From 246b52ffdd1142817a46432a378ad99815f79d1b Mon Sep 17 00:00:00 2001 From: Shiroha Date: Fri, 13 Feb 2026 17:36:28 +0800 Subject: [PATCH 001/196] Initial commit --- .gitmodules | 3 - bbdev | 1 - bbdev/LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++++++++ bbdev/README.md | 2 + 4 files changed, 203 insertions(+), 4 deletions(-) delete mode 160000 bbdev create mode 100644 bbdev/LICENSE create mode 100644 bbdev/README.md diff --git a/.gitmodules b/.gitmodules index 0c0497896..069510370 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "bbdev"] - path = bbdev - url = https://github.com/DangoSys/bbdev.git [submodule "thirdparty/waveform-mcp"] path = thirdparty/waveform-mcp url = https://github.com/jiegec/waveform-mcp.git diff --git a/bbdev b/bbdev deleted file mode 160000 index 5dcb56cbb..000000000 --- a/bbdev +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5dcb56cbbfc33a96e70b2236d8e468a68a2502bc diff --git a/bbdev/LICENSE b/bbdev/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/bbdev/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/bbdev/README.md b/bbdev/README.md new file mode 100644 index 000000000..ea27157b8 --- /dev/null +++ b/bbdev/README.md @@ -0,0 +1,2 @@ +# bbdev +Agent-friendly developer toolchain backend From bc858f7beeb9d9c6e7d0d977a8ef2a6ed8e17544 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 13 Feb 2026 18:47:42 +0800 Subject: [PATCH 002/196] feat: Add initial project structure with core functionality --- bbdev/.github/workflows/nix.yml | 16 + bbdev/.gitignore | 19 + bbdev/README.md | 16 + bbdev/api/package.json | 18 + bbdev/api/pnpm-lock.yaml | 5789 +++++++++++++++++ bbdev/api/pnpm-workspace.yaml | 1 + bbdev/api/requirements.txt | 2 + bbdev/api/services/README.md | 435 ++ bbdev/api/services/__init__.py | 84 + bbdev/api/services/agent_tools.py | 129 + bbdev/api/services/base.py | 114 + bbdev/api/services/deepwiki_tools.py | 327 + bbdev/api/services/file_tools.py | 359 + bbdev/api/services/presets.py | 106 + bbdev/api/services/registry.py | 131 + bbdev/api/services/workflow_tools.py | 71 + bbdev/api/steps/compiler/01_build_api_step.py | 26 + .../api/steps/compiler/01_build_event_step.py | 56 + bbdev/api/steps/compiler/README.md | 33 + bbdev/api/steps/firesim/.gitignore | 1 + .../firesim/01_buildbitstream_api_step.py | 26 + .../firesim/01_buildbitstream_event_step.py | 54 + .../steps/firesim/02_infrasetup_api_step.py | 27 + .../steps/firesim/02_infrasetup_event_step.py | 56 + .../steps/firesim/03_runworkload_api_step.py | 27 + .../firesim/03_runworkload_event_step.py | 56 + bbdev/api/steps/firesim/README.md | 59 + .../firesim/scripts/makefrag/firesim/build.mk | 56 + .../scripts/makefrag/firesim/config.mk | 30 + .../scripts/makefrag/firesim/driver.mk | 70 + .../scripts/makefrag/firesim/metasim.mk | 140 + .../firesim/scripts/yaml/config_build.yaml | 44 + .../scripts/yaml/config_build_recipes.yaml | 64 + .../firesim/scripts/yaml/config_hwdb.yaml | 43 + .../firesim/scripts/yaml/config_runtime.yaml | 110 + bbdev/api/steps/marshal/.gitignore | 1 + bbdev/api/steps/marshal/01_build_api_step.py | 25 + .../api/steps/marshal/01_build_event_step.py | 70 + bbdev/api/steps/marshal/02_launch_api_step.py | 26 + .../api/steps/marshal/02_launch_event_step.py | 50 + bbdev/api/steps/marshal/README.md | 53 + .../steps/marshal/scripts/interactive.json | 9 + .../marshal/scripts/interactive/host-init.sh | 20 + bbdev/api/steps/marshal/scripts/marshal | 1 + .../steps/marshal/scripts/marshal-config.yaml | 2 + .../steps/palladium/01_verilog_api_step.py | 46 + .../steps/palladium/01_verilog_event_step.py | 184 + bbdev/api/steps/sardine/01_run_api_step.py | 42 + bbdev/api/steps/sardine/01_run_event_step.py | 55 + bbdev/api/steps/sardine/README.md | 34 + .../api/steps/verilator/01_clean_api_step.py | 26 + .../steps/verilator/01_clean_event_step.py | 57 + .../steps/verilator/02_verilog_api_step.py | 46 + .../steps/verilator/02_verilog_event_step.py | 93 + .../api/steps/verilator/03_build_api_step.py | 30 + .../steps/verilator/03_build_event_step.py | 137 + .../api/steps/verilator/04_cosim_api_step.py | 40 + .../steps/verilator/04_cosim_event_step.py | 124 + bbdev/api/steps/verilator/04_sim_api_step.py | 40 + .../api/steps/verilator/04_sim_event_step.py | 124 + bbdev/api/steps/verilator/05_run_api_step.py | 53 + bbdev/api/steps/verilator/README.md | 197 + bbdev/api/steps/workload/01_buidl_api_step.py | 38 + .../api/steps/workload/01_build_event_step.py | 59 + bbdev/api/steps/workload/README.md | 39 + bbdev/api/utils/__init__.py | 10 + bbdev/api/utils/event_common.py | 111 + bbdev/api/utils/path.py | 6 + bbdev/api/utils/port.py | 16 + bbdev/api/utils/search_workload.py | 72 + bbdev/api/utils/stream_run.py | 172 + bbdev/bbdev | 533 ++ bbdev/flake.lock | 61 + bbdev/flake.nix | 46 + bbdev/nix/overlay.nix | 8 + 75 files changed, 11351 insertions(+) create mode 100644 bbdev/.github/workflows/nix.yml create mode 100644 bbdev/.gitignore create mode 100644 bbdev/api/package.json create mode 100644 bbdev/api/pnpm-lock.yaml create mode 100644 bbdev/api/pnpm-workspace.yaml create mode 100644 bbdev/api/requirements.txt create mode 100644 bbdev/api/services/README.md create mode 100644 bbdev/api/services/__init__.py create mode 100644 bbdev/api/services/agent_tools.py create mode 100644 bbdev/api/services/base.py create mode 100644 bbdev/api/services/deepwiki_tools.py create mode 100644 bbdev/api/services/file_tools.py create mode 100644 bbdev/api/services/presets.py create mode 100644 bbdev/api/services/registry.py create mode 100644 bbdev/api/services/workflow_tools.py create mode 100644 bbdev/api/steps/compiler/01_build_api_step.py create mode 100644 bbdev/api/steps/compiler/01_build_event_step.py create mode 100644 bbdev/api/steps/compiler/README.md create mode 100644 bbdev/api/steps/firesim/.gitignore create mode 100644 bbdev/api/steps/firesim/01_buildbitstream_api_step.py create mode 100644 bbdev/api/steps/firesim/01_buildbitstream_event_step.py create mode 100644 bbdev/api/steps/firesim/02_infrasetup_api_step.py create mode 100644 bbdev/api/steps/firesim/02_infrasetup_event_step.py create mode 100644 bbdev/api/steps/firesim/03_runworkload_api_step.py create mode 100644 bbdev/api/steps/firesim/03_runworkload_event_step.py create mode 100644 bbdev/api/steps/firesim/README.md create mode 100644 bbdev/api/steps/firesim/scripts/makefrag/firesim/build.mk create mode 100644 bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk create mode 100644 bbdev/api/steps/firesim/scripts/makefrag/firesim/driver.mk create mode 100644 bbdev/api/steps/firesim/scripts/makefrag/firesim/metasim.mk create mode 100644 bbdev/api/steps/firesim/scripts/yaml/config_build.yaml create mode 100644 bbdev/api/steps/firesim/scripts/yaml/config_build_recipes.yaml create mode 100644 bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml create mode 100644 bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml create mode 100644 bbdev/api/steps/marshal/.gitignore create mode 100644 bbdev/api/steps/marshal/01_build_api_step.py create mode 100644 bbdev/api/steps/marshal/01_build_event_step.py create mode 100644 bbdev/api/steps/marshal/02_launch_api_step.py create mode 100644 bbdev/api/steps/marshal/02_launch_event_step.py create mode 100644 bbdev/api/steps/marshal/README.md create mode 100644 bbdev/api/steps/marshal/scripts/interactive.json create mode 100755 bbdev/api/steps/marshal/scripts/interactive/host-init.sh create mode 120000 bbdev/api/steps/marshal/scripts/marshal create mode 100644 bbdev/api/steps/marshal/scripts/marshal-config.yaml create mode 100644 bbdev/api/steps/palladium/01_verilog_api_step.py create mode 100644 bbdev/api/steps/palladium/01_verilog_event_step.py create mode 100644 bbdev/api/steps/sardine/01_run_api_step.py create mode 100644 bbdev/api/steps/sardine/01_run_event_step.py create mode 100644 bbdev/api/steps/sardine/README.md create mode 100644 bbdev/api/steps/verilator/01_clean_api_step.py create mode 100644 bbdev/api/steps/verilator/01_clean_event_step.py create mode 100644 bbdev/api/steps/verilator/02_verilog_api_step.py create mode 100644 bbdev/api/steps/verilator/02_verilog_event_step.py create mode 100644 bbdev/api/steps/verilator/03_build_api_step.py create mode 100644 bbdev/api/steps/verilator/03_build_event_step.py create mode 100644 bbdev/api/steps/verilator/04_cosim_api_step.py create mode 100644 bbdev/api/steps/verilator/04_cosim_event_step.py create mode 100644 bbdev/api/steps/verilator/04_sim_api_step.py create mode 100644 bbdev/api/steps/verilator/04_sim_event_step.py create mode 100644 bbdev/api/steps/verilator/05_run_api_step.py create mode 100644 bbdev/api/steps/verilator/README.md create mode 100644 bbdev/api/steps/workload/01_buidl_api_step.py create mode 100644 bbdev/api/steps/workload/01_build_event_step.py create mode 100644 bbdev/api/steps/workload/README.md create mode 100644 bbdev/api/utils/__init__.py create mode 100644 bbdev/api/utils/event_common.py create mode 100644 bbdev/api/utils/path.py create mode 100644 bbdev/api/utils/port.py create mode 100644 bbdev/api/utils/search_workload.py create mode 100644 bbdev/api/utils/stream_run.py create mode 100755 bbdev/bbdev create mode 100644 bbdev/flake.lock create mode 100644 bbdev/flake.nix create mode 100644 bbdev/nix/overlay.nix diff --git a/bbdev/.github/workflows/nix.yml b/bbdev/.github/workflows/nix.yml new file mode 100644 index 000000000..ea29f7976 --- /dev/null +++ b/bbdev/.github/workflows/nix.yml @@ -0,0 +1,16 @@ +name: Nix + +on: + pull_request: + push: + branches: [main] + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@v3 + - name: Build bbdev + run: nix develop diff --git a/bbdev/.gitignore b/bbdev/.gitignore new file mode 100644 index 000000000..36283a45d --- /dev/null +++ b/bbdev/.gitignore @@ -0,0 +1,19 @@ +node_modules/ +python_modules/ +.venv/ +venv/ +.motia/ +.mermaid/ +dist/ +__pycache__/ +*.pyc +motia-workbench.json +types.d.ts +appendonlydir/ +dump.rdb +*.log +output/ +result/ + +.cursor/ +.claude/ diff --git a/bbdev/README.md b/bbdev/README.md index ea27157b8..5445e2108 100644 --- a/bbdev/README.md +++ b/bbdev/README.md @@ -1,2 +1,18 @@ # bbdev Agent-friendly developer toolchain backend + +## Build + +```bash +# Enter dev environment +nix develop + +# Install dependencies +cd api +pnpm install + +# Start dev server +pnpm dev +``` + +This Project is **Built with 💙 by [Motia](https://motia.dev)** diff --git a/bbdev/api/package.json b/bbdev/api/package.json new file mode 100644 index 000000000..4dfd49483 --- /dev/null +++ b/bbdev/api/package.json @@ -0,0 +1,18 @@ +{ + "name": "bbdev", + "version": "0.1.0", + "description": "Agent-friendly developer toolchain backend powered by Motia", + "scripts": { + "postinstall": "motia install", + "dev": "motia dev", + "dev:debug": "motia dev --debug", + "build": "motia build", + "start": "motia start", + "clean": "rm -rf dist node_modules python_modules .motia .mermaid" + }, + "keywords": ["motia"], + "dependencies": { + "motia": "0.8.2-beta.139", + "zod": "^3.24.4" + } +} diff --git a/bbdev/api/pnpm-lock.yaml b/bbdev/api/pnpm-lock.yaml new file mode 100644 index 000000000..b1468f752 --- /dev/null +++ b/bbdev/api/pnpm-lock.yaml @@ -0,0 +1,5789 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + motia: + specifier: 0.8.2-beta.139 + version: 0.8.2-beta.139(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(jiti@2.6.1)(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.8.3)(use-sync-external-store@1.6.0(react@19.2.4)) + zod: + specifier: ^3.24.4 + version: 3.25.76 + +packages: + + '@alloc/quick-lru@5.2.0': + resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} + engines: {node: '>=10'} + + '@amplitude/analytics-connector@1.6.4': + resolution: {integrity: sha512-SpIv0IQMNIq6SH3UqFGiaZyGSc7PBZwRdq7lvP0pBxW8i4Ny+8zwI0pV+VMfMHQwWY3wdIbWw5WQphNjpdq1/Q==} + + '@amplitude/analytics-core@2.38.0': + resolution: {integrity: sha512-cYFNox/mb8uWxdUFEMsS5Uj79BiWm3t1Dm+y5AS085tjbZ2RIxYsytb7ieeWCxd2cXEIpDPbcYhScU4Hm1Ky9g==} + + '@amplitude/analytics-node@1.5.35': + resolution: {integrity: sha512-V/W8Wy2g0hACizaYPYvkXqvItrRnb37CjSddIsCIs9GymZE2F0Vs9pWKXJjsPR0Pv2PB+qIZg4qjcXQlc+G42Q==} + + '@babel/code-frame@7.29.0': + resolution: {integrity: sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==} + engines: {node: '>=6.9.0'} + + '@babel/compat-data@7.29.0': + resolution: {integrity: sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg==} + engines: {node: '>=6.9.0'} + + '@babel/core@7.29.0': + resolution: {integrity: sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==} + engines: {node: '>=6.9.0'} + + '@babel/generator@7.29.1': + resolution: {integrity: sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-compilation-targets@7.28.6': + resolution: {integrity: sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==} + engines: {node: '>=6.9.0'} + + '@babel/helper-globals@7.28.0': + resolution: {integrity: sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-module-imports@7.28.6': + resolution: {integrity: sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-module-transforms@7.28.6': + resolution: {integrity: sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + + '@babel/helper-plugin-utils@7.28.6': + resolution: {integrity: sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==} + engines: {node: '>=6.9.0'} + + '@babel/helper-string-parser@7.27.1': + resolution: {integrity: sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.28.5': + resolution: {integrity: sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-option@7.27.1': + resolution: {integrity: sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==} + engines: {node: '>=6.9.0'} + + '@babel/helpers@7.28.6': + resolution: {integrity: sha512-xOBvwq86HHdB7WUDTfKfT/Vuxh7gElQ+Sfti2Cy6yIWNW05P8iUslOVcZ4/sKbE+/jQaukQAdz/gf3724kYdqw==} + engines: {node: '>=6.9.0'} + + '@babel/parser@7.29.0': + resolution: {integrity: sha512-IyDgFV5GeDUVX4YdF/3CPULtVGSXXMLh1xVIgdCgxApktqnQV0r7/8Nqthg+8YLGaAtdyIlo2qIdZrbCv4+7ww==} + engines: {node: '>=6.0.0'} + hasBin: true + + '@babel/plugin-transform-react-jsx-self@7.27.1': + resolution: {integrity: sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/plugin-transform-react-jsx-source@7.27.1': + resolution: {integrity: sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + + '@babel/runtime@7.28.6': + resolution: {integrity: sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==} + engines: {node: '>=6.9.0'} + + '@babel/template@7.28.6': + resolution: {integrity: sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==} + engines: {node: '>=6.9.0'} + + '@babel/traverse@7.29.0': + resolution: {integrity: sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==} + engines: {node: '>=6.9.0'} + + '@babel/types@7.29.0': + resolution: {integrity: sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==} + engines: {node: '>=6.9.0'} + + '@cspotcode/source-map-support@0.8.1': + resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} + engines: {node: '>=12'} + + '@esbuild/aix-ppc64@0.25.12': + resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.25.12': + resolution: {integrity: sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.25.12': + resolution: {integrity: sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.25.12': + resolution: {integrity: sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.25.12': + resolution: {integrity: sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.25.12': + resolution: {integrity: sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.25.12': + resolution: {integrity: sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.25.12': + resolution: {integrity: sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.25.12': + resolution: {integrity: sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.25.12': + resolution: {integrity: sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.25.12': + resolution: {integrity: sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.25.12': + resolution: {integrity: sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.25.12': + resolution: {integrity: sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.25.12': + resolution: {integrity: sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.25.12': + resolution: {integrity: sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.25.12': + resolution: {integrity: sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.25.12': + resolution: {integrity: sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.25.12': + resolution: {integrity: sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.25.12': + resolution: {integrity: sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.25.12': + resolution: {integrity: sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.25.12': + resolution: {integrity: sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.25.12': + resolution: {integrity: sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.25.12': + resolution: {integrity: sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.25.12': + resolution: {integrity: sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.25.12': + resolution: {integrity: sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.25.12': + resolution: {integrity: sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@eslint-community/eslint-utils@4.9.1': + resolution: {integrity: sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 + + '@eslint-community/regexpp@4.12.2': + resolution: {integrity: sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==} + engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} + + '@eslint/config-array@0.21.1': + resolution: {integrity: sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/config-helpers@0.4.2': + resolution: {integrity: sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/core@0.17.0': + resolution: {integrity: sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/eslintrc@3.3.3': + resolution: {integrity: sha512-Kr+LPIUVKz2qkx1HAMH8q1q6azbqBAsXJUxBl/ODDuVPX45Z9DfwB8tPjTi6nNZ8BuM3nbJxC5zCAg5elnBUTQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/js@9.39.2': + resolution: {integrity: sha512-q1mjIoW1VX4IvSocvM/vbTiveKC4k9eLrajNEuSsmjymSDEbpGddtpfOoN7YGAqBK3NG+uqo8ia4PDTt8buCYA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/object-schema@2.1.7': + resolution: {integrity: sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/plugin-kit@0.4.1': + resolution: {integrity: sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@floating-ui/core@1.7.4': + resolution: {integrity: sha512-C3HlIdsBxszvm5McXlB8PeOEWfBhcGBTZGkGlWc2U0KFY5IwG5OQEuQ8rq52DZmcHDlPLd+YFBK+cZcytwIFWg==} + + '@floating-ui/dom@1.7.5': + resolution: {integrity: sha512-N0bD2kIPInNHUHehXhMke1rBGs1dwqvC9O9KYMyyjK7iXt7GAhnro7UlcuYcGdS/yYOlq0MAVgrow8IbWJwyqg==} + + '@floating-ui/react-dom@2.1.7': + resolution: {integrity: sha512-0tLRojf/1Go2JgEVm+3Frg9A3IW8bJgKgdO0BN5RkF//ufuz2joZM63Npau2ff3J6lUVYgDSNzNkR+aH3IVfjg==} + peerDependencies: + react: '>=16.8.0' + react-dom: '>=16.8.0' + + '@floating-ui/utils@0.2.10': + resolution: {integrity: sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ==} + + '@humanfs/core@0.19.1': + resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} + engines: {node: '>=18.18.0'} + + '@humanfs/node@0.16.7': + resolution: {integrity: sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==} + engines: {node: '>=18.18.0'} + + '@humanwhocodes/module-importer@1.0.1': + resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} + engines: {node: '>=12.22'} + + '@humanwhocodes/retry@0.4.3': + resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==} + engines: {node: '>=18.18'} + + '@inquirer/external-editor@1.0.3': + resolution: {integrity: sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==} + engines: {node: '>=18'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@isaacs/cliui@8.0.2': + resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} + engines: {node: '>=12'} + + '@isaacs/cliui@9.0.0': + resolution: {integrity: sha512-AokJm4tuBHillT+FpMtxQ60n8ObyXBatq7jD2/JA9dxbDDokKQm8KMht5ibGzLVU9IJDIKK4TPKgMHEYMn3lMg==} + engines: {node: '>=18'} + + '@jridgewell/gen-mapping@0.3.13': + resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} + + '@jridgewell/remapping@2.3.5': + resolution: {integrity: sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==} + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + + '@jridgewell/trace-mapping@0.3.9': + resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} + + '@monaco-editor/loader@1.7.0': + resolution: {integrity: sha512-gIwR1HrJrrx+vfyOhYmCZ0/JcWqG5kbfG7+d3f/C1LXk2EvzAbHSg3MQ5lO2sMlo9izoAZ04shohfKLVT6crVA==} + + '@monaco-editor/react@4.7.0': + resolution: {integrity: sha512-cyzXQCtO47ydzxpQtCGSQGOC8Gk3ZUeBXFAxD+CWXYFo5OqZyZUonFl0DwUlTyAfRHntBfw2p3w4s9R6oe1eCA==} + peerDependencies: + monaco-editor: '>= 0.25.0 < 1' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + '@motiadev/core@0.8.2-beta.139': + resolution: {integrity: sha512-ueIGi5PsYU7TmeG2GrIXTUTdS6D9pFUzgft1vOx0rkl9YRIKevsm5IDHapjg8ZchbSLhKJyYuCeROH8AOsIXow==} + + '@motiadev/plugin-endpoint@0.8.2-beta.139': + resolution: {integrity: sha512-KKmo9hptNXhgQLbDhcyVKGuMETfowQ4Dp4utBQCfUHr/mLbZdO14lNqPpo5dEgc4VaXq0Ldc8YbgDhDorkAJ1w==} + + '@motiadev/stream-client-browser@0.8.2-beta.139': + resolution: {integrity: sha512-FF4GsAWRd6QKcWAvALL3IQ5RDreTR1uqMCTJj81qUci+YTp8joiWo7a8spMdttOgRWUu3xI1QrKPO2PjkeAzvg==} + + '@motiadev/stream-client-node@0.8.2-beta.139': + resolution: {integrity: sha512-UBZOtv6jrXGcm1nH8JgTBB2wgfgh+hmVSm5YZqrDgwxTDMr7GxXFHa7j1AnGy7B+fc+xSZwRPmzewIxPYc59Tg==} + + '@motiadev/stream-client-react@0.8.2-beta.139': + resolution: {integrity: sha512-FaTMgLYB3LkVvXhcBGGDENJCI+1ezSfH4YSvUtkId9lqYs09Kwxk69w8F/SLjTNQvf/4csarr0LD4IGdveJ2dQ==} + peerDependencies: + react: ^19.1.0 + + '@motiadev/stream-client@0.8.2-beta.139': + resolution: {integrity: sha512-NVe0r4Jl+b9QD92FK8B4bpkvvjFauhTdLtcQyv/5g2CZ7azC2IG3Fz0vw6+qe28+37L7IrpTxRWnSTkULlyazA==} + + '@motiadev/ui@0.8.2-beta.139': + resolution: {integrity: sha512-Q5yFfehQhIerhoKwQrk8elyGSB9mBrTXgEhnOeRLE/p+qzeKPjaqRxF0O1L98AT0H++aFRwwdx1io9doIhxJWw==} + peerDependencies: + react: ^19.1.0 + react-dom: ^19.1.0 + + '@motiadev/workbench@0.8.2-beta.139': + resolution: {integrity: sha512-8Ivo2aOVTPGQzx0bTg/3Qhpsrv+L4KK7B3cQmZPmkdoG88pH5ojLBql3sMZCmdTOMYFhD78puw3BMMuSb2Ev4g==} + + '@pkgjs/parseargs@0.11.0': + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} + engines: {node: '>=14'} + + '@radix-ui/number@1.1.1': + resolution: {integrity: sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g==} + + '@radix-ui/primitive@1.1.3': + resolution: {integrity: sha512-JTF99U/6XIjCBo0wqkU5sK10glYe27MRRsfwoiq5zzOEZLHU3A3KCMa5X/azekYRCJ0HlwI0crAXS/5dEHTzDg==} + + '@radix-ui/react-arrow@1.1.7': + resolution: {integrity: sha512-F+M1tLhO+mlQaOWspE8Wstg+z6PwxwRd8oQ8IXceWz92kfAmalTRf0EjrouQeo7QssEPfCn05B4Ihs1K9WQ/7w==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-checkbox@1.3.3': + resolution: {integrity: sha512-wBbpv+NQftHDdG86Qc0pIyXk5IR3tM8Vd0nWLKDcX8nNn4nXFOFwsKuqw2okA/1D/mpaAkmuyndrPJTYDNZtFw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-collapsible@1.1.12': + resolution: {integrity: sha512-Uu+mSh4agx2ib1uIGPP4/CKNULyajb3p92LsVXmH2EHVMTfZWpll88XJ0j4W0z3f8NK1eYl1+Mf/szHPmcHzyA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-collection@1.1.7': + resolution: {integrity: sha512-Fh9rGN0MoI4ZFUNyfFVNU4y9LUz93u9/0K+yLgA2bwRojxM8JU1DyvvMBabnZPBgMWREAJvU2jjVzq+LrFUglw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-compose-refs@1.1.2': + resolution: {integrity: sha512-z4eqJvfiNnFMHIIvXP3CY57y2WJs5g2v3X0zm9mEJkrkNv4rDxu+sg9Jh8EkXyeqBkB7SOcboo9dMVqhyrACIg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-context@1.1.2': + resolution: {integrity: sha512-jCi/QKUM2r1Ju5a3J64TH2A5SpKAgh0LpknyqdQ4m6DCV0xJ2HG1xARRwNGPQfi1SLdLWZ1OJz6F4OMBBNiGJA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-dialog@1.1.15': + resolution: {integrity: sha512-TCglVRtzlffRNxRMEyR36DGBLJpeusFcgMVD9PZEzAKnUs1lKCgX5u9BmC2Yg+LL9MgZDugFFs1Vl+Jp4t/PGw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-direction@1.1.1': + resolution: {integrity: sha512-1UEWRX6jnOA2y4H5WczZ44gOOjTEmlqv1uNW4GAJEO5+bauCBhv8snY65Iw5/VOS/ghKN9gr2KjnLKxrsvoMVw==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-dismissable-layer@1.1.11': + resolution: {integrity: sha512-Nqcp+t5cTB8BinFkZgXiMJniQH0PsUt2k51FUhbdfeKvc4ACcG2uQniY/8+h1Yv6Kza4Q7lD7PQV0z0oicE0Mg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-dropdown-menu@2.1.16': + resolution: {integrity: sha512-1PLGQEynI/3OX/ftV54COn+3Sud/Mn8vALg2rWnBLnRaGtJDduNW/22XjlGgPdpcIbiQxjKtb7BkcjP00nqfJw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-focus-guards@1.1.3': + resolution: {integrity: sha512-0rFg/Rj2Q62NCm62jZw0QX7a3sz6QCQU0LpZdNrJX8byRGaGVTqbrW9jAoIAHyMQqsNpeZ81YgSizOt5WXq0Pw==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-focus-scope@1.1.7': + resolution: {integrity: sha512-t2ODlkXBQyn7jkl6TNaw/MtVEVvIGelJDCG41Okq/KwUsJBwQ4XVZsHAVUkK4mBv3ewiAS3PGuUWuY2BoK4ZUw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-id@1.1.1': + resolution: {integrity: sha512-kGkGegYIdQsOb4XjsfM97rXsiHaBwco+hFI66oO4s9LU+PLAC5oJ7khdOVFxkhsmlbpUqDAvXw11CluXP+jkHg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-label@2.1.8': + resolution: {integrity: sha512-FmXs37I6hSBVDlO4y764TNz1rLgKwjJMQ0EGte6F3Cb3f4bIuHB/iLa/8I9VKkmOy+gNHq8rql3j686ACVV21A==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-menu@2.1.16': + resolution: {integrity: sha512-72F2T+PLlphrqLcAotYPp0uJMr5SjP5SL01wfEspJbru5Zs5vQaSHb4VB3ZMJPimgHHCHG7gMOeOB9H3Hdmtxg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-navigation-menu@1.2.14': + resolution: {integrity: sha512-YB9mTFQvCOAQMHU+C/jVl96WmuWeltyUEpRJJky51huhds5W2FQr1J8D/16sQlf0ozxkPK8uF3niQMdUwZPv5w==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-popper@1.2.8': + resolution: {integrity: sha512-0NJQ4LFFUuWkE7Oxf0htBKS6zLkkjBH+hM1uk7Ng705ReR8m/uelduy1DBo0PyBXPKVnBA6YBlU94MBGXrSBCw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-portal@1.1.9': + resolution: {integrity: sha512-bpIxvq03if6UNwXZ+HTK71JLh4APvnXntDc6XOX8UVq4XQOVl7lwok0AvIl+b8zgCw3fSaVTZMpAPPagXbKmHQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-presence@1.1.5': + resolution: {integrity: sha512-/jfEwNDdQVBCNvjkGit4h6pMOzq8bHkopq458dPt2lMjx+eBQUohZNG9A7DtO/O5ukSbxuaNGXMjHicgwy6rQQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-primitive@2.1.3': + resolution: {integrity: sha512-m9gTwRkhy2lvCPe6QJp4d3G1TYEUHn/FzJUtq9MjH46an1wJU+GdoGC5VLof8RX8Ft/DlpshApkhswDLZzHIcQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-primitive@2.1.4': + resolution: {integrity: sha512-9hQc4+GNVtJAIEPEqlYqW5RiYdrr8ea5XQ0ZOnD6fgru+83kqT15mq2OCcbe8KnjRZl5vF3ks69AKz3kh1jrhg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-roving-focus@1.1.11': + resolution: {integrity: sha512-7A6S9jSgm/S+7MdtNDSb+IU859vQqJ/QAtcYQcfFC6W8RS4IxIZDldLR0xqCFZ6DCyrQLjLPsxtTNch5jVA4lA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-scroll-area@1.2.10': + resolution: {integrity: sha512-tAXIa1g3sM5CGpVT0uIbUx/U3Gs5N8T52IICuCtObaos1S8fzsrPXG5WObkQN3S6NVl6wKgPhAIiBGbWnvc97A==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-select@2.2.6': + resolution: {integrity: sha512-I30RydO+bnn2PQztvo25tswPH+wFBjehVGtmagkU78yMdwTwVf12wnAOF+AeP8S2N8xD+5UPbGhkUfPyvT+mwQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-separator@1.1.8': + resolution: {integrity: sha512-sDvqVY4itsKwwSMEe0jtKgfTh+72Sy3gPmQpjqcQneqQ4PFmr/1I0YA+2/puilhggCe2gJcx5EBAYFkWkdpa5g==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-slot@1.2.3': + resolution: {integrity: sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-slot@1.2.4': + resolution: {integrity: sha512-Jl+bCv8HxKnlTLVrcDE8zTMJ09R9/ukw4qBs/oZClOfoQk/cOTbDn+NceXfV7j09YPVQUryJPHurafcSg6EVKA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-switch@1.2.6': + resolution: {integrity: sha512-bByzr1+ep1zk4VubeEVViV592vu2lHE2BZY5OnzehZqOOgogN80+mNtCqPkhn2gklJqOpxWgPoYTSnhBCqpOXQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-tabs@1.1.13': + resolution: {integrity: sha512-7xdcatg7/U+7+Udyoj2zodtI9H/IIopqo+YOIcZOq1nJwXWBZ9p8xiu5llXlekDbZkca79a/fozEYQXIA4sW6A==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-tooltip@1.2.8': + resolution: {integrity: sha512-tY7sVt1yL9ozIxvmbtN5qtmH2krXcBCfjEiCgKGLqunJHvgvZG2Pcl2oQ3kbcZARb1BGEHdkLzcYGO8ynVlieg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-use-callback-ref@1.1.1': + resolution: {integrity: sha512-FkBMwD+qbGQeMu1cOHnuGB6x4yzPjho8ap5WtbEJ26umhgqVXbhekKUQO+hZEL1vU92a3wHwdp0HAcqAUF5iDg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-controllable-state@1.2.2': + resolution: {integrity: sha512-BjasUjixPFdS+NKkypcyyN5Pmg83Olst0+c6vGov0diwTEo6mgdqVR6hxcEgFuh4QrAs7Rc+9KuGJ9TVCj0Zzg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-effect-event@0.0.2': + resolution: {integrity: sha512-Qp8WbZOBe+blgpuUT+lw2xheLP8q0oatc9UpmiemEICxGvFLYmHm9QowVZGHtJlGbS6A6yJ3iViad/2cVjnOiA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-escape-keydown@1.1.1': + resolution: {integrity: sha512-Il0+boE7w/XebUHyBjroE+DbByORGR9KKmITzbR7MyQ4akpORYP/ZmbhAr0DG7RmmBqoOnZdy2QlvajJ2QA59g==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-layout-effect@1.1.1': + resolution: {integrity: sha512-RbJRS4UWQFkzHTTwVymMTUv8EqYhOp8dOOviLj2ugtTiXRaRQS7GLGxZTLL1jWhMeoSCf5zmcZkqTl9IiYfXcQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-previous@1.1.1': + resolution: {integrity: sha512-2dHfToCj/pzca2Ck724OZ5L0EVrr3eHRNsG/b3xQJLA2hZpVCS99bLAX+hm1IHXDEnzU6by5z/5MIY794/a8NQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-rect@1.1.1': + resolution: {integrity: sha512-QTYuDesS0VtuHNNvMh+CjlKJ4LJickCMUAqjlE3+j8w+RlRpwyX3apEQKGFzbZGdo7XNG1tXa+bQqIE7HIXT2w==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-size@1.1.1': + resolution: {integrity: sha512-ewrXRDTAqAXlkl6t/fkXWNAhFX9I+CkKlw6zjEwk86RSPKwZr3xpBRso655aqYafwtnbpHLj6toFzmd6xdVptQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-visually-hidden@1.2.3': + resolution: {integrity: sha512-pzJq12tEaaIhqjbzpCuv/OypJY/BPavOofm+dbab+MHLajy277+1lLm6JFcGgF5eskJ6mquGirhXY2GD/8u8Ug==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/rect@1.1.1': + resolution: {integrity: sha512-HPwpGIzkl28mWyZqG52jiqDJ12waP11Pa1lGoiyUkIEuMLBP0oeK/C89esbXrxsky5we7dfd8U58nm0SgAWpVw==} + + '@rolldown/pluginutils@1.0.0-beta.27': + resolution: {integrity: sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==} + + '@rollup/rollup-android-arm-eabi@4.57.1': + resolution: {integrity: sha512-A6ehUVSiSaaliTxai040ZpZ2zTevHYbvu/lDoeAteHI8QnaosIzm4qwtezfRg1jOYaUmnzLX1AOD6Z+UJjtifg==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.57.1': + resolution: {integrity: sha512-dQaAddCY9YgkFHZcFNS/606Exo8vcLHwArFZ7vxXq4rigo2bb494/xKMMwRRQW6ug7Js6yXmBZhSBRuBvCCQ3w==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.57.1': + resolution: {integrity: sha512-crNPrwJOrRxagUYeMn/DZwqN88SDmwaJ8Cvi/TN1HnWBU7GwknckyosC2gd0IqYRsHDEnXf328o9/HC6OkPgOg==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.57.1': + resolution: {integrity: sha512-Ji8g8ChVbKrhFtig5QBV7iMaJrGtpHelkB3lsaKzadFBe58gmjfGXAOfI5FV0lYMH8wiqsxKQ1C9B0YTRXVy4w==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.57.1': + resolution: {integrity: sha512-R+/WwhsjmwodAcz65guCGFRkMb4gKWTcIeLy60JJQbXrJ97BOXHxnkPFrP+YwFlaS0m+uWJTstrUA9o+UchFug==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.57.1': + resolution: {integrity: sha512-IEQTCHeiTOnAUC3IDQdzRAGj3jOAYNr9kBguI7MQAAZK3caezRrg0GxAb6Hchg4lxdZEI5Oq3iov/w/hnFWY9Q==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.57.1': + resolution: {integrity: sha512-F8sWbhZ7tyuEfsmOxwc2giKDQzN3+kuBLPwwZGyVkLlKGdV1nvnNwYD0fKQ8+XS6hp9nY7B+ZeK01EBUE7aHaw==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm-musleabihf@4.57.1': + resolution: {integrity: sha512-rGfNUfn0GIeXtBP1wL5MnzSj98+PZe/AXaGBCRmT0ts80lU5CATYGxXukeTX39XBKsxzFpEeK+Mrp9faXOlmrw==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm64-gnu@4.57.1': + resolution: {integrity: sha512-MMtej3YHWeg/0klK2Qodf3yrNzz6CGjo2UntLvk2RSPlhzgLvYEB3frRvbEF2wRKh1Z2fDIg9KRPe1fawv7C+g==} + cpu: [arm64] + os: [linux] + + '@rollup/rollup-linux-arm64-musl@4.57.1': + resolution: {integrity: sha512-1a/qhaaOXhqXGpMFMET9VqwZakkljWHLmZOX48R0I/YLbhdxr1m4gtG1Hq7++VhVUmf+L3sTAf9op4JlhQ5u1Q==} + cpu: [arm64] + os: [linux] + + '@rollup/rollup-linux-loong64-gnu@4.57.1': + resolution: {integrity: sha512-QWO6RQTZ/cqYtJMtxhkRkidoNGXc7ERPbZN7dVW5SdURuLeVU7lwKMpo18XdcmpWYd0qsP1bwKPf7DNSUinhvA==} + cpu: [loong64] + os: [linux] + + '@rollup/rollup-linux-loong64-musl@4.57.1': + resolution: {integrity: sha512-xpObYIf+8gprgWaPP32xiN5RVTi/s5FCR+XMXSKmhfoJjrpRAjCuuqQXyxUa/eJTdAE6eJ+KDKaoEqjZQxh3Gw==} + cpu: [loong64] + os: [linux] + + '@rollup/rollup-linux-ppc64-gnu@4.57.1': + resolution: {integrity: sha512-4BrCgrpZo4hvzMDKRqEaW1zeecScDCR+2nZ86ATLhAoJ5FQ+lbHVD3ttKe74/c7tNT9c6F2viwB3ufwp01Oh2w==} + cpu: [ppc64] + os: [linux] + + '@rollup/rollup-linux-ppc64-musl@4.57.1': + resolution: {integrity: sha512-NOlUuzesGauESAyEYFSe3QTUguL+lvrN1HtwEEsU2rOwdUDeTMJdO5dUYl/2hKf9jWydJrO9OL/XSSf65R5+Xw==} + cpu: [ppc64] + os: [linux] + + '@rollup/rollup-linux-riscv64-gnu@4.57.1': + resolution: {integrity: sha512-ptA88htVp0AwUUqhVghwDIKlvJMD/fmL/wrQj99PRHFRAG6Z5nbWoWG4o81Nt9FT+IuqUQi+L31ZKAFeJ5Is+A==} + cpu: [riscv64] + os: [linux] + + '@rollup/rollup-linux-riscv64-musl@4.57.1': + resolution: {integrity: sha512-S51t7aMMTNdmAMPpBg7OOsTdn4tySRQvklmL3RpDRyknk87+Sp3xaumlatU+ppQ+5raY7sSTcC2beGgvhENfuw==} + cpu: [riscv64] + os: [linux] + + '@rollup/rollup-linux-s390x-gnu@4.57.1': + resolution: {integrity: sha512-Bl00OFnVFkL82FHbEqy3k5CUCKH6OEJL54KCyx2oqsmZnFTR8IoNqBF+mjQVcRCT5sB6yOvK8A37LNm/kPJiZg==} + cpu: [s390x] + os: [linux] + + '@rollup/rollup-linux-x64-gnu@4.57.1': + resolution: {integrity: sha512-ABca4ceT4N+Tv/GtotnWAeXZUZuM/9AQyCyKYyKnpk4yoA7QIAuBt6Hkgpw8kActYlew2mvckXkvx0FfoInnLg==} + cpu: [x64] + os: [linux] + + '@rollup/rollup-linux-x64-musl@4.57.1': + resolution: {integrity: sha512-HFps0JeGtuOR2convgRRkHCekD7j+gdAuXM+/i6kGzQtFhlCtQkpwtNzkNj6QhCDp7DRJ7+qC/1Vg2jt5iSOFw==} + cpu: [x64] + os: [linux] + + '@rollup/rollup-openbsd-x64@4.57.1': + resolution: {integrity: sha512-H+hXEv9gdVQuDTgnqD+SQffoWoc0Of59AStSzTEj/feWTBAnSfSD3+Dql1ZruJQxmykT/JVY0dE8Ka7z0DH1hw==} + cpu: [x64] + os: [openbsd] + + '@rollup/rollup-openharmony-arm64@4.57.1': + resolution: {integrity: sha512-4wYoDpNg6o/oPximyc/NG+mYUejZrCU2q+2w6YZqrAs2UcNUChIZXjtafAiiZSUc7On8v5NyNj34Kzj/Ltk6dQ==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.57.1': + resolution: {integrity: sha512-O54mtsV/6LW3P8qdTcamQmuC990HDfR71lo44oZMZlXU4tzLrbvTii87Ni9opq60ds0YzuAlEr/GNwuNluZyMQ==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.57.1': + resolution: {integrity: sha512-P3dLS+IerxCT/7D2q2FYcRdWRl22dNbrbBEtxdWhXrfIMPP9lQhb5h4Du04mdl5Woq05jVCDPCMF7Ub0NAjIew==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.57.1': + resolution: {integrity: sha512-VMBH2eOOaKGtIJYleXsi2B8CPVADrh+TyNxJ4mWPnKfLB/DBUmzW+5m1xUrcwWoMfSLagIRpjUFeW5CO5hyciQ==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.57.1': + resolution: {integrity: sha512-mxRFDdHIWRxg3UfIIAwCm6NzvxG0jDX/wBN6KsQFTvKFqqg9vTrWUE68qEjHt19A5wwx5X5aUi2zuZT7YR0jrA==} + cpu: [x64] + os: [win32] + + '@tailwindcss/node@4.1.18': + resolution: {integrity: sha512-DoR7U1P7iYhw16qJ49fgXUlry1t4CpXeErJHnQ44JgTSKMaZUdf17cfn5mHchfJ4KRBZRFA/Coo+MUF5+gOaCQ==} + + '@tailwindcss/oxide-android-arm64@4.1.18': + resolution: {integrity: sha512-dJHz7+Ugr9U/diKJA0W6N/6/cjI+ZTAoxPf9Iz9BFRF2GzEX8IvXxFIi/dZBloVJX/MZGvRuFA9rqwdiIEZQ0Q==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@tailwindcss/oxide-darwin-arm64@4.1.18': + resolution: {integrity: sha512-Gc2q4Qhs660bhjyBSKgq6BYvwDz4G+BuyJ5H1xfhmDR3D8HnHCmT/BSkvSL0vQLy/nkMLY20PQ2OoYMO15Jd0A==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@tailwindcss/oxide-darwin-x64@4.1.18': + resolution: {integrity: sha512-FL5oxr2xQsFrc3X9o1fjHKBYBMD1QZNyc1Xzw/h5Qu4XnEBi3dZn96HcHm41c/euGV+GRiXFfh2hUCyKi/e+yw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@tailwindcss/oxide-freebsd-x64@4.1.18': + resolution: {integrity: sha512-Fj+RHgu5bDodmV1dM9yAxlfJwkkWvLiRjbhuO2LEtwtlYlBgiAT4x/j5wQr1tC3SANAgD+0YcmWVrj8R9trVMA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + '@tailwindcss/oxide-linux-arm-gnueabihf@4.1.18': + resolution: {integrity: sha512-Fp+Wzk/Ws4dZn+LV2Nqx3IilnhH51YZoRaYHQsVq3RQvEl+71VGKFpkfHrLM/Li+kt5c0DJe/bHXK1eHgDmdiA==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@tailwindcss/oxide-linux-arm64-gnu@4.1.18': + resolution: {integrity: sha512-S0n3jboLysNbh55Vrt7pk9wgpyTTPD0fdQeh7wQfMqLPM/Hrxi+dVsLsPrycQjGKEQk85Kgbx+6+QnYNiHalnw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@tailwindcss/oxide-linux-arm64-musl@4.1.18': + resolution: {integrity: sha512-1px92582HkPQlaaCkdRcio71p8bc8i/ap5807tPRDK/uw953cauQBT8c5tVGkOwrHMfc2Yh6UuxaH4vtTjGvHg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@tailwindcss/oxide-linux-x64-gnu@4.1.18': + resolution: {integrity: sha512-v3gyT0ivkfBLoZGF9LyHmts0Isc8jHZyVcbzio6Wpzifg/+5ZJpDiRiUhDLkcr7f/r38SWNe7ucxmGW3j3Kb/g==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@tailwindcss/oxide-linux-x64-musl@4.1.18': + resolution: {integrity: sha512-bhJ2y2OQNlcRwwgOAGMY0xTFStt4/wyU6pvI6LSuZpRgKQwxTec0/3Scu91O8ir7qCR3AuepQKLU/kX99FouqQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@tailwindcss/oxide-wasm32-wasi@4.1.18': + resolution: {integrity: sha512-LffYTvPjODiP6PT16oNeUQJzNVyJl1cjIebq/rWWBF+3eDst5JGEFSc5cWxyRCJ0Mxl+KyIkqRxk1XPEs9x8TA==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + bundledDependencies: + - '@napi-rs/wasm-runtime' + - '@emnapi/core' + - '@emnapi/runtime' + - '@tybys/wasm-util' + - '@emnapi/wasi-threads' + - tslib + + '@tailwindcss/oxide-win32-arm64-msvc@4.1.18': + resolution: {integrity: sha512-HjSA7mr9HmC8fu6bdsZvZ+dhjyGCLdotjVOgLA2vEqxEBZaQo9YTX4kwgEvPCpRh8o4uWc4J/wEoFzhEmjvPbA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@tailwindcss/oxide-win32-x64-msvc@4.1.18': + resolution: {integrity: sha512-bJWbyYpUlqamC8dpR7pfjA0I7vdF6t5VpUGMWRkXVE3AXgIZjYUYAK7II1GNaxR8J1SSrSrppRar8G++JekE3Q==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@tailwindcss/oxide@4.1.18': + resolution: {integrity: sha512-EgCR5tTS5bUSKQgzeMClT6iCY3ToqE1y+ZB0AKldj809QXk1Y+3jB0upOYZrn9aGIzPtUsP7sX4QQ4XtjBB95A==} + engines: {node: '>= 10'} + + '@tailwindcss/postcss@4.1.18': + resolution: {integrity: sha512-Ce0GFnzAOuPyfV5SxjXGn0CubwGcuDB0zcdaPuCSzAa/2vII24JTkH+I6jcbXLb1ctjZMZZI6OjDaLPJQL1S0g==} + + '@tsconfig/node10@1.0.12': + resolution: {integrity: sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==} + + '@tsconfig/node12@1.0.11': + resolution: {integrity: sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==} + + '@tsconfig/node14@1.0.3': + resolution: {integrity: sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==} + + '@tsconfig/node16@1.0.4': + resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} + + '@types/babel__core@7.20.5': + resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==} + + '@types/babel__generator@7.27.0': + resolution: {integrity: sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==} + + '@types/babel__template@7.4.4': + resolution: {integrity: sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==} + + '@types/babel__traverse@7.28.0': + resolution: {integrity: sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==} + + '@types/d3-color@3.1.3': + resolution: {integrity: sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==} + + '@types/d3-drag@3.0.7': + resolution: {integrity: sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==} + + '@types/d3-interpolate@3.0.4': + resolution: {integrity: sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==} + + '@types/d3-selection@3.0.11': + resolution: {integrity: sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==} + + '@types/d3-transition@3.0.9': + resolution: {integrity: sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==} + + '@types/d3-zoom@3.0.8': + resolution: {integrity: sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==} + + '@types/estree@1.0.8': + resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + + '@types/hast@2.3.10': + resolution: {integrity: sha512-McWspRw8xx8J9HurkVBfYj0xKoE25tOFlHGdx4MJ5xORQrMGZNqJhVQWaIbm6Oyla5kYOXtDiopzKRJzEOkwJw==} + + '@types/json-schema@7.0.15': + resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + + '@types/luxon@3.7.1': + resolution: {integrity: sha512-H3iskjFIAn5SlJU7OuxUmTEpebK6TKB8rxZShDslBMZJ5u9S//KM1sbdAisiSrqwLQncVjnpi2OK2J51h+4lsg==} + + '@types/node@25.2.3': + resolution: {integrity: sha512-m0jEgYlYz+mDJZ2+F4v8D1AyQb+QzsNqRuI7xg1VQX/KlKS0qT9r1Mo16yo5F/MtifXFgaofIFsdFMox2SxIbQ==} + + '@types/trusted-types@2.0.7': + resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} + + '@types/unist@2.0.11': + resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} + + '@types/zen-observable@0.8.3': + resolution: {integrity: sha512-fbF6oTd4sGGy0xjHPKAt+eS2CrxJ3+6gQ3FGcBoIJR2TLAyCkCyI8JqZNy+FeON0AhVgNJoUumVoZQjBFUqHkw==} + + '@typescript-eslint/eslint-plugin@8.55.0': + resolution: {integrity: sha512-1y/MVSz0NglV1ijHC8OT49mPJ4qhPYjiK08YUQVbIOyu+5k862LKUHFkpKHWu//zmr7hDR2rhwUm6gnCGNmGBQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + '@typescript-eslint/parser': ^8.55.0 + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/parser@8.55.0': + resolution: {integrity: sha512-4z2nCSBfVIMnbuu8uinj+f0o4qOeggYJLbjpPHka3KH1om7e+H9yLKTYgksTaHcGco+NClhhY2vyO3HsMH1RGw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/project-service@8.55.0': + resolution: {integrity: sha512-zRcVVPFUYWa3kNnjaZGXSu3xkKV1zXy8M4nO/pElzQhFweb7PPtluDLQtKArEOGmjXoRjnUZ29NjOiF0eCDkcQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/scope-manager@8.55.0': + resolution: {integrity: sha512-fVu5Omrd3jeqeQLiB9f1YsuK/iHFOwb04bCtY4BSCLgjNbOD33ZdV6KyEqplHr+IlpgT0QTZ/iJ+wT7hvTx49Q==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/tsconfig-utils@8.55.0': + resolution: {integrity: sha512-1R9cXqY7RQd7WuqSN47PK9EDpgFUK3VqdmbYrvWJZYDd0cavROGn+74ktWBlmJ13NXUQKlZ/iAEQHI/V0kKe0Q==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/type-utils@8.55.0': + resolution: {integrity: sha512-x1iH2unH4qAt6I37I2CGlsNs+B9WGxurP2uyZLRz6UJoZWDBx9cJL1xVN/FiOmHEONEg6RIufdvyT0TEYIgC5g==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/types@8.55.0': + resolution: {integrity: sha512-ujT0Je8GI5BJWi+/mMoR0wxwVEQaxM+pi30xuMiJETlX80OPovb2p9E8ss87gnSVtYXtJoU9U1Cowcr6w2FE0w==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/typescript-estree@8.55.0': + resolution: {integrity: sha512-EwrH67bSWdx/3aRQhCoxDaHM+CrZjotc2UCCpEDVqfCE+7OjKAGWNY2HsCSTEVvWH2clYQK8pdeLp42EVs+xQw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/utils@8.55.0': + resolution: {integrity: sha512-BqZEsnPGdYpgyEIkDC1BadNY8oMwckftxBT+C8W0g1iKPdeqKZBtTfnvcq0nf60u7MkjFO8RBvpRGZBPw4L2ow==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <6.0.0' + + '@typescript-eslint/visitor-keys@8.55.0': + resolution: {integrity: sha512-AxNRwEie8Nn4eFS1FzDMJWIISMGoXMb037sgCBJ3UR6o0fQTzr2tqN9WT+DkWJPhIdQCfV7T6D387566VtnCJA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@vitejs/plugin-react@4.7.0': + resolution: {integrity: sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA==} + engines: {node: ^14.18.0 || >=16.0.0} + peerDependencies: + vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 + + '@xyflow/react@12.10.0': + resolution: {integrity: sha512-eOtz3whDMWrB4KWVatIBrKuxECHqip6PfA8fTpaS2RUGVpiEAe+nqDKsLqkViVWxDGreq0lWX71Xth/SPAzXiw==} + peerDependencies: + react: '>=17' + react-dom: '>=17' + + '@xyflow/system@0.0.74': + resolution: {integrity: sha512-7v7B/PkiVrkdZzSbL+inGAo6tkR/WQHHG0/jhSvLQToCsfa8YubOGmBYd1s08tpKpihdHDZFwzQZeR69QSBb4Q==} + + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + + accepts@1.3.8: + resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} + engines: {node: '>= 0.6'} + + acorn-jsx@5.3.2: + resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} + peerDependencies: + acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + + acorn-walk@8.3.4: + resolution: {integrity: sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==} + engines: {node: '>=0.4.0'} + + acorn@8.15.0: + resolution: {integrity: sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==} + engines: {node: '>=0.4.0'} + hasBin: true + + ajv@6.12.6: + resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} + + ajv@8.17.1: + resolution: {integrity: sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==} + + ansi-escapes@4.3.2: + resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==} + engines: {node: '>=8'} + + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + + ansi-regex@6.2.2: + resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==} + engines: {node: '>=12'} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + ansi-styles@6.2.3: + resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} + engines: {node: '>=12'} + + antlr4ts@0.5.0-alpha.4: + resolution: {integrity: sha512-WPQDt1B74OfPv/IMS2ekXAKkTZIHl88uMetg6q3OTqgFxZ/dxDXI0EWLyZid/1Pe6hTftyg5N7gel5wNAGxXyQ==} + + archiver-utils@5.0.2: + resolution: {integrity: sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==} + engines: {node: '>= 14'} + + archiver@7.0.1: + resolution: {integrity: sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==} + engines: {node: '>= 14'} + + arg@4.1.3: + resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} + + argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + + aria-hidden@1.2.6: + resolution: {integrity: sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==} + engines: {node: '>=10'} + + array-flatten@1.1.1: + resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==} + + astral-regex@2.0.0: + resolution: {integrity: sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==} + engines: {node: '>=8'} + + async@3.2.6: + resolution: {integrity: sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==} + + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + autoprefixer@10.4.24: + resolution: {integrity: sha512-uHZg7N9ULTVbutaIsDRoUkoS8/h3bdsmVJYZ5l3wv8Cp/6UIIoRDm90hZ+BwxUj/hGBEzLxdHNSKuFpn8WOyZw==} + engines: {node: ^10 || ^12 || >=14} + hasBin: true + peerDependencies: + postcss: ^8.1.0 + + axios@1.13.5: + resolution: {integrity: sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==} + + b4a@1.7.4: + resolution: {integrity: sha512-u20zJLDaSWpxaZ+zaAkEIB2dZZ1o+DF4T/MRbmsvGp9nletHOyiai19OzX1fF8xUBYsO1bPXxODvcd0978pnug==} + peerDependencies: + react-native-b4a: '*' + peerDependenciesMeta: + react-native-b4a: + optional: true + + balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + + balanced-match@4.0.2: + resolution: {integrity: sha512-x0K50QvKQ97fdEz2kPehIerj+YTeptKF9hyYkKf6egnwmMWAkADiO0QCzSp0R5xN8FTZgYaBfSaue46Ej62nMg==} + engines: {node: 20 || >=22} + + bare-events@2.8.2: + resolution: {integrity: sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==} + peerDependencies: + bare-abort-controller: '*' + peerDependenciesMeta: + bare-abort-controller: + optional: true + + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + + baseline-browser-mapping@2.9.19: + resolution: {integrity: sha512-ipDqC8FrAl/76p2SSWKSI+H9tFwm7vYqXQrItCuiVPt26Km0jS+NzSsBWAaBusvSbQcfJG+JitdMm+wZAgTYqg==} + hasBin: true + + bl@4.1.0: + resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + + body-parser@1.20.4: + resolution: {integrity: sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + + brace-expansion@1.1.12: + resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} + + brace-expansion@2.0.2: + resolution: {integrity: sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==} + + brace-expansion@5.0.2: + resolution: {integrity: sha512-Pdk8c9poy+YhOgVWw1JNN22/HcivgKWwpxKq04M/jTmHyCZn12WPJebZxdjSa5TmBqISrUSgNYU3eRORljfCCw==} + engines: {node: 20 || >=22} + + browserslist@4.28.1: + resolution: {integrity: sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==} + engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} + hasBin: true + + buffer-crc32@1.0.0: + resolution: {integrity: sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==} + engines: {node: '>=8.0.0'} + + buffer@5.7.1: + resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + + buffer@6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + + bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + + call-bound@1.0.4: + resolution: {integrity: sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==} + engines: {node: '>= 0.4'} + + callsites@3.1.0: + resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} + engines: {node: '>=6'} + + caniuse-lite@1.0.30001769: + resolution: {integrity: sha512-BCfFL1sHijQlBGWBMuJyhZUhzo7wer5sVj9hqekB/7xn0Ypy+pER/edCYQm4exbXj4WiySGp40P8UuTh6w1srg==} + + chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + + character-entities-legacy@1.1.4: + resolution: {integrity: sha512-3Xnr+7ZFS1uxeiUDvV02wQ+QDbc55o97tIV5zHScSPJpcLm/r0DFPcoY3tYRp+VZukxuMeKgXYmsXQHO05zQeA==} + + character-entities@1.2.4: + resolution: {integrity: sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw==} + + character-reference-invalid@1.1.4: + resolution: {integrity: sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg==} + + chardet@2.1.1: + resolution: {integrity: sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==} + + chokidar@4.0.3: + resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} + engines: {node: '>= 14.16.0'} + + class-variance-authority@0.7.1: + resolution: {integrity: sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg==} + + classcat@5.0.5: + resolution: {integrity: sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==} + + cli-cursor@3.1.0: + resolution: {integrity: sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==} + engines: {node: '>=8'} + + cli-spinners@2.9.2: + resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} + engines: {node: '>=6'} + + cli-width@3.0.0: + resolution: {integrity: sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==} + engines: {node: '>= 10'} + + clone@1.0.4: + resolution: {integrity: sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==} + engines: {node: '>=0.8'} + + clsx@2.1.1: + resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} + engines: {node: '>=6'} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + colors@1.4.0: + resolution: {integrity: sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==} + engines: {node: '>=0.1.90'} + + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + + comma-separated-tokens@1.0.8: + resolution: {integrity: sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==} + + commander@13.1.0: + resolution: {integrity: sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==} + engines: {node: '>=18'} + + compress-commons@6.0.2: + resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==} + engines: {node: '>= 14'} + + concat-map@0.0.1: + resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + + content-disposition@0.5.4: + resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} + engines: {node: '>= 0.6'} + + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + + convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + + cookie-signature@1.0.7: + resolution: {integrity: sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==} + + cookie@0.7.2: + resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} + engines: {node: '>= 0.6'} + + copy-to-clipboard@3.3.3: + resolution: {integrity: sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==} + + core-util-is@1.0.3: + resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} + + crc-32@1.2.2: + resolution: {integrity: sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==} + engines: {node: '>=0.8'} + hasBin: true + + crc32-stream@6.0.0: + resolution: {integrity: sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==} + engines: {node: '>= 14'} + + create-require@1.1.1: + resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} + + cron@4.4.0: + resolution: {integrity: sha512-fkdfq+b+AHI4cKdhZlppHveI/mgz2qpiYxcm+t5E5TsxX7QrLS1VE0+7GENEk9z0EeGPcpSciGv6ez24duWhwQ==} + engines: {node: '>=18.x'} + + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + + d3-color@3.1.0: + resolution: {integrity: sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==} + engines: {node: '>=12'} + + d3-dispatch@3.0.1: + resolution: {integrity: sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==} + engines: {node: '>=12'} + + d3-drag@3.0.0: + resolution: {integrity: sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==} + engines: {node: '>=12'} + + d3-ease@3.0.1: + resolution: {integrity: sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==} + engines: {node: '>=12'} + + d3-interpolate@3.0.1: + resolution: {integrity: sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==} + engines: {node: '>=12'} + + d3-selection@3.0.0: + resolution: {integrity: sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==} + engines: {node: '>=12'} + + d3-timer@3.0.1: + resolution: {integrity: sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==} + engines: {node: '>=12'} + + d3-transition@3.0.1: + resolution: {integrity: sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==} + engines: {node: '>=12'} + peerDependencies: + d3-selection: 2 - 3 + + d3-zoom@3.0.0: + resolution: {integrity: sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==} + engines: {node: '>=12'} + + dagre@0.8.5: + resolution: {integrity: sha512-/aTqmnRta7x7MCCpExk7HQL2O4owCT2h8NT//9I1OQ9vt29Pa0BzSAkR5lwFUcQ7491yVi/3CXU9jQ5o0Mn2Sw==} + + date-fns@4.1.0: + resolution: {integrity: sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==} + + debug@2.6.9: + resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + deep-is@0.1.4: + resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + + defaults@1.0.4: + resolution: {integrity: sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==} + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + depd@2.0.0: + resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} + engines: {node: '>= 0.8'} + + destroy@1.2.0: + resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + detect-node-es@1.1.0: + resolution: {integrity: sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==} + + diff@4.0.4: + resolution: {integrity: sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==} + engines: {node: '>=0.3.1'} + + dompurify@3.2.7: + resolution: {integrity: sha512-WhL/YuveyGXJaerVlMYGWhvQswa7myDG17P7Vu65EWC05o8vfeNbvNf4d/BOvH99+ZW+LlQsc1GDKMa1vNK6dw==} + + dotenv@16.6.1: + resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} + engines: {node: '>=12'} + + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + eastasianwidth@0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + + ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + + electron-to-chromium@1.5.286: + resolution: {integrity: sha512-9tfDXhJ4RKFNerfjdCcZfufu49vg620741MNs26a9+bhLThdB+plgMeou98CAaHu/WATj2iHOOHTp1hWtABj2A==} + + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + + emoji-regex@9.2.2: + resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + + encodeurl@2.0.0: + resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} + engines: {node: '>= 0.8'} + + enhanced-resolve@5.19.0: + resolution: {integrity: sha512-phv3E1Xl4tQOShqSte26C7Fl84EwUdZsyOuSSk9qtAGyyQs2s3jJzComh+Abf4g187lUUAvH+H26omrqia2aGg==} + engines: {node: '>=10.13.0'} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es-object-atoms@1.1.1: + resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} + engines: {node: '>= 0.4'} + + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + + esbuild@0.25.12: + resolution: {integrity: sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==} + engines: {node: '>=18'} + hasBin: true + + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} + engines: {node: '>=6'} + + escape-html@1.0.3: + resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + + escape-string-regexp@1.0.5: + resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} + engines: {node: '>=0.8.0'} + + escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + + eslint-scope@8.4.0: + resolution: {integrity: sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + eslint-visitor-keys@3.4.3: + resolution: {integrity: sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + eslint-visitor-keys@4.2.1: + resolution: {integrity: sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + eslint@9.39.2: + resolution: {integrity: sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + hasBin: true + peerDependencies: + jiti: '*' + peerDependenciesMeta: + jiti: + optional: true + + espree@10.4.0: + resolution: {integrity: sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + esquery@1.7.0: + resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==} + engines: {node: '>=0.10'} + + esrecurse@4.3.0: + resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==} + engines: {node: '>=4.0'} + + estraverse@5.3.0: + resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} + engines: {node: '>=4.0'} + + esutils@2.0.3: + resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} + engines: {node: '>=0.10.0'} + + etag@1.8.1: + resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} + engines: {node: '>= 0.6'} + + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + + events-universal@1.0.1: + resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} + + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + + express@4.22.1: + resolution: {integrity: sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==} + engines: {node: '>= 0.10.0'} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + + fast-json-stable-stringify@2.1.0: + resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + + fast-levenshtein@2.0.6: + resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + + fast-uri@3.1.0: + resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==} + + fault@1.0.4: + resolution: {integrity: sha512-CJ0HCB5tL5fYTEA7ToAq5+kTwd++Borf1/bifxd9iT70QcXr4MRrO3Llf8Ifs70q+SJcGHFtnIE/Nw6giCtECA==} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + figures@3.2.0: + resolution: {integrity: sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==} + engines: {node: '>=8'} + + file-entry-cache@8.0.0: + resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} + engines: {node: '>=16.0.0'} + + finalhandler@1.3.2: + resolution: {integrity: sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==} + engines: {node: '>= 0.8'} + + find-up@5.0.0: + resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} + engines: {node: '>=10'} + + flat-cache@4.0.1: + resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} + engines: {node: '>=16'} + + flatted@3.3.3: + resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==} + + follow-redirects@1.15.11: + resolution: {integrity: sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + + foreground-child@3.3.1: + resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} + engines: {node: '>=14'} + + form-data@4.0.5: + resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} + engines: {node: '>= 6'} + + format@0.2.2: + resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} + engines: {node: '>=0.4.x'} + + forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + + fraction.js@5.3.4: + resolution: {integrity: sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==} + + fresh@0.5.2: + resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} + engines: {node: '>= 0.6'} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + gensync@1.0.0-beta.2: + resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} + engines: {node: '>=6.9.0'} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-nonce@1.0.1: + resolution: {integrity: sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==} + engines: {node: '>=6'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + glob-parent@6.0.2: + resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} + engines: {node: '>=10.13.0'} + + glob@10.5.0: + resolution: {integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + hasBin: true + + glob@11.1.0: + resolution: {integrity: sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==} + engines: {node: 20 || >=22} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + hasBin: true + + globals@14.0.0: + resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==} + engines: {node: '>=18'} + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + + graceful-fs@4.2.11: + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + + graphlib@2.1.8: + resolution: {integrity: sha512-jcLLfkpoVGmH7/InMC/1hIvOPSUh38oJtGhvrOFGzioE1DZ+0YW16RgmOJhHiuWTvGiJQ9Z1Ik43JvkRPRvE+A==} + + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + + hasown@2.0.2: + resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + engines: {node: '>= 0.4'} + + hast-util-parse-selector@2.2.5: + resolution: {integrity: sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ==} + + hastscript@6.0.0: + resolution: {integrity: sha512-nDM6bvd7lIqDUiYEiu5Sl/+6ReP0BMk/2f4U/Rooccxkj0P5nm+acM5PrGJ/t5I8qPGiqZSE6hVAwZEdZIvP4w==} + + highlight.js@10.7.3: + resolution: {integrity: sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==} + + highlightjs-vue@1.0.0: + resolution: {integrity: sha512-PDEfEF102G23vHmPhLyPboFCD+BkMGu+GuJe2d9/eH4FsCwvgBpnc9n0pGE+ffKdph38s6foEZiEjdgHdzp+IA==} + + http-errors@2.0.1: + resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} + engines: {node: '>= 0.8'} + + iconv-lite@0.4.24: + resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} + engines: {node: '>=0.10.0'} + + iconv-lite@0.7.2: + resolution: {integrity: sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==} + engines: {node: '>=0.10.0'} + + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + + ignore@5.3.2: + resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} + engines: {node: '>= 4'} + + ignore@7.0.5: + resolution: {integrity: sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==} + engines: {node: '>= 4'} + + import-fresh@3.3.1: + resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} + engines: {node: '>=6'} + + imurmurhash@0.1.4: + resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} + engines: {node: '>=0.8.19'} + + inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + + inquirer@8.2.7: + resolution: {integrity: sha512-UjOaSel/iddGZJ5xP/Eixh6dY1XghiBw4XK13rCCIJcJfyhhoul/7KhLLUGtebEj6GDYM6Vnx/mVsjx2L/mFIA==} + engines: {node: '>=12.0.0'} + + ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + + is-alphabetical@1.0.4: + resolution: {integrity: sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg==} + + is-alphanumerical@1.0.4: + resolution: {integrity: sha512-UzoZUr+XfVz3t3v4KyGEniVL9BDRoQtY7tOyrRybkVNjDFWyo1yhXNGrrBTQxp3ib9BLAWs7k2YKBQsFRkZG9A==} + + is-decimal@1.0.4: + resolution: {integrity: sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw==} + + is-extglob@2.1.1: + resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} + engines: {node: '>=0.10.0'} + + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + + is-glob@4.0.3: + resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} + engines: {node: '>=0.10.0'} + + is-hexadecimal@1.0.4: + resolution: {integrity: sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw==} + + is-interactive@1.0.0: + resolution: {integrity: sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==} + engines: {node: '>=8'} + + is-stream@2.0.1: + resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} + engines: {node: '>=8'} + + is-unicode-supported@0.1.0: + resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} + engines: {node: '>=10'} + + isarray@1.0.0: + resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + + jackspeak@3.4.3: + resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + + jackspeak@4.2.3: + resolution: {integrity: sha512-ykkVRwrYvFm1nb2AJfKKYPr0emF6IiXDYUaFx4Zn9ZuIH7MrzEZ3sD5RlqGXNRpHtvUHJyOnCEFxOlNDtGo7wg==} + engines: {node: 20 || >=22} + + jiti@2.6.1: + resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==} + hasBin: true + + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + + js-yaml@4.1.1: + resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} + hasBin: true + + jsesc@3.1.0: + resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==} + engines: {node: '>=6'} + hasBin: true + + json-buffer@3.0.1: + resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} + + json-schema-traverse@0.4.1: + resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + json-schema@0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} + + json-stable-stringify-without-jsonify@1.0.1: + resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + + json5@2.2.3: + resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} + engines: {node: '>=6'} + hasBin: true + + keyv@4.5.4: + resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + + lazystream@1.0.1: + resolution: {integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==} + engines: {node: '>= 0.6.3'} + + levn@0.4.1: + resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} + engines: {node: '>= 0.8.0'} + + lightningcss-android-arm64@1.30.2: + resolution: {integrity: sha512-BH9sEdOCahSgmkVhBLeU7Hc9DWeZ1Eb6wNS6Da8igvUwAe0sqROHddIlvU06q3WyXVEOYDZ6ykBZQnjTbmo4+A==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + + lightningcss-darwin-arm64@1.30.2: + resolution: {integrity: sha512-ylTcDJBN3Hp21TdhRT5zBOIi73P6/W0qwvlFEk22fkdXchtNTOU4Qc37SkzV+EKYxLouZ6M4LG9NfZ1qkhhBWA==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + + lightningcss-darwin-x64@1.30.2: + resolution: {integrity: sha512-oBZgKchomuDYxr7ilwLcyms6BCyLn0z8J0+ZZmfpjwg9fRVZIR5/GMXd7r9RH94iDhld3UmSjBM6nXWM2TfZTQ==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + + lightningcss-freebsd-x64@1.30.2: + resolution: {integrity: sha512-c2bH6xTrf4BDpK8MoGG4Bd6zAMZDAXS569UxCAGcA7IKbHNMlhGQ89eRmvpIUGfKWNVdbhSbkQaWhEoMGmGslA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + + lightningcss-linux-arm-gnueabihf@1.30.2: + resolution: {integrity: sha512-eVdpxh4wYcm0PofJIZVuYuLiqBIakQ9uFZmipf6LF/HRj5Bgm0eb3qL/mr1smyXIS1twwOxNWndd8z0E374hiA==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + + lightningcss-linux-arm64-gnu@1.30.2: + resolution: {integrity: sha512-UK65WJAbwIJbiBFXpxrbTNArtfuznvxAJw4Q2ZGlU8kPeDIWEX1dg3rn2veBVUylA2Ezg89ktszWbaQnxD/e3A==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + + lightningcss-linux-arm64-musl@1.30.2: + resolution: {integrity: sha512-5Vh9dGeblpTxWHpOx8iauV02popZDsCYMPIgiuw97OJ5uaDsL86cnqSFs5LZkG3ghHoX5isLgWzMs+eD1YzrnA==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + + lightningcss-linux-x64-gnu@1.30.2: + resolution: {integrity: sha512-Cfd46gdmj1vQ+lR6VRTTadNHu6ALuw2pKR9lYq4FnhvgBc4zWY1EtZcAc6EffShbb1MFrIPfLDXD6Xprbnni4w==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + + lightningcss-linux-x64-musl@1.30.2: + resolution: {integrity: sha512-XJaLUUFXb6/QG2lGIW6aIk6jKdtjtcffUT0NKvIqhSBY3hh9Ch+1LCeH80dR9q9LBjG3ewbDjnumefsLsP6aiA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + + lightningcss-win32-arm64-msvc@1.30.2: + resolution: {integrity: sha512-FZn+vaj7zLv//D/192WFFVA0RgHawIcHqLX9xuWiQt7P0PtdFEVaxgF9rjM/IRYHQXNnk61/H/gb2Ei+kUQ4xQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + + lightningcss-win32-x64-msvc@1.30.2: + resolution: {integrity: sha512-5g1yc73p+iAkid5phb4oVFMB45417DkRevRbt/El/gKXJk4jid+vPFF/AXbxn05Aky8PapwzZrdJShv5C0avjw==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + + lightningcss@1.30.2: + resolution: {integrity: sha512-utfs7Pr5uJyyvDETitgsaqSyjCb2qNRAtuqUeWIAKztsOYdcACf2KtARYXg2pSvhkt+9NfoaNY7fxjl6nuMjIQ==} + engines: {node: '>= 12.0.0'} + + locate-path@6.0.0: + resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} + engines: {node: '>=10'} + + lodash.merge@4.6.2: + resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} + + lodash.truncate@4.4.2: + resolution: {integrity: sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==} + + lodash@4.17.23: + resolution: {integrity: sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==} + + log-symbols@4.1.0: + resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==} + engines: {node: '>=10'} + + lowlight@1.20.0: + resolution: {integrity: sha512-8Ktj+prEb1RoCPkEOrPMYUN/nCggB7qAWe3a7OpMjWQkh3l2RD5wKRQ+o8Q8YuI9RG/xs95waaI/E6ym/7NsTw==} + + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + + lru-cache@11.2.6: + resolution: {integrity: sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ==} + engines: {node: 20 || >=22} + + lru-cache@5.1.1: + resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} + + lucide-react@0.510.0: + resolution: {integrity: sha512-p8SQRAMVh7NhsAIETokSqDrc5CHnDLbV29mMnzaXx+Vc/hnqQzwI2r0FMWCcoTXnbw2KEjy48xwpGdEL+ck06Q==} + peerDependencies: + react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + lucide-react@0.525.0: + resolution: {integrity: sha512-Tm1txJ2OkymCGkvwoHt33Y2JpN5xucVq1slHcgE6Lk0WjDfjgKWor5CdVER8U6DvcfMwh4M8XxmpTiyzfmfDYQ==} + peerDependencies: + react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + lucide-react@0.544.0: + resolution: {integrity: sha512-t5tS44bqd825zAW45UQxpG2CvcC4urOwn2TrwSH8u+MjeE+1NnWl6QqeQ/6NdjMqdOygyiT9p3Ev0p1NJykxjw==} + peerDependencies: + react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + luxon@3.7.2: + resolution: {integrity: sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==} + engines: {node: '>=12'} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + make-error@1.3.6: + resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} + + marked@14.0.0: + resolution: {integrity: sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ==} + engines: {node: '>= 18'} + hasBin: true + + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + + media-typer@0.3.0: + resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} + engines: {node: '>= 0.6'} + + merge-descriptors@1.0.3: + resolution: {integrity: sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==} + + methods@1.1.2: + resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} + engines: {node: '>= 0.6'} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + mime@1.6.0: + resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} + engines: {node: '>=4'} + hasBin: true + + mimic-fn@2.1.0: + resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} + engines: {node: '>=6'} + + minimatch@10.2.0: + resolution: {integrity: sha512-ugkC31VaVg9cF0DFVoADH12k6061zNZkZON+aX8AWsR9GhPcErkcMBceb6znR8wLERM2AkkOxy2nWRLpT9Jq5w==} + engines: {node: 20 || >=22} + + minimatch@3.1.2: + resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} + + minimatch@5.1.6: + resolution: {integrity: sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==} + engines: {node: '>=10'} + + minimatch@9.0.5: + resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} + engines: {node: '>=16 || 14 >=14.17'} + + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + + minipass@7.1.2: + resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} + engines: {node: '>=16 || 14 >=14.17'} + + monaco-editor@0.55.1: + resolution: {integrity: sha512-jz4x+TJNFHwHtwuV9vA9rMujcZRb0CEilTEwG2rRSpe/A7Jdkuj8xPKttCgOh+v/lkHy7HsZ64oj+q3xoAFl9A==} + + motia@0.8.2-beta.139: + resolution: {integrity: sha512-s3p9DElWBruak4ngqo8GODbHR5yab1pqLYPtYybmPumi44k0E5doD80lp0KbOYgZqVDCL0at4fsMqZgUfINnQg==} + hasBin: true + + ms@2.0.0: + resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + mute-stream@0.0.8: + resolution: {integrity: sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==} + + nanoid@3.3.11: + resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + natural-compare@1.4.0: + resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + + negotiator@0.6.3: + resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} + engines: {node: '>= 0.6'} + + node-cron@3.0.3: + resolution: {integrity: sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==} + engines: {node: '>=6.0.0'} + + node-releases@2.0.27: + resolution: {integrity: sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==} + + normalize-path@3.0.0: + resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} + engines: {node: '>=0.10.0'} + + object-inspect@1.13.4: + resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} + engines: {node: '>= 0.4'} + + on-finished@2.4.1: + resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} + engines: {node: '>= 0.8'} + + onetime@5.1.2: + resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} + engines: {node: '>=6'} + + optionator@0.9.4: + resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} + engines: {node: '>= 0.8.0'} + + ora@5.4.1: + resolution: {integrity: sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==} + engines: {node: '>=10'} + + p-limit@3.1.0: + resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} + engines: {node: '>=10'} + + p-locate@5.0.0: + resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} + engines: {node: '>=10'} + + package-json-from-dist@1.0.1: + resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} + + parent-module@1.0.1: + resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} + engines: {node: '>=6'} + + parse-entities@2.0.0: + resolution: {integrity: sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==} + + parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + + path-exists@4.0.0: + resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} + engines: {node: '>=8'} + + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + + path-scurry@1.11.1: + resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} + engines: {node: '>=16 || 14 >=14.18'} + + path-scurry@2.0.1: + resolution: {integrity: sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==} + engines: {node: 20 || >=22} + + path-to-regexp@0.1.12: + resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.3: + resolution: {integrity: sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==} + engines: {node: '>=12'} + + postcss-value-parser@4.2.0: + resolution: {integrity: sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==} + + postcss@8.5.6: + resolution: {integrity: sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==} + engines: {node: ^10 || ^12 || >=14} + + prelude-ls@1.2.1: + resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} + engines: {node: '>= 0.8.0'} + + prismjs@1.27.0: + resolution: {integrity: sha512-t13BGPUlFDR7wRB5kQDG4jjl7XeuH6jbJGt11JHPL96qwsEHNX2+68tFXqc1/k+/jALsbSWJKUOT/hcYAZ5LkA==} + engines: {node: '>=6'} + + prismjs@1.30.0: + resolution: {integrity: sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==} + engines: {node: '>=6'} + + process-nextick-args@2.0.1: + resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + + process@0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + + property-information@5.6.0: + resolution: {integrity: sha512-YUHSPk+A30YPv+0Qf8i9Mbfe/C0hdPXk1s1jPVToV8pk8BQtpw10ct89Eo7OWkutrwqvT0eicAxlOg3dOAu8JA==} + + proxy-addr@2.0.7: + resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} + engines: {node: '>= 0.10'} + + proxy-from-env@1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + + python-ast@0.1.0: + resolution: {integrity: sha512-uMPE7HRMfsbHtQYPg/+EH9MJkynLfLr+0VJbeBgJHpt2wuDgf5hZrEczOIGNFKaO0W1HWiL7bSxA/EOOo70mIQ==} + + qs@6.14.2: + resolution: {integrity: sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==} + engines: {node: '>=0.6'} + + range-parser@1.2.1: + resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} + engines: {node: '>= 0.6'} + + raw-body@2.5.3: + resolution: {integrity: sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==} + engines: {node: '>= 0.8'} + + react-dom@19.2.4: + resolution: {integrity: sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==} + peerDependencies: + react: ^19.2.4 + + react-refresh@0.17.0: + resolution: {integrity: sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==} + engines: {node: '>=0.10.0'} + + react-remove-scroll-bar@2.3.8: + resolution: {integrity: sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + + react-remove-scroll@2.7.2: + resolution: {integrity: sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + react-resizable-panels@3.0.6: + resolution: {integrity: sha512-b3qKHQ3MLqOgSS+FRYKapNkJZf5EQzuf6+RLiq1/IlTHw99YrZ2NJZLk4hQIzTnnIkRg2LUqyVinu6YWWpUYew==} + peerDependencies: + react: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + react-dom: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + + react-style-singleton@2.2.3: + resolution: {integrity: sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + react-syntax-highlighter@15.6.6: + resolution: {integrity: sha512-DgXrc+AZF47+HvAPEmn7Ua/1p10jNoVZVI/LoPiYdtY+OM+/nG5yefLHKJwdKqY1adMuHFbeyBaG9j64ML7vTw==} + peerDependencies: + react: '>= 0.14.0' + + react-use-resizable@0.2.0: + resolution: {integrity: sha512-gp1cSSNzRDuRouWTYYxKE7f8VC/4cqdGpgbN9rDNdaFSwCfze/ki+WJL0Zouab/HILDpfIGW2JqAz94VmfroLA==} + peerDependencies: + react: '>=16.8.0' + + react18-json-view@0.2.9: + resolution: {integrity: sha512-z3JQgCwZRKbmWh54U94loCU6vE0ZoDBK7C8ZpcMYQB8jYMi+mR/fcgMI9jKgATeF0I6+OAF025PD+UKkXIqueQ==} + peerDependencies: + react: '>=16.8.0' + + react@19.2.4: + resolution: {integrity: sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==} + engines: {node: '>=0.10.0'} + + readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + + readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} + engines: {node: '>= 6'} + + readable-stream@4.7.0: + resolution: {integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + readdir-glob@1.1.3: + resolution: {integrity: sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==} + + readdirp@4.1.2: + resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} + engines: {node: '>= 14.18.0'} + + refractor@3.6.0: + resolution: {integrity: sha512-MY9W41IOWxxk31o+YvFCNyNzdkc9M20NoZK5vq6jkv4I/uh2zkWcfudj0Q1fovjUQJrNewS9NMzeTtqPf+n5EA==} + + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + + resolve-from@4.0.0: + resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} + engines: {node: '>=4'} + + restore-cursor@3.1.0: + resolution: {integrity: sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==} + engines: {node: '>=8'} + + rollup@4.57.1: + resolution: {integrity: sha512-oQL6lgK3e2QZeQ7gcgIkS2YZPg5slw37hYufJ3edKlfQSGGm8ICoxswK15ntSzF/a8+h7ekRy7k7oWc3BQ7y8A==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + run-async@2.4.1: + resolution: {integrity: sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==} + engines: {node: '>=0.12.0'} + + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + + safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + + safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + + scheduler@0.27.0: + resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==} + + semver@6.3.1: + resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} + hasBin: true + + semver@7.7.4: + resolution: {integrity: sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==} + engines: {node: '>=10'} + hasBin: true + + send@0.19.2: + resolution: {integrity: sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==} + engines: {node: '>= 0.8.0'} + + serve-static@1.16.3: + resolution: {integrity: sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==} + engines: {node: '>= 0.8.0'} + + setprototypeof@1.2.0: + resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + + side-channel-list@1.0.0: + resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} + engines: {node: '>= 0.4'} + + side-channel-map@1.0.1: + resolution: {integrity: sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==} + engines: {node: '>= 0.4'} + + side-channel-weakmap@1.0.2: + resolution: {integrity: sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==} + engines: {node: '>= 0.4'} + + side-channel@1.1.0: + resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} + engines: {node: '>= 0.4'} + + signal-exit@3.0.7: + resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + + slice-ansi@4.0.0: + resolution: {integrity: sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==} + engines: {node: '>=10'} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + space-separated-tokens@1.1.5: + resolution: {integrity: sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA==} + + state-local@1.0.7: + resolution: {integrity: sha512-HTEHMNieakEnoe33shBYcZ7NX83ACUjCu8c40iOGEZsngj9zRnkqS9j1pqQPXwobB0ZcVTk27REb7COQ0UR59w==} + + statuses@2.0.2: + resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} + engines: {node: '>= 0.8'} + + streamx@2.23.0: + resolution: {integrity: sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg==} + + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + + string-width@5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + + string_decoder@1.1.1: + resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + + string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + + strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + + strip-ansi@7.1.2: + resolution: {integrity: sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==} + engines: {node: '>=12'} + + strip-bom@3.0.0: + resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} + engines: {node: '>=4'} + + strip-json-comments@3.1.1: + resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} + engines: {node: '>=8'} + + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + + table@6.9.0: + resolution: {integrity: sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==} + engines: {node: '>=10.0.0'} + + tailwind-merge@3.4.0: + resolution: {integrity: sha512-uSaO4gnW+b3Y2aWoWfFpX62vn2sR3skfhbjsEnaBI81WD1wBLlHZe5sWf0AqjksNdYTbGBEd0UasQMT3SNV15g==} + + tailwindcss@4.1.18: + resolution: {integrity: sha512-4+Z+0yiYyEtUVCScyfHCxOYP06L5Ne+JiHhY2IjR2KWMIWhJOYZKLSGZaP5HkZ8+bY0cxfzwDE5uOmzFXyIwxw==} + + tapable@2.3.0: + resolution: {integrity: sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==} + engines: {node: '>=6'} + + tar-stream@3.1.7: + resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} + + text-decoder@1.2.4: + resolution: {integrity: sha512-mzlffA3tBNhziEHPK5L5InZg1d/ElNIpJhnhbDRNUtem/edZcJ5zg5FgwKKKOyklxk+6Jt+TrSu83musmvrDlg==} + + through@2.3.8: + resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==} + + tinyglobby@0.2.15: + resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} + engines: {node: '>=12.0.0'} + + toggle-selection@1.0.6: + resolution: {integrity: sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==} + + toidentifier@1.0.1: + resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} + engines: {node: '>=0.6'} + + ts-api-utils@2.4.0: + resolution: {integrity: sha512-3TaVTaAv2gTiMB35i3FiGJaRfwb3Pyn/j3m/bfAvGe8FB7CF6u+LMYqYlDh7reQf7UNvoTvdfAqHGmPGOSsPmA==} + engines: {node: '>=18.12'} + peerDependencies: + typescript: '>=4.8.4' + + ts-node@10.9.2: + resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==} + hasBin: true + peerDependencies: + '@swc/core': '>=1.2.50' + '@swc/wasm': '>=1.2.50' + '@types/node': '*' + typescript: '>=2.7' + peerDependenciesMeta: + '@swc/core': + optional: true + '@swc/wasm': + optional: true + + tsconfig-paths@4.2.0: + resolution: {integrity: sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==} + engines: {node: '>=6'} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + tw-animate-css@1.4.0: + resolution: {integrity: sha512-7bziOlRqH0hJx80h/3mbicLW7o8qLsH5+RaLR2t+OHM3D0JlWGODQKQ4cxbK7WlvmUxpcj6Kgu6EKqjrGFe3QQ==} + + type-check@0.4.0: + resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} + engines: {node: '>= 0.8.0'} + + type-fest@0.21.3: + resolution: {integrity: sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==} + engines: {node: '>=10'} + + type-is@1.6.18: + resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} + engines: {node: '>= 0.6'} + + typescript-eslint@8.55.0: + resolution: {integrity: sha512-HE4wj+r5lmDVS9gdaN0/+iqNvPZwGfnJ5lZuz7s5vLlg9ODw0bIiiETaios9LvFI1U94/VBXGm3CB2Y5cNFMpw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 + typescript: '>=4.8.4 <6.0.0' + + typescript@5.8.3: + resolution: {integrity: sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@7.16.0: + resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} + + unpipe@1.0.0: + resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} + engines: {node: '>= 0.8'} + + update-browserslist-db@1.2.3: + resolution: {integrity: sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==} + hasBin: true + peerDependencies: + browserslist: '>= 4.21.0' + + uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + + use-callback-ref@1.3.3: + resolution: {integrity: sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + use-sidecar@1.1.3: + resolution: {integrity: sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + use-sync-external-store@1.6.0: + resolution: {integrity: sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + + utils-merge@1.0.1: + resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} + engines: {node: '>= 0.4.0'} + + uuid@11.1.0: + resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} + hasBin: true + + uuid@8.3.2: + resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + hasBin: true + + v8-compile-cache-lib@3.0.1: + resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} + + vary@1.1.2: + resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} + engines: {node: '>= 0.8'} + + vite@6.4.1: + resolution: {integrity: sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + hasBin: true + peerDependencies: + '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 + jiti: '>=1.21.0' + less: '*' + lightningcss: ^1.21.0 + sass: '*' + sass-embedded: '*' + stylus: '*' + sugarss: '*' + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + jiti: + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + wcwidth@1.0.1: + resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} + + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + + word-wrap@1.2.5: + resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} + engines: {node: '>=0.10.0'} + + wrap-ansi@6.2.0: + resolution: {integrity: sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==} + engines: {node: '>=8'} + + wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + + wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + + ws@8.19.0: + resolution: {integrity: sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + + yallist@3.1.1: + resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + + yn@3.1.1: + resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} + engines: {node: '>=6'} + + yocto-queue@0.1.0: + resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} + engines: {node: '>=10'} + + zen-observable-ts@1.1.0: + resolution: {integrity: sha512-1h4zlLSqI2cRLPJUHJFL8bCWHhkpuXkF+dbGkRaWjgDIG26DmzyshUMrdV/rL3UnR+mhaX4fRq8LPouq0MYYIA==} + + zen-observable@0.8.15: + resolution: {integrity: sha512-PQ2PC7R9rslx84ndNBZB/Dkv8V8fZEpk83RLgXtYd0fwUgEjseMn1Dgajh2x6S8QbZAFa9p2qVCEuYZNgve0dQ==} + + zip-stream@6.0.1: + resolution: {integrity: sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==} + engines: {node: '>= 14'} + + zod-to-json-schema@3.25.1: + resolution: {integrity: sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA==} + peerDependencies: + zod: ^3.25 || ^4 + + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + + zustand@4.5.7: + resolution: {integrity: sha512-CHOUy7mu3lbD6o6LJLfllpjkzhHXSBlX8B9+qPddUsIfeF5S/UZ5q0kmCsnRqT1UHFQZchNFDDzMbQsuesHWlw==} + engines: {node: '>=12.7.0'} + peerDependencies: + '@types/react': '>=16.8' + immer: '>=9.0.6' + react: '>=16.8' + peerDependenciesMeta: + '@types/react': + optional: true + immer: + optional: true + react: + optional: true + + zustand@5.0.11: + resolution: {integrity: sha512-fdZY+dk7zn/vbWNCYmzZULHRrss0jx5pPFiOuMZ/5HJN6Yv3u+1Wswy/4MpZEkEGhtNH+pwxZB8OKgUBPzYAGg==} + engines: {node: '>=12.20.0'} + peerDependencies: + '@types/react': '>=18.0.0' + immer: '>=9.0.6' + react: '>=18.0.0' + use-sync-external-store: '>=1.2.0' + peerDependenciesMeta: + '@types/react': + optional: true + immer: + optional: true + react: + optional: true + use-sync-external-store: + optional: true + +snapshots: + + '@alloc/quick-lru@5.2.0': {} + + '@amplitude/analytics-connector@1.6.4': {} + + '@amplitude/analytics-core@2.38.0': + dependencies: + '@amplitude/analytics-connector': 1.6.4 + tslib: 2.8.1 + zen-observable-ts: 1.1.0 + + '@amplitude/analytics-node@1.5.35': + dependencies: + '@amplitude/analytics-core': 2.38.0 + tslib: 2.8.1 + + '@babel/code-frame@7.29.0': + dependencies: + '@babel/helper-validator-identifier': 7.28.5 + js-tokens: 4.0.0 + picocolors: 1.1.1 + + '@babel/compat-data@7.29.0': {} + + '@babel/core@7.29.0': + dependencies: + '@babel/code-frame': 7.29.0 + '@babel/generator': 7.29.1 + '@babel/helper-compilation-targets': 7.28.6 + '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0) + '@babel/helpers': 7.28.6 + '@babel/parser': 7.29.0 + '@babel/template': 7.28.6 + '@babel/traverse': 7.29.0 + '@babel/types': 7.29.0 + '@jridgewell/remapping': 2.3.5 + convert-source-map: 2.0.0 + debug: 4.4.3 + gensync: 1.0.0-beta.2 + json5: 2.2.3 + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + + '@babel/generator@7.29.1': + dependencies: + '@babel/parser': 7.29.0 + '@babel/types': 7.29.0 + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + jsesc: 3.1.0 + + '@babel/helper-compilation-targets@7.28.6': + dependencies: + '@babel/compat-data': 7.29.0 + '@babel/helper-validator-option': 7.27.1 + browserslist: 4.28.1 + lru-cache: 5.1.1 + semver: 6.3.1 + + '@babel/helper-globals@7.28.0': {} + + '@babel/helper-module-imports@7.28.6': + dependencies: + '@babel/traverse': 7.29.0 + '@babel/types': 7.29.0 + transitivePeerDependencies: + - supports-color + + '@babel/helper-module-transforms@7.28.6(@babel/core@7.29.0)': + dependencies: + '@babel/core': 7.29.0 + '@babel/helper-module-imports': 7.28.6 + '@babel/helper-validator-identifier': 7.28.5 + '@babel/traverse': 7.29.0 + transitivePeerDependencies: + - supports-color + + '@babel/helper-plugin-utils@7.28.6': {} + + '@babel/helper-string-parser@7.27.1': {} + + '@babel/helper-validator-identifier@7.28.5': {} + + '@babel/helper-validator-option@7.27.1': {} + + '@babel/helpers@7.28.6': + dependencies: + '@babel/template': 7.28.6 + '@babel/types': 7.29.0 + + '@babel/parser@7.29.0': + dependencies: + '@babel/types': 7.29.0 + + '@babel/plugin-transform-react-jsx-self@7.27.1(@babel/core@7.29.0)': + dependencies: + '@babel/core': 7.29.0 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/plugin-transform-react-jsx-source@7.27.1(@babel/core@7.29.0)': + dependencies: + '@babel/core': 7.29.0 + '@babel/helper-plugin-utils': 7.28.6 + + '@babel/runtime@7.28.6': {} + + '@babel/template@7.28.6': + dependencies: + '@babel/code-frame': 7.29.0 + '@babel/parser': 7.29.0 + '@babel/types': 7.29.0 + + '@babel/traverse@7.29.0': + dependencies: + '@babel/code-frame': 7.29.0 + '@babel/generator': 7.29.1 + '@babel/helper-globals': 7.28.0 + '@babel/parser': 7.29.0 + '@babel/template': 7.28.6 + '@babel/types': 7.29.0 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + '@babel/types@7.29.0': + dependencies: + '@babel/helper-string-parser': 7.27.1 + '@babel/helper-validator-identifier': 7.28.5 + + '@cspotcode/source-map-support@0.8.1': + dependencies: + '@jridgewell/trace-mapping': 0.3.9 + + '@esbuild/aix-ppc64@0.25.12': + optional: true + + '@esbuild/android-arm64@0.25.12': + optional: true + + '@esbuild/android-arm@0.25.12': + optional: true + + '@esbuild/android-x64@0.25.12': + optional: true + + '@esbuild/darwin-arm64@0.25.12': + optional: true + + '@esbuild/darwin-x64@0.25.12': + optional: true + + '@esbuild/freebsd-arm64@0.25.12': + optional: true + + '@esbuild/freebsd-x64@0.25.12': + optional: true + + '@esbuild/linux-arm64@0.25.12': + optional: true + + '@esbuild/linux-arm@0.25.12': + optional: true + + '@esbuild/linux-ia32@0.25.12': + optional: true + + '@esbuild/linux-loong64@0.25.12': + optional: true + + '@esbuild/linux-mips64el@0.25.12': + optional: true + + '@esbuild/linux-ppc64@0.25.12': + optional: true + + '@esbuild/linux-riscv64@0.25.12': + optional: true + + '@esbuild/linux-s390x@0.25.12': + optional: true + + '@esbuild/linux-x64@0.25.12': + optional: true + + '@esbuild/netbsd-arm64@0.25.12': + optional: true + + '@esbuild/netbsd-x64@0.25.12': + optional: true + + '@esbuild/openbsd-arm64@0.25.12': + optional: true + + '@esbuild/openbsd-x64@0.25.12': + optional: true + + '@esbuild/openharmony-arm64@0.25.12': + optional: true + + '@esbuild/sunos-x64@0.25.12': + optional: true + + '@esbuild/win32-arm64@0.25.12': + optional: true + + '@esbuild/win32-ia32@0.25.12': + optional: true + + '@esbuild/win32-x64@0.25.12': + optional: true + + '@eslint-community/eslint-utils@4.9.1(eslint@9.39.2(jiti@2.6.1))': + dependencies: + eslint: 9.39.2(jiti@2.6.1) + eslint-visitor-keys: 3.4.3 + + '@eslint-community/regexpp@4.12.2': {} + + '@eslint/config-array@0.21.1': + dependencies: + '@eslint/object-schema': 2.1.7 + debug: 4.4.3 + minimatch: 3.1.2 + transitivePeerDependencies: + - supports-color + + '@eslint/config-helpers@0.4.2': + dependencies: + '@eslint/core': 0.17.0 + + '@eslint/core@0.17.0': + dependencies: + '@types/json-schema': 7.0.15 + + '@eslint/eslintrc@3.3.3': + dependencies: + ajv: 6.12.6 + debug: 4.4.3 + espree: 10.4.0 + globals: 14.0.0 + ignore: 5.3.2 + import-fresh: 3.3.1 + js-yaml: 4.1.1 + minimatch: 3.1.2 + strip-json-comments: 3.1.1 + transitivePeerDependencies: + - supports-color + + '@eslint/js@9.39.2': {} + + '@eslint/object-schema@2.1.7': {} + + '@eslint/plugin-kit@0.4.1': + dependencies: + '@eslint/core': 0.17.0 + levn: 0.4.1 + + '@floating-ui/core@1.7.4': + dependencies: + '@floating-ui/utils': 0.2.10 + + '@floating-ui/dom@1.7.5': + dependencies: + '@floating-ui/core': 1.7.4 + '@floating-ui/utils': 0.2.10 + + '@floating-ui/react-dom@2.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@floating-ui/dom': 1.7.5 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@floating-ui/utils@0.2.10': {} + + '@humanfs/core@0.19.1': {} + + '@humanfs/node@0.16.7': + dependencies: + '@humanfs/core': 0.19.1 + '@humanwhocodes/retry': 0.4.3 + + '@humanwhocodes/module-importer@1.0.1': {} + + '@humanwhocodes/retry@0.4.3': {} + + '@inquirer/external-editor@1.0.3(@types/node@25.2.3)': + dependencies: + chardet: 2.1.1 + iconv-lite: 0.7.2 + optionalDependencies: + '@types/node': 25.2.3 + + '@isaacs/cliui@8.0.2': + dependencies: + string-width: 5.1.2 + string-width-cjs: string-width@4.2.3 + strip-ansi: 7.1.2 + strip-ansi-cjs: strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: wrap-ansi@7.0.0 + + '@isaacs/cliui@9.0.0': {} + + '@jridgewell/gen-mapping@0.3.13': + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/remapping@2.3.5': + dependencies: + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@jridgewell/trace-mapping@0.3.31': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + + '@jridgewell/trace-mapping@0.3.9': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + + '@monaco-editor/loader@1.7.0': + dependencies: + state-local: 1.0.7 + + '@monaco-editor/react@4.7.0(monaco-editor@0.55.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@monaco-editor/loader': 1.7.0 + monaco-editor: 0.55.1 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@motiadev/core@0.8.2-beta.139(@types/node@25.2.3)(typescript@5.8.3)': + dependencies: + '@amplitude/analytics-node': 1.5.35 + body-parser: 1.20.4 + colors: 1.4.0 + dotenv: 16.6.1 + express: 4.22.1 + node-cron: 3.0.3 + ts-node: 10.9.2(@types/node@25.2.3)(typescript@5.8.3) + tsconfig-paths: 4.2.0 + uuid: 11.1.0 + ws: 8.19.0 + zod: 3.25.76 + zod-to-json-schema: 3.25.1(zod@3.25.76) + transitivePeerDependencies: + - '@swc/core' + - '@swc/wasm' + - '@types/node' + - bufferutil + - supports-color + - typescript + - utf-8-validate + + '@motiadev/plugin-endpoint@0.8.2-beta.139(monaco-editor@0.55.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))': + dependencies: + '@monaco-editor/react': 4.7.0(monaco-editor@0.55.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@motiadev/stream-client-react': 0.8.2-beta.139(react@19.2.4) + '@motiadev/ui': 0.8.2-beta.139(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + clsx: 2.1.1 + json-schema: 0.4.0 + lucide-react: 0.544.0(react@19.2.4) + react18-json-view: 0.2.9(react@19.2.4) + tailwind-merge: 3.4.0 + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' + - immer + - monaco-editor + - react + - react-dom + - use-sync-external-store + + '@motiadev/stream-client-browser@0.8.2-beta.139': + dependencies: + '@motiadev/stream-client': 0.8.2-beta.139 + uuid: 11.1.0 + + '@motiadev/stream-client-node@0.8.2-beta.139': + dependencies: + '@motiadev/stream-client': 0.8.2-beta.139 + uuid: 11.1.0 + ws: 8.19.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@motiadev/stream-client-react@0.8.2-beta.139(react@19.2.4)': + dependencies: + '@motiadev/stream-client-browser': 0.8.2-beta.139 + react: 19.2.4 + + '@motiadev/stream-client@0.8.2-beta.139': + dependencies: + uuid: 11.1.0 + + '@motiadev/ui@0.8.2-beta.139(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))': + dependencies: + '@radix-ui/react-checkbox': 1.3.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-dropdown-menu': 2.1.16(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-label': 2.1.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-select': 2.2.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-separator': 1.1.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-slot': 1.2.4(react@19.2.4) + '@radix-ui/react-tabs': 1.1.13(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-tooltip': 1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + class-variance-authority: 0.7.1 + clsx: 2.1.1 + lucide-react: 0.525.0(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + react-resizable-panels: 3.0.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react-use-resizable: 0.2.0(react@19.2.4) + tailwind-merge: 3.4.0 + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' + - immer + - use-sync-external-store + + '@motiadev/workbench@0.8.2-beta.139(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(jiti@2.6.1)(lightningcss@1.30.2)(monaco-editor@0.55.1)(use-sync-external-store@1.6.0(react@19.2.4))': + dependencies: + '@monaco-editor/react': 4.7.0(monaco-editor@0.55.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@motiadev/plugin-endpoint': 0.8.2-beta.139(monaco-editor@0.55.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + '@motiadev/stream-client-react': 0.8.2-beta.139(react@19.2.4) + '@motiadev/ui': 0.8.2-beta.139(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + '@radix-ui/react-collapsible': 1.1.12(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-dialog': 1.1.15(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-dropdown-menu': 2.1.16(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-label': 2.1.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-navigation-menu': 1.2.14(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-scroll-area': 1.2.10(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-select': 2.2.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-separator': 1.1.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-slot': 1.2.4(react@19.2.4) + '@radix-ui/react-switch': 1.2.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-tabs': 1.1.13(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-tooltip': 1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@tailwindcss/postcss': 4.1.18 + '@vitejs/plugin-react': 4.7.0(vite@6.4.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2)) + '@xyflow/react': 12.10.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + autoprefixer: 10.4.24(postcss@8.5.6) + class-variance-authority: 0.7.1 + clsx: 2.1.1 + dagre: 0.8.5 + date-fns: 4.1.0 + fast-deep-equal: 3.1.3 + json-schema: 0.4.0 + lucide-react: 0.510.0(react@19.2.4) + postcss: 8.5.6 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + react-syntax-highlighter: 15.6.6(react@19.2.4) + react-use-resizable: 0.2.0(react@19.2.4) + react18-json-view: 0.2.9(react@19.2.4) + tailwind-merge: 3.4.0 + tailwindcss: 4.1.18 + tw-animate-css: 1.4.0 + typescript: 5.8.3 + typescript-eslint: 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) + vite: 6.4.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2) + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - '@types/node' + - '@types/react' + - '@types/react-dom' + - eslint + - immer + - jiti + - less + - lightningcss + - monaco-editor + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - use-sync-external-store + - yaml + + '@pkgjs/parseargs@0.11.0': + optional: true + + '@radix-ui/number@1.1.1': {} + + '@radix-ui/primitive@1.1.3': {} + + '@radix-ui/react-arrow@1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-checkbox@1.3.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + '@radix-ui/react-use-previous': 1.1.1(react@19.2.4) + '@radix-ui/react-use-size': 1.1.1(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-collapsible@1.1.12(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-collection@1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-compose-refs@1.1.2(react@19.2.4)': + dependencies: + react: 19.2.4 + + '@radix-ui/react-context@1.1.2(react@19.2.4)': + dependencies: + react: 19.2.4 + + '@radix-ui/react-dialog@1.1.15(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-dismissable-layer': 1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-focus-guards': 1.1.3(react@19.2.4) + '@radix-ui/react-focus-scope': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-portal': 1.1.9(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + aria-hidden: 1.2.6 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + react-remove-scroll: 2.7.2(react@19.2.4) + + '@radix-ui/react-direction@1.1.1(react@19.2.4)': + dependencies: + react: 19.2.4 + + '@radix-ui/react-dismissable-layer@1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + '@radix-ui/react-use-escape-keydown': 1.1.1(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-dropdown-menu@2.1.16(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-menu': 2.1.16(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-focus-guards@1.1.3(react@19.2.4)': + dependencies: + react: 19.2.4 + + '@radix-ui/react-focus-scope@1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-id@1.1.1(react@19.2.4)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + react: 19.2.4 + + '@radix-ui/react-label@2.1.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-primitive': 2.1.4(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-menu@2.1.16(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-collection': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-direction': 1.1.1(react@19.2.4) + '@radix-ui/react-dismissable-layer': 1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-focus-guards': 1.1.3(react@19.2.4) + '@radix-ui/react-focus-scope': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-popper': 1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-portal': 1.1.9(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-roving-focus': 1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + aria-hidden: 1.2.6 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + react-remove-scroll: 2.7.2(react@19.2.4) + + '@radix-ui/react-navigation-menu@1.2.14(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-collection': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-direction': 1.1.1(react@19.2.4) + '@radix-ui/react-dismissable-layer': 1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + '@radix-ui/react-use-previous': 1.1.1(react@19.2.4) + '@radix-ui/react-visually-hidden': 1.2.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-popper@1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@floating-ui/react-dom': 2.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-arrow': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + '@radix-ui/react-use-rect': 1.1.1(react@19.2.4) + '@radix-ui/react-use-size': 1.1.1(react@19.2.4) + '@radix-ui/rect': 1.1.1 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-portal@1.1.9(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-presence@1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-primitive@2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-slot': 1.2.3(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-primitive@2.1.4(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-slot': 1.2.4(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-roving-focus@1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-collection': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-direction': 1.1.1(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-scroll-area@1.2.10(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/number': 1.1.1 + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-direction': 1.1.1(react@19.2.4) + '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-select@2.2.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/number': 1.1.1 + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-collection': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-direction': 1.1.1(react@19.2.4) + '@radix-ui/react-dismissable-layer': 1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-focus-guards': 1.1.3(react@19.2.4) + '@radix-ui/react-focus-scope': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-popper': 1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-portal': 1.1.9(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + '@radix-ui/react-use-previous': 1.1.1(react@19.2.4) + '@radix-ui/react-visually-hidden': 1.2.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + aria-hidden: 1.2.6 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + react-remove-scroll: 2.7.2(react@19.2.4) + + '@radix-ui/react-separator@1.1.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-primitive': 2.1.4(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-slot@1.2.3(react@19.2.4)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + react: 19.2.4 + + '@radix-ui/react-slot@1.2.4(react@19.2.4)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + react: 19.2.4 + + '@radix-ui/react-switch@1.2.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + '@radix-ui/react-use-previous': 1.1.1(react@19.2.4) + '@radix-ui/react-use-size': 1.1.1(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-tabs@1.1.13(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-direction': 1.1.1(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-roving-focus': 1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-tooltip@1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.3 + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) + '@radix-ui/react-dismissable-layer': 1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-popper': 1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-portal': 1.1.9(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) + '@radix-ui/react-visually-hidden': 1.2.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/react-use-callback-ref@1.1.1(react@19.2.4)': + dependencies: + react: 19.2.4 + + '@radix-ui/react-use-controllable-state@1.2.2(react@19.2.4)': + dependencies: + '@radix-ui/react-use-effect-event': 0.0.2(react@19.2.4) + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + react: 19.2.4 + + '@radix-ui/react-use-effect-event@0.0.2(react@19.2.4)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + react: 19.2.4 + + '@radix-ui/react-use-escape-keydown@1.1.1(react@19.2.4)': + dependencies: + '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + react: 19.2.4 + + '@radix-ui/react-use-layout-effect@1.1.1(react@19.2.4)': + dependencies: + react: 19.2.4 + + '@radix-ui/react-use-previous@1.1.1(react@19.2.4)': + dependencies: + react: 19.2.4 + + '@radix-ui/react-use-rect@1.1.1(react@19.2.4)': + dependencies: + '@radix-ui/rect': 1.1.1 + react: 19.2.4 + + '@radix-ui/react-use-size@1.1.1(react@19.2.4)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) + react: 19.2.4 + + '@radix-ui/react-visually-hidden@1.2.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@radix-ui/rect@1.1.1': {} + + '@rolldown/pluginutils@1.0.0-beta.27': {} + + '@rollup/rollup-android-arm-eabi@4.57.1': + optional: true + + '@rollup/rollup-android-arm64@4.57.1': + optional: true + + '@rollup/rollup-darwin-arm64@4.57.1': + optional: true + + '@rollup/rollup-darwin-x64@4.57.1': + optional: true + + '@rollup/rollup-freebsd-arm64@4.57.1': + optional: true + + '@rollup/rollup-freebsd-x64@4.57.1': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.57.1': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.57.1': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.57.1': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.57.1': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.57.1': + optional: true + + '@rollup/rollup-linux-loong64-musl@4.57.1': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.57.1': + optional: true + + '@rollup/rollup-linux-ppc64-musl@4.57.1': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.57.1': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.57.1': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.57.1': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.57.1': + optional: true + + '@rollup/rollup-linux-x64-musl@4.57.1': + optional: true + + '@rollup/rollup-openbsd-x64@4.57.1': + optional: true + + '@rollup/rollup-openharmony-arm64@4.57.1': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.57.1': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.57.1': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.57.1': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.57.1': + optional: true + + '@tailwindcss/node@4.1.18': + dependencies: + '@jridgewell/remapping': 2.3.5 + enhanced-resolve: 5.19.0 + jiti: 2.6.1 + lightningcss: 1.30.2 + magic-string: 0.30.21 + source-map-js: 1.2.1 + tailwindcss: 4.1.18 + + '@tailwindcss/oxide-android-arm64@4.1.18': + optional: true + + '@tailwindcss/oxide-darwin-arm64@4.1.18': + optional: true + + '@tailwindcss/oxide-darwin-x64@4.1.18': + optional: true + + '@tailwindcss/oxide-freebsd-x64@4.1.18': + optional: true + + '@tailwindcss/oxide-linux-arm-gnueabihf@4.1.18': + optional: true + + '@tailwindcss/oxide-linux-arm64-gnu@4.1.18': + optional: true + + '@tailwindcss/oxide-linux-arm64-musl@4.1.18': + optional: true + + '@tailwindcss/oxide-linux-x64-gnu@4.1.18': + optional: true + + '@tailwindcss/oxide-linux-x64-musl@4.1.18': + optional: true + + '@tailwindcss/oxide-wasm32-wasi@4.1.18': + optional: true + + '@tailwindcss/oxide-win32-arm64-msvc@4.1.18': + optional: true + + '@tailwindcss/oxide-win32-x64-msvc@4.1.18': + optional: true + + '@tailwindcss/oxide@4.1.18': + optionalDependencies: + '@tailwindcss/oxide-android-arm64': 4.1.18 + '@tailwindcss/oxide-darwin-arm64': 4.1.18 + '@tailwindcss/oxide-darwin-x64': 4.1.18 + '@tailwindcss/oxide-freebsd-x64': 4.1.18 + '@tailwindcss/oxide-linux-arm-gnueabihf': 4.1.18 + '@tailwindcss/oxide-linux-arm64-gnu': 4.1.18 + '@tailwindcss/oxide-linux-arm64-musl': 4.1.18 + '@tailwindcss/oxide-linux-x64-gnu': 4.1.18 + '@tailwindcss/oxide-linux-x64-musl': 4.1.18 + '@tailwindcss/oxide-wasm32-wasi': 4.1.18 + '@tailwindcss/oxide-win32-arm64-msvc': 4.1.18 + '@tailwindcss/oxide-win32-x64-msvc': 4.1.18 + + '@tailwindcss/postcss@4.1.18': + dependencies: + '@alloc/quick-lru': 5.2.0 + '@tailwindcss/node': 4.1.18 + '@tailwindcss/oxide': 4.1.18 + postcss: 8.5.6 + tailwindcss: 4.1.18 + + '@tsconfig/node10@1.0.12': {} + + '@tsconfig/node12@1.0.11': {} + + '@tsconfig/node14@1.0.3': {} + + '@tsconfig/node16@1.0.4': {} + + '@types/babel__core@7.20.5': + dependencies: + '@babel/parser': 7.29.0 + '@babel/types': 7.29.0 + '@types/babel__generator': 7.27.0 + '@types/babel__template': 7.4.4 + '@types/babel__traverse': 7.28.0 + + '@types/babel__generator@7.27.0': + dependencies: + '@babel/types': 7.29.0 + + '@types/babel__template@7.4.4': + dependencies: + '@babel/parser': 7.29.0 + '@babel/types': 7.29.0 + + '@types/babel__traverse@7.28.0': + dependencies: + '@babel/types': 7.29.0 + + '@types/d3-color@3.1.3': {} + + '@types/d3-drag@3.0.7': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-interpolate@3.0.4': + dependencies: + '@types/d3-color': 3.1.3 + + '@types/d3-selection@3.0.11': {} + + '@types/d3-transition@3.0.9': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-zoom@3.0.8': + dependencies: + '@types/d3-interpolate': 3.0.4 + '@types/d3-selection': 3.0.11 + + '@types/estree@1.0.8': {} + + '@types/hast@2.3.10': + dependencies: + '@types/unist': 2.0.11 + + '@types/json-schema@7.0.15': {} + + '@types/luxon@3.7.1': {} + + '@types/node@25.2.3': + dependencies: + undici-types: 7.16.0 + + '@types/trusted-types@2.0.7': + optional: true + + '@types/unist@2.0.11': {} + + '@types/zen-observable@0.8.3': {} + + '@typescript-eslint/eslint-plugin@8.55.0(@typescript-eslint/parser@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3))(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3)': + dependencies: + '@eslint-community/regexpp': 4.12.2 + '@typescript-eslint/parser': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) + '@typescript-eslint/scope-manager': 8.55.0 + '@typescript-eslint/type-utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) + '@typescript-eslint/utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) + '@typescript-eslint/visitor-keys': 8.55.0 + eslint: 9.39.2(jiti@2.6.1) + ignore: 7.0.5 + natural-compare: 1.4.0 + ts-api-utils: 2.4.0(typescript@5.8.3) + typescript: 5.8.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/parser@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3)': + dependencies: + '@typescript-eslint/scope-manager': 8.55.0 + '@typescript-eslint/types': 8.55.0 + '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.8.3) + '@typescript-eslint/visitor-keys': 8.55.0 + debug: 4.4.3 + eslint: 9.39.2(jiti@2.6.1) + typescript: 5.8.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/project-service@8.55.0(typescript@5.8.3)': + dependencies: + '@typescript-eslint/tsconfig-utils': 8.55.0(typescript@5.8.3) + '@typescript-eslint/types': 8.55.0 + debug: 4.4.3 + typescript: 5.8.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/scope-manager@8.55.0': + dependencies: + '@typescript-eslint/types': 8.55.0 + '@typescript-eslint/visitor-keys': 8.55.0 + + '@typescript-eslint/tsconfig-utils@8.55.0(typescript@5.8.3)': + dependencies: + typescript: 5.8.3 + + '@typescript-eslint/type-utils@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3)': + dependencies: + '@typescript-eslint/types': 8.55.0 + '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.8.3) + '@typescript-eslint/utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) + debug: 4.4.3 + eslint: 9.39.2(jiti@2.6.1) + ts-api-utils: 2.4.0(typescript@5.8.3) + typescript: 5.8.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/types@8.55.0': {} + + '@typescript-eslint/typescript-estree@8.55.0(typescript@5.8.3)': + dependencies: + '@typescript-eslint/project-service': 8.55.0(typescript@5.8.3) + '@typescript-eslint/tsconfig-utils': 8.55.0(typescript@5.8.3) + '@typescript-eslint/types': 8.55.0 + '@typescript-eslint/visitor-keys': 8.55.0 + debug: 4.4.3 + minimatch: 9.0.5 + semver: 7.7.4 + tinyglobby: 0.2.15 + ts-api-utils: 2.4.0(typescript@5.8.3) + typescript: 5.8.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/utils@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3)': + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) + '@typescript-eslint/scope-manager': 8.55.0 + '@typescript-eslint/types': 8.55.0 + '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.8.3) + eslint: 9.39.2(jiti@2.6.1) + typescript: 5.8.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/visitor-keys@8.55.0': + dependencies: + '@typescript-eslint/types': 8.55.0 + eslint-visitor-keys: 4.2.1 + + '@vitejs/plugin-react@4.7.0(vite@6.4.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2))': + dependencies: + '@babel/core': 7.29.0 + '@babel/plugin-transform-react-jsx-self': 7.27.1(@babel/core@7.29.0) + '@babel/plugin-transform-react-jsx-source': 7.27.1(@babel/core@7.29.0) + '@rolldown/pluginutils': 1.0.0-beta.27 + '@types/babel__core': 7.20.5 + react-refresh: 0.17.0 + vite: 6.4.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2) + transitivePeerDependencies: + - supports-color + + '@xyflow/react@12.10.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@xyflow/system': 0.0.74 + classcat: 5.0.5 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + zustand: 4.5.7(react@19.2.4) + transitivePeerDependencies: + - '@types/react' + - immer + + '@xyflow/system@0.0.74': + dependencies: + '@types/d3-drag': 3.0.7 + '@types/d3-interpolate': 3.0.4 + '@types/d3-selection': 3.0.11 + '@types/d3-transition': 3.0.9 + '@types/d3-zoom': 3.0.8 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-zoom: 3.0.0 + + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + + accepts@1.3.8: + dependencies: + mime-types: 2.1.35 + negotiator: 0.6.3 + + acorn-jsx@5.3.2(acorn@8.15.0): + dependencies: + acorn: 8.15.0 + + acorn-walk@8.3.4: + dependencies: + acorn: 8.15.0 + + acorn@8.15.0: {} + + ajv@6.12.6: + dependencies: + fast-deep-equal: 3.1.3 + fast-json-stable-stringify: 2.1.0 + json-schema-traverse: 0.4.1 + uri-js: 4.4.1 + + ajv@8.17.1: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.0 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + + ansi-escapes@4.3.2: + dependencies: + type-fest: 0.21.3 + + ansi-regex@5.0.1: {} + + ansi-regex@6.2.2: {} + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + ansi-styles@6.2.3: {} + + antlr4ts@0.5.0-alpha.4: {} + + archiver-utils@5.0.2: + dependencies: + glob: 10.5.0 + graceful-fs: 4.2.11 + is-stream: 2.0.1 + lazystream: 1.0.1 + lodash: 4.17.23 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + + archiver@7.0.1: + dependencies: + archiver-utils: 5.0.2 + async: 3.2.6 + buffer-crc32: 1.0.0 + readable-stream: 4.7.0 + readdir-glob: 1.1.3 + tar-stream: 3.1.7 + zip-stream: 6.0.1 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + arg@4.1.3: {} + + argparse@2.0.1: {} + + aria-hidden@1.2.6: + dependencies: + tslib: 2.8.1 + + array-flatten@1.1.1: {} + + astral-regex@2.0.0: {} + + async@3.2.6: {} + + asynckit@0.4.0: {} + + autoprefixer@10.4.24(postcss@8.5.6): + dependencies: + browserslist: 4.28.1 + caniuse-lite: 1.0.30001769 + fraction.js: 5.3.4 + picocolors: 1.1.1 + postcss: 8.5.6 + postcss-value-parser: 4.2.0 + + axios@1.13.5: + dependencies: + follow-redirects: 1.15.11 + form-data: 4.0.5 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + + b4a@1.7.4: {} + + balanced-match@1.0.2: {} + + balanced-match@4.0.2: + dependencies: + jackspeak: 4.2.3 + + bare-events@2.8.2: {} + + base64-js@1.5.1: {} + + baseline-browser-mapping@2.9.19: {} + + bl@4.1.0: + dependencies: + buffer: 5.7.1 + inherits: 2.0.4 + readable-stream: 3.6.2 + + body-parser@1.20.4: + dependencies: + bytes: 3.1.2 + content-type: 1.0.5 + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + http-errors: 2.0.1 + iconv-lite: 0.4.24 + on-finished: 2.4.1 + qs: 6.14.2 + raw-body: 2.5.3 + type-is: 1.6.18 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + + brace-expansion@1.1.12: + dependencies: + balanced-match: 1.0.2 + concat-map: 0.0.1 + + brace-expansion@2.0.2: + dependencies: + balanced-match: 1.0.2 + + brace-expansion@5.0.2: + dependencies: + balanced-match: 4.0.2 + + browserslist@4.28.1: + dependencies: + baseline-browser-mapping: 2.9.19 + caniuse-lite: 1.0.30001769 + electron-to-chromium: 1.5.286 + node-releases: 2.0.27 + update-browserslist-db: 1.2.3(browserslist@4.28.1) + + buffer-crc32@1.0.0: {} + + buffer@5.7.1: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + + buffer@6.0.3: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + + bytes@3.1.2: {} + + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + + call-bound@1.0.4: + dependencies: + call-bind-apply-helpers: 1.0.2 + get-intrinsic: 1.3.0 + + callsites@3.1.0: {} + + caniuse-lite@1.0.30001769: {} + + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + character-entities-legacy@1.1.4: {} + + character-entities@1.2.4: {} + + character-reference-invalid@1.1.4: {} + + chardet@2.1.1: {} + + chokidar@4.0.3: + dependencies: + readdirp: 4.1.2 + + class-variance-authority@0.7.1: + dependencies: + clsx: 2.1.1 + + classcat@5.0.5: {} + + cli-cursor@3.1.0: + dependencies: + restore-cursor: 3.1.0 + + cli-spinners@2.9.2: {} + + cli-width@3.0.0: {} + + clone@1.0.4: {} + + clsx@2.1.1: {} + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + colors@1.4.0: {} + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + + comma-separated-tokens@1.0.8: {} + + commander@13.1.0: {} + + compress-commons@6.0.2: + dependencies: + crc-32: 1.2.2 + crc32-stream: 6.0.0 + is-stream: 2.0.1 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + + concat-map@0.0.1: {} + + content-disposition@0.5.4: + dependencies: + safe-buffer: 5.2.1 + + content-type@1.0.5: {} + + convert-source-map@2.0.0: {} + + cookie-signature@1.0.7: {} + + cookie@0.7.2: {} + + copy-to-clipboard@3.3.3: + dependencies: + toggle-selection: 1.0.6 + + core-util-is@1.0.3: {} + + crc-32@1.2.2: {} + + crc32-stream@6.0.0: + dependencies: + crc-32: 1.2.2 + readable-stream: 4.7.0 + + create-require@1.1.1: {} + + cron@4.4.0: + dependencies: + '@types/luxon': 3.7.1 + luxon: 3.7.2 + + cross-spawn@7.0.6: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + + d3-color@3.1.0: {} + + d3-dispatch@3.0.1: {} + + d3-drag@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-selection: 3.0.0 + + d3-ease@3.0.1: {} + + d3-interpolate@3.0.1: + dependencies: + d3-color: 3.1.0 + + d3-selection@3.0.0: {} + + d3-timer@3.0.1: {} + + d3-transition@3.0.1(d3-selection@3.0.0): + dependencies: + d3-color: 3.1.0 + d3-dispatch: 3.0.1 + d3-ease: 3.0.1 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-timer: 3.0.1 + + d3-zoom@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-transition: 3.0.1(d3-selection@3.0.0) + + dagre@0.8.5: + dependencies: + graphlib: 2.1.8 + lodash: 4.17.23 + + date-fns@4.1.0: {} + + debug@2.6.9: + dependencies: + ms: 2.0.0 + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + deep-is@0.1.4: {} + + defaults@1.0.4: + dependencies: + clone: 1.0.4 + + delayed-stream@1.0.0: {} + + depd@2.0.0: {} + + destroy@1.2.0: {} + + detect-libc@2.1.2: {} + + detect-node-es@1.1.0: {} + + diff@4.0.4: {} + + dompurify@3.2.7: + optionalDependencies: + '@types/trusted-types': 2.0.7 + + dotenv@16.6.1: {} + + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + eastasianwidth@0.2.0: {} + + ee-first@1.1.1: {} + + electron-to-chromium@1.5.286: {} + + emoji-regex@8.0.0: {} + + emoji-regex@9.2.2: {} + + encodeurl@2.0.0: {} + + enhanced-resolve@5.19.0: + dependencies: + graceful-fs: 4.2.11 + tapable: 2.3.0 + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es-object-atoms@1.1.1: + dependencies: + es-errors: 1.3.0 + + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.2 + + esbuild@0.25.12: + optionalDependencies: + '@esbuild/aix-ppc64': 0.25.12 + '@esbuild/android-arm': 0.25.12 + '@esbuild/android-arm64': 0.25.12 + '@esbuild/android-x64': 0.25.12 + '@esbuild/darwin-arm64': 0.25.12 + '@esbuild/darwin-x64': 0.25.12 + '@esbuild/freebsd-arm64': 0.25.12 + '@esbuild/freebsd-x64': 0.25.12 + '@esbuild/linux-arm': 0.25.12 + '@esbuild/linux-arm64': 0.25.12 + '@esbuild/linux-ia32': 0.25.12 + '@esbuild/linux-loong64': 0.25.12 + '@esbuild/linux-mips64el': 0.25.12 + '@esbuild/linux-ppc64': 0.25.12 + '@esbuild/linux-riscv64': 0.25.12 + '@esbuild/linux-s390x': 0.25.12 + '@esbuild/linux-x64': 0.25.12 + '@esbuild/netbsd-arm64': 0.25.12 + '@esbuild/netbsd-x64': 0.25.12 + '@esbuild/openbsd-arm64': 0.25.12 + '@esbuild/openbsd-x64': 0.25.12 + '@esbuild/openharmony-arm64': 0.25.12 + '@esbuild/sunos-x64': 0.25.12 + '@esbuild/win32-arm64': 0.25.12 + '@esbuild/win32-ia32': 0.25.12 + '@esbuild/win32-x64': 0.25.12 + + escalade@3.2.0: {} + + escape-html@1.0.3: {} + + escape-string-regexp@1.0.5: {} + + escape-string-regexp@4.0.0: {} + + eslint-scope@8.4.0: + dependencies: + esrecurse: 4.3.0 + estraverse: 5.3.0 + + eslint-visitor-keys@3.4.3: {} + + eslint-visitor-keys@4.2.1: {} + + eslint@9.39.2(jiti@2.6.1): + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) + '@eslint-community/regexpp': 4.12.2 + '@eslint/config-array': 0.21.1 + '@eslint/config-helpers': 0.4.2 + '@eslint/core': 0.17.0 + '@eslint/eslintrc': 3.3.3 + '@eslint/js': 9.39.2 + '@eslint/plugin-kit': 0.4.1 + '@humanfs/node': 0.16.7 + '@humanwhocodes/module-importer': 1.0.1 + '@humanwhocodes/retry': 0.4.3 + '@types/estree': 1.0.8 + ajv: 6.12.6 + chalk: 4.1.2 + cross-spawn: 7.0.6 + debug: 4.4.3 + escape-string-regexp: 4.0.0 + eslint-scope: 8.4.0 + eslint-visitor-keys: 4.2.1 + espree: 10.4.0 + esquery: 1.7.0 + esutils: 2.0.3 + fast-deep-equal: 3.1.3 + file-entry-cache: 8.0.0 + find-up: 5.0.0 + glob-parent: 6.0.2 + ignore: 5.3.2 + imurmurhash: 0.1.4 + is-glob: 4.0.3 + json-stable-stringify-without-jsonify: 1.0.1 + lodash.merge: 4.6.2 + minimatch: 3.1.2 + natural-compare: 1.4.0 + optionator: 0.9.4 + optionalDependencies: + jiti: 2.6.1 + transitivePeerDependencies: + - supports-color + + espree@10.4.0: + dependencies: + acorn: 8.15.0 + acorn-jsx: 5.3.2(acorn@8.15.0) + eslint-visitor-keys: 4.2.1 + + esquery@1.7.0: + dependencies: + estraverse: 5.3.0 + + esrecurse@4.3.0: + dependencies: + estraverse: 5.3.0 + + estraverse@5.3.0: {} + + esutils@2.0.3: {} + + etag@1.8.1: {} + + event-target-shim@5.0.1: {} + + events-universal@1.0.1: + dependencies: + bare-events: 2.8.2 + transitivePeerDependencies: + - bare-abort-controller + + events@3.3.0: {} + + express@4.22.1: + dependencies: + accepts: 1.3.8 + array-flatten: 1.1.1 + body-parser: 1.20.4 + content-disposition: 0.5.4 + content-type: 1.0.5 + cookie: 0.7.2 + cookie-signature: 1.0.7 + debug: 2.6.9 + depd: 2.0.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 1.3.2 + fresh: 0.5.2 + http-errors: 2.0.1 + merge-descriptors: 1.0.3 + methods: 1.1.2 + on-finished: 2.4.1 + parseurl: 1.3.3 + path-to-regexp: 0.1.12 + proxy-addr: 2.0.7 + qs: 6.14.2 + range-parser: 1.2.1 + safe-buffer: 5.2.1 + send: 0.19.2 + serve-static: 1.16.3 + setprototypeof: 1.2.0 + statuses: 2.0.2 + type-is: 1.6.18 + utils-merge: 1.0.1 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + + fast-deep-equal@3.1.3: {} + + fast-fifo@1.3.2: {} + + fast-json-stable-stringify@2.1.0: {} + + fast-levenshtein@2.0.6: {} + + fast-uri@3.1.0: {} + + fault@1.0.4: + dependencies: + format: 0.2.2 + + fdir@6.5.0(picomatch@4.0.3): + optionalDependencies: + picomatch: 4.0.3 + + figures@3.2.0: + dependencies: + escape-string-regexp: 1.0.5 + + file-entry-cache@8.0.0: + dependencies: + flat-cache: 4.0.1 + + finalhandler@1.3.2: + dependencies: + debug: 2.6.9 + encodeurl: 2.0.0 + escape-html: 1.0.3 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.2 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + + find-up@5.0.0: + dependencies: + locate-path: 6.0.0 + path-exists: 4.0.0 + + flat-cache@4.0.1: + dependencies: + flatted: 3.3.3 + keyv: 4.5.4 + + flatted@3.3.3: {} + + follow-redirects@1.15.11: {} + + foreground-child@3.3.1: + dependencies: + cross-spawn: 7.0.6 + signal-exit: 4.1.0 + + form-data@4.0.5: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.2 + mime-types: 2.1.35 + + format@0.2.2: {} + + forwarded@0.2.0: {} + + fraction.js@5.3.4: {} + + fresh@0.5.2: {} + + fsevents@2.3.3: + optional: true + + function-bind@1.1.2: {} + + gensync@1.0.0-beta.2: {} + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.1 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.2 + math-intrinsics: 1.1.0 + + get-nonce@1.0.1: {} + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.1 + + glob-parent@6.0.2: + dependencies: + is-glob: 4.0.3 + + glob@10.5.0: + dependencies: + foreground-child: 3.3.1 + jackspeak: 3.4.3 + minimatch: 9.0.5 + minipass: 7.1.2 + package-json-from-dist: 1.0.1 + path-scurry: 1.11.1 + + glob@11.1.0: + dependencies: + foreground-child: 3.3.1 + jackspeak: 4.2.3 + minimatch: 10.2.0 + minipass: 7.1.2 + package-json-from-dist: 1.0.1 + path-scurry: 2.0.1 + + globals@14.0.0: {} + + gopd@1.2.0: {} + + graceful-fs@4.2.11: {} + + graphlib@2.1.8: + dependencies: + lodash: 4.17.23 + + has-flag@4.0.0: {} + + has-symbols@1.1.0: {} + + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + + hasown@2.0.2: + dependencies: + function-bind: 1.1.2 + + hast-util-parse-selector@2.2.5: {} + + hastscript@6.0.0: + dependencies: + '@types/hast': 2.3.10 + comma-separated-tokens: 1.0.8 + hast-util-parse-selector: 2.2.5 + property-information: 5.6.0 + space-separated-tokens: 1.1.5 + + highlight.js@10.7.3: {} + + highlightjs-vue@1.0.0: {} + + http-errors@2.0.1: + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.2 + toidentifier: 1.0.1 + + iconv-lite@0.4.24: + dependencies: + safer-buffer: 2.1.2 + + iconv-lite@0.7.2: + dependencies: + safer-buffer: 2.1.2 + + ieee754@1.2.1: {} + + ignore@5.3.2: {} + + ignore@7.0.5: {} + + import-fresh@3.3.1: + dependencies: + parent-module: 1.0.1 + resolve-from: 4.0.0 + + imurmurhash@0.1.4: {} + + inherits@2.0.4: {} + + inquirer@8.2.7(@types/node@25.2.3): + dependencies: + '@inquirer/external-editor': 1.0.3(@types/node@25.2.3) + ansi-escapes: 4.3.2 + chalk: 4.1.2 + cli-cursor: 3.1.0 + cli-width: 3.0.0 + figures: 3.2.0 + lodash: 4.17.23 + mute-stream: 0.0.8 + ora: 5.4.1 + run-async: 2.4.1 + rxjs: 7.8.2 + string-width: 4.2.3 + strip-ansi: 6.0.1 + through: 2.3.8 + wrap-ansi: 6.2.0 + transitivePeerDependencies: + - '@types/node' + + ipaddr.js@1.9.1: {} + + is-alphabetical@1.0.4: {} + + is-alphanumerical@1.0.4: + dependencies: + is-alphabetical: 1.0.4 + is-decimal: 1.0.4 + + is-decimal@1.0.4: {} + + is-extglob@2.1.1: {} + + is-fullwidth-code-point@3.0.0: {} + + is-glob@4.0.3: + dependencies: + is-extglob: 2.1.1 + + is-hexadecimal@1.0.4: {} + + is-interactive@1.0.0: {} + + is-stream@2.0.1: {} + + is-unicode-supported@0.1.0: {} + + isarray@1.0.0: {} + + isexe@2.0.0: {} + + jackspeak@3.4.3: + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + + jackspeak@4.2.3: + dependencies: + '@isaacs/cliui': 9.0.0 + + jiti@2.6.1: {} + + js-tokens@4.0.0: {} + + js-yaml@4.1.1: + dependencies: + argparse: 2.0.1 + + jsesc@3.1.0: {} + + json-buffer@3.0.1: {} + + json-schema-traverse@0.4.1: {} + + json-schema-traverse@1.0.0: {} + + json-schema@0.4.0: {} + + json-stable-stringify-without-jsonify@1.0.1: {} + + json5@2.2.3: {} + + keyv@4.5.4: + dependencies: + json-buffer: 3.0.1 + + lazystream@1.0.1: + dependencies: + readable-stream: 2.3.8 + + levn@0.4.1: + dependencies: + prelude-ls: 1.2.1 + type-check: 0.4.0 + + lightningcss-android-arm64@1.30.2: + optional: true + + lightningcss-darwin-arm64@1.30.2: + optional: true + + lightningcss-darwin-x64@1.30.2: + optional: true + + lightningcss-freebsd-x64@1.30.2: + optional: true + + lightningcss-linux-arm-gnueabihf@1.30.2: + optional: true + + lightningcss-linux-arm64-gnu@1.30.2: + optional: true + + lightningcss-linux-arm64-musl@1.30.2: + optional: true + + lightningcss-linux-x64-gnu@1.30.2: + optional: true + + lightningcss-linux-x64-musl@1.30.2: + optional: true + + lightningcss-win32-arm64-msvc@1.30.2: + optional: true + + lightningcss-win32-x64-msvc@1.30.2: + optional: true + + lightningcss@1.30.2: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.30.2 + lightningcss-darwin-arm64: 1.30.2 + lightningcss-darwin-x64: 1.30.2 + lightningcss-freebsd-x64: 1.30.2 + lightningcss-linux-arm-gnueabihf: 1.30.2 + lightningcss-linux-arm64-gnu: 1.30.2 + lightningcss-linux-arm64-musl: 1.30.2 + lightningcss-linux-x64-gnu: 1.30.2 + lightningcss-linux-x64-musl: 1.30.2 + lightningcss-win32-arm64-msvc: 1.30.2 + lightningcss-win32-x64-msvc: 1.30.2 + + locate-path@6.0.0: + dependencies: + p-locate: 5.0.0 + + lodash.merge@4.6.2: {} + + lodash.truncate@4.4.2: {} + + lodash@4.17.23: {} + + log-symbols@4.1.0: + dependencies: + chalk: 4.1.2 + is-unicode-supported: 0.1.0 + + lowlight@1.20.0: + dependencies: + fault: 1.0.4 + highlight.js: 10.7.3 + + lru-cache@10.4.3: {} + + lru-cache@11.2.6: {} + + lru-cache@5.1.1: + dependencies: + yallist: 3.1.1 + + lucide-react@0.510.0(react@19.2.4): + dependencies: + react: 19.2.4 + + lucide-react@0.525.0(react@19.2.4): + dependencies: + react: 19.2.4 + + lucide-react@0.544.0(react@19.2.4): + dependencies: + react: 19.2.4 + + luxon@3.7.2: {} + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + + make-error@1.3.6: {} + + marked@14.0.0: {} + + math-intrinsics@1.1.0: {} + + media-typer@0.3.0: {} + + merge-descriptors@1.0.3: {} + + methods@1.1.2: {} + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + mime@1.6.0: {} + + mimic-fn@2.1.0: {} + + minimatch@10.2.0: + dependencies: + brace-expansion: 5.0.2 + + minimatch@3.1.2: + dependencies: + brace-expansion: 1.1.12 + + minimatch@5.1.6: + dependencies: + brace-expansion: 2.0.2 + + minimatch@9.0.5: + dependencies: + brace-expansion: 2.0.2 + + minimist@1.2.8: {} + + minipass@7.1.2: {} + + monaco-editor@0.55.1: + dependencies: + dompurify: 3.2.7 + marked: 14.0.0 + + motia@0.8.2-beta.139(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(jiti@2.6.1)(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.8.3)(use-sync-external-store@1.6.0(react@19.2.4)): + dependencies: + '@amplitude/analytics-node': 1.5.35 + '@motiadev/core': 0.8.2-beta.139(@types/node@25.2.3)(typescript@5.8.3) + '@motiadev/stream-client-node': 0.8.2-beta.139 + '@motiadev/workbench': 0.8.2-beta.139(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(jiti@2.6.1)(lightningcss@1.30.2)(monaco-editor@0.55.1)(use-sync-external-store@1.6.0(react@19.2.4)) + antlr4ts: 0.5.0-alpha.4 + archiver: 7.0.1 + axios: 1.13.5 + chokidar: 4.0.3 + colors: 1.4.0 + commander: 13.1.0 + cron: 4.4.0 + dotenv: 16.6.1 + esbuild: 0.25.12 + express: 4.22.1 + glob: 11.1.0 + inquirer: 8.2.7(@types/node@25.2.3) + node-cron: 3.0.3 + python-ast: 0.1.0 + table: 6.9.0 + ts-node: 10.9.2(@types/node@25.2.3)(typescript@5.8.3) + transitivePeerDependencies: + - '@swc/core' + - '@swc/wasm' + - '@types/node' + - '@types/react' + - '@types/react-dom' + - bare-abort-controller + - bufferutil + - debug + - eslint + - immer + - jiti + - less + - lightningcss + - monaco-editor + - react-native-b4a + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - typescript + - use-sync-external-store + - utf-8-validate + - yaml + + ms@2.0.0: {} + + ms@2.1.3: {} + + mute-stream@0.0.8: {} + + nanoid@3.3.11: {} + + natural-compare@1.4.0: {} + + negotiator@0.6.3: {} + + node-cron@3.0.3: + dependencies: + uuid: 8.3.2 + + node-releases@2.0.27: {} + + normalize-path@3.0.0: {} + + object-inspect@1.13.4: {} + + on-finished@2.4.1: + dependencies: + ee-first: 1.1.1 + + onetime@5.1.2: + dependencies: + mimic-fn: 2.1.0 + + optionator@0.9.4: + dependencies: + deep-is: 0.1.4 + fast-levenshtein: 2.0.6 + levn: 0.4.1 + prelude-ls: 1.2.1 + type-check: 0.4.0 + word-wrap: 1.2.5 + + ora@5.4.1: + dependencies: + bl: 4.1.0 + chalk: 4.1.2 + cli-cursor: 3.1.0 + cli-spinners: 2.9.2 + is-interactive: 1.0.0 + is-unicode-supported: 0.1.0 + log-symbols: 4.1.0 + strip-ansi: 6.0.1 + wcwidth: 1.0.1 + + p-limit@3.1.0: + dependencies: + yocto-queue: 0.1.0 + + p-locate@5.0.0: + dependencies: + p-limit: 3.1.0 + + package-json-from-dist@1.0.1: {} + + parent-module@1.0.1: + dependencies: + callsites: 3.1.0 + + parse-entities@2.0.0: + dependencies: + character-entities: 1.2.4 + character-entities-legacy: 1.1.4 + character-reference-invalid: 1.1.4 + is-alphanumerical: 1.0.4 + is-decimal: 1.0.4 + is-hexadecimal: 1.0.4 + + parseurl@1.3.3: {} + + path-exists@4.0.0: {} + + path-key@3.1.1: {} + + path-scurry@1.11.1: + dependencies: + lru-cache: 10.4.3 + minipass: 7.1.2 + + path-scurry@2.0.1: + dependencies: + lru-cache: 11.2.6 + minipass: 7.1.2 + + path-to-regexp@0.1.12: {} + + picocolors@1.1.1: {} + + picomatch@4.0.3: {} + + postcss-value-parser@4.2.0: {} + + postcss@8.5.6: + dependencies: + nanoid: 3.3.11 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + prelude-ls@1.2.1: {} + + prismjs@1.27.0: {} + + prismjs@1.30.0: {} + + process-nextick-args@2.0.1: {} + + process@0.11.10: {} + + property-information@5.6.0: + dependencies: + xtend: 4.0.2 + + proxy-addr@2.0.7: + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + + proxy-from-env@1.1.0: {} + + punycode@2.3.1: {} + + python-ast@0.1.0: + dependencies: + antlr4ts: 0.5.0-alpha.4 + + qs@6.14.2: + dependencies: + side-channel: 1.1.0 + + range-parser@1.2.1: {} + + raw-body@2.5.3: + dependencies: + bytes: 3.1.2 + http-errors: 2.0.1 + iconv-lite: 0.4.24 + unpipe: 1.0.0 + + react-dom@19.2.4(react@19.2.4): + dependencies: + react: 19.2.4 + scheduler: 0.27.0 + + react-refresh@0.17.0: {} + + react-remove-scroll-bar@2.3.8(react@19.2.4): + dependencies: + react: 19.2.4 + react-style-singleton: 2.2.3(react@19.2.4) + tslib: 2.8.1 + + react-remove-scroll@2.7.2(react@19.2.4): + dependencies: + react: 19.2.4 + react-remove-scroll-bar: 2.3.8(react@19.2.4) + react-style-singleton: 2.2.3(react@19.2.4) + tslib: 2.8.1 + use-callback-ref: 1.3.3(react@19.2.4) + use-sidecar: 1.1.3(react@19.2.4) + + react-resizable-panels@3.0.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + dependencies: + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + react-style-singleton@2.2.3(react@19.2.4): + dependencies: + get-nonce: 1.0.1 + react: 19.2.4 + tslib: 2.8.1 + + react-syntax-highlighter@15.6.6(react@19.2.4): + dependencies: + '@babel/runtime': 7.28.6 + highlight.js: 10.7.3 + highlightjs-vue: 1.0.0 + lowlight: 1.20.0 + prismjs: 1.30.0 + react: 19.2.4 + refractor: 3.6.0 + + react-use-resizable@0.2.0(react@19.2.4): + dependencies: + react: 19.2.4 + + react18-json-view@0.2.9(react@19.2.4): + dependencies: + copy-to-clipboard: 3.3.3 + react: 19.2.4 + + react@19.2.4: {} + + readable-stream@2.3.8: + dependencies: + core-util-is: 1.0.3 + inherits: 2.0.4 + isarray: 1.0.0 + process-nextick-args: 2.0.1 + safe-buffer: 5.1.2 + string_decoder: 1.1.1 + util-deprecate: 1.0.2 + + readable-stream@3.6.2: + dependencies: + inherits: 2.0.4 + string_decoder: 1.3.0 + util-deprecate: 1.0.2 + + readable-stream@4.7.0: + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + + readdir-glob@1.1.3: + dependencies: + minimatch: 5.1.6 + + readdirp@4.1.2: {} + + refractor@3.6.0: + dependencies: + hastscript: 6.0.0 + parse-entities: 2.0.0 + prismjs: 1.27.0 + + require-from-string@2.0.2: {} + + resolve-from@4.0.0: {} + + restore-cursor@3.1.0: + dependencies: + onetime: 5.1.2 + signal-exit: 3.0.7 + + rollup@4.57.1: + dependencies: + '@types/estree': 1.0.8 + optionalDependencies: + '@rollup/rollup-android-arm-eabi': 4.57.1 + '@rollup/rollup-android-arm64': 4.57.1 + '@rollup/rollup-darwin-arm64': 4.57.1 + '@rollup/rollup-darwin-x64': 4.57.1 + '@rollup/rollup-freebsd-arm64': 4.57.1 + '@rollup/rollup-freebsd-x64': 4.57.1 + '@rollup/rollup-linux-arm-gnueabihf': 4.57.1 + '@rollup/rollup-linux-arm-musleabihf': 4.57.1 + '@rollup/rollup-linux-arm64-gnu': 4.57.1 + '@rollup/rollup-linux-arm64-musl': 4.57.1 + '@rollup/rollup-linux-loong64-gnu': 4.57.1 + '@rollup/rollup-linux-loong64-musl': 4.57.1 + '@rollup/rollup-linux-ppc64-gnu': 4.57.1 + '@rollup/rollup-linux-ppc64-musl': 4.57.1 + '@rollup/rollup-linux-riscv64-gnu': 4.57.1 + '@rollup/rollup-linux-riscv64-musl': 4.57.1 + '@rollup/rollup-linux-s390x-gnu': 4.57.1 + '@rollup/rollup-linux-x64-gnu': 4.57.1 + '@rollup/rollup-linux-x64-musl': 4.57.1 + '@rollup/rollup-openbsd-x64': 4.57.1 + '@rollup/rollup-openharmony-arm64': 4.57.1 + '@rollup/rollup-win32-arm64-msvc': 4.57.1 + '@rollup/rollup-win32-ia32-msvc': 4.57.1 + '@rollup/rollup-win32-x64-gnu': 4.57.1 + '@rollup/rollup-win32-x64-msvc': 4.57.1 + fsevents: 2.3.3 + + run-async@2.4.1: {} + + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + + safe-buffer@5.1.2: {} + + safe-buffer@5.2.1: {} + + safer-buffer@2.1.2: {} + + scheduler@0.27.0: {} + + semver@6.3.1: {} + + semver@7.7.4: {} + + send@0.19.2: + dependencies: + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 0.5.2 + http-errors: 2.0.1 + mime: 1.6.0 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.2 + transitivePeerDependencies: + - supports-color + + serve-static@1.16.3: + dependencies: + encodeurl: 2.0.0 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 0.19.2 + transitivePeerDependencies: + - supports-color + + setprototypeof@1.2.0: {} + + shebang-command@2.0.0: + dependencies: + shebang-regex: 3.0.0 + + shebang-regex@3.0.0: {} + + side-channel-list@1.0.0: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + + side-channel-map@1.0.1: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + + side-channel-weakmap@1.0.2: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + side-channel-map: 1.0.1 + + side-channel@1.1.0: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + side-channel-list: 1.0.0 + side-channel-map: 1.0.1 + side-channel-weakmap: 1.0.2 + + signal-exit@3.0.7: {} + + signal-exit@4.1.0: {} + + slice-ansi@4.0.0: + dependencies: + ansi-styles: 4.3.0 + astral-regex: 2.0.0 + is-fullwidth-code-point: 3.0.0 + + source-map-js@1.2.1: {} + + space-separated-tokens@1.1.5: {} + + state-local@1.0.7: {} + + statuses@2.0.2: {} + + streamx@2.23.0: + dependencies: + events-universal: 1.0.1 + fast-fifo: 1.3.2 + text-decoder: 1.2.4 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + + string-width@5.1.2: + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.1.2 + + string_decoder@1.1.1: + dependencies: + safe-buffer: 5.1.2 + + string_decoder@1.3.0: + dependencies: + safe-buffer: 5.2.1 + + strip-ansi@6.0.1: + dependencies: + ansi-regex: 5.0.1 + + strip-ansi@7.1.2: + dependencies: + ansi-regex: 6.2.2 + + strip-bom@3.0.0: {} + + strip-json-comments@3.1.1: {} + + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + + table@6.9.0: + dependencies: + ajv: 8.17.1 + lodash.truncate: 4.4.2 + slice-ansi: 4.0.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + tailwind-merge@3.4.0: {} + + tailwindcss@4.1.18: {} + + tapable@2.3.0: {} + + tar-stream@3.1.7: + dependencies: + b4a: 1.7.4 + fast-fifo: 1.3.2 + streamx: 2.23.0 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + text-decoder@1.2.4: + dependencies: + b4a: 1.7.4 + transitivePeerDependencies: + - react-native-b4a + + through@2.3.8: {} + + tinyglobby@0.2.15: + dependencies: + fdir: 6.5.0(picomatch@4.0.3) + picomatch: 4.0.3 + + toggle-selection@1.0.6: {} + + toidentifier@1.0.1: {} + + ts-api-utils@2.4.0(typescript@5.8.3): + dependencies: + typescript: 5.8.3 + + ts-node@10.9.2(@types/node@25.2.3)(typescript@5.8.3): + dependencies: + '@cspotcode/source-map-support': 0.8.1 + '@tsconfig/node10': 1.0.12 + '@tsconfig/node12': 1.0.11 + '@tsconfig/node14': 1.0.3 + '@tsconfig/node16': 1.0.4 + '@types/node': 25.2.3 + acorn: 8.15.0 + acorn-walk: 8.3.4 + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.4 + make-error: 1.3.6 + typescript: 5.8.3 + v8-compile-cache-lib: 3.0.1 + yn: 3.1.1 + + tsconfig-paths@4.2.0: + dependencies: + json5: 2.2.3 + minimist: 1.2.8 + strip-bom: 3.0.0 + + tslib@2.8.1: {} + + tw-animate-css@1.4.0: {} + + type-check@0.4.0: + dependencies: + prelude-ls: 1.2.1 + + type-fest@0.21.3: {} + + type-is@1.6.18: + dependencies: + media-typer: 0.3.0 + mime-types: 2.1.35 + + typescript-eslint@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3): + dependencies: + '@typescript-eslint/eslint-plugin': 8.55.0(@typescript-eslint/parser@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3))(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) + '@typescript-eslint/parser': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) + '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.8.3) + '@typescript-eslint/utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) + eslint: 9.39.2(jiti@2.6.1) + typescript: 5.8.3 + transitivePeerDependencies: + - supports-color + + typescript@5.8.3: {} + + undici-types@7.16.0: {} + + unpipe@1.0.0: {} + + update-browserslist-db@1.2.3(browserslist@4.28.1): + dependencies: + browserslist: 4.28.1 + escalade: 3.2.0 + picocolors: 1.1.1 + + uri-js@4.4.1: + dependencies: + punycode: 2.3.1 + + use-callback-ref@1.3.3(react@19.2.4): + dependencies: + react: 19.2.4 + tslib: 2.8.1 + + use-sidecar@1.1.3(react@19.2.4): + dependencies: + detect-node-es: 1.1.0 + react: 19.2.4 + tslib: 2.8.1 + + use-sync-external-store@1.6.0(react@19.2.4): + dependencies: + react: 19.2.4 + + util-deprecate@1.0.2: {} + + utils-merge@1.0.1: {} + + uuid@11.1.0: {} + + uuid@8.3.2: {} + + v8-compile-cache-lib@3.0.1: {} + + vary@1.1.2: {} + + vite@6.4.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2): + dependencies: + esbuild: 0.25.12 + fdir: 6.5.0(picomatch@4.0.3) + picomatch: 4.0.3 + postcss: 8.5.6 + rollup: 4.57.1 + tinyglobby: 0.2.15 + optionalDependencies: + '@types/node': 25.2.3 + fsevents: 2.3.3 + jiti: 2.6.1 + lightningcss: 1.30.2 + + wcwidth@1.0.1: + dependencies: + defaults: 1.0.4 + + which@2.0.2: + dependencies: + isexe: 2.0.0 + + word-wrap@1.2.5: {} + + wrap-ansi@6.2.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + wrap-ansi@7.0.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + wrap-ansi@8.1.0: + dependencies: + ansi-styles: 6.2.3 + string-width: 5.1.2 + strip-ansi: 7.1.2 + + ws@8.19.0: {} + + xtend@4.0.2: {} + + yallist@3.1.1: {} + + yn@3.1.1: {} + + yocto-queue@0.1.0: {} + + zen-observable-ts@1.1.0: + dependencies: + '@types/zen-observable': 0.8.3 + zen-observable: 0.8.15 + + zen-observable@0.8.15: {} + + zip-stream@6.0.1: + dependencies: + archiver-utils: 5.0.2 + compress-commons: 6.0.2 + readable-stream: 4.7.0 + + zod-to-json-schema@3.25.1(zod@3.25.76): + dependencies: + zod: 3.25.76 + + zod@3.25.76: {} + + zustand@4.5.7(react@19.2.4): + dependencies: + use-sync-external-store: 1.6.0(react@19.2.4) + optionalDependencies: + react: 19.2.4 + + zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)): + optionalDependencies: + react: 19.2.4 + use-sync-external-store: 1.6.0(react@19.2.4) diff --git a/bbdev/api/pnpm-workspace.yaml b/bbdev/api/pnpm-workspace.yaml new file mode 100644 index 000000000..3334c0e43 --- /dev/null +++ b/bbdev/api/pnpm-workspace.yaml @@ -0,0 +1 @@ +packages: [] diff --git a/bbdev/api/requirements.txt b/bbdev/api/requirements.txt new file mode 100644 index 000000000..31a1c8927 --- /dev/null +++ b/bbdev/api/requirements.txt @@ -0,0 +1,2 @@ +pydantic>=2.0.0 +requests>=2.28.0 diff --git a/bbdev/api/services/README.md b/bbdev/api/services/README.md new file mode 100644 index 000000000..9dd3d0ba6 --- /dev/null +++ b/bbdev/api/services/README.md @@ -0,0 +1,435 @@ +# Function Calling Tool Management System + +A modular and extensible Function Calling tool management framework for AI Agent interaction with external systems. + +## 📁 Directory Structure + +``` +tools/ +├── __init__.py # Module exports +├── base.py # Tool base class and context +├── registry.py # Tool registry and manager +├── file_tools.py # File operation tools +├── presets.py # Predefined tool sets +└── README.md # This document +``` + +## 🚀 Quick Start + +### 1. Using Predefined Tool Manager + +```python +from steps.tools import create_code_agent_manager + +# Create tool manager (file_tools already registered) +manager = create_code_agent_manager() + +# Get tool definitions (send to LLM) +tools_schema = manager.get_tools_schema() + +# Call LLM +response = llm.chat( + messages=messages, + tools=tools_schema # Pass in tool definitions +) + +# Execute tool calls returned by LLM +if response.tool_calls: + for tool_call in response.tool_calls: + result = manager.execute_tool( + tool_name=tool_call.function.name, + arguments=tool_call.function.arguments, + work_dir="/path/to/project", + logger=logger + ) +``` + +### 2. Custom Tools + +```python +from steps.tools import Tool, ToolManager +import json + +class RunCommandTool(Tool): + """Command execution tool""" + + def get_name(self) -> str: + return "run_command" + + def get_description(self) -> str: + return "Execute a shell command" + + def get_parameters(self) -> dict: + return { + "type": "object", + "properties": { + "command": {"type": "string", "description": "Command to execute"} + }, + "required": ["command"] + } + + def execute(self, arguments: dict, context) -> str: + import subprocess + cmd = arguments.get("command") + + try: + result = subprocess.run( + cmd, + shell=True, + capture_output=True, + text=True, + timeout=30 + ) + + return json.dumps({ + "stdout": result.stdout, + "stderr": result.stderr, + "returncode": result.returncode + }) + except Exception as e: + return json.dumps({"error": str(e)}) + +# Register custom tool +manager = ToolManager() +manager.register_tool(RunCommandTool()) +``` + +## 📚 Core Concepts + +### Tool (Base Class) + +All tools inherit from the `Tool` base class: + +```python +class MyTool(Tool): + def get_name(self) -> str: + """Tool name (unique identifier)""" + return "my_tool" + + def get_description(self) -> str: + """Tool description (tells AI what this tool does)""" + return "My custom tool" + + def get_parameters(self) -> dict: + """Parameter definition (JSON Schema format)""" + return { + "type": "object", + "properties": { + "param1": {"type": "string", "description": "Parameter 1"}, + "param2": {"type": "integer", "description": "Parameter 2"} + }, + "required": ["param1"] + } + + def execute(self, arguments: dict, context) -> str: + """ + Execute tool logic + + Args: + arguments: Parameters passed by AI + context: ToolContext object (contains work_dir, logger, etc.) + + Returns: + Execution result (string, can be JSON) + """ + # Implement tool logic + return json.dumps({"result": "success"}) +``` + +### ToolContext (Execution Context) + +Execution environment provided to tools: + +```python +context = ToolContext( + work_dir="/path/to/project", # Working directory + logger=logger, # Logger + extra_key="extra_value" # Custom extension fields +) + +# Use in tool +class MyTool(Tool): + def execute(self, arguments, context): + context.log_info("Starting execution") + work_dir = context.work_dir + custom = context.extra.get("extra_key") + # ... +``` + +### ToolRegistry (Tool Registry) + +Manage tool registration and lookup: + +```python +from steps.tools import ToolRegistry, ReadFileTool, WriteFileTool + +registry = ToolRegistry() +registry.register(ReadFileTool()) +registry.register(WriteFileTool()) + +# Get tool +tool = registry.get("read_file") + +# List all tools +tools = registry.list_tools() # ['read_file', 'write_file'] + +# Convert to OpenAI format +schema = registry.to_openai_format() +``` + +### ToolManager (Tool Manager) + +High-level wrapper providing more convenient interface: + +```python +from steps.tools import ToolManager + +manager = ToolManager() +manager.register_tools([ReadFileTool(), WriteFileTool()]) + +# Get tool definitions +schema = manager.get_tools_schema() + +# Execute tool +result = manager.execute_tool( + tool_name="read_file", + arguments={"path": "main.py"}, + work_dir="/project" +) + +# View execution log +log = manager.get_execution_log() +``` + +## 🛠️ Built-in Tools + +### File Operation Tools + +#### read_file +Read file content + +```json +{ + "name": "read_file", + "parameters": { + "path": "relative/path/to/file.txt" + } +} +``` + +#### write_file +Write file content (automatically creates directories) + +```json +{ + "name": "write_file", + "parameters": { + "path": "output/file.txt", + "content": "file content here" + } +} +``` + +#### list_files +List files in directory + +```json +{ + "name": "list_files", + "parameters": { + "path": "src" // Optional, defaults to current directory + } +} +``` + +## 🎨 Predefined Tool Sets + +```python +from steps.tools import get_preset, list_presets + +# View available tool sets +presets = list_presets() # ['file_tools', 'code_agent'] + +# Get tool set +tools = get_preset("file_tools") # Returns [ReadFileTool, WriteFileTool, ...] + +# Create manager +from steps.tools import create_code_agent_manager +manager = create_code_agent_manager() +``` + +## 📝 Complete Examples + +### Agent Integration Example + +```python +from steps.tools import create_code_agent_manager +import httpx +import json + +async def run_agent(prompt: str, work_dir: str): + # 1. Create tool manager + manager = create_code_agent_manager() + tools_schema = manager.get_tools_schema() + + # 2. Initialize conversation + messages = [ + {"role": "system", "content": "You are a code assistant"}, + {"role": "user", "content": prompt} + ] + + # 3. AI loop + max_iterations = 10 + for i in range(max_iterations): + # Call LLM + response = await call_llm(messages, tools_schema) + assistant_msg = response["choices"][0]["message"] + messages.append(assistant_msg) + + # Check for tool calls + if not assistant_msg.get("tool_calls"): + print(f"Done! Final response: {assistant_msg['content']}") + break + + # Execute all tool calls + for tool_call in assistant_msg["tool_calls"]: + result = manager.execute_tool( + tool_name=tool_call["function"]["name"], + arguments=tool_call["function"]["arguments"], + work_dir=work_dir + ) + + # Add tool result to conversation + messages.append({ + "role": "tool", + "tool_call_id": tool_call["id"], + "content": result + }) +``` + +## 🔒 Security Features + +### Path Security + +All file operation tools include path traversal checks: + +```python +# ✅ Allowed +read_file("src/main.py") + +# ❌ Denied (path traversal) +read_file("../../../etc/passwd") +``` + +### Error Handling + +Tool execution automatically catches exceptions: + +```python +# Internal exceptions are caught and returned as error messages +result = manager.execute_tool("read_file", {"path": "nonexist.txt"}) +# Returns: {"error": "File not found: nonexist.txt"} +``` + +## 🧪 Testing + +Create test file: + +```python +# test_tools.py +from steps.tools import create_code_agent_manager +import tempfile +import os + +def test_file_tools(): + manager = create_code_agent_manager() + + with tempfile.TemporaryDirectory() as tmpdir: + # Test write_file + result = manager.execute_tool( + "write_file", + {"path": "test.txt", "content": "hello"}, + work_dir=tmpdir + ) + assert "success" in result + + # Test read_file + result = manager.execute_tool( + "read_file", + {"path": "test.txt"}, + work_dir=tmpdir + ) + assert result == "hello" + + print("✅ Test passed") + +if __name__ == "__main__": + test_file_tools() +``` + +## 📦 Extending Tools + +### Adding New Tool Categories + +Create new file `network_tools.py`: + +```python +from .base import Tool +import json +import httpx + +class HttpGetTool(Tool): + def get_name(self) -> str: + return "http_get" + + def get_description(self) -> str: + return "Make HTTP GET request" + + def get_parameters(self) -> dict: + return { + "type": "object", + "properties": { + "url": {"type": "string"} + }, + "required": ["url"] + } + + def execute(self, arguments, context): + url = arguments["url"] + response = httpx.get(url) + return json.dumps({ + "status": response.status_code, + "body": response.text[:1000] + }) +``` + +Then add to `presets.py`: + +```python +def create_network_tools(): + from .network_tools import HttpGetTool + return [HttpGetTool()] +``` + +## 🎯 Best Practices + +1. **Tool Naming**: Use clear verb+noun format (`read_file`, `list_users`) +2. **Parameter Descriptions**: Describe each parameter in detail to help AI use them correctly +3. **Error Handling**: Return JSON format error messages with `error` field +4. **Logging**: Use `context.log_info/log_error` to record key operations +5. **Security Checks**: Validate input parameters, prevent path traversal and other security issues +6. **Result Format**: Return JSON strings or plain text, maintain consistency + +## 🤝 Contributing + +Steps to add new tools: + +1. Inherit from `Tool` base class +2. Implement 4 abstract methods +3. Add to appropriate tool set in `presets.py` +4. Export in `__init__.py` +5. Update README documentation + +## 📄 License + +Same as main project diff --git a/bbdev/api/services/__init__.py b/bbdev/api/services/__init__.py new file mode 100644 index 000000000..6de3040ca --- /dev/null +++ b/bbdev/api/services/__init__.py @@ -0,0 +1,84 @@ +""" +Function Calling Tool Management Module + +This module provides a complete Function Calling tool management system, including: +- Tool base class definitions +- Tool registration and management +- Predefined tool sets +- Tool execution context + +Usage example: + +```python +from tools import create_code_agent_manager + +# Create tool manager +manager = create_code_agent_manager() + +# Get tool definitions (to send to LLM) +tools_schema = manager.get_tools_schema() + +# Execute tool call +result = manager.execute_tool( + tool_name="read_file", + arguments={"path": "main.py"}, + work_dir="/path/to/project", + logger=logger +) +``` +""" + +from .base import Tool, ToolContext +from .registry import ToolRegistry, ToolManager +from .file_tools import ( + ReadFileTool, + WriteFileTool, + ListFilesTool, + MakeDirTool, + DeleteFileTool, + GetPathTool, + GrepFilesTool, +) +from .workflow_tools import WorkflowAPITool +from .deepwiki_tools import DeepwikiAskTool, DeepwikiReadWikiTool +from .agent_tools import CallAgentTool +from .presets import ( + create_file_tools, + create_code_agent_tools, + create_default_manager, + create_code_agent_manager, + get_preset, + list_presets, +) + +__all__ = [ + # Base classes + "Tool", + "ToolContext", + # Registration and management + "ToolRegistry", + "ToolManager", + # File operation tools + "ReadFileTool", + "WriteFileTool", + "ListFilesTool", + "MakeDirTool", + "DeleteFileTool", + "GetPathTool", + "GrepFilesTool", + # Agent coordination + "CallAgentTool", + # API tools + "WorkflowAPITool", + "DeepwikiAskTool", + "DeepwikiReadWikiTool", + # Predefined tool sets + "create_file_tools", + "create_code_agent_tools", + "create_default_manager", + "create_code_agent_manager", + "get_preset", + "list_presets", +] + +__version__ = "1.0.0" diff --git a/bbdev/api/services/agent_tools.py b/bbdev/api/services/agent_tools.py new file mode 100644 index 000000000..623377aac --- /dev/null +++ b/bbdev/api/services/agent_tools.py @@ -0,0 +1,129 @@ +"""Agent coordination tools""" + +import httpx +import uuid +from typing import Dict, Any +from .base import Tool + + +class CallAgentTool(Tool): + """Tool for calling other agents""" + + def get_name(self) -> str: + return "call_agent" + + def get_description(self) -> str: + return """Call another agent (spec_agent, code_agent, review_agent, verify_agent) with a task. + Use this to delegate work to specialized agents. + Returns the agent's response and generated files.""" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "agent_role": { + "type": "string", + "enum": ["spec", "code", "review", "verify"], + "description": "Which agent to call (spec/code/review/verify)", + }, + "task_description": { + "type": "string", + "description": "Task description or instructions for the agent", + }, + "context_files": { + "type": "array", + "items": {"type": "string"}, + "description": "Optional: List of file paths the agent should read for context", + }, + "model": { + "type": "string", + "description": "Optional: LLM model to use (will inherit from parent if not specified)", + }, + }, + "required": ["agent_role", "task_description"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + agent_role = arguments.get("agent_role") + task_description = arguments.get("task_description") + context_files = arguments.get("context_files", []) + # Prefer model from parameters, otherwise inherit from context + model = arguments.get("model") or context.extra.get("model") + + # Generate temporary task file + import os + import tempfile + + # Create temporary task file + task_content = task_description + + # Add context file references if specified + if context_files: + task_content += "\n\n## Context Files\n" + for filepath in context_files: + task_content += f"- {filepath}\n" + + # Write to temporary file + temp_dir = os.path.join(context.work_dir, ".agent_tasks") + os.makedirs(temp_dir, exist_ok=True) + + task_id = str(uuid.uuid4())[:8] + task_file = os.path.join(temp_dir, f"task_{agent_role}_{task_id}.md") + + with open(task_file, "w", encoding="utf-8") as f: + f.write(task_content) + + context.log_info(f"Created task file: {task_file}") + context.log_info(f"Calling {agent_role}_agent with task") + + try: + # Get workflow API address (from environment or use defaults) + import os + + workflow_host = os.getenv("WORKFLOW_HOST", "localhost") + workflow_port = os.getenv("WORKFLOW_PORT", "3001") + base_url = f"http://{workflow_host}:{workflow_port}" + url = f"{base_url}/agent" + + context.log_info(f"Calling workflow API at: {url}") + + payload = { + "agentRole": agent_role, + "promptPath": task_file, + "workDir": context.work_dir, + } + + # Add model to payload if specified + if model: + payload["model"] = model + context.log_info(f"Using model: {model}") + + response = httpx.post(url, json=payload, timeout=600.0) + + if response.status_code == 200: + result = response.json() + + # Clean up temporary file + try: + os.remove(task_file) + except Exception: + pass + + return str( + { + "status": "success", + "agent": agent_role, + "result": result, + "files": result.get("files", []), + } + ) + else: + return str( + { + "error": f"Agent call failed with status {response.status_code}", + "response": response.text[:500], + } + ) + + except Exception as e: + return str({"error": f"Failed to call agent: {str(e)}"}) diff --git a/bbdev/api/services/base.py b/bbdev/api/services/base.py new file mode 100644 index 000000000..7b91d9ed4 --- /dev/null +++ b/bbdev/api/services/base.py @@ -0,0 +1,114 @@ +"""Function Calling tool base classes""" + +from abc import ABC, abstractmethod +from typing import Dict, Any, Optional +import json + + +class Tool(ABC): + """Tool base class""" + + def __init__(self): + self.name = self.get_name() + self.description = self.get_description() + self.parameters = self.get_parameters() + + @abstractmethod + def get_name(self) -> str: + """Return tool name""" + pass + + @abstractmethod + def get_description(self) -> str: + """Return tool description""" + pass + + @abstractmethod + def get_parameters(self) -> Dict[str, Any]: + """Return tool parameter definition (JSON Schema)""" + pass + + @abstractmethod + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + """ + Execute tool + + Args: + arguments: Tool parameters + context: Execution context (contains logger, work_dir, etc.) + + Returns: + Execution result (string format) + """ + pass + + def to_openai_format(self) -> Dict[str, Any]: + """Convert to OpenAI Function Calling format""" + return { + "type": "function", + "function": { + "name": self.name, + "description": self.description, + "parameters": self.parameters, + }, + } + + def safe_execute(self, arguments: Any, context: Any) -> str: + """ + Safely execute tool (with error handling) + + Args: + arguments: Tool parameters (can be string or dict) + context: Execution context + + Returns: + Execution result or error message + """ + try: + # Parse arguments + if isinstance(arguments, str): + args = json.loads(arguments) + else: + args = arguments + + # Execute tool + result = self.execute(args, context) + return result + + except json.JSONDecodeError as e: + error = f"Invalid JSON arguments: {str(e)}" + if hasattr(context, "logger"): + context.logger.error(f"Tool {self.name} - {error}") + return json.dumps({"error": error}) + + except Exception as e: + error = f"Tool execution failed: {str(e)}" + if hasattr(context, "logger"): + context.logger.error(f"Tool {self.name} - {error}") + return json.dumps({"error": error}) + + def __repr__(self) -> str: + return f"Tool({self.name})" + + +class ToolContext: + """Tool execution context""" + + def __init__(self, work_dir: str, logger: Any = None, **kwargs): + self.work_dir = work_dir + self.logger = logger + self.extra = kwargs + + def log_info(self, message: str): + """Log info message""" + if self.logger: + self.logger.info(message) + else: + print(f"[INFO] {message}") + + def log_error(self, message: str): + """Log error message""" + if self.logger: + self.logger.error(message) + else: + print(f"[ERROR] {message}") diff --git a/bbdev/api/services/deepwiki_tools.py b/bbdev/api/services/deepwiki_tools.py new file mode 100644 index 000000000..214cdbdaa --- /dev/null +++ b/bbdev/api/services/deepwiki_tools.py @@ -0,0 +1,327 @@ +"""Deepwiki tool wrapper - via MCP Streamable HTTP""" + +import httpx +import json +import uuid +from httpx_sse import connect_sse +from typing import Dict, Any +from .base import Tool + + +class DeepwikiAskTool(Tool): + """Deepwiki Q&A tool - via MCP""" + + def get_name(self) -> str: + return "deepwiki_ask" + + def get_description(self) -> str: + return """Ask questions about a GitHub repository using Deepwiki. + Use this to understand code, architecture, and implementation details.""" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "repo": { + "type": "string", + "description": ( + "GitHub repository in format 'owner/repo' " + "(e.g., 'DangoSys/buckyball', 'ucb-bar/gemmini')" + ), + }, + "question": { + "type": "string", + "description": "Question to ask about the repository", + }, + }, + "required": ["repo", "question"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + repo = arguments.get("repo") + question = arguments.get("question") + + try: + context.log_info( + f"Asking Deepwiki via MCP HTTP: {question[:100]}... (repo: {repo})" + ) + + # MCP tool call request (JSON-RPC 2.0) + request_payload = { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "ask_question", + "arguments": {"repoName": repo, "question": question}, + }, + } + + # Use Streamable HTTP endpoint + mcp_url = "https://mcp.deepwiki.com/mcp" + + context.log_info(f"MCP URL: {mcp_url}") + + with httpx.Client(timeout=120.0) as client: + # Step 1: Initialize session (without sessionId) + init_payload = { + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": { + "name": "buckyball-workflow", + "version": "1.0.0", + }, + }, + } + + context.log_info("Initializing MCP session (without sessionId)...") + + session_id = None + # Use POST request directly, get sessionId from response headers + init_resp = client.post( + mcp_url, + json=init_payload, + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + }, + ) + + # Get sessionId from response headers + session_id = init_resp.headers.get("mcp-session-id") + if not session_id: + context.log_error(f"Init response status: {init_resp.status_code}") + context.log_error( + f"Init response headers: {dict(init_resp.headers)}" + ) + return "Error: No session ID in init response headers" + + context.log_info(f"Session initialized, ID: {session_id}") + + # Step 2: Call tool (with sessionId) - use SSE streaming + context.log_info( + f"Calling tool: {json.dumps(request_payload, ensure_ascii=False)[:300]}" + ) + + # Use stream instead of connect_sse + with client.stream( + "POST", + mcp_url, + json=request_payload, + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + "Mcp-Session-Id": session_id, + }, + ) as tool_resp: + context.log_info(f"Tool response status: {tool_resp.status_code}") + + # Manually parse SSE stream + for line in tool_resp.iter_lines(): + if line.startswith("data: "): + # Remove "data: " prefix + data_str = line[6:] + + # Skip heartbeat + if data_str.strip() == "ping": + continue + + try: + result = json.loads(data_str) + context.log_info( + f"Tool response: {json.dumps(result, ensure_ascii=False)[:500]}" + ) + + # Handle JSON-RPC 2.0 response + if "error" in result: + error = result["error"] + error_msg = f"MCP Error: {error.get('message', 'Unknown error')}" + context.log_error(error_msg) + return error_msg + + if "result" in result: + content = result["result"].get("content", []) + if content and len(content) > 0: + answer = content[0].get("text", "") + context.log_info( + f"Deepwiki answer length: {len(answer)} chars" + ) + context.log_info( + f"Deepwiki answer preview: {answer[:200]}..." + ) + + # Limit return length + if len(answer) > 3000: + answer = answer[:3000] + "\n... (truncated)" + return answer + except json.JSONDecodeError as e: + context.log_error( + f"Failed to parse SSE data: {e}, line: {line[:100]}" + ) + continue + + return "No valid response from Deepwiki" + + except Exception as e: + error_msg = f"Error calling Deepwiki MCP: {str(e)}" + context.log_error(error_msg) + return error_msg + + +class DeepwikiReadWikiTool(Tool): + """Read Deepwiki wiki content - via MCP""" + + def get_name(self) -> str: + return "deepwiki_read_wiki" + + def get_description(self) -> str: + return """Read wiki documentation for a GitHub repository from Deepwiki. + Use this to get structured documentation about the repository.""" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "repo": { + "type": "string", + "description": "GitHub repository in format 'owner/repo'", + } + }, + "required": ["repo"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + repo = arguments.get("repo") + + try: + context.log_info(f"Reading Deepwiki wiki via MCP HTTP for: {repo}") + + # MCP tool call request + request_payload = { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "read_wiki_contents", + "arguments": {"repoName": repo}, + }, + } + + # Use Streamable HTTP endpoint + mcp_url = "https://mcp.deepwiki.com/mcp" + + context.log_info(f"MCP URL: {mcp_url}") + + with httpx.Client(timeout=120.0) as client: + # Step 1: Initialize session (without sessionId) + init_payload = { + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": { + "name": "buckyball-workflow", + "version": "1.0.0", + }, + }, + } + + context.log_info("Initializing MCP session (without sessionId)...") + + session_id = None + # Use POST request directly, get sessionId from response headers + init_resp = client.post( + mcp_url, + json=init_payload, + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + }, + ) + + # Get sessionId from response headers + session_id = init_resp.headers.get("mcp-session-id") + if not session_id: + context.log_error(f"Init response status: {init_resp.status_code}") + context.log_error( + f"Init response headers: {dict(init_resp.headers)}" + ) + return "Error: No session ID in init response headers" + + context.log_info(f"Session initialized, ID: {session_id}") + + # Step 2: Call tool (with sessionId) - use SSE streaming + context.log_info( + f"Calling tool: {json.dumps(request_payload, ensure_ascii=False)}" + ) + + # Use stream instead of connect_sse + with client.stream( + "POST", + mcp_url, + json=request_payload, + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + "Mcp-Session-Id": session_id, + }, + ) as tool_resp: + context.log_info(f"Tool response status: {tool_resp.status_code}") + + # Manually parse SSE stream + for line in tool_resp.iter_lines(): + if line.startswith("data: "): + # Remove "data: " prefix + data_str = line[6:] + + # Skip heartbeat + if data_str.strip() == "ping": + continue + + try: + result = json.loads(data_str) + context.log_info( + f"Tool response: {json.dumps(result, ensure_ascii=False)[:500]}" + ) + + # Handle JSON-RPC 2.0 response + if "error" in result: + error = result["error"] + error_msg = f"MCP Error: {error.get('message', 'Unknown error')}" + context.log_error(error_msg) + return error_msg + + if "result" in result: + content = result["result"].get("content", []) + if content and len(content) > 0: + wiki_text = content[0].get("text", "") + context.log_info( + f"Deepwiki wiki length: {len(wiki_text)} chars" + ) + context.log_info( + f"Deepwiki wiki preview: {wiki_text[:200]}..." + ) + + if len(wiki_text) > 5000: + wiki_text = ( + wiki_text[:5000] + "\n... (truncated)" + ) + return wiki_text + except json.JSONDecodeError as e: + context.log_error( + f"Failed to parse SSE data: {e}, line: {line[:100]}" + ) + continue + + return "No valid wiki content from Deepwiki" + + except Exception as e: + error_msg = f"Error reading Deepwiki wiki via MCP: {str(e)}" + context.log_error(error_msg) + return error_msg diff --git a/bbdev/api/services/file_tools.py b/bbdev/api/services/file_tools.py new file mode 100644 index 000000000..cec4ca8c1 --- /dev/null +++ b/bbdev/api/services/file_tools.py @@ -0,0 +1,359 @@ +"""File operation related tools""" + +import os +import json +import shutil +from typing import Dict, Any +from .base import Tool + + +class MakeDirTool(Tool): + """Create directory tool""" + + def get_name(self) -> str: + return "make_dir" + + def get_description(self) -> str: + return "Create a new directory (supports creating parent directories)" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "path": {"type": "string", "description": "Directory path to create"} + }, + "required": ["path"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + path = arguments.get("path") + + if not os.path.isabs(path): + path = os.path.join(context.work_dir, path) + + try: + if os.path.exists(path): + return json.dumps({"status": "exists", "path": path}) + + os.makedirs(path, exist_ok=True) + context.log_info(f"Created directory: {path}") + return json.dumps({"status": "success", "path": path}) + + except Exception as e: + return json.dumps({"error": f"Failed to create directory: {str(e)}"}) + + +class GetPathTool(Tool): + """Get path information tool""" + + def get_name(self) -> str: + return "get_path_info" + + def get_description(self) -> str: + return "Get absolute path and check if path exists" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Path to check (optional, defaults to work_dir)", + } + }, + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + path = arguments.get("path", ".") + + if not os.path.isabs(path): + path = os.path.join(context.work_dir, path) + + abs_path = os.path.abspath(path) + exists = os.path.exists(abs_path) + + info = { + "absolute_path": abs_path, + "exists": exists, + "work_dir": context.work_dir, + } + + if exists: + info["is_file"] = os.path.isfile(abs_path) + info["is_dir"] = os.path.isdir(abs_path) + + return json.dumps(info, indent=2) + + +class GrepFilesTool(Tool): + """Search file content tool""" + + def get_name(self) -> str: + return "grep_files" + + def get_description(self) -> str: + return "Search for text pattern in files" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "pattern": {"type": "string", "description": "Text pattern to search"}, + "path": {"type": "string", "description": "Directory or file path"}, + "file_ext": { + "type": "string", + "description": "File extension filter (e.g., '.scala')", + }, + }, + "required": ["pattern", "path"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + pattern = arguments.get("pattern") + path = arguments.get("path") + file_ext = arguments.get("file_ext") + + if not os.path.isabs(path): + path = os.path.join(context.work_dir, path) + + try: + results = [] + files_to_search = [] + + if os.path.isfile(path): + files_to_search = [path] + else: + for root, _, files in os.walk(path): + for file in files: + if file_ext and not file.endswith(file_ext): + continue + files_to_search.append(os.path.join(root, file)) + + # Limit file count + for filepath in files_to_search[:100]: + try: + with open(filepath, "r", encoding="utf-8", errors="ignore") as f: + for line_num, line in enumerate(f, 1): + if pattern in line: + results.append( + { + "file": filepath, + "line": line_num, + "content": line.strip()[:150], + } + ) + # Limit result count + if len(results) >= 50: + break + except Exception: + continue + + if len(results) >= 50: + break + + return json.dumps({"matches": len(results), "results": results}, indent=2) + + except Exception as e: + return json.dumps({"error": f"Search failed: {str(e)}"}) + + +class DeleteFileTool(Tool): + """Delete file tool""" + + def get_name(self) -> str: + return "delete_file" + + def get_description(self) -> str: + return "Delete a file (use with caution)" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "path": {"type": "string", "description": "Path to file to delete"} + }, + "required": ["path"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + path = arguments.get("path") + + if not os.path.isabs(path): + path = os.path.join(context.work_dir, path) + + try: + if not os.path.exists(path): + return json.dumps({"status": "not_found", "path": path}) + + if os.path.isfile(path): + os.remove(path) + context.log_info(f"Deleted file: {path}") + return json.dumps({"status": "success", "path": path}) + else: + return json.dumps({"error": "Path is a directory, not a file"}) + + except Exception as e: + return json.dumps({"error": f"Failed to delete: {str(e)}"}) + + +class ReadFileTool(Tool): + """Read file content""" + + def get_name(self) -> str: + return "read_file" + + def get_description(self) -> str: + return "Read the content of a file" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path relative to work directory", + } + }, + "required": ["path"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + file_path = arguments.get("path") + + if not file_path: + return json.dumps({"error": "Missing required parameter: path"}) + + full_path = os.path.join(context.work_dir, file_path) + + # Security check: prevent path traversal + abs_full = os.path.abspath(full_path) + abs_work = os.path.abspath(context.work_dir) + if not abs_full.startswith(abs_work): + return json.dumps({"error": "Access denied: path outside work directory"}) + + if not os.path.exists(full_path): + return json.dumps({"error": f"File not found: {file_path}"}) + + if not os.path.isfile(full_path): + return json.dumps({"error": f"Not a file: {file_path}"}) + + try: + with open(full_path, "r", encoding="utf-8") as f: + content = f.read() + + context.log_info(f"Tool: read_file({file_path}) - {len(content)} chars") + return content + + except UnicodeDecodeError: + return json.dumps( + {"error": "Cannot read file: not a text file or encoding issue"} + ) + + +class WriteFileTool(Tool): + """Write file content""" + + def get_name(self) -> str: + return "write_file" + + def get_description(self) -> str: + return "Write content to a file (creates directories if needed)" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path relative to work directory", + }, + "content": { + "type": "string", + "description": "Content to write to the file", + }, + }, + "required": ["path", "content"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + file_path = arguments.get("path") + content = arguments.get("content") + + if not file_path: + return json.dumps({"error": "Missing required parameter: path"}) + + if content is None: + return json.dumps({"error": "Missing required parameter: content"}) + + full_path = os.path.join(context.work_dir, file_path) + + # Security check: prevent path traversal + abs_full = os.path.abspath(full_path) + abs_work = os.path.abspath(context.work_dir) + if not abs_full.startswith(abs_work): + return json.dumps({"error": "Access denied: path outside work directory"}) + + try: + # Create directory + dir_path = os.path.dirname(full_path) + if dir_path: + os.makedirs(dir_path, exist_ok=True) + + # Write file + with open(full_path, "w", encoding="utf-8") as f: + f.write(content) + + context.log_info(f"Tool: write_file({file_path}) - {len(content)} chars") + + return json.dumps( + {"success": True, "path": file_path, "size": len(content)} + ) + + except Exception as e: + return json.dumps({"error": f"Failed to write file: {str(e)}"}) + + +class ListFilesTool(Tool): + """List files in directory""" + + def get_name(self) -> str: + return "list_files" + + def get_description(self) -> str: + return "List files in a directory" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Directory path relative to work directory (default: .)", + } + }, + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + dir_path = arguments.get("path", ".") + full_path = os.path.join(context.work_dir, dir_path) + + # Security check + abs_full = os.path.abspath(full_path) + abs_work = os.path.abspath(context.work_dir) + if not abs_full.startswith(abs_work): + return json.dumps({"error": "Access denied: path outside work directory"}) + + if not os.path.exists(full_path): + return json.dumps({"error": f"Directory not found: {dir_path}"}) + + if not os.path.isdir(full_path): + return json.dumps({"error": f"Not a directory: {dir_path}"}) + + try: + files = os.listdir(full_path) + context.log_info(f"Tool: list_files({dir_path}) - {len(files)} items") + + return json.dumps({"path": dir_path, "files": files, "count": len(files)}) + + except Exception as e: + return json.dumps({"error": f"Failed to list directory: {str(e)}"}) diff --git a/bbdev/api/services/presets.py b/bbdev/api/services/presets.py new file mode 100644 index 000000000..8db2aff02 --- /dev/null +++ b/bbdev/api/services/presets.py @@ -0,0 +1,106 @@ +"""Predefined tool sets""" + +from typing import List +from .base import Tool +from .file_tools import ( + ReadFileTool, + WriteFileTool, + ListFilesTool, + MakeDirTool, + DeleteFileTool, + GetPathTool, + GrepFilesTool, +) +from .workflow_tools import WorkflowAPITool +from .deepwiki_tools import DeepwikiAskTool, DeepwikiReadWikiTool +from .agent_tools import CallAgentTool +from .registry import ToolManager + + +def create_file_tools() -> List[Tool]: + """Create file operation tool set""" + return [ + ReadFileTool(), + WriteFileTool(), + ListFilesTool(), + MakeDirTool(), + DeleteFileTool(), + GetPathTool(), + GrepFilesTool(), + ] + + +def create_code_agent_tools() -> List[Tool]: + """Create Code Agent tool set (includes all required tools)""" + return [ + # File operations + ReadFileTool(), + WriteFileTool(), + ListFilesTool(), + MakeDirTool(), + DeleteFileTool(), + GetPathTool(), + GrepFilesTool(), + # Agent coordination + CallAgentTool(), + # Workflow API + WorkflowAPITool(), + # Deepwiki + DeepwikiAskTool(), + DeepwikiReadWikiTool(), + ] + + +def create_default_manager() -> ToolManager: + """Create default tool manager""" + manager = ToolManager() + manager.register_tools(create_file_tools()) + return manager + + +def create_code_agent_manager() -> ToolManager: + """Create Code Agent dedicated tool manager""" + manager = ToolManager() + manager.register_tools(create_code_agent_tools()) + return manager + + +# Predefined tool set configurations +PRESET_CONFIGS = { + "file_tools": { + "name": "File Operations", + "description": "Basic file system operations", + "tools": create_file_tools, + }, + "code_agent": { + "name": "Code Agent", + "description": "Tools for code generation and manipulation", + "tools": create_code_agent_tools, + }, +} + + +def get_preset(name: str) -> List[Tool]: + """ + Get predefined tool set + + Args: + name: Tool set name ("file_tools", "code_agent") + + Returns: + Tool list + + Raises: + ValueError: If tool set does not exist + """ + config = PRESET_CONFIGS.get(name) + if not config: + available = ", ".join(PRESET_CONFIGS.keys()) + raise ValueError(f"Unknown preset: {name}. Available: {available}") + + return config["tools"]() + + +def list_presets() -> List[str]: + """List all available predefined tool sets""" + return list(PRESET_CONFIGS.keys()) diff --git a/bbdev/api/services/registry.py b/bbdev/api/services/registry.py new file mode 100644 index 000000000..139224669 --- /dev/null +++ b/bbdev/api/services/registry.py @@ -0,0 +1,131 @@ +"""Function Calling tool registry""" + +from typing import Dict, List, Any, Optional +from .base import Tool, ToolContext + + +class ToolRegistry: + """Tool registration and management""" + + def __init__(self): + self._tools: Dict[str, Tool] = {} + + def register(self, tool: Tool): + """Register a tool""" + self._tools[tool.name] = tool + + def register_all(self, tools: List[Tool]): + """Batch register tools""" + for tool in tools: + self.register(tool) + + def get(self, name: str) -> Optional[Tool]: + """Get tool""" + return self._tools.get(name) + + def list_tools(self) -> List[str]: + """List all tool names""" + return list(self._tools.keys()) + + def to_openai_format(self) -> List[Dict[str, Any]]: + """Convert to OpenAI Function Calling format""" + return [tool.to_openai_format() for tool in self._tools.values()] + + def execute(self, tool_name: str, arguments: Any, context: ToolContext) -> str: + """ + Execute tool + + Args: + tool_name: Tool name + arguments: Tool parameters + context: Execution context + + Returns: + Execution result + """ + tool = self.get(tool_name) + + if not tool: + return f'{{"error": "Unknown tool: {tool_name}"}}' + + return tool.safe_execute(arguments, context) + + def __len__(self) -> int: + return len(self._tools) + + def __repr__(self) -> str: + return f"ToolRegistry({len(self)} tools: {', '.join(self.list_tools())})" + + +class ToolManager: + """Tool manager (high-level wrapper)""" + + def __init__(self, registry: Optional[ToolRegistry] = None): + self.registry = registry or ToolRegistry() + self._execution_log: List[Dict[str, Any]] = [] + + def register_tool(self, tool: Tool): + """Register tool""" + self.registry.register(tool) + + def register_tools(self, tools: List[Tool]): + """Batch register tools""" + self.registry.register_all(tools) + + def get_tools_schema(self) -> List[Dict[str, Any]]: + """Get tool definitions (OpenAI format)""" + return self.registry.to_openai_format() + + def execute_tool( + self, + tool_name: str, + arguments: Any, + work_dir: str, + logger: Any = None, + **kwargs, + ) -> str: + """ + Execute tool call + + Args: + tool_name: Tool name + arguments: Tool parameters + work_dir: Working directory + logger: Logger + **kwargs: Other context parameters + + Returns: + Execution result + """ + # Create context + context = ToolContext(work_dir=work_dir, logger=logger, **kwargs) + + # Execute tool + result = self.registry.execute(tool_name, arguments, context) + + # Log execution + self._execution_log.append( + { + "tool": tool_name, + "arguments": arguments, + # Truncate long results + "result": result[:200] if len(result) > 200 else result, + } + ) + + return result + + def get_execution_log(self) -> List[Dict[str, Any]]: + """Get execution log""" + return self._execution_log + + def clear_log(self): + """Clear execution log""" + self._execution_log.clear() + + def get_tool_names(self) -> List[str]: + """Get all tool names""" + return self.registry.list_tools() + + def __repr__(self) -> str: + return f"ToolManager({len(self.registry)} tools registered)" diff --git a/bbdev/api/services/workflow_tools.py b/bbdev/api/services/workflow_tools.py new file mode 100644 index 000000000..ab17e8916 --- /dev/null +++ b/bbdev/api/services/workflow_tools.py @@ -0,0 +1,71 @@ +"""Workflow internal API call tools""" + +import httpx +import asyncio +from typing import Dict, Any +from .base import Tool + + +class WorkflowAPITool(Tool): + """Generic tool for calling Workflow internal APIs""" + + def get_name(self) -> str: + return "call_workflow_api" + + def get_description(self) -> str: + return """Call internal workflow API endpoints. + Available endpoints: + - /verilator/verilog: Generate Verilog + - /verilator/build: Build verilator (params: jobs) + - /verilator/sim: Run simulation (params: binary, batch) + - /workload/build: Build workload (params: args) + - /sardine/run: Run sardine tests (params: workload)""" + + def get_parameters(self) -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "endpoint": { + "type": "string", + "description": "API endpoint path (e.g., '/verilator/build')", + }, + "params": { + "type": "object", + "description": "Request parameters as JSON object", + "additionalProperties": True, + }, + }, + "required": ["endpoint"], + } + + def execute(self, arguments: Dict[str, Any], context: Any) -> str: + endpoint = arguments.get("endpoint") + params = arguments.get("params", {}) + + # Get workflow API address + import os + + workflow_host = os.getenv("WORKFLOW_HOST", "localhost") + workflow_port = os.getenv("WORKFLOW_PORT", "3001") + base_url = f"http://{workflow_host}:{workflow_port}" + url = f"{base_url}{endpoint}" + + try: + context.log_info(f"Calling workflow API: {url}") + context.log_info(f"Parameters: {params}") + + # Synchronous call (using httpx sync client) + response = httpx.post(url, json=params, timeout=300.0) + + if response.status_code == 200: + return str(response.json()) + else: + return str( + { + "error": f"API call failed with status {response.status_code}", + "response": response.text[:500], + } + ) + + except Exception as e: + return str({"error": f"Workflow API call failed: {str(e)}"}) diff --git a/bbdev/api/steps/compiler/01_build_api_step.py b/bbdev/api/steps/compiler/01_build_api_step.py new file mode 100644 index 000000000..e24205206 --- /dev/null +++ b/bbdev/api/steps/compiler/01_build_api_step.py @@ -0,0 +1,26 @@ +import asyncio +from utils.event_common import wait_for_result + +config = { + "type": "api", + "name": "Build Compiler", + "description": "build bitstream", + "path": "/compiler/build", + "method": "POST", + "emits": ["compiler.build"], + "flows": ["compiler"], +} + + +async def handler(req, context): + body = req.get("body") or {} + await context.emit({"topic": "compiler.build", "data": body}) + + # ================================================================================== + # Wait for build result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/compiler/01_build_event_step.py b/bbdev/api/steps/compiler/01_build_event_step.py new file mode 100644 index 000000000..669830274 --- /dev/null +++ b/bbdev/api/steps/compiler/01_build_event_step.py @@ -0,0 +1,56 @@ +import os +import subprocess +import sys + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result + +config = { + "type": "event", + "name": "Build Compiler", + "description": "build bitstream", + "subscribes": ["compiler.build"], + "emits": [], + "flows": ["compiler"], +} + + +async def handler(data, context): + bbdir = get_buckyball_path() + script_dir = f"{bbdir}/workflow/steps/compiler/scripts" + yaml_dir = f"{script_dir}/yaml" + # ================================================================================== + # Execute operation + # ================================================================================== + command = f"mkdir -p {bbdir}/compiler/build" + result = stream_run_logger( + cmd=command, + logger=context.logger, + stdout_prefix="compiler build", + stderr_prefix="compiler build", + ) + command = f"cd {bbdir}/compiler/build && ninja -j{os.cpu_count()}" + result = stream_run_logger( + cmd=command, + logger=context.logger, + stdout_prefix="compiler build", + stderr_prefix="compiler build", + ) + + # ================================================================================== + # Return result to API + # ================================================================================== + success_result, failure_result = await check_result( + context, result.returncode, continue_run=False + ) + + # ================================================================================== + # Continue routing + # ================================================================================== + return diff --git a/bbdev/api/steps/compiler/README.md b/bbdev/api/steps/compiler/README.md new file mode 100644 index 000000000..993560bc7 --- /dev/null +++ b/bbdev/api/steps/compiler/README.md @@ -0,0 +1,33 @@ +# Compiler Workflow + +Compiler build workflow in the Buckyball framework for building the Buckyball compiler toolchain. + +## API Usage + +### `build` +**Endpoint**: `POST /compiler/build` + +**Function**: Build Buckyball compiler + +**Parameters**: No specific parameters + +**Example**: +```bash +bbdev compiler --build +``` + +**Response**: +```json +{ + "status": 200, + "body": { + "success": true, + "processing": false, + "return_code": 0 + } +} +``` + +## Notes + +- Ensure the system has necessary build tools and dependencies diff --git a/bbdev/api/steps/firesim/.gitignore b/bbdev/api/steps/firesim/.gitignore new file mode 100644 index 000000000..0a8db21da --- /dev/null +++ b/bbdev/api/steps/firesim/.gitignore @@ -0,0 +1 @@ +# *.yaml diff --git a/bbdev/api/steps/firesim/01_buildbitstream_api_step.py b/bbdev/api/steps/firesim/01_buildbitstream_api_step.py new file mode 100644 index 000000000..2c5fb9e95 --- /dev/null +++ b/bbdev/api/steps/firesim/01_buildbitstream_api_step.py @@ -0,0 +1,26 @@ +import asyncio +from utils.event_common import wait_for_result + +config = { + "type": "api", + "name": "Firesim Buildbitstream", + "description": "build bitstream", + "path": "/firesim/buildbitstream", + "method": "POST", + "emits": ["firesim.buildbitstream"], + "flows": ["firesim"], +} + + +async def handler(req, context): + body = req.get("body") or {} + await context.emit({"topic": "firesim.buildbitstream", "data": body}) + + # ================================================================================== + # Wait for simulation result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/firesim/01_buildbitstream_event_step.py b/bbdev/api/steps/firesim/01_buildbitstream_event_step.py new file mode 100644 index 000000000..ba943c6ea --- /dev/null +++ b/bbdev/api/steps/firesim/01_buildbitstream_event_step.py @@ -0,0 +1,54 @@ +import os +import subprocess +import sys + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result + +config = { + "type": "event", + "name": "Firesim Buildbitstream", + "description": "build bitstream", + "subscribes": ["firesim.buildbitstream"], + "emits": [], + "flows": ["firesim"], +} + + +async def handler(data, context): + bbdir = get_buckyball_path() + script_dir = f"{bbdir}/workflow/steps/firesim/scripts" + yaml_dir = f"{script_dir}/yaml" + # ================================================================================== + # Execute operation + # ================================================================================== + command = f"firesim buildbitstream " + command += f" -a {yaml_dir}/config_hwdb.yaml" + command += f" -b {yaml_dir}/config_build.yaml" + command += f" -r {yaml_dir}/config_build_recipes.yaml" + command += f" -c {yaml_dir}/config_runtime.yaml" + result = stream_run_logger( + cmd=command, + logger=context.logger, + stdout_prefix="firesim buildbitstream", + stderr_prefix="firesim buildbitstream", + ) + + # ================================================================================== + # Return result to API + # ================================================================================== + success_result, failure_result = await check_result( + context, result.returncode, continue_run=False + ) + + # ================================================================================== + # Continue routing + # ================================================================================== + + return diff --git a/bbdev/api/steps/firesim/02_infrasetup_api_step.py b/bbdev/api/steps/firesim/02_infrasetup_api_step.py new file mode 100644 index 000000000..32d3eba9a --- /dev/null +++ b/bbdev/api/steps/firesim/02_infrasetup_api_step.py @@ -0,0 +1,27 @@ +import asyncio +from utils.event_common import wait_for_result + +config = { + "type": "api", + "name": "Firesim Infrasetup", + "description": "infrasetup", + "path": "/firesim/infrasetup", + "method": "POST", + "emits": ["firesim.infrasetup"], + "flows": ["firesim"], +} + + +async def handler(req, context): + body = req.get("body") or {} + data = {"jobs": body.get("jobs", 16)} + await context.emit({"topic": "firesim.infrasetup", "data": data}) + + # ================================================================================== + # Wait for simulation result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/firesim/02_infrasetup_event_step.py b/bbdev/api/steps/firesim/02_infrasetup_event_step.py new file mode 100644 index 000000000..fb90968c0 --- /dev/null +++ b/bbdev/api/steps/firesim/02_infrasetup_event_step.py @@ -0,0 +1,56 @@ +import os +import subprocess +import sys + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result + +config = { + "type": "event", + "name": "Firesim Infrasetup", + "description": "infrasetup", + "subscribes": ["firesim.infrasetup"], + "emits": [], + "flows": ["firesim"], +} + + +async def handler(data, context): + bbdir = get_buckyball_path() + script_dir = f"{bbdir}/workflow/steps/firesim/scripts" + yaml_dir = f"{script_dir}/yaml" + # ================================================================================== + # Execute operation + # ================================================================================== + command = f"firesim infrasetup " + command += f" -a {yaml_dir}/config_hwdb.yaml" + command += f" -b {yaml_dir}/config_build.yaml" + command += f" -r {yaml_dir}/config_build_recipes.yaml" + command += f" -c {yaml_dir}/config_runtime.yaml" + result = stream_run_logger( + cmd=command, + logger=context.logger, + stdout_prefix="firesim infrasetup", + stderr_prefix="firesim infrasetup", + ) + + # ================================================================================== + # Return result to API + # ================================================================================== + success_result, failure_result = await check_result( + context, result.returncode, continue_run=False + ) + + # ================================================================================== + # Continue routing + # Routing to verilog or finish workflow + # For run workflow, continue to verilog; for standalone clean, complete + # ================================================================================== + + return diff --git a/bbdev/api/steps/firesim/03_runworkload_api_step.py b/bbdev/api/steps/firesim/03_runworkload_api_step.py new file mode 100644 index 000000000..be0705120 --- /dev/null +++ b/bbdev/api/steps/firesim/03_runworkload_api_step.py @@ -0,0 +1,27 @@ +import asyncio +from utils.event_common import wait_for_result + +config = { + "type": "api", + "name": "Firesim Runworkload", + "description": "run workload", + "path": "/firesim/runworkload", + "method": "POST", + "emits": ["firesim.runworkload"], + "flows": ["firesim"], +} + + +async def handler(req, context): + body = req.get("body") or {} + data = {"jobs": body.get("jobs", 16)} + await context.emit({"topic": "firesim.runworkload", "data": data}) + + # ================================================================================== + # Wait for simulation result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/firesim/03_runworkload_event_step.py b/bbdev/api/steps/firesim/03_runworkload_event_step.py new file mode 100644 index 000000000..d79639a48 --- /dev/null +++ b/bbdev/api/steps/firesim/03_runworkload_event_step.py @@ -0,0 +1,56 @@ +import os +import subprocess +import sys + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result + +config = { + "type": "event", + "name": "Firesim Runworkload", + "description": "run workload", + "subscribes": ["firesim.runworkload"], + "emits": [], + "flows": ["firesim"], +} + + +async def handler(data, context): + bbdir = get_buckyball_path() + script_dir = f"{bbdir}/workflow/steps/firesim/scripts" + yaml_dir = f"{script_dir}/yaml" + # ================================================================================== + # Execute operation + # ================================================================================== + command = f"firesim runworkload " + command += f" -a {yaml_dir}/config_hwdb.yaml" + command += f" -b {yaml_dir}/config_build.yaml" + command += f" -r {yaml_dir}/config_build_recipes.yaml" + command += f" -c {yaml_dir}/config_runtime.yaml" + result = stream_run_logger( + cmd=command, + logger=context.logger, + stdout_prefix="firesim runworkload", + stderr_prefix="firesim runworkload", + ) + + # ================================================================================== + # Return result to API + # ================================================================================== + success_result, failure_result = await check_result( + context, result.returncode, continue_run=False + ) + + # ================================================================================== + # Continue routing + # Routing to verilog or finish workflow + # For run workflow, continue to verilog; for standalone clean, complete + # ================================================================================== + + return diff --git a/bbdev/api/steps/firesim/README.md b/bbdev/api/steps/firesim/README.md new file mode 100644 index 000000000..e3cc4737e --- /dev/null +++ b/bbdev/api/steps/firesim/README.md @@ -0,0 +1,59 @@ +# FireSim Workflow + +FireSim FPGA simulation workflow in Buckyball framework, providing FPGA-based hardware simulation environment. + +## API Usage + +### `buildbitstream` +**Endpoint**: `POST /firesim/buildbitstream` + +**Function**: Build FPGA bitstream file + +**Parameters**: No specific parameters + +**Example**: +```bash +bbdev firesim --buildbitstream +``` + +### `infrasetup` +**Endpoint**: `POST /firesim/infrasetup` + +**Function**: Setup FireSim infrastructure + +**Parameters**: No specific parameters + +**Example**: +```bash +bbdev firesim --infrasetup +``` + +### `runworkload` +**Endpoint**: `POST /firesim/runworkload` + +**Function**: Run workload on FireSim + +**Parameters**: No specific parameters + +**Example**: +```bash +bbdev firesim --runworkload +``` + +## Typical Workflow + +```bash +# 1. Build bitstream +bbdev firesim --buildbitstream + +# 2. Setup infrastructure +bbdev firesim --infrasetup + +# 3. Run workload +bbdev firesim --runworkload +``` + +## Notes + +- Bitstream build takes several hours +- infrasetup requires cloud computing resource configuration diff --git a/bbdev/api/steps/firesim/scripts/makefrag/firesim/build.mk b/bbdev/api/steps/firesim/scripts/makefrag/firesim/build.mk new file mode 100644 index 000000000..1f3eeae5a --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/makefrag/firesim/build.mk @@ -0,0 +1,56 @@ +# See LICENSE for license details. + +CHIPYARD_STAGING_DIR := $(chipyard_dir)/sims/firesim-staging + +# target scala directories to copy into midas. used by TARGET_COPY_TO_MIDAS_SCALA_DIRS +TARGET_COPY_TO_MIDAS_SCALA_DIRS := \ + $(addprefix $(chipyard_dir)/generators/firechip/,bridgeinterfaces goldengateimplementations) + +# this rule always is run, but may not update the timestamp of the targets (depending on what the Chipyard make does). +# if that is the case (Chipyard make doesn't update it's outputs), then downstream rules *should* be skipped. +# all other chipyard collateral is located in chipyard's generated sources area. +$(FIRRTL_FILE) $(ANNO_FILE) &: SHELL := /usr/bin/env bash # needed for running source in recipe +$(FIRRTL_FILE) $(ANNO_FILE) &: firesim_target_symlink_hook + @mkdir -p $(@D) + @mkdir -p $(TARGET_SBT_DIR)/target/generated-src/$(long_name) + source $(TARGET_SBT_DIR)/../sourceme.sh + cd $(TARGET_SBT_DIR) && \ + pwd && \ + ${SBT} ";project $(TARGET_SBT_PROJECT); runMain chipyard.Generator \ + --target-dir $(TARGET_SBT_DIR)/target/generated-src/$(long_name) \ + --name $(long_name) \ + --top-module $(DESIGN_PACKAGE).$(DESIGN) \ + --legacy-configs $(TARGET_CONFIG_PACKAGE):$(TARGET_CONFIG) \ + --emit-legacy-sfc" + # Link to the generated files + ln -sf $(TARGET_SBT_DIR)/target/generated-src/$(long_name)/$(long_name).sfc.fir $(FIRRTL_FILE) + ln -sf $(TARGET_SBT_DIR)/target/generated-src/$(long_name)/$(long_name).anno.json $(ANNO_FILE) + # .d needed to run metasim CI tests + ln -sf $(TARGET_SBT_DIR)/target/generated-src/$(long_name)/$(long_name).d $(GENERATED_DIR)/$(long_name).d + +####################################### +# Setup Extra Verilator Compile Flags # +####################################### + +## default flags added for cva6 +CVA6_VERILATOR_FLAGS = \ + --unroll-count 256 \ + -Werror-PINMISSING \ + -Werror-IMPLICIT \ + -Wno-fatal \ + -Wno-PINCONNECTEMPTY \ + -Wno-ASSIGNDLY \ + -Wno-DECLFILENAME \ + -Wno-UNUSED \ + -Wno-UNOPTFLAT \ + -Wno-BLKANDNBLK \ + -Wno-style \ + -Wall + +# normal flags used for midas builds (that are incompatible with cva6) +DEFAULT_MIDAS_VERILATOR_FLAGS = \ + --assert + +# AJG: this must be evaluated after verilog generation to work (hence the =) +EXTRA_VERILATOR_FLAGS = \ + $(shell if ! grep -iq "module.*cva6" $(simulator_verilog); then echo "$(DEFAULT_MIDAS_VERILATOR_FLAGS)"; else echo "$(CVA6_VERILATOR_FLAGS)"; fi) diff --git a/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk b/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk new file mode 100644 index 000000000..555710686 --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk @@ -0,0 +1,30 @@ +# Custom configuration for Buckyball FireSim builds +# This file overrides the default TARGET_CONFIG_PACKAGE to use our custom configs + +# Only used in this projects makefrags +makefile_path := $(abspath $(lastword $(MAKEFILE_LIST))) +makefile_dir := $(patsubst %/,%,$(dir $(makefile_path))) +chipyard_dir := $(abspath $(makefile_dir)/../../../../../../arch/thirdparty/chipyard) + +# These point at the main class of the target's Chisel generator +DESIGN_PACKAGE ?= firechip.chip +DESIGN ?= FireSim + +# Override to use our custom config package +TARGET_CONFIG_PACKAGE ?= sims.firesim +TARGET_CONFIG ?= FireSimBuckyballToyConfig + +# These guide chisel elaboration of simulation components by MIDAS, +# including models and widgets. +PLATFORM_CONFIG_PACKAGE ?= firesim.firesim +PLATFORM_CONFIG ?= BaseF1Config + +# Override project for the target. +TARGET_SBT_PROJECT := buckyball + +# Point to our project directory +TARGET_SBT_DIR := $(abspath $(makefile_dir)/../../../../../../arch) +TARGET_SOURCE_DIRS := $(abspath $(makefile_dir)/../../../../../../arch/src/main/scala) + +# SBT launcher +SBT ?= java -jar $(chipyard_dir)/scripts/sbt-launch.jar $(SBT_OPTS) diff --git a/bbdev/api/steps/firesim/scripts/makefrag/firesim/driver.mk b/bbdev/api/steps/firesim/scripts/makefrag/firesim/driver.mk new file mode 100644 index 000000000..4042d8382 --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/makefrag/firesim/driver.mk @@ -0,0 +1,70 @@ +# See LICENSE for license details. + +# DOC include start: Bridge Build System Changes +########################## +# Driver Sources & Flags # +########################## + +ifeq (,$(wildcard $(RISCV)/lib/libriscv.so)) +$(warning libriscv not found) +LRISCV= +else +LRISCV=-lriscv +endif + +firechip_lib_dir = $(chipyard_dir)/generators/firechip/chip/src/main/cc +firechip_bridgestubs_lib_dir = $(chipyard_dir)/generators/firechip/bridgestubs/src/main/cc +testchipip_csrc_dir = $(chipyard_dir)/generators/testchipip/src/main/resources/testchipip/csrc + +# DRIVER_H only used to update recipe pre-reqs (ok to track more files) + +# fesvr and related srcs +DRIVER_H += \ + $(shell find $(testchipip_csrc_dir) -name "*.h") \ + $(shell find $(firechip_bridgestubs_lib_dir)/fesvr -name "*.h") +DRIVER_CC += \ + $(testchipip_csrc_dir)/cospike_impl.cc \ + $(testchipip_csrc_dir)/testchip_tsi.cc \ + $(testchipip_csrc_dir)/testchip_dtm.cc \ + $(testchipip_csrc_dir)/testchip_htif.cc \ + $(firechip_bridgestubs_lib_dir)/fesvr/firesim_tsi.cc \ + $(firechip_bridgestubs_lib_dir)/fesvr/firesim_dtm.cc \ + $(RISCV)/lib/libfesvr.a +# Disable missing override warning for testchipip. +TARGET_CXX_FLAGS += \ + -isystem $(testchipip_csrc_dir) \ + -isystem $(RISCV)/include \ + -Wno-inconsistent-missing-override +TARGET_LD_FLAGS += \ + -L$(RISCV)/lib \ + -Wl,-rpath,$(RISCV)/lib \ + $(LRISCV) + +# top-level sources +DRIVER_CC += $(addprefix $(firechip_lib_dir)/firesim/, $(addsuffix .cc, firesim_top)) +TARGET_CXX_FLAGS += -I$(firechip_bridgestubs_lib_dir)/bridge/test + +# bridge sources +DRIVER_H += $(shell find $(firechip_bridgestubs_lib_dir) -name "*.h") +DRIVER_CC += \ + $(wildcard \ + $(addprefix \ + $(firechip_bridgestubs_lib_dir)/, \ + $(addsuffix .cc,bridges/* bridges/tracerv/* bridges/cospike/*) \ + ) \ + ) +TARGET_CXX_FLAGS += \ + -I$(firechip_bridgestubs_lib_dir) \ + -I$(firechip_bridgestubs_lib_dir)/bridge \ + -I$(firechip_bridgestubs_lib_dir)/bridge/tracerv \ + -I$(firechip_bridgestubs_lib_dir)/bridge/cospike +TARGET_LD_FLAGS += \ + -l:libdwarf.so -l:libelf.so \ + -lz \ + +# other +TARGET_CXX_FLAGS += \ + -I$(GENERATED_DIR) \ + -g + +# DOC include end: Bridge Build System Changes diff --git a/bbdev/api/steps/firesim/scripts/makefrag/firesim/metasim.mk b/bbdev/api/steps/firesim/scripts/makefrag/firesim/metasim.mk new file mode 100644 index 000000000..1e7c50a35 --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/makefrag/firesim/metasim.mk @@ -0,0 +1,140 @@ +# See LICENSE for license details. + +################################################################ +# SW RTL Simulation Args -- for MIDAS- & FPGA-level Simulation # +################################################################ +TIMEOUT_CYCLES = 100000000 + +NET_SLOT ?= 0 +NET_LINK_LATENCY ?= 6405 +NET_BW ?= 100 +NET_SHMEMPORTNAME ?= $(shell printf '%0100d' $(NET_SLOT)) +NET_LOOPBACK ?= +nic-loopback0 +NET_MACADDR ?= $(shell printf '00:00:00:00:00:%02x' $$(($(NET_SLOT)+2))) +nic_args = +shmemportname0=$(NET_SHMEMPORTNAME) +macaddr0=$(NET_MACADDR) \ + +niclog0=niclog$(NET_SLOT) +linklatency0=$(NET_LINK_LATENCY) \ + +netbw0=$(NET_BW) +netburst0=8 $(NET_LOOPBACK) +tracer_args = +tracefile=TRACEFILE +blkdev_args = +blkdev-in-mem0=128 +blkdev-log0=blkdev-log$(NET_SLOT) +autocounter_args = +autocounter-readrate=1000 +autocounter-filename-base=AUTOCOUNTERFILE +# Neglecting this +arg will make the simulator use the same step size as on the +# FPGA. This will make ML simulation more closely match results seen on the +# FPGA at the expense of dramatically increased target runtime +serial_args = +fesvr-step-size=128 +#serial_args = + +COMMON_SIM_ARGS := $(COMMON_SIM_ARGS) $(serial_args) $(nic_args) $(tracer_args) $(blkdev_args) $(autocounter_args) + +# Arguments used only at a particular simulation abstraction +MIDAS_LEVEL_SIM_ARGS ?= +max-cycles=$(TIMEOUT_CYCLES) +FPGA_LEVEL_SIM_ARGS ?= + +################################ +# Verilator/VCS/XSIM execution # +################################ + +verilator = $(GENERATED_DIR)/V$(DESIGN) +verilator_debug = $(GENERATED_DIR)/V$(DESIGN)-debug +verilator_args = +vcs = $(GENERATED_DIR)/$(DESIGN) +vcs_debug = $(GENERATED_DIR)/$(DESIGN)-debug +vcs_args = +vcs+initreg+0 +vcs+initmem+0 +xsim = $(GENERATED_DIR)/$(DESIGN)-$(PLATFORM) +xcelium = $(GENERATED_DIR)/X$(DESIGN) +sim_binary_basename := $(basename $(notdir $(SIM_BINARY))) + +run-verilator: $(verilator) + cd $(dir $<) && \ + $(verilator) +permissive $(verilator_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) +which_disasm := $(shell which spike-dasm 2> /dev/null) +ifneq ($(which_disasm),) + disasm := 3>&1 1>&2 2>&3 | $(which_disasm) $(DISASM_EXTENSION) > +endif + +# Some of the generated suites use specific plus args, that are prefixed with +# the binary name. These are captured with $($*_ARGS) +$(OUTPUT_DIR)/%.run: $(OUTPUT_DIR)/% $(EMUL) + cd $(dir $($(EMUL))) && \ + ./$(notdir $($(EMUL))) $< $($*_ARGS) $($(EMUL)_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \ + 2> /dev/null 2> $@ && [ $$PIPESTATUS -eq 0 ] + +$(OUTPUT_DIR)/%.out: $(OUTPUT_DIR)/% $(EMUL) + cd $(dir $($(EMUL))) && \ + ./$(notdir $($(EMUL))) $< $($*_ARGS) $($(EMUL)_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \ + $(disasm) $@ && [ $$PIPESTATUS -eq 0 ] + +$(OUTPUT_DIR)/%.vpd: $(OUTPUT_DIR)/% $(EMUL)-debug + cd $(dir $($(EMUL)_debug)) && \ + ./$(notdir $($(EMUL)_debug)) $< +vcdplusfile=$@ $($*_ARGS) $($(EMUL)_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \ + $(disasm) $(patsubst %.vpd,%.out,$@) && [ $$PIPESTATUS -eq 0 ] + +$(OUTPUT_DIR)/%.fsdb: $(OUTPUT_DIR)/% $(EMUL)-debug + cd $(dir $($(EMUL)_debug)) && \ + ./$(notdir $($(EMUL)_debug)) $< +fsdbfile=$@ $($*_ARGS) $($(EMUL)_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \ + $(disasm) $(patsubst %.fsdb,%.out,$@) && [ $$PIPESTATUS -eq 0 ] + +.PRECIOUS: $(OUTPUT_DIR)/%.fsdb $(OUTPUT_DIR)/%.vpd $(OUTPUT_DIR)/%.out $(OUTPUT_DIR)/%.run + +# TraceGen rules + +$(OUTPUT_DIR)/tracegen.out: $($(EMUL)) + mkdir -p $(OUTPUT_DIR) && \ + cd $(dir $($(EMUL))) && \ + ./$(notdir $($(EMUL))) $($(EMUL)_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \ + 2> /dev/null 2> $@ && [ $$PIPESTATUS -eq 0 ] + +$(OUTPUT_DIR)/tracegen.result: $(OUTPUT_DIR)/tracegen.out $(AXE) + $(chipyard_dir)/scripts/check-tracegen.sh $< > $@ + +fsim-tracegen: $(OUTPUT_DIR)/tracegen.result + +.PHONY: fsim-tracegen diff --git a/bbdev/api/steps/firesim/scripts/yaml/config_build.yaml b/bbdev/api/steps/firesim/scripts/yaml/config_build.yaml new file mode 100644 index 000000000..e6ee2d5ce --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/yaml/config_build.yaml @@ -0,0 +1,44 @@ +# Build-time build design / AGFI configuration for the FireSim Simulation Manager +# See https://docs.fires.im/en/stable/Advanced-Usage/Manager/Manager-Configuration-Files.html for documentation of all of these params. + +# ------------------------------------------------------------ +# Attention! This is the example on the Local U280 Machine +# Remember to modify this file use in the same way of firesim +# ------------------------------------------------------------ + +# this refers to build farms defined in config_build_farm.yaml +build_farm: + base_recipe: build-farm-recipes/externally_provisioned.yaml + recipe_arg_overrides: + # REQUIRED: (replace this) default location of build directory on build host. + default_build_dir: /home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/FIRESIM_BUILD_DIR + # REQUIRED: List of IP addresses (or "localhost"). Each can have an OPTIONAL + # argument, called "override_build_dir", specifying to override the default + # build directory. + # + # Ex: + # build_farm_hosts: + # # use localhost and don't override the default build dir + # - localhost + # # use other IP address (don't override default build dir) + # - "111.111.1.111" + # # use other IP address (override default build dir for this build host) + # - "222.222.2.222": + # override_build_dir: /scratch/specific-build-host-build-dir + build_farm_hosts: + - localhost + +builds_to_run: + # this section references builds defined in config_build_recipes.yaml + # if you add a build here, it will be built when you run buildbitstream + # - alveo_u280_firesim_GemminiBuckyballConfig_no_nic + - alveo_u280_firesim_BuckyballToyConfig_no_nic + +agfis_to_share: + # - midasexamples_gcd + +share_with_accounts: + # To share with a specific user: + somebodysname: 123456789012 + # To share publicly: + # public: public diff --git a/bbdev/api/steps/firesim/scripts/yaml/config_build_recipes.yaml b/bbdev/api/steps/firesim/scripts/yaml/config_build_recipes.yaml new file mode 100644 index 000000000..01d8afb4f --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/yaml/config_build_recipes.yaml @@ -0,0 +1,64 @@ +# Build-time build recipe configuration for the FireSim Simulation Manager +# See https://docs.fires.im/en/stable/Advanced-Usage/Manager/Manager-Configuration-Files.html for documentation of all of these params. + +# this file contains sections that describe hardware designs that /can/ be built. +# edit config_build.yaml to actually "turn on" a config to be built when you run +# buildbitstream + +# ------------------------------------------------------------ +# Attention! This is the example on the Local U280 Machine +# Remember to modify this file use in the same way of firesim +# ------------------------------------------------------------ + +########### +# Schema: +########### +# : +# PLATFORM: +# TARGET_PROJECT: +# TARGET_PROJECT_MAKEFRAG: +# DESIGN: +# TARGET_CONFIG: +# PLATFORM_CONFIG: +# deploy_quintuplet: +# platform_config_args: +# fpga_frequency: +# build_strategy: +# post_build_hook: +# metasim_customruntimeconfig: +# bit_builder_recipe: +# # OPTIONAL: overrides for bit builder recipe +# # Arg structure should be identical to the args given +# # in the base_recipe. +# #bit_builder_arg_overrides: +# # : + +alveo_u280_firesim_GemminiBuckyballConfig_no_nic: + PLATFORM: xilinx_alveo_u280 + TARGET_PROJECT: firesim + TARGET_PROJECT_MAKEFRAG: ../makefrag/firesim + DESIGN: FireSim + TARGET_CONFIG: sims.firesim.FireSimGemminiBuckyballConfig + PLATFORM_CONFIG: BaseXilinxAlveoU280Config + deploy_quintuplet: null + platform_config_args: + fpga_frequency: 30 + build_strategy: TIMING + post_build_hook: null + metasim_customruntimeconfig: null + bit_builder_recipe: bit-builder-recipes/xilinx_alveo_u280.yaml + +alveo_u280_firesim_BuckyballToyConfig_no_nic: + PLATFORM: xilinx_alveo_u280 + TARGET_PROJECT: firesim + TARGET_PROJECT_MAKEFRAG: ../makefrag/firesim + DESIGN: FireSim + TARGET_CONFIG: sims.firesim.FireSimBuckyballToyConfig + PLATFORM_CONFIG: BaseXilinxAlveoU280Config + deploy_quintuplet: null + platform_config_args: + fpga_frequency: 30 + build_strategy: TIMING + post_build_hook: null + metasim_customruntimeconfig: null + bit_builder_recipe: bit-builder-recipes/xilinx_alveo_u280.yaml diff --git a/bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml b/bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml new file mode 100644 index 000000000..98b8155e4 --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml @@ -0,0 +1,43 @@ +# Hardware config database for FireSim Simulation Manager +# See https://docs.fires.im/en/stable/Advanced-Usage/Manager/Manager-Configuration-Files.html for documentation of all of these params. + +# ------------------------------------------------------------ +# Attention! This is the example on the Local U280 Machine +# Remember to modify this file use in the same way of firesim +# ------------------------------------------------------------ + +# Hardware configs represent a combination of: +# - an agfi/bitstream_tar with the bitstream for an fpga +# - (optional) a deployquintuplet override +# - (optional) a deploy_makefrag_override +# - (optional) a custom_runtime_config + +# The HWDBs (and their AGFI's/bitstream_tar's) provided below are public and available to all users. + +# DOCREF START: Example HWDB Entry +midasexamples_gcd: + bitstream_tar: https://raw.githubusercontent.com/invalid/address.tar.gz + deploy_quintuplet_override: null + deploy_makefrag_override: null + custom_runtime_config: null +# DOCREF END: Example HWDB Entry + +# alveo_u280_firesim_GemminiBuckyballConfig_no_nic: +# bitstream_tar: file:///home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/results-build/2025-09-24--17-27-06-alveo_u280_firesim_GemminiBuckyballConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimGemminiBuckyballConfig-BaseXilinxAlveoU280Config/firesim.tar.gz +# deploy_quintuplet_override: null +# custom_runtime_config: null + +alveo_u280_firesim_GemminiBuckyballConfig_no_nic: + bitstream_tar: file:///home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/results-build/2025-11-19--17-01-23-alveo_u280_firesim_GemminiBuckyballConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimGemminiBuckyballConfig-BaseXilinxAlveoU280Config/firesim.tar.gz + deploy_quintuplet_override: null + custom_runtime_config: null + +# alveo_u280_firesim_BuckyballToyConfig_no_nic: +# bitstream_tar: file:///home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/results-build/2025-09-29--19-18-46-alveo_u280_firesim_BuckyballToyConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimBuckyballToyConfig-BaseXilinxAlveoU280Config/firesim.tar.gz +# deploy_quintuplet_override: null +# custom_runtime_config: null + +alveo_u280_firesim_BuckyballToyConfig_no_nic: + bitstream_tar: file:///home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/results-build/2025-11-23--04-11-23-alveo_u280_firesim_BuckyballToyConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimBuckyballToyConfig-BaseXilinxAlveoU280Config/firesim.tar.gz + deploy_quintuplet_override: null + custom_runtime_config: null diff --git a/bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml b/bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml new file mode 100644 index 000000000..490db5f9b --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml @@ -0,0 +1,110 @@ +# Run-time configuration for the FireSim Simulation Manager +# See https://docs.fires.im/en/stable/Advanced-Usage/Manager/Manager-Configuration-Files.html for documentation of all of these params. + +# ------------------------------------------------------------ +# Attention! This is the example on the Local U280 Machine +# Remember to modify this file use in the same way of firesim +# ------------------------------------------------------------ + +run_farm: + base_recipe: run-farm-recipes/externally_provisioned.yaml + recipe_arg_overrides: + # REQUIRED: default platform used for run farm hosts. this is a class specifying + # how to run simulations on a run farm host. + default_platform: XilinxAlveoU280InstanceDeployManager + + # REQUIRED: default directory where simulations are run out of on the run farm hosts + default_simulation_dir: /home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/FIRESIM_RUNS_DIR + + # REQUIRED: default fpga db file that enumerates what fpgas are available on the machine (used by XilinxU* Deploy Managers) + default_fpga_db: /opt/firesim-db.json + + # REQUIRED: List of unique hostnames/IP addresses, each with their + # corresponding specification that describes the properties of the host. + # + # Ex: + # run_farm_hosts_to_use: + # # use localhost which is described by "four_fpgas_spec" below. + # - localhost: four_fpgas_spec + # # supply IP address, which points to a machine that is described + # # by "four_fpgas_spec" below. + # - "111.111.1.111": four_fpgas_spec + run_farm_hosts_to_use: + - localhost: one_fpgas_spec + +metasimulation: + metasimulation_enabled: false + # vcs or verilator. use vcs-debug or verilator-debug for waveform generation + metasimulation_host_simulator: verilator + # plusargs passed to the simulator for all metasimulations + metasimulation_only_plusargs: "+fesvr-step-size=128 +max-cycles=100000000" + # plusargs passed to the simulator ONLY FOR vcs metasimulations + metasimulation_only_vcs_plusargs: "+vcs+initreg+0 +vcs+initmem+0" + +# DOCREF START: target_config area +target_config: + topology: no_net_config + no_net_num_nodes: 1 + link_latency: 6405 + switching_latency: 10 + net_bandwidth: 200 + profile_interval: -1 + + # This references a section from config_hwdb.yaml for fpga-accelerated simulation + # or from config_build_recipes.yaml for metasimulation + # In homogeneous configurations, use this to set the hardware config deployed + # for all simulators + default_hw_config: alveo_u280_firesim_BuckyballToyConfig_no_nic + + # Advanced: Specify any extra plusargs you would like to provide when + # booting the simulator (in both FPGA-sim and metasim modes). This is + # a string, with the contents formatted as if you were passing the plusargs + # at command line, e.g. "+a=1 +b=2" + plusarg_passthrough: "" +# DOCREF END: target_config area + +tracing: + enable: no + + # Trace output formats. Only enabled if "enable" is set to "yes" above + # 0 = human readable; 1 = binary (compressed raw data); 2 = flamegraph (stack + # unwinding -> Flame Graph) + output_format: 0 + + # Trigger selector. + # 0 = no trigger; 1 = cycle count trigger; 2 = program counter trigger; 3 = + # instruction trigger + selector: 1 + start: 0 + end: -1 + +autocounter: + read_rate: 0 + +workload: + workload_name: interactive.json # br-base.json + terminate_on_completion: no + suffix_tag: null + +host_debug: + # When enabled (=yes), Zeros-out FPGA-attached DRAM before simulations + # begin (takes 2-5 minutes). + # In general, this is not required to produce deterministic simulations on + # target machines running linux. Enable if you observe simulation non-determinism. + zero_out_dram: no + # If disable_synth_asserts: no, simulation will print assertion message and + # terminate simulation if synthesized assertion fires. + # If disable_synth_asserts: yes, simulation ignores assertion firing and + # continues simulation. + disable_synth_asserts: no + +# DOCREF START: Synthesized Prints +synth_print: + # Start and end cycles for outputting synthesized prints. + # They are given in terms of the base clock and will be converted + # for each clock domain. + start: 0 + end: -1 + # When enabled (=yes), prefix print output with the target cycle at which the print was triggered + cycle_prefix: yes +# DOCREF END: Synthesized Prints diff --git a/bbdev/api/steps/marshal/.gitignore b/bbdev/api/steps/marshal/.gitignore new file mode 100644 index 000000000..0521c5fb4 --- /dev/null +++ b/bbdev/api/steps/marshal/.gitignore @@ -0,0 +1 @@ +!*.json diff --git a/bbdev/api/steps/marshal/01_build_api_step.py b/bbdev/api/steps/marshal/01_build_api_step.py new file mode 100644 index 000000000..4c0936cfe --- /dev/null +++ b/bbdev/api/steps/marshal/01_build_api_step.py @@ -0,0 +1,25 @@ +import asyncio +from utils.event_common import wait_for_result + +config = { + "type": "api", + "name": "Marshal Build", + "description": "build marshal", + "path": "/marshal/build", + "method": "POST", + "emits": ["marshal.build"], + "flows": ["marshal"], +} + + +async def handler(req, context): + body = req.get("body") or {} + await context.emit({"topic": "marshal.build", "data": body}) + # ================================================================================== + # Wait for result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/marshal/01_build_event_step.py b/bbdev/api/steps/marshal/01_build_event_step.py new file mode 100644 index 000000000..34480d8c4 --- /dev/null +++ b/bbdev/api/steps/marshal/01_build_event_step.py @@ -0,0 +1,70 @@ +import os +import subprocess +import sys + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result + +config = { + "type": "event", + "name": "Marshal Build", + "description": "build marshal", + "subscribes": ["marshal.build"], + "emits": [], + "flows": ["marshal"], +} + + +async def handler(data, context): + bbdir = get_buckyball_path() + script_dir = f"{bbdir}/workflow/steps/marshal/scripts" + # ================================================================================== + # Execute operation + # ================================================================================== + command = f"./marshal -v build interactive.json && ./marshal -v install interactive.json" + result = stream_run_logger( + cmd=command, + logger=context.logger, + cwd=script_dir, + stdout_prefix="marshal build", + stderr_prefix="marshal build", + ) + + # ================================================================================== + # Return result to API + # ================================================================================== + success_result, failure_result = await check_result( + context, result.returncode, continue_run=False + ) + + # ================================================================================== + # Continue routing + # Routing to completion or error handling + # ================================================================================== + if result.returncode == 0: + await context.emit( + { + "topic": "marshal.complete", + "data": {**data, "task": "marshal", "result": success_result}, + } + ) + else: + await context.emit( + { + "topic": "marshal.error", + "data": { + **data, + "task": "marshal", + "result": failure_result, + "returncode": result.returncode, + }, + } + ) + + return diff --git a/bbdev/api/steps/marshal/02_launch_api_step.py b/bbdev/api/steps/marshal/02_launch_api_step.py new file mode 100644 index 000000000..422f47dd2 --- /dev/null +++ b/bbdev/api/steps/marshal/02_launch_api_step.py @@ -0,0 +1,26 @@ +import asyncio +from utils.event_common import wait_for_result + +config = { + "type": "api", + "name": "Marshal Launch", + "description": "launch marshal", + "path": "/marshal/launch", + "method": "POST", + "emits": ["marshal.launch"], + "flows": ["marshal"], +} + + +async def handler(req, context): + body = req.get("body") or {} + await context.emit({"topic": "marshal.launch", "data": body}) + + # ================================================================================== + # Wait for result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/marshal/02_launch_event_step.py b/bbdev/api/steps/marshal/02_launch_event_step.py new file mode 100644 index 000000000..120e2b0d5 --- /dev/null +++ b/bbdev/api/steps/marshal/02_launch_event_step.py @@ -0,0 +1,50 @@ +import os +import subprocess +import sys + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result + +config = { + "type": "event", + "name": "Marshal Launch", + "description": "launch marshal", + "subscribes": ["marshal.launch"], + "emits": [], + "flows": ["marshal"], +} + + +async def handler(data, context): + bbdir = get_buckyball_path() + script_dir = f"{bbdir}/workflow/steps/marshal/scripts" + # ================================================================================== + # Execute operation + # ================================================================================== + command = f"./marshal -v launch interactive.json" + result = stream_run_logger( + cmd=command, + logger=context.logger, + cwd=script_dir, + stdout_prefix="marshal launch", + stderr_prefix="marshal launch", + ) + + # ================================================================================== + # Return result to API + # ================================================================================== + success_result, failure_result = await check_result( + context, result.returncode, continue_run=False + ) + + # ================================================================================== + # Continue routing + # Finish workflow + # ================================================================================== + return diff --git a/bbdev/api/steps/marshal/README.md b/bbdev/api/steps/marshal/README.md new file mode 100644 index 000000000..f8d25a32e --- /dev/null +++ b/bbdev/api/steps/marshal/README.md @@ -0,0 +1,53 @@ +# Marshal Workflow + +Marshal workflow in the Buckyball framework, used to build and launch the Marshal component. + +## API Usage Guide + +### `build` +**Endpoint**: `POST /marshal/build` + +**Function**: Build Marshal component + +**Parameters**: No specific parameters + +**Example**: +```bash +bbdev marshal --build +``` + +### `launch` +**Endpoint**: `POST /marshal/launch` + +**Function**: Launch Marshal service + +**Parameters**: No specific parameters + +**Example**: +```bash +bbdev marshal --launch +``` + +## Typical Workflow + +```bash +# 1. Build Marshal +bbdev marshal --build + +# 2. Launch Marshal service +bbdev marshal --launch +``` + +## Response Format + +All API calls return a unified format: +```json +{ + "status": 200, + "body": { + "success": true, + "processing": false, + "return_code": 0 + } +} +``` diff --git a/bbdev/api/steps/marshal/scripts/interactive.json b/bbdev/api/steps/marshal/scripts/interactive.json new file mode 100644 index 000000000..8bb194963 --- /dev/null +++ b/bbdev/api/steps/marshal/scripts/interactive.json @@ -0,0 +1,9 @@ +{ + "name" : "interactive", + "workdir" : ".", + "base" : "br-base.json", + "overlay" : "../output/overlay", + "host-init" : "./interactive/host-init.sh", + "rootfs-size" : "16GiB", + "spike-args" : "--extension=gemmini" +} diff --git a/bbdev/api/steps/marshal/scripts/interactive/host-init.sh b/bbdev/api/steps/marshal/scripts/interactive/host-init.sh new file mode 100755 index 000000000..3a481074f --- /dev/null +++ b/bbdev/api/steps/marshal/scripts/interactive/host-init.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# This script will run on the host from the workload directory +# (e.g. workloads/example-fed) every time the workload is built. +# It is recommended to call into something like a makefile because +# this script may be called multiple times. + +BBDIR=$(git rev-parse --show-toplevel) + +cd $BBDIR && source sourceme.sh + +echo "Building marshal workload" +bbdev workload --build + +mkdir -p $BBDIR/workflow/steps/marshal/output +rm -rf $BBDIR/workflow/steps/marshal/output/overlay +mkdir -p $BBDIR/workflow/steps/marshal/output/overlay/root + +# Copy workload binaries to /root directory +cp -r $BBDIR/bb-tests/output/workloads/src/* $BBDIR/workflow/steps/marshal/output/overlay/root/ diff --git a/bbdev/api/steps/marshal/scripts/marshal b/bbdev/api/steps/marshal/scripts/marshal new file mode 120000 index 000000000..6b68a6684 --- /dev/null +++ b/bbdev/api/steps/marshal/scripts/marshal @@ -0,0 +1 @@ +../../../../arch/thirdparty/chipyard/software/firemarshal/marshal \ No newline at end of file diff --git a/bbdev/api/steps/marshal/scripts/marshal-config.yaml b/bbdev/api/steps/marshal/scripts/marshal-config.yaml new file mode 100644 index 000000000..97b3664ef --- /dev/null +++ b/bbdev/api/steps/marshal/scripts/marshal-config.yaml @@ -0,0 +1,2 @@ +firesim-dir: '../../../../arch/thirdparty/chipyard/sims/firesim' +# board-dir: '../../../../arch/thirdparty/chipyard/software/firemarshal/boards/prototype' diff --git a/bbdev/api/steps/palladium/01_verilog_api_step.py b/bbdev/api/steps/palladium/01_verilog_api_step.py new file mode 100644 index 000000000..975339f67 --- /dev/null +++ b/bbdev/api/steps/palladium/01_verilog_api_step.py @@ -0,0 +1,46 @@ +import asyncio +from utils.event_common import wait_for_result + + +from utils.path import get_buckyball_path + + +config = { + "type": "api", + "name": "palladium Verilog", + "description": "generate verilog code", + "path": "/palladium/verilog", + "method": "POST", + "emits": ["palladium.verilog"], + "flows": ["palladium"], +} + + +async def handler(req, context): + bbdir = get_buckyball_path() + body = req.get("body") or {} + + # Get config name, must be provided + config_name = body.get("config") + if not config_name or config_name == "None": + return { + "status": "error", + "message": "Configuration name is required. Please specify --config_name parameter.", + "example": './bbdev palladium --verilog "--config_name sims.palladium.BuckyballToyP2EConfig"', + } + + data = { + "config": config_name, + "balltype": body.get("balltype"), + "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), + } + await context.emit({"topic": "palladium.verilog", "data": data}) + + # ================================================================================== + # Wait for simulation result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/palladium/01_verilog_event_step.py b/bbdev/api/steps/palladium/01_verilog_event_step.py new file mode 100644 index 000000000..125b3d2a2 --- /dev/null +++ b/bbdev/api/steps/palladium/01_verilog_event_step.py @@ -0,0 +1,184 @@ +import os +import subprocess +import sys + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result + +config = { + "type": "event", + "name": "make verilog", + "description": "generate verilog code", + "subscribes": ["palladium.verilog"], + "emits": ["palladium.build"], + "flows": ["palladium"], +} + + +async def handler(data, context): + bbdir = get_buckyball_path() + # Use arch/build as the base directory for chipyard.Generator + base_build_dir = f"{data.get('output_dir', f'{bbdir}/arch/build')}/palladium" + # Output directory for final Verilog files + verilog_output_dir = f"{base_build_dir}/verilog" + arch_dir = f"{bbdir}/arch" + + # Get config name, must be provided + config_name = data.get("config") + if not config_name or config_name == "None": + context.logger.error("Configuration name is required but not provided") + success_result, failure_result = await check_result( + context, + 1, + continue_run=False, + extra_fields={ + "task": "validation", + "error": "Configuration name is required. Please specify --config_name parameter.", + "example": './bbdev palladium --verilog "--config_name sims.palladium.BuckyballToyP2EConfig"', + }, + ) + return failure_result + + context.logger.info(f"Using configuration: {config_name}") + + # ================================================================================== + # Step 1: Generate FIRRTL using chipyard.Generator + # ================================================================================== + context.logger.info("Step 1: Generating FIRRTL with chipyard.Generator...") + os.system(f"mkdir -p {verilog_output_dir}") + firrtl_command = ( + f"cd {arch_dir} && " + f"sbt -J-Xmx256G -J-Xss64M -J-XX:+UseG1GC -J-XX:MaxGCPauseMillis=1000 " + f'"runMain chipyard.Generator ' + f"-td {base_build_dir} " + f"-T palladium.fpga.VCU118FPGATestHarness " + f'-C {config_name}"' + ) + + result = stream_run_logger( + cmd=firrtl_command, + logger=context.logger, + cwd=arch_dir, + stdout_prefix="palladium firrtl", + stderr_prefix="palladium firrtl", + ) + + if result.returncode != 0: + context.logger.error(f"FIRRTL generation failed with code {result.returncode}") + success_result, failure_result = await check_result( + context, + result.returncode, + continue_run=False, + extra_fields={"task": "firrtl", "step": "generate"}, + ) + return failure_result + + # ================================================================================== + # Step 2: Convert FIRRTL to SystemVerilog using firtool + # ================================================================================== + context.logger.info("Step 2: Converting FIRRTL to SystemVerilog with firtool...") + + # Extract the simple class name from the full config name + # e.g., "sims.palladium.BuckyballToyP2EConfig" -> "BuckyballToyP2EConfig" + config_class_name = config_name.split(".")[-1] + + # Find the generated FIRRTL file (in base_build_dir, not verilog_output_dir) + fir_file = f"{base_build_dir}/palladium.fpga.{config_class_name}.fir" + if not os.path.exists(fir_file): + context.logger.error(f"FIRRTL file not found: {fir_file}") + context.logger.info(f"Looking for files in {base_build_dir}...") + # List files to help debug + if os.path.exists(base_build_dir): + files = os.listdir(base_build_dir) + context.logger.info(f"Files in build dir: {files}") + success_result, failure_result = await check_result( + context, + 1, + continue_run=False, + extra_fields={"task": "firrtl", "step": "file_check"}, + ) + return failure_result + + verilog_command = ( + f"firtool {fir_file} " + f"-o {verilog_output_dir} " + f"--split-verilog " + f"--disable-all-randomization " + f"--strip-debug-info " + f"--ignore-read-enable-mem " + f"--lowering-options=disallowLocalVariables " + f"--disable-annotation-unknown" + ) + + result = stream_run_logger( + cmd=verilog_command, + logger=context.logger, + cwd=arch_dir, + stdout_prefix="palladium verilog", + stderr_prefix="palladium verilog", + ) + + if result.returncode != 0: + context.logger.error(f"Verilog generation failed with code {result.returncode}") + success_result, failure_result = await check_result( + context, + result.returncode, + continue_run=False, + extra_fields={"task": "verilog", "step": "firtool"}, + ) + return failure_result + + # ================================================================================== + # Step 3: Verify and clean up + # ================================================================================== + # context.logger.info("Step 3: Verifying generated files...") + + # # Check if top-level Verilog was generated + # top_sv_dir = f"{verilog_output_dir}/VCU118FPGATestHarness.sv" + # top_sv_file = f"{top_sv_dir}/VCU118FPGATestHarness.sv" + + # if os.path.exists(top_sv_file): + # # Count generated files + # sv_files = [f for f in os.listdir(top_sv_dir) if f.endswith('.sv')] + # context.logger.info(f"Successfully generated {len(sv_files)} SystemVerilog files") + # context.logger.info(f"Top-level module: {top_sv_file}") + # else: + # context.logger.error(f"Top-level Verilog file not found: {top_sv_file}") + + # # Remove unwanted file + # topname_file = f"{arch_dir}/TestHarness.sv" + # if os.path.exists(topname_file): + # os.remove(topname_file) + # context.logger.info(f"Removed unwanted file: {topname_file}") + + # ================================================================================== + # Return result to API + # ================================================================================== + success_result, failure_result = await check_result( + context, + result.returncode, + continue_run=data.get("from_run_workflow", False), + extra_fields={ + "task": "verilog", + "output_dir": verilog_output_dir, + "top_module": "VCU118FPGATestHarness", + }, + ) + + # ================================================================================== + # Continue routing + # Routing to verilog or finish workflow + # For run workflow, continue to verilog; for standalone clean, complete + # ================================================================================== + if data.get("from_run_workflow"): + await context.emit( + {"topic": "palladium.build", "data": {**data, "task": "run"}} + ) + + return diff --git a/bbdev/api/steps/sardine/01_run_api_step.py b/bbdev/api/steps/sardine/01_run_api_step.py new file mode 100644 index 000000000..ac1e6a9f6 --- /dev/null +++ b/bbdev/api/steps/sardine/01_run_api_step.py @@ -0,0 +1,42 @@ +import subprocess +import sys +import os +import asyncio +from utils.event_common import wait_for_result + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path + +config = { + "type": "api", + "name": "running sardine", + "description": "running sardine", + "path": "/sardine/run", + "method": "POST", + "emits": ["sardine.run"], + "flows": ["sardine"], +} + + +async def handler(req, context): + bbdir = get_buckyball_path() + + body = req.get("body") or {} + + data = {"workload": body.get("workload", "")} + + sardine_dir = f"{bbdir}/bb-tests/sardine" + + await context.emit({"topic": "sardine.run", "data": data}) + + # ================================================================================== + # Wait for execution result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/sardine/01_run_event_step.py b/bbdev/api/steps/sardine/01_run_event_step.py new file mode 100644 index 000000000..eca10cbde --- /dev/null +++ b/bbdev/api/steps/sardine/01_run_event_step.py @@ -0,0 +1,55 @@ +from contextlib import redirect_stdout +import os +import subprocess +import sys +import time + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result + +config = { + "type": "event", + "name": "running sardine", + "description": "running sardine", + "subscribes": ["sardine.run"], + "emits": [], + "flows": ["sardine"], +} + + +async def handler(data, context): + bbdir = get_buckyball_path() + + sardine_dir = f"{bbdir}/bb-tests/sardine" + + command = f"python run_tests.py --allure -m \"({data.get('workload', '')})\"" + context.logger.info( + "Executing sardine command", {"command": command, "cwd": sardine_dir} + ) + result = stream_run_logger( + cmd=command, + logger=context.logger, + cwd=sardine_dir, + executable="bash", + stdout_prefix="sardine run", + stderr_prefix="sardine run", + ) + + # ================================================================================== + # Return execution result + # ================================================================================== + success_result, failure_result = await check_result( + context, result.returncode, continue_run=False + ) + + # ================================================================================== + # finish workflow + # ================================================================================== + return diff --git a/bbdev/api/steps/sardine/README.md b/bbdev/api/steps/sardine/README.md new file mode 100644 index 000000000..90a82f2a8 --- /dev/null +++ b/bbdev/api/steps/sardine/README.md @@ -0,0 +1,34 @@ +# Sardine Workflow + +Sardine workflow in the Buckyball framework for running Sardine-related tasks. + +## API Usage + +### `run` +**Endpoint**: `POST /sardine/run` + +**Function**: Run Sardine tasks + +**Parameters**: +- **`workload`** - Specify the workload to run + +**Example**: +```bash +# Run specified workload +bbdev sardine --run "--workload /path/to/workload" + +# Run default workload +bbdev sardine --run +``` + +**Response**: +```json +{ + "status": 200, + "body": { + "success": true, + "processing": false, + "return_code": 0 + } +} +``` diff --git a/bbdev/api/steps/verilator/01_clean_api_step.py b/bbdev/api/steps/verilator/01_clean_api_step.py new file mode 100644 index 000000000..b447792ce --- /dev/null +++ b/bbdev/api/steps/verilator/01_clean_api_step.py @@ -0,0 +1,26 @@ +import asyncio +from utils.event_common import wait_for_result + +config = { + "type": "api", + "name": "Verilator Clean", + "description": "clean build directory", + "path": "/verilator/clean", + "method": "POST", + "emits": ["verilator.clean"], + "flows": ["verilator"], +} + + +async def handler(req, context): + body = req.get("body") or {} + await context.emit({"topic": "verilator.clean", "data": {**body, "task": "clean"}}) + + # ================================================================================== + # Wait for simulation result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/verilator/01_clean_event_step.py b/bbdev/api/steps/verilator/01_clean_event_step.py new file mode 100644 index 000000000..626eba3e3 --- /dev/null +++ b/bbdev/api/steps/verilator/01_clean_event_step.py @@ -0,0 +1,57 @@ +import subprocess +import os +import sys + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result + +config = { + "type": "event", + "name": "make clean", + "description": "clean build directory", + "subscribes": ["verilator.run", "verilator.clean"], + "emits": ["verilator.verilog"], + "flows": ["verilator"], +} + + +async def handler(data, context): + bbdir = get_buckyball_path() + build_dir = f"{bbdir}/arch/build" + # ================================================================================== + # Execute operation + # ================================================================================== + command = f"rm -rf {build_dir}" + result = stream_run_logger( + cmd=command, + logger=context.logger, + cwd=bbdir, + stdout_prefix="verilator clean", + stderr_prefix="verilator clean", + ) + + # ================================================================================== + # Return result to API + # ================================================================================== + success_result, failure_result = await check_result( + context, + result.returncode, + continue_run=data.get("from_run_workflow", False), + extra_fields={"task": "clean"}, + ) + + # ================================================================================== + # Continue routing + # ================================================================================== + if data.get("from_run_workflow"): + await context.emit( + {"topic": "verilator.verilog", "data": {**data, "task": "run"}} + ) + + return diff --git a/bbdev/api/steps/verilator/02_verilog_api_step.py b/bbdev/api/steps/verilator/02_verilog_api_step.py new file mode 100644 index 000000000..5f9004234 --- /dev/null +++ b/bbdev/api/steps/verilator/02_verilog_api_step.py @@ -0,0 +1,46 @@ +import asyncio +from utils.event_common import wait_for_result + + +from utils.path import get_buckyball_path + + +config = { + "type": "api", + "name": "Verilator Verilog", + "description": "generate verilog code", + "path": "/verilator/verilog", + "method": "POST", + "emits": ["verilator.verilog"], + "flows": ["verilator"], +} + + +async def handler(req, context): + bbdir = get_buckyball_path() + body = req.get("body") or {} + + # Get config name, must be provided + config_name = body.get("config") + if not config_name or config_name == "None": + return { + "status": "error", + "message": "Configuration name is required. Please specify --config parameter.", + "example": 'bbdev verilator --verilog "--config sims.verilator.BuckyballToyVerilatorConfig"', + } + + data = { + "config": config_name, + "balltype": body.get("balltype"), + "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), + } + await context.emit({"topic": "verilator.verilog", "data": data}) + + # ================================================================================== + # Wait for simulation result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/verilator/02_verilog_event_step.py b/bbdev/api/steps/verilator/02_verilog_event_step.py new file mode 100644 index 000000000..fd84c8ecc --- /dev/null +++ b/bbdev/api/steps/verilator/02_verilog_event_step.py @@ -0,0 +1,93 @@ +import os +import subprocess +import sys + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result + +config = { + "type": "event", + "name": "make verilog", + "description": "generate verilog code", + "subscribes": ["verilator.verilog"], + "emits": ["verilator.build"], + "flows": ["verilator"], +} + + +async def handler(data, context): + bbdir = get_buckyball_path() + build_dir = data.get("output_dir", f"{bbdir}/arch/build/") + arch_dir = f"{bbdir}/arch" + + # Get config name, must be provided + config_name = data.get("config") + if not config_name or config_name == "None": + context.logger.error("Configuration name is required but not provided") + success_result, failure_result = await check_result( + context, + 1, + continue_run=False, + extra_fields={ + "task": "validation", + "error": "Configuration name is required. Please specify --config parameter.", + "example": 'bbdev verilator --verilog "--config sims.verilator.BuckyballToyVerilatorConfig"', + }, + ) + return failure_result + + context.logger.info(f"Using configuration: {config_name}") + + # ================================================================================== + # Execute operation + # ================================================================================== + if data.get("balltype"): + command = ( + f"mill -i __.test.runMain sims.verify.BallTopMain {data.get('balltype')} " + ) + else: + command = f"mill -i __.test.runMain sims.verilator.Elaborate {config_name} " + + command += "--disable-annotation-unknown -strip-debug-info -O=debug " + command += f"--split-verilog -o={build_dir}" + + result = stream_run_logger( + cmd=command, + logger=context.logger, + cwd=arch_dir, + stdout_prefix="verilator verilog", + stderr_prefix="verilator verilog", + ) + + # Remove unwanted file + topname_file = f"{arch_dir}/TestHarness.sv" + if os.path.exists(topname_file): + os.remove(topname_file) + + # ================================================================================== + # Return result to API + # ================================================================================== + success_result, failure_result = await check_result( + context, + result.returncode, + continue_run=data.get("from_run_workflow", False), + extra_fields={"task": "verilog"}, + ) + + # ================================================================================== + # Continue routing + # Routing to verilog or finish workflow + # For run workflow, continue to verilog; for standalone clean, complete + # ================================================================================== + if data.get("from_run_workflow"): + await context.emit( + {"topic": "verilator.build", "data": {**data, "task": "run"}} + ) + + return diff --git a/bbdev/api/steps/verilator/03_build_api_step.py b/bbdev/api/steps/verilator/03_build_api_step.py new file mode 100644 index 000000000..2c77f19e3 --- /dev/null +++ b/bbdev/api/steps/verilator/03_build_api_step.py @@ -0,0 +1,30 @@ +import asyncio +from utils.event_common import wait_for_result + +config = { + "type": "api", + "name": "Verilator Build", + "description": "build verilator executable", + "path": "/verilator/build", + "method": "POST", + "emits": ["verilator.build"], + "flows": ["verilator"], +} + + +async def handler(req, context): + body = req.get("body") or {} + data = { + "jobs": body.get("jobs", 16), + "cosim": body.get("cosim", False), + } + await context.emit({"topic": "verilator.build", "data": data}) + + # ================================================================================== + # Wait for simulation result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/verilator/03_build_event_step.py b/bbdev/api/steps/verilator/03_build_event_step.py new file mode 100644 index 000000000..917c6a624 --- /dev/null +++ b/bbdev/api/steps/verilator/03_build_event_step.py @@ -0,0 +1,137 @@ +import os +import subprocess +import glob +import sys + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result + +config = { + "type": "event", + "name": "make build", + "description": "build verilator executable", + "subscribes": ["verilator.build"], + "emits": ["verilator.sim", "verilator.cosim"], + "flows": ["verilator"], +} + + +async def handler(data, context): + bbdir = get_buckyball_path() + arch_dir = f"{bbdir}/arch" + build_dir = f"{arch_dir}/build" + waveform_dir = f"{arch_dir}/waveform" + log_dir = f"{arch_dir}/log" + cosim = data.get("cosim", False) + + # ================================================================================== + # Execute operation + # ================================================================================== + # Find sources + vsrcs = glob.glob(f"{build_dir}/**/*.v", recursive=True) + glob.glob( + f"{build_dir}/**/*.sv", recursive=True + ) + csrcs = ( + glob.glob(f"{arch_dir}/src/csrc/**/*.c", recursive=True) + + glob.glob(f"{arch_dir}/src/csrc/**/*.cc", recursive=True) + + glob.glob(f"{arch_dir}/src/csrc/**/*.cpp", recursive=True) + + glob.glob(f"{build_dir}/**/*.c", recursive=True) + + glob.glob(f"{build_dir}/**/*.cc", recursive=True) + + glob.glob(f"{build_dir}/**/*.cpp", recursive=True) + ) + + # Setup paths: fesvr from bebop/host/spike/riscv-isa-sim (install/include, install/lib) + bebop_isa_sim = f"{bbdir}/bebop/host/spike/riscv-isa-sim" + inc_paths = [ + os.environ.get("RISCV", "") + "/include" if os.environ.get("RISCV") else "", + f"{arch_dir}/thirdparty/chipyard/tools/DRAMSim2", + f"{bebop_isa_sim}/install/include", + build_dir, + f"{arch_dir}/src/csrc/include", + ] + inc_flags = " ".join([f"-I{p}" for p in inc_paths if p]) + + if cosim: + topname = "ToyBuckyball" + else: + topname = "TestHarness" + + cflags = f"{inc_flags} -DTOP_NAME='\"V{topname}\"' -std=c++17 " + if cosim: + cflags += " -DCOSIM" + ldflags = ( + f"-lreadline -ldramsim -lfesvr -lstdc++ " + f"-L{arch_dir}/thirdparty/chipyard/tools/DRAMSim2 " + f"-L{arch_dir}/thirdparty/chipyard/toolchains/riscv-tools/riscv-isa-sim/build " + f"-L{arch_dir}/thirdparty/chipyard/toolchains/riscv-tools/riscv-isa-sim/build/lib " + f"-L{bebop_isa_sim}/install/lib" + ) + + obj_dir = f"{build_dir}/obj_dir" + subprocess.run(f"rm -rf {obj_dir}", shell=True) + os.makedirs(obj_dir, exist_ok=True) + + sources = " ".join(vsrcs + csrcs) + jobs = data.get("jobs", "") + + verilator_cmd = ( + f"verilator -MMD --build -cc --trace -O3 --x-assign fast --x-initial fast --noassert -Wno-fatal " + f"--trace-fst --trace-threads 1 --output-split 10000 --output-split-cfuncs 100 " + f"--unroll-count 256 " + f"-Wno-PINCONNECTEMPTY " + f"-Wno-ASSIGNDLY " + f"-Wno-DECLFILENAME " + f"-Wno-UNUSED " + f"-Wno-UNOPTFLAT " + f"-Wno-BLKANDNBLK " + f"-Wno-style " + f"-Wall " + f"--timing -j {jobs} +incdir+{build_dir} --top {topname} {sources} " + f"-CFLAGS '{cflags}' -LDFLAGS '{ldflags}' --Mdir {obj_dir} --exe" + ) + + result = stream_run_logger( + cmd=verilator_cmd, + logger=context.logger, + cwd=bbdir, + stdout_prefix="verilator build", + stderr_prefix="verilator build", + ) + result = stream_run_logger( + cmd=f"make -C {obj_dir} -f V{topname}.mk V{topname}", + logger=context.logger, + cwd=bbdir, + stdout_prefix="verilator build", + stderr_prefix="verilator build", + ) + + # ================================================================================== + # Return result to API + # ================================================================================== + success_result, failure_result = await check_result( + context, + result.returncode, + continue_run=data.get("from_run_workflow", False), + extra_fields={"task": "build"}, + ) + + # ================================================================================== + # Continue routing + # ================================================================================== + if data.get("from_run_workflow"): + if cosim: + await context.emit( + {"topic": "verilator.cosim", "data": {**data, "task": "run"}} + ) + else: + await context.emit( + {"topic": "verilator.sim", "data": {**data, "task": "run"}} + ) + + return diff --git a/bbdev/api/steps/verilator/04_cosim_api_step.py b/bbdev/api/steps/verilator/04_cosim_api_step.py new file mode 100644 index 000000000..cf0c28209 --- /dev/null +++ b/bbdev/api/steps/verilator/04_cosim_api_step.py @@ -0,0 +1,40 @@ +import os +import sys +import asyncio +from utils.event_common import wait_for_result + +config = { + "type": "api", + "name": "Verilator Cosim", + "description": "run verilator cosimulation", + "path": "/verilator/cosim", + "method": "POST", + "emits": ["verilator.cosim"], + "flows": ["verilator"], +} + + +async def handler(req, context): + body = req.get("body") or {} + binary = body.get("binary", "") + batch = body.get("batch", False) + if not binary: + return { + "status": 400, + "body": { + "success": False, + "failure": True, + "returncode": 400, + "message": "binary parameter is required", + }, + } + + await context.emit({"topic": "verilator.cosim", "data": {**body, "task": "cosim"}}) + # ================================================================================== + # Wait for simulation result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/verilator/04_cosim_event_step.py b/bbdev/api/steps/verilator/04_cosim_event_step.py new file mode 100644 index 000000000..66d267766 --- /dev/null +++ b/bbdev/api/steps/verilator/04_cosim_event_step.py @@ -0,0 +1,124 @@ +import os +import subprocess +import sys +from datetime import datetime + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.search_workload import search_workload +from utils.event_common import check_result + +config = { + "type": "event", + "name": "make cosim", + "description": "run cosimulation", + "subscribes": ["verilator.cosim"], + "emits": [], + "flows": ["verilator"], +} + + +async def handler(data, context): + # ================================================================================== + # Get simulation parameters + # ================================================================================== + bbdir = get_buckyball_path() + arch_dir = f"{bbdir}/arch" + build_dir = f"{arch_dir}/build" + + # Generate timestamp + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + + binary_name = data.get("binary", "") + success_result, failure_result = await check_result( + context, returncode=(binary_name == None), continue_run=True + ) + + binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) + success_result, failure_result = await check_result( + context, returncode=(binary_path == None), continue_run=True + ) + if failure_result: + context.logger.error("binary not found", failure_result) + return + + # Create log and waveform directory + log_dir = f"{arch_dir}/log/{timestamp}-{binary_name}" + waveform_dir = f"{arch_dir}/waveform/{timestamp}-{binary_name}" + topname = "ToyBuckyball" + + os.makedirs(log_dir, exist_ok=True) + os.makedirs(waveform_dir, exist_ok=True) + + bin_path = f"{build_dir}/obj_dir/V{topname}" + batch = data.get("batch", False) + + # Create log and waveform file + log_path = f"{log_dir}/bdb.log" + fst_path = f"{waveform_dir}/waveform.fst" + # Remove old waveform file + subprocess.run(f"rm -f {waveform_dir}/waveform.vcd", shell=True, check=True) + + # ================================================================================== + # Execute simulation script with streaming output + # ================================================================================== + # batch_param = "True" if batch else "False" + # sim_cmd = f"./scripts/sim.sh {bin_path} {binary_path} {log_dir}/stdout.log \ + # {log_dir}/disasm.log {batch_param} {vcd_path} {log_path}" + sim_cmd = ( + f"{bin_path} +permissive +loadmem={binary_path} +loadmem_addr=800000000 " + f"{'+batch ' if batch else ''} " + f"+fst={fst_path} +log={log_path} +permissive-off " + f"{binary_path} > >(tee {log_dir}/stdout.log) 2> >(spike-dasm > {log_dir}/disasm.log)" + ) + script_dir = os.path.dirname(__file__) + + result = stream_run_logger( + cmd=sim_cmd, + logger=context.logger, + cwd=script_dir, + stdout_prefix="verilator sim", + stderr_prefix="verilator sim", + executable="bash", + ) + success_result, failure_result = await check_result( + context, returncode=result.returncode, continue_run=True + ) + if failure_result: + context.logger.error("sim failed", failure_result) + return + + if os.path.exists(f"{waveform_dir}/waveform.fst.heir"): + subprocess.run( + f"gtkwave -f {waveform_dir}/waveform.fst -H {waveform_dir}/waveform.fst.heir", + shell=True, + check=True, + ) + + # ================================================================================== + # Return simulation result + # ================================================================================== + # This is the end point of the run workflow, status will no longer be set to processing + success_result, failure_result = await check_result( + context, + result.returncode, + continue_run=False, + extra_fields={ + "task": "sim", + "binary": binary_path, + "log_dir": log_dir, + "waveform_dir": waveform_dir, + "timestamp": timestamp, + }, + ) + + # ================================================================================== + # Finish workflow + # ================================================================================== + + return diff --git a/bbdev/api/steps/verilator/04_sim_api_step.py b/bbdev/api/steps/verilator/04_sim_api_step.py new file mode 100644 index 000000000..dbfd4d00c --- /dev/null +++ b/bbdev/api/steps/verilator/04_sim_api_step.py @@ -0,0 +1,40 @@ +import os +import sys +import asyncio +from utils.event_common import wait_for_result + +config = { + "type": "api", + "name": "Verilator Sim", + "description": "run verilator simulation", + "path": "/verilator/sim", + "method": "POST", + "emits": ["verilator.sim"], + "flows": ["verilator"], +} + + +async def handler(req, context): + body = req.get("body") or {} + binary = body.get("binary", "") + batch = body.get("batch", False) + if not binary: + return { + "status": 400, + "body": { + "success": False, + "failure": True, + "returncode": 400, + "message": "binary parameter is required", + }, + } + + await context.emit({"topic": "verilator.sim", "data": {**body, "task": "sim"}}) + # ================================================================================== + # Wait for simulation result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/verilator/04_sim_event_step.py b/bbdev/api/steps/verilator/04_sim_event_step.py new file mode 100644 index 000000000..521236ad8 --- /dev/null +++ b/bbdev/api/steps/verilator/04_sim_event_step.py @@ -0,0 +1,124 @@ +import os +import subprocess +import sys +from datetime import datetime + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.search_workload import search_workload +from utils.event_common import check_result + +config = { + "type": "event", + "name": "make sim", + "description": "run simulation", + "subscribes": ["verilator.sim"], + "emits": [], + "flows": ["verilator"], +} + + +async def handler(data, context): + # ================================================================================== + # Get simulation parameters + # ================================================================================== + bbdir = get_buckyball_path() + arch_dir = f"{bbdir}/arch" + build_dir = f"{arch_dir}/build" + + # Generate timestamp + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + + binary_name = data.get("binary", "") + success_result, failure_result = await check_result( + context, returncode=(binary_name == None), continue_run=True + ) + + binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) + success_result, failure_result = await check_result( + context, returncode=(binary_path == None), continue_run=True + ) + if failure_result: + context.logger.error("binary not found", failure_result) + return + + # Create log and waveform directory + log_dir = f"{arch_dir}/log/{timestamp}-{binary_name}" + waveform_dir = f"{arch_dir}/waveform/{timestamp}-{binary_name}" + topname = "TestHarness" + + os.makedirs(log_dir, exist_ok=True) + os.makedirs(waveform_dir, exist_ok=True) + + bin_path = f"{build_dir}/obj_dir/V{topname}" + batch = data.get("batch", False) + + # Create log and waveform file + log_path = f"{log_dir}/bdb.log" + fst_path = f"{waveform_dir}/waveform.fst" + # Remove old waveform file + subprocess.run(f"rm -f {waveform_dir}/waveform.vcd", shell=True, check=True) + + # ================================================================================== + # Execute simulation script with streaming output + # ================================================================================== + # batch_param = "True" if batch else "False" + # sim_cmd = f"./scripts/sim.sh {bin_path} {binary_path} {log_dir}/stdout.log \ + # {log_dir}/disasm.log {batch_param} {vcd_path} {log_path}" + sim_cmd = ( + f"{bin_path} +permissive +loadmem={binary_path} +loadmem_addr=800000000 " + f"{'+batch ' if batch else ''} " + f"+fst={fst_path} +log={log_path} +permissive-off " + f"{binary_path} > >(tee {log_dir}/stdout.log) 2> >(spike-dasm > {log_dir}/disasm.log)" + ) + script_dir = os.path.dirname(__file__) + + result = stream_run_logger( + cmd=sim_cmd, + logger=context.logger, + cwd=script_dir, + stdout_prefix="verilator sim", + stderr_prefix="verilator sim", + executable="bash", + ) + success_result, failure_result = await check_result( + context, returncode=result.returncode, continue_run=True + ) + if failure_result: + context.logger.error("sim failed", failure_result) + return + + if os.path.exists(f"{waveform_dir}/waveform.fst.heir"): + subprocess.run( + f"gtkwave -f {waveform_dir}/waveform.fst -H {waveform_dir}/waveform.fst.heir", + shell=True, + check=True, + ) + + # ================================================================================== + # Return simulation result + # ================================================================================== + # This is the end point of the run workflow, status will no longer be set to processing + success_result, failure_result = await check_result( + context, + result.returncode, + continue_run=False, + extra_fields={ + "task": "sim", + "binary": binary_path, + "log_dir": log_dir, + "waveform_dir": waveform_dir, + "timestamp": timestamp, + }, + ) + + # ================================================================================== + # Finish workflow + # ================================================================================== + + return diff --git a/bbdev/api/steps/verilator/05_run_api_step.py b/bbdev/api/steps/verilator/05_run_api_step.py new file mode 100644 index 000000000..b29f35083 --- /dev/null +++ b/bbdev/api/steps/verilator/05_run_api_step.py @@ -0,0 +1,53 @@ +import asyncio +from utils.event_common import wait_for_result + +config = { + "type": "api", + "name": "Verilator Complete Workflow", + "description": "trigger complete verilator workflow", + "path": "/verilator/run", + "method": "POST", + "emits": ["verilator.run"], + "flows": ["verilator"], +} + + +async def handler(req, context): + body = req.get("body") or {} + + config = { + "binary": body.get("binary", ""), + "config": body.get("config", "sims.verilator.BuckyballToyVerilatorConfig"), + "jobs": body.get("jobs", "16"), + "batch": body.get("batch", False), + "cosim": body.get("cosim", False), + "from_run_workflow": True, + } + + await context.emit({"topic": "verilator.run", "data": config}) + + # ================================================================================== + # Wait for simulation result + # + # Expected return result format: + # { + # "status": 200/400/500, + # "body": { + # "success": true/false, + # "failure": true/false, + # "processing": true/false, + # "return_code": 0, + # other fields + # } + # } + # + # Since the Motia framework wraps data in the data field, it needs to be unpacked + # if isinstance(result, dict) and 'data' in result: + # return result['data'] + # return result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/verilator/README.md b/bbdev/api/steps/verilator/README.md new file mode 100644 index 000000000..25635f69e --- /dev/null +++ b/bbdev/api/steps/verilator/README.md @@ -0,0 +1,197 @@ +# Verilator Simulation Workflow + +Hardware simulation workflow based on Verilator in the Buckyball framework, providing a complete automation flow from RTL generation to simulation execution. Verilator is a high-performance Verilog simulator that supports fast functional verification and performance analysis. + +## II. Original API Usage Guide + +#### `run` +**Endpoint**: `POST /verilator/run` + +**Function**: Execute complete workflow. Clean build directory, generate Verilog, compile Verilator into simulation file, and run simulation directly + +**Parameters**: + +- **`jobs`** - Number of parallel compilation tasks + - Default value: `16` +- **`binary`** [Required] - Test binary file path + - Default value: `""` + +**Example**: +```bash +# bbdev wrapper +bbdev verilator --run "jobs 256 --binary ${buckyball}/bb-tests/workloads/build/src/CTest/ctest_mvin_mvout_alternate_test_singlecore-baremetal --batch" + +# Raw command +curl -X POST http://localhost:5000/verilator/run -H "Content-Type: application/json" -d '{"jobs": 8, "binary": "/home/user/test.elf"}' +``` + + +#### `clean` + +**Endpoint**: `POST /verilator/clean` + +**Function**: Clean build folder + +**Parameters**: None + +**Example**: +```bash +curl -X POST http://localhost:5000/verilator/clean +``` + +#### `verilog` + +**Endpoint**: `POST /verilator/verilog` + +**Function**: Only generate Verilog code, without compilation and simulation + +**Parameters**: None + +**Example**: +```bash +curl -X POST http://localhost:5000/verilator/verilog -d '{"jobs": 8}' +``` + +#### `build` + +**Endpoint**: `POST /verilator/build` + +**Function**: Compile verilog source files and cpp source files into executable simulation file + +**Parameters**: + +- **`jobs`** - Number of parallel compilation tasks + - Default value: `16` + +**Example**: +```bash +curl -X POST http://localhost:5000/verilator/build -d '{"jobs": 16}' +``` + +#### `sim` + +**Endpoint**: `POST /verilator/sim` + +**Function**: Run existing simulation executable + +**Parameters**: + +- **`binary`** [Required] - Custom test binary file path + +**Example**: +```bash +curl -X POST http://localhost:5000/verilator/sim \ + -H "Content-Type: application/json" \ + -d '{"binary": "/home/user/test_program.elf"}' +``` + + + + +## II. Developer Documentation + +### Directory Structure + +``` +steps/verilator/ +├── 00_start_node_noop_step.py # Workflow entry node definition +├── 00_start_node_noop_step.tsx # Frontend UI component +├── 01_run_api_step.py # Complete workflow API entry +├── 01_clean_api_step.py # Clean API endpoint +├── 01_verilog_api_step.py # Verilog generation API endpoint +├── 01_build_api_step.py # Build API endpoint +├── 01_sim_api_step.py # Simulation API endpoint +├── 02_clean_event_step.py # Clean build directory +├── 03_verilog_event_step.py # Verilog code generation +├── 04_build_event_step.py # Verilator compilation +├── 05_sim_event_step.py # Simulation execution +├── 99_complete_event_step.py # Completion handling +├── 99_error_event_step.py # Error handling +└── README.md # This document +``` + +### Workflow Steps Detailed + +#### 1. Entry Node (`00_start_node_noop_step.py`) +- **Type**: `noop` node +- **Function**: Provide UI interface entry point +- **Frontend**: "Start Build Verilator" button + +#### 2. API Endpoints +- **Complete Workflow API** (`01_run_api_step.py`): `/verilator` → `verilator.run` +- **Clean API** (`01_clean_api_step.py`): `/verilator/clean` → `verilator.clean` +- **Verilog Generation API** (`01_verilog_api_step.py`): `/verilator/verilog` → `verilator.verilog` +- **Build API** (`01_build_api_step.py`): `/verilator/build` → `verilator.build` +- **Simulation API** (`01_sim_api_step.py`): `/verilator/sim` → `verilator.sim` + +#### 3. Clean Step (`02_clean_event_step.py`) +- **Type**: `event` step +- **Subscribes**: `verilator.run`, `verilator.clean` +- **Emits**: `verilator.verilog`, `verilator.complete` +- **Function**: Delete build directory, serves workflow or standalone operation + +#### 4. Verilog Generation (`03_verilog_event_step.py`) +- **Type**: `event` step +- **Subscribes**: `verilator.verilog` +- **Emits**: `verilator.build`, `verilator.complete` +- **Function**: Use mill to generate Verilog code to build directory + +#### 5. Verilator Compilation (`04_build_event_step.py`) +- **Type**: `event` step +- **Subscribes**: `verilator.build` +- **Emits**: `verilator.sim`, `verilator.complete` +- **Function**: Compile Verilog and C++ source files into executable simulation file + +#### 6. Simulation Execution (`05_sim_event_step.py`) +- **Type**: `event` step +- **Subscribes**: `verilator.sim` +- **Emits**: `verilator.complete` +- **Function**: Run simulation, supports custom binary parameter + +#### 7. Completion Handling (`99_complete_event_step.py`) +- **Type**: `event` step +- **Subscribes**: `verilator.complete` +- **Function**: Print success message, mark workflow as complete + +#### 8. Error Handling (`99_error_event_step.py`) +- **Type**: `event` step +- **Subscribes**: `verilator.error` +- **Function**: Print error message, handle workflow exceptions + +### Workflow Diagram + +```mermaid +graph TD; + API[POST /verilator
Complete Workflow] --> RUN[verilator.run] + + CLEAN_DIRECT[verilator.clean
Single-step Clean] --> CLEAN_STEP[02_clean_event_step] + VERILOG_DIRECT[verilator.verilog
Single-step Generate] --> VERILOG_STEP[03_verilog_event_step] + BUILD_DIRECT[verilator.build
Single-step Build] --> BUILD_STEP[04_build_event_step] + SIM_DIRECT[verilator.sim
Single-step Simulation] --> SIM_STEP[05_sim_event_step] + + RUN --> CLEAN_STEP + CLEAN_STEP --> |Workflow Mode| VERILOG_STEP + CLEAN_STEP --> |Single-step Mode| COMPLETE[verilator.complete] + + VERILOG_STEP --> |Workflow Mode| BUILD_STEP + VERILOG_STEP --> |Single-step Mode| COMPLETE + + BUILD_STEP --> |Workflow Mode| SIM_STEP + BUILD_STEP --> |Single-step Mode| COMPLETE + + SIM_STEP --> COMPLETE + + COMPLETE --> COMPLETE_STEP[99_complete_event_step] + + CLEAN_STEP -.-> |Error| ERROR[verilator.error] + VERILOG_STEP -.-> |Error| ERROR + BUILD_STEP -.-> |Error| ERROR + SIM_STEP -.-> |Error| ERROR + + ERROR --> ERROR_STEP[99_error_event_step] + + classDef apiNode fill:#e1f5fe + classDef eventNode fill:#f3e5f5 + classDef stepNode fill:#e8f5e8 + classDef endNode fill:#fff3e0 +``` diff --git a/bbdev/api/steps/workload/01_buidl_api_step.py b/bbdev/api/steps/workload/01_buidl_api_step.py new file mode 100644 index 000000000..daade0a02 --- /dev/null +++ b/bbdev/api/steps/workload/01_buidl_api_step.py @@ -0,0 +1,38 @@ +import subprocess +import sys +import os +import asyncio +from utils.event_common import wait_for_result + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path + +config = { + "type": "api", + "name": "build workload", + "description": "build workload", + "path": "/workload/build", + "method": "POST", + "emits": ["workload.build"], + "flows": ["workload"], +} + + +async def handler(req, context): + bbdir = get_buckyball_path() + body = req.get("body") or {} + data = {"workload": body.get("workload", "")} + workload_dir = f"{bbdir}/bb-tests/workload" + await context.emit({"topic": "workload.build", "data": data}) + + # ================================================================================== + # Wait for simulation result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/workload/01_build_event_step.py b/bbdev/api/steps/workload/01_build_event_step.py new file mode 100644 index 000000000..289cdb457 --- /dev/null +++ b/bbdev/api/steps/workload/01_build_event_step.py @@ -0,0 +1,59 @@ +from contextlib import redirect_stdout +import os +import subprocess +import sys +import time + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result + +config = { + "type": "event", + "name": "build workload", + "description": "build workload", + "subscribes": ["workload.build"], + "emits": [], + "flows": ["workload"], +} + + +async def handler(data, context): + bbdir = get_buckyball_path() + workload_dir = f"{bbdir}/bb-tests" + build_dir = f"{workload_dir}/build" + + # os.mkdir(f"{workload_dir}/build", exist_ok=True) + subprocess.run(f"rm -rf {build_dir} && mkdir -p {build_dir}", shell=True) + + command = f"cd {build_dir} && cmake -G Ninja .. && ninja -j{os.cpu_count()}" + context.logger.info( + "Executing workload command", {"command": command, "cwd": build_dir} + ) + result = stream_run_logger( + cmd=command, + logger=context.logger, + cwd=workload_dir, + executable="bash", + stdout_prefix="workload build", + stderr_prefix="workload build", + ) + + # ================================================================================== + # Return simulation result + # ================================================================================== + # This is the end of run workflow, status no longer set to processing + success_result, failure_result = await check_result( + context, result.returncode, continue_run=False + ) + + # ================================================================================== + # finish workflow + # ================================================================================== + return diff --git a/bbdev/api/steps/workload/README.md b/bbdev/api/steps/workload/README.md new file mode 100644 index 000000000..a6bb39414 --- /dev/null +++ b/bbdev/api/steps/workload/README.md @@ -0,0 +1,39 @@ +# Workload Workflow + +Workload build workflow in Buckyball framework, used to build test workloads and benchmark programs. + +## API Usage + +### `build` +**Endpoint**: `POST /workload/build` + +**Function**: Build workload + +**Parameters**: +- **`workload`** - Specify workload name to build + +**Examples**: +```bash +# Build specific workload +bbdev workload --build "--workload test_program" + +# Build all workloads +bbdev workload --build +``` + +**Response**: +```json +{ + "status": 200, + "body": { + "success": true, + "processing": false, + "return_code": 0 + } +} +``` + +## Notes + +- Workload source code located in `bb-tests/workload` directory +- Build results typically output to `bb-tests/workloads/build` directory diff --git a/bbdev/api/utils/__init__.py b/bbdev/api/utils/__init__.py new file mode 100644 index 000000000..6db5263f8 --- /dev/null +++ b/bbdev/api/utils/__init__.py @@ -0,0 +1,10 @@ +from .stream_run import stream_run +from .path import get_buckyball_path +from .port import find_available_port +from .search_workload import ( + search_workload, + search_workload_all, + search_workload_pattern, +) + +__all__ = ["stream_run", "get_buckyball_path", "find_available_port", "search_workload"] diff --git a/bbdev/api/utils/event_common.py b/bbdev/api/utils/event_common.py new file mode 100644 index 000000000..717969227 --- /dev/null +++ b/bbdev/api/utils/event_common.py @@ -0,0 +1,111 @@ +""" +Common utility functions for all event steps. +""" + + +async def check_result(context, returncode, continue_run=False, extra_fields=None): + """ + Check returncode, create appropriate result objects and set state. + + Args: + context: The event context object + returncode: The return code (int) + continue_run: If True, set processing state instead of success/failure + extra_fields: Optional dictionary of extra fields to include in result body + + Returns: + tuple: (success_result, failure_result) - one will be None based on returncode and continue_run + """ + extra_fields = extra_fields or {} + + if continue_run: + await context.state.set(context.trace_id, "processing", True) + return None, None + elif returncode != 0: + failure_result = { + "status": 500, + "body": { + "success": False, + "failure": True, + "processing": False, + "returncode": returncode, + **extra_fields, + }, + } + await context.state.set(context.trace_id, "failure", failure_result) + return None, failure_result + else: + success_result = { + "status": 200, + "body": { + "success": True, + "failure": False, + "processing": False, + "returncode": returncode, + **extra_fields, + }, + } + await context.state.set(context.trace_id, "success", success_result) + return success_result, None + + +# ================================================================================== +# API waits for event return result +# +# Expected return result format: +# { +# "status": 200/400/500, +# "body": { +# "success": true/false, +# "failure": true/false, +# "processing": true/false, +# "return_code": 0, +# other fields +# } +# } +# +# Since the Motia framework wraps data in the data field, it needs to be unpacked +# if isinstance(result, dict) and 'data' in result: +# return result['data'] +# return result +# ================================================================================== + + +async def wait_for_result(context): + """ + Check for task completion state (success or failure). + Returns result if found, None if still processing. + + Args: + context: The event context object + + Returns: + dict or None: The result data if task completed, None if still processing + """ + # Check for success result + success_result = await context.state.get(context.trace_id, "success") + if success_result and success_result.get("data"): + # Filter out invalid null state + if success_result == {"data": None} or ( + isinstance(success_result, dict) + and success_result.get("data") is None + and len(success_result) == 1 + ): + await context.state.delete(context.trace_id, "success") + return None + context.logger.info("task completed") + + if isinstance(success_result, dict) and "data" in success_result: + return success_result["data"] + return success_result + + # Check for error status + failure_result = await context.state.get(context.trace_id, "failure") + if failure_result and failure_result.get("data"): + context.logger.error("task failed", failure_result) + + if isinstance(failure_result, dict) and "data" in failure_result: + return failure_result["data"] + return failure_result + + return None diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py new file mode 100644 index 000000000..69320c09e --- /dev/null +++ b/bbdev/api/utils/path.py @@ -0,0 +1,6 @@ +import os + + +def get_buckyball_path(): + current_dir = os.path.dirname(__file__) + return os.path.dirname(os.path.dirname(current_dir)) diff --git a/bbdev/api/utils/port.py b/bbdev/api/utils/port.py new file mode 100644 index 000000000..82e6bfeda --- /dev/null +++ b/bbdev/api/utils/port.py @@ -0,0 +1,16 @@ +import socket + + +def find_available_port(start_port: int = 5000, end_port: int = 5500) -> int: + """Find an available port in the specified range""" + for port in range(start_port, end_port + 1): + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("localhost", port)) + return port + except OSError: + # Port is already in use, try next one + continue + + # If no port is available in the range, raise an exception + raise RuntimeError(f"No available port found in range {start_port}-{end_port}") diff --git a/bbdev/api/utils/search_workload.py b/bbdev/api/utils/search_workload.py new file mode 100644 index 000000000..0d4121f7d --- /dev/null +++ b/bbdev/api/utils/search_workload.py @@ -0,0 +1,72 @@ +import os +from typing import Optional, List + + +def search_workload(search_dir: str, filename: str) -> Optional[str]: + """ + Recursively search for a specified filename in the directory and its subdirectories + + Args: + search_dir: Root directory to search + filename: Filename to search for + + Returns: + Absolute path of the found file, or None if not found + """ + if not os.path.exists(search_dir): + return None + + for root, dirs, files in os.walk(search_dir): + if filename in files: + return os.path.abspath(os.path.join(root, filename)) + + return None + + +def search_workload_all(search_dir: str, filename: str) -> List[str]: + """ + Recursively search for a specified filename in the directory and its subdirectories, returning all matches + + Args: + search_dir: Root directory to search + filename: Filename to search for + + Returns: + List of absolute paths of all found files + """ + results = [] + + if not os.path.exists(search_dir): + return results + + for root, dirs, files in os.walk(search_dir): + if filename in files: + results.append(os.path.abspath(os.path.join(root, filename))) + + return results + + +def search_workload_pattern(search_dir: str, pattern: str) -> List[str]: + """ + Recursively search for files matching a specified pattern in the directory and its subdirectories + + Args: + search_dir: Root directory to search + pattern: Filename pattern (supports wildcards * and ?) + + Returns: + List of absolute paths of all matching files + """ + import fnmatch + + results = [] + + if not os.path.exists(search_dir): + return results + + for root, dirs, files in os.walk(search_dir): + for file in files: + if fnmatch.fnmatch(file, pattern): + results.append(os.path.abspath(os.path.join(root, file))) + + return results diff --git a/bbdev/api/utils/stream_run.py b/bbdev/api/utils/stream_run.py new file mode 100644 index 000000000..6bd6dcb14 --- /dev/null +++ b/bbdev/api/utils/stream_run.py @@ -0,0 +1,172 @@ +import subprocess +import threading +from typing import Optional, List, Callable + + +class StreamResult: + """Result object mimicking subprocess.CompletedProcess""" + + def __init__(self, returncode: int, stdout: str, stderr: str): + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + +def stream_run( + cmd: str, + cwd: Optional[str] = None, + shell: bool = True, + executable: Optional[str] = None, + timeout: Optional[float] = None, + on_stdout: Optional[Callable[[str], None]] = None, + on_stderr: Optional[Callable[[str], None]] = None, + stdout_prefix: str = "STDOUT", + stderr_prefix: str = "STDERR", +) -> StreamResult: + """ + Execute command and stream output in real-time + + Args: + cmd: Command to execute + cwd: Working directory + shell: Whether to execute using shell + timeout: Timeout in seconds + on_stdout: Callback function for stdout lines + on_stderr: Callback function for stderr lines + stdout_prefix: Prefix for stdout output + stderr_prefix: Prefix for stderr output + + Returns: + StreamResult: Result object containing returncode, stdout, stderr + + Example: + def log_stdout(line): + logger.info(f'[STDOUT] {line}') + + def log_stderr(line): + logger.info(f'[STDERR] {line}') + + result = stream_run( + "make build", + cwd="/path/to/project", + on_stdout=log_stdout, + on_stderr=log_stderr + ) + """ + + def read_stream( + stream, output_list: List[str], callback: Optional[Callable], prefix: str + ): + """Thread function to read stream output""" + try: + for line in iter(stream.readline, ""): + if line: + line = line.rstrip() + output_list.append(line) + if callback: + callback(line) + finally: + stream.close() + + # Start process + process = subprocess.Popen( + cmd, + cwd=cwd, + shell=shell, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + executable=executable, + ) + + stdout_lines = [] + stderr_lines = [] + + # Create threads to read stdout and stderr + stdout_thread = threading.Thread( + target=read_stream, + args=(process.stdout, stdout_lines, on_stdout, stdout_prefix), + ) + stderr_thread = threading.Thread( + target=read_stream, + args=(process.stderr, stderr_lines, on_stderr, stderr_prefix), + ) + + stdout_thread.start() + stderr_thread.start() + + try: + # Wait for process to finish (with timeout) + process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + # Kill process on timeout + process.kill() + process.wait() + + # Wait for threads to finish + stdout_thread.join() + stderr_thread.join() + + return StreamResult( + returncode=process.returncode, + stdout="\n".join(stdout_lines), + stderr="\n".join(stderr_lines), + ) + + +def stream_run_logger( + cmd: str, + logger, + cwd: Optional[str] = None, + shell: bool = True, + executable: Optional[str] = None, + timeout: Optional[float] = None, + stdout_prefix: str = "STDOUT", + stderr_prefix: str = "STDERR", + verbose: bool = False, +) -> StreamResult: + """ + Convenience function for streaming output using logger + + Args: + cmd: Command to execute + logger: Logger instance + cwd: Working directory + shell: Whether to execute using shell + timeout: Timeout in seconds + stdout_prefix: Prefix for stdout output + stderr_prefix: Prefix for stderr output + verbose: Whether to use verbose output mode (verbose mode uses logger with timestamp, non-verbose prints directly) + + Returns: + StreamResult: Result object containing returncode, stdout, stderr + """ + + def log_stdout(line): + if verbose: + # Verbose mode: use logger.info, includes timestamp and task ID + logger.info(f"[{stdout_prefix}] {line}") + else: + # Non-verbose mode: print directly, use green for STDOUT + print(f"\033[32m[{stdout_prefix}]\033[0m {line}") + + def log_stderr(line): + if verbose: + # Verbose mode: use logger.info, includes timestamp and task ID + logger.info(f"[{stderr_prefix}] {line}") + else: + # Non-verbose mode: print directly, use red for STDERR + print(f"\033[31m[{stderr_prefix}]\033[0m {line}") + + return stream_run( + cmd=cmd, + cwd=cwd, + shell=shell, + executable=executable, + timeout=timeout, + on_stdout=log_stdout, + on_stderr=log_stderr, + stdout_prefix=stdout_prefix, + stderr_prefix=stderr_prefix, + ) diff --git a/bbdev/bbdev b/bbdev/bbdev new file mode 100755 index 000000000..832f58f2d --- /dev/null +++ b/bbdev/bbdev @@ -0,0 +1,533 @@ +#! /usr/bin/env python3 + +import sys +import argparse +import subprocess +import os +import shlex +import json +import time +import requests + +from utils import find_available_port + +workflow_dir = os.path.dirname(os.path.abspath(__file__)) + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="bbdev", + description="Development tool for buckyball project", + formatter_class=argparse.RawTextHelpFormatter, + ) + + # Global parameters + parser.add_argument( + "--port", type=int, default=None, help="Port for dev server (default: None)" + ) + parser.add_argument("--server", action="store_true", help="server mode") + + # Subcommand parser + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + # ===== start subcommand ============================================================================== + start_parser = subparsers.add_parser("start", help="Start dev server") + + # ===== stop subcommand ============================================================================== + stop_parser = subparsers.add_parser("stop", help="Stop dev server") + stop_group = stop_parser.add_mutually_exclusive_group(required=False) + stop_group.add_argument("--all", action="store_true", help="Stop all servers") + + # ===== verilator subcommand ========================================================================= + verilator_parser = subparsers.add_parser("verilator", help="Verilator operations") + # Mutually exclusive option group for verilator - only one operation can be selected + verilator_group = verilator_parser.add_mutually_exclusive_group(required=True) + verilator_group.add_argument( + "--clean", action="store_true", help="Clean verilator build directory" + ) + verilator_group.add_argument( + "--verilog", + type=str, + nargs="?", + const="", + metavar="ARGS", + help='Generate verilog files from chisel. Args: "[--balltype ] [--job ] [--batch]"', + ) + verilator_group.add_argument( + "--build", + type=str, + nargs="?", + const="", + metavar="ARGS", + help='Build verilator simulation executable. Args: "[--job ]"', + ) + verilator_group.add_argument( + "--sim", + type=str, + nargs="?", + const="", + metavar="ARGS", + help='Run verilator simulation. Args: "--binary [--batch]"', + ) + verilator_group.add_argument( + "--run", + type=str, + nargs="?", + const="", + metavar="ARGS", + help='Integrated build+sim+run. Args: "--binary [--batch] [--job ]"', + ) + verilator_group.add_argument( + "--cosim", + type=str, + nargs="?", + const="", + metavar="ARGS", + help='Run verilator cosimulation. Args: "--binary [--batch]"', + ) + + # ===== vcs subcommand ========================================================================= + vcs_parser = subparsers.add_parser("vcs", help="vcs operations") + # Mutually exclusive option group for vcs - only one operation can be selected + vcs_group = vcs_parser.add_mutually_exclusive_group(required=True) + vcs_group.add_argument( + "--clean", action="store_true", help="Clean vcs build directory" + ) + vcs_group.add_argument( + "--verilog", action="store_true", help="Generate verilog files" + ) + vcs_group.add_argument( + "--build", + type=str, + nargs="?", + const="", + metavar="ARGS", + help='Build vcs simulation executable. Args: "[--job ]"', + ) + vcs_group.add_argument( + "--sim", + type=str, + nargs="?", + const="", + metavar="ARGS", + help='Run vcs simulation. Args: "--binary [--batch]"', + ) + vcs_group.add_argument( + "--run", + type=str, + nargs="?", + const="", + metavar="ARGS", + help='Integrated build+sim+run. Args: "--binary [--batch] [--job ]"', + ) + + # ===== sardine subcommand ========================================================================= + sardine_parser = subparsers.add_parser("sardine", help="sardine operations") + sardine_group = sardine_parser.add_mutually_exclusive_group(required=True) + sardine_group.add_argument( + "--run", + type=str, + nargs="?", + const="", + metavar="ARGS", + help='Run sardine. Args: "--workload "', + ) + + # ===== agent subcommand ========================================================================= + agent_parser = subparsers.add_parser("agent", help="agent operations") + agent_group = agent_parser.add_mutually_exclusive_group(required=True) + agent_group.add_argument( + "--chat", + type=str, + nargs="?", + const="", + metavar="ARGS", + help='Run agent. Args: "--message ", "--model "', + ) + + # ===== workload subcommand ========================================================================= + workload_parser = subparsers.add_parser("workload", help="workload operations") + workload_group = workload_parser.add_mutually_exclusive_group(required=True) + workload_group.add_argument( + "--build", + type=str, + nargs="?", + const="", + metavar="ARGS", + help="Build workload. ", + ) + + # ===== doc subcommand ========================================================================= + doc_parser = subparsers.add_parser("doc", help="doc operations") + doc_group = doc_parser.add_mutually_exclusive_group(required=True) + doc_group.add_argument("--deploy", action="store_true", help="Deploy doc. ") + + # ===== marshal subcommand ========================================================================= + marshal_parser = subparsers.add_parser("marshal", help="marshal operations") + marshal_group = marshal_parser.add_mutually_exclusive_group(required=True) + marshal_group.add_argument( + "--build", type=str, nargs="?", const="", metavar="ARGS", help="Build marshal. " + ) + marshal_group.add_argument( + "--launch", + type=str, + nargs="?", + const="", + metavar="ARGS", + help="Launch marshal. ", + ) + + # ===== firesim subcommand ========================================================================= + firesim_parser = subparsers.add_parser("firesim", help="firesim operations") + firesim_group = firesim_parser.add_mutually_exclusive_group(required=True) + firesim_group.add_argument( + "--buildbitstream", + type=str, + nargs="?", + const="", + metavar="ARGS", + help="Build bitstream. ", + ) + firesim_group.add_argument( + "--infrasetup", + type=str, + nargs="?", + const="", + metavar="ARGS", + help="Infrasetup. ", + ) + firesim_group.add_argument( + "--runworkload", + type=str, + nargs="?", + const="", + metavar="ARGS", + help="Run workload. ", + ) + + # ===== compiler subcommand ========================================================================= + compiler_parser = subparsers.add_parser("compiler", help="compiler operations") + compiler_group = compiler_parser.add_mutually_exclusive_group(required=True) + compiler_group.add_argument( + "--build", + type=str, + nargs="?", + const="", + metavar="ARGS", + help="Build compiler. ", + ) + + # ===== funcsim subcommand ========================================================================= + funcsim_parser = subparsers.add_parser("funcsim", help="funcsim operations") + funcsim_group = funcsim_parser.add_mutually_exclusive_group(required=True) + funcsim_group.add_argument( + "--build", type=str, nargs="?", const="", metavar="ARGS", help="Build funcsim. " + ) + funcsim_group.add_argument( + "--sim", type=str, nargs="?", const="", metavar="ARGS", help="Sim funcsim. " + ) + + # ===== uvm subcommand ========================================================================= + uvm_parser = subparsers.add_parser("uvm", help="uvm operations") + uvm_group = uvm_parser.add_mutually_exclusive_group(required=True) + uvm_group.add_argument( + "--builddut", type=str, nargs="?", const="", metavar="ARGS", help="Build dut. " + ) + uvm_group.add_argument( + "--build", type=str, nargs="?", const="", metavar="ARGS", help="Build uvm. " + ) + + # ===== palladium subcommand =================================================================== + palladium_parser = subparsers.add_parser("palladium", help="palladium operations") + palladium_group = palladium_parser.add_mutually_exclusive_group(required=True) + palladium_group.add_argument( + "--verilog", + type=str, + nargs="?", + const="", + metavar="ARGS", + help='Generate verilog files from chisel. Args example: "[--config sims.palladium.BuckyballToyP2EConfig]"', + ) + + # Parse arguments, allowing unknown arguments (so --port can work in subcommands) + args, unknown = parser.parse_known_args(argv) + + # If --port and --server are in unknown arguments, handle them manually + while unknown: + found = False + for i in range(len(unknown)): + if unknown[i] == "--port": + try: + args.port = int(unknown[i + 1]) + unknown = unknown[i + 2 :] # Remove processed arguments + found = True + break + except (ValueError, IndexError): + break + if unknown[i] == "--server": + args.server = True + unknown = unknown[i + 1 :] # Remove processed arguments + found = True + break + if not found: + break + # If there are other unknown arguments, throw error + if unknown: + parser.error(f"unrecognized arguments: {' '.join(unknown)}") + + return args + + +def extract_command_info(args): + """Generic command info extractor, applicable to all commands""" + # Basic return structure + result = {"command": getattr(args, "command", None), "operation": None, "args": {}} + + # Dynamically extract operation type (iterate through all attributes of args) + for attr_name in dir(args): + # Skip built-in attributes and known non-operation attributes + if attr_name.startswith("_") or attr_name in ["command", "port"]: + continue + + attr_value = getattr(args, attr_name) + # Find attributes with value True or non-empty string (skip None and False) + if attr_value is not None and attr_value is not False: + result["operation"] = attr_name + if attr_value is True: + # Boolean operations, such as --clean, --verilog + result["args"] = {} + else: + # Operations with arguments, such as --sim "arg_string" + # Parse argument string + args_dict = {} + if attr_value: + try: + # use shlex to parse args + arg_tokens = shlex.split(attr_value) + i = 0 + while i < len(arg_tokens): + token = arg_tokens[i] + if token.startswith("--"): + # long option + option_name = token[2:] # remove -- + if i + 1 < len(arg_tokens) and not arg_tokens[ + i + 1 + ].startswith("-"): + # next token is value + args_dict[option_name] = arg_tokens[i + 1] + i += 2 + else: + # boolean flag + args_dict[option_name] = True + i += 1 + elif token.startswith("-") and len(token) == 2: + # short option + option_name = token[1:] # remove - + if i + 1 < len(arg_tokens) and not arg_tokens[ + i + 1 + ].startswith("-"): + # next token is value + args_dict[option_name] = arg_tokens[i + 1] + i += 2 + else: + # boolean flag + args_dict[option_name] = True + i += 1 + else: + # position argument, skip + i += 1 + except ValueError as e: + print(f"Error parsing arguments: {e}") + result["args"] = args_dict + break + + return result + + +if __name__ == "__main__": + args = parse_args(sys.argv[1:]) + cmd_info = extract_command_info(args) + + # print(f"Command: {cmd_info['command']}") + # print(f"Operation: {cmd_info['operation']}") + # print(f"Arguments: {cmd_info['args']}") + + # ================================================================================== + # Two modes: server mode and script mode + # + # server mode: Manually start/stop server, can visualize and access workflows through browser + # script mode: Automatically assign port, start service when task begins, stop service when task ends + # ================================================================================== + if args.server: + if cmd_info["command"] == "start": + print(f"Starting dev server on port {args.port}") + subprocess.run( + ["npx", "motia", "dev", "--port", str(args.port)], + cwd=workflow_dir, + check=True, + ) + + elif cmd_info["command"] == "stop": + if cmd_info["operation"] == "all": + print("Stopping all servers") + subprocess.run( + "kill -9 $(ps aux | grep '[m]otia' | awk '{print $2}')", + shell=True, + check=False, + text=True, + ) + else: + print(f"Stopping server on port {args.port}") + subprocess.run( + f"kill -TERM $(lsof -t -i :{args.port})", + shell=True, + check=False, + text=True, + ) + + elif cmd_info["command"] in [ + "verilator", + "vcs", + "sardine", + "agent", + "workload", + "doc", + "marshal", + "firesim", + "compiler", + "funcsim", + "uvm", + "palladium", + ]: + api_path = f"/{cmd_info['command']}/{cmd_info['operation']}" + json_data = json.dumps(cmd_info["args"]) + subprocess.run( + [ + "curl", + "-X", + "POST", + f"http://localhost:{args.port}{api_path}", + "-H", + "Content-Type: application/json", + "-d", + json_data, + ], + cwd=workflow_dir, + check=True, + ) + else: + print(f"Unknown command: {cmd_info['command']}") + print( + "Available commands: start, stop, verilator, vcs, sardine, agent, \ + workload, doc, marshal, firesim, compiler, funcsim, uvm" + ) + sys.exit(1) + else: # script mode + if cmd_info["command"] == "start": + print(" 'start' do nothing in script mode") + elif cmd_info["command"] == "stop": + print(" 'stop' do nothing in script mode") + elif cmd_info["command"] in [ + "verilator", + "vcs", + "sardine", + "agent", + "workload", + "doc", + "marshal", + "firesim", + "compiler", + "funcsim", + "uvm", + "palladium", + ]: + # 1. Start service in background ================================ + # If port is specified, use the specified port; otherwise, automatically assign port + if args.port: + available_port = args.port + else: + available_port = find_available_port(start_port=5100, end_port=5500) + print(f"Starting server on port {available_port}...") + proc = subprocess.Popen( + ["npx", "motia", "dev", "--port", str(available_port)], cwd=workflow_dir + ) + + # Wait for service to start + max_retries = 30 + for i in range(max_retries): + try: + # Disable proxy, connect directly to localhost + # response = requests.get(f"http://localhost:{available_port}", timeout=1) + response = requests.get( + f"http://localhost:{available_port}", + timeout=1, + proxies={"http": None, "https": None, "all": None}, + ) + if response.status_code == 200: + print(f"Server is ready on port {available_port}") + break + except requests.exceptions.RequestException: + pass + time.sleep(3) + else: + print("Server failed to start within 30 seconds") + subprocess.run( + f"kill -TERM $(lsof -t -i :{available_port})", + shell=True, + check=False, + text=True, + ) + proc.terminate() + proc.wait() + sys.exit(1) + + # 2. Execute API call ================================ + api_path = f"/{cmd_info['command']}/{cmd_info['operation']}" + json_data = json.dumps(cmd_info["args"]) + print(f"Executing {cmd_info['command']} {cmd_info['operation']}...") + # Disable proxy, connect directly to localhost + result = subprocess.run( + [ + "curl", + "-sS", + "--noproxy", + "localhost", + "-X", + "POST", + f"http://localhost:{available_port}{api_path}", + "-H", + "Content-Type: application/json", + "-d", + json_data, + ], + cwd=workflow_dir, + capture_output=True, + text=True, + ) + + # 3. Shutdown service ================================ + # Give observability plugin time to finish async operations (e.g., Redis writes) + time.sleep(1) + proc.terminate() + proc.wait() + print( + f"\nTask completed. Command running on http://localhost:{available_port} is finished" + ) + + # Check the success field returned by API + try: + response = json.loads(result.stdout) + if not response.get("success", False): + print("Error: Task failed") + sys.exit(1) + except Exception: + print("Error: Invalid API response") + sys.exit(1) + + else: + print(f"Unknown command: {cmd_info['command']}") + print( + "Available commands: start, stop, verilator, vcs, sardine, agent, \ + workload, doc, marshal, firesim, compiler, funcsim, uvm" + ) + sys.exit(1) diff --git a/bbdev/flake.lock b/bbdev/flake.lock new file mode 100644 index 000000000..3009a1201 --- /dev/null +++ b/bbdev/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1770841267, + "narHash": "sha256-9xejG0KoqsoKEGp2kVbXRlEYtFFcDTHjidiuX8hGO44=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ec7c70d12ce2fc37cb92aff673dcdca89d187bae", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/bbdev/flake.nix b/bbdev/flake.nix new file mode 100644 index 000000000..c9425fc09 --- /dev/null +++ b/bbdev/flake.nix @@ -0,0 +1,46 @@ +{ + description = "bbdev - Agent-friendly developer toolchain backend powered by Motia"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + overlays = [ (import ./nix/overlay.nix) ]; + pkgs = import nixpkgs { inherit system overlays; }; + in + { + devShells.default = pkgs.mkShell { + buildInputs = with pkgs; [ + # Node.js runtime (Motia is a Node.js framework) + nodejs_22 + nodePackages.pnpm + + # Python with project dependencies (managed by Nix, not requirements.txt) + bbdevPythonPkgs + + # System tools + curl + git + ]; + + shellHook = '' + export PATH="$PWD:$PATH" + + echo "bbdev dev environment ready" + echo " Node.js: $(node --version)" + echo " Python: $(python3 --version)" + echo " pnpm: $(pnpm --version)" + echo " bbdev: $(which bbdev)" + echo "" + echo "Quick start:" + echo " pnpm install # install node deps + motia install" + echo " pnpm dev # start motia dev server" + ''; + }; + } + ); +} diff --git a/bbdev/nix/overlay.nix b/bbdev/nix/overlay.nix new file mode 100644 index 000000000..4e73a5fcc --- /dev/null +++ b/bbdev/nix/overlay.nix @@ -0,0 +1,8 @@ +# Motia Python dependencies overlay +# Provides Python packages needed by the project +final: prev: { + bbdevPythonPkgs = final.python312.withPackages (ps: with ps; [ + pydantic + requests + ]); +} From a418d8d062ea7d3f6e3517cf7758af6f768c999a Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 13 Feb 2026 19:26:13 +0800 Subject: [PATCH 003/196] feat: Add bbdev CLI package and update overlays for Python dependencies --- bbdev/flake.nix | 4 +++ bbdev/nix/bbdev-install.nix | 50 +++++++++++++++++++++++++++++++++++++ bbdev/nix/overlay.nix | 5 ++-- 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 bbdev/nix/bbdev-install.nix diff --git a/bbdev/flake.nix b/bbdev/flake.nix index c9425fc09..8197e8a1c 100644 --- a/bbdev/flake.nix +++ b/bbdev/flake.nix @@ -13,6 +13,10 @@ pkgs = import nixpkgs { inherit system overlays; }; in { + # nix build + packages.bbdev = pkgs.bbdevCli; + packages.default = pkgs.bbdevCli; + devShells.default = pkgs.mkShell { buildInputs = with pkgs; [ # Node.js runtime (Motia is a Node.js framework) diff --git a/bbdev/nix/bbdev-install.nix b/bbdev/nix/bbdev-install.nix new file mode 100644 index 000000000..3f32ed058 --- /dev/null +++ b/bbdev/nix/bbdev-install.nix @@ -0,0 +1,50 @@ +{ lib, stdenv, makeWrapper, python312, nodejs_22, nodePackages }: + +let + pythonEnv = python312.withPackages (ps: with ps; [ + pydantic + requests + ]); +in +stdenv.mkDerivation { + pname = "bbdev"; + version = "0.1.0"; + + src = ./..; + + nativeBuildInputs = [ makeWrapper ]; + + dontBuild = true; + + installPhase = '' + runHook preInstall + + # Install the api directory (Motia backend + utils + steps) + mkdir -p $out/lib/bbdev + cp -r api/* $out/lib/bbdev/ + + # Install the bbdev CLI script + mkdir -p $out/bin + cp bbdev $out/bin/.bbdev-unwrapped + chmod +x $out/bin/.bbdev-unwrapped + + # Patch workflow_dir to use BBDEV_API_DIR env var if set, + # otherwise fall back to bbdev/api relative to CWD (project root) + substituteInPlace $out/bin/.bbdev-unwrapped \ + --replace 'workflow_dir = os.path.dirname(os.path.abspath(__file__))' \ + 'workflow_dir = os.environ.get("BBDEV_API_DIR", os.path.join(os.getcwd(), "bbdev", "api"))' + + # Wrap bbdev with Python env and Node.js in PATH + makeWrapper $out/bin/.bbdev-unwrapped $out/bin/bbdev \ + --prefix PATH : ${lib.makeBinPath [ pythonEnv nodejs_22 nodePackages.pnpm ]} \ + --prefix PYTHONPATH : $out/lib/bbdev + + runHook postInstall + ''; + + meta = with lib; { + description = "Agent-friendly developer toolchain backend powered by Motia"; + license = licenses.mit; + mainProgram = "bbdev"; + }; +} diff --git a/bbdev/nix/overlay.nix b/bbdev/nix/overlay.nix index 4e73a5fcc..d13e92504 100644 --- a/bbdev/nix/overlay.nix +++ b/bbdev/nix/overlay.nix @@ -1,8 +1,9 @@ -# Motia Python dependencies overlay -# Provides Python packages needed by the project +# bbdev overlay +# Provides bbdev CLI package and Python dependencies final: prev: { bbdevPythonPkgs = final.python312.withPackages (ps: with ps; [ pydantic requests ]); + bbdevCli = final.callPackage ./bbdev-install.nix { }; } From ac36fadff81c9b10810c68d08035c893fb38fbc8 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 13 Feb 2026 19:32:25 +0800 Subject: [PATCH 004/196] fix: Preserve symlinks during installation and handle broken links in bbdev package --- bbdev/nix/bbdev-install.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bbdev/nix/bbdev-install.nix b/bbdev/nix/bbdev-install.nix index 3f32ed058..bdbbfd28e 100644 --- a/bbdev/nix/bbdev-install.nix +++ b/bbdev/nix/bbdev-install.nix @@ -20,8 +20,10 @@ stdenv.mkDerivation { runHook preInstall # Install the api directory (Motia backend + utils + steps) + # Use --no-dereference to preserve symlinks, skip broken ones after copy mkdir -p $out/lib/bbdev - cp -r api/* $out/lib/bbdev/ + cp -r --no-dereference api/* $out/lib/bbdev/ + find $out/lib/bbdev -type l ! -exec test -e {} \; -delete # Install the bbdev CLI script mkdir -p $out/bin @@ -31,7 +33,7 @@ stdenv.mkDerivation { # Patch workflow_dir to use BBDEV_API_DIR env var if set, # otherwise fall back to bbdev/api relative to CWD (project root) substituteInPlace $out/bin/.bbdev-unwrapped \ - --replace 'workflow_dir = os.path.dirname(os.path.abspath(__file__))' \ + --replace-fail 'workflow_dir = os.path.dirname(os.path.abspath(__file__))' \ 'workflow_dir = os.environ.get("BBDEV_API_DIR", os.path.join(os.getcwd(), "bbdev", "api"))' # Wrap bbdev with Python env and Node.js in PATH From 9adbf213607320e9218b34b7de6ce9be49af3f64 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 13 Feb 2026 20:01:27 +0800 Subject: [PATCH 005/196] refactor: Update bbdev script to use pnpm instead of npx and adjust workflow directory path --- bbdev/bbdev | 6 ++--- bbdev/flake.nix | 4 --- bbdev/nix/bbdev-install.nix | 52 ------------------------------------- bbdev/nix/overlay.nix | 5 ++-- 4 files changed, 5 insertions(+), 62 deletions(-) delete mode 100644 bbdev/nix/bbdev-install.nix diff --git a/bbdev/bbdev b/bbdev/bbdev index 832f58f2d..1cf99c6d0 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -11,7 +11,7 @@ import requests from utils import find_available_port -workflow_dir = os.path.dirname(os.path.abspath(__file__)) +workflow_dir = os.path.dirname(os.path.abspath(__file__)) + "/api" def parse_args(argv: list[str]) -> argparse.Namespace: @@ -362,7 +362,7 @@ if __name__ == "__main__": if cmd_info["command"] == "start": print(f"Starting dev server on port {args.port}") subprocess.run( - ["npx", "motia", "dev", "--port", str(args.port)], + ["pnpm", "dev", "--port", str(args.port)], cwd=workflow_dir, check=True, ) @@ -449,7 +449,7 @@ if __name__ == "__main__": available_port = find_available_port(start_port=5100, end_port=5500) print(f"Starting server on port {available_port}...") proc = subprocess.Popen( - ["npx", "motia", "dev", "--port", str(available_port)], cwd=workflow_dir + ["pnpm", "dev", "--port", str(available_port)], cwd=workflow_dir ) # Wait for service to start diff --git a/bbdev/flake.nix b/bbdev/flake.nix index 8197e8a1c..c9425fc09 100644 --- a/bbdev/flake.nix +++ b/bbdev/flake.nix @@ -13,10 +13,6 @@ pkgs = import nixpkgs { inherit system overlays; }; in { - # nix build - packages.bbdev = pkgs.bbdevCli; - packages.default = pkgs.bbdevCli; - devShells.default = pkgs.mkShell { buildInputs = with pkgs; [ # Node.js runtime (Motia is a Node.js framework) diff --git a/bbdev/nix/bbdev-install.nix b/bbdev/nix/bbdev-install.nix deleted file mode 100644 index bdbbfd28e..000000000 --- a/bbdev/nix/bbdev-install.nix +++ /dev/null @@ -1,52 +0,0 @@ -{ lib, stdenv, makeWrapper, python312, nodejs_22, nodePackages }: - -let - pythonEnv = python312.withPackages (ps: with ps; [ - pydantic - requests - ]); -in -stdenv.mkDerivation { - pname = "bbdev"; - version = "0.1.0"; - - src = ./..; - - nativeBuildInputs = [ makeWrapper ]; - - dontBuild = true; - - installPhase = '' - runHook preInstall - - # Install the api directory (Motia backend + utils + steps) - # Use --no-dereference to preserve symlinks, skip broken ones after copy - mkdir -p $out/lib/bbdev - cp -r --no-dereference api/* $out/lib/bbdev/ - find $out/lib/bbdev -type l ! -exec test -e {} \; -delete - - # Install the bbdev CLI script - mkdir -p $out/bin - cp bbdev $out/bin/.bbdev-unwrapped - chmod +x $out/bin/.bbdev-unwrapped - - # Patch workflow_dir to use BBDEV_API_DIR env var if set, - # otherwise fall back to bbdev/api relative to CWD (project root) - substituteInPlace $out/bin/.bbdev-unwrapped \ - --replace-fail 'workflow_dir = os.path.dirname(os.path.abspath(__file__))' \ - 'workflow_dir = os.environ.get("BBDEV_API_DIR", os.path.join(os.getcwd(), "bbdev", "api"))' - - # Wrap bbdev with Python env and Node.js in PATH - makeWrapper $out/bin/.bbdev-unwrapped $out/bin/bbdev \ - --prefix PATH : ${lib.makeBinPath [ pythonEnv nodejs_22 nodePackages.pnpm ]} \ - --prefix PYTHONPATH : $out/lib/bbdev - - runHook postInstall - ''; - - meta = with lib; { - description = "Agent-friendly developer toolchain backend powered by Motia"; - license = licenses.mit; - mainProgram = "bbdev"; - }; -} diff --git a/bbdev/nix/overlay.nix b/bbdev/nix/overlay.nix index d13e92504..4e73a5fcc 100644 --- a/bbdev/nix/overlay.nix +++ b/bbdev/nix/overlay.nix @@ -1,9 +1,8 @@ -# bbdev overlay -# Provides bbdev CLI package and Python dependencies +# Motia Python dependencies overlay +# Provides Python packages needed by the project final: prev: { bbdevPythonPkgs = final.python312.withPackages (ps: with ps; [ pydantic requests ]); - bbdevCli = final.callPackage ./bbdev-install.nix { }; } From 85cbf51859a28746308523d0d76a5f9c3e80c326 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 14 Feb 2026 00:49:27 +0800 Subject: [PATCH 006/196] fix: fix bug when bbdev used in buckyball --- bbdev/api/motia.config.ts | 10 + bbdev/api/package.json | 17 +- bbdev/api/pnpm-lock.yaml | 1898 +++++++++++++---- bbdev/api/steps/sardine/01_run_event_step.py | 2 +- .../steps/verilator/03_build_event_step.py | 5 +- .../steps/verilator/04_cosim_event_step.py | 10 +- .../api/steps/verilator/04_sim_event_step.py | 10 +- bbdev/api/utils/path.py | 3 +- bbdev/bbdev | 52 +- bbdev/flake.nix | 5 +- 10 files changed, 1603 insertions(+), 409 deletions(-) create mode 100644 bbdev/api/motia.config.ts diff --git a/bbdev/api/motia.config.ts b/bbdev/api/motia.config.ts new file mode 100644 index 000000000..d4bdc0b0c --- /dev/null +++ b/bbdev/api/motia.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from '@motiadev/core' +import endpointPlugin from '@motiadev/plugin-endpoint/plugin' +import logsPlugin from '@motiadev/plugin-logs/plugin' +import observabilityPlugin from '@motiadev/plugin-observability/plugin' +import statesPlugin from '@motiadev/plugin-states/plugin' +import bullmqPlugin from '@motiadev/plugin-bullmq/plugin' + +export default defineConfig({ + plugins: [observabilityPlugin, statesPlugin, endpointPlugin, logsPlugin, bullmqPlugin], +}) diff --git a/bbdev/api/package.json b/bbdev/api/package.json index 4dfd49483..ab2d7fdb1 100644 --- a/bbdev/api/package.json +++ b/bbdev/api/package.json @@ -10,9 +10,20 @@ "start": "motia start", "clean": "rm -rf dist node_modules python_modules .motia .mermaid" }, - "keywords": ["motia"], + "keywords": [ + "motia" + ], "dependencies": { - "motia": "0.8.2-beta.139", - "zod": "^3.24.4" + "motia": "0.17.14-beta.196", + "zod": "^3.24.4", + "@motiadev/core": "0.17.14-beta.196", + "@motiadev/plugin-logs": "0.17.14-beta.196", + "@motiadev/plugin-states": "0.17.14-beta.196", + "@motiadev/plugin-endpoint": "0.17.14-beta.196", + "@motiadev/plugin-observability": "0.17.14-beta.196", + "@motiadev/plugin-bullmq": "0.17.14-beta.196", + "@motiadev/plugin-streams": "0.17.14-beta.196", + "@motiadev/plugin-cron": "0.17.14-beta.196", + "@motiadev/plugin-ws": "0.17.14-beta.196" } } diff --git a/bbdev/api/pnpm-lock.yaml b/bbdev/api/pnpm-lock.yaml index b1468f752..5bda0622c 100644 --- a/bbdev/api/pnpm-lock.yaml +++ b/bbdev/api/pnpm-lock.yaml @@ -8,9 +8,36 @@ importers: .: dependencies: + '@motiadev/core': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/plugin-bullmq': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/adapter-bullmq-events@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)))(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))) + '@motiadev/plugin-cron': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(use-sync-external-store@1.6.0(react@19.2.4)) + '@motiadev/plugin-endpoint': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(monaco-editor@0.55.1)(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))) + '@motiadev/plugin-logs': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))) + '@motiadev/plugin-observability': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))) + '@motiadev/plugin-states': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(monaco-editor@0.55.1)(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))) + '@motiadev/plugin-streams': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))) + '@motiadev/plugin-ws': + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(use-sync-external-store@1.6.0(react@19.2.4)) motia: - specifier: 0.8.2-beta.139 - version: 0.8.2-beta.139(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(jiti@2.6.1)(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.8.3)(use-sync-external-store@1.6.0(react@19.2.4)) + specifier: 0.17.14-beta.196 + version: 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)) zod: specifier: ^3.24.4 version: 3.25.76 @@ -127,156 +154,312 @@ packages: cpu: [ppc64] os: [aix] + '@esbuild/aix-ppc64@0.27.3': + resolution: {integrity: sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + '@esbuild/android-arm64@0.25.12': resolution: {integrity: sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==} engines: {node: '>=18'} cpu: [arm64] os: [android] + '@esbuild/android-arm64@0.27.3': + resolution: {integrity: sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + '@esbuild/android-arm@0.25.12': resolution: {integrity: sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==} engines: {node: '>=18'} cpu: [arm] os: [android] + '@esbuild/android-arm@0.27.3': + resolution: {integrity: sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + '@esbuild/android-x64@0.25.12': resolution: {integrity: sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==} engines: {node: '>=18'} cpu: [x64] os: [android] + '@esbuild/android-x64@0.27.3': + resolution: {integrity: sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + '@esbuild/darwin-arm64@0.25.12': resolution: {integrity: sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==} engines: {node: '>=18'} cpu: [arm64] os: [darwin] + '@esbuild/darwin-arm64@0.27.3': + resolution: {integrity: sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + '@esbuild/darwin-x64@0.25.12': resolution: {integrity: sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==} engines: {node: '>=18'} cpu: [x64] os: [darwin] + '@esbuild/darwin-x64@0.27.3': + resolution: {integrity: sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + '@esbuild/freebsd-arm64@0.25.12': resolution: {integrity: sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==} engines: {node: '>=18'} cpu: [arm64] os: [freebsd] + '@esbuild/freebsd-arm64@0.27.3': + resolution: {integrity: sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + '@esbuild/freebsd-x64@0.25.12': resolution: {integrity: sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==} engines: {node: '>=18'} cpu: [x64] os: [freebsd] + '@esbuild/freebsd-x64@0.27.3': + resolution: {integrity: sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + '@esbuild/linux-arm64@0.25.12': resolution: {integrity: sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==} engines: {node: '>=18'} cpu: [arm64] os: [linux] + '@esbuild/linux-arm64@0.27.3': + resolution: {integrity: sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + '@esbuild/linux-arm@0.25.12': resolution: {integrity: sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==} engines: {node: '>=18'} cpu: [arm] os: [linux] + '@esbuild/linux-arm@0.27.3': + resolution: {integrity: sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + '@esbuild/linux-ia32@0.25.12': resolution: {integrity: sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==} engines: {node: '>=18'} cpu: [ia32] os: [linux] + '@esbuild/linux-ia32@0.27.3': + resolution: {integrity: sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + '@esbuild/linux-loong64@0.25.12': resolution: {integrity: sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==} engines: {node: '>=18'} cpu: [loong64] os: [linux] + '@esbuild/linux-loong64@0.27.3': + resolution: {integrity: sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + '@esbuild/linux-mips64el@0.25.12': resolution: {integrity: sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==} engines: {node: '>=18'} cpu: [mips64el] os: [linux] + '@esbuild/linux-mips64el@0.27.3': + resolution: {integrity: sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + '@esbuild/linux-ppc64@0.25.12': resolution: {integrity: sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==} engines: {node: '>=18'} cpu: [ppc64] os: [linux] + '@esbuild/linux-ppc64@0.27.3': + resolution: {integrity: sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + '@esbuild/linux-riscv64@0.25.12': resolution: {integrity: sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==} engines: {node: '>=18'} cpu: [riscv64] os: [linux] + '@esbuild/linux-riscv64@0.27.3': + resolution: {integrity: sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + '@esbuild/linux-s390x@0.25.12': resolution: {integrity: sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==} engines: {node: '>=18'} cpu: [s390x] os: [linux] + '@esbuild/linux-s390x@0.27.3': + resolution: {integrity: sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + '@esbuild/linux-x64@0.25.12': resolution: {integrity: sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==} engines: {node: '>=18'} cpu: [x64] os: [linux] + '@esbuild/linux-x64@0.27.3': + resolution: {integrity: sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + '@esbuild/netbsd-arm64@0.25.12': resolution: {integrity: sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==} engines: {node: '>=18'} cpu: [arm64] os: [netbsd] + '@esbuild/netbsd-arm64@0.27.3': + resolution: {integrity: sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + '@esbuild/netbsd-x64@0.25.12': resolution: {integrity: sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==} engines: {node: '>=18'} cpu: [x64] os: [netbsd] + '@esbuild/netbsd-x64@0.27.3': + resolution: {integrity: sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + '@esbuild/openbsd-arm64@0.25.12': resolution: {integrity: sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==} engines: {node: '>=18'} cpu: [arm64] os: [openbsd] + '@esbuild/openbsd-arm64@0.27.3': + resolution: {integrity: sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + '@esbuild/openbsd-x64@0.25.12': resolution: {integrity: sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==} engines: {node: '>=18'} cpu: [x64] os: [openbsd] + '@esbuild/openbsd-x64@0.27.3': + resolution: {integrity: sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + '@esbuild/openharmony-arm64@0.25.12': resolution: {integrity: sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==} engines: {node: '>=18'} cpu: [arm64] os: [openharmony] + '@esbuild/openharmony-arm64@0.27.3': + resolution: {integrity: sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + '@esbuild/sunos-x64@0.25.12': resolution: {integrity: sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==} engines: {node: '>=18'} cpu: [x64] os: [sunos] + '@esbuild/sunos-x64@0.27.3': + resolution: {integrity: sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + '@esbuild/win32-arm64@0.25.12': resolution: {integrity: sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==} engines: {node: '>=18'} cpu: [arm64] os: [win32] + '@esbuild/win32-arm64@0.27.3': + resolution: {integrity: sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + '@esbuild/win32-ia32@0.25.12': resolution: {integrity: sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==} engines: {node: '>=18'} cpu: [ia32] os: [win32] + '@esbuild/win32-ia32@0.27.3': + resolution: {integrity: sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + '@esbuild/win32-x64@0.25.12': resolution: {integrity: sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==} engines: {node: '>=18'} cpu: [x64] os: [win32] + '@esbuild/win32-x64@0.27.3': + resolution: {integrity: sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@eslint-community/eslint-utils@4.9.1': resolution: {integrity: sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -355,6 +538,9 @@ packages: '@types/node': optional: true + '@ioredis/commands@1.5.0': + resolution: {integrity: sha512-eUgLqrMf8nJkZxT24JvVRrQya1vZkQh8BBeYNwGDqa5I0VUi8ACx7uFvAaLxintokpTenkK6DASvo/bvNbBGow==} + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -392,34 +578,162 @@ packages: react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 - '@motiadev/core@0.8.2-beta.139': - resolution: {integrity: sha512-ueIGi5PsYU7TmeG2GrIXTUTdS6D9pFUzgft1vOx0rkl9YRIKevsm5IDHapjg8ZchbSLhKJyYuCeROH8AOsIXow==} + '@motiadev/adapter-bullmq-events@0.17.14-beta.196': + resolution: {integrity: sha512-t9ZIRuMdbs2WiN656bXAic6KsFrT4M2vhelFEPw3RT72WHyqA2yn4PcsDtuKgX46oesEnaaCxGGty3/F0O6Cng==} + peerDependencies: + '@motiadev/core': '>=0.8.0' + + '@motiadev/adapter-redis-cron@0.17.14-beta.196': + resolution: {integrity: sha512-s+VAQ8K0/hmwWH7xSL+VoWLcJgCkDYBJd+iY5GzAjokThIzhPydrJNsSzmqfmxDNRe7cPsEmKHiF3rERUTY5MA==} + peerDependencies: + '@motiadev/core': '>=0.8.0' + + '@motiadev/adapter-redis-state@0.17.14-beta.196': + resolution: {integrity: sha512-KIceJaAiwnw7AJIvH6cQzctNuZK3rzI0qWL2gtPs3ZAB4TclUigkhIbKsyGyzWad4Vc0LF9t20lX9yrRaDVl4g==} + peerDependencies: + '@motiadev/core': '>=0.8.0' + + '@motiadev/adapter-redis-streams@0.17.14-beta.196': + resolution: {integrity: sha512-KON800F7+r+H79hIHYZAD8mqZ5WPU6HEZI8jLiuLUcZ7LCFaoA+3pbnep4SJWhDqK6AYFzgTW7/LAYd5qcaFcQ==} + peerDependencies: + '@motiadev/core': '>=0.8.0' + + '@motiadev/core@0.17.14-beta.196': + resolution: {integrity: sha512-/yFTlZWe/Sa11bWWV767RlH9dwnONx7nq8Q8bESTorzrCR2EQ6x8yt7LMrIebiTbDUOcGmS+rkw7aG37/5UXeQ==} + peerDependencies: + redis: ^5.0.0 + + '@motiadev/plugin-bullmq@0.17.14-beta.196': + resolution: {integrity: sha512-OFT4Qpiti2mvcO6WxSlK7bSoAU/6PUIhmksnT0fLnYLJuYsqSl1dYTCWxWCOJE3vQ8Ctj1TVxTFg6T7ah40oCw==} + peerDependencies: + '@motiadev/adapter-bullmq-events': 0.17.14-beta.196 + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/stream-client-react': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 + lucide-react: ^0.555.0 + motia: 0.17.14-beta.196 + react: ^19.2.3 + zustand: ^5.0.9 + + '@motiadev/plugin-cron@0.17.14-beta.196': + resolution: {integrity: sha512-RkgKDKAbeZ0GSh8qeR1InqPQpx85T+eSmdTDLgNrLUbyWN1PcMcXjjZUpmxhonaJkmUgopolrKIMjQ7Jn4Zl7w==} + peerDependencies: + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/stream-client-react': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 - '@motiadev/plugin-endpoint@0.8.2-beta.139': - resolution: {integrity: sha512-KKmo9hptNXhgQLbDhcyVKGuMETfowQ4Dp4utBQCfUHr/mLbZdO14lNqPpo5dEgc4VaXq0Ldc8YbgDhDorkAJ1w==} + '@motiadev/plugin-endpoint@0.17.14-beta.196': + resolution: {integrity: sha512-0s6UrAAM2SwZ42Qb0SidxiybAQREB3Cz8SRe4cwA1JOFS6XwBNe+VDY9bLEGz3emZcZcKkxaZvhQXkFBUpYvDA==} + peerDependencies: + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/stream-client-react': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 + lucide-react: ^0.555.0 + motia: 0.17.14-beta.196 + react: ^19.2.3 + zustand: ^5.0.9 + + '@motiadev/plugin-logs@0.17.14-beta.196': + resolution: {integrity: sha512-jnKyzz4/EScXUoxBz89Zg38lWTl1wMRvhmjJD6DY1zwZC64KBGTo6d/bGCmax83VCAm8qZUo/SlblK6IAiUNjQ==} + peerDependencies: + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/stream-client-react': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 + lucide-react: ^0.555.0 + motia: 0.17.14-beta.196 + react: ^19.2.3 + zustand: ^5.0.9 + + '@motiadev/plugin-observability@0.17.14-beta.196': + resolution: {integrity: sha512-L1vjDB+ZOjOx2fkoBRbv+4gVSD8kAndtRfOHn+3Lv4kcq61VsbMcRWvhdkr0aVMs2x/nvE2lnM66T07/1hVH/w==} + peerDependencies: + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/stream-client-react': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 + lucide-react: ^0.555.0 + motia: 0.17.14-beta.196 + react: ^19.2.3 + zustand: ^5.0.9 + + '@motiadev/plugin-states@0.17.14-beta.196': + resolution: {integrity: sha512-Ql55ZP/vjuf/9svbNmr9SH3ADhXr/ntDKcP4IwrZq+BudlOpjPxsnRKZYStSICYUWYSijnmUTlrZxll1Bczqjg==} + peerDependencies: + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 + lucide-react: ^0.555.0 + motia: 0.17.14-beta.196 + react: ^19.2.3 + zustand: ^5.0.9 + + '@motiadev/plugin-streams@0.17.14-beta.196': + resolution: {integrity: sha512-pTboHvnDMxUJ3k+xUB3yPqILEhujNytw3zUvxg7+T6V1ppE4r3QyTuusbM2hlTzxI03Y/f7y5KmpRKNnWao6Ow==} + peerDependencies: + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 + lucide-react: ^0.555.0 + motia: 0.17.14-beta.196 + react: ^18.0.0 || ^19.0.0 + zustand: ^5.0.8 + + '@motiadev/plugin-ws@0.17.14-beta.196': + resolution: {integrity: sha512-804J5pBjJCio4S7GO85wO5CntzfH92cWc794yJgwmfx56vRSPCHwqUzNhQ+LAZI7lMsGYX0jRB4wlwzAjZXhww==} + peerDependencies: + '@motiadev/core': 0.17.14-beta.196 + '@motiadev/ui': 0.17.14-beta.196 - '@motiadev/stream-client-browser@0.8.2-beta.139': - resolution: {integrity: sha512-FF4GsAWRd6QKcWAvALL3IQ5RDreTR1uqMCTJj81qUci+YTp8joiWo7a8spMdttOgRWUu3xI1QrKPO2PjkeAzvg==} + '@motiadev/stream-client-browser@0.17.14-beta.196': + resolution: {integrity: sha512-X2isevw/Jg7KALDa7MjWBQqWYvgyWdIqHgL4X1k5bX+nUDgeEYXk20H0sPpTRA5z37uXwoS9jqpFIFgIzpLm9A==} - '@motiadev/stream-client-node@0.8.2-beta.139': - resolution: {integrity: sha512-UBZOtv6jrXGcm1nH8JgTBB2wgfgh+hmVSm5YZqrDgwxTDMr7GxXFHa7j1AnGy7B+fc+xSZwRPmzewIxPYc59Tg==} + '@motiadev/stream-client-node@0.17.14-beta.196': + resolution: {integrity: sha512-Bx00AB9i+3R+hP/RIOVrgH9A5z62cnt3tlOiva0vYXP3g4HkHM/aP9YUL1VdQMqlnovK9WPAoP37+ZNFkmhPlg==} - '@motiadev/stream-client-react@0.8.2-beta.139': - resolution: {integrity: sha512-FaTMgLYB3LkVvXhcBGGDENJCI+1ezSfH4YSvUtkId9lqYs09Kwxk69w8F/SLjTNQvf/4csarr0LD4IGdveJ2dQ==} + '@motiadev/stream-client-react@0.17.14-beta.196': + resolution: {integrity: sha512-SJdAFRSgUE+vd9KyPYdW9isRIbya0xgV73eQ87Cbk+Hzfc3rW0jjH0YFHcFaArvBd0/sIus4j9+zp/oi3V8nxw==} peerDependencies: - react: ^19.1.0 + react: ^19.2.3 - '@motiadev/stream-client@0.8.2-beta.139': - resolution: {integrity: sha512-NVe0r4Jl+b9QD92FK8B4bpkvvjFauhTdLtcQyv/5g2CZ7azC2IG3Fz0vw6+qe28+37L7IrpTxRWnSTkULlyazA==} + '@motiadev/stream-client@0.17.14-beta.196': + resolution: {integrity: sha512-+P/Dpl6WndLi+OgW09hRNX2fLLfReh7EY9UQvbv9AMvpU/TxN2SkuJ4EBiX0CnMFquALs1MTtbi1HrlcVvZpgA==} - '@motiadev/ui@0.8.2-beta.139': - resolution: {integrity: sha512-Q5yFfehQhIerhoKwQrk8elyGSB9mBrTXgEhnOeRLE/p+qzeKPjaqRxF0O1L98AT0H++aFRwwdx1io9doIhxJWw==} + '@motiadev/ui@0.17.14-beta.196': + resolution: {integrity: sha512-C/9FD0gPtHcHRQIvsMrgKnQ3rVddD4l5Wxqu4iEPDZXh/LRSyBlHI8v3ghjD6obDXY9ujxOi4I9Z6vaIyB6hWQ==} peerDependencies: - react: ^19.1.0 - react-dom: ^19.1.0 + lucide-react: ^0.555.0 + react: ^19.2.3 + react-dom: ^19.2.3 + + '@motiadev/workbench@0.17.14-beta.196': + resolution: {integrity: sha512-gpvC4oUFyZZlpEhPkH5SSHTvqXBzTi1UwMS0lDIoSOEuNEJQQsV57ottRp7y+ziQLORpqf4IeMDsRoIn+hsz/A==} + + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==} + cpu: [arm64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + resolution: {integrity: sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==} + cpu: [x64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + resolution: {integrity: sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==} + cpu: [arm64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + resolution: {integrity: sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==} + cpu: [arm] + os: [linux] - '@motiadev/workbench@0.8.2-beta.139': - resolution: {integrity: sha512-8Ivo2aOVTPGQzx0bTg/3Qhpsrv+L4KK7B3cQmZPmkdoG88pH5ojLBql3sMZCmdTOMYFhD78puw3BMMuSb2Ev4g==} + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + resolution: {integrity: sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==} + cpu: [x64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + resolution: {integrity: sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==} + cpu: [x64] + os: [win32] '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} @@ -457,19 +771,6 @@ packages: '@types/react-dom': optional: true - '@radix-ui/react-collapsible@1.1.12': - resolution: {integrity: sha512-Uu+mSh4agx2ib1uIGPP4/CKNULyajb3p92LsVXmH2EHVMTfZWpll88XJ0j4W0z3f8NK1eYl1+Mf/szHPmcHzyA==} - peerDependencies: - '@types/react': '*' - '@types/react-dom': '*' - react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc - react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc - peerDependenciesMeta: - '@types/react': - optional: true - '@types/react-dom': - optional: true - '@radix-ui/react-collection@1.1.7': resolution: {integrity: sha512-Fh9rGN0MoI4ZFUNyfFVNU4y9LUz93u9/0K+yLgA2bwRojxM8JU1DyvvMBabnZPBgMWREAJvU2jjVzq+LrFUglw==} peerDependencies: @@ -501,19 +802,6 @@ packages: '@types/react': optional: true - '@radix-ui/react-dialog@1.1.15': - resolution: {integrity: sha512-TCglVRtzlffRNxRMEyR36DGBLJpeusFcgMVD9PZEzAKnUs1lKCgX5u9BmC2Yg+LL9MgZDugFFs1Vl+Jp4t/PGw==} - peerDependencies: - '@types/react': '*' - '@types/react-dom': '*' - react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc - react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc - peerDependenciesMeta: - '@types/react': - optional: true - '@types/react-dom': - optional: true - '@radix-ui/react-direction@1.1.1': resolution: {integrity: sha512-1UEWRX6jnOA2y4H5WczZ44gOOjTEmlqv1uNW4GAJEO5+bauCBhv8snY65Iw5/VOS/ghKN9gr2KjnLKxrsvoMVw==} peerDependencies: @@ -606,8 +894,8 @@ packages: '@types/react-dom': optional: true - '@radix-ui/react-navigation-menu@1.2.14': - resolution: {integrity: sha512-YB9mTFQvCOAQMHU+C/jVl96WmuWeltyUEpRJJky51huhds5W2FQr1J8D/16sQlf0ozxkPK8uF3niQMdUwZPv5w==} + '@radix-ui/react-popover@1.1.15': + resolution: {integrity: sha512-kr0X2+6Yy/vJzLYJUPCZEc8SfQcf+1COFoAqauJm74umQhta9M7lNJHP7QQS3vkvcGLQUbWpMzwrXYwrYztHKA==} peerDependencies: '@types/react': '*' '@types/react-dom': '*' @@ -697,19 +985,6 @@ packages: '@types/react-dom': optional: true - '@radix-ui/react-scroll-area@1.2.10': - resolution: {integrity: sha512-tAXIa1g3sM5CGpVT0uIbUx/U3Gs5N8T52IICuCtObaos1S8fzsrPXG5WObkQN3S6NVl6wKgPhAIiBGbWnvc97A==} - peerDependencies: - '@types/react': '*' - '@types/react-dom': '*' - react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc - react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc - peerDependenciesMeta: - '@types/react': - optional: true - '@types/react-dom': - optional: true - '@radix-ui/react-select@2.2.6': resolution: {integrity: sha512-I30RydO+bnn2PQztvo25tswPH+wFBjehVGtmagkU78yMdwTwVf12wnAOF+AeP8S2N8xD+5UPbGhkUfPyvT+mwQ==} peerDependencies: @@ -754,19 +1029,6 @@ packages: '@types/react': optional: true - '@radix-ui/react-switch@1.2.6': - resolution: {integrity: sha512-bByzr1+ep1zk4VubeEVViV592vu2lHE2BZY5OnzehZqOOgogN80+mNtCqPkhn2gklJqOpxWgPoYTSnhBCqpOXQ==} - peerDependencies: - '@types/react': '*' - '@types/react-dom': '*' - react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc - react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc - peerDependenciesMeta: - '@types/react': - optional: true - '@types/react-dom': - optional: true - '@radix-ui/react-tabs@1.1.13': resolution: {integrity: sha512-7xdcatg7/U+7+Udyoj2zodtI9H/IIopqo+YOIcZOq1nJwXWBZ9p8xiu5llXlekDbZkca79a/fozEYQXIA4sW6A==} peerDependencies: @@ -881,8 +1143,36 @@ packages: '@radix-ui/rect@1.1.1': resolution: {integrity: sha512-HPwpGIzkl28mWyZqG52jiqDJ12waP11Pa1lGoiyUkIEuMLBP0oeK/C89esbXrxsky5we7dfd8U58nm0SgAWpVw==} - '@rolldown/pluginutils@1.0.0-beta.27': - resolution: {integrity: sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==} + '@redis/bloom@5.10.0': + resolution: {integrity: sha512-doIF37ob+l47n0rkpRNgU8n4iacBlKM9xLiP1LtTZTvz8TloJB8qx/MgvhMhKdYG+CvCY2aPBnN2706izFn/4A==} + engines: {node: '>= 18'} + peerDependencies: + '@redis/client': ^5.10.0 + + '@redis/client@5.10.0': + resolution: {integrity: sha512-JXmM4XCoso6C75Mr3lhKA3eNxSzkYi3nCzxDIKY+YOszYsJjuKbFgVtguVPbLMOttN4iu2fXoc2BGhdnYhIOxA==} + engines: {node: '>= 18'} + + '@redis/json@5.10.0': + resolution: {integrity: sha512-B2G8XlOmTPUuZtD44EMGbtoepQG34RCDXLZbjrtON1Djet0t5Ri7/YPXvL9aomXqP8lLTreaprtyLKF4tmXEEA==} + engines: {node: '>= 18'} + peerDependencies: + '@redis/client': ^5.10.0 + + '@redis/search@5.10.0': + resolution: {integrity: sha512-3SVcPswoSfp2HnmWbAGUzlbUPn7fOohVu2weUQ0S+EMiQi8jwjL+aN2p6V3TI65eNfVsJ8vyPvqWklm6H6esmg==} + engines: {node: '>= 18'} + peerDependencies: + '@redis/client': ^5.10.0 + + '@redis/time-series@5.10.0': + resolution: {integrity: sha512-cPkpddXH5kc/SdRhF0YG0qtjL+noqFT0AcHbQ6axhsPsO7iqPi1cjxgdkE9TNeKiBUUdCaU1DbqkR/LzbzPBhg==} + engines: {node: '>= 18'} + peerDependencies: + '@redis/client': ^5.10.0 + + '@rolldown/pluginutils@1.0.0-rc.3': + resolution: {integrity: sha512-eybk3TjzzzV97Dlj5c+XrBFW57eTNhzod66y9HrBlzJ6NsCrWCp/2kaPS3K9wJmurBC0Tdw4yPjXKZqlznim3Q==} '@rollup/rollup-android-arm-eabi@4.57.1': resolution: {integrity: sha512-A6ehUVSiSaaliTxai040ZpZ2zTevHYbvu/lDoeAteHI8QnaosIzm4qwtezfRg1jOYaUmnzLX1AOD6Z+UJjtifg==} @@ -1009,6 +1299,9 @@ packages: cpu: [x64] os: [win32] + '@standard-schema/spec@1.1.0': + resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + '@tailwindcss/node@4.1.18': resolution: {integrity: sha512-DoR7U1P7iYhw16qJ49fgXUlry1t4CpXeErJHnQ44JgTSKMaZUdf17cfn5mHchfJ4KRBZRFA/Coo+MUF5+gOaCQ==} @@ -1097,6 +1390,23 @@ packages: '@tailwindcss/postcss@4.1.18': resolution: {integrity: sha512-Ce0GFnzAOuPyfV5SxjXGn0CubwGcuDB0zcdaPuCSzAa/2vII24JTkH+I6jcbXLb1ctjZMZZI6OjDaLPJQL1S0g==} + '@tanstack/query-core@5.90.20': + resolution: {integrity: sha512-OMD2HLpNouXEfZJWcKeVKUgQ5n+n3A2JFmBaScpNDUqSrQSjiveC7dKMe53uJUg1nDG16ttFPz2xfilz6i2uVg==} + + '@tanstack/react-query@5.90.21': + resolution: {integrity: sha512-0Lu6y5t+tvlTJMTO7oh5NSpJfpg/5D41LlThfepTixPYkJ0sE2Jj0m0f6yYqujBwIXlId87e234+MxG3D3g7kg==} + peerDependencies: + react: ^18 || ^19 + + '@tanstack/react-virtual@3.13.18': + resolution: {integrity: sha512-dZkhyfahpvlaV0rIKnvQiVoWPyURppl6w4m9IwMDpuIjcJ1sD9YGWrt0wISvgU7ewACXx2Ct46WPgI6qAD4v6A==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + '@tanstack/virtual-core@3.13.18': + resolution: {integrity: sha512-Mx86Hqu1k39icq2Zusq+Ey2J6dDWTjDvEv43PJtRCoEYTLyfaPnxIQ6iy7YAOK0NV/qOEmZQ/uCufrppZxTgcg==} + '@tsconfig/node10@1.0.12': resolution: {integrity: sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==} @@ -1142,8 +1452,8 @@ packages: '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} - '@types/hast@2.3.10': - resolution: {integrity: sha512-McWspRw8xx8J9HurkVBfYj0xKoE25tOFlHGdx4MJ5xORQrMGZNqJhVQWaIbm6Oyla5kYOXtDiopzKRJzEOkwJw==} + '@types/hast@3.0.4': + resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==} '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} @@ -1154,12 +1464,18 @@ packages: '@types/node@25.2.3': resolution: {integrity: sha512-m0jEgYlYz+mDJZ2+F4v8D1AyQb+QzsNqRuI7xg1VQX/KlKS0qT9r1Mo16yo5F/MtifXFgaofIFsdFMox2SxIbQ==} + '@types/prismjs@1.26.6': + resolution: {integrity: sha512-vqlvI7qlMvcCBbVe0AKAb4f97//Hy0EBTaiW8AalRnG/xAN5zOiWWyrNqNXeq8+KAuvRewjCVY1+IPxk4RdNYw==} + '@types/trusted-types@2.0.7': resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} + '@types/yauzl@2.10.3': + resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==} + '@types/zen-observable@0.8.3': resolution: {integrity: sha512-fbF6oTd4sGGy0xjHPKAt+eS2CrxJ3+6gQ3FGcBoIJR2TLAyCkCyI8JqZNy+FeON0AhVgNJoUumVoZQjBFUqHkw==} @@ -1222,9 +1538,9 @@ packages: resolution: {integrity: sha512-AxNRwEie8Nn4eFS1FzDMJWIISMGoXMb037sgCBJ3UR6o0fQTzr2tqN9WT+DkWJPhIdQCfV7T6D387566VtnCJA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@vitejs/plugin-react@4.7.0': - resolution: {integrity: sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA==} - engines: {node: ^14.18.0 || >=16.0.0} + '@vitejs/plugin-react@5.1.4': + resolution: {integrity: sha512-VIcFLdRi/VYRU8OL/puL7QXMYafHmqOnwTZY50U1JPlCNj30PxCMx65c494b1K9be9hX83KVt0+gTEwTWLqToA==} + engines: {node: ^20.19.0 || >=22.12.0} peerDependencies: vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 @@ -1259,6 +1575,10 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} @@ -1337,6 +1657,9 @@ packages: react-native-b4a: optional: true + babel-plugin-react-compiler@1.0.0: + resolution: {integrity: sha512-Ixm8tFfoKKIPYdCCKYTsqv+Fd4IJ0DQqMyEimo+pxUOMUR9cVPlwTrFt9Avu+3cb6Zp3mAzl+t1MrG2fxxKsxw==} + balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} @@ -1381,6 +1704,9 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true + buffer-crc32@0.2.13: + resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} + buffer-crc32@1.0.0: resolution: {integrity: sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==} engines: {node: '>=8.0.0'} @@ -1391,6 +1717,9 @@ packages: buffer@6.0.3: resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + bullmq@5.69.1: + resolution: {integrity: sha512-ct9cablOCuIHq4giGK32opyhftzeSDzdoQMLi/90xlBhtJmk0XoOO4FpU64egBD8KnWSCM/u4cjy3NA1J5SYYg==} + bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -1407,6 +1736,10 @@ packages: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} + camelcase@6.3.0: + resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==} + engines: {node: '>=10'} + caniuse-lite@1.0.30001769: resolution: {integrity: sha512-BCfFL1sHijQlBGWBMuJyhZUhzo7wer5sVj9hqekB/7xn0Ypy+pER/edCYQm4exbXj4WiySGp40P8UuTh6w1srg==} @@ -1414,14 +1747,14 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} - character-entities-legacy@1.1.4: - resolution: {integrity: sha512-3Xnr+7ZFS1uxeiUDvV02wQ+QDbc55o97tIV5zHScSPJpcLm/r0DFPcoY3tYRp+VZukxuMeKgXYmsXQHO05zQeA==} + character-entities-legacy@3.0.0: + resolution: {integrity: sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==} - character-entities@1.2.4: - resolution: {integrity: sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw==} + character-entities@2.0.2: + resolution: {integrity: sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==} - character-reference-invalid@1.1.4: - resolution: {integrity: sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg==} + character-reference-invalid@2.0.1: + resolution: {integrity: sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==} chardet@2.1.1: resolution: {integrity: sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==} @@ -1430,6 +1763,10 @@ packages: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} + chownr@2.0.0: + resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} + engines: {node: '>=10'} + class-variance-authority@0.7.1: resolution: {integrity: sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg==} @@ -1456,6 +1793,10 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -1463,21 +1804,20 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} - colors@1.4.0: - resolution: {integrity: sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==} - engines: {node: '>=0.1.90'} - combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} - comma-separated-tokens@1.0.8: - resolution: {integrity: sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==} + comma-separated-tokens@2.0.3: + resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} commander@13.1.0: resolution: {integrity: sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==} engines: {node: '>=18'} + commondir@1.0.1: + resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} + compress-commons@6.0.2: resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==} engines: {node: '>= 14'} @@ -1503,6 +1843,10 @@ packages: resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} engines: {node: '>= 0.6'} + cookie@1.1.1: + resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} + engines: {node: '>=18'} + copy-to-clipboard@3.3.3: resolution: {integrity: sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==} @@ -1521,6 +1865,10 @@ packages: create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} + cron-parser@4.9.0: + resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==} + engines: {node: '>=12.0.0'} + cron@4.4.0: resolution: {integrity: sha512-fkdfq+b+AHI4cKdhZlppHveI/mgz2qpiYxcm+t5E5TsxX7QrLS1VE0+7GENEk9z0EeGPcpSciGv6ez24duWhwQ==} engines: {node: '>=18.x'} @@ -1590,6 +1938,9 @@ packages: supports-color: optional: true + decode-named-character-reference@1.3.0: + resolution: {integrity: sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==} + deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} @@ -1600,6 +1951,10 @@ packages: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} + denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -1649,6 +2004,9 @@ packages: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} + end-of-stream@1.4.5: + resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + enhanced-resolve@5.19.0: resolution: {integrity: sha512-phv3E1Xl4tQOShqSte26C7Fl84EwUdZsyOuSSk9qtAGyyQs2s3jJzComh+Abf4g187lUUAvH+H26omrqia2aGg==} engines: {node: '>=10.13.0'} @@ -1674,6 +2032,11 @@ packages: engines: {node: '>=18'} hasBin: true + esbuild@0.27.3: + resolution: {integrity: sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==} + engines: {node: '>=18'} + hasBin: true + escalade@3.2.0: resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} engines: {node: '>=6'} @@ -1750,6 +2113,11 @@ packages: resolution: {integrity: sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==} engines: {node: '>= 0.10.0'} + extract-zip@2.0.1: + resolution: {integrity: sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==} + engines: {node: '>= 10.17.0'} + hasBin: true + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -1768,6 +2136,9 @@ packages: fault@1.0.4: resolution: {integrity: sha512-CJ0HCB5tL5fYTEA7ToAq5+kTwd++Borf1/bifxd9iT70QcXr4MRrO3Llf8Ifs70q+SJcGHFtnIE/Nw6giCtECA==} + fd-slicer@1.1.0: + resolution: {integrity: sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==} + fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -1789,6 +2160,17 @@ packages: resolution: {integrity: sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==} engines: {node: '>= 0.8'} + find-cache-dir@3.3.2: + resolution: {integrity: sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==} + engines: {node: '>=8'} + + find-package-json@1.2.0: + resolution: {integrity: sha512-+SOGcLGYDJHtyqHd87ysBhmaeQ95oWspDKnMXBrnQ9Eq4OkLNqejgoaD8xVWu6GPa0B6roa6KinCMEMcVeqONw==} + + find-up@4.1.0: + resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} + engines: {node: '>=8'} + find-up@5.0.0: resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} engines: {node: '>=10'} @@ -1828,10 +2210,28 @@ packages: fraction.js@5.3.4: resolution: {integrity: sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==} + framer-motion@12.34.0: + resolution: {integrity: sha512-+/H49owhzkzQyxtn7nZeF4kdH++I2FWrESQ184Zbcw5cEqNHYkE5yxWxcTLSj5lNx3NWdbIRy5FHqUvetD8FWg==} + peerDependencies: + '@emotion/is-prop-valid': '*' + react: ^18.0.0 || ^19.0.0 + react-dom: ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + '@emotion/is-prop-valid': + optional: true + react: + optional: true + react-dom: + optional: true + fresh@0.5.2: resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} engines: {node: '>= 0.6'} + fs-minipass@2.1.0: + resolution: {integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==} + engines: {node: '>= 8'} + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -1852,10 +2252,21 @@ packages: resolution: {integrity: sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==} engines: {node: '>=6'} + get-port@5.1.1: + resolution: {integrity: sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==} + engines: {node: '>=8'} + get-proto@1.0.1: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} + get-stream@5.2.0: + resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==} + engines: {node: '>=8'} + + get-tsconfig@4.13.6: + resolution: {integrity: sha512-shZT/QMiSHc/YBLxxOkMtgSid5HFoauqCE3/exfsEcwg1WkeqjG+V40yBbBrsD+jW2HDXcs28xOfcbm2jI8Ddw==} + glob-parent@6.0.2: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} @@ -1901,11 +2312,11 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} - hast-util-parse-selector@2.2.5: - resolution: {integrity: sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ==} + hast-util-parse-selector@4.0.0: + resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==} - hastscript@6.0.0: - resolution: {integrity: sha512-nDM6bvd7lIqDUiYEiu5Sl/+6ReP0BMk/2f4U/Rooccxkj0P5nm+acM5PrGJ/t5I8qPGiqZSE6hVAwZEdZIvP4w==} + hastscript@9.0.1: + resolution: {integrity: sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w==} highlight.js@10.7.3: resolution: {integrity: sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==} @@ -1917,6 +2328,10 @@ packages: resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} engines: {node: '>= 0.8'} + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + iconv-lite@0.4.24: resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} engines: {node: '>=0.10.0'} @@ -1951,18 +2366,26 @@ packages: resolution: {integrity: sha512-UjOaSel/iddGZJ5xP/Eixh6dY1XghiBw4XK13rCCIJcJfyhhoul/7KhLLUGtebEj6GDYM6Vnx/mVsjx2L/mFIA==} engines: {node: '>=12.0.0'} + ioredis@5.9.2: + resolution: {integrity: sha512-tAAg/72/VxOUW7RQSX1pIxJVucYKcjFjfvj60L57jrZpYCHC3XN0WCQ3sNYL4Gmvv+7GPvTAjc+KSdeNuE8oWQ==} + engines: {node: '>=12.22.0'} + + ioredis@5.9.3: + resolution: {integrity: sha512-VI5tMCdeoxZWU5vjHWsiE/Su76JGhBvWF1MJnV9ZtGltHk9BmD48oDq8Tj8haZ85aceXZMxLNDQZRVo5QKNgXA==} + engines: {node: '>=12.22.0'} + ipaddr.js@1.9.1: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} - is-alphabetical@1.0.4: - resolution: {integrity: sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg==} + is-alphabetical@2.0.1: + resolution: {integrity: sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==} - is-alphanumerical@1.0.4: - resolution: {integrity: sha512-UzoZUr+XfVz3t3v4KyGEniVL9BDRoQtY7tOyrRybkVNjDFWyo1yhXNGrrBTQxp3ib9BLAWs7k2YKBQsFRkZG9A==} + is-alphanumerical@2.0.1: + resolution: {integrity: sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==} - is-decimal@1.0.4: - resolution: {integrity: sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw==} + is-decimal@2.0.1: + resolution: {integrity: sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==} is-extglob@2.1.1: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} @@ -1976,8 +2399,8 @@ packages: resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} engines: {node: '>=0.10.0'} - is-hexadecimal@1.0.4: - resolution: {integrity: sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw==} + is-hexadecimal@2.0.1: + resolution: {integrity: sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==} is-interactive@1.0.0: resolution: {integrity: sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==} @@ -2121,10 +2544,26 @@ packages: resolution: {integrity: sha512-utfs7Pr5uJyyvDETitgsaqSyjCb2qNRAtuqUeWIAKztsOYdcACf2KtARYXg2pSvhkt+9NfoaNY7fxjl6nuMjIQ==} engines: {node: '>= 12.0.0'} + locate-path@5.0.0: + resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==} + engines: {node: '>=8'} + locate-path@6.0.0: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} + lockfile@1.0.4: + resolution: {integrity: sha512-cvbTwETRfsFh4nHsL1eGWapU1XFi5Ot9E85sWAwia7Y7EgB7vfqcZhTKZ+l7hCGxSPoushMv5GKhT5PdLv03WA==} + + lodash.defaults@4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + + lodash.defaultsdeep@4.6.1: + resolution: {integrity: sha512-3j8wdDzYuWO3lM3Reg03MuQR957t287Rpcxp1njpEa8oDrikb+FwGdW3n+FELh/A6qib6yPit0j/pv9G/yeAqA==} + + lodash.isarguments@3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + lodash.merge@4.6.2: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} @@ -2151,18 +2590,8 @@ packages: lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} - lucide-react@0.510.0: - resolution: {integrity: sha512-p8SQRAMVh7NhsAIETokSqDrc5CHnDLbV29mMnzaXx+Vc/hnqQzwI2r0FMWCcoTXnbw2KEjy48xwpGdEL+ck06Q==} - peerDependencies: - react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 - - lucide-react@0.525.0: - resolution: {integrity: sha512-Tm1txJ2OkymCGkvwoHt33Y2JpN5xucVq1slHcgE6Lk0WjDfjgKWor5CdVER8U6DvcfMwh4M8XxmpTiyzfmfDYQ==} - peerDependencies: - react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 - - lucide-react@0.544.0: - resolution: {integrity: sha512-t5tS44bqd825zAW45UQxpG2CvcC4urOwn2TrwSH8u+MjeE+1NnWl6QqeQ/6NdjMqdOygyiT9p3Ev0p1NJykxjw==} + lucide-react@0.555.0: + resolution: {integrity: sha512-D8FvHUGbxWBRQM90NZeIyhAvkFfsh3u9ekrMvJ30Z6gnpBHS6HC6ldLg7tL45hwiIz/u66eKDtdA23gwwGsAHA==} peerDependencies: react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 @@ -2173,6 +2602,10 @@ packages: magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + make-dir@3.1.0: + resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} + engines: {node: '>=8'} + make-error@1.3.6: resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} @@ -2231,23 +2664,67 @@ packages: minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + minipass@3.3.6: + resolution: {integrity: sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==} + engines: {node: '>=8'} + + minipass@5.0.0: + resolution: {integrity: sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==} + engines: {node: '>=8'} + minipass@7.1.2: resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} engines: {node: '>=16 || 14 >=14.17'} + minizlib@2.1.2: + resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==} + engines: {node: '>= 8'} + + mkdirp@1.0.4: + resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} + engines: {node: '>=10'} + hasBin: true + monaco-editor@0.55.1: resolution: {integrity: sha512-jz4x+TJNFHwHtwuV9vA9rMujcZRb0CEilTEwG2rRSpe/A7Jdkuj8xPKttCgOh+v/lkHy7HsZ64oj+q3xoAFl9A==} - motia@0.8.2-beta.139: - resolution: {integrity: sha512-s3p9DElWBruak4ngqo8GODbHR5yab1pqLYPtYybmPumi44k0E5doD80lp0KbOYgZqVDCL0at4fsMqZgUfINnQg==} + motia@0.17.14-beta.196: + resolution: {integrity: sha512-/5ynL2WG8klmHt2H01SAmg3g0M0e0CQvrVQ5QxXwuTdY/gj2pY0vai8UcKsSzs394WsF9dCzNrDK6mX/ErRVTQ==} hasBin: true + motion-dom@12.34.0: + resolution: {integrity: sha512-Lql3NuEcScRDxTAO6GgUsRHBZOWI/3fnMlkMcH5NftzcN37zJta+bpbMAV9px4Nj057TuvRooMK7QrzMCgtz6Q==} + + motion-utils@12.29.2: + resolution: {integrity: sha512-G3kc34H2cX2gI63RqU+cZq+zWRRPSsNIOjpdl9TN4AQwC4sgwYPl/Q/Obf/d53nOm569T0fYK+tcoSV50BWx8A==} + + motion@12.34.0: + resolution: {integrity: sha512-01Sfa/zgsD/di8zA/uFW5Eb7/SPXoGyUfy+uMRMW5Spa8j0z/UbfQewAYvPMYFCXRlyD6e5aLHh76TxeeJD+RA==} + peerDependencies: + '@emotion/is-prop-valid': '*' + react: ^18.0.0 || ^19.0.0 + react-dom: ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + '@emotion/is-prop-valid': + optional: true + react: + optional: true + react-dom: + optional: true + ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + msgpackr-extract@3.0.3: + resolution: {integrity: sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==} + hasBin: true + + msgpackr@1.11.5: + resolution: {integrity: sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==} + mute-stream@0.0.8: resolution: {integrity: sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==} @@ -2263,10 +2740,17 @@ packages: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} + node-abort-controller@3.1.1: + resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} + node-cron@3.0.3: resolution: {integrity: sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==} engines: {node: '>=6.0.0'} + node-gyp-build-optional-packages@5.2.2: + resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} + hasBin: true + node-releases@2.0.27: resolution: {integrity: sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==} @@ -2282,6 +2766,9 @@ packages: resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} engines: {node: '>= 0.8'} + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + onetime@5.1.2: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} engines: {node: '>=6'} @@ -2294,14 +2781,26 @@ packages: resolution: {integrity: sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==} engines: {node: '>=10'} + p-limit@2.3.0: + resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} + engines: {node: '>=6'} + p-limit@3.1.0: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} + p-locate@4.1.0: + resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} + engines: {node: '>=8'} + p-locate@5.0.0: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} + p-try@2.2.0: + resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} + engines: {node: '>=6'} + package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} @@ -2309,8 +2808,8 @@ packages: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} - parse-entities@2.0.0: - resolution: {integrity: sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==} + parse-entities@4.0.2: + resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==} parseurl@1.3.3: resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} @@ -2335,6 +2834,9 @@ packages: path-to-regexp@0.1.12: resolution: {integrity: sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==} + pend@1.2.0: + resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -2342,6 +2844,10 @@ packages: resolution: {integrity: sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==} engines: {node: '>=12'} + pkg-dir@4.2.0: + resolution: {integrity: sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==} + engines: {node: '>=8'} + postcss-value-parser@4.2.0: resolution: {integrity: sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==} @@ -2353,10 +2859,6 @@ packages: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} - prismjs@1.27.0: - resolution: {integrity: sha512-t13BGPUlFDR7wRB5kQDG4jjl7XeuH6jbJGt11JHPL96qwsEHNX2+68tFXqc1/k+/jALsbSWJKUOT/hcYAZ5LkA==} - engines: {node: '>=6'} - prismjs@1.30.0: resolution: {integrity: sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==} engines: {node: '>=6'} @@ -2368,8 +2870,8 @@ packages: resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} engines: {node: '>= 0.6.0'} - property-information@5.6.0: - resolution: {integrity: sha512-YUHSPk+A30YPv+0Qf8i9Mbfe/C0hdPXk1s1jPVToV8pk8BQtpw10ct89Eo7OWkutrwqvT0eicAxlOg3dOAu8JA==} + property-information@7.1.0: + resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==} proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} @@ -2378,6 +2880,9 @@ packages: proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + pump@3.0.3: + resolution: {integrity: sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==} + punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} @@ -2402,8 +2907,8 @@ packages: peerDependencies: react: ^19.2.4 - react-refresh@0.17.0: - resolution: {integrity: sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==} + react-refresh@0.18.0: + resolution: {integrity: sha512-QgT5//D3jfjJb6Gsjxv0Slpj23ip+HtOpnNgnb2S5zU3CB26G/IDPGoy4RJB42wzFE46DRsstbW6tKHoKbhAxw==} engines: {node: '>=0.10.0'} react-remove-scroll-bar@2.3.8: @@ -2432,6 +2937,23 @@ packages: react: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc react-dom: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + react-router-dom@7.13.0: + resolution: {integrity: sha512-5CO/l5Yahi2SKC6rGZ+HDEjpjkGaG/ncEP7eWFTvFxbHP8yeeI0PxTDjimtpXYlR3b3i9/WIL4VJttPrESIf2g==} + engines: {node: '>=20.0.0'} + peerDependencies: + react: '>=18' + react-dom: '>=18' + + react-router@7.13.0: + resolution: {integrity: sha512-PZgus8ETambRT17BUm/LL8lX3Of+oiLaPuVTRH3l1eLvSPpKO3AvhAEb5N7ihAFZQrYDqkvvWfFh9p0z9VsjLw==} + engines: {node: '>=20.0.0'} + peerDependencies: + react: '>=18' + react-dom: '>=18' + peerDependenciesMeta: + react-dom: + optional: true + react-style-singleton@2.2.3: resolution: {integrity: sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==} engines: {node: '>=10'} @@ -2442,8 +2964,9 @@ packages: '@types/react': optional: true - react-syntax-highlighter@15.6.6: - resolution: {integrity: sha512-DgXrc+AZF47+HvAPEmn7Ua/1p10jNoVZVI/LoPiYdtY+OM+/nG5yefLHKJwdKqY1adMuHFbeyBaG9j64ML7vTw==} + react-syntax-highlighter@16.1.0: + resolution: {integrity: sha512-E40/hBiP5rCNwkeBN1vRP+xow1X0pndinO+z3h7HLsHyjztbyjfzNWNKuAsJj+7DLam9iT4AaaOZnueCU+Nplg==} + engines: {node: '>= 16.20.2'} peerDependencies: react: '>= 0.14.0' @@ -2479,8 +3002,25 @@ packages: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} - refractor@3.6.0: - resolution: {integrity: sha512-MY9W41IOWxxk31o+YvFCNyNzdkc9M20NoZK5vq6jkv4I/uh2zkWcfudj0Q1fovjUQJrNewS9NMzeTtqPf+n5EA==} + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-memory-server@0.14.1: + resolution: {integrity: sha512-18aoXxvUrbBSAmZ42nC4yEEBRmGA4Hew11Zk64dpgtW6du71218GvBn0h6zDKIR/gOcZhkpqYoDjJ4zIw5nsuA==} + engines: {node: '>=16'} + hasBin: true + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + + redis@5.10.0: + resolution: {integrity: sha512-0/Y+7IEiTgVGPrLFKy8oAEArSyEJkU0zvgV5xyi9NzNQ+SLZmyFbUsWIbgPcd4UdUh00opXGKlXJwMmsis5Byw==} + engines: {node: '>= 18'} + + refractor@5.0.0: + resolution: {integrity: sha512-QXOrHQF5jOpjjLfiNk5GFnWhRXvxjUVnlFxkeDmewR5sXkr3iM46Zo+CnRR8B+MDVqkULW4EcLVcRBNOPXHosw==} require-from-string@2.0.2: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} @@ -2490,10 +3030,17 @@ packages: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} + resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + restore-cursor@3.1.0: resolution: {integrity: sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==} engines: {node: '>=8'} + rimraf@5.0.10: + resolution: {integrity: sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==} + hasBin: true + rollup@4.57.1: resolution: {integrity: sha512-oQL6lgK3e2QZeQ7gcgIkS2YZPg5slw37hYufJ3edKlfQSGGm8ICoxswK15ntSzF/a8+h7ekRy7k7oWc3BQ7y8A==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} @@ -2535,6 +3082,9 @@ packages: resolution: {integrity: sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==} engines: {node: '>= 0.8.0'} + set-cookie-parser@2.7.2: + resolution: {integrity: sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==} + setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} @@ -2577,8 +3127,11 @@ packages: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} - space-separated-tokens@1.1.5: - resolution: {integrity: sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA==} + space-separated-tokens@2.0.2: + resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} + + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} state-local@1.0.7: resolution: {integrity: sha512-HTEHMNieakEnoe33shBYcZ7NX83ACUjCu8c40iOGEZsngj9zRnkqS9j1pqQPXwobB0ZcVTk27REb7COQ0UR59w==} @@ -2628,6 +3181,9 @@ packages: resolution: {integrity: sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==} engines: {node: '>=10.0.0'} + tailwind-merge@2.6.1: + resolution: {integrity: sha512-Oo6tHdpZsGpkKG88HJ8RR1rg/RdnEkQEfMoEk2x1XRI3F1AxeU+ijRXpiVUF4UbLfcxxRGw6TbUINKYdWVsQTQ==} + tailwind-merge@3.4.0: resolution: {integrity: sha512-uSaO4gnW+b3Y2aWoWfFpX62vn2sR3skfhbjsEnaBI81WD1wBLlHZe5sWf0AqjksNdYTbGBEd0UasQMT3SNV15g==} @@ -2641,6 +3197,11 @@ packages: tar-stream@3.1.7: resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} + tar@6.2.1: + resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==} + engines: {node: '>=10'} + deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + text-decoder@1.2.4: resolution: {integrity: sha512-mzlffA3tBNhziEHPK5L5InZg1d/ElNIpJhnhbDRNUtem/edZcJ5zg5FgwKKKOyklxk+6Jt+TrSu83musmvrDlg==} @@ -2651,6 +3212,10 @@ packages: resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} engines: {node: '>=12.0.0'} + tmp@0.2.5: + resolution: {integrity: sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==} + engines: {node: '>=14.14'} + toggle-selection@1.0.6: resolution: {integrity: sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==} @@ -2685,6 +3250,11 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tsx@4.21.0: + resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} + engines: {node: '>=18.0.0'} + hasBin: true + tw-animate-css@1.4.0: resolution: {integrity: sha512-7bziOlRqH0hJx80h/3mbicLW7o8qLsH5+RaLR2t+OHM3D0JlWGODQKQ4cxbK7WlvmUxpcj6Kgu6EKqjrGFe3QQ==} @@ -2707,8 +3277,8 @@ packages: eslint: ^8.57.0 || ^9.0.0 typescript: '>=4.8.4 <6.0.0' - typescript@5.8.3: - resolution: {integrity: sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==} + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} hasBin: true @@ -2764,6 +3334,10 @@ packages: resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} hasBin: true + uuid@13.0.0: + resolution: {integrity: sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==} + hasBin: true + uuid@8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} hasBin: true @@ -2775,19 +3349,19 @@ packages: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} - vite@6.4.1: - resolution: {integrity: sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==} - engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + vite@7.3.1: + resolution: {integrity: sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==} + engines: {node: ^20.19.0 || >=22.12.0} hasBin: true peerDependencies: - '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 + '@types/node': ^20.19.0 || >=22.12.0 jiti: '>=1.21.0' - less: '*' + less: ^4.0.0 lightningcss: ^1.21.0 - sass: '*' - sass-embedded: '*' - stylus: '*' - sugarss: '*' + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 terser: ^5.16.0 tsx: ^4.8.1 yaml: ^2.4.2 @@ -2839,6 +3413,9 @@ packages: resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} engines: {node: '>=12'} + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + ws@8.19.0: resolution: {integrity: sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==} engines: {node: '>=10.0.0'} @@ -2851,13 +3428,15 @@ packages: utf-8-validate: optional: true - xtend@4.0.2: - resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} - engines: {node: '>=0.4'} - yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + + yauzl@2.10.0: + resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} + yn@3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} @@ -2876,14 +3455,12 @@ packages: resolution: {integrity: sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==} engines: {node: '>= 14'} - zod-to-json-schema@3.25.1: - resolution: {integrity: sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA==} - peerDependencies: - zod: ^3.25 || ^4 - zod@3.25.76: resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + zod@4.3.6: + resolution: {integrity: sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==} + zustand@4.5.7: resolution: {integrity: sha512-CHOUy7mu3lbD6o6LJLfllpjkzhHXSBlX8B9+qPddUsIfeF5S/UZ5q0kmCsnRqT1UHFQZchNFDDzMbQsuesHWlw==} engines: {node: '>=12.7.0'} @@ -3055,81 +3632,159 @@ snapshots: '@esbuild/aix-ppc64@0.25.12': optional: true + '@esbuild/aix-ppc64@0.27.3': + optional: true + '@esbuild/android-arm64@0.25.12': optional: true + '@esbuild/android-arm64@0.27.3': + optional: true + '@esbuild/android-arm@0.25.12': optional: true + '@esbuild/android-arm@0.27.3': + optional: true + '@esbuild/android-x64@0.25.12': optional: true + '@esbuild/android-x64@0.27.3': + optional: true + '@esbuild/darwin-arm64@0.25.12': optional: true + '@esbuild/darwin-arm64@0.27.3': + optional: true + '@esbuild/darwin-x64@0.25.12': optional: true + '@esbuild/darwin-x64@0.27.3': + optional: true + '@esbuild/freebsd-arm64@0.25.12': optional: true + '@esbuild/freebsd-arm64@0.27.3': + optional: true + '@esbuild/freebsd-x64@0.25.12': optional: true + '@esbuild/freebsd-x64@0.27.3': + optional: true + '@esbuild/linux-arm64@0.25.12': optional: true + '@esbuild/linux-arm64@0.27.3': + optional: true + '@esbuild/linux-arm@0.25.12': optional: true + '@esbuild/linux-arm@0.27.3': + optional: true + '@esbuild/linux-ia32@0.25.12': optional: true + '@esbuild/linux-ia32@0.27.3': + optional: true + '@esbuild/linux-loong64@0.25.12': optional: true + '@esbuild/linux-loong64@0.27.3': + optional: true + '@esbuild/linux-mips64el@0.25.12': optional: true + '@esbuild/linux-mips64el@0.27.3': + optional: true + '@esbuild/linux-ppc64@0.25.12': optional: true + '@esbuild/linux-ppc64@0.27.3': + optional: true + '@esbuild/linux-riscv64@0.25.12': optional: true + '@esbuild/linux-riscv64@0.27.3': + optional: true + '@esbuild/linux-s390x@0.25.12': optional: true + '@esbuild/linux-s390x@0.27.3': + optional: true + '@esbuild/linux-x64@0.25.12': optional: true + '@esbuild/linux-x64@0.27.3': + optional: true + '@esbuild/netbsd-arm64@0.25.12': optional: true + '@esbuild/netbsd-arm64@0.27.3': + optional: true + '@esbuild/netbsd-x64@0.25.12': optional: true + '@esbuild/netbsd-x64@0.27.3': + optional: true + '@esbuild/openbsd-arm64@0.25.12': optional: true + '@esbuild/openbsd-arm64@0.27.3': + optional: true + '@esbuild/openbsd-x64@0.25.12': optional: true + '@esbuild/openbsd-x64@0.27.3': + optional: true + '@esbuild/openharmony-arm64@0.25.12': optional: true + '@esbuild/openharmony-arm64@0.27.3': + optional: true + '@esbuild/sunos-x64@0.25.12': optional: true + '@esbuild/sunos-x64@0.27.3': + optional: true + '@esbuild/win32-arm64@0.25.12': optional: true + '@esbuild/win32-arm64@0.27.3': + optional: true + '@esbuild/win32-ia32@0.25.12': optional: true + '@esbuild/win32-ia32@0.27.3': + optional: true + '@esbuild/win32-x64@0.25.12': optional: true + '@esbuild/win32-x64@0.27.3': + optional: true + '@eslint-community/eslint-utils@4.9.1(eslint@9.39.2(jiti@2.6.1))': dependencies: eslint: 9.39.2(jiti@2.6.1) @@ -3211,6 +3866,8 @@ snapshots: optionalDependencies: '@types/node': 25.2.3 + '@ioredis/commands@1.5.0': {} + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -3257,20 +3914,49 @@ snapshots: react: 19.2.4 react-dom: 19.2.4(react@19.2.4) - '@motiadev/core@0.8.2-beta.139(@types/node@25.2.3)(typescript@5.8.3)': + '@motiadev/adapter-bullmq-events@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + bullmq: 5.69.1 + ioredis: 5.9.3 + uuid: 11.1.0 + transitivePeerDependencies: + - supports-color + + '@motiadev/adapter-redis-cron@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + redis: 5.10.0 + uuid: 11.1.0 + + '@motiadev/adapter-redis-state@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + redis: 5.10.0 + + '@motiadev/adapter-redis-streams@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + redis: 5.10.0 + + '@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)': dependencies: '@amplitude/analytics-node': 1.5.35 + '@standard-schema/spec': 1.1.0 + ajv: 8.17.1 body-parser: 1.20.4 - colors: 1.4.0 dotenv: 16.6.1 + esbuild: 0.25.12 express: 4.22.1 node-cron: 3.0.3 - ts-node: 10.9.2(@types/node@25.2.3)(typescript@5.8.3) + picocolors: 1.1.1 + redis: 5.10.0 + ts-node: 10.9.2(@types/node@25.2.3)(typescript@5.9.3) tsconfig-paths: 4.2.0 + tsx: 4.21.0 uuid: 11.1.0 ws: 8.19.0 - zod: 3.25.76 - zod-to-json-schema: 3.25.1(zod@3.25.76) + zod: 4.3.6 transitivePeerDependencies: - '@swc/core' - '@swc/wasm' @@ -3280,50 +3966,154 @@ snapshots: - typescript - utf-8-validate - '@motiadev/plugin-endpoint@0.8.2-beta.139(monaco-editor@0.55.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))': + '@motiadev/plugin-bullmq@0.17.14-beta.196(@motiadev/adapter-bullmq-events@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)))(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))': + dependencies: + '@motiadev/adapter-bullmq-events': 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)) + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/stream-client-react': 0.17.14-beta.196(react@19.2.4) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + '@tanstack/react-query': 5.90.21(react@19.2.4) + bullmq: 5.69.1 + date-fns: 4.1.0 + ioredis: 5.9.3 + lucide-react: 0.555.0(react@19.2.4) + motia: 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)) + react: 19.2.4 + react18-json-view: 0.2.9(react@19.2.4) + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - supports-color + + '@motiadev/plugin-cron@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(use-sync-external-store@1.6.0(react@19.2.4))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/stream-client-react': 0.17.14-beta.196(react@19.2.4) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + lucide-react: 0.555.0(react@19.2.4) + react: 19.2.4 + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - '@types/react' + - immer + - use-sync-external-store + + '@motiadev/plugin-endpoint@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(monaco-editor@0.55.1)(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))': dependencies: '@monaco-editor/react': 4.7.0(monaco-editor@0.55.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@motiadev/stream-client-react': 0.8.2-beta.139(react@19.2.4) - '@motiadev/ui': 0.8.2-beta.139(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/stream-client-react': 0.17.14-beta.196(react@19.2.4) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) clsx: 2.1.1 json-schema: 0.4.0 - lucide-react: 0.544.0(react@19.2.4) + lucide-react: 0.555.0(react@19.2.4) + motia: 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)) + react: 19.2.4 + react-syntax-highlighter: 16.1.0(react@19.2.4) react18-json-view: 0.2.9(react@19.2.4) tailwind-merge: 3.4.0 zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - monaco-editor + - react-dom + + '@motiadev/plugin-logs@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/stream-client-react': 0.17.14-beta.196(react@19.2.4) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + '@tanstack/react-virtual': 3.13.18(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + lucide-react: 0.555.0(react@19.2.4) + motia: 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)) + react: 19.2.4 + react18-json-view: 0.2.9(react@19.2.4) + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - react-dom + + '@motiadev/plugin-observability@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/stream-client-react': 0.17.14-beta.196(react@19.2.4) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + '@radix-ui/react-popover': 1.1.15(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@tanstack/react-virtual': 3.13.18(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + date-fns: 4.1.0 + lucide-react: 0.555.0(react@19.2.4) + motia: 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)) + react: 19.2.4 + react18-json-view: 0.2.9(react@19.2.4) + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) transitivePeerDependencies: - '@types/react' - '@types/react-dom' - - immer + - react-dom + + '@motiadev/plugin-states@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(monaco-editor@0.55.1)(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))': + dependencies: + '@monaco-editor/react': 4.7.0(monaco-editor@0.55.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + lucide-react: 0.555.0(react@19.2.4) + motia: 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)) + react: 19.2.4 + react18-json-view: 0.2.9(react@19.2.4) + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: - monaco-editor - - react - react-dom + + '@motiadev/plugin-streams@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(lucide-react@0.555.0(react@19.2.4))(motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(zustand@5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + clsx: 2.1.1 + lucide-react: 0.555.0(react@19.2.4) + motia: 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)) + motion: 12.34.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react: 19.2.4 + react18-json-view: 0.2.9(react@19.2.4) + tailwind-merge: 2.6.1 + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - '@emotion/is-prop-valid' + - react-dom + + '@motiadev/plugin-ws@0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3))(@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)))(use-sync-external-store@1.6.0(react@19.2.4))': + dependencies: + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + lucide-react: 0.555.0(react@19.2.4) + react: 19.2.4 + zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) + transitivePeerDependencies: + - '@types/react' + - immer - use-sync-external-store - '@motiadev/stream-client-browser@0.8.2-beta.139': + '@motiadev/stream-client-browser@0.17.14-beta.196': dependencies: - '@motiadev/stream-client': 0.8.2-beta.139 - uuid: 11.1.0 + '@motiadev/stream-client': 0.17.14-beta.196 + uuid: 13.0.0 - '@motiadev/stream-client-node@0.8.2-beta.139': + '@motiadev/stream-client-node@0.17.14-beta.196': dependencies: - '@motiadev/stream-client': 0.8.2-beta.139 - uuid: 11.1.0 + '@motiadev/stream-client': 0.17.14-beta.196 + uuid: 13.0.0 ws: 8.19.0 transitivePeerDependencies: - bufferutil - utf-8-validate - '@motiadev/stream-client-react@0.8.2-beta.139(react@19.2.4)': + '@motiadev/stream-client-react@0.17.14-beta.196(react@19.2.4)': dependencies: - '@motiadev/stream-client-browser': 0.8.2-beta.139 + '@motiadev/stream-client-browser': 0.17.14-beta.196 react: 19.2.4 - '@motiadev/stream-client@0.8.2-beta.139': + '@motiadev/stream-client@0.17.14-beta.196': dependencies: - uuid: 11.1.0 + uuid: 13.0.0 - '@motiadev/ui@0.8.2-beta.139(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))': + '@motiadev/ui@0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4))': dependencies: '@radix-ui/react-checkbox': 1.3.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-dropdown-menu': 2.1.16(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -3335,7 +4125,7 @@ snapshots: '@radix-ui/react-tooltip': 1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) class-variance-authority: 0.7.1 clsx: 2.1.1 - lucide-react: 0.525.0(react@19.2.4) + lucide-react: 0.555.0(react@19.2.4) react: 19.2.4 react-dom: 19.2.4(react@19.2.4) react-resizable-panels: 3.0.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -3348,58 +4138,52 @@ snapshots: - immer - use-sync-external-store - '@motiadev/workbench@0.8.2-beta.139(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(jiti@2.6.1)(lightningcss@1.30.2)(monaco-editor@0.55.1)(use-sync-external-store@1.6.0(react@19.2.4))': + '@motiadev/workbench@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(jiti@2.6.1)(lightningcss@1.30.2)(monaco-editor@0.55.1)(redis@5.10.0)(tsx@4.21.0)(use-sync-external-store@1.6.0(react@19.2.4))': dependencies: '@monaco-editor/react': 4.7.0(monaco-editor@0.55.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@motiadev/plugin-endpoint': 0.8.2-beta.139(monaco-editor@0.55.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) - '@motiadev/stream-client-react': 0.8.2-beta.139(react@19.2.4) - '@motiadev/ui': 0.8.2-beta.139(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) - '@radix-ui/react-collapsible': 1.1.12(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-dialog': 1.1.15(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-dropdown-menu': 2.1.16(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-label': 2.1.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-navigation-menu': 1.2.14(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-scroll-area': 1.2.10(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-select': 2.2.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-separator': 1.1.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-slot': 1.2.4(react@19.2.4) - '@radix-ui/react-switch': 1.2.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-tabs': 1.1.13(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/stream-client-react': 0.17.14-beta.196(react@19.2.4) + '@motiadev/ui': 0.17.14-beta.196(lucide-react@0.555.0(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) '@radix-ui/react-tooltip': 1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@tailwindcss/postcss': 4.1.18 - '@vitejs/plugin-react': 4.7.0(vite@6.4.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2)) + '@vitejs/plugin-react': 5.1.4(vite@7.3.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)) '@xyflow/react': 12.10.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) autoprefixer: 10.4.24(postcss@8.5.6) + babel-plugin-react-compiler: 1.0.0 class-variance-authority: 0.7.1 clsx: 2.1.1 dagre: 0.8.5 - date-fns: 4.1.0 fast-deep-equal: 3.1.3 json-schema: 0.4.0 - lucide-react: 0.510.0(react@19.2.4) + lucide-react: 0.555.0(react@19.2.4) postcss: 8.5.6 react: 19.2.4 react-dom: 19.2.4(react@19.2.4) - react-syntax-highlighter: 15.6.6(react@19.2.4) - react-use-resizable: 0.2.0(react@19.2.4) + react-router-dom: 7.13.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react-syntax-highlighter: 16.1.0(react@19.2.4) react18-json-view: 0.2.9(react@19.2.4) tailwind-merge: 3.4.0 tailwindcss: 4.1.18 tw-animate-css: 1.4.0 - typescript: 5.8.3 - typescript-eslint: 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) - vite: 6.4.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2) + typescript: 5.9.3 + typescript-eslint: 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + vite: 7.3.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0) + zod: 4.3.6 zustand: 5.0.11(react@19.2.4)(use-sync-external-store@1.6.0(react@19.2.4)) transitivePeerDependencies: + - '@swc/core' + - '@swc/wasm' - '@types/node' - '@types/react' - '@types/react-dom' + - bufferutil - eslint - immer - jiti - less - lightningcss - monaco-editor + - redis - sass - sass-embedded - stylus @@ -3408,8 +4192,27 @@ snapshots: - terser - tsx - use-sync-external-store + - utf-8-validate - yaml + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + optional: true + '@pkgjs/parseargs@0.11.0': optional: true @@ -3436,54 +4239,22 @@ snapshots: react: 19.2.4 react-dom: 19.2.4(react@19.2.4) - '@radix-ui/react-collapsible@1.1.12(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': - dependencies: - '@radix-ui/primitive': 1.1.3 - '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) - '@radix-ui/react-context': 1.1.2(react@19.2.4) - '@radix-ui/react-id': 1.1.1(react@19.2.4) - '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) - '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) - react: 19.2.4 - react-dom: 19.2.4(react@19.2.4) - - '@radix-ui/react-collection@1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': - dependencies: - '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) - '@radix-ui/react-context': 1.1.2(react@19.2.4) - '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-slot': 1.2.3(react@19.2.4) - react: 19.2.4 - react-dom: 19.2.4(react@19.2.4) - - '@radix-ui/react-compose-refs@1.1.2(react@19.2.4)': - dependencies: - react: 19.2.4 - - '@radix-ui/react-context@1.1.2(react@19.2.4)': - dependencies: - react: 19.2.4 - - '@radix-ui/react-dialog@1.1.15(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': - dependencies: - '@radix-ui/primitive': 1.1.3 - '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) - '@radix-ui/react-context': 1.1.2(react@19.2.4) - '@radix-ui/react-dismissable-layer': 1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-focus-guards': 1.1.3(react@19.2.4) - '@radix-ui/react-focus-scope': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-id': 1.1.1(react@19.2.4) - '@radix-ui/react-portal': 1.1.9(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-collection@1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) + '@radix-ui/react-context': 1.1.2(react@19.2.4) '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-slot': 1.2.3(react@19.2.4) - '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) - aria-hidden: 1.2.6 react: 19.2.4 react-dom: 19.2.4(react@19.2.4) - react-remove-scroll: 2.7.2(react@19.2.4) + + '@radix-ui/react-compose-refs@1.1.2(react@19.2.4)': + dependencies: + react: 19.2.4 + + '@radix-ui/react-context@1.1.2(react@19.2.4)': + dependencies: + react: 19.2.4 '@radix-ui/react-direction@1.1.1(react@19.2.4)': dependencies: @@ -3557,24 +4328,25 @@ snapshots: react-dom: 19.2.4(react@19.2.4) react-remove-scroll: 2.7.2(react@19.2.4) - '@radix-ui/react-navigation-menu@1.2.14(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + '@radix-ui/react-popover@1.1.15(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': dependencies: '@radix-ui/primitive': 1.1.3 - '@radix-ui/react-collection': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) '@radix-ui/react-context': 1.1.2(react@19.2.4) - '@radix-ui/react-direction': 1.1.1(react@19.2.4) '@radix-ui/react-dismissable-layer': 1.1.11(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-focus-guards': 1.1.3(react@19.2.4) + '@radix-ui/react-focus-scope': 1.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-id': 1.1.1(react@19.2.4) + '@radix-ui/react-popper': 1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-portal': 1.1.9(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(react@19.2.4) '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) - '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) - '@radix-ui/react-use-previous': 1.1.1(react@19.2.4) - '@radix-ui/react-visually-hidden': 1.2.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + aria-hidden: 1.2.6 react: 19.2.4 react-dom: 19.2.4(react@19.2.4) + react-remove-scroll: 2.7.2(react@19.2.4) '@radix-ui/react-popper@1.2.8(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': dependencies: @@ -3631,20 +4403,6 @@ snapshots: react: 19.2.4 react-dom: 19.2.4(react@19.2.4) - '@radix-ui/react-scroll-area@1.2.10(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': - dependencies: - '@radix-ui/number': 1.1.1 - '@radix-ui/primitive': 1.1.3 - '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) - '@radix-ui/react-context': 1.1.2(react@19.2.4) - '@radix-ui/react-direction': 1.1.1(react@19.2.4) - '@radix-ui/react-presence': 1.1.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-use-callback-ref': 1.1.1(react@19.2.4) - '@radix-ui/react-use-layout-effect': 1.1.1(react@19.2.4) - react: 19.2.4 - react-dom: 19.2.4(react@19.2.4) - '@radix-ui/react-select@2.2.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': dependencies: '@radix-ui/number': 1.1.1 @@ -3687,18 +4445,6 @@ snapshots: '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) react: 19.2.4 - '@radix-ui/react-switch@1.2.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': - dependencies: - '@radix-ui/primitive': 1.1.3 - '@radix-ui/react-compose-refs': 1.1.2(react@19.2.4) - '@radix-ui/react-context': 1.1.2(react@19.2.4) - '@radix-ui/react-primitive': 2.1.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-use-controllable-state': 1.2.2(react@19.2.4) - '@radix-ui/react-use-previous': 1.1.1(react@19.2.4) - '@radix-ui/react-use-size': 1.1.1(react@19.2.4) - react: 19.2.4 - react-dom: 19.2.4(react@19.2.4) - '@radix-ui/react-tabs@1.1.13(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': dependencies: '@radix-ui/primitive': 1.1.3 @@ -3775,7 +4521,27 @@ snapshots: '@radix-ui/rect@1.1.1': {} - '@rolldown/pluginutils@1.0.0-beta.27': {} + '@redis/bloom@5.10.0(@redis/client@5.10.0)': + dependencies: + '@redis/client': 5.10.0 + + '@redis/client@5.10.0': + dependencies: + cluster-key-slot: 1.1.2 + + '@redis/json@5.10.0(@redis/client@5.10.0)': + dependencies: + '@redis/client': 5.10.0 + + '@redis/search@5.10.0(@redis/client@5.10.0)': + dependencies: + '@redis/client': 5.10.0 + + '@redis/time-series@5.10.0(@redis/client@5.10.0)': + dependencies: + '@redis/client': 5.10.0 + + '@rolldown/pluginutils@1.0.0-rc.3': {} '@rollup/rollup-android-arm-eabi@4.57.1': optional: true @@ -3852,6 +4618,8 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.57.1': optional: true + '@standard-schema/spec@1.1.0': {} + '@tailwindcss/node@4.1.18': dependencies: '@jridgewell/remapping': 2.3.5 @@ -3921,6 +4689,21 @@ snapshots: postcss: 8.5.6 tailwindcss: 4.1.18 + '@tanstack/query-core@5.90.20': {} + + '@tanstack/react-query@5.90.21(react@19.2.4)': + dependencies: + '@tanstack/query-core': 5.90.20 + react: 19.2.4 + + '@tanstack/react-virtual@3.13.18(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@tanstack/virtual-core': 3.13.18 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + + '@tanstack/virtual-core@3.13.18': {} + '@tsconfig/node10@1.0.12': {} '@tsconfig/node12@1.0.11': {} @@ -3973,7 +4756,7 @@ snapshots: '@types/estree@1.0.8': {} - '@types/hast@2.3.10': + '@types/hast@3.0.4': dependencies: '@types/unist': 2.0.11 @@ -3985,47 +4768,54 @@ snapshots: dependencies: undici-types: 7.16.0 + '@types/prismjs@1.26.6': {} + '@types/trusted-types@2.0.7': optional: true '@types/unist@2.0.11': {} + '@types/yauzl@2.10.3': + dependencies: + '@types/node': 25.2.3 + optional: true + '@types/zen-observable@0.8.3': {} - '@typescript-eslint/eslint-plugin@8.55.0(@typescript-eslint/parser@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3))(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3)': + '@typescript-eslint/eslint-plugin@8.55.0(@typescript-eslint/parser@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': dependencies: '@eslint-community/regexpp': 4.12.2 - '@typescript-eslint/parser': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) + '@typescript-eslint/parser': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) '@typescript-eslint/scope-manager': 8.55.0 - '@typescript-eslint/type-utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) - '@typescript-eslint/utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) + '@typescript-eslint/type-utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) '@typescript-eslint/visitor-keys': 8.55.0 eslint: 9.39.2(jiti@2.6.1) ignore: 7.0.5 natural-compare: 1.4.0 - ts-api-utils: 2.4.0(typescript@5.8.3) - typescript: 5.8.3 + ts-api-utils: 2.4.0(typescript@5.9.3) + typescript: 5.9.3 transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3)': + '@typescript-eslint/parser@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': dependencies: '@typescript-eslint/scope-manager': 8.55.0 '@typescript-eslint/types': 8.55.0 - '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.8.3) + '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.9.3) '@typescript-eslint/visitor-keys': 8.55.0 debug: 4.4.3 eslint: 9.39.2(jiti@2.6.1) - typescript: 5.8.3 + typescript: 5.9.3 transitivePeerDependencies: - supports-color - '@typescript-eslint/project-service@8.55.0(typescript@5.8.3)': + '@typescript-eslint/project-service@8.55.0(typescript@5.9.3)': dependencies: - '@typescript-eslint/tsconfig-utils': 8.55.0(typescript@5.8.3) + '@typescript-eslint/tsconfig-utils': 8.55.0(typescript@5.9.3) '@typescript-eslint/types': 8.55.0 debug: 4.4.3 - typescript: 5.8.3 + typescript: 5.9.3 transitivePeerDependencies: - supports-color @@ -4034,47 +4824,47 @@ snapshots: '@typescript-eslint/types': 8.55.0 '@typescript-eslint/visitor-keys': 8.55.0 - '@typescript-eslint/tsconfig-utils@8.55.0(typescript@5.8.3)': + '@typescript-eslint/tsconfig-utils@8.55.0(typescript@5.9.3)': dependencies: - typescript: 5.8.3 + typescript: 5.9.3 - '@typescript-eslint/type-utils@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3)': + '@typescript-eslint/type-utils@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': dependencies: '@typescript-eslint/types': 8.55.0 - '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.8.3) - '@typescript-eslint/utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) + '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.9.3) + '@typescript-eslint/utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) debug: 4.4.3 eslint: 9.39.2(jiti@2.6.1) - ts-api-utils: 2.4.0(typescript@5.8.3) - typescript: 5.8.3 + ts-api-utils: 2.4.0(typescript@5.9.3) + typescript: 5.9.3 transitivePeerDependencies: - supports-color '@typescript-eslint/types@8.55.0': {} - '@typescript-eslint/typescript-estree@8.55.0(typescript@5.8.3)': + '@typescript-eslint/typescript-estree@8.55.0(typescript@5.9.3)': dependencies: - '@typescript-eslint/project-service': 8.55.0(typescript@5.8.3) - '@typescript-eslint/tsconfig-utils': 8.55.0(typescript@5.8.3) + '@typescript-eslint/project-service': 8.55.0(typescript@5.9.3) + '@typescript-eslint/tsconfig-utils': 8.55.0(typescript@5.9.3) '@typescript-eslint/types': 8.55.0 '@typescript-eslint/visitor-keys': 8.55.0 debug: 4.4.3 minimatch: 9.0.5 semver: 7.7.4 tinyglobby: 0.2.15 - ts-api-utils: 2.4.0(typescript@5.8.3) - typescript: 5.8.3 + ts-api-utils: 2.4.0(typescript@5.9.3) + typescript: 5.9.3 transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3)': + '@typescript-eslint/utils@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3)': dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.2(jiti@2.6.1)) '@typescript-eslint/scope-manager': 8.55.0 '@typescript-eslint/types': 8.55.0 - '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.8.3) + '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.9.3) eslint: 9.39.2(jiti@2.6.1) - typescript: 5.8.3 + typescript: 5.9.3 transitivePeerDependencies: - supports-color @@ -4083,15 +4873,15 @@ snapshots: '@typescript-eslint/types': 8.55.0 eslint-visitor-keys: 4.2.1 - '@vitejs/plugin-react@4.7.0(vite@6.4.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2))': + '@vitejs/plugin-react@5.1.4(vite@7.3.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0))': dependencies: '@babel/core': 7.29.0 '@babel/plugin-transform-react-jsx-self': 7.27.1(@babel/core@7.29.0) '@babel/plugin-transform-react-jsx-source': 7.27.1(@babel/core@7.29.0) - '@rolldown/pluginutils': 1.0.0-beta.27 + '@rolldown/pluginutils': 1.0.0-rc.3 '@types/babel__core': 7.20.5 - react-refresh: 0.17.0 - vite: 6.4.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2) + react-refresh: 0.18.0 + vite: 7.3.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0) transitivePeerDependencies: - supports-color @@ -4137,6 +4927,8 @@ snapshots: acorn@8.15.0: {} + agent-base@7.1.4: {} + ajv@6.12.6: dependencies: fast-deep-equal: 3.1.3 @@ -4225,6 +5017,10 @@ snapshots: b4a@1.7.4: {} + babel-plugin-react-compiler@1.0.0: + dependencies: + '@babel/types': 7.29.0 + balanced-match@1.0.2: {} balanced-match@4.0.2: @@ -4281,6 +5077,8 @@ snapshots: node-releases: 2.0.27 update-browserslist-db: 1.2.3(browserslist@4.28.1) + buffer-crc32@0.2.13: {} + buffer-crc32@1.0.0: {} buffer@5.7.1: @@ -4293,6 +5091,18 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 + bullmq@5.69.1: + dependencies: + cron-parser: 4.9.0 + ioredis: 5.9.2 + msgpackr: 1.11.5 + node-abort-controller: 3.1.1 + semver: 7.7.4 + tslib: 2.8.1 + uuid: 11.1.0 + transitivePeerDependencies: + - supports-color + bytes@3.1.2: {} call-bind-apply-helpers@1.0.2: @@ -4307,6 +5117,8 @@ snapshots: callsites@3.1.0: {} + camelcase@6.3.0: {} + caniuse-lite@1.0.30001769: {} chalk@4.1.2: @@ -4314,11 +5126,11 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 - character-entities-legacy@1.1.4: {} + character-entities-legacy@3.0.0: {} - character-entities@1.2.4: {} + character-entities@2.0.2: {} - character-reference-invalid@1.1.4: {} + character-reference-invalid@2.0.1: {} chardet@2.1.1: {} @@ -4326,6 +5138,8 @@ snapshots: dependencies: readdirp: 4.1.2 + chownr@2.0.0: {} + class-variance-authority@0.7.1: dependencies: clsx: 2.1.1 @@ -4344,22 +5158,24 @@ snapshots: clsx@2.1.1: {} + cluster-key-slot@1.1.2: {} + color-convert@2.0.1: dependencies: color-name: 1.1.4 color-name@1.1.4: {} - colors@1.4.0: {} - combined-stream@1.0.8: dependencies: delayed-stream: 1.0.0 - comma-separated-tokens@1.0.8: {} + comma-separated-tokens@2.0.3: {} commander@13.1.0: {} + commondir@1.0.1: {} + compress-commons@6.0.2: dependencies: crc-32: 1.2.2 @@ -4382,6 +5198,8 @@ snapshots: cookie@0.7.2: {} + cookie@1.1.1: {} + copy-to-clipboard@3.3.3: dependencies: toggle-selection: 1.0.6 @@ -4397,6 +5215,10 @@ snapshots: create-require@1.1.1: {} + cron-parser@4.9.0: + dependencies: + luxon: 3.7.2 + cron@4.4.0: dependencies: '@types/luxon': 3.7.1 @@ -4459,6 +5281,10 @@ snapshots: dependencies: ms: 2.1.3 + decode-named-character-reference@1.3.0: + dependencies: + character-entities: 2.0.2 + deep-is@0.1.4: {} defaults@1.0.4: @@ -4467,6 +5293,8 @@ snapshots: delayed-stream@1.0.0: {} + denque@2.1.0: {} + depd@2.0.0: {} destroy@1.2.0: {} @@ -4501,6 +5329,10 @@ snapshots: encodeurl@2.0.0: {} + end-of-stream@1.4.5: + dependencies: + once: 1.4.0 + enhanced-resolve@5.19.0: dependencies: graceful-fs: 4.2.11 @@ -4550,6 +5382,35 @@ snapshots: '@esbuild/win32-ia32': 0.25.12 '@esbuild/win32-x64': 0.25.12 + esbuild@0.27.3: + optionalDependencies: + '@esbuild/aix-ppc64': 0.27.3 + '@esbuild/android-arm': 0.27.3 + '@esbuild/android-arm64': 0.27.3 + '@esbuild/android-x64': 0.27.3 + '@esbuild/darwin-arm64': 0.27.3 + '@esbuild/darwin-x64': 0.27.3 + '@esbuild/freebsd-arm64': 0.27.3 + '@esbuild/freebsd-x64': 0.27.3 + '@esbuild/linux-arm': 0.27.3 + '@esbuild/linux-arm64': 0.27.3 + '@esbuild/linux-ia32': 0.27.3 + '@esbuild/linux-loong64': 0.27.3 + '@esbuild/linux-mips64el': 0.27.3 + '@esbuild/linux-ppc64': 0.27.3 + '@esbuild/linux-riscv64': 0.27.3 + '@esbuild/linux-s390x': 0.27.3 + '@esbuild/linux-x64': 0.27.3 + '@esbuild/netbsd-arm64': 0.27.3 + '@esbuild/netbsd-x64': 0.27.3 + '@esbuild/openbsd-arm64': 0.27.3 + '@esbuild/openbsd-x64': 0.27.3 + '@esbuild/openharmony-arm64': 0.27.3 + '@esbuild/sunos-x64': 0.27.3 + '@esbuild/win32-arm64': 0.27.3 + '@esbuild/win32-ia32': 0.27.3 + '@esbuild/win32-x64': 0.27.3 + escalade@3.2.0: {} escape-html@1.0.3: {} @@ -4674,6 +5535,16 @@ snapshots: transitivePeerDependencies: - supports-color + extract-zip@2.0.1: + dependencies: + debug: 4.4.3 + get-stream: 5.2.0 + yauzl: 2.10.0 + optionalDependencies: + '@types/yauzl': 2.10.3 + transitivePeerDependencies: + - supports-color + fast-deep-equal@3.1.3: {} fast-fifo@1.3.2: {} @@ -4688,6 +5559,10 @@ snapshots: dependencies: format: 0.2.2 + fd-slicer@1.1.0: + dependencies: + pend: 1.2.0 + fdir@6.5.0(picomatch@4.0.3): optionalDependencies: picomatch: 4.0.3 @@ -4712,6 +5587,19 @@ snapshots: transitivePeerDependencies: - supports-color + find-cache-dir@3.3.2: + dependencies: + commondir: 1.0.1 + make-dir: 3.1.0 + pkg-dir: 4.2.0 + + find-package-json@1.2.0: {} + + find-up@4.1.0: + dependencies: + locate-path: 5.0.0 + path-exists: 4.0.0 + find-up@5.0.0: dependencies: locate-path: 6.0.0 @@ -4745,8 +5633,21 @@ snapshots: fraction.js@5.3.4: {} + framer-motion@12.34.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + dependencies: + motion-dom: 12.34.0 + motion-utils: 12.29.2 + tslib: 2.8.1 + optionalDependencies: + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + fresh@0.5.2: {} + fs-minipass@2.1.0: + dependencies: + minipass: 3.3.6 + fsevents@2.3.3: optional: true @@ -4769,11 +5670,21 @@ snapshots: get-nonce@1.0.1: {} + get-port@5.1.1: {} + get-proto@1.0.1: dependencies: dunder-proto: 1.0.1 es-object-atoms: 1.1.1 + get-stream@5.2.0: + dependencies: + pump: 3.0.3 + + get-tsconfig@4.13.6: + dependencies: + resolve-pkg-maps: 1.0.0 + glob-parent@6.0.2: dependencies: is-glob: 4.0.3 @@ -4818,15 +5729,17 @@ snapshots: dependencies: function-bind: 1.1.2 - hast-util-parse-selector@2.2.5: {} + hast-util-parse-selector@4.0.0: + dependencies: + '@types/hast': 3.0.4 - hastscript@6.0.0: + hastscript@9.0.1: dependencies: - '@types/hast': 2.3.10 - comma-separated-tokens: 1.0.8 - hast-util-parse-selector: 2.2.5 - property-information: 5.6.0 - space-separated-tokens: 1.1.5 + '@types/hast': 3.0.4 + comma-separated-tokens: 2.0.3 + hast-util-parse-selector: 4.0.0 + property-information: 7.1.0 + space-separated-tokens: 2.0.2 highlight.js@10.7.3: {} @@ -4840,6 +5753,13 @@ snapshots: statuses: 2.0.2 toidentifier: 1.0.1 + https-proxy-agent@7.0.6: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + iconv-lite@0.4.24: dependencies: safer-buffer: 2.1.2 @@ -4883,16 +5803,44 @@ snapshots: transitivePeerDependencies: - '@types/node' + ioredis@5.9.2: + dependencies: + '@ioredis/commands': 1.5.0 + cluster-key-slot: 1.1.2 + debug: 4.4.3 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + + ioredis@5.9.3: + dependencies: + '@ioredis/commands': 1.5.0 + cluster-key-slot: 1.1.2 + debug: 4.4.3 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + ipaddr.js@1.9.1: {} - is-alphabetical@1.0.4: {} + is-alphabetical@2.0.1: {} - is-alphanumerical@1.0.4: + is-alphanumerical@2.0.1: dependencies: - is-alphabetical: 1.0.4 - is-decimal: 1.0.4 + is-alphabetical: 2.0.1 + is-decimal: 2.0.1 - is-decimal@1.0.4: {} + is-decimal@2.0.1: {} is-extglob@2.1.1: {} @@ -4902,7 +5850,7 @@ snapshots: dependencies: is-extglob: 2.1.1 - is-hexadecimal@1.0.4: {} + is-hexadecimal@2.0.1: {} is-interactive@1.0.0: {} @@ -5008,10 +5956,24 @@ snapshots: lightningcss-win32-arm64-msvc: 1.30.2 lightningcss-win32-x64-msvc: 1.30.2 + locate-path@5.0.0: + dependencies: + p-locate: 4.1.0 + locate-path@6.0.0: dependencies: p-locate: 5.0.0 + lockfile@1.0.4: + dependencies: + signal-exit: 3.0.7 + + lodash.defaults@4.2.0: {} + + lodash.defaultsdeep@4.6.1: {} + + lodash.isarguments@3.1.0: {} + lodash.merge@4.6.2: {} lodash.truncate@4.4.2: {} @@ -5036,15 +5998,7 @@ snapshots: dependencies: yallist: 3.1.1 - lucide-react@0.510.0(react@19.2.4): - dependencies: - react: 19.2.4 - - lucide-react@0.525.0(react@19.2.4): - dependencies: - react: 19.2.4 - - lucide-react@0.544.0(react@19.2.4): + lucide-react@0.555.0(react@19.2.4): dependencies: react: 19.2.4 @@ -5054,6 +6008,10 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 + make-dir@3.1.0: + dependencies: + semver: 6.3.1 + make-error@1.3.6: {} marked@14.0.0: {} @@ -5094,24 +6052,40 @@ snapshots: minimist@1.2.8: {} + minipass@3.3.6: + dependencies: + yallist: 4.0.0 + + minipass@5.0.0: {} + minipass@7.1.2: {} + minizlib@2.1.2: + dependencies: + minipass: 3.3.6 + yallist: 4.0.0 + + mkdirp@1.0.4: {} + monaco-editor@0.55.1: dependencies: dompurify: 3.2.7 marked: 14.0.0 - motia@0.8.2-beta.139(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(jiti@2.6.1)(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.8.3)(use-sync-external-store@1.6.0(react@19.2.4)): + motia@0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(lightningcss@1.30.2)(monaco-editor@0.55.1)(typescript@5.9.3)(use-sync-external-store@1.6.0(react@19.2.4)): dependencies: '@amplitude/analytics-node': 1.5.35 - '@motiadev/core': 0.8.2-beta.139(@types/node@25.2.3)(typescript@5.8.3) - '@motiadev/stream-client-node': 0.8.2-beta.139 - '@motiadev/workbench': 0.8.2-beta.139(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(jiti@2.6.1)(lightningcss@1.30.2)(monaco-editor@0.55.1)(use-sync-external-store@1.6.0(react@19.2.4)) + '@motiadev/adapter-bullmq-events': 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)) + '@motiadev/adapter-redis-cron': 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)) + '@motiadev/adapter-redis-state': 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)) + '@motiadev/adapter-redis-streams': 0.17.14-beta.196(@motiadev/core@0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3)) + '@motiadev/core': 0.17.14-beta.196(@types/node@25.2.3)(redis@5.10.0)(typescript@5.9.3) + '@motiadev/stream-client-node': 0.17.14-beta.196 + '@motiadev/workbench': 0.17.14-beta.196(@types/node@25.2.3)(eslint@9.39.2(jiti@2.6.1))(jiti@2.6.1)(lightningcss@1.30.2)(monaco-editor@0.55.1)(redis@5.10.0)(tsx@4.21.0)(use-sync-external-store@1.6.0(react@19.2.4)) antlr4ts: 0.5.0-alpha.4 archiver: 7.0.1 axios: 1.13.5 chokidar: 4.0.3 - colors: 1.4.0 commander: 13.1.0 cron: 4.4.0 dotenv: 16.6.1 @@ -5119,10 +6093,16 @@ snapshots: express: 4.22.1 glob: 11.1.0 inquirer: 8.2.7(@types/node@25.2.3) + jiti: 2.6.1 node-cron: 3.0.3 + picocolors: 1.1.1 python-ast: 0.1.0 + redis: 5.10.0 + redis-memory-server: 0.14.1 table: 6.9.0 - ts-node: 10.9.2(@types/node@25.2.3)(typescript@5.8.3) + ts-node: 10.9.2(@types/node@25.2.3)(typescript@5.9.3) + tsx: 4.21.0 + zod: 4.3.6 transitivePeerDependencies: - '@swc/core' - '@swc/wasm' @@ -5134,7 +6114,6 @@ snapshots: - debug - eslint - immer - - jiti - less - lightningcss - monaco-editor @@ -5145,16 +6124,45 @@ snapshots: - sugarss - supports-color - terser - - tsx - typescript - use-sync-external-store - utf-8-validate - yaml + motion-dom@12.34.0: + dependencies: + motion-utils: 12.29.2 + + motion-utils@12.29.2: {} + + motion@12.34.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + dependencies: + framer-motion: 12.34.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + tslib: 2.8.1 + optionalDependencies: + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + ms@2.0.0: {} ms@2.1.3: {} + msgpackr-extract@3.0.3: + dependencies: + node-gyp-build-optional-packages: 5.2.2 + optionalDependencies: + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.3 + optional: true + + msgpackr@1.11.5: + optionalDependencies: + msgpackr-extract: 3.0.3 + mute-stream@0.0.8: {} nanoid@3.3.11: {} @@ -5163,10 +6171,17 @@ snapshots: negotiator@0.6.3: {} + node-abort-controller@3.1.1: {} + node-cron@3.0.3: dependencies: uuid: 8.3.2 + node-gyp-build-optional-packages@5.2.2: + dependencies: + detect-libc: 2.1.2 + optional: true + node-releases@2.0.27: {} normalize-path@3.0.0: {} @@ -5177,6 +6192,10 @@ snapshots: dependencies: ee-first: 1.1.1 + once@1.4.0: + dependencies: + wrappy: 1.0.2 + onetime@5.1.2: dependencies: mimic-fn: 2.1.0 @@ -5202,28 +6221,39 @@ snapshots: strip-ansi: 6.0.1 wcwidth: 1.0.1 + p-limit@2.3.0: + dependencies: + p-try: 2.2.0 + p-limit@3.1.0: dependencies: yocto-queue: 0.1.0 + p-locate@4.1.0: + dependencies: + p-limit: 2.3.0 + p-locate@5.0.0: dependencies: p-limit: 3.1.0 + p-try@2.2.0: {} + package-json-from-dist@1.0.1: {} parent-module@1.0.1: dependencies: callsites: 3.1.0 - parse-entities@2.0.0: + parse-entities@4.0.2: dependencies: - character-entities: 1.2.4 - character-entities-legacy: 1.1.4 - character-reference-invalid: 1.1.4 - is-alphanumerical: 1.0.4 - is-decimal: 1.0.4 - is-hexadecimal: 1.0.4 + '@types/unist': 2.0.11 + character-entities-legacy: 3.0.0 + character-reference-invalid: 2.0.1 + decode-named-character-reference: 1.3.0 + is-alphanumerical: 2.0.1 + is-decimal: 2.0.1 + is-hexadecimal: 2.0.1 parseurl@1.3.3: {} @@ -5243,10 +6273,16 @@ snapshots: path-to-regexp@0.1.12: {} + pend@1.2.0: {} + picocolors@1.1.1: {} picomatch@4.0.3: {} + pkg-dir@4.2.0: + dependencies: + find-up: 4.1.0 + postcss-value-parser@4.2.0: {} postcss@8.5.6: @@ -5257,17 +6293,13 @@ snapshots: prelude-ls@1.2.1: {} - prismjs@1.27.0: {} - prismjs@1.30.0: {} process-nextick-args@2.0.1: {} process@0.11.10: {} - property-information@5.6.0: - dependencies: - xtend: 4.0.2 + property-information@7.1.0: {} proxy-addr@2.0.7: dependencies: @@ -5276,6 +6308,11 @@ snapshots: proxy-from-env@1.1.0: {} + pump@3.0.3: + dependencies: + end-of-stream: 1.4.5 + once: 1.4.0 + punycode@2.3.1: {} python-ast@0.1.0: @@ -5300,7 +6337,7 @@ snapshots: react: 19.2.4 scheduler: 0.27.0 - react-refresh@0.17.0: {} + react-refresh@0.18.0: {} react-remove-scroll-bar@2.3.8(react@19.2.4): dependencies: @@ -5322,13 +6359,27 @@ snapshots: react: 19.2.4 react-dom: 19.2.4(react@19.2.4) + react-router-dom@7.13.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + dependencies: + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + react-router: 7.13.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + + react-router@7.13.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + dependencies: + cookie: 1.1.1 + react: 19.2.4 + set-cookie-parser: 2.7.2 + optionalDependencies: + react-dom: 19.2.4(react@19.2.4) + react-style-singleton@2.2.3(react@19.2.4): dependencies: get-nonce: 1.0.1 react: 19.2.4 tslib: 2.8.1 - react-syntax-highlighter@15.6.6(react@19.2.4): + react-syntax-highlighter@16.1.0(react@19.2.4): dependencies: '@babel/runtime': 7.28.6 highlight.js: 10.7.3 @@ -5336,7 +6387,7 @@ snapshots: lowlight: 1.20.0 prismjs: 1.30.0 react: 19.2.4 - refractor: 3.6.0 + refractor: 5.0.0 react-use-resizable@0.2.0(react@19.2.4): dependencies: @@ -5379,21 +6430,62 @@ snapshots: readdirp@4.1.2: {} - refractor@3.6.0: + redis-errors@1.2.0: {} + + redis-memory-server@0.14.1: + dependencies: + camelcase: 6.3.0 + cross-spawn: 7.0.6 + debug: 4.4.3 + extract-zip: 2.0.1 + find-cache-dir: 3.3.2 + find-package-json: 1.2.0 + get-port: 5.1.1 + https-proxy-agent: 7.0.6 + lockfile: 1.0.4 + lodash.defaultsdeep: 4.6.1 + rimraf: 5.0.10 + semver: 7.7.4 + tar: 6.2.1 + tmp: 0.2.5 + uuid: 8.3.2 + transitivePeerDependencies: + - supports-color + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + + redis@5.10.0: + dependencies: + '@redis/bloom': 5.10.0(@redis/client@5.10.0) + '@redis/client': 5.10.0 + '@redis/json': 5.10.0(@redis/client@5.10.0) + '@redis/search': 5.10.0(@redis/client@5.10.0) + '@redis/time-series': 5.10.0(@redis/client@5.10.0) + + refractor@5.0.0: dependencies: - hastscript: 6.0.0 - parse-entities: 2.0.0 - prismjs: 1.27.0 + '@types/hast': 3.0.4 + '@types/prismjs': 1.26.6 + hastscript: 9.0.1 + parse-entities: 4.0.2 require-from-string@2.0.2: {} resolve-from@4.0.0: {} + resolve-pkg-maps@1.0.0: {} + restore-cursor@3.1.0: dependencies: onetime: 5.1.2 signal-exit: 3.0.7 + rimraf@5.0.10: + dependencies: + glob: 10.5.0 + rollup@4.57.1: dependencies: '@types/estree': 1.0.8 @@ -5470,6 +6562,8 @@ snapshots: transitivePeerDependencies: - supports-color + set-cookie-parser@2.7.2: {} + setprototypeof@1.2.0: {} shebang-command@2.0.0: @@ -5518,7 +6612,9 @@ snapshots: source-map-js@1.2.1: {} - space-separated-tokens@1.1.5: {} + space-separated-tokens@2.0.2: {} + + standard-as-callback@2.1.0: {} state-local@1.0.7: {} @@ -5577,6 +6673,8 @@ snapshots: string-width: 4.2.3 strip-ansi: 6.0.1 + tailwind-merge@2.6.1: {} + tailwind-merge@3.4.0: {} tailwindcss@4.1.18: {} @@ -5592,6 +6690,15 @@ snapshots: - bare-abort-controller - react-native-b4a + tar@6.2.1: + dependencies: + chownr: 2.0.0 + fs-minipass: 2.1.0 + minipass: 5.0.0 + minizlib: 2.1.2 + mkdirp: 1.0.4 + yallist: 4.0.0 + text-decoder@1.2.4: dependencies: b4a: 1.7.4 @@ -5605,15 +6712,17 @@ snapshots: fdir: 6.5.0(picomatch@4.0.3) picomatch: 4.0.3 + tmp@0.2.5: {} + toggle-selection@1.0.6: {} toidentifier@1.0.1: {} - ts-api-utils@2.4.0(typescript@5.8.3): + ts-api-utils@2.4.0(typescript@5.9.3): dependencies: - typescript: 5.8.3 + typescript: 5.9.3 - ts-node@10.9.2(@types/node@25.2.3)(typescript@5.8.3): + ts-node@10.9.2(@types/node@25.2.3)(typescript@5.9.3): dependencies: '@cspotcode/source-map-support': 0.8.1 '@tsconfig/node10': 1.0.12 @@ -5627,7 +6736,7 @@ snapshots: create-require: 1.1.1 diff: 4.0.4 make-error: 1.3.6 - typescript: 5.8.3 + typescript: 5.9.3 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 @@ -5639,6 +6748,13 @@ snapshots: tslib@2.8.1: {} + tsx@4.21.0: + dependencies: + esbuild: 0.27.3 + get-tsconfig: 4.13.6 + optionalDependencies: + fsevents: 2.3.3 + tw-animate-css@1.4.0: {} type-check@0.4.0: @@ -5652,18 +6768,18 @@ snapshots: media-typer: 0.3.0 mime-types: 2.1.35 - typescript-eslint@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3): + typescript-eslint@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3): dependencies: - '@typescript-eslint/eslint-plugin': 8.55.0(@typescript-eslint/parser@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3))(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) - '@typescript-eslint/parser': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) - '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.8.3) - '@typescript-eslint/utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.8.3) + '@typescript-eslint/eslint-plugin': 8.55.0(@typescript-eslint/parser@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/parser': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) + '@typescript-eslint/typescript-estree': 8.55.0(typescript@5.9.3) + '@typescript-eslint/utils': 8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) eslint: 9.39.2(jiti@2.6.1) - typescript: 5.8.3 + typescript: 5.9.3 transitivePeerDependencies: - supports-color - typescript@5.8.3: {} + typescript@5.9.3: {} undici-types@7.16.0: {} @@ -5700,15 +6816,17 @@ snapshots: uuid@11.1.0: {} + uuid@13.0.0: {} + uuid@8.3.2: {} v8-compile-cache-lib@3.0.1: {} vary@1.1.2: {} - vite@6.4.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2): + vite@7.3.1(@types/node@25.2.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0): dependencies: - esbuild: 0.25.12 + esbuild: 0.27.3 fdir: 6.5.0(picomatch@4.0.3) picomatch: 4.0.3 postcss: 8.5.6 @@ -5719,6 +6837,7 @@ snapshots: fsevents: 2.3.3 jiti: 2.6.1 lightningcss: 1.30.2 + tsx: 4.21.0 wcwidth@1.0.1: dependencies: @@ -5748,12 +6867,19 @@ snapshots: string-width: 5.1.2 strip-ansi: 7.1.2 - ws@8.19.0: {} + wrappy@1.0.2: {} - xtend@4.0.2: {} + ws@8.19.0: {} yallist@3.1.1: {} + yallist@4.0.0: {} + + yauzl@2.10.0: + dependencies: + buffer-crc32: 0.2.13 + fd-slicer: 1.1.0 + yn@3.1.1: {} yocto-queue@0.1.0: {} @@ -5771,12 +6897,10 @@ snapshots: compress-commons: 6.0.2 readable-stream: 4.7.0 - zod-to-json-schema@3.25.1(zod@3.25.76): - dependencies: - zod: 3.25.76 - zod@3.25.76: {} + zod@4.3.6: {} + zustand@4.5.7(react@19.2.4): dependencies: use-sync-external-store: 1.6.0(react@19.2.4) diff --git a/bbdev/api/steps/sardine/01_run_event_step.py b/bbdev/api/steps/sardine/01_run_event_step.py index eca10cbde..2c8fe551d 100644 --- a/bbdev/api/steps/sardine/01_run_event_step.py +++ b/bbdev/api/steps/sardine/01_run_event_step.py @@ -29,7 +29,7 @@ async def handler(data, context): sardine_dir = f"{bbdir}/bb-tests/sardine" - command = f"python run_tests.py --allure -m \"({data.get('workload', '')})\"" + command = f"python3 run_tests.py --allure -m \"({data.get('workload', '')})\"" context.logger.info( "Executing sardine command", {"command": command, "cwd": sardine_dir} ) diff --git a/bbdev/api/steps/verilator/03_build_event_step.py b/bbdev/api/steps/verilator/03_build_event_step.py index 917c6a624..8b02a575f 100644 --- a/bbdev/api/steps/verilator/03_build_event_step.py +++ b/bbdev/api/steps/verilator/03_build_event_step.py @@ -67,10 +67,11 @@ async def handler(data, context): cflags += " -DCOSIM" ldflags = ( f"-lreadline -ldramsim -lfesvr -lstdc++ " + f"-L{bebop_isa_sim}/install/lib " + f"-L{bbdir}/result/lib " f"-L{arch_dir}/thirdparty/chipyard/tools/DRAMSim2 " f"-L{arch_dir}/thirdparty/chipyard/toolchains/riscv-tools/riscv-isa-sim/build " - f"-L{arch_dir}/thirdparty/chipyard/toolchains/riscv-tools/riscv-isa-sim/build/lib " - f"-L{bebop_isa_sim}/install/lib" + f"-L{arch_dir}/thirdparty/chipyard/toolchains/riscv-tools/riscv-isa-sim/build/lib" ) obj_dir = f"{build_dir}/obj_dir" diff --git a/bbdev/api/steps/verilator/04_cosim_event_step.py b/bbdev/api/steps/verilator/04_cosim_event_step.py index 66d267766..dc8c7ad99 100644 --- a/bbdev/api/steps/verilator/04_cosim_event_step.py +++ b/bbdev/api/steps/verilator/04_cosim_event_step.py @@ -67,10 +67,14 @@ async def handler(data, context): # ================================================================================== # Execute simulation script with streaming output # ================================================================================== - # batch_param = "True" if batch else "False" - # sim_cmd = f"./scripts/sim.sh {bin_path} {binary_path} {log_dir}/stdout.log \ - # {log_dir}/disasm.log {batch_param} {vcd_path} {log_path}" + bebop_isa_sim = f"{bbdir}/bebop/host/spike/riscv-isa-sim" + ld_lib_path = ( + f"{bebop_isa_sim}/install/lib:" + f"{bbdir}/result/lib:" + f"{arch_dir}/thirdparty/chipyard/tools/DRAMSim2" + ) sim_cmd = ( + f"export LD_LIBRARY_PATH=\"{ld_lib_path}:$LD_LIBRARY_PATH\"; " f"{bin_path} +permissive +loadmem={binary_path} +loadmem_addr=800000000 " f"{'+batch ' if batch else ''} " f"+fst={fst_path} +log={log_path} +permissive-off " diff --git a/bbdev/api/steps/verilator/04_sim_event_step.py b/bbdev/api/steps/verilator/04_sim_event_step.py index 521236ad8..4bd5661e1 100644 --- a/bbdev/api/steps/verilator/04_sim_event_step.py +++ b/bbdev/api/steps/verilator/04_sim_event_step.py @@ -67,10 +67,14 @@ async def handler(data, context): # ================================================================================== # Execute simulation script with streaming output # ================================================================================== - # batch_param = "True" if batch else "False" - # sim_cmd = f"./scripts/sim.sh {bin_path} {binary_path} {log_dir}/stdout.log \ - # {log_dir}/disasm.log {batch_param} {vcd_path} {log_path}" + bebop_isa_sim = f"{bbdir}/bebop/host/spike/riscv-isa-sim" + ld_lib_path = ( + f"{bebop_isa_sim}/install/lib:" + f"{bbdir}/result/lib:" + f"{arch_dir}/thirdparty/chipyard/tools/DRAMSim2" + ) sim_cmd = ( + f"export LD_LIBRARY_PATH=\"{ld_lib_path}:$LD_LIBRARY_PATH\"; " f"{bin_path} +permissive +loadmem={binary_path} +loadmem_addr=800000000 " f"{'+batch ' if batch else ''} " f"+fst={fst_path} +log={log_path} +permissive-off " diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index 69320c09e..cefb76881 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -3,4 +3,5 @@ def get_buckyball_path(): current_dir = os.path.dirname(__file__) - return os.path.dirname(os.path.dirname(current_dir)) + # bbdev/api/utils -> bbdev/api -> bbdev -> buckyball + return os.path.dirname(os.path.dirname(os.path.dirname(current_dir))) diff --git a/bbdev/bbdev b/bbdev/bbdev index 1cf99c6d0..d7775c537 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -449,11 +449,41 @@ if __name__ == "__main__": available_port = find_available_port(start_port=5100, end_port=5500) print(f"Starting server on port {available_port}...") proc = subprocess.Popen( - ["pnpm", "dev", "--port", str(available_port)], cwd=workflow_dir + ["pnpm", "dev", "--port", str(available_port)], + cwd=workflow_dir, + start_new_session=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) + # Forward stdout/stderr in background threads; stopped before + # shutdown to suppress BullMQ teardown errors and pnpm's + # ELIFECYCLE message (Motia framework limitation). + import threading + + output_active = threading.Event() + output_active.set() + + def forward_stream(stream, dest): + for line in iter(stream.readline, b""): + if not output_active.is_set(): + break + dest.write(line) + dest.flush() + + threading.Thread( + target=forward_stream, + args=(proc.stdout, sys.stdout.buffer), + daemon=True, + ).start() + threading.Thread( + target=forward_stream, + args=(proc.stderr, sys.stderr.buffer), + daemon=True, + ).start() + # Wait for service to start - max_retries = 30 + max_retries = 600 for i in range(max_retries): try: # Disable proxy, connect directly to localhost @@ -506,10 +536,22 @@ if __name__ == "__main__": ) # 3. Shutdown service ================================ - # Give observability plugin time to finish async operations (e.g., Redis writes) + output_active.clear() + import signal + time.sleep(1) - proc.terminate() - proc.wait() + try: + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + except ProcessLookupError: + pass + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() print( f"\nTask completed. Command running on http://localhost:{available_port} is finished" ) diff --git a/bbdev/flake.nix b/bbdev/flake.nix index c9425fc09..fe81a33b1 100644 --- a/bbdev/flake.nix +++ b/bbdev/flake.nix @@ -29,16 +29,13 @@ shellHook = '' export PATH="$PWD:$PATH" + source "$PWD/init.sh" echo "bbdev dev environment ready" echo " Node.js: $(node --version)" echo " Python: $(python3 --version)" echo " pnpm: $(pnpm --version)" echo " bbdev: $(which bbdev)" - echo "" - echo "Quick start:" - echo " pnpm install # install node deps + motia install" - echo " pnpm dev # start motia dev server" ''; }; } From 653103a5a66fb1ad7c4ee1a9dcaf0fd8291c5fe9 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 15 Feb 2026 20:53:05 +0800 Subject: [PATCH 007/196] refactor: removing threading in bbdev --- bbdev/api/motia.config.ts | 3 +-- bbdev/bbdev | 52 ++++----------------------------------- 2 files changed, 6 insertions(+), 49 deletions(-) diff --git a/bbdev/api/motia.config.ts b/bbdev/api/motia.config.ts index d4bdc0b0c..8e7e1684e 100644 --- a/bbdev/api/motia.config.ts +++ b/bbdev/api/motia.config.ts @@ -3,8 +3,7 @@ import endpointPlugin from '@motiadev/plugin-endpoint/plugin' import logsPlugin from '@motiadev/plugin-logs/plugin' import observabilityPlugin from '@motiadev/plugin-observability/plugin' import statesPlugin from '@motiadev/plugin-states/plugin' -import bullmqPlugin from '@motiadev/plugin-bullmq/plugin' export default defineConfig({ - plugins: [observabilityPlugin, statesPlugin, endpointPlugin, logsPlugin, bullmqPlugin], + plugins: [observabilityPlugin, statesPlugin, endpointPlugin, logsPlugin], }) diff --git a/bbdev/bbdev b/bbdev/bbdev index d7775c537..1cf99c6d0 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -449,41 +449,11 @@ if __name__ == "__main__": available_port = find_available_port(start_port=5100, end_port=5500) print(f"Starting server on port {available_port}...") proc = subprocess.Popen( - ["pnpm", "dev", "--port", str(available_port)], - cwd=workflow_dir, - start_new_session=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + ["pnpm", "dev", "--port", str(available_port)], cwd=workflow_dir ) - # Forward stdout/stderr in background threads; stopped before - # shutdown to suppress BullMQ teardown errors and pnpm's - # ELIFECYCLE message (Motia framework limitation). - import threading - - output_active = threading.Event() - output_active.set() - - def forward_stream(stream, dest): - for line in iter(stream.readline, b""): - if not output_active.is_set(): - break - dest.write(line) - dest.flush() - - threading.Thread( - target=forward_stream, - args=(proc.stdout, sys.stdout.buffer), - daemon=True, - ).start() - threading.Thread( - target=forward_stream, - args=(proc.stderr, sys.stderr.buffer), - daemon=True, - ).start() - # Wait for service to start - max_retries = 600 + max_retries = 30 for i in range(max_retries): try: # Disable proxy, connect directly to localhost @@ -536,22 +506,10 @@ if __name__ == "__main__": ) # 3. Shutdown service ================================ - output_active.clear() - import signal - + # Give observability plugin time to finish async operations (e.g., Redis writes) time.sleep(1) - try: - os.killpg(os.getpgid(proc.pid), signal.SIGTERM) - except ProcessLookupError: - pass - try: - proc.wait(timeout=10) - except subprocess.TimeoutExpired: - try: - os.killpg(os.getpgid(proc.pid), signal.SIGKILL) - except ProcessLookupError: - pass - proc.wait() + proc.terminate() + proc.wait() print( f"\nTask completed. Command running on http://localhost:{available_port} is finished" ) From 988d9894ec34032b7b01e8c0575646106772c24f Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 15 Feb 2026 21:37:06 +0800 Subject: [PATCH 008/196] fix: increase max retries and suppress stderr in bbdev server startup --- bbdev/bbdev | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bbdev/bbdev b/bbdev/bbdev index 1cf99c6d0..ec7e95c11 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -449,11 +449,13 @@ if __name__ == "__main__": available_port = find_available_port(start_port=5100, end_port=5500) print(f"Starting server on port {available_port}...") proc = subprocess.Popen( - ["pnpm", "dev", "--port", str(available_port)], cwd=workflow_dir + ["pnpm", "dev", "--port", str(available_port)], + cwd=workflow_dir, + stderr=subprocess.DEVNULL # Suppress stderr to hide BullMQ errors ) # Wait for service to start - max_retries = 30 + max_retries = 300 for i in range(max_retries): try: # Disable proxy, connect directly to localhost From 3551e22f91bfa0d119858649a6a822753999a80f Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 26 Feb 2026 02:01:33 +0800 Subject: [PATCH 009/196] fix: clean up stale BullMQ/Redis data and improve process termination in bbdev --- bbdev/api/steps/sardine/01_run_event_step.py | 2 +- bbdev/api/steps/verilator/04_sim_event_step.py | 1 + bbdev/bbdev | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bbdev/api/steps/sardine/01_run_event_step.py b/bbdev/api/steps/sardine/01_run_event_step.py index 2c8fe551d..d35899ce9 100644 --- a/bbdev/api/steps/sardine/01_run_event_step.py +++ b/bbdev/api/steps/sardine/01_run_event_step.py @@ -29,7 +29,7 @@ async def handler(data, context): sardine_dir = f"{bbdir}/bb-tests/sardine" - command = f"python3 run_tests.py --allure -m \"({data.get('workload', '')})\"" + command = f"python3 run_tests.py -m \"({data.get('workload', '')})\"" context.logger.info( "Executing sardine command", {"command": command, "cwd": sardine_dir} ) diff --git a/bbdev/api/steps/verilator/04_sim_event_step.py b/bbdev/api/steps/verilator/04_sim_event_step.py index 4bd5661e1..62f18982b 100644 --- a/bbdev/api/steps/verilator/04_sim_event_step.py +++ b/bbdev/api/steps/verilator/04_sim_event_step.py @@ -40,6 +40,7 @@ async def handler(data, context): ) binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) + context.logger.info(f"binary_path: {binary_path}") success_result, failure_result = await check_result( context, returncode=(binary_path == None), continue_run=True ) diff --git a/bbdev/bbdev b/bbdev/bbdev index ec7e95c11..1fd020ce6 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -4,9 +4,11 @@ import sys import argparse import subprocess import os +import signal import shlex import json import time +import shutil import requests from utils import find_available_port @@ -441,6 +443,11 @@ if __name__ == "__main__": "uvm", "palladium", ]: + # 0. Clean up stale BullMQ/Redis data to prevent replaying old events + aof_dir = os.path.join(workflow_dir, ".motia", "appendonlydir") + if os.path.exists(aof_dir): + shutil.rmtree(aof_dir) + # 1. Start service in background ================================ # If port is specified, use the specified port; otherwise, automatically assign port if args.port: From 2c6eabca332abda20fb6ea60a4fca9e918013250 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 6 Mar 2026 03:28:54 +0800 Subject: [PATCH 010/196] feat: faster verilator build process with lld and ccache support Attension: verilator version is updated --- .../steps/verilator/03_build_event_step.py | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/bbdev/api/steps/verilator/03_build_event_step.py b/bbdev/api/steps/verilator/03_build_event_step.py index 8b02a575f..15cae7994 100644 --- a/bbdev/api/steps/verilator/03_build_event_step.py +++ b/bbdev/api/steps/verilator/03_build_event_step.py @@ -81,8 +81,26 @@ async def handler(data, context): sources = " ".join(vsrcs + csrcs) jobs = data.get("jobs", "") + # Fix nix runtime library paths: embed rpath so binary finds nix libstdc++/glibc + libstdcpp_path = subprocess.run( + "g++ -print-file-name=libstdc++.so", shell=True, capture_output=True, text=True + ).stdout.strip() + if libstdcpp_path and "/" in libstdcpp_path: + nix_lib_dir = os.path.dirname(os.path.realpath(libstdcpp_path)) + ldflags += f" -Wl,-rpath,{nix_lib_dir}" + + # Enable ccache for verilator C++ compilation via OBJCACHE + if subprocess.run("command -v ccache", shell=True, capture_output=True).returncode == 0: + os.environ["OBJCACHE"] = "ccache" + + # Build acceleration: lld for faster linking (available in nix env) + use_lld = subprocess.run("command -v ld.lld", shell=True, capture_output=True).returncode == 0 + if use_lld: + ldflags += " -fuse-ld=lld" + + # -O1 instead of -O3: much faster C++ compilation, minimal simulation speed difference verilator_cmd = ( - f"verilator -MMD --build -cc --trace -O3 --x-assign fast --x-initial fast --noassert -Wno-fatal " + f"verilator -MMD -cc --vpi --trace -O1 --x-assign fast --x-initial fast --noassert -Wno-fatal " f"--trace-fst --trace-threads 1 --output-split 10000 --output-split-cfuncs 100 " f"--unroll-count 256 " f"-Wno-PINCONNECTEMPTY " @@ -101,11 +119,18 @@ async def handler(data, context): cmd=verilator_cmd, logger=context.logger, cwd=bbdir, - stdout_prefix="verilator build", - stderr_prefix="verilator build", + stdout_prefix="verilator verilation", + stderr_prefix="verilator verilation", ) + if result.returncode != 0: + success_result, failure_result = await check_result( + context, result.returncode, continue_run=False, extra_fields={"task": "build"} + ) + return + + make_jobs = jobs if jobs else str(os.cpu_count() or 16) result = stream_run_logger( - cmd=f"make -C {obj_dir} -f V{topname}.mk V{topname}", + cmd=f"make -j{make_jobs} VM_PARALLEL_BUILDS=1 -C {obj_dir} -f V{topname}.mk V{topname}", logger=context.logger, cwd=bbdir, stdout_prefix="verilator build", From dd2d10c0ee5a8a525533663ba5d8b19bcc7c0168 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 6 Mar 2026 03:43:43 +0800 Subject: [PATCH 011/196] fix: correct optimization level in verilator command for improved build performance --- bbdev/api/steps/verilator/03_build_event_step.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbdev/api/steps/verilator/03_build_event_step.py b/bbdev/api/steps/verilator/03_build_event_step.py index 15cae7994..11bbc3a86 100644 --- a/bbdev/api/steps/verilator/03_build_event_step.py +++ b/bbdev/api/steps/verilator/03_build_event_step.py @@ -100,7 +100,7 @@ async def handler(data, context): # -O1 instead of -O3: much faster C++ compilation, minimal simulation speed difference verilator_cmd = ( - f"verilator -MMD -cc --vpi --trace -O1 --x-assign fast --x-initial fast --noassert -Wno-fatal " + f"verilator -MMD -cc --vpi --trace -O3 --x-assign fast --x-initial fast --noassert -Wno-fatal " f"--trace-fst --trace-threads 1 --output-split 10000 --output-split-cfuncs 100 " f"--unroll-count 256 " f"-Wno-PINCONNECTEMPTY " From f8f6cd16fac8457662c9bd577a76dee99ae14774 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 6 Mar 2026 15:14:54 +0800 Subject: [PATCH 012/196] chore: remove outdated comment regarding optimization level in verilator command --- bbdev/api/steps/verilator/03_build_event_step.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bbdev/api/steps/verilator/03_build_event_step.py b/bbdev/api/steps/verilator/03_build_event_step.py index 11bbc3a86..02542e0d6 100644 --- a/bbdev/api/steps/verilator/03_build_event_step.py +++ b/bbdev/api/steps/verilator/03_build_event_step.py @@ -98,7 +98,6 @@ async def handler(data, context): if use_lld: ldflags += " -fuse-ld=lld" - # -O1 instead of -O3: much faster C++ compilation, minimal simulation speed difference verilator_cmd = ( f"verilator -MMD -cc --vpi --trace -O3 --x-assign fast --x-initial fast --noassert -Wno-fatal " f"--trace-fst --trace-threads 1 --output-split 10000 --output-split-cfuncs 100 " From 07f97a3075d409421a15b7840eb3f66f682adcbb Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 7 Mar 2026 03:29:34 +0800 Subject: [PATCH 013/196] feat: add yosys synthesis steps for area estimation, feat: add opensta for frequency evaluation --- .../api/steps/verilator/03_build_api_step.py | 1 + .../steps/verilator/03_build_event_step.py | 2 + .../steps/verilator/04_cosim_event_step.py | 26 ++- .../api/steps/verilator/04_sim_event_step.py | 26 ++- bbdev/api/steps/verilator/05_run_api_step.py | 1 + bbdev/api/steps/yosys/.gitignore | 1 + bbdev/api/steps/yosys/01_synth_api_step.py | 37 +++ bbdev/api/steps/yosys/01_synth_event_step.py | 216 ++++++++++++++++++ .../api/steps/yosys/scripts/yosys-config.yaml | 22 ++ bbdev/bbdev | 27 ++- 10 files changed, 341 insertions(+), 18 deletions(-) create mode 100644 bbdev/api/steps/yosys/.gitignore create mode 100644 bbdev/api/steps/yosys/01_synth_api_step.py create mode 100644 bbdev/api/steps/yosys/01_synth_event_step.py create mode 100644 bbdev/api/steps/yosys/scripts/yosys-config.yaml diff --git a/bbdev/api/steps/verilator/03_build_api_step.py b/bbdev/api/steps/verilator/03_build_api_step.py index 2c77f19e3..a5e4115ef 100644 --- a/bbdev/api/steps/verilator/03_build_api_step.py +++ b/bbdev/api/steps/verilator/03_build_api_step.py @@ -17,6 +17,7 @@ async def handler(req, context): data = { "jobs": body.get("jobs", 16), "cosim": body.get("cosim", False), + "coverage": body.get("coverage", False), } await context.emit({"topic": "verilator.build", "data": data}) diff --git a/bbdev/api/steps/verilator/03_build_event_step.py b/bbdev/api/steps/verilator/03_build_event_step.py index 02542e0d6..f2cfc1ac0 100644 --- a/bbdev/api/steps/verilator/03_build_event_step.py +++ b/bbdev/api/steps/verilator/03_build_event_step.py @@ -29,6 +29,7 @@ async def handler(data, context): waveform_dir = f"{arch_dir}/waveform" log_dir = f"{arch_dir}/log" cosim = data.get("cosim", False) + coverage = data.get("coverage", False) # ================================================================================== # Execute operation @@ -102,6 +103,7 @@ async def handler(data, context): f"verilator -MMD -cc --vpi --trace -O3 --x-assign fast --x-initial fast --noassert -Wno-fatal " f"--trace-fst --trace-threads 1 --output-split 10000 --output-split-cfuncs 100 " f"--unroll-count 256 " + f"{'--coverage-line ' if coverage else ''}" f"-Wno-PINCONNECTEMPTY " f"-Wno-ASSIGNDLY " f"-Wno-DECLFILENAME " diff --git a/bbdev/api/steps/verilator/04_cosim_event_step.py b/bbdev/api/steps/verilator/04_cosim_event_step.py index dc8c7ad99..ec1c524ba 100644 --- a/bbdev/api/steps/verilator/04_cosim_event_step.py +++ b/bbdev/api/steps/verilator/04_cosim_event_step.py @@ -35,6 +35,7 @@ async def handler(data, context): timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") binary_name = data.get("binary", "") + coverage = data.get("coverage", False) success_result, failure_result = await check_result( context, returncode=(binary_name == None), continue_run=True ) @@ -55,6 +56,14 @@ async def handler(data, context): os.makedirs(log_dir, exist_ok=True) os.makedirs(waveform_dir, exist_ok=True) + # Coverage data output path + coverage_flag = "" + if coverage: + coverage_dir = f"{bbdir}/bb-tests/sardine/reports/coverage" + os.makedirs(coverage_dir, exist_ok=True) + coverage_dat_path = f"{coverage_dir}/{timestamp}-{binary_name}.dat" + coverage_flag = f"+verilator+coverage+file+{coverage_dat_path}" + bin_path = f"{build_dir}/obj_dir/V{topname}" batch = data.get("batch", False) @@ -77,6 +86,7 @@ async def handler(data, context): f"export LD_LIBRARY_PATH=\"{ld_lib_path}:$LD_LIBRARY_PATH\"; " f"{bin_path} +permissive +loadmem={binary_path} +loadmem_addr=800000000 " f"{'+batch ' if batch else ''} " + f"{coverage_flag + ' ' if coverage_flag else ''}" f"+fst={fst_path} +log={log_path} +permissive-off " f"{binary_path} > >(tee {log_dir}/stdout.log) 2> >(spike-dasm > {log_dir}/disasm.log)" ) @@ -108,17 +118,21 @@ async def handler(data, context): # Return simulation result # ================================================================================== # This is the end point of the run workflow, status will no longer be set to processing - success_result, failure_result = await check_result( - context, - result.returncode, - continue_run=False, - extra_fields={ + extra_fields = { "task": "sim", "binary": binary_path, "log_dir": log_dir, "waveform_dir": waveform_dir, "timestamp": timestamp, - }, + } + if coverage: + extra_fields["coverage_dat"] = coverage_dat_path + + success_result, failure_result = await check_result( + context, + result.returncode, + continue_run=False, + extra_fields=extra_fields, ) # ================================================================================== diff --git a/bbdev/api/steps/verilator/04_sim_event_step.py b/bbdev/api/steps/verilator/04_sim_event_step.py index 62f18982b..a9c4d4e84 100644 --- a/bbdev/api/steps/verilator/04_sim_event_step.py +++ b/bbdev/api/steps/verilator/04_sim_event_step.py @@ -35,6 +35,7 @@ async def handler(data, context): timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") binary_name = data.get("binary", "") + coverage = data.get("coverage", False) success_result, failure_result = await check_result( context, returncode=(binary_name == None), continue_run=True ) @@ -56,6 +57,14 @@ async def handler(data, context): os.makedirs(log_dir, exist_ok=True) os.makedirs(waveform_dir, exist_ok=True) + # Coverage data output path + coverage_flag = "" + if coverage: + coverage_dir = f"{bbdir}/bb-tests/sardine/reports/coverage" + os.makedirs(coverage_dir, exist_ok=True) + coverage_dat_path = f"{coverage_dir}/{timestamp}-{binary_name}.dat" + coverage_flag = f"+verilator+coverage+file+{coverage_dat_path}" + bin_path = f"{build_dir}/obj_dir/V{topname}" batch = data.get("batch", False) @@ -78,6 +87,7 @@ async def handler(data, context): f"export LD_LIBRARY_PATH=\"{ld_lib_path}:$LD_LIBRARY_PATH\"; " f"{bin_path} +permissive +loadmem={binary_path} +loadmem_addr=800000000 " f"{'+batch ' if batch else ''} " + f"{coverage_flag + ' ' if coverage_flag else ''}" f"+fst={fst_path} +log={log_path} +permissive-off " f"{binary_path} > >(tee {log_dir}/stdout.log) 2> >(spike-dasm > {log_dir}/disasm.log)" ) @@ -109,17 +119,21 @@ async def handler(data, context): # Return simulation result # ================================================================================== # This is the end point of the run workflow, status will no longer be set to processing - success_result, failure_result = await check_result( - context, - result.returncode, - continue_run=False, - extra_fields={ + extra_fields = { "task": "sim", "binary": binary_path, "log_dir": log_dir, "waveform_dir": waveform_dir, "timestamp": timestamp, - }, + } + if coverage: + extra_fields["coverage_dat"] = coverage_dat_path + + success_result, failure_result = await check_result( + context, + result.returncode, + continue_run=False, + extra_fields=extra_fields, ) # ================================================================================== diff --git a/bbdev/api/steps/verilator/05_run_api_step.py b/bbdev/api/steps/verilator/05_run_api_step.py index b29f35083..02c366a61 100644 --- a/bbdev/api/steps/verilator/05_run_api_step.py +++ b/bbdev/api/steps/verilator/05_run_api_step.py @@ -21,6 +21,7 @@ async def handler(req, context): "jobs": body.get("jobs", "16"), "batch": body.get("batch", False), "cosim": body.get("cosim", False), + "coverage": body.get("coverage", False), "from_run_workflow": True, } diff --git a/bbdev/api/steps/yosys/.gitignore b/bbdev/api/steps/yosys/.gitignore new file mode 100644 index 000000000..6bfe6b19e --- /dev/null +++ b/bbdev/api/steps/yosys/.gitignore @@ -0,0 +1 @@ +log diff --git a/bbdev/api/steps/yosys/01_synth_api_step.py b/bbdev/api/steps/yosys/01_synth_api_step.py new file mode 100644 index 000000000..ef375cdfb --- /dev/null +++ b/bbdev/api/steps/yosys/01_synth_api_step.py @@ -0,0 +1,37 @@ +import asyncio +from utils.event_common import wait_for_result + + +from utils.path import get_buckyball_path + + +config = { + "type": "api", + "name": "Yosys Synth", + "description": "run yosys synthesis for area estimation", + "path": "/yosys/synth", + "method": "POST", + "emits": ["yosys.synth"], + "flows": ["yosys"], +} + + +async def handler(req, context): + bbdir = get_buckyball_path() + body = req.get("body") or {} + + data = { + "balltype": body.get("balltype"), + "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), + "top": body.get("top"), + } + await context.emit({"topic": "yosys.synth", "data": data}) + + # ================================================================================== + # Wait for synthesis result + # ================================================================================== + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/yosys/01_synth_event_step.py b/bbdev/api/steps/yosys/01_synth_event_step.py new file mode 100644 index 000000000..008a15166 --- /dev/null +++ b/bbdev/api/steps/yosys/01_synth_event_step.py @@ -0,0 +1,216 @@ +import os +import subprocess +import glob +import sys +import yaml + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result + +config = { + "type": "event", + "name": "yosys synth", + "description": "run yosys synthesis for area estimation", + "subscribes": ["yosys.synth"], + "emits": [], + "flows": ["yosys"], +} + + +def load_yosys_config(): + """Load yosys configuration from yaml file""" + config_path = os.path.join(os.path.dirname(__file__), "scripts", "yosys-config.yaml") + if os.path.exists(config_path): + with open(config_path, "r") as f: + return yaml.safe_load(f) or {} + return {} + + +async def handler(data, context): + bbdir = get_buckyball_path() + build_dir = data.get("output_dir", f"{bbdir}/arch/build/") + arch_dir = f"{bbdir}/arch" + + # Load yaml config as defaults, CLI args override + yosys_cfg = load_yosys_config() + top_module = data.get("top") or yosys_cfg.get("top") or "TargetBall" + balltype = data.get("balltype") or yosys_cfg.get("balltype") or "vecball" + liberty = yosys_cfg.get("liberty") # Path to .lib file, configured in yosys-config.yaml only + + context.logger.info(f"Ball type: {balltype}, top module: {top_module}") + + # ================================================================================== + # Step 1: Generate Verilog via BallTopMain (pure design, no DPI-C sim wrappers) + # Using BallTopMain instead of Elaborate to avoid SimTSI/SimDRAM/SimJTAG files + # that contain DPI-C imports which Yosys cannot parse + # ================================================================================== + context.logger.info("Step 1: Cleaning build directory and generating Verilog...") + + # Clean build directory first to remove stale files from previous Elaborate runs + import shutil + if os.path.exists(build_dir): + shutil.rmtree(build_dir) + os.makedirs(build_dir, exist_ok=True) + + verilog_command = ( + f"mill -i __.test.runMain sims.verify.BallTopMain {balltype} " + ) + verilog_command += "--disable-annotation-unknown -strip-debug-info -O=debug " + verilog_command += "-lowering-options=disallowLocalVariables " + verilog_command += f"--split-verilog -o={build_dir}" + + result = stream_run_logger( + cmd=verilog_command, + logger=context.logger, + cwd=arch_dir, + stdout_prefix="yosys verilog", + stderr_prefix="yosys verilog", + ) + + if result.returncode != 0: + context.logger.error(f"Verilog generation failed with code {result.returncode}") + success_result, failure_result = await check_result( + context, + result.returncode, + continue_run=False, + extra_fields={"task": "verilog"}, + ) + return failure_result + + # Remove unwanted file (BallTopMain shouldn't generate this, but clean up just in case) + for unwanted in ["TestHarness.sv", "TargetBall.sv"]: + topname_file = f"{arch_dir}/{unwanted}" + if os.path.exists(topname_file): + os.remove(topname_file) + + # ================================================================================== + # Step 2: Run Yosys synthesis for area estimation + # ================================================================================== + context.logger.info("Step 2: Running Yosys synthesis for area estimation...") + + # Collect all Verilog/SystemVerilog sources + vsrcs = glob.glob(f"{build_dir}/**/*.v", recursive=True) + glob.glob( + f"{build_dir}/**/*.sv", recursive=True + ) + + if not vsrcs: + context.logger.error(f"No Verilog source files found in {build_dir}") + success_result, failure_result = await check_result( + context, + 1, + continue_run=False, + extra_fields={"task": "synth", "error": "No Verilog source files found"}, + ) + return failure_result + + # Create yosys log directory + yosys_output_dir = os.path.join(os.path.dirname(__file__), "log") + os.makedirs(yosys_output_dir, exist_ok=True) + + # Build yosys read commands for all source files + read_commands = "\n".join([f"read_verilog -sv {src}" for src in vsrcs]) + + # Write yosys synthesis script + yosys_script = f"{yosys_output_dir}/synth_area.ys" + with open(yosys_script, "w") as f: + f.write(f"{read_commands}\n") + f.write(f"hierarchy -top {top_module}\n") + f.write("proc\n") + f.write("flatten\n") + f.write("opt\n") + f.write(f"synth -top {top_module}\n") + + if liberty and os.path.exists(liberty): + # Map to standard cells using liberty file for accurate area + context.logger.info(f"Using liberty file: {liberty}") + f.write(f"dfflibmap -liberty {liberty}\n") + f.write(f"abc -liberty {liberty}\n") + f.write(f"stat -liberty {liberty}\n") + f.write(f"tee -o {yosys_output_dir}/area_report.txt stat -liberty {liberty}\n") + # Write mapped netlist for OpenSTA timing analysis + f.write(f"write_verilog {yosys_output_dir}/synth_netlist.v\n") + else: + if liberty: + context.logger.warn(f"Liberty file not found: {liberty}, falling back to generic stat") + f.write("stat\n") + f.write(f"tee -o {yosys_output_dir}/area_report.txt stat\n") + + context.logger.info(f"Yosys script written to {yosys_script}") + context.logger.info(f"Synthesizing {len(vsrcs)} source files with top module: {top_module}") + + # Run yosys + yosys_command = f"yosys -s {yosys_script}" + + result = stream_run_logger( + cmd=yosys_command, + logger=context.logger, + cwd=build_dir, + stdout_prefix="yosys synth", + stderr_prefix="yosys synth", + ) + + # ================================================================================== + # Step 3: Run OpenSTA timing analysis (only if liberty file is available) + # ================================================================================== + extra = {"task": "synth", "output_dir": yosys_output_dir} + netlist_file = f"{yosys_output_dir}/synth_netlist.v" + timing_report_file = f"{yosys_output_dir}/timing_report.txt" + + if liberty and os.path.exists(liberty) and os.path.exists(netlist_file) and result.returncode == 0: + context.logger.info("Step 3: Running OpenSTA timing analysis...") + + # Get target clock period from config (default 10ns = 100MHz) + clock_period = yosys_cfg.get("clock_period", 10.0) + clock_name = yosys_cfg.get("clock_name", "clock") + + # Write OpenSTA TCL script + sta_script = f"{yosys_output_dir}/sta_timing.tcl" + with open(sta_script, "w") as f: + f.write(f"read_liberty {liberty}\n") + f.write(f"read_verilog {netlist_file}\n") + f.write(f"link_design {top_module}\n") + f.write(f"create_clock [get_ports {clock_name}] -name clk -period {clock_period}\n") + f.write(f"report_checks -path_delay max -format full > {timing_report_file}\n") + f.write(f"report_checks -path_delay max -format full\n") + f.write("exit\n") + + sta_result = stream_run_logger( + cmd=f"sta {sta_script}", + logger=context.logger, + cwd=yosys_output_dir, + stdout_prefix="opensta", + stderr_prefix="opensta", + ) + + if sta_result.returncode == 0 and os.path.exists(timing_report_file): + with open(timing_report_file, "r") as f: + extra["timing_report"] = f.read() + context.logger.info(f"Timing report saved to {timing_report_file}") + else: + context.logger.warn(f"OpenSTA timing analysis failed (rc={sta_result.returncode})") + elif result.returncode == 0: + context.logger.info("Skipping OpenSTA: no liberty file configured or synthesis failed") + + # ================================================================================== + # Return result to API + # ================================================================================== + report_file = f"{yosys_output_dir}/area_report.txt" + if os.path.exists(report_file): + with open(report_file, "r") as f: + extra["area_report"] = f.read() + context.logger.info(f"Area report saved to {report_file}") + + success_result, failure_result = await check_result( + context, + result.returncode, + continue_run=False, + extra_fields=extra, + ) + + return diff --git a/bbdev/api/steps/yosys/scripts/yosys-config.yaml b/bbdev/api/steps/yosys/scripts/yosys-config.yaml new file mode 100644 index 000000000..5fbe43c54 --- /dev/null +++ b/bbdev/api/steps/yosys/scripts/yosys-config.yaml @@ -0,0 +1,22 @@ +# Yosys synthesis configuration +# Users can customize liberty library paths and synthesis options here + +# Liberty (.lib) file path for ASIC area estimation +# If not specified, yosys will use generic gate-level statistics +# Example: +# liberty: /path/to/your/standard_cell.lib +liberty: + +# Default top module name (override with --top) +top: TargetBall + +# Default ball type (override with --balltype) +balltype: vecball + +# OpenSTA timing analysis settings +# Clock period in nanoseconds (used for STA constraint) +# Default 10ns = 500MHz +clock_period: 2.0 + +# Clock port name in the design +clock_name: clock diff --git a/bbdev/bbdev b/bbdev/bbdev index 1fd020ce6..9bc2fdf50 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -61,7 +61,7 @@ def parse_args(argv: list[str]) -> argparse.Namespace: nargs="?", const="", metavar="ARGS", - help='Build verilator simulation executable. Args: "[--job ]"', + help='Build verilator simulation executable. Args: "[--job ] [--coverage]"', ) verilator_group.add_argument( "--sim", @@ -69,7 +69,7 @@ def parse_args(argv: list[str]) -> argparse.Namespace: nargs="?", const="", metavar="ARGS", - help='Run verilator simulation. Args: "--binary [--batch]"', + help='Run verilator simulation. Args: "--binary [--batch] [--coverage]"', ) verilator_group.add_argument( "--run", @@ -77,7 +77,7 @@ def parse_args(argv: list[str]) -> argparse.Namespace: nargs="?", const="", metavar="ARGS", - help='Integrated build+sim+run. Args: "--binary [--batch] [--job ]"', + help='Integrated build+sim+run. Args: "--binary [--batch] [--job ] [--coverage]"', ) verilator_group.add_argument( "--cosim", @@ -132,7 +132,7 @@ def parse_args(argv: list[str]) -> argparse.Namespace: nargs="?", const="", metavar="ARGS", - help='Run sardine. Args: "--workload "', + help='Run sardine. Args: "--workload [--coverage]"', ) # ===== agent subcommand ========================================================================= @@ -239,6 +239,19 @@ def parse_args(argv: list[str]) -> argparse.Namespace: "--build", type=str, nargs="?", const="", metavar="ARGS", help="Build uvm. " ) + # ===== yosys subcommand ===================================================================== + yosys_parser = subparsers.add_parser("yosys", help="yosys synthesis operations") + yosys_group = yosys_parser.add_mutually_exclusive_group(required=True) + yosys_group.add_argument( + "--synth", + type=str, + nargs="?", + const="", + metavar="ARGS", + help='Run yosys synthesis for area estimation. Args: "[--balltype ] [--top ]"\n' + 'Liberty library can be configured in bbdev/api/steps/yosys/scripts/yosys-config.yaml', + ) + # ===== palladium subcommand =================================================================== palladium_parser = subparsers.add_parser("palladium", help="palladium operations") palladium_group = palladium_parser.add_mutually_exclusive_group(required=True) @@ -400,6 +413,7 @@ if __name__ == "__main__": "funcsim", "uvm", "palladium", + "yosys", ]: api_path = f"/{cmd_info['command']}/{cmd_info['operation']}" json_data = json.dumps(cmd_info["args"]) @@ -421,7 +435,7 @@ if __name__ == "__main__": print(f"Unknown command: {cmd_info['command']}") print( "Available commands: start, stop, verilator, vcs, sardine, agent, \ - workload, doc, marshal, firesim, compiler, funcsim, uvm" + workload, doc, marshal, firesim, compiler, funcsim, uvm, yosys" ) sys.exit(1) else: # script mode @@ -442,6 +456,7 @@ if __name__ == "__main__": "funcsim", "uvm", "palladium", + "yosys", ]: # 0. Clean up stale BullMQ/Redis data to prevent replaying old events aof_dir = os.path.join(workflow_dir, ".motia", "appendonlydir") @@ -537,6 +552,6 @@ if __name__ == "__main__": print(f"Unknown command: {cmd_info['command']}") print( "Available commands: start, stop, verilator, vcs, sardine, agent, \ - workload, doc, marshal, firesim, compiler, funcsim, uvm" + workload, doc, marshal, firesim, compiler, funcsim, uvm, yosys" ) sys.exit(1) From d114e0d148ac342e9ef044cb341547e18fd95d90 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 7 Mar 2026 04:09:10 +0800 Subject: [PATCH 014/196] feat: add coverage reporting functionality to sardine steps --- bbdev/api/steps/sardine/01_run_api_step.py | 5 +- bbdev/api/steps/sardine/01_run_event_step.py | 28 +++- .../sardine/02_coverage_report_event_step.py | 152 ++++++++++++++++++ .../steps/verilator/04_cosim_event_step.py | 4 +- .../api/steps/verilator/04_sim_event_step.py | 7 +- 5 files changed, 182 insertions(+), 14 deletions(-) create mode 100644 bbdev/api/steps/sardine/02_coverage_report_event_step.py diff --git a/bbdev/api/steps/sardine/01_run_api_step.py b/bbdev/api/steps/sardine/01_run_api_step.py index ac1e6a9f6..2c2fc1aeb 100644 --- a/bbdev/api/steps/sardine/01_run_api_step.py +++ b/bbdev/api/steps/sardine/01_run_api_step.py @@ -26,7 +26,10 @@ async def handler(req, context): body = req.get("body") or {} - data = {"workload": body.get("workload", "")} + data = { + "workload": body.get("workload", ""), + "coverage": body.get("coverage", False), + } sardine_dir = f"{bbdir}/bb-tests/sardine" diff --git a/bbdev/api/steps/sardine/01_run_event_step.py b/bbdev/api/steps/sardine/01_run_event_step.py index d35899ce9..39b1a2463 100644 --- a/bbdev/api/steps/sardine/01_run_event_step.py +++ b/bbdev/api/steps/sardine/01_run_event_step.py @@ -19,7 +19,7 @@ "name": "running sardine", "description": "running sardine", "subscribes": ["sardine.run"], - "emits": [], + "emits": ["sardine.coverage_report"], "flows": ["sardine"], } @@ -29,7 +29,12 @@ async def handler(data, context): sardine_dir = f"{bbdir}/bb-tests/sardine" + # Record start time so coverage report can filter to only this run's .dat files + run_start_time = time.time() + command = f"python3 run_tests.py -m \"({data.get('workload', '')})\"" + if data.get("coverage", False): + command += " --coverage" context.logger.info( "Executing sardine command", {"command": command, "cwd": sardine_dir} ) @@ -45,11 +50,20 @@ async def handler(data, context): # ================================================================================== # Return execution result # ================================================================================== - success_result, failure_result = await check_result( - context, result.returncode, continue_run=False - ) + coverage = data.get("coverage", False) + + if coverage: + # When coverage is enabled, always emit to coverage report step + # (even if some tests failed, coverage data is still valid) + success_result, failure_result = await check_result( + context, result.returncode, continue_run=True + ) + await context.emit( + {"topic": "sardine.coverage_report", "data": {**data, "run_start_time": run_start_time}} + ) + else: + success_result, failure_result = await check_result( + context, result.returncode, continue_run=False + ) - # ================================================================================== - # finish workflow - # ================================================================================== return diff --git a/bbdev/api/steps/sardine/02_coverage_report_event_step.py b/bbdev/api/steps/sardine/02_coverage_report_event_step.py new file mode 100644 index 000000000..c90e64c72 --- /dev/null +++ b/bbdev/api/steps/sardine/02_coverage_report_event_step.py @@ -0,0 +1,152 @@ +import os +import glob +import sys + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result + +config = { + "type": "event", + "name": "coverage report", + "description": "merge coverage data and generate annotated source + lcov HTML report", + "subscribes": ["sardine.coverage_report"], + "emits": [], + "flows": ["sardine"], +} + + +async def handler(data, context): + bbdir = get_buckyball_path() + log_dir = f"{bbdir}/arch/log" + coverage_dir = f"{bbdir}/bb-tests/sardine/reports/coverage" + os.makedirs(coverage_dir, exist_ok=True) + + # ================================================================================== + # Find coverage.dat files from this run only (created after run_start_time) + # ================================================================================== + all_dat_files = glob.glob(f"{log_dir}/*/coverage.dat") + run_start_time = data.get("run_start_time", 0) + dat_files = [f for f in all_dat_files if os.path.getmtime(f) >= run_start_time] + if not dat_files: + context.logger.error("No coverage .dat files found", {"dir": coverage_dir}) + success_result, failure_result = await check_result( + context, + returncode=1, + continue_run=False, + extra_fields={"task": "coverage_report", "error": "No .dat files found"}, + ) + return + + context.logger.info(f"Found {len(dat_files)} coverage data files") + + # ================================================================================== + # Merge all .dat files + # ================================================================================== + merged_dat = f"{coverage_dir}/merged.dat" + if len(dat_files) == 1: + # Single file, just copy + import shutil + shutil.copy2(dat_files[0], merged_dat) + else: + merge_cmd = f"verilator_coverage --merge {merged_dat} {' '.join(dat_files)}" + result = stream_run_logger( + cmd=merge_cmd, + logger=context.logger, + cwd=bbdir, + stdout_prefix="coverage merge", + stderr_prefix="coverage merge", + ) + if result.returncode != 0: + await check_result( + context, + returncode=result.returncode, + continue_run=False, + extra_fields={"task": "coverage_report", "error": "merge failed"}, + ) + return + + # ================================================================================== + # Generate annotated source report + # ================================================================================== + annotate_dir = f"{coverage_dir}/annotated" + annotate_cmd = f"verilator_coverage --annotate {annotate_dir} {merged_dat}" + result = stream_run_logger( + cmd=annotate_cmd, + logger=context.logger, + cwd=bbdir, + stdout_prefix="coverage annotate", + stderr_prefix="coverage annotate", + ) + if result.returncode != 0: + await check_result( + context, + returncode=result.returncode, + continue_run=False, + extra_fields={"task": "coverage_report", "error": "annotate failed"}, + ) + return + + # ================================================================================== + # Generate lcov info + HTML report + # ================================================================================== + lcov_info = f"{coverage_dir}/merged.info" + html_dir = f"{coverage_dir}/html" + + lcov_cmd = f"verilator_coverage --write-info {lcov_info} {merged_dat}" + result = stream_run_logger( + cmd=lcov_cmd, + logger=context.logger, + cwd=bbdir, + stdout_prefix="coverage lcov", + stderr_prefix="coverage lcov", + ) + if result.returncode != 0: + await check_result( + context, + returncode=result.returncode, + continue_run=False, + extra_fields={"task": "coverage_report", "error": "lcov export failed"}, + ) + return + + genhtml_cmd = f"genhtml {lcov_info} -o {html_dir}" + result = stream_run_logger( + cmd=genhtml_cmd, + logger=context.logger, + cwd=bbdir, + stdout_prefix="coverage html", + stderr_prefix="coverage html", + ) + if result.returncode != 0: + await check_result( + context, + returncode=result.returncode, + continue_run=False, + extra_fields={"task": "coverage_report", "error": "genhtml failed"}, + ) + return + + # ================================================================================== + # Return result + # ================================================================================== + context.logger.info(f"Coverage report generated: {html_dir}/index.html") + success_result, failure_result = await check_result( + context, + returncode=0, + continue_run=False, + extra_fields={ + "task": "coverage_report", + "coverage_dir": coverage_dir, + "annotate_dir": annotate_dir, + "html_dir": html_dir, + "dat_count": len(dat_files), + }, + ) + + return diff --git a/bbdev/api/steps/verilator/04_cosim_event_step.py b/bbdev/api/steps/verilator/04_cosim_event_step.py index ec1c524ba..f0b84ac04 100644 --- a/bbdev/api/steps/verilator/04_cosim_event_step.py +++ b/bbdev/api/steps/verilator/04_cosim_event_step.py @@ -59,9 +59,7 @@ async def handler(data, context): # Coverage data output path coverage_flag = "" if coverage: - coverage_dir = f"{bbdir}/bb-tests/sardine/reports/coverage" - os.makedirs(coverage_dir, exist_ok=True) - coverage_dat_path = f"{coverage_dir}/{timestamp}-{binary_name}.dat" + coverage_dat_path = f"{log_dir}/coverage.dat" coverage_flag = f"+verilator+coverage+file+{coverage_dat_path}" bin_path = f"{build_dir}/obj_dir/V{topname}" diff --git a/bbdev/api/steps/verilator/04_sim_event_step.py b/bbdev/api/steps/verilator/04_sim_event_step.py index a9c4d4e84..e2d64e66f 100644 --- a/bbdev/api/steps/verilator/04_sim_event_step.py +++ b/bbdev/api/steps/verilator/04_sim_event_step.py @@ -36,6 +36,7 @@ async def handler(data, context): binary_name = data.get("binary", "") coverage = data.get("coverage", False) + context.logger.info(f"[DEBUG] sim step received coverage={coverage}, data keys={list(data.keys())}") success_result, failure_result = await check_result( context, returncode=(binary_name == None), continue_run=True ) @@ -60,9 +61,7 @@ async def handler(data, context): # Coverage data output path coverage_flag = "" if coverage: - coverage_dir = f"{bbdir}/bb-tests/sardine/reports/coverage" - os.makedirs(coverage_dir, exist_ok=True) - coverage_dat_path = f"{coverage_dir}/{timestamp}-{binary_name}.dat" + coverage_dat_path = f"{log_dir}/coverage.dat" coverage_flag = f"+verilator+coverage+file+{coverage_dat_path}" bin_path = f"{build_dir}/obj_dir/V{topname}" @@ -91,6 +90,8 @@ async def handler(data, context): f"+fst={fst_path} +log={log_path} +permissive-off " f"{binary_path} > >(tee {log_dir}/stdout.log) 2> >(spike-dasm > {log_dir}/disasm.log)" ) + context.logger.info(f"[DEBUG] sim_cmd coverage_flag='{coverage_flag}'") + context.logger.info(f"[DEBUG] full sim_cmd='{sim_cmd}'") script_dir = os.path.dirname(__file__) result = stream_run_logger( From 3e58d57588f194c816d3aaa195e2b309d061ad3d Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 7 Mar 2026 04:24:53 +0800 Subject: [PATCH 015/196] fix: update verilator coverage command syntax for consistency --- bbdev/api/steps/sardine/02_coverage_report_event_step.py | 4 ++-- bbdev/api/steps/verilator/04_sim_event_step.py | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/bbdev/api/steps/sardine/02_coverage_report_event_step.py b/bbdev/api/steps/sardine/02_coverage_report_event_step.py index c90e64c72..ed1ac7a04 100644 --- a/bbdev/api/steps/sardine/02_coverage_report_event_step.py +++ b/bbdev/api/steps/sardine/02_coverage_report_event_step.py @@ -54,7 +54,7 @@ async def handler(data, context): import shutil shutil.copy2(dat_files[0], merged_dat) else: - merge_cmd = f"verilator_coverage --merge {merged_dat} {' '.join(dat_files)}" + merge_cmd = f"verilator_coverage -write {merged_dat} {' '.join(dat_files)}" result = stream_run_logger( cmd=merge_cmd, logger=context.logger, @@ -98,7 +98,7 @@ async def handler(data, context): lcov_info = f"{coverage_dir}/merged.info" html_dir = f"{coverage_dir}/html" - lcov_cmd = f"verilator_coverage --write-info {lcov_info} {merged_dat}" + lcov_cmd = f"verilator_coverage -write-info {lcov_info} {merged_dat}" result = stream_run_logger( cmd=lcov_cmd, logger=context.logger, diff --git a/bbdev/api/steps/verilator/04_sim_event_step.py b/bbdev/api/steps/verilator/04_sim_event_step.py index e2d64e66f..9f25161a6 100644 --- a/bbdev/api/steps/verilator/04_sim_event_step.py +++ b/bbdev/api/steps/verilator/04_sim_event_step.py @@ -36,7 +36,6 @@ async def handler(data, context): binary_name = data.get("binary", "") coverage = data.get("coverage", False) - context.logger.info(f"[DEBUG] sim step received coverage={coverage}, data keys={list(data.keys())}") success_result, failure_result = await check_result( context, returncode=(binary_name == None), continue_run=True ) @@ -90,8 +89,6 @@ async def handler(data, context): f"+fst={fst_path} +log={log_path} +permissive-off " f"{binary_path} > >(tee {log_dir}/stdout.log) 2> >(spike-dasm > {log_dir}/disasm.log)" ) - context.logger.info(f"[DEBUG] sim_cmd coverage_flag='{coverage_flag}'") - context.logger.info(f"[DEBUG] full sim_cmd='{sim_cmd}'") script_dir = os.path.dirname(__file__) result = stream_run_logger( From 60bc5a5765ee86c929b5ee36451cd07ee1b42876 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 7 Mar 2026 12:57:40 +0800 Subject: [PATCH 016/196] feat: add hierarchical area breakdown reporting in yosys synthesis. --- bbdev/api/steps/yosys/01_synth_api_step.py | 2 +- bbdev/api/steps/yosys/01_synth_event_step.py | 64 +++++++++++++------ .../api/steps/yosys/scripts/yosys-config.yaml | 8 +-- bbdev/bbdev | 5 +- 4 files changed, 54 insertions(+), 25 deletions(-) diff --git a/bbdev/api/steps/yosys/01_synth_api_step.py b/bbdev/api/steps/yosys/01_synth_api_step.py index ef375cdfb..5eaf482d7 100644 --- a/bbdev/api/steps/yosys/01_synth_api_step.py +++ b/bbdev/api/steps/yosys/01_synth_api_step.py @@ -21,9 +21,9 @@ async def handler(req, context): body = req.get("body") or {} data = { - "balltype": body.get("balltype"), "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), "top": body.get("top"), + "config": body.get("config"), } await context.emit({"topic": "yosys.synth", "data": data}) diff --git a/bbdev/api/steps/yosys/01_synth_event_step.py b/bbdev/api/steps/yosys/01_synth_event_step.py index 008a15166..1cc2a1968 100644 --- a/bbdev/api/steps/yosys/01_synth_event_step.py +++ b/bbdev/api/steps/yosys/01_synth_event_step.py @@ -39,31 +39,27 @@ async def handler(data, context): # Load yaml config as defaults, CLI args override yosys_cfg = load_yosys_config() - top_module = data.get("top") or yosys_cfg.get("top") or "TargetBall" - balltype = data.get("balltype") or yosys_cfg.get("balltype") or "vecball" + top_module = data.get("top") or yosys_cfg.get("top") or "BuckyballAccelerator" liberty = yosys_cfg.get("liberty") # Path to .lib file, configured in yosys-config.yaml only + elaborate_config = data.get("config") or yosys_cfg.get("elaborate_config", "sims.verilator.BuckyballToyVerilatorConfig") - context.logger.info(f"Ball type: {balltype}, top module: {top_module}") + context.logger.info(f"Top module: {top_module}, Elaborate config: {elaborate_config}") # ================================================================================== - # Step 1: Generate Verilog via BallTopMain (pure design, no DPI-C sim wrappers) - # Using BallTopMain instead of Elaborate to avoid SimTSI/SimDRAM/SimJTAG files - # that contain DPI-C imports which Yosys cannot parse + # Step 1: Generate Verilog via Elaborate (full SoC), filter DPI-C files later # ================================================================================== - context.logger.info("Step 1: Cleaning build directory and generating Verilog...") - - # Clean build directory first to remove stale files from previous Elaborate runs import shutil if os.path.exists(build_dir): shutil.rmtree(build_dir) os.makedirs(build_dir, exist_ok=True) + context.logger.info("Step 1: Generating Verilog via Elaborate...") verilog_command = ( - f"mill -i __.test.runMain sims.verify.BallTopMain {balltype} " + f"mill -i __.test.runMain sims.verilator.Elaborate {elaborate_config} " + "--disable-annotation-unknown -strip-debug-info -O=debug " + "-lowering-options=disallowLocalVariables " + f"--split-verilog -o={build_dir}" ) - verilog_command += "--disable-annotation-unknown -strip-debug-info -O=debug " - verilog_command += "-lowering-options=disallowLocalVariables " - verilog_command += f"--split-verilog -o={build_dir}" result = stream_run_logger( cmd=verilog_command, @@ -94,10 +90,25 @@ async def handler(data, context): # ================================================================================== context.logger.info("Step 2: Running Yosys synthesis for area estimation...") - # Collect all Verilog/SystemVerilog sources - vsrcs = glob.glob(f"{build_dir}/**/*.v", recursive=True) + glob.glob( - f"{build_dir}/**/*.sv", recursive=True - ) + # Collect SystemVerilog sources only (.sv), skip .v files which may contain DPI-C imports + vsrcs = glob.glob(f"{build_dir}/**/*.sv", recursive=True) + + # Filter out files with unsupported SystemVerilog syntax (e.g. '{ assignment patterns) + # These are mostly TileLink bus infra / monitors that Yosys can't parse + filtered_vsrcs = [] + skipped = [] + for f in vsrcs: + with open(f, "r") as fh: + content = fh.read() + if "'{" in content or 'import "DPI-C"' in content: + skipped.append(os.path.basename(f)) + else: + filtered_vsrcs.append(f) + vsrcs = filtered_vsrcs + + if skipped: + context.logger.info(f"Skipped {len(skipped)} files with unsupported syntax: {', '.join(skipped[:10])}{'...' if len(skipped) > 10 else ''}") + context.logger.info(f"Found {len(vsrcs)} synthesizable SystemVerilog source files") if not vsrcs: context.logger.error(f"No Verilog source files found in {build_dir}") @@ -122,15 +133,20 @@ async def handler(data, context): f.write(f"{read_commands}\n") f.write(f"hierarchy -top {top_module}\n") f.write("proc\n") - f.write("flatten\n") f.write("opt\n") f.write(f"synth -top {top_module}\n") if liberty and os.path.exists(liberty): - # Map to standard cells using liberty file for accurate area context.logger.info(f"Using liberty file: {liberty}") f.write(f"dfflibmap -liberty {liberty}\n") f.write(f"abc -liberty {liberty}\n") + + # Hierarchical area breakdown BEFORE flatten (per-module with liberty area) + f.write(f"tee -o {yosys_output_dir}/hierarchy_report.txt stat -liberty {liberty}\n") + + # Flatten and report total area + f.write("flatten\n") + f.write("opt\n") f.write(f"stat -liberty {liberty}\n") f.write(f"tee -o {yosys_output_dir}/area_report.txt stat -liberty {liberty}\n") # Write mapped netlist for OpenSTA timing analysis @@ -138,6 +154,10 @@ async def handler(data, context): else: if liberty: context.logger.warn(f"Liberty file not found: {liberty}, falling back to generic stat") + # Hierarchical breakdown before flatten + f.write(f"tee -o {yosys_output_dir}/hierarchy_report.txt stat\n") + f.write("flatten\n") + f.write("opt\n") f.write("stat\n") f.write(f"tee -o {yosys_output_dir}/area_report.txt stat\n") @@ -206,6 +226,12 @@ async def handler(data, context): extra["area_report"] = f.read() context.logger.info(f"Area report saved to {report_file}") + hierarchy_file = f"{yosys_output_dir}/hierarchy_report.txt" + if os.path.exists(hierarchy_file): + with open(hierarchy_file, "r") as f: + extra["hierarchy_report"] = f.read() + context.logger.info(f"Hierarchy report saved to {hierarchy_file}") + success_result, failure_result = await check_result( context, result.returncode, diff --git a/bbdev/api/steps/yosys/scripts/yosys-config.yaml b/bbdev/api/steps/yosys/scripts/yosys-config.yaml index 5fbe43c54..52bd76719 100644 --- a/bbdev/api/steps/yosys/scripts/yosys-config.yaml +++ b/bbdev/api/steps/yosys/scripts/yosys-config.yaml @@ -7,11 +7,11 @@ # liberty: /path/to/your/standard_cell.lib liberty: -# Default top module name (override with --top) -top: TargetBall +# Default top module name (override with --top on CLI) +top: BuckyballAccelerator -# Default ball type (override with --balltype) -balltype: vecball +# Elaborate config class (override with --config on CLI) +elaborate_config: sims.verilator.BuckyballToyVerilatorConfig # OpenSTA timing analysis settings # Clock period in nanoseconds (used for STA constraint) diff --git a/bbdev/bbdev b/bbdev/bbdev index 9bc2fdf50..850f4d65f 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -248,7 +248,10 @@ def parse_args(argv: list[str]) -> argparse.Namespace: nargs="?", const="", metavar="ARGS", - help='Run yosys synthesis for area estimation. Args: "[--balltype ] [--top ]"\n' + help='Run yosys synthesis for area estimation.\n' + 'Args: "[--top ] [--config ]"\n' + ' --top : top module for synthesis (default from yosys-config.yaml)\n' + ' --config : Elaborate config class (default: sims.verilator.BuckyballToyVerilatorConfig)\n' 'Liberty library can be configured in bbdev/api/steps/yosys/scripts/yosys-config.yaml', ) From 3d133263004747706720edc3b2e327c6113dcc3e Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 12 Mar 2026 11:06:43 +0800 Subject: [PATCH 017/196] feat: add pegasus feat: replace verilog generation with bdb --- .../api/steps/pegasus/01_verilog_api_step.py | 35 +++++ .../steps/pegasus/01_verilog_event_step.py | 81 ++++++++++ .../steps/verilator/02_verilog_event_step.py | 4 +- .../steps/verilator/03_build_event_step.py | 103 ++++++++----- .../api/steps/verilator/04_cosim_api_step.py | 40 ----- .../steps/verilator/04_cosim_event_step.py | 140 ------------------ .../api/steps/verilator/04_sim_event_step.py | 58 +++----- bbdev/api/steps/verilator/05_run_api_step.py | 3 +- bbdev/bbdev | 18 ++- 9 files changed, 222 insertions(+), 260 deletions(-) create mode 100644 bbdev/api/steps/pegasus/01_verilog_api_step.py create mode 100644 bbdev/api/steps/pegasus/01_verilog_event_step.py delete mode 100644 bbdev/api/steps/verilator/04_cosim_api_step.py delete mode 100644 bbdev/api/steps/verilator/04_cosim_event_step.py diff --git a/bbdev/api/steps/pegasus/01_verilog_api_step.py b/bbdev/api/steps/pegasus/01_verilog_api_step.py new file mode 100644 index 000000000..d43ced303 --- /dev/null +++ b/bbdev/api/steps/pegasus/01_verilog_api_step.py @@ -0,0 +1,35 @@ +import asyncio +from utils.event_common import wait_for_result +from utils.path import get_buckyball_path + + +config = { + "type": "api", + "name": "Pegasus Verilog", + "description": "Generate SystemVerilog for Pegasus FPGA (PegasusHarness + ChipTop)", + "path": "/pegasus/verilog", + "method": "POST", + "emits": ["pegasus.verilog"], + "flows": ["pegasus"], +} + + +async def handler(req, context): + bbdir = get_buckyball_path() + body = req.get("body") or {} + + # Default config for Pegasus; allow override + config_name = body.get("config", "sims.pegasus.PegasusConfig") + + data = { + "config": config_name, + "output_dir": body.get("output_dir", f"{bbdir}/arch/build/pegasus/"), + } + + await context.emit({"topic": "pegasus.verilog", "data": data}) + + while True: + result = await wait_for_result(context) + if result is not None: + return result + await asyncio.sleep(1) diff --git a/bbdev/api/steps/pegasus/01_verilog_event_step.py b/bbdev/api/steps/pegasus/01_verilog_event_step.py new file mode 100644 index 000000000..663d2bd60 --- /dev/null +++ b/bbdev/api/steps/pegasus/01_verilog_event_step.py @@ -0,0 +1,81 @@ +import os +import sys + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result + + +config = { + "type": "event", + "name": "make pegasus verilog", + "description": "Generate SystemVerilog from Chisel using ElaboratePegasus", + "subscribes": ["pegasus.verilog"], + "emits": [], + "flows": ["pegasus"], +} + + +async def handler(data, context): + bbdir = get_buckyball_path() + arch_dir = f"{bbdir}/arch" + build_dir = data.get("output_dir", f"{bbdir}/arch/build/pegasus/") + + config_name = data.get("config", "sims.pegasus.PegasusConfig") + context.logger.info(f"[pegasus] Elaborating config: {config_name}") + context.logger.info(f"[pegasus] Output directory: {build_dir}") + + os.makedirs(build_dir, exist_ok=True) + + # Use ElaboratePegasus (not the generic sims.verilator.Elaborate) + # ElaboratePegasus instantiates PegasusHarness directly, bypassing TestHarness + command = ( + f"mill -i __.test.runMain sims.pegasus.ElaboratePegasus " + f"--disable-annotation-unknown " + f"-strip-debug-info " + f"-O=debug " + f"--split-verilog " + f"-o={build_dir}" + ) + + result = stream_run_logger( + cmd=command, + logger=context.logger, + cwd=arch_dir, + stdout_prefix="pegasus verilog", + stderr_prefix="pegasus verilog", + ) + + # Clean up stray top-level file if emitted next to arch/ + for stray in ["PegasusHarness.sv", "TestHarness.sv"]: + stray_path = f"{arch_dir}/{stray}" + if os.path.exists(stray_path): + os.remove(stray_path) + + # Copy generated SV files to pegasus/vivado/generated/ for Vivado build + vivado_gen_dir = f"{bbdir}/pegasus/vivado/generated" + if result.returncode == 0 and os.path.isdir(build_dir): + import shutil + os.makedirs(vivado_gen_dir, exist_ok=True) + sv_files = [f for f in os.listdir(build_dir) if f.endswith(".sv") or f.endswith(".v")] + for f in sv_files: + shutil.copy2(os.path.join(build_dir, f), os.path.join(vivado_gen_dir, f)) + context.logger.info(f"[pegasus] Copied {len(sv_files)} files to {vivado_gen_dir}") + + success_result, failure_result = await check_result( + context, + result.returncode, + continue_run=False, + extra_fields={ + "task": "verilog", + "output_dir": build_dir, + "vivado_gen_dir": vivado_gen_dir, + "top_module": "PegasusHarness", + }, + ) + + return diff --git a/bbdev/api/steps/verilator/02_verilog_event_step.py b/bbdev/api/steps/verilator/02_verilog_event_step.py index fd84c8ecc..444d40b42 100644 --- a/bbdev/api/steps/verilator/02_verilog_event_step.py +++ b/bbdev/api/steps/verilator/02_verilog_event_step.py @@ -52,7 +52,7 @@ async def handler(data, context): f"mill -i __.test.runMain sims.verify.BallTopMain {data.get('balltype')} " ) else: - command = f"mill -i __.test.runMain sims.verilator.Elaborate {config_name} " + command = f"mill -i __.test.runMain sims.verilator.BBSimElaborate {config_name} " command += "--disable-annotation-unknown -strip-debug-info -O=debug " command += f"--split-verilog -o={build_dir}" @@ -66,7 +66,7 @@ async def handler(data, context): ) # Remove unwanted file - topname_file = f"{arch_dir}/TestHarness.sv" + topname_file = f"{arch_dir}/BBSimHarness.sv" if os.path.exists(topname_file): os.remove(topname_file) diff --git a/bbdev/api/steps/verilator/03_build_event_step.py b/bbdev/api/steps/verilator/03_build_event_step.py index f2cfc1ac0..72478a85c 100644 --- a/bbdev/api/steps/verilator/03_build_event_step.py +++ b/bbdev/api/steps/verilator/03_build_event_step.py @@ -17,7 +17,7 @@ "name": "make build", "description": "build verilator executable", "subscribes": ["verilator.build"], - "emits": ["verilator.sim", "verilator.cosim"], + "emits": ["verilator.sim"], "flows": ["verilator"], } @@ -26,15 +26,11 @@ async def handler(data, context): bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" build_dir = f"{arch_dir}/build" - waveform_dir = f"{arch_dir}/waveform" - log_dir = f"{arch_dir}/log" - cosim = data.get("cosim", False) coverage = data.get("coverage", False) - # ================================================================================== - # Execute operation # ================================================================================== # Find sources + # ================================================================================== vsrcs = glob.glob(f"{build_dir}/**/*.v", recursive=True) + glob.glob( f"{build_dir}/**/*.sv", recursive=True ) @@ -47,33 +43,71 @@ async def handler(data, context): + glob.glob(f"{build_dir}/**/*.cpp", recursive=True) ) - # Setup paths: fesvr from bebop/host/spike/riscv-isa-sim (install/include, install/lib) - bebop_isa_sim = f"{bbdir}/bebop/host/spike/riscv-isa-sim" + # Exclude testchipip's SimDRAM.cc — our SimDRAM_bb.cc overrides memory_init + csrcs = [f for f in csrcs if not f.endswith("SimDRAM.cc") or "src/csrc" in f] + + # Patch fesvr includes out of build/mm.h and build/mm.cc. + # These files are auto-copied from testchipip by Verilator as SimDRAM.v + # companion sources. They reference fesvr/memif.h which we don't have + # (fesvr is removed). The memif_t dependency was only used by SimDRAM.cc's + # load_elf — our SimDRAM_bb.cc doesn't use it. + for patch_file in [f"{build_dir}/mm.h", f"{build_dir}/mm.cc"]: + if os.path.exists(patch_file): + with open(patch_file, "r") as f: + content = f.read() + patched = "\n".join( + line for line in content.splitlines() + if "fesvr/memif.h" not in line and "fesvr/elfloader.h" not in line + ) + if patched != content: + with open(patch_file, "w") as f: + f.write(patched) + context.logger.info(f"Patched fesvr includes from {patch_file}") + + topname = "BBSimHarness" + + # ================================================================================== + # Build flags + # ================================================================================== + dramsim2_dir = f"{arch_dir}/thirdparty/chipyard/tools/DRAMSim2" + + # Find readline headers/libs in nix store (not in standard paths under nix) + rl_headers = glob.glob("/nix/store/*readline*-dev/include/readline/readline.h") + readline_inc = os.path.dirname(os.path.dirname(rl_headers[0])) if rl_headers else "" + rl_libs = glob.glob("/nix/store/*readline*/lib/libreadline.so") + readline_lib = os.path.dirname(rl_libs[0]) if rl_libs else "" + + # Find zlib headers/libs in nix store + zlib_headers = glob.glob("/nix/store/*zlib*-dev/include/zlib.h") + if not zlib_headers: + zlib_headers = glob.glob("/nix/store/*zlib*/include/zlib.h") + zlib_inc = os.path.dirname(zlib_headers[0]) if zlib_headers else "" + zlib_libs = glob.glob("/nix/store/*zlib*/lib/libz.so") + zlib_lib = os.path.dirname(zlib_libs[0]) if zlib_libs else "" + inc_paths = [ - os.environ.get("RISCV", "") + "/include" if os.environ.get("RISCV") else "", - f"{arch_dir}/thirdparty/chipyard/tools/DRAMSim2", - f"{bebop_isa_sim}/install/include", + dramsim2_dir, build_dir, f"{arch_dir}/src/csrc/include", ] + if readline_inc: + inc_paths.append(readline_inc) + if zlib_inc: + inc_paths.append(zlib_inc) inc_flags = " ".join([f"-I{p}" for p in inc_paths if p]) - if cosim: - topname = "ToyBuckyball" - else: - topname = "TestHarness" + # -DBBSIM: selects VBBSimHarness in bdb.h / main.cc + cflags = f"{inc_flags} -DBBSIM -DTOP_NAME='\"V{topname}\"' -std=c++17" - cflags = f"{inc_flags} -DTOP_NAME='\"V{topname}\"' -std=c++17 " - if cosim: - cflags += " -DCOSIM" ldflags = ( - f"-lreadline -ldramsim -lfesvr -lstdc++ " - f"-L{bebop_isa_sim}/install/lib " + f"-lreadline -ldramsim -lstdc++ -lz " f"-L{bbdir}/result/lib " - f"-L{arch_dir}/thirdparty/chipyard/tools/DRAMSim2 " - f"-L{arch_dir}/thirdparty/chipyard/toolchains/riscv-tools/riscv-isa-sim/build " - f"-L{arch_dir}/thirdparty/chipyard/toolchains/riscv-tools/riscv-isa-sim/build/lib" + f"-L{dramsim2_dir} " ) + if readline_lib: + ldflags += f"-L{readline_lib} -Wl,-rpath,{readline_lib} " + if zlib_lib: + ldflags += f"-L{zlib_lib} -Wl,-rpath,{zlib_lib} " obj_dir = f"{build_dir}/obj_dir" subprocess.run(f"rm -rf {obj_dir}", shell=True) @@ -82,7 +116,7 @@ async def handler(data, context): sources = " ".join(vsrcs + csrcs) jobs = data.get("jobs", "") - # Fix nix runtime library paths: embed rpath so binary finds nix libstdc++/glibc + # Fix nix runtime library paths libstdcpp_path = subprocess.run( "g++ -print-file-name=libstdc++.so", shell=True, capture_output=True, text=True ).stdout.strip() @@ -90,15 +124,18 @@ async def handler(data, context): nix_lib_dir = os.path.dirname(os.path.realpath(libstdcpp_path)) ldflags += f" -Wl,-rpath,{nix_lib_dir}" - # Enable ccache for verilator C++ compilation via OBJCACHE + # Enable ccache if available if subprocess.run("command -v ccache", shell=True, capture_output=True).returncode == 0: os.environ["OBJCACHE"] = "ccache" - # Build acceleration: lld for faster linking (available in nix env) + # Use lld for faster linking if available use_lld = subprocess.run("command -v ld.lld", shell=True, capture_output=True).returncode == 0 if use_lld: ldflags += " -fuse-ld=lld" + # ================================================================================== + # Run verilator + # ================================================================================== verilator_cmd = ( f"verilator -MMD -cc --vpi --trace -O3 --x-assign fast --x-initial fast --noassert -Wno-fatal " f"--trace-fst --trace-threads 1 --output-split 10000 --output-split-cfuncs 100 " @@ -148,17 +185,9 @@ async def handler(data, context): extra_fields={"task": "build"}, ) - # ================================================================================== - # Continue routing - # ================================================================================== if data.get("from_run_workflow"): - if cosim: - await context.emit( - {"topic": "verilator.cosim", "data": {**data, "task": "run"}} - ) - else: - await context.emit( - {"topic": "verilator.sim", "data": {**data, "task": "run"}} - ) + await context.emit( + {"topic": "verilator.sim", "data": {**data, "task": "run"}} + ) return diff --git a/bbdev/api/steps/verilator/04_cosim_api_step.py b/bbdev/api/steps/verilator/04_cosim_api_step.py deleted file mode 100644 index cf0c28209..000000000 --- a/bbdev/api/steps/verilator/04_cosim_api_step.py +++ /dev/null @@ -1,40 +0,0 @@ -import os -import sys -import asyncio -from utils.event_common import wait_for_result - -config = { - "type": "api", - "name": "Verilator Cosim", - "description": "run verilator cosimulation", - "path": "/verilator/cosim", - "method": "POST", - "emits": ["verilator.cosim"], - "flows": ["verilator"], -} - - -async def handler(req, context): - body = req.get("body") or {} - binary = body.get("binary", "") - batch = body.get("batch", False) - if not binary: - return { - "status": 400, - "body": { - "success": False, - "failure": True, - "returncode": 400, - "message": "binary parameter is required", - }, - } - - await context.emit({"topic": "verilator.cosim", "data": {**body, "task": "cosim"}}) - # ================================================================================== - # Wait for simulation result - # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) diff --git a/bbdev/api/steps/verilator/04_cosim_event_step.py b/bbdev/api/steps/verilator/04_cosim_event_step.py deleted file mode 100644 index f0b84ac04..000000000 --- a/bbdev/api/steps/verilator/04_cosim_event_step.py +++ /dev/null @@ -1,140 +0,0 @@ -import os -import subprocess -import sys -from datetime import datetime - -# Add the utils directory to the Python path -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from utils.search_workload import search_workload -from utils.event_common import check_result - -config = { - "type": "event", - "name": "make cosim", - "description": "run cosimulation", - "subscribes": ["verilator.cosim"], - "emits": [], - "flows": ["verilator"], -} - - -async def handler(data, context): - # ================================================================================== - # Get simulation parameters - # ================================================================================== - bbdir = get_buckyball_path() - arch_dir = f"{bbdir}/arch" - build_dir = f"{arch_dir}/build" - - # Generate timestamp - timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - - binary_name = data.get("binary", "") - coverage = data.get("coverage", False) - success_result, failure_result = await check_result( - context, returncode=(binary_name == None), continue_run=True - ) - - binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) - success_result, failure_result = await check_result( - context, returncode=(binary_path == None), continue_run=True - ) - if failure_result: - context.logger.error("binary not found", failure_result) - return - - # Create log and waveform directory - log_dir = f"{arch_dir}/log/{timestamp}-{binary_name}" - waveform_dir = f"{arch_dir}/waveform/{timestamp}-{binary_name}" - topname = "ToyBuckyball" - - os.makedirs(log_dir, exist_ok=True) - os.makedirs(waveform_dir, exist_ok=True) - - # Coverage data output path - coverage_flag = "" - if coverage: - coverage_dat_path = f"{log_dir}/coverage.dat" - coverage_flag = f"+verilator+coverage+file+{coverage_dat_path}" - - bin_path = f"{build_dir}/obj_dir/V{topname}" - batch = data.get("batch", False) - - # Create log and waveform file - log_path = f"{log_dir}/bdb.log" - fst_path = f"{waveform_dir}/waveform.fst" - # Remove old waveform file - subprocess.run(f"rm -f {waveform_dir}/waveform.vcd", shell=True, check=True) - - # ================================================================================== - # Execute simulation script with streaming output - # ================================================================================== - bebop_isa_sim = f"{bbdir}/bebop/host/spike/riscv-isa-sim" - ld_lib_path = ( - f"{bebop_isa_sim}/install/lib:" - f"{bbdir}/result/lib:" - f"{arch_dir}/thirdparty/chipyard/tools/DRAMSim2" - ) - sim_cmd = ( - f"export LD_LIBRARY_PATH=\"{ld_lib_path}:$LD_LIBRARY_PATH\"; " - f"{bin_path} +permissive +loadmem={binary_path} +loadmem_addr=800000000 " - f"{'+batch ' if batch else ''} " - f"{coverage_flag + ' ' if coverage_flag else ''}" - f"+fst={fst_path} +log={log_path} +permissive-off " - f"{binary_path} > >(tee {log_dir}/stdout.log) 2> >(spike-dasm > {log_dir}/disasm.log)" - ) - script_dir = os.path.dirname(__file__) - - result = stream_run_logger( - cmd=sim_cmd, - logger=context.logger, - cwd=script_dir, - stdout_prefix="verilator sim", - stderr_prefix="verilator sim", - executable="bash", - ) - success_result, failure_result = await check_result( - context, returncode=result.returncode, continue_run=True - ) - if failure_result: - context.logger.error("sim failed", failure_result) - return - - if os.path.exists(f"{waveform_dir}/waveform.fst.heir"): - subprocess.run( - f"gtkwave -f {waveform_dir}/waveform.fst -H {waveform_dir}/waveform.fst.heir", - shell=True, - check=True, - ) - - # ================================================================================== - # Return simulation result - # ================================================================================== - # This is the end point of the run workflow, status will no longer be set to processing - extra_fields = { - "task": "sim", - "binary": binary_path, - "log_dir": log_dir, - "waveform_dir": waveform_dir, - "timestamp": timestamp, - } - if coverage: - extra_fields["coverage_dat"] = coverage_dat_path - - success_result, failure_result = await check_result( - context, - result.returncode, - continue_run=False, - extra_fields=extra_fields, - ) - - # ================================================================================== - # Finish workflow - # ================================================================================== - - return diff --git a/bbdev/api/steps/verilator/04_sim_event_step.py b/bbdev/api/steps/verilator/04_sim_event_step.py index 9f25161a6..c07b9e55c 100644 --- a/bbdev/api/steps/verilator/04_sim_event_step.py +++ b/bbdev/api/steps/verilator/04_sim_event_step.py @@ -31,33 +31,30 @@ async def handler(data, context): arch_dir = f"{bbdir}/arch" build_dir = f"{arch_dir}/build" - # Generate timestamp timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") binary_name = data.get("binary", "") coverage = data.get("coverage", False) success_result, failure_result = await check_result( - context, returncode=(binary_name == None), continue_run=True + context, returncode=(binary_name is None), continue_run=True ) binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) - context.logger.info(f"binary_path: {binary_path}") + context.logger.info(f"binary_path: {binary_path}") success_result, failure_result = await check_result( - context, returncode=(binary_path == None), continue_run=True + context, returncode=(binary_path is None), continue_run=True ) if failure_result: context.logger.error("binary not found", failure_result) return - # Create log and waveform directory + topname = "BBSimHarness" log_dir = f"{arch_dir}/log/{timestamp}-{binary_name}" waveform_dir = f"{arch_dir}/waveform/{timestamp}-{binary_name}" - topname = "TestHarness" os.makedirs(log_dir, exist_ok=True) os.makedirs(waveform_dir, exist_ok=True) - # Coverage data output path coverage_flag = "" if coverage: coverage_dat_path = f"{log_dir}/coverage.dat" @@ -66,28 +63,27 @@ async def handler(data, context): bin_path = f"{build_dir}/obj_dir/V{topname}" batch = data.get("batch", False) - # Create log and waveform file - log_path = f"{log_dir}/bdb.log" - fst_path = f"{waveform_dir}/waveform.fst" - # Remove old waveform file - subprocess.run(f"rm -f {waveform_dir}/waveform.vcd", shell=True, check=True) + log_path = f"{log_dir}/bdb.log" + stdout_path = f"{log_dir}/stdout.log" + fst_path = f"{waveform_dir}/waveform.fst" # ================================================================================== - # Execute simulation script with streaming output + # Execute simulation + # BBSimHarness uses +elf= for ELF loading (via SimDRAM_bb.cc / libelf) + # No fesvr, no +loadmem_addr needed # ================================================================================== - bebop_isa_sim = f"{bbdir}/bebop/host/spike/riscv-isa-sim" ld_lib_path = ( - f"{bebop_isa_sim}/install/lib:" f"{bbdir}/result/lib:" f"{arch_dir}/thirdparty/chipyard/tools/DRAMSim2" ) sim_cmd = ( f"export LD_LIBRARY_PATH=\"{ld_lib_path}:$LD_LIBRARY_PATH\"; " - f"{bin_path} +permissive +loadmem={binary_path} +loadmem_addr=800000000 " - f"{'+batch ' if batch else ''} " + f"{bin_path} +permissive " + f"+elf={binary_path} " + f"{'+batch ' if batch else ''}" f"{coverage_flag + ' ' if coverage_flag else ''}" - f"+fst={fst_path} +log={log_path} +permissive-off " - f"{binary_path} > >(tee {log_dir}/stdout.log) 2> >(spike-dasm > {log_dir}/disasm.log)" + f"+fst={fst_path} +log={log_path} +stdout={stdout_path} +permissive-off " + f"{binary_path} 2> >(spike-dasm > {log_dir}/disasm.log)" ) script_dir = os.path.dirname(__file__) @@ -106,24 +102,16 @@ async def handler(data, context): context.logger.error("sim failed", failure_result) return - if os.path.exists(f"{waveform_dir}/waveform.fst.heir"): - subprocess.run( - f"gtkwave -f {waveform_dir}/waveform.fst -H {waveform_dir}/waveform.fst.heir", - shell=True, - check=True, - ) - # ================================================================================== # Return simulation result # ================================================================================== - # This is the end point of the run workflow, status will no longer be set to processing extra_fields = { - "task": "sim", - "binary": binary_path, - "log_dir": log_dir, - "waveform_dir": waveform_dir, - "timestamp": timestamp, - } + "task": "sim", + "binary": binary_path, + "log_dir": log_dir, + "waveform_dir": waveform_dir, + "timestamp": timestamp, + } if coverage: extra_fields["coverage_dat"] = coverage_dat_path @@ -134,8 +122,4 @@ async def handler(data, context): extra_fields=extra_fields, ) - # ================================================================================== - # Finish workflow - # ================================================================================== - return diff --git a/bbdev/api/steps/verilator/05_run_api_step.py b/bbdev/api/steps/verilator/05_run_api_step.py index 02c366a61..f0de83558 100644 --- a/bbdev/api/steps/verilator/05_run_api_step.py +++ b/bbdev/api/steps/verilator/05_run_api_step.py @@ -17,10 +17,9 @@ async def handler(req, context): config = { "binary": body.get("binary", ""), - "config": body.get("config", "sims.verilator.BuckyballToyVerilatorConfig"), + "config": body.get("config", "sims.verilator.BuckyballToyBBSimConfig"), "jobs": body.get("jobs", "16"), "batch": body.get("batch", False), - "cosim": body.get("cosim", False), "coverage": body.get("coverage", False), "from_run_workflow": True, } diff --git a/bbdev/bbdev b/bbdev/bbdev index 850f4d65f..8110d68a4 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -267,6 +267,18 @@ def parse_args(argv: list[str]) -> argparse.Namespace: help='Generate verilog files from chisel. Args example: "[--config sims.palladium.BuckyballToyP2EConfig]"', ) + # ===== pegasus subcommand =================================================================== + pegasus_parser = subparsers.add_parser("pegasus", help="pegasus operations") + pegasus_group = pegasus_parser.add_mutually_exclusive_group(required=True) + pegasus_group.add_argument( + "--verilog", + type=str, + nargs="?", + const="", + metavar="ARGS", + help='Generate verilog files from chisel. Args example: "[--config sims.pegasus.PegasusConfig]"', + ) + # Parse arguments, allowing unknown arguments (so --port can work in subcommands) args, unknown = parser.parse_known_args(argv) @@ -417,6 +429,7 @@ if __name__ == "__main__": "uvm", "palladium", "yosys", + "pegasus", ]: api_path = f"/{cmd_info['command']}/{cmd_info['operation']}" json_data = json.dumps(cmd_info["args"]) @@ -438,7 +451,7 @@ if __name__ == "__main__": print(f"Unknown command: {cmd_info['command']}") print( "Available commands: start, stop, verilator, vcs, sardine, agent, \ - workload, doc, marshal, firesim, compiler, funcsim, uvm, yosys" + workload, doc, marshal, firesim, compiler, funcsim, uvm, yosys, pegasus" ) sys.exit(1) else: # script mode @@ -460,6 +473,7 @@ if __name__ == "__main__": "uvm", "palladium", "yosys", + "pegasus", ]: # 0. Clean up stale BullMQ/Redis data to prevent replaying old events aof_dir = os.path.join(workflow_dir, ".motia", "appendonlydir") @@ -555,6 +569,6 @@ if __name__ == "__main__": print(f"Unknown command: {cmd_info['command']}") print( "Available commands: start, stop, verilator, vcs, sardine, agent, \ - workload, doc, marshal, firesim, compiler, funcsim, uvm, yosys" + workload, doc, marshal, firesim, compiler, funcsim, uvm, yosys, pegasus" ) sys.exit(1) From 30ec0f6c7d7b56415a65b4dc522f9020d290ea42 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 12 Mar 2026 18:17:01 +0800 Subject: [PATCH 018/196] refactor: streamline verilator API steps --- bbdev/api/steps/verilator/03_build_api_step.py | 1 - bbdev/api/steps/verilator/05_run_api_step.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bbdev/api/steps/verilator/03_build_api_step.py b/bbdev/api/steps/verilator/03_build_api_step.py index a5e4115ef..658471645 100644 --- a/bbdev/api/steps/verilator/03_build_api_step.py +++ b/bbdev/api/steps/verilator/03_build_api_step.py @@ -16,7 +16,6 @@ async def handler(req, context): body = req.get("body") or {} data = { "jobs": body.get("jobs", 16), - "cosim": body.get("cosim", False), "coverage": body.get("coverage", False), } await context.emit({"topic": "verilator.build", "data": data}) diff --git a/bbdev/api/steps/verilator/05_run_api_step.py b/bbdev/api/steps/verilator/05_run_api_step.py index f0de83558..1ce1a7636 100644 --- a/bbdev/api/steps/verilator/05_run_api_step.py +++ b/bbdev/api/steps/verilator/05_run_api_step.py @@ -4,7 +4,7 @@ config = { "type": "api", "name": "Verilator Complete Workflow", - "description": "trigger complete verilator workflow", + "description": "trigger complete verilator workflow: clean → verilog → build → sim", "path": "/verilator/run", "method": "POST", "emits": ["verilator.run"], @@ -17,7 +17,7 @@ async def handler(req, context): config = { "binary": body.get("binary", ""), - "config": body.get("config", "sims.verilator.BuckyballToyBBSimConfig"), + "config": body.get("config", "sims.verilator.BuckyballToyVerilatorConfig"), "jobs": body.get("jobs", "16"), "batch": body.get("batch", False), "coverage": body.get("coverage", False), From b7d445db85d3b723ac95c5c641d52d9b36d9ceb0 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 12 Mar 2026 19:10:05 +0800 Subject: [PATCH 019/196] refactor: patch fesvr includes in Verilator build steps and streamline include paths --- .../steps/verilator/02_verilog_event_step.py | 35 +++++++-- .../steps/verilator/03_build_event_step.py | 72 ++++++------------- 2 files changed, 52 insertions(+), 55 deletions(-) diff --git a/bbdev/api/steps/verilator/02_verilog_event_step.py b/bbdev/api/steps/verilator/02_verilog_event_step.py index 444d40b42..185015013 100644 --- a/bbdev/api/steps/verilator/02_verilog_event_step.py +++ b/bbdev/api/steps/verilator/02_verilog_event_step.py @@ -25,9 +25,8 @@ async def handler(data, context): bbdir = get_buckyball_path() build_dir = data.get("output_dir", f"{bbdir}/arch/build/") arch_dir = f"{bbdir}/arch" - - # Get config name, must be provided config_name = data.get("config") + if not config_name or config_name == "None": context.logger.error("Configuration name is required but not provided") success_result, failure_result = await check_result( @@ -65,10 +64,34 @@ async def handler(data, context): stderr_prefix="verilator verilog", ) - # Remove unwanted file - topname_file = f"{arch_dir}/BBSimHarness.sv" - if os.path.exists(topname_file): - os.remove(topname_file) + # Remove testchipip C++ sources that depend on fesvr (which we don't have). + # SimTSI.v is kept so verilator can resolve the SimTSI module reference in BBSimHarness.sv; + # tsi_tick DPI symbol is satisfied by arch/src/csrc/src/monitor/ioe/tsi_stub.cc instead. + for unwanted in [ + f"{build_dir}/testchip_htif.cc", + f"{build_dir}/testchip_htif.h", + f"{build_dir}/testchip_tsi.cc", + f"{build_dir}/testchip_tsi.h", + f"{build_dir}/SimTSI.cc", + f"{arch_dir}/BBSimHarness.sv", + ]: + if os.path.exists(unwanted): + os.remove(unwanted) + + # Patch fesvr includes out of mm.h and mm.cc (copied from testchipip resources). + # They reference fesvr/memif.h which we don't have — our SimDRAM_bb.cc doesn't use it. + for patch_file in [f"{build_dir}/mm.h", f"{build_dir}/mm.cc"]: + if os.path.exists(patch_file): + with open(patch_file, "r") as f: + content = f.read() + patched = "\n".join( + line for line in content.splitlines() + if "fesvr/memif.h" not in line and "fesvr/elfloader.h" not in line + ) + if patched != content: + with open(patch_file, "w") as f: + f.write(patched) + context.logger.info(f"Patched fesvr includes from {patch_file}") # ================================================================================== # Return result to API diff --git a/bbdev/api/steps/verilator/03_build_event_step.py b/bbdev/api/steps/verilator/03_build_event_step.py index 72478a85c..fffff2fb2 100644 --- a/bbdev/api/steps/verilator/03_build_event_step.py +++ b/bbdev/api/steps/verilator/03_build_event_step.py @@ -46,68 +46,42 @@ async def handler(data, context): # Exclude testchipip's SimDRAM.cc — our SimDRAM_bb.cc overrides memory_init csrcs = [f for f in csrcs if not f.endswith("SimDRAM.cc") or "src/csrc" in f] - # Patch fesvr includes out of build/mm.h and build/mm.cc. - # These files are auto-copied from testchipip by Verilator as SimDRAM.v - # companion sources. They reference fesvr/memif.h which we don't have - # (fesvr is removed). The memif_t dependency was only used by SimDRAM.cc's - # load_elf — our SimDRAM_bb.cc doesn't use it. - for patch_file in [f"{build_dir}/mm.h", f"{build_dir}/mm.cc"]: - if os.path.exists(patch_file): - with open(patch_file, "r") as f: - content = f.read() - patched = "\n".join( - line for line in content.splitlines() - if "fesvr/memif.h" not in line and "fesvr/elfloader.h" not in line - ) - if patched != content: - with open(patch_file, "w") as f: - f.write(patched) - context.logger.info(f"Patched fesvr includes from {patch_file}") + # Exclude testchipip's TSI/HTIF C++ sources (deleted in verilog step, but guard anyway). + # tsi_tick DPI symbol is provided by tsi_stub.cc in src/csrc instead. + _tsi_htif = {"testchip_tsi.cc", "testchip_htif.cc", "SimTSI.cc"} + csrcs = [f for f in csrcs if os.path.basename(f) not in _tsi_htif] topname = "BBSimHarness" # ================================================================================== # Build flags # ================================================================================== - dramsim2_dir = f"{arch_dir}/thirdparty/chipyard/tools/DRAMSim2" - - # Find readline headers/libs in nix store (not in standard paths under nix) - rl_headers = glob.glob("/nix/store/*readline*-dev/include/readline/readline.h") - readline_inc = os.path.dirname(os.path.dirname(rl_headers[0])) if rl_headers else "" - rl_libs = glob.glob("/nix/store/*readline*/lib/libreadline.so") - readline_lib = os.path.dirname(rl_libs[0]) if rl_libs else "" - - # Find zlib headers/libs in nix store - zlib_headers = glob.glob("/nix/store/*zlib*-dev/include/zlib.h") - if not zlib_headers: - zlib_headers = glob.glob("/nix/store/*zlib*/include/zlib.h") - zlib_inc = os.path.dirname(zlib_headers[0]) if zlib_headers else "" - zlib_libs = glob.glob("/nix/store/*zlib*/lib/libz.so") - zlib_lib = os.path.dirname(zlib_libs[0]) if zlib_libs else "" - - inc_paths = [ - dramsim2_dir, - build_dir, - f"{arch_dir}/src/csrc/include", - ] - if readline_inc: - inc_paths.append(readline_inc) - if zlib_inc: - inc_paths.append(zlib_inc) - inc_flags = " ".join([f"-I{p}" for p in inc_paths if p]) + result_dir = f"{bbdir}/result" + + def pkg_config(flag, pkg): + r = subprocess.run(["pkg-config", flag, pkg], capture_output=True, text=True) + return r.stdout.strip() if r.returncode == 0 else "" + + readline_inc = pkg_config("--variable=includedir", "readline") + readline_lib = pkg_config("--variable=libdir", "readline") + zlib_lib = pkg_config("--variable=libdir", "zlib") + + inc_flags = " ".join([ + f"-I{result_dir}/include", + f"-I{build_dir}", + f"-I{arch_dir}/src/csrc/include", + f"-I{readline_inc}", + ]) # -DBBSIM: selects VBBSimHarness in bdb.h / main.cc cflags = f"{inc_flags} -DBBSIM -DTOP_NAME='\"V{topname}\"' -std=c++17" ldflags = ( f"-lreadline -ldramsim -lstdc++ -lz " - f"-L{bbdir}/result/lib " - f"-L{dramsim2_dir} " + f"-L{result_dir}/lib " + f"-L{readline_lib} -Wl,-rpath,{readline_lib} " + f"-L{zlib_lib} -Wl,-rpath,{zlib_lib} " ) - if readline_lib: - ldflags += f"-L{readline_lib} -Wl,-rpath,{readline_lib} " - if zlib_lib: - ldflags += f"-L{zlib_lib} -Wl,-rpath,{zlib_lib} " obj_dir = f"{build_dir}/obj_dir" subprocess.run(f"rm -rf {obj_dir}", shell=True) From 098309ecf94acf17ad5e32512dc99129d3b22b92 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 12 Mar 2026 21:47:52 +0800 Subject: [PATCH 020/196] fix: update verilator command to use correct elaboration target --- bbdev/api/steps/verilator/02_verilog_event_step.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbdev/api/steps/verilator/02_verilog_event_step.py b/bbdev/api/steps/verilator/02_verilog_event_step.py index 185015013..8e73bc2e7 100644 --- a/bbdev/api/steps/verilator/02_verilog_event_step.py +++ b/bbdev/api/steps/verilator/02_verilog_event_step.py @@ -51,7 +51,7 @@ async def handler(data, context): f"mill -i __.test.runMain sims.verify.BallTopMain {data.get('balltype')} " ) else: - command = f"mill -i __.test.runMain sims.verilator.BBSimElaborate {config_name} " + command = f"mill -i __.test.runMain sims.verilator.Elaborate {config_name} " command += "--disable-annotation-unknown -strip-debug-info -O=debug " command += f"--split-verilog -o={build_dir}" From a02abd703ce5f52269b8b70fabbe4f52b9893d25 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 20 Mar 2026 19:28:33 +0800 Subject: [PATCH 021/196] refactor: enhance verilator command options and update log file naming convention --- bbdev/api/steps/verilator/02_verilog_event_step.py | 14 ++++++++++++-- bbdev/api/steps/verilator/03_build_event_step.py | 3 ++- bbdev/api/steps/verilator/04_sim_event_step.py | 4 ++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/bbdev/api/steps/verilator/02_verilog_event_step.py b/bbdev/api/steps/verilator/02_verilog_event_step.py index 8e73bc2e7..47509716f 100644 --- a/bbdev/api/steps/verilator/02_verilog_event_step.py +++ b/bbdev/api/steps/verilator/02_verilog_event_step.py @@ -53,8 +53,18 @@ async def handler(data, context): else: command = f"mill -i __.test.runMain sims.verilator.Elaborate {config_name} " - command += "--disable-annotation-unknown -strip-debug-info -O=debug " - command += f"--split-verilog -o={build_dir}" + # Firtool options (CIRCT). Current set; optional Chipyard-style options below. + command += "--disable-annotation-unknown " + command += "--strip-debug-info " + command += "-O=debug " + # command += f"-repl-seq-mem -repl-seq-mem-file={build_dir}/mem.conf " + command += f"--split-verilog -o={build_dir} " + # Optional: --disable-annotation-classless (ignore classless annotations) + # Optional: -repl-seq-mem -repl-seq-mem-file=.conf (SRAM macro replacement) + # Optional: --disable-all-randomization (disable mem/reg init; may break semantics) + # Optional: --disable-opt (no optimization) or -O=release (default is release) + # Optional: --output-annotation-file= (emit annotations after lower-to-hw) + # Optional: --no-dedup (disable module dedup); --strip-fir-debug-info (FIR locators) result = stream_run_logger( cmd=command, diff --git a/bbdev/api/steps/verilator/03_build_event_step.py b/bbdev/api/steps/verilator/03_build_event_step.py index fffff2fb2..632396727 100644 --- a/bbdev/api/steps/verilator/03_build_event_step.py +++ b/bbdev/api/steps/verilator/03_build_event_step.py @@ -74,6 +74,7 @@ def pkg_config(flag, pkg): ]) # -DBBSIM: selects VBBSimHarness in bdb.h / main.cc + # BDB NDJSON trace (+trace=...) is runtime-only; bbdev sim uses +trace=all (04_sim_event_step.py). cflags = f"{inc_flags} -DBBSIM -DTOP_NAME='\"V{topname}\"' -std=c++17" ldflags = ( @@ -84,7 +85,7 @@ def pkg_config(flag, pkg): ) obj_dir = f"{build_dir}/obj_dir" - subprocess.run(f"rm -rf {obj_dir}", shell=True) + # subprocess.run(f"rm -rf {obj_dir}", shell=True) os.makedirs(obj_dir, exist_ok=True) sources = " ".join(vsrcs + csrcs) diff --git a/bbdev/api/steps/verilator/04_sim_event_step.py b/bbdev/api/steps/verilator/04_sim_event_step.py index c07b9e55c..0ec7d1efc 100644 --- a/bbdev/api/steps/verilator/04_sim_event_step.py +++ b/bbdev/api/steps/verilator/04_sim_event_step.py @@ -63,7 +63,7 @@ async def handler(data, context): bin_path = f"{build_dir}/obj_dir/V{topname}" batch = data.get("batch", False) - log_path = f"{log_dir}/bdb.log" + log_path = f"{log_dir}/bdb.ndjson" stdout_path = f"{log_dir}/stdout.log" fst_path = f"{waveform_dir}/waveform.fst" @@ -82,7 +82,7 @@ async def handler(data, context): f"+elf={binary_path} " f"{'+batch ' if batch else ''}" f"{coverage_flag + ' ' if coverage_flag else ''}" - f"+fst={fst_path} +log={log_path} +stdout={stdout_path} +permissive-off " + f"+fst={fst_path} +log={log_path} +stdout={stdout_path} +trace=all +permissive-off " f"{binary_path} 2> >(spike-dasm > {log_dir}/disasm.log)" ) script_dir = os.path.dirname(__file__) From e3b478045fba330e635de02edc6db06503cb6fc3 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 20 Mar 2026 21:12:49 +0800 Subject: [PATCH 022/196] refactor: add simulation metadata handling and update logging paths --- bbdev/api/steps/verilator/04_sim_event_step.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bbdev/api/steps/verilator/04_sim_event_step.py b/bbdev/api/steps/verilator/04_sim_event_step.py index 0ec7d1efc..ee9c14c15 100644 --- a/bbdev/api/steps/verilator/04_sim_event_step.py +++ b/bbdev/api/steps/verilator/04_sim_event_step.py @@ -65,12 +65,17 @@ async def handler(data, context): log_path = f"{log_dir}/bdb.ndjson" stdout_path = f"{log_dir}/stdout.log" + meta_path = f"{log_dir}/sim_meta.txt" fst_path = f"{waveform_dir}/waveform.fst" # ================================================================================== # Execute simulation # BBSimHarness uses +elf= for ELF loading (via SimDRAM_bb.cc / libelf) # No fesvr, no +loadmem_addr needed + # + # disasm.log: only stderr -> spike-dasm (Rocket commit printf is stderr here; + # merging stdout with 2>&1 can break: full stdio buffering + non-DASM bytes). + # BDB_SIM_META moves NDJSON banner to sim_meta.txt so it does not pollute disasm. # ================================================================================== ld_lib_path = ( f"{bbdir}/result/lib:" @@ -78,6 +83,7 @@ async def handler(data, context): ) sim_cmd = ( f"export LD_LIBRARY_PATH=\"{ld_lib_path}:$LD_LIBRARY_PATH\"; " + f"export BDB_SIM_META=\"{meta_path}\"; " f"{bin_path} +permissive " f"+elf={binary_path} " f"{'+batch ' if batch else ''}" @@ -111,6 +117,7 @@ async def handler(data, context): "log_dir": log_dir, "waveform_dir": waveform_dir, "timestamp": timestamp, + "sim_meta": meta_path, } if coverage: extra_fields["coverage_dat"] = coverage_dat_path From 4c07c71f667c81e99f956b2c8959de72f8e76cc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E4=B8=80=E9=88=8A?= Date: Thu, 2 Apr 2026 20:29:27 +0800 Subject: [PATCH 023/196] fix: correct error reports in verilog generation (#3) --- bbdev/api/steps/palladium/01_verilog_api_step.py | 12 +++++++++--- bbdev/api/steps/verilator/02_verilog_api_step.py | 12 +++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/bbdev/api/steps/palladium/01_verilog_api_step.py b/bbdev/api/steps/palladium/01_verilog_api_step.py index 975339f67..c46fd8850 100644 --- a/bbdev/api/steps/palladium/01_verilog_api_step.py +++ b/bbdev/api/steps/palladium/01_verilog_api_step.py @@ -23,10 +23,16 @@ async def handler(req, context): # Get config name, must be provided config_name = body.get("config") if not config_name or config_name == "None": + error_msg = "Configuration name is required. Please specify --config parameter." + context.logger.error(error_msg, {"config_name": config_name}) return { - "status": "error", - "message": "Configuration name is required. Please specify --config_name parameter.", - "example": './bbdev palladium --verilog "--config_name sims.palladium.BuckyballToyP2EConfig"', + "status": 400, + "body": { + "success": False, + "failure": True, + "returncode": 400, + "message": "Configuration name is required. Please specify --config parameter.", + }, } data = { diff --git a/bbdev/api/steps/verilator/02_verilog_api_step.py b/bbdev/api/steps/verilator/02_verilog_api_step.py index 5f9004234..433dd78a8 100644 --- a/bbdev/api/steps/verilator/02_verilog_api_step.py +++ b/bbdev/api/steps/verilator/02_verilog_api_step.py @@ -23,10 +23,16 @@ async def handler(req, context): # Get config name, must be provided config_name = body.get("config") if not config_name or config_name == "None": + error_msg = "Configuration name is required. Please specify --config parameter." + context.logger.error(error_msg, {"config_name": config_name}) return { - "status": "error", - "message": "Configuration name is required. Please specify --config parameter.", - "example": 'bbdev verilator --verilog "--config sims.verilator.BuckyballToyVerilatorConfig"', + "status": 400, + "body": { + "success": False, + "failure": True, + "returncode": 400, + "message": "Configuration name is required. Please specify --config parameter.", + }, } data = { From 3b9e021eb5545b351b2eb7cfade391c548cc0639 Mon Sep 17 00:00:00 2001 From: Shiroha Date: Thu, 2 Apr 2026 23:20:04 +0800 Subject: [PATCH 024/196] feat: rebase motia-1.0-migration onto main (#6) --- bbdev/.gitignore | 4 + bbdev/api/config.yaml | 60 ++ bbdev/api/pyproject.toml | 13 + bbdev/api/steps/compiler/01_build_api.step.py | 15 + bbdev/api/steps/compiler/01_build_api_step.py | 26 - ...d_event_step.py => 01_build_event.step.py} | 24 +- .../firesim/01_buildbitstream_api.step.py | 15 + .../firesim/01_buildbitstream_api_step.py | 26 - ...tep.py => 01_buildbitstream_event.step.py} | 20 +- .../steps/firesim/02_infrasetup_api.step.py | 16 + .../steps/firesim/02_infrasetup_api_step.py | 27 - ...nt_step.py => 02_infrasetup_event.step.py} | 22 +- .../steps/firesim/03_runworkload_api.step.py | 16 + .../steps/firesim/03_runworkload_api_step.py | 27 - ...t_step.py => 03_runworkload_event.step.py} | 22 +- bbdev/api/steps/marshal/01_build_api.step.py | 15 + bbdev/api/steps/marshal/01_build_api_step.py | 25 - ...d_event_step.py => 01_build_event.step.py} | 28 +- bbdev/api/steps/marshal/02_launch_api.step.py | 15 + bbdev/api/steps/marshal/02_launch_api_step.py | 26 - ..._event_step.py => 02_launch_event.step.py} | 20 +- .../steps/palladium/01_verilog_api.step.py | 36 + .../steps/palladium/01_verilog_api_step.py | 52 - ...event_step.py => 01_verilog_event.step.py} | 96 +- ...log_api_step.py => 01_verilog_api.step.py} | 26 +- .../steps/pegasus/01_verilog_event.step.py | 122 +++ .../steps/pegasus/01_verilog_event_step.py | 81 -- .../pegasus/02_buildbitstream_api.step.py | 18 + .../pegasus/02_buildbitstream_event.step.py | 83 ++ bbdev/api/steps/sardine/01_run_api.step.py | 22 + bbdev/api/steps/sardine/01_run_api_step.py | 45 - ...run_event_step.py => 01_run_event.step.py} | 43 +- ...ep.py => 02_coverage_report_event.step.py} | 46 +- .../api/steps/verilator/01_clean_api.step.py | 15 + .../api/steps/verilator/01_clean_api_step.py | 26 - ...n_event_step.py => 01_clean_event.step.py} | 32 +- .../steps/verilator/02_verilog_api.step.py | 36 + .../steps/verilator/02_verilog_api_step.py | 52 - .../steps/verilator/02_verilog_event.step.py | 93 ++ .../steps/verilator/02_verilog_event_step.py | 126 --- .../api/steps/verilator/03_build_api.step.py | 19 + .../api/steps/verilator/03_build_api_step.py | 30 - .../steps/verilator/03_build_event.step.py | 140 +++ .../steps/verilator/03_build_event_step.py | 168 --- .../api/steps/verilator/04_cosim_api.step.py | 27 + .../steps/verilator/04_cosim_event.step.py | 128 +++ bbdev/api/steps/verilator/04_sim_api.step.py | 27 + bbdev/api/steps/verilator/04_sim_api_step.py | 40 - ...sim_event_step.py => 04_sim_event.step.py} | 107 +- bbdev/api/steps/verilator/05_run_api.step.py | 25 + bbdev/api/steps/verilator/05_run_api_step.py | 53 - bbdev/api/steps/workload/01_buidl_api.step.py | 19 + bbdev/api/steps/workload/01_buidl_api_step.py | 38 - ...d_event_step.py => 01_build_event.step.py} | 24 +- ...synth_api_step.py => 01_synth_api.step.py} | 25 +- ...h_event_step.py => 01_synth_event.step.py} | 67 +- bbdev/api/utils/event_common.py | 85 +- bbdev/api/utils/port.py | 21 +- bbdev/bbdev | 956 ++++++++---------- bbdev/flake.nix | 18 +- 60 files changed, 1739 insertions(+), 1790 deletions(-) create mode 100644 bbdev/api/config.yaml create mode 100644 bbdev/api/pyproject.toml create mode 100644 bbdev/api/steps/compiler/01_build_api.step.py delete mode 100644 bbdev/api/steps/compiler/01_build_api_step.py rename bbdev/api/steps/compiler/{01_build_event_step.py => 01_build_event.step.py} (76%) create mode 100644 bbdev/api/steps/firesim/01_buildbitstream_api.step.py delete mode 100644 bbdev/api/steps/firesim/01_buildbitstream_api_step.py rename bbdev/api/steps/firesim/{01_buildbitstream_event_step.py => 01_buildbitstream_event.step.py} (78%) create mode 100644 bbdev/api/steps/firesim/02_infrasetup_api.step.py delete mode 100644 bbdev/api/steps/firesim/02_infrasetup_api_step.py rename bbdev/api/steps/firesim/{02_infrasetup_event_step.py => 02_infrasetup_event.step.py} (78%) create mode 100644 bbdev/api/steps/firesim/03_runworkload_api.step.py delete mode 100644 bbdev/api/steps/firesim/03_runworkload_api_step.py rename bbdev/api/steps/firesim/{03_runworkload_event_step.py => 03_runworkload_event.step.py} (78%) create mode 100644 bbdev/api/steps/marshal/01_build_api.step.py delete mode 100644 bbdev/api/steps/marshal/01_build_api_step.py rename bbdev/api/steps/marshal/{01_build_event_step.py => 01_build_event.step.py} (74%) create mode 100644 bbdev/api/steps/marshal/02_launch_api.step.py delete mode 100644 bbdev/api/steps/marshal/02_launch_api_step.py rename bbdev/api/steps/marshal/{02_launch_event_step.py => 02_launch_event.step.py} (77%) create mode 100644 bbdev/api/steps/palladium/01_verilog_api.step.py delete mode 100644 bbdev/api/steps/palladium/01_verilog_api_step.py rename bbdev/api/steps/palladium/{01_verilog_event_step.py => 01_verilog_event.step.py} (64%) rename bbdev/api/steps/pegasus/{01_verilog_api_step.py => 01_verilog_api.step.py} (56%) create mode 100644 bbdev/api/steps/pegasus/01_verilog_event.step.py delete mode 100644 bbdev/api/steps/pegasus/01_verilog_event_step.py create mode 100644 bbdev/api/steps/pegasus/02_buildbitstream_api.step.py create mode 100644 bbdev/api/steps/pegasus/02_buildbitstream_event.step.py create mode 100644 bbdev/api/steps/sardine/01_run_api.step.py delete mode 100644 bbdev/api/steps/sardine/01_run_api_step.py rename bbdev/api/steps/sardine/{01_run_event_step.py => 01_run_event.step.py} (55%) rename bbdev/api/steps/sardine/{02_coverage_report_event_step.py => 02_coverage_report_event.step.py} (82%) create mode 100644 bbdev/api/steps/verilator/01_clean_api.step.py delete mode 100644 bbdev/api/steps/verilator/01_clean_api_step.py rename bbdev/api/steps/verilator/{01_clean_event_step.py => 01_clean_event.step.py} (66%) create mode 100644 bbdev/api/steps/verilator/02_verilog_api.step.py delete mode 100644 bbdev/api/steps/verilator/02_verilog_api_step.py create mode 100644 bbdev/api/steps/verilator/02_verilog_event.step.py delete mode 100644 bbdev/api/steps/verilator/02_verilog_event_step.py create mode 100644 bbdev/api/steps/verilator/03_build_api.step.py delete mode 100644 bbdev/api/steps/verilator/03_build_api_step.py create mode 100644 bbdev/api/steps/verilator/03_build_event.step.py delete mode 100644 bbdev/api/steps/verilator/03_build_event_step.py create mode 100644 bbdev/api/steps/verilator/04_cosim_api.step.py create mode 100644 bbdev/api/steps/verilator/04_cosim_event.step.py create mode 100644 bbdev/api/steps/verilator/04_sim_api.step.py delete mode 100644 bbdev/api/steps/verilator/04_sim_api_step.py rename bbdev/api/steps/verilator/{04_sim_event_step.py => 04_sim_event.step.py} (50%) create mode 100644 bbdev/api/steps/verilator/05_run_api.step.py delete mode 100644 bbdev/api/steps/verilator/05_run_api_step.py create mode 100644 bbdev/api/steps/workload/01_buidl_api.step.py delete mode 100644 bbdev/api/steps/workload/01_buidl_api_step.py rename bbdev/api/steps/workload/{01_build_event_step.py => 01_build_event.step.py} (78%) rename bbdev/api/steps/yosys/{01_synth_api_step.py => 01_synth_api.step.py} (61%) rename bbdev/api/steps/yosys/{01_synth_event_step.py => 01_synth_event.step.py} (78%) diff --git a/bbdev/.gitignore b/bbdev/.gitignore index 36283a45d..a72dce8a0 100644 --- a/bbdev/.gitignore +++ b/bbdev/.gitignore @@ -17,3 +17,7 @@ result/ .cursor/ .claude/ + +# Temp iii config files generated by bbdev script mode +api/iii-config-*.yaml +api/data/ diff --git a/bbdev/api/config.yaml b/bbdev/api/config.yaml new file mode 100644 index 000000000..80926ff06 --- /dev/null +++ b/bbdev/api/config.yaml @@ -0,0 +1,60 @@ +port: 49134 + +modules: + # ── Worker Module ────────────────────────────────────────────────────── + - class: modules::worker::WorkerModule + config: + port: 49134 + + # ── State Module ─────────────────────────────────────────────────────── + - class: modules::state::StateModule + config: + adapter: + class: modules::state::adapters::KvStore + config: + store_method: file_based + file_path: ./data/state_store.db + + # ── REST API Module ──────────────────────────────────────────────────── + - class: modules::api::RestApiModule + config: + port: 3111 + host: 0.0.0.0 + default_timeout: 86400000 + concurrency_request_limit: 1024 + cors: + allowed_origins: + - http://localhost:3000 + - http://localhost:5173 + allowed_methods: + - GET + - POST + - PUT + - DELETE + - OPTIONS + + # ── Queue Module ─────────────────────────────────────────────────────── + - class: modules::queue::QueueModule + config: + adapter: + class: modules::queue::BuiltinQueueAdapter + + # ── PubSub Module ────────────────────────────────────────────────────── + - class: modules::pubsub::PubSubModule + config: + adapter: + class: modules::pubsub::LocalAdapter + + # ── Observability Module ─────────────────────────────────────────────── + - class: modules::observability::OtelModule + config: + enabled: true + exporter: memory + + # ── Exec Module (Python) ─────────────────────────────────────────────── + - class: modules::shell::ExecModule + config: + watch: + - steps/**/*.py + exec: + - PYTHONUNBUFFERED=1 PYTHONPATH=. .venv/bin/motia dev --dir steps diff --git a/bbdev/api/pyproject.toml b/bbdev/api/pyproject.toml new file mode 100644 index 000000000..ea2c242ae --- /dev/null +++ b/bbdev/api/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "bbdev" +version = "0.1.0" +description = "Agent-friendly developer toolchain backend powered by Motia" +requires-python = ">=3.10" +dependencies = [ + "motia[otel]==1.0.0rc17", + "iii-sdk==0.2.0", + "pydantic>=2.0", +] + +[project.optional-dependencies] +dev = ["pytest>=8.0.0"] diff --git a/bbdev/api/steps/compiler/01_build_api.step.py b/bbdev/api/steps/compiler/01_build_api.step.py new file mode 100644 index 000000000..63d1040ed --- /dev/null +++ b/bbdev/api/steps/compiler/01_build_api.step.py @@ -0,0 +1,15 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "build-compiler-api", + "description": "build compiler", + "flows": ["compiler"], + "triggers": [api("POST", "/compiler/build")], + "enqueues": ["compiler.build"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + await ctx.enqueue({"topic": "compiler.build", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/compiler/01_build_api_step.py b/bbdev/api/steps/compiler/01_build_api_step.py deleted file mode 100644 index e24205206..000000000 --- a/bbdev/api/steps/compiler/01_build_api_step.py +++ /dev/null @@ -1,26 +0,0 @@ -import asyncio -from utils.event_common import wait_for_result - -config = { - "type": "api", - "name": "Build Compiler", - "description": "build bitstream", - "path": "/compiler/build", - "method": "POST", - "emits": ["compiler.build"], - "flows": ["compiler"], -} - - -async def handler(req, context): - body = req.get("body") or {} - await context.emit({"topic": "compiler.build", "data": body}) - - # ================================================================================== - # Wait for build result - # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) diff --git a/bbdev/api/steps/compiler/01_build_event_step.py b/bbdev/api/steps/compiler/01_build_event.step.py similarity index 76% rename from bbdev/api/steps/compiler/01_build_event_step.py rename to bbdev/api/steps/compiler/01_build_event.step.py index 669830274..a4969d638 100644 --- a/bbdev/api/steps/compiler/01_build_event_step.py +++ b/bbdev/api/steps/compiler/01_build_event.step.py @@ -1,7 +1,8 @@ import os -import subprocess import sys +from motia import FlowContext, queue + # Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: @@ -9,19 +10,19 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger -from utils.event_common import check_result +from utils.event_common import check_result, get_origin_trace_id config = { - "type": "event", - "name": "Build Compiler", - "description": "build bitstream", - "subscribes": ["compiler.build"], - "emits": [], + "name": "build-compiler", + "description": "build compiler", "flows": ["compiler"], + "triggers": [queue("compiler.build")], + "enqueues": [], } -async def handler(data, context): +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() script_dir = f"{bbdir}/workflow/steps/compiler/scripts" yaml_dir = f"{script_dir}/yaml" @@ -31,14 +32,14 @@ async def handler(data, context): command = f"mkdir -p {bbdir}/compiler/build" result = stream_run_logger( cmd=command, - logger=context.logger, + logger=ctx.logger, stdout_prefix="compiler build", stderr_prefix="compiler build", ) command = f"cd {bbdir}/compiler/build && ninja -j{os.cpu_count()}" result = stream_run_logger( cmd=command, - logger=context.logger, + logger=ctx.logger, stdout_prefix="compiler build", stderr_prefix="compiler build", ) @@ -47,8 +48,7 @@ async def handler(data, context): # Return result to API # ================================================================================== success_result, failure_result = await check_result( - context, result.returncode, continue_run=False - ) + ctx, result.returncode, continue_run=False, trace_id=origin_tid) # ================================================================================== # Continue routing diff --git a/bbdev/api/steps/firesim/01_buildbitstream_api.step.py b/bbdev/api/steps/firesim/01_buildbitstream_api.step.py new file mode 100644 index 000000000..0b69bdcdf --- /dev/null +++ b/bbdev/api/steps/firesim/01_buildbitstream_api.step.py @@ -0,0 +1,15 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "firesim-buildbitstream-api", + "description": "build bitstream", + "flows": ["firesim"], + "triggers": [api("POST", "/firesim/buildbitstream")], + "enqueues": ["firesim.buildbitstream"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + await ctx.enqueue({"topic": "firesim.buildbitstream", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/01_buildbitstream_api_step.py b/bbdev/api/steps/firesim/01_buildbitstream_api_step.py deleted file mode 100644 index 2c5fb9e95..000000000 --- a/bbdev/api/steps/firesim/01_buildbitstream_api_step.py +++ /dev/null @@ -1,26 +0,0 @@ -import asyncio -from utils.event_common import wait_for_result - -config = { - "type": "api", - "name": "Firesim Buildbitstream", - "description": "build bitstream", - "path": "/firesim/buildbitstream", - "method": "POST", - "emits": ["firesim.buildbitstream"], - "flows": ["firesim"], -} - - -async def handler(req, context): - body = req.get("body") or {} - await context.emit({"topic": "firesim.buildbitstream", "data": body}) - - # ================================================================================== - # Wait for simulation result - # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) diff --git a/bbdev/api/steps/firesim/01_buildbitstream_event_step.py b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py similarity index 78% rename from bbdev/api/steps/firesim/01_buildbitstream_event_step.py rename to bbdev/api/steps/firesim/01_buildbitstream_event.step.py index ba943c6ea..4550ee509 100644 --- a/bbdev/api/steps/firesim/01_buildbitstream_event_step.py +++ b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py @@ -1,7 +1,8 @@ import os -import subprocess import sys +from motia import FlowContext, queue + # Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: @@ -9,19 +10,19 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger -from utils.event_common import check_result +from utils.event_common import check_result, get_origin_trace_id config = { - "type": "event", - "name": "Firesim Buildbitstream", + "name": "firesim-buildbitstream", "description": "build bitstream", - "subscribes": ["firesim.buildbitstream"], - "emits": [], "flows": ["firesim"], + "triggers": [queue("firesim.buildbitstream")], + "enqueues": [], } -async def handler(data, context): +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() script_dir = f"{bbdir}/workflow/steps/firesim/scripts" yaml_dir = f"{script_dir}/yaml" @@ -35,7 +36,7 @@ async def handler(data, context): command += f" -c {yaml_dir}/config_runtime.yaml" result = stream_run_logger( cmd=command, - logger=context.logger, + logger=ctx.logger, stdout_prefix="firesim buildbitstream", stderr_prefix="firesim buildbitstream", ) @@ -44,8 +45,7 @@ async def handler(data, context): # Return result to API # ================================================================================== success_result, failure_result = await check_result( - context, result.returncode, continue_run=False - ) + ctx, result.returncode, continue_run=False, trace_id=origin_tid) # ================================================================================== # Continue routing diff --git a/bbdev/api/steps/firesim/02_infrasetup_api.step.py b/bbdev/api/steps/firesim/02_infrasetup_api.step.py new file mode 100644 index 000000000..5beca75ca --- /dev/null +++ b/bbdev/api/steps/firesim/02_infrasetup_api.step.py @@ -0,0 +1,16 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "firesim-infrasetup-api", + "description": "infrasetup", + "flows": ["firesim"], + "triggers": [api("POST", "/firesim/infrasetup")], + "enqueues": ["firesim.infrasetup"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + data = {"jobs": body.get("jobs", 16)} + await ctx.enqueue({"topic": "firesim.infrasetup", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/02_infrasetup_api_step.py b/bbdev/api/steps/firesim/02_infrasetup_api_step.py deleted file mode 100644 index 32d3eba9a..000000000 --- a/bbdev/api/steps/firesim/02_infrasetup_api_step.py +++ /dev/null @@ -1,27 +0,0 @@ -import asyncio -from utils.event_common import wait_for_result - -config = { - "type": "api", - "name": "Firesim Infrasetup", - "description": "infrasetup", - "path": "/firesim/infrasetup", - "method": "POST", - "emits": ["firesim.infrasetup"], - "flows": ["firesim"], -} - - -async def handler(req, context): - body = req.get("body") or {} - data = {"jobs": body.get("jobs", 16)} - await context.emit({"topic": "firesim.infrasetup", "data": data}) - - # ================================================================================== - # Wait for simulation result - # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) diff --git a/bbdev/api/steps/firesim/02_infrasetup_event_step.py b/bbdev/api/steps/firesim/02_infrasetup_event.step.py similarity index 78% rename from bbdev/api/steps/firesim/02_infrasetup_event_step.py rename to bbdev/api/steps/firesim/02_infrasetup_event.step.py index fb90968c0..2d1f3167b 100644 --- a/bbdev/api/steps/firesim/02_infrasetup_event_step.py +++ b/bbdev/api/steps/firesim/02_infrasetup_event.step.py @@ -1,7 +1,8 @@ import os -import subprocess import sys +from motia import FlowContext, queue + # Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: @@ -9,19 +10,19 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger -from utils.event_common import check_result +from utils.event_common import check_result, get_origin_trace_id config = { - "type": "event", - "name": "Firesim Infrasetup", + "name": "firesim-infrasetup", "description": "infrasetup", - "subscribes": ["firesim.infrasetup"], - "emits": [], "flows": ["firesim"], + "triggers": [queue("firesim.infrasetup")], + "enqueues": [], } -async def handler(data, context): +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() script_dir = f"{bbdir}/workflow/steps/firesim/scripts" yaml_dir = f"{script_dir}/yaml" @@ -35,7 +36,7 @@ async def handler(data, context): command += f" -c {yaml_dir}/config_runtime.yaml" result = stream_run_logger( cmd=command, - logger=context.logger, + logger=ctx.logger, stdout_prefix="firesim infrasetup", stderr_prefix="firesim infrasetup", ) @@ -44,13 +45,10 @@ async def handler(data, context): # Return result to API # ================================================================================== success_result, failure_result = await check_result( - context, result.returncode, continue_run=False - ) + ctx, result.returncode, continue_run=False, trace_id=origin_tid) # ================================================================================== # Continue routing - # Routing to verilog or finish workflow - # For run workflow, continue to verilog; for standalone clean, complete # ================================================================================== return diff --git a/bbdev/api/steps/firesim/03_runworkload_api.step.py b/bbdev/api/steps/firesim/03_runworkload_api.step.py new file mode 100644 index 000000000..de1db067b --- /dev/null +++ b/bbdev/api/steps/firesim/03_runworkload_api.step.py @@ -0,0 +1,16 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "firesim-runworkload-api", + "description": "run workload", + "flows": ["firesim"], + "triggers": [api("POST", "/firesim/runworkload")], + "enqueues": ["firesim.runworkload"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + data = {"jobs": body.get("jobs", 16)} + await ctx.enqueue({"topic": "firesim.runworkload", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/03_runworkload_api_step.py b/bbdev/api/steps/firesim/03_runworkload_api_step.py deleted file mode 100644 index be0705120..000000000 --- a/bbdev/api/steps/firesim/03_runworkload_api_step.py +++ /dev/null @@ -1,27 +0,0 @@ -import asyncio -from utils.event_common import wait_for_result - -config = { - "type": "api", - "name": "Firesim Runworkload", - "description": "run workload", - "path": "/firesim/runworkload", - "method": "POST", - "emits": ["firesim.runworkload"], - "flows": ["firesim"], -} - - -async def handler(req, context): - body = req.get("body") or {} - data = {"jobs": body.get("jobs", 16)} - await context.emit({"topic": "firesim.runworkload", "data": data}) - - # ================================================================================== - # Wait for simulation result - # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) diff --git a/bbdev/api/steps/firesim/03_runworkload_event_step.py b/bbdev/api/steps/firesim/03_runworkload_event.step.py similarity index 78% rename from bbdev/api/steps/firesim/03_runworkload_event_step.py rename to bbdev/api/steps/firesim/03_runworkload_event.step.py index d79639a48..e9a57aee7 100644 --- a/bbdev/api/steps/firesim/03_runworkload_event_step.py +++ b/bbdev/api/steps/firesim/03_runworkload_event.step.py @@ -1,7 +1,8 @@ import os -import subprocess import sys +from motia import FlowContext, queue + # Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: @@ -9,19 +10,19 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger -from utils.event_common import check_result +from utils.event_common import check_result, get_origin_trace_id config = { - "type": "event", - "name": "Firesim Runworkload", + "name": "firesim-runworkload", "description": "run workload", - "subscribes": ["firesim.runworkload"], - "emits": [], "flows": ["firesim"], + "triggers": [queue("firesim.runworkload")], + "enqueues": [], } -async def handler(data, context): +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() script_dir = f"{bbdir}/workflow/steps/firesim/scripts" yaml_dir = f"{script_dir}/yaml" @@ -35,7 +36,7 @@ async def handler(data, context): command += f" -c {yaml_dir}/config_runtime.yaml" result = stream_run_logger( cmd=command, - logger=context.logger, + logger=ctx.logger, stdout_prefix="firesim runworkload", stderr_prefix="firesim runworkload", ) @@ -44,13 +45,10 @@ async def handler(data, context): # Return result to API # ================================================================================== success_result, failure_result = await check_result( - context, result.returncode, continue_run=False - ) + ctx, result.returncode, continue_run=False, trace_id=origin_tid) # ================================================================================== # Continue routing - # Routing to verilog or finish workflow - # For run workflow, continue to verilog; for standalone clean, complete # ================================================================================== return diff --git a/bbdev/api/steps/marshal/01_build_api.step.py b/bbdev/api/steps/marshal/01_build_api.step.py new file mode 100644 index 000000000..ba9049a12 --- /dev/null +++ b/bbdev/api/steps/marshal/01_build_api.step.py @@ -0,0 +1,15 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "marshal-build-api", + "description": "build marshal", + "flows": ["marshal"], + "triggers": [api("POST", "/marshal/build")], + "enqueues": ["marshal.build"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + await ctx.enqueue({"topic": "marshal.build", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/marshal/01_build_api_step.py b/bbdev/api/steps/marshal/01_build_api_step.py deleted file mode 100644 index 4c0936cfe..000000000 --- a/bbdev/api/steps/marshal/01_build_api_step.py +++ /dev/null @@ -1,25 +0,0 @@ -import asyncio -from utils.event_common import wait_for_result - -config = { - "type": "api", - "name": "Marshal Build", - "description": "build marshal", - "path": "/marshal/build", - "method": "POST", - "emits": ["marshal.build"], - "flows": ["marshal"], -} - - -async def handler(req, context): - body = req.get("body") or {} - await context.emit({"topic": "marshal.build", "data": body}) - # ================================================================================== - # Wait for result - # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) diff --git a/bbdev/api/steps/marshal/01_build_event_step.py b/bbdev/api/steps/marshal/01_build_event.step.py similarity index 74% rename from bbdev/api/steps/marshal/01_build_event_step.py rename to bbdev/api/steps/marshal/01_build_event.step.py index 34480d8c4..4698ce5cd 100644 --- a/bbdev/api/steps/marshal/01_build_event_step.py +++ b/bbdev/api/steps/marshal/01_build_event.step.py @@ -1,7 +1,8 @@ import os -import subprocess import sys +from motia import FlowContext, queue + # Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: @@ -9,19 +10,19 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger -from utils.event_common import check_result +from utils.event_common import check_result, get_origin_trace_id config = { - "type": "event", - "name": "Marshal Build", + "name": "marshal-build", "description": "build marshal", - "subscribes": ["marshal.build"], - "emits": [], "flows": ["marshal"], + "triggers": [queue("marshal.build")], + "enqueues": ["marshal.complete", "marshal.error"], } -async def handler(data, context): +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() script_dir = f"{bbdir}/workflow/steps/marshal/scripts" # ================================================================================== @@ -30,7 +31,7 @@ async def handler(data, context): command = f"./marshal -v build interactive.json && ./marshal -v install interactive.json" result = stream_run_logger( cmd=command, - logger=context.logger, + logger=ctx.logger, cwd=script_dir, stdout_prefix="marshal build", stderr_prefix="marshal build", @@ -40,26 +41,25 @@ async def handler(data, context): # Return result to API # ================================================================================== success_result, failure_result = await check_result( - context, result.returncode, continue_run=False - ) + ctx, result.returncode, continue_run=False, trace_id=origin_tid) # ================================================================================== # Continue routing # Routing to completion or error handling # ================================================================================== if result.returncode == 0: - await context.emit( + await ctx.enqueue( { "topic": "marshal.complete", - "data": {**data, "task": "marshal", "result": success_result}, + "data": {**input_data, "task": "marshal", "result": success_result}, } ) else: - await context.emit( + await ctx.enqueue( { "topic": "marshal.error", "data": { - **data, + **input_data, "task": "marshal", "result": failure_result, "returncode": result.returncode, diff --git a/bbdev/api/steps/marshal/02_launch_api.step.py b/bbdev/api/steps/marshal/02_launch_api.step.py new file mode 100644 index 000000000..4b52daf68 --- /dev/null +++ b/bbdev/api/steps/marshal/02_launch_api.step.py @@ -0,0 +1,15 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "marshal-launch-api", + "description": "launch marshal", + "flows": ["marshal"], + "triggers": [api("POST", "/marshal/launch")], + "enqueues": ["marshal.launch"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + await ctx.enqueue({"topic": "marshal.launch", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/marshal/02_launch_api_step.py b/bbdev/api/steps/marshal/02_launch_api_step.py deleted file mode 100644 index 422f47dd2..000000000 --- a/bbdev/api/steps/marshal/02_launch_api_step.py +++ /dev/null @@ -1,26 +0,0 @@ -import asyncio -from utils.event_common import wait_for_result - -config = { - "type": "api", - "name": "Marshal Launch", - "description": "launch marshal", - "path": "/marshal/launch", - "method": "POST", - "emits": ["marshal.launch"], - "flows": ["marshal"], -} - - -async def handler(req, context): - body = req.get("body") or {} - await context.emit({"topic": "marshal.launch", "data": body}) - - # ================================================================================== - # Wait for result - # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) diff --git a/bbdev/api/steps/marshal/02_launch_event_step.py b/bbdev/api/steps/marshal/02_launch_event.step.py similarity index 77% rename from bbdev/api/steps/marshal/02_launch_event_step.py rename to bbdev/api/steps/marshal/02_launch_event.step.py index 120e2b0d5..edb11419e 100644 --- a/bbdev/api/steps/marshal/02_launch_event_step.py +++ b/bbdev/api/steps/marshal/02_launch_event.step.py @@ -1,7 +1,8 @@ import os -import subprocess import sys +from motia import FlowContext, queue + # Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: @@ -9,19 +10,19 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger -from utils.event_common import check_result +from utils.event_common import check_result, get_origin_trace_id config = { - "type": "event", - "name": "Marshal Launch", + "name": "marshal-launch", "description": "launch marshal", - "subscribes": ["marshal.launch"], - "emits": [], "flows": ["marshal"], + "triggers": [queue("marshal.launch")], + "enqueues": [], } -async def handler(data, context): +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() script_dir = f"{bbdir}/workflow/steps/marshal/scripts" # ================================================================================== @@ -30,7 +31,7 @@ async def handler(data, context): command = f"./marshal -v launch interactive.json" result = stream_run_logger( cmd=command, - logger=context.logger, + logger=ctx.logger, cwd=script_dir, stdout_prefix="marshal launch", stderr_prefix="marshal launch", @@ -40,8 +41,7 @@ async def handler(data, context): # Return result to API # ================================================================================== success_result, failure_result = await check_result( - context, result.returncode, continue_run=False - ) + ctx, result.returncode, continue_run=False, trace_id=origin_tid) # ================================================================================== # Continue routing diff --git a/bbdev/api/steps/palladium/01_verilog_api.step.py b/bbdev/api/steps/palladium/01_verilog_api.step.py new file mode 100644 index 000000000..76c250033 --- /dev/null +++ b/bbdev/api/steps/palladium/01_verilog_api.step.py @@ -0,0 +1,36 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path + +config = { + "name": "palladium-verilog-api", + "description": "generate verilog code", + "flows": ["palladium"], + "triggers": [api("POST", "/palladium/verilog")], + "enqueues": ["palladium.verilog"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = request.body or {} + + # Get config name, must be provided + config_name = body.get("config") + if not config_name or config_name == "None": + return ApiResponse( + status=400, + body={ + "status": "error", + "message": "Configuration name is required. Please specify --config_name parameter.", + "example": './bbdev palladium --verilog "--config_name sims.palladium.BuckyballToyP2EConfig"', + }, + ) + + data = { + "config": config_name, + "balltype": body.get("balltype"), + "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), + } + await ctx.enqueue({"topic": "palladium.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/palladium/01_verilog_api_step.py b/bbdev/api/steps/palladium/01_verilog_api_step.py deleted file mode 100644 index c46fd8850..000000000 --- a/bbdev/api/steps/palladium/01_verilog_api_step.py +++ /dev/null @@ -1,52 +0,0 @@ -import asyncio -from utils.event_common import wait_for_result - - -from utils.path import get_buckyball_path - - -config = { - "type": "api", - "name": "palladium Verilog", - "description": "generate verilog code", - "path": "/palladium/verilog", - "method": "POST", - "emits": ["palladium.verilog"], - "flows": ["palladium"], -} - - -async def handler(req, context): - bbdir = get_buckyball_path() - body = req.get("body") or {} - - # Get config name, must be provided - config_name = body.get("config") - if not config_name or config_name == "None": - error_msg = "Configuration name is required. Please specify --config parameter." - context.logger.error(error_msg, {"config_name": config_name}) - return { - "status": 400, - "body": { - "success": False, - "failure": True, - "returncode": 400, - "message": "Configuration name is required. Please specify --config parameter.", - }, - } - - data = { - "config": config_name, - "balltype": body.get("balltype"), - "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), - } - await context.emit({"topic": "palladium.verilog", "data": data}) - - # ================================================================================== - # Wait for simulation result - # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) diff --git a/bbdev/api/steps/palladium/01_verilog_event_step.py b/bbdev/api/steps/palladium/01_verilog_event.step.py similarity index 64% rename from bbdev/api/steps/palladium/01_verilog_event_step.py rename to bbdev/api/steps/palladium/01_verilog_event.step.py index 125b3d2a2..a8dfa4eb5 100644 --- a/bbdev/api/steps/palladium/01_verilog_event_step.py +++ b/bbdev/api/steps/palladium/01_verilog_event.step.py @@ -1,7 +1,8 @@ import os -import subprocess import sys +from motia import FlowContext, queue + # Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: @@ -9,48 +10,47 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger -from utils.event_common import check_result +from utils.event_common import check_result, get_origin_trace_id config = { - "type": "event", - "name": "make verilog", + "name": "palladium-verilog", "description": "generate verilog code", - "subscribes": ["palladium.verilog"], - "emits": ["palladium.build"], "flows": ["palladium"], + "triggers": [queue("palladium.verilog")], + "enqueues": ["palladium.build"], } -async def handler(data, context): +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() # Use arch/build as the base directory for chipyard.Generator - base_build_dir = f"{data.get('output_dir', f'{bbdir}/arch/build')}/palladium" + base_build_dir = f"{input_data.get('output_dir', f'{bbdir}/arch/build')}/palladium" # Output directory for final Verilog files verilog_output_dir = f"{base_build_dir}/verilog" arch_dir = f"{bbdir}/arch" # Get config name, must be provided - config_name = data.get("config") + config_name = input_data.get("config") if not config_name or config_name == "None": - context.logger.error("Configuration name is required but not provided") + ctx.logger.error("Configuration name is required but not provided") success_result, failure_result = await check_result( - context, + ctx, 1, continue_run=False, extra_fields={ "task": "validation", "error": "Configuration name is required. Please specify --config_name parameter.", "example": './bbdev palladium --verilog "--config_name sims.palladium.BuckyballToyP2EConfig"', - }, - ) + }, trace_id=origin_tid) return failure_result - context.logger.info(f"Using configuration: {config_name}") + ctx.logger.info(f"Using configuration: {config_name}") # ================================================================================== # Step 1: Generate FIRRTL using chipyard.Generator # ================================================================================== - context.logger.info("Step 1: Generating FIRRTL with chipyard.Generator...") + ctx.logger.info("Step 1: Generating FIRRTL with chipyard.Generator...") os.system(f"mkdir -p {verilog_output_dir}") firrtl_command = ( f"cd {arch_dir} && " @@ -63,26 +63,25 @@ async def handler(data, context): result = stream_run_logger( cmd=firrtl_command, - logger=context.logger, + logger=ctx.logger, cwd=arch_dir, stdout_prefix="palladium firrtl", stderr_prefix="palladium firrtl", ) if result.returncode != 0: - context.logger.error(f"FIRRTL generation failed with code {result.returncode}") + ctx.logger.error(f"FIRRTL generation failed with code {result.returncode}") success_result, failure_result = await check_result( - context, + ctx, result.returncode, continue_run=False, - extra_fields={"task": "firrtl", "step": "generate"}, - ) + extra_fields={"task": "firrtl", "step": "generate"}, trace_id=origin_tid) return failure_result # ================================================================================== # Step 2: Convert FIRRTL to SystemVerilog using firtool # ================================================================================== - context.logger.info("Step 2: Converting FIRRTL to SystemVerilog with firtool...") + ctx.logger.info("Step 2: Converting FIRRTL to SystemVerilog with firtool...") # Extract the simple class name from the full config name # e.g., "sims.palladium.BuckyballToyP2EConfig" -> "BuckyballToyP2EConfig" @@ -91,18 +90,17 @@ async def handler(data, context): # Find the generated FIRRTL file (in base_build_dir, not verilog_output_dir) fir_file = f"{base_build_dir}/palladium.fpga.{config_class_name}.fir" if not os.path.exists(fir_file): - context.logger.error(f"FIRRTL file not found: {fir_file}") - context.logger.info(f"Looking for files in {base_build_dir}...") + ctx.logger.error(f"FIRRTL file not found: {fir_file}") + ctx.logger.info(f"Looking for files in {base_build_dir}...") # List files to help debug if os.path.exists(base_build_dir): files = os.listdir(base_build_dir) - context.logger.info(f"Files in build dir: {files}") + ctx.logger.info(f"Files in build dir: {files}") success_result, failure_result = await check_result( - context, + ctx, 1, continue_run=False, - extra_fields={"task": "firrtl", "step": "file_check"}, - ) + extra_fields={"task": "firrtl", "step": "file_check"}, trace_id=origin_tid) return failure_result verilog_command = ( @@ -118,57 +116,33 @@ async def handler(data, context): result = stream_run_logger( cmd=verilog_command, - logger=context.logger, + logger=ctx.logger, cwd=arch_dir, stdout_prefix="palladium verilog", stderr_prefix="palladium verilog", ) if result.returncode != 0: - context.logger.error(f"Verilog generation failed with code {result.returncode}") + ctx.logger.error(f"Verilog generation failed with code {result.returncode}") success_result, failure_result = await check_result( - context, + ctx, result.returncode, continue_run=False, - extra_fields={"task": "verilog", "step": "firtool"}, - ) + extra_fields={"task": "verilog", "step": "firtool"}, trace_id=origin_tid) return failure_result - # ================================================================================== - # Step 3: Verify and clean up - # ================================================================================== - # context.logger.info("Step 3: Verifying generated files...") - - # # Check if top-level Verilog was generated - # top_sv_dir = f"{verilog_output_dir}/VCU118FPGATestHarness.sv" - # top_sv_file = f"{top_sv_dir}/VCU118FPGATestHarness.sv" - - # if os.path.exists(top_sv_file): - # # Count generated files - # sv_files = [f for f in os.listdir(top_sv_dir) if f.endswith('.sv')] - # context.logger.info(f"Successfully generated {len(sv_files)} SystemVerilog files") - # context.logger.info(f"Top-level module: {top_sv_file}") - # else: - # context.logger.error(f"Top-level Verilog file not found: {top_sv_file}") - - # # Remove unwanted file - # topname_file = f"{arch_dir}/TestHarness.sv" - # if os.path.exists(topname_file): - # os.remove(topname_file) - # context.logger.info(f"Removed unwanted file: {topname_file}") - # ================================================================================== # Return result to API # ================================================================================== success_result, failure_result = await check_result( - context, + ctx, result.returncode, - continue_run=data.get("from_run_workflow", False), + continue_run=input_data.get("from_run_workflow", False), extra_fields={ "task": "verilog", "output_dir": verilog_output_dir, "top_module": "VCU118FPGATestHarness", - }, + }, trace_id=origin_tid, ) # ================================================================================== @@ -176,9 +150,9 @@ async def handler(data, context): # Routing to verilog or finish workflow # For run workflow, continue to verilog; for standalone clean, complete # ================================================================================== - if data.get("from_run_workflow"): - await context.emit( - {"topic": "palladium.build", "data": {**data, "task": "run"}} + if input_data.get("from_run_workflow"): + await ctx.enqueue( + {"topic": "palladium.build", "data": {**input_data, "task": "run"}} ) return diff --git a/bbdev/api/steps/pegasus/01_verilog_api_step.py b/bbdev/api/steps/pegasus/01_verilog_api.step.py similarity index 56% rename from bbdev/api/steps/pegasus/01_verilog_api_step.py rename to bbdev/api/steps/pegasus/01_verilog_api.step.py index d43ced303..5e756492e 100644 --- a/bbdev/api/steps/pegasus/01_verilog_api_step.py +++ b/bbdev/api/steps/pegasus/01_verilog_api.step.py @@ -1,22 +1,20 @@ -import asyncio -from utils.event_common import wait_for_result -from utils.path import get_buckyball_path +from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.path import get_buckyball_path +from utils.event_common import wait_for_result config = { - "type": "api", "name": "Pegasus Verilog", "description": "Generate SystemVerilog for Pegasus FPGA (PegasusHarness + ChipTop)", - "path": "/pegasus/verilog", - "method": "POST", - "emits": ["pegasus.verilog"], "flows": ["pegasus"], + "triggers": [api("POST", "/pegasus/verilog")], + "enqueues": ["pegasus.verilog"], } -async def handler(req, context): +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() - body = req.get("body") or {} + body = req.body or {} # Default config for Pegasus; allow override config_name = body.get("config", "sims.pegasus.PegasusConfig") @@ -26,10 +24,6 @@ async def handler(req, context): "output_dir": body.get("output_dir", f"{bbdir}/arch/build/pegasus/"), } - await context.emit({"topic": "pegasus.verilog", "data": data}) - - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) + await ctx.enqueue({"topic": "pegasus.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + result = await wait_for_result(ctx) + return result diff --git a/bbdev/api/steps/pegasus/01_verilog_event.step.py b/bbdev/api/steps/pegasus/01_verilog_event.step.py new file mode 100644 index 000000000..82298fe72 --- /dev/null +++ b/bbdev/api/steps/pegasus/01_verilog_event.step.py @@ -0,0 +1,122 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "make pegasus verilog", + "description": "Generate SystemVerilog from Chisel using ElaboratePegasus", + "flows": ["pegasus"], + "triggers": [queue("pegasus.verilog")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + arch_dir = f"{bbdir}/arch" + build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/pegasus/") + + config_name = input_data.get("config", "sims.pegasus.PegasusConfig") + ctx.logger.info(f"[pegasus] Elaborating config: {config_name}") + ctx.logger.info(f"[pegasus] Output directory: {build_dir}") + + os.makedirs(build_dir, exist_ok=True) + + # Use ElaboratePegasus (not the generic sims.verilator.Elaborate) + # ElaboratePegasus instantiates PegasusHarness directly, bypassing TestHarness + command = ( + f"mill -i __.test.runMain sims.pegasus.ElaboratePegasus " + f"--disable-annotation-unknown " + f"-strip-debug-info " + f"-O=debug " + f"--split-verilog " + f"-o={build_dir}" + ) + + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=arch_dir, + stdout_prefix="pegasus verilog", + stderr_prefix="pegasus verilog", + ) + + # Clean up stray top-level file if emitted next to arch/ + for stray in ["PegasusHarness.sv", "TestHarness.sv"]: + stray_path = f"{arch_dir}/{stray}" + if os.path.exists(stray_path): + os.remove(stray_path) + + # Copy generated SV files to pegasus/vivado/generated/ for Vivado build + # DPI files are replaced with synth-safe stubs. + vivado_gen_dir = f"{bbdir}/pegasus/vivado/generated" + if result.returncode == 0 and os.path.isdir(build_dir): + import re + import shutil + os.makedirs(vivado_gen_dir, exist_ok=True) + for f in os.listdir(vivado_gen_dir): + if f.endswith(".sv") or f.endswith(".v"): + os.remove(os.path.join(vivado_gen_dir, f)) + + def build_dpi_stub(src_path: str) -> str: + text = open(src_path, "r", encoding="utf-8").read() + m = re.search(r"module\s+([A-Za-z_][A-Za-z0-9_]*)\s*\((.*?)\);\s*", text, re.S) + if m is None: + raise RuntimeError(f"invalid DPI module format: {src_path}") + mod_name = m.group(1) + ports_block = m.group(2).rstrip() + outputs = [] + for line in ports_block.splitlines(): + s = line.strip() + if not s.startswith("output"): + continue + s = s.rstrip(",") + s = re.sub(r"^output\s+", "", s) + s = re.sub(r"^\[[^\]]+\]\s+", "", s) + for name in s.split(","): + n = name.strip() + if n: + outputs.append(n) + stub = [f"module {mod_name}(", ports_block, ");"] + for out_name in outputs: + stub.append(f" assign {out_name} = '0;") + stub.append("endmodule") + stub.append("") + return "\n".join(stub) + + sv_files = [f for f in os.listdir(build_dir) if f.endswith(".sv") or f.endswith(".v")] + for f in sv_files: + src = os.path.join(build_dir, f) + if "DPI" in f: + stub_name = f"stub_{os.path.splitext(f)[0].replace('DPI', '')}.sv" + stub_path = os.path.join(vivado_gen_dir, stub_name) + with open(stub_path, "w", encoding="utf-8") as wf: + wf.write(build_dpi_stub(src)) + else: + shutil.copy2(src, os.path.join(vivado_gen_dir, f)) + ctx.logger.info(f"[pegasus] Copied {len(sv_files)} files to {vivado_gen_dir}") + + success_result, failure_result = await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={ + "task": "verilog", + "output_dir": build_dir, + "vivado_gen_dir": vivado_gen_dir, + "top_module": "PegasusHarness", + }, + trace_id=origin_tid, + ) + + return diff --git a/bbdev/api/steps/pegasus/01_verilog_event_step.py b/bbdev/api/steps/pegasus/01_verilog_event_step.py deleted file mode 100644 index 663d2bd60..000000000 --- a/bbdev/api/steps/pegasus/01_verilog_event_step.py +++ /dev/null @@ -1,81 +0,0 @@ -import os -import sys - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from utils.event_common import check_result - - -config = { - "type": "event", - "name": "make pegasus verilog", - "description": "Generate SystemVerilog from Chisel using ElaboratePegasus", - "subscribes": ["pegasus.verilog"], - "emits": [], - "flows": ["pegasus"], -} - - -async def handler(data, context): - bbdir = get_buckyball_path() - arch_dir = f"{bbdir}/arch" - build_dir = data.get("output_dir", f"{bbdir}/arch/build/pegasus/") - - config_name = data.get("config", "sims.pegasus.PegasusConfig") - context.logger.info(f"[pegasus] Elaborating config: {config_name}") - context.logger.info(f"[pegasus] Output directory: {build_dir}") - - os.makedirs(build_dir, exist_ok=True) - - # Use ElaboratePegasus (not the generic sims.verilator.Elaborate) - # ElaboratePegasus instantiates PegasusHarness directly, bypassing TestHarness - command = ( - f"mill -i __.test.runMain sims.pegasus.ElaboratePegasus " - f"--disable-annotation-unknown " - f"-strip-debug-info " - f"-O=debug " - f"--split-verilog " - f"-o={build_dir}" - ) - - result = stream_run_logger( - cmd=command, - logger=context.logger, - cwd=arch_dir, - stdout_prefix="pegasus verilog", - stderr_prefix="pegasus verilog", - ) - - # Clean up stray top-level file if emitted next to arch/ - for stray in ["PegasusHarness.sv", "TestHarness.sv"]: - stray_path = f"{arch_dir}/{stray}" - if os.path.exists(stray_path): - os.remove(stray_path) - - # Copy generated SV files to pegasus/vivado/generated/ for Vivado build - vivado_gen_dir = f"{bbdir}/pegasus/vivado/generated" - if result.returncode == 0 and os.path.isdir(build_dir): - import shutil - os.makedirs(vivado_gen_dir, exist_ok=True) - sv_files = [f for f in os.listdir(build_dir) if f.endswith(".sv") or f.endswith(".v")] - for f in sv_files: - shutil.copy2(os.path.join(build_dir, f), os.path.join(vivado_gen_dir, f)) - context.logger.info(f"[pegasus] Copied {len(sv_files)} files to {vivado_gen_dir}") - - success_result, failure_result = await check_result( - context, - result.returncode, - continue_run=False, - extra_fields={ - "task": "verilog", - "output_dir": build_dir, - "vivado_gen_dir": vivado_gen_dir, - "top_module": "PegasusHarness", - }, - ) - - return diff --git a/bbdev/api/steps/pegasus/02_buildbitstream_api.step.py b/bbdev/api/steps/pegasus/02_buildbitstream_api.step.py new file mode 100644 index 000000000..bad553a69 --- /dev/null +++ b/bbdev/api/steps/pegasus/02_buildbitstream_api.step.py @@ -0,0 +1,18 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import wait_for_result + +config = { + "name": "Pegasus Buildbitstream", + "description": "build pegasus bitstream", + "flows": ["pegasus"], + "triggers": [api("POST", "/pegasus/buildbitstream")], + "enqueues": ["pegasus.buildbitstream"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + await ctx.enqueue({"topic": "pegasus.buildbitstream", "data": {**body, "_trace_id": ctx.trace_id}}) + result = await wait_for_result(ctx) + return result diff --git a/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py b/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py new file mode 100644 index 000000000..20025105e --- /dev/null +++ b/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py @@ -0,0 +1,83 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "Pegasus Buildbitstream", + "description": "build pegasus bitstream", + "flows": ["pegasus"], + "triggers": [queue("pegasus.buildbitstream")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + generated_dir = input_data.get("generated_dir", f"{bbdir}/pegasus/vivado/generated") + output_dir = input_data.get("output_dir", f"{bbdir}/pegasus/vivado/build") + top_module = input_data.get("top", "PegasusHarness") + + ctx.logger.info(f"[pegasus] Generated dir: {generated_dir}") + ctx.logger.info(f"[pegasus] Output dir: {output_dir}") + + os.makedirs(output_dir, exist_ok=True) + + if not os.path.isdir(generated_dir): + ctx.logger.error(f"[pegasus] generated dir not found: {generated_dir}") + success_result, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "buildbitstream", "error": "missing generated_dir"}, + trace_id=origin_tid, + ) + return failure_result + + has_rtl = any(name.endswith(".sv") or name.endswith(".v") for name in os.listdir(generated_dir)) + if not has_rtl: + ctx.logger.error(f"[pegasus] no verilog files found in: {generated_dir}") + success_result, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "buildbitstream", "error": "empty generated_dir"}, + trace_id=origin_tid, + ) + return failure_result + + bit_cmd = ( + f"bash {bbdir}/pegasus/vivado/build-bitstream.sh " + f"--source_dir {generated_dir} " + f"--output_dir {output_dir} " + f"--top {top_module}" + ) + result = stream_run_logger( + cmd=bit_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="pegasus bitstream", + stderr_prefix="pegasus bitstream", + ) + + success_result, failure_result = await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={ + "task": "buildbitstream", + "output_dir": output_dir, + "bitstream": f"{output_dir}/{top_module}.bit", + }, + trace_id=origin_tid, + ) + return diff --git a/bbdev/api/steps/sardine/01_run_api.step.py b/bbdev/api/steps/sardine/01_run_api.step.py new file mode 100644 index 000000000..fb24f909c --- /dev/null +++ b/bbdev/api/steps/sardine/01_run_api.step.py @@ -0,0 +1,22 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path + +config = { + "name": "sardine-run-api", + "description": "running sardine", + "flows": ["sardine"], + "triggers": [api("POST", "/sardine/run")], + "enqueues": ["sardine.run"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + + body = request.body or {} + + data = {"workload": body.get("workload", "")} + + await ctx.enqueue({"topic": "sardine.run", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/sardine/01_run_api_step.py b/bbdev/api/steps/sardine/01_run_api_step.py deleted file mode 100644 index 2c2fc1aeb..000000000 --- a/bbdev/api/steps/sardine/01_run_api_step.py +++ /dev/null @@ -1,45 +0,0 @@ -import subprocess -import sys -import os -import asyncio -from utils.event_common import wait_for_result - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path - -config = { - "type": "api", - "name": "running sardine", - "description": "running sardine", - "path": "/sardine/run", - "method": "POST", - "emits": ["sardine.run"], - "flows": ["sardine"], -} - - -async def handler(req, context): - bbdir = get_buckyball_path() - - body = req.get("body") or {} - - data = { - "workload": body.get("workload", ""), - "coverage": body.get("coverage", False), - } - - sardine_dir = f"{bbdir}/bb-tests/sardine" - - await context.emit({"topic": "sardine.run", "data": data}) - - # ================================================================================== - # Wait for execution result - # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) diff --git a/bbdev/api/steps/sardine/01_run_event_step.py b/bbdev/api/steps/sardine/01_run_event.step.py similarity index 55% rename from bbdev/api/steps/sardine/01_run_event_step.py rename to bbdev/api/steps/sardine/01_run_event.step.py index 39b1a2463..1b4e1b3ab 100644 --- a/bbdev/api/steps/sardine/01_run_event_step.py +++ b/bbdev/api/steps/sardine/01_run_event.step.py @@ -1,30 +1,29 @@ -from contextlib import redirect_stdout import os -import subprocess import sys import time +from motia import FlowContext, queue + # Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) - from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger -from utils.event_common import check_result +from utils.event_common import check_result, get_origin_trace_id config = { - "type": "event", - "name": "running sardine", + "name": "sardine-run", "description": "running sardine", - "subscribes": ["sardine.run"], - "emits": ["sardine.coverage_report"], "flows": ["sardine"], + "triggers": [queue("sardine.run")], + "enqueues": ["sardine.coverage_report"], } -async def handler(data, context): +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() sardine_dir = f"{bbdir}/bb-tests/sardine" @@ -32,15 +31,15 @@ async def handler(data, context): # Record start time so coverage report can filter to only this run's .dat files run_start_time = time.time() - command = f"python3 run_tests.py -m \"({data.get('workload', '')})\"" - if data.get("coverage", False): + command = f"python3 run_tests.py -m \"({input_data.get('workload', '')})\"" + if input_data.get("coverage", False): command += " --coverage" - context.logger.info( + ctx.logger.info( "Executing sardine command", {"command": command, "cwd": sardine_dir} ) result = stream_run_logger( cmd=command, - logger=context.logger, + logger=ctx.logger, cwd=sardine_dir, executable="bash", stdout_prefix="sardine run", @@ -50,20 +49,14 @@ async def handler(data, context): # ================================================================================== # Return execution result # ================================================================================== - coverage = data.get("coverage", False) - - if coverage: - # When coverage is enabled, always emit to coverage report step + if input_data.get("coverage", False): + # When coverage is enabled, always continue to coverage report step # (even if some tests failed, coverage data is still valid) - success_result, failure_result = await check_result( - context, result.returncode, continue_run=True - ) - await context.emit( - {"topic": "sardine.coverage_report", "data": {**data, "run_start_time": run_start_time}} + await check_result(ctx, result.returncode, continue_run=True, trace_id=origin_tid) + await ctx.enqueue( + {"topic": "sardine.coverage_report", "data": {**input_data, "run_start_time": run_start_time}} ) else: - success_result, failure_result = await check_result( - context, result.returncode, continue_run=False - ) + await check_result(ctx, result.returncode, continue_run=False, trace_id=origin_tid) return diff --git a/bbdev/api/steps/sardine/02_coverage_report_event_step.py b/bbdev/api/steps/sardine/02_coverage_report_event.step.py similarity index 82% rename from bbdev/api/steps/sardine/02_coverage_report_event_step.py rename to bbdev/api/steps/sardine/02_coverage_report_event.step.py index ed1ac7a04..93f3be298 100644 --- a/bbdev/api/steps/sardine/02_coverage_report_event_step.py +++ b/bbdev/api/steps/sardine/02_coverage_report_event.step.py @@ -2,6 +2,8 @@ import glob import sys +from motia import FlowContext, queue + # Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: @@ -9,19 +11,19 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger -from utils.event_common import check_result +from utils.event_common import check_result, get_origin_trace_id config = { - "type": "event", "name": "coverage report", "description": "merge coverage data and generate annotated source + lcov HTML report", - "subscribes": ["sardine.coverage_report"], - "emits": [], "flows": ["sardine"], + "triggers": [queue("sardine.coverage_report")], + "enqueues": [], } -async def handler(data, context): +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() log_dir = f"{bbdir}/arch/log" coverage_dir = f"{bbdir}/bb-tests/sardine/reports/coverage" @@ -31,19 +33,20 @@ async def handler(data, context): # Find coverage.dat files from this run only (created after run_start_time) # ================================================================================== all_dat_files = glob.glob(f"{log_dir}/*/coverage.dat") - run_start_time = data.get("run_start_time", 0) + run_start_time = input_data.get("run_start_time", 0) dat_files = [f for f in all_dat_files if os.path.getmtime(f) >= run_start_time] if not dat_files: - context.logger.error("No coverage .dat files found", {"dir": coverage_dir}) + ctx.logger.error("No coverage .dat files found", {"dir": coverage_dir}) success_result, failure_result = await check_result( - context, + ctx, returncode=1, continue_run=False, extra_fields={"task": "coverage_report", "error": "No .dat files found"}, + trace_id=origin_tid, ) return - context.logger.info(f"Found {len(dat_files)} coverage data files") + ctx.logger.info(f"Found {len(dat_files)} coverage data files") # ================================================================================== # Merge all .dat files @@ -57,17 +60,18 @@ async def handler(data, context): merge_cmd = f"verilator_coverage -write {merged_dat} {' '.join(dat_files)}" result = stream_run_logger( cmd=merge_cmd, - logger=context.logger, + logger=ctx.logger, cwd=bbdir, stdout_prefix="coverage merge", stderr_prefix="coverage merge", ) if result.returncode != 0: await check_result( - context, + ctx, returncode=result.returncode, continue_run=False, extra_fields={"task": "coverage_report", "error": "merge failed"}, + trace_id=origin_tid, ) return @@ -78,17 +82,18 @@ async def handler(data, context): annotate_cmd = f"verilator_coverage --annotate {annotate_dir} {merged_dat}" result = stream_run_logger( cmd=annotate_cmd, - logger=context.logger, + logger=ctx.logger, cwd=bbdir, stdout_prefix="coverage annotate", stderr_prefix="coverage annotate", ) if result.returncode != 0: await check_result( - context, + ctx, returncode=result.returncode, continue_run=False, extra_fields={"task": "coverage_report", "error": "annotate failed"}, + trace_id=origin_tid, ) return @@ -101,43 +106,45 @@ async def handler(data, context): lcov_cmd = f"verilator_coverage -write-info {lcov_info} {merged_dat}" result = stream_run_logger( cmd=lcov_cmd, - logger=context.logger, + logger=ctx.logger, cwd=bbdir, stdout_prefix="coverage lcov", stderr_prefix="coverage lcov", ) if result.returncode != 0: await check_result( - context, + ctx, returncode=result.returncode, continue_run=False, extra_fields={"task": "coverage_report", "error": "lcov export failed"}, + trace_id=origin_tid, ) return genhtml_cmd = f"genhtml {lcov_info} -o {html_dir}" result = stream_run_logger( cmd=genhtml_cmd, - logger=context.logger, + logger=ctx.logger, cwd=bbdir, stdout_prefix="coverage html", stderr_prefix="coverage html", ) if result.returncode != 0: await check_result( - context, + ctx, returncode=result.returncode, continue_run=False, extra_fields={"task": "coverage_report", "error": "genhtml failed"}, + trace_id=origin_tid, ) return # ================================================================================== # Return result # ================================================================================== - context.logger.info(f"Coverage report generated: {html_dir}/index.html") + ctx.logger.info(f"Coverage report generated: {html_dir}/index.html") success_result, failure_result = await check_result( - context, + ctx, returncode=0, continue_run=False, extra_fields={ @@ -147,6 +154,7 @@ async def handler(data, context): "html_dir": html_dir, "dat_count": len(dat_files), }, + trace_id=origin_tid, ) return diff --git a/bbdev/api/steps/verilator/01_clean_api.step.py b/bbdev/api/steps/verilator/01_clean_api.step.py new file mode 100644 index 000000000..ab1361d9e --- /dev/null +++ b/bbdev/api/steps/verilator/01_clean_api.step.py @@ -0,0 +1,15 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "verilator-clean-api", + "description": "clean build directory", + "flows": ["verilator"], + "triggers": [api("POST", "/verilator/clean")], + "enqueues": ["verilator.clean"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + await ctx.enqueue({"topic": "verilator.clean", "data": {**body, "task": "clean", "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/01_clean_api_step.py b/bbdev/api/steps/verilator/01_clean_api_step.py deleted file mode 100644 index b447792ce..000000000 --- a/bbdev/api/steps/verilator/01_clean_api_step.py +++ /dev/null @@ -1,26 +0,0 @@ -import asyncio -from utils.event_common import wait_for_result - -config = { - "type": "api", - "name": "Verilator Clean", - "description": "clean build directory", - "path": "/verilator/clean", - "method": "POST", - "emits": ["verilator.clean"], - "flows": ["verilator"], -} - - -async def handler(req, context): - body = req.get("body") or {} - await context.emit({"topic": "verilator.clean", "data": {**body, "task": "clean"}}) - - # ================================================================================== - # Wait for simulation result - # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) diff --git a/bbdev/api/steps/verilator/01_clean_event_step.py b/bbdev/api/steps/verilator/01_clean_event.step.py similarity index 66% rename from bbdev/api/steps/verilator/01_clean_event_step.py rename to bbdev/api/steps/verilator/01_clean_event.step.py index 626eba3e3..4437df7ad 100644 --- a/bbdev/api/steps/verilator/01_clean_event_step.py +++ b/bbdev/api/steps/verilator/01_clean_event.step.py @@ -1,7 +1,8 @@ -import subprocess import os import sys +from motia import FlowContext, queue + # Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: @@ -9,19 +10,22 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger -from utils.event_common import check_result +from utils.event_common import check_result, get_origin_trace_id config = { - "type": "event", - "name": "make clean", + "name": "verilator-clean", "description": "clean build directory", - "subscribes": ["verilator.run", "verilator.clean"], - "emits": ["verilator.verilog"], "flows": ["verilator"], + "triggers": [ + queue("verilator.run"), + queue("verilator.clean"), + ], + "enqueues": ["verilator.verilog"], } -async def handler(data, context): +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() build_dir = f"{bbdir}/arch/build" # ================================================================================== @@ -30,7 +34,7 @@ async def handler(data, context): command = f"rm -rf {build_dir}" result = stream_run_logger( cmd=command, - logger=context.logger, + logger=ctx.logger, cwd=bbdir, stdout_prefix="verilator clean", stderr_prefix="verilator clean", @@ -40,18 +44,18 @@ async def handler(data, context): # Return result to API # ================================================================================== success_result, failure_result = await check_result( - context, + ctx, result.returncode, - continue_run=data.get("from_run_workflow", False), - extra_fields={"task": "clean"}, + continue_run=input_data.get("from_run_workflow", False), + extra_fields={"task": "clean"}, trace_id=origin_tid, ) # ================================================================================== # Continue routing # ================================================================================== - if data.get("from_run_workflow"): - await context.emit( - {"topic": "verilator.verilog", "data": {**data, "task": "run"}} + if input_data.get("from_run_workflow"): + await ctx.enqueue( + {"topic": "verilator.verilog", "data": {**input_data, "task": "run"}} ) return diff --git a/bbdev/api/steps/verilator/02_verilog_api.step.py b/bbdev/api/steps/verilator/02_verilog_api.step.py new file mode 100644 index 000000000..d128818ee --- /dev/null +++ b/bbdev/api/steps/verilator/02_verilog_api.step.py @@ -0,0 +1,36 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path + +config = { + "name": "verilator-verilog-api", + "description": "generate verilog code", + "flows": ["verilator"], + "triggers": [api("POST", "/verilator/verilog")], + "enqueues": ["verilator.verilog"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = request.body or {} + + # Get config name, must be provided + config_name = body.get("config") + if not config_name or config_name == "None": + return ApiResponse( + status=400, + body={ + "status": "error", + "message": "Configuration name is required. Please specify --config parameter.", + "example": 'bbdev verilator --verilog "--config sims.verilator.BuckyballToyVerilatorConfig"', + }, + ) + + data = { + "config": config_name, + "balltype": body.get("balltype"), + "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), + } + await ctx.enqueue({"topic": "verilator.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/02_verilog_api_step.py b/bbdev/api/steps/verilator/02_verilog_api_step.py deleted file mode 100644 index 433dd78a8..000000000 --- a/bbdev/api/steps/verilator/02_verilog_api_step.py +++ /dev/null @@ -1,52 +0,0 @@ -import asyncio -from utils.event_common import wait_for_result - - -from utils.path import get_buckyball_path - - -config = { - "type": "api", - "name": "Verilator Verilog", - "description": "generate verilog code", - "path": "/verilator/verilog", - "method": "POST", - "emits": ["verilator.verilog"], - "flows": ["verilator"], -} - - -async def handler(req, context): - bbdir = get_buckyball_path() - body = req.get("body") or {} - - # Get config name, must be provided - config_name = body.get("config") - if not config_name or config_name == "None": - error_msg = "Configuration name is required. Please specify --config parameter." - context.logger.error(error_msg, {"config_name": config_name}) - return { - "status": 400, - "body": { - "success": False, - "failure": True, - "returncode": 400, - "message": "Configuration name is required. Please specify --config parameter.", - }, - } - - data = { - "config": config_name, - "balltype": body.get("balltype"), - "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), - } - await context.emit({"topic": "verilator.verilog", "data": data}) - - # ================================================================================== - # Wait for simulation result - # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) diff --git a/bbdev/api/steps/verilator/02_verilog_event.step.py b/bbdev/api/steps/verilator/02_verilog_event.step.py new file mode 100644 index 000000000..a664ed64e --- /dev/null +++ b/bbdev/api/steps/verilator/02_verilog_event.step.py @@ -0,0 +1,93 @@ +import os +import sys + +from motia import FlowContext, queue + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "verilator-verilog", + "description": "generate verilog code", + "flows": ["verilator"], + "triggers": [queue("verilator.verilog")], + "enqueues": ["verilator.build"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/") + arch_dir = f"{bbdir}/arch" + + # Get config name, must be provided + config_name = input_data.get("config") + if not config_name or config_name == "None": + ctx.logger.error("Configuration name is required but not provided") + success_result, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "validation", + "error": "Configuration name is required. Please specify --config parameter.", + "example": 'bbdev verilator --verilog "--config sims.verilator.BuckyballToyVerilatorConfig"', + }, trace_id=origin_tid) + return failure_result + + ctx.logger.info(f"Using configuration: {config_name}") + + # ================================================================================== + # Execute operation + # ================================================================================== + if input_data.get("balltype"): + command = ( + f"mill -i __.test.runMain sims.verify.BallTopMain {input_data.get('balltype')} " + ) + else: + command = f"mill -i __.test.runMain sims.verilator.Elaborate {config_name} " + + command += "--disable-annotation-unknown -strip-debug-info -O=debug " + command += f"--split-verilog -o={build_dir}" + + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=arch_dir, + stdout_prefix="verilator verilog", + stderr_prefix="verilator verilog", + ) + + # Remove unwanted file + topname_file = f"{arch_dir}/TestHarness.sv" + if os.path.exists(topname_file): + os.remove(topname_file) + + # ================================================================================== + # Return result to API + # ================================================================================== + success_result, failure_result = await check_result( + ctx, + result.returncode, + continue_run=input_data.get("from_run_workflow", False), + extra_fields={"task": "verilog"}, trace_id=origin_tid, + ) + + # ================================================================================== + # Continue routing + # Routing to verilog or finish workflow + # For run workflow, continue to verilog; for standalone clean, complete + # ================================================================================== + if input_data.get("from_run_workflow"): + await ctx.enqueue( + {"topic": "verilator.build", "data": {**input_data, "task": "run"}} + ) + + return diff --git a/bbdev/api/steps/verilator/02_verilog_event_step.py b/bbdev/api/steps/verilator/02_verilog_event_step.py deleted file mode 100644 index 47509716f..000000000 --- a/bbdev/api/steps/verilator/02_verilog_event_step.py +++ /dev/null @@ -1,126 +0,0 @@ -import os -import subprocess -import sys - -# Add the utils directory to the Python path -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from utils.event_common import check_result - -config = { - "type": "event", - "name": "make verilog", - "description": "generate verilog code", - "subscribes": ["verilator.verilog"], - "emits": ["verilator.build"], - "flows": ["verilator"], -} - - -async def handler(data, context): - bbdir = get_buckyball_path() - build_dir = data.get("output_dir", f"{bbdir}/arch/build/") - arch_dir = f"{bbdir}/arch" - config_name = data.get("config") - - if not config_name or config_name == "None": - context.logger.error("Configuration name is required but not provided") - success_result, failure_result = await check_result( - context, - 1, - continue_run=False, - extra_fields={ - "task": "validation", - "error": "Configuration name is required. Please specify --config parameter.", - "example": 'bbdev verilator --verilog "--config sims.verilator.BuckyballToyVerilatorConfig"', - }, - ) - return failure_result - - context.logger.info(f"Using configuration: {config_name}") - - # ================================================================================== - # Execute operation - # ================================================================================== - if data.get("balltype"): - command = ( - f"mill -i __.test.runMain sims.verify.BallTopMain {data.get('balltype')} " - ) - else: - command = f"mill -i __.test.runMain sims.verilator.Elaborate {config_name} " - - # Firtool options (CIRCT). Current set; optional Chipyard-style options below. - command += "--disable-annotation-unknown " - command += "--strip-debug-info " - command += "-O=debug " - # command += f"-repl-seq-mem -repl-seq-mem-file={build_dir}/mem.conf " - command += f"--split-verilog -o={build_dir} " - # Optional: --disable-annotation-classless (ignore classless annotations) - # Optional: -repl-seq-mem -repl-seq-mem-file=.conf (SRAM macro replacement) - # Optional: --disable-all-randomization (disable mem/reg init; may break semantics) - # Optional: --disable-opt (no optimization) or -O=release (default is release) - # Optional: --output-annotation-file= (emit annotations after lower-to-hw) - # Optional: --no-dedup (disable module dedup); --strip-fir-debug-info (FIR locators) - - result = stream_run_logger( - cmd=command, - logger=context.logger, - cwd=arch_dir, - stdout_prefix="verilator verilog", - stderr_prefix="verilator verilog", - ) - - # Remove testchipip C++ sources that depend on fesvr (which we don't have). - # SimTSI.v is kept so verilator can resolve the SimTSI module reference in BBSimHarness.sv; - # tsi_tick DPI symbol is satisfied by arch/src/csrc/src/monitor/ioe/tsi_stub.cc instead. - for unwanted in [ - f"{build_dir}/testchip_htif.cc", - f"{build_dir}/testchip_htif.h", - f"{build_dir}/testchip_tsi.cc", - f"{build_dir}/testchip_tsi.h", - f"{build_dir}/SimTSI.cc", - f"{arch_dir}/BBSimHarness.sv", - ]: - if os.path.exists(unwanted): - os.remove(unwanted) - - # Patch fesvr includes out of mm.h and mm.cc (copied from testchipip resources). - # They reference fesvr/memif.h which we don't have — our SimDRAM_bb.cc doesn't use it. - for patch_file in [f"{build_dir}/mm.h", f"{build_dir}/mm.cc"]: - if os.path.exists(patch_file): - with open(patch_file, "r") as f: - content = f.read() - patched = "\n".join( - line for line in content.splitlines() - if "fesvr/memif.h" not in line and "fesvr/elfloader.h" not in line - ) - if patched != content: - with open(patch_file, "w") as f: - f.write(patched) - context.logger.info(f"Patched fesvr includes from {patch_file}") - - # ================================================================================== - # Return result to API - # ================================================================================== - success_result, failure_result = await check_result( - context, - result.returncode, - continue_run=data.get("from_run_workflow", False), - extra_fields={"task": "verilog"}, - ) - - # ================================================================================== - # Continue routing - # Routing to verilog or finish workflow - # For run workflow, continue to verilog; for standalone clean, complete - # ================================================================================== - if data.get("from_run_workflow"): - await context.emit( - {"topic": "verilator.build", "data": {**data, "task": "run"}} - ) - - return diff --git a/bbdev/api/steps/verilator/03_build_api.step.py b/bbdev/api/steps/verilator/03_build_api.step.py new file mode 100644 index 000000000..7bb01694a --- /dev/null +++ b/bbdev/api/steps/verilator/03_build_api.step.py @@ -0,0 +1,19 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "verilator-build-api", + "description": "build verilator executable", + "flows": ["verilator"], + "triggers": [api("POST", "/verilator/build")], + "enqueues": ["verilator.build"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + data = { + "jobs": body.get("jobs", 16), + "cosim": body.get("cosim", False), + } + await ctx.enqueue({"topic": "verilator.build", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/03_build_api_step.py b/bbdev/api/steps/verilator/03_build_api_step.py deleted file mode 100644 index 658471645..000000000 --- a/bbdev/api/steps/verilator/03_build_api_step.py +++ /dev/null @@ -1,30 +0,0 @@ -import asyncio -from utils.event_common import wait_for_result - -config = { - "type": "api", - "name": "Verilator Build", - "description": "build verilator executable", - "path": "/verilator/build", - "method": "POST", - "emits": ["verilator.build"], - "flows": ["verilator"], -} - - -async def handler(req, context): - body = req.get("body") or {} - data = { - "jobs": body.get("jobs", 16), - "coverage": body.get("coverage", False), - } - await context.emit({"topic": "verilator.build", "data": data}) - - # ================================================================================== - # Wait for simulation result - # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) diff --git a/bbdev/api/steps/verilator/03_build_event.step.py b/bbdev/api/steps/verilator/03_build_event.step.py new file mode 100644 index 000000000..cd3b40723 --- /dev/null +++ b/bbdev/api/steps/verilator/03_build_event.step.py @@ -0,0 +1,140 @@ +import os +import subprocess +import glob +import sys + +from motia import FlowContext, queue + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "verilator-build", + "description": "build verilator executable", + "flows": ["verilator"], + "triggers": [queue("verilator.build")], + "enqueues": ["verilator.sim", "verilator.cosim"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + arch_dir = f"{bbdir}/arch" + build_dir = f"{arch_dir}/build" + waveform_dir = f"{arch_dir}/waveform" + log_dir = f"{arch_dir}/log" + cosim = input_data.get("cosim", False) + + # ================================================================================== + # Execute operation + # ================================================================================== + # Find sources + vsrcs = glob.glob(f"{build_dir}/**/*.v", recursive=True) + glob.glob( + f"{build_dir}/**/*.sv", recursive=True + ) + csrcs = ( + glob.glob(f"{arch_dir}/src/csrc/**/*.c", recursive=True) + + glob.glob(f"{arch_dir}/src/csrc/**/*.cc", recursive=True) + + glob.glob(f"{arch_dir}/src/csrc/**/*.cpp", recursive=True) + + glob.glob(f"{build_dir}/**/*.c", recursive=True) + + glob.glob(f"{build_dir}/**/*.cc", recursive=True) + + glob.glob(f"{build_dir}/**/*.cpp", recursive=True) + ) + + # Setup paths: fesvr from bebop/host/spike/riscv-isa-sim (install/include, install/lib) + bebop_isa_sim = f"{bbdir}/bebop/host/spike/riscv-isa-sim" + inc_paths = [ + os.environ.get("RISCV", "") + "/include" if os.environ.get("RISCV") else "", + f"{arch_dir}/thirdparty/chipyard/tools/DRAMSim2", + f"{bebop_isa_sim}/install/include", + build_dir, + f"{arch_dir}/src/csrc/include", + ] + inc_flags = " ".join([f"-I{p}" for p in inc_paths if p]) + + if cosim: + topname = "ToyBuckyball" + else: + topname = "TestHarness" + + cflags = f"{inc_flags} -DTOP_NAME='\"V{topname}\"' -std=c++17 " + if cosim: + cflags += " -DCOSIM" + ldflags = ( + f"-lreadline -ldramsim -lfesvr -lstdc++ " + f"-L{bebop_isa_sim}/install/lib " + f"-L{bbdir}/result/lib " + f"-L{arch_dir}/thirdparty/chipyard/tools/DRAMSim2 " + f"-L{arch_dir}/thirdparty/chipyard/toolchains/riscv-tools/riscv-isa-sim/build " + f"-L{arch_dir}/thirdparty/chipyard/toolchains/riscv-tools/riscv-isa-sim/build/lib" + ) + + obj_dir = f"{build_dir}/obj_dir" + subprocess.run(f"rm -rf {obj_dir}", shell=True) + os.makedirs(obj_dir, exist_ok=True) + + sources = " ".join(vsrcs + csrcs) + jobs = input_data.get("jobs", "") + + verilator_cmd = ( + f"verilator -MMD --build -cc --trace -O3 --x-assign fast --x-initial fast --noassert -Wno-fatal " + f"--trace-fst --trace-threads 1 --output-split 10000 --output-split-cfuncs 100 " + f"--unroll-count 256 " + f"-Wno-PINCONNECTEMPTY " + f"-Wno-ASSIGNDLY " + f"-Wno-DECLFILENAME " + f"-Wno-UNUSED " + f"-Wno-UNOPTFLAT " + f"-Wno-BLKANDNBLK " + f"-Wno-style " + f"-Wall " + f"--timing -j {jobs} +incdir+{build_dir} --top {topname} {sources} " + f"-CFLAGS '{cflags}' -LDFLAGS '{ldflags}' --Mdir {obj_dir} --exe" + ) + + result = stream_run_logger( + cmd=verilator_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="verilator build", + stderr_prefix="verilator build", + ) + result = stream_run_logger( + cmd=f"make -C {obj_dir} -f V{topname}.mk V{topname}", + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="verilator build", + stderr_prefix="verilator build", + ) + + # ================================================================================== + # Return result to API + # ================================================================================== + success_result, failure_result = await check_result( + ctx, + result.returncode, + continue_run=input_data.get("from_run_workflow", False), + extra_fields={"task": "build"}, trace_id=origin_tid, + ) + + # ================================================================================== + # Continue routing + # ================================================================================== + if input_data.get("from_run_workflow"): + if cosim: + await ctx.enqueue( + {"topic": "verilator.cosim", "data": {**input_data, "task": "run"}} + ) + else: + await ctx.enqueue( + {"topic": "verilator.sim", "data": {**input_data, "task": "run"}} + ) + + return diff --git a/bbdev/api/steps/verilator/03_build_event_step.py b/bbdev/api/steps/verilator/03_build_event_step.py deleted file mode 100644 index 632396727..000000000 --- a/bbdev/api/steps/verilator/03_build_event_step.py +++ /dev/null @@ -1,168 +0,0 @@ -import os -import subprocess -import glob -import sys - -# Add the utils directory to the Python path -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from utils.event_common import check_result - -config = { - "type": "event", - "name": "make build", - "description": "build verilator executable", - "subscribes": ["verilator.build"], - "emits": ["verilator.sim"], - "flows": ["verilator"], -} - - -async def handler(data, context): - bbdir = get_buckyball_path() - arch_dir = f"{bbdir}/arch" - build_dir = f"{arch_dir}/build" - coverage = data.get("coverage", False) - - # ================================================================================== - # Find sources - # ================================================================================== - vsrcs = glob.glob(f"{build_dir}/**/*.v", recursive=True) + glob.glob( - f"{build_dir}/**/*.sv", recursive=True - ) - csrcs = ( - glob.glob(f"{arch_dir}/src/csrc/**/*.c", recursive=True) - + glob.glob(f"{arch_dir}/src/csrc/**/*.cc", recursive=True) - + glob.glob(f"{arch_dir}/src/csrc/**/*.cpp", recursive=True) - + glob.glob(f"{build_dir}/**/*.c", recursive=True) - + glob.glob(f"{build_dir}/**/*.cc", recursive=True) - + glob.glob(f"{build_dir}/**/*.cpp", recursive=True) - ) - - # Exclude testchipip's SimDRAM.cc — our SimDRAM_bb.cc overrides memory_init - csrcs = [f for f in csrcs if not f.endswith("SimDRAM.cc") or "src/csrc" in f] - - # Exclude testchipip's TSI/HTIF C++ sources (deleted in verilog step, but guard anyway). - # tsi_tick DPI symbol is provided by tsi_stub.cc in src/csrc instead. - _tsi_htif = {"testchip_tsi.cc", "testchip_htif.cc", "SimTSI.cc"} - csrcs = [f for f in csrcs if os.path.basename(f) not in _tsi_htif] - - topname = "BBSimHarness" - - # ================================================================================== - # Build flags - # ================================================================================== - result_dir = f"{bbdir}/result" - - def pkg_config(flag, pkg): - r = subprocess.run(["pkg-config", flag, pkg], capture_output=True, text=True) - return r.stdout.strip() if r.returncode == 0 else "" - - readline_inc = pkg_config("--variable=includedir", "readline") - readline_lib = pkg_config("--variable=libdir", "readline") - zlib_lib = pkg_config("--variable=libdir", "zlib") - - inc_flags = " ".join([ - f"-I{result_dir}/include", - f"-I{build_dir}", - f"-I{arch_dir}/src/csrc/include", - f"-I{readline_inc}", - ]) - - # -DBBSIM: selects VBBSimHarness in bdb.h / main.cc - # BDB NDJSON trace (+trace=...) is runtime-only; bbdev sim uses +trace=all (04_sim_event_step.py). - cflags = f"{inc_flags} -DBBSIM -DTOP_NAME='\"V{topname}\"' -std=c++17" - - ldflags = ( - f"-lreadline -ldramsim -lstdc++ -lz " - f"-L{result_dir}/lib " - f"-L{readline_lib} -Wl,-rpath,{readline_lib} " - f"-L{zlib_lib} -Wl,-rpath,{zlib_lib} " - ) - - obj_dir = f"{build_dir}/obj_dir" - # subprocess.run(f"rm -rf {obj_dir}", shell=True) - os.makedirs(obj_dir, exist_ok=True) - - sources = " ".join(vsrcs + csrcs) - jobs = data.get("jobs", "") - - # Fix nix runtime library paths - libstdcpp_path = subprocess.run( - "g++ -print-file-name=libstdc++.so", shell=True, capture_output=True, text=True - ).stdout.strip() - if libstdcpp_path and "/" in libstdcpp_path: - nix_lib_dir = os.path.dirname(os.path.realpath(libstdcpp_path)) - ldflags += f" -Wl,-rpath,{nix_lib_dir}" - - # Enable ccache if available - if subprocess.run("command -v ccache", shell=True, capture_output=True).returncode == 0: - os.environ["OBJCACHE"] = "ccache" - - # Use lld for faster linking if available - use_lld = subprocess.run("command -v ld.lld", shell=True, capture_output=True).returncode == 0 - if use_lld: - ldflags += " -fuse-ld=lld" - - # ================================================================================== - # Run verilator - # ================================================================================== - verilator_cmd = ( - f"verilator -MMD -cc --vpi --trace -O3 --x-assign fast --x-initial fast --noassert -Wno-fatal " - f"--trace-fst --trace-threads 1 --output-split 10000 --output-split-cfuncs 100 " - f"--unroll-count 256 " - f"{'--coverage-line ' if coverage else ''}" - f"-Wno-PINCONNECTEMPTY " - f"-Wno-ASSIGNDLY " - f"-Wno-DECLFILENAME " - f"-Wno-UNUSED " - f"-Wno-UNOPTFLAT " - f"-Wno-BLKANDNBLK " - f"-Wno-style " - f"-Wall " - f"--timing -j {jobs} +incdir+{build_dir} --top {topname} {sources} " - f"-CFLAGS '{cflags}' -LDFLAGS '{ldflags}' --Mdir {obj_dir} --exe" - ) - - result = stream_run_logger( - cmd=verilator_cmd, - logger=context.logger, - cwd=bbdir, - stdout_prefix="verilator verilation", - stderr_prefix="verilator verilation", - ) - if result.returncode != 0: - success_result, failure_result = await check_result( - context, result.returncode, continue_run=False, extra_fields={"task": "build"} - ) - return - - make_jobs = jobs if jobs else str(os.cpu_count() or 16) - result = stream_run_logger( - cmd=f"make -j{make_jobs} VM_PARALLEL_BUILDS=1 -C {obj_dir} -f V{topname}.mk V{topname}", - logger=context.logger, - cwd=bbdir, - stdout_prefix="verilator build", - stderr_prefix="verilator build", - ) - - # ================================================================================== - # Return result to API - # ================================================================================== - success_result, failure_result = await check_result( - context, - result.returncode, - continue_run=data.get("from_run_workflow", False), - extra_fields={"task": "build"}, - ) - - if data.get("from_run_workflow"): - await context.emit( - {"topic": "verilator.sim", "data": {**data, "task": "run"}} - ) - - return diff --git a/bbdev/api/steps/verilator/04_cosim_api.step.py b/bbdev/api/steps/verilator/04_cosim_api.step.py new file mode 100644 index 000000000..419336e8e --- /dev/null +++ b/bbdev/api/steps/verilator/04_cosim_api.step.py @@ -0,0 +1,27 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "verilator-cosim-api", + "description": "run verilator cosimulation", + "flows": ["verilator"], + "triggers": [api("POST", "/verilator/cosim")], + "enqueues": ["verilator.cosim"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + binary = body.get("binary", "") + if not binary: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 400, + "message": "binary parameter is required", + }, + ) + + await ctx.enqueue({"topic": "verilator.cosim", "data": {**body, "task": "cosim", "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/04_cosim_event.step.py b/bbdev/api/steps/verilator/04_cosim_event.step.py new file mode 100644 index 000000000..d84f63d87 --- /dev/null +++ b/bbdev/api/steps/verilator/04_cosim_event.step.py @@ -0,0 +1,128 @@ +import os +import subprocess +import sys +from datetime import datetime + +from motia import FlowContext, queue + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.search_workload import search_workload +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "verilator-cosim", + "description": "run cosimulation", + "flows": ["verilator"], + "triggers": [queue("verilator.cosim")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + # ================================================================================== + # Get simulation parameters + # ================================================================================== + bbdir = get_buckyball_path() + arch_dir = f"{bbdir}/arch" + build_dir = f"{arch_dir}/build" + + # Generate timestamp + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + + binary_name = input_data.get("binary", "") + success_result, failure_result = await check_result( + ctx, returncode=(binary_name == None), continue_run=True, trace_id=origin_tid, + ) + + binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) + success_result, failure_result = await check_result( + ctx, returncode=(binary_path == None), continue_run=True, trace_id=origin_tid, + ) + if failure_result: + ctx.logger.error("binary not found", failure_result) + return + + # Create log and waveform directory + log_dir = f"{arch_dir}/log/{timestamp}-{binary_name}" + waveform_dir = f"{arch_dir}/waveform/{timestamp}-{binary_name}" + topname = "ToyBuckyball" + + os.makedirs(log_dir, exist_ok=True) + os.makedirs(waveform_dir, exist_ok=True) + + bin_path = f"{build_dir}/obj_dir/V{topname}" + batch = input_data.get("batch", False) + + # Create log and waveform file + log_path = f"{log_dir}/bdb.log" + fst_path = f"{waveform_dir}/waveform.fst" + # Remove old waveform file + subprocess.run(f"rm -f {waveform_dir}/waveform.vcd", shell=True, check=True) + + # ================================================================================== + # Execute simulation script with streaming output + # ================================================================================== + bebop_isa_sim = f"{bbdir}/bebop/host/spike/riscv-isa-sim" + ld_lib_path = ( + f"{bebop_isa_sim}/install/lib:" + f"{bbdir}/result/lib:" + f"{arch_dir}/thirdparty/chipyard/tools/DRAMSim2" + ) + sim_cmd = ( + f"export LD_LIBRARY_PATH=\"{ld_lib_path}:$LD_LIBRARY_PATH\"; " + f"{bin_path} +permissive +loadmem={binary_path} +loadmem_addr=800000000 " + f"{'+batch ' if batch else ''} " + f"+fst={fst_path} +log={log_path} +permissive-off " + f"{binary_path} > >(tee {log_dir}/stdout.log) 2> >(spike-dasm > {log_dir}/disasm.log)" + ) + script_dir = os.path.dirname(__file__) + + result = stream_run_logger( + cmd=sim_cmd, + logger=ctx.logger, + cwd=script_dir, + stdout_prefix="verilator sim", + stderr_prefix="verilator sim", + executable="bash", + ) + success_result, failure_result = await check_result( + ctx, returncode=result.returncode, continue_run=True, trace_id=origin_tid) + if failure_result: + ctx.logger.error("sim failed", failure_result) + return + + if os.path.exists(f"{waveform_dir}/waveform.fst.heir"): + subprocess.run( + f"gtkwave -f {waveform_dir}/waveform.fst -H {waveform_dir}/waveform.fst.heir", + shell=True, + check=True, + ) + + # ================================================================================== + # Return simulation result + # ================================================================================== + # This is the end point of the run workflow, status will no longer be set to processing + success_result, failure_result = await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={ + "task": "sim", + "binary": binary_path, + "log_dir": log_dir, + "waveform_dir": waveform_dir, + "timestamp": timestamp, + }, trace_id=origin_tid) + + # ================================================================================== + # Finish workflow + # ================================================================================== + + return diff --git a/bbdev/api/steps/verilator/04_sim_api.step.py b/bbdev/api/steps/verilator/04_sim_api.step.py new file mode 100644 index 000000000..07ad7c5c7 --- /dev/null +++ b/bbdev/api/steps/verilator/04_sim_api.step.py @@ -0,0 +1,27 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "verilator-sim-api", + "description": "run verilator simulation", + "flows": ["verilator"], + "triggers": [api("POST", "/verilator/sim")], + "enqueues": ["verilator.sim"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + binary = body.get("binary", "") + if not binary: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 400, + "message": "binary parameter is required", + }, + ) + + await ctx.enqueue({"topic": "verilator.sim", "data": {**body, "task": "sim", "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/04_sim_api_step.py b/bbdev/api/steps/verilator/04_sim_api_step.py deleted file mode 100644 index dbfd4d00c..000000000 --- a/bbdev/api/steps/verilator/04_sim_api_step.py +++ /dev/null @@ -1,40 +0,0 @@ -import os -import sys -import asyncio -from utils.event_common import wait_for_result - -config = { - "type": "api", - "name": "Verilator Sim", - "description": "run verilator simulation", - "path": "/verilator/sim", - "method": "POST", - "emits": ["verilator.sim"], - "flows": ["verilator"], -} - - -async def handler(req, context): - body = req.get("body") or {} - binary = body.get("binary", "") - batch = body.get("batch", False) - if not binary: - return { - "status": 400, - "body": { - "success": False, - "failure": True, - "returncode": 400, - "message": "binary parameter is required", - }, - } - - await context.emit({"topic": "verilator.sim", "data": {**body, "task": "sim"}}) - # ================================================================================== - # Wait for simulation result - # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) diff --git a/bbdev/api/steps/verilator/04_sim_event_step.py b/bbdev/api/steps/verilator/04_sim_event.step.py similarity index 50% rename from bbdev/api/steps/verilator/04_sim_event_step.py rename to bbdev/api/steps/verilator/04_sim_event.step.py index ee9c14c15..a6cf81f44 100644 --- a/bbdev/api/steps/verilator/04_sim_event_step.py +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -3,6 +3,8 @@ import sys from datetime import datetime +from motia import FlowContext, queue + # Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: @@ -11,19 +13,19 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger from utils.search_workload import search_workload -from utils.event_common import check_result +from utils.event_common import check_result, get_origin_trace_id config = { - "type": "event", - "name": "make sim", + "name": "verilator-sim", "description": "run simulation", - "subscribes": ["verilator.sim"], - "emits": [], "flows": ["verilator"], + "triggers": [queue("verilator.sim")], + "enqueues": [], } -async def handler(data, context): +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) # ================================================================================== # Get simulation parameters # ================================================================================== @@ -31,102 +33,97 @@ async def handler(data, context): arch_dir = f"{bbdir}/arch" build_dir = f"{arch_dir}/build" + # Generate timestamp timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - binary_name = data.get("binary", "") - coverage = data.get("coverage", False) + binary_name = input_data.get("binary", "") success_result, failure_result = await check_result( - context, returncode=(binary_name is None), continue_run=True + ctx, returncode=(binary_name == None), continue_run=True, trace_id=origin_tid, ) binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) - context.logger.info(f"binary_path: {binary_path}") + ctx.logger.info(f"binary_path: {binary_path}") success_result, failure_result = await check_result( - context, returncode=(binary_path is None), continue_run=True + ctx, returncode=(binary_path == None), continue_run=True, trace_id=origin_tid, ) if failure_result: - context.logger.error("binary not found", failure_result) + ctx.logger.error("binary not found", failure_result) return - topname = "BBSimHarness" + # Create log and waveform directory log_dir = f"{arch_dir}/log/{timestamp}-{binary_name}" waveform_dir = f"{arch_dir}/waveform/{timestamp}-{binary_name}" + topname = "TestHarness" os.makedirs(log_dir, exist_ok=True) os.makedirs(waveform_dir, exist_ok=True) - coverage_flag = "" - if coverage: - coverage_dat_path = f"{log_dir}/coverage.dat" - coverage_flag = f"+verilator+coverage+file+{coverage_dat_path}" - bin_path = f"{build_dir}/obj_dir/V{topname}" - batch = data.get("batch", False) + batch = input_data.get("batch", False) - log_path = f"{log_dir}/bdb.ndjson" - stdout_path = f"{log_dir}/stdout.log" - meta_path = f"{log_dir}/sim_meta.txt" - fst_path = f"{waveform_dir}/waveform.fst" + # Create log and waveform file + log_path = f"{log_dir}/bdb.log" + fst_path = f"{waveform_dir}/waveform.fst" + # Remove old waveform file + subprocess.run(f"rm -f {waveform_dir}/waveform.vcd", shell=True, check=True) # ================================================================================== - # Execute simulation - # BBSimHarness uses +elf= for ELF loading (via SimDRAM_bb.cc / libelf) - # No fesvr, no +loadmem_addr needed - # - # disasm.log: only stderr -> spike-dasm (Rocket commit printf is stderr here; - # merging stdout with 2>&1 can break: full stdio buffering + non-DASM bytes). - # BDB_SIM_META moves NDJSON banner to sim_meta.txt so it does not pollute disasm. + # Execute simulation script with streaming output # ================================================================================== + bebop_isa_sim = f"{bbdir}/bebop/host/spike/riscv-isa-sim" ld_lib_path = ( + f"{bebop_isa_sim}/install/lib:" f"{bbdir}/result/lib:" f"{arch_dir}/thirdparty/chipyard/tools/DRAMSim2" ) sim_cmd = ( f"export LD_LIBRARY_PATH=\"{ld_lib_path}:$LD_LIBRARY_PATH\"; " - f"export BDB_SIM_META=\"{meta_path}\"; " - f"{bin_path} +permissive " - f"+elf={binary_path} " - f"{'+batch ' if batch else ''}" - f"{coverage_flag + ' ' if coverage_flag else ''}" - f"+fst={fst_path} +log={log_path} +stdout={stdout_path} +trace=all +permissive-off " - f"{binary_path} 2> >(spike-dasm > {log_dir}/disasm.log)" + f"{bin_path} +permissive +loadmem={binary_path} +loadmem_addr=800000000 " + f"{'+batch ' if batch else ''} " + f"+fst={fst_path} +log={log_path} +permissive-off " + f"{binary_path} > >(tee {log_dir}/stdout.log) 2> >(spike-dasm > {log_dir}/disasm.log)" ) script_dir = os.path.dirname(__file__) result = stream_run_logger( cmd=sim_cmd, - logger=context.logger, + logger=ctx.logger, cwd=script_dir, stdout_prefix="verilator sim", stderr_prefix="verilator sim", executable="bash", ) success_result, failure_result = await check_result( - context, returncode=result.returncode, continue_run=True - ) + ctx, returncode=result.returncode, continue_run=True, trace_id=origin_tid) if failure_result: - context.logger.error("sim failed", failure_result) + ctx.logger.error("sim failed", failure_result) return + if os.path.exists(f"{waveform_dir}/waveform.fst.heir"): + subprocess.run( + f"gtkwave -f {waveform_dir}/waveform.fst -H {waveform_dir}/waveform.fst.heir", + shell=True, + check=True, + ) + # ================================================================================== # Return simulation result # ================================================================================== - extra_fields = { - "task": "sim", - "binary": binary_path, - "log_dir": log_dir, - "waveform_dir": waveform_dir, - "timestamp": timestamp, - "sim_meta": meta_path, - } - if coverage: - extra_fields["coverage_dat"] = coverage_dat_path - + # This is the end point of the run workflow, status will no longer be set to processing success_result, failure_result = await check_result( - context, + ctx, result.returncode, continue_run=False, - extra_fields=extra_fields, - ) + extra_fields={ + "task": "sim", + "binary": binary_path, + "log_dir": log_dir, + "waveform_dir": waveform_dir, + "timestamp": timestamp, + }, trace_id=origin_tid) + + # ================================================================================== + # Finish workflow + # ================================================================================== return diff --git a/bbdev/api/steps/verilator/05_run_api.step.py b/bbdev/api/steps/verilator/05_run_api.step.py new file mode 100644 index 000000000..b599755b3 --- /dev/null +++ b/bbdev/api/steps/verilator/05_run_api.step.py @@ -0,0 +1,25 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "verilator-run-api", + "description": "trigger complete verilator workflow", + "flows": ["verilator"], + "triggers": [api("POST", "/verilator/run")], + "enqueues": ["verilator.run"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + + data = { + "binary": body.get("binary", ""), + "config": body.get("config", "sims.verilator.BuckyballToyVerilatorConfig"), + "jobs": body.get("jobs", "16"), + "batch": body.get("batch", False), + "cosim": body.get("cosim", False), + "from_run_workflow": True, + } + + await ctx.enqueue({"topic": "verilator.run", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/05_run_api_step.py b/bbdev/api/steps/verilator/05_run_api_step.py deleted file mode 100644 index 1ce1a7636..000000000 --- a/bbdev/api/steps/verilator/05_run_api_step.py +++ /dev/null @@ -1,53 +0,0 @@ -import asyncio -from utils.event_common import wait_for_result - -config = { - "type": "api", - "name": "Verilator Complete Workflow", - "description": "trigger complete verilator workflow: clean → verilog → build → sim", - "path": "/verilator/run", - "method": "POST", - "emits": ["verilator.run"], - "flows": ["verilator"], -} - - -async def handler(req, context): - body = req.get("body") or {} - - config = { - "binary": body.get("binary", ""), - "config": body.get("config", "sims.verilator.BuckyballToyVerilatorConfig"), - "jobs": body.get("jobs", "16"), - "batch": body.get("batch", False), - "coverage": body.get("coverage", False), - "from_run_workflow": True, - } - - await context.emit({"topic": "verilator.run", "data": config}) - - # ================================================================================== - # Wait for simulation result - # - # Expected return result format: - # { - # "status": 200/400/500, - # "body": { - # "success": true/false, - # "failure": true/false, - # "processing": true/false, - # "return_code": 0, - # other fields - # } - # } - # - # Since the Motia framework wraps data in the data field, it needs to be unpacked - # if isinstance(result, dict) and 'data' in result: - # return result['data'] - # return result - # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) diff --git a/bbdev/api/steps/workload/01_buidl_api.step.py b/bbdev/api/steps/workload/01_buidl_api.step.py new file mode 100644 index 000000000..752523d26 --- /dev/null +++ b/bbdev/api/steps/workload/01_buidl_api.step.py @@ -0,0 +1,19 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path + +config = { + "name": "workload-build-api", + "description": "build workload", + "flows": ["workload"], + "triggers": [api("POST", "/workload/build")], + "enqueues": ["workload.build"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = request.body or {} + data = {"workload": body.get("workload", "")} + await ctx.enqueue({"topic": "workload.build", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/01_buidl_api_step.py b/bbdev/api/steps/workload/01_buidl_api_step.py deleted file mode 100644 index daade0a02..000000000 --- a/bbdev/api/steps/workload/01_buidl_api_step.py +++ /dev/null @@ -1,38 +0,0 @@ -import subprocess -import sys -import os -import asyncio -from utils.event_common import wait_for_result - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path - -config = { - "type": "api", - "name": "build workload", - "description": "build workload", - "path": "/workload/build", - "method": "POST", - "emits": ["workload.build"], - "flows": ["workload"], -} - - -async def handler(req, context): - bbdir = get_buckyball_path() - body = req.get("body") or {} - data = {"workload": body.get("workload", "")} - workload_dir = f"{bbdir}/bb-tests/workload" - await context.emit({"topic": "workload.build", "data": data}) - - # ================================================================================== - # Wait for simulation result - # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) diff --git a/bbdev/api/steps/workload/01_build_event_step.py b/bbdev/api/steps/workload/01_build_event.step.py similarity index 78% rename from bbdev/api/steps/workload/01_build_event_step.py rename to bbdev/api/steps/workload/01_build_event.step.py index 289cdb457..2fee01e0c 100644 --- a/bbdev/api/steps/workload/01_build_event_step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -1,30 +1,29 @@ -from contextlib import redirect_stdout import os import subprocess import sys -import time + +from motia import FlowContext, queue # Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) - from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger -from utils.event_common import check_result +from utils.event_common import check_result, get_origin_trace_id config = { - "type": "event", - "name": "build workload", + "name": "workload-build", "description": "build workload", - "subscribes": ["workload.build"], - "emits": [], "flows": ["workload"], + "triggers": [queue("workload.build")], + "enqueues": [], } -async def handler(data, context): +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() workload_dir = f"{bbdir}/bb-tests" build_dir = f"{workload_dir}/build" @@ -33,12 +32,12 @@ async def handler(data, context): subprocess.run(f"rm -rf {build_dir} && mkdir -p {build_dir}", shell=True) command = f"cd {build_dir} && cmake -G Ninja .. && ninja -j{os.cpu_count()}" - context.logger.info( + ctx.logger.info( "Executing workload command", {"command": command, "cwd": build_dir} ) result = stream_run_logger( cmd=command, - logger=context.logger, + logger=ctx.logger, cwd=workload_dir, executable="bash", stdout_prefix="workload build", @@ -50,8 +49,7 @@ async def handler(data, context): # ================================================================================== # This is the end of run workflow, status no longer set to processing success_result, failure_result = await check_result( - context, result.returncode, continue_run=False - ) + ctx, result.returncode, continue_run=False, trace_id=origin_tid) # ================================================================================== # finish workflow diff --git a/bbdev/api/steps/yosys/01_synth_api_step.py b/bbdev/api/steps/yosys/01_synth_api.step.py similarity index 61% rename from bbdev/api/steps/yosys/01_synth_api_step.py rename to bbdev/api/steps/yosys/01_synth_api.step.py index 5eaf482d7..b50fd1926 100644 --- a/bbdev/api/steps/yosys/01_synth_api_step.py +++ b/bbdev/api/steps/yosys/01_synth_api.step.py @@ -1,37 +1,30 @@ -import asyncio -from utils.event_common import wait_for_result - +from motia import ApiRequest, ApiResponse, FlowContext, api from utils.path import get_buckyball_path - +from utils.event_common import wait_for_result config = { - "type": "api", "name": "Yosys Synth", "description": "run yosys synthesis for area estimation", - "path": "/yosys/synth", - "method": "POST", - "emits": ["yosys.synth"], "flows": ["yosys"], + "triggers": [api("POST", "/yosys/synth")], + "enqueues": ["yosys.synth"], } -async def handler(req, context): +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() - body = req.get("body") or {} + body = req.body or {} data = { "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), "top": body.get("top"), "config": body.get("config"), } - await context.emit({"topic": "yosys.synth", "data": data}) + await ctx.enqueue({"topic": "yosys.synth", "data": {**data, "_trace_id": ctx.trace_id}}) # ================================================================================== # Wait for synthesis result # ================================================================================== - while True: - result = await wait_for_result(context) - if result is not None: - return result - await asyncio.sleep(1) + result = await wait_for_result(ctx) + return result diff --git a/bbdev/api/steps/yosys/01_synth_event_step.py b/bbdev/api/steps/yosys/01_synth_event.step.py similarity index 78% rename from bbdev/api/steps/yosys/01_synth_event_step.py rename to bbdev/api/steps/yosys/01_synth_event.step.py index 1cc2a1968..56845c721 100644 --- a/bbdev/api/steps/yosys/01_synth_event_step.py +++ b/bbdev/api/steps/yosys/01_synth_event.step.py @@ -4,6 +4,8 @@ import sys import yaml +from motia import FlowContext, queue + # Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: @@ -11,15 +13,14 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger -from utils.event_common import check_result +from utils.event_common import check_result, get_origin_trace_id config = { - "type": "event", "name": "yosys synth", "description": "run yosys synthesis for area estimation", - "subscribes": ["yosys.synth"], - "emits": [], "flows": ["yosys"], + "triggers": [queue("yosys.synth")], + "enqueues": [], } @@ -32,18 +33,19 @@ def load_yosys_config(): return {} -async def handler(data, context): +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - build_dir = data.get("output_dir", f"{bbdir}/arch/build/") + build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/") arch_dir = f"{bbdir}/arch" # Load yaml config as defaults, CLI args override yosys_cfg = load_yosys_config() - top_module = data.get("top") or yosys_cfg.get("top") or "BuckyballAccelerator" + top_module = input_data.get("top") or yosys_cfg.get("top") or "BuckyballAccelerator" liberty = yosys_cfg.get("liberty") # Path to .lib file, configured in yosys-config.yaml only - elaborate_config = data.get("config") or yosys_cfg.get("elaborate_config", "sims.verilator.BuckyballToyVerilatorConfig") + elaborate_config = input_data.get("config") or yosys_cfg.get("elaborate_config", "sims.verilator.BuckyballToyVerilatorConfig") - context.logger.info(f"Top module: {top_module}, Elaborate config: {elaborate_config}") + ctx.logger.info(f"Top module: {top_module}, Elaborate config: {elaborate_config}") # ================================================================================== # Step 1: Generate Verilog via Elaborate (full SoC), filter DPI-C files later @@ -53,7 +55,7 @@ async def handler(data, context): shutil.rmtree(build_dir) os.makedirs(build_dir, exist_ok=True) - context.logger.info("Step 1: Generating Verilog via Elaborate...") + ctx.logger.info("Step 1: Generating Verilog via Elaborate...") verilog_command = ( f"mill -i __.test.runMain sims.verilator.Elaborate {elaborate_config} " "--disable-annotation-unknown -strip-debug-info -O=debug " @@ -63,19 +65,20 @@ async def handler(data, context): result = stream_run_logger( cmd=verilog_command, - logger=context.logger, + logger=ctx.logger, cwd=arch_dir, stdout_prefix="yosys verilog", stderr_prefix="yosys verilog", ) if result.returncode != 0: - context.logger.error(f"Verilog generation failed with code {result.returncode}") + ctx.logger.error(f"Verilog generation failed with code {result.returncode}") success_result, failure_result = await check_result( - context, + ctx, result.returncode, continue_run=False, extra_fields={"task": "verilog"}, + trace_id=origin_tid, ) return failure_result @@ -88,7 +91,7 @@ async def handler(data, context): # ================================================================================== # Step 2: Run Yosys synthesis for area estimation # ================================================================================== - context.logger.info("Step 2: Running Yosys synthesis for area estimation...") + ctx.logger.info("Step 2: Running Yosys synthesis for area estimation...") # Collect SystemVerilog sources only (.sv), skip .v files which may contain DPI-C imports vsrcs = glob.glob(f"{build_dir}/**/*.sv", recursive=True) @@ -107,16 +110,17 @@ async def handler(data, context): vsrcs = filtered_vsrcs if skipped: - context.logger.info(f"Skipped {len(skipped)} files with unsupported syntax: {', '.join(skipped[:10])}{'...' if len(skipped) > 10 else ''}") - context.logger.info(f"Found {len(vsrcs)} synthesizable SystemVerilog source files") + ctx.logger.info(f"Skipped {len(skipped)} files with unsupported syntax: {', '.join(skipped[:10])}{'...' if len(skipped) > 10 else ''}") + ctx.logger.info(f"Found {len(vsrcs)} synthesizable SystemVerilog source files") if not vsrcs: - context.logger.error(f"No Verilog source files found in {build_dir}") + ctx.logger.error(f"No Verilog source files found in {build_dir}") success_result, failure_result = await check_result( - context, + ctx, 1, continue_run=False, extra_fields={"task": "synth", "error": "No Verilog source files found"}, + trace_id=origin_tid, ) return failure_result @@ -137,7 +141,7 @@ async def handler(data, context): f.write(f"synth -top {top_module}\n") if liberty and os.path.exists(liberty): - context.logger.info(f"Using liberty file: {liberty}") + ctx.logger.info(f"Using liberty file: {liberty}") f.write(f"dfflibmap -liberty {liberty}\n") f.write(f"abc -liberty {liberty}\n") @@ -153,7 +157,7 @@ async def handler(data, context): f.write(f"write_verilog {yosys_output_dir}/synth_netlist.v\n") else: if liberty: - context.logger.warn(f"Liberty file not found: {liberty}, falling back to generic stat") + ctx.logger.warn(f"Liberty file not found: {liberty}, falling back to generic stat") # Hierarchical breakdown before flatten f.write(f"tee -o {yosys_output_dir}/hierarchy_report.txt stat\n") f.write("flatten\n") @@ -161,15 +165,15 @@ async def handler(data, context): f.write("stat\n") f.write(f"tee -o {yosys_output_dir}/area_report.txt stat\n") - context.logger.info(f"Yosys script written to {yosys_script}") - context.logger.info(f"Synthesizing {len(vsrcs)} source files with top module: {top_module}") + ctx.logger.info(f"Yosys script written to {yosys_script}") + ctx.logger.info(f"Synthesizing {len(vsrcs)} source files with top module: {top_module}") # Run yosys yosys_command = f"yosys -s {yosys_script}" result = stream_run_logger( cmd=yosys_command, - logger=context.logger, + logger=ctx.logger, cwd=build_dir, stdout_prefix="yosys synth", stderr_prefix="yosys synth", @@ -183,7 +187,7 @@ async def handler(data, context): timing_report_file = f"{yosys_output_dir}/timing_report.txt" if liberty and os.path.exists(liberty) and os.path.exists(netlist_file) and result.returncode == 0: - context.logger.info("Step 3: Running OpenSTA timing analysis...") + ctx.logger.info("Step 3: Running OpenSTA timing analysis...") # Get target clock period from config (default 10ns = 100MHz) clock_period = yosys_cfg.get("clock_period", 10.0) @@ -202,7 +206,7 @@ async def handler(data, context): sta_result = stream_run_logger( cmd=f"sta {sta_script}", - logger=context.logger, + logger=ctx.logger, cwd=yosys_output_dir, stdout_prefix="opensta", stderr_prefix="opensta", @@ -211,11 +215,11 @@ async def handler(data, context): if sta_result.returncode == 0 and os.path.exists(timing_report_file): with open(timing_report_file, "r") as f: extra["timing_report"] = f.read() - context.logger.info(f"Timing report saved to {timing_report_file}") + ctx.logger.info(f"Timing report saved to {timing_report_file}") else: - context.logger.warn(f"OpenSTA timing analysis failed (rc={sta_result.returncode})") + ctx.logger.warn(f"OpenSTA timing analysis failed (rc={sta_result.returncode})") elif result.returncode == 0: - context.logger.info("Skipping OpenSTA: no liberty file configured or synthesis failed") + ctx.logger.info("Skipping OpenSTA: no liberty file configured or synthesis failed") # ================================================================================== # Return result to API @@ -224,19 +228,20 @@ async def handler(data, context): if os.path.exists(report_file): with open(report_file, "r") as f: extra["area_report"] = f.read() - context.logger.info(f"Area report saved to {report_file}") + ctx.logger.info(f"Area report saved to {report_file}") hierarchy_file = f"{yosys_output_dir}/hierarchy_report.txt" if os.path.exists(hierarchy_file): with open(hierarchy_file, "r") as f: extra["hierarchy_report"] = f.read() - context.logger.info(f"Hierarchy report saved to {hierarchy_file}") + ctx.logger.info(f"Hierarchy report saved to {hierarchy_file}") success_result, failure_result = await check_result( - context, + ctx, result.returncode, continue_run=False, extra_fields=extra, + trace_id=origin_tid, ) return diff --git a/bbdev/api/utils/event_common.py b/bbdev/api/utils/event_common.py index 717969227..20081740e 100644 --- a/bbdev/api/utils/event_common.py +++ b/bbdev/api/utils/event_common.py @@ -3,23 +3,36 @@ """ -async def check_result(context, returncode, continue_run=False, extra_fields=None): +def get_origin_trace_id(input_data, ctx): + """Get the origin trace_id from input_data (passed by API step) or fall back to ctx.trace_id. + + iii 0.7+ assigns a new trace_id per handler invocation, so the API step's + trace_id must be forwarded explicitly via input_data["_trace_id"]. + """ + if isinstance(input_data, dict) and "_trace_id" in input_data: + return input_data["_trace_id"] + return ctx.trace_id + + +async def check_result(ctx, returncode, continue_run=False, extra_fields=None, trace_id=None): """ Check returncode, create appropriate result objects and set state. Args: - context: The event context object + ctx: The flow context object returncode: The return code (int) continue_run: If True, set processing state instead of success/failure extra_fields: Optional dictionary of extra fields to include in result body + trace_id: Optional trace_id to use as state scope (defaults to ctx.trace_id) Returns: tuple: (success_result, failure_result) - one will be None based on returncode and continue_run """ extra_fields = extra_fields or {} + scope = trace_id or ctx.trace_id if continue_run: - await context.state.set(context.trace_id, "processing", True) + await ctx.state.set(scope, "processing", True) return None, None elif returncode != 0: failure_result = { @@ -32,7 +45,7 @@ async def check_result(context, returncode, continue_run=False, extra_fields=Non **extra_fields, }, } - await context.state.set(context.trace_id, "failure", failure_result) + await ctx.state.set(scope, "failure", failure_result) return None, failure_result else: success_result = { @@ -45,67 +58,5 @@ async def check_result(context, returncode, continue_run=False, extra_fields=Non **extra_fields, }, } - await context.state.set(context.trace_id, "success", success_result) + await ctx.state.set(scope, "success", success_result) return success_result, None - - -# ================================================================================== -# API waits for event return result -# -# Expected return result format: -# { -# "status": 200/400/500, -# "body": { -# "success": true/false, -# "failure": true/false, -# "processing": true/false, -# "return_code": 0, -# other fields -# } -# } -# -# Since the Motia framework wraps data in the data field, it needs to be unpacked -# if isinstance(result, dict) and 'data' in result: -# return result['data'] -# return result -# ================================================================================== - - -async def wait_for_result(context): - """ - Check for task completion state (success or failure). - Returns result if found, None if still processing. - - Args: - context: The event context object - - Returns: - dict or None: The result data if task completed, None if still processing - """ - # Check for success result - success_result = await context.state.get(context.trace_id, "success") - if success_result and success_result.get("data"): - # Filter out invalid null state - if success_result == {"data": None} or ( - isinstance(success_result, dict) - and success_result.get("data") is None - and len(success_result) == 1 - ): - await context.state.delete(context.trace_id, "success") - return None - context.logger.info("task completed") - - if isinstance(success_result, dict) and "data" in success_result: - return success_result["data"] - return success_result - - # Check for error status - failure_result = await context.state.get(context.trace_id, "failure") - if failure_result and failure_result.get("data"): - context.logger.error("task failed", failure_result) - - if isinstance(failure_result, dict) and "data" in failure_result: - return failure_result["data"] - return failure_result - - return None diff --git a/bbdev/api/utils/port.py b/bbdev/api/utils/port.py index 82e6bfeda..f61b8c2d2 100644 --- a/bbdev/api/utils/port.py +++ b/bbdev/api/utils/port.py @@ -9,8 +9,25 @@ def find_available_port(start_port: int = 5000, end_port: int = 5500) -> int: sock.bind(("localhost", port)) return port except OSError: - # Port is already in use, try next one continue - # If no port is available in the range, raise an exception + raise RuntimeError(f"No available port found in range {start_port}-{end_port}") + + +def reserve_port(start_port: int = 5000, end_port: int = 5500) -> tuple[int, socket.socket]: + """Reserve a port by binding and holding the socket open. + + The caller must close the returned socket once the engine has taken over the port. + This prevents TOCTOU races when multiple bbdev instances start concurrently. + """ + for port in range(start_port, end_port + 1): + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("localhost", port)) + return port, sock + except OSError: + sock.close() + continue + raise RuntimeError(f"No available port found in range {start_port}-{end_port}") diff --git a/bbdev/bbdev b/bbdev/bbdev index 8110d68a4..a203148bc 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -8,12 +8,139 @@ import signal import shlex import json import time -import shutil +import tempfile +import threading import requests -from utils import find_available_port - -workflow_dir = os.path.dirname(os.path.abspath(__file__)) + "/api" +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "api")) +from utils.port import reserve_port + +workflow_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "api") + +NO_PROXY = {"http": None, "https": None, "all": None} + +# Subcommand definitions: {name: {operation: help_text, ...}} +# Operations with string values accept sub-arguments; True means boolean-only (no sub-args). +WORKFLOW_COMMANDS = { + "verilator": { + "clean": True, + "verilog": 'Generate verilog files. Args: "[--balltype ] [--job ] [--batch]"', + "build": 'Build simulation executable. Args: "[--job ]"', + "sim": 'Run simulation. Args: "--binary [--batch]"', + "run": 'Integrated build+sim+run. Args: "--binary [--batch] [--job ]"', + "cosim": 'Run cosimulation. Args: "--binary [--batch]"', + }, + "vcs": { + "clean": True, + "verilog": True, + "build": 'Build simulation executable. Args: "[--job ]"', + "sim": 'Run simulation. Args: "--binary [--batch]"', + "run": 'Integrated build+sim+run. Args: "--binary [--batch] [--job ]"', + }, + "sardine": { + "run": 'Run sardine. Args: "--workload "', + }, + "agent": { + "chat": 'Run agent. Args: "--message ", "--model "', + }, + "workload": { + "build": "Build workload.", + }, + "doc": { + "deploy": True, + }, + "marshal": { + "build": "Build marshal.", + "launch": "Launch marshal.", + }, + "firesim": { + "buildbitstream": "Build bitstream.", + "infrasetup": "Infrasetup.", + "runworkload": "Run workload.", + }, + "compiler": { + "build": "Build compiler.", + }, + "funcsim": { + "build": "Build funcsim.", + "sim": "Sim funcsim.", + }, + "uvm": { + "builddut": "Build dut.", + "build": "Build uvm.", + }, + "palladium": { + "verilog": 'Generate verilog files. Args: "[--config sims.palladium.BuckyballToyP2EConfig]"', + }, +} + +ALL_COMMANDS = ["start", "stop"] + list(WORKFLOW_COMMANDS) + + +def make_iii_config(port: int, worker_port: int, quiet: bool = False) -> str: + """Create a temporary config.yaml with unique HTTP and Worker ports, return the path.""" + base_config = os.path.join(workflow_dir, "config.yaml") + with open(base_config, "r") as f: + content = f.read() + # Replace top-level engine WebSocket port and WorkerModule port (both default 49134), + # then RestApiModule HTTP port (default 3111). + content = content.replace("port: 49134", f"port: {worker_port}") + content = content.replace("port: 3111", f"port: {port}", 1) + if quiet: + content = content.replace( + "exporter: memory", + "exporter: memory\n level: warn\n logs_console_output: false", + ) + tmp = tempfile.NamedTemporaryFile( + mode="w", suffix=".yaml", prefix="iii-config-", dir=workflow_dir, delete=False + ) + tmp.write(content) + tmp.close() + return tmp.name + + +def cleanup_config(config_path: str): + """Remove temp config file, ignoring if already deleted.""" + try: + os.unlink(config_path) + except FileNotFoundError: + pass + + +def _kill_proc_tree(pid: int): + """Kill a process and all its descendants via /proc. + Kills children first (bottom-up) so they are still in the process tree + and discoverable before their parent dies.""" + def _get_children(parent_pid): + children = [] + try: + for entry in os.listdir("/proc"): + if not entry.isdigit(): + continue + try: + with open(f"/proc/{entry}/stat") as f: + stat = f.read().split() + if int(stat[3]) == parent_pid: + child_pid = int(entry) + children.append(child_pid) + children.extend(_get_children(child_pid)) + except (FileNotFoundError, IndexError, ValueError, PermissionError): + continue + except FileNotFoundError: + pass + return children + + # children first, then parent — so children are still discoverable in /proc + children = _get_children(pid) + for p in reversed(children): + try: + os.kill(p, signal.SIGTERM) + except (ProcessLookupError, OSError): + pass + try: + os.kill(pid, signal.SIGTERM) + except (ProcessLookupError, OSError): + pass def parse_args(argv: list[str]) -> argparse.Namespace: @@ -23,552 +150,331 @@ def parse_args(argv: list[str]) -> argparse.Namespace: formatter_class=argparse.RawTextHelpFormatter, ) - # Global parameters - parser.add_argument( - "--port", type=int, default=None, help="Port for dev server (default: None)" + # Global parameters shared across all subcommands + common = argparse.ArgumentParser(add_help=False) + common.add_argument( + "--port", type=int, default=None, help="Port for dev server (default: auto)" ) - parser.add_argument("--server", action="store_true", help="server mode") + common.add_argument("--server", action="store_true", help="Server mode") - # Subcommand parser subparsers = parser.add_subparsers(dest="command", help="Available commands") - # ===== start subcommand ============================================================================== - start_parser = subparsers.add_parser("start", help="Start dev server") - - # ===== stop subcommand ============================================================================== - stop_parser = subparsers.add_parser("stop", help="Stop dev server") - stop_group = stop_parser.add_mutually_exclusive_group(required=False) - stop_group.add_argument("--all", action="store_true", help="Stop all servers") - - # ===== verilator subcommand ========================================================================= - verilator_parser = subparsers.add_parser("verilator", help="Verilator operations") - # Mutually exclusive option group for verilator - only one operation can be selected - verilator_group = verilator_parser.add_mutually_exclusive_group(required=True) - verilator_group.add_argument( - "--clean", action="store_true", help="Clean verilator build directory" - ) - verilator_group.add_argument( - "--verilog", - type=str, - nargs="?", - const="", - metavar="ARGS", - help='Generate verilog files from chisel. Args: "[--balltype ] [--job ] [--batch]"', - ) - verilator_group.add_argument( - "--build", - type=str, - nargs="?", - const="", - metavar="ARGS", - help='Build verilator simulation executable. Args: "[--job ] [--coverage]"', - ) - verilator_group.add_argument( - "--sim", - type=str, - nargs="?", - const="", - metavar="ARGS", - help='Run verilator simulation. Args: "--binary [--batch] [--coverage]"', - ) - verilator_group.add_argument( - "--run", - type=str, - nargs="?", - const="", - metavar="ARGS", - help='Integrated build+sim+run. Args: "--binary [--batch] [--job ] [--coverage]"', - ) - verilator_group.add_argument( - "--cosim", - type=str, - nargs="?", - const="", - metavar="ARGS", - help='Run verilator cosimulation. Args: "--binary [--batch]"', - ) - - # ===== vcs subcommand ========================================================================= - vcs_parser = subparsers.add_parser("vcs", help="vcs operations") - # Mutually exclusive option group for vcs - only one operation can be selected - vcs_group = vcs_parser.add_mutually_exclusive_group(required=True) - vcs_group.add_argument( - "--clean", action="store_true", help="Clean vcs build directory" - ) - vcs_group.add_argument( - "--verilog", action="store_true", help="Generate verilog files" - ) - vcs_group.add_argument( - "--build", - type=str, - nargs="?", - const="", - metavar="ARGS", - help='Build vcs simulation executable. Args: "[--job ]"', - ) - vcs_group.add_argument( - "--sim", - type=str, - nargs="?", - const="", - metavar="ARGS", - help='Run vcs simulation. Args: "--binary [--batch]"', - ) - vcs_group.add_argument( - "--run", - type=str, - nargs="?", - const="", - metavar="ARGS", - help='Integrated build+sim+run. Args: "--binary [--batch] [--job ]"', - ) + # start / stop + subparsers.add_parser("start", parents=[common], help="Start dev server") + stop_parser = subparsers.add_parser("stop", parents=[common], help="Stop dev server") + stop_parser.add_argument("--all", action="store_true", help="Stop all servers") + + # Build workflow subcommands from WORKFLOW_COMMANDS table + for cmd_name, operations in WORKFLOW_COMMANDS.items(): + cmd_parser = subparsers.add_parser( + cmd_name, parents=[common], help=f"{cmd_name} operations" + ) + group = cmd_parser.add_mutually_exclusive_group(required=True) + for op_name, op_help in operations.items(): + if op_help is True: + group.add_argument(f"--{op_name}", action="store_true", help=f"{op_name.capitalize()}.") + else: + group.add_argument( + f"--{op_name}", + type=str, + nargs="?", + const="", + metavar="ARGS", + help=op_help, + ) - # ===== sardine subcommand ========================================================================= - sardine_parser = subparsers.add_parser("sardine", help="sardine operations") - sardine_group = sardine_parser.add_mutually_exclusive_group(required=True) - sardine_group.add_argument( - "--run", - type=str, - nargs="?", - const="", - metavar="ARGS", - help='Run sardine. Args: "--workload [--coverage]"', - ) + return parser.parse_args(argv) - # ===== agent subcommand ========================================================================= - agent_parser = subparsers.add_parser("agent", help="agent operations") - agent_group = agent_parser.add_mutually_exclusive_group(required=True) - agent_group.add_argument( - "--chat", - type=str, - nargs="?", - const="", - metavar="ARGS", - help='Run agent. Args: "--message ", "--model "', - ) - # ===== workload subcommand ========================================================================= - workload_parser = subparsers.add_parser("workload", help="workload operations") - workload_group = workload_parser.add_mutually_exclusive_group(required=True) - workload_group.add_argument( - "--build", - type=str, - nargs="?", - const="", - metavar="ARGS", - help="Build workload. ", - ) +def extract_command_info(args: argparse.Namespace) -> dict: + """Extract command, operation, and parsed sub-arguments from argparse Namespace.""" + result = {"command": args.command, "operation": None, "args": {}} - # ===== doc subcommand ========================================================================= - doc_parser = subparsers.add_parser("doc", help="doc operations") - doc_group = doc_parser.add_mutually_exclusive_group(required=True) - doc_group.add_argument("--deploy", action="store_true", help="Deploy doc. ") + for attr_name, attr_value in vars(args).items(): + if attr_name in ("command", "port", "server"): + continue + if attr_value is None or attr_value is False: + continue - # ===== marshal subcommand ========================================================================= - marshal_parser = subparsers.add_parser("marshal", help="marshal operations") - marshal_group = marshal_parser.add_mutually_exclusive_group(required=True) - marshal_group.add_argument( - "--build", type=str, nargs="?", const="", metavar="ARGS", help="Build marshal. " - ) - marshal_group.add_argument( - "--launch", - type=str, - nargs="?", - const="", - metavar="ARGS", - help="Launch marshal. ", - ) + result["operation"] = attr_name + if attr_value is True or attr_value == "": + result["args"] = {} + else: + result["args"] = _parse_sub_args(attr_value) + break - # ===== firesim subcommand ========================================================================= - firesim_parser = subparsers.add_parser("firesim", help="firesim operations") - firesim_group = firesim_parser.add_mutually_exclusive_group(required=True) - firesim_group.add_argument( - "--buildbitstream", - type=str, - nargs="?", - const="", - metavar="ARGS", - help="Build bitstream. ", - ) - firesim_group.add_argument( - "--infrasetup", - type=str, - nargs="?", - const="", - metavar="ARGS", - help="Infrasetup. ", - ) - firesim_group.add_argument( - "--runworkload", - type=str, - nargs="?", - const="", - metavar="ARGS", - help="Run workload. ", - ) + return result - # ===== compiler subcommand ========================================================================= - compiler_parser = subparsers.add_parser("compiler", help="compiler operations") - compiler_group = compiler_parser.add_mutually_exclusive_group(required=True) - compiler_group.add_argument( - "--build", - type=str, - nargs="?", - const="", - metavar="ARGS", - help="Build compiler. ", - ) - # ===== funcsim subcommand ========================================================================= - funcsim_parser = subparsers.add_parser("funcsim", help="funcsim operations") - funcsim_group = funcsim_parser.add_mutually_exclusive_group(required=True) - funcsim_group.add_argument( - "--build", type=str, nargs="?", const="", metavar="ARGS", help="Build funcsim. " - ) - funcsim_group.add_argument( - "--sim", type=str, nargs="?", const="", metavar="ARGS", help="Sim funcsim. " - ) +def _parse_sub_args(arg_string: str) -> dict: + """Parse a sub-argument string like '--binary /path --batch' into a dict.""" + args_dict = {} + try: + tokens = shlex.split(arg_string) + except ValueError as e: + print(f"Error parsing arguments: {e}") + return args_dict + + i = 0 + while i < len(tokens): + token = tokens[i] + if token.startswith("--"): + key = token[2:] + if i + 1 < len(tokens) and not tokens[i + 1].startswith("--"): + args_dict[key] = tokens[i + 1] + i += 2 + else: + args_dict[key] = True + i += 1 + elif token.startswith("-") and len(token) == 2: + key = token[1:] + if i + 1 < len(tokens) and not tokens[i + 1].startswith("--"): + args_dict[key] = tokens[i + 1] + i += 2 + else: + args_dict[key] = True + i += 1 + else: + i += 1 + return args_dict + + +def wait_for_http(url: str, check_fn, timeout: int, error_msg: str): + """Poll an HTTP endpoint until check_fn(response) returns True, or timeout.""" + for _ in range(timeout): + try: + resp = requests.get(url, timeout=1, proxies=NO_PROXY) + if check_fn(resp): + return True + except requests.exceptions.ConnectionError: + pass + except requests.exceptions.RequestException: + return True # non-connection error means server is reachable + time.sleep(1) + print(error_msg) + return False + + +def forward_iii_output(pipe, active_event: threading.Event, quiet: bool = False): + """Forward task output to stdout. In quiet mode, suppress engine INFO/WARN logs.""" + for line in iter(pipe.readline, ""): + if not active_event.is_set(): + continue + if quiet: + # Pass through: ANSI-colored step output and ERROR lines; drop engine INFO/WARN + if line.startswith("\033[") or "[ERROR]" in line: + sys.stdout.write(line) + sys.stdout.flush() + else: + sys.stdout.write(line) + sys.stdout.flush() + pipe.close() + + +def read_state_file(path: str): + """Read JSON from an iii state file (JSON + binary padding). Returns parsed dict or None.""" + try: + with open(path, "r", errors="ignore") as f: + raw = f.read() + state_data, _ = json.JSONDecoder().raw_decode(raw) + return state_data + except (FileNotFoundError, json.JSONDecodeError, ValueError): + return None + + +def run_server_mode(args, cmd_info): + """Handle --server mode: interact with an already-running (or new) iii server.""" + command = cmd_info["command"] + + if command == "start": + print(f"Starting dev server on port {args.port}") + config_path = make_iii_config(args.port, 49134) if args.port else os.path.join(workflow_dir, "config.yaml") + try: + subprocess.run(["iii", "-c", config_path], cwd=workflow_dir, check=True) + finally: + if args.port: + cleanup_config(config_path) - # ===== uvm subcommand ========================================================================= - uvm_parser = subparsers.add_parser("uvm", help="uvm operations") - uvm_group = uvm_parser.add_mutually_exclusive_group(required=True) - uvm_group.add_argument( - "--builddut", type=str, nargs="?", const="", metavar="ARGS", help="Build dut. " - ) - uvm_group.add_argument( - "--build", type=str, nargs="?", const="", metavar="ARGS", help="Build uvm. " - ) + elif command == "stop": + if cmd_info["operation"] == "all": + print("Stopping all servers") + subprocess.run( + "kill -9 $(ps aux | grep '[i]ii' | awk '{print $2}')", + shell=True, check=False, text=True, + ) + else: + print(f"Stopping server on port {args.port}") + subprocess.run( + f"kill -TERM $(lsof -t -i :{args.port})", + shell=True, check=False, text=True, + ) - # ===== yosys subcommand ===================================================================== - yosys_parser = subparsers.add_parser("yosys", help="yosys synthesis operations") - yosys_group = yosys_parser.add_mutually_exclusive_group(required=True) - yosys_group.add_argument( - "--synth", - type=str, - nargs="?", - const="", - metavar="ARGS", - help='Run yosys synthesis for area estimation.\n' - 'Args: "[--top ] [--config ]"\n' - ' --top : top module for synthesis (default from yosys-config.yaml)\n' - ' --config : Elaborate config class (default: sims.verilator.BuckyballToyVerilatorConfig)\n' - 'Liberty library can be configured in bbdev/api/steps/yosys/scripts/yosys-config.yaml', - ) + elif command in WORKFLOW_COMMANDS: + api_path = f"/{command}/{cmd_info['operation']}" + resp = requests.post( + f"http://localhost:{args.port}{api_path}", + json=cmd_info["args"], + timeout=30, + proxies=NO_PROXY, + ) + print(resp.text) + + else: + print(f"Unknown command: {command}") + print(f"Available commands: {', '.join(ALL_COMMANDS)}") + sys.exit(1) + + +def _submit_and_poll(port: int, api_path: str, cmd_info: dict): + """Submit a task to iii and poll for its result. Returns exit code (0=ok, 1=fail).""" + base_url = f"http://localhost:{port}" + print(f"Executing {cmd_info['command']} {cmd_info['operation']}...") + try: + submit_resp = requests.post( + f"{base_url}{api_path}", + json=cmd_info["args"], + timeout=30, + proxies=NO_PROXY, + ) + submit_data = submit_resp.json() + except Exception as e: + print(f"Error: Failed to submit task: {e}") + return 1 + + if submit_resp.status_code >= 400: + print(f"Error: {submit_data}") + return 1 + + trace_id = submit_data.get("trace_id", "") + if not trace_id: + print(f"Error: No trace_id in response: {submit_data}") + return 1 + + # Poll for result + state_file = os.path.join(workflow_dir, "data", "state_store.db", f"{trace_id}.bin") + poll_timeout = 600 # 10 minute max + task_result = None + for _ in range(poll_timeout // 2): + state_data = read_state_file(state_file) + if state_data: + if "success" in state_data: + task_result = state_data["success"].get("body", state_data["success"]) + break + elif "failure" in state_data: + task_result = state_data["failure"].get("body", state_data["failure"]) + break + time.sleep(2) + else: + print("Error: Task polling timed out after 10 minutes") + return 1 + + print(f"\nTask completed on http://localhost:{port}") + if task_result and not task_result.get("success", False): + print("Error: Task failed") + return 1 + return 0 + + +def run_script_mode(args, cmd_info): + """Handle script mode: start iii, run one task, then shut down.""" + command = cmd_info["command"] + + if command in ("start", "stop"): + print(f"'{command}' does nothing in script mode") + return + + if command not in WORKFLOW_COMMANDS: + print(f"Unknown command: {command}") + print(f"Available commands: {', '.join(ALL_COMMANDS)}") + sys.exit(1) + + api_path = f"/{command}/{cmd_info['operation']}" + + # Reserve both ports atomically to prevent TOCTOU races with concurrent bbdev instances. + # Sockets stay bound until the engine has taken over (HTTP is up), then are released. + if args.port: + available_port = args.port + http_sock = None + else: + available_port, http_sock = reserve_port(start_port=5100, end_port=5500) + worker_port, worker_sock = reserve_port(start_port=49134, end_port=49500) + print(f"Starting server on port {available_port}...") + + # Clean stale state files from previous runs + state_dir = os.path.join(workflow_dir, "data", "state_store.db") + if os.path.isdir(state_dir): + for f in os.listdir(state_dir): + if f.endswith(".bin"): + try: + os.unlink(os.path.join(state_dir, f)) + except OSError: + pass - # ===== palladium subcommand =================================================================== - palladium_parser = subparsers.add_parser("palladium", help="palladium operations") - palladium_group = palladium_parser.add_mutually_exclusive_group(required=True) - palladium_group.add_argument( - "--verilog", - type=str, - nargs="?", - const="", - metavar="ARGS", - help='Generate verilog files from chisel. Args example: "[--config sims.palladium.BuckyballToyP2EConfig]"', + config_path = make_iii_config(available_port, worker_port, quiet=True) + iii_output_active = threading.Event() + iii_output_active.set() + + # Release reserved sockets just before engine starts so it can bind the same ports. + if http_sock: + http_sock.close() + worker_sock.close() + + # Set III_URL so motia SDK connects to the correct worker port + iii_env = {**os.environ, "III_URL": f"ws://localhost:{worker_port}"} + proc = subprocess.Popen( + ["iii", "-c", config_path], + cwd=workflow_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + env=iii_env, ) - # ===== pegasus subcommand =================================================================== - pegasus_parser = subparsers.add_parser("pegasus", help="pegasus operations") - pegasus_group = pegasus_parser.add_mutually_exclusive_group(required=True) - pegasus_group.add_argument( - "--verilog", - type=str, - nargs="?", - const="", - metavar="ARGS", - help='Generate verilog files from chisel. Args example: "[--config sims.pegasus.PegasusConfig]"', + output_thread = threading.Thread( + target=forward_iii_output, args=(proc.stdout, iii_output_active, True), daemon=True ) + output_thread.start() + + try: + base_url = f"http://localhost:{available_port}" + + # Phase 1: wait for iii engine HTTP + if not wait_for_http( + f"{base_url}/", + check_fn=lambda r: True, + timeout=30, + error_msg="Server failed to start within 30 seconds", + ): + sys.exit(1) - # Parse arguments, allowing unknown arguments (so --port can work in subcommands) - args, unknown = parser.parse_known_args(argv) - - # If --port and --server are in unknown arguments, handle them manually - while unknown: - found = False - for i in range(len(unknown)): - if unknown[i] == "--port": - try: - args.port = int(unknown[i + 1]) - unknown = unknown[i + 2 :] # Remove processed arguments - found = True - break - except (ValueError, IndexError): - break - if unknown[i] == "--server": - args.server = True - unknown = unknown[i + 1 :] # Remove processed arguments - found = True - break - if not found: - break - # If there are other unknown arguments, throw error - if unknown: - parser.error(f"unrecognized arguments: {' '.join(unknown)}") - - return args - - -def extract_command_info(args): - """Generic command info extractor, applicable to all commands""" - # Basic return structure - result = {"command": getattr(args, "command", None), "operation": None, "args": {}} - - # Dynamically extract operation type (iterate through all attributes of args) - for attr_name in dir(args): - # Skip built-in attributes and known non-operation attributes - if attr_name.startswith("_") or attr_name in ["command", "port"]: - continue + # Phase 2: wait for route registration + if not wait_for_http( + f"{base_url}{api_path}", + check_fn=lambda r: r.status_code != 404, + timeout=120, + error_msg="Routes failed to register within timeout", + ): + sys.exit(1) - attr_value = getattr(args, attr_name) - # Find attributes with value True or non-empty string (skip None and False) - if attr_value is not None and attr_value is not False: - result["operation"] = attr_name - if attr_value is True: - # Boolean operations, such as --clean, --verilog - result["args"] = {} - else: - # Operations with arguments, such as --sim "arg_string" - # Parse argument string - args_dict = {} - if attr_value: - try: - # use shlex to parse args - arg_tokens = shlex.split(attr_value) - i = 0 - while i < len(arg_tokens): - token = arg_tokens[i] - if token.startswith("--"): - # long option - option_name = token[2:] # remove -- - if i + 1 < len(arg_tokens) and not arg_tokens[ - i + 1 - ].startswith("-"): - # next token is value - args_dict[option_name] = arg_tokens[i + 1] - i += 2 - else: - # boolean flag - args_dict[option_name] = True - i += 1 - elif token.startswith("-") and len(token) == 2: - # short option - option_name = token[1:] # remove - - if i + 1 < len(arg_tokens) and not arg_tokens[ - i + 1 - ].startswith("-"): - # next token is value - args_dict[option_name] = arg_tokens[i + 1] - i += 2 - else: - # boolean flag - args_dict[option_name] = True - i += 1 - else: - # position argument, skip - i += 1 - except ValueError as e: - print(f"Error parsing arguments: {e}") - result["args"] = args_dict - break + print(f"Server is ready on port {available_port}") + rc = _submit_and_poll(available_port, api_path, cmd_info) + if rc != 0: + sys.exit(rc) - return result + finally: + # Cleanup: kill iii and all its descendant processes, then remove temp config + iii_output_active.clear() + time.sleep(0.5) + _kill_proc_tree(proc.pid) + proc.wait() + cleanup_config(config_path) if __name__ == "__main__": args = parse_args(sys.argv[1:]) cmd_info = extract_command_info(args) - # print(f"Command: {cmd_info['command']}") - # print(f"Operation: {cmd_info['operation']}") - # print(f"Arguments: {cmd_info['args']}") - - # ================================================================================== - # Two modes: server mode and script mode - # - # server mode: Manually start/stop server, can visualize and access workflows through browser - # script mode: Automatically assign port, start service when task begins, stop service when task ends - # ================================================================================== if args.server: - if cmd_info["command"] == "start": - print(f"Starting dev server on port {args.port}") - subprocess.run( - ["pnpm", "dev", "--port", str(args.port)], - cwd=workflow_dir, - check=True, - ) - - elif cmd_info["command"] == "stop": - if cmd_info["operation"] == "all": - print("Stopping all servers") - subprocess.run( - "kill -9 $(ps aux | grep '[m]otia' | awk '{print $2}')", - shell=True, - check=False, - text=True, - ) - else: - print(f"Stopping server on port {args.port}") - subprocess.run( - f"kill -TERM $(lsof -t -i :{args.port})", - shell=True, - check=False, - text=True, - ) - - elif cmd_info["command"] in [ - "verilator", - "vcs", - "sardine", - "agent", - "workload", - "doc", - "marshal", - "firesim", - "compiler", - "funcsim", - "uvm", - "palladium", - "yosys", - "pegasus", - ]: - api_path = f"/{cmd_info['command']}/{cmd_info['operation']}" - json_data = json.dumps(cmd_info["args"]) - subprocess.run( - [ - "curl", - "-X", - "POST", - f"http://localhost:{args.port}{api_path}", - "-H", - "Content-Type: application/json", - "-d", - json_data, - ], - cwd=workflow_dir, - check=True, - ) - else: - print(f"Unknown command: {cmd_info['command']}") - print( - "Available commands: start, stop, verilator, vcs, sardine, agent, \ - workload, doc, marshal, firesim, compiler, funcsim, uvm, yosys, pegasus" - ) - sys.exit(1) - else: # script mode - if cmd_info["command"] == "start": - print(" 'start' do nothing in script mode") - elif cmd_info["command"] == "stop": - print(" 'stop' do nothing in script mode") - elif cmd_info["command"] in [ - "verilator", - "vcs", - "sardine", - "agent", - "workload", - "doc", - "marshal", - "firesim", - "compiler", - "funcsim", - "uvm", - "palladium", - "yosys", - "pegasus", - ]: - # 0. Clean up stale BullMQ/Redis data to prevent replaying old events - aof_dir = os.path.join(workflow_dir, ".motia", "appendonlydir") - if os.path.exists(aof_dir): - shutil.rmtree(aof_dir) - - # 1. Start service in background ================================ - # If port is specified, use the specified port; otherwise, automatically assign port - if args.port: - available_port = args.port - else: - available_port = find_available_port(start_port=5100, end_port=5500) - print(f"Starting server on port {available_port}...") - proc = subprocess.Popen( - ["pnpm", "dev", "--port", str(available_port)], - cwd=workflow_dir, - stderr=subprocess.DEVNULL # Suppress stderr to hide BullMQ errors - ) - - # Wait for service to start - max_retries = 300 - for i in range(max_retries): - try: - # Disable proxy, connect directly to localhost - # response = requests.get(f"http://localhost:{available_port}", timeout=1) - response = requests.get( - f"http://localhost:{available_port}", - timeout=1, - proxies={"http": None, "https": None, "all": None}, - ) - if response.status_code == 200: - print(f"Server is ready on port {available_port}") - break - except requests.exceptions.RequestException: - pass - time.sleep(3) - else: - print("Server failed to start within 30 seconds") - subprocess.run( - f"kill -TERM $(lsof -t -i :{available_port})", - shell=True, - check=False, - text=True, - ) - proc.terminate() - proc.wait() - sys.exit(1) - - # 2. Execute API call ================================ - api_path = f"/{cmd_info['command']}/{cmd_info['operation']}" - json_data = json.dumps(cmd_info["args"]) - print(f"Executing {cmd_info['command']} {cmd_info['operation']}...") - # Disable proxy, connect directly to localhost - result = subprocess.run( - [ - "curl", - "-sS", - "--noproxy", - "localhost", - "-X", - "POST", - f"http://localhost:{available_port}{api_path}", - "-H", - "Content-Type: application/json", - "-d", - json_data, - ], - cwd=workflow_dir, - capture_output=True, - text=True, - ) - - # 3. Shutdown service ================================ - # Give observability plugin time to finish async operations (e.g., Redis writes) - time.sleep(1) - proc.terminate() - proc.wait() - print( - f"\nTask completed. Command running on http://localhost:{available_port} is finished" - ) - - # Check the success field returned by API - try: - response = json.loads(result.stdout) - if not response.get("success", False): - print("Error: Task failed") - sys.exit(1) - except Exception: - print("Error: Invalid API response") - sys.exit(1) - - else: - print(f"Unknown command: {cmd_info['command']}") - print( - "Available commands: start, stop, verilator, vcs, sardine, agent, \ - workload, doc, marshal, firesim, compiler, funcsim, uvm, yosys, pegasus" - ) - sys.exit(1) + run_server_mode(args, cmd_info) + else: + run_script_mode(args, cmd_info) diff --git a/bbdev/flake.nix b/bbdev/flake.nix index fe81a33b1..039d33b78 100644 --- a/bbdev/flake.nix +++ b/bbdev/flake.nix @@ -15,11 +15,7 @@ { devShells.default = pkgs.mkShell { buildInputs = with pkgs; [ - # Node.js runtime (Motia is a Node.js framework) - nodejs_22 - nodePackages.pnpm - - # Python with project dependencies (managed by Nix, not requirements.txt) + # Python with system-level dependencies (managed by Nix) bbdevPythonPkgs # System tools @@ -29,12 +25,18 @@ shellHook = '' export PATH="$PWD:$PATH" - source "$PWD/init.sh" + + # Create venv for pip-managed packages (motia, iii-sdk) + VENV_DIR="$PWD/api/.venv" + if [ ! -d "$VENV_DIR" ]; then + echo "Creating Python venv..." + python3 -m venv "$VENV_DIR" --system-site-packages + "$VENV_DIR/bin/pip" install -q motia[otel]==1.0.0rc17 iii-sdk==0.2.0 + fi + export PATH="$VENV_DIR/bin:$PATH" echo "bbdev dev environment ready" - echo " Node.js: $(node --version)" echo " Python: $(python3 --version)" - echo " pnpm: $(pnpm --version)" echo " bbdev: $(which bbdev)" ''; }; From 4c41382a007708c05304fc53b49e55563952310d Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 2 Apr 2026 23:57:43 +0800 Subject: [PATCH 025/196] fix: remove used wait_ function --- bbdev/api/pyproject.toml | 1 + .../api/steps/pegasus/01_verilog_api.step.py | 6 +--- .../pegasus/02_buildbitstream_api.step.py | 5 +-- bbdev/api/steps/yosys/01_synth_api.step.py | 8 +---- bbdev/bbdev | 34 ++++++++++++++++--- 5 files changed, 33 insertions(+), 21 deletions(-) diff --git a/bbdev/api/pyproject.toml b/bbdev/api/pyproject.toml index ea2c242ae..761bcaac3 100644 --- a/bbdev/api/pyproject.toml +++ b/bbdev/api/pyproject.toml @@ -7,6 +7,7 @@ dependencies = [ "motia[otel]==1.0.0rc17", "iii-sdk==0.2.0", "pydantic>=2.0", + "PyYAML>=6.0", ] [project.optional-dependencies] diff --git a/bbdev/api/steps/pegasus/01_verilog_api.step.py b/bbdev/api/steps/pegasus/01_verilog_api.step.py index 5e756492e..6094e49fa 100644 --- a/bbdev/api/steps/pegasus/01_verilog_api.step.py +++ b/bbdev/api/steps/pegasus/01_verilog_api.step.py @@ -1,7 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api from utils.path import get_buckyball_path -from utils.event_common import wait_for_result config = { "name": "Pegasus Verilog", @@ -16,14 +15,11 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = req.body or {} - # Default config for Pegasus; allow override config_name = body.get("config", "sims.pegasus.PegasusConfig") data = { "config": config_name, "output_dir": body.get("output_dir", f"{bbdir}/arch/build/pegasus/"), } - await ctx.enqueue({"topic": "pegasus.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) - result = await wait_for_result(ctx) - return result + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/pegasus/02_buildbitstream_api.step.py b/bbdev/api/steps/pegasus/02_buildbitstream_api.step.py index bad553a69..483a5afa8 100644 --- a/bbdev/api/steps/pegasus/02_buildbitstream_api.step.py +++ b/bbdev/api/steps/pegasus/02_buildbitstream_api.step.py @@ -1,7 +1,5 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import wait_for_result - config = { "name": "Pegasus Buildbitstream", "description": "build pegasus bitstream", @@ -14,5 +12,4 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} await ctx.enqueue({"topic": "pegasus.buildbitstream", "data": {**body, "_trace_id": ctx.trace_id}}) - result = await wait_for_result(ctx) - return result + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/yosys/01_synth_api.step.py b/bbdev/api/steps/yosys/01_synth_api.step.py index b50fd1926..81a12990a 100644 --- a/bbdev/api/steps/yosys/01_synth_api.step.py +++ b/bbdev/api/steps/yosys/01_synth_api.step.py @@ -1,7 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api from utils.path import get_buckyball_path -from utils.event_common import wait_for_result config = { "name": "Yosys Synth", @@ -22,9 +21,4 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: "config": body.get("config"), } await ctx.enqueue({"topic": "yosys.synth", "data": {**data, "_trace_id": ctx.trace_id}}) - - # ================================================================================== - # Wait for synthesis result - # ================================================================================== - result = await wait_for_result(ctx) - return result + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/bbdev b/bbdev/bbdev index a203148bc..8c11f6b96 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -255,16 +255,40 @@ def wait_for_http(url: str, check_fn, timeout: int, error_msg: str): return False +_ENGINE_INFRA_MODULES = ( + "iii::modules::worker", + "iii::modules::config", + "iii::modules::rest_api", + "iii::modules::api", + "iii::modules::queue", + "iii::modules::state", + "iii::modules::pubsub", + "iii::modules::shell", + "iii::modules::telemetry", + "iii::builtins", + "iii::trigger", + "iii::function", + "function_name", + "service_name", + "iii::services", + "iii::workers", + "motia.runtime", + "motia.tracing", + "motia.cli", +) + + def forward_iii_output(pipe, active_event: threading.Event, quiet: bool = False): - """Forward task output to stdout. In quiet mode, suppress engine INFO/WARN logs.""" + """Forward task output to stdout. In quiet mode, suppress engine infrastructure logs.""" for line in iter(pipe.readline, ""): if not active_event.is_set(): continue if quiet: - # Pass through: ANSI-colored step output and ERROR lines; drop engine INFO/WARN - if line.startswith("\033[") or "[ERROR]" in line: - sys.stdout.write(line) - sys.stdout.flush() + # Suppress engine infrastructure INFO/WARN; pass everything else + if any(mod in line for mod in _ENGINE_INFRA_MODULES): + continue + sys.stdout.write(line) + sys.stdout.flush() else: sys.stdout.write(line) sys.stdout.flush() From 606a8b67f2aa4a6ea9f65eb93560b08469145b75 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 3 Apr 2026 00:41:12 +0800 Subject: [PATCH 026/196] fix: add missing features --- .../steps/verilator/02_verilog_event.step.py | 59 +++++++--- .../steps/verilator/03_build_event.step.py | 102 +++++++++++++----- .../api/steps/verilator/04_sim_event.step.py | 84 ++++++++------- 3 files changed, 166 insertions(+), 79 deletions(-) diff --git a/bbdev/api/steps/verilator/02_verilog_event.step.py b/bbdev/api/steps/verilator/02_verilog_event.step.py index a664ed64e..a6439468a 100644 --- a/bbdev/api/steps/verilator/02_verilog_event.step.py +++ b/bbdev/api/steps/verilator/02_verilog_event.step.py @@ -1,4 +1,5 @@ import os +import subprocess import sys from motia import FlowContext, queue @@ -26,9 +27,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/") arch_dir = f"{bbdir}/arch" - - # Get config name, must be provided config_name = input_data.get("config") + if not config_name or config_name == "None": ctx.logger.error("Configuration name is required but not provided") success_result, failure_result = await check_result( @@ -39,7 +39,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "task": "validation", "error": "Configuration name is required. Please specify --config parameter.", "example": 'bbdev verilator --verilog "--config sims.verilator.BuckyballToyVerilatorConfig"', - }, trace_id=origin_tid) + }, + trace_id=origin_tid, + ) return failure_result ctx.logger.info(f"Using configuration: {config_name}") @@ -54,8 +56,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: else: command = f"mill -i __.test.runMain sims.verilator.Elaborate {config_name} " - command += "--disable-annotation-unknown -strip-debug-info -O=debug " - command += f"--split-verilog -o={build_dir}" + # Firtool options (CIRCT). Current set; optional Chipyard-style options below. + command += "--disable-annotation-unknown " + command += "--strip-debug-info " + command += "-O=debug " + # command += f"-repl-seq-mem -repl-seq-mem-file={build_dir}/mem.conf " + command += f"--split-verilog -o={build_dir} " + # Optional: --disable-annotation-classless (ignore classless annotations) + # Optional: -repl-seq-mem -repl-seq-mem-file=.conf (SRAM macro replacement) + # Optional: --disable-all-randomization (disable mem/reg init; may break semantics) + # Optional: --disable-opt (no optimization) or -O=release (default is release) + # Optional: --output-annotation-file= (emit annotations after lower-to-hw) + # Optional: --no-dedup (disable module dedup); --strip-fir-debug-info (FIR locators) result = stream_run_logger( cmd=command, @@ -65,10 +77,34 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="verilator verilog", ) - # Remove unwanted file - topname_file = f"{arch_dir}/TestHarness.sv" - if os.path.exists(topname_file): - os.remove(topname_file) + # Remove testchipip C++ sources that depend on fesvr (which we don't have). + # SimTSI.v is kept so verilator can resolve the SimTSI module reference in BBSimHarness.sv; + # tsi_tick DPI symbol is satisfied by arch/src/csrc/src/monitor/ioe/tsi_stub.cc instead. + for unwanted in [ + f"{build_dir}/testchip_htif.cc", + f"{build_dir}/testchip_htif.h", + f"{build_dir}/testchip_tsi.cc", + f"{build_dir}/testchip_tsi.h", + f"{build_dir}/SimTSI.cc", + f"{arch_dir}/BBSimHarness.sv", + ]: + if os.path.exists(unwanted): + os.remove(unwanted) + + # Patch fesvr includes out of mm.h and mm.cc (copied from testchipip resources). + # They reference fesvr/memif.h which we don't have — our SimDRAM_bb.cc doesn't use it. + for patch_file in [f"{build_dir}/mm.h", f"{build_dir}/mm.cc"]: + if os.path.exists(patch_file): + with open(patch_file, "r") as f: + content = f.read() + patched = "\n".join( + line for line in content.splitlines() + if "fesvr/memif.h" not in line and "fesvr/elfloader.h" not in line + ) + if patched != content: + with open(patch_file, "w") as f: + f.write(patched) + ctx.logger.info(f"Patched fesvr includes from {patch_file}") # ================================================================================== # Return result to API @@ -77,13 +113,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, result.returncode, continue_run=input_data.get("from_run_workflow", False), - extra_fields={"task": "verilog"}, trace_id=origin_tid, + extra_fields={"task": "verilog"}, + trace_id=origin_tid, ) # ================================================================================== # Continue routing - # Routing to verilog or finish workflow - # For run workflow, continue to verilog; for standalone clean, complete # ================================================================================== if input_data.get("from_run_workflow"): await ctx.enqueue( diff --git a/bbdev/api/steps/verilator/03_build_event.step.py b/bbdev/api/steps/verilator/03_build_event.step.py index cd3b40723..ddddcd66a 100644 --- a/bbdev/api/steps/verilator/03_build_event.step.py +++ b/bbdev/api/steps/verilator/03_build_event.step.py @@ -28,14 +28,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" build_dir = f"{arch_dir}/build" - waveform_dir = f"{arch_dir}/waveform" - log_dir = f"{arch_dir}/log" + coverage = input_data.get("coverage", False) cosim = input_data.get("cosim", False) - # ================================================================================== - # Execute operation # ================================================================================== # Find sources + # ================================================================================== vsrcs = glob.glob(f"{build_dir}/**/*.v", recursive=True) + glob.glob( f"{build_dir}/**/*.sv", recursive=True ) @@ -48,32 +46,50 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: + glob.glob(f"{build_dir}/**/*.cpp", recursive=True) ) - # Setup paths: fesvr from bebop/host/spike/riscv-isa-sim (install/include, install/lib) - bebop_isa_sim = f"{bbdir}/bebop/host/spike/riscv-isa-sim" - inc_paths = [ - os.environ.get("RISCV", "") + "/include" if os.environ.get("RISCV") else "", - f"{arch_dir}/thirdparty/chipyard/tools/DRAMSim2", - f"{bebop_isa_sim}/install/include", - build_dir, - f"{arch_dir}/src/csrc/include", - ] - inc_flags = " ".join([f"-I{p}" for p in inc_paths if p]) + # Exclude testchipip's SimDRAM.cc — our SimDRAM_bb.cc overrides memory_init + csrcs = [f for f in csrcs if not f.endswith("SimDRAM.cc") or "src/csrc" in f] + + # Exclude testchipip's TSI/HTIF C++ sources (deleted in verilog step, but guard anyway). + # tsi_tick DPI symbol is provided by tsi_stub.cc in src/csrc instead. + _tsi_htif = {"testchip_tsi.cc", "testchip_htif.cc", "SimTSI.cc"} + csrcs = [f for f in csrcs if os.path.basename(f) not in _tsi_htif] if cosim: topname = "ToyBuckyball" else: - topname = "TestHarness" + topname = "BBSimHarness" - cflags = f"{inc_flags} -DTOP_NAME='\"V{topname}\"' -std=c++17 " + # ================================================================================== + # Build flags + # ================================================================================== + result_dir = f"{bbdir}/result" + + def pkg_config(flag, pkg): + r = subprocess.run(["pkg-config", flag, pkg], capture_output=True, text=True) + return r.stdout.strip() if r.returncode == 0 else "" + + readline_inc = pkg_config("--variable=includedir", "readline") + readline_lib = pkg_config("--variable=libdir", "readline") + zlib_lib = pkg_config("--variable=libdir", "zlib") + + inc_flags = " ".join([ + f"-I{result_dir}/include", + f"-I{build_dir}", + f"-I{arch_dir}/src/csrc/include", + f"-I{readline_inc}", + ]) + + # -DBBSIM: selects VBBSimHarness in bdb.h / main.cc + # BDB NDJSON trace (+trace=...) is runtime-only; bbdev sim uses +trace=all (04_sim_event.step.py). + cflags = f"{inc_flags} -DBBSIM -DTOP_NAME='\"V{topname}\"' -std=c++17" if cosim: cflags += " -DCOSIM" + ldflags = ( - f"-lreadline -ldramsim -lfesvr -lstdc++ " - f"-L{bebop_isa_sim}/install/lib " - f"-L{bbdir}/result/lib " - f"-L{arch_dir}/thirdparty/chipyard/tools/DRAMSim2 " - f"-L{arch_dir}/thirdparty/chipyard/toolchains/riscv-tools/riscv-isa-sim/build " - f"-L{arch_dir}/thirdparty/chipyard/toolchains/riscv-tools/riscv-isa-sim/build/lib" + f"-lreadline -ldramsim -lstdc++ -lz " + f"-L{result_dir}/lib " + f"-L{readline_lib} -Wl,-rpath,{readline_lib} " + f"-L{zlib_lib} -Wl,-rpath,{zlib_lib} " ) obj_dir = f"{build_dir}/obj_dir" @@ -83,10 +99,31 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: sources = " ".join(vsrcs + csrcs) jobs = input_data.get("jobs", "") + # Fix nix runtime library paths + libstdcpp_path = subprocess.run( + "g++ -print-file-name=libstdc++.so", shell=True, capture_output=True, text=True + ).stdout.strip() + if libstdcpp_path and "/" in libstdcpp_path: + nix_lib_dir = os.path.dirname(os.path.realpath(libstdcpp_path)) + ldflags += f" -Wl,-rpath,{nix_lib_dir}" + + # Enable ccache if available + if subprocess.run("command -v ccache", shell=True, capture_output=True).returncode == 0: + os.environ["OBJCACHE"] = "ccache" + + # Use lld for faster linking if available + use_lld = subprocess.run("command -v ld.lld", shell=True, capture_output=True).returncode == 0 + if use_lld: + ldflags += " -fuse-ld=lld" + + # ================================================================================== + # Run verilator + # ================================================================================== verilator_cmd = ( - f"verilator -MMD --build -cc --trace -O3 --x-assign fast --x-initial fast --noassert -Wno-fatal " + f"verilator -MMD -cc --vpi --trace -O3 --x-assign fast --x-initial fast --noassert -Wno-fatal " f"--trace-fst --trace-threads 1 --output-split 10000 --output-split-cfuncs 100 " f"--unroll-count 256 " + f"{'--coverage-line ' if coverage else ''}" f"-Wno-PINCONNECTEMPTY " f"-Wno-ASSIGNDLY " f"-Wno-DECLFILENAME " @@ -103,11 +140,19 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cmd=verilator_cmd, logger=ctx.logger, cwd=bbdir, - stdout_prefix="verilator build", - stderr_prefix="verilator build", + stdout_prefix="verilator verilation", + stderr_prefix="verilator verilation", ) + if result.returncode != 0: + await check_result( + ctx, result.returncode, continue_run=False, extra_fields={"task": "build"}, + trace_id=origin_tid, + ) + return + + make_jobs = jobs if jobs else str(os.cpu_count() or 16) result = stream_run_logger( - cmd=f"make -C {obj_dir} -f V{topname}.mk V{topname}", + cmd=f"make -j{make_jobs} VM_PARALLEL_BUILDS=1 -C {obj_dir} -f V{topname}.mk V{topname}", logger=ctx.logger, cwd=bbdir, stdout_prefix="verilator build", @@ -117,11 +162,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ================================================================================== # Return result to API # ================================================================================== - success_result, failure_result = await check_result( + await check_result( ctx, result.returncode, continue_run=input_data.get("from_run_workflow", False), - extra_fields={"task": "build"}, trace_id=origin_tid, + extra_fields={"task": "build"}, + trace_id=origin_tid, ) # ================================================================================== diff --git a/bbdev/api/steps/verilator/04_sim_event.step.py b/bbdev/api/steps/verilator/04_sim_event.step.py index a6cf81f44..249dacfac 100644 --- a/bbdev/api/steps/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -25,63 +25,73 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) # ================================================================================== # Get simulation parameters # ================================================================================== + origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" build_dir = f"{arch_dir}/build" - # Generate timestamp timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") binary_name = input_data.get("binary", "") + coverage = input_data.get("coverage", False) success_result, failure_result = await check_result( - ctx, returncode=(binary_name == None), continue_run=True, trace_id=origin_tid, + ctx, returncode=(binary_name is None), continue_run=True, trace_id=origin_tid, ) binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) ctx.logger.info(f"binary_path: {binary_path}") success_result, failure_result = await check_result( - ctx, returncode=(binary_path == None), continue_run=True, trace_id=origin_tid, + ctx, returncode=(binary_path is None), continue_run=True, trace_id=origin_tid, ) if failure_result: ctx.logger.error("binary not found", failure_result) return - # Create log and waveform directory + topname = "BBSimHarness" log_dir = f"{arch_dir}/log/{timestamp}-{binary_name}" waveform_dir = f"{arch_dir}/waveform/{timestamp}-{binary_name}" - topname = "TestHarness" os.makedirs(log_dir, exist_ok=True) os.makedirs(waveform_dir, exist_ok=True) + coverage_flag = "" + if coverage: + coverage_dat_path = f"{log_dir}/coverage.dat" + coverage_flag = f"+verilator+coverage+file+{coverage_dat_path}" + bin_path = f"{build_dir}/obj_dir/V{topname}" batch = input_data.get("batch", False) - # Create log and waveform file - log_path = f"{log_dir}/bdb.log" - fst_path = f"{waveform_dir}/waveform.fst" - # Remove old waveform file - subprocess.run(f"rm -f {waveform_dir}/waveform.vcd", shell=True, check=True) + log_path = f"{log_dir}/bdb.ndjson" + stdout_path = f"{log_dir}/stdout.log" + meta_path = f"{log_dir}/sim_meta.txt" + fst_path = f"{waveform_dir}/waveform.fst" # ================================================================================== - # Execute simulation script with streaming output + # Execute simulation + # BBSimHarness uses +elf= for ELF loading (via SimDRAM_bb.cc / libelf) + # No fesvr, no +loadmem_addr needed + # + # disasm.log: only stderr -> spike-dasm (Rocket commit printf is stderr here; + # merging stdout with 2>&1 can break: full stdio buffering + non-DASM bytes). + # BDB_SIM_META moves NDJSON banner to sim_meta.txt so it does not pollute disasm. # ================================================================================== - bebop_isa_sim = f"{bbdir}/bebop/host/spike/riscv-isa-sim" ld_lib_path = ( - f"{bebop_isa_sim}/install/lib:" f"{bbdir}/result/lib:" f"{arch_dir}/thirdparty/chipyard/tools/DRAMSim2" ) sim_cmd = ( f"export LD_LIBRARY_PATH=\"{ld_lib_path}:$LD_LIBRARY_PATH\"; " - f"{bin_path} +permissive +loadmem={binary_path} +loadmem_addr=800000000 " - f"{'+batch ' if batch else ''} " - f"+fst={fst_path} +log={log_path} +permissive-off " - f"{binary_path} > >(tee {log_dir}/stdout.log) 2> >(spike-dasm > {log_dir}/disasm.log)" + f"export BDB_SIM_META=\"{meta_path}\"; " + f"{bin_path} +permissive " + f"+elf={binary_path} " + f"{'+batch ' if batch else ''}" + f"{coverage_flag + ' ' if coverage_flag else ''}" + f"+fst={fst_path} +log={log_path} +stdout={stdout_path} +trace=all +permissive-off " + f"{binary_path} 2> >(spike-dasm > {log_dir}/disasm.log)" ) script_dir = os.path.dirname(__file__) @@ -94,36 +104,32 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: executable="bash", ) success_result, failure_result = await check_result( - ctx, returncode=result.returncode, continue_run=True, trace_id=origin_tid) + ctx, returncode=result.returncode, continue_run=True, trace_id=origin_tid, + ) if failure_result: ctx.logger.error("sim failed", failure_result) return - if os.path.exists(f"{waveform_dir}/waveform.fst.heir"): - subprocess.run( - f"gtkwave -f {waveform_dir}/waveform.fst -H {waveform_dir}/waveform.fst.heir", - shell=True, - check=True, - ) - # ================================================================================== # Return simulation result # ================================================================================== - # This is the end point of the run workflow, status will no longer be set to processing - success_result, failure_result = await check_result( + extra_fields = { + "task": "sim", + "binary": binary_path, + "log_dir": log_dir, + "waveform_dir": waveform_dir, + "timestamp": timestamp, + "sim_meta": meta_path, + } + if coverage: + extra_fields["coverage_dat"] = coverage_dat_path + + await check_result( ctx, result.returncode, continue_run=False, - extra_fields={ - "task": "sim", - "binary": binary_path, - "log_dir": log_dir, - "waveform_dir": waveform_dir, - "timestamp": timestamp, - }, trace_id=origin_tid) - - # ================================================================================== - # Finish workflow - # ================================================================================== + extra_fields=extra_fields, + trace_id=origin_tid, + ) return From 0c063691d9c3d740bb2f044790ff7427f6b8fb0b Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 13 Apr 2026 19:23:24 +0800 Subject: [PATCH 027/196] feat: add pegasus --- .../steps/pegasus/01_verilog_event.step.py | 90 +++++++++++++------ .../pegasus/02_buildbitstream_event.step.py | 6 +- bbdev/bbdev | 4 + 3 files changed, 72 insertions(+), 28 deletions(-) diff --git a/bbdev/api/steps/pegasus/01_verilog_event.step.py b/bbdev/api/steps/pegasus/01_verilog_event.step.py index 82298fe72..7167963df 100644 --- a/bbdev/api/steps/pegasus/01_verilog_event.step.py +++ b/bbdev/api/steps/pegasus/01_verilog_event.step.py @@ -13,7 +13,7 @@ config = { "name": "make pegasus verilog", - "description": "Generate SystemVerilog from Chisel using ElaboratePegasus", + "description": "Generate SystemVerilog from Chisel using ElaboratePegasusTop", "flows": ["pegasus"], "triggers": [queue("pegasus.verilog")], "enqueues": [], @@ -26,19 +26,43 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: arch_dir = f"{bbdir}/arch" build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/pegasus/") - config_name = input_data.get("config", "sims.pegasus.PegasusConfig") - ctx.logger.info(f"[pegasus] Elaborating config: {config_name}") - ctx.logger.info(f"[pegasus] Output directory: {build_dir}") + build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/pegasus_top/") + soc_dir = f"{bbdir}/arch/build/pegasus/" # DigitalTop + all SoC RTL + + ctx.logger.info(f"[pegasus] Elaborating PegasusTop") + ctx.logger.info(f"[pegasus] Top output directory: {build_dir}") + ctx.logger.info(f"[pegasus] SoC RTL directory: {soc_dir}") os.makedirs(build_dir, exist_ok=True) - # Use ElaboratePegasus (not the generic sims.verilator.Elaborate) - # ElaboratePegasus instantiates PegasusHarness directly, bypassing TestHarness - command = ( + # Step 1: Elaborate the full SoC (DigitalTop + all sub-modules) + # This is the same as before, using ElaboratePegasus which runs PegasusHarness + # but we only need its generated RTL files (DigitalTop.sv, BBTile.sv, etc.) + soc_command = ( f"mill -i __.test.runMain sims.pegasus.ElaboratePegasus " f"--disable-annotation-unknown " f"-strip-debug-info " - f"-O=debug " + f"--split-verilog " + f"-o={soc_dir}" + ) + soc_result = stream_run_logger( + cmd=soc_command, + logger=ctx.logger, + cwd=arch_dir, + stdout_prefix="pegasus soc verilog", + stderr_prefix="pegasus soc verilog", + ) + if soc_result.returncode != 0: + ctx.logger.error("[pegasus] SoC elaboration failed") + await check_result(ctx, soc_result.returncode, continue_run=False, + extra_fields={"task": "verilog", "step": "soc"}, trace_id=origin_tid) + return + + # Step 2: Elaborate PegasusTop + PegasusShell (the FPGA top-level wrapper) + command = ( + f"mill -i __.test.runMain sims.pegasus.ElaboratePegasusTop " + f"--disable-annotation-unknown " + f"-strip-debug-info " f"--split-verilog " f"-o={build_dir}" ) @@ -51,15 +75,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="pegasus verilog", ) - # Clean up stray top-level file if emitted next to arch/ - for stray in ["PegasusHarness.sv", "TestHarness.sv"]: + # Clean up stray top-level files if emitted next to arch/ + for stray in ["PegasusTop.v", "PegasusTopWrapper.sv", "PegasusShell.v"]: stray_path = f"{arch_dir}/{stray}" if os.path.exists(stray_path): os.remove(stray_path) - # Copy generated SV files to pegasus/vivado/generated/ for Vivado build - # DPI files are replaced with synth-safe stubs. - vivado_gen_dir = f"{bbdir}/pegasus/vivado/generated" + # Copy generated SV/V files to pegasus/vivado/generated/ for Vivado build. + # Sources: + # soc_dir -> DigitalTop + all SoC sub-modules (DPI files stubbed out) + # build_dir -> PegasusTop.v, PegasusShell.v, PegasusTopWrapper.sv (top-level wrappers) + # PegasusHarness.sv, ChipTop.sv and harness-layer files from soc_dir are skipped. + vivado_gen_dir = f"{bbdir}/thirdparty/pegasus/vivado/generated" if result.returncode == 0 and os.path.isdir(build_dir): import re import shutil @@ -68,6 +95,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if f.endswith(".sv") or f.endswith(".v"): os.remove(os.path.join(vivado_gen_dir, f)) + HARNESS_SKIP = {"PegasusHarness.sv", "ChipTop.sv"} + def build_dpi_stub(src_path: str) -> str: text = open(src_path, "r", encoding="utf-8").read() m = re.search(r"module\s+([A-Za-z_][A-Za-z0-9_]*)\s*\((.*?)\);\s*", text, re.S) @@ -94,17 +123,28 @@ def build_dpi_stub(src_path: str) -> str: stub.append("") return "\n".join(stub) - sv_files = [f for f in os.listdir(build_dir) if f.endswith(".sv") or f.endswith(".v")] - for f in sv_files: - src = os.path.join(build_dir, f) - if "DPI" in f: - stub_name = f"stub_{os.path.splitext(f)[0].replace('DPI', '')}.sv" - stub_path = os.path.join(vivado_gen_dir, stub_name) - with open(stub_path, "w", encoding="utf-8") as wf: - wf.write(build_dpi_stub(src)) - else: - shutil.copy2(src, os.path.join(vivado_gen_dir, f)) - ctx.logger.info(f"[pegasus] Copied {len(sv_files)} files to {vivado_gen_dir}") + def copy_rtl_dir(src_dir: str, skip_set: set = None) -> int: + copied = 0 + for f in os.listdir(src_dir): + if not (f.endswith(".sv") or f.endswith(".v")): + continue + if skip_set and f in skip_set: + continue + src = os.path.join(src_dir, f) + if "DPI" in f: + stub_name = f"stub_{os.path.splitext(f)[0].replace('DPI', '')}.sv" + with open(os.path.join(vivado_gen_dir, stub_name), "w") as wf: + wf.write(build_dpi_stub(src)) + else: + shutil.copy2(src, os.path.join(vivado_gen_dir, f)) + copied += 1 + return copied + + # Copy SoC RTL (skip harness-layer files) + n_soc = copy_rtl_dir(soc_dir, skip_set=HARNESS_SKIP) + # Copy top-level wrappers (overrides any same-named file from soc_dir) + n_top = copy_rtl_dir(build_dir) + ctx.logger.info(f"[pegasus] Copied {n_soc} SoC files + {n_top} top files to {vivado_gen_dir}") success_result, failure_result = await check_result( ctx, @@ -114,7 +154,7 @@ def build_dpi_stub(src_path: str) -> str: "task": "verilog", "output_dir": build_dir, "vivado_gen_dir": vivado_gen_dir, - "top_module": "PegasusHarness", + "top_module": "PegasusTop", }, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py b/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py index 20025105e..f0f986528 100644 --- a/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py @@ -23,8 +23,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - generated_dir = input_data.get("generated_dir", f"{bbdir}/pegasus/vivado/generated") - output_dir = input_data.get("output_dir", f"{bbdir}/pegasus/vivado/build") + generated_dir = input_data.get("generated_dir", f"{bbdir}/thirdparty/pegasus/vivado/generated") + output_dir = input_data.get("output_dir", f"{bbdir}/thirdparty/pegasus/vivado/build") top_module = input_data.get("top", "PegasusHarness") ctx.logger.info(f"[pegasus] Generated dir: {generated_dir}") @@ -56,7 +56,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return failure_result bit_cmd = ( - f"bash {bbdir}/pegasus/vivado/build-bitstream.sh " + f"bash {bbdir}/thirdparty/pegasus/vivado/build-bitstream.sh " f"--source_dir {generated_dir} " f"--output_dir {output_dir} " f"--top {top_module}" diff --git a/bbdev/bbdev b/bbdev/bbdev index 8c11f6b96..b6305061a 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -72,6 +72,10 @@ WORKFLOW_COMMANDS = { "palladium": { "verilog": 'Generate verilog files. Args: "[--config sims.palladium.BuckyballToyP2EConfig]"', }, + "pegasus": { + "verilog": 'Generate SystemVerilog for Pegasus FPGA. Args: "[--config sims.pegasus.PegasusConfig]"', + "buildbitstream": "Build Vivado bitstream for AU280.", + }, } ALL_COMMANDS = ["start", "stop"] + list(WORKFLOW_COMMANDS) From 35986108ae7f2b7aee4ca70687fd6a10bc4a8b47 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 14 Apr 2026 02:31:08 +0800 Subject: [PATCH 028/196] fix: correct buildbitstream event name, which leads to step can't be redirected --- bbdev/api/steps/pegasus/02_buildbitstream_event.step.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py b/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py index f0f986528..77e10ffb4 100644 --- a/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py @@ -12,8 +12,8 @@ from utils.event_common import check_result, get_origin_trace_id config = { - "name": "Pegasus Buildbitstream", - "description": "build pegasus bitstream", + "name": "generate pegasus bitstream", + "description": "generate pegasus bitstream", "flows": ["pegasus"], "triggers": [queue("pegasus.buildbitstream")], "enqueues": [], From 96e09aa75a6b003f255f30b9142055a9680cf3ec Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 14 Apr 2026 03:27:04 +0800 Subject: [PATCH 029/196] fix: update default top module name in buildbitstream event --- bbdev/api/steps/pegasus/02_buildbitstream_event.step.py | 2 +- bbdev/bbdev | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py b/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py index 77e10ffb4..599cd1e98 100644 --- a/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py @@ -25,7 +25,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() generated_dir = input_data.get("generated_dir", f"{bbdir}/thirdparty/pegasus/vivado/generated") output_dir = input_data.get("output_dir", f"{bbdir}/thirdparty/pegasus/vivado/build") - top_module = input_data.get("top", "PegasusHarness") + top_module = input_data.get("top", "PegasusTop") ctx.logger.info(f"[pegasus] Generated dir: {generated_dir}") ctx.logger.info(f"[pegasus] Output dir: {output_dir}") diff --git a/bbdev/bbdev b/bbdev/bbdev index b6305061a..132999db4 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -380,7 +380,7 @@ def _submit_and_poll(port: int, api_path: str, cmd_info: dict): # Poll for result state_file = os.path.join(workflow_dir, "data", "state_store.db", f"{trace_id}.bin") - poll_timeout = 600 # 10 minute max + poll_timeout = 24 * 60 * 60 # 24 hours max task_result = None for _ in range(poll_timeout // 2): state_data = read_state_file(state_file) @@ -393,7 +393,7 @@ def _submit_and_poll(port: int, api_path: str, cmd_info: dict): break time.sleep(2) else: - print("Error: Task polling timed out after 10 minutes") + print("Error: Task polling timed out after 24 hours") return 1 print(f"\nTask completed on http://localhost:{port}") From 6a31cacb6713e071aada54e31b1313a7fafe0d3c Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 15 Apr 2026 01:53:45 +0800 Subject: [PATCH 030/196] fix: update kernel step to align new buckyball directory --- bbdev/api/steps/kernel/01_build_api.step.py | 15 ++ bbdev/api/steps/kernel/01_build_event.step.py | 67 +++++++ bbdev/api/steps/marshal/.gitignore | 1 - bbdev/api/steps/marshal/01_build_api.step.py | 15 -- .../api/steps/marshal/01_build_event.step.py | 70 -------- bbdev/api/steps/marshal/02_launch_api.step.py | 15 -- .../api/steps/marshal/02_launch_event.step.py | 50 ------ bbdev/api/steps/marshal/README.md | 53 ------ .../steps/marshal/scripts/interactive.json | 9 - .../marshal/scripts/interactive/host-init.sh | 20 --- bbdev/api/steps/marshal/scripts/marshal | 1 - .../steps/marshal/scripts/marshal-config.yaml | 2 - .../pegasus/03_flashbitstream_api.step.py | 15 ++ .../pegasus/03_flashbitstream_event.step.py | 165 +++++++++++++++++ .../steps/pegasus/04_runworkload_api.step.py | 15 ++ .../pegasus/04_runworkload_event.step.py | 167 ++++++++++++++++++ bbdev/bbdev | 7 +- bbdev/nix/overlay.nix | 3 + 18 files changed, 451 insertions(+), 239 deletions(-) create mode 100644 bbdev/api/steps/kernel/01_build_api.step.py create mode 100644 bbdev/api/steps/kernel/01_build_event.step.py delete mode 100644 bbdev/api/steps/marshal/.gitignore delete mode 100644 bbdev/api/steps/marshal/01_build_api.step.py delete mode 100644 bbdev/api/steps/marshal/01_build_event.step.py delete mode 100644 bbdev/api/steps/marshal/02_launch_api.step.py delete mode 100644 bbdev/api/steps/marshal/02_launch_event.step.py delete mode 100644 bbdev/api/steps/marshal/README.md delete mode 100644 bbdev/api/steps/marshal/scripts/interactive.json delete mode 100755 bbdev/api/steps/marshal/scripts/interactive/host-init.sh delete mode 120000 bbdev/api/steps/marshal/scripts/marshal delete mode 100644 bbdev/api/steps/marshal/scripts/marshal-config.yaml create mode 100644 bbdev/api/steps/pegasus/03_flashbitstream_api.step.py create mode 100644 bbdev/api/steps/pegasus/03_flashbitstream_event.step.py create mode 100644 bbdev/api/steps/pegasus/04_runworkload_api.step.py create mode 100644 bbdev/api/steps/pegasus/04_runworkload_event.step.py diff --git a/bbdev/api/steps/kernel/01_build_api.step.py b/bbdev/api/steps/kernel/01_build_api.step.py new file mode 100644 index 000000000..c5fd5c894 --- /dev/null +++ b/bbdev/api/steps/kernel/01_build_api.step.py @@ -0,0 +1,15 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "kernel-build-api", + "description": "build RISC-V kernel + rootfs for Pegasus", + "flows": ["kernel"], + "triggers": [api("POST", "/kernel/build")], + "enqueues": ["kernel.build"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + await ctx.enqueue({"topic": "kernel.build", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py new file mode 100644 index 000000000..ceb78cdfe --- /dev/null +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -0,0 +1,67 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "kernel-build", + "description": "build RISC-V kernel + rootfs for Pegasus via bb-tests/workloads/lib/kernel", + "flows": ["kernel"], + "triggers": [queue("kernel.build")], + "enqueues": ["kernel.complete", "kernel.error"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + + kernel_src = os.path.join(bbdir, "bb-tests", "workloads", "lib", "kernel") + kernel_build = os.path.join(bbdir, "bb-tests", "build", "kernel") + + # cmake configure + configure_cmd = f"cmake -B {kernel_build} -S {kernel_src}" + result = stream_run_logger( + cmd=configure_cmd, + logger=ctx.logger, + stdout_prefix="marshal build", + stderr_prefix="marshal build", + ) + if result.returncode != 0: + _, failure_result = await check_result(ctx, result.returncode, continue_run=False, trace_id=origin_tid) + await ctx.enqueue({"topic": "marshal.error", "data": {**input_data, "result": failure_result, "returncode": result.returncode}}) + return + + # cmake build + build_cmd = f"cmake --build {kernel_build} --target kernel-build" + result = stream_run_logger( + cmd=build_cmd, + logger=ctx.logger, + stdout_prefix="marshal build", + stderr_prefix="marshal build", + ) + + success_result, failure_result = await check_result( + ctx, result.returncode, continue_run=False, trace_id=origin_tid) + + if result.returncode == 0: + output_dir = os.path.join(kernel_src, "output") + await ctx.enqueue({ + "topic": "kernel.complete", + "data": {**input_data, "task": "kernel", "result": success_result, + "kernel": os.path.join(output_dir, "pegasus-bin"), + "rootfs": os.path.join(output_dir, "pegasus.img")}, + }) + else: + await ctx.enqueue({ + "topic": "kernel.error", + "data": {**input_data, "task": "kernel", "result": failure_result, "returncode": result.returncode}, + }) diff --git a/bbdev/api/steps/marshal/.gitignore b/bbdev/api/steps/marshal/.gitignore deleted file mode 100644 index 0521c5fb4..000000000 --- a/bbdev/api/steps/marshal/.gitignore +++ /dev/null @@ -1 +0,0 @@ -!*.json diff --git a/bbdev/api/steps/marshal/01_build_api.step.py b/bbdev/api/steps/marshal/01_build_api.step.py deleted file mode 100644 index ba9049a12..000000000 --- a/bbdev/api/steps/marshal/01_build_api.step.py +++ /dev/null @@ -1,15 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - -config = { - "name": "marshal-build-api", - "description": "build marshal", - "flows": ["marshal"], - "triggers": [api("POST", "/marshal/build")], - "enqueues": ["marshal.build"], -} - - -async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = request.body or {} - await ctx.enqueue({"topic": "marshal.build", "data": {**body, "_trace_id": ctx.trace_id}}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/marshal/01_build_event.step.py b/bbdev/api/steps/marshal/01_build_event.step.py deleted file mode 100644 index 4698ce5cd..000000000 --- a/bbdev/api/steps/marshal/01_build_event.step.py +++ /dev/null @@ -1,70 +0,0 @@ -import os -import sys - -from motia import FlowContext, queue - -# Add the utils directory to the Python path -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from utils.event_common import check_result, get_origin_trace_id - -config = { - "name": "marshal-build", - "description": "build marshal", - "flows": ["marshal"], - "triggers": [queue("marshal.build")], - "enqueues": ["marshal.complete", "marshal.error"], -} - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - bbdir = get_buckyball_path() - script_dir = f"{bbdir}/workflow/steps/marshal/scripts" - # ================================================================================== - # Execute operation - # ================================================================================== - command = f"./marshal -v build interactive.json && ./marshal -v install interactive.json" - result = stream_run_logger( - cmd=command, - logger=ctx.logger, - cwd=script_dir, - stdout_prefix="marshal build", - stderr_prefix="marshal build", - ) - - # ================================================================================== - # Return result to API - # ================================================================================== - success_result, failure_result = await check_result( - ctx, result.returncode, continue_run=False, trace_id=origin_tid) - - # ================================================================================== - # Continue routing - # Routing to completion or error handling - # ================================================================================== - if result.returncode == 0: - await ctx.enqueue( - { - "topic": "marshal.complete", - "data": {**input_data, "task": "marshal", "result": success_result}, - } - ) - else: - await ctx.enqueue( - { - "topic": "marshal.error", - "data": { - **input_data, - "task": "marshal", - "result": failure_result, - "returncode": result.returncode, - }, - } - ) - - return diff --git a/bbdev/api/steps/marshal/02_launch_api.step.py b/bbdev/api/steps/marshal/02_launch_api.step.py deleted file mode 100644 index 4b52daf68..000000000 --- a/bbdev/api/steps/marshal/02_launch_api.step.py +++ /dev/null @@ -1,15 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - -config = { - "name": "marshal-launch-api", - "description": "launch marshal", - "flows": ["marshal"], - "triggers": [api("POST", "/marshal/launch")], - "enqueues": ["marshal.launch"], -} - - -async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = request.body or {} - await ctx.enqueue({"topic": "marshal.launch", "data": {**body, "_trace_id": ctx.trace_id}}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/marshal/02_launch_event.step.py b/bbdev/api/steps/marshal/02_launch_event.step.py deleted file mode 100644 index edb11419e..000000000 --- a/bbdev/api/steps/marshal/02_launch_event.step.py +++ /dev/null @@ -1,50 +0,0 @@ -import os -import sys - -from motia import FlowContext, queue - -# Add the utils directory to the Python path -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from utils.event_common import check_result, get_origin_trace_id - -config = { - "name": "marshal-launch", - "description": "launch marshal", - "flows": ["marshal"], - "triggers": [queue("marshal.launch")], - "enqueues": [], -} - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - bbdir = get_buckyball_path() - script_dir = f"{bbdir}/workflow/steps/marshal/scripts" - # ================================================================================== - # Execute operation - # ================================================================================== - command = f"./marshal -v launch interactive.json" - result = stream_run_logger( - cmd=command, - logger=ctx.logger, - cwd=script_dir, - stdout_prefix="marshal launch", - stderr_prefix="marshal launch", - ) - - # ================================================================================== - # Return result to API - # ================================================================================== - success_result, failure_result = await check_result( - ctx, result.returncode, continue_run=False, trace_id=origin_tid) - - # ================================================================================== - # Continue routing - # Finish workflow - # ================================================================================== - return diff --git a/bbdev/api/steps/marshal/README.md b/bbdev/api/steps/marshal/README.md deleted file mode 100644 index f8d25a32e..000000000 --- a/bbdev/api/steps/marshal/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# Marshal Workflow - -Marshal workflow in the Buckyball framework, used to build and launch the Marshal component. - -## API Usage Guide - -### `build` -**Endpoint**: `POST /marshal/build` - -**Function**: Build Marshal component - -**Parameters**: No specific parameters - -**Example**: -```bash -bbdev marshal --build -``` - -### `launch` -**Endpoint**: `POST /marshal/launch` - -**Function**: Launch Marshal service - -**Parameters**: No specific parameters - -**Example**: -```bash -bbdev marshal --launch -``` - -## Typical Workflow - -```bash -# 1. Build Marshal -bbdev marshal --build - -# 2. Launch Marshal service -bbdev marshal --launch -``` - -## Response Format - -All API calls return a unified format: -```json -{ - "status": 200, - "body": { - "success": true, - "processing": false, - "return_code": 0 - } -} -``` diff --git a/bbdev/api/steps/marshal/scripts/interactive.json b/bbdev/api/steps/marshal/scripts/interactive.json deleted file mode 100644 index 8bb194963..000000000 --- a/bbdev/api/steps/marshal/scripts/interactive.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name" : "interactive", - "workdir" : ".", - "base" : "br-base.json", - "overlay" : "../output/overlay", - "host-init" : "./interactive/host-init.sh", - "rootfs-size" : "16GiB", - "spike-args" : "--extension=gemmini" -} diff --git a/bbdev/api/steps/marshal/scripts/interactive/host-init.sh b/bbdev/api/steps/marshal/scripts/interactive/host-init.sh deleted file mode 100755 index 3a481074f..000000000 --- a/bbdev/api/steps/marshal/scripts/interactive/host-init.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# This script will run on the host from the workload directory -# (e.g. workloads/example-fed) every time the workload is built. -# It is recommended to call into something like a makefile because -# this script may be called multiple times. - -BBDIR=$(git rev-parse --show-toplevel) - -cd $BBDIR && source sourceme.sh - -echo "Building marshal workload" -bbdev workload --build - -mkdir -p $BBDIR/workflow/steps/marshal/output -rm -rf $BBDIR/workflow/steps/marshal/output/overlay -mkdir -p $BBDIR/workflow/steps/marshal/output/overlay/root - -# Copy workload binaries to /root directory -cp -r $BBDIR/bb-tests/output/workloads/src/* $BBDIR/workflow/steps/marshal/output/overlay/root/ diff --git a/bbdev/api/steps/marshal/scripts/marshal b/bbdev/api/steps/marshal/scripts/marshal deleted file mode 120000 index 6b68a6684..000000000 --- a/bbdev/api/steps/marshal/scripts/marshal +++ /dev/null @@ -1 +0,0 @@ -../../../../arch/thirdparty/chipyard/software/firemarshal/marshal \ No newline at end of file diff --git a/bbdev/api/steps/marshal/scripts/marshal-config.yaml b/bbdev/api/steps/marshal/scripts/marshal-config.yaml deleted file mode 100644 index 97b3664ef..000000000 --- a/bbdev/api/steps/marshal/scripts/marshal-config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -firesim-dir: '../../../../arch/thirdparty/chipyard/sims/firesim' -# board-dir: '../../../../arch/thirdparty/chipyard/software/firemarshal/boards/prototype' diff --git a/bbdev/api/steps/pegasus/03_flashbitstream_api.step.py b/bbdev/api/steps/pegasus/03_flashbitstream_api.step.py new file mode 100644 index 000000000..d32d80507 --- /dev/null +++ b/bbdev/api/steps/pegasus/03_flashbitstream_api.step.py @@ -0,0 +1,15 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "Pegasus Flashbitstream", + "description": "Flash bitstream onto AU280 via hw_server", + "flows": ["pegasus"], + "triggers": [api("POST", "/pegasus/flashbitstream")], + "enqueues": ["pegasus.flashbitstream"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + await ctx.enqueue({"topic": "pegasus.flashbitstream", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/pegasus/03_flashbitstream_event.step.py b/bbdev/api/steps/pegasus/03_flashbitstream_event.step.py new file mode 100644 index 000000000..5f77a392f --- /dev/null +++ b/bbdev/api/steps/pegasus/03_flashbitstream_event.step.py @@ -0,0 +1,165 @@ +""" +pegasus flashbitstream event handler + +Sequence: + 1. Detect XDMA PCIe BDF (auto-scan /sys/bus/pci/devices for xdma driver) + 2. PCIe disconnect: clear SERR + fatal-error bits, remove device + 3. Flash via Vivado + program_fpga.tcl + 4. PCIe reconnect: rescan, re-enable memory-mapped transfers +""" +import os +import sys +import glob +import time +import subprocess + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "pegasus-flashbitstream", + "description": "Flash AU280 bitstream via Vivado hw_server + PCIe remove/rescan", + "flows": ["pegasus"], + "triggers": [queue("pegasus.flashbitstream")], + "enqueues": [], +} + +PCI_DEVICES = "/sys/bus/pci/devices" + + +def _find_xdma_bdf() -> str | None: + """Return the extended BDF (e.g. '0000:65:00.0') of the first xdma device.""" + for entry in os.listdir(PCI_DEVICES): + driver_link = os.path.join(PCI_DEVICES, entry, "driver") + if not os.path.islink(driver_link): + continue + driver_name = os.path.basename(os.readlink(driver_link)) + if driver_name == "xdma": + return entry # e.g. '0000:65:00.0' + return None + + +def _bridge_bdf(extended_bdf: str) -> str | None: + """Return the bridge BDF for a given extended device BDF.""" + dev_path = os.path.realpath(os.path.join(PCI_DEVICES, extended_bdf)) + bridge_path = os.path.dirname(dev_path) + bridge_bdf = os.path.basename(bridge_path) + if bridge_bdf.startswith("0000:"): + return bridge_bdf + return None + + +def _setpci(bdf: str, reg: str, val: str) -> bool: + r = subprocess.run(["setpci", "-s", bdf, f"{reg}={val}"], + capture_output=True) + return r.returncode == 0 + + +def _pcie_disconnect(extended_bdf: str, bridge: str | None, logger) -> None: + """Clear SERR/fatal-error bits and remove device from PCIe bus.""" + if bridge: + logger.info(f"[pegasus] clearing SERR bit on bridge {bridge}") + _setpci(bridge, "COMMAND", "0000:0100") + time.sleep(0.5) + logger.info(f"[pegasus] clearing fatal-error bit on bridge {bridge}") + _setpci(bridge, "CAP_EXP+8.w", "0000:0004") + time.sleep(0.5) + remove_path = os.path.join(PCI_DEVICES, extended_bdf, "remove") + if os.path.exists(remove_path): + logger.info(f"[pegasus] removing PCIe device {extended_bdf}") + try: + with open(remove_path, "w") as f: + f.write("1\n") + time.sleep(2) + except OSError as e: + logger.warning(f"[pegasus] remove write failed: {e}") + + +def _pcie_reconnect(bridge: str | None, device_bdf: str | None, logger) -> None: + """Rescan PCIe bus and re-enable memory-mapped transfers.""" + if bridge: + rescan_path = os.path.join(PCI_DEVICES, bridge, "rescan") + if os.path.exists(rescan_path): + logger.info(f"[pegasus] rescanning bridge {bridge}") + try: + with open(rescan_path, "w") as f: + f.write("1\n") + time.sleep(1) + except OSError: + pass + global_rescan = "/sys/bus/pci/rescan" + logger.info("[pegasus] global PCIe rescan") + try: + with open(global_rescan, "w") as f: + f.write("1\n") + time.sleep(2) + except OSError: + pass + # re-enable memory-mapped transfers on new BDF (re-detect after rescan) + new_bdf = _find_xdma_bdf() + if new_bdf: + logger.info(f"[pegasus] enabling MEM transfers on {new_bdf}") + _setpci(new_bdf, "COMMAND", "0x02") + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + + bitstream = input_data.get( + "bitstream", + os.path.join(bbdir, "thirdparty", "pegasus", "vivado", "build", "PegasusTop.bit"), + ) + serial = input_data.get("serial", "") # empty = auto (first target) + tcl_script = os.path.join( + bbdir, "thirdparty", "pegasus", "vivado", "scripts", "program_fpga.tcl" + ) + + if not os.path.exists(bitstream): + ctx.logger.error(f"[pegasus] bitstream not found: {bitstream}") + await check_result(ctx, 1, continue_run=False, + extra_fields={"error": "bitstream_not_found"}, trace_id=origin_tid) + return + + # ── 1. detect PCIe BDF ──────────────────────────────────────────────── + extended_bdf = input_data.get("bus_id") or _find_xdma_bdf() + bridge = _bridge_bdf(extended_bdf) if extended_bdf else None + ctx.logger.info(f"[pegasus] XDMA device: {extended_bdf}, bridge: {bridge}") + + # ── 2. PCIe disconnect ──────────────────────────────────────────────── + if extended_bdf: + _pcie_disconnect(extended_bdf, bridge, ctx.logger) + else: + ctx.logger.warning("[pegasus] no xdma device found, skipping PCIe disconnect") + + # ── 3. flash via Vivado ─────────────────────────────────────────────── + cmd = f"vivado -mode batch -source {tcl_script} -tclargs -bitstream_path {bitstream}" + if serial: + cmd += f" -serial {serial}" + + ctx.logger.info(f"[pegasus] flashing: {bitstream}") + result = stream_run_logger( + cmd=cmd, + logger=ctx.logger, + stdout_prefix="pegasus flash", + stderr_prefix="pegasus flash", + ) + + # ── 4. PCIe reconnect (regardless of flash result) ─────────────────── + if extended_bdf: + _pcie_reconnect(bridge, extended_bdf, ctx.logger) + + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={"bitstream": bitstream, "serial": serial}, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/pegasus/04_runworkload_api.step.py b/bbdev/api/steps/pegasus/04_runworkload_api.step.py new file mode 100644 index 000000000..62ea5c546 --- /dev/null +++ b/bbdev/api/steps/pegasus/04_runworkload_api.step.py @@ -0,0 +1,15 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "Pegasus Runworkload", + "description": "Run workload on AU280 (load Linux image + start CPU + collect UART)", + "flows": ["pegasus"], + "triggers": [api("POST", "/pegasus/runworkload")], + "enqueues": ["pegasus.runworkload"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + await ctx.enqueue({"topic": "pegasus.runworkload", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/pegasus/04_runworkload_event.step.py b/bbdev/api/steps/pegasus/04_runworkload_event.step.py new file mode 100644 index 000000000..6e0e4a9dd --- /dev/null +++ b/bbdev/api/steps/pegasus/04_runworkload_event.step.py @@ -0,0 +1,167 @@ +""" +pegasus runworkload event handler + +Sequence: + 1. Resolve marshal image paths (kernel ELF + rootfs img) + 2. Build pegasus-driver if needed + 3. pegasus-driver load --kernel --rootfs + 4. pegasus-driver run --log --timeout +""" +import os +import sys +import datetime +import subprocess + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "pegasus-runworkload", + "description": "Load Linux image into HBM2 and run on AU280", + "flows": ["pegasus"], + "triggers": [queue("pegasus.runworkload")], + "enqueues": [], +} + +KERNEL_OUTPUT_DIR = "bb-tests/output/kernel" + + +def _resolve_images(bbdir: str, workload: str, board: str) -> tuple[str, str]: + """Return (kernel_path, rootfs_path) from bbkernel output.""" + output_dir = os.path.join(bbdir, KERNEL_OUTPUT_DIR) + kernel = os.path.join(output_dir, "pegasus-bin") + rootfs = os.path.join(output_dir, "pegasus.img") + return kernel, rootfs + + +def _build_driver(bbdir: str, logger) -> str | None: + """Build pegasus-driver if not already built. Returns binary path or None.""" + driver_src = os.path.join(bbdir, "thirdparty", "pegasus", "driver") + build_dir = os.path.join(driver_src, "build") + binary = os.path.join(build_dir, "pegasus-driver") + + if os.path.exists(binary): + logger.info(f"[pegasus] driver already built: {binary}") + return binary + + logger.info("[pegasus] building pegasus-driver ...") + os.makedirs(build_dir, exist_ok=True) + + cmake_result = stream_run_logger( + cmd=f"cmake .. -DCMAKE_BUILD_TYPE=Release", + logger=logger, + cwd=build_dir, + stdout_prefix="cmake", + stderr_prefix="cmake", + ) + if cmake_result.returncode != 0: + logger.error("[pegasus] cmake configure failed") + return None + + build_result = stream_run_logger( + cmd=f"cmake --build . -j$(nproc)", + logger=logger, + cwd=build_dir, + stdout_prefix="make", + stderr_prefix="make", + ) + if build_result.returncode != 0: + logger.error("[pegasus] driver build failed") + return None + + logger.info(f"[pegasus] driver built: {binary}") + return binary + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + + workload = input_data.get("workload", "interactive") + board = input_data.get("board", "chipyard") + timeout = int(input_data.get("timeout", 300)) + uart_dev = input_data.get("uart", "/dev/ttyUSB0") + control_dev = input_data.get("control", "/dev/xdma0_control") + h2c_dev = input_data.get("h2c", "/dev/xdma0_h2c_0") + + # Log output directory + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + log_dir = os.path.join(bbdir, "arch", "log", ts) + os.makedirs(log_dir, exist_ok=True) + uart_log = os.path.join(log_dir, "pegasus_uart.log") + + # ── 1. Resolve image paths ──────────────────────────────────────────── + kernel, rootfs = _resolve_images(bbdir, workload, board) + ctx.logger.info(f"[pegasus] kernel: {kernel}") + ctx.logger.info(f"[pegasus] rootfs: {rootfs}") + + for path in (kernel, rootfs): + if not os.path.exists(path): + ctx.logger.error(f"[pegasus] image not found: {path} — run 'bbdev marshal --build' first") + await check_result(ctx, 1, continue_run=False, + extra_fields={"error": "image_not_found", "path": path}, + trace_id=origin_tid) + return + + # ── 2. Build driver ─────────────────────────────────────────────────── + driver = _build_driver(bbdir, ctx.logger) + if driver is None: + await check_result(ctx, 1, continue_run=False, + extra_fields={"error": "driver_build_failed"}, + trace_id=origin_tid) + return + + # ── 3. Load image into HBM2 ─────────────────────────────────────────── + load_cmd = ( + f"{driver} load" + f" --kernel {kernel}" + f" --rootfs {rootfs}" + f" --h2c {h2c_dev}" + ) + ctx.logger.info("[pegasus] loading images into HBM2 ...") + load_result = stream_run_logger( + cmd=load_cmd, + logger=ctx.logger, + stdout_prefix="pegasus load", + stderr_prefix="pegasus load", + ) + if load_result.returncode != 0: + await check_result(ctx, load_result.returncode, continue_run=False, + extra_fields={"error": "load_failed"}, + trace_id=origin_tid) + return + + # ── 4. Start CPU and collect UART ───────────────────────────────────── + run_cmd = ( + f"{driver} run" + f" --control {control_dev}" + f" --uart {uart_dev}" + f" --log {uart_log}" + f" --timeout {timeout}" + ) + ctx.logger.info(f"[pegasus] starting CPU, collecting UART → {uart_log}") + run_result = stream_run_logger( + cmd=run_cmd, + logger=ctx.logger, + stdout_prefix="pegasus uart", + stderr_prefix="pegasus uart", + ) + + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "workload": workload, + "board": board, + "uart_log": uart_log, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/bbdev b/bbdev/bbdev index 132999db4..9e8be937a 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -49,9 +49,8 @@ WORKFLOW_COMMANDS = { "doc": { "deploy": True, }, - "marshal": { - "build": "Build marshal.", - "launch": "Launch marshal.", + "kernel": { + "build": "Build RISC-V kernel + rootfs for Pegasus (cmake kernel-build).", }, "firesim": { "buildbitstream": "Build bitstream.", @@ -75,6 +74,8 @@ WORKFLOW_COMMANDS = { "pegasus": { "verilog": 'Generate SystemVerilog for Pegasus FPGA. Args: "[--config sims.pegasus.PegasusConfig]"', "buildbitstream": "Build Vivado bitstream for AU280.", + "flashbitstream": 'Flash bitstream onto AU280. Args: "[--bitstream ] [--serial ]"', + "runworkload": 'Run workload on AU280. Args: "[--workload ] [--board ] [--timeout ] [--uart ]"', }, } diff --git a/bbdev/nix/overlay.nix b/bbdev/nix/overlay.nix index 4e73a5fcc..2619d2f42 100644 --- a/bbdev/nix/overlay.nix +++ b/bbdev/nix/overlay.nix @@ -4,5 +4,8 @@ final: prev: { bbdevPythonPkgs = final.python312.withPackages (ps: with ps; [ pydantic requests + gitpython + humanfriendly + doit ]); } From ba4d1c2646b21c1cb1bba4de7703a3874b52d583 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 15 Apr 2026 04:28:54 +0800 Subject: [PATCH 031/196] fix: fix bugs to pass flashbitstream --- bbdev/.gitignore | 1 + bbdev/api/steps/kernel/01_build_event.step.py | 21 +-- .../steps/pegasus/01_verilog_event.step.py | 10 +- .../pegasus/02_buildbitstream_event.step.py | 8 +- .../pegasus/03_flashbitstream_event.step.py | 125 ++++++++++++------ .../pegasus/04_runworkload_event.step.py | 20 +-- 6 files changed, 105 insertions(+), 80 deletions(-) diff --git a/bbdev/.gitignore b/bbdev/.gitignore index a72dce8a0..438bbd056 100644 --- a/bbdev/.gitignore +++ b/bbdev/.gitignore @@ -12,6 +12,7 @@ types.d.ts appendonlydir/ dump.rdb *.log +*.jou output/ result/ diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index ceb78cdfe..94818c7c8 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -16,7 +16,7 @@ "description": "build RISC-V kernel + rootfs for Pegasus via bb-tests/workloads/lib/kernel", "flows": ["kernel"], "triggers": [queue("kernel.build")], - "enqueues": ["kernel.complete", "kernel.error"], + "enqueues": [], } @@ -36,8 +36,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="marshal build", ) if result.returncode != 0: - _, failure_result = await check_result(ctx, result.returncode, continue_run=False, trace_id=origin_tid) - await ctx.enqueue({"topic": "marshal.error", "data": {**input_data, "result": failure_result, "returncode": result.returncode}}) + await check_result(ctx, result.returncode, continue_run=False, trace_id=origin_tid) return # cmake build @@ -49,19 +48,5 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="marshal build", ) - success_result, failure_result = await check_result( - ctx, result.returncode, continue_run=False, trace_id=origin_tid) + await check_result(ctx, result.returncode, continue_run=False, trace_id=origin_tid) - if result.returncode == 0: - output_dir = os.path.join(kernel_src, "output") - await ctx.enqueue({ - "topic": "kernel.complete", - "data": {**input_data, "task": "kernel", "result": success_result, - "kernel": os.path.join(output_dir, "pegasus-bin"), - "rootfs": os.path.join(output_dir, "pegasus.img")}, - }) - else: - await ctx.enqueue({ - "topic": "kernel.error", - "data": {**input_data, "task": "kernel", "result": failure_result, "returncode": result.returncode}, - }) diff --git a/bbdev/api/steps/pegasus/01_verilog_event.step.py b/bbdev/api/steps/pegasus/01_verilog_event.step.py index 7167963df..8820c0f3b 100644 --- a/bbdev/api/steps/pegasus/01_verilog_event.step.py +++ b/bbdev/api/steps/pegasus/01_verilog_event.step.py @@ -29,9 +29,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/pegasus_top/") soc_dir = f"{bbdir}/arch/build/pegasus/" # DigitalTop + all SoC RTL - ctx.logger.info(f"[pegasus] Elaborating PegasusTop") - ctx.logger.info(f"[pegasus] Top output directory: {build_dir}") - ctx.logger.info(f"[pegasus] SoC RTL directory: {soc_dir}") + ctx.logger.info(f" Elaborating PegasusTop") + ctx.logger.info(f" Top output directory: {build_dir}") + ctx.logger.info(f" SoC RTL directory: {soc_dir}") os.makedirs(build_dir, exist_ok=True) @@ -53,7 +53,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="pegasus soc verilog", ) if soc_result.returncode != 0: - ctx.logger.error("[pegasus] SoC elaboration failed") + ctx.logger.error(" SoC elaboration failed") await check_result(ctx, soc_result.returncode, continue_run=False, extra_fields={"task": "verilog", "step": "soc"}, trace_id=origin_tid) return @@ -144,7 +144,7 @@ def copy_rtl_dir(src_dir: str, skip_set: set = None) -> int: n_soc = copy_rtl_dir(soc_dir, skip_set=HARNESS_SKIP) # Copy top-level wrappers (overrides any same-named file from soc_dir) n_top = copy_rtl_dir(build_dir) - ctx.logger.info(f"[pegasus] Copied {n_soc} SoC files + {n_top} top files to {vivado_gen_dir}") + ctx.logger.info(f" Copied {n_soc} SoC files + {n_top} top files to {vivado_gen_dir}") success_result, failure_result = await check_result( ctx, diff --git a/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py b/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py index 599cd1e98..60396ff74 100644 --- a/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py @@ -27,13 +27,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: output_dir = input_data.get("output_dir", f"{bbdir}/thirdparty/pegasus/vivado/build") top_module = input_data.get("top", "PegasusTop") - ctx.logger.info(f"[pegasus] Generated dir: {generated_dir}") - ctx.logger.info(f"[pegasus] Output dir: {output_dir}") + ctx.logger.info(f" Generated dir: {generated_dir}") + ctx.logger.info(f" Output dir: {output_dir}") os.makedirs(output_dir, exist_ok=True) if not os.path.isdir(generated_dir): - ctx.logger.error(f"[pegasus] generated dir not found: {generated_dir}") + ctx.logger.error(f" generated dir not found: {generated_dir}") success_result, failure_result = await check_result( ctx, 1, @@ -45,7 +45,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: has_rtl = any(name.endswith(".sv") or name.endswith(".v") for name in os.listdir(generated_dir)) if not has_rtl: - ctx.logger.error(f"[pegasus] no verilog files found in: {generated_dir}") + ctx.logger.error(f" no verilog files found in: {generated_dir}") success_result, failure_result = await check_result( ctx, 1, diff --git a/bbdev/api/steps/pegasus/03_flashbitstream_event.step.py b/bbdev/api/steps/pegasus/03_flashbitstream_event.step.py index 5f77a392f..5b8d4c476 100644 --- a/bbdev/api/steps/pegasus/03_flashbitstream_event.step.py +++ b/bbdev/api/steps/pegasus/03_flashbitstream_event.step.py @@ -2,15 +2,14 @@ pegasus flashbitstream event handler Sequence: - 1. Detect XDMA PCIe BDF (auto-scan /sys/bus/pci/devices for xdma driver) - 2. PCIe disconnect: clear SERR + fatal-error bits, remove device - 3. Flash via Vivado + program_fpga.tcl - 4. PCIe reconnect: rescan, re-enable memory-mapped transfers + 1. Detect XDMA PCIe BDF + 2. Flash via Vivado + program_fpga.tcl (JTAG — independent of PCIe) + 3. PCIe remove + rescan so new bitstream's PCIe IP re-enumerates """ import os import sys -import glob import time +import socket import subprocess from motia import FlowContext, queue @@ -56,30 +55,65 @@ def _bridge_bdf(extended_bdf: str) -> str | None: return None +def _ensure_hw_server(port: int = 3121, logger=None) -> bool: + """Ensure hw_server is listening on the given port; start it if not.""" + try: + with socket.create_connection(("localhost", port), timeout=2): + return True + except OSError: + pass + if logger: + logger.info(f"[pegasus] hw_server not on port {port}, starting ...") + subprocess.Popen( + ["hw_server"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + start_new_session=True, + ) + for _ in range(15): + time.sleep(1) + try: + with socket.create_connection(("localhost", port), timeout=1): + if logger: + logger.info(f"[pegasus] hw_server ready on port {port}") + return True + except OSError: + pass + if logger: + logger.error(f"[pegasus] hw_server failed to start on port {port}") + return False + + def _setpci(bdf: str, reg: str, val: str) -> bool: - r = subprocess.run(["setpci", "-s", bdf, f"{reg}={val}"], + r = subprocess.run(["sudo", "setpci", "-s", bdf, f"{reg}={val}"], capture_output=True) return r.returncode == 0 +def _sysfs_write(path: str, value: str, logger) -> bool: + """Write to a sysfs file via 'sudo tee' (needed for root-owned sysfs entries).""" + r = subprocess.run( + f"echo {value} | sudo tee {path}", + shell=True, capture_output=True, + ) + if r.returncode != 0: + logger.error(f" sysfs write failed: {path} <- {value}: {r.stderr.decode().strip()}") + return r.returncode == 0 + + def _pcie_disconnect(extended_bdf: str, bridge: str | None, logger) -> None: """Clear SERR/fatal-error bits and remove device from PCIe bus.""" if bridge: - logger.info(f"[pegasus] clearing SERR bit on bridge {bridge}") + logger.info(f" clearing SERR bit on bridge {bridge}") _setpci(bridge, "COMMAND", "0000:0100") time.sleep(0.5) - logger.info(f"[pegasus] clearing fatal-error bit on bridge {bridge}") + logger.info(f" clearing fatal-error bit on bridge {bridge}") _setpci(bridge, "CAP_EXP+8.w", "0000:0004") time.sleep(0.5) remove_path = os.path.join(PCI_DEVICES, extended_bdf, "remove") if os.path.exists(remove_path): - logger.info(f"[pegasus] removing PCIe device {extended_bdf}") - try: - with open(remove_path, "w") as f: - f.write("1\n") + logger.info(f" removing PCIe device {extended_bdf}") + if _sysfs_write(remove_path, "1", logger): time.sleep(2) - except OSError as e: - logger.warning(f"[pegasus] remove write failed: {e}") def _pcie_reconnect(bridge: str | None, device_bdf: str | None, logger) -> None: @@ -87,32 +121,31 @@ def _pcie_reconnect(bridge: str | None, device_bdf: str | None, logger) -> None: if bridge: rescan_path = os.path.join(PCI_DEVICES, bridge, "rescan") if os.path.exists(rescan_path): - logger.info(f"[pegasus] rescanning bridge {bridge}") - try: - with open(rescan_path, "w") as f: - f.write("1\n") - time.sleep(1) - except OSError: - pass - global_rescan = "/sys/bus/pci/rescan" - logger.info("[pegasus] global PCIe rescan") - try: - with open(global_rescan, "w") as f: - f.write("1\n") - time.sleep(2) - except OSError: - pass + logger.info(f" rescanning bridge {bridge}") + _sysfs_write(rescan_path, "1", logger) + time.sleep(1) + logger.info(" global PCIe rescan") + _sysfs_write("/sys/bus/pci/rescan", "1", logger) + time.sleep(2) # re-enable memory-mapped transfers on new BDF (re-detect after rescan) new_bdf = _find_xdma_bdf() if new_bdf: - logger.info(f"[pegasus] enabling MEM transfers on {new_bdf}") - _setpci(new_bdf, "COMMAND", "0x02") + logger.info(f" enabling MEM transfers on {new_bdf}") + _setpci(new_bdf, "COMMAND.W", "0002") async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() + # ── 0. ensure hw_server is running (Vivado can't launch it in bbdev env) ─ + hw_server_url = input_data.get("hw_server", "localhost:3121") + hw_port = int(hw_server_url.split(":")[-1]) + if not _ensure_hw_server(hw_port, ctx.logger): + await check_result(ctx, 1, continue_run=False, + extra_fields={"error": "hw_server_unavailable"}, trace_id=origin_tid) + return + bitstream = input_data.get( "bitstream", os.path.join(bbdir, "thirdparty", "pegasus", "vivado", "build", "PegasusTop.bit"), @@ -123,7 +156,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) if not os.path.exists(bitstream): - ctx.logger.error(f"[pegasus] bitstream not found: {bitstream}") + ctx.logger.error(f" bitstream not found: {bitstream}") await check_result(ctx, 1, continue_run=False, extra_fields={"error": "bitstream_not_found"}, trace_id=origin_tid) return @@ -131,20 +164,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── 1. detect PCIe BDF ──────────────────────────────────────────────── extended_bdf = input_data.get("bus_id") or _find_xdma_bdf() bridge = _bridge_bdf(extended_bdf) if extended_bdf else None - ctx.logger.info(f"[pegasus] XDMA device: {extended_bdf}, bridge: {bridge}") + ctx.logger.info(f" XDMA device: {extended_bdf}, bridge: {bridge}") - # ── 2. PCIe disconnect ──────────────────────────────────────────────── - if extended_bdf: - _pcie_disconnect(extended_bdf, bridge, ctx.logger) - else: - ctx.logger.warning("[pegasus] no xdma device found, skipping PCIe disconnect") - - # ── 3. flash via Vivado ─────────────────────────────────────────────── - cmd = f"vivado -mode batch -source {tcl_script} -tclargs -bitstream_path {bitstream}" + # ── 2. flash via Vivado (JTAG — does not require PCIe) ─────────────── + cmd = ( + f"vivado -mode tcl -source {tcl_script}" + f" -tclargs -bitstream_path {bitstream} -hw_server {hw_server_url}" + ) if serial: cmd += f" -serial {serial}" - ctx.logger.info(f"[pegasus] flashing: {bitstream}") + ctx.logger.info(f" flashing: {bitstream}") result = stream_run_logger( cmd=cmd, logger=ctx.logger, @@ -152,9 +182,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="pegasus flash", ) - # ── 4. PCIe reconnect (regardless of flash result) ─────────────────── + if result.returncode != 0: + await check_result(ctx, result.returncode, continue_run=False, + extra_fields={"bitstream": bitstream, "serial": serial}, + trace_id=origin_tid) + return + + # ── 3. PCIe remove → rescan so new bitstream's PCIe IP re-enumerates ─ if extended_bdf: + _pcie_disconnect(extended_bdf, bridge, ctx.logger) _pcie_reconnect(bridge, extended_bdf, ctx.logger) + else: + ctx.logger.warn(" no xdma device found, skipping PCIe remove/rescan") await check_result( ctx, diff --git a/bbdev/api/steps/pegasus/04_runworkload_event.step.py b/bbdev/api/steps/pegasus/04_runworkload_event.step.py index 6e0e4a9dd..81aa60c43 100644 --- a/bbdev/api/steps/pegasus/04_runworkload_event.step.py +++ b/bbdev/api/steps/pegasus/04_runworkload_event.step.py @@ -48,10 +48,10 @@ def _build_driver(bbdir: str, logger) -> str | None: binary = os.path.join(build_dir, "pegasus-driver") if os.path.exists(binary): - logger.info(f"[pegasus] driver already built: {binary}") + logger.info(f" driver already built: {binary}") return binary - logger.info("[pegasus] building pegasus-driver ...") + logger.info(" building pegasus-driver ...") os.makedirs(build_dir, exist_ok=True) cmake_result = stream_run_logger( @@ -62,7 +62,7 @@ def _build_driver(bbdir: str, logger) -> str | None: stderr_prefix="cmake", ) if cmake_result.returncode != 0: - logger.error("[pegasus] cmake configure failed") + logger.error(" cmake configure failed") return None build_result = stream_run_logger( @@ -73,10 +73,10 @@ def _build_driver(bbdir: str, logger) -> str | None: stderr_prefix="make", ) if build_result.returncode != 0: - logger.error("[pegasus] driver build failed") + logger.error(" driver build failed") return None - logger.info(f"[pegasus] driver built: {binary}") + logger.info(f" driver built: {binary}") return binary @@ -99,12 +99,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── 1. Resolve image paths ──────────────────────────────────────────── kernel, rootfs = _resolve_images(bbdir, workload, board) - ctx.logger.info(f"[pegasus] kernel: {kernel}") - ctx.logger.info(f"[pegasus] rootfs: {rootfs}") + ctx.logger.info(f" kernel: {kernel}") + ctx.logger.info(f" rootfs: {rootfs}") for path in (kernel, rootfs): if not os.path.exists(path): - ctx.logger.error(f"[pegasus] image not found: {path} — run 'bbdev marshal --build' first") + ctx.logger.error(f" image not found: {path} — run 'bbdev marshal --build' first") await check_result(ctx, 1, continue_run=False, extra_fields={"error": "image_not_found", "path": path}, trace_id=origin_tid) @@ -125,7 +125,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f" --rootfs {rootfs}" f" --h2c {h2c_dev}" ) - ctx.logger.info("[pegasus] loading images into HBM2 ...") + ctx.logger.info(" loading images into HBM2 ...") load_result = stream_run_logger( cmd=load_cmd, logger=ctx.logger, @@ -146,7 +146,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f" --log {uart_log}" f" --timeout {timeout}" ) - ctx.logger.info(f"[pegasus] starting CPU, collecting UART → {uart_log}") + ctx.logger.info(f" starting CPU, collecting UART → {uart_log}") run_result = stream_run_logger( cmd=run_cmd, logger=ctx.logger, From cac473b2b896479da150c5ec5dd09d91a24edfcd Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 15 Apr 2026 14:51:55 +0800 Subject: [PATCH 032/196] fix: enhance build process by clearing previous output and updating environment variables for pkg-config --- bbdev/api/steps/kernel/01_build_event.step.py | 7 +++++++ .../steps/pegasus/04_runworkload_event.step.py | 18 +++++++++++++----- bbdev/api/utils/stream_run.py | 4 ++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index 94818c7c8..75df33e8c 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -1,5 +1,6 @@ import os import sys +import shutil from motia import FlowContext, queue @@ -26,6 +27,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: kernel_src = os.path.join(bbdir, "bb-tests", "workloads", "lib", "kernel") kernel_build = os.path.join(bbdir, "bb-tests", "build", "kernel") + output_dir = os.path.join(bbdir, "bb-tests", "output", "kernel") + + # Clear previous output before rebuild + if os.path.isdir(output_dir): + shutil.rmtree(output_dir) + os.makedirs(output_dir, exist_ok=True) # cmake configure configure_cmd = f"cmake -B {kernel_build} -S {kernel_src}" diff --git a/bbdev/api/steps/pegasus/04_runworkload_event.step.py b/bbdev/api/steps/pegasus/04_runworkload_event.step.py index 81aa60c43..6dc36dbb6 100644 --- a/bbdev/api/steps/pegasus/04_runworkload_event.step.py +++ b/bbdev/api/steps/pegasus/04_runworkload_event.step.py @@ -54,12 +54,20 @@ def _build_driver(bbdir: str, logger) -> str | None: logger.info(" building pegasus-driver ...") os.makedirs(build_dir, exist_ok=True) + # Ensure pkg-config can find nix-managed libraries (libelf from elfutils) + result_dir = os.path.join(bbdir, "result") + pkg_config_path = os.environ.get("PKG_CONFIG_PATH", "") + nix_pc = os.path.join(result_dir, "lib", "pkgconfig") + if nix_pc not in pkg_config_path: + pkg_config_path = f"{nix_pc}:{pkg_config_path}" if pkg_config_path else nix_pc + cmake_result = stream_run_logger( - cmd=f"cmake .. -DCMAKE_BUILD_TYPE=Release", + cmd="cmake .. -DCMAKE_BUILD_TYPE=Release", logger=logger, cwd=build_dir, stdout_prefix="cmake", stderr_prefix="cmake", + env={**os.environ, "PKG_CONFIG_PATH": pkg_config_path}, ) if cmake_result.returncode != 0: logger.error(" cmake configure failed") @@ -88,7 +96,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: board = input_data.get("board", "chipyard") timeout = int(input_data.get("timeout", 300)) uart_dev = input_data.get("uart", "/dev/ttyUSB0") - control_dev = input_data.get("control", "/dev/xdma0_control") + control_dev = input_data.get("control", "/dev/xdma0_user") h2c_dev = input_data.get("h2c", "/dev/xdma0_h2c_0") # Log output directory @@ -104,7 +112,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: for path in (kernel, rootfs): if not os.path.exists(path): - ctx.logger.error(f" image not found: {path} — run 'bbdev marshal --build' first") + ctx.logger.error(f" image not found: {path} — run 'bbdev kernel --build' first") await check_result(ctx, 1, continue_run=False, extra_fields={"error": "image_not_found", "path": path}, trace_id=origin_tid) @@ -120,7 +128,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── 3. Load image into HBM2 ─────────────────────────────────────────── load_cmd = ( - f"{driver} load" + f"sudo {driver} load" f" --kernel {kernel}" f" --rootfs {rootfs}" f" --h2c {h2c_dev}" @@ -140,7 +148,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── 4. Start CPU and collect UART ───────────────────────────────────── run_cmd = ( - f"{driver} run" + f"sudo {driver} run" f" --control {control_dev}" f" --uart {uart_dev}" f" --log {uart_log}" diff --git a/bbdev/api/utils/stream_run.py b/bbdev/api/utils/stream_run.py index 6bd6dcb14..11dcd0c0c 100644 --- a/bbdev/api/utils/stream_run.py +++ b/bbdev/api/utils/stream_run.py @@ -22,6 +22,7 @@ def stream_run( on_stderr: Optional[Callable[[str], None]] = None, stdout_prefix: str = "STDOUT", stderr_prefix: str = "STDERR", + env: Optional[dict] = None, ) -> StreamResult: """ Execute command and stream output in real-time @@ -78,6 +79,7 @@ def read_stream( text=True, bufsize=1, executable=executable, + env=env, ) stdout_lines = [] @@ -125,6 +127,7 @@ def stream_run_logger( stdout_prefix: str = "STDOUT", stderr_prefix: str = "STDERR", verbose: bool = False, + env: Optional[dict] = None, ) -> StreamResult: """ Convenience function for streaming output using logger @@ -169,4 +172,5 @@ def log_stderr(line): on_stderr=log_stderr, stdout_prefix=stdout_prefix, stderr_prefix=stderr_prefix, + env=env, ) From 6d80bd841ca9a84aa19e7bab5c8a43ea5b05f7af Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 17 Apr 2026 13:25:36 +0800 Subject: [PATCH 033/196] fix: update workload event description and logging to reflect DDR usage --- bbdev/api/steps/pegasus/04_runworkload_event.step.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bbdev/api/steps/pegasus/04_runworkload_event.step.py b/bbdev/api/steps/pegasus/04_runworkload_event.step.py index 6dc36dbb6..651108aa9 100644 --- a/bbdev/api/steps/pegasus/04_runworkload_event.step.py +++ b/bbdev/api/steps/pegasus/04_runworkload_event.step.py @@ -24,7 +24,7 @@ config = { "name": "pegasus-runworkload", - "description": "Load Linux image into HBM2 and run on AU280", + "description": "Load Linux image into DDR and run on AU280", "flows": ["pegasus"], "triggers": [queue("pegasus.runworkload")], "enqueues": [], @@ -126,14 +126,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid) return - # ── 3. Load image into HBM2 ─────────────────────────────────────────── + # ── 3. Load image into DDR ─────────────────────────────────────────── load_cmd = ( f"sudo {driver} load" f" --kernel {kernel}" f" --rootfs {rootfs}" f" --h2c {h2c_dev}" ) - ctx.logger.info(" loading images into HBM2 ...") + ctx.logger.info(" loading images into DDR ...") load_result = stream_run_logger( cmd=load_cmd, logger=ctx.logger, From 90e2563fb97890aca2a1d0d598f5a931019315c8 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 22 Apr 2026 01:26:06 +0800 Subject: [PATCH 034/196] feat: add PCIe device detection and driver management in flashbitstream event --- .../pegasus/03_flashbitstream_event.step.py | 161 ++++++++++++++++-- .../pegasus/04_runworkload_event.step.py | 6 +- 2 files changed, 151 insertions(+), 16 deletions(-) diff --git a/bbdev/api/steps/pegasus/03_flashbitstream_event.step.py b/bbdev/api/steps/pegasus/03_flashbitstream_event.step.py index 5b8d4c476..e45786047 100644 --- a/bbdev/api/steps/pegasus/03_flashbitstream_event.step.py +++ b/bbdev/api/steps/pegasus/03_flashbitstream_event.step.py @@ -11,6 +11,8 @@ import time import socket import subprocess +import glob +import re from motia import FlowContext, queue @@ -31,10 +33,12 @@ } PCI_DEVICES = "/sys/bus/pci/devices" +XILINX_VENDOR = "0x10ee" +XDMA_DEVICE_IDS = {"0x903f"} def _find_xdma_bdf() -> str | None: - """Return the extended BDF (e.g. '0000:65:00.0') of the first xdma device.""" + """Return the extended BDF (e.g. '0000:65:00.0') bound to xdma driver.""" for entry in os.listdir(PCI_DEVICES): driver_link = os.path.join(PCI_DEVICES, entry, "driver") if not os.path.islink(driver_link): @@ -45,6 +49,20 @@ def _find_xdma_bdf() -> str | None: return None +def _find_xdma_pci_bdf() -> str | None: + """Return first XDMA-like Xilinx PCIe endpoint even if driver is not bound.""" + for entry in os.listdir(PCI_DEVICES): + vendor_file = os.path.join(PCI_DEVICES, entry, "vendor") + device_file = os.path.join(PCI_DEVICES, entry, "device") + if not (os.path.exists(vendor_file) and os.path.exists(device_file)): + continue + vendor = open(vendor_file, "r", encoding="utf-8").read().strip().lower() + device = open(device_file, "r", encoding="utf-8").read().strip().lower() + if vendor == XILINX_VENDOR and device in XDMA_DEVICE_IDS: + return entry + return None + + def _bridge_bdf(extended_bdf: str) -> str | None: """Return the bridge BDF for a given extended device BDF.""" dev_path = os.path.realpath(os.path.join(PCI_DEVICES, extended_bdf)) @@ -127,11 +145,115 @@ def _pcie_reconnect(bridge: str | None, device_bdf: str | None, logger) -> None: logger.info(" global PCIe rescan") _sysfs_write("/sys/bus/pci/rescan", "1", logger) time.sleep(2) - # re-enable memory-mapped transfers on new BDF (re-detect after rescan) - new_bdf = _find_xdma_bdf() - if new_bdf: - logger.info(f" enabling MEM transfers on {new_bdf}") - _setpci(new_bdf, "COMMAND.W", "0002") + # Re-detect after rescan: prefer xdma-bound BDF, fall back to raw PCIe endpoint. + new_bdf = _find_xdma_bdf() or _find_xdma_pci_bdf() + if not new_bdf: + for _ in range(5): + _sysfs_write("/sys/bus/pci/rescan", "1", logger) + time.sleep(1) + new_bdf = _find_xdma_bdf() or _find_xdma_pci_bdf() + if new_bdf: + break + if not new_bdf: + raise RuntimeError("xdma pci endpoint not found after PCIe rescan") + logger.info(f" enabling MEM+BUSMSTR on {new_bdf}") + _setpci(new_bdf, "COMMAND.W", "0006") + + +def _run_checked(cmd: str) -> None: + subprocess.run(cmd, shell=True, check=True, capture_output=True, text=True) + + +def _reload_xdma(logger) -> None: + """Align XDMA lifecycle with FireSim: unload/load with poll_mode=1.""" + logger.info(" reloading xdma kernel module") + loaded = subprocess.run( + "lsmod | grep -wq xdma", shell=True + ).returncode == 0 + if loaded: + _run_checked("sudo rmmod xdma") + _run_checked("sudo sh -c 'insmod $(find /lib/modules/$(uname -r) -name \"xdma.ko\") poll_mode=1'") + _run_checked("sudo chmod a+rw /dev/xdma*") + devs = glob.glob("/dev/xdma*_h2c_0") + if not devs: + raise RuntimeError("xdma driver loaded but /dev/xdma*_h2c_0 not found") + logger.info(f" xdma ready: {devs[0]}") + + +def _parse_mig_state(output: str) -> str: + text = output.lower() + if "calibration is still in-progress" in text: + return "busy" + + pairs = re.findall(r"^MIG_PROP\s+([^=]+)=(.*)$", output, flags=re.M) + if not pairs: + return "unknown" + + norm = [(k.strip().lower(), v.strip().lower()) for k, v in pairs] + prop_map = {k: v for k, v in norm} + + init_done = prop_map.get("init_calib_complete") + if init_done is not None: + if init_done in {"1", "true", "yes"}: + return "ready" + if init_done in {"0", "false", "no"}: + return "busy" + + if any(re.search(r"(in[-_ ]progress|busy)", v) for _, v in norm): + return "busy" + + fail_status = prop_map.get("calibration_fail.status") + has_pass = any(k.startswith("cal_status.") and v == "pass" for k, v in norm) + if has_pass and fail_status in {"false", "0", "no"}: + return "ready" + + status_ready = any( + ("status" in k or "calib" in k) and re.search(r"(done|pass|complete|success)", v) + for k, v in norm + ) + if status_ready: + return "ready" + + return "unknown" + + +def _wait_ddr_calibration(hw_server_url: str, serial: str, timeout_sec: int, tcl_script: str, logger) -> None: + if timeout_sec <= 0: + logger.info(" skip DDR calibration polling because timeout <= 0") + return + deadline = time.time() + timeout_sec + poll_sec = 5 + logger.info(f" polling DDR calibration status for up to {timeout_sec}s") + while True: + cmd = ( + f"vivado -mode tcl -source {tcl_script}" + f" -tclargs -hw_server {hw_server_url}" + ) + if serial: + cmd += f" -serial {serial}" + result = stream_run_logger( + cmd=cmd, + logger=logger, + stdout_prefix="pegasus mig", + stderr_prefix="pegasus mig", + ) + if result.returncode != 0: + raise RuntimeError( + f"mig status query failed: rc={result.returncode} stdout={result.stdout} stderr={result.stderr}" + ) + merged = f"{result.stdout}\n{result.stderr}" + state = _parse_mig_state(merged) + if state == "ready": + logger.info(" DDR calibration reported complete") + return + if state == "unknown": + raise RuntimeError("DDR calibration state unknown: no explicit MIG calibration status") + now = time.time() + if now >= deadline: + raise RuntimeError("DDR calibration did not complete before timeout") + remain = int(max(0, deadline - now)) + logger.info(f" DDR calibration still in progress, {remain}s remaining") + time.sleep(min(poll_sec, max(1, deadline - now))) async def handler(input_data: dict, ctx: FlowContext) -> None: @@ -151,9 +273,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: os.path.join(bbdir, "thirdparty", "pegasus", "vivado", "build", "PegasusTop.bit"), ) serial = input_data.get("serial", "") # empty = auto (first target) + ddr_settle_sec = int(input_data.get("ddr_settle_sec", 180)) tcl_script = os.path.join( bbdir, "thirdparty", "pegasus", "vivado", "scripts", "program_fpga.tcl" ) + mig_status_tcl = os.path.join( + bbdir, "thirdparty", "pegasus", "vivado", "scripts", "query_mig_status.tcl" + ) if not os.path.exists(bitstream): ctx.logger.error(f" bitstream not found: {bitstream}") @@ -162,7 +288,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return # ── 1. detect PCIe BDF ──────────────────────────────────────────────── - extended_bdf = input_data.get("bus_id") or _find_xdma_bdf() + extended_bdf = input_data.get("bus_id") or _find_xdma_bdf() or _find_xdma_pci_bdf() bridge = _bridge_bdf(extended_bdf) if extended_bdf else None ctx.logger.info(f" XDMA device: {extended_bdf}, bridge: {bridge}") @@ -189,11 +315,24 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return # ── 3. PCIe remove → rescan so new bitstream's PCIe IP re-enumerates ─ - if extended_bdf: - _pcie_disconnect(extended_bdf, bridge, ctx.logger) + try: + if extended_bdf: + _pcie_disconnect(extended_bdf, bridge, ctx.logger) + else: + ctx.logger.info(" no xdma device found before flash, skip disconnect and try global rescan") _pcie_reconnect(bridge, extended_bdf, ctx.logger) - else: - ctx.logger.warn(" no xdma device found, skipping PCIe remove/rescan") + _reload_xdma(ctx.logger) + _wait_ddr_calibration(hw_server_url, serial, ddr_settle_sec, mig_status_tcl, ctx.logger) + except Exception as e: + ctx.logger.error(f" post-flash PCIe/XDMA bring-up failed: {e}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "post_flash_xdma_recover_failed", "detail": str(e)}, + trace_id=origin_tid, + ) + return await check_result( ctx, diff --git a/bbdev/api/steps/pegasus/04_runworkload_event.step.py b/bbdev/api/steps/pegasus/04_runworkload_event.step.py index 651108aa9..b656e0ddb 100644 --- a/bbdev/api/steps/pegasus/04_runworkload_event.step.py +++ b/bbdev/api/steps/pegasus/04_runworkload_event.step.py @@ -42,15 +42,11 @@ def _resolve_images(bbdir: str, workload: str, board: str) -> tuple[str, str]: def _build_driver(bbdir: str, logger) -> str | None: - """Build pegasus-driver if not already built. Returns binary path or None.""" + """Build pegasus-driver with native incremental build. Returns binary path or None.""" driver_src = os.path.join(bbdir, "thirdparty", "pegasus", "driver") build_dir = os.path.join(driver_src, "build") binary = os.path.join(build_dir, "pegasus-driver") - if os.path.exists(binary): - logger.info(f" driver already built: {binary}") - return binary - logger.info(" building pegasus-driver ...") os.makedirs(build_dir, exist_ok=True) From 88d9b5e9b82012a6eaa2fe1fface720f0bb5a5bb Mon Sep 17 00:00:00 2001 From: Bohan Wang <140929282+Mikemy666@users.noreply.github.com> Date: Wed, 22 Apr 2026 13:55:02 +0800 Subject: [PATCH 035/196] feat: add dc command and workflow to bbdev (#8) --- bbdev/api/steps/dc/01_run_api.step.py | 39 +++++++ bbdev/api/steps/dc/01_run_event.step.py | 133 ++++++++++++++++++++++++ bbdev/bbdev | 30 ++++++ 3 files changed, 202 insertions(+) create mode 100644 bbdev/api/steps/dc/01_run_api.step.py create mode 100644 bbdev/api/steps/dc/01_run_event.step.py diff --git a/bbdev/api/steps/dc/01_run_api.step.py b/bbdev/api/steps/dc/01_run_api.step.py new file mode 100644 index 000000000..b81145caa --- /dev/null +++ b/bbdev/api/steps/dc/01_run_api.step.py @@ -0,0 +1,39 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path + +config = { + "name": "dc-run-api", + "description": "run Design Compiler synthesis script", + "flows": ["dc"], + "triggers": [api("POST", "/dc/run")], + "enqueues": ["dc.run"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = request.body or {} + + srcdir = body.get("srcdir") + if not srcdir: + return ApiResponse( + status=400, + body={ + "status": "error", + "message": "Missing required parameter: --srcdir", + "example": 'bbdev dc --srcdir arch/ReluBall_1 --top ReluBall', + }, + ) + + data = { + "srcdir": srcdir, + "top": body.get("top"), + "keep_hierarchy": bool(body.get("keep_hierarchy", False)), + "balltype": body.get("balltype"), + "config": body.get("config", "sims.verilator.BuckyballToyVerilatorConfig"), + "output_dir": body.get("output_dir"), + "report_dir": f"{bbdir}/bb-tests/output/dc/reports", + } + await ctx.enqueue({"topic": "dc.run", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/dc/01_run_event.step.py b/bbdev/api/steps/dc/01_run_event.step.py new file mode 100644 index 000000000..e245d8450 --- /dev/null +++ b/bbdev/api/steps/dc/01_run_event.step.py @@ -0,0 +1,133 @@ +import os +import shlex +import sys + +from motia import FlowContext, queue + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger + +config = { + "name": "dc-run", + "description": "run Design Compiler synthesis script", + "flows": ["dc"], + "triggers": [queue("dc.run")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + script_path = f"{bbdir}/evals/run-dc.sh" + + srcdir = input_data.get("srcdir") + top = input_data.get("top") + keep_hierarchy = bool(input_data.get("keep_hierarchy", False)) + balltype = input_data.get("balltype") + config_name = input_data.get("config", "sims.verilator.BuckyballToyVerilatorConfig") + output_dir = input_data.get("output_dir") + + if not srcdir: + success_result, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "dc", + "error": "Missing required parameter: srcdir", + "example": "bbdev dc --srcdir arch/ReluBall_1 --top ReluBall", + }, + trace_id=origin_tid, + ) + return failure_result + + if not os.path.exists(script_path): + success_result, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "dc", + "error": f"DC script not found: {script_path}", + }, + trace_id=origin_tid, + ) + return failure_result + + # Keep compatibility with old manual flow: ensure run-dc.sh is executable. + os.chmod(script_path, os.stat(script_path).st_mode | 0o111) + + # Optional integrated pre-step: generate Ball verilog first. + if balltype: + final_output_dir = output_dir or f"{balltype}_1" + verilog_dir = f"{bbdir}/arch/{final_output_dir}" + verilog_cmd = ( + f"mill -i __.test.runMain sims.verify.BallTopMain {shlex.quote(str(balltype))} " + "--disable-annotation-unknown --strip-debug-info -O=debug " + f"--split-verilog -o={shlex.quote(verilog_dir)}" + ) + + verilog_result = stream_run_logger( + cmd=verilog_cmd, + logger=ctx.logger, + cwd=f"{bbdir}/arch", + stdout_prefix="dc verilog", + stderr_prefix="dc verilog", + ) + + if verilog_result.returncode != 0: + success_result, failure_result = await check_result( + ctx, + verilog_result.returncode, + continue_run=False, + extra_fields={ + "task": "dc", + "error": "Failed to generate pre-DC verilog", + "balltype": balltype, + "config": config_name, + }, + trace_id=origin_tid, + ) + return failure_result + + # If user keeps default-like srcdir, force it to generated dir for consistency. + srcdir = f"arch/{final_output_dir}" + + command_parts = ["bash", shlex.quote(script_path), "--srcdir", shlex.quote(str(srcdir))] + if top: + command_parts.extend(["--top", shlex.quote(str(top))]) + if keep_hierarchy: + command_parts.append("--keep-hierarchy") + + command = " ".join(command_parts) + + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=bbdir, + executable="bash", + stdout_prefix="dc run", + stderr_prefix="dc run", + ) + + success_result, failure_result = await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={ + "task": "dc", + "report_dir": input_data.get("report_dir", f"{bbdir}/bb-tests/output/dc/reports"), + "srcdir": srcdir, + "top": top, + }, + trace_id=origin_tid, + ) + + return diff --git a/bbdev/bbdev b/bbdev/bbdev index 9e8be937a..03790fd45 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -77,6 +77,9 @@ WORKFLOW_COMMANDS = { "flashbitstream": 'Flash bitstream onto AU280. Args: "[--bitstream ] [--serial ]"', "runworkload": 'Run workload on AU280. Args: "[--workload ] [--board ] [--timeout ] [--uart ]"', }, + "dc": { + "run": 'Run Design Compiler synthesis. Args: "--srcdir [--top ] [--keep-hierarchy] [--balltype ] [--config ] [--output-dir ]"', + }, } ALL_COMMANDS = ["start", "stop"] + list(WORKFLOW_COMMANDS) @@ -169,8 +172,21 @@ def parse_args(argv: list[str]) -> argparse.Namespace: stop_parser = subparsers.add_parser("stop", parents=[common], help="Stop dev server") stop_parser.add_argument("--all", action="store_true", help="Stop all servers") + # dc direct mode (no --run wrapper): + # bbdev dc --srcdir ... [--top ...] [--keep-hierarchy] + # [--balltype ...] [--config ...] [--output-dir ...] + dc_parser = subparsers.add_parser("dc", parents=[common], help="Run DC synthesis") + dc_parser.add_argument("--srcdir", type=str, required=True, help="Design source directory relative to repo root") + dc_parser.add_argument("--top", type=str, default=None, help="Top module name") + dc_parser.add_argument("--keep-hierarchy", action="store_true", help="Keep hierarchy during compile") + dc_parser.add_argument("--balltype", type=str, default=None, help="Optional Ball type for pre-verilog generation, e.g. ReluBall") + dc_parser.add_argument("--config", type=str, default="sims.verilator.BuckyballToyVerilatorConfig", help="Verilog generation config when --balltype is set") + dc_parser.add_argument("--output-dir", type=str, default=None, help="Verilog output dir under arch/, e.g. ReluBall_1") + # Build workflow subcommands from WORKFLOW_COMMANDS table for cmd_name, operations in WORKFLOW_COMMANDS.items(): + if cmd_name == "dc": + continue cmd_parser = subparsers.add_parser( cmd_name, parents=[common], help=f"{cmd_name} operations" ) @@ -193,6 +209,20 @@ def parse_args(argv: list[str]) -> argparse.Namespace: def extract_command_info(args: argparse.Namespace) -> dict: """Extract command, operation, and parsed sub-arguments from argparse Namespace.""" + if args.command == "dc": + return { + "command": "dc", + "operation": "run", + "args": { + "srcdir": args.srcdir, + "top": args.top, + "keep_hierarchy": bool(args.keep_hierarchy), + "balltype": args.balltype, + "config": args.config, + "output_dir": args.output_dir, + }, + } + result = {"command": args.command, "operation": None, "args": {}} for attr_name, attr_value in vars(args).items(): From 9bff5da9187288c957a67e1259da761e78b9ba05 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 22 Apr 2026 14:10:53 +0800 Subject: [PATCH 036/196] refactor: simplify dc command arguments and refactor configuration management for DC synthesis --- bbdev/api/steps/dc/01_run_api.step.py | 27 +--- bbdev/api/steps/dc/01_run_event.step.py | 175 +++++++++++----------- bbdev/api/steps/dc/scripts/dc-config.yaml | 25 ++++ bbdev/api/steps/dc/scripts/run-dc.tcl | 42 ++++++ bbdev/bbdev | 29 +--- 5 files changed, 165 insertions(+), 133 deletions(-) create mode 100644 bbdev/api/steps/dc/scripts/dc-config.yaml create mode 100644 bbdev/api/steps/dc/scripts/run-dc.tcl diff --git a/bbdev/api/steps/dc/01_run_api.step.py b/bbdev/api/steps/dc/01_run_api.step.py index b81145caa..6e5ac46b0 100644 --- a/bbdev/api/steps/dc/01_run_api.step.py +++ b/bbdev/api/steps/dc/01_run_api.step.py @@ -3,37 +3,22 @@ from utils.path import get_buckyball_path config = { - "name": "dc-run-api", - "description": "run Design Compiler synthesis script", + "name": "dc-run", + "description": "run Design Compiler synthesis", "flows": ["dc"], "triggers": [api("POST", "/dc/run")], "enqueues": ["dc.run"], } -async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() - body = request.body or {} - - srcdir = body.get("srcdir") - if not srcdir: - return ApiResponse( - status=400, - body={ - "status": "error", - "message": "Missing required parameter: --srcdir", - "example": 'bbdev dc --srcdir arch/ReluBall_1 --top ReluBall', - }, - ) + body = req.body or {} data = { - "srcdir": srcdir, + "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), "top": body.get("top"), - "keep_hierarchy": bool(body.get("keep_hierarchy", False)), - "balltype": body.get("balltype"), - "config": body.get("config", "sims.verilator.BuckyballToyVerilatorConfig"), - "output_dir": body.get("output_dir"), - "report_dir": f"{bbdir}/bb-tests/output/dc/reports", + "config": body.get("config"), } await ctx.enqueue({"topic": "dc.run", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/dc/01_run_event.step.py b/bbdev/api/steps/dc/01_run_event.step.py index e245d8450..7643af6be 100644 --- a/bbdev/api/steps/dc/01_run_event.step.py +++ b/bbdev/api/steps/dc/01_run_event.step.py @@ -1,10 +1,10 @@ import os -import shlex +import shutil import sys +import yaml from motia import FlowContext, queue -# Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) @@ -15,118 +15,125 @@ config = { "name": "dc-run", - "description": "run Design Compiler synthesis script", + "description": "run Design Compiler synthesis", "flows": ["dc"], "triggers": [queue("dc.run")], "enqueues": [], } +def load_dc_config(): + config_path = os.path.join(os.path.dirname(__file__), "scripts", "dc-config.yaml") + if os.path.exists(config_path): + with open(config_path, "r") as f: + return yaml.safe_load(f) or {} + return {} + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - script_path = f"{bbdir}/evals/run-dc.sh" + arch_dir = f"{bbdir}/arch" + build_dir = input_data.get("output_dir") or f"{bbdir}/arch/build/" + + dc_cfg = load_dc_config() + top_module = input_data.get("top") or dc_cfg.get("top") or "BuckyballAccelerator" + elaborate_config = ( + input_data.get("config") + or dc_cfg.get("elaborate_config") + or "sims.verilator.BuckyballToyVerilatorConfig" + ) - srcdir = input_data.get("srcdir") - top = input_data.get("top") - keep_hierarchy = bool(input_data.get("keep_hierarchy", False)) - balltype = input_data.get("balltype") - config_name = input_data.get("config", "sims.verilator.BuckyballToyVerilatorConfig") - output_dir = input_data.get("output_dir") + ctx.logger.info(f"Top module: {top_module}, Elaborate config: {elaborate_config}") - if not srcdir: - success_result, failure_result = await check_result( - ctx, - 1, - continue_run=False, - extra_fields={ - "task": "dc", - "error": "Missing required parameter: srcdir", - "example": "bbdev dc --srcdir arch/ReluBall_1 --top ReluBall", - }, - trace_id=origin_tid, - ) - return failure_result + # Step 1: Generate Verilog via Elaborate + if os.path.exists(build_dir): + shutil.rmtree(build_dir) + os.makedirs(build_dir, exist_ok=True) - if not os.path.exists(script_path): - success_result, failure_result = await check_result( + ctx.logger.info("Step 1: Generating Verilog via Elaborate...") + verilog_command = ( + f"mill -i __.test.runMain sims.verilator.Elaborate {elaborate_config} " + "--disable-annotation-unknown -strip-debug-info -O=debug " + "-lowering-options=disallowLocalVariables " + f"--split-verilog -o={build_dir}" + ) + + result = stream_run_logger( + cmd=verilog_command, + logger=ctx.logger, + cwd=arch_dir, + stdout_prefix="dc verilog", + stderr_prefix="dc verilog", + ) + + if result.returncode != 0: + _, failure_result = await check_result( ctx, - 1, + result.returncode, continue_run=False, - extra_fields={ - "task": "dc", - "error": f"DC script not found: {script_path}", - }, + extra_fields={"task": "verilog"}, trace_id=origin_tid, ) return failure_result - # Keep compatibility with old manual flow: ensure run-dc.sh is executable. - os.chmod(script_path, os.stat(script_path).st_mode | 0o111) - - # Optional integrated pre-step: generate Ball verilog first. - if balltype: - final_output_dir = output_dir or f"{balltype}_1" - verilog_dir = f"{bbdir}/arch/{final_output_dir}" - verilog_cmd = ( - f"mill -i __.test.runMain sims.verify.BallTopMain {shlex.quote(str(balltype))} " - "--disable-annotation-unknown --strip-debug-info -O=debug " - f"--split-verilog -o={shlex.quote(verilog_dir)}" - ) - - verilog_result = stream_run_logger( - cmd=verilog_cmd, - logger=ctx.logger, - cwd=f"{bbdir}/arch", - stdout_prefix="dc verilog", - stderr_prefix="dc verilog", - ) - - if verilog_result.returncode != 0: - success_result, failure_result = await check_result( - ctx, - verilog_result.returncode, - continue_run=False, - extra_fields={ - "task": "dc", - "error": "Failed to generate pre-DC verilog", - "balltype": balltype, - "config": config_name, - }, - trace_id=origin_tid, - ) - return failure_result - - # If user keeps default-like srcdir, force it to generated dir for consistency. - srcdir = f"arch/{final_output_dir}" - - command_parts = ["bash", shlex.quote(script_path), "--srcdir", shlex.quote(str(srcdir))] - if top: - command_parts.extend(["--top", shlex.quote(str(top))]) - if keep_hierarchy: - command_parts.append("--keep-hierarchy") - - command = " ".join(command_parts) + # Step 2: Run Design Compiler + ctx.logger.info("Step 2: Running Design Compiler synthesis...") + work_dir = f"{bbdir}/bb-tests/output/dc" + design_dir = f"{work_dir}/design" + report_dir = f"{work_dir}/reports" + tmp_dir = f"{work_dir}/tmp" + + for d in (design_dir, report_dir, tmp_dir): + if os.path.exists(d): + shutil.rmtree(d) + os.makedirs(d, exist_ok=True) + shutil.copytree(build_dir, design_dir, dirs_exist_ok=True) + + tcl_script = os.path.join(os.path.dirname(__file__), "scripts", "run-dc.tcl") + target_libs = " ".join(dc_cfg.get("target_libraries") or []) + setup = ( + f'set design_dir "{design_dir}"; ' + f'set top_module "{top_module}"; ' + f'set work_dir "{work_dir}"; ' + f'set report_dir "{report_dir}"; ' + f'set tmp_dir "{tmp_dir}"; ' + f'set target_libs "{target_libs}"; ' + f'set clock_name "{dc_cfg.get("clock_name", "clock")}"; ' + f'set clock_period {dc_cfg.get("clock_period", 2.0)}; ' + f'set clock_uncertainty {dc_cfg.get("clock_uncertainty", 0.6)}; ' + f'set clock_transition {dc_cfg.get("clock_transition", 0.08)}; ' + f'set input_delay {dc_cfg.get("input_delay", 1.2)}; ' + f'set output_delay {dc_cfg.get("output_delay", 0.6)}; ' + f'set input_transition {dc_cfg.get("input_transition", 0.2)}; ' + f'set output_load {dc_cfg.get("output_load", 0.08)};' + ) + command = f'dc_shell -x \'{setup}\' -f {tcl_script}' result = stream_run_logger( cmd=command, logger=ctx.logger, cwd=bbdir, - executable="bash", stdout_prefix="dc run", stderr_prefix="dc run", ) - success_result, failure_result = await check_result( + alib = os.path.join(bbdir, "alib-52") + if os.path.exists(alib): + shutil.rmtree(alib) + + extra = {"task": "dc", "report_dir": report_dir, "top": top_module} + for name in ("area.rpt", "hierarchy.rpt", "timing.rpt", "power.rpt"): + path = os.path.join(report_dir, name) + if os.path.exists(path): + with open(path, "r") as f: + extra[name] = f.read() + + await check_result( ctx, result.returncode, continue_run=False, - extra_fields={ - "task": "dc", - "report_dir": input_data.get("report_dir", f"{bbdir}/bb-tests/output/dc/reports"), - "srcdir": srcdir, - "top": top, - }, + extra_fields=extra, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/dc/scripts/dc-config.yaml b/bbdev/api/steps/dc/scripts/dc-config.yaml new file mode 100644 index 000000000..aff0087f9 --- /dev/null +++ b/bbdev/api/steps/dc/scripts/dc-config.yaml @@ -0,0 +1,25 @@ +# Design Compiler synthesis configuration + +# Default top module name +top: BuckyballAccelerator + +# Elaborate config class for Chisel verilog generation +elaborate_config: sims.verilator.BuckyballToyVerilatorConfig + +# Standard cell .db libraries (target_library = link_library) +target_libraries: + - /data0/tools/lib/db/scc28nhkcp_hdc35p140_rvt_ffg_v0p99_0c_basic.db + - /data0/tools/lib/db/scc28nhkcp_hdc35p140_rvt_ffg_v0p99_0c_ccs.db + - /data0/tools/lib/db/scc28nhkcp_hdc35p140_rvt_ffg_v0p99_0c_ecsm.db + +# Clock constraints (ns) +clock_name: clock +clock_period: 2.0 +clock_uncertainty: 0.6 +clock_transition: 0.08 + +# IO timing constraints (ns) +input_delay: 1.2 +output_delay: 0.6 +input_transition: 0.2 +output_load: 0.08 diff --git a/bbdev/api/steps/dc/scripts/run-dc.tcl b/bbdev/api/steps/dc/scripts/run-dc.tcl new file mode 100644 index 000000000..f82625a2a --- /dev/null +++ b/bbdev/api/steps/dc/scripts/run-dc.tcl @@ -0,0 +1,42 @@ +# DC synthesis script. +# Inputs (set via `dc_shell -x` before sourcing this file): +# design_dir, top_module, work_dir, report_dir, tmp_dir +# target_libs : space-separated list of .db files +# clock_name, clock_period, clock_uncertainty, clock_transition +# input_delay, output_delay, input_transition, output_load + +set search_path [list . $design_dir] +define_design_lib work -path $tmp_dir + +set target_library $target_libs +set link_library $target_libs + +set file_list [glob -nocomplain -directory $design_dir *.sv] +analyze -format sverilog $file_list +elaborate $top_module +current_design $top_module +link + +# Keep module boundaries for hierarchical reports. +set_ungroup [get_designs *] false +set_boundary_optimization [get_designs *] false + +create_clock -name $clock_name -period $clock_period [get_ports $clock_name] +set_clock_uncertainty $clock_uncertainty [get_clocks $clock_name] +set_input_delay $input_delay -clock $clock_name [remove_from_collection [all_inputs] [get_ports $clock_name]] +set_output_delay $output_delay -clock $clock_name [all_outputs] +set_clock_equivalence $clock_name +set_load $output_load [all_outputs] +set_input_transition $input_transition [remove_from_collection [all_inputs] [get_ports $clock_name]] +set_clock_transition $clock_transition [get_clocks $clock_name] + +compile_ultra -retime -scan -no_autoungroup +write -format ddc -hierarchy -output $report_dir/design_compiled.ddc + +report_area -hierarchy -nosplit > $report_dir/area.rpt +report_hierarchy -noleaf > $report_dir/hierarchy.rpt +report_timing > $report_dir/timing.rpt +report_power -hierarchy > $report_dir/power.rpt + +write -format verilog -output $report_dir/netlist.v +exit diff --git a/bbdev/bbdev b/bbdev/bbdev index 03790fd45..85ca82ff0 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -78,7 +78,7 @@ WORKFLOW_COMMANDS = { "runworkload": 'Run workload on AU280. Args: "[--workload ] [--board ] [--timeout ] [--uart ]"', }, "dc": { - "run": 'Run Design Compiler synthesis. Args: "--srcdir [--top ] [--keep-hierarchy] [--balltype ] [--config ] [--output-dir ]"', + "run": 'Run Design Compiler synthesis. Args: "[--top ] [--config ] [--output-dir ]"', }, } @@ -172,21 +172,8 @@ def parse_args(argv: list[str]) -> argparse.Namespace: stop_parser = subparsers.add_parser("stop", parents=[common], help="Stop dev server") stop_parser.add_argument("--all", action="store_true", help="Stop all servers") - # dc direct mode (no --run wrapper): - # bbdev dc --srcdir ... [--top ...] [--keep-hierarchy] - # [--balltype ...] [--config ...] [--output-dir ...] - dc_parser = subparsers.add_parser("dc", parents=[common], help="Run DC synthesis") - dc_parser.add_argument("--srcdir", type=str, required=True, help="Design source directory relative to repo root") - dc_parser.add_argument("--top", type=str, default=None, help="Top module name") - dc_parser.add_argument("--keep-hierarchy", action="store_true", help="Keep hierarchy during compile") - dc_parser.add_argument("--balltype", type=str, default=None, help="Optional Ball type for pre-verilog generation, e.g. ReluBall") - dc_parser.add_argument("--config", type=str, default="sims.verilator.BuckyballToyVerilatorConfig", help="Verilog generation config when --balltype is set") - dc_parser.add_argument("--output-dir", type=str, default=None, help="Verilog output dir under arch/, e.g. ReluBall_1") - # Build workflow subcommands from WORKFLOW_COMMANDS table for cmd_name, operations in WORKFLOW_COMMANDS.items(): - if cmd_name == "dc": - continue cmd_parser = subparsers.add_parser( cmd_name, parents=[common], help=f"{cmd_name} operations" ) @@ -209,20 +196,6 @@ def parse_args(argv: list[str]) -> argparse.Namespace: def extract_command_info(args: argparse.Namespace) -> dict: """Extract command, operation, and parsed sub-arguments from argparse Namespace.""" - if args.command == "dc": - return { - "command": "dc", - "operation": "run", - "args": { - "srcdir": args.srcdir, - "top": args.top, - "keep_hierarchy": bool(args.keep_hierarchy), - "balltype": args.balltype, - "config": args.config, - "output_dir": args.output_dir, - }, - } - result = {"command": args.command, "operation": None, "args": {}} for attr_name, attr_value in vars(args).items(): From 5df58c010c5da548cef39ddee510acf2638a73e4 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 22 Apr 2026 17:59:56 +0800 Subject: [PATCH 037/196] del: remove palladium verilog API and related event handling from bbdev --- .../steps/palladium/01_verilog_api.step.py | 36 ---- .../steps/palladium/01_verilog_event.step.py | 158 ------------------ bbdev/bbdev | 3 - 3 files changed, 197 deletions(-) delete mode 100644 bbdev/api/steps/palladium/01_verilog_api.step.py delete mode 100644 bbdev/api/steps/palladium/01_verilog_event.step.py diff --git a/bbdev/api/steps/palladium/01_verilog_api.step.py b/bbdev/api/steps/palladium/01_verilog_api.step.py deleted file mode 100644 index 76c250033..000000000 --- a/bbdev/api/steps/palladium/01_verilog_api.step.py +++ /dev/null @@ -1,36 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - -from utils.path import get_buckyball_path - -config = { - "name": "palladium-verilog-api", - "description": "generate verilog code", - "flows": ["palladium"], - "triggers": [api("POST", "/palladium/verilog")], - "enqueues": ["palladium.verilog"], -} - - -async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() - body = request.body or {} - - # Get config name, must be provided - config_name = body.get("config") - if not config_name or config_name == "None": - return ApiResponse( - status=400, - body={ - "status": "error", - "message": "Configuration name is required. Please specify --config_name parameter.", - "example": './bbdev palladium --verilog "--config_name sims.palladium.BuckyballToyP2EConfig"', - }, - ) - - data = { - "config": config_name, - "balltype": body.get("balltype"), - "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), - } - await ctx.enqueue({"topic": "palladium.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/palladium/01_verilog_event.step.py b/bbdev/api/steps/palladium/01_verilog_event.step.py deleted file mode 100644 index a8dfa4eb5..000000000 --- a/bbdev/api/steps/palladium/01_verilog_event.step.py +++ /dev/null @@ -1,158 +0,0 @@ -import os -import sys - -from motia import FlowContext, queue - -# Add the utils directory to the Python path -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from utils.event_common import check_result, get_origin_trace_id - -config = { - "name": "palladium-verilog", - "description": "generate verilog code", - "flows": ["palladium"], - "triggers": [queue("palladium.verilog")], - "enqueues": ["palladium.build"], -} - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - bbdir = get_buckyball_path() - # Use arch/build as the base directory for chipyard.Generator - base_build_dir = f"{input_data.get('output_dir', f'{bbdir}/arch/build')}/palladium" - # Output directory for final Verilog files - verilog_output_dir = f"{base_build_dir}/verilog" - arch_dir = f"{bbdir}/arch" - - # Get config name, must be provided - config_name = input_data.get("config") - if not config_name or config_name == "None": - ctx.logger.error("Configuration name is required but not provided") - success_result, failure_result = await check_result( - ctx, - 1, - continue_run=False, - extra_fields={ - "task": "validation", - "error": "Configuration name is required. Please specify --config_name parameter.", - "example": './bbdev palladium --verilog "--config_name sims.palladium.BuckyballToyP2EConfig"', - }, trace_id=origin_tid) - return failure_result - - ctx.logger.info(f"Using configuration: {config_name}") - - # ================================================================================== - # Step 1: Generate FIRRTL using chipyard.Generator - # ================================================================================== - ctx.logger.info("Step 1: Generating FIRRTL with chipyard.Generator...") - os.system(f"mkdir -p {verilog_output_dir}") - firrtl_command = ( - f"cd {arch_dir} && " - f"sbt -J-Xmx256G -J-Xss64M -J-XX:+UseG1GC -J-XX:MaxGCPauseMillis=1000 " - f'"runMain chipyard.Generator ' - f"-td {base_build_dir} " - f"-T palladium.fpga.VCU118FPGATestHarness " - f'-C {config_name}"' - ) - - result = stream_run_logger( - cmd=firrtl_command, - logger=ctx.logger, - cwd=arch_dir, - stdout_prefix="palladium firrtl", - stderr_prefix="palladium firrtl", - ) - - if result.returncode != 0: - ctx.logger.error(f"FIRRTL generation failed with code {result.returncode}") - success_result, failure_result = await check_result( - ctx, - result.returncode, - continue_run=False, - extra_fields={"task": "firrtl", "step": "generate"}, trace_id=origin_tid) - return failure_result - - # ================================================================================== - # Step 2: Convert FIRRTL to SystemVerilog using firtool - # ================================================================================== - ctx.logger.info("Step 2: Converting FIRRTL to SystemVerilog with firtool...") - - # Extract the simple class name from the full config name - # e.g., "sims.palladium.BuckyballToyP2EConfig" -> "BuckyballToyP2EConfig" - config_class_name = config_name.split(".")[-1] - - # Find the generated FIRRTL file (in base_build_dir, not verilog_output_dir) - fir_file = f"{base_build_dir}/palladium.fpga.{config_class_name}.fir" - if not os.path.exists(fir_file): - ctx.logger.error(f"FIRRTL file not found: {fir_file}") - ctx.logger.info(f"Looking for files in {base_build_dir}...") - # List files to help debug - if os.path.exists(base_build_dir): - files = os.listdir(base_build_dir) - ctx.logger.info(f"Files in build dir: {files}") - success_result, failure_result = await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"task": "firrtl", "step": "file_check"}, trace_id=origin_tid) - return failure_result - - verilog_command = ( - f"firtool {fir_file} " - f"-o {verilog_output_dir} " - f"--split-verilog " - f"--disable-all-randomization " - f"--strip-debug-info " - f"--ignore-read-enable-mem " - f"--lowering-options=disallowLocalVariables " - f"--disable-annotation-unknown" - ) - - result = stream_run_logger( - cmd=verilog_command, - logger=ctx.logger, - cwd=arch_dir, - stdout_prefix="palladium verilog", - stderr_prefix="palladium verilog", - ) - - if result.returncode != 0: - ctx.logger.error(f"Verilog generation failed with code {result.returncode}") - success_result, failure_result = await check_result( - ctx, - result.returncode, - continue_run=False, - extra_fields={"task": "verilog", "step": "firtool"}, trace_id=origin_tid) - return failure_result - - # ================================================================================== - # Return result to API - # ================================================================================== - success_result, failure_result = await check_result( - ctx, - result.returncode, - continue_run=input_data.get("from_run_workflow", False), - extra_fields={ - "task": "verilog", - "output_dir": verilog_output_dir, - "top_module": "VCU118FPGATestHarness", - }, trace_id=origin_tid, - ) - - # ================================================================================== - # Continue routing - # Routing to verilog or finish workflow - # For run workflow, continue to verilog; for standalone clean, complete - # ================================================================================== - if input_data.get("from_run_workflow"): - await ctx.enqueue( - {"topic": "palladium.build", "data": {**input_data, "task": "run"}} - ) - - return diff --git a/bbdev/bbdev b/bbdev/bbdev index 85ca82ff0..c733f711a 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -68,9 +68,6 @@ WORKFLOW_COMMANDS = { "builddut": "Build dut.", "build": "Build uvm.", }, - "palladium": { - "verilog": 'Generate verilog files. Args: "[--config sims.palladium.BuckyballToyP2EConfig]"', - }, "pegasus": { "verilog": 'Generate SystemVerilog for Pegasus FPGA. Args: "[--config sims.pegasus.PegasusConfig]"', "buildbitstream": "Build Vivado bitstream for AU280.", From 2af83325c81054b96d23ba95af9509432201d03b Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 22 Apr 2026 22:12:11 +0800 Subject: [PATCH 038/196] fix: add yosys workflow and commands to bbdev, including run, verilog generation, and synthesis steps feat: add tests to test api avaliable and provide a example for new developer --- bbdev/api/steps/dc/01_run_api.step.py | 1 + bbdev/api/steps/dc/02_verilog_api.step.py | 24 ++++ bbdev/api/steps/dc/03_synth_api.step.py | 24 ++++ ...n_event.step.py => 03_synth_event.step.py} | 47 +------ .../01_verilator_verilog_event.step.py} | 23 ---- .../02_pegasus_verilog_event.step.py} | 14 +- .../steps/mill/03_yosys_verilog_event.step.py | 90 ++++++++++++ .../steps/mill/04_dc_verilog_event.step.py | 87 ++++++++++++ .../api/steps/verilator/03_build_api.step.py | 1 - .../steps/verilator/03_build_event.step.py | 21 +-- .../api/steps/verilator/04_cosim_api.step.py | 27 ---- .../steps/verilator/04_cosim_event.step.py | 128 ------------------ bbdev/api/steps/verilator/05_run_api.step.py | 1 - bbdev/api/steps/yosys/01_run_api.step.py | 25 ++++ bbdev/api/steps/yosys/02_verilog_api.step.py | 24 ++++ ...synth_api.step.py => 03_synth_api.step.py} | 2 +- ...h_event.step.py => 03_synth_event.step.py} | 108 +-------------- bbdev/api/tests/_api_test_helper.py | 7 + bbdev/api/tests/test_all_api_serial.py | 45 ++++++ bbdev/api/tests/test_api_compiler_build.py | 3 + bbdev/api/tests/test_api_dc_run.py | 3 + bbdev/api/tests/test_api_dc_synth.py | 3 + bbdev/api/tests/test_api_dc_verilog.py | 3 + .../tests/test_api_firesim_buildbitstream.py | 3 + .../api/tests/test_api_firesim_infrasetup.py | 3 + .../api/tests/test_api_firesim_runworkload.py | 3 + bbdev/api/tests/test_api_kernel_build.py | 3 + .../tests/test_api_pegasus_buildbitstream.py | 3 + .../tests/test_api_pegasus_flashbitstream.py | 3 + .../api/tests/test_api_pegasus_runworkload.py | 3 + bbdev/api/tests/test_api_pegasus_verilog.py | 3 + bbdev/api/tests/test_api_sardine_run.py | 3 + bbdev/api/tests/test_api_verilator_build.py | 3 + bbdev/api/tests/test_api_verilator_clean.py | 3 + bbdev/api/tests/test_api_verilator_run.py | 3 + bbdev/api/tests/test_api_verilator_sim.py | 3 + bbdev/api/tests/test_api_verilator_verilog.py | 3 + bbdev/api/tests/test_api_workload_build.py | 3 + bbdev/api/tests/test_api_yosys_run.py | 3 + bbdev/api/tests/test_api_yosys_synth.py | 3 + bbdev/api/tests/test_api_yosys_verilog.py | 3 + bbdev/bbdev | 5 + 42 files changed, 415 insertions(+), 355 deletions(-) create mode 100644 bbdev/api/steps/dc/02_verilog_api.step.py create mode 100644 bbdev/api/steps/dc/03_synth_api.step.py rename bbdev/api/steps/dc/{01_run_event.step.py => 03_synth_event.step.py} (69%) rename bbdev/api/steps/{verilator/02_verilog_event.step.py => mill/01_verilator_verilog_event.step.py} (66%) rename bbdev/api/steps/{pegasus/01_verilog_event.step.py => mill/02_pegasus_verilog_event.step.py} (85%) create mode 100644 bbdev/api/steps/mill/03_yosys_verilog_event.step.py create mode 100644 bbdev/api/steps/mill/04_dc_verilog_event.step.py delete mode 100644 bbdev/api/steps/verilator/04_cosim_api.step.py delete mode 100644 bbdev/api/steps/verilator/04_cosim_event.step.py create mode 100644 bbdev/api/steps/yosys/01_run_api.step.py create mode 100644 bbdev/api/steps/yosys/02_verilog_api.step.py rename bbdev/api/steps/yosys/{01_synth_api.step.py => 03_synth_api.step.py} (95%) rename bbdev/api/steps/yosys/{01_synth_event.step.py => 03_synth_event.step.py} (51%) create mode 100644 bbdev/api/tests/_api_test_helper.py create mode 100755 bbdev/api/tests/test_all_api_serial.py create mode 100644 bbdev/api/tests/test_api_compiler_build.py create mode 100644 bbdev/api/tests/test_api_dc_run.py create mode 100644 bbdev/api/tests/test_api_dc_synth.py create mode 100644 bbdev/api/tests/test_api_dc_verilog.py create mode 100644 bbdev/api/tests/test_api_firesim_buildbitstream.py create mode 100644 bbdev/api/tests/test_api_firesim_infrasetup.py create mode 100644 bbdev/api/tests/test_api_firesim_runworkload.py create mode 100644 bbdev/api/tests/test_api_kernel_build.py create mode 100644 bbdev/api/tests/test_api_pegasus_buildbitstream.py create mode 100644 bbdev/api/tests/test_api_pegasus_flashbitstream.py create mode 100644 bbdev/api/tests/test_api_pegasus_runworkload.py create mode 100644 bbdev/api/tests/test_api_pegasus_verilog.py create mode 100644 bbdev/api/tests/test_api_sardine_run.py create mode 100644 bbdev/api/tests/test_api_verilator_build.py create mode 100644 bbdev/api/tests/test_api_verilator_clean.py create mode 100644 bbdev/api/tests/test_api_verilator_run.py create mode 100644 bbdev/api/tests/test_api_verilator_sim.py create mode 100644 bbdev/api/tests/test_api_verilator_verilog.py create mode 100644 bbdev/api/tests/test_api_workload_build.py create mode 100644 bbdev/api/tests/test_api_yosys_run.py create mode 100644 bbdev/api/tests/test_api_yosys_synth.py create mode 100644 bbdev/api/tests/test_api_yosys_verilog.py diff --git a/bbdev/api/steps/dc/01_run_api.step.py b/bbdev/api/steps/dc/01_run_api.step.py index 6e5ac46b0..9d22ca951 100644 --- a/bbdev/api/steps/dc/01_run_api.step.py +++ b/bbdev/api/steps/dc/01_run_api.step.py @@ -19,6 +19,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), "top": body.get("top"), "config": body.get("config"), + "from_run_workflow": True, } await ctx.enqueue({"topic": "dc.run", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/dc/02_verilog_api.step.py b/bbdev/api/steps/dc/02_verilog_api.step.py new file mode 100644 index 000000000..2fb798556 --- /dev/null +++ b/bbdev/api/steps/dc/02_verilog_api.step.py @@ -0,0 +1,24 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path + +config = { + "name": "dc-verilog-api", + "description": "generate verilog for dc flow", + "flows": ["dc"], + "triggers": [api("POST", "/dc/verilog")], + "enqueues": ["dc.verilog"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = req.body or {} + + data = { + "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), + "top": body.get("top"), + "config": body.get("config"), + } + await ctx.enqueue({"topic": "dc.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/dc/03_synth_api.step.py b/bbdev/api/steps/dc/03_synth_api.step.py new file mode 100644 index 000000000..7081a0ea4 --- /dev/null +++ b/bbdev/api/steps/dc/03_synth_api.step.py @@ -0,0 +1,24 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path + +config = { + "name": "dc-synth-api", + "description": "run Design Compiler synthesis", + "flows": ["dc"], + "triggers": [api("POST", "/dc/synth")], + "enqueues": ["dc.synth"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = req.body or {} + + data = { + "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), + "top": body.get("top"), + "config": body.get("config"), + } + await ctx.enqueue({"topic": "dc.synth", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/dc/01_run_event.step.py b/bbdev/api/steps/dc/03_synth_event.step.py similarity index 69% rename from bbdev/api/steps/dc/01_run_event.step.py rename to bbdev/api/steps/dc/03_synth_event.step.py index 7643af6be..3c51a07c4 100644 --- a/bbdev/api/steps/dc/01_run_event.step.py +++ b/bbdev/api/steps/dc/03_synth_event.step.py @@ -1,8 +1,8 @@ import os import shutil import sys - import yaml + from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) @@ -14,10 +14,10 @@ from utils.stream_run import stream_run_logger config = { - "name": "dc-run", + "name": "dc-synth", "description": "run Design Compiler synthesis", "flows": ["dc"], - "triggers": [queue("dc.run")], + "triggers": [queue("dc.synth")], "enqueues": [], } @@ -33,52 +33,11 @@ def load_dc_config(): async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - arch_dir = f"{bbdir}/arch" build_dir = input_data.get("output_dir") or f"{bbdir}/arch/build/" dc_cfg = load_dc_config() top_module = input_data.get("top") or dc_cfg.get("top") or "BuckyballAccelerator" - elaborate_config = ( - input_data.get("config") - or dc_cfg.get("elaborate_config") - or "sims.verilator.BuckyballToyVerilatorConfig" - ) - - ctx.logger.info(f"Top module: {top_module}, Elaborate config: {elaborate_config}") - - # Step 1: Generate Verilog via Elaborate - if os.path.exists(build_dir): - shutil.rmtree(build_dir) - os.makedirs(build_dir, exist_ok=True) - - ctx.logger.info("Step 1: Generating Verilog via Elaborate...") - verilog_command = ( - f"mill -i __.test.runMain sims.verilator.Elaborate {elaborate_config} " - "--disable-annotation-unknown -strip-debug-info -O=debug " - "-lowering-options=disallowLocalVariables " - f"--split-verilog -o={build_dir}" - ) - - result = stream_run_logger( - cmd=verilog_command, - logger=ctx.logger, - cwd=arch_dir, - stdout_prefix="dc verilog", - stderr_prefix="dc verilog", - ) - if result.returncode != 0: - _, failure_result = await check_result( - ctx, - result.returncode, - continue_run=False, - extra_fields={"task": "verilog"}, - trace_id=origin_tid, - ) - return failure_result - - # Step 2: Run Design Compiler - ctx.logger.info("Step 2: Running Design Compiler synthesis...") work_dir = f"{bbdir}/bb-tests/output/dc" design_dir = f"{work_dir}/design" report_dir = f"{work_dir}/reports" diff --git a/bbdev/api/steps/verilator/02_verilog_event.step.py b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py similarity index 66% rename from bbdev/api/steps/verilator/02_verilog_event.step.py rename to bbdev/api/steps/mill/01_verilator_verilog_event.step.py index a6439468a..c5934f8e2 100644 --- a/bbdev/api/steps/verilator/02_verilog_event.step.py +++ b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py @@ -4,7 +4,6 @@ from motia import FlowContext, queue -# Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) @@ -46,9 +45,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"Using configuration: {config_name}") - # ================================================================================== - # Execute operation - # ================================================================================== if input_data.get("balltype"): command = ( f"mill -i __.test.runMain sims.verify.BallTopMain {input_data.get('balltype')} " @@ -56,18 +52,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: else: command = f"mill -i __.test.runMain sims.verilator.Elaborate {config_name} " - # Firtool options (CIRCT). Current set; optional Chipyard-style options below. command += "--disable-annotation-unknown " command += "--strip-debug-info " command += "-O=debug " - # command += f"-repl-seq-mem -repl-seq-mem-file={build_dir}/mem.conf " command += f"--split-verilog -o={build_dir} " - # Optional: --disable-annotation-classless (ignore classless annotations) - # Optional: -repl-seq-mem -repl-seq-mem-file=.conf (SRAM macro replacement) - # Optional: --disable-all-randomization (disable mem/reg init; may break semantics) - # Optional: --disable-opt (no optimization) or -O=release (default is release) - # Optional: --output-annotation-file= (emit annotations after lower-to-hw) - # Optional: --no-dedup (disable module dedup); --strip-fir-debug-info (FIR locators) result = stream_run_logger( cmd=command, @@ -77,9 +65,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="verilator verilog", ) - # Remove testchipip C++ sources that depend on fesvr (which we don't have). - # SimTSI.v is kept so verilator can resolve the SimTSI module reference in BBSimHarness.sv; - # tsi_tick DPI symbol is satisfied by arch/src/csrc/src/monitor/ioe/tsi_stub.cc instead. for unwanted in [ f"{build_dir}/testchip_htif.cc", f"{build_dir}/testchip_htif.h", @@ -91,8 +76,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if os.path.exists(unwanted): os.remove(unwanted) - # Patch fesvr includes out of mm.h and mm.cc (copied from testchipip resources). - # They reference fesvr/memif.h which we don't have — our SimDRAM_bb.cc doesn't use it. for patch_file in [f"{build_dir}/mm.h", f"{build_dir}/mm.cc"]: if os.path.exists(patch_file): with open(patch_file, "r") as f: @@ -106,9 +89,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f.write(patched) ctx.logger.info(f"Patched fesvr includes from {patch_file}") - # ================================================================================== - # Return result to API - # ================================================================================== success_result, failure_result = await check_result( ctx, result.returncode, @@ -117,9 +97,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) - # ================================================================================== - # Continue routing - # ================================================================================== if input_data.get("from_run_workflow"): await ctx.enqueue( {"topic": "verilator.build", "data": {**input_data, "task": "run"}} diff --git a/bbdev/api/steps/pegasus/01_verilog_event.step.py b/bbdev/api/steps/mill/02_pegasus_verilog_event.step.py similarity index 85% rename from bbdev/api/steps/pegasus/01_verilog_event.step.py rename to bbdev/api/steps/mill/02_pegasus_verilog_event.step.py index 8820c0f3b..e00c089c6 100644 --- a/bbdev/api/steps/pegasus/01_verilog_event.step.py +++ b/bbdev/api/steps/mill/02_pegasus_verilog_event.step.py @@ -27,7 +27,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/pegasus/") build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/pegasus_top/") - soc_dir = f"{bbdir}/arch/build/pegasus/" # DigitalTop + all SoC RTL + soc_dir = f"{bbdir}/arch/build/pegasus/" ctx.logger.info(f" Elaborating PegasusTop") ctx.logger.info(f" Top output directory: {build_dir}") @@ -35,9 +35,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: os.makedirs(build_dir, exist_ok=True) - # Step 1: Elaborate the full SoC (DigitalTop + all sub-modules) - # This is the same as before, using ElaboratePegasus which runs PegasusHarness - # but we only need its generated RTL files (DigitalTop.sv, BBTile.sv, etc.) soc_command = ( f"mill -i __.test.runMain sims.pegasus.ElaboratePegasus " f"--disable-annotation-unknown " @@ -58,7 +55,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={"task": "verilog", "step": "soc"}, trace_id=origin_tid) return - # Step 2: Elaborate PegasusTop + PegasusShell (the FPGA top-level wrapper) command = ( f"mill -i __.test.runMain sims.pegasus.ElaboratePegasusTop " f"--disable-annotation-unknown " @@ -75,17 +71,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="pegasus verilog", ) - # Clean up stray top-level files if emitted next to arch/ for stray in ["PegasusTop.v", "PegasusTopWrapper.sv", "PegasusShell.v"]: stray_path = f"{arch_dir}/{stray}" if os.path.exists(stray_path): os.remove(stray_path) - # Copy generated SV/V files to pegasus/vivado/generated/ for Vivado build. - # Sources: - # soc_dir -> DigitalTop + all SoC sub-modules (DPI files stubbed out) - # build_dir -> PegasusTop.v, PegasusShell.v, PegasusTopWrapper.sv (top-level wrappers) - # PegasusHarness.sv, ChipTop.sv and harness-layer files from soc_dir are skipped. vivado_gen_dir = f"{bbdir}/thirdparty/pegasus/vivado/generated" if result.returncode == 0 and os.path.isdir(build_dir): import re @@ -140,9 +130,7 @@ def copy_rtl_dir(src_dir: str, skip_set: set = None) -> int: copied += 1 return copied - # Copy SoC RTL (skip harness-layer files) n_soc = copy_rtl_dir(soc_dir, skip_set=HARNESS_SKIP) - # Copy top-level wrappers (overrides any same-named file from soc_dir) n_top = copy_rtl_dir(build_dir) ctx.logger.info(f" Copied {n_soc} SoC files + {n_top} top files to {vivado_gen_dir}") diff --git a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py new file mode 100644 index 000000000..af621bbc5 --- /dev/null +++ b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py @@ -0,0 +1,90 @@ +import os +import shutil +import sys +import yaml + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "yosys verilog", + "description": "generate verilog for yosys flow", + "flows": ["yosys"], + "triggers": [queue("yosys.run"), queue("yosys.verilog")], + "enqueues": ["yosys.synth"], +} + + +def load_yosys_config(): + bbdir = get_buckyball_path() + config_path = f"{bbdir}/bbdev/api/steps/yosys/scripts/yosys-config.yaml" + if os.path.exists(config_path): + with open(config_path, "r") as f: + return yaml.safe_load(f) or {} + return {} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/") + arch_dir = f"{bbdir}/arch" + + yosys_cfg = load_yosys_config() + elaborate_config = input_data.get("config") or yosys_cfg.get( + "elaborate_config", "sims.verilator.BuckyballToyVerilatorConfig" + ) + + if os.path.exists(build_dir): + shutil.rmtree(build_dir) + os.makedirs(build_dir, exist_ok=True) + + verilog_command = ( + f"mill -i __.test.runMain sims.verilator.Elaborate {elaborate_config} " + "--disable-annotation-unknown -strip-debug-info -O=debug " + "-lowering-options=disallowLocalVariables " + f"--split-verilog -o={build_dir}" + ) + + result = stream_run_logger( + cmd=verilog_command, + logger=ctx.logger, + cwd=arch_dir, + stdout_prefix="yosys verilog", + stderr_prefix="yosys verilog", + ) + + if result.returncode != 0: + success_result, failure_result = await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={"task": "verilog"}, + trace_id=origin_tid, + ) + return failure_result + + for unwanted in ["TestHarness.sv", "TargetBall.sv"]: + topname_file = f"{arch_dir}/{unwanted}" + if os.path.exists(topname_file): + os.remove(topname_file) + + await check_result( + ctx, + result.returncode, + continue_run=input_data.get("from_run_workflow", False), + extra_fields={"task": "verilog"}, + trace_id=origin_tid, + ) + + if input_data.get("from_run_workflow"): + await ctx.enqueue({"topic": "yosys.synth", "data": {**input_data, "task": "run"}}) + + return diff --git a/bbdev/api/steps/mill/04_dc_verilog_event.step.py b/bbdev/api/steps/mill/04_dc_verilog_event.step.py new file mode 100644 index 000000000..4656bc12b --- /dev/null +++ b/bbdev/api/steps/mill/04_dc_verilog_event.step.py @@ -0,0 +1,87 @@ +import os +import shutil +import sys +import yaml + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger + +config = { + "name": "dc-verilog", + "description": "generate verilog for dc flow", + "flows": ["dc"], + "triggers": [queue("dc.run"), queue("dc.verilog")], + "enqueues": ["dc.synth"], +} + + +def load_dc_config(): + bbdir = get_buckyball_path() + config_path = f"{bbdir}/bbdev/api/steps/dc/scripts/dc-config.yaml" + if os.path.exists(config_path): + with open(config_path, "r") as f: + return yaml.safe_load(f) or {} + return {} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + arch_dir = f"{bbdir}/arch" + build_dir = input_data.get("output_dir") or f"{bbdir}/arch/build/" + + dc_cfg = load_dc_config() + elaborate_config = ( + input_data.get("config") + or dc_cfg.get("elaborate_config") + or "sims.verilator.BuckyballToyVerilatorConfig" + ) + + if os.path.exists(build_dir): + shutil.rmtree(build_dir) + os.makedirs(build_dir, exist_ok=True) + + verilog_command = ( + f"mill -i __.test.runMain sims.verilator.Elaborate {elaborate_config} " + "--disable-annotation-unknown -strip-debug-info -O=debug " + "-lowering-options=disallowLocalVariables " + f"--split-verilog -o={build_dir}" + ) + + result = stream_run_logger( + cmd=verilog_command, + logger=ctx.logger, + cwd=arch_dir, + stdout_prefix="dc verilog", + stderr_prefix="dc verilog", + ) + + if result.returncode != 0: + _, failure_result = await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={"task": "verilog"}, + trace_id=origin_tid, + ) + return failure_result + + await check_result( + ctx, + result.returncode, + continue_run=input_data.get("from_run_workflow", False), + extra_fields={"task": "verilog"}, + trace_id=origin_tid, + ) + + if input_data.get("from_run_workflow"): + await ctx.enqueue({"topic": "dc.synth", "data": {**input_data, "task": "run"}}) + + return diff --git a/bbdev/api/steps/verilator/03_build_api.step.py b/bbdev/api/steps/verilator/03_build_api.step.py index 7bb01694a..092dd861f 100644 --- a/bbdev/api/steps/verilator/03_build_api.step.py +++ b/bbdev/api/steps/verilator/03_build_api.step.py @@ -13,7 +13,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} data = { "jobs": body.get("jobs", 16), - "cosim": body.get("cosim", False), } await ctx.enqueue({"topic": "verilator.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/03_build_event.step.py b/bbdev/api/steps/verilator/03_build_event.step.py index ddddcd66a..a3aa61ce5 100644 --- a/bbdev/api/steps/verilator/03_build_event.step.py +++ b/bbdev/api/steps/verilator/03_build_event.step.py @@ -19,7 +19,7 @@ "description": "build verilator executable", "flows": ["verilator"], "triggers": [queue("verilator.build")], - "enqueues": ["verilator.sim", "verilator.cosim"], + "enqueues": ["verilator.sim"], } @@ -29,7 +29,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: arch_dir = f"{bbdir}/arch" build_dir = f"{arch_dir}/build" coverage = input_data.get("coverage", False) - cosim = input_data.get("cosim", False) # ================================================================================== # Find sources @@ -54,10 +53,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: _tsi_htif = {"testchip_tsi.cc", "testchip_htif.cc", "SimTSI.cc"} csrcs = [f for f in csrcs if os.path.basename(f) not in _tsi_htif] - if cosim: - topname = "ToyBuckyball" - else: - topname = "BBSimHarness" + topname = "BBSimHarness" # ================================================================================== # Build flags @@ -82,8 +78,6 @@ def pkg_config(flag, pkg): # -DBBSIM: selects VBBSimHarness in bdb.h / main.cc # BDB NDJSON trace (+trace=...) is runtime-only; bbdev sim uses +trace=all (04_sim_event.step.py). cflags = f"{inc_flags} -DBBSIM -DTOP_NAME='\"V{topname}\"' -std=c++17" - if cosim: - cflags += " -DCOSIM" ldflags = ( f"-lreadline -ldramsim -lstdc++ -lz " @@ -174,13 +168,8 @@ def pkg_config(flag, pkg): # Continue routing # ================================================================================== if input_data.get("from_run_workflow"): - if cosim: - await ctx.enqueue( - {"topic": "verilator.cosim", "data": {**input_data, "task": "run"}} - ) - else: - await ctx.enqueue( - {"topic": "verilator.sim", "data": {**input_data, "task": "run"}} - ) + await ctx.enqueue( + {"topic": "verilator.sim", "data": {**input_data, "task": "run"}} + ) return diff --git a/bbdev/api/steps/verilator/04_cosim_api.step.py b/bbdev/api/steps/verilator/04_cosim_api.step.py deleted file mode 100644 index 419336e8e..000000000 --- a/bbdev/api/steps/verilator/04_cosim_api.step.py +++ /dev/null @@ -1,27 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - -config = { - "name": "verilator-cosim-api", - "description": "run verilator cosimulation", - "flows": ["verilator"], - "triggers": [api("POST", "/verilator/cosim")], - "enqueues": ["verilator.cosim"], -} - - -async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = request.body or {} - binary = body.get("binary", "") - if not binary: - return ApiResponse( - status=400, - body={ - "success": False, - "failure": True, - "returncode": 400, - "message": "binary parameter is required", - }, - ) - - await ctx.enqueue({"topic": "verilator.cosim", "data": {**body, "task": "cosim", "_trace_id": ctx.trace_id}}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/04_cosim_event.step.py b/bbdev/api/steps/verilator/04_cosim_event.step.py deleted file mode 100644 index d84f63d87..000000000 --- a/bbdev/api/steps/verilator/04_cosim_event.step.py +++ /dev/null @@ -1,128 +0,0 @@ -import os -import subprocess -import sys -from datetime import datetime - -from motia import FlowContext, queue - -# Add the utils directory to the Python path -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from utils.search_workload import search_workload -from utils.event_common import check_result, get_origin_trace_id - -config = { - "name": "verilator-cosim", - "description": "run cosimulation", - "flows": ["verilator"], - "triggers": [queue("verilator.cosim")], - "enqueues": [], -} - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - # ================================================================================== - # Get simulation parameters - # ================================================================================== - bbdir = get_buckyball_path() - arch_dir = f"{bbdir}/arch" - build_dir = f"{arch_dir}/build" - - # Generate timestamp - timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - - binary_name = input_data.get("binary", "") - success_result, failure_result = await check_result( - ctx, returncode=(binary_name == None), continue_run=True, trace_id=origin_tid, - ) - - binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) - success_result, failure_result = await check_result( - ctx, returncode=(binary_path == None), continue_run=True, trace_id=origin_tid, - ) - if failure_result: - ctx.logger.error("binary not found", failure_result) - return - - # Create log and waveform directory - log_dir = f"{arch_dir}/log/{timestamp}-{binary_name}" - waveform_dir = f"{arch_dir}/waveform/{timestamp}-{binary_name}" - topname = "ToyBuckyball" - - os.makedirs(log_dir, exist_ok=True) - os.makedirs(waveform_dir, exist_ok=True) - - bin_path = f"{build_dir}/obj_dir/V{topname}" - batch = input_data.get("batch", False) - - # Create log and waveform file - log_path = f"{log_dir}/bdb.log" - fst_path = f"{waveform_dir}/waveform.fst" - # Remove old waveform file - subprocess.run(f"rm -f {waveform_dir}/waveform.vcd", shell=True, check=True) - - # ================================================================================== - # Execute simulation script with streaming output - # ================================================================================== - bebop_isa_sim = f"{bbdir}/bebop/host/spike/riscv-isa-sim" - ld_lib_path = ( - f"{bebop_isa_sim}/install/lib:" - f"{bbdir}/result/lib:" - f"{arch_dir}/thirdparty/chipyard/tools/DRAMSim2" - ) - sim_cmd = ( - f"export LD_LIBRARY_PATH=\"{ld_lib_path}:$LD_LIBRARY_PATH\"; " - f"{bin_path} +permissive +loadmem={binary_path} +loadmem_addr=800000000 " - f"{'+batch ' if batch else ''} " - f"+fst={fst_path} +log={log_path} +permissive-off " - f"{binary_path} > >(tee {log_dir}/stdout.log) 2> >(spike-dasm > {log_dir}/disasm.log)" - ) - script_dir = os.path.dirname(__file__) - - result = stream_run_logger( - cmd=sim_cmd, - logger=ctx.logger, - cwd=script_dir, - stdout_prefix="verilator sim", - stderr_prefix="verilator sim", - executable="bash", - ) - success_result, failure_result = await check_result( - ctx, returncode=result.returncode, continue_run=True, trace_id=origin_tid) - if failure_result: - ctx.logger.error("sim failed", failure_result) - return - - if os.path.exists(f"{waveform_dir}/waveform.fst.heir"): - subprocess.run( - f"gtkwave -f {waveform_dir}/waveform.fst -H {waveform_dir}/waveform.fst.heir", - shell=True, - check=True, - ) - - # ================================================================================== - # Return simulation result - # ================================================================================== - # This is the end point of the run workflow, status will no longer be set to processing - success_result, failure_result = await check_result( - ctx, - result.returncode, - continue_run=False, - extra_fields={ - "task": "sim", - "binary": binary_path, - "log_dir": log_dir, - "waveform_dir": waveform_dir, - "timestamp": timestamp, - }, trace_id=origin_tid) - - # ================================================================================== - # Finish workflow - # ================================================================================== - - return diff --git a/bbdev/api/steps/verilator/05_run_api.step.py b/bbdev/api/steps/verilator/05_run_api.step.py index b599755b3..d784e94a2 100644 --- a/bbdev/api/steps/verilator/05_run_api.step.py +++ b/bbdev/api/steps/verilator/05_run_api.step.py @@ -17,7 +17,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "config": body.get("config", "sims.verilator.BuckyballToyVerilatorConfig"), "jobs": body.get("jobs", "16"), "batch": body.get("batch", False), - "cosim": body.get("cosim", False), "from_run_workflow": True, } diff --git a/bbdev/api/steps/yosys/01_run_api.step.py b/bbdev/api/steps/yosys/01_run_api.step.py new file mode 100644 index 000000000..f8c4dc99f --- /dev/null +++ b/bbdev/api/steps/yosys/01_run_api.step.py @@ -0,0 +1,25 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path + +config = { + "name": "yosys-run", + "description": "run yosys flow", + "flows": ["yosys"], + "triggers": [api("POST", "/yosys/run")], + "enqueues": ["yosys.run"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = req.body or {} + + data = { + "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), + "top": body.get("top"), + "config": body.get("config"), + "from_run_workflow": True, + } + await ctx.enqueue({"topic": "yosys.run", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/yosys/02_verilog_api.step.py b/bbdev/api/steps/yosys/02_verilog_api.step.py new file mode 100644 index 000000000..9d14550fc --- /dev/null +++ b/bbdev/api/steps/yosys/02_verilog_api.step.py @@ -0,0 +1,24 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path + +config = { + "name": "yosys-verilog-api", + "description": "generate verilog for yosys flow", + "flows": ["yosys"], + "triggers": [api("POST", "/yosys/verilog")], + "enqueues": ["yosys.verilog"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = req.body or {} + + data = { + "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), + "top": body.get("top"), + "config": body.get("config"), + } + await ctx.enqueue({"topic": "yosys.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/yosys/01_synth_api.step.py b/bbdev/api/steps/yosys/03_synth_api.step.py similarity index 95% rename from bbdev/api/steps/yosys/01_synth_api.step.py rename to bbdev/api/steps/yosys/03_synth_api.step.py index 81a12990a..4d4926c8a 100644 --- a/bbdev/api/steps/yosys/01_synth_api.step.py +++ b/bbdev/api/steps/yosys/03_synth_api.step.py @@ -3,7 +3,7 @@ from utils.path import get_buckyball_path config = { - "name": "Yosys Synth", + "name": "yosys-synth-api", "description": "run yosys synthesis for area estimation", "flows": ["yosys"], "triggers": [api("POST", "/yosys/synth")], diff --git a/bbdev/api/steps/yosys/01_synth_event.step.py b/bbdev/api/steps/yosys/03_synth_event.step.py similarity index 51% rename from bbdev/api/steps/yosys/01_synth_event.step.py rename to bbdev/api/steps/yosys/03_synth_event.step.py index 56845c721..7de4b7357 100644 --- a/bbdev/api/steps/yosys/01_synth_event.step.py +++ b/bbdev/api/steps/yosys/03_synth_event.step.py @@ -1,12 +1,10 @@ -import os -import subprocess import glob +import os import sys import yaml from motia import FlowContext, queue -# Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) @@ -25,7 +23,6 @@ def load_yosys_config(): - """Load yosys configuration from yaml file""" config_path = os.path.join(os.path.dirname(__file__), "scripts", "yosys-config.yaml") if os.path.exists(config_path): with open(config_path, "r") as f: @@ -37,67 +34,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/") - arch_dir = f"{bbdir}/arch" - # Load yaml config as defaults, CLI args override yosys_cfg = load_yosys_config() top_module = input_data.get("top") or yosys_cfg.get("top") or "BuckyballAccelerator" - liberty = yosys_cfg.get("liberty") # Path to .lib file, configured in yosys-config.yaml only - elaborate_config = input_data.get("config") or yosys_cfg.get("elaborate_config", "sims.verilator.BuckyballToyVerilatorConfig") - - ctx.logger.info(f"Top module: {top_module}, Elaborate config: {elaborate_config}") - - # ================================================================================== - # Step 1: Generate Verilog via Elaborate (full SoC), filter DPI-C files later - # ================================================================================== - import shutil - if os.path.exists(build_dir): - shutil.rmtree(build_dir) - os.makedirs(build_dir, exist_ok=True) - - ctx.logger.info("Step 1: Generating Verilog via Elaborate...") - verilog_command = ( - f"mill -i __.test.runMain sims.verilator.Elaborate {elaborate_config} " - "--disable-annotation-unknown -strip-debug-info -O=debug " - "-lowering-options=disallowLocalVariables " - f"--split-verilog -o={build_dir}" - ) - - result = stream_run_logger( - cmd=verilog_command, - logger=ctx.logger, - cwd=arch_dir, - stdout_prefix="yosys verilog", - stderr_prefix="yosys verilog", - ) + liberty = yosys_cfg.get("liberty") - if result.returncode != 0: - ctx.logger.error(f"Verilog generation failed with code {result.returncode}") - success_result, failure_result = await check_result( - ctx, - result.returncode, - continue_run=False, - extra_fields={"task": "verilog"}, - trace_id=origin_tid, - ) - return failure_result - - # Remove unwanted file (BallTopMain shouldn't generate this, but clean up just in case) - for unwanted in ["TestHarness.sv", "TargetBall.sv"]: - topname_file = f"{arch_dir}/{unwanted}" - if os.path.exists(topname_file): - os.remove(topname_file) - - # ================================================================================== - # Step 2: Run Yosys synthesis for area estimation - # ================================================================================== - ctx.logger.info("Step 2: Running Yosys synthesis for area estimation...") - - # Collect SystemVerilog sources only (.sv), skip .v files which may contain DPI-C imports vsrcs = glob.glob(f"{build_dir}/**/*.sv", recursive=True) - # Filter out files with unsupported SystemVerilog syntax (e.g. '{ assignment patterns) - # These are mostly TileLink bus infra / monitors that Yosys can't parse filtered_vsrcs = [] skipped = [] for f in vsrcs: @@ -110,11 +53,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: vsrcs = filtered_vsrcs if skipped: - ctx.logger.info(f"Skipped {len(skipped)} files with unsupported syntax: {', '.join(skipped[:10])}{'...' if len(skipped) > 10 else ''}") - ctx.logger.info(f"Found {len(vsrcs)} synthesizable SystemVerilog source files") + ctx.logger.info( + f"Skipped {len(skipped)} files with unsupported syntax: {', '.join(skipped[:10])}{'...' if len(skipped) > 10 else ''}" + ) if not vsrcs: - ctx.logger.error(f"No Verilog source files found in {build_dir}") success_result, failure_result = await check_result( ctx, 1, @@ -124,14 +67,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return failure_result - # Create yosys log directory yosys_output_dir = os.path.join(os.path.dirname(__file__), "log") os.makedirs(yosys_output_dir, exist_ok=True) - # Build yosys read commands for all source files read_commands = "\n".join([f"read_verilog -sv {src}" for src in vsrcs]) - - # Write yosys synthesis script yosys_script = f"{yosys_output_dir}/synth_area.ys" with open(yosys_script, "w") as f: f.write(f"{read_commands}\n") @@ -141,59 +80,36 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f.write(f"synth -top {top_module}\n") if liberty and os.path.exists(liberty): - ctx.logger.info(f"Using liberty file: {liberty}") f.write(f"dfflibmap -liberty {liberty}\n") f.write(f"abc -liberty {liberty}\n") - - # Hierarchical area breakdown BEFORE flatten (per-module with liberty area) f.write(f"tee -o {yosys_output_dir}/hierarchy_report.txt stat -liberty {liberty}\n") - - # Flatten and report total area f.write("flatten\n") f.write("opt\n") f.write(f"stat -liberty {liberty}\n") f.write(f"tee -o {yosys_output_dir}/area_report.txt stat -liberty {liberty}\n") - # Write mapped netlist for OpenSTA timing analysis f.write(f"write_verilog {yosys_output_dir}/synth_netlist.v\n") else: - if liberty: - ctx.logger.warn(f"Liberty file not found: {liberty}, falling back to generic stat") - # Hierarchical breakdown before flatten f.write(f"tee -o {yosys_output_dir}/hierarchy_report.txt stat\n") f.write("flatten\n") f.write("opt\n") f.write("stat\n") f.write(f"tee -o {yosys_output_dir}/area_report.txt stat\n") - ctx.logger.info(f"Yosys script written to {yosys_script}") - ctx.logger.info(f"Synthesizing {len(vsrcs)} source files with top module: {top_module}") - - # Run yosys - yosys_command = f"yosys -s {yosys_script}" - result = stream_run_logger( - cmd=yosys_command, + cmd=f"yosys -s {yosys_script}", logger=ctx.logger, cwd=build_dir, stdout_prefix="yosys synth", stderr_prefix="yosys synth", ) - # ================================================================================== - # Step 3: Run OpenSTA timing analysis (only if liberty file is available) - # ================================================================================== extra = {"task": "synth", "output_dir": yosys_output_dir} netlist_file = f"{yosys_output_dir}/synth_netlist.v" timing_report_file = f"{yosys_output_dir}/timing_report.txt" if liberty and os.path.exists(liberty) and os.path.exists(netlist_file) and result.returncode == 0: - ctx.logger.info("Step 3: Running OpenSTA timing analysis...") - - # Get target clock period from config (default 10ns = 100MHz) clock_period = yosys_cfg.get("clock_period", 10.0) clock_name = yosys_cfg.get("clock_name", "clock") - - # Write OpenSTA TCL script sta_script = f"{yosys_output_dir}/sta_timing.tcl" with open(sta_script, "w") as f: f.write(f"read_liberty {liberty}\n") @@ -215,28 +131,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if sta_result.returncode == 0 and os.path.exists(timing_report_file): with open(timing_report_file, "r") as f: extra["timing_report"] = f.read() - ctx.logger.info(f"Timing report saved to {timing_report_file}") - else: - ctx.logger.warn(f"OpenSTA timing analysis failed (rc={sta_result.returncode})") - elif result.returncode == 0: - ctx.logger.info("Skipping OpenSTA: no liberty file configured or synthesis failed") - # ================================================================================== - # Return result to API - # ================================================================================== report_file = f"{yosys_output_dir}/area_report.txt" if os.path.exists(report_file): with open(report_file, "r") as f: extra["area_report"] = f.read() - ctx.logger.info(f"Area report saved to {report_file}") hierarchy_file = f"{yosys_output_dir}/hierarchy_report.txt" if os.path.exists(hierarchy_file): with open(hierarchy_file, "r") as f: extra["hierarchy_report"] = f.read() - ctx.logger.info(f"Hierarchy report saved to {hierarchy_file}") - success_result, failure_result = await check_result( + await check_result( ctx, result.returncode, continue_run=False, diff --git a/bbdev/api/tests/_api_test_helper.py b/bbdev/api/tests/_api_test_helper.py new file mode 100644 index 000000000..b63177469 --- /dev/null +++ b/bbdev/api/tests/_api_test_helper.py @@ -0,0 +1,7 @@ +import subprocess + + +def run_bbdev_case(command: str): + result = subprocess.run(command, shell=True) + if result.returncode != 0: + raise RuntimeError(f"command failed({result.returncode}): {command}") diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py new file mode 100755 index 000000000..2d09f07c1 --- /dev/null +++ b/bbdev/api/tests/test_all_api_serial.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +import subprocess +from pathlib import Path + + +def main(): + test_dir = Path(__file__).resolve().parent + tests = [ + "test_api_compiler_build.py", + "test_api_workload_build.py", + "test_api_verilator_clean.py", + "test_api_verilator_verilog.py", + "test_api_verilator_build.py", + "test_api_verilator_sim.py", + "test_api_verilator_run.py", + "test_api_yosys_run.py", + "test_api_yosys_verilog.py", + "test_api_yosys_synth.py", + "test_api_dc_run.py", + "test_api_dc_verilog.py", + "test_api_dc_synth.py", + # "test_api_sardine_run.py", + # "test_api_kernel_build.py", + # "test_api_pegasus_verilog.py", + # "test_api_pegasus_buildbitstream.py", + # "test_api_pegasus_flashbitstream.py", + # "test_api_pegasus_runworkload.py", + # "test_api_firesim_buildbitstream.py", + # "test_api_firesim_infrasetup.py", + # "test_api_firesim_runworkload.py", + ] + for name in tests: + path = test_dir / name + if not path.exists(): + raise RuntimeError(f"missing test: {name}") + print(f"RUN {name}") + ret = subprocess.run(["python", str(path)], cwd=str(test_dir)) + if ret.returncode != 0: + raise RuntimeError(f"failed: {name}") + print(f"PASS: {len(tests)} api tests done.") + + +if __name__ == "__main__": + main() diff --git a/bbdev/api/tests/test_api_compiler_build.py b/bbdev/api/tests/test_api_compiler_build.py new file mode 100644 index 000000000..54c28e552 --- /dev/null +++ b/bbdev/api/tests/test_api_compiler_build.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev compiler --build") diff --git a/bbdev/api/tests/test_api_dc_run.py b/bbdev/api/tests/test_api_dc_run.py new file mode 100644 index 000000000..0f8f83720 --- /dev/null +++ b/bbdev/api/tests/test_api_dc_run.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev dc --run") diff --git a/bbdev/api/tests/test_api_dc_synth.py b/bbdev/api/tests/test_api_dc_synth.py new file mode 100644 index 000000000..b86a5c8ca --- /dev/null +++ b/bbdev/api/tests/test_api_dc_synth.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev dc --synth") diff --git a/bbdev/api/tests/test_api_dc_verilog.py b/bbdev/api/tests/test_api_dc_verilog.py new file mode 100644 index 000000000..232553bc5 --- /dev/null +++ b/bbdev/api/tests/test_api_dc_verilog.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev dc --verilog") diff --git a/bbdev/api/tests/test_api_firesim_buildbitstream.py b/bbdev/api/tests/test_api_firesim_buildbitstream.py new file mode 100644 index 000000000..4b7ceaa5d --- /dev/null +++ b/bbdev/api/tests/test_api_firesim_buildbitstream.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev firesim --buildbitstream") diff --git a/bbdev/api/tests/test_api_firesim_infrasetup.py b/bbdev/api/tests/test_api_firesim_infrasetup.py new file mode 100644 index 000000000..4e46ee0c8 --- /dev/null +++ b/bbdev/api/tests/test_api_firesim_infrasetup.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev firesim --infrasetup") diff --git a/bbdev/api/tests/test_api_firesim_runworkload.py b/bbdev/api/tests/test_api_firesim_runworkload.py new file mode 100644 index 000000000..8ebf19820 --- /dev/null +++ b/bbdev/api/tests/test_api_firesim_runworkload.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev firesim --runworkload") diff --git a/bbdev/api/tests/test_api_kernel_build.py b/bbdev/api/tests/test_api_kernel_build.py new file mode 100644 index 000000000..26f502d09 --- /dev/null +++ b/bbdev/api/tests/test_api_kernel_build.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev kernel --build") diff --git a/bbdev/api/tests/test_api_pegasus_buildbitstream.py b/bbdev/api/tests/test_api_pegasus_buildbitstream.py new file mode 100644 index 000000000..3d16e2125 --- /dev/null +++ b/bbdev/api/tests/test_api_pegasus_buildbitstream.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev pegasus --buildbitstream") diff --git a/bbdev/api/tests/test_api_pegasus_flashbitstream.py b/bbdev/api/tests/test_api_pegasus_flashbitstream.py new file mode 100644 index 000000000..d48d67dce --- /dev/null +++ b/bbdev/api/tests/test_api_pegasus_flashbitstream.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev pegasus --flashbitstream") diff --git a/bbdev/api/tests/test_api_pegasus_runworkload.py b/bbdev/api/tests/test_api_pegasus_runworkload.py new file mode 100644 index 000000000..54cbc604e --- /dev/null +++ b/bbdev/api/tests/test_api_pegasus_runworkload.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev pegasus --runworkload") diff --git a/bbdev/api/tests/test_api_pegasus_verilog.py b/bbdev/api/tests/test_api_pegasus_verilog.py new file mode 100644 index 000000000..058d6b521 --- /dev/null +++ b/bbdev/api/tests/test_api_pegasus_verilog.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev pegasus --verilog") diff --git a/bbdev/api/tests/test_api_sardine_run.py b/bbdev/api/tests/test_api_sardine_run.py new file mode 100644 index 000000000..9d469daf9 --- /dev/null +++ b/bbdev/api/tests/test_api_sardine_run.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev sardine --run") diff --git a/bbdev/api/tests/test_api_verilator_build.py b/bbdev/api/tests/test_api_verilator_build.py new file mode 100644 index 000000000..c3b131602 --- /dev/null +++ b/bbdev/api/tests/test_api_verilator_build.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev verilator --build '--jobs 16'") diff --git a/bbdev/api/tests/test_api_verilator_clean.py b/bbdev/api/tests/test_api_verilator_clean.py new file mode 100644 index 000000000..0e76ee6f9 --- /dev/null +++ b/bbdev/api/tests/test_api_verilator_clean.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev verilator --clean") diff --git a/bbdev/api/tests/test_api_verilator_run.py b/bbdev/api/tests/test_api_verilator_run.py new file mode 100644 index 000000000..f30775bdf --- /dev/null +++ b/bbdev/api/tests/test_api_verilator_run.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev verilator --run '--jobs 16 --binary ctest_vecunit_matmul_ones_singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig --batch'") diff --git a/bbdev/api/tests/test_api_verilator_sim.py b/bbdev/api/tests/test_api_verilator_sim.py new file mode 100644 index 000000000..962a28b43 --- /dev/null +++ b/bbdev/api/tests/test_api_verilator_sim.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev verilator --sim '--binary ctest_vecunit_matmul_ones_singlecore-baremetal --batch'") diff --git a/bbdev/api/tests/test_api_verilator_verilog.py b/bbdev/api/tests/test_api_verilator_verilog.py new file mode 100644 index 000000000..f55a95266 --- /dev/null +++ b/bbdev/api/tests/test_api_verilator_verilog.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev verilator --verilog '--config sims.verilator.BuckyballToyVerilatorConfig'") diff --git a/bbdev/api/tests/test_api_workload_build.py b/bbdev/api/tests/test_api_workload_build.py new file mode 100644 index 000000000..4d3343a96 --- /dev/null +++ b/bbdev/api/tests/test_api_workload_build.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev workload --build") diff --git a/bbdev/api/tests/test_api_yosys_run.py b/bbdev/api/tests/test_api_yosys_run.py new file mode 100644 index 000000000..d0b6b4ae8 --- /dev/null +++ b/bbdev/api/tests/test_api_yosys_run.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev yosys --run") diff --git a/bbdev/api/tests/test_api_yosys_synth.py b/bbdev/api/tests/test_api_yosys_synth.py new file mode 100644 index 000000000..a0e04eeea --- /dev/null +++ b/bbdev/api/tests/test_api_yosys_synth.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev yosys --synth") diff --git a/bbdev/api/tests/test_api_yosys_verilog.py b/bbdev/api/tests/test_api_yosys_verilog.py new file mode 100644 index 000000000..5a94e654a --- /dev/null +++ b/bbdev/api/tests/test_api_yosys_verilog.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev yosys --verilog") diff --git a/bbdev/bbdev b/bbdev/bbdev index c733f711a..21781460c 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -77,6 +77,11 @@ WORKFLOW_COMMANDS = { "dc": { "run": 'Run Design Compiler synthesis. Args: "[--top ] [--config ] [--output-dir ]"', }, + "yosys": { + "run": 'Run yosys flow. Args: "[--top ] [--config ] [--output-dir ]"', + "verilog": 'Generate verilog for yosys flow. Args: "[--config ] [--output-dir ]"', + "synth": 'Run yosys synthesis. Args: "[--top ] [--output-dir ]"', + }, } ALL_COMMANDS = ["start", "stop"] + list(WORKFLOW_COMMANDS) From f505a3e5fd2815198d8aa76d1e7868ea191f1092 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 22 Apr 2026 22:38:42 +0800 Subject: [PATCH 039/196] feat: spilt preparation functions for Verilog source management across multiple steps, including DC, Verilator, Pegasus, and Yosys workflows --- bbdev/api/steps/dc/03_synth_event.step.py | 30 +++- .../mill/01_verilator_verilog_event.step.py | 50 +++--- .../mill/02_pegasus_verilog_event.step.py | 137 +++++++++-------- .../steps/mill/03_yosys_verilog_event.step.py | 144 +++++++++++++++++- .../steps/mill/04_dc_verilog_event.step.py | 29 +++- bbdev/api/steps/yosys/03_synth_event.step.py | 32 ++-- 6 files changed, 315 insertions(+), 107 deletions(-) diff --git a/bbdev/api/steps/dc/03_synth_event.step.py b/bbdev/api/steps/dc/03_synth_event.step.py index 3c51a07c4..bce1796b7 100644 --- a/bbdev/api/steps/dc/03_synth_event.step.py +++ b/bbdev/api/steps/dc/03_synth_event.step.py @@ -30,6 +30,22 @@ def load_dc_config(): return {} +def prepare_dc_design(build_dir: str, design_dir: str): + source_list_path = os.path.join(build_dir, "dc_sources.list") + if not os.path.exists(source_list_path): + raise RuntimeError("missing dc_sources.list, run dc verilog first") + with open(source_list_path, "r") as f: + sources = [line.strip() for line in f.readlines() if line.strip()] + if not sources: + raise RuntimeError("empty dc_sources.list") + for src in sources: + rel = os.path.relpath(src, build_dir) + dst = os.path.join(design_dir, rel) + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy2(src, dst) + return source_list_path + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -47,7 +63,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if os.path.exists(d): shutil.rmtree(d) os.makedirs(d, exist_ok=True) - shutil.copytree(build_dir, design_dir, dirs_exist_ok=True) + try: + source_list_path = prepare_dc_design(build_dir, design_dir) + except Exception as e: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "dc", "error": str(e)}, + trace_id=origin_tid, + ) + return tcl_script = os.path.join(os.path.dirname(__file__), "scripts", "run-dc.tcl") target_libs = " ".join(dc_cfg.get("target_libraries") or []) @@ -81,7 +107,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if os.path.exists(alib): shutil.rmtree(alib) - extra = {"task": "dc", "report_dir": report_dir, "top": top_module} + extra = {"task": "dc", "report_dir": report_dir, "top": top_module, "source_list": source_list_path} for name in ("area.rpt", "hierarchy.rpt", "timing.rpt", "power.rpt"): path = os.path.join(report_dir, name) if os.path.exists(path): diff --git a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py index c5934f8e2..67edd73d2 100644 --- a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py @@ -21,6 +21,32 @@ } +def prepare_verilator_verilog(build_dir: str, arch_dir: str, logger): + for unwanted in [ + f"{build_dir}/testchip_htif.cc", + f"{build_dir}/testchip_htif.h", + f"{build_dir}/testchip_tsi.cc", + f"{build_dir}/testchip_tsi.h", + f"{build_dir}/SimTSI.cc", + f"{arch_dir}/BBSimHarness.sv", + ]: + if os.path.exists(unwanted): + os.remove(unwanted) + + for patch_file in [f"{build_dir}/mm.h", f"{build_dir}/mm.cc"]: + if os.path.exists(patch_file): + with open(patch_file, "r") as f: + content = f.read() + patched = "\n".join( + line for line in content.splitlines() + if "fesvr/memif.h" not in line and "fesvr/elfloader.h" not in line + ) + if patched != content: + with open(patch_file, "w") as f: + f.write(patched) + logger.info(f"Patched fesvr includes from {patch_file}") + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -65,29 +91,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="verilator verilog", ) - for unwanted in [ - f"{build_dir}/testchip_htif.cc", - f"{build_dir}/testchip_htif.h", - f"{build_dir}/testchip_tsi.cc", - f"{build_dir}/testchip_tsi.h", - f"{build_dir}/SimTSI.cc", - f"{arch_dir}/BBSimHarness.sv", - ]: - if os.path.exists(unwanted): - os.remove(unwanted) - - for patch_file in [f"{build_dir}/mm.h", f"{build_dir}/mm.cc"]: - if os.path.exists(patch_file): - with open(patch_file, "r") as f: - content = f.read() - patched = "\n".join( - line for line in content.splitlines() - if "fesvr/memif.h" not in line and "fesvr/elfloader.h" not in line - ) - if patched != content: - with open(patch_file, "w") as f: - f.write(patched) - ctx.logger.info(f"Patched fesvr includes from {patch_file}") + prepare_verilator_verilog(build_dir, arch_dir, ctx.logger) success_result, failure_result = await check_result( ctx, diff --git a/bbdev/api/steps/mill/02_pegasus_verilog_event.step.py b/bbdev/api/steps/mill/02_pegasus_verilog_event.step.py index e00c089c6..5ed14142d 100644 --- a/bbdev/api/steps/mill/02_pegasus_verilog_event.step.py +++ b/bbdev/api/steps/mill/02_pegasus_verilog_event.step.py @@ -20,6 +20,74 @@ } +def prepare_pegasus_verilog_output(bbdir: str, arch_dir: str, soc_dir: str, build_dir: str, logger): + for stray in ["PegasusTop.v", "PegasusTopWrapper.sv", "PegasusShell.v"]: + stray_path = f"{arch_dir}/{stray}" + if os.path.exists(stray_path): + os.remove(stray_path) + + vivado_gen_dir = f"{bbdir}/thirdparty/pegasus/vivado/generated" + if not os.path.isdir(build_dir): + return vivado_gen_dir + + import re + import shutil + os.makedirs(vivado_gen_dir, exist_ok=True) + for f in os.listdir(vivado_gen_dir): + if f.endswith(".sv") or f.endswith(".v"): + os.remove(os.path.join(vivado_gen_dir, f)) + + harness_skip = {"PegasusHarness.sv", "ChipTop.sv"} + + def build_dpi_stub(src_path: str) -> str: + text = open(src_path, "r", encoding="utf-8").read() + m = re.search(r"module\s+([A-Za-z_][A-Za-z0-9_]*)\s*\((.*?)\);\s*", text, re.S) + if m is None: + raise RuntimeError(f"invalid DPI module format: {src_path}") + mod_name = m.group(1) + ports_block = m.group(2).rstrip() + outputs = [] + for line in ports_block.splitlines(): + s = line.strip() + if not s.startswith("output"): + continue + s = s.rstrip(",") + s = re.sub(r"^output\s+", "", s) + s = re.sub(r"^\[[^\]]+\]\s+", "", s) + for name in s.split(","): + n = name.strip() + if n: + outputs.append(n) + stub = [f"module {mod_name}(", ports_block, ");"] + for out_name in outputs: + stub.append(f" assign {out_name} = '0;") + stub.append("endmodule") + stub.append("") + return "\n".join(stub) + + def copy_rtl_dir(src_dir: str, skip_set: set = None) -> int: + copied = 0 + for f in os.listdir(src_dir): + if not (f.endswith(".sv") or f.endswith(".v")): + continue + if skip_set and f in skip_set: + continue + src = os.path.join(src_dir, f) + if "DPI" in f: + stub_name = f"stub_{os.path.splitext(f)[0].replace('DPI', '')}.sv" + with open(os.path.join(vivado_gen_dir, stub_name), "w") as wf: + wf.write(build_dpi_stub(src)) + else: + shutil.copy2(src, os.path.join(vivado_gen_dir, f)) + copied += 1 + return copied + + n_soc = copy_rtl_dir(soc_dir, skip_set=harness_skip) + n_top = copy_rtl_dir(build_dir) + logger.info(f" Copied {n_soc} SoC files + {n_top} top files to {vivado_gen_dir}") + return vivado_gen_dir + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -71,68 +139,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="pegasus verilog", ) - for stray in ["PegasusTop.v", "PegasusTopWrapper.sv", "PegasusShell.v"]: - stray_path = f"{arch_dir}/{stray}" - if os.path.exists(stray_path): - os.remove(stray_path) - vivado_gen_dir = f"{bbdir}/thirdparty/pegasus/vivado/generated" - if result.returncode == 0 and os.path.isdir(build_dir): - import re - import shutil - os.makedirs(vivado_gen_dir, exist_ok=True) - for f in os.listdir(vivado_gen_dir): - if f.endswith(".sv") or f.endswith(".v"): - os.remove(os.path.join(vivado_gen_dir, f)) - - HARNESS_SKIP = {"PegasusHarness.sv", "ChipTop.sv"} - - def build_dpi_stub(src_path: str) -> str: - text = open(src_path, "r", encoding="utf-8").read() - m = re.search(r"module\s+([A-Za-z_][A-Za-z0-9_]*)\s*\((.*?)\);\s*", text, re.S) - if m is None: - raise RuntimeError(f"invalid DPI module format: {src_path}") - mod_name = m.group(1) - ports_block = m.group(2).rstrip() - outputs = [] - for line in ports_block.splitlines(): - s = line.strip() - if not s.startswith("output"): - continue - s = s.rstrip(",") - s = re.sub(r"^output\s+", "", s) - s = re.sub(r"^\[[^\]]+\]\s+", "", s) - for name in s.split(","): - n = name.strip() - if n: - outputs.append(n) - stub = [f"module {mod_name}(", ports_block, ");"] - for out_name in outputs: - stub.append(f" assign {out_name} = '0;") - stub.append("endmodule") - stub.append("") - return "\n".join(stub) - - def copy_rtl_dir(src_dir: str, skip_set: set = None) -> int: - copied = 0 - for f in os.listdir(src_dir): - if not (f.endswith(".sv") or f.endswith(".v")): - continue - if skip_set and f in skip_set: - continue - src = os.path.join(src_dir, f) - if "DPI" in f: - stub_name = f"stub_{os.path.splitext(f)[0].replace('DPI', '')}.sv" - with open(os.path.join(vivado_gen_dir, stub_name), "w") as wf: - wf.write(build_dpi_stub(src)) - else: - shutil.copy2(src, os.path.join(vivado_gen_dir, f)) - copied += 1 - return copied - - n_soc = copy_rtl_dir(soc_dir, skip_set=HARNESS_SKIP) - n_top = copy_rtl_dir(build_dir) - ctx.logger.info(f" Copied {n_soc} SoC files + {n_top} top files to {vivado_gen_dir}") + if result.returncode == 0: + vivado_gen_dir = prepare_pegasus_verilog_output( + bbdir=bbdir, + arch_dir=arch_dir, + soc_dir=soc_dir, + build_dir=build_dir, + logger=ctx.logger, + ) success_result, failure_result = await check_result( ctx, diff --git a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py index af621bbc5..d0263b2e8 100644 --- a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py +++ b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py @@ -1,6 +1,8 @@ import os import shutil import sys +import glob +import re import yaml from motia import FlowContext, queue @@ -31,6 +33,132 @@ def load_yosys_config(): return {} +def prepare_yosys_verilog(build_dir: str, yosys_log_dir: str, logger): + def _build_stub_from_header(src_path: str, content: str, force_blackbox: bool): + m = re.search(r"module\s+([A-Za-z_][A-Za-z0-9_]*)\s*\((.*?)\)\s*;", content, re.S) + if m is None: + raise RuntimeError(f"invalid module header format: {src_path}") + mod_name = m.group(1) + ports_block = m.group(2).rstrip() + head = "(* blackbox *) " if force_blackbox else "" + return f"{head}module {mod_name}(\n{ports_block}\n);\nendmodule\n" + + def _collect_defined_modules(sv_files): + defined = set() + for path in sv_files: + with open(path, "r") as f: + content = f.read() + for m in re.finditer(r"^\s*module\s+([A-Za-z_][A-Za-z0-9_]*)\b", content, re.M): + defined.add(m.group(1)) + return defined + + def _extract_external_modules(extern_path): + if not os.path.exists(extern_path): + return [] + names = [] + with open(extern_path, "r") as f: + for line in f: + m = re.match(r"\s*//\s*external module\s+([A-Za-z_][A-Za-z0-9_]*)\s*$", line) + if m: + names.append(m.group(1)) + return names + + def _collect_ports(mod_name, sv_files): + ports = set() + inst_re = re.compile(rf"\b{mod_name}\b\s+[A-Za-z_][A-Za-z0-9_]*\s*\((.*?)\)\s*;", re.S) + port_re = re.compile(r"\.([A-Za-z_][A-Za-z0-9_]*)\s*\(") + for path in sv_files: + with open(path, "r") as f: + content = f.read() + for im in inst_re.finditer(content): + block = im.group(1) + for pm in port_re.finditer(block): + ports.add(pm.group(1)) + return sorted(ports) + + def _build_blackbox(mod_name, ports): + lines = [f"(* blackbox *) module {mod_name}("] + if ports: + for idx, p in enumerate(ports): + end = "," if idx < len(ports) - 1 else "" + lines.append(f" input {p}{end}") + lines.append(");") + lines.append("endmodule") + lines.append("") + return "\n".join(lines) + + stub_dir = os.path.join(yosys_log_dir, "dpi_stubs") + os.makedirs(stub_dir, exist_ok=True) + + all_sv = glob.glob(f"{build_dir}/**/*.sv", recursive=True) + kept = [] + dpi_stubbed = [] + pattern_stubbed = [] + skipped = [] + + for path in all_sv: + with open(path, "r") as f: + content = f.read() + if 'import "DPI-C"' in content: + stub_path = os.path.join(stub_dir, f"stub_{os.path.basename(path)}") + with open(stub_path, "w") as wf: + wf.write(_build_stub_from_header(path, content, False)) + kept.append(stub_path) + dpi_stubbed.append(os.path.basename(path)) + elif "'{" in content: + stub_path = os.path.join(stub_dir, f"pattern_stub_{os.path.basename(path)}") + try: + with open(stub_path, "w") as wf: + wf.write(_build_stub_from_header(path, content, True)) + kept.append(stub_path) + pattern_stubbed.append(os.path.basename(path)) + except RuntimeError: + skipped.append(os.path.basename(path)) + else: + kept.append(path) + + extern_path = os.path.join(build_dir, "extern_modules.sv") + extern_names = _extract_external_modules(extern_path) + defined = _collect_defined_modules(kept) + ext_stubbed = [] + for mod_name in extern_names: + if mod_name in defined: + continue + ports = _collect_ports(mod_name, all_sv) + stub_path = os.path.join(stub_dir, f"ext_stub_{mod_name}.sv") + with open(stub_path, "w") as f: + f.write(_build_blackbox(mod_name, ports)) + kept.append(stub_path) + ext_stubbed.append(mod_name) + + if not kept: + raise RuntimeError("no yosys source generated") + + source_list_path = os.path.join(build_dir, "yosys_sources.list") + with open(source_list_path, "w") as f: + for path in kept: + f.write(path + "\n") + + if dpi_stubbed: + logger.info( + f"Stubbed {len(dpi_stubbed)} DPI modules: {', '.join(dpi_stubbed[:10])}{'...' if len(dpi_stubbed) > 10 else ''}" + ) + if pattern_stubbed: + logger.info( + f"Stubbed {len(pattern_stubbed)} pattern modules: {', '.join(pattern_stubbed[:10])}{'...' if len(pattern_stubbed) > 10 else ''}" + ) + if ext_stubbed: + logger.info( + f"Stubbed {len(ext_stubbed)} external modules: {', '.join(ext_stubbed[:10])}{'...' if len(ext_stubbed) > 10 else ''}" + ) + if skipped: + logger.info( + f"Skipped {len(skipped)} unsupported files: {', '.join(skipped[:10])}{'...' if len(skipped) > 10 else ''}" + ) + logger.info(f"Prepared yosys source list: {source_list_path}") + return source_list_path + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -76,11 +204,25 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if os.path.exists(topname_file): os.remove(topname_file) + yosys_log_dir = f"{bbdir}/bbdev/api/steps/yosys/log" + os.makedirs(yosys_log_dir, exist_ok=True) + try: + source_list_path = prepare_yosys_verilog(build_dir, yosys_log_dir, ctx.logger) + except Exception as e: + success_result, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "verilog", "error": str(e)}, + trace_id=origin_tid, + ) + return failure_result + await check_result( ctx, result.returncode, continue_run=input_data.get("from_run_workflow", False), - extra_fields={"task": "verilog"}, + extra_fields={"task": "verilog", "source_list": source_list_path}, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/mill/04_dc_verilog_event.step.py b/bbdev/api/steps/mill/04_dc_verilog_event.step.py index 4656bc12b..bef80ca71 100644 --- a/bbdev/api/steps/mill/04_dc_verilog_event.step.py +++ b/bbdev/api/steps/mill/04_dc_verilog_event.step.py @@ -1,6 +1,7 @@ import os import shutil import sys +import glob import yaml from motia import FlowContext, queue @@ -31,6 +32,20 @@ def load_dc_config(): return {} +def prepare_dc_verilog(build_dir: str): + vsrcs = sorted( + glob.glob(f"{build_dir}/**/*.sv", recursive=True) + + glob.glob(f"{build_dir}/**/*.v", recursive=True) + ) + if not vsrcs: + raise RuntimeError("no dc verilog source generated") + source_list_path = os.path.join(build_dir, "dc_sources.list") + with open(source_list_path, "w") as f: + for path in vsrcs: + f.write(path + "\n") + return source_list_path + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -73,11 +88,23 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return failure_result + try: + source_list_path = prepare_dc_verilog(build_dir) + except Exception as e: + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "verilog", "error": str(e)}, + trace_id=origin_tid, + ) + return failure_result + await check_result( ctx, result.returncode, continue_run=input_data.get("from_run_workflow", False), - extra_fields={"task": "verilog"}, + extra_fields={"task": "verilog", "source_list": source_list_path}, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/yosys/03_synth_event.step.py b/bbdev/api/steps/yosys/03_synth_event.step.py index 7de4b7357..c3393ca31 100644 --- a/bbdev/api/steps/yosys/03_synth_event.step.py +++ b/bbdev/api/steps/yosys/03_synth_event.step.py @@ -1,4 +1,3 @@ -import glob import os import sys import yaml @@ -29,7 +28,6 @@ def load_yosys_config(): return yaml.safe_load(f) or {} return {} - async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -39,30 +37,26 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: top_module = input_data.get("top") or yosys_cfg.get("top") or "BuckyballAccelerator" liberty = yosys_cfg.get("liberty") - vsrcs = glob.glob(f"{build_dir}/**/*.sv", recursive=True) - - filtered_vsrcs = [] - skipped = [] - for f in vsrcs: - with open(f, "r") as fh: - content = fh.read() - if "'{" in content or 'import "DPI-C"' in content: - skipped.append(os.path.basename(f)) - else: - filtered_vsrcs.append(f) - vsrcs = filtered_vsrcs - - if skipped: - ctx.logger.info( - f"Skipped {len(skipped)} files with unsupported syntax: {', '.join(skipped[:10])}{'...' if len(skipped) > 10 else ''}" + source_list_path = os.path.join(build_dir, "yosys_sources.list") + if not os.path.exists(source_list_path): + success_result, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "synth", "error": "missing yosys_sources.list, run yosys verilog first"}, + trace_id=origin_tid, ) + return failure_result + + with open(source_list_path, "r") as f: + vsrcs = [line.strip() for line in f.readlines() if line.strip()] if not vsrcs: success_result, failure_result = await check_result( ctx, 1, continue_run=False, - extra_fields={"task": "synth", "error": "No Verilog source files found"}, + extra_fields={"task": "synth", "error": "empty yosys_sources.list"}, trace_id=origin_tid, ) return failure_result From 8426485028fb0039838f39449d10bdc03e3c7a7c Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 23 Apr 2026 16:08:46 +0800 Subject: [PATCH 040/196] fix: update workload argument in sardine run test to include ctest option --- bbdev/api/tests/test_api_sardine_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbdev/api/tests/test_api_sardine_run.py b/bbdev/api/tests/test_api_sardine_run.py index 9d469daf9..070c636fa 100644 --- a/bbdev/api/tests/test_api_sardine_run.py +++ b/bbdev/api/tests/test_api_sardine_run.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev sardine --run") +run_bbdev_case("bbdev sardine --run '--workload ctest'") From 4165179783e156b74776bb31ed7f7a94bcda25bc Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 25 Apr 2026 04:07:05 +0800 Subject: [PATCH 041/196] del: remove unused ddr and tsi code from chipyard --- .../mill/01_verilator_verilog_event.step.py | 13 +++---------- bbdev/api/steps/verilator/03_build_event.step.py | 16 +++++++++------- bbdev/api/steps/verilator/04_sim_event.step.py | 2 +- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py index 67edd73d2..b581717ba 100644 --- a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py @@ -22,16 +22,9 @@ def prepare_verilator_verilog(build_dir: str, arch_dir: str, logger): - for unwanted in [ - f"{build_dir}/testchip_htif.cc", - f"{build_dir}/testchip_htif.h", - f"{build_dir}/testchip_tsi.cc", - f"{build_dir}/testchip_tsi.h", - f"{build_dir}/SimTSI.cc", - f"{arch_dir}/BBSimHarness.sv", - ]: - if os.path.exists(unwanted): - os.remove(unwanted) + unwanted_harness = f"{arch_dir}/BBSimHarness.sv" + if os.path.exists(unwanted_harness): + os.remove(unwanted_harness) for patch_file in [f"{build_dir}/mm.h", f"{build_dir}/mm.cc"]: if os.path.exists(patch_file): diff --git a/bbdev/api/steps/verilator/03_build_event.step.py b/bbdev/api/steps/verilator/03_build_event.step.py index a3aa61ce5..64a56a9ac 100644 --- a/bbdev/api/steps/verilator/03_build_event.step.py +++ b/bbdev/api/steps/verilator/03_build_event.step.py @@ -45,13 +45,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: + glob.glob(f"{build_dir}/**/*.cpp", recursive=True) ) - # Exclude testchipip's SimDRAM.cc — our SimDRAM_bb.cc overrides memory_init - csrcs = [f for f in csrcs if not f.endswith("SimDRAM.cc") or "src/csrc" in f] - - # Exclude testchipip's TSI/HTIF C++ sources (deleted in verilog step, but guard anyway). - # tsi_tick DPI symbol is provided by tsi_stub.cc in src/csrc instead. - _tsi_htif = {"testchip_tsi.cc", "testchip_htif.cc", "SimTSI.cc"} - csrcs = [f for f in csrcs if os.path.basename(f) not in _tsi_htif] + unexpected_tsi = [ + f + for f in csrcs + if os.path.basename(f) in {"testchip_tsi.cc", "testchip_htif.cc", "SimTSI.cc"} + ] + if unexpected_tsi: + raise RuntimeError( + "Unexpected TSI sources in BBSim build: " + ", ".join(unexpected_tsi) + ) topname = "BBSimHarness" diff --git a/bbdev/api/steps/verilator/04_sim_event.step.py b/bbdev/api/steps/verilator/04_sim_event.step.py index 249dacfac..3304a3c17 100644 --- a/bbdev/api/steps/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -72,7 +72,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ================================================================================== # Execute simulation - # BBSimHarness uses +elf= for ELF loading (via SimDRAM_bb.cc / libelf) + # BBSimHarness uses +elf= for ELF loading (via BBSimDRAM.cc / libelf) # No fesvr, no +loadmem_addr needed # # disasm.log: only stderr -> spike-dasm (Rocket commit printf is stderr here; From cd5dadb3916196a2c453f7b5f0f2c913bd3876d7 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 27 Apr 2026 14:55:04 +0800 Subject: [PATCH 042/196] [compiler] refactor: update new compiler path --- bbdev/api/steps/compiler/01_build_event.step.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbdev/api/steps/compiler/01_build_event.step.py b/bbdev/api/steps/compiler/01_build_event.step.py index a4969d638..cae181cca 100644 --- a/bbdev/api/steps/compiler/01_build_event.step.py +++ b/bbdev/api/steps/compiler/01_build_event.step.py @@ -29,14 +29,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ================================================================================== # Execute operation # ================================================================================== - command = f"mkdir -p {bbdir}/compiler/build" + command = f"mkdir -p {bbdir}/compiler/thirdparty/buddy-mlir/build" result = stream_run_logger( cmd=command, logger=ctx.logger, stdout_prefix="compiler build", stderr_prefix="compiler build", ) - command = f"cd {bbdir}/compiler/build && ninja -j{os.cpu_count()}" + command = f"cd {bbdir}/compiler/thirdparty/buddy-mlir/build && ninja -j{os.cpu_count()}" result = stream_run_logger( cmd=command, logger=ctx.logger, From f757093d0a982fd59338e0265df177cc82532069 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 30 Apr 2026 01:34:37 +0800 Subject: [PATCH 043/196] feat: add P2E workflow with new API and event steps for cleaning and generating Verilog files, including improved build directory management --- .../mill/01_verilator_verilog_event.step.py | 11 +- .../steps/mill/05_p2e_verilog_event.step.py | 122 ++++++++++++++++++ bbdev/api/steps/p2e/00_clean_api.step.py | 16 +++ bbdev/api/steps/p2e/00_clean_event.step.py | 45 +++++++ bbdev/api/steps/p2e/01_verilog_api.step.py | 27 ++++ .../steps/verilator/01_clean_event.step.py | 10 +- .../steps/verilator/02_verilog_api.step.py | 4 +- .../api/steps/verilator/03_build_api.step.py | 5 + .../steps/verilator/03_build_event.step.py | 11 +- .../api/steps/verilator/04_sim_event.step.py | 9 +- bbdev/api/utils/path.py | 13 ++ bbdev/bbdev | 17 ++- 12 files changed, 271 insertions(+), 19 deletions(-) create mode 100644 bbdev/api/steps/mill/05_p2e_verilog_event.step.py create mode 100644 bbdev/api/steps/p2e/00_clean_api.step.py create mode 100644 bbdev/api/steps/p2e/00_clean_event.step.py create mode 100644 bbdev/api/steps/p2e/01_verilog_api.step.py diff --git a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py index b581717ba..8cd515b60 100644 --- a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py @@ -8,7 +8,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path +from utils.path import get_buckyball_path, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id @@ -43,7 +43,11 @@ def prepare_verilator_verilog(build_dir: str, arch_dir: str, logger): async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/") + build_dir = get_verilator_build_dir( + bbdir, + input_data.get("config"), + input_data.get("output_dir"), + ) arch_dir = f"{bbdir}/arch" config_name = input_data.get("config") @@ -63,6 +67,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return failure_result ctx.logger.info(f"Using configuration: {config_name}") + ctx.logger.info(f"Using build directory: {build_dir}") if input_data.get("balltype"): command = ( @@ -96,7 +101,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if input_data.get("from_run_workflow"): await ctx.enqueue( - {"topic": "verilator.build", "data": {**input_data, "task": "run"}} + {"topic": "verilator.build", "data": {**input_data, "output_dir": build_dir, "task": "run"}} ) return diff --git a/bbdev/api/steps/mill/05_p2e_verilog_event.step.py b/bbdev/api/steps/mill/05_p2e_verilog_event.step.py new file mode 100644 index 000000000..aa2bc8919 --- /dev/null +++ b/bbdev/api/steps/mill/05_p2e_verilog_event.step.py @@ -0,0 +1,122 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.path import get_verilator_build_dir +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "make p2e verilog", + "description": "Generate DigitalTop RTL and P2E DDR4 backdoor wrapper", + "flows": ["p2e"], + "triggers": [queue("p2e.verilog")], + "enqueues": [], +} + + +def cleanup_strays(arch_dir: str): + for stray in ["P2EHarness.sv", "P2ETop.v", "P2ETopWrapper.sv"]: + path = os.path.join(arch_dir, stray) + if os.path.exists(path): + os.remove(path) + + +def normalize_p2e_timescale(build_dir: str, logger): + patched = 0 + for root, _, files in os.walk(build_dir): + for name in files: + if not name.endswith((".v", ".sv")): + continue + + path = os.path.join(root, name) + with open(path, "r") as f: + content = f.read() + + if "`timescale" in content: + continue + + with open(path, "w") as f: + f.write("`timescale 1ns/1ps\n") + f.write(content) + patched += 1 + + logger.info(f"Normalized P2E timescale in {patched} generated Verilog files") + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + arch_dir = f"{bbdir}/arch" + config_name = input_data.get("config", "sims.p2e.P2EToyConfig") + build_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("output_dir")) + + ctx.logger.info(f"Using P2E configuration: {config_name}") + ctx.logger.info(f"Using P2E output directory: {build_dir}") + + os.makedirs(build_dir, exist_ok=True) + cleanup_strays(arch_dir) + + common_firtool_opts = ( + "--disable-annotation-unknown " + "--strip-debug-info " + "-O=debug " + f"--split-verilog -o={build_dir}" + ) + + digital_command = ( + f"mill -i __.test.runMain sims.p2e.Elaborate {config_name} " + f"{common_firtool_opts}" + ) + digital_result = stream_run_logger( + cmd=digital_command, + logger=ctx.logger, + cwd=arch_dir, + stdout_prefix="p2e digital verilog", + stderr_prefix="p2e digital verilog", + ) + if digital_result.returncode != 0: + cleanup_strays(arch_dir) + await check_result( + ctx, + digital_result.returncode, + continue_run=False, + extra_fields={"task": "verilog", "step": "digital", "config": config_name}, + trace_id=origin_tid, + ) + return + + top_command = ( + f"mill -i __.test.runMain sims.p2e.ElaborateP2ETop " + f"{common_firtool_opts}" + ) + top_result = stream_run_logger( + cmd=top_command, + logger=ctx.logger, + cwd=arch_dir, + stdout_prefix="p2e top verilog", + stderr_prefix="p2e top verilog", + ) + + cleanup_strays(arch_dir) + if top_result.returncode == 0: + normalize_p2e_timescale(build_dir, ctx.logger) + + await check_result( + ctx, + top_result.returncode, + continue_run=False, + extra_fields={ + "task": "verilog", + "config": config_name, + "output_dir": build_dir, + "top_module": "P2ETop", + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/p2e/00_clean_api.step.py b/bbdev/api/steps/p2e/00_clean_api.step.py new file mode 100644 index 000000000..a4faaf6e9 --- /dev/null +++ b/bbdev/api/steps/p2e/00_clean_api.step.py @@ -0,0 +1,16 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "p2e-clean-api", + "description": "Clean P2E build directory", + "flows": ["p2e"], + "triggers": [api("POST", "/p2e/clean")], + "enqueues": ["p2e.clean"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + await ctx.enqueue({"topic": "p2e.clean", "data": {**body, "task": "clean", "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) + diff --git a/bbdev/api/steps/p2e/00_clean_event.step.py b/bbdev/api/steps/p2e/00_clean_event.step.py new file mode 100644 index 000000000..5b659fdeb --- /dev/null +++ b/bbdev/api/steps/p2e/00_clean_event.step.py @@ -0,0 +1,45 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "p2e-clean", + "description": "Clean P2E build directory", + "flows": ["p2e"], + "triggers": [queue("p2e.clean")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + config_name = input_data.get("config", "sims.p2e.P2EToyConfig") + build_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("output_dir")) + + command = f"rm -rf {build_dir}" + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="p2e clean", + stderr_prefix="p2e clean", + ) + + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={"task": "clean", "config": config_name, "output_dir": build_dir}, + trace_id=origin_tid, + ) + diff --git a/bbdev/api/steps/p2e/01_verilog_api.step.py b/bbdev/api/steps/p2e/01_verilog_api.step.py new file mode 100644 index 000000000..1642d3448 --- /dev/null +++ b/bbdev/api/steps/p2e/01_verilog_api.step.py @@ -0,0 +1,27 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path +from utils.path import get_verilator_build_dir + +config = { + "name": "p2e-verilog-api", + "description": "Generate SystemVerilog for P2E DDR4 backdoor", + "flows": ["p2e"], + "triggers": [api("POST", "/p2e/verilog")], + "enqueues": ["p2e.verilog"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = req.body or {} + + config_name = body.get("config", "sims.p2e.P2EToyConfig") + output_dir = get_verilator_build_dir(bbdir, config_name, body.get("output_dir")) + + data = { + "config": config_name, + "output_dir": output_dir, + } + await ctx.enqueue({"topic": "p2e.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/01_clean_event.step.py b/bbdev/api/steps/verilator/01_clean_event.step.py index 4437df7ad..434481ba1 100644 --- a/bbdev/api/steps/verilator/01_clean_event.step.py +++ b/bbdev/api/steps/verilator/01_clean_event.step.py @@ -8,7 +8,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path +from utils.path import get_buckyball_path, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id @@ -27,7 +27,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - build_dir = f"{bbdir}/arch/build" + build_dir = get_verilator_build_dir( + bbdir, + input_data.get("config"), + input_data.get("output_dir"), + ) # ================================================================================== # Execute operation # ================================================================================== @@ -55,7 +59,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ================================================================================== if input_data.get("from_run_workflow"): await ctx.enqueue( - {"topic": "verilator.verilog", "data": {**input_data, "task": "run"}} + {"topic": "verilator.verilog", "data": {**input_data, "output_dir": build_dir, "task": "run"}} ) return diff --git a/bbdev/api/steps/verilator/02_verilog_api.step.py b/bbdev/api/steps/verilator/02_verilog_api.step.py index d128818ee..3b3af1734 100644 --- a/bbdev/api/steps/verilator/02_verilog_api.step.py +++ b/bbdev/api/steps/verilator/02_verilog_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import get_buckyball_path +from utils.path import get_buckyball_path, get_verilator_build_dir config = { "name": "verilator-verilog-api", @@ -30,7 +30,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "config": config_name, "balltype": body.get("balltype"), - "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), + "output_dir": get_verilator_build_dir(bbdir, config_name, body.get("output_dir")), } await ctx.enqueue({"topic": "verilator.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/03_build_api.step.py b/bbdev/api/steps/verilator/03_build_api.step.py index 092dd861f..f88f07b2d 100644 --- a/bbdev/api/steps/verilator/03_build_api.step.py +++ b/bbdev/api/steps/verilator/03_build_api.step.py @@ -1,5 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.path import get_buckyball_path, get_verilator_build_dir + config = { "name": "verilator-build-api", "description": "build verilator executable", @@ -10,9 +12,12 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() body = request.body or {} data = { "jobs": body.get("jobs", 16), + "config": body.get("config"), + "output_dir": get_verilator_build_dir(bbdir, body.get("config"), body.get("output_dir")), } await ctx.enqueue({"topic": "verilator.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/03_build_event.step.py b/bbdev/api/steps/verilator/03_build_event.step.py index 64a56a9ac..9577af237 100644 --- a/bbdev/api/steps/verilator/03_build_event.step.py +++ b/bbdev/api/steps/verilator/03_build_event.step.py @@ -10,7 +10,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path +from utils.path import get_buckyball_path, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id @@ -27,8 +27,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" - build_dir = f"{arch_dir}/build" + build_dir = get_verilator_build_dir( + bbdir, + input_data.get("config"), + input_data.get("output_dir"), + ) coverage = input_data.get("coverage", False) + ctx.logger.info(f"Using build directory: {build_dir}") # ================================================================================== # Find sources @@ -171,7 +176,7 @@ def pkg_config(flag, pkg): # ================================================================================== if input_data.get("from_run_workflow"): await ctx.enqueue( - {"topic": "verilator.sim", "data": {**input_data, "task": "run"}} + {"topic": "verilator.sim", "data": {**input_data, "output_dir": build_dir, "task": "run"}} ) return diff --git a/bbdev/api/steps/verilator/04_sim_event.step.py b/bbdev/api/steps/verilator/04_sim_event.step.py index 3304a3c17..997381537 100644 --- a/bbdev/api/steps/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -10,7 +10,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path +from utils.path import get_buckyball_path, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id @@ -31,7 +31,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" - build_dir = f"{arch_dir}/build" + build_dir = get_verilator_build_dir( + bbdir, + input_data.get("config"), + input_data.get("output_dir"), + ) + ctx.logger.info(f"Using build directory: {build_dir}") timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index cefb76881..03da675fe 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -1,7 +1,20 @@ import os +import re def get_buckyball_path(): current_dir = os.path.dirname(__file__) # bbdev/api/utils -> bbdev/api -> bbdev -> buckyball return os.path.dirname(os.path.dirname(os.path.dirname(current_dir))) + + +def get_verilator_build_dir(bbdir, config=None, output_dir=None): + if output_dir: + return output_dir + + if config and config != "None": + name = re.sub(r"[^A-Za-z0-9_.-]+", "_", config).strip("._") + if name: + return f"{bbdir}/arch/build/{name}" + + return f"{bbdir}/arch/build" diff --git a/bbdev/bbdev b/bbdev/bbdev index 21781460c..03e1011c1 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -23,11 +23,11 @@ NO_PROXY = {"http": None, "https": None, "all": None} # Operations with string values accept sub-arguments; True means boolean-only (no sub-args). WORKFLOW_COMMANDS = { "verilator": { - "clean": True, - "verilog": 'Generate verilog files. Args: "[--balltype ] [--job ] [--batch]"', - "build": 'Build simulation executable. Args: "[--job ]"', - "sim": 'Run simulation. Args: "--binary [--batch]"', - "run": 'Integrated build+sim+run. Args: "--binary [--batch] [--job ]"', + "clean": 'Clean build directory. Args: "[--config ] [--output-dir ]"', + "verilog": 'Generate verilog files. Args: "[--config ] [--output-dir ] [--balltype ]"', + "build": 'Build simulation executable. Args: "[--job ] [--config ] [--output-dir ]"', + "sim": 'Run simulation. Args: "--binary [--batch] [--config ] [--output-dir ]"', + "run": 'Integrated build+sim+run. Args: "--binary [--batch] [--job ] [--config ] [--output-dir ]"', "cosim": 'Run cosimulation. Args: "--binary [--batch]"', }, "vcs": { @@ -74,6 +74,10 @@ WORKFLOW_COMMANDS = { "flashbitstream": 'Flash bitstream onto AU280. Args: "[--bitstream ] [--serial ]"', "runworkload": 'Run workload on AU280. Args: "[--workload ] [--board ] [--timeout ] [--uart ]"', }, + "p2e": { + "clean": 'Clean P2E build directory. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', + "verilog": 'Generate SystemVerilog for P2E DDR4 backdoor. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', + }, "dc": { "run": 'Run Design Compiler synthesis. Args: "[--top ] [--config ] [--output-dir ]"', }, @@ -264,7 +268,8 @@ def wait_for_http(url: str, check_fn, timeout: int, error_msg: str): print(error_msg) return False - +# If you find that bbdev has failed without any logs, +# you can disable these log filters. These contain the bbdev runtime logs. _ENGINE_INFRA_MODULES = ( "iii::modules::worker", "iii::modules::config", From 2e2e82f0cc3ee6140d76a939640928c42d386c47 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 3 May 2026 21:17:32 +0800 Subject: [PATCH 044/196] feat: implement bin to hex conversion in build step for p2e to load into ddr --- bbdev/api/steps/kernel/01_build_event.step.py | 22 +++- bbdev/api/steps/kernel/scripts/bin_to_hex.py | 104 ++++++++++++++++++ 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 bbdev/api/steps/kernel/scripts/bin_to_hex.py diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index 75df33e8c..be575cb1c 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -12,6 +12,12 @@ from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id +# Import bin_to_hex converter +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) +from bin_to_hex import bin_to_hex + config = { "name": "kernel-build", "description": "build RISC-V kernel + rootfs for Pegasus via bb-tests/workloads/lib/kernel", @@ -55,5 +61,19 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="marshal build", ) - await check_result(ctx, result.returncode, continue_run=False, trace_id=origin_tid) + if result.returncode != 0: + await check_result(ctx, result.returncode, continue_run=False, trace_id=origin_tid) + return + + # Convert bin to hex for P2E memory backdoor + bin_file = os.path.join(output_dir, "pegasus-bin") + hex_file = os.path.join(output_dir, "pegasus.hex") + + if os.path.exists(bin_file): + ctx.logger.info(f"Converting {bin_file} to Verilog hex format...") + success = bin_to_hex(bin_file, hex_file) + if not success: + ctx.logger.warning("Failed to convert bin to hex, but continuing...") + + await check_result(ctx, 0, continue_run=False, trace_id=origin_tid) diff --git a/bbdev/api/steps/kernel/scripts/bin_to_hex.py b/bbdev/api/steps/kernel/scripts/bin_to_hex.py new file mode 100644 index 000000000..a5f5d6a18 --- /dev/null +++ b/bbdev/api/steps/kernel/scripts/bin_to_hex.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 + +import subprocess +import sys +import os + +def bin_to_hex(bin_file, hex_file, objcopy_path="riscv64-unknown-elf-objcopy"): + """ + 将 RISC-V bin 文件转换为 Verilog readmemh 格式 + 地址格式：@0xNNNNNNNN + """ + print(f"Converting {bin_file} -> {hex_file}") + + # Step 1: bin -> Intel HEX + temp_hex = f"{bin_file}.tmp.hex" + try: + cmd = [objcopy_path, "-O", "ihex", bin_file, temp_hex] + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode != 0: + print(f"Error: objcopy failed: {result.stderr}") + return False + + # Step 2: Parse Intel HEX and convert to readmemh format + memory_data = {} + current_base_address = 0 + + with open(temp_hex, 'r') as f: + for line in f: + line = line.strip() + if not line or not line.startswith(':'): + continue + + record = line[1:] + if len(record) < 8: + continue + + byte_count = int(record[0:2], 16) + address = int(record[2:6], 16) + record_type = int(record[6:8], 16) + + if record_type == 0x00: # Data record + data_start = 8 + for i in range(byte_count): + if data_start + 2 <= len(record) - 2: + byte_data = record[data_start:data_start+2] + memory_address = current_base_address + address + i + memory_data[memory_address] = int(byte_data, 16) + data_start += 2 + + elif record_type == 0x04: # Extended linear address + if byte_count == 2 and len(record) >= 12: + high_address = int(record[8:12], 16) + current_base_address = high_address << 16 + + if not memory_data: + print("Error: No valid data found") + return False + + # Step 3: Write readmemh format + with open(hex_file, 'w') as f: + sorted_addresses = sorted(memory_data.keys()) + current_address = None + + for addr in sorted_addresses: + # New address section if not continuous + if current_address is None or addr != current_address + 1: + f.write(f"@0x{addr:08X}\n") + current_address = addr + else: + current_address = addr + + byte_value = memory_data[addr] + f.write(f"{byte_value:02X}\n") + + print(f"Successfully generated {hex_file}") + print(f"Address range: 0x{sorted_addresses[0]:08X} - 0x{sorted_addresses[-1]:08X}") + print(f"Total bytes: {len(memory_data)}") + return True + + except Exception as e: + print(f"Error: Conversion failed - {e}") + return False + finally: + if os.path.exists(temp_hex): + os.remove(temp_hex) + +def main(): + if len(sys.argv) != 3: + print("Usage: bin_to_hex.py ") + sys.exit(1) + + bin_file = sys.argv[1] + hex_file = sys.argv[2] + + if not os.path.exists(bin_file): + print(f"Error: File not found: {bin_file}") + sys.exit(1) + + success = bin_to_hex(bin_file, hex_file) + sys.exit(0 if success else 1) + +if __name__ == "__main__": + main() From 107340dfc854e8f0f27635c850f08f3f601d475b Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 11 May 2026 23:31:01 +0800 Subject: [PATCH 045/196] feat: add enumeratefpgas API and fix out-of-date steps for FireSim --- .../firesim/00_enumeratefpgas_api.step.py | 15 +++++ .../firesim/00_enumeratefpgas_event.step.py | 63 +++++++++++++++++++ .../firesim/01_buildbitstream_event.step.py | 2 +- .../steps/firesim/02_infrasetup_event.step.py | 2 +- .../firesim/03_runworkload_event.step.py | 2 +- .../api/steps/firesim/scripts/firesim_env.py | 26 ++++++++ bbdev/bbdev | 1 + 7 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 bbdev/api/steps/firesim/00_enumeratefpgas_api.step.py create mode 100644 bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py create mode 100644 bbdev/api/steps/firesim/scripts/firesim_env.py diff --git a/bbdev/api/steps/firesim/00_enumeratefpgas_api.step.py b/bbdev/api/steps/firesim/00_enumeratefpgas_api.step.py new file mode 100644 index 000000000..9a88bfcc1 --- /dev/null +++ b/bbdev/api/steps/firesim/00_enumeratefpgas_api.step.py @@ -0,0 +1,15 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "firesim-enumeratefpgas-api", + "description": "enumerate FPGAs", + "flows": ["firesim"], + "triggers": [api("POST", "/firesim/enumeratefpgas")], + "enqueues": ["firesim.enumeratefpgas"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + await ctx.enqueue({"topic": "firesim.enumeratefpgas", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py b/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py new file mode 100644 index 000000000..2373a5686 --- /dev/null +++ b/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py @@ -0,0 +1,63 @@ +import os +import sys + +from motia import FlowContext, queue + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +# Import firesim_env from scripts subdirectory +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "scripts")) +from firesim_env import setup_firesim_env + +config = { + "name": "firesim-enumeratefpgas", + "description": "enumerate FPGAs", + "flows": ["firesim"], + "triggers": [queue("firesim.enumeratefpgas")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + script_dir = f"{bbdir}/bbdev/api/steps/firesim/scripts" + yaml_dir = f"{script_dir}/yaml" + + # Setup FireSim environment variables and SSH agent + env = setup_firesim_env() + + # ================================================================================== + # Execute operation + # ================================================================================== + command = f"firesim enumeratefpgas" + command += f" -a {yaml_dir}/config_hwdb.yaml" + command += f" -b {yaml_dir}/config_build.yaml" + command += f" -r {yaml_dir}/config_build_recipes.yaml" + command += f" -c {yaml_dir}/config_runtime.yaml" + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + stdout_prefix="firesim enumeratefpgas", + stderr_prefix="firesim enumeratefpgas", + env=env, + ) + + # ================================================================================== + # Return result to API + # ================================================================================== + success_result, failure_result = await check_result( + ctx, result.returncode, continue_run=False, trace_id=origin_tid) + + # ================================================================================== + # Continue routing + # ================================================================================== + + return diff --git a/bbdev/api/steps/firesim/01_buildbitstream_event.step.py b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py index 4550ee509..98fc8fffe 100644 --- a/bbdev/api/steps/firesim/01_buildbitstream_event.step.py +++ b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py @@ -24,7 +24,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - script_dir = f"{bbdir}/workflow/steps/firesim/scripts" + script_dir = f"{bbdir}/bbdev/api/steps/firesim/scripts" yaml_dir = f"{script_dir}/yaml" # ================================================================================== # Execute operation diff --git a/bbdev/api/steps/firesim/02_infrasetup_event.step.py b/bbdev/api/steps/firesim/02_infrasetup_event.step.py index 2d1f3167b..14781859c 100644 --- a/bbdev/api/steps/firesim/02_infrasetup_event.step.py +++ b/bbdev/api/steps/firesim/02_infrasetup_event.step.py @@ -24,7 +24,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - script_dir = f"{bbdir}/workflow/steps/firesim/scripts" + script_dir = f"{bbdir}/bbdev/api/steps/firesim/scripts" yaml_dir = f"{script_dir}/yaml" # ================================================================================== # Execute operation diff --git a/bbdev/api/steps/firesim/03_runworkload_event.step.py b/bbdev/api/steps/firesim/03_runworkload_event.step.py index e9a57aee7..cec61e2a5 100644 --- a/bbdev/api/steps/firesim/03_runworkload_event.step.py +++ b/bbdev/api/steps/firesim/03_runworkload_event.step.py @@ -24,7 +24,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - script_dir = f"{bbdir}/workflow/steps/firesim/scripts" + script_dir = f"{bbdir}/bbdev/api/steps/firesim/scripts" yaml_dir = f"{script_dir}/yaml" # ================================================================================== # Execute operation diff --git a/bbdev/api/steps/firesim/scripts/firesim_env.py b/bbdev/api/steps/firesim/scripts/firesim_env.py new file mode 100644 index 000000000..144d0bb15 --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/firesim_env.py @@ -0,0 +1,26 @@ +"""FireSim environment setup.""" + +import os +import subprocess + + +def setup_firesim_env() -> dict: + """Setup FireSim env vars + SSH agent. Returns env dict for subprocess.""" + env = os.environ.copy() + env["FIRESIM_SOURCED"] = "1" + env["FIRESIM_RUNFARM_PREFIX"] = "" + + key = os.path.expanduser("~/.ssh/firesim.pem") + if os.path.exists(key) and "firesim.pem" not in subprocess.run( + ["ssh-add", "-l"], capture_output=True, text=True + ).stdout: + if not env.get("SSH_AUTH_SOCK"): + for line in subprocess.run( + ["ssh-agent", "-s"], capture_output=True, text=True + ).stdout.splitlines(): + if "=" in line: + k, v = line.split("=", 1) + env[k] = v.split(";", 1)[0] + subprocess.run(["ssh-add", key], env=env, capture_output=True) + + return env diff --git a/bbdev/bbdev b/bbdev/bbdev index 03e1011c1..f8c2f430b 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -53,6 +53,7 @@ WORKFLOW_COMMANDS = { "build": "Build RISC-V kernel + rootfs for Pegasus (cmake kernel-build).", }, "firesim": { + "enumeratefpgas": "Enumerate FPGAs.", "buildbitstream": "Build bitstream.", "infrasetup": "Infrasetup.", "runworkload": "Run workload.", From 800a41d87f1cbaaf29aa62c8e3c1860352a0e1a9 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 12 May 2026 16:32:36 +0800 Subject: [PATCH 046/196] feat: integrate FireSim environment setup across multiple event steps --- .../firesim/01_buildbitstream_event.step.py | 12 ++++++++++ .../steps/firesim/02_infrasetup_event.step.py | 9 ++++++++ .../firesim/03_runworkload_event.step.py | 9 ++++++++ .../api/steps/firesim/scripts/firesim_env.py | 22 ++++++++++++++++++- .../scripts/makefrag/firesim/config.mk | 8 ++++--- 5 files changed, 56 insertions(+), 4 deletions(-) diff --git a/bbdev/api/steps/firesim/01_buildbitstream_event.step.py b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py index 98fc8fffe..204ba20b2 100644 --- a/bbdev/api/steps/firesim/01_buildbitstream_event.step.py +++ b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py @@ -12,6 +12,13 @@ from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id +# Add scripts directory to path for firesim_env import +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from firesim_env import setup_firesim_env + config = { "name": "firesim-buildbitstream", "description": "build bitstream", @@ -26,6 +33,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() script_dir = f"{bbdir}/bbdev/api/steps/firesim/scripts" yaml_dir = f"{script_dir}/yaml" + + # Setup FireSim environment variables and SSH agent + env = setup_firesim_env() + # ================================================================================== # Execute operation # ================================================================================== @@ -39,6 +50,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: logger=ctx.logger, stdout_prefix="firesim buildbitstream", stderr_prefix="firesim buildbitstream", + env=env, ) # ================================================================================== diff --git a/bbdev/api/steps/firesim/02_infrasetup_event.step.py b/bbdev/api/steps/firesim/02_infrasetup_event.step.py index 14781859c..165747268 100644 --- a/bbdev/api/steps/firesim/02_infrasetup_event.step.py +++ b/bbdev/api/steps/firesim/02_infrasetup_event.step.py @@ -12,6 +12,10 @@ from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id +# Import firesim_env from scripts subdirectory +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "scripts")) +from firesim_env import setup_firesim_env + config = { "name": "firesim-infrasetup", "description": "infrasetup", @@ -26,6 +30,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() script_dir = f"{bbdir}/bbdev/api/steps/firesim/scripts" yaml_dir = f"{script_dir}/yaml" + + # Setup FireSim environment variables and SSH agent + env = setup_firesim_env() + # ================================================================================== # Execute operation # ================================================================================== @@ -39,6 +47,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: logger=ctx.logger, stdout_prefix="firesim infrasetup", stderr_prefix="firesim infrasetup", + env=env, ) # ================================================================================== diff --git a/bbdev/api/steps/firesim/03_runworkload_event.step.py b/bbdev/api/steps/firesim/03_runworkload_event.step.py index cec61e2a5..e5c9e763d 100644 --- a/bbdev/api/steps/firesim/03_runworkload_event.step.py +++ b/bbdev/api/steps/firesim/03_runworkload_event.step.py @@ -12,6 +12,10 @@ from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id +# Import firesim_env from scripts subdirectory +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "scripts")) +from firesim_env import setup_firesim_env + config = { "name": "firesim-runworkload", "description": "run workload", @@ -26,6 +30,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() script_dir = f"{bbdir}/bbdev/api/steps/firesim/scripts" yaml_dir = f"{script_dir}/yaml" + + # Setup FireSim environment variables and SSH agent + env = setup_firesim_env() + # ================================================================================== # Execute operation # ================================================================================== @@ -39,6 +47,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: logger=ctx.logger, stdout_prefix="firesim runworkload", stderr_prefix="firesim runworkload", + env=env, ) # ================================================================================== diff --git a/bbdev/api/steps/firesim/scripts/firesim_env.py b/bbdev/api/steps/firesim/scripts/firesim_env.py index 144d0bb15..c95d11c8b 100644 --- a/bbdev/api/steps/firesim/scripts/firesim_env.py +++ b/bbdev/api/steps/firesim/scripts/firesim_env.py @@ -7,9 +7,28 @@ def setup_firesim_env() -> dict: """Setup FireSim env vars + SSH agent. Returns env dict for subprocess.""" env = os.environ.copy() - env["FIRESIM_SOURCED"] = "1" + env["FIRESIM_SOURCED"] = "1" # For firesim Python CLI + env["FIRESIM_ENV_SOURCED"] = "1" # For firesim Makefile env["FIRESIM_RUNFARM_PREFIX"] = "" + # Create minimal env.sh for firesim if it doesn't exist + firesim_dir = os.path.join( + os.environ.get("HOME", "/home/mio"), + "Code/buckyball/arch/thirdparty/chipyard/sims/firesim" + ) + env_sh = os.path.join(firesim_dir, "env.sh") + if not os.path.exists(env_sh): + riscv_path = env.get("RISCV", "/home/mio/Code/buckyball/result") + with open(env_sh, "w") as f: + f.write(f"""#!/usr/bin/env bash +# Minimal env.sh for FireSim (auto-generated by firesim_env.py) + +export FIRESIM_ENV_SOURCED=1 +export FS_DIR="{firesim_dir}" +export RISCV="{riscv_path}" +""") + + # Setup SSH agent for localhost FPGA enumeration key = os.path.expanduser("~/.ssh/firesim.pem") if os.path.exists(key) and "firesim.pem" not in subprocess.run( ["ssh-add", "-l"], capture_output=True, text=True @@ -24,3 +43,4 @@ def setup_firesim_env() -> dict: subprocess.run(["ssh-add", key], env=env, capture_output=True) return env + diff --git a/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk b/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk index 555710686..9c16f60f2 100644 --- a/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk +++ b/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk @@ -4,7 +4,9 @@ # Only used in this projects makefrags makefile_path := $(abspath $(lastword $(MAKEFILE_LIST))) makefile_dir := $(patsubst %/,%,$(dir $(makefile_path))) -chipyard_dir := $(abspath $(makefile_dir)/../../../../../../arch/thirdparty/chipyard) +# makefile_dir = .../bbdev/api/steps/firesim/scripts/makefrag/firesim/ +# Need to go up 7 levels to reach buckyball root +chipyard_dir := $(abspath $(makefile_dir)/../../../../../../../arch/thirdparty/chipyard) # These point at the main class of the target's Chisel generator DESIGN_PACKAGE ?= firechip.chip @@ -23,8 +25,8 @@ PLATFORM_CONFIG ?= BaseF1Config TARGET_SBT_PROJECT := buckyball # Point to our project directory -TARGET_SBT_DIR := $(abspath $(makefile_dir)/../../../../../../arch) -TARGET_SOURCE_DIRS := $(abspath $(makefile_dir)/../../../../../../arch/src/main/scala) +TARGET_SBT_DIR := $(abspath $(makefile_dir)/../../../../../../../arch) +TARGET_SOURCE_DIRS := $(abspath $(makefile_dir)/../../../../../../../arch/src/main/scala) # SBT launcher SBT ?= java -jar $(chipyard_dir)/scripts/sbt-launch.jar $(SBT_OPTS) From af4435c90e5e400a9b587598c683099026916f8e Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 12 May 2026 23:53:30 +0800 Subject: [PATCH 047/196] fix: update AU280 board_part version in FireSim TCL script for Vivado compatibility --- .../firesim/01_buildbitstream_event.step.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/bbdev/api/steps/firesim/01_buildbitstream_event.step.py b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py index 204ba20b2..65d46bca6 100644 --- a/bbdev/api/steps/firesim/01_buildbitstream_event.step.py +++ b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py @@ -37,6 +37,23 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # Setup FireSim environment variables and SSH agent env = setup_firesim_env() + # ================================================================================== + # Fix AU280 board_part version (1.2 -> 1.0) for Vivado compatibility + # ================================================================================== + au280_tcl = f"{bbdir}/arch/thirdparty/chipyard/sims/firesim/platforms/xilinx_alveo_u280/cl_firesim/scripts/au280.tcl" + if os.path.exists(au280_tcl): + with open(au280_tcl, "r") as f: + content = f.read() + # Replace xilinx.com:au280:part0:1.2 with xilinx.com:au280:part0:1.0 + fixed_content = content.replace( + "xilinx.com:au280:part0:1.2", + "xilinx.com:au280:part0:1.0" + ) + if content != fixed_content: + with open(au280_tcl, "w") as f: + f.write(fixed_content) + ctx.logger.info("Fixed AU280 board_part version: 1.2 -> 1.0") + # ================================================================================== # Execute operation # ================================================================================== From ff249ea5c2621d54a74faef66b1828682010f4ab Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 14 May 2026 06:21:55 +0800 Subject: [PATCH 048/196] feat: add bebop and verilator workflows with new API and event steps for building, simulating, and managing P2E bitstreams and workloads --- .../api/steps/bebop/bemu/00_build_api.step.py | 15 + .../steps/bebop/bemu/00_build_event.step.py | 53 +++ bbdev/api/steps/bebop/bemu/02_sim_api.step.py | 27 ++ .../api/steps/bebop/bemu/02_sim_event.step.py | 81 +++++ .../{ => bebop}/p2e/00_clean_api.step.py | 11 +- .../{ => bebop}/p2e/00_clean_event.step.py | 13 +- .../{ => bebop}/p2e/01_verilog_api.step.py | 10 +- .../bebop/p2e/02_buildbitstream_api.step.py | 31 ++ .../bebop/p2e/02_buildbitstream_event.step.py | 114 ++++++ .../bebop/p2e/03_runworkload_api.step.py | 31 ++ .../bebop/p2e/03_runworkload_event.step.py | 110 ++++++ .../bebop/verilator/01_clean_api.step.py | 26 ++ .../bebop/verilator/01_clean_event.step.py | 53 +++ .../bebop/verilator/02_verilog_api.step.py | 35 ++ .../bebop/verilator/03_build_api.step.py | 26 ++ .../bebop/verilator/03_build_event.step.py | 77 ++++ .../steps/bebop/verilator/04_sim_api.step.py | 27 ++ .../bebop/verilator/04_sim_event.step.py | 100 +++++ .../steps/bebop/verilator/05_run_api.step.py | 48 +++ bbdev/api/steps/kernel/scripts/bin_to_hex.py | 29 +- .../mill/02_pegasus_verilog_event.step.py | 165 --------- .../06_bebop_verilator_verilog_event.step.py | 111 ++++++ ....py => 07_bebop_p2e_verilog_event.step.py} | 23 +- .../api/steps/pegasus/01_verilog_api.step.py | 25 -- .../pegasus/02_buildbitstream_api.step.py | 15 - .../pegasus/02_buildbitstream_event.step.py | 83 ----- .../pegasus/03_flashbitstream_api.step.py | 15 - .../pegasus/03_flashbitstream_event.step.py | 343 ------------------ .../steps/pegasus/04_runworkload_api.step.py | 15 - .../pegasus/04_runworkload_event.step.py | 171 --------- bbdev/api/tests/test_all_api_serial.py | 16 +- ...erilog.py => test_api_bebop_bemu_build.py} | 2 +- bbdev/api/tests/test_api_bebop_bemu_sim.py | 3 + .../test_api_bebop_p2e_buildbitstream.py | 3 + bbdev/api/tests/test_api_bebop_p2e_clean.py | 3 + .../tests/test_api_bebop_p2e_runworkload.py | 3 + bbdev/api/tests/test_api_bebop_p2e_verilog.py | 3 + .../tests/test_api_bebop_verilator_build.py | 3 + .../tests/test_api_bebop_verilator_clean.py | 3 + .../api/tests/test_api_bebop_verilator_run.py | 3 + .../api/tests/test_api_bebop_verilator_sim.py | 3 + .../tests/test_api_bebop_verilator_verilog.py | 3 + .../tests/test_api_firesim_enumeratefpgas.py | 3 + .../tests/test_api_pegasus_buildbitstream.py | 3 - .../tests/test_api_pegasus_flashbitstream.py | 3 - .../api/tests/test_api_pegasus_runworkload.py | 3 - bbdev/api/utils/stream_run.py | 3 + bbdev/bbdev | 23 +- 48 files changed, 1085 insertions(+), 884 deletions(-) create mode 100644 bbdev/api/steps/bebop/bemu/00_build_api.step.py create mode 100644 bbdev/api/steps/bebop/bemu/00_build_event.step.py create mode 100644 bbdev/api/steps/bebop/bemu/02_sim_api.step.py create mode 100644 bbdev/api/steps/bebop/bemu/02_sim_event.step.py rename bbdev/api/steps/{ => bebop}/p2e/00_clean_api.step.py (52%) rename bbdev/api/steps/{ => bebop}/p2e/00_clean_event.step.py (84%) rename bbdev/api/steps/{ => bebop}/p2e/01_verilog_api.step.py (72%) create mode 100644 bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py create mode 100644 bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py create mode 100644 bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py create mode 100644 bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py create mode 100644 bbdev/api/steps/bebop/verilator/01_clean_api.step.py create mode 100644 bbdev/api/steps/bebop/verilator/01_clean_event.step.py create mode 100644 bbdev/api/steps/bebop/verilator/02_verilog_api.step.py create mode 100644 bbdev/api/steps/bebop/verilator/03_build_api.step.py create mode 100644 bbdev/api/steps/bebop/verilator/03_build_event.step.py create mode 100644 bbdev/api/steps/bebop/verilator/04_sim_api.step.py create mode 100644 bbdev/api/steps/bebop/verilator/04_sim_event.step.py create mode 100644 bbdev/api/steps/bebop/verilator/05_run_api.step.py delete mode 100644 bbdev/api/steps/mill/02_pegasus_verilog_event.step.py create mode 100644 bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py rename bbdev/api/steps/mill/{05_p2e_verilog_event.step.py => 07_bebop_p2e_verilog_event.step.py} (83%) delete mode 100644 bbdev/api/steps/pegasus/01_verilog_api.step.py delete mode 100644 bbdev/api/steps/pegasus/02_buildbitstream_api.step.py delete mode 100644 bbdev/api/steps/pegasus/02_buildbitstream_event.step.py delete mode 100644 bbdev/api/steps/pegasus/03_flashbitstream_api.step.py delete mode 100644 bbdev/api/steps/pegasus/03_flashbitstream_event.step.py delete mode 100644 bbdev/api/steps/pegasus/04_runworkload_api.step.py delete mode 100644 bbdev/api/steps/pegasus/04_runworkload_event.step.py rename bbdev/api/tests/{test_api_pegasus_verilog.py => test_api_bebop_bemu_build.py} (51%) create mode 100644 bbdev/api/tests/test_api_bebop_bemu_sim.py create mode 100644 bbdev/api/tests/test_api_bebop_p2e_buildbitstream.py create mode 100644 bbdev/api/tests/test_api_bebop_p2e_clean.py create mode 100644 bbdev/api/tests/test_api_bebop_p2e_runworkload.py create mode 100644 bbdev/api/tests/test_api_bebop_p2e_verilog.py create mode 100644 bbdev/api/tests/test_api_bebop_verilator_build.py create mode 100644 bbdev/api/tests/test_api_bebop_verilator_clean.py create mode 100644 bbdev/api/tests/test_api_bebop_verilator_run.py create mode 100644 bbdev/api/tests/test_api_bebop_verilator_sim.py create mode 100644 bbdev/api/tests/test_api_bebop_verilator_verilog.py create mode 100644 bbdev/api/tests/test_api_firesim_enumeratefpgas.py delete mode 100644 bbdev/api/tests/test_api_pegasus_buildbitstream.py delete mode 100644 bbdev/api/tests/test_api_pegasus_flashbitstream.py delete mode 100644 bbdev/api/tests/test_api_pegasus_runworkload.py diff --git a/bbdev/api/steps/bebop/bemu/00_build_api.step.py b/bbdev/api/steps/bebop/bemu/00_build_api.step.py new file mode 100644 index 000000000..e84fd6fa5 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/00_build_api.step.py @@ -0,0 +1,15 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "bebop-bemu-build-api", + "description": "Build bebop bemu binary", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/bemu/build")], + "enqueues": ["bebop.bemu.build"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + await ctx.enqueue({"topic": "bebop.bemu.build", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/bemu/00_build_event.step.py b/bbdev/api/steps/bebop/bemu/00_build_event.step.py new file mode 100644 index 000000000..6a4ce95b9 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/00_build_event.step.py @@ -0,0 +1,53 @@ +""" +bebop bemu build event handler + +Builds bebop with bemu feature +""" +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-bemu-build", + "description": "Build bebop bemu binary", + "flows": ["bebop"], + "triggers": [queue("bebop.bemu.build")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + bebop_dir = f"{bbdir}/bebop" + + build_cmd = "cargo build --features bemu" + ctx.logger.info("Building bebop bemu ...") + build_result = stream_run_logger( + cmd=build_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop bemu build", + stderr_prefix="bebop bemu build", + ) + + bebop_bin = f"{bebop_dir}/target/debug/bebop" + await check_result( + ctx, + build_result.returncode, + continue_run=False, + extra_fields={ + "task": "build", + "binary": bebop_bin, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py new file mode 100644 index 000000000..3377bdd84 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py @@ -0,0 +1,27 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "bebop-bemu-sim-api", + "description": "Run bebop bemu emulator", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/bemu/sim")], + "enqueues": ["bebop.bemu.sim"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + binary = body.get("binary", "") + if not binary: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 400, + "message": "binary parameter is required", + }, + ) + + await ctx.enqueue({"topic": "bebop.bemu.sim", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py new file mode 100644 index 000000000..d2f7bee90 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -0,0 +1,81 @@ +""" +bebop bemu event handler + +Runs bebop bemu (Spike-based) emulator: + 1. Resolve binary (ELF) path + 2. Build bebop with bemu feature (cargo build) + 3. Run bebop bemu with the resolved ELF +""" +import os +import sys +from datetime import datetime + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.search_workload import search_workload +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-bemu-sim", + "description": "Run bebop bemu emulator", + "flows": ["bebop"], + "triggers": [queue("bebop.bemu.sim")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + bebop_dir = f"{bbdir}/bebop" + arch_dir = f"{bbdir}/arch" + + binary_name = input_data.get("binary", "") + binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) + if binary_path is None: + ctx.logger.error(f"binary not found: {binary_name}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "binary_not_found", "binary": binary_name}, + trace_id=origin_tid, + ) + return + ctx.logger.info(f"binary_path: {binary_path}") + + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + log_dir = input_data.get("log_dir") or f"{arch_dir}/log/{timestamp}-{binary_name}-bemu" + os.makedirs(log_dir, exist_ok=True) + + # ── Run bebop bemu ──────────────────────────────────────────────────── + run_cmd = ( + f"cargo run --features bemu -- bemu " + f"--elf=\"{binary_path}\" " + f"--log-dir=\"{log_dir}\"" + ) + ctx.logger.info(f"Running bebop bemu: {run_cmd}") + run_result = stream_run_logger( + cmd=run_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop bemu", + stderr_prefix="bebop bemu", + ) + + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "bemu", + "binary": binary_path, + "log_dir": log_dir, + "timestamp": timestamp, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/p2e/00_clean_api.step.py b/bbdev/api/steps/bebop/p2e/00_clean_api.step.py similarity index 52% rename from bbdev/api/steps/p2e/00_clean_api.step.py rename to bbdev/api/steps/bebop/p2e/00_clean_api.step.py index a4faaf6e9..701bd2834 100644 --- a/bbdev/api/steps/p2e/00_clean_api.step.py +++ b/bbdev/api/steps/bebop/p2e/00_clean_api.step.py @@ -1,16 +1,15 @@ from motia import ApiRequest, ApiResponse, FlowContext, api config = { - "name": "p2e-clean-api", + "name": "bebop-p2e-clean-api", "description": "Clean P2E build directory", - "flows": ["p2e"], - "triggers": [api("POST", "/p2e/clean")], - "enqueues": ["p2e.clean"], + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/p2e/clean")], + "enqueues": ["bebop.p2e.clean"], } async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - await ctx.enqueue({"topic": "p2e.clean", "data": {**body, "task": "clean", "_trace_id": ctx.trace_id}}) + await ctx.enqueue({"topic": "bebop.p2e.clean", "data": {**body, "task": "clean", "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) - diff --git a/bbdev/api/steps/p2e/00_clean_event.step.py b/bbdev/api/steps/bebop/p2e/00_clean_event.step.py similarity index 84% rename from bbdev/api/steps/p2e/00_clean_event.step.py rename to bbdev/api/steps/bebop/p2e/00_clean_event.step.py index 5b659fdeb..a37129eea 100644 --- a/bbdev/api/steps/p2e/00_clean_event.step.py +++ b/bbdev/api/steps/bebop/p2e/00_clean_event.step.py @@ -3,7 +3,7 @@ from motia import FlowContext, queue -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) @@ -12,10 +12,10 @@ from utils.event_common import check_result, get_origin_trace_id config = { - "name": "p2e-clean", + "name": "bebop-p2e-clean", "description": "Clean P2E build directory", - "flows": ["p2e"], - "triggers": [queue("p2e.clean")], + "flows": ["bebop"], + "triggers": [queue("bebop.p2e.clean")], "enqueues": [], } @@ -31,8 +31,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cmd=command, logger=ctx.logger, cwd=bbdir, - stdout_prefix="p2e clean", - stderr_prefix="p2e clean", + stdout_prefix="bebop p2e clean", + stderr_prefix="bebop p2e clean", ) await check_result( @@ -42,4 +42,3 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={"task": "clean", "config": config_name, "output_dir": build_dir}, trace_id=origin_tid, ) - diff --git a/bbdev/api/steps/p2e/01_verilog_api.step.py b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py similarity index 72% rename from bbdev/api/steps/p2e/01_verilog_api.step.py rename to bbdev/api/steps/bebop/p2e/01_verilog_api.step.py index 1642d3448..7b1cecfb6 100644 --- a/bbdev/api/steps/p2e/01_verilog_api.step.py +++ b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py @@ -4,11 +4,11 @@ from utils.path import get_verilator_build_dir config = { - "name": "p2e-verilog-api", + "name": "bebop-p2e-verilog-api", "description": "Generate SystemVerilog for P2E DDR4 backdoor", - "flows": ["p2e"], - "triggers": [api("POST", "/p2e/verilog")], - "enqueues": ["p2e.verilog"], + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/p2e/verilog")], + "enqueues": ["bebop.p2e.verilog"], } @@ -23,5 +23,5 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: "config": config_name, "output_dir": output_dir, } - await ctx.enqueue({"topic": "p2e.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue({"topic": "bebop.p2e.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py new file mode 100644 index 000000000..abaedb90f --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py @@ -0,0 +1,31 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path, get_verilator_build_dir + +config = { + "name": "bebop-p2e-buildbitstream-api", + "description": "Build P2E bitstream via bebop CLI", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/p2e/buildbitstream")], + "enqueues": ["bebop.p2e.buildbitstream"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = request.body or {} + + config_name = body.get("config", "sims.p2e.P2EToyConfig") + vsrc_dir = get_verilator_build_dir(bbdir, config_name, body.get("vsrc_dir")) + + data = { + "config": config_name, + "vsrc_dir": vsrc_dir, + "output_dir": body.get("output_dir"), + "log_dir": body.get("log_dir"), + } + await ctx.enqueue({ + "topic": "bebop.p2e.buildbitstream", + "data": {**data, "_trace_id": ctx.trace_id}, + }) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py new file mode 100644 index 000000000..013856703 --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -0,0 +1,114 @@ +""" +bebop p2e buildbitstream event handler + +Builds the FPGA bitstream via bebop CLI: + 1. Resolve verilog source directory (VSRC_PATH) from config + 2. Build bebop with p2e feature (vvacDir generated under build_dir via OUT_PATH) + 3. Run bebop p2e --buildbitstream with build_dir and output_dir +""" +import os +import sys +from datetime import datetime + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-p2e-buildbitstream", + "description": "Build P2E bitstream via bebop CLI", + "flows": ["bebop"], + "triggers": [queue("bebop.p2e.buildbitstream")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + bebop_dir = f"{bbdir}/bebop" + arch_dir = f"{bbdir}/arch" + + config_name = input_data.get("config", "sims.p2e.P2EToyConfig") + vsrc_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("vsrc_dir")) + if not os.path.isdir(vsrc_dir): + ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "vsrc_not_found", "vsrc_dir": vsrc_dir}, + trace_id=origin_tid, + ) + return + + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + build_dir = input_data.get("build_dir") or f"{bebop_dir}/build/{config_name}-{timestamp}" + output_dir = input_data.get("output_dir") or f"{arch_dir}/build/p2e-bitstream-{timestamp}" + log_dir = os.path.join(output_dir, "log") + os.makedirs(build_dir, exist_ok=True) + os.makedirs(output_dir, exist_ok=True) + os.makedirs(log_dir, exist_ok=True) + + # ── Build bebop with p2e feature ────────────────────────────────────── + build_cmd = ( + f"nix develop --ignore-environment --keep ALL_PROXY -c " + f"cargo build --features p2e " + f"--config=\"env.VSRC_PATH='{vsrc_dir}'\" " + f"--config=\"env.OUT_PATH='{build_dir}'\"" + ) + ctx.logger.info("Building bebop p2e ...") + build_result = stream_run_logger( + cmd=build_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop p2e build", + stderr_prefix="bebop p2e build", + ) + if build_result.returncode != 0: + await check_result( + ctx, build_result.returncode, continue_run=False, + extra_fields={"task": "buildbitstream", "stage": "build"}, + trace_id=origin_tid, + ) + return + + # ── Run bebop p2e --buildbitstream ──────────────────────────────────── + run_cmd = ( + f"nix develop --ignore-environment --keep HOME --keep ALL_PROXY -c " + f"cargo run --features p2e " + f"-- p2e " + f"--buildbitstream " + f"--build-dir=\"{build_dir}\" " + f"--output-dir=\"{output_dir}\"" + ) + ctx.logger.info(f"Running bebop p2e buildbitstream: {run_cmd}") + run_result = stream_run_logger( + cmd=run_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop p2e build-bitstream", + stderr_prefix="bebop p2e build-bitstream", + ) + + bitstream_path = os.path.join(output_dir, "fpgaCompDir", "bitstream.bit") + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "buildbitstream", + "config": config_name, + "vsrc_dir": vsrc_dir, + "build_dir": build_dir, + "output_dir": output_dir, + "log_dir": log_dir, + "bitstream": bitstream_path, + "timestamp": timestamp, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py new file mode 100644 index 000000000..88dace351 --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py @@ -0,0 +1,31 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "bebop-p2e-runworkload-api", + "description": "Run workload on FPGA via bebop p2e CLI", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/p2e/runworkload")], + "enqueues": ["bebop.p2e.runworkload"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + image = body.get("image", "") + bitstream = body.get("bitstream", "") + if not image or not bitstream: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 400, + "message": "image and bitstream parameters are required", + }, + ) + + await ctx.enqueue({ + "topic": "bebop.p2e.runworkload", + "data": {**body, "_trace_id": ctx.trace_id}, + }) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py new file mode 100644 index 000000000..515c997c2 --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -0,0 +1,110 @@ +""" +bebop p2e runworkload event handler + +Loads a kernel image into FPGA and runs the workload via bebop CLI: + 1. Validate image and bitstream paths + 2. Build bebop with p2e feature (VSRC_PATH from previous build, optional) + 3. Run bebop p2e --runworkload --image --bitstream +""" +import os +import sys +from datetime import datetime + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-p2e-runworkload", + "description": "Run workload on FPGA via bebop p2e CLI", + "flows": ["bebop"], + "triggers": [queue("bebop.p2e.runworkload")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + bebop_dir = f"{bbdir}/bebop" + arch_dir = f"{bbdir}/arch" + + image = input_data.get("image", "") + bitstream = input_data.get("bitstream", "") + for label, path in (("image", image), ("bitstream", bitstream)): + if not path or not os.path.exists(path): + ctx.logger.error(f"{label} not found: {path}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": f"{label}_not_found", label: path}, + trace_id=origin_tid, + ) + return + + config_name = input_data.get("config", "sims.p2e.P2EToyConfig") + vsrc_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("vsrc_dir")) + + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + output_dir = input_data.get("output_dir") or os.path.dirname(os.path.abspath(bitstream)) + log_dir = input_data.get("log_dir") or f"{arch_dir}/log/p2e-runworkload-{timestamp}" + os.makedirs(log_dir, exist_ok=True) + + # ── Build bebop with p2e feature ────────────────────────────────────── + build_cmd = "cargo build --features p2e" + if os.path.isdir(vsrc_dir): + build_cmd += f" --config=\"env.VSRC_PATH='{vsrc_dir}'\"" + ctx.logger.info("Building bebop p2e ...") + build_result = stream_run_logger( + cmd=build_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop p2e build", + stderr_prefix="bebop p2e build", + ) + if build_result.returncode != 0: + await check_result( + ctx, build_result.returncode, continue_run=False, + extra_fields={"task": "runworkload", "stage": "build"}, + trace_id=origin_tid, + ) + return + + # ── Run bebop p2e --runworkload ─────────────────────────────────────── + run_cmd = ( + f"cargo run --features p2e -- p2e " + f"--runworkload " + f"--image=\"{image}\" " + f"--bitstream=\"{bitstream}\" " + f"--log-dir=\"{log_dir}\"" + ) + ctx.logger.info(f"Running bebop p2e runworkload: {run_cmd}") + run_result = stream_run_logger( + cmd=run_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop p2e runworkload", + stderr_prefix="bebop p2e runworkload", + ) + + uart_log = os.path.join(log_dir, "uart.log") + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "runworkload", + "image": image, + "bitstream": bitstream, + "output_dir": output_dir, + "log_dir": log_dir, + "uart_log": uart_log, + "timestamp": timestamp, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py new file mode 100644 index 000000000..2a2a7d5b1 --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py @@ -0,0 +1,26 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path, get_verilator_build_dir + +config = { + "name": "bebop-verilator-clean-api", + "description": "Clean verilator build directory", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/verilator/clean")], + "enqueues": ["bebop.verilator.clean"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = req.body or {} + + config_name = body.get("config", "sims.verilator.BuckyballToyVerilatorConfig") + build_dir = get_verilator_build_dir(bbdir, config_name, body.get("output_dir")) + + data = { + "config": config_name, + "output_dir": build_dir, + } + await ctx.enqueue({"topic": "bebop.verilator.clean", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py new file mode 100644 index 000000000..28f4bbe91 --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py @@ -0,0 +1,53 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-verilator-clean", + "description": "Clean verilator build directory", + "flows": ["bebop"], + "triggers": [ + queue("bebop.verilator.run"), + queue("bebop.verilator.clean"), + ], + "enqueues": ["bebop.verilator.verilog"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + config_name = input_data.get("config", "sims.verilator.BuckyballToyVerilatorConfig") + build_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("output_dir")) + + command = f"rm -rf {build_dir}" + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="bebop verilator clean", + stderr_prefix="bebop verilator clean", + ) + + await check_result( + ctx, + result.returncode, + continue_run=input_data.get("from_run_workflow", False), + extra_fields={"task": "clean", "config": config_name, "output_dir": build_dir}, + trace_id=origin_tid, + ) + + # Continue routing to verilog if from run workflow + if input_data.get("from_run_workflow"): + await ctx.enqueue( + {"topic": "bebop.verilator.verilog", "data": {**input_data, "output_dir": build_dir, "task": "run"}} + ) diff --git a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py new file mode 100644 index 000000000..24eb9b674 --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py @@ -0,0 +1,35 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path, get_verilator_build_dir + +config = { + "name": "bebop-verilator-verilog-api", + "description": "Generate verilog code via mill", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/verilator/verilog")], + "enqueues": ["bebop.verilator.verilog"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = request.body or {} + + config_name = body.get("config") + if not config_name or config_name == "None": + return ApiResponse( + status=400, + body={ + "status": "error", + "message": "Configuration name is required. Please specify --config parameter.", + "example": 'bbdev bebop verilator --verilog "--config sims.verilator.BuckyballToyVerilatorConfig"', + }, + ) + + data = { + "config": config_name, + "balltype": body.get("balltype"), + "output_dir": get_verilator_build_dir(bbdir, config_name, body.get("output_dir")), + } + await ctx.enqueue({"topic": "bebop.verilator.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_api.step.py b/bbdev/api/steps/bebop/verilator/03_build_api.step.py new file mode 100644 index 000000000..8a9c5981b --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/03_build_api.step.py @@ -0,0 +1,26 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path, get_verilator_build_dir + +config = { + "name": "bebop-verilator-build-api", + "description": "Build bebop verilator binary", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/verilator/build")], + "enqueues": ["bebop.verilator.build"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = request.body or {} + + arch_config = body.get("config", "sims.verilator.BuckyballToyVerilatorConfig") + vsrc_dir = get_verilator_build_dir(bbdir, arch_config, body.get("vsrc_dir")) + + data = { + "config": arch_config, + "vsrc_dir": vsrc_dir, + } + await ctx.enqueue({"topic": "bebop.verilator.build", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py new file mode 100644 index 000000000..cb14e0a81 --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -0,0 +1,77 @@ +""" +bebop verilator build event handler + +Builds bebop with verilator feature and VSRC_PATH +""" +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-verilator-build", + "description": "Build bebop verilator binary", + "flows": ["bebop"], + "triggers": [queue("bebop.verilator.build")], + "enqueues": ["bebop.verilator.sim"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + bebop_dir = f"{bbdir}/bebop" + + arch_config = input_data.get("config", "sims.verilator.BuckyballToyVerilatorConfig") + vsrc_dir = get_verilator_build_dir(bbdir, arch_config, input_data.get("vsrc_dir")) + ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") + + if not os.path.isdir(vsrc_dir): + ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "vsrc_not_found", "vsrc_dir": vsrc_dir}, + trace_id=origin_tid, + ) + return + + build_cmd = ( + f"cargo build --features verilator " + f"--config=\"env.VSRC_PATH='{vsrc_dir}'\"" + ) + ctx.logger.info("Building bebop verilator ...") + build_result = stream_run_logger( + cmd=build_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop verilator build", + stderr_prefix="bebop verilator build", + ) + + bebop_bin = f"{bebop_dir}/target/debug/bebop" + await check_result( + ctx, + build_result.returncode, + continue_run=input_data.get("from_run_workflow", False), + extra_fields={ + "task": "build", + "config": arch_config, + "vsrc_dir": vsrc_dir, + "binary": bebop_bin, + }, + trace_id=origin_tid, + ) + + # Continue routing to sim if from run workflow + if input_data.get("from_run_workflow"): + await ctx.enqueue( + {"topic": "bebop.verilator.sim", "data": {**input_data, "vsrc_dir": vsrc_dir, "task": "run"}} + ) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_api.step.py b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py new file mode 100644 index 000000000..ed5249ae2 --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py @@ -0,0 +1,27 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "bebop-verilator-sim-api", + "description": "Run bebop verilator simulation", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/verilator/sim")], + "enqueues": ["bebop.verilator.sim"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + binary = body.get("binary", "") + if not binary: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 400, + "message": "binary parameter is required", + }, + ) + + await ctx.enqueue({"topic": "bebop.verilator.sim", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py new file mode 100644 index 000000000..0f896265e --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -0,0 +1,100 @@ +""" +bebop verilator event handler + +Runs bebop verilator simulation: + 1. Resolve binary path + 2. Resolve verilog source directory (VSRC_PATH) + 3. Build bebop with verilator feature (cargo build) + 4. Run bebop verilator with elf, log and fst directories +""" +import os +import sys +from datetime import datetime + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.stream_run import stream_run_logger +from utils.search_workload import search_workload +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-verilator-sim", + "description": "Run bebop verilator simulation", + "flows": ["bebop"], + "triggers": [queue("bebop.verilator.sim")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + bebop_dir = f"{bbdir}/bebop" + arch_dir = f"{bbdir}/arch" + + arch_config = input_data.get("config", "sims.verilator.BuckyballToyVerilatorConfig") + vsrc_dir = get_verilator_build_dir(bbdir, arch_config, input_data.get("vsrc_dir")) + ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") + + binary_name = input_data.get("binary", "") + binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) + if binary_path is None: + ctx.logger.error(f"binary not found: {binary_name}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "binary_not_found", "binary": binary_name}, + trace_id=origin_tid, + ) + return + ctx.logger.info(f"binary_path: {binary_path}") + + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + log_dir = input_data.get("log_dir") or f"{arch_dir}/log/{timestamp}-{binary_name}" + fst_dir = input_data.get("fst_dir") or f"{arch_dir}/waveform/{timestamp}-{binary_name}" + os.makedirs(log_dir, exist_ok=True) + os.makedirs(fst_dir, exist_ok=True) + + # ── Run bebop verilator ─────────────────────────────────────────────── + trace_flags = [] + for flag in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace"): + if input_data.get(flag, False): + trace_flags.append(f"--{flag}") + trace_args = (" " + " ".join(trace_flags)) if trace_flags else "" + + run_cmd = ( + f"cargo run --features verilator " + f"--config=\"env.ARCH_CONFIG='{arch_config}'\" " + f"-- verilator " + f"--elf=\"{binary_path}\" " + f"--log-dir=\"{log_dir}\" " + f"--fst-dir=\"{fst_dir}\"" + f"{trace_args}" + ) + ctx.logger.info(f"Running bebop verilator: {run_cmd}") + run_result = stream_run_logger( + cmd=run_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop verilator", + stderr_prefix="bebop verilator", + ) + + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "verilator", + "binary": binary_path, + "config": arch_config, + "log_dir": log_dir, + "fst_dir": fst_dir, + "timestamp": timestamp, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/verilator/05_run_api.step.py b/bbdev/api/steps/bebop/verilator/05_run_api.step.py new file mode 100644 index 000000000..0eb7ae40e --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/05_run_api.step.py @@ -0,0 +1,48 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path, get_verilator_build_dir + +config = { + "name": "bebop-verilator-run-api", + "description": "Run complete bebop verilator workflow: clean → verilog → build → sim", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/verilator/run")], + "enqueues": ["bebop.verilator.run"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = request.body or {} + + config_name = body.get("config", "sims.verilator.BuckyballToyVerilatorConfig") + build_dir = get_verilator_build_dir(bbdir, config_name, body.get("output_dir")) + + binary = body.get("binary", "") + if not binary: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 400, + "message": "binary parameter is required", + }, + ) + + data = { + "config": config_name, + "output_dir": build_dir, + "binary": binary, + "balltype": body.get("balltype"), + "itrace": body.get("itrace", False), + "mtrace": body.get("mtrace", False), + "pmctrace": body.get("pmctrace", False), + "ctrace": body.get("ctrace", False), + "banktrace": body.get("banktrace", False), + "log_dir": body.get("log_dir"), + "fst_dir": body.get("fst_dir"), + "from_run_workflow": True, + } + await ctx.enqueue({"topic": "bebop.verilator.run", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/kernel/scripts/bin_to_hex.py b/bbdev/api/steps/kernel/scripts/bin_to_hex.py index a5f5d6a18..9293cc840 100644 --- a/bbdev/api/steps/kernel/scripts/bin_to_hex.py +++ b/bbdev/api/steps/kernel/scripts/bin_to_hex.py @@ -4,12 +4,15 @@ import sys import os -def bin_to_hex(bin_file, hex_file, objcopy_path="riscv64-unknown-elf-objcopy"): +def bin_to_hex(bin_file, hex_file, objcopy_path="riscv64-unknown-elf-objcopy", base_address=0): """ 将 RISC-V bin 文件转换为 Verilog readmemh 格式 地址格式：@0xNNNNNNNN + base_address: 基地址偏移，生成的地址会减去这个值（例如 0x80000000） """ print(f"Converting {bin_file} -> {hex_file}") + if base_address != 0: + print(f"Base address offset: 0x{base_address:08X}") # Step 1: bin -> Intel HEX temp_hex = f"{bin_file}.tmp.hex" @@ -65,7 +68,14 @@ def bin_to_hex(bin_file, hex_file, objcopy_path="riscv64-unknown-elf-objcopy"): for addr in sorted_addresses: # New address section if not continuous if current_address is None or addr != current_address + 1: - f.write(f"@0x{addr:08X}\n") + relative_addr = addr - base_address + if relative_addr < 0: + print(f"Warning: address 0x{addr:08X} is below base 0x{base_address:08X}, skipping") + continue + if relative_addr == 0: + f.write(f"@0\n") + else: + f.write(f"@0x{relative_addr:X}\n") current_address = addr else: current_address = addr @@ -86,18 +96,27 @@ def bin_to_hex(bin_file, hex_file, objcopy_path="riscv64-unknown-elf-objcopy"): os.remove(temp_hex) def main(): - if len(sys.argv) != 3: - print("Usage: bin_to_hex.py ") + if len(sys.argv) < 3 or len(sys.argv) > 4: + print("Usage: bin_to_hex.py [base_address]") + print(" base_address: optional hex address to subtract (e.g., 0x80000000)") sys.exit(1) bin_file = sys.argv[1] hex_file = sys.argv[2] + base_address = 0 + + if len(sys.argv) == 4: + base_str = sys.argv[3] + if base_str.startswith('0x') or base_str.startswith('0X'): + base_address = int(base_str, 16) + else: + base_address = int(base_str, 10) if not os.path.exists(bin_file): print(f"Error: File not found: {bin_file}") sys.exit(1) - success = bin_to_hex(bin_file, hex_file) + success = bin_to_hex(bin_file, hex_file, base_address=base_address) sys.exit(0 if success else 1) if __name__ == "__main__": diff --git a/bbdev/api/steps/mill/02_pegasus_verilog_event.step.py b/bbdev/api/steps/mill/02_pegasus_verilog_event.step.py deleted file mode 100644 index 5ed14142d..000000000 --- a/bbdev/api/steps/mill/02_pegasus_verilog_event.step.py +++ /dev/null @@ -1,165 +0,0 @@ -import os -import sys - -from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from utils.event_common import check_result, get_origin_trace_id - -config = { - "name": "make pegasus verilog", - "description": "Generate SystemVerilog from Chisel using ElaboratePegasusTop", - "flows": ["pegasus"], - "triggers": [queue("pegasus.verilog")], - "enqueues": [], -} - - -def prepare_pegasus_verilog_output(bbdir: str, arch_dir: str, soc_dir: str, build_dir: str, logger): - for stray in ["PegasusTop.v", "PegasusTopWrapper.sv", "PegasusShell.v"]: - stray_path = f"{arch_dir}/{stray}" - if os.path.exists(stray_path): - os.remove(stray_path) - - vivado_gen_dir = f"{bbdir}/thirdparty/pegasus/vivado/generated" - if not os.path.isdir(build_dir): - return vivado_gen_dir - - import re - import shutil - os.makedirs(vivado_gen_dir, exist_ok=True) - for f in os.listdir(vivado_gen_dir): - if f.endswith(".sv") or f.endswith(".v"): - os.remove(os.path.join(vivado_gen_dir, f)) - - harness_skip = {"PegasusHarness.sv", "ChipTop.sv"} - - def build_dpi_stub(src_path: str) -> str: - text = open(src_path, "r", encoding="utf-8").read() - m = re.search(r"module\s+([A-Za-z_][A-Za-z0-9_]*)\s*\((.*?)\);\s*", text, re.S) - if m is None: - raise RuntimeError(f"invalid DPI module format: {src_path}") - mod_name = m.group(1) - ports_block = m.group(2).rstrip() - outputs = [] - for line in ports_block.splitlines(): - s = line.strip() - if not s.startswith("output"): - continue - s = s.rstrip(",") - s = re.sub(r"^output\s+", "", s) - s = re.sub(r"^\[[^\]]+\]\s+", "", s) - for name in s.split(","): - n = name.strip() - if n: - outputs.append(n) - stub = [f"module {mod_name}(", ports_block, ");"] - for out_name in outputs: - stub.append(f" assign {out_name} = '0;") - stub.append("endmodule") - stub.append("") - return "\n".join(stub) - - def copy_rtl_dir(src_dir: str, skip_set: set = None) -> int: - copied = 0 - for f in os.listdir(src_dir): - if not (f.endswith(".sv") or f.endswith(".v")): - continue - if skip_set and f in skip_set: - continue - src = os.path.join(src_dir, f) - if "DPI" in f: - stub_name = f"stub_{os.path.splitext(f)[0].replace('DPI', '')}.sv" - with open(os.path.join(vivado_gen_dir, stub_name), "w") as wf: - wf.write(build_dpi_stub(src)) - else: - shutil.copy2(src, os.path.join(vivado_gen_dir, f)) - copied += 1 - return copied - - n_soc = copy_rtl_dir(soc_dir, skip_set=harness_skip) - n_top = copy_rtl_dir(build_dir) - logger.info(f" Copied {n_soc} SoC files + {n_top} top files to {vivado_gen_dir}") - return vivado_gen_dir - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - bbdir = get_buckyball_path() - arch_dir = f"{bbdir}/arch" - build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/pegasus/") - - build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/pegasus_top/") - soc_dir = f"{bbdir}/arch/build/pegasus/" - - ctx.logger.info(f" Elaborating PegasusTop") - ctx.logger.info(f" Top output directory: {build_dir}") - ctx.logger.info(f" SoC RTL directory: {soc_dir}") - - os.makedirs(build_dir, exist_ok=True) - - soc_command = ( - f"mill -i __.test.runMain sims.pegasus.ElaboratePegasus " - f"--disable-annotation-unknown " - f"-strip-debug-info " - f"--split-verilog " - f"-o={soc_dir}" - ) - soc_result = stream_run_logger( - cmd=soc_command, - logger=ctx.logger, - cwd=arch_dir, - stdout_prefix="pegasus soc verilog", - stderr_prefix="pegasus soc verilog", - ) - if soc_result.returncode != 0: - ctx.logger.error(" SoC elaboration failed") - await check_result(ctx, soc_result.returncode, continue_run=False, - extra_fields={"task": "verilog", "step": "soc"}, trace_id=origin_tid) - return - - command = ( - f"mill -i __.test.runMain sims.pegasus.ElaboratePegasusTop " - f"--disable-annotation-unknown " - f"-strip-debug-info " - f"--split-verilog " - f"-o={build_dir}" - ) - - result = stream_run_logger( - cmd=command, - logger=ctx.logger, - cwd=arch_dir, - stdout_prefix="pegasus verilog", - stderr_prefix="pegasus verilog", - ) - - vivado_gen_dir = f"{bbdir}/thirdparty/pegasus/vivado/generated" - if result.returncode == 0: - vivado_gen_dir = prepare_pegasus_verilog_output( - bbdir=bbdir, - arch_dir=arch_dir, - soc_dir=soc_dir, - build_dir=build_dir, - logger=ctx.logger, - ) - - success_result, failure_result = await check_result( - ctx, - result.returncode, - continue_run=False, - extra_fields={ - "task": "verilog", - "output_dir": build_dir, - "vivado_gen_dir": vivado_gen_dir, - "top_module": "PegasusTop", - }, - trace_id=origin_tid, - ) - - return diff --git a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py new file mode 100644 index 000000000..137df50c8 --- /dev/null +++ b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py @@ -0,0 +1,111 @@ +""" +bebop verilator verilog event handler + +Generates Verilog via mill for bebop verilator +""" +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-verilator-verilog", + "description": "Generate verilog code via mill", + "flows": ["bebop"], + "triggers": [queue("bebop.verilator.verilog")], + "enqueues": ["bebop.verilator.build"], +} + + +def prepare_verilator_verilog(build_dir: str, arch_dir: str, logger): + """Remove unwanted harness and patch fesvr includes""" + unwanted_harness = f"{arch_dir}/BBSimHarness.sv" + if os.path.exists(unwanted_harness): + os.remove(unwanted_harness) + + for patch_file in [f"{build_dir}/mm.h", f"{build_dir}/mm.cc"]: + if os.path.exists(patch_file): + with open(patch_file, "r") as f: + content = f.read() + patched = "\n".join( + line for line in content.splitlines() + if "fesvr/memif.h" not in line and "fesvr/elfloader.h" not in line + ) + if patched != content: + with open(patch_file, "w") as f: + f.write(patched) + logger.info(f"Patched fesvr includes from {patch_file}") + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + build_dir = get_verilator_build_dir( + bbdir, + input_data.get("config"), + input_data.get("output_dir"), + ) + arch_dir = f"{bbdir}/arch" + config_name = input_data.get("config") + + if not config_name or config_name == "None": + ctx.logger.error("Configuration name is required but not provided") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "validation", + "error": "Configuration name is required. Please specify --config parameter.", + "example": 'bbdev bebop verilator --verilog "--config sims.verilator.BuckyballToyVerilatorConfig"', + }, + trace_id=origin_tid, + ) + return + + ctx.logger.info(f"Using configuration: {config_name}") + ctx.logger.info(f"Using build directory: {build_dir}") + + if input_data.get("balltype"): + command = ( + f"mill -i __.test.runMain sims.verify.BallTopMain {input_data.get('balltype')} " + ) + else: + command = f"mill -i __.test.runMain sims.verilator.Elaborate {config_name} " + + command += "--disable-annotation-unknown " + command += "--strip-debug-info " + command += "-O=debug " + command += f"--split-verilog -o={build_dir} " + + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=arch_dir, + stdout_prefix="bebop verilator verilog", + stderr_prefix="bebop verilator verilog", + ) + + prepare_verilator_verilog(build_dir, arch_dir, ctx.logger) + + await check_result( + ctx, + result.returncode, + continue_run=input_data.get("from_run_workflow", False), + extra_fields={"task": "verilog", "config": config_name, "output_dir": build_dir}, + trace_id=origin_tid, + ) + + # Continue routing to build if from run workflow + if input_data.get("from_run_workflow"): + await ctx.enqueue( + {"topic": "bebop.verilator.build", "data": {**input_data, "output_dir": build_dir, "vsrc_dir": build_dir, "task": "run"}} + ) diff --git a/bbdev/api/steps/mill/05_p2e_verilog_event.step.py b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py similarity index 83% rename from bbdev/api/steps/mill/05_p2e_verilog_event.step.py rename to bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py index aa2bc8919..92d2607d4 100644 --- a/bbdev/api/steps/mill/05_p2e_verilog_event.step.py +++ b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py @@ -7,21 +7,21 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path -from utils.path import get_verilator_build_dir +from utils.path import get_buckyball_path, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id config = { - "name": "make p2e verilog", - "description": "Generate DigitalTop RTL and P2E DDR4 backdoor wrapper", - "flows": ["p2e"], - "triggers": [queue("p2e.verilog")], + "name": "bebop-p2e-verilog", + "description": "Generate P2E Verilog via mill", + "flows": ["bebop"], + "triggers": [queue("bebop.p2e.verilog")], "enqueues": [], } def cleanup_strays(arch_dir: str): + """Remove stray generated files from arch directory""" for stray in ["P2EHarness.sv", "P2ETop.v", "P2ETopWrapper.sv"]: path = os.path.join(arch_dir, stray) if os.path.exists(path): @@ -29,6 +29,7 @@ def cleanup_strays(arch_dir: str): def normalize_p2e_timescale(build_dir: str, logger): + """Add timescale directive to generated Verilog files""" patched = 0 for root, _, files in os.walk(build_dir): for name in files: @@ -70,6 +71,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"--split-verilog -o={build_dir}" ) + # Generate digital top Verilog digital_command = ( f"mill -i __.test.runMain sims.p2e.Elaborate {config_name} " f"{common_firtool_opts}" @@ -78,8 +80,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cmd=digital_command, logger=ctx.logger, cwd=arch_dir, - stdout_prefix="p2e digital verilog", - stderr_prefix="p2e digital verilog", + stdout_prefix="bebop p2e digital verilog", + stderr_prefix="bebop p2e digital verilog", ) if digital_result.returncode != 0: cleanup_strays(arch_dir) @@ -92,6 +94,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return + # Generate P2E top wrapper Verilog top_command = ( f"mill -i __.test.runMain sims.p2e.ElaborateP2ETop " f"{common_firtool_opts}" @@ -100,8 +103,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cmd=top_command, logger=ctx.logger, cwd=arch_dir, - stdout_prefix="p2e top verilog", - stderr_prefix="p2e top verilog", + stdout_prefix="bebop p2e top verilog", + stderr_prefix="bebop p2e top verilog", ) cleanup_strays(arch_dir) diff --git a/bbdev/api/steps/pegasus/01_verilog_api.step.py b/bbdev/api/steps/pegasus/01_verilog_api.step.py deleted file mode 100644 index 6094e49fa..000000000 --- a/bbdev/api/steps/pegasus/01_verilog_api.step.py +++ /dev/null @@ -1,25 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - -from utils.path import get_buckyball_path - -config = { - "name": "Pegasus Verilog", - "description": "Generate SystemVerilog for Pegasus FPGA (PegasusHarness + ChipTop)", - "flows": ["pegasus"], - "triggers": [api("POST", "/pegasus/verilog")], - "enqueues": ["pegasus.verilog"], -} - - -async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() - body = req.body or {} - - config_name = body.get("config", "sims.pegasus.PegasusConfig") - - data = { - "config": config_name, - "output_dir": body.get("output_dir", f"{bbdir}/arch/build/pegasus/"), - } - await ctx.enqueue({"topic": "pegasus.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/pegasus/02_buildbitstream_api.step.py b/bbdev/api/steps/pegasus/02_buildbitstream_api.step.py deleted file mode 100644 index 483a5afa8..000000000 --- a/bbdev/api/steps/pegasus/02_buildbitstream_api.step.py +++ /dev/null @@ -1,15 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - -config = { - "name": "Pegasus Buildbitstream", - "description": "build pegasus bitstream", - "flows": ["pegasus"], - "triggers": [api("POST", "/pegasus/buildbitstream")], - "enqueues": ["pegasus.buildbitstream"], -} - - -async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = req.body or {} - await ctx.enqueue({"topic": "pegasus.buildbitstream", "data": {**body, "_trace_id": ctx.trace_id}}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py b/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py deleted file mode 100644 index 60396ff74..000000000 --- a/bbdev/api/steps/pegasus/02_buildbitstream_event.step.py +++ /dev/null @@ -1,83 +0,0 @@ -import os -import sys - -from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from utils.event_common import check_result, get_origin_trace_id - -config = { - "name": "generate pegasus bitstream", - "description": "generate pegasus bitstream", - "flows": ["pegasus"], - "triggers": [queue("pegasus.buildbitstream")], - "enqueues": [], -} - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - bbdir = get_buckyball_path() - generated_dir = input_data.get("generated_dir", f"{bbdir}/thirdparty/pegasus/vivado/generated") - output_dir = input_data.get("output_dir", f"{bbdir}/thirdparty/pegasus/vivado/build") - top_module = input_data.get("top", "PegasusTop") - - ctx.logger.info(f" Generated dir: {generated_dir}") - ctx.logger.info(f" Output dir: {output_dir}") - - os.makedirs(output_dir, exist_ok=True) - - if not os.path.isdir(generated_dir): - ctx.logger.error(f" generated dir not found: {generated_dir}") - success_result, failure_result = await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"task": "buildbitstream", "error": "missing generated_dir"}, - trace_id=origin_tid, - ) - return failure_result - - has_rtl = any(name.endswith(".sv") or name.endswith(".v") for name in os.listdir(generated_dir)) - if not has_rtl: - ctx.logger.error(f" no verilog files found in: {generated_dir}") - success_result, failure_result = await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"task": "buildbitstream", "error": "empty generated_dir"}, - trace_id=origin_tid, - ) - return failure_result - - bit_cmd = ( - f"bash {bbdir}/thirdparty/pegasus/vivado/build-bitstream.sh " - f"--source_dir {generated_dir} " - f"--output_dir {output_dir} " - f"--top {top_module}" - ) - result = stream_run_logger( - cmd=bit_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="pegasus bitstream", - stderr_prefix="pegasus bitstream", - ) - - success_result, failure_result = await check_result( - ctx, - result.returncode, - continue_run=False, - extra_fields={ - "task": "buildbitstream", - "output_dir": output_dir, - "bitstream": f"{output_dir}/{top_module}.bit", - }, - trace_id=origin_tid, - ) - return diff --git a/bbdev/api/steps/pegasus/03_flashbitstream_api.step.py b/bbdev/api/steps/pegasus/03_flashbitstream_api.step.py deleted file mode 100644 index d32d80507..000000000 --- a/bbdev/api/steps/pegasus/03_flashbitstream_api.step.py +++ /dev/null @@ -1,15 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - -config = { - "name": "Pegasus Flashbitstream", - "description": "Flash bitstream onto AU280 via hw_server", - "flows": ["pegasus"], - "triggers": [api("POST", "/pegasus/flashbitstream")], - "enqueues": ["pegasus.flashbitstream"], -} - - -async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = req.body or {} - await ctx.enqueue({"topic": "pegasus.flashbitstream", "data": {**body, "_trace_id": ctx.trace_id}}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/pegasus/03_flashbitstream_event.step.py b/bbdev/api/steps/pegasus/03_flashbitstream_event.step.py deleted file mode 100644 index e45786047..000000000 --- a/bbdev/api/steps/pegasus/03_flashbitstream_event.step.py +++ /dev/null @@ -1,343 +0,0 @@ -""" -pegasus flashbitstream event handler - -Sequence: - 1. Detect XDMA PCIe BDF - 2. Flash via Vivado + program_fpga.tcl (JTAG — independent of PCIe) - 3. PCIe remove + rescan so new bitstream's PCIe IP re-enumerates -""" -import os -import sys -import time -import socket -import subprocess -import glob -import re - -from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from utils.event_common import check_result, get_origin_trace_id - -config = { - "name": "pegasus-flashbitstream", - "description": "Flash AU280 bitstream via Vivado hw_server + PCIe remove/rescan", - "flows": ["pegasus"], - "triggers": [queue("pegasus.flashbitstream")], - "enqueues": [], -} - -PCI_DEVICES = "/sys/bus/pci/devices" -XILINX_VENDOR = "0x10ee" -XDMA_DEVICE_IDS = {"0x903f"} - - -def _find_xdma_bdf() -> str | None: - """Return the extended BDF (e.g. '0000:65:00.0') bound to xdma driver.""" - for entry in os.listdir(PCI_DEVICES): - driver_link = os.path.join(PCI_DEVICES, entry, "driver") - if not os.path.islink(driver_link): - continue - driver_name = os.path.basename(os.readlink(driver_link)) - if driver_name == "xdma": - return entry # e.g. '0000:65:00.0' - return None - - -def _find_xdma_pci_bdf() -> str | None: - """Return first XDMA-like Xilinx PCIe endpoint even if driver is not bound.""" - for entry in os.listdir(PCI_DEVICES): - vendor_file = os.path.join(PCI_DEVICES, entry, "vendor") - device_file = os.path.join(PCI_DEVICES, entry, "device") - if not (os.path.exists(vendor_file) and os.path.exists(device_file)): - continue - vendor = open(vendor_file, "r", encoding="utf-8").read().strip().lower() - device = open(device_file, "r", encoding="utf-8").read().strip().lower() - if vendor == XILINX_VENDOR and device in XDMA_DEVICE_IDS: - return entry - return None - - -def _bridge_bdf(extended_bdf: str) -> str | None: - """Return the bridge BDF for a given extended device BDF.""" - dev_path = os.path.realpath(os.path.join(PCI_DEVICES, extended_bdf)) - bridge_path = os.path.dirname(dev_path) - bridge_bdf = os.path.basename(bridge_path) - if bridge_bdf.startswith("0000:"): - return bridge_bdf - return None - - -def _ensure_hw_server(port: int = 3121, logger=None) -> bool: - """Ensure hw_server is listening on the given port; start it if not.""" - try: - with socket.create_connection(("localhost", port), timeout=2): - return True - except OSError: - pass - if logger: - logger.info(f"[pegasus] hw_server not on port {port}, starting ...") - subprocess.Popen( - ["hw_server"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - start_new_session=True, - ) - for _ in range(15): - time.sleep(1) - try: - with socket.create_connection(("localhost", port), timeout=1): - if logger: - logger.info(f"[pegasus] hw_server ready on port {port}") - return True - except OSError: - pass - if logger: - logger.error(f"[pegasus] hw_server failed to start on port {port}") - return False - - -def _setpci(bdf: str, reg: str, val: str) -> bool: - r = subprocess.run(["sudo", "setpci", "-s", bdf, f"{reg}={val}"], - capture_output=True) - return r.returncode == 0 - - -def _sysfs_write(path: str, value: str, logger) -> bool: - """Write to a sysfs file via 'sudo tee' (needed for root-owned sysfs entries).""" - r = subprocess.run( - f"echo {value} | sudo tee {path}", - shell=True, capture_output=True, - ) - if r.returncode != 0: - logger.error(f" sysfs write failed: {path} <- {value}: {r.stderr.decode().strip()}") - return r.returncode == 0 - - -def _pcie_disconnect(extended_bdf: str, bridge: str | None, logger) -> None: - """Clear SERR/fatal-error bits and remove device from PCIe bus.""" - if bridge: - logger.info(f" clearing SERR bit on bridge {bridge}") - _setpci(bridge, "COMMAND", "0000:0100") - time.sleep(0.5) - logger.info(f" clearing fatal-error bit on bridge {bridge}") - _setpci(bridge, "CAP_EXP+8.w", "0000:0004") - time.sleep(0.5) - remove_path = os.path.join(PCI_DEVICES, extended_bdf, "remove") - if os.path.exists(remove_path): - logger.info(f" removing PCIe device {extended_bdf}") - if _sysfs_write(remove_path, "1", logger): - time.sleep(2) - - -def _pcie_reconnect(bridge: str | None, device_bdf: str | None, logger) -> None: - """Rescan PCIe bus and re-enable memory-mapped transfers.""" - if bridge: - rescan_path = os.path.join(PCI_DEVICES, bridge, "rescan") - if os.path.exists(rescan_path): - logger.info(f" rescanning bridge {bridge}") - _sysfs_write(rescan_path, "1", logger) - time.sleep(1) - logger.info(" global PCIe rescan") - _sysfs_write("/sys/bus/pci/rescan", "1", logger) - time.sleep(2) - # Re-detect after rescan: prefer xdma-bound BDF, fall back to raw PCIe endpoint. - new_bdf = _find_xdma_bdf() or _find_xdma_pci_bdf() - if not new_bdf: - for _ in range(5): - _sysfs_write("/sys/bus/pci/rescan", "1", logger) - time.sleep(1) - new_bdf = _find_xdma_bdf() or _find_xdma_pci_bdf() - if new_bdf: - break - if not new_bdf: - raise RuntimeError("xdma pci endpoint not found after PCIe rescan") - logger.info(f" enabling MEM+BUSMSTR on {new_bdf}") - _setpci(new_bdf, "COMMAND.W", "0006") - - -def _run_checked(cmd: str) -> None: - subprocess.run(cmd, shell=True, check=True, capture_output=True, text=True) - - -def _reload_xdma(logger) -> None: - """Align XDMA lifecycle with FireSim: unload/load with poll_mode=1.""" - logger.info(" reloading xdma kernel module") - loaded = subprocess.run( - "lsmod | grep -wq xdma", shell=True - ).returncode == 0 - if loaded: - _run_checked("sudo rmmod xdma") - _run_checked("sudo sh -c 'insmod $(find /lib/modules/$(uname -r) -name \"xdma.ko\") poll_mode=1'") - _run_checked("sudo chmod a+rw /dev/xdma*") - devs = glob.glob("/dev/xdma*_h2c_0") - if not devs: - raise RuntimeError("xdma driver loaded but /dev/xdma*_h2c_0 not found") - logger.info(f" xdma ready: {devs[0]}") - - -def _parse_mig_state(output: str) -> str: - text = output.lower() - if "calibration is still in-progress" in text: - return "busy" - - pairs = re.findall(r"^MIG_PROP\s+([^=]+)=(.*)$", output, flags=re.M) - if not pairs: - return "unknown" - - norm = [(k.strip().lower(), v.strip().lower()) for k, v in pairs] - prop_map = {k: v for k, v in norm} - - init_done = prop_map.get("init_calib_complete") - if init_done is not None: - if init_done in {"1", "true", "yes"}: - return "ready" - if init_done in {"0", "false", "no"}: - return "busy" - - if any(re.search(r"(in[-_ ]progress|busy)", v) for _, v in norm): - return "busy" - - fail_status = prop_map.get("calibration_fail.status") - has_pass = any(k.startswith("cal_status.") and v == "pass" for k, v in norm) - if has_pass and fail_status in {"false", "0", "no"}: - return "ready" - - status_ready = any( - ("status" in k or "calib" in k) and re.search(r"(done|pass|complete|success)", v) - for k, v in norm - ) - if status_ready: - return "ready" - - return "unknown" - - -def _wait_ddr_calibration(hw_server_url: str, serial: str, timeout_sec: int, tcl_script: str, logger) -> None: - if timeout_sec <= 0: - logger.info(" skip DDR calibration polling because timeout <= 0") - return - deadline = time.time() + timeout_sec - poll_sec = 5 - logger.info(f" polling DDR calibration status for up to {timeout_sec}s") - while True: - cmd = ( - f"vivado -mode tcl -source {tcl_script}" - f" -tclargs -hw_server {hw_server_url}" - ) - if serial: - cmd += f" -serial {serial}" - result = stream_run_logger( - cmd=cmd, - logger=logger, - stdout_prefix="pegasus mig", - stderr_prefix="pegasus mig", - ) - if result.returncode != 0: - raise RuntimeError( - f"mig status query failed: rc={result.returncode} stdout={result.stdout} stderr={result.stderr}" - ) - merged = f"{result.stdout}\n{result.stderr}" - state = _parse_mig_state(merged) - if state == "ready": - logger.info(" DDR calibration reported complete") - return - if state == "unknown": - raise RuntimeError("DDR calibration state unknown: no explicit MIG calibration status") - now = time.time() - if now >= deadline: - raise RuntimeError("DDR calibration did not complete before timeout") - remain = int(max(0, deadline - now)) - logger.info(f" DDR calibration still in progress, {remain}s remaining") - time.sleep(min(poll_sec, max(1, deadline - now))) - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - bbdir = get_buckyball_path() - - # ── 0. ensure hw_server is running (Vivado can't launch it in bbdev env) ─ - hw_server_url = input_data.get("hw_server", "localhost:3121") - hw_port = int(hw_server_url.split(":")[-1]) - if not _ensure_hw_server(hw_port, ctx.logger): - await check_result(ctx, 1, continue_run=False, - extra_fields={"error": "hw_server_unavailable"}, trace_id=origin_tid) - return - - bitstream = input_data.get( - "bitstream", - os.path.join(bbdir, "thirdparty", "pegasus", "vivado", "build", "PegasusTop.bit"), - ) - serial = input_data.get("serial", "") # empty = auto (first target) - ddr_settle_sec = int(input_data.get("ddr_settle_sec", 180)) - tcl_script = os.path.join( - bbdir, "thirdparty", "pegasus", "vivado", "scripts", "program_fpga.tcl" - ) - mig_status_tcl = os.path.join( - bbdir, "thirdparty", "pegasus", "vivado", "scripts", "query_mig_status.tcl" - ) - - if not os.path.exists(bitstream): - ctx.logger.error(f" bitstream not found: {bitstream}") - await check_result(ctx, 1, continue_run=False, - extra_fields={"error": "bitstream_not_found"}, trace_id=origin_tid) - return - - # ── 1. detect PCIe BDF ──────────────────────────────────────────────── - extended_bdf = input_data.get("bus_id") or _find_xdma_bdf() or _find_xdma_pci_bdf() - bridge = _bridge_bdf(extended_bdf) if extended_bdf else None - ctx.logger.info(f" XDMA device: {extended_bdf}, bridge: {bridge}") - - # ── 2. flash via Vivado (JTAG — does not require PCIe) ─────────────── - cmd = ( - f"vivado -mode tcl -source {tcl_script}" - f" -tclargs -bitstream_path {bitstream} -hw_server {hw_server_url}" - ) - if serial: - cmd += f" -serial {serial}" - - ctx.logger.info(f" flashing: {bitstream}") - result = stream_run_logger( - cmd=cmd, - logger=ctx.logger, - stdout_prefix="pegasus flash", - stderr_prefix="pegasus flash", - ) - - if result.returncode != 0: - await check_result(ctx, result.returncode, continue_run=False, - extra_fields={"bitstream": bitstream, "serial": serial}, - trace_id=origin_tid) - return - - # ── 3. PCIe remove → rescan so new bitstream's PCIe IP re-enumerates ─ - try: - if extended_bdf: - _pcie_disconnect(extended_bdf, bridge, ctx.logger) - else: - ctx.logger.info(" no xdma device found before flash, skip disconnect and try global rescan") - _pcie_reconnect(bridge, extended_bdf, ctx.logger) - _reload_xdma(ctx.logger) - _wait_ddr_calibration(hw_server_url, serial, ddr_settle_sec, mig_status_tcl, ctx.logger) - except Exception as e: - ctx.logger.error(f" post-flash PCIe/XDMA bring-up failed: {e}") - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": "post_flash_xdma_recover_failed", "detail": str(e)}, - trace_id=origin_tid, - ) - return - - await check_result( - ctx, - result.returncode, - continue_run=False, - extra_fields={"bitstream": bitstream, "serial": serial}, - trace_id=origin_tid, - ) diff --git a/bbdev/api/steps/pegasus/04_runworkload_api.step.py b/bbdev/api/steps/pegasus/04_runworkload_api.step.py deleted file mode 100644 index 62ea5c546..000000000 --- a/bbdev/api/steps/pegasus/04_runworkload_api.step.py +++ /dev/null @@ -1,15 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - -config = { - "name": "Pegasus Runworkload", - "description": "Run workload on AU280 (load Linux image + start CPU + collect UART)", - "flows": ["pegasus"], - "triggers": [api("POST", "/pegasus/runworkload")], - "enqueues": ["pegasus.runworkload"], -} - - -async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = req.body or {} - await ctx.enqueue({"topic": "pegasus.runworkload", "data": {**body, "_trace_id": ctx.trace_id}}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/pegasus/04_runworkload_event.step.py b/bbdev/api/steps/pegasus/04_runworkload_event.step.py deleted file mode 100644 index b656e0ddb..000000000 --- a/bbdev/api/steps/pegasus/04_runworkload_event.step.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -pegasus runworkload event handler - -Sequence: - 1. Resolve marshal image paths (kernel ELF + rootfs img) - 2. Build pegasus-driver if needed - 3. pegasus-driver load --kernel --rootfs - 4. pegasus-driver run --log --timeout -""" -import os -import sys -import datetime -import subprocess - -from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from utils.event_common import check_result, get_origin_trace_id - -config = { - "name": "pegasus-runworkload", - "description": "Load Linux image into DDR and run on AU280", - "flows": ["pegasus"], - "triggers": [queue("pegasus.runworkload")], - "enqueues": [], -} - -KERNEL_OUTPUT_DIR = "bb-tests/output/kernel" - - -def _resolve_images(bbdir: str, workload: str, board: str) -> tuple[str, str]: - """Return (kernel_path, rootfs_path) from bbkernel output.""" - output_dir = os.path.join(bbdir, KERNEL_OUTPUT_DIR) - kernel = os.path.join(output_dir, "pegasus-bin") - rootfs = os.path.join(output_dir, "pegasus.img") - return kernel, rootfs - - -def _build_driver(bbdir: str, logger) -> str | None: - """Build pegasus-driver with native incremental build. Returns binary path or None.""" - driver_src = os.path.join(bbdir, "thirdparty", "pegasus", "driver") - build_dir = os.path.join(driver_src, "build") - binary = os.path.join(build_dir, "pegasus-driver") - - logger.info(" building pegasus-driver ...") - os.makedirs(build_dir, exist_ok=True) - - # Ensure pkg-config can find nix-managed libraries (libelf from elfutils) - result_dir = os.path.join(bbdir, "result") - pkg_config_path = os.environ.get("PKG_CONFIG_PATH", "") - nix_pc = os.path.join(result_dir, "lib", "pkgconfig") - if nix_pc not in pkg_config_path: - pkg_config_path = f"{nix_pc}:{pkg_config_path}" if pkg_config_path else nix_pc - - cmake_result = stream_run_logger( - cmd="cmake .. -DCMAKE_BUILD_TYPE=Release", - logger=logger, - cwd=build_dir, - stdout_prefix="cmake", - stderr_prefix="cmake", - env={**os.environ, "PKG_CONFIG_PATH": pkg_config_path}, - ) - if cmake_result.returncode != 0: - logger.error(" cmake configure failed") - return None - - build_result = stream_run_logger( - cmd=f"cmake --build . -j$(nproc)", - logger=logger, - cwd=build_dir, - stdout_prefix="make", - stderr_prefix="make", - ) - if build_result.returncode != 0: - logger.error(" driver build failed") - return None - - logger.info(f" driver built: {binary}") - return binary - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - bbdir = get_buckyball_path() - - workload = input_data.get("workload", "interactive") - board = input_data.get("board", "chipyard") - timeout = int(input_data.get("timeout", 300)) - uart_dev = input_data.get("uart", "/dev/ttyUSB0") - control_dev = input_data.get("control", "/dev/xdma0_user") - h2c_dev = input_data.get("h2c", "/dev/xdma0_h2c_0") - - # Log output directory - ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - log_dir = os.path.join(bbdir, "arch", "log", ts) - os.makedirs(log_dir, exist_ok=True) - uart_log = os.path.join(log_dir, "pegasus_uart.log") - - # ── 1. Resolve image paths ──────────────────────────────────────────── - kernel, rootfs = _resolve_images(bbdir, workload, board) - ctx.logger.info(f" kernel: {kernel}") - ctx.logger.info(f" rootfs: {rootfs}") - - for path in (kernel, rootfs): - if not os.path.exists(path): - ctx.logger.error(f" image not found: {path} — run 'bbdev kernel --build' first") - await check_result(ctx, 1, continue_run=False, - extra_fields={"error": "image_not_found", "path": path}, - trace_id=origin_tid) - return - - # ── 2. Build driver ─────────────────────────────────────────────────── - driver = _build_driver(bbdir, ctx.logger) - if driver is None: - await check_result(ctx, 1, continue_run=False, - extra_fields={"error": "driver_build_failed"}, - trace_id=origin_tid) - return - - # ── 3. Load image into DDR ─────────────────────────────────────────── - load_cmd = ( - f"sudo {driver} load" - f" --kernel {kernel}" - f" --rootfs {rootfs}" - f" --h2c {h2c_dev}" - ) - ctx.logger.info(" loading images into DDR ...") - load_result = stream_run_logger( - cmd=load_cmd, - logger=ctx.logger, - stdout_prefix="pegasus load", - stderr_prefix="pegasus load", - ) - if load_result.returncode != 0: - await check_result(ctx, load_result.returncode, continue_run=False, - extra_fields={"error": "load_failed"}, - trace_id=origin_tid) - return - - # ── 4. Start CPU and collect UART ───────────────────────────────────── - run_cmd = ( - f"sudo {driver} run" - f" --control {control_dev}" - f" --uart {uart_dev}" - f" --log {uart_log}" - f" --timeout {timeout}" - ) - ctx.logger.info(f" starting CPU, collecting UART → {uart_log}") - run_result = stream_run_logger( - cmd=run_cmd, - logger=ctx.logger, - stdout_prefix="pegasus uart", - stderr_prefix="pegasus uart", - ) - - await check_result( - ctx, - run_result.returncode, - continue_run=False, - extra_fields={ - "workload": workload, - "board": board, - "uart_log": uart_log, - }, - trace_id=origin_tid, - ) diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index 2d09f07c1..31ec62ae9 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -22,13 +22,21 @@ def main(): "test_api_dc_synth.py", # "test_api_sardine_run.py", # "test_api_kernel_build.py", - # "test_api_pegasus_verilog.py", - # "test_api_pegasus_buildbitstream.py", - # "test_api_pegasus_flashbitstream.py", - # "test_api_pegasus_runworkload.py", + # "test_api_firesim_enumeratefpgas.py", # "test_api_firesim_buildbitstream.py", # "test_api_firesim_infrasetup.py", # "test_api_firesim_runworkload.py", + # "test_api_bebop_bemu_build.py", + # "test_api_bebop_bemu_sim.py", + # "test_api_bebop_verilator_clean.py", + # "test_api_bebop_verilator_verilog.py", + # "test_api_bebop_verilator_build.py", + # "test_api_bebop_verilator_sim.py", + # "test_api_bebop_verilator_run.py", + # "test_api_bebop_p2e_clean.py", + # "test_api_bebop_p2e_verilog.py", + # "test_api_bebop_p2e_buildbitstream.py", + # "test_api_bebop_p2e_runworkload.py", ] for name in tests: path = test_dir / name diff --git a/bbdev/api/tests/test_api_pegasus_verilog.py b/bbdev/api/tests/test_api_bebop_bemu_build.py similarity index 51% rename from bbdev/api/tests/test_api_pegasus_verilog.py rename to bbdev/api/tests/test_api_bebop_bemu_build.py index 058d6b521..862fb83a3 100644 --- a/bbdev/api/tests/test_api_pegasus_verilog.py +++ b/bbdev/api/tests/test_api_bebop_bemu_build.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev pegasus --verilog") +run_bbdev_case("bbdev bebop-bemu --build") diff --git a/bbdev/api/tests/test_api_bebop_bemu_sim.py b/bbdev/api/tests/test_api_bebop_bemu_sim.py new file mode 100644 index 000000000..f9230dd65 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_bemu_sim.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-bemu --sim '--binary ctest_vecunit_matmul_ones_singlecore-baremetal --batch'") diff --git a/bbdev/api/tests/test_api_bebop_p2e_buildbitstream.py b/bbdev/api/tests/test_api_bebop_p2e_buildbitstream.py new file mode 100644 index 000000000..cfbf70b9b --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_p2e_buildbitstream.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-p2e --buildbitstream '--config sims.p2e.P2EToyConfig'") diff --git a/bbdev/api/tests/test_api_bebop_p2e_clean.py b/bbdev/api/tests/test_api_bebop_p2e_clean.py new file mode 100644 index 000000000..32926b895 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_p2e_clean.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-p2e --clean '--config sims.p2e.P2EToyConfig'") diff --git a/bbdev/api/tests/test_api_bebop_p2e_runworkload.py b/bbdev/api/tests/test_api_bebop_p2e_runworkload.py new file mode 100644 index 000000000..2e2263698 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_p2e_runworkload.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-p2e --runworkload '--image vecunit_matmul_ones --bitstream /home/wanghui/Code/buckyball/bebop/build/p2e/sims.p2e.P2EToyConfig/bitstream.bit'") diff --git a/bbdev/api/tests/test_api_bebop_p2e_verilog.py b/bbdev/api/tests/test_api_bebop_p2e_verilog.py new file mode 100644 index 000000000..3878e9e86 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_p2e_verilog.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-p2e --verilog '--config sims.p2e.P2EToyConfig'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_build.py b/bbdev/api/tests/test_api_bebop_verilator_build.py new file mode 100644 index 000000000..5ff936e8f --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_verilator_build.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-verilator --build '--jobs 16'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_clean.py b/bbdev/api/tests/test_api_bebop_verilator_clean.py new file mode 100644 index 000000000..9a46a6ddd --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_verilator_clean.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-verilator --clean") diff --git a/bbdev/api/tests/test_api_bebop_verilator_run.py b/bbdev/api/tests/test_api_bebop_verilator_run.py new file mode 100644 index 000000000..6473fb549 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_verilator_run.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-verilator --run '--jobs 16 --binary ctest_vecunit_matmul_ones_singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig --batch'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_sim.py b/bbdev/api/tests/test_api_bebop_verilator_sim.py new file mode 100644 index 000000000..61de4d83f --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_verilator_sim.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-verilator --sim '--binary ctest_vecunit_matmul_ones_singlecore-baremetal --batch'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_verilog.py b/bbdev/api/tests/test_api_bebop_verilator_verilog.py new file mode 100644 index 000000000..cc64740c7 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_verilator_verilog.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-verilator --verilog '--config sims.verilator.BuckyballToyVerilatorConfig'") diff --git a/bbdev/api/tests/test_api_firesim_enumeratefpgas.py b/bbdev/api/tests/test_api_firesim_enumeratefpgas.py new file mode 100644 index 000000000..fd80b9226 --- /dev/null +++ b/bbdev/api/tests/test_api_firesim_enumeratefpgas.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev firesim --enumeratefpgas") diff --git a/bbdev/api/tests/test_api_pegasus_buildbitstream.py b/bbdev/api/tests/test_api_pegasus_buildbitstream.py deleted file mode 100644 index 3d16e2125..000000000 --- a/bbdev/api/tests/test_api_pegasus_buildbitstream.py +++ /dev/null @@ -1,3 +0,0 @@ -from _api_test_helper import run_bbdev_case - -run_bbdev_case("bbdev pegasus --buildbitstream") diff --git a/bbdev/api/tests/test_api_pegasus_flashbitstream.py b/bbdev/api/tests/test_api_pegasus_flashbitstream.py deleted file mode 100644 index d48d67dce..000000000 --- a/bbdev/api/tests/test_api_pegasus_flashbitstream.py +++ /dev/null @@ -1,3 +0,0 @@ -from _api_test_helper import run_bbdev_case - -run_bbdev_case("bbdev pegasus --flashbitstream") diff --git a/bbdev/api/tests/test_api_pegasus_runworkload.py b/bbdev/api/tests/test_api_pegasus_runworkload.py deleted file mode 100644 index 54cbc604e..000000000 --- a/bbdev/api/tests/test_api_pegasus_runworkload.py +++ /dev/null @@ -1,3 +0,0 @@ -from _api_test_helper import run_bbdev_case - -run_bbdev_case("bbdev pegasus --runworkload") diff --git a/bbdev/api/utils/stream_run.py b/bbdev/api/utils/stream_run.py index 11dcd0c0c..ed0332598 100644 --- a/bbdev/api/utils/stream_run.py +++ b/bbdev/api/utils/stream_run.py @@ -70,6 +70,7 @@ def read_stream( stream.close() # Start process + # Use errors='replace' to handle non-UTF-8 bytes from simulation UART output process = subprocess.Popen( cmd, cwd=cwd, @@ -77,6 +78,8 @@ def read_stream( stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + encoding='utf-8', + errors='replace', # Replace invalid UTF-8 bytes with � instead of crashing bufsize=1, executable=executable, env=env, diff --git a/bbdev/bbdev b/bbdev/bbdev index f8c2f430b..9b85f6cd9 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -75,9 +75,22 @@ WORKFLOW_COMMANDS = { "flashbitstream": 'Flash bitstream onto AU280. Args: "[--bitstream ] [--serial ]"', "runworkload": 'Run workload on AU280. Args: "[--workload ] [--board ] [--timeout ] [--uart ]"', }, - "p2e": { - "clean": 'Clean P2E build directory. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', - "verilog": 'Generate SystemVerilog for P2E DDR4 backdoor. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', + "bebop-bemu": { + "build": "Build Bebop emulator.", + "sim": 'Run Bebop emulator simulation. Args: "--binary [--batch]"', + }, + "bebop-verilator": { + "clean": 'Clean Bebop Verilator build directory. Args: "[--config ] [--output-dir ]"', + "verilog": 'Generate Bebop Verilog files. Args: "[--config ] [--output-dir ]"', + "build": 'Build Bebop Verilator simulation executable. Args: "[--jobs ] [--config ] [--output-dir ]"', + "sim": 'Run Bebop Verilator simulation. Args: "--binary [--batch] [--config ] [--output-dir ]"', + "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--batch] [--jobs ] [--config ] [--output-dir ]"', + }, + "bebop-p2e": { + "clean": 'Clean Bebop P2E build directory. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', + "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', + "buildbitstream": "Build Bebop P2E bitstream.", + "runworkload": 'Run Bebop P2E workload. Args: "--image --bitstream [--config ] [--output-dir ] [--log-dir ]"', }, "dc": { "run": 'Run Design Compiler synthesis. Args: "[--top ] [--config ] [--output-dir ]"', @@ -350,7 +363,7 @@ def run_server_mode(args, cmd_info): ) elif command in WORKFLOW_COMMANDS: - api_path = f"/{command}/{cmd_info['operation']}" + api_path = f"/{command.replace('-', '/')}/{cmd_info['operation']}" resp = requests.post( f"http://localhost:{args.port}{api_path}", json=cmd_info["args"], @@ -428,7 +441,7 @@ def run_script_mode(args, cmd_info): print(f"Available commands: {', '.join(ALL_COMMANDS)}") sys.exit(1) - api_path = f"/{command}/{cmd_info['operation']}" + api_path = f"/{command.replace('-', '/')}/{cmd_info['operation']}" # Reserve both ports atomically to prevent TOCTOU races with concurrent bbdev instances. # Sockets stay bound until the engine has taken over (HTTP is up), then are released. From 89ba55adcde43aed622176829f9089686f0f6028 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 14 May 2026 06:37:42 +0800 Subject: [PATCH 049/196] refactor: simplify output directory management in bebop P2E buildbitstream step --- .../bebop/p2e/02_buildbitstream_event.step.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index 013856703..0aec1ae92 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -4,7 +4,7 @@ Builds the FPGA bitstream via bebop CLI: 1. Resolve verilog source directory (VSRC_PATH) from config 2. Build bebop with p2e feature (vvacDir generated under build_dir via OUT_PATH) - 3. Run bebop p2e --buildbitstream with build_dir and output_dir + 3. Run bebop p2e --buildbitstream with build_dir (all outputs land in build_dir) """ import os import sys @@ -33,7 +33,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" - arch_dir = f"{bbdir}/arch" config_name = input_data.get("config", "sims.p2e.P2EToyConfig") vsrc_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("vsrc_dir")) @@ -48,11 +47,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") build_dir = input_data.get("build_dir") or f"{bebop_dir}/build/{config_name}-{timestamp}" - output_dir = input_data.get("output_dir") or f"{arch_dir}/build/p2e-bitstream-{timestamp}" - log_dir = os.path.join(output_dir, "log") os.makedirs(build_dir, exist_ok=True) - os.makedirs(output_dir, exist_ok=True) - os.makedirs(log_dir, exist_ok=True) # ── Build bebop with p2e feature ────────────────────────────────────── build_cmd = ( @@ -83,8 +78,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"cargo run --features p2e " f"-- p2e " f"--buildbitstream " - f"--build-dir=\"{build_dir}\" " - f"--output-dir=\"{output_dir}\"" + f"--build-dir=\"{build_dir}\"" ) ctx.logger.info(f"Running bebop p2e buildbitstream: {run_cmd}") run_result = stream_run_logger( @@ -95,7 +89,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="bebop p2e build-bitstream", ) - bitstream_path = os.path.join(output_dir, "fpgaCompDir", "bitstream.bit") + bitstream_path = os.path.join(build_dir, "fpgaCompDir", "bitstream.bit") await check_result( ctx, run_result.returncode, @@ -105,8 +99,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "config": config_name, "vsrc_dir": vsrc_dir, "build_dir": build_dir, - "output_dir": output_dir, - "log_dir": log_dir, "bitstream": bitstream_path, "timestamp": timestamp, }, From 4a084c8197954afddd7962f22491c1c4d4c5f05e Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 14 May 2026 11:18:30 +0800 Subject: [PATCH 050/196] feat: fix p2e runworkload api --- .../bebop/p2e/03_runworkload_event.step.py | 95 +++++++++++-------- bbdev/api/steps/kernel/scripts/elf2hex.py | 78 +++++++++++++++ 2 files changed, 133 insertions(+), 40 deletions(-) create mode 100755 bbdev/api/steps/kernel/scripts/elf2hex.py diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index 515c997c2..c6531d960 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -2,10 +2,11 @@ bebop p2e runworkload event handler Loads a kernel image into FPGA and runs the workload via bebop CLI: - 1. Validate image and bitstream paths - 2. Build bebop with p2e feature (VSRC_PATH from previous build, optional) - 3. Run bebop p2e --runworkload --image --bitstream + 1. Resolve image name to .hex file under bb-tests/output/ + 2. Validate bitstream .bit file path + 3. Run bebop p2e --runworkload --image --bitstream --log-dir """ +import glob import os import sys from datetime import datetime @@ -16,7 +17,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id @@ -29,58 +30,72 @@ } +def resolve_image(bbdir: str, image_name: str) -> str: + """Search bb-tests/output/ recursively for .hex and return its absolute path.""" + output_root = f"{bbdir}/bb-tests/output" + matches = glob.glob(f"{output_root}/**/{image_name}.hex", recursive=True) + if not matches: + return "" + return matches[0] + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" - arch_dir = f"{bbdir}/arch" - image = input_data.get("image", "") + image_name = input_data.get("image", "") bitstream = input_data.get("bitstream", "") - for label, path in (("image", image), ("bitstream", bitstream)): - if not path or not os.path.exists(path): - ctx.logger.error(f"{label} not found: {path}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": f"{label}_not_found", label: path}, - trace_id=origin_tid, - ) - return - - config_name = input_data.get("config", "sims.p2e.P2EToyConfig") - vsrc_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("vsrc_dir")) - timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - output_dir = input_data.get("output_dir") or os.path.dirname(os.path.abspath(bitstream)) - log_dir = input_data.get("log_dir") or f"{arch_dir}/log/p2e-runworkload-{timestamp}" - os.makedirs(log_dir, exist_ok=True) + image_path = resolve_image(bbdir, image_name) + if not image_path: + ctx.logger.error(f"image .hex not found for name: {image_name} (searched bb-tests/output/)") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "image_not_found", "image": image_name}, + trace_id=origin_tid, + ) + return - # ── Build bebop with p2e feature ────────────────────────────────────── - build_cmd = "cargo build --features p2e" - if os.path.isdir(vsrc_dir): - build_cmd += f" --config=\"env.VSRC_PATH='{vsrc_dir}'\"" - ctx.logger.info("Building bebop p2e ...") - build_result = stream_run_logger( - cmd=build_cmd, - logger=ctx.logger, - cwd=bebop_dir, - stdout_prefix="bebop p2e build", - stderr_prefix="bebop p2e build", - ) - if build_result.returncode != 0: + if not bitstream or not os.path.isfile(bitstream): + ctx.logger.error(f"bitstream .bit file not found: {bitstream}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "bitstream_not_found", "bitstream": bitstream}, + trace_id=origin_tid, + ) + return + + # Infer build_dir from bitstream path + # bitstream is typically: /fpgaCompDir/part_b0_f0/pnrDir/xepic_vvac_top_0_0.bit + # so build_dir is 4 levels up + build_dir = input_data.get("build_dir") + if not build_dir: + bitstream_abs = os.path.abspath(bitstream) + # Go up: pnrDir -> part_b0_f0 -> fpgaCompDir -> build_dir + build_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(bitstream_abs)))) + + if not os.path.isdir(build_dir): + ctx.logger.error(f"build_dir not found (inferred from bitstream): {build_dir}") await check_result( - ctx, build_result.returncode, continue_run=False, - extra_fields={"task": "runworkload", "stage": "build"}, + ctx, 1, continue_run=False, + extra_fields={"error": "build_dir_not_found", "build_dir": build_dir}, trace_id=origin_tid, ) return + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + log_dir = input_data.get("log_dir") or f"{bebop_dir}/log/p2e-runworkload-{timestamp}" + os.makedirs(log_dir, exist_ok=True) + # ── Run bebop p2e --runworkload ─────────────────────────────────────── run_cmd = ( + f"nix develop --ignore-environment --keep HOME --keep ALL_PROXY -c " f"cargo run --features p2e -- p2e " f"--runworkload " - f"--image=\"{image}\" " + f"--image=\"{image_path}\" " f"--bitstream=\"{bitstream}\" " + f"--build-dir=\"{build_dir}\" " f"--log-dir=\"{log_dir}\"" ) ctx.logger.info(f"Running bebop p2e runworkload: {run_cmd}") @@ -99,9 +114,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "runworkload", - "image": image, + "image": image_path, "bitstream": bitstream, - "output_dir": output_dir, + "build_dir": build_dir, "log_dir": log_dir, "uart_log": uart_log, "timestamp": timestamp, diff --git a/bbdev/api/steps/kernel/scripts/elf2hex.py b/bbdev/api/steps/kernel/scripts/elf2hex.py new file mode 100755 index 000000000..e9441e77f --- /dev/null +++ b/bbdev/api/steps/kernel/scripts/elf2hex.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""ELF -> hex converter for Buckyball P2E DDR loading. + +Output format: + @0 + XX + XX + ... +where XX is one byte per line (uppercase hex). + +The hex file represents bytes starting at DDR offset 0, which maps to +CPU address 0x80000000. All LOAD segments are concatenated by physical +address, with gaps padded by zeros. +""" + +import argparse +import subprocess +import sys +from pathlib import Path + + +def elf_to_bin(elf_path: Path, bin_path: Path, objcopy: str) -> None: + subprocess.run( + [objcopy, "-O", "binary", str(elf_path), str(bin_path)], + check=True, + ) + + +def bin_to_hex(bin_path: Path, hex_path: Path) -> int: + data = bin_path.read_bytes() + lines = ["@0"] + lines.extend(f"{b:02X}" for b in data) + hex_path.write_text("\n".join(lines) + "\n") + return len(data) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("elf", type=Path, help="Input ELF file") + parser.add_argument( + "hex", + type=Path, + nargs="?", + help="Output hex file (default: .hex)", + ) + parser.add_argument( + "--objcopy", + default="riscv64-unknown-elf-objcopy", + help="objcopy binary (default: riscv64-unknown-elf-objcopy)", + ) + args = parser.parse_args() + + if not args.elf.is_file(): + print(f"error: ELF not found: {args.elf}", file=sys.stderr) + return 1 + + hex_path = args.hex or args.elf.with_suffix(".hex") + bin_path = hex_path.with_suffix(".bin") + + try: + elf_to_bin(args.elf, bin_path, args.objcopy) + except FileNotFoundError: + print(f"error: objcopy not found: {args.objcopy}", file=sys.stderr) + print("hint: run inside `nix develop` shell", file=sys.stderr) + return 1 + except subprocess.CalledProcessError as e: + print(f"error: objcopy failed: {e}", file=sys.stderr) + return 1 + + n = bin_to_hex(bin_path, hex_path) + bin_path.unlink() + + print(f"wrote {hex_path} ({n} bytes, {n + 1} lines)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 360df0b90729f209c26136cf7fad9a3ad5be8a3d Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 15 May 2026 02:13:26 +0800 Subject: [PATCH 051/196] fix: update p2e runworkload command to include output path configuration --- bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index c6531d960..d3d097fc2 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -91,7 +91,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── Run bebop p2e --runworkload ─────────────────────────────────────── run_cmd = ( f"nix develop --ignore-environment --keep HOME --keep ALL_PROXY -c " - f"cargo run --features p2e -- p2e " + f"cargo run --features p2e " + f"--config=\"env.OUT_PATH='{build_dir}'\" " + f"-- p2e " f"--runworkload " f"--image=\"{image_path}\" " f"--bitstream=\"{bitstream}\" " From 0680ddd56cca580e72842c86cf1303a088b79637 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 17 May 2026 21:33:17 +0800 Subject: [PATCH 052/196] refactor: change optimization level from debug to release in Verilator command --- bbdev/api/steps/mill/01_verilator_verilog_event.step.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py index 8cd515b60..0b62a869a 100644 --- a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py @@ -78,7 +78,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: command += "--disable-annotation-unknown " command += "--strip-debug-info " - command += "-O=debug " + command += "-O=release " command += f"--split-verilog -o={build_dir} " result = stream_run_logger( From 4cdfe2235faabba3c4b5bde016578465d65c0472 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 18 May 2026 22:58:07 +0800 Subject: [PATCH 053/196] feat: add result-query-api to retrieve task results by trace_id --- .../steps/common/01_result_query_api.step.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 bbdev/api/steps/common/01_result_query_api.step.py diff --git a/bbdev/api/steps/common/01_result_query_api.step.py b/bbdev/api/steps/common/01_result_query_api.step.py new file mode 100644 index 000000000..9ff143a9d --- /dev/null +++ b/bbdev/api/steps/common/01_result_query_api.step.py @@ -0,0 +1,64 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "result-query-api", + "description": "Query task result by trace_id", + "flows": ["common"], + "triggers": [api("GET", "/result/{trace_id}")], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + """ + Query the result of a task by its trace_id. + + Returns: + 200: {"status": "success|failure|processing", "body": {...}} + 404: {"error": "No result found for trace_id"} + """ + trace_id = request.path_params.get("trace_id", "") + + if not trace_id: + return ApiResponse( + status=400, + body={"error": "trace_id is required"} + ) + + # Check for success state + success_state = await ctx.state.get(trace_id, "success") + if success_state: + return ApiResponse( + status=200, + body={ + "status": "success", + "body": success_state.get("body", success_state) + } + ) + + # Check for failure state + failure_state = await ctx.state.get(trace_id, "failure") + if failure_state: + return ApiResponse( + status=200, + body={ + "status": "failure", + "body": failure_state.get("body", failure_state) + } + ) + + # Check for processing state + processing_state = await ctx.state.get(trace_id, "processing") + if processing_state: + return ApiResponse( + status=200, + body={ + "status": "processing", + "body": {} + } + ) + + # No state found + return ApiResponse( + status=404, + body={"error": f"No result found for trace_id: {trace_id}"} + ) From efbf84a2f6406a2533558ce32d3d0a6a3efd48dd Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 19 May 2026 16:31:21 +0800 Subject: [PATCH 054/196] feat: implement batch regression workflows for bebop bemu and p2e, including new API and event steps for managing tests and workloads --- .../api/steps/bebop/bemu/02_sim_event.step.py | 2 + .../api/steps/bebop/bemu/03_batch_api.step.py | 32 +++++ .../steps/bebop/bemu/03_batch_event.step.py | 108 ++++++++++++++ .../api/steps/bebop/bemu/scripts/nextest.toml | 19 +++ .../bebop/bemu/scripts/workloads-elf.toml | 41 ++++++ .../bebop/bemu/scripts/workloads-pk.toml | 41 ++++++ .../api/steps/bebop/p2e/04_batch_api.step.py | 47 ++++++ .../steps/bebop/p2e/04_batch_event.step.py | 123 ++++++++++++++++ .../api/steps/bebop/p2e/scripts/nextest.toml | 23 +++ .../bebop/p2e/scripts/workloads-elf.toml | 24 ++++ .../steps/bebop/p2e/scripts/workloads-pk.toml | 24 ++++ .../bebop/verilator/01_clean_api.step.py | 8 +- .../bebop/verilator/03_build_api.step.py | 8 +- .../bebop/verilator/03_build_event.step.py | 11 +- .../steps/bebop/verilator/05_run_api.step.py | 8 +- .../bebop/verilator/06_batch_api.step.py | 46 ++++++ .../bebop/verilator/06_batch_event.step.py | 135 ++++++++++++++++++ .../bebop/verilator/scripts/nextest.toml | 19 +++ .../verilator/scripts/workloads-elf.toml | 44 ++++++ .../bebop/verilator/scripts/workloads-pk.toml | 40 ++++++ bbdev/api/tests/test_all_api_serial.py | 7 +- bbdev/api/tests/test_api_bebop_bemu_batch.py | 3 + bbdev/api/tests/test_api_bebop_bemu_sim.py | 3 +- bbdev/api/tests/test_api_bebop_p2e_batch.py | 4 + .../tests/test_api_bebop_verilator_batch.py | 3 + .../tests/test_api_bebop_verilator_build.py | 2 +- .../tests/test_api_bebop_verilator_clean.py | 2 +- .../api/tests/test_api_bebop_verilator_sim.py | 2 +- bbdev/bbdev | 25 +--- 29 files changed, 823 insertions(+), 31 deletions(-) create mode 100644 bbdev/api/steps/bebop/bemu/03_batch_api.step.py create mode 100644 bbdev/api/steps/bebop/bemu/03_batch_event.step.py create mode 100644 bbdev/api/steps/bebop/bemu/scripts/nextest.toml create mode 100644 bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml create mode 100644 bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml create mode 100644 bbdev/api/steps/bebop/p2e/04_batch_api.step.py create mode 100644 bbdev/api/steps/bebop/p2e/04_batch_event.step.py create mode 100644 bbdev/api/steps/bebop/p2e/scripts/nextest.toml create mode 100644 bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml create mode 100644 bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml create mode 100644 bbdev/api/steps/bebop/verilator/06_batch_api.step.py create mode 100644 bbdev/api/steps/bebop/verilator/06_batch_event.step.py create mode 100644 bbdev/api/steps/bebop/verilator/scripts/nextest.toml create mode 100644 bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml create mode 100644 bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml create mode 100644 bbdev/api/tests/test_api_bebop_bemu_batch.py create mode 100644 bbdev/api/tests/test_api_bebop_p2e_batch.py create mode 100644 bbdev/api/tests/test_api_bebop_verilator_batch.py diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index d2f7bee90..9324b2c53 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -53,10 +53,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: os.makedirs(log_dir, exist_ok=True) # ── Run bebop bemu ──────────────────────────────────────────────────── + pk_flag = " --pk" if input_data.get("pk") else "" run_cmd = ( f"cargo run --features bemu -- bemu " f"--elf=\"{binary_path}\" " f"--log-dir=\"{log_dir}\"" + f"{pk_flag}" ) ctx.logger.info(f"Running bebop bemu: {run_cmd}") run_result = stream_run_logger( diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py new file mode 100644 index 000000000..f2f05d08a --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -0,0 +1,32 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "bebop-bemu-batch-api", + "description": "Run bebop bemu nextest batch regression", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/bemu/batch")], + "enqueues": ["bebop.bemu.batch"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + + test_type = body.get("test") + if not test_type: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --test must be specified (elf-tests or pk-tests)"} + ) + + if test_type not in ["elf-tests", "pk-tests"]: + return ApiResponse( + status=400, + body={"error": f"Invalid test type: {test_type}. Must be 'elf-tests' or 'pk-tests'"} + ) + + await ctx.enqueue({ + "topic": "bebop.bemu.batch", + "data": {"test": test_type, "_trace_id": ctx.trace_id} + }) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py new file mode 100644 index 000000000..7090cbacb --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -0,0 +1,108 @@ +""" +bebop bemu batch event handler + +Runs bebop bemu nextest batch regression: + 1. Build bebop with bemu feature + 2. Run cargo nextest with bemu-specific config +""" +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-bemu-batch", + "description": "Run bebop bemu nextest batch regression", + "flows": ["bebop"], + "triggers": [queue("bebop.bemu.batch")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + bebop_dir = f"{bbdir}/bebop" + nextest_config = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/nextest.toml" + elf_root = f"{bbdir}/bb-tests/output" + + # ── Determine workload file based on test type ──────────────────────── + test_type = input_data.get("test", "elf-tests") + if test_type == "elf-tests": + workload_toml = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/workloads-elf.toml" + elif test_type == "pk-tests": + workload_toml = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/workloads-pk.toml" + else: + ctx.logger.error(f"Invalid test type: {test_type}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_test_type", "test": test_type}, + trace_id=origin_tid, + ) + return + + ctx.logger.info(f"Running {test_type} with workload config: {workload_toml}") + + # ── Build bebop bemu ────────────────────────────────────────────────── + build_cmd = "nix develop -c cargo build --features bemu --tests" + ctx.logger.info("Building bebop bemu (tests)...") + build_result = stream_run_logger( + cmd=build_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop bemu build", + stderr_prefix="bebop bemu build", + ) + + if build_result.returncode != 0: + await check_result( + ctx, build_result.returncode, continue_run=False, + extra_fields={"task": "build", "backend": "bemu"}, + trace_id=origin_tid, + ) + return + + # ── Run nextest ─────────────────────────────────────────────────────── + # Pass parameters via environment variables (nextest doesn't support custom CLI args after `--`) + env = os.environ.copy() + env.update({ + "BEBOP_WORKLOAD_TOML": workload_toml, + "BEBOP_BB_TESTS_ROOT": elf_root, + }) + nextest_cmd = ( + f"nix develop -c cargo nextest run --features bemu --test test_bemu " + f"--config-file \"{nextest_config}\"" + ) + + ctx.logger.info(f"Running bebop bemu nextest: {nextest_cmd}") + ctx.logger.info(f"Environment: BEBOP_WORKLOAD_TOML={workload_toml}, BEBOP_BB_TESTS_ROOT={elf_root}") + run_result = stream_run_logger( + cmd=nextest_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop bemu batch", + stderr_prefix="bebop bemu batch", + env=env, + ) + + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "batch", + "backend": "bemu", + "test_type": test_type, + "nextest_config": nextest_config, + "workload_toml": workload_toml, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/bemu/scripts/nextest.toml b/bbdev/api/steps/bebop/bemu/scripts/nextest.toml new file mode 100644 index 000000000..738cb77c0 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/scripts/nextest.toml @@ -0,0 +1,19 @@ +# ============================================================================= +# Nextest configuration for bebop bemu regression +# ============================================================================= +# +# bemu = lightweight emulator tests, parallel execution (fill all cores). +# Loaded by bbdev when invoking `bbdev bebop-bemu --batch`. + +[profile.default] +failure-output = "immediate" +success-output = "never" +status-level = "all" +fail-fast = false +slow-timeout = { period = "60s", terminate-after = 5 } +retries = 0 +junit = { path = "target/nextest/junit-bbdev-bemu.xml", report-name = "bbdev-bemu-batch" } + +[[profile.default.overrides]] +filter = "binary(test_bemu)" +slow-timeout = { period = "60s", terminate-after = 5 } diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml new file mode 100644 index 000000000..1fec486f0 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml @@ -0,0 +1,41 @@ +# ============================================================================= +# Bebop bemu workload list - ELF baremetal tests +# ============================================================================= +# Test cases to run in bebop bemu batch regression with --elf flag. +# search_path is relative to bb-tests/output (or absolute path). +# Each entry under tests is the file STEM (without extension). + +[workloads] +search_path = "workloads/src/CTest/toy" +tests = [ + "ctest_mvin_mvout_test_singlecore-baremetal", + "ctest_vecunit_matmul_16xn_ones_singlecore-baremetal", + "ctest_vecunit_matmul_16xn_random1_singlecore-baremetal", + "ctest_vecunit_matmul_16xn_random2_singlecore-baremetal", + "ctest_vecunit_matmul_16xn_random3_singlecore-baremetal", + "ctest_vecunit_matmul_16xn_zero_random_singlecore-baremetal", + "ctest_vecunit_matmul_col_row_vector_singlecore-baremetal", + "ctest_vecunit_matmul_identity_random_singlecore-baremetal", + "ctest_vecunit_matmul_ones_singlecore-baremetal", + "ctest_vecunit_matmul_random1_singlecore-baremetal", + "ctest_vecunit_matmul_random2_singlecore-baremetal", + "ctest_vecunit_matmul_zero_random_singlecore-baremetal", + "ctest_relu_test_singlecore-baremetal", + "ctest_transpose_test_singlecore-baremetal", + "ctest_transpose_16xn_test_singlecore-baremetal", + "ctest_im2col_test_singlecore-baremetal", + "ctest_quant_test_singlecore-baremetal", + "ctest_dequant_test_singlecore-baremetal", + "ctest_bdb_counter_test_singlecore-baremetal", + "ctest_bfp_test_singlecore-baremetal", + "ctest_gemmini_os_cisc_basic_test_singlecore-baremetal", + "ctest_gemmini_os_cisc_atranspose_test_singlecore-baremetal", + "ctest_gemmini_os_cisc_btranspose_test_singlecore-baremetal", + "ctest_gemmini_os_cisc_shift_test_singlecore-baremetal", + "ctest_gemmini_os_risc_atranspose_test_singlecore-baremetal", + "ctest_gemmini_os_risc_btranspose_test_singlecore-baremetal", + "ctest_gemmini_os_risc_abtranspose_test_singlecore-baremetal", + "ctest_gemmini_os_risc_shift_test_singlecore-baremetal", + "ctest_gemmini_ws_cisc_conv_test_singlecore-baremetal", + "ctest_gemmini_ws_risc_basic_test_singlecore-baremetal", +] \ No newline at end of file diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml new file mode 100644 index 000000000..7d9957051 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml @@ -0,0 +1,41 @@ +# ============================================================================= +# Bebop bemu workload list - PK (proxy kernel) tests +# ============================================================================= +# Test cases to run in bebop bemu batch regression with --pk flag. +# search_path is relative to bb-tests/output (or absolute path). +# Each entry under tests is the file STEM (without extension). + +[workloads] +search_path = "workloads/src/CTest/toy" +tests = [ + "ctest_mvin_mvout_test-linux", + "ctest_vecunit_matmul_16xn_ones-linux", + "ctest_vecunit_matmul_16xn_random1-linux", + "ctest_vecunit_matmul_16xn_random2-linux", + "ctest_vecunit_matmul_16xn_random3-linux", + "ctest_vecunit_matmul_16xn_zero_random-linux", + "ctest_vecunit_matmul_col_row_vector-linux", + "ctest_vecunit_matmul_identity_random-linux", + "ctest_vecunit_matmul_ones-linux", + "ctest_vecunit_matmul_random1-linux", + "ctest_vecunit_matmul_random2-linux", + "ctest_vecunit_matmul_zero_random-linux", + "ctest_relu_test-linux", + "ctest_transpose_test-linux", + "ctest_transpose_16xn_test-linux", + "ctest_im2col_test-linux", + "ctest_quant_test-linux", + "ctest_dequant_test-linux", + "ctest_bdb_counter_test-linux", + "ctest_bfp_test-linux", + "ctest_gemmini_os_cisc_basic_test-linux", + "ctest_gemmini_os_cisc_atranspose_test-linux", + "ctest_gemmini_os_cisc_btranspose_test-linux", + "ctest_gemmini_os_cisc_shift_test-linux", + "ctest_gemmini_os_risc_atranspose_test-linux", + "ctest_gemmini_os_risc_btranspose_test-linux", + "ctest_gemmini_os_risc_abtranspose_test-linux", + "ctest_gemmini_os_risc_shift_test-linux", + "ctest_gemmini_ws_cisc_conv_test-linux", + "ctest_gemmini_ws_risc_basic_test-linux", +] \ No newline at end of file diff --git a/bbdev/api/steps/bebop/p2e/04_batch_api.step.py b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py new file mode 100644 index 000000000..9302e32ba --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py @@ -0,0 +1,47 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "bebop-p2e-batch-api", + "description": "Run bebop p2e nextest batch regression", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/p2e/batch")], + "enqueues": ["bebop.p2e.batch"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + + bitstream = body.get("bitstream", "") + build_dir = body.get("build-dir") or body.get("build_dir", "") + if not bitstream or not build_dir: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 400, + "message": "--bitstream and --build-dir parameters are required", + }, + ) + + test_type = body.get("test") + if not test_type: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --test must be specified (elf-tests or pk-tests)"} + ) + + if test_type not in ["elf-tests", "pk-tests"]: + return ApiResponse( + status=400, + body={"error": f"Invalid test type: {test_type}. Must be 'elf-tests' or 'pk-tests'"} + ) + + data = { + "bitstream": bitstream, + "build_dir": build_dir, + "test": test_type, + } + await ctx.enqueue({"topic": "bebop.p2e.batch", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py new file mode 100644 index 000000000..02fd347f3 --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py @@ -0,0 +1,123 @@ +""" +bebop p2e batch event handler + +Runs bebop p2e nextest batch regression: + 1. Build bebop with p2e feature + 2. Run cargo nextest with p2e-specific config (serial, FPGA single-resource) +""" +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-p2e-batch", + "description": "Run bebop p2e nextest batch regression", + "flows": ["bebop"], + "triggers": [queue("bebop.p2e.batch")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + bebop_dir = f"{bbdir}/bebop" + nextest_config = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/nextest.toml" + elf_root = f"{bbdir}/bb-tests/output" + + bitstream = input_data.get("bitstream", "") + build_dir = input_data.get("build_dir", "") + if not bitstream or not build_dir: + ctx.logger.error("Missing required parameters: bitstream and build_dir") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_params"}, + trace_id=origin_tid, + ) + return + + # ── Determine workload file based on test type ──────────────────────── + test_type = input_data.get("test", "elf-tests") + if test_type == "elf-tests": + workload_toml = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/workloads-elf.toml" + elif test_type == "pk-tests": + workload_toml = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/workloads-pk.toml" + else: + ctx.logger.error(f"Invalid test type: {test_type}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_test_type", "test": test_type}, + trace_id=origin_tid, + ) + return + + ctx.logger.info(f"Running {test_type} with workload config: {workload_toml}") + + # ── Build bebop p2e (tests) ─────────────────────────────────────────── + build_cmd = "nix develop -c cargo build --features p2e --tests" + ctx.logger.info("Building bebop p2e (tests)...") + build_result = stream_run_logger( + cmd=build_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop p2e build", + stderr_prefix="bebop p2e build", + ) + + if build_result.returncode != 0: + await check_result( + ctx, build_result.returncode, continue_run=False, + extra_fields={"task": "build", "backend": "p2e"}, + trace_id=origin_tid, + ) + return + + # ── Run nextest ─────────────────────────────────────────────────────── + # Pass parameters via environment variables (nextest doesn't support custom CLI args after `--`) + env = os.environ.copy() + env.update({ + "BEBOP_WORKLOAD_TOML": workload_toml, + "BEBOP_BB_TESTS_ROOT": elf_root, + "BEBOP_P2E_BITSTREAM": bitstream, + "BEBOP_P2E_BUILD_DIR": build_dir, + }) + nextest_cmd = ( + f"nix develop -c cargo nextest run --features p2e --test test_p2e " + f"--config-file \"{nextest_config}\"" + ) + + ctx.logger.info(f"Running bebop p2e nextest: {nextest_cmd}") + ctx.logger.info(f"Environment: BEBOP_WORKLOAD_TOML={workload_toml}, BEBOP_BB_TESTS_ROOT={elf_root}") + run_result = stream_run_logger( + cmd=nextest_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop p2e batch", + stderr_prefix="bebop p2e batch", + env=env, + ) + + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "batch", + "backend": "p2e", + "bitstream": bitstream, + "build_dir": build_dir, + "test_type": test_type, + "nextest_config": nextest_config, + "workload_toml": workload_toml, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/p2e/scripts/nextest.toml b/bbdev/api/steps/bebop/p2e/scripts/nextest.toml new file mode 100644 index 000000000..cc7bd5fa9 --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/scripts/nextest.toml @@ -0,0 +1,23 @@ +# ============================================================================= +# Nextest configuration for bebop p2e regression +# ============================================================================= +# +# p2e = FPGA workload tests, serial execution (single FPGA resource). +# Loaded by bbdev when invoking `bbdev bebop-p2e --batch`. + +[profile.default] +failure-output = "immediate" +success-output = "never" +status-level = "all" +fail-fast = false +slow-timeout = { period = "360s", terminate-after = 5 } +retries = 0 +junit = { path = "target/nextest/junit-bbdev-p2e.xml", report-name = "bbdev-p2e-batch" } + +[test-groups.p2e-serial] +max-threads = 1 + +[[profile.default.overrides]] +filter = "binary(test_p2e)" +test-group = "p2e-serial" +slow-timeout = { period = "360s", terminate-after = 5 } diff --git a/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml new file mode 100644 index 000000000..5d009a0a6 --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml @@ -0,0 +1,24 @@ +# ============================================================================= +# Bebop p2e workload list - ELF baremetal tests +# ============================================================================= +# search_path is relative to bb-tests/output (or absolute path). +# Each entry under tests is the file STEM (without extension). + +[workloads] +search_path = "workloads/src/CTest/toy" +tests = [ + "ctest_bdb_counter_test_singlecore-baremetal", + "ctest_bfp_test_singlecore-baremetal", + "ctest_dequant_test_singlecore-baremetal", + "ctest_gemmini_os_cisc_basic_test_singlecore-baremetal", + "ctest_gemmini_os_cisc_atranspose_test_singlecore-baremetal", + "ctest_gemmini_os_cisc_btranspose_test_singlecore-baremetal", + "ctest_gemmini_os_cisc_shift_test_singlecore-baremetal", + "ctest_gemmini_os_risc_basic_test_singlecore-baremetal", + "ctest_gemmini_os_risc_atranspose_test_singlecore-baremetal", + "ctest_gemmini_os_risc_btranspose_test_singlecore-baremetal", + "ctest_gemmini_os_risc_abtranspose_test_singlecore-baremetal", + "ctest_gemmini_os_risc_shift_test_singlecore-baremetal", + "ctest_gemmini_ws_cisc_conv_test_singlecore-baremetal", + "ctest_gemmini_ws_risc_basic_test_singlecore-baremetal", +] diff --git a/bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml new file mode 100644 index 000000000..3e898a238 --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml @@ -0,0 +1,24 @@ +# ============================================================================= +# Bebop p2e workload list - PK (proxy kernel) tests +# ============================================================================= +# search_path is relative to bb-tests/output (or absolute path). +# Each entry under tests is the file STEM (without extension). + +[workloads] +search_path = "workloads/src/CTest/toy" +tests = [ + "ctest_bdb_counter_test-linux", + "ctest_bfp_test-linux", + "ctest_dequant_test-linux", + "ctest_gemmini_os_cisc_basic_test-linux", + "ctest_gemmini_os_cisc_atranspose_test-linux", + "ctest_gemmini_os_cisc_btranspose_test-linux", + "ctest_gemmini_os_cisc_shift_test-linux", + "ctest_gemmini_os_risc_basic_test-linux", + "ctest_gemmini_os_risc_atranspose_test-linux", + "ctest_gemmini_os_risc_btranspose_test-linux", + "ctest_gemmini_os_risc_abtranspose_test-linux", + "ctest_gemmini_os_risc_shift_test-linux", + "ctest_gemmini_ws_cisc_conv_test-linux", + "ctest_gemmini_ws_risc_basic_test-linux", +] diff --git a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py index 2a2a7d5b1..bbec8298b 100644 --- a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py +++ b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py @@ -15,7 +15,13 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = req.body or {} - config_name = body.get("config", "sims.verilator.BuckyballToyVerilatorConfig") + config_name = body.get("config") + if not config_name: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --config must be specified"} + ) + build_dir = get_verilator_build_dir(bbdir, config_name, body.get("output_dir")) data = { diff --git a/bbdev/api/steps/bebop/verilator/03_build_api.step.py b/bbdev/api/steps/bebop/verilator/03_build_api.step.py index 8a9c5981b..342ddfb18 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_api.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_api.step.py @@ -15,7 +15,13 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = request.body or {} - arch_config = body.get("config", "sims.verilator.BuckyballToyVerilatorConfig") + arch_config = body.get("config") + if not arch_config: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --config must be specified"} + ) + vsrc_dir = get_verilator_build_dir(bbdir, arch_config, body.get("vsrc_dir")) data = { diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index cb14e0a81..8dcb550f4 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -30,7 +30,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" - arch_config = input_data.get("config", "sims.verilator.BuckyballToyVerilatorConfig") + arch_config = input_data.get("config") + if not arch_config: + ctx.logger.error("Missing required parameter: config must be specified") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_config"}, + trace_id=origin_tid, + ) + return + vsrc_dir = get_verilator_build_dir(bbdir, arch_config, input_data.get("vsrc_dir")) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") diff --git a/bbdev/api/steps/bebop/verilator/05_run_api.step.py b/bbdev/api/steps/bebop/verilator/05_run_api.step.py index 0eb7ae40e..b80c170da 100644 --- a/bbdev/api/steps/bebop/verilator/05_run_api.step.py +++ b/bbdev/api/steps/bebop/verilator/05_run_api.step.py @@ -15,7 +15,13 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = request.body or {} - config_name = body.get("config", "sims.verilator.BuckyballToyVerilatorConfig") + config_name = body.get("config") + if not config_name: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --config must be specified"} + ) + build_dir = get_verilator_build_dir(bbdir, config_name, body.get("output_dir")) binary = body.get("binary", "") diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py new file mode 100644 index 000000000..e347e11d6 --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -0,0 +1,46 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path, get_verilator_build_dir + +config = { + "name": "bebop-verilator-batch-api", + "description": "Run bebop verilator nextest batch regression", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/verilator/batch")], + "enqueues": ["bebop.verilator.batch"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = request.body or {} + + arch_config = body.get("config") + if not arch_config: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --config must be specified"} + ) + + test_type = body.get("test") + if not test_type: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --test must be specified (elf-tests or pk-tests)"} + ) + + if test_type not in ["elf-tests", "pk-tests"]: + return ApiResponse( + status=400, + body={"error": f"Invalid test type: {test_type}. Must be 'elf-tests' or 'pk-tests'"} + ) + + vsrc_dir = get_verilator_build_dir(bbdir, arch_config, body.get("vsrc_dir")) + + data = { + "config": arch_config, + "vsrc_dir": vsrc_dir, + "test": test_type, + } + await ctx.enqueue({"topic": "bebop.verilator.batch", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py new file mode 100644 index 000000000..18155124e --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -0,0 +1,135 @@ +""" +bebop verilator batch event handler + +Runs bebop verilator nextest batch regression: + 1. Build bebop with verilator feature and VSRC_PATH + 2. Run cargo nextest with verilator-specific config (serial execution) +""" +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "bebop-verilator-batch", + "description": "Run bebop verilator nextest batch regression", + "flows": ["bebop"], + "triggers": [queue("bebop.verilator.batch")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + bebop_dir = f"{bbdir}/bebop" + nextest_config = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/nextest.toml" + elf_root = f"{bbdir}/bb-tests/output" + + arch_config = input_data.get("config") + if not arch_config: + ctx.logger.error("Missing required parameter: config must be specified") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_config"}, + trace_id=origin_tid, + ) + return + + # ── Determine workload file based on test type ──────────────────────── + test_type = input_data.get("test", "elf-tests") + if test_type == "elf-tests": + workload_toml = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/workloads-elf.toml" + elif test_type == "pk-tests": + workload_toml = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/workloads-pk.toml" + else: + ctx.logger.error(f"Invalid test type: {test_type}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_test_type", "test": test_type}, + trace_id=origin_tid, + ) + return + + ctx.logger.info(f"Running {test_type} with workload config: {workload_toml}") + + vsrc_dir = get_verilator_build_dir(bbdir, arch_config, input_data.get("vsrc_dir")) + if not os.path.isdir(vsrc_dir): + ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "vsrc_not_found", "vsrc_dir": vsrc_dir}, + trace_id=origin_tid, + ) + return + + # ── Build bebop verilator (tests) ───────────────────────────────────── + build_cmd = ( + f"nix develop -c cargo build --features verilator --tests " + f"--config=\"env.VSRC_PATH='{vsrc_dir}'\"" + ) + ctx.logger.info("Building bebop verilator (tests)...") + build_result = stream_run_logger( + cmd=build_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop verilator build", + stderr_prefix="bebop verilator build", + ) + + if build_result.returncode != 0: + await check_result( + ctx, build_result.returncode, continue_run=False, + extra_fields={"task": "build", "backend": "verilator", "vsrc_dir": vsrc_dir}, + trace_id=origin_tid, + ) + return + + # ── Run nextest ─────────────────────────────────────────────────────── + # Pass parameters via environment variables (nextest doesn't support custom CLI args after `--`) + env = os.environ.copy() + env.update({ + "BEBOP_WORKLOAD_TOML": workload_toml, + "BEBOP_BB_TESTS_ROOT": elf_root, + "VSRC_PATH": vsrc_dir, + }) + nextest_cmd = ( + f"nix develop -c cargo nextest run --features verilator --test test_verilator " + f"--config-file \"{nextest_config}\" " + f"--config=\"env.VSRC_PATH='{vsrc_dir}'\"" + ) + + ctx.logger.info(f"Running bebop verilator nextest: {nextest_cmd}") + ctx.logger.info(f"Environment: BEBOP_WORKLOAD_TOML={workload_toml}, BEBOP_BB_TESTS_ROOT={elf_root}") + run_result = stream_run_logger( + cmd=nextest_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop verilator batch", + stderr_prefix="bebop verilator batch", + env=env, + ) + + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "batch", + "backend": "verilator", + "config": arch_config, + "vsrc_dir": vsrc_dir, + "test_type": test_type, + "nextest_config": nextest_config, + "workload_toml": workload_toml, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/verilator/scripts/nextest.toml b/bbdev/api/steps/bebop/verilator/scripts/nextest.toml new file mode 100644 index 000000000..dd2b2386f --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/scripts/nextest.toml @@ -0,0 +1,19 @@ +# ============================================================================= +# Nextest configuration for bebop verilator regression +# ============================================================================= +# +# verilator = RTL simulation, parallel execution (fill all cores). +# Loaded by bbdev when invoking `bbdev bebop-verilator --batch`. + +[profile.default] +failure-output = "immediate" +success-output = "never" +status-level = "all" +fail-fast = false +slow-timeout = { period = "360s", terminate-after = 5 } +retries = 0 +junit = { path = "target/nextest/junit-bbdev-verilator.xml", report-name = "bbdev-verilator-batch" } + +[[profile.default.overrides]] +filter = "binary(test_verilator)" +slow-timeout = { period = "360s", terminate-after = 5 } diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml new file mode 100644 index 000000000..0312c92ee --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml @@ -0,0 +1,44 @@ +# ============================================================================= +# Bebop verilator workload list - ELF baremetal tests +# ============================================================================= +# search_path is relative to bb-tests/output (or absolute path). +# Each entry under tests is the file STEM (without extension). + +[workloads] +search_path = "workloads/src/CTest/toy" +tests = [ + "ctest_mvin_mvout_test_singlecore-baremetal", + "ctest_vecunit_matmul_16xn_ones_singlecore-baremetal", + "ctest_vecunit_matmul_16xn_random1_singlecore-baremetal", + "ctest_vecunit_matmul_16xn_random2_singlecore-baremetal", + "ctest_vecunit_matmul_16xn_random3_singlecore-baremetal", + "ctest_vecunit_matmul_16xn_zero_random_singlecore-baremetal", + "ctest_vecunit_matmul_col_row_vector_singlecore-baremetal", + "ctest_vecunit_matmul_identity_random_singlecore-baremetal", + "ctest_vecunit_matmul_ones_singlecore-baremetal", + "ctest_vecunit_matmul_random1_singlecore-baremetal", + "ctest_vecunit_matmul_random2_singlecore-baremetal", + "ctest_vecunit_matmul_zero_random_singlecore-baremetal", + "ctest_relu_test_singlecore-baremetal", + "ctest_transpose_test_singlecore-baremetal", + "ctest_transpose_16xn_test_singlecore-baremetal", + "ctest_im2col_test_singlecore-baremetal", + "ctest_quant_test_singlecore-baremetal", + "ctest_dequant_test_singlecore-baremetal", + "ctest_bdb_counter_test_singlecore-baremetal", + "ctest_bfp_test_singlecore-baremetal", + "ctest_gemmini_os_cisc_basic_test_singlecore-baremetal", + "ctest_gemmini_os_cisc_atranspose_test_singlecore-baremetal", + "ctest_gemmini_os_cisc_btranspose_test_singlecore-baremetal", + "ctest_gemmini_os_cisc_shift_test_singlecore-baremetal", + "ctest_gemmini_os_risc_atranspose_test_singlecore-baremetal", + "ctest_gemmini_os_risc_btranspose_test_singlecore-baremetal", + "ctest_gemmini_os_risc_abtranspose_test_singlecore-baremetal", + "ctest_gemmini_os_risc_shift_test_singlecore-baremetal", + "ctest_gemmini_ws_cisc_conv_test_singlecore-baremetal", + "ctest_gemmini_ws_risc_basic_test_singlecore-baremetal", + + "ctest_tlb_test_singlecore-baremetal", +] + + diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml new file mode 100644 index 000000000..91a4b3c11 --- /dev/null +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml @@ -0,0 +1,40 @@ +# ============================================================================= +# Bebop verilator workload list - PK (proxy kernel) tests +# ============================================================================= +# search_path is relative to bb-tests/output (or absolute path). +# Each entry under tests is the file STEM (without extension). + +[workloads] +search_path = "workloads/src/CTest/toy" +tests = [ + "ctest_mvin_mvout_test-linux", + "ctest_vecunit_matmul_16xn_ones-linux", + "ctest_vecunit_matmul_16xn_random1-linux", + "ctest_vecunit_matmul_16xn_random2-linux", + "ctest_vecunit_matmul_16xn_random3-linux", + "ctest_vecunit_matmul_16xn_zero_random-linux", + "ctest_vecunit_matmul_col_row_vector-linux", + "ctest_vecunit_matmul_identity_random-linux", + "ctest_vecunit_matmul_ones-linux", + "ctest_vecunit_matmul_random1-linux", + "ctest_vecunit_matmul_random2-linux", + "ctest_vecunit_matmul_zero_random-linux", + "ctest_relu_test-linux", + "ctest_transpose_test-linux", + "ctest_transpose_16xn_test-linux", + "ctest_im2col_test-linux", + "ctest_quant_test-linux", + "ctest_dequant_test-linux", + "ctest_bdb_counter_test-linux", + "ctest_bfp_test-linux", + "ctest_gemmini_os_cisc_basic_test-linux", + "ctest_gemmini_os_cisc_atranspose_test-linux", + "ctest_gemmini_os_cisc_btranspose_test-linux", + "ctest_gemmini_os_cisc_shift_test-linux", + "ctest_gemmini_os_risc_atranspose_test-linux", + "ctest_gemmini_os_risc_btranspose_test-linux", + "ctest_gemmini_os_risc_abtranspose_test-linux", + "ctest_gemmini_os_risc_shift_test-linux", + "ctest_gemmini_ws_cisc_conv_test-linux", + "ctest_gemmini_ws_risc_basic_test-linux", +] diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index 31ec62ae9..7794ef5c3 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -26,17 +26,20 @@ def main(): # "test_api_firesim_buildbitstream.py", # "test_api_firesim_infrasetup.py", # "test_api_firesim_runworkload.py", - # "test_api_bebop_bemu_build.py", - # "test_api_bebop_bemu_sim.py", + "test_api_bebop_bemu_build.py", + "test_api_bebop_bemu_sim.py", + "test_api_bebop_bemu_batch.py", # "test_api_bebop_verilator_clean.py", # "test_api_bebop_verilator_verilog.py", # "test_api_bebop_verilator_build.py", # "test_api_bebop_verilator_sim.py", # "test_api_bebop_verilator_run.py", + "test_api_bebop_verilator_batch.py", # "test_api_bebop_p2e_clean.py", # "test_api_bebop_p2e_verilog.py", # "test_api_bebop_p2e_buildbitstream.py", # "test_api_bebop_p2e_runworkload.py", + # "test_api_bebop_p2e_batch.py", ] for name in tests: path = test_dir / name diff --git a/bbdev/api/tests/test_api_bebop_bemu_batch.py b/bbdev/api/tests/test_api_bebop_bemu_batch.py new file mode 100644 index 000000000..3f3f2befb --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_bemu_batch.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-bemu --batch '--test elf-tests'") diff --git a/bbdev/api/tests/test_api_bebop_bemu_sim.py b/bbdev/api/tests/test_api_bebop_bemu_sim.py index f9230dd65..20d825c2a 100644 --- a/bbdev/api/tests/test_api_bebop_bemu_sim.py +++ b/bbdev/api/tests/test_api_bebop_bemu_sim.py @@ -1,3 +1,4 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-bemu --sim '--binary ctest_vecunit_matmul_ones_singlecore-baremetal --batch'") +run_bbdev_case("bbdev bebop-bemu --sim '--binary ctest_vecunit_matmul_ones_singlecore-baremetal'") +run_bbdev_case("bbdev bebop-bemu --sim '--binary ctest_vecunit_matmul_ones-linux --pk'") diff --git a/bbdev/api/tests/test_api_bebop_p2e_batch.py b/bbdev/api/tests/test_api_bebop_p2e_batch.py new file mode 100644 index 000000000..4781f3010 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_p2e_batch.py @@ -0,0 +1,4 @@ +from _api_test_helper import run_bbdev_case + +# You need to set the bitstream and build-dir paths. +# run_bbdev_case("bbdev bebop-p2e --batch '--bitstream /path/to/bitstream.bit --build-dir /path/to/build --test elf-tests'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_batch.py b/bbdev/api/tests/test_api_bebop_verilator_batch.py new file mode 100644 index 000000000..d2053cbf7 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_verilator_batch.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev bebop-verilator --batch '--config sims.verilator.BuckyballToyVerilatorConfig --test elf-tests'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_build.py b/bbdev/api/tests/test_api_bebop_verilator_build.py index 5ff936e8f..27380320e 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_build.py +++ b/bbdev/api/tests/test_api_bebop_verilator_build.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --build '--jobs 16'") +run_bbdev_case("bbdev bebop-verilator --build '--jobs 16 --config sims.verilator.BuckyballToyVerilatorConfig'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_clean.py b/bbdev/api/tests/test_api_bebop_verilator_clean.py index 9a46a6ddd..fc16e0c20 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_clean.py +++ b/bbdev/api/tests/test_api_bebop_verilator_clean.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --clean") +run_bbdev_case("bbdev bebop-verilator --clean '--config sims.verilator.BuckyballToyVerilatorConfig'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_sim.py b/bbdev/api/tests/test_api_bebop_verilator_sim.py index 61de4d83f..fb6a1215f 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_sim.py +++ b/bbdev/api/tests/test_api_bebop_verilator_sim.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --sim '--binary ctest_vecunit_matmul_ones_singlecore-baremetal --batch'") +run_bbdev_case("bbdev bebop-verilator --sim '--binary ctest_vecunit_matmul_ones_singlecore-baremetal --batch --config sims.verilator.BuckyballToyVerilatorConfig --itrace --mtrace --pmctrace --ctrace --banktrace''") diff --git a/bbdev/bbdev b/bbdev/bbdev index 9b85f6cd9..5ea5ded48 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -40,15 +40,9 @@ WORKFLOW_COMMANDS = { "sardine": { "run": 'Run sardine. Args: "--workload "', }, - "agent": { - "chat": 'Run agent. Args: "--message ", "--model "', - }, "workload": { "build": "Build workload.", }, - "doc": { - "deploy": True, - }, "kernel": { "build": "Build RISC-V kernel + rootfs for Pegasus (cmake kernel-build).", }, @@ -61,23 +55,10 @@ WORKFLOW_COMMANDS = { "compiler": { "build": "Build compiler.", }, - "funcsim": { - "build": "Build funcsim.", - "sim": "Sim funcsim.", - }, - "uvm": { - "builddut": "Build dut.", - "build": "Build uvm.", - }, - "pegasus": { - "verilog": 'Generate SystemVerilog for Pegasus FPGA. Args: "[--config sims.pegasus.PegasusConfig]"', - "buildbitstream": "Build Vivado bitstream for AU280.", - "flashbitstream": 'Flash bitstream onto AU280. Args: "[--bitstream ] [--serial ]"', - "runworkload": 'Run workload on AU280. Args: "[--workload ] [--board ] [--timeout ] [--uart ]"', - }, "bebop-bemu": { "build": "Build Bebop emulator.", - "sim": 'Run Bebop emulator simulation. Args: "--binary [--batch]"', + "sim": 'Run Bebop emulator simulation. Args: "--binary [--pk]"', + "batch": 'Run Bebop bemu nextest batch regression (parallel). Args: "--test "', }, "bebop-verilator": { "clean": 'Clean Bebop Verilator build directory. Args: "[--config ] [--output-dir ]"', @@ -85,12 +66,14 @@ WORKFLOW_COMMANDS = { "build": 'Build Bebop Verilator simulation executable. Args: "[--jobs ] [--config ] [--output-dir ]"', "sim": 'Run Bebop Verilator simulation. Args: "--binary [--batch] [--config ] [--output-dir ]"', "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--batch] [--jobs ] [--config ] [--output-dir ]"', + "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--config --test "', }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', "buildbitstream": "Build Bebop P2E bitstream.", "runworkload": 'Run Bebop P2E workload. Args: "--image --bitstream [--config ] [--output-dir ] [--log-dir ]"', + "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--bitstream --build-dir --test "', }, "dc": { "run": 'Run Design Compiler synthesis. Args: "[--top ] [--config ] [--output-dir ]"', From aff28fe442d4335f8b34c157970b166239b5bedd Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 19 May 2026 17:02:30 +0800 Subject: [PATCH 055/196] chore: uncomment and include additional test files for API regression testing --- bbdev/api/tests/test_all_api_serial.py | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index 7794ef5c3..435b0b220 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -20,26 +20,26 @@ def main(): "test_api_dc_run.py", "test_api_dc_verilog.py", "test_api_dc_synth.py", - # "test_api_sardine_run.py", - # "test_api_kernel_build.py", - # "test_api_firesim_enumeratefpgas.py", - # "test_api_firesim_buildbitstream.py", - # "test_api_firesim_infrasetup.py", - # "test_api_firesim_runworkload.py", + "test_api_sardine_run.py", + "test_api_kernel_build.py", + "test_api_firesim_enumeratefpgas.py", + "test_api_firesim_buildbitstream.py", + "test_api_firesim_infrasetup.py", + "test_api_firesim_runworkload.py", "test_api_bebop_bemu_build.py", "test_api_bebop_bemu_sim.py", "test_api_bebop_bemu_batch.py", - # "test_api_bebop_verilator_clean.py", - # "test_api_bebop_verilator_verilog.py", - # "test_api_bebop_verilator_build.py", - # "test_api_bebop_verilator_sim.py", - # "test_api_bebop_verilator_run.py", + "test_api_bebop_verilator_clean.py", + "test_api_bebop_verilator_verilog.py", + "test_api_bebop_verilator_build.py", + "test_api_bebop_verilator_sim.py", + "test_api_bebop_verilator_run.py", "test_api_bebop_verilator_batch.py", - # "test_api_bebop_p2e_clean.py", - # "test_api_bebop_p2e_verilog.py", - # "test_api_bebop_p2e_buildbitstream.py", - # "test_api_bebop_p2e_runworkload.py", - # "test_api_bebop_p2e_batch.py", + "test_api_bebop_p2e_clean.py", + "test_api_bebop_p2e_verilog.py", + "test_api_bebop_p2e_buildbitstream.py", + "test_api_bebop_p2e_runworkload.py", + "test_api_bebop_p2e_batch.py", ] for name in tests: path = test_dir / name From 9046562c111cb13ec053ee3dc554dfdb0b8cf10e Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 20 May 2026 04:33:37 +0800 Subject: [PATCH 056/196] feat: add workload-tohex functionality with API and event steps for converting ELF files to hex --- bbdev/api/steps/kernel/01_build_event.step.py | 6 +- bbdev/api/steps/workload/02_tohex_api.step.py | 19 ++++ .../api/steps/workload/02_tohex_event.step.py | 89 +++++++++++++++++++ bbdev/api/steps/workload/scripts/elf2hex.py | 78 ++++++++++++++++ bbdev/api/tests/test_api_workload_tohex.py | 3 + bbdev/bbdev | 1 + 6 files changed, 193 insertions(+), 3 deletions(-) create mode 100644 bbdev/api/steps/workload/02_tohex_api.step.py create mode 100644 bbdev/api/steps/workload/02_tohex_event.step.py create mode 100755 bbdev/api/steps/workload/scripts/elf2hex.py create mode 100644 bbdev/api/tests/test_api_workload_tohex.py diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index be575cb1c..8af537b47 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -20,7 +20,7 @@ config = { "name": "kernel-build", - "description": "build RISC-V kernel + rootfs for Pegasus via bb-tests/workloads/lib/kernel", + "description": "build RISC-V kernel + rootfs for image via bb-tests/workloads/lib/kernel", "flows": ["kernel"], "triggers": [queue("kernel.build")], "enqueues": [], @@ -66,8 +66,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return # Convert bin to hex for P2E memory backdoor - bin_file = os.path.join(output_dir, "pegasus-bin") - hex_file = os.path.join(output_dir, "pegasus.hex") + bin_file = os.path.join(output_dir, "image.bin") + hex_file = os.path.join(output_dir, "image.hex") if os.path.exists(bin_file): ctx.logger.info(f"Converting {bin_file} to Verilog hex format...") diff --git a/bbdev/api/steps/workload/02_tohex_api.step.py b/bbdev/api/steps/workload/02_tohex_api.step.py new file mode 100644 index 000000000..8265c1c9f --- /dev/null +++ b/bbdev/api/steps/workload/02_tohex_api.step.py @@ -0,0 +1,19 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path + +config = { + "name": "workload-tohex-api", + "description": "convert elf to hex", + "flows": ["workload"], + "triggers": [api("POST", "/workload/tohex")], + "enqueues": ["workload.tohex"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = request.body or {} + data = {} + await ctx.enqueue({"topic": "workload.tohex", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/02_tohex_event.step.py b/bbdev/api/steps/workload/02_tohex_event.step.py new file mode 100644 index 000000000..0c56f339b --- /dev/null +++ b/bbdev/api/steps/workload/02_tohex_event.step.py @@ -0,0 +1,89 @@ +import os +import subprocess +import sys +from pathlib import Path + +from motia import FlowContext, queue + +# Add the utils directory to the Python path +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "workload-tohex", + "description": "convert all -baremetal ELF files under bb-tests/output/workloads/src to hex", + "flows": ["workload"], + "triggers": [queue("workload.tohex")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + script = f"{bbdir}/bbdev/api/steps/workload/scripts/elf2hex.py" + search_root = Path(f"{bbdir}/bb-tests/output/workloads/src") + + ctx.logger.info("Search root", {"path": str(search_root)}) + + if not search_root.is_dir(): + ctx.logger.error( + "Workload output directory not found", + {"path": str(search_root)}, + ) + await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) + return + + ctx.logger.info("Search root is a directory", {"path": str(search_root)}) + + elf_files = sorted(p for p in search_root.rglob("*-baremetal") if p.is_file()) + + ctx.logger.info("ELF files found", {"count": len(elf_files)}) + + if not elf_files: + ctx.logger.warn( + "No -baremetal ELF files found", + {"search_root": str(search_root)}, + ) + await check_result(ctx, 0, continue_run=False, trace_id=origin_tid) + return + + ctx.logger.info( + "Converting ELF files to hex", + {"count": len(elf_files), "search_root": str(search_root)}, + ) + + overall_rc = 0 + for elf in elf_files: + command = f"python3 {script} {elf}" + ctx.logger.info("Executing to-hex command", {"command": command}) + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=str(elf.parent), + executable="bash", + stdout_prefix=f"to-hex {elf.name}", + stderr_prefix=f"to-hex {elf.name}", + ) + if result.returncode != 0: + overall_rc = result.returncode + ctx.logger.error( + "elf2hex failed", + {"elf": str(elf), "returncode": result.returncode}, + ) + + # ================================================================================== + # Return conversion result + # ================================================================================== + success_result, failure_result = await check_result( + ctx, overall_rc, continue_run=False, trace_id=origin_tid) + + # ================================================================================== + # finish workflow + # ================================================================================== + return diff --git a/bbdev/api/steps/workload/scripts/elf2hex.py b/bbdev/api/steps/workload/scripts/elf2hex.py new file mode 100755 index 000000000..e9441e77f --- /dev/null +++ b/bbdev/api/steps/workload/scripts/elf2hex.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""ELF -> hex converter for Buckyball P2E DDR loading. + +Output format: + @0 + XX + XX + ... +where XX is one byte per line (uppercase hex). + +The hex file represents bytes starting at DDR offset 0, which maps to +CPU address 0x80000000. All LOAD segments are concatenated by physical +address, with gaps padded by zeros. +""" + +import argparse +import subprocess +import sys +from pathlib import Path + + +def elf_to_bin(elf_path: Path, bin_path: Path, objcopy: str) -> None: + subprocess.run( + [objcopy, "-O", "binary", str(elf_path), str(bin_path)], + check=True, + ) + + +def bin_to_hex(bin_path: Path, hex_path: Path) -> int: + data = bin_path.read_bytes() + lines = ["@0"] + lines.extend(f"{b:02X}" for b in data) + hex_path.write_text("\n".join(lines) + "\n") + return len(data) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("elf", type=Path, help="Input ELF file") + parser.add_argument( + "hex", + type=Path, + nargs="?", + help="Output hex file (default: .hex)", + ) + parser.add_argument( + "--objcopy", + default="riscv64-unknown-elf-objcopy", + help="objcopy binary (default: riscv64-unknown-elf-objcopy)", + ) + args = parser.parse_args() + + if not args.elf.is_file(): + print(f"error: ELF not found: {args.elf}", file=sys.stderr) + return 1 + + hex_path = args.hex or args.elf.with_suffix(".hex") + bin_path = hex_path.with_suffix(".bin") + + try: + elf_to_bin(args.elf, bin_path, args.objcopy) + except FileNotFoundError: + print(f"error: objcopy not found: {args.objcopy}", file=sys.stderr) + print("hint: run inside `nix develop` shell", file=sys.stderr) + return 1 + except subprocess.CalledProcessError as e: + print(f"error: objcopy failed: {e}", file=sys.stderr) + return 1 + + n = bin_to_hex(bin_path, hex_path) + bin_path.unlink() + + print(f"wrote {hex_path} ({n} bytes, {n + 1} lines)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bbdev/api/tests/test_api_workload_tohex.py b/bbdev/api/tests/test_api_workload_tohex.py new file mode 100644 index 000000000..add7be934 --- /dev/null +++ b/bbdev/api/tests/test_api_workload_tohex.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev workload --tohex") diff --git a/bbdev/bbdev b/bbdev/bbdev index 5ea5ded48..352905a6d 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -42,6 +42,7 @@ WORKFLOW_COMMANDS = { }, "workload": { "build": "Build workload.", + "tohex": "Convert all -baremetal ELF files under bb-tests/output/workloads/src to hex.", }, "kernel": { "build": "Build RISC-V kernel + rootfs for Pegasus (cmake kernel-build).", From aee61f94517525e2c1e77661c0fcc892d1a86cbf Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 20 May 2026 16:03:26 +0800 Subject: [PATCH 057/196] fix: correct bin_to_hex logic in kenel generation --- .../bebop/p2e/03_runworkload_event.step.py | 8 +- bbdev/api/steps/kernel/01_build_event.step.py | 19 +++- bbdev/api/steps/kernel/scripts/bin_to_hex.py | 103 +++++------------- 3 files changed, 47 insertions(+), 83 deletions(-) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index d3d097fc2..4acc85ae7 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -67,13 +67,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return # Infer build_dir from bitstream path - # bitstream is typically: /fpgaCompDir/part_b0_f0/pnrDir/xepic_vvac_top_0_0.bit - # so build_dir is 4 levels up + # bitstream is typically: /fpgaCompDir/bitstream.bit + # so build_dir is 2 levels up build_dir = input_data.get("build_dir") if not build_dir: bitstream_abs = os.path.abspath(bitstream) - # Go up: pnrDir -> part_b0_f0 -> fpgaCompDir -> build_dir - build_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(bitstream_abs)))) + # Go up: fpgaCompDir -> build_dir + build_dir = os.path.dirname(os.path.dirname(bitstream_abs)) if not os.path.isdir(build_dir): ctx.logger.error(f"build_dir not found (inferred from bitstream): {build_dir}") diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index 8af537b47..d63042f59 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -65,15 +65,28 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result(ctx, result.returncode, continue_run=False, trace_id=origin_tid) return - # Convert bin to hex for P2E memory backdoor + # Convert fw_payload.bin to hex for P2E memory backdoor + fw_payload_bin = os.path.join(output_dir, "fw_payload.bin") + fw_payload_hex = os.path.join(output_dir, "fw_payload.hex") + + if os.path.exists(fw_payload_bin): + ctx.logger.info(f"Converting {fw_payload_bin} to Verilog hex format for p2e...") + # fw_payload loads at 0x80000000, so base_address=0x80000000 to generate relative offsets + success = bin_to_hex(fw_payload_bin, fw_payload_hex, base_address=0x80000000) + if not success: + ctx.logger.error("Failed to convert fw_payload to hex, but continuing...") + else: + ctx.logger.info("fw_payload.bin not found, skipping hex generation (baremetal kernel build?)") + + # Also convert legacy image.bin for backward compatibility bin_file = os.path.join(output_dir, "image.bin") hex_file = os.path.join(output_dir, "image.hex") if os.path.exists(bin_file): ctx.logger.info(f"Converting {bin_file} to Verilog hex format...") - success = bin_to_hex(bin_file, hex_file) + success = bin_to_hex(bin_file, hex_file, base_address=0x80000000) if not success: - ctx.logger.warning("Failed to convert bin to hex, but continuing...") + ctx.logger.error("Failed to convert bin to hex, but continuing...") await check_result(ctx, 0, continue_run=False, trace_id=origin_tid) diff --git a/bbdev/api/steps/kernel/scripts/bin_to_hex.py b/bbdev/api/steps/kernel/scripts/bin_to_hex.py index 9293cc840..97fe16841 100644 --- a/bbdev/api/steps/kernel/scripts/bin_to_hex.py +++ b/bbdev/api/steps/kernel/scripts/bin_to_hex.py @@ -1,104 +1,54 @@ #!/usr/bin/env python3 -import subprocess import sys import os -def bin_to_hex(bin_file, hex_file, objcopy_path="riscv64-unknown-elf-objcopy", base_address=0): + +def bin_to_hex(bin_file, hex_file, base_address=0): """ - 将 RISC-V bin 文件转换为 Verilog readmemh 格式 - 地址格式：@0xNNNNNNNN - base_address: 基地址偏移，生成的地址会减去这个值（例如 0x80000000） + Convert a raw binary file to Verilog readmemh format. + + The output uses one byte per line with `@0xADDR` markers for the starting + offset (relative to base_address). The input is treated as a contiguous + byte stream starting at base_address; the resulting `@` offset is always 0 + because raw binaries have no internal address information. + + base_address is kept in the signature for API compatibility and is only + used to print the absolute address range. """ print(f"Converting {bin_file} -> {hex_file}") if base_address != 0: print(f"Base address offset: 0x{base_address:08X}") - # Step 1: bin -> Intel HEX - temp_hex = f"{bin_file}.tmp.hex" try: - cmd = [objcopy_path, "-O", "ihex", bin_file, temp_hex] - result = subprocess.run(cmd, capture_output=True, text=True) + with open(bin_file, "rb") as f: + data = f.read() - if result.returncode != 0: - print(f"Error: objcopy failed: {result.stderr}") + if not data: + print("Error: Input file is empty") return False - # Step 2: Parse Intel HEX and convert to readmemh format - memory_data = {} - current_base_address = 0 - - with open(temp_hex, 'r') as f: - for line in f: - line = line.strip() - if not line or not line.startswith(':'): - continue - - record = line[1:] - if len(record) < 8: - continue - - byte_count = int(record[0:2], 16) - address = int(record[2:6], 16) - record_type = int(record[6:8], 16) - - if record_type == 0x00: # Data record - data_start = 8 - for i in range(byte_count): - if data_start + 2 <= len(record) - 2: - byte_data = record[data_start:data_start+2] - memory_address = current_base_address + address + i - memory_data[memory_address] = int(byte_data, 16) - data_start += 2 - - elif record_type == 0x04: # Extended linear address - if byte_count == 2 and len(record) >= 12: - high_address = int(record[8:12], 16) - current_base_address = high_address << 16 - - if not memory_data: - print("Error: No valid data found") - return False - - # Step 3: Write readmemh format - with open(hex_file, 'w') as f: - sorted_addresses = sorted(memory_data.keys()) - current_address = None - - for addr in sorted_addresses: - # New address section if not continuous - if current_address is None or addr != current_address + 1: - relative_addr = addr - base_address - if relative_addr < 0: - print(f"Warning: address 0x{addr:08X} is below base 0x{base_address:08X}, skipping") - continue - if relative_addr == 0: - f.write(f"@0\n") - else: - f.write(f"@0x{relative_addr:X}\n") - current_address = addr - else: - current_address = addr - - byte_value = memory_data[addr] - f.write(f"{byte_value:02X}\n") + with open(hex_file, "w") as f: + f.write("@0\n") + for byte in data: + f.write(f"{byte:02X}\n") print(f"Successfully generated {hex_file}") - print(f"Address range: 0x{sorted_addresses[0]:08X} - 0x{sorted_addresses[-1]:08X}") - print(f"Total bytes: {len(memory_data)}") + print( + f"Address range: 0x{base_address:08X} - 0x{base_address + len(data) - 1:08X}" + ) + print(f"Total bytes: {len(data)}") return True except Exception as e: print(f"Error: Conversion failed - {e}") return False - finally: - if os.path.exists(temp_hex): - os.remove(temp_hex) + def main(): if len(sys.argv) < 3 or len(sys.argv) > 4: print("Usage: bin_to_hex.py [base_address]") - print(" base_address: optional hex address to subtract (e.g., 0x80000000)") + print(" base_address: optional hex address for reporting (e.g., 0x80000000)") sys.exit(1) bin_file = sys.argv[1] @@ -107,7 +57,7 @@ def main(): if len(sys.argv) == 4: base_str = sys.argv[3] - if base_str.startswith('0x') or base_str.startswith('0X'): + if base_str.startswith("0x") or base_str.startswith("0X"): base_address = int(base_str, 16) else: base_address = int(base_str, 10) @@ -119,5 +69,6 @@ def main(): success = bin_to_hex(bin_file, hex_file, base_address=base_address) sys.exit(0 if success else 1) + if __name__ == "__main__": main() From 57fdf0cbdf61d7d57d2d03e4e9ec419ecb70f98e Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 20 May 2026 21:46:15 +0800 Subject: [PATCH 058/196] fix: enforce required config input in Verilator event steps --- bbdev/api/steps/verilator/03_build_event.step.py | 5 ++++- bbdev/api/steps/verilator/04_sim_event.step.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bbdev/api/steps/verilator/03_build_event.step.py b/bbdev/api/steps/verilator/03_build_event.step.py index 9577af237..6b6acf9e3 100644 --- a/bbdev/api/steps/verilator/03_build_event.step.py +++ b/bbdev/api/steps/verilator/03_build_event.step.py @@ -25,11 +25,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) + config_name = input_data.get("config") + if not config_name or config_name == "None": + raise ValueError("Missing required input: config") bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" build_dir = get_verilator_build_dir( bbdir, - input_data.get("config"), + config_name, input_data.get("output_dir"), ) coverage = input_data.get("coverage", False) diff --git a/bbdev/api/steps/verilator/04_sim_event.step.py b/bbdev/api/steps/verilator/04_sim_event.step.py index 997381537..7f9e86349 100644 --- a/bbdev/api/steps/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -29,11 +29,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # Get simulation parameters # ================================================================================== origin_tid = get_origin_trace_id(input_data, ctx) + config_name = input_data.get("config") + if not config_name or config_name == "None": + raise ValueError("Missing required input: config") bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" build_dir = get_verilator_build_dir( bbdir, - input_data.get("config"), + config_name, input_data.get("output_dir"), ) ctx.logger.info(f"Using build directory: {build_dir}") From d5e7e5d9663d491dd8503f3deaf3f7e1434b5097 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 22 May 2026 16:28:11 +0800 Subject: [PATCH 059/196] del: remove unused image.bin --- bbdev/api/steps/kernel/01_build_event.step.py | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index d63042f59..e2f23d7c5 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -69,24 +69,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: fw_payload_bin = os.path.join(output_dir, "fw_payload.bin") fw_payload_hex = os.path.join(output_dir, "fw_payload.hex") - if os.path.exists(fw_payload_bin): - ctx.logger.info(f"Converting {fw_payload_bin} to Verilog hex format for p2e...") - # fw_payload loads at 0x80000000, so base_address=0x80000000 to generate relative offsets - success = bin_to_hex(fw_payload_bin, fw_payload_hex, base_address=0x80000000) - if not success: - ctx.logger.error("Failed to convert fw_payload to hex, but continuing...") - else: - ctx.logger.info("fw_payload.bin not found, skipping hex generation (baremetal kernel build?)") - - # Also convert legacy image.bin for backward compatibility - bin_file = os.path.join(output_dir, "image.bin") - hex_file = os.path.join(output_dir, "image.hex") - - if os.path.exists(bin_file): - ctx.logger.info(f"Converting {bin_file} to Verilog hex format...") - success = bin_to_hex(bin_file, hex_file, base_address=0x80000000) - if not success: - ctx.logger.error("Failed to convert bin to hex, but continuing...") + if not os.path.exists(fw_payload_bin): + ctx.logger.error("fw_payload.bin not found") + await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) + return + + ctx.logger.info(f"Converting {fw_payload_bin} to Verilog hex format for P2E...") + success = bin_to_hex(fw_payload_bin, fw_payload_hex, base_address=0x80000000) + if not success: + ctx.logger.error("Failed to convert fw_payload to hex") + await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) + return await check_result(ctx, 0, continue_run=False, trace_id=origin_tid) From 62ce45313e618d3e8041834a86f38cb1b16cb41e Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 22 May 2026 17:42:37 +0800 Subject: [PATCH 060/196] fix: update worker port range for server and script modes --- bbdev/bbdev | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/bbdev/bbdev b/bbdev/bbdev index 352905a6d..73f992aa9 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -18,6 +18,8 @@ from utils.port import reserve_port workflow_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "api") NO_PROXY = {"http": None, "https": None, "all": None} +WORKER_PORT_START = 49200 +WORKER_PORT_END = 49500 # Subcommand definitions: {name: {operation: help_text, ...}} # Operations with string values accept sub-arguments; True means boolean-only (no sub-args). @@ -325,7 +327,13 @@ def run_server_mode(args, cmd_info): if command == "start": print(f"Starting dev server on port {args.port}") - config_path = make_iii_config(args.port, 49134) if args.port else os.path.join(workflow_dir, "config.yaml") + if args.port: + worker_port, worker_sock = reserve_port(start_port=WORKER_PORT_START, end_port=WORKER_PORT_END) + worker_sock.close() + config_path = make_iii_config(args.port, worker_port) + print(f"Using worker port {worker_port}") + else: + config_path = os.path.join(workflow_dir, "config.yaml") try: subprocess.run(["iii", "-c", config_path], cwd=workflow_dir, check=True) finally: @@ -434,7 +442,7 @@ def run_script_mode(args, cmd_info): http_sock = None else: available_port, http_sock = reserve_port(start_port=5100, end_port=5500) - worker_port, worker_sock = reserve_port(start_port=49134, end_port=49500) + worker_port, worker_sock = reserve_port(start_port=WORKER_PORT_START, end_port=WORKER_PORT_END) print(f"Starting server on port {available_port}...") # Clean stale state files from previous runs From 4abcca1179d70b519126904db07fc6052eaa1ed2 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 25 May 2026 00:25:03 +0800 Subject: [PATCH 061/196] del: remove sardine test framework and related API steps --- bbdev/api/services/workflow_tools.py | 3 +- bbdev/api/steps/sardine/01_run_api.step.py | 22 --- bbdev/api/steps/sardine/01_run_event.step.py | 62 ------- .../sardine/02_coverage_report_event.step.py | 160 ------------------ bbdev/api/steps/sardine/README.md | 34 ---- bbdev/api/tests/test_all_api_serial.py | 1 - bbdev/api/tests/test_api_sardine_run.py | 3 - bbdev/bbdev | 3 - 8 files changed, 1 insertion(+), 287 deletions(-) delete mode 100644 bbdev/api/steps/sardine/01_run_api.step.py delete mode 100644 bbdev/api/steps/sardine/01_run_event.step.py delete mode 100644 bbdev/api/steps/sardine/02_coverage_report_event.step.py delete mode 100644 bbdev/api/steps/sardine/README.md delete mode 100644 bbdev/api/tests/test_api_sardine_run.py diff --git a/bbdev/api/services/workflow_tools.py b/bbdev/api/services/workflow_tools.py index ab17e8916..6409e37b3 100644 --- a/bbdev/api/services/workflow_tools.py +++ b/bbdev/api/services/workflow_tools.py @@ -18,8 +18,7 @@ def get_description(self) -> str: - /verilator/verilog: Generate Verilog - /verilator/build: Build verilator (params: jobs) - /verilator/sim: Run simulation (params: binary, batch) - - /workload/build: Build workload (params: args) - - /sardine/run: Run sardine tests (params: workload)""" + - /workload/build: Build workload (params: args)""" def get_parameters(self) -> Dict[str, Any]: return { diff --git a/bbdev/api/steps/sardine/01_run_api.step.py b/bbdev/api/steps/sardine/01_run_api.step.py deleted file mode 100644 index fb24f909c..000000000 --- a/bbdev/api/steps/sardine/01_run_api.step.py +++ /dev/null @@ -1,22 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - -from utils.path import get_buckyball_path - -config = { - "name": "sardine-run-api", - "description": "running sardine", - "flows": ["sardine"], - "triggers": [api("POST", "/sardine/run")], - "enqueues": ["sardine.run"], -} - - -async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() - - body = request.body or {} - - data = {"workload": body.get("workload", "")} - - await ctx.enqueue({"topic": "sardine.run", "data": {**data, "_trace_id": ctx.trace_id}}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/sardine/01_run_event.step.py b/bbdev/api/steps/sardine/01_run_event.step.py deleted file mode 100644 index 1b4e1b3ab..000000000 --- a/bbdev/api/steps/sardine/01_run_event.step.py +++ /dev/null @@ -1,62 +0,0 @@ -import os -import sys -import time - -from motia import FlowContext, queue - -# Add the utils directory to the Python path -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from utils.event_common import check_result, get_origin_trace_id - -config = { - "name": "sardine-run", - "description": "running sardine", - "flows": ["sardine"], - "triggers": [queue("sardine.run")], - "enqueues": ["sardine.coverage_report"], -} - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - bbdir = get_buckyball_path() - - sardine_dir = f"{bbdir}/bb-tests/sardine" - - # Record start time so coverage report can filter to only this run's .dat files - run_start_time = time.time() - - command = f"python3 run_tests.py -m \"({input_data.get('workload', '')})\"" - if input_data.get("coverage", False): - command += " --coverage" - ctx.logger.info( - "Executing sardine command", {"command": command, "cwd": sardine_dir} - ) - result = stream_run_logger( - cmd=command, - logger=ctx.logger, - cwd=sardine_dir, - executable="bash", - stdout_prefix="sardine run", - stderr_prefix="sardine run", - ) - - # ================================================================================== - # Return execution result - # ================================================================================== - if input_data.get("coverage", False): - # When coverage is enabled, always continue to coverage report step - # (even if some tests failed, coverage data is still valid) - await check_result(ctx, result.returncode, continue_run=True, trace_id=origin_tid) - await ctx.enqueue( - {"topic": "sardine.coverage_report", "data": {**input_data, "run_start_time": run_start_time}} - ) - else: - await check_result(ctx, result.returncode, continue_run=False, trace_id=origin_tid) - - return diff --git a/bbdev/api/steps/sardine/02_coverage_report_event.step.py b/bbdev/api/steps/sardine/02_coverage_report_event.step.py deleted file mode 100644 index 93f3be298..000000000 --- a/bbdev/api/steps/sardine/02_coverage_report_event.step.py +++ /dev/null @@ -1,160 +0,0 @@ -import os -import glob -import sys - -from motia import FlowContext, queue - -# Add the utils directory to the Python path -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from utils.event_common import check_result, get_origin_trace_id - -config = { - "name": "coverage report", - "description": "merge coverage data and generate annotated source + lcov HTML report", - "flows": ["sardine"], - "triggers": [queue("sardine.coverage_report")], - "enqueues": [], -} - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - bbdir = get_buckyball_path() - log_dir = f"{bbdir}/arch/log" - coverage_dir = f"{bbdir}/bb-tests/sardine/reports/coverage" - os.makedirs(coverage_dir, exist_ok=True) - - # ================================================================================== - # Find coverage.dat files from this run only (created after run_start_time) - # ================================================================================== - all_dat_files = glob.glob(f"{log_dir}/*/coverage.dat") - run_start_time = input_data.get("run_start_time", 0) - dat_files = [f for f in all_dat_files if os.path.getmtime(f) >= run_start_time] - if not dat_files: - ctx.logger.error("No coverage .dat files found", {"dir": coverage_dir}) - success_result, failure_result = await check_result( - ctx, - returncode=1, - continue_run=False, - extra_fields={"task": "coverage_report", "error": "No .dat files found"}, - trace_id=origin_tid, - ) - return - - ctx.logger.info(f"Found {len(dat_files)} coverage data files") - - # ================================================================================== - # Merge all .dat files - # ================================================================================== - merged_dat = f"{coverage_dir}/merged.dat" - if len(dat_files) == 1: - # Single file, just copy - import shutil - shutil.copy2(dat_files[0], merged_dat) - else: - merge_cmd = f"verilator_coverage -write {merged_dat} {' '.join(dat_files)}" - result = stream_run_logger( - cmd=merge_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="coverage merge", - stderr_prefix="coverage merge", - ) - if result.returncode != 0: - await check_result( - ctx, - returncode=result.returncode, - continue_run=False, - extra_fields={"task": "coverage_report", "error": "merge failed"}, - trace_id=origin_tid, - ) - return - - # ================================================================================== - # Generate annotated source report - # ================================================================================== - annotate_dir = f"{coverage_dir}/annotated" - annotate_cmd = f"verilator_coverage --annotate {annotate_dir} {merged_dat}" - result = stream_run_logger( - cmd=annotate_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="coverage annotate", - stderr_prefix="coverage annotate", - ) - if result.returncode != 0: - await check_result( - ctx, - returncode=result.returncode, - continue_run=False, - extra_fields={"task": "coverage_report", "error": "annotate failed"}, - trace_id=origin_tid, - ) - return - - # ================================================================================== - # Generate lcov info + HTML report - # ================================================================================== - lcov_info = f"{coverage_dir}/merged.info" - html_dir = f"{coverage_dir}/html" - - lcov_cmd = f"verilator_coverage -write-info {lcov_info} {merged_dat}" - result = stream_run_logger( - cmd=lcov_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="coverage lcov", - stderr_prefix="coverage lcov", - ) - if result.returncode != 0: - await check_result( - ctx, - returncode=result.returncode, - continue_run=False, - extra_fields={"task": "coverage_report", "error": "lcov export failed"}, - trace_id=origin_tid, - ) - return - - genhtml_cmd = f"genhtml {lcov_info} -o {html_dir}" - result = stream_run_logger( - cmd=genhtml_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="coverage html", - stderr_prefix="coverage html", - ) - if result.returncode != 0: - await check_result( - ctx, - returncode=result.returncode, - continue_run=False, - extra_fields={"task": "coverage_report", "error": "genhtml failed"}, - trace_id=origin_tid, - ) - return - - # ================================================================================== - # Return result - # ================================================================================== - ctx.logger.info(f"Coverage report generated: {html_dir}/index.html") - success_result, failure_result = await check_result( - ctx, - returncode=0, - continue_run=False, - extra_fields={ - "task": "coverage_report", - "coverage_dir": coverage_dir, - "annotate_dir": annotate_dir, - "html_dir": html_dir, - "dat_count": len(dat_files), - }, - trace_id=origin_tid, - ) - - return diff --git a/bbdev/api/steps/sardine/README.md b/bbdev/api/steps/sardine/README.md deleted file mode 100644 index 90a82f2a8..000000000 --- a/bbdev/api/steps/sardine/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Sardine Workflow - -Sardine workflow in the Buckyball framework for running Sardine-related tasks. - -## API Usage - -### `run` -**Endpoint**: `POST /sardine/run` - -**Function**: Run Sardine tasks - -**Parameters**: -- **`workload`** - Specify the workload to run - -**Example**: -```bash -# Run specified workload -bbdev sardine --run "--workload /path/to/workload" - -# Run default workload -bbdev sardine --run -``` - -**Response**: -```json -{ - "status": 200, - "body": { - "success": true, - "processing": false, - "return_code": 0 - } -} -``` diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index 435b0b220..79dff79f6 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -20,7 +20,6 @@ def main(): "test_api_dc_run.py", "test_api_dc_verilog.py", "test_api_dc_synth.py", - "test_api_sardine_run.py", "test_api_kernel_build.py", "test_api_firesim_enumeratefpgas.py", "test_api_firesim_buildbitstream.py", diff --git a/bbdev/api/tests/test_api_sardine_run.py b/bbdev/api/tests/test_api_sardine_run.py deleted file mode 100644 index 070c636fa..000000000 --- a/bbdev/api/tests/test_api_sardine_run.py +++ /dev/null @@ -1,3 +0,0 @@ -from _api_test_helper import run_bbdev_case - -run_bbdev_case("bbdev sardine --run '--workload ctest'") diff --git a/bbdev/bbdev b/bbdev/bbdev index 73f992aa9..7b4c6cb33 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -39,9 +39,6 @@ WORKFLOW_COMMANDS = { "sim": 'Run simulation. Args: "--binary [--batch]"', "run": 'Integrated build+sim+run. Args: "--binary [--batch] [--job ]"', }, - "sardine": { - "run": 'Run sardine. Args: "--workload "', - }, "workload": { "build": "Build workload.", "tohex": "Convert all -baremetal ELF files under bb-tests/output/workloads/src to hex.", From 98aa0e647de70489b9ab81a570c18cf145f89e9b Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 25 May 2026 18:32:52 +0800 Subject: [PATCH 062/196] refactor: replace quantization tests with fp2int and int2fp tests in workload configurations for bemu and verilator --- bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml | 4 ++-- bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml | 4 ++-- bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml | 4 ++-- bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml index 1fec486f0..6410abf15 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml @@ -24,8 +24,8 @@ tests = [ "ctest_transpose_test_singlecore-baremetal", "ctest_transpose_16xn_test_singlecore-baremetal", "ctest_im2col_test_singlecore-baremetal", - "ctest_quant_test_singlecore-baremetal", - "ctest_dequant_test_singlecore-baremetal", + "ctest_fp2int_test_singlecore-baremetal", + "ctest_int2fp_test_singlecore-baremetal", "ctest_bdb_counter_test_singlecore-baremetal", "ctest_bfp_test_singlecore-baremetal", "ctest_gemmini_os_cisc_basic_test_singlecore-baremetal", diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml index 7d9957051..53832ec7e 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml @@ -24,8 +24,8 @@ tests = [ "ctest_transpose_test-linux", "ctest_transpose_16xn_test-linux", "ctest_im2col_test-linux", - "ctest_quant_test-linux", - "ctest_dequant_test-linux", + "ctest_fp2int_test-linux", + "ctest_int2fp_test-linux", "ctest_bdb_counter_test-linux", "ctest_bfp_test-linux", "ctest_gemmini_os_cisc_basic_test-linux", diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml index 0312c92ee..1d23de637 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml @@ -23,8 +23,8 @@ tests = [ "ctest_transpose_test_singlecore-baremetal", "ctest_transpose_16xn_test_singlecore-baremetal", "ctest_im2col_test_singlecore-baremetal", - "ctest_quant_test_singlecore-baremetal", - "ctest_dequant_test_singlecore-baremetal", + "ctest_fp2int_test_singlecore-baremetal", + "ctest_int2fp_test_singlecore-baremetal", "ctest_bdb_counter_test_singlecore-baremetal", "ctest_bfp_test_singlecore-baremetal", "ctest_gemmini_os_cisc_basic_test_singlecore-baremetal", diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml index 91a4b3c11..10efb3faf 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml @@ -23,8 +23,8 @@ tests = [ "ctest_transpose_test-linux", "ctest_transpose_16xn_test-linux", "ctest_im2col_test-linux", - "ctest_quant_test-linux", - "ctest_dequant_test-linux", + "ctest_fp2int_test-linux", + "ctest_int2fp_test-linux", "ctest_bdb_counter_test-linux", "ctest_bfp_test-linux", "ctest_gemmini_os_cisc_basic_test-linux", From 6989a7be2620638a8a6ca38503c2446eb85ba62d Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 25 May 2026 20:09:02 +0800 Subject: [PATCH 063/196] feat: add new minimal and fp2int tests to workload configurations for bemu and verilator --- bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml | 2 ++ bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml | 2 ++ bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml | 2 ++ bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml | 2 ++ 4 files changed, 8 insertions(+) diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml index 6410abf15..2727b923e 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml @@ -38,4 +38,6 @@ tests = [ "ctest_gemmini_os_risc_shift_test_singlecore-baremetal", "ctest_gemmini_ws_cisc_conv_test_singlecore-baremetal", "ctest_gemmini_ws_risc_basic_test_singlecore-baremetal", + "ctest_mmio_minimal_test_singlecore-baremetal", + "ctest_mxfp2int_test_singlecore-baremetal", ] \ No newline at end of file diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml index 53832ec7e..a4ece5187 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml @@ -38,4 +38,6 @@ tests = [ "ctest_gemmini_os_risc_shift_test-linux", "ctest_gemmini_ws_cisc_conv_test-linux", "ctest_gemmini_ws_risc_basic_test-linux", + "ctest_mmio_minimal_test-linux", + "ctest_mxfp2int_test-linux", ] \ No newline at end of file diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml index 1d23de637..e594565d8 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml @@ -37,6 +37,8 @@ tests = [ "ctest_gemmini_os_risc_shift_test_singlecore-baremetal", "ctest_gemmini_ws_cisc_conv_test_singlecore-baremetal", "ctest_gemmini_ws_risc_basic_test_singlecore-baremetal", + "ctest_mmio_minimal_test_singlecore-baremetal", + "ctest_mxfp2int_test_singlecore-baremetal", "ctest_tlb_test_singlecore-baremetal", ] diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml index 10efb3faf..2458ea857 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml @@ -37,4 +37,6 @@ tests = [ "ctest_gemmini_os_risc_shift_test-linux", "ctest_gemmini_ws_cisc_conv_test-linux", "ctest_gemmini_ws_risc_basic_test-linux", + "ctest_mmio_minimal_test-linux", + "ctest_mxfp2int_test-linux", ] From 0ce488b18c2cd7cb8141a5888d932cb1cf09d5c1 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 26 May 2026 22:40:14 +0800 Subject: [PATCH 064/196] refactor: update search path and add new workload tests for bemu and verilator --- bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml | 6 +++++- bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml | 8 ++++++-- .../api/steps/bebop/verilator/scripts/workloads-elf.toml | 2 +- bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml index 2727b923e..6a5533929 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml @@ -6,7 +6,7 @@ # Each entry under tests is the file STEM (without extension). [workloads] -search_path = "workloads/src/CTest/toy" +search_path = "workloads/src" tests = [ "ctest_mvin_mvout_test_singlecore-baremetal", "ctest_vecunit_matmul_16xn_ones_singlecore-baremetal", @@ -40,4 +40,8 @@ tests = [ "ctest_gemmini_ws_risc_basic_test_singlecore-baremetal", "ctest_mmio_minimal_test_singlecore-baremetal", "ctest_mxfp2int_test_singlecore-baremetal", + + "optest_buckyball_matmul_64x16_16x64_singlecore-baremetal", + "optest_buckyball_matmul_1024x16_16x1024_singlecore-baremetal", + "optest_tile_matmul_127x17_17x127_singlecore-baremetal", ] \ No newline at end of file diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml index a4ece5187..344c00441 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml @@ -6,7 +6,7 @@ # Each entry under tests is the file STEM (without extension). [workloads] -search_path = "workloads/src/CTest/toy" +search_path = "workloads/src" tests = [ "ctest_mvin_mvout_test-linux", "ctest_vecunit_matmul_16xn_ones-linux", @@ -40,4 +40,8 @@ tests = [ "ctest_gemmini_ws_risc_basic_test-linux", "ctest_mmio_minimal_test-linux", "ctest_mxfp2int_test-linux", -] \ No newline at end of file + + "optest_buckyball_matmul_64x16_16x64-linux", + "optest_buckyball_matmul_1024x16_16x1024-linux", + "optest_tile_matmul_127x17_17x127-linux", +] diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml index e594565d8..d571318ad 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml @@ -5,7 +5,7 @@ # Each entry under tests is the file STEM (without extension). [workloads] -search_path = "workloads/src/CTest/toy" +search_path = "workloads/src" tests = [ "ctest_mvin_mvout_test_singlecore-baremetal", "ctest_vecunit_matmul_16xn_ones_singlecore-baremetal", diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml index 2458ea857..adf8b5ad9 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml @@ -5,7 +5,7 @@ # Each entry under tests is the file STEM (without extension). [workloads] -search_path = "workloads/src/CTest/toy" +search_path = "workloads/src" tests = [ "ctest_mvin_mvout_test-linux", "ctest_vecunit_matmul_16xn_ones-linux", From 95cb22a676b6d4577d29a0eaebf7ae2639f61d30 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 27 May 2026 15:16:49 +0800 Subject: [PATCH 065/196] feat: add new stride tests and update configurations for bemu and verilator workloads --- bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml | 3 ++- bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml | 1 + bbdev/api/steps/bebop/verilator/03_build_event.step.py | 2 ++ bbdev/api/steps/bebop/verilator/04_sim_event.step.py | 1 + .../api/steps/bebop/verilator/scripts/workloads-elf.toml | 8 +++++--- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml index 6a5533929..8a029dc4a 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml @@ -9,6 +9,7 @@ search_path = "workloads/src" tests = [ "ctest_mvin_mvout_test_singlecore-baremetal", + "ctest_mvin_mvout_stride_test_singlecore-baremetal", "ctest_vecunit_matmul_16xn_ones_singlecore-baremetal", "ctest_vecunit_matmul_16xn_random1_singlecore-baremetal", "ctest_vecunit_matmul_16xn_random2_singlecore-baremetal", @@ -44,4 +45,4 @@ tests = [ "optest_buckyball_matmul_64x16_16x64_singlecore-baremetal", "optest_buckyball_matmul_1024x16_16x1024_singlecore-baremetal", "optest_tile_matmul_127x17_17x127_singlecore-baremetal", -] \ No newline at end of file +] diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml index 344c00441..11a7e42e2 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml @@ -9,6 +9,7 @@ search_path = "workloads/src" tests = [ "ctest_mvin_mvout_test-linux", + "ctest_mvin_mvout_stride_test-linux", "ctest_vecunit_matmul_16xn_ones-linux", "ctest_vecunit_matmul_16xn_random1-linux", "ctest_vecunit_matmul_16xn_random2-linux", diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index 8dcb550f4..153f53f84 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -78,6 +78,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: }, trace_id=origin_tid, ) + if build_result.returncode != 0: + return # Continue routing to sim if from run workflow if input_data.get("from_run_workflow"): diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index 0f896265e..e48f09e44 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -69,6 +69,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: run_cmd = ( f"cargo run --features verilator " f"--config=\"env.ARCH_CONFIG='{arch_config}'\" " + f"--config=\"env.VSRC_PATH='{vsrc_dir}'\" " f"-- verilator " f"--elf=\"{binary_path}\" " f"--log-dir=\"{log_dir}\" " diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml index d571318ad..f8a110bb4 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml @@ -26,7 +26,6 @@ tests = [ "ctest_fp2int_test_singlecore-baremetal", "ctest_int2fp_test_singlecore-baremetal", "ctest_bdb_counter_test_singlecore-baremetal", - "ctest_bfp_test_singlecore-baremetal", "ctest_gemmini_os_cisc_basic_test_singlecore-baremetal", "ctest_gemmini_os_cisc_atranspose_test_singlecore-baremetal", "ctest_gemmini_os_cisc_btranspose_test_singlecore-baremetal", @@ -41,6 +40,9 @@ tests = [ "ctest_mxfp2int_test_singlecore-baremetal", "ctest_tlb_test_singlecore-baremetal", -] - + "ctest_mvin_mvout_stride_test_singlecore-baremetal", + "ctest_transpose_matmul_singlecore-baremetal", + "optest_buckyball_matmul_16x64_64x16_singlecore-baremetal", + "optest_buckyball_matmul_64x16_16x64_singlecore-baremetal", +] From 60fdee8f3e4d71f629f424bf2ccda1ddb25c4503 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 27 May 2026 21:03:27 +0800 Subject: [PATCH 066/196] feat: add model test into workload and bemu --- .../api/steps/bebop/bemu/02_sim_event.step.py | 5 ++-- bbdev/api/steps/workload/01_buidl_api.step.py | 5 +++- .../api/steps/workload/01_build_event.step.py | 23 +++++++++++++++---- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 9324b2c53..68c4ee08d 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -47,6 +47,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return ctx.logger.info(f"binary_path: {binary_path}") + binary_dir = os.path.dirname(binary_path) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") log_dir = input_data.get("log_dir") or f"{arch_dir}/log/{timestamp}-{binary_name}-bemu" @@ -55,7 +56,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── Run bebop bemu ──────────────────────────────────────────────────── pk_flag = " --pk" if input_data.get("pk") else "" run_cmd = ( - f"cargo run --features bemu -- bemu " + f"cargo run --manifest-path \"{bebop_dir}/Cargo.toml\" --features bemu -- bemu " f"--elf=\"{binary_path}\" " f"--log-dir=\"{log_dir}\"" f"{pk_flag}" @@ -64,7 +65,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: run_result = stream_run_logger( cmd=run_cmd, logger=ctx.logger, - cwd=bebop_dir, + cwd=binary_dir, stdout_prefix="bebop bemu", stderr_prefix="bebop bemu", ) diff --git a/bbdev/api/steps/workload/01_buidl_api.step.py b/bbdev/api/steps/workload/01_buidl_api.step.py index 752523d26..aa491ce84 100644 --- a/bbdev/api/steps/workload/01_buidl_api.step.py +++ b/bbdev/api/steps/workload/01_buidl_api.step.py @@ -14,6 +14,9 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = request.body or {} - data = {"workload": body.get("workload", "")} + data = { + "workload": body.get("workload", ""), + "model": body.get("model", ""), + } await ctx.enqueue({"topic": "workload.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index 2fee01e0c..8c355396b 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -1,5 +1,4 @@ import os -import subprocess import sys from motia import FlowContext, queue @@ -27,11 +26,27 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() workload_dir = f"{bbdir}/bb-tests" build_dir = f"{workload_dir}/build" + model = input_data.get("model", "") - # os.mkdir(f"{workload_dir}/build", exist_ok=True) - subprocess.run(f"rm -rf {build_dir} && mkdir -p {build_dir}", shell=True) + model_targets = { + "lenet": "buddy-buckyball-lenet-run", + } + target = "" + if model: + target = model_targets.get(model.lower()) + if target is None: + ctx.logger.error(f"Unknown model: {model}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "unknown_model", "model": model}, + trace_id=origin_tid, + ) + return - command = f"cd {build_dir} && cmake -G Ninja .. && ninja -j{os.cpu_count()}" + os.makedirs(build_dir, exist_ok=True) + + ninja_target = f" {target}" if target else "" + command = f"cd {build_dir} && cmake -G Ninja .. && ninja -j{os.cpu_count()}{ninja_target}" ctx.logger.info( "Executing workload command", {"command": command, "cwd": build_dir} ) From 367bea49ecdcc79c862a143c209fe6e8ed3756fe Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 27 May 2026 21:31:48 +0800 Subject: [PATCH 067/196] fix: correct workload build --- bbdev/api/steps/workload/01_build_event.step.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index 8c355396b..2ee8c55ac 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -30,6 +30,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: model_targets = { "lenet": "buddy-buckyball-lenet-run", + "mobilenet": "buddy-buckyball-mobilenetv3-run", + "resnet": "buddy-buckyball-resnet-run", + "yolo": "buddy-buckyball-yolo26-run", } target = "" if model: @@ -46,7 +49,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: os.makedirs(build_dir, exist_ok=True) ninja_target = f" {target}" if target else "" - command = f"cd {build_dir} && cmake -G Ninja .. && ninja -j{os.cpu_count()}{ninja_target}" + command = f"cd {bbdir} && nix develop -c bash -c 'cd {build_dir} && cmake -G Ninja .. && ninja -j{os.cpu_count()}{ninja_target}'" ctx.logger.info( "Executing workload command", {"command": command, "cwd": build_dir} ) From acd99abc36d253d68f3df0cb2807a1ceb04ece67 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 28 May 2026 04:01:02 +0800 Subject: [PATCH 068/196] feat: add new convolutional tests to workloads for bemu and pk --- bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml | 3 +++ bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml index 8a029dc4a..00301a15b 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml @@ -44,5 +44,8 @@ tests = [ "optest_buckyball_matmul_64x16_16x64_singlecore-baremetal", "optest_buckyball_matmul_1024x16_16x1024_singlecore-baremetal", + "optest_linalg_conv2d_nhwc_hwcf_singlecore-baremetal", + "optest_linalg_conv2d_nhwc_fhwc_singlecore-baremetal", + "optest_linalg_conv2d_nchw_fchw_singlecore-baremetal", "optest_tile_matmul_127x17_17x127_singlecore-baremetal", ] diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml index 11a7e42e2..c68ade5f9 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml @@ -44,5 +44,8 @@ tests = [ "optest_buckyball_matmul_64x16_16x64-linux", "optest_buckyball_matmul_1024x16_16x1024-linux", + "optest_linalg_conv2d_nhwc_hwcf-linux", + "optest_linalg_conv2d_nhwc_fhwc-linux", + "optest_linalg_conv2d_nchw_fchw-linux", "optest_tile_matmul_127x17_17x127-linux", ] From 05d2b2cd170be452d40c9a590cd01cfbca63690a Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 29 May 2026 15:09:06 +0800 Subject: [PATCH 069/196] feat: enhance runworkload command with wave options and update workload tests --- .../bebop/p2e/03_runworkload_event.step.py | 45 ++++++++++++++++++- .../verilator/scripts/workloads-elf.toml | 3 ++ bbdev/bbdev | 2 +- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index 4acc85ae7..4aac4c1ee 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -4,7 +4,7 @@ Loads a kernel image into FPGA and runs the workload via bebop CLI: 1. Resolve image name to .hex file under bb-tests/output/ 2. Validate bitstream .bit file path - 3. Run bebop p2e --runworkload --image --bitstream --log-dir + 3. Run bebop p2e --runworkload --image --bitstream [--wave] [--wave-start ] --log-dir """ import glob import os @@ -46,6 +46,37 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: image_name = input_data.get("image", "") bitstream = input_data.get("bitstream", "") + wave = bool(input_data.get("wave", False)) + if "wave_start" in input_data: + ctx.logger.error("invalid parameter: --wave_start (use --wave-start)") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_parameter", "parameter": "wave_start"}, + trace_id=origin_tid, + ) + return + wave_start_raw = input_data.get("wave-start") + wave_start = None + if wave_start_raw is not None: + try: + wave_start = int(wave_start_raw) + except (TypeError, ValueError): + ctx.logger.error(f"invalid wave_start: {wave_start_raw}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_wave_start", "wave_start": wave_start_raw}, + trace_id=origin_tid, + ) + return + if wave_start < 0: + ctx.logger.error(f"wave_start must be >= 0: {wave_start}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_wave_start", "wave_start": wave_start}, + trace_id=origin_tid, + ) + return + wave = True image_path = resolve_image(bbdir, image_name) if not image_path: @@ -70,6 +101,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # bitstream is typically: /fpgaCompDir/bitstream.bit # so build_dir is 2 levels up build_dir = input_data.get("build_dir") + if not build_dir: + build_dir = input_data.get("build-dir") if not build_dir: bitstream_abs = os.path.abspath(bitstream) # Go up: fpgaCompDir -> build_dir @@ -85,7 +118,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - log_dir = input_data.get("log_dir") or f"{bebop_dir}/log/p2e-runworkload-{timestamp}" + log_dir = input_data.get("log_dir") + if not log_dir: + log_dir = input_data.get("log-dir") + if not log_dir: + log_dir = f"{bebop_dir}/log/p2e-runworkload-{timestamp}" os.makedirs(log_dir, exist_ok=True) # ── Run bebop p2e --runworkload ─────────────────────────────────────── @@ -100,6 +137,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"--build-dir=\"{build_dir}\" " f"--log-dir=\"{log_dir}\"" ) + if wave: + run_cmd += " --wave" + if wave_start is not None: + run_cmd += f" --wave-start=\"{wave_start}\"" ctx.logger.info(f"Running bebop p2e runworkload: {run_cmd}") run_result = stream_run_logger( cmd=run_cmd, diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml index f8a110bb4..863194b34 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml @@ -45,4 +45,7 @@ tests = [ "ctest_transpose_matmul_singlecore-baremetal", "optest_buckyball_matmul_16x64_64x16_singlecore-baremetal", "optest_buckyball_matmul_64x16_16x64_singlecore-baremetal", + "optest_linalg_conv2d_nhwc_hwcf_singlecore-baremetal", + "optest_linalg_conv2d_nhwc_fhwc_singlecore-baremetal", + "optest_linalg_conv2d_nchw_fchw_singlecore-baremetal", ] diff --git a/bbdev/bbdev b/bbdev/bbdev index 7b4c6cb33..000ca6573 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -72,7 +72,7 @@ WORKFLOW_COMMANDS = { "clean": 'Clean Bebop P2E build directory. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', "buildbitstream": "Build Bebop P2E bitstream.", - "runworkload": 'Run Bebop P2E workload. Args: "--image --bitstream [--config ] [--output-dir ] [--log-dir ]"', + "runworkload": 'Run Bebop P2E workload. Args: "--image --bitstream [--wave] [--wave-start ] [--config ] [--output-dir ] [--log-dir ]"', "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--bitstream --build-dir --test "', }, "dc": { From 034a63d0ef76985648e0bc115741ae0e21ddc2cb Mon Sep 17 00:00:00 2001 From: sjm Date: Fri, 29 May 2026 18:33:20 +0800 Subject: [PATCH 070/196] feat: add trace for lenet --- .../api/steps/bebop/bemu/02_sim_event.step.py | 7 +++++++ bbdev/api/steps/workload/01_buidl_api.step.py | 1 + .../api/steps/workload/01_build_event.step.py | 18 +++++++++++++++++- bbdev/api/steps/workload/README.md | 3 +++ .../api/tests/test_api_workload_build_trace.py | 3 +++ bbdev/bbdev | 2 +- 6 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 bbdev/api/tests/test_api_workload_build_trace.py diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 68c4ee08d..dc64435dc 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -52,6 +52,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") log_dir = input_data.get("log_dir") or f"{arch_dir}/log/{timestamp}-{binary_name}-bemu" os.makedirs(log_dir, exist_ok=True) + trace_path = os.path.join(log_dir, "layer-trace.ndjson") + run_env = os.environ.copy() + run_env.setdefault("BB_LAYER_TRACE_PATH", trace_path) # ── Run bebop bemu ──────────────────────────────────────────────────── pk_flag = " --pk" if input_data.get("pk") else "" @@ -68,7 +71,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cwd=binary_dir, stdout_prefix="bebop bemu", stderr_prefix="bebop bemu", + env=run_env, ) + fallback_trace_path = os.path.join(binary_dir, "layer-trace.ndjson") + if not os.path.exists(trace_path) and os.path.exists(fallback_trace_path): + os.replace(fallback_trace_path, trace_path) await check_result( ctx, diff --git a/bbdev/api/steps/workload/01_buidl_api.step.py b/bbdev/api/steps/workload/01_buidl_api.step.py index aa491ce84..cef07655a 100644 --- a/bbdev/api/steps/workload/01_buidl_api.step.py +++ b/bbdev/api/steps/workload/01_buidl_api.step.py @@ -17,6 +17,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "workload": body.get("workload", ""), "model": body.get("model", ""), + "trace": body.get("trace", False), } await ctx.enqueue({"topic": "workload.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index 2ee8c55ac..1f49afc47 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -27,6 +27,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: workload_dir = f"{bbdir}/bb-tests" build_dir = f"{workload_dir}/build" model = input_data.get("model", "") + trace = bool(input_data.get("trace", False)) model_targets = { "lenet": "buddy-buckyball-lenet-run", @@ -34,9 +35,24 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "resnet": "buddy-buckyball-resnet-run", "yolo": "buddy-buckyball-yolo26-run", } + trace_model_targets = { + "lenet": "buddy-buckyball-lenet-trace-run", + } target = "" if model: - target = model_targets.get(model.lower()) + model_key = model.lower() + if trace: + target = trace_model_targets.get(model_key) + if target is None: + ctx.logger.error(f"Trace build is not supported for model: {model}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "trace_unsupported_model", "model": model}, + trace_id=origin_tid, + ) + return + else: + target = model_targets.get(model_key) if target is None: ctx.logger.error(f"Unknown model: {model}") await check_result( diff --git a/bbdev/api/steps/workload/README.md b/bbdev/api/steps/workload/README.md index a6bb39414..29848bd73 100644 --- a/bbdev/api/steps/workload/README.md +++ b/bbdev/api/steps/workload/README.md @@ -17,6 +17,9 @@ Workload build workflow in Buckyball framework, used to build test workloads and # Build specific workload bbdev workload --build "--workload test_program" +# Build traced LeNet workload +bbdev workload --build "--model lenet --trace" + # Build all workloads bbdev workload --build ``` diff --git a/bbdev/api/tests/test_api_workload_build_trace.py b/bbdev/api/tests/test_api_workload_build_trace.py new file mode 100644 index 000000000..e5455a6b5 --- /dev/null +++ b/bbdev/api/tests/test_api_workload_build_trace.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev workload --build '--model lenet --trace'") diff --git a/bbdev/bbdev b/bbdev/bbdev index 000ca6573..f60fd178c 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -40,7 +40,7 @@ WORKFLOW_COMMANDS = { "run": 'Integrated build+sim+run. Args: "--binary [--batch] [--job ]"', }, "workload": { - "build": "Build workload.", + "build": 'Build workload. Args: "[--model ] [--trace]"', "tohex": "Convert all -baremetal ELF files under bb-tests/output/workloads/src to hex.", }, "kernel": { From 6aa37d891baba27230d1490953caf6042a6a65dc Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 30 May 2026 21:28:41 +0800 Subject: [PATCH 071/196] refactor: remove trace functionality from workload build and related tests --- bbdev/api/steps/bebop/bemu/02_sim_event.step.py | 7 ------- bbdev/api/steps/workload/01_buidl_api.step.py | 4 ---- bbdev/api/steps/workload/01_build_event.step.py | 17 +---------------- bbdev/api/steps/workload/README.md | 3 --- .../api/tests/test_api_workload_build_trace.py | 3 --- bbdev/bbdev | 2 +- 6 files changed, 2 insertions(+), 34 deletions(-) delete mode 100644 bbdev/api/tests/test_api_workload_build_trace.py diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index dc64435dc..68c4ee08d 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -52,9 +52,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") log_dir = input_data.get("log_dir") or f"{arch_dir}/log/{timestamp}-{binary_name}-bemu" os.makedirs(log_dir, exist_ok=True) - trace_path = os.path.join(log_dir, "layer-trace.ndjson") - run_env = os.environ.copy() - run_env.setdefault("BB_LAYER_TRACE_PATH", trace_path) # ── Run bebop bemu ──────────────────────────────────────────────────── pk_flag = " --pk" if input_data.get("pk") else "" @@ -71,11 +68,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cwd=binary_dir, stdout_prefix="bebop bemu", stderr_prefix="bebop bemu", - env=run_env, ) - fallback_trace_path = os.path.join(binary_dir, "layer-trace.ndjson") - if not os.path.exists(trace_path) and os.path.exists(fallback_trace_path): - os.replace(fallback_trace_path, trace_path) await check_result( ctx, diff --git a/bbdev/api/steps/workload/01_buidl_api.step.py b/bbdev/api/steps/workload/01_buidl_api.step.py index cef07655a..254bdb94b 100644 --- a/bbdev/api/steps/workload/01_buidl_api.step.py +++ b/bbdev/api/steps/workload/01_buidl_api.step.py @@ -1,7 +1,5 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import get_buckyball_path - config = { "name": "workload-build-api", "description": "build workload", @@ -12,12 +10,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = request.body or {} data = { "workload": body.get("workload", ""), "model": body.get("model", ""), - "trace": body.get("trace", False), } await ctx.enqueue({"topic": "workload.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index 1f49afc47..14a85f1c8 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -27,7 +27,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: workload_dir = f"{bbdir}/bb-tests" build_dir = f"{workload_dir}/build" model = input_data.get("model", "") - trace = bool(input_data.get("trace", False)) model_targets = { "lenet": "buddy-buckyball-lenet-run", @@ -35,24 +34,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "resnet": "buddy-buckyball-resnet-run", "yolo": "buddy-buckyball-yolo26-run", } - trace_model_targets = { - "lenet": "buddy-buckyball-lenet-trace-run", - } target = "" if model: model_key = model.lower() - if trace: - target = trace_model_targets.get(model_key) - if target is None: - ctx.logger.error(f"Trace build is not supported for model: {model}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "trace_unsupported_model", "model": model}, - trace_id=origin_tid, - ) - return - else: - target = model_targets.get(model_key) + target = model_targets.get(model_key) if target is None: ctx.logger.error(f"Unknown model: {model}") await check_result( diff --git a/bbdev/api/steps/workload/README.md b/bbdev/api/steps/workload/README.md index 29848bd73..a6bb39414 100644 --- a/bbdev/api/steps/workload/README.md +++ b/bbdev/api/steps/workload/README.md @@ -17,9 +17,6 @@ Workload build workflow in Buckyball framework, used to build test workloads and # Build specific workload bbdev workload --build "--workload test_program" -# Build traced LeNet workload -bbdev workload --build "--model lenet --trace" - # Build all workloads bbdev workload --build ``` diff --git a/bbdev/api/tests/test_api_workload_build_trace.py b/bbdev/api/tests/test_api_workload_build_trace.py deleted file mode 100644 index e5455a6b5..000000000 --- a/bbdev/api/tests/test_api_workload_build_trace.py +++ /dev/null @@ -1,3 +0,0 @@ -from _api_test_helper import run_bbdev_case - -run_bbdev_case("bbdev workload --build '--model lenet --trace'") diff --git a/bbdev/bbdev b/bbdev/bbdev index f60fd178c..e21626191 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -40,7 +40,7 @@ WORKFLOW_COMMANDS = { "run": 'Integrated build+sim+run. Args: "--binary [--batch] [--job ]"', }, "workload": { - "build": 'Build workload. Args: "[--model ] [--trace]"', + "build": 'Build workload. Args: "[--model ]"', "tohex": "Convert all -baremetal ELF files under bb-tests/output/workloads/src to hex.", }, "kernel": { From dd6ee0deb06b2f94203530f78cfd05a97ec2e8d6 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 31 May 2026 18:54:31 +0800 Subject: [PATCH 072/196] feat: update kernel build command with hart count parameters validation --- bbdev/api/steps/kernel/01_build_event.step.py | 34 +++++++++++++++++-- bbdev/bbdev | 2 +- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index e2f23d7c5..dd372c9fc 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -27,6 +27,25 @@ } +def hart_count_params(input_data: dict) -> dict: + visible = int(input_data.get("visible-hart-count", 64)) + total = int(input_data.get("total-hart-count", visible)) + hidden_base = int(input_data.get("hidden-hart-base", visible)) + + if visible < 1: + raise ValueError("visible-hart-count must be at least 1") + if total < visible: + raise ValueError("total-hart-count must cover visible harts") + if hidden_base < visible: + raise ValueError("hidden-hart-base must be after visible harts") + + return { + "visible": visible, + "total": total, + "hidden_base": hidden_base, + } + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -40,8 +59,20 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: shutil.rmtree(output_dir) os.makedirs(output_dir, exist_ok=True) + try: + hart_params = hart_count_params(input_data) + except ValueError as e: + ctx.logger.error(str(e)) + await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) + return + # cmake configure - configure_cmd = f"cmake -B {kernel_build} -S {kernel_src}" + configure_cmd = ( + f"cmake -B {kernel_build} -S {kernel_src} " + f"-DBUCKYBALL_VISIBLE_HART_COUNT={hart_params['visible']} " + f"-DBUCKYBALL_TOTAL_HART_COUNT={hart_params['total']} " + f"-DBUCKYBALL_HIDDEN_HART_BASE={hart_params['hidden_base']}" + ) result = stream_run_logger( cmd=configure_cmd, logger=ctx.logger, @@ -82,4 +113,3 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return await check_result(ctx, 0, continue_run=False, trace_id=origin_tid) - diff --git a/bbdev/bbdev b/bbdev/bbdev index e21626191..29e6fbea7 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -44,7 +44,7 @@ WORKFLOW_COMMANDS = { "tohex": "Convert all -baremetal ELF files under bb-tests/output/workloads/src to hex.", }, "kernel": { - "build": "Build RISC-V kernel + rootfs for Pegasus (cmake kernel-build).", + "build": 'Build RISC-V kernel + rootfs for Pegasus. Args: "[--visible-hart-count ] [--total-hart-count ] [--hidden-hart-base ]"', }, "firesim": { "enumeratefpgas": "Enumerate FPGAs.", From 12886fe143b99a1807927b13f1d03bb51f814395 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 31 May 2026 20:05:32 +0800 Subject: [PATCH 073/196] feat: enhance yosys commands and logging functionality with log directory support --- bbdev/api/steps/mill/03_yosys_verilog_event.step.py | 3 ++- bbdev/api/steps/yosys/01_run_api.step.py | 9 ++++++--- bbdev/api/steps/yosys/02_verilog_api.step.py | 9 ++++++--- bbdev/api/steps/yosys/03_synth_api.step.py | 9 ++++++--- bbdev/api/steps/yosys/03_synth_event.step.py | 6 +++++- bbdev/api/utils/yosys_log.py | 12 ++++++++++++ bbdev/bbdev | 6 +++--- 7 files changed, 40 insertions(+), 14 deletions(-) create mode 100644 bbdev/api/utils/yosys_log.py diff --git a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py index d0263b2e8..bb1d1f5eb 100644 --- a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py +++ b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py @@ -163,7 +163,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/") + yosys_log_dir = input_data.get("log_dir", f"{bbdir}/bbdev/api/steps/yosys/log") arch_dir = f"{bbdir}/arch" + ctx.logger.info(f"Yosys log dir: {yosys_log_dir}") yosys_cfg = load_yosys_config() elaborate_config = input_data.get("config") or yosys_cfg.get( @@ -204,7 +206,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if os.path.exists(topname_file): os.remove(topname_file) - yosys_log_dir = f"{bbdir}/bbdev/api/steps/yosys/log" os.makedirs(yosys_log_dir, exist_ok=True) try: source_list_path = prepare_yosys_verilog(build_dir, yosys_log_dir, ctx.logger) diff --git a/bbdev/api/steps/yosys/01_run_api.step.py b/bbdev/api/steps/yosys/01_run_api.step.py index f8c4dc99f..609400289 100644 --- a/bbdev/api/steps/yosys/01_run_api.step.py +++ b/bbdev/api/steps/yosys/01_run_api.step.py @@ -1,6 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api from utils.path import get_buckyball_path +from utils.yosys_log import make_yosys_log_dir, req_arg config = { "name": "yosys-run", @@ -14,11 +15,13 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = req.body or {} + log_dir = req_arg(body, "log_dir") or make_yosys_log_dir(bbdir, ctx.trace_id) data = { - "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), - "top": body.get("top"), - "config": body.get("config"), + "output_dir": req_arg(body, "output_dir") or f"{bbdir}/arch/build/", + "log_dir": log_dir, + "top": req_arg(body, "top"), + "config": req_arg(body, "config"), "from_run_workflow": True, } await ctx.enqueue({"topic": "yosys.run", "data": {**data, "_trace_id": ctx.trace_id}}) diff --git a/bbdev/api/steps/yosys/02_verilog_api.step.py b/bbdev/api/steps/yosys/02_verilog_api.step.py index 9d14550fc..88c8f199f 100644 --- a/bbdev/api/steps/yosys/02_verilog_api.step.py +++ b/bbdev/api/steps/yosys/02_verilog_api.step.py @@ -1,6 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api from utils.path import get_buckyball_path +from utils.yosys_log import make_yosys_log_dir, req_arg config = { "name": "yosys-verilog-api", @@ -14,11 +15,13 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = req.body or {} + log_dir = req_arg(body, "log_dir") or make_yosys_log_dir(bbdir, ctx.trace_id) data = { - "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), - "top": body.get("top"), - "config": body.get("config"), + "output_dir": req_arg(body, "output_dir") or f"{bbdir}/arch/build/", + "log_dir": log_dir, + "top": req_arg(body, "top"), + "config": req_arg(body, "config"), } await ctx.enqueue({"topic": "yosys.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/yosys/03_synth_api.step.py b/bbdev/api/steps/yosys/03_synth_api.step.py index 4d4926c8a..09055c907 100644 --- a/bbdev/api/steps/yosys/03_synth_api.step.py +++ b/bbdev/api/steps/yosys/03_synth_api.step.py @@ -1,6 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api from utils.path import get_buckyball_path +from utils.yosys_log import make_yosys_log_dir, req_arg config = { "name": "yosys-synth-api", @@ -14,11 +15,13 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = req.body or {} + log_dir = req_arg(body, "log_dir") or make_yosys_log_dir(bbdir, ctx.trace_id) data = { - "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), - "top": body.get("top"), - "config": body.get("config"), + "output_dir": req_arg(body, "output_dir") or f"{bbdir}/arch/build/", + "log_dir": log_dir, + "top": req_arg(body, "top"), + "config": req_arg(body, "config"), } await ctx.enqueue({"topic": "yosys.synth", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/yosys/03_synth_event.step.py b/bbdev/api/steps/yosys/03_synth_event.step.py index c3393ca31..d075cfe28 100644 --- a/bbdev/api/steps/yosys/03_synth_event.step.py +++ b/bbdev/api/steps/yosys/03_synth_event.step.py @@ -1,6 +1,7 @@ import os import sys import yaml +from datetime import datetime from motia import FlowContext, queue @@ -61,8 +62,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return failure_result - yosys_output_dir = os.path.join(os.path.dirname(__file__), "log") + stamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") + fallback_log_dir = os.path.join(os.path.dirname(__file__), "log", f"{stamp}-{origin_tid[:8]}") + yosys_output_dir = input_data.get("log_dir", fallback_log_dir) os.makedirs(yosys_output_dir, exist_ok=True) + ctx.logger.info(f"Yosys log dir: {yosys_output_dir}") read_commands = "\n".join([f"read_verilog -sv {src}" for src in vsrcs]) yosys_script = f"{yosys_output_dir}/synth_area.ys" diff --git a/bbdev/api/utils/yosys_log.py b/bbdev/api/utils/yosys_log.py new file mode 100644 index 000000000..613cee49e --- /dev/null +++ b/bbdev/api/utils/yosys_log.py @@ -0,0 +1,12 @@ +import os +from datetime import datetime + + +def req_arg(body: dict, name: str): + return body.get(name) or body.get(name.replace("_", "-")) + + +def make_yosys_log_dir(bbdir: str, trace_id: str) -> str: + stamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") + suffix = trace_id[:8] if trace_id else "no-trace" + return os.path.join(bbdir, "bbdev", "api", "steps", "yosys", "log", f"{stamp}-{suffix}") diff --git a/bbdev/bbdev b/bbdev/bbdev index 29e6fbea7..8d14249eb 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -79,9 +79,9 @@ WORKFLOW_COMMANDS = { "run": 'Run Design Compiler synthesis. Args: "[--top ] [--config ] [--output-dir ]"', }, "yosys": { - "run": 'Run yosys flow. Args: "[--top ] [--config ] [--output-dir ]"', - "verilog": 'Generate verilog for yosys flow. Args: "[--config ] [--output-dir ]"', - "synth": 'Run yosys synthesis. Args: "[--top ] [--output-dir ]"', + "run": 'Run yosys flow. Args: "[--top ] [--config ] [--output-dir ] [--log-dir ]"', + "verilog": 'Generate verilog for yosys flow. Args: "[--config ] [--output-dir ] [--log-dir ]"', + "synth": 'Run yosys synthesis. Args: "[--top ] [--output-dir ] [--log-dir ]"', }, } From 657aa83bf881270ef97a97246027c7a9978d78c7 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 3 Jun 2026 13:38:13 +0800 Subject: [PATCH 074/196] feat: add multicore kernel build and runworkload commands with multi-FPGA support and validation --- .../bebop/p2e/03_runworkload_event.step.py | 5 ++- bbdev/api/steps/kernel/01_build_event.step.py | 41 ++++++++++++++----- bbdev/api/tests/test_api_kernel_build.py | 1 + bbdev/bbdev | 4 +- 4 files changed, 37 insertions(+), 14 deletions(-) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index 4aac4c1ee..640921693 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -4,7 +4,7 @@ Loads a kernel image into FPGA and runs the workload via bebop CLI: 1. Resolve image name to .hex file under bb-tests/output/ 2. Validate bitstream .bit file path - 3. Run bebop p2e --runworkload --image --bitstream [--wave] [--wave-start ] --log-dir + 3. Run bebop p2e --runworkload --image --bitstream [--multi-fpga] [--wave] [--wave-start ] --log-dir """ import glob import os @@ -46,6 +46,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: image_name = input_data.get("image", "") bitstream = input_data.get("bitstream", "") + multi_fpga = bool(input_data.get("multi-fpga", False)) wave = bool(input_data.get("wave", False)) if "wave_start" in input_data: ctx.logger.error("invalid parameter: --wave_start (use --wave-start)") @@ -137,6 +138,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"--build-dir=\"{build_dir}\" " f"--log-dir=\"{log_dir}\"" ) + if multi_fpga: + run_cmd += " --multi-fpga" if wave: run_cmd += " --wave" if wave_start is not None: diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index dd372c9fc..b8311e015 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -1,6 +1,5 @@ import os import sys -import shutil from motia import FlowContext, queue @@ -28,16 +27,22 @@ def hart_count_params(input_data: dict) -> dict: + allowed = {"visible-hart-count", "total-hart-count", "_trace_id"} + unknown = sorted(k for k in input_data if k not in allowed) + if unknown: + raise ValueError(f"unknown kernel build parameter(s): {', '.join(unknown)}") + + if "hidden-hart-base" in input_data: + raise ValueError("hidden-hart-base is not supported; hidden harts must start at visible-hart-count") + visible = int(input_data.get("visible-hart-count", 64)) total = int(input_data.get("total-hart-count", visible)) - hidden_base = int(input_data.get("hidden-hart-base", visible)) + hidden_base = visible if visible < 1: raise ValueError("visible-hart-count must be at least 1") if total < visible: raise ValueError("total-hart-count must cover visible harts") - if hidden_base < visible: - raise ValueError("hidden-hart-base must be after visible harts") return { "visible": visible, @@ -46,17 +51,29 @@ def hart_count_params(input_data: dict) -> dict: } +def kernel_build_dir(bbdir: str, hart_params: dict) -> str: + visible = hart_params["visible"] + total = hart_params["total"] + if visible == 64 and total == 64: + return os.path.join(bbdir, "bb-tests", "build", "kernel") + return os.path.join(bbdir, "bb-tests", "build", f"kernel-v{visible}-t{total}") + + +def fw_payload_name(hart_params: dict) -> str: + visible = hart_params["visible"] + total = hart_params["total"] + if visible == 64 and total == 64: + return "fw_payload" + return f"fw_payload-v{visible}-t{total}" + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() kernel_src = os.path.join(bbdir, "bb-tests", "workloads", "lib", "kernel") - kernel_build = os.path.join(bbdir, "bb-tests", "build", "kernel") output_dir = os.path.join(bbdir, "bb-tests", "output", "kernel") - # Clear previous output before rebuild - if os.path.isdir(output_dir): - shutil.rmtree(output_dir) os.makedirs(output_dir, exist_ok=True) try: @@ -65,6 +82,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.error(str(e)) await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) return + kernel_build = kernel_build_dir(bbdir, hart_params) # cmake configure configure_cmd = ( @@ -97,11 +115,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return # Convert fw_payload.bin to hex for P2E memory backdoor - fw_payload_bin = os.path.join(output_dir, "fw_payload.bin") - fw_payload_hex = os.path.join(output_dir, "fw_payload.hex") + payload_name = fw_payload_name(hart_params) + fw_payload_bin = os.path.join(output_dir, f"{payload_name}.bin") + fw_payload_hex = os.path.join(output_dir, f"{payload_name}.hex") if not os.path.exists(fw_payload_bin): - ctx.logger.error("fw_payload.bin not found") + ctx.logger.error(f"{payload_name}.bin not found") await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) return diff --git a/bbdev/api/tests/test_api_kernel_build.py b/bbdev/api/tests/test_api_kernel_build.py index 26f502d09..46514658a 100644 --- a/bbdev/api/tests/test_api_kernel_build.py +++ b/bbdev/api/tests/test_api_kernel_build.py @@ -1,3 +1,4 @@ from _api_test_helper import run_bbdev_case run_bbdev_case("bbdev kernel --build") +run_bbdev_case("bbdev kernel --build '--visible-hart-count 64 --total-hart-count 256'") diff --git a/bbdev/bbdev b/bbdev/bbdev index 8d14249eb..4228e4bbb 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -44,7 +44,7 @@ WORKFLOW_COMMANDS = { "tohex": "Convert all -baremetal ELF files under bb-tests/output/workloads/src to hex.", }, "kernel": { - "build": 'Build RISC-V kernel + rootfs for Pegasus. Args: "[--visible-hart-count ] [--total-hart-count ] [--hidden-hart-base ]"', + "build": 'Build RISC-V kernel + rootfs for Pegasus. Args: "[--visible-hart-count ] [--total-hart-count ]"', }, "firesim": { "enumeratefpgas": "Enumerate FPGAs.", @@ -72,7 +72,7 @@ WORKFLOW_COMMANDS = { "clean": 'Clean Bebop P2E build directory. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', "buildbitstream": "Build Bebop P2E bitstream.", - "runworkload": 'Run Bebop P2E workload. Args: "--image --bitstream [--wave] [--wave-start ] [--config ] [--output-dir ] [--log-dir ]"', + "runworkload": 'Run Bebop P2E workload. Args: "--image --bitstream [--multi-fpga] [--wave] [--wave-start ] [--config ] [--output-dir ] [--log-dir ]"', "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--bitstream --build-dir --test "', }, "dc": { From 6777517e78676e09dece070ee90ae118c0a70948 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 4 Jun 2026 14:45:47 +0800 Subject: [PATCH 075/196] feat: extend kernel build command to include model parameter and validate against predefined models --- bbdev/api/steps/kernel/01_build_event.step.py | 42 ++++++++++++++++--- bbdev/bbdev | 2 +- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index b8311e015..894e40eef 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -17,6 +17,19 @@ sys.path.insert(0, scripts_path) from bin_to_hex import bin_to_hex +KERNEL_MODELS = { + "bert", + "deepseekr1", + "gemma4", + "lenet", + "llama2", + "mobilenet", + "qwen3", + "resnet", + "stablediffusion", + "yolo", +} + config = { "name": "kernel-build", "description": "build RISC-V kernel + rootfs for image via bb-tests/workloads/lib/kernel", @@ -27,7 +40,7 @@ def hart_count_params(input_data: dict) -> dict: - allowed = {"visible-hart-count", "total-hart-count", "_trace_id"} + allowed = {"visible-hart-count", "total-hart-count", "model", "_trace_id"} unknown = sorted(k for k in input_data if k not in allowed) if unknown: raise ValueError(f"unknown kernel build parameter(s): {', '.join(unknown)}") @@ -51,12 +64,27 @@ def hart_count_params(input_data: dict) -> dict: } -def kernel_build_dir(bbdir: str, hart_params: dict) -> str: +def kernel_model(input_data: dict) -> str: + model = input_data.get("model", "") + if model in ("", None): + return "" + if not isinstance(model, str): + raise ValueError("model must be a string") + + model = model.lower() + if model not in KERNEL_MODELS: + valid = ", ".join(sorted(KERNEL_MODELS)) + raise ValueError(f"unknown kernel model: {model}; valid models: {valid}") + return model + + +def kernel_build_dir(bbdir: str, hart_params: dict, model: str = "") -> str: visible = hart_params["visible"] total = hart_params["total"] + model_suffix = f"-model-{model}" if model else "" if visible == 64 and total == 64: - return os.path.join(bbdir, "bb-tests", "build", "kernel") - return os.path.join(bbdir, "bb-tests", "build", f"kernel-v{visible}-t{total}") + return os.path.join(bbdir, "bb-tests", "build", f"kernel{model_suffix}") + return os.path.join(bbdir, "bb-tests", "build", f"kernel-v{visible}-t{total}{model_suffix}") def fw_payload_name(hart_params: dict) -> str: @@ -78,18 +106,20 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: try: hart_params = hart_count_params(input_data) + model = kernel_model(input_data) except ValueError as e: ctx.logger.error(str(e)) await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) return - kernel_build = kernel_build_dir(bbdir, hart_params) + kernel_build = kernel_build_dir(bbdir, hart_params, model) # cmake configure configure_cmd = ( f"cmake -B {kernel_build} -S {kernel_src} " f"-DBUCKYBALL_VISIBLE_HART_COUNT={hart_params['visible']} " f"-DBUCKYBALL_TOTAL_HART_COUNT={hart_params['total']} " - f"-DBUCKYBALL_HIDDEN_HART_BASE={hart_params['hidden_base']}" + f"-DBUCKYBALL_HIDDEN_HART_BASE={hart_params['hidden_base']} " + f"-DBUCKYBALL_KERNEL_MODEL={model}" ) result = stream_run_logger( cmd=configure_cmd, diff --git a/bbdev/bbdev b/bbdev/bbdev index 4228e4bbb..071a9ffb0 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -44,7 +44,7 @@ WORKFLOW_COMMANDS = { "tohex": "Convert all -baremetal ELF files under bb-tests/output/workloads/src to hex.", }, "kernel": { - "build": 'Build RISC-V kernel + rootfs for Pegasus. Args: "[--visible-hart-count ] [--total-hart-count ]"', + "build": 'Build RISC-V kernel + rootfs for Pegasus. Args: "[--visible-hart-count ] [--total-hart-count ] [--model ]"', }, "firesim": { "enumeratefpgas": "Enumerate FPGAs.", From 09ff7470d0c31fed308f80c1b3dc4043ea032a30 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 5 Jun 2026 15:32:24 +0800 Subject: [PATCH 076/196] refactor: remove deprecated Design Compiler API steps and update related commands for RTL generation --- bbdev/api/steps/dc/01_run_api.step.py | 25 ---- bbdev/api/steps/dc/02_verilog_api.step.py | 10 +- bbdev/api/steps/dc/03_synth_api.step.py | 24 ---- bbdev/api/steps/dc/03_synth_event.step.py | 125 ------------------ bbdev/api/steps/dc/scripts/dc-config.yaml | 25 ---- bbdev/api/steps/dc/scripts/run-dc.tcl | 42 ------ .../steps/mill/04_dc_verilog_event.step.py | 56 ++++---- bbdev/api/tests/test_all_api_serial.py | 2 - bbdev/api/tests/test_api_dc_run.py | 3 - bbdev/api/tests/test_api_dc_synth.py | 3 - bbdev/api/tests/test_api_dc_verilog.py | 2 +- bbdev/api/utils/path.py | 44 +++++- bbdev/bbdev | 2 +- 13 files changed, 81 insertions(+), 282 deletions(-) delete mode 100644 bbdev/api/steps/dc/01_run_api.step.py delete mode 100644 bbdev/api/steps/dc/03_synth_api.step.py delete mode 100644 bbdev/api/steps/dc/03_synth_event.step.py delete mode 100644 bbdev/api/steps/dc/scripts/dc-config.yaml delete mode 100644 bbdev/api/steps/dc/scripts/run-dc.tcl delete mode 100644 bbdev/api/tests/test_api_dc_run.py delete mode 100644 bbdev/api/tests/test_api_dc_synth.py diff --git a/bbdev/api/steps/dc/01_run_api.step.py b/bbdev/api/steps/dc/01_run_api.step.py deleted file mode 100644 index 9d22ca951..000000000 --- a/bbdev/api/steps/dc/01_run_api.step.py +++ /dev/null @@ -1,25 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - -from utils.path import get_buckyball_path - -config = { - "name": "dc-run", - "description": "run Design Compiler synthesis", - "flows": ["dc"], - "triggers": [api("POST", "/dc/run")], - "enqueues": ["dc.run"], -} - - -async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() - body = req.body or {} - - data = { - "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), - "top": body.get("top"), - "config": body.get("config"), - "from_run_workflow": True, - } - await ctx.enqueue({"topic": "dc.run", "data": {**data, "_trace_id": ctx.trace_id}}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/dc/02_verilog_api.step.py b/bbdev/api/steps/dc/02_verilog_api.step.py index 2fb798556..970565a01 100644 --- a/bbdev/api/steps/dc/02_verilog_api.step.py +++ b/bbdev/api/steps/dc/02_verilog_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import get_buckyball_path +from utils.path import check_dc_rtl_args, get_buckyball_path, get_dc_rtl_dir config = { "name": "dc-verilog-api", @@ -14,10 +14,14 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = req.body or {} + try: + check_dc_rtl_args(body) + output_dir = get_dc_rtl_dir(bbdir, body.get("config"), body.get("dir")) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) data = { - "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), - "top": body.get("top"), + "output_dir": output_dir, "config": body.get("config"), } await ctx.enqueue({"topic": "dc.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) diff --git a/bbdev/api/steps/dc/03_synth_api.step.py b/bbdev/api/steps/dc/03_synth_api.step.py deleted file mode 100644 index 7081a0ea4..000000000 --- a/bbdev/api/steps/dc/03_synth_api.step.py +++ /dev/null @@ -1,24 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - -from utils.path import get_buckyball_path - -config = { - "name": "dc-synth-api", - "description": "run Design Compiler synthesis", - "flows": ["dc"], - "triggers": [api("POST", "/dc/synth")], - "enqueues": ["dc.synth"], -} - - -async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() - body = req.body or {} - - data = { - "output_dir": body.get("output_dir", f"{bbdir}/arch/build/"), - "top": body.get("top"), - "config": body.get("config"), - } - await ctx.enqueue({"topic": "dc.synth", "data": {**data, "_trace_id": ctx.trace_id}}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/dc/03_synth_event.step.py b/bbdev/api/steps/dc/03_synth_event.step.py deleted file mode 100644 index bce1796b7..000000000 --- a/bbdev/api/steps/dc/03_synth_event.step.py +++ /dev/null @@ -1,125 +0,0 @@ -import os -import shutil -import sys -import yaml - -from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.event_common import check_result, get_origin_trace_id -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger - -config = { - "name": "dc-synth", - "description": "run Design Compiler synthesis", - "flows": ["dc"], - "triggers": [queue("dc.synth")], - "enqueues": [], -} - - -def load_dc_config(): - config_path = os.path.join(os.path.dirname(__file__), "scripts", "dc-config.yaml") - if os.path.exists(config_path): - with open(config_path, "r") as f: - return yaml.safe_load(f) or {} - return {} - - -def prepare_dc_design(build_dir: str, design_dir: str): - source_list_path = os.path.join(build_dir, "dc_sources.list") - if not os.path.exists(source_list_path): - raise RuntimeError("missing dc_sources.list, run dc verilog first") - with open(source_list_path, "r") as f: - sources = [line.strip() for line in f.readlines() if line.strip()] - if not sources: - raise RuntimeError("empty dc_sources.list") - for src in sources: - rel = os.path.relpath(src, build_dir) - dst = os.path.join(design_dir, rel) - os.makedirs(os.path.dirname(dst), exist_ok=True) - shutil.copy2(src, dst) - return source_list_path - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - bbdir = get_buckyball_path() - build_dir = input_data.get("output_dir") or f"{bbdir}/arch/build/" - - dc_cfg = load_dc_config() - top_module = input_data.get("top") or dc_cfg.get("top") or "BuckyballAccelerator" - - work_dir = f"{bbdir}/bb-tests/output/dc" - design_dir = f"{work_dir}/design" - report_dir = f"{work_dir}/reports" - tmp_dir = f"{work_dir}/tmp" - - for d in (design_dir, report_dir, tmp_dir): - if os.path.exists(d): - shutil.rmtree(d) - os.makedirs(d, exist_ok=True) - try: - source_list_path = prepare_dc_design(build_dir, design_dir) - except Exception as e: - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"task": "dc", "error": str(e)}, - trace_id=origin_tid, - ) - return - - tcl_script = os.path.join(os.path.dirname(__file__), "scripts", "run-dc.tcl") - target_libs = " ".join(dc_cfg.get("target_libraries") or []) - setup = ( - f'set design_dir "{design_dir}"; ' - f'set top_module "{top_module}"; ' - f'set work_dir "{work_dir}"; ' - f'set report_dir "{report_dir}"; ' - f'set tmp_dir "{tmp_dir}"; ' - f'set target_libs "{target_libs}"; ' - f'set clock_name "{dc_cfg.get("clock_name", "clock")}"; ' - f'set clock_period {dc_cfg.get("clock_period", 2.0)}; ' - f'set clock_uncertainty {dc_cfg.get("clock_uncertainty", 0.6)}; ' - f'set clock_transition {dc_cfg.get("clock_transition", 0.08)}; ' - f'set input_delay {dc_cfg.get("input_delay", 1.2)}; ' - f'set output_delay {dc_cfg.get("output_delay", 0.6)}; ' - f'set input_transition {dc_cfg.get("input_transition", 0.2)}; ' - f'set output_load {dc_cfg.get("output_load", 0.08)};' - ) - command = f'dc_shell -x \'{setup}\' -f {tcl_script}' - - result = stream_run_logger( - cmd=command, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="dc run", - stderr_prefix="dc run", - ) - - alib = os.path.join(bbdir, "alib-52") - if os.path.exists(alib): - shutil.rmtree(alib) - - extra = {"task": "dc", "report_dir": report_dir, "top": top_module, "source_list": source_list_path} - for name in ("area.rpt", "hierarchy.rpt", "timing.rpt", "power.rpt"): - path = os.path.join(report_dir, name) - if os.path.exists(path): - with open(path, "r") as f: - extra[name] = f.read() - - await check_result( - ctx, - result.returncode, - continue_run=False, - extra_fields=extra, - trace_id=origin_tid, - ) - - return diff --git a/bbdev/api/steps/dc/scripts/dc-config.yaml b/bbdev/api/steps/dc/scripts/dc-config.yaml deleted file mode 100644 index aff0087f9..000000000 --- a/bbdev/api/steps/dc/scripts/dc-config.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Design Compiler synthesis configuration - -# Default top module name -top: BuckyballAccelerator - -# Elaborate config class for Chisel verilog generation -elaborate_config: sims.verilator.BuckyballToyVerilatorConfig - -# Standard cell .db libraries (target_library = link_library) -target_libraries: - - /data0/tools/lib/db/scc28nhkcp_hdc35p140_rvt_ffg_v0p99_0c_basic.db - - /data0/tools/lib/db/scc28nhkcp_hdc35p140_rvt_ffg_v0p99_0c_ccs.db - - /data0/tools/lib/db/scc28nhkcp_hdc35p140_rvt_ffg_v0p99_0c_ecsm.db - -# Clock constraints (ns) -clock_name: clock -clock_period: 2.0 -clock_uncertainty: 0.6 -clock_transition: 0.08 - -# IO timing constraints (ns) -input_delay: 1.2 -output_delay: 0.6 -input_transition: 0.2 -output_load: 0.08 diff --git a/bbdev/api/steps/dc/scripts/run-dc.tcl b/bbdev/api/steps/dc/scripts/run-dc.tcl deleted file mode 100644 index f82625a2a..000000000 --- a/bbdev/api/steps/dc/scripts/run-dc.tcl +++ /dev/null @@ -1,42 +0,0 @@ -# DC synthesis script. -# Inputs (set via `dc_shell -x` before sourcing this file): -# design_dir, top_module, work_dir, report_dir, tmp_dir -# target_libs : space-separated list of .db files -# clock_name, clock_period, clock_uncertainty, clock_transition -# input_delay, output_delay, input_transition, output_load - -set search_path [list . $design_dir] -define_design_lib work -path $tmp_dir - -set target_library $target_libs -set link_library $target_libs - -set file_list [glob -nocomplain -directory $design_dir *.sv] -analyze -format sverilog $file_list -elaborate $top_module -current_design $top_module -link - -# Keep module boundaries for hierarchical reports. -set_ungroup [get_designs *] false -set_boundary_optimization [get_designs *] false - -create_clock -name $clock_name -period $clock_period [get_ports $clock_name] -set_clock_uncertainty $clock_uncertainty [get_clocks $clock_name] -set_input_delay $input_delay -clock $clock_name [remove_from_collection [all_inputs] [get_ports $clock_name]] -set_output_delay $output_delay -clock $clock_name [all_outputs] -set_clock_equivalence $clock_name -set_load $output_load [all_outputs] -set_input_transition $input_transition [remove_from_collection [all_inputs] [get_ports $clock_name]] -set_clock_transition $clock_transition [get_clocks $clock_name] - -compile_ultra -retime -scan -no_autoungroup -write -format ddc -hierarchy -output $report_dir/design_compiled.ddc - -report_area -hierarchy -nosplit > $report_dir/area.rpt -report_hierarchy -noleaf > $report_dir/hierarchy.rpt -report_timing > $report_dir/timing.rpt -report_power -hierarchy > $report_dir/power.rpt - -write -format verilog -output $report_dir/netlist.v -exit diff --git a/bbdev/api/steps/mill/04_dc_verilog_event.step.py b/bbdev/api/steps/mill/04_dc_verilog_event.step.py index bef80ca71..1633a48c6 100644 --- a/bbdev/api/steps/mill/04_dc_verilog_event.step.py +++ b/bbdev/api/steps/mill/04_dc_verilog_event.step.py @@ -2,7 +2,6 @@ import shutil import sys import glob -import yaml from motia import FlowContext, queue @@ -16,22 +15,13 @@ config = { "name": "dc-verilog", - "description": "generate verilog for dc flow", + "description": "generate RTL and memory metadata for downstream DC/tapeout flow", "flows": ["dc"], - "triggers": [queue("dc.run"), queue("dc.verilog")], - "enqueues": ["dc.synth"], + "triggers": [queue("dc.verilog")], + "enqueues": [], } -def load_dc_config(): - bbdir = get_buckyball_path() - config_path = f"{bbdir}/bbdev/api/steps/dc/scripts/dc-config.yaml" - if os.path.exists(config_path): - with open(config_path, "r") as f: - return yaml.safe_load(f) or {} - return {} - - def prepare_dc_verilog(build_dir: str): vsrcs = sorted( glob.glob(f"{build_dir}/**/*.sv", recursive=True) @@ -50,23 +40,38 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" - build_dir = input_data.get("output_dir") or f"{bbdir}/arch/build/" - - dc_cfg = load_dc_config() - elaborate_config = ( - input_data.get("config") - or dc_cfg.get("elaborate_config") - or "sims.verilator.BuckyballToyVerilatorConfig" - ) + elaborate_config = input_data.get("config") + if not elaborate_config: + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "validation", "error": "missing required parameter: config"}, + trace_id=origin_tid, + ) + return failure_result + build_dir = input_data.get("output_dir") + if not build_dir: + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "validation", "error": "missing required parameter: output_dir"}, + trace_id=origin_tid, + ) + return failure_result + ctx.logger.info(f"Using DC RTL output directory: {build_dir}") if os.path.exists(build_dir): shutil.rmtree(build_dir) os.makedirs(build_dir, exist_ok=True) + mem_conf = os.path.join(build_dir, "dc.mems.conf") verilog_command = ( f"mill -i __.test.runMain sims.verilator.Elaborate {elaborate_config} " - "--disable-annotation-unknown -strip-debug-info -O=debug " + "--disable-annotation-unknown --strip-debug-info -O=debug " "-lowering-options=disallowLocalVariables " + f"--repl-seq-mem --repl-seq-mem-file={mem_conf} " f"--split-verilog -o={build_dir}" ) @@ -103,12 +108,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result( ctx, result.returncode, - continue_run=input_data.get("from_run_workflow", False), - extra_fields={"task": "verilog", "source_list": source_list_path}, + continue_run=False, + extra_fields={"task": "verilog", "source_list": source_list_path, "mem_conf": mem_conf}, trace_id=origin_tid, ) - if input_data.get("from_run_workflow"): - await ctx.enqueue({"topic": "dc.synth", "data": {**input_data, "task": "run"}}) - return diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index 79dff79f6..8cb368e33 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -17,9 +17,7 @@ def main(): "test_api_yosys_run.py", "test_api_yosys_verilog.py", "test_api_yosys_synth.py", - "test_api_dc_run.py", "test_api_dc_verilog.py", - "test_api_dc_synth.py", "test_api_kernel_build.py", "test_api_firesim_enumeratefpgas.py", "test_api_firesim_buildbitstream.py", diff --git a/bbdev/api/tests/test_api_dc_run.py b/bbdev/api/tests/test_api_dc_run.py deleted file mode 100644 index 0f8f83720..000000000 --- a/bbdev/api/tests/test_api_dc_run.py +++ /dev/null @@ -1,3 +0,0 @@ -from _api_test_helper import run_bbdev_case - -run_bbdev_case("bbdev dc --run") diff --git a/bbdev/api/tests/test_api_dc_synth.py b/bbdev/api/tests/test_api_dc_synth.py deleted file mode 100644 index b86a5c8ca..000000000 --- a/bbdev/api/tests/test_api_dc_synth.py +++ /dev/null @@ -1,3 +0,0 @@ -from _api_test_helper import run_bbdev_case - -run_bbdev_case("bbdev dc --synth") diff --git a/bbdev/api/tests/test_api_dc_verilog.py b/bbdev/api/tests/test_api_dc_verilog.py index 232553bc5..ff77de9e7 100644 --- a/bbdev/api/tests/test_api_dc_verilog.py +++ b/bbdev/api/tests/test_api_dc_verilog.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev dc --verilog") +run_bbdev_case("bbdev dc --verilog '--config sims.verilator.BuckyballToyVerilatorConfig'") diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index 03da675fe..73faaa3ca 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -12,9 +12,51 @@ def get_verilator_build_dir(bbdir, config=None, output_dir=None): if output_dir: return output_dir + return get_config_build_dir(bbdir, config) + + +def sanitize_config_name(config=None): if config and config != "None": name = re.sub(r"[^A-Za-z0-9_.-]+", "_", config).strip("._") if name: - return f"{bbdir}/arch/build/{name}" + return name + return None + + +def get_config_build_dir(bbdir, config=None, output_dir=None, output_root=None): + if output_dir: + return output_dir + + name = sanitize_config_name(config) + if output_root: + if not name: + raise ValueError("output_root requires a valid config name") + return os.path.join(output_root, name) + + if name: + return f"{bbdir}/arch/build/{name}" return f"{bbdir}/arch/build" + + +def get_dc_rtl_dir(bbdir, config=None, base_dir=None): + if not config or config is True: + raise ValueError("missing required parameter: config") + + name = sanitize_config_name(config) + if not name: + raise ValueError("invalid config name") + + if base_dir is True: + raise ValueError("dir requires a path value") + root = base_dir or os.path.join(bbdir, "arch", "build") + if not os.path.isabs(root): + raise ValueError("dir must be an absolute path") + return os.path.join(root, name) + + +def check_dc_rtl_args(body: dict): + allowed = {"config", "dir"} + for name in body: + if name not in allowed: + raise ValueError(f"unexpected parameter: {name}") diff --git a/bbdev/bbdev b/bbdev/bbdev index 071a9ffb0..2f7d2d222 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -76,7 +76,7 @@ WORKFLOW_COMMANDS = { "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--bitstream --build-dir --test "', }, "dc": { - "run": 'Run Design Compiler synthesis. Args: "[--top ] [--config ] [--output-dir ]"', + "verilog": 'Generate DC RTL. Args: "--config [--dir ]"', }, "yosys": { "run": 'Run yosys flow. Args: "[--top ] [--config ] [--output-dir ] [--log-dir ]"', From bf646d1a0ba45b5cc08c0c477792cd71a4627f32 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 5 Jun 2026 18:06:48 +0800 Subject: [PATCH 077/196] feat: add DPI-C source filtering in DC Verilog preparation and log skipped sources --- .../steps/mill/04_dc_verilog_event.step.py | 49 +++++++++++++++++-- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/bbdev/api/steps/mill/04_dc_verilog_event.step.py b/bbdev/api/steps/mill/04_dc_verilog_event.step.py index 1633a48c6..eb2093bd9 100644 --- a/bbdev/api/steps/mill/04_dc_verilog_event.step.py +++ b/bbdev/api/steps/mill/04_dc_verilog_event.step.py @@ -2,6 +2,7 @@ import shutil import sys import glob +import re from motia import FlowContext, queue @@ -22,18 +23,52 @@ } +def is_dpi_source(path: str) -> bool: + with open(path, "r", encoding="utf-8", errors="ignore") as f: + return 'import "DPI-C"' in f.read() + + +def build_stub_from_header(src_path: str) -> str: + with open(src_path, "r", encoding="utf-8", errors="ignore") as f: + content = f.read() + m = re.search( + r"module\s+([A-Za-z_][A-Za-z0-9_]*)\s*(#\s*\(.*?\)\s*)?\((.*?)\)\s*;", + content, + re.S, + ) + if m is None: + raise RuntimeError(f"invalid module header format: {src_path}") + mod_name = m.group(1) + params_block = m.group(2) or "" + ports_block = m.group(3).rstrip() + return f"(* blackbox *) module {mod_name} {params_block}(\n{ports_block}\n);\nendmodule\n" + + def prepare_dc_verilog(build_dir: str): vsrcs = sorted( glob.glob(f"{build_dir}/**/*.sv", recursive=True) + glob.glob(f"{build_dir}/**/*.v", recursive=True) ) - if not vsrcs: + stub_dir = os.path.join(build_dir, "dc_stubs") + os.makedirs(stub_dir, exist_ok=True) + kept = [] + stubbed_dpi = [] + for path in vsrcs: + if is_dpi_source(path): + stub_path = os.path.join(stub_dir, f"stub_{os.path.basename(path)}") + with open(stub_path, "w") as f: + f.write(build_stub_from_header(path)) + kept.append(stub_path) + stubbed_dpi.append(path) + else: + kept.append(path) + if not kept: raise RuntimeError("no dc verilog source generated") source_list_path = os.path.join(build_dir, "dc_sources.list") with open(source_list_path, "w") as f: - for path in vsrcs: + for path in kept: f.write(path + "\n") - return source_list_path + return source_list_path, stubbed_dpi async def handler(input_data: dict, ctx: FlowContext) -> None: @@ -94,7 +129,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return failure_result try: - source_list_path = prepare_dc_verilog(build_dir) + source_list_path, stubbed_dpi = prepare_dc_verilog(build_dir) except Exception as e: _, failure_result = await check_result( ctx, @@ -104,6 +139,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return failure_result + if stubbed_dpi: + ctx.logger.info( + f"Stubbed {len(stubbed_dpi)} DPI-C sources for DC: " + f"{', '.join(os.path.basename(path) for path in stubbed_dpi[:10])}" + f"{'...' if len(stubbed_dpi) > 10 else ''}" + ) await check_result( ctx, From 69f33511bd60506cbba18c4291295cf6f2c2b2d5 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 8 Jun 2026 20:42:12 +0800 Subject: [PATCH 078/196] fix: harden bbdev cleanup and run workflow --- .../bebop/verilator/01_clean_event.step.py | 9 +- .../bebop/verilator/03_build_api.step.py | 8 +- .../bebop/verilator/03_build_event.step.py | 53 +++- .../bebop/verilator/04_sim_event.step.py | 84 ++++++- .../steps/bebop/verilator/05_run_api.step.py | 13 +- .../06_bebop_verilator_verilog_event.step.py | 56 ++++- bbdev/api/utils/bebop_verilator.py | 30 +++ bbdev/api/utils/event_common.py | 8 +- bbdev/api/utils/path.py | 8 + bbdev/api/utils/port.py | 26 +- bbdev/api/utils/process.py | 236 ++++++++++++++++++ bbdev/api/utils/stream_run.py | 4 +- bbdev/bbdev | 171 ++++++++----- 13 files changed, 599 insertions(+), 107 deletions(-) create mode 100644 bbdev/api/utils/bebop_verilator.py create mode 100644 bbdev/api/utils/process.py diff --git a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py index 28f4bbe91..f151b27fe 100644 --- a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py +++ b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py @@ -16,10 +16,10 @@ "description": "Clean verilator build directory", "flows": ["bebop"], "triggers": [ - queue("bebop.verilator.run"), queue("bebop.verilator.clean"), + queue("bebop.verilator.run.clean"), ], - "enqueues": ["bebop.verilator.verilog"], + "enqueues": ["bebop.verilator.verilog", "bebop.verilator.run.verilog"], } @@ -48,6 +48,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # Continue routing to verilog if from run workflow if input_data.get("from_run_workflow"): + data = {k: v for k, v in input_data.items() if k not in ("output_dir", "vsrc_dir")} + if input_data.get("_explicit_output_dir"): + data["output_dir"] = build_dir await ctx.enqueue( - {"topic": "bebop.verilator.verilog", "data": {**input_data, "output_dir": build_dir, "task": "run"}} + {"topic": "bebop.verilator.run.verilog", "data": {**data, "task": "run"}} ) diff --git a/bbdev/api/steps/bebop/verilator/03_build_api.step.py b/bbdev/api/steps/bebop/verilator/03_build_api.step.py index 342ddfb18..bed479674 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_api.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_api.step.py @@ -1,7 +1,5 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import get_buckyball_path, get_verilator_build_dir - config = { "name": "bebop-verilator-build-api", "description": "Build bebop verilator binary", @@ -12,7 +10,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = request.body or {} arch_config = body.get("config") @@ -22,11 +19,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body={"error": "Missing required parameter: --config must be specified"} ) - vsrc_dir = get_verilator_build_dir(bbdir, arch_config, body.get("vsrc_dir")) - data = { "config": arch_config, - "vsrc_dir": vsrc_dir, } + if "vsrc_dir" in body: + data["vsrc_dir"] = body.get("vsrc_dir") await ctx.enqueue({"topic": "bebop.verilator.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index 153f53f84..24d4649a4 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -15,16 +15,41 @@ from utils.path import get_buckyball_path, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id +from utils.bebop_verilator import write_build_marker config = { "name": "bebop-verilator-build", "description": "Build bebop verilator binary", "flows": ["bebop"], - "triggers": [queue("bebop.verilator.build")], - "enqueues": ["bebop.verilator.sim"], + "triggers": [queue("bebop.verilator.build"), queue("bebop.verilator.run.build")], + "enqueues": ["bebop.verilator.sim", "bebop.verilator.run.sim"], } +def describe_path(path: str) -> dict: + info = { + "cwd": os.getcwd(), + "uid": os.getuid(), + "exists": os.path.exists(path), + "is_dir": os.path.isdir(path), + "parent": os.path.dirname(path), + } + try: + stat = os.stat(path) + info.update({ + "mode": oct(stat.st_mode), + "owner_uid": stat.st_uid, + "owner_gid": stat.st_gid, + }) + except OSError as e: + info.update({"stat_error": str(e), "errno": e.errno}) + try: + info["parent_entries"] = sorted(os.listdir(info["parent"]))[:20] + except OSError as e: + info["parent_error"] = str(e) + return info + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -44,10 +69,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") if not os.path.isdir(vsrc_dir): - ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}") + path_info = describe_path(vsrc_dir) + ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}; path_info={path_info}") await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "vsrc_not_found", "vsrc_dir": vsrc_dir}, + extra_fields={ + "error": "vsrc_not_found", + "source": "bebop.verilator.build", + "vsrc_dir": vsrc_dir, + "path_info": path_info, + }, trace_id=origin_tid, ) return @@ -66,6 +97,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) bebop_bin = f"{bebop_dir}/target/debug/bebop" + if build_result.returncode == 0: + try: + write_build_marker(bebop_dir, arch_config, vsrc_dir, bebop_bin) + except OSError as e: + ctx.logger.error(f"failed to write bebop verilator build marker: {e}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "build_marker_write_failed", "detail": str(e)}, + trace_id=origin_tid, + ) + return + await check_result( ctx, build_result.returncode, @@ -84,5 +127,5 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # Continue routing to sim if from run workflow if input_data.get("from_run_workflow"): await ctx.enqueue( - {"topic": "bebop.verilator.sim", "data": {**input_data, "vsrc_dir": vsrc_dir, "task": "run"}} + {"topic": "bebop.verilator.run.sim", "data": {**input_data, "vsrc_dir": vsrc_dir, "task": "run"}} ) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index e48f09e44..108714dc9 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -4,11 +4,12 @@ Runs bebop verilator simulation: 1. Resolve binary path 2. Resolve verilog source directory (VSRC_PATH) - 3. Build bebop with verilator feature (cargo build) - 4. Run bebop verilator with elf, log and fst directories + 3. Run the built bebop binary with elf, log and fst directories """ import os +import shlex import sys +import json from datetime import datetime from motia import FlowContext, queue @@ -21,12 +22,13 @@ from utils.stream_run import stream_run_logger from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id +from utils.bebop_verilator import build_marker_path, read_build_marker config = { "name": "bebop-verilator-sim", "description": "Run bebop verilator simulation", "flows": ["bebop"], - "triggers": [queue("bebop.verilator.sim")], + "triggers": [queue("bebop.verilator.sim"), queue("bebop.verilator.run.sim")], "enqueues": [], } @@ -40,6 +42,66 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: arch_config = input_data.get("config", "sims.verilator.BuckyballToyVerilatorConfig") vsrc_dir = get_verilator_build_dir(bbdir, arch_config, input_data.get("vsrc_dir")) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") + if not os.path.isdir(vsrc_dir): + ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "vsrc_not_found", "vsrc_dir": vsrc_dir}, + trace_id=origin_tid, + ) + return + + bebop_bin = f"{bebop_dir}/target/debug/bebop" + if not os.path.isfile(bebop_bin): + ctx.logger.error(f"bebop binary does not exist: {bebop_bin}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "bebop_binary_not_found", "binary": bebop_bin}, + trace_id=origin_tid, + ) + return + try: + marker = read_build_marker(bebop_dir) + except (OSError, json.JSONDecodeError) as e: + marker_path = build_marker_path(bebop_dir) + ctx.logger.error(f"failed to read bebop verilator build marker: {e}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "build_marker_read_failed", "marker": marker_path, "detail": str(e)}, + trace_id=origin_tid, + ) + return + expect_vsrc = os.path.abspath(vsrc_dir) + expect_bin = os.path.abspath(bebop_bin) + if marker is None: + marker_path = build_marker_path(bebop_dir) + ctx.logger.error(f"bebop verilator build marker does not exist: {marker_path}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "build_marker_not_found", "marker": marker_path}, + trace_id=origin_tid, + ) + return + if ( + marker.get("config") != arch_config + or marker.get("vsrc_dir") != expect_vsrc + or marker.get("binary") != expect_bin + ): + ctx.logger.error(f"bebop verilator build marker mismatch: {marker}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={ + "error": "build_marker_mismatch", + "marker": marker, + "expected": { + "config": arch_config, + "vsrc_dir": expect_vsrc, + "binary": expect_bin, + }, + }, + trace_id=origin_tid, + ) + return binary_name = input_data.get("binary", "") binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) @@ -67,13 +129,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_args = (" " + " ".join(trace_flags)) if trace_flags else "" run_cmd = ( - f"cargo run --features verilator " - f"--config=\"env.ARCH_CONFIG='{arch_config}'\" " - f"--config=\"env.VSRC_PATH='{vsrc_dir}'\" " - f"-- verilator " - f"--elf=\"{binary_path}\" " - f"--log-dir=\"{log_dir}\" " - f"--fst-dir=\"{fst_dir}\"" + f"{shlex.quote(bebop_bin)} verilator " + f"--elf={shlex.quote(binary_path)} " + f"--log-dir={shlex.quote(log_dir)} " + f"--fst-dir={shlex.quote(fst_dir)}" f"{trace_args}" ) ctx.logger.info(f"Running bebop verilator: {run_cmd}") @@ -83,6 +142,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cwd=bebop_dir, stdout_prefix="bebop verilator", stderr_prefix="bebop verilator", + env={ + **os.environ, + "ARCH_CONFIG": arch_config, + "VSRC_PATH": vsrc_dir, + }, ) await check_result( diff --git a/bbdev/api/steps/bebop/verilator/05_run_api.step.py b/bbdev/api/steps/bebop/verilator/05_run_api.step.py index b80c170da..b7014b0e5 100644 --- a/bbdev/api/steps/bebop/verilator/05_run_api.step.py +++ b/bbdev/api/steps/bebop/verilator/05_run_api.step.py @@ -1,18 +1,15 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import get_buckyball_path, get_verilator_build_dir - config = { "name": "bebop-verilator-run-api", "description": "Run complete bebop verilator workflow: clean → verilog → build → sim", "flows": ["bebop"], "triggers": [api("POST", "/bebop/verilator/run")], - "enqueues": ["bebop.verilator.run"], + "enqueues": ["bebop.verilator.run.clean"], } async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = request.body or {} config_name = body.get("config") @@ -22,8 +19,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body={"error": "Missing required parameter: --config must be specified"} ) - build_dir = get_verilator_build_dir(bbdir, config_name, body.get("output_dir")) - binary = body.get("binary", "") if not binary: return ApiResponse( @@ -38,7 +33,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "config": config_name, - "output_dir": build_dir, "binary": binary, "balltype": body.get("balltype"), "itrace": body.get("itrace", False), @@ -50,5 +44,8 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "fst_dir": body.get("fst_dir"), "from_run_workflow": True, } - await ctx.enqueue({"topic": "bebop.verilator.run", "data": {**data, "_trace_id": ctx.trace_id}}) + if "output_dir" in body: + data["output_dir"] = body.get("output_dir") + data["_explicit_output_dir"] = True + await ctx.enqueue({"topic": "bebop.verilator.run.clean", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py index 137df50c8..66de11280 100644 --- a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py @@ -20,8 +20,8 @@ "name": "bebop-verilator-verilog", "description": "Generate verilog code via mill", "flows": ["bebop"], - "triggers": [queue("bebop.verilator.verilog")], - "enqueues": ["bebop.verilator.build"], + "triggers": [queue("bebop.verilator.verilog"), queue("bebop.verilator.run.verilog")], + "enqueues": ["bebop.verilator.build", "bebop.verilator.run.build"], } @@ -45,6 +45,19 @@ def prepare_verilator_verilog(build_dir: str, arch_dir: str, logger): logger.info(f"Patched fesvr includes from {patch_file}") +def check_verilog_output(build_dir: str) -> dict: + exists = os.path.exists(build_dir) + is_dir = os.path.isdir(build_dir) + sv_count = 0 + if is_dir: + sv_count = sum(1 for name in os.listdir(build_dir) if name.endswith((".sv", ".v"))) + return { + "exists": exists, + "is_dir": is_dir, + "sv_count": sv_count, + } + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -96,6 +109,43 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: prepare_verilator_verilog(build_dir, arch_dir, ctx.logger) + output_status = check_verilog_output(build_dir) + if result.returncode != 0: + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={ + "task": "verilog", + "config": config_name, + "output_dir": build_dir, + "output_status": output_status, + }, + trace_id=origin_tid, + ) + return + + if not output_status["is_dir"] or output_status["sv_count"] == 0: + ctx.logger.error( + f"Verilog output is invalid: {build_dir} " + f"(exists={output_status['exists']}, is_dir={output_status['is_dir']}, " + f"sv_count={output_status['sv_count']})" + ) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "verilog", + "error": "invalid_verilog_output", + "config": config_name, + "output_dir": build_dir, + "output_status": output_status, + }, + trace_id=origin_tid, + ) + return + await check_result( ctx, result.returncode, @@ -107,5 +157,5 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # Continue routing to build if from run workflow if input_data.get("from_run_workflow"): await ctx.enqueue( - {"topic": "bebop.verilator.build", "data": {**input_data, "output_dir": build_dir, "vsrc_dir": build_dir, "task": "run"}} + {"topic": "bebop.verilator.run.build", "data": {**input_data, "output_dir": build_dir, "vsrc_dir": build_dir, "task": "run"}} ) diff --git a/bbdev/api/utils/bebop_verilator.py b/bbdev/api/utils/bebop_verilator.py new file mode 100644 index 000000000..d4726d27e --- /dev/null +++ b/bbdev/api/utils/bebop_verilator.py @@ -0,0 +1,30 @@ +import json +import os + + +def build_marker_path(bebop_dir: str) -> str: + return os.path.join(bebop_dir, "target", "debug", ".bbdev-verilator-build.json") + + +def write_build_marker(bebop_dir: str, config: str, vsrc_dir: str, binary: str): + marker = build_marker_path(bebop_dir) + os.makedirs(os.path.dirname(marker), exist_ok=True) + data = { + "config": config, + "vsrc_dir": os.path.abspath(vsrc_dir), + "binary": os.path.abspath(binary), + } + tmp = f"{marker}.tmp" + with open(tmp, "w") as f: + json.dump(data, f, indent=2, sort_keys=True) + f.write("\n") + os.replace(tmp, marker) + + +def read_build_marker(bebop_dir: str) -> dict | None: + marker = build_marker_path(bebop_dir) + try: + with open(marker) as f: + return json.load(f) + except FileNotFoundError: + return None diff --git a/bbdev/api/utils/event_common.py b/bbdev/api/utils/event_common.py index 20081740e..102547aec 100644 --- a/bbdev/api/utils/event_common.py +++ b/bbdev/api/utils/event_common.py @@ -31,10 +31,7 @@ async def check_result(ctx, returncode, continue_run=False, extra_fields=None, t extra_fields = extra_fields or {} scope = trace_id or ctx.trace_id - if continue_run: - await ctx.state.set(scope, "processing", True) - return None, None - elif returncode != 0: + if returncode != 0: failure_result = { "status": 500, "body": { @@ -47,6 +44,9 @@ async def check_result(ctx, returncode, continue_run=False, extra_fields=None, t } await ctx.state.set(scope, "failure", failure_result) return None, failure_result + elif continue_run: + await ctx.state.set(scope, "processing", {"processing": True, **extra_fields}) + return None, None else: success_result = { "status": 200, diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index 73faaa3ca..815ce86c3 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -3,6 +3,14 @@ def get_buckyball_path(): + root = os.environ.get("BUCKYBALL_ROOT") + if root: + if not os.path.isabs(root): + raise ValueError("BUCKYBALL_ROOT must be an absolute path") + if not os.path.isdir(root): + raise ValueError(f"BUCKYBALL_ROOT does not exist: {root}") + return root + current_dir = os.path.dirname(__file__) # bbdev/api/utils -> bbdev/api -> bbdev -> buckyball return os.path.dirname(os.path.dirname(os.path.dirname(current_dir))) diff --git a/bbdev/api/utils/port.py b/bbdev/api/utils/port.py index f61b8c2d2..05afc3fbe 100644 --- a/bbdev/api/utils/port.py +++ b/bbdev/api/utils/port.py @@ -1,12 +1,14 @@ import socket +HOST = "127.0.0.1" + def find_available_port(start_port: int = 5000, end_port: int = 5500) -> int: """Find an available port in the specified range""" for port in range(start_port, end_port + 1): try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind(("localhost", port)) + sock.bind((HOST, port)) return port except OSError: continue @@ -14,17 +16,33 @@ def find_available_port(start_port: int = 5000, end_port: int = 5500) -> int: raise RuntimeError(f"No available port found in range {start_port}-{end_port}") -def reserve_port(start_port: int = 5000, end_port: int = 5500) -> tuple[int, socket.socket]: +def _port_order(start_port: int, end_port: int, preferred_port: int | None = None): + if preferred_port is None: + yield from range(start_port, end_port + 1) + return + + if preferred_port < start_port or preferred_port > end_port: + raise ValueError("preferred_port must be within the requested range") + + yield from range(preferred_port, end_port + 1) + yield from range(start_port, preferred_port) + + +def reserve_port( + start_port: int = 5000, + end_port: int = 5500, + preferred_port: int | None = None, +) -> tuple[int, socket.socket]: """Reserve a port by binding and holding the socket open. The caller must close the returned socket once the engine has taken over the port. This prevents TOCTOU races when multiple bbdev instances start concurrently. """ - for port in range(start_port, end_port + 1): + for port in _port_order(start_port, end_port, preferred_port): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(("localhost", port)) + sock.bind((HOST, port)) return port, sock except OSError: sock.close() diff --git a/bbdev/api/utils/process.py b/bbdev/api/utils/process.py new file mode 100644 index 000000000..0105f3441 --- /dev/null +++ b/bbdev/api/utils/process.py @@ -0,0 +1,236 @@ +import os +import signal +import shutil +import subprocess +import time + + +def _stat(pid: int): + try: + with open(f"/proc/{pid}/stat") as f: + return f.read().split() + except (FileNotFoundError, PermissionError): + return None + + +def _exists(pid: int) -> bool: + stat = _stat(pid) + return stat is not None and stat[2] != "Z" + + +def _cmdline(pid: int) -> list[str]: + try: + with open(f"/proc/{pid}/cmdline", "rb") as f: + raw = f.read() + except (FileNotFoundError, PermissionError): + return [] + return [part.decode(errors="replace") for part in raw.split(b"\0") if part] + + +def _cwd(pid: int) -> str | None: + try: + return os.readlink(f"/proc/{pid}/cwd") + except (FileNotFoundError, PermissionError, OSError): + return None + + +def _env(pid: int) -> dict[str, str]: + try: + with open(f"/proc/{pid}/environ", "rb") as f: + raw = f.read() + except (FileNotFoundError, PermissionError): + return {} + + env = {} + for item in raw.split(b"\0"): + if not item or b"=" not in item: + continue + key, value = item.split(b"=", 1) + env[key.decode(errors="replace")] = value.decode(errors="replace") + return env + + +def _own(pid: int) -> bool: + try: + return os.stat(f"/proc/{pid}").st_uid == os.getuid() + except (FileNotFoundError, PermissionError): + return False + + +def _children(pid: int) -> list[int]: + children = [] + try: + entries = os.listdir("/proc") + except FileNotFoundError: + return children + + for entry in entries: + if not entry.isdigit(): + continue + child = int(entry) + stat = _stat(child) + if not stat: + continue + try: + if int(stat[3]) == pid: + children.append(child) + children.extend(_children(child)) + except (IndexError, ValueError): + continue + return children + + +def _wait_gone(pids: set[int], timeout: float) -> set[int]: + deadline = time.time() + timeout + alive = set(pids) + while alive and time.time() < deadline: + alive = {pid for pid in alive if _exists(pid)} + if alive: + time.sleep(0.1) + return {pid for pid in alive if _exists(pid)} + + +def _signal(pids: set[int], sig: signal.Signals): + for pid in sorted(pids, reverse=True): + if pid == os.getpid(): + continue + try: + os.kill(pid, sig) + except (ProcessLookupError, PermissionError, OSError): + pass + + +def kill_tree(pid: int, term_timeout: float = 5.0): + targets = set(_children(pid)) + targets.add(pid) + _signal(targets, signal.SIGTERM) + alive = _wait_gone(targets, term_timeout) + if alive: + _signal(alive, signal.SIGKILL) + _wait_gone(alive, 1.0) + + +def terminate_group(proc: subprocess.Popen, term_timeout: float = 5.0): + if proc.poll() is not None: + return + + try: + os.killpg(proc.pid, signal.SIGTERM) + except (ProcessLookupError, PermissionError, OSError): + kill_tree(proc.pid, term_timeout=term_timeout) + + try: + proc.wait(timeout=term_timeout) + return + except subprocess.TimeoutExpired: + pass + + try: + os.killpg(proc.pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError, OSError): + kill_tree(proc.pid, term_timeout=1.0) + + try: + proc.wait(timeout=1) + except subprocess.TimeoutExpired: + pass + + +def _is_workspace_server(pid: int, workflow_dir: str, worker_url: str | None = None) -> bool: + repo_dir = os.path.dirname(workflow_dir) + root_dir = os.path.dirname(repo_dir) + if not _own(pid): + return False + + cwd = _cwd(pid) + if cwd not in (workflow_dir, repo_dir, root_dir): + return False + + cmd = _cmdline(pid) + if not cmd: + return False + + exe = os.path.basename(cmd[0]) + is_bbdev = len(cmd) >= 2 and os.path.abspath(cmd[1]) == os.path.join(repo_dir, "bbdev") + if is_bbdev and worker_url is None: + return True + + if cwd != workflow_dir: + return False + if worker_url and _env(pid).get("III_URL") != worker_url: + return False + + text = " ".join(cmd) + return exe == "iii" or ("motia" in text and "dev" in cmd and "--dir" in cmd and "steps" in cmd) + + +def _workspace_pids(workflow_dir: str, worker_url: str | None = None) -> set[int]: + pids = set() + try: + entries = os.listdir("/proc") + except FileNotFoundError: + return pids + + for entry in entries: + if entry.isdigit() and _is_workspace_server(int(entry), workflow_dir, worker_url): + pids.add(int(entry)) + return pids + + +def _port_pids( + workflow_dir: str, + port: int | None, + worker_url: str | None = None, +) -> set[int]: + if port is None: + return set() + + result = subprocess.run( + ["lsof", "-ti", f":{port}"], + capture_output=True, + text=True, + check=False, + ) + pids = set() + for line in result.stdout.splitlines(): + try: + pid = int(line) + if _is_workspace_server(pid, workflow_dir, worker_url): + pids.add(pid) + except ValueError: + pass + return pids + + +def stop_workspace_servers( + workflow_dir: str, + port: int | None = None, + worker_url: str | None = None, +) -> int: + pids = _workspace_pids(workflow_dir, worker_url=worker_url) + pids.update(_port_pids(workflow_dir, port, worker_url=worker_url)) + if not pids: + return 0 + + targets = set(pids) + for pid in list(pids): + targets.update(_children(pid)) + + _signal(targets, signal.SIGTERM) + alive = _wait_gone(targets, 5.0) + if alive: + _signal(alive, signal.SIGKILL) + _wait_gone(alive, 1.0) + return len(targets) + + +def clean_python_caches(workflow_dir: str): + for rel in ("steps", "utils"): + root = os.path.join(workflow_dir, rel) + if not os.path.isdir(root): + continue + for dirpath, dirnames, _ in os.walk(root): + if "__pycache__" not in dirnames: + continue + shutil.rmtree(os.path.join(dirpath, "__pycache__"), ignore_errors=True) + dirnames.remove("__pycache__") diff --git a/bbdev/api/utils/stream_run.py b/bbdev/api/utils/stream_run.py index ed0332598..f5ddd54ee 100644 --- a/bbdev/api/utils/stream_run.py +++ b/bbdev/api/utils/stream_run.py @@ -155,7 +155,7 @@ def log_stdout(line): logger.info(f"[{stdout_prefix}] {line}") else: # Non-verbose mode: print directly, use green for STDOUT - print(f"\033[32m[{stdout_prefix}]\033[0m {line}") + print(f"\033[32m[{stdout_prefix}]\033[0m {line}", flush=True) def log_stderr(line): if verbose: @@ -163,7 +163,7 @@ def log_stderr(line): logger.info(f"[{stderr_prefix}] {line}") else: # Non-verbose mode: print directly, use red for STDERR - print(f"\033[31m[{stderr_prefix}]\033[0m {line}") + print(f"\033[31m[{stderr_prefix}]\033[0m {line}", flush=True) return stream_run( cmd=cmd, diff --git a/bbdev/bbdev b/bbdev/bbdev index 2f7d2d222..ba918142b 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -4,7 +4,6 @@ import sys import argparse import subprocess import os -import signal import shlex import json import time @@ -13,13 +12,16 @@ import threading import requests sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "api")) -from utils.port import reserve_port +from utils.port import HOST, reserve_port +from utils.process import clean_python_caches, stop_workspace_servers, terminate_group workflow_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "api") NO_PROXY = {"http": None, "https": None, "all": None} WORKER_PORT_START = 49200 WORKER_PORT_END = 49500 +HTTP_PORT_START = 5100 +HTTP_PORT_END = 5500 # Subcommand definitions: {name: {operation: help_text, ...}} # Operations with string values accept sub-arguments; True means boolean-only (no sub-args). @@ -88,7 +90,7 @@ WORKFLOW_COMMANDS = { ALL_COMMANDS = ["start", "stop"] + list(WORKFLOW_COMMANDS) -def make_iii_config(port: int, worker_port: int, quiet: bool = False) -> str: +def make_iii_config(port: int, worker_port: int, quiet: bool = False, exec_module: bool = True) -> str: """Create a temporary config.yaml with unique HTTP and Worker ports, return the path.""" base_config = os.path.join(workflow_dir, "config.yaml") with open(base_config, "r") as f: @@ -97,11 +99,17 @@ def make_iii_config(port: int, worker_port: int, quiet: bool = False) -> str: # then RestApiModule HTTP port (default 3111). content = content.replace("port: 49134", f"port: {worker_port}") content = content.replace("port: 3111", f"port: {port}", 1) + content = content.replace("host: 0.0.0.0", f"host: {HOST}") if quiet: content = content.replace( "exporter: memory", "exporter: memory\n level: warn\n logs_console_output: false", ) + if not exec_module: + marker = "\n # ── Exec Module (Python) ───────────────────────────────────────────────" + if marker not in content: + raise RuntimeError("failed to remove ExecModule from iii config") + content = content.split(marker, 1)[0].rstrip() + "\n" tmp = tempfile.NamedTemporaryFile( mode="w", suffix=".yaml", prefix="iii-config-", dir=workflow_dir, delete=False ) @@ -110,6 +118,12 @@ def make_iii_config(port: int, worker_port: int, quiet: bool = False) -> str: return tmp.name +def user_preferred_port(start: int, end: int) -> int: + size = end - start + 1 + offset = (os.getuid() * 37) % size + return start + offset + + def cleanup_config(config_path: str): """Remove temp config file, ignoring if already deleted.""" try: @@ -118,42 +132,6 @@ def cleanup_config(config_path: str): pass -def _kill_proc_tree(pid: int): - """Kill a process and all its descendants via /proc. - Kills children first (bottom-up) so they are still in the process tree - and discoverable before their parent dies.""" - def _get_children(parent_pid): - children = [] - try: - for entry in os.listdir("/proc"): - if not entry.isdigit(): - continue - try: - with open(f"/proc/{entry}/stat") as f: - stat = f.read().split() - if int(stat[3]) == parent_pid: - child_pid = int(entry) - children.append(child_pid) - children.extend(_get_children(child_pid)) - except (FileNotFoundError, IndexError, ValueError, PermissionError): - continue - except FileNotFoundError: - pass - return children - - # children first, then parent — so children are still discoverable in /proc - children = _get_children(pid) - for p in reversed(children): - try: - os.kill(p, signal.SIGTERM) - except (ProcessLookupError, OSError): - pass - try: - os.kill(pid, signal.SIGTERM) - except (ProcessLookupError, OSError): - pass - - def parse_args(argv: list[str]) -> argparse.Namespace: parser = argparse.ArgumentParser( prog="bbdev", @@ -328,33 +306,47 @@ def run_server_mode(args, cmd_info): worker_port, worker_sock = reserve_port(start_port=WORKER_PORT_START, end_port=WORKER_PORT_END) worker_sock.close() config_path = make_iii_config(args.port, worker_port) + worker_url = f"ws://{HOST}:{worker_port}" print(f"Using worker port {worker_port}") else: config_path = os.path.join(workflow_dir, "config.yaml") + worker_url = f"ws://{HOST}:49134" + proc = None try: - subprocess.run(["iii", "-c", config_path], cwd=workflow_dir, check=True) + env = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"} + if worker_url: + env["III_URL"] = worker_url + proc = subprocess.Popen( + ["iii", "-c", config_path], + cwd=workflow_dir, + env=env, + start_new_session=True, + ) + rc = proc.wait() + if rc != 0: + sys.exit(rc) finally: + if proc: + terminate_group(proc) + stop_workspace_servers(workflow_dir, worker_url=worker_url) if args.port: cleanup_config(config_path) elif command == "stop": if cmd_info["operation"] == "all": - print("Stopping all servers") - subprocess.run( - "kill -9 $(ps aux | grep '[i]ii' | awk '{print $2}')", - shell=True, check=False, text=True, - ) + count = stop_workspace_servers(workflow_dir) + print(f"Stopped {count} process(es) for this workspace") else: - print(f"Stopping server on port {args.port}") - subprocess.run( - f"kill -TERM $(lsof -t -i :{args.port})", - shell=True, check=False, text=True, - ) + count = stop_workspace_servers(workflow_dir, port=args.port) + if args.port: + print(f"Stopped {count} process(es) on port {args.port}") + else: + print(f"Stopped {count} process(es) for this workspace") elif command in WORKFLOW_COMMANDS: api_path = f"/{command.replace('-', '/')}/{cmd_info['operation']}" resp = requests.post( - f"http://localhost:{args.port}{api_path}", + f"http://{HOST}:{args.port}{api_path}", json=cmd_info["args"], timeout=30, proxies=NO_PROXY, @@ -369,7 +361,7 @@ def run_server_mode(args, cmd_info): def _submit_and_poll(port: int, api_path: str, cmd_info: dict): """Submit a task to iii and poll for its result. Returns exit code (0=ok, 1=fail).""" - base_url = f"http://localhost:{port}" + base_url = f"http://{HOST}:{port}" print(f"Executing {cmd_info['command']} {cmd_info['operation']}...") try: submit_resp = requests.post( @@ -396,6 +388,7 @@ def _submit_and_poll(port: int, api_path: str, cmd_info: dict): state_file = os.path.join(workflow_dir, "data", "state_store.db", f"{trace_id}.bin") poll_timeout = 24 * 60 * 60 # 24 hours max task_result = None + legacy_processing_count = 0 for _ in range(poll_timeout // 2): state_data = read_state_file(state_file) if state_data: @@ -405,12 +398,19 @@ def _submit_and_poll(port: int, api_path: str, cmd_info: dict): elif "failure" in state_data: task_result = state_data["failure"].get("body", state_data["failure"]) break + elif state_data.get("processing") is True: + legacy_processing_count += 1 + if legacy_processing_count >= 30: + print("Error: Task is stuck in legacy processing state; worker did not advance the queue") + return 1 + else: + legacy_processing_count = 0 time.sleep(2) else: print("Error: Task polling timed out after 24 hours") return 1 - print(f"\nTask completed on http://localhost:{port}") + print(f"\nTask completed on http://{HOST}:{port}") if task_result and not task_result.get("success", False): print("Error: Task failed") return 1 @@ -421,9 +421,16 @@ def run_script_mode(args, cmd_info): """Handle script mode: start iii, run one task, then shut down.""" command = cmd_info["command"] - if command in ("start", "stop"): + if command == "start": print(f"'{command}' does nothing in script mode") return + if command == "stop": + count = stop_workspace_servers(workflow_dir, port=args.port) + if args.port: + print(f"Stopped {count} process(es) on port {args.port}") + else: + print(f"Stopped {count} process(es) for this workspace") + return if command not in WORKFLOW_COMMANDS: print(f"Unknown command: {command}") @@ -438,8 +445,16 @@ def run_script_mode(args, cmd_info): available_port = args.port http_sock = None else: - available_port, http_sock = reserve_port(start_port=5100, end_port=5500) - worker_port, worker_sock = reserve_port(start_port=WORKER_PORT_START, end_port=WORKER_PORT_END) + available_port, http_sock = reserve_port( + start_port=HTTP_PORT_START, + end_port=HTTP_PORT_END, + preferred_port=user_preferred_port(HTTP_PORT_START, HTTP_PORT_END), + ) + worker_port, worker_sock = reserve_port( + start_port=WORKER_PORT_START, + end_port=WORKER_PORT_END, + preferred_port=user_preferred_port(WORKER_PORT_START, WORKER_PORT_END), + ) print(f"Starting server on port {available_port}...") # Clean stale state files from previous runs @@ -451,8 +466,14 @@ def run_script_mode(args, cmd_info): os.unlink(os.path.join(state_dir, f)) except OSError: pass + clean_python_caches(workflow_dir) - config_path = make_iii_config(available_port, worker_port, quiet=True) + config_path = make_iii_config( + available_port, + worker_port, + quiet=True, + exec_module=False, + ) iii_output_active = threading.Event() iii_output_active.set() @@ -462,7 +483,12 @@ def run_script_mode(args, cmd_info): worker_sock.close() # Set III_URL so motia SDK connects to the correct worker port - iii_env = {**os.environ, "III_URL": f"ws://localhost:{worker_port}"} + iii_env = { + **os.environ, + "III_URL": f"ws://{HOST}:{worker_port}", + "PYTHONDONTWRITEBYTECODE": "1", + "BUCKYBALL_ROOT": os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + } proc = subprocess.Popen( ["iii", "-c", config_path], cwd=workflow_dir, @@ -470,7 +496,9 @@ def run_script_mode(args, cmd_info): stderr=subprocess.STDOUT, text=True, env=iii_env, + start_new_session=True, ) + worker_proc = None output_thread = threading.Thread( target=forward_iii_output, args=(proc.stdout, iii_output_active, True), daemon=True @@ -478,7 +506,7 @@ def run_script_mode(args, cmd_info): output_thread.start() try: - base_url = f"http://localhost:{available_port}" + base_url = f"http://{HOST}:{available_port}" # Phase 1: wait for iii engine HTTP if not wait_for_http( @@ -489,6 +517,22 @@ def run_script_mode(args, cmd_info): ): sys.exit(1) + worker_proc = subprocess.Popen( + [".venv/bin/motia", "dev", "--dir", "steps"], + cwd=workflow_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + env=iii_env, + start_new_session=True, + ) + worker_output_thread = threading.Thread( + target=forward_iii_output, + args=(worker_proc.stdout, iii_output_active, True), + daemon=True, + ) + worker_output_thread.start() + # Phase 2: wait for route registration if not wait_for_http( f"{base_url}{api_path}", @@ -497,6 +541,7 @@ def run_script_mode(args, cmd_info): error_msg="Routes failed to register within timeout", ): sys.exit(1) + time.sleep(2) print(f"Server is ready on port {available_port}") rc = _submit_and_poll(available_port, api_path, cmd_info) @@ -504,11 +549,13 @@ def run_script_mode(args, cmd_info): sys.exit(rc) finally: - # Cleanup: kill iii and all its descendant processes, then remove temp config + # Cleanup: stop iii, its process group, and any workspace worker it spawned. iii_output_active.clear() time.sleep(0.5) - _kill_proc_tree(proc.pid) - proc.wait() + if worker_proc: + terminate_group(worker_proc) + terminate_group(proc) + stop_workspace_servers(workflow_dir, worker_url=iii_env["III_URL"]) cleanup_config(config_path) From d432115a56a7bddb1ba8191629519df44c97e295 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 21 Jun 2026 19:26:11 +0800 Subject: [PATCH 079/196] feat: add new TLB test cases to workloads for baremetal singlecore --- bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml | 7 +++++++ .../api/steps/bebop/verilator/scripts/workloads-elf.toml | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml index 5d009a0a6..ab86585c1 100644 --- a/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml @@ -21,4 +21,11 @@ tests = [ "ctest_gemmini_os_risc_shift_test_singlecore-baremetal", "ctest_gemmini_ws_cisc_conv_test_singlecore-baremetal", "ctest_gemmini_ws_risc_basic_test_singlecore-baremetal", + "ctest_tlb_bare_mode_test_singlecore-baremetal", + "ctest_tlb_sv39_identity_test_singlecore-baremetal", + "ctest_tlb_sfence_flush_test_singlecore-baremetal", + "ctest_tlb_repeated_sfence_test_singlecore-baremetal", + "ctest_tlb_sv39_nonidentity_test_singlecore-baremetal", + "ctest_tlb_sv39_4k_alias_test_singlecore-baremetal", + "ctest_tlb_sfence_remap_test_singlecore-baremetal", ] diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml index 863194b34..8d514db76 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml @@ -39,7 +39,13 @@ tests = [ "ctest_mmio_minimal_test_singlecore-baremetal", "ctest_mxfp2int_test_singlecore-baremetal", - "ctest_tlb_test_singlecore-baremetal", + "ctest_tlb_bare_mode_test_singlecore-baremetal", + "ctest_tlb_sv39_identity_test_singlecore-baremetal", + "ctest_tlb_sfence_flush_test_singlecore-baremetal", + "ctest_tlb_repeated_sfence_test_singlecore-baremetal", + "ctest_tlb_sv39_nonidentity_test_singlecore-baremetal", + "ctest_tlb_sv39_4k_alias_test_singlecore-baremetal", + "ctest_tlb_sfence_remap_test_singlecore-baremetal", "ctest_mvin_mvout_stride_test_singlecore-baremetal", "ctest_transpose_matmul_singlecore-baremetal", From 2c217dd87a77026cfdd048ea7d8825e1e5b4a02a Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 21 Jun 2026 23:59:39 +0800 Subject: [PATCH 080/196] feat: update fw_payload_name function to include model parameter for dynamic naming --- bbdev/api/steps/kernel/01_build_event.step.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index 894e40eef..3b5050253 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -87,12 +87,15 @@ def kernel_build_dir(bbdir: str, hart_params: dict, model: str = "") -> str: return os.path.join(bbdir, "bb-tests", "build", f"kernel-v{visible}-t{total}{model_suffix}") -def fw_payload_name(hart_params: dict) -> str: +def fw_payload_name(hart_params: dict, model: str = "") -> str: visible = hart_params["visible"] total = hart_params["total"] - if visible == 64 and total == 64: - return "fw_payload" - return f"fw_payload-v{visible}-t{total}" + name = "fw_payload" + if visible != 64 or total != 64: + name = f"{name}-v{visible}-t{total}" + if model: + name = f"{name}-{model}" + return name async def handler(input_data: dict, ctx: FlowContext) -> None: @@ -145,7 +148,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return # Convert fw_payload.bin to hex for P2E memory backdoor - payload_name = fw_payload_name(hart_params) + payload_name = fw_payload_name(hart_params, model) fw_payload_bin = os.path.join(output_dir, f"{payload_name}.bin") fw_payload_hex = os.path.join(output_dir, f"{payload_name}.hex") From 728e131f4a6b7d5a84c66374cd803dd44d079d2f Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 23 Jun 2026 21:03:41 +0800 Subject: [PATCH 081/196] feat: add new matrix multiplication test case for baremetal singlecore --- .../steps/bebop/verilator/scripts/workloads-elf.toml | 1 + bbdev/api/steps/compiler/01_build_event.step.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml index 8d514db76..81580a4ce 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml @@ -51,6 +51,7 @@ tests = [ "ctest_transpose_matmul_singlecore-baremetal", "optest_buckyball_matmul_16x64_64x16_singlecore-baremetal", "optest_buckyball_matmul_64x16_16x64_singlecore-baremetal", + "optest_tile_matmul_1x84_84x10_singlecore-baremetal", "optest_linalg_conv2d_nhwc_hwcf_singlecore-baremetal", "optest_linalg_conv2d_nhwc_fhwc_singlecore-baremetal", "optest_linalg_conv2d_nchw_fchw_singlecore-baremetal", diff --git a/bbdev/api/steps/compiler/01_build_event.step.py b/bbdev/api/steps/compiler/01_build_event.step.py index cae181cca..a14edc8f7 100644 --- a/bbdev/api/steps/compiler/01_build_event.step.py +++ b/bbdev/api/steps/compiler/01_build_event.step.py @@ -36,6 +36,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stdout_prefix="compiler build", stderr_prefix="compiler build", ) + command = ( + f"cd {bbdir}/compiler/thirdparty/buddy-mlir/build " + "&& cmake --build . --target python-package-buddy" + ) + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + stdout_prefix="compiler build", + stderr_prefix="compiler build", + ) command = f"cd {bbdir}/compiler/thirdparty/buddy-mlir/build && ninja -j{os.cpu_count()}" result = stream_run_logger( cmd=command, From a9b492e698ac3e7f0f8b225b4a97327a79ca483f Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 26 Jun 2026 23:43:16 +0800 Subject: [PATCH 082/196] refactor: update to latest bebop API --- .../api/steps/bebop/bemu/00_build_api.step.py | 15 ---- .../steps/bebop/bemu/00_build_event.step.py | 53 ------------ .../api/steps/bebop/bemu/02_sim_event.step.py | 4 +- .../bebop/p2e/02_buildbitstream_api.step.py | 3 +- .../bebop/p2e/02_buildbitstream_event.step.py | 80 ++++++++++--------- .../bebop/p2e/03_runworkload_event.step.py | 22 ++--- .../bebop/verilator/04_sim_event.step.py | 14 ++-- bbdev/api/tests/test_all_api_serial.py | 1 - bbdev/api/tests/test_api_bebop_bemu_build.py | 3 - bbdev/bbdev | 11 ++- 10 files changed, 68 insertions(+), 138 deletions(-) delete mode 100644 bbdev/api/steps/bebop/bemu/00_build_api.step.py delete mode 100644 bbdev/api/steps/bebop/bemu/00_build_event.step.py delete mode 100644 bbdev/api/tests/test_api_bebop_bemu_build.py diff --git a/bbdev/api/steps/bebop/bemu/00_build_api.step.py b/bbdev/api/steps/bebop/bemu/00_build_api.step.py deleted file mode 100644 index e84fd6fa5..000000000 --- a/bbdev/api/steps/bebop/bemu/00_build_api.step.py +++ /dev/null @@ -1,15 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - -config = { - "name": "bebop-bemu-build-api", - "description": "Build bebop bemu binary", - "flows": ["bebop"], - "triggers": [api("POST", "/bebop/bemu/build")], - "enqueues": ["bebop.bemu.build"], -} - - -async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = request.body or {} - await ctx.enqueue({"topic": "bebop.bemu.build", "data": {**body, "_trace_id": ctx.trace_id}}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/bemu/00_build_event.step.py b/bbdev/api/steps/bebop/bemu/00_build_event.step.py deleted file mode 100644 index 6a4ce95b9..000000000 --- a/bbdev/api/steps/bebop/bemu/00_build_event.step.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -bebop bemu build event handler - -Builds bebop with bemu feature -""" -import os -import sys - -from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from utils.event_common import check_result, get_origin_trace_id - -config = { - "name": "bebop-bemu-build", - "description": "Build bebop bemu binary", - "flows": ["bebop"], - "triggers": [queue("bebop.bemu.build")], - "enqueues": [], -} - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - bbdir = get_buckyball_path() - bebop_dir = f"{bbdir}/bebop" - - build_cmd = "cargo build --features bemu" - ctx.logger.info("Building bebop bemu ...") - build_result = stream_run_logger( - cmd=build_cmd, - logger=ctx.logger, - cwd=bebop_dir, - stdout_prefix="bebop bemu build", - stderr_prefix="bebop bemu build", - ) - - bebop_bin = f"{bebop_dir}/target/debug/bebop" - await check_result( - ctx, - build_result.returncode, - continue_run=False, - extra_fields={ - "task": "build", - "binary": bebop_bin, - }, - trace_id=origin_tid, - ) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 68c4ee08d..5ece824ed 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -4,7 +4,7 @@ Runs bebop bemu (Spike-based) emulator: 1. Resolve binary (ELF) path 2. Build bebop with bemu feature (cargo build) - 3. Run bebop bemu with the resolved ELF + 3. Run bebop run bemu with the resolved ELF """ import os import sys @@ -56,7 +56,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── Run bebop bemu ──────────────────────────────────────────────────── pk_flag = " --pk" if input_data.get("pk") else "" run_cmd = ( - f"cargo run --manifest-path \"{bebop_dir}/Cargo.toml\" --features bemu -- bemu " + f"cargo run --manifest-path \"{bebop_dir}/Cargo.toml\" --features bemu -- run bemu " f"--elf=\"{binary_path}\" " f"--log-dir=\"{log_dir}\"" f"{pk_flag}" diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py index abaedb90f..75cdfc4db 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py @@ -4,7 +4,7 @@ config = { "name": "bebop-p2e-buildbitstream-api", - "description": "Build P2E bitstream via bebop CLI", + "description": "Build Bebop P2E runtime case", "flows": ["bebop"], "triggers": [api("POST", "/bebop/p2e/buildbitstream")], "enqueues": ["bebop.p2e.buildbitstream"], @@ -22,7 +22,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "config": config_name, "vsrc_dir": vsrc_dir, "output_dir": body.get("output_dir"), - "log_dir": body.get("log_dir"), } await ctx.enqueue({ "topic": "bebop.p2e.buildbitstream", diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index 0aec1ae92..50affc62a 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -1,10 +1,10 @@ """ bebop p2e buildbitstream event handler -Builds the FPGA bitstream via bebop CLI: - 1. Resolve verilog source directory (VSRC_PATH) from config - 2. Build bebop with p2e feature (vvacDir generated under build_dir via OUT_PATH) - 3. Run bebop p2e --buildbitstream with build_dir (all outputs land in build_dir) +Builds the P2E VVAC runtime case via bebop CLI: + 1. Resolve Verilog source directory (VSRC_PATH) from config + 2. Run bebop build p2e with rtl_dir and out_dir + 3. Validate runtime artifacts used by bebop run p2e """ import os import sys @@ -22,7 +22,7 @@ config = { "name": "bebop-p2e-buildbitstream", - "description": "Build P2E bitstream via bebop CLI", + "description": "Build Bebop P2E runtime case", "flows": ["bebop"], "triggers": [queue("bebop.p2e.buildbitstream")], "enqueues": [], @@ -46,17 +46,22 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - build_dir = input_data.get("build_dir") or f"{bebop_dir}/build/{config_name}-{timestamp}" + build_dir = ( + input_data.get("build_dir") + or input_data.get("build-dir") + or input_data.get("output_dir") + or input_data.get("output-dir") + or f"{bebop_dir}/build/{config_name}-{timestamp}" + ) os.makedirs(build_dir, exist_ok=True) - # ── Build bebop with p2e feature ────────────────────────────────────── build_cmd = ( f"nix develop --ignore-environment --keep ALL_PROXY -c " - f"cargo build --features p2e " - f"--config=\"env.VSRC_PATH='{vsrc_dir}'\" " - f"--config=\"env.OUT_PATH='{build_dir}'\"" + f"cargo run --features p2e -- build p2e " + f"--rtl-dir=\"{vsrc_dir}\" " + f"--out-dir=\"{build_dir}\"" ) - ctx.logger.info("Building bebop p2e ...") + ctx.logger.info("Building bebop p2e runtime case ...") build_result = stream_run_logger( cmd=build_cmd, logger=ctx.logger, @@ -64,42 +69,41 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stdout_prefix="bebop p2e build", stderr_prefix="bebop p2e build", ) - if build_result.returncode != 0: - await check_result( - ctx, build_result.returncode, continue_run=False, - extra_fields={"task": "buildbitstream", "stage": "build"}, - trace_id=origin_tid, - ) - return - # ── Run bebop p2e --buildbitstream ──────────────────────────────────── - run_cmd = ( - f"nix develop --ignore-environment --keep HOME --keep ALL_PROXY -c " - f"cargo run --features p2e " - f"-- p2e " - f"--buildbitstream " - f"--build-dir=\"{build_dir}\"" - ) - ctx.logger.info(f"Running bebop p2e buildbitstream: {run_cmd}") - run_result = stream_run_logger( - cmd=run_cmd, - logger=ctx.logger, - cwd=bebop_dir, - stdout_prefix="bebop p2e build-bitstream", - stderr_prefix="bebop p2e build-bitstream", - ) + rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") + libvctb_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so") + if build_result.returncode == 0: + missing = [path for path in (rtcfg_path, libvctb_path) if not os.path.exists(path)] + if missing: + ctx.logger.error(f"P2E runtime artifacts missing: {missing}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "build", + "config": config_name, + "vsrc_dir": vsrc_dir, + "build_dir": build_dir, + "missing": missing, + "error": "runtime_artifact_not_found", + "timestamp": timestamp, + }, + trace_id=origin_tid, + ) + return - bitstream_path = os.path.join(build_dir, "fpgaCompDir", "bitstream.bit") await check_result( ctx, - run_result.returncode, + build_result.returncode, continue_run=False, extra_fields={ - "task": "buildbitstream", + "task": "build", "config": config_name, "vsrc_dir": vsrc_dir, "build_dir": build_dir, - "bitstream": bitstream_path, + "rtcfg": rtcfg_path, + "libvCtb": libvctb_path, "timestamp": timestamp, }, trace_id=origin_tid, diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index 640921693..23a0e146f 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -4,7 +4,7 @@ Loads a kernel image into FPGA and runs the workload via bebop CLI: 1. Resolve image name to .hex file under bb-tests/output/ 2. Validate bitstream .bit file path - 3. Run bebop p2e --runworkload --image --bitstream [--multi-fpga] [--wave] [--wave-start ] --log-dir + 3. Run bebop run p2e --image --bitstream [--multi-fpga] [--wave] [--wave-start ] """ import glob import os @@ -126,17 +126,21 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: log_dir = f"{bebop_dir}/log/p2e-runworkload-{timestamp}" os.makedirs(log_dir, exist_ok=True) - # ── Run bebop p2e --runworkload ─────────────────────────────────────── + if input_data.get("log_dir") or input_data.get("log-dir"): + ctx.logger.warning( + "bebop run p2e no longer accepts a separate output/build directory; " + "using build_dir as --log-dir so VVAC runtime artifacts can be found" + ) + + # ── Run bebop run p2e ───────────────────────────────────────────────── run_cmd = ( f"nix develop --ignore-environment --keep HOME --keep ALL_PROXY -c " f"cargo run --features p2e " f"--config=\"env.OUT_PATH='{build_dir}'\" " - f"-- p2e " - f"--runworkload " + f"-- run p2e " f"--image=\"{image_path}\" " f"--bitstream=\"{bitstream}\" " - f"--build-dir=\"{build_dir}\" " - f"--log-dir=\"{log_dir}\"" + f"--log-dir=\"{build_dir}\"" ) if multi_fpga: run_cmd += " --multi-fpga" @@ -153,7 +157,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="bebop p2e runworkload", ) - uart_log = os.path.join(log_dir, "uart.log") await check_result( ctx, run_result.returncode, @@ -163,8 +166,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "image": image_path, "bitstream": bitstream, "build_dir": build_dir, - "log_dir": log_dir, - "uart_log": uart_log, + "log_dir": build_dir, + "requested_log_dir": log_dir, + "uart_log": os.path.join(build_dir, "uart.log"), "timestamp": timestamp, }, trace_id=origin_tid, diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index 108714dc9..30951d852 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -4,7 +4,7 @@ Runs bebop verilator simulation: 1. Resolve binary path 2. Resolve verilog source directory (VSRC_PATH) - 3. Run the built bebop binary with elf, log and fst directories + 3. Run the built bebop binary with the new run/verilator CLI """ import os import shlex @@ -121,19 +121,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: os.makedirs(log_dir, exist_ok=True) os.makedirs(fst_dir, exist_ok=True) - # ── Run bebop verilator ─────────────────────────────────────────────── - trace_flags = [] - for flag in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace"): - if input_data.get(flag, False): - trace_flags.append(f"--{flag}") - trace_args = (" " + " ".join(trace_flags)) if trace_flags else "" + # ── Run bebop run verilator ────────────────────────────────────────── + wave_arg = " --no-wave" if input_data.get("no-wave", False) or input_data.get("no_wave", False) else "" run_cmd = ( - f"{shlex.quote(bebop_bin)} verilator " + f"{shlex.quote(bebop_bin)} run verilator " f"--elf={shlex.quote(binary_path)} " f"--log-dir={shlex.quote(log_dir)} " f"--fst-dir={shlex.quote(fst_dir)}" - f"{trace_args}" + f"{wave_arg}" ) ctx.logger.info(f"Running bebop verilator: {run_cmd}") run_result = stream_run_logger( diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index 8cb368e33..f2dfda3fe 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -23,7 +23,6 @@ def main(): "test_api_firesim_buildbitstream.py", "test_api_firesim_infrasetup.py", "test_api_firesim_runworkload.py", - "test_api_bebop_bemu_build.py", "test_api_bebop_bemu_sim.py", "test_api_bebop_bemu_batch.py", "test_api_bebop_verilator_clean.py", diff --git a/bbdev/api/tests/test_api_bebop_bemu_build.py b/bbdev/api/tests/test_api_bebop_bemu_build.py deleted file mode 100644 index 862fb83a3..000000000 --- a/bbdev/api/tests/test_api_bebop_bemu_build.py +++ /dev/null @@ -1,3 +0,0 @@ -from _api_test_helper import run_bbdev_case - -run_bbdev_case("bbdev bebop-bemu --build") diff --git a/bbdev/bbdev b/bbdev/bbdev index ba918142b..72677739f 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -58,23 +58,22 @@ WORKFLOW_COMMANDS = { "build": "Build compiler.", }, "bebop-bemu": { - "build": "Build Bebop emulator.", - "sim": 'Run Bebop emulator simulation. Args: "--binary [--pk]"', + "sim": 'Run Bebop emulator simulation. Args: "--binary [--pk] [--log-dir ]"', "batch": 'Run Bebop bemu nextest batch regression (parallel). Args: "--test "', }, "bebop-verilator": { "clean": 'Clean Bebop Verilator build directory. Args: "[--config ] [--output-dir ]"', "verilog": 'Generate Bebop Verilog files. Args: "[--config ] [--output-dir ]"', "build": 'Build Bebop Verilator simulation executable. Args: "[--jobs ] [--config ] [--output-dir ]"', - "sim": 'Run Bebop Verilator simulation. Args: "--binary [--batch] [--config ] [--output-dir ]"', - "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--batch] [--jobs ] [--config ] [--output-dir ]"', + "sim": 'Run Bebop Verilator simulation. Args: "--binary [--batch] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', + "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--config --test "', }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', - "buildbitstream": "Build Bebop P2E bitstream.", - "runworkload": 'Run Bebop P2E workload. Args: "--image --bitstream [--multi-fpga] [--wave] [--wave-start ] [--config ] [--output-dir ] [--log-dir ]"', + "buildbitstream": 'Build Bebop P2E runtime case. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', + "runworkload": 'Run Bebop P2E workload. Args: "--image --bitstream [--multi-fpga] [--wave] [--wave-start ] [--build-dir ]"', "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--bitstream --build-dir --test "', }, "dc": { From 06afc92a75a86688801da403ebf3c5b535c40dd4 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 30 Jun 2026 17:45:20 +0800 Subject: [PATCH 083/196] feat: implement model trace cleanup and Perfetto generation in bebop bemu event handler --- .../api/steps/bebop/bemu/02_sim_event.step.py | 88 ++++++++++++++++++- bbdev/api/steps/kernel/01_build_event.step.py | 2 +- .../api/steps/workload/01_build_event.step.py | 7 ++ 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 5ece824ed..287cd623d 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -4,11 +4,13 @@ Runs bebop bemu (Spike-based) emulator: 1. Resolve binary (ELF) path 2. Build bebop with bemu feature (cargo build) - 3. Run bebop run bemu with the resolved ELF + 3. Run bebop bemu with the resolved ELF """ import os +import shlex import sys from datetime import datetime +from pathlib import Path from motia import FlowContext, queue @@ -21,6 +23,10 @@ from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id +PERFETTO_TARGETS = { + "buddy-buckyball-lenet-run": "buddy-buckyball-lenet-perfetto", +} + config = { "name": "bebop-bemu-sim", "description": "Run bebop bemu emulator", @@ -30,6 +36,31 @@ } +def clean_model_trace(binary_dir: str) -> None: + trace_dir = Path(binary_dir) / "trace" + for subdir in ("cycle", "tensor"): + target_dir = trace_dir / subdir + if not target_dir.exists(): + continue + if not target_dir.is_dir(): + raise NotADirectoryError(f"trace path is not a directory: {target_dir}") + for path in target_dir.glob("trace-*.txt"): + if not path.is_file(): + raise FileNotFoundError(f"trace path is not a file: {path}") + path.unlink() + summary = target_dir / "summary.txt" + if summary.exists(): + if not summary.is_file(): + raise FileNotFoundError(f"trace summary path is not a file: {summary}") + summary.unlink() + + perfetto = trace_dir / "perfetto.json" + if perfetto.exists(): + if not perfetto.is_file(): + raise FileNotFoundError(f"perfetto path is not a file: {perfetto}") + perfetto.unlink() + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -48,6 +79,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return ctx.logger.info(f"binary_path: {binary_path}") binary_dir = os.path.dirname(binary_path) + perfetto_target = PERFETTO_TARGETS.get(binary_name) + if perfetto_target: + clean_model_trace(binary_dir) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") log_dir = input_data.get("log_dir") or f"{arch_dir}/log/{timestamp}-{binary_name}-bemu" @@ -56,7 +90,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── Run bebop bemu ──────────────────────────────────────────────────── pk_flag = " --pk" if input_data.get("pk") else "" run_cmd = ( - f"cargo run --manifest-path \"{bebop_dir}/Cargo.toml\" --features bemu -- run bemu " + f"cargo run --manifest-path \"{bebop_dir}/Cargo.toml\" --features bemu -- bemu " f"--elf=\"{binary_path}\" " f"--log-dir=\"{log_dir}\"" f"{pk_flag}" @@ -69,16 +103,64 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stdout_prefix="bebop bemu", stderr_prefix="bebop bemu", ) + if run_result.returncode != 0: + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "bemu", + "binary": binary_path, + "log_dir": log_dir, + "timestamp": timestamp, + }, + trace_id=origin_tid, + ) + return + + perfetto_path = None + if perfetto_target: + perfetto_cmd = ( + f"cmake --build {shlex.quote(f'{bbdir}/bb-tests/build')} " + f"--target {shlex.quote(perfetto_target)}" + ) + ctx.logger.info(f"Generating Perfetto trace: {perfetto_cmd}") + perfetto_result = stream_run_logger( + cmd=perfetto_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="perfetto", + stderr_prefix="perfetto", + ) + perfetto_path = f"{binary_dir}/trace/perfetto.json" + if perfetto_result.returncode != 0: + await check_result( + ctx, + perfetto_result.returncode, + continue_run=False, + extra_fields={ + "task": "perfetto", + "binary": binary_path, + "log_dir": log_dir, + "timestamp": timestamp, + "perfetto_target": perfetto_target, + "perfetto": perfetto_path, + }, + trace_id=origin_tid, + ) + return await check_result( ctx, - run_result.returncode, + 0, continue_run=False, extra_fields={ "task": "bemu", "binary": binary_path, "log_dir": log_dir, "timestamp": timestamp, + "perfetto_target": perfetto_target, + "perfetto": perfetto_path, }, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index 3b5050253..2bb287595 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -26,7 +26,7 @@ "mobilenet", "qwen3", "resnet", - "stablediffusion", + "stable-diffusion", "yolo", } diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index 14a85f1c8..4889456fe 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -33,6 +33,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "mobilenet": "buddy-buckyball-mobilenetv3-run", "resnet": "buddy-buckyball-resnet-run", "yolo": "buddy-buckyball-yolo26-run", + "bert": "buddy-buckyball-bert-run", + "qwen3": "buddy-buckyball-qwen3-run", + "gemma4": "buddy-buckyball-gemma4-run", + "deepseekr1": "buddy-buckyball-deepseekr1-run", + "llama2": "buddy-buckyball-llama2-run", + "stable-diffusion": "buddy-buckyball-stable-diffusion-run", + "whisper": "buddy-buckyball-whisper-run", } target = "" if model: From da8330689c7729b75378a4bff30030d2d03f2182 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 4 Jul 2026 01:03:34 +0800 Subject: [PATCH 084/196] feat: enhance bebop workflow commands and add UVM simulation support --- bbdev/api/steps/bebop/bemu/02_sim_api.step.py | 24 +++ .../api/steps/bebop/bemu/02_sim_event.step.py | 51 +++++- .../api/steps/bebop/bemu/03_batch_api.step.py | 21 ++- .../steps/bebop/bemu/03_batch_event.step.py | 37 +++- .../api/steps/bebop/p2e/00_clean_api.step.py | 6 + .../steps/bebop/p2e/00_clean_event.step.py | 10 +- .../steps/bebop/p2e/01_verilog_api.step.py | 7 +- .../bebop/p2e/02_buildbitstream_api.step.py | 7 +- .../bebop/p2e/02_buildbitstream_event.step.py | 10 +- .../bebop/p2e/scripts/workloads-elf.toml | 3 +- .../steps/bebop/p2e/scripts/workloads-pk.toml | 3 +- .../bebop/verilator/01_clean_api.step.py | 2 +- .../bebop/verilator/01_clean_event.step.py | 10 +- .../bebop/verilator/02_verilog_api.step.py | 2 +- .../bebop/verilator/03_build_api.step.py | 2 +- .../bebop/verilator/03_build_event.step.py | 2 +- .../steps/bebop/verilator/04_sim_api.step.py | 6 + .../bebop/verilator/04_sim_event.step.py | 10 +- .../steps/bebop/verilator/05_run_api.step.py | 2 +- .../bebop/verilator/06_batch_api.step.py | 2 +- .../bebop/verilator/06_batch_event.step.py | 20 ++- bbdev/api/steps/compiler/01_build_api.step.py | 50 ++++++ .../api/steps/compiler/01_build_event.step.py | 133 ++++++++++---- bbdev/api/steps/compiler/README.md | 8 +- .../mill/07_bebop_p2e_verilog_event.step.py | 10 +- bbdev/api/steps/uvm/01_build_api.step.py | 32 ++++ bbdev/api/steps/uvm/01_build_event.step.py | 37 ++++ bbdev/api/steps/uvm/02_run_api.step.py | 34 ++++ bbdev/api/steps/uvm/02_run_event.step.py | 77 +++++++++ bbdev/api/steps/uvm/uvm_common.py | 163 ++++++++++++++++++ .../api/steps/verilator/01_clean_api.step.py | 6 + .../steps/verilator/01_clean_event.step.py | 11 +- .../steps/verilator/02_verilog_api.step.py | 2 +- .../api/steps/verilator/03_build_api.step.py | 10 +- .../steps/verilator/03_build_event.step.py | 2 +- bbdev/api/steps/verilator/04_sim_api.step.py | 6 + .../api/steps/verilator/04_sim_event.step.py | 2 +- bbdev/api/steps/verilator/05_run_api.step.py | 21 ++- bbdev/api/steps/verilator/README.md | 2 +- bbdev/api/steps/workload/01_buidl_api.step.py | 15 +- .../api/steps/workload/01_build_event.step.py | 31 +++- bbdev/api/steps/workload/README.md | 15 +- bbdev/api/tests/test_api_bebop_bemu_batch.py | 2 +- bbdev/api/tests/test_api_bebop_bemu_sim.py | 4 +- bbdev/api/tests/test_api_compiler_build.py | 2 +- bbdev/api/tests/test_api_verilator_build.py | 2 +- bbdev/api/tests/test_api_verilator_clean.py | 2 +- bbdev/api/tests/test_api_verilator_sim.py | 2 +- bbdev/api/utils/bemu.py | 41 +++++ bbdev/bbdev | 123 ++++++++++--- 50 files changed, 961 insertions(+), 121 deletions(-) create mode 100644 bbdev/api/steps/uvm/01_build_api.step.py create mode 100644 bbdev/api/steps/uvm/01_build_event.step.py create mode 100644 bbdev/api/steps/uvm/02_run_api.step.py create mode 100644 bbdev/api/steps/uvm/02_run_event.step.py create mode 100644 bbdev/api/steps/uvm/uvm_common.py create mode 100644 bbdev/api/utils/bemu.py diff --git a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py index 3377bdd84..4c35bd264 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py @@ -1,5 +1,15 @@ +import os +import sys + from motia import ApiRequest, ApiResponse, FlowContext, api +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.bemu import validate_bemu_chip +from utils.path import get_buckyball_path + config = { "name": "bebop-bemu-sim-api", "description": "Run bebop bemu emulator", @@ -11,6 +21,20 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + chip = body.get("chip", "") + try: + validate_bemu_chip(chip, get_buckyball_path()) + except ValueError as e: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 400, + "message": str(e), + }, + ) + binary = body.get("binary", "") if not binary: return ApiResponse( diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 287cd623d..b36270ee1 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -22,6 +22,7 @@ from utils.stream_run import stream_run_logger from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id +from utils.bemu import bemu_feature PERFETTO_TARGETS = { "buddy-buckyball-lenet-run": "buddy-buckyball-lenet-perfetto", @@ -67,6 +68,26 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bebop_dir = f"{bbdir}/bebop" arch_dir = f"{bbdir}/arch" + chip = input_data.get("chip") + if not chip: + ctx.logger.error("Missing required parameter: chip must be specified") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_chip"}, + trace_id=origin_tid, + ) + return + try: + feature = bemu_feature(chip) + except ValueError as e: + ctx.logger.error(str(e)) + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_chip", "chip": chip}, + trace_id=origin_tid, + ) + return + binary_name = input_data.get("binary", "") binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) if binary_path is None: @@ -88,18 +109,30 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: os.makedirs(log_dir, exist_ok=True) # ── Run bebop bemu ──────────────────────────────────────────────────── - pk_flag = " --pk" if input_data.get("pk") else "" - run_cmd = ( - f"cargo run --manifest-path \"{bebop_dir}/Cargo.toml\" --features bemu -- bemu " - f"--elf=\"{binary_path}\" " - f"--log-dir=\"{log_dir}\"" - f"{pk_flag}" - ) + cargo_args = [ + "cargo", + "run", + "--manifest-path", + f"{bebop_dir}/Cargo.toml", + "--features", + feature, + "--", + "run", + "bemu", + "--elf", + binary_path, + "--log-dir", + log_dir, + ] + if input_data.get("pk"): + cargo_args.append("--pk") + inner_cmd = f"cd {shlex.quote(binary_dir)} && {shlex.join(cargo_args)}" + run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" ctx.logger.info(f"Running bebop bemu: {run_cmd}") run_result = stream_run_logger( cmd=run_cmd, logger=ctx.logger, - cwd=binary_dir, + cwd=bbdir, stdout_prefix="bebop bemu", stderr_prefix="bebop bemu", ) @@ -111,6 +144,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "task": "bemu", "binary": binary_path, + "chip": chip, "log_dir": log_dir, "timestamp": timestamp, }, @@ -157,6 +191,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "task": "bemu", "binary": binary_path, + "chip": chip, "log_dir": log_dir, "timestamp": timestamp, "perfetto_target": perfetto_target, diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py index f2f05d08a..f72cb2b2d 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -1,5 +1,15 @@ +import os +import sys + from motia import ApiRequest, ApiResponse, FlowContext, api +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.bemu import validate_bemu_chip +from utils.path import get_buckyball_path + config = { "name": "bebop-bemu-batch-api", "description": "Run bebop bemu nextest batch regression", @@ -12,6 +22,15 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + chip = body.get("chip") + try: + validate_bemu_chip(chip, get_buckyball_path()) + except ValueError as e: + return ApiResponse( + status=400, + body={"error": str(e)} + ) + test_type = body.get("test") if not test_type: return ApiResponse( @@ -27,6 +46,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: await ctx.enqueue({ "topic": "bebop.bemu.batch", - "data": {"test": test_type, "_trace_id": ctx.trace_id} + "data": {"chip": chip, "test": test_type, "_trace_id": ctx.trace_id} }) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index 7090cbacb..977397924 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -6,6 +6,7 @@ 2. Run cargo nextest with bemu-specific config """ import os +import shlex import sys from motia import FlowContext, queue @@ -17,6 +18,7 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id +from utils.bemu import bemu_feature config = { "name": "bebop-bemu-batch", @@ -34,6 +36,26 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: nextest_config = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/nextest.toml" elf_root = f"{bbdir}/bb-tests/output" + chip = input_data.get("chip") + if not chip: + ctx.logger.error("Missing required parameter: chip must be specified") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_chip"}, + trace_id=origin_tid, + ) + return + try: + feature = bemu_feature(chip) + except ValueError as e: + ctx.logger.error(str(e)) + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_chip", "chip": chip}, + trace_id=origin_tid, + ) + return + # ── Determine workload file based on test type ──────────────────────── test_type = input_data.get("test", "elf-tests") if test_type == "elf-tests": @@ -52,12 +74,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"Running {test_type} with workload config: {workload_toml}") # ── Build bebop bemu ────────────────────────────────────────────────── - build_cmd = "nix develop -c cargo build --features bemu --tests" + build_cmd = ( + f"nix develop -c cargo build --manifest-path {shlex.quote(f'{bebop_dir}/Cargo.toml')} " + f"--features {feature} --tests" + ) ctx.logger.info("Building bebop bemu (tests)...") build_result = stream_run_logger( cmd=build_cmd, logger=ctx.logger, - cwd=bebop_dir, + cwd=bbdir, stdout_prefix="bebop bemu build", stderr_prefix="bebop bemu build", ) @@ -78,8 +103,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "BEBOP_BB_TESTS_ROOT": elf_root, }) nextest_cmd = ( - f"nix develop -c cargo nextest run --features bemu --test test_bemu " - f"--config-file \"{nextest_config}\"" + f"nix develop -c cargo nextest run --manifest-path {shlex.quote(f'{bebop_dir}/Cargo.toml')} " + f"--features {feature} --test test_bemu " + f"--config-file {shlex.quote(nextest_config)}" ) ctx.logger.info(f"Running bebop bemu nextest: {nextest_cmd}") @@ -87,7 +113,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: run_result = stream_run_logger( cmd=nextest_cmd, logger=ctx.logger, - cwd=bebop_dir, + cwd=bbdir, stdout_prefix="bebop bemu batch", stderr_prefix="bebop bemu batch", env=env, @@ -100,6 +126,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "task": "batch", "backend": "bemu", + "chip": chip, "test_type": test_type, "nextest_config": nextest_config, "workload_toml": workload_toml, diff --git a/bbdev/api/steps/bebop/p2e/00_clean_api.step.py b/bbdev/api/steps/bebop/p2e/00_clean_api.step.py index 701bd2834..5f34d2a4d 100644 --- a/bbdev/api/steps/bebop/p2e/00_clean_api.step.py +++ b/bbdev/api/steps/bebop/p2e/00_clean_api.step.py @@ -11,5 +11,11 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} + config_name = body.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --config must be specified"}, + ) await ctx.enqueue({"topic": "bebop.p2e.clean", "data": {**body, "task": "clean", "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/00_clean_event.step.py b/bbdev/api/steps/bebop/p2e/00_clean_event.step.py index a37129eea..453942382 100644 --- a/bbdev/api/steps/bebop/p2e/00_clean_event.step.py +++ b/bbdev/api/steps/bebop/p2e/00_clean_event.step.py @@ -23,7 +23,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - config_name = input_data.get("config", "sims.p2e.P2EToyConfig") + config_name = input_data.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + ctx.logger.error("Missing required parameter: config") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_config"}, + trace_id=origin_tid, + ) + return build_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("output_dir")) command = f"rm -rf {build_dir}" diff --git a/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py index 7b1cecfb6..c680a9eb2 100644 --- a/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py +++ b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py @@ -16,7 +16,12 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = req.body or {} - config_name = body.get("config", "sims.p2e.P2EToyConfig") + config_name = body.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --config must be specified"}, + ) output_dir = get_verilator_build_dir(bbdir, config_name, body.get("output_dir")) data = { diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py index 75cdfc4db..7b00fabe9 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py @@ -15,7 +15,12 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = request.body or {} - config_name = body.get("config", "sims.p2e.P2EToyConfig") + config_name = body.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --config must be specified"}, + ) vsrc_dir = get_verilator_build_dir(bbdir, config_name, body.get("vsrc_dir")) data = { diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index 50affc62a..f3a7f5112 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -34,7 +34,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" - config_name = input_data.get("config", "sims.p2e.P2EToyConfig") + config_name = input_data.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + ctx.logger.error("Missing required parameter: config") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_config"}, + trace_id=origin_tid, + ) + return vsrc_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("vsrc_dir")) if not os.path.isdir(vsrc_dir): ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}") diff --git a/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml index ab86585c1..573fa4115 100644 --- a/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml @@ -5,11 +5,10 @@ # Each entry under tests is the file STEM (without extension). [workloads] -search_path = "workloads/src/CTest/toy" +search_path = "workloads/src" tests = [ "ctest_bdb_counter_test_singlecore-baremetal", "ctest_bfp_test_singlecore-baremetal", - "ctest_dequant_test_singlecore-baremetal", "ctest_gemmini_os_cisc_basic_test_singlecore-baremetal", "ctest_gemmini_os_cisc_atranspose_test_singlecore-baremetal", "ctest_gemmini_os_cisc_btranspose_test_singlecore-baremetal", diff --git a/bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml index 3e898a238..c2c91389b 100644 --- a/bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml @@ -5,11 +5,10 @@ # Each entry under tests is the file STEM (without extension). [workloads] -search_path = "workloads/src/CTest/toy" +search_path = "workloads/src" tests = [ "ctest_bdb_counter_test-linux", "ctest_bfp_test-linux", - "ctest_dequant_test-linux", "ctest_gemmini_os_cisc_basic_test-linux", "ctest_gemmini_os_cisc_atranspose_test-linux", "ctest_gemmini_os_cisc_btranspose_test-linux", diff --git a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py index bbec8298b..0ff94e3ab 100644 --- a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py +++ b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py @@ -16,7 +16,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} config_name = body.get("config") - if not config_name: + if not isinstance(config_name, str) or not config_name or config_name == "None": return ApiResponse( status=400, body={"error": "Missing required parameter: --config must be specified"} diff --git a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py index f151b27fe..85989037b 100644 --- a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py +++ b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py @@ -26,7 +26,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - config_name = input_data.get("config", "sims.verilator.BuckyballToyVerilatorConfig") + config_name = input_data.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + ctx.logger.error("Missing required parameter: config") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_config"}, + trace_id=origin_tid, + ) + return build_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("output_dir")) command = f"rm -rf {build_dir}" diff --git a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py index 24eb9b674..aaac2420e 100644 --- a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py +++ b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py @@ -16,7 +16,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} config_name = body.get("config") - if not config_name or config_name == "None": + if not isinstance(config_name, str) or not config_name or config_name == "None": return ApiResponse( status=400, body={ diff --git a/bbdev/api/steps/bebop/verilator/03_build_api.step.py b/bbdev/api/steps/bebop/verilator/03_build_api.step.py index bed479674..4e5032cfa 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_api.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_api.step.py @@ -13,7 +13,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} arch_config = body.get("config") - if not arch_config: + if not isinstance(arch_config, str) or not arch_config or arch_config == "None": return ApiResponse( status=400, body={"error": "Missing required parameter: --config must be specified"} diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index 24d4649a4..27e8dc694 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -56,7 +56,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bebop_dir = f"{bbdir}/bebop" arch_config = input_data.get("config") - if not arch_config: + if not isinstance(arch_config, str) or not arch_config or arch_config == "None": ctx.logger.error("Missing required parameter: config must be specified") await check_result( ctx, 1, continue_run=False, diff --git a/bbdev/api/steps/bebop/verilator/04_sim_api.step.py b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py index ed5249ae2..2e72a80b4 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_api.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py @@ -22,6 +22,12 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "message": "binary parameter is required", }, ) + config_name = body.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --config must be specified"}, + ) await ctx.enqueue({"topic": "bebop.verilator.sim", "data": {**body, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index 30951d852..a849de947 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -39,7 +39,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bebop_dir = f"{bbdir}/bebop" arch_dir = f"{bbdir}/arch" - arch_config = input_data.get("config", "sims.verilator.BuckyballToyVerilatorConfig") + arch_config = input_data.get("config") + if not isinstance(arch_config, str) or not arch_config or arch_config == "None": + ctx.logger.error("Missing required parameter: config") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_config"}, + trace_id=origin_tid, + ) + return vsrc_dir = get_verilator_build_dir(bbdir, arch_config, input_data.get("vsrc_dir")) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") if not os.path.isdir(vsrc_dir): diff --git a/bbdev/api/steps/bebop/verilator/05_run_api.step.py b/bbdev/api/steps/bebop/verilator/05_run_api.step.py index b7014b0e5..d5080ef57 100644 --- a/bbdev/api/steps/bebop/verilator/05_run_api.step.py +++ b/bbdev/api/steps/bebop/verilator/05_run_api.step.py @@ -13,7 +13,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} config_name = body.get("config") - if not config_name: + if not isinstance(config_name, str) or not config_name or config_name == "None": return ApiResponse( status=400, body={"error": "Missing required parameter: --config must be specified"} diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py index e347e11d6..d74ea0903 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -16,7 +16,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} arch_config = body.get("config") - if not arch_config: + if not isinstance(arch_config, str) or not arch_config or arch_config == "None": return ApiResponse( status=400, body={"error": "Missing required parameter: --config must be specified"} diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 18155124e..facab9e21 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -6,6 +6,7 @@ 2. Run cargo nextest with verilator-specific config (serial execution) """ import os +import shlex import sys from motia import FlowContext, queue @@ -35,7 +36,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: elf_root = f"{bbdir}/bb-tests/output" arch_config = input_data.get("config") - if not arch_config: + if not isinstance(arch_config, str) or not arch_config or arch_config == "None": ctx.logger.error("Missing required parameter: config must be specified") await check_result( ctx, 1, continue_run=False, @@ -72,15 +73,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return # ── Build bebop verilator (tests) ───────────────────────────────────── + vsrc_config = shlex.quote(f"env.VSRC_PATH='{vsrc_dir}'") build_cmd = ( - f"nix develop -c cargo build --features verilator --tests " - f"--config=\"env.VSRC_PATH='{vsrc_dir}'\"" + f"nix develop -c cargo build --manifest-path {shlex.quote(f'{bebop_dir}/Cargo.toml')} " + "--features verilator --tests " + f"--config={vsrc_config}" ) ctx.logger.info("Building bebop verilator (tests)...") build_result = stream_run_logger( cmd=build_cmd, logger=ctx.logger, - cwd=bebop_dir, + cwd=bbdir, stdout_prefix="bebop verilator build", stderr_prefix="bebop verilator build", ) @@ -102,9 +105,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "VSRC_PATH": vsrc_dir, }) nextest_cmd = ( - f"nix develop -c cargo nextest run --features verilator --test test_verilator " - f"--config-file \"{nextest_config}\" " - f"--config=\"env.VSRC_PATH='{vsrc_dir}'\"" + f"nix develop -c cargo nextest run --manifest-path {shlex.quote(f'{bebop_dir}/Cargo.toml')} " + "--features verilator --test test_verilator " + f"--config-file {shlex.quote(nextest_config)} " + f"--config={vsrc_config}" ) ctx.logger.info(f"Running bebop verilator nextest: {nextest_cmd}") @@ -112,7 +116,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: run_result = stream_run_logger( cmd=nextest_cmd, logger=ctx.logger, - cwd=bebop_dir, + cwd=bbdir, stdout_prefix="bebop verilator batch", stderr_prefix="bebop verilator batch", env=env, diff --git a/bbdev/api/steps/compiler/01_build_api.step.py b/bbdev/api/steps/compiler/01_build_api.step.py index 63d1040ed..851b62039 100644 --- a/bbdev/api/steps/compiler/01_build_api.step.py +++ b/bbdev/api/steps/compiler/01_build_api.step.py @@ -1,5 +1,16 @@ +import os +import re +import sys +from pathlib import Path + from motia import ApiRequest, ApiResponse, FlowContext, api +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path + config = { "name": "build-compiler-api", "description": "build compiler", @@ -9,7 +20,46 @@ } +def available_chips() -> list[str]: + chips_dir = Path(get_buckyball_path()) / "examples" / "chips" + return sorted( + path.name + for path in chips_dir.iterdir() + if (path / "compiler" / "CMakeLists.txt").is_file() + ) + + async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + + stable = body.get("stable", False) + if not isinstance(stable, bool): + return ApiResponse( + status=400, + body={"error": "Invalid parameter: stable must be a boolean flag"}, + ) + + chip = body.get("chip") + if not chip: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --chip must be specified"}, + ) + if not re.fullmatch(r"[A-Za-z0-9_-]+", chip): + return ApiResponse( + status=400, + body={"error": f"Invalid chip: {chip}"}, + ) + + chips = available_chips() + chip_dir = Path(get_buckyball_path()) / "examples" / "chips" / chip / "compiler" + if not (chip_dir / "CMakeLists.txt").is_file(): + return ApiResponse( + status=400, + body={ + "error": f"Compiler chip does not exist: {chip}; available compiler chips are: {', '.join(chips)}", + }, + ) + await ctx.enqueue({"topic": "compiler.build", "data": {**body, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/compiler/01_build_event.step.py b/bbdev/api/steps/compiler/01_build_event.step.py index a14edc8f7..e63797e0a 100644 --- a/bbdev/api/steps/compiler/01_build_event.step.py +++ b/bbdev/api/steps/compiler/01_build_event.step.py @@ -1,5 +1,8 @@ import os +import re +import shlex import sys +from pathlib import Path from motia import FlowContext, queue @@ -21,44 +24,114 @@ } +def available_chips(bbdir: str) -> list[str]: + chips_dir = Path(bbdir) / "examples" / "chips" + return sorted( + path.name + for path in chips_dir.iterdir() + if (path / "compiler" / "CMakeLists.txt").is_file() + ) + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - script_dir = f"{bbdir}/workflow/steps/compiler/scripts" - yaml_dir = f"{script_dir}/yaml" - # ================================================================================== - # Execute operation - # ================================================================================== - command = f"mkdir -p {bbdir}/compiler/thirdparty/buddy-mlir/build" - result = stream_run_logger( - cmd=command, - logger=ctx.logger, - stdout_prefix="compiler build", - stderr_prefix="compiler build", - ) - command = ( - f"cd {bbdir}/compiler/thirdparty/buddy-mlir/build " - "&& cmake --build . --target python-package-buddy" - ) - result = stream_run_logger( - cmd=command, - logger=ctx.logger, - stdout_prefix="compiler build", - stderr_prefix="compiler build", - ) - command = f"cd {bbdir}/compiler/thirdparty/buddy-mlir/build && ninja -j{os.cpu_count()}" - result = stream_run_logger( - cmd=command, - logger=ctx.logger, - stdout_prefix="compiler build", - stderr_prefix="compiler build", - ) + + stable = input_data.get("stable", False) + if not isinstance(stable, bool): + ctx.logger.error("Invalid parameter: stable must be a boolean flag") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_stable", "stable": stable}, + trace_id=origin_tid, + ) + return + + chip = input_data.get("chip") + if not chip: + ctx.logger.error("Missing required parameter: chip must be specified") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_chip"}, + trace_id=origin_tid, + ) + return + if not re.fullmatch(r"[A-Za-z0-9_-]+", chip): + ctx.logger.error(f"Invalid chip: {chip}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_chip", "chip": chip}, + trace_id=origin_tid, + ) + return + + chips = available_chips(bbdir) + chip_dir = Path(bbdir) / "examples" / "chips" / chip / "compiler" + if not (chip_dir / "CMakeLists.txt").is_file(): + ctx.logger.error(f"Compiler chip does not exist: {chip}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={ + "error": "unknown_chip", + "chip": chip, + "available_chips": chips, + }, + trace_id=origin_tid, + ) + return + + buddy_dir = f"{bbdir}/compiler/thirdparty/buddy-mlir" + build_dir = f"{buddy_dir}/build" + llvm_build_dir = f"{buddy_dir}/llvm/build" + + def nix_cmd(args: list[str]) -> str: + return shlex.join(["nix", "develop", "-c", *args]) + + async def run_stage(stage: str, command: str): + ctx.logger.info(f"Running compiler build stage '{stage}': {command}") + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="compiler build", + stderr_prefix="compiler build", + ) + if result.returncode != 0: + await check_result( + ctx, result.returncode, continue_run=False, + extra_fields={"task": stage, "chip": chip, "stable": stable}, + trace_id=origin_tid, + ) + return False + return True + + cmake_args = [ + "cmake", "-G", "Ninja", + "-S", buddy_dir, + "-B", build_dir, + f"-DBUDDY_EXTERNAL_DIALECTS_DIR={chip_dir}", + f"-DMLIR_DIR={llvm_build_dir}/lib/cmake/mlir", + f"-DLLVM_DIR={llvm_build_dir}/lib/cmake/llvm", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=RELEASE", + "-DBUDDY_MLIR_ENABLE_PYTHON_PACKAGES=ON", + "-DPython3_EXECUTABLE=python3", + "-DPython_EXECUTABLE=python", + ] + if not await run_stage("configure", nix_cmd(cmake_args)): + return + + command = nix_cmd(["ninja", "-C", build_dir, + "buddy-opt", "buddy-translate", "buddy-llc"]) + mode = "stable" if stable else "custom" + if not await run_stage(f"compiler-{chip}-{mode}", command): + return # ================================================================================== # Return result to API # ================================================================================== success_result, failure_result = await check_result( - ctx, result.returncode, continue_run=False, trace_id=origin_tid) + ctx, 0, continue_run=False, trace_id=origin_tid) # ================================================================================== # Continue routing diff --git a/bbdev/api/steps/compiler/README.md b/bbdev/api/steps/compiler/README.md index 993560bc7..92684307f 100644 --- a/bbdev/api/steps/compiler/README.md +++ b/bbdev/api/steps/compiler/README.md @@ -9,11 +9,14 @@ Compiler build workflow in the Buckyball framework for building the Buckyball co **Function**: Build Buckyball compiler -**Parameters**: No specific parameters +**Parameters**: +- `chip`: compiler chip package used as `BUDDY_EXTERNAL_DIALECTS_DIR`; valid values include `toy`, `goban`, and `pebble` +- `stable`: optional boolean flag; for `pebble`, build the compiler package that can use the stable LLVM backend lowering path **Example**: ```bash -bbdev compiler --build +bbdev compiler --build '--chip toy' +bbdev compiler --build '--chip pebble --stable' ``` **Response**: @@ -31,3 +34,4 @@ bbdev compiler --build ## Notes - Ensure the system has necessary build tools and dependencies +- `pebble` supports both custom `xbuckyball` lowering and stable LLVM backend lowering; the selected lowering mode is controlled by pass/workload options diff --git a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py index 92d2607d4..6b733fddd 100644 --- a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py +++ b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py @@ -55,7 +55,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" - config_name = input_data.get("config", "sims.p2e.P2EToyConfig") + config_name = input_data.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + ctx.logger.error("Missing required parameter: config") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_config"}, + trace_id=origin_tid, + ) + return build_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("output_dir")) ctx.logger.info(f"Using P2E configuration: {config_name}") diff --git a/bbdev/api/steps/uvm/01_build_api.step.py b/bbdev/api/steps/uvm/01_build_api.step.py new file mode 100644 index 000000000..11dd79613 --- /dev/null +++ b/bbdev/api/steps/uvm/01_build_api.step.py @@ -0,0 +1,32 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "uvm-build-api", + "description": "Build a Ball UVM simulation", + "flows": ["uvm"], + "triggers": [api("POST", "/uvm/build")], + "enqueues": ["uvm.build"], +} + + +def check_args(body: dict) -> str | None: + allowed = {"ball", "filelist"} + for key in body: + if key not in allowed: + return f"Unexpected parameter: --{key}" + ball = body.get("ball") + if not ball or ball is True: + return "Missing required parameter: --ball=" + if body.get("filelist") is True: + return "Parameter --filelist requires a path value" + return None + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + error = check_args(body) + if error: + return ApiResponse(status=400, body={"error": error}) + + await ctx.enqueue({"topic": "uvm.build", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/uvm/01_build_event.step.py b/bbdev/api/steps/uvm/01_build_event.step.py new file mode 100644 index 000000000..5e2baf7ac --- /dev/null +++ b/bbdev/api/steps/uvm/01_build_event.step.py @@ -0,0 +1,37 @@ +import os +import sys + +from motia import FlowContext, queue + +step_dir = os.path.dirname(os.path.abspath(__file__)) +utils_path = os.path.abspath(os.path.join(step_dir, "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +if step_dir not in sys.path: + sys.path.insert(0, step_dir) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path +from uvm_common import run_uvm_build + +config = { + "name": "uvm-build", + "description": "Build a Ball UVM simulation", + "flows": ["uvm"], + "triggers": [queue("uvm.build")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + + result, info = run_uvm_build(bbdir, input_data, ctx) + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields=info, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/uvm/02_run_api.step.py b/bbdev/api/steps/uvm/02_run_api.step.py new file mode 100644 index 000000000..22c916373 --- /dev/null +++ b/bbdev/api/steps/uvm/02_run_api.step.py @@ -0,0 +1,34 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "uvm-run-api", + "description": "Build and run a Ball UVM simulation", + "flows": ["uvm"], + "triggers": [api("POST", "/uvm/run")], + "enqueues": ["uvm.run"], +} + + +def check_args(body: dict) -> str | None: + allowed = {"ball", "filelist", "test"} + for key in body: + if key not in allowed: + return f"Unexpected parameter: --{key}" + ball = body.get("ball") + if not ball or ball is True: + return "Missing required parameter: --ball=" + if body.get("filelist") is True: + return "Parameter --filelist requires a path value" + if body.get("test") is True: + return "Parameter --test requires a value" + return None + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + error = check_args(body) + if error: + return ApiResponse(status=400, body={"error": error}) + + await ctx.enqueue({"topic": "uvm.run", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/uvm/02_run_event.step.py b/bbdev/api/steps/uvm/02_run_event.step.py new file mode 100644 index 000000000..a0286e83c --- /dev/null +++ b/bbdev/api/steps/uvm/02_run_event.step.py @@ -0,0 +1,77 @@ +import os +import shlex +import sys + +from motia import FlowContext, queue + +step_dir = os.path.dirname(os.path.abspath(__file__)) +utils_path = os.path.abspath(os.path.join(step_dir, "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +if step_dir not in sys.path: + sys.path.insert(0, step_dir) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from uvm_common import default_test_name, run_uvm_build, uvm_paths + +config = { + "name": "uvm-run", + "description": "Build and run a Ball UVM simulation", + "flows": ["uvm"], + "triggers": [queue("uvm.run")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + + build_result, info = run_uvm_build(bbdir, input_data, ctx) + if build_result.returncode != 0: + await check_result( + ctx, + build_result.returncode, + continue_run=False, + extra_fields=info, + trace_id=origin_tid, + ) + return + + paths = uvm_paths(bbdir, input_data) + test = input_data.get("test") or default_test_name(paths["ball"]) + if test is True: + ctx.logger.error("Parameter --test requires a value") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={**info, "error": "invalid_test"}, + trace_id=origin_tid, + ) + return + + script = ( + f"cd {shlex.quote(paths['verify_dir'])} && " + f"{shlex.quote(paths['simv'])} +UVM_TESTNAME={shlex.quote(test)}" + ) + cmd = f"nix develop {shlex.quote(paths['verify_env'])} --command zsh -lc {shlex.quote(script)}" + + ctx.logger.info(f"Running UVM test {test} for ball {paths['ball']}") + run_result = stream_run_logger( + cmd=cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm run", + stderr_prefix="uvm run", + ) + + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={**info, "task": "run", "test": test}, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/uvm/uvm_common.py b/bbdev/api/steps/uvm/uvm_common.py new file mode 100644 index 000000000..bc88471d1 --- /dev/null +++ b/bbdev/api/steps/uvm/uvm_common.py @@ -0,0 +1,163 @@ +import os +import re +import shlex +import sys +import tomllib + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.stream_run import StreamResult, stream_run_logger + + +def default_test_name(ball: str) -> str: + return f"{ball}_ball_test" + + +def uvm_paths(bbdir: str, input_data: dict) -> dict: + ball = input_data.get("ball") + if not isinstance(ball, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", ball): + raise ValueError("invalid ball name") + + ball_dir = os.path.join(bbdir, "examples", "balls", ball) + verify_dir = os.path.join(ball_dir, "verify") + casegen_dir = os.path.join(verify_dir, "casegen") + verify_env = os.path.join(bbdir, "verify") + sim_dir = os.path.join(verify_dir, "build") + simv = os.path.join(sim_dir, "simv") + csrc_dir = os.path.join(sim_dir, "csrc") + + return { + "ball": ball, + "ball_dir": ball_dir, + "verify_dir": verify_dir, + "casegen_dir": casegen_dir, + "verify_env": verify_env, + "sim_dir": sim_dir, + "simv": simv, + "csrc_dir": csrc_dir, + } + + +def checked_paths(bbdir: str, input_data: dict) -> dict: + paths = uvm_paths(bbdir, input_data) + for key in ("ball_dir", "verify_dir", "casegen_dir", "verify_env"): + if not os.path.isdir(paths[key]): + raise FileNotFoundError(f"{key} not found: {paths[key]}") + + cargo_toml = os.path.join(paths["casegen_dir"], "Cargo.toml") + if not os.path.isfile(cargo_toml): + raise FileNotFoundError(f"Cargo.toml not found: {cargo_toml}") + + filelist = resolve_filelist(paths["verify_dir"], input_data.get("filelist"), paths["ball"]) + crate = read_crate_name(cargo_toml) + dpi_lib = os.path.join(paths["casegen_dir"], "target", "debug", f"lib{crate.replace('-', '_')}") + + paths.update({ + "cargo_toml": cargo_toml, + "filelist": filelist, + "filelist_arg": filelist_arg(paths["verify_dir"], filelist), + "crate": crate, + "dpi_lib": dpi_lib, + }) + return paths + + +def resolve_filelist(verify_dir: str, arg, ball: str) -> str: + if arg is True: + raise ValueError("parameter --filelist requires a path value") + + if arg: + path = arg if os.path.isabs(arg) else os.path.join(verify_dir, arg) + if not os.path.isfile(path): + raise FileNotFoundError(f"filelist not found: {path}") + return os.path.abspath(path) + + default = os.path.join(verify_dir, "filelists", f"{ball}_ball_toy.f") + if os.path.isfile(default): + return default + + filelist_dir = os.path.join(verify_dir, "filelists") + if not os.path.isdir(filelist_dir): + raise FileNotFoundError(f"filelist directory not found: {filelist_dir}") + found = [ + os.path.join(filelist_dir, name) + for name in os.listdir(filelist_dir) + if name.endswith(".f") + ] + if len(found) != 1: + raise RuntimeError(f"expected exactly one filelist under {filelist_dir}, found {len(found)}") + return found[0] + + +def filelist_arg(verify_dir: str, filelist: str) -> str: + rel = os.path.relpath(filelist, verify_dir) + if rel.startswith(".."): + return filelist + return rel + + +def read_crate_name(cargo_toml: str) -> str: + with open(cargo_toml, "rb") as f: + data = tomllib.load(f) + name = data.get("package", {}).get("name") + if not name: + raise ValueError(f"package.name not found in {cargo_toml}") + return name + + +def failed_result(message: str) -> StreamResult: + return StreamResult(returncode=1, stdout="", stderr=message) + + +def run_uvm_build(bbdir: str, input_data: dict, ctx) -> tuple[StreamResult, dict]: + try: + paths = checked_paths(bbdir, input_data) + except Exception as e: + ctx.logger.error(str(e)) + return failed_result(str(e)), {"task": "build", "error": str(e)} + + info = { + "task": "build", + "ball": paths["ball"], + "verify_dir": paths["verify_dir"], + "filelist": paths["filelist"], + "simv": paths["simv"], + } + + cargo_cmd = ( + f"nix develop {shlex.quote(paths['verify_env'])} --command " + f"cargo build --manifest-path {shlex.quote(paths['cargo_toml'])}" + ) + ctx.logger.info(f"Building UVM DPI reference for ball {paths['ball']}") + cargo_result = stream_run_logger( + cmd=cargo_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm dpi build", + stderr_prefix="uvm dpi build", + ) + if cargo_result.returncode != 0: + return cargo_result, info + + script = ( + f"cd {shlex.quote(paths['verify_dir'])} && " + f"mkdir -p {shlex.quote(paths['sim_dir'])} {shlex.quote(paths['csrc_dir'])} && " + "vcs -full64 -sverilog -timescale=1ns/1ps " + "$VCS_UVM_ARGS " + f"-sv_lib {shlex.quote(paths['dpi_lib'])} " + f"-Mdir {shlex.quote(paths['csrc_dir'])} " + f"-o {shlex.quote(paths['simv'])} " + f"-f {shlex.quote(paths['filelist_arg'])}" + ) + vcs_cmd = f"nix develop {shlex.quote(paths['verify_env'])} --command zsh -lc {shlex.quote(script)}" + + ctx.logger.info(f"Building UVM simulation for ball {paths['ball']}") + return stream_run_logger( + cmd=vcs_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm vcs build", + stderr_prefix="uvm vcs build", + ), info diff --git a/bbdev/api/steps/verilator/01_clean_api.step.py b/bbdev/api/steps/verilator/01_clean_api.step.py index ab1361d9e..a5b0a826e 100644 --- a/bbdev/api/steps/verilator/01_clean_api.step.py +++ b/bbdev/api/steps/verilator/01_clean_api.step.py @@ -11,5 +11,11 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + config_name = body.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --config must be specified"}, + ) await ctx.enqueue({"topic": "verilator.clean", "data": {**body, "task": "clean", "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/01_clean_event.step.py b/bbdev/api/steps/verilator/01_clean_event.step.py index 434481ba1..af1e0678b 100644 --- a/bbdev/api/steps/verilator/01_clean_event.step.py +++ b/bbdev/api/steps/verilator/01_clean_event.step.py @@ -26,10 +26,19 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) + config_name = input_data.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + ctx.logger.error("Missing required parameter: config") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_config"}, + trace_id=origin_tid, + ) + return bbdir = get_buckyball_path() build_dir = get_verilator_build_dir( bbdir, - input_data.get("config"), + config_name, input_data.get("output_dir"), ) # ================================================================================== diff --git a/bbdev/api/steps/verilator/02_verilog_api.step.py b/bbdev/api/steps/verilator/02_verilog_api.step.py index 3b3af1734..d5054f5fd 100644 --- a/bbdev/api/steps/verilator/02_verilog_api.step.py +++ b/bbdev/api/steps/verilator/02_verilog_api.step.py @@ -17,7 +17,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: # Get config name, must be provided config_name = body.get("config") - if not config_name or config_name == "None": + if not isinstance(config_name, str) or not config_name or config_name == "None": return ApiResponse( status=400, body={ diff --git a/bbdev/api/steps/verilator/03_build_api.step.py b/bbdev/api/steps/verilator/03_build_api.step.py index f88f07b2d..4c7b3ed7f 100644 --- a/bbdev/api/steps/verilator/03_build_api.step.py +++ b/bbdev/api/steps/verilator/03_build_api.step.py @@ -14,10 +14,16 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = request.body or {} + config_name = body.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --config must be specified"}, + ) data = { "jobs": body.get("jobs", 16), - "config": body.get("config"), - "output_dir": get_verilator_build_dir(bbdir, body.get("config"), body.get("output_dir")), + "config": config_name, + "output_dir": get_verilator_build_dir(bbdir, config_name, body.get("output_dir")), } await ctx.enqueue({"topic": "verilator.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/03_build_event.step.py b/bbdev/api/steps/verilator/03_build_event.step.py index 6b6acf9e3..64806092e 100644 --- a/bbdev/api/steps/verilator/03_build_event.step.py +++ b/bbdev/api/steps/verilator/03_build_event.step.py @@ -26,7 +26,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) config_name = input_data.get("config") - if not config_name or config_name == "None": + if not isinstance(config_name, str) or not config_name or config_name == "None": raise ValueError("Missing required input: config") bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" diff --git a/bbdev/api/steps/verilator/04_sim_api.step.py b/bbdev/api/steps/verilator/04_sim_api.step.py index 07ad7c5c7..069d941b5 100644 --- a/bbdev/api/steps/verilator/04_sim_api.step.py +++ b/bbdev/api/steps/verilator/04_sim_api.step.py @@ -22,6 +22,12 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "message": "binary parameter is required", }, ) + config_name = body.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --config must be specified"}, + ) await ctx.enqueue({"topic": "verilator.sim", "data": {**body, "task": "sim", "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/04_sim_event.step.py b/bbdev/api/steps/verilator/04_sim_event.step.py index 7f9e86349..d880c5a02 100644 --- a/bbdev/api/steps/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -30,7 +30,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ================================================================================== origin_tid = get_origin_trace_id(input_data, ctx) config_name = input_data.get("config") - if not config_name or config_name == "None": + if not isinstance(config_name, str) or not config_name or config_name == "None": raise ValueError("Missing required input: config") bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" diff --git a/bbdev/api/steps/verilator/05_run_api.step.py b/bbdev/api/steps/verilator/05_run_api.step.py index d784e94a2..79c872901 100644 --- a/bbdev/api/steps/verilator/05_run_api.step.py +++ b/bbdev/api/steps/verilator/05_run_api.step.py @@ -11,10 +11,27 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + config_name = body.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --config must be specified"}, + ) + binary = body.get("binary", "") + if not binary: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 400, + "message": "binary parameter is required", + }, + ) data = { - "binary": body.get("binary", ""), - "config": body.get("config", "sims.verilator.BuckyballToyVerilatorConfig"), + "binary": binary, + "config": config_name, "jobs": body.get("jobs", "16"), "batch": body.get("batch", False), "from_run_workflow": True, diff --git a/bbdev/api/steps/verilator/README.md b/bbdev/api/steps/verilator/README.md index 25635f69e..9da810c68 100644 --- a/bbdev/api/steps/verilator/README.md +++ b/bbdev/api/steps/verilator/README.md @@ -19,7 +19,7 @@ Hardware simulation workflow based on Verilator in the Buckyball framework, prov **Example**: ```bash # bbdev wrapper -bbdev verilator --run "jobs 256 --binary ${buckyball}/bb-tests/workloads/build/src/CTest/ctest_mvin_mvout_alternate_test_singlecore-baremetal --batch" +bbdev verilator --run "jobs 256 --binary ${buckyball}/bb-tests/output/workloads/src/ctest_mvin_mvout_alternate_test_singlecore-baremetal --batch" # Raw command curl -X POST http://localhost:5000/verilator/run -H "Content-Type: application/json" -d '{"jobs": 8, "binary": "/home/user/test.elf"}' diff --git a/bbdev/api/steps/workload/01_buidl_api.step.py b/bbdev/api/steps/workload/01_buidl_api.step.py index 254bdb94b..db6db307f 100644 --- a/bbdev/api/steps/workload/01_buidl_api.step.py +++ b/bbdev/api/steps/workload/01_buidl_api.step.py @@ -11,9 +11,22 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + allowed = {"model", "stable"} + unknown = sorted(k for k in body if k not in allowed) + if unknown: + return ApiResponse( + status=400, + body={"error": f"Unknown workload build parameter(s): {', '.join(unknown)}"}, + ) + stable = body.get("stable", False) + if not isinstance(stable, bool): + return ApiResponse( + status=400, + body={"error": "Invalid parameter: stable must be a boolean flag"}, + ) data = { - "workload": body.get("workload", ""), "model": body.get("model", ""), + "stable": stable, } await ctx.enqueue({"topic": "workload.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index 4889456fe..230f8adfd 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -1,5 +1,6 @@ import os import sys +import shlex from motia import FlowContext, queue @@ -26,7 +27,27 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() workload_dir = f"{bbdir}/bb-tests" build_dir = f"{workload_dir}/build" + allowed = {"model", "stable", "_trace_id"} + unknown = sorted(k for k in input_data if k not in allowed) + if unknown: + ctx.logger.error(f"Unknown workload build parameter(s): {', '.join(unknown)}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "unknown_parameter", "parameters": unknown}, + trace_id=origin_tid, + ) + return model = input_data.get("model", "") + stable = input_data.get("stable", False) + + if not isinstance(stable, bool): + ctx.logger.error("Invalid parameter: stable must be a boolean flag") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_stable", "stable": stable}, + trace_id=origin_tid, + ) + return model_targets = { "lenet": "buddy-buckyball-lenet-run", @@ -56,8 +77,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: os.makedirs(build_dir, exist_ok=True) - ninja_target = f" {target}" if target else "" - command = f"cd {bbdir} && nix develop -c bash -c 'cd {build_dir} && cmake -G Ninja .. && ninja -j{os.cpu_count()}{ninja_target}'" + stable_arg = "-DBUCKYBALL_STABLE=ON" if stable else "-DBUCKYBALL_STABLE=OFF" + ninja_target = f" {shlex.quote(target)}" if target else "" + inner = ( + f"cd {shlex.quote(build_dir)} && " + f"cmake -G Ninja {stable_arg} .. && " + f"ninja -j{os.cpu_count()}{ninja_target}" + ) + command = f"cd {shlex.quote(bbdir)} && nix develop -c bash -c {shlex.quote(inner)}" ctx.logger.info( "Executing workload command", {"command": command, "cwd": build_dir} ) diff --git a/bbdev/api/steps/workload/README.md b/bbdev/api/steps/workload/README.md index a6bb39414..6769fabc1 100644 --- a/bbdev/api/steps/workload/README.md +++ b/bbdev/api/steps/workload/README.md @@ -10,12 +10,16 @@ Workload build workflow in Buckyball framework, used to build test workloads and **Function**: Build workload **Parameters**: -- **`workload`** - Specify workload name to build +- **`model`** - Optional model name to build. If omitted, build all workloads. +- **`stable`** - Optional boolean flag. If set, build with stable LLVM Buckyball extensions. **Examples**: ```bash -# Build specific workload -bbdev workload --build "--workload test_program" +# Build one model with the default xbuckyball path +bbdev workload --build "--model lenet" + +# Build one model with stable LLVM Buckyball extensions +bbdev workload --build "--model lenet --stable" # Build all workloads bbdev workload --build @@ -35,5 +39,6 @@ bbdev workload --build ## Notes -- Workload source code located in `bb-tests/workload` directory -- Build results typically output to `bb-tests/workloads/build` directory +- Workload build entry is `bb-tests` +- Workload source code is distributed under `bb-tests/workloads/src` and `examples/*/*/workloads` +- Workload binaries are emitted under `bb-tests/output/workloads/src` diff --git a/bbdev/api/tests/test_api_bebop_bemu_batch.py b/bbdev/api/tests/test_api_bebop_bemu_batch.py index 3f3f2befb..7eff4a36e 100644 --- a/bbdev/api/tests/test_api_bebop_bemu_batch.py +++ b/bbdev/api/tests/test_api_bebop_bemu_batch.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-bemu --batch '--test elf-tests'") +run_bbdev_case("bbdev bebop-bemu --batch '--chip toy --test elf-tests'") diff --git a/bbdev/api/tests/test_api_bebop_bemu_sim.py b/bbdev/api/tests/test_api_bebop_bemu_sim.py index 20d825c2a..0cc425e71 100644 --- a/bbdev/api/tests/test_api_bebop_bemu_sim.py +++ b/bbdev/api/tests/test_api_bebop_bemu_sim.py @@ -1,4 +1,4 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-bemu --sim '--binary ctest_vecunit_matmul_ones_singlecore-baremetal'") -run_bbdev_case("bbdev bebop-bemu --sim '--binary ctest_vecunit_matmul_ones-linux --pk'") +run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary ctest_vecunit_matmul_ones_singlecore-baremetal'") +run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary ctest_vecunit_matmul_ones-linux --pk'") diff --git a/bbdev/api/tests/test_api_compiler_build.py b/bbdev/api/tests/test_api_compiler_build.py index 54c28e552..ef786df81 100644 --- a/bbdev/api/tests/test_api_compiler_build.py +++ b/bbdev/api/tests/test_api_compiler_build.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev compiler --build") +run_bbdev_case("bbdev compiler --build '--chip toy'") diff --git a/bbdev/api/tests/test_api_verilator_build.py b/bbdev/api/tests/test_api_verilator_build.py index c3b131602..4c984d6fa 100644 --- a/bbdev/api/tests/test_api_verilator_build.py +++ b/bbdev/api/tests/test_api_verilator_build.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --build '--jobs 16'") +run_bbdev_case("bbdev verilator --build '--jobs 16 --config sims.verilator.BuckyballToyVerilatorConfig'") diff --git a/bbdev/api/tests/test_api_verilator_clean.py b/bbdev/api/tests/test_api_verilator_clean.py index 0e76ee6f9..f40ad020b 100644 --- a/bbdev/api/tests/test_api_verilator_clean.py +++ b/bbdev/api/tests/test_api_verilator_clean.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --clean") +run_bbdev_case("bbdev verilator --clean '--config sims.verilator.BuckyballToyVerilatorConfig'") diff --git a/bbdev/api/tests/test_api_verilator_sim.py b/bbdev/api/tests/test_api_verilator_sim.py index 962a28b43..849493831 100644 --- a/bbdev/api/tests/test_api_verilator_sim.py +++ b/bbdev/api/tests/test_api_verilator_sim.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --sim '--binary ctest_vecunit_matmul_ones_singlecore-baremetal --batch'") +run_bbdev_case("bbdev verilator --sim '--binary ctest_vecunit_matmul_ones_singlecore-baremetal --batch --config sims.verilator.BuckyballToyVerilatorConfig'") diff --git a/bbdev/api/utils/bemu.py b/bbdev/api/utils/bemu.py new file mode 100644 index 000000000..8a54ecd7d --- /dev/null +++ b/bbdev/api/utils/bemu.py @@ -0,0 +1,41 @@ +import os +import tomllib +from pathlib import Path + +from utils.path import get_buckyball_path + + +def available_bemu_chips(bbdir: str | None = None) -> list[str]: + root = Path(bbdir or get_buckyball_path()) + cargo_toml = root / "bebop" / "Cargo.toml" + with open(cargo_toml, "rb") as f: + manifest = tomllib.load(f) + + features = manifest.get("features", {}) + chips = [] + for name in features: + if not name.startswith("bemu-"): + continue + chip = name.removeprefix("bemu-") + if (root / "examples" / "chips" / chip / "emu" / "Cargo.toml").is_file(): + chips.append(chip) + return sorted(chips) + + +def bemu_feature(chip: str, bbdir: str | None = None) -> str: + chips = available_bemu_chips(bbdir) + if chip not in chips: + available = ", ".join(chips) if chips else "none" + raise ValueError(f"invalid chip: {chip}; available bemu chips: {available}") + return f"bemu-{chip}" + + +def validate_bemu_chip(chip: str | None, bbdir: str | None = None) -> str: + if not chip: + raise ValueError("missing required parameter: chip") + if not isinstance(chip, str): + raise ValueError("chip must be a string") + if os.path.sep in chip or chip in {".", ".."}: + raise ValueError(f"invalid chip: {chip}") + bemu_feature(chip, bbdir) + return chip diff --git a/bbdev/bbdev b/bbdev/bbdev index 72677739f..8c9e28349 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -27,11 +27,11 @@ HTTP_PORT_END = 5500 # Operations with string values accept sub-arguments; True means boolean-only (no sub-args). WORKFLOW_COMMANDS = { "verilator": { - "clean": 'Clean build directory. Args: "[--config ] [--output-dir ]"', - "verilog": 'Generate verilog files. Args: "[--config ] [--output-dir ] [--balltype ]"', - "build": 'Build simulation executable. Args: "[--job ] [--config ] [--output-dir ]"', - "sim": 'Run simulation. Args: "--binary [--batch] [--config ] [--output-dir ]"', - "run": 'Integrated build+sim+run. Args: "--binary [--batch] [--job ] [--config ] [--output-dir ]"', + "clean": 'Clean build directory. Args: "--config [--output-dir ]"', + "verilog": 'Generate verilog files. Args: "--config [--output-dir ] [--balltype ]"', + "build": 'Build simulation executable. Args: "--config [--jobs ] [--output-dir ]"', + "sim": 'Run simulation. Args: "--config --binary [--batch] [--output-dir ]"', + "run": 'Integrated build+sim+run. Args: "--config --binary [--batch] [--jobs ] [--output-dir ]"', "cosim": 'Run cosimulation. Args: "--binary [--batch]"', }, "vcs": { @@ -42,7 +42,7 @@ WORKFLOW_COMMANDS = { "run": 'Integrated build+sim+run. Args: "--binary [--batch] [--job ]"', }, "workload": { - "build": 'Build workload. Args: "[--model ]"', + "build": 'Build workload. Args: "[--model ] [--stable]"', "tohex": "Convert all -baremetal ELF files under bb-tests/output/workloads/src to hex.", }, "kernel": { @@ -55,24 +55,24 @@ WORKFLOW_COMMANDS = { "runworkload": "Run workload.", }, "compiler": { - "build": "Build compiler.", + "build": 'Build compiler. Args: "--chip [--stable]"', }, "bebop-bemu": { - "sim": 'Run Bebop emulator simulation. Args: "--binary [--pk] [--log-dir ]"', - "batch": 'Run Bebop bemu nextest batch regression (parallel). Args: "--test "', + "sim": 'Run Bebop emulator simulation. Args: "--chip --binary [--pk] [--log-dir ]"', + "batch": 'Run Bebop bemu nextest batch regression (parallel). Args: "--chip --test "', }, "bebop-verilator": { - "clean": 'Clean Bebop Verilator build directory. Args: "[--config ] [--output-dir ]"', - "verilog": 'Generate Bebop Verilog files. Args: "[--config ] [--output-dir ]"', - "build": 'Build Bebop Verilator simulation executable. Args: "[--jobs ] [--config ] [--output-dir ]"', - "sim": 'Run Bebop Verilator simulation. Args: "--binary [--batch] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', + "clean": 'Clean Bebop Verilator build directory. Args: "--config [--output-dir ]"', + "verilog": 'Generate Bebop Verilog files. Args: "--config [--output-dir ]"', + "build": 'Build Bebop Verilator simulation executable. Args: "--config [--jobs ] [--output-dir ]"', + "sim": 'Run Bebop Verilator simulation. Args: "--config --binary [--batch] [--log-dir ] [--fst-dir ] [--no-wave]"', "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--config --test "', }, "bebop-p2e": { - "clean": 'Clean Bebop P2E build directory. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', - "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', - "buildbitstream": 'Build Bebop P2E runtime case. Args: "[--config sims.p2e.P2EToyConfig] [--output-dir ]"', + "clean": 'Clean Bebop P2E build directory. Args: "--config [--output-dir ]"', + "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--config [--output-dir ]"', + "buildbitstream": 'Build Bebop P2E runtime case. Args: "--config [--vsrc-dir ] [--output-dir ]"', "runworkload": 'Run Bebop P2E workload. Args: "--image --bitstream [--multi-fpga] [--wave] [--wave-start ] [--build-dir ]"', "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--bitstream --build-dir --test "', }, @@ -84,6 +84,10 @@ WORKFLOW_COMMANDS = { "verilog": 'Generate verilog for yosys flow. Args: "[--config ] [--output-dir ] [--log-dir ]"', "synth": 'Run yosys synthesis. Args: "[--top ] [--output-dir ] [--log-dir ]"', }, + "uvm": { + "build": 'Build a Ball UVM simulation. Args: "--ball= [--filelist ]"', + "run": 'Build and run a Ball UVM simulation. Args: "--ball= [--filelist ] [--test ]"', + }, } ALL_COMMANDS = ["start", "stop"] + list(WORKFLOW_COMMANDS) @@ -132,6 +136,12 @@ def cleanup_config(config_path: str): def parse_args(argv: list[str]) -> argparse.Namespace: + try: + argv = _normalize_workflow_args(argv) + except ValueError as e: + print(f"bbdev: error: {e}", file=sys.stderr) + sys.exit(2) + parser = argparse.ArgumentParser( prog="bbdev", description="Development tool for buckyball project", @@ -174,6 +184,55 @@ def parse_args(argv: list[str]) -> argparse.Namespace: return parser.parse_args(argv) +def _normalize_workflow_args(argv: list[str]) -> list[str]: + """Attach operation sub-arguments to the operation flag before argparse sees them.""" + if not argv: + return argv + + command = argv[0] + if command not in WORKFLOW_COMMANDS: + return argv + + operations = WORKFLOW_COMMANDS[command] + out = [command] + i = 1 + while i < len(argv): + token = argv[i] + if token.startswith("--") and "=" in token: + op_name, _value = token[2:].split("=", 1) + if op_name in operations and operations[op_name] is not True: + raise ValueError( + f"--{op_name} sub-arguments must be passed as a separate quoted string" + ) + if token.startswith("--") and "=" not in token: + op_name = token[2:] + if op_name in operations and operations[op_name] is not True: + values = [] + j = i + 1 + while j < len(argv): + next_token = argv[j] + next_key = next_token[2:].split("=", 1)[0] if next_token.startswith("--") else "" + if next_key in operations: + break + values.append(next_token) + j += 1 + if values: + if len(values) > 1 and values[0].startswith("--"): + raise ValueError( + f"{token} sub-arguments must be quoted as one string" + ) + value = values[0] if len(values) == 1 else shlex.join(values) + out.append(f"{token}={value}") + else: + out.append(token) + i = j + continue + out.append(token) + i += 1 + + return out + + def extract_command_info(args: argparse.Namespace) -> dict: """Extract command, operation, and parsed sub-arguments from argparse Namespace.""" result = {"command": args.command, "operation": None, "args": {}} @@ -188,7 +247,11 @@ def extract_command_info(args: argparse.Namespace) -> dict: if attr_value is True or attr_value == "": result["args"] = {} else: - result["args"] = _parse_sub_args(attr_value) + try: + result["args"] = _parse_sub_args(attr_value) + except ValueError as e: + print(f"bbdev: error: {e}", file=sys.stderr) + sys.exit(2) break return result @@ -200,20 +263,28 @@ def _parse_sub_args(arg_string: str) -> dict: try: tokens = shlex.split(arg_string) except ValueError as e: - print(f"Error parsing arguments: {e}") - return args_dict + raise ValueError(f"failed to parse sub-arguments: {e}") from e i = 0 while i < len(tokens): token = tokens[i] if token.startswith("--"): - key = token[2:] - if i + 1 < len(tokens) and not tokens[i + 1].startswith("--"): - args_dict[key] = tokens[i + 1] - i += 2 - else: - args_dict[key] = True + if "=" in token: + key, value = token[2:].split("=", 1) + if not key or value == "": + raise ValueError(f"Invalid argument: {token}") + args_dict[key] = value i += 1 + else: + key = token[2:] + if not key: + raise ValueError(f"Invalid argument: {token}") + if i + 1 < len(tokens) and not tokens[i + 1].startswith("--"): + args_dict[key] = tokens[i + 1] + i += 2 + else: + args_dict[key] = True + i += 1 elif token.startswith("-") and len(token) == 2: key = token[1:] if i + 1 < len(tokens) and not tokens[i + 1].startswith("--"): @@ -223,7 +294,7 @@ def _parse_sub_args(arg_string: str) -> dict: args_dict[key] = True i += 1 else: - i += 1 + raise ValueError(f"Unexpected argument: {token}") return args_dict From e7a138eb783e7ac1ea359014dbbeb782bf39adc2 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 4 Jul 2026 19:10:38 +0800 Subject: [PATCH 085/196] refactor: reorganize bebop bemu and yosys scripts, moving common functions to dedicated modules --- bbdev/api/steps/bebop/bemu/02_sim_api.step.py | 5 ++- .../api/steps/bebop/bemu/02_sim_event.step.py | 11 +++-- .../api/steps/bebop/bemu/03_batch_api.step.py | 5 ++- .../steps/bebop/bemu/03_batch_event.step.py | 14 ++++--- .../steps/bebop/bemu/scripts/bemu_common.py | 28 +++++++++++++ .../bebop/verilator/03_build_event.step.py | 5 ++- .../bebop/verilator/04_sim_event.step.py | 5 ++- .../bebop/verilator/scripts/build_marker.py} | 0 bbdev/api/steps/yosys/01_run_api.step.py | 10 ++++- bbdev/api/steps/yosys/02_verilog_api.step.py | 10 ++++- bbdev/api/steps/yosys/03_synth_api.step.py | 10 ++++- bbdev/api/steps/yosys/scripts/yosys_log.py | 12 ++++++ bbdev/api/utils/bemu.py | 41 ------------------- bbdev/api/utils/yosys_log.py | 12 ------ 14 files changed, 100 insertions(+), 68 deletions(-) create mode 100644 bbdev/api/steps/bebop/bemu/scripts/bemu_common.py rename bbdev/api/{utils/bebop_verilator.py => steps/bebop/verilator/scripts/build_marker.py} (100%) create mode 100644 bbdev/api/steps/yosys/scripts/yosys_log.py delete mode 100644 bbdev/api/utils/bemu.py delete mode 100644 bbdev/api/utils/yosys_log.py diff --git a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py index 4c35bd264..24b467ad1 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py @@ -6,9 +6,12 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) -from utils.bemu import validate_bemu_chip from utils.path import get_buckyball_path +from bemu_common import validate_bemu_chip config = { "name": "bebop-bemu-sim-api", diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index b36270ee1..68b9568f5 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -17,12 +17,15 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id -from utils.bemu import bemu_feature +from bemu_common import bemu_env PERFETTO_TARGETS = { "buddy-buckyball-lenet-run": "buddy-buckyball-lenet-perfetto", @@ -78,7 +81,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return try: - feature = bemu_feature(chip) + env = os.environ.copy() + env.update(bemu_env(chip, bbdir)) except ValueError as e: ctx.logger.error(str(e)) await check_result( @@ -115,7 +119,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--manifest-path", f"{bebop_dir}/Cargo.toml", "--features", - feature, + "bemu", "--", "run", "bemu", @@ -135,6 +139,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cwd=bbdir, stdout_prefix="bebop bemu", stderr_prefix="bebop bemu", + env=env, ) if run_result.returncode != 0: await check_result( diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py index f72cb2b2d..763b6a9db 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -6,9 +6,12 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) -from utils.bemu import validate_bemu_chip from utils.path import get_buckyball_path +from bemu_common import validate_bemu_chip config = { "name": "bebop-bemu-batch-api", diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index 977397924..fbf0ad26d 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -14,11 +14,14 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id -from utils.bemu import bemu_feature +from bemu_common import bemu_env config = { "name": "bebop-bemu-batch", @@ -46,7 +49,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return try: - feature = bemu_feature(chip) + env = os.environ.copy() + env.update(bemu_env(chip, bbdir)) except ValueError as e: ctx.logger.error(str(e)) await check_result( @@ -76,7 +80,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── Build bebop bemu ────────────────────────────────────────────────── build_cmd = ( f"nix develop -c cargo build --manifest-path {shlex.quote(f'{bebop_dir}/Cargo.toml')} " - f"--features {feature} --tests" + "--features bemu --tests" ) ctx.logger.info("Building bebop bemu (tests)...") build_result = stream_run_logger( @@ -85,6 +89,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cwd=bbdir, stdout_prefix="bebop bemu build", stderr_prefix="bebop bemu build", + env=env, ) if build_result.returncode != 0: @@ -97,14 +102,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── Run nextest ─────────────────────────────────────────────────────── # Pass parameters via environment variables (nextest doesn't support custom CLI args after `--`) - env = os.environ.copy() env.update({ "BEBOP_WORKLOAD_TOML": workload_toml, "BEBOP_BB_TESTS_ROOT": elf_root, }) nextest_cmd = ( f"nix develop -c cargo nextest run --manifest-path {shlex.quote(f'{bebop_dir}/Cargo.toml')} " - f"--features {feature} --test test_bemu " + "--features bemu --test test_bemu " f"--config-file {shlex.quote(nextest_config)}" ) diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py new file mode 100644 index 000000000..ddfaba406 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py @@ -0,0 +1,28 @@ +import os +from pathlib import Path + +from utils.path import get_buckyball_path + + +def validate_bemu_chip(chip: str | None, bbdir: str | None = None) -> str: + _ = bbdir + if not chip: + raise ValueError("missing required parameter: chip") + if not isinstance(chip, str): + raise ValueError("chip must be a string") + if os.path.sep in chip or chip in {".", ".."}: + raise ValueError(f"invalid chip: {chip}") + return chip + + +def bemu_chip_inst(chip: str, bbdir: str | None = None) -> Path: + chip = validate_bemu_chip(chip, bbdir) + root = Path(bbdir or get_buckyball_path()) + inst = root / "examples" / "chips" / chip / "emu" / "src" / "lib.rs" + if not inst.is_file(): + raise ValueError(f"bemu chip instruction file does not exist: {inst}") + return inst + + +def bemu_env(chip: str, bbdir: str | None = None) -> dict[str, str]: + return {"BEBOP_BEMU_CHIP_INST": str(bemu_chip_inst(chip, bbdir))} diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index 27e8dc694..30d4ca37c 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -11,11 +11,14 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) from utils.path import get_buckyball_path, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id -from utils.bebop_verilator import write_build_marker +from build_marker import write_build_marker config = { "name": "bebop-verilator-build", diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index a849de947..414eeef00 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -17,12 +17,15 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) from utils.path import get_buckyball_path, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id -from utils.bebop_verilator import build_marker_path, read_build_marker +from build_marker import build_marker_path, read_build_marker config = { "name": "bebop-verilator-sim", diff --git a/bbdev/api/utils/bebop_verilator.py b/bbdev/api/steps/bebop/verilator/scripts/build_marker.py similarity index 100% rename from bbdev/api/utils/bebop_verilator.py rename to bbdev/api/steps/bebop/verilator/scripts/build_marker.py diff --git a/bbdev/api/steps/yosys/01_run_api.step.py b/bbdev/api/steps/yosys/01_run_api.step.py index 609400289..6bf7bcca9 100644 --- a/bbdev/api/steps/yosys/01_run_api.step.py +++ b/bbdev/api/steps/yosys/01_run_api.step.py @@ -1,7 +1,15 @@ +import os +import sys + from motia import ApiRequest, ApiResponse, FlowContext, api from utils.path import get_buckyball_path -from utils.yosys_log import make_yosys_log_dir, req_arg + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from yosys_log import make_yosys_log_dir, req_arg config = { "name": "yosys-run", diff --git a/bbdev/api/steps/yosys/02_verilog_api.step.py b/bbdev/api/steps/yosys/02_verilog_api.step.py index 88c8f199f..a472ae2d8 100644 --- a/bbdev/api/steps/yosys/02_verilog_api.step.py +++ b/bbdev/api/steps/yosys/02_verilog_api.step.py @@ -1,7 +1,15 @@ +import os +import sys + from motia import ApiRequest, ApiResponse, FlowContext, api from utils.path import get_buckyball_path -from utils.yosys_log import make_yosys_log_dir, req_arg + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from yosys_log import make_yosys_log_dir, req_arg config = { "name": "yosys-verilog-api", diff --git a/bbdev/api/steps/yosys/03_synth_api.step.py b/bbdev/api/steps/yosys/03_synth_api.step.py index 09055c907..88dd2990e 100644 --- a/bbdev/api/steps/yosys/03_synth_api.step.py +++ b/bbdev/api/steps/yosys/03_synth_api.step.py @@ -1,7 +1,15 @@ +import os +import sys + from motia import ApiRequest, ApiResponse, FlowContext, api from utils.path import get_buckyball_path -from utils.yosys_log import make_yosys_log_dir, req_arg + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from yosys_log import make_yosys_log_dir, req_arg config = { "name": "yosys-synth-api", diff --git a/bbdev/api/steps/yosys/scripts/yosys_log.py b/bbdev/api/steps/yosys/scripts/yosys_log.py new file mode 100644 index 000000000..e58811beb --- /dev/null +++ b/bbdev/api/steps/yosys/scripts/yosys_log.py @@ -0,0 +1,12 @@ +import os +from datetime import datetime + + +def req_arg(body: dict, name: str): + return body.get(name) or body.get(name.replace("_", "-")) + + +def make_yosys_log_dir(bbdir: str, trace_id: str) -> str: + stamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") + suffix = trace_id[:8] if trace_id else "no-trace" + return os.path.join(bbdir, "bbdev", "api", "steps", "yosys", "log", f"{stamp}-{suffix}") diff --git a/bbdev/api/utils/bemu.py b/bbdev/api/utils/bemu.py deleted file mode 100644 index 8a54ecd7d..000000000 --- a/bbdev/api/utils/bemu.py +++ /dev/null @@ -1,41 +0,0 @@ -import os -import tomllib -from pathlib import Path - -from utils.path import get_buckyball_path - - -def available_bemu_chips(bbdir: str | None = None) -> list[str]: - root = Path(bbdir or get_buckyball_path()) - cargo_toml = root / "bebop" / "Cargo.toml" - with open(cargo_toml, "rb") as f: - manifest = tomllib.load(f) - - features = manifest.get("features", {}) - chips = [] - for name in features: - if not name.startswith("bemu-"): - continue - chip = name.removeprefix("bemu-") - if (root / "examples" / "chips" / chip / "emu" / "Cargo.toml").is_file(): - chips.append(chip) - return sorted(chips) - - -def bemu_feature(chip: str, bbdir: str | None = None) -> str: - chips = available_bemu_chips(bbdir) - if chip not in chips: - available = ", ".join(chips) if chips else "none" - raise ValueError(f"invalid chip: {chip}; available bemu chips: {available}") - return f"bemu-{chip}" - - -def validate_bemu_chip(chip: str | None, bbdir: str | None = None) -> str: - if not chip: - raise ValueError("missing required parameter: chip") - if not isinstance(chip, str): - raise ValueError("chip must be a string") - if os.path.sep in chip or chip in {".", ".."}: - raise ValueError(f"invalid chip: {chip}") - bemu_feature(chip, bbdir) - return chip diff --git a/bbdev/api/utils/yosys_log.py b/bbdev/api/utils/yosys_log.py deleted file mode 100644 index 613cee49e..000000000 --- a/bbdev/api/utils/yosys_log.py +++ /dev/null @@ -1,12 +0,0 @@ -import os -from datetime import datetime - - -def req_arg(body: dict, name: str): - return body.get(name) or body.get(name.replace("_", "-")) - - -def make_yosys_log_dir(bbdir: str, trace_id: str) -> str: - stamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") - suffix = trace_id[:8] if trace_id else "no-trace" - return os.path.join(bbdir, "bbdev", "api", "steps", "yosys", "log", f"{stamp}-{suffix}") From 95243724b25f5ff0731fb97b79f6f1d9a7325983 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 4 Jul 2026 21:27:27 +0800 Subject: [PATCH 086/196] fix: update UVM API steps --- bbdev/api/steps/uvm/01_build_api.step.py | 10 +- bbdev/api/steps/uvm/01_build_event.step.py | 2 +- bbdev/api/steps/uvm/02_run_event.step.py | 41 ++- bbdev/api/steps/uvm/scripts/__init__.py | 1 + bbdev/api/steps/uvm/scripts/uvm_common.py | 276 +++++++++++++++++++++ bbdev/api/steps/uvm/uvm_common.py | 163 ------------ 6 files changed, 316 insertions(+), 177 deletions(-) create mode 100644 bbdev/api/steps/uvm/scripts/__init__.py create mode 100644 bbdev/api/steps/uvm/scripts/uvm_common.py delete mode 100644 bbdev/api/steps/uvm/uvm_common.py diff --git a/bbdev/api/steps/uvm/01_build_api.step.py b/bbdev/api/steps/uvm/01_build_api.step.py index 11dd79613..3684868e5 100644 --- a/bbdev/api/steps/uvm/01_build_api.step.py +++ b/bbdev/api/steps/uvm/01_build_api.step.py @@ -10,13 +10,15 @@ def check_args(body: dict) -> str | None: - allowed = {"ball", "filelist"} + allowed = {"ball", "config", "filelist"} for key in body: if key not in allowed: return f"Unexpected parameter: --{key}" - ball = body.get("ball") - if not ball or ball is True: - return "Missing required parameter: --ball=" + config = body.get("config") + if not config or config is True: + return "Missing required parameter: --config=" + if body.get("ball") is True: + return "Parameter --ball requires a value" if body.get("filelist") is True: return "Parameter --filelist requires a path value" return None diff --git a/bbdev/api/steps/uvm/01_build_event.step.py b/bbdev/api/steps/uvm/01_build_event.step.py index 5e2baf7ac..56bf71d24 100644 --- a/bbdev/api/steps/uvm/01_build_event.step.py +++ b/bbdev/api/steps/uvm/01_build_event.step.py @@ -12,7 +12,7 @@ from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path -from uvm_common import run_uvm_build +from scripts.uvm_common import run_uvm_build config = { "name": "uvm-build", diff --git a/bbdev/api/steps/uvm/02_run_event.step.py b/bbdev/api/steps/uvm/02_run_event.step.py index a0286e83c..291c952e1 100644 --- a/bbdev/api/steps/uvm/02_run_event.step.py +++ b/bbdev/api/steps/uvm/02_run_event.step.py @@ -14,7 +14,7 @@ from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger -from uvm_common import default_test_name, run_uvm_build, uvm_paths +from scripts.uvm_common import checked_run_paths, default_test_name config = { "name": "uvm-run", @@ -28,19 +28,40 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() + info = {"task": "run", "ball": input_data.get("ball")} - build_result, info = run_uvm_build(bbdir, input_data, ctx) - if build_result.returncode != 0: + try: + paths = checked_run_paths(bbdir, input_data) + except Exception as e: + ctx.logger.error(str(e)) await check_result( ctx, - build_result.returncode, + 1, + continue_run=False, + extra_fields={**info, "error": str(e)}, + trace_id=origin_tid, + ) + return + + info.update({ + "ball": paths["ball"], + "verify_dir": paths["verify_dir"], + "simv": paths["simv"], + "dpi_lib": paths["dpi_lib"], + }) + + if not os.path.isfile(paths["simv"]): + error = f"simv not found: {paths['simv']}. Run `bbdev uvm --build \"--config \"` first." + ctx.logger.error(error) + await check_result( + ctx, + 1, continue_run=False, - extra_fields=info, + extra_fields={**info, "error": error}, trace_id=origin_tid, ) return - paths = uvm_paths(bbdir, input_data) test = input_data.get("test") or default_test_name(paths["ball"]) if test is True: ctx.logger.error("Parameter --test requires a value") @@ -55,9 +76,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: script = ( f"cd {shlex.quote(paths['verify_dir'])} && " - f"{shlex.quote(paths['simv'])} +UVM_TESTNAME={shlex.quote(test)}" + f"{shlex.quote(paths['simv'])} " + f"-sv_lib {shlex.quote(paths['dpi_lib'])} " + f"+UVM_TESTNAME={shlex.quote(test)}" ) - cmd = f"nix develop {shlex.quote(paths['verify_env'])} --command zsh -lc {shlex.quote(script)}" + cmd = f"nix develop {shlex.quote(paths['verify_env'])} --command bash -lc {shlex.quote(script)}" ctx.logger.info(f"Running UVM test {test} for ball {paths['ball']}") run_result = stream_run_logger( @@ -72,6 +95,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, run_result.returncode, continue_run=False, - extra_fields={**info, "task": "run", "test": test}, + extra_fields={**info, "test": test}, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/uvm/scripts/__init__.py b/bbdev/api/steps/uvm/scripts/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/__init__.py @@ -0,0 +1 @@ + diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py new file mode 100644 index 000000000..43de3896d --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -0,0 +1,276 @@ +import os +import re +import shlex +import sys +import tomllib + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.stream_run import StreamResult, stream_run_logger + + +def default_test_name(ball: str) -> str: + return f"{ball}_ball_test" + + +def config_dir_name(config: str) -> str: + if not isinstance(config, str) or not re.fullmatch(r"[A-Za-z0-9_.-]+", config): + raise ValueError("invalid config name") + return config + + +def uvm_paths(bbdir: str, input_data: dict, ball_override: str | None = None) -> dict: + ball = ball_override or input_data.get("ball") + if not isinstance(ball, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", ball): + raise ValueError("invalid ball name") + + config = input_data.get("config") + config_dir = config_dir_name(config) if config else None + ball_dir = os.path.join(bbdir, "examples", "balls", ball) + verify_dir = os.path.join(ball_dir, "verify") + casegen_dir = os.path.join(verify_dir, "casegen") + verify_env = os.path.join(bbdir, "verify") + build_dir = os.path.join(verify_dir, "build") + sim_dir = os.path.join(build_dir, config_dir) if config_dir else os.path.join(build_dir, "current") + simv = os.path.join(sim_dir, "simv") + csrc_dir = os.path.join(sim_dir, "csrc") + rtl_dir = os.path.join(bbdir, "arch", "build", config_dir) if config_dir else None + + return { + "ball": ball, + "config": config_dir, + "ball_dir": ball_dir, + "verify_dir": verify_dir, + "casegen_dir": casegen_dir, + "verify_env": verify_env, + "build_dir": build_dir, + "sim_dir": sim_dir, + "simv": simv, + "csrc_dir": csrc_dir, + "rtl_dir": rtl_dir, + } + + +def checked_paths(bbdir: str, input_data: dict, ball_override: str | None = None) -> dict: + paths = uvm_paths(bbdir, input_data, ball_override) + for key in ("ball_dir", "verify_dir", "casegen_dir", "verify_env"): + if not os.path.isdir(paths[key]): + raise FileNotFoundError(f"{key} not found: {paths[key]}") + if paths["rtl_dir"] and not os.path.isdir(paths["rtl_dir"]): + raise FileNotFoundError(f"rtl_dir not found: {paths['rtl_dir']}") + + cargo_toml = os.path.join(paths["casegen_dir"], "Cargo.toml") + if not os.path.isfile(cargo_toml): + raise FileNotFoundError(f"Cargo.toml not found: {cargo_toml}") + + filelist = resolve_filelist(paths["verify_dir"], input_data.get("filelist"), paths["ball"]) + crate = read_crate_name(cargo_toml) + dpi_lib = os.path.join(paths["casegen_dir"], "target", "debug", f"lib{crate.replace('-', '_')}") + + paths.update({ + "cargo_toml": cargo_toml, + "filelist": filelist, + "filelist_arg": prepare_filelist(paths, filelist), + "crate": crate, + "dpi_lib": dpi_lib, + }) + return paths + + +def checked_run_paths(bbdir: str, input_data: dict) -> dict: + paths = uvm_paths(bbdir, input_data) + for key in ("ball_dir", "verify_dir", "casegen_dir", "verify_env"): + if not os.path.isdir(paths[key]): + raise FileNotFoundError(f"{key} not found: {paths[key]}") + + cargo_toml = os.path.join(paths["casegen_dir"], "Cargo.toml") + if not os.path.isfile(cargo_toml): + raise FileNotFoundError(f"Cargo.toml not found: {cargo_toml}") + + crate = read_crate_name(cargo_toml) + dpi_lib = os.path.join(paths["casegen_dir"], "target", "debug", f"lib{crate.replace('-', '_')}") + paths.update({ + "cargo_toml": cargo_toml, + "crate": crate, + "dpi_lib": dpi_lib, + }) + return paths + + +def discover_uvm_balls(bbdir: str) -> list[str]: + balls_dir = os.path.join(bbdir, "examples", "balls") + if not os.path.isdir(balls_dir): + return [] + balls = [] + for name in sorted(os.listdir(balls_dir)): + if not re.fullmatch(r"[A-Za-z0-9_-]+", name): + continue + verify_dir = os.path.join(balls_dir, name, "verify") + filelist_dir = os.path.join(verify_dir, "filelists") + casegen_toml = os.path.join(verify_dir, "casegen", "Cargo.toml") + if os.path.isdir(filelist_dir) and os.path.isfile(casegen_toml): + balls.append(name) + return balls + + +def resolve_filelist(verify_dir: str, arg, ball: str) -> str: + if arg is True: + raise ValueError("parameter --filelist requires a path value") + + if arg: + path = arg if os.path.isabs(arg) else os.path.join(verify_dir, arg) + if not os.path.isfile(path): + raise FileNotFoundError(f"filelist not found: {path}") + return os.path.abspath(path) + + default = os.path.join(verify_dir, "filelists", f"{ball}_ball_toy.f") + if os.path.isfile(default): + return default + + filelist_dir = os.path.join(verify_dir, "filelists") + if not os.path.isdir(filelist_dir): + raise FileNotFoundError(f"filelist directory not found: {filelist_dir}") + found = [ + os.path.join(filelist_dir, name) + for name in os.listdir(filelist_dir) + if name.endswith(".f") + ] + if len(found) != 1: + raise RuntimeError(f"expected exactly one filelist under {filelist_dir}, found {len(found)}") + return found[0] + + +def filelist_arg(verify_dir: str, filelist: str) -> str: + rel = os.path.relpath(filelist, verify_dir) + if rel.startswith(".."): + return filelist + return rel + + +def prepare_filelist(paths: dict, filelist: str) -> str: + os.makedirs(paths["sim_dir"], exist_ok=True) + generated = os.path.join(paths["sim_dir"], os.path.basename(filelist)) + rel_rtl_dir = os.path.relpath(paths["rtl_dir"], paths["verify_dir"]) if paths["rtl_dir"] else None + rel_uvm_dir = os.path.relpath(os.path.join(paths["verify_env"], "uvm"), paths["verify_dir"]) + with open(filelist, "r") as src, open(generated, "w") as dst: + for line in src: + original = line + line = line.replace("@UVM@", rel_uvm_dir) + if "@RTL@" in line: + if not rel_rtl_dir: + raise ValueError("filelist uses @RTL@ but --config was not provided") + line = line.replace("@RTL@", rel_rtl_dir) + elif re.search(r"(?:\.\./)+arch/build/[^/\s]+/", original): + raise ValueError("filelist must use @RTL@ instead of hard-coded arch/build/") + if re.search(r"(?:\.\./)+verify/uvm/", original): + raise ValueError("filelist must use @UVM@ instead of hard-coded verify/uvm") + dst.write(line) + return filelist_arg(paths["verify_dir"], generated) + + +def read_crate_name(cargo_toml: str) -> str: + with open(cargo_toml, "rb") as f: + data = tomllib.load(f) + name = data.get("package", {}).get("name") + if not name: + raise ValueError(f"package.name not found in {cargo_toml}") + return name + + +def failed_result(message: str) -> StreamResult: + return StreamResult(returncode=1, stdout="", stderr=message) + + +def run_uvm_build_one(bbdir: str, input_data: dict, ctx, ball: str) -> tuple[StreamResult, dict]: + try: + paths = checked_paths(bbdir, input_data, ball) + except Exception as e: + ctx.logger.error(str(e)) + return failed_result(str(e)), {"task": "build", "error": str(e)} + + info = { + "task": "build", + "ball": paths["ball"], + "config": paths["config"], + "verify_dir": paths["verify_dir"], + "filelist": paths["filelist"], + "resolved_filelist": os.path.join(paths["verify_dir"], paths["filelist_arg"]), + "simv": paths["simv"], + "dpi_lib": paths["dpi_lib"], + } + + cargo_cmd = ( + f"nix develop {shlex.quote(paths['verify_env'])} --command " + f"cargo build --manifest-path {shlex.quote(paths['cargo_toml'])}" + ) + ctx.logger.info(f"Building UVM DPI reference for ball {paths['ball']}") + cargo_result = stream_run_logger( + cmd=cargo_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm dpi build", + stderr_prefix="uvm dpi build", + ) + if cargo_result.returncode != 0: + return cargo_result, info + + script = ( + f"cd {shlex.quote(paths['verify_dir'])} && " + f"rm -rf {shlex.quote(paths['csrc_dir'])} {shlex.quote(paths['simv'])} {shlex.quote(paths['simv'])}.daidir && " + f"mkdir -p {shlex.quote(paths['sim_dir'])} {shlex.quote(paths['csrc_dir'])} && " + "vcs -full64 -sverilog -timescale=1ns/1ps -hsopt=off " + "$VCS_UVM_ARGS " + f"-Mdir={shlex.quote(paths['csrc_dir'])} " + f"-o {shlex.quote(paths['simv'])} " + f"-f {shlex.quote(paths['filelist_arg'])}" + ) + vcs_cmd = f"nix develop {shlex.quote(paths['verify_env'])} --command bash -lc {shlex.quote(script)}" + + ctx.logger.info(f"Building UVM simulation for ball {paths['ball']} config {paths['config']}") + vcs_result = stream_run_logger( + cmd=vcs_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm vcs build", + stderr_prefix="uvm vcs build", + ) + if vcs_result.returncode == 0: + current = os.path.join(paths["build_dir"], "current") + os.makedirs(paths["build_dir"], exist_ok=True) + tmp = f"{current}.tmp" + try: + os.unlink(tmp) + except FileNotFoundError: + pass + os.symlink(paths["config"], tmp) + os.replace(tmp, current) + return vcs_result, info + + +def run_uvm_build(bbdir: str, input_data: dict, ctx) -> tuple[StreamResult, dict]: + config = input_data.get("config") + if not config or config is True: + return failed_result("Missing required parameter: --config="), { + "task": "build", + "error": "missing_config", + } + + ball = input_data.get("ball") + balls = [ball] if ball and ball is not True else discover_uvm_balls(bbdir) + if not balls: + return failed_result("No UVM balls found"), {"task": "build", "error": "no_uvm_balls"} + + built = [] + for one_ball in balls: + result, info = run_uvm_build_one(bbdir, input_data, ctx, one_ball) + if result.returncode != 0: + return result, {**info, "built": built} + built.append(one_ball) + + return StreamResult(returncode=0, stdout="", stderr=""), { + "task": "build", + "config": config, + "built": built, + } diff --git a/bbdev/api/steps/uvm/uvm_common.py b/bbdev/api/steps/uvm/uvm_common.py deleted file mode 100644 index bc88471d1..000000000 --- a/bbdev/api/steps/uvm/uvm_common.py +++ /dev/null @@ -1,163 +0,0 @@ -import os -import re -import shlex -import sys -import tomllib - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.stream_run import StreamResult, stream_run_logger - - -def default_test_name(ball: str) -> str: - return f"{ball}_ball_test" - - -def uvm_paths(bbdir: str, input_data: dict) -> dict: - ball = input_data.get("ball") - if not isinstance(ball, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", ball): - raise ValueError("invalid ball name") - - ball_dir = os.path.join(bbdir, "examples", "balls", ball) - verify_dir = os.path.join(ball_dir, "verify") - casegen_dir = os.path.join(verify_dir, "casegen") - verify_env = os.path.join(bbdir, "verify") - sim_dir = os.path.join(verify_dir, "build") - simv = os.path.join(sim_dir, "simv") - csrc_dir = os.path.join(sim_dir, "csrc") - - return { - "ball": ball, - "ball_dir": ball_dir, - "verify_dir": verify_dir, - "casegen_dir": casegen_dir, - "verify_env": verify_env, - "sim_dir": sim_dir, - "simv": simv, - "csrc_dir": csrc_dir, - } - - -def checked_paths(bbdir: str, input_data: dict) -> dict: - paths = uvm_paths(bbdir, input_data) - for key in ("ball_dir", "verify_dir", "casegen_dir", "verify_env"): - if not os.path.isdir(paths[key]): - raise FileNotFoundError(f"{key} not found: {paths[key]}") - - cargo_toml = os.path.join(paths["casegen_dir"], "Cargo.toml") - if not os.path.isfile(cargo_toml): - raise FileNotFoundError(f"Cargo.toml not found: {cargo_toml}") - - filelist = resolve_filelist(paths["verify_dir"], input_data.get("filelist"), paths["ball"]) - crate = read_crate_name(cargo_toml) - dpi_lib = os.path.join(paths["casegen_dir"], "target", "debug", f"lib{crate.replace('-', '_')}") - - paths.update({ - "cargo_toml": cargo_toml, - "filelist": filelist, - "filelist_arg": filelist_arg(paths["verify_dir"], filelist), - "crate": crate, - "dpi_lib": dpi_lib, - }) - return paths - - -def resolve_filelist(verify_dir: str, arg, ball: str) -> str: - if arg is True: - raise ValueError("parameter --filelist requires a path value") - - if arg: - path = arg if os.path.isabs(arg) else os.path.join(verify_dir, arg) - if not os.path.isfile(path): - raise FileNotFoundError(f"filelist not found: {path}") - return os.path.abspath(path) - - default = os.path.join(verify_dir, "filelists", f"{ball}_ball_toy.f") - if os.path.isfile(default): - return default - - filelist_dir = os.path.join(verify_dir, "filelists") - if not os.path.isdir(filelist_dir): - raise FileNotFoundError(f"filelist directory not found: {filelist_dir}") - found = [ - os.path.join(filelist_dir, name) - for name in os.listdir(filelist_dir) - if name.endswith(".f") - ] - if len(found) != 1: - raise RuntimeError(f"expected exactly one filelist under {filelist_dir}, found {len(found)}") - return found[0] - - -def filelist_arg(verify_dir: str, filelist: str) -> str: - rel = os.path.relpath(filelist, verify_dir) - if rel.startswith(".."): - return filelist - return rel - - -def read_crate_name(cargo_toml: str) -> str: - with open(cargo_toml, "rb") as f: - data = tomllib.load(f) - name = data.get("package", {}).get("name") - if not name: - raise ValueError(f"package.name not found in {cargo_toml}") - return name - - -def failed_result(message: str) -> StreamResult: - return StreamResult(returncode=1, stdout="", stderr=message) - - -def run_uvm_build(bbdir: str, input_data: dict, ctx) -> tuple[StreamResult, dict]: - try: - paths = checked_paths(bbdir, input_data) - except Exception as e: - ctx.logger.error(str(e)) - return failed_result(str(e)), {"task": "build", "error": str(e)} - - info = { - "task": "build", - "ball": paths["ball"], - "verify_dir": paths["verify_dir"], - "filelist": paths["filelist"], - "simv": paths["simv"], - } - - cargo_cmd = ( - f"nix develop {shlex.quote(paths['verify_env'])} --command " - f"cargo build --manifest-path {shlex.quote(paths['cargo_toml'])}" - ) - ctx.logger.info(f"Building UVM DPI reference for ball {paths['ball']}") - cargo_result = stream_run_logger( - cmd=cargo_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="uvm dpi build", - stderr_prefix="uvm dpi build", - ) - if cargo_result.returncode != 0: - return cargo_result, info - - script = ( - f"cd {shlex.quote(paths['verify_dir'])} && " - f"mkdir -p {shlex.quote(paths['sim_dir'])} {shlex.quote(paths['csrc_dir'])} && " - "vcs -full64 -sverilog -timescale=1ns/1ps " - "$VCS_UVM_ARGS " - f"-sv_lib {shlex.quote(paths['dpi_lib'])} " - f"-Mdir {shlex.quote(paths['csrc_dir'])} " - f"-o {shlex.quote(paths['simv'])} " - f"-f {shlex.quote(paths['filelist_arg'])}" - ) - vcs_cmd = f"nix develop {shlex.quote(paths['verify_env'])} --command zsh -lc {shlex.quote(script)}" - - ctx.logger.info(f"Building UVM simulation for ball {paths['ball']}") - return stream_run_logger( - cmd=vcs_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="uvm vcs build", - stderr_prefix="uvm vcs build", - ), info From 9c23d8cd15b5edc0ff52f93dd276268d17519e1e Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 5 Jul 2026 11:35:04 +0800 Subject: [PATCH 087/196] refactor: replace validate_bemu_chip with bemu_chip_inst in bebop API steps --- bbdev/api/steps/bebop/bemu/02_sim_api.step.py | 4 ++-- bbdev/api/steps/bebop/bemu/03_batch_api.step.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py index 24b467ad1..1b26a061a 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py @@ -11,7 +11,7 @@ sys.path.insert(0, scripts_path) from utils.path import get_buckyball_path -from bemu_common import validate_bemu_chip +from bemu_common import bemu_chip_inst config = { "name": "bebop-bemu-sim-api", @@ -26,7 +26,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} chip = body.get("chip", "") try: - validate_bemu_chip(chip, get_buckyball_path()) + bemu_chip_inst(chip, get_buckyball_path()) except ValueError as e: return ApiResponse( status=400, diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py index 763b6a9db..1970d7b3d 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -11,7 +11,7 @@ sys.path.insert(0, scripts_path) from utils.path import get_buckyball_path -from bemu_common import validate_bemu_chip +from bemu_common import bemu_chip_inst config = { "name": "bebop-bemu-batch-api", @@ -27,7 +27,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: chip = body.get("chip") try: - validate_bemu_chip(chip, get_buckyball_path()) + bemu_chip_inst(chip, get_buckyball_path()) except ValueError as e: return ApiResponse( status=400, From 440ed20539cc2298ddf60dd98fb8de15e94afd42 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 5 Jul 2026 14:17:49 +0800 Subject: [PATCH 088/196] fix: update test names in bebop workload configuration files for consistency --- .../bebop/bemu/scripts/workloads-elf.toml | 66 +++++++-------- .../bebop/bemu/scripts/workloads-pk.toml | 66 +++++++-------- .../bebop/p2e/scripts/workloads-elf.toml | 40 +++++----- .../steps/bebop/p2e/scripts/workloads-pk.toml | 26 +++--- .../verilator/scripts/workloads-elf.toml | 80 +++++++++---------- .../bebop/verilator/scripts/workloads-pk.toml | 64 +++++++-------- 6 files changed, 171 insertions(+), 171 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml index 00301a15b..5effb996e 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml @@ -8,39 +8,39 @@ [workloads] search_path = "workloads/src" tests = [ - "ctest_mvin_mvout_test_singlecore-baremetal", - "ctest_mvin_mvout_stride_test_singlecore-baremetal", - "ctest_vecunit_matmul_16xn_ones_singlecore-baremetal", - "ctest_vecunit_matmul_16xn_random1_singlecore-baremetal", - "ctest_vecunit_matmul_16xn_random2_singlecore-baremetal", - "ctest_vecunit_matmul_16xn_random3_singlecore-baremetal", - "ctest_vecunit_matmul_16xn_zero_random_singlecore-baremetal", - "ctest_vecunit_matmul_col_row_vector_singlecore-baremetal", - "ctest_vecunit_matmul_identity_random_singlecore-baremetal", - "ctest_vecunit_matmul_ones_singlecore-baremetal", - "ctest_vecunit_matmul_random1_singlecore-baremetal", - "ctest_vecunit_matmul_random2_singlecore-baremetal", - "ctest_vecunit_matmul_zero_random_singlecore-baremetal", - "ctest_relu_test_singlecore-baremetal", - "ctest_transpose_test_singlecore-baremetal", - "ctest_transpose_16xn_test_singlecore-baremetal", - "ctest_im2col_test_singlecore-baremetal", - "ctest_fp2int_test_singlecore-baremetal", - "ctest_int2fp_test_singlecore-baremetal", - "ctest_bdb_counter_test_singlecore-baremetal", - "ctest_bfp_test_singlecore-baremetal", - "ctest_gemmini_os_cisc_basic_test_singlecore-baremetal", - "ctest_gemmini_os_cisc_atranspose_test_singlecore-baremetal", - "ctest_gemmini_os_cisc_btranspose_test_singlecore-baremetal", - "ctest_gemmini_os_cisc_shift_test_singlecore-baremetal", - "ctest_gemmini_os_risc_atranspose_test_singlecore-baremetal", - "ctest_gemmini_os_risc_btranspose_test_singlecore-baremetal", - "ctest_gemmini_os_risc_abtranspose_test_singlecore-baremetal", - "ctest_gemmini_os_risc_shift_test_singlecore-baremetal", - "ctest_gemmini_ws_cisc_conv_test_singlecore-baremetal", - "ctest_gemmini_ws_risc_basic_test_singlecore-baremetal", - "ctest_mmio_minimal_test_singlecore-baremetal", - "ctest_mxfp2int_test_singlecore-baremetal", + "mvin_mvout_test-singlecore-baremetal", + "mvin_mvout_stride_test-singlecore-baremetal", + "vecunit_matmul_16xn_ones-singlecore-baremetal", + "vecunit_matmul_16xn_random1-singlecore-baremetal", + "vecunit_matmul_16xn_random2-singlecore-baremetal", + "vecunit_matmul_16xn_random3-singlecore-baremetal", + "vecunit_matmul_16xn_zero_random-singlecore-baremetal", + "vecunit_matmul_col_row_vector-singlecore-baremetal", + "vecunit_matmul_identity_random-singlecore-baremetal", + "vecunit_matmul_ones-singlecore-baremetal", + "vecunit_matmul_random1-singlecore-baremetal", + "vecunit_matmul_random2-singlecore-baremetal", + "vecunit_matmul_zero_random-singlecore-baremetal", + "relu_test-singlecore-baremetal", + "transpose_test-singlecore-baremetal", + "transpose_16xn_test-singlecore-baremetal", + "im2col_test-singlecore-baremetal", + "fp2int_test-singlecore-baremetal", + "int2fp_test-singlecore-baremetal", + "bdb_counter_test-singlecore-baremetal", + "bfp_test-singlecore-baremetal", + "gemmini_os_cisc_basic_test-singlecore-baremetal", + "gemmini_os_cisc_atranspose_test-singlecore-baremetal", + "gemmini_os_cisc_btranspose_test-singlecore-baremetal", + "gemmini_os_cisc_shift_test-singlecore-baremetal", + "gemmini_os_risc_atranspose_test-singlecore-baremetal", + "gemmini_os_risc_btranspose_test-singlecore-baremetal", + "gemmini_os_risc_abtranspose_test-singlecore-baremetal", + "gemmini_os_risc_shift_test-singlecore-baremetal", + "gemmini_ws_cisc_conv_test-singlecore-baremetal", + "gemmini_ws_risc_basic_test-singlecore-baremetal", + "mmio_minimal_test-singlecore-baremetal", + "mxfp2int_test-singlecore-baremetal", "optest_buckyball_matmul_64x16_16x64_singlecore-baremetal", "optest_buckyball_matmul_1024x16_16x1024_singlecore-baremetal", diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml index c68ade5f9..cd8c4aa00 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml @@ -8,39 +8,39 @@ [workloads] search_path = "workloads/src" tests = [ - "ctest_mvin_mvout_test-linux", - "ctest_mvin_mvout_stride_test-linux", - "ctest_vecunit_matmul_16xn_ones-linux", - "ctest_vecunit_matmul_16xn_random1-linux", - "ctest_vecunit_matmul_16xn_random2-linux", - "ctest_vecunit_matmul_16xn_random3-linux", - "ctest_vecunit_matmul_16xn_zero_random-linux", - "ctest_vecunit_matmul_col_row_vector-linux", - "ctest_vecunit_matmul_identity_random-linux", - "ctest_vecunit_matmul_ones-linux", - "ctest_vecunit_matmul_random1-linux", - "ctest_vecunit_matmul_random2-linux", - "ctest_vecunit_matmul_zero_random-linux", - "ctest_relu_test-linux", - "ctest_transpose_test-linux", - "ctest_transpose_16xn_test-linux", - "ctest_im2col_test-linux", - "ctest_fp2int_test-linux", - "ctest_int2fp_test-linux", - "ctest_bdb_counter_test-linux", - "ctest_bfp_test-linux", - "ctest_gemmini_os_cisc_basic_test-linux", - "ctest_gemmini_os_cisc_atranspose_test-linux", - "ctest_gemmini_os_cisc_btranspose_test-linux", - "ctest_gemmini_os_cisc_shift_test-linux", - "ctest_gemmini_os_risc_atranspose_test-linux", - "ctest_gemmini_os_risc_btranspose_test-linux", - "ctest_gemmini_os_risc_abtranspose_test-linux", - "ctest_gemmini_os_risc_shift_test-linux", - "ctest_gemmini_ws_cisc_conv_test-linux", - "ctest_gemmini_ws_risc_basic_test-linux", - "ctest_mmio_minimal_test-linux", - "ctest_mxfp2int_test-linux", + "mvin_mvout_test-linux", + "mvin_mvout_stride_test-linux", + "vecunit_matmul_16xn_ones-linux", + "vecunit_matmul_16xn_random1-linux", + "vecunit_matmul_16xn_random2-linux", + "vecunit_matmul_16xn_random3-linux", + "vecunit_matmul_16xn_zero_random-linux", + "vecunit_matmul_col_row_vector-linux", + "vecunit_matmul_identity_random-linux", + "vecunit_matmul_ones-linux", + "vecunit_matmul_random1-linux", + "vecunit_matmul_random2-linux", + "vecunit_matmul_zero_random-linux", + "relu_test-linux", + "transpose_test-linux", + "transpose_16xn_test-linux", + "im2col_test-linux", + "fp2int_test-linux", + "int2fp_test-linux", + "bdb_counter_test-linux", + "bfp_test-linux", + "gemmini_os_cisc_basic_test-linux", + "gemmini_os_cisc_atranspose_test-linux", + "gemmini_os_cisc_btranspose_test-linux", + "gemmini_os_cisc_shift_test-linux", + "gemmini_os_risc_atranspose_test-linux", + "gemmini_os_risc_btranspose_test-linux", + "gemmini_os_risc_abtranspose_test-linux", + "gemmini_os_risc_shift_test-linux", + "gemmini_ws_cisc_conv_test-linux", + "gemmini_ws_risc_basic_test-linux", + "mmio_minimal_test-linux", + "mxfp2int_test-linux", "optest_buckyball_matmul_64x16_16x64-linux", "optest_buckyball_matmul_1024x16_16x1024-linux", diff --git a/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml index 573fa4115..df6bd7b9d 100644 --- a/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml @@ -7,24 +7,24 @@ [workloads] search_path = "workloads/src" tests = [ - "ctest_bdb_counter_test_singlecore-baremetal", - "ctest_bfp_test_singlecore-baremetal", - "ctest_gemmini_os_cisc_basic_test_singlecore-baremetal", - "ctest_gemmini_os_cisc_atranspose_test_singlecore-baremetal", - "ctest_gemmini_os_cisc_btranspose_test_singlecore-baremetal", - "ctest_gemmini_os_cisc_shift_test_singlecore-baremetal", - "ctest_gemmini_os_risc_basic_test_singlecore-baremetal", - "ctest_gemmini_os_risc_atranspose_test_singlecore-baremetal", - "ctest_gemmini_os_risc_btranspose_test_singlecore-baremetal", - "ctest_gemmini_os_risc_abtranspose_test_singlecore-baremetal", - "ctest_gemmini_os_risc_shift_test_singlecore-baremetal", - "ctest_gemmini_ws_cisc_conv_test_singlecore-baremetal", - "ctest_gemmini_ws_risc_basic_test_singlecore-baremetal", - "ctest_tlb_bare_mode_test_singlecore-baremetal", - "ctest_tlb_sv39_identity_test_singlecore-baremetal", - "ctest_tlb_sfence_flush_test_singlecore-baremetal", - "ctest_tlb_repeated_sfence_test_singlecore-baremetal", - "ctest_tlb_sv39_nonidentity_test_singlecore-baremetal", - "ctest_tlb_sv39_4k_alias_test_singlecore-baremetal", - "ctest_tlb_sfence_remap_test_singlecore-baremetal", + "bdb_counter_test-singlecore-baremetal", + "bfp_test-singlecore-baremetal", + "gemmini_os_cisc_basic_test-singlecore-baremetal", + "gemmini_os_cisc_atranspose_test-singlecore-baremetal", + "gemmini_os_cisc_btranspose_test-singlecore-baremetal", + "gemmini_os_cisc_shift_test-singlecore-baremetal", + "gemmini_os_risc_basic_test-singlecore-baremetal", + "gemmini_os_risc_atranspose_test-singlecore-baremetal", + "gemmini_os_risc_btranspose_test-singlecore-baremetal", + "gemmini_os_risc_abtranspose_test-singlecore-baremetal", + "gemmini_os_risc_shift_test-singlecore-baremetal", + "gemmini_ws_cisc_conv_test-singlecore-baremetal", + "gemmini_ws_risc_basic_test-singlecore-baremetal", + "tlb_bare_mode_test-singlecore-baremetal", + "tlb_sv39_identity_test-singlecore-baremetal", + "tlb_sfence_flush_test-singlecore-baremetal", + "tlb_repeated_sfence_test-singlecore-baremetal", + "tlb_sv39_nonidentity_test-singlecore-baremetal", + "tlb_sv39_4k_alias_test-singlecore-baremetal", + "tlb_sfence_remap_test-singlecore-baremetal", ] diff --git a/bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml index c2c91389b..5c31fdc2e 100644 --- a/bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml @@ -7,17 +7,17 @@ [workloads] search_path = "workloads/src" tests = [ - "ctest_bdb_counter_test-linux", - "ctest_bfp_test-linux", - "ctest_gemmini_os_cisc_basic_test-linux", - "ctest_gemmini_os_cisc_atranspose_test-linux", - "ctest_gemmini_os_cisc_btranspose_test-linux", - "ctest_gemmini_os_cisc_shift_test-linux", - "ctest_gemmini_os_risc_basic_test-linux", - "ctest_gemmini_os_risc_atranspose_test-linux", - "ctest_gemmini_os_risc_btranspose_test-linux", - "ctest_gemmini_os_risc_abtranspose_test-linux", - "ctest_gemmini_os_risc_shift_test-linux", - "ctest_gemmini_ws_cisc_conv_test-linux", - "ctest_gemmini_ws_risc_basic_test-linux", + "bdb_counter_test-linux", + "bfp_test-linux", + "gemmini_os_cisc_basic_test-linux", + "gemmini_os_cisc_atranspose_test-linux", + "gemmini_os_cisc_btranspose_test-linux", + "gemmini_os_cisc_shift_test-linux", + "gemmini_os_risc_basic_test-linux", + "gemmini_os_risc_atranspose_test-linux", + "gemmini_os_risc_btranspose_test-linux", + "gemmini_os_risc_abtranspose_test-linux", + "gemmini_os_risc_shift_test-linux", + "gemmini_ws_cisc_conv_test-linux", + "gemmini_ws_risc_basic_test-linux", ] diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml index 81580a4ce..44fd3a72b 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml @@ -7,48 +7,48 @@ [workloads] search_path = "workloads/src" tests = [ - "ctest_mvin_mvout_test_singlecore-baremetal", - "ctest_vecunit_matmul_16xn_ones_singlecore-baremetal", - "ctest_vecunit_matmul_16xn_random1_singlecore-baremetal", - "ctest_vecunit_matmul_16xn_random2_singlecore-baremetal", - "ctest_vecunit_matmul_16xn_random3_singlecore-baremetal", - "ctest_vecunit_matmul_16xn_zero_random_singlecore-baremetal", - "ctest_vecunit_matmul_col_row_vector_singlecore-baremetal", - "ctest_vecunit_matmul_identity_random_singlecore-baremetal", - "ctest_vecunit_matmul_ones_singlecore-baremetal", - "ctest_vecunit_matmul_random1_singlecore-baremetal", - "ctest_vecunit_matmul_random2_singlecore-baremetal", - "ctest_vecunit_matmul_zero_random_singlecore-baremetal", - "ctest_relu_test_singlecore-baremetal", - "ctest_transpose_test_singlecore-baremetal", - "ctest_transpose_16xn_test_singlecore-baremetal", - "ctest_im2col_test_singlecore-baremetal", - "ctest_fp2int_test_singlecore-baremetal", - "ctest_int2fp_test_singlecore-baremetal", - "ctest_bdb_counter_test_singlecore-baremetal", - "ctest_gemmini_os_cisc_basic_test_singlecore-baremetal", - "ctest_gemmini_os_cisc_atranspose_test_singlecore-baremetal", - "ctest_gemmini_os_cisc_btranspose_test_singlecore-baremetal", - "ctest_gemmini_os_cisc_shift_test_singlecore-baremetal", - "ctest_gemmini_os_risc_atranspose_test_singlecore-baremetal", - "ctest_gemmini_os_risc_btranspose_test_singlecore-baremetal", - "ctest_gemmini_os_risc_abtranspose_test_singlecore-baremetal", - "ctest_gemmini_os_risc_shift_test_singlecore-baremetal", - "ctest_gemmini_ws_cisc_conv_test_singlecore-baremetal", - "ctest_gemmini_ws_risc_basic_test_singlecore-baremetal", - "ctest_mmio_minimal_test_singlecore-baremetal", - "ctest_mxfp2int_test_singlecore-baremetal", + "mvin_mvout_test-singlecore-baremetal", + "vecunit_matmul_16xn_ones-singlecore-baremetal", + "vecunit_matmul_16xn_random1-singlecore-baremetal", + "vecunit_matmul_16xn_random2-singlecore-baremetal", + "vecunit_matmul_16xn_random3-singlecore-baremetal", + "vecunit_matmul_16xn_zero_random-singlecore-baremetal", + "vecunit_matmul_col_row_vector-singlecore-baremetal", + "vecunit_matmul_identity_random-singlecore-baremetal", + "vecunit_matmul_ones-singlecore-baremetal", + "vecunit_matmul_random1-singlecore-baremetal", + "vecunit_matmul_random2-singlecore-baremetal", + "vecunit_matmul_zero_random-singlecore-baremetal", + "relu_test-singlecore-baremetal", + "transpose_test-singlecore-baremetal", + "transpose_16xn_test-singlecore-baremetal", + "im2col_test-singlecore-baremetal", + "fp2int_test-singlecore-baremetal", + "int2fp_test-singlecore-baremetal", + "bdb_counter_test-singlecore-baremetal", + "gemmini_os_cisc_basic_test-singlecore-baremetal", + "gemmini_os_cisc_atranspose_test-singlecore-baremetal", + "gemmini_os_cisc_btranspose_test-singlecore-baremetal", + "gemmini_os_cisc_shift_test-singlecore-baremetal", + "gemmini_os_risc_atranspose_test-singlecore-baremetal", + "gemmini_os_risc_btranspose_test-singlecore-baremetal", + "gemmini_os_risc_abtranspose_test-singlecore-baremetal", + "gemmini_os_risc_shift_test-singlecore-baremetal", + "gemmini_ws_cisc_conv_test-singlecore-baremetal", + "gemmini_ws_risc_basic_test-singlecore-baremetal", + "mmio_minimal_test-singlecore-baremetal", + "mxfp2int_test-singlecore-baremetal", - "ctest_tlb_bare_mode_test_singlecore-baremetal", - "ctest_tlb_sv39_identity_test_singlecore-baremetal", - "ctest_tlb_sfence_flush_test_singlecore-baremetal", - "ctest_tlb_repeated_sfence_test_singlecore-baremetal", - "ctest_tlb_sv39_nonidentity_test_singlecore-baremetal", - "ctest_tlb_sv39_4k_alias_test_singlecore-baremetal", - "ctest_tlb_sfence_remap_test_singlecore-baremetal", + "tlb_bare_mode_test-singlecore-baremetal", + "tlb_sv39_identity_test-singlecore-baremetal", + "tlb_sfence_flush_test-singlecore-baremetal", + "tlb_repeated_sfence_test-singlecore-baremetal", + "tlb_sv39_nonidentity_test-singlecore-baremetal", + "tlb_sv39_4k_alias_test-singlecore-baremetal", + "tlb_sfence_remap_test-singlecore-baremetal", - "ctest_mvin_mvout_stride_test_singlecore-baremetal", - "ctest_transpose_matmul_singlecore-baremetal", + "mvin_mvout_stride_test-singlecore-baremetal", + "transpose_matmul-singlecore-baremetal", "optest_buckyball_matmul_16x64_64x16_singlecore-baremetal", "optest_buckyball_matmul_64x16_16x64_singlecore-baremetal", "optest_tile_matmul_1x84_84x10_singlecore-baremetal", diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml index adf8b5ad9..8b2b13c7b 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml @@ -7,36 +7,36 @@ [workloads] search_path = "workloads/src" tests = [ - "ctest_mvin_mvout_test-linux", - "ctest_vecunit_matmul_16xn_ones-linux", - "ctest_vecunit_matmul_16xn_random1-linux", - "ctest_vecunit_matmul_16xn_random2-linux", - "ctest_vecunit_matmul_16xn_random3-linux", - "ctest_vecunit_matmul_16xn_zero_random-linux", - "ctest_vecunit_matmul_col_row_vector-linux", - "ctest_vecunit_matmul_identity_random-linux", - "ctest_vecunit_matmul_ones-linux", - "ctest_vecunit_matmul_random1-linux", - "ctest_vecunit_matmul_random2-linux", - "ctest_vecunit_matmul_zero_random-linux", - "ctest_relu_test-linux", - "ctest_transpose_test-linux", - "ctest_transpose_16xn_test-linux", - "ctest_im2col_test-linux", - "ctest_fp2int_test-linux", - "ctest_int2fp_test-linux", - "ctest_bdb_counter_test-linux", - "ctest_bfp_test-linux", - "ctest_gemmini_os_cisc_basic_test-linux", - "ctest_gemmini_os_cisc_atranspose_test-linux", - "ctest_gemmini_os_cisc_btranspose_test-linux", - "ctest_gemmini_os_cisc_shift_test-linux", - "ctest_gemmini_os_risc_atranspose_test-linux", - "ctest_gemmini_os_risc_btranspose_test-linux", - "ctest_gemmini_os_risc_abtranspose_test-linux", - "ctest_gemmini_os_risc_shift_test-linux", - "ctest_gemmini_ws_cisc_conv_test-linux", - "ctest_gemmini_ws_risc_basic_test-linux", - "ctest_mmio_minimal_test-linux", - "ctest_mxfp2int_test-linux", + "mvin_mvout_test-linux", + "vecunit_matmul_16xn_ones-linux", + "vecunit_matmul_16xn_random1-linux", + "vecunit_matmul_16xn_random2-linux", + "vecunit_matmul_16xn_random3-linux", + "vecunit_matmul_16xn_zero_random-linux", + "vecunit_matmul_col_row_vector-linux", + "vecunit_matmul_identity_random-linux", + "vecunit_matmul_ones-linux", + "vecunit_matmul_random1-linux", + "vecunit_matmul_random2-linux", + "vecunit_matmul_zero_random-linux", + "relu_test-linux", + "transpose_test-linux", + "transpose_16xn_test-linux", + "im2col_test-linux", + "fp2int_test-linux", + "int2fp_test-linux", + "bdb_counter_test-linux", + "bfp_test-linux", + "gemmini_os_cisc_basic_test-linux", + "gemmini_os_cisc_atranspose_test-linux", + "gemmini_os_cisc_btranspose_test-linux", + "gemmini_os_cisc_shift_test-linux", + "gemmini_os_risc_atranspose_test-linux", + "gemmini_os_risc_btranspose_test-linux", + "gemmini_os_risc_abtranspose_test-linux", + "gemmini_os_risc_shift_test-linux", + "gemmini_ws_cisc_conv_test-linux", + "gemmini_ws_risc_basic_test-linux", + "mmio_minimal_test-linux", + "mxfp2int_test-linux", ] From 51119d680fe66598dae4fed4b1fc1359b113c5ce Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 5 Jul 2026 17:01:32 +0800 Subject: [PATCH 089/196] feat: add chip parameter to workload build API and update related configurations --- bbdev/api/services/workflow_tools.py | 2 +- .../bebop/bemu/scripts/workloads-elf.toml | 7 ---- .../bebop/bemu/scripts/workloads-pk.toml | 7 ---- bbdev/api/steps/workload/01_buidl_api.step.py | 31 ++++++++++++++++- .../api/steps/workload/01_build_event.step.py | 33 +++++++++++++++++-- bbdev/api/steps/workload/README.md | 7 ++-- bbdev/api/tests/test_api_workload_build.py | 2 +- 7 files changed, 67 insertions(+), 22 deletions(-) diff --git a/bbdev/api/services/workflow_tools.py b/bbdev/api/services/workflow_tools.py index 6409e37b3..389372acd 100644 --- a/bbdev/api/services/workflow_tools.py +++ b/bbdev/api/services/workflow_tools.py @@ -18,7 +18,7 @@ def get_description(self) -> str: - /verilator/verilog: Generate Verilog - /verilator/build: Build verilator (params: jobs) - /verilator/sim: Run simulation (params: binary, batch) - - /workload/build: Build workload (params: args)""" + - /workload/build: Build workload (params: chip, model, stable)""" def get_parameters(self) -> Dict[str, Any]: return { diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml index 5effb996e..9e18f8f65 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml @@ -41,11 +41,4 @@ tests = [ "gemmini_ws_risc_basic_test-singlecore-baremetal", "mmio_minimal_test-singlecore-baremetal", "mxfp2int_test-singlecore-baremetal", - - "optest_buckyball_matmul_64x16_16x64_singlecore-baremetal", - "optest_buckyball_matmul_1024x16_16x1024_singlecore-baremetal", - "optest_linalg_conv2d_nhwc_hwcf_singlecore-baremetal", - "optest_linalg_conv2d_nhwc_fhwc_singlecore-baremetal", - "optest_linalg_conv2d_nchw_fchw_singlecore-baremetal", - "optest_tile_matmul_127x17_17x127_singlecore-baremetal", ] diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml index cd8c4aa00..ed8f0a728 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml @@ -41,11 +41,4 @@ tests = [ "gemmini_ws_risc_basic_test-linux", "mmio_minimal_test-linux", "mxfp2int_test-linux", - - "optest_buckyball_matmul_64x16_16x64-linux", - "optest_buckyball_matmul_1024x16_16x1024-linux", - "optest_linalg_conv2d_nhwc_hwcf-linux", - "optest_linalg_conv2d_nhwc_fhwc-linux", - "optest_linalg_conv2d_nchw_fchw-linux", - "optest_tile_matmul_127x17_17x127-linux", ] diff --git a/bbdev/api/steps/workload/01_buidl_api.step.py b/bbdev/api/steps/workload/01_buidl_api.step.py index db6db307f..cbbe3c631 100644 --- a/bbdev/api/steps/workload/01_buidl_api.step.py +++ b/bbdev/api/steps/workload/01_buidl_api.step.py @@ -1,5 +1,16 @@ +import os +import re +import sys +from pathlib import Path + from motia import ApiRequest, ApiResponse, FlowContext, api +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path + config = { "name": "workload-build-api", "description": "build workload", @@ -11,13 +22,30 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - allowed = {"model", "stable"} + allowed = {"chip", "model", "stable"} unknown = sorted(k for k in body if k not in allowed) if unknown: return ApiResponse( status=400, body={"error": f"Unknown workload build parameter(s): {', '.join(unknown)}"}, ) + chip = body.get("chip") + if not chip: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --chip must be specified"}, + ) + if not isinstance(chip, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", chip): + return ApiResponse( + status=400, + body={"error": f"Invalid chip: {chip}"}, + ) + chip_dir = Path(get_buckyball_path()) / "examples" / "chips" / chip + if not chip_dir.is_dir(): + return ApiResponse( + status=400, + body={"error": f"Workload chip does not exist: {chip}"}, + ) stable = body.get("stable", False) if not isinstance(stable, bool): return ApiResponse( @@ -25,6 +53,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body={"error": "Invalid parameter: stable must be a boolean flag"}, ) data = { + "chip": chip, "model": body.get("model", ""), "stable": stable, } diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index 230f8adfd..2ee37b6d0 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -1,6 +1,8 @@ import os import sys import shlex +import re +from pathlib import Path from motia import FlowContext, queue @@ -27,7 +29,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() workload_dir = f"{bbdir}/bb-tests" build_dir = f"{workload_dir}/build" - allowed = {"model", "stable", "_trace_id"} + allowed = {"chip", "model", "stable", "_trace_id"} unknown = sorted(k for k in input_data if k not in allowed) if unknown: ctx.logger.error(f"Unknown workload build parameter(s): {', '.join(unknown)}") @@ -37,6 +39,32 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + chip = input_data.get("chip") + if not chip: + ctx.logger.error("Missing required parameter: chip must be specified") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_chip"}, + trace_id=origin_tid, + ) + return + if not isinstance(chip, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", chip): + ctx.logger.error(f"Invalid chip: {chip}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_chip", "chip": chip}, + trace_id=origin_tid, + ) + return + chip_dir = Path(bbdir) / "examples" / "chips" / chip + if not chip_dir.is_dir(): + ctx.logger.error(f"Workload chip does not exist: {chip}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "unknown_chip", "chip": chip}, + trace_id=origin_tid, + ) + return model = input_data.get("model", "") stable = input_data.get("stable", False) @@ -77,11 +105,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: os.makedirs(build_dir, exist_ok=True) + chip_arg = f"-DBUCKYBALL_WORKLOAD_CHIP={shlex.quote(chip)}" stable_arg = "-DBUCKYBALL_STABLE=ON" if stable else "-DBUCKYBALL_STABLE=OFF" ninja_target = f" {shlex.quote(target)}" if target else "" inner = ( f"cd {shlex.quote(build_dir)} && " - f"cmake -G Ninja {stable_arg} .. && " + f"cmake -G Ninja {chip_arg} {stable_arg} .. && " f"ninja -j{os.cpu_count()}{ninja_target}" ) command = f"cd {shlex.quote(bbdir)} && nix develop -c bash -c {shlex.quote(inner)}" diff --git a/bbdev/api/steps/workload/README.md b/bbdev/api/steps/workload/README.md index 6769fabc1..73252a9e1 100644 --- a/bbdev/api/steps/workload/README.md +++ b/bbdev/api/steps/workload/README.md @@ -10,19 +10,20 @@ Workload build workflow in Buckyball framework, used to build test workloads and **Function**: Build workload **Parameters**: +- **`chip`** - Required chip name. Selects chip-specific workloads. - **`model`** - Optional model name to build. If omitted, build all workloads. - **`stable`** - Optional boolean flag. If set, build with stable LLVM Buckyball extensions. **Examples**: ```bash # Build one model with the default xbuckyball path -bbdev workload --build "--model lenet" +bbdev workload --build "--chip toy --model lenet" # Build one model with stable LLVM Buckyball extensions -bbdev workload --build "--model lenet --stable" +bbdev workload --build "--chip toy --model lenet --stable" # Build all workloads -bbdev workload --build +bbdev workload --build "--chip toy" ``` **Response**: diff --git a/bbdev/api/tests/test_api_workload_build.py b/bbdev/api/tests/test_api_workload_build.py index 4d3343a96..f4e5b5bef 100644 --- a/bbdev/api/tests/test_api_workload_build.py +++ b/bbdev/api/tests/test_api_workload_build.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev workload --build") +run_bbdev_case("bbdev workload --build '--chip toy'") From 23b5a45c0450ae0daba8bdb018849ff84d5dd688 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 5 Jul 2026 22:10:37 +0800 Subject: [PATCH 090/196] feat: add multiple new linear algebra test cases --- bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml | 8 ++++++++ bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml | 8 ++++++++ .../api/steps/bebop/verilator/scripts/workloads-elf.toml | 3 +++ .../api/steps/bebop/verilator/scripts/workloads-pk.toml | 9 +++++++++ 4 files changed, 28 insertions(+) diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml index 9e18f8f65..603a09f06 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml @@ -41,4 +41,12 @@ tests = [ "gemmini_ws_risc_basic_test-singlecore-baremetal", "mmio_minimal_test-singlecore-baremetal", "mxfp2int_test-singlecore-baremetal", + "optest_buckyball_matmul_16x64_64x16_singlecore-baremetal", + "optest_buckyball_matmul_64x16_16x64_singlecore-baremetal", + "optest_buckyball_matmul_1024x16_16x1024_singlecore-baremetal", + "optest_tile_matmul_127x17_17x127_singlecore-baremetal", + "optest_tile_matmul_1x84_84x10_singlecore-baremetal", + "optest_linalg_conv2d_nhwc_hwcf_singlecore-baremetal", + "optest_linalg_conv2d_nhwc_fhwc_singlecore-baremetal", + "optest_linalg_conv2d_nchw_fchw_singlecore-baremetal", ] diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml index ed8f0a728..c1160f04d 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml @@ -41,4 +41,12 @@ tests = [ "gemmini_ws_risc_basic_test-linux", "mmio_minimal_test-linux", "mxfp2int_test-linux", + "optest_buckyball_matmul_16x64_64x16-linux", + "optest_buckyball_matmul_64x16_16x64-linux", + "optest_buckyball_matmul_1024x16_16x1024-linux", + "optest_tile_matmul_127x17_17x127-linux", + "optest_tile_matmul_1x84_84x10-linux", + "optest_linalg_conv2d_nhwc_hwcf-linux", + "optest_linalg_conv2d_nhwc_fhwc-linux", + "optest_linalg_conv2d_nchw_fchw-linux", ] diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml index 44fd3a72b..9a86575a1 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml @@ -26,6 +26,7 @@ tests = [ "fp2int_test-singlecore-baremetal", "int2fp_test-singlecore-baremetal", "bdb_counter_test-singlecore-baremetal", + "bfp_test-singlecore-baremetal", "gemmini_os_cisc_basic_test-singlecore-baremetal", "gemmini_os_cisc_atranspose_test-singlecore-baremetal", "gemmini_os_cisc_btranspose_test-singlecore-baremetal", @@ -51,6 +52,8 @@ tests = [ "transpose_matmul-singlecore-baremetal", "optest_buckyball_matmul_16x64_64x16_singlecore-baremetal", "optest_buckyball_matmul_64x16_16x64_singlecore-baremetal", + "optest_buckyball_matmul_1024x16_16x1024_singlecore-baremetal", + "optest_tile_matmul_127x17_17x127_singlecore-baremetal", "optest_tile_matmul_1x84_84x10_singlecore-baremetal", "optest_linalg_conv2d_nhwc_hwcf_singlecore-baremetal", "optest_linalg_conv2d_nhwc_fhwc_singlecore-baremetal", diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml index 8b2b13c7b..f9ddf6d0d 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml @@ -8,6 +8,7 @@ search_path = "workloads/src" tests = [ "mvin_mvout_test-linux", + "mvin_mvout_stride_test-linux", "vecunit_matmul_16xn_ones-linux", "vecunit_matmul_16xn_random1-linux", "vecunit_matmul_16xn_random2-linux", @@ -39,4 +40,12 @@ tests = [ "gemmini_ws_risc_basic_test-linux", "mmio_minimal_test-linux", "mxfp2int_test-linux", + "optest_buckyball_matmul_16x64_64x16-linux", + "optest_buckyball_matmul_64x16_16x64-linux", + "optest_buckyball_matmul_1024x16_16x1024-linux", + "optest_tile_matmul_127x17_17x127-linux", + "optest_tile_matmul_1x84_84x10-linux", + "optest_linalg_conv2d_nhwc_hwcf-linux", + "optest_linalg_conv2d_nhwc_fhwc-linux", + "optest_linalg_conv2d_nchw_fchw-linux", ] From 140625b1c6e3948375ab3aabbc590708aca8314b Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 6 Jul 2026 12:05:54 +0800 Subject: [PATCH 091/196] chore: disable slow tests in workload configurations --- bbdev/api/steps/bebop/verilator/06_batch_event.step.py | 2 +- bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml | 5 +++-- bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index facab9e21..449627054 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -3,7 +3,7 @@ Runs bebop verilator nextest batch regression: 1. Build bebop with verilator feature and VSRC_PATH - 2. Run cargo nextest with verilator-specific config (serial execution) + 2. Run cargo nextest with verilator-specific config """ import os import shlex diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml index 9a86575a1..09b0ce40d 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml @@ -52,8 +52,9 @@ tests = [ "transpose_matmul-singlecore-baremetal", "optest_buckyball_matmul_16x64_64x16_singlecore-baremetal", "optest_buckyball_matmul_64x16_16x64_singlecore-baremetal", - "optest_buckyball_matmul_1024x16_16x1024_singlecore-baremetal", - "optest_tile_matmul_127x17_17x127_singlecore-baremetal", + # Disabled for Verilator batch because these tests are too slow. + # "optest_buckyball_matmul_1024x16_16x1024_singlecore-baremetal", + # "optest_tile_matmul_127x17_17x127_singlecore-baremetal", "optest_tile_matmul_1x84_84x10_singlecore-baremetal", "optest_linalg_conv2d_nhwc_hwcf_singlecore-baremetal", "optest_linalg_conv2d_nhwc_fhwc_singlecore-baremetal", diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml index f9ddf6d0d..30aea7b5f 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml +++ b/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml @@ -42,8 +42,9 @@ tests = [ "mxfp2int_test-linux", "optest_buckyball_matmul_16x64_64x16-linux", "optest_buckyball_matmul_64x16_16x64-linux", - "optest_buckyball_matmul_1024x16_16x1024-linux", - "optest_tile_matmul_127x17_17x127-linux", + # Disabled for Verilator batch because these tests are too slow. + # "optest_buckyball_matmul_1024x16_16x1024-linux", + # "optest_tile_matmul_127x17_17x127-linux", "optest_tile_matmul_1x84_84x10-linux", "optest_linalg_conv2d_nhwc_hwcf-linux", "optest_linalg_conv2d_nhwc_fhwc-linux", From 9c205a18d14f06f60553c29e93868b113b113186 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 9 Jul 2026 16:00:44 +0800 Subject: [PATCH 092/196] fix: update p2e to 0.0.1 api --- .../api/steps/bebop/bemu/02_sim_event.step.py | 20 +++++++++++++++++-- .../bebop/p2e/02_buildbitstream_event.step.py | 16 ++++++++++----- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 68b9568f5..cb6033ba9 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -23,7 +23,7 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger -from utils.search_workload import search_workload +from utils.search_workload import search_workload, search_workload_all from utils.event_common import check_result, get_origin_trace_id from bemu_common import bemu_env @@ -65,6 +65,22 @@ def clean_model_trace(binary_dir: str) -> None: perfetto.unlink() +def resolve_bemu_binary(bbdir: str, chip: str, binary_name: str) -> str | None: + search_root = f"{bbdir}/bb-tests/output/workloads/src" + chip_root = f"{search_root}/CTest/chips/{chip}" + + chip_binary = search_workload(chip_root, binary_name) + if chip_binary is not None: + return chip_binary + + matches = search_workload_all(search_root, binary_name) + chip_marker = f"{os.path.sep}CTest{os.path.sep}chips{os.path.sep}" + non_chip_matches = [path for path in matches if chip_marker not in path] + if len(non_chip_matches) == 1: + return non_chip_matches[0] + return None + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -93,7 +109,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return binary_name = input_data.get("binary", "") - binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) + binary_path = resolve_bemu_binary(bbdir, chip, binary_name) if binary_path is None: ctx.logger.error(f"binary not found: {binary_name}") await check_result( diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index f3a7f5112..03d9d7e93 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -4,7 +4,7 @@ Builds the P2E VVAC runtime case via bebop CLI: 1. Resolve Verilog source directory (VSRC_PATH) from config 2. Run bebop build p2e with rtl_dir and out_dir - 3. Validate runtime artifacts used by bebop run p2e + 3. Validate generated bitstream and runtime artifacts """ import os import sys @@ -64,7 +64,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: os.makedirs(build_dir, exist_ok=True) build_cmd = ( - f"nix develop --ignore-environment --keep ALL_PROXY -c " + f"nix develop --ignore-environment --keep HOME --keep ALL_PROXY -c " f"cargo run --features p2e -- build p2e " f"--rtl-dir=\"{vsrc_dir}\" " f"--out-dir=\"{build_dir}\"" @@ -80,10 +80,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") libvctb_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so") + bitstream_path = os.path.join(build_dir, "fpgaCompDir", "bitstream.bit") if build_result.returncode == 0: - missing = [path for path in (rtcfg_path, libvctb_path) if not os.path.exists(path)] + missing = [ + path + for path in (rtcfg_path, libvctb_path, bitstream_path) + if not os.path.exists(path) + ] if missing: - ctx.logger.error(f"P2E runtime artifacts missing: {missing}") + ctx.logger.error(f"P2E build artifacts missing: {missing}") await check_result( ctx, 1, @@ -94,7 +99,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "vsrc_dir": vsrc_dir, "build_dir": build_dir, "missing": missing, - "error": "runtime_artifact_not_found", + "error": "p2e_artifact_not_found", "timestamp": timestamp, }, trace_id=origin_tid, @@ -112,6 +117,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "build_dir": build_dir, "rtcfg": rtcfg_path, "libvCtb": libvctb_path, + "bitstream": bitstream_path, "timestamp": timestamp, }, trace_id=origin_tid, From bab6f8f22a857474713b831f895d03885da4cdc6 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 10 Jul 2026 02:10:41 +0800 Subject: [PATCH 093/196] feat: add workload clean API and enhance bebop batch regression with clean-before option --- bbdev/api/services/workflow_tools.py | 1 + .../api/steps/bebop/bemu/03_batch_api.step.py | 7 ++- .../steps/bebop/bemu/03_batch_event.step.py | 5 ++ .../bebop/verilator/06_batch_api.step.py | 1 + .../bebop/verilator/06_batch_event.step.py | 5 ++ bbdev/api/steps/workload/00_clean_api.step.py | 19 +++++++ .../api/steps/workload/00_clean_event.step.py | 54 +++++++++++++++++++ bbdev/api/steps/workload/README.md | 14 +++++ bbdev/api/tests/test_all_api_serial.py | 1 + bbdev/api/tests/test_api_workload_clean.py | 3 ++ bbdev/bbdev | 7 +-- 11 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 bbdev/api/steps/workload/00_clean_api.step.py create mode 100644 bbdev/api/steps/workload/00_clean_event.step.py create mode 100644 bbdev/api/tests/test_api_workload_clean.py diff --git a/bbdev/api/services/workflow_tools.py b/bbdev/api/services/workflow_tools.py index 389372acd..339bc1b56 100644 --- a/bbdev/api/services/workflow_tools.py +++ b/bbdev/api/services/workflow_tools.py @@ -18,6 +18,7 @@ def get_description(self) -> str: - /verilator/verilog: Generate Verilog - /verilator/build: Build verilator (params: jobs) - /verilator/sim: Run simulation (params: binary, batch) + - /workload/clean: Clean workload output directory - /workload/build: Build workload (params: chip, model, stable)""" def get_parameters(self) -> Dict[str, Any]: diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py index 1970d7b3d..e85abd8c4 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -49,6 +49,11 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: await ctx.enqueue({ "topic": "bebop.bemu.batch", - "data": {"chip": chip, "test": test_type, "_trace_id": ctx.trace_id} + "data": { + "chip": chip, + "test": test_type, + "clean-before": body.get("clean-before", body.get("clean_before", False)), + "_trace_id": ctx.trace_id, + } }) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index fbf0ad26d..1b7a93c3d 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -6,6 +6,7 @@ 2. Run cargo nextest with bemu-specific config """ import os +import shutil import shlex import sys @@ -102,6 +103,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── Run nextest ─────────────────────────────────────────────────────── # Pass parameters via environment variables (nextest doesn't support custom CLI args after `--`) + if input_data.get("clean-before", input_data.get("clean_before", False)): + shutil.rmtree(f"{bebop_dir}/test-artifacts", ignore_errors=True) + ctx.logger.info("Cleaned previous bebop test artifacts") + env.update({ "BEBOP_WORKLOAD_TOML": workload_toml, "BEBOP_BB_TESTS_ROOT": elf_root, diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py index d74ea0903..2db727c8c 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -41,6 +41,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "config": arch_config, "vsrc_dir": vsrc_dir, "test": test_type, + "clean-before": body.get("clean-before", body.get("clean_before", False)), } await ctx.enqueue({"topic": "bebop.verilator.batch", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 449627054..7c1f27daa 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -6,6 +6,7 @@ 2. Run cargo nextest with verilator-specific config """ import os +import shutil import shlex import sys @@ -98,6 +99,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── Run nextest ─────────────────────────────────────────────────────── # Pass parameters via environment variables (nextest doesn't support custom CLI args after `--`) + if input_data.get("clean-before", input_data.get("clean_before", False)): + shutil.rmtree(f"{bebop_dir}/test-artifacts", ignore_errors=True) + ctx.logger.info("Cleaned previous bebop test artifacts") + env = os.environ.copy() env.update({ "BEBOP_WORKLOAD_TOML": workload_toml, diff --git a/bbdev/api/steps/workload/00_clean_api.step.py b/bbdev/api/steps/workload/00_clean_api.step.py new file mode 100644 index 000000000..7132babe8 --- /dev/null +++ b/bbdev/api/steps/workload/00_clean_api.step.py @@ -0,0 +1,19 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "workload-clean-api", + "description": "clean workload output directory", + "flows": ["workload"], + "triggers": [api("POST", "/workload/clean")], + "enqueues": ["workload.clean"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + if body: + unknown = ", ".join(sorted(body)) + return ApiResponse(status=400, body={"error": f"Unknown workload clean parameter(s): {unknown}"}) + + await ctx.enqueue({"topic": "workload.clean", "data": {"_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/00_clean_event.step.py b/bbdev/api/steps/workload/00_clean_event.step.py new file mode 100644 index 000000000..3256d7227 --- /dev/null +++ b/bbdev/api/steps/workload/00_clean_event.step.py @@ -0,0 +1,54 @@ +import os +import shutil +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path + +config = { + "name": "workload-clean", + "description": "clean workload output directory", + "flows": ["workload"], + "triggers": [queue("workload.clean")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + allowed = {"_trace_id"} + unknown = sorted(k for k in input_data if k not in allowed) + if unknown: + ctx.logger.error(f"Unknown workload clean parameter(s): {', '.join(unknown)}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "unknown_parameter", "parameters": unknown}, + trace_id=origin_tid, + ) + return + + bbdir = get_buckyball_path() + paths = [os.path.join(bbdir, "bb-tests", "output")] + + for path in paths: + if os.path.exists(path): + ctx.logger.info("Removing workload directory", {"path": path}) + shutil.rmtree(path) + else: + ctx.logger.info("Workload directory already clean", {"path": path}) + + await check_result( + ctx, + 0, + continue_run=False, + extra_fields={"task": "clean", "paths": paths}, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/workload/README.md b/bbdev/api/steps/workload/README.md index 73252a9e1..d0f4ab69f 100644 --- a/bbdev/api/steps/workload/README.md +++ b/bbdev/api/steps/workload/README.md @@ -4,6 +4,18 @@ Workload build workflow in Buckyball framework, used to build test workloads and ## API Usage +### `clean` +**Endpoint**: `POST /workload/clean` + +**Function**: Clean workload output directory. + +**Parameters**: None. + +**Examples**: +```bash +bbdev workload --clean +``` + ### `build` **Endpoint**: `POST /workload/build` @@ -14,6 +26,8 @@ Workload build workflow in Buckyball framework, used to build test workloads and - **`model`** - Optional model name to build. If omitted, build all workloads. - **`stable`** - Optional boolean flag. If set, build with stable LLVM Buckyball extensions. +For chip workloads under paths like `*/chips/`, only the directory selected by `chip` is synced to `bb-tests/output/workloads`. + **Examples**: ```bash # Build one model with the default xbuckyball path diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index f2dfda3fe..2f3cd8819 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -8,6 +8,7 @@ def main(): test_dir = Path(__file__).resolve().parent tests = [ "test_api_compiler_build.py", + "test_api_workload_clean.py", "test_api_workload_build.py", "test_api_verilator_clean.py", "test_api_verilator_verilog.py", diff --git a/bbdev/api/tests/test_api_workload_clean.py b/bbdev/api/tests/test_api_workload_clean.py new file mode 100644 index 000000000..b9d59f7ed --- /dev/null +++ b/bbdev/api/tests/test_api_workload_clean.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev workload --clean") diff --git a/bbdev/bbdev b/bbdev/bbdev index 8c9e28349..64c01987e 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -42,7 +42,8 @@ WORKFLOW_COMMANDS = { "run": 'Integrated build+sim+run. Args: "--binary [--batch] [--job ]"', }, "workload": { - "build": 'Build workload. Args: "[--model ] [--stable]"', + "clean": "Clean workload output directory.", + "build": 'Build workload. Args: "--chip [--model ] [--stable]"', "tohex": "Convert all -baremetal ELF files under bb-tests/output/workloads/src to hex.", }, "kernel": { @@ -59,7 +60,7 @@ WORKFLOW_COMMANDS = { }, "bebop-bemu": { "sim": 'Run Bebop emulator simulation. Args: "--chip --binary [--pk] [--log-dir ]"', - "batch": 'Run Bebop bemu nextest batch regression (parallel). Args: "--chip --test "', + "batch": 'Run Bebop bemu nextest batch regression (parallel). Args: "--chip --test [--clean-before]"', }, "bebop-verilator": { "clean": 'Clean Bebop Verilator build directory. Args: "--config [--output-dir ]"', @@ -67,7 +68,7 @@ WORKFLOW_COMMANDS = { "build": 'Build Bebop Verilator simulation executable. Args: "--config [--jobs ] [--output-dir ]"', "sim": 'Run Bebop Verilator simulation. Args: "--config --binary [--batch] [--log-dir ] [--fst-dir ] [--no-wave]"', "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', - "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--config --test "', + "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--config --test [--clean-before]"', }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--config [--output-dir ]"', From d1921b3461861fee361d73ee1521f068faa0cd62 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 10 Jul 2026 04:11:21 +0800 Subject: [PATCH 094/196] feat: enhance bebop batch regression by adding chip parameter and updating workload configurations --- .../steps/bebop/bemu/03_batch_event.step.py | 17 ++--- .../steps/bebop/bemu/scripts/bemu_common.py | 13 +--- .../bebop/bemu/scripts/workloads-elf.toml | 52 ---------------- .../bebop/bemu/scripts/workloads-pk.toml | 52 ---------------- .../api/steps/bebop/p2e/04_batch_api.step.py | 8 +++ .../steps/bebop/p2e/04_batch_event.step.py | 28 ++++++--- .../bebop/p2e/scripts/workloads-elf.toml | 30 --------- .../steps/bebop/p2e/scripts/workloads-pk.toml | 23 ------- bbdev/api/steps/bebop/regression.py | 39 ++++++++++++ .../bebop/verilator/06_batch_api.step.py | 8 +++ .../bebop/verilator/06_batch_event.step.py | 28 ++++++--- .../verilator/scripts/workloads-elf.toml | 62 ------------------- .../bebop/verilator/scripts/workloads-pk.toml | 52 ---------------- .../tests/test_api_bebop_verilator_batch.py | 2 +- bbdev/bbdev | 4 +- 15 files changed, 108 insertions(+), 310 deletions(-) delete mode 100644 bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml delete mode 100644 bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml delete mode 100644 bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml delete mode 100644 bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml create mode 100644 bbdev/api/steps/bebop/regression.py delete mode 100644 bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml delete mode 100644 bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index 1b7a93c3d..61030a870 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -15,6 +15,9 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +bebop_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +if bebop_path not in sys.path: + sys.path.insert(0, bebop_path) scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) @@ -23,6 +26,7 @@ from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id from bemu_common import bemu_env +from regression import regression_workload_toml config = { "name": "bebop-bemu-batch", @@ -61,17 +65,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - # ── Determine workload file based on test type ──────────────────────── test_type = input_data.get("test", "elf-tests") - if test_type == "elf-tests": - workload_toml = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/workloads-elf.toml" - elif test_type == "pk-tests": - workload_toml = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/workloads-pk.toml" - else: - ctx.logger.error(f"Invalid test type: {test_type}") + try: + workload_toml = regression_workload_toml(chip, "bemu", test_type, bbdir) + except ValueError as e: + ctx.logger.error(str(e)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "invalid_test_type", "test": test_type}, + extra_fields={"error": "invalid_regression", "test": test_type, "chip": chip}, trace_id=origin_tid, ) return diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py index ddfaba406..e652c10f7 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py @@ -1,22 +1,11 @@ -import os from pathlib import Path from utils.path import get_buckyball_path -def validate_bemu_chip(chip: str | None, bbdir: str | None = None) -> str: - _ = bbdir +def bemu_chip_inst(chip: str, bbdir: str | None = None) -> Path: if not chip: raise ValueError("missing required parameter: chip") - if not isinstance(chip, str): - raise ValueError("chip must be a string") - if os.path.sep in chip or chip in {".", ".."}: - raise ValueError(f"invalid chip: {chip}") - return chip - - -def bemu_chip_inst(chip: str, bbdir: str | None = None) -> Path: - chip = validate_bemu_chip(chip, bbdir) root = Path(bbdir or get_buckyball_path()) inst = root / "examples" / "chips" / chip / "emu" / "src" / "lib.rs" if not inst.is_file(): diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml deleted file mode 100644 index 603a09f06..000000000 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-elf.toml +++ /dev/null @@ -1,52 +0,0 @@ -# ============================================================================= -# Bebop bemu workload list - ELF baremetal tests -# ============================================================================= -# Test cases to run in bebop bemu batch regression with --elf flag. -# search_path is relative to bb-tests/output (or absolute path). -# Each entry under tests is the file STEM (without extension). - -[workloads] -search_path = "workloads/src" -tests = [ - "mvin_mvout_test-singlecore-baremetal", - "mvin_mvout_stride_test-singlecore-baremetal", - "vecunit_matmul_16xn_ones-singlecore-baremetal", - "vecunit_matmul_16xn_random1-singlecore-baremetal", - "vecunit_matmul_16xn_random2-singlecore-baremetal", - "vecunit_matmul_16xn_random3-singlecore-baremetal", - "vecunit_matmul_16xn_zero_random-singlecore-baremetal", - "vecunit_matmul_col_row_vector-singlecore-baremetal", - "vecunit_matmul_identity_random-singlecore-baremetal", - "vecunit_matmul_ones-singlecore-baremetal", - "vecunit_matmul_random1-singlecore-baremetal", - "vecunit_matmul_random2-singlecore-baremetal", - "vecunit_matmul_zero_random-singlecore-baremetal", - "relu_test-singlecore-baremetal", - "transpose_test-singlecore-baremetal", - "transpose_16xn_test-singlecore-baremetal", - "im2col_test-singlecore-baremetal", - "fp2int_test-singlecore-baremetal", - "int2fp_test-singlecore-baremetal", - "bdb_counter_test-singlecore-baremetal", - "bfp_test-singlecore-baremetal", - "gemmini_os_cisc_basic_test-singlecore-baremetal", - "gemmini_os_cisc_atranspose_test-singlecore-baremetal", - "gemmini_os_cisc_btranspose_test-singlecore-baremetal", - "gemmini_os_cisc_shift_test-singlecore-baremetal", - "gemmini_os_risc_atranspose_test-singlecore-baremetal", - "gemmini_os_risc_btranspose_test-singlecore-baremetal", - "gemmini_os_risc_abtranspose_test-singlecore-baremetal", - "gemmini_os_risc_shift_test-singlecore-baremetal", - "gemmini_ws_cisc_conv_test-singlecore-baremetal", - "gemmini_ws_risc_basic_test-singlecore-baremetal", - "mmio_minimal_test-singlecore-baremetal", - "mxfp2int_test-singlecore-baremetal", - "optest_buckyball_matmul_16x64_64x16_singlecore-baremetal", - "optest_buckyball_matmul_64x16_16x64_singlecore-baremetal", - "optest_buckyball_matmul_1024x16_16x1024_singlecore-baremetal", - "optest_tile_matmul_127x17_17x127_singlecore-baremetal", - "optest_tile_matmul_1x84_84x10_singlecore-baremetal", - "optest_linalg_conv2d_nhwc_hwcf_singlecore-baremetal", - "optest_linalg_conv2d_nhwc_fhwc_singlecore-baremetal", - "optest_linalg_conv2d_nchw_fchw_singlecore-baremetal", -] diff --git a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml deleted file mode 100644 index c1160f04d..000000000 --- a/bbdev/api/steps/bebop/bemu/scripts/workloads-pk.toml +++ /dev/null @@ -1,52 +0,0 @@ -# ============================================================================= -# Bebop bemu workload list - PK (proxy kernel) tests -# ============================================================================= -# Test cases to run in bebop bemu batch regression with --pk flag. -# search_path is relative to bb-tests/output (or absolute path). -# Each entry under tests is the file STEM (without extension). - -[workloads] -search_path = "workloads/src" -tests = [ - "mvin_mvout_test-linux", - "mvin_mvout_stride_test-linux", - "vecunit_matmul_16xn_ones-linux", - "vecunit_matmul_16xn_random1-linux", - "vecunit_matmul_16xn_random2-linux", - "vecunit_matmul_16xn_random3-linux", - "vecunit_matmul_16xn_zero_random-linux", - "vecunit_matmul_col_row_vector-linux", - "vecunit_matmul_identity_random-linux", - "vecunit_matmul_ones-linux", - "vecunit_matmul_random1-linux", - "vecunit_matmul_random2-linux", - "vecunit_matmul_zero_random-linux", - "relu_test-linux", - "transpose_test-linux", - "transpose_16xn_test-linux", - "im2col_test-linux", - "fp2int_test-linux", - "int2fp_test-linux", - "bdb_counter_test-linux", - "bfp_test-linux", - "gemmini_os_cisc_basic_test-linux", - "gemmini_os_cisc_atranspose_test-linux", - "gemmini_os_cisc_btranspose_test-linux", - "gemmini_os_cisc_shift_test-linux", - "gemmini_os_risc_atranspose_test-linux", - "gemmini_os_risc_btranspose_test-linux", - "gemmini_os_risc_abtranspose_test-linux", - "gemmini_os_risc_shift_test-linux", - "gemmini_ws_cisc_conv_test-linux", - "gemmini_ws_risc_basic_test-linux", - "mmio_minimal_test-linux", - "mxfp2int_test-linux", - "optest_buckyball_matmul_16x64_64x16-linux", - "optest_buckyball_matmul_64x16_16x64-linux", - "optest_buckyball_matmul_1024x16_16x1024-linux", - "optest_tile_matmul_127x17_17x127-linux", - "optest_tile_matmul_1x84_84x10-linux", - "optest_linalg_conv2d_nhwc_hwcf-linux", - "optest_linalg_conv2d_nhwc_fhwc-linux", - "optest_linalg_conv2d_nchw_fchw-linux", -] diff --git a/bbdev/api/steps/bebop/p2e/04_batch_api.step.py b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py index 9302e32ba..668b8e2c3 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_api.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py @@ -25,6 +25,13 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: }, ) + chip = body.get("chip") + if not chip: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --chip must be specified"} + ) + test_type = body.get("test") if not test_type: return ApiResponse( @@ -39,6 +46,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: ) data = { + "chip": chip, "bitstream": bitstream, "build_dir": build_dir, "test": test_type, diff --git a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py index 02fd347f3..f501d788a 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py @@ -13,10 +13,14 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +bebop_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +if bebop_path not in sys.path: + sys.path.insert(0, bebop_path) from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id +from regression import regression_workload_toml config = { "name": "bebop-p2e-batch", @@ -45,17 +49,24 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - # ── Determine workload file based on test type ──────────────────────── + chip = input_data.get("chip") + if not chip: + ctx.logger.error("Missing required parameter: chip must be specified") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_chip"}, + trace_id=origin_tid, + ) + return + test_type = input_data.get("test", "elf-tests") - if test_type == "elf-tests": - workload_toml = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/workloads-elf.toml" - elif test_type == "pk-tests": - workload_toml = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/workloads-pk.toml" - else: - ctx.logger.error(f"Invalid test type: {test_type}") + try: + workload_toml = regression_workload_toml(chip, "p2e", test_type, bbdir) + except ValueError as e: + ctx.logger.error(str(e)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "invalid_test_type", "test": test_type}, + extra_fields={"error": "invalid_regression", "test": test_type, "chip": chip}, trace_id=origin_tid, ) return @@ -113,6 +124,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "task": "batch", "backend": "p2e", + "chip": chip, "bitstream": bitstream, "build_dir": build_dir, "test_type": test_type, diff --git a/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml deleted file mode 100644 index df6bd7b9d..000000000 --- a/bbdev/api/steps/bebop/p2e/scripts/workloads-elf.toml +++ /dev/null @@ -1,30 +0,0 @@ -# ============================================================================= -# Bebop p2e workload list - ELF baremetal tests -# ============================================================================= -# search_path is relative to bb-tests/output (or absolute path). -# Each entry under tests is the file STEM (without extension). - -[workloads] -search_path = "workloads/src" -tests = [ - "bdb_counter_test-singlecore-baremetal", - "bfp_test-singlecore-baremetal", - "gemmini_os_cisc_basic_test-singlecore-baremetal", - "gemmini_os_cisc_atranspose_test-singlecore-baremetal", - "gemmini_os_cisc_btranspose_test-singlecore-baremetal", - "gemmini_os_cisc_shift_test-singlecore-baremetal", - "gemmini_os_risc_basic_test-singlecore-baremetal", - "gemmini_os_risc_atranspose_test-singlecore-baremetal", - "gemmini_os_risc_btranspose_test-singlecore-baremetal", - "gemmini_os_risc_abtranspose_test-singlecore-baremetal", - "gemmini_os_risc_shift_test-singlecore-baremetal", - "gemmini_ws_cisc_conv_test-singlecore-baremetal", - "gemmini_ws_risc_basic_test-singlecore-baremetal", - "tlb_bare_mode_test-singlecore-baremetal", - "tlb_sv39_identity_test-singlecore-baremetal", - "tlb_sfence_flush_test-singlecore-baremetal", - "tlb_repeated_sfence_test-singlecore-baremetal", - "tlb_sv39_nonidentity_test-singlecore-baremetal", - "tlb_sv39_4k_alias_test-singlecore-baremetal", - "tlb_sfence_remap_test-singlecore-baremetal", -] diff --git a/bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml deleted file mode 100644 index 5c31fdc2e..000000000 --- a/bbdev/api/steps/bebop/p2e/scripts/workloads-pk.toml +++ /dev/null @@ -1,23 +0,0 @@ -# ============================================================================= -# Bebop p2e workload list - PK (proxy kernel) tests -# ============================================================================= -# search_path is relative to bb-tests/output (or absolute path). -# Each entry under tests is the file STEM (without extension). - -[workloads] -search_path = "workloads/src" -tests = [ - "bdb_counter_test-linux", - "bfp_test-linux", - "gemmini_os_cisc_basic_test-linux", - "gemmini_os_cisc_atranspose_test-linux", - "gemmini_os_cisc_btranspose_test-linux", - "gemmini_os_cisc_shift_test-linux", - "gemmini_os_risc_basic_test-linux", - "gemmini_os_risc_atranspose_test-linux", - "gemmini_os_risc_btranspose_test-linux", - "gemmini_os_risc_abtranspose_test-linux", - "gemmini_os_risc_shift_test-linux", - "gemmini_ws_cisc_conv_test-linux", - "gemmini_ws_risc_basic_test-linux", -] diff --git a/bbdev/api/steps/bebop/regression.py b/bbdev/api/steps/bebop/regression.py new file mode 100644 index 000000000..36f62b844 --- /dev/null +++ b/bbdev/api/steps/bebop/regression.py @@ -0,0 +1,39 @@ +import os +from pathlib import Path + +from utils.path import get_buckyball_path + + +def validate_chip(chip: str | None) -> str: + if not chip: + raise ValueError("missing required parameter: chip") + if not isinstance(chip, str): + raise ValueError("chip must be a string") + if os.path.sep in chip or chip in {".", ".."}: + raise ValueError(f"invalid chip: {chip}") + return chip + + +def chip_regression_dir(chip: str, backend: str, bbdir: str | None = None) -> Path: + chip = validate_chip(chip) + root = Path(bbdir or get_buckyball_path()) + return root / "examples" / "chips" / chip / "regression" / "batch" / backend + + +def regression_workload_toml( + chip: str, + backend: str, + test_type: str, + bbdir: str | None = None, +) -> str: + if test_type == "elf-tests": + suffix = "elf" + elif test_type == "pk-tests": + suffix = "pk" + else: + raise ValueError(f"invalid test type: {test_type}") + + toml = chip_regression_dir(chip, backend, bbdir) / f"workloads-{suffix}.toml" + if not toml.is_file(): + raise ValueError(f"regression workload toml does not exist: {toml}") + return str(toml) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py index 2db727c8c..65740f5dd 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -22,6 +22,13 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body={"error": "Missing required parameter: --config must be specified"} ) + chip = body.get("chip") + if not chip: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --chip must be specified"} + ) + test_type = body.get("test") if not test_type: return ApiResponse( @@ -38,6 +45,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: vsrc_dir = get_verilator_build_dir(bbdir, arch_config, body.get("vsrc_dir")) data = { + "chip": chip, "config": arch_config, "vsrc_dir": vsrc_dir, "test": test_type, diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 7c1f27daa..9b523f528 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -15,10 +15,14 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +bebop_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +if bebop_path not in sys.path: + sys.path.insert(0, bebop_path) from utils.path import get_buckyball_path, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id +from regression import regression_workload_toml config = { "name": "bebop-verilator-batch", @@ -46,17 +50,24 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - # ── Determine workload file based on test type ──────────────────────── + chip = input_data.get("chip") + if not chip: + ctx.logger.error("Missing required parameter: chip must be specified") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_chip"}, + trace_id=origin_tid, + ) + return + test_type = input_data.get("test", "elf-tests") - if test_type == "elf-tests": - workload_toml = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/workloads-elf.toml" - elif test_type == "pk-tests": - workload_toml = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/workloads-pk.toml" - else: - ctx.logger.error(f"Invalid test type: {test_type}") + try: + workload_toml = regression_workload_toml(chip, "verilator", test_type, bbdir) + except ValueError as e: + ctx.logger.error(str(e)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "invalid_test_type", "test": test_type}, + extra_fields={"error": "invalid_regression", "test": test_type, "chip": chip}, trace_id=origin_tid, ) return @@ -134,6 +145,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "task": "batch", "backend": "verilator", + "chip": chip, "config": arch_config, "vsrc_dir": vsrc_dir, "test_type": test_type, diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml deleted file mode 100644 index 09b0ce40d..000000000 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-elf.toml +++ /dev/null @@ -1,62 +0,0 @@ -# ============================================================================= -# Bebop verilator workload list - ELF baremetal tests -# ============================================================================= -# search_path is relative to bb-tests/output (or absolute path). -# Each entry under tests is the file STEM (without extension). - -[workloads] -search_path = "workloads/src" -tests = [ - "mvin_mvout_test-singlecore-baremetal", - "vecunit_matmul_16xn_ones-singlecore-baremetal", - "vecunit_matmul_16xn_random1-singlecore-baremetal", - "vecunit_matmul_16xn_random2-singlecore-baremetal", - "vecunit_matmul_16xn_random3-singlecore-baremetal", - "vecunit_matmul_16xn_zero_random-singlecore-baremetal", - "vecunit_matmul_col_row_vector-singlecore-baremetal", - "vecunit_matmul_identity_random-singlecore-baremetal", - "vecunit_matmul_ones-singlecore-baremetal", - "vecunit_matmul_random1-singlecore-baremetal", - "vecunit_matmul_random2-singlecore-baremetal", - "vecunit_matmul_zero_random-singlecore-baremetal", - "relu_test-singlecore-baremetal", - "transpose_test-singlecore-baremetal", - "transpose_16xn_test-singlecore-baremetal", - "im2col_test-singlecore-baremetal", - "fp2int_test-singlecore-baremetal", - "int2fp_test-singlecore-baremetal", - "bdb_counter_test-singlecore-baremetal", - "bfp_test-singlecore-baremetal", - "gemmini_os_cisc_basic_test-singlecore-baremetal", - "gemmini_os_cisc_atranspose_test-singlecore-baremetal", - "gemmini_os_cisc_btranspose_test-singlecore-baremetal", - "gemmini_os_cisc_shift_test-singlecore-baremetal", - "gemmini_os_risc_atranspose_test-singlecore-baremetal", - "gemmini_os_risc_btranspose_test-singlecore-baremetal", - "gemmini_os_risc_abtranspose_test-singlecore-baremetal", - "gemmini_os_risc_shift_test-singlecore-baremetal", - "gemmini_ws_cisc_conv_test-singlecore-baremetal", - "gemmini_ws_risc_basic_test-singlecore-baremetal", - "mmio_minimal_test-singlecore-baremetal", - "mxfp2int_test-singlecore-baremetal", - - "tlb_bare_mode_test-singlecore-baremetal", - "tlb_sv39_identity_test-singlecore-baremetal", - "tlb_sfence_flush_test-singlecore-baremetal", - "tlb_repeated_sfence_test-singlecore-baremetal", - "tlb_sv39_nonidentity_test-singlecore-baremetal", - "tlb_sv39_4k_alias_test-singlecore-baremetal", - "tlb_sfence_remap_test-singlecore-baremetal", - - "mvin_mvout_stride_test-singlecore-baremetal", - "transpose_matmul-singlecore-baremetal", - "optest_buckyball_matmul_16x64_64x16_singlecore-baremetal", - "optest_buckyball_matmul_64x16_16x64_singlecore-baremetal", - # Disabled for Verilator batch because these tests are too slow. - # "optest_buckyball_matmul_1024x16_16x1024_singlecore-baremetal", - # "optest_tile_matmul_127x17_17x127_singlecore-baremetal", - "optest_tile_matmul_1x84_84x10_singlecore-baremetal", - "optest_linalg_conv2d_nhwc_hwcf_singlecore-baremetal", - "optest_linalg_conv2d_nhwc_fhwc_singlecore-baremetal", - "optest_linalg_conv2d_nchw_fchw_singlecore-baremetal", -] diff --git a/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml b/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml deleted file mode 100644 index 30aea7b5f..000000000 --- a/bbdev/api/steps/bebop/verilator/scripts/workloads-pk.toml +++ /dev/null @@ -1,52 +0,0 @@ -# ============================================================================= -# Bebop verilator workload list - PK (proxy kernel) tests -# ============================================================================= -# search_path is relative to bb-tests/output (or absolute path). -# Each entry under tests is the file STEM (without extension). - -[workloads] -search_path = "workloads/src" -tests = [ - "mvin_mvout_test-linux", - "mvin_mvout_stride_test-linux", - "vecunit_matmul_16xn_ones-linux", - "vecunit_matmul_16xn_random1-linux", - "vecunit_matmul_16xn_random2-linux", - "vecunit_matmul_16xn_random3-linux", - "vecunit_matmul_16xn_zero_random-linux", - "vecunit_matmul_col_row_vector-linux", - "vecunit_matmul_identity_random-linux", - "vecunit_matmul_ones-linux", - "vecunit_matmul_random1-linux", - "vecunit_matmul_random2-linux", - "vecunit_matmul_zero_random-linux", - "relu_test-linux", - "transpose_test-linux", - "transpose_16xn_test-linux", - "im2col_test-linux", - "fp2int_test-linux", - "int2fp_test-linux", - "bdb_counter_test-linux", - "bfp_test-linux", - "gemmini_os_cisc_basic_test-linux", - "gemmini_os_cisc_atranspose_test-linux", - "gemmini_os_cisc_btranspose_test-linux", - "gemmini_os_cisc_shift_test-linux", - "gemmini_os_risc_atranspose_test-linux", - "gemmini_os_risc_btranspose_test-linux", - "gemmini_os_risc_abtranspose_test-linux", - "gemmini_os_risc_shift_test-linux", - "gemmini_ws_cisc_conv_test-linux", - "gemmini_ws_risc_basic_test-linux", - "mmio_minimal_test-linux", - "mxfp2int_test-linux", - "optest_buckyball_matmul_16x64_64x16-linux", - "optest_buckyball_matmul_64x16_16x64-linux", - # Disabled for Verilator batch because these tests are too slow. - # "optest_buckyball_matmul_1024x16_16x1024-linux", - # "optest_tile_matmul_127x17_17x127-linux", - "optest_tile_matmul_1x84_84x10-linux", - "optest_linalg_conv2d_nhwc_hwcf-linux", - "optest_linalg_conv2d_nhwc_fhwc-linux", - "optest_linalg_conv2d_nchw_fchw-linux", -] diff --git a/bbdev/api/tests/test_api_bebop_verilator_batch.py b/bbdev/api/tests/test_api_bebop_verilator_batch.py index d2053cbf7..1c383a2ef 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_batch.py +++ b/bbdev/api/tests/test_api_bebop_verilator_batch.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --batch '--config sims.verilator.BuckyballToyVerilatorConfig --test elf-tests'") +run_bbdev_case("bbdev bebop-verilator --batch '--chip toy --config sims.verilator.BuckyballToyVerilatorConfig --test elf-tests'") diff --git a/bbdev/bbdev b/bbdev/bbdev index 64c01987e..8ecbbc575 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -68,14 +68,14 @@ WORKFLOW_COMMANDS = { "build": 'Build Bebop Verilator simulation executable. Args: "--config [--jobs ] [--output-dir ]"', "sim": 'Run Bebop Verilator simulation. Args: "--config --binary [--batch] [--log-dir ] [--fst-dir ] [--no-wave]"', "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', - "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--config --test [--clean-before]"', + "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--clean-before]"', }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--config [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--config [--output-dir ]"', "buildbitstream": 'Build Bebop P2E runtime case. Args: "--config [--vsrc-dir ] [--output-dir ]"', "runworkload": 'Run Bebop P2E workload. Args: "--image --bitstream [--multi-fpga] [--wave] [--wave-start ] [--build-dir ]"', - "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--bitstream --build-dir --test "', + "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--chip --bitstream --build-dir --test "', }, "dc": { "verilog": 'Generate DC RTL. Args: "--config [--dir ]"', From a2c7c925275c9b2df0c60a70dd1a81e8f943b68e Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 19 Jul 2026 20:04:47 +0800 Subject: [PATCH 095/196] fix: don't allow --batch to build --- .../bebop/verilator/03_build_event.step.py | 2 +- .../bebop/verilator/04_sim_event.step.py | 52 ++----------------- .../bebop/verilator/06_batch_event.step.py | 37 +++++-------- .../bebop/verilator/scripts/build_marker.py | 32 ++++++++++++ 4 files changed, 50 insertions(+), 73 deletions(-) diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index 30d4ca37c..521517606 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -87,7 +87,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return build_cmd = ( - f"cargo build --features verilator " + f"cargo build --features verilator --tests " f"--config=\"env.VSRC_PATH='{vsrc_dir}'\"" ) ctx.logger.info("Building bebop verilator ...") diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index 414eeef00..6d4d02ff2 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -9,7 +9,6 @@ import os import shlex import sys -import json from datetime import datetime from motia import FlowContext, queue @@ -25,7 +24,7 @@ from utils.stream_run import stream_run_logger from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id -from build_marker import build_marker_path, read_build_marker +from build_marker import verify_build_marker config = { "name": "bebop-verilator-sim", @@ -63,53 +62,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return bebop_bin = f"{bebop_dir}/target/debug/bebop" - if not os.path.isfile(bebop_bin): - ctx.logger.error(f"bebop binary does not exist: {bebop_bin}") + marker_err = verify_build_marker(bebop_dir, arch_config, vsrc_dir) + if marker_err is not None: + ctx.logger.error(f"bebop verilator build check failed: {marker_err}") await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "bebop_binary_not_found", "binary": bebop_bin}, - trace_id=origin_tid, - ) - return - try: - marker = read_build_marker(bebop_dir) - except (OSError, json.JSONDecodeError) as e: - marker_path = build_marker_path(bebop_dir) - ctx.logger.error(f"failed to read bebop verilator build marker: {e}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "build_marker_read_failed", "marker": marker_path, "detail": str(e)}, - trace_id=origin_tid, - ) - return - expect_vsrc = os.path.abspath(vsrc_dir) - expect_bin = os.path.abspath(bebop_bin) - if marker is None: - marker_path = build_marker_path(bebop_dir) - ctx.logger.error(f"bebop verilator build marker does not exist: {marker_path}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "build_marker_not_found", "marker": marker_path}, - trace_id=origin_tid, - ) - return - if ( - marker.get("config") != arch_config - or marker.get("vsrc_dir") != expect_vsrc - or marker.get("binary") != expect_bin - ): - ctx.logger.error(f"bebop verilator build marker mismatch: {marker}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={ - "error": "build_marker_mismatch", - "marker": marker, - "expected": { - "config": arch_config, - "vsrc_dir": expect_vsrc, - "binary": expect_bin, - }, - }, + extra_fields=marker_err, trace_id=origin_tid, ) return diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 9b523f528..cd5a0022e 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -1,9 +1,7 @@ """ bebop verilator batch event handler -Runs bebop verilator nextest batch regression: - 1. Build bebop with verilator feature and VSRC_PATH - 2. Run cargo nextest with verilator-specific config +Runs bebop verilator nextest batch regression (requires prior --build). """ import os import shutil @@ -15,6 +13,9 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) bebop_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) if bebop_path not in sys.path: sys.path.insert(0, bebop_path) @@ -23,6 +24,7 @@ from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id from regression import regression_workload_toml +from build_marker import verify_build_marker config = { "name": "bebop-verilator-batch", @@ -84,32 +86,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - # ── Build bebop verilator (tests) ───────────────────────────────────── - vsrc_config = shlex.quote(f"env.VSRC_PATH='{vsrc_dir}'") - build_cmd = ( - f"nix develop -c cargo build --manifest-path {shlex.quote(f'{bebop_dir}/Cargo.toml')} " - "--features verilator --tests " - f"--config={vsrc_config}" - ) - ctx.logger.info("Building bebop verilator (tests)...") - build_result = stream_run_logger( - cmd=build_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="bebop verilator build", - stderr_prefix="bebop verilator build", - ) - - if build_result.returncode != 0: + marker_err = verify_build_marker(bebop_dir, arch_config, vsrc_dir) + if marker_err is not None: + ctx.logger.error(f"bebop verilator build check failed: {marker_err}") await check_result( - ctx, build_result.returncode, continue_run=False, - extra_fields={"task": "build", "backend": "verilator", "vsrc_dir": vsrc_dir}, + ctx, 1, continue_run=False, + extra_fields=marker_err, trace_id=origin_tid, ) return - # ── Run nextest ─────────────────────────────────────────────────────── - # Pass parameters via environment variables (nextest doesn't support custom CLI args after `--`) + vsrc_config = shlex.quote(f"env.VSRC_PATH='{vsrc_dir}'") if input_data.get("clean-before", input_data.get("clean_before", False)): shutil.rmtree(f"{bebop_dir}/test-artifacts", ignore_errors=True) ctx.logger.info("Cleaned previous bebop test artifacts") @@ -122,7 +109,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: }) nextest_cmd = ( f"nix develop -c cargo nextest run --manifest-path {shlex.quote(f'{bebop_dir}/Cargo.toml')} " - "--features verilator --test test_verilator " + "--features verilator --test test_verilator --no-build " f"--config-file {shlex.quote(nextest_config)} " f"--config={vsrc_config}" ) diff --git a/bbdev/api/steps/bebop/verilator/scripts/build_marker.py b/bbdev/api/steps/bebop/verilator/scripts/build_marker.py index d4726d27e..4596789b1 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/build_marker.py +++ b/bbdev/api/steps/bebop/verilator/scripts/build_marker.py @@ -28,3 +28,35 @@ def read_build_marker(bebop_dir: str) -> dict | None: return json.load(f) except FileNotFoundError: return None + + +def verify_build_marker(bebop_dir: str, arch_config: str, vsrc_dir: str) -> dict | None: + bebop_bin = os.path.join(bebop_dir, "target", "debug", "bebop") + if not os.path.isfile(bebop_bin): + return {"error": "bebop_binary_not_found", "binary": bebop_bin} + + try: + marker = read_build_marker(bebop_dir) + except (OSError, json.JSONDecodeError) as e: + marker_path = build_marker_path(bebop_dir) + return {"error": "build_marker_read_failed", "marker": marker_path, "detail": str(e)} + + expect_vsrc = os.path.abspath(vsrc_dir) + expect_bin = os.path.abspath(bebop_bin) + if marker is None: + return {"error": "build_marker_not_found", "marker": build_marker_path(bebop_dir)} + if ( + marker.get("config") != arch_config + or marker.get("vsrc_dir") != expect_vsrc + or marker.get("binary") != expect_bin + ): + return { + "error": "build_marker_mismatch", + "marker": marker, + "expected": { + "config": arch_config, + "vsrc_dir": expect_vsrc, + "binary": expect_bin, + }, + } + return None From 93502bc62502aa17b604d7c4af0563b0de729a2a Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 19 Jul 2026 20:40:31 +0800 Subject: [PATCH 096/196] fix: delete unused verify --- .../bebop/verilator/04_sim_event.step.py | 54 ++++++++++++++++--- .../bebop/verilator/06_batch_event.step.py | 16 +----- .../bebop/verilator/scripts/build_marker.py | 41 ++------------ .../firesim/scripts/yaml/config_build.yaml | 4 +- .../firesim/scripts/yaml/config_hwdb.yaml | 7 ++- .../firesim/scripts/yaml/config_runtime.yaml | 2 +- 6 files changed, 61 insertions(+), 63 deletions(-) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index 6d4d02ff2..15c0c2f4a 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -6,6 +6,7 @@ 2. Resolve verilog source directory (VSRC_PATH) 3. Run the built bebop binary with the new run/verilator CLI """ +import json import os import shlex import sys @@ -24,7 +25,7 @@ from utils.stream_run import stream_run_logger from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id -from build_marker import verify_build_marker +from build_marker import build_marker_path, read_build_marker config = { "name": "bebop-verilator-sim", @@ -62,12 +63,54 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return bebop_bin = f"{bebop_dir}/target/debug/bebop" - marker_err = verify_build_marker(bebop_dir, arch_config, vsrc_dir) - if marker_err is not None: - ctx.logger.error(f"bebop verilator build check failed: {marker_err}") + if not os.path.isfile(bebop_bin): + ctx.logger.error(f"bebop binary does not exist: {bebop_bin}") await check_result( ctx, 1, continue_run=False, - extra_fields=marker_err, + extra_fields={"error": "bebop_binary_not_found", "binary": bebop_bin}, + trace_id=origin_tid, + ) + return + + marker_path = build_marker_path(bebop_dir) + try: + marker = read_build_marker(bebop_dir) + except FileNotFoundError: + ctx.logger.error(f"bebop verilator build marker does not exist: {marker_path}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "build_marker_not_found", "marker": marker_path}, + trace_id=origin_tid, + ) + return + except (OSError, json.JSONDecodeError) as e: + ctx.logger.error(f"failed to read bebop verilator build marker: {e}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "build_marker_read_failed", "marker": marker_path, "detail": str(e)}, + trace_id=origin_tid, + ) + return + + expect_vsrc = os.path.abspath(vsrc_dir) + expect_bin = os.path.abspath(bebop_bin) + if ( + marker.get("config") != arch_config + or marker.get("vsrc_dir") != expect_vsrc + or marker.get("binary") != expect_bin + ): + ctx.logger.error(f"bebop verilator build marker mismatch: {marker}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={ + "error": "build_marker_mismatch", + "marker": marker, + "expected": { + "config": arch_config, + "vsrc_dir": expect_vsrc, + "binary": expect_bin, + }, + }, trace_id=origin_tid, ) return @@ -90,7 +133,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: os.makedirs(log_dir, exist_ok=True) os.makedirs(fst_dir, exist_ok=True) - # ── Run bebop run verilator ────────────────────────────────────────── wave_arg = " --no-wave" if input_data.get("no-wave", False) or input_data.get("no_wave", False) else "" run_cmd = ( diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index cd5a0022e..25c8b0325 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -13,9 +13,6 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -scripts_path = os.path.join(os.path.dirname(__file__), "scripts") -if scripts_path not in sys.path: - sys.path.insert(0, scripts_path) bebop_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) if bebop_path not in sys.path: sys.path.insert(0, bebop_path) @@ -24,7 +21,6 @@ from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id from regression import regression_workload_toml -from build_marker import verify_build_marker config = { "name": "bebop-verilator-batch", @@ -86,16 +82,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - marker_err = verify_build_marker(bebop_dir, arch_config, vsrc_dir) - if marker_err is not None: - ctx.logger.error(f"bebop verilator build check failed: {marker_err}") - await check_result( - ctx, 1, continue_run=False, - extra_fields=marker_err, - trace_id=origin_tid, - ) - return - vsrc_config = shlex.quote(f"env.VSRC_PATH='{vsrc_dir}'") if input_data.get("clean-before", input_data.get("clean_before", False)): shutil.rmtree(f"{bebop_dir}/test-artifacts", ignore_errors=True) @@ -109,7 +95,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: }) nextest_cmd = ( f"nix develop -c cargo nextest run --manifest-path {shlex.quote(f'{bebop_dir}/Cargo.toml')} " - "--features verilator --test test_verilator --no-build " + "--features verilator --test test_verilator " f"--config-file {shlex.quote(nextest_config)} " f"--config={vsrc_config}" ) diff --git a/bbdev/api/steps/bebop/verilator/scripts/build_marker.py b/bbdev/api/steps/bebop/verilator/scripts/build_marker.py index 4596789b1..ec726443a 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/build_marker.py +++ b/bbdev/api/steps/bebop/verilator/scripts/build_marker.py @@ -21,42 +21,7 @@ def write_build_marker(bebop_dir: str, config: str, vsrc_dir: str, binary: str): os.replace(tmp, marker) -def read_build_marker(bebop_dir: str) -> dict | None: +def read_build_marker(bebop_dir: str) -> dict: marker = build_marker_path(bebop_dir) - try: - with open(marker) as f: - return json.load(f) - except FileNotFoundError: - return None - - -def verify_build_marker(bebop_dir: str, arch_config: str, vsrc_dir: str) -> dict | None: - bebop_bin = os.path.join(bebop_dir, "target", "debug", "bebop") - if not os.path.isfile(bebop_bin): - return {"error": "bebop_binary_not_found", "binary": bebop_bin} - - try: - marker = read_build_marker(bebop_dir) - except (OSError, json.JSONDecodeError) as e: - marker_path = build_marker_path(bebop_dir) - return {"error": "build_marker_read_failed", "marker": marker_path, "detail": str(e)} - - expect_vsrc = os.path.abspath(vsrc_dir) - expect_bin = os.path.abspath(bebop_bin) - if marker is None: - return {"error": "build_marker_not_found", "marker": build_marker_path(bebop_dir)} - if ( - marker.get("config") != arch_config - or marker.get("vsrc_dir") != expect_vsrc - or marker.get("binary") != expect_bin - ): - return { - "error": "build_marker_mismatch", - "marker": marker, - "expected": { - "config": arch_config, - "vsrc_dir": expect_vsrc, - "binary": expect_bin, - }, - } - return None + with open(marker) as f: + return json.load(f) diff --git a/bbdev/api/steps/firesim/scripts/yaml/config_build.yaml b/bbdev/api/steps/firesim/scripts/yaml/config_build.yaml index e6ee2d5ce..b6c400207 100644 --- a/bbdev/api/steps/firesim/scripts/yaml/config_build.yaml +++ b/bbdev/api/steps/firesim/scripts/yaml/config_build.yaml @@ -31,8 +31,8 @@ build_farm: builds_to_run: # this section references builds defined in config_build_recipes.yaml # if you add a build here, it will be built when you run buildbitstream - # - alveo_u280_firesim_GemminiBuckyballConfig_no_nic - - alveo_u280_firesim_BuckyballToyConfig_no_nic + - alveo_u280_firesim_GemminiBuckyballConfig_no_nic + # - alveo_u280_firesim_BuckyballToyConfig_no_nic agfis_to_share: # - midasexamples_gcd diff --git a/bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml b/bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml index 98b8155e4..f5fd36d94 100644 --- a/bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml +++ b/bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml @@ -38,6 +38,11 @@ alveo_u280_firesim_GemminiBuckyballConfig_no_nic: # custom_runtime_config: null alveo_u280_firesim_BuckyballToyConfig_no_nic: - bitstream_tar: file:///home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/results-build/2025-11-23--04-11-23-alveo_u280_firesim_BuckyballToyConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimBuckyballToyConfig-BaseXilinxAlveoU280Config/firesim.tar.gz + bitstream_tar: file:///home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/results-build/2026-05-12--15-52-50-alveo_u280_firesim_BuckyballToyConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimBuckyballToyConfig-BaseXilinxAlveoU280Config/firesim.tar.gz + deploy_quintuplet_override: null + custom_runtime_config: null + +alveo_u280_firesim_GemminiBuckyballConfig_no_nic: + bitstream_tar: file:///home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/results-build/2026-05-12--19-18-24-alveo_u280_firesim_GemminiBuckyballConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimGemminiBuckyballConfig-BaseXilinxAlveoU280Config/firesim.tar.gz deploy_quintuplet_override: null custom_runtime_config: null diff --git a/bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml b/bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml index 490db5f9b..807fafc6b 100644 --- a/bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml +++ b/bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml @@ -54,7 +54,7 @@ target_config: # or from config_build_recipes.yaml for metasimulation # In homogeneous configurations, use this to set the hardware config deployed # for all simulators - default_hw_config: alveo_u280_firesim_BuckyballToyConfig_no_nic + default_hw_config: alveo_u280_firesim_GemminiBuckyballConfig_no_nic # Advanced: Specify any extra plusargs you would like to provide when # booting the simulator (in both FPGA-sim and metasim modes). This is From 0634f0f61d022037ee0ec18c0bb49632ed9c268f Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 23 Jul 2026 15:25:45 +0800 Subject: [PATCH 097/196] update: bump dramsim2 to dramsim3 --- .../bebop/p2e/03_runworkload_event.step.py | 110 +++++++++++++----- .../bebop/verilator/04_sim_event.step.py | 5 + .../steps/verilator/03_build_event.step.py | 2 +- .../api/steps/verilator/04_sim_event.step.py | 3 +- bbdev/api/steps/workload/scripts/elf2hex.py | 5 +- bbdev/api/tests/test_api_bebop_bemu_sim.py | 2 +- .../api/tests/test_api_bebop_verilator_run.py | 2 +- .../api/tests/test_api_bebop_verilator_sim.py | 2 +- bbdev/api/tests/test_api_verilator_run.py | 2 +- bbdev/api/tests/test_api_verilator_sim.py | 2 +- 10 files changed, 98 insertions(+), 37 deletions(-) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index 23a0e146f..48be411c2 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -8,6 +8,7 @@ """ import glob import os +import re import sys from datetime import datetime @@ -17,7 +18,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path +from utils.path import get_buckyball_path, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id @@ -39,6 +40,21 @@ def resolve_image(bbdir: str, image_name: str) -> str: return matches[0] +def resolve_runtime_config(bitstream: str, requested_config: object) -> str: + if isinstance(requested_config, str) and requested_config: + return requested_config + + build_dir = os.path.dirname(os.path.dirname(os.path.abspath(bitstream))) + case_name = os.path.basename(build_dir) + return re.sub(r"-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}$", "", case_name) + + +def case_uses_multi_fpga(build_dir: str) -> bool: + fpga_comp_dir = os.path.join(build_dir, "fpgaCompDir") + part_dirs = glob.glob(os.path.join(fpga_comp_dir, "part_b*_f*")) + return len([path for path in part_dirs if os.path.isdir(path)]) > 1 + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -98,19 +114,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - # Infer build_dir from bitstream path - # bitstream is typically: /fpgaCompDir/bitstream.bit - # so build_dir is 2 levels up - build_dir = input_data.get("build_dir") - if not build_dir: - build_dir = input_data.get("build-dir") - if not build_dir: - bitstream_abs = os.path.abspath(bitstream) - # Go up: fpgaCompDir -> build_dir - build_dir = os.path.dirname(os.path.dirname(bitstream_abs)) - + bitstream = os.path.abspath(bitstream) + build_dir = os.path.dirname(os.path.dirname(bitstream)) if not os.path.isdir(build_dir): - ctx.logger.error(f"build_dir not found (inferred from bitstream): {build_dir}") + ctx.logger.error(f"P2E build case not found for bitstream: {build_dir}") await check_result( ctx, 1, continue_run=False, extra_fields={"error": "build_dir_not_found", "build_dir": build_dir}, @@ -118,29 +125,74 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return + if not multi_fpga and case_uses_multi_fpga(build_dir): + multi_fpga = True + ctx.logger.info(f"Detected multi-FPGA P2E case: {build_dir}") + + config_name = resolve_runtime_config(bitstream, input_data.get("config")) + vsrc_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("vsrc_dir")) + if not os.path.isdir(vsrc_dir): + ctx.logger.error(f"VSRC_PATH does not exist for P2E runtime: {vsrc_dir}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={ + "error": "vsrc_not_found", + "config": config_name, + "vsrc_dir": vsrc_dir, + }, + trace_id=origin_tid, + ) + return + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - log_dir = input_data.get("log_dir") - if not log_dir: - log_dir = input_data.get("log-dir") - if not log_dir: - log_dir = f"{bebop_dir}/log/p2e-runworkload-{timestamp}" + log_dir = f"{bbdir}/log/{timestamp}-p2e-{image_name}" os.makedirs(log_dir, exist_ok=True) - if input_data.get("log_dir") or input_data.get("log-dir"): - ctx.logger.warning( - "bebop run p2e no longer accepts a separate output/build directory; " - "using build_dir as --log-dir so VVAC runtime artifacts can be found" + # Rebuild the VVAC host runtime in the bitstream case. The bitstream is + # deliberately left in place, so runtime/DPIC changes never trigger FPGA synthesis. + runtime_cmd = ( + f"env BEBOP_P2E_RUNTIME_ONLY=1 BEBOP_P2E_REBUILD_RUNTIME=1 " + f"cargo run --features p2e -- build p2e " + f"--rtl-dir=\"{vsrc_dir}\" " + f"--out-dir=\"{build_dir}\"" + ) + ctx.logger.info("Preparing bebop p2e runtime for the selected bitstream ...") + runtime_result = stream_run_logger( + cmd=runtime_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop p2e runtime", + stderr_prefix="bebop p2e runtime", + ) + rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") + libvctb_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so") + if runtime_result.returncode != 0 or not all(os.path.isfile(path) for path in (rtcfg_path, libvctb_path)): + missing = [path for path in (rtcfg_path, libvctb_path) if not os.path.isfile(path)] + if missing: + ctx.logger.error(f"P2E runtime artifacts missing: {missing}") + await check_result( + ctx, + runtime_result.returncode or 1, + continue_run=False, + extra_fields={ + "task": "runtime", + "config": config_name, + "vsrc_dir": vsrc_dir, + "build_dir": build_dir, + "missing": missing, + }, + trace_id=origin_tid, ) + return # ── Run bebop run p2e ───────────────────────────────────────────────── run_cmd = ( - f"nix develop --ignore-environment --keep HOME --keep ALL_PROXY -c " f"cargo run --features p2e " f"--config=\"env.OUT_PATH='{build_dir}'\" " f"-- run p2e " f"--image=\"{image_path}\" " f"--bitstream=\"{bitstream}\" " - f"--log-dir=\"{build_dir}\"" + f"--log-dir=\"{log_dir}\"" ) if multi_fpga: run_cmd += " --multi-fpga" @@ -148,6 +200,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: run_cmd += " --wave" if wave_start is not None: run_cmd += f" --wave-start=\"{wave_start}\"" + for trace_name in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace"): + if input_data.get(trace_name, False): + run_cmd += f" --{trace_name}" ctx.logger.info(f"Running bebop p2e runworkload: {run_cmd}") run_result = stream_run_logger( cmd=run_cmd, @@ -165,10 +220,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "task": "runworkload", "image": image_path, "bitstream": bitstream, + "config": config_name, "build_dir": build_dir, - "log_dir": build_dir, - "requested_log_dir": log_dir, - "uart_log": os.path.join(build_dir, "uart.log"), + "log_dir": log_dir, + "bdb_trace": os.path.join(log_dir, "bdb.ndjson"), + "uart_log": os.path.join(log_dir, "uart.log"), "timestamp": timestamp, }, trace_id=origin_tid, diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index 15c0c2f4a..e20ccc865 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -134,6 +134,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: os.makedirs(fst_dir, exist_ok=True) wave_arg = " --no-wave" if input_data.get("no-wave", False) or input_data.get("no_wave", False) else "" + trace_args = "" + for trace_name in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace"): + if input_data.get(trace_name, False): + trace_args += f" --{trace_name}" run_cmd = ( f"{shlex.quote(bebop_bin)} run verilator " @@ -141,6 +145,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"--log-dir={shlex.quote(log_dir)} " f"--fst-dir={shlex.quote(fst_dir)}" f"{wave_arg}" + f"{trace_args}" ) ctx.logger.info(f"Running bebop verilator: {run_cmd}") run_result = stream_run_logger( diff --git a/bbdev/api/steps/verilator/03_build_event.step.py b/bbdev/api/steps/verilator/03_build_event.step.py index 64806092e..58831ba31 100644 --- a/bbdev/api/steps/verilator/03_build_event.step.py +++ b/bbdev/api/steps/verilator/03_build_event.step.py @@ -90,7 +90,7 @@ def pkg_config(flag, pkg): cflags = f"{inc_flags} -DBBSIM -DTOP_NAME='\"V{topname}\"' -std=c++17" ldflags = ( - f"-lreadline -ldramsim -lstdc++ -lz " + f"-lreadline -ldramsim3 -lstdc++ -lz " f"-L{result_dir}/lib " f"-L{readline_lib} -Wl,-rpath,{readline_lib} " f"-L{zlib_lib} -Wl,-rpath,{zlib_lib} " diff --git a/bbdev/api/steps/verilator/04_sim_event.step.py b/bbdev/api/steps/verilator/04_sim_event.step.py index d880c5a02..3cc6aae9b 100644 --- a/bbdev/api/steps/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -88,8 +88,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # BDB_SIM_META moves NDJSON banner to sim_meta.txt so it does not pollute disasm. # ================================================================================== ld_lib_path = ( - f"{bbdir}/result/lib:" - f"{arch_dir}/thirdparty/chipyard/tools/DRAMSim2" + f"{bbdir}/result/lib" ) sim_cmd = ( f"export LD_LIBRARY_PATH=\"{ld_lib_path}:$LD_LIBRARY_PATH\"; " diff --git a/bbdev/api/steps/workload/scripts/elf2hex.py b/bbdev/api/steps/workload/scripts/elf2hex.py index e9441e77f..782f2b080 100755 --- a/bbdev/api/steps/workload/scripts/elf2hex.py +++ b/bbdev/api/steps/workload/scripts/elf2hex.py @@ -2,7 +2,7 @@ """ELF -> hex converter for Buckyball P2E DDR loading. Output format: - @0 + @0x0 XX XX ... @@ -28,7 +28,8 @@ def elf_to_bin(elf_path: Path, bin_path: Path, objcopy: str) -> None: def bin_to_hex(bin_path: Path, hex_path: Path) -> int: data = bin_path.read_bytes() - lines = ["@0"] + # VDBG accepts hexadecimal address markers only, including address zero. + lines = ["@0x0"] lines.extend(f"{b:02X}" for b in data) hex_path.write_text("\n".join(lines) + "\n") return len(data) diff --git a/bbdev/api/tests/test_api_bebop_bemu_sim.py b/bbdev/api/tests/test_api_bebop_bemu_sim.py index 0cc425e71..04460ed3e 100644 --- a/bbdev/api/tests/test_api_bebop_bemu_sim.py +++ b/bbdev/api/tests/test_api_bebop_bemu_sim.py @@ -1,4 +1,4 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary ctest_vecunit_matmul_ones_singlecore-baremetal'") +run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary vecunit_matmul_ones-singlecore-baremetal'") run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary ctest_vecunit_matmul_ones-linux --pk'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_run.py b/bbdev/api/tests/test_api_bebop_verilator_run.py index 6473fb549..e71520f06 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_run.py +++ b/bbdev/api/tests/test_api_bebop_verilator_run.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --run '--jobs 16 --binary ctest_vecunit_matmul_ones_singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig --batch'") +run_bbdev_case("bbdev bebop-verilator --run '--jobs 16 --binary vecunit_matmul_ones-singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig --batch'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_sim.py b/bbdev/api/tests/test_api_bebop_verilator_sim.py index fb6a1215f..8920e115f 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_sim.py +++ b/bbdev/api/tests/test_api_bebop_verilator_sim.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --sim '--binary ctest_vecunit_matmul_ones_singlecore-baremetal --batch --config sims.verilator.BuckyballToyVerilatorConfig --itrace --mtrace --pmctrace --ctrace --banktrace''") +run_bbdev_case("bbdev bebop-verilator --sim '--binary vecunit_matmul_ones-singlecore-baremetal --batch --config sims.verilator.BuckyballToyVerilatorConfig --itrace --mtrace --pmctrace --ctrace --banktrace'") diff --git a/bbdev/api/tests/test_api_verilator_run.py b/bbdev/api/tests/test_api_verilator_run.py index f30775bdf..a8deb2651 100644 --- a/bbdev/api/tests/test_api_verilator_run.py +++ b/bbdev/api/tests/test_api_verilator_run.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --run '--jobs 16 --binary ctest_vecunit_matmul_ones_singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig --batch'") +run_bbdev_case("bbdev verilator --run '--jobs 16 --binary vecunit_matmul_ones-singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig --batch'") diff --git a/bbdev/api/tests/test_api_verilator_sim.py b/bbdev/api/tests/test_api_verilator_sim.py index 849493831..07978928c 100644 --- a/bbdev/api/tests/test_api_verilator_sim.py +++ b/bbdev/api/tests/test_api_verilator_sim.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --sim '--binary ctest_vecunit_matmul_ones_singlecore-baremetal --batch --config sims.verilator.BuckyballToyVerilatorConfig'") +run_bbdev_case("bbdev verilator --sim '--binary vecunit_matmul_ones-singlecore-baremetal --batch --config sims.verilator.BuckyballToyVerilatorConfig'") From b325fa57dc8efd8871bd826e7aa49cc8cfa358cc Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 30 Jul 2026 03:04:47 +0800 Subject: [PATCH 098/196] refactor: replace bemu_chip_inst with bemu_manifest in API steps for improved clarity and functionality --- bbdev/api/steps/bebop/bemu/02_sim_api.step.py | 4 ++-- bbdev/api/steps/bebop/bemu/02_sim_event.step.py | 14 +++++--------- bbdev/api/steps/bebop/bemu/03_batch_api.step.py | 4 ++-- .../api/steps/bebop/bemu/03_batch_event.step.py | 16 ++++++++-------- .../api/steps/bebop/bemu/scripts/bemu_common.py | 14 +++++--------- 5 files changed, 22 insertions(+), 30 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py index 1b26a061a..4ee24b404 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py @@ -11,7 +11,7 @@ sys.path.insert(0, scripts_path) from utils.path import get_buckyball_path -from bemu_common import bemu_chip_inst +from bemu_common import bemu_manifest config = { "name": "bebop-bemu-sim-api", @@ -26,7 +26,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} chip = body.get("chip", "") try: - bemu_chip_inst(chip, get_buckyball_path()) + bemu_manifest(chip, get_buckyball_path()) except ValueError as e: return ApiResponse( status=400, diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index cb6033ba9..38fb50a26 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -3,8 +3,8 @@ Runs bebop bemu (Spike-based) emulator: 1. Resolve binary (ELF) path - 2. Build bebop with bemu feature (cargo build) - 3. Run bebop bemu with the resolved ELF + 2. Build the selected chip's BEMU wrapper + 3. Run BEMU with the resolved ELF """ import os import shlex @@ -25,7 +25,7 @@ from utils.stream_run import stream_run_logger from utils.search_workload import search_workload, search_workload_all from utils.event_common import check_result, get_origin_trace_id -from bemu_common import bemu_env +from bemu_common import bemu_manifest PERFETTO_TARGETS = { "buddy-buckyball-lenet-run": "buddy-buckyball-lenet-perfetto", @@ -97,8 +97,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return try: - env = os.environ.copy() - env.update(bemu_env(chip, bbdir)) + bemu_cargo_manifest = bemu_manifest(chip, bbdir) except ValueError as e: ctx.logger.error(str(e)) await check_result( @@ -133,9 +132,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "cargo", "run", "--manifest-path", - f"{bebop_dir}/Cargo.toml", - "--features", - "bemu", + str(bemu_cargo_manifest), "--", "run", "bemu", @@ -155,7 +152,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cwd=bbdir, stdout_prefix="bebop bemu", stderr_prefix="bebop bemu", - env=env, ) if run_result.returncode != 0: await check_result( diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py index e85abd8c4..8699fcdf7 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -11,7 +11,7 @@ sys.path.insert(0, scripts_path) from utils.path import get_buckyball_path -from bemu_common import bemu_chip_inst +from bemu_common import bemu_manifest config = { "name": "bebop-bemu-batch-api", @@ -27,7 +27,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: chip = body.get("chip") try: - bemu_chip_inst(chip, get_buckyball_path()) + bemu_manifest(chip, get_buckyball_path()) except ValueError as e: return ApiResponse( status=400, diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index 61030a870..9a111bc29 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -2,7 +2,7 @@ bebop bemu batch event handler Runs bebop bemu nextest batch regression: - 1. Build bebop with bemu feature + 1. Build the selected chip's BEMU wrapper 2. Run cargo nextest with bemu-specific config """ import os @@ -25,7 +25,7 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id -from bemu_common import bemu_env +from bemu_common import bemu_manifest from regression import regression_workload_toml config = { @@ -55,7 +55,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return try: env = os.environ.copy() - env.update(bemu_env(chip, bbdir)) + bemu_cargo_manifest = bemu_manifest(chip, bbdir) except ValueError as e: ctx.logger.error(str(e)) await check_result( @@ -81,8 +81,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── Build bebop bemu ────────────────────────────────────────────────── build_cmd = ( - f"nix develop -c cargo build --manifest-path {shlex.quote(f'{bebop_dir}/Cargo.toml')} " - "--features bemu --tests" + f"nix develop -c cargo build --manifest-path {shlex.quote(str(bemu_cargo_manifest))} " + "--tests" ) ctx.logger.info("Building bebop bemu (tests)...") build_result = stream_run_logger( @@ -103,7 +103,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return # ── Run nextest ─────────────────────────────────────────────────────── - # Pass parameters via environment variables (nextest doesn't support custom CLI args after `--`) + # Pass test harness parameters through nextest's process environment. if input_data.get("clean-before", input_data.get("clean_before", False)): shutil.rmtree(f"{bebop_dir}/test-artifacts", ignore_errors=True) ctx.logger.info("Cleaned previous bebop test artifacts") @@ -113,8 +113,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "BEBOP_BB_TESTS_ROOT": elf_root, }) nextest_cmd = ( - f"nix develop -c cargo nextest run --manifest-path {shlex.quote(f'{bebop_dir}/Cargo.toml')} " - "--features bemu --test test_bemu " + f"nix develop -c cargo nextest run --manifest-path {shlex.quote(str(bemu_cargo_manifest))} " + "--test test_bemu " f"--config-file {shlex.quote(nextest_config)}" ) diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py index e652c10f7..79e32b03a 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py @@ -3,15 +3,11 @@ from utils.path import get_buckyball_path -def bemu_chip_inst(chip: str, bbdir: str | None = None) -> Path: +def bemu_manifest(chip: str, bbdir: str | None = None) -> Path: if not chip: raise ValueError("missing required parameter: chip") root = Path(bbdir or get_buckyball_path()) - inst = root / "examples" / "chips" / chip / "emu" / "src" / "lib.rs" - if not inst.is_file(): - raise ValueError(f"bemu chip instruction file does not exist: {inst}") - return inst - - -def bemu_env(chip: str, bbdir: str | None = None) -> dict[str, str]: - return {"BEBOP_BEMU_CHIP_INST": str(bemu_chip_inst(chip, bbdir))} + manifest = root / "examples" / "chips" / chip / "emu" / "Cargo.toml" + if not manifest.is_file(): + raise ValueError(f"bemu chip manifest does not exist: {manifest}") + return manifest From 3f75f3f3eced411ff5004efe820b61d416296f94 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 30 Jul 2026 09:08:27 +0800 Subject: [PATCH 099/196] feat: enhance bebop verilator API with improved error handling and support for additional parameters --- .../bebop/verilator/03_build_api.step.py | 18 ++++++++- .../bebop/verilator/03_build_event.step.py | 36 +++++------------- .../steps/bebop/verilator/05_run_api.step.py | 12 +++++- bbdev/api/utils/stream_run.py | 38 +++++++++++++------ bbdev/bbdev | 9 +++++ 5 files changed, 70 insertions(+), 43 deletions(-) diff --git a/bbdev/api/steps/bebop/verilator/03_build_api.step.py b/bbdev/api/steps/bebop/verilator/03_build_api.step.py index 4e5032cfa..7a27f4c5f 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_api.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_api.step.py @@ -1,5 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.path import get_buckyball_path, get_verilator_build_dir + config = { "name": "bebop-verilator-build-api", "description": "Build bebop verilator binary", @@ -10,8 +12,16 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() body = request.body or {} + unsupported = [name for name in ("vsrc_dir", "output_dir") if name in body] + if unsupported: + return ApiResponse( + status=400, + body={"error": f"Unsupported parameter(s): {', '.join(unsupported)}. Use --vsrc-dir/--output-dir."}, + ) + arch_config = body.get("config") if not isinstance(arch_config, str) or not arch_config or arch_config == "None": return ApiResponse( @@ -21,8 +31,12 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "config": arch_config, + "jobs": body.get("jobs", 16), + "vsrc_dir": get_verilator_build_dir( + bbdir, + arch_config, + body.get("vsrc-dir") or body.get("output-dir"), + ), } - if "vsrc_dir" in body: - data["vsrc_dir"] = body.get("vsrc_dir") await ctx.enqueue({"topic": "bebop.verilator.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index 521517606..1f3faff03 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -4,6 +4,7 @@ Builds bebop with verilator feature and VSRC_PATH """ import os +import shlex import sys from motia import FlowContext, queue @@ -29,28 +30,8 @@ } -def describe_path(path: str) -> dict: - info = { - "cwd": os.getcwd(), - "uid": os.getuid(), - "exists": os.path.exists(path), - "is_dir": os.path.isdir(path), - "parent": os.path.dirname(path), - } - try: - stat = os.stat(path) - info.update({ - "mode": oct(stat.st_mode), - "owner_uid": stat.st_uid, - "owner_gid": stat.st_gid, - }) - except OSError as e: - info.update({"stat_error": str(e), "errno": e.errno}) - try: - info["parent_entries"] = sorted(os.listdir(info["parent"]))[:20] - except OSError as e: - info["parent_error"] = str(e) - return info +def configure_fast_dev_profile(env: dict) -> None: + env.setdefault("CARGO_INCREMENTAL", "1") async def handler(input_data: dict, ctx: FlowContext) -> None: @@ -72,24 +53,24 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") if not os.path.isdir(vsrc_dir): - path_info = describe_path(vsrc_dir) - ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}; path_info={path_info}") + ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}") await check_result( ctx, 1, continue_run=False, extra_fields={ "error": "vsrc_not_found", "source": "bebop.verilator.build", "vsrc_dir": vsrc_dir, - "path_info": path_info, }, trace_id=origin_tid, ) return + jobs = input_data.get("jobs", 16) build_cmd = ( - f"cargo build --features verilator --tests " - f"--config=\"env.VSRC_PATH='{vsrc_dir}'\"" + f"cargo build --bin bebop --features verilator --jobs {shlex.quote(str(jobs))}" ) + env = {**os.environ, "VSRC_PATH": vsrc_dir} + configure_fast_dev_profile(env) ctx.logger.info("Building bebop verilator ...") build_result = stream_run_logger( cmd=build_cmd, @@ -97,6 +78,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cwd=bebop_dir, stdout_prefix="bebop verilator build", stderr_prefix="bebop verilator build", + env=env, ) bebop_bin = f"{bebop_dir}/target/debug/bebop" diff --git a/bbdev/api/steps/bebop/verilator/05_run_api.step.py b/bbdev/api/steps/bebop/verilator/05_run_api.step.py index d5080ef57..7915ef78c 100644 --- a/bbdev/api/steps/bebop/verilator/05_run_api.step.py +++ b/bbdev/api/steps/bebop/verilator/05_run_api.step.py @@ -12,6 +12,12 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + if "output_dir" in body: + return ApiResponse( + status=400, + body={"error": "Unsupported parameter: output_dir. Use --output-dir."}, + ) + config_name = body.get("config") if not isinstance(config_name, str) or not config_name or config_name == "None": return ApiResponse( @@ -42,10 +48,12 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "banktrace": body.get("banktrace", False), "log_dir": body.get("log_dir"), "fst_dir": body.get("fst_dir"), + "jobs": body.get("jobs", 16), "from_run_workflow": True, } - if "output_dir" in body: - data["output_dir"] = body.get("output_dir") + output_dir = body.get("output-dir") + if output_dir is not None: + data["output_dir"] = output_dir data["_explicit_output_dir"] = True await ctx.enqueue({"topic": "bebop.verilator.run.clean", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/utils/stream_run.py b/bbdev/api/utils/stream_run.py index f5ddd54ee..08edb4f36 100644 --- a/bbdev/api/utils/stream_run.py +++ b/bbdev/api/utils/stream_run.py @@ -55,17 +55,31 @@ def log_stderr(line): ) """ - def read_stream( - stream, output_list: List[str], callback: Optional[Callable], prefix: str - ): - """Thread function to read stream output""" + def read_stream(stream, output_list: List[str], callback: Optional[Callable]): + """Thread function to read stream output as soon as a line/progress record is available.""" + pending = [] + + def emit(): + if not pending: + return + line = "".join(pending).rstrip() + pending.clear() + if not line: + return + output_list.append(line) + if callback: + callback(line) + try: - for line in iter(stream.readline, ""): - if line: - line = line.rstrip() - output_list.append(line) - if callback: - callback(line) + while True: + char = stream.read(1) + if char == "": + emit() + break + if char in ("\n", "\r"): + emit() + else: + pending.append(char) finally: stream.close() @@ -91,11 +105,11 @@ def read_stream( # Create threads to read stdout and stderr stdout_thread = threading.Thread( target=read_stream, - args=(process.stdout, stdout_lines, on_stdout, stdout_prefix), + args=(process.stdout, stdout_lines, on_stdout), ) stderr_thread = threading.Thread( target=read_stream, - args=(process.stderr, stderr_lines, on_stderr, stderr_prefix), + args=(process.stderr, stderr_lines, on_stderr), ) stdout_thread.start() diff --git a/bbdev/bbdev b/bbdev/bbdev index 8ecbbc575..3be712d98 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -460,6 +460,7 @@ def _submit_and_poll(port: int, api_path: str, cmd_info: dict): poll_timeout = 24 * 60 * 60 # 24 hours max task_result = None legacy_processing_count = 0 + last_heartbeat = time.monotonic() for _ in range(poll_timeout // 2): state_data = read_state_file(state_file) if state_data: @@ -476,6 +477,10 @@ def _submit_and_poll(port: int, api_path: str, cmd_info: dict): return 1 else: legacy_processing_count = 0 + now = time.monotonic() + if now - last_heartbeat >= 600: + print(f"[bbdev] still running trace {trace_id}...", flush=True) + last_heartbeat = now time.sleep(2) else: print("Error: Task polling timed out after 24 hours") @@ -557,6 +562,8 @@ def run_script_mode(args, cmd_info): iii_env = { **os.environ, "III_URL": f"ws://{HOST}:{worker_port}", + "PYTHONUNBUFFERED": "1", + "PYTHONIOENCODING": "utf-8", "PYTHONDONTWRITEBYTECODE": "1", "BUCKYBALL_ROOT": os.path.dirname(os.path.dirname(os.path.abspath(__file__))), } @@ -566,6 +573,7 @@ def run_script_mode(args, cmd_info): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + bufsize=1, env=iii_env, start_new_session=True, ) @@ -594,6 +602,7 @@ def run_script_mode(args, cmd_info): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + bufsize=1, env=iii_env, start_new_session=True, ) From 731d26b81d405fd99d35499e026b803f846483eb Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 2 Aug 2026 01:36:32 +0800 Subject: [PATCH 100/196] refactor: remove unused bebop_dir variable from API steps to streamline code --- bbdev/api/steps/bebop/bemu/02_sim_event.step.py | 1 - bbdev/api/steps/bebop/bemu/03_batch_event.step.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 38fb50a26..102627050 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -84,7 +84,6 @@ def resolve_bemu_binary(bbdir: str, chip: str, binary_name: str) -> str | None: async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - bebop_dir = f"{bbdir}/bebop" arch_dir = f"{bbdir}/arch" chip = input_data.get("chip") diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index 9a111bc29..96ce5ed43 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -40,7 +40,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - bebop_dir = f"{bbdir}/bebop" nextest_config = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/nextest.toml" elf_root = f"{bbdir}/bb-tests/output" @@ -105,7 +104,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── Run nextest ─────────────────────────────────────────────────────── # Pass test harness parameters through nextest's process environment. if input_data.get("clean-before", input_data.get("clean_before", False)): - shutil.rmtree(f"{bebop_dir}/test-artifacts", ignore_errors=True) + shutil.rmtree(bemu_cargo_manifest.parent / "test-artifacts", ignore_errors=True) ctx.logger.info("Cleaned previous bebop test artifacts") env.update({ From 1b289a9d0e4a6fafc74b8e50c5044c4aefb4531c Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 2 Aug 2026 22:12:29 +0800 Subject: [PATCH 101/196] feat: update test cases to use new toy binary for bebop and verilator simulations --- bbdev/api/tests/test_api_bebop_bemu_sim.py | 4 ++-- bbdev/api/tests/test_api_bebop_verilator_run.py | 2 +- bbdev/api/tests/test_api_bebop_verilator_sim.py | 2 +- bbdev/api/tests/test_api_verilator_run.py | 2 +- bbdev/api/tests/test_api_verilator_sim.py | 2 +- bbdev/bbdev | 7 ++++++- 6 files changed, 12 insertions(+), 7 deletions(-) diff --git a/bbdev/api/tests/test_api_bebop_bemu_sim.py b/bbdev/api/tests/test_api_bebop_bemu_sim.py index 04460ed3e..7a8e4bc1a 100644 --- a/bbdev/api/tests/test_api_bebop_bemu_sim.py +++ b/bbdev/api/tests/test_api_bebop_bemu_sim.py @@ -1,4 +1,4 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary vecunit_matmul_ones-singlecore-baremetal'") -run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary ctest_vecunit_matmul_ones-linux --pk'") +run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_vecunit_matmul_ones-singlecore-baremetal'") +run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_vecunit_matmul_ones-linux --pk'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_run.py b/bbdev/api/tests/test_api_bebop_verilator_run.py index e71520f06..71ee1d37e 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_run.py +++ b/bbdev/api/tests/test_api_bebop_verilator_run.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --run '--jobs 16 --binary vecunit_matmul_ones-singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig --batch'") +run_bbdev_case("bbdev bebop-verilator --run '--jobs 16 --binary toy_vecunit_matmul_ones-singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig --batch'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_sim.py b/bbdev/api/tests/test_api_bebop_verilator_sim.py index 8920e115f..5e6768dd1 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_sim.py +++ b/bbdev/api/tests/test_api_bebop_verilator_sim.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --sim '--binary vecunit_matmul_ones-singlecore-baremetal --batch --config sims.verilator.BuckyballToyVerilatorConfig --itrace --mtrace --pmctrace --ctrace --banktrace'") +run_bbdev_case("bbdev bebop-verilator --sim '--binary toy_vecunit_matmul_ones-singlecore-baremetal --batch --config sims.verilator.BuckyballToyVerilatorConfig --itrace --mtrace --pmctrace --ctrace --banktrace'") diff --git a/bbdev/api/tests/test_api_verilator_run.py b/bbdev/api/tests/test_api_verilator_run.py index a8deb2651..c03dfdcc3 100644 --- a/bbdev/api/tests/test_api_verilator_run.py +++ b/bbdev/api/tests/test_api_verilator_run.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --run '--jobs 16 --binary vecunit_matmul_ones-singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig --batch'") +run_bbdev_case("bbdev verilator --run '--jobs 16 --binary toy_vecunit_matmul_ones-singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig --batch'") diff --git a/bbdev/api/tests/test_api_verilator_sim.py b/bbdev/api/tests/test_api_verilator_sim.py index 07978928c..52500c753 100644 --- a/bbdev/api/tests/test_api_verilator_sim.py +++ b/bbdev/api/tests/test_api_verilator_sim.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --sim '--binary vecunit_matmul_ones-singlecore-baremetal --batch --config sims.verilator.BuckyballToyVerilatorConfig'") +run_bbdev_case("bbdev verilator --sim '--binary toy_vecunit_matmul_ones-singlecore-baremetal --batch --config sims.verilator.BuckyballToyVerilatorConfig'") diff --git a/bbdev/bbdev b/bbdev/bbdev index 3be712d98..3d7ecb0e4 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -384,7 +384,12 @@ def run_server_mode(args, cmd_info): worker_url = f"ws://{HOST}:49134" proc = None try: - env = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"} + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + env = { + **os.environ, + "PYTHONDONTWRITEBYTECODE": "1", + "BUCKYBALL_ROOT": repo_root, + } if worker_url: env["III_URL"] = worker_url proc = subprocess.Popen( From 4349cd91a802a15c51bb5652292e2a57980d7d01 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 3 Aug 2026 01:18:35 +0800 Subject: [PATCH 102/196] feat: introduce MCP package with core functionality and tools for buckyball-dev --- bbdev/mcp/__init__.py | 1 + bbdev/mcp/__main__.py | 16 + bbdev/mcp/common.py | 446 ++++++++++++++++++++ bbdev/mcp/server.py | 14 + bbdev/mcp/tools/__init__.py | 80 ++++ bbdev/mcp/tools/bebop_p2e_batch.py | 39 ++ bbdev/mcp/tools/bebop_p2e_buildbitstream.py | 27 ++ bbdev/mcp/tools/bebop_p2e_clean.py | 16 + bbdev/mcp/tools/bebop_p2e_runworkload.py | 23 + bbdev/mcp/tools/bebop_p2e_verilog.py | 25 ++ bbdev/mcp/tools/bebop_verilator_batch.py | 35 ++ bbdev/mcp/tools/bebop_verilator_build.py | 18 + bbdev/mcp/tools/bebop_verilator_clean.py | 16 + bbdev/mcp/tools/bebop_verilator_run.py | 43 ++ bbdev/mcp/tools/bebop_verilator_sim.py | 43 ++ bbdev/mcp/tools/bebop_verilator_verilog.py | 16 + bbdev/mcp/tools/bemu_batch.py | 24 ++ bbdev/mcp/tools/bemu_sim.py | 26 ++ bbdev/mcp/tools/compiler_build.py | 18 + bbdev/mcp/tools/dc_verilog.py | 25 ++ bbdev/mcp/tools/firesim_buildbitstream.py | 14 + bbdev/mcp/tools/firesim_enumeratefpgas.py | 14 + bbdev/mcp/tools/firesim_infrasetup.py | 14 + bbdev/mcp/tools/firesim_runworkload.py | 14 + bbdev/mcp/tools/kernel_build.py | 25 ++ bbdev/mcp/tools/uvm_build.py | 24 ++ bbdev/mcp/tools/uvm_run.py | 26 ++ bbdev/mcp/tools/validate.py | 22 + bbdev/mcp/tools/verilator_build.py | 21 + bbdev/mcp/tools/verilator_clean.py | 14 + bbdev/mcp/tools/verilator_run.py | 42 ++ bbdev/mcp/tools/verilator_sim.py | 39 ++ bbdev/mcp/tools/verilator_verilog.py | 14 + bbdev/mcp/tools/workload_build.py | 20 + bbdev/mcp/tools/workload_clean.py | 14 + bbdev/mcp/tools/workload_tohex.py | 14 + bbdev/mcp/tools/yosys_run.py | 32 ++ bbdev/mcp/tools/yosys_synth.py | 32 ++ bbdev/mcp/tools/yosys_verilog.py | 32 ++ 39 files changed, 1378 insertions(+) create mode 100644 bbdev/mcp/__init__.py create mode 100644 bbdev/mcp/__main__.py create mode 100644 bbdev/mcp/common.py create mode 100644 bbdev/mcp/server.py create mode 100644 bbdev/mcp/tools/__init__.py create mode 100644 bbdev/mcp/tools/bebop_p2e_batch.py create mode 100644 bbdev/mcp/tools/bebop_p2e_buildbitstream.py create mode 100644 bbdev/mcp/tools/bebop_p2e_clean.py create mode 100644 bbdev/mcp/tools/bebop_p2e_runworkload.py create mode 100644 bbdev/mcp/tools/bebop_p2e_verilog.py create mode 100644 bbdev/mcp/tools/bebop_verilator_batch.py create mode 100644 bbdev/mcp/tools/bebop_verilator_build.py create mode 100644 bbdev/mcp/tools/bebop_verilator_clean.py create mode 100644 bbdev/mcp/tools/bebop_verilator_run.py create mode 100644 bbdev/mcp/tools/bebop_verilator_sim.py create mode 100644 bbdev/mcp/tools/bebop_verilator_verilog.py create mode 100644 bbdev/mcp/tools/bemu_batch.py create mode 100644 bbdev/mcp/tools/bemu_sim.py create mode 100644 bbdev/mcp/tools/compiler_build.py create mode 100644 bbdev/mcp/tools/dc_verilog.py create mode 100644 bbdev/mcp/tools/firesim_buildbitstream.py create mode 100644 bbdev/mcp/tools/firesim_enumeratefpgas.py create mode 100644 bbdev/mcp/tools/firesim_infrasetup.py create mode 100644 bbdev/mcp/tools/firesim_runworkload.py create mode 100644 bbdev/mcp/tools/kernel_build.py create mode 100644 bbdev/mcp/tools/uvm_build.py create mode 100644 bbdev/mcp/tools/uvm_run.py create mode 100644 bbdev/mcp/tools/validate.py create mode 100644 bbdev/mcp/tools/verilator_build.py create mode 100644 bbdev/mcp/tools/verilator_clean.py create mode 100644 bbdev/mcp/tools/verilator_run.py create mode 100644 bbdev/mcp/tools/verilator_sim.py create mode 100644 bbdev/mcp/tools/verilator_verilog.py create mode 100644 bbdev/mcp/tools/workload_build.py create mode 100644 bbdev/mcp/tools/workload_clean.py create mode 100644 bbdev/mcp/tools/workload_tohex.py create mode 100644 bbdev/mcp/tools/yosys_run.py create mode 100644 bbdev/mcp/tools/yosys_synth.py create mode 100644 bbdev/mcp/tools/yosys_verilog.py diff --git a/bbdev/mcp/__init__.py b/bbdev/mcp/__init__.py new file mode 100644 index 000000000..6edf60f97 --- /dev/null +++ b/bbdev/mcp/__init__.py @@ -0,0 +1 @@ +"""buckyball-dev MCP package.""" diff --git a/bbdev/mcp/__main__.py b/bbdev/mcp/__main__.py new file mode 100644 index 000000000..9b9d78ddd --- /dev/null +++ b/bbdev/mcp/__main__.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Run: python -u bbdev/mcp/__main__.py (stdio MCP).""" + +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from server import main + +if __name__ == "__main__": + main() diff --git a/bbdev/mcp/common.py b/bbdev/mcp/common.py new file mode 100644 index 000000000..457ec6300 --- /dev/null +++ b/bbdev/mcp/common.py @@ -0,0 +1,446 @@ +"""Shared helpers for buckyball-dev MCP (bbdev HTTP lifecycle + polling).""" + +from __future__ import annotations + +import atexit +import json +import shutil +import socket +import subprocess +import sys +import time +import tomllib +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any, Dict, Optional + +# bbdev/mcp -> repo root +REPO = Path(__file__).resolve().parents[2] +BBDEV = REPO / "bbdev" / "bbdev" +API = REPO / "bbdev" / "api" +MOTIA = API / ".venv" / "bin" / "motia" +LOG = REPO / "bbdev" / "server.log" +STATE_DIR = API / "data" / "state_store.db" +_proc: Optional[subprocess.Popen] = None +_port: Optional[int] = None +_log_fh = None + + +def _log(msg: str) -> None: + print(f"[buckyball-dev] {msg}", file=sys.stderr, flush=True) + + +def _fmt(payload: Any) -> str: + return json.dumps(payload, ensure_ascii=False, indent=2) + + +def _err(msg: str) -> str: + return _fmt({"success": False, "failure": True, "error": msg}) + + +def _need(name: str, value: Optional[str]) -> Optional[str]: + if value is None or not str(value).strip(): + return f"missing required parameter: {name}" + return None + + +def _opt(params: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: + for k, v in kwargs.items(): + if v is None: + continue + if isinstance(v, str) and not v.strip(): + continue + params[k] = v + return params + + +def _free_port(lo: int = 5100, hi: int = 5500) -> int: + for p in range(lo, hi + 1): + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", p)) + return p + except OSError: + continue + raise RuntimeError(f"No available port in {lo}-{hi}") + + +def _http( + method: str, url: str, payload: Optional[Dict[str, Any]] = None, timeout: int = 30 +): + data = None + headers = {} + if payload is not None: + data = json.dumps(payload).encode("utf-8") + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + body = resp.read().decode("utf-8") + return resp.status, json.loads(body) if body else {} + except urllib.error.HTTPError as e: + raw = e.read().decode("utf-8", errors="replace") + try: + return e.code, json.loads(raw) if raw else {} + except json.JSONDecodeError: + return e.code, {"response_body": raw} + + +def _ready(port: int) -> bool: + try: + code, _ = _http("GET", f"http://127.0.0.1:{port}/compiler/build", timeout=2) + return code != 404 + except Exception: + return False + + +def _stop() -> None: + global _proc, _port, _log_fh + if _port is not None and BBDEV.is_file(): + subprocess.run( + [str(BBDEV), "stop", "--server", "--port", str(_port)], + cwd=str(BBDEV.parent), + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if _proc is not None: + _proc.terminate() + try: + _proc.wait(timeout=5) + except subprocess.TimeoutExpired: + _proc.kill() + _proc = None + _port = None + if _log_fh is not None: + _log_fh.close() + _log_fh = None + + +def _ensure() -> int: + """Start `bbdev start --server` if needed. Returns HTTP port.""" + global _proc, _port, _log_fh + + if ( + _port is not None + and _proc is not None + and _proc.poll() is None + and _ready(_port) + ): + return _port + if _proc is not None: + _log(f"bbdev on port {_port} died; restarting") + _stop() + + if not BBDEV.is_file(): + raise RuntimeError(f"missing bbdev CLI: {BBDEV}") + if shutil.which("iii") is None: + raise RuntimeError( + "iii not found; run MCP via scripts/claude/run_mcp_server.sh" + ) + if not MOTIA.is_file(): + raise RuntimeError( + f"missing {MOTIA}; install with: " + f"cd {API} && uv venv .venv --python python3 --seed && " + "uv pip install --python .venv/bin/python -r pyproject.toml" + ) + + port = _free_port() + LOG.parent.mkdir(parents=True, exist_ok=True) + _log_fh = open(LOG, "a", encoding="utf-8") + _proc = subprocess.Popen( + [str(BBDEV), "start", "--server", "--port", str(port)], + cwd=str(BBDEV.parent), + stdout=_log_fh, + stderr=_log_fh, + start_new_session=True, + ) + _port = port + _log(f"starting bbdev on port {port} (log: {LOG})") + + for _ in range(120): + if _proc.poll() is not None: + tail = "" + try: + tail = LOG.read_text(encoding="utf-8", errors="replace")[-2000:] + except OSError: + pass + _stop() + raise RuntimeError( + f"bbdev exited early; see {LOG}\n--- log tail ---\n{tail}" + ) + if _ready(port): + _log(f"bbdev ready on port {port}") + return port + time.sleep(1) + + _stop() + raise RuntimeError(f"bbdev failed to start on port {port} within 120s; see {LOG}") + + +def _read_state(trace_id: str) -> Optional[Dict[str, Any]]: + """Same as bbdev CLI: poll iii file state store (HTTP /result path_params are broken).""" + path = STATE_DIR / f"{trace_id}.bin" + if not path.is_file(): + return None + try: + raw = path.read_text(encoding="utf-8", errors="ignore") + data, _ = json.JSONDecoder().raw_decode(raw) + except (OSError, json.JSONDecodeError, ValueError): + return None + if not isinstance(data, dict): + raise RuntimeError(f"state file root must be object: {path}") + return data + + +def _call(endpoint: str, params: Dict[str, Any], timeout: int = 600) -> Dict[str, Any]: + port = _ensure() + base = f"http://127.0.0.1:{port}" + _log(f"POST {endpoint} params={params}") + + status, submit = _http("POST", f"{base}{endpoint}", params, timeout=30) + if status >= 400: + return { + "success": False, + "failure": True, + "status_code": status, + "error": submit, + "server_log": str(LOG), + "port": port, + } + + trace_id = submit.get("trace_id") + if not trace_id: + return { + "success": False, + "failure": True, + "error": "no trace_id in submit response", + "response": submit, + "server_log": str(LOG), + "port": port, + } + + deadline = time.monotonic() + timeout + beat = time.monotonic() + stuck = 0 + while time.monotonic() < deadline: + state = _read_state(trace_id) + if state is None: + time.sleep(2) + continue + if "success" in state: + out = state["success"].get("body", state["success"]) + if not isinstance(out, dict): + raise RuntimeError(f"success body must be object: {out!r}") + out.setdefault("success", True) + out.setdefault("trace_id", trace_id) + out.setdefault("port", port) + return out + if "failure" in state: + body = state["failure"].get("body", state["failure"]) + return { + "success": False, + "failure": True, + "trace_id": trace_id, + "port": port, + "body": body, + "server_log": str(LOG), + } + if state.get("processing") is True: + stuck += 1 + if stuck >= 30: + return { + "success": False, + "failure": True, + "error": "stuck in legacy processing state", + "trace_id": trace_id, + "port": port, + "server_log": str(LOG), + } + else: + stuck = 0 + + now = time.monotonic() + if now - beat >= 30: + _log(f"still running {endpoint} trace={trace_id}") + beat = now + time.sleep(2) + + return { + "success": False, + "failure": True, + "error": f"timed out after {timeout}s waiting for {endpoint}", + "trace_id": trace_id, + "server_log": str(LOG), + "port": port, + } + + +def _load_toml(path: Path) -> Dict[str, Any]: + with path.open("rb") as f: + data = tomllib.load(f) + if not isinstance(data, dict): + raise ValueError(f"TOML root must be a table: {path}") + return data + + +def _balldomain_path(chip: str, balldomain: Optional[str]) -> Path: + chip_root = REPO / "examples" / "chips" / chip + if not chip_root.is_dir(): + raise FileNotFoundError(f"chip does not exist: {chip_root}") + + domains = chip_root / "configs" / "tiles" / "cores" / "balldomains" + cores = chip_root / "configs" / "tiles" / "cores" + + if balldomain is None: + default = cores / "default.toml" + if not default.is_file(): + raise FileNotFoundError(f"missing {default}; pass balldomain= explicitly") + rel = _load_toml(default).get("balldomain") + if not isinstance(rel, str) or not rel: + raise ValueError(f"cores/default.toml missing balldomain=: {default}") + path = (cores / rel).resolve() + else: + raw = Path(balldomain) + if raw.is_absolute(): + path = raw + elif balldomain.endswith(".toml"): + candidates = [ + domains / raw.name, + cores / balldomain, + chip_root / balldomain, + ] + path = next( + (p.resolve() for p in candidates if p.is_file()), + candidates[0].resolve(), + ) + else: + path = (domains / f"{balldomain}.toml").resolve() + + if not path.is_file(): + raise FileNotFoundError(f"balldomain toml does not exist: {path}") + return path + + +def _validate(path: Path) -> Dict[str, Any]: + cfg = _load_toml(path) + mappings = cfg.get("ballIdMappings", []) + isa = cfg.get("ballISA", []) + if not isinstance(mappings, list) or not isinstance(isa, list): + raise ValueError(f"{path}: ballIdMappings/ballISA must be arrays") + + ids = [m.get("ballId") for m in mappings] + names = [m.get("ballName") for m in mappings] + funct7s = [e.get("funct7") for e in isa] + mnemonics = [e.get("mnemonic") for e in isa] + bids = [e.get("bid") for e in isa] + id_set = set(ids) + + missing_config = [] + bad_bw = [] + for m in mappings: + name = m.get("ballName") + if not m.get("ballClass"): + bad_bw.append({"ballName": name, "error": "missing ballClass"}) + in_bw, out_bw = m.get("inBW"), m.get("outBW") + if ( + not isinstance(in_bw, int) + or in_bw <= 0 + or not isinstance(out_bw, int) + or out_bw <= 0 + ): + bad_bw.append({"ballName": name, "inBW": in_bw, "outBW": out_bw}) + cfg_rel = m.get("config") + if cfg_rel is None: + continue + if not isinstance(cfg_rel, str) or not cfg_rel: + missing_config.append( + {"ballName": name, "config": cfg_rel, "error": "empty"} + ) + continue + cfg_path = (path.parent / cfg_rel).resolve() + if not cfg_path.is_file(): + missing_config.append( + {"ballName": name, "config": cfg_rel, "resolved": str(cfg_path)} + ) + + orphan = sorted(id_set - set(bids)) + unknown = sorted(set(bids) - id_set) + + def dups(xs): + return sorted(x for x in set(xs) if xs.count(x) > 1) + + checks = { + "ballNum_matches_count": { + "pass": cfg.get("ballNum") == len(mappings), + "expected": len(mappings), + "actual": cfg.get("ballNum"), + }, + "ballId_strict_increment": {"pass": ids == list(range(len(ids))), "ids": ids}, + "ballId_no_duplicates": { + "pass": len(ids) == len(set(ids)), + "duplicates": dups(ids), + }, + "ballName_no_duplicates": { + "pass": len(names) == len(set(names)), + "duplicates": dups(names), + }, + "funct7_no_duplicates": { + "pass": len(funct7s) == len(set(funct7s)), + "duplicates": dups(funct7s), + }, + "mnemonic_no_duplicates": { + "pass": len(mnemonics) == len(set(mnemonics)), + "duplicates": dups(mnemonics), + }, + "isa_bid_in_mappings": {"pass": not unknown, "unknown_bids": unknown}, + "every_ball_has_isa": {"pass": not orphan, "ballIds_without_isa": orphan}, + "ball_config_files_exist": { + "pass": not missing_config, + "missing": missing_config, + }, + "bandwidth_positive": {"pass": not bad_bw, "invalid": bad_bw}, + } + + id_to_isa: Dict[Any, list] = {} + for e in isa: + id_to_isa.setdefault(e.get("bid"), []).append(e) + balls = [ + { + "ballId": m.get("ballId"), + "ballName": m.get("ballName"), + "ballClass": m.get("ballClass"), + "inBW": m.get("inBW"), + "outBW": m.get("outBW"), + "config": m.get("config"), + "isa": [ + {"mnemonic": e.get("mnemonic"), "funct7": e.get("funct7")} + for e in id_to_isa.get(m.get("ballId"), []) + ], + } + for m in mappings + ] + + return { + "passed": all(c["pass"] for c in checks.values()), + "chip_balldomain": str(path.relative_to(REPO)), + "checks": checks, + "balls": balls, + } + + + +atexit.register(_stop) + + +# Public aliases for tool modules +log = _log +fmt = _fmt +err = _err +need = _need +opt = _opt +call = _call +balldomain_path = _balldomain_path +validate_toml = _validate diff --git a/bbdev/mcp/server.py b/bbdev/mcp/server.py new file mode 100644 index 000000000..c0918605c --- /dev/null +++ b/bbdev/mcp/server.py @@ -0,0 +1,14 @@ +"""FastMCP entry for buckyball-dev.""" + +from __future__ import annotations + +from mcp.server.fastmcp import FastMCP + +from tools import register_all + +mcp = FastMCP("buckyball-dev") +register_all(mcp) + + +def main() -> None: + mcp.run(transport="stdio") diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py new file mode 100644 index 000000000..0bb826a32 --- /dev/null +++ b/bbdev/mcp/tools/__init__.py @@ -0,0 +1,80 @@ +"""Register all MCP tools.""" + +from __future__ import annotations + +from . import validate +from . import compiler_build +from . import workload_clean +from . import workload_build +from . import bemu_sim +from . import bemu_batch +from . import bebop_verilator_clean +from . import bebop_verilator_verilog +from . import bebop_verilator_build +from . import bebop_verilator_sim +from . import bebop_verilator_run +from . import bebop_verilator_batch +from . import verilator_clean +from . import verilator_verilog +from . import verilator_build +from . import verilator_sim +from . import verilator_run +from . import uvm_build +from . import uvm_run +from . import workload_tohex +from . import bebop_p2e_clean +from . import bebop_p2e_verilog +from . import bebop_p2e_buildbitstream +from . import bebop_p2e_runworkload +from . import bebop_p2e_batch +from . import dc_verilog +from . import firesim_enumeratefpgas +from . import firesim_buildbitstream +from . import firesim_infrasetup +from . import firesim_runworkload +from . import kernel_build +from . import yosys_run +from . import yosys_verilog +from . import yosys_synth + +MODULES = [ + validate, + compiler_build, + workload_clean, + workload_build, + bemu_sim, + bemu_batch, + bebop_verilator_clean, + bebop_verilator_verilog, + bebop_verilator_build, + bebop_verilator_sim, + bebop_verilator_run, + bebop_verilator_batch, + verilator_clean, + verilator_verilog, + verilator_build, + verilator_sim, + verilator_run, + uvm_build, + uvm_run, + workload_tohex, + bebop_p2e_clean, + bebop_p2e_verilog, + bebop_p2e_buildbitstream, + bebop_p2e_runworkload, + bebop_p2e_batch, + dc_verilog, + firesim_enumeratefpgas, + firesim_buildbitstream, + firesim_infrasetup, + firesim_runworkload, + kernel_build, + yosys_run, + yosys_verilog, + yosys_synth, +] + + +def register_all(mcp) -> None: + for mod in MODULES: + mod.register(mcp) diff --git a/bbdev/mcp/tools/bebop_p2e_batch.py b/bbdev/mcp/tools/bebop_p2e_batch.py new file mode 100644 index 000000000..0bb644ed5 --- /dev/null +++ b/bbdev/mcp/tools/bebop_p2e_batch.py @@ -0,0 +1,39 @@ +"""MCP tool: bbdev_bebop_p2e_batch.""" + +from __future__ import annotations + + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_p2e_batch( + chip: str, + bitstream: str, + build_dir: str, + test: str, + ) -> str: + """Batch bebop-p2e regression. test: elf-tests|pk-tests. POST /bebop/p2e/batch.""" + for n, v in ( + ("chip", chip), + ("bitstream", bitstream), + ("build_dir", build_dir), + ): + if e := need(n, v): + return err(e) + if test not in ("elf-tests", "pk-tests"): + return err("test must be elf-tests or pk-tests") + return fmt( + call( + "/bebop/p2e/batch", + { + "chip": chip, + "bitstream": bitstream, + "build-dir": build_dir, + "test": test, + }, + timeout=14400, + ) + ) + diff --git a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py new file mode 100644 index 000000000..0cda45734 --- /dev/null +++ b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py @@ -0,0 +1,27 @@ +"""MCP tool: bbdev_bebop_p2e_buildbitstream.""" + +from __future__ import annotations + +from typing import Optional + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_p2e_buildbitstream( + config: str, + vsrc_dir: Optional[str] = None, + output_dir: Optional[str] = None, + ) -> str: + """Build bebop-p2e bitstream. POST /bebop/p2e/buildbitstream.""" + if e := need("config", config): + return err(e) + return fmt( + call( + "/bebop/p2e/buildbitstream", + opt({"config": config}, vsrc_dir=vsrc_dir, output_dir=output_dir), + timeout=14400, + ) + ) + diff --git a/bbdev/mcp/tools/bebop_p2e_clean.py b/bbdev/mcp/tools/bebop_p2e_clean.py new file mode 100644 index 000000000..7a7baf970 --- /dev/null +++ b/bbdev/mcp/tools/bebop_p2e_clean.py @@ -0,0 +1,16 @@ +"""MCP tool: bbdev_bebop_p2e_clean.""" + +from __future__ import annotations + + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_p2e_clean(config: str) -> str: + """Clean bebop-p2e build. POST /bebop/p2e/clean.""" + if e := need("config", config): + return err(e) + return fmt(call("/bebop/p2e/clean", {"config": config}, timeout=300)) + diff --git a/bbdev/mcp/tools/bebop_p2e_runworkload.py b/bbdev/mcp/tools/bebop_p2e_runworkload.py new file mode 100644 index 000000000..b04dbd246 --- /dev/null +++ b/bbdev/mcp/tools/bebop_p2e_runworkload.py @@ -0,0 +1,23 @@ +"""MCP tool: bbdev_bebop_p2e_runworkload.""" + +from __future__ import annotations + + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_p2e_runworkload(image: str, bitstream: str) -> str: + """Run one image on bebop-p2e FPGA. POST /bebop/p2e/runworkload.""" + for n, v in (("image", image), ("bitstream", bitstream)): + if e := need(n, v): + return err(e) + return fmt( + call( + "/bebop/p2e/runworkload", + {"image": image, "bitstream": bitstream}, + timeout=7200, + ) + ) + diff --git a/bbdev/mcp/tools/bebop_p2e_verilog.py b/bbdev/mcp/tools/bebop_p2e_verilog.py new file mode 100644 index 000000000..c6d55c979 --- /dev/null +++ b/bbdev/mcp/tools/bebop_p2e_verilog.py @@ -0,0 +1,25 @@ +"""MCP tool: bbdev_bebop_p2e_verilog.""" + +from __future__ import annotations + +from typing import Optional + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_p2e_verilog( + config: str, output_dir: Optional[str] = None + ) -> str: + """Generate Verilog for bebop-p2e. POST /bebop/p2e/verilog.""" + if e := need("config", config): + return err(e) + return fmt( + call( + "/bebop/p2e/verilog", + opt({"config": config}, output_dir=output_dir), + timeout=1800, + ) + ) + diff --git a/bbdev/mcp/tools/bebop_verilator_batch.py b/bbdev/mcp/tools/bebop_verilator_batch.py new file mode 100644 index 000000000..f8c249b61 --- /dev/null +++ b/bbdev/mcp/tools/bebop_verilator_batch.py @@ -0,0 +1,35 @@ +"""MCP tool: bbdev_bebop_verilator_batch.""" + +from __future__ import annotations + + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_verilator_batch( + chip: str, + config: str, + test: str, + clean_before: bool = False, + ) -> str: + """Batch bebop-verilator regression. POST /bebop/verilator/batch.""" + for n, v in (("chip", chip), ("config", config)): + if e := need(n, v): + return err(e) + if test not in ("elf-tests", "pk-tests"): + return err("test must be elf-tests or pk-tests") + return fmt( + call( + "/bebop/verilator/batch", + { + "chip": chip, + "config": config, + "test": test, + "clean-before": clean_before, + }, + timeout=14400, + ) + ) + diff --git a/bbdev/mcp/tools/bebop_verilator_build.py b/bbdev/mcp/tools/bebop_verilator_build.py new file mode 100644 index 000000000..0cbfa30cd --- /dev/null +++ b/bbdev/mcp/tools/bebop_verilator_build.py @@ -0,0 +1,18 @@ +"""MCP tool: bbdev_bebop_verilator_build.""" + +from __future__ import annotations + + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_verilator_build(config: str, jobs: int = 16) -> str: + """Build bebop verilator binary. POST /bebop/verilator/build.""" + if e := need("config", config): + return err(e) + return fmt( + call("/bebop/verilator/build", {"config": config, "jobs": jobs}, timeout=3600) + ) + diff --git a/bbdev/mcp/tools/bebop_verilator_clean.py b/bbdev/mcp/tools/bebop_verilator_clean.py new file mode 100644 index 000000000..69bc55641 --- /dev/null +++ b/bbdev/mcp/tools/bebop_verilator_clean.py @@ -0,0 +1,16 @@ +"""MCP tool: bbdev_bebop_verilator_clean.""" + +from __future__ import annotations + + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_verilator_clean(config: str) -> str: + """Clean bebop-verilator build. POST /bebop/verilator/clean.""" + if e := need("config", config): + return err(e) + return fmt(call("/bebop/verilator/clean", {"config": config}, timeout=300)) + diff --git a/bbdev/mcp/tools/bebop_verilator_run.py b/bbdev/mcp/tools/bebop_verilator_run.py new file mode 100644 index 000000000..f5348f2f6 --- /dev/null +++ b/bbdev/mcp/tools/bebop_verilator_run.py @@ -0,0 +1,43 @@ +"""MCP tool: bbdev_bebop_verilator_run.""" + +from __future__ import annotations + + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_verilator_run( + binary: str, + config: str, + jobs: int = 16, + itrace: bool = False, + mtrace: bool = False, + pmctrace: bool = False, + ctrace: bool = False, + banktrace: bool = False, + no_wave: bool = False, + ) -> str: + """Full bebop-verilator flow. POST /bebop/verilator/run.""" + for n, v in (("binary", binary), ("config", config)): + if e := need(n, v): + return err(e) + return fmt( + call( + "/bebop/verilator/run", + { + "binary": binary, + "config": config, + "jobs": jobs, + "itrace": itrace, + "mtrace": mtrace, + "pmctrace": pmctrace, + "ctrace": ctrace, + "banktrace": banktrace, + "no-wave": no_wave, + }, + timeout=14400, + ) + ) + diff --git a/bbdev/mcp/tools/bebop_verilator_sim.py b/bbdev/mcp/tools/bebop_verilator_sim.py new file mode 100644 index 000000000..f716767d3 --- /dev/null +++ b/bbdev/mcp/tools/bebop_verilator_sim.py @@ -0,0 +1,43 @@ +"""MCP tool: bbdev_bebop_verilator_sim.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_verilator_sim( + binary: str, + config: str, + itrace: bool = False, + mtrace: bool = False, + pmctrace: bool = False, + ctrace: bool = False, + banktrace: bool = False, + no_wave: bool = False, + log_dir: Optional[str] = None, + fst_dir: Optional[str] = None, + ) -> str: + """Run one workload on bebop-verilator. POST /bebop/verilator/sim.""" + for n, v in (("binary", binary), ("config", config)): + if e := need(n, v): + return err(e) + params: Dict[str, Any] = { + "binary": binary, + "config": config, + "itrace": itrace, + "mtrace": mtrace, + "pmctrace": pmctrace, + "ctrace": ctrace, + "banktrace": banktrace, + "no-wave": no_wave, + } + if log_dir: + params["log_dir"] = log_dir + if fst_dir: + params["fst_dir"] = fst_dir + return fmt(call("/bebop/verilator/sim", params, timeout=7200)) + diff --git a/bbdev/mcp/tools/bebop_verilator_verilog.py b/bbdev/mcp/tools/bebop_verilator_verilog.py new file mode 100644 index 000000000..db598acc7 --- /dev/null +++ b/bbdev/mcp/tools/bebop_verilator_verilog.py @@ -0,0 +1,16 @@ +"""MCP tool: bbdev_bebop_verilator_verilog.""" + +from __future__ import annotations + + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_verilator_verilog(config: str) -> str: + """Generate Verilog for bebop-verilator. POST /bebop/verilator/verilog.""" + if e := need("config", config): + return err(e) + return fmt(call("/bebop/verilator/verilog", {"config": config}, timeout=1800)) + diff --git a/bbdev/mcp/tools/bemu_batch.py b/bbdev/mcp/tools/bemu_batch.py new file mode 100644 index 000000000..668751979 --- /dev/null +++ b/bbdev/mcp/tools/bemu_batch.py @@ -0,0 +1,24 @@ +"""MCP tool: bbdev_bemu_batch.""" + +from __future__ import annotations + + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bemu_batch(chip: str, test: str, clean_before: bool = False) -> str: + """Batch bemu regression. test: elf-tests|pk-tests. POST /bebop/bemu/batch.""" + if e := need("chip", chip): + return err(e) + if test not in ("elf-tests", "pk-tests"): + return err("test must be elf-tests or pk-tests") + return fmt( + call( + "/bebop/bemu/batch", + {"chip": chip, "test": test, "clean-before": clean_before}, + timeout=7200, + ) + ) + diff --git a/bbdev/mcp/tools/bemu_sim.py b/bbdev/mcp/tools/bemu_sim.py new file mode 100644 index 000000000..f09f76f96 --- /dev/null +++ b/bbdev/mcp/tools/bemu_sim.py @@ -0,0 +1,26 @@ +"""MCP tool: bbdev_bemu_sim.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bemu_sim( + chip: str, + binary: str, + pk: bool = False, + log_dir: Optional[str] = None, + ) -> str: + """Run one workload on bebop-bemu. POST /bebop/bemu/sim.""" + for n, v in (("chip", chip), ("binary", binary)): + if e := need(n, v): + return err(e) + params: Dict[str, Any] = {"chip": chip, "binary": binary, "pk": pk} + if log_dir: + params["log_dir"] = log_dir + return fmt(call("/bebop/bemu/sim", params, timeout=1800)) + diff --git a/bbdev/mcp/tools/compiler_build.py b/bbdev/mcp/tools/compiler_build.py new file mode 100644 index 000000000..ea1ac814c --- /dev/null +++ b/bbdev/mcp/tools/compiler_build.py @@ -0,0 +1,18 @@ +"""MCP tool: bbdev_compiler_build.""" + +from __future__ import annotations + + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_compiler_build(chip: str, stable: bool = False) -> str: + """Build buddy-mlir compiler for a chip. POST /compiler/build.""" + if e := need("chip", chip): + return err(e) + return fmt( + call("/compiler/build", {"chip": chip, "stable": stable}, timeout=1800) + ) + diff --git a/bbdev/mcp/tools/dc_verilog.py b/bbdev/mcp/tools/dc_verilog.py new file mode 100644 index 000000000..31a94347f --- /dev/null +++ b/bbdev/mcp/tools/dc_verilog.py @@ -0,0 +1,25 @@ +"""MCP tool: bbdev_dc_verilog.""" + +from __future__ import annotations + +from typing import Optional + +from common import call, err, fmt, opt + + +def register(mcp): + @mcp.tool() + def bbdev_dc_verilog( + config: Optional[str] = None, rtl_dir: Optional[str] = None + ) -> str: + """Generate Verilog for DC flow. POST /dc/verilog (rtl_dir maps to dir).""" + if not config and not rtl_dir: + return err("config or rtl_dir is required") + return fmt( + call( + "/dc/verilog", + opt({}, config=config, dir=rtl_dir), + timeout=1800, + ) + ) + diff --git a/bbdev/mcp/tools/firesim_buildbitstream.py b/bbdev/mcp/tools/firesim_buildbitstream.py new file mode 100644 index 000000000..17d215c38 --- /dev/null +++ b/bbdev/mcp/tools/firesim_buildbitstream.py @@ -0,0 +1,14 @@ +"""MCP tool: bbdev_firesim_buildbitstream.""" + +from __future__ import annotations + + +from common import call, fmt, opt + + +def register(mcp): + @mcp.tool() + def bbdev_firesim_buildbitstream() -> str: + """Build FireSim bitstream. POST /firesim/buildbitstream.""" + return fmt(call("/firesim/buildbitstream", {}, timeout=14400)) + diff --git a/bbdev/mcp/tools/firesim_enumeratefpgas.py b/bbdev/mcp/tools/firesim_enumeratefpgas.py new file mode 100644 index 000000000..ba9c68e58 --- /dev/null +++ b/bbdev/mcp/tools/firesim_enumeratefpgas.py @@ -0,0 +1,14 @@ +"""MCP tool: bbdev_firesim_enumeratefpgas.""" + +from __future__ import annotations + + +from common import call, fmt, opt + + +def register(mcp): + @mcp.tool() + def bbdev_firesim_enumeratefpgas() -> str: + """Enumerate FireSim FPGAs. POST /firesim/enumeratefpgas.""" + return fmt(call("/firesim/enumeratefpgas", {}, timeout=1800)) + diff --git a/bbdev/mcp/tools/firesim_infrasetup.py b/bbdev/mcp/tools/firesim_infrasetup.py new file mode 100644 index 000000000..91838b914 --- /dev/null +++ b/bbdev/mcp/tools/firesim_infrasetup.py @@ -0,0 +1,14 @@ +"""MCP tool: bbdev_firesim_infrasetup.""" + +from __future__ import annotations + + +from common import call, fmt, opt + + +def register(mcp): + @mcp.tool() + def bbdev_firesim_infrasetup(jobs: int = 16) -> str: + """FireSim infrasetup. POST /firesim/infrasetup.""" + return fmt(call("/firesim/infrasetup", {"jobs": jobs}, timeout=7200)) + diff --git a/bbdev/mcp/tools/firesim_runworkload.py b/bbdev/mcp/tools/firesim_runworkload.py new file mode 100644 index 000000000..88836f08c --- /dev/null +++ b/bbdev/mcp/tools/firesim_runworkload.py @@ -0,0 +1,14 @@ +"""MCP tool: bbdev_firesim_runworkload.""" + +from __future__ import annotations + + +from common import call, fmt, opt + + +def register(mcp): + @mcp.tool() + def bbdev_firesim_runworkload(jobs: int = 16) -> str: + """Run FireSim workload. POST /firesim/runworkload.""" + return fmt(call("/firesim/runworkload", {"jobs": jobs}, timeout=14400)) + diff --git a/bbdev/mcp/tools/kernel_build.py b/bbdev/mcp/tools/kernel_build.py new file mode 100644 index 000000000..98260249c --- /dev/null +++ b/bbdev/mcp/tools/kernel_build.py @@ -0,0 +1,25 @@ +"""MCP tool: bbdev_kernel_build.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from common import call, fmt, opt + + +def register(mcp): + @mcp.tool() + def bbdev_kernel_build( + model: Optional[str] = None, + visible_hart_count: Optional[int] = None, + total_hart_count: Optional[int] = None, + ) -> str: + """Build RISC-V kernel + rootfs. POST /kernel/build.""" + params: Dict[str, Any] = {} + opt(params, model=model) + if visible_hart_count is not None: + params["visible-hart-count"] = visible_hart_count + if total_hart_count is not None: + params["total-hart-count"] = total_hart_count + return fmt(call("/kernel/build", params, timeout=7200)) + diff --git a/bbdev/mcp/tools/uvm_build.py b/bbdev/mcp/tools/uvm_build.py new file mode 100644 index 000000000..8df86bd7a --- /dev/null +++ b/bbdev/mcp/tools/uvm_build.py @@ -0,0 +1,24 @@ +"""MCP tool: bbdev_uvm_build.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_uvm_build( + config: str, ball: Optional[str] = None, filelist: Optional[str] = None + ) -> str: + """Build a Ball UVM simulation. POST /uvm/build.""" + if e := need("config", config): + return err(e) + params: Dict[str, Any] = {"config": config} + if ball: + params["ball"] = ball + if filelist: + params["filelist"] = filelist + return fmt(call("/uvm/build", params, timeout=3600)) + diff --git a/bbdev/mcp/tools/uvm_run.py b/bbdev/mcp/tools/uvm_run.py new file mode 100644 index 000000000..d4902fdb3 --- /dev/null +++ b/bbdev/mcp/tools/uvm_run.py @@ -0,0 +1,26 @@ +"""MCP tool: bbdev_uvm_run.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_uvm_run( + ball: str, + filelist: Optional[str] = None, + test: Optional[str] = None, + ) -> str: + """Build and run a Ball UVM simulation. POST /uvm/run.""" + if e := need("ball", ball): + return err(e) + params: Dict[str, Any] = {"ball": ball} + if filelist: + params["filelist"] = filelist + if test: + params["test"] = test + return fmt(call("/uvm/run", params, timeout=7200)) + diff --git a/bbdev/mcp/tools/validate.py b/bbdev/mcp/tools/validate.py new file mode 100644 index 000000000..d12cf2a19 --- /dev/null +++ b/bbdev/mcp/tools/validate.py @@ -0,0 +1,22 @@ +"""MCP tool: validate.""" + +from __future__ import annotations + +from typing import Optional + +from common import balldomain_path, err, fmt, need, validate_toml + + +def register(mcp): + @mcp.tool() + def validate(chip: str = "toy", balldomain: Optional[str] = None) -> str: + """Validate chip balldomain TOML registration.""" + if e := need("chip", chip): + return err(e) + try: + return fmt(validate_toml(balldomain_path(chip, balldomain))) + except Exception as ex: + return fmt( + {"passed": False, "success": False, "failure": True, "error": str(ex)} + ) + diff --git a/bbdev/mcp/tools/verilator_build.py b/bbdev/mcp/tools/verilator_build.py new file mode 100644 index 000000000..55b39058d --- /dev/null +++ b/bbdev/mcp/tools/verilator_build.py @@ -0,0 +1,21 @@ +"""MCP tool: bbdev_verilator_build.""" + +from __future__ import annotations + +from common import call, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_verilator_build( + config: str, + jobs: int = 16, + coverage: bool = False, + ) -> str: + """Build non-bebop Verilator sim binary. POST /verilator/build.""" + if e := need("config", config): + return err(e) + params = {"config": config, "jobs": jobs} + if coverage: + params["coverage"] = True + return fmt(call("/verilator/build", params, timeout=3600)) diff --git a/bbdev/mcp/tools/verilator_clean.py b/bbdev/mcp/tools/verilator_clean.py new file mode 100644 index 000000000..f6c81e507 --- /dev/null +++ b/bbdev/mcp/tools/verilator_clean.py @@ -0,0 +1,14 @@ +"""MCP tool: bbdev_verilator_clean.""" + +from __future__ import annotations + +from common import call, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_verilator_clean(config: str) -> str: + """Clean non-bebop Verilator build. POST /verilator/clean.""" + if e := need("config", config): + return err(e) + return fmt(call("/verilator/clean", {"config": config}, timeout=300)) diff --git a/bbdev/mcp/tools/verilator_run.py b/bbdev/mcp/tools/verilator_run.py new file mode 100644 index 000000000..fb3f80b68 --- /dev/null +++ b/bbdev/mcp/tools/verilator_run.py @@ -0,0 +1,42 @@ +"""MCP tool: bbdev_verilator_run.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from common import call, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_verilator_run( + binary: str, + config: str, + batch: bool = True, + coverage: bool = False, + jobs: Optional[int] = None, + itrace: bool = False, + mtrace: bool = False, + pmctrace: bool = False, + ctrace: bool = False, + banktrace: bool = False, + ) -> str: + """Full non-bebop Verilator flow. POST /verilator/run.""" + if e := need("binary", binary): + return err(e) + if e := need("config", config): + return err(e) + params: Dict[str, Any] = { + "binary": binary, + "config": config, + "batch": batch, + "coverage": coverage, + "itrace": itrace, + "mtrace": mtrace, + "pmctrace": pmctrace, + "ctrace": ctrace, + "banktrace": banktrace, + } + if jobs is not None: + params["jobs"] = jobs + return fmt(call("/verilator/run", params, timeout=14400)) diff --git a/bbdev/mcp/tools/verilator_sim.py b/bbdev/mcp/tools/verilator_sim.py new file mode 100644 index 000000000..99c1ae50c --- /dev/null +++ b/bbdev/mcp/tools/verilator_sim.py @@ -0,0 +1,39 @@ +"""MCP tool: bbdev_verilator_sim.""" + +from __future__ import annotations + +from typing import Any, Dict + +from common import call, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_verilator_sim( + binary: str, + config: str, + batch: bool = True, + coverage: bool = False, + itrace: bool = False, + mtrace: bool = False, + pmctrace: bool = False, + ctrace: bool = False, + banktrace: bool = False, + ) -> str: + """Run one workload on non-bebop Verilator. POST /verilator/sim.""" + for n, v in (("binary", binary), ("config", config)): + if e := need(n, v): + return err(e) + params: Dict[str, Any] = { + "binary": binary, + "config": config, + "batch": batch, + "itrace": itrace, + "mtrace": mtrace, + "pmctrace": pmctrace, + "ctrace": ctrace, + "banktrace": banktrace, + } + if coverage: + params["coverage"] = True + return fmt(call("/verilator/sim", params, timeout=7200)) diff --git a/bbdev/mcp/tools/verilator_verilog.py b/bbdev/mcp/tools/verilator_verilog.py new file mode 100644 index 000000000..4f04d6a3f --- /dev/null +++ b/bbdev/mcp/tools/verilator_verilog.py @@ -0,0 +1,14 @@ +"""MCP tool: bbdev_verilator_verilog.""" + +from __future__ import annotations + +from common import call, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_verilator_verilog(config: str) -> str: + """Generate Verilog for non-bebop Verilator. POST /verilator/verilog.""" + if e := need("config", config): + return err(e) + return fmt(call("/verilator/verilog", {"config": config}, timeout=1800)) diff --git a/bbdev/mcp/tools/workload_build.py b/bbdev/mcp/tools/workload_build.py new file mode 100644 index 000000000..770a3eeed --- /dev/null +++ b/bbdev/mcp/tools/workload_build.py @@ -0,0 +1,20 @@ +"""MCP tool: bbdev_workload_build.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from common import call, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_workload_build(chip: str, model: Optional[str] = None) -> str: + """Build workloads for a chip. POST /workload/build.""" + if e := need("chip", chip): + return err(e) + params: Dict[str, Any] = {"chip": chip} + if model: + params["model"] = model + return fmt(call("/workload/build", params, timeout=1800)) + diff --git a/bbdev/mcp/tools/workload_clean.py b/bbdev/mcp/tools/workload_clean.py new file mode 100644 index 000000000..d8beccfba --- /dev/null +++ b/bbdev/mcp/tools/workload_clean.py @@ -0,0 +1,14 @@ +"""MCP tool: bbdev_workload_clean.""" + +from __future__ import annotations + + +from common import call, fmt, opt + + +def register(mcp): + @mcp.tool() + def bbdev_workload_clean() -> str: + """Clean workload artifacts. POST /workload/clean.""" + return fmt(call("/workload/clean", {}, timeout=120)) + diff --git a/bbdev/mcp/tools/workload_tohex.py b/bbdev/mcp/tools/workload_tohex.py new file mode 100644 index 000000000..018a5d708 --- /dev/null +++ b/bbdev/mcp/tools/workload_tohex.py @@ -0,0 +1,14 @@ +"""MCP tool: bbdev_workload_tohex.""" + +from __future__ import annotations + + +from common import call, fmt, opt + + +def register(mcp): + @mcp.tool() + def bbdev_workload_tohex() -> str: + """Convert workload ELF to hex. POST /workload/tohex.""" + return fmt(call("/workload/tohex", {}, timeout=1800)) + diff --git a/bbdev/mcp/tools/yosys_run.py b/bbdev/mcp/tools/yosys_run.py new file mode 100644 index 000000000..8b85a24fb --- /dev/null +++ b/bbdev/mcp/tools/yosys_run.py @@ -0,0 +1,32 @@ +"""MCP tool: bbdev_yosys_run.""" + +from __future__ import annotations + +from typing import Optional + +from common import call, fmt, opt + + +def register(mcp): + @mcp.tool() + def bbdev_yosys_run( + top: Optional[str] = None, + config: Optional[str] = None, + output_dir: Optional[str] = None, + log_dir: Optional[str] = None, + ) -> str: + """Full yosys flow. POST /yosys/run.""" + return fmt( + call( + "/yosys/run", + opt( + {}, + top=top, + config=config, + output_dir=output_dir, + log_dir=log_dir, + ), + timeout=3600, + ) + ) + diff --git a/bbdev/mcp/tools/yosys_synth.py b/bbdev/mcp/tools/yosys_synth.py new file mode 100644 index 000000000..9287ef159 --- /dev/null +++ b/bbdev/mcp/tools/yosys_synth.py @@ -0,0 +1,32 @@ +"""MCP tool: bbdev_yosys_synth.""" + +from __future__ import annotations + +from typing import Optional + +from common import call, fmt, opt + + +def register(mcp): + @mcp.tool() + def bbdev_yosys_synth( + top: Optional[str] = None, + config: Optional[str] = None, + output_dir: Optional[str] = None, + log_dir: Optional[str] = None, + ) -> str: + """Yosys synthesis + OpenSTA. POST /yosys/synth.""" + return fmt( + call( + "/yosys/synth", + opt( + {}, + top=top, + config=config, + output_dir=output_dir, + log_dir=log_dir, + ), + timeout=3600, + ) + ) + diff --git a/bbdev/mcp/tools/yosys_verilog.py b/bbdev/mcp/tools/yosys_verilog.py new file mode 100644 index 000000000..6ad6c5500 --- /dev/null +++ b/bbdev/mcp/tools/yosys_verilog.py @@ -0,0 +1,32 @@ +"""MCP tool: bbdev_yosys_verilog.""" + +from __future__ import annotations + +from typing import Optional + +from common import call, fmt, opt + + +def register(mcp): + @mcp.tool() + def bbdev_yosys_verilog( + top: Optional[str] = None, + config: Optional[str] = None, + output_dir: Optional[str] = None, + log_dir: Optional[str] = None, + ) -> str: + """Generate Verilog for yosys flow. POST /yosys/verilog.""" + return fmt( + call( + "/yosys/verilog", + opt( + {}, + top=top, + config=config, + output_dir=output_dir, + log_dir=log_dir, + ), + timeout=1800, + ) + ) + From cf5e47f4dc525b28933264e40f85dd93a3db030a Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 3 Aug 2026 12:58:00 +0800 Subject: [PATCH 103/196] feat: refactor common API to use submit function for asynchronous task submission and add task status retrieval functionality --- bbdev/mcp/common.py | 103 ++++++++++---------- bbdev/mcp/tests/test_common.py | 99 +++++++++++++++++++ bbdev/mcp/tools/__init__.py | 2 + bbdev/mcp/tools/bebop_p2e_batch.py | 5 +- bbdev/mcp/tools/bebop_p2e_buildbitstream.py | 5 +- bbdev/mcp/tools/bebop_p2e_clean.py | 4 +- bbdev/mcp/tools/bebop_p2e_runworkload.py | 5 +- bbdev/mcp/tools/bebop_p2e_verilog.py | 5 +- bbdev/mcp/tools/bebop_verilator_batch.py | 5 +- bbdev/mcp/tools/bebop_verilator_build.py | 4 +- bbdev/mcp/tools/bebop_verilator_clean.py | 4 +- bbdev/mcp/tools/bebop_verilator_run.py | 5 +- bbdev/mcp/tools/bebop_verilator_sim.py | 4 +- bbdev/mcp/tools/bebop_verilator_verilog.py | 4 +- bbdev/mcp/tools/bemu_batch.py | 5 +- bbdev/mcp/tools/bemu_sim.py | 4 +- bbdev/mcp/tools/compiler_build.py | 4 +- bbdev/mcp/tools/dc_verilog.py | 5 +- bbdev/mcp/tools/firesim_buildbitstream.py | 4 +- bbdev/mcp/tools/firesim_enumeratefpgas.py | 4 +- bbdev/mcp/tools/firesim_infrasetup.py | 4 +- bbdev/mcp/tools/firesim_runworkload.py | 4 +- bbdev/mcp/tools/kernel_build.py | 4 +- bbdev/mcp/tools/task_status.py | 17 ++++ bbdev/mcp/tools/uvm_build.py | 4 +- bbdev/mcp/tools/uvm_run.py | 4 +- bbdev/mcp/tools/verilator_build.py | 4 +- bbdev/mcp/tools/verilator_clean.py | 4 +- bbdev/mcp/tools/verilator_run.py | 4 +- bbdev/mcp/tools/verilator_sim.py | 4 +- bbdev/mcp/tools/verilator_verilog.py | 4 +- bbdev/mcp/tools/workload_build.py | 4 +- bbdev/mcp/tools/workload_clean.py | 4 +- bbdev/mcp/tools/workload_tohex.py | 4 +- bbdev/mcp/tools/yosys_run.py | 5 +- bbdev/mcp/tools/yosys_synth.py | 5 +- bbdev/mcp/tools/yosys_verilog.py | 5 +- 37 files changed, 235 insertions(+), 129 deletions(-) create mode 100644 bbdev/mcp/tests/test_common.py create mode 100644 bbdev/mcp/tools/task_status.py diff --git a/bbdev/mcp/common.py b/bbdev/mcp/common.py index 457ec6300..fded013a1 100644 --- a/bbdev/mcp/common.py +++ b/bbdev/mcp/common.py @@ -25,6 +25,7 @@ _proc: Optional[subprocess.Popen] = None _port: Optional[int] = None _log_fh = None +_submitted_trace_ids: set[str] = set() def _log(msg: str) -> None: @@ -195,7 +196,8 @@ def _read_state(trace_id: str) -> Optional[Dict[str, Any]]: return data -def _call(endpoint: str, params: Dict[str, Any], timeout: int = 600) -> Dict[str, Any]: +def submit(endpoint: str, params: Dict[str, Any]) -> Dict[str, Any]: + """Submit an asynchronous bbdev task without waiting for completion.""" port = _ensure() base = f"http://127.0.0.1:{port}" _log(f"POST {endpoint} params={params}") @@ -222,62 +224,60 @@ def _call(endpoint: str, params: Dict[str, Any], timeout: int = 600) -> Dict[str "port": port, } - deadline = time.monotonic() + timeout - beat = time.monotonic() - stuck = 0 - while time.monotonic() < deadline: - state = _read_state(trace_id) - if state is None: - time.sleep(2) - continue - if "success" in state: - out = state["success"].get("body", state["success"]) - if not isinstance(out, dict): - raise RuntimeError(f"success body must be object: {out!r}") - out.setdefault("success", True) - out.setdefault("trace_id", trace_id) - out.setdefault("port", port) - return out - if "failure" in state: - body = state["failure"].get("body", state["failure"]) - return { - "success": False, - "failure": True, - "trace_id": trace_id, - "port": port, - "body": body, - "server_log": str(LOG), - } - if state.get("processing") is True: - stuck += 1 - if stuck >= 30: - return { - "success": False, - "failure": True, - "error": "stuck in legacy processing state", - "trace_id": trace_id, - "port": port, - "server_log": str(LOG), - } - else: - stuck = 0 - - now = time.monotonic() - if now - beat >= 30: - _log(f"still running {endpoint} trace={trace_id}") - beat = now - time.sleep(2) - + _submitted_trace_ids.add(trace_id) return { - "success": False, - "failure": True, - "error": f"timed out after {timeout}s waiting for {endpoint}", + "accepted": True, + "processing": True, "trace_id": trace_id, - "server_log": str(LOG), "port": port, } +def task_status(trace_id: str) -> Dict[str, Any]: + """Read the terminal or in-progress state for a submitted bbdev task.""" + if not trace_id: + raise ValueError("trace_id is required") + + state = _read_state(trace_id) + if state is None: + if trace_id not in _submitted_trace_ids: + raise RuntimeError(f"unknown task trace_id: {trace_id}") + return { + "accepted": True, + "processing": True, + "queued": True, + "trace_id": trace_id, + } + if "success" in state: + _submitted_trace_ids.discard(trace_id) + out = state["success"].get("body", state["success"]) + if not isinstance(out, dict): + raise RuntimeError(f"success body must be object: {out!r}") + out.setdefault("success", True) + out.setdefault("failure", False) + out.setdefault("processing", False) + out.setdefault("trace_id", trace_id) + return out + if "failure" in state: + _submitted_trace_ids.discard(trace_id) + body = state["failure"].get("body", state["failure"]) + return { + "success": False, + "failure": True, + "processing": False, + "trace_id": trace_id, + "body": body, + "server_log": str(LOG), + } + if state.get("processing") is True: + out = dict(state) + out.setdefault("accepted", True) + out.setdefault("processing", True) + out.setdefault("trace_id", trace_id) + return out + raise RuntimeError(f"invalid task state for trace_id {trace_id}: {state!r}") + + def _load_toml(path: Path) -> Dict[str, Any]: with path.open("rb") as f: data = tomllib.load(f) @@ -441,6 +441,5 @@ def dups(xs): err = _err need = _need opt = _opt -call = _call balldomain_path = _balldomain_path validate_toml = _validate diff --git a/bbdev/mcp/tests/test_common.py b/bbdev/mcp/tests/test_common.py new file mode 100644 index 000000000..95c31ae94 --- /dev/null +++ b/bbdev/mcp/tests/test_common.py @@ -0,0 +1,99 @@ +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import common + + +class SubmitTest(unittest.TestCase): + def setUp(self): + common._submitted_trace_ids.clear() + + @patch("common._http") + @patch("common._ensure", return_value=5100) + def test_submit_returns_trace_without_reading_state(self, ensure, http): + http.return_value = 202, {"trace_id": "trace-1"} + + result = common.submit("/workload/build", {"chip": "pebble"}) + + self.assertEqual( + result, + { + "accepted": True, + "processing": True, + "trace_id": "trace-1", + "port": 5100, + }, + ) + ensure.assert_called_once_with() + http.assert_called_once_with( + "POST", + "http://127.0.0.1:5100/workload/build", + {"chip": "pebble"}, + timeout=30, + ) + + @patch("common._read_state", return_value=None) + def test_task_status_reports_queued_before_state_exists(self, read_state): + common._submitted_trace_ids.add("trace-1") + self.assertEqual( + common.task_status("trace-1"), + { + "accepted": True, + "processing": True, + "queued": True, + "trace_id": "trace-1", + }, + ) + read_state.assert_called_once_with("trace-1") + + @patch("common._read_state", return_value=None) + def test_task_status_rejects_unknown_trace(self, read_state): + with self.assertRaisesRegex(RuntimeError, "unknown task trace_id"): + common.task_status("unknown") + + @patch("common._read_state") + def test_task_status_reports_success(self, read_state): + read_state.return_value = { + "success": {"body": {"returncode": 0, "task": "build"}} + } + + self.assertEqual( + common.task_status("trace-1"), + { + "returncode": 0, + "task": "build", + "success": True, + "failure": False, + "processing": False, + "trace_id": "trace-1", + }, + ) + + @patch("common._read_state") + def test_task_status_reports_failure(self, read_state): + read_state.return_value = {"failure": {"body": {"returncode": 1}}} + + self.assertEqual( + common.task_status("trace-1"), + { + "success": False, + "failure": True, + "processing": False, + "trace_id": "trace-1", + "body": {"returncode": 1}, + "server_log": str(common.LOG), + }, + ) + + @patch("common._read_state", return_value={"unknown": True}) + def test_task_status_rejects_invalid_state(self, read_state): + with self.assertRaisesRegex(RuntimeError, "invalid task state"): + common.task_status("trace-1") + + +if __name__ == "__main__": + unittest.main() diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index 0bb826a32..d495299bd 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -3,6 +3,7 @@ from __future__ import annotations from . import validate +from . import task_status from . import compiler_build from . import workload_clean from . import workload_build @@ -39,6 +40,7 @@ MODULES = [ validate, + task_status, compiler_build, workload_clean, workload_build, diff --git a/bbdev/mcp/tools/bebop_p2e_batch.py b/bbdev/mcp/tools/bebop_p2e_batch.py index 0bb644ed5..885dc4f2e 100644 --- a/bbdev/mcp/tools/bebop_p2e_batch.py +++ b/bbdev/mcp/tools/bebop_p2e_batch.py @@ -3,7 +3,7 @@ from __future__ import annotations -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -25,7 +25,7 @@ def bbdev_bebop_p2e_batch( if test not in ("elf-tests", "pk-tests"): return err("test must be elf-tests or pk-tests") return fmt( - call( + submit( "/bebop/p2e/batch", { "chip": chip, @@ -33,7 +33,6 @@ def bbdev_bebop_p2e_batch( "build-dir": build_dir, "test": test, }, - timeout=14400, ) ) diff --git a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py index 0cda45734..c28d48b4e 100644 --- a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py +++ b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py @@ -4,7 +4,7 @@ from typing import Optional -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -18,10 +18,9 @@ def bbdev_bebop_p2e_buildbitstream( if e := need("config", config): return err(e) return fmt( - call( + submit( "/bebop/p2e/buildbitstream", opt({"config": config}, vsrc_dir=vsrc_dir, output_dir=output_dir), - timeout=14400, ) ) diff --git a/bbdev/mcp/tools/bebop_p2e_clean.py b/bbdev/mcp/tools/bebop_p2e_clean.py index 7a7baf970..9ad741f92 100644 --- a/bbdev/mcp/tools/bebop_p2e_clean.py +++ b/bbdev/mcp/tools/bebop_p2e_clean.py @@ -3,7 +3,7 @@ from __future__ import annotations -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -12,5 +12,5 @@ def bbdev_bebop_p2e_clean(config: str) -> str: """Clean bebop-p2e build. POST /bebop/p2e/clean.""" if e := need("config", config): return err(e) - return fmt(call("/bebop/p2e/clean", {"config": config}, timeout=300)) + return fmt(submit("/bebop/p2e/clean", {"config": config})) diff --git a/bbdev/mcp/tools/bebop_p2e_runworkload.py b/bbdev/mcp/tools/bebop_p2e_runworkload.py index b04dbd246..cad65fcc4 100644 --- a/bbdev/mcp/tools/bebop_p2e_runworkload.py +++ b/bbdev/mcp/tools/bebop_p2e_runworkload.py @@ -3,7 +3,7 @@ from __future__ import annotations -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -14,10 +14,9 @@ def bbdev_bebop_p2e_runworkload(image: str, bitstream: str) -> str: if e := need(n, v): return err(e) return fmt( - call( + submit( "/bebop/p2e/runworkload", {"image": image, "bitstream": bitstream}, - timeout=7200, ) ) diff --git a/bbdev/mcp/tools/bebop_p2e_verilog.py b/bbdev/mcp/tools/bebop_p2e_verilog.py index c6d55c979..5e09de66a 100644 --- a/bbdev/mcp/tools/bebop_p2e_verilog.py +++ b/bbdev/mcp/tools/bebop_p2e_verilog.py @@ -4,7 +4,7 @@ from typing import Optional -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -16,10 +16,9 @@ def bbdev_bebop_p2e_verilog( if e := need("config", config): return err(e) return fmt( - call( + submit( "/bebop/p2e/verilog", opt({"config": config}, output_dir=output_dir), - timeout=1800, ) ) diff --git a/bbdev/mcp/tools/bebop_verilator_batch.py b/bbdev/mcp/tools/bebop_verilator_batch.py index f8c249b61..4588a1f8f 100644 --- a/bbdev/mcp/tools/bebop_verilator_batch.py +++ b/bbdev/mcp/tools/bebop_verilator_batch.py @@ -3,7 +3,7 @@ from __future__ import annotations -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -21,7 +21,7 @@ def bbdev_bebop_verilator_batch( if test not in ("elf-tests", "pk-tests"): return err("test must be elf-tests or pk-tests") return fmt( - call( + submit( "/bebop/verilator/batch", { "chip": chip, @@ -29,7 +29,6 @@ def bbdev_bebop_verilator_batch( "test": test, "clean-before": clean_before, }, - timeout=14400, ) ) diff --git a/bbdev/mcp/tools/bebop_verilator_build.py b/bbdev/mcp/tools/bebop_verilator_build.py index 0cbfa30cd..b9d745bdd 100644 --- a/bbdev/mcp/tools/bebop_verilator_build.py +++ b/bbdev/mcp/tools/bebop_verilator_build.py @@ -3,7 +3,7 @@ from __future__ import annotations -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -13,6 +13,6 @@ def bbdev_bebop_verilator_build(config: str, jobs: int = 16) -> str: if e := need("config", config): return err(e) return fmt( - call("/bebop/verilator/build", {"config": config, "jobs": jobs}, timeout=3600) + submit("/bebop/verilator/build", {"config": config, "jobs": jobs}) ) diff --git a/bbdev/mcp/tools/bebop_verilator_clean.py b/bbdev/mcp/tools/bebop_verilator_clean.py index 69bc55641..a2d93970d 100644 --- a/bbdev/mcp/tools/bebop_verilator_clean.py +++ b/bbdev/mcp/tools/bebop_verilator_clean.py @@ -3,7 +3,7 @@ from __future__ import annotations -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -12,5 +12,5 @@ def bbdev_bebop_verilator_clean(config: str) -> str: """Clean bebop-verilator build. POST /bebop/verilator/clean.""" if e := need("config", config): return err(e) - return fmt(call("/bebop/verilator/clean", {"config": config}, timeout=300)) + return fmt(submit("/bebop/verilator/clean", {"config": config})) diff --git a/bbdev/mcp/tools/bebop_verilator_run.py b/bbdev/mcp/tools/bebop_verilator_run.py index f5348f2f6..7805207d8 100644 --- a/bbdev/mcp/tools/bebop_verilator_run.py +++ b/bbdev/mcp/tools/bebop_verilator_run.py @@ -3,7 +3,7 @@ from __future__ import annotations -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -24,7 +24,7 @@ def bbdev_bebop_verilator_run( if e := need(n, v): return err(e) return fmt( - call( + submit( "/bebop/verilator/run", { "binary": binary, @@ -37,7 +37,6 @@ def bbdev_bebop_verilator_run( "banktrace": banktrace, "no-wave": no_wave, }, - timeout=14400, ) ) diff --git a/bbdev/mcp/tools/bebop_verilator_sim.py b/bbdev/mcp/tools/bebop_verilator_sim.py index f716767d3..eb8daa3e1 100644 --- a/bbdev/mcp/tools/bebop_verilator_sim.py +++ b/bbdev/mcp/tools/bebop_verilator_sim.py @@ -4,7 +4,7 @@ from typing import Any, Dict, Optional -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -39,5 +39,5 @@ def bbdev_bebop_verilator_sim( params["log_dir"] = log_dir if fst_dir: params["fst_dir"] = fst_dir - return fmt(call("/bebop/verilator/sim", params, timeout=7200)) + return fmt(submit("/bebop/verilator/sim", params)) diff --git a/bbdev/mcp/tools/bebop_verilator_verilog.py b/bbdev/mcp/tools/bebop_verilator_verilog.py index db598acc7..31bc509b2 100644 --- a/bbdev/mcp/tools/bebop_verilator_verilog.py +++ b/bbdev/mcp/tools/bebop_verilator_verilog.py @@ -3,7 +3,7 @@ from __future__ import annotations -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -12,5 +12,5 @@ def bbdev_bebop_verilator_verilog(config: str) -> str: """Generate Verilog for bebop-verilator. POST /bebop/verilator/verilog.""" if e := need("config", config): return err(e) - return fmt(call("/bebop/verilator/verilog", {"config": config}, timeout=1800)) + return fmt(submit("/bebop/verilator/verilog", {"config": config})) diff --git a/bbdev/mcp/tools/bemu_batch.py b/bbdev/mcp/tools/bemu_batch.py index 668751979..df17d92a0 100644 --- a/bbdev/mcp/tools/bemu_batch.py +++ b/bbdev/mcp/tools/bemu_batch.py @@ -3,7 +3,7 @@ from __future__ import annotations -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -15,10 +15,9 @@ def bbdev_bemu_batch(chip: str, test: str, clean_before: bool = False) -> str: if test not in ("elf-tests", "pk-tests"): return err("test must be elf-tests or pk-tests") return fmt( - call( + submit( "/bebop/bemu/batch", {"chip": chip, "test": test, "clean-before": clean_before}, - timeout=7200, ) ) diff --git a/bbdev/mcp/tools/bemu_sim.py b/bbdev/mcp/tools/bemu_sim.py index f09f76f96..5381e1ff6 100644 --- a/bbdev/mcp/tools/bemu_sim.py +++ b/bbdev/mcp/tools/bemu_sim.py @@ -4,7 +4,7 @@ from typing import Any, Dict, Optional -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -22,5 +22,5 @@ def bbdev_bemu_sim( params: Dict[str, Any] = {"chip": chip, "binary": binary, "pk": pk} if log_dir: params["log_dir"] = log_dir - return fmt(call("/bebop/bemu/sim", params, timeout=1800)) + return fmt(submit("/bebop/bemu/sim", params)) diff --git a/bbdev/mcp/tools/compiler_build.py b/bbdev/mcp/tools/compiler_build.py index ea1ac814c..d6180eb69 100644 --- a/bbdev/mcp/tools/compiler_build.py +++ b/bbdev/mcp/tools/compiler_build.py @@ -3,7 +3,7 @@ from __future__ import annotations -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -13,6 +13,6 @@ def bbdev_compiler_build(chip: str, stable: bool = False) -> str: if e := need("chip", chip): return err(e) return fmt( - call("/compiler/build", {"chip": chip, "stable": stable}, timeout=1800) + submit("/compiler/build", {"chip": chip, "stable": stable}) ) diff --git a/bbdev/mcp/tools/dc_verilog.py b/bbdev/mcp/tools/dc_verilog.py index 31a94347f..619bc0318 100644 --- a/bbdev/mcp/tools/dc_verilog.py +++ b/bbdev/mcp/tools/dc_verilog.py @@ -4,7 +4,7 @@ from typing import Optional -from common import call, err, fmt, opt +from common import submit, err, fmt, opt def register(mcp): @@ -16,10 +16,9 @@ def bbdev_dc_verilog( if not config and not rtl_dir: return err("config or rtl_dir is required") return fmt( - call( + submit( "/dc/verilog", opt({}, config=config, dir=rtl_dir), - timeout=1800, ) ) diff --git a/bbdev/mcp/tools/firesim_buildbitstream.py b/bbdev/mcp/tools/firesim_buildbitstream.py index 17d215c38..68d818b14 100644 --- a/bbdev/mcp/tools/firesim_buildbitstream.py +++ b/bbdev/mcp/tools/firesim_buildbitstream.py @@ -3,12 +3,12 @@ from __future__ import annotations -from common import call, fmt, opt +from common import submit, fmt, opt def register(mcp): @mcp.tool() def bbdev_firesim_buildbitstream() -> str: """Build FireSim bitstream. POST /firesim/buildbitstream.""" - return fmt(call("/firesim/buildbitstream", {}, timeout=14400)) + return fmt(submit("/firesim/buildbitstream", {})) diff --git a/bbdev/mcp/tools/firesim_enumeratefpgas.py b/bbdev/mcp/tools/firesim_enumeratefpgas.py index ba9c68e58..4d1ca5468 100644 --- a/bbdev/mcp/tools/firesim_enumeratefpgas.py +++ b/bbdev/mcp/tools/firesim_enumeratefpgas.py @@ -3,12 +3,12 @@ from __future__ import annotations -from common import call, fmt, opt +from common import submit, fmt, opt def register(mcp): @mcp.tool() def bbdev_firesim_enumeratefpgas() -> str: """Enumerate FireSim FPGAs. POST /firesim/enumeratefpgas.""" - return fmt(call("/firesim/enumeratefpgas", {}, timeout=1800)) + return fmt(submit("/firesim/enumeratefpgas", {})) diff --git a/bbdev/mcp/tools/firesim_infrasetup.py b/bbdev/mcp/tools/firesim_infrasetup.py index 91838b914..b008e72b3 100644 --- a/bbdev/mcp/tools/firesim_infrasetup.py +++ b/bbdev/mcp/tools/firesim_infrasetup.py @@ -3,12 +3,12 @@ from __future__ import annotations -from common import call, fmt, opt +from common import submit, fmt, opt def register(mcp): @mcp.tool() def bbdev_firesim_infrasetup(jobs: int = 16) -> str: """FireSim infrasetup. POST /firesim/infrasetup.""" - return fmt(call("/firesim/infrasetup", {"jobs": jobs}, timeout=7200)) + return fmt(submit("/firesim/infrasetup", {"jobs": jobs})) diff --git a/bbdev/mcp/tools/firesim_runworkload.py b/bbdev/mcp/tools/firesim_runworkload.py index 88836f08c..005f13340 100644 --- a/bbdev/mcp/tools/firesim_runworkload.py +++ b/bbdev/mcp/tools/firesim_runworkload.py @@ -3,12 +3,12 @@ from __future__ import annotations -from common import call, fmt, opt +from common import submit, fmt, opt def register(mcp): @mcp.tool() def bbdev_firesim_runworkload(jobs: int = 16) -> str: """Run FireSim workload. POST /firesim/runworkload.""" - return fmt(call("/firesim/runworkload", {"jobs": jobs}, timeout=14400)) + return fmt(submit("/firesim/runworkload", {"jobs": jobs})) diff --git a/bbdev/mcp/tools/kernel_build.py b/bbdev/mcp/tools/kernel_build.py index 98260249c..ffffa5883 100644 --- a/bbdev/mcp/tools/kernel_build.py +++ b/bbdev/mcp/tools/kernel_build.py @@ -4,7 +4,7 @@ from typing import Any, Dict, Optional -from common import call, fmt, opt +from common import submit, fmt, opt def register(mcp): @@ -21,5 +21,5 @@ def bbdev_kernel_build( params["visible-hart-count"] = visible_hart_count if total_hart_count is not None: params["total-hart-count"] = total_hart_count - return fmt(call("/kernel/build", params, timeout=7200)) + return fmt(submit("/kernel/build", params)) diff --git a/bbdev/mcp/tools/task_status.py b/bbdev/mcp/tools/task_status.py new file mode 100644 index 000000000..902832afa --- /dev/null +++ b/bbdev/mcp/tools/task_status.py @@ -0,0 +1,17 @@ +"""MCP tool: bbdev_task_status.""" + +from __future__ import annotations + +from common import err, fmt, need, task_status + + +def register(mcp): + @mcp.tool() + def bbdev_task_status(trace_id: str) -> str: + """Read a bbdev task result by trace_id without blocking.""" + if error := need("trace_id", trace_id): + return err(error) + try: + return fmt(task_status(trace_id)) + except Exception as ex: + return fmt({"success": False, "failure": True, "error": str(ex)}) diff --git a/bbdev/mcp/tools/uvm_build.py b/bbdev/mcp/tools/uvm_build.py index 8df86bd7a..9a5a51afc 100644 --- a/bbdev/mcp/tools/uvm_build.py +++ b/bbdev/mcp/tools/uvm_build.py @@ -4,7 +4,7 @@ from typing import Any, Dict, Optional -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -20,5 +20,5 @@ def bbdev_uvm_build( params["ball"] = ball if filelist: params["filelist"] = filelist - return fmt(call("/uvm/build", params, timeout=3600)) + return fmt(submit("/uvm/build", params)) diff --git a/bbdev/mcp/tools/uvm_run.py b/bbdev/mcp/tools/uvm_run.py index d4902fdb3..7e763df66 100644 --- a/bbdev/mcp/tools/uvm_run.py +++ b/bbdev/mcp/tools/uvm_run.py @@ -4,7 +4,7 @@ from typing import Any, Dict, Optional -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -22,5 +22,5 @@ def bbdev_uvm_run( params["filelist"] = filelist if test: params["test"] = test - return fmt(call("/uvm/run", params, timeout=7200)) + return fmt(submit("/uvm/run", params)) diff --git a/bbdev/mcp/tools/verilator_build.py b/bbdev/mcp/tools/verilator_build.py index 55b39058d..4961f60d8 100644 --- a/bbdev/mcp/tools/verilator_build.py +++ b/bbdev/mcp/tools/verilator_build.py @@ -2,7 +2,7 @@ from __future__ import annotations -from common import call, err, fmt, need +from common import submit, err, fmt, need def register(mcp): @@ -18,4 +18,4 @@ def bbdev_verilator_build( params = {"config": config, "jobs": jobs} if coverage: params["coverage"] = True - return fmt(call("/verilator/build", params, timeout=3600)) + return fmt(submit("/verilator/build", params)) diff --git a/bbdev/mcp/tools/verilator_clean.py b/bbdev/mcp/tools/verilator_clean.py index f6c81e507..c56c5308a 100644 --- a/bbdev/mcp/tools/verilator_clean.py +++ b/bbdev/mcp/tools/verilator_clean.py @@ -2,7 +2,7 @@ from __future__ import annotations -from common import call, err, fmt, need +from common import submit, err, fmt, need def register(mcp): @@ -11,4 +11,4 @@ def bbdev_verilator_clean(config: str) -> str: """Clean non-bebop Verilator build. POST /verilator/clean.""" if e := need("config", config): return err(e) - return fmt(call("/verilator/clean", {"config": config}, timeout=300)) + return fmt(submit("/verilator/clean", {"config": config})) diff --git a/bbdev/mcp/tools/verilator_run.py b/bbdev/mcp/tools/verilator_run.py index fb3f80b68..865e05fd2 100644 --- a/bbdev/mcp/tools/verilator_run.py +++ b/bbdev/mcp/tools/verilator_run.py @@ -4,7 +4,7 @@ from typing import Any, Dict, Optional -from common import call, err, fmt, need +from common import submit, err, fmt, need def register(mcp): @@ -39,4 +39,4 @@ def bbdev_verilator_run( } if jobs is not None: params["jobs"] = jobs - return fmt(call("/verilator/run", params, timeout=14400)) + return fmt(submit("/verilator/run", params)) diff --git a/bbdev/mcp/tools/verilator_sim.py b/bbdev/mcp/tools/verilator_sim.py index 99c1ae50c..05a830a96 100644 --- a/bbdev/mcp/tools/verilator_sim.py +++ b/bbdev/mcp/tools/verilator_sim.py @@ -4,7 +4,7 @@ from typing import Any, Dict -from common import call, err, fmt, need +from common import submit, err, fmt, need def register(mcp): @@ -36,4 +36,4 @@ def bbdev_verilator_sim( } if coverage: params["coverage"] = True - return fmt(call("/verilator/sim", params, timeout=7200)) + return fmt(submit("/verilator/sim", params)) diff --git a/bbdev/mcp/tools/verilator_verilog.py b/bbdev/mcp/tools/verilator_verilog.py index 4f04d6a3f..beff8676b 100644 --- a/bbdev/mcp/tools/verilator_verilog.py +++ b/bbdev/mcp/tools/verilator_verilog.py @@ -2,7 +2,7 @@ from __future__ import annotations -from common import call, err, fmt, need +from common import submit, err, fmt, need def register(mcp): @@ -11,4 +11,4 @@ def bbdev_verilator_verilog(config: str) -> str: """Generate Verilog for non-bebop Verilator. POST /verilator/verilog.""" if e := need("config", config): return err(e) - return fmt(call("/verilator/verilog", {"config": config}, timeout=1800)) + return fmt(submit("/verilator/verilog", {"config": config})) diff --git a/bbdev/mcp/tools/workload_build.py b/bbdev/mcp/tools/workload_build.py index 770a3eeed..5e3d6762a 100644 --- a/bbdev/mcp/tools/workload_build.py +++ b/bbdev/mcp/tools/workload_build.py @@ -4,7 +4,7 @@ from typing import Any, Dict, Optional -from common import call, err, fmt, need, opt +from common import submit, err, fmt, need, opt def register(mcp): @@ -16,5 +16,5 @@ def bbdev_workload_build(chip: str, model: Optional[str] = None) -> str: params: Dict[str, Any] = {"chip": chip} if model: params["model"] = model - return fmt(call("/workload/build", params, timeout=1800)) + return fmt(submit("/workload/build", params)) diff --git a/bbdev/mcp/tools/workload_clean.py b/bbdev/mcp/tools/workload_clean.py index d8beccfba..74f8c9e43 100644 --- a/bbdev/mcp/tools/workload_clean.py +++ b/bbdev/mcp/tools/workload_clean.py @@ -3,12 +3,12 @@ from __future__ import annotations -from common import call, fmt, opt +from common import submit, fmt, opt def register(mcp): @mcp.tool() def bbdev_workload_clean() -> str: """Clean workload artifacts. POST /workload/clean.""" - return fmt(call("/workload/clean", {}, timeout=120)) + return fmt(submit("/workload/clean", {})) diff --git a/bbdev/mcp/tools/workload_tohex.py b/bbdev/mcp/tools/workload_tohex.py index 018a5d708..9f30ddd1e 100644 --- a/bbdev/mcp/tools/workload_tohex.py +++ b/bbdev/mcp/tools/workload_tohex.py @@ -3,12 +3,12 @@ from __future__ import annotations -from common import call, fmt, opt +from common import submit, fmt, opt def register(mcp): @mcp.tool() def bbdev_workload_tohex() -> str: """Convert workload ELF to hex. POST /workload/tohex.""" - return fmt(call("/workload/tohex", {}, timeout=1800)) + return fmt(submit("/workload/tohex", {})) diff --git a/bbdev/mcp/tools/yosys_run.py b/bbdev/mcp/tools/yosys_run.py index 8b85a24fb..72ee431f8 100644 --- a/bbdev/mcp/tools/yosys_run.py +++ b/bbdev/mcp/tools/yosys_run.py @@ -4,7 +4,7 @@ from typing import Optional -from common import call, fmt, opt +from common import submit, fmt, opt def register(mcp): @@ -17,7 +17,7 @@ def bbdev_yosys_run( ) -> str: """Full yosys flow. POST /yosys/run.""" return fmt( - call( + submit( "/yosys/run", opt( {}, @@ -26,7 +26,6 @@ def bbdev_yosys_run( output_dir=output_dir, log_dir=log_dir, ), - timeout=3600, ) ) diff --git a/bbdev/mcp/tools/yosys_synth.py b/bbdev/mcp/tools/yosys_synth.py index 9287ef159..520b101e3 100644 --- a/bbdev/mcp/tools/yosys_synth.py +++ b/bbdev/mcp/tools/yosys_synth.py @@ -4,7 +4,7 @@ from typing import Optional -from common import call, fmt, opt +from common import submit, fmt, opt def register(mcp): @@ -17,7 +17,7 @@ def bbdev_yosys_synth( ) -> str: """Yosys synthesis + OpenSTA. POST /yosys/synth.""" return fmt( - call( + submit( "/yosys/synth", opt( {}, @@ -26,7 +26,6 @@ def bbdev_yosys_synth( output_dir=output_dir, log_dir=log_dir, ), - timeout=3600, ) ) diff --git a/bbdev/mcp/tools/yosys_verilog.py b/bbdev/mcp/tools/yosys_verilog.py index 6ad6c5500..05eae8b4c 100644 --- a/bbdev/mcp/tools/yosys_verilog.py +++ b/bbdev/mcp/tools/yosys_verilog.py @@ -4,7 +4,7 @@ from typing import Optional -from common import call, fmt, opt +from common import submit, fmt, opt def register(mcp): @@ -17,7 +17,7 @@ def bbdev_yosys_verilog( ) -> str: """Generate Verilog for yosys flow. POST /yosys/verilog.""" return fmt( - call( + submit( "/yosys/verilog", opt( {}, @@ -26,7 +26,6 @@ def bbdev_yosys_verilog( output_dir=output_dir, log_dir=log_dir, ), - timeout=1800, ) ) From 5a19d9bc74778a5368b5397554af9963875c6e05 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 9 Aug 2026 17:52:28 +0800 Subject: [PATCH 104/196] feat: enhance workflow commands and API to support rushB backend for bebop and verilator simulations --- bbdev/api/services/workflow_tools.py | 2 +- .../api/steps/bebop/bemu/02_sim_event.step.py | 69 ++++++++++- .../bebop/p2e/02_buildbitstream_event.step.py | 2 +- .../bebop/p2e/03_runworkload_event.step.py | 4 +- .../api/steps/bebop/p2e/04_batch_api.step.py | 6 +- .../steps/bebop/p2e/04_batch_event.step.py | 114 ++++++++++++++--- .../bebop/verilator/03_build_event.step.py | 10 +- .../bebop/verilator/04_sim_event.step.py | 75 +++++++---- .../steps/bebop/verilator/05_run_api.step.py | 3 +- .../bebop/verilator/06_batch_event.step.py | 2 +- bbdev/api/steps/kernel/01_build_event.step.py | 87 +++++++++++-- bbdev/api/steps/workload/01_buidl_api.step.py | 9 +- .../api/steps/workload/01_build_event.step.py | 116 +++++++++++++++--- bbdev/api/steps/workload/README.md | 8 ++ bbdev/api/tests/test_api_bebop_p2e_batch.py | 4 +- bbdev/api/utils/port.py | 7 +- bbdev/bbdev | 14 +-- bbdev/flake.lock | 6 +- bbdev/mcp/tools/bebop_p2e_batch.py | 11 +- bbdev/mcp/tools/bebop_p2e_runworkload.py | 25 ++-- bbdev/mcp/tools/bebop_verilator_run.py | 3 +- bbdev/mcp/tools/bebop_verilator_sim.py | 13 +- bbdev/mcp/tools/bemu_sim.py | 20 +-- bbdev/mcp/tools/kernel_build.py | 13 +- bbdev/mcp/tools/workload_build.py | 11 +- 25 files changed, 486 insertions(+), 148 deletions(-) diff --git a/bbdev/api/services/workflow_tools.py b/bbdev/api/services/workflow_tools.py index 339bc1b56..ac742100f 100644 --- a/bbdev/api/services/workflow_tools.py +++ b/bbdev/api/services/workflow_tools.py @@ -19,7 +19,7 @@ def get_description(self) -> str: - /verilator/build: Build verilator (params: jobs) - /verilator/sim: Run simulation (params: binary, batch) - /workload/clean: Clean workload output directory - - /workload/build: Build workload (params: chip, model, stable)""" + - /workload/build: Build workload (params: chip, model, stable, rushB)""" def get_parameters(self) -> Dict[str, Any]: return { diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 102627050..1b29eebf4 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -1,10 +1,7 @@ """ bebop bemu event handler -Runs bebop bemu (Spike-based) emulator: - 1. Resolve binary (ELF) path - 2. Build the selected chip's BEMU wrapper - 3. Run BEMU with the resolved ELF +Run BEMU either through its guest-ELF emulator or the rushB native ABI. """ import os import shlex @@ -81,10 +78,18 @@ def resolve_bemu_binary(bbdir: str, chip: str, binary_name: str) -> str | None: return None +def is_native_host_elf(path: str) -> bool: + try: + with open(path, "rb") as binary: + header = binary.read(20) + except OSError: + return False + return header[:4] == b"\x7fELF" and header[18:20] == b"\x3e\x00" + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - arch_dir = f"{bbdir}/arch" chip = input_data.get("chip") if not chip: @@ -123,13 +128,61 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: clean_model_trace(binary_dir) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - log_dir = input_data.get("log_dir") or f"{arch_dir}/log/{timestamp}-{binary_name}-bemu" + log_dir = f"{bbdir}/log/{timestamp}-bemu-{binary_name}" os.makedirs(log_dir, exist_ok=True) + if input_data.get("rushB"): + # rushB binaries are host executables. Rebuilding and co-locating the + # backend library keeps the ABI selection explicit and avoids sending + # a host ELF through Spike's guest-ELF path. + if not is_native_host_elf(binary_path): + ctx.logger.error( + f"--rushB requires a native x86_64 rushB runner; got guest ELF: {binary_path}" + ) + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "rushB_requires_native_runner", "binary": binary_path}, + trace_id=origin_tid, + ) + return + bemu_library = bemu_cargo_manifest.parent / "target/release/libbebop_bemu.so" + build_cmd = shlex.join([ + "cargo", "build", "--release", "--manifest-path", str(bemu_cargo_manifest), "--lib", + ]) + copy_cmd = shlex.join([ + "cmake", "-E", "copy_if_different", str(bemu_library), binary_dir, + ]) + inner_cmd = f"cd {shlex.quote(binary_dir)} && {build_cmd} && {copy_cmd} && exec {shlex.quote(binary_path)}" + run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" + ctx.logger.info(f"Running rushB BEMU: {run_cmd}") + run_result = stream_run_logger( + cmd=run_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="rushB bemu", + stderr_prefix="rushB bemu", + ) + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "bemu", + "backend": "rushB", + "binary": binary_path, + "chip": chip, + "log_dir": log_dir, + "timestamp": timestamp, + }, + trace_id=origin_tid, + ) + return + # ── Run bebop bemu ──────────────────────────────────────────────────── cargo_args = [ "cargo", "run", + "--release", "--manifest-path", str(bemu_cargo_manifest), "--", @@ -142,6 +195,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ] if input_data.get("pk"): cargo_args.append("--pk") + if input_data.get("disasm"): + cargo_args.append("--disasm") + if input_data.get("tool-profile"): + cargo_args.append("--tool-profile") inner_cmd = f"cd {shlex.quote(binary_dir)} && {shlex.join(cargo_args)}" run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" ctx.logger.info(f"Running bebop bemu: {run_cmd}") diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index 03d9d7e93..23648ec4a 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -65,7 +65,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: build_cmd = ( f"nix develop --ignore-environment --keep HOME --keep ALL_PROXY -c " - f"cargo run --features p2e -- build p2e " + f"cargo run --release --features p2e -- build p2e " f"--rtl-dir=\"{vsrc_dir}\" " f"--out-dir=\"{build_dir}\"" ) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index 48be411c2..fd784ec35 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -152,7 +152,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # deliberately left in place, so runtime/DPIC changes never trigger FPGA synthesis. runtime_cmd = ( f"env BEBOP_P2E_RUNTIME_ONLY=1 BEBOP_P2E_REBUILD_RUNTIME=1 " - f"cargo run --features p2e -- build p2e " + f"cargo run --release --features p2e -- build p2e " f"--rtl-dir=\"{vsrc_dir}\" " f"--out-dir=\"{build_dir}\"" ) @@ -187,7 +187,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── Run bebop run p2e ───────────────────────────────────────────────── run_cmd = ( - f"cargo run --features p2e " + f"cargo run --release --features p2e " f"--config=\"env.OUT_PATH='{build_dir}'\" " f"-- run p2e " f"--image=\"{image_path}\" " diff --git a/bbdev/api/steps/bebop/p2e/04_batch_api.step.py b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py index 668b8e2c3..843a445ff 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_api.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py @@ -13,15 +13,14 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} bitstream = body.get("bitstream", "") - build_dir = body.get("build-dir") or body.get("build_dir", "") - if not bitstream or not build_dir: + if not bitstream: return ApiResponse( status=400, body={ "success": False, "failure": True, "returncode": 400, - "message": "--bitstream and --build-dir parameters are required", + "message": "--bitstream parameter is required", }, ) @@ -48,7 +47,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "chip": chip, "bitstream": bitstream, - "build_dir": build_dir, "test": test_type, } await ctx.enqueue({"topic": "bebop.p2e.batch", "data": {**data, "_trace_id": ctx.trace_id}}) diff --git a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py index f501d788a..bdb1cc2f5 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py @@ -1,11 +1,13 @@ """ bebop p2e batch event handler -Runs bebop p2e nextest batch regression: - 1. Build bebop with p2e feature - 2. Run cargo nextest with p2e-specific config (serial, FPGA single-resource) +Runs bebop p2e nextest batch regression (aligned with runworkload): + 1. Resolve VSRC from bitstream case and rebuild VVAC runtime into case dir + 2. Build bebop with p2e feature and OUT_PATH= + 3. Run cargo nextest with the same OUT_PATH (serial, single FPGA) """ import os +import re import sys from motia import FlowContext, queue @@ -17,7 +19,7 @@ if bebop_path not in sys.path: sys.path.insert(0, bebop_path) -from utils.path import get_buckyball_path +from utils.path import get_buckyball_path, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id from regression import regression_workload_toml @@ -31,6 +33,15 @@ } +def resolve_runtime_config(bitstream: str, requested_config: object) -> str: + if isinstance(requested_config, str) and requested_config: + return requested_config + + build_dir = os.path.dirname(os.path.dirname(os.path.abspath(bitstream))) + case_name = os.path.basename(build_dir) + return re.sub(r"-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}$", "", case_name) + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -39,12 +50,23 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: elf_root = f"{bbdir}/bb-tests/output" bitstream = input_data.get("bitstream", "") - build_dir = input_data.get("build_dir", "") - if not bitstream or not build_dir: - ctx.logger.error("Missing required parameters: bitstream and build_dir") + if not bitstream or not os.path.isfile(bitstream): + ctx.logger.error(f"bitstream .bit file not found: {bitstream}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "bitstream_not_found", "bitstream": bitstream}, + trace_id=origin_tid, + ) + return + + bitstream = os.path.abspath(bitstream) + # Same as runworkload: case home is parent of fpgaCompDir/ + build_dir = os.path.dirname(os.path.dirname(bitstream)) + if not os.path.isdir(build_dir): + ctx.logger.error(f"P2E build case not found for bitstream: {build_dir}") await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_params"}, + extra_fields={"error": "build_dir_not_found", "build_dir": build_dir}, trace_id=origin_tid, ) return @@ -72,9 +94,66 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return ctx.logger.info(f"Running {test_type} with workload config: {workload_toml}") + ctx.logger.info(f"P2E case dir (from bitstream): {build_dir}") - # ── Build bebop p2e (tests) ─────────────────────────────────────────── - build_cmd = "nix develop -c cargo build --features p2e --tests" + config_name = resolve_runtime_config(bitstream, input_data.get("config")) + vsrc_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("vsrc_dir")) + if not os.path.isdir(vsrc_dir): + ctx.logger.error(f"VSRC_PATH does not exist for P2E runtime: {vsrc_dir}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={ + "error": "vsrc_not_found", + "config": config_name, + "vsrc_dir": vsrc_dir, + }, + trace_id=origin_tid, + ) + return + + # Rebuild VVAC host runtime in the bitstream case (same as runworkload). + runtime_cmd = ( + f"env BEBOP_P2E_RUNTIME_ONLY=1 BEBOP_P2E_REBUILD_RUNTIME=1 " + f"cargo run --release --features p2e -- build p2e " + f"--rtl-dir=\"{vsrc_dir}\" " + f"--out-dir=\"{build_dir}\"" + ) + ctx.logger.info("Preparing bebop p2e runtime for the selected bitstream ...") + runtime_result = stream_run_logger( + cmd=runtime_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop p2e runtime", + stderr_prefix="bebop p2e runtime", + ) + rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") + libvctb_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so") + if runtime_result.returncode != 0 or not all(os.path.isfile(path) for path in (rtcfg_path, libvctb_path)): + missing = [path for path in (rtcfg_path, libvctb_path) if not os.path.isfile(path)] + if missing: + ctx.logger.error(f"P2E runtime artifacts missing: {missing}") + await check_result( + ctx, + runtime_result.returncode or 1, + continue_run=False, + extra_fields={ + "task": "runtime", + "config": config_name, + "vsrc_dir": vsrc_dir, + "build_dir": build_dir, + "missing": missing, + }, + trace_id=origin_tid, + ) + return + + # cargo global --config so OUT_PATH reaches bebop-p2e build.rs (same as runworkload). + cargo_out = f"--config=\"env.OUT_PATH='{build_dir}'\"" + + # ── Build bebop p2e (tests), linked against case libvCtb ─────────────── + build_cmd = ( + f"nix develop -c cargo {cargo_out} build --release --features p2e --tests" + ) ctx.logger.info("Building bebop p2e (tests)...") build_result = stream_run_logger( cmd=build_cmd, @@ -87,27 +166,29 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if build_result.returncode != 0: await check_result( ctx, build_result.returncode, continue_run=False, - extra_fields={"task": "build", "backend": "p2e"}, + extra_fields={"task": "build", "backend": "p2e", "build_dir": build_dir}, trace_id=origin_tid, ) return # ── Run nextest ─────────────────────────────────────────────────────── - # Pass parameters via environment variables (nextest doesn't support custom CLI args after `--`) env = os.environ.copy() env.update({ "BEBOP_WORKLOAD_TOML": workload_toml, "BEBOP_BB_TESTS_ROOT": elf_root, "BEBOP_P2E_BITSTREAM": bitstream, - "BEBOP_P2E_BUILD_DIR": build_dir, + "OUT_PATH": build_dir, }) nextest_cmd = ( - f"nix develop -c cargo nextest run --features p2e --test test_p2e " - f"--config-file \"{nextest_config}\"" + f"nix develop -c cargo {cargo_out} nextest run --release --features p2e " + f"--test test_p2e --config-file \"{nextest_config}\"" ) ctx.logger.info(f"Running bebop p2e nextest: {nextest_cmd}") - ctx.logger.info(f"Environment: BEBOP_WORKLOAD_TOML={workload_toml}, BEBOP_BB_TESTS_ROOT={elf_root}") + ctx.logger.info( + f"Environment: BEBOP_WORKLOAD_TOML={workload_toml}, " + f"BEBOP_BB_TESTS_ROOT={elf_root}, OUT_PATH={build_dir}" + ) run_result = stream_run_logger( cmd=nextest_cmd, logger=ctx.logger, @@ -126,6 +207,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "backend": "p2e", "chip": chip, "bitstream": bitstream, + "config": config_name, "build_dir": build_dir, "test_type": test_type, "nextest_config": nextest_config, diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index 1f3faff03..b11413ffd 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -29,11 +29,6 @@ "enqueues": ["bebop.verilator.sim", "bebop.verilator.run.sim"], } - -def configure_fast_dev_profile(env: dict) -> None: - env.setdefault("CARGO_INCREMENTAL", "1") - - async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -67,10 +62,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: jobs = input_data.get("jobs", 16) build_cmd = ( - f"cargo build --bin bebop --features verilator --jobs {shlex.quote(str(jobs))}" + f"cargo build --release --bin bebop --features verilator --jobs {shlex.quote(str(jobs))}" ) env = {**os.environ, "VSRC_PATH": vsrc_dir} - configure_fast_dev_profile(env) ctx.logger.info("Building bebop verilator ...") build_result = stream_run_logger( cmd=build_cmd, @@ -81,7 +75,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: env=env, ) - bebop_bin = f"{bebop_dir}/target/debug/bebop" + bebop_bin = f"{bebop_dir}/target/release/bebop" if build_result.returncode == 0: try: write_build_marker(bebop_dir, arch_config, vsrc_dir, bebop_bin) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index e20ccc865..b8807b0ab 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -1,10 +1,7 @@ """ bebop verilator event handler -Runs bebop verilator simulation: - 1. Resolve binary path - 2. Resolve verilog source directory (VSRC_PATH) - 3. Run the built bebop binary with the new run/verilator CLI +Run Verilator either through its guest-ELF simulator or the rushB native ABI. """ import json import os @@ -36,11 +33,19 @@ } +def is_native_host_elf(path: str) -> bool: + try: + with open(path, "rb") as binary: + header = binary.read(20) + except OSError: + return False + return header[:4] == b"\x7fELF" and header[18:20] == b"\x3e\x00" + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" - arch_dir = f"{bbdir}/arch" arch_config = input_data.get("config") if not isinstance(arch_config, str) or not arch_config or arch_config == "None": @@ -62,7 +67,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - bebop_bin = f"{bebop_dir}/target/debug/bebop" + bebop_bin = f"{bebop_dir}/target/release/bebop" if not os.path.isfile(bebop_bin): ctx.logger.error(f"bebop binary does not exist: {bebop_bin}") await check_result( @@ -128,10 +133,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"binary_path: {binary_path}") timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - log_dir = input_data.get("log_dir") or f"{arch_dir}/log/{timestamp}-{binary_name}" - fst_dir = input_data.get("fst_dir") or f"{arch_dir}/waveform/{timestamp}-{binary_name}" + log_dir = f"{bbdir}/log/{timestamp}-verilator-{binary_name}" os.makedirs(log_dir, exist_ok=True) - os.makedirs(fst_dir, exist_ok=True) wave_arg = " --no-wave" if input_data.get("no-wave", False) or input_data.get("no_wave", False) else "" trace_args = "" @@ -139,26 +142,46 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if input_data.get(trace_name, False): trace_args += f" --{trace_name}" - run_cmd = ( - f"{shlex.quote(bebop_bin)} run verilator " - f"--elf={shlex.quote(binary_path)} " - f"--log-dir={shlex.quote(log_dir)} " - f"--fst-dir={shlex.quote(fst_dir)}" - f"{wave_arg}" - f"{trace_args}" - ) - ctx.logger.info(f"Running bebop verilator: {run_cmd}") + if input_data.get("rushB"): + # rushB owns the Verilator instance inside libbebop_verilator. It must + # execute as a native program, never as the normal guest ELF input. + if not is_native_host_elf(binary_path): + ctx.logger.error( + f"--rushB requires a native x86_64 rushB runner; got guest ELF: {binary_path}" + ) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "rushB_requires_native_runner", "binary": binary_path}, + trace_id=origin_tid, + ) + return + backend_library = f"{bebop_dir}/target/release/deps/libbebop_verilator.so" + binary_dir = os.path.dirname(binary_path) + copy_cmd = shlex.join(["cmake", "-E", "copy_if_different", backend_library, binary_dir]) + inner_cmd = f"cd {shlex.quote(binary_dir)} && {copy_cmd} && exec {shlex.quote(binary_path)}" + run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" + ctx.logger.info(f"Running rushB Verilator: {run_cmd}") + stdout_prefix = "rushB verilator" + stderr_prefix = "rushB verilator" + else: + run_cmd = ( + f"{shlex.quote(bebop_bin)} run verilator " + f"--elf={shlex.quote(binary_path)} " + f"--log-dir={shlex.quote(log_dir)}" + f"{wave_arg}" + f"{trace_args}" + ) + ctx.logger.info(f"Running bebop verilator: {run_cmd}") + stdout_prefix = "bebop verilator" + stderr_prefix = "bebop verilator" run_result = stream_run_logger( cmd=run_cmd, logger=ctx.logger, cwd=bebop_dir, - stdout_prefix="bebop verilator", - stderr_prefix="bebop verilator", - env={ - **os.environ, - "ARCH_CONFIG": arch_config, - "VSRC_PATH": vsrc_dir, - }, + stdout_prefix=stdout_prefix, + stderr_prefix=stderr_prefix, ) await check_result( @@ -167,10 +190,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "verilator", + "backend": "rushB" if input_data.get("rushB") else "guest-elf", "binary": binary_path, "config": arch_config, "log_dir": log_dir, - "fst_dir": fst_dir, "timestamp": timestamp, }, trace_id=origin_tid, diff --git a/bbdev/api/steps/bebop/verilator/05_run_api.step.py b/bbdev/api/steps/bebop/verilator/05_run_api.step.py index 7915ef78c..772a906ef 100644 --- a/bbdev/api/steps/bebop/verilator/05_run_api.step.py +++ b/bbdev/api/steps/bebop/verilator/05_run_api.step.py @@ -46,8 +46,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "pmctrace": body.get("pmctrace", False), "ctrace": body.get("ctrace", False), "banktrace": body.get("banktrace", False), - "log_dir": body.get("log_dir"), - "fst_dir": body.get("fst_dir"), + "rushB": body.get("rushB", False), "jobs": body.get("jobs", 16), "from_run_workflow": True, } diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 25c8b0325..f1d75edfe 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -94,7 +94,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "VSRC_PATH": vsrc_dir, }) nextest_cmd = ( - f"nix develop -c cargo nextest run --manifest-path {shlex.quote(f'{bebop_dir}/Cargo.toml')} " + f"nix develop -c cargo nextest run --release --manifest-path {shlex.quote(f'{bebop_dir}/Cargo.toml')} " "--features verilator --test test_verilator " f"--config-file {shlex.quote(nextest_config)} " f"--config={vsrc_config}" diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index 2bb287595..1848bd178 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -40,7 +40,14 @@ def hart_count_params(input_data: dict) -> dict: - allowed = {"visible-hart-count", "total-hart-count", "model", "_trace_id"} + allowed = { + "visible-hart-count", + "total-hart-count", + "model", + "chip", + "interactive", + "_trace_id", + } unknown = sorted(k for k in input_data if k not in allowed) if unknown: raise ValueError(f"unknown kernel build parameter(s): {', '.join(unknown)}") @@ -64,6 +71,13 @@ def hart_count_params(input_data: dict) -> dict: } +def kernel_interactive(input_data: dict) -> bool: + interactive = input_data.get("interactive", False) + if not isinstance(interactive, bool): + raise ValueError("interactive must be a boolean flag") + return interactive + + def kernel_model(input_data: dict) -> str: model = input_data.get("model", "") if model in ("", None): @@ -78,22 +92,52 @@ def kernel_model(input_data: dict) -> str: return model -def kernel_build_dir(bbdir: str, hart_params: dict, model: str = "") -> str: +def kernel_chip(input_data: dict, bbdir: str) -> str: + chip = input_data.get("chip", "") + if chip in ("", None): + return "" + if not isinstance(chip, str): + raise ValueError("chip must be a string") + if os.path.sep in chip or chip in {".", ".."}: + raise ValueError(f"invalid chip: {chip}") + + chip_kernel = os.path.join(bbdir, "examples", "chips", chip, "kernel") + chip_init = os.path.join(chip_kernel, "overlay", "init") + if not os.path.isfile(chip_init): + raise ValueError(f"chip OS overlay init not found: {chip_init}") + return chip + + +def kernel_build_dir( + bbdir: str, + hart_params: dict, + model: str = "", + chip: str = "", + interactive: bool = False, +) -> str: visible = hart_params["visible"] total = hart_params["total"] - model_suffix = f"-model-{model}" if model else "" + suffix = "" + if chip: + suffix += f"-chip-{chip}" + if model: + suffix += f"-model-{model}" + if interactive: + suffix += "-interactive" if visible == 64 and total == 64: - return os.path.join(bbdir, "bb-tests", "build", f"kernel{model_suffix}") - return os.path.join(bbdir, "bb-tests", "build", f"kernel-v{visible}-t{total}{model_suffix}") + return os.path.join(bbdir, "bb-tests", "build", f"kernel{suffix}") + return os.path.join(bbdir, "bb-tests", "build", f"kernel-v{visible}-t{total}{suffix}") -def fw_payload_name(hart_params: dict, model: str = "") -> str: +def fw_payload_name(hart_params: dict, model: str = "", chip: str = "") -> str: visible = hart_params["visible"] total = hart_params["total"] name = "fw_payload" if visible != 64 or total != 64: name = f"{name}-v{visible}-t{total}" - if model: + if chip: + name = f"{name}-{chip}-pk" + elif model: name = f"{name}-{model}" return name @@ -110,11 +154,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: try: hart_params = hart_count_params(input_data) model = kernel_model(input_data) + chip = kernel_chip(input_data, bbdir) + interactive = kernel_interactive(input_data) + if chip and model: + raise ValueError("chip and model cannot be set together") except ValueError as e: ctx.logger.error(str(e)) await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) return - kernel_build = kernel_build_dir(bbdir, hart_params, model) + kernel_build = kernel_build_dir( + bbdir, hart_params, model, chip, interactive=interactive + ) + interactive_arg = "ON" if interactive else "OFF" # cmake configure configure_cmd = ( @@ -122,8 +173,24 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"-DBUCKYBALL_VISIBLE_HART_COUNT={hart_params['visible']} " f"-DBUCKYBALL_TOTAL_HART_COUNT={hart_params['total']} " f"-DBUCKYBALL_HIDDEN_HART_BASE={hart_params['hidden_base']} " - f"-DBUCKYBALL_KERNEL_MODEL={model}" + f"-DBUCKYBALL_KERNEL_MODEL={model} " + f"-DBUCKYBALL_KERNEL_CHIP={chip} " + f"-DBUCKYBALL_KERNEL_INTERACTIVE={interactive_arg}" ) + if chip: + chip_kernel = os.path.join(bbdir, "examples", "chips", chip, "kernel") + pk_toml = os.path.join( + bbdir, "examples", "chips", chip, "regression", "batch", "bemu", "workloads-pk.toml" + ) + if not os.path.isfile(pk_toml): + ctx.logger.error(f"chip pk workload toml not found: {pk_toml}") + await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) + return + configure_cmd += ( + f" -DBUCKYBALL_CHIP_KERNEL_DIR={chip_kernel}" + f" -DBUCKYBALL_PK_WORKLOAD_TOML={pk_toml}" + ) + result = stream_run_logger( cmd=configure_cmd, logger=ctx.logger, @@ -148,7 +215,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return # Convert fw_payload.bin to hex for P2E memory backdoor - payload_name = fw_payload_name(hart_params, model) + payload_name = fw_payload_name(hart_params, model, chip) fw_payload_bin = os.path.join(output_dir, f"{payload_name}.bin") fw_payload_hex = os.path.join(output_dir, f"{payload_name}.hex") diff --git a/bbdev/api/steps/workload/01_buidl_api.step.py b/bbdev/api/steps/workload/01_buidl_api.step.py index cbbe3c631..4e697a02b 100644 --- a/bbdev/api/steps/workload/01_buidl_api.step.py +++ b/bbdev/api/steps/workload/01_buidl_api.step.py @@ -22,7 +22,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - allowed = {"chip", "model", "stable"} + allowed = {"chip", "model", "stable", "rushB"} unknown = sorted(k for k in body if k not in allowed) if unknown: return ApiResponse( @@ -52,10 +52,17 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: status=400, body={"error": "Invalid parameter: stable must be a boolean flag"}, ) + rushb_backend = body.get("rushB") + if rushb_backend is not None and rushb_backend not in {"bemu", "verilator"}: + return ApiResponse( + status=400, + body={"error": "Invalid --rushB: expected bemu or verilator"}, + ) data = { "chip": chip, "model": body.get("model", ""), "stable": stable, + "rushB": rushb_backend, } await ctx.enqueue({"topic": "workload.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index 2ee37b6d0..36aab1e37 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -23,13 +23,79 @@ "enqueues": [], } +MODEL_TARGETS = { + "lenet": "buddy-buckyball-lenet-run", + "mobilenet": "buddy-buckyball-mobilenetv3-run", + "resnet": "buddy-buckyball-resnet-run", + "yolo": "buddy-buckyball-yolo26-run", + "bert": "buddy-buckyball-bert-run", + "qwen3": "buddy-buckyball-qwen3-run", + "gemma4": "buddy-buckyball-gemma4-run", + "deepseekr1": "buddy-buckyball-deepseekr1-run", + "llama2": "buddy-buckyball-llama2-run", + "stable-diffusion": "buddy-buckyball-stable-diffusion-run", + "whisper": "buddy-buckyball-whisper-run", + "buddynext": "buddy-buckyball-buddynext-all-run", +} + +RUSHB_TARGETS = { + "lenet": { + "bemu": "buddy-buckyball-lenet-rushB-bemu-run", + "verilator": "buddy-buckyball-lenet-rushB-verilator-run", + }, + "mobilenet": { + "bemu": "buddy-buckyball-mobilenetv3-rushB-bemu-run", + "verilator": "buddy-buckyball-mobilenetv3-rushB-verilator-run", + }, + "resnet": { + "bemu": "buddy-buckyball-resnet-rushB-bemu-run", + "verilator": "buddy-buckyball-resnet-rushB-verilator-run", + }, + "yolo": { + "bemu": "buddy-buckyball-yolo26-rushB-bemu-run", + "verilator": "buddy-buckyball-yolo26-rushB-verilator-run", + }, + "bert": { + "bemu": "buddy-buckyball-bert-rushB-bemu-run", + "verilator": "buddy-buckyball-bert-rushB-verilator-run", + }, + "qwen3": { + "bemu": "buddy-buckyball-qwen3-rushB-bemu-run", + "verilator": "buddy-buckyball-qwen3-rushB-verilator-run", + }, + "gemma4": { + "bemu": "buddy-buckyball-gemma4-rushB-bemu-run", + "verilator": "buddy-buckyball-gemma4-rushB-verilator-run", + }, + "deepseekr1": { + "bemu": "buddy-buckyball-deepseekr1-rushB-bemu-run", + "verilator": "buddy-buckyball-deepseekr1-rushB-verilator-run", + }, + "llama2": { + "bemu": "buddy-buckyball-llama2-rushB-bemu-run", + "verilator": "buddy-buckyball-llama2-rushB-verilator-run", + }, + "stable-diffusion": { + "bemu": "buddy-buckyball-stable-diffusion-rushB-bemu-run", + "verilator": "buddy-buckyball-stable-diffusion-rushB-verilator-run", + }, + "whisper": { + "bemu": "buddy-buckyball-whisper-rushB-bemu-run", + "verilator": "buddy-buckyball-whisper-rushB-verilator-run", + }, + "buddynext": { + "bemu": "buddy-buckyball-buddynext-rushB-bemu-run", + "verilator": "buddy-buckyball-buddynext-rushB-verilator-run", + }, +} + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() workload_dir = f"{bbdir}/bb-tests" build_dir = f"{workload_dir}/build" - allowed = {"chip", "model", "stable", "_trace_id"} + allowed = {"chip", "model", "stable", "rushB", "_trace_id"} unknown = sorted(k for k in input_data if k not in allowed) if unknown: ctx.logger.error(f"Unknown workload build parameter(s): {', '.join(unknown)}") @@ -67,6 +133,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return model = input_data.get("model", "") stable = input_data.get("stable", False) + rushb_backend = input_data.get("rushB") if not isinstance(stable, bool): ctx.logger.error("Invalid parameter: stable must be a boolean flag") @@ -77,23 +144,19 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - model_targets = { - "lenet": "buddy-buckyball-lenet-run", - "mobilenet": "buddy-buckyball-mobilenetv3-run", - "resnet": "buddy-buckyball-resnet-run", - "yolo": "buddy-buckyball-yolo26-run", - "bert": "buddy-buckyball-bert-run", - "qwen3": "buddy-buckyball-qwen3-run", - "gemma4": "buddy-buckyball-gemma4-run", - "deepseekr1": "buddy-buckyball-deepseekr1-run", - "llama2": "buddy-buckyball-llama2-run", - "stable-diffusion": "buddy-buckyball-stable-diffusion-run", - "whisper": "buddy-buckyball-whisper-run", - } + if rushb_backend is not None and rushb_backend not in {"bemu", "verilator"}: + ctx.logger.error("Invalid rushB backend: expected bemu or verilator") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_rushB", "rushB": rushb_backend}, + trace_id=origin_tid, + ) + return + target = "" if model: model_key = model.lower() - target = model_targets.get(model_key) + target = MODEL_TARGETS.get(model_key) if target is None: ctx.logger.error(f"Unknown model: {model}") await check_result( @@ -102,6 +165,22 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + if rushb_backend is not None: + target = RUSHB_TARGETS.get(model_key, {}).get(rushb_backend) + if target is None: + ctx.logger.error(f"Missing rushB target registration for model {model}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={ + "error": "missing_rushB_target", + "model": model, + "rushB": rushb_backend, + }, + trace_id=origin_tid, + ) + return + elif rushb_backend is not None: + target = f"rushB-{rushb_backend}-workloads-build" os.makedirs(build_dir, exist_ok=True) @@ -110,7 +189,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ninja_target = f" {shlex.quote(target)}" if target else "" inner = ( f"cd {shlex.quote(build_dir)} && " - f"cmake -G Ninja {chip_arg} {stable_arg} .. && " + f"cmake -G Ninja {chip_arg} {stable_arg} " + f"-DPython3_EXECUTABLE=\"$(which python3)\" .. && " f"ninja -j{os.cpu_count()}{ninja_target}" ) command = f"cd {shlex.quote(bbdir)} && nix develop -c bash -c {shlex.quote(inner)}" @@ -131,7 +211,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ================================================================================== # This is the end of run workflow, status no longer set to processing success_result, failure_result = await check_result( - ctx, result.returncode, continue_run=False, trace_id=origin_tid) + ctx, result.returncode, continue_run=False, + extra_fields={"chip": chip, "model": model, "rushB": rushb_backend}, + trace_id=origin_tid) # ================================================================================== # finish workflow diff --git a/bbdev/api/steps/workload/README.md b/bbdev/api/steps/workload/README.md index d0f4ab69f..e6671d788 100644 --- a/bbdev/api/steps/workload/README.md +++ b/bbdev/api/steps/workload/README.md @@ -25,6 +25,8 @@ bbdev workload --clean - **`chip`** - Required chip name. Selects chip-specific workloads. - **`model`** - Optional model name to build. If omitted, build all workloads. - **`stable`** - Optional boolean flag. If set, build with stable LLVM Buckyball extensions. +- **`rushB`** - Optional host-native backend: `bemu` or `verilator`. Requires + `model`. It builds host-native CPU and rushB-lowered accelerator objects. For chip workloads under paths like `*/chips/`, only the directory selected by `chip` is synced to `bb-tests/output/workloads`. @@ -36,6 +38,12 @@ bbdev workload --build "--chip toy --model lenet" # Build one model with stable LLVM Buckyball extensions bbdev workload --build "--chip toy --model lenet --stable" +# Build a model's rushB BEMU runner +bbdev workload --build "--chip pebble --model lenet --rushB bemu" + +# Build a model's rushB Verilator runner +bbdev workload --build "--chip pebble --model lenet --rushB verilator" + # Build all workloads bbdev workload --build "--chip toy" ``` diff --git a/bbdev/api/tests/test_api_bebop_p2e_batch.py b/bbdev/api/tests/test_api_bebop_p2e_batch.py index 4781f3010..fc1952f53 100644 --- a/bbdev/api/tests/test_api_bebop_p2e_batch.py +++ b/bbdev/api/tests/test_api_bebop_p2e_batch.py @@ -1,4 +1,4 @@ from _api_test_helper import run_bbdev_case -# You need to set the bitstream and build-dir paths. -# run_bbdev_case("bbdev bebop-p2e --batch '--bitstream /path/to/bitstream.bit --build-dir /path/to/build --test elf-tests'") +# You need to set the bitstream path (case dir is derived from it). +# run_bbdev_case("bbdev bebop-p2e --batch '--chip pebble --bitstream /path/to/bitstream.bit --test elf-tests'") diff --git a/bbdev/api/utils/port.py b/bbdev/api/utils/port.py index 05afc3fbe..431e9b9d2 100644 --- a/bbdev/api/utils/port.py +++ b/bbdev/api/utils/port.py @@ -35,13 +35,14 @@ def reserve_port( ) -> tuple[int, socket.socket]: """Reserve a port by binding and holding the socket open. + Do not set SO_REUSEADDR: with it, two processes can both bind the same + port, then both fail when iii tries to listen. Without it, bind is exclusive. + The caller must close the returned socket once the engine has taken over the port. - This prevents TOCTOU races when multiple bbdev instances start concurrently. """ for port in _port_order(start_port, end_port, preferred_port): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind((HOST, port)) return port, sock except OSError: diff --git a/bbdev/bbdev b/bbdev/bbdev index 3d7ecb0e4..eaac0485b 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -43,11 +43,11 @@ WORKFLOW_COMMANDS = { }, "workload": { "clean": "Clean workload output directory.", - "build": 'Build workload. Args: "--chip [--model ] [--stable]"', + "build": 'Build workload. Args: "--chip [--model ] [--stable] [--rushB ]"; without --model, --rushB builds CTest and MLIR rushB runners.', "tohex": "Convert all -baremetal ELF files under bb-tests/output/workloads/src to hex.", }, "kernel": { - "build": 'Build RISC-V kernel + rootfs for Pegasus. Args: "[--visible-hart-count ] [--total-hart-count ] [--model ]"', + "build": 'Build RISC-V kernel + rootfs. Args: "[--chip ] [--model ] [--interactive] [--visible-hart-count ] [--total-hart-count ]"', }, "firesim": { "enumeratefpgas": "Enumerate FPGAs.", @@ -59,23 +59,23 @@ WORKFLOW_COMMANDS = { "build": 'Build compiler. Args: "--chip [--stable]"', }, "bebop-bemu": { - "sim": 'Run Bebop emulator simulation. Args: "--chip --binary [--pk] [--log-dir ]"', + "sim": 'Run Bebop emulator simulation. Args: "--chip --binary [--rushB] [--pk] [--disasm] [--tool-profile] [--log-dir ]"', "batch": 'Run Bebop bemu nextest batch regression (parallel). Args: "--chip --test [--clean-before]"', }, "bebop-verilator": { "clean": 'Clean Bebop Verilator build directory. Args: "--config [--output-dir ]"', "verilog": 'Generate Bebop Verilog files. Args: "--config [--output-dir ]"', "build": 'Build Bebop Verilator simulation executable. Args: "--config [--jobs ] [--output-dir ]"', - "sim": 'Run Bebop Verilator simulation. Args: "--config --binary [--batch] [--log-dir ] [--fst-dir ] [--no-wave]"', - "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', + "sim": 'Run Bebop Verilator simulation. Args: "--config --binary [--rushB] [--batch] [--log-dir ] [--fst-dir ] [--no-wave]"', + "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--rushB] [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--clean-before]"', }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--config [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--config [--output-dir ]"', "buildbitstream": 'Build Bebop P2E runtime case. Args: "--config [--vsrc-dir ] [--output-dir ]"', - "runworkload": 'Run Bebop P2E workload. Args: "--image --bitstream [--multi-fpga] [--wave] [--wave-start ] [--build-dir ]"', - "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--chip --bitstream --build-dir --test "', + "runworkload": 'Run Bebop P2E workload. Args: "--image --bitstream [--multi-fpga] [--wave] [--wave-start ] [--itrace] [--mtrace] [--pmctrace]"', + "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--chip --bitstream --test "', }, "dc": { "verilog": 'Generate DC RTL. Args: "--config [--dir ]"', diff --git a/bbdev/flake.lock b/bbdev/flake.lock index 3009a1201..a57f1a193 100644 --- a/bbdev/flake.lock +++ b/bbdev/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1770841267, - "narHash": "sha256-9xejG0KoqsoKEGp2kVbXRlEYtFFcDTHjidiuX8hGO44=", + "lastModified": 1785454630, + "narHash": "sha256-LQy14TZp77TwbQf40gg1V3jo8FwJG0jGDkAH+zRHqg8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ec7c70d12ce2fc37cb92aff673dcdca89d187bae", + "rev": "1559d3daa3ecc813a650b79375ea61b6741b8746", "type": "github" }, "original": { diff --git a/bbdev/mcp/tools/bebop_p2e_batch.py b/bbdev/mcp/tools/bebop_p2e_batch.py index 885dc4f2e..d306c61f3 100644 --- a/bbdev/mcp/tools/bebop_p2e_batch.py +++ b/bbdev/mcp/tools/bebop_p2e_batch.py @@ -3,7 +3,7 @@ from __future__ import annotations -from common import submit, err, fmt, need, opt +from common import submit, err, fmt, need def register(mcp): @@ -11,15 +11,10 @@ def register(mcp): def bbdev_bebop_p2e_batch( chip: str, bitstream: str, - build_dir: str, test: str, ) -> str: """Batch bebop-p2e regression. test: elf-tests|pk-tests. POST /bebop/p2e/batch.""" - for n, v in ( - ("chip", chip), - ("bitstream", bitstream), - ("build_dir", build_dir), - ): + for n, v in (("chip", chip), ("bitstream", bitstream)): if e := need(n, v): return err(e) if test not in ("elf-tests", "pk-tests"): @@ -30,9 +25,7 @@ def bbdev_bebop_p2e_batch( { "chip": chip, "bitstream": bitstream, - "build-dir": build_dir, "test": test, }, ) ) - diff --git a/bbdev/mcp/tools/bebop_p2e_runworkload.py b/bbdev/mcp/tools/bebop_p2e_runworkload.py index cad65fcc4..e658f949e 100644 --- a/bbdev/mcp/tools/bebop_p2e_runworkload.py +++ b/bbdev/mcp/tools/bebop_p2e_runworkload.py @@ -2,21 +2,30 @@ from __future__ import annotations +from typing import Any, Dict -from common import submit, err, fmt, need, opt +from common import submit, err, fmt, need def register(mcp): @mcp.tool() - def bbdev_bebop_p2e_runworkload(image: str, bitstream: str) -> str: + def bbdev_bebop_p2e_runworkload( + image: str, + bitstream: str, + itrace: bool = False, + mtrace: bool = False, + pmctrace: bool = False, + ) -> str: """Run one image on bebop-p2e FPGA. POST /bebop/p2e/runworkload.""" for n, v in (("image", image), ("bitstream", bitstream)): if e := need(n, v): return err(e) - return fmt( - submit( - "/bebop/p2e/runworkload", - {"image": image, "bitstream": bitstream}, - ) - ) + params: Dict[str, Any] = {"image": image, "bitstream": bitstream} + if itrace: + params["itrace"] = True + if mtrace: + params["mtrace"] = True + if pmctrace: + params["pmctrace"] = True + return fmt(submit("/bebop/p2e/runworkload", params)) diff --git a/bbdev/mcp/tools/bebop_verilator_run.py b/bbdev/mcp/tools/bebop_verilator_run.py index 7805207d8..35a7ae9a0 100644 --- a/bbdev/mcp/tools/bebop_verilator_run.py +++ b/bbdev/mcp/tools/bebop_verilator_run.py @@ -18,6 +18,7 @@ def bbdev_bebop_verilator_run( ctrace: bool = False, banktrace: bool = False, no_wave: bool = False, + rushB: bool = False, ) -> str: """Full bebop-verilator flow. POST /bebop/verilator/run.""" for n, v in (("binary", binary), ("config", config)): @@ -36,7 +37,7 @@ def bbdev_bebop_verilator_run( "ctrace": ctrace, "banktrace": banktrace, "no-wave": no_wave, + "rushB": rushB, }, ) ) - diff --git a/bbdev/mcp/tools/bebop_verilator_sim.py b/bbdev/mcp/tools/bebop_verilator_sim.py index eb8daa3e1..dc8cdb375 100644 --- a/bbdev/mcp/tools/bebop_verilator_sim.py +++ b/bbdev/mcp/tools/bebop_verilator_sim.py @@ -2,9 +2,9 @@ from __future__ import annotations -from typing import Any, Dict, Optional +from typing import Any, Dict -from common import submit, err, fmt, need, opt +from common import submit, err, fmt, need def register(mcp): @@ -18,8 +18,7 @@ def bbdev_bebop_verilator_sim( ctrace: bool = False, banktrace: bool = False, no_wave: bool = False, - log_dir: Optional[str] = None, - fst_dir: Optional[str] = None, + rushB: bool = False, ) -> str: """Run one workload on bebop-verilator. POST /bebop/verilator/sim.""" for n, v in (("binary", binary), ("config", config)): @@ -34,10 +33,6 @@ def bbdev_bebop_verilator_sim( "ctrace": ctrace, "banktrace": banktrace, "no-wave": no_wave, + "rushB": rushB, } - if log_dir: - params["log_dir"] = log_dir - if fst_dir: - params["fst_dir"] = fst_dir return fmt(submit("/bebop/verilator/sim", params)) - diff --git a/bbdev/mcp/tools/bemu_sim.py b/bbdev/mcp/tools/bemu_sim.py index 5381e1ff6..79f061cfd 100644 --- a/bbdev/mcp/tools/bemu_sim.py +++ b/bbdev/mcp/tools/bemu_sim.py @@ -2,9 +2,9 @@ from __future__ import annotations -from typing import Any, Dict, Optional +from typing import Any, Dict -from common import submit, err, fmt, need, opt +from common import submit, err, fmt, need def register(mcp): @@ -13,14 +13,20 @@ def bbdev_bemu_sim( chip: str, binary: str, pk: bool = False, - log_dir: Optional[str] = None, + disasm: bool = False, + tool_profile: bool = False, + rushB: bool = False, ) -> str: """Run one workload on bebop-bemu. POST /bebop/bemu/sim.""" for n, v in (("chip", chip), ("binary", binary)): if e := need(n, v): return err(e) - params: Dict[str, Any] = {"chip": chip, "binary": binary, "pk": pk} - if log_dir: - params["log_dir"] = log_dir + params: Dict[str, Any] = { + "chip": chip, + "binary": binary, + "pk": pk, + "disasm": disasm, + "tool-profile": tool_profile, + "rushB": rushB, + } return fmt(submit("/bebop/bemu/sim", params)) - diff --git a/bbdev/mcp/tools/kernel_build.py b/bbdev/mcp/tools/kernel_build.py index ffffa5883..1c5cb2126 100644 --- a/bbdev/mcp/tools/kernel_build.py +++ b/bbdev/mcp/tools/kernel_build.py @@ -10,13 +10,22 @@ def register(mcp): @mcp.tool() def bbdev_kernel_build( + chip: Optional[str] = None, model: Optional[str] = None, + interactive: bool = False, visible_hart_count: Optional[int] = None, total_hart_count: Optional[int] = None, ) -> str: - """Build RISC-V kernel + rootfs. POST /kernel/build.""" + """Build RISC-V kernel + rootfs. POST /kernel/build. + + With --chip, uses examples/chips//kernel OS overlay and packs + that chip's bemu workloads-pk.toml into fw_payload--pk. + With --interactive, keep shared /init shell and do not auto-run. + """ params: Dict[str, Any] = {} - opt(params, model=model) + opt(params, chip=chip, model=model) + if interactive: + params["interactive"] = True if visible_hart_count is not None: params["visible-hart-count"] = visible_hart_count if total_hart_count is not None: diff --git a/bbdev/mcp/tools/workload_build.py b/bbdev/mcp/tools/workload_build.py index 5e3d6762a..8148ae19a 100644 --- a/bbdev/mcp/tools/workload_build.py +++ b/bbdev/mcp/tools/workload_build.py @@ -9,12 +9,19 @@ def register(mcp): @mcp.tool() - def bbdev_workload_build(chip: str, model: Optional[str] = None) -> str: + def bbdev_workload_build( + chip: str, + model: Optional[str] = None, + rushB: Optional[str] = None, + ) -> str: """Build workloads for a chip. POST /workload/build.""" if e := need("chip", chip): return err(e) params: Dict[str, Any] = {"chip": chip} if model: params["model"] = model + if rushB: + if rushB not in {"bemu", "verilator"}: + return err("rushB must be bemu or verilator") + params["rushB"] = rushB return fmt(submit("/workload/build", params)) - From fcbc9aaf62aa866875a07bd773d0a7854c829447 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 9 Aug 2026 22:11:16 +0800 Subject: [PATCH 105/196] feat: add rushB parameter support across bebop and verilator batch processes for enhanced regression capabilities --- bbdev/api/steps/bebop/bemu/03_batch_api.step.py | 1 + bbdev/api/steps/bebop/bemu/03_batch_event.step.py | 12 +++++++++--- bbdev/api/steps/bebop/regression.py | 4 +++- .../api/steps/bebop/verilator/06_batch_api.step.py | 1 + .../steps/bebop/verilator/06_batch_event.step.py | 12 +++++++++--- bbdev/bbdev | 4 ++-- bbdev/mcp/tools/bebop_verilator_batch.py | 5 +++-- bbdev/mcp/tools/bemu_batch.py | 14 ++++++++++---- 8 files changed, 38 insertions(+), 15 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py index 8699fcdf7..8dc416cb5 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -53,6 +53,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "chip": chip, "test": test_type, "clean-before": body.get("clean-before", body.get("clean_before", False)), + "rushB": bool(body.get("rushB", False)), "_trace_id": ctx.trace_id, } }) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index 96ce5ed43..f98628764 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -65,8 +65,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return test_type = input_data.get("test", "elf-tests") + rushB = bool(input_data.get("rushB", False)) try: - workload_toml = regression_workload_toml(chip, "bemu", test_type, bbdir) + workload_toml = regression_workload_toml( + chip, "bemu", test_type, bbdir, rushB=rushB + ) except ValueError as e: ctx.logger.error(str(e)) await check_result( @@ -76,7 +79,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - ctx.logger.info(f"Running {test_type} with workload config: {workload_toml}") + ctx.logger.info( + f"Running {test_type} with workload config: {workload_toml} rushB={rushB}" + ) # ── Build bebop bemu ────────────────────────────────────────────────── build_cmd = ( @@ -134,9 +139,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "batch", - "backend": "bemu", + "backend": "bemu-rushB" if rushB else "bemu", "chip": chip, "test_type": test_type, + "rushB": rushB, "nextest_config": nextest_config, "workload_toml": workload_toml, }, diff --git a/bbdev/api/steps/bebop/regression.py b/bbdev/api/steps/bebop/regression.py index 36f62b844..9fa8fa101 100644 --- a/bbdev/api/steps/bebop/regression.py +++ b/bbdev/api/steps/bebop/regression.py @@ -25,6 +25,7 @@ def regression_workload_toml( backend: str, test_type: str, bbdir: str | None = None, + rushB: bool = False, ) -> str: if test_type == "elf-tests": suffix = "elf" @@ -33,7 +34,8 @@ def regression_workload_toml( else: raise ValueError(f"invalid test type: {test_type}") - toml = chip_regression_dir(chip, backend, bbdir) / f"workloads-{suffix}.toml" + name = f"workloads-{suffix}-rushB.toml" if rushB else f"workloads-{suffix}.toml" + toml = chip_regression_dir(chip, backend, bbdir) / name if not toml.is_file(): raise ValueError(f"regression workload toml does not exist: {toml}") return str(toml) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py index 65740f5dd..2ae5dbdd0 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -50,6 +50,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "vsrc_dir": vsrc_dir, "test": test_type, "clean-before": body.get("clean-before", body.get("clean_before", False)), + "rushB": bool(body.get("rushB", False)), } await ctx.enqueue({"topic": "bebop.verilator.batch", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index f1d75edfe..4150063aa 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -59,8 +59,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return test_type = input_data.get("test", "elf-tests") + rushB = bool(input_data.get("rushB", False)) try: - workload_toml = regression_workload_toml(chip, "verilator", test_type, bbdir) + workload_toml = regression_workload_toml( + chip, "verilator", test_type, bbdir, rushB=rushB + ) except ValueError as e: ctx.logger.error(str(e)) await check_result( @@ -70,7 +73,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - ctx.logger.info(f"Running {test_type} with workload config: {workload_toml}") + ctx.logger.info( + f"Running {test_type} with workload config: {workload_toml} rushB={rushB}" + ) vsrc_dir = get_verilator_build_dir(bbdir, arch_config, input_data.get("vsrc_dir")) if not os.path.isdir(vsrc_dir): @@ -117,11 +122,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "batch", - "backend": "verilator", + "backend": "verilator-rushB" if rushB else "verilator", "chip": chip, "config": arch_config, "vsrc_dir": vsrc_dir, "test_type": test_type, + "rushB": rushB, "nextest_config": nextest_config, "workload_toml": workload_toml, }, diff --git a/bbdev/bbdev b/bbdev/bbdev index eaac0485b..b7ca6ce7c 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -60,7 +60,7 @@ WORKFLOW_COMMANDS = { }, "bebop-bemu": { "sim": 'Run Bebop emulator simulation. Args: "--chip --binary [--rushB] [--pk] [--disasm] [--tool-profile] [--log-dir ]"', - "batch": 'Run Bebop bemu nextest batch regression (parallel). Args: "--chip --test [--clean-before]"', + "batch": 'Run Bebop bemu nextest batch regression (parallel). Args: "--chip --test [--rushB] [--clean-before]"', }, "bebop-verilator": { "clean": 'Clean Bebop Verilator build directory. Args: "--config [--output-dir ]"', @@ -68,7 +68,7 @@ WORKFLOW_COMMANDS = { "build": 'Build Bebop Verilator simulation executable. Args: "--config [--jobs ] [--output-dir ]"', "sim": 'Run Bebop Verilator simulation. Args: "--config --binary [--rushB] [--batch] [--log-dir ] [--fst-dir ] [--no-wave]"', "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--rushB] [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', - "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--clean-before]"', + "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--rushB] [--clean-before]"', }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--config [--output-dir ]"', diff --git a/bbdev/mcp/tools/bebop_verilator_batch.py b/bbdev/mcp/tools/bebop_verilator_batch.py index 4588a1f8f..584cb11a6 100644 --- a/bbdev/mcp/tools/bebop_verilator_batch.py +++ b/bbdev/mcp/tools/bebop_verilator_batch.py @@ -3,7 +3,7 @@ from __future__ import annotations -from common import submit, err, fmt, need, opt +from common import submit, err, fmt, need def register(mcp): @@ -13,6 +13,7 @@ def bbdev_bebop_verilator_batch( config: str, test: str, clean_before: bool = False, + rushB: bool = False, ) -> str: """Batch bebop-verilator regression. POST /bebop/verilator/batch.""" for n, v in (("chip", chip), ("config", config)): @@ -28,7 +29,7 @@ def bbdev_bebop_verilator_batch( "config": config, "test": test, "clean-before": clean_before, + "rushB": rushB, }, ) ) - diff --git a/bbdev/mcp/tools/bemu_batch.py b/bbdev/mcp/tools/bemu_batch.py index df17d92a0..65509578d 100644 --- a/bbdev/mcp/tools/bemu_batch.py +++ b/bbdev/mcp/tools/bemu_batch.py @@ -3,12 +3,14 @@ from __future__ import annotations -from common import submit, err, fmt, need, opt +from common import submit, err, fmt, need def register(mcp): @mcp.tool() - def bbdev_bemu_batch(chip: str, test: str, clean_before: bool = False) -> str: + def bbdev_bemu_batch( + chip: str, test: str, clean_before: bool = False, rushB: bool = False + ) -> str: """Batch bemu regression. test: elf-tests|pk-tests. POST /bebop/bemu/batch.""" if e := need("chip", chip): return err(e) @@ -17,7 +19,11 @@ def bbdev_bemu_batch(chip: str, test: str, clean_before: bool = False) -> str: return fmt( submit( "/bebop/bemu/batch", - {"chip": chip, "test": test, "clean-before": clean_before}, + { + "chip": chip, + "test": test, + "clean-before": clean_before, + "rushB": rushB, + }, ) ) - From 1fcc155a3504c9fc0f4047c60a52a1f4019c675f Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 10 Aug 2026 01:03:04 +0800 Subject: [PATCH 106/196] feat: enhance compiler API to support core and chip parameters, improving error handling and validation for compiler targets --- bbdev/api/steps/compiler/01_build_api.step.py | 33 +++--- .../api/steps/compiler/01_build_event.step.py | 56 +++++----- bbdev/api/steps/compiler/README.md | 4 +- bbdev/api/steps/workload/01_buidl_api.step.py | 5 + .../api/steps/workload/01_build_event.step.py | 14 ++- bbdev/api/utils/chip.py | 100 ++++++++++++++++++ bbdev/mcp/common.py | 31 ++++-- bbdev/mcp/tools/validate.py | 3 +- 8 files changed, 190 insertions(+), 56 deletions(-) create mode 100644 bbdev/api/utils/chip.py diff --git a/bbdev/api/steps/compiler/01_build_api.step.py b/bbdev/api/steps/compiler/01_build_api.step.py index 851b62039..465ced4d9 100644 --- a/bbdev/api/steps/compiler/01_build_api.step.py +++ b/bbdev/api/steps/compiler/01_build_api.step.py @@ -1,7 +1,6 @@ import os import re import sys -from pathlib import Path from motia import ApiRequest, ApiResponse, FlowContext, api @@ -9,6 +8,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) +from utils.chip import available_compiler_chips, available_cores, resolve_chip_compiler_core, resolve_core from utils.path import get_buckyball_path config = { @@ -20,15 +20,6 @@ } -def available_chips() -> list[str]: - chips_dir = Path(get_buckyball_path()) / "examples" / "chips" - return sorted( - path.name - for path in chips_dir.iterdir() - if (path / "compiler" / "CMakeLists.txt").is_file() - ) - - async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} @@ -40,24 +31,30 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: ) chip = body.get("chip") - if not chip: + core = body.get("core") + if bool(chip) == bool(core): return ApiResponse( status=400, - body={"error": "Missing required parameter: --chip must be specified"}, + body={"error": "Specify exactly one compiler target: --core or --chip"}, ) - if not re.fullmatch(r"[A-Za-z0-9_-]+", chip): + target_name = core or chip + if not isinstance(target_name, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", target_name): return ApiResponse( status=400, - body={"error": f"Invalid chip: {chip}"}, + body={"error": f"Invalid compiler target: {target_name}"}, ) - chips = available_chips() - chip_dir = Path(get_buckyball_path()) / "examples" / "chips" / chip / "compiler" - if not (chip_dir / "CMakeLists.txt").is_file(): + try: + if core: + resolve_core(get_buckyball_path(), core, require_compiler=True) + else: + resolve_chip_compiler_core(get_buckyball_path(), chip) + except ValueError as error: + choices = available_cores(get_buckyball_path()) if core else available_compiler_chips(get_buckyball_path()) return ApiResponse( status=400, body={ - "error": f"Compiler chip does not exist: {chip}; available compiler chips are: {', '.join(chips)}", + "error": f"{error}; available {'cores' if core else 'chips'} are: {', '.join(choices)}", }, ) diff --git a/bbdev/api/steps/compiler/01_build_event.step.py b/bbdev/api/steps/compiler/01_build_event.step.py index e63797e0a..8162cca6e 100644 --- a/bbdev/api/steps/compiler/01_build_event.step.py +++ b/bbdev/api/steps/compiler/01_build_event.step.py @@ -2,7 +2,6 @@ import re import shlex import sys -from pathlib import Path from motia import FlowContext, queue @@ -11,6 +10,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) +from utils.chip import available_compiler_chips, available_cores, resolve_chip_compiler_core, resolve_core from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id @@ -24,15 +24,6 @@ } -def available_chips(bbdir: str) -> list[str]: - chips_dir = Path(bbdir) / "examples" / "chips" - return sorted( - path.name - for path in chips_dir.iterdir() - if (path / "compiler" / "CMakeLists.txt").is_file() - ) - - async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -48,37 +39,47 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return chip = input_data.get("chip") - if not chip: - ctx.logger.error("Missing required parameter: chip must be specified") + core = input_data.get("core") + if bool(chip) == bool(core): + ctx.logger.error("Specify exactly one compiler target: chip or core") await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + extra_fields={"error": "invalid_target_selection"}, trace_id=origin_tid, ) return - if not re.fullmatch(r"[A-Za-z0-9_-]+", chip): - ctx.logger.error(f"Invalid chip: {chip}") + target_name = core or chip + if not isinstance(target_name, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", target_name): + ctx.logger.error(f"Invalid compiler target: {target_name}") await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "invalid_chip", "chip": chip}, + extra_fields={"error": "invalid_target", "target": target_name}, trace_id=origin_tid, ) return - chips = available_chips(bbdir) - chip_dir = Path(bbdir) / "examples" / "chips" / chip / "compiler" - if not (chip_dir / "CMakeLists.txt").is_file(): - ctx.logger.error(f"Compiler chip does not exist: {chip}") + try: + core_package = ( + resolve_core(bbdir, core, require_compiler=True) + if core + else resolve_chip_compiler_core(bbdir, chip) + ) + except ValueError as error: + choices = available_cores(bbdir) if core else available_compiler_chips(bbdir) + ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, extra_fields={ - "error": "unknown_chip", + "error": "unknown_core" if core else "unknown_chip", + "core": core, "chip": chip, - "available_chips": chips, + "available": choices, }, trace_id=origin_tid, ) return + compiler_dir = core_package.compiler + assert compiler_dir is not None buddy_dir = f"{bbdir}/compiler/thirdparty/buddy-mlir" build_dir = f"{buddy_dir}/build" @@ -99,7 +100,12 @@ async def run_stage(stage: str, command: str): if result.returncode != 0: await check_result( ctx, result.returncode, continue_run=False, - extra_fields={"task": stage, "chip": chip, "stable": stable}, + extra_fields={ + "task": stage, + "chip": chip, + "core": core_package.name, + "stable": stable, + }, trace_id=origin_tid, ) return False @@ -109,7 +115,7 @@ async def run_stage(stage: str, command: str): "cmake", "-G", "Ninja", "-S", buddy_dir, "-B", build_dir, - f"-DBUDDY_EXTERNAL_DIALECTS_DIR={chip_dir}", + f"-DBUDDY_EXTERNAL_DIALECTS_DIR={compiler_dir}", f"-DMLIR_DIR={llvm_build_dir}/lib/cmake/mlir", f"-DLLVM_DIR={llvm_build_dir}/lib/cmake/llvm", "-DLLVM_ENABLE_ASSERTIONS=ON", @@ -124,7 +130,7 @@ async def run_stage(stage: str, command: str): command = nix_cmd(["ninja", "-C", build_dir, "buddy-opt", "buddy-translate", "buddy-llc"]) mode = "stable" if stable else "custom" - if not await run_stage(f"compiler-{chip}-{mode}", command): + if not await run_stage(f"compiler-{core_package.name}-{mode}", command): return # ================================================================================== diff --git a/bbdev/api/steps/compiler/README.md b/bbdev/api/steps/compiler/README.md index 92684307f..cf5815592 100644 --- a/bbdev/api/steps/compiler/README.md +++ b/bbdev/api/steps/compiler/README.md @@ -10,12 +10,14 @@ Compiler build workflow in the Buckyball framework for building the Buckyball co **Function**: Build Buckyball compiler **Parameters**: -- `chip`: compiler chip package used as `BUDDY_EXTERNAL_DIALECTS_DIR`; valid values include `toy`, `goban`, and `pebble` +- `core`: direct compiler Core target, such as `toy`, `goban`, or `pebble` +- `chip`: Chip runtime target; its `chip.toml` selects the default compiler Core. Specify exactly one of `core` or `chip`. - `stable`: optional boolean flag; for `pebble`, build the compiler package that can use the stable LLVM backend lowering path **Example**: ```bash bbdev compiler --build '--chip toy' +bbdev compiler --build '--core goban' bbdev compiler --build '--chip pebble --stable' ``` diff --git a/bbdev/api/steps/workload/01_buidl_api.step.py b/bbdev/api/steps/workload/01_buidl_api.step.py index 4e697a02b..59f469838 100644 --- a/bbdev/api/steps/workload/01_buidl_api.step.py +++ b/bbdev/api/steps/workload/01_buidl_api.step.py @@ -9,6 +9,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) +from utils.chip import resolve_chip_compiler_core from utils.path import get_buckyball_path config = { @@ -46,6 +47,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: status=400, body={"error": f"Workload chip does not exist: {chip}"}, ) + try: + resolve_chip_compiler_core(get_buckyball_path(), chip) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) stable = body.get("stable", False) if not isinstance(stable, bool): return ApiResponse( diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index 36aab1e37..e3b1dc405 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -11,6 +11,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) +from utils.chip import resolve_chip_compiler_core from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id @@ -131,6 +132,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + try: + core_package = resolve_chip_compiler_core(bbdir, chip) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_chip_manifest", "chip": chip}, + trace_id=origin_tid, + ) + return model = input_data.get("model", "") stable = input_data.get("stable", False) rushb_backend = input_data.get("rushB") @@ -185,11 +196,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: os.makedirs(build_dir, exist_ok=True) chip_arg = f"-DBUCKYBALL_WORKLOAD_CHIP={shlex.quote(chip)}" + core_arg = f"-DBUCKYBALL_WORKLOAD_CORE={shlex.quote(core_package.name)}" stable_arg = "-DBUCKYBALL_STABLE=ON" if stable else "-DBUCKYBALL_STABLE=OFF" ninja_target = f" {shlex.quote(target)}" if target else "" inner = ( f"cd {shlex.quote(build_dir)} && " - f"cmake -G Ninja {chip_arg} {stable_arg} " + f"cmake -G Ninja {chip_arg} {core_arg} {stable_arg} " f"-DPython3_EXECUTABLE=\"$(which python3)\" .. && " f"ninja -j{os.cpu_count()}{ninja_target}" ) diff --git a/bbdev/api/utils/chip.py b/bbdev/api/utils/chip.py new file mode 100644 index 000000000..514cb53b5 --- /dev/null +++ b/bbdev/api/utils/chip.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +import tomllib + + +@dataclass(frozen=True) +class CorePackage: + name: str + root: Path + config: Path + compiler: Path | None + + +def _load_toml(path: Path) -> dict: + with path.open("rb") as source: + data = tomllib.load(source) + if not isinstance(data, dict): + raise ValueError(f"TOML root must be a table: {path}") + return data + + +def _core_table(path: Path) -> dict: + core = _load_toml(path).get("core") + if not isinstance(core, dict): + raise ValueError(f"missing [core] table: {path}") + return core + + +def available_cores(bbdir: str) -> list[str]: + roots = Path(bbdir) / "examples" / "cores" + if not roots.is_dir(): + return [] + return sorted( + path.name + for path in roots.iterdir() + if (path / "manifest.toml").is_file() + ) + + +def resolve_core(bbdir: str, name: str, require_compiler: bool = False) -> CorePackage: + root = Path(bbdir) / "examples" / "cores" / name + manifest = root / "manifest.toml" + if not manifest.is_file(): + raise ValueError(f"Core does not exist: {name}") + + core = _core_table(manifest) + if core.get("name") != name: + raise ValueError(f"Core manifest name mismatch: {manifest}") + + config_rel = core.get("config") + if not isinstance(config_rel, str) or not config_rel: + raise ValueError(f"Core manifest missing core.config: {manifest}") + config = (root / config_rel).resolve() + if not config.is_file(): + raise ValueError(f"Core configuration does not exist: {config}") + + compiler_rel = core.get("compiler") + compiler = (root / compiler_rel).resolve() if isinstance(compiler_rel, str) else None + if compiler is not None and not (compiler / "CMakeLists.txt").is_file(): + raise ValueError(f"Core compiler does not exist: {compiler}") + if require_compiler and compiler is None: + raise ValueError(f"Core has no compiler package: {name}") + + return CorePackage(name=name, root=root, config=config, compiler=compiler) + + +def available_compiler_chips(bbdir: str) -> list[str]: + chips = Path(bbdir) / "examples" / "chips" + if not chips.is_dir(): + return [] + + result = [] + for root in chips.iterdir(): + manifest = root / "chip.toml" + if not manifest.is_file(): + continue + try: + core_name = _load_toml(manifest).get("chip", {}).get("compilerCore") + if isinstance(core_name, str): + resolve_core(bbdir, core_name, require_compiler=True) + result.append(root.name) + except (ValueError, tomllib.TOMLDecodeError): + continue + return sorted(result) + + +def resolve_chip_compiler_core(bbdir: str, chip: str) -> CorePackage: + manifest = Path(bbdir) / "examples" / "chips" / chip / "chip.toml" + if not manifest.is_file(): + raise ValueError(f"Chip manifest does not exist: {manifest}") + + chip_table = _load_toml(manifest).get("chip") + if not isinstance(chip_table, dict) or chip_table.get("name") != chip: + raise ValueError(f"Chip manifest name mismatch: {manifest}") + core_name = chip_table.get("compilerCore") + if not isinstance(core_name, str) or not core_name: + raise ValueError(f"Chip manifest missing chip.compilerCore: {manifest}") + return resolve_core(bbdir, core_name, require_compiler=True) diff --git a/bbdev/mcp/common.py b/bbdev/mcp/common.py index fded013a1..7f6c986a8 100644 --- a/bbdev/mcp/common.py +++ b/bbdev/mcp/common.py @@ -291,17 +291,30 @@ def _balldomain_path(chip: str, balldomain: Optional[str]) -> Path: if not chip_root.is_dir(): raise FileNotFoundError(f"chip does not exist: {chip_root}") - domains = chip_root / "configs" / "tiles" / "cores" / "balldomains" - cores = chip_root / "configs" / "tiles" / "cores" + chip_manifest = chip_root / "chip.toml" + if not chip_manifest.is_file(): + raise FileNotFoundError(f"chip manifest does not exist: {chip_manifest}") + chip_config = _load_toml(chip_manifest).get("chip") + if not isinstance(chip_config, dict): + raise ValueError(f"missing [chip] table: {chip_manifest}") + core_name = chip_config.get("compilerCore") + if not isinstance(core_name, str) or not core_name: + raise ValueError(f"chip manifest missing compilerCore: {chip_manifest}") + + core_root = REPO / "examples" / "cores" / core_name + core_manifest = core_root / "manifest.toml" + if not core_manifest.is_file(): + raise FileNotFoundError(f"core manifest does not exist: {core_manifest}") + domains = core_root / "configs" / "balldomains" + core_config = core_root / "core.toml" if balldomain is None: - default = cores / "default.toml" - if not default.is_file(): - raise FileNotFoundError(f"missing {default}; pass balldomain= explicitly") - rel = _load_toml(default).get("balldomain") + if not core_config.is_file(): + raise FileNotFoundError(f"missing {core_config}; pass balldomain= explicitly") + rel = _load_toml(core_config).get("balldomain") if not isinstance(rel, str) or not rel: - raise ValueError(f"cores/default.toml missing balldomain=: {default}") - path = (cores / rel).resolve() + raise ValueError(f"core.toml missing balldomain=: {core_config}") + path = (core_root / rel).resolve() else: raw = Path(balldomain) if raw.is_absolute(): @@ -309,7 +322,7 @@ def _balldomain_path(chip: str, balldomain: Optional[str]) -> Path: elif balldomain.endswith(".toml"): candidates = [ domains / raw.name, - cores / balldomain, + core_root / balldomain, chip_root / balldomain, ] path = next( diff --git a/bbdev/mcp/tools/validate.py b/bbdev/mcp/tools/validate.py index d12cf2a19..c1accb82a 100644 --- a/bbdev/mcp/tools/validate.py +++ b/bbdev/mcp/tools/validate.py @@ -10,7 +10,7 @@ def register(mcp): @mcp.tool() def validate(chip: str = "toy", balldomain: Optional[str] = None) -> str: - """Validate chip balldomain TOML registration.""" + """Validate the compiler Core's BallDomain registration for a Chip.""" if e := need("chip", chip): return err(e) try: @@ -19,4 +19,3 @@ def validate(chip: str = "toy", balldomain: Optional[str] = None) -> str: return fmt( {"passed": False, "success": False, "failure": True, "error": str(ex)} ) - From 50c7868f9a1cb6a56bade61f664ece0e3177efa1 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 10 Aug 2026 01:42:28 +0800 Subject: [PATCH 107/196] feat: extend build command in compiler API to include additional targets for enhanced functionality --- bbdev/api/steps/compiler/01_build_event.step.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bbdev/api/steps/compiler/01_build_event.step.py b/bbdev/api/steps/compiler/01_build_event.step.py index 8162cca6e..defe76c5a 100644 --- a/bbdev/api/steps/compiler/01_build_event.step.py +++ b/bbdev/api/steps/compiler/01_build_event.step.py @@ -127,8 +127,11 @@ async def run_stage(stage: str, command: str): if not await run_stage("configure", nix_cmd(cmake_args)): return - command = nix_cmd(["ninja", "-C", build_dir, - "buddy-opt", "buddy-translate", "buddy-llc"]) + command = nix_cmd([ + "ninja", "-C", build_dir, + "buddy-opt", "buddy-translate", "buddy-llc", + "python-package-buddy", "BuddyMLIRPythonModules", + ]) mode = "stable" if stable else "custom" if not await run_stage(f"compiler-{core_package.name}-{mode}", command): return From 76922e903f6ba8c9311f96202eb7cee9da88dac8 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 10 Aug 2026 16:01:52 +0800 Subject: [PATCH 108/196] feat: implement task cancellation API and process management for improved task control and user experience --- .../api/steps/bebop/bemu/02_sim_event.step.py | 68 ++++++++++++++----- .../steps/bebop/bemu/03_batch_event.step.py | 4 +- .../steps/bebop/bemu/scripts/bemu_common.py | 41 ++++++++++- .../steps/common/01_result_query_api.step.py | 10 +++ bbdev/api/steps/common/02_cancel_api.step.py | 29 ++++++++ .../api/steps/common/02_cancel_event.step.py | 39 +++++++++++ bbdev/api/utils/event_common.py | 8 ++- bbdev/api/utils/process_registry.py | 67 ++++++++++++++++++ bbdev/api/utils/stream_run.py | 17 +++++ bbdev/mcp/common.py | 20 +++++- bbdev/mcp/tools/__init__.py | 2 + bbdev/mcp/tools/task_cancel.py | 17 +++++ 12 files changed, 296 insertions(+), 26 deletions(-) create mode 100644 bbdev/api/steps/common/02_cancel_api.step.py create mode 100644 bbdev/api/steps/common/02_cancel_event.step.py create mode 100644 bbdev/api/utils/process_registry.py create mode 100644 bbdev/mcp/tools/task_cancel.py diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 1b29eebf4..c71f329eb 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -6,6 +6,7 @@ import os import shlex import sys +import tomllib from datetime import datetime from pathlib import Path @@ -22,7 +23,8 @@ from utils.stream_run import stream_run_logger from utils.search_workload import search_workload, search_workload_all from utils.event_common import check_result, get_origin_trace_id -from bemu_common import bemu_manifest +from utils.process_registry import cancellation_requested +from bemu_common import bemu_core_manifest, bemu_manifest, bemu_tile PERFETTO_TARGETS = { "buddy-buckyball-lenet-run": "buddy-buckyball-lenet-perfetto", @@ -63,14 +65,18 @@ def clean_model_trace(binary_dir: str) -> None: def resolve_bemu_binary(bbdir: str, chip: str, binary_name: str) -> str | None: - search_root = f"{bbdir}/bb-tests/output/workloads/src" - chip_root = f"{search_root}/CTest/chips/{chip}" + if Path(binary_name).name != binary_name: + return None - chip_binary = search_workload(chip_root, binary_name) - if chip_binary is not None: - return chip_binary + build_root = f"{bbdir}/bb-tests/build/workloads/src" + output_root = f"{bbdir}/bb-tests/output/workloads/src" + for search_root in (build_root, output_root): + chip_root = f"{search_root}/CTest/chips/{chip}" + chip_binary = search_workload(chip_root, binary_name) + if chip_binary is not None: + return chip_binary - matches = search_workload_all(search_root, binary_name) + matches = search_workload_all(output_root, binary_name) chip_marker = f"{os.path.sep}CTest{os.path.sep}chips{os.path.sep}" non_chip_matches = [path for path in matches if chip_marker not in path] if len(non_chip_matches) == 1: @@ -87,6 +93,22 @@ def is_native_host_elf(path: str) -> bool: return header[:4] == b"\x7fELF" and header[18:20] == b"\x3e\x00" +def rushb_bemu_library(manifest: Path) -> Path: + """Return the cdylib emitted by the Core selected for a rushB run.""" + with manifest.open("rb") as source: + cargo = tomllib.load(source) + lib_name = cargo.get("lib", {}).get("name") + if not lib_name: + lib_name = cargo["package"]["name"].replace("-", "_") + return manifest.parent / "target/release" / f"lib{lib_name}.so" + + +def rushb_bemu_library_dirs(manifest: Path) -> list[Path]: + """Locate native shared-library dependencies produced by the Core build.""" + build_dir = manifest.parent / "target/release/build" + return sorted({library.parent for library in build_dir.glob("*/out/**/libriscv.so")}) + + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() @@ -128,10 +150,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: clean_model_trace(binary_dir) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - log_dir = f"{bbdir}/log/{timestamp}-bemu-{binary_name}" + log_dir = f"{bbdir}/log/{timestamp}-bemu-{Path(binary_path).name}" os.makedirs(log_dir, exist_ok=True) if input_data.get("rushB"): + # rushB is a native ABI for one accelerator instance, so it remains a + # Core-level backend even when guest-ELF BEMU runs a whole tile. + bemu_cargo_manifest = bemu_core_manifest(chip, bbdir) # rushB binaries are host executables. Rebuilding and co-locating the # backend library keeps the ABI selection explicit and avoids sending # a host ELF through Spike's guest-ELF path. @@ -145,14 +170,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - bemu_library = bemu_cargo_manifest.parent / "target/release/libbebop_bemu.so" + bemu_library = rushb_bemu_library(bemu_cargo_manifest) + bemu_runtime_library = Path(binary_dir) / "libbebop_bemu.so" + dependency_dirs = rushb_bemu_library_dirs(bemu_cargo_manifest) build_cmd = shlex.join([ "cargo", "build", "--release", "--manifest-path", str(bemu_cargo_manifest), "--lib", ]) copy_cmd = shlex.join([ - "cmake", "-E", "copy_if_different", str(bemu_library), binary_dir, + "cmake", "-E", "copy_if_different", str(bemu_library), str(bemu_runtime_library), ]) - inner_cmd = f"cd {shlex.quote(binary_dir)} && {build_cmd} && {copy_cmd} && exec {shlex.quote(binary_path)}" + dependency_path = os.pathsep.join(str(path) for path in dependency_dirs) + library_env = f"LD_LIBRARY_PATH={shlex.quote(dependency_path)}:${{LD_LIBRARY_PATH:-}}" + inner_cmd = f"cd {shlex.quote(binary_dir)} && {build_cmd} && {copy_cmd} && {library_env} exec {shlex.quote(binary_path)}" run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" ctx.logger.info(f"Running rushB BEMU: {run_cmd}") run_result = stream_run_logger( @@ -161,7 +190,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cwd=bbdir, stdout_prefix="rushB bemu", stderr_prefix="rushB bemu", + task_scope=origin_tid, ) + if cancellation_requested(origin_tid): + return await check_result( ctx, run_result.returncode, @@ -186,13 +218,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--manifest-path", str(bemu_cargo_manifest), "--", - "run", - "bemu", - "--elf", - binary_path, - "--log-dir", - log_dir, ] + tile = bemu_tile(chip, bbdir) + if tile: + cargo_args.extend(["--tile", str(tile), "--elf", binary_path, "--log-dir", log_dir]) + else: + cargo_args.extend(["--", "run", "bemu", "--elf", binary_path, "--log-dir", log_dir]) if input_data.get("pk"): cargo_args.append("--pk") if input_data.get("disasm"): @@ -208,7 +239,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cwd=bbdir, stdout_prefix="bebop bemu", stderr_prefix="bebop bemu", + task_scope=origin_tid, ) + if cancellation_requested(origin_tid): + return if run_result.returncode != 0: await check_result( ctx, diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index f98628764..afde5bb19 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -25,7 +25,7 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id -from bemu_common import bemu_manifest +from bemu_common import bemu_core_manifest from regression import regression_workload_toml config = { @@ -54,7 +54,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return try: env = os.environ.copy() - bemu_cargo_manifest = bemu_manifest(chip, bbdir) + bemu_cargo_manifest = bemu_core_manifest(chip, bbdir) except ValueError as e: ctx.logger.error(str(e)) await check_result( diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py index 79e32b03a..fb53ba895 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py @@ -1,4 +1,5 @@ from pathlib import Path +import tomllib from utils.path import get_buckyball_path @@ -7,7 +8,45 @@ def bemu_manifest(chip: str, bbdir: str | None = None) -> Path: if not chip: raise ValueError("missing required parameter: chip") root = Path(bbdir or get_buckyball_path()) - manifest = root / "examples" / "chips" / chip / "emu" / "Cargo.toml" + chip_manifest = root / "examples" / "chips" / chip / "chip.toml" + if not chip_manifest.is_file(): + raise ValueError(f"chip manifest does not exist: {chip_manifest}") + + with chip_manifest.open("rb") as f: + data = tomllib.load(f) + if not isinstance(data, dict): + raise ValueError(f"TOML root must be a table: {chip_manifest}") + + runtime = data.get("runtime") + if not isinstance(runtime, dict): + raise ValueError(f"chip manifest missing [runtime]: {chip_manifest}") + rel = runtime.get("bemu") + if not isinstance(rel, str) or not rel: + raise ValueError(f"chip manifest missing runtime.bemu: {chip_manifest}") + + manifest = (chip_manifest.parent / rel).resolve() if not manifest.is_file(): raise ValueError(f"bemu chip manifest does not exist: {manifest}") return manifest + + +def bemu_core_manifest(chip: str, bbdir: str | None = None) -> Path: + root = Path(bbdir or get_buckyball_path()) + chip_manifest = root / "examples" / "chips" / chip / "chip.toml" + with chip_manifest.open("rb") as f: + data = tomllib.load(f) + runtime = data.get("runtime", {}) + rel = runtime.get("bemuCore") or runtime.get("bemu") + manifest = (chip_manifest.parent / rel).resolve() + if not manifest.is_file(): + raise ValueError(f"BEMU Core manifest does not exist: {manifest}") + return manifest + + +def bemu_tile(chip: str, bbdir: str | None = None) -> Path | None: + root = Path(bbdir or get_buckyball_path()) + chip_manifest = root / "examples" / "chips" / chip / "chip.toml" + with chip_manifest.open("rb") as f: + data = tomllib.load(f) + rel = data.get("runtime", {}).get("bemuTile") + return (chip_manifest.parent / rel).resolve() if rel else None diff --git a/bbdev/api/steps/common/01_result_query_api.step.py b/bbdev/api/steps/common/01_result_query_api.step.py index 9ff143a9d..88257656f 100644 --- a/bbdev/api/steps/common/01_result_query_api.step.py +++ b/bbdev/api/steps/common/01_result_query_api.step.py @@ -24,6 +24,16 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body={"error": "trace_id is required"} ) + cancelled_state = await ctx.state.get(trace_id, "cancelled") + if cancelled_state: + return ApiResponse( + status=200, + body={ + "status": "cancelled", + "body": cancelled_state.get("body", cancelled_state), + }, + ) + # Check for success state success_state = await ctx.state.get(trace_id, "success") if success_state: diff --git a/bbdev/api/steps/common/02_cancel_api.step.py b/bbdev/api/steps/common/02_cancel_api.step.py new file mode 100644 index 000000000..7da7dfbca --- /dev/null +++ b/bbdev/api/steps/common/02_cancel_api.step.py @@ -0,0 +1,29 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + + +config = { + "name": "task-cancel-api", + "description": "Cancel a queued or running bbdev task by trace_id", + "flows": ["common"], + "triggers": [api("POST", "/task/{trace_id}/cancel")], + "enqueues": ["common.task.cancel"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + trace_id = request.path_params.get("trace_id", "") + if not trace_id: + return ApiResponse(status=400, body={"error": "trace_id is required"}) + + for key in ("success", "failure", "cancelled"): + if await ctx.state.get(trace_id, key): + return ApiResponse( + status=409, + body={"error": "task is already terminal", "trace_id": trace_id}, + ) + + await ctx.enqueue({ + "topic": "common.task.cancel", + "data": {"_trace_id": trace_id}, + }) + return ApiResponse(status=202, body={"trace_id": trace_id, "status": "cancelling"}) diff --git a/bbdev/api/steps/common/02_cancel_event.step.py b/bbdev/api/steps/common/02_cancel_event.step.py new file mode 100644 index 000000000..64c63d428 --- /dev/null +++ b/bbdev/api/steps/common/02_cancel_event.step.py @@ -0,0 +1,39 @@ +import asyncio + +from motia import FlowContext, queue + +from utils.event_common import get_origin_trace_id +from utils.process_registry import cancel_process, force_cancel_process, process_pid + + +config = { + "name": "task-cancel", + "description": "Terminate a task process group", + "flows": ["common"], + "triggers": [queue("common.task.cancel")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + trace_id = get_origin_trace_id(input_data, ctx) + pid = process_pid(trace_id) + found = cancel_process(trace_id) + if pid is not None: + await asyncio.sleep(5) + force_cancel_process(trace_id, pid) + await ctx.state.set( + trace_id, + "cancelled", + { + "status": 200, + "body": { + "success": False, + "failure": False, + "cancelled": True, + "processing": False, + "trace_id": trace_id, + "process_found": found, + }, + }, + ) diff --git a/bbdev/api/utils/event_common.py b/bbdev/api/utils/event_common.py index 102547aec..6903277a9 100644 --- a/bbdev/api/utils/event_common.py +++ b/bbdev/api/utils/event_common.py @@ -2,6 +2,8 @@ Common utility functions for all event steps. """ +from utils.process_registry import set_current_task_scope + def get_origin_trace_id(input_data, ctx): """Get the origin trace_id from input_data (passed by API step) or fall back to ctx.trace_id. @@ -9,9 +11,9 @@ def get_origin_trace_id(input_data, ctx): iii 0.7+ assigns a new trace_id per handler invocation, so the API step's trace_id must be forwarded explicitly via input_data["_trace_id"]. """ - if isinstance(input_data, dict) and "_trace_id" in input_data: - return input_data["_trace_id"] - return ctx.trace_id + trace_id = input_data["_trace_id"] if isinstance(input_data, dict) and "_trace_id" in input_data else ctx.trace_id + set_current_task_scope(trace_id) + return trace_id async def check_result(ctx, returncode, continue_run=False, extra_fields=None, trace_id=None): diff --git a/bbdev/api/utils/process_registry.py b/bbdev/api/utils/process_registry.py new file mode 100644 index 000000000..27ec1a4ee --- /dev/null +++ b/bbdev/api/utils/process_registry.py @@ -0,0 +1,67 @@ +"""In-process registry for cancellable bbdev subprocesses.""" + +import os +import signal +import threading +from contextvars import ContextVar +from typing import Dict + + +_lock = threading.Lock() +_processes: Dict[str, int] = {} +_cancelled: set[str] = set() +_current_task_scope: ContextVar[str | None] = ContextVar("bbdev_task_scope", default=None) + + +def set_current_task_scope(scope: str) -> None: + _current_task_scope.set(scope) + + +def current_task_scope() -> str | None: + return _current_task_scope.get() + + +def register_process(scope: str, pid: int) -> None: + with _lock: + _processes[scope] = pid + _cancelled.discard(scope) + + +def unregister_process(scope: str) -> None: + with _lock: + _processes.pop(scope, None) + + +def cancel_process(scope: str) -> bool: + with _lock: + pid = _processes.get(scope) + _cancelled.add(scope) + if pid is None: + return False + try: + os.killpg(pid, signal.SIGTERM) + except ProcessLookupError: + return False + return True + + +def force_cancel_process(scope: str, pid: int) -> bool: + """Escalate a still-registered task process group after its grace period.""" + with _lock: + if _processes.get(scope) != pid: + return False + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + return False + return True + + +def process_pid(scope: str) -> int | None: + with _lock: + return _processes.get(scope) + + +def cancellation_requested(scope: str) -> bool: + with _lock: + return scope in _cancelled diff --git a/bbdev/api/utils/stream_run.py b/bbdev/api/utils/stream_run.py index 08edb4f36..101900821 100644 --- a/bbdev/api/utils/stream_run.py +++ b/bbdev/api/utils/stream_run.py @@ -2,6 +2,8 @@ import threading from typing import Optional, List, Callable +from utils.process_registry import current_task_scope, register_process, unregister_process + class StreamResult: """Result object mimicking subprocess.CompletedProcess""" @@ -23,6 +25,7 @@ def stream_run( stdout_prefix: str = "STDOUT", stderr_prefix: str = "STDERR", env: Optional[dict] = None, + task_scope: Optional[str] = None, ) -> StreamResult: """ Execute command and stream output in real-time @@ -85,10 +88,15 @@ def emit(): # Start process # Use errors='replace' to handle non-UTF-8 bytes from simulation UART output + task_scope = task_scope or current_task_scope() process = subprocess.Popen( cmd, cwd=cwd, shell=shell, + # MCP workers do not provide interactive input. Closing the child's + # stdin makes batch workloads observe EOF instead of waiting forever + # at an application prompt. + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, @@ -97,7 +105,12 @@ def emit(): bufsize=1, executable=executable, env=env, + # A task owns a process group, so a generic cancel request also + # terminates shells and descendants such as nix/cargo/ninja. + start_new_session=True, ) + if task_scope: + register_process(task_scope, process.pid) stdout_lines = [] stderr_lines = [] @@ -127,6 +140,8 @@ def emit(): stdout_thread.join() stderr_thread.join() + if task_scope: + unregister_process(task_scope) return StreamResult( returncode=process.returncode, stdout="\n".join(stdout_lines), @@ -145,6 +160,7 @@ def stream_run_logger( stderr_prefix: str = "STDERR", verbose: bool = False, env: Optional[dict] = None, + task_scope: Optional[str] = None, ) -> StreamResult: """ Convenience function for streaming output using logger @@ -190,4 +206,5 @@ def log_stderr(line): stdout_prefix=stdout_prefix, stderr_prefix=stderr_prefix, env=env, + task_scope=task_scope, ) diff --git a/bbdev/mcp/common.py b/bbdev/mcp/common.py index 7f6c986a8..fc25b8d41 100644 --- a/bbdev/mcp/common.py +++ b/bbdev/mcp/common.py @@ -248,6 +248,17 @@ def task_status(trace_id: str) -> Dict[str, Any]: "queued": True, "trace_id": trace_id, } + if "cancelled" in state: + _submitted_trace_ids.discard(trace_id) + body = state["cancelled"].get("body", state["cancelled"]) + if not isinstance(body, dict): + raise RuntimeError(f"cancelled body must be object: {body!r}") + body.setdefault("success", False) + body.setdefault("failure", False) + body.setdefault("cancelled", True) + body.setdefault("processing", False) + body.setdefault("trace_id", trace_id) + return body if "success" in state: _submitted_trace_ids.discard(trace_id) out = state["success"].get("body", state["success"]) @@ -306,15 +317,18 @@ def _balldomain_path(chip: str, balldomain: Optional[str]) -> Path: if not core_manifest.is_file(): raise FileNotFoundError(f"core manifest does not exist: {core_manifest}") domains = core_root / "configs" / "balldomains" - core_config = core_root / "core.toml" + core_manifest_config = _load_toml(core_manifest).get("core", {}).get("config") + if not isinstance(core_manifest_config, str) or not core_manifest_config: + raise ValueError(f"Core manifest missing core.config: {core_manifest}") + core_config = (core_root / core_manifest_config).resolve() if balldomain is None: if not core_config.is_file(): raise FileNotFoundError(f"missing {core_config}; pass balldomain= explicitly") rel = _load_toml(core_config).get("balldomain") if not isinstance(rel, str) or not rel: - raise ValueError(f"core.toml missing balldomain=: {core_config}") - path = (core_root / rel).resolve() + raise ValueError(f"Core configuration missing balldomain=: {core_config}") + path = (core_config.parent / rel).resolve() else: raw = Path(balldomain) if raw.is_absolute(): diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index d495299bd..1bc5d93ad 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -4,6 +4,7 @@ from . import validate from . import task_status +from . import task_cancel from . import compiler_build from . import workload_clean from . import workload_build @@ -41,6 +42,7 @@ MODULES = [ validate, task_status, + task_cancel, compiler_build, workload_clean, workload_build, diff --git a/bbdev/mcp/tools/task_cancel.py b/bbdev/mcp/tools/task_cancel.py new file mode 100644 index 000000000..7b12eca54 --- /dev/null +++ b/bbdev/mcp/tools/task_cancel.py @@ -0,0 +1,17 @@ +"""MCP tool: bbdev_task_cancel.""" + +from __future__ import annotations + +from common import err, fmt, need, submit + + +def register(mcp): + @mcp.tool() + def bbdev_task_cancel(trace_id: str) -> str: + """Cancel a queued or running bbdev task and terminate its process group.""" + if error := need("trace_id", trace_id): + return err(error) + try: + return fmt(submit(f"/task/{trace_id}/cancel", {})) + except Exception as ex: + return fmt({"success": False, "failure": True, "error": str(ex)}) From 6400616d6ab70f16b131648d7eb61c3e665ca588 Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Mon, 10 Aug 2026 22:28:00 +0800 Subject: [PATCH 109/196] feat: add Bebop bank difftest workflow --- .../steps/bebop/difftest/01_run_api.step.py | 79 ++++++ .../steps/bebop/difftest/01_run_event.step.py | 228 ++++++++++++++++++ bbdev/api/tests/test_all_api_serial.py | 1 + bbdev/api/tests/test_api_bebop_difftest.py | 8 + bbdev/bbdev | 3 + bbdev/mcp/tools/__init__.py | 2 + bbdev/mcp/tools/bebop_difftest_run.py | 41 ++++ 7 files changed, 362 insertions(+) create mode 100644 bbdev/api/steps/bebop/difftest/01_run_api.step.py create mode 100644 bbdev/api/steps/bebop/difftest/01_run_event.step.py create mode 100644 bbdev/api/tests/test_api_bebop_difftest.py create mode 100644 bbdev/mcp/tools/bebop_difftest_run.py diff --git a/bbdev/api/steps/bebop/difftest/01_run_api.step.py b/bbdev/api/steps/bebop/difftest/01_run_api.step.py new file mode 100644 index 000000000..cc8128d56 --- /dev/null +++ b/bbdev/api/steps/bebop/difftest/01_run_api.step.py @@ -0,0 +1,79 @@ +from pathlib import Path + +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path, get_verilator_build_dir + + +config = { + "name": "bebop-difftest-run-api", + "description": "Run Verilator+BEMU Bank DiffTest", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/difftest/run")], + "enqueues": ["bebop.difftest.run"], +} + + +def _required_string(body: dict, name: str) -> str | None: + value = body.get(name) + if not isinstance(value, str) or not value.strip() or value == "None": + return None + return value + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + required = {name: _required_string(body, name) for name in ("chip", "config", "binary")} + missing = [name for name, value in required.items() if value is None] + if missing: + missing_args = ", ".join("--" + name for name in missing) + return ApiResponse( + status=400, + body={ + "success": False, + "error": f"Missing required parameter(s): {missing_args}", + }, + ) + + unsupported = [name for name in ("rushB", "pk", "fast") if body.get(name)] + if unsupported: + unsupported_args = ", ".join("--" + name for name in unsupported) + return ApiResponse( + status=400, + body={ + "success": False, + "error": ( + "Bank DiffTest only supports normal baremetal ELF mode; " + f"unsupported: {unsupported_args}" + ), + }, + ) + + bbdir = get_buckyball_path() + chip = required["chip"] + manifest = Path(bbdir) / "examples" / "chips" / chip / "emu" / "Cargo.toml" + if not manifest.is_file(): + return ApiResponse( + status=400, + body={"success": False, "error": f"Chip BEMU manifest does not exist: {manifest}"}, + ) + + data = { + **required, + "jobs": body.get("jobs", 16), + "vsrc_dir": get_verilator_build_dir( + bbdir, + required["config"], + body.get("vsrc-dir") or body.get("output-dir"), + ), + "no_wave": body.get("no-wave", body.get("no_wave", False)), + "itrace": body.get("itrace", False), + "mtrace": body.get("mtrace", False), + "pmctrace": body.get("pmctrace", False), + "ctrace": body.get("ctrace", False), + "banktrace": body.get("banktrace", False), + "log_dir": body.get("log-dir") or body.get("log_dir"), + "_trace_id": ctx.trace_id, + } + await ctx.enqueue({"topic": "bebop.difftest.run", "data": data}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/difftest/01_run_event.step.py b/bbdev/api/steps/bebop/difftest/01_run_event.step.py new file mode 100644 index 000000000..0bea2ef03 --- /dev/null +++ b/bbdev/api/steps/bebop/difftest/01_run_event.step.py @@ -0,0 +1,228 @@ +"""Build and run a Chip-specific Verilator+BEMU Bank DiffTest workload.""" + +import os +import shlex +import sys +from datetime import datetime +from pathlib import Path + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path +from utils.search_workload import search_workload, search_workload_all +from utils.stream_run import stream_run_logger + + +config = { + "name": "bebop-difftest-run", + "description": "Build and run Verilator+BEMU Bank DiffTest", + "flows": ["bebop"], + "triggers": [queue("bebop.difftest.run")], + "enqueues": [], +} + + +def resolve_workload(bbdir: str, chip: str, binary: str) -> str | None: + candidate = Path(binary) + if not candidate.is_absolute(): + candidate = Path(bbdir) / candidate + if candidate.is_file(): + return str(candidate.resolve()) + + search_root = f"{bbdir}/bb-tests/output/workloads/src" + chip_match = search_workload(f"{search_root}/CTest/chips/{chip}", binary) + if chip_match is not None: + return chip_match + + matches = search_workload_all(search_root, binary) + chip_marker = f"{os.path.sep}CTest{os.path.sep}chips{os.path.sep}" + non_chip_matches = [path for path in matches if chip_marker not in path] + return non_chip_matches[0] if len(non_chip_matches) == 1 else None + + +async def fail(ctx, trace_id: str, error: str, **fields) -> None: + ctx.logger.error(fields.pop("message", error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": error, **fields}, + trace_id=trace_id, + ) + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + chip = input_data["chip"] + arch_config = input_data["config"] + binary_name = input_data["binary"] + + chip_emu_dir = Path(bbdir) / "examples" / "chips" / chip / "emu" + manifest = chip_emu_dir / "Cargo.toml" + if not manifest.is_file(): + await fail(ctx, origin_tid, "invalid_chip", chip=chip, manifest=str(manifest)) + return + + vsrc_dir = os.path.abspath(input_data["vsrc_dir"]) + if not os.path.isdir(vsrc_dir): + await fail( + ctx, + origin_tid, + "vsrc_not_found", + config=arch_config, + vsrc_dir=vsrc_dir, + message=f"RTL directory does not exist: {vsrc_dir}; generate Verilog first", + ) + return + + binary_path = resolve_workload(bbdir, chip, binary_name) + if binary_path is None: + await fail(ctx, origin_tid, "binary_not_found", chip=chip, binary=binary_name) + return + + try: + jobs = int(input_data.get("jobs", 16)) + if jobs <= 0: + raise ValueError + except (TypeError, ValueError): + await fail(ctx, origin_tid, "invalid_jobs", jobs=input_data.get("jobs")) + return + + # sourceme.sh points RISCV at the repository's `result` environment. Keep + # it current when a newly required native dependency is not present. + dramsim_header = Path(bbdir) / "result" / "include" / "dramsim3.h" + if not dramsim_header.is_file(): + ctx.logger.info( + f"Nix environment is missing {dramsim_header}; rebuilding result" + ) + env_result = stream_run_logger( + cmd="nix build", + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="bebop difftest environment", + stderr_prefix="bebop difftest environment", + ) + if env_result.returncode != 0: + await check_result( + ctx, + env_result.returncode, + continue_run=False, + extra_fields={ + "task": "environment", + "backend": "verilator+bemu-difftest", + "chip": chip, + "config": arch_config, + "dependency": str(dramsim_header), + }, + trace_id=origin_tid, + ) + return + + bebop_bin = chip_emu_dir / "target" / "release" / "bebop" + cargo_build_cmd = shlex.join( + [ + "cargo", + "build", + "--release", + "--bin", + "bebop", + "--features", + "verilator,bemu,difftest", + "--jobs", + str(jobs), + ] + ) + build_inner = f"cd {shlex.quote(str(chip_emu_dir))} && exec {cargo_build_cmd}" + build_cmd = f"nix develop -c sh -c {shlex.quote(build_inner)}" + ctx.logger.info(f"Building Chip DiffTest executable: {build_cmd}") + build_result = stream_run_logger( + cmd=build_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="bebop difftest build", + stderr_prefix="bebop difftest build", + env={**os.environ, "VSRC_PATH": vsrc_dir}, + ) + if build_result.returncode != 0: + await check_result( + ctx, + build_result.returncode, + continue_run=False, + extra_fields={ + "task": "build", + "backend": "verilator+bemu-difftest", + "chip": chip, + "config": arch_config, + "binary": str(bebop_bin), + "vsrc_dir": vsrc_dir, + }, + trace_id=origin_tid, + ) + return + + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + requested_log_dir = input_data.get("log_dir") + if requested_log_dir: + log_dir = os.path.abspath(os.path.join(bbdir, requested_log_dir)) + else: + log_dir = f"{bbdir}/log/{timestamp}-difftest-{binary_name}" + os.makedirs(log_dir, exist_ok=True) + + run_args = [ + str(bebop_bin), + "run", + "verilator", + f"--elf={binary_path}", + f"--log-dir={log_dir}", + "--diff", + ] + if input_data.get("no_wave"): + run_args.append("--no-wave") + for trace_name in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace"): + if input_data.get(trace_name): + run_args.append(f"--{trace_name}") + + preload = os.pathsep.join( + path + for path in ( + f"{bbdir}/result/lib/libdramsim3.so", + os.environ.get("LD_PRELOAD", ""), + ) + if path + ) + run_inner = ( + f"cd {shlex.quote(str(chip_emu_dir))} && " + f"export LD_PRELOAD={shlex.quote(preload)} && exec {shlex.join(run_args)}" + ) + run_cmd = f"nix develop -c sh -c {shlex.quote(run_inner)}" + ctx.logger.info(f"Running Bank DiffTest: {run_cmd}") + run_result = stream_run_logger( + cmd=run_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="bebop difftest", + stderr_prefix="bebop difftest", + ) + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "difftest", + "backend": "verilator+bemu-difftest", + "chip": chip, + "config": arch_config, + "binary": binary_path, + "vsrc_dir": vsrc_dir, + "log_dir": log_dir, + "bank_diff": os.path.join(log_dir, "bank_diff.ndjson"), + "timestamp": timestamp, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index 2f3cd8819..37bc8763f 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -32,6 +32,7 @@ def main(): "test_api_bebop_verilator_sim.py", "test_api_bebop_verilator_run.py", "test_api_bebop_verilator_batch.py", + "test_api_bebop_difftest.py", "test_api_bebop_p2e_clean.py", "test_api_bebop_p2e_verilog.py", "test_api_bebop_p2e_buildbitstream.py", diff --git a/bbdev/api/tests/test_api_bebop_difftest.py b/bbdev/api/tests/test_api_bebop_difftest.py new file mode 100644 index 000000000..986f62d20 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_difftest.py @@ -0,0 +1,8 @@ +from _api_test_helper import run_bbdev_case + + +run_bbdev_case( + "bbdev bebop-difftest --run " + "'--chip toy --config sims.verilator.BuckyballToyVerilatorConfig " + "--binary toy_im2col_k3_test-singlecore-baremetal --no-wave'" +) diff --git a/bbdev/bbdev b/bbdev/bbdev index b7ca6ce7c..abfb7151f 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -70,6 +70,9 @@ WORKFLOW_COMMANDS = { "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--rushB] [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--rushB] [--clean-before]"', }, + "bebop-difftest": { + "run": 'Run Verilator+BEMU Bank DiffTest. Args: "--chip --config --binary [--jobs ] [--log-dir ] [--no-wave]"', + }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--config [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--config [--output-dir ]"', diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index 1bc5d93ad..c8483d527 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -16,6 +16,7 @@ from . import bebop_verilator_sim from . import bebop_verilator_run from . import bebop_verilator_batch +from . import bebop_difftest_run from . import verilator_clean from . import verilator_verilog from . import verilator_build @@ -54,6 +55,7 @@ bebop_verilator_sim, bebop_verilator_run, bebop_verilator_batch, + bebop_difftest_run, verilator_clean, verilator_verilog, verilator_build, diff --git a/bbdev/mcp/tools/bebop_difftest_run.py b/bbdev/mcp/tools/bebop_difftest_run.py new file mode 100644 index 000000000..5122ca52a --- /dev/null +++ b/bbdev/mcp/tools/bebop_difftest_run.py @@ -0,0 +1,41 @@ +"""MCP tool: bbdev_bebop_difftest_run.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from common import submit, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_difftest_run( + chip: str, + config: str, + binary: str, + jobs: int = 16, + no_wave: bool = True, + itrace: bool = False, + mtrace: bool = False, + pmctrace: bool = False, + ctrace: bool = False, + banktrace: bool = False, + log_dir: Optional[str] = None, + ) -> str: + """Build and run Verilator+BEMU Bank DiffTest. POST /bebop/difftest/run.""" + for name, value in (("chip", chip), ("config", config), ("binary", binary)): + if error := need(name, value): + return err(error) + params: Dict[str, Any] = { + "chip": chip, + "config": config, + "binary": binary, + "jobs": jobs, + "no-wave": no_wave, + "itrace": itrace, + "mtrace": mtrace, + "pmctrace": pmctrace, + "ctrace": ctrace, + "banktrace": banktrace, + } + return fmt(submit("/bebop/difftest/run", opt(params, **{"log-dir": log_dir}))) From e5d3fa3209605005c5b5813c031c6eca9bb0f59b Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Tue, 11 Aug 2026 01:16:03 +0800 Subject: [PATCH 110/196] refactor: make difftest workflow backend-oriented --- .../{bebop => }/difftest/01_run_api.step.py | 14 ++++----- .../{bebop => }/difftest/01_run_event.step.py | 30 +++++++++---------- bbdev/api/tests/test_all_api_serial.py | 2 +- ...bebop_difftest.py => test_api_difftest.py} | 2 +- bbdev/bbdev | 4 +-- bbdev/mcp/tools/__init__.py | 4 +-- ...{bebop_difftest_run.py => difftest_run.py} | 8 ++--- 7 files changed, 32 insertions(+), 32 deletions(-) rename bbdev/api/steps/{bebop => }/difftest/01_run_api.step.py (85%) rename bbdev/api/steps/{bebop => }/difftest/01_run_event.step.py (90%) rename bbdev/api/tests/{test_api_bebop_difftest.py => test_api_difftest.py} (85%) rename bbdev/mcp/tools/{bebop_difftest_run.py => difftest_run.py} (80%) diff --git a/bbdev/api/steps/bebop/difftest/01_run_api.step.py b/bbdev/api/steps/difftest/01_run_api.step.py similarity index 85% rename from bbdev/api/steps/bebop/difftest/01_run_api.step.py rename to bbdev/api/steps/difftest/01_run_api.step.py index cc8128d56..293dbd938 100644 --- a/bbdev/api/steps/bebop/difftest/01_run_api.step.py +++ b/bbdev/api/steps/difftest/01_run_api.step.py @@ -6,11 +6,11 @@ config = { - "name": "bebop-difftest-run-api", - "description": "Run Verilator+BEMU Bank DiffTest", - "flows": ["bebop"], - "triggers": [api("POST", "/bebop/difftest/run")], - "enqueues": ["bebop.difftest.run"], + "name": "difftest-run-api", + "description": "Run Bank DiffTest on Verilator RTL with BEMU as the reference model", + "flows": ["difftest"], + "triggers": [api("POST", "/difftest/run")], + "enqueues": ["difftest.run"], } @@ -55,7 +55,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: if not manifest.is_file(): return ApiResponse( status=400, - body={"success": False, "error": f"Chip BEMU manifest does not exist: {manifest}"}, + body={"success": False, "error": f"Chip emulator manifest does not exist: {manifest}"}, ) data = { @@ -75,5 +75,5 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "log_dir": body.get("log-dir") or body.get("log_dir"), "_trace_id": ctx.trace_id, } - await ctx.enqueue({"topic": "bebop.difftest.run", "data": data}) + await ctx.enqueue({"topic": "difftest.run", "data": data}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/difftest/01_run_event.step.py b/bbdev/api/steps/difftest/01_run_event.step.py similarity index 90% rename from bbdev/api/steps/bebop/difftest/01_run_event.step.py rename to bbdev/api/steps/difftest/01_run_event.step.py index 0bea2ef03..1556d8df2 100644 --- a/bbdev/api/steps/bebop/difftest/01_run_event.step.py +++ b/bbdev/api/steps/difftest/01_run_event.step.py @@ -1,4 +1,4 @@ -"""Build and run a Chip-specific Verilator+BEMU Bank DiffTest workload.""" +"""Run Bank DiffTest on Verilator RTL using BEMU as the reference model.""" import os import shlex @@ -8,7 +8,7 @@ from motia import FlowContext, queue -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) @@ -19,10 +19,10 @@ config = { - "name": "bebop-difftest-run", - "description": "Build and run Verilator+BEMU Bank DiffTest", - "flows": ["bebop"], - "triggers": [queue("bebop.difftest.run")], + "name": "difftest-run", + "description": "Run Bank DiffTest on Verilator RTL with BEMU as the reference model", + "flows": ["difftest"], + "triggers": [queue("difftest.run")], "enqueues": [], } @@ -105,8 +105,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cmd="nix build", logger=ctx.logger, cwd=bbdir, - stdout_prefix="bebop difftest environment", - stderr_prefix="bebop difftest environment", + stdout_prefix="difftest environment", + stderr_prefix="difftest environment", ) if env_result.returncode != 0: await check_result( @@ -115,7 +115,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "environment", - "backend": "verilator+bemu-difftest", + "backend": "verilator", "chip": chip, "config": arch_config, "dependency": str(dramsim_header), @@ -145,8 +145,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cmd=build_cmd, logger=ctx.logger, cwd=bbdir, - stdout_prefix="bebop difftest build", - stderr_prefix="bebop difftest build", + stdout_prefix="difftest build", + stderr_prefix="difftest build", env={**os.environ, "VSRC_PATH": vsrc_dir}, ) if build_result.returncode != 0: @@ -156,7 +156,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "build", - "backend": "verilator+bemu-difftest", + "backend": "verilator", "chip": chip, "config": arch_config, "binary": str(bebop_bin), @@ -206,8 +206,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cmd=run_cmd, logger=ctx.logger, cwd=bbdir, - stdout_prefix="bebop difftest", - stderr_prefix="bebop difftest", + stdout_prefix="difftest", + stderr_prefix="difftest", ) await check_result( ctx, @@ -215,7 +215,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "difftest", - "backend": "verilator+bemu-difftest", + "backend": "verilator", "chip": chip, "config": arch_config, "binary": binary_path, diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index 37bc8763f..a85b211fd 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -32,7 +32,7 @@ def main(): "test_api_bebop_verilator_sim.py", "test_api_bebop_verilator_run.py", "test_api_bebop_verilator_batch.py", - "test_api_bebop_difftest.py", + "test_api_difftest.py", "test_api_bebop_p2e_clean.py", "test_api_bebop_p2e_verilog.py", "test_api_bebop_p2e_buildbitstream.py", diff --git a/bbdev/api/tests/test_api_bebop_difftest.py b/bbdev/api/tests/test_api_difftest.py similarity index 85% rename from bbdev/api/tests/test_api_bebop_difftest.py rename to bbdev/api/tests/test_api_difftest.py index 986f62d20..731694be6 100644 --- a/bbdev/api/tests/test_api_bebop_difftest.py +++ b/bbdev/api/tests/test_api_difftest.py @@ -2,7 +2,7 @@ run_bbdev_case( - "bbdev bebop-difftest --run " + "bbdev difftest --run " "'--chip toy --config sims.verilator.BuckyballToyVerilatorConfig " "--binary toy_im2col_k3_test-singlecore-baremetal --no-wave'" ) diff --git a/bbdev/bbdev b/bbdev/bbdev index abfb7151f..3392a02aa 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -70,8 +70,8 @@ WORKFLOW_COMMANDS = { "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--rushB] [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--rushB] [--clean-before]"', }, - "bebop-difftest": { - "run": 'Run Verilator+BEMU Bank DiffTest. Args: "--chip --config --binary [--jobs ] [--log-dir ] [--no-wave]"', + "difftest": { + "run": 'Use BEMU as the reference model to run Bank DiffTest on Verilator RTL. Args: "--chip --config --binary [--jobs ] [--log-dir ] [--no-wave]"', }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--config [--output-dir ]"', diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index c8483d527..9b1d9fbce 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -16,7 +16,7 @@ from . import bebop_verilator_sim from . import bebop_verilator_run from . import bebop_verilator_batch -from . import bebop_difftest_run +from . import difftest_run from . import verilator_clean from . import verilator_verilog from . import verilator_build @@ -55,7 +55,7 @@ bebop_verilator_sim, bebop_verilator_run, bebop_verilator_batch, - bebop_difftest_run, + difftest_run, verilator_clean, verilator_verilog, verilator_build, diff --git a/bbdev/mcp/tools/bebop_difftest_run.py b/bbdev/mcp/tools/difftest_run.py similarity index 80% rename from bbdev/mcp/tools/bebop_difftest_run.py rename to bbdev/mcp/tools/difftest_run.py index 5122ca52a..3c9b487c8 100644 --- a/bbdev/mcp/tools/bebop_difftest_run.py +++ b/bbdev/mcp/tools/difftest_run.py @@ -1,4 +1,4 @@ -"""MCP tool: bbdev_bebop_difftest_run.""" +"""MCP tool: bbdev_difftest_run.""" from __future__ import annotations @@ -9,7 +9,7 @@ def register(mcp): @mcp.tool() - def bbdev_bebop_difftest_run( + def bbdev_difftest_run( chip: str, config: str, binary: str, @@ -22,7 +22,7 @@ def bbdev_bebop_difftest_run( banktrace: bool = False, log_dir: Optional[str] = None, ) -> str: - """Build and run Verilator+BEMU Bank DiffTest. POST /bebop/difftest/run.""" + """Run Bank DiffTest on Verilator RTL using BEMU as reference. POST /difftest/run.""" for name, value in (("chip", chip), ("config", config), ("binary", binary)): if error := need(name, value): return err(error) @@ -38,4 +38,4 @@ def bbdev_bebop_difftest_run( "ctrace": ctrace, "banktrace": banktrace, } - return fmt(submit("/bebop/difftest/run", opt(params, **{"log-dir": log_dir}))) + return fmt(submit("/difftest/run", opt(params, **{"log-dir": log_dir}))) From 262878cd32bab737289c28b503c8557ad2768b43 Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Tue, 11 Aug 2026 01:25:01 +0800 Subject: [PATCH 111/196] feat: select difftest backend explicitly --- bbdev/api/steps/difftest/01_run_api.step.py | 38 ++++++++++++++++- bbdev/api/steps/difftest/01_run_event.step.py | 21 ++++++++-- bbdev/api/tests/test_api_difftest.py | 3 +- bbdev/bbdev | 2 +- bbdev/mcp/tools/difftest_run.py | 41 ++++++++++++++----- 5 files changed, 86 insertions(+), 19 deletions(-) diff --git a/bbdev/api/steps/difftest/01_run_api.step.py b/bbdev/api/steps/difftest/01_run_api.step.py index 293dbd938..8319b2004 100644 --- a/bbdev/api/steps/difftest/01_run_api.step.py +++ b/bbdev/api/steps/difftest/01_run_api.step.py @@ -7,7 +7,7 @@ config = { "name": "difftest-run-api", - "description": "Run Bank DiffTest on Verilator RTL with BEMU as the reference model", + "description": "Run Bank DiffTest on a selected backend with BEMU as the reference model", "flows": ["difftest"], "triggers": [api("POST", "/difftest/run")], "enqueues": ["difftest.run"], @@ -23,7 +23,40 @@ def _required_string(body: dict, name: str) -> str | None: async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - required = {name: _required_string(body, name) for name in ("chip", "config", "binary")} + backend = _required_string(body, "backend") + if backend is None: + return ApiResponse( + status=400, + body={ + "success": False, + "error": "Missing required parameter(s): --backend", + }, + ) + + known_backends = ("verilator", "p2e") + if backend not in known_backends: + return ApiResponse( + status=400, + body={ + "success": False, + "error": ( + f"Unsupported DiffTest backend: {backend}; " + f"available backends: {', '.join(known_backends)}" + ), + }, + ) + if backend == "p2e": + return ApiResponse( + status=501, + body={ + "success": False, + "error": "DiffTest backend p2e is reserved but not implemented yet", + }, + ) + + required = { + name: _required_string(body, name) for name in ("chip", "config", "binary") + } missing = [name for name, value in required.items() if value is None] if missing: missing_args = ", ".join("--" + name for name in missing) @@ -59,6 +92,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: ) data = { + "backend": backend, **required, "jobs": body.get("jobs", 16), "vsrc_dir": get_verilator_build_dir( diff --git a/bbdev/api/steps/difftest/01_run_event.step.py b/bbdev/api/steps/difftest/01_run_event.step.py index 1556d8df2..2c24beb32 100644 --- a/bbdev/api/steps/difftest/01_run_event.step.py +++ b/bbdev/api/steps/difftest/01_run_event.step.py @@ -58,6 +58,19 @@ async def fail(ctx, trace_id: str, error: str, **fields) -> None: async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) + backend = input_data.get("backend") + if backend != "verilator": + error = "backend_not_implemented" if backend == "p2e" else "unsupported_backend" + await fail( + ctx, + origin_tid, + error, + backend=backend, + available_backends=["verilator", "p2e"], + implemented_backends=["verilator"], + ) + return + bbdir = get_buckyball_path() chip = input_data["chip"] arch_config = input_data["config"] @@ -115,7 +128,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "environment", - "backend": "verilator", + "backend": backend, "chip": chip, "config": arch_config, "dependency": str(dramsim_header), @@ -156,7 +169,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "build", - "backend": "verilator", + "backend": backend, "chip": chip, "config": arch_config, "binary": str(bebop_bin), @@ -177,7 +190,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: run_args = [ str(bebop_bin), "run", - "verilator", + backend, f"--elf={binary_path}", f"--log-dir={log_dir}", "--diff", @@ -215,7 +228,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "difftest", - "backend": "verilator", + "backend": backend, "chip": chip, "config": arch_config, "binary": binary_path, diff --git a/bbdev/api/tests/test_api_difftest.py b/bbdev/api/tests/test_api_difftest.py index 731694be6..24ae9e4a8 100644 --- a/bbdev/api/tests/test_api_difftest.py +++ b/bbdev/api/tests/test_api_difftest.py @@ -3,6 +3,7 @@ run_bbdev_case( "bbdev difftest --run " - "'--chip toy --config sims.verilator.BuckyballToyVerilatorConfig " + "'--backend verilator --chip toy " + "--config sims.verilator.BuckyballToyVerilatorConfig " "--binary toy_im2col_k3_test-singlecore-baremetal --no-wave'" ) diff --git a/bbdev/bbdev b/bbdev/bbdev index 3392a02aa..d52679f28 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -71,7 +71,7 @@ WORKFLOW_COMMANDS = { "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--rushB] [--clean-before]"', }, "difftest": { - "run": 'Use BEMU as the reference model to run Bank DiffTest on Verilator RTL. Args: "--chip --config --binary [--jobs ] [--log-dir ] [--no-wave]"', + "run": 'Use BEMU as the reference model to run Bank DiffTest on a selected backend. Args: "--backend --chip --config --binary [--jobs ] [--log-dir ] [--no-wave]"; p2e is reserved but not implemented yet.', }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--config [--output-dir ]"', diff --git a/bbdev/mcp/tools/difftest_run.py b/bbdev/mcp/tools/difftest_run.py index 3c9b487c8..3e91c268b 100644 --- a/bbdev/mcp/tools/difftest_run.py +++ b/bbdev/mcp/tools/difftest_run.py @@ -10,9 +10,10 @@ def register(mcp): @mcp.tool() def bbdev_difftest_run( - chip: str, - config: str, - binary: str, + backend: str, + chip: Optional[str] = None, + config: Optional[str] = None, + binary: Optional[str] = None, jobs: int = 16, no_wave: bool = True, itrace: bool = False, @@ -22,14 +23,21 @@ def bbdev_difftest_run( banktrace: bool = False, log_dir: Optional[str] = None, ) -> str: - """Run Bank DiffTest on Verilator RTL using BEMU as reference. POST /difftest/run.""" - for name, value in (("chip", chip), ("config", config), ("binary", binary)): - if error := need(name, value): - return err(error) + """Run Bank DiffTest on a selected backend using BEMU as reference. POST /difftest/run.""" + if error := need("backend", backend): + return err(error) + known_backends = ("verilator", "p2e") + if backend not in known_backends: + return err( + f"Unsupported DiffTest backend: {backend}; " + f"available backends: {', '.join(known_backends)}" + ) + if backend == "verilator": + for name, value in (("chip", chip), ("config", config), ("binary", binary)): + if error := need(name, value): + return err(error) params: Dict[str, Any] = { - "chip": chip, - "config": config, - "binary": binary, + "backend": backend, "jobs": jobs, "no-wave": no_wave, "itrace": itrace, @@ -38,4 +46,15 @@ def bbdev_difftest_run( "ctrace": ctrace, "banktrace": banktrace, } - return fmt(submit("/difftest/run", opt(params, **{"log-dir": log_dir}))) + return fmt( + submit( + "/difftest/run", + opt( + params, + chip=chip, + config=config, + binary=binary, + **{"log-dir": log_dir}, + ), + ) + ) From 5620cec4cff922345e2b5e8a0da7ca5e6fc704aa Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Tue, 11 Aug 2026 13:34:23 +0800 Subject: [PATCH 112/196] refactor: integrate difftest into Bebop Verilator run --- .../bebop/verilator/03_build_event.step.py | 98 ++++++- .../bebop/verilator/04_sim_event.step.py | 79 +++++- .../steps/bebop/verilator/05_run_api.step.py | 13 +- .../bebop/verilator/scripts/build_marker.py | 9 +- bbdev/api/steps/difftest/01_run_api.step.py | 113 -------- bbdev/api/steps/difftest/01_run_event.step.py | 241 ------------------ bbdev/api/tests/test_all_api_serial.py | 1 - .../api/tests/test_api_bebop_verilator_run.py | 2 +- bbdev/api/tests/test_api_difftest.py | 9 - bbdev/api/utils/path.py | 30 +++ bbdev/bbdev | 5 +- bbdev/mcp/tools/__init__.py | 2 - bbdev/mcp/tools/bebop_verilator_run.py | 4 + bbdev/mcp/tools/bebop_verilator_sim.py | 2 + bbdev/mcp/tools/difftest_run.py | 60 ----- 15 files changed, 218 insertions(+), 450 deletions(-) delete mode 100644 bbdev/api/steps/difftest/01_run_api.step.py delete mode 100644 bbdev/api/steps/difftest/01_run_event.step.py delete mode 100644 bbdev/api/tests/test_api_difftest.py delete mode 100644 bbdev/mcp/tools/difftest_run.py diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index b11413ffd..624231624 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -16,7 +16,7 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.path import get_buckyball_path, get_chip_from_config, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id from build_marker import write_build_marker @@ -60,25 +60,97 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return + diff = bool(input_data.get("diff", False)) + build_dir = bebop_dir + chip = None + features = ["verilator"] + if diff: + try: + chip = get_chip_from_config(bbdir, arch_config) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "chip_resolution_failed", "detail": str(error)}, + trace_id=origin_tid, + ) + return + + build_dir = os.path.join(bbdir, "examples", "chips", chip, "emu") + manifest = os.path.join(build_dir, "Cargo.toml") + if not os.path.isfile(manifest): + ctx.logger.error(f"Chip emulator manifest does not exist: {manifest}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "chip_manifest_not_found", "manifest": manifest}, + trace_id=origin_tid, + ) + return + features.extend(["bemu", "difftest"]) + + dramsim_header = os.path.join(bbdir, "result", "include", "dramsim3.h") + if not os.path.isfile(dramsim_header): + ctx.logger.info(f"Nix environment is missing {dramsim_header}; rebuilding result") + env_result = stream_run_logger( + cmd="nix build", + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="bebop verilator difftest environment", + stderr_prefix="bebop verilator difftest environment", + ) + if env_result.returncode != 0: + await check_result( + ctx, + env_result.returncode, + continue_run=False, + extra_fields={ + "task": "environment", + "diff": True, + "dependency": dramsim_header, + }, + trace_id=origin_tid, + ) + return + jobs = input_data.get("jobs", 16) - build_cmd = ( - f"cargo build --release --bin bebop --features verilator --jobs {shlex.quote(str(jobs))}" + cargo_build_cmd = shlex.join( + [ + "cargo", + "build", + "--release", + "--bin", + "bebop", + "--features", + ",".join(features), + "--jobs", + str(jobs), + ] ) + command_cwd = build_dir + build_cmd = cargo_build_cmd + if diff: + build_inner = f"cd {shlex.quote(build_dir)} && exec {cargo_build_cmd}" + build_cmd = f"nix develop -c sh -c {shlex.quote(build_inner)}" + command_cwd = bbdir env = {**os.environ, "VSRC_PATH": vsrc_dir} - ctx.logger.info("Building bebop verilator ...") + ctx.logger.info(f"Building bebop verilator (diff={diff}) ...") build_result = stream_run_logger( cmd=build_cmd, logger=ctx.logger, - cwd=bebop_dir, + cwd=command_cwd, stdout_prefix="bebop verilator build", stderr_prefix="bebop verilator build", env=env, ) - bebop_bin = f"{bebop_dir}/target/release/bebop" + bebop_bin = os.path.join(build_dir, "target", "release", "bebop") if build_result.returncode == 0: try: - write_build_marker(bebop_dir, arch_config, vsrc_dir, bebop_bin) + write_build_marker(build_dir, arch_config, vsrc_dir, bebop_bin, diff=diff) except OSError as e: ctx.logger.error(f"failed to write bebop verilator build marker: {e}") await check_result( @@ -97,6 +169,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "config": arch_config, "vsrc_dir": vsrc_dir, "binary": bebop_bin, + "diff": diff, + "chip": chip, }, trace_id=origin_tid, ) @@ -106,5 +180,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # Continue routing to sim if from run workflow if input_data.get("from_run_workflow"): await ctx.enqueue( - {"topic": "bebop.verilator.run.sim", "data": {**input_data, "vsrc_dir": vsrc_dir, "task": "run"}} + { + "topic": "bebop.verilator.run.sim", + "data": { + **input_data, + "vsrc_dir": vsrc_dir, + "chip": chip, + "task": "run", + }, + } ) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index b8807b0ab..8c0b4cb65 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -18,7 +18,7 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.path import get_buckyball_path, get_chip_from_config, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id @@ -56,6 +56,20 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + + diff = bool(input_data.get("diff", False)) + batch = bool(input_data.get("batch", False)) + if diff and input_data.get("rushB"): + ctx.logger.error("--diff and --rushB cannot be used together") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "diff_conflicts_with_rushB"}, + trace_id=origin_tid, + ) + return + vsrc_dir = get_verilator_build_dir(bbdir, arch_config, input_data.get("vsrc_dir")) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") if not os.path.isdir(vsrc_dir): @@ -67,7 +81,24 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - bebop_bin = f"{bebop_dir}/target/release/bebop" + build_dir = bebop_dir + chip = input_data.get("chip") + if diff: + try: + chip = chip or get_chip_from_config(bbdir, arch_config) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "chip_resolution_failed", "detail": str(error)}, + trace_id=origin_tid, + ) + return + build_dir = os.path.join(bbdir, "examples", "chips", chip, "emu") + + bebop_bin = os.path.join(build_dir, "target", "release", "bebop") if not os.path.isfile(bebop_bin): ctx.logger.error(f"bebop binary does not exist: {bebop_bin}") await check_result( @@ -77,9 +108,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - marker_path = build_marker_path(bebop_dir) + marker_path = build_marker_path(build_dir) try: - marker = read_build_marker(bebop_dir) + marker = read_build_marker(build_dir) except FileNotFoundError: ctx.logger.error(f"bebop verilator build marker does not exist: {marker_path}") await check_result( @@ -103,6 +134,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: marker.get("config") != arch_config or marker.get("vsrc_dir") != expect_vsrc or marker.get("binary") != expect_bin + or bool(marker.get("diff", False)) != diff ): ctx.logger.error(f"bebop verilator build marker mismatch: {marker}") await check_result( @@ -114,6 +146,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "config": arch_config, "vsrc_dir": expect_vsrc, "binary": expect_bin, + "diff": diff, }, }, trace_id=origin_tid, @@ -121,7 +154,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return binary_name = input_data.get("binary", "") - binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) + workload_root = f"{bbdir}/bb-tests/output/workloads/src" + binary_path = None + if diff and chip: + binary_path = search_workload( + f"{workload_root}/CTest/chips/{chip}", binary_name + ) + if binary_path is None: + binary_path = search_workload(workload_root, binary_name) if binary_path is None: ctx.logger.error(f"binary not found: {binary_name}") await check_result( @@ -133,7 +173,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"binary_path: {binary_path}") timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - log_dir = f"{bbdir}/log/{timestamp}-verilator-{binary_name}" + mode = "difftest" if diff else "verilator" + log_dir = f"{bbdir}/log/{timestamp}-{mode}-{binary_name}" os.makedirs(log_dir, exist_ok=True) wave_arg = " --no-wave" if input_data.get("no-wave", False) or input_data.get("no_wave", False) else "" @@ -170,12 +211,28 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"{shlex.quote(bebop_bin)} run verilator " f"--elf={shlex.quote(binary_path)} " f"--log-dir={shlex.quote(log_dir)}" + f"{' --diff' if diff else ''}" + f"{' --batch' if batch else ''}" f"{wave_arg}" f"{trace_args}" ) - ctx.logger.info(f"Running bebop verilator: {run_cmd}") - stdout_prefix = "bebop verilator" - stderr_prefix = "bebop verilator" + if diff: + preload = os.pathsep.join( + path + for path in ( + f"{bbdir}/result/lib/libdramsim3.so", + os.environ.get("LD_PRELOAD", ""), + ) + if path + ) + run_inner = ( + f"cd {shlex.quote(build_dir)} && " + f"export LD_PRELOAD={shlex.quote(preload)} && exec {run_cmd}" + ) + run_cmd = f"nix develop -c sh -c {shlex.quote(run_inner)}" + ctx.logger.info(f"Running bebop verilator (diff={diff}): {run_cmd}") + stdout_prefix = "bebop verilator difftest" if diff else "bebop verilator" + stderr_prefix = stdout_prefix run_result = stream_run_logger( cmd=run_cmd, logger=ctx.logger, @@ -191,9 +248,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "task": "verilator", "backend": "rushB" if input_data.get("rushB") else "guest-elf", + "diff": diff, + "batch": batch, + "chip": chip, "binary": binary_path, "config": arch_config, "log_dir": log_dir, + "bank_diff": os.path.join(log_dir, "bank_diff.ndjson") if diff else None, "timestamp": timestamp, }, trace_id=origin_tid, diff --git a/bbdev/api/steps/bebop/verilator/05_run_api.step.py b/bbdev/api/steps/bebop/verilator/05_run_api.step.py index 772a906ef..003d5776b 100644 --- a/bbdev/api/steps/bebop/verilator/05_run_api.step.py +++ b/bbdev/api/steps/bebop/verilator/05_run_api.step.py @@ -37,6 +37,14 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: }, ) + diff = body.get("diff", False) + rushb = body.get("rushB", False) + if diff and rushb: + return ApiResponse( + status=400, + body={"error": "--diff and --rushB cannot be used together"}, + ) + data = { "config": config_name, "binary": binary, @@ -46,7 +54,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "pmctrace": body.get("pmctrace", False), "ctrace": body.get("ctrace", False), "banktrace": body.get("banktrace", False), - "rushB": body.get("rushB", False), + "rushB": rushb, + "diff": diff, + "batch": body.get("batch", False), + "no-wave": body.get("no-wave", body.get("no_wave", False)), "jobs": body.get("jobs", 16), "from_run_workflow": True, } diff --git a/bbdev/api/steps/bebop/verilator/scripts/build_marker.py b/bbdev/api/steps/bebop/verilator/scripts/build_marker.py index ec726443a..17fb887d2 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/build_marker.py +++ b/bbdev/api/steps/bebop/verilator/scripts/build_marker.py @@ -6,13 +6,20 @@ def build_marker_path(bebop_dir: str) -> str: return os.path.join(bebop_dir, "target", "debug", ".bbdev-verilator-build.json") -def write_build_marker(bebop_dir: str, config: str, vsrc_dir: str, binary: str): +def write_build_marker( + bebop_dir: str, + config: str, + vsrc_dir: str, + binary: str, + diff: bool = False, +): marker = build_marker_path(bebop_dir) os.makedirs(os.path.dirname(marker), exist_ok=True) data = { "config": config, "vsrc_dir": os.path.abspath(vsrc_dir), "binary": os.path.abspath(binary), + "diff": diff, } tmp = f"{marker}.tmp" with open(tmp, "w") as f: diff --git a/bbdev/api/steps/difftest/01_run_api.step.py b/bbdev/api/steps/difftest/01_run_api.step.py deleted file mode 100644 index 8319b2004..000000000 --- a/bbdev/api/steps/difftest/01_run_api.step.py +++ /dev/null @@ -1,113 +0,0 @@ -from pathlib import Path - -from motia import ApiRequest, ApiResponse, FlowContext, api - -from utils.path import get_buckyball_path, get_verilator_build_dir - - -config = { - "name": "difftest-run-api", - "description": "Run Bank DiffTest on a selected backend with BEMU as the reference model", - "flows": ["difftest"], - "triggers": [api("POST", "/difftest/run")], - "enqueues": ["difftest.run"], -} - - -def _required_string(body: dict, name: str) -> str | None: - value = body.get(name) - if not isinstance(value, str) or not value.strip() or value == "None": - return None - return value - - -async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = request.body or {} - backend = _required_string(body, "backend") - if backend is None: - return ApiResponse( - status=400, - body={ - "success": False, - "error": "Missing required parameter(s): --backend", - }, - ) - - known_backends = ("verilator", "p2e") - if backend not in known_backends: - return ApiResponse( - status=400, - body={ - "success": False, - "error": ( - f"Unsupported DiffTest backend: {backend}; " - f"available backends: {', '.join(known_backends)}" - ), - }, - ) - if backend == "p2e": - return ApiResponse( - status=501, - body={ - "success": False, - "error": "DiffTest backend p2e is reserved but not implemented yet", - }, - ) - - required = { - name: _required_string(body, name) for name in ("chip", "config", "binary") - } - missing = [name for name, value in required.items() if value is None] - if missing: - missing_args = ", ".join("--" + name for name in missing) - return ApiResponse( - status=400, - body={ - "success": False, - "error": f"Missing required parameter(s): {missing_args}", - }, - ) - - unsupported = [name for name in ("rushB", "pk", "fast") if body.get(name)] - if unsupported: - unsupported_args = ", ".join("--" + name for name in unsupported) - return ApiResponse( - status=400, - body={ - "success": False, - "error": ( - "Bank DiffTest only supports normal baremetal ELF mode; " - f"unsupported: {unsupported_args}" - ), - }, - ) - - bbdir = get_buckyball_path() - chip = required["chip"] - manifest = Path(bbdir) / "examples" / "chips" / chip / "emu" / "Cargo.toml" - if not manifest.is_file(): - return ApiResponse( - status=400, - body={"success": False, "error": f"Chip emulator manifest does not exist: {manifest}"}, - ) - - data = { - "backend": backend, - **required, - "jobs": body.get("jobs", 16), - "vsrc_dir": get_verilator_build_dir( - bbdir, - required["config"], - body.get("vsrc-dir") or body.get("output-dir"), - ), - "no_wave": body.get("no-wave", body.get("no_wave", False)), - "itrace": body.get("itrace", False), - "mtrace": body.get("mtrace", False), - "pmctrace": body.get("pmctrace", False), - "ctrace": body.get("ctrace", False), - "banktrace": body.get("banktrace", False), - "log_dir": body.get("log-dir") or body.get("log_dir"), - "_trace_id": ctx.trace_id, - } - await ctx.enqueue({"topic": "difftest.run", "data": data}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/difftest/01_run_event.step.py b/bbdev/api/steps/difftest/01_run_event.step.py deleted file mode 100644 index 2c24beb32..000000000 --- a/bbdev/api/steps/difftest/01_run_event.step.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Run Bank DiffTest on Verilator RTL using BEMU as the reference model.""" - -import os -import shlex -import sys -from datetime import datetime -from pathlib import Path - -from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.event_common import check_result, get_origin_trace_id -from utils.path import get_buckyball_path -from utils.search_workload import search_workload, search_workload_all -from utils.stream_run import stream_run_logger - - -config = { - "name": "difftest-run", - "description": "Run Bank DiffTest on Verilator RTL with BEMU as the reference model", - "flows": ["difftest"], - "triggers": [queue("difftest.run")], - "enqueues": [], -} - - -def resolve_workload(bbdir: str, chip: str, binary: str) -> str | None: - candidate = Path(binary) - if not candidate.is_absolute(): - candidate = Path(bbdir) / candidate - if candidate.is_file(): - return str(candidate.resolve()) - - search_root = f"{bbdir}/bb-tests/output/workloads/src" - chip_match = search_workload(f"{search_root}/CTest/chips/{chip}", binary) - if chip_match is not None: - return chip_match - - matches = search_workload_all(search_root, binary) - chip_marker = f"{os.path.sep}CTest{os.path.sep}chips{os.path.sep}" - non_chip_matches = [path for path in matches if chip_marker not in path] - return non_chip_matches[0] if len(non_chip_matches) == 1 else None - - -async def fail(ctx, trace_id: str, error: str, **fields) -> None: - ctx.logger.error(fields.pop("message", error)) - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": error, **fields}, - trace_id=trace_id, - ) - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - backend = input_data.get("backend") - if backend != "verilator": - error = "backend_not_implemented" if backend == "p2e" else "unsupported_backend" - await fail( - ctx, - origin_tid, - error, - backend=backend, - available_backends=["verilator", "p2e"], - implemented_backends=["verilator"], - ) - return - - bbdir = get_buckyball_path() - chip = input_data["chip"] - arch_config = input_data["config"] - binary_name = input_data["binary"] - - chip_emu_dir = Path(bbdir) / "examples" / "chips" / chip / "emu" - manifest = chip_emu_dir / "Cargo.toml" - if not manifest.is_file(): - await fail(ctx, origin_tid, "invalid_chip", chip=chip, manifest=str(manifest)) - return - - vsrc_dir = os.path.abspath(input_data["vsrc_dir"]) - if not os.path.isdir(vsrc_dir): - await fail( - ctx, - origin_tid, - "vsrc_not_found", - config=arch_config, - vsrc_dir=vsrc_dir, - message=f"RTL directory does not exist: {vsrc_dir}; generate Verilog first", - ) - return - - binary_path = resolve_workload(bbdir, chip, binary_name) - if binary_path is None: - await fail(ctx, origin_tid, "binary_not_found", chip=chip, binary=binary_name) - return - - try: - jobs = int(input_data.get("jobs", 16)) - if jobs <= 0: - raise ValueError - except (TypeError, ValueError): - await fail(ctx, origin_tid, "invalid_jobs", jobs=input_data.get("jobs")) - return - - # sourceme.sh points RISCV at the repository's `result` environment. Keep - # it current when a newly required native dependency is not present. - dramsim_header = Path(bbdir) / "result" / "include" / "dramsim3.h" - if not dramsim_header.is_file(): - ctx.logger.info( - f"Nix environment is missing {dramsim_header}; rebuilding result" - ) - env_result = stream_run_logger( - cmd="nix build", - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="difftest environment", - stderr_prefix="difftest environment", - ) - if env_result.returncode != 0: - await check_result( - ctx, - env_result.returncode, - continue_run=False, - extra_fields={ - "task": "environment", - "backend": backend, - "chip": chip, - "config": arch_config, - "dependency": str(dramsim_header), - }, - trace_id=origin_tid, - ) - return - - bebop_bin = chip_emu_dir / "target" / "release" / "bebop" - cargo_build_cmd = shlex.join( - [ - "cargo", - "build", - "--release", - "--bin", - "bebop", - "--features", - "verilator,bemu,difftest", - "--jobs", - str(jobs), - ] - ) - build_inner = f"cd {shlex.quote(str(chip_emu_dir))} && exec {cargo_build_cmd}" - build_cmd = f"nix develop -c sh -c {shlex.quote(build_inner)}" - ctx.logger.info(f"Building Chip DiffTest executable: {build_cmd}") - build_result = stream_run_logger( - cmd=build_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="difftest build", - stderr_prefix="difftest build", - env={**os.environ, "VSRC_PATH": vsrc_dir}, - ) - if build_result.returncode != 0: - await check_result( - ctx, - build_result.returncode, - continue_run=False, - extra_fields={ - "task": "build", - "backend": backend, - "chip": chip, - "config": arch_config, - "binary": str(bebop_bin), - "vsrc_dir": vsrc_dir, - }, - trace_id=origin_tid, - ) - return - - timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") - requested_log_dir = input_data.get("log_dir") - if requested_log_dir: - log_dir = os.path.abspath(os.path.join(bbdir, requested_log_dir)) - else: - log_dir = f"{bbdir}/log/{timestamp}-difftest-{binary_name}" - os.makedirs(log_dir, exist_ok=True) - - run_args = [ - str(bebop_bin), - "run", - backend, - f"--elf={binary_path}", - f"--log-dir={log_dir}", - "--diff", - ] - if input_data.get("no_wave"): - run_args.append("--no-wave") - for trace_name in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace"): - if input_data.get(trace_name): - run_args.append(f"--{trace_name}") - - preload = os.pathsep.join( - path - for path in ( - f"{bbdir}/result/lib/libdramsim3.so", - os.environ.get("LD_PRELOAD", ""), - ) - if path - ) - run_inner = ( - f"cd {shlex.quote(str(chip_emu_dir))} && " - f"export LD_PRELOAD={shlex.quote(preload)} && exec {shlex.join(run_args)}" - ) - run_cmd = f"nix develop -c sh -c {shlex.quote(run_inner)}" - ctx.logger.info(f"Running Bank DiffTest: {run_cmd}") - run_result = stream_run_logger( - cmd=run_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="difftest", - stderr_prefix="difftest", - ) - await check_result( - ctx, - run_result.returncode, - continue_run=False, - extra_fields={ - "task": "difftest", - "backend": backend, - "chip": chip, - "config": arch_config, - "binary": binary_path, - "vsrc_dir": vsrc_dir, - "log_dir": log_dir, - "bank_diff": os.path.join(log_dir, "bank_diff.ndjson"), - "timestamp": timestamp, - }, - trace_id=origin_tid, - ) diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index a85b211fd..2f3cd8819 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -32,7 +32,6 @@ def main(): "test_api_bebop_verilator_sim.py", "test_api_bebop_verilator_run.py", "test_api_bebop_verilator_batch.py", - "test_api_difftest.py", "test_api_bebop_p2e_clean.py", "test_api_bebop_p2e_verilog.py", "test_api_bebop_p2e_buildbitstream.py", diff --git a/bbdev/api/tests/test_api_bebop_verilator_run.py b/bbdev/api/tests/test_api_bebop_verilator_run.py index 71ee1d37e..4860a4c78 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_run.py +++ b/bbdev/api/tests/test_api_bebop_verilator_run.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --run '--jobs 16 --binary toy_vecunit_matmul_ones-singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig --batch'") +run_bbdev_case("bbdev bebop-verilator --run '--diff --batch --no-wave --jobs 16 --binary toy_im2col_k3_test-singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig'") diff --git a/bbdev/api/tests/test_api_difftest.py b/bbdev/api/tests/test_api_difftest.py deleted file mode 100644 index 24ae9e4a8..000000000 --- a/bbdev/api/tests/test_api_difftest.py +++ /dev/null @@ -1,9 +0,0 @@ -from _api_test_helper import run_bbdev_case - - -run_bbdev_case( - "bbdev difftest --run " - "'--backend verilator --chip toy " - "--config sims.verilator.BuckyballToyVerilatorConfig " - "--binary toy_im2col_k3_test-singlecore-baremetal --no-wave'" -) diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index 815ce86c3..48ebfc902 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -1,5 +1,6 @@ import os import re +from pathlib import Path def get_buckyball_path(): @@ -23,6 +24,35 @@ def get_verilator_build_dir(bbdir, config=None, output_dir=None): return get_config_build_dir(bbdir, config) +def get_chip_from_config(bbdir, config): + """Resolve the owning chip from a fully qualified Scala config name.""" + if not isinstance(config, str) or not config or config == "None": + raise ValueError("invalid config name") + + config_class = config.rsplit(".", 1)[-1] + declaration = re.compile(rf"\b(?:class|object)\s+{re.escape(config_class)}\b") + chips_root = Path(bbdir) / "examples" / "chips" + matches = [] + for chip_dir in sorted(path for path in chips_root.iterdir() if path.is_dir()): + scala_root = chip_dir / "arch" / "src" / "main" / "scala" + if not scala_root.is_dir(): + continue + for source in scala_root.rglob("*.scala"): + try: + contents = source.read_text(encoding="utf-8") + except OSError: + continue + if declaration.search(contents): + matches.append(chip_dir.name) + break + + if len(matches) != 1: + raise ValueError( + f"config {config} must resolve to exactly one chip; matches={matches}" + ) + return matches[0] + + def sanitize_config_name(config=None): if config and config != "None": name = re.sub(r"[^A-Za-z0-9_.-]+", "_", config).strip("._") diff --git a/bbdev/bbdev b/bbdev/bbdev index d52679f28..edf786471 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -67,12 +67,9 @@ WORKFLOW_COMMANDS = { "verilog": 'Generate Bebop Verilog files. Args: "--config [--output-dir ]"', "build": 'Build Bebop Verilator simulation executable. Args: "--config [--jobs ] [--output-dir ]"', "sim": 'Run Bebop Verilator simulation. Args: "--config --binary [--rushB] [--batch] [--log-dir ] [--fst-dir ] [--no-wave]"', - "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--rushB] [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', + "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--rushB] [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--rushB] [--clean-before]"', }, - "difftest": { - "run": 'Use BEMU as the reference model to run Bank DiffTest on a selected backend. Args: "--backend --chip --config --binary [--jobs ] [--log-dir ] [--no-wave]"; p2e is reserved but not implemented yet.', - }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--config [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--config [--output-dir ]"', diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index 9b1d9fbce..1bc5d93ad 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -16,7 +16,6 @@ from . import bebop_verilator_sim from . import bebop_verilator_run from . import bebop_verilator_batch -from . import difftest_run from . import verilator_clean from . import verilator_verilog from . import verilator_build @@ -55,7 +54,6 @@ bebop_verilator_sim, bebop_verilator_run, bebop_verilator_batch, - difftest_run, verilator_clean, verilator_verilog, verilator_build, diff --git a/bbdev/mcp/tools/bebop_verilator_run.py b/bbdev/mcp/tools/bebop_verilator_run.py index 35a7ae9a0..7680c28b1 100644 --- a/bbdev/mcp/tools/bebop_verilator_run.py +++ b/bbdev/mcp/tools/bebop_verilator_run.py @@ -19,6 +19,8 @@ def bbdev_bebop_verilator_run( banktrace: bool = False, no_wave: bool = False, rushB: bool = False, + diff: bool = False, + batch: bool = False, ) -> str: """Full bebop-verilator flow. POST /bebop/verilator/run.""" for n, v in (("binary", binary), ("config", config)): @@ -38,6 +40,8 @@ def bbdev_bebop_verilator_run( "banktrace": banktrace, "no-wave": no_wave, "rushB": rushB, + "diff": diff, + "batch": batch, }, ) ) diff --git a/bbdev/mcp/tools/bebop_verilator_sim.py b/bbdev/mcp/tools/bebop_verilator_sim.py index dc8cdb375..31765dbc8 100644 --- a/bbdev/mcp/tools/bebop_verilator_sim.py +++ b/bbdev/mcp/tools/bebop_verilator_sim.py @@ -19,6 +19,7 @@ def bbdev_bebop_verilator_sim( banktrace: bool = False, no_wave: bool = False, rushB: bool = False, + batch: bool = False, ) -> str: """Run one workload on bebop-verilator. POST /bebop/verilator/sim.""" for n, v in (("binary", binary), ("config", config)): @@ -34,5 +35,6 @@ def bbdev_bebop_verilator_sim( "banktrace": banktrace, "no-wave": no_wave, "rushB": rushB, + "batch": batch, } return fmt(submit("/bebop/verilator/sim", params)) diff --git a/bbdev/mcp/tools/difftest_run.py b/bbdev/mcp/tools/difftest_run.py deleted file mode 100644 index 3e91c268b..000000000 --- a/bbdev/mcp/tools/difftest_run.py +++ /dev/null @@ -1,60 +0,0 @@ -"""MCP tool: bbdev_difftest_run.""" - -from __future__ import annotations - -from typing import Any, Dict, Optional - -from common import submit, err, fmt, need, opt - - -def register(mcp): - @mcp.tool() - def bbdev_difftest_run( - backend: str, - chip: Optional[str] = None, - config: Optional[str] = None, - binary: Optional[str] = None, - jobs: int = 16, - no_wave: bool = True, - itrace: bool = False, - mtrace: bool = False, - pmctrace: bool = False, - ctrace: bool = False, - banktrace: bool = False, - log_dir: Optional[str] = None, - ) -> str: - """Run Bank DiffTest on a selected backend using BEMU as reference. POST /difftest/run.""" - if error := need("backend", backend): - return err(error) - known_backends = ("verilator", "p2e") - if backend not in known_backends: - return err( - f"Unsupported DiffTest backend: {backend}; " - f"available backends: {', '.join(known_backends)}" - ) - if backend == "verilator": - for name, value in (("chip", chip), ("config", config), ("binary", binary)): - if error := need(name, value): - return err(error) - params: Dict[str, Any] = { - "backend": backend, - "jobs": jobs, - "no-wave": no_wave, - "itrace": itrace, - "mtrace": mtrace, - "pmctrace": pmctrace, - "ctrace": ctrace, - "banktrace": banktrace, - } - return fmt( - submit( - "/difftest/run", - opt( - params, - chip=chip, - config=config, - binary=binary, - **{"log-dir": log_dir}, - ), - ) - ) From 7d2a4dc32f16c4c344c648217f81a9a2e500a633 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 11 Aug 2026 20:20:38 +0800 Subject: [PATCH 113/196] feat: add IP replacement API and enhance workflow commands for behavioral RTL processing --- .../firesim/01_buildbitstream_event.step.py | 2 +- bbdev/api/steps/firesim/scripts/build.sc | 2 + .../api/steps/firesim/scripts/firesim_env.py | 9 +- .../firesim/scripts/makefrag/firesim/build.mk | 10 +- .../scripts/makefrag/firesim/config.mk | 1 + .../firesim/scripts/yaml/config_build.yaml | 4 +- .../firesim/scripts/yaml/config_hwdb.yaml | 10 +- .../firesim/scripts/yaml/config_runtime.yaml | 4 +- bbdev/api/steps/ip-replace/01_run_api.step.py | 33 ++ .../api/steps/ip-replace/01_run_event.step.py | 129 ++++++ .../steps/ip-replace/scripts/sram_replace.py | 412 ++++++++++++++++++ .../steps/mill/03_yosys_verilog_event.step.py | 32 +- .../steps/mill/04_dc_verilog_event.step.py | 29 +- bbdev/api/steps/yosys/01_run_api.step.py | 1 + bbdev/api/steps/yosys/03_synth_api.step.py | 17 +- bbdev/api/steps/yosys/03_synth_event.step.py | 120 ++++- .../api/steps/yosys/scripts/yosys-config.yaml | 20 +- bbdev/api/utils/verilog.py | 12 + bbdev/bbdev | 7 +- bbdev/mcp/tools/__init__.py | 2 + bbdev/mcp/tools/ip_replace.py | 26 ++ bbdev/mcp/tools/yosys_run.py | 3 +- bbdev/mcp/tools/yosys_synth.py | 3 +- 23 files changed, 830 insertions(+), 58 deletions(-) create mode 100644 bbdev/api/steps/firesim/scripts/build.sc create mode 100644 bbdev/api/steps/ip-replace/01_run_api.step.py create mode 100644 bbdev/api/steps/ip-replace/01_run_event.step.py create mode 100755 bbdev/api/steps/ip-replace/scripts/sram_replace.py create mode 100644 bbdev/api/utils/verilog.py create mode 100644 bbdev/mcp/tools/ip_replace.py diff --git a/bbdev/api/steps/firesim/01_buildbitstream_event.step.py b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py index 65d46bca6..240d05f94 100644 --- a/bbdev/api/steps/firesim/01_buildbitstream_event.step.py +++ b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py @@ -40,7 +40,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ================================================================================== # Fix AU280 board_part version (1.2 -> 1.0) for Vivado compatibility # ================================================================================== - au280_tcl = f"{bbdir}/arch/thirdparty/chipyard/sims/firesim/platforms/xilinx_alveo_u280/cl_firesim/scripts/au280.tcl" + au280_tcl = f"{bbdir}/thirdparty/firesim/platforms/xilinx_alveo_u280/cl_firesim/scripts/au280.tcl" if os.path.exists(au280_tcl): with open(au280_tcl, "r") as f: content = f.read() diff --git a/bbdev/api/steps/firesim/scripts/build.sc b/bbdev/api/steps/firesim/scripts/build.sc new file mode 100644 index 000000000..1e818e7a9 --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/build.sc @@ -0,0 +1,2 @@ +// FireSim uses the dedicated Mill graph in arch/build.sc. +// This file documents the entry point used by the FireSim step. diff --git a/bbdev/api/steps/firesim/scripts/firesim_env.py b/bbdev/api/steps/firesim/scripts/firesim_env.py index c95d11c8b..01af74d1c 100644 --- a/bbdev/api/steps/firesim/scripts/firesim_env.py +++ b/bbdev/api/steps/firesim/scripts/firesim_env.py @@ -4,6 +4,7 @@ import subprocess + def setup_firesim_env() -> dict: """Setup FireSim env vars + SSH agent. Returns env dict for subprocess.""" env = os.environ.copy() @@ -12,13 +13,11 @@ def setup_firesim_env() -> dict: env["FIRESIM_RUNFARM_PREFIX"] = "" # Create minimal env.sh for firesim if it doesn't exist - firesim_dir = os.path.join( - os.environ.get("HOME", "/home/mio"), - "Code/buckyball/arch/thirdparty/chipyard/sims/firesim" - ) + firesim_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..", "thirdparty", "firesim")) + env["PATH"] = os.path.join(firesim_dir, "deploy") + os.pathsep + env.get("PATH", "") env_sh = os.path.join(firesim_dir, "env.sh") if not os.path.exists(env_sh): - riscv_path = env.get("RISCV", "/home/mio/Code/buckyball/result") + riscv_path = env.get("RISCV", os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..", "result"))) with open(env_sh, "w") as f: f.write(f"""#!/usr/bin/env bash # Minimal env.sh for FireSim (auto-generated by firesim_env.py) diff --git a/bbdev/api/steps/firesim/scripts/makefrag/firesim/build.mk b/bbdev/api/steps/firesim/scripts/makefrag/firesim/build.mk index 1f3eeae5a..ece8ee0e2 100644 --- a/bbdev/api/steps/firesim/scripts/makefrag/firesim/build.mk +++ b/bbdev/api/steps/firesim/scripts/makefrag/firesim/build.mk @@ -1,6 +1,6 @@ # See LICENSE for license details. -CHIPYARD_STAGING_DIR := $(chipyard_dir)/sims/firesim-staging +CHIPYARD_STAGING_DIR := $(abspath $(makefile_dir)/../../../../../../../thirdparty/firesim-staging) # target scala directories to copy into midas. used by TARGET_COPY_TO_MIDAS_SCALA_DIRS TARGET_COPY_TO_MIDAS_SCALA_DIRS := \ @@ -13,15 +13,13 @@ $(FIRRTL_FILE) $(ANNO_FILE) &: SHELL := /usr/bin/env bash # needed for running s $(FIRRTL_FILE) $(ANNO_FILE) &: firesim_target_symlink_hook @mkdir -p $(@D) @mkdir -p $(TARGET_SBT_DIR)/target/generated-src/$(long_name) - source $(TARGET_SBT_DIR)/../sourceme.sh - cd $(TARGET_SBT_DIR) && \ - pwd && \ - ${SBT} ";project $(TARGET_SBT_PROJECT); runMain chipyard.Generator \ + cd $(TARGET_MILL_DIR) && \ + mill -i firesim.runMain chipyard.Generator \ --target-dir $(TARGET_SBT_DIR)/target/generated-src/$(long_name) \ --name $(long_name) \ --top-module $(DESIGN_PACKAGE).$(DESIGN) \ --legacy-configs $(TARGET_CONFIG_PACKAGE):$(TARGET_CONFIG) \ - --emit-legacy-sfc" + --emit-legacy-sfc # Link to the generated files ln -sf $(TARGET_SBT_DIR)/target/generated-src/$(long_name)/$(long_name).sfc.fir $(FIRRTL_FILE) ln -sf $(TARGET_SBT_DIR)/target/generated-src/$(long_name)/$(long_name).anno.json $(ANNO_FILE) diff --git a/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk b/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk index 9c16f60f2..8ce70ab26 100644 --- a/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk +++ b/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk @@ -26,6 +26,7 @@ TARGET_SBT_PROJECT := buckyball # Point to our project directory TARGET_SBT_DIR := $(abspath $(makefile_dir)/../../../../../../../arch) +TARGET_MILL_DIR := $(TARGET_SBT_DIR) TARGET_SOURCE_DIRS := $(abspath $(makefile_dir)/../../../../../../../arch/src/main/scala) # SBT launcher diff --git a/bbdev/api/steps/firesim/scripts/yaml/config_build.yaml b/bbdev/api/steps/firesim/scripts/yaml/config_build.yaml index b6c400207..da98a5c66 100644 --- a/bbdev/api/steps/firesim/scripts/yaml/config_build.yaml +++ b/bbdev/api/steps/firesim/scripts/yaml/config_build.yaml @@ -11,7 +11,7 @@ build_farm: base_recipe: build-farm-recipes/externally_provisioned.yaml recipe_arg_overrides: # REQUIRED: (replace this) default location of build directory on build host. - default_build_dir: /home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/FIRESIM_BUILD_DIR + default_build_dir: /home/mio/Code/buckyball/thirdparty/firesim/deploy/FIRESIM_BUILD_DIR # REQUIRED: List of IP addresses (or "localhost"). Each can have an OPTIONAL # argument, called "override_build_dir", specifying to override the default # build directory. @@ -31,7 +31,7 @@ build_farm: builds_to_run: # this section references builds defined in config_build_recipes.yaml # if you add a build here, it will be built when you run buildbitstream - - alveo_u280_firesim_GemminiBuckyballConfig_no_nic + - alveo_u280_firesim_BuckyballToyConfig_no_nic # - alveo_u280_firesim_BuckyballToyConfig_no_nic agfis_to_share: diff --git a/bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml b/bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml index f5fd36d94..b411d547e 100644 --- a/bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml +++ b/bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml @@ -23,26 +23,26 @@ midasexamples_gcd: # DOCREF END: Example HWDB Entry # alveo_u280_firesim_GemminiBuckyballConfig_no_nic: -# bitstream_tar: file:///home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/results-build/2025-09-24--17-27-06-alveo_u280_firesim_GemminiBuckyballConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimGemminiBuckyballConfig-BaseXilinxAlveoU280Config/firesim.tar.gz +# bitstream_tar: file:///home/mio/Code/buckyball/thirdparty/firesim/deploy/results-build/2025-09-24--17-27-06-alveo_u280_firesim_GemminiBuckyballConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimGemminiBuckyballConfig-BaseXilinxAlveoU280Config/firesim.tar.gz # deploy_quintuplet_override: null # custom_runtime_config: null alveo_u280_firesim_GemminiBuckyballConfig_no_nic: - bitstream_tar: file:///home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/results-build/2025-11-19--17-01-23-alveo_u280_firesim_GemminiBuckyballConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimGemminiBuckyballConfig-BaseXilinxAlveoU280Config/firesim.tar.gz + bitstream_tar: file:///home/mio/Code/buckyball/thirdparty/firesim/deploy/results-build/2025-11-19--17-01-23-alveo_u280_firesim_GemminiBuckyballConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimGemminiBuckyballConfig-BaseXilinxAlveoU280Config/firesim.tar.gz deploy_quintuplet_override: null custom_runtime_config: null # alveo_u280_firesim_BuckyballToyConfig_no_nic: -# bitstream_tar: file:///home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/results-build/2025-09-29--19-18-46-alveo_u280_firesim_BuckyballToyConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimBuckyballToyConfig-BaseXilinxAlveoU280Config/firesim.tar.gz +# bitstream_tar: file:///home/mio/Code/buckyball/thirdparty/firesim/deploy/results-build/2025-09-29--19-18-46-alveo_u280_firesim_BuckyballToyConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimBuckyballToyConfig-BaseXilinxAlveoU280Config/firesim.tar.gz # deploy_quintuplet_override: null # custom_runtime_config: null alveo_u280_firesim_BuckyballToyConfig_no_nic: - bitstream_tar: file:///home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/results-build/2026-05-12--15-52-50-alveo_u280_firesim_BuckyballToyConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimBuckyballToyConfig-BaseXilinxAlveoU280Config/firesim.tar.gz + bitstream_tar: file:///home/mio/Code/buckyball/thirdparty/firesim/deploy/results-build/2026-05-12--15-52-50-alveo_u280_firesim_BuckyballToyConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimBuckyballToyConfig-BaseXilinxAlveoU280Config/firesim.tar.gz deploy_quintuplet_override: null custom_runtime_config: null alveo_u280_firesim_GemminiBuckyballConfig_no_nic: - bitstream_tar: file:///home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/results-build/2026-05-12--19-18-24-alveo_u280_firesim_GemminiBuckyballConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimGemminiBuckyballConfig-BaseXilinxAlveoU280Config/firesim.tar.gz + bitstream_tar: file:///home/mio/Code/buckyball/thirdparty/firesim/deploy/results-build/2026-05-12--19-18-24-alveo_u280_firesim_GemminiBuckyballConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimGemminiBuckyballConfig-BaseXilinxAlveoU280Config/firesim.tar.gz deploy_quintuplet_override: null custom_runtime_config: null diff --git a/bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml b/bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml index 807fafc6b..56ecca649 100644 --- a/bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml +++ b/bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml @@ -14,7 +14,7 @@ run_farm: default_platform: XilinxAlveoU280InstanceDeployManager # REQUIRED: default directory where simulations are run out of on the run farm hosts - default_simulation_dir: /home/mio/Code/buckyball/arch/thirdparty/chipyard/sims/firesim/deploy/FIRESIM_RUNS_DIR + default_simulation_dir: /home/mio/Code/buckyball/thirdparty/firesim/deploy/FIRESIM_RUNS_DIR # REQUIRED: default fpga db file that enumerates what fpgas are available on the machine (used by XilinxU* Deploy Managers) default_fpga_db: /opt/firesim-db.json @@ -54,7 +54,7 @@ target_config: # or from config_build_recipes.yaml for metasimulation # In homogeneous configurations, use this to set the hardware config deployed # for all simulators - default_hw_config: alveo_u280_firesim_GemminiBuckyballConfig_no_nic + default_hw_config: alveo_u280_firesim_BuckyballToyConfig_no_nic # Advanced: Specify any extra plusargs you would like to provide when # booting the simulator (in both FPGA-sim and metasim modes). This is diff --git a/bbdev/api/steps/ip-replace/01_run_api.step.py b/bbdev/api/steps/ip-replace/01_run_api.step.py new file mode 100644 index 000000000..80f2b33d7 --- /dev/null +++ b/bbdev/api/steps/ip-replace/01_run_api.step.py @@ -0,0 +1,33 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + + +def req_arg(body: dict, name: str): + return body.get(name) or body.get(name.replace("_", "-")) + + +config = { + "name": "ip-replace-api", + "description": "replace behavioral IP RTL with compiler macro wrappers", + "flows": ["ip-replace"], + "triggers": [api("POST", "/ip/replace/run")], + "enqueues": ["ip-replace.run"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + source_list = req_arg(body, "source_list") + if not isinstance(source_list, str) or not source_list: + return ApiResponse(status=400, body={"error": "source_list is required"}) + output_dir = req_arg(body, "output_dir") + if not isinstance(output_dir, str) or not output_dir: + return ApiResponse(status=400, body={"error": "output_dir is required"}) + + data = { + "source_list": source_list, + "ip_replace_output_dir": output_dir, + "top": req_arg(body, "top"), + "consumer": req_arg(body, "consumer") or "generic", + } + await ctx.enqueue({"topic": "ip-replace.run", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/ip-replace/01_run_event.step.py b/bbdev/api/steps/ip-replace/01_run_event.step.py new file mode 100644 index 000000000..418a3ac1b --- /dev/null +++ b/bbdev/api/steps/ip-replace/01_run_event.step.py @@ -0,0 +1,129 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import check_result, get_origin_trace_id + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from sram_replace import prepare_ip_replacement + + +config = { + "name": "ip-replace", + "description": "replace behavioral SRAM/IP RTL with compiler macro wrappers", + "flows": ["ip-replace", "dc", "yosys"], + "triggers": [queue("ip-replace.run")], + "enqueues": ["yosys.synth"], +} + + +def default_source_list(input_data: dict) -> str | None: + source_list = input_data.get("source_list") + if isinstance(source_list, str) and source_list: + return source_list + build_dir = input_data.get("output_dir") + consumer = input_data.get("consumer") + if not isinstance(build_dir, str): + return None + if consumer == "dc": + return os.path.join(build_dir, "dc_sources.list") + if consumer == "yosys": + return os.path.join(build_dir, "yosys_sources.list") + return None + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + source_list_path = default_source_list(input_data) + if not source_list_path or not os.path.isfile(source_list_path): + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "ip-replace", "error": "missing source list"}, + trace_id=origin_tid, + ) + return failure_result + + with open(source_list_path) as handle: + sources = [line.strip() for line in handle if line.strip()] + if not sources: + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "ip-replace", "error": "empty source list"}, + trace_id=origin_tid, + ) + return failure_result + + consumer = input_data.get("consumer", "generic") + output_dir = input_data.get("ip_replace_output_dir") + if not isinstance(output_dir, str) or not output_dir: + output_dir = os.path.join(os.path.dirname(source_list_path), "ip-replace") + os.makedirs(output_dir, exist_ok=True) + + try: + replacement = prepare_ip_replacement( + sources, + output_dir, + os.environ.get("SKY130_ROOT"), + input_data.get("top"), + ) + except Exception as exc: + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "ip-replace", "error": str(exc)}, + trace_id=origin_tid, + ) + return failure_result + + replaced_source_list = os.path.join(output_dir, f"{consumer}_sources.list") + with open(replaced_source_list, "w") as handle: + for path in replacement["source_paths"]: + handle.write(f"{path}\n") + + replacement["source_list"] = replaced_source_list + extra = { + "task": "ip-replace", + "consumer": consumer, + "source_list": replaced_source_list, + "ip_manifest": replacement.get("manifest"), + "ip_mapped_memories": replacement["mapped"], + "ip_macro_area": replacement["macro_area"], + } + if input_data.get("mem_conf"): + extra["mem_conf"] = input_data["mem_conf"] + if replacement["mapped"]: + ctx.logger.info( + f"Mapped {replacement['mapped']} behavioral memories to compiler macros; " + f"estimated macro area {replacement['macro_area']:.3f} um^2" + ) + + next_topic = input_data.get("next_topic") + if isinstance(next_topic, str) and next_topic: + await check_result(ctx, 0, continue_run=True, extra_fields=extra, trace_id=origin_tid) + await ctx.enqueue( + { + "topic": next_topic, + "data": { + **input_data, + "source_list": replaced_source_list, + "ip_replacement": replacement, + "_trace_id": origin_tid, + }, + } + ) + return + + await check_result(ctx, 0, continue_run=False, extra_fields=extra, trace_id=origin_tid) diff --git a/bbdev/api/steps/ip-replace/scripts/sram_replace.py b/bbdev/api/steps/ip-replace/scripts/sram_replace.py new file mode 100755 index 000000000..507e8d9ef --- /dev/null +++ b/bbdev/api/steps/ip-replace/scripts/sram_replace.py @@ -0,0 +1,412 @@ +"""Generate SRAM replacement collateral for behavioral memories. + +The generated artifacts mirror the DC tapeout flow: behavioral memory modules +are replaced by same-named wrappers around compiler macros, while the original +RTL is removed from the synthesis source list. Unsupported port topologies +remain in RTL and are recorded in the manifest instead of being approximated. +""" + +from __future__ import annotations + +import json +import math +import os +import re +import argparse +import tomllib +from dataclasses import dataclass +from pathlib import Path + + +MODULE_RE = re.compile(r"\bmodule\s+([A-Za-z_][A-Za-z0-9_$]*)\s*\((.*?)\)\s*;", re.S) +MEMORY_RE = re.compile( + r"\breg\s+(?:\[\s*(\d+)\s*:\s*(\d+)\s*\]\s+)?Memory\s*\[\s*(\d+)\s*:\s*(\d+)\s*\]", + re.S, +) +SIZE_RE = re.compile(r"\bSIZE\s+([0-9.]+)\s+BY\s+([0-9.]+)\s*;", re.I) +RANGE_RE = re.compile(r"\[\s*(\d+)\s*:\s*(\d+)\s*\]\s*{name}\b") + + +@dataclass(frozen=True) +class Macro: + name: str + depth: int + width: int + write_size: int + area: float + liberty: Path + verilog: Path + lef: Path + + +@dataclass(frozen=True) +class Memory: + name: str + source: Path + header: str + depth: int + width: int + addr_width: int + mask_width: int | None + + +def width_from_range(text: str, name: str) -> int | None: + match = re.search(RANGE_RE.pattern.format(name=re.escape(name)), text) + if match is None: + return None + return abs(int(match.group(1)) - int(match.group(2))) + 1 + + +def ceil_log2(value: int) -> int: + return max(1, math.ceil(math.log2(value))) + + +def load_macros(sky130_root: Path) -> list[Macro]: + macro_root = sky130_root / "share" / "sram22_sky130_macros" + macros: list[Macro] = [] + for toml_path in sorted(macro_root.glob("*/sram22.toml")): + name = toml_path.parent.name + if name.endswith("_test"): + continue + with toml_path.open("rb") as handle: + config = tomllib.load(handle) + liberty = toml_path.parent / f"{name}_tt_025C_1v80.rc.lib" + verilog = toml_path.parent / f"{name}.v" + lef = toml_path.parent / f"{name}.lef" + if not (liberty.is_file() and verilog.is_file() and lef.is_file()): + continue + size = SIZE_RE.search(lef.read_text(errors="replace")) + if size is None: + continue + macros.append( + Macro( + name=name, + depth=int(config["num_words"]), + width=int(config["data_width"]), + write_size=int(config["write_size"]), + area=float(size.group(1)) * float(size.group(2)), + liberty=liberty, + verilog=verilog, + lef=lef, + ) + ) + if not macros: + raise RuntimeError(f"no usable SRAM22 macros found under {macro_root}") + return macros + + +def discover_memories(source_paths: list[Path]) -> tuple[list[Memory], list[dict[str, object]]]: + memories: list[Memory] = [] + unsupported: list[dict[str, object]] = [] + for source in source_paths: + text = source.read_text(errors="replace") + module_match = MODULE_RE.search(text) + memory_match = MEMORY_RE.search(text) + if module_match is None or memory_match is None: + continue + name, header = module_match.groups() + ports = set(re.findall(r"\b(?:RW0|R0|W0)_[A-Za-z0-9_]+\b", header)) + if not {"RW0_addr", "RW0_en", "RW0_clk", "RW0_wmode", "RW0_wdata", "RW0_rdata"}.issubset(ports): + unsupported.append( + { + "module": name, + "source": str(source), + "status": "unsupported", + "reason": "requires a multi-port SRAM macro; SRAM22 provides only 1RW macros", + } + ) + continue + data_msb = int(memory_match.group(1) or 0) + data_lsb = int(memory_match.group(2) or 0) + depth_a = int(memory_match.group(3)) + depth_b = int(memory_match.group(4)) + addr_width = width_from_range(header, "RW0_addr") + if addr_width is None: + addr_width = ceil_log2(abs(depth_a - depth_b) + 1) + memories.append( + Memory( + name=name, + source=source, + header=header, + depth=abs(depth_a - depth_b) + 1, + width=abs(data_msb - data_lsb) + 1, + addr_width=addr_width, + mask_width=width_from_range(header, "RW0_wmask"), + ) + ) + return memories, unsupported + + +def reachable_modules(source_paths: list[Path], top_module: str | None) -> set[str] | None: + if not top_module: + return None + modules: dict[str, str] = {} + for source in source_paths: + text = source.read_text(errors="replace") + match = MODULE_RE.search(text) + if match is not None: + modules[match.group(1)] = text + if top_module not in modules: + return {top_module} + + # Split Verilog module instantiations are line-oriented in CIRCT output. + # The pattern intentionally requires a module type and an instance name, + # excluding ``module``, ``always`` and other language constructs. + instance_re = re.compile( + r"^\s*([A-Za-z_][A-Za-z0-9_$]*)\s*(?:#\s*\(.*?\))?\s+([A-Za-z_][A-Za-z0-9_$]*)\s*\(", + re.M | re.S, + ) + reachable = {top_module} + pending = [top_module] + while pending: + module = pending.pop() + for child, _ in instance_re.findall(modules.get(module, "")): + if child in modules and child not in reachable: + reachable.add(child) + pending.append(child) + return reachable + + +def choose_macro(memory: Memory, macros: list[Macro]) -> Macro: + source_granularity = memory.width // memory.mask_width if memory.mask_width else None + candidates = [ + macro + for macro in macros + if source_granularity is None or source_granularity % macro.write_size == 0 + ] + if not candidates: + raise RuntimeError(f"no SRAM22 macro has a compatible write mask for {memory.name}") + return min( + candidates, + key=lambda macro: ( + math.ceil(memory.depth / macro.depth) + * math.ceil(memory.width / macro.width) + * macro.area, + -macro.width, + ), + ) + + +def addr_expr(signal: str, source_width: int, macro_width: int) -> str: + if source_width == macro_width: + return signal + if source_width > macro_width: + return f"{signal}[{macro_width - 1}:0]" + return f"{{{{{macro_width - source_width}{{1'b0}}}}, {signal}}}" + + +def data_expr(signal: str, lo: int, logical_width: int, physical_width: int) -> str: + if logical_width == physical_width: + return f"{signal}[{lo} +: {logical_width}]" + return f"{{{{{physical_width - logical_width}{{1'b0}}}}, {signal}[{lo} +: {logical_width}]}}" + + +def bank_select_expr(memory: Memory, macro: Macro, bank: int, banks: int) -> str: + if banks == 1: + return "1'b1" + macro_addr_width = ceil_log2(macro.depth) + bank_bits = ceil_log2(banks) + return f"(RW0_addr[{macro_addr_width + bank_bits - 1}:{macro_addr_width}] == {bank_bits}'d{bank})" + + +def emit_wrapper(memory: Memory, macro: Macro) -> tuple[str, dict[str, object]]: + macro_addr_width = ceil_log2(macro.depth) + banks = math.ceil(memory.depth / macro.depth) + bank_bits = ceil_log2(banks) if banks > 1 else 0 + slices = math.ceil(memory.width / macro.width) + lines = [f"module {memory.name}({memory.header});", ""] + lines.extend( + [ + " logic ren_d0;", + " logic wmode_d0;", + f" logic [{memory.width - 1}:0] rdata_comb;", + ] + ) + if bank_bits: + lines.append(f" logic [{bank_bits - 1}:0] bank_d0;") + lines.extend(["", " always_ff @(posedge RW0_clk) begin", " ren_d0 <= RW0_en;", " wmode_d0 <= RW0_wmode;"]) + if bank_bits: + lines.append(f" bank_d0 <= RW0_addr[{macro_addr_width + bank_bits - 1}:{macro_addr_width}];") + lines.extend([" end", ""]) + + for slice_index in range(slices): + lo = slice_index * macro.width + logical_width = min(macro.width, memory.width - lo) + lines.append(f" logic [{macro.width - 1}:0] q_{slice_index} [0:{banks - 1}];") + lines.append(f" logic [{macro.width - 1}:0] d_{slice_index};") + lines.append(f" logic [{macro.width // macro.write_size - 1}:0] wmask_{slice_index};") + lines.append(f" assign d_{slice_index} = {data_expr('RW0_wdata', lo, logical_width, macro.width)};") + for bit in range(macro.width // macro.write_size): + if memory.mask_width is None: + mask = "1'b1" + else: + source_granularity = memory.width // memory.mask_width + source_bit = (lo + bit * macro.write_size) // source_granularity + mask = f"RW0_wmask[{source_bit}]" if source_bit < memory.mask_width else "1'b0" + lines.append(f" assign wmask_{slice_index}[{bit}] = {mask};") + lines.append("") + for bank in range(banks): + select = bank_select_expr(memory, macro, bank, banks) + lines.extend( + [ + f" {macro.name} u_sram_{slice_index}_{bank} (", + " .clk(RW0_clk),", + f" .we(RW0_en & RW0_wmode & {select}),", + f" .wmask(wmask_{slice_index}),", + f" .addr({addr_expr('RW0_addr', memory.addr_width, macro_addr_width)}),", + f" .din(d_{slice_index}),", + f" .dout(q_{slice_index}[{bank}])", + " );", + ] + ) + lines.append("") + + lines.append(" always_comb begin") + lines.append(" rdata_comb = 'x;") + for slice_index in range(slices): + lo = slice_index * macro.width + logical_width = min(macro.width, memory.width - lo) + if banks == 1: + lines.append(f" rdata_comb[{lo} +: {logical_width}] = q_{slice_index}[0][{logical_width - 1}:0];") + else: + lines.append(" case (bank_d0)") + for bank in range(banks): + lines.append( + f" {bank_bits}'d{bank}: rdata_comb[{lo} +: {logical_width}] = q_{slice_index}[{bank}][{logical_width - 1}:0];" + ) + lines.append(" default: rdata_comb = 'x;") + lines.append(" endcase") + lines.extend( + [ + " end", + f" assign RW0_rdata = (ren_d0 && !wmode_d0) ? rdata_comb : {memory.width}'bx;", + "endmodule", + "", + ] + ) + return "\n".join(lines), { + "module": memory.name, + "source": str(memory.source), + "status": "mapped", + "port_type": "1RW", + "logical_depth": memory.depth, + "logical_width": memory.width, + "macro": macro.name, + "macro_depth": macro.depth, + "macro_width": macro.width, + "macro_write_size": macro.write_size, + "banks": banks, + "slices": slices, + "instances": banks * slices, + "macro_area": macro.area, + "macro_total_area": macro.area * banks * slices, + "liberty": str(macro.liberty), + "lef": str(macro.lef), + "verilog": str(macro.verilog), + } + + +def emit_blackboxes(macros: list[Macro]) -> str: + lines = ["// SRAM22 blackboxes for synthesis; physical views are in manifest.", ""] + for macro in sorted({macro.name: macro for macro in macros}.values(), key=lambda item: item.name): + addr_width = ceil_log2(macro.depth) + mask_width = macro.width // macro.write_size + lines.extend( + [ + "(* blackbox *)", + f"module {macro.name}(", + " input clk,", + " input we,", + f" input [{mask_width - 1}:0] wmask,", + f" input [{addr_width - 1}:0] addr,", + f" input [{macro.width - 1}:0] din,", + f" output [{macro.width - 1}:0] dout", + ");", + "endmodule", + "", + ] + ) + return "\n".join(lines) + + +def prepare_ip_replacement( + source_paths: list[str], output_dir: str, sky130_root: str | None, top_module: str | None = None +) -> dict[str, object]: + """Create IP replacement collateral for DC and Yosys consumers.""" + if not sky130_root: + return {"source_paths": source_paths, "macro_liberties": [], "macro_area": 0.0, "mapped": 0} + root = Path(os.path.expandvars(os.path.expanduser(sky130_root))) + if not root.is_dir(): + return {"source_paths": source_paths, "macro_liberties": [], "macro_area": 0.0, "mapped": 0} + + macros = load_macros(root) + sources = [Path(path) for path in source_paths] + memories, unsupported = discover_memories(sources) + reachable = reachable_modules(sources, top_module) + if reachable is not None: + memories = [memory for memory in memories if memory.name in reachable] + unsupported = [row for row in unsupported if row["module"] in reachable] + output = Path(output_dir) + output.mkdir(parents=True, exist_ok=True) + + wrappers: list[str] = [] + manifest: list[dict[str, object]] = unsupported[:] + replaced_sources: set[Path] = set() + used_macros: list[Macro] = [] + for memory in memories: + try: + macro = choose_macro(memory, macros) + wrapper, row = emit_wrapper(memory, macro) + except Exception as exc: + manifest.append( + {"module": memory.name, "source": str(memory.source), "status": "unsupported", "reason": str(exc)} + ) + continue + wrappers.append(wrapper) + manifest.append(row) + replaced_sources.add(memory.source.resolve()) + used_macros.append(macro) + + wrapper_path = output / "sram22_replacements.sv" + blackbox_path = output / "sram22_blackboxes.v" + manifest_path = output / "sram22_manifest.json" + wrapper_path.write_text("\n".join(wrappers)) + blackbox_path.write_text(emit_blackboxes(used_macros)) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + + macro_liberties = sorted({str(macro.liberty) for macro in used_macros}) + macro_area = sum(float(row.get("macro_total_area", 0.0)) for row in manifest) + remaining_sources = [path for path in source_paths if Path(path).resolve() not in replaced_sources] + extra_sources = [str(blackbox_path), str(wrapper_path)] if used_macros else [] + return { + "source_paths": extra_sources + remaining_sources, + "macro_liberties": macro_liberties, + "macro_area": macro_area, + "mapped": len(used_macros), + "manifest": str(manifest_path), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-list", type=Path, help="file containing one Verilog source path per line") + parser.add_argument("--source-dir", type=Path, help="directory to scan when --source-list is omitted") + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--sky130-root", default=os.environ.get("SKY130_ROOT")) + parser.add_argument("--top", default=None) + args = parser.parse_args() + + if args.source_list: + sources = [line.strip() for line in args.source_list.read_text().splitlines() if line.strip()] + elif args.source_dir: + sources = [str(path) for path in sorted(args.source_dir.glob("*.sv"))] + else: + parser.error("one of --source-list or --source-dir is required") + + result = prepare_ip_replacement(sources, str(args.output_dir), args.sky130_root, args.top) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py index bb1d1f5eb..95a696e07 100644 --- a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py +++ b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py @@ -14,13 +14,14 @@ from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id +from utils.verilog import seq_mem_elaboration_command config = { "name": "yosys verilog", "description": "generate verilog for yosys flow", "flows": ["yosys"], "triggers": [queue("yosys.run"), queue("yosys.verilog")], - "enqueues": ["yosys.synth"], + "enqueues": ["ip-replace.run"], } @@ -176,12 +177,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: shutil.rmtree(build_dir) os.makedirs(build_dir, exist_ok=True) - verilog_command = ( - f"mill -i __.test.runMain sims.verilator.Elaborate {elaborate_config} " - "--disable-annotation-unknown -strip-debug-info -O=debug " - "-lowering-options=disallowLocalVariables " - f"--split-verilog -o={build_dir}" - ) + mem_conf = os.path.join(build_dir, "mems.conf") + verilog_command = seq_mem_elaboration_command(elaborate_config, build_dir, mem_conf) result = stream_run_logger( cmd=verilog_command, @@ -222,12 +219,25 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result( ctx, result.returncode, - continue_run=input_data.get("from_run_workflow", False), - extra_fields={"task": "verilog", "source_list": source_list_path}, + continue_run=True, + extra_fields={"task": "verilog", "source_list": source_list_path, "mem_conf": mem_conf}, trace_id=origin_tid, ) - if input_data.get("from_run_workflow"): - await ctx.enqueue({"topic": "yosys.synth", "data": {**input_data, "task": "run"}}) + await ctx.enqueue( + { + "topic": "ip-replace.run", + "data": { + **input_data, + "source_list": source_list_path, + "ip_replace_output_dir": yosys_log_dir, + "consumer": "yosys", + "mem_conf": mem_conf, + "next_topic": "yosys.synth" if input_data.get("from_run_workflow") else None, + "task": "run" if input_data.get("from_run_workflow") else "verilog", + "_trace_id": origin_tid, + }, + } + ) return diff --git a/bbdev/api/steps/mill/04_dc_verilog_event.step.py b/bbdev/api/steps/mill/04_dc_verilog_event.step.py index eb2093bd9..f10c3f36e 100644 --- a/bbdev/api/steps/mill/04_dc_verilog_event.step.py +++ b/bbdev/api/steps/mill/04_dc_verilog_event.step.py @@ -13,13 +13,14 @@ from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger +from utils.verilog import seq_mem_elaboration_command config = { "name": "dc-verilog", "description": "generate RTL and memory metadata for downstream DC/tapeout flow", "flows": ["dc"], "triggers": [queue("dc.verilog")], - "enqueues": [], + "enqueues": ["ip-replace.run"], } @@ -101,14 +102,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: shutil.rmtree(build_dir) os.makedirs(build_dir, exist_ok=True) - mem_conf = os.path.join(build_dir, "dc.mems.conf") - verilog_command = ( - f"mill -i __.test.runMain sims.verilator.Elaborate {elaborate_config} " - "--disable-annotation-unknown --strip-debug-info -O=debug " - "-lowering-options=disallowLocalVariables " - f"--repl-seq-mem --repl-seq-mem-file={mem_conf} " - f"--split-verilog -o={build_dir}" - ) + mem_conf = os.path.join(build_dir, "mems.conf") + verilog_command = seq_mem_elaboration_command(elaborate_config, build_dir, mem_conf) result = stream_run_logger( cmd=verilog_command, @@ -149,9 +144,23 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result( ctx, result.returncode, - continue_run=False, + continue_run=True, extra_fields={"task": "verilog", "source_list": source_list_path, "mem_conf": mem_conf}, trace_id=origin_tid, ) + await ctx.enqueue( + { + "topic": "ip-replace.run", + "data": { + **input_data, + "source_list": source_list_path, + "ip_replace_output_dir": os.path.join(build_dir, "ip-replace"), + "consumer": "dc", + "mem_conf": mem_conf, + "_trace_id": origin_tid, + }, + } + ) + return diff --git a/bbdev/api/steps/yosys/01_run_api.step.py b/bbdev/api/steps/yosys/01_run_api.step.py index 6bf7bcca9..dbe56c426 100644 --- a/bbdev/api/steps/yosys/01_run_api.step.py +++ b/bbdev/api/steps/yosys/01_run_api.step.py @@ -30,6 +30,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: "log_dir": log_dir, "top": req_arg(body, "top"), "config": req_arg(body, "config"), + "vcd": req_arg(body, "vcd"), "from_run_workflow": True, } await ctx.enqueue({"topic": "yosys.run", "data": {**data, "_trace_id": ctx.trace_id}}) diff --git a/bbdev/api/steps/yosys/03_synth_api.step.py b/bbdev/api/steps/yosys/03_synth_api.step.py index 88dd2990e..363afc37c 100644 --- a/bbdev/api/steps/yosys/03_synth_api.step.py +++ b/bbdev/api/steps/yosys/03_synth_api.step.py @@ -16,7 +16,7 @@ "description": "run yosys synthesis for area estimation", "flows": ["yosys"], "triggers": [api("POST", "/yosys/synth")], - "enqueues": ["yosys.synth"], + "enqueues": ["ip-replace.run"], } @@ -30,6 +30,19 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: "log_dir": log_dir, "top": req_arg(body, "top"), "config": req_arg(body, "config"), + "vcd": req_arg(body, "vcd"), } - await ctx.enqueue({"topic": "yosys.synth", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "ip-replace.run", + "data": { + **data, + "source_list": f"{data['output_dir']}/yosys_sources.list", + "ip_replace_output_dir": log_dir, + "consumer": "yosys", + "next_topic": "yosys.synth", + "_trace_id": ctx.trace_id, + }, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/yosys/03_synth_event.step.py b/bbdev/api/steps/yosys/03_synth_event.step.py index d075cfe28..7ae3dfb61 100644 --- a/bbdev/api/steps/yosys/03_synth_event.step.py +++ b/bbdev/api/steps/yosys/03_synth_event.step.py @@ -1,4 +1,5 @@ import os +import re import sys import yaml from datetime import datetime @@ -22,11 +23,41 @@ } +POWER_TOTAL_RE = re.compile( + r"^Total\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)", + re.M, +) + + +def dynamic_power_report(power_report: str, activity_source: str) -> tuple[str, float] | None: + """Return OpenSTA's internal-plus-switching power, excluding leakage.""" + match = POWER_TOTAL_RE.search(power_report) + if match is None: + return None + internal, switching, _leakage, _total = (float(value) for value in match.groups()) + dynamic = internal + switching + return ( + "OpenSTA dynamic power\n" + f" Activity source: {activity_source}\n" + f" Internal: {internal:.6e} W\n" + f" Switching: {switching:.6e} W\n" + f" Dynamic (internal + switching): {dynamic:.6e} W\n", + dynamic, + ) + + def load_yosys_config(): config_path = os.path.join(os.path.dirname(__file__), "scripts", "yosys-config.yaml") if os.path.exists(config_path): with open(config_path, "r") as f: - return yaml.safe_load(f) or {} + config = yaml.safe_load(f) or {} + # Keep paths portable between the Nix shell and explicit user + # configurations while retaining normal YAML scalar values. + if isinstance(config.get("liberty"), str): + config["liberty"] = os.path.expandvars(os.path.expanduser(config["liberty"])) + if isinstance(config.get("vcd"), str) and config["vcd"]: + config["vcd"] = os.path.expandvars(os.path.expanduser(config["vcd"])) + return config return {} async def handler(input_data: dict, ctx: FlowContext) -> None: @@ -38,7 +69,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: top_module = input_data.get("top") or yosys_cfg.get("top") or "BuckyballAccelerator" liberty = yosys_cfg.get("liberty") - source_list_path = os.path.join(build_dir, "yosys_sources.list") + source_list_path = input_data.get("source_list") or os.path.join(build_dir, "yosys_sources.list") if not os.path.exists(source_list_path): success_result, failure_result = await check_result( ctx, @@ -68,6 +99,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: os.makedirs(yosys_output_dir, exist_ok=True) ctx.logger.info(f"Yosys log dir: {yosys_output_dir}") + ip_replacement = input_data.get("ip_replacement") or {} + macro_liberties = ip_replacement.get("macro_liberties", []) + macro_area = ip_replacement.get("macro_area", 0.0) + mapped_memories = ip_replacement.get("mapped", 0) + read_commands = "\n".join([f"read_verilog -sv {src}" for src in vsrcs]) yosys_script = f"{yosys_output_dir}/synth_area.ys" with open(yosys_script, "w") as f: @@ -101,21 +137,68 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="yosys synth", ) - extra = {"task": "synth", "output_dir": yosys_output_dir} + extra = { + "task": "synth", + "output_dir": yosys_output_dir, + "ip_manifest": ip_replacement.get("manifest"), + "ip_mapped_memories": mapped_memories, + "ip_macro_area": macro_area, + } netlist_file = f"{yosys_output_dir}/synth_netlist.v" timing_report_file = f"{yosys_output_dir}/timing_report.txt" + power_report_file = f"{yosys_output_dir}/power_report.txt" + dynamic_power_report_file = f"{yosys_output_dir}/dynamic_power_report.txt" + sta_returncode = 0 if liberty and os.path.exists(liberty) and os.path.exists(netlist_file) and result.returncode == 0: clock_period = yosys_cfg.get("clock_period", 10.0) clock_name = yosys_cfg.get("clock_name", "clock") + vcd = input_data.get("vcd") or yosys_cfg.get("vcd") + if isinstance(vcd, str): + vcd = os.path.expandvars(os.path.expanduser(vcd)) + if vcd and not os.path.isfile(vcd): + extra["error"] = f"VCD file does not exist: {vcd}" + sta_returncode = 1 + vcd = None + if sta_returncode: + await check_result( + ctx, + sta_returncode, + continue_run=False, + extra_fields=extra, + trace_id=origin_tid, + ) + return sta_script = f"{yosys_output_dir}/sta_timing.tcl" with open(sta_script, "w") as f: f.write(f"read_liberty {liberty}\n") + for macro_liberty in macro_liberties: + f.write(f"read_liberty {macro_liberty}\n") f.write(f"read_verilog {netlist_file}\n") f.write(f"link_design {top_module}\n") - f.write(f"create_clock [get_ports {clock_name}] -name clk -period {clock_period}\n") + # Prefer the configured port, then use the generated-memory clock + # naming convention. Some small modules are combinational and + # therefore have no clock at all. + f.write(f"set bb_clock_port [get_ports -quiet {clock_name}]\n") + f.write("if {[llength $bb_clock_port] == 0} { set bb_clock_port [get_ports -quiet *clk*] }\n") + f.write(f"if {{[llength $bb_clock_port] > 0}} {{ create_clock $bb_clock_port -name clk -period {clock_period} }}\n") f.write(f"report_checks -path_delay max -format full > {timing_report_file}\n") f.write(f"report_checks -path_delay max -format full\n") + if vcd and os.path.exists(vcd): + f.write(f"read_vcd {vcd} -scope {top_module}\n") + extra["power_activity_source"] = vcd + else: + input_activity = yosys_cfg.get("input_activity", 0.1) + input_duty = yosys_cfg.get("input_duty", 0.5) + clock_activity = yosys_cfg.get("clock_activity", 1.0) + clock_duty = yosys_cfg.get("clock_duty", 0.5) + f.write(f"set_power_activity -input -activity {input_activity} -duty {input_duty}\n") + # OpenSTA expects the clock collection after ``-clock``. + # Applying the default rate to all clocks keeps this usable + # for generated designs whose clock port is renamed. + f.write(f"if {{[llength [all_clocks]] > 0}} {{ set_power_activity -clock [all_clocks] -activity {clock_activity} -duty {clock_duty} }}\n") + extra["power_activity_source"] = "default" + f.write(f"report_power -digits 4 > {power_report_file}\n") f.write("exit\n") sta_result = stream_run_logger( @@ -129,11 +212,38 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if sta_result.returncode == 0 and os.path.exists(timing_report_file): with open(timing_report_file, "r") as f: extra["timing_report"] = f.read() + sta_returncode = sta_result.returncode + if sta_result.returncode == 0 and os.path.exists(power_report_file): + with open(power_report_file, "r") as f: + extra["power_report"] = f.read() + dynamic_result = dynamic_power_report( + extra["power_report"], extra.get("power_activity_source", "unknown") + ) + if dynamic_result is not None: + dynamic_report, dynamic_power = dynamic_result + with open(dynamic_power_report_file, "w") as f: + f.write(dynamic_report) + extra["dynamic_power_report"] = dynamic_report + extra["dynamic_power_w"] = dynamic_power report_file = f"{yosys_output_dir}/area_report.txt" if os.path.exists(report_file): with open(report_file, "r") as f: extra["area_report"] = f.read() + if macro_area: + chip_area = re.search(r"Chip area for module .*?:\s+([0-9.]+)", extra["area_report"]) + standard_area = float(chip_area.group(1)) if chip_area else 0.0 + total_area = standard_area + macro_area + ip_area_report = ( + "\nIP macro area summary\n" + f" Macro area: {macro_area:.6f} um^2\n" + f" Standard-cell area: {standard_area:.6f} um^2\n" + f" Total estimated area: {total_area:.6f} um^2\n" + ) + with open(report_file, "a") as f: + f.write(ip_area_report) + extra["area_report"] += ip_area_report + extra["estimated_total_area"] = total_area hierarchy_file = f"{yosys_output_dir}/hierarchy_report.txt" if os.path.exists(hierarchy_file): @@ -142,7 +252,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result( ctx, - result.returncode, + result.returncode or sta_returncode, continue_run=False, extra_fields=extra, trace_id=origin_tid, diff --git a/bbdev/api/steps/yosys/scripts/yosys-config.yaml b/bbdev/api/steps/yosys/scripts/yosys-config.yaml index 52bd76719..61be109d3 100644 --- a/bbdev/api/steps/yosys/scripts/yosys-config.yaml +++ b/bbdev/api/steps/yosys/scripts/yosys-config.yaml @@ -3,9 +3,9 @@ # Liberty (.lib) file path for ASIC area estimation # If not specified, yosys will use generic gate-level statistics -# Example: -# liberty: /path/to/your/standard_cell.lib -liberty: +# The default Nix EDA shell exports SKY130_ROOT. bbdev expands environment +# variables in this file before invoking Yosys/OpenSTA. +liberty: ${SKY130_ROOT}/lib/sky130_fd_sc_hd__tt_025C_1v80.lib # Default top module name (override with --top on CLI) top: BuckyballAccelerator @@ -15,8 +15,18 @@ elaborate_config: sims.verilator.BuckyballToyVerilatorConfig # OpenSTA timing analysis settings # Clock period in nanoseconds (used for STA constraint) -# Default 10ns = 500MHz -clock_period: 2.0 +# Default 10ns = 100MHz +clock_period: 10.0 # Clock port name in the design clock_name: clock + +# Power estimation reports dynamic power as Internal + Switching; leakage is +# retained in OpenSTA's raw report but not included in dynamic_power_report.txt. +# Supply a workload VCD through `bbdev yosys --synth --vcd ` for an +# activity-driven result. Empty vcd uses the default toggle-rate estimate below. +vcd: +input_activity: 0.1 +input_duty: 0.5 +clock_activity: 1.0 +clock_duty: 0.5 diff --git a/bbdev/api/utils/verilog.py b/bbdev/api/utils/verilog.py new file mode 100644 index 000000000..3a255a303 --- /dev/null +++ b/bbdev/api/utils/verilog.py @@ -0,0 +1,12 @@ +"""Shared RTL elaboration commands for synthesis-oriented flows.""" + + +def seq_mem_elaboration_command(elaborate_config: str, build_dir: str, mem_conf: str) -> str: + """Emit split Verilog with behavioral memories replaced by named modules.""" + return ( + f"mill -i __.test.runMain sims.verilator.Elaborate {elaborate_config} " + "--disable-annotation-unknown --strip-debug-info -O=debug " + "-lowering-options=disallowLocalVariables " + f"--repl-seq-mem --repl-seq-mem-file={mem_conf} " + f"--split-verilog -o={build_dir}" + ) diff --git a/bbdev/bbdev b/bbdev/bbdev index edf786471..00ef16143 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -80,10 +80,13 @@ WORKFLOW_COMMANDS = { "dc": { "verilog": 'Generate DC RTL. Args: "--config [--dir ]"', }, + "ip-replace": { + "run": 'Replace behavioral IP RTL. Args: "--source-list --output-dir [--top ] [--consumer ]"', + }, "yosys": { - "run": 'Run yosys flow. Args: "[--top ] [--config ] [--output-dir ] [--log-dir ]"', + "run": 'Run yosys flow. Args: "[--top ] [--config ] [--output-dir ] [--log-dir ] [--vcd ]"', "verilog": 'Generate verilog for yosys flow. Args: "[--config ] [--output-dir ] [--log-dir ]"', - "synth": 'Run yosys synthesis. Args: "[--top ] [--output-dir ] [--log-dir ]"', + "synth": 'Run yosys synthesis. Args: "[--top ] [--output-dir ] [--log-dir ] [--vcd ]"', }, "uvm": { "build": 'Build a Ball UVM simulation. Args: "--ball= [--filelist ]"', diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index 1bc5d93ad..80d0d4925 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -30,6 +30,7 @@ from . import bebop_p2e_runworkload from . import bebop_p2e_batch from . import dc_verilog +from . import ip_replace from . import firesim_enumeratefpgas from . import firesim_buildbitstream from . import firesim_infrasetup @@ -68,6 +69,7 @@ bebop_p2e_runworkload, bebop_p2e_batch, dc_verilog, + ip_replace, firesim_enumeratefpgas, firesim_buildbitstream, firesim_infrasetup, diff --git a/bbdev/mcp/tools/ip_replace.py b/bbdev/mcp/tools/ip_replace.py new file mode 100644 index 000000000..ecec7daea --- /dev/null +++ b/bbdev/mcp/tools/ip_replace.py @@ -0,0 +1,26 @@ +"""MCP tool: bbdev_ip_replace_run.""" + +from __future__ import annotations + +from typing import Optional + +from common import err, fmt, opt, submit + + +def register(mcp): + @mcp.tool() + def bbdev_ip_replace_run( + source_list: str, + output_dir: str, + top: Optional[str] = None, + consumer: Optional[str] = None, + ) -> str: + """Replace behavioral IP RTL. POST /ip/replace/run.""" + if not source_list or not output_dir: + return err("source_list and output_dir are required") + return fmt( + submit( + "/ip/replace/run", + opt({}, source_list=source_list, output_dir=output_dir, top=top, consumer=consumer), + ) + ) diff --git a/bbdev/mcp/tools/yosys_run.py b/bbdev/mcp/tools/yosys_run.py index 72ee431f8..2b71d6770 100644 --- a/bbdev/mcp/tools/yosys_run.py +++ b/bbdev/mcp/tools/yosys_run.py @@ -14,6 +14,7 @@ def bbdev_yosys_run( config: Optional[str] = None, output_dir: Optional[str] = None, log_dir: Optional[str] = None, + vcd: Optional[str] = None, ) -> str: """Full yosys flow. POST /yosys/run.""" return fmt( @@ -25,7 +26,7 @@ def bbdev_yosys_run( config=config, output_dir=output_dir, log_dir=log_dir, + vcd=vcd, ), ) ) - diff --git a/bbdev/mcp/tools/yosys_synth.py b/bbdev/mcp/tools/yosys_synth.py index 520b101e3..8e67f3407 100644 --- a/bbdev/mcp/tools/yosys_synth.py +++ b/bbdev/mcp/tools/yosys_synth.py @@ -14,6 +14,7 @@ def bbdev_yosys_synth( config: Optional[str] = None, output_dir: Optional[str] = None, log_dir: Optional[str] = None, + vcd: Optional[str] = None, ) -> str: """Yosys synthesis + OpenSTA. POST /yosys/synth.""" return fmt( @@ -25,7 +26,7 @@ def bbdev_yosys_synth( config=config, output_dir=output_dir, log_dir=log_dir, + vcd=vcd, ), ) ) - From 5bb69b7624005d72f2b02faca6d418dc39a94c5b Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Mon, 10 Aug 2026 22:28:00 +0800 Subject: [PATCH 114/196] feat: add Bebop bank difftest workflow --- .../steps/bebop/difftest/01_run_api.step.py | 79 ++++++ .../steps/bebop/difftest/01_run_event.step.py | 228 ++++++++++++++++++ bbdev/api/tests/test_all_api_serial.py | 1 + bbdev/api/tests/test_api_bebop_difftest.py | 8 + bbdev/bbdev | 3 + bbdev/mcp/tools/__init__.py | 2 + bbdev/mcp/tools/bebop_difftest_run.py | 41 ++++ 7 files changed, 362 insertions(+) create mode 100644 bbdev/api/steps/bebop/difftest/01_run_api.step.py create mode 100644 bbdev/api/steps/bebop/difftest/01_run_event.step.py create mode 100644 bbdev/api/tests/test_api_bebop_difftest.py create mode 100644 bbdev/mcp/tools/bebop_difftest_run.py diff --git a/bbdev/api/steps/bebop/difftest/01_run_api.step.py b/bbdev/api/steps/bebop/difftest/01_run_api.step.py new file mode 100644 index 000000000..cc8128d56 --- /dev/null +++ b/bbdev/api/steps/bebop/difftest/01_run_api.step.py @@ -0,0 +1,79 @@ +from pathlib import Path + +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path, get_verilator_build_dir + + +config = { + "name": "bebop-difftest-run-api", + "description": "Run Verilator+BEMU Bank DiffTest", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/difftest/run")], + "enqueues": ["bebop.difftest.run"], +} + + +def _required_string(body: dict, name: str) -> str | None: + value = body.get(name) + if not isinstance(value, str) or not value.strip() or value == "None": + return None + return value + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + required = {name: _required_string(body, name) for name in ("chip", "config", "binary")} + missing = [name for name, value in required.items() if value is None] + if missing: + missing_args = ", ".join("--" + name for name in missing) + return ApiResponse( + status=400, + body={ + "success": False, + "error": f"Missing required parameter(s): {missing_args}", + }, + ) + + unsupported = [name for name in ("rushB", "pk", "fast") if body.get(name)] + if unsupported: + unsupported_args = ", ".join("--" + name for name in unsupported) + return ApiResponse( + status=400, + body={ + "success": False, + "error": ( + "Bank DiffTest only supports normal baremetal ELF mode; " + f"unsupported: {unsupported_args}" + ), + }, + ) + + bbdir = get_buckyball_path() + chip = required["chip"] + manifest = Path(bbdir) / "examples" / "chips" / chip / "emu" / "Cargo.toml" + if not manifest.is_file(): + return ApiResponse( + status=400, + body={"success": False, "error": f"Chip BEMU manifest does not exist: {manifest}"}, + ) + + data = { + **required, + "jobs": body.get("jobs", 16), + "vsrc_dir": get_verilator_build_dir( + bbdir, + required["config"], + body.get("vsrc-dir") or body.get("output-dir"), + ), + "no_wave": body.get("no-wave", body.get("no_wave", False)), + "itrace": body.get("itrace", False), + "mtrace": body.get("mtrace", False), + "pmctrace": body.get("pmctrace", False), + "ctrace": body.get("ctrace", False), + "banktrace": body.get("banktrace", False), + "log_dir": body.get("log-dir") or body.get("log_dir"), + "_trace_id": ctx.trace_id, + } + await ctx.enqueue({"topic": "bebop.difftest.run", "data": data}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/difftest/01_run_event.step.py b/bbdev/api/steps/bebop/difftest/01_run_event.step.py new file mode 100644 index 000000000..0bea2ef03 --- /dev/null +++ b/bbdev/api/steps/bebop/difftest/01_run_event.step.py @@ -0,0 +1,228 @@ +"""Build and run a Chip-specific Verilator+BEMU Bank DiffTest workload.""" + +import os +import shlex +import sys +from datetime import datetime +from pathlib import Path + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path +from utils.search_workload import search_workload, search_workload_all +from utils.stream_run import stream_run_logger + + +config = { + "name": "bebop-difftest-run", + "description": "Build and run Verilator+BEMU Bank DiffTest", + "flows": ["bebop"], + "triggers": [queue("bebop.difftest.run")], + "enqueues": [], +} + + +def resolve_workload(bbdir: str, chip: str, binary: str) -> str | None: + candidate = Path(binary) + if not candidate.is_absolute(): + candidate = Path(bbdir) / candidate + if candidate.is_file(): + return str(candidate.resolve()) + + search_root = f"{bbdir}/bb-tests/output/workloads/src" + chip_match = search_workload(f"{search_root}/CTest/chips/{chip}", binary) + if chip_match is not None: + return chip_match + + matches = search_workload_all(search_root, binary) + chip_marker = f"{os.path.sep}CTest{os.path.sep}chips{os.path.sep}" + non_chip_matches = [path for path in matches if chip_marker not in path] + return non_chip_matches[0] if len(non_chip_matches) == 1 else None + + +async def fail(ctx, trace_id: str, error: str, **fields) -> None: + ctx.logger.error(fields.pop("message", error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": error, **fields}, + trace_id=trace_id, + ) + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + chip = input_data["chip"] + arch_config = input_data["config"] + binary_name = input_data["binary"] + + chip_emu_dir = Path(bbdir) / "examples" / "chips" / chip / "emu" + manifest = chip_emu_dir / "Cargo.toml" + if not manifest.is_file(): + await fail(ctx, origin_tid, "invalid_chip", chip=chip, manifest=str(manifest)) + return + + vsrc_dir = os.path.abspath(input_data["vsrc_dir"]) + if not os.path.isdir(vsrc_dir): + await fail( + ctx, + origin_tid, + "vsrc_not_found", + config=arch_config, + vsrc_dir=vsrc_dir, + message=f"RTL directory does not exist: {vsrc_dir}; generate Verilog first", + ) + return + + binary_path = resolve_workload(bbdir, chip, binary_name) + if binary_path is None: + await fail(ctx, origin_tid, "binary_not_found", chip=chip, binary=binary_name) + return + + try: + jobs = int(input_data.get("jobs", 16)) + if jobs <= 0: + raise ValueError + except (TypeError, ValueError): + await fail(ctx, origin_tid, "invalid_jobs", jobs=input_data.get("jobs")) + return + + # sourceme.sh points RISCV at the repository's `result` environment. Keep + # it current when a newly required native dependency is not present. + dramsim_header = Path(bbdir) / "result" / "include" / "dramsim3.h" + if not dramsim_header.is_file(): + ctx.logger.info( + f"Nix environment is missing {dramsim_header}; rebuilding result" + ) + env_result = stream_run_logger( + cmd="nix build", + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="bebop difftest environment", + stderr_prefix="bebop difftest environment", + ) + if env_result.returncode != 0: + await check_result( + ctx, + env_result.returncode, + continue_run=False, + extra_fields={ + "task": "environment", + "backend": "verilator+bemu-difftest", + "chip": chip, + "config": arch_config, + "dependency": str(dramsim_header), + }, + trace_id=origin_tid, + ) + return + + bebop_bin = chip_emu_dir / "target" / "release" / "bebop" + cargo_build_cmd = shlex.join( + [ + "cargo", + "build", + "--release", + "--bin", + "bebop", + "--features", + "verilator,bemu,difftest", + "--jobs", + str(jobs), + ] + ) + build_inner = f"cd {shlex.quote(str(chip_emu_dir))} && exec {cargo_build_cmd}" + build_cmd = f"nix develop -c sh -c {shlex.quote(build_inner)}" + ctx.logger.info(f"Building Chip DiffTest executable: {build_cmd}") + build_result = stream_run_logger( + cmd=build_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="bebop difftest build", + stderr_prefix="bebop difftest build", + env={**os.environ, "VSRC_PATH": vsrc_dir}, + ) + if build_result.returncode != 0: + await check_result( + ctx, + build_result.returncode, + continue_run=False, + extra_fields={ + "task": "build", + "backend": "verilator+bemu-difftest", + "chip": chip, + "config": arch_config, + "binary": str(bebop_bin), + "vsrc_dir": vsrc_dir, + }, + trace_id=origin_tid, + ) + return + + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + requested_log_dir = input_data.get("log_dir") + if requested_log_dir: + log_dir = os.path.abspath(os.path.join(bbdir, requested_log_dir)) + else: + log_dir = f"{bbdir}/log/{timestamp}-difftest-{binary_name}" + os.makedirs(log_dir, exist_ok=True) + + run_args = [ + str(bebop_bin), + "run", + "verilator", + f"--elf={binary_path}", + f"--log-dir={log_dir}", + "--diff", + ] + if input_data.get("no_wave"): + run_args.append("--no-wave") + for trace_name in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace"): + if input_data.get(trace_name): + run_args.append(f"--{trace_name}") + + preload = os.pathsep.join( + path + for path in ( + f"{bbdir}/result/lib/libdramsim3.so", + os.environ.get("LD_PRELOAD", ""), + ) + if path + ) + run_inner = ( + f"cd {shlex.quote(str(chip_emu_dir))} && " + f"export LD_PRELOAD={shlex.quote(preload)} && exec {shlex.join(run_args)}" + ) + run_cmd = f"nix develop -c sh -c {shlex.quote(run_inner)}" + ctx.logger.info(f"Running Bank DiffTest: {run_cmd}") + run_result = stream_run_logger( + cmd=run_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="bebop difftest", + stderr_prefix="bebop difftest", + ) + await check_result( + ctx, + run_result.returncode, + continue_run=False, + extra_fields={ + "task": "difftest", + "backend": "verilator+bemu-difftest", + "chip": chip, + "config": arch_config, + "binary": binary_path, + "vsrc_dir": vsrc_dir, + "log_dir": log_dir, + "bank_diff": os.path.join(log_dir, "bank_diff.ndjson"), + "timestamp": timestamp, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index 2f3cd8819..37bc8763f 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -32,6 +32,7 @@ def main(): "test_api_bebop_verilator_sim.py", "test_api_bebop_verilator_run.py", "test_api_bebop_verilator_batch.py", + "test_api_bebop_difftest.py", "test_api_bebop_p2e_clean.py", "test_api_bebop_p2e_verilog.py", "test_api_bebop_p2e_buildbitstream.py", diff --git a/bbdev/api/tests/test_api_bebop_difftest.py b/bbdev/api/tests/test_api_bebop_difftest.py new file mode 100644 index 000000000..986f62d20 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_difftest.py @@ -0,0 +1,8 @@ +from _api_test_helper import run_bbdev_case + + +run_bbdev_case( + "bbdev bebop-difftest --run " + "'--chip toy --config sims.verilator.BuckyballToyVerilatorConfig " + "--binary toy_im2col_k3_test-singlecore-baremetal --no-wave'" +) diff --git a/bbdev/bbdev b/bbdev/bbdev index 00ef16143..b19233fef 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -70,6 +70,9 @@ WORKFLOW_COMMANDS = { "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--rushB] [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--rushB] [--clean-before]"', }, + "bebop-difftest": { + "run": 'Run Verilator+BEMU Bank DiffTest. Args: "--chip --config --binary [--jobs ] [--log-dir ] [--no-wave]"', + }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--config [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--config [--output-dir ]"', diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index 80d0d4925..0c017f470 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -16,6 +16,7 @@ from . import bebop_verilator_sim from . import bebop_verilator_run from . import bebop_verilator_batch +from . import bebop_difftest_run from . import verilator_clean from . import verilator_verilog from . import verilator_build @@ -55,6 +56,7 @@ bebop_verilator_sim, bebop_verilator_run, bebop_verilator_batch, + bebop_difftest_run, verilator_clean, verilator_verilog, verilator_build, diff --git a/bbdev/mcp/tools/bebop_difftest_run.py b/bbdev/mcp/tools/bebop_difftest_run.py new file mode 100644 index 000000000..5122ca52a --- /dev/null +++ b/bbdev/mcp/tools/bebop_difftest_run.py @@ -0,0 +1,41 @@ +"""MCP tool: bbdev_bebop_difftest_run.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from common import submit, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_bebop_difftest_run( + chip: str, + config: str, + binary: str, + jobs: int = 16, + no_wave: bool = True, + itrace: bool = False, + mtrace: bool = False, + pmctrace: bool = False, + ctrace: bool = False, + banktrace: bool = False, + log_dir: Optional[str] = None, + ) -> str: + """Build and run Verilator+BEMU Bank DiffTest. POST /bebop/difftest/run.""" + for name, value in (("chip", chip), ("config", config), ("binary", binary)): + if error := need(name, value): + return err(error) + params: Dict[str, Any] = { + "chip": chip, + "config": config, + "binary": binary, + "jobs": jobs, + "no-wave": no_wave, + "itrace": itrace, + "mtrace": mtrace, + "pmctrace": pmctrace, + "ctrace": ctrace, + "banktrace": banktrace, + } + return fmt(submit("/bebop/difftest/run", opt(params, **{"log-dir": log_dir}))) From 70e06a90da4711c5a063a5adc5c4ec6428b97d27 Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Tue, 11 Aug 2026 01:16:03 +0800 Subject: [PATCH 115/196] refactor: make difftest workflow backend-oriented --- .../{bebop => }/difftest/01_run_api.step.py | 14 ++++----- .../{bebop => }/difftest/01_run_event.step.py | 30 +++++++++---------- bbdev/api/tests/test_all_api_serial.py | 2 +- ...bebop_difftest.py => test_api_difftest.py} | 2 +- bbdev/bbdev | 4 +-- bbdev/mcp/tools/__init__.py | 4 +-- ...{bebop_difftest_run.py => difftest_run.py} | 8 ++--- 7 files changed, 32 insertions(+), 32 deletions(-) rename bbdev/api/steps/{bebop => }/difftest/01_run_api.step.py (85%) rename bbdev/api/steps/{bebop => }/difftest/01_run_event.step.py (90%) rename bbdev/api/tests/{test_api_bebop_difftest.py => test_api_difftest.py} (85%) rename bbdev/mcp/tools/{bebop_difftest_run.py => difftest_run.py} (80%) diff --git a/bbdev/api/steps/bebop/difftest/01_run_api.step.py b/bbdev/api/steps/difftest/01_run_api.step.py similarity index 85% rename from bbdev/api/steps/bebop/difftest/01_run_api.step.py rename to bbdev/api/steps/difftest/01_run_api.step.py index cc8128d56..293dbd938 100644 --- a/bbdev/api/steps/bebop/difftest/01_run_api.step.py +++ b/bbdev/api/steps/difftest/01_run_api.step.py @@ -6,11 +6,11 @@ config = { - "name": "bebop-difftest-run-api", - "description": "Run Verilator+BEMU Bank DiffTest", - "flows": ["bebop"], - "triggers": [api("POST", "/bebop/difftest/run")], - "enqueues": ["bebop.difftest.run"], + "name": "difftest-run-api", + "description": "Run Bank DiffTest on Verilator RTL with BEMU as the reference model", + "flows": ["difftest"], + "triggers": [api("POST", "/difftest/run")], + "enqueues": ["difftest.run"], } @@ -55,7 +55,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: if not manifest.is_file(): return ApiResponse( status=400, - body={"success": False, "error": f"Chip BEMU manifest does not exist: {manifest}"}, + body={"success": False, "error": f"Chip emulator manifest does not exist: {manifest}"}, ) data = { @@ -75,5 +75,5 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "log_dir": body.get("log-dir") or body.get("log_dir"), "_trace_id": ctx.trace_id, } - await ctx.enqueue({"topic": "bebop.difftest.run", "data": data}) + await ctx.enqueue({"topic": "difftest.run", "data": data}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/difftest/01_run_event.step.py b/bbdev/api/steps/difftest/01_run_event.step.py similarity index 90% rename from bbdev/api/steps/bebop/difftest/01_run_event.step.py rename to bbdev/api/steps/difftest/01_run_event.step.py index 0bea2ef03..1556d8df2 100644 --- a/bbdev/api/steps/bebop/difftest/01_run_event.step.py +++ b/bbdev/api/steps/difftest/01_run_event.step.py @@ -1,4 +1,4 @@ -"""Build and run a Chip-specific Verilator+BEMU Bank DiffTest workload.""" +"""Run Bank DiffTest on Verilator RTL using BEMU as the reference model.""" import os import shlex @@ -8,7 +8,7 @@ from motia import FlowContext, queue -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) @@ -19,10 +19,10 @@ config = { - "name": "bebop-difftest-run", - "description": "Build and run Verilator+BEMU Bank DiffTest", - "flows": ["bebop"], - "triggers": [queue("bebop.difftest.run")], + "name": "difftest-run", + "description": "Run Bank DiffTest on Verilator RTL with BEMU as the reference model", + "flows": ["difftest"], + "triggers": [queue("difftest.run")], "enqueues": [], } @@ -105,8 +105,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cmd="nix build", logger=ctx.logger, cwd=bbdir, - stdout_prefix="bebop difftest environment", - stderr_prefix="bebop difftest environment", + stdout_prefix="difftest environment", + stderr_prefix="difftest environment", ) if env_result.returncode != 0: await check_result( @@ -115,7 +115,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "environment", - "backend": "verilator+bemu-difftest", + "backend": "verilator", "chip": chip, "config": arch_config, "dependency": str(dramsim_header), @@ -145,8 +145,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cmd=build_cmd, logger=ctx.logger, cwd=bbdir, - stdout_prefix="bebop difftest build", - stderr_prefix="bebop difftest build", + stdout_prefix="difftest build", + stderr_prefix="difftest build", env={**os.environ, "VSRC_PATH": vsrc_dir}, ) if build_result.returncode != 0: @@ -156,7 +156,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "build", - "backend": "verilator+bemu-difftest", + "backend": "verilator", "chip": chip, "config": arch_config, "binary": str(bebop_bin), @@ -206,8 +206,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cmd=run_cmd, logger=ctx.logger, cwd=bbdir, - stdout_prefix="bebop difftest", - stderr_prefix="bebop difftest", + stdout_prefix="difftest", + stderr_prefix="difftest", ) await check_result( ctx, @@ -215,7 +215,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "difftest", - "backend": "verilator+bemu-difftest", + "backend": "verilator", "chip": chip, "config": arch_config, "binary": binary_path, diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index 37bc8763f..a85b211fd 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -32,7 +32,7 @@ def main(): "test_api_bebop_verilator_sim.py", "test_api_bebop_verilator_run.py", "test_api_bebop_verilator_batch.py", - "test_api_bebop_difftest.py", + "test_api_difftest.py", "test_api_bebop_p2e_clean.py", "test_api_bebop_p2e_verilog.py", "test_api_bebop_p2e_buildbitstream.py", diff --git a/bbdev/api/tests/test_api_bebop_difftest.py b/bbdev/api/tests/test_api_difftest.py similarity index 85% rename from bbdev/api/tests/test_api_bebop_difftest.py rename to bbdev/api/tests/test_api_difftest.py index 986f62d20..731694be6 100644 --- a/bbdev/api/tests/test_api_bebop_difftest.py +++ b/bbdev/api/tests/test_api_difftest.py @@ -2,7 +2,7 @@ run_bbdev_case( - "bbdev bebop-difftest --run " + "bbdev difftest --run " "'--chip toy --config sims.verilator.BuckyballToyVerilatorConfig " "--binary toy_im2col_k3_test-singlecore-baremetal --no-wave'" ) diff --git a/bbdev/bbdev b/bbdev/bbdev index b19233fef..bfdc22211 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -70,8 +70,8 @@ WORKFLOW_COMMANDS = { "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--rushB] [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--rushB] [--clean-before]"', }, - "bebop-difftest": { - "run": 'Run Verilator+BEMU Bank DiffTest. Args: "--chip --config --binary [--jobs ] [--log-dir ] [--no-wave]"', + "difftest": { + "run": 'Use BEMU as the reference model to run Bank DiffTest on Verilator RTL. Args: "--chip --config --binary [--jobs ] [--log-dir ] [--no-wave]"', }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--config [--output-dir ]"', diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index 0c017f470..29a35d1a7 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -16,7 +16,7 @@ from . import bebop_verilator_sim from . import bebop_verilator_run from . import bebop_verilator_batch -from . import bebop_difftest_run +from . import difftest_run from . import verilator_clean from . import verilator_verilog from . import verilator_build @@ -56,7 +56,7 @@ bebop_verilator_sim, bebop_verilator_run, bebop_verilator_batch, - bebop_difftest_run, + difftest_run, verilator_clean, verilator_verilog, verilator_build, diff --git a/bbdev/mcp/tools/bebop_difftest_run.py b/bbdev/mcp/tools/difftest_run.py similarity index 80% rename from bbdev/mcp/tools/bebop_difftest_run.py rename to bbdev/mcp/tools/difftest_run.py index 5122ca52a..3c9b487c8 100644 --- a/bbdev/mcp/tools/bebop_difftest_run.py +++ b/bbdev/mcp/tools/difftest_run.py @@ -1,4 +1,4 @@ -"""MCP tool: bbdev_bebop_difftest_run.""" +"""MCP tool: bbdev_difftest_run.""" from __future__ import annotations @@ -9,7 +9,7 @@ def register(mcp): @mcp.tool() - def bbdev_bebop_difftest_run( + def bbdev_difftest_run( chip: str, config: str, binary: str, @@ -22,7 +22,7 @@ def bbdev_bebop_difftest_run( banktrace: bool = False, log_dir: Optional[str] = None, ) -> str: - """Build and run Verilator+BEMU Bank DiffTest. POST /bebop/difftest/run.""" + """Run Bank DiffTest on Verilator RTL using BEMU as reference. POST /difftest/run.""" for name, value in (("chip", chip), ("config", config), ("binary", binary)): if error := need(name, value): return err(error) @@ -38,4 +38,4 @@ def bbdev_bebop_difftest_run( "ctrace": ctrace, "banktrace": banktrace, } - return fmt(submit("/bebop/difftest/run", opt(params, **{"log-dir": log_dir}))) + return fmt(submit("/difftest/run", opt(params, **{"log-dir": log_dir}))) From f4c3241bde02ab8e7f283fd94e3f5f08bade96ef Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Tue, 11 Aug 2026 01:25:01 +0800 Subject: [PATCH 116/196] feat: select difftest backend explicitly --- bbdev/api/steps/difftest/01_run_api.step.py | 38 ++++++++++++++++- bbdev/api/steps/difftest/01_run_event.step.py | 21 ++++++++-- bbdev/api/tests/test_api_difftest.py | 3 +- bbdev/bbdev | 2 +- bbdev/mcp/tools/difftest_run.py | 41 ++++++++++++++----- 5 files changed, 86 insertions(+), 19 deletions(-) diff --git a/bbdev/api/steps/difftest/01_run_api.step.py b/bbdev/api/steps/difftest/01_run_api.step.py index 293dbd938..8319b2004 100644 --- a/bbdev/api/steps/difftest/01_run_api.step.py +++ b/bbdev/api/steps/difftest/01_run_api.step.py @@ -7,7 +7,7 @@ config = { "name": "difftest-run-api", - "description": "Run Bank DiffTest on Verilator RTL with BEMU as the reference model", + "description": "Run Bank DiffTest on a selected backend with BEMU as the reference model", "flows": ["difftest"], "triggers": [api("POST", "/difftest/run")], "enqueues": ["difftest.run"], @@ -23,7 +23,40 @@ def _required_string(body: dict, name: str) -> str | None: async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - required = {name: _required_string(body, name) for name in ("chip", "config", "binary")} + backend = _required_string(body, "backend") + if backend is None: + return ApiResponse( + status=400, + body={ + "success": False, + "error": "Missing required parameter(s): --backend", + }, + ) + + known_backends = ("verilator", "p2e") + if backend not in known_backends: + return ApiResponse( + status=400, + body={ + "success": False, + "error": ( + f"Unsupported DiffTest backend: {backend}; " + f"available backends: {', '.join(known_backends)}" + ), + }, + ) + if backend == "p2e": + return ApiResponse( + status=501, + body={ + "success": False, + "error": "DiffTest backend p2e is reserved but not implemented yet", + }, + ) + + required = { + name: _required_string(body, name) for name in ("chip", "config", "binary") + } missing = [name for name, value in required.items() if value is None] if missing: missing_args = ", ".join("--" + name for name in missing) @@ -59,6 +92,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: ) data = { + "backend": backend, **required, "jobs": body.get("jobs", 16), "vsrc_dir": get_verilator_build_dir( diff --git a/bbdev/api/steps/difftest/01_run_event.step.py b/bbdev/api/steps/difftest/01_run_event.step.py index 1556d8df2..2c24beb32 100644 --- a/bbdev/api/steps/difftest/01_run_event.step.py +++ b/bbdev/api/steps/difftest/01_run_event.step.py @@ -58,6 +58,19 @@ async def fail(ctx, trace_id: str, error: str, **fields) -> None: async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) + backend = input_data.get("backend") + if backend != "verilator": + error = "backend_not_implemented" if backend == "p2e" else "unsupported_backend" + await fail( + ctx, + origin_tid, + error, + backend=backend, + available_backends=["verilator", "p2e"], + implemented_backends=["verilator"], + ) + return + bbdir = get_buckyball_path() chip = input_data["chip"] arch_config = input_data["config"] @@ -115,7 +128,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "environment", - "backend": "verilator", + "backend": backend, "chip": chip, "config": arch_config, "dependency": str(dramsim_header), @@ -156,7 +169,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "build", - "backend": "verilator", + "backend": backend, "chip": chip, "config": arch_config, "binary": str(bebop_bin), @@ -177,7 +190,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: run_args = [ str(bebop_bin), "run", - "verilator", + backend, f"--elf={binary_path}", f"--log-dir={log_dir}", "--diff", @@ -215,7 +228,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "difftest", - "backend": "verilator", + "backend": backend, "chip": chip, "config": arch_config, "binary": binary_path, diff --git a/bbdev/api/tests/test_api_difftest.py b/bbdev/api/tests/test_api_difftest.py index 731694be6..24ae9e4a8 100644 --- a/bbdev/api/tests/test_api_difftest.py +++ b/bbdev/api/tests/test_api_difftest.py @@ -3,6 +3,7 @@ run_bbdev_case( "bbdev difftest --run " - "'--chip toy --config sims.verilator.BuckyballToyVerilatorConfig " + "'--backend verilator --chip toy " + "--config sims.verilator.BuckyballToyVerilatorConfig " "--binary toy_im2col_k3_test-singlecore-baremetal --no-wave'" ) diff --git a/bbdev/bbdev b/bbdev/bbdev index bfdc22211..488ef6ec2 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -71,7 +71,7 @@ WORKFLOW_COMMANDS = { "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--rushB] [--clean-before]"', }, "difftest": { - "run": 'Use BEMU as the reference model to run Bank DiffTest on Verilator RTL. Args: "--chip --config --binary [--jobs ] [--log-dir ] [--no-wave]"', + "run": 'Use BEMU as the reference model to run Bank DiffTest on a selected backend. Args: "--backend --chip --config --binary [--jobs ] [--log-dir ] [--no-wave]"; p2e is reserved but not implemented yet.', }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--config [--output-dir ]"', diff --git a/bbdev/mcp/tools/difftest_run.py b/bbdev/mcp/tools/difftest_run.py index 3c9b487c8..3e91c268b 100644 --- a/bbdev/mcp/tools/difftest_run.py +++ b/bbdev/mcp/tools/difftest_run.py @@ -10,9 +10,10 @@ def register(mcp): @mcp.tool() def bbdev_difftest_run( - chip: str, - config: str, - binary: str, + backend: str, + chip: Optional[str] = None, + config: Optional[str] = None, + binary: Optional[str] = None, jobs: int = 16, no_wave: bool = True, itrace: bool = False, @@ -22,14 +23,21 @@ def bbdev_difftest_run( banktrace: bool = False, log_dir: Optional[str] = None, ) -> str: - """Run Bank DiffTest on Verilator RTL using BEMU as reference. POST /difftest/run.""" - for name, value in (("chip", chip), ("config", config), ("binary", binary)): - if error := need(name, value): - return err(error) + """Run Bank DiffTest on a selected backend using BEMU as reference. POST /difftest/run.""" + if error := need("backend", backend): + return err(error) + known_backends = ("verilator", "p2e") + if backend not in known_backends: + return err( + f"Unsupported DiffTest backend: {backend}; " + f"available backends: {', '.join(known_backends)}" + ) + if backend == "verilator": + for name, value in (("chip", chip), ("config", config), ("binary", binary)): + if error := need(name, value): + return err(error) params: Dict[str, Any] = { - "chip": chip, - "config": config, - "binary": binary, + "backend": backend, "jobs": jobs, "no-wave": no_wave, "itrace": itrace, @@ -38,4 +46,15 @@ def bbdev_difftest_run( "ctrace": ctrace, "banktrace": banktrace, } - return fmt(submit("/difftest/run", opt(params, **{"log-dir": log_dir}))) + return fmt( + submit( + "/difftest/run", + opt( + params, + chip=chip, + config=config, + binary=binary, + **{"log-dir": log_dir}, + ), + ) + ) From ade31fd709941d7fc1473998606a8587ee690e8f Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Tue, 11 Aug 2026 13:34:23 +0800 Subject: [PATCH 117/196] refactor: integrate difftest into Bebop Verilator run --- bbdev/api/steps/difftest/01_run_api.step.py | 113 -------- bbdev/api/steps/difftest/01_run_event.step.py | 241 ------------------ bbdev/api/tests/test_all_api_serial.py | 1 - bbdev/api/tests/test_api_difftest.py | 9 - bbdev/bbdev | 3 - bbdev/mcp/tools/__init__.py | 2 - bbdev/mcp/tools/difftest_run.py | 60 ----- 7 files changed, 429 deletions(-) delete mode 100644 bbdev/api/steps/difftest/01_run_api.step.py delete mode 100644 bbdev/api/steps/difftest/01_run_event.step.py delete mode 100644 bbdev/api/tests/test_api_difftest.py delete mode 100644 bbdev/mcp/tools/difftest_run.py diff --git a/bbdev/api/steps/difftest/01_run_api.step.py b/bbdev/api/steps/difftest/01_run_api.step.py deleted file mode 100644 index 8319b2004..000000000 --- a/bbdev/api/steps/difftest/01_run_api.step.py +++ /dev/null @@ -1,113 +0,0 @@ -from pathlib import Path - -from motia import ApiRequest, ApiResponse, FlowContext, api - -from utils.path import get_buckyball_path, get_verilator_build_dir - - -config = { - "name": "difftest-run-api", - "description": "Run Bank DiffTest on a selected backend with BEMU as the reference model", - "flows": ["difftest"], - "triggers": [api("POST", "/difftest/run")], - "enqueues": ["difftest.run"], -} - - -def _required_string(body: dict, name: str) -> str | None: - value = body.get(name) - if not isinstance(value, str) or not value.strip() or value == "None": - return None - return value - - -async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = request.body or {} - backend = _required_string(body, "backend") - if backend is None: - return ApiResponse( - status=400, - body={ - "success": False, - "error": "Missing required parameter(s): --backend", - }, - ) - - known_backends = ("verilator", "p2e") - if backend not in known_backends: - return ApiResponse( - status=400, - body={ - "success": False, - "error": ( - f"Unsupported DiffTest backend: {backend}; " - f"available backends: {', '.join(known_backends)}" - ), - }, - ) - if backend == "p2e": - return ApiResponse( - status=501, - body={ - "success": False, - "error": "DiffTest backend p2e is reserved but not implemented yet", - }, - ) - - required = { - name: _required_string(body, name) for name in ("chip", "config", "binary") - } - missing = [name for name, value in required.items() if value is None] - if missing: - missing_args = ", ".join("--" + name for name in missing) - return ApiResponse( - status=400, - body={ - "success": False, - "error": f"Missing required parameter(s): {missing_args}", - }, - ) - - unsupported = [name for name in ("rushB", "pk", "fast") if body.get(name)] - if unsupported: - unsupported_args = ", ".join("--" + name for name in unsupported) - return ApiResponse( - status=400, - body={ - "success": False, - "error": ( - "Bank DiffTest only supports normal baremetal ELF mode; " - f"unsupported: {unsupported_args}" - ), - }, - ) - - bbdir = get_buckyball_path() - chip = required["chip"] - manifest = Path(bbdir) / "examples" / "chips" / chip / "emu" / "Cargo.toml" - if not manifest.is_file(): - return ApiResponse( - status=400, - body={"success": False, "error": f"Chip emulator manifest does not exist: {manifest}"}, - ) - - data = { - "backend": backend, - **required, - "jobs": body.get("jobs", 16), - "vsrc_dir": get_verilator_build_dir( - bbdir, - required["config"], - body.get("vsrc-dir") or body.get("output-dir"), - ), - "no_wave": body.get("no-wave", body.get("no_wave", False)), - "itrace": body.get("itrace", False), - "mtrace": body.get("mtrace", False), - "pmctrace": body.get("pmctrace", False), - "ctrace": body.get("ctrace", False), - "banktrace": body.get("banktrace", False), - "log_dir": body.get("log-dir") or body.get("log_dir"), - "_trace_id": ctx.trace_id, - } - await ctx.enqueue({"topic": "difftest.run", "data": data}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/difftest/01_run_event.step.py b/bbdev/api/steps/difftest/01_run_event.step.py deleted file mode 100644 index 2c24beb32..000000000 --- a/bbdev/api/steps/difftest/01_run_event.step.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Run Bank DiffTest on Verilator RTL using BEMU as the reference model.""" - -import os -import shlex -import sys -from datetime import datetime -from pathlib import Path - -from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.event_common import check_result, get_origin_trace_id -from utils.path import get_buckyball_path -from utils.search_workload import search_workload, search_workload_all -from utils.stream_run import stream_run_logger - - -config = { - "name": "difftest-run", - "description": "Run Bank DiffTest on Verilator RTL with BEMU as the reference model", - "flows": ["difftest"], - "triggers": [queue("difftest.run")], - "enqueues": [], -} - - -def resolve_workload(bbdir: str, chip: str, binary: str) -> str | None: - candidate = Path(binary) - if not candidate.is_absolute(): - candidate = Path(bbdir) / candidate - if candidate.is_file(): - return str(candidate.resolve()) - - search_root = f"{bbdir}/bb-tests/output/workloads/src" - chip_match = search_workload(f"{search_root}/CTest/chips/{chip}", binary) - if chip_match is not None: - return chip_match - - matches = search_workload_all(search_root, binary) - chip_marker = f"{os.path.sep}CTest{os.path.sep}chips{os.path.sep}" - non_chip_matches = [path for path in matches if chip_marker not in path] - return non_chip_matches[0] if len(non_chip_matches) == 1 else None - - -async def fail(ctx, trace_id: str, error: str, **fields) -> None: - ctx.logger.error(fields.pop("message", error)) - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": error, **fields}, - trace_id=trace_id, - ) - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - backend = input_data.get("backend") - if backend != "verilator": - error = "backend_not_implemented" if backend == "p2e" else "unsupported_backend" - await fail( - ctx, - origin_tid, - error, - backend=backend, - available_backends=["verilator", "p2e"], - implemented_backends=["verilator"], - ) - return - - bbdir = get_buckyball_path() - chip = input_data["chip"] - arch_config = input_data["config"] - binary_name = input_data["binary"] - - chip_emu_dir = Path(bbdir) / "examples" / "chips" / chip / "emu" - manifest = chip_emu_dir / "Cargo.toml" - if not manifest.is_file(): - await fail(ctx, origin_tid, "invalid_chip", chip=chip, manifest=str(manifest)) - return - - vsrc_dir = os.path.abspath(input_data["vsrc_dir"]) - if not os.path.isdir(vsrc_dir): - await fail( - ctx, - origin_tid, - "vsrc_not_found", - config=arch_config, - vsrc_dir=vsrc_dir, - message=f"RTL directory does not exist: {vsrc_dir}; generate Verilog first", - ) - return - - binary_path = resolve_workload(bbdir, chip, binary_name) - if binary_path is None: - await fail(ctx, origin_tid, "binary_not_found", chip=chip, binary=binary_name) - return - - try: - jobs = int(input_data.get("jobs", 16)) - if jobs <= 0: - raise ValueError - except (TypeError, ValueError): - await fail(ctx, origin_tid, "invalid_jobs", jobs=input_data.get("jobs")) - return - - # sourceme.sh points RISCV at the repository's `result` environment. Keep - # it current when a newly required native dependency is not present. - dramsim_header = Path(bbdir) / "result" / "include" / "dramsim3.h" - if not dramsim_header.is_file(): - ctx.logger.info( - f"Nix environment is missing {dramsim_header}; rebuilding result" - ) - env_result = stream_run_logger( - cmd="nix build", - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="difftest environment", - stderr_prefix="difftest environment", - ) - if env_result.returncode != 0: - await check_result( - ctx, - env_result.returncode, - continue_run=False, - extra_fields={ - "task": "environment", - "backend": backend, - "chip": chip, - "config": arch_config, - "dependency": str(dramsim_header), - }, - trace_id=origin_tid, - ) - return - - bebop_bin = chip_emu_dir / "target" / "release" / "bebop" - cargo_build_cmd = shlex.join( - [ - "cargo", - "build", - "--release", - "--bin", - "bebop", - "--features", - "verilator,bemu,difftest", - "--jobs", - str(jobs), - ] - ) - build_inner = f"cd {shlex.quote(str(chip_emu_dir))} && exec {cargo_build_cmd}" - build_cmd = f"nix develop -c sh -c {shlex.quote(build_inner)}" - ctx.logger.info(f"Building Chip DiffTest executable: {build_cmd}") - build_result = stream_run_logger( - cmd=build_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="difftest build", - stderr_prefix="difftest build", - env={**os.environ, "VSRC_PATH": vsrc_dir}, - ) - if build_result.returncode != 0: - await check_result( - ctx, - build_result.returncode, - continue_run=False, - extra_fields={ - "task": "build", - "backend": backend, - "chip": chip, - "config": arch_config, - "binary": str(bebop_bin), - "vsrc_dir": vsrc_dir, - }, - trace_id=origin_tid, - ) - return - - timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") - requested_log_dir = input_data.get("log_dir") - if requested_log_dir: - log_dir = os.path.abspath(os.path.join(bbdir, requested_log_dir)) - else: - log_dir = f"{bbdir}/log/{timestamp}-difftest-{binary_name}" - os.makedirs(log_dir, exist_ok=True) - - run_args = [ - str(bebop_bin), - "run", - backend, - f"--elf={binary_path}", - f"--log-dir={log_dir}", - "--diff", - ] - if input_data.get("no_wave"): - run_args.append("--no-wave") - for trace_name in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace"): - if input_data.get(trace_name): - run_args.append(f"--{trace_name}") - - preload = os.pathsep.join( - path - for path in ( - f"{bbdir}/result/lib/libdramsim3.so", - os.environ.get("LD_PRELOAD", ""), - ) - if path - ) - run_inner = ( - f"cd {shlex.quote(str(chip_emu_dir))} && " - f"export LD_PRELOAD={shlex.quote(preload)} && exec {shlex.join(run_args)}" - ) - run_cmd = f"nix develop -c sh -c {shlex.quote(run_inner)}" - ctx.logger.info(f"Running Bank DiffTest: {run_cmd}") - run_result = stream_run_logger( - cmd=run_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="difftest", - stderr_prefix="difftest", - ) - await check_result( - ctx, - run_result.returncode, - continue_run=False, - extra_fields={ - "task": "difftest", - "backend": backend, - "chip": chip, - "config": arch_config, - "binary": binary_path, - "vsrc_dir": vsrc_dir, - "log_dir": log_dir, - "bank_diff": os.path.join(log_dir, "bank_diff.ndjson"), - "timestamp": timestamp, - }, - trace_id=origin_tid, - ) diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index a85b211fd..2f3cd8819 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -32,7 +32,6 @@ def main(): "test_api_bebop_verilator_sim.py", "test_api_bebop_verilator_run.py", "test_api_bebop_verilator_batch.py", - "test_api_difftest.py", "test_api_bebop_p2e_clean.py", "test_api_bebop_p2e_verilog.py", "test_api_bebop_p2e_buildbitstream.py", diff --git a/bbdev/api/tests/test_api_difftest.py b/bbdev/api/tests/test_api_difftest.py deleted file mode 100644 index 24ae9e4a8..000000000 --- a/bbdev/api/tests/test_api_difftest.py +++ /dev/null @@ -1,9 +0,0 @@ -from _api_test_helper import run_bbdev_case - - -run_bbdev_case( - "bbdev difftest --run " - "'--backend verilator --chip toy " - "--config sims.verilator.BuckyballToyVerilatorConfig " - "--binary toy_im2col_k3_test-singlecore-baremetal --no-wave'" -) diff --git a/bbdev/bbdev b/bbdev/bbdev index 488ef6ec2..00ef16143 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -70,9 +70,6 @@ WORKFLOW_COMMANDS = { "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--rushB] [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--rushB] [--clean-before]"', }, - "difftest": { - "run": 'Use BEMU as the reference model to run Bank DiffTest on a selected backend. Args: "--backend --chip --config --binary [--jobs ] [--log-dir ] [--no-wave]"; p2e is reserved but not implemented yet.', - }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--config [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--config [--output-dir ]"', diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index 29a35d1a7..80d0d4925 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -16,7 +16,6 @@ from . import bebop_verilator_sim from . import bebop_verilator_run from . import bebop_verilator_batch -from . import difftest_run from . import verilator_clean from . import verilator_verilog from . import verilator_build @@ -56,7 +55,6 @@ bebop_verilator_sim, bebop_verilator_run, bebop_verilator_batch, - difftest_run, verilator_clean, verilator_verilog, verilator_build, diff --git a/bbdev/mcp/tools/difftest_run.py b/bbdev/mcp/tools/difftest_run.py deleted file mode 100644 index 3e91c268b..000000000 --- a/bbdev/mcp/tools/difftest_run.py +++ /dev/null @@ -1,60 +0,0 @@ -"""MCP tool: bbdev_difftest_run.""" - -from __future__ import annotations - -from typing import Any, Dict, Optional - -from common import submit, err, fmt, need, opt - - -def register(mcp): - @mcp.tool() - def bbdev_difftest_run( - backend: str, - chip: Optional[str] = None, - config: Optional[str] = None, - binary: Optional[str] = None, - jobs: int = 16, - no_wave: bool = True, - itrace: bool = False, - mtrace: bool = False, - pmctrace: bool = False, - ctrace: bool = False, - banktrace: bool = False, - log_dir: Optional[str] = None, - ) -> str: - """Run Bank DiffTest on a selected backend using BEMU as reference. POST /difftest/run.""" - if error := need("backend", backend): - return err(error) - known_backends = ("verilator", "p2e") - if backend not in known_backends: - return err( - f"Unsupported DiffTest backend: {backend}; " - f"available backends: {', '.join(known_backends)}" - ) - if backend == "verilator": - for name, value in (("chip", chip), ("config", config), ("binary", binary)): - if error := need(name, value): - return err(error) - params: Dict[str, Any] = { - "backend": backend, - "jobs": jobs, - "no-wave": no_wave, - "itrace": itrace, - "mtrace": mtrace, - "pmctrace": pmctrace, - "ctrace": ctrace, - "banktrace": banktrace, - } - return fmt( - submit( - "/difftest/run", - opt( - params, - chip=chip, - config=config, - binary=binary, - **{"log-dir": log_dir}, - ), - ) - ) From e655a587db1601c6abeeae52b79e7fe2dfa18ffe Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Tue, 11 Aug 2026 23:05:31 +0800 Subject: [PATCH 118/196] feat: add Bebop difftest batch workflow --- bbdev/api/steps/bebop/regression.py | 9 ++- .../bebop/verilator/03_build_api.step.py | 1 + .../bebop/verilator/06_batch_api.step.py | 11 +++- .../bebop/verilator/06_batch_event.step.py | 63 ++++++++++++++++--- bbdev/bbdev | 4 +- bbdev/mcp/tools/bebop_verilator_batch.py | 2 + bbdev/mcp/tools/bebop_verilator_build.py | 10 ++- 7 files changed, 84 insertions(+), 16 deletions(-) diff --git a/bbdev/api/steps/bebop/regression.py b/bbdev/api/steps/bebop/regression.py index 9fa8fa101..f31dc7632 100644 --- a/bbdev/api/steps/bebop/regression.py +++ b/bbdev/api/steps/bebop/regression.py @@ -26,6 +26,7 @@ def regression_workload_toml( test_type: str, bbdir: str | None = None, rushB: bool = False, + diff: bool = False, ) -> str: if test_type == "elf-tests": suffix = "elf" @@ -34,8 +35,14 @@ def regression_workload_toml( else: raise ValueError(f"invalid test type: {test_type}") + regression_dir = chip_regression_dir(chip, backend, bbdir) + if diff: + diff_toml = regression_dir / f"workloads-{suffix}-diff.toml" + if diff_toml.is_file(): + return str(diff_toml) + name = f"workloads-{suffix}-rushB.toml" if rushB else f"workloads-{suffix}.toml" - toml = chip_regression_dir(chip, backend, bbdir) / name + toml = regression_dir / name if not toml.is_file(): raise ValueError(f"regression workload toml does not exist: {toml}") return str(toml) diff --git a/bbdev/api/steps/bebop/verilator/03_build_api.step.py b/bbdev/api/steps/bebop/verilator/03_build_api.step.py index 7a27f4c5f..a6b1d3f57 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_api.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_api.step.py @@ -32,6 +32,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "config": arch_config, "jobs": body.get("jobs", 16), + "diff": bool(body.get("diff", False)), "vsrc_dir": get_verilator_build_dir( bbdir, arch_config, diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py index 2ae5dbdd0..439e0e8d3 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -42,6 +42,14 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body={"error": f"Invalid test type: {test_type}. Must be 'elf-tests' or 'pk-tests'"} ) + diff = bool(body.get("diff", False)) + rushB = bool(body.get("rushB", False)) + if diff and rushB: + return ApiResponse( + status=400, + body={"error": "--diff and --rushB cannot be used together"}, + ) + vsrc_dir = get_verilator_build_dir(bbdir, arch_config, body.get("vsrc_dir")) data = { @@ -50,7 +58,8 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "vsrc_dir": vsrc_dir, "test": test_type, "clean-before": body.get("clean-before", body.get("clean_before", False)), - "rushB": bool(body.get("rushB", False)), + "rushB": rushB, + "diff": diff, } await ctx.enqueue({"topic": "bebop.verilator.batch", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 4150063aa..40608bbf3 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -60,9 +60,20 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: test_type = input_data.get("test", "elf-tests") rushB = bool(input_data.get("rushB", False)) + diff = bool(input_data.get("diff", False)) + if diff and rushB: + ctx.logger.error("--diff and --rushB cannot be used together") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "diff_conflicts_with_rushB"}, + trace_id=origin_tid, + ) + return try: workload_toml = regression_workload_toml( - chip, "verilator", test_type, bbdir, rushB=rushB + chip, "verilator", test_type, bbdir, rushB=rushB, diff=diff ) except ValueError as e: ctx.logger.error(str(e)) @@ -74,7 +85,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return ctx.logger.info( - f"Running {test_type} with workload config: {workload_toml} rushB={rushB}" + f"Running {test_type} with workload config: {workload_toml} " + f"rushB={rushB} diff={diff}" ) vsrc_dir = get_verilator_build_dir(bbdir, arch_config, input_data.get("vsrc_dir")) @@ -88,19 +100,51 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return vsrc_config = shlex.quote(f"env.VSRC_PATH='{vsrc_dir}'") - if input_data.get("clean-before", input_data.get("clean_before", False)): - shutil.rmtree(f"{bebop_dir}/test-artifacts", ignore_errors=True) - ctx.logger.info("Cleaned previous bebop test artifacts") - env = os.environ.copy() env.update({ "BEBOP_WORKLOAD_TOML": workload_toml, "BEBOP_BB_TESTS_ROOT": elf_root, + "BEBOP_ARCH_CONFIG": arch_config, "VSRC_PATH": vsrc_dir, }) + + manifest = f"{bebop_dir}/Cargo.toml" + features = "verilator" + if diff: + manifest = f"{bbdir}/examples/chips/{chip}/emu/Cargo.toml" + if not os.path.isfile(manifest): + ctx.logger.error(f"Chip emulator manifest does not exist: {manifest}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "chip_manifest_not_found", "manifest": manifest}, + trace_id=origin_tid, + ) + return + features = "verilator,bemu,difftest" + preload = os.pathsep.join( + path + for path in ( + f"{bbdir}/result/lib/libdramsim3.so", + os.environ.get("LD_PRELOAD", ""), + ) + if path + ) + env.update({ + "BEBOP_VERILATOR_DIFF": "1", + "BEBOP_DIFF_LD_PRELOAD": preload, + "BEBOP_DIFF_RUN_DIR": os.path.dirname(manifest), + }) + + if input_data.get("clean-before", input_data.get("clean_before", False)): + artifact_dir = os.path.join(os.path.dirname(manifest), "test-artifacts") + shutil.rmtree(artifact_dir, ignore_errors=True) + ctx.logger.info(f"Cleaned previous bebop test artifacts: {artifact_dir}") + nextest_cmd = ( - f"nix develop -c cargo nextest run --release --manifest-path {shlex.quote(f'{bebop_dir}/Cargo.toml')} " - "--features verilator --test test_verilator " + f"nix develop -c cargo nextest run --release --manifest-path {shlex.quote(manifest)} " + f"--features {shlex.quote(features)} --test test_verilator " f"--config-file {shlex.quote(nextest_config)} " f"--config={vsrc_config}" ) @@ -122,12 +166,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "batch", - "backend": "verilator-rushB" if rushB else "verilator", + "backend": "verilator-rushB" if rushB else ("difftest" if diff else "verilator"), "chip": chip, "config": arch_config, "vsrc_dir": vsrc_dir, "test_type": test_type, "rushB": rushB, + "diff": diff, "nextest_config": nextest_config, "workload_toml": workload_toml, }, diff --git a/bbdev/bbdev b/bbdev/bbdev index 00ef16143..1cb9c481d 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -65,10 +65,10 @@ WORKFLOW_COMMANDS = { "bebop-verilator": { "clean": 'Clean Bebop Verilator build directory. Args: "--config [--output-dir ]"', "verilog": 'Generate Bebop Verilog files. Args: "--config [--output-dir ]"', - "build": 'Build Bebop Verilator simulation executable. Args: "--config [--jobs ] [--output-dir ]"', + "build": 'Build Bebop Verilator simulation executable. Args: "--config [--diff] [--jobs ] [--output-dir ]"', "sim": 'Run Bebop Verilator simulation. Args: "--config --binary [--rushB] [--batch] [--log-dir ] [--fst-dir ] [--no-wave]"', "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--rushB] [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', - "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--rushB] [--clean-before]"', + "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--diff] [--rushB] [--clean-before]"', }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--config [--output-dir ]"', diff --git a/bbdev/mcp/tools/bebop_verilator_batch.py b/bbdev/mcp/tools/bebop_verilator_batch.py index 584cb11a6..d7ca136d0 100644 --- a/bbdev/mcp/tools/bebop_verilator_batch.py +++ b/bbdev/mcp/tools/bebop_verilator_batch.py @@ -14,6 +14,7 @@ def bbdev_bebop_verilator_batch( test: str, clean_before: bool = False, rushB: bool = False, + diff: bool = False, ) -> str: """Batch bebop-verilator regression. POST /bebop/verilator/batch.""" for n, v in (("chip", chip), ("config", config)): @@ -30,6 +31,7 @@ def bbdev_bebop_verilator_batch( "test": test, "clean-before": clean_before, "rushB": rushB, + "diff": diff, }, ) ) diff --git a/bbdev/mcp/tools/bebop_verilator_build.py b/bbdev/mcp/tools/bebop_verilator_build.py index b9d745bdd..4d6a1e478 100644 --- a/bbdev/mcp/tools/bebop_verilator_build.py +++ b/bbdev/mcp/tools/bebop_verilator_build.py @@ -8,11 +8,15 @@ def register(mcp): @mcp.tool() - def bbdev_bebop_verilator_build(config: str, jobs: int = 16) -> str: + def bbdev_bebop_verilator_build( + config: str, jobs: int = 16, diff: bool = False + ) -> str: """Build bebop verilator binary. POST /bebop/verilator/build.""" if e := need("config", config): return err(e) return fmt( - submit("/bebop/verilator/build", {"config": config, "jobs": jobs}) + submit( + "/bebop/verilator/build", + {"config": config, "jobs": jobs, "diff": diff}, + ) ) - From 21a81657026c9057115e76e6cda4c9297ae8e3ce Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 12 Aug 2026 18:47:06 +0800 Subject: [PATCH 119/196] feat: implement DC synthesis and power analysis workflows with API integration --- bbdev/api/steps/dc/01_area_api.step.py | 36 ++ bbdev/api/steps/dc/02_verilog_api.step.py | 3 +- bbdev/api/steps/dc/03_area_event.step.py | 135 ++++++ bbdev/api/steps/dc/04_power_api.step.py | 37 ++ bbdev/api/steps/dc/05_sim_event.step.py | 153 +++++++ bbdev/api/steps/dc/06_ptpx_event.step.py | 117 +++++ bbdev/api/steps/dc/power.py | 39 ++ bbdev/api/steps/dc/scripts/dc.tcl | 56 +++ .../steps/dc/scripts/emit_seq_mem_models.py | 119 +++++ bbdev/api/steps/dc/scripts/gate_dpi.cc | 42 ++ bbdev/api/steps/dc/scripts/ptpx.tcl | 31 ++ bbdev/api/steps/dc/tapeout.py | 179 ++++++++ bbdev/api/steps/ip-replace/01_run_api.step.py | 2 +- .../api/steps/ip-replace/01_run_event.step.py | 35 +- .../steps/ip-replace/scripts/sram_replace.py | 422 ++++-------------- .../steps/mill/03_yosys_verilog_event.step.py | 7 +- .../steps/mill/04_dc_verilog_event.step.py | 27 +- bbdev/api/{utils => steps/mill}/verilog.py | 0 bbdev/api/steps/vcs/01_clean_api.step.py | 31 ++ bbdev/api/steps/vcs/01_clean_event.step.py | 31 ++ bbdev/api/steps/vcs/02_verilog_api.step.py | 32 ++ bbdev/api/steps/vcs/02_verilog_event.step.py | 52 +++ bbdev/api/steps/vcs/03_build_api.step.py | 34 ++ bbdev/api/steps/vcs/03_build_event.step.py | 218 +++++++++ bbdev/api/steps/vcs/04_sim_api.step.py | 22 + bbdev/api/steps/vcs/04_sim_event.step.py | 96 ++++ bbdev/api/steps/vcs/05_run_api.step.py | 40 ++ bbdev/api/steps/vcs/README.md | 18 + bbdev/api/steps/yosys/01_run_api.step.py | 2 +- bbdev/api/steps/yosys/02_verilog_api.step.py | 2 +- bbdev/api/steps/yosys/03_synth_api.step.py | 2 +- bbdev/api/steps/yosys/03_synth_event.step.py | 32 +- .../api/steps/yosys/scripts/yosys-config.yaml | 9 +- bbdev/api/tests/test_api_dc_verilog.py | 13 + bbdev/api/utils/path.py | 76 +++- bbdev/bbdev | 49 +- bbdev/mcp/tools/__init__.py | 4 + bbdev/mcp/tools/dc_area.py | 16 + bbdev/mcp/tools/dc_power.py | 46 ++ bbdev/mcp/tools/dc_verilog.py | 12 +- 40 files changed, 1872 insertions(+), 405 deletions(-) create mode 100644 bbdev/api/steps/dc/01_area_api.step.py create mode 100644 bbdev/api/steps/dc/03_area_event.step.py create mode 100644 bbdev/api/steps/dc/04_power_api.step.py create mode 100644 bbdev/api/steps/dc/05_sim_event.step.py create mode 100644 bbdev/api/steps/dc/06_ptpx_event.step.py create mode 100644 bbdev/api/steps/dc/power.py create mode 100755 bbdev/api/steps/dc/scripts/dc.tcl create mode 100644 bbdev/api/steps/dc/scripts/emit_seq_mem_models.py create mode 100644 bbdev/api/steps/dc/scripts/gate_dpi.cc create mode 100755 bbdev/api/steps/dc/scripts/ptpx.tcl create mode 100644 bbdev/api/steps/dc/tapeout.py rename bbdev/api/{utils => steps/mill}/verilog.py (100%) create mode 100644 bbdev/api/steps/vcs/01_clean_api.step.py create mode 100644 bbdev/api/steps/vcs/01_clean_event.step.py create mode 100644 bbdev/api/steps/vcs/02_verilog_api.step.py create mode 100644 bbdev/api/steps/vcs/02_verilog_event.step.py create mode 100644 bbdev/api/steps/vcs/03_build_api.step.py create mode 100644 bbdev/api/steps/vcs/03_build_event.step.py create mode 100644 bbdev/api/steps/vcs/04_sim_api.step.py create mode 100644 bbdev/api/steps/vcs/04_sim_event.step.py create mode 100644 bbdev/api/steps/vcs/05_run_api.step.py create mode 100644 bbdev/api/steps/vcs/README.md create mode 100644 bbdev/mcp/tools/dc_area.py create mode 100644 bbdev/mcp/tools/dc_power.py diff --git a/bbdev/api/steps/dc/01_area_api.step.py b/bbdev/api/steps/dc/01_area_api.step.py new file mode 100644 index 000000000..e6c5aa7f6 --- /dev/null +++ b/bbdev/api/steps/dc/01_area_api.step.py @@ -0,0 +1,36 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import check_dc_rtl_args, get_buckyball_path, get_dc_analysis_dir, get_dc_rtl_dir + +config = { + "name": "dc-area-api", + "description": "run DC synthesis and area reports", + "flows": ["dc"], + "triggers": [api("POST", "/dc/area")], + "enqueues": ["dc.verilog"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = req.body or {} + try: + check_dc_rtl_args(body) + rtl_dir = get_dc_rtl_dir(bbdir, body.get("config")) + analysis_dir = get_dc_analysis_dir(bbdir, body.get("config"), "area") + except ValueError as exc: + return ApiResponse(status=400, body={"error": str(exc)}) + await ctx.enqueue( + { + "topic": "dc.verilog", + "data": { + "output_dir": rtl_dir, + "analysis_dir": analysis_dir, + "config": body.get("config"), + "top": body.get("top") or "DigitalTop", + "from_area_workflow": True, + "_trace_id": ctx.trace_id, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/dc/02_verilog_api.step.py b/bbdev/api/steps/dc/02_verilog_api.step.py index 970565a01..a4bf4df08 100644 --- a/bbdev/api/steps/dc/02_verilog_api.step.py +++ b/bbdev/api/steps/dc/02_verilog_api.step.py @@ -16,13 +16,14 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} try: check_dc_rtl_args(body) - output_dir = get_dc_rtl_dir(bbdir, body.get("config"), body.get("dir")) + output_dir = get_dc_rtl_dir(bbdir, body.get("config")) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) data = { "output_dir": output_dir, "config": body.get("config"), + "top": body.get("top") or "DigitalTop", } await ctx.enqueue({"topic": "dc.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/dc/03_area_event.step.py b/bbdev/api/steps/dc/03_area_event.step.py new file mode 100644 index 000000000..1b75a3d2f --- /dev/null +++ b/bbdev/api/steps/dc/03_area_event.step.py @@ -0,0 +1,135 @@ +import os +import shlex +import shutil +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) +step_path = os.path.dirname(__file__) +if step_path not in sys.path: + sys.path.insert(0, step_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger +from tapeout import get_tapeout_contract, technology_settings, write_run_tcl + +config = { + "name": "dc-area", + "description": "run Synopsys Design Compiler and area reports on prepared DC RTL", + "flows": ["dc"], + "triggers": [queue("dc.area")], + "enqueues": ["dc.sim"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + source_list_path = input_data.get("source_list") + top_module = input_data.get("top") or "DigitalTop" + analysis_dir = input_data.get("analysis_dir") + if not isinstance(source_list_path, str) or not os.path.isfile(source_list_path): + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "dc", "error": "missing prepared DC source list"}, + trace_id=origin_tid, + ) + return + if not isinstance(analysis_dir, str) or not analysis_dir: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "dc", "error": "missing fixed DC analysis output directory"}, + trace_id=origin_tid, + ) + return + if shutil.which("dc_shell") is None: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "dc", + "error": "dc_shell is not on PATH; source the DC host environment before running bbdev dc --area", + }, + trace_id=origin_tid, + ) + return + + try: + contract = get_tapeout_contract(get_buckyball_path(), input_data.get("config"), top_module) + except (OSError, ValueError) as exc: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "dc", "error": str(exc)}, + trace_id=origin_tid, + ) + return + + output_dir = os.path.join(analysis_dir, "outputs") + report_dir = os.path.join(analysis_dir, "reports") + os.makedirs(output_dir, exist_ok=True) + os.makedirs(report_dir, exist_ok=True) + try: + tech = technology_settings() + run_config = write_run_tcl( + os.path.join(analysis_dir, "run.tcl"), + { + "top": top_module, + "source_list": source_list_path, + "output_dir": output_dir, + "report_dir": report_dir, + "clock_port": contract.clock_port, + "clock_period_ns": contract.clock_period_ns, + **tech, + }, + ) + except (OSError, ValueError) as exc: + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "dc", "error": str(exc)}, trace_id=origin_tid) + return + script_path = contract.dc_script + dc_log = os.path.join(analysis_dir, "dc_shell.log") + ctx.logger.info(f"Running chip-owned DC synthesis for {contract.chip}, top {top_module}: {script_path}") + result = stream_run_logger( + cmd=( + f"dc_shell -f {shlex.quote(str(script_path))} " + f"-x {shlex.quote('set RUN_CONFIG ' + '{' + str(run_config) + '}')} " + f"> {shlex.quote(dc_log)} 2>&1" + ), + logger=ctx.logger, + cwd=os.path.dirname(script_path), + stdout_prefix="dc", + stderr_prefix="dc", + ) + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={ + "task": "area", + "top_module": top_module, + "dc_script": str(script_path), + "dc_output_dir": output_dir, + "dc_report_dir": report_dir, + "dc_log": dc_log, + "run_config": str(run_config), + "chip": contract.chip, + "tapeout_dir": str(contract.root), + "sram_manifest": (input_data.get("sram_collateral") or {}).get("sram_manifest"), + }, + trace_id=origin_tid, + ) + if result.returncode == 0 and input_data.get("from_power_workflow"): + await ctx.enqueue({"topic": "dc.sim", "data": {**input_data, "area_dir": output_dir, "chip": contract.chip, "_trace_id": origin_tid}}) diff --git a/bbdev/api/steps/dc/04_power_api.step.py b/bbdev/api/steps/dc/04_power_api.step.py new file mode 100644 index 000000000..4c46baf39 --- /dev/null +++ b/bbdev/api/steps/dc/04_power_api.step.py @@ -0,0 +1,37 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import check_dc_power_args, get_buckyball_path, get_dc_analysis_dir, get_dc_rtl_dir + +config = { + "name": "dc-power-api", + "description": "run DC synthesis and PrimeTime PX power analysis", + "flows": ["dc"], + "triggers": [api("POST", "/dc/power")], + "enqueues": ["dc.verilog"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + bbdir = get_buckyball_path() + body = req.body or {} + try: + check_dc_power_args(body) + rtl_dir = get_dc_rtl_dir(bbdir, body.get("config")) + analysis_dir = get_dc_analysis_dir(bbdir, body.get("config"), "power") + except ValueError as exc: + return ApiResponse(status=400, body={"error": str(exc)}) + await ctx.enqueue( + { + "topic": "dc.verilog", + "data": { + **body, + "output_dir": rtl_dir, + "analysis_dir": analysis_dir, + "config": body.get("config"), + "top": body.get("top") or "DigitalTop", + "from_power_workflow": True, + "_trace_id": ctx.trace_id, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/dc/05_sim_event.step.py b/bbdev/api/steps/dc/05_sim_event.step.py new file mode 100644 index 000000000..98c0ace8e --- /dev/null +++ b/bbdev/api/steps/dc/05_sim_event.step.py @@ -0,0 +1,153 @@ +import os +import shlex +import shutil +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +step_path = os.path.dirname(__file__) +if step_path not in sys.path: + sys.path.insert(0, step_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path +from utils.search_workload import search_workload +from utils.stream_run import stream_run_logger +from tapeout import get_tapeout_contract, resolve_power_window, write_run_env + +config = { + "name": "dc-sim", + "description": "rerun chip-owned simulation and produce activity for PTPX", + "flows": ["dc"], + "triggers": [queue("dc.sim")], + "enqueues": ["dc.power"], +} + + +def _first(data: dict, *names: str): + for name in names: + value = data.get(name) + if value is not None and value != "": + return value + return None + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + config_name = input_data.get("config") + top_module = input_data.get("top") or "DigitalTop" + analysis_dir = input_data.get("analysis_dir") + if not isinstance(analysis_dir, str) or not analysis_dir: + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": "missing analysis directory"}, trace_id=origin_tid) + return + + try: + contract = get_tapeout_contract(get_buckyball_path(), config_name, top_module) + except (OSError, ValueError) as exc: + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": str(exc)}, trace_id=origin_tid) + return + + activity_format = str(_first(input_data, "format") or contract.power_format).lower() + if activity_format not in {"saif", "vcd", "fsdb"}: + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": f"unsupported activity format: {activity_format}"}, trace_id=origin_tid) + return + + # An explicit activity file is retained as a debugging escape hatch. Normal + # power runs leave it unset and always regenerate activity below. + explicit_activity = _first(input_data, "activity") + if explicit_activity: + if not os.path.isfile(explicit_activity): + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": f"activity file does not exist: {explicit_activity}"}, trace_id=origin_tid) + return + await ctx.enqueue({"topic": "dc.power", "data": {**input_data, "activity": explicit_activity, "format": activity_format, "_trace_id": origin_tid}}) + return + + if shutil.which("bash") is None: + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": "bash is required for chip power_sim.sh"}, trace_id=origin_tid) + return + + activity_dir = os.path.join(analysis_dir, "activity") + os.makedirs(activity_dir, exist_ok=True) + workload_name = _first(input_data, "workload") or contract.power_workload + if not workload_name: + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": "missing workload name; pass --workload "}, trace_id=origin_tid) + return + if os.path.isabs(str(workload_name)): + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": "workload must be a built ELF name, not an absolute path"}, trace_id=origin_tid) + return + workload = None + for root in ("output/workloads/src", "build/workloads/src"): + workload = search_workload(os.path.join(get_buckyball_path(), "bb-tests", root), str(workload_name)) + if workload: + break + if not workload: + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": f"workload not found in bb-tests output/build directories: {workload_name}"}, trace_id=origin_tid) + return + try: + start_ns, end_ns = resolve_power_window( + contract, + _first(input_data, "start_ns", "start-ns"), + _first(input_data, "end_ns", "end-ns"), + ) + except ValueError as exc: + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": str(exc)}, trace_id=origin_tid) + return + netlist = os.path.join(input_data.get("area_dir") or "", f"{top_module}.v") + sdc = os.path.join(input_data.get("area_dir") or "", f"{top_module}.sdc") + activity_path = os.path.join(activity_dir, f"activity.{activity_format}") + run_env = write_run_env( + os.path.join(activity_dir, "run.env"), + { + "OUTPUT_DIR": activity_dir, + "ACTIVITY_FILE": activity_path, + "ACTIVITY_FORMAT": activity_format, + "TOP": top_module, + "CONFIG": str(config_name or ""), + "NETLIST": netlist, + "SDC": sdc, + "WORKLOAD": workload, + "START_NS": start_ns, + "END_NS": end_ns, + "ACTIVITY_STRIP_PATH": contract.power_strip_path, + }, + ) + + ctx.logger.info( + f"Running chip-owned power simulation for {contract.chip}; " + f"format={activity_format}, workload={workload or ''}, " + f"window={start_ns or ''}..{end_ns or ''} ns" + ) + result = stream_run_logger( + cmd=f"bash {shlex.quote(str(contract.power_sim_script))} {shlex.quote(str(run_env))}", + logger=ctx.logger, + cwd=str(contract.root), + stdout_prefix="power sim", + stderr_prefix="power sim", + ) + if result.returncode == 0 and not os.path.isfile(activity_path): + result.returncode = 1 + error = f"chip power simulation completed without producing {activity_path}" + else: + error = None + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={ + "task": "sim", + "chip": contract.chip, + "power_sim_script": str(contract.power_sim_script), + "activity": activity_path, + "format": activity_format, + "workload": workload, + "start_ns": start_ns, + "end_ns": end_ns, + **({"error": error} if error else {}), + }, + trace_id=origin_tid, + ) + if result.returncode == 0: + await ctx.enqueue({"topic": "dc.power", "data": {**input_data, "activity": activity_path, "format": activity_format, "strip_path": contract.power_strip_path, "_trace_id": origin_tid}}) diff --git a/bbdev/api/steps/dc/06_ptpx_event.step.py b/bbdev/api/steps/dc/06_ptpx_event.step.py new file mode 100644 index 000000000..a5c459922 --- /dev/null +++ b/bbdev/api/steps/dc/06_ptpx_event.step.py @@ -0,0 +1,117 @@ +import os +import shlex +import shutil +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) +step_path = os.path.dirname(__file__) +if step_path not in sys.path: + sys.path.insert(0, step_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path +from power import read_dynamic_power +from utils.stream_run import stream_run_logger +from tapeout import get_tapeout_contract, resolve_power_window, technology_settings, write_run_tcl + +config = { + "name": "dc-ptpx", + "description": "run PrimeTime PX power analysis on DC output", + "flows": ["dc"], + "triggers": [queue("dc.power")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + top_module = input_data.get("top") or "DigitalTop" + activity_path = input_data.get("activity") + activity_format = input_data.get("format") + analysis_dir = input_data.get("analysis_dir") + if not isinstance(analysis_dir, str) or not analysis_dir: + error = "missing fixed DC analysis output directory" + elif not isinstance(activity_path, str) or not activity_path: + error = "missing required --activity" + elif not isinstance(activity_format, str) or activity_format not in {"saif", "vcd", "fsdb"}: + error = "missing or invalid --format; expected saif, vcd, or fsdb" + elif shutil.which("pt_shell") is None: + error = "pt_shell is not on PATH; source the DC host environment before running bbdev dc --power" + else: + error = None + if error: + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "power", "error": error}, trace_id=origin_tid) + return + + try: + netlist = os.path.join(analysis_dir, "outputs", f"{top_module}.v") + sdc = os.path.join(analysis_dir, "outputs", f"{top_module}.sdc") + contract = get_tapeout_contract(get_buckyball_path(), input_data.get("config"), top_module) + start_ns, end_ns = resolve_power_window( + contract, + input_data.get("start_ns", input_data.get("start-ns")), + input_data.get("end_ns", input_data.get("end-ns")), + ) + tech = technology_settings() + run_config = write_run_tcl( + os.path.join(analysis_dir, "power-run.tcl"), + { + "top": top_module, + "netlist": netlist, + "sdc": sdc, + "activity": activity_path, + "activity_format": activity_format, + "report_dir": os.path.join(analysis_dir, "power-reports"), + "strip_path": input_data.get("strip_path") or "", + "start_ns": start_ns or "", + "end_ns": end_ns or "", + **tech, + }, + ) + script = contract.power_script + except (OSError, ValueError) as exc: + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "power", "error": str(exc)}, trace_id=origin_tid) + return + + result = stream_run_logger( + cmd=( + f"pt_shell -f {shlex.quote(str(script))} " + f"-x {shlex.quote('set RUN_CONFIG ' + '{' + str(run_config) + '}')} " + f"> {shlex.quote(os.path.join(analysis_dir, 'pt_shell.log'))} 2>&1" + ), + logger=ctx.logger, + cwd=os.path.dirname(script), + stdout_prefix="ptpx", + stderr_prefix="ptpx", + ) + report_dir = os.path.join(analysis_dir, "power-reports") + dynamic_power = read_dynamic_power(os.path.join(report_dir, "power_total.rpt")) + extra_fields = { + "task": "power", + "top_module": top_module, + "activity": activity_path, + "format": activity_format, + "power_script": str(script), + "run_config": str(run_config), + "pt_log": os.path.join(analysis_dir, "pt_shell.log"), + "power_report_dir": report_dir, + "power_scope": "dynamic (internal + switching)", + "start_ns": start_ns, + "end_ns": end_ns, + } + if dynamic_power: + extra_fields["dynamic_power"] = dynamic_power + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields=extra_fields, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/dc/power.py b/bbdev/api/steps/dc/power.py new file mode 100644 index 000000000..48849237b --- /dev/null +++ b/bbdev/api/steps/dc/power.py @@ -0,0 +1,39 @@ +"""Small report helpers for dynamic-power-only summaries.""" + +from __future__ import annotations + +import re +from pathlib import Path + + +_POWER_VALUE = re.compile(r"([-+]?\d+(?:\.\d*)?(?:[eE][-+]?\d+)?)\s*([munp]?W)\b") +_UNIT_TO_WATTS = {"W": 1.0, "mW": 1e-3, "uW": 1e-6, "nW": 1e-9, "pW": 1e-12} + + +def _watts(value: str) -> float: + match = _POWER_VALUE.fullmatch(value) + if match is None: + raise ValueError(f"unsupported power value: {value}") + return float(match.group(1)) * _UNIT_TO_WATTS[match.group(2)] + + +def read_dynamic_power(report: str | Path) -> dict[str, str] | None: + path = Path(report) + if not path.is_file(): + return None + values: dict[str, str] = {} + patterns = { + "internal_power": r"Cell Internal Power\s*=\s*([-+]?\d+(?:\.\d*)?(?:[eE][-+]?\d+)?\s*[munp]?W)\b", + "switching_power": r"Net Switching Power\s*=\s*([-+]?\d+(?:\.\d*)?(?:[eE][-+]?\d+)?\s*[munp]?W)\b", + "total_power": r"Total Power\s*=\s*([-+]?\d+(?:\.\d*)?(?:[eE][-+]?\d+)?\s*[munp]?W)\b", + } + text = path.read_text(encoding="utf-8", errors="ignore") + for key, pattern in patterns.items(): + match = re.search(pattern, text) + if match: + values[key] = match.group(1).strip() + if "internal_power" not in values or "switching_power" not in values: + return None + dynamic_watts = _watts(values["internal_power"]) + _watts(values["switching_power"]) + values["dynamic_power"] = f"{dynamic_watts:.12g} W" + return values diff --git a/bbdev/api/steps/dc/scripts/dc.tcl b/bbdev/api/steps/dc/scripts/dc.tcl new file mode 100755 index 000000000..b525d3845 --- /dev/null +++ b/bbdev/api/steps/dc/scripts/dc.tcl @@ -0,0 +1,56 @@ +# Shared Design Compiler implementation. Chip tapeout/dc.tcl files are thin +# ownership wrappers; all RUN_* values are supplied by bbdev. +if {![info exists RUN_CONFIG] || $RUN_CONFIG eq ""} { + error "bbdev must pass -x {set RUN_CONFIG }" +} +source [file normalize $RUN_CONFIG] + +proc bbdev_read_filelist {path} { + set fh [open $path r] + set files [list] + while {[gets $fh line] >= 0} { + set line [string trim $line] + if {$line eq "" || [string match "#*" $line]} { continue } + lappend files [file normalize $line] + } + close $fh + if {[llength $files] == 0} { error "empty DC source list: $path" } + return $files +} + +set top $RUN_TOP +set output_dir [file normalize $RUN_OUTPUT_DIR] +set report_dir [file normalize $RUN_REPORT_DIR] +file mkdir $output_dir $report_dir [file join $report_dir work] +set target_library [list $RUN_TARGET_LIBRARY] +set synthetic_library $RUN_SYNTHETIC_LIBRARY +set link_library [concat [list *] $target_library $synthetic_library $RUN_LINK_LIBRARY] +set_host_options -max_cores $RUN_MAX_CORES +define_design_lib WORK -path [file join $report_dir work] +set search_path [list .] +set_app_var verilogout_no_tri true +set_app_var verilogout_equation false +analyze -format sverilog -define {SYNTHESIS DC_SYN} [bbdev_read_filelist $RUN_SOURCE_LIST] +elaborate $top +current_design $top +link +set bb_clock [get_ports -quiet $RUN_CLOCK_PORT] +if {[sizeof_collection $bb_clock] != 1} { error "clock port '$RUN_CLOCK_PORT' was not found exactly once" } +create_clock -name bb_clock -period $RUN_CLOCK_PERIOD_NS $bb_clock +set_clock_uncertainty [expr {$RUN_CLOCK_PERIOD_NS * 0.30}] [get_clocks bb_clock] +set_clock_transition [expr {$RUN_CLOCK_PERIOD_NS * 0.10}] [get_clocks bb_clock] +set_input_delay [expr {$RUN_CLOCK_PERIOD_NS * 0.70}] -clock bb_clock [remove_from_collection [all_inputs] $bb_clock] +set_output_delay [expr {$RUN_CLOCK_PERIOD_NS * 0.70}] -clock bb_clock [all_outputs] +set_load 2.0 [all_outputs] +compile_ultra -area_high_effort_script -no_autoungroup -no_boundary_optimization +set_fix_multiple_port_nets -all -buffer_constants +change_names -hierarchy -rules verilog +write -format ddc -hierarchy -output [file join $output_dir ${top}.ddc] +write -format verilog -hierarchy -output [file join $output_dir ${top}.v] +write_sdc [file join $output_dir ${top}.sdc] +report_constraint -all_violators > [file join $report_dir constraint.rpt] +report_timing -delay max -max_paths 50 > [file join $report_dir timing_max.rpt] +report_timing -delay min -max_paths 50 > [file join $report_dir timing_min.rpt] +report_area -hierarchy > [file join $report_dir area.rpt] +report_reference > [file join $report_dir reference.rpt] +exit diff --git a/bbdev/api/steps/dc/scripts/emit_seq_mem_models.py b/bbdev/api/steps/dc/scripts/emit_seq_mem_models.py new file mode 100644 index 000000000..4dc110cba --- /dev/null +++ b/bbdev/api/steps/dc/scripts/emit_seq_mem_models.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Emit behavioral models for FIRRTL externalized sequential memories. + +These models are used only by gate-level activity simulation when a chip has +not yet supplied PDK SRAM macro simulation collateral. They retain the +external-memory interface from ``mems.conf`` so the DC netlist is simulated +with functional memories rather than X-producing black boxes. +""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + + +LINE = re.compile( + r"name\s+(?P\S+)\s+depth\s+(?P\d+)\s+width\s+(?P\d+)" + r"\s+ports\s+(?P\S+)(?:\s+mask_gran\s+(?P\d+))?$" +) + + +def write_masked(lines: list[str], *, addr: str, data: str, mask: str, width: int, gran: int) -> None: + lines.extend( + [ + f" for (int i = 0; i < {width // gran}; i++) begin", + f" if ({mask}[i]) memory[{addr}][i*{gran} +: {gran}] <= {data}[i*{gran} +: {gran}];", + " end", + ] + ) + + +def emit(name: str, depth: int, width: int, ports: str, mask_gran: int | None) -> str: + lines = [f"module {name}("] + if ports == "mrw": + lines += [ + " input logic [$clog2(" + str(depth) + ")-1:0] RW0_addr,", + " input logic RW0_en, RW0_clk, RW0_wmode,", + f" input logic [{width - 1}:0] RW0_wdata,", + f" output logic [{width - 1}:0] RW0_rdata,", + f" input logic [{width // mask_gran - 1}:0] RW0_wmask", + ");", + f" logic [{width - 1}:0] memory [0:{depth - 1}];", + " always @(posedge RW0_clk) begin", + " if (RW0_en) begin", + " if (RW0_wmode) begin", + ] + write_masked(lines, addr="RW0_addr", data="RW0_wdata", mask="RW0_wmask", width=width, gran=mask_gran or width) + lines += [" end", " RW0_rdata <= memory[RW0_addr];", " end", " end"] + elif ports == "rw": + lines += [ + " input logic [$clog2(" + str(depth) + ")-1:0] RW0_addr,", + " input logic RW0_en, RW0_clk, RW0_wmode,", + f" input logic [{width - 1}:0] RW0_wdata,", + f" output logic [{width - 1}:0] RW0_rdata", + ");", + f" logic [{width - 1}:0] memory [0:{depth - 1}];", + " always @(posedge RW0_clk) begin", + " if (RW0_en) begin", + " if (RW0_wmode) memory[RW0_addr] <= RW0_wdata;", + " RW0_rdata <= memory[RW0_addr];", + " end", + " end", + ] + elif ports in {"mwrite,read", "write,read"}: + has_mask = ports.startswith("mwrite") + lines += [ + " input logic [$clog2(" + str(depth) + ")-1:0] R0_addr,", + " input logic R0_en, R0_clk,", + f" output logic [{width - 1}:0] R0_data,", + " input logic [$clog2(" + str(depth) + ")-1:0] W0_addr,", + " input logic W0_en, W0_clk,", + f" input logic [{width - 1}:0] W0_data" + ("," if has_mask else ""), + ] + if has_mask: + lines.append(f" input logic [{width // mask_gran - 1}:0] W0_mask") + lines += [");", f" logic [{width - 1}:0] memory [0:{depth - 1}];", " always @(posedge R0_clk) if (R0_en) R0_data <= memory[R0_addr];", " always @(posedge W0_clk) if (W0_en) begin"] + if has_mask: + write_masked(lines, addr="W0_addr", data="W0_data", mask="W0_mask", width=width, gran=mask_gran or width) + else: + lines.append(" memory[W0_addr] <= W0_data;") + lines += [" end"] + else: + raise ValueError(f"unsupported memory port topology for {name}: {ports}") + lines += ["endmodule", ""] + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--mem-conf", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + models = [] + for raw in args.mem_conf.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line: + continue + match = LINE.fullmatch(line) + if match is None: + raise ValueError(f"invalid mems.conf entry: {line}") + values = match.groupdict() + models.append( + emit( + values["name"], + int(values["depth"]), + int(values["width"]), + values["ports"], + int(values["mask"]) if values["mask"] else None, + ) + ) + if not models: + raise ValueError(f"no memories found in {args.mem_conf}") + args.output.write_text("`timescale 1ns/1ps\n\n" + "\n".join(models), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bbdev/api/steps/dc/scripts/gate_dpi.cc b/bbdev/api/steps/dc/scripts/gate_dpi.cc new file mode 100644 index 000000000..77d9f053e --- /dev/null +++ b/bbdev/api/steps/dc/scripts/gate_dpi.cc @@ -0,0 +1,42 @@ +#include +#include + +extern "C" void dpi_bdb_set_clk(unsigned long long) {} + +extern "C" void scu_uart_write(std::uint32_t, std::uint32_t ch) { + std::fputc(static_cast(ch & 0xff), stdout); + std::fflush(stdout); +} + +// Gate power runs use the requested measurement horizon, rather than ending +// when a workload's SCU writes its completion register. +extern "C" void scu_sim_exit(std::uint32_t hart, std::uint32_t code) { + std::fprintf(stderr, "[SCU] hart %u: workload exit code %u\n", hart, code); + std::fflush(stderr); +} + +extern "C" void scu_uart_rx_sample(std::uint32_t, std::uint32_t, + std::uint32_t *valid, + std::uint32_t *data) { + *valid = 0; + *data = 0; +} + +extern "C" void dpi_itrace(std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t) {} +extern "C" void dpi_mtrace(std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t) {} +extern "C" void dpi_pmctrace(std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t) {} +extern "C" void dpi_mem_pmctrace(std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t) {} +extern "C" void dpi_mtrace_issue(std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t) {} diff --git a/bbdev/api/steps/dc/scripts/ptpx.tcl b/bbdev/api/steps/dc/scripts/ptpx.tcl new file mode 100755 index 000000000..7139c27d2 --- /dev/null +++ b/bbdev/api/steps/dc/scripts/ptpx.tcl @@ -0,0 +1,31 @@ +# Shared PrimeTime PX implementation. Chip tapeout/power.tcl files are thin +# wrappers; this reports only standard-cell dynamic power. +if {![info exists RUN_CONFIG] || $RUN_CONFIG eq ""} { error "bbdev must pass -x {set RUN_CONFIG }" } +source [file normalize $RUN_CONFIG] +set report_dir [file normalize $RUN_REPORT_DIR] +file mkdir $report_dir +set target_library [list $RUN_TARGET_LIBRARY] +set link_library [concat [list *] $target_library $RUN_SYNTHETIC_LIBRARY $RUN_LINK_LIBRARY] +read_verilog [file normalize $RUN_NETLIST] +current_design $RUN_TOP +link +read_sdc [file normalize $RUN_SDC] +set opts [list] +if {$RUN_START_NS ne "" || $RUN_END_NS ne ""} { + lappend opts -time [list "${RUN_START_NS}ns" "${RUN_END_NS}ns"] +} +set format [string tolower $RUN_ACTIVITY_FORMAT] +if {$format eq "fsdb"} { + if {$RUN_STRIP_PATH ne ""} { read_fsdb {*}$opts -strip_path $RUN_STRIP_PATH [file normalize $RUN_ACTIVITY] } else { read_fsdb {*}$opts [file normalize $RUN_ACTIVITY] } +} elseif {$format eq "vcd"} { + if {$RUN_STRIP_PATH ne ""} { read_vcd {*}$opts -strip_path $RUN_STRIP_PATH [file normalize $RUN_ACTIVITY] } else { read_vcd {*}$opts [file normalize $RUN_ACTIVITY] } +} elseif {$format eq "saif"} { + read_saif {*}$opts [file normalize $RUN_ACTIVITY] +} else { error "unsupported activity format: $format" } +set power_enable_analysis true +set power_model_preference nlpm +update_power +report_power -hierarchy -levels 3 > [file join $report_dir power_hierarchy.rpt] +report_power -verbose > [file join $report_dir power_total.rpt] +report_power -hierarchy -levels 2 -sort_by total_power > [file join $report_dir power_hierarchy_sorted.rpt] +exit diff --git a/bbdev/api/steps/dc/tapeout.py b/bbdev/api/steps/dc/tapeout.py new file mode 100644 index 000000000..e202e1b98 --- /dev/null +++ b/bbdev/api/steps/dc/tapeout.py @@ -0,0 +1,179 @@ +"""Resolve chip-owned tapeout flow contracts.""" + +from __future__ import annotations + +import os +import shlex +import tomllib +from dataclasses import dataclass +from pathlib import Path + +from utils.path import get_chip_from_config + + +def _tcl_word(value: str) -> str: + return "{" + str(value).replace("\\", "\\\\").replace("}", "\\}") + "}" + + +def _tcl_list(values: list[str]) -> str: + return "[list " + " ".join(_tcl_word(value) for value in values) + "]" + + +@dataclass(frozen=True) +class TapeoutContract: + chip: str + root: Path + dc_script: Path + power_script: Path + power_sim_script: Path + top_module: str + clock_port: str + clock_period_ns: float + power_format: str + power_start_ns: str | None + power_end_ns: str | None + power_workload: str | None + power_strip_path: str + + +def resolve_power_window( + contract: TapeoutContract, start_ns: object | None, end_ns: object | None +) -> tuple[str | None, str | None]: + """Merge optional CLI window bounds with a chip default and validate them.""" + start = str(start_ns) if start_ns is not None and str(start_ns) != "" else contract.power_start_ns + end = str(end_ns) if end_ns is not None and str(end_ns) != "" else contract.power_end_ns + if (start is None) != (end is None): + raise ValueError("power start_ns and end_ns must be supplied together") + if start is not None: + try: + if float(start) < 0 or float(start) >= float(end): + raise ValueError + except ValueError as exc: + raise ValueError("power start_ns must be smaller than end_ns") from exc + return start, end + + +def _read_config(root: Path) -> dict: + path = root / "config.toml" + if not path.is_file(): + return {} + with path.open("rb") as handle: + data = tomllib.load(handle) + section = data.get("tapeout", data) + if not isinstance(section, dict): + raise ValueError(f"{path} must contain a [tapeout] table") + return section + + +def get_tapeout_contract(bbdir: str | os.PathLike[str], config: str, top: str | None = None) -> TapeoutContract: + """Resolve the tapeout scripts owned by the chip selected by a Scala config.""" + chip = get_chip_from_config(str(bbdir), config) + root = Path(bbdir) / "examples" / "chips" / chip / "tapeout" + if not root.is_dir(): + raise ValueError(f"chip {chip} has no tapeout directory: {root}") + + settings = _read_config(root) + top_module = top or settings.get("top", "DigitalTop") + clock_port = str(settings.get("clock_port", "")).strip() + if not clock_port: + raise ValueError(f"missing tapeout.clock_port in {root / 'config.toml'}") + try: + clock_period_ns = float(settings.get("clock_period_ns")) + except (TypeError, ValueError) as exc: + raise ValueError(f"invalid tapeout.clock_period_ns in {root / 'config.toml'}") from exc + if clock_period_ns <= 0: + raise ValueError(f"tapeout.clock_period_ns must be positive in {root / 'config.toml'}") + + power_format = str(settings.get("power_format", "fsdb")).lower() + if power_format not in {"saif", "vcd", "fsdb"}: + raise ValueError(f"unsupported tapeout.power_format in {root / 'config.toml'}: {power_format}") + + def optional_string(name: str) -> str | None: + value = settings.get(name) + return None if value is None or str(value).strip() == "" else str(value) + + power_start_ns = optional_string("power_start_ns") + power_end_ns = optional_string("power_end_ns") + if (power_start_ns is None) != (power_end_ns is None): + raise ValueError(f"tapeout power_start_ns and power_end_ns must be set together in {root / 'config.toml'}") + if power_start_ns is not None: + try: + if float(power_start_ns) < 0 or float(power_start_ns) >= float(power_end_ns): + raise ValueError + except ValueError as exc: + raise ValueError(f"invalid tapeout power window in {root / 'config.toml'}") from exc + + scripts = { + "dc_script": root / str(settings.get("dc_script", "dc.tcl")), + "power_script": root / str(settings.get("power_script", "power.tcl")), + "power_sim_script": root / str(settings.get("power_sim_script", "power_sim.sh")), + } + missing = [str(path) for path in scripts.values() if not path.is_file()] + if missing: + raise ValueError(f"chip {chip} tapeout contract is missing: {', '.join(missing)}") + + return TapeoutContract( + chip=chip, + root=root, + dc_script=scripts["dc_script"], + power_script=scripts["power_script"], + power_sim_script=scripts["power_sim_script"], + top_module=str(top_module), + clock_port=clock_port, + clock_period_ns=clock_period_ns, + power_format=power_format, + power_start_ns=power_start_ns, + power_end_ns=power_end_ns, + power_workload=optional_string("power_workload"), + power_strip_path=str(settings.get("power_strip_path", "")), + ) + + +def technology_settings() -> dict[str, object]: + """Read the small technology contract exported by the host setup.""" + target = os.environ.get("TARGET_LIBRARY", "").strip() + synthetic = [item for item in os.environ.get("SYNTHETIC_LIBRARY", "").split(os.pathsep) if item] + link = [item for item in os.environ.get("LINK_LIBRARY", "").split(os.pathsep) if item] + if not target: + raise ValueError("missing TARGET_LIBRARY; source sourceme_host.sh") + if not Path(target).is_file(): + raise ValueError(f"TARGET_LIBRARY does not exist: {target}") + missing = [item for item in synthetic + link if not Path(item).is_file()] + if missing: + raise ValueError("technology library does not exist: " + ", ".join(missing)) + if not synthetic: + raise ValueError("missing SYNTHETIC_LIBRARY; source sourceme_host.sh") + return { + "target_library": target, + "synthetic_library": synthetic, + "link_library": link, + "max_cores": 8, + } + + +def write_run_tcl(path: str | os.PathLike[str], values: dict[str, object]) -> Path: + """Write a Tcl variable file consumed by chip-owned DC/PT scripts.""" + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + lines = [] + for key, value in values.items(): + name = "RUN_" + key.upper() + if isinstance(value, list): + lines.append(f"set {name} {_tcl_list([str(item) for item in value])}") + elif value is None: + lines.append(f"set {name} {{}}") + elif isinstance(value, (int, float)): + lines.append(f"set {name} {value}") + else: + lines.append(f"set {name} {_tcl_word(str(value))}") + output.write_text("\n".join(lines) + "\n", encoding="utf-8") + return output + + +def write_run_env(path: str | os.PathLike[str], values: dict[str, object]) -> Path: + """Write shell assignments for a chip-owned power simulation wrapper.""" + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + lines = [f"{key}={shlex.quote(str(value))}" for key, value in values.items() if value is not None] + output.write_text("\n".join(lines) + "\n", encoding="utf-8") + return output diff --git a/bbdev/api/steps/ip-replace/01_run_api.step.py b/bbdev/api/steps/ip-replace/01_run_api.step.py index 80f2b33d7..03afddc79 100644 --- a/bbdev/api/steps/ip-replace/01_run_api.step.py +++ b/bbdev/api/steps/ip-replace/01_run_api.step.py @@ -7,7 +7,7 @@ def req_arg(body: dict, name: str): config = { "name": "ip-replace-api", - "description": "replace behavioral IP RTL with compiler macro wrappers", + "description": "prepare top-scoped synthesis RTL and SRAM metadata", "flows": ["ip-replace"], "triggers": [api("POST", "/ip/replace/run")], "enqueues": ["ip-replace.run"], diff --git a/bbdev/api/steps/ip-replace/01_run_event.step.py b/bbdev/api/steps/ip-replace/01_run_event.step.py index 418a3ac1b..09aa4c51b 100644 --- a/bbdev/api/steps/ip-replace/01_run_event.step.py +++ b/bbdev/api/steps/ip-replace/01_run_event.step.py @@ -13,15 +13,15 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from sram_replace import prepare_ip_replacement +from sram_replace import prepare_sram_collateral config = { "name": "ip-replace", - "description": "replace behavioral SRAM/IP RTL with compiler macro wrappers", + "description": "prepare top-scoped synthesis RTL and SRAM metadata", "flows": ["ip-replace", "dc", "yosys"], "triggers": [queue("ip-replace.run")], - "enqueues": ["yosys.synth"], + "enqueues": ["dc.area", "yosys.synth"], } @@ -66,18 +66,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return failure_result consumer = input_data.get("consumer", "generic") + top_module = input_data.get("top") or "DigitalTop" output_dir = input_data.get("ip_replace_output_dir") if not isinstance(output_dir, str) or not output_dir: output_dir = os.path.join(os.path.dirname(source_list_path), "ip-replace") os.makedirs(output_dir, exist_ok=True) try: - replacement = prepare_ip_replacement( - sources, - output_dir, - os.environ.get("SKY130_ROOT"), - input_data.get("top"), - ) + collateral = prepare_sram_collateral(sources, output_dir, top_module) except Exception as exc: _, failure_result = await check_result( ctx, @@ -90,25 +86,24 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: replaced_source_list = os.path.join(output_dir, f"{consumer}_sources.list") with open(replaced_source_list, "w") as handle: - for path in replacement["source_paths"]: + for path in collateral["source_paths"]: handle.write(f"{path}\n") - replacement["source_list"] = replaced_source_list + collateral["source_list"] = replaced_source_list extra = { "task": "ip-replace", "consumer": consumer, "source_list": replaced_source_list, - "ip_manifest": replacement.get("manifest"), - "ip_mapped_memories": replacement["mapped"], - "ip_macro_area": replacement["macro_area"], + "sram_manifest": collateral["sram_manifest"], + "sram_memory_count": collateral["sram_memory_count"], + "top_module": collateral["top_module"], } if input_data.get("mem_conf"): extra["mem_conf"] = input_data["mem_conf"] - if replacement["mapped"]: - ctx.logger.info( - f"Mapped {replacement['mapped']} behavioral memories to compiler macros; " - f"estimated macro area {replacement['macro_area']:.3f} um^2" - ) + ctx.logger.info( + f"Prepared {collateral['sram_memory_count']} technology-neutral SRAM modules " + f"for top {collateral['top_module']}" + ) next_topic = input_data.get("next_topic") if isinstance(next_topic, str) and next_topic: @@ -119,7 +114,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "data": { **input_data, "source_list": replaced_source_list, - "ip_replacement": replacement, + "sram_collateral": collateral, "_trace_id": origin_tid, }, } diff --git a/bbdev/api/steps/ip-replace/scripts/sram_replace.py b/bbdev/api/steps/ip-replace/scripts/sram_replace.py index 507e8d9ef..61c3e9f92 100755 --- a/bbdev/api/steps/ip-replace/scripts/sram_replace.py +++ b/bbdev/api/steps/ip-replace/scripts/sram_replace.py @@ -1,53 +1,46 @@ -"""Generate SRAM replacement collateral for behavioral memories. +"""Prepare technology-neutral SRAM collateral for synthesis. -The generated artifacts mirror the DC tapeout flow: behavioral memory modules -are replaced by same-named wrappers around compiler macros, while the original -RTL is removed from the synthesis source list. Unsupported port topologies -remain in RTL and are recorded in the manifest instead of being approximated. +The elaborator has already split sequential memories into separate SystemVerilog +modules. This module preserves those implementations, selects the RTL reachable +from a synthesis top, and records memory interfaces in a manifest. A PDK-owned +flow can consume that manifest to choose macros and replace modules later. """ from __future__ import annotations +import argparse import json import math -import os import re -import argparse -import tomllib from dataclasses import dataclass from pathlib import Path -MODULE_RE = re.compile(r"\bmodule\s+([A-Za-z_][A-Za-z0-9_$]*)\s*\((.*?)\)\s*;", re.S) +MODULE_NAME_RE = re.compile(r"^\s*module\s+([A-Za-z_][A-Za-z0-9_$]*)\b", re.M) +MODULE_HEADER_RE = re.compile( + r"\bmodule\s+([A-Za-z_][A-Za-z0-9_$]*)\s*(?:#\s*\(.*?\)\s*)?\((.*?)\)\s*;", + re.S, +) MEMORY_RE = re.compile( r"\breg\s+(?:\[\s*(\d+)\s*:\s*(\d+)\s*\]\s+)?Memory\s*\[\s*(\d+)\s*:\s*(\d+)\s*\]", re.S, ) -SIZE_RE = re.compile(r"\bSIZE\s+([0-9.]+)\s+BY\s+([0-9.]+)\s*;", re.I) RANGE_RE = re.compile(r"\[\s*(\d+)\s*:\s*(\d+)\s*\]\s*{name}\b") - - -@dataclass(frozen=True) -class Macro: - name: str - depth: int - width: int - write_size: int - area: float - liberty: Path - verilog: Path - lef: Path +INSTANCE_RE = re.compile( + r"^\s*([A-Za-z_][A-Za-z0-9_$]*)\s*(?:#\s*\(.*?\)\s*)?([A-Za-z_][A-Za-z0-9_$]*)\s*\(", + re.M | re.S, +) @dataclass(frozen=True) class Memory: name: str source: Path - header: str depth: int width: int - addr_width: int - mask_width: int | None + port_type: str + address_width: int | None + write_mask_width: int | None def width_from_range(text: str, name: str) -> int | None: @@ -61,339 +54,120 @@ def ceil_log2(value: int) -> int: return max(1, math.ceil(math.log2(value))) -def load_macros(sky130_root: Path) -> list[Macro]: - macro_root = sky130_root / "share" / "sram22_sky130_macros" - macros: list[Macro] = [] - for toml_path in sorted(macro_root.glob("*/sram22.toml")): - name = toml_path.parent.name - if name.endswith("_test"): - continue - with toml_path.open("rb") as handle: - config = tomllib.load(handle) - liberty = toml_path.parent / f"{name}_tt_025C_1v80.rc.lib" - verilog = toml_path.parent / f"{name}.v" - lef = toml_path.parent / f"{name}.lef" - if not (liberty.is_file() and verilog.is_file() and lef.is_file()): - continue - size = SIZE_RE.search(lef.read_text(errors="replace")) - if size is None: - continue - macros.append( - Macro( - name=name, - depth=int(config["num_words"]), - width=int(config["data_width"]), - write_size=int(config["write_size"]), - area=float(size.group(1)) * float(size.group(2)), - liberty=liberty, - verilog=verilog, - lef=lef, - ) - ) - if not macros: - raise RuntimeError(f"no usable SRAM22 macros found under {macro_root}") - return macros +def module_sources(source_paths: list[Path]) -> dict[str, Path]: + sources: dict[str, Path] = {} + for source in source_paths: + match = MODULE_NAME_RE.search(source.read_text(errors="replace")) + if match is not None: + sources.setdefault(match.group(1), source) + return sources -def discover_memories(source_paths: list[Path]) -> tuple[list[Memory], list[dict[str, object]]]: +def reachable_modules(source_paths: list[Path], top_module: str) -> set[str]: + sources = module_sources(source_paths) + if top_module not in sources: + raise ValueError(f"synthesis top is not defined in source list: {top_module}") + + module_text = { + module: path.read_text(errors="replace") for module, path in sources.items() + } + reachable = {top_module} + pending = [top_module] + while pending: + module = pending.pop() + for child, _instance in INSTANCE_RE.findall(module_text[module]): + if child in sources and child not in reachable: + reachable.add(child) + pending.append(child) + return reachable + + +def classify_ports(header: str) -> str: + names = set(re.findall(r"\b((?:RW|R|W)\d+)_[A-Za-z0-9_]+\b", header)) + rw_ports = {name for name in names if name.startswith("RW")} + if rw_ports: + return f"{len(rw_ports)}RW" + reads = {name for name in names if name.startswith("R")} + writes = {name for name in names if name.startswith("W")} + return f"{len(reads)}R{len(writes)}W" + + +def discover_memories(source_paths: list[Path]) -> list[Memory]: memories: list[Memory] = [] - unsupported: list[dict[str, object]] = [] for source in source_paths: text = source.read_text(errors="replace") - module_match = MODULE_RE.search(text) + module_match = MODULE_HEADER_RE.search(text) memory_match = MEMORY_RE.search(text) if module_match is None or memory_match is None: continue name, header = module_match.groups() - ports = set(re.findall(r"\b(?:RW0|R0|W0)_[A-Za-z0-9_]+\b", header)) - if not {"RW0_addr", "RW0_en", "RW0_clk", "RW0_wmode", "RW0_wdata", "RW0_rdata"}.issubset(ports): - unsupported.append( - { - "module": name, - "source": str(source), - "status": "unsupported", - "reason": "requires a multi-port SRAM macro; SRAM22 provides only 1RW macros", - } - ) - continue data_msb = int(memory_match.group(1) or 0) data_lsb = int(memory_match.group(2) or 0) depth_a = int(memory_match.group(3)) depth_b = int(memory_match.group(4)) - addr_width = width_from_range(header, "RW0_addr") - if addr_width is None: - addr_width = ceil_log2(abs(depth_a - depth_b) + 1) + depth = abs(depth_a - depth_b) + 1 memories.append( Memory( name=name, source=source, - header=header, - depth=abs(depth_a - depth_b) + 1, + depth=depth, width=abs(data_msb - data_lsb) + 1, - addr_width=addr_width, - mask_width=width_from_range(header, "RW0_wmask"), + port_type=classify_ports(header), + address_width=width_from_range(header, "RW0_addr") or ceil_log2(depth), + write_mask_width=( + width_from_range(header, "RW0_wmask") + or (1 if "RW0_wmask" in header else None) + ), ) ) - return memories, unsupported + return memories -def reachable_modules(source_paths: list[Path], top_module: str | None) -> set[str] | None: - if not top_module: - return None - modules: dict[str, str] = {} - for source in source_paths: - text = source.read_text(errors="replace") - match = MODULE_RE.search(text) - if match is not None: - modules[match.group(1)] = text - if top_module not in modules: - return {top_module} - - # Split Verilog module instantiations are line-oriented in CIRCT output. - # The pattern intentionally requires a module type and an instance name, - # excluding ``module``, ``always`` and other language constructs. - instance_re = re.compile( - r"^\s*([A-Za-z_][A-Za-z0-9_$]*)\s*(?:#\s*\(.*?\))?\s+([A-Za-z_][A-Za-z0-9_$]*)\s*\(", - re.M | re.S, - ) - reachable = {top_module} - pending = [top_module] - while pending: - module = pending.pop() - for child, _ in instance_re.findall(modules.get(module, "")): - if child in modules and child not in reachable: - reachable.add(child) - pending.append(child) - return reachable - - -def choose_macro(memory: Memory, macros: list[Macro]) -> Macro: - source_granularity = memory.width // memory.mask_width if memory.mask_width else None - candidates = [ - macro - for macro in macros - if source_granularity is None or source_granularity % macro.write_size == 0 - ] - if not candidates: - raise RuntimeError(f"no SRAM22 macro has a compatible write mask for {memory.name}") - return min( - candidates, - key=lambda macro: ( - math.ceil(memory.depth / macro.depth) - * math.ceil(memory.width / macro.width) - * macro.area, - -macro.width, - ), - ) - - -def addr_expr(signal: str, source_width: int, macro_width: int) -> str: - if source_width == macro_width: - return signal - if source_width > macro_width: - return f"{signal}[{macro_width - 1}:0]" - return f"{{{{{macro_width - source_width}{{1'b0}}}}, {signal}}}" - - -def data_expr(signal: str, lo: int, logical_width: int, physical_width: int) -> str: - if logical_width == physical_width: - return f"{signal}[{lo} +: {logical_width}]" - return f"{{{{{physical_width - logical_width}{{1'b0}}}}, {signal}[{lo} +: {logical_width}]}}" - - -def bank_select_expr(memory: Memory, macro: Macro, bank: int, banks: int) -> str: - if banks == 1: - return "1'b1" - macro_addr_width = ceil_log2(macro.depth) - bank_bits = ceil_log2(banks) - return f"(RW0_addr[{macro_addr_width + bank_bits - 1}:{macro_addr_width}] == {bank_bits}'d{bank})" - - -def emit_wrapper(memory: Memory, macro: Macro) -> tuple[str, dict[str, object]]: - macro_addr_width = ceil_log2(macro.depth) - banks = math.ceil(memory.depth / macro.depth) - bank_bits = ceil_log2(banks) if banks > 1 else 0 - slices = math.ceil(memory.width / macro.width) - lines = [f"module {memory.name}({memory.header});", ""] - lines.extend( - [ - " logic ren_d0;", - " logic wmode_d0;", - f" logic [{memory.width - 1}:0] rdata_comb;", - ] - ) - if bank_bits: - lines.append(f" logic [{bank_bits - 1}:0] bank_d0;") - lines.extend(["", " always_ff @(posedge RW0_clk) begin", " ren_d0 <= RW0_en;", " wmode_d0 <= RW0_wmode;"]) - if bank_bits: - lines.append(f" bank_d0 <= RW0_addr[{macro_addr_width + bank_bits - 1}:{macro_addr_width}];") - lines.extend([" end", ""]) - - for slice_index in range(slices): - lo = slice_index * macro.width - logical_width = min(macro.width, memory.width - lo) - lines.append(f" logic [{macro.width - 1}:0] q_{slice_index} [0:{banks - 1}];") - lines.append(f" logic [{macro.width - 1}:0] d_{slice_index};") - lines.append(f" logic [{macro.width // macro.write_size - 1}:0] wmask_{slice_index};") - lines.append(f" assign d_{slice_index} = {data_expr('RW0_wdata', lo, logical_width, macro.width)};") - for bit in range(macro.width // macro.write_size): - if memory.mask_width is None: - mask = "1'b1" - else: - source_granularity = memory.width // memory.mask_width - source_bit = (lo + bit * macro.write_size) // source_granularity - mask = f"RW0_wmask[{source_bit}]" if source_bit < memory.mask_width else "1'b0" - lines.append(f" assign wmask_{slice_index}[{bit}] = {mask};") - lines.append("") - for bank in range(banks): - select = bank_select_expr(memory, macro, bank, banks) - lines.extend( - [ - f" {macro.name} u_sram_{slice_index}_{bank} (", - " .clk(RW0_clk),", - f" .we(RW0_en & RW0_wmode & {select}),", - f" .wmask(wmask_{slice_index}),", - f" .addr({addr_expr('RW0_addr', memory.addr_width, macro_addr_width)}),", - f" .din(d_{slice_index}),", - f" .dout(q_{slice_index}[{bank}])", - " );", - ] - ) - lines.append("") - - lines.append(" always_comb begin") - lines.append(" rdata_comb = 'x;") - for slice_index in range(slices): - lo = slice_index * macro.width - logical_width = min(macro.width, memory.width - lo) - if banks == 1: - lines.append(f" rdata_comb[{lo} +: {logical_width}] = q_{slice_index}[0][{logical_width - 1}:0];") - else: - lines.append(" case (bank_d0)") - for bank in range(banks): - lines.append( - f" {bank_bits}'d{bank}: rdata_comb[{lo} +: {logical_width}] = q_{slice_index}[{bank}][{logical_width - 1}:0];" - ) - lines.append(" default: rdata_comb = 'x;") - lines.append(" endcase") - lines.extend( - [ - " end", - f" assign RW0_rdata = (ren_d0 && !wmode_d0) ? rdata_comb : {memory.width}'bx;", - "endmodule", - "", - ] - ) - return "\n".join(lines), { - "module": memory.name, - "source": str(memory.source), - "status": "mapped", - "port_type": "1RW", - "logical_depth": memory.depth, - "logical_width": memory.width, - "macro": macro.name, - "macro_depth": macro.depth, - "macro_width": macro.width, - "macro_write_size": macro.write_size, - "banks": banks, - "slices": slices, - "instances": banks * slices, - "macro_area": macro.area, - "macro_total_area": macro.area * banks * slices, - "liberty": str(macro.liberty), - "lef": str(macro.lef), - "verilog": str(macro.verilog), - } - - -def emit_blackboxes(macros: list[Macro]) -> str: - lines = ["// SRAM22 blackboxes for synthesis; physical views are in manifest.", ""] - for macro in sorted({macro.name: macro for macro in macros}.values(), key=lambda item: item.name): - addr_width = ceil_log2(macro.depth) - mask_width = macro.width // macro.write_size - lines.extend( - [ - "(* blackbox *)", - f"module {macro.name}(", - " input clk,", - " input we,", - f" input [{mask_width - 1}:0] wmask,", - f" input [{addr_width - 1}:0] addr,", - f" input [{macro.width - 1}:0] din,", - f" output [{macro.width - 1}:0] dout", - ");", - "endmodule", - "", - ] - ) - return "\n".join(lines) - - -def prepare_ip_replacement( - source_paths: list[str], output_dir: str, sky130_root: str | None, top_module: str | None = None +def prepare_sram_collateral( + source_paths: list[str], output_dir: str, top_module: str ) -> dict[str, object]: - """Create IP replacement collateral for DC and Yosys consumers.""" - if not sky130_root: - return {"source_paths": source_paths, "macro_liberties": [], "macro_area": 0.0, "mapped": 0} - root = Path(os.path.expandvars(os.path.expanduser(sky130_root))) - if not root.is_dir(): - return {"source_paths": source_paths, "macro_liberties": [], "macro_area": 0.0, "mapped": 0} - - macros = load_macros(root) - sources = [Path(path) for path in source_paths] - memories, unsupported = discover_memories(sources) + """Emit top-scoped source and memory manifests without choosing a PDK macro.""" + sources = [Path(path).resolve() for path in source_paths] reachable = reachable_modules(sources, top_module) - if reachable is not None: - memories = [memory for memory in memories if memory.name in reachable] - unsupported = [row for row in unsupported if row["module"] in reachable] + source_modules = module_sources(sources) + selected_paths = {source_modules[module] for module in reachable} + selected = [path for path in sources if path in selected_paths] + memories = discover_memories(selected) + output = Path(output_dir) output.mkdir(parents=True, exist_ok=True) - - wrappers: list[str] = [] - manifest: list[dict[str, object]] = unsupported[:] - replaced_sources: set[Path] = set() - used_macros: list[Macro] = [] - for memory in memories: - try: - macro = choose_macro(memory, macros) - wrapper, row = emit_wrapper(memory, macro) - except Exception as exc: - manifest.append( - {"module": memory.name, "source": str(memory.source), "status": "unsupported", "reason": str(exc)} - ) - continue - wrappers.append(wrapper) - manifest.append(row) - replaced_sources.add(memory.source.resolve()) - used_macros.append(macro) - - wrapper_path = output / "sram22_replacements.sv" - blackbox_path = output / "sram22_blackboxes.v" - manifest_path = output / "sram22_manifest.json" - wrapper_path.write_text("\n".join(wrappers)) - blackbox_path.write_text(emit_blackboxes(used_macros)) + manifest_path = output / "sram_manifest.json" + manifest = { + "schema_version": 1, + "top_module": top_module, + "memories": [ + { + "module": memory.name, + "source": str(memory.source), + "depth": memory.depth, + "width": memory.width, + "port_type": memory.port_type, + "address_width": memory.address_width, + "write_mask_width": memory.write_mask_width, + } + for memory in memories + ], + } manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") - - macro_liberties = sorted({str(macro.liberty) for macro in used_macros}) - macro_area = sum(float(row.get("macro_total_area", 0.0)) for row in manifest) - remaining_sources = [path for path in source_paths if Path(path).resolve() not in replaced_sources] - extra_sources = [str(blackbox_path), str(wrapper_path)] if used_macros else [] return { - "source_paths": extra_sources + remaining_sources, - "macro_liberties": macro_liberties, - "macro_area": macro_area, - "mapped": len(used_macros), - "manifest": str(manifest_path), + "source_paths": [str(path) for path in selected], + "sram_manifest": str(manifest_path), + "sram_memory_count": len(memories), + "top_module": top_module, } def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--source-list", type=Path, help="file containing one Verilog source path per line") + parser.add_argument("--source-list", type=Path, help="one Verilog source path per line") parser.add_argument("--source-dir", type=Path, help="directory to scan when --source-list is omitted") parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--sky130-root", default=os.environ.get("SKY130_ROOT")) - parser.add_argument("--top", default=None) + parser.add_argument("--top", required=True) args = parser.parse_args() if args.source_list: @@ -403,7 +177,7 @@ def main() -> int: else: parser.error("one of --source-list or --source-dir is required") - result = prepare_ip_replacement(sources, str(args.output_dir), args.sky130_root, args.top) + result = prepare_sram_collateral(sources, str(args.output_dir), args.top) print(json.dumps(result, indent=2, sort_keys=True)) return 0 diff --git a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py index 95a696e07..496b6bf58 100644 --- a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py +++ b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py @@ -10,11 +10,14 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +step_path = os.path.dirname(__file__) +if step_path not in sys.path: + sys.path.insert(0, step_path) from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id -from utils.verilog import seq_mem_elaboration_command +from verilog import seq_mem_elaboration_command config = { "name": "yosys verilog", @@ -169,6 +172,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"Yosys log dir: {yosys_log_dir}") yosys_cfg = load_yosys_config() + top_module = input_data.get("top") or yosys_cfg.get("top") or "DigitalTop" elaborate_config = input_data.get("config") or yosys_cfg.get( "elaborate_config", "sims.verilator.BuckyballToyVerilatorConfig" ) @@ -232,6 +236,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "source_list": source_list_path, "ip_replace_output_dir": yosys_log_dir, "consumer": "yosys", + "top": top_module, "mem_conf": mem_conf, "next_topic": "yosys.synth" if input_data.get("from_run_workflow") else None, "task": "run" if input_data.get("from_run_workflow") else "verilog", diff --git a/bbdev/api/steps/mill/04_dc_verilog_event.step.py b/bbdev/api/steps/mill/04_dc_verilog_event.step.py index f10c3f36e..7ce603099 100644 --- a/bbdev/api/steps/mill/04_dc_verilog_event.step.py +++ b/bbdev/api/steps/mill/04_dc_verilog_event.step.py @@ -3,17 +3,21 @@ import sys import glob import re +import shlex from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +step_path = os.path.dirname(__file__) +if step_path not in sys.path: + sys.path.insert(0, step_path) from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger -from utils.verilog import seq_mem_elaboration_command +from verilog import seq_mem_elaboration_command config = { "name": "dc-verilog", @@ -77,6 +81,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" elaborate_config = input_data.get("config") + top_module = input_data.get("top") or "DigitalTop" if not elaborate_config: _, failure_result = await check_result( ctx, @@ -96,7 +101,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return failure_result - ctx.logger.info(f"Using DC RTL output directory: {build_dir}") + ctx.logger.info(f"Using DC RTL output directory: {build_dir}, synthesis top: {top_module}") if os.path.exists(build_dir): shutil.rmtree(build_dir) @@ -104,9 +109,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: mem_conf = os.path.join(build_dir, "mems.conf") verilog_command = seq_mem_elaboration_command(elaborate_config, build_dir, mem_conf) + elaboration_log = os.path.join(build_dir, "dc_elaboration.log") result = stream_run_logger( - cmd=verilog_command, + # Elaboration emits the complete device tree and environment, which is + # useful for postmortem debugging but too large for a bbdev terminal. + # Keep it with the generated collateral and leave the CLI responsive. + cmd=f"{verilog_command} > {shlex.quote(elaboration_log)} 2>&1", logger=ctx.logger, cwd=arch_dir, stdout_prefix="dc verilog", @@ -118,7 +127,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, result.returncode, continue_run=False, - extra_fields={"task": "verilog"}, + extra_fields={"task": "verilog", "elaboration_log": elaboration_log}, trace_id=origin_tid, ) return failure_result @@ -145,7 +154,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, result.returncode, continue_run=True, - extra_fields={"task": "verilog", "source_list": source_list_path, "mem_conf": mem_conf}, + extra_fields={ + "task": "verilog", + "source_list": source_list_path, + "mem_conf": mem_conf, + "top_module": top_module, + "elaboration_log": elaboration_log, + }, trace_id=origin_tid, ) @@ -158,6 +173,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "ip_replace_output_dir": os.path.join(build_dir, "ip-replace"), "consumer": "dc", "mem_conf": mem_conf, + "top": top_module, + "next_topic": "dc.area" if input_data.get("from_area_workflow") or input_data.get("from_power_workflow") else None, "_trace_id": origin_tid, }, } diff --git a/bbdev/api/utils/verilog.py b/bbdev/api/steps/mill/verilog.py similarity index 100% rename from bbdev/api/utils/verilog.py rename to bbdev/api/steps/mill/verilog.py diff --git a/bbdev/api/steps/vcs/01_clean_api.step.py b/bbdev/api/steps/vcs/01_clean_api.step.py new file mode 100644 index 000000000..cebeb4af5 --- /dev/null +++ b/bbdev/api/steps/vcs/01_clean_api.step.py @@ -0,0 +1,31 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path, get_vcs_build_dir + + +config = { + "name": "vcs-clean-api", + "description": "clean VCS artifacts without deleting generated RTL", + "flows": ["vcs"], + "triggers": [api("POST", "/vcs/clean")], + "enqueues": ["vcs.clean"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + config_name = body.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + return ApiResponse(status=400, body={"error": "Missing required parameter: --config must be specified"}) + bbdir = get_buckyball_path() + await ctx.enqueue( + { + "topic": "vcs.clean", + "data": { + "config": config_name, + "output_dir": get_vcs_build_dir(bbdir, config_name, body.get("output_dir")), + "_trace_id": ctx.trace_id, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/01_clean_event.step.py b/bbdev/api/steps/vcs/01_clean_event.step.py new file mode 100644 index 000000000..26a4e1534 --- /dev/null +++ b/bbdev/api/steps/vcs/01_clean_event.step.py @@ -0,0 +1,31 @@ +import os +import shutil +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path, get_vcs_build_dir + + +config = { + "name": "vcs-clean", + "description": "remove simulator-specific VCS artifacts", + "flows": ["vcs"], + "triggers": [queue("vcs.clean")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + build_dir = get_vcs_build_dir(bbdir, input_data.get("config"), input_data.get("output_dir")) + artifact_dir = os.path.join(build_dir, "vcs") + if os.path.isdir(artifact_dir): + shutil.rmtree(artifact_dir) + await check_result(ctx, 0, extra_fields={"task": "clean", "artifact_dir": artifact_dir}, trace_id=origin_tid) diff --git a/bbdev/api/steps/vcs/02_verilog_api.step.py b/bbdev/api/steps/vcs/02_verilog_api.step.py new file mode 100644 index 000000000..f0ed9e14c --- /dev/null +++ b/bbdev/api/steps/vcs/02_verilog_api.step.py @@ -0,0 +1,32 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path, get_vcs_build_dir + + +config = { + "name": "vcs-verilog-api", + "description": "generate BBSimHarness RTL for VCS", + "flows": ["vcs"], + "triggers": [api("POST", "/vcs/verilog")], + "enqueues": ["vcs.verilog"], +} + + +def _request_data(body: dict) -> dict: + config_name = body.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + raise ValueError("Missing required parameter: --config must be specified") + bbdir = get_buckyball_path() + return { + "config": config_name, + "output_dir": get_vcs_build_dir(bbdir, config_name, body.get("output_dir")), + } + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + try: + data = _request_data(req.body or {}) + except ValueError as exc: + return ApiResponse(status=400, body={"error": str(exc)}) + await ctx.enqueue({"topic": "vcs.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/02_verilog_event.step.py b/bbdev/api/steps/vcs/02_verilog_event.step.py new file mode 100644 index 000000000..c2bd6f666 --- /dev/null +++ b/bbdev/api/steps/vcs/02_verilog_event.step.py @@ -0,0 +1,52 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path, get_vcs_build_dir +from utils.stream_run import stream_run_logger + + +config = { + "name": "vcs-verilog", + "description": "generate BBSimHarness RTL for VCS", + "flows": ["vcs"], + "triggers": [queue("vcs.verilog")], + "enqueues": ["vcs.build"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + config_name = input_data.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + await check_result(ctx, 1, extra_fields={"task": "verilog", "error": "missing config"}, trace_id=origin_tid) + return + bbdir = get_buckyball_path() + build_dir = get_vcs_build_dir(bbdir, config_name, input_data.get("output_dir")) + command = ( + f"mill -i __.test.runMain sims.verilator.Elaborate {config_name} " + "--disable-annotation-unknown --strip-debug-info -O=release " + f"--split-verilog -o={build_dir}" + ) + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=f"{bbdir}/arch", + stdout_prefix="vcs verilog", + stderr_prefix="vcs verilog", + ) + await check_result( + ctx, + result.returncode, + continue_run=input_data.get("from_run_workflow", False), + extra_fields={"task": "verilog", "output_dir": build_dir}, + trace_id=origin_tid, + ) + if result.returncode == 0 and input_data.get("from_run_workflow"): + await ctx.enqueue({"topic": "vcs.build", "data": {**input_data, "output_dir": build_dir, "_trace_id": origin_tid}}) diff --git a/bbdev/api/steps/vcs/03_build_api.step.py b/bbdev/api/steps/vcs/03_build_api.step.py new file mode 100644 index 000000000..eaa8586ee --- /dev/null +++ b/bbdev/api/steps/vcs/03_build_api.step.py @@ -0,0 +1,34 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path, get_vcs_build_dir + + +config = { + "name": "vcs-build-api", + "description": "build a VCS BBSimHarness executable", + "flows": ["vcs"], + "triggers": [api("POST", "/vcs/build")], + "enqueues": ["vcs.build"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + config_name = body.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + return ApiResponse(status=400, body={"error": "Missing required parameter: --config must be specified"}) + jobs = body.get("jobs", 16) + try: + jobs = int(jobs) + except (TypeError, ValueError): + return ApiResponse(status=400, body={"error": "jobs must be a positive integer"}) + if jobs <= 0: + return ApiResponse(status=400, body={"error": "jobs must be a positive integer"}) + bbdir = get_buckyball_path() + data = { + "config": config_name, + "jobs": jobs, + "output_dir": get_vcs_build_dir(bbdir, config_name, body.get("output_dir")), + } + await ctx.enqueue({"topic": "vcs.build", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/03_build_event.step.py b/bbdev/api/steps/vcs/03_build_event.step.py new file mode 100644 index 000000000..d836614d4 --- /dev/null +++ b/bbdev/api/steps/vcs/03_build_event.step.py @@ -0,0 +1,218 @@ +import glob +import os +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path, get_vcs_build_dir +from utils.stream_run import stream_run_logger + + +config = { + "name": "vcs-build", + "description": "compile BBSimHarness with Synopsys VCS", + "flows": ["vcs"], + "triggers": [queue("vcs.build")], + "enqueues": ["vcs.sim"], +} + + +TESTBENCH = r'''`timescale 1ns/1ps +module BBSimVcsHarness; + reg clock = 1'b0; + reg reset = 1'b1; + integer timeout_ns = 100000000; + + BBSimHarness dut (.clock(clock), .reset(reset)); + + always #0.5 clock = ~clock; + + initial begin + if ($value$plusargs("timeout-ns=%d", timeout_ns)) begin end + repeat (10) @(posedge clock); + reset = 1'b0; + #(timeout_ns); + $fatal(1, "VCS simulation timed out after %0d ns", timeout_ns); + end + + initial begin + if ($test$plusargs("vpd")) $vcdpluson(0, dut); + if ($test$plusargs("vcd")) begin : enable_vcd + string vcd_file; + if (!$value$plusargs("vcd=%s", vcd_file)) vcd_file = "activity.vcd"; + $dumpfile(vcd_file); + $dumpvars(0, dut); + end + end +endmodule +''' + + +# The normal BBSim monitor has a Verilator-specific C++ main. VCS runs from +# a SystemVerilog testbench, so it needs only the memory model plus these DPI +# callbacks. Trace callbacks are intentionally no-ops; functional execution, +# UART, and SCU exit remain available to every chip using BBSimHarness. +VCS_DPI = r'''#include +#include +#include + +extern "C" void dpi_bdb_set_clk(unsigned long long) {} +extern "C" void bbsim_memory_print_stats(); + +extern "C" void scu_uart_write(std::uint32_t, std::uint32_t ch) { + std::fputc(static_cast(ch & 0xff), stdout); + std::fflush(stdout); +} + +extern "C" void scu_sim_exit(std::uint32_t hart, std::uint32_t code) { + std::fprintf(stderr, "[SCU] hart %u: simulation exit code %u\\n", hart, code); + std::fflush(stderr); + bbsim_memory_print_stats(); + vpi_control(vpiFinish, static_cast(code)); +} + +extern "C" void scu_uart_rx_sample(std::uint32_t, std::uint32_t, + std::uint32_t *valid, std::uint32_t *data) { + *valid = 0; + *data = 0; +} + +extern "C" void dpi_itrace(std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t) {} +extern "C" void dpi_mtrace(std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t) {} +extern "C" void dpi_pmctrace(std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t) {} +extern "C" void dpi_mem_pmctrace(std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t) {} +extern "C" void dpi_mtrace_issue(std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t) {} +''' + + +def _pkg_config(flag: str, package: str) -> str: + result = subprocess.run(["pkg-config", flag, package], capture_output=True, text=True) + return result.stdout.strip() if result.returncode == 0 else "" + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + config_name = input_data.get("config") + if not isinstance(config_name, str) or not config_name or config_name == "None": + await check_result(ctx, 1, extra_fields={"task": "build", "error": "missing config"}, trace_id=origin_tid) + return + if shutil.which("vcs") is None: + await check_result( + ctx, + 1, + extra_fields={"task": "build", "error": "vcs is not on PATH; source the EDA host environment before running bbdev vcs"}, + trace_id=origin_tid, + ) + return + + bbdir = get_buckyball_path() + build_dir = get_vcs_build_dir(bbdir, config_name, input_data.get("output_dir")) + vsrcs = sorted( + path for path in ( + glob.glob(f"{build_dir}/**/*.v", recursive=True) + + glob.glob(f"{build_dir}/**/*.sv", recursive=True) + ) + if Path(path).name != "BBSimVcsHarness.sv" + ) + if not vsrcs: + await check_result(ctx, 1, extra_fields={"task": "build", "error": f"no RTL found under {build_dir}; run bbdev vcs --verilog first"}, trace_id=origin_tid) + return + + artifact_dir = Path(build_dir) / "vcs" + if artifact_dir.exists(): + shutil.rmtree(artifact_dir) + artifact_dir.mkdir(parents=True) + host_gxx = Path(__file__).parent / "scripts" / "host_gxx.sh" + host_gxx.chmod(0o755) + testbench = artifact_dir / "BBSimVcsHarness.sv" + dpi_shim = artifact_dir / "vcs_dpi.cc" + testbench.write_text(TESTBENCH, encoding="utf-8") + dpi_shim.write_text(VCS_DPI, encoding="utf-8") + + arch_dir = Path(bbdir) / "arch" + csrcs = [ + arch_dir / "src/csrc/src/monitor/ioe/BBSimDRAM.cc", + arch_dir / "src/csrc/src/monitor/ioe/mm.cc", + dpi_shim, + ] + missing = [str(path) for path in csrcs if not path.is_file()] + if missing: + await check_result(ctx, 1, extra_fields={"task": "build", "error": f"missing VCS DPI source(s): {', '.join(missing)}"}, trace_id=origin_tid) + return + + result_dir = Path(bbdir) / "result" + # VCS itself is a host (glibc 2.28) Synopsys binary. Do not let the Nix + # shell's compiler or pkg-config libraries leak into simv: that produces a + # glibc-2.42 executable which cannot load the host VCS runtime. + cflags = " ".join( + flag for flag in ( + "-std=c++17 -DVCS_NO_DRAMSIM", + f"-I{result_dir / 'include'}", + f"-I{build_dir}", + f"-I{arch_dir / 'src/csrc/include'}", + "", + ) if flag + ) + ldflags = " ".join( + flag for flag in ( + "-lz -lstdc++", + "-L/usr/lib64", + ) if flag + ) + jobs = int(input_data.get("jobs", 16)) + simv = artifact_dir / "simv" + compile_log = artifact_dir / "vcs_compile.log" + sources = [*vsrcs, str(testbench), *(str(path) for path in csrcs)] + command = " ".join( + [ + "env -u NIX_LDFLAGS -u NIX_CFLAGS_COMPILE -u NIX_LDFLAGS_FOR_TARGET -u NIX_CFLAGS_COMPILE_FOR_TARGET -u CPATH -u LIBRARY_PATH -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH -u CFLAGS -u CXXFLAGS -u LDFLAGS", + f"vcs -full64 -sverilog -timescale=1ns/1ps -cpp {shlex.quote(str(host_gxx))} -cc /usr/bin/gcc -ld {shlex.quote(str(host_gxx))}", + "-top BBSimVcsHarness -debug_access+all -hsopt=off", + f"-j {jobs}", + f"-Mdir={shlex.quote(str(artifact_dir / 'csrc'))}", + f"-o {shlex.quote(str(simv))}", + f"-l {shlex.quote(str(compile_log))}", + f"+incdir+{shlex.quote(build_dir)}", + f"-CFLAGS {shlex.quote(cflags)}", + f"-LDFLAGS {shlex.quote(ldflags)}", + *(shlex.quote(source) for source in sources), + ] + ) + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=artifact_dir, + stdout_prefix="vcs build", + stderr_prefix="vcs build", + ) + await check_result( + ctx, + result.returncode, + continue_run=input_data.get("from_run_workflow", False), + extra_fields={"task": "build", "simv": str(simv), "build_dir": build_dir, "compile_log": str(compile_log)}, + trace_id=origin_tid, + ) + if result.returncode == 0 and input_data.get("from_run_workflow"): + await ctx.enqueue({"topic": "vcs.sim", "data": {**input_data, "output_dir": build_dir, "_trace_id": origin_tid}}) diff --git a/bbdev/api/steps/vcs/04_sim_api.step.py b/bbdev/api/steps/vcs/04_sim_api.step.py new file mode 100644 index 000000000..1ea7201d5 --- /dev/null +++ b/bbdev/api/steps/vcs/04_sim_api.step.py @@ -0,0 +1,22 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + + +config = { + "name": "vcs-sim-api", + "description": "run a built VCS BBSimHarness simulation", + "flows": ["vcs"], + "triggers": [api("POST", "/vcs/sim")], + "enqueues": ["vcs.sim"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + config_name = body.get("config") + binary = body.get("binary") + if not isinstance(config_name, str) or not config_name or config_name == "None": + return ApiResponse(status=400, body={"error": "Missing required parameter: --config must be specified"}) + if not isinstance(binary, str) or not binary: + return ApiResponse(status=400, body={"error": "Missing required parameter: --binary must be specified"}) + await ctx.enqueue({"topic": "vcs.sim", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/04_sim_event.step.py b/bbdev/api/steps/vcs/04_sim_event.step.py new file mode 100644 index 000000000..2302522ed --- /dev/null +++ b/bbdev/api/steps/vcs/04_sim_event.step.py @@ -0,0 +1,96 @@ +import os +import shlex +import sys +from datetime import datetime +from pathlib import Path + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path, get_vcs_build_dir +from utils.search_workload import search_workload +from utils.stream_run import stream_run_logger + + +config = { + "name": "vcs-sim", + "description": "run a VCS BBSimHarness simulation", + "flows": ["vcs"], + "triggers": [queue("vcs.sim")], + "enqueues": [], +} + + +def _resolve_binary(bbdir: str, binary: str) -> str | None: + if os.path.isfile(binary): + return os.path.abspath(binary) + for root in ("output/workloads/src", "build/workloads/src"): + found = search_workload(f"{bbdir}/bb-tests/{root}", binary) + if found: + return found + return None + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + config_name = input_data.get("config") + binary_name = input_data.get("binary") + if not isinstance(config_name, str) or not config_name or config_name == "None": + await check_result(ctx, 1, extra_fields={"task": "sim", "error": "missing config"}, trace_id=origin_tid) + return + if not isinstance(binary_name, str) or not binary_name: + await check_result(ctx, 1, extra_fields={"task": "sim", "error": "missing binary"}, trace_id=origin_tid) + return + bbdir = get_buckyball_path() + binary = _resolve_binary(bbdir, binary_name) + if binary is None: + await check_result(ctx, 1, extra_fields={"task": "sim", "error": f"binary not found: {binary_name}"}, trace_id=origin_tid) + return + build_dir = get_vcs_build_dir(bbdir, config_name, input_data.get("output_dir")) + artifact_dir = Path(build_dir) / "vcs" + simv = artifact_dir / "simv" + if not simv.is_file(): + await check_result(ctx, 1, extra_fields={"task": "sim", "error": f"VCS executable missing: {simv}; run bbdev vcs --build first"}, trace_id=origin_tid) + return + + timestamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") + log_dir = Path(bbdir) / "log" / f"{timestamp}-vcs-{config_name.replace('/', '_')}" + log_dir.mkdir(parents=True) + timeout_ns = input_data.get("timeout_ns", input_data.get("timeout-ns", "100000000")) + try: + if float(timeout_ns) <= 0: + raise ValueError + except (TypeError, ValueError): + await check_result(ctx, 1, extra_fields={"task": "sim", "error": "timeout-ns must be a positive number"}, trace_id=origin_tid) + return + wave = input_data.get("waveform", input_data.get("wave")) + wave_arg = "+vcd" if str(wave).lower() == "vcd" else ("+vpd" if wave else "") + command = " ".join( + [ + "env -u LD_LIBRARY_PATH", + shlex.quote(str(simv)), + "+permissive", + f"+elf={shlex.quote(binary)}", + "+batch" if input_data.get("batch", False) else "", + wave_arg, + f"+timeout-ns={shlex.quote(str(timeout_ns))}", + "+permissive-off", + ] + ) + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=artifact_dir, + stdout_prefix="vcs sim", + stderr_prefix="vcs sim", + ) + await check_result( + ctx, + result.returncode, + extra_fields={"task": "sim", "simv": str(simv), "binary": binary, "log_dir": str(log_dir)}, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/vcs/05_run_api.step.py b/bbdev/api/steps/vcs/05_run_api.step.py new file mode 100644 index 000000000..d7fbc7fbb --- /dev/null +++ b/bbdev/api/steps/vcs/05_run_api.step.py @@ -0,0 +1,40 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.path import get_buckyball_path, get_vcs_build_dir + + +config = { + "name": "vcs-run-api", + "description": "generate RTL, build, and run VCS", + "flows": ["vcs"], + "triggers": [api("POST", "/vcs/run")], + "enqueues": ["vcs.verilog"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + config_name = body.get("config") + binary = body.get("binary") + if not isinstance(config_name, str) or not config_name or config_name == "None": + return ApiResponse(status=400, body={"error": "Missing required parameter: --config must be specified"}) + if not isinstance(binary, str) or not binary: + return ApiResponse(status=400, body={"error": "Missing required parameter: --binary must be specified"}) + jobs = body.get("jobs", 16) + try: + jobs = int(jobs) + except (TypeError, ValueError): + return ApiResponse(status=400, body={"error": "jobs must be a positive integer"}) + if jobs <= 0: + return ApiResponse(status=400, body={"error": "jobs must be a positive integer"}) + bbdir = get_buckyball_path() + data = { + "config": config_name, + "binary": binary, + "batch": bool(body.get("batch", False)), + "jobs": jobs, + "output_dir": get_vcs_build_dir(bbdir, config_name, body.get("output_dir")), + "from_run_workflow": True, + } + await ctx.enqueue({"topic": "vcs.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/README.md b/bbdev/api/steps/vcs/README.md new file mode 100644 index 000000000..3ff698e8e --- /dev/null +++ b/bbdev/api/steps/vcs/README.md @@ -0,0 +1,18 @@ +# VCS workflow + +VCS follows the same user-facing stages as Verilator while keeping its own +simulator products below `arch/build//vcs/`: + +```text +vcs.verilog → vcs.build → vcs.sim +``` + +`bbdev vcs --run` routes through all three stages. The generated RTL is the +standard `BBSimHarness` elaboration; `vcs.build` adds a small SystemVerilog +testbench and a VCS-only DPI shim. The shim supplies BBSimDRAM, SCU UART, and +SCU exit callbacks without linking the Verilator-specific BBSim C++ main. + +The VCS flow is intended for RTL functional simulation. Gate-level power +simulation remains chip-owned under `examples/chips//tapeout/`, because +that flow must define the chip's workload, reset/boot skip interval, and SDF +policy. diff --git a/bbdev/api/steps/yosys/01_run_api.step.py b/bbdev/api/steps/yosys/01_run_api.step.py index dbe56c426..95c097150 100644 --- a/bbdev/api/steps/yosys/01_run_api.step.py +++ b/bbdev/api/steps/yosys/01_run_api.step.py @@ -28,7 +28,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "output_dir": req_arg(body, "output_dir") or f"{bbdir}/arch/build/", "log_dir": log_dir, - "top": req_arg(body, "top"), + "top": req_arg(body, "top") or "DigitalTop", "config": req_arg(body, "config"), "vcd": req_arg(body, "vcd"), "from_run_workflow": True, diff --git a/bbdev/api/steps/yosys/02_verilog_api.step.py b/bbdev/api/steps/yosys/02_verilog_api.step.py index a472ae2d8..f03ae57a7 100644 --- a/bbdev/api/steps/yosys/02_verilog_api.step.py +++ b/bbdev/api/steps/yosys/02_verilog_api.step.py @@ -28,7 +28,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "output_dir": req_arg(body, "output_dir") or f"{bbdir}/arch/build/", "log_dir": log_dir, - "top": req_arg(body, "top"), + "top": req_arg(body, "top") or "DigitalTop", "config": req_arg(body, "config"), } await ctx.enqueue({"topic": "yosys.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) diff --git a/bbdev/api/steps/yosys/03_synth_api.step.py b/bbdev/api/steps/yosys/03_synth_api.step.py index 363afc37c..d3da70b9b 100644 --- a/bbdev/api/steps/yosys/03_synth_api.step.py +++ b/bbdev/api/steps/yosys/03_synth_api.step.py @@ -28,7 +28,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "output_dir": req_arg(body, "output_dir") or f"{bbdir}/arch/build/", "log_dir": log_dir, - "top": req_arg(body, "top"), + "top": req_arg(body, "top") or "DigitalTop", "config": req_arg(body, "config"), "vcd": req_arg(body, "vcd"), } diff --git a/bbdev/api/steps/yosys/03_synth_event.step.py b/bbdev/api/steps/yosys/03_synth_event.step.py index 7ae3dfb61..497ef1761 100644 --- a/bbdev/api/steps/yosys/03_synth_event.step.py +++ b/bbdev/api/steps/yosys/03_synth_event.step.py @@ -66,8 +66,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/") yosys_cfg = load_yosys_config() - top_module = input_data.get("top") or yosys_cfg.get("top") or "BuckyballAccelerator" - liberty = yosys_cfg.get("liberty") + top_module = input_data.get("top") or yosys_cfg.get("top") or "DigitalTop" + liberty = yosys_cfg.get("liberty") or os.environ.get("YOSYS_LIBERTY") + if isinstance(liberty, str): + liberty = os.path.expandvars(os.path.expanduser(liberty)) source_list_path = input_data.get("source_list") or os.path.join(build_dir, "yosys_sources.list") if not os.path.exists(source_list_path): @@ -99,10 +101,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: os.makedirs(yosys_output_dir, exist_ok=True) ctx.logger.info(f"Yosys log dir: {yosys_output_dir}") - ip_replacement = input_data.get("ip_replacement") or {} - macro_liberties = ip_replacement.get("macro_liberties", []) - macro_area = ip_replacement.get("macro_area", 0.0) - mapped_memories = ip_replacement.get("mapped", 0) + sram_collateral = input_data.get("sram_collateral") or {} read_commands = "\n".join([f"read_verilog -sv {src}" for src in vsrcs]) yosys_script = f"{yosys_output_dir}/synth_area.ys" @@ -140,9 +139,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra = { "task": "synth", "output_dir": yosys_output_dir, - "ip_manifest": ip_replacement.get("manifest"), - "ip_mapped_memories": mapped_memories, - "ip_macro_area": macro_area, + "sram_manifest": sram_collateral.get("sram_manifest"), + "sram_memory_count": sram_collateral.get("sram_memory_count", 0), } netlist_file = f"{yosys_output_dir}/synth_netlist.v" timing_report_file = f"{yosys_output_dir}/timing_report.txt" @@ -172,8 +170,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: sta_script = f"{yosys_output_dir}/sta_timing.tcl" with open(sta_script, "w") as f: f.write(f"read_liberty {liberty}\n") - for macro_liberty in macro_liberties: - f.write(f"read_liberty {macro_liberty}\n") f.write(f"read_verilog {netlist_file}\n") f.write(f"link_design {top_module}\n") # Prefer the configured port, then use the generated-memory clock @@ -230,20 +226,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if os.path.exists(report_file): with open(report_file, "r") as f: extra["area_report"] = f.read() - if macro_area: - chip_area = re.search(r"Chip area for module .*?:\s+([0-9.]+)", extra["area_report"]) - standard_area = float(chip_area.group(1)) if chip_area else 0.0 - total_area = standard_area + macro_area - ip_area_report = ( - "\nIP macro area summary\n" - f" Macro area: {macro_area:.6f} um^2\n" - f" Standard-cell area: {standard_area:.6f} um^2\n" - f" Total estimated area: {total_area:.6f} um^2\n" - ) - with open(report_file, "a") as f: - f.write(ip_area_report) - extra["area_report"] += ip_area_report - extra["estimated_total_area"] = total_area hierarchy_file = f"{yosys_output_dir}/hierarchy_report.txt" if os.path.exists(hierarchy_file): diff --git a/bbdev/api/steps/yosys/scripts/yosys-config.yaml b/bbdev/api/steps/yosys/scripts/yosys-config.yaml index 61be109d3..49ed4ed51 100644 --- a/bbdev/api/steps/yosys/scripts/yosys-config.yaml +++ b/bbdev/api/steps/yosys/scripts/yosys-config.yaml @@ -2,13 +2,12 @@ # Users can customize liberty library paths and synthesis options here # Liberty (.lib) file path for ASIC area estimation -# If not specified, yosys will use generic gate-level statistics -# The default Nix EDA shell exports SKY130_ROOT. bbdev expands environment -# variables in this file before invoking Yosys/OpenSTA. -liberty: ${SKY130_ROOT}/lib/sky130_fd_sc_hd__tt_025C_1v80.lib +# If not specified, yosys will use generic gate-level statistics. A technology +# owner may set YOSYS_LIBERTY in the host environment. +liberty: # Default top module name (override with --top on CLI) -top: BuckyballAccelerator +top: DigitalTop # Elaborate config class (override with --config on CLI) elaborate_config: sims.verilator.BuckyballToyVerilatorConfig diff --git a/bbdev/api/tests/test_api_dc_verilog.py b/bbdev/api/tests/test_api_dc_verilog.py index ff77de9e7..397326394 100644 --- a/bbdev/api/tests/test_api_dc_verilog.py +++ b/bbdev/api/tests/test_api_dc_verilog.py @@ -1,3 +1,16 @@ +import json +from pathlib import Path + from _api_test_helper import run_bbdev_case run_bbdev_case("bbdev dc --verilog '--config sims.verilator.BuckyballToyVerilatorConfig'") + +root = Path(__file__).resolve().parents[3] +output_dir = root / "arch" / "build" / "sims.verilator.BuckyballToyVerilatorConfig" +manifest = json.loads((output_dir / "ip-replace" / "sram_manifest.json").read_text()) +source_list = (output_dir / "ip-replace" / "dc_sources.list").read_text().splitlines() + +assert manifest["top_module"] == "DigitalTop" +assert manifest["memories"] +assert any(path.endswith("/DigitalTop.sv") for path in source_list) +assert not any(path.endswith("/BBSimHarness.sv") for path in source_list) diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index 48ebfc902..f6ecb21ab 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -1,5 +1,6 @@ import os import re +from datetime import datetime from pathlib import Path @@ -24,6 +25,15 @@ def get_verilator_build_dir(bbdir, config=None, output_dir=None): return get_config_build_dir(bbdir, config) +def get_vcs_build_dir(bbdir, config=None, output_dir=None): + """Return the shared generated-RTL directory used by the VCS flow. + + VCS and Verilator elaborate the same BBSimHarness RTL. Simulator-specific + products live below ``vcs/`` so neither flow overwrites the other. + """ + return get_config_build_dir(bbdir, config, output_dir) + + def get_chip_from_config(bbdir, config): """Resolve the owning chip from a fully qualified Scala config name.""" if not isinstance(config, str) or not config or config == "None": @@ -77,7 +87,7 @@ def get_config_build_dir(bbdir, config=None, output_dir=None, output_root=None): return f"{bbdir}/arch/build" -def get_dc_rtl_dir(bbdir, config=None, base_dir=None): +def get_dc_rtl_dir(bbdir, config=None): if not config or config is True: raise ValueError("missing required parameter: config") @@ -85,16 +95,66 @@ def get_dc_rtl_dir(bbdir, config=None, base_dir=None): if not name: raise ValueError("invalid config name") - if base_dir is True: - raise ValueError("dir requires a path value") - root = base_dir or os.path.join(bbdir, "arch", "build") - if not os.path.isabs(root): - raise ValueError("dir must be an absolute path") - return os.path.join(root, name) + return os.path.join(bbdir, "arch", "build", name) + + +def get_dc_analysis_dir(bbdir, config=None, stage="area"): + if not config or config is True: + raise ValueError("missing required parameter: config") + name = sanitize_config_name(config) + if not name: + raise ValueError("invalid config name") + if stage not in {"area", "power"}: + raise ValueError(f"invalid DC analysis stage: {stage}") + timestamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") + return os.path.join(bbdir, "log", f"{timestamp}-dc-{name}", stage) def check_dc_rtl_args(body: dict): - allowed = {"config", "dir"} + allowed = {"config", "top"} + for name in body: + if name not in allowed: + raise ValueError(f"unexpected parameter: {name}") + top = body.get("top") + if top is not None and (not isinstance(top, str) or not top): + raise ValueError("top must be a non-empty module name") + if isinstance(top, str) and not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_$]*", top): + raise ValueError("top must be a legal unescaped Verilog module name") + + +def check_dc_power_args(body: dict): + allowed = {"config", "top", "activity", "format", "strip_path", "workload", "start_ns", "end_ns", "start-ns", "end-ns"} for name in body: if name not in allowed: raise ValueError(f"unexpected parameter: {name}") + common = {key: body[key] for key in ("config", "top") if key in body} + check_dc_rtl_args(common) + activity = body.get("activity") + activity_format = body.get("format") + if activity is not None and (not isinstance(activity, str) or not activity): + raise ValueError("activity must be a non-empty path when provided") + if activity_format is not None and activity_format not in {"saif", "vcd", "fsdb"}: + raise ValueError("format must be one of saif, vcd, or fsdb") + if (activity is None) != (activity_format is None): + raise ValueError("activity and format must be provided together") + strip_path = body.get("strip_path") + if strip_path is not None and not isinstance(strip_path, str): + raise ValueError("strip_path must be a string") + for name in ("workload",): + value = body.get(name) + if value is not None and (not isinstance(value, str) or not value): + raise ValueError(f"{name} must be a non-empty path when provided") + if isinstance(value, str) and os.path.isabs(value): + raise ValueError(f"{name} must be a built ELF name, not an absolute path") + for name in ("start_ns", "end_ns", "start-ns", "end-ns"): + value = body.get(name) + if value is not None: + try: + if float(value) < 0: + raise ValueError + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be a non-negative number") from exc + start = body.get("start_ns", body.get("start-ns")) + end = body.get("end_ns", body.get("end-ns")) + if start is not None and end is not None and float(start) >= float(end): + raise ValueError("start_ns must be smaller than end_ns") diff --git a/bbdev/bbdev b/bbdev/bbdev index 1cb9c481d..811fa607b 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -10,12 +10,14 @@ import time import tempfile import threading import requests +import shutil sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "api")) from utils.port import HOST, reserve_port from utils.process import clean_python_caches, stop_workspace_servers, terminate_group workflow_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "api") +workspace_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) NO_PROXY = {"http": None, "https": None, "all": None} WORKER_PORT_START = 49200 @@ -35,11 +37,11 @@ WORKFLOW_COMMANDS = { "cosim": 'Run cosimulation. Args: "--binary [--batch]"', }, "vcs": { - "clean": True, - "verilog": True, - "build": 'Build simulation executable. Args: "[--job ]"', - "sim": 'Run simulation. Args: "--binary [--batch]"', - "run": 'Integrated build+sim+run. Args: "--binary [--batch] [--job ]"', + "clean": 'Clean VCS artifacts. Args: "--config [--output-dir ]"', + "verilog": 'Generate RTL for VCS. Args: "--config [--output-dir ]"', + "build": 'Build VCS simulation executable. Args: "--config [--jobs ] [--output-dir ]"', + "sim": 'Run a built VCS simulation. Args: "--config --binary [--batch] [--output-dir ]"', + "run": 'Generate RTL, build, and run VCS. Args: "--config --binary [--batch] [--jobs ] [--output-dir ]"', }, "workload": { "clean": "Clean workload output directory.", @@ -78,7 +80,9 @@ WORKFLOW_COMMANDS = { "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--chip --bitstream --test "', }, "dc": { - "verilog": 'Generate DC RTL. Args: "--config [--dir ]"', + "verilog": 'Generate top-scoped DC RTL collateral under arch/build/. Args: "--config [--top ]"', + "area": 'Generate RTL, run DC synthesis, and report area under log/-dc-/area. Args: "--config [--top ]".', + "power": 'Generate RTL, run chip-owned power simulation, then run PrimeTime PX under log/-dc-/power. Args: "--config [--top ] [--workload ] [--start-ns ] [--end-ns ] [--activity --format ]".', }, "ip-replace": { "run": 'Replace behavioral IP RTL. Args: "--source-list --output-dir [--top ] [--consumer ]"', @@ -139,6 +143,37 @@ def cleanup_config(config_path: str): pass +def load_host_environment() -> None: + """Import the optional host EDA contract for worker subprocesses. + + ``nix develop -c`` does not start an interactive zsh, so a sourceme file + installed in zshrc is otherwise invisible to Motia workers. Import only + when a tool is missing and keep the operation scoped to this workspace. + """ + if shutil.which("dc_shell") and shutil.which("pt_shell") and shutil.which("vcs"): + return + sourceme = os.path.join(workspace_dir, "sourceme_host.sh") + if not os.path.isfile(sourceme): + return + try: + result = subprocess.run( + ["bash", "-c", 'source "$1" >/dev/null 2>&1; env -0', "bbdev", sourceme], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + except (OSError, subprocess.CalledProcessError): + return + for item in result.stdout.split(b"\0"): + if b"=" not in item: + continue + key, value = item.split(b"=", 1) + try: + os.environ[key.decode()] = value.decode() + except UnicodeDecodeError: + continue + + def parse_args(argv: list[str]) -> argparse.Namespace: try: argv = _normalize_workflow_args(argv) @@ -521,6 +556,8 @@ def run_script_mode(args, cmd_info): print(f"Available commands: {', '.join(ALL_COMMANDS)}") sys.exit(1) + load_host_environment() + api_path = f"/{command.replace('-', '/')}/{cmd_info['operation']}" # Reserve both ports atomically to prevent TOCTOU races with concurrent bbdev instances. diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index 80d0d4925..775c08e0c 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -30,6 +30,8 @@ from . import bebop_p2e_runworkload from . import bebop_p2e_batch from . import dc_verilog +from . import dc_area +from . import dc_power from . import ip_replace from . import firesim_enumeratefpgas from . import firesim_buildbitstream @@ -69,6 +71,8 @@ bebop_p2e_runworkload, bebop_p2e_batch, dc_verilog, + dc_area, + dc_power, ip_replace, firesim_enumeratefpgas, firesim_buildbitstream, diff --git a/bbdev/mcp/tools/dc_area.py b/bbdev/mcp/tools/dc_area.py new file mode 100644 index 000000000..dfd2342ac --- /dev/null +++ b/bbdev/mcp/tools/dc_area.py @@ -0,0 +1,16 @@ +"""MCP tool: bbdev_dc_area.""" + +from __future__ import annotations + +from typing import Optional + +from common import err, fmt, opt, submit + + +def register(mcp): + @mcp.tool() + def bbdev_dc_area(config: Optional[str] = None, top: Optional[str] = None) -> str: + """Run DC synthesis and area reporting. Defaults to DigitalTop.""" + if not config: + return err("config is required") + return fmt(submit("/dc/area", opt({}, config=config, top=top))) diff --git a/bbdev/mcp/tools/dc_power.py b/bbdev/mcp/tools/dc_power.py new file mode 100644 index 000000000..c7c3f46f8 --- /dev/null +++ b/bbdev/mcp/tools/dc_power.py @@ -0,0 +1,46 @@ +"""MCP tool: bbdev_dc_power.""" + +from __future__ import annotations + +from typing import Optional + +from common import err, fmt, opt, submit + + +def register(mcp): + @mcp.tool() + def bbdev_dc_power( + config: Optional[str] = None, + top: Optional[str] = None, + activity: Optional[str] = None, + activity_format: Optional[str] = None, + strip_path: Optional[str] = None, + workload: Optional[str] = None, + start_ns: Optional[str] = None, + end_ns: Optional[str] = None, + ) -> str: + """Run DC synthesis, chip-owned activity simulation, and PrimeTime PX. + + Omit activity/activity_format for the normal rerun-and-measure flow. + Supply both only as a debugging override. + """ + if not config: + return err("config is required") + if bool(activity) != bool(activity_format): + return err("activity and activity_format must be provided together") + return fmt( + submit( + "/dc/power", + opt( + {}, + config=config, + top=top, + activity=activity, + format=activity_format, + strip_path=strip_path, + workload=workload, + start_ns=start_ns, + end_ns=end_ns, + ), + ) + ) diff --git a/bbdev/mcp/tools/dc_verilog.py b/bbdev/mcp/tools/dc_verilog.py index 619bc0318..44c98f0cb 100644 --- a/bbdev/mcp/tools/dc_verilog.py +++ b/bbdev/mcp/tools/dc_verilog.py @@ -10,15 +10,15 @@ def register(mcp): @mcp.tool() def bbdev_dc_verilog( - config: Optional[str] = None, rtl_dir: Optional[str] = None + config: Optional[str] = None, + top: Optional[str] = None, ) -> str: - """Generate Verilog for DC flow. POST /dc/verilog (rtl_dir maps to dir).""" - if not config and not rtl_dir: - return err("config or rtl_dir is required") + """Generate top-scoped DC RTL collateral. Defaults to DigitalTop.""" + if not config: + return err("config is required") return fmt( submit( "/dc/verilog", - opt({}, config=config, dir=rtl_dir), + opt({}, config=config, top=top), ) ) - From cd4b306e28dfb56df00cee14eda70b846fbe50ac Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 12 Aug 2026 19:23:26 +0800 Subject: [PATCH 120/196] refactor: update VCS build and simulation steps to improve configuration and include DRAMSim3 support --- bbdev/api/steps/vcs/03_build_event.step.py | 14 +++++--------- bbdev/api/steps/vcs/04_sim_event.step.py | 1 + 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/bbdev/api/steps/vcs/03_build_event.step.py b/bbdev/api/steps/vcs/03_build_event.step.py index d836614d4..d53690ad3 100644 --- a/bbdev/api/steps/vcs/03_build_event.step.py +++ b/bbdev/api/steps/vcs/03_build_event.step.py @@ -144,8 +144,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if artifact_dir.exists(): shutil.rmtree(artifact_dir) artifact_dir.mkdir(parents=True) - host_gxx = Path(__file__).parent / "scripts" / "host_gxx.sh" - host_gxx.chmod(0o755) testbench = artifact_dir / "BBSimVcsHarness.sv" dpi_shim = artifact_dir / "vcs_dpi.cc" testbench.write_text(TESTBENCH, encoding="utf-8") @@ -155,6 +153,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: csrcs = [ arch_dir / "src/csrc/src/monitor/ioe/BBSimDRAM.cc", arch_dir / "src/csrc/src/monitor/ioe/mm.cc", + arch_dir / "src/csrc/src/monitor/ioe/mm_dramsim3.cc", dpi_shim, ] missing = [str(path) for path in csrcs if not path.is_file()] @@ -163,12 +162,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return result_dir = Path(bbdir) / "result" - # VCS itself is a host (glibc 2.28) Synopsys binary. Do not let the Nix - # shell's compiler or pkg-config libraries leak into simv: that produces a - # glibc-2.42 executable which cannot load the host VCS runtime. cflags = " ".join( flag for flag in ( - "-std=c++17 -DVCS_NO_DRAMSIM", + "-std=c++17", f"-I{result_dir / 'include'}", f"-I{build_dir}", f"-I{arch_dir / 'src/csrc/include'}", @@ -177,8 +173,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) ldflags = " ".join( flag for flag in ( - "-lz -lstdc++", - "-L/usr/lib64", + "-ldramsim3 -lz -lstdc++", + f"-L{result_dir / 'lib'} -Wl,-rpath,{result_dir / 'lib'}", ) if flag ) jobs = int(input_data.get("jobs", 16)) @@ -188,7 +184,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: command = " ".join( [ "env -u NIX_LDFLAGS -u NIX_CFLAGS_COMPILE -u NIX_LDFLAGS_FOR_TARGET -u NIX_CFLAGS_COMPILE_FOR_TARGET -u CPATH -u LIBRARY_PATH -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH -u CFLAGS -u CXXFLAGS -u LDFLAGS", - f"vcs -full64 -sverilog -timescale=1ns/1ps -cpp {shlex.quote(str(host_gxx))} -cc /usr/bin/gcc -ld {shlex.quote(str(host_gxx))}", + "vcs -full64 -sverilog -timescale=1ns/1ps -cpp g++ -cc g++ -ld g++", "-top BBSimVcsHarness -debug_access+all -hsopt=off", f"-j {jobs}", f"-Mdir={shlex.quote(str(artifact_dir / 'csrc'))}", diff --git a/bbdev/api/steps/vcs/04_sim_event.step.py b/bbdev/api/steps/vcs/04_sim_event.step.py index 2302522ed..ce46f6781 100644 --- a/bbdev/api/steps/vcs/04_sim_event.step.py +++ b/bbdev/api/steps/vcs/04_sim_event.step.py @@ -77,6 +77,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"+elf={shlex.quote(binary)}", "+batch" if input_data.get("batch", False) else "", wave_arg, + f"+dramsim_ini_dir={shlex.quote(str(Path(bbdir) / 'result/share/dramsim3/configs'))}", f"+timeout-ns={shlex.quote(str(timeout_ns))}", "+permissive-off", ] From 079f54ef67c259f4697ee6a056c61cc0e4cf67d2 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 13 Aug 2026 01:22:43 +0800 Subject: [PATCH 121/196] feat: add model layout and CMake configuration for various models in workload step --- .../api/steps/workload/01_build_event.step.py | 91 ++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index e3b1dc405..86c3c0032 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -39,6 +39,63 @@ "buddynext": "buddy-buckyball-buddynext-all-run", } +# CLI model → layout dir under archs/buckyball// +MODEL_LAYOUT = { + "lenet": "LeNet", + "mobilenet": "MobileNetV3", + "resnet": "ResNet18", + "yolo": "YOLO26", + "bert": "Bert", + "qwen3": "Qwen3", + "gemma4": "Gemma4", + "deepseekr1": "DeepSeekR1", + "llama2": "llama2", + "stable-diffusion": "StableDiffusion", + "whisper": "Whisper", + "buddynext": "BuddyNext", +} + +# CLI model → -DMODEL= value (CMake MODEL_ flags) +MODEL_CMAKE = { + "lenet": "lenet", + "mobilenet": "mobilenetv3", + "resnet": "resnet18", + "yolo": "yolo26", + "bert": "bert", + "qwen3": "qwen3", + "gemma4": "gemma4", + "deepseekr1": "deepseekr1", + "llama2": "llama2", + "stable-diffusion": "stablediffusion", + "whisper": "whisper", + "buddynext": "buddynext", +} + + +def chips_for_model(bbdir: str, model_key: str) -> set[str]: + """Chips that currently ship a layout for this model (many-to-many).""" + layout = MODEL_LAYOUT.get(model_key) + if layout is None: + return set() + root = ( + Path(bbdir) + / "bb-tests" + / "workloads" + / "src" + / "ModelTest" + / "e2e" + / "models" + / "archs" + / "buckyball" + ) + if not root.is_dir(): + return set() + return { + p.name + for p in root.iterdir() + if p.is_dir() and (p / layout).is_dir() + } + RUSHB_TARGETS = { "lenet": { "bemu": "buddy-buckyball-lenet-rushB-bemu-run", @@ -176,6 +233,34 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + layout = MODEL_LAYOUT.get(model_key) + if layout is None: + ctx.logger.error(f"Missing layout mapping for model {model}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_model_layout", "model": model}, + trace_id=origin_tid, + ) + return + supported_chips = chips_for_model(bbdir, model_key) + if chip not in supported_chips: + allowed = ", ".join(sorted(supported_chips)) if supported_chips else "(none)" + ctx.logger.error( + f"Model '{model}' has no Buckyball layout on chip '{chip}' " + f"(layout dir '{layout}'; chips with layout: {allowed})" + ) + await check_result( + ctx, 1, continue_run=False, + extra_fields={ + "error": "unsupported_chip_model", + "chip": chip, + "model": model, + "layout": layout, + "supported_chips": sorted(supported_chips), + }, + trace_id=origin_tid, + ) + return if rushb_backend is not None: target = RUSHB_TARGETS.get(model_key, {}).get(rushb_backend) if target is None: @@ -198,10 +283,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: chip_arg = f"-DBUCKYBALL_WORKLOAD_CHIP={shlex.quote(chip)}" core_arg = f"-DBUCKYBALL_WORKLOAD_CORE={shlex.quote(core_package.name)}" stable_arg = "-DBUCKYBALL_STABLE=ON" if stable else "-DBUCKYBALL_STABLE=OFF" + model_arg = "" + if model: + cmake_model = MODEL_CMAKE[model.lower()] + model_arg = f" -DMODEL={shlex.quote(cmake_model)}" ninja_target = f" {shlex.quote(target)}" if target else "" inner = ( f"cd {shlex.quote(build_dir)} && " - f"cmake -G Ninja {chip_arg} {core_arg} {stable_arg} " + f"cmake -G Ninja {chip_arg} {core_arg} {stable_arg}{model_arg} " f"-DPython3_EXECUTABLE=\"$(which python3)\" .. && " f"ninja -j{os.cpu_count()}{ninja_target}" ) From 67577a1ea3f54459486e00e0012f38812a2164bb Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Fri, 14 Aug 2026 05:54:44 +0800 Subject: [PATCH 122/196] refactor: remove Verilator batch simulation option (#11) --- bbdev/api/steps/bebop/verilator/04_sim_event.step.py | 3 --- bbdev/api/steps/bebop/verilator/05_run_api.step.py | 1 - bbdev/api/tests/test_api_bebop_verilator_run.py | 2 +- bbdev/api/tests/test_api_bebop_verilator_sim.py | 2 +- bbdev/bbdev | 4 ++-- 5 files changed, 4 insertions(+), 8 deletions(-) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index 8c0b4cb65..ce0429669 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -58,7 +58,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return diff = bool(input_data.get("diff", False)) - batch = bool(input_data.get("batch", False)) if diff and input_data.get("rushB"): ctx.logger.error("--diff and --rushB cannot be used together") await check_result( @@ -212,7 +211,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"--elf={shlex.quote(binary_path)} " f"--log-dir={shlex.quote(log_dir)}" f"{' --diff' if diff else ''}" - f"{' --batch' if batch else ''}" f"{wave_arg}" f"{trace_args}" ) @@ -249,7 +247,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "task": "verilator", "backend": "rushB" if input_data.get("rushB") else "guest-elf", "diff": diff, - "batch": batch, "chip": chip, "binary": binary_path, "config": arch_config, diff --git a/bbdev/api/steps/bebop/verilator/05_run_api.step.py b/bbdev/api/steps/bebop/verilator/05_run_api.step.py index 003d5776b..ffab895b4 100644 --- a/bbdev/api/steps/bebop/verilator/05_run_api.step.py +++ b/bbdev/api/steps/bebop/verilator/05_run_api.step.py @@ -56,7 +56,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "banktrace": body.get("banktrace", False), "rushB": rushb, "diff": diff, - "batch": body.get("batch", False), "no-wave": body.get("no-wave", body.get("no_wave", False)), "jobs": body.get("jobs", 16), "from_run_workflow": True, diff --git a/bbdev/api/tests/test_api_bebop_verilator_run.py b/bbdev/api/tests/test_api_bebop_verilator_run.py index 4860a4c78..1d9a88ae4 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_run.py +++ b/bbdev/api/tests/test_api_bebop_verilator_run.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --run '--diff --batch --no-wave --jobs 16 --binary toy_im2col_k3_test-singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig'") +run_bbdev_case("bbdev bebop-verilator --run '--diff --no-wave --jobs 16 --binary toy_im2col_k3_test-singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_sim.py b/bbdev/api/tests/test_api_bebop_verilator_sim.py index 5e6768dd1..80c1e4a2e 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_sim.py +++ b/bbdev/api/tests/test_api_bebop_verilator_sim.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --sim '--binary toy_vecunit_matmul_ones-singlecore-baremetal --batch --config sims.verilator.BuckyballToyVerilatorConfig --itrace --mtrace --pmctrace --ctrace --banktrace'") +run_bbdev_case("bbdev bebop-verilator --sim '--binary toy_vecunit_matmul_ones-singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig --itrace --mtrace --pmctrace --ctrace --banktrace'") diff --git a/bbdev/bbdev b/bbdev/bbdev index 811fa607b..a68a6b1e9 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -68,8 +68,8 @@ WORKFLOW_COMMANDS = { "clean": 'Clean Bebop Verilator build directory. Args: "--config [--output-dir ]"', "verilog": 'Generate Bebop Verilog files. Args: "--config [--output-dir ]"', "build": 'Build Bebop Verilator simulation executable. Args: "--config [--diff] [--jobs ] [--output-dir ]"', - "sim": 'Run Bebop Verilator simulation. Args: "--config --binary [--rushB] [--batch] [--log-dir ] [--fst-dir ] [--no-wave]"', - "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--rushB] [--batch] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', + "sim": 'Run Bebop Verilator simulation. Args: "--config --binary [--rushB] [--log-dir ] [--fst-dir ] [--no-wave]"', + "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--rushB] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--diff] [--rushB] [--clean-before]"', }, "bebop-p2e": { From 787a6a2116c21566c19d01900701acd649d8c954 Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Thu, 20 Aug 2026 22:51:27 +0800 Subject: [PATCH 123/196] fix: resolve Verilator difftest runtime manifests (#12) --- .../steps/bebop/bemu/scripts/bemu_common.py | 23 ++--------------- .../bebop/verilator/03_build_event.step.py | 12 +++++---- .../bebop/verilator/04_sim_event.step.py | 4 ++- .../bebop/verilator/06_batch_event.step.py | 10 +++++--- bbdev/api/utils/chip.py | 25 +++++++++++++++++++ 5 files changed, 43 insertions(+), 31 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py index fb53ba895..ddda6c822 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py @@ -1,33 +1,14 @@ from pathlib import Path import tomllib +from utils.chip import resolve_chip_runtime_manifest from utils.path import get_buckyball_path def bemu_manifest(chip: str, bbdir: str | None = None) -> Path: if not chip: raise ValueError("missing required parameter: chip") - root = Path(bbdir or get_buckyball_path()) - chip_manifest = root / "examples" / "chips" / chip / "chip.toml" - if not chip_manifest.is_file(): - raise ValueError(f"chip manifest does not exist: {chip_manifest}") - - with chip_manifest.open("rb") as f: - data = tomllib.load(f) - if not isinstance(data, dict): - raise ValueError(f"TOML root must be a table: {chip_manifest}") - - runtime = data.get("runtime") - if not isinstance(runtime, dict): - raise ValueError(f"chip manifest missing [runtime]: {chip_manifest}") - rel = runtime.get("bemu") - if not isinstance(rel, str) or not rel: - raise ValueError(f"chip manifest missing runtime.bemu: {chip_manifest}") - - manifest = (chip_manifest.parent / rel).resolve() - if not manifest.is_file(): - raise ValueError(f"bemu chip manifest does not exist: {manifest}") - return manifest + return resolve_chip_runtime_manifest(bbdir or get_buckyball_path(), chip, "bemu") def bemu_core_manifest(chip: str, bbdir: str | None = None) -> Path: diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index 624231624..9d8b3c062 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -16,6 +16,7 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) +from utils.chip import resolve_chip_runtime_manifest from utils.path import get_buckyball_path, get_chip_from_config, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id @@ -78,18 +79,19 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - build_dir = os.path.join(bbdir, "examples", "chips", chip, "emu") - manifest = os.path.join(build_dir, "Cargo.toml") - if not os.path.isfile(manifest): - ctx.logger.error(f"Chip emulator manifest does not exist: {manifest}") + try: + manifest = resolve_chip_runtime_manifest(bbdir, chip, "bemu") + except ValueError as error: + ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "chip_manifest_not_found", "manifest": manifest}, + extra_fields={"error": "chip_manifest_not_found", "detail": str(error)}, trace_id=origin_tid, ) return + build_dir = str(manifest.parent) features.extend(["bemu", "difftest"]) dramsim_header = os.path.join(bbdir, "result", "include", "dramsim3.h") diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index ce0429669..26b3aa0f9 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -18,6 +18,7 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) +from utils.chip import resolve_chip_runtime_manifest from utils.path import get_buckyball_path, get_chip_from_config, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.search_workload import search_workload @@ -85,6 +86,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if diff: try: chip = chip or get_chip_from_config(bbdir, arch_config) + manifest = resolve_chip_runtime_manifest(bbdir, chip, "bemu") except ValueError as error: ctx.logger.error(str(error)) await check_result( @@ -95,7 +97,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - build_dir = os.path.join(bbdir, "examples", "chips", chip, "emu") + build_dir = str(manifest.parent) bebop_bin = os.path.join(build_dir, "target", "release", "bebop") if not os.path.isfile(bebop_bin): diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 40608bbf3..dbb643f69 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -17,6 +17,7 @@ if bebop_path not in sys.path: sys.path.insert(0, bebop_path) +from utils.chip import resolve_chip_runtime_manifest from utils.path import get_buckyball_path, get_verilator_build_dir from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id @@ -111,14 +112,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: manifest = f"{bebop_dir}/Cargo.toml" features = "verilator" if diff: - manifest = f"{bbdir}/examples/chips/{chip}/emu/Cargo.toml" - if not os.path.isfile(manifest): - ctx.logger.error(f"Chip emulator manifest does not exist: {manifest}") + try: + manifest = str(resolve_chip_runtime_manifest(bbdir, chip, "bemu")) + except ValueError as error: + ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "chip_manifest_not_found", "manifest": manifest}, + extra_fields={"error": "chip_manifest_not_found", "detail": str(error)}, trace_id=origin_tid, ) return diff --git a/bbdev/api/utils/chip.py b/bbdev/api/utils/chip.py index 514cb53b5..13e16f90e 100644 --- a/bbdev/api/utils/chip.py +++ b/bbdev/api/utils/chip.py @@ -21,6 +21,31 @@ def _load_toml(path: Path) -> dict: return data +def resolve_chip_runtime_manifest(bbdir: str, chip: str, runtime_name: str) -> Path: + """Resolve a Cargo manifest declared by a chip's ``[runtime]`` table.""" + chip_root = Path(bbdir) / "examples" / "chips" / chip + chip_manifest = chip_root / "chip.toml" + if not chip_manifest.is_file(): + raise ValueError(f"Chip manifest does not exist: {chip_manifest}") + + data = _load_toml(chip_manifest) + chip_table = data.get("chip") + if not isinstance(chip_table, dict) or chip_table.get("name") != chip: + raise ValueError(f"Chip manifest name mismatch: {chip_manifest}") + + runtime = data.get("runtime") + if not isinstance(runtime, dict): + raise ValueError(f"Chip manifest missing [runtime]: {chip_manifest}") + manifest_rel = runtime.get(runtime_name) + if not isinstance(manifest_rel, str) or not manifest_rel: + raise ValueError(f"Chip manifest missing runtime.{runtime_name}: {chip_manifest}") + + manifest = (chip_root / manifest_rel).resolve() + if not manifest.is_file(): + raise ValueError(f"Chip runtime manifest does not exist: {manifest}") + return manifest + + def _core_table(path: Path) -> dict: core = _load_toml(path).get("core") if not isinstance(core, dict): From 40b1e5ff573136cb28fe73d4aa08709d01657dc1 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 23 Aug 2026 20:57:45 +0800 Subject: [PATCH 124/196] feat: enhance Bebop and P2E workflows with new command options and async logging --- bbdev/api/config.yaml | 1 + bbdev/api/requirements.txt | 1 + .../api/steps/bebop/bemu/02_sim_event.step.py | 102 +++-- .../steps/bebop/bemu/03_batch_event.step.py | 27 +- .../steps/bebop/bemu/04_analysis_api.step.py | 76 ++++ .../bebop/bemu/04_analysis_event.step.py | 84 ++++ .../steps/bebop/bemu/scripts/bemu_analysis.py | 262 ++++++++++++ .../steps/bebop/bemu/scripts/bemu_common.py | 74 ++-- .../bebop/bemu/scripts/test_bemu_analysis.py | 120 ++++++ .../api/steps/bebop/p2e/00_clean_api.step.py | 18 +- .../steps/bebop/p2e/00_clean_event.step.py | 20 +- .../steps/bebop/p2e/01_verilog_api.step.py | 23 +- .../bebop/p2e/02_buildbitstream_api.step.py | 18 +- .../bebop/p2e/02_buildbitstream_event.step.py | 45 +- .../bebop/p2e/03_runworkload_event.step.py | 48 ++- .../steps/bebop/p2e/04_batch_event.step.py | 70 ++-- .../api/steps/bebop/p2e/scripts/nextest.toml | 4 +- bbdev/api/steps/bebop/regression_harness.py | 19 + .../bebop/verilator/01_clean_api.step.py | 19 +- .../bebop/verilator/01_clean_event.step.py | 20 +- .../bebop/verilator/02_verilog_api.step.py | 52 ++- .../bebop/verilator/03_build_api.step.py | 16 +- .../bebop/verilator/03_build_event.step.py | 71 ++-- .../steps/bebop/verilator/04_sim_api.step.py | 18 +- .../bebop/verilator/04_sim_event.step.py | 52 ++- .../steps/bebop/verilator/05_run_api.step.py | 15 +- .../bebop/verilator/06_batch_api.step.py | 25 +- .../bebop/verilator/06_batch_event.step.py | 44 +- .../bebop/verilator/scripts/build_marker.py | 40 +- bbdev/api/steps/compiler/01_build_api.step.py | 6 +- .../api/steps/compiler/01_build_event.step.py | 84 +--- bbdev/api/steps/compiler/README.md | 2 + bbdev/api/steps/dc/01_area_api.step.py | 16 +- bbdev/api/steps/dc/02_verilog_api.step.py | 14 +- bbdev/api/steps/dc/03_area_event.step.py | 69 ++- bbdev/api/steps/dc/04_power_api.step.py | 13 +- bbdev/api/steps/dc/05_sim_event.step.py | 25 +- bbdev/api/steps/dc/06_ptpx_event.step.py | 6 +- bbdev/api/steps/dc/tapeout.py | 19 +- .../firesim/00_enumeratefpgas_event.step.py | 4 +- .../firesim/01_buildbitstream_event.step.py | 4 +- .../steps/firesim/02_infrasetup_event.step.py | 4 +- .../firesim/03_runworkload_event.step.py | 4 +- bbdev/api/steps/kernel/01_build_event.step.py | 49 ++- .../mill/01_verilator_verilog_event.step.py | 124 +++--- .../steps/mill/03_yosys_verilog_event.step.py | 64 ++- .../steps/mill/04_dc_verilog_event.step.py | 90 ++-- .../06_bebop_verilator_verilog_event.step.py | 133 +++--- .../mill/07_bebop_p2e_verilog_event.step.py | 125 ++---- bbdev/api/steps/mill/verilog.py | 12 - .../regression/00_buildbitstream_api.step.py | 37 ++ .../00_buildbitstream_event.step.py | 56 +++ .../api/steps/regression/01_check_api.step.py | 48 +++ .../steps/regression/01_check_event.step.py | 70 ++++ .../02_eval_performance_api.step.py | 48 +++ .../02_eval_performance_event.step.py | 396 ++++++++++++++++++ .../regression/03_eval_area_power_api.step.py | 45 ++ .../03_eval_area_power_event.step.py | 71 ++++ bbdev/api/steps/regression/__init__.py | 0 .../api/steps/regression/scripts/accuracy.py | 91 ++++ bbdev/api/steps/regression/scripts/dc_area.py | 10 + .../steps/regression/scripts/model_layout.py | 123 ++++++ .../steps/regression/scripts/models_toml.py | 20 + .../regression/scripts/perfetto_latency.py | 34 ++ bbdev/api/steps/regression/scripts/result.py | 27 ++ bbdev/api/steps/uvm/01_build_api.step.py | 5 +- bbdev/api/steps/uvm/01_build_event.step.py | 3 +- bbdev/api/steps/uvm/02_run_api.step.py | 5 +- bbdev/api/steps/uvm/02_run_event.step.py | 33 +- bbdev/api/steps/uvm/scripts/uvm_common.py | 76 +++- bbdev/api/steps/vcs/01_clean_api.step.py | 12 +- bbdev/api/steps/vcs/02_verilog_api.step.py | 15 +- bbdev/api/steps/vcs/02_verilog_event.step.py | 39 +- bbdev/api/steps/vcs/03_build_api.step.py | 12 +- bbdev/api/steps/vcs/03_build_event.step.py | 14 +- bbdev/api/steps/vcs/04_sim_api.step.py | 15 +- bbdev/api/steps/vcs/04_sim_event.step.py | 34 +- bbdev/api/steps/vcs/05_run_api.step.py | 12 +- .../api/steps/verilator/01_clean_api.step.py | 18 +- .../steps/verilator/01_clean_event.step.py | 22 +- .../steps/verilator/02_verilog_api.step.py | 53 ++- .../api/steps/verilator/03_build_api.step.py | 15 +- .../steps/verilator/03_build_event.step.py | 46 +- bbdev/api/steps/verilator/04_sim_api.step.py | 18 +- .../api/steps/verilator/04_sim_event.step.py | 65 ++- bbdev/api/steps/verilator/05_run_api.step.py | 37 +- bbdev/api/steps/verilator/README.md | 2 +- bbdev/api/steps/verilator/dramsim3.json | 1 + bbdev/api/steps/verilator/dramsim3.txt | 78 ++++ bbdev/api/steps/workload/00_clean_api.step.py | 20 +- .../api/steps/workload/00_clean_event.step.py | 35 +- bbdev/api/steps/workload/01_buidl_api.step.py | 5 - .../api/steps/workload/01_build_event.step.py | 183 +------- bbdev/api/steps/workload/02_tohex_api.step.py | 20 +- .../api/steps/workload/02_tohex_event.step.py | 44 +- bbdev/api/steps/workload/README.md | 13 +- bbdev/api/steps/yosys/01_run_api.step.py | 20 +- bbdev/api/steps/yosys/02_verilog_api.step.py | 25 +- bbdev/api/steps/yosys/03_synth_api.step.py | 23 +- bbdev/api/steps/yosys/03_synth_event.step.py | 25 +- .../api/steps/yosys/scripts/yosys-config.yaml | 3 - .../tests/fixtures/bemu-analysis/bdb.ndjson | 4 + bbdev/api/tests/test_all_api_serial.py | 1 + .../api/tests/test_api_bebop_bemu_analysis.py | 12 + bbdev/api/tests/test_api_bebop_bemu_sim.py | 4 +- .../test_api_bebop_p2e_buildbitstream.py | 2 +- bbdev/api/tests/test_api_bebop_p2e_clean.py | 2 +- bbdev/api/tests/test_api_bebop_p2e_verilog.py | 2 +- .../tests/test_api_bebop_verilator_batch.py | 2 +- .../tests/test_api_bebop_verilator_build.py | 2 +- .../tests/test_api_bebop_verilator_clean.py | 2 +- .../api/tests/test_api_bebop_verilator_run.py | 2 +- .../api/tests/test_api_bebop_verilator_sim.py | 2 +- .../tests/test_api_bebop_verilator_verilog.py | 2 +- bbdev/api/tests/test_api_dc_verilog.py | 4 +- .../test_api_regression_buildbitstream.py | 272 ++++++++++++ bbdev/api/tests/test_api_regression_check.py | 239 +++++++++++ .../test_api_regression_eval_area_power.py | 378 +++++++++++++++++ .../test_api_regression_eval_performance.py | 111 +++++ ...t_api_regression_eval_performance_event.py | 312 ++++++++++++++ bbdev/api/tests/test_api_verilator_build.py | 2 +- bbdev/api/tests/test_api_verilator_clean.py | 2 +- bbdev/api/tests/test_api_verilator_run.py | 2 +- bbdev/api/tests/test_api_verilator_sim.py | 2 +- bbdev/api/tests/test_api_verilator_verilog.py | 2 +- bbdev/api/tests/test_api_workload_clean.py | 2 +- bbdev/api/tests/test_api_workload_tohex.py | 2 +- bbdev/api/tests/test_api_yosys_run.py | 2 +- bbdev/api/tests/test_api_yosys_synth.py | 2 +- bbdev/api/tests/test_api_yosys_verilog.py | 2 +- bbdev/api/utils/build.py | 371 ++++++++++++++++ bbdev/api/utils/chip.py | 167 ++++---- bbdev/api/utils/path.py | 226 ++++++---- bbdev/api/utils/rtl.py | 122 ++++++ bbdev/api/utils/stream_run.py | 31 ++ bbdev/api/utils/workers.py | 170 ++++++++ bbdev/bbdev | 212 ++++++---- bbdev/mcp/common.py | 123 ++++-- bbdev/mcp/tests/test_common.py | 71 +++- bbdev/mcp/tools/__init__.py | 10 + bbdev/mcp/tools/bebop_p2e_buildbitstream.py | 7 +- bbdev/mcp/tools/bebop_p2e_clean.py | 7 +- bbdev/mcp/tools/bebop_p2e_verilog.py | 7 +- bbdev/mcp/tools/bebop_verilator_batch.py | 7 +- bbdev/mcp/tools/bebop_verilator_build.py | 6 +- bbdev/mcp/tools/bebop_verilator_clean.py | 7 +- bbdev/mcp/tools/bebop_verilator_run.py | 6 +- bbdev/mcp/tools/bebop_verilator_sim.py | 6 +- bbdev/mcp/tools/bebop_verilator_verilog.py | 7 +- bbdev/mcp/tools/bemu_analysis.py | 37 ++ bbdev/mcp/tools/bemu_sim.py | 4 + bbdev/mcp/tools/compiler_build.py | 24 +- bbdev/mcp/tools/dc_area.py | 8 +- bbdev/mcp/tools/dc_power.py | 8 +- bbdev/mcp/tools/dc_verilog.py | 8 +- bbdev/mcp/tools/kernel_build.py | 6 +- bbdev/mcp/tools/regression_buildbitstream.py | 24 ++ bbdev/mcp/tools/regression_check.py | 24 ++ bbdev/mcp/tools/regression_eval_area_power.py | 24 ++ .../mcp/tools/regression_eval_performance.py | 24 ++ bbdev/mcp/tools/uvm_build.py | 7 +- bbdev/mcp/tools/verilator_build.py | 6 +- bbdev/mcp/tools/verilator_clean.py | 6 +- bbdev/mcp/tools/verilator_run.py | 6 +- bbdev/mcp/tools/verilator_sim.py | 6 +- bbdev/mcp/tools/verilator_verilog.py | 6 +- bbdev/mcp/tools/workload_clean.py | 10 +- bbdev/mcp/tools/workload_tohex.py | 10 +- bbdev/mcp/tools/yosys_run.py | 9 +- bbdev/mcp/tools/yosys_synth.py | 9 +- bbdev/mcp/tools/yosys_verilog.py | 10 +- 171 files changed, 5880 insertions(+), 1707 deletions(-) create mode 100644 bbdev/api/steps/bebop/bemu/04_analysis_api.step.py create mode 100644 bbdev/api/steps/bebop/bemu/04_analysis_event.step.py create mode 100644 bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py create mode 100644 bbdev/api/steps/bebop/bemu/scripts/test_bemu_analysis.py create mode 100644 bbdev/api/steps/bebop/regression_harness.py delete mode 100644 bbdev/api/steps/mill/verilog.py create mode 100644 bbdev/api/steps/regression/00_buildbitstream_api.step.py create mode 100644 bbdev/api/steps/regression/00_buildbitstream_event.step.py create mode 100644 bbdev/api/steps/regression/01_check_api.step.py create mode 100644 bbdev/api/steps/regression/01_check_event.step.py create mode 100644 bbdev/api/steps/regression/02_eval_performance_api.step.py create mode 100644 bbdev/api/steps/regression/02_eval_performance_event.step.py create mode 100644 bbdev/api/steps/regression/03_eval_area_power_api.step.py create mode 100644 bbdev/api/steps/regression/03_eval_area_power_event.step.py create mode 100644 bbdev/api/steps/regression/__init__.py create mode 100644 bbdev/api/steps/regression/scripts/accuracy.py create mode 100644 bbdev/api/steps/regression/scripts/dc_area.py create mode 100644 bbdev/api/steps/regression/scripts/model_layout.py create mode 100644 bbdev/api/steps/regression/scripts/models_toml.py create mode 100644 bbdev/api/steps/regression/scripts/perfetto_latency.py create mode 100644 bbdev/api/steps/regression/scripts/result.py create mode 100644 bbdev/api/steps/verilator/dramsim3.json create mode 100644 bbdev/api/steps/verilator/dramsim3.txt create mode 100644 bbdev/api/tests/fixtures/bemu-analysis/bdb.ndjson create mode 100644 bbdev/api/tests/test_api_bebop_bemu_analysis.py create mode 100644 bbdev/api/tests/test_api_regression_buildbitstream.py create mode 100644 bbdev/api/tests/test_api_regression_check.py create mode 100644 bbdev/api/tests/test_api_regression_eval_area_power.py create mode 100644 bbdev/api/tests/test_api_regression_eval_performance.py create mode 100644 bbdev/api/tests/test_api_regression_eval_performance_event.py create mode 100644 bbdev/api/utils/build.py create mode 100644 bbdev/api/utils/rtl.py create mode 100644 bbdev/api/utils/workers.py create mode 100644 bbdev/mcp/tools/bemu_analysis.py create mode 100644 bbdev/mcp/tools/regression_buildbitstream.py create mode 100644 bbdev/mcp/tools/regression_check.py create mode 100644 bbdev/mcp/tools/regression_eval_area_power.py create mode 100644 bbdev/mcp/tools/regression_eval_performance.py diff --git a/bbdev/api/config.yaml b/bbdev/api/config.yaml index 80926ff06..e2884dc20 100644 --- a/bbdev/api/config.yaml +++ b/bbdev/api/config.yaml @@ -5,6 +5,7 @@ modules: - class: modules::worker::WorkerModule config: port: 49134 + host: 0.0.0.0 # ── State Module ─────────────────────────────────────────────────────── - class: modules::state::StateModule diff --git a/bbdev/api/requirements.txt b/bbdev/api/requirements.txt index 31a1c8927..7d31babef 100644 --- a/bbdev/api/requirements.txt +++ b/bbdev/api/requirements.txt @@ -1,2 +1,3 @@ pydantic>=2.0.0 requests>=2.28.0 +protobuf>=5.0.0 diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index c71f329eb..626eb70f6 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -19,12 +19,12 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger +from utils.path import bebop_cargo_env, bebop_target_dir, get_buckyball_path, get_bemu_log_dir, workload_tests_root +from utils.stream_run import stream_run_logger_async from utils.search_workload import search_workload, search_workload_all from utils.event_common import check_result, get_origin_trace_id from utils.process_registry import cancellation_requested -from bemu_common import bemu_core_manifest, bemu_manifest, bemu_tile +from bemu_common import bemu_core_manifest, bemu_manifest, bemu_tile_index, chip_emu_manifest PERFETTO_TARGETS = { "buddy-buckyball-lenet-run": "buddy-buckyball-lenet-perfetto", @@ -68,15 +68,13 @@ def resolve_bemu_binary(bbdir: str, chip: str, binary_name: str) -> str | None: if Path(binary_name).name != binary_name: return None - build_root = f"{bbdir}/bb-tests/build/workloads/src" - output_root = f"{bbdir}/bb-tests/output/workloads/src" - for search_root in (build_root, output_root): - chip_root = f"{search_root}/CTest/chips/{chip}" - chip_binary = search_workload(chip_root, binary_name) - if chip_binary is not None: - return chip_binary + workload_root = workload_tests_root(bbdir, chip) + chip_root = f"{workload_root}/CTest/chips/{chip}" + chip_binary = search_workload(chip_root, binary_name) + if chip_binary is not None: + return chip_binary - matches = search_workload_all(output_root, binary_name) + matches = search_workload_all(workload_root, binary_name) chip_marker = f"{os.path.sep}CTest{os.path.sep}chips{os.path.sep}" non_chip_matches = [path for path in matches if chip_marker not in path] if len(non_chip_matches) == 1: @@ -93,19 +91,19 @@ def is_native_host_elf(path: str) -> bool: return header[:4] == b"\x7fELF" and header[18:20] == b"\x3e\x00" -def rushb_bemu_library(manifest: Path) -> Path: - """Return the cdylib emitted by the Core selected for a rushB run.""" +def rushb_bemu_library(manifest: Path, bbdir: str, chip: str) -> Path: + """Return the cdylib emitted by bebop-bemu for a rushB run.""" with manifest.open("rb") as source: cargo = tomllib.load(source) lib_name = cargo.get("lib", {}).get("name") if not lib_name: lib_name = cargo["package"]["name"].replace("-", "_") - return manifest.parent / "target/release" / f"lib{lib_name}.so" + return Path(bebop_target_dir(bbdir, chip)) / "release" / f"lib{lib_name}.so" -def rushb_bemu_library_dirs(manifest: Path) -> list[Path]: - """Locate native shared-library dependencies produced by the Core build.""" - build_dir = manifest.parent / "target/release/build" +def rushb_bemu_library_dirs(manifest: Path, bbdir: str, chip: str) -> list[Path]: + """Locate native shared-library dependencies produced by the bebop-bemu build.""" + build_dir = Path(bebop_target_dir(bbdir, chip)) / "release" / "build" return sorted({library.parent for library in build_dir.glob("*/out/**/libriscv.so")}) @@ -150,7 +148,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: clean_model_trace(binary_dir) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - log_dir = f"{bbdir}/log/{timestamp}-bemu-{Path(binary_path).name}" + log_dir = get_bemu_log_dir(bbdir, chip, timestamp, binary_name) os.makedirs(log_dir, exist_ok=True) if input_data.get("rushB"): @@ -170,9 +168,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - bemu_library = rushb_bemu_library(bemu_cargo_manifest) + bemu_library = rushb_bemu_library(bemu_cargo_manifest, bbdir, chip) bemu_runtime_library = Path(binary_dir) / "libbebop_bemu.so" - dependency_dirs = rushb_bemu_library_dirs(bemu_cargo_manifest) + dependency_dirs = rushb_bemu_library_dirs(bemu_cargo_manifest, bbdir, chip) build_cmd = shlex.join([ "cargo", "build", "--release", "--manifest-path", str(bemu_cargo_manifest), "--lib", ]) @@ -184,13 +182,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: inner_cmd = f"cd {shlex.quote(binary_dir)} && {build_cmd} && {copy_cmd} && {library_env} exec {shlex.quote(binary_path)}" run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" ctx.logger.info(f"Running rushB BEMU: {run_cmd}") - run_result = stream_run_logger( + run_result = await stream_run_logger_async( cmd=run_cmd, logger=ctx.logger, cwd=bbdir, stdout_prefix="rushB bemu", stderr_prefix="rushB bemu", task_scope=origin_tid, + env={**os.environ.copy(), **bebop_cargo_env(bbdir, chip)}, ) if cancellation_requested(origin_tid): return @@ -211,35 +210,68 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return # ── Run bebop bemu ──────────────────────────────────────────────────── - cargo_args = [ - "cargo", - "run", - "--release", - "--manifest-path", - str(bemu_cargo_manifest), - "--", - ] - tile = bemu_tile(chip, bbdir) - if tile: - cargo_args.extend(["--tile", str(tile), "--elf", binary_path, "--log-dir", log_dir]) + tile_index = bemu_tile_index(chip, bbdir) + chip_emu = chip_emu_manifest(chip, bbdir) + if (tile_index is not None) != bool(chip_emu): + ctx.logger.error( + f"chip {chip}: bundle bemu.chipMain and emu/Cargo.toml must both exist or both be absent" + ) + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "chip_emu_entry_mismatch", "chip": chip}, + trace_id=origin_tid, + ) + return + if chip_emu: + cargo_args = [ + "cargo", + "run", + "--release", + "--manifest-path", + str(chip_emu), + "--", + "--tile-index", + str(tile_index), + "--elf", + binary_path, + "--log-dir", + log_dir, + ] else: - cargo_args.extend(["--", "run", "bemu", "--elf", binary_path, "--log-dir", log_dir]) + cargo_args = [ + "cargo", + "run", + "--release", + "--manifest-path", + str(bemu_cargo_manifest), + "--bin", + "bebop-bemu", + "--", + "--elf", + binary_path, + "--log-dir", + log_dir, + ] if input_data.get("pk"): cargo_args.append("--pk") if input_data.get("disasm"): cargo_args.append("--disasm") if input_data.get("tool-profile"): cargo_args.append("--tool-profile") + for trace_name in ("itrace", "mtrace"): + if input_data.get(trace_name, False): + cargo_args.append(f"--{trace_name}") inner_cmd = f"cd {shlex.quote(binary_dir)} && {shlex.join(cargo_args)}" run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" ctx.logger.info(f"Running bebop bemu: {run_cmd}") - run_result = stream_run_logger( + run_result = await stream_run_logger_async( cmd=run_cmd, logger=ctx.logger, cwd=bbdir, stdout_prefix="bebop bemu", stderr_prefix="bebop bemu", task_scope=origin_tid, + env={**os.environ.copy(), **bebop_cargo_env(bbdir, chip)}, ) if cancellation_requested(origin_tid): return @@ -266,7 +298,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"--target {shlex.quote(perfetto_target)}" ) ctx.logger.info(f"Generating Perfetto trace: {perfetto_cmd}") - perfetto_result = stream_run_logger( + perfetto_result = await stream_run_logger_async( cmd=perfetto_cmd, logger=ctx.logger, cwd=bbdir, diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index afde5bb19..00e6252bb 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -22,11 +22,12 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger +from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id -from bemu_common import bemu_core_manifest +from bemu_common import bemu_manifest from regression import regression_workload_toml +from regression_harness import nextest_harness_args config = { "name": "bebop-bemu-batch", @@ -41,7 +42,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() nextest_config = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/nextest.toml" - elf_root = f"{bbdir}/bb-tests/output" chip = input_data.get("chip") if not chip: @@ -52,9 +52,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + elf_root = chip_output_root(bbdir, chip) try: - env = os.environ.copy() - bemu_cargo_manifest = bemu_core_manifest(chip, bbdir) + bemu_cargo_manifest = bemu_manifest(chip, bbdir) except ValueError as e: ctx.logger.error(str(e)) await check_result( @@ -64,6 +64,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return + env = os.environ.copy() + env.update(bebop_cargo_env(bbdir, chip)) test_type = input_data.get("test", "elf-tests") rushB = bool(input_data.get("rushB", False)) try: @@ -89,7 +91,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--tests" ) ctx.logger.info("Building bebop bemu (tests)...") - build_result = stream_run_logger( + build_result = await stream_run_logger_async( cmd=build_cmd, logger=ctx.logger, cwd=bbdir, @@ -112,19 +114,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: shutil.rmtree(bemu_cargo_manifest.parent / "test-artifacts", ignore_errors=True) ctx.logger.info("Cleaned previous bebop test artifacts") - env.update({ - "BEBOP_WORKLOAD_TOML": workload_toml, - "BEBOP_BB_TESTS_ROOT": elf_root, - }) + harness = nextest_harness_args(workload_toml, elf_root) nextest_cmd = ( f"nix develop -c cargo nextest run --manifest-path {shlex.quote(str(bemu_cargo_manifest))} " "--test test_bemu " - f"--config-file {shlex.quote(nextest_config)}" + f"--config-file {shlex.quote(nextest_config)} " + f"{harness}" ) ctx.logger.info(f"Running bebop bemu nextest: {nextest_cmd}") - ctx.logger.info(f"Environment: BEBOP_WORKLOAD_TOML={workload_toml}, BEBOP_BB_TESTS_ROOT={elf_root}") - run_result = stream_run_logger( + run_result = await stream_run_logger_async( cmd=nextest_cmd, logger=ctx.logger, cwd=bbdir, diff --git a/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py b/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py new file mode 100644 index 000000000..20ce60588 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py @@ -0,0 +1,76 @@ +import os +import sys + +from motia import ApiRequest, ApiResponse, FlowContext, api + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from utils.path import get_buckyball_path +from bemu_analysis import abs_log_dir, chip_maps + +config = { + "name": "bebop-bemu-analysis-api", + "description": "Analyze bebop bemu itrace/mtrace", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/bemu/analysis")], + "enqueues": ["bebop.bemu.analysis"], +} + +ALLOWED = {"chip", "log-dir", "log_dir", "itrace", "mtrace"} + + +def _fail(message: str, status: int = 400) -> ApiResponse: + return ApiResponse( + status=status, + body={ + "success": False, + "failure": True, + "returncode": status, + "message": message, + }, + ) + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + unknown = sorted(k for k in body if k not in ALLOWED) + if unknown: + return _fail(f"unknown parameter(s): {', '.join(unknown)}") + + chip = body.get("chip", "") + if not chip: + return _fail("chip parameter is required") + try: + chip_maps(get_buckyball_path(), chip) + except ValueError as e: + return _fail(str(e)) + + itrace = bool(body.get("itrace", False)) + mtrace = bool(body.get("mtrace", False)) + if not itrace and not mtrace: + return _fail("need --itrace and/or --mtrace") + + log_dir = body.get("log-dir", body.get("log_dir", "")) + try: + log_dir = str(abs_log_dir(log_dir)) + except ValueError as e: + return _fail(str(e)) + + await ctx.enqueue( + { + "topic": "bebop.bemu.analysis", + "data": { + "chip": chip, + "log-dir": log_dir, + "itrace": itrace, + "mtrace": mtrace, + "_trace_id": ctx.trace_id, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/bemu/04_analysis_event.step.py b/bbdev/api/steps/bebop/bemu/04_analysis_event.step.py new file mode 100644 index 000000000..e2e25be77 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/04_analysis_event.step.py @@ -0,0 +1,84 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from utils.path import get_buckyball_path +from utils.event_common import check_result, get_origin_trace_id +from bemu_analysis import abs_log_dir, analysis_dir, chip_maps, write_report + +config = { + "name": "bebop-bemu-analysis", + "description": "Analyze bebop bemu itrace/mtrace", + "flows": ["bebop"], + "triggers": [queue("bebop.bemu.analysis")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + chip = input_data.get("chip") + if not chip: + ctx.logger.error("Missing required parameter: chip") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_chip"}, + trace_id=origin_tid, + ) + return + + try: + log_path = abs_log_dir(input_data.get("log-dir") or input_data.get("log_dir")) + except ValueError as e: + ctx.logger.error(str(e)) + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "log_dir_not_absolute", "message": str(e)}, + trace_id=origin_tid, + ) + return + + itrace = bool(input_data.get("itrace", False)) + mtrace = bool(input_data.get("mtrace", False)) + try: + names, matrix, depth = chip_maps(get_buckyball_path(), chip) + text = analysis_dir( + log_path, + names=names, + matrix=matrix, + bank_depth=depth, + itrace=itrace, + mtrace=mtrace, + ) + out = write_report(log_path, text) + except ValueError as e: + ctx.logger.error(str(e)) + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "analysis_failed", "message": str(e), "log_dir": str(log_path)}, + trace_id=origin_tid, + ) + return + + for line in text.splitlines(): + ctx.logger.info(line) + await check_result( + ctx, + 0, + continue_run=False, + extra_fields={ + "task": "bemu-analysis", + "chip": chip, + "log_dir": str(log_path), + "analysis": str(out), + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py new file mode 100644 index 000000000..dcbb3c231 --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +"""Summarize bebop-bemu itrace/mtrace in bdb.ndjson.""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections import Counter +from pathlib import Path + +ISA = { + 0: "fence", + 1: "barrier", + 16: "mvout", + 32: "mset", + 33: "mvin", + 34: "mmio_set", + 35: "mvin_mmio", +} + +MATRIX_MNEMONICS = frozenset({"MATRIX", "MATRIX_F32"}) + +_API = Path(__file__).resolve().parents[4] +if str(_API) not in sys.path: + sys.path.insert(0, str(_API)) + + +def abs_log_dir(value: object) -> Path: + if not isinstance(value, str) or not value: + raise ValueError("log-dir must be an absolute path") + path = Path(value) + if not path.is_absolute(): + raise ValueError(f"log-dir must be an absolute path: {value}") + return path + + +def _hex_u(value: object, field: str, line_no: int) -> int: + if not isinstance(value, str) or not value.startswith("0x"): + raise ValueError(f"bdb.ndjson:{line_no}: {field} must be a 0x hex string") + return int(value, 16) + + +def chip_maps(bbdir: str, chip: str) -> tuple[dict[int, str], set[int], int]: + parse = Path(bbdir) / "bazel" / "configparse" + if str(parse) not in sys.path: + sys.path.insert(0, str(parse)) + from chip_json import compiler_core, core_entry, load_topology + + data = load_topology(Path(bbdir), chip) + core_name = compiler_core(data) + if not core_name: + raise ValueError(f"chip {chip} has no unique topology core") + core = core_entry(data, core_name) + ball = core.get("balldomain") + if not isinstance(ball, dict): + raise ValueError(f"chip {chip} core {core_name} has no balldomain") + isa = ball.get("ballISA") + ball_path = ball.get("_file") + if not isinstance(isa, list) or not isa: + raise ValueError(f"balldomain missing ballISA: {ball_path}") + + names: dict[int, str] = dict(ISA) + matrix: set[int] = set() + seen: set[int] = set() + for entry in isa: + if not isinstance(entry, dict): + raise ValueError(f"ballISA entry must be a table: {ball_path}") + funct = entry.get("funct7") + mnemonic = entry.get("mnemonic") + if not isinstance(funct, int) or funct < 0: + raise ValueError(f"ballISA funct7 must be a non-negative int: {ball_path}") + if not isinstance(mnemonic, str) or not mnemonic: + raise ValueError(f"ballISA mnemonic must be a non-empty string: {ball_path}") + if funct in seen: + raise ValueError(f"duplicate ballISA funct7 {funct}: {ball_path}") + if funct in ISA: + raise ValueError( + f"ballISA funct7 {funct} ({mnemonic}) collides with ISA {ISA[funct]}: {ball_path}" + ) + seen.add(funct) + names[funct] = mnemonic.lower() + if mnemonic in MATRIX_MNEMONICS: + matrix.add(funct) + + mem = core.get("memdomain") + if not isinstance(mem, dict): + raise ValueError(f"chip {chip} core {core_name} has no memdomain") + bank = mem.get("bank") + mem_path = mem.get("_file") + if not isinstance(bank, dict): + raise ValueError(f"memdomain missing [bank]: {mem_path}") + entries = bank.get("entries") + if not isinstance(entries, int) or entries <= 0: + raise ValueError(f"memdomain bank.entries must be a positive int: {mem_path}") + return names, matrix, entries + + +def _mnk(rs2: int) -> tuple[int, int, int]: + return rs2 & 0xFFF, (rs2 >> 12) & 0xFFF, (rs2 >> 24) & 0xFFF + + +def analysis_dir( + log_dir: Path, + *, + names: dict[int, str], + matrix: set[int], + bank_depth: int, + itrace: bool, + mtrace: bool, +) -> str: + if bank_depth <= 0: + raise ValueError(f"bank_depth must be a positive int, got {bank_depth}") + if not log_dir.is_dir(): + raise ValueError(f"log-dir is not a directory: {log_dir}") + bdb = log_dir / "bdb.ndjson" + if not bdb.is_file(): + raise ValueError(f"missing bdb.ndjson: {bdb}") + + n_itrace = 0 + n_mtrace = 0 + prev_clk: int | None = None + funct_n: Counter[str] = Counter() + funct_cyc: Counter[str] = Counter() + mnk_n: Counter[tuple[int, int, int]] = Counter() + rows_n: Counter[int] = Counter() + rows_sum = 0 + rows_1 = 0 + + with bdb.open(encoding="utf-8") as source: + for line_no, raw in enumerate(source, start=1): + line = raw.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"bdb.ndjson:{line_no}: invalid JSON") from exc + if not isinstance(obj, dict): + raise ValueError(f"bdb.ndjson:{line_no}: line must be a JSON object") + kind = obj.get("type") + if kind == "itrace": + n_itrace += 1 + if "clk" not in obj: + raise ValueError(f"bdb.ndjson:{line_no}: itrace missing clk") + clk = obj["clk"] + if not isinstance(clk, int) or clk < 0: + raise ValueError(f"bdb.ndjson:{line_no}: itrace clk must be a non-negative int") + if prev_clk is not None and clk < prev_clk: + raise ValueError( + f"bdb.ndjson:{line_no}: itrace clk went backwards {prev_clk} -> {clk}" + ) + funct = _hex_u(obj.get("funct"), "funct", line_no) + if funct not in names: + known = ", ".join(f"{v}=0x{k:02x}" for k, v in sorted(names.items())) + raise ValueError( + f"bdb.ndjson:{line_no}: unknown funct 0x{funct:02x}; known: {known}" + ) + name = names[funct] + delta = clk if prev_clk is None else clk - prev_clk + funct_n[name] += 1 + funct_cyc[name] += delta + prev_clk = clk + if funct in matrix: + rs2 = _hex_u(obj.get("rs2"), "rs2", line_no) + mnk_n[_mnk(rs2)] += 1 + elif kind == "mtrace": + n_mtrace += 1 + rows = obj.get("rows") + if not isinstance(rows, int) or rows < 0: + raise ValueError(f"bdb.ndjson:{line_no}: mtrace rows must be a non-negative int") + rows_n[rows] += 1 + rows_sum += rows + if rows == 1: + rows_1 += 1 + elif kind is None: + raise ValueError(f"bdb.ndjson:{line_no}: missing type") + + if itrace and n_itrace == 0: + raise ValueError(f"no itrace events in {bdb}") + if mtrace and n_mtrace == 0: + raise ValueError(f"no mtrace events in {bdb}") + + lines: list[str] = [f"log_dir: {log_dir}"] + if itrace: + total_cyc = sum(funct_cyc.values()) + if total_cyc == 0: + raise ValueError(f"itrace span is 0 cycles in {bdb}") + lines.append("== itrace ==") + lines.append(f"events: {n_itrace}") + lines.append(f"span_cycles: {total_cyc}") + lines.append(f"{'funct':<16} {'n':>10} {'cycles':>14} {'pct':>8}") + for name, count in funct_n.most_common(): + cyc = funct_cyc[name] + pct = 100.0 * cyc / total_cyc + lines.append(f"{name:<16} {count:>10} {cyc:>14} {pct:>7.1f}%") + lines.append(f"fence: {funct_n.get('fence', 0)}") + lines.append(f"mset: {funct_n.get('mset', 0)}") + if matrix: + lines.append("matrix (M,N,K):") + if not mnk_n: + lines.append(" (none)") + else: + for (m, n, k), count in mnk_n.most_common(): + lines.append(f" ({m},{n},{k}) n={count}") + if mtrace: + mean = rows_sum / n_mtrace + lines.append("== mtrace ==") + lines.append(f"events: {n_mtrace}") + lines.append(f"bank_depth: {bank_depth}") + lines.append(f"mean_rows: {mean:.2f}") + lines.append(f"mean_rows/bank_depth: {mean / bank_depth:.4f}") + lines.append(f"rows=1: {rows_1} ({100.0 * rows_1 / n_mtrace:.1f}%)") + lines.append("rows histogram:") + for rows, count in sorted(rows_n.items()): + lines.append(f" {rows}: {count}") + return "\n".join(lines) + "\n" + + +def write_report(log_dir: Path, text: str) -> Path: + out = log_dir / "analysis.txt" + out.write_text(text, encoding="utf-8") + return out + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--chip", required=True) + parser.add_argument("--log-dir", required=True, type=Path) + parser.add_argument("--bbdir", type=Path) + parser.add_argument("--itrace", action="store_true") + parser.add_argument("--mtrace", action="store_true") + args = parser.parse_args(argv) + if not args.itrace and not args.mtrace: + raise SystemExit("need --itrace and/or --mtrace") + log_dir = abs_log_dir(str(args.log_dir)) + api_root = Path(__file__).resolve().parents[4] + if str(api_root) not in sys.path: + sys.path.insert(0, str(api_root)) + from utils.path import get_buckyball_path + + bbdir = args.bbdir if args.bbdir is not None else Path(get_buckyball_path()) + names, matrix, depth = chip_maps(str(bbdir), args.chip) + text = analysis_dir( + log_dir, + names=names, + matrix=matrix, + bank_depth=depth, + itrace=args.itrace, + mtrace=args.mtrace, + ) + write_report(log_dir, text) + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except ValueError as exc: + print(exc, file=sys.stderr) + raise SystemExit(1) diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py index ddda6c822..962d96b5c 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py @@ -1,33 +1,57 @@ +import json from pathlib import Path -import tomllib -from utils.chip import resolve_chip_runtime_manifest -from utils.path import get_buckyball_path +from utils.build import install_bundle +from utils.chip import require_chip + + +def _repo(bbdir: str | None) -> Path: + from utils.path import get_buckyball_path + + return Path(bbdir or get_buckyball_path()) + + +def _chip_json(chip: str, bbdir: str | None = None) -> dict: + root = _repo(bbdir) + path = root / "examples" / "chips" / chip / "generated" / "chip.json" + if not path.is_file(): + install_bundle(str(root), chip) + if not path.is_file(): + raise ValueError(f"missing {path} after bundle build") + return json.loads(path.read_text(encoding="utf-8")) def bemu_manifest(chip: str, bbdir: str | None = None) -> Path: - if not chip: - raise ValueError("missing required parameter: chip") - return resolve_chip_runtime_manifest(bbdir or get_buckyball_path(), chip, "bemu") + chip = require_chip({"chip": chip}) + root = _repo(bbdir) + install_bundle(str(root), chip) + manifest = root / "examples" / "chips" / chip / "generated" / "bemu" / "Cargo.toml" + if not manifest.is_file(): + raise ValueError(f"missing {manifest}; bundle install failed") + return manifest def bemu_core_manifest(chip: str, bbdir: str | None = None) -> Path: - root = Path(bbdir or get_buckyball_path()) - chip_manifest = root / "examples" / "chips" / chip / "chip.toml" - with chip_manifest.open("rb") as f: - data = tomllib.load(f) - runtime = data.get("runtime", {}) - rel = runtime.get("bemuCore") or runtime.get("bemu") - manifest = (chip_manifest.parent / rel).resolve() - if not manifest.is_file(): - raise ValueError(f"BEMU Core manifest does not exist: {manifest}") - return manifest - - -def bemu_tile(chip: str, bbdir: str | None = None) -> Path | None: - root = Path(bbdir or get_buckyball_path()) - chip_manifest = root / "examples" / "chips" / chip / "chip.toml" - with chip_manifest.open("rb") as f: - data = tomllib.load(f) - rel = data.get("runtime", {}).get("bemuTile") - return (chip_manifest.parent / rel).resolve() if rel else None + return bemu_manifest(chip, bbdir) + + +def chip_emu_manifest(chip: str, bbdir: str | None = None) -> Path | None: + chip = require_chip({"chip": chip}) + root = _repo(bbdir) + main = _chip_json(chip, bbdir).get("bemu", {}).get("chipMain", "") + if not main: + return None + manifest = root / "examples" / "chips" / chip / "emu" / "Cargo.toml" + if not manifest.is_file(): + raise ValueError(f"missing chip emu manifest: {manifest}") + return manifest + + +def bemu_tile_index(chip: str, bbdir: str | None = None) -> int | None: + bemu = _chip_json(chip, bbdir).get("bemu", {}) + if not bemu.get("chipMain", ""): + return None + index = bemu.get("tileIndex", 0) + if not isinstance(index, int) or index < 0: + raise ValueError(f"chip {chip}: bemu.tileIndex must be a non-negative int") + return index diff --git a/bbdev/api/steps/bebop/bemu/scripts/test_bemu_analysis.py b/bbdev/api/steps/bebop/bemu/scripts/test_bemu_analysis.py new file mode 100644 index 000000000..eaf6c2c3e --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/scripts/test_bemu_analysis.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +import tempfile +import unittest +from pathlib import Path + +from bemu_analysis import abs_log_dir, analysis_dir, chip_maps + +BBDIR = Path(__file__).resolve().parents[6] + + +def _write_bdb(text: str) -> Path: + tmp = Path(tempfile.mkdtemp()) + (tmp / "bdb.ndjson").write_text(text, encoding="utf-8") + return tmp + + +class AbsLogDirTest(unittest.TestCase): + def test_relative_fails(self): + with self.assertRaisesRegex(ValueError, "absolute path"): + abs_log_dir("log/2026-08-19-13-00-bemu-buddy-buckyball-yolo26-run") + + def test_absolute_ok(self): + path = abs_log_dir("/tmp/bemu-log") + self.assertTrue(path.is_absolute()) + + +class ChipMapsTest(unittest.TestCase): + def test_pebble_loads_ballisa(self): + names, matrix, depth = chip_maps(str(BBDIR), "pebble") + self.assertEqual(names[0], "fence") + self.assertEqual(names[32], "mset") + self.assertEqual(names[65], "smatmul") + self.assertEqual(names[48], "im2col") + self.assertEqual(names[49], "transpose") + self.assertEqual(names[51], "fp2int") + self.assertEqual(names[52], "int2fp") + self.assertEqual(matrix, {65}) + self.assertEqual(depth, 1024) + + +class AnalysisDirTest(unittest.TestCase): + def setUp(self): + self.names, self.matrix, self.depth = chip_maps(str(BBDIR), "pebble") + + def test_itrace_mtrace(self): + log_dir = _write_bdb( + "\n".join( + [ + '{"type":"itrace","clk":10,"event":"complete","funct":"0x00","pc":"0x0","rs1":"0x0","rs2":"0x0"}', + '{"type":"itrace","clk":20,"event":"complete","funct":"0x41","pc":"0x0","rs1":"0x0","rs2":"0x0000000100010400"}', + '{"type":"mtrace","clk":15,"event":"read","addr":"0x0","rows":1024,"line_bytes":16,"row_stride":16,"vbank_id":0,"pbank_id":0,"group_id":0}', + '{"type":"mtrace","clk":16,"event":"write","addr":"0x0","rows":1,"line_bytes":16,"row_stride":16,"vbank_id":1,"pbank_id":1,"group_id":0}', + "", + ] + ) + ) + text = analysis_dir( + log_dir, + names=self.names, + matrix=self.matrix, + bank_depth=self.depth, + itrace=True, + mtrace=True, + ) + self.assertIn("span_cycles: 20", text) + self.assertIn("smatmul", text) + self.assertIn("(1024,16,256) n=1", text) + self.assertIn(f"bank_depth: {self.depth}", text) + self.assertIn("mean_rows/bank_depth:", text) + self.assertNotIn("mean_rows/1024:", text) + self.assertIn("rows=1: 1 (50.0%)", text) + + def test_bank_depth_from_arg(self): + log_dir = _write_bdb( + '{"type":"mtrace","clk":1,"event":"read","addr":"0x0","rows":32,"line_bytes":16,"row_stride":16,"vbank_id":0,"pbank_id":0,"group_id":0}\n' + ) + text = analysis_dir( + log_dir, + names=self.names, + matrix=self.matrix, + bank_depth=64, + itrace=False, + mtrace=True, + ) + self.assertIn("bank_depth: 64", text) + self.assertIn("mean_rows: 32.00", text) + self.assertIn("mean_rows/bank_depth: 0.5000", text) + self.assertNotIn("mean_rows/1024", text) + + def test_unknown_funct_fails(self): + log_dir = _write_bdb( + '{"type":"itrace","clk":1,"event":"complete","funct":"0x99","pc":"0x0","rs1":"0x0","rs2":"0x0"}\n' + ) + with self.assertRaisesRegex(ValueError, "unknown funct 0x99"): + analysis_dir( + log_dir, + names=self.names, + matrix=self.matrix, + bank_depth=self.depth, + itrace=True, + mtrace=False, + ) + + def test_missing_itrace_fails(self): + log_dir = _write_bdb( + '{"type":"mtrace","clk":1,"event":"read","addr":"0x0","rows":1,"line_bytes":16,"row_stride":16,"vbank_id":0,"pbank_id":0,"group_id":0}\n' + ) + with self.assertRaisesRegex(ValueError, "no itrace events"): + analysis_dir( + log_dir, + names=self.names, + matrix=self.matrix, + bank_depth=self.depth, + itrace=True, + mtrace=False, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/bbdev/api/steps/bebop/p2e/00_clean_api.step.py b/bbdev/api/steps/bebop/p2e/00_clean_api.step.py index 5f34d2a4d..847d385cb 100644 --- a/bbdev/api/steps/bebop/p2e/00_clean_api.step.py +++ b/bbdev/api/steps/bebop/p2e/00_clean_api.step.py @@ -1,5 +1,8 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.chip import require_chip +from utils.path import get_buckyball_path + config = { "name": "bebop-p2e-clean-api", "description": "Clean P2E build directory", @@ -11,11 +14,12 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - config_name = body.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --config must be specified"}, - ) - await ctx.enqueue({"topic": "bebop.p2e.clean", "data": {**body, "task": "clean", "_trace_id": ctx.trace_id}}) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + await ctx.enqueue({ + "topic": "bebop.p2e.clean", + "data": {**body, "chip": chip, "task": "clean", "_trace_id": ctx.trace_id}, + }) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/00_clean_event.step.py b/bbdev/api/steps/bebop/p2e/00_clean_event.step.py index 453942382..ba3141705 100644 --- a/bbdev/api/steps/bebop/p2e/00_clean_event.step.py +++ b/bbdev/api/steps/bebop/p2e/00_clean_event.step.py @@ -7,8 +7,9 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path, get_verilator_build_dir -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.path import get_buckyball_path, rtl_dir_for_clean +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id config = { @@ -23,19 +24,20 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - config_name = input_data.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - ctx.logger.error("Missing required parameter: config") + try: + chip = require_chip(input_data) + except ValueError as error: + ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_config"}, + extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) return - build_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("output_dir")) + build_dir = rtl_dir_for_clean(bbdir, chip, "p2e", input_data.get("output_dir")) command = f"rm -rf {build_dir}" - result = stream_run_logger( + result = await stream_run_logger_async( cmd=command, logger=ctx.logger, cwd=bbdir, @@ -47,6 +49,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, result.returncode, continue_run=False, - extra_fields={"task": "clean", "config": config_name, "output_dir": build_dir}, + extra_fields={"task": "clean", "output_dir": build_dir}, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py index c680a9eb2..c3b9a2483 100644 --- a/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py +++ b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py @@ -1,7 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.chip import require_chip from utils.path import get_buckyball_path -from utils.path import get_verilator_build_dir config = { "name": "bebop-p2e-verilog-api", @@ -13,20 +13,13 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = req.body or {} - - config_name = body.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --config must be specified"}, - ) - output_dir = get_verilator_build_dir(bbdir, config_name, body.get("output_dir")) - - data = { - "config": config_name, - "output_dir": output_dir, - } + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + data = {"chip": chip} + if body.get("output_dir"): + data["output_dir"] = body["output_dir"] await ctx.enqueue({"topic": "bebop.p2e.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py index 7b00fabe9..cb9371083 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py @@ -1,6 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.chip import require_chip +from utils.path import get_buckyball_path, get_p2e_build_dir config = { "name": "bebop-p2e-buildbitstream-api", @@ -14,17 +15,14 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = request.body or {} - - config_name = body.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --config must be specified"}, - ) - vsrc_dir = get_verilator_build_dir(bbdir, config_name, body.get("vsrc_dir")) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + vsrc_dir = get_p2e_build_dir(bbdir, chip, body.get("vsrc_dir")) data = { - "config": config_name, + "chip": chip, "vsrc_dir": vsrc_dir, "output_dir": body.get("output_dir"), } diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index 23648ec4a..0976edabe 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -16,8 +16,9 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path, get_verilator_build_dir -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.path import bebop_cargo_env, get_buckyball_path, get_p2e_build_dir +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id config = { @@ -34,16 +35,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" - config_name = input_data.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - ctx.logger.error("Missing required parameter: config") + try: + chip = require_chip(input_data) + except ValueError as error: + ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_config"}, + extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) return - vsrc_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("vsrc_dir")) + vsrc_dir = get_p2e_build_dir(bbdir, chip, input_data.get("vsrc_dir")) if not os.path.isdir(vsrc_dir): ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}") await check_result( @@ -59,7 +61,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: or input_data.get("build-dir") or input_data.get("output_dir") or input_data.get("output-dir") - or f"{bebop_dir}/build/{config_name}-{timestamp}" + or f"{bebop_dir}/build/{chip}-{timestamp}" ) os.makedirs(build_dir, exist_ok=True) @@ -70,12 +72,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"--out-dir=\"{build_dir}\"" ) ctx.logger.info("Building bebop p2e runtime case ...") - build_result = stream_run_logger( + build_result = await stream_run_logger_async( cmd=build_cmd, logger=ctx.logger, cwd=bebop_dir, stdout_prefix="bebop p2e build", stderr_prefix="bebop p2e build", + env={**os.environ, **bebop_cargo_env(bbdir, chip)}, ) rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") @@ -95,7 +98,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "build", - "config": config_name, "vsrc_dir": vsrc_dir, "build_dir": build_dir, "missing": missing, @@ -106,19 +108,22 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return + extra_fields = { + "task": "build", + "vsrc_dir": vsrc_dir, + "build_dir": build_dir, + "rtcfg": rtcfg_path, + "libvCtb": libvctb_path, + "bitstream": bitstream_path, + "timestamp": timestamp, + } + if input_data.get("from_regression_buildbitstream") and build_result.returncode == 0: + extra_fields["bitstream"] = os.path.abspath(bitstream_path) + await check_result( ctx, build_result.returncode, continue_run=False, - extra_fields={ - "task": "build", - "config": config_name, - "vsrc_dir": vsrc_dir, - "build_dir": build_dir, - "rtcfg": rtcfg_path, - "libvCtb": libvctb_path, - "bitstream": bitstream_path, - "timestamp": timestamp, - }, + extra_fields=extra_fields, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index fd784ec35..679894edc 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -18,8 +18,9 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path, get_verilator_build_dir -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path, get_p2e_build_dir, get_run_log_dir +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id config = { @@ -31,12 +32,16 @@ } -def resolve_image(bbdir: str, image_name: str) -> str: - """Search bb-tests/output/ recursively for .hex and return its absolute path.""" - output_root = f"{bbdir}/bb-tests/output" - matches = glob.glob(f"{output_root}/**/{image_name}.hex", recursive=True) +def resolve_image(bbdir: str, image_name: str, chip: str) -> str: + """Search chip workload build output recursively for .hex.""" + workload_root = os.path.join(chip_output_root(bbdir, chip), "workloads") + matches = glob.glob(f"{workload_root}/**/{image_name}.hex", recursive=True) if not matches: return "" + if len(matches) > 1: + raise ValueError( + f"multiple .hex files for {image_name!r} under {workload_root}: {matches}" + ) return matches[0] @@ -57,6 +62,16 @@ def case_uses_multi_fpga(build_dir: str) -> bool: async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) + try: + chip = require_chip(input_data) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_chip"}, + trace_id=origin_tid, + ) + return bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" @@ -95,9 +110,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return wave = True - image_path = resolve_image(bbdir, image_name) + image_path = resolve_image(bbdir, image_name, chip) if not image_path: - ctx.logger.error(f"image .hex not found for name: {image_name} (searched bb-tests/output/)") + ctx.logger.error( + f"image .hex not found for name: {image_name} " + f"(searched bb-tests/output/{chip}/workloads/)" + ) await check_result( ctx, 1, continue_run=False, extra_fields={"error": "image_not_found", "image": image_name}, @@ -129,15 +147,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: multi_fpga = True ctx.logger.info(f"Detected multi-FPGA P2E case: {build_dir}") - config_name = resolve_runtime_config(bitstream, input_data.get("config")) - vsrc_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("vsrc_dir")) + vsrc_dir = get_p2e_build_dir(bbdir, chip, input_data.get("vsrc_dir")) if not os.path.isdir(vsrc_dir): ctx.logger.error(f"VSRC_PATH does not exist for P2E runtime: {vsrc_dir}") await check_result( ctx, 1, continue_run=False, extra_fields={ "error": "vsrc_not_found", - "config": config_name, "vsrc_dir": vsrc_dir, }, trace_id=origin_tid, @@ -145,7 +161,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - log_dir = f"{bbdir}/log/{timestamp}-p2e-{image_name}" + log_dir = get_run_log_dir(bbdir, chip, "p2e", timestamp, "p2e", image_name, input_data.get("vsrc_dir")) os.makedirs(log_dir, exist_ok=True) # Rebuild the VVAC host runtime in the bitstream case. The bitstream is @@ -157,12 +173,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"--out-dir=\"{build_dir}\"" ) ctx.logger.info("Preparing bebop p2e runtime for the selected bitstream ...") - runtime_result = stream_run_logger( + runtime_result = await stream_run_logger_async( cmd=runtime_cmd, logger=ctx.logger, cwd=bebop_dir, stdout_prefix="bebop p2e runtime", stderr_prefix="bebop p2e runtime", + env={**os.environ.copy(), **bebop_cargo_env(bbdir, chip)}, ) rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") libvctb_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so") @@ -176,7 +193,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "runtime", - "config": config_name, "vsrc_dir": vsrc_dir, "build_dir": build_dir, "missing": missing, @@ -204,12 +220,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if input_data.get(trace_name, False): run_cmd += f" --{trace_name}" ctx.logger.info(f"Running bebop p2e runworkload: {run_cmd}") - run_result = stream_run_logger( + run_result = await stream_run_logger_async( cmd=run_cmd, logger=ctx.logger, cwd=bebop_dir, stdout_prefix="bebop p2e runworkload", stderr_prefix="bebop p2e runworkload", + env={**os.environ.copy(), **bebop_cargo_env(bbdir, chip)}, ) await check_result( @@ -220,7 +237,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "task": "runworkload", "image": image_path, "bitstream": bitstream, - "config": config_name, "build_dir": build_dir, "log_dir": log_dir, "bdb_trace": os.path.join(log_dir, "bdb.ndjson"), diff --git a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py index bdb1cc2f5..f60df7753 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py @@ -19,10 +19,12 @@ if bebop_path not in sys.path: sys.path.insert(0, bebop_path) -from utils.path import get_buckyball_path, get_verilator_build_dir -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path, get_p2e_build_dir +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from regression import regression_workload_toml +from regression_harness import nextest_harness_args config = { "name": "bebop-p2e-batch", @@ -47,7 +49,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" nextest_config = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/nextest.toml" - elf_root = f"{bbdir}/bb-tests/output" bitstream = input_data.get("bitstream", "") if not bitstream or not os.path.isfile(bitstream): @@ -80,6 +81,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + elf_root = chip_output_root(bbdir, chip) + cargo_env = bebop_cargo_env(bbdir, chip) test_type = input_data.get("test", "elf-tests") try: @@ -96,15 +99,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"Running {test_type} with workload config: {workload_toml}") ctx.logger.info(f"P2E case dir (from bitstream): {build_dir}") - config_name = resolve_runtime_config(bitstream, input_data.get("config")) - vsrc_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("vsrc_dir")) + vsrc_dir = get_p2e_build_dir(bbdir, chip, input_data.get("vsrc_dir")) if not os.path.isdir(vsrc_dir): ctx.logger.error(f"VSRC_PATH does not exist for P2E runtime: {vsrc_dir}") await check_result( ctx, 1, continue_run=False, extra_fields={ "error": "vsrc_not_found", - "config": config_name, "vsrc_dir": vsrc_dir, }, trace_id=origin_tid, @@ -119,12 +120,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"--out-dir=\"{build_dir}\"" ) ctx.logger.info("Preparing bebop p2e runtime for the selected bitstream ...") - runtime_result = stream_run_logger( + runtime_result = await stream_run_logger_async( cmd=runtime_cmd, logger=ctx.logger, cwd=bebop_dir, stdout_prefix="bebop p2e runtime", stderr_prefix="bebop p2e runtime", + env={**os.environ.copy(), **cargo_env}, ) rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") libvctb_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so") @@ -138,7 +140,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "runtime", - "config": config_name, "vsrc_dir": vsrc_dir, "build_dir": build_dir, "missing": missing, @@ -155,12 +156,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"nix develop -c cargo {cargo_out} build --release --features p2e --tests" ) ctx.logger.info("Building bebop p2e (tests)...") - build_result = stream_run_logger( + build_result = await stream_run_logger_async( cmd=build_cmd, logger=ctx.logger, cwd=bebop_dir, stdout_prefix="bebop p2e build", stderr_prefix="bebop p2e build", + env={**os.environ.copy(), **cargo_env}, ) if build_result.returncode != 0: @@ -173,23 +175,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── Run nextest ─────────────────────────────────────────────────────── env = os.environ.copy() - env.update({ - "BEBOP_WORKLOAD_TOML": workload_toml, - "BEBOP_BB_TESTS_ROOT": elf_root, - "BEBOP_P2E_BITSTREAM": bitstream, - "OUT_PATH": build_dir, - }) + env.update(cargo_env) + env["OUT_PATH"] = build_dir + harness = nextest_harness_args(workload_toml, elf_root, p2e_bitstream=bitstream) nextest_cmd = ( f"nix develop -c cargo {cargo_out} nextest run --release --features p2e " - f"--test test_p2e --config-file \"{nextest_config}\"" + f"--test test_p2e --config-file \"{nextest_config}\" {harness}" ) ctx.logger.info(f"Running bebop p2e nextest: {nextest_cmd}") - ctx.logger.info( - f"Environment: BEBOP_WORKLOAD_TOML={workload_toml}, " - f"BEBOP_BB_TESTS_ROOT={elf_root}, OUT_PATH={build_dir}" - ) - run_result = stream_run_logger( + run_result = await stream_run_logger_async( cmd=nextest_cmd, logger=ctx.logger, cwd=bebop_dir, @@ -198,20 +193,31 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: env=env, ) + extra_fields = { + "task": "batch", + "backend": "p2e", + "chip": chip, + "bitstream": bitstream, + "build_dir": build_dir, + "test_type": test_type, + "nextest_config": nextest_config, + "workload_toml": workload_toml, + } + + if input_data.get("from_regression_check"): + # Model accuracy is filled by eval-performance, not pk pass rate. + if run_result.returncode != 0: + await check_result( + ctx, run_result.returncode, continue_run=False, + extra_fields={**extra_fields, "error": "pk_tests_failed"}, + trace_id=origin_tid, + ) + return + await check_result( ctx, run_result.returncode, continue_run=False, - extra_fields={ - "task": "batch", - "backend": "p2e", - "chip": chip, - "bitstream": bitstream, - "config": config_name, - "build_dir": build_dir, - "test_type": test_type, - "nextest_config": nextest_config, - "workload_toml": workload_toml, - }, + extra_fields=extra_fields, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/bebop/p2e/scripts/nextest.toml b/bbdev/api/steps/bebop/p2e/scripts/nextest.toml index cc7bd5fa9..13f1966ca 100644 --- a/bbdev/api/steps/bebop/p2e/scripts/nextest.toml +++ b/bbdev/api/steps/bebop/p2e/scripts/nextest.toml @@ -10,7 +10,7 @@ failure-output = "immediate" success-output = "never" status-level = "all" fail-fast = false -slow-timeout = { period = "360s", terminate-after = 5 } +slow-timeout = { period = "600s", terminate-after = 18 } retries = 0 junit = { path = "target/nextest/junit-bbdev-p2e.xml", report-name = "bbdev-p2e-batch" } @@ -20,4 +20,4 @@ max-threads = 1 [[profile.default.overrides]] filter = "binary(test_p2e)" test-group = "p2e-serial" -slow-timeout = { period = "360s", terminate-after = 5 } +slow-timeout = { period = "600s", terminate-after = 18 } diff --git a/bbdev/api/steps/bebop/regression_harness.py b/bbdev/api/steps/bebop/regression_harness.py new file mode 100644 index 000000000..3f52a8718 --- /dev/null +++ b/bbdev/api/steps/bebop/regression_harness.py @@ -0,0 +1,19 @@ +import shlex + + +def nextest_harness_args( + workload_toml: str, + bb_tests_root: str, + *, + p2e_bitstream: str | None = None, +) -> str: + args = [ + "--", + "--workload-toml", + shlex.quote(workload_toml), + "--bb-tests-root", + shlex.quote(bb_tests_root), + ] + if p2e_bitstream: + args.extend(["--p2e-bitstream", shlex.quote(p2e_bitstream)]) + return " ".join(args) diff --git a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py index 0ff94e3ab..4f98d1cee 100644 --- a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py +++ b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py @@ -1,6 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.chip import require_chip +from utils.path import get_buckyball_path, rtl_dir_for_clean config = { "name": "bebop-verilator-clean-api", @@ -14,18 +15,14 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = req.body or {} + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) - config_name = body.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --config must be specified"} - ) - - build_dir = get_verilator_build_dir(bbdir, config_name, body.get("output_dir")) - + build_dir = rtl_dir_for_clean(bbdir, chip, "verilog", body.get("output_dir")) data = { - "config": config_name, + "chip": chip, "output_dir": build_dir, } await ctx.enqueue({"topic": "bebop.verilator.clean", "data": {**data, "_trace_id": ctx.trace_id}}) diff --git a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py index 85989037b..26d42c559 100644 --- a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py +++ b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py @@ -7,8 +7,9 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path, get_verilator_build_dir -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.path import get_buckyball_path, rtl_dir_for_clean +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id config = { @@ -26,19 +27,20 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - config_name = input_data.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - ctx.logger.error("Missing required parameter: config") + try: + chip = require_chip(input_data) + except ValueError as error: + ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_config"}, + extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) return - build_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("output_dir")) + build_dir = rtl_dir_for_clean(bbdir, chip, "verilog", input_data.get("output_dir")) command = f"rm -rf {build_dir}" - result = stream_run_logger( + result = await stream_run_logger_async( cmd=command, logger=ctx.logger, cwd=bbdir, @@ -50,7 +52,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, result.returncode, continue_run=input_data.get("from_run_workflow", False), - extra_fields={"task": "clean", "config": config_name, "output_dir": build_dir}, + extra_fields={"task": "clean", "output_dir": build_dir}, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py index aaac2420e..17f4c20cf 100644 --- a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py +++ b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.chip import require_chip config = { "name": "bebop-verilator-verilog-api", @@ -12,24 +12,40 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = request.body or {} - config_name = body.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse( - status=400, - body={ - "status": "error", - "message": "Configuration name is required. Please specify --config parameter.", - "example": 'bbdev bebop verilator --verilog "--config sims.verilator.BuckyballToyVerilatorConfig"', - }, - ) - - data = { - "config": config_name, - "balltype": body.get("balltype"), - "output_dir": get_verilator_build_dir(bbdir, config_name, body.get("output_dir")), - } + if body.get("balltype"): + if body.get("chip") or ( + isinstance(body.get("config"), str) and body.get("config") and body.get("config") != "None" + ): + return ApiResponse( + status=400, + body={"error": "balltype mill does not take --chip or --config"}, + ) + output_dir = body.get("output_dir") + if not output_dir: + return ApiResponse( + status=400, + body={"error": "balltype mill requires output_dir"}, + ) + data = { + "balltype": body.get("balltype"), + "output_dir": output_dir, + } + else: + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse( + status=400, + body={ + "status": "error", + "message": str(e), + "example": 'bbdev bebop verilator --verilog "--chip toy"', + }, + ) + data = {"chip": chip, "balltype": body.get("balltype")} + if body.get("output_dir"): + data["output_dir"] = body["output_dir"] await ctx.enqueue({"topic": "bebop.verilator.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_api.step.py b/bbdev/api/steps/bebop/verilator/03_build_api.step.py index a6b1d3f57..132d603bf 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_api.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_api.step.py @@ -1,5 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.chip import require_chip from utils.path import get_buckyball_path, get_verilator_build_dir config = { @@ -22,20 +23,17 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body={"error": f"Unsupported parameter(s): {', '.join(unsupported)}. Use --vsrc-dir/--output-dir."}, ) - arch_config = body.get("config") - if not isinstance(arch_config, str) or not arch_config or arch_config == "None": - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --config must be specified"} - ) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) data = { - "config": arch_config, + "chip": chip, "jobs": body.get("jobs", 16), "diff": bool(body.get("diff", False)), "vsrc_dir": get_verilator_build_dir( - bbdir, - arch_config, + bbdir, chip, body.get("vsrc-dir") or body.get("output-dir"), ), } diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index 9d8b3c062..4984fe3e8 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -6,6 +6,7 @@ import os import shlex import sys +from pathlib import Path from motia import FlowContext, queue @@ -16,9 +17,10 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.chip import resolve_chip_runtime_manifest -from utils.path import get_buckyball_path, get_chip_from_config, get_verilator_build_dir -from utils.stream_run import stream_run_logger +from utils.build import install_bundle +from utils.chip import bebop_cargo, require_chip +from utils.path import bebop_cargo_env, bebop_target_dir, get_buckyball_path, get_verilator_build_dir +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from build_marker import write_build_marker @@ -35,17 +37,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" - arch_config = input_data.get("config") - if not isinstance(arch_config, str) or not arch_config or arch_config == "None": - ctx.logger.error("Missing required parameter: config must be specified") + try: + chip = require_chip(input_data) + except ValueError as error: + ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_config"}, + extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) return - vsrc_dir = get_verilator_build_dir(bbdir, arch_config, input_data.get("vsrc_dir")) + vsrc_dir = get_verilator_build_dir(bbdir, chip, input_data.get("vsrc_dir")) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") if not os.path.isdir(vsrc_dir): @@ -63,24 +66,22 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: diff = bool(input_data.get("diff", False)) build_dir = bebop_dir - chip = None + manifest = bebop_cargo(bbdir) + chip = input_data.get("chip") features = ["verilator"] if diff: - try: - chip = get_chip_from_config(bbdir, arch_config) - except ValueError as error: - ctx.logger.error(str(error)) + if not isinstance(chip, str) or not chip: + ctx.logger.error("Missing required parameter: --chip (required with --diff)") await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "chip_resolution_failed", "detail": str(error)}, + extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) return - try: - manifest = resolve_chip_runtime_manifest(bbdir, chip, "bemu") + install_bundle(bbdir, chip) except ValueError as error: ctx.logger.error(str(error)) await check_result( @@ -91,13 +92,22 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - build_dir = str(manifest.parent) + manifest = Path(bbdir) / "examples" / "chips" / chip / "generated" / "bebop" / "Cargo.toml" + build_dir = manifest.parent + if not manifest.is_file(): + ctx.logger.error(f"missing verilator bebop shim: {manifest}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "bebop_shim_not_found", "manifest": str(manifest)}, + trace_id=origin_tid, + ) + return features.extend(["bemu", "difftest"]) dramsim_header = os.path.join(bbdir, "result", "include", "dramsim3.h") if not os.path.isfile(dramsim_header): ctx.logger.info(f"Nix environment is missing {dramsim_header}; rebuilding result") - env_result = stream_run_logger( + env_result = await stream_run_logger_async( cmd="nix build", logger=ctx.logger, cwd=bbdir, @@ -124,6 +134,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "cargo", "build", "--release", + "--manifest-path", + str(manifest), "--bin", "bebop", "--features", @@ -132,15 +144,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: str(jobs), ] ) - command_cwd = build_dir - build_cmd = cargo_build_cmd - if diff: - build_inner = f"cd {shlex.quote(build_dir)} && exec {cargo_build_cmd}" - build_cmd = f"nix develop -c sh -c {shlex.quote(build_inner)}" - command_cwd = bbdir - env = {**os.environ, "VSRC_PATH": vsrc_dir} + build_inner = ( + f"cd {shlex.quote(build_dir)} && " + f"exec {cargo_build_cmd}" + ) + build_cmd = f"nix develop -c sh -c {shlex.quote(build_inner)}" + command_cwd = bbdir + env = {**os.environ, "VSRC_PATH": vsrc_dir, **bebop_cargo_env(bbdir, chip)} ctx.logger.info(f"Building bebop verilator (diff={diff}) ...") - build_result = stream_run_logger( + build_result = await stream_run_logger_async( cmd=build_cmd, logger=ctx.logger, cwd=command_cwd, @@ -149,10 +161,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: env=env, ) - bebop_bin = os.path.join(build_dir, "target", "release", "bebop") + bebop_bin = os.path.join(bebop_target_dir(bbdir, chip), "release", "bebop") + target_dir = bebop_target_dir(bbdir, chip) if build_result.returncode == 0: try: - write_build_marker(build_dir, arch_config, vsrc_dir, bebop_bin, diff=diff) + write_build_marker(target_dir, chip, vsrc_dir, bebop_bin, diff=diff) except OSError as e: ctx.logger.error(f"failed to write bebop verilator build marker: {e}") await check_result( @@ -161,14 +174,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - await check_result( ctx, build_result.returncode, continue_run=input_data.get("from_run_workflow", False), extra_fields={ "task": "build", - "config": arch_config, "vsrc_dir": vsrc_dir, "binary": bebop_bin, "diff": diff, diff --git a/bbdev/api/steps/bebop/verilator/04_sim_api.step.py b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py index 2e72a80b4..b9a2fef5e 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_api.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py @@ -1,5 +1,8 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.chip import require_chip +from utils.path import get_buckyball_path + config = { "name": "bebop-verilator-sim-api", "description": "Run bebop verilator simulation", @@ -22,12 +25,13 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "message": "binary parameter is required", }, ) - config_name = body.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --config must be specified"}, - ) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) - await ctx.enqueue({"topic": "bebop.verilator.sim", "data": {**body, "_trace_id": ctx.trace_id}}) + await ctx.enqueue({ + "topic": "bebop.verilator.sim", + "data": {**body, "chip": chip, "_trace_id": ctx.trace_id}, + }) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index 26b3aa0f9..b6ee22572 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -18,9 +18,10 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.chip import resolve_chip_runtime_manifest -from utils.path import get_buckyball_path, get_chip_from_config, get_verilator_build_dir -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.build import install_bundle +from utils.path import bebop_target_dir, get_buckyball_path, get_run_log_dir, get_verilator_build_dir, workload_tests_root +from utils.stream_run import stream_run_logger_async from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id from build_marker import build_marker_path, read_build_marker @@ -48,12 +49,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" - arch_config = input_data.get("config") - if not isinstance(arch_config, str) or not arch_config or arch_config == "None": - ctx.logger.error("Missing required parameter: config") + try: + chip = require_chip(input_data) + except ValueError as error: + ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_config"}, + extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) return @@ -70,7 +72,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - vsrc_dir = get_verilator_build_dir(bbdir, arch_config, input_data.get("vsrc_dir")) + vsrc_dir = get_verilator_build_dir(bbdir, chip, input_data.get("vsrc_dir")) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") if not os.path.isdir(vsrc_dir): ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}") @@ -84,9 +86,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: build_dir = bebop_dir chip = input_data.get("chip") if diff: + if not isinstance(chip, str) or not chip: + ctx.logger.error("Missing required parameter: --chip (required with --diff)") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_chip"}, + trace_id=origin_tid, + ) + return try: - chip = chip or get_chip_from_config(bbdir, arch_config) - manifest = resolve_chip_runtime_manifest(bbdir, chip, "bemu") + install_bundle(bbdir, chip) except ValueError as error: ctx.logger.error(str(error)) await check_result( @@ -97,9 +108,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - build_dir = str(manifest.parent) - bebop_bin = os.path.join(build_dir, "target", "release", "bebop") + bebop_bin = os.path.join(bebop_target_dir(bbdir, chip), "release", "bebop") if not os.path.isfile(bebop_bin): ctx.logger.error(f"bebop binary does not exist: {bebop_bin}") await check_result( @@ -109,9 +119,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - marker_path = build_marker_path(build_dir) + target_dir = bebop_target_dir(bbdir, chip) + marker_path = build_marker_path(target_dir) try: - marker = read_build_marker(build_dir) + marker = read_build_marker(target_dir) except FileNotFoundError: ctx.logger.error(f"bebop verilator build marker does not exist: {marker_path}") await check_result( @@ -132,7 +143,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: expect_vsrc = os.path.abspath(vsrc_dir) expect_bin = os.path.abspath(bebop_bin) if ( - marker.get("config") != arch_config + marker.get("config") != chip or marker.get("vsrc_dir") != expect_vsrc or marker.get("binary") != expect_bin or bool(marker.get("diff", False)) != diff @@ -144,7 +155,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "error": "build_marker_mismatch", "marker": marker, "expected": { - "config": arch_config, "vsrc_dir": expect_vsrc, "binary": expect_bin, "diff": diff, @@ -155,7 +165,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return binary_name = input_data.get("binary", "") - workload_root = f"{bbdir}/bb-tests/output/workloads/src" + workload_root = workload_tests_root(bbdir, chip) binary_path = None if diff and chip: binary_path = search_workload( @@ -174,8 +184,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"binary_path: {binary_path}") timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - mode = "difftest" if diff else "verilator" - log_dir = f"{bbdir}/log/{timestamp}-{mode}-{binary_name}" + log_dir = get_run_log_dir(bbdir, chip, "verilog", timestamp, "verilator", binary_name, input_data.get("vsrc_dir")) os.makedirs(log_dir, exist_ok=True) wave_arg = " --no-wave" if input_data.get("no-wave", False) or input_data.get("no_wave", False) else "" @@ -199,7 +208,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - backend_library = f"{bebop_dir}/target/release/deps/libbebop_verilator.so" + backend_library = f"{bebop_target_dir(bbdir, chip)}/release/deps/libbebop_verilator.so" binary_dir = os.path.dirname(binary_path) copy_cmd = shlex.join(["cmake", "-E", "copy_if_different", backend_library, binary_dir]) inner_cmd = f"cd {shlex.quote(binary_dir)} && {copy_cmd} && exec {shlex.quote(binary_path)}" @@ -233,7 +242,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"Running bebop verilator (diff={diff}): {run_cmd}") stdout_prefix = "bebop verilator difftest" if diff else "bebop verilator" stderr_prefix = stdout_prefix - run_result = stream_run_logger( + run_result = await stream_run_logger_async( cmd=run_cmd, logger=ctx.logger, cwd=bebop_dir, @@ -251,7 +260,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "diff": diff, "chip": chip, "binary": binary_path, - "config": arch_config, "log_dir": log_dir, "bank_diff": os.path.join(log_dir, "bank_diff.ndjson") if diff else None, "timestamp": timestamp, diff --git a/bbdev/api/steps/bebop/verilator/05_run_api.step.py b/bbdev/api/steps/bebop/verilator/05_run_api.step.py index ffab895b4..b3d356518 100644 --- a/bbdev/api/steps/bebop/verilator/05_run_api.step.py +++ b/bbdev/api/steps/bebop/verilator/05_run_api.step.py @@ -1,5 +1,8 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.chip import require_chip +from utils.path import get_buckyball_path + config = { "name": "bebop-verilator-run-api", "description": "Run complete bebop verilator workflow: clean → verilog → build → sim", @@ -18,12 +21,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body={"error": "Unsupported parameter: output_dir. Use --output-dir."}, ) - config_name = body.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --config must be specified"} - ) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) binary = body.get("binary", "") if not binary: @@ -46,7 +47,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: ) data = { - "config": config_name, + "chip": chip, "binary": binary, "balltype": body.get("balltype"), "itrace": body.get("itrace", False), diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py index 439e0e8d3..bcd45e3bb 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -1,5 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.chip import require_chip from utils.path import get_buckyball_path, get_verilator_build_dir config = { @@ -15,31 +16,22 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = request.body or {} - arch_config = body.get("config") - if not isinstance(arch_config, str) or not arch_config or arch_config == "None": - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --config must be specified"} - ) - - chip = body.get("chip") - if not chip: - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --chip must be specified"} - ) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) test_type = body.get("test") if not test_type: return ApiResponse( status=400, - body={"error": "Missing required parameter: --test must be specified (elf-tests or pk-tests)"} + body={"error": "Missing required parameter: --test must be specified (elf-tests or pk-tests)"}, ) if test_type not in ["elf-tests", "pk-tests"]: return ApiResponse( status=400, - body={"error": f"Invalid test type: {test_type}. Must be 'elf-tests' or 'pk-tests'"} + body={"error": f"Invalid test type: {test_type}. Must be 'elf-tests' or 'pk-tests'"}, ) diff = bool(body.get("diff", False)) @@ -50,11 +42,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body={"error": "--diff and --rushB cannot be used together"}, ) - vsrc_dir = get_verilator_build_dir(bbdir, arch_config, body.get("vsrc_dir")) + vsrc_dir = get_verilator_build_dir(bbdir, chip, body.get("vsrc_dir")) data = { "chip": chip, - "config": arch_config, "vsrc_dir": vsrc_dir, "test": test_type, "clean-before": body.get("clean-before", body.get("clean_before", False)), diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index dbb643f69..698e9f87c 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -17,11 +17,13 @@ if bebop_path not in sys.path: sys.path.insert(0, bebop_path) -from utils.chip import resolve_chip_runtime_manifest -from utils.path import get_buckyball_path, get_verilator_build_dir -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.build import install_bundle +from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path, get_verilator_build_dir +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from regression import regression_workload_toml +from regression_harness import nextest_harness_args config = { "name": "bebop-verilator-batch", @@ -37,27 +39,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" nextest_config = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/nextest.toml" - elf_root = f"{bbdir}/bb-tests/output" - arch_config = input_data.get("config") - if not isinstance(arch_config, str) or not arch_config or arch_config == "None": - ctx.logger.error("Missing required parameter: config must be specified") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "missing_config"}, - trace_id=origin_tid, - ) - return - - chip = input_data.get("chip") - if not chip: - ctx.logger.error("Missing required parameter: chip must be specified") + try: + chip = require_chip(input_data) + except ValueError as error: + ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) return + elf_root = chip_output_root(bbdir, chip) test_type = input_data.get("test", "elf-tests") rushB = bool(input_data.get("rushB", False)) @@ -90,7 +83,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"rushB={rushB} diff={diff}" ) - vsrc_dir = get_verilator_build_dir(bbdir, arch_config, input_data.get("vsrc_dir")) + vsrc_dir = get_verilator_build_dir(bbdir, chip, input_data.get("vsrc_dir")) if not os.path.isdir(vsrc_dir): ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}") await check_result( @@ -102,10 +95,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: vsrc_config = shlex.quote(f"env.VSRC_PATH='{vsrc_dir}'") env = os.environ.copy() + env.update(bebop_cargo_env(bbdir, chip)) env.update({ - "BEBOP_WORKLOAD_TOML": workload_toml, - "BEBOP_BB_TESTS_ROOT": elf_root, - "BEBOP_ARCH_CONFIG": arch_config, + "BEBOP_ARCH_CONFIG": chip, "VSRC_PATH": vsrc_dir, }) @@ -113,7 +105,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: features = "verilator" if diff: try: - manifest = str(resolve_chip_runtime_manifest(bbdir, chip, "bemu")) + install_bundle(bbdir, chip) except ValueError as error: ctx.logger.error(str(error)) await check_result( @@ -144,16 +136,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: shutil.rmtree(artifact_dir, ignore_errors=True) ctx.logger.info(f"Cleaned previous bebop test artifacts: {artifact_dir}") + harness = nextest_harness_args(workload_toml, elf_root) nextest_cmd = ( f"nix develop -c cargo nextest run --release --manifest-path {shlex.quote(manifest)} " f"--features {shlex.quote(features)} --test test_verilator " f"--config-file {shlex.quote(nextest_config)} " - f"--config={vsrc_config}" + f"--config={vsrc_config} " + f"{harness}" ) ctx.logger.info(f"Running bebop verilator nextest: {nextest_cmd}") - ctx.logger.info(f"Environment: BEBOP_WORKLOAD_TOML={workload_toml}, BEBOP_BB_TESTS_ROOT={elf_root}") - run_result = stream_run_logger( + run_result = await stream_run_logger_async( cmd=nextest_cmd, logger=ctx.logger, cwd=bbdir, @@ -170,7 +163,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "task": "batch", "backend": "verilator-rushB" if rushB else ("difftest" if diff else "verilator"), "chip": chip, - "config": arch_config, "vsrc_dir": vsrc_dir, "test_type": test_type, "rushB": rushB, diff --git a/bbdev/api/steps/bebop/verilator/scripts/build_marker.py b/bbdev/api/steps/bebop/verilator/scripts/build_marker.py index 17fb887d2..852c75933 100644 --- a/bbdev/api/steps/bebop/verilator/scripts/build_marker.py +++ b/bbdev/api/steps/bebop/verilator/scripts/build_marker.py @@ -2,33 +2,33 @@ import os -def build_marker_path(bebop_dir: str) -> str: - return os.path.join(bebop_dir, "target", "debug", ".bbdev-verilator-build.json") +def build_marker_path(target_dir: str) -> str: + return os.path.join(target_dir, ".bbdev-verilator-build.json") def write_build_marker( - bebop_dir: str, + target_dir: str, config: str, vsrc_dir: str, binary: str, diff: bool = False, ): - marker = build_marker_path(bebop_dir) - os.makedirs(os.path.dirname(marker), exist_ok=True) - data = { - "config": config, - "vsrc_dir": os.path.abspath(vsrc_dir), - "binary": os.path.abspath(binary), - "diff": diff, - } - tmp = f"{marker}.tmp" - with open(tmp, "w") as f: - json.dump(data, f, indent=2, sort_keys=True) - f.write("\n") - os.replace(tmp, marker) + marker = build_marker_path(target_dir) + os.makedirs(os.path.dirname(marker), exist_ok=True) + data = { + "config": config, + "vsrc_dir": os.path.abspath(vsrc_dir), + "binary": os.path.abspath(binary), + "diff": diff, + } + tmp = f"{marker}.tmp" + with open(tmp, "w") as f: + json.dump(data, f, indent=2, sort_keys=True) + f.write("\n") + os.replace(tmp, marker) -def read_build_marker(bebop_dir: str) -> dict: - marker = build_marker_path(bebop_dir) - with open(marker) as f: - return json.load(f) +def read_build_marker(target_dir: str) -> dict: + marker = build_marker_path(target_dir) + with open(marker) as f: + return json.load(f) diff --git a/bbdev/api/steps/compiler/01_build_api.step.py b/bbdev/api/steps/compiler/01_build_api.step.py index 465ced4d9..8e8f9815f 100644 --- a/bbdev/api/steps/compiler/01_build_api.step.py +++ b/bbdev/api/steps/compiler/01_build_api.step.py @@ -8,7 +8,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import available_compiler_chips, available_cores, resolve_chip_compiler_core, resolve_core +from utils.chip import available_compiler_chips, available_cores, resolve_core from utils.path import get_buckyball_path config = { @@ -47,8 +47,8 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: try: if core: resolve_core(get_buckyball_path(), core, require_compiler=True) - else: - resolve_chip_compiler_core(get_buckyball_path(), chip) + elif chip not in available_compiler_chips(get_buckyball_path()): + raise ValueError(f"Chip does not exist: {chip}") except ValueError as error: choices = available_cores(get_buckyball_path()) if core else available_compiler_chips(get_buckyball_path()) return ApiResponse( diff --git a/bbdev/api/steps/compiler/01_build_event.step.py b/bbdev/api/steps/compiler/01_build_event.step.py index defe76c5a..cf360aa1a 100644 --- a/bbdev/api/steps/compiler/01_build_event.step.py +++ b/bbdev/api/steps/compiler/01_build_event.step.py @@ -1,18 +1,16 @@ import os import re -import shlex import sys from motia import FlowContext, queue -# Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import available_compiler_chips, available_cores, resolve_chip_compiler_core, resolve_core +from utils.chip import available_compiler_chips, available_cores, resolve_core +from utils.build import build_compiler from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id config = { @@ -59,12 +57,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return try: - core_package = ( + if chip: + build_compiler(bbdir, chip=chip) + else: resolve_core(bbdir, core, require_compiler=True) - if core - else resolve_chip_compiler_core(bbdir, chip) - ) - except ValueError as error: + build_compiler(bbdir, core=core) + except (ValueError, RuntimeError) as error: choices = available_cores(bbdir) if core else available_compiler_chips(bbdir) ctx.logger.error(str(error)) await check_result( @@ -78,71 +76,5 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - compiler_dir = core_package.compiler - assert compiler_dir is not None - - buddy_dir = f"{bbdir}/compiler/thirdparty/buddy-mlir" - build_dir = f"{buddy_dir}/build" - llvm_build_dir = f"{buddy_dir}/llvm/build" - - def nix_cmd(args: list[str]) -> str: - return shlex.join(["nix", "develop", "-c", *args]) - - async def run_stage(stage: str, command: str): - ctx.logger.info(f"Running compiler build stage '{stage}': {command}") - result = stream_run_logger( - cmd=command, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="compiler build", - stderr_prefix="compiler build", - ) - if result.returncode != 0: - await check_result( - ctx, result.returncode, continue_run=False, - extra_fields={ - "task": stage, - "chip": chip, - "core": core_package.name, - "stable": stable, - }, - trace_id=origin_tid, - ) - return False - return True - - cmake_args = [ - "cmake", "-G", "Ninja", - "-S", buddy_dir, - "-B", build_dir, - f"-DBUDDY_EXTERNAL_DIALECTS_DIR={compiler_dir}", - f"-DMLIR_DIR={llvm_build_dir}/lib/cmake/mlir", - f"-DLLVM_DIR={llvm_build_dir}/lib/cmake/llvm", - "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DCMAKE_BUILD_TYPE=RELEASE", - "-DBUDDY_MLIR_ENABLE_PYTHON_PACKAGES=ON", - "-DPython3_EXECUTABLE=python3", - "-DPython_EXECUTABLE=python", - ] - if not await run_stage("configure", nix_cmd(cmake_args)): - return - - command = nix_cmd([ - "ninja", "-C", build_dir, - "buddy-opt", "buddy-translate", "buddy-llc", - "python-package-buddy", "BuddyMLIRPythonModules", - ]) - mode = "stable" if stable else "custom" - if not await run_stage(f"compiler-{core_package.name}-{mode}", command): - return - - # ================================================================================== - # Return result to API - # ================================================================================== - success_result, failure_result = await check_result( - ctx, 0, continue_run=False, trace_id=origin_tid) - # ================================================================================== - # Continue routing - # ================================================================================== - return + await check_result(ctx, 0, continue_run=False, trace_id=origin_tid) diff --git a/bbdev/api/steps/compiler/README.md b/bbdev/api/steps/compiler/README.md index cf5815592..e45c5e6dd 100644 --- a/bbdev/api/steps/compiler/README.md +++ b/bbdev/api/steps/compiler/README.md @@ -36,4 +36,6 @@ bbdev compiler --build '--chip pebble --stable' ## Notes - Ensure the system has necessary build tools and dependencies +- Each `compilerCore` builds under `compiler/thirdparty/buddy-mlir/build/cores//`; different cores can be built in parallel on one checkout - `pebble` supports both custom `xbuckyball` lowering and stable LLVM backend lowering; the selected lowering mode is controlled by pass/workload options +- Interactive shell: `export BUCKYBALL_COMPILER_CORE=` then re-source `sourceme.sh` for `buddy-opt` on PATH; bbdev injects `BUDDY_MLIR_BUILD_DIR` automatically in subprocesses diff --git a/bbdev/api/steps/dc/01_area_api.step.py b/bbdev/api/steps/dc/01_area_api.step.py index e6c5aa7f6..b7064f2e4 100644 --- a/bbdev/api/steps/dc/01_area_api.step.py +++ b/bbdev/api/steps/dc/01_area_api.step.py @@ -1,6 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import check_dc_rtl_args, get_buckyball_path, get_dc_analysis_dir, get_dc_rtl_dir +from utils.chip import require_chip +from utils.path import check_dc_rtl_args config = { "name": "dc-area-api", @@ -12,21 +13,20 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = req.body or {} try: - check_dc_rtl_args(body) - rtl_dir = get_dc_rtl_dir(bbdir, body.get("config")) - analysis_dir = get_dc_analysis_dir(bbdir, body.get("config"), "area") + chip = require_chip(body) + subset = {"chip": chip} + if "top" in body: + subset["top"] = body.get("top") + check_dc_rtl_args(subset) except ValueError as exc: return ApiResponse(status=400, body={"error": str(exc)}) await ctx.enqueue( { "topic": "dc.verilog", "data": { - "output_dir": rtl_dir, - "analysis_dir": analysis_dir, - "config": body.get("config"), + "chip": chip, "top": body.get("top") or "DigitalTop", "from_area_workflow": True, "_trace_id": ctx.trace_id, diff --git a/bbdev/api/steps/dc/02_verilog_api.step.py b/bbdev/api/steps/dc/02_verilog_api.step.py index a4bf4df08..5f496ae3a 100644 --- a/bbdev/api/steps/dc/02_verilog_api.step.py +++ b/bbdev/api/steps/dc/02_verilog_api.step.py @@ -1,6 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import check_dc_rtl_args, get_buckyball_path, get_dc_rtl_dir +from utils.chip import require_chip +from utils.path import check_dc_rtl_args config = { "name": "dc-verilog-api", @@ -12,17 +13,18 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = req.body or {} try: - check_dc_rtl_args(body) - output_dir = get_dc_rtl_dir(bbdir, body.get("config")) + chip = require_chip(body) + subset = {"chip": chip} + if "top" in body: + subset["top"] = body.get("top") + check_dc_rtl_args(subset) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) data = { - "output_dir": output_dir, - "config": body.get("config"), + "chip": chip, "top": body.get("top") or "DigitalTop", } await ctx.enqueue({"topic": "dc.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) diff --git a/bbdev/api/steps/dc/03_area_event.step.py b/bbdev/api/steps/dc/03_area_event.step.py index 1b75a3d2f..5edecfe41 100644 --- a/bbdev/api/steps/dc/03_area_event.step.py +++ b/bbdev/api/steps/dc/03_area_event.step.py @@ -18,7 +18,7 @@ from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger +from utils.stream_run import stream_run_logger_async from tapeout import get_tapeout_contract, technology_settings, write_run_tcl config = { @@ -67,7 +67,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return try: - contract = get_tapeout_contract(get_buckyball_path(), input_data.get("config"), top_module) + contract = get_tapeout_contract(get_buckyball_path(), input_data.get("chip"), top_module) except (OSError, ValueError) as exc: await check_result( ctx, @@ -102,7 +102,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: script_path = contract.dc_script dc_log = os.path.join(analysis_dir, "dc_shell.log") ctx.logger.info(f"Running chip-owned DC synthesis for {contract.chip}, top {top_module}: {script_path}") - result = stream_run_logger( + result = await stream_run_logger_async( cmd=( f"dc_shell -f {shlex.quote(str(script_path))} " f"-x {shlex.quote('set RUN_CONFIG ' + '{' + str(run_config) + '}')} " @@ -113,22 +113,61 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stdout_prefix="dc", stderr_prefix="dc", ) + extra_fields = { + "task": "area", + "top_module": top_module, + "dc_script": str(script_path), + "dc_output_dir": output_dir, + "dc_report_dir": report_dir, + "dc_log": dc_log, + "run_config": str(run_config), + "chip": contract.chip, + "tapeout_dir": str(contract.root), + "sram_manifest": (input_data.get("sram_collateral") or {}).get("sram_manifest"), + } + + if result.returncode == 0 and input_data.get("from_regression_area_power"): + reg_scripts = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "regression", "scripts") + ) + if reg_scripts not in sys.path: + sys.path.insert(0, reg_scripts) + from dc_area import area_mm2_from_rpt + from result import merge_metrics + + rpt = os.path.join(report_dir, "area.rpt") + if not os.path.isfile(rpt): + await check_result( + ctx, 1, continue_run=False, + extra_fields={**extra_fields, "error": f"missing area.rpt: {rpt}", "area_rpt": rpt}, + trace_id=origin_tid, + ) + return + try: + with open(rpt, encoding="utf-8") as handle: + area = area_mm2_from_rpt(handle.read()) + except ValueError as exc: + await check_result( + ctx, 1, continue_run=False, + extra_fields={**extra_fields, "error": str(exc), "area_rpt": rpt}, + trace_id=origin_tid, + ) + return + freq = 1000.0 / contract.clock_period_ns + merge_metrics(get_buckyball_path(), area=area, freq=freq) + extra_fields["area"] = area + extra_fields["freq"] = freq + await check_result( + ctx, result.returncode, continue_run=False, + extra_fields=extra_fields, trace_id=origin_tid, + ) + return + await check_result( ctx, result.returncode, continue_run=False, - extra_fields={ - "task": "area", - "top_module": top_module, - "dc_script": str(script_path), - "dc_output_dir": output_dir, - "dc_report_dir": report_dir, - "dc_log": dc_log, - "run_config": str(run_config), - "chip": contract.chip, - "tapeout_dir": str(contract.root), - "sram_manifest": (input_data.get("sram_collateral") or {}).get("sram_manifest"), - }, + extra_fields=extra_fields, trace_id=origin_tid, ) if result.returncode == 0 and input_data.get("from_power_workflow"): diff --git a/bbdev/api/steps/dc/04_power_api.step.py b/bbdev/api/steps/dc/04_power_api.step.py index 4c46baf39..138bb79b7 100644 --- a/bbdev/api/steps/dc/04_power_api.step.py +++ b/bbdev/api/steps/dc/04_power_api.step.py @@ -1,6 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import check_dc_power_args, get_buckyball_path, get_dc_analysis_dir, get_dc_rtl_dir +from utils.chip import require_chip +from utils.path import check_dc_power_args config = { "name": "dc-power-api", @@ -12,12 +13,10 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = req.body or {} try: - check_dc_power_args(body) - rtl_dir = get_dc_rtl_dir(bbdir, body.get("config")) - analysis_dir = get_dc_analysis_dir(bbdir, body.get("config"), "power") + chip = require_chip(body) + check_dc_power_args({k: v for k, v in body.items() if k != "config"}) except ValueError as exc: return ApiResponse(status=400, body={"error": str(exc)}) await ctx.enqueue( @@ -25,9 +24,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: "topic": "dc.verilog", "data": { **body, - "output_dir": rtl_dir, - "analysis_dir": analysis_dir, - "config": body.get("config"), + "chip": chip, "top": body.get("top") or "DigitalTop", "from_power_workflow": True, "_trace_id": ctx.trace_id, diff --git a/bbdev/api/steps/dc/05_sim_event.step.py b/bbdev/api/steps/dc/05_sim_event.step.py index 98c0ace8e..ac1b8ef62 100644 --- a/bbdev/api/steps/dc/05_sim_event.step.py +++ b/bbdev/api/steps/dc/05_sim_event.step.py @@ -13,9 +13,9 @@ sys.path.insert(0, step_path) from utils.event_common import check_result, get_origin_trace_id -from utils.path import get_buckyball_path +from utils.path import get_buckyball_path, workload_build_dir, workload_tests_root from utils.search_workload import search_workload -from utils.stream_run import stream_run_logger +from utils.stream_run import stream_run_logger_async from tapeout import get_tapeout_contract, resolve_power_window, write_run_env config = { @@ -37,7 +37,7 @@ def _first(data: dict, *names: str): async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) - config_name = input_data.get("config") + chip = input_data.get("chip") top_module = input_data.get("top") or "DigitalTop" analysis_dir = input_data.get("analysis_dir") if not isinstance(analysis_dir, str) or not analysis_dir: @@ -45,7 +45,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return try: - contract = get_tapeout_contract(get_buckyball_path(), config_name, top_module) + contract = get_tapeout_contract(get_buckyball_path(), input_data.get("chip"), top_module) except (OSError, ValueError) as exc: await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": str(exc)}, trace_id=origin_tid) return @@ -78,13 +78,20 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if os.path.isabs(str(workload_name)): await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": "workload must be a built ELF name, not an absolute path"}, trace_id=origin_tid) return + if not chip: + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": "missing chip"}, trace_id=origin_tid) + return + bbdir = get_buckyball_path() workload = None - for root in ("output/workloads/src", "build/workloads/src"): - workload = search_workload(os.path.join(get_buckyball_path(), "bb-tests", root), str(workload_name)) + for root in ( + workload_tests_root(bbdir, chip), + os.path.join(workload_build_dir(bbdir, chip), "src"), + ): + workload = search_workload(root, str(workload_name)) if workload: break if not workload: - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": f"workload not found in bb-tests output/build directories: {workload_name}"}, trace_id=origin_tid) + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": f"workload not found for chip {chip}: {workload_name}"}, trace_id=origin_tid) return try: start_ns, end_ns = resolve_power_window( @@ -105,7 +112,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "ACTIVITY_FILE": activity_path, "ACTIVITY_FORMAT": activity_format, "TOP": top_module, - "CONFIG": str(config_name or ""), + "CONFIG": chip, "NETLIST": netlist, "SDC": sdc, "WORKLOAD": workload, @@ -120,7 +127,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"format={activity_format}, workload={workload or ''}, " f"window={start_ns or ''}..{end_ns or ''} ns" ) - result = stream_run_logger( + result = await stream_run_logger_async( cmd=f"bash {shlex.quote(str(contract.power_sim_script))} {shlex.quote(str(run_env))}", logger=ctx.logger, cwd=str(contract.root), diff --git a/bbdev/api/steps/dc/06_ptpx_event.step.py b/bbdev/api/steps/dc/06_ptpx_event.step.py index a5c459922..b3c1334d0 100644 --- a/bbdev/api/steps/dc/06_ptpx_event.step.py +++ b/bbdev/api/steps/dc/06_ptpx_event.step.py @@ -18,7 +18,7 @@ from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path from power import read_dynamic_power -from utils.stream_run import stream_run_logger +from utils.stream_run import stream_run_logger_async from tapeout import get_tapeout_contract, resolve_power_window, technology_settings, write_run_tcl config = { @@ -53,7 +53,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: try: netlist = os.path.join(analysis_dir, "outputs", f"{top_module}.v") sdc = os.path.join(analysis_dir, "outputs", f"{top_module}.sdc") - contract = get_tapeout_contract(get_buckyball_path(), input_data.get("config"), top_module) + contract = get_tapeout_contract(get_buckyball_path(), input_data.get("chip"), top_module) start_ns, end_ns = resolve_power_window( contract, input_data.get("start_ns", input_data.get("start-ns")), @@ -80,7 +80,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result(ctx, 1, continue_run=False, extra_fields={"task": "power", "error": str(exc)}, trace_id=origin_tid) return - result = stream_run_logger( + result = await stream_run_logger_async( cmd=( f"pt_shell -f {shlex.quote(str(script))} " f"-x {shlex.quote('set RUN_CONFIG ' + '{' + str(run_config) + '}')} " diff --git a/bbdev/api/steps/dc/tapeout.py b/bbdev/api/steps/dc/tapeout.py index e202e1b98..fd829a8e7 100644 --- a/bbdev/api/steps/dc/tapeout.py +++ b/bbdev/api/steps/dc/tapeout.py @@ -8,9 +8,6 @@ from dataclasses import dataclass from pathlib import Path -from utils.path import get_chip_from_config - - def _tcl_word(value: str) -> str: return "{" + str(value).replace("\\", "\\\\").replace("}", "\\}") + "}" @@ -65,9 +62,10 @@ def _read_config(root: Path) -> dict: return section -def get_tapeout_contract(bbdir: str | os.PathLike[str], config: str, top: str | None = None) -> TapeoutContract: - """Resolve the tapeout scripts owned by the chip selected by a Scala config.""" - chip = get_chip_from_config(str(bbdir), config) +def get_tapeout_contract(bbdir: str | os.PathLike[str], chip: str, top: str | None = None) -> TapeoutContract: + """Resolve the tapeout scripts owned by the chip.""" + if not isinstance(chip, str) or not chip: + raise ValueError("Missing required parameter: --chip") root = Path(bbdir) / "examples" / "chips" / chip / "tapeout" if not root.is_dir(): raise ValueError(f"chip {chip} has no tapeout directory: {root}") @@ -135,14 +133,19 @@ def technology_settings() -> dict[str, object]: synthetic = [item for item in os.environ.get("SYNTHETIC_LIBRARY", "").split(os.pathsep) if item] link = [item for item in os.environ.get("LINK_LIBRARY", "").split(os.pathsep) if item] if not target: - raise ValueError("missing TARGET_LIBRARY; source sourceme_host.sh") + raise ValueError("missing TARGET_LIBRARY; export it in zshrc") + if any(ch.isspace() for ch in target): + raise ValueError( + "TARGET_LIBRARY contains whitespace/newlines; " + f"fix the zshrc export (got {target!r})" + ) if not Path(target).is_file(): raise ValueError(f"TARGET_LIBRARY does not exist: {target}") missing = [item for item in synthetic + link if not Path(item).is_file()] if missing: raise ValueError("technology library does not exist: " + ", ".join(missing)) if not synthetic: - raise ValueError("missing SYNTHETIC_LIBRARY; source sourceme_host.sh") + raise ValueError("missing SYNTHETIC_LIBRARY; export it in zshrc") return { "target_library": target, "synthetic_library": synthetic, diff --git a/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py b/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py index 2373a5686..e1b56a6a6 100644 --- a/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py +++ b/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py @@ -9,7 +9,7 @@ sys.path.insert(0, utils_path) from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id # Import firesim_env from scripts subdirectory @@ -42,7 +42,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: command += f" -b {yaml_dir}/config_build.yaml" command += f" -r {yaml_dir}/config_build_recipes.yaml" command += f" -c {yaml_dir}/config_runtime.yaml" - result = stream_run_logger( + result = await stream_run_logger_async( cmd=command, logger=ctx.logger, stdout_prefix="firesim enumeratefpgas", diff --git a/bbdev/api/steps/firesim/01_buildbitstream_event.step.py b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py index 240d05f94..be857b5ce 100644 --- a/bbdev/api/steps/firesim/01_buildbitstream_event.step.py +++ b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py @@ -9,7 +9,7 @@ sys.path.insert(0, utils_path) from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id # Add scripts directory to path for firesim_env import @@ -62,7 +62,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: command += f" -b {yaml_dir}/config_build.yaml" command += f" -r {yaml_dir}/config_build_recipes.yaml" command += f" -c {yaml_dir}/config_runtime.yaml" - result = stream_run_logger( + result = await stream_run_logger_async( cmd=command, logger=ctx.logger, stdout_prefix="firesim buildbitstream", diff --git a/bbdev/api/steps/firesim/02_infrasetup_event.step.py b/bbdev/api/steps/firesim/02_infrasetup_event.step.py index 165747268..d122ad760 100644 --- a/bbdev/api/steps/firesim/02_infrasetup_event.step.py +++ b/bbdev/api/steps/firesim/02_infrasetup_event.step.py @@ -9,7 +9,7 @@ sys.path.insert(0, utils_path) from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id # Import firesim_env from scripts subdirectory @@ -42,7 +42,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: command += f" -b {yaml_dir}/config_build.yaml" command += f" -r {yaml_dir}/config_build_recipes.yaml" command += f" -c {yaml_dir}/config_runtime.yaml" - result = stream_run_logger( + result = await stream_run_logger_async( cmd=command, logger=ctx.logger, stdout_prefix="firesim infrasetup", diff --git a/bbdev/api/steps/firesim/03_runworkload_event.step.py b/bbdev/api/steps/firesim/03_runworkload_event.step.py index e5c9e763d..d9fe92da4 100644 --- a/bbdev/api/steps/firesim/03_runworkload_event.step.py +++ b/bbdev/api/steps/firesim/03_runworkload_event.step.py @@ -9,7 +9,7 @@ sys.path.insert(0, utils_path) from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id # Import firesim_env from scripts subdirectory @@ -42,7 +42,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: command += f" -b {yaml_dir}/config_build.yaml" command += f" -r {yaml_dir}/config_build_recipes.yaml" command += f" -c {yaml_dir}/config_runtime.yaml" - result = stream_run_logger( + result = await stream_run_logger_async( cmd=command, logger=ctx.logger, stdout_prefix="firesim runworkload", diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index 1848bd178..f8f1acb3a 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -8,7 +8,7 @@ sys.path.insert(0, utils_path) from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id # Import bin_to_hex converter @@ -92,7 +92,7 @@ def kernel_model(input_data: dict) -> str: return model -def kernel_chip(input_data: dict, bbdir: str) -> str: +def kernel_chip(input_data: dict, bbdir: str, *, require_overlay: bool) -> str: chip = input_data.get("chip", "") if chip in ("", None): return "" @@ -101,13 +101,23 @@ def kernel_chip(input_data: dict, bbdir: str) -> str: if os.path.sep in chip or chip in {".", ".."}: raise ValueError(f"invalid chip: {chip}") - chip_kernel = os.path.join(bbdir, "examples", "chips", chip, "kernel") - chip_init = os.path.join(chip_kernel, "overlay", "init") - if not os.path.isfile(chip_init): - raise ValueError(f"chip OS overlay init not found: {chip_init}") + chip_dir = os.path.join(bbdir, "examples", "chips", chip) + if not os.path.isdir(chip_dir): + raise ValueError(f"unknown chip: {chip}") + + if require_overlay: + chip_init = os.path.join(chip_dir, "kernel", "overlay", "init") + if not os.path.isfile(chip_init): + raise ValueError(f"chip OS overlay init not found: {chip_init}") return chip +def cmake_model(model: str) -> str: + if model == "stable-diffusion": + return "stablediffusion" + return model + + def kernel_build_dir( bbdir: str, hart_params: dict, @@ -135,10 +145,10 @@ def fw_payload_name(hart_params: dict, model: str = "", chip: str = "") -> str: name = "fw_payload" if visible != 64 or total != 64: name = f"{name}-v{visible}-t{total}" - if chip: - name = f"{name}-{chip}-pk" - elif model: + if model: name = f"{name}-{model}" + elif chip: + name = f"{name}-{chip}-pk" return name @@ -154,10 +164,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: try: hart_params = hart_count_params(input_data) model = kernel_model(input_data) - chip = kernel_chip(input_data, bbdir) + if model and not input_data.get("chip"): + raise ValueError("--model requires --chip") interactive = kernel_interactive(input_data) - if chip and model: - raise ValueError("chip and model cannot be set together") + # pk mode needs OS overlay; model mode only needs chip for ModelTest path + chip = kernel_chip(input_data, bbdir, require_overlay=not bool(model)) except ValueError as e: ctx.logger.error(str(e)) await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) @@ -166,6 +177,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir, hart_params, model, chip, interactive=interactive ) interactive_arg = "ON" if interactive else "OFF" + cmake_model_name = cmake_model(model) # cmake configure configure_cmd = ( @@ -173,11 +185,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"-DBUCKYBALL_VISIBLE_HART_COUNT={hart_params['visible']} " f"-DBUCKYBALL_TOTAL_HART_COUNT={hart_params['total']} " f"-DBUCKYBALL_HIDDEN_HART_BASE={hart_params['hidden_base']} " - f"-DBUCKYBALL_KERNEL_MODEL={model} " + f"-DBUCKYBALL_KERNEL_MODEL={cmake_model_name} " f"-DBUCKYBALL_KERNEL_CHIP={chip} " - f"-DBUCKYBALL_KERNEL_INTERACTIVE={interactive_arg}" + f"-DBUCKYBALL_KERNEL_INTERACTIVE={interactive_arg} " + f"-DBUCKYBALL_MODEL_DATASET=" ) - if chip: + if chip and not model: chip_kernel = os.path.join(bbdir, "examples", "chips", chip, "kernel") pk_toml = os.path.join( bbdir, "examples", "chips", chip, "regression", "batch", "bemu", "workloads-pk.toml" @@ -191,7 +204,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f" -DBUCKYBALL_PK_WORKLOAD_TOML={pk_toml}" ) - result = stream_run_logger( + result = await stream_run_logger_async( cmd=configure_cmd, logger=ctx.logger, stdout_prefix="marshal build", @@ -203,7 +216,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # cmake build build_cmd = f"cmake --build {kernel_build} --target kernel-build" - result = stream_run_logger( + result = await stream_run_logger_async( cmd=build_cmd, logger=ctx.logger, stdout_prefix="marshal build", @@ -215,7 +228,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return # Convert fw_payload.bin to hex for P2E memory backdoor - payload_name = fw_payload_name(hart_params, model, chip) + payload_name = fw_payload_name(hart_params, cmake_model_name if model else "", chip) fw_payload_bin = os.path.join(output_dir, f"{payload_name}.bin") fw_payload_hex = os.path.join(output_dir, f"{payload_name}.hex") diff --git a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py index 0b62a869a..f0bdd7c8d 100644 --- a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py @@ -1,5 +1,4 @@ import os -import subprocess import sys from motia import FlowContext, queue @@ -8,8 +7,10 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path, get_verilator_build_dir -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.path import get_buckyball_path +from utils.rtl import run_chip_mill +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id config = { @@ -21,87 +22,72 @@ } -def prepare_verilator_verilog(build_dir: str, arch_dir: str, logger): - unwanted_harness = f"{arch_dir}/BBSimHarness.sv" - if os.path.exists(unwanted_harness): - os.remove(unwanted_harness) - - for patch_file in [f"{build_dir}/mm.h", f"{build_dir}/mm.cc"]: - if os.path.exists(patch_file): - with open(patch_file, "r") as f: - content = f.read() - patched = "\n".join( - line for line in content.splitlines() - if "fesvr/memif.h" not in line and "fesvr/elfloader.h" not in line - ) - if patched != content: - with open(patch_file, "w") as f: - f.write(patched) - logger.info(f"Patched fesvr includes from {patch_file}") - - async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - build_dir = get_verilator_build_dir( - bbdir, - input_data.get("config"), - input_data.get("output_dir"), - ) - arch_dir = f"{bbdir}/arch" - config_name = input_data.get("config") - - if not config_name or config_name == "None": - ctx.logger.error("Configuration name is required but not provided") - success_result, failure_result = await check_result( - ctx, - 1, - continue_run=False, - extra_fields={ - "task": "validation", - "error": "Configuration name is required. Please specify --config parameter.", - "example": 'bbdev verilator --verilog "--config sims.verilator.BuckyballToyVerilatorConfig"', - }, - trace_id=origin_tid, - ) - return failure_result - - ctx.logger.info(f"Using configuration: {config_name}") - ctx.logger.info(f"Using build directory: {build_dir}") if input_data.get("balltype"): + build_dir = input_data.get("output_dir") + if not build_dir: + ctx.logger.error("balltype mill requires output_dir") + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "validation", + "error": "balltype mill requires output_dir", + }, + trace_id=origin_tid, + ) + return failure_result + os.makedirs(build_dir, exist_ok=True) command = ( f"mill -i __.test.runMain sims.verify.BallTopMain {input_data.get('balltype')} " + "--disable-annotation-unknown --strip-debug-info -O=release " + f"--split-verilog -o={build_dir} " ) + ctx.logger.info(f"Using build directory: {build_dir}") + result = await stream_run_logger_async( + cmd=command, + logger=ctx.logger, + cwd=f"{bbdir}/arch", + stdout_prefix="verilator verilog", + stderr_prefix="verilator verilog", + ) + returncode = result.returncode else: - command = f"mill -i __.test.runMain sims.verilator.Elaborate {config_name} " - - command += "--disable-annotation-unknown " - command += "--strip-debug-info " - command += "-O=release " - command += f"--split-verilog -o={build_dir} " - - result = stream_run_logger( - cmd=command, - logger=ctx.logger, - cwd=arch_dir, - stdout_prefix="verilator verilog", - stderr_prefix="verilator verilog", - ) - - prepare_verilator_verilog(build_dir, arch_dir, ctx.logger) + try: + chip = require_chip(input_data) + build_dir, returncode = await run_chip_mill( + ctx, bbdir, chip, "verilog", "verilator verilog", + input_data.get("output_dir"), + ) + except (ValueError, RuntimeError) as error: + ctx.logger.error(str(error)) + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "validation", + "error": str(error), + "example": 'bbdev verilator --verilog "--chip toy"', + }, + trace_id=origin_tid, + ) + return failure_result + ctx.logger.info(f"Using chip: {chip}") - success_result, failure_result = await check_result( + await check_result( ctx, - result.returncode, - continue_run=input_data.get("from_run_workflow", False), + returncode, + continue_run=input_data.get("from_run_workflow", False) and returncode == 0, extra_fields={"task": "verilog"}, trace_id=origin_tid, ) - if input_data.get("from_run_workflow"): + if input_data.get("from_run_workflow") and returncode == 0: await ctx.enqueue( {"topic": "verilator.build", "data": {**input_data, "output_dir": build_dir, "task": "run"}} ) - - return diff --git a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py index 496b6bf58..da8f479f9 100644 --- a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py +++ b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py @@ -1,9 +1,9 @@ import os -import shutil import sys import glob import re import yaml +from datetime import datetime from motia import FlowContext, queue @@ -14,10 +14,10 @@ if step_path not in sys.path: sys.path.insert(0, step_path) -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.path import get_buckyball_path, get_run_log_dir +from utils.rtl import run_chip_mill from utils.event_common import check_result, get_origin_trace_id -from verilog import seq_mem_elaboration_command config = { "name": "yosys verilog", @@ -166,41 +166,39 @@ def _build_blackbox(mod_name, ports): async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/") - yosys_log_dir = input_data.get("log_dir", f"{bbdir}/bbdev/api/steps/yosys/log") - arch_dir = f"{bbdir}/arch" - ctx.logger.info(f"Yosys log dir: {yosys_log_dir}") - - yosys_cfg = load_yosys_config() - top_module = input_data.get("top") or yosys_cfg.get("top") or "DigitalTop" - elaborate_config = input_data.get("config") or yosys_cfg.get( - "elaborate_config", "sims.verilator.BuckyballToyVerilatorConfig" - ) - - if os.path.exists(build_dir): - shutil.rmtree(build_dir) - os.makedirs(build_dir, exist_ok=True) - - mem_conf = os.path.join(build_dir, "mems.conf") - verilog_command = seq_mem_elaboration_command(elaborate_config, build_dir, mem_conf) - - result = stream_run_logger( - cmd=verilog_command, - logger=ctx.logger, - cwd=arch_dir, - stdout_prefix="yosys verilog", - stderr_prefix="yosys verilog", - ) - - if result.returncode != 0: + try: + chip = require_chip(input_data) + build_dir, returncode = await run_chip_mill( + ctx, bbdir, chip, "synth", "yosys verilog", + input_data.get("output_dir"), + ) + except (ValueError, RuntimeError) as error: success_result, failure_result = await check_result( ctx, - result.returncode, + 1, + continue_run=False, + extra_fields={"task": "validation", "error": str(error)}, + trace_id=origin_tid, + ) + return failure_result + if returncode != 0: + success_result, failure_result = await check_result( + ctx, + returncode, continue_run=False, extra_fields={"task": "verilog"}, trace_id=origin_tid, ) return failure_result + yosys_cfg = load_yosys_config() + top_module = input_data.get("top") or yosys_cfg.get("top") or "DigitalTop" + yosys_log_dir = input_data.get("log_dir") or get_run_log_dir( + bbdir, chip, "synth", datetime.now().strftime("%Y%m%d-%H%M%S-%f"), + "yosys", top_module, input_data.get("output_dir"), + ) + ctx.logger.info(f"Yosys log dir: {yosys_log_dir}") + mem_conf = os.path.join(build_dir, "mems.conf") + arch_dir = f"{bbdir}/arch" for unwanted in ["TestHarness.sv", "TargetBall.sv"]: topname_file = f"{arch_dir}/{unwanted}" @@ -222,7 +220,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result( ctx, - result.returncode, + 0, continue_run=True, extra_fields={"task": "verilog", "source_list": source_list_path, "mem_conf": mem_conf}, trace_id=origin_tid, diff --git a/bbdev/api/steps/mill/04_dc_verilog_event.step.py b/bbdev/api/steps/mill/04_dc_verilog_event.step.py index 7ce603099..34ef224fa 100644 --- a/bbdev/api/steps/mill/04_dc_verilog_event.step.py +++ b/bbdev/api/steps/mill/04_dc_verilog_event.step.py @@ -1,9 +1,7 @@ import os -import shutil import sys import glob import re -import shlex from motia import FlowContext, queue @@ -15,9 +13,9 @@ sys.path.insert(0, step_path) from utils.event_common import check_result, get_origin_trace_id -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger -from verilog import seq_mem_elaboration_command +from utils.chip import require_chip +from utils.path import get_buckyball_path, get_dc_analysis_dir +from utils.rtl import run_chip_mill config = { "name": "dc-verilog", @@ -79,58 +77,33 @@ def prepare_dc_verilog(build_dir: str): async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - arch_dir = f"{bbdir}/arch" - elaborate_config = input_data.get("config") - top_module = input_data.get("top") or "DigitalTop" - if not elaborate_config: + try: + chip = require_chip(input_data) + build_dir, returncode = await run_chip_mill( + ctx, bbdir, chip, "synth", "dc verilog", + input_data.get("output_dir"), + ) + except (ValueError, RuntimeError) as error: _, failure_result = await check_result( ctx, 1, continue_run=False, - extra_fields={"task": "validation", "error": "missing required parameter: config"}, + extra_fields={"task": "validation", "error": str(error)}, trace_id=origin_tid, ) return failure_result - build_dir = input_data.get("output_dir") - if not build_dir: + if returncode != 0: _, failure_result = await check_result( ctx, - 1, + returncode, continue_run=False, - extra_fields={"task": "validation", "error": "missing required parameter: output_dir"}, + extra_fields={"task": "verilog"}, trace_id=origin_tid, ) return failure_result + top_module = input_data.get("top") or "DigitalTop" ctx.logger.info(f"Using DC RTL output directory: {build_dir}, synthesis top: {top_module}") - - if os.path.exists(build_dir): - shutil.rmtree(build_dir) - os.makedirs(build_dir, exist_ok=True) - mem_conf = os.path.join(build_dir, "mems.conf") - verilog_command = seq_mem_elaboration_command(elaborate_config, build_dir, mem_conf) - elaboration_log = os.path.join(build_dir, "dc_elaboration.log") - - result = stream_run_logger( - # Elaboration emits the complete device tree and environment, which is - # useful for postmortem debugging but too large for a bbdev terminal. - # Keep it with the generated collateral and leave the CLI responsive. - cmd=f"{verilog_command} > {shlex.quote(elaboration_log)} 2>&1", - logger=ctx.logger, - cwd=arch_dir, - stdout_prefix="dc verilog", - stderr_prefix="dc verilog", - ) - - if result.returncode != 0: - _, failure_result = await check_result( - ctx, - result.returncode, - continue_run=False, - extra_fields={"task": "verilog", "elaboration_log": elaboration_log}, - trace_id=origin_tid, - ) - return failure_result try: source_list_path, stubbed_dpi = prepare_dc_verilog(build_dir) @@ -152,32 +125,31 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result( ctx, - result.returncode, + 0, continue_run=True, extra_fields={ "task": "verilog", "source_list": source_list_path, "mem_conf": mem_conf, "top_module": top_module, - "elaboration_log": elaboration_log, }, trace_id=origin_tid, ) - await ctx.enqueue( - { - "topic": "ip-replace.run", - "data": { - **input_data, - "source_list": source_list_path, - "ip_replace_output_dir": os.path.join(build_dir, "ip-replace"), - "consumer": "dc", - "mem_conf": mem_conf, - "top": top_module, - "next_topic": "dc.area" if input_data.get("from_area_workflow") or input_data.get("from_power_workflow") else None, - "_trace_id": origin_tid, - }, - } - ) + payload = { + **input_data, + "source_list": source_list_path, + "ip_replace_output_dir": os.path.join(build_dir, "ip-replace"), + "consumer": "dc", + "mem_conf": mem_conf, + "top": top_module, + "next_topic": "dc.area" if input_data.get("from_area_workflow") or input_data.get("from_power_workflow") else None, + "_trace_id": origin_tid, + } + if input_data.get("from_power_workflow"): + payload["analysis_dir"] = get_dc_analysis_dir(bbdir, chip, "power") + elif input_data.get("from_area_workflow"): + payload["analysis_dir"] = get_dc_analysis_dir(bbdir, chip, "area") + await ctx.enqueue({"topic": "ip-replace.run", "data": payload}) return diff --git a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py index 66de11280..5de510fd5 100644 --- a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py @@ -12,8 +12,10 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path, get_verilator_build_dir -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.path import get_buckyball_path +from utils.rtl import run_chip_mill +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id config = { @@ -26,23 +28,10 @@ def prepare_verilator_verilog(build_dir: str, arch_dir: str, logger): - """Remove unwanted harness and patch fesvr includes""" - unwanted_harness = f"{arch_dir}/BBSimHarness.sv" - if os.path.exists(unwanted_harness): - os.remove(unwanted_harness) - - for patch_file in [f"{build_dir}/mm.h", f"{build_dir}/mm.cc"]: - if os.path.exists(patch_file): - with open(patch_file, "r") as f: - content = f.read() - patched = "\n".join( - line for line in content.splitlines() - if "fesvr/memif.h" not in line and "fesvr/elfloader.h" not in line - ) - if patched != content: - with open(patch_file, "w") as f: - f.write(patched) - logger.info(f"Patched fesvr includes from {patch_file}") + unwanted = f"{arch_dir}/BBSimHarness.sv" + if os.path.isfile(unwanted): + os.remove(unwanted) + logger.info(f"Removed leaked mill output: {unwanted}") def check_verilog_output(build_dir: str) -> dict: @@ -61,63 +50,72 @@ def check_verilog_output(build_dir: str) -> dict: async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - build_dir = get_verilator_build_dir( - bbdir, - input_data.get("config"), - input_data.get("output_dir"), - ) arch_dir = f"{bbdir}/arch" - config_name = input_data.get("config") - - if not config_name or config_name == "None": - ctx.logger.error("Configuration name is required but not provided") - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={ - "task": "validation", - "error": "Configuration name is required. Please specify --config parameter.", - "example": 'bbdev bebop verilator --verilog "--config sims.verilator.BuckyballToyVerilatorConfig"', - }, - trace_id=origin_tid, - ) - return - - ctx.logger.info(f"Using configuration: {config_name}") - ctx.logger.info(f"Using build directory: {build_dir}") + chip = None if input_data.get("balltype"): + build_dir = input_data.get("output_dir") + if not build_dir: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "validation", + "error": "balltype mill requires output_dir", + "example": 'bbdev bebop verilator --verilog "--balltype foo --output-dir "', + }, + trace_id=origin_tid, + ) + return + os.makedirs(build_dir, exist_ok=True) command = ( f"mill -i __.test.runMain sims.verify.BallTopMain {input_data.get('balltype')} " + "--disable-annotation-unknown --strip-debug-info -O=debug " + f"--split-verilog -o={build_dir} " + ) + ctx.logger.info(f"Using build directory: {build_dir}") + result = await stream_run_logger_async( + cmd=command, + logger=ctx.logger, + cwd=arch_dir, + stdout_prefix="bebop verilator verilog", + stderr_prefix="bebop verilator verilog", ) + returncode = result.returncode + prepare_verilator_verilog(build_dir, arch_dir, ctx.logger) else: - command = f"mill -i __.test.runMain sims.verilator.Elaborate {config_name} " - - command += "--disable-annotation-unknown " - command += "--strip-debug-info " - command += "-O=debug " - command += f"--split-verilog -o={build_dir} " - - result = stream_run_logger( - cmd=command, - logger=ctx.logger, - cwd=arch_dir, - stdout_prefix="bebop verilator verilog", - stderr_prefix="bebop verilator verilog", - ) - - prepare_verilator_verilog(build_dir, arch_dir, ctx.logger) + try: + chip = require_chip(input_data) + build_dir, returncode = await run_chip_mill( + ctx, bbdir, chip, "verilog", "bebop verilator verilog", + input_data.get("output_dir"), + ) + except (ValueError, RuntimeError) as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "validation", + "error": str(error), + "example": 'bbdev bebop verilator --verilog "--chip toy"', + }, + trace_id=origin_tid, + ) + return + ctx.logger.info(f"Using chip: {chip}") output_status = check_verilog_output(build_dir) - if result.returncode != 0: + if returncode != 0: await check_result( ctx, - result.returncode, + returncode, continue_run=False, extra_fields={ "task": "verilog", - "config": config_name, + "chip": chip, "output_dir": build_dir, "output_status": output_status, }, @@ -138,7 +136,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "task": "verilog", "error": "invalid_verilog_output", - "config": config_name, + "chip": chip, "output_dir": build_dir, "output_status": output_status, }, @@ -148,14 +146,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result( ctx, - result.returncode, - continue_run=input_data.get("from_run_workflow", False), - extra_fields={"task": "verilog", "config": config_name, "output_dir": build_dir}, + returncode, + continue_run=input_data.get("from_run_workflow", False) and returncode == 0, + extra_fields={"task": "verilog", "chip": chip, "output_dir": build_dir}, trace_id=origin_tid, ) - # Continue routing to build if from run workflow - if input_data.get("from_run_workflow"): + if input_data.get("from_run_workflow") and returncode == 0: await ctx.enqueue( {"topic": "bebop.verilator.run.build", "data": {**input_data, "output_dir": build_dir, "vsrc_dir": build_dir, "task": "run"}} ) diff --git a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py index 6b733fddd..f5ae693ba 100644 --- a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py +++ b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py @@ -7,8 +7,9 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path, get_verilator_build_dir -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.path import get_buckyball_path +from utils.rtl import run_chip_mill from utils.event_common import check_result, get_origin_trace_id config = { @@ -16,118 +17,52 @@ "description": "Generate P2E Verilog via mill", "flows": ["bebop"], "triggers": [queue("bebop.p2e.verilog")], - "enqueues": [], + "enqueues": ["bebop.p2e.buildbitstream"], } -def cleanup_strays(arch_dir: str): - """Remove stray generated files from arch directory""" - for stray in ["P2EHarness.sv", "P2ETop.v", "P2ETopWrapper.sv"]: - path = os.path.join(arch_dir, stray) - if os.path.exists(path): - os.remove(path) - - -def normalize_p2e_timescale(build_dir: str, logger): - """Add timescale directive to generated Verilog files""" - patched = 0 - for root, _, files in os.walk(build_dir): - for name in files: - if not name.endswith((".v", ".sv")): - continue - - path = os.path.join(root, name) - with open(path, "r") as f: - content = f.read() - - if "`timescale" in content: - continue - - with open(path, "w") as f: - f.write("`timescale 1ns/1ps\n") - f.write(content) - patched += 1 - - logger.info(f"Normalized P2E timescale in {patched} generated Verilog files") - - async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - arch_dir = f"{bbdir}/arch" - config_name = input_data.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - ctx.logger.error("Missing required parameter: config") + try: + chip = require_chip(input_data) + build_dir, returncode = await run_chip_mill( + ctx, bbdir, chip, "p2e", "bebop p2e verilog", + input_data.get("output_dir"), + ) + except (ValueError, RuntimeError) as error: + ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_config"}, + extra_fields={"error": "missing_chip", "detail": str(error)}, trace_id=origin_tid, ) return - build_dir = get_verilator_build_dir(bbdir, config_name, input_data.get("output_dir")) - - ctx.logger.info(f"Using P2E configuration: {config_name}") + ctx.logger.info(f"Using P2E chip: {chip}") ctx.logger.info(f"Using P2E output directory: {build_dir}") - os.makedirs(build_dir, exist_ok=True) - cleanup_strays(arch_dir) - - common_firtool_opts = ( - "--disable-annotation-unknown " - "--strip-debug-info " - "-O=debug " - f"--split-verilog -o={build_dir}" - ) - - # Generate digital top Verilog - digital_command = ( - f"mill -i __.test.runMain sims.p2e.Elaborate {config_name} " - f"{common_firtool_opts}" - ) - digital_result = stream_run_logger( - cmd=digital_command, - logger=ctx.logger, - cwd=arch_dir, - stdout_prefix="bebop p2e digital verilog", - stderr_prefix="bebop p2e digital verilog", - ) - if digital_result.returncode != 0: - cleanup_strays(arch_dir) - await check_result( - ctx, - digital_result.returncode, - continue_run=False, - extra_fields={"task": "verilog", "step": "digital", "config": config_name}, - trace_id=origin_tid, - ) - return - - # Generate P2E top wrapper Verilog - top_command = ( - f"mill -i __.test.runMain sims.p2e.ElaborateP2ETop " - f"{common_firtool_opts}" - ) - top_result = stream_run_logger( - cmd=top_command, - logger=ctx.logger, - cwd=arch_dir, - stdout_prefix="bebop p2e top verilog", - stderr_prefix="bebop p2e top verilog", - ) - - cleanup_strays(arch_dir) - if top_result.returncode == 0: - normalize_p2e_timescale(build_dir, ctx.logger) - + from_reg = bool(input_data.get("from_regression_buildbitstream")) await check_result( ctx, - top_result.returncode, - continue_run=False, + returncode, + continue_run=from_reg and returncode == 0, extra_fields={ "task": "verilog", - "config": config_name, + "chip": chip, "output_dir": build_dir, "top_module": "P2ETop", }, trace_id=origin_tid, ) + if from_reg and returncode == 0: + await ctx.enqueue( + { + "topic": "bebop.p2e.buildbitstream", + "data": { + **input_data, + "vsrc_dir": build_dir, + "from_regression_buildbitstream": True, + "_trace_id": origin_tid, + }, + } + ) diff --git a/bbdev/api/steps/mill/verilog.py b/bbdev/api/steps/mill/verilog.py deleted file mode 100644 index 3a255a303..000000000 --- a/bbdev/api/steps/mill/verilog.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Shared RTL elaboration commands for synthesis-oriented flows.""" - - -def seq_mem_elaboration_command(elaborate_config: str, build_dir: str, mem_conf: str) -> str: - """Emit split Verilog with behavioral memories replaced by named modules.""" - return ( - f"mill -i __.test.runMain sims.verilator.Elaborate {elaborate_config} " - "--disable-annotation-unknown --strip-debug-info -O=debug " - "-lowering-options=disallowLocalVariables " - f"--repl-seq-mem --repl-seq-mem-file={mem_conf} " - f"--split-verilog -o={build_dir}" - ) diff --git a/bbdev/api/steps/regression/00_buildbitstream_api.step.py b/bbdev/api/steps/regression/00_buildbitstream_api.step.py new file mode 100644 index 000000000..1ef76e640 --- /dev/null +++ b/bbdev/api/steps/regression/00_buildbitstream_api.step.py @@ -0,0 +1,37 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.chip import require_chip + +config = { + "name": "regression-buildbitstream-api", + "description": "Chain p2e verilog and buildbitstream for regression", + "flows": ["regression"], + "triggers": [api("POST", "/regression/buildbitstream")], + "enqueues": ["regression.buildbitstream"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + chip = require_chip(body) + except ValueError as exc: + return ApiResponse(status=400, body={"error": str(exc)}) + + data = { + "chip": chip, + "_trace_id": ctx.trace_id, + } + output_dir = body.get("output_dir") + if output_dir is None: + output_dir = body.get("output-dir") + if output_dir is not None: + if not isinstance(output_dir, str) or not output_dir: + return ApiResponse( + status=400, + body={"error": "output_dir must be a non-empty path"}, + ) + data["output_dir"] = output_dir + + await ctx.enqueue({"topic": "regression.buildbitstream", "data": data}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/regression/00_buildbitstream_event.step.py b/bbdev/api/steps/regression/00_buildbitstream_event.step.py new file mode 100644 index 000000000..ddc07c1f7 --- /dev/null +++ b/bbdev/api/steps/regression/00_buildbitstream_event.step.py @@ -0,0 +1,56 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.chip import require_chip +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "regression-buildbitstream", + "description": "Enqueue p2e verilog then buildbitstream for regression", + "flows": ["regression"], + "triggers": [queue("regression.buildbitstream")], + "enqueues": ["bebop.p2e.verilog"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + try: + chip = require_chip(input_data) + except ValueError as exc: + ctx.logger.error(str(exc)) + extra = {"error": "missing_chip"} if str(exc).startswith("Missing required parameter: --chip") else {"error": str(exc)} + await check_result( + ctx, 1, continue_run=False, + extra_fields=extra, + trace_id=origin_tid, + ) + return + + await check_result( + ctx, + 0, + continue_run=True, + extra_fields={ + "task": "regression.buildbitstream", + "chip": chip, + }, + trace_id=origin_tid, + ) + + data = { + "chip": chip, + "from_regression_buildbitstream": True, + "_trace_id": origin_tid, + } + output_dir = input_data.get("output_dir") or input_data.get("output-dir") + if output_dir: + data["build_dir"] = output_dir + + await ctx.enqueue({"topic": "bebop.p2e.verilog", "data": data}) diff --git a/bbdev/api/steps/regression/01_check_api.step.py b/bbdev/api/steps/regression/01_check_api.step.py new file mode 100644 index 000000000..db1019125 --- /dev/null +++ b/bbdev/api/steps/regression/01_check_api.step.py @@ -0,0 +1,48 @@ +import os + +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "regression-check-api", + "description": "Run p2e pk-tests and record accuracy for regression", + "flows": ["regression"], + "triggers": [api("POST", "/regression/check")], + "enqueues": ["regression.check"], +} + + +def _req_str(body: dict, key: str): + value = body.get(key) + if not isinstance(value, str) or not value or value == "None": + return None + return value + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + chip = _req_str(body, "chip") + if chip is None: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --chip must be specified"}, + ) + bitstream = _req_str(body, "bitstream") + if bitstream is None: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --bitstream must be specified"}, + ) + if not os.path.isfile(bitstream): + return ApiResponse( + status=400, + body={"error": f"bitstream file not found: {bitstream}"}, + ) + + data = { + "chip": chip, + "bitstream": bitstream, + "_trace_id": ctx.trace_id, + } + + await ctx.enqueue({"topic": "regression.check", "data": data}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/regression/01_check_event.step.py b/bbdev/api/steps/regression/01_check_event.step.py new file mode 100644 index 000000000..3c50076bb --- /dev/null +++ b/bbdev/api/steps/regression/01_check_event.step.py @@ -0,0 +1,70 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "regression-check", + "description": "Enqueue p2e pk-tests batch for regression accuracy", + "flows": ["regression"], + "triggers": [queue("regression.check")], + "enqueues": ["bebop.p2e.batch"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + chip = input_data.get("chip") + bitstream = input_data.get("bitstream") + if not isinstance(chip, str) or not chip or chip == "None": + ctx.logger.error("Missing required parameter: chip") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_chip"}, + trace_id=origin_tid, + ) + return + if not isinstance(bitstream, str) or not bitstream or bitstream == "None": + ctx.logger.error("Missing required parameter: bitstream") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_bitstream"}, + trace_id=origin_tid, + ) + return + if not os.path.isfile(bitstream): + ctx.logger.error(f"bitstream file not found: {bitstream}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "bitstream_not_found", "bitstream": bitstream}, + trace_id=origin_tid, + ) + return + + await check_result( + ctx, + 0, + continue_run=True, + extra_fields={ + "task": "regression.check", + "chip": chip, + "bitstream": bitstream, + }, + trace_id=origin_tid, + ) + + data = { + "chip": chip, + "bitstream": bitstream, + "test": "pk-tests", + "from_regression_check": True, + "_trace_id": origin_tid, + } + + await ctx.enqueue({"topic": "bebop.p2e.batch", "data": data}) diff --git a/bbdev/api/steps/regression/02_eval_performance_api.step.py b/bbdev/api/steps/regression/02_eval_performance_api.step.py new file mode 100644 index 000000000..b4bcaf987 --- /dev/null +++ b/bbdev/api/steps/regression/02_eval_performance_api.step.py @@ -0,0 +1,48 @@ +import os + +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "regression-eval-performance-api", + "description": "Run p2e model perfetto latency for regression", + "flows": ["regression"], + "triggers": [api("POST", "/regression/eval-performance")], + "enqueues": ["regression.eval-performance"], +} + + +def _req_str(body: dict, key: str): + value = body.get(key) + if not isinstance(value, str) or not value or value == "None": + return None + return value + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + chip = _req_str(body, "chip") + if chip is None: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --chip must be specified"}, + ) + bitstream = _req_str(body, "bitstream") + if bitstream is None: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --bitstream must be specified"}, + ) + if not os.path.isfile(bitstream): + return ApiResponse( + status=400, + body={"error": f"bitstream file not found: {bitstream}"}, + ) + + data = { + "chip": chip, + "bitstream": bitstream, + "_trace_id": ctx.trace_id, + } + + await ctx.enqueue({"topic": "regression.eval-performance", "data": data}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/regression/02_eval_performance_event.step.py b/bbdev/api/steps/regression/02_eval_performance_event.step.py new file mode 100644 index 000000000..12b399e61 --- /dev/null +++ b/bbdev/api/steps/regression/02_eval_performance_event.step.py @@ -0,0 +1,396 @@ +"""regression --eval-performance event. + +Per model (synchronous shells inside one regression event): + 1. workload build --chip --model + 2. bridge chip layout output -> flat path the kernel expects + 3. kernel build --model with dataset packed from e2e/datasets/ + -> fw_payload-.hex; /init runs binary with --dataset/--max-samples + 4. p2e runworkload; parse per-model accuracy from uart (top1= / map=) + 5. require non-empty log_dir/trace/cycle/; run perfetto.py; cycles_i + +After all models: cycles = mean(cycles_i); accuracy = mean(accuracy_i); +merge_metrics(cycles=..., accuracy=..., models=[...]). +Any model failure aborts the whole stage (no partial average). +""" +import importlib.util +import json +import os +import shlex +import sys +from datetime import datetime +from pathlib import Path + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.build import build_workload +from utils.path import get_buckyball_path, get_p2e_build_dir, get_run_log_dir +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id + +regression_scripts = os.path.join(os.path.dirname(__file__), "scripts") +if regression_scripts not in sys.path: + sys.path.insert(0, regression_scripts) +from model_layout import bridge_model_layout, perfetto_inputs +from models_toml import load_eval_models +from accuracy import accuracy_from_uart, load_eval_accuracy, mean_accuracy +from perfetto_latency import e2e_cycles_from_perfetto, mean_cycles +from result import merge_metrics + +config = { + "name": "regression-eval-performance", + "description": "Per-model p2e perfetto cycles for regression", + "flows": ["regression"], + "triggers": [queue("regression.eval-performance")], + "enqueues": [], +} + + +def _load_sibling(rel): + path = os.path.normpath(os.path.join(os.path.dirname(__file__), rel)) + name = os.path.basename(path).replace(".", "_") + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load sibling step: {path}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +_workload = _load_sibling("../workload/01_build_event.step.py") +_kernel = _load_sibling("../kernel/01_build_event.step.py") +_p2e = _load_sibling("../bebop/p2e/03_runworkload_event.step.py") + +MODEL_LAYOUT = _workload.MODEL_LAYOUT +PERFETTO_SCRIPT = ( + Path(get_buckyball_path()) + / "bb-tests" / "workloads" / "src" / "ModelTest" + / "e2e" / "framework" / "trace" / "perfetto.py" +) + + +def _fail(ctx, origin_tid, error, **extra): + fields = {"task": "regression.eval-performance", "error": error} + fields.update(extra) + return check_result(ctx, 1, continue_run=False, extra_fields=fields, trace_id=origin_tid) + + +def _workload_build(bbdir, chip, model): + model_key = model.lower() + if MODEL_LAYOUT.get(model_key) is None: + raise ValueError(f"Unknown model: {model}") + build_workload(bbdir, chip, model=model_key) + + +def _kernel_build_cmds(bbdir, model, dataset=""): + kernel_src = os.path.join(bbdir, "bb-tests", "workloads", "lib", "kernel") + hart_params = {"visible": 64, "total": 64, "hidden_base": 64} + kernel_build = _kernel.kernel_build_dir(bbdir, hart_params, model=model) + ds_arg = "" + if dataset: + ds_arg = f" -DBUCKYBALL_MODEL_DATASET={shlex.quote(dataset)}" + configure = ( + f"cmake -B {kernel_build} -S {kernel_src} " + f"-DBUCKYBALL_VISIBLE_HART_COUNT=64 " + f"-DBUCKYBALL_TOTAL_HART_COUNT=64 " + f"-DBUCKYBALL_HIDDEN_HART_BASE=64 " + f"-DBUCKYBALL_KERNEL_MODEL={model} " + f"-DBUCKYBALL_KERNEL_CHIP= " + f"-DBUCKYBALL_KERNEL_INTERACTIVE=OFF" + f"{ds_arg}" + ) + build = f"cmake --build {kernel_build} --target kernel-build" + payload = _kernel.fw_payload_name(hart_params, model=model) + fw_bin = os.path.join(bbdir, "bb-tests", "output", "kernel", f"{payload}.bin") + fw_hex = os.path.join(bbdir, "bb-tests", "output", "kernel", f"{payload}.hex") + return configure, build, fw_bin, fw_hex + + +def _p2e_run_cmds(bbdir, bitstream, image_name, chip, input_data): + image_path = _p2e.resolve_image(bbdir, image_name, chip) + if not image_path: + raise FileNotFoundError( + f"image .hex not found for name: {image_name} " + f"(searched bb-tests/output/{chip}/workloads/)" + ) + bitstream = os.path.abspath(bitstream) + build_dir = os.path.dirname(os.path.dirname(bitstream)) + if not os.path.isdir(build_dir): + raise FileNotFoundError( + f"P2E build case not found for bitstream: {build_dir}" + ) + multi_fpga = bool(input_data.get("multi-fpga", False)) + if not multi_fpga and _p2e.case_uses_multi_fpga(build_dir): + multi_fpga = True + vsrc_dir = get_p2e_build_dir(bbdir, chip, input_data.get("vsrc_dir")) + if not os.path.isdir(vsrc_dir): + raise FileNotFoundError(f"VSRC_PATH does not exist for P2E runtime: {vsrc_dir}") + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + log_dir = get_run_log_dir(bbdir, chip, "p2e", timestamp, "p2e", image_name, input_data.get("vsrc_dir")) + os.makedirs(log_dir, exist_ok=True) + runtime_cmd = ( + f"env BEBOP_P2E_RUNTIME_ONLY=1 BEBOP_P2E_REBUILD_RUNTIME=1 " + f'cargo run --release --features p2e -- build p2e ' + f'--rtl-dir="{vsrc_dir}" ' + f'--out-dir="{build_dir}"' + ) + run_cmd = ( + f"cargo run --release --features p2e " + f"--config=\"env.OUT_PATH='{build_dir}'\" " + f"-- run p2e " + f'--image="{image_path}" ' + f'--bitstream="{bitstream}" ' + f'--log-dir="{log_dir}"' + ) + if multi_fpga: + run_cmd += " --multi-fpga" + return runtime_cmd, run_cmd, f"{bbdir}/bebop", log_dir, build_dir + + +def _perfetto_cmd(trace_dir, trace_toml, mlir_files): + args = [str(trace_dir), str(trace_toml)] + for mlir in mlir_files: + args += ["--mlir", str(mlir)] + return ( + f"{sys.executable} {shlex.quote(str(PERFETTO_SCRIPT))} " + + " ".join(shlex.quote(a) for a in args) + ) + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + + chip = input_data.get("chip") + if not isinstance(chip, str) or not chip or chip == "None": + ctx.logger.error("Missing required parameter: chip") + await _fail(ctx, origin_tid, "missing_chip") + return + bitstream = input_data.get("bitstream") + if not isinstance(bitstream, str) or not bitstream or bitstream == "None": + ctx.logger.error("Missing required parameter: bitstream") + await _fail(ctx, origin_tid, "missing_bitstream") + return + if not os.path.isfile(bitstream): + ctx.logger.error(f"bitstream file not found: {bitstream}") + await _fail(ctx, origin_tid, "bitstream_not_found", bitstream=bitstream) + return + + try: + models = load_eval_models(chip, bbdir) + acc_specs = load_eval_accuracy(chip, bbdir, models) + except ValueError as e: + ctx.logger.error(str(e)) + await _fail(ctx, origin_tid, "models_toml_invalid", chip=chip) + return + + await check_result( + ctx, 0, continue_run=True, + extra_fields={ + "task": "regression.eval-performance", + "chip": chip, "bitstream": bitstream, "models": models, + }, + trace_id=origin_tid, + ) + + model_results = [] + for model in models: + model_key = model.lower() + if MODEL_LAYOUT.get(model_key) is None: + ctx.logger.error(f"Unknown model: {model}") + await _fail(ctx, origin_tid, "unknown_model", model=model) + return + + ctx.logger.info(f"[eval-performance] model {model}: workload build") + try: + _workload_build(bbdir, chip, model) + except (ValueError, RuntimeError) as e: + ctx.logger.error(str(e)) + await _fail(ctx, origin_tid, "workload_build_cmd", model=model) + return + + ctx.logger.info(f"[eval-performance] model {model}: bridge layout") + try: + bridge_model_layout(bbdir, chip, model) + except (FileNotFoundError, FileExistsError) as e: + ctx.logger.error(str(e)) + await _fail(ctx, origin_tid, "layout_bridge_failed", model=model) + return + + ctx.logger.info(f"[eval-performance] model {model}: kernel build") + spec = acc_specs[model] + try: + k_cfg, k_build, fw_bin, fw_hex = _kernel_build_cmds( + bbdir, model, dataset=spec["dataset"], + ) + except ValueError as e: + ctx.logger.error(str(e)) + await _fail(ctx, origin_tid, "kernel_build_cmd", model=model) + return + cfg_result = await stream_run_logger_async( + cmd=k_cfg, logger=ctx.logger, + stdout_prefix=f"kernel configure {model}", + stderr_prefix=f"kernel configure {model}", + ) + if cfg_result.returncode != 0: + await _fail(ctx, origin_tid, "kernel_configure_failed", + model=model, returncode=cfg_result.returncode) + return + build_result = await stream_run_logger_async( + cmd=k_build, logger=ctx.logger, + stdout_prefix=f"kernel build {model}", + stderr_prefix=f"kernel build {model}", + ) + if build_result.returncode != 0: + await _fail(ctx, origin_tid, "kernel_build_failed", + model=model, returncode=build_result.returncode) + return + if not os.path.isfile(fw_bin): + ctx.logger.error(f"fw_payload bin not found: {fw_bin}") + await _fail(ctx, origin_tid, "kernel_bin_missing", + model=model, fw_bin=fw_bin) + return + if not _kernel.bin_to_hex(fw_bin, fw_hex, base_address=0x80000000): + ctx.logger.error(f"bin_to_hex failed: {fw_bin} -> {fw_hex}") + await _fail(ctx, origin_tid, "kernel_tohex_failed", model=model) + return + + ctx.logger.info(f"[eval-performance] model {model}: p2e runworkload") + image_name = os.path.splitext(os.path.basename(fw_hex))[0] + try: + runtime_cmd, run_cmd, bebop_cwd, log_dir, build_dir = _p2e_run_cmds( + bbdir, bitstream, image_name, chip, input_data + ) + except FileNotFoundError as e: + ctx.logger.error(str(e)) + err = "p2e_run_cmd" + extra = {"model": model} + if str(e).startswith("P2E build case not found"): + err = "build_dir_not_found" + extra["build_dir"] = os.path.dirname( + os.path.dirname(os.path.abspath(bitstream)) + ) + await _fail(ctx, origin_tid, err, **extra) + return + rt_result = await stream_run_logger_async( + cmd=runtime_cmd, logger=ctx.logger, cwd=bebop_cwd, + stdout_prefix=f"p2e runtime {model}", + stderr_prefix=f"p2e runtime {model}", + ) + rtcfg = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") + libvctb = os.path.join( + build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so" + ) + if rt_result.returncode != 0 or not all( + os.path.isfile(p) for p in (rtcfg, libvctb) + ): + await _fail(ctx, origin_tid, "p2e_runtime_failed", + model=model, returncode=rt_result.returncode) + return + run_result = await stream_run_logger_async( + cmd=run_cmd, logger=ctx.logger, cwd=bebop_cwd, + stdout_prefix=f"p2e run {model}", + stderr_prefix=f"p2e run {model}", + ) + if run_result.returncode != 0: + await _fail(ctx, origin_tid, "p2e_run_failed", + model=model, returncode=run_result.returncode, + log_dir=log_dir) + return + + uart_path = Path(log_dir) / "uart_hart_0.log" + if not uart_path.is_file(): + uart_path = Path(log_dir) / "uart.log" + if not uart_path.is_file(): + ctx.logger.error(f"uart log missing under {log_dir}") + await _fail(ctx, origin_tid, "uart_missing", model=model, log_dir=log_dir) + return + try: + acc_i = accuracy_from_uart(spec["metric"], uart_path.read_text()) + except ValueError as e: + ctx.logger.error(f"accuracy parse failed for {model}: {e}") + await _fail(ctx, origin_tid, "accuracy_unparsable", model=model) + return + + cycle_dir = Path(log_dir) / "trace" / "cycle" + if not cycle_dir.is_dir(): + ctx.logger.error(f"cycle trace dir missing: {cycle_dir}") + await _fail(ctx, origin_tid, "cycle_trace_missing", + model=model, cycle_dir=str(cycle_dir)) + return + cycle_files = [ + p for p in cycle_dir.iterdir() + if p.is_file() and p.name.startswith("trace-") + and p.name.endswith(".txt") + ] + if not cycle_files: + ctx.logger.error(f"cycle trace dir empty: {cycle_dir}") + await _fail(ctx, origin_tid, "cycle_trace_empty", + model=model, cycle_dir=str(cycle_dir)) + return + + ctx.logger.info(f"[eval-performance] model {model}: perfetto") + try: + pinputs = perfetto_inputs(bbdir, chip, model) + except (KeyError, FileNotFoundError) as e: + ctx.logger.error(str(e)) + await _fail(ctx, origin_tid, "perfetto_inputs_missing", model=model) + return + trace_dir = Path(log_dir) / "trace" + perf_cmd = _perfetto_cmd( + trace_dir, pinputs["trace_toml"], pinputs["mlir_files"] + ) + perf_out = trace_dir / "perfetto.json" + perf_result = await stream_run_logger_async( + cmd=perf_cmd, logger=ctx.logger, + stdout_prefix=f"perfetto {model}", + stderr_prefix=f"perfetto {model}", + ) + if perf_result.returncode != 0 or not perf_out.is_file(): + await _fail(ctx, origin_tid, "perfetto_failed", + model=model, returncode=perf_result.returncode) + return + try: + perf_data = json.loads(perf_out.read_text()) + except (OSError, json.JSONDecodeError) as e: + ctx.logger.error(f"perfetto json unreadable: {e}") + await _fail(ctx, origin_tid, "perfetto_json_unreadable", model=model) + return + try: + cycles_i = e2e_cycles_from_perfetto(perf_data) + except ValueError as e: + ctx.logger.error(str(e)) + await _fail(ctx, origin_tid, "perfetto_cycles_failed", model=model) + return + + dest = Path(log_dir) / f"perfetto-{model}.json" + dest.write_text(json.dumps(perf_data, indent=2)) + model_results.append({ + "name": model, + "cycles": cycles_i, + "accuracy": acc_i, + "metric": spec["metric"], + "dataset": spec["dataset"], + "perfetto": str(dest), + }) + + try: + cycles = mean_cycles([r["cycles"] for r in model_results]) + accuracy = mean_accuracy([r["accuracy"] for r in model_results]) + except ValueError as e: + ctx.logger.error(str(e)) + await _fail(ctx, origin_tid, "cycles_mean_failed", models=model_results) + return + merge_metrics(bbdir, cycles=cycles, accuracy=accuracy, models=model_results) + await check_result( + ctx, 0, continue_run=False, + extra_fields={ + "task": "regression.eval-performance", + "chip": chip, "cycles": cycles, "accuracy": accuracy, + "models": model_results, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/regression/03_eval_area_power_api.step.py b/bbdev/api/steps/regression/03_eval_area_power_api.step.py new file mode 100644 index 000000000..664b32511 --- /dev/null +++ b/bbdev/api/steps/regression/03_eval_area_power_api.step.py @@ -0,0 +1,45 @@ +import os + +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.chip import require_chip +from utils.path import check_dc_rtl_args, get_buckyball_path + +config = { + "name": "regression-eval-area-power-api", + "description": "DC area+freq for regression", + "flows": ["regression"], + "triggers": [api("POST", "/regression/eval-area-power")], + "enqueues": ["regression.eval-area-power"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + chip = require_chip(body) + subset = {"chip": chip} + if "top" in body: + subset["top"] = body.get("top") + check_dc_rtl_args(subset) + except ValueError as exc: + return ApiResponse(status=400, body={"error": str(exc)}) + + bbdir = get_buckyball_path() + tapeout = os.path.join(bbdir, "examples", "chips", chip, "tapeout") + if not os.path.isdir(tapeout): + return ApiResponse( + status=400, + body={"error": f"chip {chip} has no tapeout directory: {tapeout}"}, + ) + + data = { + "chip": chip, + "_trace_id": ctx.trace_id, + } + top = body.get("top") + if isinstance(top, str) and top: + data["top"] = top + + await ctx.enqueue({"topic": "regression.eval-area-power", "data": data}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/regression/03_eval_area_power_event.step.py b/bbdev/api/steps/regression/03_eval_area_power_event.step.py new file mode 100644 index 000000000..c7a98a302 --- /dev/null +++ b/bbdev/api/steps/regression/03_eval_area_power_event.step.py @@ -0,0 +1,71 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.chip import require_chip +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path + +config = { + "name": "regression-eval-area-power", + "description": "Enqueue DC area workflow for regression area+freq", + "flows": ["regression"], + "triggers": [queue("regression.eval-area-power")], + "enqueues": ["dc.verilog"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + try: + chip = require_chip(input_data) + except ValueError as exc: + ctx.logger.error(str(exc)) + extra = {"error": "missing_chip"} if str(exc).startswith("Missing required parameter: --chip") else {"error": str(exc)} + await check_result( + ctx, 1, continue_run=False, + extra_fields=extra, + trace_id=origin_tid, + ) + return + tapeout = os.path.join(bbdir, "examples", "chips", chip, "tapeout") + if not os.path.isdir(tapeout): + ctx.logger.error(f"chip {chip} has no tapeout directory: {tapeout}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_tapeout", "chip": chip, "tapeout": tapeout}, + trace_id=origin_tid, + ) + return + + top = input_data.get("top") or "DigitalTop" + + await check_result( + ctx, + 0, + continue_run=True, + extra_fields={ + "task": "regression.eval-area-power", + "chip": chip, + }, + trace_id=origin_tid, + ) + + await ctx.enqueue( + { + "topic": "dc.verilog", + "data": { + "chip": chip, + "top": top, + "from_area_workflow": True, + "from_regression_area_power": True, + "_trace_id": origin_tid, + }, + } + ) diff --git a/bbdev/api/steps/regression/__init__.py b/bbdev/api/steps/regression/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/bbdev/api/steps/regression/scripts/accuracy.py b/bbdev/api/steps/regression/scripts/accuracy.py new file mode 100644 index 000000000..f19cdee10 --- /dev/null +++ b/bbdev/api/steps/regression/scripts/accuracy.py @@ -0,0 +1,91 @@ +"""Per-model accuracy parsing for regression eval-performance. + +UART contracts (fail loud if missing / malformed): + top1 -> line "top1=/" + map -> line "map=" +""" +from __future__ import annotations + +import re +from typing import Any + +_TOP1 = re.compile(r"^top1=(\d+)/(\d+)\s*$", re.M) +_MAP = re.compile(r"^map=([0-9]*\.?[0-9]+)\s*$", re.M) + + +def accuracy_from_uart(metric: str, uart: str) -> float: + if not isinstance(metric, str) or not metric: + raise ValueError("accuracy metric must be a non-empty string") + if not isinstance(uart, str) or not uart: + raise ValueError("uart log is empty; cannot parse accuracy") + m = metric.lower() + if m == "top1": + return _top1(uart) + if m == "map": + return _map(uart) + raise ValueError(f"unknown accuracy metric: {metric}") + + +def _top1(uart: str) -> float: + hits = _TOP1.findall(uart) + if not hits: + raise ValueError("uart missing top1=/ line") + correct_s, total_s = hits[-1] + correct = int(correct_s) + total = int(total_s) + if total <= 0: + raise ValueError(f"top1 total must be > 0, got {total}") + if correct < 0 or correct > total: + raise ValueError(f"top1 correct out of range: {correct}/{total}") + return correct / total + + +def _map(uart: str) -> float: + hits = _MAP.findall(uart) + if not hits: + raise ValueError("uart missing map= line") + val = float(hits[-1]) + if val < 0.0 or val > 1.0: + raise ValueError(f"map out of range [0,1]: {val}") + return val + + +def mean_accuracy(values: list[float]) -> float: + if not values: + raise ValueError("cannot mean empty accuracy list") + return sum(values) / len(values) + + +def load_eval_accuracy(chip: str, bbdir: str, models: list[str]) -> dict[str, dict[str, Any]]: + """Load [eval.accuracy.] for each model. Missing section -> error.""" + import tomllib + from pathlib import Path + + path = Path(bbdir) / "examples" / "chips" / chip / "regression" / "eval" / "models.toml" + if not path.is_file(): + raise ValueError(f"eval models toml does not exist: {path}") + with open(path, "rb") as f: + data = tomllib.load(f) + eval_sec = data.get("eval") + if not isinstance(eval_sec, dict): + raise ValueError(f"missing [eval] section: {path}") + acc_root = eval_sec.get("accuracy") + if not isinstance(acc_root, dict): + raise ValueError(f"missing [eval.accuracy] tables: {path}") + out: dict[str, dict[str, Any]] = {} + for name in models: + spec = acc_root.get(name) + if not isinstance(spec, dict): + raise ValueError(f"missing [eval.accuracy.{name}]: {path}") + metric = spec.get("metric") + dataset = spec.get("dataset") + if not isinstance(metric, str) or not metric: + raise ValueError(f"[eval.accuracy.{name}].metric must be non-empty string") + if not isinstance(dataset, str) or not dataset: + raise ValueError(f"[eval.accuracy.{name}].dataset must be non-empty string") + if dataset.startswith("/") or ".." in dataset.split("/"): + raise ValueError( + f"[eval.accuracy.{name}].dataset must be a relative path without '..': {dataset}" + ) + out[name] = {"metric": metric, "dataset": dataset} + return out diff --git a/bbdev/api/steps/regression/scripts/dc_area.py b/bbdev/api/steps/regression/scripts/dc_area.py new file mode 100644 index 000000000..b49b7f92a --- /dev/null +++ b/bbdev/api/steps/regression/scripts/dc_area.py @@ -0,0 +1,10 @@ +import re + +_AREA = re.compile(r"Total cell area:\s+([\d.]+)") + + +def area_mm2_from_rpt(text: str) -> float: + m = _AREA.search(text) + if not m: + raise ValueError("Total cell area not found") + return float(m.group(1)) / 1e6 diff --git a/bbdev/api/steps/regression/scripts/model_layout.py b/bbdev/api/steps/regression/scripts/model_layout.py new file mode 100644 index 000000000..e8fe08492 --- /dev/null +++ b/bbdev/api/steps/regression/scripts/model_layout.py @@ -0,0 +1,123 @@ +"""Bridge pebble chip layout output -> flat archs/buckyball/ path the kernel expects. + +Workload `--chip --model ` writes under + bb-tests/output//workloads/src/ModelTest/e2e/models/archs/buckyball/// +Kernel `--model ` expects the flat + bb-tests/output//workloads/src/ModelTest/e2e/models/archs/buckyball// +""" +import os +from pathlib import Path + +MODEL_LAYOUT = { + "lenet": "LeNet", + "mobilenet": "MobileNetV3", + "resnet": "ResNet18", + "yolo": "YOLO26", + "bert": "Bert", + "qwen3": "Qwen3", + "gemma4": "Gemma4", + "deepseekr1": "DeepSeekR1", + "llama2": "llama2", + "stable-diffusion": "StableDiffusion", + "whisper": "Whisper", + "buddynext": "BuddyNext", +} + +MODEL_PERFETTO = { + "lenet": { + "trace_toml": "bb-tests/workloads/src/ModelTest/e2e/models/models/LeNet/trace/trace-nodes.toml", + "mlir_files": [ + "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/LeNet/subgraph0_linalg.mlir", + "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/LeNet/subgraph0_buckyball.mlir", + ], + }, + "mobilenet": { + "trace_toml": "bb-tests/workloads/src/ModelTest/e2e/models/models/MobileNetV3/trace/trace.toml", + "mlir_files": [ + "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/MobileNetV3/subgraph0_linalg.mlir", + "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/MobileNetV3/subgraph0_buckyball.mlir", + ], + }, + "resnet": { + "trace_toml": "bb-tests/workloads/src/ModelTest/e2e/models/models/ResNet18/trace/trace.toml", + "mlir_files": [ + "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/ResNet18/subgraph0_linalg.mlir", + "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/ResNet18/subgraph0_buckyball.mlir", + ], + }, + "yolo": { + "trace_toml": "bb-tests/workloads/src/ModelTest/e2e/models/models/YOLO26/trace/trace.toml", + "mlir_files": [ + "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/YOLO26/subgraph0_linalg.mlir", + "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/YOLO26/subgraph0_buckyball.mlir", + ], + }, +} + + +def _archs_root(bbdir: str, chip: str) -> Path: + return ( + Path(bbdir) + / "bb-tests" + / "output" + / chip + / "workloads" + / "src" + / "ModelTest" + / "e2e" + / "models" + / "archs" + / "buckyball" + ) + + +def layout_name(model: str) -> str: + layout = MODEL_LAYOUT.get(model.lower()) + if layout is None: + raise ValueError(f"missing layout mapping for model: {model}") + return layout + + +def chip_output_dir(bbdir: str, chip: str, model: str) -> Path: + return _archs_root(bbdir, chip) / chip / layout_name(model) + + +def flat_output_dir(bbdir: str, chip: str, model: str) -> Path: + return _archs_root(bbdir, chip) / layout_name(model) + + +def bridge_model_layout(bbdir: str, chip: str, model: str) -> Path: + source = chip_output_dir(bbdir, chip, model) + if not source.is_dir(): + raise FileNotFoundError( + f"chip layout output dir missing for model '{model}' on chip '{chip}': {source}" + ) + flat = flat_output_dir(bbdir, chip, model) + if flat.is_symlink(): + target = os.readlink(flat) + if os.path.abspath(target) == os.path.abspath(str(source)): + return flat + flat.unlink() + elif flat.exists(): + raise FileExistsError( + f"refusing to clobber existing non-symlink at flat layout path: {flat}" + ) + flat.parent.mkdir(parents=True, exist_ok=True) + os.symlink(str(source), str(flat)) + return flat + + +def perfetto_inputs(bbdir: str, chip: str, model: str) -> dict: + spec = MODEL_PERFETTO.get(model.lower()) + if spec is None: + raise KeyError(f"no perfetto spec for model: {model}") + trace_toml = Path(bbdir) / spec["trace_toml"] + if not trace_toml.is_file(): + raise FileNotFoundError(f"perfetto trace toml missing: {trace_toml}") + mlir_files = [] + for rel in spec["mlir_files"]: + path = Path(bbdir) / rel.replace("{chip}", chip) + if not path.is_file(): + raise FileNotFoundError(f"perfetto mlir file missing: {path}") + mlir_files.append(path) + return {"trace_toml": trace_toml, "mlir_files": mlir_files} diff --git a/bbdev/api/steps/regression/scripts/models_toml.py b/bbdev/api/steps/regression/scripts/models_toml.py new file mode 100644 index 000000000..1cf98ca7b --- /dev/null +++ b/bbdev/api/steps/regression/scripts/models_toml.py @@ -0,0 +1,20 @@ +import tomllib +from pathlib import Path + + +def load_eval_models(chip: str, bbdir: str) -> list[str]: + path = Path(bbdir) / "examples" / "chips" / chip / "regression" / "eval" / "models.toml" + if not path.is_file(): + raise ValueError(f"eval models toml does not exist: {path}") + with open(path, "rb") as f: + data = tomllib.load(f) + eval_sec = data.get("eval") + if not isinstance(eval_sec, dict): + raise ValueError(f"missing [eval] section: {path}") + models = eval_sec.get("models") + if not isinstance(models, list) or not models: + raise ValueError(f"[eval].models must be a non-empty list: {path}") + for name in models: + if not isinstance(name, str) or not name: + raise ValueError(f"[eval].models entries must be non-empty strings: {path}") + return models diff --git a/bbdev/api/steps/regression/scripts/perfetto_latency.py b/bbdev/api/steps/regression/scripts/perfetto_latency.py new file mode 100644 index 000000000..7956db138 --- /dev/null +++ b/bbdev/api/steps/regression/scripts/perfetto_latency.py @@ -0,0 +1,34 @@ +def _cycle_int(value, label: str) -> int: + try: + num = float(value) + except (TypeError, ValueError) as e: + raise ValueError(f"{label} is not a number: {value!r}") from e + if num != int(num): + raise ValueError(f"{label} is not an integer cycle count: {value!r}") + return int(num) + + +def e2e_cycles_from_perfetto(data: dict) -> int: + """End-to-end cycle count: min(ts) .. max(ts+dur) over complete (ph=X) events. + + Perfetto ts/dur in this stack are cycle offsets (see elapsed_cycle), not wall ns. + """ + events = data.get("traceEvents") + if not isinstance(events, list): + raise ValueError("perfetto missing traceEvents list") + xs = [e for e in events if isinstance(e, dict) and e.get("ph") == "X"] + if not xs: + raise ValueError("perfetto has no complete (ph=X) events") + t0 = min(_cycle_int(e["ts"], "ts") for e in xs) + t1 = max(_cycle_int(e["ts"], "ts") + _cycle_int(e["dur"], "dur") for e in xs) + return t1 - t0 + + +def mean_cycles(values: list[int]) -> int: + if not values: + raise ValueError("mean_cycles requires a non-empty list") + total = sum(values) + n = len(values) + if total % n != 0: + raise ValueError(f"mean cycles not integral: sum={total} n={n} values={values}") + return total // n diff --git a/bbdev/api/steps/regression/scripts/result.py b/bbdev/api/steps/regression/scripts/result.py new file mode 100644 index 000000000..b09f0c63d --- /dev/null +++ b/bbdev/api/steps/regression/scripts/result.py @@ -0,0 +1,27 @@ +import json +import os +from pathlib import Path + + +def result_path(bbdir: str) -> Path: + rel = os.environ.get("EVAL_RESULT_PATH", "chipcrowd-eval-result.json") + root = Path(bbdir).resolve() + target = (root / rel).resolve() + try: + target.relative_to(root) + except ValueError as e: + raise ValueError(f"EVAL_RESULT_PATH escapes repo: {rel}") from e + return target + + +def merge_metrics(bbdir: str, **fields) -> dict: + path = result_path(bbdir) + data = {} + if path.is_file(): + data = json.loads(path.read_text()) + if not isinstance(data, dict): + raise ValueError(f"eval result is not an object: {path}") + data.update(fields) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2) + "\n") + return data diff --git a/bbdev/api/steps/uvm/01_build_api.step.py b/bbdev/api/steps/uvm/01_build_api.step.py index 3684868e5..9f9a87e0d 100644 --- a/bbdev/api/steps/uvm/01_build_api.step.py +++ b/bbdev/api/steps/uvm/01_build_api.step.py @@ -10,13 +10,16 @@ def check_args(body: dict) -> str | None: - allowed = {"ball", "config", "filelist"} + allowed = {"ball", "config", "core_config", "filelist"} for key in body: if key not in allowed: return f"Unexpected parameter: --{key}" config = body.get("config") if not config or config is True: return "Missing required parameter: --config=" + core_config = body.get("core_config") + if not core_config or core_config is True: + return "Missing required parameter: --core-config=" if body.get("ball") is True: return "Parameter --ball requires a value" if body.get("filelist") is True: diff --git a/bbdev/api/steps/uvm/01_build_event.step.py b/bbdev/api/steps/uvm/01_build_event.step.py index 56bf71d24..ac89ab5ff 100644 --- a/bbdev/api/steps/uvm/01_build_event.step.py +++ b/bbdev/api/steps/uvm/01_build_event.step.py @@ -1,3 +1,4 @@ +import asyncio import os import sys @@ -27,7 +28,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - result, info = run_uvm_build(bbdir, input_data, ctx) + result, info = await asyncio.to_thread(run_uvm_build, bbdir, input_data, ctx) await check_result( ctx, result.returncode, diff --git a/bbdev/api/steps/uvm/02_run_api.step.py b/bbdev/api/steps/uvm/02_run_api.step.py index 22c916373..17246805b 100644 --- a/bbdev/api/steps/uvm/02_run_api.step.py +++ b/bbdev/api/steps/uvm/02_run_api.step.py @@ -10,7 +10,7 @@ def check_args(body: dict) -> str | None: - allowed = {"ball", "filelist", "test"} + allowed = {"ball", "filelist", "test", "plusargs"} for key in body: if key not in allowed: return f"Unexpected parameter: --{key}" @@ -21,6 +21,9 @@ def check_args(body: dict) -> str | None: return "Parameter --filelist requires a path value" if body.get("test") is True: return "Parameter --test requires a value" + plusargs = body.get("plusargs") + if not isinstance(plusargs, str) or not plusargs: + return "Missing required parameter: --plusargs='+BID='" return None diff --git a/bbdev/api/steps/uvm/02_run_event.step.py b/bbdev/api/steps/uvm/02_run_event.step.py index 291c952e1..641addf8c 100644 --- a/bbdev/api/steps/uvm/02_run_event.step.py +++ b/bbdev/api/steps/uvm/02_run_event.step.py @@ -13,7 +13,7 @@ from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger +from utils.stream_run import stream_run_logger_async from scripts.uvm_common import checked_run_paths, default_test_name config = { @@ -74,16 +74,41 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return + plusargs = input_data.get("plusargs") + if not isinstance(plusargs, str) or not plusargs: + ctx.logger.error("Missing required parameter: --plusargs='+BID='") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={**info, "error": "missing_plusargs"}, + trace_id=origin_tid, + ) + return + try: + plusarg_words = shlex.split(plusargs) + except ValueError as e: + ctx.logger.error(str(e)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={**info, "error": str(e)}, + trace_id=origin_tid, + ) + return + script = ( f"cd {shlex.quote(paths['verify_dir'])} && " f"{shlex.quote(paths['simv'])} " f"-sv_lib {shlex.quote(paths['dpi_lib'])} " - f"+UVM_TESTNAME={shlex.quote(test)}" + f"+UVM_TESTNAME={shlex.quote(test)} " + + " ".join(shlex.quote(word) for word in plusarg_words) ) - cmd = f"nix develop {shlex.quote(paths['verify_env'])} --command bash -lc {shlex.quote(script)}" + cmd = f"nix develop {shlex.quote(paths['verify_env'])} --command zsh -ic {shlex.quote(script)}" ctx.logger.info(f"Running UVM test {test} for ball {paths['ball']}") - run_result = stream_run_logger( + run_result = await stream_run_logger_async( cmd=cmd, logger=ctx.logger, cwd=bbdir, diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index 43de3896d..da8f4c91b 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -12,6 +12,10 @@ def default_test_name(ball: str) -> str: + if ball == "fp2int": + return "fp2int_scale_rows_test" + if ball == "int2fp": + return "int2fp_channel_two_rows_test" return f"{ball}_ball_test" @@ -21,6 +25,59 @@ def config_dir_name(config: str) -> str: return config +def core_ball_isa_defines(bbdir: str, input_data: dict, ball: str) -> list[str]: + core_config = input_data.get("core_config") + if not isinstance(core_config, str) or not core_config: + raise ValueError("Missing required parameter: --core-config=") + path = core_config if os.path.isabs(core_config) else os.path.join(bbdir, core_config) + path = os.path.abspath(path) + if not os.path.isfile(path): + raise FileNotFoundError(f"core_config not found: {path}") + with open(path, "rb") as src: + core = tomllib.load(src) + rel_domain = core.get("balldomain") + if not isinstance(rel_domain, str) or not rel_domain: + raise ValueError(f"core_config missing balldomain: {path}") + domain_path = os.path.abspath(os.path.join(os.path.dirname(path), rel_domain)) + if not os.path.isfile(domain_path): + raise FileNotFoundError(f"balldomain not found: {domain_path}") + with open(domain_path, "rb") as src: + domain = tomllib.load(src) + entries = domain.get("ballISA") + if not isinstance(entries, list): + raise ValueError(f"ballISA missing from {domain_path}") + required = { + "fp2int": {"FP2INT": "FP2INT_FUNCT7"}, + "int2fp": { + "INT2FP_TENSOR": "INT2FP_TENSOR_FUNCT7", + "INT2FP_CHANNEL": "INT2FP_CHANNEL_FUNCT7", + }, + "smatmul": { + "SMATMUL_OS": "SMATMUL_OS_FUNCT7", + "SMATMUL_WS": "SMATMUL_WS_FUNCT7", + }, + "im2col": {"IM2COL": "IM2COL_FUNCT7"}, + "transpose": {"TRANSPOSE": "TRANSPOSE_FUNCT7"}, + }.get(ball) + if required is None: + return [] + values = {} + for entry in entries: + if not isinstance(entry, dict): + raise ValueError(f"invalid ballISA entry in {domain_path}") + mnemonic = entry.get("mnemonic") + funct7 = entry.get("funct7") + if mnemonic in required: + if not isinstance(funct7, int) or funct7 < 0 or funct7 > 127: + raise ValueError(f"invalid funct7 for {mnemonic} in {domain_path}") + if mnemonic in values: + raise ValueError(f"duplicate {mnemonic} in {domain_path}") + values[mnemonic] = funct7 + if set(values) != set(required): + raise ValueError(f"{domain_path} missing required {ball} ISA entries") + return [f"+define+{macro}={values[mnemonic]}" for mnemonic, macro in required.items()] + + def uvm_paths(bbdir: str, input_data: dict, ball_override: str | None = None) -> dict: ball = ball_override or input_data.get("ball") if not isinstance(ball, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", ball): @@ -36,7 +93,14 @@ def uvm_paths(bbdir: str, input_data: dict, ball_override: str | None = None) -> sim_dir = os.path.join(build_dir, config_dir) if config_dir else os.path.join(build_dir, "current") simv = os.path.join(sim_dir, "simv") csrc_dir = os.path.join(sim_dir, "csrc") - rtl_dir = os.path.join(bbdir, "arch", "build", config_dir) if config_dir else None + core_config = input_data.get("core_config") + core_dir = None + if config_dir: + if not isinstance(core_config, str) or not core_config: + raise ValueError("Missing required parameter: --core-config=") + core_path = core_config if os.path.isabs(core_config) else os.path.join(bbdir, core_config) + core_dir = os.path.basename(os.path.dirname(os.path.dirname(os.path.abspath(core_path)))) + rtl_dir = os.path.join(bbdir, "arch", "build", core_dir, config_dir) if config_dir else None return { "ball": ball, @@ -186,6 +250,7 @@ def failed_result(message: str) -> StreamResult: def run_uvm_build_one(bbdir: str, input_data: dict, ctx, ball: str) -> tuple[StreamResult, dict]: try: paths = checked_paths(bbdir, input_data, ball) + isa_defines = core_ball_isa_defines(bbdir, input_data, ball) except Exception as e: ctx.logger.error(str(e)) return failed_result(str(e)), {"task": "build", "error": str(e)} @@ -199,6 +264,7 @@ def run_uvm_build_one(bbdir: str, input_data: dict, ctx, ball: str) -> tuple[Str "resolved_filelist": os.path.join(paths["verify_dir"], paths["filelist_arg"]), "simv": paths["simv"], "dpi_lib": paths["dpi_lib"], + "core_config": input_data.get("core_config"), } cargo_cmd = ( @@ -220,13 +286,15 @@ def run_uvm_build_one(bbdir: str, input_data: dict, ctx, ball: str) -> tuple[Str f"cd {shlex.quote(paths['verify_dir'])} && " f"rm -rf {shlex.quote(paths['csrc_dir'])} {shlex.quote(paths['simv'])} {shlex.quote(paths['simv'])}.daidir && " f"mkdir -p {shlex.quote(paths['sim_dir'])} {shlex.quote(paths['csrc_dir'])} && " - "vcs -full64 -sverilog -timescale=1ns/1ps -hsopt=off " - "$VCS_UVM_ARGS " + "vcs -full64 -sverilog -timescale=1ns/1ps -debug_access+all " + "${=VCS_UVM_ARGS} " + + " ".join(shlex.quote(value) for value in isa_defines) + + " " f"-Mdir={shlex.quote(paths['csrc_dir'])} " f"-o {shlex.quote(paths['simv'])} " f"-f {shlex.quote(paths['filelist_arg'])}" ) - vcs_cmd = f"nix develop {shlex.quote(paths['verify_env'])} --command bash -lc {shlex.quote(script)}" + vcs_cmd = f"nix develop {shlex.quote(paths['verify_env'])} --command zsh -c {shlex.quote(script)}" ctx.logger.info(f"Building UVM simulation for ball {paths['ball']} config {paths['config']}") vcs_result = stream_run_logger( diff --git a/bbdev/api/steps/vcs/01_clean_api.step.py b/bbdev/api/steps/vcs/01_clean_api.step.py index cebeb4af5..2e5529a39 100644 --- a/bbdev/api/steps/vcs/01_clean_api.step.py +++ b/bbdev/api/steps/vcs/01_clean_api.step.py @@ -1,5 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.chip import require_chip from utils.path import get_buckyball_path, get_vcs_build_dir @@ -14,16 +15,17 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - config_name = body.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse(status=400, body={"error": "Missing required parameter: --config must be specified"}) bbdir = get_buckyball_path() + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) await ctx.enqueue( { "topic": "vcs.clean", "data": { - "config": config_name, - "output_dir": get_vcs_build_dir(bbdir, config_name, body.get("output_dir")), + "chip": chip, + "output_dir": get_vcs_build_dir(bbdir, chip, body.get("output_dir")), "_trace_id": ctx.trace_id, }, } diff --git a/bbdev/api/steps/vcs/02_verilog_api.step.py b/bbdev/api/steps/vcs/02_verilog_api.step.py index f0ed9e14c..7ac3e782b 100644 --- a/bbdev/api/steps/vcs/02_verilog_api.step.py +++ b/bbdev/api/steps/vcs/02_verilog_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import get_buckyball_path, get_vcs_build_dir +from utils.chip import require_chip config = { @@ -13,14 +13,11 @@ def _request_data(body: dict) -> dict: - config_name = body.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - raise ValueError("Missing required parameter: --config must be specified") - bbdir = get_buckyball_path() - return { - "config": config_name, - "output_dir": get_vcs_build_dir(bbdir, config_name, body.get("output_dir")), - } + chip = require_chip(body) + data = {"chip": chip} + if body.get("output_dir"): + data["output_dir"] = body["output_dir"] + return data async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: diff --git a/bbdev/api/steps/vcs/02_verilog_event.step.py b/bbdev/api/steps/vcs/02_verilog_event.step.py index c2bd6f666..d25ee16eb 100644 --- a/bbdev/api/steps/vcs/02_verilog_event.step.py +++ b/bbdev/api/steps/vcs/02_verilog_event.step.py @@ -8,8 +8,9 @@ sys.path.insert(0, utils_path) from utils.event_common import check_result, get_origin_trace_id -from utils.path import get_buckyball_path, get_vcs_build_dir -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.path import get_buckyball_path +from utils.rtl import run_chip_mill config = { @@ -23,30 +24,26 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) - config_name = input_data.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - await check_result(ctx, 1, extra_fields={"task": "verilog", "error": "missing config"}, trace_id=origin_tid) + try: + chip = require_chip(input_data) + except ValueError as exc: + await check_result(ctx, 1, extra_fields={"task": "verilog", "error": str(exc)}, trace_id=origin_tid) return bbdir = get_buckyball_path() - build_dir = get_vcs_build_dir(bbdir, config_name, input_data.get("output_dir")) - command = ( - f"mill -i __.test.runMain sims.verilator.Elaborate {config_name} " - "--disable-annotation-unknown --strip-debug-info -O=release " - f"--split-verilog -o={build_dir}" - ) - result = stream_run_logger( - cmd=command, - logger=ctx.logger, - cwd=f"{bbdir}/arch", - stdout_prefix="vcs verilog", - stderr_prefix="vcs verilog", - ) + try: + build_dir, returncode = await run_chip_mill( + ctx, bbdir, chip, "verilog", "vcs verilog", + input_data.get("output_dir"), + ) + except (ValueError, RuntimeError) as exc: + await check_result(ctx, 1, extra_fields={"task": "verilog", "error": str(exc)}, trace_id=origin_tid) + return await check_result( ctx, - result.returncode, - continue_run=input_data.get("from_run_workflow", False), + returncode, + continue_run=input_data.get("from_run_workflow", False) and returncode == 0, extra_fields={"task": "verilog", "output_dir": build_dir}, trace_id=origin_tid, ) - if result.returncode == 0 and input_data.get("from_run_workflow"): + if input_data.get("from_run_workflow") and returncode == 0: await ctx.enqueue({"topic": "vcs.build", "data": {**input_data, "output_dir": build_dir, "_trace_id": origin_tid}}) diff --git a/bbdev/api/steps/vcs/03_build_api.step.py b/bbdev/api/steps/vcs/03_build_api.step.py index eaa8586ee..a2c8916d5 100644 --- a/bbdev/api/steps/vcs/03_build_api.step.py +++ b/bbdev/api/steps/vcs/03_build_api.step.py @@ -1,5 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.chip import require_chip from utils.path import get_buckyball_path, get_vcs_build_dir @@ -14,9 +15,10 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - config_name = body.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse(status=400, body={"error": "Missing required parameter: --config must be specified"}) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) jobs = body.get("jobs", 16) try: jobs = int(jobs) @@ -26,9 +28,9 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: return ApiResponse(status=400, body={"error": "jobs must be a positive integer"}) bbdir = get_buckyball_path() data = { - "config": config_name, + "chip": chip, "jobs": jobs, - "output_dir": get_vcs_build_dir(bbdir, config_name, body.get("output_dir")), + "output_dir": get_vcs_build_dir(bbdir, chip, body.get("output_dir")), } await ctx.enqueue({"topic": "vcs.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/03_build_event.step.py b/bbdev/api/steps/vcs/03_build_event.step.py index d53690ad3..1b6e1d8a1 100644 --- a/bbdev/api/steps/vcs/03_build_event.step.py +++ b/bbdev/api/steps/vcs/03_build_event.step.py @@ -13,8 +13,9 @@ sys.path.insert(0, utils_path) from utils.event_common import check_result, get_origin_trace_id +from utils.chip import require_chip from utils.path import get_buckyball_path, get_vcs_build_dir -from utils.stream_run import stream_run_logger +from utils.stream_run import stream_run_logger_async config = { @@ -114,9 +115,10 @@ def _pkg_config(flag: str, package: str) -> str: async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) - config_name = input_data.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - await check_result(ctx, 1, extra_fields={"task": "build", "error": "missing config"}, trace_id=origin_tid) + try: + chip = require_chip(input_data) + except ValueError as exc: + await check_result(ctx, 1, extra_fields={"task": "build", "error": str(exc)}, trace_id=origin_tid) return if shutil.which("vcs") is None: await check_result( @@ -128,7 +130,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return bbdir = get_buckyball_path() - build_dir = get_vcs_build_dir(bbdir, config_name, input_data.get("output_dir")) + build_dir = get_vcs_build_dir(bbdir, chip, input_data.get("output_dir")) vsrcs = sorted( path for path in ( glob.glob(f"{build_dir}/**/*.v", recursive=True) @@ -196,7 +198,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: *(shlex.quote(source) for source in sources), ] ) - result = stream_run_logger( + result = await stream_run_logger_async( cmd=command, logger=ctx.logger, cwd=artifact_dir, diff --git a/bbdev/api/steps/vcs/04_sim_api.step.py b/bbdev/api/steps/vcs/04_sim_api.step.py index 1ea7201d5..bc935db45 100644 --- a/bbdev/api/steps/vcs/04_sim_api.step.py +++ b/bbdev/api/steps/vcs/04_sim_api.step.py @@ -1,5 +1,8 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.chip import require_chip +from utils.path import get_buckyball_path + config = { "name": "vcs-sim-api", @@ -12,11 +15,15 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - config_name = body.get("config") binary = body.get("binary") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse(status=400, body={"error": "Missing required parameter: --config must be specified"}) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) if not isinstance(binary, str) or not binary: return ApiResponse(status=400, body={"error": "Missing required parameter: --binary must be specified"}) - await ctx.enqueue({"topic": "vcs.sim", "data": {**body, "_trace_id": ctx.trace_id}}) + await ctx.enqueue({ + "topic": "vcs.sim", + "data": {**body, "chip": chip, "_trace_id": ctx.trace_id}, + }) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/04_sim_event.step.py b/bbdev/api/steps/vcs/04_sim_event.step.py index ce46f6781..41a52e3cb 100644 --- a/bbdev/api/steps/vcs/04_sim_event.step.py +++ b/bbdev/api/steps/vcs/04_sim_event.step.py @@ -11,9 +11,10 @@ sys.path.insert(0, utils_path) from utils.event_common import check_result, get_origin_trace_id -from utils.path import get_buckyball_path, get_vcs_build_dir +from utils.chip import require_chip +from utils.path import get_buckyball_path, get_run_log_dir, get_vcs_build_dir, workload_build_dir, workload_tests_root from utils.search_workload import search_workload -from utils.stream_run import stream_run_logger +from utils.stream_run import stream_run_logger_async config = { @@ -25,11 +26,14 @@ } -def _resolve_binary(bbdir: str, binary: str) -> str | None: +def _resolve_binary(bbdir: str, chip: str, binary: str) -> str | None: if os.path.isfile(binary): return os.path.abspath(binary) - for root in ("output/workloads/src", "build/workloads/src"): - found = search_workload(f"{bbdir}/bb-tests/{root}", binary) + for root in ( + workload_tests_root(bbdir, chip), + os.path.join(workload_build_dir(bbdir, chip), "src"), + ): + found = search_workload(root, binary) if found: return found return None @@ -37,20 +41,21 @@ def _resolve_binary(bbdir: str, binary: str) -> str | None: async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) - config_name = input_data.get("config") - binary_name = input_data.get("binary") - if not isinstance(config_name, str) or not config_name or config_name == "None": - await check_result(ctx, 1, extra_fields={"task": "sim", "error": "missing config"}, trace_id=origin_tid) + try: + chip = require_chip(input_data) + except ValueError as exc: + await check_result(ctx, 1, extra_fields={"task": "sim", "error": str(exc)}, trace_id=origin_tid) return + binary_name = input_data.get("binary") if not isinstance(binary_name, str) or not binary_name: await check_result(ctx, 1, extra_fields={"task": "sim", "error": "missing binary"}, trace_id=origin_tid) return bbdir = get_buckyball_path() - binary = _resolve_binary(bbdir, binary_name) + binary = _resolve_binary(bbdir, chip, binary_name) if binary is None: await check_result(ctx, 1, extra_fields={"task": "sim", "error": f"binary not found: {binary_name}"}, trace_id=origin_tid) return - build_dir = get_vcs_build_dir(bbdir, config_name, input_data.get("output_dir")) + build_dir = get_vcs_build_dir(bbdir, chip, input_data.get("output_dir")) artifact_dir = Path(build_dir) / "vcs" simv = artifact_dir / "simv" if not simv.is_file(): @@ -58,7 +63,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return timestamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") - log_dir = Path(bbdir) / "log" / f"{timestamp}-vcs-{config_name.replace('/', '_')}" + log_dir = Path(get_run_log_dir( + bbdir, chip, "verilog", timestamp, "vcs", binary_name, + input_data.get("output_dir"), + )) log_dir.mkdir(parents=True) timeout_ns = input_data.get("timeout_ns", input_data.get("timeout-ns", "100000000")) try: @@ -82,7 +90,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "+permissive-off", ] ) - result = stream_run_logger( + result = await stream_run_logger_async( cmd=command, logger=ctx.logger, cwd=artifact_dir, diff --git a/bbdev/api/steps/vcs/05_run_api.step.py b/bbdev/api/steps/vcs/05_run_api.step.py index d7fbc7fbb..eeb77a842 100644 --- a/bbdev/api/steps/vcs/05_run_api.step.py +++ b/bbdev/api/steps/vcs/05_run_api.step.py @@ -1,5 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.chip import require_chip from utils.path import get_buckyball_path, get_vcs_build_dir @@ -14,10 +15,11 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - config_name = body.get("config") binary = body.get("binary") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse(status=400, body={"error": "Missing required parameter: --config must be specified"}) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) if not isinstance(binary, str) or not binary: return ApiResponse(status=400, body={"error": "Missing required parameter: --binary must be specified"}) jobs = body.get("jobs", 16) @@ -29,11 +31,11 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: return ApiResponse(status=400, body={"error": "jobs must be a positive integer"}) bbdir = get_buckyball_path() data = { - "config": config_name, + "chip": chip, "binary": binary, "batch": bool(body.get("batch", False)), "jobs": jobs, - "output_dir": get_vcs_build_dir(bbdir, config_name, body.get("output_dir")), + "output_dir": get_vcs_build_dir(bbdir, chip, body.get("output_dir")), "from_run_workflow": True, } await ctx.enqueue({"topic": "vcs.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) diff --git a/bbdev/api/steps/verilator/01_clean_api.step.py b/bbdev/api/steps/verilator/01_clean_api.step.py index a5b0a826e..77c6291fd 100644 --- a/bbdev/api/steps/verilator/01_clean_api.step.py +++ b/bbdev/api/steps/verilator/01_clean_api.step.py @@ -1,5 +1,8 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.chip import require_chip +from utils.path import get_buckyball_path + config = { "name": "verilator-clean-api", "description": "clean build directory", @@ -11,11 +14,12 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - config_name = body.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --config must be specified"}, - ) - await ctx.enqueue({"topic": "verilator.clean", "data": {**body, "task": "clean", "_trace_id": ctx.trace_id}}) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + await ctx.enqueue({ + "topic": "verilator.clean", + "data": {**body, "chip": chip, "task": "clean", "_trace_id": ctx.trace_id}, + }) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/01_clean_event.step.py b/bbdev/api/steps/verilator/01_clean_event.step.py index af1e0678b..06ae1ad90 100644 --- a/bbdev/api/steps/verilator/01_clean_event.step.py +++ b/bbdev/api/steps/verilator/01_clean_event.step.py @@ -8,8 +8,9 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path, get_verilator_build_dir -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.path import get_buckyball_path, rtl_dir_for_clean +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id config = { @@ -26,26 +27,27 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) - config_name = input_data.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - ctx.logger.error("Missing required parameter: config") + try: + chip = require_chip(input_data) + except ValueError as error: + ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_config"}, + extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) return bbdir = get_buckyball_path() - build_dir = get_verilator_build_dir( - bbdir, - config_name, + build_dir = rtl_dir_for_clean( + bbdir, chip, + "verilog", input_data.get("output_dir"), ) # ================================================================================== # Execute operation # ================================================================================== command = f"rm -rf {build_dir}" - result = stream_run_logger( + result = await stream_run_logger_async( cmd=command, logger=ctx.logger, cwd=bbdir, diff --git a/bbdev/api/steps/verilator/02_verilog_api.step.py b/bbdev/api/steps/verilator/02_verilog_api.step.py index d5054f5fd..67ade991e 100644 --- a/bbdev/api/steps/verilator/02_verilog_api.step.py +++ b/bbdev/api/steps/verilator/02_verilog_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.chip import require_chip config = { "name": "verilator-verilog-api", @@ -12,25 +12,40 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = request.body or {} - # Get config name, must be provided - config_name = body.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse( - status=400, - body={ - "status": "error", - "message": "Configuration name is required. Please specify --config parameter.", - "example": 'bbdev verilator --verilog "--config sims.verilator.BuckyballToyVerilatorConfig"', - }, - ) - - data = { - "config": config_name, - "balltype": body.get("balltype"), - "output_dir": get_verilator_build_dir(bbdir, config_name, body.get("output_dir")), - } + if body.get("balltype"): + if body.get("chip") or ( + isinstance(body.get("config"), str) and body.get("config") and body.get("config") != "None" + ): + return ApiResponse( + status=400, + body={"error": "balltype mill does not take --chip or --config"}, + ) + output_dir = body.get("output_dir") + if not output_dir: + return ApiResponse( + status=400, + body={"error": "balltype mill requires output_dir"}, + ) + data = { + "balltype": body.get("balltype"), + "output_dir": output_dir, + } + else: + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse( + status=400, + body={ + "status": "error", + "message": str(e), + "example": 'bbdev verilator --verilog "--chip toy"', + }, + ) + data = {"chip": chip} + if body.get("output_dir"): + data["output_dir"] = body["output_dir"] await ctx.enqueue({"topic": "verilator.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/03_build_api.step.py b/bbdev/api/steps/verilator/03_build_api.step.py index 4c7b3ed7f..7771a78a4 100644 --- a/bbdev/api/steps/verilator/03_build_api.step.py +++ b/bbdev/api/steps/verilator/03_build_api.step.py @@ -1,5 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.chip import require_chip from utils.path import get_buckyball_path, get_verilator_build_dir config = { @@ -14,16 +15,14 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = request.body or {} - config_name = body.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --config must be specified"}, - ) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) data = { + "chip": chip, "jobs": body.get("jobs", 16), - "config": config_name, - "output_dir": get_verilator_build_dir(bbdir, config_name, body.get("output_dir")), + "output_dir": get_verilator_build_dir(bbdir, chip, body.get("output_dir")), } await ctx.enqueue({"topic": "verilator.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/03_build_event.step.py b/bbdev/api/steps/verilator/03_build_event.step.py index 58831ba31..a52261b8b 100644 --- a/bbdev/api/steps/verilator/03_build_event.step.py +++ b/bbdev/api/steps/verilator/03_build_event.step.py @@ -10,8 +10,9 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path, get_verilator_build_dir -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.path import gcc_lib_dir, get_buckyball_path, get_verilator_build_dir +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id config = { @@ -25,14 +26,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) - config_name = input_data.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - raise ValueError("Missing required input: config") + chip = require_chip(input_data) bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" build_dir = get_verilator_build_dir( - bbdir, - config_name, + bbdir, chip, input_data.get("output_dir"), ) coverage = input_data.get("coverage", False) @@ -69,31 +67,29 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # Build flags # ================================================================================== result_dir = f"{bbdir}/result" - - def pkg_config(flag, pkg): - r = subprocess.run(["pkg-config", flag, pkg], capture_output=True, text=True) - return r.stdout.strip() if r.returncode == 0 else "" - - readline_inc = pkg_config("--variable=includedir", "readline") - readline_lib = pkg_config("--variable=libdir", "readline") - zlib_lib = pkg_config("--variable=libdir", "zlib") + result_lib = f"{result_dir}/lib" + if not os.path.isdir(result_lib): + raise RuntimeError(f"missing {result_lib}; run nix build") inc_flags = " ".join([ f"-I{result_dir}/include", f"-I{build_dir}", f"-I{arch_dir}/src/csrc/include", - f"-I{readline_inc}", ]) # -DBBSIM: selects VBBSimHarness in bdb.h / main.cc # BDB NDJSON trace (+trace=...) is runtime-only; bbdev sim uses +trace=all (04_sim_event.step.py). cflags = f"{inc_flags} -DBBSIM -DTOP_NAME='\"V{topname}\"' -std=c++17" + rpath_dirs = [] + for lib_dir in (result_lib, gcc_lib_dir("liblz4.so"), gcc_lib_dir("libstdc++.so")): + if lib_dir not in rpath_dirs: + rpath_dirs.append(lib_dir) + rpath_flags = " ".join(f"-Wl,-rpath,{lib_dir}" for lib_dir in rpath_dirs) + ldflags = ( f"-lreadline -ldramsim3 -lstdc++ -lz " - f"-L{result_dir}/lib " - f"-L{readline_lib} -Wl,-rpath,{readline_lib} " - f"-L{zlib_lib} -Wl,-rpath,{zlib_lib} " + f"-L{result_lib} {rpath_flags} " ) obj_dir = f"{build_dir}/obj_dir" @@ -103,14 +99,6 @@ def pkg_config(flag, pkg): sources = " ".join(vsrcs + csrcs) jobs = input_data.get("jobs", "") - # Fix nix runtime library paths - libstdcpp_path = subprocess.run( - "g++ -print-file-name=libstdc++.so", shell=True, capture_output=True, text=True - ).stdout.strip() - if libstdcpp_path and "/" in libstdcpp_path: - nix_lib_dir = os.path.dirname(os.path.realpath(libstdcpp_path)) - ldflags += f" -Wl,-rpath,{nix_lib_dir}" - # Enable ccache if available if subprocess.run("command -v ccache", shell=True, capture_output=True).returncode == 0: os.environ["OBJCACHE"] = "ccache" @@ -140,7 +128,7 @@ def pkg_config(flag, pkg): f"-CFLAGS '{cflags}' -LDFLAGS '{ldflags}' --Mdir {obj_dir} --exe" ) - result = stream_run_logger( + result = await stream_run_logger_async( cmd=verilator_cmd, logger=ctx.logger, cwd=bbdir, @@ -155,7 +143,7 @@ def pkg_config(flag, pkg): return make_jobs = jobs if jobs else str(os.cpu_count() or 16) - result = stream_run_logger( + result = await stream_run_logger_async( cmd=f"make -j{make_jobs} VM_PARALLEL_BUILDS=1 -C {obj_dir} -f V{topname}.mk V{topname}", logger=ctx.logger, cwd=bbdir, diff --git a/bbdev/api/steps/verilator/04_sim_api.step.py b/bbdev/api/steps/verilator/04_sim_api.step.py index 069d941b5..1f1281efd 100644 --- a/bbdev/api/steps/verilator/04_sim_api.step.py +++ b/bbdev/api/steps/verilator/04_sim_api.step.py @@ -1,5 +1,8 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.chip import require_chip +from utils.path import get_buckyball_path + config = { "name": "verilator-sim-api", "description": "run verilator simulation", @@ -22,12 +25,13 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "message": "binary parameter is required", }, ) - config_name = body.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --config must be specified"}, - ) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) - await ctx.enqueue({"topic": "verilator.sim", "data": {**body, "task": "sim", "_trace_id": ctx.trace_id}}) + await ctx.enqueue({ + "topic": "verilator.sim", + "data": {**body, "chip": chip, "task": "sim", "_trace_id": ctx.trace_id}, + }) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/04_sim_event.step.py b/bbdev/api/steps/verilator/04_sim_event.step.py index 3cc6aae9b..81609d872 100644 --- a/bbdev/api/steps/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -10,8 +10,9 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path, get_verilator_build_dir -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.path import gcc_lib_dir, get_buckyball_path, get_run_log_dir, get_verilator_build_dir, workload_tests_root +from utils.stream_run import stream_run_logger_async from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id @@ -29,14 +30,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # Get simulation parameters # ================================================================================== origin_tid = get_origin_trace_id(input_data, ctx) - config_name = input_data.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - raise ValueError("Missing required input: config") + chip = require_chip(input_data) bbdir = get_buckyball_path() - arch_dir = f"{bbdir}/arch" build_dir = get_verilator_build_dir( - bbdir, - config_name, + bbdir, chip, input_data.get("output_dir"), ) ctx.logger.info(f"Using build directory: {build_dir}") @@ -44,23 +41,34 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") binary_name = input_data.get("binary", "") + if not binary_name: + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "binary_required"}, + trace_id=origin_tid, + ) + return coverage = input_data.get("coverage", False) - success_result, failure_result = await check_result( - ctx, returncode=(binary_name is None), continue_run=True, trace_id=origin_tid, - ) - binary_path = search_workload(f"{bbdir}/bb-tests/output/workloads/src", binary_name) + workload_root = workload_tests_root(bbdir, chip) + binary_path = search_workload(workload_root, binary_name) ctx.logger.info(f"binary_path: {binary_path}") - success_result, failure_result = await check_result( - ctx, returncode=(binary_path is None), continue_run=True, trace_id=origin_tid, - ) - if failure_result: - ctx.logger.error("binary not found", failure_result) + if binary_path is None: + ctx.logger.error(f"binary not found: {binary_name}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={ + "error": "binary_not_found", + "binary": binary_name, + "search_dir": workload_root, + }, + trace_id=origin_tid, + ) return topname = "BBSimHarness" - log_dir = f"{arch_dir}/log/{timestamp}-{binary_name}" - waveform_dir = f"{arch_dir}/waveform/{timestamp}-{binary_name}" + log_dir = get_run_log_dir(bbdir, chip, "verilog", timestamp, "verilator", binary_name, input_data.get("output_dir")) + waveform_dir = os.path.join(log_dir, "waveform") os.makedirs(log_dir, exist_ok=True) os.makedirs(waveform_dir, exist_ok=True) @@ -71,6 +79,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: coverage_flag = f"+verilator+coverage+file+{coverage_dat_path}" bin_path = f"{build_dir}/obj_dir/V{topname}" + if not os.path.isfile(bin_path): + ctx.logger.error(f"simulator binary not found: {bin_path}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "simulator_not_found", "simulator": bin_path}, + trace_id=origin_tid, + ) + return batch = input_data.get("batch", False) log_path = f"{log_dir}/bdb.ndjson" @@ -87,9 +103,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # merging stdout with 2>&1 can break: full stdio buffering + non-DASM bytes). # BDB_SIM_META moves NDJSON banner to sim_meta.txt so it does not pollute disasm. # ================================================================================== - ld_lib_path = ( - f"{bbdir}/result/lib" - ) + result_lib = f"{bbdir}/result/lib" + if not os.path.isdir(result_lib): + raise RuntimeError(f"missing {result_lib}; run nix build") + lib_dirs = [result_lib, gcc_lib_dir("liblz4.so"), gcc_lib_dir("libstdc++.so")] + ld_lib_path = ":".join(dict.fromkeys(lib_dirs)) + ctx.logger.info(f"LD_LIBRARY_PATH prefix: {ld_lib_path}") sim_cmd = ( f"export LD_LIBRARY_PATH=\"{ld_lib_path}:$LD_LIBRARY_PATH\"; " f"export BDB_SIM_META=\"{meta_path}\"; " @@ -102,7 +121,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) script_dir = os.path.dirname(__file__) - result = stream_run_logger( + result = await stream_run_logger_async( cmd=sim_cmd, logger=ctx.logger, cwd=script_dir, diff --git a/bbdev/api/steps/verilator/05_run_api.step.py b/bbdev/api/steps/verilator/05_run_api.step.py index 79c872901..39283ce12 100644 --- a/bbdev/api/steps/verilator/05_run_api.step.py +++ b/bbdev/api/steps/verilator/05_run_api.step.py @@ -1,5 +1,16 @@ +import os +import sys + from motia import ApiRequest, ApiResponse, FlowContext, api +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.chip import require_chip +from utils.path import get_buckyball_path, workload_tests_root +from utils.search_workload import search_workload + config = { "name": "verilator-run-api", "description": "trigger complete verilator workflow", @@ -11,12 +22,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - config_name = body.get("config") - if not isinstance(config_name, str) or not config_name or config_name == "None": - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --config must be specified"}, - ) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) binary = body.get("binary", "") if not binary: return ApiResponse( @@ -29,9 +38,23 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: }, ) + search_dir = workload_tests_root(get_buckyball_path(), chip) + if search_workload(search_dir, binary) is None: + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 1, + "error": "binary_not_found", + "binary": binary, + "search_dir": search_dir, + }, + ) + data = { + "chip": chip, "binary": binary, - "config": config_name, "jobs": body.get("jobs", "16"), "batch": body.get("batch", False), "from_run_workflow": True, diff --git a/bbdev/api/steps/verilator/README.md b/bbdev/api/steps/verilator/README.md index 9da810c68..5ca961b79 100644 --- a/bbdev/api/steps/verilator/README.md +++ b/bbdev/api/steps/verilator/README.md @@ -19,7 +19,7 @@ Hardware simulation workflow based on Verilator in the Buckyball framework, prov **Example**: ```bash # bbdev wrapper -bbdev verilator --run "jobs 256 --binary ${buckyball}/bb-tests/output/workloads/src/ctest_mvin_mvout_alternate_test_singlecore-baremetal --batch" +bbdev verilator --run "jobs 256 --binary ${buckyball}/bb-tests/output/toy/workloads/src/ctest_mvin_mvout_alternate_test-baremetal --batch" # Raw command curl -X POST http://localhost:5000/verilator/run -H "Content-Type: application/json" -d '{"jobs": 8, "binary": "/home/user/test.elf"}' diff --git a/bbdev/api/steps/verilator/dramsim3.json b/bbdev/api/steps/verilator/dramsim3.json new file mode 100644 index 000000000..01aba8e2d --- /dev/null +++ b/bbdev/api/steps/verilator/dramsim3.json @@ -0,0 +1 @@ +{"0":{"act_energy":43200.0,"act_stb_energy":{"0":3013588.8000000003,"1":1462579.2},"all_bank_idle_cycles":{"0":33121,"1":39365},"average_bandwidth":0.13577000419861668,"average_interarrival":313.7708333333333,"average_power":440.3248756988487,"average_read_latency":17.416666666666668,"channel":0,"epoch_num":0,"hbm_dual_cmds":0,"interarrival_latency":{"1":2,"103":1,"106":1,"107":1,"113":1,"117":1,"12":1,"1216":1,"123":1,"124":1,"127":1,"14":1,"143":1,"15":1,"151":1,"164":1,"172":1,"18":1,"183":1,"2":1,"212":1,"262":1,"266":1,"279":1,"3":3,"301":1,"315":1,"32":14,"32868":1,"350":1,"4":1,"45":6,"46":3,"47":4,"48":2,"49":7,"5":2,"50":3,"51":5,"52":3,"53":1,"54":1,"55":17,"56":1,"57":1,"58":3,"59":3,"60":6,"63":1,"64":2,"65":4,"662":1,"67":3,"68":1,"69":2,"70":2,"74":1,"75":1,"76":1,"79":1,"803":1,"81":1,"82":2,"86":1,"9":1,"91":1,"93":1,"94":1},"interarrival_latency[-0]":0,"interarrival_latency[0-9]":10,"interarrival_latency[10-19]":4,"interarrival_latency[100-]":24,"interarrival_latency[20-29]":0,"interarrival_latency[30-39]":14,"interarrival_latency[40-49]":22,"interarrival_latency[50-59]":38,"interarrival_latency[60-69]":19,"interarrival_latency[70-79]":6,"interarrival_latency[80-89]":4,"interarrival_latency[90-99]":3,"num_act_cmds":10,"num_cycles":45253,"num_ondemand_pres":0,"num_pre_cmds":8,"num_read_cmds":144,"num_read_row_hits":134,"num_reads_done":144,"num_ref_cmds":17,"num_refb_cmds":0,"num_srefe_cmds":0,"num_srefx_cmds":0,"num_write_buf_hits":0,"num_write_cmds":0,"num_write_row_hits":0,"num_writes_done":0,"pre_stb_energy":{"0":6081015.600000001,"1":7227414.000000001},"rank_active_cycles":{"0":12132,"1":5888},"read_energy":304819.2,"read_latency":{"15":125,"16":1,"17":3,"18":2,"19":1,"25":7,"33":1,"61":1,"73":1,"80":1,"89":1},"read_latency[-0]":0,"read_latency[0-19]":132,"read_latency[100-119]":0,"read_latency[120-139]":0,"read_latency[140-159]":0,"read_latency[160-179]":0,"read_latency[180-199]":0,"read_latency[20-39]":8,"read_latency[200-]":0,"read_latency[40-59]":0,"read_latency[60-79]":2,"read_latency[80-99]":2,"ref_energy":1793404.8,"refb_energy":-0.0,"sref_cycles":{"0":0,"1":0},"sref_energy":{"0":0.0,"1":0.0},"total_energy":19926021.6,"write_energy":0.0,"write_latency":null,"write_latency[-0]":0,"write_latency[0-19]":0,"write_latency[100-119]":0,"write_latency[120-139]":0,"write_latency[140-159]":0,"write_latency[160-179]":0,"write_latency[180-199]":0,"write_latency[20-39]":0,"write_latency[200-]":0,"write_latency[40-59]":0,"write_latency[60-79]":0,"write_latency[80-99]":0}} \ No newline at end of file diff --git a/bbdev/api/steps/verilator/dramsim3.txt b/bbdev/api/steps/verilator/dramsim3.txt new file mode 100644 index 000000000..a479c6c98 --- /dev/null +++ b/bbdev/api/steps/verilator/dramsim3.txt @@ -0,0 +1,78 @@ +########################################### +## Statistics of Channel 0 +########################################### +num_srefx_cmds = 0 # Number of SREFX commands +num_srefe_cmds = 0 # Number of SREFE commands +num_refb_cmds = 0 # Number of REFb commands +num_cycles = 45253 # Number of DRAM cycles +epoch_num = 0 # Number of epochs +num_write_buf_hits = 0 # Number of write buffer hits +num_write_cmds = 0 # Number of WRITE/WRITEP commands +num_reads_done = 144 # Number of read requests issued +hbm_dual_cmds = 0 # Number of cycles dual cmds issued +num_ref_cmds = 17 # Number of REF commands +num_read_row_hits = 134 # Number of read row buffer hits +num_read_cmds = 144 # Number of READ/READP commands +num_writes_done = 0 # Number of read requests issued +num_write_row_hits = 0 # Number of write row buffer hits +num_act_cmds = 10 # Number of ACT commands +num_pre_cmds = 8 # Number of PRE commands +num_ondemand_pres = 0 # Number of ondemend PRE commands +sref_cycles.0 = 0 # Cyles of rank in SREF mode rank.0 +sref_cycles.1 = 0 # Cyles of rank in SREF mode rank.1 +rank_active_cycles.0 = 12132 # Cyles of rank active rank.0 +rank_active_cycles.1 = 5888 # Cyles of rank active rank.1 +all_bank_idle_cycles.0 = 33121 # Cyles of all bank idle in rank rank.0 +all_bank_idle_cycles.1 = 39365 # Cyles of all bank idle in rank rank.1 +interarrival_latency[-0] = 0 # Request interarrival latency (cycles) +interarrival_latency[0-9] = 10 # Request interarrival latency (cycles) +interarrival_latency[10-19] = 4 # Request interarrival latency (cycles) +interarrival_latency[20-29] = 0 # Request interarrival latency (cycles) +interarrival_latency[30-39] = 14 # Request interarrival latency (cycles) +interarrival_latency[40-49] = 22 # Request interarrival latency (cycles) +interarrival_latency[50-59] = 38 # Request interarrival latency (cycles) +interarrival_latency[60-69] = 19 # Request interarrival latency (cycles) +interarrival_latency[70-79] = 6 # Request interarrival latency (cycles) +interarrival_latency[80-89] = 4 # Request interarrival latency (cycles) +interarrival_latency[90-99] = 3 # Request interarrival latency (cycles) +interarrival_latency[100-] = 24 # Request interarrival latency (cycles) +write_latency[-0] = 0 # Write cmd latency (cycles) +write_latency[0-19] = 0 # Write cmd latency (cycles) +write_latency[20-39] = 0 # Write cmd latency (cycles) +write_latency[40-59] = 0 # Write cmd latency (cycles) +write_latency[60-79] = 0 # Write cmd latency (cycles) +write_latency[80-99] = 0 # Write cmd latency (cycles) +write_latency[100-119] = 0 # Write cmd latency (cycles) +write_latency[120-139] = 0 # Write cmd latency (cycles) +write_latency[140-159] = 0 # Write cmd latency (cycles) +write_latency[160-179] = 0 # Write cmd latency (cycles) +write_latency[180-199] = 0 # Write cmd latency (cycles) +write_latency[200-] = 0 # Write cmd latency (cycles) +read_latency[-0] = 0 # Read request latency (cycles) +read_latency[0-19] = 132 # Read request latency (cycles) +read_latency[20-39] = 8 # Read request latency (cycles) +read_latency[40-59] = 0 # Read request latency (cycles) +read_latency[60-79] = 2 # Read request latency (cycles) +read_latency[80-99] = 2 # Read request latency (cycles) +read_latency[100-119] = 0 # Read request latency (cycles) +read_latency[120-139] = 0 # Read request latency (cycles) +read_latency[140-159] = 0 # Read request latency (cycles) +read_latency[160-179] = 0 # Read request latency (cycles) +read_latency[180-199] = 0 # Read request latency (cycles) +read_latency[200-] = 0 # Read request latency (cycles) +ref_energy = 1.7934e+06 # Refresh energy +refb_energy = -0 # Refresh-bank energy +write_energy = 0 # Write energy +read_energy = 304819 # Read energy +act_energy = 43200 # Activation energy +sref_energy.0 = 0 # SREF energy rank.0 +sref_energy.1 = 0 # SREF energy rank.1 +pre_stb_energy.0 = 6.08102e+06 # Precharge standby energy rank.0 +pre_stb_energy.1 = 7.22741e+06 # Precharge standby energy rank.1 +act_stb_energy.0 = 3.01359e+06 # Active standby energy rank.0 +act_stb_energy.1 = 1.46258e+06 # Active standby energy rank.1 +average_read_latency = 17.4167 # Average read request latency (cycles) +average_interarrival = 313.771 # Average request interarrival latency (cycles) +total_energy = 1.9926e+07 # Total energy (pJ) +average_power = 440.325 # Average power (mW) +average_bandwidth = 0.13577 # Average bandwidth diff --git a/bbdev/api/steps/workload/00_clean_api.step.py b/bbdev/api/steps/workload/00_clean_api.step.py index 7132babe8..53aa4a440 100644 --- a/bbdev/api/steps/workload/00_clean_api.step.py +++ b/bbdev/api/steps/workload/00_clean_api.step.py @@ -1,8 +1,10 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.chip import require_chip + config = { "name": "workload-clean-api", - "description": "clean workload output directory", + "description": "clean workload output directory for one chip", "flows": ["workload"], "triggers": [api("POST", "/workload/clean")], "enqueues": ["workload.clean"], @@ -11,9 +13,17 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - if body: - unknown = ", ".join(sorted(body)) - return ApiResponse(status=400, body={"error": f"Unknown workload clean parameter(s): {unknown}"}) + try: + chip = require_chip(body) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + + unknown = sorted(k for k in body if k not in {"chip"}) + if unknown: + return ApiResponse( + status=400, + body={"error": f"Unknown workload clean parameter(s): {', '.join(unknown)}"}, + ) - await ctx.enqueue({"topic": "workload.clean", "data": {"_trace_id": ctx.trace_id}}) + await ctx.enqueue({"topic": "workload.clean", "data": {**body, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/00_clean_event.step.py b/bbdev/api/steps/workload/00_clean_event.step.py index 3256d7227..9621177fb 100644 --- a/bbdev/api/steps/workload/00_clean_event.step.py +++ b/bbdev/api/steps/workload/00_clean_event.step.py @@ -8,12 +8,13 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) +from utils.chip import require_chip from utils.event_common import check_result, get_origin_trace_id -from utils.path import get_buckyball_path +from utils.path import chip_output_root, get_buckyball_path config = { "name": "workload-clean", - "description": "clean workload output directory", + "description": "clean workload output directory for one chip", "flows": ["workload"], "triggers": [queue("workload.clean")], "enqueues": [], @@ -22,7 +23,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) - allowed = {"_trace_id"} + allowed = {"chip", "_trace_id"} unknown = sorted(k for k in input_data if k not in allowed) if unknown: ctx.logger.error(f"Unknown workload clean parameter(s): {', '.join(unknown)}") @@ -35,20 +36,32 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return + try: + chip = require_chip(input_data) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_chip"}, + trace_id=origin_tid, + ) + return + bbdir = get_buckyball_path() - paths = [os.path.join(bbdir, "bb-tests", "output")] + path = chip_output_root(bbdir, chip) - for path in paths: - if os.path.exists(path): - ctx.logger.info("Removing workload directory", {"path": path}) - shutil.rmtree(path) - else: - ctx.logger.info("Workload directory already clean", {"path": path}) + if os.path.exists(path): + ctx.logger.info("Removing workload output directory", {"path": path}) + shutil.rmtree(path) + else: + ctx.logger.info("Workload output directory already clean", {"path": path}) await check_result( ctx, 0, continue_run=False, - extra_fields={"task": "clean", "paths": paths}, + extra_fields={"task": "clean", "chip": chip, "path": path}, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/workload/01_buidl_api.step.py b/bbdev/api/steps/workload/01_buidl_api.step.py index 59f469838..4e697a02b 100644 --- a/bbdev/api/steps/workload/01_buidl_api.step.py +++ b/bbdev/api/steps/workload/01_buidl_api.step.py @@ -9,7 +9,6 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import resolve_chip_compiler_core from utils.path import get_buckyball_path config = { @@ -47,10 +46,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: status=400, body={"error": f"Workload chip does not exist: {chip}"}, ) - try: - resolve_chip_compiler_core(get_buckyball_path(), chip) - except ValueError as error: - return ApiResponse(status=400, body={"error": str(error)}) stable = body.get("stable", False) if not isinstance(stable, bool): return ApiResponse( diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index 86c3c0032..bb8ddf110 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -1,6 +1,5 @@ import os import sys -import shlex import re from pathlib import Path @@ -11,9 +10,8 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import resolve_chip_compiler_core +from utils.build import build_workload from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger from utils.event_common import check_result, get_origin_trace_id config = { @@ -24,21 +22,6 @@ "enqueues": [], } -MODEL_TARGETS = { - "lenet": "buddy-buckyball-lenet-run", - "mobilenet": "buddy-buckyball-mobilenetv3-run", - "resnet": "buddy-buckyball-resnet-run", - "yolo": "buddy-buckyball-yolo26-run", - "bert": "buddy-buckyball-bert-run", - "qwen3": "buddy-buckyball-qwen3-run", - "gemma4": "buddy-buckyball-gemma4-run", - "deepseekr1": "buddy-buckyball-deepseekr1-run", - "llama2": "buddy-buckyball-llama2-run", - "stable-diffusion": "buddy-buckyball-stable-diffusion-run", - "whisper": "buddy-buckyball-whisper-run", - "buddynext": "buddy-buckyball-buddynext-all-run", -} - # CLI model → layout dir under archs/buckyball// MODEL_LAYOUT = { "lenet": "LeNet", @@ -55,22 +38,6 @@ "buddynext": "BuddyNext", } -# CLI model → -DMODEL= value (CMake MODEL_ flags) -MODEL_CMAKE = { - "lenet": "lenet", - "mobilenet": "mobilenetv3", - "resnet": "resnet18", - "yolo": "yolo26", - "bert": "bert", - "qwen3": "qwen3", - "gemma4": "gemma4", - "deepseekr1": "deepseekr1", - "llama2": "llama2", - "stable-diffusion": "stablediffusion", - "whisper": "whisper", - "buddynext": "buddynext", -} - def chips_for_model(bbdir: str, model_key: str) -> set[str]: """Chips that currently ship a layout for this model (many-to-many).""" @@ -96,63 +63,10 @@ def chips_for_model(bbdir: str, model_key: str) -> set[str]: if p.is_dir() and (p / layout).is_dir() } -RUSHB_TARGETS = { - "lenet": { - "bemu": "buddy-buckyball-lenet-rushB-bemu-run", - "verilator": "buddy-buckyball-lenet-rushB-verilator-run", - }, - "mobilenet": { - "bemu": "buddy-buckyball-mobilenetv3-rushB-bemu-run", - "verilator": "buddy-buckyball-mobilenetv3-rushB-verilator-run", - }, - "resnet": { - "bemu": "buddy-buckyball-resnet-rushB-bemu-run", - "verilator": "buddy-buckyball-resnet-rushB-verilator-run", - }, - "yolo": { - "bemu": "buddy-buckyball-yolo26-rushB-bemu-run", - "verilator": "buddy-buckyball-yolo26-rushB-verilator-run", - }, - "bert": { - "bemu": "buddy-buckyball-bert-rushB-bemu-run", - "verilator": "buddy-buckyball-bert-rushB-verilator-run", - }, - "qwen3": { - "bemu": "buddy-buckyball-qwen3-rushB-bemu-run", - "verilator": "buddy-buckyball-qwen3-rushB-verilator-run", - }, - "gemma4": { - "bemu": "buddy-buckyball-gemma4-rushB-bemu-run", - "verilator": "buddy-buckyball-gemma4-rushB-verilator-run", - }, - "deepseekr1": { - "bemu": "buddy-buckyball-deepseekr1-rushB-bemu-run", - "verilator": "buddy-buckyball-deepseekr1-rushB-verilator-run", - }, - "llama2": { - "bemu": "buddy-buckyball-llama2-rushB-bemu-run", - "verilator": "buddy-buckyball-llama2-rushB-verilator-run", - }, - "stable-diffusion": { - "bemu": "buddy-buckyball-stable-diffusion-rushB-bemu-run", - "verilator": "buddy-buckyball-stable-diffusion-rushB-verilator-run", - }, - "whisper": { - "bemu": "buddy-buckyball-whisper-rushB-bemu-run", - "verilator": "buddy-buckyball-whisper-rushB-verilator-run", - }, - "buddynext": { - "bemu": "buddy-buckyball-buddynext-rushB-bemu-run", - "verilator": "buddy-buckyball-buddynext-rushB-verilator-run", - }, -} - async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - workload_dir = f"{bbdir}/bb-tests" - build_dir = f"{workload_dir}/build" allowed = {"chip", "model", "stable", "rushB", "_trace_id"} unknown = sorted(k for k in input_data if k not in allowed) if unknown: @@ -189,16 +103,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - try: - core_package = resolve_chip_compiler_core(bbdir, chip) - except ValueError as error: - ctx.logger.error(str(error)) - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "invalid_chip_manifest", "chip": chip}, - trace_id=origin_tid, - ) - return model = input_data.get("model", "") stable = input_data.get("stable", False) rushb_backend = input_data.get("rushB") @@ -221,24 +125,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - target = "" if model: model_key = model.lower() - target = MODEL_TARGETS.get(model_key) - if target is None: - ctx.logger.error(f"Unknown model: {model}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "unknown_model", "model": model}, - trace_id=origin_tid, - ) - return layout = MODEL_LAYOUT.get(model_key) if layout is None: - ctx.logger.error(f"Missing layout mapping for model {model}") + ctx.logger.error(f"Unknown model: {model}") await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_model_layout", "model": model}, + extra_fields={"error": "unknown_model", "model": model}, trace_id=origin_tid, ) return @@ -261,62 +155,27 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - if rushb_backend is not None: - target = RUSHB_TARGETS.get(model_key, {}).get(rushb_backend) - if target is None: - ctx.logger.error(f"Missing rushB target registration for model {model}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={ - "error": "missing_rushB_target", - "model": model, - "rushB": rushb_backend, - }, - trace_id=origin_tid, - ) - return - elif rushb_backend is not None: - target = f"rushB-{rushb_backend}-workloads-build" - os.makedirs(build_dir, exist_ok=True) - - chip_arg = f"-DBUCKYBALL_WORKLOAD_CHIP={shlex.quote(chip)}" - core_arg = f"-DBUCKYBALL_WORKLOAD_CORE={shlex.quote(core_package.name)}" - stable_arg = "-DBUCKYBALL_STABLE=ON" if stable else "-DBUCKYBALL_STABLE=OFF" - model_arg = "" - if model: - cmake_model = MODEL_CMAKE[model.lower()] - model_arg = f" -DMODEL={shlex.quote(cmake_model)}" - ninja_target = f" {shlex.quote(target)}" if target else "" - inner = ( - f"cd {shlex.quote(build_dir)} && " - f"cmake -G Ninja {chip_arg} {core_arg} {stable_arg}{model_arg} " - f"-DPython3_EXECUTABLE=\"$(which python3)\" .. && " - f"ninja -j{os.cpu_count()}{ninja_target}" - ) - command = f"cd {shlex.quote(bbdir)} && nix develop -c bash -c {shlex.quote(inner)}" - ctx.logger.info( - "Executing workload command", {"command": command, "cwd": build_dir} - ) - result = stream_run_logger( - cmd=command, - logger=ctx.logger, - cwd=workload_dir, - executable="bash", - stdout_prefix="workload build", - stderr_prefix="workload build", - ) + try: + build_workload( + bbdir, + chip, + model=model.lower() if model else "", + rushb=rushb_backend, + stable=stable, + ) + except (ValueError, RuntimeError) as error: + ctx.logger.error(str(error)) + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "workload_build_failed", "chip": chip, "detail": str(error)}, + trace_id=origin_tid, + ) + return - # ================================================================================== - # Return simulation result - # ================================================================================== - # This is the end of run workflow, status no longer set to processing - success_result, failure_result = await check_result( - ctx, result.returncode, continue_run=False, + await check_result( + ctx, 0, continue_run=False, extra_fields={"chip": chip, "model": model, "rushB": rushb_backend}, trace_id=origin_tid) - # ================================================================================== - # finish workflow - # ================================================================================== return diff --git a/bbdev/api/steps/workload/02_tohex_api.step.py b/bbdev/api/steps/workload/02_tohex_api.step.py index 8265c1c9f..1400909bf 100644 --- a/bbdev/api/steps/workload/02_tohex_api.step.py +++ b/bbdev/api/steps/workload/02_tohex_api.step.py @@ -1,10 +1,10 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import get_buckyball_path +from utils.chip import require_chip config = { "name": "workload-tohex-api", - "description": "convert elf to hex", + "description": "convert chip workload ELF to hex", "flows": ["workload"], "triggers": [api("POST", "/workload/tohex")], "enqueues": ["workload.tohex"], @@ -12,8 +12,18 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = request.body or {} - data = {} - await ctx.enqueue({"topic": "workload.tohex", "data": {**data, "_trace_id": ctx.trace_id}}) + try: + chip = require_chip(body) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + + unknown = sorted(k for k in body if k not in {"chip"}) + if unknown: + return ApiResponse( + status=400, + body={"error": f"Unknown workload tohex parameter(s): {', '.join(unknown)}"}, + ) + + await ctx.enqueue({"topic": "workload.tohex", "data": {**body, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/02_tohex_event.step.py b/bbdev/api/steps/workload/02_tohex_event.step.py index 0c56f339b..c6260ea98 100644 --- a/bbdev/api/steps/workload/02_tohex_event.step.py +++ b/bbdev/api/steps/workload/02_tohex_event.step.py @@ -5,18 +5,18 @@ from motia import FlowContext, queue -# Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger +from utils.chip import require_chip from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path, workload_tests_root +from utils.stream_run import stream_run_logger_async config = { "name": "workload-tohex", - "description": "convert all -baremetal ELF files under bb-tests/output/workloads/src to hex", + "description": "convert all -baremetal ELF files under chip workload output to hex", "flows": ["workload"], "triggers": [queue("workload.tohex")], "enqueues": [], @@ -26,8 +26,19 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() + try: + chip = require_chip(input_data) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "missing_chip"}, + trace_id=origin_tid, + ) + return + script = f"{bbdir}/bbdev/api/steps/workload/scripts/elf2hex.py" - search_root = Path(f"{bbdir}/bb-tests/output/workloads/src") + search_root = Path(workload_tests_root(bbdir, chip)) ctx.logger.info("Search root", {"path": str(search_root)}) @@ -39,12 +50,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) return - ctx.logger.info("Search root is a directory", {"path": str(search_root)}) - elf_files = sorted(p for p in search_root.rglob("*-baremetal") if p.is_file()) - ctx.logger.info("ELF files found", {"count": len(elf_files)}) - if not elf_files: ctx.logger.warn( "No -baremetal ELF files found", @@ -53,16 +60,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result(ctx, 0, continue_run=False, trace_id=origin_tid) return - ctx.logger.info( - "Converting ELF files to hex", - {"count": len(elf_files), "search_root": str(search_root)}, - ) - overall_rc = 0 for elf in elf_files: command = f"python3 {script} {elf}" - ctx.logger.info("Executing to-hex command", {"command": command}) - result = stream_run_logger( + result = await stream_run_logger_async( cmd=command, logger=ctx.logger, cwd=str(elf.parent), @@ -77,13 +78,4 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: {"elf": str(elf), "returncode": result.returncode}, ) - # ================================================================================== - # Return conversion result - # ================================================================================== - success_result, failure_result = await check_result( - ctx, overall_rc, continue_run=False, trace_id=origin_tid) - - # ================================================================================== - # finish workflow - # ================================================================================== - return + await check_result(ctx, overall_rc, continue_run=False, trace_id=origin_tid) diff --git a/bbdev/api/steps/workload/README.md b/bbdev/api/steps/workload/README.md index e6671d788..8be5a5c41 100644 --- a/bbdev/api/steps/workload/README.md +++ b/bbdev/api/steps/workload/README.md @@ -7,13 +7,14 @@ Workload build workflow in Buckyball framework, used to build test workloads and ### `clean` **Endpoint**: `POST /workload/clean` -**Function**: Clean workload output directory. +**Function**: Clean workload output directory for one chip. -**Parameters**: None. +**Parameters**: +- **`chip`** - Required chip name. **Examples**: ```bash -bbdev workload --clean +bbdev workload --clean "--chip toy" ``` ### `build` @@ -28,7 +29,7 @@ bbdev workload --clean - **`rushB`** - Optional host-native backend: `bemu` or `verilator`. Requires `model`. It builds host-native CPU and rushB-lowered accelerator objects. -For chip workloads under paths like `*/chips/`, only the directory selected by `chip` is synced to `bb-tests/output/workloads`. +For chip workloads under paths like `*/chips/`, only the directory selected by `chip` is synced to `bb-tests/output//workloads`. **Examples**: ```bash @@ -62,6 +63,6 @@ bbdev workload --build "--chip toy" ## Notes -- Workload build entry is `bb-tests` +- Workload build directory is `bb-tests/workloads/build/` - Workload source code is distributed under `bb-tests/workloads/src` and `examples/*/*/workloads` -- Workload binaries are emitted under `bb-tests/output/workloads/src` +- Workload binaries are emitted under `bb-tests/output//workloads/src` diff --git a/bbdev/api/steps/yosys/01_run_api.step.py b/bbdev/api/steps/yosys/01_run_api.step.py index 95c097150..fe2385bd7 100644 --- a/bbdev/api/steps/yosys/01_run_api.step.py +++ b/bbdev/api/steps/yosys/01_run_api.step.py @@ -3,13 +3,13 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import get_buckyball_path +from utils.chip import require_chip scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from yosys_log import make_yosys_log_dir, req_arg +from yosys_log import req_arg config = { "name": "yosys-run", @@ -21,17 +21,23 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = req.body or {} - log_dir = req_arg(body, "log_dir") or make_yosys_log_dir(bbdir, ctx.trace_id) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) data = { - "output_dir": req_arg(body, "output_dir") or f"{bbdir}/arch/build/", - "log_dir": log_dir, + "chip": chip, "top": req_arg(body, "top") or "DigitalTop", - "config": req_arg(body, "config"), "vcd": req_arg(body, "vcd"), "from_run_workflow": True, } + output_dir = req_arg(body, "output_dir") + if output_dir: + data["output_dir"] = output_dir + log_dir = req_arg(body, "log_dir") + if log_dir: + data["log_dir"] = log_dir await ctx.enqueue({"topic": "yosys.run", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/yosys/02_verilog_api.step.py b/bbdev/api/steps/yosys/02_verilog_api.step.py index f03ae57a7..c927410a9 100644 --- a/bbdev/api/steps/yosys/02_verilog_api.step.py +++ b/bbdev/api/steps/yosys/02_verilog_api.step.py @@ -3,13 +3,13 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import get_buckyball_path +from utils.chip import require_chip scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from yosys_log import make_yosys_log_dir, req_arg +from yosys_log import req_arg config = { "name": "yosys-verilog-api", @@ -21,15 +21,18 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = req.body or {} - log_dir = req_arg(body, "log_dir") or make_yosys_log_dir(bbdir, ctx.trace_id) - - data = { - "output_dir": req_arg(body, "output_dir") or f"{bbdir}/arch/build/", - "log_dir": log_dir, - "top": req_arg(body, "top") or "DigitalTop", - "config": req_arg(body, "config"), - } + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + + data = {"chip": chip, "top": req_arg(body, "top") or "DigitalTop"} + output_dir = req_arg(body, "output_dir") + if output_dir: + data["output_dir"] = output_dir + log_dir = req_arg(body, "log_dir") + if log_dir: + data["log_dir"] = log_dir await ctx.enqueue({"topic": "yosys.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/yosys/03_synth_api.step.py b/bbdev/api/steps/yosys/03_synth_api.step.py index d3da70b9b..1b3cd9bc9 100644 --- a/bbdev/api/steps/yosys/03_synth_api.step.py +++ b/bbdev/api/steps/yosys/03_synth_api.step.py @@ -3,13 +3,14 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.path import get_buckyball_path +from utils.chip import require_chip +from utils.path import get_arch_build_dir, get_buckyball_path scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from yosys_log import make_yosys_log_dir, req_arg +from yosys_log import req_arg config = { "name": "yosys-synth-api", @@ -23,22 +24,28 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = req.body or {} - log_dir = req_arg(body, "log_dir") or make_yosys_log_dir(bbdir, ctx.trace_id) + try: + chip = require_chip(body) + rtl = get_arch_build_dir(bbdir, chip, req_arg(body, "output_dir")) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) data = { - "output_dir": req_arg(body, "output_dir") or f"{bbdir}/arch/build/", - "log_dir": log_dir, + "chip": chip, + "output_dir": rtl, "top": req_arg(body, "top") or "DigitalTop", - "config": req_arg(body, "config"), "vcd": req_arg(body, "vcd"), } + log_dir = req_arg(body, "log_dir") + if log_dir: + data["log_dir"] = log_dir await ctx.enqueue( { "topic": "ip-replace.run", "data": { **data, - "source_list": f"{data['output_dir']}/yosys_sources.list", - "ip_replace_output_dir": log_dir, + "source_list": os.path.join(rtl, "yosys_sources.list"), + "ip_replace_output_dir": log_dir or rtl, "consumer": "yosys", "next_topic": "yosys.synth", "_trace_id": ctx.trace_id, diff --git a/bbdev/api/steps/yosys/03_synth_event.step.py b/bbdev/api/steps/yosys/03_synth_event.step.py index 497ef1761..014507117 100644 --- a/bbdev/api/steps/yosys/03_synth_event.step.py +++ b/bbdev/api/steps/yosys/03_synth_event.step.py @@ -10,8 +10,9 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger +from utils.chip import require_chip +from utils.path import get_arch_build_dir, get_buckyball_path, get_run_log_dir +from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id config = { @@ -63,7 +64,16 @@ def load_yosys_config(): async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - build_dir = input_data.get("output_dir", f"{bbdir}/arch/build/") + try: + chip = require_chip(input_data) + build_dir = get_arch_build_dir(bbdir, chip, input_data.get("output_dir")) + except ValueError as exc: + await check_result( + ctx, 1, continue_run=False, + extra_fields={"task": "synth", "error": str(exc)}, + trace_id=origin_tid, + ) + return yosys_cfg = load_yosys_config() top_module = input_data.get("top") or yosys_cfg.get("top") or "DigitalTop" @@ -96,8 +106,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return failure_result stamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") - fallback_log_dir = os.path.join(os.path.dirname(__file__), "log", f"{stamp}-{origin_tid[:8]}") - yosys_output_dir = input_data.get("log_dir", fallback_log_dir) + yosys_output_dir = input_data.get("log_dir") or get_run_log_dir( + bbdir, chip, "synth", stamp, "yosys", top_module, input_data.get("output_dir"), + ) os.makedirs(yosys_output_dir, exist_ok=True) ctx.logger.info(f"Yosys log dir: {yosys_output_dir}") @@ -128,7 +139,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f.write("stat\n") f.write(f"tee -o {yosys_output_dir}/area_report.txt stat\n") - result = stream_run_logger( + result = await stream_run_logger_async( cmd=f"yosys -s {yosys_script}", logger=ctx.logger, cwd=build_dir, @@ -197,7 +208,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f.write(f"report_power -digits 4 > {power_report_file}\n") f.write("exit\n") - sta_result = stream_run_logger( + sta_result = await stream_run_logger_async( cmd=f"sta {sta_script}", logger=ctx.logger, cwd=yosys_output_dir, diff --git a/bbdev/api/steps/yosys/scripts/yosys-config.yaml b/bbdev/api/steps/yosys/scripts/yosys-config.yaml index 49ed4ed51..005e2cdf5 100644 --- a/bbdev/api/steps/yosys/scripts/yosys-config.yaml +++ b/bbdev/api/steps/yosys/scripts/yosys-config.yaml @@ -9,9 +9,6 @@ liberty: # Default top module name (override with --top on CLI) top: DigitalTop -# Elaborate config class (override with --config on CLI) -elaborate_config: sims.verilator.BuckyballToyVerilatorConfig - # OpenSTA timing analysis settings # Clock period in nanoseconds (used for STA constraint) # Default 10ns = 100MHz diff --git a/bbdev/api/tests/fixtures/bemu-analysis/bdb.ndjson b/bbdev/api/tests/fixtures/bemu-analysis/bdb.ndjson new file mode 100644 index 000000000..5cafa8fed --- /dev/null +++ b/bbdev/api/tests/fixtures/bemu-analysis/bdb.ndjson @@ -0,0 +1,4 @@ +{"type":"itrace","clk":10,"event":"complete","funct":"0x00","pc":"0x0","rs1":"0x0","rs2":"0x0"} +{"type":"itrace","clk":20,"event":"complete","funct":"0x41","pc":"0x0","rs1":"0x0","rs2":"0x0000000100010400"} +{"type":"mtrace","clk":15,"event":"read","addr":"0x0","rows":1024,"line_bytes":16,"row_stride":16,"vbank_id":0,"pbank_id":0,"group_id":0} +{"type":"mtrace","clk":16,"event":"write","addr":"0x0","rows":1,"line_bytes":16,"row_stride":16,"vbank_id":1,"pbank_id":1,"group_id":0} diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index 2f3cd8819..2f9843f7d 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -25,6 +25,7 @@ def main(): "test_api_firesim_infrasetup.py", "test_api_firesim_runworkload.py", "test_api_bebop_bemu_sim.py", + "test_api_bebop_bemu_analysis.py", "test_api_bebop_bemu_batch.py", "test_api_bebop_verilator_clean.py", "test_api_bebop_verilator_verilog.py", diff --git a/bbdev/api/tests/test_api_bebop_bemu_analysis.py b/bbdev/api/tests/test_api_bebop_bemu_analysis.py new file mode 100644 index 000000000..4aba36d88 --- /dev/null +++ b/bbdev/api/tests/test_api_bebop_bemu_analysis.py @@ -0,0 +1,12 @@ +from pathlib import Path +import shutil +import tempfile + +from _api_test_helper import run_bbdev_case + +src = Path(__file__).resolve().parent / "fixtures" / "bemu-analysis" +dst = Path(tempfile.mkdtemp()) / "bemu-analysis" +shutil.copytree(src, dst) +run_bbdev_case( + f"bbdev bebop-bemu --analysis '--chip pebble --log-dir {dst} --itrace --mtrace'" +) diff --git a/bbdev/api/tests/test_api_bebop_bemu_sim.py b/bbdev/api/tests/test_api_bebop_bemu_sim.py index 7a8e4bc1a..179a87d2a 100644 --- a/bbdev/api/tests/test_api_bebop_bemu_sim.py +++ b/bbdev/api/tests/test_api_bebop_bemu_sim.py @@ -1,4 +1,4 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_vecunit_matmul_ones-singlecore-baremetal'") -run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_vecunit_matmul_ones-linux --pk'") +run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_relu_test-singlecore-baremetal'") +run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_relu_test-linux --pk'") diff --git a/bbdev/api/tests/test_api_bebop_p2e_buildbitstream.py b/bbdev/api/tests/test_api_bebop_p2e_buildbitstream.py index cfbf70b9b..ddac6f048 100644 --- a/bbdev/api/tests/test_api_bebop_p2e_buildbitstream.py +++ b/bbdev/api/tests/test_api_bebop_p2e_buildbitstream.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-p2e --buildbitstream '--config sims.p2e.P2EToyConfig'") +run_bbdev_case("bbdev bebop-p2e --buildbitstream '--chip toy'") diff --git a/bbdev/api/tests/test_api_bebop_p2e_clean.py b/bbdev/api/tests/test_api_bebop_p2e_clean.py index 32926b895..c67a885ad 100644 --- a/bbdev/api/tests/test_api_bebop_p2e_clean.py +++ b/bbdev/api/tests/test_api_bebop_p2e_clean.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-p2e --clean '--config sims.p2e.P2EToyConfig'") +run_bbdev_case("bbdev bebop-p2e --clean '--chip toy'") diff --git a/bbdev/api/tests/test_api_bebop_p2e_verilog.py b/bbdev/api/tests/test_api_bebop_p2e_verilog.py index 3878e9e86..b3363f1ad 100644 --- a/bbdev/api/tests/test_api_bebop_p2e_verilog.py +++ b/bbdev/api/tests/test_api_bebop_p2e_verilog.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-p2e --verilog '--config sims.p2e.P2EToyConfig'") +run_bbdev_case("bbdev bebop-p2e --verilog '--chip toy'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_batch.py b/bbdev/api/tests/test_api_bebop_verilator_batch.py index 1c383a2ef..5546cc79d 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_batch.py +++ b/bbdev/api/tests/test_api_bebop_verilator_batch.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --batch '--chip toy --config sims.verilator.BuckyballToyVerilatorConfig --test elf-tests'") +run_bbdev_case("bbdev bebop-verilator --batch '--chip toy --test elf-tests'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_build.py b/bbdev/api/tests/test_api_bebop_verilator_build.py index 27380320e..9dee15463 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_build.py +++ b/bbdev/api/tests/test_api_bebop_verilator_build.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --build '--jobs 16 --config sims.verilator.BuckyballToyVerilatorConfig'") +run_bbdev_case("bbdev bebop-verilator --build '--jobs 16 --chip toy'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_clean.py b/bbdev/api/tests/test_api_bebop_verilator_clean.py index fc16e0c20..780de8680 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_clean.py +++ b/bbdev/api/tests/test_api_bebop_verilator_clean.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --clean '--config sims.verilator.BuckyballToyVerilatorConfig'") +run_bbdev_case("bbdev bebop-verilator --clean '--chip toy'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_run.py b/bbdev/api/tests/test_api_bebop_verilator_run.py index 1d9a88ae4..712e0479f 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_run.py +++ b/bbdev/api/tests/test_api_bebop_verilator_run.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --run '--diff --no-wave --jobs 16 --binary toy_im2col_k3_test-singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig'") +run_bbdev_case("bbdev bebop-verilator --run '--diff --no-wave --jobs 16 --binary toy_relu_test-singlecore-baremetal --chip toy'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_sim.py b/bbdev/api/tests/test_api_bebop_verilator_sim.py index 80c1e4a2e..4b4cccdc8 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_sim.py +++ b/bbdev/api/tests/test_api_bebop_verilator_sim.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --sim '--binary toy_vecunit_matmul_ones-singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig --itrace --mtrace --pmctrace --ctrace --banktrace'") +run_bbdev_case("bbdev bebop-verilator --sim '--binary toy_relu_test-singlecore-baremetal --chip toy --itrace --mtrace --pmctrace --ctrace --banktrace'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_verilog.py b/bbdev/api/tests/test_api_bebop_verilator_verilog.py index cc64740c7..1702bc0ae 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_verilog.py +++ b/bbdev/api/tests/test_api_bebop_verilator_verilog.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --verilog '--config sims.verilator.BuckyballToyVerilatorConfig'") +run_bbdev_case("bbdev bebop-verilator --verilog '--chip toy'") diff --git a/bbdev/api/tests/test_api_dc_verilog.py b/bbdev/api/tests/test_api_dc_verilog.py index 397326394..41aa489fe 100644 --- a/bbdev/api/tests/test_api_dc_verilog.py +++ b/bbdev/api/tests/test_api_dc_verilog.py @@ -3,10 +3,10 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev dc --verilog '--config sims.verilator.BuckyballToyVerilatorConfig'") +run_bbdev_case("bbdev dc --verilog '--chip toy'") root = Path(__file__).resolve().parents[3] -output_dir = root / "arch" / "build" / "sims.verilator.BuckyballToyVerilatorConfig" +output_dir = root / "arch" / "build" / "toy" / "sims.verilator.BuckyballToyVerilatorConfig" manifest = json.loads((output_dir / "ip-replace" / "sram_manifest.json").read_text()) source_list = (output_dir / "ip-replace" / "dc_sources.list").read_text().splitlines() diff --git a/bbdev/api/tests/test_api_regression_buildbitstream.py b/bbdev/api/tests/test_api_regression_buildbitstream.py new file mode 100644 index 000000000..c434898d9 --- /dev/null +++ b/bbdev/api/tests/test_api_regression_buildbitstream.py @@ -0,0 +1,272 @@ +import asyncio +import importlib.util +import os +import sys +from pathlib import Path +from types import SimpleNamespace + +API = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(API)) + + +def load_step(rel: str): + path = API / "steps" / rel + name = path.name.replace(".", "_") + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load step: {path}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class FakeState: + def __init__(self): + self.store = {} + + async def set(self, scope, key, val): + self.store[(scope, key)] = val + + +class FakeCtx: + def __init__(self, trace_id="tid-origin"): + self.trace_id = trace_id + self.enqueued = [] + self.state = FakeState() + self.logger = SimpleNamespace( + info=lambda *a, **k: None, + error=lambda *a, **k: None, + ) + + async def enqueue(self, item): + self.enqueued.append(item) + + +def run(coro): + return asyncio.run(coro) + + +def test_api_missing_chip(): + mod = load_step("regression/00_buildbitstream_api.step.py") + ctx = FakeCtx() + resp = run(mod.handler(SimpleNamespace(body={"config": "sims.p2e.P2EPebbleLinuxConfig"}), ctx)) + assert resp.status == 400 + assert "chip" in resp.body["error"] + assert ctx.enqueued == [] + + +def test_api_missing_config(): + mod = load_step("regression/00_buildbitstream_api.step.py") + ctx = FakeCtx() + resp = run(mod.handler(SimpleNamespace(body={"chip": "pebble"}), ctx)) + assert resp.status == 400 + assert "config" in resp.body["error"] + assert ctx.enqueued == [] + + +def test_api_enqueues_regression_event(): + mod = load_step("regression/00_buildbitstream_api.step.py") + ctx = FakeCtx("tid-api") + resp = run( + mod.handler( + SimpleNamespace( + body={ + "chip": "pebble", + "config": "sims.p2e.P2EPebbleLinuxConfig", + "output_dir": "/tmp/bs", + } + ), + ctx, + ) + ) + assert resp.status == 202 + assert resp.body == {"trace_id": "tid-api"} + assert len(ctx.enqueued) == 1 + item = ctx.enqueued[0] + assert item["topic"] == "regression.buildbitstream" + assert item["data"]["chip"] == "pebble" + assert item["data"]["config"] == "sims.p2e.P2EPebbleLinuxConfig" + assert item["data"]["output_dir"] == "/tmp/bs" + assert item["data"]["_trace_id"] == "tid-api" + assert "from_regression_buildbitstream" not in item["data"] + + +def test_event_sets_processing_and_enqueues_verilog(): + mod = load_step("regression/00_buildbitstream_event.step.py") + ctx = FakeCtx("tid-evt") + run( + mod.handler( + { + "chip": "pebble", + "config": "sims.p2e.P2EPebbleLinuxConfig", + "output_dir": "/tmp/bs", + "_trace_id": "tid-origin", + }, + ctx, + ) + ) + assert ctx.state.store[("tid-origin", "processing")]["processing"] is True + assert ("tid-origin", "success") not in ctx.state.store + assert ("tid-origin", "failure") not in ctx.state.store + assert len(ctx.enqueued) == 1 + item = ctx.enqueued[0] + assert item["topic"] == "bebop.p2e.verilog" + data = item["data"] + assert data["from_regression_buildbitstream"] is True + assert data["chip"] == "pebble" + assert data["config"] == "sims.p2e.P2EPebbleLinuxConfig" + assert data["_trace_id"] == "tid-origin" + assert data["build_dir"] == "/tmp/bs" + assert "output_dir" not in data + + +def _patch_verilog(mod, monkeypatch, rc=0): + monkeypatch.setattr(mod, "get_buckyball_path", lambda: "/tmp/bb") + monkeypatch.setattr( + mod, "get_verilator_build_dir", lambda bb, cfg, out: out or "/tmp/bb/arch/build/cfg" + ) + monkeypatch.setattr(mod, "cleanup_strays", lambda *a, **k: None) + monkeypatch.setattr(mod, "normalize_p2e_timescale", lambda *a, **k: None) + monkeypatch.setattr(mod.os, "makedirs", lambda *a, **k: None) + monkeypatch.setattr( + mod, "stream_run_logger", lambda **k: SimpleNamespace(returncode=rc) + ) + + +def test_verilog_without_flag_finalizes(monkeypatch): + mod = load_step("mill/07_bebop_p2e_verilog_event.step.py") + _patch_verilog(mod, monkeypatch, rc=0) + ctx = FakeCtx() + run(mod.handler({"config": "sims.p2e.P2EPebbleLinuxConfig", "_trace_id": "tid-origin"}, ctx)) + assert ctx.enqueued == [] + success = ctx.state.store[("tid-origin", "success")] + assert success["body"]["success"] is True + assert success["body"]["task"] == "verilog" + assert ("tid-origin", "processing") not in ctx.state.store + + +def test_verilog_with_flag_continues_and_enqueues(monkeypatch): + mod = load_step("mill/07_bebop_p2e_verilog_event.step.py") + _patch_verilog(mod, monkeypatch, rc=0) + ctx = FakeCtx() + run( + mod.handler( + { + "chip": "pebble", + "config": "sims.p2e.P2EPebbleLinuxConfig", + "from_regression_buildbitstream": True, + "build_dir": "/tmp/bs", + "_trace_id": "tid-origin", + }, + ctx, + ) + ) + proc = ctx.state.store[("tid-origin", "processing")] + assert proc["processing"] is True + assert ("tid-origin", "success") not in ctx.state.store + assert len(ctx.enqueued) == 1 + item = ctx.enqueued[0] + assert item["topic"] == "bebop.p2e.buildbitstream" + data = item["data"] + assert data["from_regression_buildbitstream"] is True + assert data["chip"] == "pebble" + assert data["vsrc_dir"] == "/tmp/bb/arch/build/cfg" + assert data["build_dir"] == "/tmp/bs" + assert data["_trace_id"] == "tid-origin" + + +def test_verilog_failure_with_flag_does_not_enqueue(monkeypatch): + mod = load_step("mill/07_bebop_p2e_verilog_event.step.py") + _patch_verilog(mod, monkeypatch, rc=7) + ctx = FakeCtx() + run( + mod.handler( + { + "config": "sims.p2e.P2EPebbleLinuxConfig", + "from_regression_buildbitstream": True, + "_trace_id": "tid-origin", + }, + ctx, + ) + ) + assert ctx.enqueued == [] + failure = ctx.state.store[("tid-origin", "failure")] + assert failure["body"]["failure"] is True + assert failure["body"]["returncode"] == 7 + assert ("tid-origin", "success") not in ctx.state.store + + +def _patch_buildbitstream(mod, monkeypatch): + monkeypatch.setattr(mod, "get_buckyball_path", lambda: "/tmp/bb") + monkeypatch.setattr(mod, "get_verilator_build_dir", lambda bb, cfg, vsrc: vsrc or "/tmp/vsrc") + monkeypatch.setattr(mod.os.path, "isdir", lambda p: True) + monkeypatch.setattr(mod.os.path, "exists", lambda p: True) + monkeypatch.setattr(mod.os, "makedirs", lambda *a, **k: None) + monkeypatch.setattr(mod, "stream_run_logger", lambda **k: SimpleNamespace(returncode=0)) + + +def test_buildbitstream_without_flag_keeps_relative_bitstream(monkeypatch): + mod = load_step("bebop/p2e/02_buildbitstream_event.step.py") + _patch_buildbitstream(mod, monkeypatch) + ctx = FakeCtx() + run( + mod.handler( + { + "config": "sims.p2e.P2EPebbleLinuxConfig", + "vsrc_dir": "/tmp/vsrc", + "output_dir": "rel-bs", + "_trace_id": "tid-origin", + }, + ctx, + ) + ) + success = ctx.state.store[("tid-origin", "success")]["body"] + assert success["success"] is True + assert success["bitstream"] == os.path.join("rel-bs", "fpgaCompDir", "bitstream.bit") + + +def test_buildbitstream_with_flag_failure_keeps_relative_bitstream(monkeypatch): + mod = load_step("bebop/p2e/02_buildbitstream_event.step.py") + _patch_buildbitstream(mod, monkeypatch) + monkeypatch.setattr(mod, "stream_run_logger", lambda **k: SimpleNamespace(returncode=1)) + ctx = FakeCtx() + run( + mod.handler( + { + "config": "sims.p2e.P2EPebbleLinuxConfig", + "vsrc_dir": "/tmp/vsrc", + "output_dir": "rel-bs", + "from_regression_buildbitstream": True, + "_trace_id": "tid-origin", + }, + ctx, + ) + ) + failure = ctx.state.store[("tid-origin", "failure")]["body"] + assert failure["failure"] is True + assert failure["bitstream"] == os.path.join("rel-bs", "fpgaCompDir", "bitstream.bit") + assert not os.path.isabs(failure["bitstream"]) + + +def test_buildbitstream_with_flag_absolutes_bitstream(monkeypatch): + mod = load_step("bebop/p2e/02_buildbitstream_event.step.py") + _patch_buildbitstream(mod, monkeypatch) + ctx = FakeCtx() + run( + mod.handler( + { + "config": "sims.p2e.P2EPebbleLinuxConfig", + "vsrc_dir": "/tmp/vsrc", + "output_dir": "rel-bs", + "from_regression_buildbitstream": True, + "_trace_id": "tid-origin", + }, + ctx, + ) + ) + success = ctx.state.store[("tid-origin", "success")]["body"] + assert success["success"] is True + assert success["bitstream"] == os.path.abspath( + os.path.join("rel-bs", "fpgaCompDir", "bitstream.bit") + ) + assert os.path.isabs(success["bitstream"]) diff --git a/bbdev/api/tests/test_api_regression_check.py b/bbdev/api/tests/test_api_regression_check.py new file mode 100644 index 000000000..fc0fd47dc --- /dev/null +++ b/bbdev/api/tests/test_api_regression_check.py @@ -0,0 +1,239 @@ +import asyncio +import importlib.util +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +API = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(API)) + + +def load_step(rel: str): + path = API / "steps" / rel + name = path.name.replace(".", "_") + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load step: {path}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class FakeState: + def __init__(self): + self.store = {} + + async def set(self, scope, key, val): + self.store[(scope, key)] = val + + +class FakeCtx: + def __init__(self, trace_id="tid-origin"): + self.trace_id = trace_id + self.enqueued = [] + self.state = FakeState() + self.logger = SimpleNamespace( + info=lambda *a, **k: None, + error=lambda *a, **k: None, + ) + + async def enqueue(self, item): + self.enqueued.append(item) + + +def run(coro): + return asyncio.run(coro) + + +def test_api_missing_chip(tmp_path): + bs = tmp_path / "bitstream.bit" + bs.write_text("x") + mod = load_step("regression/01_check_api.step.py") + ctx = FakeCtx() + resp = run(mod.handler(SimpleNamespace(body={"bitstream": str(bs)}), ctx)) + assert resp.status == 400 + assert "chip" in resp.body["error"] + assert ctx.enqueued == [] + + +def test_api_missing_bitstream(): + mod = load_step("regression/01_check_api.step.py") + ctx = FakeCtx() + resp = run(mod.handler(SimpleNamespace(body={"chip": "pebble"}), ctx)) + assert resp.status == 400 + assert "bitstream" in resp.body["error"] + assert ctx.enqueued == [] + + +def test_api_bitstream_not_found(): + mod = load_step("regression/01_check_api.step.py") + ctx = FakeCtx() + resp = run( + mod.handler( + SimpleNamespace(body={"chip": "pebble", "bitstream": "/no/such/bitstream.bit"}), + ctx, + ) + ) + assert resp.status == 400 + assert "not found" in resp.body["error"] + assert ctx.enqueued == [] + + +def test_api_enqueues_regression_event(tmp_path): + bs = tmp_path / "bitstream.bit" + bs.write_text("x") + mod = load_step("regression/01_check_api.step.py") + ctx = FakeCtx("tid-api") + resp = run( + mod.handler( + SimpleNamespace(body={"chip": "pebble", "bitstream": str(bs)}), + ctx, + ) + ) + assert resp.status == 202 + assert resp.body == {"trace_id": "tid-api"} + assert len(ctx.enqueued) == 1 + item = ctx.enqueued[0] + assert item["topic"] == "regression.check" + assert item["data"]["chip"] == "pebble" + assert item["data"]["bitstream"] == str(bs) + assert item["data"]["_trace_id"] == "tid-api" + assert "from_regression_check" not in item["data"] + + +def test_event_sets_processing_and_enqueues_batch(tmp_path): + bs = tmp_path / "bitstream.bit" + bs.write_text("x") + mod = load_step("regression/01_check_event.step.py") + ctx = FakeCtx("tid-evt") + run( + mod.handler( + { + "chip": "pebble", + "bitstream": str(bs), + "_trace_id": "tid-origin", + }, + ctx, + ) + ) + assert ctx.state.store[("tid-origin", "processing")]["processing"] is True + assert ("tid-origin", "success") not in ctx.state.store + assert ("tid-origin", "failure") not in ctx.state.store + assert len(ctx.enqueued) == 1 + item = ctx.enqueued[0] + assert item["topic"] == "bebop.p2e.batch" + data = item["data"] + assert data["from_regression_check"] is True + assert data["test"] == "pk-tests" + assert data["chip"] == "pebble" + assert data["bitstream"] == str(bs) + assert data["_trace_id"] == "tid-origin" + + +def _prep_batch_case(tmp_path): + build = tmp_path / "case" + fpga = build / "fpgaCompDir" + fpga.mkdir(parents=True) + bs = fpga / "bitstream.bit" + bs.write_text("x") + rt = build / "vvacDir" / "runtimeDir" + rt.mkdir(parents=True) + (rt / "rtcfg").write_text("x") + lib = rt / "lib" / "lib_arm" + lib.mkdir(parents=True) + (lib / "libvCtb.so").write_text("x") + vsrc = tmp_path / "vsrc" + vsrc.mkdir() + return str(bs), str(vsrc) + + +def _patch_batch(mod, monkeypatch, tmp_path, vsrc, stdout="", stderr="", rc=0): + monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) + monkeypatch.setattr(mod, "get_verilator_build_dir", lambda *a, **k: vsrc) + monkeypatch.setattr(mod, "regression_workload_toml", lambda *a, **k: "/tmp/workloads-pk.toml") + + def fake_stream(**k): + prefix = k.get("stdout_prefix", "") + if prefix == "bebop p2e batch": + return SimpleNamespace(returncode=rc, stdout=stdout, stderr=stderr) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(mod, "stream_run_logger", fake_stream) + + +def test_batch_without_flag_skips_accuracy(tmp_path, monkeypatch): + bs, vsrc = _prep_batch_case(tmp_path) + mod = load_step("bebop/p2e/04_batch_event.step.py") + _patch_batch( + mod, monkeypatch, tmp_path, vsrc, + stdout="Summary [ 1.2s] 4 tests run: 3 passed, 1 failed", + ) + ctx = FakeCtx() + run( + mod.handler( + { + "chip": "pebble", + "bitstream": bs, + "test": "pk-tests", + "_trace_id": "tid-origin", + }, + ctx, + ) + ) + success = ctx.state.store[("tid-origin", "success")]["body"] + assert success["success"] is True + assert "accuracy" not in success + assert not (tmp_path / "chipcrowd-eval-result.json").exists() + + +def test_batch_flag_merges_accuracy(tmp_path, monkeypatch): + bs, vsrc = _prep_batch_case(tmp_path) + mod = load_step("bebop/p2e/04_batch_event.step.py") + _patch_batch( + mod, monkeypatch, tmp_path, vsrc, + stdout="Summary [ 1.2s] 4 tests run: 3 passed, 1 failed", + rc=1, + ) + ctx = FakeCtx() + run( + mod.handler( + { + "chip": "pebble", + "bitstream": bs, + "test": "pk-tests", + "from_regression_check": True, + "_trace_id": "tid-origin", + }, + ctx, + ) + ) + failure = ctx.state.store[("tid-origin", "failure")]["body"] + assert failure["failure"] is True + assert failure["returncode"] == 1 + assert failure["accuracy"] == 0.75 + data = json.loads((tmp_path / "chipcrowd-eval-result.json").read_text()) + assert data["accuracy"] == 0.75 + + +def test_batch_flag_unparsable_fails(tmp_path, monkeypatch): + bs, vsrc = _prep_batch_case(tmp_path) + mod = load_step("bebop/p2e/04_batch_event.step.py") + _patch_batch(mod, monkeypatch, tmp_path, vsrc, stdout="no summary here", rc=0) + ctx = FakeCtx() + run( + mod.handler( + { + "chip": "pebble", + "bitstream": bs, + "test": "pk-tests", + "from_regression_check": True, + "_trace_id": "tid-origin", + }, + ctx, + ) + ) + failure = ctx.state.store[("tid-origin", "failure")]["body"] + assert failure["failure"] is True + assert failure["error"] == "accuracy_unparsable" + assert not (tmp_path / "chipcrowd-eval-result.json").exists() diff --git a/bbdev/api/tests/test_api_regression_eval_area_power.py b/bbdev/api/tests/test_api_regression_eval_area_power.py new file mode 100644 index 000000000..19e9243bc --- /dev/null +++ b/bbdev/api/tests/test_api_regression_eval_area_power.py @@ -0,0 +1,378 @@ +import asyncio +import importlib.util +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +API = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(API)) + +PEBBLE_CFG = "sims.verilator.BuckyballPebbleVerilatorConfig" + + +def load_step(rel: str): + path = API / "steps" / rel + name = path.name.replace(".", "_") + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load step: {path}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class FakeState: + def __init__(self): + self.store = {} + + async def set(self, scope, key, val): + self.store[(scope, key)] = val + + +class FakeCtx: + def __init__(self, trace_id="tid-origin"): + self.trace_id = trace_id + self.enqueued = [] + self.state = FakeState() + self.logger = SimpleNamespace( + info=lambda *a, **k: None, + error=lambda *a, **k: None, + ) + + async def enqueue(self, item): + self.enqueued.append(item) + + +def run(coro): + return asyncio.run(coro) + + +def test_api_missing_chip(): + mod = load_step("regression/03_eval_area_power_api.step.py") + ctx = FakeCtx() + resp = run(mod.handler(SimpleNamespace(body={"config": PEBBLE_CFG}), ctx)) + assert resp.status == 400 + assert "chip" in resp.body["error"] + assert ctx.enqueued == [] + + +def test_api_rejects_config(): + mod = load_step("regression/03_eval_area_power_api.step.py") + ctx = FakeCtx() + resp = run(mod.handler(SimpleNamespace(body={"chip": "pebble", "config": PEBBLE_CFG}), ctx)) + assert resp.status == 400 + assert "config" in resp.body["error"] + assert ctx.enqueued == [] + + +def test_api_missing_tapeout(): + mod = load_step("regression/03_eval_area_power_api.step.py") + ctx = FakeCtx() + resp = run( + mod.handler( + SimpleNamespace(body={"chip": "toy"}), + ctx, + ) + ) + assert resp.status == 400 + assert "tapeout" in resp.body["error"] + assert ctx.enqueued == [] + + +def test_api_invalid_top(): + mod = load_step("regression/03_eval_area_power_api.step.py") + ctx = FakeCtx() + resp = run( + mod.handler( + SimpleNamespace( + body={"chip": "pebble", "top": "not a module"} + ), + ctx, + ) + ) + assert resp.status == 400 + assert "top" in resp.body["error"] + assert ctx.enqueued == [] + + +def test_api_enqueues_regression_event(): + mod = load_step("regression/03_eval_area_power_api.step.py") + ctx = FakeCtx("tid-api") + resp = run( + mod.handler( + SimpleNamespace(body={"chip": "pebble"}), + ctx, + ) + ) + assert resp.status == 202 + assert resp.body == {"trace_id": "tid-api"} + assert len(ctx.enqueued) == 1 + item = ctx.enqueued[0] + assert item["topic"] == "regression.eval-area-power" + assert item["data"]["chip"] == "pebble" + assert "config" not in item["data"] + assert item["data"]["_trace_id"] == "tid-api" + assert "from_regression_area_power" not in item["data"] + assert "from_area_workflow" not in item["data"] + + +def test_api_forward_top(): + mod = load_step("regression/03_eval_area_power_api.step.py") + ctx = FakeCtx("tid-api") + resp = run( + mod.handler( + SimpleNamespace( + body={"chip": "pebble", "top": "DigitalTop"} + ), + ctx, + ) + ) + assert resp.status == 202 + assert ctx.enqueued[0]["data"]["top"] == "DigitalTop" + + +def test_event_missing_chip(tmp_path, monkeypatch): + mod = load_step("regression/03_eval_area_power_event.step.py") + monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) + ctx = FakeCtx() + run(mod.handler({"config": PEBBLE_CFG, "_trace_id": "tid-origin"}, ctx)) + failure = ctx.state.store[("tid-origin", "failure")]["body"] + assert failure["error"] == "missing_chip" + assert ctx.enqueued == [] + + +def test_event_missing_tapeout_without_dir(tmp_path, monkeypatch): + mod = load_step("regression/03_eval_area_power_event.step.py") + monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) + ctx = FakeCtx() + run(mod.handler({"chip": "pebble", "_trace_id": "tid-origin"}, ctx)) + failure = ctx.state.store[("tid-origin", "failure")]["body"] + assert failure["error"] == "missing_tapeout" + assert ctx.enqueued == [] + + +def test_event_missing_tapeout(tmp_path, monkeypatch): + _write_pebble_chip_toml(tmp_path) + mod = load_step("regression/03_eval_area_power_event.step.py") + monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) + ctx = FakeCtx() + run( + mod.handler( + {"chip": "pebble", "_trace_id": "tid-origin"}, + ctx, + ) + ) + failure = ctx.state.store[("tid-origin", "failure")]["body"] + assert failure["error"] == "missing_tapeout" + assert ctx.enqueued == [] + + +def _write_pebble_chip_toml(root: Path) -> Path: + chip_dir = root / "examples" / "chips" / "pebble" + chip_dir.mkdir(parents=True, exist_ok=True) + (chip_dir / "chip.toml").write_text( + '[chip]\nverilatorConfig = "sims.verilator.BuckyballPebbleVerilatorConfig"\n', + encoding="utf-8", + ) + return chip_dir + + +def _prep_event_tapeout(tmp_path): + chip_dir = _write_pebble_chip_toml(tmp_path) + tapeout = chip_dir / "tapeout" + tapeout.mkdir() + return tapeout + + +def test_event_sets_processing_and_enqueues_dc_verilog(tmp_path, monkeypatch): + _prep_event_tapeout(tmp_path) + mod = load_step("regression/03_eval_area_power_event.step.py") + monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) + ctx = FakeCtx("tid-evt") + run( + mod.handler( + { + "chip": "pebble", + "top": "DigitalTop", + "_trace_id": "tid-origin", + }, + ctx, + ) + ) + assert ctx.state.store[("tid-origin", "processing")]["processing"] is True + assert ("tid-origin", "success") not in ctx.state.store + assert ("tid-origin", "failure") not in ctx.state.store + assert len(ctx.enqueued) == 1 + item = ctx.enqueued[0] + assert item["topic"] == "dc.verilog" + data = item["data"] + assert data["from_area_workflow"] is True + assert data["from_regression_area_power"] is True + assert "config" not in data + assert data["top"] == "DigitalTop" + assert "output_dir" not in data + assert "analysis_dir" not in data + assert data["_trace_id"] == "tid-origin" + + +def _patch_area(mod, monkeypatch, tmp_path, analysis_dir, rpt_text=None, rc=0): + monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) + monkeypatch.setattr(mod.shutil, "which", lambda name: "/bin/dc_shell") + monkeypatch.setattr( + mod, + "get_tapeout_contract", + lambda *a, **k: SimpleNamespace( + chip="pebble", + clock_port="clk", + clock_period_ns=10.0, + dc_script=str(tmp_path / "dc.tcl"), + root=tmp_path / "tapeout", + ), + ) + monkeypatch.setattr(mod, "technology_settings", lambda: {}) + monkeypatch.setattr(mod, "write_run_tcl", lambda *a, **k: tmp_path / "run.tcl") + + async def fake_stream(**k): + if rpt_text is not None: + rpt = Path(analysis_dir) / "reports" / "area.rpt" + rpt.parent.mkdir(parents=True, exist_ok=True) + rpt.write_text(rpt_text) + return SimpleNamespace(returncode=rc) + + monkeypatch.setattr(mod, "stream_run_logger_async", fake_stream) + + +def _area_input(tmp_path, analysis_dir, **extra): + src = tmp_path / "dc_sources.list" + src.write_text("a.sv\n") + data = { + "source_list": str(src), + "analysis_dir": str(analysis_dir), + "chip": "pebble", + "top": "DigitalTop", + "_trace_id": "tid-origin", + } + data.update(extra) + return data + + +def test_area_without_flag_skips_metrics(tmp_path, monkeypatch): + analysis_dir = tmp_path / "analysis" + mod = load_step("dc/03_area_event.step.py") + _patch_area( + mod, monkeypatch, tmp_path, analysis_dir, + rpt_text="Total cell area: 1000000.0000\n", + ) + ctx = FakeCtx() + run(mod.handler(_area_input(tmp_path, analysis_dir), ctx)) + success = ctx.state.store[("tid-origin", "success")]["body"] + assert success["success"] is True + assert "area" not in success + assert "freq" not in success + assert ctx.enqueued == [] + assert not (tmp_path / "chipcrowd-eval-result.json").exists() + + +def test_area_without_flag_enqueues_sim_for_power(tmp_path, monkeypatch): + analysis_dir = tmp_path / "analysis" + mod = load_step("dc/03_area_event.step.py") + _patch_area(mod, monkeypatch, tmp_path, analysis_dir, rpt_text="x") + ctx = FakeCtx() + run( + mod.handler( + _area_input(tmp_path, analysis_dir, from_power_workflow=True), + ctx, + ) + ) + success = ctx.state.store[("tid-origin", "success")]["body"] + assert success["success"] is True + assert len(ctx.enqueued) == 1 + assert ctx.enqueued[0]["topic"] == "dc.sim" + assert not (tmp_path / "chipcrowd-eval-result.json").exists() + + +def test_area_flag_merges_area_freq(tmp_path, monkeypatch): + analysis_dir = tmp_path / "analysis" + mod = load_step("dc/03_area_event.step.py") + _patch_area( + mod, monkeypatch, tmp_path, analysis_dir, + rpt_text="Total cell area: 1000000.0000\n", + ) + ctx = FakeCtx() + run( + mod.handler( + _area_input( + tmp_path, analysis_dir, + from_area_workflow=True, + from_regression_area_power=True, + ), + ctx, + ) + ) + success = ctx.state.store[("tid-origin", "success")]["body"] + assert success["success"] is True + assert success["area"] == 1.0 + assert success["freq"] == 100.0 + assert ctx.enqueued == [] + data = json.loads((tmp_path / "chipcrowd-eval-result.json").read_text()) + assert data["area"] == 1.0 + assert data["freq"] == 100.0 + + +def test_area_flag_does_not_enqueue_sim(tmp_path, monkeypatch): + analysis_dir = tmp_path / "analysis" + mod = load_step("dc/03_area_event.step.py") + _patch_area( + mod, monkeypatch, tmp_path, analysis_dir, + rpt_text="Total cell area: 1000000.0000\n", + ) + ctx = FakeCtx() + run( + mod.handler( + _area_input( + tmp_path, analysis_dir, + from_power_workflow=True, + from_regression_area_power=True, + ), + ctx, + ) + ) + assert ctx.enqueued == [] + assert (tmp_path / "chipcrowd-eval-result.json").is_file() + + +def test_area_flag_missing_rpt_fails(tmp_path, monkeypatch): + analysis_dir = tmp_path / "analysis" + mod = load_step("dc/03_area_event.step.py") + _patch_area(mod, monkeypatch, tmp_path, analysis_dir, rpt_text=None) + ctx = FakeCtx() + run( + mod.handler( + _area_input(tmp_path, analysis_dir, from_regression_area_power=True), + ctx, + ) + ) + failure = ctx.state.store[("tid-origin", "failure")]["body"] + assert failure["failure"] is True + assert "missing area.rpt" in failure["error"] + assert ctx.enqueued == [] + assert not (tmp_path / "chipcrowd-eval-result.json").exists() + + +def test_area_flag_unparsable_rpt_fails(tmp_path, monkeypatch): + analysis_dir = tmp_path / "analysis" + mod = load_step("dc/03_area_event.step.py") + _patch_area(mod, monkeypatch, tmp_path, analysis_dir, rpt_text="no area here\n") + ctx = FakeCtx() + run( + mod.handler( + _area_input(tmp_path, analysis_dir, from_regression_area_power=True), + ctx, + ) + ) + failure = ctx.state.store[("tid-origin", "failure")]["body"] + assert failure["failure"] is True + assert failure["error"] == "Total cell area not found" + assert ctx.enqueued == [] + assert not (tmp_path / "chipcrowd-eval-result.json").exists() diff --git a/bbdev/api/tests/test_api_regression_eval_performance.py b/bbdev/api/tests/test_api_regression_eval_performance.py new file mode 100644 index 000000000..6e2576a94 --- /dev/null +++ b/bbdev/api/tests/test_api_regression_eval_performance.py @@ -0,0 +1,111 @@ +import asyncio +import importlib.util +import sys +from pathlib import Path +from types import SimpleNamespace + +API = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(API)) + + +def load_step(rel): + path = API / "steps" / rel + name = path.name.replace(".", "_") + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load step: {path}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class FakeCtx: + def __init__(self, trace_id="tid-api"): + self.trace_id = trace_id + self.enqueued = [] + self.logger = SimpleNamespace(info=lambda *a, **k: None, + error=lambda *a, **k: None) + + async def enqueue(self, item): + self.enqueued.append(item) + + +def run(coro): + return asyncio.run(coro) + + +def test_api_missing_chip(tmp_path): + bs = tmp_path / "bitstream.bit" + bs.write_text("x") + mod = load_step("regression/02_eval_performance_api.step.py") + ctx = FakeCtx() + resp = run(mod.handler(SimpleNamespace(body={"bitstream": str(bs)}), ctx)) + assert resp.status == 400 + assert "chip" in resp.body["error"] + assert ctx.enqueued == [] + + +def test_api_missing_bitstream(): + mod = load_step("regression/02_eval_performance_api.step.py") + ctx = FakeCtx() + resp = run(mod.handler(SimpleNamespace(body={"chip": "pebble"}), ctx)) + assert resp.status == 400 + assert "bitstream" in resp.body["error"] + assert ctx.enqueued == [] + + +def test_api_bitstream_not_found(): + mod = load_step("regression/02_eval_performance_api.step.py") + ctx = FakeCtx() + resp = run( + mod.handler( + SimpleNamespace( + body={"chip": "pebble", "bitstream": "/no/such/bitstream.bit"}, + ), + ctx, + ) + ) + assert resp.status == 400 + assert "not found" in resp.body["error"] + assert ctx.enqueued == [] + + +def test_api_enqueues_regression_event(tmp_path): + bs = tmp_path / "bitstream.bit" + bs.write_text("x") + mod = load_step("regression/02_eval_performance_api.step.py") + ctx = FakeCtx("tid-api") + resp = run( + mod.handler( + SimpleNamespace(body={"chip": "pebble", "bitstream": str(bs)}), ctx) + ) + assert resp.status == 202 + assert resp.body == {"trace_id": "tid-api"} + assert len(ctx.enqueued) == 1 + item = ctx.enqueued[0] + assert item["topic"] == "regression.eval-performance" + assert item["data"]["chip"] == "pebble" + assert item["data"]["bitstream"] == str(bs) + assert item["data"]["_trace_id"] == "tid-api" + assert "config" not in item["data"] + + +def test_api_forward_config(tmp_path): + bs = tmp_path / "bitstream.bit" + bs.write_text("x") + mod = load_step("regression/02_eval_performance_api.step.py") + ctx = FakeCtx("tid-api") + resp = run( + mod.handler( + SimpleNamespace( + body={ + "chip": "pebble", + "bitstream": str(bs), + "config": "sims.p2e.P2EPebbleLinuxConfig", + } + ), + ctx, + ) + ) + assert resp.status == 202 + assert ctx.enqueued[0]["data"]["config"] == "sims.p2e.P2EPebbleLinuxConfig" diff --git a/bbdev/api/tests/test_api_regression_eval_performance_event.py b/bbdev/api/tests/test_api_regression_eval_performance_event.py new file mode 100644 index 000000000..ea2a1999f --- /dev/null +++ b/bbdev/api/tests/test_api_regression_eval_performance_event.py @@ -0,0 +1,312 @@ +import asyncio +import importlib.util +import json +import os +import re +import sys +from pathlib import Path +from types import SimpleNamespace + +API = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(API)) + + +def load_step(rel): + path = API / "steps" / rel + name = path.name.replace(".", "_") + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load step: {path}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class FakeState: + def __init__(self): + self.store = {} + + async def set(self, scope, key, val): + self.store[(scope, key)] = val + + +class FakeCtx: + def __init__(self, trace_id="tid-evt"): + self.trace_id = trace_id + self.enqueued = [] + self.state = FakeState() + self.logger = SimpleNamespace(info=lambda *a, **k: None, + error=lambda *a, **k: None) + + async def enqueue(self, item): + self.enqueued.append(item) + + +def run(coro): + return asyncio.run(coro) + + +def _make_bitstream(tmp_path): + build = tmp_path / "case" + fpga = build / "fpgaCompDir" + fpga.mkdir(parents=True) + bs = fpga / "bitstream.bit" + bs.write_text("x") + rt = build / "vvacDir" / "runtimeDir" + lib = rt / "lib" / "lib_arm" + lib.mkdir(parents=True) + (lib / "libvCtb.so").write_text("x") + (rt / "rtcfg").write_text("x") + vsrc = tmp_path / "vsrc" + vsrc.mkdir() + return str(bs), str(vsrc), str(build) + + +def _patch_paths(mod, monkeypatch, tmp_path, vsrc): + monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) + monkeypatch.setattr(mod, "resolve_chip_compiler_core", + lambda bb, chip: SimpleNamespace(name="pebbleCore")) + monkeypatch.setattr(mod, "get_verilator_build_dir", + lambda bb, cfg, out: out or str(vsrc)) + + +def _fake_stream_factory(tmp_path): + def fake(**k): + prefix = k.get("stdout_prefix", "") + cmd = k.get("cmd", "") + rc = 0 + if prefix.startswith("workload build"): + src = (tmp_path / "bb-tests" / "output" / "workloads" / "src" + / "ModelTest" / "e2e" / "models" / "archs" / "buckyball" + / "pebble" / "LeNet") + src.mkdir(parents=True, exist_ok=True) + (src / "buddy-buckyball-lenet-run").write_text("x") + bld = (tmp_path / "bb-tests" / "build" / "workloads" / "src" + / "ModelTest" / "e2e" / "models" / "archs" / "buckyball" + / "pebble" / "LeNet") + bld.mkdir(parents=True, exist_ok=True) + (bld / "subgraph0_linalg.mlir").write_text("linalg") + (bld / "subgraph0_buckyball.mlir").write_text("buckyball") + elif prefix.startswith("kernel build"): + out = tmp_path / "bb-tests" / "output" / "kernel" + out.mkdir(parents=True, exist_ok=True) + (out / "fw_payload-lenet.bin").write_text("bin") + elif prefix.startswith("p2e run"): + m = re.search(r'--log-dir="([^"]+)"', cmd) + if m: + cycle = Path(m.group(1)) / "trace" / "cycle" + cycle.mkdir(parents=True, exist_ok=True) + (cycle / "trace-0.txt").write_text("start 10\nend 20\nelapsed 10\n") + elif prefix.startswith("perfetto"): + m = re.search(r"^(\S+) \S+ (\S+) (\S+)", cmd) + trace_dir = Path(m.group(2)) if m else None + if trace_dir: + out = trace_dir / "perfetto.json" + out.write_text(json.dumps({ + "traceEvents": [{"ph": "X", "ts": 10, "dur": 5}] + })) + return SimpleNamespace(returncode=rc, stdout="", stderr="") + return fake + + +def test_event_missing_bitstream(tmp_path, monkeypatch): + mod = load_step("regression/02_eval_performance_event.step.py") + _patch_paths(mod, monkeypatch, tmp_path, tmp_path / "vsrc") + ctx = FakeCtx() + run(mod.handler({"chip": "pebble", "bitstream": "/no/such.bit", + "_trace_id": "tid-origin"}, ctx)) + failure = ctx.state.store[("tid-origin", "failure")]["body"] + assert failure["failure"] is True + assert failure["error"] == "bitstream_not_found" + assert ctx.enqueued == [] + + +def test_event_missing_models_toml(tmp_path, monkeypatch): + bs, vsrc, _ = _make_bitstream(tmp_path) + mod = load_step("regression/02_eval_performance_event.step.py") + _patch_paths(mod, monkeypatch, tmp_path, vsrc) + ctx = FakeCtx() + run(mod.handler({"chip": "pebble", "bitstream": bs, + "_trace_id": "tid-origin"}, ctx)) + failure = ctx.state.store[("tid-origin", "failure")]["body"] + assert failure["failure"] is True + assert failure["error"] == "models_toml_invalid" + assert not (tmp_path / "chipcrowd-eval-result.json").exists() + + +def test_event_missing_chip(tmp_path, monkeypatch): + bs, vsrc, _ = _make_bitstream(tmp_path) + mod = load_step("regression/02_eval_performance_event.step.py") + _patch_paths(mod, monkeypatch, tmp_path, vsrc) + ctx = FakeCtx() + run(mod.handler({"bitstream": bs, "_trace_id": "tid-origin"}, ctx)) + failure = ctx.state.store[("tid-origin", "failure")]["body"] + assert failure["error"] == "missing_chip" + + +def _seed_models_toml(tmp_path, models=None): + if models is None: + models = ["lenet"] + d = (tmp_path / "examples" / "chips" / "pebble" / "regression" / "eval") + d.mkdir(parents=True, exist_ok=True) + (d / "models.toml").write_text( + "[eval]\nmodels = " + json.dumps(models) + "\n" + ) + + +def _seed_perfetto_sources(tmp_path): + toml = (tmp_path / "bb-tests" / "workloads" / "src" / "ModelTest" + / "e2e" / "models" / "models" / "LeNet" / "trace" + / "trace-linalg-buckyball.toml") + toml.parent.mkdir(parents=True, exist_ok=True) + toml.write_text("[trace]\n") + + +def test_event_happy_path_merges_latency(tmp_path, monkeypatch): + bs, vsrc, _ = _make_bitstream(tmp_path) + _seed_models_toml(tmp_path) + _seed_perfetto_sources(tmp_path) + mod = load_step("regression/02_eval_performance_event.step.py") + _patch_paths(mod, monkeypatch, tmp_path, vsrc) + monkeypatch.setattr(mod, "stream_run_logger", _fake_stream_factory(tmp_path)) + monkeypatch.setattr(mod._kernel, "bin_to_hex", + lambda bin_p, hex_p, base_address=0: ( + Path(hex_p).parent.mkdir(parents=True, exist_ok=True), + Path(hex_p).write_text("hex"), + True, + )[-1]) + ctx = FakeCtx() + run(mod.handler({"chip": "pebble", "bitstream": bs, + "_trace_id": "tid-origin"}, ctx)) + success = ctx.state.store[("tid-origin", "success")]["body"] + assert success["success"] is True + assert success["latency"] == 5.0 + assert len(success["models"]) == 1 + assert success["models"][0]["name"] == "lenet" + assert success["models"][0]["latency"] == 5.0 + data = json.loads((tmp_path / "chipcrowd-eval-result.json").read_text()) + assert data["latency"] == 5.0 + assert data["models"][0]["name"] == "lenet" + assert data["models"][0]["perfetto"].endswith("perfetto-lenet.json") + + +def test_event_unknown_model_fails_cleanly(tmp_path, monkeypatch): + bs, vsrc, _ = _make_bitstream(tmp_path) + _seed_models_toml(tmp_path, models=["not-a-real-model"]) + mod = load_step("regression/02_eval_performance_event.step.py") + _patch_paths(mod, monkeypatch, tmp_path, vsrc) + ctx = FakeCtx() + run(mod.handler({"chip": "pebble", "bitstream": bs, + "_trace_id": "tid-origin"}, ctx)) + failure = ctx.state.store[("tid-origin", "failure")]["body"] + assert failure["failure"] is True + assert failure["error"] == "unknown_model" + assert failure["model"] == "not-a-real-model" + assert not (tmp_path / "chipcrowd-eval-result.json").exists() + + +def test_workload_build_cmd_unknown_model_is_valueerror(tmp_path, monkeypatch): + mod = load_step("regression/02_eval_performance_event.step.py") + monkeypatch.setattr(mod, "resolve_chip_compiler_core", + lambda bb, chip: SimpleNamespace(name="pebbleCore")) + try: + mod._workload_build_cmd(str(tmp_path), "pebble", "not-a-real-model") + assert False + except ValueError as e: + assert "Unknown model" in str(e) + except KeyError: + assert False, "unknown model must raise ValueError, not KeyError" + + +def test_event_missing_model_layout_fails_cleanly(tmp_path, monkeypatch): + bs, vsrc, _ = _make_bitstream(tmp_path) + _seed_models_toml(tmp_path, models=["lenet"]) + mod = load_step("regression/02_eval_performance_event.step.py") + _patch_paths(mod, monkeypatch, tmp_path, vsrc) + monkeypatch.setattr(mod, "MODEL_LAYOUT", {}) + ctx = FakeCtx() + run(mod.handler({"chip": "pebble", "bitstream": bs, + "_trace_id": "tid-origin"}, ctx)) + failure = ctx.state.store[("tid-origin", "failure")]["body"] + assert failure["error"] == "missing_model_layout" + assert failure["model"] == "lenet" + assert not (tmp_path / "chipcrowd-eval-result.json").exists() + + +def test_event_missing_build_dir_fails_cleanly(tmp_path, monkeypatch): + bs, vsrc, build = _make_bitstream(tmp_path) + _seed_models_toml(tmp_path) + _seed_perfetto_sources(tmp_path) + mod = load_step("regression/02_eval_performance_event.step.py") + _patch_paths(mod, monkeypatch, tmp_path, vsrc) + monkeypatch.setattr(mod, "stream_run_logger", _fake_stream_factory(tmp_path)) + monkeypatch.setattr(mod._kernel, "bin_to_hex", + lambda bin_p, hex_p, base_address=0: ( + Path(hex_p).parent.mkdir(parents=True, exist_ok=True), + Path(hex_p).write_text("hex"), + True, + )[-1]) + orig_isdir = os.path.isdir + + def fake_isdir(path): + if os.path.abspath(path) == os.path.abspath(build): + return False + return orig_isdir(path) + + monkeypatch.setattr(mod.os.path, "isdir", fake_isdir) + ctx = FakeCtx() + run(mod.handler({"chip": "pebble", "bitstream": bs, + "_trace_id": "tid-origin"}, ctx)) + failure = ctx.state.store[("tid-origin", "failure")]["body"] + assert failure["error"] == "build_dir_not_found" + assert failure["model"] == "lenet" + assert failure["build_dir"] == os.path.abspath(build) + assert not (tmp_path / "chipcrowd-eval-result.json").exists() + + +def test_p2e_run_cmds_missing_build_dir(tmp_path, monkeypatch): + mod = load_step("regression/02_eval_performance_event.step.py") + hex_dir = tmp_path / "bb-tests" / "output" / "kernel" + hex_dir.mkdir(parents=True) + (hex_dir / "fw.hex").write_text("x") + fpga = tmp_path / "case" / "fpgaCompDir" + fpga.mkdir(parents=True) + bs = fpga / "bitstream.bit" + bs.write_text("x") + build = str(tmp_path / "case") + orig_isdir = os.path.isdir + + def fake_isdir(path): + if os.path.abspath(path) == os.path.abspath(build): + return False + return orig_isdir(path) + + monkeypatch.setattr(mod.os.path, "isdir", fake_isdir) + try: + mod._p2e_run_cmds(str(tmp_path), str(bs), "fw", None, {}) + assert False + except FileNotFoundError as e: + assert "P2E build case not found" in str(e) + assert build in str(e) + + +def test_event_workload_build_failure_aborts(tmp_path, monkeypatch): + bs, vsrc, _ = _make_bitstream(tmp_path) + _seed_models_toml(tmp_path) + _seed_perfetto_sources(tmp_path) + mod = load_step("regression/02_eval_performance_event.step.py") + _patch_paths(mod, monkeypatch, tmp_path, vsrc) + + def fake(**k): + prefix = k.get("stdout_prefix", "") + if prefix.startswith("workload build"): + return SimpleNamespace(returncode=1, stdout="", stderr="") + return SimpleNamespace(returncode=0, stdout="", stderr="") + monkeypatch.setattr(mod, "stream_run_logger", fake) + ctx = FakeCtx() + run(mod.handler({"chip": "pebble", "bitstream": bs, + "_trace_id": "tid-origin"}, ctx)) + failure = ctx.state.store[("tid-origin", "failure")]["body"] + assert failure["error"] == "workload_build_failed" + assert failure["model"] == "lenet" + assert not (tmp_path / "chipcrowd-eval-result.json").exists() diff --git a/bbdev/api/tests/test_api_verilator_build.py b/bbdev/api/tests/test_api_verilator_build.py index 4c984d6fa..300da1ff2 100644 --- a/bbdev/api/tests/test_api_verilator_build.py +++ b/bbdev/api/tests/test_api_verilator_build.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --build '--jobs 16 --config sims.verilator.BuckyballToyVerilatorConfig'") +run_bbdev_case("bbdev verilator --build '--jobs 16 --chip toy'") diff --git a/bbdev/api/tests/test_api_verilator_clean.py b/bbdev/api/tests/test_api_verilator_clean.py index f40ad020b..e94bbb161 100644 --- a/bbdev/api/tests/test_api_verilator_clean.py +++ b/bbdev/api/tests/test_api_verilator_clean.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --clean '--config sims.verilator.BuckyballToyVerilatorConfig'") +run_bbdev_case("bbdev verilator --clean '--chip toy'") diff --git a/bbdev/api/tests/test_api_verilator_run.py b/bbdev/api/tests/test_api_verilator_run.py index c03dfdcc3..dbd5fb3e9 100644 --- a/bbdev/api/tests/test_api_verilator_run.py +++ b/bbdev/api/tests/test_api_verilator_run.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --run '--jobs 16 --binary toy_vecunit_matmul_ones-singlecore-baremetal --config sims.verilator.BuckyballToyVerilatorConfig --batch'") +run_bbdev_case("bbdev verilator --run '--jobs 16 --binary toy_relu_test-singlecore-baremetal --chip toy --batch'") diff --git a/bbdev/api/tests/test_api_verilator_sim.py b/bbdev/api/tests/test_api_verilator_sim.py index 52500c753..99addce13 100644 --- a/bbdev/api/tests/test_api_verilator_sim.py +++ b/bbdev/api/tests/test_api_verilator_sim.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --sim '--binary toy_vecunit_matmul_ones-singlecore-baremetal --batch --config sims.verilator.BuckyballToyVerilatorConfig'") +run_bbdev_case("bbdev verilator --sim '--binary toy_relu_test-singlecore-baremetal --batch --chip toy'") diff --git a/bbdev/api/tests/test_api_verilator_verilog.py b/bbdev/api/tests/test_api_verilator_verilog.py index f55a95266..ddc5e8882 100644 --- a/bbdev/api/tests/test_api_verilator_verilog.py +++ b/bbdev/api/tests/test_api_verilator_verilog.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --verilog '--config sims.verilator.BuckyballToyVerilatorConfig'") +run_bbdev_case("bbdev verilator --verilog '--chip toy'") diff --git a/bbdev/api/tests/test_api_workload_clean.py b/bbdev/api/tests/test_api_workload_clean.py index b9d59f7ed..da87ffbb8 100644 --- a/bbdev/api/tests/test_api_workload_clean.py +++ b/bbdev/api/tests/test_api_workload_clean.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev workload --clean") +run_bbdev_case("bbdev workload --clean '--chip toy'") diff --git a/bbdev/api/tests/test_api_workload_tohex.py b/bbdev/api/tests/test_api_workload_tohex.py index add7be934..5f3d36964 100644 --- a/bbdev/api/tests/test_api_workload_tohex.py +++ b/bbdev/api/tests/test_api_workload_tohex.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev workload --tohex") +run_bbdev_case("bbdev workload --tohex '--chip toy'") diff --git a/bbdev/api/tests/test_api_yosys_run.py b/bbdev/api/tests/test_api_yosys_run.py index d0b6b4ae8..c8d133217 100644 --- a/bbdev/api/tests/test_api_yosys_run.py +++ b/bbdev/api/tests/test_api_yosys_run.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev yosys --run") +run_bbdev_case("bbdev yosys --run '--chip toy'") diff --git a/bbdev/api/tests/test_api_yosys_synth.py b/bbdev/api/tests/test_api_yosys_synth.py index a0e04eeea..3f88bc892 100644 --- a/bbdev/api/tests/test_api_yosys_synth.py +++ b/bbdev/api/tests/test_api_yosys_synth.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev yosys --synth") +run_bbdev_case("bbdev yosys --synth '--chip toy'") diff --git a/bbdev/api/tests/test_api_yosys_verilog.py b/bbdev/api/tests/test_api_yosys_verilog.py index 5a94e654a..057ad8cf8 100644 --- a/bbdev/api/tests/test_api_yosys_verilog.py +++ b/bbdev/api/tests/test_api_yosys_verilog.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev yosys --verilog") +run_bbdev_case("bbdev yosys --verilog '--chip toy'") diff --git a/bbdev/api/utils/build.py b/bbdev/api/utils/build.py new file mode 100644 index 000000000..f314a94ea --- /dev/null +++ b/bbdev/api/utils/build.py @@ -0,0 +1,371 @@ +from __future__ import annotations + +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +from utils.path import bebop_cargo_env, workload_build_dir + +_MODELS: dict[str, tuple[str, str]] = { + "lenet": ("lenet", "buddy-buckyball-lenet-run"), + "mobilenet": ("mobilenetv3", "buddy-buckyball-mobilenetv3-run"), + "resnet": ("resnet18", "buddy-buckyball-resnet-run"), + "yolo": ("yolo26", "buddy-buckyball-yolo26-run"), + "bert": ("bert", "buddy-buckyball-bert-run"), + "qwen3": ("qwen3", "buddy-buckyball-qwen3-run"), + "gemma4": ("gemma4", "buddy-buckyball-gemma4-run"), + "deepseekr1": ("deepseekr1", "buddy-buckyball-deepseekr1-run"), + "llama2": ("llama2", "buddy-buckyball-llama2-run"), + "stable-diffusion": ("stablediffusion", "buddy-buckyball-stable-diffusion-run"), + "whisper": ("whisper", "buddy-buckyball-whisper-run"), + "buddynext": ("buddynext", "buddy-buckyball-buddynext-all-run"), +} + +_RUSHB: dict[str, dict[str, str]] = { + "lenet": { + "bemu": "buddy-buckyball-lenet-rushB-bemu-run", + "verilator": "buddy-buckyball-lenet-rushB-verilator-run", + }, + "mobilenet": { + "bemu": "buddy-buckyball-mobilenetv3-rushB-bemu-run", + "verilator": "buddy-buckyball-mobilenetv3-rushB-verilator-run", + }, + "resnet": { + "bemu": "buddy-buckyball-resnet-rushB-bemu-run", + "verilator": "buddy-buckyball-resnet-rushB-verilator-run", + }, + "yolo": { + "bemu": "buddy-buckyball-yolo26-rushB-bemu-run", + "verilator": "buddy-buckyball-yolo26-rushB-verilator-run", + }, + "bert": { + "bemu": "buddy-buckyball-bert-rushB-bemu-run", + "verilator": "buddy-buckyball-bert-rushB-verilator-run", + }, + "qwen3": { + "bemu": "buddy-buckyball-qwen3-rushB-bemu-run", + "verilator": "buddy-buckyball-qwen3-rushB-verilator-run", + }, + "gemma4": { + "bemu": "buddy-buckyball-gemma4-rushB-bemu-run", + "verilator": "buddy-buckyball-gemma4-rushB-verilator-run", + }, + "deepseekr1": { + "bemu": "buddy-buckyball-deepseekr1-rushB-bemu-run", + "verilator": "buddy-buckyball-deepseekr1-rushB-verilator-run", + }, + "llama2": { + "bemu": "buddy-buckyball-llama2-rushB-bemu-run", + "verilator": "buddy-buckyball-llama2-rushB-verilator-run", + }, + "stable-diffusion": { + "bemu": "buddy-buckyball-stable-diffusion-rushB-bemu-run", + "verilator": "buddy-buckyball-stable-diffusion-rushB-verilator-run", + }, + "whisper": { + "bemu": "buddy-buckyball-whisper-rushB-bemu-run", + "verilator": "buddy-buckyball-whisper-rushB-verilator-run", + }, + "buddynext": { + "bemu": "buddy-buckyball-buddynext-rushB-bemu-run", + "verilator": "buddy-buckyball-buddynext-rushB-verilator-run", + }, +} + + +def _repo(bbdir: str) -> Path: + root = Path(bbdir).resolve() + if not root.is_dir(): + raise ValueError(f"repo does not exist: {root}") + return root + + +def _configparse(bbdir: str) -> Path: + path = _repo(bbdir) / "bazel" / "configparse" + if not path.is_dir(): + raise ValueError(f"missing configparse: {path}") + return path + + +def _pythonpath(bbdir: str) -> dict[str, str]: + env = os.environ.copy() + parse = _configparse(bbdir) + prev = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = f"{parse}:{prev}" if prev else str(parse) + return env + + +def _run( + cmd: list[str], + *, + bbdir: str, + cwd: Path | None = None, + extra_env: dict[str, str] | None = None, +) -> None: + env = _pythonpath(bbdir) + if extra_env: + env.update(extra_env) + result = subprocess.run(cmd, cwd=cwd or _repo(bbdir), env=env) + if result.returncode != 0: + raise RuntimeError(f"command failed ({result.returncode}): {' '.join(cmd)}") + + +def _require_riscv() -> Path: + raw = os.environ.get("RISCV", "") + if not raw: + raise RuntimeError("RISCV is unset; enter nix develop") + root = Path(raw) + if not root.is_dir(): + raise RuntimeError(f"RISCV is not a directory: {root}") + return root + + +def install_bundle(bbdir: str, chip: str) -> None: + if not re.fullmatch(r"[A-Za-z0-9_-]+", chip): + raise ValueError(f"invalid chip: {chip}") + script = _configparse(bbdir) / "chip_bundle.py" + _run([sys.executable, str(script), "--repo", str(_repo(bbdir)), "--chip", chip, "--all"], bbdir=bbdir) + + +def _compiler_core(bbdir: str, chip: str | None, core: str | None) -> str: + if core: + return core + if not chip: + raise ValueError("chip or core is required") + sys.path.insert(0, str(_configparse(bbdir))) + from chip_bundle import chip_index # type: ignore + + picked = chip_index(_repo(bbdir), chip).get("compilerCore", "") + if not isinstance(picked, str) or not picked: + raise ValueError(f"chip {chip} has no compilerCore") + return picked + + +def build_compiler(bbdir: str, *, chip: str | None = None, core: str | None = None) -> None: + root = _repo(bbdir) + picked = _compiler_core(bbdir, chip, core) + buddy = root / "compiler" / "thirdparty" / "buddy-mlir" + dialects = root / "examples" / "cores" / picked / "compiler" + llvm = buddy / "llvm" / "build" + build_root = buddy / "build" + build = build_root / "cores" / picked + if not (dialects / "CMakeLists.txt").is_file(): + raise RuntimeError(f"missing compiler package: {dialects / 'CMakeLists.txt'}") + if not (llvm / "lib" / "cmake" / "mlir").is_dir(): + raise RuntimeError(f"missing LLVM/MLIR cmake at {llvm}") + python = sys.executable + py_include = subprocess.check_output( + [python, "-c", "import sysconfig; print(sysconfig.get_path('include'))"], + text=True, + ).strip() + py_lib = subprocess.check_output( + [ + python, + "-c", + "import sysconfig; print(sysconfig.get_config_var('LIBDIR') + '/' + sysconfig.get_config_var('LDLIBRARY'))", + ], + text=True, + ).strip() + cache = llvm / "CMakeCache.txt" + nanobind = subprocess.check_output( + ["sed", "-n", r"s/^NB_DIR:INTERNAL=\(.*\)$/\1\/cmake/p", str(cache)], + text=True, + ).strip() + if not (Path(nanobind) / "nanobind-config.cmake").is_file(): + raise RuntimeError(f"missing nanobind cmake package: {nanobind}") + build.mkdir(parents=True, exist_ok=True) + cmake = [ + "cmake", + "-G", + "Ninja", + "-S", + str(buddy), + "-B", + str(build), + f"-DBUDDY_EXTERNAL_DIALECTS_DIR={dialects}", + f"-DMLIR_DIR={llvm / 'lib' / 'cmake' / 'mlir'}", + f"-DLLVM_DIR={llvm / 'lib' / 'cmake' / 'llvm'}", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=RELEASE", + "-DBUDDY_MLIR_ENABLE_PYTHON_PACKAGES=ON", + f"-Dnanobind_DIR={nanobind}", + f"-DPython3_EXECUTABLE={python}", + f"-DPython3_INCLUDE_DIR={py_include}", + f"-DPython3_LIBRARY={py_lib}", + f"-DPython_EXECUTABLE={python}", + f"-DPython_INCLUDE_DIR={py_include}", + f"-DPython_LIBRARY={py_lib}", + ] + _run(cmake, bbdir=bbdir) + _run( + ["ninja", "-C", str(build), "-j", str(os.cpu_count() or 1), "buddy-opt", "buddy-translate", "buddy-llc", "python-package-buddy", "BuddyMLIRPythonModules"], + bbdir=bbdir, + ) + public_bin = build_root / "bin" + public_bin.mkdir(parents=True, exist_ok=True) + for tool in ("buddy-opt", "buddy-translate", "buddy-llc"): + link = public_bin / f"{tool}-{picked}" + if link.is_symlink(): + link.unlink() + link.symlink_to(Path("..") / "cores" / picked / "bin" / tool) + + +def _ninja_target(model: str, rushb: str | None) -> tuple[str, str]: + cmake_model = "" + ninja_arg = "" + if model: + if model not in _MODELS: + raise ValueError(f"unknown workload model: {model}") + cmake_model, ninja_arg = _MODELS[model] + if rushb: + mapped = _RUSHB.get(model, {}).get(rushb) + if not mapped: + raise ValueError(f"no rushB target for model {model!r} backend {rushb!r}") + ninja_arg = mapped + elif rushb: + ninja_arg = f"rushB-{rushb}-workloads-build" + return cmake_model, ninja_arg + + +def _workload_core(bbdir: str, chip: str) -> str: + defs_py = _configparse(bbdir) / "workload_cmake_defs.py" + out = subprocess.check_output( + [sys.executable, str(defs_py), "--repo", str(_repo(bbdir)), "--chip", chip], + text=True, + env=_pythonpath(bbdir), + ) + for line in out.splitlines(): + key, _, value = line.partition("=") + if key == "BUCKYBALL_WORKLOAD_CORE": + if not value: + raise RuntimeError("workload_cmake_defs missing BUCKYBALL_WORKLOAD_CORE") + return value + raise RuntimeError("workload_cmake_defs missing BUCKYBALL_WORKLOAD_CORE") + + +def _bemu_manifest(bbdir: str, chip: str) -> str: + defs_py = _configparse(bbdir) / "workload_cmake_defs.py" + out = subprocess.check_output( + [sys.executable, str(defs_py), "--repo", str(_repo(bbdir)), "--chip", chip], + text=True, + env=_pythonpath(bbdir), + ) + for line in out.splitlines(): + key, _, value = line.partition("=") + if key == "BUCKYBALL_RUSHB_BEMU_MANIFEST": + if not value: + raise RuntimeError("workload_cmake_defs missing BUCKYBALL_RUSHB_BEMU_MANIFEST") + return value + raise RuntimeError("workload_cmake_defs missing BUCKYBALL_RUSHB_BEMU_MANIFEST") + + +def gen_ball_isa(bbdir: str, core: str) -> Path: + root = _repo(bbdir) + balldomain = root / "examples" / "cores" / core / "configs" / "balldomains" / "default.toml" + if not balldomain.is_file(): + raise RuntimeError(f"missing balldomain: {balldomain}") + out = root / "examples" / "cores" / core / "isa" / "ballISA.h" + out.parent.mkdir(parents=True, exist_ok=True) + parse = _configparse(bbdir) + with tempfile.NamedTemporaryFile(suffix=".json") as tmp: + _run( + [sys.executable, str(parse / "toml2json.py"), str(balldomain), tmp.name, "--repo", str(root)], + bbdir=bbdir, + ) + _run( + [sys.executable, str(parse / "json_to_ball_isa.py"), tmp.name, str(out)], + bbdir=bbdir, + ) + return out + + +def build_workload( + bbdir: str, + chip: str, + *, + model: str = "", + rushb: str | None = None, + stable: bool = False, +) -> None: + if not re.fullmatch(r"[A-Za-z0-9_-]+", chip): + raise ValueError(f"invalid chip: {chip}") + if rushb is not None and rushb not in {"bemu", "verilator"}: + raise ValueError(f"rushB must be bemu|verilator, got {rushb!r}") + root = _repo(bbdir) + install_bundle(bbdir, chip) + core = _workload_core(bbdir, chip) + build_compiler(bbdir, core=core) + isa = gen_ball_isa(bbdir, core) + bemu_manifest = _bemu_manifest(bbdir, chip) + _run( + ["cargo", "build", "--release", "--manifest-path", bemu_manifest, "--lib"], + bbdir=bbdir, + extra_env=bebop_cargo_env(bbdir, chip), + ) + + riscv = _require_riscv() + linux_cc = riscv / "bin" / "riscv64-unknown-linux-gnu-gcc" + linux_cxx = riscv / "bin" / "riscv64-unknown-linux-gnu-g++" + if not linux_cc.is_file() or not linux_cxx.is_file(): + raise RuntimeError(f"missing RISC-V linux toolchain under {riscv / 'bin'}") + + wl = root / "bb-tests" / "workloads" + build = Path(workload_build_dir(str(root), chip)) + defs_py = _configparse(bbdir) / "workload_cmake_defs.py" + defs = subprocess.check_output( + [sys.executable, str(defs_py), "--repo", str(root), "--chip", chip], + text=True, + env=_pythonpath(bbdir), + ) + cmake_model, ninja_arg = _ninja_target(model.lower(), rushb) + compiler_build = ( + root / "compiler" / "thirdparty" / "buddy-mlir" / "build" / "cores" / core + ) + cache = compiler_build / "CMakeCache.txt" + python_exec = subprocess.check_output( + ["sed", "-n", r"s/^Python3_EXECUTABLE:FILEPATH=//p", str(cache)], + text=True, + ).splitlines() + if not python_exec: + raise RuntimeError(f"Python3_EXECUTABLE not found in {cache}") + env = os.environ.copy() + env.update(bebop_cargo_env(bbdir, chip)) + env["PATH"] = f"{riscv / 'bin'}:{env.get('PATH', '')}" + env["RISCV"] = str(riscv) + env["CC"] = str(linux_cc) + env["CXX"] = str(linux_cxx) + env["BUDDY_MLIR_BUILD_DIR"] = str(compiler_build) + build.parent.mkdir(parents=True, exist_ok=True) + if build.is_dir(): + import shutil + + shutil.rmtree(build) + build.mkdir(parents=True) + cmake_args = [ + "cmake", + "-G", + "Ninja", + f"-DBUCKYBALL_STABLE={'ON' if stable else 'OFF'}", + f"-DBB_BALL_ISA_INCLUDE_DIR={isa.parent}", + f"-DPython3_EXECUTABLE={python_exec[0]}", + f"-DCMAKE_C_COMPILER={linux_cc}", + f"-DCMAKE_CXX_COMPILER={linux_cxx}", + ] + for line in defs.splitlines(): + if line: + cmake_args.append(f"-D{line}") + if cmake_model: + cmake_args.extend(["-DMODEL=" + cmake_model, "-DARCH=buckyball"]) + cmake_args.append("..") + result = subprocess.run(cmake_args, cwd=build, env=env) + if result.returncode != 0: + raise RuntimeError(f"cmake failed ({result.returncode})") + if ninja_arg: + result = subprocess.run(["ninja", f"-j{os.cpu_count() or 1}", ninja_arg], cwd=build, env=env) + else: + result = subprocess.run(["ninja", f"-j{os.cpu_count() or 1}"], cwd=build, env=env) + if result.returncode != 0: + raise RuntimeError(f"ninja failed ({result.returncode})") diff --git a/bbdev/api/utils/chip.py b/bbdev/api/utils/chip.py index 13e16f90e..145e9ddeb 100644 --- a/bbdev/api/utils/chip.py +++ b/bbdev/api/utils/chip.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from pathlib import Path +import re import tomllib @@ -9,48 +10,90 @@ class CorePackage: name: str root: Path - config: Path compiler: Path | None + config: Path | None = None -def _load_toml(path: Path) -> dict: - with path.open("rb") as source: - data = tomllib.load(source) - if not isinstance(data, dict): - raise ValueError(f"TOML root must be a table: {path}") - return data +_SKIP_CORE_DIRS = frozenset() +_CHIP_RE = re.compile(r"^[A-Za-z0-9_-]+$") +_MILL_KEYS = frozenset({"verilatorConfig", "p2eConfig"}) +_DERIVED_KEYS = frozenset({"name", "topology", "compilerCore", "bemuTileIndex"}) -def resolve_chip_runtime_manifest(bbdir: str, chip: str, runtime_name: str) -> Path: - """Resolve a Cargo manifest declared by a chip's ``[runtime]`` table.""" - chip_root = Path(bbdir) / "examples" / "chips" / chip - chip_manifest = chip_root / "chip.toml" - if not chip_manifest.is_file(): - raise ValueError(f"Chip manifest does not exist: {chip_manifest}") - - data = _load_toml(chip_manifest) - chip_table = data.get("chip") - if not isinstance(chip_table, dict) or chip_table.get("name") != chip: - raise ValueError(f"Chip manifest name mismatch: {chip_manifest}") +def bebop_cargo(bbdir: str) -> Path: + manifest = Path(bbdir) / "bebop" / "Cargo.toml" + if not manifest.is_file(): + raise ValueError(f"bebop Cargo.toml does not exist: {manifest}") + return manifest - runtime = data.get("runtime") - if not isinstance(runtime, dict): - raise ValueError(f"Chip manifest missing [runtime]: {chip_manifest}") - manifest_rel = runtime.get(runtime_name) - if not isinstance(manifest_rel, str) or not manifest_rel: - raise ValueError(f"Chip manifest missing runtime.{runtime_name}: {chip_manifest}") - manifest = (chip_root / manifest_rel).resolve() +def bebop_bemu_cargo(bbdir: str, chip: str) -> Path: + manifest = Path(bbdir) / "examples" / "chips" / chip / "generated" / "bemu" / "Cargo.toml" if not manifest.is_file(): - raise ValueError(f"Chip runtime manifest does not exist: {manifest}") + raise ValueError(f"bebop-bemu manifest does not exist: {manifest}") return manifest -def _core_table(path: Path) -> dict: - core = _load_toml(path).get("core") - if not isinstance(core, dict): - raise ValueError(f"missing [core] table: {path}") - return core +def require_chip(data: dict) -> str: + chip = data.get("chip") + if not isinstance(chip, str) or not chip: + raise ValueError("Missing required parameter: --chip") + extra = data.get("config") + if isinstance(extra, str) and extra and extra != "None": + raise ValueError( + "do not pass --config; mill class comes from chip.toml [chip] " + "verilatorConfig / p2eConfig" + ) + if not _CHIP_RE.fullmatch(chip): + raise ValueError(f"invalid chip: {chip}") + return chip + + +def _chip_table(data: dict, chip: str) -> dict: + if "runtime" in data: + raise ValueError( + f"{chip}: [runtime] is removed; mill class is [chip].verilatorConfig / p2eConfig" + ) + extra = sorted(k for k in data if k != "chip") + if extra: + raise ValueError(f"{chip}: unexpected top-level keys in chip.toml: {extra}") + table = data.get("chip") + if not isinstance(table, dict): + raise ValueError(f"{chip}: missing [chip] in chip.toml") + for key, value in table.items(): + if key in _DERIVED_KEYS: + raise ValueError( + f"{chip}: {key} is derived from the chip directory / topology; do not set it" + ) + if key not in _MILL_KEYS: + raise ValueError(f"{chip}: unexpected [chip].{key} in chip.toml") + if not isinstance(value, str) or not value: + raise ValueError(f"{chip}: [chip].{key} must be a non-empty string") + return table + + +def chip_toml(bbdir: str, chip: str) -> dict: + chip = require_chip({"chip": chip}) + path = Path(bbdir) / "examples" / "chips" / chip / "chip.toml" + if not path.is_file(): + raise ValueError(f"missing chip.toml: {path}") + with path.open("rb") as handle: + data = tomllib.load(handle) + if not isinstance(data, dict): + raise ValueError(f"chip.toml root must be a table: {path}") + _chip_table(data, chip) + return data + + +def chip_field(bbdir: str, chip: str, key: str) -> str: + if key not in _MILL_KEYS: + raise ValueError(f"invalid chip field: {key}") + value = chip_toml(bbdir, chip)["chip"].get(key) + if not isinstance(value, str) or not value: + raise ValueError(f"{chip}: missing [chip].{key} in chip.toml") + if "/" in value or "\\" in value or value in (".", ".."): + raise ValueError(f"{chip}: [chip].{key} is not a single path component: {value!r}") + return value def available_cores(bbdir: str) -> list[str]: @@ -60,66 +103,34 @@ def available_cores(bbdir: str) -> list[str]: return sorted( path.name for path in roots.iterdir() - if (path / "manifest.toml").is_file() + if path.is_dir() + and path.name not in _SKIP_CORE_DIRS + and not path.name.startswith(".") + and (path / "configs").is_dir() ) def resolve_core(bbdir: str, name: str, require_compiler: bool = False) -> CorePackage: root = Path(bbdir) / "examples" / "cores" / name - manifest = root / "manifest.toml" - if not manifest.is_file(): + if not root.is_dir() or name in _SKIP_CORE_DIRS: raise ValueError(f"Core does not exist: {name}") + if not (root / "configs").is_dir(): + raise ValueError(f"Core missing configs/: {root}") - core = _core_table(manifest) - if core.get("name") != name: - raise ValueError(f"Core manifest name mismatch: {manifest}") - - config_rel = core.get("config") - if not isinstance(config_rel, str) or not config_rel: - raise ValueError(f"Core manifest missing core.config: {manifest}") - config = (root / config_rel).resolve() - if not config.is_file(): - raise ValueError(f"Core configuration does not exist: {config}") - - compiler_rel = core.get("compiler") - compiler = (root / compiler_rel).resolve() if isinstance(compiler_rel, str) else None - if compiler is not None and not (compiler / "CMakeLists.txt").is_file(): - raise ValueError(f"Core compiler does not exist: {compiler}") + compiler_dir = root / "compiler" + compiler = compiler_dir if (compiler_dir / "CMakeLists.txt").is_file() else None if require_compiler and compiler is None: raise ValueError(f"Core has no compiler package: {name}") - return CorePackage(name=name, root=root, config=config, compiler=compiler) + return CorePackage(name=name, root=root, compiler=compiler, config=None) def available_compiler_chips(bbdir: str) -> list[str]: chips = Path(bbdir) / "examples" / "chips" if not chips.is_dir(): return [] - - result = [] - for root in chips.iterdir(): - manifest = root / "chip.toml" - if not manifest.is_file(): - continue - try: - core_name = _load_toml(manifest).get("chip", {}).get("compilerCore") - if isinstance(core_name, str): - resolve_core(bbdir, core_name, require_compiler=True) - result.append(root.name) - except (ValueError, tomllib.TOMLDecodeError): - continue - return sorted(result) - - -def resolve_chip_compiler_core(bbdir: str, chip: str) -> CorePackage: - manifest = Path(bbdir) / "examples" / "chips" / chip / "chip.toml" - if not manifest.is_file(): - raise ValueError(f"Chip manifest does not exist: {manifest}") - - chip_table = _load_toml(manifest).get("chip") - if not isinstance(chip_table, dict) or chip_table.get("name") != chip: - raise ValueError(f"Chip manifest name mismatch: {manifest}") - core_name = chip_table.get("compilerCore") - if not isinstance(core_name, str) or not core_name: - raise ValueError(f"Chip manifest missing chip.compilerCore: {manifest}") - return resolve_core(bbdir, core_name, require_compiler=True) + return sorted( + path.name + for path in chips.iterdir() + if path.is_dir() and (path / "chip.toml").is_file() + ) diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index f6ecb21ab..71f70bd2f 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -1,117 +1,203 @@ import os import re +import subprocess from datetime import datetime from pathlib import Path +from utils.chip import chip_field, chip_toml + def get_buckyball_path(): + current_dir = os.path.dirname(__file__) + # bbdev/api/utils -> bbdev/api -> bbdev -> buckyball + inferred = os.path.realpath(os.path.dirname(os.path.dirname(os.path.dirname(current_dir)))) + root = os.environ.get("BUCKYBALL_ROOT") if root: if not os.path.isabs(root): raise ValueError("BUCKYBALL_ROOT must be an absolute path") if not os.path.isdir(root): raise ValueError(f"BUCKYBALL_ROOT does not exist: {root}") - return root + root = os.path.realpath(root) + if root != inferred: + raise ValueError( + f"BUCKYBALL_ROOT={root} does not match bbdev tree root {inferred}" + ) + else: + root = inferred - current_dir = os.path.dirname(__file__) - # bbdev/api/utils -> bbdev/api -> bbdev -> buckyball - return os.path.dirname(os.path.dirname(os.path.dirname(current_dir))) + home = os.path.realpath(str(Path.home())) + if not Path(root).is_relative_to(home): + raise ValueError(f"buckyball root must be under $HOME ({home}): {root}") + return root -def get_verilator_build_dir(bbdir, config=None, output_dir=None): - if output_dir: - return output_dir +def _chip_name(chip): + if not isinstance(chip, str) or not chip or chip == "None": + raise ValueError("missing required parameter: chip") + if not re.fullmatch(r"[A-Za-z0-9_.-]+", chip): + raise ValueError(f"invalid chip name: {chip}") + return chip + + +def chip_output_root(bbdir, chip): + chip = _chip_name(chip) + return os.path.join(bbdir, "bb-tests", "output", chip) + - return get_config_build_dir(bbdir, config) +def workload_build_dir(bbdir, chip): + chip = _chip_name(chip) + return os.path.join(bbdir, "bb-tests", "workloads", "build", chip) -def get_vcs_build_dir(bbdir, config=None, output_dir=None): - """Return the shared generated-RTL directory used by the VCS flow. +def workload_tests_root(bbdir, chip): + return os.path.join(chip_output_root(bbdir, chip), "workloads", "src") - VCS and Verilator elaborate the same BBSimHarness RTL. Simulator-specific - products live below ``vcs/`` so neither flow overwrites the other. - """ - return get_config_build_dir(bbdir, config, output_dir) +def bebop_target_dir(bbdir, chip): + chip = _chip_name(chip) + return os.path.join(bbdir, "bebop", "target", chip) -def get_chip_from_config(bbdir, config): - """Resolve the owning chip from a fully qualified Scala config name.""" - if not isinstance(config, str) or not config or config == "None": - raise ValueError("invalid config name") - config_class = config.rsplit(".", 1)[-1] - declaration = re.compile(rf"\b(?:class|object)\s+{re.escape(config_class)}\b") - chips_root = Path(bbdir) / "examples" / "chips" - matches = [] - for chip_dir in sorted(path for path in chips_root.iterdir() if path.is_dir()): - scala_root = chip_dir / "arch" / "src" / "main" / "scala" - if not scala_root.is_dir(): +def bebop_cargo_env(bbdir, chip): + return {"CARGO_TARGET_DIR": bebop_target_dir(bbdir, chip)} + + +def gcc_lib_dir(soname: str) -> str: + candidates = [] + printed = subprocess.run( + ["g++", f"-print-file-name={soname}"], + capture_output=True, + text=True, + ) + printed_path = (printed.stdout or "").strip() + if printed.returncode == 0 and printed_path and "/" in printed_path: + candidates.append(printed_path) + for directory in os.environ.get("LIBRARY_PATH", "").split(":"): + if directory: + candidates.append(os.path.join(directory, soname)) + for match in re.finditer(r"-L(\S+)", os.environ.get("NIX_LDFLAGS", "")): + candidates.append(os.path.join(match.group(1), soname)) + + runtime_names = [soname] + if soname.endswith(".so"): + runtime_names.append(soname + ".1") + + for path in candidates: + if not os.path.isfile(path): continue - for source in scala_root.rglob("*.scala"): - try: - contents = source.read_text(encoding="utf-8") - except OSError: - continue - if declaration.search(contents): - matches.append(chip_dir.name) - break + lib_dir = os.path.dirname(os.path.realpath(path)) + if any(os.path.isfile(os.path.join(lib_dir, name)) for name in runtime_names): + return lib_dir - if len(matches) != 1: - raise ValueError( - f"config {config} must resolve to exactly one chip; matches={matches}" - ) - return matches[0] + detail = printed_path or (printed.stderr or "").strip() or f"exit {printed.returncode}" + raise RuntimeError( + f"cannot locate {soname} (g++ -print-file-name={soname} -> {detail!r}; " + f"LIBRARY_PATH={os.environ.get('LIBRARY_PATH', '')!r}; " + f"NIX_LDFLAGS={os.environ.get('NIX_LDFLAGS', '')!r})" + ) -def sanitize_config_name(config=None): - if config and config != "None": - name = re.sub(r"[^A-Za-z0-9_.-]+", "_", config).strip("._") - if name: - return name - return None +_RTL_FIELD = { + "verilog": "verilatorConfig", + "synth": "verilatorConfig", + "p2e": "p2eConfig", +} -def get_config_build_dir(bbdir, config=None, output_dir=None, output_root=None): +def chip_arch_root(bbdir, chip): + return os.path.join(bbdir, "arch", "build", _chip_name(chip)) + + +def rtl_dir(bbdir, chip, product, output_dir=None): if output_dir: return output_dir + if product not in _RTL_FIELD: + raise ValueError(f"invalid rtl product: {product}") + cfg = chip_field(bbdir, chip, _RTL_FIELD[product]) + return os.path.join(chip_arch_root(bbdir, chip), cfg) + + +def rtl_dir_for_clean(bbdir, chip, product, output_dir=None): + return rtl_dir(bbdir, chip, product, output_dir) + + +def rtl_config_name(bbdir, chip, product, output_dir=None): + return os.path.basename(os.path.normpath(rtl_dir(bbdir, chip, product, output_dir))) + + +def _log_part(value, what): + if not isinstance(value, str) or not value: + raise ValueError(f"log dir missing {what}") + if os.path.basename(value) != value: + raise ValueError(f"log {what} must be a single path component: {value!r}") + if not re.fullmatch(r"[A-Za-z0-9_.-]+", value): + raise ValueError(f"invalid log {what}: {value!r}") + return value + + +def get_run_log_dir(bbdir, chip, product, stamp, tool, name, output_dir=None): + chip = _chip_name(chip) + config = _log_part(rtl_config_name(bbdir, chip, product, output_dir), "config") + stamp = _log_part(stamp, "timestamp") + tool = _log_part(tool, "tool") + name = _log_part(os.path.basename(name), "name") + return os.path.join(bbdir, "log", f"{stamp}-{chip}-{config}-{tool}-{name}") + + +def get_bemu_log_dir(bbdir, chip, stamp, name): + chip = _chip_name(chip) + data = chip_toml(bbdir, chip) + table = data.get("chip") + if not isinstance(table, dict): + raise ValueError(f"{chip}: missing [chip] in chip.toml") + verilator = table.get("verilatorConfig") + p2e = table.get("p2eConfig") + if isinstance(verilator, str) and verilator: + config = verilator + elif isinstance(p2e, str) and p2e: + config = p2e + else: + raise ValueError(f"{chip}: missing [chip].verilatorConfig and [chip].p2eConfig") + stamp = _log_part(stamp, "timestamp") + name = _log_part(os.path.basename(name), "name") + config = _log_part(config, "config") + return os.path.join(bbdir, "log", f"{stamp}-{chip}-{config}-bemu-{name}") + + +def get_verilator_build_dir(bbdir, chip=None, output_dir=None): + if output_dir: + return output_dir + if chip is None: + raise ValueError("missing required parameter: chip") + return rtl_dir(bbdir, chip, "verilog") + - name = sanitize_config_name(config) - if output_root: - if not name: - raise ValueError("output_root requires a valid config name") - return os.path.join(output_root, name) +def get_p2e_build_dir(bbdir, chip, output_dir=None): + return rtl_dir(bbdir, chip, "p2e", output_dir) - if name: - return f"{bbdir}/arch/build/{name}" - return f"{bbdir}/arch/build" +def get_vcs_build_dir(bbdir, chip=None, output_dir=None): + return get_verilator_build_dir(bbdir, chip, output_dir) -def get_dc_rtl_dir(bbdir, config=None): - if not config or config is True: - raise ValueError("missing required parameter: config") +def get_arch_build_dir(bbdir, chip, output_dir=None): + return rtl_dir(bbdir, chip, "synth", output_dir) - name = sanitize_config_name(config) - if not name: - raise ValueError("invalid config name") - return os.path.join(bbdir, "arch", "build", name) +def get_dc_rtl_dir(bbdir, chip=None): + return get_arch_build_dir(bbdir, chip) -def get_dc_analysis_dir(bbdir, config=None, stage="area"): - if not config or config is True: - raise ValueError("missing required parameter: config") - name = sanitize_config_name(config) - if not name: - raise ValueError("invalid config name") +def get_dc_analysis_dir(bbdir, chip=None, stage="area"): if stage not in {"area", "power"}: raise ValueError(f"invalid DC analysis stage: {stage}") timestamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") - return os.path.join(bbdir, "log", f"{timestamp}-dc-{name}", stage) + return get_run_log_dir(bbdir, chip, "synth", timestamp, "dc", stage) def check_dc_rtl_args(body: dict): - allowed = {"config", "top"} + allowed = {"chip", "top"} for name in body: if name not in allowed: raise ValueError(f"unexpected parameter: {name}") @@ -123,11 +209,11 @@ def check_dc_rtl_args(body: dict): def check_dc_power_args(body: dict): - allowed = {"config", "top", "activity", "format", "strip_path", "workload", "start_ns", "end_ns", "start-ns", "end-ns"} + allowed = {"chip", "top", "activity", "format", "strip_path", "workload", "start_ns", "end_ns", "start-ns", "end-ns"} for name in body: if name not in allowed: raise ValueError(f"unexpected parameter: {name}") - common = {key: body[key] for key in ("config", "top") if key in body} + common = {key: body[key] for key in ("chip", "top") if key in body} check_dc_rtl_args(common) activity = body.get("activity") activity_format = body.get("format") diff --git a/bbdev/api/utils/rtl.py b/bbdev/api/utils/rtl.py new file mode 100644 index 000000000..660cf8d1a --- /dev/null +++ b/bbdev/api/utils/rtl.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import os +import shlex +from pathlib import Path + +from utils.chip import chip_field +from utils.path import chip_arch_root +from utils.stream_run import stream_run_logger_async + +_KIND = { + "verilog": ("verilatorConfig", "sims.verilator.Elaborate", False, "fesvr"), + "synth": ("verilatorConfig", "sims.verilator.Elaborate", True, None), + "p2e": ("p2eConfig", "sims.p2e.Elaborate", False, "p2e"), +} + + +def mill_cmd(main: str, mill_config: str, out: str, *, seq_mem: bool = False) -> str: + extra = "" + if seq_mem: + extra = ( + " -lowering-options=disallowLocalVariables" + f" --repl-seq-mem --repl-seq-mem-file={shlex.quote(os.path.join(out, 'mems.conf'))}" + ) + return ( + f"mill -i __.test.runMain {main} {mill_config} " + "--disable-annotation-unknown --strip-debug-info -O=debug " + f"--split-verilog -o={shlex.quote(out)}{extra}" + ) + + +def p2e_top_cmd(out: str) -> str: + return ( + "mill -i __.test.runMain sims.p2e.ElaborateP2ETop " + "--disable-annotation-unknown --strip-debug-info -O=debug " + f"--split-verilog -o={shlex.quote(out)}" + ) + + +def patch_fesvr(out: str, arch: str) -> None: + harness = os.path.join(arch, "BBSimHarness.sv") + if os.path.isfile(harness): + os.remove(harness) + drop = ("fesvr/memif.h", "fesvr/elfloader.h") + for name in ("mm.h", "mm.cc"): + path = os.path.join(out, name) + if not os.path.isfile(path): + continue + text = Path(path).read_text() + Path(path).write_text( + "".join(line for line in text.splitlines(True) if not any(token in line for token in drop)) + ) + + +def drop_p2e_harness(arch: str) -> None: + for name in ("P2EHarness.sv", "P2ETop.v", "P2ETopWrapper.sv"): + path = os.path.join(arch, name) + if os.path.isfile(path): + os.remove(path) + + +def normalize_timescale(out: str) -> None: + n = 0 + for path in Path(out).rglob("*"): + if path.suffix not in {".v", ".sv"}: + continue + text = path.read_text() + if "`timescale" in text: + continue + path.write_text("`timescale 1ns/1ps\n" + text) + n += 1 + return n + + +def assert_sv(out: str) -> None: + n = sum(1 for path in Path(out).rglob("*") if path.suffix in {".v", ".sv"}) + if n == 0: + raise RuntimeError(f"no .sv/.v emitted under {out}") + + +async def run_chip_mill(ctx, bbdir: str, chip: str, kind: str, prefix: str, output_dir=None): + if kind not in _KIND: + raise ValueError(f"invalid mill kind: {kind}") + arch = os.path.join(bbdir, "arch") + if not os.path.isdir(arch): + raise ValueError(f"missing {arch}") + key, main, seq_mem, post = _KIND[kind] + mill_config = chip_field(bbdir, chip, key) + out = output_dir or os.path.join(chip_arch_root(bbdir, chip), mill_config) + os.makedirs(out, exist_ok=True) + ctx.logger.info(f"Using mill config: {mill_config}") + ctx.logger.info(f"Using build directory: {out}") + result = await stream_run_logger_async( + cmd=mill_cmd(main, mill_config, out, seq_mem=seq_mem), + logger=ctx.logger, + cwd=arch, + stdout_prefix=prefix, + stderr_prefix=prefix, + ) + if result.returncode != 0: + return out, result.returncode + if post == "fesvr": + patch_fesvr(out, arch) + if post == "p2e": + top = await stream_run_logger_async( + cmd=p2e_top_cmd(out), + logger=ctx.logger, + cwd=arch, + stdout_prefix=prefix, + stderr_prefix=prefix, + ) + if top.returncode != 0: + return out, top.returncode + drop_p2e_harness(arch) + n = normalize_timescale(out) + ctx.logger.info(f"normalized timescale in {n} files") + if seq_mem: + mems = os.path.join(out, "mems.conf") + if not os.path.isfile(mems): + raise RuntimeError(f"mill seq-mem did not emit {mems}") + assert_sv(out) + return out, 0 diff --git a/bbdev/api/utils/stream_run.py b/bbdev/api/utils/stream_run.py index 101900821..699efa87c 100644 --- a/bbdev/api/utils/stream_run.py +++ b/bbdev/api/utils/stream_run.py @@ -1,3 +1,4 @@ +import asyncio import subprocess import threading from typing import Optional, List, Callable @@ -208,3 +209,33 @@ def log_stderr(line): env=env, task_scope=task_scope, ) + + +async def stream_run_logger_async( + cmd: str, + logger, + cwd: Optional[str] = None, + shell: bool = True, + executable: Optional[str] = None, + timeout: Optional[float] = None, + stdout_prefix: str = "STDOUT", + stderr_prefix: str = "STDERR", + verbose: bool = False, + env: Optional[dict] = None, + task_scope: Optional[str] = None, +) -> StreamResult: + """Run stream_run_logger off the Motia event loop so long jobs do not stall the worker.""" + return await asyncio.to_thread( + stream_run_logger, + cmd, + logger, + cwd, + shell, + executable, + timeout, + stdout_prefix, + stderr_prefix, + verbose, + env, + task_scope, + ) diff --git a/bbdev/api/utils/workers.py b/bbdev/api/utils/workers.py new file mode 100644 index 000000000..e7542df6c --- /dev/null +++ b/bbdev/api/utils/workers.py @@ -0,0 +1,170 @@ +import json +import os +import subprocess +from pathlib import Path + +from utils.port import HOST + +SERVER_PORTS_FILE = "bbdev-server.json" + + +def server_ports_path(workflow_dir: str) -> str: + return os.path.join(workflow_dir, "data", SERVER_PORTS_FILE) + + +def write_server_ports(workflow_dir: str, http_port: int, worker_port: int, bb_root: str) -> None: + path = server_ports_path(workflow_dir) + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = f"{path}.tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump( + { + "http_port": http_port, + "worker_port": worker_port, + "bb_root": os.path.realpath(bb_root), + }, + f, + ) + f.write("\n") + os.replace(tmp, path) + + +def read_server_ports(workflow_dir: str) -> dict: + path = server_ports_path(workflow_dir) + with open(path, encoding="utf-8") as f: + data = json.load(f) + if not isinstance(data, dict): + raise RuntimeError(f"invalid server ports file: {path}") + for key in ("http_port", "worker_port", "bb_root"): + if key not in data: + raise RuntimeError(f"server ports file missing {key}: {path}") + return data + + +def list_engine_workers(worker_port: int, timeout_s: float = 10.0) -> list: + result = subprocess.run( + [ + "iii", + "trigger", + "--function-id", + "engine::workers::list", + "--address", + HOST, + "--port", + str(worker_port), + "--timeout-ms", + str(int(timeout_s * 1000)), + ], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"engine::workers::list failed (rc={result.returncode}): " + f"{result.stderr.strip() or result.stdout.strip()}" + ) + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as e: + raise RuntimeError(f"engine::workers::list returned non-JSON: {result.stdout[:500]}") from e + workers = payload.get("workers") + if not isinstance(workers, list): + raise RuntimeError(f"engine::workers::list missing workers list: {payload}") + return workers + + +def _worker_pid(worker: dict) -> int | None: + name = worker.get("name") + if not isinstance(name, str) or ":" not in name: + return None + tail = name.rsplit(":", 1)[-1] + if not tail.isdigit(): + return None + return int(tail) + + +def _cwd(pid: int) -> str | None: + try: + return os.readlink(f"/proc/{pid}/cwd") + except (FileNotFoundError, PermissionError, OSError): + return None + + +def motia_worker_count() -> int: + raw = os.environ.get("BBDEV_MOTIA_WORKERS", "1") + try: + n = int(raw) + except ValueError as e: + raise RuntimeError(f"BBDEV_MOTIA_WORKERS must be an int, got {raw!r}") from e + if n < 1: + raise RuntimeError(f"BBDEV_MOTIA_WORKERS must be >= 1, got {n}") + return n + + +def classify_workspace_workers(worker_port: int, bb_root: str) -> tuple[list, list]: + """Return (local, foreign) Motia workers attached to this iii.""" + root = os.path.realpath(bb_root) + workers = list_engine_workers(worker_port) + foreign = [] + local = [] + for worker in workers: + funcs = worker.get("function_count") or 0 + runtime = worker.get("runtime") + if funcs <= 0 and runtime not in ("python", "motia"): + continue + pid = _worker_pid(worker) + if pid is None: + foreign.append(worker) + continue + cwd = _cwd(pid) + if cwd is None: + foreign.append(worker) + continue + cwd_real = os.path.realpath(cwd) + if Path(cwd_real).is_relative_to(root): + local.append(worker) + else: + foreign.append(worker) + return local, foreign + + +def assert_sole_workspace_workers( + worker_port: int, bb_root: str, min_local: int | None = None +) -> int: + """Refuse to run if any Motia worker outside this repo is connected to our iii. + + Shared machines often leave orphan Motia processes that reconnect to whatever + process later binds the same worker port; iii then load-balances jobs into + the foreign tree (e.g. /home/other/buckyball). + + Returns the number of local Motia workers when isolation checks pass. + """ + root = os.path.realpath(bb_root) + need = motia_worker_count() if min_local is None else min_local + if need < 1: + raise RuntimeError(f"min_local must be >= 1, got {need}") + local, foreign = classify_workspace_workers(worker_port, bb_root) + + if foreign: + detail = [] + for worker in foreign: + pid = _worker_pid(worker) + detail.append( + { + "name": worker.get("name"), + "id": worker.get("id"), + "function_count": worker.get("function_count"), + "pid_cwd": _cwd(pid) if pid is not None else None, + } + ) + raise RuntimeError( + "foreign Motia worker(s) connected to this iii engine; " + f"refusing to run (worker_port={worker_port}, bb_root={root}): {detail}" + ) + if len(local) < need: + raise RuntimeError( + f"need {need} Motia worker(s) from this workspace, found {len(local)} " + f"(worker_port={worker_port}, bb_root={root})" + ) + return len(local) diff --git a/bbdev/bbdev b/bbdev/bbdev index a68a6b1e9..32986b4c9 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -15,6 +15,7 @@ import shutil sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "api")) from utils.port import HOST, reserve_port from utils.process import clean_python_caches, stop_workspace_servers, terminate_group +from utils.workers import assert_sole_workspace_workers, motia_worker_count, write_server_ports workflow_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "api") workspace_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -29,27 +30,27 @@ HTTP_PORT_END = 5500 # Operations with string values accept sub-arguments; True means boolean-only (no sub-args). WORKFLOW_COMMANDS = { "verilator": { - "clean": 'Clean build directory. Args: "--config [--output-dir ]"', - "verilog": 'Generate verilog files. Args: "--config [--output-dir ] [--balltype ]"', - "build": 'Build simulation executable. Args: "--config [--jobs ] [--output-dir ]"', - "sim": 'Run simulation. Args: "--config --binary [--batch] [--output-dir ]"', - "run": 'Integrated build+sim+run. Args: "--config --binary [--batch] [--jobs ] [--output-dir ]"', + "clean": 'Clean build directory. Args: "--chip [--output-dir ]"', + "verilog": 'Generate verilog files. Args: "--chip [--output-dir ] [--balltype ]"', + "build": 'Build simulation executable. Args: "--chip [--jobs ] [--output-dir ]"', + "sim": 'Run simulation. Args: "--chip --binary [--batch] [--output-dir ]"', + "run": 'Integrated build+sim+run. Args: "--chip --binary [--batch] [--jobs ] [--output-dir ]"', "cosim": 'Run cosimulation. Args: "--binary [--batch]"', }, "vcs": { - "clean": 'Clean VCS artifacts. Args: "--config [--output-dir ]"', - "verilog": 'Generate RTL for VCS. Args: "--config [--output-dir ]"', - "build": 'Build VCS simulation executable. Args: "--config [--jobs ] [--output-dir ]"', - "sim": 'Run a built VCS simulation. Args: "--config --binary [--batch] [--output-dir ]"', - "run": 'Generate RTL, build, and run VCS. Args: "--config --binary [--batch] [--jobs ] [--output-dir ]"', + "clean": 'Clean VCS artifacts. Args: "--chip [--output-dir ]"', + "verilog": 'Generate RTL for VCS. Args: "--chip [--output-dir ]"', + "build": 'Build VCS simulation executable. Args: "--chip [--jobs ] [--output-dir ]"', + "sim": 'Run a built VCS simulation. Args: "--chip --binary [--batch] [--output-dir ]"', + "run": 'Generate RTL, build, and run VCS. Args: "--chip --binary [--batch] [--jobs ] [--output-dir ]"', }, "workload": { - "clean": "Clean workload output directory.", + "clean": 'Clean workload output directory. Args: "--chip "', "build": 'Build workload. Args: "--chip [--model ] [--stable] [--rushB ]"; without --model, --rushB builds CTest and MLIR rushB runners.', - "tohex": "Convert all -baremetal ELF files under bb-tests/output/workloads/src to hex.", + "tohex": 'Convert chip workload ELF to hex. Args: "--chip "', }, "kernel": { - "build": 'Build RISC-V kernel + rootfs. Args: "[--chip ] [--model ] [--interactive] [--visible-hart-count ] [--total-hart-count ]"', + "build": 'Build RISC-V kernel + rootfs. Args: "[--chip ] [--model ] [--interactive] [--visible-hart-count ] [--total-hart-count ]"; --model requires --chip', }, "firesim": { "enumeratefpgas": "Enumerate FPGAs.", @@ -58,49 +59,59 @@ WORKFLOW_COMMANDS = { "runworkload": "Run workload.", }, "compiler": { - "build": 'Build compiler. Args: "--chip [--stable]"', + "build": 'Build compiler. Args: "--chip |--core [--stable]"', }, "bebop-bemu": { - "sim": 'Run Bebop emulator simulation. Args: "--chip --binary [--rushB] [--pk] [--disasm] [--tool-profile] [--log-dir ]"', + "sim": 'Run Bebop emulator simulation. Args: "--chip --binary [--rushB] [--pk] [--disasm] [--tool-profile] [--itrace] [--mtrace]"', + "analysis": 'Analyze itrace/mtrace in an existing log dir. Args: "--chip --log-dir --itrace|--mtrace"', "batch": 'Run Bebop bemu nextest batch regression (parallel). Args: "--chip --test [--rushB] [--clean-before]"', }, "bebop-verilator": { - "clean": 'Clean Bebop Verilator build directory. Args: "--config [--output-dir ]"', - "verilog": 'Generate Bebop Verilog files. Args: "--config [--output-dir ]"', - "build": 'Build Bebop Verilator simulation executable. Args: "--config [--diff] [--jobs ] [--output-dir ]"', - "sim": 'Run Bebop Verilator simulation. Args: "--config --binary [--rushB] [--log-dir ] [--fst-dir ] [--no-wave]"', - "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--rushB] [--jobs ] [--config ] [--log-dir ] [--fst-dir ] [--no-wave]"', - "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --config --test [--diff] [--rushB] [--clean-before]"', + "clean": 'Clean Bebop Verilator build directory. Args: "--chip [--output-dir ]"', + "verilog": 'Generate Bebop Verilog files. Args: "--chip [--output-dir ]"', + "build": 'Build Bebop Verilator simulation executable. Args: "--chip [--diff] [--jobs ] [--output-dir ]"', + "sim": 'Run Bebop Verilator simulation. Args: "--chip --binary [--rushB] [--log-dir ] [--fst-dir ] [--no-wave]"', + "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--rushB] [--jobs ] [--chip ] [--log-dir ] [--fst-dir ] [--no-wave]"', + "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --test [--diff] [--rushB] [--clean-before]"', }, "bebop-p2e": { - "clean": 'Clean Bebop P2E build directory. Args: "--config [--output-dir ]"', - "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--config [--output-dir ]"', - "buildbitstream": 'Build Bebop P2E runtime case. Args: "--config [--vsrc-dir ] [--output-dir ]"', + "clean": 'Clean Bebop P2E build directory. Args: "--chip [--output-dir ]"', + "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--chip [--output-dir ]"', + "buildbitstream": 'Build Bebop P2E runtime case. Args: "--chip [--vsrc-dir ] [--output-dir ]"', "runworkload": 'Run Bebop P2E workload. Args: "--image --bitstream [--multi-fpga] [--wave] [--wave-start ] [--itrace] [--mtrace] [--pmctrace]"', "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--chip --bitstream --test "', }, "dc": { - "verilog": 'Generate top-scoped DC RTL collateral under arch/build/. Args: "--config [--top ]"', - "area": 'Generate RTL, run DC synthesis, and report area under log/-dc-/area. Args: "--config [--top ]".', - "power": 'Generate RTL, run chip-owned power simulation, then run PrimeTime PX under log/-dc-/power. Args: "--config [--top ] [--workload ] [--start-ns ] [--end-ns ] [--activity --format ]".', + "verilog": 'Generate top-scoped DC RTL under arch/build//. Args: "--chip [--top ]"', + "area": 'Generate RTL, run DC synthesis, and report area under log////area. Args: "--chip [--top ]".', + "power": 'Generate RTL, run chip-owned power simulation, then run PrimeTime PX under log////power. Args: "--chip [--top ] [--workload ] [--start-ns ] [--end-ns ] [--activity --format ]".', }, "ip-replace": { "run": 'Replace behavioral IP RTL. Args: "--source-list --output-dir [--top ] [--consumer ]"', }, "yosys": { - "run": 'Run yosys flow. Args: "[--top ] [--config ] [--output-dir ] [--log-dir ] [--vcd ]"', - "verilog": 'Generate verilog for yosys flow. Args: "[--config ] [--output-dir ] [--log-dir ]"', + "run": 'Run yosys flow. Args: "[--top ] [--chip ] [--output-dir ] [--log-dir ] [--vcd ]"', + "verilog": 'Generate verilog for yosys flow. Args: "[--chip ] [--output-dir ] [--log-dir ]"', "synth": 'Run yosys synthesis. Args: "[--top ] [--output-dir ] [--log-dir ] [--vcd ]"', }, "uvm": { "build": 'Build a Ball UVM simulation. Args: "--ball= [--filelist ]"', "run": 'Build and run a Ball UVM simulation. Args: "--ball= [--filelist ] [--test ]"', }, + "regression": { + "buildbitstream": 'Build pebble p2e bitstream. Args: "--chip [--output-dir ]"', + "check": 'P2E pk-tests. Args: "--chip --bitstream [--chip ]"', + "eval-performance": 'P2E model perfetto cycles. Args: "--chip --bitstream [--chip ]"', + "eval-area-power": 'DC area+freq. Args: "--chip [--top ]"', + }, } ALL_COMMANDS = ["start", "stop"] + list(WORKFLOW_COMMANDS) +MOTIA_EXEC = "PYTHONUNBUFFERED=1 PYTHONPATH=. .venv/bin/motia dev --dir steps" + + def make_iii_config(port: int, worker_port: int, quiet: bool = False, exec_module: bool = True) -> str: """Create a temporary config.yaml with unique HTTP and Worker ports, return the path.""" base_config = os.path.join(workflow_dir, "config.yaml") @@ -121,6 +132,13 @@ def make_iii_config(port: int, worker_port: int, quiet: bool = False, exec_modul if marker not in content: raise RuntimeError("failed to remove ExecModule from iii config") content = content.split(marker, 1)[0].rstrip() + "\n" + else: + single = f" exec:\n - {MOTIA_EXEC}\n" + if single not in content: + raise RuntimeError("failed to expand Motia worker exec list in iii config") + n = motia_worker_count() + multi = " exec:\n" + "".join(f" - {MOTIA_EXEC}\n" for _ in range(n)) + content = content.replace(single, multi, 1) tmp = tempfile.NamedTemporaryFile( mode="w", suffix=".yaml", prefix="iii-config-", dir=workflow_dir, delete=False ) @@ -144,34 +162,8 @@ def cleanup_config(config_path: str): def load_host_environment() -> None: - """Import the optional host EDA contract for worker subprocesses. - - ``nix develop -c`` does not start an interactive zsh, so a sourceme file - installed in zshrc is otherwise invisible to Motia workers. Import only - when a tool is missing and keep the operation scoped to this workspace. - """ - if shutil.which("dc_shell") and shutil.which("pt_shell") and shutil.which("vcs"): - return - sourceme = os.path.join(workspace_dir, "sourceme_host.sh") - if not os.path.isfile(sourceme): - return - try: - result = subprocess.run( - ["bash", "-c", 'source "$1" >/dev/null 2>&1; env -0', "bbdev", sourceme], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - ) - except (OSError, subprocess.CalledProcessError): - return - for item in result.stdout.split(b"\0"): - if b"=" not in item: - continue - key, value = item.split(b"=", 1) - try: - os.environ[key.decode()] = value.decode() - except UnicodeDecodeError: - continue + """No-op placeholder: host EDA env (TARGET_LIBRARY etc.) comes from the user shell (zshrc).""" + return def parse_args(argv: list[str]) -> argparse.Namespace: @@ -411,31 +403,60 @@ def run_server_mode(args, cmd_info): if command == "start": print(f"Starting dev server on port {args.port}") - if args.port: - worker_port, worker_sock = reserve_port(start_port=WORKER_PORT_START, end_port=WORKER_PORT_END) - worker_sock.close() - config_path = make_iii_config(args.port, worker_port) - worker_url = f"ws://{HOST}:{worker_port}" - print(f"Using worker port {worker_port}") - else: - config_path = os.path.join(workflow_dir, "config.yaml") - worker_url = f"ws://{HOST}:49134" + if not args.port: + print("Error: --server start requires --port", file=sys.stderr) + sys.exit(2) + load_host_environment() + preferred_worker = user_preferred_port(WORKER_PORT_START, WORKER_PORT_END) + worker_port, worker_sock = reserve_port( + start_port=WORKER_PORT_START, + end_port=WORKER_PORT_END, + preferred_port=preferred_worker, + ) + worker_sock.close() + config_path = make_iii_config(args.port, worker_port) + worker_url = f"ws://{HOST}:{worker_port}" + print(f"Using worker port {worker_port} (preferred {preferred_worker})") proc = None try: repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + write_server_ports(workflow_dir, args.port, worker_port, repo_root) env = { **os.environ, "PYTHONDONTWRITEBYTECODE": "1", "BUCKYBALL_ROOT": repo_root, + "III_URL": worker_url, } - if worker_url: - env["III_URL"] = worker_url proc = subprocess.Popen( ["iii", "-c", config_path], cwd=workflow_dir, env=env, start_new_session=True, ) + need = motia_worker_count() + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if proc.poll() is not None: + sys.exit(proc.returncode or 1) + try: + n = assert_sole_workspace_workers(worker_port, repo_root, min_local=need) + break + except RuntimeError as e: + msg = str(e) + if "foreign Motia worker" in msg: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + time.sleep(0.5) + continue + else: + print( + f"Error: need {need} Motia worker(s) from this workspace within 60s", + file=sys.stderr, + ) + sys.exit(1) + print( + f"Workspace Motia isolation ok: {n} worker(s) on port {worker_port}" + ) rc = proc.wait() if rc != 0: sys.exit(rc) @@ -443,8 +464,7 @@ def run_server_mode(args, cmd_info): if proc: terminate_group(proc) stop_workspace_servers(workflow_dir, worker_url=worker_url) - if args.port: - cleanup_config(config_path) + cleanup_config(config_path) elif command == "stop": if cmd_info["operation"] == "all": @@ -603,6 +623,9 @@ def run_script_mode(args, cmd_info): http_sock.close() worker_sock.close() + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + write_server_ports(workflow_dir, available_port, worker_port, repo_root) + # Set III_URL so motia SDK connects to the correct worker port iii_env = { **os.environ, @@ -610,8 +633,9 @@ def run_script_mode(args, cmd_info): "PYTHONUNBUFFERED": "1", "PYTHONIOENCODING": "utf-8", "PYTHONDONTWRITEBYTECODE": "1", - "BUCKYBALL_ROOT": os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "BUCKYBALL_ROOT": repo_root, } + worker_procs = [] proc = subprocess.Popen( ["iii", "-c", config_path], cwd=workflow_dir, @@ -622,7 +646,6 @@ def run_script_mode(args, cmd_info): env=iii_env, start_new_session=True, ) - worker_proc = None output_thread = threading.Thread( target=forward_iii_output, args=(proc.stdout, iii_output_active, True), daemon=True @@ -641,22 +664,25 @@ def run_script_mode(args, cmd_info): ): sys.exit(1) - worker_proc = subprocess.Popen( - [".venv/bin/motia", "dev", "--dir", "steps"], - cwd=workflow_dir, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - bufsize=1, - env=iii_env, - start_new_session=True, - ) - worker_output_thread = threading.Thread( - target=forward_iii_output, - args=(worker_proc.stdout, iii_output_active, True), - daemon=True, - ) - worker_output_thread.start() + need = motia_worker_count() + for _ in range(need): + worker_proc = subprocess.Popen( + [".venv/bin/motia", "dev", "--dir", "steps"], + cwd=workflow_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + env=iii_env, + start_new_session=True, + ) + worker_procs.append(worker_proc) + worker_output_thread = threading.Thread( + target=forward_iii_output, + args=(worker_proc.stdout, iii_output_active, True), + daemon=True, + ) + worker_output_thread.start() # Phase 2: wait for route registration if not wait_for_http( @@ -668,7 +694,13 @@ def run_script_mode(args, cmd_info): sys.exit(1) time.sleep(2) - print(f"Server is ready on port {available_port}") + try: + n = assert_sole_workspace_workers(worker_port, repo_root, min_local=need) + except RuntimeError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + print(f"Server is ready on port {available_port} with {n} Motia worker(s)") rc = _submit_and_poll(available_port, api_path, cmd_info) if rc != 0: sys.exit(rc) @@ -677,7 +709,7 @@ def run_script_mode(args, cmd_info): # Cleanup: stop iii, its process group, and any workspace worker it spawned. iii_output_active.clear() time.sleep(0.5) - if worker_proc: + for worker_proc in worker_procs: terminate_group(worker_proc) terminate_group(proc) stop_workspace_servers(workflow_dir, worker_url=iii_env["III_URL"]) diff --git a/bbdev/mcp/common.py b/bbdev/mcp/common.py index fc25b8d41..13d000541 100644 --- a/bbdev/mcp/common.py +++ b/bbdev/mcp/common.py @@ -4,6 +4,7 @@ import atexit import json +import os import shutil import socket import subprocess @@ -56,8 +57,15 @@ def _opt(params: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: return params +def _preferred_http_port(lo: int = 5100, hi: int = 5500) -> int: + size = hi - lo + 1 + return lo + (os.getuid() * 37) % size + + def _free_port(lo: int = 5100, hi: int = 5500) -> int: - for p in range(lo, hi + 1): + preferred = _preferred_http_port(lo, hi) + order = list(range(preferred, hi + 1)) + list(range(lo, preferred)) + for p in order: try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(("127.0.0.1", p)) @@ -67,6 +75,14 @@ def _free_port(lo: int = 5100, hi: int = 5500) -> int: raise RuntimeError(f"No available port in {lo}-{hi}") +def _assert_workspace_workers() -> None: + sys.path.insert(0, str(API)) + from utils.workers import assert_sole_workspace_workers, read_server_ports + + ports = read_server_ports(str(API)) + assert_sole_workspace_workers(int(ports["worker_port"]), str(ports["bb_root"])) + + def _http( method: str, url: str, payload: Optional[Dict[str, Any]] = None, timeout: int = 30 ): @@ -100,7 +116,7 @@ def _stop() -> None: global _proc, _port, _log_fh if _port is not None and BBDEV.is_file(): subprocess.run( - [str(BBDEV), "stop", "--server", "--port", str(_port)], + ["nix", "develop", "--command", str(BBDEV), "stop", "--server", "--port", str(_port)], cwd=str(BBDEV.parent), capture_output=True, text=True, @@ -130,6 +146,7 @@ def _ensure() -> int: and _proc.poll() is None and _ready(_port) ): + _assert_workspace_workers() return _port if _proc is not None: _log(f"bbdev on port {_port} died; restarting") @@ -137,9 +154,9 @@ def _ensure() -> int: if not BBDEV.is_file(): raise RuntimeError(f"missing bbdev CLI: {BBDEV}") - if shutil.which("iii") is None: + if shutil.which("nix") is None: raise RuntimeError( - "iii not found; run MCP via scripts/claude/run_mcp_server.sh" + "nix not found; MCP must start bbdev through the project development environment" ) if not MOTIA.is_file(): raise RuntimeError( @@ -152,7 +169,7 @@ def _ensure() -> int: LOG.parent.mkdir(parents=True, exist_ok=True) _log_fh = open(LOG, "a", encoding="utf-8") _proc = subprocess.Popen( - [str(BBDEV), "start", "--server", "--port", str(port)], + ["nix", "develop", "--command", str(BBDEV), "start", "--server", "--port", str(port)], cwd=str(BBDEV.parent), stdout=_log_fh, stderr=_log_fh, @@ -173,6 +190,18 @@ def _ensure() -> int: f"bbdev exited early; see {LOG}\n--- log tail ---\n{tail}" ) if _ready(port): + try: + _assert_workspace_workers() + except RuntimeError as e: + msg = str(e) + if ( + ("need" in msg and "Motia worker" in msg) + or "engine::workers::list failed" in msg + ): + time.sleep(1) + continue + _stop() + raise _log(f"bbdev ready on port {port}") return port time.sleep(1) @@ -199,6 +228,7 @@ def _read_state(trace_id: str) -> Optional[Dict[str, Any]]: def submit(endpoint: str, params: Dict[str, Any]) -> Dict[str, Any]: """Submit an asynchronous bbdev task without waiting for completion.""" port = _ensure() + _assert_workspace_workers() base = f"http://127.0.0.1:{port}" _log(f"POST {endpoint} params={params}") @@ -280,10 +310,16 @@ def task_status(trace_id: str) -> Dict[str, Any]: "body": body, "server_log": str(LOG), } - if state.get("processing") is True: - out = dict(state) + if "processing" in state: + body = state["processing"] + if body is True: + out = {k: v for k, v in state.items() if k != "processing"} + elif isinstance(body, dict): + out = dict(body) + else: + raise RuntimeError(f"processing body must be object or true: {body!r}") out.setdefault("accepted", True) - out.setdefault("processing", True) + out["processing"] = True out.setdefault("trace_id", trace_id) return out raise RuntimeError(f"invalid task state for trace_id {trace_id}: {state!r}") @@ -302,33 +338,25 @@ def _balldomain_path(chip: str, balldomain: Optional[str]) -> Path: if not chip_root.is_dir(): raise FileNotFoundError(f"chip does not exist: {chip_root}") - chip_manifest = chip_root / "chip.toml" - if not chip_manifest.is_file(): - raise FileNotFoundError(f"chip manifest does not exist: {chip_manifest}") - chip_config = _load_toml(chip_manifest).get("chip") - if not isinstance(chip_config, dict): - raise ValueError(f"missing [chip] table: {chip_manifest}") - core_name = chip_config.get("compilerCore") - if not isinstance(core_name, str) or not core_name: - raise ValueError(f"chip manifest missing compilerCore: {chip_manifest}") + parse = REPO / "bazel" / "configparse" + if str(parse) not in sys.path: + sys.path.insert(0, str(parse)) + from chip_json import compiler_core, core_entry, load_topology + + data = load_topology(REPO, chip) + core_name = compiler_core(data) + if not core_name: + raise ValueError(f"chip {chip} has no unique topology core") + core = core_entry(data, core_name) core_root = REPO / "examples" / "cores" / core_name - core_manifest = core_root / "manifest.toml" - if not core_manifest.is_file(): - raise FileNotFoundError(f"core manifest does not exist: {core_manifest}") domains = core_root / "configs" / "balldomains" - core_manifest_config = _load_toml(core_manifest).get("core", {}).get("config") - if not isinstance(core_manifest_config, str) or not core_manifest_config: - raise ValueError(f"Core manifest missing core.config: {core_manifest}") - core_config = (core_root / core_manifest_config).resolve() if balldomain is None: - if not core_config.is_file(): - raise FileNotFoundError(f"missing {core_config}; pass balldomain= explicitly") - rel = _load_toml(core_config).get("balldomain") - if not isinstance(rel, str) or not rel: - raise ValueError(f"Core configuration missing balldomain=: {core_config}") - path = (core_config.parent / rel).resolve() + bd = core.get("balldomain") + if not isinstance(bd, dict) or not isinstance(bd.get("_file"), str): + raise ValueError(f"chip {chip} core {core_name} has no balldomain") + path = (REPO / bd["_file"]).resolve() else: raw = Path(balldomain) if raw.is_absolute(): @@ -354,15 +382,23 @@ def _balldomain_path(chip: str, balldomain: Optional[str]) -> Path: def _validate(path: Path) -> Dict[str, Any]: cfg = _load_toml(path) mappings = cfg.get("ballIdMappings", []) - isa = cfg.get("ballISA", []) - if not isinstance(mappings, list) or not isinstance(isa, list): - raise ValueError(f"{path}: ballIdMappings/ballISA must be arrays") + if not isinstance(mappings, list): + raise ValueError(f"{path}: ballIdMappings must be an array") + isa = cfg.get("ballISA") + if not isinstance(isa, list) or not isa: + raise ValueError(f"{path}: missing or empty ballISA") + for i, e in enumerate(isa): + if not isinstance(e, dict): + raise ValueError(f"{path}: ballISA[{i}] must be a table") + if not isinstance(e.get("mnemonic"), str) or not e.get("mnemonic"): + raise ValueError(f"{path}: ballISA[{i}].mnemonic invalid") + if not isinstance(e.get("funct7"), int) or e.get("funct7") < 0: + raise ValueError(f"{path}: ballISA[{i}].funct7 invalid") + if not isinstance(e.get("bid"), int) or e.get("bid") < 0: + raise ValueError(f"{path}: ballISA[{i}].bid invalid") ids = [m.get("ballId") for m in mappings] names = [m.get("ballName") for m in mappings] - funct7s = [e.get("funct7") for e in isa] - mnemonics = [e.get("mnemonic") for e in isa] - bids = [e.get("bid") for e in isa] id_set = set(ids) missing_config = [] @@ -380,11 +416,9 @@ def _validate(path: Path) -> Dict[str, Any]: ): bad_bw.append({"ballName": name, "inBW": in_bw, "outBW": out_bw}) cfg_rel = m.get("config") - if cfg_rel is None: - continue if not isinstance(cfg_rel, str) or not cfg_rel: missing_config.append( - {"ballName": name, "config": cfg_rel, "error": "empty"} + {"ballName": name, "config": cfg_rel, "error": "missing"} ) continue cfg_path = (path.parent / cfg_rel).resolve() @@ -392,6 +426,17 @@ def _validate(path: Path) -> Dict[str, Any]: missing_config.append( {"ballName": name, "config": cfg_rel, "resolved": str(cfg_path)} ) + continue + ball_cfg = _load_toml(cfg_path) + ball = ball_cfg.get("ball") + if not isinstance(ball, dict): + missing_config.append( + {"ballName": name, "config": cfg_rel, "error": "missing [ball]"} + ) + + funct7s = [e.get("funct7") for e in isa] + mnemonics = [e.get("mnemonic") for e in isa] + bids = [e.get("bid") for e in isa] orphan = sorted(id_set - set(bids)) unknown = sorted(set(bids) - id_set) diff --git a/bbdev/mcp/tests/test_common.py b/bbdev/mcp/tests/test_common.py index 95c31ae94..70b05e46e 100644 --- a/bbdev/mcp/tests/test_common.py +++ b/bbdev/mcp/tests/test_common.py @@ -1,7 +1,7 @@ import sys import unittest from pathlib import Path -from unittest.mock import patch +from unittest.mock import mock_open, patch sys.path.insert(0, str(Path(__file__).resolve().parents[1])) @@ -13,8 +13,9 @@ def setUp(self): common._submitted_trace_ids.clear() @patch("common._http") + @patch("common._assert_workspace_workers") @patch("common._ensure", return_value=5100) - def test_submit_returns_trace_without_reading_state(self, ensure, http): + def test_submit_returns_trace_without_reading_state(self, ensure, workers, http): http.return_value = 202, {"trace_id": "trace-1"} result = common.submit("/workload/build", {"chip": "pebble"}) @@ -29,6 +30,7 @@ def test_submit_returns_trace_without_reading_state(self, ensure, http): }, ) ensure.assert_called_once_with() + workers.assert_called_once_with() http.assert_called_once_with( "POST", "http://127.0.0.1:5100/workload/build", @@ -55,6 +57,50 @@ def test_task_status_rejects_unknown_trace(self, read_state): with self.assertRaisesRegex(RuntimeError, "unknown task trace_id"): common.task_status("unknown") + +class EnsureTest(unittest.TestCase): + def setUp(self): + self.proc = common._proc + self.port = common._port + self.log_fh = common._log_fh + common._proc = None + common._port = None + common._log_fh = None + + def tearDown(self): + common._proc = self.proc + common._port = self.port + common._log_fh = self.log_fh + + def test_ensure_starts_server_in_nix_environment(self): + with ( + patch("common.shutil.which", return_value="/usr/bin/nix"), + patch("common._free_port", return_value=5199), + patch("common._ready", return_value=True), + patch("common._assert_workspace_workers"), + patch("common.open", mock_open()), + patch("common.subprocess.Popen") as popen, + ): + popen.return_value.poll.return_value = None + self.assertEqual(common._ensure(), 5199) + + popen.assert_called_once_with( + [ + "nix", + "develop", + "--command", + str(common.BBDEV), + "start", + "--server", + "--port", + "5199", + ], + cwd=str(common.BBDEV.parent), + stdout=common._log_fh, + stderr=common._log_fh, + start_new_session=True, + ) + @patch("common._read_state") def test_task_status_reports_success(self, read_state): read_state.return_value = { @@ -89,6 +135,27 @@ def test_task_status_reports_failure(self, read_state): }, ) + @patch("common._read_state") + def test_task_status_reports_nested_processing(self, read_state): + read_state.return_value = { + "processing": { + "processing": True, + "task": "regression.buildbitstream", + "chip": "pebble", + } + } + + self.assertEqual( + common.task_status("trace-1"), + { + "processing": True, + "task": "regression.buildbitstream", + "chip": "pebble", + "accepted": True, + "trace_id": "trace-1", + }, + ) + @patch("common._read_state", return_value={"unknown": True}) def test_task_status_rejects_invalid_state(self, read_state): with self.assertRaisesRegex(RuntimeError, "invalid task state"): diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index 775c08e0c..3656d0725 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -9,6 +9,7 @@ from . import workload_clean from . import workload_build from . import bemu_sim +from . import bemu_analysis from . import bemu_batch from . import bebop_verilator_clean from . import bebop_verilator_verilog @@ -41,6 +42,10 @@ from . import yosys_run from . import yosys_verilog from . import yosys_synth +from . import regression_buildbitstream +from . import regression_check +from . import regression_eval_performance +from . import regression_eval_area_power MODULES = [ validate, @@ -50,6 +55,7 @@ workload_clean, workload_build, bemu_sim, + bemu_analysis, bemu_batch, bebop_verilator_clean, bebop_verilator_verilog, @@ -82,6 +88,10 @@ yosys_run, yosys_verilog, yosys_synth, + regression_buildbitstream, + regression_check, + regression_eval_performance, + regression_eval_area_power, ] diff --git a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py index c28d48b4e..faacd4172 100644 --- a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py +++ b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py @@ -10,17 +10,16 @@ def register(mcp): @mcp.tool() def bbdev_bebop_p2e_buildbitstream( - config: str, + chip: str, vsrc_dir: Optional[str] = None, output_dir: Optional[str] = None, ) -> str: """Build bebop-p2e bitstream. POST /bebop/p2e/buildbitstream.""" - if e := need("config", config): + if e := need("chip", chip): return err(e) return fmt( submit( "/bebop/p2e/buildbitstream", - opt({"config": config}, vsrc_dir=vsrc_dir, output_dir=output_dir), + opt({"chip": chip}, vsrc_dir=vsrc_dir, output_dir=output_dir), ) ) - diff --git a/bbdev/mcp/tools/bebop_p2e_clean.py b/bbdev/mcp/tools/bebop_p2e_clean.py index 9ad741f92..63e8148a8 100644 --- a/bbdev/mcp/tools/bebop_p2e_clean.py +++ b/bbdev/mcp/tools/bebop_p2e_clean.py @@ -8,9 +8,8 @@ def register(mcp): @mcp.tool() - def bbdev_bebop_p2e_clean(config: str) -> str: + def bbdev_bebop_p2e_clean(chip: str) -> str: """Clean bebop-p2e build. POST /bebop/p2e/clean.""" - if e := need("config", config): + if e := need("chip", chip): return err(e) - return fmt(submit("/bebop/p2e/clean", {"config": config})) - + return fmt(submit("/bebop/p2e/clean", {"chip": chip})) diff --git a/bbdev/mcp/tools/bebop_p2e_verilog.py b/bbdev/mcp/tools/bebop_p2e_verilog.py index 5e09de66a..b3e35b600 100644 --- a/bbdev/mcp/tools/bebop_p2e_verilog.py +++ b/bbdev/mcp/tools/bebop_p2e_verilog.py @@ -10,15 +10,14 @@ def register(mcp): @mcp.tool() def bbdev_bebop_p2e_verilog( - config: str, output_dir: Optional[str] = None + chip: str, output_dir: Optional[str] = None ) -> str: """Generate Verilog for bebop-p2e. POST /bebop/p2e/verilog.""" - if e := need("config", config): + if e := need("chip", chip): return err(e) return fmt( submit( "/bebop/p2e/verilog", - opt({"config": config}, output_dir=output_dir), + opt({"chip": chip}, output_dir=output_dir), ) ) - diff --git a/bbdev/mcp/tools/bebop_verilator_batch.py b/bbdev/mcp/tools/bebop_verilator_batch.py index d7ca136d0..507ee051d 100644 --- a/bbdev/mcp/tools/bebop_verilator_batch.py +++ b/bbdev/mcp/tools/bebop_verilator_batch.py @@ -10,16 +10,14 @@ def register(mcp): @mcp.tool() def bbdev_bebop_verilator_batch( chip: str, - config: str, test: str, clean_before: bool = False, rushB: bool = False, diff: bool = False, ) -> str: """Batch bebop-verilator regression. POST /bebop/verilator/batch.""" - for n, v in (("chip", chip), ("config", config)): - if e := need(n, v): - return err(e) + if e := need("chip", chip): + return err(e) if test not in ("elf-tests", "pk-tests"): return err("test must be elf-tests or pk-tests") return fmt( @@ -27,7 +25,6 @@ def bbdev_bebop_verilator_batch( "/bebop/verilator/batch", { "chip": chip, - "config": config, "test": test, "clean-before": clean_before, "rushB": rushB, diff --git a/bbdev/mcp/tools/bebop_verilator_build.py b/bbdev/mcp/tools/bebop_verilator_build.py index 4d6a1e478..b61e1a35d 100644 --- a/bbdev/mcp/tools/bebop_verilator_build.py +++ b/bbdev/mcp/tools/bebop_verilator_build.py @@ -9,14 +9,14 @@ def register(mcp): @mcp.tool() def bbdev_bebop_verilator_build( - config: str, jobs: int = 16, diff: bool = False + chip: str, jobs: int = 16, diff: bool = False ) -> str: """Build bebop verilator binary. POST /bebop/verilator/build.""" - if e := need("config", config): + if e := need("chip", chip): return err(e) return fmt( submit( "/bebop/verilator/build", - {"config": config, "jobs": jobs, "diff": diff}, + {"chip": chip, "jobs": jobs, "diff": diff}, ) ) diff --git a/bbdev/mcp/tools/bebop_verilator_clean.py b/bbdev/mcp/tools/bebop_verilator_clean.py index a2d93970d..7660ae14e 100644 --- a/bbdev/mcp/tools/bebop_verilator_clean.py +++ b/bbdev/mcp/tools/bebop_verilator_clean.py @@ -8,9 +8,8 @@ def register(mcp): @mcp.tool() - def bbdev_bebop_verilator_clean(config: str) -> str: + def bbdev_bebop_verilator_clean(chip: str) -> str: """Clean bebop-verilator build. POST /bebop/verilator/clean.""" - if e := need("config", config): + if e := need("chip", chip): return err(e) - return fmt(submit("/bebop/verilator/clean", {"config": config})) - + return fmt(submit("/bebop/verilator/clean", {"chip": chip})) diff --git a/bbdev/mcp/tools/bebop_verilator_run.py b/bbdev/mcp/tools/bebop_verilator_run.py index 7680c28b1..5cf59e75b 100644 --- a/bbdev/mcp/tools/bebop_verilator_run.py +++ b/bbdev/mcp/tools/bebop_verilator_run.py @@ -10,7 +10,7 @@ def register(mcp): @mcp.tool() def bbdev_bebop_verilator_run( binary: str, - config: str, + chip: str, jobs: int = 16, itrace: bool = False, mtrace: bool = False, @@ -23,7 +23,7 @@ def bbdev_bebop_verilator_run( batch: bool = False, ) -> str: """Full bebop-verilator flow. POST /bebop/verilator/run.""" - for n, v in (("binary", binary), ("config", config)): + for n, v in (("binary", binary), ("chip", chip)): if e := need(n, v): return err(e) return fmt( @@ -31,7 +31,7 @@ def bbdev_bebop_verilator_run( "/bebop/verilator/run", { "binary": binary, - "config": config, + "chip": chip, "jobs": jobs, "itrace": itrace, "mtrace": mtrace, diff --git a/bbdev/mcp/tools/bebop_verilator_sim.py b/bbdev/mcp/tools/bebop_verilator_sim.py index 31765dbc8..59f2b84e6 100644 --- a/bbdev/mcp/tools/bebop_verilator_sim.py +++ b/bbdev/mcp/tools/bebop_verilator_sim.py @@ -11,7 +11,7 @@ def register(mcp): @mcp.tool() def bbdev_bebop_verilator_sim( binary: str, - config: str, + chip: str, itrace: bool = False, mtrace: bool = False, pmctrace: bool = False, @@ -22,12 +22,12 @@ def bbdev_bebop_verilator_sim( batch: bool = False, ) -> str: """Run one workload on bebop-verilator. POST /bebop/verilator/sim.""" - for n, v in (("binary", binary), ("config", config)): + for n, v in (("binary", binary), ("chip", chip)): if e := need(n, v): return err(e) params: Dict[str, Any] = { "binary": binary, - "config": config, + "chip": chip, "itrace": itrace, "mtrace": mtrace, "pmctrace": pmctrace, diff --git a/bbdev/mcp/tools/bebop_verilator_verilog.py b/bbdev/mcp/tools/bebop_verilator_verilog.py index 31bc509b2..959a80d83 100644 --- a/bbdev/mcp/tools/bebop_verilator_verilog.py +++ b/bbdev/mcp/tools/bebop_verilator_verilog.py @@ -8,9 +8,8 @@ def register(mcp): @mcp.tool() - def bbdev_bebop_verilator_verilog(config: str) -> str: + def bbdev_bebop_verilator_verilog(chip: str) -> str: """Generate Verilog for bebop-verilator. POST /bebop/verilator/verilog.""" - if e := need("config", config): + if e := need("chip", chip): return err(e) - return fmt(submit("/bebop/verilator/verilog", {"config": config})) - + return fmt(submit("/bebop/verilator/verilog", {"chip": chip})) diff --git a/bbdev/mcp/tools/bemu_analysis.py b/bbdev/mcp/tools/bemu_analysis.py new file mode 100644 index 000000000..7189fb0dd --- /dev/null +++ b/bbdev/mcp/tools/bemu_analysis.py @@ -0,0 +1,37 @@ +"""MCP tool: bbdev_bemu_analysis.""" + +from __future__ import annotations + +from pathlib import Path + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_bemu_analysis( + chip: str, + log_dir: str, + itrace: bool = False, + mtrace: bool = False, + ) -> str: + """Analyze existing bebop-bemu traces. POST /bebop/bemu/analysis.""" + if e := need("chip", chip): + return err(e) + if e := need("log_dir", log_dir): + return err(e) + if not Path(log_dir).is_absolute(): + return err(f"log_dir must be an absolute path: {log_dir}") + if not itrace and not mtrace: + return err("need itrace and/or mtrace") + return fmt( + submit( + "/bebop/bemu/analysis", + { + "chip": chip, + "log-dir": log_dir, + "itrace": itrace, + "mtrace": mtrace, + }, + ) + ) diff --git a/bbdev/mcp/tools/bemu_sim.py b/bbdev/mcp/tools/bemu_sim.py index 79f061cfd..1f5b59270 100644 --- a/bbdev/mcp/tools/bemu_sim.py +++ b/bbdev/mcp/tools/bemu_sim.py @@ -15,6 +15,8 @@ def bbdev_bemu_sim( pk: bool = False, disasm: bool = False, tool_profile: bool = False, + itrace: bool = False, + mtrace: bool = False, rushB: bool = False, ) -> str: """Run one workload on bebop-bemu. POST /bebop/bemu/sim.""" @@ -27,6 +29,8 @@ def bbdev_bemu_sim( "pk": pk, "disasm": disasm, "tool-profile": tool_profile, + "itrace": itrace, + "mtrace": mtrace, "rushB": rushB, } return fmt(submit("/bebop/bemu/sim", params)) diff --git a/bbdev/mcp/tools/compiler_build.py b/bbdev/mcp/tools/compiler_build.py index d6180eb69..faa82efcd 100644 --- a/bbdev/mcp/tools/compiler_build.py +++ b/bbdev/mcp/tools/compiler_build.py @@ -3,16 +3,22 @@ from __future__ import annotations -from common import submit, err, fmt, need, opt +from typing import Any, Dict, Optional + +from common import submit, err, fmt def register(mcp): @mcp.tool() - def bbdev_compiler_build(chip: str, stable: bool = False) -> str: - """Build buddy-mlir compiler for a chip. POST /compiler/build.""" - if e := need("chip", chip): - return err(e) - return fmt( - submit("/compiler/build", {"chip": chip, "stable": stable}) - ) - + def bbdev_compiler_build( + core: Optional[str] = None, chip: Optional[str] = None, stable: bool = False + ) -> str: + """Build buddy-mlir compiler for exactly one Core or chip. POST /compiler/build.""" + if bool(core) == bool(chip): + return err("specify exactly one compiler target: core or chip") + params: Dict[str, Any] = {"stable": stable} + if core: + params["core"] = core + else: + params["chip"] = chip + return fmt(submit("/compiler/build", params)) diff --git a/bbdev/mcp/tools/dc_area.py b/bbdev/mcp/tools/dc_area.py index dfd2342ac..cef17d4ed 100644 --- a/bbdev/mcp/tools/dc_area.py +++ b/bbdev/mcp/tools/dc_area.py @@ -9,8 +9,8 @@ def register(mcp): @mcp.tool() - def bbdev_dc_area(config: Optional[str] = None, top: Optional[str] = None) -> str: + def bbdev_dc_area(chip: Optional[str] = None, top: Optional[str] = None) -> str: """Run DC synthesis and area reporting. Defaults to DigitalTop.""" - if not config: - return err("config is required") - return fmt(submit("/dc/area", opt({}, config=config, top=top))) + if not chip: + return err("chip is required") + return fmt(submit("/dc/area", opt({}, chip=chip, top=top))) diff --git a/bbdev/mcp/tools/dc_power.py b/bbdev/mcp/tools/dc_power.py index c7c3f46f8..961cac1e1 100644 --- a/bbdev/mcp/tools/dc_power.py +++ b/bbdev/mcp/tools/dc_power.py @@ -10,7 +10,7 @@ def register(mcp): @mcp.tool() def bbdev_dc_power( - config: Optional[str] = None, + chip: Optional[str] = None, top: Optional[str] = None, activity: Optional[str] = None, activity_format: Optional[str] = None, @@ -24,8 +24,8 @@ def bbdev_dc_power( Omit activity/activity_format for the normal rerun-and-measure flow. Supply both only as a debugging override. """ - if not config: - return err("config is required") + if not chip: + return err("chip is required") if bool(activity) != bool(activity_format): return err("activity and activity_format must be provided together") return fmt( @@ -33,7 +33,7 @@ def bbdev_dc_power( "/dc/power", opt( {}, - config=config, + chip=chip, top=top, activity=activity, format=activity_format, diff --git a/bbdev/mcp/tools/dc_verilog.py b/bbdev/mcp/tools/dc_verilog.py index 44c98f0cb..df93fe7b8 100644 --- a/bbdev/mcp/tools/dc_verilog.py +++ b/bbdev/mcp/tools/dc_verilog.py @@ -10,15 +10,15 @@ def register(mcp): @mcp.tool() def bbdev_dc_verilog( - config: Optional[str] = None, + chip: Optional[str] = None, top: Optional[str] = None, ) -> str: """Generate top-scoped DC RTL collateral. Defaults to DigitalTop.""" - if not config: - return err("config is required") + if not chip: + return err("chip is required") return fmt( submit( "/dc/verilog", - opt({}, config=config, top=top), + opt({}, chip=chip, top=top), ) ) diff --git a/bbdev/mcp/tools/kernel_build.py b/bbdev/mcp/tools/kernel_build.py index 1c5cb2126..6b10291d6 100644 --- a/bbdev/mcp/tools/kernel_build.py +++ b/bbdev/mcp/tools/kernel_build.py @@ -18,9 +18,11 @@ def bbdev_kernel_build( ) -> str: """Build RISC-V kernel + rootfs. POST /kernel/build. - With --chip, uses examples/chips//kernel OS overlay and packs + With --chip only: uses examples/chips//kernel OS overlay and packs that chip's bemu workloads-pk.toml into fw_payload--pk. - With --interactive, keep shared /init shell and do not auto-run. + With --model: requires --chip; packs ModelTest runtime from + archs/buckyball// into fw_payload-. + With --interactive: keep shared /init shell and do not auto-run. """ params: Dict[str, Any] = {} opt(params, chip=chip, model=model) diff --git a/bbdev/mcp/tools/regression_buildbitstream.py b/bbdev/mcp/tools/regression_buildbitstream.py new file mode 100644 index 000000000..091632c51 --- /dev/null +++ b/bbdev/mcp/tools/regression_buildbitstream.py @@ -0,0 +1,24 @@ +"""MCP tool: bbdev_regression_buildbitstream.""" + +from __future__ import annotations + +from typing import Optional + +from common import submit, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_regression_buildbitstream( + chip: str, + output_dir: Optional[str] = None, + ) -> str: + """Build p2e bitstream. POST /regression/buildbitstream.""" + if e := need("chip", chip): + return err(e) + return fmt( + submit( + "/regression/buildbitstream", + opt({"chip": chip}, output_dir=output_dir), + ) + ) diff --git a/bbdev/mcp/tools/regression_check.py b/bbdev/mcp/tools/regression_check.py new file mode 100644 index 000000000..9acee1625 --- /dev/null +++ b/bbdev/mcp/tools/regression_check.py @@ -0,0 +1,24 @@ +"""MCP tool: bbdev_regression_check.""" + +from __future__ import annotations + + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_regression_check( + chip: str, + bitstream: str, + ) -> str: + """P2E pk-tests. POST /regression/check.""" + for n, v in (("chip", chip), ("bitstream", bitstream)): + if e := need(n, v): + return err(e) + return fmt( + submit( + "/regression/check", + {"chip": chip, "bitstream": bitstream}, + ) + ) diff --git a/bbdev/mcp/tools/regression_eval_area_power.py b/bbdev/mcp/tools/regression_eval_area_power.py new file mode 100644 index 000000000..a402b947b --- /dev/null +++ b/bbdev/mcp/tools/regression_eval_area_power.py @@ -0,0 +1,24 @@ +"""MCP tool: bbdev_regression_eval_area_power.""" + +from __future__ import annotations + +from typing import Optional + +from common import submit, err, fmt, need, opt + + +def register(mcp): + @mcp.tool() + def bbdev_regression_eval_area_power( + chip: str, + top: Optional[str] = None, + ) -> str: + """DC area+freq. POST /regression/eval-area-power.""" + if e := need("chip", chip): + return err(e) + return fmt( + submit( + "/regression/eval-area-power", + opt({"chip": chip}, top=top), + ) + ) diff --git a/bbdev/mcp/tools/regression_eval_performance.py b/bbdev/mcp/tools/regression_eval_performance.py new file mode 100644 index 000000000..71bd1740c --- /dev/null +++ b/bbdev/mcp/tools/regression_eval_performance.py @@ -0,0 +1,24 @@ +"""MCP tool: bbdev_regression_eval_performance.""" + +from __future__ import annotations + + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_regression_eval_performance( + chip: str, + bitstream: str, + ) -> str: + """P2E model perfetto latency. POST /regression/eval-performance.""" + for n, v in (("chip", chip), ("bitstream", bitstream)): + if e := need(n, v): + return err(e) + return fmt( + submit( + "/regression/eval-performance", + {"chip": chip, "bitstream": bitstream}, + ) + ) diff --git a/bbdev/mcp/tools/uvm_build.py b/bbdev/mcp/tools/uvm_build.py index 9a5a51afc..d56a14a8d 100644 --- a/bbdev/mcp/tools/uvm_build.py +++ b/bbdev/mcp/tools/uvm_build.py @@ -10,15 +10,16 @@ def register(mcp): @mcp.tool() def bbdev_uvm_build( - config: str, ball: Optional[str] = None, filelist: Optional[str] = None + config: str, core_config: str, ball: Optional[str] = None, filelist: Optional[str] = None ) -> str: """Build a Ball UVM simulation. POST /uvm/build.""" if e := need("config", config): return err(e) - params: Dict[str, Any] = {"config": config} + if e := need("core_config", core_config): + return err(e) + params: Dict[str, Any] = {"config": config, "core_config": core_config} if ball: params["ball"] = ball if filelist: params["filelist"] = filelist return fmt(submit("/uvm/build", params)) - diff --git a/bbdev/mcp/tools/verilator_build.py b/bbdev/mcp/tools/verilator_build.py index 4961f60d8..6aa35d3ed 100644 --- a/bbdev/mcp/tools/verilator_build.py +++ b/bbdev/mcp/tools/verilator_build.py @@ -8,14 +8,14 @@ def register(mcp): @mcp.tool() def bbdev_verilator_build( - config: str, + chip: str, jobs: int = 16, coverage: bool = False, ) -> str: """Build non-bebop Verilator sim binary. POST /verilator/build.""" - if e := need("config", config): + if e := need("chip", chip): return err(e) - params = {"config": config, "jobs": jobs} + params = {"chip": chip, "jobs": jobs} if coverage: params["coverage"] = True return fmt(submit("/verilator/build", params)) diff --git a/bbdev/mcp/tools/verilator_clean.py b/bbdev/mcp/tools/verilator_clean.py index c56c5308a..89fada8e0 100644 --- a/bbdev/mcp/tools/verilator_clean.py +++ b/bbdev/mcp/tools/verilator_clean.py @@ -7,8 +7,8 @@ def register(mcp): @mcp.tool() - def bbdev_verilator_clean(config: str) -> str: + def bbdev_verilator_clean(chip: str) -> str: """Clean non-bebop Verilator build. POST /verilator/clean.""" - if e := need("config", config): + if e := need("chip", chip): return err(e) - return fmt(submit("/verilator/clean", {"config": config})) + return fmt(submit("/verilator/clean", {"chip": chip})) diff --git a/bbdev/mcp/tools/verilator_run.py b/bbdev/mcp/tools/verilator_run.py index 865e05fd2..918766e03 100644 --- a/bbdev/mcp/tools/verilator_run.py +++ b/bbdev/mcp/tools/verilator_run.py @@ -11,7 +11,7 @@ def register(mcp): @mcp.tool() def bbdev_verilator_run( binary: str, - config: str, + chip: str, batch: bool = True, coverage: bool = False, jobs: Optional[int] = None, @@ -24,11 +24,11 @@ def bbdev_verilator_run( """Full non-bebop Verilator flow. POST /verilator/run.""" if e := need("binary", binary): return err(e) - if e := need("config", config): + if e := need("chip", chip): return err(e) params: Dict[str, Any] = { "binary": binary, - "config": config, + "chip": chip, "batch": batch, "coverage": coverage, "itrace": itrace, diff --git a/bbdev/mcp/tools/verilator_sim.py b/bbdev/mcp/tools/verilator_sim.py index 05a830a96..fc40ef799 100644 --- a/bbdev/mcp/tools/verilator_sim.py +++ b/bbdev/mcp/tools/verilator_sim.py @@ -11,7 +11,7 @@ def register(mcp): @mcp.tool() def bbdev_verilator_sim( binary: str, - config: str, + chip: str, batch: bool = True, coverage: bool = False, itrace: bool = False, @@ -21,12 +21,12 @@ def bbdev_verilator_sim( banktrace: bool = False, ) -> str: """Run one workload on non-bebop Verilator. POST /verilator/sim.""" - for n, v in (("binary", binary), ("config", config)): + for n, v in (("binary", binary), ("chip", chip)): if e := need(n, v): return err(e) params: Dict[str, Any] = { "binary": binary, - "config": config, + "chip": chip, "batch": batch, "itrace": itrace, "mtrace": mtrace, diff --git a/bbdev/mcp/tools/verilator_verilog.py b/bbdev/mcp/tools/verilator_verilog.py index beff8676b..5ce8bdcd1 100644 --- a/bbdev/mcp/tools/verilator_verilog.py +++ b/bbdev/mcp/tools/verilator_verilog.py @@ -7,8 +7,8 @@ def register(mcp): @mcp.tool() - def bbdev_verilator_verilog(config: str) -> str: + def bbdev_verilator_verilog(chip: str) -> str: """Generate Verilog for non-bebop Verilator. POST /verilator/verilog.""" - if e := need("config", config): + if e := need("chip", chip): return err(e) - return fmt(submit("/verilator/verilog", {"config": config})) + return fmt(submit("/verilator/verilog", {"chip": chip})) diff --git a/bbdev/mcp/tools/workload_clean.py b/bbdev/mcp/tools/workload_clean.py index 74f8c9e43..35000f2af 100644 --- a/bbdev/mcp/tools/workload_clean.py +++ b/bbdev/mcp/tools/workload_clean.py @@ -2,13 +2,11 @@ from __future__ import annotations - -from common import submit, fmt, opt +from common import submit, fmt def register(mcp): @mcp.tool() - def bbdev_workload_clean() -> str: - """Clean workload artifacts. POST /workload/clean.""" - return fmt(submit("/workload/clean", {})) - + def bbdev_workload_clean(chip: str) -> str: + """Clean workload artifacts for one chip. POST /workload/clean.""" + return fmt(submit("/workload/clean", {"chip": chip})) diff --git a/bbdev/mcp/tools/workload_tohex.py b/bbdev/mcp/tools/workload_tohex.py index 9f30ddd1e..1aab7a5db 100644 --- a/bbdev/mcp/tools/workload_tohex.py +++ b/bbdev/mcp/tools/workload_tohex.py @@ -2,13 +2,11 @@ from __future__ import annotations - -from common import submit, fmt, opt +from common import submit, fmt def register(mcp): @mcp.tool() - def bbdev_workload_tohex() -> str: - """Convert workload ELF to hex. POST /workload/tohex.""" - return fmt(submit("/workload/tohex", {})) - + def bbdev_workload_tohex(chip: str) -> str: + """Convert chip workload ELF to hex. POST /workload/tohex.""" + return fmt(submit("/workload/tohex", {"chip": chip})) diff --git a/bbdev/mcp/tools/yosys_run.py b/bbdev/mcp/tools/yosys_run.py index 2b71d6770..c53a9f1c6 100644 --- a/bbdev/mcp/tools/yosys_run.py +++ b/bbdev/mcp/tools/yosys_run.py @@ -4,26 +4,27 @@ from typing import Optional -from common import submit, fmt, opt +from common import submit, err, fmt, need, opt def register(mcp): @mcp.tool() def bbdev_yosys_run( + chip: str, top: Optional[str] = None, - config: Optional[str] = None, output_dir: Optional[str] = None, log_dir: Optional[str] = None, vcd: Optional[str] = None, ) -> str: """Full yosys flow. POST /yosys/run.""" + if e := need("chip", chip): + return err(e) return fmt( submit( "/yosys/run", opt( - {}, + {"chip": chip}, top=top, - config=config, output_dir=output_dir, log_dir=log_dir, vcd=vcd, diff --git a/bbdev/mcp/tools/yosys_synth.py b/bbdev/mcp/tools/yosys_synth.py index 8e67f3407..944fdb2fc 100644 --- a/bbdev/mcp/tools/yosys_synth.py +++ b/bbdev/mcp/tools/yosys_synth.py @@ -4,26 +4,27 @@ from typing import Optional -from common import submit, fmt, opt +from common import submit, err, fmt, need, opt def register(mcp): @mcp.tool() def bbdev_yosys_synth( + chip: str, top: Optional[str] = None, - config: Optional[str] = None, output_dir: Optional[str] = None, log_dir: Optional[str] = None, vcd: Optional[str] = None, ) -> str: """Yosys synthesis + OpenSTA. POST /yosys/synth.""" + if e := need("chip", chip): + return err(e) return fmt( submit( "/yosys/synth", opt( - {}, + {"chip": chip}, top=top, - config=config, output_dir=output_dir, log_dir=log_dir, vcd=vcd, diff --git a/bbdev/mcp/tools/yosys_verilog.py b/bbdev/mcp/tools/yosys_verilog.py index 05eae8b4c..d6d926f5f 100644 --- a/bbdev/mcp/tools/yosys_verilog.py +++ b/bbdev/mcp/tools/yosys_verilog.py @@ -4,28 +4,28 @@ from typing import Optional -from common import submit, fmt, opt +from common import submit, err, fmt, need, opt def register(mcp): @mcp.tool() def bbdev_yosys_verilog( + chip: str, top: Optional[str] = None, - config: Optional[str] = None, output_dir: Optional[str] = None, log_dir: Optional[str] = None, ) -> str: """Generate Verilog for yosys flow. POST /yosys/verilog.""" + if e := need("chip", chip): + return err(e) return fmt( submit( "/yosys/verilog", opt( - {}, + {"chip": chip}, top=top, - config=config, output_dir=output_dir, log_dir=log_dir, ), ) ) - From 4dbf61c063274fbb0c3dc3ca950def2c684295ac Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 25 Aug 2026 23:53:02 +0800 Subject: [PATCH 125/196] refactor: streamline Bebop and Verilator workflows by consolidating chip requirement handling and updating directory management --- bbdev/api/config.yaml | 2 + bbdev/api/steps/bebop/bemu/02_sim_api.step.py | 4 +- .../api/steps/bebop/bemu/02_sim_event.step.py | 15 +- .../api/steps/bebop/bemu/03_batch_api.step.py | 4 +- .../steps/bebop/bemu/03_batch_event.step.py | 3 +- .../steps/bebop/bemu/scripts/bemu_analysis.py | 60 +- .../steps/bebop/bemu/scripts/bemu_common.py | 48 +- .../api/steps/bebop/p2e/00_clean_api.step.py | 2 +- .../steps/bebop/p2e/00_clean_event.step.py | 6 +- .../steps/bebop/p2e/01_verilog_api.step.py | 2 +- .../bebop/p2e/02_buildbitstream_api.step.py | 6 +- .../bebop/p2e/02_buildbitstream_event.step.py | 15 +- .../bebop/p2e/03_runworkload_event.step.py | 28 +- .../steps/bebop/p2e/04_batch_event.step.py | 18 +- .../bebop/verilator/01_clean_api.step.py | 6 +- .../bebop/verilator/01_clean_event.step.py | 6 +- .../bebop/verilator/02_verilog_api.step.py | 56 +- .../bebop/verilator/03_build_api.step.py | 9 +- .../bebop/verilator/03_build_event.step.py | 101 +-- .../steps/bebop/verilator/04_sim_api.step.py | 2 +- .../bebop/verilator/04_sim_event.step.py | 41 +- .../steps/bebop/verilator/05_run_api.step.py | 3 +- .../bebop/verilator/06_batch_api.step.py | 6 +- .../bebop/verilator/06_batch_event.step.py | 31 +- bbdev/api/steps/compiler/01_build_api.step.py | 40 +- .../api/steps/compiler/01_build_event.step.py | 65 +- bbdev/api/steps/compiler/README.md | 41 -- bbdev/api/steps/compiler/scripts/build.py | 181 +++++ bbdev/api/steps/config/.gitignore | 1 + bbdev/api/steps/config/01_install_api.step.py | 22 + .../api/steps/config/01_install_event.step.py | 155 +++++ bbdev/api/steps/config/scripts/1_toml2json.py | 122 ++++ .../config/scripts/2_parameter_derivation.py | 645 ++++++++++++++++++ .../api/steps/config/scripts/3_json2proto.py | 283 ++++++++ bbdev/api/steps/config/scripts/4_install.py | 148 ++++ .../api/steps/config/scripts/proto/chip.proto | 245 +++++++ bbdev/api/steps/dc/01_area_api.step.py | 12 +- bbdev/api/steps/dc/02_verilog_api.step.py | 28 +- bbdev/api/steps/dc/04_power_api.step.py | 9 +- .../mill/01_verilator_verilog_event.step.py | 88 ++- .../02_vcs_verilog_event.step.py} | 31 +- .../steps/mill/03_yosys_verilog_event.step.py | 36 +- .../steps/mill/04_dc_verilog_event.step.py | 39 +- .../06_bebop_verilator_verilog_event.step.py | 96 +-- .../mill/07_bebop_p2e_verilog_event.step.py | 31 +- bbdev/api/steps/mill/scripts/mill.py | 18 + .../regression/00_buildbitstream_api.step.py | 2 +- .../00_buildbitstream_event.step.py | 2 +- .../02_eval_performance_event.step.py | 30 +- .../regression/03_eval_area_power_api.step.py | 24 +- .../03_eval_area_power_event.step.py | 2 +- .../api/steps/uvm/00_ball_verilog_api.step.py | 39 ++ .../steps/uvm/00_ball_verilog_event.step.py | 80 +++ bbdev/api/steps/vcs/01_clean_api.step.py | 6 +- bbdev/api/steps/vcs/01_clean_event.step.py | 4 +- bbdev/api/steps/vcs/02_verilog_api.step.py | 2 +- bbdev/api/steps/vcs/03_build_api.step.py | 6 +- bbdev/api/steps/vcs/03_build_event.step.py | 6 +- bbdev/api/steps/vcs/04_sim_api.step.py | 2 +- bbdev/api/steps/vcs/04_sim_event.step.py | 24 +- bbdev/api/steps/vcs/05_run_api.step.py | 6 +- bbdev/api/steps/vcs/README.md | 3 + .../api/steps/verilator/01_clean_api.step.py | 2 +- .../steps/verilator/01_clean_event.step.py | 6 +- .../steps/verilator/02_verilog_api.step.py | 58 +- .../api/steps/verilator/03_build_api.step.py | 6 +- .../steps/verilator/03_build_event.step.py | 23 +- bbdev/api/steps/verilator/04_sim_api.step.py | 2 +- .../api/steps/verilator/04_sim_event.step.py | 64 +- bbdev/api/steps/verilator/05_run_api.step.py | 2 +- bbdev/api/steps/workload/00_clean_api.step.py | 2 +- .../api/steps/workload/00_clean_event.step.py | 2 +- .../api/steps/workload/01_build_event.step.py | 6 +- bbdev/api/steps/workload/02_tohex_api.step.py | 2 +- .../api/steps/workload/02_tohex_event.step.py | 2 +- bbdev/api/steps/yosys/01_run_api.step.py | 2 +- bbdev/api/steps/yosys/02_verilog_api.step.py | 2 +- bbdev/api/steps/yosys/03_synth_api.step.py | 6 +- bbdev/api/steps/yosys/03_synth_event.step.py | 8 +- .../test_api_regression_buildbitstream.py | 4 +- bbdev/api/tests/test_api_regression_check.py | 2 +- ...t_api_regression_eval_performance_event.py | 2 +- bbdev/api/utils/build.py | 371 ---------- bbdev/api/utils/chip.py | 136 ---- bbdev/api/utils/event_common.py | 18 + bbdev/api/utils/path.py | 185 +---- bbdev/api/utils/rtl.py | 122 ---- bbdev/bbdev | 8 +- bbdev/mcp/common.py | 44 +- bbdev/mcp/tools/compiler_build.py | 19 +- 90 files changed, 2543 insertions(+), 1611 deletions(-) delete mode 100644 bbdev/api/steps/compiler/README.md create mode 100644 bbdev/api/steps/compiler/scripts/build.py create mode 100644 bbdev/api/steps/config/.gitignore create mode 100644 bbdev/api/steps/config/01_install_api.step.py create mode 100644 bbdev/api/steps/config/01_install_event.step.py create mode 100644 bbdev/api/steps/config/scripts/1_toml2json.py create mode 100644 bbdev/api/steps/config/scripts/2_parameter_derivation.py create mode 100644 bbdev/api/steps/config/scripts/3_json2proto.py create mode 100644 bbdev/api/steps/config/scripts/4_install.py create mode 100644 bbdev/api/steps/config/scripts/proto/chip.proto rename bbdev/api/steps/{vcs/02_verilog_event.step.py => mill/02_vcs_verilog_event.step.py} (62%) create mode 100644 bbdev/api/steps/mill/scripts/mill.py create mode 100644 bbdev/api/steps/uvm/00_ball_verilog_api.step.py create mode 100644 bbdev/api/steps/uvm/00_ball_verilog_event.step.py delete mode 100644 bbdev/api/utils/build.py delete mode 100644 bbdev/api/utils/chip.py delete mode 100644 bbdev/api/utils/rtl.py diff --git a/bbdev/api/config.yaml b/bbdev/api/config.yaml index e2884dc20..6f0ed8878 100644 --- a/bbdev/api/config.yaml +++ b/bbdev/api/config.yaml @@ -39,6 +39,8 @@ modules: config: adapter: class: modules::queue::BuiltinQueueAdapter + config: + max_attempts: 1 # ── PubSub Module ────────────────────────────────────────────────────── - class: modules::pubsub::PubSubModule diff --git a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py index 4ee24b404..678949cb8 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py @@ -10,8 +10,8 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) +from utils.event_common import require_chip from utils.path import get_buckyball_path -from bemu_common import bemu_manifest config = { "name": "bebop-bemu-sim-api", @@ -26,7 +26,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} chip = body.get("chip", "") try: - bemu_manifest(chip, get_buckyball_path()) + require_chip({"chip": chip}) except ValueError as e: return ApiResponse( status=400, diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 626eb70f6..a38e2dab2 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -19,13 +19,14 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.path import bebop_cargo_env, bebop_target_dir, get_buckyball_path, get_bemu_log_dir, workload_tests_root +from utils.path import bebop_cargo_env, bebop_target_dir, get_buckyball_path, log_dir, workload_tests_root from utils.stream_run import stream_run_logger_async from utils.search_workload import search_workload, search_workload_all from utils.event_common import check_result, get_origin_trace_id from utils.process_registry import cancellation_requested from bemu_common import bemu_core_manifest, bemu_manifest, bemu_tile_index, chip_emu_manifest + PERFETTO_TARGETS = { "buddy-buckyball-lenet-run": "buddy-buckyball-lenet-perfetto", } @@ -148,8 +149,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: clean_model_trace(binary_dir) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - log_dir = get_bemu_log_dir(bbdir, chip, timestamp, binary_name) - os.makedirs(log_dir, exist_ok=True) + run_log = log_dir(bbdir, chip, "verilog", timestamp, "bemu", binary_name) + os.makedirs(run_log, exist_ok=True) if input_data.get("rushB"): # rushB is a native ABI for one accelerator instance, so it remains a @@ -202,7 +203,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "backend": "rushB", "binary": binary_path, "chip": chip, - "log_dir": log_dir, + "log_dir": run_log, "timestamp": timestamp, }, trace_id=origin_tid, @@ -284,7 +285,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "task": "bemu", "binary": binary_path, "chip": chip, - "log_dir": log_dir, + "log_dir": run_log, "timestamp": timestamp, }, trace_id=origin_tid, @@ -314,7 +315,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "task": "perfetto", "binary": binary_path, - "log_dir": log_dir, + "log_dir": run_log, "timestamp": timestamp, "perfetto_target": perfetto_target, "perfetto": perfetto_path, @@ -331,7 +332,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "task": "bemu", "binary": binary_path, "chip": chip, - "log_dir": log_dir, + "log_dir": run_log, "timestamp": timestamp, "perfetto_target": perfetto_target, "perfetto": perfetto_path, diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py index 8dc416cb5..6372d8a40 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -10,8 +10,8 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) +from utils.event_common import require_chip from utils.path import get_buckyball_path -from bemu_common import bemu_manifest config = { "name": "bebop-bemu-batch-api", @@ -27,7 +27,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: chip = body.get("chip") try: - bemu_manifest(chip, get_buckyball_path()) + require_chip({"chip": chip}) except ValueError as e: return ApiResponse( status=400, diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index 00e6252bb..bd915a685 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -29,6 +29,7 @@ from regression import regression_workload_toml from regression_harness import nextest_harness_args + config = { "name": "bebop-bemu-batch", "description": "Run bebop bemu nextest batch regression", @@ -52,7 +53,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - elf_root = chip_output_root(bbdir, chip) try: bemu_cargo_manifest = bemu_manifest(chip, bbdir) except ValueError as e: @@ -63,6 +63,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + elf_root = chip_output_root(bbdir, chip) env = os.environ.copy() env.update(bebop_cargo_env(bbdir, chip)) diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py index dcbb3c231..c03da4789 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py @@ -42,23 +42,38 @@ def _hex_u(value: object, field: str, line_no: int) -> int: def chip_maps(bbdir: str, chip: str) -> tuple[dict[int, str], set[int], int]: - parse = Path(bbdir) / "bazel" / "configparse" - if str(parse) not in sys.path: - sys.path.insert(0, str(parse)) - from chip_json import compiler_core, core_entry, load_topology + path = ( + Path(bbdir) + / "examples" + / "chips" + / chip + / "configs" + / "generated" + / "config" + / "config.json" + ) + data = json.loads(path.read_text(encoding="utf-8")) + cores = [] + for tile in data["designs"]["tiles"]: + cores.extend(tile["cores"]) + if len(cores) != 1: + # isomorphic tiles share the same core pkg; take first unique balldomain+_file + seen = [] + uniq = [] + for core in cores: + key = core["balldomain"]["_file"] + if key not in seen: + seen.append(key) + uniq.append(core) + if len(uniq) != 1: + raise ValueError(f"chip {chip} has no unique topology core") + core = uniq[0] + else: + core = cores[0] - data = load_topology(Path(bbdir), chip) - core_name = compiler_core(data) - if not core_name: - raise ValueError(f"chip {chip} has no unique topology core") - core = core_entry(data, core_name) - ball = core.get("balldomain") - if not isinstance(ball, dict): - raise ValueError(f"chip {chip} core {core_name} has no balldomain") - isa = ball.get("ballISA") - ball_path = ball.get("_file") - if not isinstance(isa, list) or not isa: - raise ValueError(f"balldomain missing ballISA: {ball_path}") + ball = core["balldomain"] + isa = ball["ballISA"] + ball_path = ball["_file"] names: dict[int, str] = dict(ISA) matrix: set[int] = set() @@ -83,16 +98,9 @@ def chip_maps(bbdir: str, chip: str) -> tuple[dict[int, str], set[int], int]: if mnemonic in MATRIX_MNEMONICS: matrix.add(funct) - mem = core.get("memdomain") - if not isinstance(mem, dict): - raise ValueError(f"chip {chip} core {core_name} has no memdomain") - bank = mem.get("bank") - mem_path = mem.get("_file") - if not isinstance(bank, dict): - raise ValueError(f"memdomain missing [bank]: {mem_path}") - entries = bank.get("entries") - if not isinstance(entries, int) or entries <= 0: - raise ValueError(f"memdomain bank.entries must be a positive int: {mem_path}") + mem = core["memdomain"] + bank = mem["bank"] + entries = bank["entries"] return names, matrix, entries diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py index 962d96b5c..727bc32bd 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py @@ -1,8 +1,7 @@ -import json +import sys from pathlib import Path -from utils.build import install_bundle -from utils.chip import require_chip +from utils.event_common import require_chip def _repo(bbdir: str | None) -> Path: @@ -11,24 +10,29 @@ def _repo(bbdir: str | None) -> Path: return Path(bbdir or get_buckyball_path()) -def _chip_json(chip: str, bbdir: str | None = None) -> dict: +def _chip(chip: str, bbdir: str | None = None): + chip = require_chip({"chip": chip}) root = _repo(bbdir) - path = root / "examples" / "chips" / chip / "generated" / "chip.json" - if not path.is_file(): - install_bundle(str(root), chip) + scripts = root / "bbdev" / "api" / "steps" / "config" / "scripts" + if str(scripts) not in sys.path: + sys.path.insert(0, str(scripts)) + import chip_pb2 + + path = root / "examples" / "chips" / chip / "configs" / "generated" / "chip.pb" if not path.is_file(): - raise ValueError(f"missing {path} after bundle build") - return json.loads(path.read_text(encoding="utf-8")) + raise FileNotFoundError(f"missing {path}; run bbdev config --install") + msg = chip_pb2.Chip() + msg.ParseFromString(path.read_bytes()) + return msg def bemu_manifest(chip: str, bbdir: str | None = None) -> Path: chip = require_chip({"chip": chip}) root = _repo(bbdir) - install_bundle(str(root), chip) - manifest = root / "examples" / "chips" / chip / "generated" / "bemu" / "Cargo.toml" - if not manifest.is_file(): - raise ValueError(f"missing {manifest}; bundle install failed") - return manifest + path = root / "examples" / "chips" / chip / "configs" / "generated" / "bemu" / "Cargo.toml" + if not path.is_file(): + raise FileNotFoundError(f"missing {path}; run bbdev config --install") + return path def bemu_core_manifest(chip: str, bbdir: str | None = None) -> Path: @@ -38,20 +42,14 @@ def bemu_core_manifest(chip: str, bbdir: str | None = None) -> Path: def chip_emu_manifest(chip: str, bbdir: str | None = None) -> Path | None: chip = require_chip({"chip": chip}) root = _repo(bbdir) - main = _chip_json(chip, bbdir).get("bemu", {}).get("chipMain", "") + main = _chip(chip, bbdir).bemu.chip_main if not main: return None - manifest = root / "examples" / "chips" / chip / "emu" / "Cargo.toml" - if not manifest.is_file(): - raise ValueError(f"missing chip emu manifest: {manifest}") - return manifest + return root / "examples" / "chips" / chip / "emu" / "Cargo.toml" def bemu_tile_index(chip: str, bbdir: str | None = None) -> int | None: - bemu = _chip_json(chip, bbdir).get("bemu", {}) - if not bemu.get("chipMain", ""): + bemu = _chip(chip, bbdir).bemu + if not bemu.chip_main: return None - index = bemu.get("tileIndex", 0) - if not isinstance(index, int) or index < 0: - raise ValueError(f"chip {chip}: bemu.tileIndex must be a non-negative int") - return index + return bemu.tile_index diff --git a/bbdev/api/steps/bebop/p2e/00_clean_api.step.py b/bbdev/api/steps/bebop/p2e/00_clean_api.step.py index 847d385cb..e7001eb9a 100644 --- a/bbdev/api/steps/bebop/p2e/00_clean_api.step.py +++ b/bbdev/api/steps/bebop/p2e/00_clean_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip +from utils.event_common import require_chip from utils.path import get_buckyball_path config = { diff --git a/bbdev/api/steps/bebop/p2e/00_clean_event.step.py b/bbdev/api/steps/bebop/p2e/00_clean_event.step.py index ba3141705..b479cb159 100644 --- a/bbdev/api/steps/bebop/p2e/00_clean_event.step.py +++ b/bbdev/api/steps/bebop/p2e/00_clean_event.step.py @@ -7,8 +7,8 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import require_chip -from utils.path import get_buckyball_path, rtl_dir_for_clean +from utils.event_common import require_chip +from utils.path import get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -34,7 +34,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - build_dir = rtl_dir_for_clean(bbdir, chip, "p2e", input_data.get("output_dir")) + build_dir = rtl_dir(bbdir, chip, "p2e", input_data.get("output_dir")) command = f"rm -rf {build_dir}" result = await stream_run_logger_async( diff --git a/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py index c3b9a2483..e0fd8b9f5 100644 --- a/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py +++ b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip +from utils.event_common import require_chip from utils.path import get_buckyball_path config = { diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py index cb9371083..b024d8cc6 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py @@ -1,7 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip -from utils.path import get_buckyball_path, get_p2e_build_dir +from utils.event_common import require_chip +from utils.path import get_buckyball_path, rtl_dir config = { "name": "bebop-p2e-buildbitstream-api", @@ -19,7 +19,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: chip = require_chip(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - vsrc_dir = get_p2e_build_dir(bbdir, chip, body.get("vsrc_dir")) + vsrc_dir = rtl_dir(bbdir, chip, "p2e", body.get("vsrc_dir")) data = { "chip": chip, diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index 0976edabe..c3590fd75 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -16,8 +16,8 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import require_chip -from utils.path import bebop_cargo_env, get_buckyball_path, get_p2e_build_dir +from utils.event_common import require_chip +from utils.path import bebop_cargo_env, get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -45,16 +45,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - vsrc_dir = get_p2e_build_dir(bbdir, chip, input_data.get("vsrc_dir")) - if not os.path.isdir(vsrc_dir): - ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "vsrc_not_found", "vsrc_dir": vsrc_dir}, - trace_id=origin_tid, - ) - return - + vsrc_dir = rtl_dir(bbdir, chip, "p2e", input_data.get("vsrc_dir")) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") build_dir = ( input_data.get("build_dir") diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index 679894edc..bd94fad91 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -18,8 +18,8 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import require_chip -from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path, get_p2e_build_dir, get_run_log_dir +from utils.event_common import require_chip +from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path, rtl_dir, log_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -147,22 +147,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: multi_fpga = True ctx.logger.info(f"Detected multi-FPGA P2E case: {build_dir}") - vsrc_dir = get_p2e_build_dir(bbdir, chip, input_data.get("vsrc_dir")) - if not os.path.isdir(vsrc_dir): - ctx.logger.error(f"VSRC_PATH does not exist for P2E runtime: {vsrc_dir}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={ - "error": "vsrc_not_found", - "vsrc_dir": vsrc_dir, - }, - trace_id=origin_tid, - ) - return - + vsrc_dir = rtl_dir(bbdir, chip, "p2e", input_data.get("vsrc_dir")) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - log_dir = get_run_log_dir(bbdir, chip, "p2e", timestamp, "p2e", image_name, input_data.get("vsrc_dir")) - os.makedirs(log_dir, exist_ok=True) + run_log = log_dir(bbdir, chip, "p2e", timestamp, "p2e", image_name, input_data.get("vsrc_dir")) + os.makedirs(run_log, exist_ok=True) # Rebuild the VVAC host runtime in the bitstream case. The bitstream is # deliberately left in place, so runtime/DPIC changes never trigger FPGA synthesis. @@ -238,9 +226,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "image": image_path, "bitstream": bitstream, "build_dir": build_dir, - "log_dir": log_dir, - "bdb_trace": os.path.join(log_dir, "bdb.ndjson"), - "uart_log": os.path.join(log_dir, "uart.log"), + "log_dir": run_log, + "bdb_trace": os.path.join(run_log, "bdb.ndjson"), + "uart_log": os.path.join(run_log, "uart.log"), "timestamp": timestamp, }, trace_id=origin_tid, diff --git a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py index f60df7753..a5641e4c1 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py @@ -19,8 +19,8 @@ if bebop_path not in sys.path: sys.path.insert(0, bebop_path) -from utils.chip import require_chip -from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path, get_p2e_build_dir +from utils.event_common import require_chip +from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from regression import regression_workload_toml @@ -99,19 +99,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"Running {test_type} with workload config: {workload_toml}") ctx.logger.info(f"P2E case dir (from bitstream): {build_dir}") - vsrc_dir = get_p2e_build_dir(bbdir, chip, input_data.get("vsrc_dir")) - if not os.path.isdir(vsrc_dir): - ctx.logger.error(f"VSRC_PATH does not exist for P2E runtime: {vsrc_dir}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={ - "error": "vsrc_not_found", - "vsrc_dir": vsrc_dir, - }, - trace_id=origin_tid, - ) - return - + vsrc_dir = rtl_dir(bbdir, chip, "p2e", input_data.get("vsrc_dir")) # Rebuild VVAC host runtime in the bitstream case (same as runworkload). runtime_cmd = ( f"env BEBOP_P2E_RUNTIME_ONLY=1 BEBOP_P2E_REBUILD_RUNTIME=1 " diff --git a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py index 4f98d1cee..451f35fda 100644 --- a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py +++ b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py @@ -1,7 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip -from utils.path import get_buckyball_path, rtl_dir_for_clean +from utils.event_common import require_chip +from utils.path import get_buckyball_path, rtl_dir config = { "name": "bebop-verilator-clean-api", @@ -20,7 +20,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - build_dir = rtl_dir_for_clean(bbdir, chip, "verilog", body.get("output_dir")) + build_dir = rtl_dir(bbdir, chip, "verilog", body.get("output_dir")) data = { "chip": chip, "output_dir": build_dir, diff --git a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py index 26d42c559..35883221d 100644 --- a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py +++ b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py @@ -7,8 +7,8 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import require_chip -from utils.path import get_buckyball_path, rtl_dir_for_clean +from utils.event_common import require_chip +from utils.path import get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -37,7 +37,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - build_dir = rtl_dir_for_clean(bbdir, chip, "verilog", input_data.get("output_dir")) + build_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("output_dir")) command = f"rm -rf {build_dir}" result = await stream_run_logger_async( diff --git a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py index 17f4c20cf..2a023d1bf 100644 --- a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py +++ b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip +from utils.event_common import require_chip config = { "name": "bebop-verilator-verilog-api", @@ -15,37 +15,27 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} if body.get("balltype"): - if body.get("chip") or ( - isinstance(body.get("config"), str) and body.get("config") and body.get("config") != "None" - ): - return ApiResponse( - status=400, - body={"error": "balltype mill does not take --chip or --config"}, - ) - output_dir = body.get("output_dir") - if not output_dir: - return ApiResponse( - status=400, - body={"error": "balltype mill requires output_dir"}, - ) - data = { - "balltype": body.get("balltype"), - "output_dir": output_dir, - } - else: - try: - chip = require_chip(body) - except ValueError as e: - return ApiResponse( - status=400, - body={ - "status": "error", - "message": str(e), - "example": 'bbdev bebop verilator --verilog "--chip toy"', - }, - ) - data = {"chip": chip, "balltype": body.get("balltype")} - if body.get("output_dir"): - data["output_dir"] = body["output_dir"] + return ApiResponse( + status=400, + body={ + "error": "ball-only verilog belongs to uvm; use bbdev uvm --verilog", + }, + ) + + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse( + status=400, + body={ + "status": "error", + "message": str(e), + "example": 'bbdev bebop verilator --verilog "--chip toy"', + }, + ) + + data = {"chip": chip} + if body.get("output_dir"): + data["output_dir"] = body["output_dir"] await ctx.enqueue({"topic": "bebop.verilator.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_api.step.py b/bbdev/api/steps/bebop/verilator/03_build_api.step.py index 132d603bf..660338657 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_api.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_api.step.py @@ -1,7 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip -from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.event_common import require_chip +from utils.path import get_buckyball_path, rtl_dir config = { "name": "bebop-verilator-build-api", @@ -32,9 +32,8 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "chip": chip, "jobs": body.get("jobs", 16), "diff": bool(body.get("diff", False)), - "vsrc_dir": get_verilator_build_dir( - bbdir, chip, - body.get("vsrc-dir") or body.get("output-dir"), + "vsrc_dir": rtl_dir( + bbdir, chip, "verilog", body.get("vsrc-dir") or body.get("output-dir"), ), } await ctx.enqueue({"topic": "bebop.verilator.build", "data": {**data, "_trace_id": ctx.trace_id}}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index 4984fe3e8..079b84091 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -17,13 +17,13 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.build import install_bundle -from utils.chip import bebop_cargo, require_chip -from utils.path import bebop_cargo_env, bebop_target_dir, get_buckyball_path, get_verilator_build_dir +from utils.event_common import require_chip +from utils.path import bebop_cargo_env, bebop_target_dir, get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from build_marker import write_build_marker + config = { "name": "bebop-verilator-build", "description": "Build bebop verilator binary", @@ -32,78 +32,24 @@ "enqueues": ["bebop.verilator.sim", "bebop.verilator.run.sim"], } + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" + chip = require_chip(input_data) - try: - chip = require_chip(input_data) - except ValueError as error: - ctx.logger.error(str(error)) - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, - trace_id=origin_tid, - ) - return - - vsrc_dir = get_verilator_build_dir(bbdir, chip, input_data.get("vsrc_dir")) + vsrc_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("vsrc_dir")) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") - if not os.path.isdir(vsrc_dir): - ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={ - "error": "vsrc_not_found", - "source": "bebop.verilator.build", - "vsrc_dir": vsrc_dir, - }, - trace_id=origin_tid, - ) - return - diff = bool(input_data.get("diff", False)) build_dir = bebop_dir - manifest = bebop_cargo(bbdir) - chip = input_data.get("chip") + manifest = Path(bbdir) / "bebop" / "Cargo.toml" features = ["verilator"] if diff: - if not isinstance(chip, str) or not chip: - ctx.logger.error("Missing required parameter: --chip (required with --diff)") - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": "missing_chip"}, - trace_id=origin_tid, - ) - return - try: - install_bundle(bbdir, chip) - except ValueError as error: - ctx.logger.error(str(error)) - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": "chip_manifest_not_found", "detail": str(error)}, - trace_id=origin_tid, - ) - return manifest = Path(bbdir) / "examples" / "chips" / chip / "generated" / "bebop" / "Cargo.toml" build_dir = manifest.parent - if not manifest.is_file(): - ctx.logger.error(f"missing verilator bebop shim: {manifest}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "bebop_shim_not_found", "manifest": str(manifest)}, - trace_id=origin_tid, - ) - return features.extend(["bemu", "difftest"]) - dramsim_header = os.path.join(bbdir, "result", "include", "dramsim3.h") if not os.path.isfile(dramsim_header): ctx.logger.info(f"Nix environment is missing {dramsim_header}; rebuilding result") @@ -115,18 +61,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="bebop verilator difftest environment", ) if env_result.returncode != 0: - await check_result( - ctx, - env_result.returncode, - continue_run=False, - extra_fields={ - "task": "environment", - "diff": True, - "dependency": dramsim_header, - }, - trace_id=origin_tid, - ) - return + raise RuntimeError(f"nix build failed: {env_result.returncode}") jobs = input_data.get("jobs", 16) cargo_build_cmd = shlex.join( @@ -144,18 +79,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: str(jobs), ] ) - build_inner = ( - f"cd {shlex.quote(build_dir)} && " - f"exec {cargo_build_cmd}" - ) + build_inner = f"cd {shlex.quote(build_dir)} && exec {cargo_build_cmd}" build_cmd = f"nix develop -c sh -c {shlex.quote(build_inner)}" - command_cwd = bbdir env = {**os.environ, "VSRC_PATH": vsrc_dir, **bebop_cargo_env(bbdir, chip)} ctx.logger.info(f"Building bebop verilator (diff={diff}) ...") build_result = await stream_run_logger_async( cmd=build_cmd, logger=ctx.logger, - cwd=command_cwd, + cwd=bbdir, stdout_prefix="bebop verilator build", stderr_prefix="bebop verilator build", env=env, @@ -164,16 +95,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bebop_bin = os.path.join(bebop_target_dir(bbdir, chip), "release", "bebop") target_dir = bebop_target_dir(bbdir, chip) if build_result.returncode == 0: - try: - write_build_marker(target_dir, chip, vsrc_dir, bebop_bin, diff=diff) - except OSError as e: - ctx.logger.error(f"failed to write bebop verilator build marker: {e}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "build_marker_write_failed", "detail": str(e)}, - trace_id=origin_tid, - ) - return + write_build_marker(target_dir, chip, vsrc_dir, bebop_bin, diff=diff) await check_result( ctx, build_result.returncode, @@ -190,7 +112,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if build_result.returncode != 0: return - # Continue routing to sim if from run workflow if input_data.get("from_run_workflow"): await ctx.enqueue( { diff --git a/bbdev/api/steps/bebop/verilator/04_sim_api.step.py b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py index b9a2fef5e..837659b50 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_api.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip +from utils.event_common import require_chip from utils.path import get_buckyball_path config = { diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index b6ee22572..642b4f6bf 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -8,6 +8,7 @@ import shlex import sys from datetime import datetime +from pathlib import Path from motia import FlowContext, queue @@ -18,14 +19,14 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.chip import require_chip -from utils.build import install_bundle -from utils.path import bebop_target_dir, get_buckyball_path, get_run_log_dir, get_verilator_build_dir, workload_tests_root +from utils.event_common import require_chip +from utils.path import bebop_target_dir, get_buckyball_path, log_dir, rtl_dir, workload_tests_root from utils.stream_run import stream_run_logger_async from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id from build_marker import build_marker_path, read_build_marker + config = { "name": "bebop-verilator-sim", "description": "Run bebop verilator simulation", @@ -60,6 +61,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return + diff = bool(input_data.get("diff", False)) if diff and input_data.get("rushB"): ctx.logger.error("--diff and --rushB cannot be used together") @@ -72,17 +74,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - vsrc_dir = get_verilator_build_dir(bbdir, chip, input_data.get("vsrc_dir")) + vsrc_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("vsrc_dir")) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") - if not os.path.isdir(vsrc_dir): - ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "vsrc_not_found", "vsrc_dir": vsrc_dir}, - trace_id=origin_tid, - ) - return - build_dir = bebop_dir chip = input_data.get("chip") if diff: @@ -96,18 +89,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - try: - install_bundle(bbdir, chip) - except ValueError as error: - ctx.logger.error(str(error)) - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": "chip_resolution_failed", "detail": str(error)}, - trace_id=origin_tid, - ) - return bebop_bin = os.path.join(bebop_target_dir(bbdir, chip), "release", "bebop") if not os.path.isfile(bebop_bin): @@ -184,8 +165,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"binary_path: {binary_path}") timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - log_dir = get_run_log_dir(bbdir, chip, "verilog", timestamp, "verilator", binary_name, input_data.get("vsrc_dir")) - os.makedirs(log_dir, exist_ok=True) + run_log = log_dir(bbdir, chip, "verilog", timestamp, "verilator", binary_name, input_data.get("vsrc_dir")) + os.makedirs(run_log, exist_ok=True) wave_arg = " --no-wave" if input_data.get("no-wave", False) or input_data.get("no_wave", False) else "" trace_args = "" @@ -220,7 +201,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: run_cmd = ( f"{shlex.quote(bebop_bin)} run verilator " f"--elf={shlex.quote(binary_path)} " - f"--log-dir={shlex.quote(log_dir)}" + f"--log-dir={shlex.quote(run_log)}" f"{' --diff' if diff else ''}" f"{wave_arg}" f"{trace_args}" @@ -260,8 +241,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "diff": diff, "chip": chip, "binary": binary_path, - "log_dir": log_dir, - "bank_diff": os.path.join(log_dir, "bank_diff.ndjson") if diff else None, + "log_dir": run_log, + "bank_diff": os.path.join(run_log, "bank_diff.ndjson") if diff else None, "timestamp": timestamp, }, trace_id=origin_tid, diff --git a/bbdev/api/steps/bebop/verilator/05_run_api.step.py b/bbdev/api/steps/bebop/verilator/05_run_api.step.py index b3d356518..c9344dccb 100644 --- a/bbdev/api/steps/bebop/verilator/05_run_api.step.py +++ b/bbdev/api/steps/bebop/verilator/05_run_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip +from utils.event_common import require_chip from utils.path import get_buckyball_path config = { @@ -49,7 +49,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "chip": chip, "binary": binary, - "balltype": body.get("balltype"), "itrace": body.get("itrace", False), "mtrace": body.get("mtrace", False), "pmctrace": body.get("pmctrace", False), diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py index bcd45e3bb..979bae0b4 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -1,7 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip -from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.event_common import require_chip +from utils.path import get_buckyball_path, rtl_dir config = { "name": "bebop-verilator-batch-api", @@ -42,7 +42,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body={"error": "--diff and --rushB cannot be used together"}, ) - vsrc_dir = get_verilator_build_dir(bbdir, chip, body.get("vsrc_dir")) + vsrc_dir = rtl_dir(bbdir, chip, "verilog", body.get("vsrc_dir")) data = { "chip": chip, diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 698e9f87c..eb7166a61 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -7,6 +7,7 @@ import shutil import shlex import sys +from pathlib import Path from motia import FlowContext, queue @@ -17,14 +18,14 @@ if bebop_path not in sys.path: sys.path.insert(0, bebop_path) -from utils.chip import require_chip -from utils.build import install_bundle -from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path, get_verilator_build_dir +from utils.event_common import require_chip +from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from regression import regression_workload_toml from regression_harness import nextest_harness_args + config = { "name": "bebop-verilator-batch", "description": "Run bebop verilator nextest batch regression", @@ -50,6 +51,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + elf_root = chip_output_root(bbdir, chip) test_type = input_data.get("test", "elf-tests") @@ -83,16 +85,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"rushB={rushB} diff={diff}" ) - vsrc_dir = get_verilator_build_dir(bbdir, chip, input_data.get("vsrc_dir")) - if not os.path.isdir(vsrc_dir): - ctx.logger.error(f"VSRC_PATH does not exist: {vsrc_dir}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "vsrc_not_found", "vsrc_dir": vsrc_dir}, - trace_id=origin_tid, - ) - return - + vsrc_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("vsrc_dir")) vsrc_config = shlex.quote(f"env.VSRC_PATH='{vsrc_dir}'") env = os.environ.copy() env.update(bebop_cargo_env(bbdir, chip)) @@ -104,18 +97,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: manifest = f"{bebop_dir}/Cargo.toml" features = "verilator" if diff: - try: - install_bundle(bbdir, chip) - except ValueError as error: - ctx.logger.error(str(error)) - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": "chip_manifest_not_found", "detail": str(error)}, - trace_id=origin_tid, - ) - return features = "verilator,bemu,difftest" preload = os.pathsep.join( path diff --git a/bbdev/api/steps/compiler/01_build_api.step.py b/bbdev/api/steps/compiler/01_build_api.step.py index 8e8f9815f..44a6fecff 100644 --- a/bbdev/api/steps/compiler/01_build_api.step.py +++ b/bbdev/api/steps/compiler/01_build_api.step.py @@ -8,7 +8,6 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import available_compiler_chips, available_cores, resolve_core from utils.path import get_buckyball_path config = { @@ -22,41 +21,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - - stable = body.get("stable", False) - if not isinstance(stable, bool): - return ApiResponse( - status=400, - body={"error": "Invalid parameter: stable must be a boolean flag"}, - ) - + if "core" in body: + raise ValueError("compiler build takes --chip, not --core") chip = body.get("chip") - core = body.get("core") - if bool(chip) == bool(core): - return ApiResponse( - status=400, - body={"error": "Specify exactly one compiler target: --core or --chip"}, - ) - target_name = core or chip - if not isinstance(target_name, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", target_name): - return ApiResponse( - status=400, - body={"error": f"Invalid compiler target: {target_name}"}, - ) - - try: - if core: - resolve_core(get_buckyball_path(), core, require_compiler=True) - elif chip not in available_compiler_chips(get_buckyball_path()): - raise ValueError(f"Chip does not exist: {chip}") - except ValueError as error: - choices = available_cores(get_buckyball_path()) if core else available_compiler_chips(get_buckyball_path()) - return ApiResponse( - status=400, - body={ - "error": f"{error}; available {'cores' if core else 'chips'} are: {', '.join(choices)}", - }, - ) - + if not isinstance(chip, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", chip): + raise ValueError(f"Invalid compiler chip: {chip}") await ctx.enqueue({"topic": "compiler.build", "data": {**body, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/compiler/01_build_event.step.py b/bbdev/api/steps/compiler/01_build_event.step.py index cf360aa1a..1359c504e 100644 --- a/bbdev/api/steps/compiler/01_build_event.step.py +++ b/bbdev/api/steps/compiler/01_build_event.step.py @@ -1,3 +1,4 @@ +import asyncio import os import re import sys @@ -8,9 +9,11 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import available_compiler_chips, available_cores, resolve_core -from utils.build import build_compiler from utils.path import get_buckyball_path + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "scripts")) +import build as compiler_build # noqa: E402 + from utils.event_common import check_result, get_origin_trace_id config = { @@ -26,53 +29,25 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - stable = input_data.get("stable", False) - if not isinstance(stable, bool): - ctx.logger.error("Invalid parameter: stable must be a boolean flag") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "invalid_stable", "stable": stable}, - trace_id=origin_tid, - ) - return - chip = input_data.get("chip") - core = input_data.get("core") - if bool(chip) == bool(core): - ctx.logger.error("Specify exactly one compiler target: chip or core") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "invalid_target_selection"}, - trace_id=origin_tid, - ) - return - target_name = core or chip - if not isinstance(target_name, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", target_name): - ctx.logger.error(f"Invalid compiler target: {target_name}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "invalid_target", "target": target_name}, - trace_id=origin_tid, - ) - return + if not isinstance(chip, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", chip): + raise ValueError(f"Invalid compiler chip: {chip}") try: - if chip: - build_compiler(bbdir, chip=chip) - else: - resolve_core(bbdir, core, require_compiler=True) - build_compiler(bbdir, core=core) - except (ValueError, RuntimeError) as error: - choices = available_cores(bbdir) if core else available_compiler_chips(bbdir) - ctx.logger.error(str(error)) + await asyncio.to_thread( + compiler_build.build_compiler, + bbdir, + chip=chip, + logger=ctx.logger, + task_scope=origin_tid, + ) + except Exception as exc: + ctx.logger.error(f"Compiler build failed: {exc}") await check_result( - ctx, 1, continue_run=False, - extra_fields={ - "error": "unknown_core" if core else "unknown_chip", - "core": core, - "chip": chip, - "available": choices, - }, + ctx, + 1, + continue_run=False, + extra_fields={"task": "build", "chip": chip, "error": str(exc)}, trace_id=origin_tid, ) return diff --git a/bbdev/api/steps/compiler/README.md b/bbdev/api/steps/compiler/README.md deleted file mode 100644 index e45c5e6dd..000000000 --- a/bbdev/api/steps/compiler/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Compiler Workflow - -Compiler build workflow in the Buckyball framework for building the Buckyball compiler toolchain. - -## API Usage - -### `build` -**Endpoint**: `POST /compiler/build` - -**Function**: Build Buckyball compiler - -**Parameters**: -- `core`: direct compiler Core target, such as `toy`, `goban`, or `pebble` -- `chip`: Chip runtime target; its `chip.toml` selects the default compiler Core. Specify exactly one of `core` or `chip`. -- `stable`: optional boolean flag; for `pebble`, build the compiler package that can use the stable LLVM backend lowering path - -**Example**: -```bash -bbdev compiler --build '--chip toy' -bbdev compiler --build '--core goban' -bbdev compiler --build '--chip pebble --stable' -``` - -**Response**: -```json -{ - "status": 200, - "body": { - "success": true, - "processing": false, - "return_code": 0 - } -} -``` - -## Notes - -- Ensure the system has necessary build tools and dependencies -- Each `compilerCore` builds under `compiler/thirdparty/buddy-mlir/build/cores//`; different cores can be built in parallel on one checkout -- `pebble` supports both custom `xbuckyball` lowering and stable LLVM backend lowering; the selected lowering mode is controlled by pass/workload options -- Interactive shell: `export BUCKYBALL_COMPILER_CORE=` then re-source `sourceme.sh` for `buddy-opt` on PATH; bbdev injects `BUDDY_MLIR_BUILD_DIR` automatically in subprocesses diff --git a/bbdev/api/steps/compiler/scripts/build.py b/bbdev/api/steps/compiler/scripts/build.py new file mode 100644 index 000000000..fcafd3fad --- /dev/null +++ b/bbdev/api/steps/compiler/scripts/build.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import os +import shlex +import shutil +import subprocess +from pathlib import Path + + +def _repo(raw: str | Path) -> Path: + root = Path(raw).resolve() + if not root.is_dir(): + raise ValueError(f"repo does not exist: {root}") + return root + + +def _run( + cmd: list[str], + *, + repo: Path, + cwd: Path | None = None, + logger: object | None = None, + task_scope: str | None = None, + output_prefix: str, +) -> None: + if logger is None: + result = subprocess.run(cmd, cwd=cwd or repo) + else: + from utils.stream_run import stream_run_logger + + result = stream_run_logger( + cmd=shlex.join(cmd), + logger=logger, + cwd=str(cwd or repo), + stdout_prefix=output_prefix, + stderr_prefix=output_prefix, + task_scope=task_scope, + ) + if result.returncode != 0: + detail = result.stderr.strip() if logger is not None else "" + if not detail and logger is not None: + detail = "\n".join(result.stdout.splitlines()[-20:]) + if detail: + raise RuntimeError( + f"command failed ({result.returncode}): {' '.join(cmd)}\n{detail}" + ) + raise RuntimeError(f"command failed ({result.returncode}): {' '.join(cmd)}") + + +def compiler_python() -> str: + python = shutil.which("python3") + if not python: + raise RuntimeError("python3 not in PATH; enter nix develop") + probe = subprocess.run([python, "-c", "import nanobind"], capture_output=True) + if probe.returncode != 0: + raise RuntimeError(f"python3 lacks nanobind ({python}); enter nix develop") + return python + + +def build_llvm( + repo: str | Path, *, logger: object | None = None, task_scope: str | None = None +) -> Path: + root = _repo(repo) + buddy = root / "compiler" / "thirdparty" / "buddy-mlir" + llvm_src = buddy / "llvm" / "llvm" + llvm_build = buddy / "llvm" / "build" + if not llvm_src.is_dir(): + raise RuntimeError(f"missing LLVM source: {llvm_src}") + + python = compiler_python() + cmake = [ + "cmake", + "-G", + "Ninja", + "-S", + str(llvm_src), + "-B", + str(llvm_build), + "-DLLVM_ENABLE_PROJECTS=mlir;clang", + "-DLLVM_ENABLE_RUNTIMES=openmp", + "-DLLVM_TARGETS_TO_BUILD=host;RISCV", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DOPENMP_ENABLE_LIBOMPTARGET=OFF", + "-DLIBOMP_LIBFLAGS=-lrt", + "-DCMAKE_BUILD_TYPE=RELEASE", + "-DMLIR_ENABLE_BINDINGS_PYTHON=ON", + f"-DPython3_EXECUTABLE={python}", + f"-DPython_EXECUTABLE={python}", + ] + if not (llvm_build / "build.ninja").is_file(): + _run( + cmake, + repo=root, + cwd=buddy, + logger=logger, + task_scope=task_scope, + output_prefix="compiler llvm configure", + ) + _run( + ["ninja", "-C", str(llvm_build), "-j", str(os.cpu_count() or 1)], + repo=root, + cwd=buddy, + logger=logger, + task_scope=task_scope, + output_prefix="compiler llvm build", + ) + mlir_cmake = llvm_build / "lib" / "cmake" / "mlir" + if not mlir_cmake.is_dir(): + raise RuntimeError(f"LLVM/MLIR build failed: missing {mlir_cmake}") + return llvm_build + + +def compiler_build_dir(repo: str | Path, chip: str) -> Path: + return _repo(repo) / "compiler" / "thirdparty" / "buddy-mlir" / "build" / chip + + +def build_compiler( + repo: str | Path, + *, + chip: str, + logger: object | None = None, + task_scope: str | None = None, +) -> Path: + root = _repo(repo) + if not chip: + raise ValueError("chip is required") + chip_pb = root / "examples" / "chips" / chip / "configs" / "generated" / "chip.pb" + if not chip_pb.is_file(): + raise RuntimeError(f"missing {chip_pb}; run bbdev config --install") + buddy = root / "compiler" / "thirdparty" / "buddy-mlir" + llvm = build_llvm(root, logger=logger, task_scope=task_scope) + python = compiler_python() + build = compiler_build_dir(root, chip) + build.mkdir(parents=True, exist_ok=True) + cmake = [ + "cmake", + "-G", + "Ninja", + "-S", + str(buddy), + "-B", + str(build), + f"-DBUDDY_EXTERNAL_DIALECTS_DIR={root / 'compiler'}", + f"-DBUCKYBALL_CHIP_PB={chip_pb}", + f"-DMLIR_DIR={llvm / 'lib' / 'cmake' / 'mlir'}", + f"-DLLVM_DIR={llvm / 'lib' / 'cmake' / 'llvm'}", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=RELEASE", + "-DBUDDY_MLIR_ENABLE_PYTHON_PACKAGES=ON", + f"-DPython3_EXECUTABLE={python}", + f"-DPython_EXECUTABLE={python}", + ] + _run( + cmake, + repo=root, + logger=logger, + task_scope=task_scope, + output_prefix="compiler configure", + ) + _run( + [ + "ninja", + "-C", + str(build), + "-j", + str(os.cpu_count() or 1), + "buddy-opt", + "buddy-translate", + "buddy-llc", + "python-package-buddy", + "BuddyMLIRPythonModules", + ], + repo=root, + logger=logger, + task_scope=task_scope, + output_prefix="compiler build", + ) + for tool in ("buddy-opt", "buddy-translate", "buddy-llc"): + if not (build / "bin" / tool).is_file(): + raise RuntimeError(f"compiler build failed: missing {build / 'bin' / tool}") + return build diff --git a/bbdev/api/steps/config/.gitignore b/bbdev/api/steps/config/.gitignore new file mode 100644 index 000000000..e3efcb307 --- /dev/null +++ b/bbdev/api/steps/config/.gitignore @@ -0,0 +1 @@ +scripts/chip_pb2.py diff --git a/bbdev/api/steps/config/01_install_api.step.py b/bbdev/api/steps/config/01_install_api.step.py new file mode 100644 index 000000000..5b82ecdd5 --- /dev/null +++ b/bbdev/api/steps/config/01_install_api.step.py @@ -0,0 +1,22 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "config-install-api", + "description": "install all chip configs under examples/chips", + "flows": ["config"], + "triggers": [api("POST", "/config/install")], + "enqueues": ["config.install"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + if body: + unknown = sorted(body.keys()) + return ApiResponse( + status=400, + body={"error": f"config install takes no parameters: {', '.join(unknown)}"}, + ) + + await ctx.enqueue({"topic": "config.install", "data": {"_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/config/01_install_event.step.py b/bbdev/api/steps/config/01_install_event.step.py new file mode 100644 index 000000000..b7522758d --- /dev/null +++ b/bbdev/api/steps/config/01_install_event.step.py @@ -0,0 +1,155 @@ +import importlib.util +import json +import os +import shlex +import sys +from pathlib import Path + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +scripts_dir = os.path.join(os.path.dirname(__file__), "scripts") + +from utils.event_common import check_result, get_origin_trace_id +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger_async + +config = { + "name": "config-install", + "description": "install all chip configs under examples/chips", + "flows": ["config"], + "triggers": [queue("config.install")], + "enqueues": [], +} + + +def _load_script(name: str): + path = os.path.join(scripts_dir, name) + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise ValueError(f"cannot load {path}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = Path(get_buckyball_path()) + chips = bbdir / "examples" / "chips" + + tops = sorted(chips.glob("*/configs/chip.toml")) + if not tops: + raise ValueError(f"no configs/chip.toml under {chips}") + + toml2json = _load_script("1_toml2json.py") + derive = _load_script("2_parameter_derivation.py") + + # step 1: generate chip config JSONs from toml files + json_outs: list[Path] = [] + for top in tops: + data = toml2json.toml2json(top) + out = top.parent / "generated" / "config" / "config.json" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + json_outs.append(out) + ctx.logger.info(f"Wrote chip config JSON: {out}") + + # step 2: derive computed parameters from config JSONs + derived_outs: list[Path] = [] + for top, json_path in zip(tops, json_outs): + chip = top.parent.parent.name + data = json.loads(json_path.read_text(encoding="utf-8")) + derived_path = json_path.parent / "derived.json" + derive.write_derived(data, bbdir, chip, derived_path) + derived_outs.append(derived_path) + ctx.logger.info(f"Wrote derived config: {derived_path}") + + # step 3.1: generate chip_pb2.py from chip.proto + scripts = Path(scripts_dir) + cmd = ( + f"protoc -I{shlex.quote(str(scripts / 'proto'))} " + f"--python_out={shlex.quote(str(scripts))} " + f"{shlex.quote(str(scripts / 'proto' / 'chip.proto'))}" + ) + ctx.logger.info(f"Running protoc: {cmd}") + result = await stream_run_logger_async( + cmd=cmd, + logger=ctx.logger, + stdout_prefix="protoc", + stderr_prefix="protoc", + task_scope=origin_tid, + ) + if result.returncode != 0: + raise RuntimeError(f"protoc failed with returncode={result.returncode}") + ctx.logger.info(f"Generated chip python bindings: {scripts / 'chip_pb2.py'}") + + # step 3.2: generate chip.pb for all components build + json2proto = _load_script("3_json2proto.py") + pb_outs: list[Path] = [] + for json_path, derived_path in zip(json_outs, derived_outs): + out = json_path.parent / "chip.pb" + json2proto.write_chip_pb(json_path, derived_path, out, bbdir) + pb_outs.append(out) + ctx.logger.info(f"Wrote chip protobuf: {out}") + + # step 4: translate PB to different build systems + step4 = _load_script("4_install.py") + installed: list[dict[str, str]] = [] + for top, src_pb, json_path in zip(tops, pb_outs, json_outs): + chip_name = top.parent.parent.name + chip = step4.load_chip(src_pb) + gen = top.parent / "generated" + + # A. TO Arch BUILD SYSTEM + arch = step4.install_arch(src_pb, gen) + ctx.logger.info(f"Installed arch pb: {arch}") + + # B. TO BEMU BUILD SYSTEM + bemu = step4.install_bemu(chip, bbdir, gen) + ctx.logger.info(f"Installed bemu crate: {bemu}") + + # C. TO Workload BUILD SYSTEM + workload = step4.install_workload(chip, bbdir, chip_name, gen) + ctx.logger.info(f"Installed workload defs: {workload}") + + default_design = Path(json.loads(json_path.read_text(encoding="utf-8"))["designs"]["_file"]).resolve() + designs_dir = top.parent / "designs" + if not designs_dir.is_dir(): + raise ValueError(f"missing {designs_dir}") + for design in sorted(designs_dir.glob("*.toml")): + if design.resolve() == default_design: + continue + vdata = toml2json.toml2json(top, design=design) + vjson = gen / "config" / f"{design.stem}.json" + vjson.write_text(json.dumps(vdata, indent=2, sort_keys=True) + "\n", encoding="utf-8") + vderived = gen / "config" / f"{design.stem}.derived.json" + derive.write_derived(vdata, bbdir, chip_name, vderived) + vpb = json2proto.write_chip_pb(vjson, vderived, gen / f"{design.stem}.pb", bbdir) + ctx.logger.info(f"Installed arch variant pb: {vpb}") + + installed.append( + { + "chip": chip_name, + "arch": str(arch), + "bemu": str(bemu), + "workload": str(workload), + } + ) + + await check_result( + ctx, + 0, + continue_run=False, + extra_fields={ + "task": "install", + "json": [str(p) for p in json_outs], + "derived": [str(p) for p in derived_outs], + "pb2": str(scripts / "chip_pb2.py"), + "install": installed, + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/config/scripts/1_toml2json.py b/bbdev/api/steps/config/scripts/1_toml2json.py new file mode 100644 index 000000000..671b822ea --- /dev/null +++ b/bbdev/api/steps/config/scripts/1_toml2json.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Recursively expand one TOML entry and every include / *.toml reference into JSON.""" + +from __future__ import annotations + +import json +import sys +import tomllib +from pathlib import Path +from typing import Any + + +def _die(msg: str) -> None: + print(f"1_toml2json: {msg}", file=sys.stderr) + raise SystemExit(1) + + +def _load(path: Path) -> dict[str, Any]: + if not path.is_file(): + _die(f"missing TOML: {path}") + with path.open("rb") as f: + data = tomllib.load(f) + if not isinstance(data, dict): + _die(f"TOML root must be a table: {path}") + return data + + +def _walk_file( + path: Path, + includes: list[str], + stack: list[str], +) -> dict[str, Any]: + path = path.resolve() + key = str(path) + if key in stack: + _die("include cycle: %s" % " -> ".join(stack + [key])) + if key not in includes: + includes.append(key) + data = _load(path) + stack.append(key) + out = _walk(data, path.parent, includes, stack) + stack.pop() + if not isinstance(out, dict): + _die(f"{path}: root must be a table") + if "_file" in out: + _die(f"{path}: reserved key '_file'") + out["_file"] = key + return out + + +def _walk( + obj: Any, + base: Path, + includes: list[str], + stack: list[str], +) -> Any: + if isinstance(obj, dict): + inc = obj.get("include") + if inc is not None: + if not isinstance(inc, str) or not inc: + _die(f"{base}: include must be a non-empty string") + loaded = _walk_file((base / inc).resolve(), includes, stack) + rest = {k: v for k, v in obj.items() if k != "include"} + walked = _walk(rest, base, includes, stack) + if not isinstance(walked, dict): + _die(f"{base}: include siblings must be a table") + overlap = (set(loaded) & set(walked)) - {"_file"} + if overlap: + _die(f"{base / inc}: include key collision: {sorted(overlap)}") + merged = dict(loaded) + merged.update(walked) + return merged + return {k: _walk(v, base, includes, stack) for k, v in obj.items()} + if isinstance(obj, list): + return [_walk(x, base, includes, stack) for x in obj] + if isinstance(obj, str) and obj.endswith(".toml"): + path = (base / obj).resolve() + if not path.is_file(): + _die(f"toml path does not exist: {path} (from {base} + {obj!r})") + return _walk_file(path, includes, stack) + return obj + + +def toml2json(top: Path, design: Path | None = None) -> dict[str, Any]: + path = Path(top) + if not path.is_file(): + raise FileNotFoundError(f"missing TOML: {path}") + path = path.resolve() + includes: list[str] = [] + if design is None: + data = _walk_file(path, includes, []) + else: + design = Path(design) + if not design.is_file(): + raise FileNotFoundError(f"missing TOML: {design}") + design = design.resolve() + raw = _load(path) + if not isinstance(raw.get("designs"), dict): + _die(f"{path}: missing [designs]") + raw["designs"] = {"include": design.relative_to(path.parent).as_posix()} + includes.append(str(path)) + data = _walk(raw, path.parent, includes, [str(path)]) + if not isinstance(data, dict): + _die(f"{path}: root must be a table") + if "_file" in data: + _die(f"{path}: reserved key '_file'") + data["_file"] = str(path) + if "includes" in data: + _die(f"{path}: reserved key 'includes'") + data["includes"] = includes + return data + + +def main() -> None: + if len(sys.argv) != 2: + _die(f"usage: {sys.argv[0]} TOP.toml") + out = toml2json(Path(sys.argv[1])) + sys.stdout.write(json.dumps(out, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/bbdev/api/steps/config/scripts/2_parameter_derivation.py b/bbdev/api/steps/config/scripts/2_parameter_derivation.py new file mode 100644 index 000000000..e275646e4 --- /dev/null +++ b/bbdev/api/steps/config/scripts/2_parameter_derivation.py @@ -0,0 +1,645 @@ +#!/usr/bin/env python3 +"""Derive computed chip parameters from step1 config.json.""" + +from __future__ import annotations + +import json +import sys +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +_SYSTEM_FUNCTS = frozenset({0, 1, 16, 32, 33, 34, 35}) + + +def _die(msg: str) -> None: + raise ValueError(msg) + + +def _repo_rel(repo: Path, path: str | Path) -> str: + p = Path(path) + if not p.is_absolute(): + return p.as_posix() + return p.resolve().relative_to(repo.resolve()).as_posix() + + +def _walk(obj: Any) -> Iterator[Any]: + if isinstance(obj, dict): + yield obj + for key, value in obj.items(): + if key != "includes": + yield from _walk(value) + elif isinstance(obj, list): + for item in obj: + yield from _walk(item) + + +def core_pkg(rel: str) -> str | None: + parts = Path(rel).parts + if "cores" not in parts: + return None + idx = parts.index("cores") + if idx + 1 >= len(parts): + return None + return parts[idx + 1] + + +def _require_id(obj: dict[str, Any], key: str, ctx: str) -> int: + val = obj.get(key) + if not isinstance(val, int) or val < 0: + _die(f"{ctx}: missing {key} (non-negative int required)") + return val + + +def _require_id_list(obj: dict[str, Any], key: str, count: int, ctx: str) -> list[int]: + val = obj.get(key) + if not isinstance(val, list) or len(val) != count: + _die(f"{ctx}: {key} must be a list of length {count}") + out: list[int] = [] + for item in val: + if not isinstance(item, int) or item < 0: + _die(f"{ctx}: {key} entries must be non-negative ints") + out.append(item) + if len(set(out)) != len(out): + _die(f"{ctx}: {key} has duplicates: {out}") + return out + + +def _tile_cores(tile: dict[str, Any]) -> list[dict[str, Any]]: + cores = tile.get("cores") + if isinstance(cores, list) and cores: + out: list[dict[str, Any]] = [] + seen: set[int] = set() + for i, core in enumerate(cores): + if not isinstance(core, dict): + _die("cores entry must be a table") + core_id = _require_id(core, "core_id", f"cores[{i}]") + if core_id in seen: + _die(f"duplicate core_id {core_id} in tile") + seen.add(core_id) + out.append(core) + expected = set(range(len(out))) + if seen != expected: + _die(f"tile core_id must be exactly 0..{len(out)-1}, got {sorted(seen)}") + out.sort(key=lambda c: c["core_id"]) + return out + + template = tile.get("coreTemplate") + if isinstance(template, dict): + count = template.get("count") + if not isinstance(count, int) or count < 1: + _die("[coreTemplate].count must be a positive int") + core_ids = _require_id_list(template, "core_ids", count, "coreTemplate") + expected = set(range(count)) + if set(core_ids) != expected: + _die(f"coreTemplate.core_ids must be exactly 0..{count-1}, got {core_ids}") + base = { + k: v + for k, v in template.items() + if k not in ("count", "core_ids") + } + out = [] + for core_id in sorted(core_ids): + core = dict(base) + core["core_id"] = core_id + out.append(core) + return out + _die("tile must define [[cores]] or [coreTemplate]") + + +def iter_topology_tiles(topo: dict[str, Any]) -> list[dict[str, Any]]: + tiles = topo.get("tiles") + if isinstance(tiles, list) and tiles: + out: list[dict[str, Any]] = [] + seen: set[int] = set() + for i, tile in enumerate(tiles): + if not isinstance(tile, dict): + _die("tiles entry must be a table") + tile_id = _require_id(tile, "tile_id", f"tiles[{i}]") + if tile_id in seen: + _die(f"duplicate tile_id {tile_id}") + seen.add(tile_id) + out.append(tile) + expected = set(range(len(out))) + if seen != expected: + _die(f"tile_id must be exactly 0..{len(out)-1}, got {sorted(seen)}") + out.sort(key=lambda t: t["tile_id"]) + return out + + template = topo.get("tileTemplate") + if isinstance(template, dict): + count = template.get("count") + if not isinstance(count, int) or count < 1: + _die("[tileTemplate].count must be a positive int") + tile_ids = _require_id_list(template, "tile_ids", count, "tileTemplate") + expected = set(range(count)) + if set(tile_ids) != expected: + _die(f"tileTemplate.tile_ids must be exactly 0..{count-1}, got {tile_ids}") + base = { + k: v + for k, v in template.items() + if k not in ("count", "tile_ids") + } + out = [] + for tile_id in sorted(tile_ids): + tile = dict(base) + tile["tile_id"] = tile_id + out.append(tile) + return out + _die("topology must define [[tiles]] or [tileTemplate]") + + +def iter_cores(topo: dict[str, Any]) -> Iterator[dict[str, Any]]: + for tile in iter_topology_tiles(topo): + yield from _tile_cores(tile) + + +def _tile_signature(tile: dict[str, Any]) -> tuple[tuple[int, str, str], ...]: + sig = [] + for core in _tile_cores(tile): + rel = core.get("_file") + if not isinstance(rel, str): + _die("core config missing _file") + pkg = core_pkg(rel) + if not pkg: + _die(f"unsupported core config path: {rel}") + role = core.get("name") + if role is not None and not isinstance(role, str): + _die("core name must be a string") + sig.append((core["core_id"], pkg, role or "")) + return tuple(sig) + + +def _assert_isomorphic_tiles(topo: dict[str, Any]) -> None: + tiles = iter_topology_tiles(topo) + if not tiles: + _die("no tiles") + ref = _tile_signature(tiles[0]) + if not ref: + _die("tile has no cores") + for tile in tiles[1:]: + sig = _tile_signature(tile) + if sig != ref: + _die( + f"tiles must be isomorphic; tile_id={tile['tile_id']} " + f"signature {sig} != tile_id={tiles[0]['tile_id']} {ref}" + ) + + +def hart_id(tile_id: int, core_id: int, cores_per_tile: int) -> int: + if core_id < 0 or core_id >= cores_per_tile: + _die(f"core_id {core_id} out of range for cores_per_tile={cores_per_tile}") + return tile_id * cores_per_tile + core_id + + +def unique_cores(topo: dict[str, Any]) -> list[str]: + seen: set[str] = set() + out: list[str] = [] + for obj in _walk(topo): + if not isinstance(obj, dict): + continue + rel = obj.get("_file") + if not isinstance(rel, str): + continue + pkg = core_pkg(rel) + if pkg and pkg not in seen: + seen.add(pkg) + out.append(pkg) + if not out: + _die("topology has no cores//configs/*.toml include") + return out + + +def tile_files(topo: dict[str, Any]) -> list[str]: + seen: set[str] = set() + tiles: list[str] = [] + for obj in _walk(topo): + if not isinstance(obj, dict): + continue + rel = obj.get("_file") + if not isinstance(rel, str) or "/tiles/" not in rel or rel in seen: + continue + seen.add(rel) + tiles.append(rel) + tiles.sort() + if not tiles: + _die("topology has no tile file") + return tiles + + +def bank_params(core: dict, pkg: str) -> tuple[int, int, int]: + mem = core.get("memdomain") + if not isinstance(mem, dict): + _die(f"{pkg}: missing memdomain") + bank = mem.get("bank") + if not isinstance(bank, dict): + _die(f"{pkg}: missing [bank]") + num, width, entries = bank.get("num"), bank.get("width"), bank.get("entries") + if not isinstance(num, int) or num <= 0: + _die(f"{pkg}: bank.num must be a positive int") + if not isinstance(width, int) or width < 8 or width % 8 != 0: + _die(f"{pkg}: bank.width must be a positive multiple of 8") + if not isinstance(entries, int) or entries <= 0: + _die(f"{pkg}: bank.entries must be a positive int") + return num, width, entries + + +def _ball_dir(ball_class: str) -> str: + if not ball_class.startswith("examples.balls."): + _die(f"malformed ballClass: {ball_class!r}") + directory = ball_class[len("examples.balls.") :].split(".", 1)[0] + if not directory: + _die(f"malformed ballClass: {ball_class!r}") + return directory + + +def _balldomain_base_dir(repo: Path, core: dict, pkg: str) -> str: + bd = core.get("balldomain") + if isinstance(bd, dict): + rel = bd.get("_file") + if isinstance(rel, str): + return _repo_rel(repo, Path(rel).parent) + return f"examples/cores/{pkg}/configs/balldomains" + + +def _config_path(repo: Path, config: object) -> str: + if isinstance(config, str): + return _repo_rel(repo, config) + if isinstance(config, dict): + raw = config.get("_file") + if isinstance(raw, str): + return _repo_rel(repo, raw) + _die(f"ball mapping config must be a path or include _file: {config!r}") + + +def _ball_num(core: dict) -> int: + bd = core.get("balldomain") + if not isinstance(bd, dict): + return 0 + ball_num = bd.get("ballNum") + if ball_num is None: + return 0 + if not isinstance(ball_num, int) or ball_num < 0: + _die("balldomain.ballNum must be a non-negative int") + return ball_num + + +def _n_tiles(topo: dict[str, Any]) -> int: + top = topo.get("top") + if not isinstance(top, dict): + _die("topology missing [top]") + n = top.get("nTiles") + if not isinstance(n, int) or n < 1: + _die("[top].nTiles must be a positive int") + return n + + +def _derive_cores(repo: Path, topo: dict[str, Any]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for index, core in enumerate(iter_cores(topo)): + rel = core.get("_file") + if not isinstance(rel, str): + _die("core config missing _file") + pkg = core_pkg(rel) + if not pkg: + _die(f"unsupported core config path: {rel}") + role = core.get("name") + if role is not None and not isinstance(role, str): + _die("core name must be a string") + core_id = _require_id(core, "core_id", f"core index {index}") + ball_num = _ball_num(core) + if isinstance(core.get("memdomain"), dict): + num, width, entries = bank_params(core, pkg) + elif ball_num > 0: + _die(f"{pkg}: buckyball core missing memdomain") + else: + num, width, entries = 0, 0, 0 + entry: dict[str, Any] = { + "index": index, + "core_id": core_id, + "pkg": pkg, + "role": role or "", + "config_path": _repo_rel(repo, rel), + "balldomain_base_dir": _balldomain_base_dir(repo, core, pkg), + "bank_num": num, + "bank_width": width, + "bank_entries": entries, + "ball_num": ball_num, + } + if ball_num > 0: + bd = core.get("balldomain") + if not isinstance(bd, dict): + _die(f"{pkg}: missing balldomain") + mappings = bd.get("ballIdMappings") + if not isinstance(mappings, list): + _die(f"{pkg}: missing ballIdMappings") + entry["mappings"] = [] + for mapping in mappings: + if not isinstance(mapping, dict): + _die("ballIdMappings entry must be a table") + ball_class = mapping.get("ballClass") + if not isinstance(ball_class, str) or not ball_class: + _die(f"ballClass must be a non-empty string: {mapping!r}") + entry["mappings"].append( + { + "ball_id": mapping.get("ballId"), + "ball_dir": _ball_dir(ball_class), + "config_path": _config_path(repo, mapping.get("config")), + } + ) + out.append(entry) + return out + + +def _derive_tiles( + repo: Path, topo: dict[str, Any], cores: list[dict[str, Any]] +) -> list[dict[str, Any]]: + placements: list[dict[str, Any]] = [] + offset = 0 + for tile in iter_topology_tiles(topo): + tile_id = _require_id(tile, "tile_id", "tile") + tile_path = tile.get("_file") + if not isinstance(tile_path, str): + _die("tile missing _file") + tile_cores = _tile_cores(tile) + n = len(tile_cores) + shared = tile.get("sharedMem") + vbc = 0 + if isinstance(shared, dict): + raw = shared.get("virtualBankCount") + if isinstance(raw, int) and raw > 0: + vbc = raw + if vbc == 0 and n > 0: + bank_nums = [cores[i]["bank_num"] for i in range(offset, offset + n)] + if any(bank_nums): + vbc = max(bank_nums) + indices = list(range(offset, offset + n)) + has_buckyball = any(cores[i]["ball_num"] > 0 for i in indices) + mem_ball_channel_num = 0 + if has_buckyball: + raw = tile.get("memBallChannelNum") + if not isinstance(raw, int): + _die("tile with Buckyball cores must define memBallChannelNum") + mem_ball_channel_num = raw + placements.append( + { + "tile_id": tile_id, + "path": _repo_rel(repo, tile_path), + "core_indices": indices, + "cores_per_tile": n, + "virtual_bank_count": vbc, + "mem_ball_channel_num": mem_ball_channel_num, + } + ) + offset += n + return placements + + +def _bemu_balls(repo: Path, topo: dict[str, Any]) -> list[dict[str, str]]: + seen: set[str] = set() + balls: list[dict[str, str]] = [] + for core in iter_cores(topo): + bd = core.get("balldomain") + if not isinstance(bd, dict): + continue + mappings = bd.get("ballIdMappings") + isa = bd.get("ballISA") + if not isinstance(mappings, list) or not isinstance(isa, list): + continue + bid_to_class = { + m["ballId"]: m["ballClass"] + for m in mappings + if isinstance(m, dict) + and isinstance(m.get("ballId"), int) + and isinstance(m.get("ballClass"), str) + } + pkg = core_pkg(core.get("_file", "")) or "core" + for entry in isa: + if not isinstance(entry, dict): + _die("ballISA entry must be a table") + funct7 = entry.get("funct7") + bid = entry.get("bid") + if not isinstance(funct7, int) or not isinstance(bid, int): + _die(f"ballISA entry must have funct7 and bid: {entry!r}") + if funct7 in _SYSTEM_FUNCTS: + continue + ball_class = bid_to_class.get(bid) + if not ball_class: + _die( + f"core {pkg}: ballISA funct7 {funct7} " + f"references missing bid {bid}" + ) + if ball_class in seen: + continue + seen.add(ball_class) + ball_dir = _ball_dir(ball_class) + emu_lib = repo / "examples" / "balls" / ball_dir / "emu" / "src" / "lib.rs" + if not emu_lib.is_file(): + _die(f"missing BEMU ball source for {ball_class}: {emu_lib}") + balls.append( + { + "ball_class": ball_class, + "ball_dir": ball_dir, + "emu_lib": _repo_rel(repo, emu_lib), + } + ) + return balls + + +def _bemu_paths(repo: Path, chip: str, topo: dict[str, Any]) -> tuple[str, int]: + main = repo / "examples" / "chips" / chip / "emu" / "src" / "main.rs" + tiles = tile_files(topo) + if main.is_file(): + if len(tiles) != 1: + _die(f"chip {chip}: emu requires exactly one tile file, got {tiles}") + return f"examples/chips/{chip}/emu/src/main.rs", 0 + return "", 0 + + +def _ball_ctest_dirs(repo: Path, core: dict[str, Any]) -> list[str]: + bd = core.get("balldomain") + if not isinstance(bd, dict): + _die("core missing balldomain for ctest dirs") + mappings = bd.get("ballIdMappings") + if not isinstance(mappings, list): + _die("core missing ballIdMappings for ctest dirs") + dirs: list[str] = [] + for mapping in mappings: + if not isinstance(mapping, dict): + _die("ballIdMappings entry must be a table") + ball_class = mapping.get("ballClass") + if not isinstance(ball_class, str): + _die("ballIdMappings entry missing ballClass") + ball_dir = _ball_dir(ball_class) + path = repo / "examples" / "balls" / ball_dir / "workloads" / "ctests" + if not path.is_dir(): + _die(f"missing {path}") + dirs.append(_repo_rel(repo, path)) + return dirs + + +def _target_name(role: str, pkg: str) -> str: + if role: + return role + return pkg + + +def _derive_targets( + repo: Path, topo: dict[str, Any], cores: list[dict[str, Any]] +) -> dict[str, dict[str, Any]]: + targets: dict[str, dict[str, Any]] = {} + topo_by_pkg: dict[str, dict[str, Any]] = {} + for core in iter_cores(topo): + rel = core.get("_file") + if not isinstance(rel, str): + continue + pkg = core_pkg(rel) + if pkg and pkg not in topo_by_pkg: + topo_by_pkg[pkg] = core + + for inst in cores: + name = _target_name(inst["role"], inst["pkg"]) + if name in targets: + continue + pkg = inst["pkg"] + topo_core = topo_by_pkg.get(pkg) + if topo_core is None: + _die(f"topology missing core package {pkg}") + target: dict[str, Any] = { + "pkg": pkg, + "compiler": "compiler", + "bank": { + "num": inst["bank_num"], + "width": inst["bank_width"], + "entries": inst["bank_entries"], + }, + "ball_ctest_dirs": ( + _ball_ctest_dirs(repo, topo_core) if inst["ball_num"] > 0 else [] + ), + } + targets[name] = target + return targets + + +def _derive_harts( + tiles: list[dict[str, Any]], cores: list[dict[str, Any]] +) -> list[dict[str, Any]]: + if not tiles: + _die("no tiles") + cores_per_tile = tiles[0]["cores_per_tile"] + for tile in tiles[1:]: + if tile["cores_per_tile"] != cores_per_tile: + _die("tiles must be isomorphic (cores_per_tile mismatch)") + + harts: list[dict[str, Any]] = [] + for tile in tiles: + tile_id = tile["tile_id"] + for local, core_index in enumerate(tile["core_indices"]): + inst = cores[core_index] + core_id = inst["core_id"] + if core_id != local: + _die( + f"tile_id={tile_id}: core_id={core_id} != sorted slot {local}" + ) + hid = hart_id(tile_id, core_id, cores_per_tile) + harts.append( + { + "hart_id": hid, + "tile_id": tile_id, + "core_id": core_id, + "core_index": core_index, + "target": _target_name(inst["role"], inst["pkg"]), + "pkg": inst["pkg"], + "role": inst["role"], + } + ) + harts.sort(key=lambda h: h["hart_id"]) + ids = [h["hart_id"] for h in harts] + if ids != list(range(len(harts))): + _die(f"hart_id must be exactly 0..{len(harts)-1}, got {ids}") + if len(harts) != len(cores): + _die(f"hart count {len(harts)} != core count {len(cores)}") + return harts + + +def _workload(chip: str, targets: dict[str, dict[str, Any]], repo: Path) -> dict[str, Any]: + defs: dict[str, str] = { + "BUCKYBALL_WORKLOAD_CHIP": chip, + "BUCKYBALL_CARGO_TARGET_DIR": str(repo / "bebop" / "target" / chip), + } + return {"cmake_param": defs} + + +def derive(data: dict[str, Any], repo: Path, chip: str) -> dict[str, Any]: + repo = repo.resolve() + topo = data.get("designs") + if not isinstance(topo, dict): + _die(f"{chip}: missing designs") + + root_file = data.get("_file") + if not isinstance(root_file, str): + _die(f"{chip}: missing root _file") + design_file = topo.get("_file") + if not isinstance(design_file, str): + _die(f"{chip}: designs missing _file") + + includes_raw = data.get("includes") + if not isinstance(includes_raw, list): + _die(f"{chip}: includes must be a list") + + _assert_isomorphic_tiles(topo) + cores = _derive_cores(repo, topo) + tiles = _derive_tiles(repo, topo, cores) + targets = _derive_targets(repo, topo, cores) + harts = _derive_harts(tiles, cores) + n_tiles = _n_tiles(topo) + if len(tiles) != n_tiles: + _die(f"{chip}: [top].nTiles={n_tiles} but got {len(tiles)} tile(s)") + + chip_main, tile_index = _bemu_paths(repo, chip, topo) + + return { + "chip": chip, + "name": Path(_repo_rel(repo, design_file)).stem, + "chip_path": _repo_rel(repo, root_file), + "tile_config_path": _repo_rel(repo, design_file), + "includes": [_repo_rel(repo, p) for p in includes_raw], + "n_tiles": n_tiles, + "targets": targets, + "harts": harts, + "cores": cores, + "tiles": tiles, + "bemu": { + "chip_main": chip_main, + "tile_index": tile_index, + "balls": _bemu_balls(repo, topo), + }, + "workload": _workload(chip, targets, repo), + } + + +def write_derived(data: dict[str, Any], repo: Path, chip: str, out: Path) -> dict[str, Any]: + derived = derive(data, repo, chip) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(derived, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return derived + + +def main() -> None: + if len(sys.argv) not in (4, 5): + _die(f"usage: {sys.argv[0]} REPO CHIP CONFIG.json [OUT.json]") + repo = Path(sys.argv[1]) + chip = sys.argv[2] + config = Path(sys.argv[3]) + out = ( + Path(sys.argv[4]) + if len(sys.argv) == 5 + else config.parent / "derived.json" + ) + data = json.loads(config.read_text(encoding="utf-8")) + derived = write_derived(data, repo, chip, out) + sys.stdout.write(json.dumps(derived, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/bbdev/api/steps/config/scripts/3_json2proto.py b/bbdev/api/steps/config/scripts/3_json2proto.py new file mode 100644 index 000000000..17fe44086 --- /dev/null +++ b/bbdev/api/steps/config/scripts/3_json2proto.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +"""Build Chip protobuf from config.json + derived.json.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from typing import Any + +_SCRIPTS = Path(__file__).resolve().parent +if str(_SCRIPTS) not in sys.path: + sys.path.insert(0, str(_SCRIPTS)) + +import chip_pb2 as pb # noqa: E402 + + +def _load_derive(): + path = _SCRIPTS / "2_parameter_derivation.py" + spec = importlib.util.spec_from_file_location("parameter_derivation", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {path}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +_derive = _load_derive() + + +def _rel(bbdir: Path, path: str | Path) -> str: + p = Path(path) + if p.is_absolute(): + return p.resolve().relative_to(bbdir.resolve()).as_posix() + return p.as_posix() + + +def _ball_dir(ball_class: str) -> str: + parts = ball_class.split(".") + if len(parts) < 3 or parts[0] != "examples" or parts[1] != "balls": + raise ValueError(f"bad ballClass: {ball_class}") + return parts[2] + + +def _raw_core_list(designs: dict[str, Any]) -> list[dict[str, Any]]: + return list(_derive.iter_cores(designs)) + + +def _raw_tile_proto(designs: dict[str, Any]) -> dict[str, Any]: + return _derive.iter_topology_tiles(designs)[0] + + +def _fill_bank(msg: pb.BankConfig, d: dict[str, Any]) -> None: + msg.num = d["num"] + msg.width = d["width"] + msg.entries = d["entries"] + msg.mask_len = d["maskLen"] + msg.channel = d["channel"] + + +def _fill_mem(msg: pb.MemDomainConfig, d: dict[str, Any], bbdir: Path) -> None: + msg.source_path = _rel(bbdir, d["_file"]) + _fill_bank(msg.bank, d["bank"]) + dma = d["dma"] + msg.dma.n_xacts = dma["nXacts"] + msg.dma.burst_max_bytes = dma["burstMaxBytes"] + msg.dma.bus_width = dma["busWidth"] + msg.dma.max_in_flight_mem_reqs = dma["maxInFlightMemReqs"] + msg.tlb.size = d["tlb"]["size"] + msg.tma.read_channel = d["tma"]["readChannel"] + msg.tma.write_channel = d["tma"]["writeChannel"] + mmio = d["mmio"] + msg.mmio.enable = mmio["enable"] + msg.mmio.bank_num = mmio["bankNum"] + msg.mmio.bank_entries = mmio["bankEntries"] + msg.mmio.bank_width = mmio["bankWidth"] + msg.mmio.read_width = mmio["readWidth"] + msg.mem.addr_len = d["mem"]["addrLen"] + + +def _fill_ball(msg: pb.BallDomain, d: dict[str, Any], bbdir: Path) -> None: + msg.source_path = _rel(bbdir, d["_file"]) + msg.ball_num = d["ballNum"] + for m in d["ballIdMappings"]: + e = msg.mappings.add() + e.ball_id = m["ballId"] + e.ball_name = m["ballName"] + e.ball_class = m["ballClass"] + e.ball_dir = _ball_dir(m["ballClass"]) + e.config_path = _rel(bbdir, m["config"]["_file"]) + e.in_bw = m["inBW"] + e.out_bw = m["outBW"] + if "mmioReadBW" in m: + e.mmio_read_bw = m["mmioReadBW"] + if "mmioWriteBW" in m: + e.mmio_write_bw = m["mmioWriteBW"] + for item in d["ballISA"]: + e = msg.isa.add() + e.mnemonic = item["mnemonic"] + e.funct7 = item["funct7"] + e.bid = item["bid"] + + +def _fill_rocket(msg: pb.RocketCoreConfig, d: dict[str, Any]) -> None: + msg.x_len = d["xLen"] + msg.pg_levels = d["pgLevels"] + msg.use_vm = d["useVM"] + msg.use_zba = d["useZba"] + msg.use_zbb = d["useZbb"] + msg.use_zbs = d["useZbs"] + msg.have_c_flush = d["haveCFlush"] + md = d["mulDiv"] + msg.mul_div.enable = md["enable"] + msg.mul_div.mul_unroll = md["mulUnroll"] + msg.mul_div.mul_early_out = md["mulEarlyOut"] + msg.mul_div.div_early_out = md["divEarlyOut"] + fpu = d["fpu"] + msg.fpu.enable = fpu["enable"] + msg.fpu.min_f_len = fpu["minFLen"] + msg.fpu.f_len = fpu["fLen"] + dc = d["dcache"] + msg.dcache.n_sets = dc["nSets"] + msg.dcache.n_ways = dc["nWays"] + msg.dcache.n_mshrs = dc["nMSHRs"] + ic = d["icache"] + msg.icache.n_sets = ic["nSets"] + msg.icache.n_ways = ic["nWays"] + btb = d["btb"] + msg.btb.enable = btb["enable"] + msg.btb.n_entries = btb["nEntries"] + msg.btb.n_ras = btb["nRAS"] + + +def _fill_frontend(msg: pb.FrontendConfig, d: dict[str, Any], bbdir: Path) -> None: + msg.source_path = _rel(bbdir, d["_file"]) + msg.rob_entries = d["robEntries"] + msg.rs_out_of_order_response = d["rsOutOfOrderResponse"] + msg.bank_id_len = d["bankIdLen"] + msg.vbank_id_upper_bound = d["vbankIdUpperBound"] + msg.shared_bank_id_base = d["sharedBankIdBase"] + msg.iter_len = d["iterLen"] + msg.sub_rob_enable = d["subRobEnable"] + msg.sub_rob_depth = d["subRobDepth"] + + +def _fill_gp(msg: pb.GpDomainConfig, d: dict[str, Any], bbdir: Path) -> None: + msg.source_path = _rel(bbdir, d["_file"]) + msg.lane_number = d["laneNumber"] + msg.chaining_size = d["chainingSize"] + msg.v_len = d["vLen"] + msg.d_len = d["dLen"] + msg.e_len = d["eLen"] + msg.lane_scale = d["laneScale"] + + +def _fill_core_param(msg: pb.CoreParamConfig, d: dict[str, Any], bbdir: Path) -> None: + msg.source_path = _rel(bbdir, d["_file"]) + msg.core_data_bytes = d["coreDataBytes"] + msg.x_len = d["xLen"] + msg.vaddr_bits = d["vaddrBits"] + msg.paddr_bits = d["paddrBits"] + msg.pg_idx_bits = d["pgIdxBits"] + msg.n_pmps = d["nPMPs"] + + +def _fill_core(ci: pb.CoreInstance, raw: dict[str, Any], meta: dict[str, Any], bbdir: Path) -> None: + ci.index = meta["index"] + ci.role = meta["role"] + ci.pkg = meta["pkg"] + ci.config_path = meta["config_path"] + ci.balldomain_base_dir = meta["balldomain_base_dir"] + if "balldomain" in raw: + _fill_ball(ci.balldomain, raw["balldomain"], bbdir) + if "memdomain" in raw: + _fill_mem(ci.mem, raw["memdomain"], bbdir) + if "rocketCore" in raw: + _fill_rocket(ci.rocket_core, raw["rocketCore"]) + if "frontend" in raw: + _fill_frontend(ci.frontend, raw["frontend"], bbdir) + if "gpdomain" in raw: + _fill_gp(ci.gp_domain, raw["gpdomain"], bbdir) + if "core" in raw: + _fill_core_param(ci.core, raw["core"], bbdir) + + +def _fill_tile(tp: pb.TilePlacement, meta: dict[str, Any], proto: dict[str, Any]) -> None: + tp.path = meta["path"] + tp.virtual_bank_count = meta["virtual_bank_count"] + tp.core_indices.extend(meta["core_indices"]) + tp.mem_ball_channel_num = meta["mem_ball_channel_num"] + dc = proto["privateDCache"] + tp.private_dcache.enable = dc["enable"] + tp.private_dcache.ways = dc["ways"] + tp.private_dcache.capacity_kb = dc["capacityKB"] + tp.private_dcache.write_bytes = dc["writeBytes"] + tp.private_dcache.port_factor = dc["portFactor"] + tp.private_dcache.mem_cycles = dc["memCycles"] + sm = proto["sharedMem"] + tp.shared_mem.enable = sm["enable"] + tp.shared_mem.entries = sm["entries"] + tp.shared_mem.input_channels = sm["inputChannels"] + tp.shared_mem.default_group_count = sm["defaultGroupCount"] + if "virtualBankCount" in sm: + tp.shared_mem.virtual_bank_count = sm["virtualBankCount"] + if "memCycles" in sm: + tp.shared_mem.mem_cycles = sm["memCycles"] + + + +def fill_chip(config: dict[str, Any], derived: dict[str, Any], bbdir: Path) -> pb.Chip: + """ + Hardware numbers live in config.json; identities/paths live in derived.json. + chip.pb is a curated schema, so we pick from both instead of dumping either file. + + Core/tile lists vs templates are already expanded in step2 — import that walker, + do not write a second one. + + Attention: All tiles on a chip must match, so SRAM/dcache + is copied from the first tile onto every TilePlacement. + """ + b = pb.Chip() + b.name = derived["name"] + b.chip_path = derived["chip_path"] + b.topology_path = derived["tile_config_path"] + b.n_tiles = derived["n_tiles"] + b.includes.extend(derived["includes"]) + + sims = config["sims"] + b.mill.verilator_config = sims["verilator"] + b.mill.p2e_config = sims["p2e"] + + raw_cores = _raw_core_list(config["designs"]) + meta_cores = derived["cores"] + if len(raw_cores) != len(meta_cores): + raise ValueError( + f"core count mismatch: config={len(raw_cores)} derived={len(meta_cores)}" + ) + for raw, meta in zip(raw_cores, meta_cores): + _fill_core(b.cores.add(), raw, meta, bbdir) + + tile_proto = _raw_tile_proto(config["designs"]) + for meta in derived["tiles"]: + _fill_tile(b.tiles.add(), meta, tile_proto) + + targets = derived["targets"] + for name, target in targets.items(): + p = b.profiles.add() + p.name = name + p.pkg = target["pkg"] + p.compiler = target["compiler"] + bank = target["bank"] + p.bank_num = bank["num"] + p.bank_width = bank["width"] + p.bank_entries = bank["entries"] + p.ball_ctest_dirs.extend(target["ball_ctest_dirs"]) + + bemu = derived["bemu"] + b.bemu.chip_main = bemu["chip_main"] + b.bemu.tile_index = bemu["tile_index"] + for ball in bemu["balls"]: + e = b.bemu.balls.add() + e.ball_class = ball["ball_class"] + e.ball_dir = ball["ball_dir"] + e.emu_lib = ball["emu_lib"] + + for key, val in derived["workload"]["cmake_param"].items(): + b.workload.cmake_param[key] = val + return b + + +def write_chip_pb( + config_path: Path, + derived_path: Path, + out_path: Path, + bbdir: Path, +) -> Path: + config = json.loads(config_path.read_text(encoding="utf-8")) + derived = json.loads(derived_path.read_text(encoding="utf-8")) + chip = fill_chip(config, derived, bbdir) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(chip.SerializeToString()) + return out_path diff --git a/bbdev/api/steps/config/scripts/4_install.py b/bbdev/api/steps/config/scripts/4_install.py new file mode 100644 index 000000000..138a9b7fd --- /dev/null +++ b/bbdev/api/steps/config/scripts/4_install.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Install Chip.pb into arch / bemu / compiler / workload build trees.""" + +from __future__ import annotations + +import os +import shutil +import sys +from pathlib import Path + +_SCRIPTS = Path(__file__).resolve().parent +if str(_SCRIPTS) not in sys.path: + sys.path.insert(0, str(_SCRIPTS)) + +import chip_pb2 as pb # noqa: E402 + +_ENGINE = Path("bebop/src/nodes/bemu") +_CHIP_RS_DIR = _ENGINE / "src" +_BEMU_CRATE = _ENGINE / "chip" + + +def load_chip(pb_path: Path) -> pb.Chip: + chip = pb.Chip() + chip.ParseFromString(pb_path.read_bytes()) + if not chip.name: + raise ValueError(f"empty Chip.name in {pb_path}") + if not chip.cores: + raise ValueError(f"no cores in {pb_path}") + return chip + + +def _emit_dispatch(chip: pb.Chip, bbdir: Path, out: Path) -> None: + balls = list(chip.bemu.balls) + src_dir = bbdir / _CHIP_RS_DIR + if not balls: + out.write_text( + "use crate::inst::instruction::ExecContext;\n\n" + "pub fn execute_known(\n" + " _ball_class: &str,\n" + " _funct: u32,\n" + " _xs1: u64,\n" + " _xs2: u64,\n" + " _ctx: &mut ExecContext,\n" + ") -> u64 {\n" + ' panic!("no BEMU ball implementation")\n' + "}\n\n" + "pub fn cycles_after_issue(_ball_class: &str, _funct: u32, _xs1: u64, _xs2: u64) -> u64 {\n" + ' panic!("no BEMU ball latency implementation")\n' + "}\n", + encoding="utf-8", + ) + return + + lines = ["use crate::inst::instruction::ExecContext;", ""] + for ball in balls: + if not ball.ball_dir.isidentifier(): + raise ValueError(f"ball_dir is not a Rust identifier: {ball.ball_dir!r}") + lib = bbdir / ball.emu_lib + if not lib.is_file(): + raise FileNotFoundError(f"missing ball emu: {lib}") + rel = Path(os.path.relpath(lib, src_dir)).as_posix() + if '"' in rel or "\\" in rel: + raise ValueError(f"ball emu path not usable in rust #[path]: {rel}") + lines.append(f'#[path = "{rel}"]') + lines.append(f"mod {ball.ball_dir};") + lines.append("") + + def chain(fn: str, ctx: bool, panic_msg: str) -> list[str]: + extra = ", ctx" if ctx else "" + body = [ + f" {balls[0].ball_dir}::{fn}(ball_class, funct, xs1, xs2{extra})" + ] + for ball in balls[1:]: + body.append( + f" .or_else(|| {ball.ball_dir}::{fn}(ball_class, funct, xs1, xs2{extra}))" + ) + body.append(f' .unwrap_or_else(|| panic!("{panic_msg}"))') + return body + + lines += [ + "pub fn execute_known(", + " ball_class: &str,", + " funct: u32,", + " xs1: u64,", + " xs2: u64,", + " ctx: &mut ExecContext,", + ") -> u64 {", + ] + lines += chain( + "execute_known", + True, + "no BEMU ball implementation for ballClass={ball_class} funct7={funct}", + ) + lines += ["}", ""] + lines += [ + "pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> u64 {", + ] + lines += chain( + "cycles_after_issue", + False, + "no BEMU ball latency implementation for ballClass={ball_class} funct7={funct}", + ) + lines += ["}", ""] + out.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def install_arch(src_pb: Path, gen: Path) -> Path: + dest = gen / "chip.pb" + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_pb, dest) + return dest + + +def install_bemu(chip: pb.Chip, bbdir: Path, gen: Path) -> Path: + src = bbdir / _BEMU_CRATE + cargo = src / "Cargo.toml" + build_rs = src / "build.rs" + if not cargo.is_file(): + raise FileNotFoundError(f"missing {cargo}") + if not build_rs.is_file(): + raise FileNotFoundError(f"missing {build_rs}") + bemu = gen / "bemu" + bemu.mkdir(parents=True, exist_ok=True) + _emit_dispatch(chip, bbdir, bemu / "dispatch.rs") + shutil.copy2(cargo, bemu / "Cargo.toml") + shutil.copy2(build_rs, bemu / "build.rs") + return bemu / "Cargo.toml" + + +def install_workload(chip: pb.Chip, bbdir: Path, name: str, gen: Path) -> Path: + if not chip.profiles: + raise ValueError(f"chip {name}: no compiler profiles") + cargo = bbdir / "bebop" / "target" / name + defs = { + "BUCKYBALL_WORKLOAD_CHIP": name, + "BUCKYBALL_CARGO_TARGET_DIR": str(cargo), + "BUCKYBALL_RUSHB_BEMU_MANIFEST": str(gen / "bemu" / "Cargo.toml"), + "BUCKYBALL_RUSHB_BEMU_LIBRARY": str(cargo / "release" / "libbebop_bemu.so"), + "BUCKYBALL_RUSHB_VERILATOR_LIBRARY": str( + cargo / "release" / "deps" / "libbebop_verilator.so" + ), + "BUCKYBALL_CHIP_PB": str(gen / "chip.pb"), + } + out = gen / "workload" / "cmake.defs" + out.parent.mkdir(parents=True, exist_ok=True) + body = "".join(f"{k}={v}\n" for k, v in sorted(defs.items())) + out.write_text(body, encoding="utf-8") + return out diff --git a/bbdev/api/steps/config/scripts/proto/chip.proto b/bbdev/api/steps/config/scripts/proto/chip.proto new file mode 100644 index 000000000..52b71151d --- /dev/null +++ b/bbdev/api/steps/config/scripts/proto/chip.proto @@ -0,0 +1,245 @@ +// chip.proto is the Chip schema. +// Java: mill protoJava; Python: chip_pb2.py (generated by config install step3). +syntax = "proto3"; + +package buckyball.config; + +option java_package = "buckyball.config"; +option java_multiple_files = true; + +message MillConfig { + string verilator_config = 1; + string p2e_config = 2; +} + +message BankConfig { + uint32 num = 1; + uint32 width = 2; + uint32 entries = 3; + uint32 mask_len = 4; + uint32 channel = 5; +} + +message MmioConfig { + bool enable = 1; + uint32 bank_num = 2; + uint32 bank_entries = 3; + uint32 bank_width = 4; + uint32 read_width = 5; +} + +message DmaConfig { + uint32 n_xacts = 1; + uint32 burst_max_bytes = 2; + uint32 bus_width = 3; + uint32 max_in_flight_mem_reqs = 4; +} + +message TlbConfig { + uint32 size = 1; +} + +message TmaConfig { + uint32 read_channel = 1; + uint32 write_channel = 2; +} + +message MemAddrConfig { + uint32 addr_len = 1; +} + +message MemDomainConfig { + BankConfig bank = 1; + DmaConfig dma = 2; + TlbConfig tlb = 3; + TmaConfig tma = 4; + MmioConfig mmio = 5; + MemAddrConfig mem = 6; + string source_path = 7; +} + +message BallIdMapping { + uint32 ball_id = 1; + string ball_name = 2; + string ball_class = 3; + string ball_dir = 4; + string config_path = 5; + uint32 in_bw = 6; + uint32 out_bw = 7; + uint32 mmio_read_bw = 8; + uint32 mmio_write_bw = 9; +} + +message BallIsaEntry { + string mnemonic = 1; + uint32 funct7 = 2; + uint32 bid = 3; +} + +message BallDomain { + uint32 ball_num = 1; + repeated BallIdMapping mappings = 2; + repeated BallIsaEntry isa = 3; + string source_path = 4; +} + +message RocketMulDivConfig { + bool enable = 1; + uint32 mul_unroll = 2; + bool mul_early_out = 3; + bool div_early_out = 4; +} + +message RocketFpuConfig { + bool enable = 1; + uint32 min_f_len = 2; + uint32 f_len = 3; +} + +message RocketDCacheConfig { + uint32 n_sets = 1; + uint32 n_ways = 2; + uint32 n_mshrs = 3; +} + +message RocketICacheConfig { + uint32 n_sets = 1; + uint32 n_ways = 2; +} + +message RocketBtbConfig { + bool enable = 1; + uint32 n_entries = 2; + uint32 n_ras = 3; +} + +message RocketCoreConfig { + uint32 x_len = 1; + uint32 pg_levels = 2; + bool use_vm = 3; + bool use_zba = 4; + bool use_zbb = 5; + bool use_zbs = 6; + bool have_c_flush = 7; + RocketMulDivConfig mul_div = 8; + RocketFpuConfig fpu = 9; + RocketDCacheConfig dcache = 10; + RocketICacheConfig icache = 11; + RocketBtbConfig btb = 12; +} + +message FrontendConfig { + uint32 rob_entries = 1; + bool rs_out_of_order_response = 2; + uint32 bank_id_len = 3; + uint32 vbank_id_upper_bound = 4; + uint32 shared_bank_id_base = 5; + uint32 iter_len = 6; + bool sub_rob_enable = 7; + uint32 sub_rob_depth = 8; + string source_path = 9; +} + +message GpDomainConfig { + uint32 lane_number = 1; + uint32 chaining_size = 2; + uint32 v_len = 3; + uint32 d_len = 4; + uint32 e_len = 5; + uint32 lane_scale = 6; + string source_path = 7; +} + +message CoreParamConfig { + uint32 core_data_bytes = 1; + uint32 x_len = 2; + uint32 vaddr_bits = 3; + uint32 paddr_bits = 4; + uint32 pg_idx_bits = 5; + uint32 n_pmps = 6; + string source_path = 7; +} + +message CoreInstance { + uint32 index = 1; + string role = 2; + string pkg = 3; + string config_path = 4; + BallDomain balldomain = 5; + MemDomainConfig mem = 6; + string balldomain_base_dir = 7; + RocketCoreConfig rocket_core = 8; + FrontendConfig frontend = 9; + GpDomainConfig gp_domain = 10; + CoreParamConfig core = 11; +} + +message BemuBall { + string ball_class = 1; + string ball_dir = 2; + string emu_lib = 3; +} + +message BemuConfig { + string chip_main = 1; + uint32 tile_index = 2; + repeated BemuBall balls = 3; +} + +message SharedMemConfig { + bool enable = 1; + uint32 entries = 2; + uint32 input_channels = 3; + uint32 default_group_count = 4; + uint32 virtual_bank_count = 5; + uint32 mem_cycles = 6; +} + +message PrivateDCacheConfig { + bool enable = 1; + uint32 ways = 2; + uint32 capacity_kb = 3; + uint32 write_bytes = 4; + uint32 port_factor = 5; + uint32 mem_cycles = 6; +} + +message TilePlacement { + string path = 1; + uint32 virtual_bank_count = 2; + repeated uint32 core_indices = 3; + uint32 mem_ball_channel_num = 4; + PrivateDCacheConfig private_dcache = 5; + SharedMemConfig shared_mem = 6; +} + +message WorkloadConfig { + reserved 1; + map cmake_param = 2; +} + +message CompilerProfile { + string name = 1; + string pkg = 2; + string compiler = 3; + uint32 bank_num = 4; + uint32 bank_width = 5; + uint32 bank_entries = 6; + repeated string ball_ctest_dirs = 7; +} + +message Chip { + string name = 1; + MillConfig mill = 2; + string topology_path = 3; + repeated string includes = 4; + reserved 5; + uint32 n_tiles = 6; + repeated TilePlacement tiles = 7; + repeated CoreInstance cores = 8; + reserved 9; + BemuConfig bemu = 10; + WorkloadConfig workload = 11; + string chip_path = 12; + repeated CompilerProfile profiles = 13; +} diff --git a/bbdev/api/steps/dc/01_area_api.step.py b/bbdev/api/steps/dc/01_area_api.step.py index b7064f2e4..1a03447b6 100644 --- a/bbdev/api/steps/dc/01_area_api.step.py +++ b/bbdev/api/steps/dc/01_area_api.step.py @@ -1,7 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip -from utils.path import check_dc_rtl_args +from utils.event_common import require_chip config = { "name": "dc-area-api", @@ -14,14 +13,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - try: - chip = require_chip(body) - subset = {"chip": chip} - if "top" in body: - subset["top"] = body.get("top") - check_dc_rtl_args(subset) - except ValueError as exc: - return ApiResponse(status=400, body={"error": str(exc)}) + chip = require_chip(body) await ctx.enqueue( { "topic": "dc.verilog", diff --git a/bbdev/api/steps/dc/02_verilog_api.step.py b/bbdev/api/steps/dc/02_verilog_api.step.py index 5f496ae3a..72dd1f2de 100644 --- a/bbdev/api/steps/dc/02_verilog_api.step.py +++ b/bbdev/api/steps/dc/02_verilog_api.step.py @@ -1,7 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip -from utils.path import check_dc_rtl_args +from utils.event_common import require_chip config = { "name": "dc-verilog-api", @@ -14,18 +13,15 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - try: - chip = require_chip(body) - subset = {"chip": chip} - if "top" in body: - subset["top"] = body.get("top") - check_dc_rtl_args(subset) - except ValueError as e: - return ApiResponse(status=400, body={"error": str(e)}) - - data = { - "chip": chip, - "top": body.get("top") or "DigitalTop", - } - await ctx.enqueue({"topic": "dc.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + chip = require_chip(body) + await ctx.enqueue( + { + "topic": "dc.verilog", + "data": { + "chip": chip, + "top": body.get("top") or "DigitalTop", + "_trace_id": ctx.trace_id, + }, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/dc/04_power_api.step.py b/bbdev/api/steps/dc/04_power_api.step.py index 138bb79b7..c9fd1e8e6 100644 --- a/bbdev/api/steps/dc/04_power_api.step.py +++ b/bbdev/api/steps/dc/04_power_api.step.py @@ -1,7 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip -from utils.path import check_dc_power_args +from utils.event_common import require_chip config = { "name": "dc-power-api", @@ -14,11 +13,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - try: - chip = require_chip(body) - check_dc_power_args({k: v for k, v in body.items() if k != "config"}) - except ValueError as exc: - return ApiResponse(status=400, body={"error": str(exc)}) + chip = require_chip(body) await ctx.enqueue( { "topic": "dc.verilog", diff --git a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py index f0bdd7c8d..d661ce5fb 100644 --- a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py @@ -6,16 +6,23 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +mill_dir = os.path.dirname(__file__) +mill_scripts = os.path.join(mill_dir, "scripts") +if mill_dir not in sys.path: + sys.path.insert(0, mill_dir) +if mill_scripts not in sys.path: + sys.path.insert(0, mill_scripts) -from utils.chip import require_chip +from utils.event_common import require_chip from utils.path import get_buckyball_path -from utils.rtl import run_chip_mill -from utils.stream_run import stream_run_logger_async +from utils.path import rtl_out +import mill as mill_run from utils.event_common import check_result, get_origin_trace_id +from utils.stream_run import stream_run_logger_async config = { "name": "verilator-verilog", - "description": "generate verilog code", + "description": "generate chip verilog code", "flows": ["verilator"], "triggers": [queue("verilator.verilog")], "enqueues": ["verilator.build"], @@ -25,59 +32,40 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() + arch = os.path.join(bbdir, "arch") - if input_data.get("balltype"): - build_dir = input_data.get("output_dir") - if not build_dir: - ctx.logger.error("balltype mill requires output_dir") - _, failure_result = await check_result( - ctx, - 1, - continue_run=False, - extra_fields={ - "task": "validation", - "error": "balltype mill requires output_dir", - }, - trace_id=origin_tid, - ) - return failure_result - os.makedirs(build_dir, exist_ok=True) - command = ( - f"mill -i __.test.runMain sims.verify.BallTopMain {input_data.get('balltype')} " - "--disable-annotation-unknown --strip-debug-info -O=release " - f"--split-verilog -o={build_dir} " + try: + chip = require_chip(input_data) + mill_config, build_dir = rtl_out( + bbdir, chip, "verilog", input_data.get("output_dir"), ) + os.makedirs(build_dir, exist_ok=True) + ctx.logger.info(f"Using mill config: {mill_config}") ctx.logger.info(f"Using build directory: {build_dir}") - result = await stream_run_logger_async( - cmd=command, + returncode = ( + await stream_run_logger_async( + cmd=mill_run.elaborate_cmd("sims.verilator.Elaborate", mill_config, build_dir), logger=ctx.logger, - cwd=f"{bbdir}/arch", + cwd=arch, stdout_prefix="verilator verilog", stderr_prefix="verilator verilog", - ) - returncode = result.returncode - else: - try: - chip = require_chip(input_data) - build_dir, returncode = await run_chip_mill( - ctx, bbdir, chip, "verilog", "verilator verilog", - input_data.get("output_dir"), ) - except (ValueError, RuntimeError) as error: - ctx.logger.error(str(error)) - _, failure_result = await check_result( - ctx, - 1, - continue_run=False, - extra_fields={ - "task": "validation", - "error": str(error), - "example": 'bbdev verilator --verilog "--chip toy"', - }, - trace_id=origin_tid, - ) - return failure_result - ctx.logger.info(f"Using chip: {chip}") + ).returncode + except (ValueError, RuntimeError) as error: + ctx.logger.error(str(error)) + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "validation", + "error": str(error), + "example": 'bbdev verilator --verilog "--chip toy"', + }, + trace_id=origin_tid, + ) + return failure_result + ctx.logger.info(f"Using chip: {chip}") await check_result( ctx, diff --git a/bbdev/api/steps/vcs/02_verilog_event.step.py b/bbdev/api/steps/mill/02_vcs_verilog_event.step.py similarity index 62% rename from bbdev/api/steps/vcs/02_verilog_event.step.py rename to bbdev/api/steps/mill/02_vcs_verilog_event.step.py index d25ee16eb..6333bbfbe 100644 --- a/bbdev/api/steps/vcs/02_verilog_event.step.py +++ b/bbdev/api/steps/mill/02_vcs_verilog_event.step.py @@ -6,12 +6,19 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +mill_dir = os.path.dirname(__file__) +mill_scripts = os.path.join(mill_dir, "scripts") +if mill_dir not in sys.path: + sys.path.insert(0, mill_dir) +if mill_scripts not in sys.path: + sys.path.insert(0, mill_scripts) -from utils.event_common import check_result, get_origin_trace_id -from utils.chip import require_chip +from utils.event_common import require_chip from utils.path import get_buckyball_path -from utils.rtl import run_chip_mill - +from utils.path import rtl_out +import mill as mill_run +from utils.event_common import check_result, get_origin_trace_id +from utils.stream_run import stream_run_logger_async config = { "name": "vcs-verilog", @@ -30,11 +37,21 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result(ctx, 1, extra_fields={"task": "verilog", "error": str(exc)}, trace_id=origin_tid) return bbdir = get_buckyball_path() + arch = os.path.join(bbdir, "arch") try: - build_dir, returncode = await run_chip_mill( - ctx, bbdir, chip, "verilog", "vcs verilog", - input_data.get("output_dir"), + mill_config, build_dir = rtl_out( + bbdir, chip, "verilog", input_data.get("output_dir"), ) + os.makedirs(build_dir, exist_ok=True) + returncode = ( + await stream_run_logger_async( + cmd=mill_run.elaborate_cmd("sims.verilator.Elaborate", mill_config, build_dir), + logger=ctx.logger, + cwd=arch, + stdout_prefix="vcs verilog", + stderr_prefix="vcs verilog", + ) + ).returncode except (ValueError, RuntimeError) as exc: await check_result(ctx, 1, extra_fields={"task": "verilog", "error": str(exc)}, trace_id=origin_tid) return diff --git a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py index da8f479f9..4af13ec19 100644 --- a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py +++ b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py @@ -13,10 +13,18 @@ step_path = os.path.dirname(__file__) if step_path not in sys.path: sys.path.insert(0, step_path) +mill_dir = os.path.dirname(__file__) +mill_scripts = os.path.join(mill_dir, "scripts") +if mill_dir not in sys.path: + sys.path.insert(0, mill_dir) +if mill_scripts not in sys.path: + sys.path.insert(0, mill_scripts) -from utils.chip import require_chip -from utils.path import get_buckyball_path, get_run_log_dir -from utils.rtl import run_chip_mill +from utils.event_common import require_chip +from utils.path import get_buckyball_path, log_dir +from utils.stream_run import stream_run_logger_async +from utils.path import rtl_out +import mill as mill_run from utils.event_common import check_result, get_origin_trace_id config = { @@ -166,12 +174,26 @@ def _build_blackbox(mod_name, ports): async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() + arch = os.path.join(bbdir, "arch") try: chip = require_chip(input_data) - build_dir, returncode = await run_chip_mill( - ctx, bbdir, chip, "synth", "yosys verilog", - input_data.get("output_dir"), + mill_config, build_dir = rtl_out( + bbdir, chip, "verilog", input_data.get("output_dir"), ) + os.makedirs(build_dir, exist_ok=True) + ctx.logger.info(f"Using mill config: {mill_config}") + ctx.logger.info(f"Using build directory: {build_dir}") + returncode = ( + await stream_run_logger_async( + cmd=mill_run.elaborate_cmd( + "sims.verilator.Elaborate", mill_config, build_dir, seq_mem=True, + ), + logger=ctx.logger, + cwd=arch, + stdout_prefix="yosys verilog", + stderr_prefix="yosys verilog", + ) + ).returncode except (ValueError, RuntimeError) as error: success_result, failure_result = await check_result( ctx, @@ -192,7 +214,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return failure_result yosys_cfg = load_yosys_config() top_module = input_data.get("top") or yosys_cfg.get("top") or "DigitalTop" - yosys_log_dir = input_data.get("log_dir") or get_run_log_dir( + yosys_log_dir = input_data.get("log_dir") or log_dir( bbdir, chip, "synth", datetime.now().strftime("%Y%m%d-%H%M%S-%f"), "yosys", top_module, input_data.get("output_dir"), ) diff --git a/bbdev/api/steps/mill/04_dc_verilog_event.step.py b/bbdev/api/steps/mill/04_dc_verilog_event.step.py index 34ef224fa..5d1ddd862 100644 --- a/bbdev/api/steps/mill/04_dc_verilog_event.step.py +++ b/bbdev/api/steps/mill/04_dc_verilog_event.step.py @@ -1,4 +1,5 @@ import os +from datetime import datetime import sys import glob import re @@ -13,9 +14,17 @@ sys.path.insert(0, step_path) from utils.event_common import check_result, get_origin_trace_id -from utils.chip import require_chip -from utils.path import get_buckyball_path, get_dc_analysis_dir -from utils.rtl import run_chip_mill +from utils.event_common import require_chip +from utils.path import get_buckyball_path, log_dir +from utils.stream_run import stream_run_logger_async +mill_dir = os.path.join(os.path.dirname(__file__)) +mill_scripts = os.path.join(mill_dir, "scripts") +if mill_dir not in sys.path: + sys.path.insert(0, mill_dir) +if mill_scripts not in sys.path: + sys.path.insert(0, mill_scripts) +from utils.path import rtl_out +import mill as mill_run config = { "name": "dc-verilog", @@ -77,12 +86,26 @@ def prepare_dc_verilog(build_dir: str): async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() + arch = os.path.join(bbdir, "arch") try: chip = require_chip(input_data) - build_dir, returncode = await run_chip_mill( - ctx, bbdir, chip, "synth", "dc verilog", - input_data.get("output_dir"), + mill_config, build_dir = rtl_out( + bbdir, chip, "verilog", input_data.get("output_dir"), ) + os.makedirs(build_dir, exist_ok=True) + ctx.logger.info(f"Using mill config: {mill_config}") + ctx.logger.info(f"Using build directory: {build_dir}") + returncode = ( + await stream_run_logger_async( + cmd=mill_run.elaborate_cmd( + "sims.verilator.Elaborate", mill_config, build_dir, seq_mem=True, + ), + logger=ctx.logger, + cwd=arch, + stdout_prefix="dc verilog", + stderr_prefix="dc verilog", + ) + ).returncode except (ValueError, RuntimeError) as error: _, failure_result = await check_result( ctx, @@ -147,9 +170,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "_trace_id": origin_tid, } if input_data.get("from_power_workflow"): - payload["analysis_dir"] = get_dc_analysis_dir(bbdir, chip, "power") + payload["analysis_dir"] = log_dir(bbdir, chip, "synth", datetime.now().strftime("%Y%m%d-%H%M%S-%f"), "dc", "power") elif input_data.get("from_area_workflow"): - payload["analysis_dir"] = get_dc_analysis_dir(bbdir, chip, "area") + payload["analysis_dir"] = log_dir(bbdir, chip, "synth", datetime.now().strftime("%Y%m%d-%H%M%S-%f"), "dc", "area") await ctx.enqueue({"topic": "ip-replace.run", "data": payload}) return diff --git a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py index 5de510fd5..b3ab033af 100644 --- a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py @@ -11,11 +11,18 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) - -from utils.chip import require_chip +mill_dir = os.path.dirname(__file__) +mill_scripts = os.path.join(mill_dir, "scripts") +if mill_dir not in sys.path: + sys.path.insert(0, mill_dir) +if mill_scripts not in sys.path: + sys.path.insert(0, mill_scripts) + +from utils.event_common import require_chip from utils.path import get_buckyball_path -from utils.rtl import run_chip_mill from utils.stream_run import stream_run_logger_async +from utils.path import rtl_out +import mill as mill_run from utils.event_common import check_result, get_origin_trace_id config = { @@ -27,13 +34,6 @@ } -def prepare_verilator_verilog(build_dir: str, arch_dir: str, logger): - unwanted = f"{arch_dir}/BBSimHarness.sv" - if os.path.isfile(unwanted): - os.remove(unwanted) - logger.info(f"Removed leaked mill output: {unwanted}") - - def check_verilog_output(build_dir: str) -> dict: exists = os.path.exists(build_dir) is_dir = os.path.isdir(build_dir) @@ -50,62 +50,40 @@ def check_verilog_output(build_dir: str) -> dict: async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - arch_dir = f"{bbdir}/arch" + arch = os.path.join(bbdir, "arch") chip = None - - if input_data.get("balltype"): - build_dir = input_data.get("output_dir") - if not build_dir: - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={ - "task": "validation", - "error": "balltype mill requires output_dir", - "example": 'bbdev bebop verilator --verilog "--balltype foo --output-dir "', - }, - trace_id=origin_tid, - ) - return - os.makedirs(build_dir, exist_ok=True) - command = ( - f"mill -i __.test.runMain sims.verify.BallTopMain {input_data.get('balltype')} " - "--disable-annotation-unknown --strip-debug-info -O=debug " - f"--split-verilog -o={build_dir} " + try: + chip = require_chip(input_data) + mill_config, build_dir = rtl_out( + bbdir, chip, "verilog", input_data.get("output_dir"), ) + os.makedirs(build_dir, exist_ok=True) + ctx.logger.info(f"Using mill config: {mill_config}") ctx.logger.info(f"Using build directory: {build_dir}") - result = await stream_run_logger_async( - cmd=command, + returncode = ( + await stream_run_logger_async( + cmd=mill_run.elaborate_cmd("sims.verilator.Elaborate", mill_config, build_dir), logger=ctx.logger, - cwd=arch_dir, + cwd=arch, stdout_prefix="bebop verilator verilog", stderr_prefix="bebop verilator verilog", - ) - returncode = result.returncode - prepare_verilator_verilog(build_dir, arch_dir, ctx.logger) - else: - try: - chip = require_chip(input_data) - build_dir, returncode = await run_chip_mill( - ctx, bbdir, chip, "verilog", "bebop verilator verilog", - input_data.get("output_dir"), - ) - except (ValueError, RuntimeError) as error: - ctx.logger.error(str(error)) - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={ - "task": "validation", - "error": str(error), - "example": 'bbdev bebop verilator --verilog "--chip toy"', - }, - trace_id=origin_tid, ) - return - ctx.logger.info(f"Using chip: {chip}") + ).returncode + except (ValueError, RuntimeError) as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "validation", + "error": str(error), + "example": 'bbdev bebop verilator --verilog "--chip toy"', + }, + trace_id=origin_tid, + ) + return + ctx.logger.info(f"Using chip: {chip}") output_status = check_verilog_output(build_dir) if returncode != 0: diff --git a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py index f5ae693ba..e4317835e 100644 --- a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py +++ b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py @@ -6,10 +6,18 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +mill_dir = os.path.dirname(__file__) +mill_scripts = os.path.join(mill_dir, "scripts") +if mill_dir not in sys.path: + sys.path.insert(0, mill_dir) +if mill_scripts not in sys.path: + sys.path.insert(0, mill_scripts) -from utils.chip import require_chip +from utils.event_common import require_chip from utils.path import get_buckyball_path -from utils.rtl import run_chip_mill +from utils.path import rtl_out +from utils.stream_run import stream_run_logger_async +import mill as mill_run from utils.event_common import check_result, get_origin_trace_id config = { @@ -24,12 +32,25 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() + arch = os.path.join(bbdir, "arch") try: chip = require_chip(input_data) - build_dir, returncode = await run_chip_mill( - ctx, bbdir, chip, "p2e", "bebop p2e verilog", - input_data.get("output_dir"), + mill_config, build_dir = rtl_out( + bbdir, chip, "p2e", input_data.get("output_dir"), ) + os.makedirs(build_dir, exist_ok=True) + ctx.logger.info(f"Using mill config: {mill_config}") + ctx.logger.info(f"Using build directory: {build_dir}") + prefix = "bebop p2e verilog" + returncode = ( + await stream_run_logger_async( + cmd=mill_run.elaborate_cmd("sims.p2e.Elaborate", mill_config, build_dir), + logger=ctx.logger, + cwd=arch, + stdout_prefix=prefix, + stderr_prefix=prefix, + ) + ).returncode except (ValueError, RuntimeError) as error: ctx.logger.error(str(error)) await check_result( diff --git a/bbdev/api/steps/mill/scripts/mill.py b/bbdev/api/steps/mill/scripts/mill.py new file mode 100644 index 000000000..4fe212310 --- /dev/null +++ b/bbdev/api/steps/mill/scripts/mill.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import os +import shlex + + +def elaborate_cmd(main: str, config: str, out: str, *, seq_mem: bool = False) -> str: + extra = "" + if seq_mem: + extra = ( + " -lowering-options=disallowLocalVariables" + f" --repl-seq-mem --repl-seq-mem-file={shlex.quote(os.path.join(out, 'mems.conf'))}" + ) + return ( + f"mill -i __.test.runMain {main} {config} " + "--disable-annotation-unknown --strip-debug-info -O=debug " + f"--split-verilog -o={shlex.quote(out)}{extra}" + ) diff --git a/bbdev/api/steps/regression/00_buildbitstream_api.step.py b/bbdev/api/steps/regression/00_buildbitstream_api.step.py index 1ef76e640..06b697f9d 100644 --- a/bbdev/api/steps/regression/00_buildbitstream_api.step.py +++ b/bbdev/api/steps/regression/00_buildbitstream_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip +from utils.event_common import require_chip config = { "name": "regression-buildbitstream-api", diff --git a/bbdev/api/steps/regression/00_buildbitstream_event.step.py b/bbdev/api/steps/regression/00_buildbitstream_event.step.py index ddc07c1f7..a93f0e17e 100644 --- a/bbdev/api/steps/regression/00_buildbitstream_event.step.py +++ b/bbdev/api/steps/regression/00_buildbitstream_event.step.py @@ -7,7 +7,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import require_chip +from utils.event_common import require_chip from utils.event_common import check_result, get_origin_trace_id config = { diff --git a/bbdev/api/steps/regression/02_eval_performance_event.step.py b/bbdev/api/steps/regression/02_eval_performance_event.step.py index 12b399e61..ecee5c1b0 100644 --- a/bbdev/api/steps/regression/02_eval_performance_event.step.py +++ b/bbdev/api/steps/regression/02_eval_performance_event.step.py @@ -26,11 +26,13 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.build import build_workload -from utils.path import get_buckyball_path, get_p2e_build_dir, get_run_log_dir +from utils.path import get_buckyball_path, rtl_dir, log_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id +sys.path.insert(0, os.path.join(get_buckyball_path(), "bb-tests", "workloads", "scripts")) +import build as workload_build # noqa: E402 + regression_scripts = os.path.join(os.path.dirname(__file__), "scripts") if regression_scripts not in sys.path: sys.path.insert(0, regression_scripts) @@ -82,7 +84,7 @@ def _workload_build(bbdir, chip, model): model_key = model.lower() if MODEL_LAYOUT.get(model_key) is None: raise ValueError(f"Unknown model: {model}") - build_workload(bbdir, chip, model=model_key) + workload_build.build_workload(bbdir, chip, model=model_key) def _kernel_build_cmds(bbdir, model, dataset=""): @@ -125,12 +127,10 @@ def _p2e_run_cmds(bbdir, bitstream, image_name, chip, input_data): multi_fpga = bool(input_data.get("multi-fpga", False)) if not multi_fpga and _p2e.case_uses_multi_fpga(build_dir): multi_fpga = True - vsrc_dir = get_p2e_build_dir(bbdir, chip, input_data.get("vsrc_dir")) - if not os.path.isdir(vsrc_dir): - raise FileNotFoundError(f"VSRC_PATH does not exist for P2E runtime: {vsrc_dir}") + vsrc_dir = rtl_dir(bbdir, chip, "p2e", input_data.get("vsrc_dir")) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - log_dir = get_run_log_dir(bbdir, chip, "p2e", timestamp, "p2e", image_name, input_data.get("vsrc_dir")) - os.makedirs(log_dir, exist_ok=True) + run_log = log_dir(bbdir, chip, "p2e", timestamp, "p2e", image_name, input_data.get("vsrc_dir")) + os.makedirs(run_log, exist_ok=True) runtime_cmd = ( f"env BEBOP_P2E_RUNTIME_ONLY=1 BEBOP_P2E_REBUILD_RUNTIME=1 " f'cargo run --release --features p2e -- build p2e ' @@ -147,7 +147,7 @@ def _p2e_run_cmds(bbdir, bitstream, image_name, chip, input_data): ) if multi_fpga: run_cmd += " --multi-fpga" - return runtime_cmd, run_cmd, f"{bbdir}/bebop", log_dir, build_dir + return runtime_cmd, run_cmd, f"{bbdir}/bebop", run_log, build_dir def _perfetto_cmd(trace_dir, trace_toml, mlir_files): @@ -261,7 +261,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"[eval-performance] model {model}: p2e runworkload") image_name = os.path.splitext(os.path.basename(fw_hex))[0] try: - runtime_cmd, run_cmd, bebop_cwd, log_dir, build_dir = _p2e_run_cmds( + runtime_cmd, run_cmd, bebop_cwd, run_log, build_dir = _p2e_run_cmds( bbdir, bitstream, image_name, chip, input_data ) except FileNotFoundError as e: @@ -301,9 +301,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: log_dir=log_dir) return - uart_path = Path(log_dir) / "uart_hart_0.log" + uart_path = Path(run_log) / "uart_hart_0.log" if not uart_path.is_file(): - uart_path = Path(log_dir) / "uart.log" + uart_path = Path(run_log) / "uart.log" if not uart_path.is_file(): ctx.logger.error(f"uart log missing under {log_dir}") await _fail(ctx, origin_tid, "uart_missing", model=model, log_dir=log_dir) @@ -315,7 +315,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await _fail(ctx, origin_tid, "accuracy_unparsable", model=model) return - cycle_dir = Path(log_dir) / "trace" / "cycle" + cycle_dir = Path(run_log) / "trace" / "cycle" if not cycle_dir.is_dir(): ctx.logger.error(f"cycle trace dir missing: {cycle_dir}") await _fail(ctx, origin_tid, "cycle_trace_missing", @@ -339,7 +339,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.error(str(e)) await _fail(ctx, origin_tid, "perfetto_inputs_missing", model=model) return - trace_dir = Path(log_dir) / "trace" + trace_dir = Path(run_log) / "trace" perf_cmd = _perfetto_cmd( trace_dir, pinputs["trace_toml"], pinputs["mlir_files"] ) @@ -366,7 +366,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await _fail(ctx, origin_tid, "perfetto_cycles_failed", model=model) return - dest = Path(log_dir) / f"perfetto-{model}.json" + dest = Path(run_log) / f"perfetto-{model}.json" dest.write_text(json.dumps(perf_data, indent=2)) model_results.append({ "name": model, diff --git a/bbdev/api/steps/regression/03_eval_area_power_api.step.py b/bbdev/api/steps/regression/03_eval_area_power_api.step.py index 664b32511..117f49f90 100644 --- a/bbdev/api/steps/regression/03_eval_area_power_api.step.py +++ b/bbdev/api/steps/regression/03_eval_area_power_api.step.py @@ -1,9 +1,6 @@ -import os - from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip -from utils.path import check_dc_rtl_args, get_buckyball_path +from utils.event_common import require_chip config = { "name": "regression-eval-area-power-api", @@ -16,23 +13,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - try: - chip = require_chip(body) - subset = {"chip": chip} - if "top" in body: - subset["top"] = body.get("top") - check_dc_rtl_args(subset) - except ValueError as exc: - return ApiResponse(status=400, body={"error": str(exc)}) - - bbdir = get_buckyball_path() - tapeout = os.path.join(bbdir, "examples", "chips", chip, "tapeout") - if not os.path.isdir(tapeout): - return ApiResponse( - status=400, - body={"error": f"chip {chip} has no tapeout directory: {tapeout}"}, - ) - + chip = require_chip(body) data = { "chip": chip, "_trace_id": ctx.trace_id, @@ -40,6 +21,5 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: top = body.get("top") if isinstance(top, str) and top: data["top"] = top - await ctx.enqueue({"topic": "regression.eval-area-power", "data": data}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/regression/03_eval_area_power_event.step.py b/bbdev/api/steps/regression/03_eval_area_power_event.step.py index c7a98a302..b619dc5f3 100644 --- a/bbdev/api/steps/regression/03_eval_area_power_event.step.py +++ b/bbdev/api/steps/regression/03_eval_area_power_event.step.py @@ -7,7 +7,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import require_chip +from utils.event_common import require_chip from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path diff --git a/bbdev/api/steps/uvm/00_ball_verilog_api.step.py b/bbdev/api/steps/uvm/00_ball_verilog_api.step.py new file mode 100644 index 000000000..76e2df8ca --- /dev/null +++ b/bbdev/api/steps/uvm/00_ball_verilog_api.step.py @@ -0,0 +1,39 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +config = { + "name": "uvm-ball-verilog-api", + "description": "generate isolated ball verilog for UVM", + "flows": ["uvm"], + "triggers": [api("POST", "/uvm/verilog")], + "enqueues": ["uvm.verilog"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + + if body.get("chip") or ( + isinstance(body.get("config"), str) and body.get("config") and body.get("config") != "None" + ): + return ApiResponse( + status=400, + body={"error": "uvm ball verilog does not take --chip or --config"}, + ) + + balltype = body.get("balltype") + if not isinstance(balltype, str) or not balltype: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --balltype"}, + ) + + output_dir = body.get("output_dir") + if not isinstance(output_dir, str) or not output_dir: + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --output-dir"}, + ) + + data = {"balltype": balltype, "output_dir": output_dir} + await ctx.enqueue({"topic": "uvm.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/uvm/00_ball_verilog_event.step.py b/bbdev/api/steps/uvm/00_ball_verilog_event.step.py new file mode 100644 index 000000000..a93c56e17 --- /dev/null +++ b/bbdev/api/steps/uvm/00_ball_verilog_event.step.py @@ -0,0 +1,80 @@ +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path +from utils.stream_run import stream_run_logger_async +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "uvm-ball-verilog", + "description": "generate isolated ball verilog for UVM", + "flows": ["uvm"], + "triggers": [queue("uvm.verilog")], + "enqueues": [], +} + + +def _balltype(data: dict) -> str: + balltype = data.get("balltype") + if not isinstance(balltype, str) or not balltype: + raise ValueError("Missing required parameter: --balltype") + return balltype + + +def _output_dir(data: dict) -> str: + output_dir = data.get("output_dir") + if not isinstance(output_dir, str) or not output_dir: + raise ValueError("Missing required parameter: --output-dir") + return output_dir + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + + try: + balltype = _balltype(input_data) + build_dir = _output_dir(input_data) + except ValueError as error: + ctx.logger.error(str(error)) + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "validation", + "error": str(error), + "example": 'bbdev uvm --verilog "--balltype ReluBall --output-dir "', + }, + trace_id=origin_tid, + ) + return failure_result + + os.makedirs(build_dir, exist_ok=True) + command = ( + f"mill -i __.test.runMain sims.verify.BallTopMain {balltype} " + "--disable-annotation-unknown --strip-debug-info -O=release " + f"--split-verilog -o={build_dir} " + ) + ctx.logger.info(f"Using build directory: {build_dir}") + result = await stream_run_logger_async( + cmd=command, + logger=ctx.logger, + cwd=f"{bbdir}/arch", + stdout_prefix="uvm ball verilog", + stderr_prefix="uvm ball verilog", + ) + + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={"task": "verilog", "balltype": balltype, "output_dir": build_dir}, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/vcs/01_clean_api.step.py b/bbdev/api/steps/vcs/01_clean_api.step.py index 2e5529a39..71adc3ff5 100644 --- a/bbdev/api/steps/vcs/01_clean_api.step.py +++ b/bbdev/api/steps/vcs/01_clean_api.step.py @@ -1,7 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip -from utils.path import get_buckyball_path, get_vcs_build_dir +from utils.event_common import require_chip +from utils.path import get_buckyball_path, rtl_dir config = { @@ -25,7 +25,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: "topic": "vcs.clean", "data": { "chip": chip, - "output_dir": get_vcs_build_dir(bbdir, chip, body.get("output_dir")), + "output_dir": rtl_dir(bbdir, chip, "verilog", body.get("output_dir")), "_trace_id": ctx.trace_id, }, } diff --git a/bbdev/api/steps/vcs/01_clean_event.step.py b/bbdev/api/steps/vcs/01_clean_event.step.py index 26a4e1534..7b5b0b35c 100644 --- a/bbdev/api/steps/vcs/01_clean_event.step.py +++ b/bbdev/api/steps/vcs/01_clean_event.step.py @@ -9,7 +9,7 @@ sys.path.insert(0, utils_path) from utils.event_common import check_result, get_origin_trace_id -from utils.path import get_buckyball_path, get_vcs_build_dir +from utils.path import get_buckyball_path, rtl_dir config = { @@ -24,7 +24,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - build_dir = get_vcs_build_dir(bbdir, input_data.get("config"), input_data.get("output_dir")) + build_dir = rtl_dir(bbdir, input_data.get("config", "verilog"), input_data.get("output_dir")) artifact_dir = os.path.join(build_dir, "vcs") if os.path.isdir(artifact_dir): shutil.rmtree(artifact_dir) diff --git a/bbdev/api/steps/vcs/02_verilog_api.step.py b/bbdev/api/steps/vcs/02_verilog_api.step.py index 7ac3e782b..029129fab 100644 --- a/bbdev/api/steps/vcs/02_verilog_api.step.py +++ b/bbdev/api/steps/vcs/02_verilog_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip +from utils.event_common import require_chip config = { diff --git a/bbdev/api/steps/vcs/03_build_api.step.py b/bbdev/api/steps/vcs/03_build_api.step.py index a2c8916d5..c1f5d788b 100644 --- a/bbdev/api/steps/vcs/03_build_api.step.py +++ b/bbdev/api/steps/vcs/03_build_api.step.py @@ -1,7 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip -from utils.path import get_buckyball_path, get_vcs_build_dir +from utils.event_common import require_chip +from utils.path import get_buckyball_path, rtl_dir config = { @@ -30,7 +30,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "chip": chip, "jobs": jobs, - "output_dir": get_vcs_build_dir(bbdir, chip, body.get("output_dir")), + "output_dir": rtl_dir(bbdir, chip, "verilog", body.get("output_dir")), } await ctx.enqueue({"topic": "vcs.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/03_build_event.step.py b/bbdev/api/steps/vcs/03_build_event.step.py index 1b6e1d8a1..7c7ca9aef 100644 --- a/bbdev/api/steps/vcs/03_build_event.step.py +++ b/bbdev/api/steps/vcs/03_build_event.step.py @@ -13,8 +13,8 @@ sys.path.insert(0, utils_path) from utils.event_common import check_result, get_origin_trace_id -from utils.chip import require_chip -from utils.path import get_buckyball_path, get_vcs_build_dir +from utils.event_common import require_chip +from utils.path import get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async @@ -130,7 +130,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return bbdir = get_buckyball_path() - build_dir = get_vcs_build_dir(bbdir, chip, input_data.get("output_dir")) + build_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("output_dir")) vsrcs = sorted( path for path in ( glob.glob(f"{build_dir}/**/*.v", recursive=True) diff --git a/bbdev/api/steps/vcs/04_sim_api.step.py b/bbdev/api/steps/vcs/04_sim_api.step.py index bc935db45..4f17357e2 100644 --- a/bbdev/api/steps/vcs/04_sim_api.step.py +++ b/bbdev/api/steps/vcs/04_sim_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip +from utils.event_common import require_chip from utils.path import get_buckyball_path diff --git a/bbdev/api/steps/vcs/04_sim_event.step.py b/bbdev/api/steps/vcs/04_sim_event.step.py index 41a52e3cb..b19cbef27 100644 --- a/bbdev/api/steps/vcs/04_sim_event.step.py +++ b/bbdev/api/steps/vcs/04_sim_event.step.py @@ -11,8 +11,8 @@ sys.path.insert(0, utils_path) from utils.event_common import check_result, get_origin_trace_id -from utils.chip import require_chip -from utils.path import get_buckyball_path, get_run_log_dir, get_vcs_build_dir, workload_build_dir, workload_tests_root +from utils.event_common import require_chip +from utils.path import get_buckyball_path, log_dir, rtl_dir, workload_build_dir, workload_tests_root from utils.search_workload import search_workload from utils.stream_run import stream_run_logger_async @@ -55,26 +55,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if binary is None: await check_result(ctx, 1, extra_fields={"task": "sim", "error": f"binary not found: {binary_name}"}, trace_id=origin_tid) return - build_dir = get_vcs_build_dir(bbdir, chip, input_data.get("output_dir")) + build_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("output_dir")) artifact_dir = Path(build_dir) / "vcs" simv = artifact_dir / "simv" - if not simv.is_file(): - await check_result(ctx, 1, extra_fields={"task": "sim", "error": f"VCS executable missing: {simv}; run bbdev vcs --build first"}, trace_id=origin_tid) - return - timestamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") - log_dir = Path(get_run_log_dir( + run_log = Path(log_dir( bbdir, chip, "verilog", timestamp, "vcs", binary_name, input_data.get("output_dir"), )) - log_dir.mkdir(parents=True) + run_log.mkdir(parents=True) timeout_ns = input_data.get("timeout_ns", input_data.get("timeout-ns", "100000000")) - try: - if float(timeout_ns) <= 0: - raise ValueError - except (TypeError, ValueError): - await check_result(ctx, 1, extra_fields={"task": "sim", "error": "timeout-ns must be a positive number"}, trace_id=origin_tid) - return + if float(timeout_ns) <= 0: + raise ValueError("timeout-ns must be a positive number") wave = input_data.get("waveform", input_data.get("wave")) wave_arg = "+vcd" if str(wave).lower() == "vcd" else ("+vpd" if wave else "") command = " ".join( @@ -100,6 +92,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result( ctx, result.returncode, - extra_fields={"task": "sim", "simv": str(simv), "binary": binary, "log_dir": str(log_dir)}, + extra_fields={"task": "sim", "simv": str(simv), "binary": binary, "log_dir": str(run_log)}, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/vcs/05_run_api.step.py b/bbdev/api/steps/vcs/05_run_api.step.py index eeb77a842..2b2494a8e 100644 --- a/bbdev/api/steps/vcs/05_run_api.step.py +++ b/bbdev/api/steps/vcs/05_run_api.step.py @@ -1,7 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip -from utils.path import get_buckyball_path, get_vcs_build_dir +from utils.event_common import require_chip +from utils.path import get_buckyball_path, rtl_dir config = { @@ -35,7 +35,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: "binary": binary, "batch": bool(body.get("batch", False)), "jobs": jobs, - "output_dir": get_vcs_build_dir(bbdir, chip, body.get("output_dir")), + "output_dir": rtl_dir(bbdir, chip, "verilog", body.get("output_dir")), "from_run_workflow": True, } await ctx.enqueue({"topic": "vcs.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) diff --git a/bbdev/api/steps/vcs/README.md b/bbdev/api/steps/vcs/README.md index 3ff698e8e..f4de374c9 100644 --- a/bbdev/api/steps/vcs/README.md +++ b/bbdev/api/steps/vcs/README.md @@ -3,6 +3,9 @@ VCS follows the same user-facing stages as Verilator while keeping its own simulator products below `arch/build//vcs/`: +`vcs.verilog` is implemented under `steps/mill/02_vcs_verilog_event.step.py` +(same mill elaboration as Verilator). Flow: + ```text vcs.verilog → vcs.build → vcs.sim ``` diff --git a/bbdev/api/steps/verilator/01_clean_api.step.py b/bbdev/api/steps/verilator/01_clean_api.step.py index 77c6291fd..2376aa29a 100644 --- a/bbdev/api/steps/verilator/01_clean_api.step.py +++ b/bbdev/api/steps/verilator/01_clean_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip +from utils.event_common import require_chip from utils.path import get_buckyball_path config = { diff --git a/bbdev/api/steps/verilator/01_clean_event.step.py b/bbdev/api/steps/verilator/01_clean_event.step.py index 06ae1ad90..5f4550cb3 100644 --- a/bbdev/api/steps/verilator/01_clean_event.step.py +++ b/bbdev/api/steps/verilator/01_clean_event.step.py @@ -8,8 +8,8 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import require_chip -from utils.path import get_buckyball_path, rtl_dir_for_clean +from utils.event_common import require_chip +from utils.path import get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -38,7 +38,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return bbdir = get_buckyball_path() - build_dir = rtl_dir_for_clean( + build_dir = rtl_dir( bbdir, chip, "verilog", input_data.get("output_dir"), diff --git a/bbdev/api/steps/verilator/02_verilog_api.step.py b/bbdev/api/steps/verilator/02_verilog_api.step.py index 67ade991e..395fbfe45 100644 --- a/bbdev/api/steps/verilator/02_verilog_api.step.py +++ b/bbdev/api/steps/verilator/02_verilog_api.step.py @@ -1,10 +1,10 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip +from utils.event_common import require_chip config = { "name": "verilator-verilog-api", - "description": "generate verilog code", + "description": "generate chip verilog code", "flows": ["verilator"], "triggers": [api("POST", "/verilator/verilog")], "enqueues": ["verilator.verilog"], @@ -15,37 +15,27 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} if body.get("balltype"): - if body.get("chip") or ( - isinstance(body.get("config"), str) and body.get("config") and body.get("config") != "None" - ): - return ApiResponse( - status=400, - body={"error": "balltype mill does not take --chip or --config"}, - ) - output_dir = body.get("output_dir") - if not output_dir: - return ApiResponse( - status=400, - body={"error": "balltype mill requires output_dir"}, - ) - data = { - "balltype": body.get("balltype"), - "output_dir": output_dir, - } - else: - try: - chip = require_chip(body) - except ValueError as e: - return ApiResponse( - status=400, - body={ - "status": "error", - "message": str(e), - "example": 'bbdev verilator --verilog "--chip toy"', - }, - ) - data = {"chip": chip} - if body.get("output_dir"): - data["output_dir"] = body["output_dir"] + return ApiResponse( + status=400, + body={ + "error": "ball-only verilog belongs to uvm; use bbdev uvm --verilog", + }, + ) + + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse( + status=400, + body={ + "status": "error", + "message": str(e), + "example": 'bbdev verilator --verilog "--chip toy"', + }, + ) + + data = {"chip": chip} + if body.get("output_dir"): + data["output_dir"] = body["output_dir"] await ctx.enqueue({"topic": "verilator.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/03_build_api.step.py b/bbdev/api/steps/verilator/03_build_api.step.py index 7771a78a4..023bbbc16 100644 --- a/bbdev/api/steps/verilator/03_build_api.step.py +++ b/bbdev/api/steps/verilator/03_build_api.step.py @@ -1,7 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip -from utils.path import get_buckyball_path, get_verilator_build_dir +from utils.event_common import require_chip +from utils.path import get_buckyball_path, rtl_dir config = { "name": "verilator-build-api", @@ -22,7 +22,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "chip": chip, "jobs": body.get("jobs", 16), - "output_dir": get_verilator_build_dir(bbdir, chip, body.get("output_dir")), + "output_dir": rtl_dir(bbdir, chip, "verilog", body.get("output_dir")), } await ctx.enqueue({"topic": "verilator.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/03_build_event.step.py b/bbdev/api/steps/verilator/03_build_event.step.py index a52261b8b..0d634cc49 100644 --- a/bbdev/api/steps/verilator/03_build_event.step.py +++ b/bbdev/api/steps/verilator/03_build_event.step.py @@ -10,8 +10,8 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import require_chip -from utils.path import gcc_lib_dir, get_buckyball_path, get_verilator_build_dir +from utils.event_common import require_chip +from utils.path import get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -29,9 +29,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: chip = require_chip(input_data) bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" - build_dir = get_verilator_build_dir( - bbdir, chip, - input_data.get("output_dir"), + build_dir = rtl_dir( + bbdir, chip, "verilog", input_data.get("output_dir"), ) coverage = input_data.get("coverage", False) ctx.logger.info(f"Using build directory: {build_dir}") @@ -68,9 +67,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ================================================================================== result_dir = f"{bbdir}/result" result_lib = f"{result_dir}/lib" - if not os.path.isdir(result_lib): - raise RuntimeError(f"missing {result_lib}; run nix build") - inc_flags = " ".join([ f"-I{result_dir}/include", f"-I{build_dir}", @@ -81,8 +77,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # BDB NDJSON trace (+trace=...) is runtime-only; bbdev sim uses +trace=all (04_sim_event.step.py). cflags = f"{inc_flags} -DBBSIM -DTOP_NAME='\"V{topname}\"' -std=c++17" + def _gcc_lib_dir(soname: str) -> str: + printed = subprocess.run( + ["g++", f"-print-file-name={soname}"], + capture_output=True, + text=True, + check=True, + ) + return os.path.dirname(os.path.realpath(printed.stdout.strip())) + rpath_dirs = [] - for lib_dir in (result_lib, gcc_lib_dir("liblz4.so"), gcc_lib_dir("libstdc++.so")): + for lib_dir in (result_lib, _gcc_lib_dir("liblz4.so"), _gcc_lib_dir("libstdc++.so")): if lib_dir not in rpath_dirs: rpath_dirs.append(lib_dir) rpath_flags = " ".join(f"-Wl,-rpath,{lib_dir}" for lib_dir in rpath_dirs) diff --git a/bbdev/api/steps/verilator/04_sim_api.step.py b/bbdev/api/steps/verilator/04_sim_api.step.py index 1f1281efd..e9cf26916 100644 --- a/bbdev/api/steps/verilator/04_sim_api.step.py +++ b/bbdev/api/steps/verilator/04_sim_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip +from utils.event_common import require_chip from utils.path import get_buckyball_path config = { diff --git a/bbdev/api/steps/verilator/04_sim_event.step.py b/bbdev/api/steps/verilator/04_sim_event.step.py index 81609d872..961cb9080 100644 --- a/bbdev/api/steps/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -10,8 +10,8 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import require_chip -from utils.path import gcc_lib_dir, get_buckyball_path, get_run_log_dir, get_verilator_build_dir, workload_tests_root +from utils.event_common import require_chip +from utils.path import get_buckyball_path, log_dir, rtl_dir, workload_tests_root from utils.stream_run import stream_run_logger_async from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id @@ -32,9 +32,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) chip = require_chip(input_data) bbdir = get_buckyball_path() - build_dir = get_verilator_build_dir( - bbdir, chip, - input_data.get("output_dir"), + build_dir = rtl_dir( + bbdir, chip, "verilog", input_data.get("output_dir"), ) ctx.logger.info(f"Using build directory: {build_dir}") @@ -42,56 +41,31 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: binary_name = input_data.get("binary", "") if not binary_name: - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "binary_required"}, - trace_id=origin_tid, - ) - return + raise ValueError("binary required") coverage = input_data.get("coverage", False) workload_root = workload_tests_root(bbdir, chip) binary_path = search_workload(workload_root, binary_name) ctx.logger.info(f"binary_path: {binary_path}") - if binary_path is None: - ctx.logger.error(f"binary not found: {binary_name}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={ - "error": "binary_not_found", - "binary": binary_name, - "search_dir": workload_root, - }, - trace_id=origin_tid, - ) - return topname = "BBSimHarness" - log_dir = get_run_log_dir(bbdir, chip, "verilog", timestamp, "verilator", binary_name, input_data.get("output_dir")) - waveform_dir = os.path.join(log_dir, "waveform") + run_log = log_dir(bbdir, chip, "verilog", timestamp, "verilator", binary_name, input_data.get("output_dir")) + waveform_dir = os.path.join(run_log, "waveform") - os.makedirs(log_dir, exist_ok=True) + os.makedirs(run_log, exist_ok=True) os.makedirs(waveform_dir, exist_ok=True) coverage_flag = "" if coverage: - coverage_dat_path = f"{log_dir}/coverage.dat" + coverage_dat_path = f"{run_log}/coverage.dat" coverage_flag = f"+verilator+coverage+file+{coverage_dat_path}" bin_path = f"{build_dir}/obj_dir/V{topname}" - if not os.path.isfile(bin_path): - ctx.logger.error(f"simulator binary not found: {bin_path}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "simulator_not_found", "simulator": bin_path}, - trace_id=origin_tid, - ) - return batch = input_data.get("batch", False) - log_path = f"{log_dir}/bdb.ndjson" - stdout_path = f"{log_dir}/stdout.log" - meta_path = f"{log_dir}/sim_meta.txt" + log_path = f"{run_log}/bdb.ndjson" + stdout_path = f"{run_log}/stdout.log" + meta_path = f"{run_log}/sim_meta.txt" fst_path = f"{waveform_dir}/waveform.fst" # ================================================================================== @@ -104,9 +78,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # BDB_SIM_META moves NDJSON banner to sim_meta.txt so it does not pollute disasm. # ================================================================================== result_lib = f"{bbdir}/result/lib" - if not os.path.isdir(result_lib): - raise RuntimeError(f"missing {result_lib}; run nix build") - lib_dirs = [result_lib, gcc_lib_dir("liblz4.so"), gcc_lib_dir("libstdc++.so")] + def _gcc_lib_dir(soname: str) -> str: + printed = subprocess.run( + ["g++", f"-print-file-name={soname}"], + capture_output=True, + text=True, + check=True, + ) + return os.path.dirname(os.path.realpath(printed.stdout.strip())) + lib_dirs = [result_lib, _gcc_lib_dir("liblz4.so"), _gcc_lib_dir("libstdc++.so")] ld_lib_path = ":".join(dict.fromkeys(lib_dirs)) ctx.logger.info(f"LD_LIBRARY_PATH prefix: {ld_lib_path}") sim_cmd = ( @@ -142,7 +122,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields = { "task": "sim", "binary": binary_path, - "log_dir": log_dir, + "log_dir": run_log, "waveform_dir": waveform_dir, "timestamp": timestamp, "sim_meta": meta_path, diff --git a/bbdev/api/steps/verilator/05_run_api.step.py b/bbdev/api/steps/verilator/05_run_api.step.py index 39283ce12..c04c198fb 100644 --- a/bbdev/api/steps/verilator/05_run_api.step.py +++ b/bbdev/api/steps/verilator/05_run_api.step.py @@ -7,7 +7,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import require_chip +from utils.event_common import require_chip from utils.path import get_buckyball_path, workload_tests_root from utils.search_workload import search_workload diff --git a/bbdev/api/steps/workload/00_clean_api.step.py b/bbdev/api/steps/workload/00_clean_api.step.py index 53aa4a440..b0c9503f5 100644 --- a/bbdev/api/steps/workload/00_clean_api.step.py +++ b/bbdev/api/steps/workload/00_clean_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip +from utils.event_common import require_chip config = { "name": "workload-clean-api", diff --git a/bbdev/api/steps/workload/00_clean_event.step.py b/bbdev/api/steps/workload/00_clean_event.step.py index 9621177fb..29e1d6846 100644 --- a/bbdev/api/steps/workload/00_clean_event.step.py +++ b/bbdev/api/steps/workload/00_clean_event.step.py @@ -8,7 +8,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import require_chip +from utils.event_common import require_chip from utils.event_common import check_result, get_origin_trace_id from utils.path import chip_output_root, get_buckyball_path diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index bb8ddf110..a3eec8148 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -10,10 +10,12 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.build import build_workload from utils.path import get_buckyball_path from utils.event_common import check_result, get_origin_trace_id +sys.path.insert(0, os.path.join(get_buckyball_path(), "bb-tests", "workloads", "scripts")) +import build as workload_build # noqa: E402 + config = { "name": "workload-build", "description": "build workload", @@ -157,7 +159,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return try: - build_workload( + workload_build.build_workload( bbdir, chip, model=model.lower() if model else "", diff --git a/bbdev/api/steps/workload/02_tohex_api.step.py b/bbdev/api/steps/workload/02_tohex_api.step.py index 1400909bf..b87b73f76 100644 --- a/bbdev/api/steps/workload/02_tohex_api.step.py +++ b/bbdev/api/steps/workload/02_tohex_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip +from utils.event_common import require_chip config = { "name": "workload-tohex-api", diff --git a/bbdev/api/steps/workload/02_tohex_event.step.py b/bbdev/api/steps/workload/02_tohex_event.step.py index c6260ea98..c7ef314cd 100644 --- a/bbdev/api/steps/workload/02_tohex_event.step.py +++ b/bbdev/api/steps/workload/02_tohex_event.step.py @@ -9,7 +9,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import require_chip +from utils.event_common import require_chip from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path, workload_tests_root from utils.stream_run import stream_run_logger_async diff --git a/bbdev/api/steps/yosys/01_run_api.step.py b/bbdev/api/steps/yosys/01_run_api.step.py index fe2385bd7..667f52b8f 100644 --- a/bbdev/api/steps/yosys/01_run_api.step.py +++ b/bbdev/api/steps/yosys/01_run_api.step.py @@ -3,7 +3,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip +from utils.event_common import require_chip scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: diff --git a/bbdev/api/steps/yosys/02_verilog_api.step.py b/bbdev/api/steps/yosys/02_verilog_api.step.py index c927410a9..ed0a5a0d4 100644 --- a/bbdev/api/steps/yosys/02_verilog_api.step.py +++ b/bbdev/api/steps/yosys/02_verilog_api.step.py @@ -3,7 +3,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip +from utils.event_common import require_chip scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: diff --git a/bbdev/api/steps/yosys/03_synth_api.step.py b/bbdev/api/steps/yosys/03_synth_api.step.py index 1b3cd9bc9..7e393a9d3 100644 --- a/bbdev/api/steps/yosys/03_synth_api.step.py +++ b/bbdev/api/steps/yosys/03_synth_api.step.py @@ -3,8 +3,8 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.chip import require_chip -from utils.path import get_arch_build_dir, get_buckyball_path +from utils.event_common import require_chip +from utils.path import rtl_dir, get_buckyball_path scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: @@ -26,7 +26,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} try: chip = require_chip(body) - rtl = get_arch_build_dir(bbdir, chip, req_arg(body, "output_dir")) + rtl = rtl_dir(bbdir, chip, "synth", req_arg(body, "output_dir")) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) diff --git a/bbdev/api/steps/yosys/03_synth_event.step.py b/bbdev/api/steps/yosys/03_synth_event.step.py index 014507117..2aee9db3f 100644 --- a/bbdev/api/steps/yosys/03_synth_event.step.py +++ b/bbdev/api/steps/yosys/03_synth_event.step.py @@ -10,8 +10,8 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.chip import require_chip -from utils.path import get_arch_build_dir, get_buckyball_path, get_run_log_dir +from utils.event_common import require_chip +from utils.path import rtl_dir, get_buckyball_path, log_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -66,7 +66,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() try: chip = require_chip(input_data) - build_dir = get_arch_build_dir(bbdir, chip, input_data.get("output_dir")) + build_dir = rtl_dir(bbdir, chip, "synth", input_data.get("output_dir")) except ValueError as exc: await check_result( ctx, 1, continue_run=False, @@ -106,7 +106,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return failure_result stamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") - yosys_output_dir = input_data.get("log_dir") or get_run_log_dir( + yosys_output_dir = input_data.get("log_dir") or log_dir( bbdir, chip, "synth", stamp, "yosys", top_module, input_data.get("output_dir"), ) os.makedirs(yosys_output_dir, exist_ok=True) diff --git a/bbdev/api/tests/test_api_regression_buildbitstream.py b/bbdev/api/tests/test_api_regression_buildbitstream.py index c434898d9..78158fd9d 100644 --- a/bbdev/api/tests/test_api_regression_buildbitstream.py +++ b/bbdev/api/tests/test_api_regression_buildbitstream.py @@ -123,7 +123,7 @@ def test_event_sets_processing_and_enqueues_verilog(): def _patch_verilog(mod, monkeypatch, rc=0): monkeypatch.setattr(mod, "get_buckyball_path", lambda: "/tmp/bb") monkeypatch.setattr( - mod, "get_verilator_build_dir", lambda bb, cfg, out: out or "/tmp/bb/arch/build/cfg" + mod, "rtl_dir", lambda bb, cfg, out: out or "/tmp/bb/arch/build/cfg" ) monkeypatch.setattr(mod, "cleanup_strays", lambda *a, **k: None) monkeypatch.setattr(mod, "normalize_p2e_timescale", lambda *a, **k: None) @@ -198,7 +198,7 @@ def test_verilog_failure_with_flag_does_not_enqueue(monkeypatch): def _patch_buildbitstream(mod, monkeypatch): monkeypatch.setattr(mod, "get_buckyball_path", lambda: "/tmp/bb") - monkeypatch.setattr(mod, "get_verilator_build_dir", lambda bb, cfg, vsrc: vsrc or "/tmp/vsrc") + monkeypatch.setattr(mod, "rtl_dir", lambda bb, cfg, vsrc: vsrc or "/tmp/vsrc") monkeypatch.setattr(mod.os.path, "isdir", lambda p: True) monkeypatch.setattr(mod.os.path, "exists", lambda p: True) monkeypatch.setattr(mod.os, "makedirs", lambda *a, **k: None) diff --git a/bbdev/api/tests/test_api_regression_check.py b/bbdev/api/tests/test_api_regression_check.py index fc0fd47dc..665c3c446 100644 --- a/bbdev/api/tests/test_api_regression_check.py +++ b/bbdev/api/tests/test_api_regression_check.py @@ -150,7 +150,7 @@ def _prep_batch_case(tmp_path): def _patch_batch(mod, monkeypatch, tmp_path, vsrc, stdout="", stderr="", rc=0): monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) - monkeypatch.setattr(mod, "get_verilator_build_dir", lambda *a, **k: vsrc) + monkeypatch.setattr(mod, "rtl_dir", lambda *a, **k: vsrc) monkeypatch.setattr(mod, "regression_workload_toml", lambda *a, **k: "/tmp/workloads-pk.toml") def fake_stream(**k): diff --git a/bbdev/api/tests/test_api_regression_eval_performance_event.py b/bbdev/api/tests/test_api_regression_eval_performance_event.py index ea2a1999f..a1c11991d 100644 --- a/bbdev/api/tests/test_api_regression_eval_performance_event.py +++ b/bbdev/api/tests/test_api_regression_eval_performance_event.py @@ -66,7 +66,7 @@ def _patch_paths(mod, monkeypatch, tmp_path, vsrc): monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) monkeypatch.setattr(mod, "resolve_chip_compiler_core", lambda bb, chip: SimpleNamespace(name="pebbleCore")) - monkeypatch.setattr(mod, "get_verilator_build_dir", + monkeypatch.setattr(mod, "rtl_dir", lambda bb, cfg, out: out or str(vsrc)) diff --git a/bbdev/api/utils/build.py b/bbdev/api/utils/build.py deleted file mode 100644 index f314a94ea..000000000 --- a/bbdev/api/utils/build.py +++ /dev/null @@ -1,371 +0,0 @@ -from __future__ import annotations - -import os -import re -import subprocess -import sys -import tempfile -from pathlib import Path - -from utils.path import bebop_cargo_env, workload_build_dir - -_MODELS: dict[str, tuple[str, str]] = { - "lenet": ("lenet", "buddy-buckyball-lenet-run"), - "mobilenet": ("mobilenetv3", "buddy-buckyball-mobilenetv3-run"), - "resnet": ("resnet18", "buddy-buckyball-resnet-run"), - "yolo": ("yolo26", "buddy-buckyball-yolo26-run"), - "bert": ("bert", "buddy-buckyball-bert-run"), - "qwen3": ("qwen3", "buddy-buckyball-qwen3-run"), - "gemma4": ("gemma4", "buddy-buckyball-gemma4-run"), - "deepseekr1": ("deepseekr1", "buddy-buckyball-deepseekr1-run"), - "llama2": ("llama2", "buddy-buckyball-llama2-run"), - "stable-diffusion": ("stablediffusion", "buddy-buckyball-stable-diffusion-run"), - "whisper": ("whisper", "buddy-buckyball-whisper-run"), - "buddynext": ("buddynext", "buddy-buckyball-buddynext-all-run"), -} - -_RUSHB: dict[str, dict[str, str]] = { - "lenet": { - "bemu": "buddy-buckyball-lenet-rushB-bemu-run", - "verilator": "buddy-buckyball-lenet-rushB-verilator-run", - }, - "mobilenet": { - "bemu": "buddy-buckyball-mobilenetv3-rushB-bemu-run", - "verilator": "buddy-buckyball-mobilenetv3-rushB-verilator-run", - }, - "resnet": { - "bemu": "buddy-buckyball-resnet-rushB-bemu-run", - "verilator": "buddy-buckyball-resnet-rushB-verilator-run", - }, - "yolo": { - "bemu": "buddy-buckyball-yolo26-rushB-bemu-run", - "verilator": "buddy-buckyball-yolo26-rushB-verilator-run", - }, - "bert": { - "bemu": "buddy-buckyball-bert-rushB-bemu-run", - "verilator": "buddy-buckyball-bert-rushB-verilator-run", - }, - "qwen3": { - "bemu": "buddy-buckyball-qwen3-rushB-bemu-run", - "verilator": "buddy-buckyball-qwen3-rushB-verilator-run", - }, - "gemma4": { - "bemu": "buddy-buckyball-gemma4-rushB-bemu-run", - "verilator": "buddy-buckyball-gemma4-rushB-verilator-run", - }, - "deepseekr1": { - "bemu": "buddy-buckyball-deepseekr1-rushB-bemu-run", - "verilator": "buddy-buckyball-deepseekr1-rushB-verilator-run", - }, - "llama2": { - "bemu": "buddy-buckyball-llama2-rushB-bemu-run", - "verilator": "buddy-buckyball-llama2-rushB-verilator-run", - }, - "stable-diffusion": { - "bemu": "buddy-buckyball-stable-diffusion-rushB-bemu-run", - "verilator": "buddy-buckyball-stable-diffusion-rushB-verilator-run", - }, - "whisper": { - "bemu": "buddy-buckyball-whisper-rushB-bemu-run", - "verilator": "buddy-buckyball-whisper-rushB-verilator-run", - }, - "buddynext": { - "bemu": "buddy-buckyball-buddynext-rushB-bemu-run", - "verilator": "buddy-buckyball-buddynext-rushB-verilator-run", - }, -} - - -def _repo(bbdir: str) -> Path: - root = Path(bbdir).resolve() - if not root.is_dir(): - raise ValueError(f"repo does not exist: {root}") - return root - - -def _configparse(bbdir: str) -> Path: - path = _repo(bbdir) / "bazel" / "configparse" - if not path.is_dir(): - raise ValueError(f"missing configparse: {path}") - return path - - -def _pythonpath(bbdir: str) -> dict[str, str]: - env = os.environ.copy() - parse = _configparse(bbdir) - prev = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = f"{parse}:{prev}" if prev else str(parse) - return env - - -def _run( - cmd: list[str], - *, - bbdir: str, - cwd: Path | None = None, - extra_env: dict[str, str] | None = None, -) -> None: - env = _pythonpath(bbdir) - if extra_env: - env.update(extra_env) - result = subprocess.run(cmd, cwd=cwd or _repo(bbdir), env=env) - if result.returncode != 0: - raise RuntimeError(f"command failed ({result.returncode}): {' '.join(cmd)}") - - -def _require_riscv() -> Path: - raw = os.environ.get("RISCV", "") - if not raw: - raise RuntimeError("RISCV is unset; enter nix develop") - root = Path(raw) - if not root.is_dir(): - raise RuntimeError(f"RISCV is not a directory: {root}") - return root - - -def install_bundle(bbdir: str, chip: str) -> None: - if not re.fullmatch(r"[A-Za-z0-9_-]+", chip): - raise ValueError(f"invalid chip: {chip}") - script = _configparse(bbdir) / "chip_bundle.py" - _run([sys.executable, str(script), "--repo", str(_repo(bbdir)), "--chip", chip, "--all"], bbdir=bbdir) - - -def _compiler_core(bbdir: str, chip: str | None, core: str | None) -> str: - if core: - return core - if not chip: - raise ValueError("chip or core is required") - sys.path.insert(0, str(_configparse(bbdir))) - from chip_bundle import chip_index # type: ignore - - picked = chip_index(_repo(bbdir), chip).get("compilerCore", "") - if not isinstance(picked, str) or not picked: - raise ValueError(f"chip {chip} has no compilerCore") - return picked - - -def build_compiler(bbdir: str, *, chip: str | None = None, core: str | None = None) -> None: - root = _repo(bbdir) - picked = _compiler_core(bbdir, chip, core) - buddy = root / "compiler" / "thirdparty" / "buddy-mlir" - dialects = root / "examples" / "cores" / picked / "compiler" - llvm = buddy / "llvm" / "build" - build_root = buddy / "build" - build = build_root / "cores" / picked - if not (dialects / "CMakeLists.txt").is_file(): - raise RuntimeError(f"missing compiler package: {dialects / 'CMakeLists.txt'}") - if not (llvm / "lib" / "cmake" / "mlir").is_dir(): - raise RuntimeError(f"missing LLVM/MLIR cmake at {llvm}") - python = sys.executable - py_include = subprocess.check_output( - [python, "-c", "import sysconfig; print(sysconfig.get_path('include'))"], - text=True, - ).strip() - py_lib = subprocess.check_output( - [ - python, - "-c", - "import sysconfig; print(sysconfig.get_config_var('LIBDIR') + '/' + sysconfig.get_config_var('LDLIBRARY'))", - ], - text=True, - ).strip() - cache = llvm / "CMakeCache.txt" - nanobind = subprocess.check_output( - ["sed", "-n", r"s/^NB_DIR:INTERNAL=\(.*\)$/\1\/cmake/p", str(cache)], - text=True, - ).strip() - if not (Path(nanobind) / "nanobind-config.cmake").is_file(): - raise RuntimeError(f"missing nanobind cmake package: {nanobind}") - build.mkdir(parents=True, exist_ok=True) - cmake = [ - "cmake", - "-G", - "Ninja", - "-S", - str(buddy), - "-B", - str(build), - f"-DBUDDY_EXTERNAL_DIALECTS_DIR={dialects}", - f"-DMLIR_DIR={llvm / 'lib' / 'cmake' / 'mlir'}", - f"-DLLVM_DIR={llvm / 'lib' / 'cmake' / 'llvm'}", - "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DCMAKE_BUILD_TYPE=RELEASE", - "-DBUDDY_MLIR_ENABLE_PYTHON_PACKAGES=ON", - f"-Dnanobind_DIR={nanobind}", - f"-DPython3_EXECUTABLE={python}", - f"-DPython3_INCLUDE_DIR={py_include}", - f"-DPython3_LIBRARY={py_lib}", - f"-DPython_EXECUTABLE={python}", - f"-DPython_INCLUDE_DIR={py_include}", - f"-DPython_LIBRARY={py_lib}", - ] - _run(cmake, bbdir=bbdir) - _run( - ["ninja", "-C", str(build), "-j", str(os.cpu_count() or 1), "buddy-opt", "buddy-translate", "buddy-llc", "python-package-buddy", "BuddyMLIRPythonModules"], - bbdir=bbdir, - ) - public_bin = build_root / "bin" - public_bin.mkdir(parents=True, exist_ok=True) - for tool in ("buddy-opt", "buddy-translate", "buddy-llc"): - link = public_bin / f"{tool}-{picked}" - if link.is_symlink(): - link.unlink() - link.symlink_to(Path("..") / "cores" / picked / "bin" / tool) - - -def _ninja_target(model: str, rushb: str | None) -> tuple[str, str]: - cmake_model = "" - ninja_arg = "" - if model: - if model not in _MODELS: - raise ValueError(f"unknown workload model: {model}") - cmake_model, ninja_arg = _MODELS[model] - if rushb: - mapped = _RUSHB.get(model, {}).get(rushb) - if not mapped: - raise ValueError(f"no rushB target for model {model!r} backend {rushb!r}") - ninja_arg = mapped - elif rushb: - ninja_arg = f"rushB-{rushb}-workloads-build" - return cmake_model, ninja_arg - - -def _workload_core(bbdir: str, chip: str) -> str: - defs_py = _configparse(bbdir) / "workload_cmake_defs.py" - out = subprocess.check_output( - [sys.executable, str(defs_py), "--repo", str(_repo(bbdir)), "--chip", chip], - text=True, - env=_pythonpath(bbdir), - ) - for line in out.splitlines(): - key, _, value = line.partition("=") - if key == "BUCKYBALL_WORKLOAD_CORE": - if not value: - raise RuntimeError("workload_cmake_defs missing BUCKYBALL_WORKLOAD_CORE") - return value - raise RuntimeError("workload_cmake_defs missing BUCKYBALL_WORKLOAD_CORE") - - -def _bemu_manifest(bbdir: str, chip: str) -> str: - defs_py = _configparse(bbdir) / "workload_cmake_defs.py" - out = subprocess.check_output( - [sys.executable, str(defs_py), "--repo", str(_repo(bbdir)), "--chip", chip], - text=True, - env=_pythonpath(bbdir), - ) - for line in out.splitlines(): - key, _, value = line.partition("=") - if key == "BUCKYBALL_RUSHB_BEMU_MANIFEST": - if not value: - raise RuntimeError("workload_cmake_defs missing BUCKYBALL_RUSHB_BEMU_MANIFEST") - return value - raise RuntimeError("workload_cmake_defs missing BUCKYBALL_RUSHB_BEMU_MANIFEST") - - -def gen_ball_isa(bbdir: str, core: str) -> Path: - root = _repo(bbdir) - balldomain = root / "examples" / "cores" / core / "configs" / "balldomains" / "default.toml" - if not balldomain.is_file(): - raise RuntimeError(f"missing balldomain: {balldomain}") - out = root / "examples" / "cores" / core / "isa" / "ballISA.h" - out.parent.mkdir(parents=True, exist_ok=True) - parse = _configparse(bbdir) - with tempfile.NamedTemporaryFile(suffix=".json") as tmp: - _run( - [sys.executable, str(parse / "toml2json.py"), str(balldomain), tmp.name, "--repo", str(root)], - bbdir=bbdir, - ) - _run( - [sys.executable, str(parse / "json_to_ball_isa.py"), tmp.name, str(out)], - bbdir=bbdir, - ) - return out - - -def build_workload( - bbdir: str, - chip: str, - *, - model: str = "", - rushb: str | None = None, - stable: bool = False, -) -> None: - if not re.fullmatch(r"[A-Za-z0-9_-]+", chip): - raise ValueError(f"invalid chip: {chip}") - if rushb is not None and rushb not in {"bemu", "verilator"}: - raise ValueError(f"rushB must be bemu|verilator, got {rushb!r}") - root = _repo(bbdir) - install_bundle(bbdir, chip) - core = _workload_core(bbdir, chip) - build_compiler(bbdir, core=core) - isa = gen_ball_isa(bbdir, core) - bemu_manifest = _bemu_manifest(bbdir, chip) - _run( - ["cargo", "build", "--release", "--manifest-path", bemu_manifest, "--lib"], - bbdir=bbdir, - extra_env=bebop_cargo_env(bbdir, chip), - ) - - riscv = _require_riscv() - linux_cc = riscv / "bin" / "riscv64-unknown-linux-gnu-gcc" - linux_cxx = riscv / "bin" / "riscv64-unknown-linux-gnu-g++" - if not linux_cc.is_file() or not linux_cxx.is_file(): - raise RuntimeError(f"missing RISC-V linux toolchain under {riscv / 'bin'}") - - wl = root / "bb-tests" / "workloads" - build = Path(workload_build_dir(str(root), chip)) - defs_py = _configparse(bbdir) / "workload_cmake_defs.py" - defs = subprocess.check_output( - [sys.executable, str(defs_py), "--repo", str(root), "--chip", chip], - text=True, - env=_pythonpath(bbdir), - ) - cmake_model, ninja_arg = _ninja_target(model.lower(), rushb) - compiler_build = ( - root / "compiler" / "thirdparty" / "buddy-mlir" / "build" / "cores" / core - ) - cache = compiler_build / "CMakeCache.txt" - python_exec = subprocess.check_output( - ["sed", "-n", r"s/^Python3_EXECUTABLE:FILEPATH=//p", str(cache)], - text=True, - ).splitlines() - if not python_exec: - raise RuntimeError(f"Python3_EXECUTABLE not found in {cache}") - env = os.environ.copy() - env.update(bebop_cargo_env(bbdir, chip)) - env["PATH"] = f"{riscv / 'bin'}:{env.get('PATH', '')}" - env["RISCV"] = str(riscv) - env["CC"] = str(linux_cc) - env["CXX"] = str(linux_cxx) - env["BUDDY_MLIR_BUILD_DIR"] = str(compiler_build) - build.parent.mkdir(parents=True, exist_ok=True) - if build.is_dir(): - import shutil - - shutil.rmtree(build) - build.mkdir(parents=True) - cmake_args = [ - "cmake", - "-G", - "Ninja", - f"-DBUCKYBALL_STABLE={'ON' if stable else 'OFF'}", - f"-DBB_BALL_ISA_INCLUDE_DIR={isa.parent}", - f"-DPython3_EXECUTABLE={python_exec[0]}", - f"-DCMAKE_C_COMPILER={linux_cc}", - f"-DCMAKE_CXX_COMPILER={linux_cxx}", - ] - for line in defs.splitlines(): - if line: - cmake_args.append(f"-D{line}") - if cmake_model: - cmake_args.extend(["-DMODEL=" + cmake_model, "-DARCH=buckyball"]) - cmake_args.append("..") - result = subprocess.run(cmake_args, cwd=build, env=env) - if result.returncode != 0: - raise RuntimeError(f"cmake failed ({result.returncode})") - if ninja_arg: - result = subprocess.run(["ninja", f"-j{os.cpu_count() or 1}", ninja_arg], cwd=build, env=env) - else: - result = subprocess.run(["ninja", f"-j{os.cpu_count() or 1}"], cwd=build, env=env) - if result.returncode != 0: - raise RuntimeError(f"ninja failed ({result.returncode})") diff --git a/bbdev/api/utils/chip.py b/bbdev/api/utils/chip.py deleted file mode 100644 index 145e9ddeb..000000000 --- a/bbdev/api/utils/chip.py +++ /dev/null @@ -1,136 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -import re -import tomllib - - -@dataclass(frozen=True) -class CorePackage: - name: str - root: Path - compiler: Path | None - config: Path | None = None - - -_SKIP_CORE_DIRS = frozenset() -_CHIP_RE = re.compile(r"^[A-Za-z0-9_-]+$") -_MILL_KEYS = frozenset({"verilatorConfig", "p2eConfig"}) -_DERIVED_KEYS = frozenset({"name", "topology", "compilerCore", "bemuTileIndex"}) - - -def bebop_cargo(bbdir: str) -> Path: - manifest = Path(bbdir) / "bebop" / "Cargo.toml" - if not manifest.is_file(): - raise ValueError(f"bebop Cargo.toml does not exist: {manifest}") - return manifest - - -def bebop_bemu_cargo(bbdir: str, chip: str) -> Path: - manifest = Path(bbdir) / "examples" / "chips" / chip / "generated" / "bemu" / "Cargo.toml" - if not manifest.is_file(): - raise ValueError(f"bebop-bemu manifest does not exist: {manifest}") - return manifest - - -def require_chip(data: dict) -> str: - chip = data.get("chip") - if not isinstance(chip, str) or not chip: - raise ValueError("Missing required parameter: --chip") - extra = data.get("config") - if isinstance(extra, str) and extra and extra != "None": - raise ValueError( - "do not pass --config; mill class comes from chip.toml [chip] " - "verilatorConfig / p2eConfig" - ) - if not _CHIP_RE.fullmatch(chip): - raise ValueError(f"invalid chip: {chip}") - return chip - - -def _chip_table(data: dict, chip: str) -> dict: - if "runtime" in data: - raise ValueError( - f"{chip}: [runtime] is removed; mill class is [chip].verilatorConfig / p2eConfig" - ) - extra = sorted(k for k in data if k != "chip") - if extra: - raise ValueError(f"{chip}: unexpected top-level keys in chip.toml: {extra}") - table = data.get("chip") - if not isinstance(table, dict): - raise ValueError(f"{chip}: missing [chip] in chip.toml") - for key, value in table.items(): - if key in _DERIVED_KEYS: - raise ValueError( - f"{chip}: {key} is derived from the chip directory / topology; do not set it" - ) - if key not in _MILL_KEYS: - raise ValueError(f"{chip}: unexpected [chip].{key} in chip.toml") - if not isinstance(value, str) or not value: - raise ValueError(f"{chip}: [chip].{key} must be a non-empty string") - return table - - -def chip_toml(bbdir: str, chip: str) -> dict: - chip = require_chip({"chip": chip}) - path = Path(bbdir) / "examples" / "chips" / chip / "chip.toml" - if not path.is_file(): - raise ValueError(f"missing chip.toml: {path}") - with path.open("rb") as handle: - data = tomllib.load(handle) - if not isinstance(data, dict): - raise ValueError(f"chip.toml root must be a table: {path}") - _chip_table(data, chip) - return data - - -def chip_field(bbdir: str, chip: str, key: str) -> str: - if key not in _MILL_KEYS: - raise ValueError(f"invalid chip field: {key}") - value = chip_toml(bbdir, chip)["chip"].get(key) - if not isinstance(value, str) or not value: - raise ValueError(f"{chip}: missing [chip].{key} in chip.toml") - if "/" in value or "\\" in value or value in (".", ".."): - raise ValueError(f"{chip}: [chip].{key} is not a single path component: {value!r}") - return value - - -def available_cores(bbdir: str) -> list[str]: - roots = Path(bbdir) / "examples" / "cores" - if not roots.is_dir(): - return [] - return sorted( - path.name - for path in roots.iterdir() - if path.is_dir() - and path.name not in _SKIP_CORE_DIRS - and not path.name.startswith(".") - and (path / "configs").is_dir() - ) - - -def resolve_core(bbdir: str, name: str, require_compiler: bool = False) -> CorePackage: - root = Path(bbdir) / "examples" / "cores" / name - if not root.is_dir() or name in _SKIP_CORE_DIRS: - raise ValueError(f"Core does not exist: {name}") - if not (root / "configs").is_dir(): - raise ValueError(f"Core missing configs/: {root}") - - compiler_dir = root / "compiler" - compiler = compiler_dir if (compiler_dir / "CMakeLists.txt").is_file() else None - if require_compiler and compiler is None: - raise ValueError(f"Core has no compiler package: {name}") - - return CorePackage(name=name, root=root, compiler=compiler, config=None) - - -def available_compiler_chips(bbdir: str) -> list[str]: - chips = Path(bbdir) / "examples" / "chips" - if not chips.is_dir(): - return [] - return sorted( - path.name - for path in chips.iterdir() - if path.is_dir() and (path / "chip.toml").is_file() - ) diff --git a/bbdev/api/utils/event_common.py b/bbdev/api/utils/event_common.py index 6903277a9..d2c6eca26 100644 --- a/bbdev/api/utils/event_common.py +++ b/bbdev/api/utils/event_common.py @@ -2,8 +2,26 @@ Common utility functions for all event steps. """ +import re + from utils.process_registry import set_current_task_scope +_CHIP_RE = re.compile(r"^[A-Za-z0-9_-]+$") + + +def require_chip(data: dict) -> str: + chip = data.get("chip") + if not isinstance(chip, str) or not chip: + raise ValueError("Missing required parameter: --chip") + extra = data.get("config") + if isinstance(extra, str) and extra and extra != "None": + raise ValueError( + "do not pass --config; mill class comes from chip config sims" + ) + if not _CHIP_RE.fullmatch(chip): + raise ValueError(f"invalid chip: {chip}") + return chip + def get_origin_trace_id(input_data, ctx): """Get the origin trace_id from input_data (passed by API step) or fall back to ctx.trace_id. diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index 71f70bd2f..807a6d2a8 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -1,10 +1,10 @@ +import json import os import re import subprocess -from datetime import datetime from pathlib import Path -from utils.chip import chip_field, chip_toml +_PRODUCT = {"verilog": "verilator", "synth": "verilator", "p2e": "p2e"} def get_buckyball_path(): @@ -63,67 +63,36 @@ def bebop_cargo_env(bbdir, chip): return {"CARGO_TARGET_DIR": bebop_target_dir(bbdir, chip)} -def gcc_lib_dir(soname: str) -> str: - candidates = [] - printed = subprocess.run( - ["g++", f"-print-file-name={soname}"], - capture_output=True, - text=True, - ) - printed_path = (printed.stdout or "").strip() - if printed.returncode == 0 and printed_path and "/" in printed_path: - candidates.append(printed_path) - for directory in os.environ.get("LIBRARY_PATH", "").split(":"): - if directory: - candidates.append(os.path.join(directory, soname)) - for match in re.finditer(r"-L(\S+)", os.environ.get("NIX_LDFLAGS", "")): - candidates.append(os.path.join(match.group(1), soname)) - - runtime_names = [soname] - if soname.endswith(".so"): - runtime_names.append(soname + ".1") - - for path in candidates: - if not os.path.isfile(path): - continue - lib_dir = os.path.dirname(os.path.realpath(path)) - if any(os.path.isfile(os.path.join(lib_dir, name)) for name in runtime_names): - return lib_dir - - detail = printed_path or (printed.stderr or "").strip() or f"exit {printed.returncode}" - raise RuntimeError( - f"cannot locate {soname} (g++ -print-file-name={soname} -> {detail!r}; " - f"LIBRARY_PATH={os.environ.get('LIBRARY_PATH', '')!r}; " - f"NIX_LDFLAGS={os.environ.get('NIX_LDFLAGS', '')!r})" - ) - - -_RTL_FIELD = { - "verilog": "verilatorConfig", - "synth": "verilatorConfig", - "p2e": "p2eConfig", -} - - def chip_arch_root(bbdir, chip): return os.path.join(bbdir, "arch", "build", _chip_name(chip)) -def rtl_dir(bbdir, chip, product, output_dir=None): - if output_dir: - return output_dir - if product not in _RTL_FIELD: +def sim_name(bbdir, chip, product): + if product not in _PRODUCT: raise ValueError(f"invalid rtl product: {product}") - cfg = chip_field(bbdir, chip, _RTL_FIELD[product]) - return os.path.join(chip_arch_root(bbdir, chip), cfg) + path = ( + Path(bbdir) + / "examples" + / "chips" + / _chip_name(chip) + / "configs" + / "generated" + / "config" + / "config.json" + ) + return json.loads(path.read_text(encoding="utf-8"))["sims"][_PRODUCT[product]] -def rtl_dir_for_clean(bbdir, chip, product, output_dir=None): - return rtl_dir(bbdir, chip, product, output_dir) +def rtl_dir(bbdir, chip, product, output_dir=None): + if output_dir: + return output_dir + return os.path.join(chip_arch_root(bbdir, chip), sim_name(bbdir, chip, product)) -def rtl_config_name(bbdir, chip, product, output_dir=None): - return os.path.basename(os.path.normpath(rtl_dir(bbdir, chip, product, output_dir))) +def rtl_out(bbdir, chip, product, output_dir=None): + name = sim_name(bbdir, chip, product) + out = output_dir or os.path.join(chip_arch_root(bbdir, chip), name) + return name, out def _log_part(value, what): @@ -136,111 +105,13 @@ def _log_part(value, what): return value -def get_run_log_dir(bbdir, chip, product, stamp, tool, name, output_dir=None): +def log_dir(bbdir, chip, product, stamp, tool, name, output_dir=None): chip = _chip_name(chip) - config = _log_part(rtl_config_name(bbdir, chip, product, output_dir), "config") + config = _log_part( + os.path.basename(os.path.normpath(rtl_dir(bbdir, chip, product, output_dir))), + "config", + ) stamp = _log_part(stamp, "timestamp") tool = _log_part(tool, "tool") name = _log_part(os.path.basename(name), "name") return os.path.join(bbdir, "log", f"{stamp}-{chip}-{config}-{tool}-{name}") - - -def get_bemu_log_dir(bbdir, chip, stamp, name): - chip = _chip_name(chip) - data = chip_toml(bbdir, chip) - table = data.get("chip") - if not isinstance(table, dict): - raise ValueError(f"{chip}: missing [chip] in chip.toml") - verilator = table.get("verilatorConfig") - p2e = table.get("p2eConfig") - if isinstance(verilator, str) and verilator: - config = verilator - elif isinstance(p2e, str) and p2e: - config = p2e - else: - raise ValueError(f"{chip}: missing [chip].verilatorConfig and [chip].p2eConfig") - stamp = _log_part(stamp, "timestamp") - name = _log_part(os.path.basename(name), "name") - config = _log_part(config, "config") - return os.path.join(bbdir, "log", f"{stamp}-{chip}-{config}-bemu-{name}") - - -def get_verilator_build_dir(bbdir, chip=None, output_dir=None): - if output_dir: - return output_dir - if chip is None: - raise ValueError("missing required parameter: chip") - return rtl_dir(bbdir, chip, "verilog") - - -def get_p2e_build_dir(bbdir, chip, output_dir=None): - return rtl_dir(bbdir, chip, "p2e", output_dir) - - -def get_vcs_build_dir(bbdir, chip=None, output_dir=None): - return get_verilator_build_dir(bbdir, chip, output_dir) - - -def get_arch_build_dir(bbdir, chip, output_dir=None): - return rtl_dir(bbdir, chip, "synth", output_dir) - - -def get_dc_rtl_dir(bbdir, chip=None): - return get_arch_build_dir(bbdir, chip) - - -def get_dc_analysis_dir(bbdir, chip=None, stage="area"): - if stage not in {"area", "power"}: - raise ValueError(f"invalid DC analysis stage: {stage}") - timestamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") - return get_run_log_dir(bbdir, chip, "synth", timestamp, "dc", stage) - - -def check_dc_rtl_args(body: dict): - allowed = {"chip", "top"} - for name in body: - if name not in allowed: - raise ValueError(f"unexpected parameter: {name}") - top = body.get("top") - if top is not None and (not isinstance(top, str) or not top): - raise ValueError("top must be a non-empty module name") - if isinstance(top, str) and not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_$]*", top): - raise ValueError("top must be a legal unescaped Verilog module name") - - -def check_dc_power_args(body: dict): - allowed = {"chip", "top", "activity", "format", "strip_path", "workload", "start_ns", "end_ns", "start-ns", "end-ns"} - for name in body: - if name not in allowed: - raise ValueError(f"unexpected parameter: {name}") - common = {key: body[key] for key in ("chip", "top") if key in body} - check_dc_rtl_args(common) - activity = body.get("activity") - activity_format = body.get("format") - if activity is not None and (not isinstance(activity, str) or not activity): - raise ValueError("activity must be a non-empty path when provided") - if activity_format is not None and activity_format not in {"saif", "vcd", "fsdb"}: - raise ValueError("format must be one of saif, vcd, or fsdb") - if (activity is None) != (activity_format is None): - raise ValueError("activity and format must be provided together") - strip_path = body.get("strip_path") - if strip_path is not None and not isinstance(strip_path, str): - raise ValueError("strip_path must be a string") - for name in ("workload",): - value = body.get(name) - if value is not None and (not isinstance(value, str) or not value): - raise ValueError(f"{name} must be a non-empty path when provided") - if isinstance(value, str) and os.path.isabs(value): - raise ValueError(f"{name} must be a built ELF name, not an absolute path") - for name in ("start_ns", "end_ns", "start-ns", "end-ns"): - value = body.get(name) - if value is not None: - try: - if float(value) < 0: - raise ValueError - except (TypeError, ValueError) as exc: - raise ValueError(f"{name} must be a non-negative number") from exc - start = body.get("start_ns", body.get("start-ns")) - end = body.get("end_ns", body.get("end-ns")) - if start is not None and end is not None and float(start) >= float(end): - raise ValueError("start_ns must be smaller than end_ns") diff --git a/bbdev/api/utils/rtl.py b/bbdev/api/utils/rtl.py deleted file mode 100644 index 660cf8d1a..000000000 --- a/bbdev/api/utils/rtl.py +++ /dev/null @@ -1,122 +0,0 @@ -from __future__ import annotations - -import os -import shlex -from pathlib import Path - -from utils.chip import chip_field -from utils.path import chip_arch_root -from utils.stream_run import stream_run_logger_async - -_KIND = { - "verilog": ("verilatorConfig", "sims.verilator.Elaborate", False, "fesvr"), - "synth": ("verilatorConfig", "sims.verilator.Elaborate", True, None), - "p2e": ("p2eConfig", "sims.p2e.Elaborate", False, "p2e"), -} - - -def mill_cmd(main: str, mill_config: str, out: str, *, seq_mem: bool = False) -> str: - extra = "" - if seq_mem: - extra = ( - " -lowering-options=disallowLocalVariables" - f" --repl-seq-mem --repl-seq-mem-file={shlex.quote(os.path.join(out, 'mems.conf'))}" - ) - return ( - f"mill -i __.test.runMain {main} {mill_config} " - "--disable-annotation-unknown --strip-debug-info -O=debug " - f"--split-verilog -o={shlex.quote(out)}{extra}" - ) - - -def p2e_top_cmd(out: str) -> str: - return ( - "mill -i __.test.runMain sims.p2e.ElaborateP2ETop " - "--disable-annotation-unknown --strip-debug-info -O=debug " - f"--split-verilog -o={shlex.quote(out)}" - ) - - -def patch_fesvr(out: str, arch: str) -> None: - harness = os.path.join(arch, "BBSimHarness.sv") - if os.path.isfile(harness): - os.remove(harness) - drop = ("fesvr/memif.h", "fesvr/elfloader.h") - for name in ("mm.h", "mm.cc"): - path = os.path.join(out, name) - if not os.path.isfile(path): - continue - text = Path(path).read_text() - Path(path).write_text( - "".join(line for line in text.splitlines(True) if not any(token in line for token in drop)) - ) - - -def drop_p2e_harness(arch: str) -> None: - for name in ("P2EHarness.sv", "P2ETop.v", "P2ETopWrapper.sv"): - path = os.path.join(arch, name) - if os.path.isfile(path): - os.remove(path) - - -def normalize_timescale(out: str) -> None: - n = 0 - for path in Path(out).rglob("*"): - if path.suffix not in {".v", ".sv"}: - continue - text = path.read_text() - if "`timescale" in text: - continue - path.write_text("`timescale 1ns/1ps\n" + text) - n += 1 - return n - - -def assert_sv(out: str) -> None: - n = sum(1 for path in Path(out).rglob("*") if path.suffix in {".v", ".sv"}) - if n == 0: - raise RuntimeError(f"no .sv/.v emitted under {out}") - - -async def run_chip_mill(ctx, bbdir: str, chip: str, kind: str, prefix: str, output_dir=None): - if kind not in _KIND: - raise ValueError(f"invalid mill kind: {kind}") - arch = os.path.join(bbdir, "arch") - if not os.path.isdir(arch): - raise ValueError(f"missing {arch}") - key, main, seq_mem, post = _KIND[kind] - mill_config = chip_field(bbdir, chip, key) - out = output_dir or os.path.join(chip_arch_root(bbdir, chip), mill_config) - os.makedirs(out, exist_ok=True) - ctx.logger.info(f"Using mill config: {mill_config}") - ctx.logger.info(f"Using build directory: {out}") - result = await stream_run_logger_async( - cmd=mill_cmd(main, mill_config, out, seq_mem=seq_mem), - logger=ctx.logger, - cwd=arch, - stdout_prefix=prefix, - stderr_prefix=prefix, - ) - if result.returncode != 0: - return out, result.returncode - if post == "fesvr": - patch_fesvr(out, arch) - if post == "p2e": - top = await stream_run_logger_async( - cmd=p2e_top_cmd(out), - logger=ctx.logger, - cwd=arch, - stdout_prefix=prefix, - stderr_prefix=prefix, - ) - if top.returncode != 0: - return out, top.returncode - drop_p2e_harness(arch) - n = normalize_timescale(out) - ctx.logger.info(f"normalized timescale in {n} files") - if seq_mem: - mems = os.path.join(out, "mems.conf") - if not os.path.isfile(mems): - raise RuntimeError(f"mill seq-mem did not emit {mems}") - assert_sv(out) - return out, 0 diff --git a/bbdev/bbdev b/bbdev/bbdev index 32986b4c9..0486ef171 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -31,7 +31,7 @@ HTTP_PORT_END = 5500 WORKFLOW_COMMANDS = { "verilator": { "clean": 'Clean build directory. Args: "--chip [--output-dir ]"', - "verilog": 'Generate verilog files. Args: "--chip [--output-dir ] [--balltype ]"', + "verilog": 'Generate verilog files. Args: "--chip [--output-dir ]"', "build": 'Build simulation executable. Args: "--chip [--jobs ] [--output-dir ]"', "sim": 'Run simulation. Args: "--chip --binary [--batch] [--output-dir ]"', "run": 'Integrated build+sim+run. Args: "--chip --binary [--batch] [--jobs ] [--output-dir ]"', @@ -59,7 +59,7 @@ WORKFLOW_COMMANDS = { "runworkload": "Run workload.", }, "compiler": { - "build": 'Build compiler. Args: "--chip |--core [--stable]"', + "build": 'Build compiler. Args: "--chip "', }, "bebop-bemu": { "sim": 'Run Bebop emulator simulation. Args: "--chip --binary [--rushB] [--pk] [--disasm] [--tool-profile] [--itrace] [--mtrace]"', @@ -95,9 +95,13 @@ WORKFLOW_COMMANDS = { "synth": 'Run yosys synthesis. Args: "[--top ] [--output-dir ] [--log-dir ] [--vcd ]"', }, "uvm": { + "verilog": 'Generate isolated Ball RTL for UVM. Args: "--balltype --output-dir "', "build": 'Build a Ball UVM simulation. Args: "--ball= [--filelist ]"', "run": 'Build and run a Ball UVM simulation. Args: "--ball= [--filelist ] [--test ]"', }, + "config": { + "install": True, + }, "regression": { "buildbitstream": 'Build pebble p2e bitstream. Args: "--chip [--output-dir ]"', "check": 'P2E pk-tests. Args: "--chip --bitstream [--chip ]"', diff --git a/bbdev/mcp/common.py b/bbdev/mcp/common.py index 13d000541..b7ec6ad09 100644 --- a/bbdev/mcp/common.py +++ b/bbdev/mcp/common.py @@ -335,28 +335,36 @@ def _load_toml(path: Path) -> Dict[str, Any]: def _balldomain_path(chip: str, balldomain: Optional[str]) -> Path: chip_root = REPO / "examples" / "chips" / chip - if not chip_root.is_dir(): - raise FileNotFoundError(f"chip does not exist: {chip_root}") - - parse = REPO / "bazel" / "configparse" - if str(parse) not in sys.path: - sys.path.insert(0, str(parse)) - from chip_json import compiler_core, core_entry, load_topology - - data = load_topology(REPO, chip) - core_name = compiler_core(data) - if not core_name: + cfg = json.loads( + (REPO / "examples" / "chips" / chip / "configs" / "generated" / "config" / "config.json") + .read_text(encoding="utf-8") + ) + cores = [] + for tile in cfg["designs"]["tiles"]: + cores.extend(tile["cores"]) + seen = [] + uniq = [] + for core in cores: + key = core["balldomain"]["_file"] + if key not in seen: + seen.append(key) + uniq.append(core) + if len(uniq) != 1: raise ValueError(f"chip {chip} has no unique topology core") - core = core_entry(data, core_name) - + core = uniq[0] + core_file = Path(core["_file"]) + parts = core_file.parts + core_name = parts[parts.index("cores") + 1] core_root = REPO / "examples" / "cores" / core_name domains = core_root / "configs" / "balldomains" if balldomain is None: - bd = core.get("balldomain") - if not isinstance(bd, dict) or not isinstance(bd.get("_file"), str): - raise ValueError(f"chip {chip} core {core_name} has no balldomain") - path = (REPO / bd["_file"]).resolve() + bd = core["balldomain"] + path = Path(bd["_file"]) + if not path.is_absolute(): + path = (REPO / bd["_file"]).resolve() + else: + path = path.resolve() else: raw = Path(balldomain) if raw.is_absolute(): @@ -374,8 +382,6 @@ def _balldomain_path(chip: str, balldomain: Optional[str]) -> Path: else: path = (domains / f"{balldomain}.toml").resolve() - if not path.is_file(): - raise FileNotFoundError(f"balldomain toml does not exist: {path}") return path diff --git a/bbdev/mcp/tools/compiler_build.py b/bbdev/mcp/tools/compiler_build.py index faa82efcd..7c5a43470 100644 --- a/bbdev/mcp/tools/compiler_build.py +++ b/bbdev/mcp/tools/compiler_build.py @@ -3,22 +3,11 @@ from __future__ import annotations -from typing import Any, Dict, Optional - -from common import submit, err, fmt +from common import submit, fmt def register(mcp): @mcp.tool() - def bbdev_compiler_build( - core: Optional[str] = None, chip: Optional[str] = None, stable: bool = False - ) -> str: - """Build buddy-mlir compiler for exactly one Core or chip. POST /compiler/build.""" - if bool(core) == bool(chip): - return err("specify exactly one compiler target: core or chip") - params: Dict[str, Any] = {"stable": stable} - if core: - params["core"] = core - else: - params["chip"] = chip - return fmt(submit("/compiler/build", params)) + def bbdev_compiler_build(chip: str) -> str: + """Build buddy-mlir compiler for one chip. POST /compiler/build.""" + return fmt(submit("/compiler/build", {"chip": chip})) From 6dd077dfaf1289b4ad35bd2e4ffa6802635bbde3 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 26 Aug 2026 02:46:13 +0800 Subject: [PATCH 126/196] refactor: improve module loading and error handling in build steps for compiler and workload --- .../api/steps/bebop/bemu/03_batch_event.step.py | 2 +- .../api/steps/bebop/p2e/04_batch_event.step.py | 2 +- bbdev/api/steps/bebop/regression_harness.py | 17 +++++------------ .../bebop/verilator/06_batch_event.step.py | 2 +- bbdev/api/steps/compiler/01_build_event.step.py | 9 +++++++-- bbdev/api/steps/config/scripts/4_install.py | 12 +++++------- .../02_eval_performance_event.step.py | 10 ++++++---- bbdev/api/steps/workload/01_build_event.step.py | 15 ++++++++++++--- 8 files changed, 38 insertions(+), 31 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index bd915a685..be62f9a52 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -115,7 +115,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: shutil.rmtree(bemu_cargo_manifest.parent / "test-artifacts", ignore_errors=True) ctx.logger.info("Cleaned previous bebop test artifacts") - harness = nextest_harness_args(workload_toml, elf_root) + harness = nextest_harness_args(workload_toml, elf_root, env) nextest_cmd = ( f"nix develop -c cargo nextest run --manifest-path {shlex.quote(str(bemu_cargo_manifest))} " "--test test_bemu " diff --git a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py index a5641e4c1..b1445398e 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py @@ -165,7 +165,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: env = os.environ.copy() env.update(cargo_env) env["OUT_PATH"] = build_dir - harness = nextest_harness_args(workload_toml, elf_root, p2e_bitstream=bitstream) + harness = nextest_harness_args(workload_toml, elf_root, env, p2e_bitstream=bitstream) nextest_cmd = ( f"nix develop -c cargo {cargo_out} nextest run --release --features p2e " f"--test test_p2e --config-file \"{nextest_config}\" {harness}" diff --git a/bbdev/api/steps/bebop/regression_harness.py b/bbdev/api/steps/bebop/regression_harness.py index 3f52a8718..280bad97d 100644 --- a/bbdev/api/steps/bebop/regression_harness.py +++ b/bbdev/api/steps/bebop/regression_harness.py @@ -1,19 +1,12 @@ -import shlex - - def nextest_harness_args( workload_toml: str, bb_tests_root: str, + env: dict, *, p2e_bitstream: str | None = None, ) -> str: - args = [ - "--", - "--workload-toml", - shlex.quote(workload_toml), - "--bb-tests-root", - shlex.quote(bb_tests_root), - ] + env["BEBOP_WORKLOAD_TOML"] = workload_toml + env["BEBOP_BB_TESTS_ROOT"] = bb_tests_root if p2e_bitstream: - args.extend(["--p2e-bitstream", shlex.quote(p2e_bitstream)]) - return " ".join(args) + env["BEBOP_P2E_BITSTREAM"] = p2e_bitstream + return "" diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index eb7166a61..e6771b5c8 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -117,7 +117,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: shutil.rmtree(artifact_dir, ignore_errors=True) ctx.logger.info(f"Cleaned previous bebop test artifacts: {artifact_dir}") - harness = nextest_harness_args(workload_toml, elf_root) + harness = nextest_harness_args(workload_toml, elf_root, env) nextest_cmd = ( f"nix develop -c cargo nextest run --release --manifest-path {shlex.quote(manifest)} " f"--features {shlex.quote(features)} --test test_verilator " diff --git a/bbdev/api/steps/compiler/01_build_event.step.py b/bbdev/api/steps/compiler/01_build_event.step.py index 1359c504e..432ccc861 100644 --- a/bbdev/api/steps/compiler/01_build_event.step.py +++ b/bbdev/api/steps/compiler/01_build_event.step.py @@ -1,4 +1,5 @@ import asyncio +import importlib.util import os import re import sys @@ -11,8 +12,12 @@ from utils.path import get_buckyball_path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "scripts")) -import build as compiler_build # noqa: E402 +_compiler_build_path = os.path.join(os.path.dirname(__file__), "scripts", "build.py") +_spec = importlib.util.spec_from_file_location("compiler_build_module", _compiler_build_path) +if _spec is None or _spec.loader is None: + raise RuntimeError(f"cannot load {_compiler_build_path}") +compiler_build = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(compiler_build) from utils.event_common import check_result, get_origin_trace_id diff --git a/bbdev/api/steps/config/scripts/4_install.py b/bbdev/api/steps/config/scripts/4_install.py index 138a9b7fd..291fc7790 100644 --- a/bbdev/api/steps/config/scripts/4_install.py +++ b/bbdev/api/steps/config/scripts/4_install.py @@ -15,7 +15,6 @@ import chip_pb2 as pb # noqa: E402 _ENGINE = Path("bebop/src/nodes/bemu") -_CHIP_RS_DIR = _ENGINE / "src" _BEMU_CRATE = _ENGINE / "chip" @@ -31,7 +30,6 @@ def load_chip(pb_path: Path) -> pb.Chip: def _emit_dispatch(chip: pb.Chip, bbdir: Path, out: Path) -> None: balls = list(chip.bemu.balls) - src_dir = bbdir / _CHIP_RS_DIR if not balls: out.write_text( "use crate::inst::instruction::ExecContext;\n\n" @@ -55,13 +53,13 @@ def _emit_dispatch(chip: pb.Chip, bbdir: Path, out: Path) -> None: for ball in balls: if not ball.ball_dir.isidentifier(): raise ValueError(f"ball_dir is not a Rust identifier: {ball.ball_dir!r}") - lib = bbdir / ball.emu_lib + lib = (bbdir / ball.emu_lib).resolve() if not lib.is_file(): raise FileNotFoundError(f"missing ball emu: {lib}") - rel = Path(os.path.relpath(lib, src_dir)).as_posix() - if '"' in rel or "\\" in rel: - raise ValueError(f"ball emu path not usable in rust #[path]: {rel}") - lines.append(f'#[path = "{rel}"]') + path = str(lib) + if '"' in path or "\\" in path: + raise ValueError(f"ball emu path not usable in rust #[path]: {path}") + lines.append(f'#[path = "{path}"]') lines.append(f"mod {ball.ball_dir};") lines.append("") diff --git a/bbdev/api/steps/regression/02_eval_performance_event.step.py b/bbdev/api/steps/regression/02_eval_performance_event.step.py index ecee5c1b0..5d1deb656 100644 --- a/bbdev/api/steps/regression/02_eval_performance_event.step.py +++ b/bbdev/api/steps/regression/02_eval_performance_event.step.py @@ -80,11 +80,13 @@ def _fail(ctx, origin_tid, error, **extra): return check_result(ctx, 1, continue_run=False, extra_fields=fields, trace_id=origin_tid) -def _workload_build(bbdir, chip, model): +def _workload_build(bbdir, chip, model, logger, task_scope): model_key = model.lower() if MODEL_LAYOUT.get(model_key) is None: raise ValueError(f"Unknown model: {model}") - workload_build.build_workload(bbdir, chip, model=model_key) + workload_build.build_workload( + bbdir, chip, model=model_key, logger=logger, task_scope=task_scope + ) def _kernel_build_cmds(bbdir, model, dataset=""): @@ -206,8 +208,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"[eval-performance] model {model}: workload build") try: - _workload_build(bbdir, chip, model) - except (ValueError, RuntimeError) as e: + _workload_build(bbdir, chip, model, ctx.logger, origin_tid) + except Exception as e: ctx.logger.error(str(e)) await _fail(ctx, origin_tid, "workload_build_cmd", model=model) return diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index a3eec8148..732c22d51 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -1,3 +1,4 @@ +import importlib.util import os import sys import re @@ -13,8 +14,14 @@ from utils.path import get_buckyball_path from utils.event_common import check_result, get_origin_trace_id -sys.path.insert(0, os.path.join(get_buckyball_path(), "bb-tests", "workloads", "scripts")) -import build as workload_build # noqa: E402 +_workload_build_path = os.path.join( + get_buckyball_path(), "bb-tests", "workloads", "scripts", "build.py" +) +_spec = importlib.util.spec_from_file_location("workload_build_module", _workload_build_path) +if _spec is None or _spec.loader is None: + raise RuntimeError(f"cannot load {_workload_build_path}") +workload_build = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(workload_build) config = { "name": "workload-build", @@ -165,8 +172,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: model=model.lower() if model else "", rushb=rushb_backend, stable=stable, + logger=ctx.logger, + task_scope=origin_tid, ) - except (ValueError, RuntimeError) as error: + except Exception as error: ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, From d58c12f12ab395133c74678a5867936e846b186d Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 26 Aug 2026 02:53:31 +0800 Subject: [PATCH 127/196] chore: add protobuf to system tools in flake.nix --- bbdev/flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/bbdev/flake.nix b/bbdev/flake.nix index 039d33b78..972baad46 100644 --- a/bbdev/flake.nix +++ b/bbdev/flake.nix @@ -21,6 +21,7 @@ # System tools curl git + protobuf ]; shellHook = '' From 812230b99f72b7d97a1db08398c60b79b012d704 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 26 Aug 2026 15:32:05 +0800 Subject: [PATCH 128/196] chore: remove protobuf from system tools in flake.nix --- bbdev/flake.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/bbdev/flake.nix b/bbdev/flake.nix index 972baad46..039d33b78 100644 --- a/bbdev/flake.nix +++ b/bbdev/flake.nix @@ -21,7 +21,6 @@ # System tools curl git - protobuf ]; shellHook = '' From 7e5b4ed48b3d2e96dc8c84dcc8abb6e65bb7c8d8 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 27 Aug 2026 20:57:42 +0800 Subject: [PATCH 129/196] feat: add support for ctest and mlirtest flags in workload build and API, enhance error handling for conflicting options --- .../api/steps/bebop/bemu/02_sim_event.step.py | 4 +- .../bebop/p2e/02_buildbitstream_event.step.py | 2 + .../mill/07_bebop_p2e_verilog_event.step.py | 3 + bbdev/api/steps/uvm/scripts/uvm_common.py | 15 +- bbdev/api/steps/workload/01_buidl_api.step.py | 26 +- .../api/steps/workload/01_build_event.step.py | 38 +- bbdev/api/steps/workload/README.md | 8 + bbdev/api/tests/test_all_api_serial.py | 4 + .../test_api_regression_buildbitstream.py | 273 +------------ bbdev/api/tests/test_api_regression_check.py | 241 +---------- .../test_api_regression_eval_area_power.py | 379 +----------------- .../test_api_regression_eval_performance.py | 113 +----- ...t_api_regression_eval_performance_event.py | 312 -------------- bbdev/api/utils/p2e_timescale.py | 22 + bbdev/bbdev | 2 +- bbdev/mcp/tools/workload_build.py | 10 + 16 files changed, 137 insertions(+), 1315 deletions(-) delete mode 100644 bbdev/api/tests/test_api_regression_eval_performance_event.py create mode 100644 bbdev/api/utils/p2e_timescale.py diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index a38e2dab2..732bf1537 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -236,7 +236,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--elf", binary_path, "--log-dir", - log_dir, + run_log, ] else: cargo_args = [ @@ -251,7 +251,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--elf", binary_path, "--log-dir", - log_dir, + run_log, ] if input_data.get("pk"): cargo_args.append("--pk") diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index c3590fd75..fc953a9c5 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -18,6 +18,7 @@ from utils.event_common import require_chip from utils.path import bebop_cargo_env, get_buckyball_path, rtl_dir +from utils.p2e_timescale import normalize_p2e_timescale from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -46,6 +47,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return vsrc_dir = rtl_dir(bbdir, chip, "p2e", input_data.get("vsrc_dir")) + normalize_p2e_timescale(vsrc_dir, ctx.logger) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") build_dir = ( input_data.get("build_dir") diff --git a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py index e4317835e..2b8784bb7 100644 --- a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py +++ b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py @@ -16,6 +16,7 @@ from utils.event_common import require_chip from utils.path import get_buckyball_path from utils.path import rtl_out +from utils.p2e_timescale import normalize_p2e_timescale from utils.stream_run import stream_run_logger_async import mill as mill_run from utils.event_common import check_result, get_origin_trace_id @@ -51,6 +52,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix=prefix, ) ).returncode + if returncode == 0: + normalize_p2e_timescale(build_dir, ctx.logger) except (ValueError, RuntimeError) as error: ctx.logger.error(str(error)) await check_result( diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index da8f4c91b..0d4760115 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -75,7 +75,20 @@ def core_ball_isa_defines(bbdir: str, input_data: dict, ball: str) -> list[str]: values[mnemonic] = funct7 if set(values) != set(required): raise ValueError(f"{domain_path} missing required {ball} ISA entries") - return [f"+define+{macro}={values[mnemonic]}" for mnemonic, macro in required.items()] + defines = [f"+define+{macro}={values[mnemonic]}" for mnemonic, macro in required.items()] + if ball == "smatmul": + mappings = domain.get("ballIdMappings") + if not isinstance(mappings, list): + raise ValueError(f"ballIdMappings missing from {domain_path}") + matching = [entry for entry in mappings + if isinstance(entry, dict) and entry.get("ballName") == "SMatMulBall"] + if len(matching) != 1: + raise ValueError(f"{domain_path} must define exactly one SMatMulBall mapping") + out_bw = matching[0].get("outBW") + if not isinstance(out_bw, int) or out_bw <= 0 or out_bw > 4 or 4 % out_bw: + raise ValueError(f"invalid SMatMulBall outBW in {domain_path}") + defines.append(f"+define+SMATMUL_OUT_BW={out_bw}") + return defines def uvm_paths(bbdir: str, input_data: dict, ball_override: str | None = None) -> dict: diff --git a/bbdev/api/steps/workload/01_buidl_api.step.py b/bbdev/api/steps/workload/01_buidl_api.step.py index 4e697a02b..c998aca14 100644 --- a/bbdev/api/steps/workload/01_buidl_api.step.py +++ b/bbdev/api/steps/workload/01_buidl_api.step.py @@ -22,7 +22,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - allowed = {"chip", "model", "stable", "rushB"} + allowed = {"chip", "model", "stable", "rushB", "ctest", "mlirtest"} unknown = sorted(k for k in body if k not in allowed) if unknown: return ApiResponse( @@ -58,11 +58,35 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: status=400, body={"error": "Invalid --rushB: expected bemu or verilator"}, ) + ctest = body.get("ctest", False) + mlirtest = body.get("mlirtest", False) + if not isinstance(ctest, bool) or not isinstance(mlirtest, bool): + return ApiResponse( + status=400, + body={"error": "--ctest and --mlirtest must be boolean flags"}, + ) + if ctest and mlirtest: + return ApiResponse( + status=400, + body={"error": "--ctest and --mlirtest cannot be used together"}, + ) + if (ctest or mlirtest) and body.get("model"): + return ApiResponse( + status=400, + body={"error": "--ctest and --mlirtest cannot be used with --model"}, + ) + if (ctest or mlirtest) and rushb_backend: + return ApiResponse( + status=400, + body={"error": "--ctest and --mlirtest cannot be used with --rushB"}, + ) data = { "chip": chip, "model": body.get("model", ""), "stable": stable, "rushB": rushb_backend, + "ctest": ctest, + "mlirtest": mlirtest, } await ctx.enqueue({"topic": "workload.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index 732c22d51..385c3684d 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -76,7 +76,7 @@ def chips_for_model(bbdir: str, model_key: str) -> set[str]: async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - allowed = {"chip", "model", "stable", "rushB", "_trace_id"} + allowed = {"chip", "model", "stable", "rushB", "ctest", "mlirtest", "_trace_id"} unknown = sorted(k for k in input_data if k not in allowed) if unknown: ctx.logger.error(f"Unknown workload build parameter(s): {', '.join(unknown)}") @@ -133,6 +133,32 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + ctest = input_data.get("ctest", False) + mlirtest = input_data.get("mlirtest", False) + if not isinstance(ctest, bool) or not isinstance(mlirtest, bool): + ctx.logger.error("--ctest and --mlirtest must be boolean flags") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_test_scope"}, + trace_id=origin_tid, + ) + return + if ctest and mlirtest: + ctx.logger.error("--ctest and --mlirtest cannot be used together") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "conflicting_test_scope"}, + trace_id=origin_tid, + ) + return + if (ctest or mlirtest) and (model or rushb_backend): + ctx.logger.error("--ctest and --mlirtest cannot be used with --model or --rushB") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "test_scope_conflicts_with_workload"}, + trace_id=origin_tid, + ) + return if model: model_key = model.lower() @@ -171,6 +197,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: chip, model=model.lower() if model else "", rushb=rushb_backend, + ctest=ctest, + mlirtest=mlirtest, stable=stable, logger=ctx.logger, task_scope=origin_tid, @@ -186,7 +214,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result( ctx, 0, continue_run=False, - extra_fields={"chip": chip, "model": model, "rushB": rushb_backend}, + extra_fields={ + "chip": chip, + "model": model, + "rushB": rushb_backend, + "ctest": ctest, + "mlirtest": mlirtest, + }, trace_id=origin_tid) return diff --git a/bbdev/api/steps/workload/README.md b/bbdev/api/steps/workload/README.md index 8be5a5c41..d9e0ad086 100644 --- a/bbdev/api/steps/workload/README.md +++ b/bbdev/api/steps/workload/README.md @@ -28,6 +28,8 @@ bbdev workload --clean "--chip toy" - **`stable`** - Optional boolean flag. If set, build with stable LLVM Buckyball extensions. - **`rushB`** - Optional host-native backend: `bemu` or `verilator`. Requires `model`. It builds host-native CPU and rushB-lowered accelerator objects. +- **`ctest`** - Build CTest workloads only. Cannot be combined with `model` or `rushB`. +- **`mlirtest`** - Build MLIRTest workloads only. Cannot be combined with `model` or `rushB`. For chip workloads under paths like `*/chips/`, only the directory selected by `chip` is synced to `bb-tests/output//workloads`. @@ -47,6 +49,12 @@ bbdev workload --build "--chip pebble --model lenet --rushB verilator" # Build all workloads bbdev workload --build "--chip toy" + +# Build only CTest workloads +bbdev workload --build "--chip pebble --ctest" + +# Build only MLIRTest workloads +bbdev workload --build "--chip pebble --mlirtest" ``` **Response**: diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index 2f9843f7d..3cca9e0b7 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -38,6 +38,10 @@ def main(): "test_api_bebop_p2e_buildbitstream.py", "test_api_bebop_p2e_runworkload.py", "test_api_bebop_p2e_batch.py", + "test_api_regression_buildbitstream.py", + "test_api_regression_check.py", + "test_api_regression_eval_performance.py", + "test_api_regression_eval_area_power.py", ] for name in tests: path = test_dir / name diff --git a/bbdev/api/tests/test_api_regression_buildbitstream.py b/bbdev/api/tests/test_api_regression_buildbitstream.py index 78158fd9d..bc316bd2a 100644 --- a/bbdev/api/tests/test_api_regression_buildbitstream.py +++ b/bbdev/api/tests/test_api_regression_buildbitstream.py @@ -1,272 +1,3 @@ -import asyncio -import importlib.util -import os -import sys -from pathlib import Path -from types import SimpleNamespace +from _api_test_helper import run_bbdev_case -API = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(API)) - - -def load_step(rel: str): - path = API / "steps" / rel - name = path.name.replace(".", "_") - spec = importlib.util.spec_from_file_location(name, path) - if spec is None or spec.loader is None: - raise RuntimeError(f"cannot load step: {path}") - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - - -class FakeState: - def __init__(self): - self.store = {} - - async def set(self, scope, key, val): - self.store[(scope, key)] = val - - -class FakeCtx: - def __init__(self, trace_id="tid-origin"): - self.trace_id = trace_id - self.enqueued = [] - self.state = FakeState() - self.logger = SimpleNamespace( - info=lambda *a, **k: None, - error=lambda *a, **k: None, - ) - - async def enqueue(self, item): - self.enqueued.append(item) - - -def run(coro): - return asyncio.run(coro) - - -def test_api_missing_chip(): - mod = load_step("regression/00_buildbitstream_api.step.py") - ctx = FakeCtx() - resp = run(mod.handler(SimpleNamespace(body={"config": "sims.p2e.P2EPebbleLinuxConfig"}), ctx)) - assert resp.status == 400 - assert "chip" in resp.body["error"] - assert ctx.enqueued == [] - - -def test_api_missing_config(): - mod = load_step("regression/00_buildbitstream_api.step.py") - ctx = FakeCtx() - resp = run(mod.handler(SimpleNamespace(body={"chip": "pebble"}), ctx)) - assert resp.status == 400 - assert "config" in resp.body["error"] - assert ctx.enqueued == [] - - -def test_api_enqueues_regression_event(): - mod = load_step("regression/00_buildbitstream_api.step.py") - ctx = FakeCtx("tid-api") - resp = run( - mod.handler( - SimpleNamespace( - body={ - "chip": "pebble", - "config": "sims.p2e.P2EPebbleLinuxConfig", - "output_dir": "/tmp/bs", - } - ), - ctx, - ) - ) - assert resp.status == 202 - assert resp.body == {"trace_id": "tid-api"} - assert len(ctx.enqueued) == 1 - item = ctx.enqueued[0] - assert item["topic"] == "regression.buildbitstream" - assert item["data"]["chip"] == "pebble" - assert item["data"]["config"] == "sims.p2e.P2EPebbleLinuxConfig" - assert item["data"]["output_dir"] == "/tmp/bs" - assert item["data"]["_trace_id"] == "tid-api" - assert "from_regression_buildbitstream" not in item["data"] - - -def test_event_sets_processing_and_enqueues_verilog(): - mod = load_step("regression/00_buildbitstream_event.step.py") - ctx = FakeCtx("tid-evt") - run( - mod.handler( - { - "chip": "pebble", - "config": "sims.p2e.P2EPebbleLinuxConfig", - "output_dir": "/tmp/bs", - "_trace_id": "tid-origin", - }, - ctx, - ) - ) - assert ctx.state.store[("tid-origin", "processing")]["processing"] is True - assert ("tid-origin", "success") not in ctx.state.store - assert ("tid-origin", "failure") not in ctx.state.store - assert len(ctx.enqueued) == 1 - item = ctx.enqueued[0] - assert item["topic"] == "bebop.p2e.verilog" - data = item["data"] - assert data["from_regression_buildbitstream"] is True - assert data["chip"] == "pebble" - assert data["config"] == "sims.p2e.P2EPebbleLinuxConfig" - assert data["_trace_id"] == "tid-origin" - assert data["build_dir"] == "/tmp/bs" - assert "output_dir" not in data - - -def _patch_verilog(mod, monkeypatch, rc=0): - monkeypatch.setattr(mod, "get_buckyball_path", lambda: "/tmp/bb") - monkeypatch.setattr( - mod, "rtl_dir", lambda bb, cfg, out: out or "/tmp/bb/arch/build/cfg" - ) - monkeypatch.setattr(mod, "cleanup_strays", lambda *a, **k: None) - monkeypatch.setattr(mod, "normalize_p2e_timescale", lambda *a, **k: None) - monkeypatch.setattr(mod.os, "makedirs", lambda *a, **k: None) - monkeypatch.setattr( - mod, "stream_run_logger", lambda **k: SimpleNamespace(returncode=rc) - ) - - -def test_verilog_without_flag_finalizes(monkeypatch): - mod = load_step("mill/07_bebop_p2e_verilog_event.step.py") - _patch_verilog(mod, monkeypatch, rc=0) - ctx = FakeCtx() - run(mod.handler({"config": "sims.p2e.P2EPebbleLinuxConfig", "_trace_id": "tid-origin"}, ctx)) - assert ctx.enqueued == [] - success = ctx.state.store[("tid-origin", "success")] - assert success["body"]["success"] is True - assert success["body"]["task"] == "verilog" - assert ("tid-origin", "processing") not in ctx.state.store - - -def test_verilog_with_flag_continues_and_enqueues(monkeypatch): - mod = load_step("mill/07_bebop_p2e_verilog_event.step.py") - _patch_verilog(mod, monkeypatch, rc=0) - ctx = FakeCtx() - run( - mod.handler( - { - "chip": "pebble", - "config": "sims.p2e.P2EPebbleLinuxConfig", - "from_regression_buildbitstream": True, - "build_dir": "/tmp/bs", - "_trace_id": "tid-origin", - }, - ctx, - ) - ) - proc = ctx.state.store[("tid-origin", "processing")] - assert proc["processing"] is True - assert ("tid-origin", "success") not in ctx.state.store - assert len(ctx.enqueued) == 1 - item = ctx.enqueued[0] - assert item["topic"] == "bebop.p2e.buildbitstream" - data = item["data"] - assert data["from_regression_buildbitstream"] is True - assert data["chip"] == "pebble" - assert data["vsrc_dir"] == "/tmp/bb/arch/build/cfg" - assert data["build_dir"] == "/tmp/bs" - assert data["_trace_id"] == "tid-origin" - - -def test_verilog_failure_with_flag_does_not_enqueue(monkeypatch): - mod = load_step("mill/07_bebop_p2e_verilog_event.step.py") - _patch_verilog(mod, monkeypatch, rc=7) - ctx = FakeCtx() - run( - mod.handler( - { - "config": "sims.p2e.P2EPebbleLinuxConfig", - "from_regression_buildbitstream": True, - "_trace_id": "tid-origin", - }, - ctx, - ) - ) - assert ctx.enqueued == [] - failure = ctx.state.store[("tid-origin", "failure")] - assert failure["body"]["failure"] is True - assert failure["body"]["returncode"] == 7 - assert ("tid-origin", "success") not in ctx.state.store - - -def _patch_buildbitstream(mod, monkeypatch): - monkeypatch.setattr(mod, "get_buckyball_path", lambda: "/tmp/bb") - monkeypatch.setattr(mod, "rtl_dir", lambda bb, cfg, vsrc: vsrc or "/tmp/vsrc") - monkeypatch.setattr(mod.os.path, "isdir", lambda p: True) - monkeypatch.setattr(mod.os.path, "exists", lambda p: True) - monkeypatch.setattr(mod.os, "makedirs", lambda *a, **k: None) - monkeypatch.setattr(mod, "stream_run_logger", lambda **k: SimpleNamespace(returncode=0)) - - -def test_buildbitstream_without_flag_keeps_relative_bitstream(monkeypatch): - mod = load_step("bebop/p2e/02_buildbitstream_event.step.py") - _patch_buildbitstream(mod, monkeypatch) - ctx = FakeCtx() - run( - mod.handler( - { - "config": "sims.p2e.P2EPebbleLinuxConfig", - "vsrc_dir": "/tmp/vsrc", - "output_dir": "rel-bs", - "_trace_id": "tid-origin", - }, - ctx, - ) - ) - success = ctx.state.store[("tid-origin", "success")]["body"] - assert success["success"] is True - assert success["bitstream"] == os.path.join("rel-bs", "fpgaCompDir", "bitstream.bit") - - -def test_buildbitstream_with_flag_failure_keeps_relative_bitstream(monkeypatch): - mod = load_step("bebop/p2e/02_buildbitstream_event.step.py") - _patch_buildbitstream(mod, monkeypatch) - monkeypatch.setattr(mod, "stream_run_logger", lambda **k: SimpleNamespace(returncode=1)) - ctx = FakeCtx() - run( - mod.handler( - { - "config": "sims.p2e.P2EPebbleLinuxConfig", - "vsrc_dir": "/tmp/vsrc", - "output_dir": "rel-bs", - "from_regression_buildbitstream": True, - "_trace_id": "tid-origin", - }, - ctx, - ) - ) - failure = ctx.state.store[("tid-origin", "failure")]["body"] - assert failure["failure"] is True - assert failure["bitstream"] == os.path.join("rel-bs", "fpgaCompDir", "bitstream.bit") - assert not os.path.isabs(failure["bitstream"]) - - -def test_buildbitstream_with_flag_absolutes_bitstream(monkeypatch): - mod = load_step("bebop/p2e/02_buildbitstream_event.step.py") - _patch_buildbitstream(mod, monkeypatch) - ctx = FakeCtx() - run( - mod.handler( - { - "config": "sims.p2e.P2EPebbleLinuxConfig", - "vsrc_dir": "/tmp/vsrc", - "output_dir": "rel-bs", - "from_regression_buildbitstream": True, - "_trace_id": "tid-origin", - }, - ctx, - ) - ) - success = ctx.state.store[("tid-origin", "success")]["body"] - assert success["success"] is True - assert success["bitstream"] == os.path.abspath( - os.path.join("rel-bs", "fpgaCompDir", "bitstream.bit") - ) - assert os.path.isabs(success["bitstream"]) +run_bbdev_case("bbdev regression --buildbitstream '--chip pebble'") diff --git a/bbdev/api/tests/test_api_regression_check.py b/bbdev/api/tests/test_api_regression_check.py index 665c3c446..9fae0db63 100644 --- a/bbdev/api/tests/test_api_regression_check.py +++ b/bbdev/api/tests/test_api_regression_check.py @@ -1,239 +1,4 @@ -import asyncio -import importlib.util -import json -import sys -from pathlib import Path -from types import SimpleNamespace +from _api_test_helper import run_bbdev_case -API = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(API)) - - -def load_step(rel: str): - path = API / "steps" / rel - name = path.name.replace(".", "_") - spec = importlib.util.spec_from_file_location(name, path) - if spec is None or spec.loader is None: - raise RuntimeError(f"cannot load step: {path}") - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - - -class FakeState: - def __init__(self): - self.store = {} - - async def set(self, scope, key, val): - self.store[(scope, key)] = val - - -class FakeCtx: - def __init__(self, trace_id="tid-origin"): - self.trace_id = trace_id - self.enqueued = [] - self.state = FakeState() - self.logger = SimpleNamespace( - info=lambda *a, **k: None, - error=lambda *a, **k: None, - ) - - async def enqueue(self, item): - self.enqueued.append(item) - - -def run(coro): - return asyncio.run(coro) - - -def test_api_missing_chip(tmp_path): - bs = tmp_path / "bitstream.bit" - bs.write_text("x") - mod = load_step("regression/01_check_api.step.py") - ctx = FakeCtx() - resp = run(mod.handler(SimpleNamespace(body={"bitstream": str(bs)}), ctx)) - assert resp.status == 400 - assert "chip" in resp.body["error"] - assert ctx.enqueued == [] - - -def test_api_missing_bitstream(): - mod = load_step("regression/01_check_api.step.py") - ctx = FakeCtx() - resp = run(mod.handler(SimpleNamespace(body={"chip": "pebble"}), ctx)) - assert resp.status == 400 - assert "bitstream" in resp.body["error"] - assert ctx.enqueued == [] - - -def test_api_bitstream_not_found(): - mod = load_step("regression/01_check_api.step.py") - ctx = FakeCtx() - resp = run( - mod.handler( - SimpleNamespace(body={"chip": "pebble", "bitstream": "/no/such/bitstream.bit"}), - ctx, - ) - ) - assert resp.status == 400 - assert "not found" in resp.body["error"] - assert ctx.enqueued == [] - - -def test_api_enqueues_regression_event(tmp_path): - bs = tmp_path / "bitstream.bit" - bs.write_text("x") - mod = load_step("regression/01_check_api.step.py") - ctx = FakeCtx("tid-api") - resp = run( - mod.handler( - SimpleNamespace(body={"chip": "pebble", "bitstream": str(bs)}), - ctx, - ) - ) - assert resp.status == 202 - assert resp.body == {"trace_id": "tid-api"} - assert len(ctx.enqueued) == 1 - item = ctx.enqueued[0] - assert item["topic"] == "regression.check" - assert item["data"]["chip"] == "pebble" - assert item["data"]["bitstream"] == str(bs) - assert item["data"]["_trace_id"] == "tid-api" - assert "from_regression_check" not in item["data"] - - -def test_event_sets_processing_and_enqueues_batch(tmp_path): - bs = tmp_path / "bitstream.bit" - bs.write_text("x") - mod = load_step("regression/01_check_event.step.py") - ctx = FakeCtx("tid-evt") - run( - mod.handler( - { - "chip": "pebble", - "bitstream": str(bs), - "_trace_id": "tid-origin", - }, - ctx, - ) - ) - assert ctx.state.store[("tid-origin", "processing")]["processing"] is True - assert ("tid-origin", "success") not in ctx.state.store - assert ("tid-origin", "failure") not in ctx.state.store - assert len(ctx.enqueued) == 1 - item = ctx.enqueued[0] - assert item["topic"] == "bebop.p2e.batch" - data = item["data"] - assert data["from_regression_check"] is True - assert data["test"] == "pk-tests" - assert data["chip"] == "pebble" - assert data["bitstream"] == str(bs) - assert data["_trace_id"] == "tid-origin" - - -def _prep_batch_case(tmp_path): - build = tmp_path / "case" - fpga = build / "fpgaCompDir" - fpga.mkdir(parents=True) - bs = fpga / "bitstream.bit" - bs.write_text("x") - rt = build / "vvacDir" / "runtimeDir" - rt.mkdir(parents=True) - (rt / "rtcfg").write_text("x") - lib = rt / "lib" / "lib_arm" - lib.mkdir(parents=True) - (lib / "libvCtb.so").write_text("x") - vsrc = tmp_path / "vsrc" - vsrc.mkdir() - return str(bs), str(vsrc) - - -def _patch_batch(mod, monkeypatch, tmp_path, vsrc, stdout="", stderr="", rc=0): - monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) - monkeypatch.setattr(mod, "rtl_dir", lambda *a, **k: vsrc) - monkeypatch.setattr(mod, "regression_workload_toml", lambda *a, **k: "/tmp/workloads-pk.toml") - - def fake_stream(**k): - prefix = k.get("stdout_prefix", "") - if prefix == "bebop p2e batch": - return SimpleNamespace(returncode=rc, stdout=stdout, stderr=stderr) - return SimpleNamespace(returncode=0, stdout="", stderr="") - - monkeypatch.setattr(mod, "stream_run_logger", fake_stream) - - -def test_batch_without_flag_skips_accuracy(tmp_path, monkeypatch): - bs, vsrc = _prep_batch_case(tmp_path) - mod = load_step("bebop/p2e/04_batch_event.step.py") - _patch_batch( - mod, monkeypatch, tmp_path, vsrc, - stdout="Summary [ 1.2s] 4 tests run: 3 passed, 1 failed", - ) - ctx = FakeCtx() - run( - mod.handler( - { - "chip": "pebble", - "bitstream": bs, - "test": "pk-tests", - "_trace_id": "tid-origin", - }, - ctx, - ) - ) - success = ctx.state.store[("tid-origin", "success")]["body"] - assert success["success"] is True - assert "accuracy" not in success - assert not (tmp_path / "chipcrowd-eval-result.json").exists() - - -def test_batch_flag_merges_accuracy(tmp_path, monkeypatch): - bs, vsrc = _prep_batch_case(tmp_path) - mod = load_step("bebop/p2e/04_batch_event.step.py") - _patch_batch( - mod, monkeypatch, tmp_path, vsrc, - stdout="Summary [ 1.2s] 4 tests run: 3 passed, 1 failed", - rc=1, - ) - ctx = FakeCtx() - run( - mod.handler( - { - "chip": "pebble", - "bitstream": bs, - "test": "pk-tests", - "from_regression_check": True, - "_trace_id": "tid-origin", - }, - ctx, - ) - ) - failure = ctx.state.store[("tid-origin", "failure")]["body"] - assert failure["failure"] is True - assert failure["returncode"] == 1 - assert failure["accuracy"] == 0.75 - data = json.loads((tmp_path / "chipcrowd-eval-result.json").read_text()) - assert data["accuracy"] == 0.75 - - -def test_batch_flag_unparsable_fails(tmp_path, monkeypatch): - bs, vsrc = _prep_batch_case(tmp_path) - mod = load_step("bebop/p2e/04_batch_event.step.py") - _patch_batch(mod, monkeypatch, tmp_path, vsrc, stdout="no summary here", rc=0) - ctx = FakeCtx() - run( - mod.handler( - { - "chip": "pebble", - "bitstream": bs, - "test": "pk-tests", - "from_regression_check": True, - "_trace_id": "tid-origin", - }, - ctx, - ) - ) - failure = ctx.state.store[("tid-origin", "failure")]["body"] - assert failure["failure"] is True - assert failure["error"] == "accuracy_unparsable" - assert not (tmp_path / "chipcrowd-eval-result.json").exists() +# You need to set the bitstream path. +# run_bbdev_case("bbdev regression --check '--chip pebble --bitstream /path/to/bitstream.bit'") diff --git a/bbdev/api/tests/test_api_regression_eval_area_power.py b/bbdev/api/tests/test_api_regression_eval_area_power.py index 19e9243bc..e303dc2bd 100644 --- a/bbdev/api/tests/test_api_regression_eval_area_power.py +++ b/bbdev/api/tests/test_api_regression_eval_area_power.py @@ -1,378 +1,3 @@ -import asyncio -import importlib.util -import json -import sys -from pathlib import Path -from types import SimpleNamespace +from _api_test_helper import run_bbdev_case -API = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(API)) - -PEBBLE_CFG = "sims.verilator.BuckyballPebbleVerilatorConfig" - - -def load_step(rel: str): - path = API / "steps" / rel - name = path.name.replace(".", "_") - spec = importlib.util.spec_from_file_location(name, path) - if spec is None or spec.loader is None: - raise RuntimeError(f"cannot load step: {path}") - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - - -class FakeState: - def __init__(self): - self.store = {} - - async def set(self, scope, key, val): - self.store[(scope, key)] = val - - -class FakeCtx: - def __init__(self, trace_id="tid-origin"): - self.trace_id = trace_id - self.enqueued = [] - self.state = FakeState() - self.logger = SimpleNamespace( - info=lambda *a, **k: None, - error=lambda *a, **k: None, - ) - - async def enqueue(self, item): - self.enqueued.append(item) - - -def run(coro): - return asyncio.run(coro) - - -def test_api_missing_chip(): - mod = load_step("regression/03_eval_area_power_api.step.py") - ctx = FakeCtx() - resp = run(mod.handler(SimpleNamespace(body={"config": PEBBLE_CFG}), ctx)) - assert resp.status == 400 - assert "chip" in resp.body["error"] - assert ctx.enqueued == [] - - -def test_api_rejects_config(): - mod = load_step("regression/03_eval_area_power_api.step.py") - ctx = FakeCtx() - resp = run(mod.handler(SimpleNamespace(body={"chip": "pebble", "config": PEBBLE_CFG}), ctx)) - assert resp.status == 400 - assert "config" in resp.body["error"] - assert ctx.enqueued == [] - - -def test_api_missing_tapeout(): - mod = load_step("regression/03_eval_area_power_api.step.py") - ctx = FakeCtx() - resp = run( - mod.handler( - SimpleNamespace(body={"chip": "toy"}), - ctx, - ) - ) - assert resp.status == 400 - assert "tapeout" in resp.body["error"] - assert ctx.enqueued == [] - - -def test_api_invalid_top(): - mod = load_step("regression/03_eval_area_power_api.step.py") - ctx = FakeCtx() - resp = run( - mod.handler( - SimpleNamespace( - body={"chip": "pebble", "top": "not a module"} - ), - ctx, - ) - ) - assert resp.status == 400 - assert "top" in resp.body["error"] - assert ctx.enqueued == [] - - -def test_api_enqueues_regression_event(): - mod = load_step("regression/03_eval_area_power_api.step.py") - ctx = FakeCtx("tid-api") - resp = run( - mod.handler( - SimpleNamespace(body={"chip": "pebble"}), - ctx, - ) - ) - assert resp.status == 202 - assert resp.body == {"trace_id": "tid-api"} - assert len(ctx.enqueued) == 1 - item = ctx.enqueued[0] - assert item["topic"] == "regression.eval-area-power" - assert item["data"]["chip"] == "pebble" - assert "config" not in item["data"] - assert item["data"]["_trace_id"] == "tid-api" - assert "from_regression_area_power" not in item["data"] - assert "from_area_workflow" not in item["data"] - - -def test_api_forward_top(): - mod = load_step("regression/03_eval_area_power_api.step.py") - ctx = FakeCtx("tid-api") - resp = run( - mod.handler( - SimpleNamespace( - body={"chip": "pebble", "top": "DigitalTop"} - ), - ctx, - ) - ) - assert resp.status == 202 - assert ctx.enqueued[0]["data"]["top"] == "DigitalTop" - - -def test_event_missing_chip(tmp_path, monkeypatch): - mod = load_step("regression/03_eval_area_power_event.step.py") - monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) - ctx = FakeCtx() - run(mod.handler({"config": PEBBLE_CFG, "_trace_id": "tid-origin"}, ctx)) - failure = ctx.state.store[("tid-origin", "failure")]["body"] - assert failure["error"] == "missing_chip" - assert ctx.enqueued == [] - - -def test_event_missing_tapeout_without_dir(tmp_path, monkeypatch): - mod = load_step("regression/03_eval_area_power_event.step.py") - monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) - ctx = FakeCtx() - run(mod.handler({"chip": "pebble", "_trace_id": "tid-origin"}, ctx)) - failure = ctx.state.store[("tid-origin", "failure")]["body"] - assert failure["error"] == "missing_tapeout" - assert ctx.enqueued == [] - - -def test_event_missing_tapeout(tmp_path, monkeypatch): - _write_pebble_chip_toml(tmp_path) - mod = load_step("regression/03_eval_area_power_event.step.py") - monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) - ctx = FakeCtx() - run( - mod.handler( - {"chip": "pebble", "_trace_id": "tid-origin"}, - ctx, - ) - ) - failure = ctx.state.store[("tid-origin", "failure")]["body"] - assert failure["error"] == "missing_tapeout" - assert ctx.enqueued == [] - - -def _write_pebble_chip_toml(root: Path) -> Path: - chip_dir = root / "examples" / "chips" / "pebble" - chip_dir.mkdir(parents=True, exist_ok=True) - (chip_dir / "chip.toml").write_text( - '[chip]\nverilatorConfig = "sims.verilator.BuckyballPebbleVerilatorConfig"\n', - encoding="utf-8", - ) - return chip_dir - - -def _prep_event_tapeout(tmp_path): - chip_dir = _write_pebble_chip_toml(tmp_path) - tapeout = chip_dir / "tapeout" - tapeout.mkdir() - return tapeout - - -def test_event_sets_processing_and_enqueues_dc_verilog(tmp_path, monkeypatch): - _prep_event_tapeout(tmp_path) - mod = load_step("regression/03_eval_area_power_event.step.py") - monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) - ctx = FakeCtx("tid-evt") - run( - mod.handler( - { - "chip": "pebble", - "top": "DigitalTop", - "_trace_id": "tid-origin", - }, - ctx, - ) - ) - assert ctx.state.store[("tid-origin", "processing")]["processing"] is True - assert ("tid-origin", "success") not in ctx.state.store - assert ("tid-origin", "failure") not in ctx.state.store - assert len(ctx.enqueued) == 1 - item = ctx.enqueued[0] - assert item["topic"] == "dc.verilog" - data = item["data"] - assert data["from_area_workflow"] is True - assert data["from_regression_area_power"] is True - assert "config" not in data - assert data["top"] == "DigitalTop" - assert "output_dir" not in data - assert "analysis_dir" not in data - assert data["_trace_id"] == "tid-origin" - - -def _patch_area(mod, monkeypatch, tmp_path, analysis_dir, rpt_text=None, rc=0): - monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) - monkeypatch.setattr(mod.shutil, "which", lambda name: "/bin/dc_shell") - monkeypatch.setattr( - mod, - "get_tapeout_contract", - lambda *a, **k: SimpleNamespace( - chip="pebble", - clock_port="clk", - clock_period_ns=10.0, - dc_script=str(tmp_path / "dc.tcl"), - root=tmp_path / "tapeout", - ), - ) - monkeypatch.setattr(mod, "technology_settings", lambda: {}) - monkeypatch.setattr(mod, "write_run_tcl", lambda *a, **k: tmp_path / "run.tcl") - - async def fake_stream(**k): - if rpt_text is not None: - rpt = Path(analysis_dir) / "reports" / "area.rpt" - rpt.parent.mkdir(parents=True, exist_ok=True) - rpt.write_text(rpt_text) - return SimpleNamespace(returncode=rc) - - monkeypatch.setattr(mod, "stream_run_logger_async", fake_stream) - - -def _area_input(tmp_path, analysis_dir, **extra): - src = tmp_path / "dc_sources.list" - src.write_text("a.sv\n") - data = { - "source_list": str(src), - "analysis_dir": str(analysis_dir), - "chip": "pebble", - "top": "DigitalTop", - "_trace_id": "tid-origin", - } - data.update(extra) - return data - - -def test_area_without_flag_skips_metrics(tmp_path, monkeypatch): - analysis_dir = tmp_path / "analysis" - mod = load_step("dc/03_area_event.step.py") - _patch_area( - mod, monkeypatch, tmp_path, analysis_dir, - rpt_text="Total cell area: 1000000.0000\n", - ) - ctx = FakeCtx() - run(mod.handler(_area_input(tmp_path, analysis_dir), ctx)) - success = ctx.state.store[("tid-origin", "success")]["body"] - assert success["success"] is True - assert "area" not in success - assert "freq" not in success - assert ctx.enqueued == [] - assert not (tmp_path / "chipcrowd-eval-result.json").exists() - - -def test_area_without_flag_enqueues_sim_for_power(tmp_path, monkeypatch): - analysis_dir = tmp_path / "analysis" - mod = load_step("dc/03_area_event.step.py") - _patch_area(mod, monkeypatch, tmp_path, analysis_dir, rpt_text="x") - ctx = FakeCtx() - run( - mod.handler( - _area_input(tmp_path, analysis_dir, from_power_workflow=True), - ctx, - ) - ) - success = ctx.state.store[("tid-origin", "success")]["body"] - assert success["success"] is True - assert len(ctx.enqueued) == 1 - assert ctx.enqueued[0]["topic"] == "dc.sim" - assert not (tmp_path / "chipcrowd-eval-result.json").exists() - - -def test_area_flag_merges_area_freq(tmp_path, monkeypatch): - analysis_dir = tmp_path / "analysis" - mod = load_step("dc/03_area_event.step.py") - _patch_area( - mod, monkeypatch, tmp_path, analysis_dir, - rpt_text="Total cell area: 1000000.0000\n", - ) - ctx = FakeCtx() - run( - mod.handler( - _area_input( - tmp_path, analysis_dir, - from_area_workflow=True, - from_regression_area_power=True, - ), - ctx, - ) - ) - success = ctx.state.store[("tid-origin", "success")]["body"] - assert success["success"] is True - assert success["area"] == 1.0 - assert success["freq"] == 100.0 - assert ctx.enqueued == [] - data = json.loads((tmp_path / "chipcrowd-eval-result.json").read_text()) - assert data["area"] == 1.0 - assert data["freq"] == 100.0 - - -def test_area_flag_does_not_enqueue_sim(tmp_path, monkeypatch): - analysis_dir = tmp_path / "analysis" - mod = load_step("dc/03_area_event.step.py") - _patch_area( - mod, monkeypatch, tmp_path, analysis_dir, - rpt_text="Total cell area: 1000000.0000\n", - ) - ctx = FakeCtx() - run( - mod.handler( - _area_input( - tmp_path, analysis_dir, - from_power_workflow=True, - from_regression_area_power=True, - ), - ctx, - ) - ) - assert ctx.enqueued == [] - assert (tmp_path / "chipcrowd-eval-result.json").is_file() - - -def test_area_flag_missing_rpt_fails(tmp_path, monkeypatch): - analysis_dir = tmp_path / "analysis" - mod = load_step("dc/03_area_event.step.py") - _patch_area(mod, monkeypatch, tmp_path, analysis_dir, rpt_text=None) - ctx = FakeCtx() - run( - mod.handler( - _area_input(tmp_path, analysis_dir, from_regression_area_power=True), - ctx, - ) - ) - failure = ctx.state.store[("tid-origin", "failure")]["body"] - assert failure["failure"] is True - assert "missing area.rpt" in failure["error"] - assert ctx.enqueued == [] - assert not (tmp_path / "chipcrowd-eval-result.json").exists() - - -def test_area_flag_unparsable_rpt_fails(tmp_path, monkeypatch): - analysis_dir = tmp_path / "analysis" - mod = load_step("dc/03_area_event.step.py") - _patch_area(mod, monkeypatch, tmp_path, analysis_dir, rpt_text="no area here\n") - ctx = FakeCtx() - run( - mod.handler( - _area_input(tmp_path, analysis_dir, from_regression_area_power=True), - ctx, - ) - ) - failure = ctx.state.store[("tid-origin", "failure")]["body"] - assert failure["failure"] is True - assert failure["error"] == "Total cell area not found" - assert ctx.enqueued == [] - assert not (tmp_path / "chipcrowd-eval-result.json").exists() +run_bbdev_case("bbdev regression --eval-area-power '--chip pebble'") diff --git a/bbdev/api/tests/test_api_regression_eval_performance.py b/bbdev/api/tests/test_api_regression_eval_performance.py index 6e2576a94..2798f1c37 100644 --- a/bbdev/api/tests/test_api_regression_eval_performance.py +++ b/bbdev/api/tests/test_api_regression_eval_performance.py @@ -1,111 +1,4 @@ -import asyncio -import importlib.util -import sys -from pathlib import Path -from types import SimpleNamespace +from _api_test_helper import run_bbdev_case -API = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(API)) - - -def load_step(rel): - path = API / "steps" / rel - name = path.name.replace(".", "_") - spec = importlib.util.spec_from_file_location(name, path) - if spec is None or spec.loader is None: - raise RuntimeError(f"cannot load step: {path}") - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - - -class FakeCtx: - def __init__(self, trace_id="tid-api"): - self.trace_id = trace_id - self.enqueued = [] - self.logger = SimpleNamespace(info=lambda *a, **k: None, - error=lambda *a, **k: None) - - async def enqueue(self, item): - self.enqueued.append(item) - - -def run(coro): - return asyncio.run(coro) - - -def test_api_missing_chip(tmp_path): - bs = tmp_path / "bitstream.bit" - bs.write_text("x") - mod = load_step("regression/02_eval_performance_api.step.py") - ctx = FakeCtx() - resp = run(mod.handler(SimpleNamespace(body={"bitstream": str(bs)}), ctx)) - assert resp.status == 400 - assert "chip" in resp.body["error"] - assert ctx.enqueued == [] - - -def test_api_missing_bitstream(): - mod = load_step("regression/02_eval_performance_api.step.py") - ctx = FakeCtx() - resp = run(mod.handler(SimpleNamespace(body={"chip": "pebble"}), ctx)) - assert resp.status == 400 - assert "bitstream" in resp.body["error"] - assert ctx.enqueued == [] - - -def test_api_bitstream_not_found(): - mod = load_step("regression/02_eval_performance_api.step.py") - ctx = FakeCtx() - resp = run( - mod.handler( - SimpleNamespace( - body={"chip": "pebble", "bitstream": "/no/such/bitstream.bit"}, - ), - ctx, - ) - ) - assert resp.status == 400 - assert "not found" in resp.body["error"] - assert ctx.enqueued == [] - - -def test_api_enqueues_regression_event(tmp_path): - bs = tmp_path / "bitstream.bit" - bs.write_text("x") - mod = load_step("regression/02_eval_performance_api.step.py") - ctx = FakeCtx("tid-api") - resp = run( - mod.handler( - SimpleNamespace(body={"chip": "pebble", "bitstream": str(bs)}), ctx) - ) - assert resp.status == 202 - assert resp.body == {"trace_id": "tid-api"} - assert len(ctx.enqueued) == 1 - item = ctx.enqueued[0] - assert item["topic"] == "regression.eval-performance" - assert item["data"]["chip"] == "pebble" - assert item["data"]["bitstream"] == str(bs) - assert item["data"]["_trace_id"] == "tid-api" - assert "config" not in item["data"] - - -def test_api_forward_config(tmp_path): - bs = tmp_path / "bitstream.bit" - bs.write_text("x") - mod = load_step("regression/02_eval_performance_api.step.py") - ctx = FakeCtx("tid-api") - resp = run( - mod.handler( - SimpleNamespace( - body={ - "chip": "pebble", - "bitstream": str(bs), - "config": "sims.p2e.P2EPebbleLinuxConfig", - } - ), - ctx, - ) - ) - assert resp.status == 202 - assert ctx.enqueued[0]["data"]["config"] == "sims.p2e.P2EPebbleLinuxConfig" +# You need to set the bitstream path. +# run_bbdev_case("bbdev regression --eval-performance '--chip pebble --bitstream /path/to/bitstream.bit'") diff --git a/bbdev/api/tests/test_api_regression_eval_performance_event.py b/bbdev/api/tests/test_api_regression_eval_performance_event.py deleted file mode 100644 index a1c11991d..000000000 --- a/bbdev/api/tests/test_api_regression_eval_performance_event.py +++ /dev/null @@ -1,312 +0,0 @@ -import asyncio -import importlib.util -import json -import os -import re -import sys -from pathlib import Path -from types import SimpleNamespace - -API = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(API)) - - -def load_step(rel): - path = API / "steps" / rel - name = path.name.replace(".", "_") - spec = importlib.util.spec_from_file_location(name, path) - if spec is None or spec.loader is None: - raise RuntimeError(f"cannot load step: {path}") - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - - -class FakeState: - def __init__(self): - self.store = {} - - async def set(self, scope, key, val): - self.store[(scope, key)] = val - - -class FakeCtx: - def __init__(self, trace_id="tid-evt"): - self.trace_id = trace_id - self.enqueued = [] - self.state = FakeState() - self.logger = SimpleNamespace(info=lambda *a, **k: None, - error=lambda *a, **k: None) - - async def enqueue(self, item): - self.enqueued.append(item) - - -def run(coro): - return asyncio.run(coro) - - -def _make_bitstream(tmp_path): - build = tmp_path / "case" - fpga = build / "fpgaCompDir" - fpga.mkdir(parents=True) - bs = fpga / "bitstream.bit" - bs.write_text("x") - rt = build / "vvacDir" / "runtimeDir" - lib = rt / "lib" / "lib_arm" - lib.mkdir(parents=True) - (lib / "libvCtb.so").write_text("x") - (rt / "rtcfg").write_text("x") - vsrc = tmp_path / "vsrc" - vsrc.mkdir() - return str(bs), str(vsrc), str(build) - - -def _patch_paths(mod, monkeypatch, tmp_path, vsrc): - monkeypatch.setattr(mod, "get_buckyball_path", lambda: str(tmp_path)) - monkeypatch.setattr(mod, "resolve_chip_compiler_core", - lambda bb, chip: SimpleNamespace(name="pebbleCore")) - monkeypatch.setattr(mod, "rtl_dir", - lambda bb, cfg, out: out or str(vsrc)) - - -def _fake_stream_factory(tmp_path): - def fake(**k): - prefix = k.get("stdout_prefix", "") - cmd = k.get("cmd", "") - rc = 0 - if prefix.startswith("workload build"): - src = (tmp_path / "bb-tests" / "output" / "workloads" / "src" - / "ModelTest" / "e2e" / "models" / "archs" / "buckyball" - / "pebble" / "LeNet") - src.mkdir(parents=True, exist_ok=True) - (src / "buddy-buckyball-lenet-run").write_text("x") - bld = (tmp_path / "bb-tests" / "build" / "workloads" / "src" - / "ModelTest" / "e2e" / "models" / "archs" / "buckyball" - / "pebble" / "LeNet") - bld.mkdir(parents=True, exist_ok=True) - (bld / "subgraph0_linalg.mlir").write_text("linalg") - (bld / "subgraph0_buckyball.mlir").write_text("buckyball") - elif prefix.startswith("kernel build"): - out = tmp_path / "bb-tests" / "output" / "kernel" - out.mkdir(parents=True, exist_ok=True) - (out / "fw_payload-lenet.bin").write_text("bin") - elif prefix.startswith("p2e run"): - m = re.search(r'--log-dir="([^"]+)"', cmd) - if m: - cycle = Path(m.group(1)) / "trace" / "cycle" - cycle.mkdir(parents=True, exist_ok=True) - (cycle / "trace-0.txt").write_text("start 10\nend 20\nelapsed 10\n") - elif prefix.startswith("perfetto"): - m = re.search(r"^(\S+) \S+ (\S+) (\S+)", cmd) - trace_dir = Path(m.group(2)) if m else None - if trace_dir: - out = trace_dir / "perfetto.json" - out.write_text(json.dumps({ - "traceEvents": [{"ph": "X", "ts": 10, "dur": 5}] - })) - return SimpleNamespace(returncode=rc, stdout="", stderr="") - return fake - - -def test_event_missing_bitstream(tmp_path, monkeypatch): - mod = load_step("regression/02_eval_performance_event.step.py") - _patch_paths(mod, monkeypatch, tmp_path, tmp_path / "vsrc") - ctx = FakeCtx() - run(mod.handler({"chip": "pebble", "bitstream": "/no/such.bit", - "_trace_id": "tid-origin"}, ctx)) - failure = ctx.state.store[("tid-origin", "failure")]["body"] - assert failure["failure"] is True - assert failure["error"] == "bitstream_not_found" - assert ctx.enqueued == [] - - -def test_event_missing_models_toml(tmp_path, monkeypatch): - bs, vsrc, _ = _make_bitstream(tmp_path) - mod = load_step("regression/02_eval_performance_event.step.py") - _patch_paths(mod, monkeypatch, tmp_path, vsrc) - ctx = FakeCtx() - run(mod.handler({"chip": "pebble", "bitstream": bs, - "_trace_id": "tid-origin"}, ctx)) - failure = ctx.state.store[("tid-origin", "failure")]["body"] - assert failure["failure"] is True - assert failure["error"] == "models_toml_invalid" - assert not (tmp_path / "chipcrowd-eval-result.json").exists() - - -def test_event_missing_chip(tmp_path, monkeypatch): - bs, vsrc, _ = _make_bitstream(tmp_path) - mod = load_step("regression/02_eval_performance_event.step.py") - _patch_paths(mod, monkeypatch, tmp_path, vsrc) - ctx = FakeCtx() - run(mod.handler({"bitstream": bs, "_trace_id": "tid-origin"}, ctx)) - failure = ctx.state.store[("tid-origin", "failure")]["body"] - assert failure["error"] == "missing_chip" - - -def _seed_models_toml(tmp_path, models=None): - if models is None: - models = ["lenet"] - d = (tmp_path / "examples" / "chips" / "pebble" / "regression" / "eval") - d.mkdir(parents=True, exist_ok=True) - (d / "models.toml").write_text( - "[eval]\nmodels = " + json.dumps(models) + "\n" - ) - - -def _seed_perfetto_sources(tmp_path): - toml = (tmp_path / "bb-tests" / "workloads" / "src" / "ModelTest" - / "e2e" / "models" / "models" / "LeNet" / "trace" - / "trace-linalg-buckyball.toml") - toml.parent.mkdir(parents=True, exist_ok=True) - toml.write_text("[trace]\n") - - -def test_event_happy_path_merges_latency(tmp_path, monkeypatch): - bs, vsrc, _ = _make_bitstream(tmp_path) - _seed_models_toml(tmp_path) - _seed_perfetto_sources(tmp_path) - mod = load_step("regression/02_eval_performance_event.step.py") - _patch_paths(mod, monkeypatch, tmp_path, vsrc) - monkeypatch.setattr(mod, "stream_run_logger", _fake_stream_factory(tmp_path)) - monkeypatch.setattr(mod._kernel, "bin_to_hex", - lambda bin_p, hex_p, base_address=0: ( - Path(hex_p).parent.mkdir(parents=True, exist_ok=True), - Path(hex_p).write_text("hex"), - True, - )[-1]) - ctx = FakeCtx() - run(mod.handler({"chip": "pebble", "bitstream": bs, - "_trace_id": "tid-origin"}, ctx)) - success = ctx.state.store[("tid-origin", "success")]["body"] - assert success["success"] is True - assert success["latency"] == 5.0 - assert len(success["models"]) == 1 - assert success["models"][0]["name"] == "lenet" - assert success["models"][0]["latency"] == 5.0 - data = json.loads((tmp_path / "chipcrowd-eval-result.json").read_text()) - assert data["latency"] == 5.0 - assert data["models"][0]["name"] == "lenet" - assert data["models"][0]["perfetto"].endswith("perfetto-lenet.json") - - -def test_event_unknown_model_fails_cleanly(tmp_path, monkeypatch): - bs, vsrc, _ = _make_bitstream(tmp_path) - _seed_models_toml(tmp_path, models=["not-a-real-model"]) - mod = load_step("regression/02_eval_performance_event.step.py") - _patch_paths(mod, monkeypatch, tmp_path, vsrc) - ctx = FakeCtx() - run(mod.handler({"chip": "pebble", "bitstream": bs, - "_trace_id": "tid-origin"}, ctx)) - failure = ctx.state.store[("tid-origin", "failure")]["body"] - assert failure["failure"] is True - assert failure["error"] == "unknown_model" - assert failure["model"] == "not-a-real-model" - assert not (tmp_path / "chipcrowd-eval-result.json").exists() - - -def test_workload_build_cmd_unknown_model_is_valueerror(tmp_path, monkeypatch): - mod = load_step("regression/02_eval_performance_event.step.py") - monkeypatch.setattr(mod, "resolve_chip_compiler_core", - lambda bb, chip: SimpleNamespace(name="pebbleCore")) - try: - mod._workload_build_cmd(str(tmp_path), "pebble", "not-a-real-model") - assert False - except ValueError as e: - assert "Unknown model" in str(e) - except KeyError: - assert False, "unknown model must raise ValueError, not KeyError" - - -def test_event_missing_model_layout_fails_cleanly(tmp_path, monkeypatch): - bs, vsrc, _ = _make_bitstream(tmp_path) - _seed_models_toml(tmp_path, models=["lenet"]) - mod = load_step("regression/02_eval_performance_event.step.py") - _patch_paths(mod, monkeypatch, tmp_path, vsrc) - monkeypatch.setattr(mod, "MODEL_LAYOUT", {}) - ctx = FakeCtx() - run(mod.handler({"chip": "pebble", "bitstream": bs, - "_trace_id": "tid-origin"}, ctx)) - failure = ctx.state.store[("tid-origin", "failure")]["body"] - assert failure["error"] == "missing_model_layout" - assert failure["model"] == "lenet" - assert not (tmp_path / "chipcrowd-eval-result.json").exists() - - -def test_event_missing_build_dir_fails_cleanly(tmp_path, monkeypatch): - bs, vsrc, build = _make_bitstream(tmp_path) - _seed_models_toml(tmp_path) - _seed_perfetto_sources(tmp_path) - mod = load_step("regression/02_eval_performance_event.step.py") - _patch_paths(mod, monkeypatch, tmp_path, vsrc) - monkeypatch.setattr(mod, "stream_run_logger", _fake_stream_factory(tmp_path)) - monkeypatch.setattr(mod._kernel, "bin_to_hex", - lambda bin_p, hex_p, base_address=0: ( - Path(hex_p).parent.mkdir(parents=True, exist_ok=True), - Path(hex_p).write_text("hex"), - True, - )[-1]) - orig_isdir = os.path.isdir - - def fake_isdir(path): - if os.path.abspath(path) == os.path.abspath(build): - return False - return orig_isdir(path) - - monkeypatch.setattr(mod.os.path, "isdir", fake_isdir) - ctx = FakeCtx() - run(mod.handler({"chip": "pebble", "bitstream": bs, - "_trace_id": "tid-origin"}, ctx)) - failure = ctx.state.store[("tid-origin", "failure")]["body"] - assert failure["error"] == "build_dir_not_found" - assert failure["model"] == "lenet" - assert failure["build_dir"] == os.path.abspath(build) - assert not (tmp_path / "chipcrowd-eval-result.json").exists() - - -def test_p2e_run_cmds_missing_build_dir(tmp_path, monkeypatch): - mod = load_step("regression/02_eval_performance_event.step.py") - hex_dir = tmp_path / "bb-tests" / "output" / "kernel" - hex_dir.mkdir(parents=True) - (hex_dir / "fw.hex").write_text("x") - fpga = tmp_path / "case" / "fpgaCompDir" - fpga.mkdir(parents=True) - bs = fpga / "bitstream.bit" - bs.write_text("x") - build = str(tmp_path / "case") - orig_isdir = os.path.isdir - - def fake_isdir(path): - if os.path.abspath(path) == os.path.abspath(build): - return False - return orig_isdir(path) - - monkeypatch.setattr(mod.os.path, "isdir", fake_isdir) - try: - mod._p2e_run_cmds(str(tmp_path), str(bs), "fw", None, {}) - assert False - except FileNotFoundError as e: - assert "P2E build case not found" in str(e) - assert build in str(e) - - -def test_event_workload_build_failure_aborts(tmp_path, monkeypatch): - bs, vsrc, _ = _make_bitstream(tmp_path) - _seed_models_toml(tmp_path) - _seed_perfetto_sources(tmp_path) - mod = load_step("regression/02_eval_performance_event.step.py") - _patch_paths(mod, monkeypatch, tmp_path, vsrc) - - def fake(**k): - prefix = k.get("stdout_prefix", "") - if prefix.startswith("workload build"): - return SimpleNamespace(returncode=1, stdout="", stderr="") - return SimpleNamespace(returncode=0, stdout="", stderr="") - monkeypatch.setattr(mod, "stream_run_logger", fake) - ctx = FakeCtx() - run(mod.handler({"chip": "pebble", "bitstream": bs, - "_trace_id": "tid-origin"}, ctx)) - failure = ctx.state.store[("tid-origin", "failure")]["body"] - assert failure["error"] == "workload_build_failed" - assert failure["model"] == "lenet" - assert not (tmp_path / "chipcrowd-eval-result.json").exists() diff --git a/bbdev/api/utils/p2e_timescale.py b/bbdev/api/utils/p2e_timescale.py new file mode 100644 index 000000000..5156875e2 --- /dev/null +++ b/bbdev/api/utils/p2e_timescale.py @@ -0,0 +1,22 @@ +import os + + +def normalize_p2e_timescale(build_dir, logger): + if not os.path.isdir(build_dir): + raise FileNotFoundError(f"P2E rtl dir not found: {build_dir}") + patched = 0 + for root, _, files in os.walk(build_dir): + for name in files: + if not name.endswith((".v", ".sv")): + continue + path = os.path.join(root, name) + with open(path) as f: + content = f.read() + if "`timescale" in content: + continue + with open(path, "w") as f: + f.write("`timescale 1ns/1ps\n") + f.write(content) + patched += 1 + logger.info(f"Normalized P2E timescale in {patched} generated Verilog files") + return patched diff --git a/bbdev/bbdev b/bbdev/bbdev index 0486ef171..933af683b 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -46,7 +46,7 @@ WORKFLOW_COMMANDS = { }, "workload": { "clean": 'Clean workload output directory. Args: "--chip "', - "build": 'Build workload. Args: "--chip [--model ] [--stable] [--rushB ]"; without --model, --rushB builds CTest and MLIR rushB runners.', + "build": 'Build workload. Args: "--chip [--model ] [--stable] [--rushB ] [--ctest|--mlirtest]"; --ctest and --mlirtest are mutually exclusive local builds and cannot be combined with --model or --rushB.', "tohex": 'Convert chip workload ELF to hex. Args: "--chip "', }, "kernel": { diff --git a/bbdev/mcp/tools/workload_build.py b/bbdev/mcp/tools/workload_build.py index 8148ae19a..f62b05fe7 100644 --- a/bbdev/mcp/tools/workload_build.py +++ b/bbdev/mcp/tools/workload_build.py @@ -13,6 +13,8 @@ def bbdev_workload_build( chip: str, model: Optional[str] = None, rushB: Optional[str] = None, + ctest: bool = False, + mlirtest: bool = False, ) -> str: """Build workloads for a chip. POST /workload/build.""" if e := need("chip", chip): @@ -24,4 +26,12 @@ def bbdev_workload_build( if rushB not in {"bemu", "verilator"}: return err("rushB must be bemu or verilator") params["rushB"] = rushB + if ctest and mlirtest: + return err("ctest and mlirtest cannot be used together") + if (ctest or mlirtest) and (model or rushB): + return err("ctest and mlirtest cannot be used with model or rushB") + if ctest: + params["ctest"] = True + if mlirtest: + params["mlirtest"] = True return fmt(submit("/workload/build", params)) From 71486824937914447b8a690b127db9217aca0e67 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 27 Aug 2026 22:36:45 +0800 Subject: [PATCH 130/196] fix: update simulation command to include LD_LIBRARY_PATH for proper runtime library configuration --- bbdev/api/steps/uvm/02_run_event.step.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbdev/api/steps/uvm/02_run_event.step.py b/bbdev/api/steps/uvm/02_run_event.step.py index 641addf8c..efc7ef1c3 100644 --- a/bbdev/api/steps/uvm/02_run_event.step.py +++ b/bbdev/api/steps/uvm/02_run_event.step.py @@ -100,7 +100,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: script = ( f"cd {shlex.quote(paths['verify_dir'])} && " - f"{shlex.quote(paths['simv'])} " + f"env LD_LIBRARY_PATH=\"$VCS_RUNTIME_LIBRARY_PATH\" {shlex.quote(paths['simv'])} " f"-sv_lib {shlex.quote(paths['dpi_lib'])} " f"+UVM_TESTNAME={shlex.quote(test)} " + " ".join(shlex.quote(word) for word in plusarg_words) From a45cfa173ce13c10cde6703ce6df7a1136d872a1 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 27 Aug 2026 22:44:16 +0800 Subject: [PATCH 131/196] feat: add MATADD support in core_ball_isa_defines for enhanced functionality --- bbdev/api/steps/uvm/scripts/uvm_common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index 0d4760115..6ac613d63 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -56,6 +56,7 @@ def core_ball_isa_defines(bbdir: str, input_data: dict, ball: str) -> list[str]: "SMATMUL_OS": "SMATMUL_OS_FUNCT7", "SMATMUL_WS": "SMATMUL_WS_FUNCT7", }, + "matadd": {"MATADD": "MATADD_FUNCT7"}, "im2col": {"IM2COL": "IM2COL_FUNCT7"}, "transpose": {"TRANSPOSE": "TRANSPOSE_FUNCT7"}, }.get(ball) From 837ddb9e68d9c2df6209edcd84bbc38db650ba13 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 28 Aug 2026 22:42:13 +0800 Subject: [PATCH 132/196] feat: add dynamic bank number definitions for chip profiles in workload installation --- bbdev/api/steps/config/scripts/4_install.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bbdev/api/steps/config/scripts/4_install.py b/bbdev/api/steps/config/scripts/4_install.py index 291fc7790..37db41402 100644 --- a/bbdev/api/steps/config/scripts/4_install.py +++ b/bbdev/api/steps/config/scripts/4_install.py @@ -139,6 +139,8 @@ def install_workload(chip: pb.Chip, bbdir: Path, name: str, gen: Path) -> Path: ), "BUCKYBALL_CHIP_PB": str(gen / "chip.pb"), } + for profile in chip.profiles: + defs[f"BUCKYBALL_BANK_NUM_{profile.name}"] = str(profile.bank_num) out = gen / "workload" / "cmake.defs" out.parent.mkdir(parents=True, exist_ok=True) body = "".join(f"{k}={v}\n" for k, v in sorted(defs.items())) From 4b88d1a0e85578961cbd5894c572e23aa898135c Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 29 Aug 2026 16:52:13 +0800 Subject: [PATCH 133/196] refactor: remove dynamic bank number definitions from workload installation process --- bbdev/api/steps/config/scripts/4_install.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bbdev/api/steps/config/scripts/4_install.py b/bbdev/api/steps/config/scripts/4_install.py index 37db41402..291fc7790 100644 --- a/bbdev/api/steps/config/scripts/4_install.py +++ b/bbdev/api/steps/config/scripts/4_install.py @@ -139,8 +139,6 @@ def install_workload(chip: pb.Chip, bbdir: Path, name: str, gen: Path) -> Path: ), "BUCKYBALL_CHIP_PB": str(gen / "chip.pb"), } - for profile in chip.profiles: - defs[f"BUCKYBALL_BANK_NUM_{profile.name}"] = str(profile.bank_num) out = gen / "workload" / "cmake.defs" out.parent.mkdir(parents=True, exist_ok=True) body = "".join(f"{k}={v}\n" for k, v in sorted(defs.items())) From 45160ae7eaa5236c6ef0407a9f8d995ae632b63c Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Mon, 31 Aug 2026 18:47:59 +0800 Subject: [PATCH 134/196] fix: select RushB RTL through bbdev (#14) * fix(verilator): select RushB RTL through bbdev * [BBDev][Verilator] Load the current RushB backend --- .../bebop/verilator/01_clean_api.step.py | 6 +++- .../bebop/verilator/01_clean_event.step.py | 8 +++++- .../bebop/verilator/02_verilog_api.step.py | 2 +- .../bebop/verilator/03_build_api.step.py | 8 +++++- .../bebop/verilator/03_build_event.step.py | 8 +++++- .../bebop/verilator/04_sim_event.step.py | 14 ++++++++-- .../bebop/verilator/06_batch_api.step.py | 4 ++- .../bebop/verilator/06_batch_event.step.py | 8 +++++- bbdev/api/steps/compiler/scripts/build.py | 28 ++++++++++++------- .../06_bebop_verilator_verilog_event.step.py | 6 +++- bbdev/api/utils/path.py | 19 +++++++++---- 11 files changed, 85 insertions(+), 26 deletions(-) diff --git a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py index 451f35fda..0a288846c 100644 --- a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py +++ b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py @@ -20,10 +20,14 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - build_dir = rtl_dir(bbdir, chip, "verilog", body.get("output_dir")) + rushb = bool(body.get("rushB", False)) + build_dir = rtl_dir( + bbdir, chip, "verilog", body.get("output_dir"), rushb=rushb + ) data = { "chip": chip, "output_dir": build_dir, + "rushB": rushb, } await ctx.enqueue({"topic": "bebop.verilator.clean", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py index 35883221d..f364a2176 100644 --- a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py +++ b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py @@ -37,7 +37,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - build_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("output_dir")) + build_dir = rtl_dir( + bbdir, + chip, + "verilog", + input_data.get("output_dir"), + rushb=bool(input_data.get("rushB", False)), + ) command = f"rm -rf {build_dir}" result = await stream_run_logger_async( diff --git a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py index 2a023d1bf..fc470fe80 100644 --- a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py +++ b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py @@ -34,7 +34,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: }, ) - data = {"chip": chip} + data = {"chip": chip, "rushB": bool(body.get("rushB", False))} if body.get("output_dir"): data["output_dir"] = body["output_dir"] await ctx.enqueue({"topic": "bebop.verilator.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_api.step.py b/bbdev/api/steps/bebop/verilator/03_build_api.step.py index 660338657..05654630a 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_api.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_api.step.py @@ -28,12 +28,18 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) + rushb = bool(body.get("rushB", False)) data = { "chip": chip, "jobs": body.get("jobs", 16), "diff": bool(body.get("diff", False)), + "rushB": rushb, "vsrc_dir": rtl_dir( - bbdir, chip, "verilog", body.get("vsrc-dir") or body.get("output-dir"), + bbdir, + chip, + "verilog", + body.get("vsrc-dir") or body.get("output-dir"), + rushb=rushb, ), } await ctx.enqueue({"topic": "bebop.verilator.build", "data": {**data, "_trace_id": ctx.trace_id}}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index 079b84091..7de3a6b45 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -39,7 +39,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bebop_dir = f"{bbdir}/bebop" chip = require_chip(input_data) - vsrc_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("vsrc_dir")) + vsrc_dir = rtl_dir( + bbdir, + chip, + "verilog", + input_data.get("vsrc_dir"), + rushb=bool(input_data.get("rushB", False)), + ) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") diff = bool(input_data.get("diff", False)) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index 642b4f6bf..a308b7233 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -74,7 +74,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - vsrc_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("vsrc_dir")) + vsrc_dir = rtl_dir( + bbdir, + chip, + "verilog", + input_data.get("vsrc_dir"), + rushb=bool(input_data.get("rushB", False)), + ) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") build_dir = bebop_dir chip = input_data.get("chip") @@ -192,7 +198,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: backend_library = f"{bebop_target_dir(bbdir, chip)}/release/deps/libbebop_verilator.so" binary_dir = os.path.dirname(binary_path) copy_cmd = shlex.join(["cmake", "-E", "copy_if_different", backend_library, binary_dir]) - inner_cmd = f"cd {shlex.quote(binary_dir)} && {copy_cmd} && exec {shlex.quote(binary_path)}" + library_env = f"LD_LIBRARY_PATH={shlex.quote(binary_dir)}:${{LD_LIBRARY_PATH:-}}" + inner_cmd = ( + f"cd {shlex.quote(binary_dir)} && {copy_cmd} && " + f"{library_env} exec {shlex.quote(binary_path)}" + ) run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" ctx.logger.info(f"Running rushB Verilator: {run_cmd}") stdout_prefix = "rushB verilator" diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py index 979bae0b4..d2b3b7ea8 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -42,7 +42,9 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body={"error": "--diff and --rushB cannot be used together"}, ) - vsrc_dir = rtl_dir(bbdir, chip, "verilog", body.get("vsrc_dir")) + vsrc_dir = rtl_dir( + bbdir, chip, "verilog", body.get("vsrc_dir"), rushb=rushB + ) data = { "chip": chip, diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index e6771b5c8..6a850df05 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -85,7 +85,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"rushB={rushB} diff={diff}" ) - vsrc_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("vsrc_dir")) + vsrc_dir = rtl_dir( + bbdir, + chip, + "verilog", + input_data.get("vsrc_dir"), + rushb=rushB, + ) vsrc_config = shlex.quote(f"env.VSRC_PATH='{vsrc_dir}'") env = os.environ.copy() env.update(bebop_cargo_env(bbdir, chip)) diff --git a/bbdev/api/steps/compiler/scripts/build.py b/bbdev/api/steps/compiler/scripts/build.py index fcafd3fad..bba660b06 100644 --- a/bbdev/api/steps/compiler/scripts/build.py +++ b/bbdev/api/steps/compiler/scripts/build.py @@ -47,14 +47,22 @@ def _run( raise RuntimeError(f"command failed ({result.returncode}): {' '.join(cmd)}") -def compiler_python() -> str: - python = shutil.which("python3") - if not python: - raise RuntimeError("python3 not in PATH; enter nix develop") - probe = subprocess.run([python, "-c", "import nanobind"], capture_output=True) - if probe.returncode != 0: - raise RuntimeError(f"python3 lacks nanobind ({python}); enter nix develop") - return python +def compiler_python(repo: str | Path) -> str: + root = _repo(repo) + candidates = [root / "result" / "bin" / "python3"] + path_python = shutil.which("python3") + if path_python: + candidates.append(Path(path_python)) + for candidate in candidates: + if not candidate.is_file(): + continue + probe = subprocess.run( + [str(candidate), "-c", "import nanobind"], capture_output=True + ) + if probe.returncode == 0: + return str(candidate) + checked = ", ".join(str(candidate) for candidate in candidates) + raise RuntimeError(f"python3 with nanobind not found (checked: {checked})") def build_llvm( @@ -67,7 +75,7 @@ def build_llvm( if not llvm_src.is_dir(): raise RuntimeError(f"missing LLVM source: {llvm_src}") - python = compiler_python() + python = compiler_python(root) cmake = [ "cmake", "-G", @@ -129,7 +137,7 @@ def build_compiler( raise RuntimeError(f"missing {chip_pb}; run bbdev config --install") buddy = root / "compiler" / "thirdparty" / "buddy-mlir" llvm = build_llvm(root, logger=logger, task_scope=task_scope) - python = compiler_python() + python = compiler_python(root) build = compiler_build_dir(root, chip) build.mkdir(parents=True, exist_ok=True) cmake = [ diff --git a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py index b3ab033af..09ee4bb3b 100644 --- a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py @@ -55,7 +55,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: try: chip = require_chip(input_data) mill_config, build_dir = rtl_out( - bbdir, chip, "verilog", input_data.get("output_dir"), + bbdir, + chip, + "verilog", + input_data.get("output_dir"), + rushb=bool(input_data.get("rushB", False)), ) os.makedirs(build_dir, exist_ok=True) ctx.logger.info(f"Using mill config: {mill_config}") diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index 807a6d2a8..e61fc89b4 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -67,7 +67,7 @@ def chip_arch_root(bbdir, chip): return os.path.join(bbdir, "arch", "build", _chip_name(chip)) -def sim_name(bbdir, chip, product): +def sim_name(bbdir, chip, product, *, rushb=False): if product not in _PRODUCT: raise ValueError(f"invalid rtl product: {product}") path = ( @@ -80,17 +80,24 @@ def sim_name(bbdir, chip, product): / "config" / "config.json" ) - return json.loads(path.read_text(encoding="utf-8"))["sims"][_PRODUCT[product]] + name = json.loads(path.read_text(encoding="utf-8"))["sims"][_PRODUCT[product]] + if not rushb: + return name + if product != "verilog" or not name.endswith("VerilatorConfig"): + raise ValueError(f"rushB RTL is not supported for product {product}: {name}") + return f"{name[:-len('VerilatorConfig')]}RushBVerilatorConfig" -def rtl_dir(bbdir, chip, product, output_dir=None): +def rtl_dir(bbdir, chip, product, output_dir=None, *, rushb=False): if output_dir: return output_dir - return os.path.join(chip_arch_root(bbdir, chip), sim_name(bbdir, chip, product)) + return os.path.join( + chip_arch_root(bbdir, chip), sim_name(bbdir, chip, product, rushb=rushb) + ) -def rtl_out(bbdir, chip, product, output_dir=None): - name = sim_name(bbdir, chip, product) +def rtl_out(bbdir, chip, product, output_dir=None, *, rushb=False): + name = sim_name(bbdir, chip, product, rushb=rushb) out = output_dir or os.path.join(chip_arch_root(bbdir, chip), name) return name, out From 79a3aa795d30b8a19b41a0e023820c755a656d27 Mon Sep 17 00:00:00 2001 From: Shiroha Date: Tue, 1 Sep 2026 05:19:12 -0400 Subject: [PATCH 135/196] feat: introduce SRAM macro generation and replacement APIs (#15) --- bbdev/.gitignore | 2 + .../bebop/p2e/03_runworkload_event.step.py | 2 +- bbdev/api/steps/compiler/scripts/build.py | 41 ++-- ...ea_event.step.py => 01_area_event.step.py} | 44 +++-- ...sim_event.step.py => 03_sim_event.step.py} | 12 +- bbdev/api/steps/dc/scripts/dc.tcl | 56 ------ bbdev/api/steps/dc/scripts/tapeout.py | 145 ++++++++++++++ bbdev/api/steps/dc/tapeout.py | 182 ----------------- bbdev/api/steps/ip-replace/01_run_api.step.py | 33 ---- .../api/steps/ip-replace/01_run_event.step.py | 124 ------------ .../steps/ip-replace/scripts/sram_replace.py | 186 ------------------ bbdev/api/steps/ip/01_generate_api.step.py | 44 +++++ bbdev/api/steps/ip/01_generate_event.step.py | 72 +++++++ bbdev/api/steps/ip/02_replace_api.step.py | 44 +++++ bbdev/api/steps/ip/02_replace_event.step.py | 116 +++++++++++ bbdev/api/steps/ip/scripts/generate.py | 148 ++++++++++++++ bbdev/api/steps/ip/scripts/macro_compiler.py | 41 ++++ bbdev/api/steps/ip/scripts/replace.py | 89 +++++++++ bbdev/api/steps/ip/scripts/sram_compiler.py | 43 ++++ .../steps/mill/03_yosys_verilog_event.step.py | 20 +- .../steps/mill/04_dc_verilog_event.step.py | 32 +-- .../01_ptpx_event.step.py} | 41 ++-- bbdev/api/steps/{dc => pt/scripts}/power.py | 0 bbdev/api/steps/vcs/04_sim_event.step.py | 2 +- bbdev/api/steps/yosys/03_synth_api.step.py | 26 ++- bbdev/api/steps/yosys/03_synth_event.step.py | 19 +- bbdev/api/steps/yosys/scripts/yosys_log.py | 8 +- bbdev/api/tests/test_api_dc_verilog.py | 13 -- bbdev/api/utils/path.py | 10 +- bbdev/bbdev | 8 +- bbdev/mcp/tools/__init__.py | 2 + bbdev/mcp/tools/ip_generate.py | 33 ++++ bbdev/mcp/tools/ip_replace.py | 29 +-- 33 files changed, 945 insertions(+), 722 deletions(-) rename bbdev/api/steps/dc/{03_area_event.step.py => 01_area_event.step.py} (76%) rename bbdev/api/steps/dc/{05_sim_event.step.py => 03_sim_event.step.py} (92%) delete mode 100755 bbdev/api/steps/dc/scripts/dc.tcl create mode 100644 bbdev/api/steps/dc/scripts/tapeout.py delete mode 100644 bbdev/api/steps/dc/tapeout.py delete mode 100644 bbdev/api/steps/ip-replace/01_run_api.step.py delete mode 100644 bbdev/api/steps/ip-replace/01_run_event.step.py delete mode 100755 bbdev/api/steps/ip-replace/scripts/sram_replace.py create mode 100644 bbdev/api/steps/ip/01_generate_api.step.py create mode 100644 bbdev/api/steps/ip/01_generate_event.step.py create mode 100644 bbdev/api/steps/ip/02_replace_api.step.py create mode 100644 bbdev/api/steps/ip/02_replace_event.step.py create mode 100644 bbdev/api/steps/ip/scripts/generate.py create mode 100644 bbdev/api/steps/ip/scripts/macro_compiler.py create mode 100644 bbdev/api/steps/ip/scripts/replace.py create mode 100644 bbdev/api/steps/ip/scripts/sram_compiler.py rename bbdev/api/steps/{dc/06_ptpx_event.step.py => pt/01_ptpx_event.step.py} (79%) rename bbdev/api/steps/{dc => pt/scripts}/power.py (100%) create mode 100644 bbdev/mcp/tools/ip_generate.py diff --git a/bbdev/.gitignore b/bbdev/.gitignore index 438bbd056..d6f53a209 100644 --- a/bbdev/.gitignore +++ b/bbdev/.gitignore @@ -22,3 +22,5 @@ result/ # Temp iii config files generated by bbdev script mode api/iii-config-*.yaml api/data/ + +api/steps/ip/scripts/_smoke_out/ diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index bd94fad91..639a6839a 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -19,7 +19,7 @@ sys.path.insert(0, utils_path) from utils.event_common import require_chip -from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path, rtl_dir, log_dir +from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path, log_dir, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id diff --git a/bbdev/api/steps/compiler/scripts/build.py b/bbdev/api/steps/compiler/scripts/build.py index bba660b06..74b7c15a0 100644 --- a/bbdev/api/steps/compiler/scripts/build.py +++ b/bbdev/api/steps/compiler/scripts/build.py @@ -1,5 +1,6 @@ from __future__ import annotations +import fcntl import os import shlex import shutil @@ -7,13 +8,6 @@ from pathlib import Path -def _repo(raw: str | Path) -> Path: - root = Path(raw).resolve() - if not root.is_dir(): - raise ValueError(f"repo does not exist: {root}") - return root - - def _run( cmd: list[str], *, @@ -48,7 +42,7 @@ def _run( def compiler_python(repo: str | Path) -> str: - root = _repo(repo) + root = Path(repo).resolve() candidates = [root / "result" / "bin" / "python3"] path_python = shutil.which("python3") if path_python: @@ -68,7 +62,7 @@ def compiler_python(repo: str | Path) -> str: def build_llvm( repo: str | Path, *, logger: object | None = None, task_scope: str | None = None ) -> Path: - root = _repo(repo) + root = Path(repo).resolve() buddy = root / "compiler" / "thirdparty" / "buddy-mlir" llvm_src = buddy / "llvm" / "llvm" llvm_build = buddy / "llvm" / "build" @@ -95,23 +89,26 @@ def build_llvm( f"-DPython3_EXECUTABLE={python}", f"-DPython_EXECUTABLE={python}", ] - if not (llvm_build / "build.ninja").is_file(): + llvm_build.mkdir(parents=True, exist_ok=True) + with open(llvm_build / ".bbdev.lock", "a+b") as lockf: + fcntl.flock(lockf.fileno(), fcntl.LOCK_EX) + if not (llvm_build / "build.ninja").is_file(): + _run( + cmake, + repo=root, + cwd=buddy, + logger=logger, + task_scope=task_scope, + output_prefix="compiler llvm configure", + ) _run( - cmake, + ["ninja", "-C", str(llvm_build), "-j", str(os.cpu_count() or 1)], repo=root, cwd=buddy, logger=logger, task_scope=task_scope, - output_prefix="compiler llvm configure", + output_prefix="compiler llvm build", ) - _run( - ["ninja", "-C", str(llvm_build), "-j", str(os.cpu_count() or 1)], - repo=root, - cwd=buddy, - logger=logger, - task_scope=task_scope, - output_prefix="compiler llvm build", - ) mlir_cmake = llvm_build / "lib" / "cmake" / "mlir" if not mlir_cmake.is_dir(): raise RuntimeError(f"LLVM/MLIR build failed: missing {mlir_cmake}") @@ -119,7 +116,7 @@ def build_llvm( def compiler_build_dir(repo: str | Path, chip: str) -> Path: - return _repo(repo) / "compiler" / "thirdparty" / "buddy-mlir" / "build" / chip + return Path(repo).resolve() / "compiler" / "thirdparty" / "buddy-mlir" / "build" / chip def build_compiler( @@ -129,7 +126,7 @@ def build_compiler( logger: object | None = None, task_scope: str | None = None, ) -> Path: - root = _repo(repo) + root = Path(repo).resolve() if not chip: raise ValueError("chip is required") chip_pb = root / "examples" / "chips" / chip / "configs" / "generated" / "chip.pb" diff --git a/bbdev/api/steps/dc/03_area_event.step.py b/bbdev/api/steps/dc/01_area_event.step.py similarity index 76% rename from bbdev/api/steps/dc/03_area_event.step.py rename to bbdev/api/steps/dc/01_area_event.step.py index 5edecfe41..2705f4a37 100644 --- a/bbdev/api/steps/dc/03_area_event.step.py +++ b/bbdev/api/steps/dc/01_area_event.step.py @@ -1,3 +1,4 @@ +import json import os import shlex import shutil @@ -12,14 +13,14 @@ scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -step_path = os.path.dirname(__file__) -if step_path not in sys.path: - sys.path.insert(0, step_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger_async -from tapeout import get_tapeout_contract, technology_settings, write_run_tcl +from tapeout import clock_period_ns, get_tapeout_contract, write_run_tcl config = { "name": "dc-area", @@ -83,7 +84,27 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: os.makedirs(output_dir, exist_ok=True) os.makedirs(report_dir, exist_ok=True) try: - tech = technology_settings() + gman = input_data.get("generate_manifest") + if not isinstance(gman, str) or not gman: + rman = input_data.get("replace_manifest") + if isinstance(rman, str) and os.path.isfile(rman): + with open(rman, encoding="utf-8") as handle: + gman = json.load(handle).get("generate_manifest") + if not isinstance(gman, str) or not os.path.isfile(gman): + raise ValueError("missing generate_manifest with link_dbs") + with open(gman, encoding="utf-8") as handle: + link_dbs = json.load(handle).get("link_dbs") + if not isinstance(link_dbs, list) or not link_dbs: + raise ValueError(f"generate_manifest.link_dbs empty: {gman}") + for path in link_dbs: + if not isinstance(path, str) or not os.path.isfile(path): + raise ValueError(f"sram db missing: {path}") + tech = { + "target_library": contract.target_library, + "synthetic_library": contract.synthetic_library, + "link_library": list(contract.link_library) + list(link_dbs), + "max_cores": contract.max_cores, + } run_config = write_run_tcl( os.path.join(analysis_dir, "run.tcl"), { @@ -91,12 +112,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "source_list": source_list_path, "output_dir": output_dir, "report_dir": report_dir, - "clock_port": contract.clock_port, - "clock_period_ns": contract.clock_period_ns, + "sdc": str(contract.constraints_sdc), **tech, }, ) - except (OSError, ValueError) as exc: + except (OSError, ValueError, json.JSONDecodeError) as exc: await check_result(ctx, 1, continue_run=False, extra_fields={"task": "dc", "error": str(exc)}, trace_id=origin_tid) return script_path = contract.dc_script @@ -104,9 +124,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"Running chip-owned DC synthesis for {contract.chip}, top {top_module}: {script_path}") result = await stream_run_logger_async( cmd=( - f"dc_shell -f {shlex.quote(str(script_path))} " + f"set -o pipefail; dc_shell -f {shlex.quote(str(script_path))} " f"-x {shlex.quote('set RUN_CONFIG ' + '{' + str(run_config) + '}')} " - f"> {shlex.quote(dc_log)} 2>&1" + f"2>&1 | tee {shlex.quote(dc_log)}" ), logger=ctx.logger, cwd=os.path.dirname(script_path), @@ -123,7 +143,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "run_config": str(run_config), "chip": contract.chip, "tapeout_dir": str(contract.root), - "sram_manifest": (input_data.get("sram_collateral") or {}).get("sram_manifest"), + "replace_manifest": input_data.get("replace_manifest"), } if result.returncode == 0 and input_data.get("from_regression_area_power"): @@ -153,7 +173,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - freq = 1000.0 / contract.clock_period_ns + freq = 1000.0 / clock_period_ns(contract) merge_metrics(get_buckyball_path(), area=area, freq=freq) extra_fields["area"] = area extra_fields["freq"] = freq diff --git a/bbdev/api/steps/dc/05_sim_event.step.py b/bbdev/api/steps/dc/03_sim_event.step.py similarity index 92% rename from bbdev/api/steps/dc/05_sim_event.step.py rename to bbdev/api/steps/dc/03_sim_event.step.py index ac1b8ef62..863113176 100644 --- a/bbdev/api/steps/dc/05_sim_event.step.py +++ b/bbdev/api/steps/dc/03_sim_event.step.py @@ -8,9 +8,9 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -step_path = os.path.dirname(__file__) -if step_path not in sys.path: - sys.path.insert(0, step_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path, workload_build_dir, workload_tests_root @@ -23,7 +23,7 @@ "description": "rerun chip-owned simulation and produce activity for PTPX", "flows": ["dc"], "triggers": [queue("dc.sim")], - "enqueues": ["dc.power"], + "enqueues": ["pt.run"], } @@ -62,7 +62,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if not os.path.isfile(explicit_activity): await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": f"activity file does not exist: {explicit_activity}"}, trace_id=origin_tid) return - await ctx.enqueue({"topic": "dc.power", "data": {**input_data, "activity": explicit_activity, "format": activity_format, "_trace_id": origin_tid}}) + await ctx.enqueue({"topic": "pt.run", "data": {**input_data, "activity": explicit_activity, "format": activity_format, "_trace_id": origin_tid}}) return if shutil.which("bash") is None: @@ -157,4 +157,4 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) if result.returncode == 0: - await ctx.enqueue({"topic": "dc.power", "data": {**input_data, "activity": activity_path, "format": activity_format, "strip_path": contract.power_strip_path, "_trace_id": origin_tid}}) + await ctx.enqueue({"topic": "pt.run", "data": {**input_data, "activity": activity_path, "format": activity_format, "strip_path": contract.power_strip_path, "_trace_id": origin_tid}}) diff --git a/bbdev/api/steps/dc/scripts/dc.tcl b/bbdev/api/steps/dc/scripts/dc.tcl deleted file mode 100755 index b525d3845..000000000 --- a/bbdev/api/steps/dc/scripts/dc.tcl +++ /dev/null @@ -1,56 +0,0 @@ -# Shared Design Compiler implementation. Chip tapeout/dc.tcl files are thin -# ownership wrappers; all RUN_* values are supplied by bbdev. -if {![info exists RUN_CONFIG] || $RUN_CONFIG eq ""} { - error "bbdev must pass -x {set RUN_CONFIG }" -} -source [file normalize $RUN_CONFIG] - -proc bbdev_read_filelist {path} { - set fh [open $path r] - set files [list] - while {[gets $fh line] >= 0} { - set line [string trim $line] - if {$line eq "" || [string match "#*" $line]} { continue } - lappend files [file normalize $line] - } - close $fh - if {[llength $files] == 0} { error "empty DC source list: $path" } - return $files -} - -set top $RUN_TOP -set output_dir [file normalize $RUN_OUTPUT_DIR] -set report_dir [file normalize $RUN_REPORT_DIR] -file mkdir $output_dir $report_dir [file join $report_dir work] -set target_library [list $RUN_TARGET_LIBRARY] -set synthetic_library $RUN_SYNTHETIC_LIBRARY -set link_library [concat [list *] $target_library $synthetic_library $RUN_LINK_LIBRARY] -set_host_options -max_cores $RUN_MAX_CORES -define_design_lib WORK -path [file join $report_dir work] -set search_path [list .] -set_app_var verilogout_no_tri true -set_app_var verilogout_equation false -analyze -format sverilog -define {SYNTHESIS DC_SYN} [bbdev_read_filelist $RUN_SOURCE_LIST] -elaborate $top -current_design $top -link -set bb_clock [get_ports -quiet $RUN_CLOCK_PORT] -if {[sizeof_collection $bb_clock] != 1} { error "clock port '$RUN_CLOCK_PORT' was not found exactly once" } -create_clock -name bb_clock -period $RUN_CLOCK_PERIOD_NS $bb_clock -set_clock_uncertainty [expr {$RUN_CLOCK_PERIOD_NS * 0.30}] [get_clocks bb_clock] -set_clock_transition [expr {$RUN_CLOCK_PERIOD_NS * 0.10}] [get_clocks bb_clock] -set_input_delay [expr {$RUN_CLOCK_PERIOD_NS * 0.70}] -clock bb_clock [remove_from_collection [all_inputs] $bb_clock] -set_output_delay [expr {$RUN_CLOCK_PERIOD_NS * 0.70}] -clock bb_clock [all_outputs] -set_load 2.0 [all_outputs] -compile_ultra -area_high_effort_script -no_autoungroup -no_boundary_optimization -set_fix_multiple_port_nets -all -buffer_constants -change_names -hierarchy -rules verilog -write -format ddc -hierarchy -output [file join $output_dir ${top}.ddc] -write -format verilog -hierarchy -output [file join $output_dir ${top}.v] -write_sdc [file join $output_dir ${top}.sdc] -report_constraint -all_violators > [file join $report_dir constraint.rpt] -report_timing -delay max -max_paths 50 > [file join $report_dir timing_max.rpt] -report_timing -delay min -max_paths 50 > [file join $report_dir timing_min.rpt] -report_area -hierarchy > [file join $report_dir area.rpt] -report_reference > [file join $report_dir reference.rpt] -exit diff --git a/bbdev/api/steps/dc/scripts/tapeout.py b/bbdev/api/steps/dc/scripts/tapeout.py new file mode 100644 index 000000000..8bf1a4f03 --- /dev/null +++ b/bbdev/api/steps/dc/scripts/tapeout.py @@ -0,0 +1,145 @@ +"""Resolve chip-owned tapeout flow contracts.""" + +from __future__ import annotations + +import os +import re +import shlex +import tomllib +from dataclasses import dataclass +from pathlib import Path + + +def _tcl_word(value: str) -> str: + return "{" + str(value).replace("\\", "\\\\").replace("}", "\\}") + "}" + + +def _tcl_list(values: list[str]) -> str: + return "[list " + " ".join(_tcl_word(value) for value in values) + "]" + + +@dataclass(frozen=True) +class SramGeom: + name: str + words: int + mux: int + bits: int + + +@dataclass(frozen=True) +class TapeoutContract: + chip: str + root: Path + dc_script: Path + constraints_sdc: Path + power_script: Path + power_sim_script: Path + sram_mdf: Path + top_module: str + target_library: str + synthetic_library: list[str] + link_library: list[str] + max_cores: int + power_format: str + power_start_ns: str | None + power_end_ns: str | None + power_workload: str | None + power_strip_path: str + sram_process: str + sram_corner: str + lc_shell: Path + sram_table: dict[str, SramGeom] + + +def _p(root: Path, value: str) -> Path: + p = Path(value) + return p if p.is_absolute() else root / p + + +def resolve_power_window( + contract: TapeoutContract, start_ns: object | None, end_ns: object | None +) -> tuple[str | None, str | None]: + start = str(start_ns) if start_ns is not None and str(start_ns) != "" else contract.power_start_ns + end = str(end_ns) if end_ns is not None and str(end_ns) != "" else contract.power_end_ns + if (start is None) != (end is None): + raise ValueError("power start_ns and end_ns must be supplied together") + if start is not None: + try: + if float(start) < 0 or float(start) >= float(end): + raise ValueError + except ValueError as exc: + raise ValueError("power start_ns must be smaller than end_ns") from exc + return start, end + + +def clock_period_ns(contract: TapeoutContract) -> float: + text = contract.constraints_sdc.read_text(encoding="utf-8") + match = re.search(r"create_clock\b[^;\n]*-period\s+([0-9]+(?:\.[0-9]+)?)", text) + if match is None: + raise ValueError(f"no create_clock -period in {contract.constraints_sdc}") + period = float(match.group(1)) + if period <= 0: + raise ValueError(f"invalid create_clock -period in {contract.constraints_sdc}") + return period + + +def get_tapeout_contract(bbdir: str | os.PathLike[str], chip: str, top: str | None = None) -> TapeoutContract: + root = Path(bbdir) / "examples" / "chips" / chip / "tapeout" + with (root / "config.toml").open("rb") as f: + s = tomllib.load(f)["tapeout"] + + sram_table = { + row["name"]: SramGeom(row["name"], row["words"], row["mux"], row["bits"]) + for row in s["sram"] + } + + pw = s.get("power_workload") + return TapeoutContract( + chip=chip, + root=root, + dc_script=_p(root, s["dc_script"]), + constraints_sdc=_p(root, s["constraints_sdc"]), + power_script=_p(root, s["power_script"]), + power_sim_script=_p(root, s["power_sim_script"]), + sram_mdf=_p(root, s["sram_mdf"]), + top_module=top or s["top"], + target_library=str(_p(root, s["target_library"])), + synthetic_library=[str(_p(root, p)) for p in s["synthetic_library"]], + link_library=[str(_p(root, p)) for p in s.get("link_library", [])], + max_cores=s["max_cores"], + power_format=s["power_format"], + power_start_ns=s.get("power_start_ns"), + power_end_ns=s.get("power_end_ns"), + power_workload=None if pw in (None, "") else str(pw), + power_strip_path=s.get("power_strip_path", ""), + sram_process=s["sram_process"], + sram_corner=s["sram_corner"], + lc_shell=_p(root, s["lc_shell"]), + sram_table=sram_table, + ) + + +def write_run_tcl(path: str | os.PathLike[str], values: dict[str, object]) -> Path: + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + lines = [] + for key, value in values.items(): + name = "RUN_" + key.upper() + if isinstance(value, list): + lines.append(f"set {name} {_tcl_list([str(item) for item in value])}") + elif value is None: + lines.append(f"set {name} {{}}") + elif isinstance(value, (int, float)): + lines.append(f"set {name} {value}") + else: + lines.append(f"set {name} {_tcl_word(str(value))}") + output.write_text("\n".join(lines) + "\n", encoding="utf-8") + return output + + +def write_run_env(path: str | os.PathLike[str], values: dict[str, object]) -> Path: + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + lines = [f"{key}={shlex.quote(str(value))}" for key, value in values.items() if value is not None] + output.write_text("\n".join(lines) + "\n", encoding="utf-8") + return output diff --git a/bbdev/api/steps/dc/tapeout.py b/bbdev/api/steps/dc/tapeout.py deleted file mode 100644 index fd829a8e7..000000000 --- a/bbdev/api/steps/dc/tapeout.py +++ /dev/null @@ -1,182 +0,0 @@ -"""Resolve chip-owned tapeout flow contracts.""" - -from __future__ import annotations - -import os -import shlex -import tomllib -from dataclasses import dataclass -from pathlib import Path - -def _tcl_word(value: str) -> str: - return "{" + str(value).replace("\\", "\\\\").replace("}", "\\}") + "}" - - -def _tcl_list(values: list[str]) -> str: - return "[list " + " ".join(_tcl_word(value) for value in values) + "]" - - -@dataclass(frozen=True) -class TapeoutContract: - chip: str - root: Path - dc_script: Path - power_script: Path - power_sim_script: Path - top_module: str - clock_port: str - clock_period_ns: float - power_format: str - power_start_ns: str | None - power_end_ns: str | None - power_workload: str | None - power_strip_path: str - - -def resolve_power_window( - contract: TapeoutContract, start_ns: object | None, end_ns: object | None -) -> tuple[str | None, str | None]: - """Merge optional CLI window bounds with a chip default and validate them.""" - start = str(start_ns) if start_ns is not None and str(start_ns) != "" else contract.power_start_ns - end = str(end_ns) if end_ns is not None and str(end_ns) != "" else contract.power_end_ns - if (start is None) != (end is None): - raise ValueError("power start_ns and end_ns must be supplied together") - if start is not None: - try: - if float(start) < 0 or float(start) >= float(end): - raise ValueError - except ValueError as exc: - raise ValueError("power start_ns must be smaller than end_ns") from exc - return start, end - - -def _read_config(root: Path) -> dict: - path = root / "config.toml" - if not path.is_file(): - return {} - with path.open("rb") as handle: - data = tomllib.load(handle) - section = data.get("tapeout", data) - if not isinstance(section, dict): - raise ValueError(f"{path} must contain a [tapeout] table") - return section - - -def get_tapeout_contract(bbdir: str | os.PathLike[str], chip: str, top: str | None = None) -> TapeoutContract: - """Resolve the tapeout scripts owned by the chip.""" - if not isinstance(chip, str) or not chip: - raise ValueError("Missing required parameter: --chip") - root = Path(bbdir) / "examples" / "chips" / chip / "tapeout" - if not root.is_dir(): - raise ValueError(f"chip {chip} has no tapeout directory: {root}") - - settings = _read_config(root) - top_module = top or settings.get("top", "DigitalTop") - clock_port = str(settings.get("clock_port", "")).strip() - if not clock_port: - raise ValueError(f"missing tapeout.clock_port in {root / 'config.toml'}") - try: - clock_period_ns = float(settings.get("clock_period_ns")) - except (TypeError, ValueError) as exc: - raise ValueError(f"invalid tapeout.clock_period_ns in {root / 'config.toml'}") from exc - if clock_period_ns <= 0: - raise ValueError(f"tapeout.clock_period_ns must be positive in {root / 'config.toml'}") - - power_format = str(settings.get("power_format", "fsdb")).lower() - if power_format not in {"saif", "vcd", "fsdb"}: - raise ValueError(f"unsupported tapeout.power_format in {root / 'config.toml'}: {power_format}") - - def optional_string(name: str) -> str | None: - value = settings.get(name) - return None if value is None or str(value).strip() == "" else str(value) - - power_start_ns = optional_string("power_start_ns") - power_end_ns = optional_string("power_end_ns") - if (power_start_ns is None) != (power_end_ns is None): - raise ValueError(f"tapeout power_start_ns and power_end_ns must be set together in {root / 'config.toml'}") - if power_start_ns is not None: - try: - if float(power_start_ns) < 0 or float(power_start_ns) >= float(power_end_ns): - raise ValueError - except ValueError as exc: - raise ValueError(f"invalid tapeout power window in {root / 'config.toml'}") from exc - - scripts = { - "dc_script": root / str(settings.get("dc_script", "dc.tcl")), - "power_script": root / str(settings.get("power_script", "power.tcl")), - "power_sim_script": root / str(settings.get("power_sim_script", "power_sim.sh")), - } - missing = [str(path) for path in scripts.values() if not path.is_file()] - if missing: - raise ValueError(f"chip {chip} tapeout contract is missing: {', '.join(missing)}") - - return TapeoutContract( - chip=chip, - root=root, - dc_script=scripts["dc_script"], - power_script=scripts["power_script"], - power_sim_script=scripts["power_sim_script"], - top_module=str(top_module), - clock_port=clock_port, - clock_period_ns=clock_period_ns, - power_format=power_format, - power_start_ns=power_start_ns, - power_end_ns=power_end_ns, - power_workload=optional_string("power_workload"), - power_strip_path=str(settings.get("power_strip_path", "")), - ) - - -def technology_settings() -> dict[str, object]: - """Read the small technology contract exported by the host setup.""" - target = os.environ.get("TARGET_LIBRARY", "").strip() - synthetic = [item for item in os.environ.get("SYNTHETIC_LIBRARY", "").split(os.pathsep) if item] - link = [item for item in os.environ.get("LINK_LIBRARY", "").split(os.pathsep) if item] - if not target: - raise ValueError("missing TARGET_LIBRARY; export it in zshrc") - if any(ch.isspace() for ch in target): - raise ValueError( - "TARGET_LIBRARY contains whitespace/newlines; " - f"fix the zshrc export (got {target!r})" - ) - if not Path(target).is_file(): - raise ValueError(f"TARGET_LIBRARY does not exist: {target}") - missing = [item for item in synthetic + link if not Path(item).is_file()] - if missing: - raise ValueError("technology library does not exist: " + ", ".join(missing)) - if not synthetic: - raise ValueError("missing SYNTHETIC_LIBRARY; export it in zshrc") - return { - "target_library": target, - "synthetic_library": synthetic, - "link_library": link, - "max_cores": 8, - } - - -def write_run_tcl(path: str | os.PathLike[str], values: dict[str, object]) -> Path: - """Write a Tcl variable file consumed by chip-owned DC/PT scripts.""" - output = Path(path) - output.parent.mkdir(parents=True, exist_ok=True) - lines = [] - for key, value in values.items(): - name = "RUN_" + key.upper() - if isinstance(value, list): - lines.append(f"set {name} {_tcl_list([str(item) for item in value])}") - elif value is None: - lines.append(f"set {name} {{}}") - elif isinstance(value, (int, float)): - lines.append(f"set {name} {value}") - else: - lines.append(f"set {name} {_tcl_word(str(value))}") - output.write_text("\n".join(lines) + "\n", encoding="utf-8") - return output - - -def write_run_env(path: str | os.PathLike[str], values: dict[str, object]) -> Path: - """Write shell assignments for a chip-owned power simulation wrapper.""" - output = Path(path) - output.parent.mkdir(parents=True, exist_ok=True) - lines = [f"{key}={shlex.quote(str(value))}" for key, value in values.items() if value is not None] - output.write_text("\n".join(lines) + "\n", encoding="utf-8") - return output diff --git a/bbdev/api/steps/ip-replace/01_run_api.step.py b/bbdev/api/steps/ip-replace/01_run_api.step.py deleted file mode 100644 index 03afddc79..000000000 --- a/bbdev/api/steps/ip-replace/01_run_api.step.py +++ /dev/null @@ -1,33 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - - -def req_arg(body: dict, name: str): - return body.get(name) or body.get(name.replace("_", "-")) - - -config = { - "name": "ip-replace-api", - "description": "prepare top-scoped synthesis RTL and SRAM metadata", - "flows": ["ip-replace"], - "triggers": [api("POST", "/ip/replace/run")], - "enqueues": ["ip-replace.run"], -} - - -async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = req.body or {} - source_list = req_arg(body, "source_list") - if not isinstance(source_list, str) or not source_list: - return ApiResponse(status=400, body={"error": "source_list is required"}) - output_dir = req_arg(body, "output_dir") - if not isinstance(output_dir, str) or not output_dir: - return ApiResponse(status=400, body={"error": "output_dir is required"}) - - data = { - "source_list": source_list, - "ip_replace_output_dir": output_dir, - "top": req_arg(body, "top"), - "consumer": req_arg(body, "consumer") or "generic", - } - await ctx.enqueue({"topic": "ip-replace.run", "data": {**data, "_trace_id": ctx.trace_id}}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/ip-replace/01_run_event.step.py b/bbdev/api/steps/ip-replace/01_run_event.step.py deleted file mode 100644 index 09aa4c51b..000000000 --- a/bbdev/api/steps/ip-replace/01_run_event.step.py +++ /dev/null @@ -1,124 +0,0 @@ -import os -import sys - -from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.event_common import check_result, get_origin_trace_id - -scripts_path = os.path.join(os.path.dirname(__file__), "scripts") -if scripts_path not in sys.path: - sys.path.insert(0, scripts_path) - -from sram_replace import prepare_sram_collateral - - -config = { - "name": "ip-replace", - "description": "prepare top-scoped synthesis RTL and SRAM metadata", - "flows": ["ip-replace", "dc", "yosys"], - "triggers": [queue("ip-replace.run")], - "enqueues": ["dc.area", "yosys.synth"], -} - - -def default_source_list(input_data: dict) -> str | None: - source_list = input_data.get("source_list") - if isinstance(source_list, str) and source_list: - return source_list - build_dir = input_data.get("output_dir") - consumer = input_data.get("consumer") - if not isinstance(build_dir, str): - return None - if consumer == "dc": - return os.path.join(build_dir, "dc_sources.list") - if consumer == "yosys": - return os.path.join(build_dir, "yosys_sources.list") - return None - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - source_list_path = default_source_list(input_data) - if not source_list_path or not os.path.isfile(source_list_path): - _, failure_result = await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"task": "ip-replace", "error": "missing source list"}, - trace_id=origin_tid, - ) - return failure_result - - with open(source_list_path) as handle: - sources = [line.strip() for line in handle if line.strip()] - if not sources: - _, failure_result = await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"task": "ip-replace", "error": "empty source list"}, - trace_id=origin_tid, - ) - return failure_result - - consumer = input_data.get("consumer", "generic") - top_module = input_data.get("top") or "DigitalTop" - output_dir = input_data.get("ip_replace_output_dir") - if not isinstance(output_dir, str) or not output_dir: - output_dir = os.path.join(os.path.dirname(source_list_path), "ip-replace") - os.makedirs(output_dir, exist_ok=True) - - try: - collateral = prepare_sram_collateral(sources, output_dir, top_module) - except Exception as exc: - _, failure_result = await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"task": "ip-replace", "error": str(exc)}, - trace_id=origin_tid, - ) - return failure_result - - replaced_source_list = os.path.join(output_dir, f"{consumer}_sources.list") - with open(replaced_source_list, "w") as handle: - for path in collateral["source_paths"]: - handle.write(f"{path}\n") - - collateral["source_list"] = replaced_source_list - extra = { - "task": "ip-replace", - "consumer": consumer, - "source_list": replaced_source_list, - "sram_manifest": collateral["sram_manifest"], - "sram_memory_count": collateral["sram_memory_count"], - "top_module": collateral["top_module"], - } - if input_data.get("mem_conf"): - extra["mem_conf"] = input_data["mem_conf"] - ctx.logger.info( - f"Prepared {collateral['sram_memory_count']} technology-neutral SRAM modules " - f"for top {collateral['top_module']}" - ) - - next_topic = input_data.get("next_topic") - if isinstance(next_topic, str) and next_topic: - await check_result(ctx, 0, continue_run=True, extra_fields=extra, trace_id=origin_tid) - await ctx.enqueue( - { - "topic": next_topic, - "data": { - **input_data, - "source_list": replaced_source_list, - "sram_collateral": collateral, - "_trace_id": origin_tid, - }, - } - ) - return - - await check_result(ctx, 0, continue_run=False, extra_fields=extra, trace_id=origin_tid) diff --git a/bbdev/api/steps/ip-replace/scripts/sram_replace.py b/bbdev/api/steps/ip-replace/scripts/sram_replace.py deleted file mode 100755 index 61c3e9f92..000000000 --- a/bbdev/api/steps/ip-replace/scripts/sram_replace.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Prepare technology-neutral SRAM collateral for synthesis. - -The elaborator has already split sequential memories into separate SystemVerilog -modules. This module preserves those implementations, selects the RTL reachable -from a synthesis top, and records memory interfaces in a manifest. A PDK-owned -flow can consume that manifest to choose macros and replace modules later. -""" - -from __future__ import annotations - -import argparse -import json -import math -import re -from dataclasses import dataclass -from pathlib import Path - - -MODULE_NAME_RE = re.compile(r"^\s*module\s+([A-Za-z_][A-Za-z0-9_$]*)\b", re.M) -MODULE_HEADER_RE = re.compile( - r"\bmodule\s+([A-Za-z_][A-Za-z0-9_$]*)\s*(?:#\s*\(.*?\)\s*)?\((.*?)\)\s*;", - re.S, -) -MEMORY_RE = re.compile( - r"\breg\s+(?:\[\s*(\d+)\s*:\s*(\d+)\s*\]\s+)?Memory\s*\[\s*(\d+)\s*:\s*(\d+)\s*\]", - re.S, -) -RANGE_RE = re.compile(r"\[\s*(\d+)\s*:\s*(\d+)\s*\]\s*{name}\b") -INSTANCE_RE = re.compile( - r"^\s*([A-Za-z_][A-Za-z0-9_$]*)\s*(?:#\s*\(.*?\)\s*)?([A-Za-z_][A-Za-z0-9_$]*)\s*\(", - re.M | re.S, -) - - -@dataclass(frozen=True) -class Memory: - name: str - source: Path - depth: int - width: int - port_type: str - address_width: int | None - write_mask_width: int | None - - -def width_from_range(text: str, name: str) -> int | None: - match = re.search(RANGE_RE.pattern.format(name=re.escape(name)), text) - if match is None: - return None - return abs(int(match.group(1)) - int(match.group(2))) + 1 - - -def ceil_log2(value: int) -> int: - return max(1, math.ceil(math.log2(value))) - - -def module_sources(source_paths: list[Path]) -> dict[str, Path]: - sources: dict[str, Path] = {} - for source in source_paths: - match = MODULE_NAME_RE.search(source.read_text(errors="replace")) - if match is not None: - sources.setdefault(match.group(1), source) - return sources - - -def reachable_modules(source_paths: list[Path], top_module: str) -> set[str]: - sources = module_sources(source_paths) - if top_module not in sources: - raise ValueError(f"synthesis top is not defined in source list: {top_module}") - - module_text = { - module: path.read_text(errors="replace") for module, path in sources.items() - } - reachable = {top_module} - pending = [top_module] - while pending: - module = pending.pop() - for child, _instance in INSTANCE_RE.findall(module_text[module]): - if child in sources and child not in reachable: - reachable.add(child) - pending.append(child) - return reachable - - -def classify_ports(header: str) -> str: - names = set(re.findall(r"\b((?:RW|R|W)\d+)_[A-Za-z0-9_]+\b", header)) - rw_ports = {name for name in names if name.startswith("RW")} - if rw_ports: - return f"{len(rw_ports)}RW" - reads = {name for name in names if name.startswith("R")} - writes = {name for name in names if name.startswith("W")} - return f"{len(reads)}R{len(writes)}W" - - -def discover_memories(source_paths: list[Path]) -> list[Memory]: - memories: list[Memory] = [] - for source in source_paths: - text = source.read_text(errors="replace") - module_match = MODULE_HEADER_RE.search(text) - memory_match = MEMORY_RE.search(text) - if module_match is None or memory_match is None: - continue - name, header = module_match.groups() - data_msb = int(memory_match.group(1) or 0) - data_lsb = int(memory_match.group(2) or 0) - depth_a = int(memory_match.group(3)) - depth_b = int(memory_match.group(4)) - depth = abs(depth_a - depth_b) + 1 - memories.append( - Memory( - name=name, - source=source, - depth=depth, - width=abs(data_msb - data_lsb) + 1, - port_type=classify_ports(header), - address_width=width_from_range(header, "RW0_addr") or ceil_log2(depth), - write_mask_width=( - width_from_range(header, "RW0_wmask") - or (1 if "RW0_wmask" in header else None) - ), - ) - ) - return memories - - -def prepare_sram_collateral( - source_paths: list[str], output_dir: str, top_module: str -) -> dict[str, object]: - """Emit top-scoped source and memory manifests without choosing a PDK macro.""" - sources = [Path(path).resolve() for path in source_paths] - reachable = reachable_modules(sources, top_module) - source_modules = module_sources(sources) - selected_paths = {source_modules[module] for module in reachable} - selected = [path for path in sources if path in selected_paths] - memories = discover_memories(selected) - - output = Path(output_dir) - output.mkdir(parents=True, exist_ok=True) - manifest_path = output / "sram_manifest.json" - manifest = { - "schema_version": 1, - "top_module": top_module, - "memories": [ - { - "module": memory.name, - "source": str(memory.source), - "depth": memory.depth, - "width": memory.width, - "port_type": memory.port_type, - "address_width": memory.address_width, - "write_mask_width": memory.write_mask_width, - } - for memory in memories - ], - } - manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") - return { - "source_paths": [str(path) for path in selected], - "sram_manifest": str(manifest_path), - "sram_memory_count": len(memories), - "top_module": top_module, - } - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--source-list", type=Path, help="one Verilog source path per line") - parser.add_argument("--source-dir", type=Path, help="directory to scan when --source-list is omitted") - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--top", required=True) - args = parser.parse_args() - - if args.source_list: - sources = [line.strip() for line in args.source_list.read_text().splitlines() if line.strip()] - elif args.source_dir: - sources = [str(path) for path in sorted(args.source_dir.glob("*.sv"))] - else: - parser.error("one of --source-list or --source-dir is required") - - result = prepare_sram_collateral(sources, str(args.output_dir), args.top) - print(json.dumps(result, indent=2, sort_keys=True)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/bbdev/api/steps/ip/01_generate_api.step.py b/bbdev/api/steps/ip/01_generate_api.step.py new file mode 100644 index 000000000..d12372730 --- /dev/null +++ b/bbdev/api/steps/ip/01_generate_api.step.py @@ -0,0 +1,44 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_chip + + +def req_arg(body: dict, name: str): + return body.get(name) or body.get(name.replace("_", "-")) + + +config = { + "name": "ip-generate-api", + "description": "generate SRAM macros from elaborator mems.conf via MacroCompiler", + "flows": ["ip", "dc", "yosys"], + "triggers": [api("POST", "/ip/generate")], + "enqueues": ["ip.generate"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + + data = { + "chip": chip, + "consumer": req_arg(body, "consumer") or "dc", + "top": req_arg(body, "top") or "DigitalTop", + } + if data["consumer"] not in ("dc", "yosys"): + return ApiResponse(status=400, body={"error": "consumer must be dc or yosys"}) + output_dir = req_arg(body, "output_dir") + if output_dir is not None: + if not isinstance(output_dir, str) or not output_dir: + return ApiResponse(status=400, body={"error": "output_dir must be a non-empty string"}) + data["output_dir"] = output_dir + next_topic = req_arg(body, "next_topic") + if next_topic is not None: + if not isinstance(next_topic, str) or not next_topic: + return ApiResponse(status=400, body={"error": "next_topic must be a non-empty string"}) + data["next_topic"] = next_topic + await ctx.enqueue({"topic": "ip.generate", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/ip/01_generate_event.step.py b/bbdev/api/steps/ip/01_generate_event.step.py new file mode 100644 index 000000000..f44e43a9f --- /dev/null +++ b/bbdev/api/steps/ip/01_generate_event.step.py @@ -0,0 +1,72 @@ +import os +import sys +from pathlib import Path + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from utils.event_common import check_result, get_origin_trace_id, require_chip +from utils.path import get_buckyball_path, rtl_dir +from generate import generate_sram + + +config = { + "name": "ip-generate", + "description": "generate SRAM macros from elaborator mems.conf via MacroCompiler", + "flows": ["ip", "dc", "yosys"], + "triggers": [queue("ip.generate")], + "enqueues": ["ip.replace"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + try: + chip = require_chip(input_data) + consumer = input_data.get("consumer") or "dc" + if consumer not in ("dc", "yosys"): + raise ValueError("consumer must be dc or yosys") + top = input_data.get("top") or "DigitalTop" + if not isinstance(top, str) or not top: + raise ValueError("top must be a non-empty string") + bbdir = Path(get_buckyball_path()).resolve() + build_dir = Path(rtl_dir(bbdir, chip, "verilog", input_data.get("output_dir"))).resolve() + man = generate_sram(bbdir=bbdir, chip=chip, build_dir=build_dir) + except Exception as exc: + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "ip.generate", "error": str(exc)}, + trace_id=origin_tid, + ) + return failure_result + + extra = { + "task": "ip.generate", + "chip": man["chip"], + "generate_manifest": man["generate_manifest"], + "sram_macros_v": man["sram_macros_v"], + "mem_count": man["mem_count"], + "build_dir": str(build_dir), + } + await check_result(ctx, 0, continue_run=True, extra_fields=extra, trace_id=origin_tid) + await ctx.enqueue( + { + "topic": "ip.replace", + "data": { + **input_data, + "chip": chip, + "consumer": consumer, + "top": top, + "_trace_id": origin_tid, + }, + } + ) diff --git a/bbdev/api/steps/ip/02_replace_api.step.py b/bbdev/api/steps/ip/02_replace_api.step.py new file mode 100644 index 000000000..8e3118687 --- /dev/null +++ b/bbdev/api/steps/ip/02_replace_api.step.py @@ -0,0 +1,44 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_chip + + +def req_arg(body: dict, name: str): + return body.get(name) or body.get(name.replace("_", "-")) + + +config = { + "name": "ip-replace-api", + "description": "assemble top-scoped synthesis RTL plus generated SRAM macros", + "flows": ["ip", "dc", "yosys"], + "triggers": [api("POST", "/ip/replace")], + "enqueues": ["ip.replace"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + + data = { + "chip": chip, + "consumer": req_arg(body, "consumer") or "dc", + "top": req_arg(body, "top") or "DigitalTop", + } + if data["consumer"] not in ("dc", "yosys"): + return ApiResponse(status=400, body={"error": "consumer must be dc or yosys"}) + output_dir = req_arg(body, "output_dir") + if output_dir is not None: + if not isinstance(output_dir, str) or not output_dir: + return ApiResponse(status=400, body={"error": "output_dir must be a non-empty string"}) + data["output_dir"] = output_dir + next_topic = req_arg(body, "next_topic") + if next_topic is not None: + if not isinstance(next_topic, str) or not next_topic: + return ApiResponse(status=400, body={"error": "next_topic must be a non-empty string"}) + data["next_topic"] = next_topic + await ctx.enqueue({"topic": "ip.replace", "data": {**data, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/ip/02_replace_event.step.py b/bbdev/api/steps/ip/02_replace_event.step.py new file mode 100644 index 000000000..10d5234c7 --- /dev/null +++ b/bbdev/api/steps/ip/02_replace_event.step.py @@ -0,0 +1,116 @@ +import os +import sys +from pathlib import Path + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) + +from utils.event_common import check_result, get_origin_trace_id, require_chip +from utils.path import get_buckyball_path, rtl_dir +from replace import replace_sources + +_SOURCE_LIST = {"dc": "dc_sources.list", "yosys": "yosys_sources.list"} + + +config = { + "name": "ip-replace", + "description": "assemble top-scoped synthesis RTL plus generated SRAM macros", + "flows": ["ip", "dc", "yosys"], + "triggers": [queue("ip.replace")], + "enqueues": ["dc.area", "yosys.synth"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + try: + chip = require_chip(input_data) + consumer = input_data.get("consumer") or "dc" + if consumer not in _SOURCE_LIST: + raise ValueError("consumer must be dc or yosys") + top_module = input_data.get("top") or "DigitalTop" + if not isinstance(top_module, str) or not top_module: + raise ValueError("top must be a non-empty string") + + bbdir = Path(get_buckyball_path()).resolve() + build_dir = Path(rtl_dir(bbdir, chip, "verilog", input_data.get("output_dir"))).resolve() + source_list_path = build_dir / _SOURCE_LIST[consumer] + if not source_list_path.is_file(): + raise FileNotFoundError(f"missing source list: {source_list_path}") + sources = [line.strip() for line in source_list_path.read_text().splitlines() if line.strip()] + if not sources: + raise ValueError(f"empty source list: {source_list_path}") + + gen_dir = build_dir / "ip-generate" + sram_macros_v = gen_dir / "sram_macros.v" + generate_manifest = gen_dir / "generate_manifest.json" + if not generate_manifest.is_file(): + raise FileNotFoundError(f"missing generate_manifest: {generate_manifest}") + + result = replace_sources( + source_paths=sources, + top_module=top_module, + sram_macros_v=sram_macros_v, + output_dir=build_dir / "ip-replace", + consumer=consumer, + generate_manifest=generate_manifest, + ) + except Exception as exc: + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "ip.replace", "error": str(exc)}, + trace_id=origin_tid, + ) + return failure_result + + extra = { + "task": "ip.replace", + "consumer": consumer, + "source_list": result["source_list"], + "replace_manifest": result["replace_manifest"], + "source_count": result["source_count"], + "top_module": top_module, + "build_dir": str(build_dir), + } + if generate_manifest.is_file(): + extra["generate_manifest"] = str(generate_manifest) + + next_topic = input_data.get("next_topic") + if next_topic is not None: + if not isinstance(next_topic, str) or not next_topic: + _, failure_result = await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "ip.replace", "error": "next_topic must be a non-empty string"}, + trace_id=origin_tid, + ) + return failure_result + await check_result(ctx, 0, continue_run=True, extra_fields=extra, trace_id=origin_tid) + await ctx.enqueue( + { + "topic": next_topic, + "data": { + **input_data, + "chip": chip, + "consumer": consumer, + "top": top_module, + "source_list": result["source_list"], + "replace_manifest": result["replace_manifest"], + "generate_manifest": str(generate_manifest), + "_trace_id": origin_tid, + }, + } + ) + return + + await check_result(ctx, 0, continue_run=False, extra_fields=extra, trace_id=origin_tid) diff --git a/bbdev/api/steps/ip/scripts/generate.py b/bbdev/api/steps/ip/scripts/generate.py new file mode 100644 index 000000000..f883eb742 --- /dev/null +++ b/bbdev/api/steps/ip/scripts/generate.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path + +_DC = Path(__file__).resolve().parents[2] / "dc" / "scripts" +if str(_DC) not in sys.path: + sys.path.insert(0, str(_DC)) + +from tapeout import SramGeom, get_tapeout_contract +from macro_compiler import run_macro_compiler +from sram_compiler import generate_sram_dbs, leaf_names_from_macros + + +def pad_mems_conf(text: str) -> str: + lines = [] + for raw in text.splitlines(): + if not raw.strip(): + continue + lines.append(raw.rstrip() + " ") + if not lines: + raise ValueError("empty mems.conf") + return "\n".join(lines) + "\n" + + +def geoms_from_macros( + *, + macros_v: Path, + sram_table: dict[str, SramGeom], + mdf: Path, +) -> list[SramGeom]: + raw = json.loads(mdf.read_text()) + if not isinstance(raw, list): + raise ValueError(f"mdf must be a list: {mdf}") + index: dict[str, dict] = {} + for entry in raw: + if not isinstance(entry, dict): + raise ValueError(f"mdf entry must be object: {mdf}") + name = entry.get("name") + if not isinstance(name, str) or not name: + raise ValueError(f"mdf entry missing name: {mdf}") + if name in index: + raise ValueError(f"duplicate mdf name {name}: {mdf}") + index[name] = entry + + names = leaf_names_from_macros(macros_v, set(sram_table) | set(index)) + geoms: list[SramGeom] = [] + for name in names: + if name not in sram_table: + raise ValueError(f"sram leaf {name} missing from tapeout.sram") + if name not in index: + raise ValueError(f"sram leaf {name} missing from mdf: {mdf}") + entry = index[name] + try: + depth = int(entry["depth"]) + width = int(entry["width"]) + except (KeyError, TypeError, ValueError) as exc: + raise ValueError(f"mdf entry {name} missing depth/width") from exc + geom = sram_table[name] + if depth != geom.words: + raise ValueError( + f"sram leaf {name}: mdf depth {depth} != tapeout.sram words {geom.words}" + ) + if width != geom.bits: + raise ValueError( + f"sram leaf {name}: mdf width {width} != tapeout.sram bits {geom.bits}" + ) + geoms.append(geom) + return geoms + + +def _leaf_paths(cache_dir: Path, name: str, corner: str) -> dict[str, str]: + leaf = cache_dir / name + return { + "v": str((leaf / f"{name}.v").resolve()), + "lib": str((leaf / f"{name}_{corner}.lib").resolve()), + "db": str((leaf / f"{name}_{corner}.db").resolve()), + } + + +def generate_sram( + *, + bbdir: str | Path, + chip: str, + build_dir: str | Path, + out_dir: str | Path | None = None, +) -> dict: + bbdir = Path(bbdir).resolve() + build_dir = Path(build_dir).resolve() + src = build_dir / "mems.conf" + if not src.is_file(): + raise FileNotFoundError(f"missing elaborator mems.conf: {src}") + padded = pad_mems_conf(src.read_text()) + contract = get_tapeout_contract(bbdir, chip) + if out_dir is None: + dest = build_dir / "ip-generate" + else: + if isinstance(out_dir, str) and not out_dir.strip(): + raise ValueError("empty out_dir") + dest = Path(out_dir).resolve() + dest.mkdir(parents=True, exist_ok=True) + mems_out = dest / "mems.conf" + verilog = dest / "sram_macros.v" + firrtl = dest / "sram_macros.fir" + mems_out.write_text(padded) + arch_dir = bbdir / "arch" + run_macro_compiler( + mems_conf=mems_out, + mdf=contract.sram_mdf, + verilog=verilog, + firrtl=firrtl, + arch_dir=arch_dir, + ) + geoms = geoms_from_macros( + macros_v=verilog, + sram_table=contract.sram_table, + mdf=contract.sram_mdf, + ) + ip_db = build_dir.parent / f"{build_dir.name}-ip-db" + db_paths, corner_tag = generate_sram_dbs( + geoms=geoms, + process=contract.sram_process, + corner=contract.sram_corner, + cache_dir=ip_db, + lc_shell=contract.lc_shell, + ) + names = [g.name for g in geoms] + man = { + "chip": chip, + "sram_mdf": str(contract.sram_mdf), + "mems_conf": str(mems_out), + "sram_macros_v": str(verilog), + "sram_macros_fir": str(firrtl), + "sram_corner": corner_tag, + "ip_db": str(ip_db), + "leaves": names, + "leaf_paths": { + g.name: _leaf_paths(ip_db, g.name, corner_tag) + for g in geoms + }, + "link_dbs": [str(p) for p in db_paths], + "leaf_count": len(names), + "mem_count": sum(1 for line in padded.splitlines() if line.strip()), + } + manifest_path = dest / "generate_manifest.json" + manifest_path.write_text(json.dumps(man, indent=2) + "\n") + return {**man, "generate_manifest": str(manifest_path)} diff --git a/bbdev/api/steps/ip/scripts/macro_compiler.py b/bbdev/api/steps/ip/scripts/macro_compiler.py new file mode 100644 index 000000000..d6c4b6319 --- /dev/null +++ b/bbdev/api/steps/ip/scripts/macro_compiler.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import os +import shlex +import sys +from pathlib import Path + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.stream_run import stream_run + + +def run_macro_compiler( + *, mems_conf: Path, mdf: Path, verilog: Path, firrtl: Path, arch_dir: Path +) -> None: + verilog.parent.mkdir(parents=True, exist_ok=True) + cmd = shlex.join( + [ + "mill", + "-i", + "tapeout.runMain", + "tapeout.macros.MacroCompiler", + "-n", + str(mems_conf), + "-v", + str(verilog), + "-f", + str(firrtl), + "--library", + str(mdf), + "--mode", + "strict", + ] + ) + r = stream_run(cmd, cwd=str(arch_dir)) + if r.returncode != 0: + raise RuntimeError(f"MacroCompiler failed:\n{r.stdout}\n{r.stderr}") + if not verilog.is_file() or verilog.stat().st_size == 0: + raise RuntimeError(f"MacroCompiler produced empty verilog: {verilog}") diff --git a/bbdev/api/steps/ip/scripts/replace.py b/bbdev/api/steps/ip/scripts/replace.py new file mode 100644 index 000000000..4440051bc --- /dev/null +++ b/bbdev/api/steps/ip/scripts/replace.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import json +import re +from pathlib import Path + +MODULE_NAME_RE = re.compile( + r"^\s*(?:\(\*.*?\*\)\s*)*module\s+([A-Za-z_][A-Za-z0-9_$]*)\b", + re.M, +) +INSTANCE_RE = re.compile( + r"^\s*([A-Za-z_][A-Za-z0-9_$]*)\s*(?:#\s*\(.*?\)\s*)?([A-Za-z_][A-Za-z0-9_$]*)\s*\(", + re.M | re.S, +) +COMMENT_RE = re.compile(r"/\*.*?\*/|//.*?$", re.M | re.S) +MODULE_TOKEN_RE = re.compile(r"\bmodule\b") + + +def replace_sources( + *, + source_paths: list[str], + top_module: str, + sram_macros_v: str | Path, + output_dir: str | Path, + consumer: str, + generate_manifest: str | Path | None = None, +) -> dict: + paths = [Path(path).resolve() for path in source_paths] + macros = Path(sram_macros_v).resolve() + + modules: dict[str, Path] = {} + texts: dict[str, str] = {} + for path in paths: + text = path.read_text() + match = MODULE_NAME_RE.search(text) + if match is None: + stripped = COMMENT_RE.sub("", text) + if MODULE_TOKEN_RE.search(stripped): + raise RuntimeError(f"no module declaration: {path}") + continue + name = match.group(1) + if name in modules: + raise RuntimeError(f"duplicate module {name}: {modules[name]} vs {path}") + modules[name] = path + texts[name] = text + + if top_module not in modules: + raise ValueError(f"synthesis top is not defined in source list: {top_module}") + if not macros.is_file(): + raise FileNotFoundError(f"missing sram_macros.v: {macros}") + + reachable = {top_module} + pending = [top_module] + while pending: + cur = pending.pop() + for child, _inst in INSTANCE_RE.findall(texts[cur]): + if child in modules and child not in reachable: + reachable.add(child) + pending.append(child) + + selected_paths = {modules[name] for name in reachable} + selected = [path for path in paths if path in selected_paths] + if macros not in selected: + selected.append(macros) + + out = Path(output_dir) + out.mkdir(parents=True, exist_ok=True) + source_list_path = out / f"{consumer}_sources.list" + source_list_path.write_text("\n".join(str(path) for path in selected) + "\n") + + manifest = { + "top_module": top_module, + "consumer": consumer, + "source_count": len(selected), + "sram_macros_v": str(macros), + } + if generate_manifest is not None: + gman = Path(generate_manifest).resolve() + if not gman.is_file(): + raise FileNotFoundError(f"missing generate_manifest: {gman}") + manifest["generate_manifest"] = str(gman) + manifest_path = out / "replace_manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + return { + "source_list": str(source_list_path), + "replace_manifest": str(manifest_path), + "source_count": len(selected), + } diff --git a/bbdev/api/steps/ip/scripts/sram_compiler.py b/bbdev/api/steps/ip/scripts/sram_compiler.py new file mode 100644 index 000000000..b5adda9e2 --- /dev/null +++ b/bbdev/api/steps/ip/scripts/sram_compiler.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import re +import sys +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +_API = _HERE.parents[2] +_IP = _HERE.parents[4] / "thirdparty" / "soc-framework" / "ip" +for _p in (str(_API), str(_IP)): + if _p not in sys.path: + sys.path.insert(0, _p) + +from smic180.compiler import generate_smic180_sram_dbs + + +def leaf_names_from_macros(macros_v: Path, known: set[str]) -> list[str]: + if not macros_v.is_file(): + raise FileNotFoundError(f"missing macros verilog: {macros_v}") + text = macros_v.read_text() + ordered: list[str] = [] + for name in sorted(known): + pat = re.compile(rf"(?m)^\s*{re.escape(name)}\s+[A-Za-z_][A-Za-z0-9_$]*\s*\(") + if pat.search(text): + ordered.append(name) + if not ordered: + raise RuntimeError(f"no known sram leaf instances in {macros_v}") + return ordered + + +def generate_sram_dbs( + *, + geoms: list, + process: str, + corner: str, + cache_dir: Path, + lc_shell: Path, +) -> tuple[list[Path], str]: + if process == "smic180": + return generate_smic180_sram_dbs(geoms, corner, cache_dir, lc_shell) + elif process == "tsmc28": + raise ValueError("tsmc28 sram is not supported yet") + raise ValueError(f"unknown sram process {process!r}") diff --git a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py index 4af13ec19..eff2a57f7 100644 --- a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py +++ b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py @@ -32,7 +32,7 @@ "description": "generate verilog for yosys flow", "flows": ["yosys"], "triggers": [queue("yosys.run"), queue("yosys.verilog")], - "enqueues": ["ip-replace.run"], + "enqueues": ["ip.generate"], } @@ -215,7 +215,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: yosys_cfg = load_yosys_config() top_module = input_data.get("top") or yosys_cfg.get("top") or "DigitalTop" yosys_log_dir = input_data.get("log_dir") or log_dir( - bbdir, chip, "synth", datetime.now().strftime("%Y%m%d-%H%M%S-%f"), + bbdir, chip, "synth", datetime.now().strftime("%Y-%m-%d-%H-%M"), "yosys", top_module, input_data.get("output_dir"), ) ctx.logger.info(f"Yosys log dir: {yosys_log_dir}") @@ -240,26 +240,28 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return failure_result + need_ip = bool(input_data.get("from_run_workflow")) await check_result( ctx, 0, - continue_run=True, + continue_run=need_ip, extra_fields={"task": "verilog", "source_list": source_list_path, "mem_conf": mem_conf}, trace_id=origin_tid, ) + if not need_ip: + return await ctx.enqueue( { - "topic": "ip-replace.run", + "topic": "ip.generate", "data": { **input_data, - "source_list": source_list_path, - "ip_replace_output_dir": yosys_log_dir, + "chip": chip, "consumer": "yosys", "top": top_module, - "mem_conf": mem_conf, - "next_topic": "yosys.synth" if input_data.get("from_run_workflow") else None, - "task": "run" if input_data.get("from_run_workflow") else "verilog", + "log_dir": yosys_log_dir, + "next_topic": "yosys.synth", + "task": "run", "_trace_id": origin_tid, }, } diff --git a/bbdev/api/steps/mill/04_dc_verilog_event.step.py b/bbdev/api/steps/mill/04_dc_verilog_event.step.py index 5d1ddd862..80c45fdd4 100644 --- a/bbdev/api/steps/mill/04_dc_verilog_event.step.py +++ b/bbdev/api/steps/mill/04_dc_verilog_event.step.py @@ -1,8 +1,8 @@ import os -from datetime import datetime import sys import glob import re +from datetime import datetime from motia import FlowContext, queue @@ -31,7 +31,7 @@ "description": "generate RTL and memory metadata for downstream DC/tapeout flow", "flows": ["dc"], "triggers": [queue("dc.verilog")], - "enqueues": ["ip-replace.run"], + "enqueues": ["ip.generate"], } @@ -63,9 +63,12 @@ def prepare_dc_verilog(build_dir: str): ) stub_dir = os.path.join(build_dir, "dc_stubs") os.makedirs(stub_dir, exist_ok=True) + skip_dirs = {os.path.join(build_dir, name) for name in ("ip-generate", "ip-replace", "dc_stubs")} kept = [] stubbed_dpi = [] for path in vsrcs: + if any(path == d or path.startswith(d + os.sep) for d in skip_dirs): + continue if is_dpi_source(path): stub_path = os.path.join(stub_dir, f"stub_{os.path.basename(path)}") with open(stub_path, "w") as f: @@ -146,10 +149,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"{'...' if len(stubbed_dpi) > 10 else ''}" ) + need_ip = bool( + input_data.get("from_area_workflow") or input_data.get("from_power_workflow") + ) await check_result( ctx, 0, - continue_run=True, + continue_run=need_ip, extra_fields={ "task": "verilog", "source_list": source_list_path, @@ -158,21 +164,25 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: }, trace_id=origin_tid, ) + if not need_ip: + return payload = { **input_data, - "source_list": source_list_path, - "ip_replace_output_dir": os.path.join(build_dir, "ip-replace"), + "chip": chip, "consumer": "dc", - "mem_conf": mem_conf, "top": top_module, - "next_topic": "dc.area" if input_data.get("from_area_workflow") or input_data.get("from_power_workflow") else None, + "next_topic": "dc.area", "_trace_id": origin_tid, } if input_data.get("from_power_workflow"): - payload["analysis_dir"] = log_dir(bbdir, chip, "synth", datetime.now().strftime("%Y%m%d-%H%M%S-%f"), "dc", "power") - elif input_data.get("from_area_workflow"): - payload["analysis_dir"] = log_dir(bbdir, chip, "synth", datetime.now().strftime("%Y%m%d-%H%M%S-%f"), "dc", "area") - await ctx.enqueue({"topic": "ip-replace.run", "data": payload}) + payload["analysis_dir"] = log_dir( + bbdir, chip, "synth", datetime.now().strftime("%Y-%m-%d-%H-%M"), "dc", "power" + ) + else: + payload["analysis_dir"] = log_dir( + bbdir, chip, "synth", datetime.now().strftime("%Y-%m-%d-%H-%M"), "dc", "area" + ) + await ctx.enqueue({"topic": "ip.generate", "data": payload}) return diff --git a/bbdev/api/steps/dc/06_ptpx_event.step.py b/bbdev/api/steps/pt/01_ptpx_event.step.py similarity index 79% rename from bbdev/api/steps/dc/06_ptpx_event.step.py rename to bbdev/api/steps/pt/01_ptpx_event.step.py index b3c1334d0..d952f6231 100644 --- a/bbdev/api/steps/dc/06_ptpx_event.step.py +++ b/bbdev/api/steps/pt/01_ptpx_event.step.py @@ -11,21 +11,21 @@ scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -step_path = os.path.dirname(__file__) -if step_path not in sys.path: - sys.path.insert(0, step_path) +dc_scripts_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "dc", "scripts")) +if dc_scripts_path not in sys.path: + sys.path.insert(0, dc_scripts_path) from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path -from power import read_dynamic_power from utils.stream_run import stream_run_logger_async -from tapeout import get_tapeout_contract, resolve_power_window, technology_settings, write_run_tcl +from power import read_dynamic_power +from tapeout import get_tapeout_contract, resolve_power_window, write_run_tcl config = { - "name": "dc-ptpx", - "description": "run PrimeTime PX power analysis on DC output", - "flows": ["dc"], - "triggers": [queue("dc.power")], + "name": "pt-ptpx", + "description": "run PrimeTime PX power analysis", + "flows": ["pt", "dc"], + "triggers": [queue("pt.run")], "enqueues": [], } @@ -43,11 +43,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: elif not isinstance(activity_format, str) or activity_format not in {"saif", "vcd", "fsdb"}: error = "missing or invalid --format; expected saif, vcd, or fsdb" elif shutil.which("pt_shell") is None: - error = "pt_shell is not on PATH; source the DC host environment before running bbdev dc --power" + error = "pt_shell is not on PATH" else: error = None if error: - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "power", "error": error}, trace_id=origin_tid) + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "pt", "error": error}, trace_id=origin_tid) return try: @@ -59,7 +59,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: input_data.get("start_ns", input_data.get("start-ns")), input_data.get("end_ns", input_data.get("end-ns")), ) - tech = technology_settings() + tech = { + "target_library": contract.target_library, + "synthetic_library": contract.synthetic_library, + "link_library": contract.link_library, + "max_cores": contract.max_cores, + } run_config = write_run_tcl( os.path.join(analysis_dir, "power-run.tcl"), { @@ -77,24 +82,24 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) script = contract.power_script except (OSError, ValueError) as exc: - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "power", "error": str(exc)}, trace_id=origin_tid) + await check_result(ctx, 1, continue_run=False, extra_fields={"task": "pt", "error": str(exc)}, trace_id=origin_tid) return result = await stream_run_logger_async( cmd=( - f"pt_shell -f {shlex.quote(str(script))} " + f"set -o pipefail; pt_shell -f {shlex.quote(str(script))} " f"-x {shlex.quote('set RUN_CONFIG ' + '{' + str(run_config) + '}')} " - f"> {shlex.quote(os.path.join(analysis_dir, 'pt_shell.log'))} 2>&1" + f"2>&1 | tee {shlex.quote(os.path.join(analysis_dir, 'pt_shell.log'))}" ), logger=ctx.logger, cwd=os.path.dirname(script), - stdout_prefix="ptpx", - stderr_prefix="ptpx", + stdout_prefix="pt", + stderr_prefix="pt", ) report_dir = os.path.join(analysis_dir, "power-reports") dynamic_power = read_dynamic_power(os.path.join(report_dir, "power_total.rpt")) extra_fields = { - "task": "power", + "task": "pt", "top_module": top_module, "activity": activity_path, "format": activity_format, diff --git a/bbdev/api/steps/dc/power.py b/bbdev/api/steps/pt/scripts/power.py similarity index 100% rename from bbdev/api/steps/dc/power.py rename to bbdev/api/steps/pt/scripts/power.py diff --git a/bbdev/api/steps/vcs/04_sim_event.step.py b/bbdev/api/steps/vcs/04_sim_event.step.py index b19cbef27..82977b913 100644 --- a/bbdev/api/steps/vcs/04_sim_event.step.py +++ b/bbdev/api/steps/vcs/04_sim_event.step.py @@ -58,7 +58,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: build_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("output_dir")) artifact_dir = Path(build_dir) / "vcs" simv = artifact_dir / "simv" - timestamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") run_log = Path(log_dir( bbdir, chip, "verilog", timestamp, "vcs", binary_name, input_data.get("output_dir"), diff --git a/bbdev/api/steps/yosys/03_synth_api.step.py b/bbdev/api/steps/yosys/03_synth_api.step.py index 7e393a9d3..2994e98dc 100644 --- a/bbdev/api/steps/yosys/03_synth_api.step.py +++ b/bbdev/api/steps/yosys/03_synth_api.step.py @@ -4,7 +4,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api from utils.event_common import require_chip -from utils.path import rtl_dir, get_buckyball_path scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: @@ -17,39 +16,36 @@ "description": "run yosys synthesis for area estimation", "flows": ["yosys"], "triggers": [api("POST", "/yosys/synth")], - "enqueues": ["ip-replace.run"], + "enqueues": ["ip.generate"], } async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = req.body or {} try: chip = require_chip(body) - rtl = rtl_dir(bbdir, chip, "synth", req_arg(body, "output_dir")) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) data = { "chip": chip, - "output_dir": rtl, + "consumer": "yosys", "top": req_arg(body, "top") or "DigitalTop", - "vcd": req_arg(body, "vcd"), + "next_topic": "yosys.synth", } + output_dir = req_arg(body, "output_dir") + if output_dir: + data["output_dir"] = output_dir + vcd = req_arg(body, "vcd") + if vcd: + data["vcd"] = vcd log_dir = req_arg(body, "log_dir") if log_dir: data["log_dir"] = log_dir await ctx.enqueue( { - "topic": "ip-replace.run", - "data": { - **data, - "source_list": os.path.join(rtl, "yosys_sources.list"), - "ip_replace_output_dir": log_dir or rtl, - "consumer": "yosys", - "next_topic": "yosys.synth", - "_trace_id": ctx.trace_id, - }, + "topic": "ip.generate", + "data": {**data, "_trace_id": ctx.trace_id}, } ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/yosys/03_synth_event.step.py b/bbdev/api/steps/yosys/03_synth_event.step.py index 2aee9db3f..3c297bbc1 100644 --- a/bbdev/api/steps/yosys/03_synth_event.step.py +++ b/bbdev/api/steps/yosys/03_synth_event.step.py @@ -81,13 +81,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if isinstance(liberty, str): liberty = os.path.expandvars(os.path.expanduser(liberty)) - source_list_path = input_data.get("source_list") or os.path.join(build_dir, "yosys_sources.list") - if not os.path.exists(source_list_path): - success_result, failure_result = await check_result( + source_list_path = input_data.get("source_list") + if not isinstance(source_list_path, str) or not os.path.isfile(source_list_path): + _, failure_result = await check_result( ctx, 1, continue_run=False, - extra_fields={"task": "synth", "error": "missing yosys_sources.list, run yosys verilog first"}, + extra_fields={"task": "synth", "error": "missing prepared yosys source list"}, trace_id=origin_tid, ) return failure_result @@ -96,24 +96,22 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: vsrcs = [line.strip() for line in f.readlines() if line.strip()] if not vsrcs: - success_result, failure_result = await check_result( + _, failure_result = await check_result( ctx, 1, continue_run=False, - extra_fields={"task": "synth", "error": "empty yosys_sources.list"}, + extra_fields={"task": "synth", "error": "empty source list"}, trace_id=origin_tid, ) return failure_result - stamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") yosys_output_dir = input_data.get("log_dir") or log_dir( bbdir, chip, "synth", stamp, "yosys", top_module, input_data.get("output_dir"), ) os.makedirs(yosys_output_dir, exist_ok=True) ctx.logger.info(f"Yosys log dir: {yosys_output_dir}") - sram_collateral = input_data.get("sram_collateral") or {} - read_commands = "\n".join([f"read_verilog -sv {src}" for src in vsrcs]) yosys_script = f"{yosys_output_dir}/synth_area.ys" with open(yosys_script, "w") as f: @@ -150,8 +148,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra = { "task": "synth", "output_dir": yosys_output_dir, - "sram_manifest": sram_collateral.get("sram_manifest"), - "sram_memory_count": sram_collateral.get("sram_memory_count", 0), + "replace_manifest": input_data.get("replace_manifest"), } netlist_file = f"{yosys_output_dir}/synth_netlist.v" timing_report_file = f"{yosys_output_dir}/timing_report.txt" diff --git a/bbdev/api/steps/yosys/scripts/yosys_log.py b/bbdev/api/steps/yosys/scripts/yosys_log.py index e58811beb..a79c3deef 100644 --- a/bbdev/api/steps/yosys/scripts/yosys_log.py +++ b/bbdev/api/steps/yosys/scripts/yosys_log.py @@ -3,10 +3,10 @@ def req_arg(body: dict, name: str): - return body.get(name) or body.get(name.replace("_", "-")) + return body.get(name) or body.get(name.replace("_", "-")) def make_yosys_log_dir(bbdir: str, trace_id: str) -> str: - stamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f") - suffix = trace_id[:8] if trace_id else "no-trace" - return os.path.join(bbdir, "bbdev", "api", "steps", "yosys", "log", f"{stamp}-{suffix}") + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + suffix = trace_id[:8] if trace_id else "no-trace" + return os.path.join(bbdir, "bbdev", "api", "steps", "yosys", "log", f"{stamp}-{suffix}") diff --git a/bbdev/api/tests/test_api_dc_verilog.py b/bbdev/api/tests/test_api_dc_verilog.py index 41aa489fe..a6247c22f 100644 --- a/bbdev/api/tests/test_api_dc_verilog.py +++ b/bbdev/api/tests/test_api_dc_verilog.py @@ -1,16 +1,3 @@ -import json -from pathlib import Path - from _api_test_helper import run_bbdev_case run_bbdev_case("bbdev dc --verilog '--chip toy'") - -root = Path(__file__).resolve().parents[3] -output_dir = root / "arch" / "build" / "toy" / "sims.verilator.BuckyballToyVerilatorConfig" -manifest = json.loads((output_dir / "ip-replace" / "sram_manifest.json").read_text()) -source_list = (output_dir / "ip-replace" / "dc_sources.list").read_text().splitlines() - -assert manifest["top_module"] == "DigitalTop" -assert manifest["memories"] -assert any(path.endswith("/DigitalTop.sv") for path in source_list) -assert not any(path.endswith("/BBSimHarness.sv") for path in source_list) diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index e61fc89b4..35d74b787 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -4,8 +4,6 @@ import subprocess from pathlib import Path -_PRODUCT = {"verilog": "verilator", "synth": "verilator", "p2e": "p2e"} - def get_buckyball_path(): current_dir = os.path.dirname(__file__) @@ -68,7 +66,11 @@ def chip_arch_root(bbdir, chip): def sim_name(bbdir, chip, product, *, rushb=False): - if product not in _PRODUCT: + if product == "verilog" or product == "synth": + sim_key = "verilator" + elif product == "p2e": + sim_key = "p2e" + else: raise ValueError(f"invalid rtl product: {product}") path = ( Path(bbdir) @@ -80,7 +82,7 @@ def sim_name(bbdir, chip, product, *, rushb=False): / "config" / "config.json" ) - name = json.loads(path.read_text(encoding="utf-8"))["sims"][_PRODUCT[product]] + name = json.loads(path.read_text(encoding="utf-8"))["sims"][sim_key] if not rushb: return name if product != "verilog" or not name.endswith("VerilatorConfig"): diff --git a/bbdev/bbdev b/bbdev/bbdev index 933af683b..a4e7ba6b2 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -86,8 +86,9 @@ WORKFLOW_COMMANDS = { "area": 'Generate RTL, run DC synthesis, and report area under log////area. Args: "--chip [--top ]".', "power": 'Generate RTL, run chip-owned power simulation, then run PrimeTime PX under log////power. Args: "--chip [--top ] [--workload ] [--start-ns ] [--end-ns ] [--activity --format ]".', }, - "ip-replace": { - "run": 'Replace behavioral IP RTL. Args: "--source-list --output-dir [--top ] [--consumer ]"', + "ip": { + "generate": 'Generate SRAM macros from elaborator mems.conf under arch/build//. Args: "--chip [--consumer ] [--top ] [--output-dir ] [--next-topic ]"', + "replace": 'Assemble top-scoped synthesis RTL plus SRAM macros under arch/build//. Args: "--chip [--consumer ] [--top ] [--output-dir ] [--next-topic ]"', }, "yosys": { "run": 'Run yosys flow. Args: "[--top ] [--chip ] [--output-dir ] [--log-dir ] [--vcd ]"', @@ -555,7 +556,8 @@ def _submit_and_poll(port: int, api_path: str, cmd_info: dict): print(f"\nTask completed on http://{HOST}:{port}") if task_result and not task_result.get("success", False): - print("Error: Task failed") + err = task_result.get("error") or task_result + print(f"Error: Task failed: {err}") return 1 return 0 diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index 3656d0725..bb31b3f71 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -33,6 +33,7 @@ from . import dc_verilog from . import dc_area from . import dc_power +from . import ip_generate from . import ip_replace from . import firesim_enumeratefpgas from . import firesim_buildbitstream @@ -79,6 +80,7 @@ dc_verilog, dc_area, dc_power, + ip_generate, ip_replace, firesim_enumeratefpgas, firesim_buildbitstream, diff --git a/bbdev/mcp/tools/ip_generate.py b/bbdev/mcp/tools/ip_generate.py new file mode 100644 index 000000000..be8bd5404 --- /dev/null +++ b/bbdev/mcp/tools/ip_generate.py @@ -0,0 +1,33 @@ +"""MCP tool: bbdev_ip_generate.""" + +from __future__ import annotations + +from typing import Optional + +from common import err, fmt, need, opt, submit + + +def register(mcp): + @mcp.tool() + def bbdev_ip_generate( + chip: str, + consumer: Optional[str] = None, + top: Optional[str] = None, + output_dir: Optional[str] = None, + next_topic: Optional[str] = None, + ) -> str: + """Generate SRAM macros from mems.conf. POST /ip/generate.""" + if e := need("chip", chip): + return err(e) + return fmt( + submit( + "/ip/generate", + opt( + {"chip": chip}, + consumer=consumer, + top=top, + output_dir=output_dir, + next_topic=next_topic, + ), + ) + ) diff --git a/bbdev/mcp/tools/ip_replace.py b/bbdev/mcp/tools/ip_replace.py index ecec7daea..dccafa35e 100644 --- a/bbdev/mcp/tools/ip_replace.py +++ b/bbdev/mcp/tools/ip_replace.py @@ -1,26 +1,33 @@ -"""MCP tool: bbdev_ip_replace_run.""" +"""MCP tool: bbdev_ip_replace.""" from __future__ import annotations from typing import Optional -from common import err, fmt, opt, submit +from common import err, fmt, need, opt, submit def register(mcp): @mcp.tool() - def bbdev_ip_replace_run( - source_list: str, - output_dir: str, - top: Optional[str] = None, + def bbdev_ip_replace( + chip: str, consumer: Optional[str] = None, + top: Optional[str] = None, + output_dir: Optional[str] = None, + next_topic: Optional[str] = None, ) -> str: - """Replace behavioral IP RTL. POST /ip/replace/run.""" - if not source_list or not output_dir: - return err("source_list and output_dir are required") + """Assemble synthesis source list plus SRAM macros. POST /ip/replace.""" + if e := need("chip", chip): + return err(e) return fmt( submit( - "/ip/replace/run", - opt({}, source_list=source_list, output_dir=output_dir, top=top, consumer=consumer), + "/ip/replace", + opt( + {"chip": chip}, + consumer=consumer, + top=top, + output_dir=output_dir, + next_topic=next_topic, + ), ) ) From 02de97b766fde1ad0fd16f3db4748c88ad1d4e44 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 3 Sep 2026 15:57:03 +0800 Subject: [PATCH 136/196] feat: update build and simulation steps to support tapeout output directory --- bbdev/api/lc_output.txt | 29 ++++ bbdev/api/steps/ip/01_generate_event.step.py | 2 +- bbdev/api/steps/ip/02_replace_event.step.py | 2 +- bbdev/api/steps/ip/scripts/generate.py | 124 ++++++++++++++++-- .../steps/mill/02_vcs_verilog_event.step.py | 2 +- .../steps/mill/04_dc_verilog_event.step.py | 6 +- bbdev/api/steps/uvm/scripts/uvm_common.py | 9 +- bbdev/api/steps/vcs/01_clean_api.step.py | 2 +- bbdev/api/steps/vcs/01_clean_event.step.py | 3 +- bbdev/api/steps/vcs/03_build_api.step.py | 2 +- bbdev/api/steps/vcs/03_build_event.step.py | 2 +- bbdev/api/steps/vcs/04_sim_event.step.py | 4 +- bbdev/api/steps/vcs/05_run_api.step.py | 2 +- bbdev/api/utils/path.py | 2 + 14 files changed, 160 insertions(+), 31 deletions(-) create mode 100644 bbdev/api/lc_output.txt diff --git a/bbdev/api/lc_output.txt b/bbdev/api/lc_output.txt new file mode 100644 index 000000000..85f5db05d --- /dev/null +++ b/bbdev/api/lc_output.txt @@ -0,0 +1,29 @@ + + Library Compiler (TM) + DesignWare (R) + + Version R-2020.09-SP5 for linux64 - Apr 14, 2021 + + Copyright (c) 1988 - 2021 Synopsys, Inc. + This software and the associated documentation are proprietary to Synopsys, + Inc. This software may only be used in accordance with the terms and conditions + of a written license agreement with Synopsys, Inc. All other use, reproduction, + or distribution of this software is strictly prohibited. + +Initializing... +read_lib {/home/wanghui/Code/buckyball/arch/build/pebble/sims.tapeout.BuckyballPebbleTapeoutConfig-ip-db/chipyard_sram_64x8/chipyard_sram_64x8_ss_1.62_125.lib} +Reading '/home/wanghui/Code/buckyball/arch/build/pebble/sims.tapeout.BuckyballPebbleTapeoutConfig-ip-db/chipyard_sram_64x8/chipyard_sram_64x8_ss_1.62_125.lib' ... +Warning: Line 50, The 'internal_power_calculation' attribute in char_config group is required for NLPM library. + No default can be applied to this attribute. (LBDB-366) +Information: Line 235, Cell 'chipyard_sram_64x8', The cell 'chipyard_sram_64x8' contiains memory/memory_read/memory_write groups which are deprecated, use the is_memory_cell attribute instead. (LIBG-280) +Technology library 'chipyard_sram_64x8_ss_1.62_125' read successfully +1 +write_lib [get_object_name [get_libs *]] -format db -output {/home/wanghui/Code/buckyball/arch/build/pebble/sims.tapeout.BuckyballPebbleTapeoutConfig-ip-db/chipyard_sram_64x8/chipyard_sram_64x8_ss_1.62_125.db} +Wrote the 'chipyard_sram_64x8_ss_1.62_125' library to '/home/wanghui/Code/buckyball/arch/build/pebble/sims.tapeout.BuckyballPebbleTapeoutConfig-ip-db/chipyard_sram_64x8/chipyard_sram_64x8_ss_1.62_125.db' successfully +1 +exit +Maximum memory usage for this session: 84.32 MB +CPU usage for this session: 1 seconds ( 0.00 hours) +Elapsed time for this session: 1 seconds ( 0.00 hours) +Thank you for using Library Compiler. + diff --git a/bbdev/api/steps/ip/01_generate_event.step.py b/bbdev/api/steps/ip/01_generate_event.step.py index f44e43a9f..994d82d48 100644 --- a/bbdev/api/steps/ip/01_generate_event.step.py +++ b/bbdev/api/steps/ip/01_generate_event.step.py @@ -37,7 +37,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if not isinstance(top, str) or not top: raise ValueError("top must be a non-empty string") bbdir = Path(get_buckyball_path()).resolve() - build_dir = Path(rtl_dir(bbdir, chip, "verilog", input_data.get("output_dir"))).resolve() + build_dir = Path(rtl_dir(bbdir, chip, "tapeout" if consumer == "dc" else "verilog", input_data.get("output_dir"))).resolve() man = generate_sram(bbdir=bbdir, chip=chip, build_dir=build_dir) except Exception as exc: _, failure_result = await check_result( diff --git a/bbdev/api/steps/ip/02_replace_event.step.py b/bbdev/api/steps/ip/02_replace_event.step.py index 10d5234c7..873ee2c00 100644 --- a/bbdev/api/steps/ip/02_replace_event.step.py +++ b/bbdev/api/steps/ip/02_replace_event.step.py @@ -40,7 +40,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: raise ValueError("top must be a non-empty string") bbdir = Path(get_buckyball_path()).resolve() - build_dir = Path(rtl_dir(bbdir, chip, "verilog", input_data.get("output_dir"))).resolve() + build_dir = Path(rtl_dir(bbdir, chip, "tapeout" if consumer == "dc" else "verilog", input_data.get("output_dir"))).resolve() source_list_path = build_dir / _SOURCE_LIST[consumer] if not source_list_path.is_file(): raise FileNotFoundError(f"missing source list: {source_list_path}") diff --git a/bbdev/api/steps/ip/scripts/generate.py b/bbdev/api/steps/ip/scripts/generate.py index f883eb742..2bb845a50 100644 --- a/bbdev/api/steps/ip/scripts/generate.py +++ b/bbdev/api/steps/ip/scripts/generate.py @@ -1,7 +1,9 @@ from __future__ import annotations import json +import re import sys +from collections import Counter from pathlib import Path _DC = Path(__file__).resolve().parents[2] / "dc" / "scripts" @@ -12,6 +14,9 @@ from macro_compiler import run_macro_compiler from sram_compiler import generate_sram_dbs, leaf_names_from_macros +_MODULE = re.compile(r"(?m)^module\s+(\w+)\s*\(") +_INST = re.compile(r"(?m)^\s*([A-Za-z_][A-Za-z0-9_$]*)\s+[A-Za-z_][A-Za-z0-9_$]*\s*\(") + def pad_mems_conf(text: str) -> str: lines = [] @@ -24,16 +29,11 @@ def pad_mems_conf(text: str) -> str: return "\n".join(lines) + "\n" -def geoms_from_macros( - *, - macros_v: Path, - sram_table: dict[str, SramGeom], - mdf: Path, -) -> list[SramGeom]: +def load_mdf(mdf: Path) -> dict[str, tuple[int, int]]: raw = json.loads(mdf.read_text()) if not isinstance(raw, list): raise ValueError(f"mdf must be a list: {mdf}") - index: dict[str, dict] = {} + index: dict[str, tuple[int, int]] = {} for entry in raw: if not isinstance(entry, dict): raise ValueError(f"mdf entry must be object: {mdf}") @@ -42,8 +42,104 @@ def geoms_from_macros( raise ValueError(f"mdf entry missing name: {mdf}") if name in index: raise ValueError(f"duplicate mdf name {name}: {mdf}") - index[name] = entry + try: + depth = int(entry["depth"]) + width = int(entry["width"]) + except (KeyError, TypeError, ValueError) as exc: + raise ValueError(f"mdf entry {name} missing depth/width") from exc + if depth <= 0 or width <= 0: + raise ValueError(f"mdf entry {name}: depth/width must be positive") + index[name] = (depth, width) + return index + + +def parse_mems(text: str) -> list[tuple[str, int, int]]: + out: list[tuple[str, int, int]] = [] + for i, raw in enumerate(text.splitlines(), 1): + line = raw.strip() + if not line: + continue + parts = line.split() + if len(parts) < 6 or parts[0] != "name" or parts[2] != "depth" or parts[4] != "width": + raise ValueError(f"mems.conf:{i}: expected 'name depth width ...', got {line!r}") + try: + depth = int(parts[3]) + width = int(parts[5]) + except ValueError as exc: + raise ValueError(f"mems.conf:{i}: depth/width not int") from exc + if depth <= 0 or width <= 0: + raise ValueError(f"mems.conf:{i}: depth/width must be positive") + out.append((parts[1], depth, width)) + if not out: + raise ValueError("empty mems.conf") + return out + + +def module_insts(text: str) -> dict[str, Counter[str]]: + hits = list(_MODULE.finditer(text)) + out: dict[str, Counter[str]] = {} + for i, m in enumerate(hits): + name = m.group(1) + if name in out: + raise ValueError(f"duplicate module {name}") + start = m.end() + end = hits[i + 1].start() if i + 1 < len(hits) else len(text) + counts: Counter[str] = Counter() + for inst in _INST.finditer(text[start:end]): + leaf = inst.group(1) + if leaf in ("module", "assign", "wire", "input", "output"): + raise ValueError(f"module {name}: refused instance parse {leaf!r}") + counts[leaf] += 1 + out[name] = counts + return out + +def check_mapping(*, mems_text: str, macros_v: Path, mdf: dict[str, tuple[int, int]]) -> None: + if not macros_v.is_file(): + raise FileNotFoundError(f"missing macros verilog: {macros_v}") + insts = module_insts(macros_v.read_text()) + lib = ", ".join(f"{n}={d}x{w}" for n, (d, w) in sorted(mdf.items())) + errs: list[str] = [] + for name, depth, width in parse_mems(mems_text): + need = depth * width + if name not in insts: + errs.append(f"{name}: missing module for {depth}x{width} ({need} bits)") + continue + counts = insts[name] + if not counts: + errs.append(f"{name}: empty mapping for {depth}x{width} ({need} bits)") + continue + got = 0 + parts = [] + unknown = False + for leaf, n in sorted(counts.items()): + if leaf not in mdf: + errs.append(f"{name}: unknown leaf {leaf}") + unknown = True + break + d, w = mdf[leaf] + got += n * d * w + parts.append(f"{n}x {leaf} ({d}x{w})") + if unknown: + continue + if got != need: + errs.append( + f"{name}: need {depth}x{width} = {need} bits, " + f"got {' + '.join(parts)} = {got} bits" + ) + if errs: + raise RuntimeError( + "SRAM mapping mismatch:\n " + "\n ".join(errs) + f"\nlibrary: {lib}" + ) + + +def geoms_from_macros( + *, + macros_v: Path, + sram_table: dict[str, SramGeom], + mdf: Path, +) -> list[SramGeom]: + index = load_mdf(mdf) names = leaf_names_from_macros(macros_v, set(sram_table) | set(index)) geoms: list[SramGeom] = [] for name in names: @@ -51,12 +147,7 @@ def geoms_from_macros( raise ValueError(f"sram leaf {name} missing from tapeout.sram") if name not in index: raise ValueError(f"sram leaf {name} missing from mdf: {mdf}") - entry = index[name] - try: - depth = int(entry["depth"]) - width = int(entry["width"]) - except (KeyError, TypeError, ValueError) as exc: - raise ValueError(f"mdf entry {name} missing depth/width") from exc + depth, width = index[name] geom = sram_table[name] if depth != geom.words: raise ValueError( @@ -112,6 +203,11 @@ def generate_sram( firrtl=firrtl, arch_dir=arch_dir, ) + check_mapping( + mems_text=padded, + macros_v=verilog, + mdf=load_mdf(contract.sram_mdf), + ) geoms = geoms_from_macros( macros_v=verilog, sram_table=contract.sram_table, diff --git a/bbdev/api/steps/mill/02_vcs_verilog_event.step.py b/bbdev/api/steps/mill/02_vcs_verilog_event.step.py index 6333bbfbe..eb6c1feda 100644 --- a/bbdev/api/steps/mill/02_vcs_verilog_event.step.py +++ b/bbdev/api/steps/mill/02_vcs_verilog_event.step.py @@ -40,7 +40,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: arch = os.path.join(bbdir, "arch") try: mill_config, build_dir = rtl_out( - bbdir, chip, "verilog", input_data.get("output_dir"), + bbdir, chip, "tapeout", input_data.get("output_dir"), ) os.makedirs(build_dir, exist_ok=True) returncode = ( diff --git a/bbdev/api/steps/mill/04_dc_verilog_event.step.py b/bbdev/api/steps/mill/04_dc_verilog_event.step.py index 80c45fdd4..6b8ed804c 100644 --- a/bbdev/api/steps/mill/04_dc_verilog_event.step.py +++ b/bbdev/api/steps/mill/04_dc_verilog_event.step.py @@ -93,7 +93,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: try: chip = require_chip(input_data) mill_config, build_dir = rtl_out( - bbdir, chip, "verilog", input_data.get("output_dir"), + bbdir, chip, "tapeout", input_data.get("output_dir"), ) os.makedirs(build_dir, exist_ok=True) ctx.logger.info(f"Using mill config: {mill_config}") @@ -177,11 +177,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: } if input_data.get("from_power_workflow"): payload["analysis_dir"] = log_dir( - bbdir, chip, "synth", datetime.now().strftime("%Y-%m-%d-%H-%M"), "dc", "power" + bbdir, chip, "tapeout", datetime.now().strftime("%Y-%m-%d-%H-%M"), "dc", "power" ) else: payload["analysis_dir"] = log_dir( - bbdir, chip, "synth", datetime.now().strftime("%Y-%m-%d-%H-%M"), "dc", "area" + bbdir, chip, "tapeout", datetime.now().strftime("%Y-%m-%d-%H-%M"), "dc", "area" ) await ctx.enqueue({"topic": "ip.generate", "data": payload}) diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index 6ac613d63..9db37cbb9 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -54,7 +54,7 @@ def core_ball_isa_defines(bbdir: str, input_data: dict, ball: str) -> list[str]: }, "smatmul": { "SMATMUL_OS": "SMATMUL_OS_FUNCT7", - "SMATMUL_WS": "SMATMUL_WS_FUNCT7", + "SMATMUL_BIAS": "SMATMUL_BIAS_FUNCT7", }, "matadd": {"MATADD": "MATADD_FUNCT7"}, "im2col": {"IM2COL": "IM2COL_FUNCT7"}, @@ -85,10 +85,11 @@ def core_ball_isa_defines(bbdir: str, input_data: dict, ball: str) -> list[str]: if isinstance(entry, dict) and entry.get("ballName") == "SMatMulBall"] if len(matching) != 1: raise ValueError(f"{domain_path} must define exactly one SMatMulBall mapping") + in_bw = matching[0].get("inBW") out_bw = matching[0].get("outBW") - if not isinstance(out_bw, int) or out_bw <= 0 or out_bw > 4 or 4 % out_bw: - raise ValueError(f"invalid SMatMulBall outBW in {domain_path}") - defines.append(f"+define+SMATMUL_OUT_BW={out_bw}") + if in_bw != 2 or out_bw != 1: + raise ValueError(f"SMatMulBall must use inBW=2,outBW=1 in {domain_path}") + defines.append("+define+SMATMUL_OUT_BW=1") return defines diff --git a/bbdev/api/steps/vcs/01_clean_api.step.py b/bbdev/api/steps/vcs/01_clean_api.step.py index 71adc3ff5..899228ed2 100644 --- a/bbdev/api/steps/vcs/01_clean_api.step.py +++ b/bbdev/api/steps/vcs/01_clean_api.step.py @@ -25,7 +25,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: "topic": "vcs.clean", "data": { "chip": chip, - "output_dir": rtl_dir(bbdir, chip, "verilog", body.get("output_dir")), + "output_dir": rtl_dir(bbdir, chip, "tapeout", body.get("output_dir")), "_trace_id": ctx.trace_id, }, } diff --git a/bbdev/api/steps/vcs/01_clean_event.step.py b/bbdev/api/steps/vcs/01_clean_event.step.py index 7b5b0b35c..c7a9c7ac7 100644 --- a/bbdev/api/steps/vcs/01_clean_event.step.py +++ b/bbdev/api/steps/vcs/01_clean_event.step.py @@ -24,7 +24,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - build_dir = rtl_dir(bbdir, input_data.get("config", "verilog"), input_data.get("output_dir")) + chip = input_data.get("chip") + build_dir = rtl_dir(bbdir, chip, "tapeout", input_data.get("output_dir")) artifact_dir = os.path.join(build_dir, "vcs") if os.path.isdir(artifact_dir): shutil.rmtree(artifact_dir) diff --git a/bbdev/api/steps/vcs/03_build_api.step.py b/bbdev/api/steps/vcs/03_build_api.step.py index c1f5d788b..901fba742 100644 --- a/bbdev/api/steps/vcs/03_build_api.step.py +++ b/bbdev/api/steps/vcs/03_build_api.step.py @@ -30,7 +30,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "chip": chip, "jobs": jobs, - "output_dir": rtl_dir(bbdir, chip, "verilog", body.get("output_dir")), + "output_dir": rtl_dir(bbdir, chip, "tapeout", body.get("output_dir")), } await ctx.enqueue({"topic": "vcs.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/03_build_event.step.py b/bbdev/api/steps/vcs/03_build_event.step.py index 7c7ca9aef..52e282025 100644 --- a/bbdev/api/steps/vcs/03_build_event.step.py +++ b/bbdev/api/steps/vcs/03_build_event.step.py @@ -130,7 +130,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return bbdir = get_buckyball_path() - build_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("output_dir")) + build_dir = rtl_dir(bbdir, chip, "tapeout", input_data.get("output_dir")) vsrcs = sorted( path for path in ( glob.glob(f"{build_dir}/**/*.v", recursive=True) diff --git a/bbdev/api/steps/vcs/04_sim_event.step.py b/bbdev/api/steps/vcs/04_sim_event.step.py index 82977b913..1ea55cdee 100644 --- a/bbdev/api/steps/vcs/04_sim_event.step.py +++ b/bbdev/api/steps/vcs/04_sim_event.step.py @@ -55,12 +55,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if binary is None: await check_result(ctx, 1, extra_fields={"task": "sim", "error": f"binary not found: {binary_name}"}, trace_id=origin_tid) return - build_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("output_dir")) + build_dir = rtl_dir(bbdir, chip, "tapeout", input_data.get("output_dir")) artifact_dir = Path(build_dir) / "vcs" simv = artifact_dir / "simv" timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") run_log = Path(log_dir( - bbdir, chip, "verilog", timestamp, "vcs", binary_name, + bbdir, chip, "tapeout", timestamp, "vcs", binary_name, input_data.get("output_dir"), )) run_log.mkdir(parents=True) diff --git a/bbdev/api/steps/vcs/05_run_api.step.py b/bbdev/api/steps/vcs/05_run_api.step.py index 2b2494a8e..f90f21290 100644 --- a/bbdev/api/steps/vcs/05_run_api.step.py +++ b/bbdev/api/steps/vcs/05_run_api.step.py @@ -35,7 +35,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: "binary": binary, "batch": bool(body.get("batch", False)), "jobs": jobs, - "output_dir": rtl_dir(bbdir, chip, "verilog", body.get("output_dir")), + "output_dir": rtl_dir(bbdir, chip, "tapeout", body.get("output_dir")), "from_run_workflow": True, } await ctx.enqueue({"topic": "vcs.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index 35d74b787..60e332efc 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -68,6 +68,8 @@ def chip_arch_root(bbdir, chip): def sim_name(bbdir, chip, product, *, rushb=False): if product == "verilog" or product == "synth": sim_key = "verilator" + elif product == "tapeout": + sim_key = "tapeout" elif product == "p2e": sim_key = "p2e" else: From 6bd07c0fe6efc46e11662f1dbcb674d25b5632d8 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 4 Sep 2026 02:12:02 +0800 Subject: [PATCH 137/196] fix: update execution command for motia worker in bbdev and config.yaml to fix timeout bug --- bbdev/bbdev | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bbdev/bbdev b/bbdev/bbdev index a4e7ba6b2..b1bcdea15 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -206,10 +206,16 @@ def parse_args(argv: list[str]) -> argparse.Namespace: group = cmd_parser.add_mutually_exclusive_group(required=True) for op_name, op_help in operations.items(): if op_help is True: - group.add_argument(f"--{op_name}", action="store_true", help=f"{op_name.capitalize()}.") + group.add_argument( + f"--{op_name}", + dest=op_name, + action="store_true", + help=f"{op_name.capitalize()}.", + ) else: group.add_argument( f"--{op_name}", + dest=op_name, type=str, nargs="?", const="", From c45d8fed790c0af68ab8327152a862e266f3348e Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 4 Sep 2026 05:07:53 +0800 Subject: [PATCH 138/196] refactor: streamline workload directory cleanup process by consolidating path handling and logging --- .../api/steps/workload/00_clean_event.step.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/bbdev/api/steps/workload/00_clean_event.step.py b/bbdev/api/steps/workload/00_clean_event.step.py index 29e1d6846..01d92ed53 100644 --- a/bbdev/api/steps/workload/00_clean_event.step.py +++ b/bbdev/api/steps/workload/00_clean_event.step.py @@ -50,18 +50,25 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return bbdir = get_buckyball_path() - path = chip_output_root(bbdir, chip) + output_path = chip_output_root(bbdir, chip) + build_path = os.path.join(bbdir, "bb-tests", "workloads", "build", chip) - if os.path.exists(path): - ctx.logger.info("Removing workload output directory", {"path": path}) - shutil.rmtree(path) - else: - ctx.logger.info("Workload output directory already clean", {"path": path}) + for path in (output_path, build_path): + if os.path.exists(path): + ctx.logger.info("Removing workload directory", {"path": path}) + shutil.rmtree(path) + else: + ctx.logger.info("Workload directory already clean", {"path": path}) await check_result( ctx, 0, continue_run=False, - extra_fields={"task": "clean", "chip": chip, "path": path}, + extra_fields={ + "task": "clean", + "chip": chip, + "output_path": output_path, + "build_path": build_path, + }, trace_id=origin_tid, ) From 444dce577682a1f85ddd6221ef7d5da28518772a Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 4 Sep 2026 10:21:32 +0800 Subject: [PATCH 139/196] refactor: enhance path handling in workload cleanup by introducing workload_build_dir function --- bbdev/api/steps/workload/00_clean_event.step.py | 4 ++-- bbdev/mcp/common.py | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/bbdev/api/steps/workload/00_clean_event.step.py b/bbdev/api/steps/workload/00_clean_event.step.py index 01d92ed53..a710be5dd 100644 --- a/bbdev/api/steps/workload/00_clean_event.step.py +++ b/bbdev/api/steps/workload/00_clean_event.step.py @@ -10,7 +10,7 @@ from utils.event_common import require_chip from utils.event_common import check_result, get_origin_trace_id -from utils.path import chip_output_root, get_buckyball_path +from utils.path import chip_output_root, get_buckyball_path, workload_build_dir config = { "name": "workload-clean", @@ -51,7 +51,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() output_path = chip_output_root(bbdir, chip) - build_path = os.path.join(bbdir, "bb-tests", "workloads", "build", chip) + build_path = workload_build_dir(bbdir, chip) for path in (output_path, build_path): if os.path.exists(path): diff --git a/bbdev/mcp/common.py b/bbdev/mcp/common.py index b7ec6ad09..d9938bd86 100644 --- a/bbdev/mcp/common.py +++ b/bbdev/mcp/common.py @@ -433,13 +433,6 @@ def _validate(path: Path) -> Dict[str, Any]: {"ballName": name, "config": cfg_rel, "resolved": str(cfg_path)} ) continue - ball_cfg = _load_toml(cfg_path) - ball = ball_cfg.get("ball") - if not isinstance(ball, dict): - missing_config.append( - {"ballName": name, "config": cfg_rel, "error": "missing [ball]"} - ) - funct7s = [e.get("funct7") for e in isa] mnemonics = [e.get("mnemonic") for e in isa] bids = [e.get("bid") for e in isa] From f1782ac80db68ef4382083be2e4a97958e659778 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 4 Sep 2026 17:12:38 +0800 Subject: [PATCH 140/196] chore: update .gitignore to exclude lc_output.txt and enhance tapeout.py and generate.py with bitwrite validation --- bbdev/.gitignore | 1 + bbdev/api/lc_output.txt | 29 ------------------- bbdev/api/steps/dc/scripts/tapeout.py | 14 ++++++--- bbdev/api/steps/ip/scripts/generate.py | 40 ++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 33 deletions(-) delete mode 100644 bbdev/api/lc_output.txt diff --git a/bbdev/.gitignore b/bbdev/.gitignore index d6f53a209..505616b23 100644 --- a/bbdev/.gitignore +++ b/bbdev/.gitignore @@ -24,3 +24,4 @@ api/iii-config-*.yaml api/data/ api/steps/ip/scripts/_smoke_out/ +lc_output.txt diff --git a/bbdev/api/lc_output.txt b/bbdev/api/lc_output.txt deleted file mode 100644 index 85f5db05d..000000000 --- a/bbdev/api/lc_output.txt +++ /dev/null @@ -1,29 +0,0 @@ - - Library Compiler (TM) - DesignWare (R) - - Version R-2020.09-SP5 for linux64 - Apr 14, 2021 - - Copyright (c) 1988 - 2021 Synopsys, Inc. - This software and the associated documentation are proprietary to Synopsys, - Inc. This software may only be used in accordance with the terms and conditions - of a written license agreement with Synopsys, Inc. All other use, reproduction, - or distribution of this software is strictly prohibited. - -Initializing... -read_lib {/home/wanghui/Code/buckyball/arch/build/pebble/sims.tapeout.BuckyballPebbleTapeoutConfig-ip-db/chipyard_sram_64x8/chipyard_sram_64x8_ss_1.62_125.lib} -Reading '/home/wanghui/Code/buckyball/arch/build/pebble/sims.tapeout.BuckyballPebbleTapeoutConfig-ip-db/chipyard_sram_64x8/chipyard_sram_64x8_ss_1.62_125.lib' ... -Warning: Line 50, The 'internal_power_calculation' attribute in char_config group is required for NLPM library. - No default can be applied to this attribute. (LBDB-366) -Information: Line 235, Cell 'chipyard_sram_64x8', The cell 'chipyard_sram_64x8' contiains memory/memory_read/memory_write groups which are deprecated, use the is_memory_cell attribute instead. (LIBG-280) -Technology library 'chipyard_sram_64x8_ss_1.62_125' read successfully -1 -write_lib [get_object_name [get_libs *]] -format db -output {/home/wanghui/Code/buckyball/arch/build/pebble/sims.tapeout.BuckyballPebbleTapeoutConfig-ip-db/chipyard_sram_64x8/chipyard_sram_64x8_ss_1.62_125.db} -Wrote the 'chipyard_sram_64x8_ss_1.62_125' library to '/home/wanghui/Code/buckyball/arch/build/pebble/sims.tapeout.BuckyballPebbleTapeoutConfig-ip-db/chipyard_sram_64x8/chipyard_sram_64x8_ss_1.62_125.db' successfully -1 -exit -Maximum memory usage for this session: 84.32 MB -CPU usage for this session: 1 seconds ( 0.00 hours) -Elapsed time for this session: 1 seconds ( 0.00 hours) -Thank you for using Library Compiler. - diff --git a/bbdev/api/steps/dc/scripts/tapeout.py b/bbdev/api/steps/dc/scripts/tapeout.py index 8bf1a4f03..6dbed01a2 100644 --- a/bbdev/api/steps/dc/scripts/tapeout.py +++ b/bbdev/api/steps/dc/scripts/tapeout.py @@ -24,6 +24,7 @@ class SramGeom: words: int mux: int bits: int + bitwrite: bool = False @dataclass(frozen=True) @@ -88,10 +89,15 @@ def get_tapeout_contract(bbdir: str | os.PathLike[str], chip: str, top: str | No with (root / "config.toml").open("rb") as f: s = tomllib.load(f)["tapeout"] - sram_table = { - row["name"]: SramGeom(row["name"], row["words"], row["mux"], row["bits"]) - for row in s["sram"] - } + sram_table = {} + for row in s["sram"]: + name = row["name"] + if name in sram_table: + raise ValueError(f"duplicate tapeout.sram name {name}") + bitwrite = row["bitwrite"] if "bitwrite" in row else False + if not isinstance(bitwrite, bool): + raise ValueError(f"tapeout.sram {name}: bitwrite must be bool, got {bitwrite!r}") + sram_table[name] = SramGeom(name, row["words"], row["mux"], row["bits"], bitwrite) pw = s.get("power_workload") return TapeoutContract( diff --git a/bbdev/api/steps/ip/scripts/generate.py b/bbdev/api/steps/ip/scripts/generate.py index 2bb845a50..6ed0ca1d8 100644 --- a/bbdev/api/steps/ip/scripts/generate.py +++ b/bbdev/api/steps/ip/scripts/generate.py @@ -161,6 +161,45 @@ def geoms_from_macros( return geoms +def check_bitwrite_contract(geoms: list[SramGeom], mdf: Path, macros_v: Path, sram_table: dict[str, SramGeom]) -> None: + raw = json.loads(mdf.read_text()) + if not isinstance(raw, list): + raise ValueError(f"mdf must be a list: {mdf}") + by_name: dict[str, dict] = {} + for entry in raw: + if not isinstance(entry, dict): + raise ValueError(f"mdf entry must be object: {mdf}") + name = entry.get("name") + if not isinstance(name, str) or not name: + raise ValueError(f"mdf entry missing name: {mdf}") + if name in by_name: + raise ValueError(f"duplicate mdf name {name}: {mdf}") + by_name[name] = entry + for g in geoms: + if g.name not in by_name: + raise ValueError(f"sram leaf {g.name} missing from mdf: {mdf}") + ports = by_name[g.name].get("ports") + if not isinstance(ports, list) or len(ports) != 1: + raise ValueError(f"{g.name}: expected exactly 1 mdf port") + p = ports[0] + if not isinstance(p, dict): + raise ValueError(f"{g.name}: mdf port must be object") + mask_name = p.get("mask port name") + if g.bitwrite: + if mask_name != "BWEN": + raise ValueError(f"{g.name}: bitwrite requires mask port BWEN, got {mask_name!r}") + if p.get("mask port polarity") != "active low": + raise ValueError(f"{g.name}: BWEN polarity must be active low") + if p.get("mask granularity") != 1: + raise ValueError(f"{g.name}: BWEN mask granularity must be 1") + elif mask_name is not None: + raise ValueError(f"{g.name}: mask port {mask_name!r} but bitwrite is false") + used = {g.name for g in geoms} + for geom in sram_table.values(): + if geom.bitwrite and geom.name not in used: + raise RuntimeError(f"bitwrite sram {geom.name} not instantiated in {macros_v}") + + def _leaf_paths(cache_dir: Path, name: str, corner: str) -> dict[str, str]: leaf = cache_dir / name return { @@ -213,6 +252,7 @@ def generate_sram( sram_table=contract.sram_table, mdf=contract.sram_mdf, ) + check_bitwrite_contract(geoms, contract.sram_mdf, verilog, contract.sram_table) ip_db = build_dir.parent / f"{build_dir.name}-ip-db" db_paths, corner_tag = generate_sram_dbs( geoms=geoms, From 0dc29ce7be22915fb23b5e625a1fc9619991f384 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 5 Sep 2026 16:13:02 +0800 Subject: [PATCH 141/196] refactor: enhance UVM and Verilator workflows by updating parameter handling and adding ELF validation --- bbdev/.gitignore | 1 + .../steps/bebop/bemu/scripts/bemu_analysis.py | 9 +- .../api/steps/uvm/00_ball_verilog_api.step.py | 39 -- .../steps/uvm/00_ball_verilog_event.step.py | 80 --- bbdev/api/steps/uvm/01_build_api.step.py | 12 +- bbdev/api/steps/uvm/01_build_event.step.py | 22 +- bbdev/api/steps/uvm/02_run_api.step.py | 16 +- bbdev/api/steps/uvm/02_run_event.step.py | 89 +-- bbdev/api/steps/uvm/scripts/uvm_common.py | 535 +++++++----------- .../api/steps/verilator/04_sim_event.step.py | 161 +++++- bbdev/api/steps/verilator/05_run_api.step.py | 7 + bbdev/api/steps/verilator/README.md | 17 +- bbdev/api/steps/verilator/dramsim3.json | 1 - bbdev/api/steps/verilator/dramsim3.txt | 78 --- bbdev/api/tests/test_api_verilator_run.py | 2 +- bbdev/api/tests/test_api_verilator_sim.py | 2 +- bbdev/api/tests/test_uvm_chip_pb.py | 53 ++ bbdev/api/utils/search_workload.py | 3 + bbdev/bbdev | 12 +- bbdev/mcp/tools/uvm_build.py | 16 +- bbdev/mcp/tools/uvm_run.py | 21 +- bbdev/mcp/tools/verilator_run.py | 4 +- bbdev/mcp/tools/verilator_sim.py | 4 +- 23 files changed, 488 insertions(+), 696 deletions(-) delete mode 100644 bbdev/api/steps/uvm/00_ball_verilog_api.step.py delete mode 100644 bbdev/api/steps/uvm/00_ball_verilog_event.step.py delete mode 100644 bbdev/api/steps/verilator/dramsim3.json delete mode 100644 bbdev/api/steps/verilator/dramsim3.txt create mode 100644 bbdev/api/tests/test_uvm_chip_pb.py diff --git a/bbdev/.gitignore b/bbdev/.gitignore index 505616b23..2574abf3b 100644 --- a/bbdev/.gitignore +++ b/bbdev/.gitignore @@ -13,6 +13,7 @@ appendonlydir/ dump.rdb *.log *.jou +dramsim3* output/ result/ diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py index c03da4789..a7d91e549 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py @@ -19,7 +19,7 @@ 35: "mvin_mmio", } -MATRIX_MNEMONICS = frozenset({"MATRIX", "MATRIX_F32"}) +MATRIX_MNEMONICS = frozenset({"MATRIX", "MATRIX_F32", "SMATMUL_OS"}) _API = Path(__file__).resolve().parents[4] if str(_API) not in sys.path: @@ -104,8 +104,8 @@ def chip_maps(bbdir: str, chip: str) -> tuple[dict[int, str], set[int], int]: return names, matrix, entries -def _mnk(rs2: int) -> tuple[int, int, int]: - return rs2 & 0xFFF, (rs2 >> 12) & 0xFFF, (rs2 >> 24) & 0xFFF +def _mnk(rs1: int, rs2: int) -> tuple[int, int, int]: + return rs2 & 0xFFF, (rs2 >> 12) & 0xFFF, rs1 >> 30 def analysis_dir( @@ -170,8 +170,9 @@ def analysis_dir( funct_cyc[name] += delta prev_clk = clk if funct in matrix: + rs1 = _hex_u(obj.get("rs1"), "rs1", line_no) rs2 = _hex_u(obj.get("rs2"), "rs2", line_no) - mnk_n[_mnk(rs2)] += 1 + mnk_n[_mnk(rs1, rs2)] += 1 elif kind == "mtrace": n_mtrace += 1 rows = obj.get("rows") diff --git a/bbdev/api/steps/uvm/00_ball_verilog_api.step.py b/bbdev/api/steps/uvm/00_ball_verilog_api.step.py deleted file mode 100644 index 76e2df8ca..000000000 --- a/bbdev/api/steps/uvm/00_ball_verilog_api.step.py +++ /dev/null @@ -1,39 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - -config = { - "name": "uvm-ball-verilog-api", - "description": "generate isolated ball verilog for UVM", - "flows": ["uvm"], - "triggers": [api("POST", "/uvm/verilog")], - "enqueues": ["uvm.verilog"], -} - - -async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = request.body or {} - - if body.get("chip") or ( - isinstance(body.get("config"), str) and body.get("config") and body.get("config") != "None" - ): - return ApiResponse( - status=400, - body={"error": "uvm ball verilog does not take --chip or --config"}, - ) - - balltype = body.get("balltype") - if not isinstance(balltype, str) or not balltype: - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --balltype"}, - ) - - output_dir = body.get("output_dir") - if not isinstance(output_dir, str) or not output_dir: - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --output-dir"}, - ) - - data = {"balltype": balltype, "output_dir": output_dir} - await ctx.enqueue({"topic": "uvm.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/uvm/00_ball_verilog_event.step.py b/bbdev/api/steps/uvm/00_ball_verilog_event.step.py deleted file mode 100644 index a93c56e17..000000000 --- a/bbdev/api/steps/uvm/00_ball_verilog_event.step.py +++ /dev/null @@ -1,80 +0,0 @@ -import os -import sys - -from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger_async -from utils.event_common import check_result, get_origin_trace_id - -config = { - "name": "uvm-ball-verilog", - "description": "generate isolated ball verilog for UVM", - "flows": ["uvm"], - "triggers": [queue("uvm.verilog")], - "enqueues": [], -} - - -def _balltype(data: dict) -> str: - balltype = data.get("balltype") - if not isinstance(balltype, str) or not balltype: - raise ValueError("Missing required parameter: --balltype") - return balltype - - -def _output_dir(data: dict) -> str: - output_dir = data.get("output_dir") - if not isinstance(output_dir, str) or not output_dir: - raise ValueError("Missing required parameter: --output-dir") - return output_dir - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - bbdir = get_buckyball_path() - - try: - balltype = _balltype(input_data) - build_dir = _output_dir(input_data) - except ValueError as error: - ctx.logger.error(str(error)) - _, failure_result = await check_result( - ctx, - 1, - continue_run=False, - extra_fields={ - "task": "validation", - "error": str(error), - "example": 'bbdev uvm --verilog "--balltype ReluBall --output-dir "', - }, - trace_id=origin_tid, - ) - return failure_result - - os.makedirs(build_dir, exist_ok=True) - command = ( - f"mill -i __.test.runMain sims.verify.BallTopMain {balltype} " - "--disable-annotation-unknown --strip-debug-info -O=release " - f"--split-verilog -o={build_dir} " - ) - ctx.logger.info(f"Using build directory: {build_dir}") - result = await stream_run_logger_async( - cmd=command, - logger=ctx.logger, - cwd=f"{bbdir}/arch", - stdout_prefix="uvm ball verilog", - stderr_prefix="uvm ball verilog", - ) - - await check_result( - ctx, - result.returncode, - continue_run=False, - extra_fields={"task": "verilog", "balltype": balltype, "output_dir": build_dir}, - trace_id=origin_tid, - ) diff --git a/bbdev/api/steps/uvm/01_build_api.step.py b/bbdev/api/steps/uvm/01_build_api.step.py index 9f9a87e0d..6217fe51e 100644 --- a/bbdev/api/steps/uvm/01_build_api.step.py +++ b/bbdev/api/steps/uvm/01_build_api.step.py @@ -10,20 +10,14 @@ def check_args(body: dict) -> str | None: - allowed = {"ball", "config", "core_config", "filelist"} + allowed = {"chip", "ball"} for key in body: if key not in allowed: return f"Unexpected parameter: --{key}" - config = body.get("config") - if not config or config is True: - return "Missing required parameter: --config=" - core_config = body.get("core_config") - if not core_config or core_config is True: - return "Missing required parameter: --core-config=" + if not body.get("chip") or body.get("chip") is True: + return "Missing required parameter: --chip" if body.get("ball") is True: return "Parameter --ball requires a value" - if body.get("filelist") is True: - return "Parameter --filelist requires a path value" return None diff --git a/bbdev/api/steps/uvm/01_build_event.step.py b/bbdev/api/steps/uvm/01_build_event.step.py index ac89ab5ff..0106e0a24 100644 --- a/bbdev/api/steps/uvm/01_build_event.step.py +++ b/bbdev/api/steps/uvm/01_build_event.step.py @@ -13,7 +13,7 @@ from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path -from scripts.uvm_common import run_uvm_build +from scripts.uvm_common import run_chip config = { "name": "uvm-build", @@ -28,11 +28,25 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - result, info = await asyncio.to_thread(run_uvm_build, bbdir, input_data, ctx) + try: + info = await asyncio.to_thread( + run_chip, bbdir, input_data["chip"], input_data.get("ball"), ctx, False + ) + except Exception as e: + ctx.logger.error(str(e)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": str(e)}, + trace_id=origin_tid, + ) + return + await check_result( ctx, - result.returncode, + 0, continue_run=False, - extra_fields=info, + extra_fields={"task": "build", **info}, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/uvm/02_run_api.step.py b/bbdev/api/steps/uvm/02_run_api.step.py index 17246805b..5cf65fd63 100644 --- a/bbdev/api/steps/uvm/02_run_api.step.py +++ b/bbdev/api/steps/uvm/02_run_api.step.py @@ -10,20 +10,14 @@ def check_args(body: dict) -> str | None: - allowed = {"ball", "filelist", "test", "plusargs"} + allowed = {"chip", "ball"} for key in body: if key not in allowed: return f"Unexpected parameter: --{key}" - ball = body.get("ball") - if not ball or ball is True: - return "Missing required parameter: --ball=" - if body.get("filelist") is True: - return "Parameter --filelist requires a path value" - if body.get("test") is True: - return "Parameter --test requires a value" - plusargs = body.get("plusargs") - if not isinstance(plusargs, str) or not plusargs: - return "Missing required parameter: --plusargs='+BID='" + if not body.get("chip") or body.get("chip") is True: + return "Missing required parameter: --chip" + if body.get("ball") is True: + return "Parameter --ball requires a value" return None diff --git a/bbdev/api/steps/uvm/02_run_event.step.py b/bbdev/api/steps/uvm/02_run_event.step.py index efc7ef1c3..e948a6d17 100644 --- a/bbdev/api/steps/uvm/02_run_event.step.py +++ b/bbdev/api/steps/uvm/02_run_event.step.py @@ -1,5 +1,5 @@ +import asyncio import os -import shlex import sys from motia import FlowContext, queue @@ -13,8 +13,7 @@ from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path -from utils.stream_run import stream_run_logger_async -from scripts.uvm_common import checked_run_paths, default_test_name +from scripts.uvm_common import run_chip config = { "name": "uvm-run", @@ -28,98 +27,26 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - info = {"task": "run", "ball": input_data.get("ball")} try: - paths = checked_run_paths(bbdir, input_data) - except Exception as e: - ctx.logger.error(str(e)) - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={**info, "error": str(e)}, - trace_id=origin_tid, - ) - return - - info.update({ - "ball": paths["ball"], - "verify_dir": paths["verify_dir"], - "simv": paths["simv"], - "dpi_lib": paths["dpi_lib"], - }) - - if not os.path.isfile(paths["simv"]): - error = f"simv not found: {paths['simv']}. Run `bbdev uvm --build \"--config \"` first." - ctx.logger.error(error) - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={**info, "error": error}, - trace_id=origin_tid, + info = await asyncio.to_thread( + run_chip, bbdir, input_data["chip"], input_data.get("ball"), ctx, True ) - return - - test = input_data.get("test") or default_test_name(paths["ball"]) - if test is True: - ctx.logger.error("Parameter --test requires a value") - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={**info, "error": "invalid_test"}, - trace_id=origin_tid, - ) - return - - plusargs = input_data.get("plusargs") - if not isinstance(plusargs, str) or not plusargs: - ctx.logger.error("Missing required parameter: --plusargs='+BID='") - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={**info, "error": "missing_plusargs"}, - trace_id=origin_tid, - ) - return - try: - plusarg_words = shlex.split(plusargs) - except ValueError as e: + except Exception as e: ctx.logger.error(str(e)) await check_result( ctx, 1, continue_run=False, - extra_fields={**info, "error": str(e)}, + extra_fields={"error": str(e)}, trace_id=origin_tid, ) return - script = ( - f"cd {shlex.quote(paths['verify_dir'])} && " - f"env LD_LIBRARY_PATH=\"$VCS_RUNTIME_LIBRARY_PATH\" {shlex.quote(paths['simv'])} " - f"-sv_lib {shlex.quote(paths['dpi_lib'])} " - f"+UVM_TESTNAME={shlex.quote(test)} " - + " ".join(shlex.quote(word) for word in plusarg_words) - ) - cmd = f"nix develop {shlex.quote(paths['verify_env'])} --command zsh -ic {shlex.quote(script)}" - - ctx.logger.info(f"Running UVM test {test} for ball {paths['ball']}") - run_result = await stream_run_logger_async( - cmd=cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="uvm run", - stderr_prefix="uvm run", - ) - await check_result( ctx, - run_result.returncode, + 0, continue_run=False, - extra_fields={**info, "test": test}, + extra_fields={"task": "run", **info}, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index 9db37cbb9..6e6069d8e 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -1,359 +1,206 @@ import os -import re import shlex import sys import tomllib +from datetime import datetime +from pathlib import Path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) - -from utils.stream_run import StreamResult, stream_run_logger - - -def default_test_name(ball: str) -> str: - if ball == "fp2int": - return "fp2int_scale_rows_test" - if ball == "int2fp": - return "int2fp_channel_two_rows_test" - return f"{ball}_ball_test" - - -def config_dir_name(config: str) -> str: - if not isinstance(config, str) or not re.fullmatch(r"[A-Za-z0-9_.-]+", config): - raise ValueError("invalid config name") - return config - - -def core_ball_isa_defines(bbdir: str, input_data: dict, ball: str) -> list[str]: - core_config = input_data.get("core_config") - if not isinstance(core_config, str) or not core_config: - raise ValueError("Missing required parameter: --core-config=") - path = core_config if os.path.isabs(core_config) else os.path.join(bbdir, core_config) - path = os.path.abspath(path) - if not os.path.isfile(path): - raise FileNotFoundError(f"core_config not found: {path}") - with open(path, "rb") as src: - core = tomllib.load(src) - rel_domain = core.get("balldomain") - if not isinstance(rel_domain, str) or not rel_domain: - raise ValueError(f"core_config missing balldomain: {path}") - domain_path = os.path.abspath(os.path.join(os.path.dirname(path), rel_domain)) - if not os.path.isfile(domain_path): - raise FileNotFoundError(f"balldomain not found: {domain_path}") - with open(domain_path, "rb") as src: - domain = tomllib.load(src) - entries = domain.get("ballISA") - if not isinstance(entries, list): - raise ValueError(f"ballISA missing from {domain_path}") - required = { - "fp2int": {"FP2INT": "FP2INT_FUNCT7"}, - "int2fp": { - "INT2FP_TENSOR": "INT2FP_TENSOR_FUNCT7", - "INT2FP_CHANNEL": "INT2FP_CHANNEL_FUNCT7", - }, - "smatmul": { - "SMATMUL_OS": "SMATMUL_OS_FUNCT7", - "SMATMUL_BIAS": "SMATMUL_BIAS_FUNCT7", - }, - "matadd": {"MATADD": "MATADD_FUNCT7"}, - "im2col": {"IM2COL": "IM2COL_FUNCT7"}, - "transpose": {"TRANSPOSE": "TRANSPOSE_FUNCT7"}, - }.get(ball) - if required is None: - return [] - values = {} - for entry in entries: - if not isinstance(entry, dict): - raise ValueError(f"invalid ballISA entry in {domain_path}") - mnemonic = entry.get("mnemonic") - funct7 = entry.get("funct7") - if mnemonic in required: - if not isinstance(funct7, int) or funct7 < 0 or funct7 > 127: - raise ValueError(f"invalid funct7 for {mnemonic} in {domain_path}") - if mnemonic in values: - raise ValueError(f"duplicate {mnemonic} in {domain_path}") - values[mnemonic] = funct7 - if set(values) != set(required): - raise ValueError(f"{domain_path} missing required {ball} ISA entries") - defines = [f"+define+{macro}={values[mnemonic]}" for mnemonic, macro in required.items()] - if ball == "smatmul": - mappings = domain.get("ballIdMappings") - if not isinstance(mappings, list): - raise ValueError(f"ballIdMappings missing from {domain_path}") - matching = [entry for entry in mappings - if isinstance(entry, dict) and entry.get("ballName") == "SMatMulBall"] - if len(matching) != 1: - raise ValueError(f"{domain_path} must define exactly one SMatMulBall mapping") - in_bw = matching[0].get("inBW") - out_bw = matching[0].get("outBW") - if in_bw != 2 or out_bw != 1: - raise ValueError(f"SMatMulBall must use inBW=2,outBW=1 in {domain_path}") - defines.append("+define+SMATMUL_OUT_BW=1") - return defines - - -def uvm_paths(bbdir: str, input_data: dict, ball_override: str | None = None) -> dict: - ball = ball_override or input_data.get("ball") - if not isinstance(ball, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", ball): - raise ValueError("invalid ball name") - - config = input_data.get("config") - config_dir = config_dir_name(config) if config else None - ball_dir = os.path.join(bbdir, "examples", "balls", ball) - verify_dir = os.path.join(ball_dir, "verify") - casegen_dir = os.path.join(verify_dir, "casegen") - verify_env = os.path.join(bbdir, "verify") - build_dir = os.path.join(verify_dir, "build") - sim_dir = os.path.join(build_dir, config_dir) if config_dir else os.path.join(build_dir, "current") - simv = os.path.join(sim_dir, "simv") - csrc_dir = os.path.join(sim_dir, "csrc") - core_config = input_data.get("core_config") - core_dir = None - if config_dir: - if not isinstance(core_config, str) or not core_config: - raise ValueError("Missing required parameter: --core-config=") - core_path = core_config if os.path.isabs(core_config) else os.path.join(bbdir, core_config) - core_dir = os.path.basename(os.path.dirname(os.path.dirname(os.path.abspath(core_path)))) - rtl_dir = os.path.join(bbdir, "arch", "build", core_dir, config_dir) if config_dir else None - - return { - "ball": ball, - "config": config_dir, - "ball_dir": ball_dir, - "verify_dir": verify_dir, - "casegen_dir": casegen_dir, - "verify_env": verify_env, - "build_dir": build_dir, - "sim_dir": sim_dir, - "simv": simv, - "csrc_dir": csrc_dir, - "rtl_dir": rtl_dir, - } - - -def checked_paths(bbdir: str, input_data: dict, ball_override: str | None = None) -> dict: - paths = uvm_paths(bbdir, input_data, ball_override) - for key in ("ball_dir", "verify_dir", "casegen_dir", "verify_env"): - if not os.path.isdir(paths[key]): - raise FileNotFoundError(f"{key} not found: {paths[key]}") - if paths["rtl_dir"] and not os.path.isdir(paths["rtl_dir"]): - raise FileNotFoundError(f"rtl_dir not found: {paths['rtl_dir']}") - - cargo_toml = os.path.join(paths["casegen_dir"], "Cargo.toml") - if not os.path.isfile(cargo_toml): - raise FileNotFoundError(f"Cargo.toml not found: {cargo_toml}") - - filelist = resolve_filelist(paths["verify_dir"], input_data.get("filelist"), paths["ball"]) - crate = read_crate_name(cargo_toml) - dpi_lib = os.path.join(paths["casegen_dir"], "target", "debug", f"lib{crate.replace('-', '_')}") - - paths.update({ - "cargo_toml": cargo_toml, - "filelist": filelist, - "filelist_arg": prepare_filelist(paths, filelist), - "crate": crate, - "dpi_lib": dpi_lib, - }) - return paths - - -def checked_run_paths(bbdir: str, input_data: dict) -> dict: - paths = uvm_paths(bbdir, input_data) - for key in ("ball_dir", "verify_dir", "casegen_dir", "verify_env"): - if not os.path.isdir(paths[key]): - raise FileNotFoundError(f"{key} not found: {paths[key]}") - - cargo_toml = os.path.join(paths["casegen_dir"], "Cargo.toml") - if not os.path.isfile(cargo_toml): - raise FileNotFoundError(f"Cargo.toml not found: {cargo_toml}") - - crate = read_crate_name(cargo_toml) - dpi_lib = os.path.join(paths["casegen_dir"], "target", "debug", f"lib{crate.replace('-', '_')}") - paths.update({ - "cargo_toml": cargo_toml, - "crate": crate, - "dpi_lib": dpi_lib, - }) - return paths - - -def discover_uvm_balls(bbdir: str) -> list[str]: - balls_dir = os.path.join(bbdir, "examples", "balls") - if not os.path.isdir(balls_dir): - return [] - balls = [] - for name in sorted(os.listdir(balls_dir)): - if not re.fullmatch(r"[A-Za-z0-9_-]+", name): - continue - verify_dir = os.path.join(balls_dir, name, "verify") - filelist_dir = os.path.join(verify_dir, "filelists") - casegen_toml = os.path.join(verify_dir, "casegen", "Cargo.toml") - if os.path.isdir(filelist_dir) and os.path.isfile(casegen_toml): - balls.append(name) - return balls - - -def resolve_filelist(verify_dir: str, arg, ball: str) -> str: - if arg is True: - raise ValueError("parameter --filelist requires a path value") - - if arg: - path = arg if os.path.isabs(arg) else os.path.join(verify_dir, arg) - if not os.path.isfile(path): - raise FileNotFoundError(f"filelist not found: {path}") - return os.path.abspath(path) - - default = os.path.join(verify_dir, "filelists", f"{ball}_ball_toy.f") - if os.path.isfile(default): - return default - - filelist_dir = os.path.join(verify_dir, "filelists") - if not os.path.isdir(filelist_dir): - raise FileNotFoundError(f"filelist directory not found: {filelist_dir}") - found = [ - os.path.join(filelist_dir, name) - for name in os.listdir(filelist_dir) - if name.endswith(".f") +config_scripts = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "config", "scripts")) +if config_scripts not in sys.path: + sys.path.insert(0, config_scripts) + +import chip_pb2 +from utils.path import get_buckyball_path, log_dir +from utils.stream_run import stream_run_logger + + +def load_chip(bbdir: str, chip: str): + path = Path(bbdir) / "examples" / "chips" / chip / "configs" / "generated" / "chip.pb" + msg = chip_pb2.Chip() + msg.ParseFromString(path.read_bytes()) + if not msg.name or not msg.cores: + raise ValueError(f"empty chip.pb: {path}") + return msg + + +def ball_domain(chip): + d0 = chip.cores[0].balldomain + key = [(m.ball_id, m.ball_dir, m.in_bw, m.out_bw) for m in d0.mappings] + isa = [(e.mnemonic, e.funct7, e.bid) for e in d0.isa] + for core in chip.cores[1:]: + k = [(m.ball_id, m.ball_dir, m.in_bw, m.out_bw) for m in core.balldomain.mappings] + i = [(e.mnemonic, e.funct7, e.bid) for e in core.balldomain.isa] + if k != key or i != isa: + raise ValueError("chip.pb cores have different balldomains") + return d0 + + +def selected_mappings(domain, ball: str | None): + if ball is None: + return list(domain.mappings) + for m in domain.mappings: + if m.ball_dir == ball: + return [m] + raise ValueError(f"ball {ball!r} not in chip.pb") + + +def vcs_defines(domain, mapping): + defs = [ + f"+define+BB_IN_BW={mapping.in_bw}", + f"+define+BB_OUT_BW={mapping.out_bw}", + f"+define+BB_MMIO_READ_BW={mapping.mmio_read_bw}", + f"+define+BB_MMIO_WRITE_BW={mapping.mmio_write_bw}", ] - if len(found) != 1: - raise RuntimeError(f"expected exactly one filelist under {filelist_dir}, found {len(found)}") - return found[0] - - -def filelist_arg(verify_dir: str, filelist: str) -> str: - rel = os.path.relpath(filelist, verify_dir) - if rel.startswith(".."): - return filelist - return rel - - -def prepare_filelist(paths: dict, filelist: str) -> str: - os.makedirs(paths["sim_dir"], exist_ok=True) - generated = os.path.join(paths["sim_dir"], os.path.basename(filelist)) - rel_rtl_dir = os.path.relpath(paths["rtl_dir"], paths["verify_dir"]) if paths["rtl_dir"] else None - rel_uvm_dir = os.path.relpath(os.path.join(paths["verify_env"], "uvm"), paths["verify_dir"]) - with open(filelist, "r") as src, open(generated, "w") as dst: - for line in src: - original = line - line = line.replace("@UVM@", rel_uvm_dir) - if "@RTL@" in line: - if not rel_rtl_dir: - raise ValueError("filelist uses @RTL@ but --config was not provided") - line = line.replace("@RTL@", rel_rtl_dir) - elif re.search(r"(?:\.\./)+arch/build/[^/\s]+/", original): - raise ValueError("filelist must use @RTL@ instead of hard-coded arch/build/") - if re.search(r"(?:\.\./)+verify/uvm/", original): - raise ValueError("filelist must use @UVM@ instead of hard-coded verify/uvm") - dst.write(line) - return filelist_arg(paths["verify_dir"], generated) - - -def read_crate_name(cargo_toml: str) -> str: - with open(cargo_toml, "rb") as f: - data = tomllib.load(f) - name = data.get("package", {}).get("name") - if not name: - raise ValueError(f"package.name not found in {cargo_toml}") - return name - - -def failed_result(message: str) -> StreamResult: - return StreamResult(returncode=1, stdout="", stderr=message) - - -def run_uvm_build_one(bbdir: str, input_data: dict, ctx, ball: str) -> tuple[StreamResult, dict]: - try: - paths = checked_paths(bbdir, input_data, ball) - isa_defines = core_ball_isa_defines(bbdir, input_data, ball) - except Exception as e: - ctx.logger.error(str(e)) - return failed_result(str(e)), {"task": "build", "error": str(e)} - - info = { - "task": "build", - "ball": paths["ball"], - "config": paths["config"], - "verify_dir": paths["verify_dir"], - "filelist": paths["filelist"], - "resolved_filelist": os.path.join(paths["verify_dir"], paths["filelist_arg"]), - "simv": paths["simv"], - "dpi_lib": paths["dpi_lib"], - "core_config": input_data.get("core_config"), - } - - cargo_cmd = ( - f"nix develop {shlex.quote(paths['verify_env'])} --command " - f"cargo build --manifest-path {shlex.quote(paths['cargo_toml'])}" - ) - ctx.logger.info(f"Building UVM DPI reference for ball {paths['ball']}") - cargo_result = stream_run_logger( - cmd=cargo_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="uvm dpi build", - stderr_prefix="uvm dpi build", + for e in domain.isa: + if e.bid == mapping.ball_id: + defs.append(f"+define+{e.mnemonic}_FUNCT7={e.funct7}") + return defs + + +def _filelist(verify_dir: Path, ball_dir: str, uvm_rel: str, rtl_rel: str, sim_dir: Path) -> str: + src = verify_dir / "filelists" / f"{ball_dir}_ball.f" + dst = sim_dir / f"{ball_dir}_ball.f" + sim_dir.mkdir(parents=True, exist_ok=True) + text = src.read_text() + dst.write_text(text.replace("@UVM@", uvm_rel).replace("@RTL@", rtl_rel)) + return str(dst.relative_to(verify_dir)) + + +def build_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx) -> None: + ball = mapping.ball_dir + verify_dir = Path(bbdir) / "examples" / "balls" / ball / "verify" + casegen = verify_dir / "casegen" / "Cargo.toml" + rtl_dir = Path(bbdir) / "arch" / "build" / chip_name / mill_cfg + sim_dir = verify_dir / "build" / chip_name + uvm_rel = os.path.relpath(Path(bbdir) / "verify" / "uvm", verify_dir) + rtl_rel = os.path.relpath(rtl_dir, verify_dir) + flist = _filelist(verify_dir, ball, uvm_rel, rtl_rel, sim_dir) + cargo = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"cargo build --manifest-path {shlex.quote(str(casegen))}" ) - if cargo_result.returncode != 0: - return cargo_result, info - + r = stream_run_logger(cmd=cargo, logger=ctx.logger, cwd=bbdir, stdout_prefix="uvm dpi", stderr_prefix="uvm dpi") + if r.returncode != 0: + raise RuntimeError(f"cargo build failed for {ball}") + crate = tomllib.loads(casegen.read_text()).get("package", {}).get("name") + if not crate: + raise ValueError(f"package.name missing in {casegen}") + simv = sim_dir / "simv" + csrc = sim_dir / "csrc" + hier = sim_dir / "cm_hier.cfg" + sim_dir.mkdir(parents=True, exist_ok=True) + hier.write_text("+tree tb_top.dut\n") script = ( - f"cd {shlex.quote(paths['verify_dir'])} && " - f"rm -rf {shlex.quote(paths['csrc_dir'])} {shlex.quote(paths['simv'])} {shlex.quote(paths['simv'])}.daidir && " - f"mkdir -p {shlex.quote(paths['sim_dir'])} {shlex.quote(paths['csrc_dir'])} && " + f"cd {shlex.quote(str(verify_dir))} && " + f"rm -rf {shlex.quote(str(csrc))} {shlex.quote(str(simv))} {shlex.quote(str(simv))}.daidir && " + f"mkdir -p {shlex.quote(str(sim_dir))} {shlex.quote(str(csrc))} && " "vcs -full64 -sverilog -timescale=1ns/1ps -debug_access+all " "${=VCS_UVM_ARGS} " - + " ".join(shlex.quote(value) for value in isa_defines) - + " " - f"-Mdir={shlex.quote(paths['csrc_dir'])} " - f"-o {shlex.quote(paths['simv'])} " - f"-f {shlex.quote(paths['filelist_arg'])}" + + " ".join(shlex.quote(d) for d in vcs_defines(domain, mapping)) + + f" -cm line+cond+tgl+assert -cm_hier {shlex.quote(str(hier))} " + f"-Mdir={shlex.quote(str(csrc))} -o {shlex.quote(str(simv))} " + f"-f {shlex.quote(flist)}" ) - vcs_cmd = f"nix develop {shlex.quote(paths['verify_env'])} --command zsh -c {shlex.quote(script)}" - - ctx.logger.info(f"Building UVM simulation for ball {paths['ball']} config {paths['config']}") - vcs_result = stream_run_logger( - cmd=vcs_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="uvm vcs build", - stderr_prefix="uvm vcs build", + vcs = f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command zsh -c {shlex.quote(script)}" + r = stream_run_logger(cmd=vcs, logger=ctx.logger, cwd=bbdir, stdout_prefix="uvm vcs", stderr_prefix="uvm vcs") + if r.returncode != 0: + raise RuntimeError(f"vcs failed for {ball}") + + +def run_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx, cov_dir: str) -> None: + ball = mapping.ball_dir + verify_dir = Path(bbdir) / "examples" / "balls" / ball / "verify" + simv = verify_dir / "build" / chip_name / "simv" + if not simv.is_file(): + build_ball(bbdir, chip_name, mill_cfg, domain, mapping, ctx) + crate = tomllib.loads((verify_dir / "casegen" / "Cargo.toml").read_text())["package"]["name"] + dpi = verify_dir / "casegen" / "target" / "debug" / f"lib{crate.replace('-', '_')}" + test = f"{ball}_ball_test" + script = ( + f"cd {shlex.quote(str(verify_dir))} && " + f"env LD_LIBRARY_PATH=\"$VCS_RUNTIME_LIBRARY_PATH\" {shlex.quote(str(simv))} " + f"-sv_lib {shlex.quote(str(dpi))} " + f"+UVM_TESTNAME={shlex.quote(test)} +BID={mapping.ball_id} " + f"-cm line+cond+tgl+assert -cm_name {shlex.quote(test)}" ) - if vcs_result.returncode == 0: - current = os.path.join(paths["build_dir"], "current") - os.makedirs(paths["build_dir"], exist_ok=True) - tmp = f"{current}.tmp" - try: - os.unlink(tmp) - except FileNotFoundError: - pass - os.symlink(paths["config"], tmp) - os.replace(tmp, current) - return vcs_result, info - - -def run_uvm_build(bbdir: str, input_data: dict, ctx) -> tuple[StreamResult, dict]: - config = input_data.get("config") - if not config or config is True: - return failed_result("Missing required parameter: --config="), { - "task": "build", - "error": "missing_config", - } - - ball = input_data.get("ball") - balls = [ball] if ball and ball is not True else discover_uvm_balls(bbdir) - if not balls: - return failed_result("No UVM balls found"), {"task": "build", "error": "no_uvm_balls"} - - built = [] - for one_ball in balls: - result, info = run_uvm_build_one(bbdir, input_data, ctx, one_ball) - if result.returncode != 0: - return result, {**info, "built": built} - built.append(one_ball) - - return StreamResult(returncode=0, stdout="", stderr=""), { - "task": "build", - "config": config, - "built": built, - } + cmd = f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command zsh -ic {shlex.quote(script)}" + r = stream_run_logger(cmd=cmd, logger=ctx.logger, cwd=bbdir, stdout_prefix="uvm run", stderr_prefix="uvm run") + if r.returncode != 0: + raise RuntimeError(f"uvm run failed for {ball} test={test}") + Path(cov_dir).mkdir(parents=True, exist_ok=True) + urg = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"urg -dir {shlex.quote(str(simv))}.vdb -format text -report {shlex.quote(cov_dir)}" + ) + r = stream_run_logger(cmd=urg, logger=ctx.logger, cwd=str(verify_dir), stdout_prefix="uvm urg", stderr_prefix="uvm urg") + if r.returncode != 0: + raise RuntimeError(f"urg failed for {ball}") + + +def dashboard_summary(cov_dir: str) -> dict[str, str]: + path = Path(cov_dir) / "dashboard.txt" + lines = path.read_text().splitlines() + i = 0 + while i < len(lines) and "Total Coverage Summary" not in lines[i]: + i += 1 + if i >= len(lines): + raise RuntimeError(f"Total Coverage Summary missing in {path}") + i += 1 + while i < len(lines) and not lines[i].strip().startswith("SCORE"): + i += 1 + if i + 1 >= len(lines): + raise RuntimeError(f"SCORE row missing in {path}") + keys = lines[i].split() + vals = lines[i + 1].split() + if len(vals) != len(keys): + raise RuntimeError(f"SCORE/value mismatch in {path}: {keys!r} vs {vals!r}") + return dict(zip(keys, vals)) + + +def run_chip(bbdir: str, chip: str, ball: str | None, ctx, do_run: bool) -> dict: + msg = load_chip(bbdir, chip) + domain = ball_domain(msg) + mill_cfg = msg.mill.verilator_config + maps = selected_mappings(domain, ball) + ran = [] + covs = [] + run_root = None + if do_run: + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + run_name = "all" if ball is None else maps[0].ball_dir + run_root = log_dir(bbdir, chip, "verilog", stamp, "uvm", run_name) + for m in maps: + if do_run: + cov = ( + os.path.join(run_root, m.ball_dir, "coverage") + if ball is None + else os.path.join(run_root, "coverage") + ) + run_ball(bbdir, chip, mill_cfg, domain, m, ctx, cov) + if not (Path(cov) / "dashboard.txt").is_file(): + raise RuntimeError(f"urg wrote no dashboard.txt under {cov}") + covs.append((m.ball_dir, cov)) + else: + build_ball(bbdir, chip, mill_cfg, domain, m, ctx) + ran.append(m.ball_dir) + info = {"chip": chip, "mill": mill_cfg, "balls": ran} + if do_run and ball is None: + index_dir = Path(run_root) / "coverage" + index_dir.mkdir(parents=True, exist_ok=True) + body = ["ball_dir score line cond toggle assert group result"] + for b, p in covs: + s = dashboard_summary(p) + body.append( + f"{b} {s['SCORE']} {s['LINE']} {s['COND']} {s['TOGGLE']} " + f"{s.get('ASSERT', '--')} {s['GROUP']} pass" + ) + index = index_dir / "index.txt" + index.write_text("\n".join(body) + "\n") + info["index"] = str(index) + info["log"] = run_root + elif do_run: + info["log"] = run_root + return info diff --git a/bbdev/api/steps/verilator/04_sim_event.step.py b/bbdev/api/steps/verilator/04_sim_event.step.py index 961cb9080..5bac2a837 100644 --- a/bbdev/api/steps/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -1,4 +1,6 @@ import os +import shlex +import struct import subprocess import sys from datetime import datetime @@ -25,12 +27,77 @@ } +DRAM_BASE = 0x80000000 +DRAM_SIZE = 0x10000000 + + +def validate_bbsim_elf(path: str) -> tuple[bool, str]: + """Validate the ELF layout accepted by BBSimDRAM before starting Verilator.""" + try: + with open(path, "rb") as binary: + header = binary.read(64) + if len(header) != 64: + return False, "not a complete ELF64 header" + if header[:4] != b"\x7fELF" or header[4] != 2 or header[5] != 1: + return False, "requires a little-endian ELF64 binary" + + (_, _, machine, _, _, phoff, _, _, ehsize, phentsize, phnum, _, _, _) = ( + struct.unpack("<16sHHIQQQIHHHHHH", header) + ) + if machine != 243: + return False, "requires a RISC-V ELF" + if ehsize != 64 or phentsize != 56 or phnum == 0: + return False, "has an unsupported ELF program-header layout" + + file_size = os.fstat(binary.fileno()).st_size + if phoff + phnum * phentsize > file_size: + return False, "program headers extend past the end of the file" + + load_segments = 0 + for index in range(phnum): + binary.seek(phoff + index * phentsize) + program_header = binary.read(phentsize) + if len(program_header) != phentsize: + return False, "cannot read ELF program headers" + p_type, _, p_offset, _, p_paddr, p_filesz, p_memsz, _ = struct.unpack( + " p_memsz or p_offset + p_filesz > file_size: + return False, "has an invalid loadable segment" + if p_paddr < DRAM_BASE or p_paddr + p_memsz > DRAM_BASE + DRAM_SIZE: + return ( + False, + f"load segment {index} at 0x{p_paddr:x} is outside " + f"BBSimDRAM [0x{DRAM_BASE:x}, 0x{DRAM_BASE + DRAM_SIZE:x})", + ) + except (OSError, struct.error): + return False, "cannot read a valid ELF64 program-header table" + + if not load_segments: + return False, "has no loadable ELF segments" + return True, "" + + async def handler(input_data: dict, ctx: FlowContext) -> None: # ================================================================================== # Get simulation parameters # ================================================================================== origin_tid = get_origin_trace_id(input_data, ctx) - chip = require_chip(input_data) + try: + chip = require_chip(input_data) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_chip"}, + trace_id=origin_tid, + ) + return bbdir = get_buckyball_path() build_dir = rtl_dir( bbdir, chip, "verilog", input_data.get("output_dir"), @@ -41,19 +108,58 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: binary_name = input_data.get("binary", "") if not binary_name: - raise ValueError("binary required") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "binary_required"}, + trace_id=origin_tid, + ) + return coverage = input_data.get("coverage", False) workload_root = workload_tests_root(bbdir, chip) binary_path = search_workload(workload_root, binary_name) ctx.logger.info(f"binary_path: {binary_path}") + if binary_path is None: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "binary_not_found", + "binary": binary_name, + "search_dir": workload_root, + }, + trace_id=origin_tid, + ) + return + valid_elf, elf_error = validate_bbsim_elf(binary_path) + if not valid_elf: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "unsupported_elf", + "binary": binary_path, + "detail": ( + "direct Verilator requires a baremetal RISC-V ELF in BBSimDRAM; " + f"{elf_error}. Use bebop-verilator for Linux ELFs" + ), + }, + trace_id=origin_tid, + ) + return topname = "BBSimHarness" run_log = log_dir(bbdir, chip, "verilog", timestamp, "verilator", binary_name, input_data.get("output_dir")) waveform_dir = os.path.join(run_log, "waveform") os.makedirs(run_log, exist_ok=True) - os.makedirs(waveform_dir, exist_ok=True) + no_wave = bool(input_data.get("no-wave", input_data.get("no_wave", False))) + if not no_wave: + os.makedirs(waveform_dir, exist_ok=True) coverage_flag = "" if coverage: @@ -61,8 +167,24 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: coverage_flag = f"+verilator+coverage+file+{coverage_dat_path}" bin_path = f"{build_dir}/obj_dir/V{topname}" + if not os.path.isfile(bin_path): + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "simulator_not_found", "simulator": bin_path}, + trace_id=origin_tid, + ) + return batch = input_data.get("batch", False) + trace_names = [ + name + for name in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace") + if input_data.get(name, False) + ] + trace_arg = ",".join(trace_names) if trace_names else "none" + log_path = f"{run_log}/bdb.ndjson" stdout_path = f"{run_log}/stdout.log" meta_path = f"{run_log}/sim_meta.txt" @@ -89,15 +211,30 @@ def _gcc_lib_dir(soname: str) -> str: lib_dirs = [result_lib, _gcc_lib_dir("liblz4.so"), _gcc_lib_dir("libstdc++.so")] ld_lib_path = ":".join(dict.fromkeys(lib_dirs)) ctx.logger.info(f"LD_LIBRARY_PATH prefix: {ld_lib_path}") + sim_args = [ + bin_path, + "+permissive", + f"+elf={binary_path}", + ] + if batch: + sim_args.append("+batch") + if coverage_flag: + sim_args.append(coverage_flag) + if no_wave: + sim_args.append("+no-wave") + else: + sim_args.append(f"+fst={fst_path}") + sim_args.extend([ + f"+log={log_path}", + f"+stdout={stdout_path}", + f"+trace={trace_arg}", + "+permissive-off", + ]) sim_cmd = ( - f"export LD_LIBRARY_PATH=\"{ld_lib_path}:$LD_LIBRARY_PATH\"; " - f"export BDB_SIM_META=\"{meta_path}\"; " - f"{bin_path} +permissive " - f"+elf={binary_path} " - f"{'+batch ' if batch else ''}" - f"{coverage_flag + ' ' if coverage_flag else ''}" - f"+fst={fst_path} +log={log_path} +stdout={stdout_path} +trace=all +permissive-off " - f"{binary_path} 2> >(spike-dasm > {log_dir}/disasm.log)" + f"export LD_LIBRARY_PATH={shlex.quote(ld_lib_path)}:$LD_LIBRARY_PATH; " + f"export BDB_SIM_META={shlex.quote(meta_path)}; " + f"{shlex.join(sim_args)} " + f"2> >(spike-dasm > {shlex.quote(os.path.join(run_log, 'disasm.log'))})" ) script_dir = os.path.dirname(__file__) @@ -123,7 +260,7 @@ def _gcc_lib_dir(soname: str) -> str: "task": "sim", "binary": binary_path, "log_dir": run_log, - "waveform_dir": waveform_dir, + "waveform_dir": None if no_wave else waveform_dir, "timestamp": timestamp, "sim_meta": meta_path, } diff --git a/bbdev/api/steps/verilator/05_run_api.step.py b/bbdev/api/steps/verilator/05_run_api.step.py index c04c198fb..030fc223e 100644 --- a/bbdev/api/steps/verilator/05_run_api.step.py +++ b/bbdev/api/steps/verilator/05_run_api.step.py @@ -57,6 +57,13 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "binary": binary, "jobs": body.get("jobs", "16"), "batch": body.get("batch", False), + "coverage": body.get("coverage", False), + "no-wave": body.get("no-wave", body.get("no_wave", False)), + "itrace": body.get("itrace", False), + "mtrace": body.get("mtrace", False), + "pmctrace": body.get("pmctrace", False), + "ctrace": body.get("ctrace", False), + "banktrace": body.get("banktrace", False), "from_run_workflow": True, } diff --git a/bbdev/api/steps/verilator/README.md b/bbdev/api/steps/verilator/README.md index 5ca961b79..8369f21c6 100644 --- a/bbdev/api/steps/verilator/README.md +++ b/bbdev/api/steps/verilator/README.md @@ -15,14 +15,21 @@ Hardware simulation workflow based on Verilator in the Buckyball framework, prov - Default value: `16` - **`binary`** [Required] - Test binary file path - Default value: `""` +- **`batch`** - Run without the interactive monitor +- **`no-wave`** - Disable FST waveform generation +- **`itrace`**, **`mtrace`**, **`pmctrace`**, **`ctrace`**, **`banktrace`** - Enable selected BDB traces + +Direct Verilator uses `BBSimHarness` and therefore requires a baremetal RISC-V +ELF linked into the simulated `0x80000000` DRAM window. Linux workload ELFs +must be run with `bebop-verilator` instead. **Example**: ```bash # bbdev wrapper -bbdev verilator --run "jobs 256 --binary ${buckyball}/bb-tests/output/toy/workloads/src/ctest_mvin_mvout_alternate_test-baremetal --batch" +bbdev verilator --run "--chip toy --jobs 16 --binary toy-toy-ctest-mvin_mvout_test-baremetal --batch" # Raw command -curl -X POST http://localhost:5000/verilator/run -H "Content-Type: application/json" -d '{"jobs": 8, "binary": "/home/user/test.elf"}' +curl -X POST http://localhost:5000/verilator/run -H "Content-Type: application/json" -d '{"chip":"toy", "jobs":8, "binary":"toy-toy-ctest-mvin_mvout_test-baremetal", "batch":true}' ``` @@ -77,6 +84,12 @@ curl -X POST http://localhost:5000/verilator/build -d '{"jobs": 16}' **Parameters**: - **`binary`** [Required] - Custom test binary file path +- **`batch`** - Run without the interactive monitor +- **`no-wave`** - Disable FST waveform generation +- **`itrace`**, **`mtrace`**, **`pmctrace`**, **`ctrace`**, **`banktrace`** - Enable selected BDB traces + +The binary may be a workload filename or an existing absolute path. It must be +a baremetal ELF; Linux workload ELFs require `bebop-verilator`. **Example**: ```bash diff --git a/bbdev/api/steps/verilator/dramsim3.json b/bbdev/api/steps/verilator/dramsim3.json deleted file mode 100644 index 01aba8e2d..000000000 --- a/bbdev/api/steps/verilator/dramsim3.json +++ /dev/null @@ -1 +0,0 @@ -{"0":{"act_energy":43200.0,"act_stb_energy":{"0":3013588.8000000003,"1":1462579.2},"all_bank_idle_cycles":{"0":33121,"1":39365},"average_bandwidth":0.13577000419861668,"average_interarrival":313.7708333333333,"average_power":440.3248756988487,"average_read_latency":17.416666666666668,"channel":0,"epoch_num":0,"hbm_dual_cmds":0,"interarrival_latency":{"1":2,"103":1,"106":1,"107":1,"113":1,"117":1,"12":1,"1216":1,"123":1,"124":1,"127":1,"14":1,"143":1,"15":1,"151":1,"164":1,"172":1,"18":1,"183":1,"2":1,"212":1,"262":1,"266":1,"279":1,"3":3,"301":1,"315":1,"32":14,"32868":1,"350":1,"4":1,"45":6,"46":3,"47":4,"48":2,"49":7,"5":2,"50":3,"51":5,"52":3,"53":1,"54":1,"55":17,"56":1,"57":1,"58":3,"59":3,"60":6,"63":1,"64":2,"65":4,"662":1,"67":3,"68":1,"69":2,"70":2,"74":1,"75":1,"76":1,"79":1,"803":1,"81":1,"82":2,"86":1,"9":1,"91":1,"93":1,"94":1},"interarrival_latency[-0]":0,"interarrival_latency[0-9]":10,"interarrival_latency[10-19]":4,"interarrival_latency[100-]":24,"interarrival_latency[20-29]":0,"interarrival_latency[30-39]":14,"interarrival_latency[40-49]":22,"interarrival_latency[50-59]":38,"interarrival_latency[60-69]":19,"interarrival_latency[70-79]":6,"interarrival_latency[80-89]":4,"interarrival_latency[90-99]":3,"num_act_cmds":10,"num_cycles":45253,"num_ondemand_pres":0,"num_pre_cmds":8,"num_read_cmds":144,"num_read_row_hits":134,"num_reads_done":144,"num_ref_cmds":17,"num_refb_cmds":0,"num_srefe_cmds":0,"num_srefx_cmds":0,"num_write_buf_hits":0,"num_write_cmds":0,"num_write_row_hits":0,"num_writes_done":0,"pre_stb_energy":{"0":6081015.600000001,"1":7227414.000000001},"rank_active_cycles":{"0":12132,"1":5888},"read_energy":304819.2,"read_latency":{"15":125,"16":1,"17":3,"18":2,"19":1,"25":7,"33":1,"61":1,"73":1,"80":1,"89":1},"read_latency[-0]":0,"read_latency[0-19]":132,"read_latency[100-119]":0,"read_latency[120-139]":0,"read_latency[140-159]":0,"read_latency[160-179]":0,"read_latency[180-199]":0,"read_latency[20-39]":8,"read_latency[200-]":0,"read_latency[40-59]":0,"read_latency[60-79]":2,"read_latency[80-99]":2,"ref_energy":1793404.8,"refb_energy":-0.0,"sref_cycles":{"0":0,"1":0},"sref_energy":{"0":0.0,"1":0.0},"total_energy":19926021.6,"write_energy":0.0,"write_latency":null,"write_latency[-0]":0,"write_latency[0-19]":0,"write_latency[100-119]":0,"write_latency[120-139]":0,"write_latency[140-159]":0,"write_latency[160-179]":0,"write_latency[180-199]":0,"write_latency[20-39]":0,"write_latency[200-]":0,"write_latency[40-59]":0,"write_latency[60-79]":0,"write_latency[80-99]":0}} \ No newline at end of file diff --git a/bbdev/api/steps/verilator/dramsim3.txt b/bbdev/api/steps/verilator/dramsim3.txt deleted file mode 100644 index a479c6c98..000000000 --- a/bbdev/api/steps/verilator/dramsim3.txt +++ /dev/null @@ -1,78 +0,0 @@ -########################################### -## Statistics of Channel 0 -########################################### -num_srefx_cmds = 0 # Number of SREFX commands -num_srefe_cmds = 0 # Number of SREFE commands -num_refb_cmds = 0 # Number of REFb commands -num_cycles = 45253 # Number of DRAM cycles -epoch_num = 0 # Number of epochs -num_write_buf_hits = 0 # Number of write buffer hits -num_write_cmds = 0 # Number of WRITE/WRITEP commands -num_reads_done = 144 # Number of read requests issued -hbm_dual_cmds = 0 # Number of cycles dual cmds issued -num_ref_cmds = 17 # Number of REF commands -num_read_row_hits = 134 # Number of read row buffer hits -num_read_cmds = 144 # Number of READ/READP commands -num_writes_done = 0 # Number of read requests issued -num_write_row_hits = 0 # Number of write row buffer hits -num_act_cmds = 10 # Number of ACT commands -num_pre_cmds = 8 # Number of PRE commands -num_ondemand_pres = 0 # Number of ondemend PRE commands -sref_cycles.0 = 0 # Cyles of rank in SREF mode rank.0 -sref_cycles.1 = 0 # Cyles of rank in SREF mode rank.1 -rank_active_cycles.0 = 12132 # Cyles of rank active rank.0 -rank_active_cycles.1 = 5888 # Cyles of rank active rank.1 -all_bank_idle_cycles.0 = 33121 # Cyles of all bank idle in rank rank.0 -all_bank_idle_cycles.1 = 39365 # Cyles of all bank idle in rank rank.1 -interarrival_latency[-0] = 0 # Request interarrival latency (cycles) -interarrival_latency[0-9] = 10 # Request interarrival latency (cycles) -interarrival_latency[10-19] = 4 # Request interarrival latency (cycles) -interarrival_latency[20-29] = 0 # Request interarrival latency (cycles) -interarrival_latency[30-39] = 14 # Request interarrival latency (cycles) -interarrival_latency[40-49] = 22 # Request interarrival latency (cycles) -interarrival_latency[50-59] = 38 # Request interarrival latency (cycles) -interarrival_latency[60-69] = 19 # Request interarrival latency (cycles) -interarrival_latency[70-79] = 6 # Request interarrival latency (cycles) -interarrival_latency[80-89] = 4 # Request interarrival latency (cycles) -interarrival_latency[90-99] = 3 # Request interarrival latency (cycles) -interarrival_latency[100-] = 24 # Request interarrival latency (cycles) -write_latency[-0] = 0 # Write cmd latency (cycles) -write_latency[0-19] = 0 # Write cmd latency (cycles) -write_latency[20-39] = 0 # Write cmd latency (cycles) -write_latency[40-59] = 0 # Write cmd latency (cycles) -write_latency[60-79] = 0 # Write cmd latency (cycles) -write_latency[80-99] = 0 # Write cmd latency (cycles) -write_latency[100-119] = 0 # Write cmd latency (cycles) -write_latency[120-139] = 0 # Write cmd latency (cycles) -write_latency[140-159] = 0 # Write cmd latency (cycles) -write_latency[160-179] = 0 # Write cmd latency (cycles) -write_latency[180-199] = 0 # Write cmd latency (cycles) -write_latency[200-] = 0 # Write cmd latency (cycles) -read_latency[-0] = 0 # Read request latency (cycles) -read_latency[0-19] = 132 # Read request latency (cycles) -read_latency[20-39] = 8 # Read request latency (cycles) -read_latency[40-59] = 0 # Read request latency (cycles) -read_latency[60-79] = 2 # Read request latency (cycles) -read_latency[80-99] = 2 # Read request latency (cycles) -read_latency[100-119] = 0 # Read request latency (cycles) -read_latency[120-139] = 0 # Read request latency (cycles) -read_latency[140-159] = 0 # Read request latency (cycles) -read_latency[160-179] = 0 # Read request latency (cycles) -read_latency[180-199] = 0 # Read request latency (cycles) -read_latency[200-] = 0 # Read request latency (cycles) -ref_energy = 1.7934e+06 # Refresh energy -refb_energy = -0 # Refresh-bank energy -write_energy = 0 # Write energy -read_energy = 304819 # Read energy -act_energy = 43200 # Activation energy -sref_energy.0 = 0 # SREF energy rank.0 -sref_energy.1 = 0 # SREF energy rank.1 -pre_stb_energy.0 = 6.08102e+06 # Precharge standby energy rank.0 -pre_stb_energy.1 = 7.22741e+06 # Precharge standby energy rank.1 -act_stb_energy.0 = 3.01359e+06 # Active standby energy rank.0 -act_stb_energy.1 = 1.46258e+06 # Active standby energy rank.1 -average_read_latency = 17.4167 # Average read request latency (cycles) -average_interarrival = 313.771 # Average request interarrival latency (cycles) -total_energy = 1.9926e+07 # Total energy (pJ) -average_power = 440.325 # Average power (mW) -average_bandwidth = 0.13577 # Average bandwidth diff --git a/bbdev/api/tests/test_api_verilator_run.py b/bbdev/api/tests/test_api_verilator_run.py index dbd5fb3e9..c12691182 100644 --- a/bbdev/api/tests/test_api_verilator_run.py +++ b/bbdev/api/tests/test_api_verilator_run.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --run '--jobs 16 --binary toy_relu_test-singlecore-baremetal --chip toy --batch'") +run_bbdev_case("bbdev verilator --run '--jobs 16 --binary toy-toy-ctest-mvin_mvout_test-baremetal --chip toy --batch'") diff --git a/bbdev/api/tests/test_api_verilator_sim.py b/bbdev/api/tests/test_api_verilator_sim.py index 99addce13..6c9ad70b2 100644 --- a/bbdev/api/tests/test_api_verilator_sim.py +++ b/bbdev/api/tests/test_api_verilator_sim.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --sim '--binary toy_relu_test-singlecore-baremetal --batch --chip toy'") +run_bbdev_case("bbdev verilator --sim '--binary toy-toy-ctest-mvin_mvout_test-baremetal --batch --chip toy'") diff --git a/bbdev/api/tests/test_uvm_chip_pb.py b/bbdev/api/tests/test_uvm_chip_pb.py new file mode 100644 index 000000000..6aa2cb75e --- /dev/null +++ b/bbdev/api/tests/test_uvm_chip_pb.py @@ -0,0 +1,53 @@ +import sys +from pathlib import Path + +API = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(API / "steps" / "config" / "scripts")) +sys.path.insert(0, str(API / "steps" / "uvm" / "scripts")) + +import chip_pb2 +from uvm_common import ball_domain, selected_mappings, vcs_defines + + +def _chip(): + c = chip_pb2.Chip() + core = c.cores.add() + m = core.balldomain.mappings.add() + m.ball_id = 5 + m.ball_dir = "lut" + m.in_bw = 2 + m.out_bw = 1 + isa = core.balldomain.isa.add() + isa.mnemonic = "LUT" + isa.funct7 = 66 + isa.bid = 5 + return c + + +def test_defines(): + c = _chip() + d = ball_domain(c) + m = selected_mappings(d, "lut")[0] + defs = vcs_defines(d, m) + assert defs == [ + "+define+BB_IN_BW=2", + "+define+BB_OUT_BW=1", + "+define+BB_MMIO_READ_BW=0", + "+define+BB_MMIO_WRITE_BW=0", + "+define+LUT_FUNCT7=66", + ] + + +def test_unknown_ball(): + try: + selected_mappings(ball_domain(_chip()), "transpose") + except ValueError as e: + assert "transpose" in str(e) + return + raise AssertionError("expected ValueError") + + +if __name__ == "__main__": + test_defines() + test_unknown_ball() + print("ok") diff --git a/bbdev/api/utils/search_workload.py b/bbdev/api/utils/search_workload.py index 0d4121f7d..21ec94655 100644 --- a/bbdev/api/utils/search_workload.py +++ b/bbdev/api/utils/search_workload.py @@ -13,6 +13,9 @@ def search_workload(search_dir: str, filename: str) -> Optional[str]: Returns: Absolute path of the found file, or None if not found """ + if os.path.isfile(filename): + return os.path.abspath(filename) + if not os.path.exists(search_dir): return None diff --git a/bbdev/bbdev b/bbdev/bbdev index b1bcdea15..4e0464cbe 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -167,8 +167,16 @@ def cleanup_config(config_path: str): def load_host_environment() -> None: - """No-op placeholder: host EDA env (TARGET_LIBRARY etc.) comes from the user shell (zshrc).""" - return + """Import the host login-shell environment used by the EDA tools.""" + result = subprocess.run( + ["zsh", "-lic", "env -0"], + check=True, + capture_output=True, + ) + for item in result.stdout.split(b"\0"): + if item: + key, value = item.split(b"=", 1) + os.environ[key.decode()] = value.decode() def parse_args(argv: list[str]) -> argparse.Namespace: diff --git a/bbdev/mcp/tools/uvm_build.py b/bbdev/mcp/tools/uvm_build.py index d56a14a8d..0d4792c6d 100644 --- a/bbdev/mcp/tools/uvm_build.py +++ b/bbdev/mcp/tools/uvm_build.py @@ -2,24 +2,18 @@ from __future__ import annotations -from typing import Any, Dict, Optional +from typing import Optional -from common import submit, err, fmt, need, opt +from common import submit, err, fmt, need def register(mcp): @mcp.tool() - def bbdev_uvm_build( - config: str, core_config: str, ball: Optional[str] = None, filelist: Optional[str] = None - ) -> str: + def bbdev_uvm_build(chip: str, ball: Optional[str] = None) -> str: """Build a Ball UVM simulation. POST /uvm/build.""" - if e := need("config", config): + if e := need("chip", chip): return err(e) - if e := need("core_config", core_config): - return err(e) - params: Dict[str, Any] = {"config": config, "core_config": core_config} + params = {"chip": chip} if ball: params["ball"] = ball - if filelist: - params["filelist"] = filelist return fmt(submit("/uvm/build", params)) diff --git a/bbdev/mcp/tools/uvm_run.py b/bbdev/mcp/tools/uvm_run.py index 7e763df66..93f537c9f 100644 --- a/bbdev/mcp/tools/uvm_run.py +++ b/bbdev/mcp/tools/uvm_run.py @@ -2,25 +2,18 @@ from __future__ import annotations -from typing import Any, Dict, Optional +from typing import Optional -from common import submit, err, fmt, need, opt +from common import submit, err, fmt, need def register(mcp): @mcp.tool() - def bbdev_uvm_run( - ball: str, - filelist: Optional[str] = None, - test: Optional[str] = None, - ) -> str: + def bbdev_uvm_run(chip: str, ball: Optional[str] = None) -> str: """Build and run a Ball UVM simulation. POST /uvm/run.""" - if e := need("ball", ball): + if e := need("chip", chip): return err(e) - params: Dict[str, Any] = {"ball": ball} - if filelist: - params["filelist"] = filelist - if test: - params["test"] = test + params = {"chip": chip} + if ball: + params["ball"] = ball return fmt(submit("/uvm/run", params)) - diff --git a/bbdev/mcp/tools/verilator_run.py b/bbdev/mcp/tools/verilator_run.py index 918766e03..4aa75920a 100644 --- a/bbdev/mcp/tools/verilator_run.py +++ b/bbdev/mcp/tools/verilator_run.py @@ -12,8 +12,9 @@ def register(mcp): def bbdev_verilator_run( binary: str, chip: str, - batch: bool = True, + batch: bool = False, coverage: bool = False, + no_wave: bool = False, jobs: Optional[int] = None, itrace: bool = False, mtrace: bool = False, @@ -31,6 +32,7 @@ def bbdev_verilator_run( "chip": chip, "batch": batch, "coverage": coverage, + "no-wave": no_wave, "itrace": itrace, "mtrace": mtrace, "pmctrace": pmctrace, diff --git a/bbdev/mcp/tools/verilator_sim.py b/bbdev/mcp/tools/verilator_sim.py index fc40ef799..ad2df6817 100644 --- a/bbdev/mcp/tools/verilator_sim.py +++ b/bbdev/mcp/tools/verilator_sim.py @@ -12,8 +12,9 @@ def register(mcp): def bbdev_verilator_sim( binary: str, chip: str, - batch: bool = True, + batch: bool = False, coverage: bool = False, + no_wave: bool = False, itrace: bool = False, mtrace: bool = False, pmctrace: bool = False, @@ -28,6 +29,7 @@ def bbdev_verilator_sim( "binary": binary, "chip": chip, "batch": batch, + "no-wave": no_wave, "itrace": itrace, "mtrace": mtrace, "pmctrace": pmctrace, From b00d5825f7c0c257452c7517491dcced4c7ffa87 Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Thu, 10 Sep 2026 18:27:25 +0800 Subject: [PATCH 142/196] fix: align per-chip build and P2E runtime paths (#16) --- .../bebop/p2e/03_runworkload_event.step.py | 50 +++++++++++-------- bbdev/api/steps/compiler/scripts/build.py | 3 +- bbdev/api/steps/kernel/01_build_event.step.py | 8 +-- .../02_eval_performance_event.step.py | 19 ++++--- 4 files changed, 47 insertions(+), 33 deletions(-) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index 639a6839a..e39e2e0bd 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -11,6 +11,7 @@ import re import sys from datetime import datetime +from pathlib import Path from motia import FlowContext, queue @@ -19,7 +20,7 @@ sys.path.insert(0, utils_path) from utils.event_common import require_chip -from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path, log_dir, rtl_dir +from utils.path import bebop_cargo_env, get_buckyball_path, log_dir, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -33,16 +34,11 @@ def resolve_image(bbdir: str, image_name: str, chip: str) -> str: - """Search chip workload build output recursively for .hex.""" - workload_root = os.path.join(chip_output_root(bbdir, chip), "workloads") - matches = glob.glob(f"{workload_root}/**/{image_name}.hex", recursive=True) - if not matches: - return "" - if len(matches) > 1: - raise ValueError( - f"multiple .hex files for {image_name!r} under {workload_root}: {matches}" - ) - return matches[0] + """Resolve a kernel image name to its deterministic output path.""" + image_name = image_name.replace(r"\_", "_") + filename = image_name if image_name.endswith(".hex") else f"{image_name}.hex" + path = Path(bbdir) / "bb-tests" / "output" / "kernel" / chip / filename + return str(path) if path.is_file() else "" def resolve_runtime_config(bitstream: str, requested_config: object) -> str: @@ -114,7 +110,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if not image_path: ctx.logger.error( f"image .hex not found for name: {image_name} " - f"(searched bb-tests/output/{chip}/workloads/)" + f"(expected bb-tests/output/kernel/{chip}/)" ) await check_result( ctx, 1, continue_run=False, @@ -169,10 +165,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="bebop p2e runtime", env={**os.environ.copy(), **bebop_cargo_env(bbdir, chip)}, ) + runtime_lib_dir = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm") rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") - libvctb_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so") - if runtime_result.returncode != 0 or not all(os.path.isfile(path) for path in (rtcfg_path, libvctb_path)): - missing = [path for path in (rtcfg_path, libvctb_path) if not os.path.isfile(path)] + libvctb_path = os.path.join(runtime_lib_dir, "libvCtb.so") + libstdcxx_path = os.path.join(runtime_lib_dir, "libstdc++.so.6") + bebop_p2e_path = os.path.join(build_dir, "bebop-p2e") + runtime_artifacts = (rtcfg_path, libvctb_path, libstdcxx_path, bebop_p2e_path) + if runtime_result.returncode != 0 or not all(os.path.isfile(path) for path in runtime_artifacts): + missing = [path for path in runtime_artifacts if not os.path.isfile(path)] if missing: ctx.logger.error(f"P2E runtime artifacts missing: {missing}") await check_result( @@ -189,14 +189,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - # ── Run bebop run p2e ───────────────────────────────────────────────── + # Run the case-local executable produced above. It must load the VVAC + # runtime's libstdc++ (6.0.25), not Cargo/Nix's newer libstdc++; mixing the + # two ABIs crashes inside ICtbMgr::init(). Apply LD_LIBRARY_PATH only to + # the runtime process so Cargo itself keeps its normal library environment. run_cmd = ( - f"cargo run --release --features p2e " - f"--config=\"env.OUT_PATH='{build_dir}'\" " - f"-- run p2e " + f"\"{bebop_p2e_path}\" run p2e " f"--image=\"{image_path}\" " f"--bitstream=\"{bitstream}\" " - f"--log-dir=\"{log_dir}\"" + f"--log-dir=\"{run_log}\"" ) if multi_fpga: run_cmd += " --multi-fpga" @@ -208,13 +209,20 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if input_data.get(trace_name, False): run_cmd += f" --{trace_name}" ctx.logger.info(f"Running bebop p2e runworkload: {run_cmd}") + run_env = {**os.environ.copy(), **bebop_cargo_env(bbdir, chip)} + inherited_library_path = run_env.get("LD_LIBRARY_PATH") + run_env["LD_LIBRARY_PATH"] = ( + f"{runtime_lib_dir}:{inherited_library_path}" + if inherited_library_path + else runtime_lib_dir + ) run_result = await stream_run_logger_async( cmd=run_cmd, logger=ctx.logger, cwd=bebop_dir, stdout_prefix="bebop p2e runworkload", stderr_prefix="bebop p2e runworkload", - env={**os.environ.copy(), **bebop_cargo_env(bbdir, chip)}, + env=run_env, ) await check_result( diff --git a/bbdev/api/steps/compiler/scripts/build.py b/bbdev/api/steps/compiler/scripts/build.py index 74b7c15a0..1bc03ad74 100644 --- a/bbdev/api/steps/compiler/scripts/build.py +++ b/bbdev/api/steps/compiler/scripts/build.py @@ -172,6 +172,7 @@ def build_compiler( "buddy-opt", "buddy-translate", "buddy-llc", + "rax-pack", "python-package-buddy", "BuddyMLIRPythonModules", ], @@ -180,7 +181,7 @@ def build_compiler( task_scope=task_scope, output_prefix="compiler build", ) - for tool in ("buddy-opt", "buddy-translate", "buddy-llc"): + for tool in ("buddy-opt", "buddy-translate", "buddy-llc", "rax-pack"): if not (build / "bin" / tool).is_file(): raise RuntimeError(f"compiler build failed: missing {build / 'bin' / tool}") return build diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index f8f1acb3a..9bc9f1e5f 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -157,10 +157,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() kernel_src = os.path.join(bbdir, "bb-tests", "workloads", "lib", "kernel") - output_dir = os.path.join(bbdir, "bb-tests", "output", "kernel") - - os.makedirs(output_dir, exist_ok=True) - try: hart_params = hart_count_params(input_data) model = kernel_model(input_data) @@ -173,6 +169,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.error(str(e)) await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) return + output_dir = os.path.join(bbdir, "bb-tests", "output", "kernel") + if chip: + output_dir = os.path.join(output_dir, chip) + os.makedirs(output_dir, exist_ok=True) kernel_build = kernel_build_dir( bbdir, hart_params, model, chip, interactive=interactive ) diff --git a/bbdev/api/steps/regression/02_eval_performance_event.step.py b/bbdev/api/steps/regression/02_eval_performance_event.step.py index 5d1deb656..75b723f7f 100644 --- a/bbdev/api/steps/regression/02_eval_performance_event.step.py +++ b/bbdev/api/steps/regression/02_eval_performance_event.step.py @@ -89,10 +89,12 @@ def _workload_build(bbdir, chip, model, logger, task_scope): ) -def _kernel_build_cmds(bbdir, model, dataset=""): +def _kernel_build_cmds(bbdir, chip, model, dataset=""): kernel_src = os.path.join(bbdir, "bb-tests", "workloads", "lib", "kernel") hart_params = {"visible": 64, "total": 64, "hidden_base": 64} - kernel_build = _kernel.kernel_build_dir(bbdir, hart_params, model=model) + kernel_build = _kernel.kernel_build_dir( + bbdir, hart_params, model=model, chip=chip + ) ds_arg = "" if dataset: ds_arg = f" -DBUCKYBALL_MODEL_DATASET={shlex.quote(dataset)}" @@ -102,14 +104,17 @@ def _kernel_build_cmds(bbdir, model, dataset=""): f"-DBUCKYBALL_TOTAL_HART_COUNT=64 " f"-DBUCKYBALL_HIDDEN_HART_BASE=64 " f"-DBUCKYBALL_KERNEL_MODEL={model} " - f"-DBUCKYBALL_KERNEL_CHIP= " + f"-DBUCKYBALL_KERNEL_CHIP={chip} " f"-DBUCKYBALL_KERNEL_INTERACTIVE=OFF" f"{ds_arg}" ) build = f"cmake --build {kernel_build} --target kernel-build" payload = _kernel.fw_payload_name(hart_params, model=model) - fw_bin = os.path.join(bbdir, "bb-tests", "output", "kernel", f"{payload}.bin") - fw_hex = os.path.join(bbdir, "bb-tests", "output", "kernel", f"{payload}.hex") + kernel_output = os.path.join( + bbdir, "bb-tests", "output", "kernel", chip + ) + fw_bin = os.path.join(kernel_output, f"{payload}.bin") + fw_hex = os.path.join(kernel_output, f"{payload}.hex") return configure, build, fw_bin, fw_hex @@ -118,7 +123,7 @@ def _p2e_run_cmds(bbdir, bitstream, image_name, chip, input_data): if not image_path: raise FileNotFoundError( f"image .hex not found for name: {image_name} " - f"(searched bb-tests/output/{chip}/workloads/)" + f"(expected bb-tests/output/kernel/{chip}/)" ) bitstream = os.path.abspath(bitstream) build_dir = os.path.dirname(os.path.dirname(bitstream)) @@ -226,7 +231,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: spec = acc_specs[model] try: k_cfg, k_build, fw_bin, fw_hex = _kernel_build_cmds( - bbdir, model, dataset=spec["dataset"], + bbdir, chip, model, dataset=spec["dataset"], ) except ValueError as e: ctx.logger.error(str(e)) From 1eca7c492b6516a9d97772fc64f510cae537d727 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 11 Sep 2026 20:53:52 +0800 Subject: [PATCH 143/196] feat: add VCS support to Bebop workflow with new APIs for verilog generation, build, and batch execution --- .../api/steps/bebop/bemu/02_sim_event.step.py | 17 +---- .../steps/bebop/bemu/03_batch_event.step.py | 4 +- .../steps/bebop/p2e/04_batch_event.step.py | 4 +- .../steps/bebop/vcs/01_verilog_api.step.py | 10 +++ .../api/steps/bebop/vcs/02_build_api.step.py | 11 ++++ .../api/steps/bebop/vcs/03_batch_api.step.py | 11 ++++ .../steps/bebop/vcs/03_batch_event.step.py | 62 +++++++++++++++++++ .../bebop/verilator/04_sim_event.step.py | 11 +--- .../bebop/verilator/06_batch_event.step.py | 4 +- bbdev/api/steps/compiler/scripts/build.py | 53 +++++++++++++--- bbdev/api/steps/dc/03_sim_event.step.py | 4 +- .../steps/mill/02_vcs_verilog_event.step.py | 5 +- bbdev/api/steps/uvm/00_waive_event.step.py | 43 +++++++++++++ bbdev/api/steps/uvm/scripts/test_report.py | 13 ++++ bbdev/api/steps/uvm/scripts/waive.py | 26 ++++++++ bbdev/api/steps/vcs/03_build_event.step.py | 9 ++- bbdev/api/steps/vcs/04_sim_event.step.py | 4 +- .../api/steps/verilator/04_sim_event.step.py | 4 +- bbdev/api/steps/verilator/05_run_api.step.py | 4 +- bbdev/api/utils/path.py | 4 ++ bbdev/api/utils/search_workload.py | 24 ++++--- bbdev/bbdev | 5 ++ 22 files changed, 276 insertions(+), 56 deletions(-) create mode 100644 bbdev/api/steps/bebop/vcs/01_verilog_api.step.py create mode 100644 bbdev/api/steps/bebop/vcs/02_build_api.step.py create mode 100644 bbdev/api/steps/bebop/vcs/03_batch_api.step.py create mode 100644 bbdev/api/steps/bebop/vcs/03_batch_event.step.py create mode 100644 bbdev/api/steps/uvm/00_waive_event.step.py create mode 100644 bbdev/api/steps/uvm/scripts/test_report.py create mode 100644 bbdev/api/steps/uvm/scripts/waive.py diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 732bf1537..704b6cb73 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -19,9 +19,9 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.path import bebop_cargo_env, bebop_target_dir, get_buckyball_path, log_dir, workload_tests_root +from utils.path import bebop_cargo_env, bebop_target_dir, get_buckyball_path, log_dir, workloads_output_root from utils.stream_run import stream_run_logger_async -from utils.search_workload import search_workload, search_workload_all +from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id from utils.process_registry import cancellation_requested from bemu_common import bemu_core_manifest, bemu_manifest, bemu_tile_index, chip_emu_manifest @@ -69,18 +69,7 @@ def resolve_bemu_binary(bbdir: str, chip: str, binary_name: str) -> str | None: if Path(binary_name).name != binary_name: return None - workload_root = workload_tests_root(bbdir, chip) - chip_root = f"{workload_root}/CTest/chips/{chip}" - chip_binary = search_workload(chip_root, binary_name) - if chip_binary is not None: - return chip_binary - - matches = search_workload_all(workload_root, binary_name) - chip_marker = f"{os.path.sep}CTest{os.path.sep}chips{os.path.sep}" - non_chip_matches = [path for path in matches if chip_marker not in path] - if len(non_chip_matches) == 1: - return non_chip_matches[0] - return None + return search_workload(workloads_output_root(bbdir), binary_name) def is_native_host_elf(path: str) -> bool: diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index be62f9a52..7de4c4a9e 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -22,7 +22,7 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path +from utils.path import bebop_cargo_env, get_buckyball_path, workloads_output_root from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from bemu_common import bemu_manifest @@ -63,7 +63,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - elf_root = chip_output_root(bbdir, chip) + elf_root = workloads_output_root(bbdir) env = os.environ.copy() env.update(bebop_cargo_env(bbdir, chip)) diff --git a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py index b1445398e..aee02f005 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py @@ -20,7 +20,7 @@ sys.path.insert(0, bebop_path) from utils.event_common import require_chip -from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path, rtl_dir +from utils.path import bebop_cargo_env, get_buckyball_path, rtl_dir, workloads_output_root from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from regression import regression_workload_toml @@ -81,7 +81,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - elf_root = chip_output_root(bbdir, chip) + elf_root = workloads_output_root(bbdir) cargo_env = bebop_cargo_env(bbdir, chip) test_type = input_data.get("test", "elf-tests") diff --git a/bbdev/api/steps/bebop/vcs/01_verilog_api.step.py b/bbdev/api/steps/bebop/vcs/01_verilog_api.step.py new file mode 100644 index 000000000..27b6bd565 --- /dev/null +++ b/bbdev/api/steps/bebop/vcs/01_verilog_api.step.py @@ -0,0 +1,10 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_chip + +config = {"name": "bebop-vcs-verilog-api", "flows": ["bebop"], "triggers": [api("POST", "/bebop/vcs/verilog")], "enqueues": ["vcs.verilog"]} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + await ctx.enqueue({"topic": "vcs.verilog", "data": {"chip": require_chip(req.body or {}), "bebop_vcs": True, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/vcs/02_build_api.step.py b/bbdev/api/steps/bebop/vcs/02_build_api.step.py new file mode 100644 index 000000000..aee98a5c7 --- /dev/null +++ b/bbdev/api/steps/bebop/vcs/02_build_api.step.py @@ -0,0 +1,11 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_chip + +config = {"name": "bebop-vcs-build-api", "flows": ["bebop"], "triggers": [api("POST", "/bebop/vcs/build")], "enqueues": ["vcs.build"]} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + await ctx.enqueue({"topic": "vcs.build", "data": {"chip": require_chip(body), "jobs": body.get("jobs", 16), "bebop_vcs": True, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/vcs/03_batch_api.step.py b/bbdev/api/steps/bebop/vcs/03_batch_api.step.py new file mode 100644 index 000000000..ab9762d23 --- /dev/null +++ b/bbdev/api/steps/bebop/vcs/03_batch_api.step.py @@ -0,0 +1,11 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api + +from utils.event_common import require_chip + +config = {"name": "bebop-vcs-batch-api", "flows": ["bebop"], "triggers": [api("POST", "/bebop/vcs/batch")], "enqueues": ["bebop.vcs.batch"]} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + await ctx.enqueue({"topic": "bebop.vcs.batch", "data": {"chip": require_chip(body), "test": body.get("test", "elf-tests"), "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/vcs/03_batch_event.step.py b/bbdev/api/steps/bebop/vcs/03_batch_event.step.py new file mode 100644 index 000000000..4d0cb1d93 --- /dev/null +++ b/bbdev/api/steps/bebop/vcs/03_batch_event.step.py @@ -0,0 +1,62 @@ +import os +import shlex +import subprocess +import sys +import tomllib +from datetime import datetime +from pathlib import Path + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +bebop_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +if bebop_path not in sys.path: + sys.path.insert(0, bebop_path) + +from regression import regression_workload_toml +from utils.event_common import check_result, get_origin_trace_id, require_chip +from utils.path import get_buckyball_path, log_dir, rtl_dir, workloads_output_root +from utils.search_workload import search_workload +from utils.stream_run import stream_run_logger_async + +config = {"name": "bebop-vcs-batch", "flows": ["bebop"], "triggers": [queue("bebop.vcs.batch")], "enqueues": []} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + try: + chip = require_chip(input_data) + bbdir = get_buckyball_path() + test_type = input_data.get("test", "elf-tests") + manifest = regression_workload_toml(chip, "verilator", test_type, bbdir) + except ValueError as error: + await check_result(ctx, 1, extra_fields={"error": str(error)}, trace_id=origin_tid) + return + simv = Path(rtl_dir(bbdir, chip, "verilog")) / "vcs" / "simv" + if not simv.is_file(): + await check_result(ctx, 1, extra_fields={"error": "missing VCS simv: run bebop-vcs --verilog and --build first"}, trace_id=origin_tid) + return + tests = tomllib.loads(Path(manifest).read_text())["workloads"]["tests"] + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + report = Path(log_dir(bbdir, chip, "verilog", stamp, "bebop-vcs", test_type)) + report.mkdir(parents=True) + libc = subprocess.check_output(["g++", "-print-file-name=libc.so.6"], text=True).strip() + ball_names = [path.name for path in (Path(bbdir) / "examples" / "balls").iterdir() if path.is_dir()] + counts = {ball: [0, 0] for ball in ball_names} + passed = 0 + for name in tests: + ball = next((item for item in ball_names if f"-ctest-{item}" in name), "framework") + counts.setdefault(ball, [0, 0])[1] += 1 + binary = search_workload(workloads_output_root(bbdir), name) + if binary is None: + continue + command = " ".join(("env", f"LD_LIBRARY_PATH={shlex.quote(str(Path(libc).parent))}", shlex.quote(str(simv)), f"+elf={shlex.quote(binary)}", "+batch", "+trace=none", "+no-wave", f"+dramsim_ini_dir={shlex.quote(str(Path(bbdir) / 'result/share/dramsim3/configs'))}")) + result = await stream_run_logger_async(cmd=command, logger=ctx.logger, cwd=str(simv.parent), stdout_prefix="bebop vcs batch", stderr_prefix="bebop vcs batch") + if result.returncode == 0: + counts[ball][0] += 1 + passed += 1 + index = report / "test-index.txt" + index.write_text("ball_dir vcs\n" + "\n".join(f"{ball} {ok}/{total}" for ball, (ok, total) in sorted(counts.items()) if total) + "\n") + await check_result(ctx, 0 if passed == len(tests) else 1, extra_fields={"task": "batch", "passed": passed, "total": len(tests), "test_index": str(index)}, trace_id=origin_tid) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index a308b7233..4990b421f 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -20,7 +20,7 @@ sys.path.insert(0, scripts_path) from utils.event_common import require_chip -from utils.path import bebop_target_dir, get_buckyball_path, log_dir, rtl_dir, workload_tests_root +from utils.path import bebop_target_dir, get_buckyball_path, log_dir, rtl_dir, workloads_output_root from utils.stream_run import stream_run_logger_async from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id @@ -152,14 +152,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return binary_name = input_data.get("binary", "") - workload_root = workload_tests_root(bbdir, chip) - binary_path = None - if diff and chip: - binary_path = search_workload( - f"{workload_root}/CTest/chips/{chip}", binary_name - ) - if binary_path is None: - binary_path = search_workload(workload_root, binary_name) + binary_path = search_workload(workloads_output_root(bbdir), binary_name) if binary_path is None: ctx.logger.error(f"binary not found: {binary_name}") await check_result( diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 6a850df05..2175811f6 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -19,7 +19,7 @@ sys.path.insert(0, bebop_path) from utils.event_common import require_chip -from utils.path import bebop_cargo_env, chip_output_root, get_buckyball_path, rtl_dir +from utils.path import bebop_cargo_env, get_buckyball_path, rtl_dir, workloads_output_root from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from regression import regression_workload_toml @@ -52,7 +52,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - elf_root = chip_output_root(bbdir, chip) + elf_root = workloads_output_root(bbdir) test_type = input_data.get("test", "elf-tests") rushB = bool(input_data.get("rushB", False)) diff --git a/bbdev/api/steps/compiler/scripts/build.py b/bbdev/api/steps/compiler/scripts/build.py index 1bc03ad74..05b55555c 100644 --- a/bbdev/api/steps/compiler/scripts/build.py +++ b/bbdev/api/steps/compiler/scripts/build.py @@ -1,7 +1,9 @@ from __future__ import annotations +import argparse import fcntl import os +import re import shlex import shutil import subprocess @@ -115,27 +117,45 @@ def build_llvm( return llvm_build -def compiler_build_dir(repo: str | Path, chip: str) -> Path: - return Path(repo).resolve() / "compiler" / "thirdparty" / "buddy-mlir" / "build" / chip +def compiler_build_dir(repo: str | Path, instance: str) -> Path: + return ( + Path(repo).resolve() + / "compiler" + / "thirdparty" + / "buddy-mlir" + / "build" + / instance + ) def build_compiler( repo: str | Path, *, chip: str, + instance: str | None = None, + chip_pb: str | Path | None = None, logger: object | None = None, task_scope: str | None = None, ) -> Path: root = Path(repo).resolve() if not chip: raise ValueError("chip is required") - chip_pb = root / "examples" / "chips" / chip / "configs" / "generated" / "chip.pb" - if not chip_pb.is_file(): - raise RuntimeError(f"missing {chip_pb}; run bbdev config --install") + build_instance = instance or chip + if not re.fullmatch(r"[A-Za-z0-9_-]+", build_instance): + raise ValueError(f"invalid compiler build instance: {build_instance}") + if (instance is None) != (chip_pb is None): + raise ValueError("compiler variant requires both instance and chip_pb") + selected_pb = ( + Path(chip_pb).resolve() + if chip_pb is not None + else root / "examples" / "chips" / chip / "configs" / "generated" / "chip.pb" + ) + if not selected_pb.is_file(): + raise RuntimeError(f"missing {selected_pb}; generate the requested chip.pb first") buddy = root / "compiler" / "thirdparty" / "buddy-mlir" llvm = build_llvm(root, logger=logger, task_scope=task_scope) python = compiler_python(root) - build = compiler_build_dir(root, chip) + build = compiler_build_dir(root, build_instance) build.mkdir(parents=True, exist_ok=True) cmake = [ "cmake", @@ -146,7 +166,7 @@ def build_compiler( "-B", str(build), f"-DBUDDY_EXTERNAL_DIALECTS_DIR={root / 'compiler'}", - f"-DBUCKYBALL_CHIP_PB={chip_pb}", + f"-DBUCKYBALL_CHIP_PB={selected_pb}", f"-DMLIR_DIR={llvm / 'lib' / 'cmake' / 'mlir'}", f"-DLLVM_DIR={llvm / 'lib' / 'cmake' / 'llvm'}", "-DLLVM_ENABLE_ASSERTIONS=ON", @@ -185,3 +205,22 @@ def build_compiler( if not (build / "bin" / tool).is_file(): raise RuntimeError(f"compiler build failed: missing {build / 'bin' / tool}") return build + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build a chip compiler instance") + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--chip", required=True) + parser.add_argument("--instance") + parser.add_argument("--chip-pb", type=Path) + args = parser.parse_args() + build_compiler( + args.repo, + chip=args.chip, + instance=args.instance, + chip_pb=args.chip_pb, + ) + + +if __name__ == "__main__": + main() diff --git a/bbdev/api/steps/dc/03_sim_event.step.py b/bbdev/api/steps/dc/03_sim_event.step.py index 863113176..833638430 100644 --- a/bbdev/api/steps/dc/03_sim_event.step.py +++ b/bbdev/api/steps/dc/03_sim_event.step.py @@ -13,7 +13,7 @@ sys.path.insert(0, scripts_path) from utils.event_common import check_result, get_origin_trace_id -from utils.path import get_buckyball_path, workload_build_dir, workload_tests_root +from utils.path import get_buckyball_path, workload_build_dir, workloads_output_root from utils.search_workload import search_workload from utils.stream_run import stream_run_logger_async from tapeout import get_tapeout_contract, resolve_power_window, write_run_env @@ -84,7 +84,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() workload = None for root in ( - workload_tests_root(bbdir, chip), + workloads_output_root(bbdir), os.path.join(workload_build_dir(bbdir, chip), "src"), ): workload = search_workload(root, str(workload_name)) diff --git a/bbdev/api/steps/mill/02_vcs_verilog_event.step.py b/bbdev/api/steps/mill/02_vcs_verilog_event.step.py index eb6c1feda..acc8c98b8 100644 --- a/bbdev/api/steps/mill/02_vcs_verilog_event.step.py +++ b/bbdev/api/steps/mill/02_vcs_verilog_event.step.py @@ -40,7 +40,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: arch = os.path.join(bbdir, "arch") try: mill_config, build_dir = rtl_out( - bbdir, chip, "tapeout", input_data.get("output_dir"), + bbdir, + chip, + "verilog" if input_data.get("bebop_vcs") else "tapeout", + input_data.get("output_dir"), ) os.makedirs(build_dir, exist_ok=True) returncode = ( diff --git a/bbdev/api/steps/uvm/00_waive_event.step.py b/bbdev/api/steps/uvm/00_waive_event.step.py new file mode 100644 index 000000000..90538afc1 --- /dev/null +++ b/bbdev/api/steps/uvm/00_waive_event.step.py @@ -0,0 +1,43 @@ +import asyncio +import os +import sys +from pathlib import Path + +from motia import FlowContext, queue + +step_dir = os.path.dirname(os.path.abspath(__file__)) +utils_path = os.path.abspath(os.path.join(step_dir, "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) +if step_dir not in sys.path: + sys.path.insert(0, step_dir) + +from scripts.waive import apply_waivers +from utils.event_common import check_result, get_origin_trace_id, require_chip +from utils.path import get_buckyball_path, rtl_dir + +config = { + "name": "uvm-waive", + "description": "apply common Chisel coverage waivers to generated RTL", + "flows": ["uvm"], + "triggers": [queue("uvm.waive")], + "enqueues": ["uvm.build", "uvm.run"], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + try: + chip = require_chip(input_data) + task = input_data["uvm_task"] + if task not in ("build", "run"): + raise ValueError(f"invalid UVM task: {task}") + bbdir = get_buckyball_path() + count = await asyncio.to_thread(apply_waivers, Path(rtl_dir(bbdir, chip, "tapeout"))) + except (KeyError, ValueError) as error: + await check_result(ctx, 1, extra_fields={"task": "waive", "error": str(error)}, trace_id=origin_tid) + return + await check_result(ctx, 0, continue_run=True, extra_fields={"task": "waive", "count": count}, trace_id=origin_tid) + await ctx.enqueue( + {"topic": f"uvm.{task}", "data": {**input_data, "force_rebuild": True, "_trace_id": origin_tid}} + ) diff --git a/bbdev/api/steps/uvm/scripts/test_report.py b/bbdev/api/steps/uvm/scripts/test_report.py new file mode 100644 index 000000000..17daa1657 --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/test_report.py @@ -0,0 +1,13 @@ +from pathlib import Path + + +def discover_tests(bbdir: str, chip: str, ball: str) -> list[Path]: + root = Path(bbdir) / "bb-tests" / "output" / chip / "workloads" / "src" / "CTest" / "chips" / chip / "balls" / chip / ball / "ctests" + return sorted(root.glob("*-baremetal")) + + +def write_test_index(path: Path, rows: list[tuple[str, str, str]]) -> None: + path.write_text( + "ball_dir bemu verilator\n" + "\n".join(f"{ball} {bemu} {verilator}" for ball, bemu, verilator in rows) + "\n" + ) + diff --git a/bbdev/api/steps/uvm/scripts/waive.py b/bbdev/api/steps/uvm/scripts/waive.py new file mode 100644 index 000000000..d64ebe1d0 --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/waive.py @@ -0,0 +1,26 @@ +import re +from pathlib import Path + + +WAIVE_LINE_PATTERNS = ( + re.compile(r"^\s*(?:wire|reg|logic)\b.*\b_(?:GEN|T)(?:_\d+)?\b"), +) + + +def apply_waivers(rtl_dir: Path) -> int: + if not rtl_dir.is_dir(): + raise ValueError(f"missing generated RTL directory: {rtl_dir}") + changed = 0 + for path in sorted(rtl_dir.glob("*.sv")): + lines = path.read_text().splitlines(keepends=True) + output = [] + for line in lines: + if any(pattern.search(line) for pattern in WAIVE_LINE_PATTERNS) and ( + not output or output[-1] != "//VCS coverage off\n" + ): + output.extend(("//VCS coverage off\n", line, "//VCS coverage on\n")) + changed += 1 + else: + output.append(line) + path.write_text("".join(output)) + return changed diff --git a/bbdev/api/steps/vcs/03_build_event.step.py b/bbdev/api/steps/vcs/03_build_event.step.py index 52e282025..4a049531f 100644 --- a/bbdev/api/steps/vcs/03_build_event.step.py +++ b/bbdev/api/steps/vcs/03_build_event.step.py @@ -40,6 +40,7 @@ initial begin if ($value$plusargs("timeout-ns=%d", timeout_ns)) begin end repeat (10) @(posedge clock); + @(negedge clock); reset = 1'b0; #(timeout_ns); $fatal(1, "VCS simulation timed out after %0d ns", timeout_ns); @@ -130,7 +131,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return bbdir = get_buckyball_path() - build_dir = rtl_dir(bbdir, chip, "tapeout", input_data.get("output_dir")) + build_dir = rtl_dir( + bbdir, + chip, + "verilog" if input_data.get("bebop_vcs") else "tapeout", + input_data.get("output_dir"), + ) vsrcs = sorted( path for path in ( glob.glob(f"{build_dir}/**/*.v", recursive=True) @@ -187,6 +193,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: [ "env -u NIX_LDFLAGS -u NIX_CFLAGS_COMPILE -u NIX_LDFLAGS_FOR_TARGET -u NIX_CFLAGS_COMPILE_FOR_TARGET -u CPATH -u LIBRARY_PATH -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH -u CFLAGS -u CXXFLAGS -u LDFLAGS", "vcs -full64 -sverilog -timescale=1ns/1ps -cpp g++ -cc g++ -ld g++", + "+define+PRINTF_COND=0" if input_data.get("bebop_vcs") else "", "-top BBSimVcsHarness -debug_access+all -hsopt=off", f"-j {jobs}", f"-Mdir={shlex.quote(str(artifact_dir / 'csrc'))}", diff --git a/bbdev/api/steps/vcs/04_sim_event.step.py b/bbdev/api/steps/vcs/04_sim_event.step.py index 1ea55cdee..0102b9eb0 100644 --- a/bbdev/api/steps/vcs/04_sim_event.step.py +++ b/bbdev/api/steps/vcs/04_sim_event.step.py @@ -12,7 +12,7 @@ from utils.event_common import check_result, get_origin_trace_id from utils.event_common import require_chip -from utils.path import get_buckyball_path, log_dir, rtl_dir, workload_build_dir, workload_tests_root +from utils.path import get_buckyball_path, log_dir, rtl_dir, workload_build_dir, workloads_output_root from utils.search_workload import search_workload from utils.stream_run import stream_run_logger_async @@ -30,7 +30,7 @@ def _resolve_binary(bbdir: str, chip: str, binary: str) -> str | None: if os.path.isfile(binary): return os.path.abspath(binary) for root in ( - workload_tests_root(bbdir, chip), + workloads_output_root(bbdir), os.path.join(workload_build_dir(bbdir, chip), "src"), ): found = search_workload(root, binary) diff --git a/bbdev/api/steps/verilator/04_sim_event.step.py b/bbdev/api/steps/verilator/04_sim_event.step.py index 5bac2a837..ba88a0042 100644 --- a/bbdev/api/steps/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -13,7 +13,7 @@ sys.path.insert(0, utils_path) from utils.event_common import require_chip -from utils.path import get_buckyball_path, log_dir, rtl_dir, workload_tests_root +from utils.path import get_buckyball_path, log_dir, rtl_dir, workloads_output_root from utils.stream_run import stream_run_logger_async from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id @@ -118,7 +118,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return coverage = input_data.get("coverage", False) - workload_root = workload_tests_root(bbdir, chip) + workload_root = workloads_output_root(bbdir) binary_path = search_workload(workload_root, binary_name) ctx.logger.info(f"binary_path: {binary_path}") if binary_path is None: diff --git a/bbdev/api/steps/verilator/05_run_api.step.py b/bbdev/api/steps/verilator/05_run_api.step.py index 030fc223e..3810a06bf 100644 --- a/bbdev/api/steps/verilator/05_run_api.step.py +++ b/bbdev/api/steps/verilator/05_run_api.step.py @@ -8,7 +8,7 @@ sys.path.insert(0, utils_path) from utils.event_common import require_chip -from utils.path import get_buckyball_path, workload_tests_root +from utils.path import get_buckyball_path, workloads_output_root from utils.search_workload import search_workload config = { @@ -38,7 +38,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: }, ) - search_dir = workload_tests_root(get_buckyball_path(), chip) + search_dir = workloads_output_root(get_buckyball_path()) if search_workload(search_dir, binary) is None: return ApiResponse( status=400, diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index 60e332efc..cd29c9695 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -43,6 +43,10 @@ def chip_output_root(bbdir, chip): return os.path.join(bbdir, "bb-tests", "output", chip) +def workloads_output_root(bbdir): + return os.path.join(bbdir, "bb-tests", "output") + + def workload_build_dir(bbdir, chip): chip = _chip_name(chip) return os.path.join(bbdir, "bb-tests", "workloads", "build", chip) diff --git a/bbdev/api/utils/search_workload.py b/bbdev/api/utils/search_workload.py index 21ec94655..522b17792 100644 --- a/bbdev/api/utils/search_workload.py +++ b/bbdev/api/utils/search_workload.py @@ -4,26 +4,28 @@ def search_workload(search_dir: str, filename: str) -> Optional[str]: """ - Recursively search for a specified filename in the directory and its subdirectories + Recursively resolve one workload file name in the directory and its subdirectories. Args: search_dir: Root directory to search filename: Filename to search for Returns: - Absolute path of the found file, or None if not found + Absolute path of the file, or None if not found. + + Raises: + ValueError: the file name has more than one match. """ if os.path.isfile(filename): return os.path.abspath(filename) - if not os.path.exists(search_dir): - return None - - for root, dirs, files in os.walk(search_dir): - if filename in files: - return os.path.abspath(os.path.join(root, filename)) - - return None + matches = search_workload_all(search_dir, filename) + if len(matches) > 1: + raise ValueError( + f"workload file name {filename!r} is ambiguous under {search_dir}: " + + ", ".join(matches) + ) + return matches[0] if matches else None def search_workload_all(search_dir: str, filename: str) -> List[str]: @@ -43,6 +45,7 @@ def search_workload_all(search_dir: str, filename: str) -> List[str]: return results for root, dirs, files in os.walk(search_dir): + dirs.sort() if filename in files: results.append(os.path.abspath(os.path.join(root, filename))) @@ -68,6 +71,7 @@ def search_workload_pattern(search_dir: str, pattern: str) -> List[str]: return results for root, dirs, files in os.walk(search_dir): + dirs.sort() for file in files: if fnmatch.fnmatch(file, pattern): results.append(os.path.abspath(os.path.join(root, file))) diff --git a/bbdev/bbdev b/bbdev/bbdev index 4e0464cbe..f93936c61 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -74,6 +74,11 @@ WORKFLOW_COMMANDS = { "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--rushB] [--jobs ] [--chip ] [--log-dir ] [--fst-dir ] [--no-wave]"', "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --test [--diff] [--rushB] [--clean-before]"', }, + "bebop-vcs": { + "verilog": 'Generate Bebop VCS RTL once. Args: "--chip "', + "build": 'Build the reusable Bebop VCS simv. Args: "--chip [--jobs ]"', + "batch": 'Run Bebop VCS workload batch. Args: "--chip --test "', + }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--chip [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--chip [--output-dir ]"', From ce235c216a22221ecae0af484fe92c408f739688 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 11 Sep 2026 21:13:22 +0800 Subject: [PATCH 144/196] feat: add support for dynamic bank entries configuration in UVM simulation script --- bbdev/api/steps/uvm/scripts/uvm_common.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index 6e6069d8e..68cd1b681 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -119,9 +119,12 @@ def run_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx, co crate = tomllib.loads((verify_dir / "casegen" / "Cargo.toml").read_text())["package"]["name"] dpi = verify_dir / "casegen" / "target" / "debug" / f"lib{crate.replace('-', '_')}" test = f"{ball}_ball_test" + verify_config = verify_dir / "build" / chip_name / "verify_config.env" + bank_entries = load_chip(bbdir, chip_name).cores[0].mem.bank.entries + verify_config.write_text(f"bank_entries={bank_entries}\nball_id={mapping.ball_id}\n") script = ( f"cd {shlex.quote(str(verify_dir))} && " - f"env LD_LIBRARY_PATH=\"$VCS_RUNTIME_LIBRARY_PATH\" {shlex.quote(str(simv))} " + f"env LD_LIBRARY_PATH=\"$VCS_RUNTIME_LIBRARY_PATH\" BB_VERIFY_CONFIG={shlex.quote(str(verify_config))} {shlex.quote(str(simv))} " f"-sv_lib {shlex.quote(str(dpi))} " f"+UVM_TESTNAME={shlex.quote(test)} +BID={mapping.ball_id} " f"-cm line+cond+tgl+assert -cm_name {shlex.quote(test)}" @@ -195,7 +198,7 @@ def run_chip(bbdir: str, chip: str, ball: str | None, ctx, do_run: bool) -> dict s = dashboard_summary(p) body.append( f"{b} {s['SCORE']} {s['LINE']} {s['COND']} {s['TOGGLE']} " - f"{s.get('ASSERT', '--')} {s['GROUP']} pass" + f"{s.get('ASSERT', '--')} {s.get('GROUP', '--')} pass" ) index = index_dir / "index.txt" index.write_text("\n".join(body) + "\n") From 450869af8de10a6636509326e7923858ecb136f3 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 12 Sep 2026 03:10:00 +0800 Subject: [PATCH 145/196] feat: implement server startup locking mechanism in bbdev and enhance chip emulation handling in batch event step --- .../api/steps/bebop/bemu/03_batch_event.step.py | 8 ++++++-- bbdev/bbdev | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index 7de4c4a9e..93d24b45e 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -25,7 +25,7 @@ from utils.path import bebop_cargo_env, get_buckyball_path, workloads_output_root from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id -from bemu_common import bemu_manifest +from bemu_common import bemu_manifest, chip_emu_manifest from regression import regression_workload_toml from regression_harness import nextest_harness_args @@ -53,8 +53,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + rushB = bool(input_data.get("rushB", False)) try: bemu_cargo_manifest = bemu_manifest(chip, bbdir) + if not rushB: + chip_emu = chip_emu_manifest(chip, bbdir) + if chip_emu is not None and 'name = "test_bemu"' in chip_emu.read_text(encoding="utf-8"): + bemu_cargo_manifest = chip_emu except ValueError as e: ctx.logger.error(str(e)) await check_result( @@ -68,7 +73,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: env = os.environ.copy() env.update(bebop_cargo_env(bbdir, chip)) test_type = input_data.get("test", "elf-tests") - rushB = bool(input_data.get("rushB", False)) try: workload_toml = regression_workload_toml( chip, "bemu", test_type, bbdir, rushB=rushB diff --git a/bbdev/bbdev b/bbdev/bbdev index f93936c61..4e7e86e17 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -2,6 +2,7 @@ import sys import argparse +import fcntl import subprocess import os import shlex @@ -25,6 +26,7 @@ WORKER_PORT_START = 49200 WORKER_PORT_END = 49500 HTTP_PORT_START = 5100 HTTP_PORT_END = 5500 +SERVER_START_LOCK = "/tmp/buckyball-bbdev-server-start.lock" # Subcommand definitions: {name: {operation: help_text, ...}} # Operations with string values accept sub-arguments; True means boolean-only (no sub-args). @@ -159,7 +161,7 @@ def make_iii_config(port: int, worker_port: int, quiet: bool = False, exec_modul def user_preferred_port(start: int, end: int) -> int: size = end - start + 1 - offset = (os.getuid() * 37) % size + offset = (os.getuid() * 37 + os.getpid()) % size return start + offset @@ -605,6 +607,12 @@ def run_script_mode(args, cmd_info): api_path = f"/{command.replace('-', '/')}/{cmd_info['operation']}" + # Serialize only engine startup. The reservation socket must be released before + # iii can bind it, so without this lock another bbdev can select the same port + # during that handoff. + server_start_lock = open(SERVER_START_LOCK, "w") + fcntl.flock(server_start_lock, fcntl.LOCK_EX) + # Reserve both ports atomically to prevent TOCTOU races with concurrent bbdev instances. # Sockets stay bound until the engine has taken over (HTTP is up), then are released. if args.port: @@ -689,6 +697,10 @@ def run_script_mode(args, cmd_info): ): sys.exit(1) + fcntl.flock(server_start_lock, fcntl.LOCK_UN) + server_start_lock.close() + server_start_lock = None + need = motia_worker_count() for _ in range(need): worker_proc = subprocess.Popen( @@ -731,6 +743,9 @@ def run_script_mode(args, cmd_info): sys.exit(rc) finally: + if server_start_lock is not None: + fcntl.flock(server_start_lock, fcntl.LOCK_UN) + server_start_lock.close() # Cleanup: stop iii, its process group, and any workspace worker it spawned. iii_output_active.clear() time.sleep(0.5) From bb22ad8ee39dfeafe52295a87f56ffc29a8c0835 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 12 Sep 2026 03:39:44 +0800 Subject: [PATCH 146/196] fix: update server start lock path in bbdev to use dynamic directory structure --- bbdev/bbdev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbdev/bbdev b/bbdev/bbdev index 4e7e86e17..fafff47e1 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -26,7 +26,7 @@ WORKER_PORT_START = 49200 WORKER_PORT_END = 49500 HTTP_PORT_START = 5100 HTTP_PORT_END = 5500 -SERVER_START_LOCK = "/tmp/buckyball-bbdev-server-start.lock" +SERVER_START_LOCK = os.path.join(workflow_dir, "data", "server-start.lock") # Subcommand definitions: {name: {operation: help_text, ...}} # Operations with string values accept sub-arguments; True means boolean-only (no sub-args). From f79bc12c99c6646eec0ef8c61f0936cd0f0640e2 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 13 Sep 2026 15:33:41 +0800 Subject: [PATCH 147/196] feat: add ball_params mapping to BallIdMapping in proto and update JSON to proto conversion --- bbdev/api/steps/config/scripts/3_json2proto.py | 2 ++ bbdev/api/steps/config/scripts/proto/chip.proto | 1 + 2 files changed, 3 insertions(+) diff --git a/bbdev/api/steps/config/scripts/3_json2proto.py b/bbdev/api/steps/config/scripts/3_json2proto.py index 17fe44086..980a0ef08 100644 --- a/bbdev/api/steps/config/scripts/3_json2proto.py +++ b/bbdev/api/steps/config/scripts/3_json2proto.py @@ -89,6 +89,8 @@ def _fill_ball(msg: pb.BallDomain, d: dict[str, Any], bbdir: Path) -> None: e.ball_class = m["ballClass"] e.ball_dir = _ball_dir(m["ballClass"]) e.config_path = _rel(bbdir, m["config"]["_file"]) + for key, value in m["config"]["ball"].items(): + e.ball_params[key] = str(value) e.in_bw = m["inBW"] e.out_bw = m["outBW"] if "mmioReadBW" in m: diff --git a/bbdev/api/steps/config/scripts/proto/chip.proto b/bbdev/api/steps/config/scripts/proto/chip.proto index 52b71151d..22a3729a1 100644 --- a/bbdev/api/steps/config/scripts/proto/chip.proto +++ b/bbdev/api/steps/config/scripts/proto/chip.proto @@ -68,6 +68,7 @@ message BallIdMapping { uint32 out_bw = 7; uint32 mmio_read_bw = 8; uint32 mmio_write_bw = 9; + map ball_params = 10; } message BallIsaEntry { From 7ac9189f5a629913703a10980de13f9ad621cceb Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 13 Sep 2026 18:01:35 +0800 Subject: [PATCH 148/196] refactor: improve code readability and consistency in bebop regression and batch event steps by standardizing indentation and formatting --- .../steps/bebop/bemu/03_batch_event.step.py | 36 +++++++--- bbdev/api/steps/bebop/regression.py | 68 +++++++++---------- .../bebop/verilator/06_batch_api.step.py | 22 +++--- .../bebop/verilator/06_batch_event.step.py | 49 ++++++++----- bbdev/api/utils/path.py | 4 +- bbdev/mcp/tools/bebop_verilator_batch.py | 4 +- 6 files changed, 111 insertions(+), 72 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index 93d24b45e..0422530f9 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -5,6 +5,7 @@ 1. Build the selected chip's BEMU wrapper 2. Run cargo nextest with bemu-specific config """ + import os import shutil import shlex @@ -29,7 +30,6 @@ from regression import regression_workload_toml from regression_harness import nextest_harness_args - config = { "name": "bebop-bemu-batch", "description": "Run bebop bemu nextest batch regression", @@ -42,13 +42,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - nextest_config = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/nextest.toml" + nextest_config = ( + f"{os.path.dirname(os.path.abspath(__file__))}/scripts/nextest.toml" + ) chip = input_data.get("chip") if not chip: ctx.logger.error("Missing required parameter: chip must be specified") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) @@ -58,12 +62,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bemu_cargo_manifest = bemu_manifest(chip, bbdir) if not rushB: chip_emu = chip_emu_manifest(chip, bbdir) - if chip_emu is not None and 'name = "test_bemu"' in chip_emu.read_text(encoding="utf-8"): + if chip_emu is not None and 'name = "test_bemu"' in chip_emu.read_text( + encoding="utf-8" + ): bemu_cargo_manifest = chip_emu except ValueError as e: ctx.logger.error(str(e)) await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "invalid_chip", "chip": chip}, trace_id=origin_tid, ) @@ -80,8 +88,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: except ValueError as e: ctx.logger.error(str(e)) await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "invalid_regression", "test": test_type, "chip": chip}, + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "invalid_regression", + "test": test_type, + "chip": chip, + }, trace_id=origin_tid, ) return @@ -107,7 +121,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if build_result.returncode != 0: await check_result( - ctx, build_result.returncode, continue_run=False, + ctx, + build_result.returncode, + continue_run=False, extra_fields={"task": "build", "backend": "bemu"}, trace_id=origin_tid, ) @@ -116,7 +132,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ── Run nextest ─────────────────────────────────────────────────────── # Pass test harness parameters through nextest's process environment. if input_data.get("clean-before", input_data.get("clean_before", False)): - shutil.rmtree(bemu_cargo_manifest.parent / "test-artifacts", ignore_errors=True) + shutil.rmtree( + os.path.join(env["CARGO_TARGET_DIR"], "test-artifacts"), ignore_errors=True + ) ctx.logger.info("Cleaned previous bebop test artifacts") harness = nextest_harness_args(workload_toml, elf_root, env) diff --git a/bbdev/api/steps/bebop/regression.py b/bbdev/api/steps/bebop/regression.py index f31dc7632..89108cfd1 100644 --- a/bbdev/api/steps/bebop/regression.py +++ b/bbdev/api/steps/bebop/regression.py @@ -5,44 +5,44 @@ def validate_chip(chip: str | None) -> str: - if not chip: - raise ValueError("missing required parameter: chip") - if not isinstance(chip, str): - raise ValueError("chip must be a string") - if os.path.sep in chip or chip in {".", ".."}: - raise ValueError(f"invalid chip: {chip}") - return chip + if not chip: + raise ValueError("missing required parameter: chip") + if not isinstance(chip, str): + raise ValueError("chip must be a string") + if os.path.sep in chip or chip in {".", ".."}: + raise ValueError(f"invalid chip: {chip}") + return chip def chip_regression_dir(chip: str, backend: str, bbdir: str | None = None) -> Path: - chip = validate_chip(chip) - root = Path(bbdir or get_buckyball_path()) - return root / "examples" / "chips" / chip / "regression" / "batch" / backend + chip = validate_chip(chip) + root = Path(bbdir or get_buckyball_path()) + return root / "examples" / "chips" / chip / "regression" / "batch" / backend def regression_workload_toml( - chip: str, - backend: str, - test_type: str, - bbdir: str | None = None, - rushB: bool = False, - diff: bool = False, + chip: str, + backend: str, + test_type: str, + bbdir: str | None = None, + rushB: bool = False, + diff: bool = False, ) -> str: - if test_type == "elf-tests": - suffix = "elf" - elif test_type == "pk-tests": - suffix = "pk" - else: - raise ValueError(f"invalid test type: {test_type}") - - regression_dir = chip_regression_dir(chip, backend, bbdir) - if diff: - diff_toml = regression_dir / f"workloads-{suffix}-diff.toml" - if diff_toml.is_file(): - return str(diff_toml) - - name = f"workloads-{suffix}-rushB.toml" if rushB else f"workloads-{suffix}.toml" - toml = regression_dir / name - if not toml.is_file(): - raise ValueError(f"regression workload toml does not exist: {toml}") - return str(toml) + if test_type == "elf-tests": + suffix = "elf" + elif test_type == "pk-tests": + suffix = "pk" + else: + raise ValueError(f"invalid test type: {test_type}") + + regression_dir = chip_regression_dir(chip, backend, bbdir) + if diff: + diff_toml = regression_dir / f"workloads-{suffix}-diff.toml" + if diff_toml.is_file(): + return str(diff_toml) + + name = f"workloads-{suffix}.toml" + toml = regression_dir / name + if not toml.is_file(): + raise ValueError(f"regression workload toml does not exist: {toml}") + return str(toml) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py index d2b3b7ea8..9ebc6e40f 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -21,30 +21,32 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) + diff = bool(body.get("diff", False)) + rushB = bool(body.get("rushB", False)) test_type = body.get("test") if not test_type: return ApiResponse( status=400, - body={"error": "Missing required parameter: --test must be specified (elf-tests or pk-tests)"}, + body={ + "error": "Missing required parameter: --test must be specified (elf-tests)" + }, ) - if test_type not in ["elf-tests", "pk-tests"]: + if test_type != "elf-tests" and not (rushB and test_type == "pk-tests"): return ApiResponse( status=400, - body={"error": f"Invalid test type: {test_type}. Must be 'elf-tests' or 'pk-tests'"}, + body={ + "error": f"Invalid test type: {test_type}. Must be 'elf-tests', or 'pk-tests' with --rushB" + }, ) - diff = bool(body.get("diff", False)) - rushB = bool(body.get("rushB", False)) if diff and rushB: return ApiResponse( status=400, body={"error": "--diff and --rushB cannot be used together"}, ) - vsrc_dir = rtl_dir( - bbdir, chip, "verilog", body.get("vsrc_dir"), rushb=rushB - ) + vsrc_dir = rtl_dir(bbdir, chip, "verilog", body.get("vsrc_dir"), rushb=rushB) data = { "chip": chip, @@ -54,5 +56,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "rushB": rushB, "diff": diff, } - await ctx.enqueue({"topic": "bebop.verilator.batch", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + {"topic": "bebop.verilator.batch", "data": {**data, "_trace_id": ctx.trace_id}} + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 2175811f6..77244dc76 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -3,6 +3,7 @@ Runs bebop verilator nextest batch regression (requires prior --build). """ + import os import shutil import shlex @@ -19,13 +20,17 @@ sys.path.insert(0, bebop_path) from utils.event_common import require_chip -from utils.path import bebop_cargo_env, get_buckyball_path, rtl_dir, workloads_output_root +from utils.path import ( + bebop_cargo_env, + get_buckyball_path, + rtl_dir, + workloads_output_root, +) from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from regression import regression_workload_toml from regression_harness import nextest_harness_args - config = { "name": "bebop-verilator-batch", "description": "Run bebop verilator nextest batch regression", @@ -39,14 +44,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" - nextest_config = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/nextest.toml" + nextest_config = ( + f"{os.path.dirname(os.path.abspath(__file__))}/scripts/nextest.toml" + ) try: chip = require_chip(input_data) except ValueError as error: ctx.logger.error(str(error)) await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) @@ -74,8 +83,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: except ValueError as e: ctx.logger.error(str(e)) await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "invalid_regression", "test": test_type, "chip": chip}, + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "invalid_regression", + "test": test_type, + "chip": chip, + }, trace_id=origin_tid, ) return @@ -95,10 +110,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: vsrc_config = shlex.quote(f"env.VSRC_PATH='{vsrc_dir}'") env = os.environ.copy() env.update(bebop_cargo_env(bbdir, chip)) - env.update({ - "BEBOP_ARCH_CONFIG": chip, - "VSRC_PATH": vsrc_dir, - }) + env.update( + { + "BEBOP_ARCH_CONFIG": chip, + "VSRC_PATH": vsrc_dir, + } + ) manifest = f"{bebop_dir}/Cargo.toml" features = "verilator" @@ -112,14 +129,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) if path ) - env.update({ - "BEBOP_VERILATOR_DIFF": "1", - "BEBOP_DIFF_LD_PRELOAD": preload, - "BEBOP_DIFF_RUN_DIR": os.path.dirname(manifest), - }) + env["LD_PRELOAD"] = preload if input_data.get("clean-before", input_data.get("clean_before", False)): - artifact_dir = os.path.join(os.path.dirname(manifest), "test-artifacts") + artifact_dir = os.path.join(env["CARGO_TARGET_DIR"], "test-artifacts") shutil.rmtree(artifact_dir, ignore_errors=True) ctx.logger.info(f"Cleaned previous bebop test artifacts: {artifact_dir}") @@ -148,7 +161,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "batch", - "backend": "verilator-rushB" if rushB else ("difftest" if diff else "verilator"), + "backend": ( + "verilator-rushB" if rushB else ("difftest" if diff else "verilator") + ), "chip": chip, "vsrc_dir": vsrc_dir, "test_type": test_type, diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index cd29c9695..d9121a04f 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -8,7 +8,9 @@ def get_buckyball_path(): current_dir = os.path.dirname(__file__) # bbdev/api/utils -> bbdev/api -> bbdev -> buckyball - inferred = os.path.realpath(os.path.dirname(os.path.dirname(os.path.dirname(current_dir)))) + inferred = os.path.realpath( + os.path.dirname(os.path.dirname(os.path.dirname(current_dir))) + ) root = os.environ.get("BUCKYBALL_ROOT") if root: diff --git a/bbdev/mcp/tools/bebop_verilator_batch.py b/bbdev/mcp/tools/bebop_verilator_batch.py index 507ee051d..8b9aad923 100644 --- a/bbdev/mcp/tools/bebop_verilator_batch.py +++ b/bbdev/mcp/tools/bebop_verilator_batch.py @@ -18,8 +18,8 @@ def bbdev_bebop_verilator_batch( """Batch bebop-verilator regression. POST /bebop/verilator/batch.""" if e := need("chip", chip): return err(e) - if test not in ("elf-tests", "pk-tests"): - return err("test must be elf-tests or pk-tests") + if test != "elf-tests" and not (rushB and test == "pk-tests"): + return err("test must be elf-tests, or pk-tests with rushB") return fmt( submit( "/bebop/verilator/batch", From d278e4527d56de20d5e8d359eaf306045aa16efa Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 14 Sep 2026 03:46:10 +0800 Subject: [PATCH 149/196] feat: update runworkload command to include chip parameter and improve image resolution logic --- .../bebop/p2e/03_runworkload_event.step.py | 35 ++++++++++--------- .../02_eval_performance_event.step.py | 9 ++--- .../tests/test_api_bebop_p2e_runworkload.py | 2 +- bbdev/bbdev | 2 +- bbdev/mcp/tools/bebop_p2e_runworkload.py | 6 ++-- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index e39e2e0bd..f9b91afd1 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -11,7 +11,6 @@ import re import sys from datetime import datetime -from pathlib import Path from motia import FlowContext, queue @@ -34,11 +33,26 @@ def resolve_image(bbdir: str, image_name: str, chip: str) -> str: - """Resolve a kernel image name to its deterministic output path.""" + """Resolve one workload hex image for the selected chip.""" image_name = image_name.replace(r"\_", "_") - filename = image_name if image_name.endswith(".hex") else f"{image_name}.hex" - path = Path(bbdir) / "bb-tests" / "output" / "kernel" / chip / filename - return str(path) if path.is_file() else "" + matches = glob.glob( + os.path.join( + bbdir, + "bb-tests", + "output", + chip, + "workloads", + "**", + f"{image_name}.hex", + ), + recursive=True, + ) + if len(matches) != 1: + raise ValueError( + f"expected one workload hex for {image_name!r} under " + f"bb-tests/output/{chip}/workloads/, found {len(matches)}" + ) + return matches[0] def resolve_runtime_config(bitstream: str, requested_config: object) -> str: @@ -107,17 +121,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: wave = True image_path = resolve_image(bbdir, image_name, chip) - if not image_path: - ctx.logger.error( - f"image .hex not found for name: {image_name} " - f"(expected bb-tests/output/kernel/{chip}/)" - ) - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "image_not_found", "image": image_name}, - trace_id=origin_tid, - ) - return if not bitstream or not os.path.isfile(bitstream): ctx.logger.error(f"bitstream .bit file not found: {bitstream}") diff --git a/bbdev/api/steps/regression/02_eval_performance_event.step.py b/bbdev/api/steps/regression/02_eval_performance_event.step.py index 75b723f7f..d2db347e9 100644 --- a/bbdev/api/steps/regression/02_eval_performance_event.step.py +++ b/bbdev/api/steps/regression/02_eval_performance_event.step.py @@ -119,12 +119,9 @@ def _kernel_build_cmds(bbdir, chip, model, dataset=""): def _p2e_run_cmds(bbdir, bitstream, image_name, chip, input_data): - image_path = _p2e.resolve_image(bbdir, image_name, chip) - if not image_path: - raise FileNotFoundError( - f"image .hex not found for name: {image_name} " - f"(expected bb-tests/output/kernel/{chip}/)" - ) + image_path = os.path.join( + bbdir, "bb-tests", "output", "kernel", chip, f"{image_name}.hex" + ) bitstream = os.path.abspath(bitstream) build_dir = os.path.dirname(os.path.dirname(bitstream)) if not os.path.isdir(build_dir): diff --git a/bbdev/api/tests/test_api_bebop_p2e_runworkload.py b/bbdev/api/tests/test_api_bebop_p2e_runworkload.py index 2e2263698..1d1b26d85 100644 --- a/bbdev/api/tests/test_api_bebop_p2e_runworkload.py +++ b/bbdev/api/tests/test_api_bebop_p2e_runworkload.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-p2e --runworkload '--image vecunit_matmul_ones --bitstream /home/wanghui/Code/buckyball/bebop/build/p2e/sims.p2e.P2EToyConfig/bitstream.bit'") +run_bbdev_case("bbdev bebop-p2e --runworkload '--chip toy --image vecunit_matmul_ones --bitstream /home/wanghui/Code/buckyball/bebop/build/p2e/sims.p2e.P2EToyConfig/bitstream.bit'") diff --git a/bbdev/bbdev b/bbdev/bbdev index fafff47e1..f1e218bc2 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -85,7 +85,7 @@ WORKFLOW_COMMANDS = { "clean": 'Clean Bebop P2E build directory. Args: "--chip [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--chip [--output-dir ]"', "buildbitstream": 'Build Bebop P2E runtime case. Args: "--chip [--vsrc-dir ] [--output-dir ]"', - "runworkload": 'Run Bebop P2E workload. Args: "--image --bitstream [--multi-fpga] [--wave] [--wave-start ] [--itrace] [--mtrace] [--pmctrace]"', + "runworkload": 'Run Bebop P2E workload. Args: "--chip --image --bitstream [--multi-fpga] [--wave] [--wave-start ] [--itrace] [--mtrace] [--pmctrace]"', "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--chip --bitstream --test "', }, "dc": { diff --git a/bbdev/mcp/tools/bebop_p2e_runworkload.py b/bbdev/mcp/tools/bebop_p2e_runworkload.py index e658f949e..4ccd2a951 100644 --- a/bbdev/mcp/tools/bebop_p2e_runworkload.py +++ b/bbdev/mcp/tools/bebop_p2e_runworkload.py @@ -10,6 +10,7 @@ def register(mcp): @mcp.tool() def bbdev_bebop_p2e_runworkload( + chip: str, image: str, bitstream: str, itrace: bool = False, @@ -17,10 +18,10 @@ def bbdev_bebop_p2e_runworkload( pmctrace: bool = False, ) -> str: """Run one image on bebop-p2e FPGA. POST /bebop/p2e/runworkload.""" - for n, v in (("image", image), ("bitstream", bitstream)): + for n, v in (("chip", chip), ("image", image), ("bitstream", bitstream)): if e := need(n, v): return err(e) - params: Dict[str, Any] = {"image": image, "bitstream": bitstream} + params: Dict[str, Any] = {"chip": chip, "image": image, "bitstream": bitstream} if itrace: params["itrace"] = True if mtrace: @@ -28,4 +29,3 @@ def bbdev_bebop_p2e_runworkload( if pmctrace: params["pmctrace"] = True return fmt(submit("/bebop/p2e/runworkload", params)) - From cd95ebdd10d6a4877adc16ed2028e70db51eb2d7 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 14 Sep 2026 15:12:19 +0800 Subject: [PATCH 150/196] fix: modularize image resolution logic by moving it to a new script and update workload event step to utilize the new function --- .../bebop/p2e/03_runworkload_event.step.py | 27 +++-------------- .../steps/bebop/p2e/scripts/resolve_image.py | 30 +++++++++++++++++++ bbdev/api/steps/uvm/scripts/uvm_common.py | 23 +++++++++++--- 3 files changed, 53 insertions(+), 27 deletions(-) create mode 100644 bbdev/api/steps/bebop/p2e/scripts/resolve_image.py diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index f9b91afd1..7f0a9c098 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -17,11 +17,15 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) from utils.event_common import require_chip from utils.path import bebop_cargo_env, get_buckyball_path, log_dir, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id +from resolve_image import resolve_image config = { "name": "bebop-p2e-runworkload", @@ -32,29 +36,6 @@ } -def resolve_image(bbdir: str, image_name: str, chip: str) -> str: - """Resolve one workload hex image for the selected chip.""" - image_name = image_name.replace(r"\_", "_") - matches = glob.glob( - os.path.join( - bbdir, - "bb-tests", - "output", - chip, - "workloads", - "**", - f"{image_name}.hex", - ), - recursive=True, - ) - if len(matches) != 1: - raise ValueError( - f"expected one workload hex for {image_name!r} under " - f"bb-tests/output/{chip}/workloads/, found {len(matches)}" - ) - return matches[0] - - def resolve_runtime_config(bitstream: str, requested_config: object) -> str: if isinstance(requested_config, str) and requested_config: return requested_config diff --git a/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py b/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py new file mode 100644 index 000000000..e6c59b92b --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py @@ -0,0 +1,30 @@ +import glob +import os + + +def resolve_image(bbdir: str, image_name: str, chip: str) -> str: + image_name = image_name.replace(r"\_", "_") + matches = glob.glob( + os.path.join( + bbdir, + "bb-tests", + "output", + chip, + "workloads", + "**", + f"{image_name}.hex", + ), + recursive=True, + ) + kernel_image = os.path.join( + bbdir, "bb-tests", "output", "kernel", chip, f"{image_name}.hex" + ) + if os.path.isfile(kernel_image): + matches.append(kernel_image) + if len(matches) != 1: + raise ValueError( + f"expected one image for {image_name!r} under " + f"bb-tests/output/{chip}/workloads/ or bb-tests/output/kernel/{chip}/, " + f"found {len(matches)}" + ) + return matches[0] diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index 68cd1b681..7534f8522 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -1,4 +1,5 @@ import os +import re import shlex import sys import tomllib @@ -47,12 +48,13 @@ def selected_mappings(domain, ball: str | None): raise ValueError(f"ball {ball!r} not in chip.pb") -def vcs_defines(domain, mapping): +def vcs_defines(domain, mapping, bank_entries: int): defs = [ f"+define+BB_IN_BW={mapping.in_bw}", f"+define+BB_OUT_BW={mapping.out_bw}", f"+define+BB_MMIO_READ_BW={mapping.mmio_read_bw}", f"+define+BB_MMIO_WRITE_BW={mapping.mmio_write_bw}", + f"+define+BB_BANK_ADDR_W={(bank_entries - 1).bit_length()}", ] for e in domain.isa: if e.bid == mapping.ball_id: @@ -60,11 +62,23 @@ def vcs_defines(domain, mapping): return defs -def _filelist(verify_dir: Path, ball_dir: str, uvm_rel: str, rtl_rel: str, sim_dir: Path) -> str: +def smatmul_accumulator_filename(rtl_dir: Path) -> str: + text = (rtl_dir / "SMatMulUnit.sv").read_text() + modules = re.findall(r"\b(accumulator_\d+x\d+)\s+accumulator_ext\s*\(", text) + if len(modules) != 1: + raise ValueError(f"expected one SMatMul accumulator instance, found {modules!r}") + return f"{modules[0]}.sv" + + +def _filelist( + verify_dir: Path, ball_dir: str, uvm_rel: str, rtl_rel: str, rtl_dir: Path, sim_dir: Path +) -> str: src = verify_dir / "filelists" / f"{ball_dir}_ball.f" dst = sim_dir / f"{ball_dir}_ball.f" sim_dir.mkdir(parents=True, exist_ok=True) text = src.read_text() + if "@SMATMUL_ACCUMULATOR@" in text: + text = text.replace("@SMATMUL_ACCUMULATOR@", smatmul_accumulator_filename(rtl_dir)) dst.write_text(text.replace("@UVM@", uvm_rel).replace("@RTL@", rtl_rel)) return str(dst.relative_to(verify_dir)) @@ -74,10 +88,11 @@ def build_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx) verify_dir = Path(bbdir) / "examples" / "balls" / ball / "verify" casegen = verify_dir / "casegen" / "Cargo.toml" rtl_dir = Path(bbdir) / "arch" / "build" / chip_name / mill_cfg + bank_entries = load_chip(bbdir, chip_name).cores[0].mem.bank.entries sim_dir = verify_dir / "build" / chip_name uvm_rel = os.path.relpath(Path(bbdir) / "verify" / "uvm", verify_dir) rtl_rel = os.path.relpath(rtl_dir, verify_dir) - flist = _filelist(verify_dir, ball, uvm_rel, rtl_rel, sim_dir) + flist = _filelist(verify_dir, ball, uvm_rel, rtl_rel, rtl_dir, sim_dir) cargo = ( f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " f"cargo build --manifest-path {shlex.quote(str(casegen))}" @@ -99,7 +114,7 @@ def build_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx) f"mkdir -p {shlex.quote(str(sim_dir))} {shlex.quote(str(csrc))} && " "vcs -full64 -sverilog -timescale=1ns/1ps -debug_access+all " "${=VCS_UVM_ARGS} " - + " ".join(shlex.quote(d) for d in vcs_defines(domain, mapping)) + + " ".join(shlex.quote(d) for d in vcs_defines(domain, mapping, bank_entries)) + f" -cm line+cond+tgl+assert -cm_hier {shlex.quote(str(hier))} " f"-Mdir={shlex.quote(str(csrc))} -o {shlex.quote(str(simv))} " f"-f {shlex.quote(flist)}" From 1226a0e23f7a6f25606808489d1ce000d0835240 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 14 Sep 2026 15:32:27 +0800 Subject: [PATCH 151/196] fix: re-import firesim step --- bbdev/api/steps/firesim/.gitignore | 2 +- .../firesim/00_enumeratefpgas_api.step.py | 13 +- .../firesim/00_enumeratefpgas_event.step.py | 60 ++++--- .../firesim/01_buildbitstream_api.step.py | 13 +- .../firesim/01_buildbitstream_event.step.py | 60 +++---- .../steps/firesim/02_infrasetup_api.step.py | 14 +- .../steps/firesim/02_infrasetup_event.step.py | 67 ++++--- .../steps/firesim/03_runworkload_api.step.py | 12 +- .../firesim/03_runworkload_event.step.py | 67 ++++--- bbdev/api/steps/firesim/README.md | 61 ++----- bbdev/api/steps/firesim/scripts/chip_yaml.py | 168 ++++++++++++++++++ .../firesim/scripts/yaml/config_build.yaml | 44 ----- .../scripts/yaml/config_build_recipes.yaml | 64 ------- .../firesim/scripts/yaml/config_hwdb.yaml | 48 ----- .../firesim/scripts/yaml/config_runtime.yaml | 110 ------------ bbdev/api/steps/uvm/scripts/uvm_common.py | 17 +- .../tests/test_api_firesim_buildbitstream.py | 2 +- .../tests/test_api_firesim_enumeratefpgas.py | 2 +- .../api/tests/test_api_firesim_infrasetup.py | 2 +- .../api/tests/test_api_firesim_runworkload.py | 2 +- bbdev/api/utils/path.py | 9 +- bbdev/bbdev | 8 +- bbdev/mcp/tools/firesim_buildbitstream.py | 10 +- bbdev/mcp/tools/firesim_enumeratefpgas.py | 10 +- bbdev/mcp/tools/firesim_infrasetup.py | 10 +- bbdev/mcp/tools/firesim_runworkload.py | 10 +- 26 files changed, 409 insertions(+), 476 deletions(-) create mode 100644 bbdev/api/steps/firesim/scripts/chip_yaml.py delete mode 100644 bbdev/api/steps/firesim/scripts/yaml/config_build.yaml delete mode 100644 bbdev/api/steps/firesim/scripts/yaml/config_build_recipes.yaml delete mode 100644 bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml delete mode 100644 bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml diff --git a/bbdev/api/steps/firesim/.gitignore b/bbdev/api/steps/firesim/.gitignore index 0a8db21da..2a6160516 100644 --- a/bbdev/api/steps/firesim/.gitignore +++ b/bbdev/api/steps/firesim/.gitignore @@ -1 +1 @@ -# *.yaml +*.yaml \ No newline at end of file diff --git a/bbdev/api/steps/firesim/00_enumeratefpgas_api.step.py b/bbdev/api/steps/firesim/00_enumeratefpgas_api.step.py index 9a88bfcc1..da11565ad 100644 --- a/bbdev/api/steps/firesim/00_enumeratefpgas_api.step.py +++ b/bbdev/api/steps/firesim/00_enumeratefpgas_api.step.py @@ -1,5 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.event_common import require_chip + config = { "name": "firesim-enumeratefpgas-api", "description": "enumerate FPGAs", @@ -11,5 +13,14 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - await ctx.enqueue({"topic": "firesim.enumeratefpgas", "data": {**body, "_trace_id": ctx.trace_id}}) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + await ctx.enqueue( + { + "topic": "firesim.enumeratefpgas", + "data": {"chip": chip, "_trace_id": ctx.trace_id}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py b/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py index e1b56a6a6..d05a9fce1 100644 --- a/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py +++ b/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py @@ -3,18 +3,20 @@ from motia import FlowContext, queue -# Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path +from utils.path import get_buckyball_path, sim_name from utils.stream_run import stream_run_logger_async -from utils.event_common import check_result, get_origin_trace_id +from utils.event_common import check_result, get_origin_trace_id, require_chip + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) -# Import firesim_env from scripts subdirectory -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "scripts")) from firesim_env import setup_firesim_env +from chip_yaml import firesim_cmd, prepare_yamls config = { "name": "firesim-enumeratefpgas", @@ -28,36 +30,36 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - script_dir = f"{bbdir}/bbdev/api/steps/firesim/scripts" - yaml_dir = f"{script_dir}/yaml" - # Setup FireSim environment variables and SSH agent - env = setup_firesim_env() + try: + chip = require_chip(input_data) + target_config = sim_name(bbdir, chip, "firesim") + yaml_dir = prepare_yamls(bbdir, chip, target_config) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": str(error)}, + trace_id=origin_tid, + ) + return - # ================================================================================== - # Execute operation - # ================================================================================== - command = f"firesim enumeratefpgas" - command += f" -a {yaml_dir}/config_hwdb.yaml" - command += f" -b {yaml_dir}/config_build.yaml" - command += f" -r {yaml_dir}/config_build_recipes.yaml" - command += f" -c {yaml_dir}/config_runtime.yaml" + env = setup_firesim_env() + ctx.logger.info(f"firesim enumeratefpgas chip={chip}") result = await stream_run_logger_async( - cmd=command, + cmd=firesim_cmd("enumeratefpgas", yaml_dir), logger=ctx.logger, stdout_prefix="firesim enumeratefpgas", stderr_prefix="firesim enumeratefpgas", env=env, ) - # ================================================================================== - # Return result to API - # ================================================================================== - success_result, failure_result = await check_result( - ctx, result.returncode, continue_run=False, trace_id=origin_tid) - - # ================================================================================== - # Continue routing - # ================================================================================== - - return + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={"chip": chip, "yaml_dir": yaml_dir}, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/firesim/01_buildbitstream_api.step.py b/bbdev/api/steps/firesim/01_buildbitstream_api.step.py index 0b69bdcdf..106f7e7e3 100644 --- a/bbdev/api/steps/firesim/01_buildbitstream_api.step.py +++ b/bbdev/api/steps/firesim/01_buildbitstream_api.step.py @@ -1,5 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.event_common import require_chip + config = { "name": "firesim-buildbitstream-api", "description": "build bitstream", @@ -11,5 +13,14 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - await ctx.enqueue({"topic": "firesim.buildbitstream", "data": {**body, "_trace_id": ctx.trace_id}}) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + await ctx.enqueue( + { + "topic": "firesim.buildbitstream", + "data": {"chip": chip, "_trace_id": ctx.trace_id}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/01_buildbitstream_event.step.py b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py index be857b5ce..eecd2e726 100644 --- a/bbdev/api/steps/firesim/01_buildbitstream_event.step.py +++ b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py @@ -3,21 +3,20 @@ from motia import FlowContext, queue -# Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path +from utils.path import get_buckyball_path, sim_name from utils.stream_run import stream_run_logger_async -from utils.event_common import check_result, get_origin_trace_id +from utils.event_common import check_result, get_origin_trace_id, require_chip -# Add scripts directory to path for firesim_env import scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) from firesim_env import setup_firesim_env +from chip_yaml import firesim_cmd, prepare_yamls config = { "name": "firesim-buildbitstream", @@ -31,53 +30,50 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - script_dir = f"{bbdir}/bbdev/api/steps/firesim/scripts" - yaml_dir = f"{script_dir}/yaml" - # Setup FireSim environment variables and SSH agent + try: + chip = require_chip(input_data) + target_config = sim_name(bbdir, chip, "firesim") + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": str(error)}, + trace_id=origin_tid, + ) + return + + yaml_dir = prepare_yamls(bbdir, chip, target_config) env = setup_firesim_env() - # ================================================================================== - # Fix AU280 board_part version (1.2 -> 1.0) for Vivado compatibility - # ================================================================================== au280_tcl = f"{bbdir}/thirdparty/firesim/platforms/xilinx_alveo_u280/cl_firesim/scripts/au280.tcl" if os.path.exists(au280_tcl): with open(au280_tcl, "r") as f: content = f.read() - # Replace xilinx.com:au280:part0:1.2 with xilinx.com:au280:part0:1.0 fixed_content = content.replace( "xilinx.com:au280:part0:1.2", - "xilinx.com:au280:part0:1.0" + "xilinx.com:au280:part0:1.0", ) if content != fixed_content: with open(au280_tcl, "w") as f: f.write(fixed_content) ctx.logger.info("Fixed AU280 board_part version: 1.2 -> 1.0") - # ================================================================================== - # Execute operation - # ================================================================================== - command = f"firesim buildbitstream " - command += f" -a {yaml_dir}/config_hwdb.yaml" - command += f" -b {yaml_dir}/config_build.yaml" - command += f" -r {yaml_dir}/config_build_recipes.yaml" - command += f" -c {yaml_dir}/config_runtime.yaml" + ctx.logger.info(f"firesim buildbitstream chip={chip} TARGET_CONFIG={target_config}") result = await stream_run_logger_async( - cmd=command, + cmd=firesim_cmd("buildbitstream", yaml_dir), logger=ctx.logger, stdout_prefix="firesim buildbitstream", stderr_prefix="firesim buildbitstream", env=env, ) - # ================================================================================== - # Return result to API - # ================================================================================== - success_result, failure_result = await check_result( - ctx, result.returncode, continue_run=False, trace_id=origin_tid) - - # ================================================================================== - # Continue routing - # ================================================================================== - - return + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={"chip": chip, "target_config": target_config, "yaml_dir": yaml_dir}, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/firesim/02_infrasetup_api.step.py b/bbdev/api/steps/firesim/02_infrasetup_api.step.py index 5beca75ca..b1f87b6ba 100644 --- a/bbdev/api/steps/firesim/02_infrasetup_api.step.py +++ b/bbdev/api/steps/firesim/02_infrasetup_api.step.py @@ -1,5 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.event_common import require_chip + config = { "name": "firesim-infrasetup-api", "description": "infrasetup", @@ -11,6 +13,14 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - data = {"jobs": body.get("jobs", 16)} - await ctx.enqueue({"topic": "firesim.infrasetup", "data": {**data, "_trace_id": ctx.trace_id}}) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + await ctx.enqueue( + { + "topic": "firesim.infrasetup", + "data": {"chip": chip, "_trace_id": ctx.trace_id}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/02_infrasetup_event.step.py b/bbdev/api/steps/firesim/02_infrasetup_event.step.py index d122ad760..55ba00c5a 100644 --- a/bbdev/api/steps/firesim/02_infrasetup_event.step.py +++ b/bbdev/api/steps/firesim/02_infrasetup_event.step.py @@ -3,18 +3,20 @@ from motia import FlowContext, queue -# Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path +from utils.path import get_buckyball_path, sim_name from utils.stream_run import stream_run_logger_async -from utils.event_common import check_result, get_origin_trace_id +from utils.event_common import check_result, get_origin_trace_id, require_chip + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) -# Import firesim_env from scripts subdirectory -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "scripts")) from firesim_env import setup_firesim_env +from chip_yaml import firesim_cmd, prepare_yamls, recipe_name config = { "name": "firesim-infrasetup", @@ -28,36 +30,43 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - script_dir = f"{bbdir}/bbdev/api/steps/firesim/scripts" - yaml_dir = f"{script_dir}/yaml" - # Setup FireSim environment variables and SSH agent - env = setup_firesim_env() + try: + chip = require_chip(input_data) + target_config = sim_name(bbdir, chip, "firesim") + yaml_dir = prepare_yamls(bbdir, chip, target_config) + hwdb = os.path.join(yaml_dir, "config_hwdb.yaml") + text = open(hwdb, encoding="utf-8").read() + if "bitstream_tar: null" in text or "bitstream_tar: file://" not in text: + raise ValueError( + f"no bitstream for recipe {recipe_name(chip)}; " + f"run bbdev firesim --buildbitstream '--chip {chip}' first" + ) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": str(error)}, + trace_id=origin_tid, + ) + return - # ================================================================================== - # Execute operation - # ================================================================================== - command = f"firesim infrasetup " - command += f" -a {yaml_dir}/config_hwdb.yaml" - command += f" -b {yaml_dir}/config_build.yaml" - command += f" -r {yaml_dir}/config_build_recipes.yaml" - command += f" -c {yaml_dir}/config_runtime.yaml" + env = setup_firesim_env() + ctx.logger.info(f"firesim infrasetup chip={chip} TARGET_CONFIG={target_config}") result = await stream_run_logger_async( - cmd=command, + cmd=firesim_cmd("infrasetup", yaml_dir), logger=ctx.logger, stdout_prefix="firesim infrasetup", stderr_prefix="firesim infrasetup", env=env, ) - # ================================================================================== - # Return result to API - # ================================================================================== - success_result, failure_result = await check_result( - ctx, result.returncode, continue_run=False, trace_id=origin_tid) - - # ================================================================================== - # Continue routing - # ================================================================================== - - return + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={"chip": chip, "target_config": target_config, "yaml_dir": yaml_dir}, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/firesim/03_runworkload_api.step.py b/bbdev/api/steps/firesim/03_runworkload_api.step.py index de1db067b..36f9ff897 100644 --- a/bbdev/api/steps/firesim/03_runworkload_api.step.py +++ b/bbdev/api/steps/firesim/03_runworkload_api.step.py @@ -1,5 +1,7 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.event_common import require_chip + config = { "name": "firesim-runworkload-api", "description": "run workload", @@ -11,6 +13,12 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - data = {"jobs": body.get("jobs", 16)} - await ctx.enqueue({"topic": "firesim.runworkload", "data": {**data, "_trace_id": ctx.trace_id}}) + try: + chip = require_chip(body) + except ValueError as e: + return ApiResponse(status=400, body={"error": str(e)}) + data = {"chip": chip, "_trace_id": ctx.trace_id} + if body.get("jobs") is not None: + data["jobs"] = body["jobs"] + await ctx.enqueue({"topic": "firesim.runworkload", "data": data}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/03_runworkload_event.step.py b/bbdev/api/steps/firesim/03_runworkload_event.step.py index d9fe92da4..3edec98db 100644 --- a/bbdev/api/steps/firesim/03_runworkload_event.step.py +++ b/bbdev/api/steps/firesim/03_runworkload_event.step.py @@ -3,18 +3,20 @@ from motia import FlowContext, queue -# Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path +from utils.path import get_buckyball_path, sim_name from utils.stream_run import stream_run_logger_async -from utils.event_common import check_result, get_origin_trace_id +from utils.event_common import check_result, get_origin_trace_id, require_chip + +scripts_path = os.path.join(os.path.dirname(__file__), "scripts") +if scripts_path not in sys.path: + sys.path.insert(0, scripts_path) -# Import firesim_env from scripts subdirectory -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "scripts")) from firesim_env import setup_firesim_env +from chip_yaml import firesim_cmd, prepare_yamls, recipe_name config = { "name": "firesim-runworkload", @@ -28,36 +30,43 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - script_dir = f"{bbdir}/bbdev/api/steps/firesim/scripts" - yaml_dir = f"{script_dir}/yaml" - # Setup FireSim environment variables and SSH agent - env = setup_firesim_env() + try: + chip = require_chip(input_data) + target_config = sim_name(bbdir, chip, "firesim") + yaml_dir = prepare_yamls(bbdir, chip, target_config) + hwdb = os.path.join(yaml_dir, "config_hwdb.yaml") + text = open(hwdb, encoding="utf-8").read() + if "bitstream_tar: null" in text or "bitstream_tar: file://" not in text: + raise ValueError( + f"no bitstream for recipe {recipe_name(chip)}; " + f"run bbdev firesim --buildbitstream '--chip {chip}' first" + ) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": str(error)}, + trace_id=origin_tid, + ) + return - # ================================================================================== - # Execute operation - # ================================================================================== - command = f"firesim runworkload " - command += f" -a {yaml_dir}/config_hwdb.yaml" - command += f" -b {yaml_dir}/config_build.yaml" - command += f" -r {yaml_dir}/config_build_recipes.yaml" - command += f" -c {yaml_dir}/config_runtime.yaml" + env = setup_firesim_env() + ctx.logger.info(f"firesim runworkload chip={chip} TARGET_CONFIG={target_config}") result = await stream_run_logger_async( - cmd=command, + cmd=firesim_cmd("runworkload", yaml_dir), logger=ctx.logger, stdout_prefix="firesim runworkload", stderr_prefix="firesim runworkload", env=env, ) - # ================================================================================== - # Return result to API - # ================================================================================== - success_result, failure_result = await check_result( - ctx, result.returncode, continue_run=False, trace_id=origin_tid) - - # ================================================================================== - # Continue routing - # ================================================================================== - - return + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={"chip": chip, "target_config": target_config, "yaml_dir": yaml_dir}, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/firesim/README.md b/bbdev/api/steps/firesim/README.md index e3cc4737e..8a9a69ea2 100644 --- a/bbdev/api/steps/firesim/README.md +++ b/bbdev/api/steps/firesim/README.md @@ -1,59 +1,18 @@ # FireSim Workflow -FireSim FPGA simulation workflow in Buckyball framework, providing FPGA-based hardware simulation environment. +FireSim FPGA simulation. All commands take `--chip`; mill class comes from +`examples/chips//configs/chip.toml` `[sims].firesim`. -## API Usage +Manager YAMLs are generated per chip under `scripts/yaml//`. -### `buildbitstream` -**Endpoint**: `POST /firesim/buildbitstream` +## Commands -**Function**: Build FPGA bitstream file - -**Parameters**: No specific parameters - -**Example**: ```bash -bbdev firesim --buildbitstream +bbdev firesim --enumeratefpgas '--chip toy' +bbdev firesim --buildbitstream '--chip toy' +bbdev firesim --infrasetup '--chip toy' +bbdev firesim --runworkload '--chip toy' ``` -### `infrasetup` -**Endpoint**: `POST /firesim/infrasetup` - -**Function**: Setup FireSim infrastructure - -**Parameters**: No specific parameters - -**Example**: -```bash -bbdev firesim --infrasetup -``` - -### `runworkload` -**Endpoint**: `POST /firesim/runworkload` - -**Function**: Run workload on FireSim - -**Parameters**: No specific parameters - -**Example**: -```bash -bbdev firesim --runworkload -``` - -## Typical Workflow - -```bash -# 1. Build bitstream -bbdev firesim --buildbitstream - -# 2. Setup infrastructure -bbdev firesim --infrasetup - -# 3. Run workload -bbdev firesim --runworkload -``` - -## Notes - -- Bitstream build takes several hours -- infrasetup requires cloud computing resource configuration +`buildbitstream` takes hours. `infrasetup` / `runworkload` require a prior +bitstream under `thirdparty/firesim/deploy/results-build` for that chip recipe. diff --git a/bbdev/api/steps/firesim/scripts/chip_yaml.py b/bbdev/api/steps/firesim/scripts/chip_yaml.py new file mode 100644 index 000000000..52b30c179 --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/chip_yaml.py @@ -0,0 +1,168 @@ +"""Prepare per-chip FireSim manager YAMLs from chip sims.firesim.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Optional + + +def recipe_name(chip: str) -> str: + return f"alveo_u280_firesim_{chip}_no_nic" + + +def _latest_bitstream(results: Path, recipe: str) -> Optional[Path]: + if not results.is_dir(): + return None + dirs = sorted( + (p for p in results.iterdir() if p.is_dir() and recipe in p.name), + key=lambda p: p.stat().st_mtime, + ) + for d in reversed(dirs): + matches = list(d.rglob("firesim.tar.gz")) + if matches: + return matches[0] + return None + + +def prepare_yamls(bbdir: str, chip: str, target_config: str) -> str: + """Write chip-scoped firesim yamls under scripts/yaml/. Return that dir.""" + if not target_config or "." not in target_config: + raise ValueError(f"invalid firesim TARGET_CONFIG: {target_config!r}") + recipe = recipe_name(chip) + script_dir = Path(bbdir) / "bbdev" / "api" / "steps" / "firesim" / "scripts" + out = script_dir / "yaml" / chip + out.mkdir(parents=True, exist_ok=True) + + deploy = Path(bbdir) / "thirdparty" / "firesim" / "deploy" + makefrag = script_dir / "makefrag" / "firesim" + if not makefrag.is_dir(): + raise FileNotFoundError(f"missing firesim makefrag: {makefrag}") + + build_dir = deploy / "FIRESIM_BUILD_DIR" + runs_dir = deploy / "FIRESIM_RUNS_DIR" + build_dir.mkdir(parents=True, exist_ok=True) + runs_dir.mkdir(parents=True, exist_ok=True) + + bitstream = _latest_bitstream(deploy / "results-build", recipe) + if bitstream is None: + bitstream_line = ( + " # bitstream_tar filled after buildbitstream; required for infrasetup/runworkload\n" + " bitstream_tar: null" + ) + else: + bitstream_line = f" bitstream_tar: file://{bitstream.resolve()}" + + (out / "config_build_recipes.yaml").write_text( + f"""# Auto-generated for chip={chip}. Do not edit by hand. +{recipe}: + PLATFORM: xilinx_alveo_u280 + TARGET_PROJECT: firesim + TARGET_PROJECT_MAKEFRAG: {makefrag} + DESIGN: FireSim + TARGET_CONFIG: {target_config} + PLATFORM_CONFIG: BaseXilinxAlveoU280Config + deploy_quintuplet: null + platform_config_args: + fpga_frequency: 30 + build_strategy: TIMING + post_build_hook: null + metasim_customruntimeconfig: null + bit_builder_recipe: bit-builder-recipes/xilinx_alveo_u280.yaml +""", + encoding="utf-8", + ) + + (out / "config_build.yaml").write_text( + f"""# Auto-generated for chip={chip}. Do not edit by hand. +build_farm: + base_recipe: build-farm-recipes/externally_provisioned.yaml + recipe_arg_overrides: + default_build_dir: {build_dir} + build_farm_hosts: + - localhost + +builds_to_run: + - {recipe} + +agfis_to_share: [] + +share_with_accounts: {{}} +""", + encoding="utf-8", + ) + + (out / "config_runtime.yaml").write_text( + f"""# Auto-generated for chip={chip}. Do not edit by hand. +run_farm: + base_recipe: run-farm-recipes/externally_provisioned.yaml + recipe_arg_overrides: + default_platform: XilinxAlveoU280InstanceDeployManager + default_simulation_dir: {runs_dir} + default_fpga_db: /opt/firesim-db.json + run_farm_hosts_to_use: + - localhost: one_fpgas_spec + +metasimulation: + metasimulation_enabled: false + metasimulation_host_simulator: verilator + metasimulation_only_plusargs: "+fesvr-step-size=128 +max-cycles=100000000" + metasimulation_only_vcs_plusargs: "+vcs+initreg+0 +vcs+initmem+0" + +target_config: + topology: no_net_config + no_net_num_nodes: 1 + link_latency: 6405 + switching_latency: 10 + net_bandwidth: 200 + profile_interval: -1 + default_hw_config: {recipe} + plusarg_passthrough: "" + +tracing: + enable: no + output_format: 0 + selector: 1 + start: 0 + end: -1 + +autocounter: + read_rate: 0 + +workload: + workload_name: interactive.json + terminate_on_completion: no + suffix_tag: null + +host_debug: + zero_out_dram: no + disable_synth_asserts: no + +synth_print: + start: 0 + end: -1 + cycle_prefix: yes +""", + encoding="utf-8", + ) + + (out / "config_hwdb.yaml").write_text( + f"""# Auto-generated for chip={chip}. bitstream_tar updated by buildbitstream / results-build scan. +{recipe}: +{bitstream_line} + deploy_quintuplet_override: null + custom_runtime_config: null +""", + encoding="utf-8", + ) + + return str(out) + + +def firesim_cmd(action: str, yaml_dir: str) -> str: + return ( + f"firesim {action}" + f" -a {yaml_dir}/config_hwdb.yaml" + f" -b {yaml_dir}/config_build.yaml" + f" -r {yaml_dir}/config_build_recipes.yaml" + f" -c {yaml_dir}/config_runtime.yaml" + ) diff --git a/bbdev/api/steps/firesim/scripts/yaml/config_build.yaml b/bbdev/api/steps/firesim/scripts/yaml/config_build.yaml deleted file mode 100644 index da98a5c66..000000000 --- a/bbdev/api/steps/firesim/scripts/yaml/config_build.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# Build-time build design / AGFI configuration for the FireSim Simulation Manager -# See https://docs.fires.im/en/stable/Advanced-Usage/Manager/Manager-Configuration-Files.html for documentation of all of these params. - -# ------------------------------------------------------------ -# Attention! This is the example on the Local U280 Machine -# Remember to modify this file use in the same way of firesim -# ------------------------------------------------------------ - -# this refers to build farms defined in config_build_farm.yaml -build_farm: - base_recipe: build-farm-recipes/externally_provisioned.yaml - recipe_arg_overrides: - # REQUIRED: (replace this) default location of build directory on build host. - default_build_dir: /home/mio/Code/buckyball/thirdparty/firesim/deploy/FIRESIM_BUILD_DIR - # REQUIRED: List of IP addresses (or "localhost"). Each can have an OPTIONAL - # argument, called "override_build_dir", specifying to override the default - # build directory. - # - # Ex: - # build_farm_hosts: - # # use localhost and don't override the default build dir - # - localhost - # # use other IP address (don't override default build dir) - # - "111.111.1.111" - # # use other IP address (override default build dir for this build host) - # - "222.222.2.222": - # override_build_dir: /scratch/specific-build-host-build-dir - build_farm_hosts: - - localhost - -builds_to_run: - # this section references builds defined in config_build_recipes.yaml - # if you add a build here, it will be built when you run buildbitstream - - alveo_u280_firesim_BuckyballToyConfig_no_nic - # - alveo_u280_firesim_BuckyballToyConfig_no_nic - -agfis_to_share: - # - midasexamples_gcd - -share_with_accounts: - # To share with a specific user: - somebodysname: 123456789012 - # To share publicly: - # public: public diff --git a/bbdev/api/steps/firesim/scripts/yaml/config_build_recipes.yaml b/bbdev/api/steps/firesim/scripts/yaml/config_build_recipes.yaml deleted file mode 100644 index 01d8afb4f..000000000 --- a/bbdev/api/steps/firesim/scripts/yaml/config_build_recipes.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# Build-time build recipe configuration for the FireSim Simulation Manager -# See https://docs.fires.im/en/stable/Advanced-Usage/Manager/Manager-Configuration-Files.html for documentation of all of these params. - -# this file contains sections that describe hardware designs that /can/ be built. -# edit config_build.yaml to actually "turn on" a config to be built when you run -# buildbitstream - -# ------------------------------------------------------------ -# Attention! This is the example on the Local U280 Machine -# Remember to modify this file use in the same way of firesim -# ------------------------------------------------------------ - -########### -# Schema: -########### -# : -# PLATFORM: -# TARGET_PROJECT: -# TARGET_PROJECT_MAKEFRAG: -# DESIGN: -# TARGET_CONFIG: -# PLATFORM_CONFIG: -# deploy_quintuplet: -# platform_config_args: -# fpga_frequency: -# build_strategy: -# post_build_hook: -# metasim_customruntimeconfig: -# bit_builder_recipe: -# # OPTIONAL: overrides for bit builder recipe -# # Arg structure should be identical to the args given -# # in the base_recipe. -# #bit_builder_arg_overrides: -# # : - -alveo_u280_firesim_GemminiBuckyballConfig_no_nic: - PLATFORM: xilinx_alveo_u280 - TARGET_PROJECT: firesim - TARGET_PROJECT_MAKEFRAG: ../makefrag/firesim - DESIGN: FireSim - TARGET_CONFIG: sims.firesim.FireSimGemminiBuckyballConfig - PLATFORM_CONFIG: BaseXilinxAlveoU280Config - deploy_quintuplet: null - platform_config_args: - fpga_frequency: 30 - build_strategy: TIMING - post_build_hook: null - metasim_customruntimeconfig: null - bit_builder_recipe: bit-builder-recipes/xilinx_alveo_u280.yaml - -alveo_u280_firesim_BuckyballToyConfig_no_nic: - PLATFORM: xilinx_alveo_u280 - TARGET_PROJECT: firesim - TARGET_PROJECT_MAKEFRAG: ../makefrag/firesim - DESIGN: FireSim - TARGET_CONFIG: sims.firesim.FireSimBuckyballToyConfig - PLATFORM_CONFIG: BaseXilinxAlveoU280Config - deploy_quintuplet: null - platform_config_args: - fpga_frequency: 30 - build_strategy: TIMING - post_build_hook: null - metasim_customruntimeconfig: null - bit_builder_recipe: bit-builder-recipes/xilinx_alveo_u280.yaml diff --git a/bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml b/bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml deleted file mode 100644 index b411d547e..000000000 --- a/bbdev/api/steps/firesim/scripts/yaml/config_hwdb.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# Hardware config database for FireSim Simulation Manager -# See https://docs.fires.im/en/stable/Advanced-Usage/Manager/Manager-Configuration-Files.html for documentation of all of these params. - -# ------------------------------------------------------------ -# Attention! This is the example on the Local U280 Machine -# Remember to modify this file use in the same way of firesim -# ------------------------------------------------------------ - -# Hardware configs represent a combination of: -# - an agfi/bitstream_tar with the bitstream for an fpga -# - (optional) a deployquintuplet override -# - (optional) a deploy_makefrag_override -# - (optional) a custom_runtime_config - -# The HWDBs (and their AGFI's/bitstream_tar's) provided below are public and available to all users. - -# DOCREF START: Example HWDB Entry -midasexamples_gcd: - bitstream_tar: https://raw.githubusercontent.com/invalid/address.tar.gz - deploy_quintuplet_override: null - deploy_makefrag_override: null - custom_runtime_config: null -# DOCREF END: Example HWDB Entry - -# alveo_u280_firesim_GemminiBuckyballConfig_no_nic: -# bitstream_tar: file:///home/mio/Code/buckyball/thirdparty/firesim/deploy/results-build/2025-09-24--17-27-06-alveo_u280_firesim_GemminiBuckyballConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimGemminiBuckyballConfig-BaseXilinxAlveoU280Config/firesim.tar.gz -# deploy_quintuplet_override: null -# custom_runtime_config: null - -alveo_u280_firesim_GemminiBuckyballConfig_no_nic: - bitstream_tar: file:///home/mio/Code/buckyball/thirdparty/firesim/deploy/results-build/2025-11-19--17-01-23-alveo_u280_firesim_GemminiBuckyballConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimGemminiBuckyballConfig-BaseXilinxAlveoU280Config/firesim.tar.gz - deploy_quintuplet_override: null - custom_runtime_config: null - -# alveo_u280_firesim_BuckyballToyConfig_no_nic: -# bitstream_tar: file:///home/mio/Code/buckyball/thirdparty/firesim/deploy/results-build/2025-09-29--19-18-46-alveo_u280_firesim_BuckyballToyConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimBuckyballToyConfig-BaseXilinxAlveoU280Config/firesim.tar.gz -# deploy_quintuplet_override: null -# custom_runtime_config: null - -alveo_u280_firesim_BuckyballToyConfig_no_nic: - bitstream_tar: file:///home/mio/Code/buckyball/thirdparty/firesim/deploy/results-build/2026-05-12--15-52-50-alveo_u280_firesim_BuckyballToyConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimBuckyballToyConfig-BaseXilinxAlveoU280Config/firesim.tar.gz - deploy_quintuplet_override: null - custom_runtime_config: null - -alveo_u280_firesim_GemminiBuckyballConfig_no_nic: - bitstream_tar: file:///home/mio/Code/buckyball/thirdparty/firesim/deploy/results-build/2026-05-12--19-18-24-alveo_u280_firesim_GemminiBuckyballConfig_no_nic/cl_xilinx_alveo_u280-firesim-FireSim-sims.firesim.FireSimGemminiBuckyballConfig-BaseXilinxAlveoU280Config/firesim.tar.gz - deploy_quintuplet_override: null - custom_runtime_config: null diff --git a/bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml b/bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml deleted file mode 100644 index 56ecca649..000000000 --- a/bbdev/api/steps/firesim/scripts/yaml/config_runtime.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# Run-time configuration for the FireSim Simulation Manager -# See https://docs.fires.im/en/stable/Advanced-Usage/Manager/Manager-Configuration-Files.html for documentation of all of these params. - -# ------------------------------------------------------------ -# Attention! This is the example on the Local U280 Machine -# Remember to modify this file use in the same way of firesim -# ------------------------------------------------------------ - -run_farm: - base_recipe: run-farm-recipes/externally_provisioned.yaml - recipe_arg_overrides: - # REQUIRED: default platform used for run farm hosts. this is a class specifying - # how to run simulations on a run farm host. - default_platform: XilinxAlveoU280InstanceDeployManager - - # REQUIRED: default directory where simulations are run out of on the run farm hosts - default_simulation_dir: /home/mio/Code/buckyball/thirdparty/firesim/deploy/FIRESIM_RUNS_DIR - - # REQUIRED: default fpga db file that enumerates what fpgas are available on the machine (used by XilinxU* Deploy Managers) - default_fpga_db: /opt/firesim-db.json - - # REQUIRED: List of unique hostnames/IP addresses, each with their - # corresponding specification that describes the properties of the host. - # - # Ex: - # run_farm_hosts_to_use: - # # use localhost which is described by "four_fpgas_spec" below. - # - localhost: four_fpgas_spec - # # supply IP address, which points to a machine that is described - # # by "four_fpgas_spec" below. - # - "111.111.1.111": four_fpgas_spec - run_farm_hosts_to_use: - - localhost: one_fpgas_spec - -metasimulation: - metasimulation_enabled: false - # vcs or verilator. use vcs-debug or verilator-debug for waveform generation - metasimulation_host_simulator: verilator - # plusargs passed to the simulator for all metasimulations - metasimulation_only_plusargs: "+fesvr-step-size=128 +max-cycles=100000000" - # plusargs passed to the simulator ONLY FOR vcs metasimulations - metasimulation_only_vcs_plusargs: "+vcs+initreg+0 +vcs+initmem+0" - -# DOCREF START: target_config area -target_config: - topology: no_net_config - no_net_num_nodes: 1 - link_latency: 6405 - switching_latency: 10 - net_bandwidth: 200 - profile_interval: -1 - - # This references a section from config_hwdb.yaml for fpga-accelerated simulation - # or from config_build_recipes.yaml for metasimulation - # In homogeneous configurations, use this to set the hardware config deployed - # for all simulators - default_hw_config: alveo_u280_firesim_BuckyballToyConfig_no_nic - - # Advanced: Specify any extra plusargs you would like to provide when - # booting the simulator (in both FPGA-sim and metasim modes). This is - # a string, with the contents formatted as if you were passing the plusargs - # at command line, e.g. "+a=1 +b=2" - plusarg_passthrough: "" -# DOCREF END: target_config area - -tracing: - enable: no - - # Trace output formats. Only enabled if "enable" is set to "yes" above - # 0 = human readable; 1 = binary (compressed raw data); 2 = flamegraph (stack - # unwinding -> Flame Graph) - output_format: 0 - - # Trigger selector. - # 0 = no trigger; 1 = cycle count trigger; 2 = program counter trigger; 3 = - # instruction trigger - selector: 1 - start: 0 - end: -1 - -autocounter: - read_rate: 0 - -workload: - workload_name: interactive.json # br-base.json - terminate_on_completion: no - suffix_tag: null - -host_debug: - # When enabled (=yes), Zeros-out FPGA-attached DRAM before simulations - # begin (takes 2-5 minutes). - # In general, this is not required to produce deterministic simulations on - # target machines running linux. Enable if you observe simulation non-determinism. - zero_out_dram: no - # If disable_synth_asserts: no, simulation will print assertion message and - # terminate simulation if synthesized assertion fires. - # If disable_synth_asserts: yes, simulation ignores assertion firing and - # continues simulation. - disable_synth_asserts: no - -# DOCREF START: Synthesized Prints -synth_print: - # Start and end cycles for outputting synthesized prints. - # They are given in terms of the base clock and will be converted - # for each clock domain. - start: 0 - end: -1 - # When enabled (=yes), prefix print output with the target cycle at which the print was triggered - cycle_prefix: yes -# DOCREF END: Synthesized Prints diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index 7534f8522..d29f03a08 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -9,17 +9,26 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -config_scripts = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "config", "scripts")) -if config_scripts not in sys.path: - sys.path.insert(0, config_scripts) -import chip_pb2 from utils.path import get_buckyball_path, log_dir from utils.stream_run import stream_run_logger def load_chip(bbdir: str, chip: str): + config_scripts = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..", "config", "scripts") + ) + if config_scripts not in sys.path: + sys.path.insert(0, config_scripts) + try: + import chip_pb2 + except ImportError as e: + raise FileNotFoundError( + f"missing {os.path.join(config_scripts, 'chip_pb2.py')}; run bbdev config --install" + ) from e path = Path(bbdir) / "examples" / "chips" / chip / "configs" / "generated" / "chip.pb" + if not path.is_file(): + raise FileNotFoundError(f"missing {path}; run bbdev config --install") msg = chip_pb2.Chip() msg.ParseFromString(path.read_bytes()) if not msg.name or not msg.cores: diff --git a/bbdev/api/tests/test_api_firesim_buildbitstream.py b/bbdev/api/tests/test_api_firesim_buildbitstream.py index 4b7ceaa5d..3edefe465 100644 --- a/bbdev/api/tests/test_api_firesim_buildbitstream.py +++ b/bbdev/api/tests/test_api_firesim_buildbitstream.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev firesim --buildbitstream") +run_bbdev_case("bbdev firesim --buildbitstream '--chip toy'") diff --git a/bbdev/api/tests/test_api_firesim_enumeratefpgas.py b/bbdev/api/tests/test_api_firesim_enumeratefpgas.py index fd80b9226..337070c26 100644 --- a/bbdev/api/tests/test_api_firesim_enumeratefpgas.py +++ b/bbdev/api/tests/test_api_firesim_enumeratefpgas.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev firesim --enumeratefpgas") +run_bbdev_case("bbdev firesim --enumeratefpgas '--chip toy'") diff --git a/bbdev/api/tests/test_api_firesim_infrasetup.py b/bbdev/api/tests/test_api_firesim_infrasetup.py index 4e46ee0c8..7a619caa5 100644 --- a/bbdev/api/tests/test_api_firesim_infrasetup.py +++ b/bbdev/api/tests/test_api_firesim_infrasetup.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev firesim --infrasetup") +run_bbdev_case("bbdev firesim --infrasetup '--chip toy'") diff --git a/bbdev/api/tests/test_api_firesim_runworkload.py b/bbdev/api/tests/test_api_firesim_runworkload.py index 8ebf19820..e9c941513 100644 --- a/bbdev/api/tests/test_api_firesim_runworkload.py +++ b/bbdev/api/tests/test_api_firesim_runworkload.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev firesim --runworkload") +run_bbdev_case("bbdev firesim --runworkload '--chip toy'") diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index d9121a04f..e55646acb 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -78,6 +78,8 @@ def sim_name(bbdir, chip, product, *, rushb=False): sim_key = "tapeout" elif product == "p2e": sim_key = "p2e" + elif product == "firesim": + sim_key = "firesim" else: raise ValueError(f"invalid rtl product: {product}") path = ( @@ -90,7 +92,12 @@ def sim_name(bbdir, chip, product, *, rushb=False): / "config" / "config.json" ) - name = json.loads(path.read_text(encoding="utf-8"))["sims"][sim_key] + if not path.is_file(): + raise ValueError(f"missing {path}; run bbdev config --install") + sims = json.loads(path.read_text(encoding="utf-8"))["sims"] + if sim_key not in sims: + raise ValueError(f"chip {chip!r} has no sims.{sim_key} in {path}") + name = sims[sim_key] if not rushb: return name if product != "verilog" or not name.endswith("VerilatorConfig"): diff --git a/bbdev/bbdev b/bbdev/bbdev index f1e218bc2..98a4090b9 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -55,10 +55,10 @@ WORKFLOW_COMMANDS = { "build": 'Build RISC-V kernel + rootfs. Args: "[--chip ] [--model ] [--interactive] [--visible-hart-count ] [--total-hart-count ]"; --model requires --chip', }, "firesim": { - "enumeratefpgas": "Enumerate FPGAs.", - "buildbitstream": "Build bitstream.", - "infrasetup": "Infrasetup.", - "runworkload": "Run workload.", + "enumeratefpgas": 'Enumerate FPGAs. Args: "--chip "', + "buildbitstream": 'Build bitstream. Args: "--chip "', + "infrasetup": 'Infrasetup. Args: "--chip "', + "runworkload": 'Run workload. Args: "--chip "', }, "compiler": { "build": 'Build compiler. Args: "--chip "', diff --git a/bbdev/mcp/tools/firesim_buildbitstream.py b/bbdev/mcp/tools/firesim_buildbitstream.py index 68d818b14..eaced3979 100644 --- a/bbdev/mcp/tools/firesim_buildbitstream.py +++ b/bbdev/mcp/tools/firesim_buildbitstream.py @@ -2,13 +2,13 @@ from __future__ import annotations - -from common import submit, fmt, opt +from common import submit, err, fmt, need def register(mcp): @mcp.tool() - def bbdev_firesim_buildbitstream() -> str: + def bbdev_firesim_buildbitstream(chip: str) -> str: """Build FireSim bitstream. POST /firesim/buildbitstream.""" - return fmt(submit("/firesim/buildbitstream", {})) - + if e := need("chip", chip): + return err(e) + return fmt(submit("/firesim/buildbitstream", {"chip": chip})) diff --git a/bbdev/mcp/tools/firesim_enumeratefpgas.py b/bbdev/mcp/tools/firesim_enumeratefpgas.py index 4d1ca5468..66a060ea8 100644 --- a/bbdev/mcp/tools/firesim_enumeratefpgas.py +++ b/bbdev/mcp/tools/firesim_enumeratefpgas.py @@ -2,13 +2,13 @@ from __future__ import annotations - -from common import submit, fmt, opt +from common import submit, err, fmt, need def register(mcp): @mcp.tool() - def bbdev_firesim_enumeratefpgas() -> str: + def bbdev_firesim_enumeratefpgas(chip: str) -> str: """Enumerate FireSim FPGAs. POST /firesim/enumeratefpgas.""" - return fmt(submit("/firesim/enumeratefpgas", {})) - + if e := need("chip", chip): + return err(e) + return fmt(submit("/firesim/enumeratefpgas", {"chip": chip})) diff --git a/bbdev/mcp/tools/firesim_infrasetup.py b/bbdev/mcp/tools/firesim_infrasetup.py index b008e72b3..d6caca6fb 100644 --- a/bbdev/mcp/tools/firesim_infrasetup.py +++ b/bbdev/mcp/tools/firesim_infrasetup.py @@ -2,13 +2,13 @@ from __future__ import annotations - -from common import submit, fmt, opt +from common import submit, err, fmt, need def register(mcp): @mcp.tool() - def bbdev_firesim_infrasetup(jobs: int = 16) -> str: + def bbdev_firesim_infrasetup(chip: str) -> str: """FireSim infrasetup. POST /firesim/infrasetup.""" - return fmt(submit("/firesim/infrasetup", {"jobs": jobs})) - + if e := need("chip", chip): + return err(e) + return fmt(submit("/firesim/infrasetup", {"chip": chip})) diff --git a/bbdev/mcp/tools/firesim_runworkload.py b/bbdev/mcp/tools/firesim_runworkload.py index 005f13340..2c4ef4d1b 100644 --- a/bbdev/mcp/tools/firesim_runworkload.py +++ b/bbdev/mcp/tools/firesim_runworkload.py @@ -2,13 +2,13 @@ from __future__ import annotations - -from common import submit, fmt, opt +from common import submit, err, fmt, need def register(mcp): @mcp.tool() - def bbdev_firesim_runworkload(jobs: int = 16) -> str: + def bbdev_firesim_runworkload(chip: str) -> str: """Run FireSim workload. POST /firesim/runworkload.""" - return fmt(submit("/firesim/runworkload", {"jobs": jobs})) - + if e := need("chip", chip): + return err(e) + return fmt(submit("/firesim/runworkload", {"chip": chip})) From ea81545f3c56140e2138d7c1205fb68502afebbb Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 14 Sep 2026 15:28:44 +0800 Subject: [PATCH 152/196] refactor: remove unused smatmul_accumulator_filename function and simplify _filelist parameters in uvm_common.py --- bbdev/api/steps/uvm/scripts/uvm_common.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index d29f03a08..e40ee9a1c 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -1,5 +1,4 @@ import os -import re import shlex import sys import tomllib @@ -71,23 +70,13 @@ def vcs_defines(domain, mapping, bank_entries: int): return defs -def smatmul_accumulator_filename(rtl_dir: Path) -> str: - text = (rtl_dir / "SMatMulUnit.sv").read_text() - modules = re.findall(r"\b(accumulator_\d+x\d+)\s+accumulator_ext\s*\(", text) - if len(modules) != 1: - raise ValueError(f"expected one SMatMul accumulator instance, found {modules!r}") - return f"{modules[0]}.sv" - - def _filelist( - verify_dir: Path, ball_dir: str, uvm_rel: str, rtl_rel: str, rtl_dir: Path, sim_dir: Path + verify_dir: Path, ball_dir: str, uvm_rel: str, rtl_rel: str, sim_dir: Path ) -> str: src = verify_dir / "filelists" / f"{ball_dir}_ball.f" dst = sim_dir / f"{ball_dir}_ball.f" sim_dir.mkdir(parents=True, exist_ok=True) text = src.read_text() - if "@SMATMUL_ACCUMULATOR@" in text: - text = text.replace("@SMATMUL_ACCUMULATOR@", smatmul_accumulator_filename(rtl_dir)) dst.write_text(text.replace("@UVM@", uvm_rel).replace("@RTL@", rtl_rel)) return str(dst.relative_to(verify_dir)) @@ -101,7 +90,7 @@ def build_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx) sim_dir = verify_dir / "build" / chip_name uvm_rel = os.path.relpath(Path(bbdir) / "verify" / "uvm", verify_dir) rtl_rel = os.path.relpath(rtl_dir, verify_dir) - flist = _filelist(verify_dir, ball, uvm_rel, rtl_rel, rtl_dir, sim_dir) + flist = _filelist(verify_dir, ball, uvm_rel, rtl_rel, sim_dir) cargo = ( f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " f"cargo build --manifest-path {shlex.quote(str(casegen))}" From 274eb16c9bf18bb32e81369b455202eba3b762ec Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 14 Sep 2026 15:48:03 +0800 Subject: [PATCH 153/196] fix: enhance subprocess handling in load_host_environment by adding stdin and start_new_session parameters --- bbdev/bbdev | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bbdev/bbdev b/bbdev/bbdev index 98a4090b9..1319b372f 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -179,6 +179,8 @@ def load_host_environment() -> None: ["zsh", "-lic", "env -0"], check=True, capture_output=True, + stdin=subprocess.DEVNULL, + start_new_session=True, ) for item in result.stdout.split(b"\0"): if item: From 62da7d402ec235a76ec800498bd34afaab6a4c63 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 14 Sep 2026 16:19:56 +0800 Subject: [PATCH 154/196] fix: improve error detail extraction in subprocess handling by capturing relevant output lines on failure --- bbdev/api/steps/compiler/scripts/build.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bbdev/api/steps/compiler/scripts/build.py b/bbdev/api/steps/compiler/scripts/build.py index 05b55555c..75373a38b 100644 --- a/bbdev/api/steps/compiler/scripts/build.py +++ b/bbdev/api/steps/compiler/scripts/build.py @@ -35,7 +35,13 @@ def _run( if result.returncode != 0: detail = result.stderr.strip() if logger is not None else "" if not detail and logger is not None: - detail = "\n".join(result.stdout.splitlines()[-20:]) + lines = result.stdout.splitlines() + failed = [i for i, line in enumerate(lines) if "FAILED:" in line] + if failed: + start = failed[0] + detail = "\n".join(lines[start:]) + else: + detail = "\n".join(lines[-20:]) if detail: raise RuntimeError( f"command failed ({result.returncode}): {' '.join(cmd)}\n{detail}" From 469ecc22581d82a6aaca193cf42e016ec38d7da9 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 16 Sep 2026 15:30:49 +0800 Subject: [PATCH 155/196] fix: update runworkload command to include fpga-location parameter and improve error handling in workload event step --- .../api/steps/bebop/bemu/02_sim_event.step.py | 4 +++- .../bebop/p2e/02_buildbitstream_api.step.py | 6 +++-- .../bebop/p2e/03_runworkload_event.step.py | 24 +++++++++++++++++-- .../steps/bebop/p2e/scripts/resolve_image.py | 4 ++++ bbdev/bbdev | 2 +- bbdev/mcp/tools/bebop_p2e_buildbitstream.py | 6 ++++- 6 files changed, 39 insertions(+), 7 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 704b6cb73..d5a1f5a80 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -133,7 +133,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return ctx.logger.info(f"binary_path: {binary_path}") binary_dir = os.path.dirname(binary_path) - perfetto_target = PERFETTO_TARGETS.get(binary_name) + perfetto_target = ( + PERFETTO_TARGETS.get(binary_name) if input_data.get("tool-profile") else None + ) if perfetto_target: clean_model_trace(binary_dir) diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py index b024d8cc6..61dc9572b 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py @@ -19,12 +19,14 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: chip = require_chip(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - vsrc_dir = rtl_dir(bbdir, chip, "p2e", body.get("vsrc_dir")) + vsrc_dir = rtl_dir( + bbdir, chip, "p2e", body.get("vsrc_dir") or body.get("vsrc-dir") + ) data = { "chip": chip, "vsrc_dir": vsrc_dir, - "output_dir": body.get("output_dir"), + "output_dir": body.get("output_dir") or body.get("output-dir"), } await ctx.enqueue({ "topic": "bebop.p2e.buildbitstream", diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index 7f0a9c098..f1ab863f6 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -69,6 +69,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: image_name = input_data.get("image", "") bitstream = input_data.get("bitstream", "") multi_fpga = bool(input_data.get("multi-fpga", False)) + fpga_location = input_data.get("fpga-location") or input_data.get("fpga_location") or "0.A" + if not isinstance(fpga_location, str) or not re.fullmatch(r"[0-3]\.[A-E]", fpga_location): + ctx.logger.error(f"invalid FPGA location: {fpga_location}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_fpga_location", "fpga_location": fpga_location}, + trace_id=origin_tid, + ) + return wave = bool(input_data.get("wave", False)) if "wave_start" in input_data: ctx.logger.error("invalid parameter: --wave_start (use --wave-start)") @@ -101,7 +110,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return wave = True - image_path = resolve_image(bbdir, image_name, chip) + try: + image_path = resolve_image(bbdir, image_name, chip) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "image_not_found", "image": image_name}, + trace_id=origin_tid, + ) + return if not bitstream or not os.path.isfile(bitstream): ctx.logger.error(f"bitstream .bit file not found: {bitstream}") @@ -181,7 +199,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"\"{bebop_p2e_path}\" run p2e " f"--image=\"{image_path}\" " f"--bitstream=\"{bitstream}\" " - f"--log-dir=\"{run_log}\"" + f"--log-dir=\"{run_log}\" " + f"--fpga-location=\"{fpga_location}\"" ) if multi_fpga: run_cmd += " --multi-fpga" @@ -219,6 +238,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "bitstream": bitstream, "build_dir": build_dir, "log_dir": run_log, + "fpga_location": fpga_location, "bdb_trace": os.path.join(run_log, "bdb.ndjson"), "uart_log": os.path.join(run_log, "uart.log"), "timestamp": timestamp, diff --git a/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py b/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py index e6c59b92b..db0c18264 100644 --- a/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py +++ b/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py @@ -4,6 +4,10 @@ def resolve_image(bbdir: str, image_name: str, chip: str) -> str: image_name = image_name.replace(r"\_", "_") + if os.path.isabs(image_name): + raise ValueError( + f"image must be a logical name, not an absolute path: {image_name!r}" + ) matches = glob.glob( os.path.join( bbdir, diff --git a/bbdev/bbdev b/bbdev/bbdev index 1319b372f..58bef3883 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -85,7 +85,7 @@ WORKFLOW_COMMANDS = { "clean": 'Clean Bebop P2E build directory. Args: "--chip [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--chip [--output-dir ]"', "buildbitstream": 'Build Bebop P2E runtime case. Args: "--chip [--vsrc-dir ] [--output-dir ]"', - "runworkload": 'Run Bebop P2E workload. Args: "--chip --image --bitstream [--multi-fpga] [--wave] [--wave-start ] [--itrace] [--mtrace] [--pmctrace]"', + "runworkload": 'Run Bebop P2E workload. Args: "--chip --image --bitstream [--fpga-location ] [--multi-fpga] [--wave] [--wave-start ] [--itrace] [--mtrace] [--pmctrace]"', "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--chip --bitstream --test "', }, "dc": { diff --git a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py index faacd4172..ec0094499 100644 --- a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py +++ b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py @@ -20,6 +20,10 @@ def bbdev_bebop_p2e_buildbitstream( return fmt( submit( "/bebop/p2e/buildbitstream", - opt({"chip": chip}, vsrc_dir=vsrc_dir, output_dir=output_dir), + opt( + {"chip": chip}, + vsrc_dir=vsrc_dir, + output_dir=output_dir, + ), ) ) From 8cc740c63a27327a588f38e72e21ef97ccab949f Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 16 Sep 2026 15:55:00 +0800 Subject: [PATCH 156/196] fix: add trace-config parameter to workload build commands and improve error handling for trace configuration --- bbdev/api/steps/workload/01_buidl_api.step.py | 11 +++++++++- .../api/steps/workload/01_build_event.step.py | 21 ++++++++++++++++++- bbdev/bbdev | 2 +- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/bbdev/api/steps/workload/01_buidl_api.step.py b/bbdev/api/steps/workload/01_buidl_api.step.py index c998aca14..39f72af3f 100644 --- a/bbdev/api/steps/workload/01_buidl_api.step.py +++ b/bbdev/api/steps/workload/01_buidl_api.step.py @@ -22,7 +22,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - allowed = {"chip", "model", "stable", "rushB", "ctest", "mlirtest"} + allowed = {"chip", "model", "stable", "rushB", "ctest", "mlirtest", "trace-config"} unknown = sorted(k for k in body if k not in allowed) if unknown: return ApiResponse( @@ -52,6 +52,9 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: status=400, body={"error": "Invalid parameter: stable must be a boolean flag"}, ) + trace_config = body.get("trace-config", "") + if not isinstance(trace_config, str) or (trace_config and not os.path.isabs(trace_config)): + return ApiResponse(status=400, body={"error": "Invalid trace config"}) rushb_backend = body.get("rushB") if rushb_backend is not None and rushb_backend not in {"bemu", "verilator"}: return ApiResponse( @@ -80,6 +83,11 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: status=400, body={"error": "--ctest and --mlirtest cannot be used with --rushB"}, ) + if trace_config and not body.get("model"): + return ApiResponse( + status=400, + body={"error": "--trace-config requires --model"}, + ) data = { "chip": chip, "model": body.get("model", ""), @@ -87,6 +95,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "rushB": rushb_backend, "ctest": ctest, "mlirtest": mlirtest, + "trace-config": trace_config, } await ctx.enqueue({"topic": "workload.build", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index 385c3684d..d9d8d4784 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -76,7 +76,7 @@ def chips_for_model(bbdir: str, model_key: str) -> set[str]: async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - allowed = {"chip", "model", "stable", "rushB", "ctest", "mlirtest", "_trace_id"} + allowed = {"chip", "model", "stable", "rushB", "ctest", "mlirtest", "trace-config", "_trace_id"} unknown = sorted(k for k in input_data if k not in allowed) if unknown: ctx.logger.error(f"Unknown workload build parameter(s): {', '.join(unknown)}") @@ -113,6 +113,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return model = input_data.get("model", "") + trace_config = input_data.get("trace-config", "") stable = input_data.get("stable", False) rushb_backend = input_data.get("rushB") @@ -124,6 +125,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + if not isinstance(trace_config, str) or (trace_config and not os.path.isabs(trace_config)): + ctx.logger.error("Invalid trace config") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "invalid_trace_config", "trace_config": trace_config}, + trace_id=origin_tid, + ) + return if rushb_backend is not None and rushb_backend not in {"bemu", "verilator"}: ctx.logger.error("Invalid rushB backend: expected bemu or verilator") @@ -159,6 +168,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + if trace_config and not model: + ctx.logger.error("--trace-config requires --model") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "trace_config_requires_model"}, + trace_id=origin_tid, + ) + return if model: model_key = model.lower() @@ -200,6 +217,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctest=ctest, mlirtest=mlirtest, stable=stable, + trace_config=trace_config, logger=ctx.logger, task_scope=origin_tid, ) @@ -220,6 +238,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "rushB": rushb_backend, "ctest": ctest, "mlirtest": mlirtest, + "trace_config": trace_config, }, trace_id=origin_tid) diff --git a/bbdev/bbdev b/bbdev/bbdev index 58bef3883..485d6b3a0 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -48,7 +48,7 @@ WORKFLOW_COMMANDS = { }, "workload": { "clean": 'Clean workload output directory. Args: "--chip "', - "build": 'Build workload. Args: "--chip [--model ] [--stable] [--rushB ] [--ctest|--mlirtest]"; --ctest and --mlirtest are mutually exclusive local builds and cannot be combined with --model or --rushB.', + "build": 'Build workload. Args: "--chip [--model ] [--trace-config ] [--stable] [--rushB ] [--ctest|--mlirtest]"; --ctest and --mlirtest are mutually exclusive local builds and cannot be combined with --model or --rushB.', "tohex": 'Convert chip workload ELF to hex. Args: "--chip "', }, "kernel": { From dae31784200920b8b41f07ea941e25af915669d1 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 16 Sep 2026 23:53:39 +0800 Subject: [PATCH 157/196] feat: add BoomCoreConfig and related validation logic to JSON to proto conversion --- .../api/steps/config/scripts/3_json2proto.py | 41 ++++++++++++++++++- .../api/steps/config/scripts/proto/chip.proto | 21 ++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/bbdev/api/steps/config/scripts/3_json2proto.py b/bbdev/api/steps/config/scripts/3_json2proto.py index 980a0ef08..068e0665d 100644 --- a/bbdev/api/steps/config/scripts/3_json2proto.py +++ b/bbdev/api/steps/config/scripts/3_json2proto.py @@ -134,6 +134,41 @@ def _fill_rocket(msg: pb.RocketCoreConfig, d: dict[str, Any]) -> None: msg.btb.n_ras = btb["nRAS"] +def _fill_boom(msg: pb.BoomCoreConfig, d: dict[str, Any]) -> None: + msg.fetch_width = d["fetchWidth"] + msg.decode_width = d["decodeWidth"] + msg.num_rob_entries = d["numRobEntries"] + dc = d["dcache"] + msg.dcache.n_sets = dc["nSets"] + msg.dcache.n_ways = dc["nWays"] + msg.dcache.n_mshrs = dc["nMSHRs"] + ic = d["icache"] + msg.icache.n_sets = ic["nSets"] + msg.icache.n_ways = ic["nWays"] + + +def _check_core_kind(raw: dict[str, Any], pkg: str) -> str: + kind = raw.get("kind") + if kind not in ("rocket", "boom"): + raise ValueError(f"{pkg}: kind must be 'rocket' or 'boom', got {kind!r}") + has_rocket = "rocketCore" in raw + has_boom = "boomCore" in raw + if kind == "rocket": + if not has_rocket: + raise ValueError(f"{pkg}: kind=rocket requires rocketCore") + if has_boom: + raise ValueError(f"{pkg}: kind=rocket forbids boomCore") + else: + if not has_boom: + raise ValueError(f"{pkg}: kind=boom requires boomCore") + if has_rocket: + raise ValueError(f"{pkg}: kind=boom forbids rocketCore") + bd = raw.get("balldomain") + if isinstance(bd, dict) and bd.get("ballNum", 0): + raise ValueError(f"{pkg}: kind=boom forbids balldomain.ballNum > 0") + return kind + + def _fill_frontend(msg: pb.FrontendConfig, d: dict[str, Any], bbdir: Path) -> None: msg.source_path = _rel(bbdir, d["_file"]) msg.rob_entries = d["robEntries"] @@ -172,12 +207,16 @@ def _fill_core(ci: pb.CoreInstance, raw: dict[str, Any], meta: dict[str, Any], b ci.pkg = meta["pkg"] ci.config_path = meta["config_path"] ci.balldomain_base_dir = meta["balldomain_base_dir"] + kind = _check_core_kind(raw, meta["pkg"]) + ci.kind = kind if "balldomain" in raw: _fill_ball(ci.balldomain, raw["balldomain"], bbdir) if "memdomain" in raw: _fill_mem(ci.mem, raw["memdomain"], bbdir) - if "rocketCore" in raw: + if kind == "rocket": _fill_rocket(ci.rocket_core, raw["rocketCore"]) + else: + _fill_boom(ci.boom_core, raw["boomCore"]) if "frontend" in raw: _fill_frontend(ci.frontend, raw["frontend"], bbdir) if "gpdomain" in raw: diff --git a/bbdev/api/steps/config/scripts/proto/chip.proto b/bbdev/api/steps/config/scripts/proto/chip.proto index 22a3729a1..71c62aca9 100644 --- a/bbdev/api/steps/config/scripts/proto/chip.proto +++ b/bbdev/api/steps/config/scripts/proto/chip.proto @@ -129,6 +129,25 @@ message RocketCoreConfig { RocketBtbConfig btb = 12; } +message BoomDCacheConfig { + uint32 n_sets = 1; + uint32 n_ways = 2; + uint32 n_mshrs = 3; +} + +message BoomICacheConfig { + uint32 n_sets = 1; + uint32 n_ways = 2; +} + +message BoomCoreConfig { + uint32 fetch_width = 1; + uint32 decode_width = 2; + uint32 num_rob_entries = 3; + BoomDCacheConfig dcache = 4; + BoomICacheConfig icache = 5; +} + message FrontendConfig { uint32 rob_entries = 1; bool rs_out_of_order_response = 2; @@ -173,6 +192,8 @@ message CoreInstance { FrontendConfig frontend = 9; GpDomainConfig gp_domain = 10; CoreParamConfig core = 11; + string kind = 12; + BoomCoreConfig boom_core = 13; } message BemuBall { From d887ed2f10476af54f10bd0f4c8510ef1cb3db28 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 19 Sep 2026 15:52:16 +0800 Subject: [PATCH 158/196] feat: enhance UVM simulation support by adding IP parameter handling and validation logic --- bbdev/api/steps/uvm/01_build_api.step.py | 6 +- bbdev/api/steps/uvm/01_build_event.step.py | 10 +- bbdev/api/steps/uvm/02_run_api.step.py | 6 +- bbdev/api/steps/uvm/02_run_event.step.py | 10 +- bbdev/api/steps/uvm/scripts/uvm_common.py | 189 +++++++++++++++++++++ bbdev/api/tests/test_uvm_chip_pb.py | 15 +- bbdev/api/tests/test_uvm_waive.py | 24 +++ bbdev/mcp/tools/uvm_build.py | 10 +- bbdev/mcp/tools/uvm_run.py | 10 +- 9 files changed, 267 insertions(+), 13 deletions(-) create mode 100644 bbdev/api/tests/test_uvm_waive.py diff --git a/bbdev/api/steps/uvm/01_build_api.step.py b/bbdev/api/steps/uvm/01_build_api.step.py index 6217fe51e..4224cdf28 100644 --- a/bbdev/api/steps/uvm/01_build_api.step.py +++ b/bbdev/api/steps/uvm/01_build_api.step.py @@ -10,7 +10,7 @@ def check_args(body: dict) -> str | None: - allowed = {"chip", "ball"} + allowed = {"chip", "ball", "ip"} for key in body: if key not in allowed: return f"Unexpected parameter: --{key}" @@ -18,6 +18,10 @@ def check_args(body: dict) -> str | None: return "Missing required parameter: --chip" if body.get("ball") is True: return "Parameter --ball requires a value" + if body.get("ip") is True: + return "Parameter --ip requires a value" + if body.get("ball") and body.get("ip"): + return "Parameters --ball and --ip are mutually exclusive" return None diff --git a/bbdev/api/steps/uvm/01_build_event.step.py b/bbdev/api/steps/uvm/01_build_event.step.py index 0106e0a24..103c1c305 100644 --- a/bbdev/api/steps/uvm/01_build_event.step.py +++ b/bbdev/api/steps/uvm/01_build_event.step.py @@ -13,7 +13,7 @@ from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path -from scripts.uvm_common import run_chip +from scripts.uvm_common import run_uvm config = { "name": "uvm-build", @@ -30,7 +30,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: try: info = await asyncio.to_thread( - run_chip, bbdir, input_data["chip"], input_data.get("ball"), ctx, False + run_uvm, + bbdir, + input_data["chip"], + input_data.get("ball"), + input_data.get("ip"), + ctx, + False, ) except Exception as e: ctx.logger.error(str(e)) diff --git a/bbdev/api/steps/uvm/02_run_api.step.py b/bbdev/api/steps/uvm/02_run_api.step.py index 5cf65fd63..a0872cebb 100644 --- a/bbdev/api/steps/uvm/02_run_api.step.py +++ b/bbdev/api/steps/uvm/02_run_api.step.py @@ -10,7 +10,7 @@ def check_args(body: dict) -> str | None: - allowed = {"chip", "ball"} + allowed = {"chip", "ball", "ip"} for key in body: if key not in allowed: return f"Unexpected parameter: --{key}" @@ -18,6 +18,10 @@ def check_args(body: dict) -> str | None: return "Missing required parameter: --chip" if body.get("ball") is True: return "Parameter --ball requires a value" + if body.get("ip") is True: + return "Parameter --ip requires a value" + if body.get("ball") and body.get("ip"): + return "Parameters --ball and --ip are mutually exclusive" return None diff --git a/bbdev/api/steps/uvm/02_run_event.step.py b/bbdev/api/steps/uvm/02_run_event.step.py index e948a6d17..a8ab800a5 100644 --- a/bbdev/api/steps/uvm/02_run_event.step.py +++ b/bbdev/api/steps/uvm/02_run_event.step.py @@ -13,7 +13,7 @@ from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path -from scripts.uvm_common import run_chip +from scripts.uvm_common import run_uvm config = { "name": "uvm-run", @@ -30,7 +30,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: try: info = await asyncio.to_thread( - run_chip, bbdir, input_data["chip"], input_data.get("ball"), ctx, True + run_uvm, + bbdir, + input_data["chip"], + input_data.get("ball"), + input_data.get("ip"), + ctx, + True, ) except Exception as e: ctx.logger.error(str(e)) diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index e40ee9a1c..314b14a7f 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -11,6 +11,7 @@ from utils.path import get_buckyball_path, log_dir from utils.stream_run import stream_run_logger +from .waive import apply_waivers def load_chip(bbdir: str, chip: str): @@ -81,6 +82,183 @@ def _filelist( return str(dst.relative_to(verify_dir)) +def load_ip(bbdir: str, name: str) -> dict: + registry = Path(bbdir) / "verify" / "uvm" / "ip.toml" + if name not in tomllib.loads(registry.read_text()).get("ips", []): + raise ValueError(f"IP {name!r} is not registered in {registry}") + root = Path(bbdir) / "arch" / "src" / "main" / "scala" / "framework" / "mem-core" / name + resources = root / "src" / "main" / "resources" + targets = [ + { + "name": filelist.stem, + "filelist": filelist.name, + "top": f"{filelist.stem}_tb", + "test": "protocol_test", + } + for filelist in sorted(resources.glob("*.f")) + ] + if not targets: + raise ValueError(f"registered IP {name!r} has no filelists under {resources}") + return { + "root": str(root.relative_to(bbdir)), + "mill_module": name, + "model": "src/csrc/Cargo.toml", + "resources": "src/main/resources", + "targets": targets, + } + + +def _ip_filelist(resources: Path, source: str, sim_dir: Path, bbdir: str, rtl: Path) -> Path: + dst = sim_dir / source + sim_dir.mkdir(parents=True, exist_ok=True) + text = (resources / source).read_text() + dst.write_text( + text.replace("@VERIFY@", str(Path(bbdir) / "verify")) + .replace("@RESOURCES@", str(resources)) + .replace("@RTL@", str(rtl)) + ) + return dst + + +def build_ip(bbdir: str, chip: str, name: str, ctx) -> dict: + config = load_ip(bbdir, name) + root = Path(bbdir) / config["root"] + resources = root / config["resources"] + manifest = root / config["model"] + rtl = root / "build" + sim_root = rtl / "uvm" / chip + + cargo = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"cargo build --manifest-path {shlex.quote(str(manifest))}" + ) + result = stream_run_logger( + cmd=cargo, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm dpi", + stderr_prefix="uvm dpi", + ) + if result.returncode != 0: + raise RuntimeError(f"cargo build failed for IP {name}") + + emit = ( + f"{shlex.quote(str(Path(bbdir) / 'result' / 'bin' / 'mill'))} " + f"{shlex.quote(config['mill_module'] + '.run')}" + ) + result = stream_run_logger( + cmd=emit, + logger=ctx.logger, + cwd=str(root), + stdout_prefix="uvm rtl", + stderr_prefix="uvm rtl", + ) + if result.returncode != 0: + raise RuntimeError(f"RTL generation failed for IP {name}") + apply_waivers(rtl) + + for target in config["targets"]: + sim_dir = sim_root / target["name"] + filelist = _ip_filelist(resources, target["filelist"], sim_dir, bbdir, rtl) + simv = sim_dir / "simv" + csrc = sim_dir / "csrc" + hier = sim_dir / "cm_hier.cfg" + hier.write_text( + f"+tree {target['top']}.dut\n" + f"+tree {target['top']}.source_if\n" + f"+tree {target['top']}.sink_if\n" + ) + script = ( + f"cd {shlex.quote(str(sim_dir))} && " + f"rm -rf {shlex.quote(str(csrc))} {shlex.quote(str(simv))} {shlex.quote(str(simv))}.daidir && " + f"mkdir -p {shlex.quote(str(csrc))} && " + "vcs -full64 -sverilog -timescale=1ns/1ps -debug_access+all " + "${=VCS_UVM_ARGS} " + f"-cm line+cond+tgl+assert -cm_hier {shlex.quote(str(hier))} " + f"-cm_assert_hier {shlex.quote(str(hier))} " + f"-Mdir={shlex.quote(str(csrc))} -o {shlex.quote(str(simv))} " + f"-f {shlex.quote(str(filelist))}" + ) + command = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"zsh -c {shlex.quote(script)}" + ) + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm vcs", + stderr_prefix="uvm vcs", + ) + if result.returncode != 0: + raise RuntimeError(f"VCS failed for IP {name} target={target['name']}") + return config + + +def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str) -> dict: + config = load_ip(bbdir, name) + root = Path(bbdir) / config["root"] + manifest = root / config["model"] + crate = tomllib.loads(manifest.read_text())["package"]["name"].replace("-", "_") + model = manifest.parent / "target" / "debug" / f"lib{crate}" + sim_root = root / "build" / "uvm" / chip + + if any(not (sim_root / target["name"] / "simv").is_file() for target in config["targets"]): + config = build_ip(bbdir, chip, name, ctx) + + coverage = [] + for target in config["targets"]: + simv = sim_root / target["name"] / "simv" + cov_dir = Path(cov_root) / target["name"] / "coverage" + script = ( + f"env LD_LIBRARY_PATH=\"$VCS_RUNTIME_LIBRARY_PATH\" {shlex.quote(str(simv))} " + f"-sv_lib {shlex.quote(str(model))} +UVM_TESTNAME={shlex.quote(target['test'])} " + f"-cm line+cond+tgl+assert -cm_name {shlex.quote(target['test'])}" + ) + command = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"zsh -ic {shlex.quote(script)}" + ) + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=str(root), + stdout_prefix="uvm run", + stderr_prefix="uvm run", + ) + if result.returncode != 0: + raise RuntimeError(f"UVM failed for IP {name} target={target['name']}") + + cov_dir.mkdir(parents=True, exist_ok=True) + urg = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"urg -dir {shlex.quote(str(simv))}.vdb -format text -report {shlex.quote(str(cov_dir))}" + ) + result = stream_run_logger( + cmd=urg, + logger=ctx.logger, + cwd=str(root), + stdout_prefix="uvm urg", + stderr_prefix="uvm urg", + ) + if result.returncode != 0: + raise RuntimeError(f"URG failed for IP {name} target={target['name']}") + coverage.append((target["name"], cov_dir)) + + index_dir = Path(cov_root) / "coverage" + index_dir.mkdir(parents=True, exist_ok=True) + rows = ["target score line cond toggle assert group result"] + for target, path in coverage: + summary = dashboard_summary(str(path)) + rows.append( + f"{target} {summary['SCORE']} {summary['LINE']} {summary['COND']} " + f"{summary['TOGGLE']} {summary.get('ASSERT', '--')} {summary.get('GROUP', '--')} pass" + ) + index = index_dir / "index.txt" + index.write_text("\n".join(rows) + "\n") + return {"chip": chip, "ip": name, "targets": [t["name"] for t in config["targets"]], "index": str(index), "log": cov_root} + + def build_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx) -> None: ball = mapping.ball_dir verify_dir = Path(bbdir) / "examples" / "balls" / ball / "verify" @@ -220,3 +398,14 @@ def run_chip(bbdir: str, chip: str, ball: str | None, ctx, do_run: bool) -> dict elif do_run: info["log"] = run_root return info + + +def run_uvm(bbdir: str, chip: str, ball: str | None, ip: str | None, ctx, do_run: bool) -> dict: + if ip is None: + return run_chip(bbdir, chip, ball, ctx, do_run) + if do_run: + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + run_root = log_dir(bbdir, chip, "verilog", stamp, "uvm", ip) + return run_ip(bbdir, chip, ip, ctx, run_root) + config = build_ip(bbdir, chip, ip, ctx) + return {"chip": chip, "ip": ip, "targets": [target["name"] for target in config["targets"]]} diff --git a/bbdev/api/tests/test_uvm_chip_pb.py b/bbdev/api/tests/test_uvm_chip_pb.py index 6aa2cb75e..bc0bac1fb 100644 --- a/bbdev/api/tests/test_uvm_chip_pb.py +++ b/bbdev/api/tests/test_uvm_chip_pb.py @@ -3,10 +3,10 @@ API = Path(__file__).resolve().parents[1] sys.path.insert(0, str(API / "steps" / "config" / "scripts")) -sys.path.insert(0, str(API / "steps" / "uvm" / "scripts")) +sys.path.insert(0, str(API / "steps" / "uvm")) import chip_pb2 -from uvm_common import ball_domain, selected_mappings, vcs_defines +from scripts.uvm_common import ball_domain, load_ip, selected_mappings, vcs_defines def _chip(): @@ -28,16 +28,24 @@ def test_defines(): c = _chip() d = ball_domain(c) m = selected_mappings(d, "lut")[0] - defs = vcs_defines(d, m) + defs = vcs_defines(d, m, 4096) assert defs == [ "+define+BB_IN_BW=2", "+define+BB_OUT_BW=1", "+define+BB_MMIO_READ_BW=0", "+define+BB_MMIO_WRITE_BW=0", + "+define+BB_BANK_ADDR_W=12", "+define+LUT_FUNCT7=66", ] +def test_registered_ip(): + root = str(Path(__file__).resolve().parents[3]) + axis = load_ip(root, "axis") + assert axis["mill_module"] == "axis" + assert [target["name"] for target in axis["targets"]] == ["arbiter", "master", "slave"] + + def test_unknown_ball(): try: selected_mappings(ball_domain(_chip()), "transpose") @@ -49,5 +57,6 @@ def test_unknown_ball(): if __name__ == "__main__": test_defines() + test_registered_ip() test_unknown_ball() print("ok") diff --git a/bbdev/api/tests/test_uvm_waive.py b/bbdev/api/tests/test_uvm_waive.py new file mode 100644 index 000000000..2ca2af0b0 --- /dev/null +++ b/bbdev/api/tests/test_uvm_waive.py @@ -0,0 +1,24 @@ +import sys +from pathlib import Path + +API = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(API / "steps" / "uvm")) + +from scripts.waive import apply_waivers + + +def test_waivers_are_target_agnostic_and_idempotent(tmp_path): + rtl = tmp_path / "rtl" + rtl.mkdir() + source = rtl / "Dut.sv" + source.write_text("module Dut;\n wire _GEN = 1'b0;\nendmodule\n") + + assert apply_waivers(rtl) == 1 + assert source.read_text() == ( + "module Dut;\n" + "//VCS coverage off\n" + " wire _GEN = 1'b0;\n" + "//VCS coverage on\n" + "endmodule\n" + ) + assert apply_waivers(rtl) == 0 diff --git a/bbdev/mcp/tools/uvm_build.py b/bbdev/mcp/tools/uvm_build.py index 0d4792c6d..8132b7055 100644 --- a/bbdev/mcp/tools/uvm_build.py +++ b/bbdev/mcp/tools/uvm_build.py @@ -9,11 +9,17 @@ def register(mcp): @mcp.tool() - def bbdev_uvm_build(chip: str, ball: Optional[str] = None) -> str: - """Build a Ball UVM simulation. POST /uvm/build.""" + def bbdev_uvm_build( + chip: str, ball: Optional[str] = None, ip: Optional[str] = None + ) -> str: + """Build a Ball or IP UVM simulation. POST /uvm/build.""" if e := need("chip", chip): return err(e) params = {"chip": chip} + if ball and ip: + return err("Parameters --ball and --ip are mutually exclusive") if ball: params["ball"] = ball + if ip: + params["ip"] = ip return fmt(submit("/uvm/build", params)) diff --git a/bbdev/mcp/tools/uvm_run.py b/bbdev/mcp/tools/uvm_run.py index 93f537c9f..c285dbf80 100644 --- a/bbdev/mcp/tools/uvm_run.py +++ b/bbdev/mcp/tools/uvm_run.py @@ -9,11 +9,17 @@ def register(mcp): @mcp.tool() - def bbdev_uvm_run(chip: str, ball: Optional[str] = None) -> str: - """Build and run a Ball UVM simulation. POST /uvm/run.""" + def bbdev_uvm_run( + chip: str, ball: Optional[str] = None, ip: Optional[str] = None + ) -> str: + """Build and run a Ball or IP UVM simulation. POST /uvm/run.""" if e := need("chip", chip): return err(e) params = {"chip": chip} + if ball and ip: + return err("Parameters --ball and --ip are mutually exclusive") if ball: params["ball"] = ball + if ip: + params["ip"] = ip return fmt(submit("/uvm/run", params)) From 950dbf0e71f18379885126339b4862e83d09144f Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Tue, 22 Sep 2026 10:58:00 +0800 Subject: [PATCH 159/196] [node/p2e] Support P2E bank DiffTest (#17) * feat: support P2E bank difftest * fix: create bbdev runtime data directory * fix: defer generated chip protobuf import * fix: serialize concurrent mill builds * fix: handle UVM chip domains and generated accumulator names * fix: reject unsupported UVM domains and missing filelists * refactor: update CPU configuration handling in JSON to proto conversion and enhance UVM script functionality * feat: add support for diff option in Bebop P2E and Verilator workflows * feat: integrate diff option into Bebop P2E build and runtime processes * refactor: improve build directory validation and update CPU configuration messages in proto files * refactor: streamline preset configuration management and enhance error handling in Bebop workflows --------- Co-authored-by: shirohasuki --- bbdev/api/services/presets.py | 30 +-- .../api/steps/bebop/bemu/02_sim_event.step.py | 19 +- .../steps/bebop/bemu/03_batch_event.step.py | 12 +- .../steps/bebop/bemu/04_analysis_api.step.py | 6 +- .../steps/bebop/bemu/scripts/bemu_analysis.py | 34 ++- .../bebop/bemu/scripts/test_bemu_analysis.py | 120 --------- .../steps/bebop/p2e/01_verilog_api.step.py | 2 +- .../bebop/p2e/02_buildbitstream_api.step.py | 1 + .../bebop/p2e/02_buildbitstream_event.step.py | 74 +++--- .../bebop/p2e/03_runworkload_api.step.py | 1 - .../bebop/p2e/03_runworkload_event.step.py | 57 ++--- .../api/steps/bebop/p2e/04_batch_api.step.py | 2 + .../steps/bebop/p2e/04_batch_event.step.py | 78 +++--- bbdev/api/steps/bebop/regression_harness.py | 12 - .../steps/bebop/vcs/03_batch_event.step.py | 4 +- .../bebop/verilator/02_verilog_api.step.py | 2 +- .../bebop/verilator/03_build_event.step.py | 6 +- .../bebop/verilator/04_sim_event.step.py | 2 +- .../bebop/verilator/06_batch_event.step.py | 30 +-- .../api/steps/config/01_install_event.step.py | 4 + .../config/scripts/2_parameter_derivation.py | 4 +- .../api/steps/config/scripts/3_json2proto.py | 54 ++-- bbdev/api/steps/config/scripts/4_install.py | 80 +++++- .../api/steps/config/scripts/proto/chip.proto | 27 +- .../steps/dc/scripts/emit_seq_mem_models.py | 12 +- bbdev/api/steps/ip/02_replace_event.step.py | 7 +- bbdev/api/steps/ip/scripts/generate.py | 16 +- bbdev/api/steps/ip/scripts/replace.py | 28 +-- bbdev/api/steps/ip/scripts/sram_compiler.py | 12 +- bbdev/api/steps/kernel/01_build_event.step.py | 45 ++-- .../06_bebop_verilator_verilog_event.step.py | 9 +- .../mill/07_bebop_p2e_verilog_event.step.py | 9 +- bbdev/api/steps/mill/scripts/mill.py | 4 +- bbdev/api/steps/pt/scripts/power.py | 9 +- .../00_buildbitstream_event.step.py | 2 +- .../02_eval_performance_event.step.py | 85 +++---- .../api/steps/regression/scripts/accuracy.py | 8 +- bbdev/api/steps/regression/scripts/dc_area.py | 5 +- .../steps/regression/scripts/model_layout.py | 70 ++---- bbdev/api/steps/regression/scripts/result.py | 10 +- bbdev/api/steps/uvm/scripts/uvm_common.py | 230 ++++++++++++++---- bbdev/api/steps/uvm/scripts/waive.py | 7 +- bbdev/api/steps/vcs/03_build_event.step.py | 10 +- .../steps/verilator/03_build_event.step.py | 6 +- .../api/steps/verilator/04_sim_event.step.py | 10 +- .../api/steps/workload/01_build_event.step.py | 28 +-- bbdev/api/steps/yosys/03_synth_event.step.py | 14 +- bbdev/api/tests/test_all_api_serial.py | 2 + .../api/tests/test_api_bebop_bemu_analysis.py | 11 +- bbdev/api/tests/test_api_bebop_bemu_sim.py | 2 +- bbdev/api/tests/test_api_uvm_build.py | 3 + bbdev/api/tests/test_api_uvm_run.py | 3 + bbdev/api/tests/test_uvm_chip_pb.py | 62 ----- bbdev/api/tests/test_uvm_waive.py | 24 -- bbdev/api/utils/event_common.py | 5 +- bbdev/api/utils/model.py | 19 ++ bbdev/api/utils/path.py | 18 +- bbdev/api/utils/port.py | 7 +- bbdev/api/utils/workers.py | 9 +- .../workload_manifest.py} | 4 +- bbdev/bbdev | 13 +- bbdev/mcp/__main__.py | 5 +- bbdev/mcp/common.py | 28 ++- bbdev/mcp/tools/__init__.py | 7 +- bbdev/mcp/tools/bebop_p2e_batch.py | 2 + bbdev/mcp/tools/bebop_p2e_buildbitstream.py | 3 +- bbdev/mcp/tools/bebop_p2e_runworkload.py | 3 + bbdev/mcp/tools/bebop_p2e_verilog.py | 4 +- bbdev/mcp/tools/bebop_verilator_verilog.py | 4 +- 69 files changed, 718 insertions(+), 817 deletions(-) delete mode 100644 bbdev/api/steps/bebop/bemu/scripts/test_bemu_analysis.py delete mode 100644 bbdev/api/steps/bebop/regression_harness.py create mode 100644 bbdev/api/tests/test_api_uvm_build.py create mode 100644 bbdev/api/tests/test_api_uvm_run.py delete mode 100644 bbdev/api/tests/test_uvm_chip_pb.py delete mode 100644 bbdev/api/tests/test_uvm_waive.py create mode 100644 bbdev/api/utils/model.py rename bbdev/api/{steps/bebop/regression.py => utils/workload_manifest.py} (94%) diff --git a/bbdev/api/services/presets.py b/bbdev/api/services/presets.py index 8db2aff02..6ef8be893 100644 --- a/bbdev/api/services/presets.py +++ b/bbdev/api/services/presets.py @@ -65,19 +65,11 @@ def create_code_agent_manager() -> ToolManager: return manager -# Predefined tool set configurations -PRESET_CONFIGS = { - "file_tools": { - "name": "File Operations", - "description": "Basic file system operations", - "tools": create_file_tools, - }, - "code_agent": { - "name": "Code Agent", - "description": "Tools for code generation and manipulation", - "tools": create_code_agent_tools, - }, -} +def preset_configs(): + return { + "file_tools": create_file_tools, + "code_agent": create_code_agent_tools, + } def get_preset(name: str) -> List[Tool]: @@ -93,14 +85,14 @@ def get_preset(name: str) -> List[Tool]: Raises: ValueError: If tool set does not exist """ - config = PRESET_CONFIGS.get(name) - if not config: - available = ", ".join(PRESET_CONFIGS.keys()) + presets = preset_configs() + factory = presets.get(name) + if factory is None: + available = ", ".join(presets) raise ValueError(f"Unknown preset: {name}. Available: {available}") - - return config["tools"]() + return factory() def list_presets() -> List[str]: """List all available predefined tool sets""" - return list(PRESET_CONFIGS.keys()) + return list(preset_configs()) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index d5a1f5a80..12d0103f7 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -27,10 +27,6 @@ from bemu_common import bemu_core_manifest, bemu_manifest, bemu_tile_index, chip_emu_manifest -PERFETTO_TARGETS = { - "buddy-buckyball-lenet-run": "buddy-buckyball-lenet-perfetto", -} - config = { "name": "bebop-bemu-sim", "description": "Run bebop bemu emulator", @@ -69,7 +65,12 @@ def resolve_bemu_binary(bbdir: str, chip: str, binary_name: str) -> str | None: if Path(binary_name).name != binary_name: return None - return search_workload(workloads_output_root(bbdir), binary_name) + workload = search_workload(workloads_output_root(bbdir), binary_name) + if workload is not None: + return workload + + kernel = Path(bbdir) / "bb-tests" / "output" / "kernel" / chip / binary_name + return str(kernel) if kernel.is_file() else None def is_native_host_elf(path: str) -> bool: @@ -133,9 +134,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return ctx.logger.info(f"binary_path: {binary_path}") binary_dir = os.path.dirname(binary_path) - perfetto_target = ( - PERFETTO_TARGETS.get(binary_name) if input_data.get("tool-profile") else None - ) + perfetto_target = None + if input_data.get("tool-profile"): + if not binary_name.endswith("-run"): + raise ValueError(f"tool-profile binary must end with '-run': {binary_name}") + perfetto_target = f"{binary_name.removesuffix('-run')}-perfetto" if perfetto_target: clean_model_trace(binary_dir) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index 0422530f9..f5bb21593 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -27,8 +27,7 @@ from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from bemu_common import bemu_manifest, chip_emu_manifest -from regression import regression_workload_toml -from regression_harness import nextest_harness_args +from utils.workload_manifest import resolve_workload_toml config = { "name": "bebop-bemu-batch", @@ -82,7 +81,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: env.update(bebop_cargo_env(bbdir, chip)) test_type = input_data.get("test", "elf-tests") try: - workload_toml = regression_workload_toml( + workload_toml = resolve_workload_toml( chip, "bemu", test_type, bbdir, rushB=rushB ) except ValueError as e: @@ -137,7 +136,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) ctx.logger.info("Cleaned previous bebop test artifacts") - harness = nextest_harness_args(workload_toml, elf_root, env) + harness = shlex.join([ + "--", + "--workload-toml", workload_toml, + "--bb-tests-root", elf_root, + *(["--rushb-backend", "bemu"] if rushB else []), + ]) nextest_cmd = ( f"nix develop -c cargo nextest run --manifest-path {shlex.quote(str(bemu_cargo_manifest))} " "--test test_bemu " diff --git a/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py b/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py index 20ce60588..eaef1930d 100644 --- a/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py +++ b/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py @@ -21,9 +21,6 @@ "enqueues": ["bebop.bemu.analysis"], } -ALLOWED = {"chip", "log-dir", "log_dir", "itrace", "mtrace"} - - def _fail(message: str, status: int = 400) -> ApiResponse: return ApiResponse( status=status, @@ -38,7 +35,8 @@ def _fail(message: str, status: int = 400) -> ApiResponse: async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - unknown = sorted(k for k in body if k not in ALLOWED) + allowed = {"chip", "log-dir", "log_dir", "itrace", "mtrace"} + unknown = sorted(k for k in body if k not in allowed) if unknown: return _fail(f"unknown parameter(s): {', '.join(unknown)}") diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py index a7d91e549..e12b4e667 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py @@ -9,21 +9,8 @@ from collections import Counter from pathlib import Path -ISA = { - 0: "fence", - 1: "barrier", - 16: "mvout", - 32: "mset", - 33: "mvin", - 34: "mmio_set", - 35: "mvin_mmio", -} - -MATRIX_MNEMONICS = frozenset({"MATRIX", "MATRIX_F32", "SMATMUL_OS"}) - -_API = Path(__file__).resolve().parents[4] -if str(_API) not in sys.path: - sys.path.insert(0, str(_API)) +if str(Path(__file__).resolve().parents[4]) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parents[4])) def abs_log_dir(value: object) -> Path: @@ -42,6 +29,15 @@ def _hex_u(value: object, field: str, line_no: int) -> int: def chip_maps(bbdir: str, chip: str) -> tuple[dict[int, str], set[int], int]: + system_isa = { + 0: "fence", + 1: "barrier", + 16: "mvout", + 32: "mset", + 33: "mvin", + 34: "mmio_set", + 35: "mvin_mmio", + } path = ( Path(bbdir) / "examples" @@ -75,7 +71,7 @@ def chip_maps(bbdir: str, chip: str) -> tuple[dict[int, str], set[int], int]: isa = ball["ballISA"] ball_path = ball["_file"] - names: dict[int, str] = dict(ISA) + names: dict[int, str] = dict(system_isa) matrix: set[int] = set() seen: set[int] = set() for entry in isa: @@ -89,13 +85,13 @@ def chip_maps(bbdir: str, chip: str) -> tuple[dict[int, str], set[int], int]: raise ValueError(f"ballISA mnemonic must be a non-empty string: {ball_path}") if funct in seen: raise ValueError(f"duplicate ballISA funct7 {funct}: {ball_path}") - if funct in ISA: + if funct in system_isa: raise ValueError( - f"ballISA funct7 {funct} ({mnemonic}) collides with ISA {ISA[funct]}: {ball_path}" + f"ballISA funct7 {funct} ({mnemonic}) collides with ISA {system_isa[funct]}: {ball_path}" ) seen.add(funct) names[funct] = mnemonic.lower() - if mnemonic in MATRIX_MNEMONICS: + if mnemonic in {"MATRIX", "MATRIX_F32", "SMATMUL_OS"}: matrix.add(funct) mem = core["memdomain"] diff --git a/bbdev/api/steps/bebop/bemu/scripts/test_bemu_analysis.py b/bbdev/api/steps/bebop/bemu/scripts/test_bemu_analysis.py deleted file mode 100644 index eaf6c2c3e..000000000 --- a/bbdev/api/steps/bebop/bemu/scripts/test_bemu_analysis.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python3 -import tempfile -import unittest -from pathlib import Path - -from bemu_analysis import abs_log_dir, analysis_dir, chip_maps - -BBDIR = Path(__file__).resolve().parents[6] - - -def _write_bdb(text: str) -> Path: - tmp = Path(tempfile.mkdtemp()) - (tmp / "bdb.ndjson").write_text(text, encoding="utf-8") - return tmp - - -class AbsLogDirTest(unittest.TestCase): - def test_relative_fails(self): - with self.assertRaisesRegex(ValueError, "absolute path"): - abs_log_dir("log/2026-08-19-13-00-bemu-buddy-buckyball-yolo26-run") - - def test_absolute_ok(self): - path = abs_log_dir("/tmp/bemu-log") - self.assertTrue(path.is_absolute()) - - -class ChipMapsTest(unittest.TestCase): - def test_pebble_loads_ballisa(self): - names, matrix, depth = chip_maps(str(BBDIR), "pebble") - self.assertEqual(names[0], "fence") - self.assertEqual(names[32], "mset") - self.assertEqual(names[65], "smatmul") - self.assertEqual(names[48], "im2col") - self.assertEqual(names[49], "transpose") - self.assertEqual(names[51], "fp2int") - self.assertEqual(names[52], "int2fp") - self.assertEqual(matrix, {65}) - self.assertEqual(depth, 1024) - - -class AnalysisDirTest(unittest.TestCase): - def setUp(self): - self.names, self.matrix, self.depth = chip_maps(str(BBDIR), "pebble") - - def test_itrace_mtrace(self): - log_dir = _write_bdb( - "\n".join( - [ - '{"type":"itrace","clk":10,"event":"complete","funct":"0x00","pc":"0x0","rs1":"0x0","rs2":"0x0"}', - '{"type":"itrace","clk":20,"event":"complete","funct":"0x41","pc":"0x0","rs1":"0x0","rs2":"0x0000000100010400"}', - '{"type":"mtrace","clk":15,"event":"read","addr":"0x0","rows":1024,"line_bytes":16,"row_stride":16,"vbank_id":0,"pbank_id":0,"group_id":0}', - '{"type":"mtrace","clk":16,"event":"write","addr":"0x0","rows":1,"line_bytes":16,"row_stride":16,"vbank_id":1,"pbank_id":1,"group_id":0}', - "", - ] - ) - ) - text = analysis_dir( - log_dir, - names=self.names, - matrix=self.matrix, - bank_depth=self.depth, - itrace=True, - mtrace=True, - ) - self.assertIn("span_cycles: 20", text) - self.assertIn("smatmul", text) - self.assertIn("(1024,16,256) n=1", text) - self.assertIn(f"bank_depth: {self.depth}", text) - self.assertIn("mean_rows/bank_depth:", text) - self.assertNotIn("mean_rows/1024:", text) - self.assertIn("rows=1: 1 (50.0%)", text) - - def test_bank_depth_from_arg(self): - log_dir = _write_bdb( - '{"type":"mtrace","clk":1,"event":"read","addr":"0x0","rows":32,"line_bytes":16,"row_stride":16,"vbank_id":0,"pbank_id":0,"group_id":0}\n' - ) - text = analysis_dir( - log_dir, - names=self.names, - matrix=self.matrix, - bank_depth=64, - itrace=False, - mtrace=True, - ) - self.assertIn("bank_depth: 64", text) - self.assertIn("mean_rows: 32.00", text) - self.assertIn("mean_rows/bank_depth: 0.5000", text) - self.assertNotIn("mean_rows/1024", text) - - def test_unknown_funct_fails(self): - log_dir = _write_bdb( - '{"type":"itrace","clk":1,"event":"complete","funct":"0x99","pc":"0x0","rs1":"0x0","rs2":"0x0"}\n' - ) - with self.assertRaisesRegex(ValueError, "unknown funct 0x99"): - analysis_dir( - log_dir, - names=self.names, - matrix=self.matrix, - bank_depth=self.depth, - itrace=True, - mtrace=False, - ) - - def test_missing_itrace_fails(self): - log_dir = _write_bdb( - '{"type":"mtrace","clk":1,"event":"read","addr":"0x0","rows":1,"line_bytes":16,"row_stride":16,"vbank_id":0,"pbank_id":0,"group_id":0}\n' - ) - with self.assertRaisesRegex(ValueError, "no itrace events"): - analysis_dir( - log_dir, - names=self.names, - matrix=self.matrix, - bank_depth=self.depth, - itrace=True, - mtrace=False, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py index e0fd8b9f5..0c2c17169 100644 --- a/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py +++ b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py @@ -18,7 +18,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: chip = require_chip(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - data = {"chip": chip} + data = {"chip": chip, "diff": bool(body.get("diff", False))} if body.get("output_dir"): data["output_dir"] = body["output_dir"] await ctx.enqueue({"topic": "bebop.p2e.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py index 61dc9572b..ba2a845ee 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py @@ -25,6 +25,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "chip": chip, + "diff": bool(body.get("diff", False)), "vsrc_dir": vsrc_dir, "output_dir": body.get("output_dir") or body.get("output-dir"), } diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index fc953a9c5..b9f8bc3bd 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -7,8 +7,11 @@ 3. Validate generated bitstream and runtime artifacts """ import os +import shlex +import shutil import sys from datetime import datetime +from pathlib import Path from motia import FlowContext, queue @@ -50,57 +53,52 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: normalize_p2e_timescale(vsrc_dir, ctx.logger) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") build_dir = ( - input_data.get("build_dir") - or input_data.get("build-dir") - or input_data.get("output_dir") - or input_data.get("output-dir") + input_data.get("output_dir") or f"{bebop_dir}/build/{chip}-{timestamp}" ) - os.makedirs(build_dir, exist_ok=True) + build_path = Path(build_dir).resolve() + build_root = (Path(bebop_dir) / "build").resolve() + if build_root not in build_path.parents: + raise ValueError(f"P2E output_dir must be under {build_root}: {build_path}") + if build_path.exists(): + shutil.rmtree(build_path) + build_path.mkdir(parents=True) - build_cmd = ( - f"nix develop --ignore-environment --keep HOME --keep ALL_PROXY -c " - f"cargo run --release --features p2e -- build p2e " - f"--rtl-dir=\"{vsrc_dir}\" " - f"--out-dir=\"{build_dir}\"" - ) + diff = bool(input_data.get("diff", False)) + manifest = Path(bbdir) / "examples" / "chips" / chip / "generated" / "bebop" / "Cargo.toml" + features = ["p2e"] + build_env = {**os.environ, **bebop_cargo_env(bbdir, chip)} + if diff: + features.append("bemu") + build_env["CARGO_TARGET_DIR"] = os.path.join(bebop_dir, "target", f"{chip}-p2e-diff") + build_cmd = shlex.join([ + "nix", "develop", "--ignore-env", + "--keep-env-var", "HOME", + "--keep-env-var", "ALL_PROXY", + "--keep-env-var", "CARGO_TARGET_DIR", + "-c", "cargo", "run", "--release", + "--manifest-path", str(manifest), + "--bin", "bebop", + "--features", ",".join(features), + "--", "build", "p2e", + "--rtl-dir", vsrc_dir, + "--out-dir", build_dir, + *(["--diff"] if diff else []), + ]) ctx.logger.info("Building bebop p2e runtime case ...") build_result = await stream_run_logger_async( cmd=build_cmd, logger=ctx.logger, - cwd=bebop_dir, + cwd=str(manifest.parent), stdout_prefix="bebop p2e build", stderr_prefix="bebop p2e build", - env={**os.environ, **bebop_cargo_env(bbdir, chip)}, + env=build_env, ) rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") libvctb_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so") bitstream_path = os.path.join(build_dir, "fpgaCompDir", "bitstream.bit") - if build_result.returncode == 0: - missing = [ - path - for path in (rtcfg_path, libvctb_path, bitstream_path) - if not os.path.exists(path) - ] - if missing: - ctx.logger.error(f"P2E build artifacts missing: {missing}") - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={ - "task": "build", - "vsrc_dir": vsrc_dir, - "build_dir": build_dir, - "missing": missing, - "error": "p2e_artifact_not_found", - "timestamp": timestamp, - }, - trace_id=origin_tid, - ) - return - + runtime_path = os.path.join(build_dir, "bebop-p2e") extra_fields = { "task": "build", "vsrc_dir": vsrc_dir, @@ -108,6 +106,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "rtcfg": rtcfg_path, "libvCtb": libvctb_path, "bitstream": bitstream_path, + "runtime": runtime_path, + "diff": diff, "timestamp": timestamp, } if input_data.get("from_regression_buildbitstream") and build_result.returncode == 0: diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py index 88dace351..472cf701b 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py @@ -23,7 +23,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "message": "image and bitstream parameters are required", }, ) - await ctx.enqueue({ "topic": "bebop.p2e.runworkload", "data": {**body, "_trace_id": ctx.trace_id}, diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index f1ab863f6..c96bdb4a1 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -9,6 +9,7 @@ import glob import os import re +import shlex import sys from datetime import datetime @@ -22,7 +23,7 @@ sys.path.insert(0, scripts_path) from utils.event_common import require_chip -from utils.path import bebop_cargo_env, get_buckyball_path, log_dir, rtl_dir +from utils.path import bebop_cargo_env, get_buckyball_path, log_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from resolve_image import resolve_image @@ -79,6 +80,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return wave = bool(input_data.get("wave", False)) + diff = bool(input_data.get("diff", False)) if "wave_start" in input_data: ctx.logger.error("invalid parameter: --wave_start (use --wave-start)") await check_result( @@ -121,6 +123,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return + image_base = os.path.splitext(image_path)[0] + elf_path = f"{image_base}.elf" if os.path.isfile(f"{image_base}.elf") else image_base + if diff and not os.path.isfile(elf_path): + ctx.logger.error(f"workload ELF for P2E DiffTest not found: {elf_path}") + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "workload_elf_not_found", "elf": elf_path}, + trace_id=origin_tid, + ) + return + if not bitstream or not os.path.isfile(bitstream): ctx.logger.error(f"bitstream .bit file not found: {bitstream}") await check_result( @@ -145,45 +158,27 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: multi_fpga = True ctx.logger.info(f"Detected multi-FPGA P2E case: {build_dir}") - vsrc_dir = rtl_dir(bbdir, chip, "p2e", input_data.get("vsrc_dir")) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") run_log = log_dir(bbdir, chip, "p2e", timestamp, "p2e", image_name, input_data.get("vsrc_dir")) os.makedirs(run_log, exist_ok=True) - # Rebuild the VVAC host runtime in the bitstream case. The bitstream is - # deliberately left in place, so runtime/DPIC changes never trigger FPGA synthesis. - runtime_cmd = ( - f"env BEBOP_P2E_RUNTIME_ONLY=1 BEBOP_P2E_REBUILD_RUNTIME=1 " - f"cargo run --release --features p2e -- build p2e " - f"--rtl-dir=\"{vsrc_dir}\" " - f"--out-dir=\"{build_dir}\"" - ) - ctx.logger.info("Preparing bebop p2e runtime for the selected bitstream ...") - runtime_result = await stream_run_logger_async( - cmd=runtime_cmd, - logger=ctx.logger, - cwd=bebop_dir, - stdout_prefix="bebop p2e runtime", - stderr_prefix="bebop p2e runtime", - env={**os.environ.copy(), **bebop_cargo_env(bbdir, chip)}, - ) runtime_lib_dir = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm") rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") libvctb_path = os.path.join(runtime_lib_dir, "libvCtb.so") - libstdcxx_path = os.path.join(runtime_lib_dir, "libstdc++.so.6") bebop_p2e_path = os.path.join(build_dir, "bebop-p2e") - runtime_artifacts = (rtcfg_path, libvctb_path, libstdcxx_path, bebop_p2e_path) - if runtime_result.returncode != 0 or not all(os.path.isfile(path) for path in runtime_artifacts): + runtime_artifacts = [rtcfg_path, libvctb_path, bebop_p2e_path] + if diff: + runtime_artifacts.append(os.path.join(build_dir, "libriscv.so")) + if not all(os.path.isfile(path) for path in runtime_artifacts): missing = [path for path in runtime_artifacts if not os.path.isfile(path)] if missing: ctx.logger.error(f"P2E runtime artifacts missing: {missing}") await check_result( ctx, - runtime_result.returncode or 1, + 1, continue_run=False, extra_fields={ "task": "runtime", - "vsrc_dir": vsrc_dir, "build_dir": build_dir, "missing": missing, }, @@ -191,10 +186,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - # Run the case-local executable produced above. It must load the VVAC - # runtime's libstdc++ (6.0.25), not Cargo/Nix's newer libstdc++; mixing the - # two ABIs crashes inside ICtbMgr::init(). Apply LD_LIBRARY_PATH only to - # the runtime process so Cargo itself keeps its normal library environment. run_cmd = ( f"\"{bebop_p2e_path}\" run p2e " f"--image=\"{image_path}\" " @@ -208,17 +199,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: run_cmd += " --wave" if wave_start is not None: run_cmd += f" --wave-start=\"{wave_start}\"" + if diff: + run_cmd += f" --diff --image-elf={shlex.quote(elf_path)}" for trace_name in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace"): if input_data.get(trace_name, False): run_cmd += f" --{trace_name}" ctx.logger.info(f"Running bebop p2e runworkload: {run_cmd}") run_env = {**os.environ.copy(), **bebop_cargo_env(bbdir, chip)} - inherited_library_path = run_env.get("LD_LIBRARY_PATH") - run_env["LD_LIBRARY_PATH"] = ( - f"{runtime_lib_dir}:{inherited_library_path}" - if inherited_library_path - else runtime_lib_dir - ) run_result = await stream_run_logger_async( cmd=run_cmd, logger=ctx.logger, @@ -241,6 +228,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "fpga_location": fpga_location, "bdb_trace": os.path.join(run_log, "bdb.ndjson"), "uart_log": os.path.join(run_log, "uart.log"), + "bank_diff": os.path.join(run_log, "diff.ndjson") if diff else None, + "diff": diff, "timestamp": timestamp, }, trace_id=origin_tid, diff --git a/bbdev/api/steps/bebop/p2e/04_batch_api.step.py b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py index 843a445ff..95fe93d23 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_api.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py @@ -44,10 +44,12 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body={"error": f"Invalid test type: {test_type}. Must be 'elf-tests' or 'pk-tests'"} ) + diff = bool(body.get("diff", False)) data = { "chip": chip, "bitstream": bitstream, "test": test_type, + "diff": diff, } await ctx.enqueue({"topic": "bebop.p2e.batch", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py index aee02f005..52080a0a6 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py @@ -2,13 +2,15 @@ bebop p2e batch event handler Runs bebop p2e nextest batch regression (aligned with runworkload): - 1. Resolve VSRC from bitstream case and rebuild VVAC runtime into case dir - 2. Build bebop with p2e feature and OUT_PATH= - 3. Run cargo nextest with the same OUT_PATH (serial, single FPGA) + 1. Resolve the runtime from the bitstream case + 2. Build the test harness against that runtime + 3. Run cargo nextest serially on one FPGA """ import os import re +import shlex import sys +from pathlib import Path from motia import FlowContext, queue @@ -20,11 +22,10 @@ sys.path.insert(0, bebop_path) from utils.event_common import require_chip -from utils.path import bebop_cargo_env, get_buckyball_path, rtl_dir, workloads_output_root +from utils.path import bebop_cargo_env, get_buckyball_path, workloads_output_root from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id -from regression import regression_workload_toml -from regression_harness import nextest_harness_args +from utils.workload_manifest import resolve_workload_toml config = { "name": "bebop-p2e-batch", @@ -61,7 +62,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return bitstream = os.path.abspath(bitstream) - # Same as runworkload: case home is parent of fpgaCompDir/ build_dir = os.path.dirname(os.path.dirname(bitstream)) if not os.path.isdir(build_dir): ctx.logger.error(f"P2E build case not found for bitstream: {build_dir}") @@ -85,8 +85,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cargo_env = bebop_cargo_env(bbdir, chip) test_type = input_data.get("test", "elf-tests") + diff = bool(input_data.get("diff", False)) try: - workload_toml = regression_workload_toml(chip, "p2e", test_type, bbdir) + workload_toml = resolve_workload_toml(chip, "p2e", test_type, bbdir, diff=diff) except ValueError as e: ctx.logger.error(str(e)) await check_result( @@ -96,39 +97,32 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - ctx.logger.info(f"Running {test_type} with workload config: {workload_toml}") + ctx.logger.info(f"Running {test_type} with workload config: {workload_toml} diff={diff}") ctx.logger.info(f"P2E case dir (from bitstream): {build_dir}") - vsrc_dir = rtl_dir(bbdir, chip, "p2e", input_data.get("vsrc_dir")) - # Rebuild VVAC host runtime in the bitstream case (same as runworkload). - runtime_cmd = ( - f"env BEBOP_P2E_RUNTIME_ONLY=1 BEBOP_P2E_REBUILD_RUNTIME=1 " - f"cargo run --release --features p2e -- build p2e " - f"--rtl-dir=\"{vsrc_dir}\" " - f"--out-dir=\"{build_dir}\"" - ) - ctx.logger.info("Preparing bebop p2e runtime for the selected bitstream ...") - runtime_result = await stream_run_logger_async( - cmd=runtime_cmd, - logger=ctx.logger, - cwd=bebop_dir, - stdout_prefix="bebop p2e runtime", - stderr_prefix="bebop p2e runtime", - env={**os.environ.copy(), **cargo_env}, - ) + manifest = Path(bebop_dir) / "Cargo.toml" + features = ["p2e"] + runtime_env = {**os.environ.copy(), **cargo_env} + if diff: + manifest = Path(bbdir) / "examples" / "chips" / chip / "generated" / "bebop" / "Cargo.toml" + features.append("bemu") + runtime_env["CARGO_TARGET_DIR"] = os.path.join(bebop_dir, "target", f"{chip}-p2e-diff") + feature_arg = ",".join(features) rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") libvctb_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so") - if runtime_result.returncode != 0 or not all(os.path.isfile(path) for path in (rtcfg_path, libvctb_path)): - missing = [path for path in (rtcfg_path, libvctb_path) if not os.path.isfile(path)] + runtime_artifacts = [rtcfg_path, libvctb_path, os.path.join(build_dir, "bebop-p2e")] + if diff: + runtime_artifacts.append(os.path.join(build_dir, "libriscv.so")) + if not all(os.path.isfile(path) for path in runtime_artifacts): + missing = [path for path in runtime_artifacts if not os.path.isfile(path)] if missing: ctx.logger.error(f"P2E runtime artifacts missing: {missing}") await check_result( ctx, - runtime_result.returncode or 1, + 1, continue_run=False, extra_fields={ "task": "runtime", - "vsrc_dir": vsrc_dir, "build_dir": build_dir, "missing": missing, }, @@ -136,12 +130,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - # cargo global --config so OUT_PATH reaches bebop-p2e build.rs (same as runworkload). + # OUT_PATH makes the test harness link against this bitstream case's runtime. cargo_out = f"--config=\"env.OUT_PATH='{build_dir}'\"" # ── Build bebop p2e (tests), linked against case libvCtb ─────────────── build_cmd = ( - f"nix develop -c cargo {cargo_out} build --release --features p2e --tests" + f"nix develop -c cargo {cargo_out} build --release " + f"--manifest-path {shlex.quote(str(manifest))} --features {shlex.quote(feature_arg)} --tests" ) ctx.logger.info("Building bebop p2e (tests)...") build_result = await stream_run_logger_async( @@ -150,7 +145,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cwd=bebop_dir, stdout_prefix="bebop p2e build", stderr_prefix="bebop p2e build", - env={**os.environ.copy(), **cargo_env}, + env=runtime_env, ) if build_result.returncode != 0: @@ -162,12 +157,20 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return # ── Run nextest ─────────────────────────────────────────────────────── - env = os.environ.copy() - env.update(cargo_env) + env = runtime_env.copy() env["OUT_PATH"] = build_dir - harness = nextest_harness_args(workload_toml, elf_root, env, p2e_bitstream=bitstream) + harness_args = [ + "--", + "--workload-toml", workload_toml, + "--bb-tests-root", elf_root, + "--p2e-bitstream", bitstream, + ] + if diff: + harness_args.append("--diff") + harness = shlex.join(harness_args) nextest_cmd = ( - f"nix develop -c cargo {cargo_out} nextest run --release --features p2e " + f"nix develop -c cargo {cargo_out} nextest run --release " + f"--manifest-path {shlex.quote(str(manifest))} --features {shlex.quote(feature_arg)} " f"--test test_p2e --config-file \"{nextest_config}\" {harness}" ) @@ -188,6 +191,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "bitstream": bitstream, "build_dir": build_dir, "test_type": test_type, + "diff": diff, "nextest_config": nextest_config, "workload_toml": workload_toml, } diff --git a/bbdev/api/steps/bebop/regression_harness.py b/bbdev/api/steps/bebop/regression_harness.py deleted file mode 100644 index 280bad97d..000000000 --- a/bbdev/api/steps/bebop/regression_harness.py +++ /dev/null @@ -1,12 +0,0 @@ -def nextest_harness_args( - workload_toml: str, - bb_tests_root: str, - env: dict, - *, - p2e_bitstream: str | None = None, -) -> str: - env["BEBOP_WORKLOAD_TOML"] = workload_toml - env["BEBOP_BB_TESTS_ROOT"] = bb_tests_root - if p2e_bitstream: - env["BEBOP_P2E_BITSTREAM"] = p2e_bitstream - return "" diff --git a/bbdev/api/steps/bebop/vcs/03_batch_event.step.py b/bbdev/api/steps/bebop/vcs/03_batch_event.step.py index 4d0cb1d93..d0e786b2e 100644 --- a/bbdev/api/steps/bebop/vcs/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/vcs/03_batch_event.step.py @@ -15,7 +15,7 @@ if bebop_path not in sys.path: sys.path.insert(0, bebop_path) -from regression import regression_workload_toml +from utils.workload_manifest import resolve_workload_toml from utils.event_common import check_result, get_origin_trace_id, require_chip from utils.path import get_buckyball_path, log_dir, rtl_dir, workloads_output_root from utils.search_workload import search_workload @@ -30,7 +30,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: chip = require_chip(input_data) bbdir = get_buckyball_path() test_type = input_data.get("test", "elf-tests") - manifest = regression_workload_toml(chip, "verilator", test_type, bbdir) + manifest = resolve_workload_toml(chip, "verilator", test_type, bbdir) except ValueError as error: await check_result(ctx, 1, extra_fields={"error": str(error)}, trace_id=origin_tid) return diff --git a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py index fc470fe80..dc2061484 100644 --- a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py +++ b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py @@ -34,7 +34,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: }, ) - data = {"chip": chip, "rushB": bool(body.get("rushB", False))} + data = {"chip": chip, "rushB": bool(body.get("rushB", False)), "diff": bool(body.get("diff", False))} if body.get("output_dir"): data["output_dir"] = body["output_dir"] await ctx.enqueue({"topic": "bebop.verilator.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index 7de3a6b45..47ea93663 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -54,8 +54,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: features = ["verilator"] if diff: manifest = Path(bbdir) / "examples" / "chips" / chip / "generated" / "bebop" / "Cargo.toml" - build_dir = manifest.parent - features.extend(["bemu", "difftest"]) + build_dir = str(manifest.parent) + features.append("bemu") dramsim_header = os.path.join(bbdir, "result", "include", "dramsim3.h") if not os.path.isfile(dramsim_header): ctx.logger.info(f"Nix environment is missing {dramsim_header}; rebuilding result") @@ -85,7 +85,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: str(jobs), ] ) - build_inner = f"cd {shlex.quote(build_dir)} && exec {cargo_build_cmd}" + build_inner = f"cd {shlex.quote(str(build_dir))} && exec {cargo_build_cmd}" build_cmd = f"nix develop -c sh -c {shlex.quote(build_inner)}" env = {**os.environ, "VSRC_PATH": vsrc_dir, **bebop_cargo_env(bbdir, chip)} ctx.logger.info(f"Building bebop verilator (diff={diff}) ...") diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index 4990b421f..7ea6c2a35 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -245,7 +245,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "chip": chip, "binary": binary_path, "log_dir": run_log, - "bank_diff": os.path.join(run_log, "bank_diff.ndjson") if diff else None, + "bank_diff": os.path.join(run_log, "diff.ndjson") if diff else None, "timestamp": timestamp, }, trace_id=origin_tid, diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 77244dc76..fad0b33a9 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -28,8 +28,7 @@ ) from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id -from regression import regression_workload_toml -from regression_harness import nextest_harness_args +from utils.workload_manifest import resolve_workload_toml config = { "name": "bebop-verilator-batch", @@ -77,7 +76,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return try: - workload_toml = regression_workload_toml( + workload_toml = resolve_workload_toml( chip, "verilator", test_type, bbdir, rushB=rushB, diff=diff ) except ValueError as e: @@ -112,7 +111,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: env.update(bebop_cargo_env(bbdir, chip)) env.update( { - "BEBOP_ARCH_CONFIG": chip, "VSRC_PATH": vsrc_dir, } ) @@ -120,23 +118,25 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: manifest = f"{bebop_dir}/Cargo.toml" features = "verilator" if diff: - features = "verilator,bemu,difftest" - preload = os.pathsep.join( - path - for path in ( - f"{bbdir}/result/lib/libdramsim3.so", - os.environ.get("LD_PRELOAD", ""), - ) - if path - ) - env["LD_PRELOAD"] = preload + manifest = f"{bbdir}/examples/chips/{chip}/generated/bebop/Cargo.toml" + features = "verilator,bemu" if input_data.get("clean-before", input_data.get("clean_before", False)): artifact_dir = os.path.join(env["CARGO_TARGET_DIR"], "test-artifacts") shutil.rmtree(artifact_dir, ignore_errors=True) ctx.logger.info(f"Cleaned previous bebop test artifacts: {artifact_dir}") - harness = nextest_harness_args(workload_toml, elf_root, env) + harness_args = [ + "--", + "--workload-toml", workload_toml, + "--bb-tests-root", elf_root, + "--arch-config", chip, + ] + if diff: + harness_args.append("--diff") + if rushB: + harness_args.extend(["--rushb-backend", "verilator"]) + harness = shlex.join(harness_args) nextest_cmd = ( f"nix develop -c cargo nextest run --release --manifest-path {shlex.quote(manifest)} " f"--features {shlex.quote(features)} --test test_verilator " diff --git a/bbdev/api/steps/config/01_install_event.step.py b/bbdev/api/steps/config/01_install_event.step.py index b7522758d..0012a32ef 100644 --- a/bbdev/api/steps/config/01_install_event.step.py +++ b/bbdev/api/steps/config/01_install_event.step.py @@ -112,6 +112,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bemu = step4.install_bemu(chip, bbdir, gen) ctx.logger.info(f"Installed bemu crate: {bemu}") + bebop = step4.install_bebop(gen) + ctx.logger.info(f"Installed bebop crate: {bebop}") + # C. TO Workload BUILD SYSTEM workload = step4.install_workload(chip, bbdir, chip_name, gen) ctx.logger.info(f"Installed workload defs: {workload}") @@ -136,6 +139,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "chip": chip_name, "arch": str(arch), "bemu": str(bemu), + "bebop": str(bebop), "workload": str(workload), } ) diff --git a/bbdev/api/steps/config/scripts/2_parameter_derivation.py b/bbdev/api/steps/config/scripts/2_parameter_derivation.py index e275646e4..285ba09ef 100644 --- a/bbdev/api/steps/config/scripts/2_parameter_derivation.py +++ b/bbdev/api/steps/config/scripts/2_parameter_derivation.py @@ -9,8 +9,6 @@ from pathlib import Path from typing import Any -_SYSTEM_FUNCTS = frozenset({0, 1, 16, 32, 33, 34, 35}) - def _die(msg: str) -> None: raise ValueError(msg) @@ -421,7 +419,7 @@ def _bemu_balls(repo: Path, topo: dict[str, Any]) -> list[dict[str, str]]: bid = entry.get("bid") if not isinstance(funct7, int) or not isinstance(bid, int): _die(f"ballISA entry must have funct7 and bid: {entry!r}") - if funct7 in _SYSTEM_FUNCTS: + if funct7 in {0, 1, 16, 32, 33, 34, 35}: continue ball_class = bid_to_class.get(bid) if not ball_class: diff --git a/bbdev/api/steps/config/scripts/3_json2proto.py b/bbdev/api/steps/config/scripts/3_json2proto.py index 068e0665d..bf8cafcb6 100644 --- a/bbdev/api/steps/config/scripts/3_json2proto.py +++ b/bbdev/api/steps/config/scripts/3_json2proto.py @@ -9,15 +9,14 @@ from pathlib import Path from typing import Any -_SCRIPTS = Path(__file__).resolve().parent -if str(_SCRIPTS) not in sys.path: - sys.path.insert(0, str(_SCRIPTS)) +if str(Path(__file__).resolve().parent) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parent)) import chip_pb2 as pb # noqa: E402 def _load_derive(): - path = _SCRIPTS / "2_parameter_derivation.py" + path = Path(__file__).resolve().with_name("2_parameter_derivation.py") spec = importlib.util.spec_from_file_location("parameter_derivation", path) if spec is None or spec.loader is None: raise RuntimeError(f"cannot load {path}") @@ -104,9 +103,7 @@ def _fill_ball(msg: pb.BallDomain, d: dict[str, Any], bbdir: Path) -> None: e.bid = item["bid"] -def _fill_rocket(msg: pb.RocketCoreConfig, d: dict[str, Any]) -> None: - msg.x_len = d["xLen"] - msg.pg_levels = d["pgLevels"] +def _fill_rocket(msg: pb.RocketCpuConfig, d: dict[str, Any]) -> None: msg.use_vm = d["useVM"] msg.use_zba = d["useZba"] msg.use_zbb = d["useZbb"] @@ -134,7 +131,7 @@ def _fill_rocket(msg: pb.RocketCoreConfig, d: dict[str, Any]) -> None: msg.btb.n_ras = btb["nRAS"] -def _fill_boom(msg: pb.BoomCoreConfig, d: dict[str, Any]) -> None: +def _fill_boom(msg: pb.BoomCpuConfig, d: dict[str, Any]) -> None: msg.fetch_width = d["fetchWidth"] msg.decode_width = d["decodeWidth"] msg.num_rob_entries = d["numRobEntries"] @@ -147,25 +144,12 @@ def _fill_boom(msg: pb.BoomCoreConfig, d: dict[str, Any]) -> None: msg.icache.n_ways = ic["nWays"] -def _check_core_kind(raw: dict[str, Any], pkg: str) -> str: - kind = raw.get("kind") +def _check_cpu_kind(cpu: dict[str, Any], pkg: str) -> str: + kind = cpu.get("kind") if kind not in ("rocket", "boom"): raise ValueError(f"{pkg}: kind must be 'rocket' or 'boom', got {kind!r}") - has_rocket = "rocketCore" in raw - has_boom = "boomCore" in raw - if kind == "rocket": - if not has_rocket: - raise ValueError(f"{pkg}: kind=rocket requires rocketCore") - if has_boom: - raise ValueError(f"{pkg}: kind=rocket forbids boomCore") - else: - if not has_boom: - raise ValueError(f"{pkg}: kind=boom requires boomCore") - if has_rocket: - raise ValueError(f"{pkg}: kind=boom forbids rocketCore") - bd = raw.get("balldomain") - if isinstance(bd, dict) and bd.get("ballNum", 0): - raise ValueError(f"{pkg}: kind=boom forbids balldomain.ballNum > 0") + if not isinstance(cpu.get("config"), dict): + raise ValueError(f"{pkg}: cpu.config must reference a TOML file") return kind @@ -191,13 +175,13 @@ def _fill_gp(msg: pb.GpDomainConfig, d: dict[str, Any], bbdir: Path) -> None: msg.lane_scale = d["laneScale"] -def _fill_core_param(msg: pb.CoreParamConfig, d: dict[str, Any], bbdir: Path) -> None: - msg.source_path = _rel(bbdir, d["_file"]) +def _fill_tile_params(msg: pb.TileParamConfig, d: dict[str, Any]) -> None: msg.core_data_bytes = d["coreDataBytes"] msg.x_len = d["xLen"] msg.vaddr_bits = d["vaddrBits"] msg.paddr_bits = d["paddrBits"] msg.pg_idx_bits = d["pgIdxBits"] + msg.pg_levels = d["pgLevels"] msg.n_pmps = d["nPMPs"] @@ -207,22 +191,25 @@ def _fill_core(ci: pb.CoreInstance, raw: dict[str, Any], meta: dict[str, Any], b ci.pkg = meta["pkg"] ci.config_path = meta["config_path"] ci.balldomain_base_dir = meta["balldomain_base_dir"] - kind = _check_core_kind(raw, meta["pkg"]) - ci.kind = kind + cpu = raw["cpu"] + kind = _check_cpu_kind(cpu, meta["pkg"]) + ci.cpu.kind = kind + ci.cpu.source_path = _rel(bbdir, cpu["_file"]) if "balldomain" in raw: _fill_ball(ci.balldomain, raw["balldomain"], bbdir) if "memdomain" in raw: _fill_mem(ci.mem, raw["memdomain"], bbdir) if kind == "rocket": - _fill_rocket(ci.rocket_core, raw["rocketCore"]) + _fill_rocket(ci.cpu.rocket, cpu["config"]) else: - _fill_boom(ci.boom_core, raw["boomCore"]) + _fill_boom(ci.cpu.boom, cpu["config"]) + bd = raw.get("balldomain") + if isinstance(bd, dict) and bd.get("ballNum", 0): + raise ValueError(f"{meta['pkg']}: kind=boom forbids balldomain.ballNum > 0") if "frontend" in raw: _fill_frontend(ci.frontend, raw["frontend"], bbdir) if "gpdomain" in raw: _fill_gp(ci.gp_domain, raw["gpdomain"], bbdir) - if "core" in raw: - _fill_core_param(ci.core, raw["core"], bbdir) def _fill_tile(tp: pb.TilePlacement, meta: dict[str, Any], proto: dict[str, Any]) -> None: @@ -230,6 +217,7 @@ def _fill_tile(tp: pb.TilePlacement, meta: dict[str, Any], proto: dict[str, Any] tp.virtual_bank_count = meta["virtual_bank_count"] tp.core_indices.extend(meta["core_indices"]) tp.mem_ball_channel_num = meta["mem_ball_channel_num"] + _fill_tile_params(tp.param, proto) dc = proto["privateDCache"] tp.private_dcache.enable = dc["enable"] tp.private_dcache.ways = dc["ways"] diff --git a/bbdev/api/steps/config/scripts/4_install.py b/bbdev/api/steps/config/scripts/4_install.py index 291fc7790..ed08879d9 100644 --- a/bbdev/api/steps/config/scripts/4_install.py +++ b/bbdev/api/steps/config/scripts/4_install.py @@ -8,16 +8,11 @@ import sys from pathlib import Path -_SCRIPTS = Path(__file__).resolve().parent -if str(_SCRIPTS) not in sys.path: - sys.path.insert(0, str(_SCRIPTS)) +if str(Path(__file__).resolve().parent) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parent)) import chip_pb2 as pb # noqa: E402 -_ENGINE = Path("bebop/src/nodes/bemu") -_BEMU_CRATE = _ENGINE / "chip" - - def load_chip(pb_path: Path) -> pb.Chip: chip = pb.Chip() chip.ParseFromString(pb_path.read_bytes()) @@ -110,7 +105,7 @@ def install_arch(src_pb: Path, gen: Path) -> Path: def install_bemu(chip: pb.Chip, bbdir: Path, gen: Path) -> Path: - src = bbdir / _BEMU_CRATE + src = bbdir / "bebop" / "src" / "nodes" / "bemu" / "chip" cargo = src / "Cargo.toml" build_rs = src / "build.rs" if not cargo.is_file(): @@ -125,6 +120,75 @@ def install_bemu(chip: pb.Chip, bbdir: Path, gen: Path) -> Path: return bemu / "Cargo.toml" +def install_bebop(gen: Path) -> Path: + bebop = gen.parent.parent / "generated" / "bebop" + bebop.mkdir(parents=True, exist_ok=True) + manifest = bebop / "Cargo.toml" + manifest.write_text( + """[workspace] +resolver = "2" + +[package] +name = "bebop" +version = "0.1.0" +edition = "2021" +build = "../../../../../bebop/build.rs" + +[[bin]] +name = "bebop" +path = "../../../../../bebop/src/main.rs" + +[[test]] +name = "test_verilator" +path = "../../../../../bebop/tests/test_verilator.rs" +harness = false +required-features = ["verilator"] + +[[test]] +name = "test_p2e" +path = "../../../../../bebop/tests/test_p2e.rs" +harness = false +required-features = ["p2e"] + +[features] +default = [] +verilator = ["dep:bebop-verilator"] +p2e = ["dep:bebop-p2e", "bebop-bemu?/p2e"] +bemu = ["dep:bebop-bemu"] + +[dependencies] +bebop-verilator = { path = "../../../../../bebop/src/nodes/verilator", optional = true } +bebop-p2e = { path = "../../../../../bebop/src/nodes/p2e", optional = true } +bebop-bemu = { path = "../../configs/generated/bemu", optional = true, default-features = false } +bebop-dasm = { path = "../../../../../bebop/src/nodes/lib/dasm" } +bebop-bank-hash = { path = "../../../../../bebop/src/nodes/lib/bank-hash" } +bebop-bemu-profile = { path = "../../../../../bebop/src/nodes/lib/bemu-profile" } +bebop-fd-redirect = { path = "../../../../../bebop/src/nodes/lib/fd-redirect" } +bebop-rtl-trace = { path = "../../../../../bebop/src/nodes/lib/rtl-trace" } +bebop-uart = { path = "../../../../../bebop/src/nodes/lib/uart" } +clap = { version = "4", features = ["derive"] } +libc = "0.2" +log = "0.4" +env_logger = "0.11" +nix = { version = "0.29", features = ["fs", "mman", "signal", "process"] } +toml = "0.8" +camino = "1.1" +snafu = "0.8" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +duct = "0.13" + +[dev-dependencies] +libtest-mimic = "0.8" +assert_cmd = "2" +walkdir = "2" +chrono = { version = "0.4", default-features = false, features = ["clock"] } +""", + encoding="utf-8", + ) + return manifest + + def install_workload(chip: pb.Chip, bbdir: Path, name: str, gen: Path) -> Path: if not chip.profiles: raise ValueError(f"chip {name}: no compiler profiles") diff --git a/bbdev/api/steps/config/scripts/proto/chip.proto b/bbdev/api/steps/config/scripts/proto/chip.proto index 71c62aca9..90e3ad00e 100644 --- a/bbdev/api/steps/config/scripts/proto/chip.proto +++ b/bbdev/api/steps/config/scripts/proto/chip.proto @@ -114,9 +114,7 @@ message RocketBtbConfig { uint32 n_ras = 3; } -message RocketCoreConfig { - uint32 x_len = 1; - uint32 pg_levels = 2; +message RocketCpuConfig { bool use_vm = 3; bool use_zba = 4; bool use_zbb = 5; @@ -140,7 +138,7 @@ message BoomICacheConfig { uint32 n_ways = 2; } -message BoomCoreConfig { +message BoomCpuConfig { uint32 fetch_width = 1; uint32 decode_width = 2; uint32 num_rob_entries = 3; @@ -170,14 +168,23 @@ message GpDomainConfig { string source_path = 7; } -message CoreParamConfig { +message TileParamConfig { uint32 core_data_bytes = 1; uint32 x_len = 2; uint32 vaddr_bits = 3; uint32 paddr_bits = 4; uint32 pg_idx_bits = 5; - uint32 n_pmps = 6; - string source_path = 7; + uint32 pg_levels = 6; + uint32 n_pmps = 7; +} + +message CpuConfig { + string kind = 1; + oneof cpu { + RocketCpuConfig rocket = 2; + BoomCpuConfig boom = 3; + } + string source_path = 4; } message CoreInstance { @@ -188,12 +195,9 @@ message CoreInstance { BallDomain balldomain = 5; MemDomainConfig mem = 6; string balldomain_base_dir = 7; - RocketCoreConfig rocket_core = 8; + CpuConfig cpu = 8; FrontendConfig frontend = 9; GpDomainConfig gp_domain = 10; - CoreParamConfig core = 11; - string kind = 12; - BoomCoreConfig boom_core = 13; } message BemuBall { @@ -233,6 +237,7 @@ message TilePlacement { uint32 mem_ball_channel_num = 4; PrivateDCacheConfig private_dcache = 5; SharedMemConfig shared_mem = 6; + TileParamConfig param = 7; } message WorkloadConfig { diff --git a/bbdev/api/steps/dc/scripts/emit_seq_mem_models.py b/bbdev/api/steps/dc/scripts/emit_seq_mem_models.py index 4dc110cba..2a18a4e77 100644 --- a/bbdev/api/steps/dc/scripts/emit_seq_mem_models.py +++ b/bbdev/api/steps/dc/scripts/emit_seq_mem_models.py @@ -14,12 +14,6 @@ from pathlib import Path -LINE = re.compile( - r"name\s+(?P\S+)\s+depth\s+(?P\d+)\s+width\s+(?P\d+)" - r"\s+ports\s+(?P\S+)(?:\s+mask_gran\s+(?P\d+))?$" -) - - def write_masked(lines: list[str], *, addr: str, data: str, mask: str, width: int, gran: int) -> None: lines.extend( [ @@ -96,7 +90,11 @@ def main() -> int: line = raw.strip() if not line: continue - match = LINE.fullmatch(line) + match = re.fullmatch( + r"name\s+(?P\S+)\s+depth\s+(?P\d+)\s+width\s+(?P\d+)" + r"\s+ports\s+(?P\S+)(?:\s+mask_gran\s+(?P\d+))?", + line, + ) if match is None: raise ValueError(f"invalid mems.conf entry: {line}") values = match.groupdict() diff --git a/bbdev/api/steps/ip/02_replace_event.step.py b/bbdev/api/steps/ip/02_replace_event.step.py index 873ee2c00..6312ec87f 100644 --- a/bbdev/api/steps/ip/02_replace_event.step.py +++ b/bbdev/api/steps/ip/02_replace_event.step.py @@ -16,9 +16,6 @@ from utils.path import get_buckyball_path, rtl_dir from replace import replace_sources -_SOURCE_LIST = {"dc": "dc_sources.list", "yosys": "yosys_sources.list"} - - config = { "name": "ip-replace", "description": "assemble top-scoped synthesis RTL plus generated SRAM macros", @@ -33,7 +30,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: try: chip = require_chip(input_data) consumer = input_data.get("consumer") or "dc" - if consumer not in _SOURCE_LIST: + if consumer not in {"dc", "yosys"}: raise ValueError("consumer must be dc or yosys") top_module = input_data.get("top") or "DigitalTop" if not isinstance(top_module, str) or not top_module: @@ -41,7 +38,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = Path(get_buckyball_path()).resolve() build_dir = Path(rtl_dir(bbdir, chip, "tapeout" if consumer == "dc" else "verilog", input_data.get("output_dir"))).resolve() - source_list_path = build_dir / _SOURCE_LIST[consumer] + source_list_path = build_dir / f"{consumer}_sources.list" if not source_list_path.is_file(): raise FileNotFoundError(f"missing source list: {source_list_path}") sources = [line.strip() for line in source_list_path.read_text().splitlines() if line.strip()] diff --git a/bbdev/api/steps/ip/scripts/generate.py b/bbdev/api/steps/ip/scripts/generate.py index 6ed0ca1d8..20fd4a41c 100644 --- a/bbdev/api/steps/ip/scripts/generate.py +++ b/bbdev/api/steps/ip/scripts/generate.py @@ -6,18 +6,13 @@ from collections import Counter from pathlib import Path -_DC = Path(__file__).resolve().parents[2] / "dc" / "scripts" -if str(_DC) not in sys.path: - sys.path.insert(0, str(_DC)) +if str(Path(__file__).resolve().parents[2] / "dc" / "scripts") not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "dc" / "scripts")) from tapeout import SramGeom, get_tapeout_contract from macro_compiler import run_macro_compiler from sram_compiler import generate_sram_dbs, leaf_names_from_macros -_MODULE = re.compile(r"(?m)^module\s+(\w+)\s*\(") -_INST = re.compile(r"(?m)^\s*([A-Za-z_][A-Za-z0-9_$]*)\s+[A-Za-z_][A-Za-z0-9_$]*\s*\(") - - def pad_mems_conf(text: str) -> str: lines = [] for raw in text.splitlines(): @@ -76,7 +71,7 @@ def parse_mems(text: str) -> list[tuple[str, int, int]]: def module_insts(text: str) -> dict[str, Counter[str]]: - hits = list(_MODULE.finditer(text)) + hits = list(re.finditer(r"(?m)^module\s+(\w+)\s*\(", text)) out: dict[str, Counter[str]] = {} for i, m in enumerate(hits): name = m.group(1) @@ -85,7 +80,10 @@ def module_insts(text: str) -> dict[str, Counter[str]]: start = m.end() end = hits[i + 1].start() if i + 1 < len(hits) else len(text) counts: Counter[str] = Counter() - for inst in _INST.finditer(text[start:end]): + for inst in re.finditer( + r"(?m)^\s*([A-Za-z_][A-Za-z0-9_$]*)\s+[A-Za-z_][A-Za-z0-9_$]*\s*\(", + text[start:end], + ): leaf = inst.group(1) if leaf in ("module", "assign", "wire", "input", "output"): raise ValueError(f"module {name}: refused instance parse {leaf!r}") diff --git a/bbdev/api/steps/ip/scripts/replace.py b/bbdev/api/steps/ip/scripts/replace.py index 4440051bc..264a6a2c2 100644 --- a/bbdev/api/steps/ip/scripts/replace.py +++ b/bbdev/api/steps/ip/scripts/replace.py @@ -4,18 +4,6 @@ import re from pathlib import Path -MODULE_NAME_RE = re.compile( - r"^\s*(?:\(\*.*?\*\)\s*)*module\s+([A-Za-z_][A-Za-z0-9_$]*)\b", - re.M, -) -INSTANCE_RE = re.compile( - r"^\s*([A-Za-z_][A-Za-z0-9_$]*)\s*(?:#\s*\(.*?\)\s*)?([A-Za-z_][A-Za-z0-9_$]*)\s*\(", - re.M | re.S, -) -COMMENT_RE = re.compile(r"/\*.*?\*/|//.*?$", re.M | re.S) -MODULE_TOKEN_RE = re.compile(r"\bmodule\b") - - def replace_sources( *, source_paths: list[str], @@ -32,10 +20,14 @@ def replace_sources( texts: dict[str, str] = {} for path in paths: text = path.read_text() - match = MODULE_NAME_RE.search(text) + match = re.search( + r"^\s*(?:\(\*.*?\*\)\s*)*module\s+([A-Za-z_][A-Za-z0-9_$]*)\b", + text, + re.M, + ) if match is None: - stripped = COMMENT_RE.sub("", text) - if MODULE_TOKEN_RE.search(stripped): + stripped = re.sub(r"/\*.*?\*/|//.*?$", "", text, flags=re.M | re.S) + if re.search(r"\bmodule\b", stripped): raise RuntimeError(f"no module declaration: {path}") continue name = match.group(1) @@ -53,7 +45,11 @@ def replace_sources( pending = [top_module] while pending: cur = pending.pop() - for child, _inst in INSTANCE_RE.findall(texts[cur]): + for child, _inst in re.findall( + r"^\s*([A-Za-z_][A-Za-z0-9_$]*)\s*(?:#\s*\(.*?\)\s*)?([A-Za-z_][A-Za-z0-9_$]*)\s*\(", + texts[cur], + re.M | re.S, + ): if child in modules and child not in reachable: reachable.add(child) pending.append(child) diff --git a/bbdev/api/steps/ip/scripts/sram_compiler.py b/bbdev/api/steps/ip/scripts/sram_compiler.py index b5adda9e2..44795be83 100644 --- a/bbdev/api/steps/ip/scripts/sram_compiler.py +++ b/bbdev/api/steps/ip/scripts/sram_compiler.py @@ -4,12 +4,12 @@ import sys from pathlib import Path -_HERE = Path(__file__).resolve().parent -_API = _HERE.parents[2] -_IP = _HERE.parents[4] / "thirdparty" / "soc-framework" / "ip" -for _p in (str(_API), str(_IP)): - if _p not in sys.path: - sys.path.insert(0, _p) +for path in ( + Path(__file__).resolve().parents[2], + Path(__file__).resolve().parents[4] / "thirdparty" / "soc-framework" / "ip", +): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) from smic180.compiler import generate_smic180_sram_dbs diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index 9bc9f1e5f..e31caf371 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -1,4 +1,5 @@ import os +import shutil import sys from motia import FlowContext, queue @@ -17,19 +18,6 @@ sys.path.insert(0, scripts_path) from bin_to_hex import bin_to_hex -KERNEL_MODELS = { - "bert", - "deepseekr1", - "gemma4", - "lenet", - "llama2", - "mobilenet", - "qwen3", - "resnet", - "stable-diffusion", - "yolo", -} - config = { "name": "kernel-build", "description": "build RISC-V kernel + rootfs for image via bb-tests/workloads/lib/kernel", @@ -86,8 +74,20 @@ def kernel_model(input_data: dict) -> str: raise ValueError("model must be a string") model = model.lower() - if model not in KERNEL_MODELS: - valid = ", ".join(sorted(KERNEL_MODELS)) + supported = { + "bert", + "deepseekr1", + "gemma4", + "lenet", + "llama2", + "mobilenet", + "qwen3", + "resnet", + "stable-diffusion", + "yolo", + } + if model not in supported: + valid = ", ".join(sorted(supported)) raise ValueError(f"unknown kernel model: {model}; valid models: {valid}") return model @@ -231,11 +231,26 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: payload_name = fw_payload_name(hart_params, cmake_model_name if model else "", chip) fw_payload_bin = os.path.join(output_dir, f"{payload_name}.bin") fw_payload_hex = os.path.join(output_dir, f"{payload_name}.hex") + fw_payload_elf = os.path.join(output_dir, f"{payload_name}.elf") + built_payload_elf = os.path.join( + kernel_build, + "opensbi", + "platform", + "buckyball", + "firmware", + "fw_payload.elf", + ) if not os.path.exists(fw_payload_bin): ctx.logger.error(f"{payload_name}.bin not found") await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) return + if not os.path.isfile(built_payload_elf): + ctx.logger.error(f"fw_payload.elf not found: {built_payload_elf}") + await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) + return + + shutil.copy2(built_payload_elf, fw_payload_elf) ctx.logger.info(f"Converting {fw_payload_bin} to Verilog hex format for P2E...") success = bin_to_hex(fw_payload_bin, fw_payload_hex, base_address=0x80000000) diff --git a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py index 09ee4bb3b..9696d013d 100644 --- a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py @@ -4,6 +4,7 @@ Generates Verilog via mill for bebop verilator """ import os +import shutil import sys from motia import FlowContext, queue @@ -61,12 +62,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: input_data.get("output_dir"), rushb=bool(input_data.get("rushB", False)), ) - os.makedirs(build_dir, exist_ok=True) + shutil.rmtree(build_dir, ignore_errors=True) + os.makedirs(build_dir) ctx.logger.info(f"Using mill config: {mill_config}") ctx.logger.info(f"Using build directory: {build_dir}") + command = mill_run.elaborate_cmd("sims.verilator.Elaborate", mill_config, build_dir) + if input_data.get("diff"): + command += " --difftest" returncode = ( await stream_run_logger_async( - cmd=mill_run.elaborate_cmd("sims.verilator.Elaborate", mill_config, build_dir), + cmd=command, logger=ctx.logger, cwd=arch, stdout_prefix="bebop verilator verilog", diff --git a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py index 2b8784bb7..d89e891c4 100644 --- a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py +++ b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py @@ -1,4 +1,5 @@ import os +import shutil import sys from motia import FlowContext, queue @@ -39,13 +40,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: mill_config, build_dir = rtl_out( bbdir, chip, "p2e", input_data.get("output_dir"), ) - os.makedirs(build_dir, exist_ok=True) + shutil.rmtree(build_dir, ignore_errors=True) + os.makedirs(build_dir) ctx.logger.info(f"Using mill config: {mill_config}") ctx.logger.info(f"Using build directory: {build_dir}") prefix = "bebop p2e verilog" + command = mill_run.elaborate_cmd("sims.p2e.Elaborate", mill_config, build_dir) + if input_data.get("diff"): + command += " --difftest" returncode = ( await stream_run_logger_async( - cmd=mill_run.elaborate_cmd("sims.p2e.Elaborate", mill_config, build_dir), + cmd=command, logger=ctx.logger, cwd=arch, stdout_prefix=prefix, diff --git a/bbdev/api/steps/mill/scripts/mill.py b/bbdev/api/steps/mill/scripts/mill.py index 4fe212310..bd6b3dbea 100644 --- a/bbdev/api/steps/mill/scripts/mill.py +++ b/bbdev/api/steps/mill/scripts/mill.py @@ -7,12 +7,12 @@ def elaborate_cmd(main: str, config: str, out: str, *, seq_mem: bool = False) -> str: extra = "" if seq_mem: - extra = ( + extra += ( " -lowering-options=disallowLocalVariables" f" --repl-seq-mem --repl-seq-mem-file={shlex.quote(os.path.join(out, 'mems.conf'))}" ) return ( - f"mill -i __.test.runMain {main} {config} " + f"flock /tmp/buckyball-mill.lock mill -i __.test.runMain {main} {config} " "--disable-annotation-unknown --strip-debug-info -O=debug " f"--split-verilog -o={shlex.quote(out)}{extra}" ) diff --git a/bbdev/api/steps/pt/scripts/power.py b/bbdev/api/steps/pt/scripts/power.py index 48849237b..fcc3251d3 100644 --- a/bbdev/api/steps/pt/scripts/power.py +++ b/bbdev/api/steps/pt/scripts/power.py @@ -6,15 +6,12 @@ from pathlib import Path -_POWER_VALUE = re.compile(r"([-+]?\d+(?:\.\d*)?(?:[eE][-+]?\d+)?)\s*([munp]?W)\b") -_UNIT_TO_WATTS = {"W": 1.0, "mW": 1e-3, "uW": 1e-6, "nW": 1e-9, "pW": 1e-12} - - def _watts(value: str) -> float: - match = _POWER_VALUE.fullmatch(value) + match = re.fullmatch(r"([-+]?\d+(?:\.\d*)?(?:[eE][-+]?\d+)?)\s*([munp]?W)\b", value) if match is None: raise ValueError(f"unsupported power value: {value}") - return float(match.group(1)) * _UNIT_TO_WATTS[match.group(2)] + scale = {"W": 1.0, "mW": 1e-3, "uW": 1e-6, "nW": 1e-9, "pW": 1e-12} + return float(match.group(1)) * scale[match.group(2)] def read_dynamic_power(report: str | Path) -> dict[str, str] | None: diff --git a/bbdev/api/steps/regression/00_buildbitstream_event.step.py b/bbdev/api/steps/regression/00_buildbitstream_event.step.py index a93f0e17e..b314f037e 100644 --- a/bbdev/api/steps/regression/00_buildbitstream_event.step.py +++ b/bbdev/api/steps/regression/00_buildbitstream_event.step.py @@ -51,6 +51,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: } output_dir = input_data.get("output_dir") or input_data.get("output-dir") if output_dir: - data["build_dir"] = output_dir + data["output_dir"] = output_dir await ctx.enqueue({"topic": "bebop.p2e.verilog", "data": data}) diff --git a/bbdev/api/steps/regression/02_eval_performance_event.step.py b/bbdev/api/steps/regression/02_eval_performance_event.step.py index d2db347e9..02f16ad6b 100644 --- a/bbdev/api/steps/regression/02_eval_performance_event.step.py +++ b/bbdev/api/steps/regression/02_eval_performance_event.step.py @@ -26,9 +26,10 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path, rtl_dir, log_dir +from utils.path import bebop_cargo_env, get_buckyball_path, log_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id +from utils.model import model_layout_name sys.path.insert(0, os.path.join(get_buckyball_path(), "bb-tests", "workloads", "scripts")) import build as workload_build # noqa: E402 @@ -62,17 +63,9 @@ def _load_sibling(rel): return mod -_workload = _load_sibling("../workload/01_build_event.step.py") _kernel = _load_sibling("../kernel/01_build_event.step.py") _p2e = _load_sibling("../bebop/p2e/03_runworkload_event.step.py") -MODEL_LAYOUT = _workload.MODEL_LAYOUT -PERFETTO_SCRIPT = ( - Path(get_buckyball_path()) - / "bb-tests" / "workloads" / "src" / "ModelTest" - / "e2e" / "framework" / "trace" / "perfetto.py" -) - def _fail(ctx, origin_tid, error, **extra): fields = {"task": "regression.eval-performance", "error": error} @@ -82,8 +75,7 @@ def _fail(ctx, origin_tid, error, **extra): def _workload_build(bbdir, chip, model, logger, task_scope): model_key = model.lower() - if MODEL_LAYOUT.get(model_key) is None: - raise ValueError(f"Unknown model: {model}") + model_layout_name(model_key) workload_build.build_workload( bbdir, chip, model=model_key, logger=logger, task_scope=task_scope ) @@ -131,27 +123,33 @@ def _p2e_run_cmds(bbdir, bitstream, image_name, chip, input_data): multi_fpga = bool(input_data.get("multi-fpga", False)) if not multi_fpga and _p2e.case_uses_multi_fpga(build_dir): multi_fpga = True - vsrc_dir = rtl_dir(bbdir, chip, "p2e", input_data.get("vsrc_dir")) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") run_log = log_dir(bbdir, chip, "p2e", timestamp, "p2e", image_name, input_data.get("vsrc_dir")) os.makedirs(run_log, exist_ok=True) - runtime_cmd = ( - f"env BEBOP_P2E_RUNTIME_ONLY=1 BEBOP_P2E_REBUILD_RUNTIME=1 " - f'cargo run --release --features p2e -- build p2e ' - f'--rtl-dir="{vsrc_dir}" ' - f'--out-dir="{build_dir}"' - ) - run_cmd = ( - f"cargo run --release --features p2e " - f"--config=\"env.OUT_PATH='{build_dir}'\" " - f"-- run p2e " - f'--image="{image_path}" ' - f'--bitstream="{bitstream}" ' - f'--log-dir="{log_dir}"' + runtime_lib_dir = os.path.join( + build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm" ) + runtime = os.path.join(build_dir, "bebop-p2e") + artifacts = [ + os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg"), + os.path.join(runtime_lib_dir, "libvCtb.so"), + runtime, + ] + missing = [path for path in artifacts if not os.path.isfile(path)] + if missing: + raise FileNotFoundError(f"P2E runtime artifacts missing: {missing}") + + run_args = [ + runtime, + "run", "p2e", + "--image", image_path, + "--bitstream", bitstream, + "--log-dir", run_log, + ] if multi_fpga: - run_cmd += " --multi-fpga" - return runtime_cmd, run_cmd, f"{bbdir}/bebop", run_log, build_dir + run_args.append("--multi-fpga") + run_env = {**os.environ, **bebop_cargo_env(bbdir, chip)} + return shlex.join(run_args), run_env, f"{bbdir}/bebop", run_log def _perfetto_cmd(trace_dir, trace_toml, mlir_files): @@ -159,7 +157,7 @@ def _perfetto_cmd(trace_dir, trace_toml, mlir_files): for mlir in mlir_files: args += ["--mlir", str(mlir)] return ( - f"{sys.executable} {shlex.quote(str(PERFETTO_SCRIPT))} " + f"{sys.executable} {shlex.quote(str(Path(get_buckyball_path()) / 'bb-tests/workloads/src/ModelTest/e2e/framework/trace/perfetto.py'))} " + " ".join(shlex.quote(a) for a in args) ) @@ -203,7 +201,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: model_results = [] for model in models: model_key = model.lower() - if MODEL_LAYOUT.get(model_key) is None: + try: + model_layout_name(model_key) + except ValueError: ctx.logger.error(f"Unknown model: {model}") await _fail(ctx, origin_tid, "unknown_model", model=model) return @@ -265,7 +265,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"[eval-performance] model {model}: p2e runworkload") image_name = os.path.splitext(os.path.basename(fw_hex))[0] try: - runtime_cmd, run_cmd, bebop_cwd, run_log, build_dir = _p2e_run_cmds( + run_cmd, run_env, bebop_cwd, run_log = _p2e_run_cmds( bbdir, bitstream, image_name, chip, input_data ) except FileNotFoundError as e: @@ -277,40 +277,27 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra["build_dir"] = os.path.dirname( os.path.dirname(os.path.abspath(bitstream)) ) + elif str(e).startswith("P2E runtime artifacts missing"): + err = "p2e_runtime_missing" await _fail(ctx, origin_tid, err, **extra) return - rt_result = await stream_run_logger_async( - cmd=runtime_cmd, logger=ctx.logger, cwd=bebop_cwd, - stdout_prefix=f"p2e runtime {model}", - stderr_prefix=f"p2e runtime {model}", - ) - rtcfg = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") - libvctb = os.path.join( - build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so" - ) - if rt_result.returncode != 0 or not all( - os.path.isfile(p) for p in (rtcfg, libvctb) - ): - await _fail(ctx, origin_tid, "p2e_runtime_failed", - model=model, returncode=rt_result.returncode) - return run_result = await stream_run_logger_async( - cmd=run_cmd, logger=ctx.logger, cwd=bebop_cwd, + cmd=run_cmd, logger=ctx.logger, cwd=bebop_cwd, env=run_env, stdout_prefix=f"p2e run {model}", stderr_prefix=f"p2e run {model}", ) if run_result.returncode != 0: await _fail(ctx, origin_tid, "p2e_run_failed", model=model, returncode=run_result.returncode, - log_dir=log_dir) + log_dir=run_log) return uart_path = Path(run_log) / "uart_hart_0.log" if not uart_path.is_file(): uart_path = Path(run_log) / "uart.log" if not uart_path.is_file(): - ctx.logger.error(f"uart log missing under {log_dir}") - await _fail(ctx, origin_tid, "uart_missing", model=model, log_dir=log_dir) + ctx.logger.error(f"uart log missing under {run_log}") + await _fail(ctx, origin_tid, "uart_missing", model=model, log_dir=run_log) return try: acc_i = accuracy_from_uart(spec["metric"], uart_path.read_text()) diff --git a/bbdev/api/steps/regression/scripts/accuracy.py b/bbdev/api/steps/regression/scripts/accuracy.py index f19cdee10..601ad77b1 100644 --- a/bbdev/api/steps/regression/scripts/accuracy.py +++ b/bbdev/api/steps/regression/scripts/accuracy.py @@ -9,10 +9,6 @@ import re from typing import Any -_TOP1 = re.compile(r"^top1=(\d+)/(\d+)\s*$", re.M) -_MAP = re.compile(r"^map=([0-9]*\.?[0-9]+)\s*$", re.M) - - def accuracy_from_uart(metric: str, uart: str) -> float: if not isinstance(metric, str) or not metric: raise ValueError("accuracy metric must be a non-empty string") @@ -27,7 +23,7 @@ def accuracy_from_uart(metric: str, uart: str) -> float: def _top1(uart: str) -> float: - hits = _TOP1.findall(uart) + hits = re.findall(r"^top1=(\d+)/(\d+)\s*$", uart, re.M) if not hits: raise ValueError("uart missing top1=/ line") correct_s, total_s = hits[-1] @@ -41,7 +37,7 @@ def _top1(uart: str) -> float: def _map(uart: str) -> float: - hits = _MAP.findall(uart) + hits = re.findall(r"^map=([0-9]*\.?[0-9]+)\s*$", uart, re.M) if not hits: raise ValueError("uart missing map= line") val = float(hits[-1]) diff --git a/bbdev/api/steps/regression/scripts/dc_area.py b/bbdev/api/steps/regression/scripts/dc_area.py index b49b7f92a..64c07d186 100644 --- a/bbdev/api/steps/regression/scripts/dc_area.py +++ b/bbdev/api/steps/regression/scripts/dc_area.py @@ -1,10 +1,7 @@ import re -_AREA = re.compile(r"Total cell area:\s+([\d.]+)") - - def area_mm2_from_rpt(text: str) -> float: - m = _AREA.search(text) + m = re.search(r"Total cell area:\s+([\d.]+)", text) if not m: raise ValueError("Total cell area not found") return float(m.group(1)) / 1e6 diff --git a/bbdev/api/steps/regression/scripts/model_layout.py b/bbdev/api/steps/regression/scripts/model_layout.py index e8fe08492..b4d2c491a 100644 --- a/bbdev/api/steps/regression/scripts/model_layout.py +++ b/bbdev/api/steps/regression/scripts/model_layout.py @@ -7,52 +7,7 @@ """ import os from pathlib import Path - -MODEL_LAYOUT = { - "lenet": "LeNet", - "mobilenet": "MobileNetV3", - "resnet": "ResNet18", - "yolo": "YOLO26", - "bert": "Bert", - "qwen3": "Qwen3", - "gemma4": "Gemma4", - "deepseekr1": "DeepSeekR1", - "llama2": "llama2", - "stable-diffusion": "StableDiffusion", - "whisper": "Whisper", - "buddynext": "BuddyNext", -} - -MODEL_PERFETTO = { - "lenet": { - "trace_toml": "bb-tests/workloads/src/ModelTest/e2e/models/models/LeNet/trace/trace-nodes.toml", - "mlir_files": [ - "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/LeNet/subgraph0_linalg.mlir", - "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/LeNet/subgraph0_buckyball.mlir", - ], - }, - "mobilenet": { - "trace_toml": "bb-tests/workloads/src/ModelTest/e2e/models/models/MobileNetV3/trace/trace.toml", - "mlir_files": [ - "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/MobileNetV3/subgraph0_linalg.mlir", - "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/MobileNetV3/subgraph0_buckyball.mlir", - ], - }, - "resnet": { - "trace_toml": "bb-tests/workloads/src/ModelTest/e2e/models/models/ResNet18/trace/trace.toml", - "mlir_files": [ - "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/ResNet18/subgraph0_linalg.mlir", - "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/ResNet18/subgraph0_buckyball.mlir", - ], - }, - "yolo": { - "trace_toml": "bb-tests/workloads/src/ModelTest/e2e/models/models/YOLO26/trace/trace.toml", - "mlir_files": [ - "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/YOLO26/subgraph0_linalg.mlir", - "bb-tests/output/{chip}/workloads/src/ModelTest/e2e/models/archs/buckyball/{chip}/YOLO26/subgraph0_buckyball.mlir", - ], - }, -} +from utils.model import model_layout_name def _archs_root(bbdir: str, chip: str) -> Path: @@ -72,10 +27,7 @@ def _archs_root(bbdir: str, chip: str) -> Path: def layout_name(model: str) -> str: - layout = MODEL_LAYOUT.get(model.lower()) - if layout is None: - raise ValueError(f"missing layout mapping for model: {model}") - return layout + return model_layout_name(model) def chip_output_dir(bbdir: str, chip: str, model: str) -> Path: @@ -108,15 +60,23 @@ def bridge_model_layout(bbdir: str, chip: str, model: str) -> Path: def perfetto_inputs(bbdir: str, chip: str, model: str) -> dict: - spec = MODEL_PERFETTO.get(model.lower()) - if spec is None: + if model.lower() not in {"lenet", "mobilenet", "resnet", "yolo"}: raise KeyError(f"no perfetto spec for model: {model}") - trace_toml = Path(bbdir) / spec["trace_toml"] + layout = layout_name(model) + trace_name = "trace-nodes.toml" if model.lower() == "lenet" else "trace.toml" + trace_toml = ( + Path(bbdir) + / "bb-tests/workloads/src/ModelTest/e2e/models/models" + / layout + / "trace" + / trace_name + ) if not trace_toml.is_file(): raise FileNotFoundError(f"perfetto trace toml missing: {trace_toml}") + output = _archs_root(bbdir, chip) / chip / layout mlir_files = [] - for rel in spec["mlir_files"]: - path = Path(bbdir) / rel.replace("{chip}", chip) + for name in ("subgraph0_linalg.mlir", "subgraph0_buckyball.mlir"): + path = output / name if not path.is_file(): raise FileNotFoundError(f"perfetto mlir file missing: {path}") mlir_files.append(path) diff --git a/bbdev/api/steps/regression/scripts/result.py b/bbdev/api/steps/regression/scripts/result.py index b09f0c63d..954e44c41 100644 --- a/bbdev/api/steps/regression/scripts/result.py +++ b/bbdev/api/steps/regression/scripts/result.py @@ -1,17 +1,9 @@ import json -import os from pathlib import Path def result_path(bbdir: str) -> Path: - rel = os.environ.get("EVAL_RESULT_PATH", "chipcrowd-eval-result.json") - root = Path(bbdir).resolve() - target = (root / rel).resolve() - try: - target.relative_to(root) - except ValueError as e: - raise ValueError(f"EVAL_RESULT_PATH escapes repo: {rel}") from e - return target + return Path(bbdir).resolve() / "chipcrowd-eval-result.json" def merge_metrics(bbdir: str, **fields) -> dict: diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index 314b14a7f..f67d91238 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -1,4 +1,5 @@ import os +import re import shlex import sys import tomllib @@ -13,20 +14,18 @@ from utils.stream_run import stream_run_logger from .waive import apply_waivers +config_scripts = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..", "config", "scripts") +) +sys.path.insert(0, config_scripts) + def load_chip(bbdir: str, chip: str): - config_scripts = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "..", "config", "scripts") + import chip_pb2 + + path = ( + Path(bbdir) / "examples" / "chips" / chip / "configs" / "generated" / "chip.pb" ) - if config_scripts not in sys.path: - sys.path.insert(0, config_scripts) - try: - import chip_pb2 - except ImportError as e: - raise FileNotFoundError( - f"missing {os.path.join(config_scripts, 'chip_pb2.py')}; run bbdev config --install" - ) from e - path = Path(bbdir) / "examples" / "chips" / chip / "configs" / "generated" / "chip.pb" if not path.is_file(): raise FileNotFoundError(f"missing {path}; run bbdev config --install") msg = chip_pb2.Chip() @@ -36,16 +35,40 @@ def load_chip(bbdir: str, chip: str): return msg -def ball_domain(chip): - d0 = chip.cores[0].balldomain - key = [(m.ball_id, m.ball_dir, m.in_bw, m.out_bw) for m in d0.mappings] - isa = [(e.mnemonic, e.funct7, e.bid) for e in d0.isa] - for core in chip.cores[1:]: - k = [(m.ball_id, m.ball_dir, m.in_bw, m.out_bw) for m in core.balldomain.mappings] - i = [(e.mnemonic, e.funct7, e.bid) for e in core.balldomain.isa] - if k != key or i != isa: - raise ValueError("chip.pb cores have different balldomains") - return d0 +def load_chip_uvm(bbdir: str, chip: str) -> dict[str, list[str]]: + path = Path(bbdir) / "examples" / "chips" / chip / "configs" / "chip.toml" + config = tomllib.loads(path.read_text())["uvm"] + if set(config) != {"balls", "ips"}: + raise ValueError(f"{path}: [uvm] must define exactly balls and ips") + if not config["balls"] and not config["ips"]: + raise ValueError(f"{path}: [uvm] has no targets") + return config + + +def ball_domain(chip, ball: str | None = None): + if ball is None: + domains = [core.balldomain for core in chip.cores] + domain = domains[0] + for other in domains[1:]: + if other.mappings != domain.mappings or other.isa != domain.isa: + raise ValueError("chip.pb cores have different balldomains") + return domain + + matches = [] + for core in chip.cores: + mappings = [m for m in core.balldomain.mappings if m.ball_dir == ball] + if mappings: + mapping = mappings[0] + isa = [e for e in core.balldomain.isa if e.bid == mapping.ball_id] + matches.append((core.balldomain, mapping, isa)) + if not matches: + raise ValueError(f"ball {ball!r} not in chip.pb") + + domain, mapping, isa = matches[0] + for _, other_mapping, other_isa in matches[1:]: + if other_mapping != mapping or other_isa != isa: + raise ValueError(f"ball {ball!r} has different definitions across cores") + return domain def selected_mappings(domain, ball: str | None): @@ -71,13 +94,32 @@ def vcs_defines(domain, mapping, bank_entries: int): return defs +def smatmul_accumulator_filename(rtl_dir: Path) -> str: + text = (rtl_dir / "SMatMulUnit.sv").read_text() + modules = re.findall(r"\b(accumulator_\d+x\d+)\s+accumulator_ext\s*\(", text) + if len(modules) != 1: + raise ValueError( + f"expected one SMatMul accumulator instance, found {modules!r}" + ) + return f"{modules[0]}.sv" + + def _filelist( - verify_dir: Path, ball_dir: str, uvm_rel: str, rtl_rel: str, sim_dir: Path + verify_dir: Path, + ball_dir: str, + uvm_rel: str, + rtl_rel: str, + rtl_dir: Path, + sim_dir: Path, ) -> str: src = verify_dir / "filelists" / f"{ball_dir}_ball.f" dst = sim_dir / f"{ball_dir}_ball.f" sim_dir.mkdir(parents=True, exist_ok=True) text = src.read_text() + if "@SMATMUL_ACCUMULATOR@" in text: + text = text.replace( + "@SMATMUL_ACCUMULATOR@", smatmul_accumulator_filename(rtl_dir) + ) dst.write_text(text.replace("@UVM@", uvm_rel).replace("@RTL@", rtl_rel)) return str(dst.relative_to(verify_dir)) @@ -86,7 +128,16 @@ def load_ip(bbdir: str, name: str) -> dict: registry = Path(bbdir) / "verify" / "uvm" / "ip.toml" if name not in tomllib.loads(registry.read_text()).get("ips", []): raise ValueError(f"IP {name!r} is not registered in {registry}") - root = Path(bbdir) / "arch" / "src" / "main" / "scala" / "framework" / "mem-core" / name + root = ( + Path(bbdir) + / "arch" + / "src" + / "main" + / "scala" + / "framework" + / "mem-core" + / name + ) resources = root / "src" / "main" / "resources" targets = [ { @@ -108,7 +159,9 @@ def load_ip(bbdir: str, name: str) -> dict: } -def _ip_filelist(resources: Path, source: str, sim_dir: Path, bbdir: str, rtl: Path) -> Path: +def _ip_filelist( + resources: Path, source: str, sim_dir: Path, bbdir: str, rtl: Path +) -> Path: dst = sim_dir / source sim_dir.mkdir(parents=True, exist_ok=True) text = (resources / source).read_text() @@ -120,6 +173,20 @@ def _ip_filelist(resources: Path, source: str, sim_dir: Path, bbdir: str, rtl: P return dst +def check_uvm_result(result, target: str) -> None: + if result.returncode != 0: + raise RuntimeError(f"UVM failed for {target}") + counts = dict( + re.findall(r"UVM_(ERROR|FATAL)\s*:\s*(\d+)", result.stdout + result.stderr) + ) + if set(counts) != {"ERROR", "FATAL"}: + raise RuntimeError(f"UVM report summary missing for {target}") + if counts != {"ERROR": "0", "FATAL": "0"}: + raise RuntimeError( + f"UVM failed for {target}: errors={counts['ERROR']} fatals={counts['FATAL']}" + ) + + def build_ip(bbdir: str, chip: str, name: str, ctx) -> dict: config = load_ip(bbdir, name) root = Path(bbdir) / config["root"] @@ -203,7 +270,10 @@ def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str) -> dict: model = manifest.parent / "target" / "debug" / f"lib{crate}" sim_root = root / "build" / "uvm" / chip - if any(not (sim_root / target["name"] / "simv").is_file() for target in config["targets"]): + if any( + not (sim_root / target["name"] / "simv").is_file() + for target in config["targets"] + ): config = build_ip(bbdir, chip, name, ctx) coverage = [] @@ -211,7 +281,7 @@ def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str) -> dict: simv = sim_root / target["name"] / "simv" cov_dir = Path(cov_root) / target["name"] / "coverage" script = ( - f"env LD_LIBRARY_PATH=\"$VCS_RUNTIME_LIBRARY_PATH\" {shlex.quote(str(simv))} " + f'env LD_LIBRARY_PATH="$VCS_RUNTIME_LIBRARY_PATH" {shlex.quote(str(simv))} ' f"-sv_lib {shlex.quote(str(model))} +UVM_TESTNAME={shlex.quote(target['test'])} " f"-cm line+cond+tgl+assert -cm_name {shlex.quote(target['test'])}" ) @@ -226,8 +296,7 @@ def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str) -> dict: stdout_prefix="uvm run", stderr_prefix="uvm run", ) - if result.returncode != 0: - raise RuntimeError(f"UVM failed for IP {name} target={target['name']}") + check_uvm_result(result, f"IP {name} target={target['name']}") cov_dir.mkdir(parents=True, exist_ok=True) urg = ( @@ -256,7 +325,13 @@ def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str) -> dict: ) index = index_dir / "index.txt" index.write_text("\n".join(rows) + "\n") - return {"chip": chip, "ip": name, "targets": [t["name"] for t in config["targets"]], "index": str(index), "log": cov_root} + return { + "chip": chip, + "ip": name, + "targets": [t["name"] for t in config["targets"]], + "index": str(index), + "log": cov_root, + } def build_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx) -> None: @@ -268,12 +343,18 @@ def build_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx) sim_dir = verify_dir / "build" / chip_name uvm_rel = os.path.relpath(Path(bbdir) / "verify" / "uvm", verify_dir) rtl_rel = os.path.relpath(rtl_dir, verify_dir) - flist = _filelist(verify_dir, ball, uvm_rel, rtl_rel, sim_dir) + flist = _filelist(verify_dir, ball, uvm_rel, rtl_rel, rtl_dir, sim_dir) cargo = ( f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " f"cargo build --manifest-path {shlex.quote(str(casegen))}" ) - r = stream_run_logger(cmd=cargo, logger=ctx.logger, cwd=bbdir, stdout_prefix="uvm dpi", stderr_prefix="uvm dpi") + r = stream_run_logger( + cmd=cargo, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm dpi", + stderr_prefix="uvm dpi", + ) if r.returncode != 0: raise RuntimeError(f"cargo build failed for {ball}") crate = tomllib.loads(casegen.read_text()).get("package", {}).get("name") @@ -296,40 +377,64 @@ def build_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx) f"-f {shlex.quote(flist)}" ) vcs = f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command zsh -c {shlex.quote(script)}" - r = stream_run_logger(cmd=vcs, logger=ctx.logger, cwd=bbdir, stdout_prefix="uvm vcs", stderr_prefix="uvm vcs") + r = stream_run_logger( + cmd=vcs, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm vcs", + stderr_prefix="uvm vcs", + ) if r.returncode != 0: raise RuntimeError(f"vcs failed for {ball}") -def run_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx, cov_dir: str) -> None: +def run_ball( + bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx, cov_dir: str +) -> None: ball = mapping.ball_dir verify_dir = Path(bbdir) / "examples" / "balls" / ball / "verify" simv = verify_dir / "build" / chip_name / "simv" if not simv.is_file(): build_ball(bbdir, chip_name, mill_cfg, domain, mapping, ctx) - crate = tomllib.loads((verify_dir / "casegen" / "Cargo.toml").read_text())["package"]["name"] + crate = tomllib.loads((verify_dir / "casegen" / "Cargo.toml").read_text())[ + "package" + ]["name"] dpi = verify_dir / "casegen" / "target" / "debug" / f"lib{crate.replace('-', '_')}" test = f"{ball}_ball_test" verify_config = verify_dir / "build" / chip_name / "verify_config.env" bank_entries = load_chip(bbdir, chip_name).cores[0].mem.bank.entries - verify_config.write_text(f"bank_entries={bank_entries}\nball_id={mapping.ball_id}\n") + verify_config.write_text( + f"bank_entries={bank_entries}\nball_id={mapping.ball_id}\n" + ) script = ( f"cd {shlex.quote(str(verify_dir))} && " - f"env LD_LIBRARY_PATH=\"$VCS_RUNTIME_LIBRARY_PATH\" BB_VERIFY_CONFIG={shlex.quote(str(verify_config))} {shlex.quote(str(simv))} " + f'env LD_LIBRARY_PATH="$VCS_RUNTIME_LIBRARY_PATH" BB_VERIFY_CONFIG={shlex.quote(str(verify_config))} ' + f"{shlex.quote(str(simv))} " f"-sv_lib {shlex.quote(str(dpi))} " f"+UVM_TESTNAME={shlex.quote(test)} +BID={mapping.ball_id} " f"-cm line+cond+tgl+assert -cm_name {shlex.quote(test)}" ) cmd = f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command zsh -ic {shlex.quote(script)}" - r = stream_run_logger(cmd=cmd, logger=ctx.logger, cwd=bbdir, stdout_prefix="uvm run", stderr_prefix="uvm run") - if r.returncode != 0: - raise RuntimeError(f"uvm run failed for {ball} test={test}") + r = stream_run_logger( + cmd=cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm run", + stderr_prefix="uvm run", + ) + check_uvm_result(r, f"{ball} test={test}") Path(cov_dir).mkdir(parents=True, exist_ok=True) urg = ( f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " f"urg -dir {shlex.quote(str(simv))}.vdb -format text -report {shlex.quote(cov_dir)}" ) - r = stream_run_logger(cmd=urg, logger=ctx.logger, cwd=str(verify_dir), stdout_prefix="uvm urg", stderr_prefix="uvm urg") + r = stream_run_logger( + cmd=urg, + logger=ctx.logger, + cwd=str(verify_dir), + stdout_prefix="uvm urg", + stderr_prefix="uvm urg", + ) if r.returncode != 0: raise RuntimeError(f"urg failed for {ball}") @@ -356,7 +461,7 @@ def dashboard_summary(cov_dir: str) -> dict[str, str]: def run_chip(bbdir: str, chip: str, ball: str | None, ctx, do_run: bool) -> dict: msg = load_chip(bbdir, chip) - domain = ball_domain(msg) + domain = ball_domain(msg, ball) mill_cfg = msg.mill.verilator_config maps = selected_mappings(domain, ball) ran = [] @@ -400,12 +505,39 @@ def run_chip(bbdir: str, chip: str, ball: str | None, ctx, do_run: bool) -> dict return info -def run_uvm(bbdir: str, chip: str, ball: str | None, ip: str | None, ctx, do_run: bool) -> dict: - if ip is None: +def run_uvm( + bbdir: str, chip: str, ball: str | None, ip: str | None, ctx, do_run: bool +) -> dict: + if ball is not None: return run_chip(bbdir, chip, ball, ctx, do_run) - if do_run: - stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - run_root = log_dir(bbdir, chip, "verilog", stamp, "uvm", ip) - return run_ip(bbdir, chip, ip, ctx, run_root) - config = build_ip(bbdir, chip, ip, ctx) - return {"chip": chip, "ip": ip, "targets": [target["name"] for target in config["targets"]]} + if ip is not None: + if do_run: + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + run_root = log_dir(bbdir, chip, "verilog", stamp, "uvm", ip) + return run_ip(bbdir, chip, ip, ctx, run_root) + config = build_ip(bbdir, chip, ip, ctx) + return { + "chip": chip, + "ip": ip, + "targets": [target["name"] for target in config["targets"]], + } + + targets = load_chip_uvm(bbdir, chip) + results = [ + run_chip(bbdir, chip, target, ctx, do_run) for target in targets["balls"] + ] + for target in targets["ips"]: + if do_run: + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + run_root = log_dir(bbdir, chip, "verilog", stamp, "uvm", target) + results.append(run_ip(bbdir, chip, target, ctx, run_root)) + else: + config = build_ip(bbdir, chip, target, ctx) + results.append( + { + "chip": chip, + "ip": target, + "targets": [item["name"] for item in config["targets"]], + } + ) + return {"chip": chip, "results": results} diff --git a/bbdev/api/steps/uvm/scripts/waive.py b/bbdev/api/steps/uvm/scripts/waive.py index d64ebe1d0..3450ec580 100644 --- a/bbdev/api/steps/uvm/scripts/waive.py +++ b/bbdev/api/steps/uvm/scripts/waive.py @@ -2,11 +2,6 @@ from pathlib import Path -WAIVE_LINE_PATTERNS = ( - re.compile(r"^\s*(?:wire|reg|logic)\b.*\b_(?:GEN|T)(?:_\d+)?\b"), -) - - def apply_waivers(rtl_dir: Path) -> int: if not rtl_dir.is_dir(): raise ValueError(f"missing generated RTL directory: {rtl_dir}") @@ -15,7 +10,7 @@ def apply_waivers(rtl_dir: Path) -> int: lines = path.read_text().splitlines(keepends=True) output = [] for line in lines: - if any(pattern.search(line) for pattern in WAIVE_LINE_PATTERNS) and ( + if re.search(r"^\s*(?:wire|reg|logic)\b.*\b_(?:GEN|T)(?:_\d+)?\b", line) and ( not output or output[-1] != "//VCS coverage off\n" ): output.extend(("//VCS coverage off\n", line, "//VCS coverage on\n")) diff --git a/bbdev/api/steps/vcs/03_build_event.step.py b/bbdev/api/steps/vcs/03_build_event.step.py index 4a049531f..f1265cf80 100644 --- a/bbdev/api/steps/vcs/03_build_event.step.py +++ b/bbdev/api/steps/vcs/03_build_event.step.py @@ -27,7 +27,8 @@ } -TESTBENCH = r'''`timescale 1ns/1ps +def testbench_source() -> str: + return r'''`timescale 1ns/1ps module BBSimVcsHarness; reg clock = 1'b0; reg reset = 1'b1; @@ -63,7 +64,8 @@ # a SystemVerilog testbench, so it needs only the memory model plus these DPI # callbacks. Trace callbacks are intentionally no-ops; functional execution, # UART, and SCU exit remain available to every chip using BBSimHarness. -VCS_DPI = r'''#include +def dpi_source() -> str: + return r'''#include #include #include @@ -154,8 +156,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: artifact_dir.mkdir(parents=True) testbench = artifact_dir / "BBSimVcsHarness.sv" dpi_shim = artifact_dir / "vcs_dpi.cc" - testbench.write_text(TESTBENCH, encoding="utf-8") - dpi_shim.write_text(VCS_DPI, encoding="utf-8") + testbench.write_text(testbench_source(), encoding="utf-8") + dpi_shim.write_text(dpi_source(), encoding="utf-8") arch_dir = Path(bbdir) / "arch" csrcs = [ diff --git a/bbdev/api/steps/verilator/03_build_event.step.py b/bbdev/api/steps/verilator/03_build_event.step.py index 0d634cc49..8b6a5bb02 100644 --- a/bbdev/api/steps/verilator/03_build_event.step.py +++ b/bbdev/api/steps/verilator/03_build_event.step.py @@ -104,9 +104,9 @@ def _gcc_lib_dir(soname: str) -> str: sources = " ".join(vsrcs + csrcs) jobs = input_data.get("jobs", "") - # Enable ccache if available + build_env = os.environ.copy() if subprocess.run("command -v ccache", shell=True, capture_output=True).returncode == 0: - os.environ["OBJCACHE"] = "ccache" + build_env["OBJCACHE"] = "ccache" # Use lld for faster linking if available use_lld = subprocess.run("command -v ld.lld", shell=True, capture_output=True).returncode == 0 @@ -139,6 +139,7 @@ def _gcc_lib_dir(soname: str) -> str: cwd=bbdir, stdout_prefix="verilator verilation", stderr_prefix="verilator verilation", + env=build_env, ) if result.returncode != 0: await check_result( @@ -154,6 +155,7 @@ def _gcc_lib_dir(soname: str) -> str: cwd=bbdir, stdout_prefix="verilator build", stderr_prefix="verilator build", + env=build_env, ) # ================================================================================== diff --git a/bbdev/api/steps/verilator/04_sim_event.step.py b/bbdev/api/steps/verilator/04_sim_event.step.py index ba88a0042..f47bf3e1e 100644 --- a/bbdev/api/steps/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -27,10 +27,6 @@ } -DRAM_BASE = 0x80000000 -DRAM_SIZE = 0x10000000 - - def validate_bbsim_elf(path: str) -> tuple[bool, str]: """Validate the ELF layout accepted by BBSimDRAM before starting Verilator.""" try: @@ -53,6 +49,8 @@ def validate_bbsim_elf(path: str) -> tuple[bool, str]: if phoff + phnum * phentsize > file_size: return False, "program headers extend past the end of the file" + dram_base = 0x80000000 + dram_size = 0x10000000 load_segments = 0 for index in range(phnum): binary.seek(phoff + index * phentsize) @@ -67,11 +65,11 @@ def validate_bbsim_elf(path: str) -> tuple[bool, str]: load_segments += 1 if p_filesz > p_memsz or p_offset + p_filesz > file_size: return False, "has an invalid loadable segment" - if p_paddr < DRAM_BASE or p_paddr + p_memsz > DRAM_BASE + DRAM_SIZE: + if p_paddr < dram_base or p_paddr + p_memsz > dram_base + dram_size: return ( False, f"load segment {index} at 0x{p_paddr:x} is outside " - f"BBSimDRAM [0x{DRAM_BASE:x}, 0x{DRAM_BASE + DRAM_SIZE:x})", + f"BBSimDRAM [0x{dram_base:x}, 0x{dram_base + dram_size:x})", ) except (OSError, struct.error): return False, "cannot read a valid ELF64 program-header table" diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index d9d8d4784..bda8d8fa9 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -13,6 +13,7 @@ from utils.path import get_buckyball_path from utils.event_common import check_result, get_origin_trace_id +from utils.model import model_layout_name _workload_build_path = os.path.join( get_buckyball_path(), "bb-tests", "workloads", "scripts", "build.py" @@ -31,27 +32,11 @@ "enqueues": [], } -# CLI model → layout dir under archs/buckyball// -MODEL_LAYOUT = { - "lenet": "LeNet", - "mobilenet": "MobileNetV3", - "resnet": "ResNet18", - "yolo": "YOLO26", - "bert": "Bert", - "qwen3": "Qwen3", - "gemma4": "Gemma4", - "deepseekr1": "DeepSeekR1", - "llama2": "llama2", - "stable-diffusion": "StableDiffusion", - "whisper": "Whisper", - "buddynext": "BuddyNext", -} - - def chips_for_model(bbdir: str, model_key: str) -> set[str]: """Chips that currently ship a layout for this model (many-to-many).""" - layout = MODEL_LAYOUT.get(model_key) - if layout is None: + try: + layout = model_layout_name(model_key) + except ValueError: return set() root = ( Path(bbdir) @@ -179,8 +164,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if model: model_key = model.lower() - layout = MODEL_LAYOUT.get(model_key) - if layout is None: + try: + layout = model_layout_name(model_key) + except ValueError: ctx.logger.error(f"Unknown model: {model}") await check_result( ctx, 1, continue_run=False, diff --git a/bbdev/api/steps/yosys/03_synth_event.step.py b/bbdev/api/steps/yosys/03_synth_event.step.py index 3c297bbc1..0a0a493ad 100644 --- a/bbdev/api/steps/yosys/03_synth_event.step.py +++ b/bbdev/api/steps/yosys/03_synth_event.step.py @@ -24,15 +24,13 @@ } -POWER_TOTAL_RE = re.compile( - r"^Total\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)", - re.M, -) - - def dynamic_power_report(power_report: str, activity_source: str) -> tuple[str, float] | None: """Return OpenSTA's internal-plus-switching power, excluding leakage.""" - match = POWER_TOTAL_RE.search(power_report) + match = re.search( + r"^Total\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)", + power_report, + re.M, + ) if match is None: return None internal, switching, _leakage, _total = (float(value) for value in match.groups()) @@ -77,7 +75,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: yosys_cfg = load_yosys_config() top_module = input_data.get("top") or yosys_cfg.get("top") or "DigitalTop" - liberty = yosys_cfg.get("liberty") or os.environ.get("YOSYS_LIBERTY") + liberty = yosys_cfg.get("liberty") if isinstance(liberty, str): liberty = os.path.expandvars(os.path.expanduser(liberty)) diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index 3cca9e0b7..f68793435 100755 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -19,6 +19,8 @@ def main(): "test_api_yosys_verilog.py", "test_api_yosys_synth.py", "test_api_dc_verilog.py", + "test_api_uvm_build.py", + "test_api_uvm_run.py", "test_api_kernel_build.py", "test_api_firesim_enumeratefpgas.py", "test_api_firesim_buildbitstream.py", diff --git a/bbdev/api/tests/test_api_bebop_bemu_analysis.py b/bbdev/api/tests/test_api_bebop_bemu_analysis.py index 4aba36d88..3f86a6681 100644 --- a/bbdev/api/tests/test_api_bebop_bemu_analysis.py +++ b/bbdev/api/tests/test_api_bebop_bemu_analysis.py @@ -1,12 +1,3 @@ -from pathlib import Path -import shutil -import tempfile - from _api_test_helper import run_bbdev_case -src = Path(__file__).resolve().parent / "fixtures" / "bemu-analysis" -dst = Path(tempfile.mkdtemp()) / "bemu-analysis" -shutil.copytree(src, dst) -run_bbdev_case( - f"bbdev bebop-bemu --analysis '--chip pebble --log-dir {dst} --itrace --mtrace'" -) +run_bbdev_case("bbdev bebop-bemu --analysis '--chip pebble --log-dir fixtures/bemu-analysis --itrace --mtrace'") # fmt: skip diff --git a/bbdev/api/tests/test_api_bebop_bemu_sim.py b/bbdev/api/tests/test_api_bebop_bemu_sim.py index 179a87d2a..82774073a 100644 --- a/bbdev/api/tests/test_api_bebop_bemu_sim.py +++ b/bbdev/api/tests/test_api_bebop_bemu_sim.py @@ -1,4 +1,4 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_relu_test-singlecore-baremetal'") +run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_relu_test-singlecore-baremetal'") # fmt: skip run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_relu_test-linux --pk'") diff --git a/bbdev/api/tests/test_api_uvm_build.py b/bbdev/api/tests/test_api_uvm_build.py new file mode 100644 index 000000000..eba674164 --- /dev/null +++ b/bbdev/api/tests/test_api_uvm_build.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev uvm --build '--chip toy --ip axis'") diff --git a/bbdev/api/tests/test_api_uvm_run.py b/bbdev/api/tests/test_api_uvm_run.py new file mode 100644 index 000000000..eaca26f4a --- /dev/null +++ b/bbdev/api/tests/test_api_uvm_run.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev uvm --run '--chip toy --ip axis'") diff --git a/bbdev/api/tests/test_uvm_chip_pb.py b/bbdev/api/tests/test_uvm_chip_pb.py deleted file mode 100644 index bc0bac1fb..000000000 --- a/bbdev/api/tests/test_uvm_chip_pb.py +++ /dev/null @@ -1,62 +0,0 @@ -import sys -from pathlib import Path - -API = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(API / "steps" / "config" / "scripts")) -sys.path.insert(0, str(API / "steps" / "uvm")) - -import chip_pb2 -from scripts.uvm_common import ball_domain, load_ip, selected_mappings, vcs_defines - - -def _chip(): - c = chip_pb2.Chip() - core = c.cores.add() - m = core.balldomain.mappings.add() - m.ball_id = 5 - m.ball_dir = "lut" - m.in_bw = 2 - m.out_bw = 1 - isa = core.balldomain.isa.add() - isa.mnemonic = "LUT" - isa.funct7 = 66 - isa.bid = 5 - return c - - -def test_defines(): - c = _chip() - d = ball_domain(c) - m = selected_mappings(d, "lut")[0] - defs = vcs_defines(d, m, 4096) - assert defs == [ - "+define+BB_IN_BW=2", - "+define+BB_OUT_BW=1", - "+define+BB_MMIO_READ_BW=0", - "+define+BB_MMIO_WRITE_BW=0", - "+define+BB_BANK_ADDR_W=12", - "+define+LUT_FUNCT7=66", - ] - - -def test_registered_ip(): - root = str(Path(__file__).resolve().parents[3]) - axis = load_ip(root, "axis") - assert axis["mill_module"] == "axis" - assert [target["name"] for target in axis["targets"]] == ["arbiter", "master", "slave"] - - -def test_unknown_ball(): - try: - selected_mappings(ball_domain(_chip()), "transpose") - except ValueError as e: - assert "transpose" in str(e) - return - raise AssertionError("expected ValueError") - - -if __name__ == "__main__": - test_defines() - test_registered_ip() - test_unknown_ball() - print("ok") diff --git a/bbdev/api/tests/test_uvm_waive.py b/bbdev/api/tests/test_uvm_waive.py deleted file mode 100644 index 2ca2af0b0..000000000 --- a/bbdev/api/tests/test_uvm_waive.py +++ /dev/null @@ -1,24 +0,0 @@ -import sys -from pathlib import Path - -API = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(API / "steps" / "uvm")) - -from scripts.waive import apply_waivers - - -def test_waivers_are_target_agnostic_and_idempotent(tmp_path): - rtl = tmp_path / "rtl" - rtl.mkdir() - source = rtl / "Dut.sv" - source.write_text("module Dut;\n wire _GEN = 1'b0;\nendmodule\n") - - assert apply_waivers(rtl) == 1 - assert source.read_text() == ( - "module Dut;\n" - "//VCS coverage off\n" - " wire _GEN = 1'b0;\n" - "//VCS coverage on\n" - "endmodule\n" - ) - assert apply_waivers(rtl) == 0 diff --git a/bbdev/api/utils/event_common.py b/bbdev/api/utils/event_common.py index d2c6eca26..b7574390b 100644 --- a/bbdev/api/utils/event_common.py +++ b/bbdev/api/utils/event_common.py @@ -6,9 +6,6 @@ from utils.process_registry import set_current_task_scope -_CHIP_RE = re.compile(r"^[A-Za-z0-9_-]+$") - - def require_chip(data: dict) -> str: chip = data.get("chip") if not isinstance(chip, str) or not chip: @@ -18,7 +15,7 @@ def require_chip(data: dict) -> str: raise ValueError( "do not pass --config; mill class comes from chip config sims" ) - if not _CHIP_RE.fullmatch(chip): + if not re.fullmatch(r"[A-Za-z0-9_-]+", chip): raise ValueError(f"invalid chip: {chip}") return chip diff --git a/bbdev/api/utils/model.py b/bbdev/api/utils/model.py new file mode 100644 index 000000000..ab2bf263a --- /dev/null +++ b/bbdev/api/utils/model.py @@ -0,0 +1,19 @@ +def model_layout_name(model: str) -> str: + aliases = { + "lenet": "LeNet", + "mobilenet": "MobileNetV3", + "resnet": "ResNet18", + "yolo": "YOLO26", + "bert": "Bert", + "qwen3": "Qwen3", + "gemma4": "Gemma4", + "deepseekr1": "DeepSeekR1", + "llama2": "llama2", + "stable-diffusion": "StableDiffusion", + "whisper": "Whisper", + "buddynext": "BuddyNext", + } + try: + return aliases[model.lower()] + except KeyError as error: + raise ValueError(f"unknown model: {model}") from error diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index e55646acb..6d32ed35f 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -7,25 +7,9 @@ def get_buckyball_path(): current_dir = os.path.dirname(__file__) - # bbdev/api/utils -> bbdev/api -> bbdev -> buckyball - inferred = os.path.realpath( + root = os.path.realpath( os.path.dirname(os.path.dirname(os.path.dirname(current_dir))) ) - - root = os.environ.get("BUCKYBALL_ROOT") - if root: - if not os.path.isabs(root): - raise ValueError("BUCKYBALL_ROOT must be an absolute path") - if not os.path.isdir(root): - raise ValueError(f"BUCKYBALL_ROOT does not exist: {root}") - root = os.path.realpath(root) - if root != inferred: - raise ValueError( - f"BUCKYBALL_ROOT={root} does not match bbdev tree root {inferred}" - ) - else: - root = inferred - home = os.path.realpath(str(Path.home())) if not Path(root).is_relative_to(home): raise ValueError(f"buckyball root must be under $HOME ({home}): {root}") diff --git a/bbdev/api/utils/port.py b/bbdev/api/utils/port.py index 431e9b9d2..e450ba414 100644 --- a/bbdev/api/utils/port.py +++ b/bbdev/api/utils/port.py @@ -1,14 +1,11 @@ import socket -HOST = "127.0.0.1" - - def find_available_port(start_port: int = 5000, end_port: int = 5500) -> int: """Find an available port in the specified range""" for port in range(start_port, end_port + 1): try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind((HOST, port)) + sock.bind(("127.0.0.1", port)) return port except OSError: continue @@ -43,7 +40,7 @@ def reserve_port( for port in _port_order(start_port, end_port, preferred_port): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: - sock.bind((HOST, port)) + sock.bind(("127.0.0.1", port)) return port, sock except OSError: sock.close() diff --git a/bbdev/api/utils/workers.py b/bbdev/api/utils/workers.py index e7542df6c..226243097 100644 --- a/bbdev/api/utils/workers.py +++ b/bbdev/api/utils/workers.py @@ -3,13 +3,8 @@ import subprocess from pathlib import Path -from utils.port import HOST - -SERVER_PORTS_FILE = "bbdev-server.json" - - def server_ports_path(workflow_dir: str) -> str: - return os.path.join(workflow_dir, "data", SERVER_PORTS_FILE) + return os.path.join(workflow_dir, "data", "bbdev-server.json") def write_server_ports(workflow_dir: str, http_port: int, worker_port: int, bb_root: str) -> None: @@ -49,7 +44,7 @@ def list_engine_workers(worker_port: int, timeout_s: float = 10.0) -> list: "--function-id", "engine::workers::list", "--address", - HOST, + "127.0.0.1", "--port", str(worker_port), "--timeout-ms", diff --git a/bbdev/api/steps/bebop/regression.py b/bbdev/api/utils/workload_manifest.py similarity index 94% rename from bbdev/api/steps/bebop/regression.py rename to bbdev/api/utils/workload_manifest.py index 89108cfd1..89d68d71d 100644 --- a/bbdev/api/steps/bebop/regression.py +++ b/bbdev/api/utils/workload_manifest.py @@ -1,7 +1,7 @@ import os from pathlib import Path -from utils.path import get_buckyball_path +from .path import get_buckyball_path def validate_chip(chip: str | None) -> str: @@ -20,7 +20,7 @@ def chip_regression_dir(chip: str, backend: str, bbdir: str | None = None) -> Pa return root / "examples" / "chips" / chip / "regression" / "batch" / backend -def regression_workload_toml( +def resolve_workload_toml( chip: str, backend: str, test_type: str, diff --git a/bbdev/bbdev b/bbdev/bbdev index 485d6b3a0..5c19b0bdb 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -70,7 +70,7 @@ WORKFLOW_COMMANDS = { }, "bebop-verilator": { "clean": 'Clean Bebop Verilator build directory. Args: "--chip [--output-dir ]"', - "verilog": 'Generate Bebop Verilog files. Args: "--chip [--output-dir ]"', + "verilog": 'Generate Bebop Verilog files. Args: "--chip [--diff] [--output-dir ]"', "build": 'Build Bebop Verilator simulation executable. Args: "--chip [--diff] [--jobs ] [--output-dir ]"', "sim": 'Run Bebop Verilator simulation. Args: "--chip --binary [--rushB] [--log-dir ] [--fst-dir ] [--no-wave]"', "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--rushB] [--jobs ] [--chip ] [--log-dir ] [--fst-dir ] [--no-wave]"', @@ -83,10 +83,10 @@ WORKFLOW_COMMANDS = { }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--chip [--output-dir ]"', - "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--chip [--output-dir ]"', - "buildbitstream": 'Build Bebop P2E runtime case. Args: "--chip [--vsrc-dir ] [--output-dir ]"', - "runworkload": 'Run Bebop P2E workload. Args: "--chip --image --bitstream [--fpga-location ] [--multi-fpga] [--wave] [--wave-start ] [--itrace] [--mtrace] [--pmctrace]"', - "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--chip --bitstream --test "', + "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--chip [--diff] [--output-dir ]"', + "buildbitstream": 'Build Bebop P2E runtime case. Args: "--chip [--diff] [--vsrc-dir ] [--output-dir ]"', + "runworkload": 'Run Bebop P2E workload. Args: "--chip --image --bitstream [--fpga-location ] [--multi-fpga] [--wave] [--wave-start ] [--diff] [--itrace] [--mtrace] [--pmctrace]"', + "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--chip --bitstream --test [--diff]"', }, "dc": { "verilog": 'Generate top-scoped DC RTL under arch/build//. Args: "--chip [--top ]"', @@ -332,6 +332,8 @@ def _parse_sub_args(arg_string: str) -> dict: key, value = token[2:].split("=", 1) if not key or value == "": raise ValueError(f"Invalid argument: {token}") + if value in ("true", "false"): + value = value == "true" args_dict[key] = value i += 1 else: @@ -612,6 +614,7 @@ def run_script_mode(args, cmd_info): # Serialize only engine startup. The reservation socket must be released before # iii can bind it, so without this lock another bbdev can select the same port # during that handoff. + os.makedirs(os.path.dirname(SERVER_START_LOCK), exist_ok=True) server_start_lock = open(SERVER_START_LOCK, "w") fcntl.flock(server_start_lock, fcntl.LOCK_EX) diff --git a/bbdev/mcp/__main__.py b/bbdev/mcp/__main__.py index 9b9d78ddd..5b5037498 100644 --- a/bbdev/mcp/__main__.py +++ b/bbdev/mcp/__main__.py @@ -6,9 +6,8 @@ import sys from pathlib import Path -ROOT = Path(__file__).resolve().parent -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) +if str(Path(__file__).resolve().parent) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parent)) from server import main diff --git a/bbdev/mcp/common.py b/bbdev/mcp/common.py index d9938bd86..15d12e468 100644 --- a/bbdev/mcp/common.py +++ b/bbdev/mcp/common.py @@ -17,12 +17,28 @@ from typing import Any, Dict, Optional # bbdev/mcp -> repo root -REPO = Path(__file__).resolve().parents[2] -BBDEV = REPO / "bbdev" / "bbdev" -API = REPO / "bbdev" / "api" -MOTIA = API / ".venv" / "bin" / "motia" -LOG = REPO / "bbdev" / "server.log" -STATE_DIR = API / "data" / "state_store.db" +def repo_path() -> Path: + return Path(__file__).resolve().parents[2] + + +def bbdev_path() -> Path: + return repo_path() / "bbdev" / "bbdev" + + +def api_path() -> Path: + return repo_path() / "bbdev" / "api" + + +def motia_path() -> Path: + return api_path() / ".venv" / "bin" / "motia" + + +def log_path() -> Path: + return repo_path() / "bbdev" / "server.log" + + +def state_dir() -> Path: + return api_path() / "data" / "state_store.db" _proc: Optional[subprocess.Popen] = None _port: Optional[int] = None _log_fh = None diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index bb31b3f71..70ff219a5 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -48,7 +48,8 @@ from . import regression_eval_performance from . import regression_eval_area_power -MODULES = [ +def modules(): + return [ validate, task_status, task_cancel, @@ -94,9 +95,9 @@ regression_check, regression_eval_performance, regression_eval_area_power, -] + ] def register_all(mcp) -> None: - for mod in MODULES: + for mod in modules(): mod.register(mcp) diff --git a/bbdev/mcp/tools/bebop_p2e_batch.py b/bbdev/mcp/tools/bebop_p2e_batch.py index d306c61f3..ccbb71ea6 100644 --- a/bbdev/mcp/tools/bebop_p2e_batch.py +++ b/bbdev/mcp/tools/bebop_p2e_batch.py @@ -12,6 +12,7 @@ def bbdev_bebop_p2e_batch( chip: str, bitstream: str, test: str, + diff: bool = False, ) -> str: """Batch bebop-p2e regression. test: elf-tests|pk-tests. POST /bebop/p2e/batch.""" for n, v in (("chip", chip), ("bitstream", bitstream)): @@ -26,6 +27,7 @@ def bbdev_bebop_p2e_batch( "chip": chip, "bitstream": bitstream, "test": test, + "diff": diff, }, ) ) diff --git a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py index ec0094499..52423bd16 100644 --- a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py +++ b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py @@ -11,6 +11,7 @@ def register(mcp): @mcp.tool() def bbdev_bebop_p2e_buildbitstream( chip: str, + diff: bool = False, vsrc_dir: Optional[str] = None, output_dir: Optional[str] = None, ) -> str: @@ -21,7 +22,7 @@ def bbdev_bebop_p2e_buildbitstream( submit( "/bebop/p2e/buildbitstream", opt( - {"chip": chip}, + {"chip": chip, "diff": diff}, vsrc_dir=vsrc_dir, output_dir=output_dir, ), diff --git a/bbdev/mcp/tools/bebop_p2e_runworkload.py b/bbdev/mcp/tools/bebop_p2e_runworkload.py index 4ccd2a951..41cd24826 100644 --- a/bbdev/mcp/tools/bebop_p2e_runworkload.py +++ b/bbdev/mcp/tools/bebop_p2e_runworkload.py @@ -16,6 +16,7 @@ def bbdev_bebop_p2e_runworkload( itrace: bool = False, mtrace: bool = False, pmctrace: bool = False, + diff: bool = False, ) -> str: """Run one image on bebop-p2e FPGA. POST /bebop/p2e/runworkload.""" for n, v in (("chip", chip), ("image", image), ("bitstream", bitstream)): @@ -28,4 +29,6 @@ def bbdev_bebop_p2e_runworkload( params["mtrace"] = True if pmctrace: params["pmctrace"] = True + if diff: + params["diff"] = True return fmt(submit("/bebop/p2e/runworkload", params)) diff --git a/bbdev/mcp/tools/bebop_p2e_verilog.py b/bbdev/mcp/tools/bebop_p2e_verilog.py index b3e35b600..4037f547b 100644 --- a/bbdev/mcp/tools/bebop_p2e_verilog.py +++ b/bbdev/mcp/tools/bebop_p2e_verilog.py @@ -10,7 +10,7 @@ def register(mcp): @mcp.tool() def bbdev_bebop_p2e_verilog( - chip: str, output_dir: Optional[str] = None + chip: str, output_dir: Optional[str] = None, diff: bool = False ) -> str: """Generate Verilog for bebop-p2e. POST /bebop/p2e/verilog.""" if e := need("chip", chip): @@ -18,6 +18,6 @@ def bbdev_bebop_p2e_verilog( return fmt( submit( "/bebop/p2e/verilog", - opt({"chip": chip}, output_dir=output_dir), + opt({"chip": chip, "diff": diff}, output_dir=output_dir), ) ) diff --git a/bbdev/mcp/tools/bebop_verilator_verilog.py b/bbdev/mcp/tools/bebop_verilator_verilog.py index 959a80d83..c94457dd5 100644 --- a/bbdev/mcp/tools/bebop_verilator_verilog.py +++ b/bbdev/mcp/tools/bebop_verilator_verilog.py @@ -8,8 +8,8 @@ def register(mcp): @mcp.tool() - def bbdev_bebop_verilator_verilog(chip: str) -> str: + def bbdev_bebop_verilator_verilog(chip: str, diff: bool = False) -> str: """Generate Verilog for bebop-verilator. POST /bebop/verilator/verilog.""" if e := need("chip", chip): return err(e) - return fmt(submit("/bebop/verilator/verilog", {"chip": chip})) + return fmt(submit("/bebop/verilator/verilog", {"chip": chip, "diff": diff})) From 46d39f6171ef4cfe63b4d5f724c1250258f702f8 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 22 Sep 2026 10:57:18 +0800 Subject: [PATCH 160/196] refactor: update path handling for bbdev and related components in common module --- bbdev/api/services/agent_tools.py | 8 +-- bbdev/api/services/workflow_tools.py | 9 +-- bbdev/bbdev | 4 +- bbdev/mcp/common.py | 65 +++++++++++--------- bbdev/mcp/tests/test_common.py | 6 +- bbdev/mcp/tools/__init__.py | 90 ++++++++++++++-------------- 6 files changed, 90 insertions(+), 92 deletions(-) diff --git a/bbdev/api/services/agent_tools.py b/bbdev/api/services/agent_tools.py index 623377aac..fa0153d73 100644 --- a/bbdev/api/services/agent_tools.py +++ b/bbdev/api/services/agent_tools.py @@ -77,13 +77,7 @@ def execute(self, arguments: Dict[str, Any], context: Any) -> str: context.log_info(f"Calling {agent_role}_agent with task") try: - # Get workflow API address (from environment or use defaults) - import os - - workflow_host = os.getenv("WORKFLOW_HOST", "localhost") - workflow_port = os.getenv("WORKFLOW_PORT", "3001") - base_url = f"http://{workflow_host}:{workflow_port}" - url = f"{base_url}/agent" + url = "http://localhost:3001/agent" context.log_info(f"Calling workflow API at: {url}") diff --git a/bbdev/api/services/workflow_tools.py b/bbdev/api/services/workflow_tools.py index ac742100f..ace61fff6 100644 --- a/bbdev/api/services/workflow_tools.py +++ b/bbdev/api/services/workflow_tools.py @@ -1,7 +1,6 @@ """Workflow internal API call tools""" import httpx -import asyncio from typing import Dict, Any from .base import Tool @@ -42,13 +41,7 @@ def execute(self, arguments: Dict[str, Any], context: Any) -> str: endpoint = arguments.get("endpoint") params = arguments.get("params", {}) - # Get workflow API address - import os - - workflow_host = os.getenv("WORKFLOW_HOST", "localhost") - workflow_port = os.getenv("WORKFLOW_PORT", "3001") - base_url = f"http://{workflow_host}:{workflow_port}" - url = f"{base_url}{endpoint}" + url = f"http://localhost:3001{endpoint}" try: context.log_info(f"Calling workflow API: {url}") diff --git a/bbdev/bbdev b/bbdev/bbdev index 5c19b0bdb..33c0aac09 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -14,7 +14,7 @@ import requests import shutil sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "api")) -from utils.port import HOST, reserve_port +from utils.port import reserve_port from utils.process import clean_python_caches, stop_workspace_servers, terminate_group from utils.workers import assert_sole_workspace_workers, motia_worker_count, write_server_ports @@ -133,7 +133,7 @@ def make_iii_config(port: int, worker_port: int, quiet: bool = False, exec_modul # then RestApiModule HTTP port (default 3111). content = content.replace("port: 49134", f"port: {worker_port}") content = content.replace("port: 3111", f"port: {port}", 1) - content = content.replace("host: 0.0.0.0", f"host: {HOST}") + content = content.replace("host: 0.0.0.0", "host: 127.0.0.1") if quiet: content = content.replace( "exporter: memory", diff --git a/bbdev/mcp/common.py b/bbdev/mcp/common.py index 15d12e468..4e0532736 100644 --- a/bbdev/mcp/common.py +++ b/bbdev/mcp/common.py @@ -92,10 +92,10 @@ def _free_port(lo: int = 5100, hi: int = 5500) -> int: def _assert_workspace_workers() -> None: - sys.path.insert(0, str(API)) + sys.path.insert(0, str(api_path())) from utils.workers import assert_sole_workspace_workers, read_server_ports - ports = read_server_ports(str(API)) + ports = read_server_ports(str(api_path())) assert_sole_workspace_workers(int(ports["worker_port"]), str(ports["bb_root"])) @@ -130,10 +130,10 @@ def _ready(port: int) -> bool: def _stop() -> None: global _proc, _port, _log_fh - if _port is not None and BBDEV.is_file(): + if _port is not None and bbdev_path().is_file(): subprocess.run( - ["nix", "develop", "--command", str(BBDEV), "stop", "--server", "--port", str(_port)], - cwd=str(BBDEV.parent), + ["nix", "develop", "--command", str(bbdev_path()), "stop", "--server", "--port", str(_port)], + cwd=str(bbdev_path().parent), capture_output=True, text=True, timeout=30, @@ -168,42 +168,42 @@ def _ensure() -> int: _log(f"bbdev on port {_port} died; restarting") _stop() - if not BBDEV.is_file(): - raise RuntimeError(f"missing bbdev CLI: {BBDEV}") + if not bbdev_path().is_file(): + raise RuntimeError(f"missing bbdev CLI: {bbdev_path()}") if shutil.which("nix") is None: raise RuntimeError( "nix not found; MCP must start bbdev through the project development environment" ) - if not MOTIA.is_file(): + if not motia_path().is_file(): raise RuntimeError( - f"missing {MOTIA}; install with: " - f"cd {API} && uv venv .venv --python python3 --seed && " + f"missing {motia_path()}; install with: " + f"cd {api_path()} && uv venv .venv --python python3 --seed && " "uv pip install --python .venv/bin/python -r pyproject.toml" ) port = _free_port() - LOG.parent.mkdir(parents=True, exist_ok=True) - _log_fh = open(LOG, "a", encoding="utf-8") + log_path().parent.mkdir(parents=True, exist_ok=True) + _log_fh = open(log_path(), "a", encoding="utf-8") _proc = subprocess.Popen( - ["nix", "develop", "--command", str(BBDEV), "start", "--server", "--port", str(port)], - cwd=str(BBDEV.parent), + ["nix", "develop", "--command", str(bbdev_path()), "start", "--server", "--port", str(port)], + cwd=str(bbdev_path().parent), stdout=_log_fh, stderr=_log_fh, start_new_session=True, ) _port = port - _log(f"starting bbdev on port {port} (log: {LOG})") + _log(f"starting bbdev on port {port} (log: {log_path()})") for _ in range(120): if _proc.poll() is not None: tail = "" try: - tail = LOG.read_text(encoding="utf-8", errors="replace")[-2000:] + tail = log_path().read_text(encoding="utf-8", errors="replace")[-2000:] except OSError: pass _stop() raise RuntimeError( - f"bbdev exited early; see {LOG}\n--- log tail ---\n{tail}" + f"bbdev exited early; see {log_path()}\n--- log tail ---\n{tail}" ) if _ready(port): try: @@ -223,12 +223,14 @@ def _ensure() -> int: time.sleep(1) _stop() - raise RuntimeError(f"bbdev failed to start on port {port} within 120s; see {LOG}") + raise RuntimeError( + f"bbdev failed to start on port {port} within 120s; see {log_path()}" + ) def _read_state(trace_id: str) -> Optional[Dict[str, Any]]: """Same as bbdev CLI: poll iii file state store (HTTP /result path_params are broken).""" - path = STATE_DIR / f"{trace_id}.bin" + path = state_dir() / f"{trace_id}.bin" if not path.is_file(): return None try: @@ -255,7 +257,7 @@ def submit(endpoint: str, params: Dict[str, Any]) -> Dict[str, Any]: "failure": True, "status_code": status, "error": submit, - "server_log": str(LOG), + "server_log": str(log_path()), "port": port, } @@ -266,7 +268,7 @@ def submit(endpoint: str, params: Dict[str, Any]) -> Dict[str, Any]: "failure": True, "error": "no trace_id in submit response", "response": submit, - "server_log": str(LOG), + "server_log": str(log_path()), "port": port, } @@ -324,7 +326,7 @@ def task_status(trace_id: str) -> Dict[str, Any]: "processing": False, "trace_id": trace_id, "body": body, - "server_log": str(LOG), + "server_log": str(log_path()), } if "processing" in state: body = state["processing"] @@ -350,9 +352,18 @@ def _load_toml(path: Path) -> Dict[str, Any]: def _balldomain_path(chip: str, balldomain: Optional[str]) -> Path: - chip_root = REPO / "examples" / "chips" / chip + chip_root = repo_path() / "examples" / "chips" / chip cfg = json.loads( - (REPO / "examples" / "chips" / chip / "configs" / "generated" / "config" / "config.json") + ( + repo_path() + / "examples" + / "chips" + / chip + / "configs" + / "generated" + / "config" + / "config.json" + ) .read_text(encoding="utf-8") ) cores = [] @@ -371,14 +382,14 @@ def _balldomain_path(chip: str, balldomain: Optional[str]) -> Path: core_file = Path(core["_file"]) parts = core_file.parts core_name = parts[parts.index("cores") + 1] - core_root = REPO / "examples" / "cores" / core_name + core_root = repo_path() / "examples" / "cores" / core_name domains = core_root / "configs" / "balldomains" if balldomain is None: bd = core["balldomain"] path = Path(bd["_file"]) if not path.is_absolute(): - path = (REPO / bd["_file"]).resolve() + path = (repo_path() / bd["_file"]).resolve() else: path = path.resolve() else: @@ -512,7 +523,7 @@ def dups(xs): return { "passed": all(c["pass"] for c in checks.values()), - "chip_balldomain": str(path.relative_to(REPO)), + "chip_balldomain": str(path.relative_to(repo_path())), "checks": checks, "balls": balls, } diff --git a/bbdev/mcp/tests/test_common.py b/bbdev/mcp/tests/test_common.py index 70b05e46e..03afd25ed 100644 --- a/bbdev/mcp/tests/test_common.py +++ b/bbdev/mcp/tests/test_common.py @@ -89,13 +89,13 @@ def test_ensure_starts_server_in_nix_environment(self): "nix", "develop", "--command", - str(common.BBDEV), + str(common.bbdev_path()), "start", "--server", "--port", "5199", ], - cwd=str(common.BBDEV.parent), + cwd=str(common.bbdev_path().parent), stdout=common._log_fh, stderr=common._log_fh, start_new_session=True, @@ -131,7 +131,7 @@ def test_task_status_reports_failure(self, read_state): "processing": False, "trace_id": "trace-1", "body": {"returncode": 1}, - "server_log": str(common.LOG), + "server_log": str(common.log_path()), }, ) diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index 70ff219a5..96c5d4c05 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -50,51 +50,51 @@ def modules(): return [ - validate, - task_status, - task_cancel, - compiler_build, - workload_clean, - workload_build, - bemu_sim, - bemu_analysis, - bemu_batch, - bebop_verilator_clean, - bebop_verilator_verilog, - bebop_verilator_build, - bebop_verilator_sim, - bebop_verilator_run, - bebop_verilator_batch, - verilator_clean, - verilator_verilog, - verilator_build, - verilator_sim, - verilator_run, - uvm_build, - uvm_run, - workload_tohex, - bebop_p2e_clean, - bebop_p2e_verilog, - bebop_p2e_buildbitstream, - bebop_p2e_runworkload, - bebop_p2e_batch, - dc_verilog, - dc_area, - dc_power, - ip_generate, - ip_replace, - firesim_enumeratefpgas, - firesim_buildbitstream, - firesim_infrasetup, - firesim_runworkload, - kernel_build, - yosys_run, - yosys_verilog, - yosys_synth, - regression_buildbitstream, - regression_check, - regression_eval_performance, - regression_eval_area_power, + validate, + task_status, + task_cancel, + compiler_build, + workload_clean, + workload_build, + bemu_sim, + bemu_analysis, + bemu_batch, + bebop_verilator_clean, + bebop_verilator_verilog, + bebop_verilator_build, + bebop_verilator_sim, + bebop_verilator_run, + bebop_verilator_batch, + verilator_clean, + verilator_verilog, + verilator_build, + verilator_sim, + verilator_run, + uvm_build, + uvm_run, + workload_tohex, + bebop_p2e_clean, + bebop_p2e_verilog, + bebop_p2e_buildbitstream, + bebop_p2e_runworkload, + bebop_p2e_batch, + dc_verilog, + dc_area, + dc_power, + ip_generate, + ip_replace, + firesim_enumeratefpgas, + firesim_buildbitstream, + firesim_infrasetup, + firesim_runworkload, + kernel_build, + yosys_run, + yosys_verilog, + yosys_synth, + regression_buildbitstream, + regression_check, + regression_eval_performance, + regression_eval_area_power, ] From 3ab66f9a6cc3a4b9fd054d7a29e3a103870f779b Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 22 Sep 2026 11:02:26 +0800 Subject: [PATCH 161/196] refactor: clean up bbdev configuration and improve logging handling --- .../api/steps/yosys/scripts/yosys-config.yaml | 3 +- bbdev/bbdev | 119 +++++++++--------- 2 files changed, 57 insertions(+), 65 deletions(-) diff --git a/bbdev/api/steps/yosys/scripts/yosys-config.yaml b/bbdev/api/steps/yosys/scripts/yosys-config.yaml index 005e2cdf5..a50cb07c4 100644 --- a/bbdev/api/steps/yosys/scripts/yosys-config.yaml +++ b/bbdev/api/steps/yosys/scripts/yosys-config.yaml @@ -2,8 +2,7 @@ # Users can customize liberty library paths and synthesis options here # Liberty (.lib) file path for ASIC area estimation -# If not specified, yosys will use generic gate-level statistics. A technology -# owner may set YOSYS_LIBERTY in the host environment. +# If not specified, yosys will use generic gate-level statistics. liberty: # Default top module name (override with --top on CLI) diff --git a/bbdev/bbdev b/bbdev/bbdev index 33c0aac09..132fe055b 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -21,13 +21,6 @@ from utils.workers import assert_sole_workspace_workers, motia_worker_count, wri workflow_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "api") workspace_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -NO_PROXY = {"http": None, "https": None, "all": None} -WORKER_PORT_START = 49200 -WORKER_PORT_END = 49500 -HTTP_PORT_START = 5100 -HTTP_PORT_END = 5500 -SERVER_START_LOCK = os.path.join(workflow_dir, "data", "server-start.lock") - # Subcommand definitions: {name: {operation: help_text, ...}} # Operations with string values accept sub-arguments; True means boolean-only (no sub-args). WORKFLOW_COMMANDS = { @@ -118,12 +111,6 @@ WORKFLOW_COMMANDS = { }, } -ALL_COMMANDS = ["start", "stop"] + list(WORKFLOW_COMMANDS) - - -MOTIA_EXEC = "PYTHONUNBUFFERED=1 PYTHONPATH=. .venv/bin/motia dev --dir steps" - - def make_iii_config(port: int, worker_port: int, quiet: bool = False, exec_module: bool = True) -> str: """Create a temporary config.yaml with unique HTTP and Worker ports, return the path.""" base_config = os.path.join(workflow_dir, "config.yaml") @@ -145,11 +132,12 @@ def make_iii_config(port: int, worker_port: int, quiet: bool = False, exec_modul raise RuntimeError("failed to remove ExecModule from iii config") content = content.split(marker, 1)[0].rstrip() + "\n" else: - single = f" exec:\n - {MOTIA_EXEC}\n" + motia_exec = "PYTHONUNBUFFERED=1 PYTHONPATH=. .venv/bin/motia dev --dir steps" + single = f" exec:\n - {motia_exec}\n" if single not in content: raise RuntimeError("failed to expand Motia worker exec list in iii config") n = motia_worker_count() - multi = " exec:\n" + "".join(f" - {MOTIA_EXEC}\n" for _ in range(n)) + multi = " exec:\n" + "".join(f" - {motia_exec}\n" for _ in range(n)) content = content.replace(single, multi, 1) tmp = tempfile.NamedTemporaryFile( mode="w", suffix=".yaml", prefix="iii-config-", dir=workflow_dir, delete=False @@ -363,7 +351,9 @@ def wait_for_http(url: str, check_fn, timeout: int, error_msg: str): """Poll an HTTP endpoint until check_fn(response) returns True, or timeout.""" for _ in range(timeout): try: - resp = requests.get(url, timeout=1, proxies=NO_PROXY) + resp = requests.get( + url, timeout=1, proxies={"http": None, "https": None, "all": None} + ) if check_fn(resp): return True except requests.exceptions.ConnectionError: @@ -376,29 +366,6 @@ def wait_for_http(url: str, check_fn, timeout: int, error_msg: str): # If you find that bbdev has failed without any logs, # you can disable these log filters. These contain the bbdev runtime logs. -_ENGINE_INFRA_MODULES = ( - "iii::modules::worker", - "iii::modules::config", - "iii::modules::rest_api", - "iii::modules::api", - "iii::modules::queue", - "iii::modules::state", - "iii::modules::pubsub", - "iii::modules::shell", - "iii::modules::telemetry", - "iii::builtins", - "iii::trigger", - "iii::function", - "function_name", - "service_name", - "iii::services", - "iii::workers", - "motia.runtime", - "motia.tracing", - "motia.cli", -) - - def forward_iii_output(pipe, active_event: threading.Event, quiet: bool = False): """Forward task output to stdout. In quiet mode, suppress engine infrastructure logs.""" for line in iter(pipe.readline, ""): @@ -406,7 +373,30 @@ def forward_iii_output(pipe, active_event: threading.Event, quiet: bool = False) continue if quiet: # Suppress engine infrastructure INFO/WARN; pass everything else - if any(mod in line for mod in _ENGINE_INFRA_MODULES): + if any( + module in line + for module in ( + "iii::modules::worker", + "iii::modules::config", + "iii::modules::rest_api", + "iii::modules::api", + "iii::modules::queue", + "iii::modules::state", + "iii::modules::pubsub", + "iii::modules::shell", + "iii::modules::telemetry", + "iii::builtins", + "iii::trigger", + "iii::function", + "function_name", + "service_name", + "iii::services", + "iii::workers", + "motia.runtime", + "motia.tracing", + "motia.cli", + ) + ): continue sys.stdout.write(line) sys.stdout.flush() @@ -437,15 +427,16 @@ def run_server_mode(args, cmd_info): print("Error: --server start requires --port", file=sys.stderr) sys.exit(2) load_host_environment() - preferred_worker = user_preferred_port(WORKER_PORT_START, WORKER_PORT_END) + worker_port_start, worker_port_end = 49200, 49500 + preferred_worker = user_preferred_port(worker_port_start, worker_port_end) worker_port, worker_sock = reserve_port( - start_port=WORKER_PORT_START, - end_port=WORKER_PORT_END, + start_port=worker_port_start, + end_port=worker_port_end, preferred_port=preferred_worker, ) worker_sock.close() config_path = make_iii_config(args.port, worker_port) - worker_url = f"ws://{HOST}:{worker_port}" + worker_url = f"ws://127.0.0.1:{worker_port}" print(f"Using worker port {worker_port} (preferred {preferred_worker})") proc = None try: @@ -454,7 +445,6 @@ def run_server_mode(args, cmd_info): env = { **os.environ, "PYTHONDONTWRITEBYTECODE": "1", - "BUCKYBALL_ROOT": repo_root, "III_URL": worker_url, } proc = subprocess.Popen( @@ -510,29 +500,29 @@ def run_server_mode(args, cmd_info): elif command in WORKFLOW_COMMANDS: api_path = f"/{command.replace('-', '/')}/{cmd_info['operation']}" resp = requests.post( - f"http://{HOST}:{args.port}{api_path}", + f"http://127.0.0.1:{args.port}{api_path}", json=cmd_info["args"], timeout=30, - proxies=NO_PROXY, + proxies={"http": None, "https": None, "all": None}, ) print(resp.text) else: print(f"Unknown command: {command}") - print(f"Available commands: {', '.join(ALL_COMMANDS)}") + print(f"Available commands: {', '.join(['start', 'stop', *WORKFLOW_COMMANDS])}") sys.exit(1) def _submit_and_poll(port: int, api_path: str, cmd_info: dict): """Submit a task to iii and poll for its result. Returns exit code (0=ok, 1=fail).""" - base_url = f"http://{HOST}:{port}" + base_url = f"http://127.0.0.1:{port}" print(f"Executing {cmd_info['command']} {cmd_info['operation']}...") try: submit_resp = requests.post( f"{base_url}{api_path}", json=cmd_info["args"], timeout=30, - proxies=NO_PROXY, + proxies={"http": None, "https": None, "all": None}, ) submit_data = submit_resp.json() except Exception as e: @@ -579,7 +569,7 @@ def _submit_and_poll(port: int, api_path: str, cmd_info: dict): print("Error: Task polling timed out after 24 hours") return 1 - print(f"\nTask completed on http://{HOST}:{port}") + print(f"\nTask completed on http://127.0.0.1:{port}") if task_result and not task_result.get("success", False): err = task_result.get("error") or task_result print(f"Error: Task failed: {err}") @@ -604,7 +594,7 @@ def run_script_mode(args, cmd_info): if command not in WORKFLOW_COMMANDS: print(f"Unknown command: {command}") - print(f"Available commands: {', '.join(ALL_COMMANDS)}") + print(f"Available commands: {', '.join(['start', 'stop', *WORKFLOW_COMMANDS])}") sys.exit(1) load_host_environment() @@ -614,10 +604,14 @@ def run_script_mode(args, cmd_info): # Serialize only engine startup. The reservation socket must be released before # iii can bind it, so without this lock another bbdev can select the same port # during that handoff. - os.makedirs(os.path.dirname(SERVER_START_LOCK), exist_ok=True) - server_start_lock = open(SERVER_START_LOCK, "w") + server_start_lock_path = os.path.join(workflow_dir, "data", "server-start.lock") + os.makedirs(os.path.dirname(server_start_lock_path), exist_ok=True) + server_start_lock = open(server_start_lock_path, "w") fcntl.flock(server_start_lock, fcntl.LOCK_EX) + http_port_start, http_port_end = 5100, 5500 + worker_port_start, worker_port_end = 49200, 49500 + # Reserve both ports atomically to prevent TOCTOU races with concurrent bbdev instances. # Sockets stay bound until the engine has taken over (HTTP is up), then are released. if args.port: @@ -625,14 +619,14 @@ def run_script_mode(args, cmd_info): http_sock = None else: available_port, http_sock = reserve_port( - start_port=HTTP_PORT_START, - end_port=HTTP_PORT_END, - preferred_port=user_preferred_port(HTTP_PORT_START, HTTP_PORT_END), + start_port=http_port_start, + end_port=http_port_end, + preferred_port=user_preferred_port(http_port_start, http_port_end), ) worker_port, worker_sock = reserve_port( - start_port=WORKER_PORT_START, - end_port=WORKER_PORT_END, - preferred_port=user_preferred_port(WORKER_PORT_START, WORKER_PORT_END), + start_port=worker_port_start, + end_port=worker_port_end, + preferred_port=user_preferred_port(worker_port_start, worker_port_end), ) print(f"Starting server on port {available_port}...") @@ -667,11 +661,10 @@ def run_script_mode(args, cmd_info): # Set III_URL so motia SDK connects to the correct worker port iii_env = { **os.environ, - "III_URL": f"ws://{HOST}:{worker_port}", + "III_URL": f"ws://127.0.0.1:{worker_port}", "PYTHONUNBUFFERED": "1", "PYTHONIOENCODING": "utf-8", "PYTHONDONTWRITEBYTECODE": "1", - "BUCKYBALL_ROOT": repo_root, } worker_procs = [] proc = subprocess.Popen( @@ -691,7 +684,7 @@ def run_script_mode(args, cmd_info): output_thread.start() try: - base_url = f"http://{HOST}:{available_port}" + base_url = f"http://127.0.0.1:{available_port}" # Phase 1: wait for iii engine HTTP if not wait_for_http( From cc83f050919169dd7144093e495086c98d8ca926 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 22 Sep 2026 11:37:21 +0800 Subject: [PATCH 162/196] refactor: update bebop bemu batch regression descriptions and remove unused nextest configuration --- .../api/steps/bebop/bemu/03_batch_api.step.py | 2 +- .../steps/bebop/bemu/03_batch_event.step.py | 23 +++++++------------ .../api/steps/bebop/bemu/scripts/nextest.toml | 19 --------------- bbdev/bbdev | 2 +- 4 files changed, 10 insertions(+), 36 deletions(-) delete mode 100644 bbdev/api/steps/bebop/bemu/scripts/nextest.toml diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py index 6372d8a40..2c8eb89ab 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -15,7 +15,7 @@ config = { "name": "bebop-bemu-batch-api", - "description": "Run bebop bemu nextest batch regression", + "description": "Run bebop bemu batch regression", "flows": ["bebop"], "triggers": [api("POST", "/bebop/bemu/batch")], "enqueues": ["bebop.bemu.batch"], diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index f5bb21593..afac30d7b 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -1,9 +1,9 @@ """ bebop bemu batch event handler -Runs bebop bemu nextest batch regression: +Runs bebop bemu batch regression: 1. Build the selected chip's BEMU wrapper - 2. Run cargo nextest with bemu-specific config + 2. Run the BEMU regression harness """ import os @@ -31,7 +31,7 @@ config = { "name": "bebop-bemu-batch", - "description": "Run bebop bemu nextest batch regression", + "description": "Run bebop bemu batch regression", "flows": ["bebop"], "triggers": [queue("bebop.bemu.batch")], "enqueues": [], @@ -41,10 +41,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - nextest_config = ( - f"{os.path.dirname(os.path.abspath(__file__))}/scripts/nextest.toml" - ) - chip = input_data.get("chip") if not chip: ctx.logger.error("Missing required parameter: chip must be specified") @@ -128,8 +124,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - # ── Run nextest ─────────────────────────────────────────────────────── - # Pass test harness parameters through nextest's process environment. + # ── Run regression ──────────────────────────────────────────────────── if input_data.get("clean-before", input_data.get("clean_before", False)): shutil.rmtree( os.path.join(env["CARGO_TARGET_DIR"], "test-artifacts"), ignore_errors=True @@ -142,16 +137,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--bb-tests-root", elf_root, *(["--rushb-backend", "bemu"] if rushB else []), ]) - nextest_cmd = ( - f"nix develop -c cargo nextest run --manifest-path {shlex.quote(str(bemu_cargo_manifest))} " + test_cmd = ( + f"nix develop -c cargo test --manifest-path {shlex.quote(str(bemu_cargo_manifest))} " "--test test_bemu " - f"--config-file {shlex.quote(nextest_config)} " f"{harness}" ) - ctx.logger.info(f"Running bebop bemu nextest: {nextest_cmd}") + ctx.logger.info(f"Running bebop bemu regression: {test_cmd}") run_result = await stream_run_logger_async( - cmd=nextest_cmd, + cmd=test_cmd, logger=ctx.logger, cwd=bbdir, stdout_prefix="bebop bemu batch", @@ -169,7 +163,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "chip": chip, "test_type": test_type, "rushB": rushB, - "nextest_config": nextest_config, "workload_toml": workload_toml, }, trace_id=origin_tid, diff --git a/bbdev/api/steps/bebop/bemu/scripts/nextest.toml b/bbdev/api/steps/bebop/bemu/scripts/nextest.toml deleted file mode 100644 index 738cb77c0..000000000 --- a/bbdev/api/steps/bebop/bemu/scripts/nextest.toml +++ /dev/null @@ -1,19 +0,0 @@ -# ============================================================================= -# Nextest configuration for bebop bemu regression -# ============================================================================= -# -# bemu = lightweight emulator tests, parallel execution (fill all cores). -# Loaded by bbdev when invoking `bbdev bebop-bemu --batch`. - -[profile.default] -failure-output = "immediate" -success-output = "never" -status-level = "all" -fail-fast = false -slow-timeout = { period = "60s", terminate-after = 5 } -retries = 0 -junit = { path = "target/nextest/junit-bbdev-bemu.xml", report-name = "bbdev-bemu-batch" } - -[[profile.default.overrides]] -filter = "binary(test_bemu)" -slow-timeout = { period = "60s", terminate-after = 5 } diff --git a/bbdev/bbdev b/bbdev/bbdev index 132fe055b..903fce662 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -59,7 +59,7 @@ WORKFLOW_COMMANDS = { "bebop-bemu": { "sim": 'Run Bebop emulator simulation. Args: "--chip --binary [--rushB] [--pk] [--disasm] [--tool-profile] [--itrace] [--mtrace]"', "analysis": 'Analyze itrace/mtrace in an existing log dir. Args: "--chip --log-dir --itrace|--mtrace"', - "batch": 'Run Bebop bemu nextest batch regression (parallel). Args: "--chip --test [--rushB] [--clean-before]"', + "batch": 'Run Bebop BEMU batch regression. Args: "--chip --test [--rushB] [--clean-before]"', }, "bebop-verilator": { "clean": 'Clean Bebop Verilator build directory. Args: "--chip [--output-dir ]"', From 7dd6562f5de75604cfb653e1498e32dee7cb3542 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 22 Sep 2026 13:02:52 +0800 Subject: [PATCH 163/196] refactor: update batch regression descriptions and remove unused nextest configuration files --- .../api/steps/bebop/p2e/04_batch_api.step.py | 2 +- .../steps/bebop/p2e/04_batch_event.step.py | 21 +++++----- .../api/steps/bebop/p2e/scripts/nextest.toml | 23 ----------- .../bebop/verilator/06_batch_api.step.py | 2 +- .../bebop/verilator/06_batch_event.step.py | 20 ++++------ .../bebop/verilator/scripts/nextest.toml | 19 --------- bbdev/api/steps/config/scripts/4_install.py | 2 +- bbdev/bbdev | 39 +++++++++++-------- 8 files changed, 41 insertions(+), 87 deletions(-) delete mode 100644 bbdev/api/steps/bebop/p2e/scripts/nextest.toml delete mode 100644 bbdev/api/steps/bebop/verilator/scripts/nextest.toml diff --git a/bbdev/api/steps/bebop/p2e/04_batch_api.step.py b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py index 95fe93d23..5f8265f72 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_api.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py @@ -2,7 +2,7 @@ config = { "name": "bebop-p2e-batch-api", - "description": "Run bebop p2e nextest batch regression", + "description": "Run bebop p2e batch regression", "flows": ["bebop"], "triggers": [api("POST", "/bebop/p2e/batch")], "enqueues": ["bebop.p2e.batch"], diff --git a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py index 52080a0a6..d1d26d325 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py @@ -1,10 +1,10 @@ """ bebop p2e batch event handler -Runs bebop p2e nextest batch regression (aligned with runworkload): +Runs bebop p2e batch regression (aligned with runworkload): 1. Resolve the runtime from the bitstream case 2. Build the test harness against that runtime - 3. Run cargo nextest serially on one FPGA + 3. Run the P2E regression harness serially on one FPGA """ import os import re @@ -29,7 +29,7 @@ config = { "name": "bebop-p2e-batch", - "description": "Run bebop p2e nextest batch regression", + "description": "Run bebop p2e batch regression", "flows": ["bebop"], "triggers": [queue("bebop.p2e.batch")], "enqueues": [], @@ -49,8 +49,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" - nextest_config = f"{os.path.dirname(os.path.abspath(__file__))}/scripts/nextest.toml" - bitstream = input_data.get("bitstream", "") if not bitstream or not os.path.isfile(bitstream): ctx.logger.error(f"bitstream .bit file not found: {bitstream}") @@ -156,7 +154,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - # ── Run nextest ─────────────────────────────────────────────────────── + # ── Run regression ──────────────────────────────────────────────────── env = runtime_env.copy() env["OUT_PATH"] = build_dir harness_args = [ @@ -168,15 +166,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if diff: harness_args.append("--diff") harness = shlex.join(harness_args) - nextest_cmd = ( - f"nix develop -c cargo {cargo_out} nextest run --release " + test_cmd = ( + f"nix develop -c cargo {cargo_out} test --release " f"--manifest-path {shlex.quote(str(manifest))} --features {shlex.quote(feature_arg)} " - f"--test test_p2e --config-file \"{nextest_config}\" {harness}" + f"--test test_p2e {harness}" ) - ctx.logger.info(f"Running bebop p2e nextest: {nextest_cmd}") + ctx.logger.info(f"Running bebop p2e regression: {test_cmd}") run_result = await stream_run_logger_async( - cmd=nextest_cmd, + cmd=test_cmd, logger=ctx.logger, cwd=bebop_dir, stdout_prefix="bebop p2e batch", @@ -192,7 +190,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "build_dir": build_dir, "test_type": test_type, "diff": diff, - "nextest_config": nextest_config, "workload_toml": workload_toml, } diff --git a/bbdev/api/steps/bebop/p2e/scripts/nextest.toml b/bbdev/api/steps/bebop/p2e/scripts/nextest.toml deleted file mode 100644 index 13f1966ca..000000000 --- a/bbdev/api/steps/bebop/p2e/scripts/nextest.toml +++ /dev/null @@ -1,23 +0,0 @@ -# ============================================================================= -# Nextest configuration for bebop p2e regression -# ============================================================================= -# -# p2e = FPGA workload tests, serial execution (single FPGA resource). -# Loaded by bbdev when invoking `bbdev bebop-p2e --batch`. - -[profile.default] -failure-output = "immediate" -success-output = "never" -status-level = "all" -fail-fast = false -slow-timeout = { period = "600s", terminate-after = 18 } -retries = 0 -junit = { path = "target/nextest/junit-bbdev-p2e.xml", report-name = "bbdev-p2e-batch" } - -[test-groups.p2e-serial] -max-threads = 1 - -[[profile.default.overrides]] -filter = "binary(test_p2e)" -test-group = "p2e-serial" -slow-timeout = { period = "600s", terminate-after = 18 } diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py index 9ebc6e40f..108b71e8c 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -5,7 +5,7 @@ config = { "name": "bebop-verilator-batch-api", - "description": "Run bebop verilator nextest batch regression", + "description": "Run bebop verilator batch regression", "flows": ["bebop"], "triggers": [api("POST", "/bebop/verilator/batch")], "enqueues": ["bebop.verilator.batch"], diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index fad0b33a9..6a3cf807d 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -1,7 +1,7 @@ """ bebop verilator batch event handler -Runs bebop verilator nextest batch regression (requires prior --build). +Runs bebop verilator batch regression (requires prior --build). """ import os @@ -32,7 +32,7 @@ config = { "name": "bebop-verilator-batch", - "description": "Run bebop verilator nextest batch regression", + "description": "Run bebop verilator batch regression", "flows": ["bebop"], "triggers": [queue("bebop.verilator.batch")], "enqueues": [], @@ -43,10 +43,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" - nextest_config = ( - f"{os.path.dirname(os.path.abspath(__file__))}/scripts/nextest.toml" - ) - try: chip = require_chip(input_data) except ValueError as error: @@ -137,17 +133,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if rushB: harness_args.extend(["--rushb-backend", "verilator"]) harness = shlex.join(harness_args) - nextest_cmd = ( - f"nix develop -c cargo nextest run --release --manifest-path {shlex.quote(manifest)} " + test_cmd = ( + f"nix develop -c cargo --config={vsrc_config} test --release " + f"--manifest-path {shlex.quote(manifest)} " f"--features {shlex.quote(features)} --test test_verilator " - f"--config-file {shlex.quote(nextest_config)} " - f"--config={vsrc_config} " f"{harness}" ) - ctx.logger.info(f"Running bebop verilator nextest: {nextest_cmd}") + ctx.logger.info(f"Running bebop verilator regression: {test_cmd}") run_result = await stream_run_logger_async( - cmd=nextest_cmd, + cmd=test_cmd, logger=ctx.logger, cwd=bbdir, stdout_prefix="bebop verilator batch", @@ -169,7 +164,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "test_type": test_type, "rushB": rushB, "diff": diff, - "nextest_config": nextest_config, "workload_toml": workload_toml, }, trace_id=origin_tid, diff --git a/bbdev/api/steps/bebop/verilator/scripts/nextest.toml b/bbdev/api/steps/bebop/verilator/scripts/nextest.toml deleted file mode 100644 index dd2b2386f..000000000 --- a/bbdev/api/steps/bebop/verilator/scripts/nextest.toml +++ /dev/null @@ -1,19 +0,0 @@ -# ============================================================================= -# Nextest configuration for bebop verilator regression -# ============================================================================= -# -# verilator = RTL simulation, parallel execution (fill all cores). -# Loaded by bbdev when invoking `bbdev bebop-verilator --batch`. - -[profile.default] -failure-output = "immediate" -success-output = "never" -status-level = "all" -fail-fast = false -slow-timeout = { period = "360s", terminate-after = 5 } -retries = 0 -junit = { path = "target/nextest/junit-bbdev-verilator.xml", report-name = "bbdev-verilator-batch" } - -[[profile.default.overrides]] -filter = "binary(test_verilator)" -slow-timeout = { period = "360s", terminate-after = 5 } diff --git a/bbdev/api/steps/config/scripts/4_install.py b/bbdev/api/steps/config/scripts/4_install.py index ed08879d9..564f12a65 100644 --- a/bbdev/api/steps/config/scripts/4_install.py +++ b/bbdev/api/steps/config/scripts/4_install.py @@ -159,7 +159,7 @@ def install_bebop(gen: Path) -> Path: [dependencies] bebop-verilator = { path = "../../../../../bebop/src/nodes/verilator", optional = true } bebop-p2e = { path = "../../../../../bebop/src/nodes/p2e", optional = true } -bebop-bemu = { path = "../../configs/generated/bemu", optional = true, default-features = false } +bebop-bemu = { path = "../../configs/generated/bemu", optional = true } bebop-dasm = { path = "../../../../../bebop/src/nodes/lib/dasm" } bebop-bank-hash = { path = "../../../../../bebop/src/nodes/lib/bank-hash" } bebop-bemu-profile = { path = "../../../../../bebop/src/nodes/lib/bemu-profile" } diff --git a/bbdev/bbdev b/bbdev/bbdev index 903fce662..bb1c60281 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -67,7 +67,7 @@ WORKFLOW_COMMANDS = { "build": 'Build Bebop Verilator simulation executable. Args: "--chip [--diff] [--jobs ] [--output-dir ]"', "sim": 'Run Bebop Verilator simulation. Args: "--chip --binary [--rushB] [--log-dir ] [--fst-dir ] [--no-wave]"', "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--rushB] [--jobs ] [--chip ] [--log-dir ] [--fst-dir ] [--no-wave]"', - "batch": 'Run Bebop verilator nextest batch regression (parallel). Args: "--chip --test [--diff] [--rushB] [--clean-before]"', + "batch": 'Run Bebop Verilator batch regression. Args: "--chip --test [--diff] [--rushB] [--clean-before]"', }, "bebop-vcs": { "verilog": 'Generate Bebop VCS RTL once. Args: "--chip "', @@ -79,7 +79,7 @@ WORKFLOW_COMMANDS = { "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--chip [--diff] [--output-dir ]"', "buildbitstream": 'Build Bebop P2E runtime case. Args: "--chip [--diff] [--vsrc-dir ] [--output-dir ]"', "runworkload": 'Run Bebop P2E workload. Args: "--chip --image --bitstream [--fpga-location ] [--multi-fpga] [--wave] [--wave-start ] [--diff] [--itrace] [--mtrace] [--pmctrace]"', - "batch": 'Run Bebop p2e nextest batch regression (serial). Args: "--chip --bitstream --test [--diff]"', + "batch": 'Run Bebop P2E batch regression. Args: "--chip --bitstream --test [--diff]"', }, "dc": { "verilog": 'Generate top-scoped DC RTL under arch/build//. Args: "--chip [--top ]"', @@ -111,7 +111,13 @@ WORKFLOW_COMMANDS = { }, } -def make_iii_config(port: int, worker_port: int, quiet: bool = False, exec_module: bool = True) -> str: +def make_iii_config( + port: int, + worker_port: int, + quiet: bool = False, + exec_module: bool = True, + state_dir: str | None = None, +) -> str: """Create a temporary config.yaml with unique HTTP and Worker ports, return the path.""" base_config = os.path.join(workflow_dir, "config.yaml") with open(base_config, "r") as f: @@ -121,6 +127,11 @@ def make_iii_config(port: int, worker_port: int, quiet: bool = False, exec_modul content = content.replace("port: 49134", f"port: {worker_port}") content = content.replace("port: 3111", f"port: {port}", 1) content = content.replace("host: 0.0.0.0", "host: 127.0.0.1") + if state_dir is not None: + content = content.replace( + "file_path: ./data/state_store.db", + f"file_path: {state_dir}", + ) if quiet: content = content.replace( "exporter: memory", @@ -513,7 +524,7 @@ def run_server_mode(args, cmd_info): sys.exit(1) -def _submit_and_poll(port: int, api_path: str, cmd_info: dict): +def _submit_and_poll(port: int, api_path: str, cmd_info: dict, state_dir: str): """Submit a task to iii and poll for its result. Returns exit code (0=ok, 1=fail).""" base_url = f"http://127.0.0.1:{port}" print(f"Executing {cmd_info['command']} {cmd_info['operation']}...") @@ -539,7 +550,7 @@ def _submit_and_poll(port: int, api_path: str, cmd_info: dict): return 1 # Poll for result - state_file = os.path.join(workflow_dir, "data", "state_store.db", f"{trace_id}.bin") + state_file = os.path.join(state_dir, f"{trace_id}.bin") poll_timeout = 24 * 60 * 60 # 24 hours max task_result = None legacy_processing_count = 0 @@ -630,15 +641,9 @@ def run_script_mode(args, cmd_info): ) print(f"Starting server on port {available_port}...") - # Clean stale state files from previous runs - state_dir = os.path.join(workflow_dir, "data", "state_store.db") - if os.path.isdir(state_dir): - for f in os.listdir(state_dir): - if f.endswith(".bin"): - try: - os.unlink(os.path.join(state_dir, f)) - except OSError: - pass + data_dir = os.path.join(workflow_dir, "data") + os.makedirs(data_dir, exist_ok=True) + state_dir = tempfile.mkdtemp(prefix="state-", dir=data_dir) clean_python_caches(workflow_dir) config_path = make_iii_config( @@ -646,6 +651,7 @@ def run_script_mode(args, cmd_info): worker_port, quiet=True, exec_module=False, + state_dir=state_dir, ) iii_output_active = threading.Event() iii_output_active.set() @@ -656,8 +662,6 @@ def run_script_mode(args, cmd_info): worker_sock.close() repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - write_server_ports(workflow_dir, available_port, worker_port, repo_root) - # Set III_URL so motia SDK connects to the correct worker port iii_env = { **os.environ, @@ -736,7 +740,7 @@ def run_script_mode(args, cmd_info): sys.exit(1) print(f"Server is ready on port {available_port} with {n} Motia worker(s)") - rc = _submit_and_poll(available_port, api_path, cmd_info) + rc = _submit_and_poll(available_port, api_path, cmd_info, state_dir) if rc != 0: sys.exit(rc) @@ -752,6 +756,7 @@ def run_script_mode(args, cmd_info): terminate_group(proc) stop_workspace_servers(workflow_dir, worker_url=iii_env["III_URL"]) cleanup_config(config_path) + shutil.rmtree(state_dir) if __name__ == "__main__": From 137e8c45e17515f36a4fa43a95c3c6664a252abe Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Wed, 23 Sep 2026 13:52:37 +0800 Subject: [PATCH 164/196] refactor: streamline test cases by removing unused regression scripts and updating command execution in helper --- .../api/steps/config/scripts/3_json2proto.py | 13 ++--- bbdev/api/steps/uvm/scripts/uvm_common.py | 55 ++++++++++++++++--- bbdev/api/tests/_api_test_helper.py | 5 +- bbdev/api/tests/test_all_api_serial.py | 3 - .../api/tests/test_api_bebop_bemu_analysis.py | 2 +- bbdev/api/tests/test_api_bebop_bemu_sim.py | 2 +- bbdev/api/tests/test_api_bebop_p2e_batch.py | 4 -- bbdev/api/tests/test_api_regression_check.py | 4 -- .../test_api_regression_eval_performance.py | 4 -- 9 files changed, 54 insertions(+), 38 deletions(-) mode change 100755 => 100644 bbdev/api/tests/test_all_api_serial.py delete mode 100644 bbdev/api/tests/test_api_bebop_p2e_batch.py delete mode 100644 bbdev/api/tests/test_api_regression_check.py delete mode 100644 bbdev/api/tests/test_api_regression_eval_performance.py diff --git a/bbdev/api/steps/config/scripts/3_json2proto.py b/bbdev/api/steps/config/scripts/3_json2proto.py index bf8cafcb6..a34121ff3 100644 --- a/bbdev/api/steps/config/scripts/3_json2proto.py +++ b/bbdev/api/steps/config/scripts/3_json2proto.py @@ -46,10 +46,6 @@ def _raw_core_list(designs: dict[str, Any]) -> list[dict[str, Any]]: return list(_derive.iter_cores(designs)) -def _raw_tile_proto(designs: dict[str, Any]) -> dict[str, Any]: - return _derive.iter_topology_tiles(designs)[0] - - def _fill_bank(msg: pb.BankConfig, d: dict[str, Any]) -> None: msg.num = d["num"] msg.width = d["width"] @@ -245,8 +241,7 @@ def fill_chip(config: dict[str, Any], derived: dict[str, Any], bbdir: Path) -> p Core/tile lists vs templates are already expanded in step2 — import that walker, do not write a second one. - Attention: All tiles on a chip must match, so SRAM/dcache - is copied from the first tile onto every TilePlacement. + Pair each tile's hardware parameters with its own derived placement. """ b = pb.Chip() b.name = derived["name"] @@ -268,9 +263,9 @@ def fill_chip(config: dict[str, Any], derived: dict[str, Any], bbdir: Path) -> p for raw, meta in zip(raw_cores, meta_cores): _fill_core(b.cores.add(), raw, meta, bbdir) - tile_proto = _raw_tile_proto(config["designs"]) - for meta in derived["tiles"]: - _fill_tile(b.tiles.add(), meta, tile_proto) + raw_tiles = _derive.iter_topology_tiles(config["designs"]) + for raw, meta in zip(raw_tiles, derived["tiles"], strict=True): + _fill_tile(b.tiles.add(), meta, raw) targets = derived["targets"] for name, target in targets.items(): diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index f67d91238..81278dfaa 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -1,3 +1,4 @@ +import fcntl import os import re import shlex @@ -80,17 +81,30 @@ def selected_mappings(domain, ball: str | None): raise ValueError(f"ball {ball!r} not in chip.pb") -def vcs_defines(domain, mapping, bank_entries: int): +def vcs_defines(domain, mapping, core, tile): + shared_bank_num = ( + tile.shared_mem.entries // core.mem.bank.entries + if tile.shared_mem.enable + else 0 + ) + max_group_count = max(core.mem.bank.num, shared_bank_num) defs = [ f"+define+BB_IN_BW={mapping.in_bw}", f"+define+BB_OUT_BW={mapping.out_bw}", f"+define+BB_MMIO_READ_BW={mapping.mmio_read_bw}", f"+define+BB_MMIO_WRITE_BW={mapping.mmio_write_bw}", - f"+define+BB_BANK_ADDR_W={(bank_entries - 1).bit_length()}", + f"+define+BB_BANK_ID_W={(tile.virtual_bank_count - 1).bit_length()}", + f"+define+BB_GROUP_COUNT_W={max_group_count.bit_length()}", + f"+define+BB_GROUP_ID_W={(max_group_count - 1).bit_length()}", + f"+define+BB_BANK_ADDR_W={(core.mem.bank.entries - 1).bit_length()}", + f"+define+BB_ROB_ID_W={(core.frontend.rob_entries - 1).bit_length()}", + f"+define+BB_SUB_ROB_ID_W={(core.frontend.sub_rob_depth * 4 - 1).bit_length()}", ] for e in domain.isa: if e.bid == mapping.ball_id: defs.append(f"+define+{e.mnemonic}_FUNCT7={e.funct7}") + if mapping.ball_dir == "smatmul": + defs.append(f"+define+SMATMUL_TILE_ROWS={mapping.ball_params['tileRows']}") return defs @@ -120,6 +134,16 @@ def _filelist( text = text.replace( "@SMATMUL_ACCUMULATOR@", smatmul_accumulator_filename(rtl_dir) ) + if "@IM2COL_BUFFER@" in text: + modules = set( + re.findall( + r"\b(buf_\d+x\d+)\s+buf(?:Lo|Hi)_ext\s*\(", + (rtl_dir / "LineBufferManager.sv").read_text(), + ) + ) + if len(modules) != 1: + raise ValueError(f"expected one Im2col buffer module, found {modules!r}") + text = text.replace("@IM2COL_BUFFER@", f"{modules.pop()}.sv") dst.write_text(text.replace("@UVM@", uvm_rel).replace("@RTL@", rtl_rel)) return str(dst.relative_to(verify_dir)) @@ -194,6 +218,9 @@ def build_ip(bbdir: str, chip: str, name: str, ctx) -> dict: manifest = root / config["model"] rtl = root / "build" sim_root = rtl / "uvm" / chip + rtl.mkdir(parents=True, exist_ok=True) + build_lock = (rtl / ".build.lock").open("w") + fcntl.flock(build_lock, fcntl.LOCK_EX) cargo = ( f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " @@ -259,6 +286,7 @@ def build_ip(bbdir: str, chip: str, name: str, ctx) -> dict: ) if result.returncode != 0: raise RuntimeError(f"VCS failed for IP {name} target={target['name']}") + build_lock.close() return config @@ -279,9 +307,12 @@ def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str) -> dict: coverage = [] for target in config["targets"]: simv = sim_root / target["name"] / "simv" + simv_q = shlex.quote(str(simv)) cov_dir = Path(cov_root) / target["name"] / "coverage" script = ( - f'env LD_LIBRARY_PATH="$VCS_RUNTIME_LIBRARY_PATH" {shlex.quote(str(simv))} ' + f'loader="$(patchelf --print-interpreter {simv_q})"; ' + f'library_path="$(patchelf --print-rpath {simv_q})"; ' + f'"$loader" --library-path "$library_path" {simv_q} -no_save ' f"-sv_lib {shlex.quote(str(model))} +UVM_TESTNAME={shlex.quote(target['test'])} " f"-cm line+cond+tgl+assert -cm_name {shlex.quote(target['test'])}" ) @@ -339,7 +370,9 @@ def build_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx) verify_dir = Path(bbdir) / "examples" / "balls" / ball / "verify" casegen = verify_dir / "casegen" / "Cargo.toml" rtl_dir = Path(bbdir) / "arch" / "build" / chip_name / mill_cfg - bank_entries = load_chip(bbdir, chip_name).cores[0].mem.bank.entries + chip = load_chip(bbdir, chip_name) + core = next(core for core in chip.cores if any(item.ball_dir == ball for item in core.balldomain.mappings)) + tile = next(tile for tile in chip.tiles if core.index in tile.core_indices) sim_dir = verify_dir / "build" / chip_name uvm_rel = os.path.relpath(Path(bbdir) / "verify" / "uvm", verify_dir) rtl_rel = os.path.relpath(rtl_dir, verify_dir) @@ -371,7 +404,7 @@ def build_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx) f"mkdir -p {shlex.quote(str(sim_dir))} {shlex.quote(str(csrc))} && " "vcs -full64 -sverilog -timescale=1ns/1ps -debug_access+all " "${=VCS_UVM_ARGS} " - + " ".join(shlex.quote(d) for d in vcs_defines(domain, mapping, bank_entries)) + + " ".join(shlex.quote(d) for d in vcs_defines(domain, mapping, core, tile)) + f" -cm line+cond+tgl+assert -cm_hier {shlex.quote(str(hier))} " f"-Mdir={shlex.quote(str(csrc))} -o {shlex.quote(str(simv))} " f"-f {shlex.quote(flist)}" @@ -402,14 +435,18 @@ def run_ball( dpi = verify_dir / "casegen" / "target" / "debug" / f"lib{crate.replace('-', '_')}" test = f"{ball}_ball_test" verify_config = verify_dir / "build" / chip_name / "verify_config.env" - bank_entries = load_chip(bbdir, chip_name).cores[0].mem.bank.entries + chip = load_chip(bbdir, chip_name) + core = next(core for core in chip.cores if any(item.ball_dir == ball for item in core.balldomain.mappings)) verify_config.write_text( - f"bank_entries={bank_entries}\nball_id={mapping.ball_id}\n" + f"bank_entries={core.mem.bank.entries}\nball_id={mapping.ball_id}\n" ) + simv_q = shlex.quote(str(simv)) script = ( f"cd {shlex.quote(str(verify_dir))} && " - f'env LD_LIBRARY_PATH="$VCS_RUNTIME_LIBRARY_PATH" BB_VERIFY_CONFIG={shlex.quote(str(verify_config))} ' - f"{shlex.quote(str(simv))} " + f'loader="$(patchelf --print-interpreter {simv_q})"; ' + f'library_path="$(patchelf --print-rpath {simv_q})"; ' + f'BB_VERIFY_CONFIG={shlex.quote(str(verify_config))} ' + f'"$loader" --library-path "$library_path" {simv_q} -no_save ' f"-sv_lib {shlex.quote(str(dpi))} " f"+UVM_TESTNAME={shlex.quote(test)} +BID={mapping.ball_id} " f"-cm line+cond+tgl+assert -cm_name {shlex.quote(test)}" diff --git a/bbdev/api/tests/_api_test_helper.py b/bbdev/api/tests/_api_test_helper.py index b63177469..7da793d8c 100644 --- a/bbdev/api/tests/_api_test_helper.py +++ b/bbdev/api/tests/_api_test_helper.py @@ -1,7 +1,6 @@ +import shlex import subprocess def run_bbdev_case(command: str): - result = subprocess.run(command, shell=True) - if result.returncode != 0: - raise RuntimeError(f"command failed({result.returncode}): {command}") + subprocess.run(shlex.split(command), check=True) diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py old mode 100755 new mode 100644 index f68793435..3ef7ff3bb --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -39,10 +39,7 @@ def main(): "test_api_bebop_p2e_verilog.py", "test_api_bebop_p2e_buildbitstream.py", "test_api_bebop_p2e_runworkload.py", - "test_api_bebop_p2e_batch.py", "test_api_regression_buildbitstream.py", - "test_api_regression_check.py", - "test_api_regression_eval_performance.py", "test_api_regression_eval_area_power.py", ] for name in tests: diff --git a/bbdev/api/tests/test_api_bebop_bemu_analysis.py b/bbdev/api/tests/test_api_bebop_bemu_analysis.py index 3f86a6681..542aba417 100644 --- a/bbdev/api/tests/test_api_bebop_bemu_analysis.py +++ b/bbdev/api/tests/test_api_bebop_bemu_analysis.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-bemu --analysis '--chip pebble --log-dir fixtures/bemu-analysis --itrace --mtrace'") # fmt: skip +run_bbdev_case("bbdev bebop-bemu --analysis '--chip pebble --log-dir fixtures/bemu-analysis --itrace --mtrace'") diff --git a/bbdev/api/tests/test_api_bebop_bemu_sim.py b/bbdev/api/tests/test_api_bebop_bemu_sim.py index 82774073a..179a87d2a 100644 --- a/bbdev/api/tests/test_api_bebop_bemu_sim.py +++ b/bbdev/api/tests/test_api_bebop_bemu_sim.py @@ -1,4 +1,4 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_relu_test-singlecore-baremetal'") # fmt: skip +run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_relu_test-singlecore-baremetal'") run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_relu_test-linux --pk'") diff --git a/bbdev/api/tests/test_api_bebop_p2e_batch.py b/bbdev/api/tests/test_api_bebop_p2e_batch.py deleted file mode 100644 index fc1952f53..000000000 --- a/bbdev/api/tests/test_api_bebop_p2e_batch.py +++ /dev/null @@ -1,4 +0,0 @@ -from _api_test_helper import run_bbdev_case - -# You need to set the bitstream path (case dir is derived from it). -# run_bbdev_case("bbdev bebop-p2e --batch '--chip pebble --bitstream /path/to/bitstream.bit --test elf-tests'") diff --git a/bbdev/api/tests/test_api_regression_check.py b/bbdev/api/tests/test_api_regression_check.py deleted file mode 100644 index 9fae0db63..000000000 --- a/bbdev/api/tests/test_api_regression_check.py +++ /dev/null @@ -1,4 +0,0 @@ -from _api_test_helper import run_bbdev_case - -# You need to set the bitstream path. -# run_bbdev_case("bbdev regression --check '--chip pebble --bitstream /path/to/bitstream.bit'") diff --git a/bbdev/api/tests/test_api_regression_eval_performance.py b/bbdev/api/tests/test_api_regression_eval_performance.py deleted file mode 100644 index 2798f1c37..000000000 --- a/bbdev/api/tests/test_api_regression_eval_performance.py +++ /dev/null @@ -1,4 +0,0 @@ -from _api_test_helper import run_bbdev_case - -# You need to set the bitstream path. -# run_bbdev_case("bbdev regression --eval-performance '--chip pebble --bitstream /path/to/bitstream.bit'") From 5cc4c1e8765b4ff19e87b68c4298f43c9f240139 Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Wed, 23 Sep 2026 21:04:16 +0800 Subject: [PATCH 165/196] fix: pass P2E build environment through Nix --- .../steps/bebop/p2e/02_buildbitstream_event.step.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index b9f8bc3bd..9c5112e84 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -67,7 +67,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: diff = bool(input_data.get("diff", False)) manifest = Path(bbdir) / "examples" / "chips" / chip / "generated" / "bebop" / "Cargo.toml" features = ["p2e"] - build_env = {**os.environ, **bebop_cargo_env(bbdir, chip)} + build_env = { + **os.environ, + **bebop_cargo_env(bbdir, chip), + "VSRC_PATH": vsrc_dir, + "OUT_PATH": str(build_path), + } if diff: features.append("bemu") build_env["CARGO_TARGET_DIR"] = os.path.join(bebop_dir, "target", f"{chip}-p2e-diff") @@ -76,6 +81,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--keep-env-var", "HOME", "--keep-env-var", "ALL_PROXY", "--keep-env-var", "CARGO_TARGET_DIR", + "--keep-env-var", "VSRC_PATH", + "--keep-env-var", "OUT_PATH", "-c", "cargo", "run", "--release", "--manifest-path", str(manifest), "--bin", "bebop", @@ -89,7 +96,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: build_result = await stream_run_logger_async( cmd=build_cmd, logger=ctx.logger, - cwd=str(manifest.parent), + cwd=bebop_dir, stdout_prefix="bebop p2e build", stderr_prefix="bebop p2e build", env=build_env, From 8f591f90fb9b4f0138b0f38081099138d1d1dc8c Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Thu, 24 Sep 2026 18:07:43 +0800 Subject: [PATCH 166/196] fix: improve server lock handling in bbdev and streamline IP build process in uvm_common --- bbdev/api/steps/uvm/scripts/uvm_common.py | 11 ++--------- bbdev/bbdev | 8 ++++---- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index 81278dfaa..fbef56b61 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -291,19 +291,13 @@ def build_ip(bbdir: str, chip: str, name: str, ctx) -> dict: def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str) -> dict: - config = load_ip(bbdir, name) + config = build_ip(bbdir, chip, name, ctx) root = Path(bbdir) / config["root"] manifest = root / config["model"] crate = tomllib.loads(manifest.read_text())["package"]["name"].replace("-", "_") model = manifest.parent / "target" / "debug" / f"lib{crate}" sim_root = root / "build" / "uvm" / chip - if any( - not (sim_root / target["name"] / "simv").is_file() - for target in config["targets"] - ): - config = build_ip(bbdir, chip, name, ctx) - coverage = [] for target in config["targets"]: simv = sim_root / target["name"] / "simv" @@ -427,8 +421,7 @@ def run_ball( ball = mapping.ball_dir verify_dir = Path(bbdir) / "examples" / "balls" / ball / "verify" simv = verify_dir / "build" / chip_name / "simv" - if not simv.is_file(): - build_ball(bbdir, chip_name, mill_cfg, domain, mapping, ctx) + build_ball(bbdir, chip_name, mill_cfg, domain, mapping, ctx) crate = tomllib.loads((verify_dir / "casegen" / "Cargo.toml").read_text())[ "package" ]["name"] diff --git a/bbdev/bbdev b/bbdev/bbdev index bb1c60281..e7e74c57d 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -699,10 +699,6 @@ def run_script_mode(args, cmd_info): ): sys.exit(1) - fcntl.flock(server_start_lock, fcntl.LOCK_UN) - server_start_lock.close() - server_start_lock = None - need = motia_worker_count() for _ in range(need): worker_proc = subprocess.Popen( @@ -739,6 +735,10 @@ def run_script_mode(args, cmd_info): print(f"Error: {e}", file=sys.stderr) sys.exit(1) + fcntl.flock(server_start_lock, fcntl.LOCK_UN) + server_start_lock.close() + server_start_lock = None + print(f"Server is ready on port {available_port} with {n} Motia worker(s)") rc = _submit_and_poll(available_port, api_path, cmd_info, state_dir) if rc != 0: From 2b7568e63e01d0e20523a8acc2dfc34698559dcb Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Thu, 24 Sep 2026 23:13:10 +0800 Subject: [PATCH 167/196] fix: terminate detached descendants on bbdev cancellation --- bbdev/api/tests/test_process_cleanup.py | 72 +++++++++++++++++++++++++ bbdev/api/utils/process.py | 23 +------- bbdev/api/utils/process_registry.py | 7 ++- bbdev/api/utils/stream_run.py | 4 +- 4 files changed, 79 insertions(+), 27 deletions(-) create mode 100644 bbdev/api/tests/test_process_cleanup.py diff --git a/bbdev/api/tests/test_process_cleanup.py b/bbdev/api/tests/test_process_cleanup.py new file mode 100644 index 000000000..7fe5f8246 --- /dev/null +++ b/bbdev/api/tests/test_process_cleanup.py @@ -0,0 +1,72 @@ +import os +import shlex +import signal +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from utils.process import _exists, terminate_group +from utils.process_registry import cancel_process, register_process, unregister_process +from utils.stream_run import stream_run + + +class ProcessCleanupTest(unittest.TestCase): + def test_nested_session_exits_on_shutdown_and_cancel(self): + for cancel in (False, True): + with self.subTest(cancel=cancel): + parent = subprocess.Popen( + [ + sys.executable, + "-c", + "import subprocess, sys, time; " + "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(300)'], start_new_session=True); " + "print(child.pid, flush=True); time.sleep(300)", + ], + stdout=subprocess.PIPE, + text=True, + start_new_session=True, + ) + child_pid = int(parent.stdout.readline()) + try: + if cancel: + register_process("nested-session-test", parent.pid) + self.assertTrue(cancel_process("nested-session-test")) + parent.wait(timeout=2) + else: + terminate_group(parent, term_timeout=1) + self.assertFalse(_exists(child_pid)) + finally: + unregister_process("nested-session-test") + if _exists(child_pid): + os.kill(child_pid, signal.SIGKILL) + if parent.poll() is None: + parent.kill() + parent.wait() + parent.stdout.close() + + def test_timeout_exits_nested_session(self): + with tempfile.TemporaryDirectory() as directory: + child_pid_file = Path(directory) / "child.pid" + command = shlex.join( + [ + sys.executable, + "-c", + "import subprocess, sys, time; " + "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(300)'], start_new_session=True); " + "open(sys.argv[1], 'w').write(str(child.pid)); time.sleep(300)", + str(child_pid_file), + ] + ) + try: + result = stream_run(command, timeout=1) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(_exists(int(child_pid_file.read_text()))) + finally: + if child_pid_file.exists(): + child_pid = int(child_pid_file.read_text()) + if _exists(child_pid): + os.kill(child_pid, signal.SIGKILL) diff --git a/bbdev/api/utils/process.py b/bbdev/api/utils/process.py index 0105f3441..f0039659f 100644 --- a/bbdev/api/utils/process.py +++ b/bbdev/api/utils/process.py @@ -113,27 +113,8 @@ def kill_tree(pid: int, term_timeout: float = 5.0): def terminate_group(proc: subprocess.Popen, term_timeout: float = 5.0): if proc.poll() is not None: return - - try: - os.killpg(proc.pid, signal.SIGTERM) - except (ProcessLookupError, PermissionError, OSError): - kill_tree(proc.pid, term_timeout=term_timeout) - - try: - proc.wait(timeout=term_timeout) - return - except subprocess.TimeoutExpired: - pass - - try: - os.killpg(proc.pid, signal.SIGKILL) - except (ProcessLookupError, PermissionError, OSError): - kill_tree(proc.pid, term_timeout=1.0) - - try: - proc.wait(timeout=1) - except subprocess.TimeoutExpired: - pass + kill_tree(proc.pid, term_timeout=term_timeout) + proc.wait(timeout=1) def _is_workspace_server(pid: int, workflow_dir: str, worker_url: str | None = None) -> bool: diff --git a/bbdev/api/utils/process_registry.py b/bbdev/api/utils/process_registry.py index 27ec1a4ee..f2acd257b 100644 --- a/bbdev/api/utils/process_registry.py +++ b/bbdev/api/utils/process_registry.py @@ -6,6 +6,8 @@ from contextvars import ContextVar from typing import Dict +from utils.process import kill_tree + _lock = threading.Lock() _processes: Dict[str, int] = {} @@ -38,10 +40,7 @@ def cancel_process(scope: str) -> bool: _cancelled.add(scope) if pid is None: return False - try: - os.killpg(pid, signal.SIGTERM) - except ProcessLookupError: - return False + kill_tree(pid) return True diff --git a/bbdev/api/utils/stream_run.py b/bbdev/api/utils/stream_run.py index 699efa87c..0fb2a02ab 100644 --- a/bbdev/api/utils/stream_run.py +++ b/bbdev/api/utils/stream_run.py @@ -3,6 +3,7 @@ import threading from typing import Optional, List, Callable +from utils.process import kill_tree from utils.process_registry import current_task_scope, register_process, unregister_process @@ -133,8 +134,7 @@ def emit(): # Wait for process to finish (with timeout) process.wait(timeout=timeout) except subprocess.TimeoutExpired: - # Kill process on timeout - process.kill() + kill_tree(process.pid) process.wait() # Wait for threads to finish From d372dd9358a46f5be45d7c0fa17ebfbe5cd6c956 Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Fri, 25 Sep 2026 07:48:23 +0800 Subject: [PATCH 168/196] test: move process cleanup unit tests out of API examples --- bbdev/api/{tests => unit_tests}/test_process_cleanup.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename bbdev/api/{tests => unit_tests}/test_process_cleanup.py (100%) diff --git a/bbdev/api/tests/test_process_cleanup.py b/bbdev/api/unit_tests/test_process_cleanup.py similarity index 100% rename from bbdev/api/tests/test_process_cleanup.py rename to bbdev/api/unit_tests/test_process_cleanup.py From 1faa8f04818e4b704a86ed3d3ff0fb7919a852c0 Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Fri, 25 Sep 2026 07:37:42 +0800 Subject: [PATCH 169/196] fix: run P2E workloads with waveform tools available --- bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index c96bdb4a1..4074ca014 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -187,7 +187,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return run_cmd = ( - f"\"{bebop_p2e_path}\" run p2e " + f"nix develop -c \"{bebop_p2e_path}\" run p2e " f"--image=\"{image_path}\" " f"--bitstream=\"{bitstream}\" " f"--log-dir=\"{run_log}\" " From 055ab6464de060b50ced71087e94b13bd83c1510 Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Fri, 25 Sep 2026 23:35:04 +0800 Subject: [PATCH 170/196] Capture BEMU run output in log directory (#23) --- bbdev/api/steps/bebop/bemu/02_sim_event.step.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 12d0103f7..bd6d4813d 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -176,11 +176,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: library_env = f"LD_LIBRARY_PATH={shlex.quote(dependency_path)}:${{LD_LIBRARY_PATH:-}}" inner_cmd = f"cd {shlex.quote(binary_dir)} && {build_cmd} && {copy_cmd} && {library_env} exec {shlex.quote(binary_path)}" run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" + run_cmd = f"set -o pipefail; {run_cmd} 2>&1 | tee {shlex.quote(str(Path(run_log) / 'run.log'))}" ctx.logger.info(f"Running rushB BEMU: {run_cmd}") run_result = await stream_run_logger_async( cmd=run_cmd, logger=ctx.logger, cwd=bbdir, + executable="/bin/bash", stdout_prefix="rushB bemu", stderr_prefix="rushB bemu", task_scope=origin_tid, @@ -258,11 +260,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cargo_args.append(f"--{trace_name}") inner_cmd = f"cd {shlex.quote(binary_dir)} && {shlex.join(cargo_args)}" run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" + run_cmd = f"set -o pipefail; {run_cmd} 2>&1 | tee {shlex.quote(str(Path(run_log) / 'run.log'))}" ctx.logger.info(f"Running bebop bemu: {run_cmd}") run_result = await stream_run_logger_async( cmd=run_cmd, logger=ctx.logger, cwd=bbdir, + executable="/bin/bash", stdout_prefix="bebop bemu", stderr_prefix="bebop bemu", task_scope=origin_tid, From 3e4b176bb9bacd898f840113d3899262be3ffc9d Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Fri, 25 Sep 2026 23:43:16 +0800 Subject: [PATCH 171/196] Refresh P2E runtime before workloads (#22) * fix: distinguish P2E diff build directories * fix: refresh P2E runtime before running workloads --- .../bebop/p2e/02_buildbitstream_event.step.py | 4 +- .../bebop/p2e/03_runworkload_event.step.py | 49 +++++++++++++++++-- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index 9c5112e84..1dc5782fa 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -52,9 +52,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: vsrc_dir = rtl_dir(bbdir, chip, "p2e", input_data.get("vsrc_dir")) normalize_p2e_timescale(vsrc_dir, ctx.logger) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + diff = bool(input_data.get("diff", False)) build_dir = ( input_data.get("output_dir") - or f"{bebop_dir}/build/{chip}-{timestamp}" + or f"{bebop_dir}/build/{chip}{'-diff' if diff else ''}-{timestamp}" ) build_path = Path(build_dir).resolve() build_root = (Path(bebop_dir) / "build").resolve() @@ -64,7 +65,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: shutil.rmtree(build_path) build_path.mkdir(parents=True) - diff = bool(input_data.get("diff", False)) manifest = Path(bbdir) / "examples" / "chips" / chip / "generated" / "bebop" / "Cargo.toml" features = ["p2e"] build_env = { diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index 4074ca014..90cd70cea 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -10,6 +10,7 @@ import os import re import shlex +import shutil import sys from datetime import datetime @@ -166,9 +167,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") libvctb_path = os.path.join(runtime_lib_dir, "libvCtb.so") bebop_p2e_path = os.path.join(build_dir, "bebop-p2e") - runtime_artifacts = [rtcfg_path, libvctb_path, bebop_p2e_path] - if diff: - runtime_artifacts.append(os.path.join(build_dir, "libriscv.so")) + runtime_artifacts = [rtcfg_path, libvctb_path] if not all(os.path.isfile(path) for path in runtime_artifacts): missing = [path for path in runtime_artifacts if not os.path.isfile(path)] if missing: @@ -186,6 +185,48 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return + manifest = os.path.join(bbdir, "examples", "chips", chip, "generated", "bebop", "Cargo.toml") + build_env = {**os.environ, **bebop_cargo_env(bbdir, chip), "OUT_PATH": build_dir} + if diff: + build_env["CARGO_TARGET_DIR"] = os.path.join(bebop_dir, "target", f"{chip}-p2e-diff") + build_cmd = shlex.join([ + "nix", "develop", "--ignore-env", + "--keep-env-var", "HOME", + "--keep-env-var", "ALL_PROXY", + "--keep-env-var", "CARGO_TARGET_DIR", + "--keep-env-var", "OUT_PATH", + "-c", "cargo", "build", "--release", + "--manifest-path", manifest, + "--bin", "bebop", + "--features", "p2e,bemu" if diff else "p2e", + ]) + ctx.logger.info("Building bebop p2e runtime for the selected case...") + build_result = await stream_run_logger_async( + cmd=build_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop p2e runtime build", + stderr_prefix="bebop p2e runtime build", + env=build_env, + ) + if build_result.returncode != 0: + await check_result( + ctx, build_result.returncode, continue_run=False, + extra_fields={"task": "runtime_build", "build_dir": build_dir}, + trace_id=origin_tid, + ) + return + + target_release = os.path.join(build_env["CARGO_TARGET_DIR"], "release") + staged_runtime = f"{bebop_p2e_path}.new" + shutil.copy2(os.path.join(target_release, "bebop"), staged_runtime) + os.replace(staged_runtime, bebop_p2e_path) + if diff: + libriscv_path = os.path.join(build_dir, "libriscv.so") + staged_libriscv = f"{libriscv_path}.new" + shutil.copy2(os.path.join(target_release, "bemu-runtime", "libriscv.so"), staged_libriscv) + os.replace(staged_libriscv, libriscv_path) + run_cmd = ( f"nix develop -c \"{bebop_p2e_path}\" run p2e " f"--image=\"{image_path}\" " @@ -205,7 +246,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if input_data.get(trace_name, False): run_cmd += f" --{trace_name}" ctx.logger.info(f"Running bebop p2e runworkload: {run_cmd}") - run_env = {**os.environ.copy(), **bebop_cargo_env(bbdir, chip)} + run_env = build_env run_result = await stream_run_logger_async( cmd=run_cmd, logger=ctx.logger, From 77940ebc2e2f81bf5596fb51cbd461cb8fa53fdd Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 29 Sep 2026 16:24:30 +0800 Subject: [PATCH 172/196] feat: Enhance BEMU and model simulation workflows --- bbdev/api/services/workflow_tools.py | 14 +- bbdev/api/steps/bebop/bemu/02_sim_api.step.py | 11 +- .../api/steps/bebop/bemu/02_sim_event.step.py | 165 ++++++----------- .../api/steps/bebop/bemu/03_batch_api.step.py | 41 ++--- .../steps/bebop/bemu/03_batch_event.step.py | 53 ++---- .../api/steps/bebop/bemu/scripts/model_sim.py | 48 +++++ .../bebop/p2e/02_buildbitstream_event.step.py | 2 +- .../bebop/p2e/03_runworkload_event.step.py | 147 ++++++++------- .../steps/bebop/p2e/04_batch_event.step.py | 102 ++++++----- .../bebop/p2e/scripts/timescale.py} | 0 .../bebop/verilator/01_clean_api.step.py | 15 +- .../bebop/verilator/01_clean_event.step.py | 27 +-- .../bebop/verilator/02_verilog_api.step.py | 17 +- .../bebop/verilator/03_build_api.step.py | 20 +-- .../bebop/verilator/03_build_event.step.py | 31 ++-- .../bebop/verilator/04_sim_event.step.py | 162 +++++++---------- .../steps/bebop/verilator/05_run_api.step.py | 21 +-- .../bebop/verilator/06_batch_api.step.py | 22 +-- .../bebop/verilator/06_batch_event.step.py | 65 ++----- bbdev/api/steps/compiler/scripts/build.py | 15 +- bbdev/api/steps/config/scripts/4_install.py | 100 ++--------- bbdev/api/steps/config/scripts/model_pb2.py | 42 +++++ .../steps/config/scripts/proto/model.proto | 17 ++ bbdev/api/steps/ip/scripts/sram_compiler.py | 2 +- bbdev/api/steps/kernel/01_build_event.step.py | 29 +-- .../06_bebop_verilator_verilog_event.step.py | 56 +++--- .../mill/07_bebop_p2e_verilog_event.step.py | 2 +- bbdev/api/steps/model/01_build_api.step.py | 26 +++ bbdev/api/steps/model/01_build_event.step.py | 36 ++++ bbdev/api/steps/model/scripts/build.py | 67 +++++++ bbdev/api/steps/model/scripts/config.py | 82 +++++++++ .../02_eval_performance_event.step.py | 42 ++--- .../steps/regression/scripts/model_layout.py | 84 +-------- bbdev/api/steps/uvm/scripts/uvm_common.py | 87 +++++++-- bbdev/api/steps/uvm/scripts/waive.py | 17 +- .../api/steps/verilator/04_sim_event.step.py | 133 ++++++-------- bbdev/api/steps/workload/01_buidl_api.step.py | 57 ++---- .../api/steps/workload/01_build_event.step.py | 169 +++++------------- bbdev/api/steps/workload/README.md | 19 +- bbdev/api/utils/model.py | 21 ++- bbdev/api/utils/path.py | 31 ++-- bbdev/api/utils/workload_manifest.py | 4 - bbdev/bbdev | 15 +- bbdev/mcp/common.py | 75 ++------ bbdev/mcp/tools/__init__.py | 2 + bbdev/mcp/tools/bebop_verilator_batch.py | 14 +- bbdev/mcp/tools/bebop_verilator_run.py | 7 +- bbdev/mcp/tools/bebop_verilator_sim.py | 5 +- bbdev/mcp/tools/bemu_batch.py | 14 +- bbdev/mcp/tools/bemu_sim.py | 9 +- bbdev/mcp/tools/kernel_build.py | 4 +- bbdev/mcp/tools/model_build.py | 15 ++ bbdev/mcp/tools/workload_build.py | 17 +- 53 files changed, 1032 insertions(+), 1246 deletions(-) create mode 100644 bbdev/api/steps/bebop/bemu/scripts/model_sim.py rename bbdev/api/{utils/p2e_timescale.py => steps/bebop/p2e/scripts/timescale.py} (100%) create mode 100644 bbdev/api/steps/config/scripts/model_pb2.py create mode 100644 bbdev/api/steps/config/scripts/proto/model.proto create mode 100644 bbdev/api/steps/model/01_build_api.step.py create mode 100644 bbdev/api/steps/model/01_build_event.step.py create mode 100644 bbdev/api/steps/model/scripts/build.py create mode 100644 bbdev/api/steps/model/scripts/config.py create mode 100644 bbdev/mcp/tools/model_build.py diff --git a/bbdev/api/services/workflow_tools.py b/bbdev/api/services/workflow_tools.py index ace61fff6..67a2a83f9 100644 --- a/bbdev/api/services/workflow_tools.py +++ b/bbdev/api/services/workflow_tools.py @@ -12,13 +12,7 @@ def get_name(self) -> str: return "call_workflow_api" def get_description(self) -> str: - return """Call internal workflow API endpoints. - Available endpoints: - - /verilator/verilog: Generate Verilog - - /verilator/build: Build verilator (params: jobs) - - /verilator/sim: Run simulation (params: binary, batch) - - /workload/clean: Clean workload output directory - - /workload/build: Build workload (params: chip, model, stable, rushB)""" + return "Call internal workflow API endpoints.\n Available endpoints:\n - /verilator/verilog: Generate Verilog\n - /verilator/build: Build verilator (params: jobs)\n - /verilator/sim: Run simulation (params: binary, batch)\n - /workload/clean: Clean workload output directory\n - /workload/build: Build workload (params: chip, stable, ctest, mlirtest)\n - /model/build: Build model recipe (params: chip, model, ctrace, dtrace)" def get_parameters(self) -> Dict[str, Any]: return { @@ -40,16 +34,11 @@ def get_parameters(self) -> Dict[str, Any]: def execute(self, arguments: Dict[str, Any], context: Any) -> str: endpoint = arguments.get("endpoint") params = arguments.get("params", {}) - url = f"http://localhost:3001{endpoint}" - try: context.log_info(f"Calling workflow API: {url}") context.log_info(f"Parameters: {params}") - - # Synchronous call (using httpx sync client) response = httpx.post(url, json=params, timeout=300.0) - if response.status_code == 200: return str(response.json()) else: @@ -59,6 +48,5 @@ def execute(self, arguments: Dict[str, Any], context: Any) -> str: "response": response.text[:500], } ) - except Exception as e: return str({"error": f"Workflow API call failed: {str(e)}"}) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py index 678949cb8..c9913bbd1 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py @@ -12,6 +12,7 @@ from utils.event_common import require_chip from utils.path import get_buckyball_path +from steps.bebop.bemu.scripts.model_sim import model_run_commands config = { "name": "bebop-bemu-sim-api", @@ -39,14 +40,20 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: ) binary = body.get("binary", "") - if not binary: + if body.get("model"): + try: + model_run_commands(get_buckyball_path(), body) + except (ValueError, KeyError, OSError) as error: + return ApiResponse(status=400, body={"success": False, "failure": True, + "returncode": 400, "message": str(error)}) + elif not binary: return ApiResponse( status=400, body={ "success": False, "failure": True, "returncode": 400, - "message": "binary parameter is required", + "message": "model or binary parameter is required", }, ) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index bd6d4813d..5109a340a 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -1,15 +1,14 @@ """ bebop bemu event handler -Run BEMU either through its guest-ELF emulator or the rushB native ABI. +Run a guest ELF through BEMU. """ + import os import shlex import sys -import tomllib from datetime import datetime from pathlib import Path - from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) @@ -18,14 +17,18 @@ scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) - -from utils.path import bebop_cargo_env, bebop_target_dir, get_buckyball_path, log_dir, workloads_output_root +from utils.path import ( + bebop_cargo_env, + get_buckyball_path, + log_dir, + workloads_output_root, +) from utils.stream_run import stream_run_logger_async from utils.search_workload import search_workload +from steps.bebop.bemu.scripts.model_sim import model_run_commands from utils.event_common import check_result, get_origin_trace_id from utils.process_registry import cancellation_requested -from bemu_common import bemu_core_manifest, bemu_manifest, bemu_tile_index, chip_emu_manifest - +from bemu_common import bemu_manifest, bemu_tile_index, chip_emu_manifest config = { "name": "bebop-bemu-sim", @@ -53,7 +56,6 @@ def clean_model_trace(binary_dir: str) -> None: if not summary.is_file(): raise FileNotFoundError(f"trace summary path is not a file: {summary}") summary.unlink() - perfetto = trace_dir / "perfetto.json" if perfetto.exists(): if not perfetto.is_file(): @@ -64,49 +66,23 @@ def clean_model_trace(binary_dir: str) -> None: def resolve_bemu_binary(bbdir: str, chip: str, binary_name: str) -> str | None: if Path(binary_name).name != binary_name: return None - workload = search_workload(workloads_output_root(bbdir), binary_name) if workload is not None: return workload - kernel = Path(bbdir) / "bb-tests" / "output" / "kernel" / chip / binary_name return str(kernel) if kernel.is_file() else None -def is_native_host_elf(path: str) -> bool: - try: - with open(path, "rb") as binary: - header = binary.read(20) - except OSError: - return False - return header[:4] == b"\x7fELF" and header[18:20] == b"\x3e\x00" - - -def rushb_bemu_library(manifest: Path, bbdir: str, chip: str) -> Path: - """Return the cdylib emitted by bebop-bemu for a rushB run.""" - with manifest.open("rb") as source: - cargo = tomllib.load(source) - lib_name = cargo.get("lib", {}).get("name") - if not lib_name: - lib_name = cargo["package"]["name"].replace("-", "_") - return Path(bebop_target_dir(bbdir, chip)) / "release" / f"lib{lib_name}.so" - - -def rushb_bemu_library_dirs(manifest: Path, bbdir: str, chip: str) -> list[Path]: - """Locate native shared-library dependencies produced by the bebop-bemu build.""" - build_dir = Path(bebop_target_dir(bbdir, chip)) / "release" / "build" - return sorted({library.parent for library in build_dir.glob("*/out/**/libriscv.so")}) - - async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - chip = input_data.get("chip") if not chip: ctx.logger.error("Missing required parameter: chip must be specified") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) @@ -116,18 +92,41 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: except ValueError as e: ctx.logger.error(str(e)) await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "invalid_chip", "chip": chip}, trace_id=origin_tid, ) return - + if input_data.get("model"): + params = {key: value for key, value in input_data.items() if key != "_trace_id"} + commands, run_log = model_run_commands(bbdir, params) + run_log.mkdir(parents=True, exist_ok=True) + ctx.logger.info(f"Model simulation logs: {run_log}") + for command, filename in zip(commands, ("build.log", "run.log"), strict=True): + command_line = shlex.join(["nix", "develop", "-c", *command]) + command_line = f"set -o pipefail; {command_line} 2>&1 | tee {shlex.quote(str(run_log / filename))}" + result = await stream_run_logger_async( + cmd=command_line, executable="/bin/bash", + logger=ctx.logger, cwd=bbdir, task_scope=origin_tid, + stdout_prefix="model simulation", stderr_prefix="model simulation") + if cancellation_requested(origin_tid): + return + if result.returncode: + break + await check_result(ctx, result.returncode, continue_run=False, + extra_fields={"chip": chip, "model": params["model"], "log_dir": str(run_log)}, + trace_id=origin_tid) + return binary_name = input_data.get("binary", "") binary_path = resolve_bemu_binary(bbdir, chip, binary_name) if binary_path is None: ctx.logger.error(f"binary not found: {binary_name}") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "binary_not_found", "binary": binary_name}, trace_id=origin_tid, ) @@ -141,72 +140,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: perfetto_target = f"{binary_name.removesuffix('-run')}-perfetto" if perfetto_target: clean_model_trace(binary_dir) - timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") run_log = log_dir(bbdir, chip, "verilog", timestamp, "bemu", binary_name) os.makedirs(run_log, exist_ok=True) - - if input_data.get("rushB"): - # rushB is a native ABI for one accelerator instance, so it remains a - # Core-level backend even when guest-ELF BEMU runs a whole tile. - bemu_cargo_manifest = bemu_core_manifest(chip, bbdir) - # rushB binaries are host executables. Rebuilding and co-locating the - # backend library keeps the ABI selection explicit and avoids sending - # a host ELF through Spike's guest-ELF path. - if not is_native_host_elf(binary_path): - ctx.logger.error( - f"--rushB requires a native x86_64 rushB runner; got guest ELF: {binary_path}" - ) - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "rushB_requires_native_runner", "binary": binary_path}, - trace_id=origin_tid, - ) - return - bemu_library = rushb_bemu_library(bemu_cargo_manifest, bbdir, chip) - bemu_runtime_library = Path(binary_dir) / "libbebop_bemu.so" - dependency_dirs = rushb_bemu_library_dirs(bemu_cargo_manifest, bbdir, chip) - build_cmd = shlex.join([ - "cargo", "build", "--release", "--manifest-path", str(bemu_cargo_manifest), "--lib", - ]) - copy_cmd = shlex.join([ - "cmake", "-E", "copy_if_different", str(bemu_library), str(bemu_runtime_library), - ]) - dependency_path = os.pathsep.join(str(path) for path in dependency_dirs) - library_env = f"LD_LIBRARY_PATH={shlex.quote(dependency_path)}:${{LD_LIBRARY_PATH:-}}" - inner_cmd = f"cd {shlex.quote(binary_dir)} && {build_cmd} && {copy_cmd} && {library_env} exec {shlex.quote(binary_path)}" - run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" - run_cmd = f"set -o pipefail; {run_cmd} 2>&1 | tee {shlex.quote(str(Path(run_log) / 'run.log'))}" - ctx.logger.info(f"Running rushB BEMU: {run_cmd}") - run_result = await stream_run_logger_async( - cmd=run_cmd, - logger=ctx.logger, - cwd=bbdir, - executable="/bin/bash", - stdout_prefix="rushB bemu", - stderr_prefix="rushB bemu", - task_scope=origin_tid, - env={**os.environ.copy(), **bebop_cargo_env(bbdir, chip)}, - ) - if cancellation_requested(origin_tid): - return - await check_result( - ctx, - run_result.returncode, - continue_run=False, - extra_fields={ - "task": "bemu", - "backend": "rushB", - "binary": binary_path, - "chip": chip, - "log_dir": run_log, - "timestamp": timestamp, - }, - trace_id=origin_tid, - ) - return - - # ── Run bebop bemu ──────────────────────────────────────────────────── + core_index = input_data.get("core_index") tile_index = bemu_tile_index(chip, bbdir) chip_emu = chip_emu_manifest(chip, bbdir) if (tile_index is not None) != bool(chip_emu): @@ -214,12 +151,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"chip {chip}: bundle bemu.chipMain and emu/Cargo.toml must both exist or both be absent" ) await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "chip_emu_entry_mismatch", "chip": chip}, trace_id=origin_tid, ) return - if chip_emu: + if chip_emu and core_index is None: cargo_args = [ "cargo", "run", @@ -249,8 +188,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--log-dir", run_log, ] + if core_index is not None: + cargo_args.extend(["--core-index", str(core_index)]) if input_data.get("pk"): cargo_args.append("--pk") + if input_data.get("host-io"): + cargo_args.append("--host-io") if input_data.get("disasm"): cargo_args.append("--disasm") if input_data.get("tool-profile"): @@ -258,6 +201,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: for trace_name in ("itrace", "mtrace"): if input_data.get(trace_name, False): cargo_args.append(f"--{trace_name}") + arguments = input_data.get("arguments", []) + if arguments: + if not isinstance(arguments, list) or not all(isinstance(arg, str) for arg in arguments): + raise ValueError("arguments must be a list of strings") + cargo_args.extend(["--", *arguments]) inner_cmd = f"cd {shlex.quote(binary_dir)} && {shlex.join(cargo_args)}" run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" run_cmd = f"set -o pipefail; {run_cmd} 2>&1 | tee {shlex.quote(str(Path(run_log) / 'run.log'))}" @@ -289,13 +237,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - perfetto_path = None if perfetto_target: - perfetto_cmd = ( - f"cmake --build {shlex.quote(f'{bbdir}/bb-tests/build')} " - f"--target {shlex.quote(perfetto_target)}" - ) + perfetto_cmd = f"cmake --build {shlex.quote(f'{bbdir}/bb-tests/build')} --target {shlex.quote(perfetto_target)}" ctx.logger.info(f"Generating Perfetto trace: {perfetto_cmd}") perfetto_result = await stream_run_logger_async( cmd=perfetto_cmd, @@ -321,7 +265,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - await check_result( ctx, 0, diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py index 2c8eb89ab..0b4e0cb59 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -1,6 +1,5 @@ import os import sys - from motia import ApiRequest, ApiResponse, FlowContext, api utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) @@ -9,9 +8,7 @@ scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) - from utils.event_common import require_chip -from utils.path import get_buckyball_path config = { "name": "bebop-bemu-batch-api", @@ -24,37 +21,37 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - chip = body.get("chip") try: require_chip({"chip": chip}) except ValueError as e: - return ApiResponse( - status=400, - body={"error": str(e)} - ) - + return ApiResponse(status=400, body={"error": str(e)}) test_type = body.get("test") if not test_type: return ApiResponse( status=400, - body={"error": "Missing required parameter: --test must be specified (elf-tests or pk-tests)"} + body={ + "error": "Missing required parameter: --test must be specified (elf-tests or pk-tests)" + }, ) - if test_type not in ["elf-tests", "pk-tests"]: return ApiResponse( status=400, - body={"error": f"Invalid test type: {test_type}. Must be 'elf-tests' or 'pk-tests'"} + body={ + "error": f"Invalid test type: {test_type}. Must be 'elf-tests' or 'pk-tests'" + }, ) - - await ctx.enqueue({ - "topic": "bebop.bemu.batch", - "data": { - "chip": chip, - "test": test_type, - "clean-before": body.get("clean-before", body.get("clean_before", False)), - "rushB": bool(body.get("rushB", False)), - "_trace_id": ctx.trace_id, + await ctx.enqueue( + { + "topic": "bebop.bemu.batch", + "data": { + "chip": chip, + "test": test_type, + "clean-before": body.get( + "clean-before", body.get("clean_before", False) + ), + "_trace_id": ctx.trace_id, + }, } - }) + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index afac30d7b..d9ed4efa2 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -10,7 +10,6 @@ import shutil import shlex import sys - from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) @@ -22,7 +21,6 @@ scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) - from utils.path import bebop_cargo_env, get_buckyball_path, workloads_output_root from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -52,15 +50,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - rushB = bool(input_data.get("rushB", False)) try: bemu_cargo_manifest = bemu_manifest(chip, bbdir) - if not rushB: - chip_emu = chip_emu_manifest(chip, bbdir) - if chip_emu is not None and 'name = "test_bemu"' in chip_emu.read_text( - encoding="utf-8" - ): - bemu_cargo_manifest = chip_emu + chip_emu = chip_emu_manifest(chip, bbdir) + if chip_emu is not None and 'name = "test_bemu"' in chip_emu.read_text( + encoding="utf-8" + ): + bemu_cargo_manifest = chip_emu except ValueError as e: ctx.logger.error(str(e)) await check_result( @@ -72,14 +68,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return elf_root = workloads_output_root(bbdir) - env = os.environ.copy() env.update(bebop_cargo_env(bbdir, chip)) test_type = input_data.get("test", "elf-tests") try: - workload_toml = resolve_workload_toml( - chip, "bemu", test_type, bbdir, rushB=rushB - ) + workload_toml = resolve_workload_toml(chip, "bemu", test_type, bbdir) except ValueError as e: ctx.logger.error(str(e)) await check_result( @@ -94,16 +87,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - - ctx.logger.info( - f"Running {test_type} with workload config: {workload_toml} rushB={rushB}" - ) - - # ── Build bebop bemu ────────────────────────────────────────────────── - build_cmd = ( - f"nix develop -c cargo build --manifest-path {shlex.quote(str(bemu_cargo_manifest))} " - "--tests" - ) + ctx.logger.info(f"Running {test_type} with workload config: {workload_toml}") + build_cmd = f"nix develop -c cargo build --manifest-path {shlex.quote(str(bemu_cargo_manifest))} --tests" ctx.logger.info("Building bebop bemu (tests)...") build_result = await stream_run_logger_async( cmd=build_cmd, @@ -113,7 +98,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="bebop bemu build", env=env, ) - if build_result.returncode != 0: await check_result( ctx, @@ -123,26 +107,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - - # ── Run regression ──────────────────────────────────────────────────── if input_data.get("clean-before", input_data.get("clean_before", False)): shutil.rmtree( os.path.join(env["CARGO_TARGET_DIR"], "test-artifacts"), ignore_errors=True ) ctx.logger.info("Cleaned previous bebop test artifacts") - - harness = shlex.join([ - "--", - "--workload-toml", workload_toml, - "--bb-tests-root", elf_root, - *(["--rushb-backend", "bemu"] if rushB else []), - ]) - test_cmd = ( - f"nix develop -c cargo test --manifest-path {shlex.quote(str(bemu_cargo_manifest))} " - "--test test_bemu " - f"{harness}" + harness = shlex.join( + ["--", "--workload-toml", workload_toml, "--bb-tests-root", elf_root, *[]] ) - + test_cmd = f"nix develop -c cargo test --manifest-path {shlex.quote(str(bemu_cargo_manifest))} --test test_bemu {harness}" ctx.logger.info(f"Running bebop bemu regression: {test_cmd}") run_result = await stream_run_logger_async( cmd=test_cmd, @@ -152,17 +125,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="bebop bemu batch", env=env, ) - await check_result( ctx, run_result.returncode, continue_run=False, extra_fields={ "task": "batch", - "backend": "bemu-rushB" if rushB else "bemu", + "backend": "bemu", "chip": chip, "test_type": test_type, - "rushB": rushB, "workload_toml": workload_toml, }, trace_id=origin_tid, diff --git a/bbdev/api/steps/bebop/bemu/scripts/model_sim.py b/bbdev/api/steps/bebop/bemu/scripts/model_sim.py new file mode 100644 index 000000000..bc3561fdb --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/scripts/model_sim.py @@ -0,0 +1,48 @@ +import json +import re +import tomllib +from datetime import datetime +from pathlib import Path + +from utils.path import log_dir + + +def model_run_commands(repo: str, params: dict): + unknown = params.keys() - {"chip", "model"} + if unknown: + raise ValueError(f"Unknown model simulation parameters: {sorted(unknown)}") + for key in ("chip", "model"): + if not isinstance(params.get(key), str) or not re.fullmatch(r"[A-Za-z0-9_-]+", params[key]): + raise ValueError(f"Invalid or missing {key}") + root = Path(repo) + chip, model = params["chip"], params["model"] + config = root / "examples/models" / chip / model / "configs/running-param.toml" + build = root / "stack/models/build" / chip / model + packages = list((build / "artifact").glob("*.rax")) + if len(packages) != 1: + raise ValueError(f"Expected one built RAX under {build / 'artifact'}; build the model first") + layout = json.loads((build / "artifact/layout.json").read_text()) + kind = layout["execution"]["kind"] + if kind == "native": + manifest = root / "examples/chips" / chip / "configs/generated/bemu/Cargo.toml" + binary = "bebop-bemu" + python = root / "result/bin/python3" + elif kind == "python": + manifest = root / "examples/chips" / chip / "emu/Cargo.toml" + with manifest.open("rb") as source: + binary = tomllib.load(source)["bin"][0]["name"] + python = root / "stack/serving/.venv/bin/python" + else: + raise ValueError(f"Unknown model execution kind: {kind}") + target = root / "bebop/target" / chip + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + log = Path(log_dir(repo, chip, "verilog", stamp, "bemu", model)) + commands = [ + ["cargo", "build", "--release", "--manifest-path", str(manifest), + "--target-dir", str(target), "--bin", binary], + [str(python), "-u", "-m", "stack.serving.cli", "--model", str(packages[0]), + "--loader", str(build / "loader/load-model"), + "--simulator", str(target / "release" / binary), + "--log-dir", str(log), "--run-config", str(config)], + ] + return commands, log diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index 1dc5782fa..389c7a4ef 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -21,7 +21,7 @@ from utils.event_common import require_chip from utils.path import bebop_cargo_env, get_buckyball_path, rtl_dir -from utils.p2e_timescale import normalize_p2e_timescale +from steps.bebop.p2e.scripts.timescale import normalize_p2e_timescale from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index 90cd70cea..8637b3cc7 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -6,6 +6,7 @@ 2. Validate bitstream .bit file path 3. Run bebop run p2e --image --bitstream [--multi-fpga] [--wave] [--wave-start ] """ + import glob import os import re @@ -13,7 +14,6 @@ import shutil import sys from datetime import datetime - from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) @@ -22,7 +22,6 @@ scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) - from utils.event_common import require_chip from utils.path import bebop_cargo_env, get_buckyball_path, log_dir from utils.stream_run import stream_run_logger_async @@ -41,10 +40,9 @@ def resolve_runtime_config(bitstream: str, requested_config: object) -> str: if isinstance(requested_config, str) and requested_config: return requested_config - build_dir = os.path.dirname(os.path.dirname(os.path.abspath(bitstream))) case_name = os.path.basename(build_dir) - return re.sub(r"-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}$", "", case_name) + return re.sub("-\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}$", "", case_name) def case_uses_multi_fpga(build_dir: str) -> bool: @@ -60,23 +58,33 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: except ValueError as error: ctx.logger.error(str(error)) await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) return bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" - image_name = input_data.get("image", "") bitstream = input_data.get("bitstream", "") multi_fpga = bool(input_data.get("multi-fpga", False)) - fpga_location = input_data.get("fpga-location") or input_data.get("fpga_location") or "0.A" - if not isinstance(fpga_location, str) or not re.fullmatch(r"[0-3]\.[A-E]", fpga_location): + fpga_location = ( + input_data.get("fpga-location") or input_data.get("fpga_location") or "0.A" + ) + if not isinstance(fpga_location, str) or not re.fullmatch( + "[0-3]\\.[A-E]", fpga_location + ): ctx.logger.error(f"invalid FPGA location: {fpga_location}") await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "invalid_fpga_location", "fpga_location": fpga_location}, + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "invalid_fpga_location", + "fpga_location": fpga_location, + }, trace_id=origin_tid, ) return @@ -85,7 +93,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if "wave_start" in input_data: ctx.logger.error("invalid parameter: --wave_start (use --wave-start)") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "invalid_parameter", "parameter": "wave_start"}, trace_id=origin_tid, ) @@ -98,77 +108,89 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: except (TypeError, ValueError): ctx.logger.error(f"invalid wave_start: {wave_start_raw}") await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "invalid_wave_start", "wave_start": wave_start_raw}, + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "invalid_wave_start", + "wave_start": wave_start_raw, + }, trace_id=origin_tid, ) return if wave_start < 0: ctx.logger.error(f"wave_start must be >= 0: {wave_start}") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "invalid_wave_start", "wave_start": wave_start}, trace_id=origin_tid, ) return wave = True - try: image_path = resolve_image(bbdir, image_name, chip) except ValueError as error: ctx.logger.error(str(error)) await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "image_not_found", "image": image_name}, trace_id=origin_tid, ) return - image_base = os.path.splitext(image_path)[0] - elf_path = f"{image_base}.elf" if os.path.isfile(f"{image_base}.elf") else image_base - if diff and not os.path.isfile(elf_path): + elf_path = ( + f"{image_base}.elf" if os.path.isfile(f"{image_base}.elf") else image_base + ) + if diff and (not os.path.isfile(elf_path)): ctx.logger.error(f"workload ELF for P2E DiffTest not found: {elf_path}") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "workload_elf_not_found", "elf": elf_path}, trace_id=origin_tid, ) return - if not bitstream or not os.path.isfile(bitstream): ctx.logger.error(f"bitstream .bit file not found: {bitstream}") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "bitstream_not_found", "bitstream": bitstream}, trace_id=origin_tid, ) return - bitstream = os.path.abspath(bitstream) build_dir = os.path.dirname(os.path.dirname(bitstream)) if not os.path.isdir(build_dir): ctx.logger.error(f"P2E build case not found for bitstream: {build_dir}") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "build_dir_not_found", "build_dir": build_dir}, trace_id=origin_tid, ) return - if not multi_fpga and case_uses_multi_fpga(build_dir): multi_fpga = True ctx.logger.info(f"Detected multi-FPGA P2E case: {build_dir}") - timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - run_log = log_dir(bbdir, chip, "p2e", timestamp, "p2e", image_name, input_data.get("vsrc_dir")) + run_log = log_dir( + bbdir, chip, "p2e", timestamp, "p2e", image_name, input_data.get("vsrc_dir") + ) os.makedirs(run_log, exist_ok=True) - runtime_lib_dir = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm") rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") libvctb_path = os.path.join(runtime_lib_dir, "libvCtb.so") bebop_p2e_path = os.path.join(build_dir, "bebop-p2e") runtime_artifacts = [rtcfg_path, libvctb_path] - if not all(os.path.isfile(path) for path in runtime_artifacts): + if not all((os.path.isfile(path) for path in runtime_artifacts)): missing = [path for path in runtime_artifacts if not os.path.isfile(path)] if missing: ctx.logger.error(f"P2E runtime artifacts missing: {missing}") @@ -184,22 +206,39 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - - manifest = os.path.join(bbdir, "examples", "chips", chip, "generated", "bebop", "Cargo.toml") + manifest = os.path.join( + bbdir, "examples", "chips", chip, "generated", "bebop", "Cargo.toml" + ) build_env = {**os.environ, **bebop_cargo_env(bbdir, chip), "OUT_PATH": build_dir} if diff: - build_env["CARGO_TARGET_DIR"] = os.path.join(bebop_dir, "target", f"{chip}-p2e-diff") - build_cmd = shlex.join([ - "nix", "develop", "--ignore-env", - "--keep-env-var", "HOME", - "--keep-env-var", "ALL_PROXY", - "--keep-env-var", "CARGO_TARGET_DIR", - "--keep-env-var", "OUT_PATH", - "-c", "cargo", "build", "--release", - "--manifest-path", manifest, - "--bin", "bebop", - "--features", "p2e,bemu" if diff else "p2e", - ]) + build_env["CARGO_TARGET_DIR"] = os.path.join( + bebop_dir, "target", f"{chip}-p2e-diff" + ) + build_cmd = shlex.join( + [ + "nix", + "develop", + "--ignore-env", + "--keep-env-var", + "HOME", + "--keep-env-var", + "ALL_PROXY", + "--keep-env-var", + "CARGO_TARGET_DIR", + "--keep-env-var", + "OUT_PATH", + "-c", + "cargo", + "build", + "--release", + "--manifest-path", + manifest, + "--bin", + "bebop", + "--features", + "p2e,bemu" if diff else "p2e", + ] + ) ctx.logger.info("Building bebop p2e runtime for the selected case...") build_result = await stream_run_logger_async( cmd=build_cmd, @@ -211,35 +250,24 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) if build_result.returncode != 0: await check_result( - ctx, build_result.returncode, continue_run=False, + ctx, + build_result.returncode, + continue_run=False, extra_fields={"task": "runtime_build", "build_dir": build_dir}, trace_id=origin_tid, ) return - target_release = os.path.join(build_env["CARGO_TARGET_DIR"], "release") staged_runtime = f"{bebop_p2e_path}.new" shutil.copy2(os.path.join(target_release, "bebop"), staged_runtime) os.replace(staged_runtime, bebop_p2e_path) - if diff: - libriscv_path = os.path.join(build_dir, "libriscv.so") - staged_libriscv = f"{libriscv_path}.new" - shutil.copy2(os.path.join(target_release, "bemu-runtime", "libriscv.so"), staged_libriscv) - os.replace(staged_libriscv, libriscv_path) - - run_cmd = ( - f"nix develop -c \"{bebop_p2e_path}\" run p2e " - f"--image=\"{image_path}\" " - f"--bitstream=\"{bitstream}\" " - f"--log-dir=\"{run_log}\" " - f"--fpga-location=\"{fpga_location}\"" - ) + run_cmd = f'nix develop -c "{bebop_p2e_path}" run p2e --image="{image_path}" --bitstream="{bitstream}" --log-dir="{run_log}" --fpga-location="{fpga_location}"' if multi_fpga: run_cmd += " --multi-fpga" if wave: run_cmd += " --wave" if wave_start is not None: - run_cmd += f" --wave-start=\"{wave_start}\"" + run_cmd += f' --wave-start="{wave_start}"' if diff: run_cmd += f" --diff --image-elf={shlex.quote(elf_path)}" for trace_name in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace"): @@ -255,7 +283,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="bebop p2e runworkload", env=run_env, ) - await check_result( ctx, run_result.returncode, diff --git a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py index d1d26d325..5ee40cda7 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py @@ -6,12 +6,12 @@ 2. Build the test harness against that runtime 3. Run the P2E regression harness serially on one FPGA """ + import os import re import shlex import sys from pathlib import Path - from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) @@ -20,8 +20,6 @@ bebop_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) if bebop_path not in sys.path: sys.path.insert(0, bebop_path) - -from utils.event_common import require_chip from utils.path import bebop_cargo_env, get_buckyball_path, workloads_output_root from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -39,10 +37,9 @@ def resolve_runtime_config(bitstream: str, requested_config: object) -> str: if isinstance(requested_config, str) and requested_config: return requested_config - build_dir = os.path.dirname(os.path.dirname(os.path.abspath(bitstream))) case_name = os.path.basename(build_dir) - return re.sub(r"-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}$", "", case_name) + return re.sub("-\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}$", "", case_name) async def handler(input_data: dict, ctx: FlowContext) -> None: @@ -53,35 +50,38 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if not bitstream or not os.path.isfile(bitstream): ctx.logger.error(f"bitstream .bit file not found: {bitstream}") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "bitstream_not_found", "bitstream": bitstream}, trace_id=origin_tid, ) return - bitstream = os.path.abspath(bitstream) build_dir = os.path.dirname(os.path.dirname(bitstream)) if not os.path.isdir(build_dir): ctx.logger.error(f"P2E build case not found for bitstream: {build_dir}") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "build_dir_not_found", "build_dir": build_dir}, trace_id=origin_tid, ) return - chip = input_data.get("chip") if not chip: ctx.logger.error("Missing required parameter: chip must be specified") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) return elf_root = workloads_output_root(bbdir) cargo_env = bebop_cargo_env(bbdir, chip) - test_type = input_data.get("test", "elf-tests") diff = bool(input_data.get("diff", False)) try: @@ -89,29 +89,45 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: except ValueError as e: ctx.logger.error(str(e)) await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "invalid_regression", "test": test_type, "chip": chip}, + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "invalid_regression", + "test": test_type, + "chip": chip, + }, trace_id=origin_tid, ) return - - ctx.logger.info(f"Running {test_type} with workload config: {workload_toml} diff={diff}") + ctx.logger.info( + f"Running {test_type} with workload config: {workload_toml} diff={diff}" + ) ctx.logger.info(f"P2E case dir (from bitstream): {build_dir}") - manifest = Path(bebop_dir) / "Cargo.toml" features = ["p2e"] runtime_env = {**os.environ.copy(), **cargo_env} if diff: - manifest = Path(bbdir) / "examples" / "chips" / chip / "generated" / "bebop" / "Cargo.toml" + manifest = ( + Path(bbdir) + / "examples" + / "chips" + / chip + / "generated" + / "bebop" + / "Cargo.toml" + ) features.append("bemu") - runtime_env["CARGO_TARGET_DIR"] = os.path.join(bebop_dir, "target", f"{chip}-p2e-diff") + runtime_env["CARGO_TARGET_DIR"] = os.path.join( + bebop_dir, "target", f"{chip}-p2e-diff" + ) feature_arg = ",".join(features) rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") - libvctb_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so") + libvctb_path = os.path.join( + build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so" + ) runtime_artifacts = [rtcfg_path, libvctb_path, os.path.join(build_dir, "bebop-p2e")] - if diff: - runtime_artifacts.append(os.path.join(build_dir, "libriscv.so")) - if not all(os.path.isfile(path) for path in runtime_artifacts): + if not all((os.path.isfile(path) for path in runtime_artifacts)): missing = [path for path in runtime_artifacts if not os.path.isfile(path)] if missing: ctx.logger.error(f"P2E runtime artifacts missing: {missing}") @@ -127,15 +143,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - - # OUT_PATH makes the test harness link against this bitstream case's runtime. - cargo_out = f"--config=\"env.OUT_PATH='{build_dir}'\"" - - # ── Build bebop p2e (tests), linked against case libvCtb ─────────────── - build_cmd = ( - f"nix develop -c cargo {cargo_out} build --release " - f"--manifest-path {shlex.quote(str(manifest))} --features {shlex.quote(feature_arg)} --tests" - ) + cargo_out = f'''--config="env.OUT_PATH='{build_dir}'"''' + build_cmd = f"nix develop -c cargo {cargo_out} build --release --manifest-path {shlex.quote(str(manifest))} --features {shlex.quote(feature_arg)} --tests" ctx.logger.info("Building bebop p2e (tests)...") build_result = await stream_run_logger_async( cmd=build_cmd, @@ -145,33 +154,30 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="bebop p2e build", env=runtime_env, ) - if build_result.returncode != 0: await check_result( - ctx, build_result.returncode, continue_run=False, + ctx, + build_result.returncode, + continue_run=False, extra_fields={"task": "build", "backend": "p2e", "build_dir": build_dir}, trace_id=origin_tid, ) return - - # ── Run regression ──────────────────────────────────────────────────── env = runtime_env.copy() env["OUT_PATH"] = build_dir harness_args = [ "--", - "--workload-toml", workload_toml, - "--bb-tests-root", elf_root, - "--p2e-bitstream", bitstream, + "--workload-toml", + workload_toml, + "--bb-tests-root", + elf_root, + "--p2e-bitstream", + bitstream, ] if diff: harness_args.append("--diff") harness = shlex.join(harness_args) - test_cmd = ( - f"nix develop -c cargo {cargo_out} test --release " - f"--manifest-path {shlex.quote(str(manifest))} --features {shlex.quote(feature_arg)} " - f"--test test_p2e {harness}" - ) - + test_cmd = f"nix develop -c cargo {cargo_out} test --release --manifest-path {shlex.quote(str(manifest))} --features {shlex.quote(feature_arg)} --test test_p2e {harness}" ctx.logger.info(f"Running bebop p2e regression: {test_cmd}") run_result = await stream_run_logger_async( cmd=test_cmd, @@ -181,7 +187,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="bebop p2e batch", env=env, ) - extra_fields = { "task": "batch", "backend": "p2e", @@ -192,17 +197,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "diff": diff, "workload_toml": workload_toml, } - if input_data.get("from_regression_check"): - # Model accuracy is filled by eval-performance, not pk pass rate. if run_result.returncode != 0: await check_result( - ctx, run_result.returncode, continue_run=False, + ctx, + run_result.returncode, + continue_run=False, extra_fields={**extra_fields, "error": "pk_tests_failed"}, trace_id=origin_tid, ) return - await check_result( ctx, run_result.returncode, diff --git a/bbdev/api/utils/p2e_timescale.py b/bbdev/api/steps/bebop/p2e/scripts/timescale.py similarity index 100% rename from bbdev/api/utils/p2e_timescale.py rename to bbdev/api/steps/bebop/p2e/scripts/timescale.py diff --git a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py index 0a288846c..0126a91be 100644 --- a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py +++ b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py @@ -1,5 +1,4 @@ from motia import ApiRequest, ApiResponse, FlowContext, api - from utils.event_common import require_chip from utils.path import get_buckyball_path, rtl_dir @@ -19,15 +18,9 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: chip = require_chip(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - - rushb = bool(body.get("rushB", False)) - build_dir = rtl_dir( - bbdir, chip, "verilog", body.get("output_dir"), rushb=rushb + build_dir = rtl_dir(bbdir, chip, "verilog", body.get("output_dir")) + data = {"chip": chip, "output_dir": build_dir} + await ctx.enqueue( + {"topic": "bebop.verilator.clean", "data": {**data, "_trace_id": ctx.trace_id}} ) - data = { - "chip": chip, - "output_dir": build_dir, - "rushB": rushb, - } - await ctx.enqueue({"topic": "bebop.verilator.clean", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py index f364a2176..ff645b680 100644 --- a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py +++ b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py @@ -1,12 +1,10 @@ import os import sys - from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) - from utils.event_common import require_chip from utils.path import get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async @@ -16,10 +14,7 @@ "name": "bebop-verilator-clean", "description": "Clean verilator build directory", "flows": ["bebop"], - "triggers": [ - queue("bebop.verilator.clean"), - queue("bebop.verilator.run.clean"), - ], + "triggers": [queue("bebop.verilator.clean"), queue("bebop.verilator.run.clean")], "enqueues": ["bebop.verilator.verilog", "bebop.verilator.run.verilog"], } @@ -32,19 +27,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: except ValueError as error: ctx.logger.error(str(error)) await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) return - build_dir = rtl_dir( - bbdir, - chip, - "verilog", - input_data.get("output_dir"), - rushb=bool(input_data.get("rushB", False)), - ) - + build_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("output_dir")) command = f"rm -rf {build_dir}" result = await stream_run_logger_async( cmd=command, @@ -53,7 +43,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stdout_prefix="bebop verilator clean", stderr_prefix="bebop verilator clean", ) - await check_result( ctx, result.returncode, @@ -61,10 +50,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={"task": "clean", "output_dir": build_dir}, trace_id=origin_tid, ) - - # Continue routing to verilog if from run workflow if input_data.get("from_run_workflow"): - data = {k: v for k, v in input_data.items() if k not in ("output_dir", "vsrc_dir")} + data = { + k: v for k, v in input_data.items() if k not in ("output_dir", "vsrc_dir") + } if input_data.get("_explicit_output_dir"): data["output_dir"] = build_dir await ctx.enqueue( diff --git a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py index dc2061484..e092718bd 100644 --- a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py +++ b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py @@ -1,5 +1,4 @@ from motia import ApiRequest, ApiResponse, FlowContext, api - from utils.event_common import require_chip config = { @@ -13,15 +12,11 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - if body.get("balltype"): return ApiResponse( status=400, - body={ - "error": "ball-only verilog belongs to uvm; use bbdev uvm --verilog", - }, + body={"error": "ball-only verilog belongs to uvm; use bbdev uvm --verilog"}, ) - try: chip = require_chip(body) except ValueError as e: @@ -33,9 +28,13 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "example": 'bbdev bebop verilator --verilog "--chip toy"', }, ) - - data = {"chip": chip, "rushB": bool(body.get("rushB", False)), "diff": bool(body.get("diff", False))} + data = {"chip": chip, "diff": bool(body.get("diff", False))} if body.get("output_dir"): data["output_dir"] = body["output_dir"] - await ctx.enqueue({"topic": "bebop.verilator.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "bebop.verilator.verilog", + "data": {**data, "_trace_id": ctx.trace_id}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_api.step.py b/bbdev/api/steps/bebop/verilator/03_build_api.step.py index 05654630a..4c20084ed 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_api.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_api.step.py @@ -1,5 +1,4 @@ from motia import ApiRequest, ApiResponse, FlowContext, api - from utils.event_common import require_chip from utils.path import get_buckyball_path, rtl_dir @@ -15,32 +14,27 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = request.body or {} - unsupported = [name for name in ("vsrc_dir", "output_dir") if name in body] if unsupported: return ApiResponse( status=400, - body={"error": f"Unsupported parameter(s): {', '.join(unsupported)}. Use --vsrc-dir/--output-dir."}, + body={ + "error": f"Unsupported parameter(s): {', '.join(unsupported)}. Use --vsrc-dir/--output-dir." + }, ) - try: chip = require_chip(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - - rushb = bool(body.get("rushB", False)) data = { "chip": chip, "jobs": body.get("jobs", 16), "diff": bool(body.get("diff", False)), - "rushB": rushb, "vsrc_dir": rtl_dir( - bbdir, - chip, - "verilog", - body.get("vsrc-dir") or body.get("output-dir"), - rushb=rushb, + bbdir, chip, "verilog", body.get("vsrc-dir") or body.get("output-dir") ), } - await ctx.enqueue({"topic": "bebop.verilator.build", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + {"topic": "bebop.verilator.build", "data": {**data, "_trace_id": ctx.trace_id}} + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index 47ea93663..ec52e4119 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -3,11 +3,11 @@ Builds bebop with verilator feature and VSRC_PATH """ + import os import shlex import sys from pathlib import Path - from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) @@ -16,14 +16,12 @@ scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) - from utils.event_common import require_chip from utils.path import bebop_cargo_env, bebop_target_dir, get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from build_marker import write_build_marker - config = { "name": "bebop-verilator-build", "description": "Build bebop verilator binary", @@ -38,27 +36,29 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" chip = require_chip(input_data) - - vsrc_dir = rtl_dir( - bbdir, - chip, - "verilog", - input_data.get("vsrc_dir"), - rushb=bool(input_data.get("rushB", False)), - ) + vsrc_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("vsrc_dir")) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") - diff = bool(input_data.get("diff", False)) build_dir = bebop_dir manifest = Path(bbdir) / "bebop" / "Cargo.toml" features = ["verilator"] if diff: - manifest = Path(bbdir) / "examples" / "chips" / chip / "generated" / "bebop" / "Cargo.toml" + manifest = ( + Path(bbdir) + / "examples" + / "chips" + / chip + / "generated" + / "bebop" + / "Cargo.toml" + ) build_dir = str(manifest.parent) features.append("bemu") dramsim_header = os.path.join(bbdir, "result", "include", "dramsim3.h") if not os.path.isfile(dramsim_header): - ctx.logger.info(f"Nix environment is missing {dramsim_header}; rebuilding result") + ctx.logger.info( + f"Nix environment is missing {dramsim_header}; rebuilding result" + ) env_result = await stream_run_logger_async( cmd="nix build", logger=ctx.logger, @@ -68,7 +68,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) if env_result.returncode != 0: raise RuntimeError(f"nix build failed: {env_result.returncode}") - jobs = input_data.get("jobs", 16) cargo_build_cmd = shlex.join( [ @@ -97,7 +96,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="bebop verilator build", env=env, ) - bebop_bin = os.path.join(bebop_target_dir(bbdir, chip), "release", "bebop") target_dir = bebop_target_dir(bbdir, chip) if build_result.returncode == 0: @@ -117,7 +115,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) if build_result.returncode != 0: return - if input_data.get("from_run_workflow"): await ctx.enqueue( { diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index 7ea6c2a35..6deb926da 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -1,15 +1,14 @@ """ bebop verilator event handler -Run Verilator either through its guest-ELF simulator or the rushB native ABI. +Run a guest ELF through Verilator. """ + import json import os import shlex import sys from datetime import datetime -from pathlib import Path - from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) @@ -18,15 +17,19 @@ scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) - from utils.event_common import require_chip -from utils.path import bebop_target_dir, get_buckyball_path, log_dir, rtl_dir, workloads_output_root +from utils.path import ( + bebop_target_dir, + get_buckyball_path, + log_dir, + rtl_dir, + workloads_output_root, +) from utils.stream_run import stream_run_logger_async from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id from build_marker import build_marker_path, read_build_marker - config = { "name": "bebop-verilator-sim", "description": "Run bebop verilator simulation", @@ -36,57 +39,32 @@ } -def is_native_host_elf(path: str) -> bool: - try: - with open(path, "rb") as binary: - header = binary.read(20) - except OSError: - return False - return header[:4] == b"\x7fELF" and header[18:20] == b"\x3e\x00" - - async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" - try: chip = require_chip(input_data) except ValueError as error: ctx.logger.error(str(error)) - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, - trace_id=origin_tid, - ) - return - - - diff = bool(input_data.get("diff", False)) - if diff and input_data.get("rushB"): - ctx.logger.error("--diff and --rushB cannot be used together") await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "diff_conflicts_with_rushB"}, + extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) return - - vsrc_dir = rtl_dir( - bbdir, - chip, - "verilog", - input_data.get("vsrc_dir"), - rushb=bool(input_data.get("rushB", False)), - ) + diff = bool(input_data.get("diff", False)) + vsrc_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("vsrc_dir")) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") build_dir = bebop_dir chip = input_data.get("chip") if diff: if not isinstance(chip, str) or not chip: - ctx.logger.error("Missing required parameter: --chip (required with --diff)") + ctx.logger.error( + "Missing required parameter: --chip (required with --diff)" + ) await check_result( ctx, 1, @@ -95,17 +73,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - bebop_bin = os.path.join(bebop_target_dir(bbdir, chip), "release", "bebop") if not os.path.isfile(bebop_bin): ctx.logger.error(f"bebop binary does not exist: {bebop_bin}") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "bebop_binary_not_found", "binary": bebop_bin}, trace_id=origin_tid, ) return - target_dir = bebop_target_dir(bbdir, chip) marker_path = build_marker_path(target_dir) try: @@ -113,7 +91,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: except FileNotFoundError: ctx.logger.error(f"bebop verilator build marker does not exist: {marker_path}") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "build_marker_not_found", "marker": marker_path}, trace_id=origin_tid, ) @@ -121,23 +101,30 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: except (OSError, json.JSONDecodeError) as e: ctx.logger.error(f"failed to read bebop verilator build marker: {e}") await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "build_marker_read_failed", "marker": marker_path, "detail": str(e)}, + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "build_marker_read_failed", + "marker": marker_path, + "detail": str(e), + }, trace_id=origin_tid, ) return - expect_vsrc = os.path.abspath(vsrc_dir) expect_bin = os.path.abspath(bebop_bin) if ( marker.get("config") != chip or marker.get("vsrc_dir") != expect_vsrc or marker.get("binary") != expect_bin - or bool(marker.get("diff", False)) != diff + or (bool(marker.get("diff", False)) != diff) ): ctx.logger.error(f"bebop verilator build marker mismatch: {marker}") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={ "error": "build_marker_mismatch", "marker": marker, @@ -150,67 +137,43 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - binary_name = input_data.get("binary", "") binary_path = search_workload(workloads_output_root(bbdir), binary_name) if binary_path is None: ctx.logger.error(f"binary not found: {binary_name}") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "binary_not_found", "binary": binary_name}, trace_id=origin_tid, ) return ctx.logger.info(f"binary_path: {binary_path}") - timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - run_log = log_dir(bbdir, chip, "verilog", timestamp, "verilator", binary_name, input_data.get("vsrc_dir")) + run_log = log_dir( + bbdir, + chip, + "verilog", + timestamp, + "verilator", + binary_name, + input_data.get("vsrc_dir"), + ) os.makedirs(run_log, exist_ok=True) - - wave_arg = " --no-wave" if input_data.get("no-wave", False) or input_data.get("no_wave", False) else "" + wave_arg = ( + " --no-wave" + if input_data.get("no-wave", False) or input_data.get("no_wave", False) + else "" + ) trace_args = "" for trace_name in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace"): if input_data.get(trace_name, False): trace_args += f" --{trace_name}" - - if input_data.get("rushB"): - # rushB owns the Verilator instance inside libbebop_verilator. It must - # execute as a native program, never as the normal guest ELF input. - if not is_native_host_elf(binary_path): - ctx.logger.error( - f"--rushB requires a native x86_64 rushB runner; got guest ELF: {binary_path}" - ) - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": "rushB_requires_native_runner", "binary": binary_path}, - trace_id=origin_tid, - ) - return - backend_library = f"{bebop_target_dir(bbdir, chip)}/release/deps/libbebop_verilator.so" - binary_dir = os.path.dirname(binary_path) - copy_cmd = shlex.join(["cmake", "-E", "copy_if_different", backend_library, binary_dir]) - library_env = f"LD_LIBRARY_PATH={shlex.quote(binary_dir)}:${{LD_LIBRARY_PATH:-}}" - inner_cmd = ( - f"cd {shlex.quote(binary_dir)} && {copy_cmd} && " - f"{library_env} exec {shlex.quote(binary_path)}" - ) - run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" - ctx.logger.info(f"Running rushB Verilator: {run_cmd}") - stdout_prefix = "rushB verilator" - stderr_prefix = "rushB verilator" - else: - run_cmd = ( - f"{shlex.quote(bebop_bin)} run verilator " - f"--elf={shlex.quote(binary_path)} " - f"--log-dir={shlex.quote(run_log)}" - f"{' --diff' if diff else ''}" - f"{wave_arg}" - f"{trace_args}" - ) - if diff: - preload = os.pathsep.join( + run_cmd = f"{shlex.quote(bebop_bin)} run verilator --elf={shlex.quote(binary_path)} --log-dir={shlex.quote(run_log)}{(' --diff' if diff else '')}{wave_arg}{trace_args}" + if diff: + preload = os.pathsep.join( + ( path for path in ( f"{bbdir}/result/lib/libdramsim3.so", @@ -218,14 +181,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) if path ) - run_inner = ( - f"cd {shlex.quote(build_dir)} && " - f"export LD_PRELOAD={shlex.quote(preload)} && exec {run_cmd}" - ) - run_cmd = f"nix develop -c sh -c {shlex.quote(run_inner)}" - ctx.logger.info(f"Running bebop verilator (diff={diff}): {run_cmd}") - stdout_prefix = "bebop verilator difftest" if diff else "bebop verilator" - stderr_prefix = stdout_prefix + ) + run_inner = f"cd {shlex.quote(build_dir)} && export LD_PRELOAD={shlex.quote(preload)} && exec {run_cmd}" + run_cmd = f"nix develop -c sh -c {shlex.quote(run_inner)}" + ctx.logger.info(f"Running bebop verilator (diff={diff}): {run_cmd}") + stdout_prefix = "bebop verilator difftest" if diff else "bebop verilator" + stderr_prefix = stdout_prefix run_result = await stream_run_logger_async( cmd=run_cmd, logger=ctx.logger, @@ -233,14 +194,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stdout_prefix=stdout_prefix, stderr_prefix=stderr_prefix, ) - await check_result( ctx, run_result.returncode, continue_run=False, extra_fields={ "task": "verilator", - "backend": "rushB" if input_data.get("rushB") else "guest-elf", + "backend": "guest-elf", "diff": diff, "chip": chip, "binary": binary_path, diff --git a/bbdev/api/steps/bebop/verilator/05_run_api.step.py b/bbdev/api/steps/bebop/verilator/05_run_api.step.py index c9344dccb..5e945fd19 100644 --- a/bbdev/api/steps/bebop/verilator/05_run_api.step.py +++ b/bbdev/api/steps/bebop/verilator/05_run_api.step.py @@ -1,7 +1,5 @@ from motia import ApiRequest, ApiResponse, FlowContext, api - from utils.event_common import require_chip -from utils.path import get_buckyball_path config = { "name": "bebop-verilator-run-api", @@ -14,18 +12,15 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - if "output_dir" in body: return ApiResponse( status=400, body={"error": "Unsupported parameter: output_dir. Use --output-dir."}, ) - try: chip = require_chip(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - binary = body.get("binary", "") if not binary: return ApiResponse( @@ -37,15 +32,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "message": "binary parameter is required", }, ) - diff = body.get("diff", False) - rushb = body.get("rushB", False) - if diff and rushb: - return ApiResponse( - status=400, - body={"error": "--diff and --rushB cannot be used together"}, - ) - data = { "chip": chip, "binary": binary, @@ -54,7 +41,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "pmctrace": body.get("pmctrace", False), "ctrace": body.get("ctrace", False), "banktrace": body.get("banktrace", False), - "rushB": rushb, "diff": diff, "no-wave": body.get("no-wave", body.get("no_wave", False)), "jobs": body.get("jobs", 16), @@ -64,5 +50,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: if output_dir is not None: data["output_dir"] = output_dir data["_explicit_output_dir"] = True - await ctx.enqueue({"topic": "bebop.verilator.run.clean", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "bebop.verilator.run.clean", + "data": {**data, "_trace_id": ctx.trace_id}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py index 108b71e8c..3e3443d92 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -1,5 +1,4 @@ from motia import ApiRequest, ApiResponse, FlowContext, api - from utils.event_common import require_chip from utils.path import get_buckyball_path, rtl_dir @@ -15,14 +14,11 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = request.body or {} - try: chip = require_chip(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - diff = bool(body.get("diff", False)) - rushB = bool(body.get("rushB", False)) test_type = body.get("test") if not test_type: return ApiResponse( @@ -31,29 +27,17 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "error": "Missing required parameter: --test must be specified (elf-tests)" }, ) - - if test_type != "elf-tests" and not (rushB and test_type == "pk-tests"): - return ApiResponse( - status=400, - body={ - "error": f"Invalid test type: {test_type}. Must be 'elf-tests', or 'pk-tests' with --rushB" - }, - ) - - if diff and rushB: + if test_type != "elf-tests": return ApiResponse( status=400, - body={"error": "--diff and --rushB cannot be used together"}, + body={"error": f"Invalid test type: {test_type}. Must be 'elf-tests'"}, ) - - vsrc_dir = rtl_dir(bbdir, chip, "verilog", body.get("vsrc_dir"), rushb=rushB) - + vsrc_dir = rtl_dir(bbdir, chip, "verilog", body.get("vsrc_dir")) data = { "chip": chip, "vsrc_dir": vsrc_dir, "test": test_type, "clean-before": body.get("clean-before", body.get("clean_before", False)), - "rushB": rushB, "diff": diff, } await ctx.enqueue( diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 6a3cf807d..3110aa1df 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -8,8 +8,6 @@ import shutil import shlex import sys -from pathlib import Path - from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) @@ -18,7 +16,6 @@ bebop_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) if bebop_path not in sys.path: sys.path.insert(0, bebop_path) - from utils.event_common import require_chip from utils.path import ( bebop_cargo_env, @@ -55,25 +52,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - elf_root = workloads_output_root(bbdir) - test_type = input_data.get("test", "elf-tests") - rushB = bool(input_data.get("rushB", False)) diff = bool(input_data.get("diff", False)) - if diff and rushB: - ctx.logger.error("--diff and --rushB cannot be used together") - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": "diff_conflicts_with_rushB"}, - trace_id=origin_tid, - ) - return try: workload_toml = resolve_workload_toml( - chip, "verilator", test_type, bbdir, rushB=rushB, diff=diff + chip, "verilator", test_type, bbdir, diff=diff ) except ValueError as e: ctx.logger.error(str(e)) @@ -89,57 +73,36 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - ctx.logger.info( - f"Running {test_type} with workload config: {workload_toml} " - f"rushB={rushB} diff={diff}" - ) - - vsrc_dir = rtl_dir( - bbdir, - chip, - "verilog", - input_data.get("vsrc_dir"), - rushb=rushB, + f"Running {test_type} with workload config: {workload_toml} diff={diff}" ) + vsrc_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("vsrc_dir")) vsrc_config = shlex.quote(f"env.VSRC_PATH='{vsrc_dir}'") env = os.environ.copy() env.update(bebop_cargo_env(bbdir, chip)) - env.update( - { - "VSRC_PATH": vsrc_dir, - } - ) - + env.update({"VSRC_PATH": vsrc_dir}) manifest = f"{bebop_dir}/Cargo.toml" features = "verilator" if diff: manifest = f"{bbdir}/examples/chips/{chip}/generated/bebop/Cargo.toml" features = "verilator,bemu" - if input_data.get("clean-before", input_data.get("clean_before", False)): artifact_dir = os.path.join(env["CARGO_TARGET_DIR"], "test-artifacts") shutil.rmtree(artifact_dir, ignore_errors=True) ctx.logger.info(f"Cleaned previous bebop test artifacts: {artifact_dir}") - harness_args = [ "--", - "--workload-toml", workload_toml, - "--bb-tests-root", elf_root, - "--arch-config", chip, + "--workload-toml", + workload_toml, + "--bb-tests-root", + elf_root, + "--arch-config", + chip, ] if diff: harness_args.append("--diff") - if rushB: - harness_args.extend(["--rushb-backend", "verilator"]) harness = shlex.join(harness_args) - test_cmd = ( - f"nix develop -c cargo --config={vsrc_config} test --release " - f"--manifest-path {shlex.quote(manifest)} " - f"--features {shlex.quote(features)} --test test_verilator " - f"{harness}" - ) - + test_cmd = f"nix develop -c cargo --config={vsrc_config} test --release --manifest-path {shlex.quote(manifest)} --features {shlex.quote(features)} --test test_verilator {harness}" ctx.logger.info(f"Running bebop verilator regression: {test_cmd}") run_result = await stream_run_logger_async( cmd=test_cmd, @@ -149,20 +112,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="bebop verilator batch", env=env, ) - await check_result( ctx, run_result.returncode, continue_run=False, extra_fields={ "task": "batch", - "backend": ( - "verilator-rushB" if rushB else ("difftest" if diff else "verilator") - ), + "backend": "difftest" if diff else "verilator", "chip": chip, "vsrc_dir": vsrc_dir, "test_type": test_type, - "rushB": rushB, "diff": diff, "workload_toml": workload_toml, }, diff --git a/bbdev/api/steps/compiler/scripts/build.py b/bbdev/api/steps/compiler/scripts/build.py index 75373a38b..a4530078a 100644 --- a/bbdev/api/steps/compiler/scripts/build.py +++ b/bbdev/api/steps/compiler/scripts/build.py @@ -14,20 +14,20 @@ def _run( cmd: list[str], *, repo: Path, - cwd: Path | None = None, logger: object | None = None, task_scope: str | None = None, output_prefix: str, ) -> None: + cmd = ["nix", "develop", str(repo), "-c", *cmd] if logger is None: - result = subprocess.run(cmd, cwd=cwd or repo) + result = subprocess.run(cmd, cwd=repo) else: from utils.stream_run import stream_run_logger result = stream_run_logger( cmd=shlex.join(cmd), logger=logger, - cwd=str(cwd or repo), + cwd=str(repo), stdout_prefix=output_prefix, stderr_prefix=output_prefix, task_scope=task_scope, @@ -71,7 +71,7 @@ def build_llvm( repo: str | Path, *, logger: object | None = None, task_scope: str | None = None ) -> Path: root = Path(repo).resolve() - buddy = root / "compiler" / "thirdparty" / "buddy-mlir" + buddy = root / "stack" / "compiler" / "thirdparty" / "buddy-mlir" llvm_src = buddy / "llvm" / "llvm" llvm_build = buddy / "llvm" / "build" if not llvm_src.is_dir(): @@ -104,7 +104,6 @@ def build_llvm( _run( cmake, repo=root, - cwd=buddy, logger=logger, task_scope=task_scope, output_prefix="compiler llvm configure", @@ -112,7 +111,6 @@ def build_llvm( _run( ["ninja", "-C", str(llvm_build), "-j", str(os.cpu_count() or 1)], repo=root, - cwd=buddy, logger=logger, task_scope=task_scope, output_prefix="compiler llvm build", @@ -126,6 +124,7 @@ def build_llvm( def compiler_build_dir(repo: str | Path, instance: str) -> Path: return ( Path(repo).resolve() + / "stack" / "compiler" / "thirdparty" / "buddy-mlir" @@ -158,7 +157,7 @@ def build_compiler( ) if not selected_pb.is_file(): raise RuntimeError(f"missing {selected_pb}; generate the requested chip.pb first") - buddy = root / "compiler" / "thirdparty" / "buddy-mlir" + buddy = root / "stack" / "compiler" / "thirdparty" / "buddy-mlir" llvm = build_llvm(root, logger=logger, task_scope=task_scope) python = compiler_python(root) build = compiler_build_dir(root, build_instance) @@ -171,7 +170,7 @@ def build_compiler( str(buddy), "-B", str(build), - f"-DBUDDY_EXTERNAL_DIALECTS_DIR={root / 'compiler'}", + f"-DBUDDY_EXTERNAL_DIALECTS_DIR={root / 'stack' / 'compiler'}", f"-DBUCKYBALL_CHIP_PB={selected_pb}", f"-DMLIR_DIR={llvm / 'lib' / 'cmake' / 'mlir'}", f"-DLLVM_DIR={llvm / 'lib' / 'cmake' / 'llvm'}", diff --git a/bbdev/api/steps/config/scripts/4_install.py b/bbdev/api/steps/config/scripts/4_install.py index 564f12a65..2c729126f 100644 --- a/bbdev/api/steps/config/scripts/4_install.py +++ b/bbdev/api/steps/config/scripts/4_install.py @@ -1,8 +1,6 @@ -#!/usr/bin/env python3 """Install Chip.pb into arch / bemu / compiler / workload build trees.""" from __future__ import annotations - import os import shutil import sys @@ -10,8 +8,8 @@ if str(Path(__file__).resolve().parent) not in sys.path: sys.path.insert(0, str(Path(__file__).resolve().parent)) +import chip_pb2 as pb -import chip_pb2 as pb # noqa: E402 def load_chip(pb_path: Path) -> pb.Chip: chip = pb.Chip() @@ -27,23 +25,10 @@ def _emit_dispatch(chip: pb.Chip, bbdir: Path, out: Path) -> None: balls = list(chip.bemu.balls) if not balls: out.write_text( - "use crate::inst::instruction::ExecContext;\n\n" - "pub fn execute_known(\n" - " _ball_class: &str,\n" - " _funct: u32,\n" - " _xs1: u64,\n" - " _xs2: u64,\n" - " _ctx: &mut ExecContext,\n" - ") -> u64 {\n" - ' panic!("no BEMU ball implementation")\n' - "}\n\n" - "pub fn cycles_after_issue(_ball_class: &str, _funct: u32, _xs1: u64, _xs2: u64) -> u64 {\n" - ' panic!("no BEMU ball latency implementation")\n' - "}\n", + 'use crate::inst::instruction::ExecContext;\n\npub fn execute_known(\n _ball_class: &str,\n _funct: u32,\n _xs1: u64,\n _xs2: u64,\n _ctx: &mut ExecContext,\n) -> u64 {\n panic!("no BEMU ball implementation")\n}\n\npub fn cycles_after_issue(_ball_class: &str, _funct: u32, _xs1: u64, _xs2: u64) -> u64 {\n panic!("no BEMU ball latency implementation")\n}\n', encoding="utf-8", ) return - lines = ["use crate::inst::instruction::ExecContext;", ""] for ball in balls: if not ball.ball_dir.isidentifier(): @@ -60,9 +45,7 @@ def _emit_dispatch(chip: pb.Chip, bbdir: Path, out: Path) -> None: def chain(fn: str, ctx: bool, panic_msg: str) -> list[str]: extra = ", ctx" if ctx else "" - body = [ - f" {balls[0].ball_dir}::{fn}(ball_class, funct, xs1, xs2{extra})" - ] + body = [f" {balls[0].ball_dir}::{fn}(ball_class, funct, xs1, xs2{extra})"] for ball in balls[1:]: body.append( f" .or_else(|| {ball.ball_dir}::{fn}(ball_class, funct, xs1, xs2{extra}))" @@ -86,7 +69,7 @@ def chain(fn: str, ctx: bool, panic_msg: str) -> list[str]: ) lines += ["}", ""] lines += [ - "pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> u64 {", + "pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> u64 {" ] lines += chain( "cycles_after_issue", @@ -115,8 +98,10 @@ def install_bemu(chip: pb.Chip, bbdir: Path, gen: Path) -> Path: bemu = gen / "bemu" bemu.mkdir(parents=True, exist_ok=True) _emit_dispatch(chip, bbdir, bemu / "dispatch.rs") - shutil.copy2(cargo, bemu / "Cargo.toml") - shutil.copy2(build_rs, bemu / "build.rs") + for source in (cargo, build_rs): + target = bemu / source.name + target.unlink(missing_ok=True) + target.symlink_to(os.path.relpath(source, bemu)) return bemu / "Cargo.toml" @@ -125,65 +110,7 @@ def install_bebop(gen: Path) -> Path: bebop.mkdir(parents=True, exist_ok=True) manifest = bebop / "Cargo.toml" manifest.write_text( - """[workspace] -resolver = "2" - -[package] -name = "bebop" -version = "0.1.0" -edition = "2021" -build = "../../../../../bebop/build.rs" - -[[bin]] -name = "bebop" -path = "../../../../../bebop/src/main.rs" - -[[test]] -name = "test_verilator" -path = "../../../../../bebop/tests/test_verilator.rs" -harness = false -required-features = ["verilator"] - -[[test]] -name = "test_p2e" -path = "../../../../../bebop/tests/test_p2e.rs" -harness = false -required-features = ["p2e"] - -[features] -default = [] -verilator = ["dep:bebop-verilator"] -p2e = ["dep:bebop-p2e", "bebop-bemu?/p2e"] -bemu = ["dep:bebop-bemu"] - -[dependencies] -bebop-verilator = { path = "../../../../../bebop/src/nodes/verilator", optional = true } -bebop-p2e = { path = "../../../../../bebop/src/nodes/p2e", optional = true } -bebop-bemu = { path = "../../configs/generated/bemu", optional = true } -bebop-dasm = { path = "../../../../../bebop/src/nodes/lib/dasm" } -bebop-bank-hash = { path = "../../../../../bebop/src/nodes/lib/bank-hash" } -bebop-bemu-profile = { path = "../../../../../bebop/src/nodes/lib/bemu-profile" } -bebop-fd-redirect = { path = "../../../../../bebop/src/nodes/lib/fd-redirect" } -bebop-rtl-trace = { path = "../../../../../bebop/src/nodes/lib/rtl-trace" } -bebop-uart = { path = "../../../../../bebop/src/nodes/lib/uart" } -clap = { version = "4", features = ["derive"] } -libc = "0.2" -log = "0.4" -env_logger = "0.11" -nix = { version = "0.29", features = ["fs", "mman", "signal", "process"] } -toml = "0.8" -camino = "1.1" -snafu = "0.8" -serde = { version = "1", features = ["derive"] } -serde_json = "1" -duct = "0.13" - -[dev-dependencies] -libtest-mimic = "0.8" -assert_cmd = "2" -walkdir = "2" -chrono = { version = "0.4", default-features = false, features = ["clock"] } -""", + '[workspace]\nresolver = "2"\n\n[package]\nname = "bebop"\nversion = "0.1.0"\nedition = "2021"\nbuild = "../../../../../bebop/build.rs"\n\n[[bin]]\nname = "bebop"\npath = "../../../../../bebop/src/main.rs"\n\n[[test]]\nname = "test_verilator"\npath = "../../../../../bebop/tests/test_verilator.rs"\nharness = false\nrequired-features = ["verilator"]\n\n[[test]]\nname = "test_p2e"\npath = "../../../../../bebop/tests/test_p2e.rs"\nharness = false\nrequired-features = ["p2e"]\n\n[features]\ndefault = []\nverilator = ["dep:bebop-verilator"]\np2e = ["dep:bebop-p2e", "bebop-bemu?/p2e"]\nbemu = ["dep:bebop-bemu"]\n\n[dependencies]\nbebop-verilator = { path = "../../../../../bebop/src/nodes/verilator", optional = true }\nbebop-p2e = { path = "../../../../../bebop/src/nodes/p2e", optional = true }\nbebop-bemu = { path = "../../configs/generated/bemu", optional = true }\nbebop-dasm = { path = "../../../../../bebop/src/nodes/lib/dasm" }\nbebop-bank-hash = { path = "../../../../../bebop/src/nodes/lib/bank-hash" }\nbebop-bemu-profile = { path = "../../../../../bebop/src/nodes/lib/bemu-profile" }\nbebop-fd-redirect = { path = "../../../../../bebop/src/nodes/lib/fd-redirect" }\nbebop-rtl-trace = { path = "../../../../../bebop/src/nodes/lib/rtl-trace" }\nbebop-uart = { path = "../../../../../bebop/src/nodes/lib/uart" }\nclap = { version = "4", features = ["derive"] }\nlibc = "0.2"\nlog = "0.4"\nenv_logger = "0.11"\nnix = { version = "0.29", features = ["fs", "mman", "signal", "process"] }\ntoml = "0.8"\ncamino = "1.1"\nsnafu = "0.8"\nserde = { version = "1", features = ["derive"] }\nserde_json = "1"\nduct = "0.13"\n\n[dev-dependencies]\nlibtest-mimic = "0.8"\nassert_cmd = "2"\nwalkdir = "2"\nchrono = { version = "0.4", default-features = false, features = ["clock"] }\n', encoding="utf-8", ) return manifest @@ -192,19 +119,12 @@ def install_bebop(gen: Path) -> Path: def install_workload(chip: pb.Chip, bbdir: Path, name: str, gen: Path) -> Path: if not chip.profiles: raise ValueError(f"chip {name}: no compiler profiles") - cargo = bbdir / "bebop" / "target" / name defs = { "BUCKYBALL_WORKLOAD_CHIP": name, - "BUCKYBALL_CARGO_TARGET_DIR": str(cargo), - "BUCKYBALL_RUSHB_BEMU_MANIFEST": str(gen / "bemu" / "Cargo.toml"), - "BUCKYBALL_RUSHB_BEMU_LIBRARY": str(cargo / "release" / "libbebop_bemu.so"), - "BUCKYBALL_RUSHB_VERILATOR_LIBRARY": str( - cargo / "release" / "deps" / "libbebop_verilator.so" - ), "BUCKYBALL_CHIP_PB": str(gen / "chip.pb"), } out = gen / "workload" / "cmake.defs" out.parent.mkdir(parents=True, exist_ok=True) - body = "".join(f"{k}={v}\n" for k, v in sorted(defs.items())) + body = "".join((f"{k}={v}\n" for k, v in sorted(defs.items()))) out.write_text(body, encoding="utf-8") return out diff --git a/bbdev/api/steps/config/scripts/model_pb2.py b/bbdev/api/steps/config/scripts/model_pb2.py new file mode 100644 index 000000000..3d6be48f8 --- /dev/null +++ b/bbdev/api/steps/config/scripts/model_pb2.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: model.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 7, + 35, + 1, + '', + 'model.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0bmodel.proto\x12\x10\x62uckyball.config\"_\n\x0eModelParameter\x12\x11\n\x07integer\x18\x01 \x01(\x12H\x00\x12\x0e\n\x04real\x18\x02 \x01(\x01H\x00\x12\x0e\n\x04text\x18\x03 \x01(\tH\x00\x12\x11\n\x07\x62oolean\x18\x04 \x01(\x08H\x00\x42\x07\n\x05value\"\xc4\x01\n\x0bModelConfig\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x41\n\nparameters\x18\x03 \x03(\x0b\x32-.buckyball.config.ModelConfig.ParametersEntry\x1aS\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0b\x32 .buckyball.config.ModelParameter:\x02\x38\x01\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'model_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_MODELCONFIG_PARAMETERSENTRY']._loaded_options = None + _globals['_MODELCONFIG_PARAMETERSENTRY']._serialized_options = b'8\001' + _globals['_MODELPARAMETER']._serialized_start=33 + _globals['_MODELPARAMETER']._serialized_end=128 + _globals['_MODELCONFIG']._serialized_start=131 + _globals['_MODELCONFIG']._serialized_end=327 + _globals['_MODELCONFIG_PARAMETERSENTRY']._serialized_start=244 + _globals['_MODELCONFIG_PARAMETERSENTRY']._serialized_end=327 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/proto/model.proto b/bbdev/api/steps/config/scripts/proto/model.proto new file mode 100644 index 000000000..5e12c9682 --- /dev/null +++ b/bbdev/api/steps/config/scripts/proto/model.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; +package buckyball.config; + +message ModelParameter { + oneof value { + sint64 integer = 1; + double real = 2; + string text = 3; + bool boolean = 4; + } +} + +message ModelConfig { + uint32 version = 1; + string name = 2; + map parameters = 3; +} diff --git a/bbdev/api/steps/ip/scripts/sram_compiler.py b/bbdev/api/steps/ip/scripts/sram_compiler.py index 44795be83..1e2a27cdd 100644 --- a/bbdev/api/steps/ip/scripts/sram_compiler.py +++ b/bbdev/api/steps/ip/scripts/sram_compiler.py @@ -6,7 +6,7 @@ for path in ( Path(__file__).resolve().parents[2], - Path(__file__).resolve().parents[4] / "thirdparty" / "soc-framework" / "ip", + Path(__file__).resolve().parents[5] / "thirdparty" / "soc-framework" / "ip", ): if str(path) not in sys.path: sys.path.insert(0, str(path)) diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index e31caf371..eb7d56f46 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -1,4 +1,5 @@ import os +import re import shutil import sys @@ -73,22 +74,8 @@ def kernel_model(input_data: dict) -> str: if not isinstance(model, str): raise ValueError("model must be a string") - model = model.lower() - supported = { - "bert", - "deepseekr1", - "gemma4", - "lenet", - "llama2", - "mobilenet", - "qwen3", - "resnet", - "stable-diffusion", - "yolo", - } - if model not in supported: - valid = ", ".join(sorted(supported)) - raise ValueError(f"unknown kernel model: {model}; valid models: {valid}") + if not re.fullmatch(r"[a-z0-9][a-z0-9-]*", model): + raise ValueError(f"invalid model name: {model}") return model @@ -112,12 +99,6 @@ def kernel_chip(input_data: dict, bbdir: str, *, require_overlay: bool) -> str: return chip -def cmake_model(model: str) -> str: - if model == "stable-diffusion": - return "stablediffusion" - return model - - def kernel_build_dir( bbdir: str, hart_params: dict, @@ -163,7 +144,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if model and not input_data.get("chip"): raise ValueError("--model requires --chip") interactive = kernel_interactive(input_data) - # pk mode needs OS overlay; model mode only needs chip for ModelTest path + # pk mode needs OS overlay; model mode only needs chip for the model artifact path chip = kernel_chip(input_data, bbdir, require_overlay=not bool(model)) except ValueError as e: ctx.logger.error(str(e)) @@ -177,7 +158,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir, hart_params, model, chip, interactive=interactive ) interactive_arg = "ON" if interactive else "OFF" - cmake_model_name = cmake_model(model) + cmake_model_name = model # cmake configure configure_cmd = ( diff --git a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py index 9696d013d..199a1b68d 100644 --- a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py @@ -3,10 +3,10 @@ Generates Verilog via mill for bebop verilator """ + import os import shutil import sys - from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) @@ -18,7 +18,6 @@ sys.path.insert(0, mill_dir) if mill_scripts not in sys.path: sys.path.insert(0, mill_scripts) - from utils.event_common import require_chip from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger_async @@ -30,7 +29,10 @@ "name": "bebop-verilator-verilog", "description": "Generate verilog code via mill", "flows": ["bebop"], - "triggers": [queue("bebop.verilator.verilog"), queue("bebop.verilator.run.verilog")], + "triggers": [ + queue("bebop.verilator.verilog"), + queue("bebop.verilator.run.verilog"), + ], "enqueues": ["bebop.verilator.build", "bebop.verilator.run.build"], } @@ -40,12 +42,10 @@ def check_verilog_output(build_dir: str) -> dict: is_dir = os.path.isdir(build_dir) sv_count = 0 if is_dir: - sv_count = sum(1 for name in os.listdir(build_dir) if name.endswith((".sv", ".v"))) - return { - "exists": exists, - "is_dir": is_dir, - "sv_count": sv_count, - } + sv_count = sum( + (1 for name in os.listdir(build_dir) if name.endswith((".sv", ".v"))) + ) + return {"exists": exists, "is_dir": is_dir, "sv_count": sv_count} async def handler(input_data: dict, ctx: FlowContext) -> None: @@ -56,26 +56,24 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: try: chip = require_chip(input_data) mill_config, build_dir = rtl_out( - bbdir, - chip, - "verilog", - input_data.get("output_dir"), - rushb=bool(input_data.get("rushB", False)), + bbdir, chip, "verilog", input_data.get("output_dir") ) shutil.rmtree(build_dir, ignore_errors=True) os.makedirs(build_dir) ctx.logger.info(f"Using mill config: {mill_config}") ctx.logger.info(f"Using build directory: {build_dir}") - command = mill_run.elaborate_cmd("sims.verilator.Elaborate", mill_config, build_dir) + command = mill_run.elaborate_cmd( + "sims.verilator.Elaborate", mill_config, build_dir + ) if input_data.get("diff"): command += " --difftest" returncode = ( await stream_run_logger_async( - cmd=command, - logger=ctx.logger, - cwd=arch, - stdout_prefix="bebop verilator verilog", - stderr_prefix="bebop verilator verilog", + cmd=command, + logger=ctx.logger, + cwd=arch, + stdout_prefix="bebop verilator verilog", + stderr_prefix="bebop verilator verilog", ) ).returncode except (ValueError, RuntimeError) as error: @@ -93,7 +91,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return ctx.logger.info(f"Using chip: {chip}") - output_status = check_verilog_output(build_dir) if returncode != 0: await check_result( @@ -109,12 +106,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - if not output_status["is_dir"] or output_status["sv_count"] == 0: ctx.logger.error( - f"Verilog output is invalid: {build_dir} " - f"(exists={output_status['exists']}, is_dir={output_status['is_dir']}, " - f"sv_count={output_status['sv_count']})" + f"Verilog output is invalid: {build_dir} (exists={output_status['exists']}, is_dir={output_status['is_dir']}, sv_count={output_status['sv_count']})" ) await check_result( ctx, @@ -130,7 +124,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - await check_result( ctx, returncode, @@ -138,8 +131,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={"task": "verilog", "chip": chip, "output_dir": build_dir}, trace_id=origin_tid, ) - if input_data.get("from_run_workflow") and returncode == 0: await ctx.enqueue( - {"topic": "bebop.verilator.run.build", "data": {**input_data, "output_dir": build_dir, "vsrc_dir": build_dir, "task": "run"}} + { + "topic": "bebop.verilator.run.build", + "data": { + **input_data, + "output_dir": build_dir, + "vsrc_dir": build_dir, + "task": "run", + }, + } ) diff --git a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py index d89e891c4..4dcef5e1b 100644 --- a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py +++ b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py @@ -17,7 +17,7 @@ from utils.event_common import require_chip from utils.path import get_buckyball_path from utils.path import rtl_out -from utils.p2e_timescale import normalize_p2e_timescale +from steps.bebop.p2e.scripts.timescale import normalize_p2e_timescale from utils.stream_run import stream_run_logger_async import mill as mill_run from utils.event_common import check_result, get_origin_trace_id diff --git a/bbdev/api/steps/model/01_build_api.step.py b/bbdev/api/steps/model/01_build_api.step.py new file mode 100644 index 000000000..2be9d208c --- /dev/null +++ b/bbdev/api/steps/model/01_build_api.step.py @@ -0,0 +1,26 @@ +import os +import sys + +from motia import ApiRequest, ApiResponse, FlowContext, api + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) +from steps.model.scripts.build import validate_model_build +from utils.path import get_buckyball_path + +config = { + "name": "model-build-api", + "description": "Build a chip model recipe", + "flows": ["model"], + "triggers": [api("POST", "/model/build")], + "enqueues": ["model.build"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + validate_model_build(get_buckyball_path(), body) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + await ctx.enqueue({"topic": "model.build", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/model/01_build_event.step.py b/bbdev/api/steps/model/01_build_event.step.py new file mode 100644 index 000000000..704daae83 --- /dev/null +++ b/bbdev/api/steps/model/01_build_event.step.py @@ -0,0 +1,36 @@ +import os +import sys + +from motia import FlowContext, queue + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) +from utils.event_common import check_result, get_origin_trace_id +from steps.model.scripts.build import build_model, validate_model_build +from utils.path import get_buckyball_path + +config = { + "name": "model-build", + "description": "Build a chip model recipe", + "flows": ["model"], + "triggers": [queue("model.build")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + trace_id = get_origin_trace_id(input_data, ctx) + params = {key: value for key, value in input_data.items() if key != "_trace_id"} + try: + repo = get_buckyball_path() + validate_model_build(repo, params) + build_model(repo, params["chip"], params.get("model"), + ctrace=params.get("ctrace", False), + dtrace=params.get("dtrace", False), + logger=ctx.logger, task_scope=trace_id) + except Exception as error: + ctx.logger.error(str(error)) + await check_result(ctx, 1, continue_run=False, + extra_fields={"error": "model_build_failed", "detail": str(error)}, + trace_id=trace_id) + return + await check_result(ctx, 0, continue_run=False, extra_fields=params, trace_id=trace_id) diff --git a/bbdev/api/steps/model/scripts/build.py b/bbdev/api/steps/model/scripts/build.py new file mode 100644 index 000000000..1d20936e1 --- /dev/null +++ b/bbdev/api/steps/model/scripts/build.py @@ -0,0 +1,67 @@ +import re +import shlex +from pathlib import Path + +from utils.stream_run import stream_run_logger +from steps.model.scripts.config import compile_config, cmake_arguments + + +def validate_model_build(repo: str, params: dict) -> list[str]: + unknown = params.keys() - {"chip", "model", "ctrace", "dtrace"} + if unknown: + raise ValueError(f"Unknown model build parameters: {', '.join(sorted(unknown))}") + for name in ("chip", "model"): + value = params.get(name) + if name == "model" and value is None: + continue + if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", value): + raise ValueError(f"Invalid or missing {name}: {value}") + recipes = Path(repo) / "examples/models" / params["chip"] + model = params.get("model") + if model is not None: + if not (recipes / model / "CMakeLists.txt").is_file(): + raise ValueError(f"Model recipe does not exist: {recipes / model}") + models = [model] + else: + models = sorted(path.parent.name for path in recipes.glob("*/CMakeLists.txt")) + if not models: + raise ValueError(f"No model recipes for chip: {params['chip']}") + for name in ("ctrace", "dtrace"): + if not isinstance(params.get(name, False), bool): + raise ValueError(f"{name} must be a boolean flag") + return models + + +def build_model(repo: str, chip: str, model: str | None = None, *, ctrace: bool = False, dtrace: bool = False, + logger, task_scope: str) -> None: + models = validate_model_build(repo, {"chip": chip, "model": model, "ctrace": ctrace, "dtrace": dtrace}) + source = Path(repo) / "stack/models" + compiler = Path(repo) / "stack/compiler/thirdparty/buddy-mlir/build" / chip + command = ["cmake", "--build", str(compiler), "--target", "buddy-opt", + "buddy-translate", "buddy-llc", "rax-pack", "python-package-buddy", "--parallel", "4"] + result = stream_run_logger(cmd=shlex.join(command), cwd=repo, logger=logger, + stdout_prefix="model compiler", stderr_prefix="model compiler", task_scope=task_scope) + if result.returncode: + raise RuntimeError(f"Model compiler build failed ({result.returncode}): {shlex.join(command)}") + for model in models: + build = source / "build" / chip / model + config = source / "models" / model / "configs/model.toml" + model_args = [] + if config.is_file(): + pb = build / "config/model.pb" + compile_config(config, pb, model) + model_args = cmake_arguments(pb, model) + commands = [ + ["cmake", "-S", str(source), "-B", str(build), "-G", "Ninja", + f"-DCHIP={chip}", f"-DMODEL={model}", f"-DCTRACE={'ON' if ctrace else 'OFF'}", + f"-DDTRACE={'ON' if dtrace else 'OFF'}", *model_args], + ["cmake", "--build", str(build), "--parallel", "1"], + ] + for command in commands: + result = stream_run_logger( + cmd=shlex.join(command), cwd=repo, logger=logger, + stdout_prefix=f"model {model}", stderr_prefix=f"model {model}", + task_scope=task_scope, + ) + if result.returncode: + raise RuntimeError(f"Model build failed ({result.returncode}): {shlex.join(command)}") diff --git a/bbdev/api/steps/model/scripts/config.py b/bbdev/api/steps/model/scripts/config.py new file mode 100644 index 000000000..81ab5edb6 --- /dev/null +++ b/bbdev/api/steps/model/scripts/config.py @@ -0,0 +1,82 @@ +"""Compile model-owned TOML parameters to protobuf and ordinary build arguments.""" +import ast +import json +import math +import operator +import re +import tomllib +from pathlib import Path + +from steps.config.scripts.model_pb2 import ModelConfig + + +_OPERATORS = {ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, + ast.FloorDiv: operator.floordiv, ast.Mod: operator.mod} + + +def compile_config(source: Path, destination: Path, name: str) -> None: + config = tomllib.loads(source.read_text()) + unknown = config.keys() - {"parameters", "derived"} + if unknown: + raise ValueError(f"Unknown model config sections: {sorted(unknown)}") + values = dict(config["parameters"]) + derived = config.get("derived", {}) + if values.keys() & derived.keys(): + raise ValueError("A model parameter cannot be both supplied and derived") + visiting = set() + + def resolve(key): + if key in values: + return values[key] + if key in visiting: + raise ValueError(f"Cyclic model parameter: {key}") + visiting.add(key) + value = evaluate(ast.parse(derived[key], mode="eval").body) + visiting.remove(key) + values[key] = value + return value + + def evaluate(node): + if isinstance(node, ast.Name): + return resolve(node.id) + if isinstance(node, ast.Constant) and type(node.value) in (int, float): + return node.value + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub): + return -evaluate(node.operand) + if isinstance(node, ast.BinOp) and type(node.op) in _OPERATORS: + left, right = evaluate(node.left), evaluate(node.right) + if type(left) not in (int, float) or type(right) not in (int, float): + raise ValueError("Derived arithmetic requires numeric parameters") + return _OPERATORS[type(node.op)](left, right) + raise ValueError("Model expressions support names, numbers, +, -, *, // and % only") + + for key in derived: + resolve(key) + message = ModelConfig(version=1, name=name) + fields = {int: "integer", float: "real", str: "text", bool: "boolean"} + for key, value in values.items(): + if not re.fullmatch(r"[a-z][a-z0-9_]*", key): + raise ValueError(f"Invalid model parameter name: {key}") + if type(value) not in fields or isinstance(value, float) and not math.isfinite(value): + raise ValueError(f"Invalid model parameter value: {key}") + setattr(message.parameters[key], fields[type(value)], value) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(message.SerializeToString(deterministic=True)) + destination.with_suffix(".json").write_text(json.dumps(values, indent=2, sort_keys=True) + "\n") + + +def cmake_arguments(path: Path, name: str) -> list[str]: + message = ModelConfig.FromString(path.read_bytes()) + if message.version != 1 or message.name != name: + raise ValueError(f"Invalid model config version or model name: {path}") + arguments = [] + for key, parameter in sorted(message.parameters.items()): + field = parameter.WhichOneof("value") + if field is None: + raise ValueError(f"Model parameter has no value: {key}") + value = getattr(parameter, field) + if field == "boolean": + arguments.append(f"-D{key.upper()}:BOOL={'ON' if value else 'OFF'}") + else: + arguments.append(f"-D{key.upper()}:STRING={value}") + return arguments diff --git a/bbdev/api/steps/regression/02_eval_performance_event.step.py b/bbdev/api/steps/regression/02_eval_performance_event.step.py index 02f16ad6b..7f784e557 100644 --- a/bbdev/api/steps/regression/02_eval_performance_event.step.py +++ b/bbdev/api/steps/regression/02_eval_performance_event.step.py @@ -1,12 +1,11 @@ """regression --eval-performance event. Per model (synchronous shells inside one regression event): - 1. workload build --chip --model - 2. bridge chip layout output -> flat path the kernel expects - 3. kernel build --model with dataset packed from e2e/datasets/ - -> fw_payload-.hex; /init runs binary with --dataset/--max-samples - 4. p2e runworkload; parse per-model accuracy from uart (top1= / map=) - 5. require non-empty log_dir/trace/cycle/; run perfetto.py; cycles_i + 1. model build --chip --model + 2. kernel build --model with dataset packed from the model build dataset directory + -> fw_payload-.hex; /init runs binary with --dataset + 3. p2e runworkload; parse per-model accuracy from uart (top1= / map=) + 4. require non-empty log_dir/trace/cycle/; run perfetto.py; cycles_i After all models: cycles = mean(cycles_i); accuracy = mean(accuracy_i); merge_metrics(cycles=..., accuracy=..., models=[...]). @@ -31,13 +30,12 @@ from utils.event_common import check_result, get_origin_trace_id from utils.model import model_layout_name -sys.path.insert(0, os.path.join(get_buckyball_path(), "bb-tests", "workloads", "scripts")) -import build as workload_build # noqa: E402 +from steps.model.scripts.build import build_model regression_scripts = os.path.join(os.path.dirname(__file__), "scripts") if regression_scripts not in sys.path: sys.path.insert(0, regression_scripts) -from model_layout import bridge_model_layout, perfetto_inputs +from model_layout import perfetto_inputs from models_toml import load_eval_models from accuracy import accuracy_from_uart, load_eval_accuracy, mean_accuracy from perfetto_latency import e2e_cycles_from_perfetto, mean_cycles @@ -73,15 +71,13 @@ def _fail(ctx, origin_tid, error, **extra): return check_result(ctx, 1, continue_run=False, extra_fields=fields, trace_id=origin_tid) -def _workload_build(bbdir, chip, model, logger, task_scope): - model_key = model.lower() - model_layout_name(model_key) - workload_build.build_workload( - bbdir, chip, model=model_key, logger=logger, task_scope=task_scope - ) +def _model_build(bbdir, chip, model, logger, task_scope): + build_model(bbdir, chip, model_layout_name(model.lower()), + ctrace=True, logger=logger, task_scope=task_scope) def _kernel_build_cmds(bbdir, chip, model, dataset=""): + model = model_layout_name(model) kernel_src = os.path.join(bbdir, "bb-tests", "workloads", "lib", "kernel") hart_params = {"visible": 64, "total": 64, "hidden_base": 64} kernel_build = _kernel.kernel_build_dir( @@ -157,7 +153,7 @@ def _perfetto_cmd(trace_dir, trace_toml, mlir_files): for mlir in mlir_files: args += ["--mlir", str(mlir)] return ( - f"{sys.executable} {shlex.quote(str(Path(get_buckyball_path()) / 'bb-tests/workloads/src/ModelTest/e2e/framework/trace/perfetto.py'))} " + f"{sys.executable} {shlex.quote(str(Path(get_buckyball_path()) / 'stack/serving/trace/perfetto.py'))} " + " ".join(shlex.quote(a) for a in args) ) @@ -208,20 +204,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await _fail(ctx, origin_tid, "unknown_model", model=model) return - ctx.logger.info(f"[eval-performance] model {model}: workload build") + ctx.logger.info(f"[eval-performance] model {model}: model build") try: - _workload_build(bbdir, chip, model, ctx.logger, origin_tid) + _model_build(bbdir, chip, model, ctx.logger, origin_tid) except Exception as e: ctx.logger.error(str(e)) - await _fail(ctx, origin_tid, "workload_build_cmd", model=model) - return - - ctx.logger.info(f"[eval-performance] model {model}: bridge layout") - try: - bridge_model_layout(bbdir, chip, model) - except (FileNotFoundError, FileExistsError) as e: - ctx.logger.error(str(e)) - await _fail(ctx, origin_tid, "layout_bridge_failed", model=model) + await _fail(ctx, origin_tid, "model_build_cmd", model=model) return ctx.logger.info(f"[eval-performance] model {model}: kernel build") diff --git a/bbdev/api/steps/regression/scripts/model_layout.py b/bbdev/api/steps/regression/scripts/model_layout.py index b4d2c491a..49e299dca 100644 --- a/bbdev/api/steps/regression/scripts/model_layout.py +++ b/bbdev/api/steps/regression/scripts/model_layout.py @@ -1,83 +1,13 @@ -"""Bridge pebble chip layout output -> flat archs/buckyball/ path the kernel expects. - -Workload `--chip --model ` writes under - bb-tests/output//workloads/src/ModelTest/e2e/models/archs/buckyball/// -Kernel `--model ` expects the flat - bb-tests/output//workloads/src/ModelTest/e2e/models/archs/buckyball// -""" -import os from pathlib import Path from utils.model import model_layout_name -def _archs_root(bbdir: str, chip: str) -> Path: - return ( - Path(bbdir) - / "bb-tests" - / "output" - / chip - / "workloads" - / "src" - / "ModelTest" - / "e2e" - / "models" - / "archs" - / "buckyball" - ) - - -def layout_name(model: str) -> str: - return model_layout_name(model) - - -def chip_output_dir(bbdir: str, chip: str, model: str) -> Path: - return _archs_root(bbdir, chip) / chip / layout_name(model) - - -def flat_output_dir(bbdir: str, chip: str, model: str) -> Path: - return _archs_root(bbdir, chip) / layout_name(model) - - -def bridge_model_layout(bbdir: str, chip: str, model: str) -> Path: - source = chip_output_dir(bbdir, chip, model) - if not source.is_dir(): - raise FileNotFoundError( - f"chip layout output dir missing for model '{model}' on chip '{chip}': {source}" - ) - flat = flat_output_dir(bbdir, chip, model) - if flat.is_symlink(): - target = os.readlink(flat) - if os.path.abspath(target) == os.path.abspath(str(source)): - return flat - flat.unlink() - elif flat.exists(): - raise FileExistsError( - f"refusing to clobber existing non-symlink at flat layout path: {flat}" - ) - flat.parent.mkdir(parents=True, exist_ok=True) - os.symlink(str(source), str(flat)) - return flat - - def perfetto_inputs(bbdir: str, chip: str, model: str) -> dict: - if model.lower() not in {"lenet", "mobilenet", "resnet", "yolo"}: - raise KeyError(f"no perfetto spec for model: {model}") - layout = layout_name(model) - trace_name = "trace-nodes.toml" if model.lower() == "lenet" else "trace.toml" - trace_toml = ( - Path(bbdir) - / "bb-tests/workloads/src/ModelTest/e2e/models/models" - / layout - / "trace" - / trace_name - ) - if not trace_toml.is_file(): - raise FileNotFoundError(f"perfetto trace toml missing: {trace_toml}") - output = _archs_root(bbdir, chip) / chip / layout - mlir_files = [] - for name in ("subgraph0_linalg.mlir", "subgraph0_buckyball.mlir"): - path = output / name + name = model_layout_name(model) + root = Path(bbdir) + trace_toml = root / "examples/models" / chip / name / "trace/trace.toml" + mlir = root / "stack/models/build" / chip / name / "generated/subgraph0.mlir" + for path in (trace_toml, mlir): if not path.is_file(): - raise FileNotFoundError(f"perfetto mlir file missing: {path}") - mlir_files.append(path) - return {"trace_toml": trace_toml, "mlir_files": mlir_files} + raise FileNotFoundError(path) + return {"trace_toml": trace_toml, "mlir_files": [mlir]} diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index fbef56b61..b58bf3bd5 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -290,6 +290,73 @@ def build_ip(bbdir: str, chip: str, name: str, ctx) -> dict: return config +def validate_ip_exclusions(exclusions: Path, exported: Path) -> None: + current = set() + for path in exported.glob("fullexclude.*"): + for block in path.read_text().split('// CHECKSUM: "')[1:]: + checksum = block.split('"', 1)[0] + instance = re.search(r"^// INSTANCE: (\S+)", block, re.MULTILINE) + if instance: + current.add((instance[1], checksum)) + checksum = None + for line in exclusions.read_text().splitlines(): + if line.startswith("CHECKSUM:"): + checksum = line.split('"')[1] + elif line.startswith("INSTANCE:"): + instance = line.split()[1] + if (instance, checksum) not in current: + raise RuntimeError( + f"coverage exclusions require review: {exclusions}: {instance}" + ) + elif line.startswith("MODULE:"): + raise ValueError(f"IP exclusions must use instance scopes: {exclusions}") + + +def report_ip_coverage( + bbdir: str, root: Path, target: dict, simv: Path, cov_dir: Path, ctx +) -> None: + cov_dir.mkdir(parents=True, exist_ok=True) + exclusions = root / "src" / "main" / "resources" / f"{target['name']}.el" + reports = [(cov_dir, "")] + if exclusions.is_file(): + reports[0] = (cov_dir, " -dump full_exclusions") + hierarchy = simv.parent / "rtl_hier.cfg" + hierarchy.write_text(f"+tree {target['top']}.dut\n") + scope = f" -hier {shlex.quote(str(hierarchy))}" + reports.extend( + [ + (cov_dir / "rtl_raw", scope), + ( + cov_dir / "rtl", + scope + f" -elfile {shlex.quote(str(exclusions))} -excl_strict", + ), + ] + ) + for report, options in reports: + if report.name == "rtl": + validate_ip_exclusions(exclusions, simv.parent) + command = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"urg -dir {shlex.quote(str(simv))}.vdb -format text" + f"{options} -report {shlex.quote(str(report))}" + ) + result = stream_run_logger( + cmd=command, + logger=ctx.logger, + cwd=str(simv.parent), + stdout_prefix="uvm urg", + stderr_prefix="uvm urg", + ) + if result.returncode != 0: + raise RuntimeError(f"URG failed for {target['name']}: {report}") + if report.name == "rtl" and re.search( + r"(?:Warning|Error)-\[", result.stdout + result.stderr + ): + raise RuntimeError( + f"coverage exclusions require review for {target['name']}: {report}" + ) + + def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str) -> dict: config = build_ip(bbdir, chip, name, ctx) root = Path(bbdir) / config["root"] @@ -323,20 +390,7 @@ def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str) -> dict: ) check_uvm_result(result, f"IP {name} target={target['name']}") - cov_dir.mkdir(parents=True, exist_ok=True) - urg = ( - f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " - f"urg -dir {shlex.quote(str(simv))}.vdb -format text -report {shlex.quote(str(cov_dir))}" - ) - result = stream_run_logger( - cmd=urg, - logger=ctx.logger, - cwd=str(root), - stdout_prefix="uvm urg", - stderr_prefix="uvm urg", - ) - if result.returncode != 0: - raise RuntimeError(f"URG failed for IP {name} target={target['name']}") + report_ip_coverage(bbdir, root, target, simv, cov_dir, ctx) coverage.append((target["name"], cov_dir)) index_dir = Path(cov_root) / "coverage" @@ -355,6 +409,11 @@ def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str) -> dict: "ip": name, "targets": [t["name"] for t in config["targets"]], "index": str(index), + "rtl_coverage": { + target["name"]: str(Path(cov_root) / target["name"] / "coverage" / "rtl") + for target in config["targets"] + if (root / "src/main/resources" / f"{target['name']}.el").is_file() + }, "log": cov_root, } diff --git a/bbdev/api/steps/uvm/scripts/waive.py b/bbdev/api/steps/uvm/scripts/waive.py index 3450ec580..a1f40125e 100644 --- a/bbdev/api/steps/uvm/scripts/waive.py +++ b/bbdev/api/steps/uvm/scripts/waive.py @@ -9,13 +9,18 @@ def apply_waivers(rtl_dir: Path) -> int: for path in sorted(rtl_dir.glob("*.sv")): lines = path.read_text().splitlines(keepends=True) output = [] + excluded_module = False for line in lines: - if re.search(r"^\s*(?:wire|reg|logic)\b.*\b_(?:GEN|T)(?:_\d+)?\b", line) and ( - not output or output[-1] != "//VCS coverage off\n" - ): - output.extend(("//VCS coverage off\n", line, "//VCS coverage on\n")) + if line.strip() == "// VCS coverage exclude_file": + # FIRRTL memory models share a file with the DUT in unsplit RTL. + output.append("//VCS coverage off\n") + excluded_module = True changed += 1 - else: - output.append(line) + continue + if excluded_module and re.match(r"^\s*endmodule\b", line): + output.extend((line, "//VCS coverage on\n")) + excluded_module = False + continue + output.append(line) path.write_text("".join(output)) return changed diff --git a/bbdev/api/steps/verilator/04_sim_event.step.py b/bbdev/api/steps/verilator/04_sim_event.step.py index f47bf3e1e..b17cf15d5 100644 --- a/bbdev/api/steps/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -4,14 +4,11 @@ import subprocess import sys from datetime import datetime - from motia import FlowContext, queue -# Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) - from utils.event_common import require_chip from utils.path import get_buckyball_path, log_dir, rtl_dir, workloads_output_root from utils.stream_run import stream_run_logger_async @@ -33,30 +30,27 @@ def validate_bbsim_elf(path: str) -> tuple[bool, str]: with open(path, "rb") as binary: header = binary.read(64) if len(header) != 64: - return False, "not a complete ELF64 header" + return (False, "not a complete ELF64 header") if header[:4] != b"\x7fELF" or header[4] != 2 or header[5] != 1: - return False, "requires a little-endian ELF64 binary" - - (_, _, machine, _, _, phoff, _, _, ehsize, phentsize, phnum, _, _, _) = ( + return (False, "requires a little-endian ELF64 binary") + _, _, machine, _, _, phoff, _, _, ehsize, phentsize, phnum, _, _, _ = ( struct.unpack("<16sHHIQQQIHHHHHH", header) ) if machine != 243: - return False, "requires a RISC-V ELF" + return (False, "requires a RISC-V ELF") if ehsize != 64 or phentsize != 56 or phnum == 0: - return False, "has an unsupported ELF program-header layout" - + return (False, "has an unsupported ELF program-header layout") file_size = os.fstat(binary.fileno()).st_size if phoff + phnum * phentsize > file_size: - return False, "program headers extend past the end of the file" - - dram_base = 0x80000000 - dram_size = 0x10000000 + return (False, "program headers extend past the end of the file") + dram_base = 2147483648 + dram_size = 268435456 load_segments = 0 for index in range(phnum): binary.seek(phoff + index * phentsize) program_header = binary.read(phentsize) if len(program_header) != phentsize: - return False, "cannot read ELF program headers" + return (False, "cannot read ELF program headers") p_type, _, p_offset, _, p_paddr, p_filesz, p_memsz, _ = struct.unpack( " tuple[bool, str]: continue load_segments += 1 if p_filesz > p_memsz or p_offset + p_filesz > file_size: - return False, "has an invalid loadable segment" + return (False, "has an invalid loadable segment") if p_paddr < dram_base or p_paddr + p_memsz > dram_base + dram_size: return ( False, - f"load segment {index} at 0x{p_paddr:x} is outside " - f"BBSimDRAM [0x{dram_base:x}, 0x{dram_base + dram_size:x})", + f"load segment {index} at 0x{p_paddr:x} is outside BBSimDRAM [0x{dram_base:x}, 0x{dram_base + dram_size:x})", ) except (OSError, struct.error): - return False, "cannot read a valid ELF64 program-header table" - + return (False, "cannot read a valid ELF64 program-header table") if not load_segments: - return False, "has no loadable ELF segments" - return True, "" + return (False, "has no loadable ELF segments") + return (True, "") async def handler(input_data: dict, ctx: FlowContext) -> None: - # ================================================================================== - # Get simulation parameters - # ================================================================================== origin_tid = get_origin_trace_id(input_data, ctx) try: chip = require_chip(input_data) @@ -97,13 +86,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return bbdir = get_buckyball_path() - build_dir = rtl_dir( - bbdir, chip, "verilog", input_data.get("output_dir"), - ) + build_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("output_dir")) ctx.logger.info(f"Using build directory: {build_dir}") - timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - binary_name = input_data.get("binary", "") if not binary_name: await check_result( @@ -115,7 +100,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return coverage = input_data.get("coverage", False) - workload_root = workloads_output_root(bbdir) binary_path = search_workload(workload_root, binary_name) ctx.logger.info(f"binary_path: {binary_path}") @@ -141,29 +125,30 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "error": "unsupported_elf", "binary": binary_path, - "detail": ( - "direct Verilator requires a baremetal RISC-V ELF in BBSimDRAM; " - f"{elf_error}. Use bebop-verilator for Linux ELFs" - ), + "detail": f"direct Verilator requires a baremetal RISC-V ELF in BBSimDRAM; {elf_error}. Use bebop-verilator for Linux ELFs", }, trace_id=origin_tid, ) return - topname = "BBSimHarness" - run_log = log_dir(bbdir, chip, "verilog", timestamp, "verilator", binary_name, input_data.get("output_dir")) + run_log = log_dir( + bbdir, + chip, + "verilog", + timestamp, + "verilator", + binary_name, + input_data.get("output_dir"), + ) waveform_dir = os.path.join(run_log, "waveform") - os.makedirs(run_log, exist_ok=True) no_wave = bool(input_data.get("no-wave", input_data.get("no_wave", False))) if not no_wave: os.makedirs(waveform_dir, exist_ok=True) - coverage_flag = "" if coverage: coverage_dat_path = f"{run_log}/coverage.dat" coverage_flag = f"+verilator+coverage+file+{coverage_dat_path}" - bin_path = f"{build_dir}/obj_dir/V{topname}" if not os.path.isfile(bin_path): await check_result( @@ -175,29 +160,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return batch = input_data.get("batch", False) - trace_names = [ name for name in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace") if input_data.get(name, False) ] trace_arg = ",".join(trace_names) if trace_names else "none" - - log_path = f"{run_log}/bdb.ndjson" + log_path = f"{run_log}/bdb.ndjson" stdout_path = f"{run_log}/stdout.log" - meta_path = f"{run_log}/sim_meta.txt" - fst_path = f"{waveform_dir}/waveform.fst" - - # ================================================================================== - # Execute simulation - # BBSimHarness uses +elf= for ELF loading (via BBSimDRAM.cc / libelf) - # No fesvr, no +loadmem_addr needed - # - # disasm.log: only stderr -> spike-dasm (Rocket commit printf is stderr here; - # merging stdout with 2>&1 can break: full stdio buffering + non-DASM bytes). - # BDB_SIM_META moves NDJSON banner to sim_meta.txt so it does not pollute disasm. - # ================================================================================== + meta_path = f"{run_log}/sim_meta.txt" + fst_path = f"{waveform_dir}/waveform.fst" result_lib = f"{bbdir}/result/lib" + def _gcc_lib_dir(soname: str) -> str: printed = subprocess.run( ["g++", f"-print-file-name={soname}"], @@ -206,14 +180,11 @@ def _gcc_lib_dir(soname: str) -> str: check=True, ) return os.path.dirname(os.path.realpath(printed.stdout.strip())) + lib_dirs = [result_lib, _gcc_lib_dir("liblz4.so"), _gcc_lib_dir("libstdc++.so")] ld_lib_path = ":".join(dict.fromkeys(lib_dirs)) ctx.logger.info(f"LD_LIBRARY_PATH prefix: {ld_lib_path}") - sim_args = [ - bin_path, - "+permissive", - f"+elf={binary_path}", - ] + sim_args = [bin_path, "+permissive", f"+elf={binary_path}"] if batch: sim_args.append("+batch") if coverage_flag: @@ -222,20 +193,30 @@ def _gcc_lib_dir(soname: str) -> str: sim_args.append("+no-wave") else: sim_args.append(f"+fst={fst_path}") - sim_args.extend([ - f"+log={log_path}", - f"+stdout={stdout_path}", - f"+trace={trace_arg}", - "+permissive-off", - ]) - sim_cmd = ( - f"export LD_LIBRARY_PATH={shlex.quote(ld_lib_path)}:$LD_LIBRARY_PATH; " - f"export BDB_SIM_META={shlex.quote(meta_path)}; " - f"{shlex.join(sim_args)} " - f"2> >(spike-dasm > {shlex.quote(os.path.join(run_log, 'disasm.log'))})" + sim_args.extend( + [ + f"+log={log_path}", + f"+stdout={stdout_path}", + f"+trace={trace_arg}", + "+permissive-off", + ] + ) + dasm_cmd = shlex.join( + [ + "cargo", + "run", + "--quiet", + "--release", + "--manifest-path", + os.path.join(bbdir, "bebop", "Cargo.toml"), + "--bin", + "bebop", + "--", + "dasm", + ] ) + sim_cmd = f"export LD_LIBRARY_PATH={shlex.quote(ld_lib_path)}:$LD_LIBRARY_PATH; export BDB_SIM_META={shlex.quote(meta_path)}; {shlex.join(sim_args)} 2> >({dasm_cmd} > {shlex.quote(os.path.join(run_log, 'disasm.log'))})" script_dir = os.path.dirname(__file__) - result = await stream_run_logger_async( cmd=sim_cmd, logger=ctx.logger, @@ -245,15 +226,11 @@ def _gcc_lib_dir(soname: str) -> str: executable="bash", ) success_result, failure_result = await check_result( - ctx, returncode=result.returncode, continue_run=True, trace_id=origin_tid, + ctx, returncode=result.returncode, continue_run=True, trace_id=origin_tid ) if failure_result: ctx.logger.error("sim failed", failure_result) return - - # ================================================================================== - # Return simulation result - # ================================================================================== extra_fields = { "task": "sim", "binary": binary_path, @@ -264,7 +241,6 @@ def _gcc_lib_dir(soname: str) -> str: } if coverage: extra_fields["coverage_dat"] = coverage_dat_path - await check_result( ctx, result.returncode, @@ -272,5 +248,4 @@ def _gcc_lib_dir(soname: str) -> str: extra_fields=extra_fields, trace_id=origin_tid, ) - return diff --git a/bbdev/api/steps/workload/01_buidl_api.step.py b/bbdev/api/steps/workload/01_buidl_api.step.py index 39f72af3f..86164c3f5 100644 --- a/bbdev/api/steps/workload/01_buidl_api.step.py +++ b/bbdev/api/steps/workload/01_buidl_api.step.py @@ -2,13 +2,11 @@ import re import sys from pathlib import Path - from motia import ApiRequest, ApiResponse, FlowContext, api utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) - from utils.path import get_buckyball_path config = { @@ -22,12 +20,14 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - allowed = {"chip", "model", "stable", "rushB", "ctest", "mlirtest", "trace-config"} - unknown = sorted(k for k in body if k not in allowed) + allowed = {"chip", "stable", "ctest", "mlirtest"} + unknown = sorted((k for k in body if k not in allowed)) if unknown: return ApiResponse( status=400, - body={"error": f"Unknown workload build parameter(s): {', '.join(unknown)}"}, + body={ + "error": f"Unknown workload build parameter(s): {', '.join(unknown)}" + }, ) chip = body.get("chip") if not chip: @@ -35,16 +35,12 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: status=400, body={"error": "Missing required parameter: --chip must be specified"}, ) - if not isinstance(chip, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", chip): - return ApiResponse( - status=400, - body={"error": f"Invalid chip: {chip}"}, - ) + if not isinstance(chip, str) or not re.fullmatch("[A-Za-z0-9_-]+", chip): + return ApiResponse(status=400, body={"error": f"Invalid chip: {chip}"}) chip_dir = Path(get_buckyball_path()) / "examples" / "chips" / chip if not chip_dir.is_dir(): return ApiResponse( - status=400, - body={"error": f"Workload chip does not exist: {chip}"}, + status=400, body={"error": f"Workload chip does not exist: {chip}"} ) stable = body.get("stable", False) if not isinstance(stable, bool): @@ -52,50 +48,23 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: status=400, body={"error": "Invalid parameter: stable must be a boolean flag"}, ) - trace_config = body.get("trace-config", "") - if not isinstance(trace_config, str) or (trace_config and not os.path.isabs(trace_config)): - return ApiResponse(status=400, body={"error": "Invalid trace config"}) - rushb_backend = body.get("rushB") - if rushb_backend is not None and rushb_backend not in {"bemu", "verilator"}: - return ApiResponse( - status=400, - body={"error": "Invalid --rushB: expected bemu or verilator"}, - ) ctest = body.get("ctest", False) mlirtest = body.get("mlirtest", False) if not isinstance(ctest, bool) or not isinstance(mlirtest, bool): return ApiResponse( - status=400, - body={"error": "--ctest and --mlirtest must be boolean flags"}, + status=400, body={"error": "--ctest and --mlirtest must be boolean flags"} ) if ctest and mlirtest: return ApiResponse( - status=400, - body={"error": "--ctest and --mlirtest cannot be used together"}, - ) - if (ctest or mlirtest) and body.get("model"): - return ApiResponse( - status=400, - body={"error": "--ctest and --mlirtest cannot be used with --model"}, - ) - if (ctest or mlirtest) and rushb_backend: - return ApiResponse( - status=400, - body={"error": "--ctest and --mlirtest cannot be used with --rushB"}, - ) - if trace_config and not body.get("model"): - return ApiResponse( - status=400, - body={"error": "--trace-config requires --model"}, + status=400, body={"error": "--ctest and --mlirtest cannot be used together"} ) data = { "chip": chip, - "model": body.get("model", ""), "stable": stable, - "rushB": rushb_backend, "ctest": ctest, "mlirtest": mlirtest, - "trace-config": trace_config, } - await ctx.enqueue({"topic": "workload.build", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + {"topic": "workload.build", "data": {**data, "_trace_id": ctx.trace_id}} + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index bda8d8fa9..51b2f4c0e 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -3,27 +3,24 @@ import sys import re from pathlib import Path - from motia import FlowContext, queue -# Add the utils directory to the Python path utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) - from utils.path import get_buckyball_path from utils.event_common import check_result, get_origin_trace_id -from utils.model import model_layout_name _workload_build_path = os.path.join( get_buckyball_path(), "bb-tests", "workloads", "scripts", "build.py" ) -_spec = importlib.util.spec_from_file_location("workload_build_module", _workload_build_path) +_spec = importlib.util.spec_from_file_location( + "workload_build_module", _workload_build_path +) if _spec is None or _spec.loader is None: raise RuntimeError(f"cannot load {_workload_build_path}") workload_build = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(workload_build) - config = { "name": "workload-build", "description": "build workload", @@ -32,41 +29,24 @@ "enqueues": [], } -def chips_for_model(bbdir: str, model_key: str) -> set[str]: - """Chips that currently ship a layout for this model (many-to-many).""" - try: - layout = model_layout_name(model_key) - except ValueError: - return set() - root = ( - Path(bbdir) - / "bb-tests" - / "workloads" - / "src" - / "ModelTest" - / "e2e" - / "models" - / "archs" - / "buckyball" - ) - if not root.is_dir(): - return set() - return { - p.name - for p in root.iterdir() - if p.is_dir() and (p / layout).is_dir() - } - async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - allowed = {"chip", "model", "stable", "rushB", "ctest", "mlirtest", "trace-config", "_trace_id"} - unknown = sorted(k for k in input_data if k not in allowed) + allowed = { + "chip", + "stable", + "ctest", + "mlirtest", + "_trace_id", + } + unknown = sorted((k for k in input_data if k not in allowed)) if unknown: ctx.logger.error(f"Unknown workload build parameter(s): {', '.join(unknown)}") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "unknown_parameter", "parameters": unknown}, trace_id=origin_tid, ) @@ -75,15 +55,19 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if not chip: ctx.logger.error("Missing required parameter: chip must be specified") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "missing_chip"}, trace_id=origin_tid, ) return - if not isinstance(chip, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", chip): + if not isinstance(chip, str) or not re.fullmatch("[A-Za-z0-9_-]+", chip): ctx.logger.error(f"Invalid chip: {chip}") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "invalid_chip", "chip": chip}, trace_id=origin_tid, ) @@ -92,47 +76,32 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if not chip_dir.is_dir(): ctx.logger.error(f"Workload chip does not exist: {chip}") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "unknown_chip", "chip": chip}, trace_id=origin_tid, ) return - model = input_data.get("model", "") - trace_config = input_data.get("trace-config", "") stable = input_data.get("stable", False) - rushb_backend = input_data.get("rushB") - if not isinstance(stable, bool): ctx.logger.error("Invalid parameter: stable must be a boolean flag") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "invalid_stable", "stable": stable}, trace_id=origin_tid, ) return - if not isinstance(trace_config, str) or (trace_config and not os.path.isabs(trace_config)): - ctx.logger.error("Invalid trace config") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "invalid_trace_config", "trace_config": trace_config}, - trace_id=origin_tid, - ) - return - - if rushb_backend is not None and rushb_backend not in {"bemu", "verilator"}: - ctx.logger.error("Invalid rushB backend: expected bemu or verilator") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "invalid_rushB", "rushB": rushb_backend}, - trace_id=origin_tid, - ) - return ctest = input_data.get("ctest", False) mlirtest = input_data.get("mlirtest", False) if not isinstance(ctest, bool) or not isinstance(mlirtest, bool): ctx.logger.error("--ctest and --mlirtest must be boolean flags") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "invalid_test_scope"}, trace_id=origin_tid, ) @@ -140,92 +109,46 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if ctest and mlirtest: ctx.logger.error("--ctest and --mlirtest cannot be used together") await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"error": "conflicting_test_scope"}, trace_id=origin_tid, ) return - if (ctest or mlirtest) and (model or rushb_backend): - ctx.logger.error("--ctest and --mlirtest cannot be used with --model or --rushB") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "test_scope_conflicts_with_workload"}, - trace_id=origin_tid, - ) - return - if trace_config and not model: - ctx.logger.error("--trace-config requires --model") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "trace_config_requires_model"}, - trace_id=origin_tid, - ) - return - - if model: - model_key = model.lower() - try: - layout = model_layout_name(model_key) - except ValueError: - ctx.logger.error(f"Unknown model: {model}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "unknown_model", "model": model}, - trace_id=origin_tid, - ) - return - supported_chips = chips_for_model(bbdir, model_key) - if chip not in supported_chips: - allowed = ", ".join(sorted(supported_chips)) if supported_chips else "(none)" - ctx.logger.error( - f"Model '{model}' has no Buckyball layout on chip '{chip}' " - f"(layout dir '{layout}'; chips with layout: {allowed})" - ) - await check_result( - ctx, 1, continue_run=False, - extra_fields={ - "error": "unsupported_chip_model", - "chip": chip, - "model": model, - "layout": layout, - "supported_chips": sorted(supported_chips), - }, - trace_id=origin_tid, - ) - return - try: workload_build.build_workload( bbdir, chip, - model=model.lower() if model else "", - rushb=rushb_backend, ctest=ctest, mlirtest=mlirtest, stable=stable, - trace_config=trace_config, logger=ctx.logger, task_scope=origin_tid, ) except Exception as error: ctx.logger.error(str(error)) await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "workload_build_failed", "chip": chip, "detail": str(error)}, + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "workload_build_failed", + "chip": chip, + "detail": str(error), + }, trace_id=origin_tid, ) return - await check_result( - ctx, 0, continue_run=False, + ctx, + 0, + continue_run=False, extra_fields={ "chip": chip, - "model": model, - "rushB": rushb_backend, "ctest": ctest, "mlirtest": mlirtest, - "trace_config": trace_config, }, - trace_id=origin_tid) - + trace_id=origin_tid, + ) return diff --git a/bbdev/api/steps/workload/README.md b/bbdev/api/steps/workload/README.md index d9e0ad086..b852dc172 100644 --- a/bbdev/api/steps/workload/README.md +++ b/bbdev/api/steps/workload/README.md @@ -24,29 +24,14 @@ bbdev workload --clean "--chip toy" **Parameters**: - **`chip`** - Required chip name. Selects chip-specific workloads. -- **`model`** - Optional model name to build. If omitted, build all workloads. - **`stable`** - Optional boolean flag. If set, build with stable LLVM Buckyball extensions. -- **`rushB`** - Optional host-native backend: `bemu` or `verilator`. Requires - `model`. It builds host-native CPU and rushB-lowered accelerator objects. -- **`ctest`** - Build CTest workloads only. Cannot be combined with `model` or `rushB`. -- **`mlirtest`** - Build MLIRTest workloads only. Cannot be combined with `model` or `rushB`. +- **`ctest`** - Build CTest workloads only. +- **`mlirtest`** - Build MLIRTest workloads only. For chip workloads under paths like `*/chips/`, only the directory selected by `chip` is synced to `bb-tests/output//workloads`. **Examples**: ```bash -# Build one model with the default xbuckyball path -bbdev workload --build "--chip toy --model lenet" - -# Build one model with stable LLVM Buckyball extensions -bbdev workload --build "--chip toy --model lenet --stable" - -# Build a model's rushB BEMU runner -bbdev workload --build "--chip pebble --model lenet --rushB bemu" - -# Build a model's rushB Verilator runner -bbdev workload --build "--chip pebble --model lenet --rushB verilator" - # Build all workloads bbdev workload --build "--chip toy" diff --git a/bbdev/api/utils/model.py b/bbdev/api/utils/model.py index ab2bf263a..d19d8fbbf 100644 --- a/bbdev/api/utils/model.py +++ b/bbdev/api/utils/model.py @@ -1,17 +1,16 @@ def model_layout_name(model: str) -> str: aliases = { - "lenet": "LeNet", - "mobilenet": "MobileNetV3", - "resnet": "ResNet18", - "yolo": "YOLO26", - "bert": "Bert", - "qwen3": "Qwen3", - "gemma4": "Gemma4", - "deepseekr1": "DeepSeekR1", + "lenet": "lenet", + "mobilenet": "mobilenet-v3-small", + "resnet": "resnet18", + "yolo": "yolo26n", + "bert": "bert", + "qwen3": "qwen3-8b", + "gemma4": "gemma4-e2b-it", + "deepseekr1": "deepseek-r1-0528-qwen3-8b", "llama2": "llama2", - "stable-diffusion": "StableDiffusion", - "whisper": "Whisper", - "buddynext": "BuddyNext", + "stable-diffusion": "stable-diffusion", + "whisper": "whisper", } try: return aliases[model.lower()] diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index 6d32ed35f..4f2214c42 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -1,7 +1,6 @@ import json import os import re -import subprocess from pathlib import Path @@ -19,7 +18,7 @@ def get_buckyball_path(): def _chip_name(chip): if not isinstance(chip, str) or not chip or chip == "None": raise ValueError("missing required parameter: chip") - if not re.fullmatch(r"[A-Za-z0-9_.-]+", chip): + if not re.fullmatch("[A-Za-z0-9_.-]+", chip): raise ValueError(f"invalid chip name: {chip}") return chip @@ -55,7 +54,7 @@ def chip_arch_root(bbdir, chip): return os.path.join(bbdir, "arch", "build", _chip_name(chip)) -def sim_name(bbdir, chip, product, *, rushb=False): +def sim_name(bbdir, chip, product): if product == "verilog" or product == "synth": sim_key = "verilator" elif product == "tapeout": @@ -82,25 +81,19 @@ def sim_name(bbdir, chip, product, *, rushb=False): if sim_key not in sims: raise ValueError(f"chip {chip!r} has no sims.{sim_key} in {path}") name = sims[sim_key] - if not rushb: - return name - if product != "verilog" or not name.endswith("VerilatorConfig"): - raise ValueError(f"rushB RTL is not supported for product {product}: {name}") - return f"{name[:-len('VerilatorConfig')]}RushBVerilatorConfig" + return name -def rtl_dir(bbdir, chip, product, output_dir=None, *, rushb=False): +def rtl_dir(bbdir, chip, product, output_dir=None): if output_dir: return output_dir - return os.path.join( - chip_arch_root(bbdir, chip), sim_name(bbdir, chip, product, rushb=rushb) - ) + return os.path.join(chip_arch_root(bbdir, chip), sim_name(bbdir, chip, product)) -def rtl_out(bbdir, chip, product, output_dir=None, *, rushb=False): - name = sim_name(bbdir, chip, product, rushb=rushb) +def rtl_out(bbdir, chip, product, output_dir=None): + name = sim_name(bbdir, chip, product) out = output_dir or os.path.join(chip_arch_root(bbdir, chip), name) - return name, out + return (name, out) def _log_part(value, what): @@ -108,18 +101,14 @@ def _log_part(value, what): raise ValueError(f"log dir missing {what}") if os.path.basename(value) != value: raise ValueError(f"log {what} must be a single path component: {value!r}") - if not re.fullmatch(r"[A-Za-z0-9_.-]+", value): + if not re.fullmatch("[A-Za-z0-9_.-]+", value): raise ValueError(f"invalid log {what}: {value!r}") return value def log_dir(bbdir, chip, product, stamp, tool, name, output_dir=None): chip = _chip_name(chip) - config = _log_part( - os.path.basename(os.path.normpath(rtl_dir(bbdir, chip, product, output_dir))), - "config", - ) stamp = _log_part(stamp, "timestamp") tool = _log_part(tool, "tool") name = _log_part(os.path.basename(name), "name") - return os.path.join(bbdir, "log", f"{stamp}-{chip}-{config}-{tool}-{name}") + return os.path.join(bbdir, "log", f"{stamp}-{chip}-{tool}-{name}") diff --git a/bbdev/api/utils/workload_manifest.py b/bbdev/api/utils/workload_manifest.py index 89d68d71d..27822d5ce 100644 --- a/bbdev/api/utils/workload_manifest.py +++ b/bbdev/api/utils/workload_manifest.py @@ -1,6 +1,5 @@ import os from pathlib import Path - from .path import get_buckyball_path @@ -25,7 +24,6 @@ def resolve_workload_toml( backend: str, test_type: str, bbdir: str | None = None, - rushB: bool = False, diff: bool = False, ) -> str: if test_type == "elf-tests": @@ -34,13 +32,11 @@ def resolve_workload_toml( suffix = "pk" else: raise ValueError(f"invalid test type: {test_type}") - regression_dir = chip_regression_dir(chip, backend, bbdir) if diff: diff_toml = regression_dir / f"workloads-{suffix}-diff.toml" if diff_toml.is_file(): return str(diff_toml) - name = f"workloads-{suffix}.toml" toml = regression_dir / name if not toml.is_file(): diff --git a/bbdev/bbdev b/bbdev/bbdev index e7e74c57d..d87bebfe0 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -41,9 +41,12 @@ WORKFLOW_COMMANDS = { }, "workload": { "clean": 'Clean workload output directory. Args: "--chip "', - "build": 'Build workload. Args: "--chip [--model ] [--trace-config ] [--stable] [--rushB ] [--ctest|--mlirtest]"; --ctest and --mlirtest are mutually exclusive local builds and cannot be combined with --model or --rushB.', + "build": 'Build test workloads. Args: "--chip [--stable] [--ctest|--mlirtest]"', "tohex": 'Convert chip workload ELF to hex. Args: "--chip "', }, + "model": { + "build": 'Build model recipes (all chip models when --model is omitted). Args: "--chip [--model ] [--ctrace] [--dtrace]"', + }, "kernel": { "build": 'Build RISC-V kernel + rootfs. Args: "[--chip ] [--model ] [--interactive] [--visible-hart-count ] [--total-hart-count ]"; --model requires --chip', }, @@ -57,17 +60,17 @@ WORKFLOW_COMMANDS = { "build": 'Build compiler. Args: "--chip "', }, "bebop-bemu": { - "sim": 'Run Bebop emulator simulation. Args: "--chip --binary [--rushB] [--pk] [--disasm] [--tool-profile] [--itrace] [--mtrace]"', + "sim": 'Run Bebop emulator simulation. Model args: "--chip --model ". ELF args: "--chip --binary [--pk] [--disasm] [--tool-profile] [--itrace] [--mtrace]"', "analysis": 'Analyze itrace/mtrace in an existing log dir. Args: "--chip --log-dir --itrace|--mtrace"', - "batch": 'Run Bebop BEMU batch regression. Args: "--chip --test [--rushB] [--clean-before]"', + "batch": 'Run Bebop BEMU batch regression. Args: "--chip --test [--clean-before]"', }, "bebop-verilator": { "clean": 'Clean Bebop Verilator build directory. Args: "--chip [--output-dir ]"', "verilog": 'Generate Bebop Verilog files. Args: "--chip [--diff] [--output-dir ]"', "build": 'Build Bebop Verilator simulation executable. Args: "--chip [--diff] [--jobs ] [--output-dir ]"', - "sim": 'Run Bebop Verilator simulation. Args: "--chip --binary [--rushB] [--log-dir ] [--fst-dir ] [--no-wave]"', - "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--rushB] [--jobs ] [--chip ] [--log-dir ] [--fst-dir ] [--no-wave]"', - "batch": 'Run Bebop Verilator batch regression. Args: "--chip --test [--diff] [--rushB] [--clean-before]"', + "sim": 'Run Bebop Verilator simulation. Args: "--chip --binary [--log-dir ] [--fst-dir ] [--no-wave]"', + "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--jobs ] [--chip ] [--log-dir ] [--fst-dir ] [--no-wave]"', + "batch": 'Run Bebop Verilator batch regression. Args: "--chip --test [--diff] [--clean-before]"', }, "bebop-vcs": { "verilog": 'Generate Bebop VCS RTL once. Args: "--chip "', diff --git a/bbdev/mcp/common.py b/bbdev/mcp/common.py index 4e0532736..48cedb1de 100644 --- a/bbdev/mcp/common.py +++ b/bbdev/mcp/common.py @@ -133,7 +133,7 @@ def _stop() -> None: if _port is not None and bbdev_path().is_file(): subprocess.run( ["nix", "develop", "--command", str(bbdev_path()), "stop", "--server", "--port", str(_port)], - cwd=str(bbdev_path().parent), + cwd=str(repo_path()), capture_output=True, text=True, timeout=30, @@ -186,7 +186,7 @@ def _ensure() -> int: _log_fh = open(log_path(), "a", encoding="utf-8") _proc = subprocess.Popen( ["nix", "develop", "--command", str(bbdev_path()), "start", "--server", "--port", str(port)], - cwd=str(bbdev_path().parent), + cwd=str(repo_path()), stdout=_log_fh, stderr=_log_fh, start_new_session=True, @@ -352,64 +352,21 @@ def _load_toml(path: Path) -> Dict[str, Any]: def _balldomain_path(chip: str, balldomain: Optional[str]) -> Path: - chip_root = repo_path() / "examples" / "chips" / chip - cfg = json.loads( - ( - repo_path() - / "examples" - / "chips" - / chip - / "configs" - / "generated" - / "config" - / "config.json" - ) - .read_text(encoding="utf-8") - ) - cores = [] - for tile in cfg["designs"]["tiles"]: - cores.extend(tile["cores"]) - seen = [] - uniq = [] - for core in cores: - key = core["balldomain"]["_file"] - if key not in seen: - seen.append(key) - uniq.append(core) - if len(uniq) != 1: - raise ValueError(f"chip {chip} has no unique topology core") - core = uniq[0] - core_file = Path(core["_file"]) - parts = core_file.parts - core_name = parts[parts.index("cores") + 1] - core_root = repo_path() / "examples" / "cores" / core_name - domains = core_root / "configs" / "balldomains" - - if balldomain is None: - bd = core["balldomain"] - path = Path(bd["_file"]) - if not path.is_absolute(): - path = (repo_path() / bd["_file"]).resolve() - else: - path = path.resolve() - else: + if balldomain: raw = Path(balldomain) - if raw.is_absolute(): - path = raw - elif balldomain.endswith(".toml"): - candidates = [ - domains / raw.name, - core_root / balldomain, - chip_root / balldomain, - ] - path = next( - (p.resolve() for p in candidates if p.is_file()), - candidates[0].resolve(), - ) - else: - path = (domains / f"{balldomain}.toml").resolve() - - return path + if raw.is_absolute() or len(raw.parts) > 1: + return (repo_path() / raw).resolve() + derived = repo_path() / "examples" / "chips" / chip / "configs/generated/config/derived.json" + cfg = json.loads(derived.read_text(encoding="utf-8")) + domains = {core["balldomain_base_dir"] for core in cfg["cores"]} + if len(domains) != 1: + raise ValueError(f"chip {chip} has multiple BallDomains; provide an explicit TOML path") + directory = repo_path() / domains.pop() + if balldomain: + return directory / (balldomain if balldomain.endswith(".toml") else f"{balldomain}.toml") + core_file = repo_path() / cfg["cores"][0]["config_path"] + include = _load_toml(core_file)["balldomain"] + return (core_file.parent / include).resolve() def _validate(path: Path) -> Dict[str, Any]: diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index 96c5d4c05..c532545ff 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -8,6 +8,7 @@ from . import compiler_build from . import workload_clean from . import workload_build +from . import model_build from . import bemu_sim from . import bemu_analysis from . import bemu_batch @@ -56,6 +57,7 @@ def modules(): compiler_build, workload_clean, workload_build, + model_build, bemu_sim, bemu_analysis, bemu_batch, diff --git a/bbdev/mcp/tools/bebop_verilator_batch.py b/bbdev/mcp/tools/bebop_verilator_batch.py index 8b9aad923..c1a53dba9 100644 --- a/bbdev/mcp/tools/bebop_verilator_batch.py +++ b/bbdev/mcp/tools/bebop_verilator_batch.py @@ -1,25 +1,20 @@ """MCP tool: bbdev_bebop_verilator_batch.""" from __future__ import annotations - - from common import submit, err, fmt, need def register(mcp): + @mcp.tool() def bbdev_bebop_verilator_batch( - chip: str, - test: str, - clean_before: bool = False, - rushB: bool = False, - diff: bool = False, + chip: str, test: str, clean_before: bool = False, diff: bool = False ) -> str: """Batch bebop-verilator regression. POST /bebop/verilator/batch.""" if e := need("chip", chip): return err(e) - if test != "elf-tests" and not (rushB and test == "pk-tests"): - return err("test must be elf-tests, or pk-tests with rushB") + if test != "elf-tests": + return err("test must be elf-tests") return fmt( submit( "/bebop/verilator/batch", @@ -27,7 +22,6 @@ def bbdev_bebop_verilator_batch( "chip": chip, "test": test, "clean-before": clean_before, - "rushB": rushB, "diff": diff, }, ) diff --git a/bbdev/mcp/tools/bebop_verilator_run.py b/bbdev/mcp/tools/bebop_verilator_run.py index 5cf59e75b..4f3faa645 100644 --- a/bbdev/mcp/tools/bebop_verilator_run.py +++ b/bbdev/mcp/tools/bebop_verilator_run.py @@ -1,12 +1,11 @@ """MCP tool: bbdev_bebop_verilator_run.""" from __future__ import annotations - - -from common import submit, err, fmt, need, opt +from common import submit, err, fmt, need def register(mcp): + @mcp.tool() def bbdev_bebop_verilator_run( binary: str, @@ -18,7 +17,6 @@ def bbdev_bebop_verilator_run( ctrace: bool = False, banktrace: bool = False, no_wave: bool = False, - rushB: bool = False, diff: bool = False, batch: bool = False, ) -> str: @@ -39,7 +37,6 @@ def bbdev_bebop_verilator_run( "ctrace": ctrace, "banktrace": banktrace, "no-wave": no_wave, - "rushB": rushB, "diff": diff, "batch": batch, }, diff --git a/bbdev/mcp/tools/bebop_verilator_sim.py b/bbdev/mcp/tools/bebop_verilator_sim.py index 59f2b84e6..2c0b70cd8 100644 --- a/bbdev/mcp/tools/bebop_verilator_sim.py +++ b/bbdev/mcp/tools/bebop_verilator_sim.py @@ -1,13 +1,12 @@ """MCP tool: bbdev_bebop_verilator_sim.""" from __future__ import annotations - from typing import Any, Dict - from common import submit, err, fmt, need def register(mcp): + @mcp.tool() def bbdev_bebop_verilator_sim( binary: str, @@ -18,7 +17,6 @@ def bbdev_bebop_verilator_sim( ctrace: bool = False, banktrace: bool = False, no_wave: bool = False, - rushB: bool = False, batch: bool = False, ) -> str: """Run one workload on bebop-verilator. POST /bebop/verilator/sim.""" @@ -34,7 +32,6 @@ def bbdev_bebop_verilator_sim( "ctrace": ctrace, "banktrace": banktrace, "no-wave": no_wave, - "rushB": rushB, "batch": batch, } return fmt(submit("/bebop/verilator/sim", params)) diff --git a/bbdev/mcp/tools/bemu_batch.py b/bbdev/mcp/tools/bemu_batch.py index 65509578d..cb4ac11cf 100644 --- a/bbdev/mcp/tools/bemu_batch.py +++ b/bbdev/mcp/tools/bemu_batch.py @@ -1,16 +1,13 @@ """MCP tool: bbdev_bemu_batch.""" from __future__ import annotations - - from common import submit, err, fmt, need def register(mcp): + @mcp.tool() - def bbdev_bemu_batch( - chip: str, test: str, clean_before: bool = False, rushB: bool = False - ) -> str: + def bbdev_bemu_batch(chip: str, test: str, clean_before: bool = False) -> str: """Batch bemu regression. test: elf-tests|pk-tests. POST /bebop/bemu/batch.""" if e := need("chip", chip): return err(e) @@ -19,11 +16,6 @@ def bbdev_bemu_batch( return fmt( submit( "/bebop/bemu/batch", - { - "chip": chip, - "test": test, - "clean-before": clean_before, - "rushB": rushB, - }, + {"chip": chip, "test": test, "clean-before": clean_before}, ) ) diff --git a/bbdev/mcp/tools/bemu_sim.py b/bbdev/mcp/tools/bemu_sim.py index 1f5b59270..8ab005501 100644 --- a/bbdev/mcp/tools/bemu_sim.py +++ b/bbdev/mcp/tools/bemu_sim.py @@ -1,13 +1,12 @@ """MCP tool: bbdev_bemu_sim.""" from __future__ import annotations - from typing import Any, Dict - from common import submit, err, fmt, need def register(mcp): + @mcp.tool() def bbdev_bemu_sim( chip: str, @@ -17,7 +16,8 @@ def bbdev_bemu_sim( tool_profile: bool = False, itrace: bool = False, mtrace: bool = False, - rushB: bool = False, + core_index: int | None = None, + arguments: list[str] | None = None, ) -> str: """Run one workload on bebop-bemu. POST /bebop/bemu/sim.""" for n, v in (("chip", chip), ("binary", binary)): @@ -31,6 +31,7 @@ def bbdev_bemu_sim( "tool-profile": tool_profile, "itrace": itrace, "mtrace": mtrace, - "rushB": rushB, + "core_index": core_index, + "arguments": arguments if arguments is not None else [], } return fmt(submit("/bebop/bemu/sim", params)) diff --git a/bbdev/mcp/tools/kernel_build.py b/bbdev/mcp/tools/kernel_build.py index 6b10291d6..951a80d01 100644 --- a/bbdev/mcp/tools/kernel_build.py +++ b/bbdev/mcp/tools/kernel_build.py @@ -20,8 +20,8 @@ def bbdev_kernel_build( With --chip only: uses examples/chips//kernel OS overlay and packs that chip's bemu workloads-pk.toml into fw_payload--pk. - With --model: requires --chip; packs ModelTest runtime from - archs/buckyball// into fw_payload-. + With --model: requires --chip; packs a native model artifact from + stack/models/build///artifact into fw_payload-. With --interactive: keep shared /init shell and do not auto-run. """ params: Dict[str, Any] = {} diff --git a/bbdev/mcp/tools/model_build.py b/bbdev/mcp/tools/model_build.py new file mode 100644 index 000000000..647688499 --- /dev/null +++ b/bbdev/mcp/tools/model_build.py @@ -0,0 +1,15 @@ +"""MCP tool: bbdev_model_build.""" + +from common import submit, err, fmt, need + + +def register(mcp): + @mcp.tool() + def bbdev_model_build(chip: str, model: str | None = None, ctrace: bool = False, dtrace: bool = False) -> str: + """Build one model recipe, or all recipes for the chip when model is omitted. POST /model/build.""" + if error := need("chip", chip): + return err(error) + params = {"chip": chip, "ctrace": ctrace, "dtrace": dtrace} + if model is not None: + params["model"] = model + return fmt(submit("/model/build", params)) diff --git a/bbdev/mcp/tools/workload_build.py b/bbdev/mcp/tools/workload_build.py index f62b05fe7..ecff3e8e6 100644 --- a/bbdev/mcp/tools/workload_build.py +++ b/bbdev/mcp/tools/workload_build.py @@ -1,18 +1,15 @@ """MCP tool: bbdev_workload_build.""" from __future__ import annotations - -from typing import Any, Dict, Optional - -from common import submit, err, fmt, need, opt +from typing import Any, Dict +from common import submit, err, fmt, need def register(mcp): + @mcp.tool() def bbdev_workload_build( chip: str, - model: Optional[str] = None, - rushB: Optional[str] = None, ctest: bool = False, mlirtest: bool = False, ) -> str: @@ -20,16 +17,8 @@ def bbdev_workload_build( if e := need("chip", chip): return err(e) params: Dict[str, Any] = {"chip": chip} - if model: - params["model"] = model - if rushB: - if rushB not in {"bemu", "verilator"}: - return err("rushB must be bemu or verilator") - params["rushB"] = rushB if ctest and mlirtest: return err("ctest and mlirtest cannot be used together") - if (ctest or mlirtest) and (model or rushB): - return err("ctest and mlirtest cannot be used with model or rushB") if ctest: params["ctest"] = True if mlirtest: From a0bba72e4bed6fc526fa9eb2efab84efbedf69d0 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 29 Sep 2026 17:10:46 +0800 Subject: [PATCH 173/196] fix: clean up object directory before build in Verilator step --- .../steps/verilator/03_build_event.step.py | 58 +++++++++++++------ 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/bbdev/api/steps/verilator/03_build_event.step.py b/bbdev/api/steps/verilator/03_build_event.step.py index 8b6a5bb02..70c574473 100644 --- a/bbdev/api/steps/verilator/03_build_event.step.py +++ b/bbdev/api/steps/verilator/03_build_event.step.py @@ -1,4 +1,5 @@ import os +import shutil import subprocess import glob import sys @@ -30,11 +31,19 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" build_dir = rtl_dir( - bbdir, chip, "verilog", input_data.get("output_dir"), + bbdir, + chip, + "verilog", + input_data.get("output_dir"), ) coverage = input_data.get("coverage", False) ctx.logger.info(f"Using build directory: {build_dir}") + obj_dir = f"{build_dir}/obj_dir" + if os.path.exists(obj_dir): + shutil.rmtree(obj_dir) + os.makedirs(obj_dir) + # ================================================================================== # Find sources # ================================================================================== @@ -67,11 +76,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ================================================================================== result_dir = f"{bbdir}/result" result_lib = f"{result_dir}/lib" - inc_flags = " ".join([ - f"-I{result_dir}/include", - f"-I{build_dir}", - f"-I{arch_dir}/src/csrc/include", - ]) + inc_flags = " ".join( + [ + f"-I{result_dir}/include", + f"-I{build_dir}", + f"-I{arch_dir}/src/csrc/include", + ] + ) # -DBBSIM: selects VBBSimHarness in bdb.h / main.cc # BDB NDJSON trace (+trace=...) is runtime-only; bbdev sim uses +trace=all (04_sim_event.step.py). @@ -87,29 +98,32 @@ def _gcc_lib_dir(soname: str) -> str: return os.path.dirname(os.path.realpath(printed.stdout.strip())) rpath_dirs = [] - for lib_dir in (result_lib, _gcc_lib_dir("liblz4.so"), _gcc_lib_dir("libstdc++.so")): + for lib_dir in ( + result_lib, + _gcc_lib_dir("liblz4.so"), + _gcc_lib_dir("libstdc++.so"), + ): if lib_dir not in rpath_dirs: rpath_dirs.append(lib_dir) rpath_flags = " ".join(f"-Wl,-rpath,{lib_dir}" for lib_dir in rpath_dirs) - ldflags = ( - f"-lreadline -ldramsim3 -lstdc++ -lz " - f"-L{result_lib} {rpath_flags} " - ) - - obj_dir = f"{build_dir}/obj_dir" - subprocess.run(f"rm -rf {obj_dir}", shell=True) - os.makedirs(obj_dir, exist_ok=True) + ldflags = f"-lreadline -ldramsim3 -lstdc++ -lz " f"-L{result_lib} {rpath_flags} " sources = " ".join(vsrcs + csrcs) jobs = input_data.get("jobs", "") build_env = os.environ.copy() - if subprocess.run("command -v ccache", shell=True, capture_output=True).returncode == 0: + if ( + subprocess.run("command -v ccache", shell=True, capture_output=True).returncode + == 0 + ): build_env["OBJCACHE"] = "ccache" # Use lld for faster linking if available - use_lld = subprocess.run("command -v ld.lld", shell=True, capture_output=True).returncode == 0 + use_lld = ( + subprocess.run("command -v ld.lld", shell=True, capture_output=True).returncode + == 0 + ) if use_lld: ldflags += " -fuse-ld=lld" @@ -143,7 +157,10 @@ def _gcc_lib_dir(soname: str) -> str: ) if result.returncode != 0: await check_result( - ctx, result.returncode, continue_run=False, extra_fields={"task": "build"}, + ctx, + result.returncode, + continue_run=False, + extra_fields={"task": "build"}, trace_id=origin_tid, ) return @@ -174,7 +191,10 @@ def _gcc_lib_dir(soname: str) -> str: # ================================================================================== if input_data.get("from_run_workflow"): await ctx.enqueue( - {"topic": "verilator.sim", "data": {**input_data, "output_dir": build_dir, "task": "run"}} + { + "topic": "verilator.sim", + "data": {**input_data, "output_dir": build_dir, "task": "run"}, + } ) return From 2b1cc6ab484587dce0f3ce2cd710e317a8c1649c Mon Sep 17 00:00:00 2001 From: daiyongyuan <1533208939@qq.com> Date: Wed, 30 Sep 2026 01:21:21 +0800 Subject: [PATCH 174/196] fix(p2e): pass diff mode to bitstream build --- bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index 389c7a4ef..0165981f5 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -72,6 +72,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: **bebop_cargo_env(bbdir, chip), "VSRC_PATH": vsrc_dir, "OUT_PATH": str(build_path), + "P2E_DIFF": "1" if diff else "0", } if diff: features.append("bemu") @@ -83,6 +84,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--keep-env-var", "CARGO_TARGET_DIR", "--keep-env-var", "VSRC_PATH", "--keep-env-var", "OUT_PATH", + "--keep-env-var", "P2E_DIFF", "-c", "cargo", "run", "--release", "--manifest-path", str(manifest), "--bin", "bebop", From 9ff49f127854fd10c7ad4f8573edb14e13915b07 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 4 Oct 2026 20:08:42 +0800 Subject: [PATCH 175/196] feat: update performance reporting and P2E workflow management del: remove regression --- .../api/steps/bebop/bemu/02_sim_event.step.py | 73 +++- .../steps/bebop/bemu/scripts/bemu_analysis.py | 66 ++- .../api/steps/bebop/bemu/scripts/model_sim.py | 63 ++- .../bebop/p2e/02_buildbitstream_api.step.py | 28 +- .../bebop/p2e/02_buildbitstream_event.step.py | 30 +- .../bebop/p2e/03_runworkload_api.step.py | 21 +- .../bebop/p2e/03_runworkload_event.step.py | 164 ++++---- .../steps/bebop/p2e/04_batch_event.step.py | 11 +- .../steps/bebop/p2e/scripts/runtime_case.py | 33 ++ bbdev/api/steps/bebop/performance_report.py | 223 +++++++++++ .../bebop/verilator/02_verilog_api.step.py | 4 +- .../steps/bebop/verilator/04_sim_api.step.py | 5 +- .../bebop/verilator/04_sim_event.step.py | 25 +- .../config/scripts/2_parameter_derivation.py | 179 ++++----- .../api/steps/config/scripts/3_json2proto.py | 77 ++-- bbdev/api/steps/config/scripts/4_install.py | 13 +- .../steps/config/scripts/ball_normalize.py | 71 ++++ .../api/steps/config/scripts/proto/chip.proto | 36 +- bbdev/api/steps/dc/01_area_event.step.py | 44 +- bbdev/api/steps/dc/scripts/report.py | 31 ++ .../scripts/makefrag/firesim/config.mk | 4 +- bbdev/api/steps/kernel/01_build_api.step.py | 8 + bbdev/api/steps/kernel/01_build_event.step.py | 37 +- .../mill/07_bebop_p2e_verilog_event.step.py | 17 +- .../regression/00_buildbitstream_api.step.py | 37 -- .../00_buildbitstream_event.step.py | 56 --- .../api/steps/regression/01_check_api.step.py | 48 --- .../steps/regression/01_check_event.step.py | 70 ---- .../02_eval_performance_api.step.py | 48 --- .../02_eval_performance_event.step.py | 375 ------------------ .../regression/03_eval_area_power_api.step.py | 25 -- .../03_eval_area_power_event.step.py | 71 ---- bbdev/api/steps/regression/__init__.py | 0 .../api/steps/regression/scripts/accuracy.py | 87 ---- bbdev/api/steps/regression/scripts/dc_area.py | 7 - .../steps/regression/scripts/model_layout.py | 13 - .../steps/regression/scripts/models_toml.py | 20 - .../regression/scripts/perfetto_latency.py | 34 -- bbdev/api/steps/regression/scripts/result.py | 19 - bbdev/api/steps/uvm/01_build_api.step.py | 7 +- bbdev/api/steps/uvm/01_build_event.step.py | 1 + bbdev/api/steps/uvm/02_run_api.step.py | 7 +- bbdev/api/steps/uvm/02_run_event.step.py | 17 +- bbdev/api/steps/uvm/scripts/report.py | 94 +++++ bbdev/api/steps/uvm/scripts/uvm_common.py | 260 +++++++----- bbdev/api/tests/test_all_api_serial.py | 2 - .../test_api_regression_buildbitstream.py | 3 - .../test_api_regression_eval_area_power.py | 3 - bbdev/api/tests/test_bemu_binary.py | 50 +++ bbdev/api/tests/test_p2e_resume.py | 67 ++++ bbdev/api/tests/test_p2e_runtime_reuse.py | 54 +++ bbdev/api/tests/test_uvm_ip_profiles.py | 51 +++ bbdev/api/utils/process.py | 6 +- bbdev/api/utils/report_schema.py | 183 +++++++++ bbdev/api/utils/reports.py | 125 ++++++ bbdev/bbdev | 23 +- bbdev/mcp/common.py | 60 ++- bbdev/mcp/tests/test_common.py | 4 +- bbdev/mcp/tools/__init__.py | 8 - bbdev/mcp/tools/bebop_p2e_buildbitstream.py | 5 +- bbdev/mcp/tools/bebop_p2e_runworkload.py | 22 +- bbdev/mcp/tools/bebop_verilator_sim.py | 2 + bbdev/mcp/tools/bemu_sim.py | 16 +- bbdev/mcp/tools/kernel_build.py | 4 +- bbdev/mcp/tools/regression_buildbitstream.py | 24 -- bbdev/mcp/tools/regression_check.py | 24 -- bbdev/mcp/tools/regression_eval_area_power.py | 24 -- .../mcp/tools/regression_eval_performance.py | 24 -- bbdev/mcp/tools/uvm_build.py | 6 +- bbdev/mcp/tools/uvm_run.py | 6 +- bbdev/mcp/tools/validate.py | 4 +- 71 files changed, 1813 insertions(+), 1546 deletions(-) create mode 100644 bbdev/api/steps/bebop/p2e/scripts/runtime_case.py create mode 100644 bbdev/api/steps/bebop/performance_report.py create mode 100644 bbdev/api/steps/config/scripts/ball_normalize.py create mode 100644 bbdev/api/steps/dc/scripts/report.py delete mode 100644 bbdev/api/steps/regression/00_buildbitstream_api.step.py delete mode 100644 bbdev/api/steps/regression/00_buildbitstream_event.step.py delete mode 100644 bbdev/api/steps/regression/01_check_api.step.py delete mode 100644 bbdev/api/steps/regression/01_check_event.step.py delete mode 100644 bbdev/api/steps/regression/02_eval_performance_api.step.py delete mode 100644 bbdev/api/steps/regression/02_eval_performance_event.step.py delete mode 100644 bbdev/api/steps/regression/03_eval_area_power_api.step.py delete mode 100644 bbdev/api/steps/regression/03_eval_area_power_event.step.py delete mode 100644 bbdev/api/steps/regression/__init__.py delete mode 100644 bbdev/api/steps/regression/scripts/accuracy.py delete mode 100644 bbdev/api/steps/regression/scripts/dc_area.py delete mode 100644 bbdev/api/steps/regression/scripts/model_layout.py delete mode 100644 bbdev/api/steps/regression/scripts/models_toml.py delete mode 100644 bbdev/api/steps/regression/scripts/perfetto_latency.py delete mode 100644 bbdev/api/steps/regression/scripts/result.py create mode 100644 bbdev/api/steps/uvm/scripts/report.py delete mode 100644 bbdev/api/tests/test_api_regression_buildbitstream.py delete mode 100644 bbdev/api/tests/test_api_regression_eval_area_power.py create mode 100644 bbdev/api/tests/test_bemu_binary.py create mode 100644 bbdev/api/tests/test_p2e_resume.py create mode 100644 bbdev/api/tests/test_p2e_runtime_reuse.py create mode 100644 bbdev/api/tests/test_uvm_ip_profiles.py create mode 100644 bbdev/api/utils/report_schema.py create mode 100644 bbdev/api/utils/reports.py delete mode 100644 bbdev/mcp/tools/regression_buildbitstream.py delete mode 100644 bbdev/mcp/tools/regression_check.py delete mode 100644 bbdev/mcp/tools/regression_eval_area_power.py delete mode 100644 bbdev/mcp/tools/regression_eval_performance.py diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 5109a340a..2d5634941 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -29,6 +29,8 @@ from utils.event_common import check_result, get_origin_trace_id from utils.process_registry import cancellation_requested from bemu_common import bemu_manifest, bemu_tile_index, chip_emu_manifest +from steps.bebop.performance_report import performance_report +from utils.reports import source_context config = { "name": "bebop-bemu-sim", @@ -66,7 +68,8 @@ def clean_model_trace(binary_dir: str) -> None: def resolve_bemu_binary(bbdir: str, chip: str, binary_name: str) -> str | None: if Path(binary_name).name != binary_name: return None - workload = search_workload(workloads_output_root(bbdir), binary_name) + workload_root = Path(workloads_output_root(bbdir)) / chip / "workloads" + workload = search_workload(str(workload_root), binary_name) if workload is not None: return workload kernel = Path(bbdir) / "bb-tests" / "output" / "kernel" / chip / binary_name @@ -100,27 +103,59 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return if input_data.get("model"): + report_context = source_context(bbdir, chip) params = {key: value for key, value in input_data.items() if key != "_trace_id"} commands, run_log = model_run_commands(bbdir, params) run_log.mkdir(parents=True, exist_ok=True) ctx.logger.info(f"Model simulation logs: {run_log}") - for command, filename in zip(commands, ("build.log", "run.log"), strict=True): + filenames = ("run.log",) if input_data.get("reuse-simulator", False) else ("build.log", "run.log") + for command, filename in zip(commands, filenames, strict=True): command_line = shlex.join(["nix", "develop", "-c", *command]) command_line = f"set -o pipefail; {command_line} 2>&1 | tee {shlex.quote(str(run_log / filename))}" result = await stream_run_logger_async( - cmd=command_line, executable="/bin/bash", - logger=ctx.logger, cwd=bbdir, task_scope=origin_tid, - stdout_prefix="model simulation", stderr_prefix="model simulation") + cmd=command_line, + executable="/bin/bash", + logger=ctx.logger, + cwd=bbdir, + task_scope=origin_tid, + stdout_prefix="model simulation", + stderr_prefix="model simulation", + ) if cancellation_requested(origin_tid): return if result.returncode: break - await check_result(ctx, result.returncode, continue_run=False, - extra_fields={"chip": chip, "model": params["model"], "log_dir": str(run_log)}, - trace_id=origin_tid) + try: + report_path = performance_report(bbdir, report_context, origin_tid, "bemu", run_log, (run_log / "run.log").read_text(), result.returncode) + except (OSError, ValueError, KeyError) as error: + await check_result(ctx, 1, extra_fields={"task": "report", "error": str(error), "log_dir": str(run_log)}, trace_id=origin_tid) + return + await check_result( + ctx, + result.returncode, + continue_run=False, + extra_fields={ + "chip": chip, + "model": params["model"], + "report_path": report_path, + "log_dir": str(run_log), + }, + trace_id=origin_tid, + ) return binary_name = input_data.get("binary", "") - binary_path = resolve_bemu_binary(bbdir, chip, binary_name) + try: + binary_path = resolve_bemu_binary(bbdir, chip, binary_name) + except ValueError as error: + ctx.logger.error(str(error)) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "binary_resolution_error", "binary": binary_name}, + trace_id=origin_tid, + ) + return if binary_path is None: ctx.logger.error(f"binary not found: {binary_name}") await check_result( @@ -188,6 +223,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--log-dir", run_log, ] + cargo_args[1:1] = [ + "--config", + 'build.rustflags=["-C", "target-cpu=native"]', + "--config", + 'profile.release.lto="thin"', + "--config", + "profile.release.codegen-units=1", + ] if core_index is not None: cargo_args.extend(["--core-index", str(core_index)]) if input_data.get("pk"): @@ -203,13 +246,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cargo_args.append(f"--{trace_name}") arguments = input_data.get("arguments", []) if arguments: - if not isinstance(arguments, list) or not all(isinstance(arg, str) for arg in arguments): + if not isinstance(arguments, list) or not all( + isinstance(arg, str) for arg in arguments + ): raise ValueError("arguments must be a list of strings") cargo_args.extend(["--", *arguments]) inner_cmd = f"cd {shlex.quote(binary_dir)} && {shlex.join(cargo_args)}" run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" run_cmd = f"set -o pipefail; {run_cmd} 2>&1 | tee {shlex.quote(str(Path(run_log) / 'run.log'))}" ctx.logger.info(f"Running bebop bemu: {run_cmd}") + report_context = source_context(bbdir, chip) run_result = await stream_run_logger_async( cmd=run_cmd, logger=ctx.logger, @@ -222,6 +268,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) if cancellation_requested(origin_tid): return + try: + report_path = performance_report(bbdir, report_context, origin_tid, "bemu", run_log, (Path(run_log) / "run.log").read_text(), run_result.returncode) + except (OSError, ValueError, KeyError) as error: + await check_result(ctx, 1, extra_fields={"task": "report", "error": str(error), "log_dir": run_log}, trace_id=origin_tid) + return if run_result.returncode != 0: await check_result( ctx, @@ -229,6 +280,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "bemu", + "report_path": report_path, "binary": binary_path, "chip": chip, "log_dir": run_log, @@ -271,6 +323,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "bemu", + "report_path": report_path, "binary": binary_path, "chip": chip, "log_dir": run_log, diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py index e12b4e667..204addaf3 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py @@ -82,7 +82,9 @@ def chip_maps(bbdir: str, chip: str) -> tuple[dict[int, str], set[int], int]: if not isinstance(funct, int) or funct < 0: raise ValueError(f"ballISA funct7 must be a non-negative int: {ball_path}") if not isinstance(mnemonic, str) or not mnemonic: - raise ValueError(f"ballISA mnemonic must be a non-empty string: {ball_path}") + raise ValueError( + f"ballISA mnemonic must be a non-empty string: {ball_path}" + ) if funct in seen: raise ValueError(f"duplicate ballISA funct7 {funct}: {ball_path}") if funct in system_isa: @@ -123,13 +125,11 @@ def analysis_dir( n_itrace = 0 n_mtrace = 0 - prev_clk: int | None = None + prev_event: int | None = None funct_n: Counter[str] = Counter() - funct_cyc: Counter[str] = Counter() mnk_n: Counter[tuple[int, int, int]] = Counter() - rows_n: Counter[int] = Counter() - rows_sum = 0 - rows_1 = 0 + bank_n: Counter[int] = Counter() + access_n: Counter[str] = Counter() with bdb.open(encoding="utf-8") as source: for line_no, raw in enumerate(source, start=1): @@ -145,39 +145,34 @@ def analysis_dir( kind = obj.get("type") if kind == "itrace": n_itrace += 1 - if "clk" not in obj: - raise ValueError(f"bdb.ndjson:{line_no}: itrace missing clk") - clk = obj["clk"] - if not isinstance(clk, int) or clk < 0: - raise ValueError(f"bdb.ndjson:{line_no}: itrace clk must be a non-negative int") - if prev_clk is not None and clk < prev_clk: + event_index = obj["event_index"] + if not isinstance(event_index, int) or event_index < 0: raise ValueError( - f"bdb.ndjson:{line_no}: itrace clk went backwards {prev_clk} -> {clk}" + f"bdb.ndjson:{line_no}: event_index must be a non-negative int" + ) + if prev_event is not None and event_index <= prev_event: + raise ValueError( + f"bdb.ndjson:{line_no}: instruction event order is not increasing" ) funct = _hex_u(obj.get("funct"), "funct", line_no) if funct not in names: - known = ", ".join(f"{v}=0x{k:02x}" for k, v in sorted(names.items())) + known = ", ".join( + f"{v}=0x{k:02x}" for k, v in sorted(names.items()) + ) raise ValueError( f"bdb.ndjson:{line_no}: unknown funct 0x{funct:02x}; known: {known}" ) name = names[funct] - delta = clk if prev_clk is None else clk - prev_clk funct_n[name] += 1 - funct_cyc[name] += delta - prev_clk = clk + prev_event = event_index if funct in matrix: rs1 = _hex_u(obj.get("rs1"), "rs1", line_no) rs2 = _hex_u(obj.get("rs2"), "rs2", line_no) mnk_n[_mnk(rs1, rs2)] += 1 elif kind == "mtrace": n_mtrace += 1 - rows = obj.get("rows") - if not isinstance(rows, int) or rows < 0: - raise ValueError(f"bdb.ndjson:{line_no}: mtrace rows must be a non-negative int") - rows_n[rows] += 1 - rows_sum += rows - if rows == 1: - rows_1 += 1 + bank_n[obj["vbank_id"]] += 1 + access_n[obj["event"]] += 1 elif kind is None: raise ValueError(f"bdb.ndjson:{line_no}: missing type") @@ -188,17 +183,12 @@ def analysis_dir( lines: list[str] = [f"log_dir: {log_dir}"] if itrace: - total_cyc = sum(funct_cyc.values()) - if total_cyc == 0: - raise ValueError(f"itrace span is 0 cycles in {bdb}") lines.append("== itrace ==") lines.append(f"events: {n_itrace}") - lines.append(f"span_cycles: {total_cyc}") - lines.append(f"{'funct':<16} {'n':>10} {'cycles':>14} {'pct':>8}") + lines.append(f"{'funct':<16} {'n':>10} {'pct':>8}") for name, count in funct_n.most_common(): - cyc = funct_cyc[name] - pct = 100.0 * cyc / total_cyc - lines.append(f"{name:<16} {count:>10} {cyc:>14} {pct:>7.1f}%") + pct = 100.0 * count / n_itrace + lines.append(f"{name:<16} {count:>10} {pct:>7.1f}%") lines.append(f"fence: {funct_n.get('fence', 0)}") lines.append(f"mset: {funct_n.get('mset', 0)}") if matrix: @@ -209,16 +199,14 @@ def analysis_dir( for (m, n, k), count in mnk_n.most_common(): lines.append(f" ({m},{n},{k}) n={count}") if mtrace: - mean = rows_sum / n_mtrace lines.append("== mtrace ==") lines.append(f"events: {n_mtrace}") lines.append(f"bank_depth: {bank_depth}") - lines.append(f"mean_rows: {mean:.2f}") - lines.append(f"mean_rows/bank_depth: {mean / bank_depth:.4f}") - lines.append(f"rows=1: {rows_1} ({100.0 * rows_1 / n_mtrace:.1f}%)") - lines.append("rows histogram:") - for rows, count in sorted(rows_n.items()): - lines.append(f" {rows}: {count}") + lines.append(f"reads: {access_n['read']}") + lines.append(f"writes: {access_n['write']}") + lines.append("bank events:") + for bank, count in sorted(bank_n.items()): + lines.append(f" {bank}: {count}") return "\n".join(lines) + "\n" diff --git a/bbdev/api/steps/bebop/bemu/scripts/model_sim.py b/bbdev/api/steps/bebop/bemu/scripts/model_sim.py index bc3561fdb..f24196bed 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/model_sim.py +++ b/bbdev/api/steps/bebop/bemu/scripts/model_sim.py @@ -8,41 +8,80 @@ def model_run_commands(repo: str, params: dict): - unknown = params.keys() - {"chip", "model"} + unknown = params.keys() - {"chip", "model", "reuse-simulator"} if unknown: raise ValueError(f"Unknown model simulation parameters: {sorted(unknown)}") for key in ("chip", "model"): - if not isinstance(params.get(key), str) or not re.fullmatch(r"[A-Za-z0-9_-]+", params[key]): + if not isinstance(params.get(key), str) or not re.fullmatch( + r"[A-Za-z0-9_-]+", params[key] + ): raise ValueError(f"Invalid or missing {key}") root = Path(repo) + if not isinstance(params.get("reuse-simulator", False), bool): + raise ValueError("reuse-simulator must be a boolean") chip, model = params["chip"], params["model"] config = root / "examples/models" / chip / model / "configs/running-param.toml" build = root / "stack/models/build" / chip / model packages = list((build / "artifact").glob("*.rax")) if len(packages) != 1: - raise ValueError(f"Expected one built RAX under {build / 'artifact'}; build the model first") + raise ValueError( + f"Expected one built RAX under {build / 'artifact'}; build the model first" + ) layout = json.loads((build / "artifact/layout.json").read_text()) kind = layout["execution"]["kind"] - if kind == "native": + settings = tomllib.loads(config.read_text()) + if kind == "native" and "tile_indices" not in settings: manifest = root / "examples/chips" / chip / "configs/generated/bemu/Cargo.toml" binary = "bebop-bemu" python = root / "result/bin/python3" - elif kind == "python": + elif kind == "python" or (kind == "native" and "tile_indices" in settings): manifest = root / "examples/chips" / chip / "emu/Cargo.toml" with manifest.open("rb") as source: binary = tomllib.load(source)["bin"][0]["name"] - python = root / "stack/serving/.venv/bin/python" + python = root / ("result/bin/python3" if kind == "native" else "stack/serving/.venv/bin/python") else: raise ValueError(f"Unknown model execution kind: {kind}") target = root / "bebop/target" / chip stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") log = Path(log_dir(repo, chip, "verilog", stamp, "bemu", model)) commands = [ - ["cargo", "build", "--release", "--manifest-path", str(manifest), - "--target-dir", str(target), "--bin", binary], - [str(python), "-u", "-m", "stack.serving.cli", "--model", str(packages[0]), - "--loader", str(build / "loader/load-model"), - "--simulator", str(target / "release" / binary), - "--log-dir", str(log), "--run-config", str(config)], + [ + "cargo", + "build", + "--release", + "--config", + 'build.rustflags=["-C", "target-cpu=native"]', + "--config", + 'profile.release.lto="thin"', + "--config", + "profile.release.codegen-units=1", + "--manifest-path", + str(manifest), + "--target-dir", + str(target), + "--bin", + binary, + ], + [ + str(python), + "-u", + "-m", + "stack.serving.cli", + "--model", + str(packages[0]), + "--loader", + str(build / "loader/load-model"), + "--simulator", + str(target / "release" / binary), + "--log-dir", + str(log), + "--run-config", + str(config), + ], ] + if params.get("reuse-simulator", False): + simulator = target / "release" / binary + if not simulator.is_file(): + raise ValueError(f"Reusable model simulator missing: {simulator}") + commands = commands[1:] return commands, log diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py index ba2a845ee..a1aa093e2 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py @@ -1,4 +1,5 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from pathlib import Path from utils.event_common import require_chip from utils.path import get_buckyball_path, rtl_dir @@ -23,11 +24,36 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir, chip, "p2e", body.get("vsrc_dir") or body.get("vsrc-dir") ) + resume = body.get("resume_post_route", body.get("resume-post-route", False)) + if not isinstance(resume, bool): + return ApiResponse(status=400, body={"error": "resume-post-route must be a boolean flag"}) + stop_after = body.get("stop_after", body.get("stop-after")) + if stop_after is not None and stop_after not in ("vsyn", "vcom"): + return ApiResponse(status=400, body={"error": "stop-after must be vsyn or vcom"}) + if resume and stop_after is not None: + return ApiResponse(status=400, body={"error": "stop-after conflicts with resume-post-route"}) + output_dir = body.get("output_dir") or body.get("output-dir") + if stop_after and output_dir and Path(output_dir).exists(): + return ApiResponse(status=400, body={"error": "Resource assessment requires a fresh output_dir"}) + if resume: + if not isinstance(output_dir, str) or not output_dir: + return ApiResponse(status=400, body={"error": "Post-route resume requires output_dir"}) + case = Path(output_dir).resolve() + build_root = (Path(bbdir) / "bebop/build").resolve() + if build_root not in case.parents or not case.is_dir(): + return ApiResponse(status=400, body={"error": "P2E resume requires an existing case under bebop/build"}) + for artifact in ("xepic_vvac_top_0_0_route.dcp", "xepic_vvac_top_0_0.bit"): + path = case / "fpgaCompDir/part_b0_f0/pnrDir" / artifact + if not path.is_file(): + return ApiResponse(status=400, body={"error": f"Post-route resume requires {path}"}) + data = { "chip": chip, "diff": bool(body.get("diff", False)), "vsrc_dir": vsrc_dir, - "output_dir": body.get("output_dir") or body.get("output-dir"), + "output_dir": output_dir, + "resume_post_route": resume, + "stop_after": stop_after, } await ctx.enqueue({ "topic": "bebop.p2e.buildbitstream", diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index 0165981f5..a02393289 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -53,6 +53,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: normalize_p2e_timescale(vsrc_dir, ctx.logger) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") diff = bool(input_data.get("diff", False)) + resume = input_data.get("resume_post_route", False) + stop_after = input_data.get("stop_after") + if resume and not input_data.get("output_dir"): + raise ValueError("Post-route resume requires an explicit output_dir") build_dir = ( input_data.get("output_dir") or f"{bebop_dir}/build/{chip}{'-diff' if diff else ''}-{timestamp}" @@ -61,9 +65,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: build_root = (Path(bebop_dir) / "build").resolve() if build_root not in build_path.parents: raise ValueError(f"P2E output_dir must be under {build_root}: {build_path}") - if build_path.exists(): + if resume and not build_path.is_dir(): + raise ValueError(f"P2E resume case does not exist: {build_path}") + if stop_after and build_path.exists(): + raise ValueError(f"Resource assessment requires a fresh output_dir: {build_path}") + if build_path.exists() and not resume: shutil.rmtree(build_path) - build_path.mkdir(parents=True) + build_path.mkdir(parents=True, exist_ok=resume) manifest = Path(bbdir) / "examples" / "chips" / chip / "generated" / "bebop" / "Cargo.toml" features = ["p2e"] @@ -72,7 +80,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: **bebop_cargo_env(bbdir, chip), "VSRC_PATH": vsrc_dir, "OUT_PATH": str(build_path), - "P2E_DIFF": "1" if diff else "0", } if diff: features.append("bemu") @@ -84,8 +91,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--keep-env-var", "CARGO_TARGET_DIR", "--keep-env-var", "VSRC_PATH", "--keep-env-var", "OUT_PATH", - "--keep-env-var", "P2E_DIFF", - "-c", "cargo", "run", "--release", + "-c", "env", "-u", "LD_LIBRARY_PATH", "cargo", "run", "--release", "--manifest-path", str(manifest), "--bin", "bebop", "--features", ",".join(features), @@ -93,7 +99,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--rtl-dir", vsrc_dir, "--out-dir", build_dir, *(["--diff"] if diff else []), + *(["--resume-post-route"] if resume else []), + *(["--stop-after", stop_after] if stop_after else []), ]) + build_cmd = f"cd {shlex.quote(bbdir)} && {build_cmd}" ctx.logger.info("Building bebop p2e runtime case ...") build_result = await stream_run_logger_async( cmd=build_cmd, @@ -119,9 +128,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "diff": diff, "timestamp": timestamp, } - if input_data.get("from_regression_buildbitstream") and build_result.returncode == 0: - extra_fields["bitstream"] = os.path.abspath(bitstream_path) - + if stop_after: + extra_fields = { + "task": "resource-assessment", "stop_after": stop_after, + "runtime_ready": False, "vsrc_dir": vsrc_dir, "build_dir": build_dir, + "diff": diff, "timestamp": timestamp, + } + else: + extra_fields["runtime_ready"] = build_result.returncode == 0 await check_result( ctx, build_result.returncode, diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py index 472cf701b..b34465116 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py @@ -1,4 +1,5 @@ from motia import ApiRequest, ApiResponse, FlowContext, api +from steps.bebop.p2e.scripts.runtime_case import validate_runtime_reuse, validate_cold_load_case config = { "name": "bebop-p2e-runworkload-api", @@ -11,18 +12,34 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + if not isinstance(body.get("reuse-runtime", False), bool): + return ApiResponse(status=400, body={"error": "reuse-runtime must be a boolean"}) image = body.get("image", "") + manifest = body.get("load-manifest", "") bitstream = body.get("bitstream", "") - if not image or not bitstream: + if not all(isinstance(value, str) for value in (image, manifest, bitstream)): + return ApiResponse(status=400, body={"error": "image, load-manifest and bitstream must be strings"}) + if bool(image) == bool(manifest) or not bitstream: return ApiResponse( status=400, body={ "success": False, "failure": True, "returncode": 400, - "message": "image and bitstream parameters are required", + "message": "exactly one of image and load-manifest, and bitstream, are required", }, ) + if manifest and body.get("diff", False): + return ApiResponse(status=400, body={"error": "load-manifest cannot use single-ELF diff"}) + try: + validate_cold_load_case(bitstream) + except (ValueError, OSError, TypeError) as error: + return ApiResponse(status=400, body={"error": str(error)}) + if body.get("reuse-runtime", False): + try: + validate_runtime_reuse(bitstream, bool(body.get("diff", False))) + except (ValueError, OSError) as error: + return ApiResponse(status=400, body={"error": str(error)}) await ctx.enqueue({ "topic": "bebop.p2e.runworkload", "data": {**body, "_trace_id": ctx.trace_id}, diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index 8637b3cc7..48a9f8809 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -27,6 +27,10 @@ from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from resolve_image import resolve_image +from steps.bebop.p2e.scripts.runtime_case import validate_runtime_reuse, validate_cold_load_case +from steps.bebop.performance_report import performance_report +from utils.reports import source_context +from pathlib import Path config = { "name": "bebop-p2e-runworkload", @@ -68,6 +72,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" image_name = input_data.get("image", "") + manifest_path = input_data.get("load-manifest", "") bitstream = input_data.get("bitstream", "") multi_fpga = bool(input_data.get("multi-fpga", False)) fpga_location = ( @@ -130,30 +135,29 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return wave = True try: - image_path = resolve_image(bbdir, image_name, chip) - except ValueError as error: + if bool(image_name) == bool(manifest_path): + raise ValueError("exactly one of image and load-manifest is required") + if manifest_path and diff: + raise ValueError("load-manifest cannot use single-ELF diff") + if manifest_path: + manifest_path = str(Path(manifest_path).resolve(strict=True)) + if not Path(manifest_path).is_file(): + raise ValueError("load-manifest must be a regular JSON file") + image_path = None + elf_path = None + image_name = Path(manifest_path).stem + else: + image_path = resolve_image(bbdir, image_name, chip) + image_base = os.path.splitext(image_path)[0] + elf_path = f"{image_base}.elf" if os.path.isfile(f"{image_base}.elf") else image_base + if diff and not os.path.isfile(elf_path): + raise ValueError(f"workload ELF for P2E DiffTest not found: {elf_path}") + validate_cold_load_case(bitstream) + except (ValueError, OSError, TypeError) as error: ctx.logger.error(str(error)) - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": "image_not_found", "image": image_name}, - trace_id=origin_tid, - ) - return - image_base = os.path.splitext(image_path)[0] - elf_path = ( - f"{image_base}.elf" if os.path.isfile(f"{image_base}.elf") else image_base - ) - if diff and (not os.path.isfile(elf_path)): - ctx.logger.error(f"workload ELF for P2E DiffTest not found: {elf_path}") - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": "workload_elf_not_found", "elf": elf_path}, - trace_id=origin_tid, - ) + await check_result(ctx, 1, continue_run=False, + extra_fields={"error": "invalid_cold_load_inputs", "message": str(error)}, + trace_id=origin_tid) return if not bitstream or not os.path.isfile(bitstream): ctx.logger.error(f"bitstream .bit file not found: {bitstream}") @@ -214,54 +218,64 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: build_env["CARGO_TARGET_DIR"] = os.path.join( bebop_dir, "target", f"{chip}-p2e-diff" ) - build_cmd = shlex.join( - [ - "nix", - "develop", - "--ignore-env", - "--keep-env-var", - "HOME", - "--keep-env-var", - "ALL_PROXY", - "--keep-env-var", - "CARGO_TARGET_DIR", - "--keep-env-var", - "OUT_PATH", - "-c", - "cargo", - "build", - "--release", - "--manifest-path", - manifest, - "--bin", - "bebop", - "--features", - "p2e,bemu" if diff else "p2e", - ] - ) - ctx.logger.info("Building bebop p2e runtime for the selected case...") - build_result = await stream_run_logger_async( - cmd=build_cmd, - logger=ctx.logger, - cwd=bebop_dir, - stdout_prefix="bebop p2e runtime build", - stderr_prefix="bebop p2e runtime build", - env=build_env, - ) - if build_result.returncode != 0: - await check_result( - ctx, - build_result.returncode, - continue_run=False, - extra_fields={"task": "runtime_build", "build_dir": build_dir}, - trace_id=origin_tid, + reuse_runtime = input_data.get("reuse-runtime", False) + if reuse_runtime: + validate_runtime_reuse(bitstream, diff) + ctx.logger.info(f"Reusing selected P2E runtime without rebuilding: {bebop_p2e_path}") + else: + build_cmd = shlex.join( + [ + "nix", + "develop", + "--ignore-env", + "--keep-env-var", + "HOME", + "--keep-env-var", + "ALL_PROXY", + "--keep-env-var", + "CARGO_TARGET_DIR", + "--keep-env-var", + "OUT_PATH", + "-c", + "cargo", + "build", + "--release", + "--manifest-path", + manifest, + "--bin", + "bebop", + "--features", + "p2e,bemu" if diff else "p2e", + ] ) - return - target_release = os.path.join(build_env["CARGO_TARGET_DIR"], "release") - staged_runtime = f"{bebop_p2e_path}.new" - shutil.copy2(os.path.join(target_release, "bebop"), staged_runtime) - os.replace(staged_runtime, bebop_p2e_path) - run_cmd = f'nix develop -c "{bebop_p2e_path}" run p2e --image="{image_path}" --bitstream="{bitstream}" --log-dir="{run_log}" --fpga-location="{fpga_location}"' + ctx.logger.info("Building bebop p2e runtime for the selected case...") + build_result = await stream_run_logger_async( + cmd=build_cmd, + logger=ctx.logger, + cwd=bebop_dir, + stdout_prefix="bebop p2e runtime build", + stderr_prefix="bebop p2e runtime build", + env=build_env, + ) + if build_result.returncode != 0: + await check_result( + ctx, + build_result.returncode, + continue_run=False, + extra_fields={"task": "runtime_build", "build_dir": build_dir}, + trace_id=origin_tid, + ) + return + target_release = os.path.join(build_env["CARGO_TARGET_DIR"], "release") + staged_runtime = f"{bebop_p2e_path}.new" + shutil.copy2(os.path.join(target_release, "bebop"), staged_runtime) + os.replace(staged_runtime, bebop_p2e_path) + run_args = ["nix", "develop", "-c", "env", "-u", "LD_LIBRARY_PATH", bebop_p2e_path, + "run", "p2e", "--bitstream", bitstream, "--log-dir", run_log, + "--fpga-location", fpga_location] + run_args += ["--load-manifest", manifest_path] if manifest_path else ["--image", image_path] + run_cmd = f"cd {shlex.quote(bbdir)} && {shlex.join(run_args)}" + report_context = source_context(bbdir, chip) if multi_fpga: run_cmd += " --multi-fpga" if wave: @@ -283,19 +297,27 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="bebop p2e runworkload", env=run_env, ) + uart_logs = sorted(Path(run_log).glob("uart_hart_*.log")) + try: + report_path = performance_report(bbdir, report_context, origin_tid, "p2e", run_log, "\n".join(path.read_text() for path in uart_logs), run_result.returncode) + except (OSError, ValueError, KeyError) as error: + await check_result(ctx, 1, extra_fields={"task": "report", "log_dir": run_log, "error": str(error)}, trace_id=origin_tid) + return await check_result( ctx, run_result.returncode, continue_run=False, extra_fields={ "task": "runworkload", + "report_path": report_path, "image": image_path, + "load_manifest": manifest_path or None, "bitstream": bitstream, "build_dir": build_dir, "log_dir": run_log, "fpga_location": fpga_location, "bdb_trace": os.path.join(run_log, "bdb.ndjson"), - "uart_log": os.path.join(run_log, "uart.log"), + "uart_logs": [str(path) for path in uart_logs], "bank_diff": os.path.join(run_log, "diff.ndjson") if diff else None, "diff": diff, "timestamp": timestamp, diff --git a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py index 5ee40cda7..ec2c71c2a 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py @@ -197,16 +197,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "diff": diff, "workload_toml": workload_toml, } - if input_data.get("from_regression_check"): - if run_result.returncode != 0: - await check_result( - ctx, - run_result.returncode, - continue_run=False, - extra_fields={**extra_fields, "error": "pk_tests_failed"}, - trace_id=origin_tid, - ) - return + await check_result( ctx, run_result.returncode, diff --git a/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py b/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py new file mode 100644 index 000000000..13d83b108 --- /dev/null +++ b/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py @@ -0,0 +1,33 @@ +from pathlib import Path +import os + + +def validate_cold_load_case(bitstream: str) -> Path: + bit = Path(bitstream).resolve(strict=True) + if not bit.is_file(): + raise ValueError(f"P2E bitstream is not a file: {bit}") + case = bit.parent.parent + marker = case / "p2e-cold-load.cap" + if marker.read_text() != "p2e-cold-load-v1\n": + raise ValueError(f"P2E case does not support cold loading: {case}") + return case + + +def validate_runtime_reuse(bitstream: str, diff: bool) -> Path: + bit = Path(bitstream).resolve() + case = bit.parent.parent + executable = case / "bebop-p2e" + files = [bit, executable, case / "vvacDir/runtimeDir/rtcfg", + case / "vvacDir/runtimeDir/lib/lib_arm/libvCtb.so", + case / "p2e_trace_mode"] + missing = [str(path) for path in files if not path.is_file()] + if missing: + raise ValueError(f"Reusable P2E case is incomplete: {missing}") + if not os.access(executable, os.X_OK): + raise ValueError(f"Reusable P2E runtime is not executable: {executable}") + mode = (case / "p2e_trace_mode").read_text().strip() + if mode not in ("none", "btrace", "btrace_nb_v1"): + raise ValueError(f"Unknown reusable P2E trace mode: {mode}") + if diff and mode == "none": + raise ValueError("Reusable P2E case does not support diff") + return executable diff --git a/bbdev/api/steps/bebop/performance_report.py b/bbdev/api/steps/bebop/performance_report.py new file mode 100644 index 000000000..402a4bce9 --- /dev/null +++ b/bbdev/api/steps/bebop/performance_report.py @@ -0,0 +1,223 @@ +import hashlib +import importlib.util +import json +import re +import tomllib +from pathlib import Path + +from utils.reports import new_report, attachment, finish_report +from utils.report_schema import number, nonempty, CATEGORIES +from bisect import bisect_right + + +def parse_trace(data, categories): + events = data["traceEvents"] + groups = {} + for event in events: + if event.get("ph") != "X" or event.get("cat") != "buddy.trace": + continue + args = event["args"] + path = args["id_path"] + if not isinstance(path, list) or not path: + raise ValueError("trace id_path must be a non-empty list") + for part in path: + number(part, "trace id", integer=True) + identity = "-".join(str(int(part)) for part in path) + start = number(args["start_cycle"], "start_cycle", integer=True) + end = number(args["end_cycle"], "end_cycle", integer=True) + elapsed = number(args["elapsed_cycle"], "elapsed_cycle", integer=True) + if end < start or end - start != elapsed or event["dur"] != elapsed: + raise ValueError(f"inconsistent cycle interval for {identity}") + name = nonempty(event["name"], "operator name") + category = categories.get(name, "Other") + if category not in CATEGORIES: + raise ValueError(f"unknown operator category: {category}") + groups.setdefault(identity, []).append( + { + "id": identity, + "name": name, + "category": category, + "cycles": int(elapsed), + "start_cycle": int(start), + "level": len(path) - 1, + } + ) + if not groups: + raise ValueError("no buddy.trace cycle intervals found") + starts = {} + for identity, records in groups.items(): + records.sort(key=lambda op: op["start_cycle"]) + starts[identity] = [op["start_cycle"] for op in records] + for previous, current in zip(records, records[1:]): + if previous["name"] != current["name"]: + raise ValueError(f"trace id has inconsistent names: {identity}") + if previous["start_cycle"] + previous["cycles"] > current["start_cycle"]: + raise ValueError(f"overlapping calls of trace {identity}") + for identity, records in groups.items(): + parts = identity.split("-") + if len(parts) > 1: + parent_id = "-".join(parts[:-1]) + if parent_id not in groups: + raise ValueError(f"missing parent for trace {identity}") + for op in records: + index = bisect_right(starts[parent_id], op["start_cycle"]) - 1 + if index < 0: + raise ValueError(f"nested trace outside parent: {identity}") + parent = groups[parent_id][index] + if ( + op["start_cycle"] + op["cycles"] + > parent["start_cycle"] + parent["cycles"] + ): + raise ValueError(f"nested trace outside parent: {identity}") + start = min(records[0]["start_cycle"] for records in groups.values()) + end = max( + records[-1]["start_cycle"] + records[-1]["cycles"] + for records in groups.values() + ) + if start == end: + raise ValueError("trace span must be positive") + operators = [] + for records in groups.values(): + op = dict(records[0]) + op["cycles"] = sum(record["cycles"] for record in records) + op["calls"] = len(records) + op["start_cycle"] -= start + operators.append(op) + return int(end - start), operators + + +def performance_report(repo, context, trace_id, backend, log, stdout, returncode): + repo, log = Path(repo), Path(log) + records = [ + json.loads(line.split("@BBPERF ", 1)[1]) + for line in stdout.splitlines() + if "@BBPERF " in line + ] + if not records: + report = new_report(context, f"{backend}.execution", trace_id, backend) + report["execution"] = { + "status": "pass" if returncode == 0 else "fail", + "returncode": returncode, + "performance": "not_collected", + } + return finish_report(report, log / "report") + models = {record["model"] for record in records} + if len(models) != 1: + raise ValueError("one model is required per performance run") + if len({record["unit"] for record in records}) != 1: + raise ValueError("benchmark throughput units differ") + model = models.pop() + if not re.fullmatch(r"[A-Za-z0-9_-]+", model): + raise ValueError("invalid benchmark model name") + recipe = repo / f"examples/models/{context['chip']}/{model}" + settings = tomllib.loads((recipe / "configs/report.toml").read_text()) + report = new_report(context, f"{backend}.performance", trace_id, backend) + output = log / "report" + report["clock_hz"] = settings.get(backend, {}).get("clock_hz") + expected = settings.get("expected_classes") + correct = samples = elapsed = cycles = 0 + for index, record in enumerate(records): + number(record["samples"], "samples", positive=True, integer=True) + number(record["elapsed_ns"], "elapsed_ns", positive=True, integer=True) + number(record["cycles"], "cycles", positive=True, integer=True) + samples += record["samples"] + elapsed += record["elapsed_ns"] + cycles += record["cycles"] + if "correct" in record: + number(record["correct"], "correct samples", integer=True) + if record["correct"] > record["samples"]: + raise ValueError("correct samples exceed sample count") + correct += record["correct"] + elif expected is not None: + if len(expected) != len(records): + raise ValueError("reference labels do not match the benchmark inputs") + if sum(record["predictions"].values()) != record["samples"]: + raise ValueError("prediction counts do not match sample count") + correct += record["predictions"].get(str(expected[index]), 0) + accuracy = ( + correct / samples + if expected is not None or all("correct" in record for record in records) + else None + ) + build = repo / f"stack/models/build/{context['chip']}/{model}" + workload = hashlib.sha256() + for file in sorted((build / "artifact").rglob("*")): + if file.is_file(): + workload.update(file.relative_to(build / "artifact").as_posix().encode()) + workload.update(file.read_bytes()) + run_config = recipe / "configs/running-param.toml" + inputs = tomllib.loads(run_config.read_text())["inputs"] + dataset = hashlib.sha256() + for value in inputs: + if isinstance(value, str) and (run_config.parent / value).is_file(): + dataset.update((run_config.parent / value).read_bytes()) + else: + dataset.update(json.dumps(value, sort_keys=True).encode()) + operators = {} + all_events = [] + trace_span = 0 + trace_roots = sorted( + { + file.parent.parent + for file in log.rglob("trace-*.txt") + if file.parent.name == "cycle" + } + ) + if trace_roots: + spec = importlib.util.spec_from_file_location( + "bbdev_perfetto", repo / "stack/serving/trace/perfetto.py" + ) + perfetto = importlib.util.module_from_spec(spec) + spec.loader.exec_module(perfetto) + metadata = [build / f"models/{model}/subgraph0.trace.mlir"] + for index, root in enumerate(trace_roots): + data = perfetto.build_perfetto(root, recipe / "trace/trace.toml", metadata) + span, ops = parse_trace(data, settings.get("operator_categories", {})) + trace_span += span + for op in ops: + if op["id"] in operators: + operators[op["id"]]["cycles"] += op["cycles"] + operators[op["id"]]["calls"] += op["calls"] + else: + operators[op["id"]] = op + for event in data["traceEvents"]: + event["pid"] = index + all_events.extend(data["traceEvents"]) + result = { + "name": model, + "status": "pass" if returncode == 0 else "fail", + "workload_hash": workload.hexdigest(), + "dataset_hash": dataset.hexdigest(), + "cycles": trace_span if trace_roots else cycles, + "accuracy": accuracy, + "accuracy_metric": records[0].get( + "accuracy_metric", settings["accuracy_metric"] + ), + "latency_ms": elapsed / samples / 1e6, + "throughput": {"value": samples * 1e9 / elapsed, "unit": records[0]["unit"]}, + "error": None, + "trace_url": None, + "operators": list(operators.values()), + "benchmark": { + "samples": samples, + "elapsed_ns": elapsed, + "scope": "model_execution", + }, + } + if returncode != 0: + result.update( + cycles=None, + accuracy=None, + latency_ms=None, + throughput=None, + operators=[], + error=f"model exited with {returncode}", + ) + elif all_events: + path = log / "perfetto.json" + path.write_text(json.dumps({"traceEvents": all_events}) + "\n") + result["trace_url"] = attachment( + report, output, path, f"model-{model}-perfetto.json" + ) + report["models"].append(result) + return finish_report(report, output) diff --git a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py index e092718bd..ce30cc603 100644 --- a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py +++ b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py @@ -29,8 +29,8 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: }, ) data = {"chip": chip, "diff": bool(body.get("diff", False))} - if body.get("output_dir"): - data["output_dir"] = body["output_dir"] + if body.get("output-dir"): + data["output_dir"] = body["output-dir"] await ctx.enqueue( { "topic": "bebop.verilator.verilog", diff --git a/bbdev/api/steps/bebop/verilator/04_sim_api.step.py b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py index 837659b50..989ac97fb 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_api.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py @@ -30,8 +30,11 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) + data = {**body, "chip": chip, "_trace_id": ctx.trace_id} + if "vsrc-dir" in data: + data["vsrc_dir"] = data.pop("vsrc-dir") await ctx.enqueue({ "topic": "bebop.verilator.sim", - "data": {**body, "chip": chip, "_trace_id": ctx.trace_id}, + "data": data, }) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index 6deb926da..da26044cd 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -7,6 +7,7 @@ import json import os import shlex +import subprocess import sys from datetime import datetime from motia import FlowContext, queue @@ -172,18 +173,36 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_args += f" --{trace_name}" run_cmd = f"{shlex.quote(bebop_bin)} run verilator --elf={shlex.quote(binary_path)} --log-dir={shlex.quote(run_log)}{(' --diff' if diff else '')}{wave_arg}{trace_args}" if diff: + dram_library = f"{bbdir}/result/lib/libdramsim3.so" + dram_rpath = subprocess.check_output( + ["patchelf", "--print-rpath", dram_library], text=True + ).strip().split(os.pathsep) + cpp_libraries = [ + os.path.join(path, "libstdc++.so.6") for path in dram_rpath + if os.path.isfile(os.path.join(path, "libstdc++.so.6")) + ] + if len(cpp_libraries) != 1: + raise ValueError(f"DRAMSim requires one C++ runtime in its RPATH: {dram_rpath}") preload = os.pathsep.join( ( path for path in ( - f"{bbdir}/result/lib/libdramsim3.so", + dram_library, + cpp_libraries[0], os.environ.get("LD_PRELOAD", ""), ) if path ) ) - run_inner = f"cd {shlex.quote(build_dir)} && export LD_PRELOAD={shlex.quote(preload)} && exec {run_cmd}" - run_cmd = f"nix develop -c sh -c {shlex.quote(run_inner)}" + # VVAC embeds its older C++ runtime in the executable's search path. + # DRAMSim is built by the active Nix toolchain and needs that toolchain's + # runtime. Resolve it from DRAMSim and preload only the simulator process. + run_inner = ( + f"cd {shlex.quote(build_dir)} && " + f"export LD_PRELOAD={shlex.quote(preload)} && " + f"exec {run_cmd}" + ) + run_cmd = f"cd {shlex.quote(bbdir)} && nix develop -c sh -c {shlex.quote(run_inner)}" ctx.logger.info(f"Running bebop verilator (diff={diff}): {run_cmd}") stdout_prefix = "bebop verilator difftest" if diff else "bebop verilator" stderr_prefix = stdout_prefix diff --git a/bbdev/api/steps/config/scripts/2_parameter_derivation.py b/bbdev/api/steps/config/scripts/2_parameter_derivation.py index 285ba09ef..d9ee14852 100644 --- a/bbdev/api/steps/config/scripts/2_parameter_derivation.py +++ b/bbdev/api/steps/config/scripts/2_parameter_derivation.py @@ -9,6 +9,10 @@ from pathlib import Path from typing import Any +if str(Path(__file__).resolve().parent) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parent)) +from ball_normalize import normalize_ball_domain + def _die(msg: str) -> None: raise ValueError(msg) @@ -106,45 +110,24 @@ def _tile_cores(tile: dict[str, Any]) -> list[dict[str, Any]]: def iter_topology_tiles(topo: dict[str, Any]) -> list[dict[str, Any]]: - tiles = topo.get("tiles") - if isinstance(tiles, list) and tiles: - out: list[dict[str, Any]] = [] - seen: set[int] = set() - for i, tile in enumerate(tiles): - if not isinstance(tile, dict): - _die("tiles entry must be a table") - tile_id = _require_id(tile, "tile_id", f"tiles[{i}]") - if tile_id in seen: - _die(f"duplicate tile_id {tile_id}") - seen.add(tile_id) - out.append(tile) - expected = set(range(len(out))) - if seen != expected: - _die(f"tile_id must be exactly 0..{len(out)-1}, got {sorted(seen)}") - out.sort(key=lambda t: t["tile_id"]) - return out - - template = topo.get("tileTemplate") - if isinstance(template, dict): - count = template.get("count") - if not isinstance(count, int) or count < 1: - _die("[tileTemplate].count must be a positive int") - tile_ids = _require_id_list(template, "tile_ids", count, "tileTemplate") - expected = set(range(count)) - if set(tile_ids) != expected: - _die(f"tileTemplate.tile_ids must be exactly 0..{count-1}, got {tile_ids}") - base = { - k: v - for k, v in template.items() - if k not in ("count", "tile_ids") - } - out = [] - for tile_id in sorted(tile_ids): - tile = dict(base) - tile["tile_id"] = tile_id - out.append(tile) - return out - _die("topology must define [[tiles]] or [tileTemplate]") + """The main tile is tile 0; homogeneous compute tiles follow as tiles 1..count.""" + for legacy in ("tiles", "tileTemplate", "main_core"): + if legacy in topo: + _die(f"[{legacy}] is replaced by [main_tile] and [compute_tiles]") + main = topo.get("main_tile") + if not isinstance(main, dict): + _die("topology must define [main_tile]") + out = [dict(main, tile_id=0, kind="main")] + compute = topo.get("compute_tiles") + if compute is not None: + if not isinstance(compute, dict): + _die("[compute_tiles] must be a table") + count = compute.get("count") + if type(count) is not int or count < 1: + _die("[compute_tiles].count must be a positive int") + base = {k: v for k, v in compute.items() if k != "count"} + out += [dict(base, tile_id=i + 1, kind="compute") for i in range(count)] + return out def iter_cores(topo: dict[str, Any]) -> Iterator[dict[str, Any]]: @@ -170,24 +153,13 @@ def _tile_signature(tile: dict[str, Any]) -> tuple[tuple[int, str, str], ...]: def _assert_isomorphic_tiles(topo: dict[str, Any]) -> None: tiles = iter_topology_tiles(topo) - if not tiles: - _die("no tiles") - ref = _tile_signature(tiles[0]) - if not ref: - _die("tile has no cores") - for tile in tiles[1:]: - sig = _tile_signature(tile) - if sig != ref: - _die( - f"tiles must be isomorphic; tile_id={tile['tile_id']} " - f"signature {sig} != tile_id={tiles[0]['tile_id']} {ref}" - ) - - -def hart_id(tile_id: int, core_id: int, cores_per_tile: int) -> int: - if core_id < 0 or core_id >= cores_per_tile: - _die(f"core_id {core_id} out of range for cores_per_tile={cores_per_tile}") - return tile_id * cores_per_tile + core_id + for tile in tiles: + if not _tile_signature(tile): + _die(f"tile_id={tile['tile_id']} has no cores") + compute = [t for t in tiles if t["kind"] == "compute"] + for tile in compute[1:]: + if _tile_signature(tile) != _tile_signature(compute[0]): + _die(f"compute tiles must be isomorphic; tile_id={tile['tile_id']} differs from tile_id=1") def unique_cores(topo: dict[str, Any]) -> list[str]: @@ -271,7 +243,7 @@ def _config_path(repo: Path, config: object) -> str: def _ball_num(core: dict) -> int: - bd = core.get("balldomain") + bd = normalize_ball_domain(core) if not isinstance(bd, dict): return 0 ball_num = bd.get("ballNum") @@ -283,13 +255,9 @@ def _ball_num(core: dict) -> int: def _n_tiles(topo: dict[str, Any]) -> int: - top = topo.get("top") - if not isinstance(top, dict): - _die("topology missing [top]") - n = top.get("nTiles") - if not isinstance(n, int) or n < 1: - _die("[top].nTiles must be a positive int") - return n + if "top" in topo: + _die("[top] is retired: tile counts and hart layout follow [main_tile] and [compute_tiles]") + return len(iter_topology_tiles(topo)) def _derive_cores(repo: Path, topo: dict[str, Any]) -> list[dict[str, Any]]: @@ -325,7 +293,7 @@ def _derive_cores(repo: Path, topo: dict[str, Any]) -> list[dict[str, Any]]: "ball_num": ball_num, } if ball_num > 0: - bd = core.get("balldomain") + bd = normalize_ball_domain(core) if not isinstance(bd, dict): _die(f"{pkg}: missing balldomain") mappings = bd.get("ballIdMappings") @@ -341,8 +309,9 @@ def _derive_cores(repo: Path, topo: dict[str, Any]) -> list[dict[str, Any]]: entry["mappings"].append( { "ball_id": mapping.get("ballId"), - "ball_dir": _ball_dir(ball_class), - "config_path": _config_path(repo, mapping.get("config")), + "ball_dir": "" if mapping.get("builtin") else _ball_dir(ball_class), + "config_path": "" if mapping.get("builtin") else _config_path(repo, mapping.get("config")), + "builtin": mapping.get("builtin", ""), } ) out.append(entry) @@ -361,6 +330,10 @@ def _derive_tiles( _die("tile missing _file") tile_cores = _tile_cores(tile) n = len(tile_cores) + if "controllerCore" in tile: + _die(f"tile_id={tile_id}: controllerCore is retired; a compute tile's controller is core 0") + if tile["kind"] == "compute" and n < 2: + _die(f"tile_id={tile_id}: a compute tile needs its controller and at least one worker") shared = tile.get("sharedMem") vbc = 0 if isinstance(shared, dict): @@ -382,6 +355,7 @@ def _derive_tiles( placements.append( { "tile_id": tile_id, + "kind": tile["kind"], "path": _repo_rel(repo, tile_path), "core_indices": indices, "cores_per_tile": n, @@ -389,6 +363,8 @@ def _derive_tiles( "mem_ball_channel_num": mem_ball_channel_num, } ) + if tile["kind"] == "compute": + placements[-1]["controller_core_index"] = indices[0] offset += n return placements @@ -397,7 +373,7 @@ def _bemu_balls(repo: Path, topo: dict[str, Any]) -> list[dict[str, str]]: seen: set[str] = set() balls: list[dict[str, str]] = [] for core in iter_cores(topo): - bd = core.get("balldomain") + bd = normalize_ball_domain(core) if not isinstance(bd, dict): continue mappings = bd.get("ballIdMappings") @@ -411,6 +387,7 @@ def _bemu_balls(repo: Path, topo: dict[str, Any]) -> list[dict[str, str]]: and isinstance(m.get("ballId"), int) and isinstance(m.get("ballClass"), str) } + builtin_ids = {m["ballId"] for m in mappings if m.get("builtin")} pkg = core_pkg(core.get("_file", "")) or "core" for entry in isa: if not isinstance(entry, dict): @@ -421,6 +398,8 @@ def _bemu_balls(repo: Path, topo: dict[str, Any]) -> list[dict[str, str]]: _die(f"ballISA entry must have funct7 and bid: {entry!r}") if funct7 in {0, 1, 16, 32, 33, 34, 35}: continue + if bid in builtin_ids: + continue ball_class = bid_to_class.get(bid) if not ball_class: _die( @@ -446,16 +425,19 @@ def _bemu_balls(repo: Path, topo: dict[str, Any]) -> list[dict[str, str]]: def _bemu_paths(repo: Path, chip: str, topo: dict[str, Any]) -> tuple[str, int]: main = repo / "examples" / "chips" / chip / "emu" / "src" / "main.rs" - tiles = tile_files(topo) - if main.is_file(): - if len(tiles) != 1: - _die(f"chip {chip}: emu requires exactly one tile file, got {tiles}") - return f"examples/chips/{chip}/emu/src/main.rs", 0 - return "", 0 + if not main.is_file(): + return "", 0 + tiles = iter_topology_tiles(topo) + for kind in ("main", "compute"): + files = {t.get("_file") for t in tiles if t["kind"] == kind} + if len(files) > 1: + _die(f"chip {chip}: emu requires one {kind} tile file, got {sorted(files)}") + compute = [t["tile_id"] for t in tiles if t["kind"] == "compute"] + return f"examples/chips/{chip}/emu/src/main.rs", compute[0] if compute else 0 def _ball_ctest_dirs(repo: Path, core: dict[str, Any]) -> list[str]: - bd = core.get("balldomain") + bd = normalize_ball_domain(core) if not isinstance(bd, dict): _die("core missing balldomain for ctest dirs") mappings = bd.get("ballIdMappings") @@ -465,6 +447,8 @@ def _ball_ctest_dirs(repo: Path, core: dict[str, Any]) -> list[str]: for mapping in mappings: if not isinstance(mapping, dict): _die("ballIdMappings entry must be a table") + if mapping.get("builtin"): + continue ball_class = mapping.get("ballClass") if not isinstance(ball_class, str): _die("ballIdMappings entry missing ballClass") @@ -519,44 +503,41 @@ def _derive_targets( return targets -def _derive_harts( - tiles: list[dict[str, Any]], cores: list[dict[str, Any]] -) -> list[dict[str, Any]]: - if not tiles: - _die("no tiles") - cores_per_tile = tiles[0]["cores_per_tile"] - for tile in tiles[1:]: - if tile["cores_per_tile"] != cores_per_tile: - _die("tiles must be isomorphic (cores_per_tile mismatch)") - +def _derive_harts(tiles: list[dict[str, Any]], cores: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Main tile cores are harts 0..m-1 and the only Linux-visible harts. Compute tile t + (1..N) has its controller at m+t-1; workers follow from m+N, tile by tile.""" + main = tiles[0] + compute = tiles[1:] + m = main["cores_per_tile"] + workers_per_tile = compute[0]["cores_per_tile"] - 1 if compute else 0 harts: list[dict[str, Any]] = [] for tile in tiles: tile_id = tile["tile_id"] for local, core_index in enumerate(tile["core_indices"]): inst = cores[core_index] - core_id = inst["core_id"] - if core_id != local: - _die( - f"tile_id={tile_id}: core_id={core_id} != sorted slot {local}" - ) - hid = hart_id(tile_id, core_id, cores_per_tile) + if inst["core_id"] != local: + _die(f"tile_id={tile_id}: core_id={inst['core_id']} != sorted slot {local}") + if tile["kind"] == "main": + hid = local + elif local == 0: + hid = m + tile_id - 1 + else: + hid = m + len(compute) + (tile_id - 1) * workers_per_tile + local - 1 harts.append( { "hart_id": hid, "tile_id": tile_id, - "core_id": core_id, + "core_id": local, "core_index": core_index, + "visible": tile["kind"] == "main", "target": _target_name(inst["role"], inst["pkg"]), "pkg": inst["pkg"], "role": inst["role"], } ) harts.sort(key=lambda h: h["hart_id"]) - ids = [h["hart_id"] for h in harts] - if ids != list(range(len(harts))): - _die(f"hart_id must be exactly 0..{len(harts)-1}, got {ids}") - if len(harts) != len(cores): - _die(f"hart count {len(harts)} != core count {len(cores)}") + if [h["hart_id"] for h in harts] != list(range(len(cores))): + _die("chip hart_ids must be exactly 0..total_harts-1") return harts @@ -591,8 +572,6 @@ def derive(data: dict[str, Any], repo: Path, chip: str) -> dict[str, Any]: targets = _derive_targets(repo, topo, cores) harts = _derive_harts(tiles, cores) n_tiles = _n_tiles(topo) - if len(tiles) != n_tiles: - _die(f"{chip}: [top].nTiles={n_tiles} but got {len(tiles)} tile(s)") chip_main, tile_index = _bemu_paths(repo, chip, topo) diff --git a/bbdev/api/steps/config/scripts/3_json2proto.py b/bbdev/api/steps/config/scripts/3_json2proto.py index a34121ff3..6fa41fc08 100644 --- a/bbdev/api/steps/config/scripts/3_json2proto.py +++ b/bbdev/api/steps/config/scripts/3_json2proto.py @@ -13,6 +13,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import chip_pb2 as pb # noqa: E402 +from ball_normalize import normalize_ball_domain def _load_derive(): @@ -82,9 +83,14 @@ def _fill_ball(msg: pb.BallDomain, d: dict[str, Any], bbdir: Path) -> None: e.ball_id = m["ballId"] e.ball_name = m["ballName"] e.ball_class = m["ballClass"] - e.ball_dir = _ball_dir(m["ballClass"]) - e.config_path = _rel(bbdir, m["config"]["_file"]) - for key, value in m["config"]["ball"].items(): + if m.get("builtin"): + e.builtin = m["builtin"] + params = m["ball_params"] + else: + e.ball_dir = _ball_dir(m["ballClass"]) + e.config_path = _rel(bbdir, m["config"]["_file"]) + params = m["config"]["ball"] + for key, value in params.items(): e.ball_params[key] = str(value) e.in_bw = m["inBW"] e.out_bw = m["outBW"] @@ -161,14 +167,16 @@ def _fill_frontend(msg: pb.FrontendConfig, d: dict[str, Any], bbdir: Path) -> No msg.sub_rob_depth = d["subRobDepth"] -def _fill_gp(msg: pb.GpDomainConfig, d: dict[str, Any], bbdir: Path) -> None: +def _fill_rvv(msg: pb.RvvConfig, d: dict[str, Any], bbdir: Path) -> None: + if type(d.get("enable", False)) is not bool: + raise ValueError("rvv.enable must be a boolean") + msg.enable = d.get("enable", False) msg.source_path = _rel(bbdir, d["_file"]) msg.lane_number = d["laneNumber"] - msg.chaining_size = d["chainingSize"] msg.v_len = d["vLen"] - msg.d_len = d["dLen"] msg.e_len = d["eLen"] - msg.lane_scale = d["laneScale"] + msg.i_buf_words = d["iBufWords"] + msg.memory_ports = d["memoryPorts"] def _fill_tile_params(msg: pb.TileParamConfig, d: dict[str, Any]) -> None: @@ -181,46 +189,48 @@ def _fill_tile_params(msg: pb.TileParamConfig, d: dict[str, Any]) -> None: msg.n_pmps = d["nPMPs"] +def _fill_cpu(msg: pb.CpuConfig, cpu: dict[str, Any], context: str, bbdir: Path) -> str: + kind = _check_cpu_kind(cpu, context) + msg.kind = kind + msg.source_path = _rel(bbdir, cpu["_file"]) + if kind == "rocket": + _fill_rocket(msg.rocket, cpu["config"]) + else: + _fill_boom(msg.boom, cpu["config"]) + return kind + + def _fill_core(ci: pb.CoreInstance, raw: dict[str, Any], meta: dict[str, Any], bbdir: Path) -> None: ci.index = meta["index"] ci.role = meta["role"] ci.pkg = meta["pkg"] ci.config_path = meta["config_path"] ci.balldomain_base_dir = meta["balldomain_base_dir"] - cpu = raw["cpu"] - kind = _check_cpu_kind(cpu, meta["pkg"]) - ci.cpu.kind = kind - ci.cpu.source_path = _rel(bbdir, cpu["_file"]) - if "balldomain" in raw: - _fill_ball(ci.balldomain, raw["balldomain"], bbdir) + kind = _fill_cpu(ci.cpu, raw["cpu"], meta["pkg"], bbdir) + domain = normalize_ball_domain(raw) + if domain is not None: + _fill_ball(ci.balldomain, domain, bbdir) if "memdomain" in raw: _fill_mem(ci.mem, raw["memdomain"], bbdir) - if kind == "rocket": - _fill_rocket(ci.cpu.rocket, cpu["config"]) - else: - _fill_boom(ci.cpu.boom, cpu["config"]) - bd = raw.get("balldomain") + if kind == "boom": + bd = domain if isinstance(bd, dict) and bd.get("ballNum", 0): raise ValueError(f"{meta['pkg']}: kind=boom forbids balldomain.ballNum > 0") if "frontend" in raw: _fill_frontend(ci.frontend, raw["frontend"], bbdir) - if "gpdomain" in raw: - _fill_gp(ci.gp_domain, raw["gpdomain"], bbdir) + if ci.balldomain.ball_num > 0: + _fill_rvv(ci.rvv, raw["rvv"], bbdir) def _fill_tile(tp: pb.TilePlacement, meta: dict[str, Any], proto: dict[str, Any]) -> None: tp.path = meta["path"] + tp.kind = pb.TILE_KIND_MAIN if meta["kind"] == "main" else pb.TILE_KIND_COMPUTE tp.virtual_bank_count = meta["virtual_bank_count"] tp.core_indices.extend(meta["core_indices"]) + if "controller_core_index" in meta: + tp.controller_core_index = meta["controller_core_index"] tp.mem_ball_channel_num = meta["mem_ball_channel_num"] _fill_tile_params(tp.param, proto) - dc = proto["privateDCache"] - tp.private_dcache.enable = dc["enable"] - tp.private_dcache.ways = dc["ways"] - tp.private_dcache.capacity_kb = dc["capacityKB"] - tp.private_dcache.write_bytes = dc["writeBytes"] - tp.private_dcache.port_factor = dc["portFactor"] - tp.private_dcache.mem_cycles = dc["memCycles"] sm = proto["sharedMem"] tp.shared_mem.enable = sm["enable"] tp.shared_mem.entries = sm["entries"] @@ -251,8 +261,10 @@ def fill_chip(config: dict[str, Any], derived: dict[str, Any], bbdir: Path) -> p b.includes.extend(derived["includes"]) sims = config["sims"] - b.mill.verilator_config = sims["verilator"] - b.mill.p2e_config = sims["p2e"] + if "verilator" in sims: + b.mill.verilator_config = sims["verilator"] + if "p2e" in sims: + b.mill.p2e_config = sims["p2e"] raw_cores = _raw_core_list(config["designs"]) meta_cores = derived["cores"] @@ -260,8 +272,11 @@ def fill_chip(config: dict[str, Any], derived: dict[str, Any], bbdir: Path) -> p raise ValueError( f"core count mismatch: config={len(raw_cores)} derived={len(meta_cores)}" ) - for raw, meta in zip(raw_cores, meta_cores): - _fill_core(b.cores.add(), raw, meta, bbdir) + hart_ids = {hart["core_index"]: hart["hart_id"] for hart in derived["harts"]} + for index, (raw, meta) in enumerate(zip(raw_cores, meta_cores)): + core = b.cores.add() + _fill_core(core, raw, meta, bbdir) + core.hart_id = hart_ids[index] raw_tiles = _derive.iter_topology_tiles(config["designs"]) for raw, meta in zip(raw_tiles, derived["tiles"], strict=True): diff --git a/bbdev/api/steps/config/scripts/4_install.py b/bbdev/api/steps/config/scripts/4_install.py index 2c729126f..cfe205e3d 100644 --- a/bbdev/api/steps/config/scripts/4_install.py +++ b/bbdev/api/steps/config/scripts/4_install.py @@ -25,7 +25,7 @@ def _emit_dispatch(chip: pb.Chip, bbdir: Path, out: Path) -> None: balls = list(chip.bemu.balls) if not balls: out.write_text( - 'use crate::inst::instruction::ExecContext;\n\npub fn execute_known(\n _ball_class: &str,\n _funct: u32,\n _xs1: u64,\n _xs2: u64,\n _ctx: &mut ExecContext,\n) -> u64 {\n panic!("no BEMU ball implementation")\n}\n\npub fn cycles_after_issue(_ball_class: &str, _funct: u32, _xs1: u64, _xs2: u64) -> u64 {\n panic!("no BEMU ball latency implementation")\n}\n', + 'use crate::inst::instruction::ExecContext;\n\npub fn execute_known(\n _ball_class: &str,\n _funct: u32,\n _xs1: u64,\n _xs2: u64,\n _ctx: &mut ExecContext,\n) -> u64 {\n panic!("no BEMU ball implementation")\n}\n\npub fn cycles_after_issue(_ball_class: &str, _funct: u32, _xs1: u64, _xs2: u64) -> u64 { panic!("no BEMU latency contract") }\n', encoding="utf-8", ) return @@ -69,13 +69,10 @@ def chain(fn: str, ctx: bool, panic_msg: str) -> list[str]: ) lines += ["}", ""] lines += [ - "pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> u64 {" + "pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> u64 {", ] - lines += chain( - "cycles_after_issue", - False, - "no BEMU ball latency implementation for ballClass={ball_class} funct7={funct}", - ) + lines += chain("cycles_after_issue", False, + "no BEMU latency contract for ballClass={ball_class} funct7={funct}") lines += ["}", ""] out.write_text("\n".join(lines) + "\n", encoding="utf-8") @@ -110,7 +107,7 @@ def install_bebop(gen: Path) -> Path: bebop.mkdir(parents=True, exist_ok=True) manifest = bebop / "Cargo.toml" manifest.write_text( - '[workspace]\nresolver = "2"\n\n[package]\nname = "bebop"\nversion = "0.1.0"\nedition = "2021"\nbuild = "../../../../../bebop/build.rs"\n\n[[bin]]\nname = "bebop"\npath = "../../../../../bebop/src/main.rs"\n\n[[test]]\nname = "test_verilator"\npath = "../../../../../bebop/tests/test_verilator.rs"\nharness = false\nrequired-features = ["verilator"]\n\n[[test]]\nname = "test_p2e"\npath = "../../../../../bebop/tests/test_p2e.rs"\nharness = false\nrequired-features = ["p2e"]\n\n[features]\ndefault = []\nverilator = ["dep:bebop-verilator"]\np2e = ["dep:bebop-p2e", "bebop-bemu?/p2e"]\nbemu = ["dep:bebop-bemu"]\n\n[dependencies]\nbebop-verilator = { path = "../../../../../bebop/src/nodes/verilator", optional = true }\nbebop-p2e = { path = "../../../../../bebop/src/nodes/p2e", optional = true }\nbebop-bemu = { path = "../../configs/generated/bemu", optional = true }\nbebop-dasm = { path = "../../../../../bebop/src/nodes/lib/dasm" }\nbebop-bank-hash = { path = "../../../../../bebop/src/nodes/lib/bank-hash" }\nbebop-bemu-profile = { path = "../../../../../bebop/src/nodes/lib/bemu-profile" }\nbebop-fd-redirect = { path = "../../../../../bebop/src/nodes/lib/fd-redirect" }\nbebop-rtl-trace = { path = "../../../../../bebop/src/nodes/lib/rtl-trace" }\nbebop-uart = { path = "../../../../../bebop/src/nodes/lib/uart" }\nclap = { version = "4", features = ["derive"] }\nlibc = "0.2"\nlog = "0.4"\nenv_logger = "0.11"\nnix = { version = "0.29", features = ["fs", "mman", "signal", "process"] }\ntoml = "0.8"\ncamino = "1.1"\nsnafu = "0.8"\nserde = { version = "1", features = ["derive"] }\nserde_json = "1"\nduct = "0.13"\n\n[dev-dependencies]\nlibtest-mimic = "0.8"\nassert_cmd = "2"\nwalkdir = "2"\nchrono = { version = "0.4", default-features = false, features = ["clock"] }\n', + '[workspace]\nresolver = "2"\n\n[package]\nname = "bebop"\nversion = "0.1.0"\nedition = "2021"\nbuild = "../../../../../bebop/build.rs"\n\n[[bin]]\nname = "bebop"\npath = "../../../../../bebop/src/main.rs"\n\n[[test]]\nname = "test_verilator"\npath = "../../../../../bebop/tests/test_verilator.rs"\nharness = false\nrequired-features = ["verilator"]\n\n[[test]]\nname = "test_p2e"\npath = "../../../../../bebop/tests/test_p2e.rs"\nharness = false\nrequired-features = ["p2e"]\n\n[features]\ndefault = []\nverilator = ["dep:bebop-verilator"]\np2e = ["dep:bebop-p2e", "bebop-bemu?/p2e"]\nbemu = ["dep:bebop-bemu", "bebop-p2e?/diff"]\n\n[dependencies]\nbebop-verilator = { path = "../../../../../bebop/src/nodes/verilator", optional = true }\nbebop-p2e = { path = "../../../../../bebop/src/nodes/p2e", optional = true }\nbebop-bemu = { path = "../../configs/generated/bemu", optional = true }\nbebop-dasm = { path = "../../../../../bebop/src/nodes/lib/dasm" }\nbebop-bank-hash = { path = "../../../../../bebop/src/nodes/lib/bank-hash" }\nbebop-bemu-profile = { path = "../../../../../bebop/src/nodes/lib/bemu-profile" }\nbebop-fd-redirect = { path = "../../../../../bebop/src/nodes/lib/fd-redirect" }\nbebop-rtl-trace = { path = "../../../../../bebop/src/nodes/lib/rtl-trace" }\nbebop-uart = { path = "../../../../../bebop/src/nodes/lib/uart" }\nclap = { version = "4", features = ["derive"] }\nlibc = "0.2"\nlog = "0.4"\nenv_logger = "0.11"\nnix = { version = "0.29", features = ["fs", "mman", "signal", "process"] }\ntoml = "0.8"\ncamino = "1.1"\nsnafu = "0.8"\nserde = { version = "1", features = ["derive"] }\nserde_json = "1"\nduct = "0.13"\n\n[dev-dependencies]\nlibtest-mimic = "0.8"\nassert_cmd = "2"\nwalkdir = "2"\nchrono = { version = "0.4", default-features = false, features = ["clock"] }\n', encoding="utf-8", ) return manifest diff --git a/bbdev/api/steps/config/scripts/ball_normalize.py b/bbdev/api/steps/config/scripts/ball_normalize.py new file mode 100644 index 000000000..3f665f3b2 --- /dev/null +++ b/bbdev/api/steps/config/scripts/ball_normalize.py @@ -0,0 +1,71 @@ +"""Normalize external Ball mappings and the explicitly enabled RVV kernel.""" + +from copy import deepcopy +from typing import Any + + +def normalize_ball_domain(core: dict[str, Any]) -> dict[str, Any] | None: + rvv = core.get("rvv") + enabled = False + if rvv is not None: + if not isinstance(rvv, dict) or ("enable" in rvv and type(rvv["enable"]) is not bool): + raise ValueError("rvv.enable must be an explicit boolean") + enabled = rvv.get("enable", False) + raw = core.get("balldomain") + if raw is None and not enabled: + return None + if not isinstance(raw, dict): + raise ValueError("RVV kernel requires a Ball domain") + domain = deepcopy(raw) + mappings = domain.get("ballIdMappings") + isa = domain.get("ballISA") + if not isinstance(mappings, list) or not isinstance(isa, list): + raise ValueError("Ball domain requires ballIdMappings and ballISA lists") + if any(not isinstance(entry, dict) for entry in isa): + raise ValueError("Ball ISA entries must be tables") + if type(domain.get("ballNum")) is not int or domain["ballNum"] != len(mappings): + raise ValueError("ballNum must equal the number of mappings") + kernel = None + for index, mapping in enumerate(mappings): + if not isinstance(mapping, dict) or type(mapping.get("ballId")) is not int or mapping["ballId"] != index: + raise ValueError("Ball IDs must enumerate mappings without gaps or reordering") + if "builtin" in mapping: + if mapping["builtin"] != "kernel": + raise ValueError("unknown builtin Ball") + if kernel is not None or index != len(mappings) - 1: + raise ValueError("builtin kernel must occur once at the final Ball ID") + kernel = mapping + continue + klass = mapping.get("ballClass") + config = mapping.get("config") + if not isinstance(klass, str) or not klass.startswith("examples.balls."): + raise ValueError("external Ball class must be under examples.balls") + if not isinstance(config, dict) or not isinstance(config.get("_file"), str) or not isinstance(config.get("ball"), dict): + raise ValueError("external Ball requires a resolved config with [ball] parameters") + if any(type(mapping.get(key)) is not int or mapping[key] <= 0 for key in ("inBW", "outBW")): + raise ValueError("external Ball inBW/outBW must be positive integers") + if not enabled: + if kernel is not None: + raise ValueError("builtin kernel requires rvv.enable=true") + return domain + params = {} + for key in ("laneNumber", "vLen", "eLen", "iBufWords", "memoryPorts"): + value = rvv.get(key) + if type(value) is not int or value <= 0: + raise ValueError(f"rvv.{key} must be a positive integer") + params[key] = str(value) + bid = len(mappings) if kernel is None else len(mappings) - 1 + expected = {"ballId": bid, "ballName": "kernel", "ballClass": "framework.balldomain.kernel.KernelBall", + "builtin": "kernel", "inBW": 0, "outBW": 0, "ball_params": params} + instruction = {"mnemonic": "RUN_KERNEL", "funct7": 15, "bid": bid} + conflicts = [entry for entry in isa if entry.get("funct7") == 15 or entry.get("mnemonic") == "RUN_KERNEL"] + if kernel is not None: + if kernel != expected or conflicts != [instruction]: + raise ValueError("builtin kernel mapping or RUN_KERNEL ISA differs from RVV configuration") + else: + if conflicts: + raise ValueError("RUN_KERNEL/funct7=15 conflicts with an existing ISA entry") + mappings.append(expected) + isa.append(instruction) + domain["ballNum"] += 1 + return domain diff --git a/bbdev/api/steps/config/scripts/proto/chip.proto b/bbdev/api/steps/config/scripts/proto/chip.proto index 90e3ad00e..0c209b683 100644 --- a/bbdev/api/steps/config/scripts/proto/chip.proto +++ b/bbdev/api/steps/config/scripts/proto/chip.proto @@ -69,6 +69,7 @@ message BallIdMapping { uint32 mmio_read_bw = 8; uint32 mmio_write_bw = 9; map ball_params = 10; + string builtin = 11; } message BallIsaEntry { @@ -158,14 +159,14 @@ message FrontendConfig { string source_path = 9; } -message GpDomainConfig { +message RvvConfig { uint32 lane_number = 1; - uint32 chaining_size = 2; - uint32 v_len = 3; - uint32 d_len = 4; - uint32 e_len = 5; - uint32 lane_scale = 6; - string source_path = 7; + uint32 v_len = 2; + uint32 e_len = 3; + uint32 i_buf_words = 4; + uint32 memory_ports = 5; + string source_path = 6; + optional bool enable = 7; } message TileParamConfig { @@ -197,7 +198,8 @@ message CoreInstance { string balldomain_base_dir = 7; CpuConfig cpu = 8; FrontendConfig frontend = 9; - GpDomainConfig gp_domain = 10; + optional uint32 hart_id = 10; + RvvConfig rvv = 11; } message BemuBall { @@ -221,13 +223,11 @@ message SharedMemConfig { uint32 mem_cycles = 6; } -message PrivateDCacheConfig { - bool enable = 1; - uint32 ways = 2; - uint32 capacity_kb = 3; - uint32 write_bytes = 4; - uint32 port_factor = 5; - uint32 mem_cycles = 6; +// Tile 0 is the main tile: every core is Linux visible. Compute tiles follow and are +// homogeneous: core 0 controls the tile and the other cores are hidden task workers. +enum TileKind { + TILE_KIND_MAIN = 0; + TILE_KIND_COMPUTE = 1; } message TilePlacement { @@ -235,9 +235,12 @@ message TilePlacement { uint32 virtual_bank_count = 2; repeated uint32 core_indices = 3; uint32 mem_ball_channel_num = 4; - PrivateDCacheConfig private_dcache = 5; + reserved 5; SharedMemConfig shared_mem = 6; TileParamConfig param = 7; + // Set for compute tiles only. + optional uint32 controller_core_index = 8; + TileKind kind = 9; } message WorkloadConfig { @@ -269,4 +272,5 @@ message Chip { WorkloadConfig workload = 11; string chip_path = 12; repeated CompilerProfile profiles = 13; + reserved 14; } diff --git a/bbdev/api/steps/dc/01_area_event.step.py b/bbdev/api/steps/dc/01_area_event.step.py index 2705f4a37..cbd165031 100644 --- a/bbdev/api/steps/dc/01_area_event.step.py +++ b/bbdev/api/steps/dc/01_area_event.step.py @@ -20,7 +20,9 @@ from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger_async -from tapeout import clock_period_ns, get_tapeout_contract, write_run_tcl +from tapeout import get_tapeout_contract, write_run_tcl +from steps.dc.scripts.report import area_report +from utils.reports import source_context config = { "name": "dc-area", @@ -120,6 +122,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result(ctx, 1, continue_run=False, extra_fields={"task": "dc", "error": str(exc)}, trace_id=origin_tid) return script_path = contract.dc_script + report_context = source_context(get_buckyball_path(), contract.chip) dc_log = os.path.join(analysis_dir, "dc_shell.log") ctx.logger.info(f"Running chip-owned DC synthesis for {contract.chip}, top {top_module}: {script_path}") result = await stream_run_logger_async( @@ -145,43 +148,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "tapeout_dir": str(contract.root), "replace_manifest": input_data.get("replace_manifest"), } - - if result.returncode == 0 and input_data.get("from_regression_area_power"): - reg_scripts = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "regression", "scripts") - ) - if reg_scripts not in sys.path: - sys.path.insert(0, reg_scripts) - from dc_area import area_mm2_from_rpt - from result import merge_metrics - - rpt = os.path.join(report_dir, "area.rpt") - if not os.path.isfile(rpt): - await check_result( - ctx, 1, continue_run=False, - extra_fields={**extra_fields, "error": f"missing area.rpt: {rpt}", "area_rpt": rpt}, - trace_id=origin_tid, - ) - return + if result.returncode == 0: try: - with open(rpt, encoding="utf-8") as handle: - area = area_mm2_from_rpt(handle.read()) - except ValueError as exc: - await check_result( - ctx, 1, continue_run=False, - extra_fields={**extra_fields, "error": str(exc), "area_rpt": rpt}, - trace_id=origin_tid, - ) + extra_fields["report_path"] = area_report(report_context, origin_tid, report_dir, contract) + except (OSError, ValueError) as error: + await check_result(ctx, 1, extra_fields={**extra_fields, "error": str(error)}, trace_id=origin_tid) return - freq = 1000.0 / clock_period_ns(contract) - merge_metrics(get_buckyball_path(), area=area, freq=freq) - extra_fields["area"] = area - extra_fields["freq"] = freq - await check_result( - ctx, result.returncode, continue_run=False, - extra_fields=extra_fields, trace_id=origin_tid, - ) - return await check_result( ctx, diff --git a/bbdev/api/steps/dc/scripts/report.py b/bbdev/api/steps/dc/scripts/report.py new file mode 100644 index 000000000..76d5bc8c5 --- /dev/null +++ b/bbdev/api/steps/dc/scripts/report.py @@ -0,0 +1,31 @@ +import hashlib +import re +from pathlib import Path + +from utils.reports import new_report, attachment, finish_report + + +def area_report(context, trace_id, report_dir, contract): + report_dir = Path(report_dir) + output = report_dir.parent / "report" + report = new_report(context, "dc.area", trace_id, "dc") + values = re.findall( + r"Total cell area:\s*([0-9.eE+-]+)", (report_dir / "area.rpt").read_text() + ) + if len(values) != 1: + raise ValueError("DC report must contain one total cell area") + constraints = (report_dir / "constraint.rpt").read_text() + if "Report : constraint" not in constraints: + raise ValueError("invalid DC constraint report") + report["area"] = { + "status": "pass", + "cell_um2": float(values[0]), + "scope": contract.top_module, + "conditions": f"DC · {Path(contract.target_library).name} · SDC {hashlib.sha256(contract.constraints_sdc.read_bytes()).hexdigest()}", + "constraints_met": "(VIOLATED)" not in constraints, + "report_url": attachment(report, output, report_dir / "area.rpt", "area.rpt"), + "constraints_report_url": attachment( + report, output, report_dir / "constraint.rpt", "area-constraints.rpt" + ), + } + return finish_report(report, output) diff --git a/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk b/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk index 8ce70ab26..a7d2d7e26 100644 --- a/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk +++ b/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk @@ -14,7 +14,9 @@ DESIGN ?= FireSim # Override to use our custom config package TARGET_CONFIG_PACKAGE ?= sims.firesim -TARGET_CONFIG ?= FireSimBuckyballToyConfig +ifndef TARGET_CONFIG +$(error TARGET_CONFIG must name an available FireSim target) +endif # These guide chisel elaboration of simulation components by MIDAS, # including models and widgets. diff --git a/bbdev/api/steps/kernel/01_build_api.step.py b/bbdev/api/steps/kernel/01_build_api.step.py index c5fd5c894..bdec4672b 100644 --- a/bbdev/api/steps/kernel/01_build_api.step.py +++ b/bbdev/api/steps/kernel/01_build_api.step.py @@ -11,5 +11,13 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + memory = body.get("guest-memory-mib", 512) + if isinstance(memory, bool) or not str(memory).isdigit() or not 1 <= int(memory) <= 16384: + return ApiResponse(status=400, body={"error": "guest-memory-mib must be an integer in 1..16384 (P2E DDR capacity)"}) + storage = body.get("model-storage", "initramfs") + if storage not in ("initramfs", "pmem"): + return ApiResponse(status=400, body={"error": "model-storage must be initramfs or pmem"}) + if storage == "pmem" and (not body.get("model") or int(memory) >= 16384): + return ApiResponse(status=400, body={"error": "pmem requires --model and guest-memory-mib < 16384"}) await ctx.enqueue({"topic": "kernel.build", "data": {**body, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index eb7d56f46..7760f9cb2 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -35,6 +35,8 @@ def hart_count_params(input_data: dict) -> dict: "model", "chip", "interactive", + "guest-memory-mib", + "model-storage", "_trace_id", } unknown = sorted(k for k in input_data if k not in allowed) @@ -105,22 +107,26 @@ def kernel_build_dir( model: str = "", chip: str = "", interactive: bool = False, + guest_memory_mib: int = 512, + model_storage: str = "initramfs", ) -> str: visible = hart_params["visible"] total = hart_params["total"] - suffix = "" + suffix = "" if guest_memory_mib == 512 else f"-mem{guest_memory_mib}M" if chip: suffix += f"-chip-{chip}" if model: suffix += f"-model-{model}" if interactive: suffix += "-interactive" + if model_storage == "pmem": + suffix += "-pmem" if visible == 64 and total == 64: return os.path.join(bbdir, "bb-tests", "build", f"kernel{suffix}") return os.path.join(bbdir, "bb-tests", "build", f"kernel-v{visible}-t{total}{suffix}") -def fw_payload_name(hart_params: dict, model: str = "", chip: str = "") -> str: +def fw_payload_name(hart_params: dict, model: str = "", chip: str = "", guest_memory_mib: int = 512, model_storage: str = "initramfs") -> str: visible = hart_params["visible"] total = hart_params["total"] name = "fw_payload" @@ -130,6 +136,10 @@ def fw_payload_name(hart_params: dict, model: str = "", chip: str = "") -> str: name = f"{name}-{model}" elif chip: name = f"{name}-{chip}-pk" + if guest_memory_mib != 512: + name += f"-mem{guest_memory_mib}M" + if model_storage == "pmem": + name += "-pmem" return name @@ -139,6 +149,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: kernel_src = os.path.join(bbdir, "bb-tests", "workloads", "lib", "kernel") try: + memory_value = input_data.get("guest-memory-mib", 512) + if isinstance(memory_value, bool) or not str(memory_value).isdigit() or not 1 <= int(memory_value) <= 16384: + raise ValueError("guest-memory-mib must be an integer in 1..16384 (P2E DDR capacity)") + guest_memory_mib = int(memory_value) + model_storage = input_data.get("model-storage", "initramfs") + if model_storage not in ("initramfs", "pmem"): + raise ValueError("model-storage must be initramfs or pmem") + if model_storage == "pmem" and (not input_data.get("model") or guest_memory_mib >= 16384): + raise ValueError("pmem requires --model and guest-memory-mib < 16384") hart_params = hart_count_params(input_data) model = kernel_model(input_data) if model and not input_data.get("chip"): @@ -155,7 +174,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: output_dir = os.path.join(output_dir, chip) os.makedirs(output_dir, exist_ok=True) kernel_build = kernel_build_dir( - bbdir, hart_params, model, chip, interactive=interactive + bbdir, hart_params, model, chip, interactive=interactive, guest_memory_mib=guest_memory_mib, model_storage=model_storage ) interactive_arg = "ON" if interactive else "OFF" cmake_model_name = model @@ -166,6 +185,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"-DBUCKYBALL_VISIBLE_HART_COUNT={hart_params['visible']} " f"-DBUCKYBALL_TOTAL_HART_COUNT={hart_params['total']} " f"-DBUCKYBALL_HIDDEN_HART_BASE={hart_params['hidden_base']} " + f"-DBUCKYBALL_GUEST_MEMORY_MIB={guest_memory_mib} " + f"-DBUCKYBALL_MODEL_STORAGE={model_storage} " f"-DBUCKYBALL_KERNEL_MODEL={cmake_model_name} " f"-DBUCKYBALL_KERNEL_CHIP={chip} " f"-DBUCKYBALL_KERNEL_INTERACTIVE={interactive_arg} " @@ -209,7 +230,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return # Convert fw_payload.bin to hex for P2E memory backdoor - payload_name = fw_payload_name(hart_params, cmake_model_name if model else "", chip) + payload_name = fw_payload_name(hart_params, cmake_model_name if model else "", chip, guest_memory_mib, model_storage) fw_payload_bin = os.path.join(output_dir, f"{payload_name}.bin") fw_payload_hex = os.path.join(output_dir, f"{payload_name}.hex") fw_payload_elf = os.path.join(output_dir, f"{payload_name}.elf") @@ -240,4 +261,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) return - await check_result(ctx, 0, continue_run=False, trace_id=origin_tid) + storage_fields = {"model_storage": model_storage} + if model_storage == "pmem": + manifest = os.path.join(output_dir, f"{payload_name}.load.json") + if not os.path.isfile(manifest): + raise ValueError(f"Missing pmem load manifest: {manifest}") + storage_fields["load_manifest"] = manifest + await check_result(ctx, 0, continue_run=False, extra_fields=storage_fields, trace_id=origin_tid) diff --git a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py index 4dcef5e1b..74478fba0 100644 --- a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py +++ b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py @@ -27,7 +27,7 @@ "description": "Generate P2E Verilog via mill", "flows": ["bebop"], "triggers": [queue("bebop.p2e.verilog")], - "enqueues": ["bebop.p2e.buildbitstream"], + "enqueues": [], } @@ -70,11 +70,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"Using P2E chip: {chip}") ctx.logger.info(f"Using P2E output directory: {build_dir}") - from_reg = bool(input_data.get("from_regression_buildbitstream")) await check_result( ctx, returncode, - continue_run=from_reg and returncode == 0, + continue_run=False, extra_fields={ "task": "verilog", "chip": chip, @@ -83,15 +82,3 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: }, trace_id=origin_tid, ) - if from_reg and returncode == 0: - await ctx.enqueue( - { - "topic": "bebop.p2e.buildbitstream", - "data": { - **input_data, - "vsrc_dir": build_dir, - "from_regression_buildbitstream": True, - "_trace_id": origin_tid, - }, - } - ) diff --git a/bbdev/api/steps/regression/00_buildbitstream_api.step.py b/bbdev/api/steps/regression/00_buildbitstream_api.step.py deleted file mode 100644 index 06b697f9d..000000000 --- a/bbdev/api/steps/regression/00_buildbitstream_api.step.py +++ /dev/null @@ -1,37 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - -from utils.event_common import require_chip - -config = { - "name": "regression-buildbitstream-api", - "description": "Chain p2e verilog and buildbitstream for regression", - "flows": ["regression"], - "triggers": [api("POST", "/regression/buildbitstream")], - "enqueues": ["regression.buildbitstream"], -} - - -async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = req.body or {} - try: - chip = require_chip(body) - except ValueError as exc: - return ApiResponse(status=400, body={"error": str(exc)}) - - data = { - "chip": chip, - "_trace_id": ctx.trace_id, - } - output_dir = body.get("output_dir") - if output_dir is None: - output_dir = body.get("output-dir") - if output_dir is not None: - if not isinstance(output_dir, str) or not output_dir: - return ApiResponse( - status=400, - body={"error": "output_dir must be a non-empty path"}, - ) - data["output_dir"] = output_dir - - await ctx.enqueue({"topic": "regression.buildbitstream", "data": data}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/regression/00_buildbitstream_event.step.py b/bbdev/api/steps/regression/00_buildbitstream_event.step.py deleted file mode 100644 index b314f037e..000000000 --- a/bbdev/api/steps/regression/00_buildbitstream_event.step.py +++ /dev/null @@ -1,56 +0,0 @@ -import os -import sys - -from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.event_common import require_chip -from utils.event_common import check_result, get_origin_trace_id - -config = { - "name": "regression-buildbitstream", - "description": "Enqueue p2e verilog then buildbitstream for regression", - "flows": ["regression"], - "triggers": [queue("regression.buildbitstream")], - "enqueues": ["bebop.p2e.verilog"], -} - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - try: - chip = require_chip(input_data) - except ValueError as exc: - ctx.logger.error(str(exc)) - extra = {"error": "missing_chip"} if str(exc).startswith("Missing required parameter: --chip") else {"error": str(exc)} - await check_result( - ctx, 1, continue_run=False, - extra_fields=extra, - trace_id=origin_tid, - ) - return - - await check_result( - ctx, - 0, - continue_run=True, - extra_fields={ - "task": "regression.buildbitstream", - "chip": chip, - }, - trace_id=origin_tid, - ) - - data = { - "chip": chip, - "from_regression_buildbitstream": True, - "_trace_id": origin_tid, - } - output_dir = input_data.get("output_dir") or input_data.get("output-dir") - if output_dir: - data["output_dir"] = output_dir - - await ctx.enqueue({"topic": "bebop.p2e.verilog", "data": data}) diff --git a/bbdev/api/steps/regression/01_check_api.step.py b/bbdev/api/steps/regression/01_check_api.step.py deleted file mode 100644 index db1019125..000000000 --- a/bbdev/api/steps/regression/01_check_api.step.py +++ /dev/null @@ -1,48 +0,0 @@ -import os - -from motia import ApiRequest, ApiResponse, FlowContext, api - -config = { - "name": "regression-check-api", - "description": "Run p2e pk-tests and record accuracy for regression", - "flows": ["regression"], - "triggers": [api("POST", "/regression/check")], - "enqueues": ["regression.check"], -} - - -def _req_str(body: dict, key: str): - value = body.get(key) - if not isinstance(value, str) or not value or value == "None": - return None - return value - - -async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = req.body or {} - chip = _req_str(body, "chip") - if chip is None: - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --chip must be specified"}, - ) - bitstream = _req_str(body, "bitstream") - if bitstream is None: - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --bitstream must be specified"}, - ) - if not os.path.isfile(bitstream): - return ApiResponse( - status=400, - body={"error": f"bitstream file not found: {bitstream}"}, - ) - - data = { - "chip": chip, - "bitstream": bitstream, - "_trace_id": ctx.trace_id, - } - - await ctx.enqueue({"topic": "regression.check", "data": data}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/regression/01_check_event.step.py b/bbdev/api/steps/regression/01_check_event.step.py deleted file mode 100644 index 3c50076bb..000000000 --- a/bbdev/api/steps/regression/01_check_event.step.py +++ /dev/null @@ -1,70 +0,0 @@ -import os -import sys - -from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.event_common import check_result, get_origin_trace_id - -config = { - "name": "regression-check", - "description": "Enqueue p2e pk-tests batch for regression accuracy", - "flows": ["regression"], - "triggers": [queue("regression.check")], - "enqueues": ["bebop.p2e.batch"], -} - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - chip = input_data.get("chip") - bitstream = input_data.get("bitstream") - if not isinstance(chip, str) or not chip or chip == "None": - ctx.logger.error("Missing required parameter: chip") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, - trace_id=origin_tid, - ) - return - if not isinstance(bitstream, str) or not bitstream or bitstream == "None": - ctx.logger.error("Missing required parameter: bitstream") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "missing_bitstream"}, - trace_id=origin_tid, - ) - return - if not os.path.isfile(bitstream): - ctx.logger.error(f"bitstream file not found: {bitstream}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "bitstream_not_found", "bitstream": bitstream}, - trace_id=origin_tid, - ) - return - - await check_result( - ctx, - 0, - continue_run=True, - extra_fields={ - "task": "regression.check", - "chip": chip, - "bitstream": bitstream, - }, - trace_id=origin_tid, - ) - - data = { - "chip": chip, - "bitstream": bitstream, - "test": "pk-tests", - "from_regression_check": True, - "_trace_id": origin_tid, - } - - await ctx.enqueue({"topic": "bebop.p2e.batch", "data": data}) diff --git a/bbdev/api/steps/regression/02_eval_performance_api.step.py b/bbdev/api/steps/regression/02_eval_performance_api.step.py deleted file mode 100644 index b4bcaf987..000000000 --- a/bbdev/api/steps/regression/02_eval_performance_api.step.py +++ /dev/null @@ -1,48 +0,0 @@ -import os - -from motia import ApiRequest, ApiResponse, FlowContext, api - -config = { - "name": "regression-eval-performance-api", - "description": "Run p2e model perfetto latency for regression", - "flows": ["regression"], - "triggers": [api("POST", "/regression/eval-performance")], - "enqueues": ["regression.eval-performance"], -} - - -def _req_str(body: dict, key: str): - value = body.get(key) - if not isinstance(value, str) or not value or value == "None": - return None - return value - - -async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = req.body or {} - chip = _req_str(body, "chip") - if chip is None: - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --chip must be specified"}, - ) - bitstream = _req_str(body, "bitstream") - if bitstream is None: - return ApiResponse( - status=400, - body={"error": "Missing required parameter: --bitstream must be specified"}, - ) - if not os.path.isfile(bitstream): - return ApiResponse( - status=400, - body={"error": f"bitstream file not found: {bitstream}"}, - ) - - data = { - "chip": chip, - "bitstream": bitstream, - "_trace_id": ctx.trace_id, - } - - await ctx.enqueue({"topic": "regression.eval-performance", "data": data}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/regression/02_eval_performance_event.step.py b/bbdev/api/steps/regression/02_eval_performance_event.step.py deleted file mode 100644 index 7f784e557..000000000 --- a/bbdev/api/steps/regression/02_eval_performance_event.step.py +++ /dev/null @@ -1,375 +0,0 @@ -"""regression --eval-performance event. - -Per model (synchronous shells inside one regression event): - 1. model build --chip --model - 2. kernel build --model with dataset packed from the model build dataset directory - -> fw_payload-.hex; /init runs binary with --dataset - 3. p2e runworkload; parse per-model accuracy from uart (top1= / map=) - 4. require non-empty log_dir/trace/cycle/; run perfetto.py; cycles_i - -After all models: cycles = mean(cycles_i); accuracy = mean(accuracy_i); -merge_metrics(cycles=..., accuracy=..., models=[...]). -Any model failure aborts the whole stage (no partial average). -""" -import importlib.util -import json -import os -import shlex -import sys -from datetime import datetime -from pathlib import Path - -from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import bebop_cargo_env, get_buckyball_path, log_dir -from utils.stream_run import stream_run_logger_async -from utils.event_common import check_result, get_origin_trace_id -from utils.model import model_layout_name - -from steps.model.scripts.build import build_model - -regression_scripts = os.path.join(os.path.dirname(__file__), "scripts") -if regression_scripts not in sys.path: - sys.path.insert(0, regression_scripts) -from model_layout import perfetto_inputs -from models_toml import load_eval_models -from accuracy import accuracy_from_uart, load_eval_accuracy, mean_accuracy -from perfetto_latency import e2e_cycles_from_perfetto, mean_cycles -from result import merge_metrics - -config = { - "name": "regression-eval-performance", - "description": "Per-model p2e perfetto cycles for regression", - "flows": ["regression"], - "triggers": [queue("regression.eval-performance")], - "enqueues": [], -} - - -def _load_sibling(rel): - path = os.path.normpath(os.path.join(os.path.dirname(__file__), rel)) - name = os.path.basename(path).replace(".", "_") - spec = importlib.util.spec_from_file_location(name, path) - if spec is None or spec.loader is None: - raise RuntimeError(f"cannot load sibling step: {path}") - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - - -_kernel = _load_sibling("../kernel/01_build_event.step.py") -_p2e = _load_sibling("../bebop/p2e/03_runworkload_event.step.py") - - -def _fail(ctx, origin_tid, error, **extra): - fields = {"task": "regression.eval-performance", "error": error} - fields.update(extra) - return check_result(ctx, 1, continue_run=False, extra_fields=fields, trace_id=origin_tid) - - -def _model_build(bbdir, chip, model, logger, task_scope): - build_model(bbdir, chip, model_layout_name(model.lower()), - ctrace=True, logger=logger, task_scope=task_scope) - - -def _kernel_build_cmds(bbdir, chip, model, dataset=""): - model = model_layout_name(model) - kernel_src = os.path.join(bbdir, "bb-tests", "workloads", "lib", "kernel") - hart_params = {"visible": 64, "total": 64, "hidden_base": 64} - kernel_build = _kernel.kernel_build_dir( - bbdir, hart_params, model=model, chip=chip - ) - ds_arg = "" - if dataset: - ds_arg = f" -DBUCKYBALL_MODEL_DATASET={shlex.quote(dataset)}" - configure = ( - f"cmake -B {kernel_build} -S {kernel_src} " - f"-DBUCKYBALL_VISIBLE_HART_COUNT=64 " - f"-DBUCKYBALL_TOTAL_HART_COUNT=64 " - f"-DBUCKYBALL_HIDDEN_HART_BASE=64 " - f"-DBUCKYBALL_KERNEL_MODEL={model} " - f"-DBUCKYBALL_KERNEL_CHIP={chip} " - f"-DBUCKYBALL_KERNEL_INTERACTIVE=OFF" - f"{ds_arg}" - ) - build = f"cmake --build {kernel_build} --target kernel-build" - payload = _kernel.fw_payload_name(hart_params, model=model) - kernel_output = os.path.join( - bbdir, "bb-tests", "output", "kernel", chip - ) - fw_bin = os.path.join(kernel_output, f"{payload}.bin") - fw_hex = os.path.join(kernel_output, f"{payload}.hex") - return configure, build, fw_bin, fw_hex - - -def _p2e_run_cmds(bbdir, bitstream, image_name, chip, input_data): - image_path = os.path.join( - bbdir, "bb-tests", "output", "kernel", chip, f"{image_name}.hex" - ) - bitstream = os.path.abspath(bitstream) - build_dir = os.path.dirname(os.path.dirname(bitstream)) - if not os.path.isdir(build_dir): - raise FileNotFoundError( - f"P2E build case not found for bitstream: {build_dir}" - ) - multi_fpga = bool(input_data.get("multi-fpga", False)) - if not multi_fpga and _p2e.case_uses_multi_fpga(build_dir): - multi_fpga = True - timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - run_log = log_dir(bbdir, chip, "p2e", timestamp, "p2e", image_name, input_data.get("vsrc_dir")) - os.makedirs(run_log, exist_ok=True) - runtime_lib_dir = os.path.join( - build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm" - ) - runtime = os.path.join(build_dir, "bebop-p2e") - artifacts = [ - os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg"), - os.path.join(runtime_lib_dir, "libvCtb.so"), - runtime, - ] - missing = [path for path in artifacts if not os.path.isfile(path)] - if missing: - raise FileNotFoundError(f"P2E runtime artifacts missing: {missing}") - - run_args = [ - runtime, - "run", "p2e", - "--image", image_path, - "--bitstream", bitstream, - "--log-dir", run_log, - ] - if multi_fpga: - run_args.append("--multi-fpga") - run_env = {**os.environ, **bebop_cargo_env(bbdir, chip)} - return shlex.join(run_args), run_env, f"{bbdir}/bebop", run_log - - -def _perfetto_cmd(trace_dir, trace_toml, mlir_files): - args = [str(trace_dir), str(trace_toml)] - for mlir in mlir_files: - args += ["--mlir", str(mlir)] - return ( - f"{sys.executable} {shlex.quote(str(Path(get_buckyball_path()) / 'stack/serving/trace/perfetto.py'))} " - + " ".join(shlex.quote(a) for a in args) - ) - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - bbdir = get_buckyball_path() - - chip = input_data.get("chip") - if not isinstance(chip, str) or not chip or chip == "None": - ctx.logger.error("Missing required parameter: chip") - await _fail(ctx, origin_tid, "missing_chip") - return - bitstream = input_data.get("bitstream") - if not isinstance(bitstream, str) or not bitstream or bitstream == "None": - ctx.logger.error("Missing required parameter: bitstream") - await _fail(ctx, origin_tid, "missing_bitstream") - return - if not os.path.isfile(bitstream): - ctx.logger.error(f"bitstream file not found: {bitstream}") - await _fail(ctx, origin_tid, "bitstream_not_found", bitstream=bitstream) - return - - try: - models = load_eval_models(chip, bbdir) - acc_specs = load_eval_accuracy(chip, bbdir, models) - except ValueError as e: - ctx.logger.error(str(e)) - await _fail(ctx, origin_tid, "models_toml_invalid", chip=chip) - return - - await check_result( - ctx, 0, continue_run=True, - extra_fields={ - "task": "regression.eval-performance", - "chip": chip, "bitstream": bitstream, "models": models, - }, - trace_id=origin_tid, - ) - - model_results = [] - for model in models: - model_key = model.lower() - try: - model_layout_name(model_key) - except ValueError: - ctx.logger.error(f"Unknown model: {model}") - await _fail(ctx, origin_tid, "unknown_model", model=model) - return - - ctx.logger.info(f"[eval-performance] model {model}: model build") - try: - _model_build(bbdir, chip, model, ctx.logger, origin_tid) - except Exception as e: - ctx.logger.error(str(e)) - await _fail(ctx, origin_tid, "model_build_cmd", model=model) - return - - ctx.logger.info(f"[eval-performance] model {model}: kernel build") - spec = acc_specs[model] - try: - k_cfg, k_build, fw_bin, fw_hex = _kernel_build_cmds( - bbdir, chip, model, dataset=spec["dataset"], - ) - except ValueError as e: - ctx.logger.error(str(e)) - await _fail(ctx, origin_tid, "kernel_build_cmd", model=model) - return - cfg_result = await stream_run_logger_async( - cmd=k_cfg, logger=ctx.logger, - stdout_prefix=f"kernel configure {model}", - stderr_prefix=f"kernel configure {model}", - ) - if cfg_result.returncode != 0: - await _fail(ctx, origin_tid, "kernel_configure_failed", - model=model, returncode=cfg_result.returncode) - return - build_result = await stream_run_logger_async( - cmd=k_build, logger=ctx.logger, - stdout_prefix=f"kernel build {model}", - stderr_prefix=f"kernel build {model}", - ) - if build_result.returncode != 0: - await _fail(ctx, origin_tid, "kernel_build_failed", - model=model, returncode=build_result.returncode) - return - if not os.path.isfile(fw_bin): - ctx.logger.error(f"fw_payload bin not found: {fw_bin}") - await _fail(ctx, origin_tid, "kernel_bin_missing", - model=model, fw_bin=fw_bin) - return - if not _kernel.bin_to_hex(fw_bin, fw_hex, base_address=0x80000000): - ctx.logger.error(f"bin_to_hex failed: {fw_bin} -> {fw_hex}") - await _fail(ctx, origin_tid, "kernel_tohex_failed", model=model) - return - - ctx.logger.info(f"[eval-performance] model {model}: p2e runworkload") - image_name = os.path.splitext(os.path.basename(fw_hex))[0] - try: - run_cmd, run_env, bebop_cwd, run_log = _p2e_run_cmds( - bbdir, bitstream, image_name, chip, input_data - ) - except FileNotFoundError as e: - ctx.logger.error(str(e)) - err = "p2e_run_cmd" - extra = {"model": model} - if str(e).startswith("P2E build case not found"): - err = "build_dir_not_found" - extra["build_dir"] = os.path.dirname( - os.path.dirname(os.path.abspath(bitstream)) - ) - elif str(e).startswith("P2E runtime artifacts missing"): - err = "p2e_runtime_missing" - await _fail(ctx, origin_tid, err, **extra) - return - run_result = await stream_run_logger_async( - cmd=run_cmd, logger=ctx.logger, cwd=bebop_cwd, env=run_env, - stdout_prefix=f"p2e run {model}", - stderr_prefix=f"p2e run {model}", - ) - if run_result.returncode != 0: - await _fail(ctx, origin_tid, "p2e_run_failed", - model=model, returncode=run_result.returncode, - log_dir=run_log) - return - - uart_path = Path(run_log) / "uart_hart_0.log" - if not uart_path.is_file(): - uart_path = Path(run_log) / "uart.log" - if not uart_path.is_file(): - ctx.logger.error(f"uart log missing under {run_log}") - await _fail(ctx, origin_tid, "uart_missing", model=model, log_dir=run_log) - return - try: - acc_i = accuracy_from_uart(spec["metric"], uart_path.read_text()) - except ValueError as e: - ctx.logger.error(f"accuracy parse failed for {model}: {e}") - await _fail(ctx, origin_tid, "accuracy_unparsable", model=model) - return - - cycle_dir = Path(run_log) / "trace" / "cycle" - if not cycle_dir.is_dir(): - ctx.logger.error(f"cycle trace dir missing: {cycle_dir}") - await _fail(ctx, origin_tid, "cycle_trace_missing", - model=model, cycle_dir=str(cycle_dir)) - return - cycle_files = [ - p for p in cycle_dir.iterdir() - if p.is_file() and p.name.startswith("trace-") - and p.name.endswith(".txt") - ] - if not cycle_files: - ctx.logger.error(f"cycle trace dir empty: {cycle_dir}") - await _fail(ctx, origin_tid, "cycle_trace_empty", - model=model, cycle_dir=str(cycle_dir)) - return - - ctx.logger.info(f"[eval-performance] model {model}: perfetto") - try: - pinputs = perfetto_inputs(bbdir, chip, model) - except (KeyError, FileNotFoundError) as e: - ctx.logger.error(str(e)) - await _fail(ctx, origin_tid, "perfetto_inputs_missing", model=model) - return - trace_dir = Path(run_log) / "trace" - perf_cmd = _perfetto_cmd( - trace_dir, pinputs["trace_toml"], pinputs["mlir_files"] - ) - perf_out = trace_dir / "perfetto.json" - perf_result = await stream_run_logger_async( - cmd=perf_cmd, logger=ctx.logger, - stdout_prefix=f"perfetto {model}", - stderr_prefix=f"perfetto {model}", - ) - if perf_result.returncode != 0 or not perf_out.is_file(): - await _fail(ctx, origin_tid, "perfetto_failed", - model=model, returncode=perf_result.returncode) - return - try: - perf_data = json.loads(perf_out.read_text()) - except (OSError, json.JSONDecodeError) as e: - ctx.logger.error(f"perfetto json unreadable: {e}") - await _fail(ctx, origin_tid, "perfetto_json_unreadable", model=model) - return - try: - cycles_i = e2e_cycles_from_perfetto(perf_data) - except ValueError as e: - ctx.logger.error(str(e)) - await _fail(ctx, origin_tid, "perfetto_cycles_failed", model=model) - return - - dest = Path(run_log) / f"perfetto-{model}.json" - dest.write_text(json.dumps(perf_data, indent=2)) - model_results.append({ - "name": model, - "cycles": cycles_i, - "accuracy": acc_i, - "metric": spec["metric"], - "dataset": spec["dataset"], - "perfetto": str(dest), - }) - - try: - cycles = mean_cycles([r["cycles"] for r in model_results]) - accuracy = mean_accuracy([r["accuracy"] for r in model_results]) - except ValueError as e: - ctx.logger.error(str(e)) - await _fail(ctx, origin_tid, "cycles_mean_failed", models=model_results) - return - merge_metrics(bbdir, cycles=cycles, accuracy=accuracy, models=model_results) - await check_result( - ctx, 0, continue_run=False, - extra_fields={ - "task": "regression.eval-performance", - "chip": chip, "cycles": cycles, "accuracy": accuracy, - "models": model_results, - }, - trace_id=origin_tid, - ) diff --git a/bbdev/api/steps/regression/03_eval_area_power_api.step.py b/bbdev/api/steps/regression/03_eval_area_power_api.step.py deleted file mode 100644 index 117f49f90..000000000 --- a/bbdev/api/steps/regression/03_eval_area_power_api.step.py +++ /dev/null @@ -1,25 +0,0 @@ -from motia import ApiRequest, ApiResponse, FlowContext, api - -from utils.event_common import require_chip - -config = { - "name": "regression-eval-area-power-api", - "description": "DC area+freq for regression", - "flows": ["regression"], - "triggers": [api("POST", "/regression/eval-area-power")], - "enqueues": ["regression.eval-area-power"], -} - - -async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = req.body or {} - chip = require_chip(body) - data = { - "chip": chip, - "_trace_id": ctx.trace_id, - } - top = body.get("top") - if isinstance(top, str) and top: - data["top"] = top - await ctx.enqueue({"topic": "regression.eval-area-power", "data": data}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/regression/03_eval_area_power_event.step.py b/bbdev/api/steps/regression/03_eval_area_power_event.step.py deleted file mode 100644 index b619dc5f3..000000000 --- a/bbdev/api/steps/regression/03_eval_area_power_event.step.py +++ /dev/null @@ -1,71 +0,0 @@ -import os -import sys - -from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.event_common import require_chip -from utils.event_common import check_result, get_origin_trace_id -from utils.path import get_buckyball_path - -config = { - "name": "regression-eval-area-power", - "description": "Enqueue DC area workflow for regression area+freq", - "flows": ["regression"], - "triggers": [queue("regression.eval-area-power")], - "enqueues": ["dc.verilog"], -} - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - bbdir = get_buckyball_path() - try: - chip = require_chip(input_data) - except ValueError as exc: - ctx.logger.error(str(exc)) - extra = {"error": "missing_chip"} if str(exc).startswith("Missing required parameter: --chip") else {"error": str(exc)} - await check_result( - ctx, 1, continue_run=False, - extra_fields=extra, - trace_id=origin_tid, - ) - return - tapeout = os.path.join(bbdir, "examples", "chips", chip, "tapeout") - if not os.path.isdir(tapeout): - ctx.logger.error(f"chip {chip} has no tapeout directory: {tapeout}") - await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "missing_tapeout", "chip": chip, "tapeout": tapeout}, - trace_id=origin_tid, - ) - return - - top = input_data.get("top") or "DigitalTop" - - await check_result( - ctx, - 0, - continue_run=True, - extra_fields={ - "task": "regression.eval-area-power", - "chip": chip, - }, - trace_id=origin_tid, - ) - - await ctx.enqueue( - { - "topic": "dc.verilog", - "data": { - "chip": chip, - "top": top, - "from_area_workflow": True, - "from_regression_area_power": True, - "_trace_id": origin_tid, - }, - } - ) diff --git a/bbdev/api/steps/regression/__init__.py b/bbdev/api/steps/regression/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/bbdev/api/steps/regression/scripts/accuracy.py b/bbdev/api/steps/regression/scripts/accuracy.py deleted file mode 100644 index 601ad77b1..000000000 --- a/bbdev/api/steps/regression/scripts/accuracy.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Per-model accuracy parsing for regression eval-performance. - -UART contracts (fail loud if missing / malformed): - top1 -> line "top1=/" - map -> line "map=" -""" -from __future__ import annotations - -import re -from typing import Any - -def accuracy_from_uart(metric: str, uart: str) -> float: - if not isinstance(metric, str) or not metric: - raise ValueError("accuracy metric must be a non-empty string") - if not isinstance(uart, str) or not uart: - raise ValueError("uart log is empty; cannot parse accuracy") - m = metric.lower() - if m == "top1": - return _top1(uart) - if m == "map": - return _map(uart) - raise ValueError(f"unknown accuracy metric: {metric}") - - -def _top1(uart: str) -> float: - hits = re.findall(r"^top1=(\d+)/(\d+)\s*$", uart, re.M) - if not hits: - raise ValueError("uart missing top1=/ line") - correct_s, total_s = hits[-1] - correct = int(correct_s) - total = int(total_s) - if total <= 0: - raise ValueError(f"top1 total must be > 0, got {total}") - if correct < 0 or correct > total: - raise ValueError(f"top1 correct out of range: {correct}/{total}") - return correct / total - - -def _map(uart: str) -> float: - hits = re.findall(r"^map=([0-9]*\.?[0-9]+)\s*$", uart, re.M) - if not hits: - raise ValueError("uart missing map= line") - val = float(hits[-1]) - if val < 0.0 or val > 1.0: - raise ValueError(f"map out of range [0,1]: {val}") - return val - - -def mean_accuracy(values: list[float]) -> float: - if not values: - raise ValueError("cannot mean empty accuracy list") - return sum(values) / len(values) - - -def load_eval_accuracy(chip: str, bbdir: str, models: list[str]) -> dict[str, dict[str, Any]]: - """Load [eval.accuracy.] for each model. Missing section -> error.""" - import tomllib - from pathlib import Path - - path = Path(bbdir) / "examples" / "chips" / chip / "regression" / "eval" / "models.toml" - if not path.is_file(): - raise ValueError(f"eval models toml does not exist: {path}") - with open(path, "rb") as f: - data = tomllib.load(f) - eval_sec = data.get("eval") - if not isinstance(eval_sec, dict): - raise ValueError(f"missing [eval] section: {path}") - acc_root = eval_sec.get("accuracy") - if not isinstance(acc_root, dict): - raise ValueError(f"missing [eval.accuracy] tables: {path}") - out: dict[str, dict[str, Any]] = {} - for name in models: - spec = acc_root.get(name) - if not isinstance(spec, dict): - raise ValueError(f"missing [eval.accuracy.{name}]: {path}") - metric = spec.get("metric") - dataset = spec.get("dataset") - if not isinstance(metric, str) or not metric: - raise ValueError(f"[eval.accuracy.{name}].metric must be non-empty string") - if not isinstance(dataset, str) or not dataset: - raise ValueError(f"[eval.accuracy.{name}].dataset must be non-empty string") - if dataset.startswith("/") or ".." in dataset.split("/"): - raise ValueError( - f"[eval.accuracy.{name}].dataset must be a relative path without '..': {dataset}" - ) - out[name] = {"metric": metric, "dataset": dataset} - return out diff --git a/bbdev/api/steps/regression/scripts/dc_area.py b/bbdev/api/steps/regression/scripts/dc_area.py deleted file mode 100644 index 64c07d186..000000000 --- a/bbdev/api/steps/regression/scripts/dc_area.py +++ /dev/null @@ -1,7 +0,0 @@ -import re - -def area_mm2_from_rpt(text: str) -> float: - m = re.search(r"Total cell area:\s+([\d.]+)", text) - if not m: - raise ValueError("Total cell area not found") - return float(m.group(1)) / 1e6 diff --git a/bbdev/api/steps/regression/scripts/model_layout.py b/bbdev/api/steps/regression/scripts/model_layout.py deleted file mode 100644 index 49e299dca..000000000 --- a/bbdev/api/steps/regression/scripts/model_layout.py +++ /dev/null @@ -1,13 +0,0 @@ -from pathlib import Path -from utils.model import model_layout_name - - -def perfetto_inputs(bbdir: str, chip: str, model: str) -> dict: - name = model_layout_name(model) - root = Path(bbdir) - trace_toml = root / "examples/models" / chip / name / "trace/trace.toml" - mlir = root / "stack/models/build" / chip / name / "generated/subgraph0.mlir" - for path in (trace_toml, mlir): - if not path.is_file(): - raise FileNotFoundError(path) - return {"trace_toml": trace_toml, "mlir_files": [mlir]} diff --git a/bbdev/api/steps/regression/scripts/models_toml.py b/bbdev/api/steps/regression/scripts/models_toml.py deleted file mode 100644 index 1cf98ca7b..000000000 --- a/bbdev/api/steps/regression/scripts/models_toml.py +++ /dev/null @@ -1,20 +0,0 @@ -import tomllib -from pathlib import Path - - -def load_eval_models(chip: str, bbdir: str) -> list[str]: - path = Path(bbdir) / "examples" / "chips" / chip / "regression" / "eval" / "models.toml" - if not path.is_file(): - raise ValueError(f"eval models toml does not exist: {path}") - with open(path, "rb") as f: - data = tomllib.load(f) - eval_sec = data.get("eval") - if not isinstance(eval_sec, dict): - raise ValueError(f"missing [eval] section: {path}") - models = eval_sec.get("models") - if not isinstance(models, list) or not models: - raise ValueError(f"[eval].models must be a non-empty list: {path}") - for name in models: - if not isinstance(name, str) or not name: - raise ValueError(f"[eval].models entries must be non-empty strings: {path}") - return models diff --git a/bbdev/api/steps/regression/scripts/perfetto_latency.py b/bbdev/api/steps/regression/scripts/perfetto_latency.py deleted file mode 100644 index 7956db138..000000000 --- a/bbdev/api/steps/regression/scripts/perfetto_latency.py +++ /dev/null @@ -1,34 +0,0 @@ -def _cycle_int(value, label: str) -> int: - try: - num = float(value) - except (TypeError, ValueError) as e: - raise ValueError(f"{label} is not a number: {value!r}") from e - if num != int(num): - raise ValueError(f"{label} is not an integer cycle count: {value!r}") - return int(num) - - -def e2e_cycles_from_perfetto(data: dict) -> int: - """End-to-end cycle count: min(ts) .. max(ts+dur) over complete (ph=X) events. - - Perfetto ts/dur in this stack are cycle offsets (see elapsed_cycle), not wall ns. - """ - events = data.get("traceEvents") - if not isinstance(events, list): - raise ValueError("perfetto missing traceEvents list") - xs = [e for e in events if isinstance(e, dict) and e.get("ph") == "X"] - if not xs: - raise ValueError("perfetto has no complete (ph=X) events") - t0 = min(_cycle_int(e["ts"], "ts") for e in xs) - t1 = max(_cycle_int(e["ts"], "ts") + _cycle_int(e["dur"], "dur") for e in xs) - return t1 - t0 - - -def mean_cycles(values: list[int]) -> int: - if not values: - raise ValueError("mean_cycles requires a non-empty list") - total = sum(values) - n = len(values) - if total % n != 0: - raise ValueError(f"mean cycles not integral: sum={total} n={n} values={values}") - return total // n diff --git a/bbdev/api/steps/regression/scripts/result.py b/bbdev/api/steps/regression/scripts/result.py deleted file mode 100644 index 954e44c41..000000000 --- a/bbdev/api/steps/regression/scripts/result.py +++ /dev/null @@ -1,19 +0,0 @@ -import json -from pathlib import Path - - -def result_path(bbdir: str) -> Path: - return Path(bbdir).resolve() / "chipcrowd-eval-result.json" - - -def merge_metrics(bbdir: str, **fields) -> dict: - path = result_path(bbdir) - data = {} - if path.is_file(): - data = json.loads(path.read_text()) - if not isinstance(data, dict): - raise ValueError(f"eval result is not an object: {path}") - data.update(fields) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data, indent=2) + "\n") - return data diff --git a/bbdev/api/steps/uvm/01_build_api.step.py b/bbdev/api/steps/uvm/01_build_api.step.py index 4224cdf28..123e9f793 100644 --- a/bbdev/api/steps/uvm/01_build_api.step.py +++ b/bbdev/api/steps/uvm/01_build_api.step.py @@ -10,7 +10,7 @@ def check_args(body: dict) -> str | None: - allowed = {"chip", "ball", "ip"} + allowed = {"chip", "ball", "ip", "target"} for key in body: if key not in allowed: return f"Unexpected parameter: --{key}" @@ -22,6 +22,11 @@ def check_args(body: dict) -> str | None: return "Parameter --ip requires a value" if body.get("ball") and body.get("ip"): return "Parameters --ball and --ip are mutually exclusive" + if "target" in body: + if not isinstance(body["target"], str) or not body["target"].strip(): + return "Parameter --target requires a value" + if not body.get("ip"): + return "Parameter --target requires --ip" return None diff --git a/bbdev/api/steps/uvm/01_build_event.step.py b/bbdev/api/steps/uvm/01_build_event.step.py index 103c1c305..ce319c645 100644 --- a/bbdev/api/steps/uvm/01_build_event.step.py +++ b/bbdev/api/steps/uvm/01_build_event.step.py @@ -37,6 +37,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: input_data.get("ip"), ctx, False, + input_data.get("target"), ) except Exception as e: ctx.logger.error(str(e)) diff --git a/bbdev/api/steps/uvm/02_run_api.step.py b/bbdev/api/steps/uvm/02_run_api.step.py index a0872cebb..3532fde4f 100644 --- a/bbdev/api/steps/uvm/02_run_api.step.py +++ b/bbdev/api/steps/uvm/02_run_api.step.py @@ -10,7 +10,7 @@ def check_args(body: dict) -> str | None: - allowed = {"chip", "ball", "ip"} + allowed = {"chip", "ball", "ip", "target"} for key in body: if key not in allowed: return f"Unexpected parameter: --{key}" @@ -22,6 +22,11 @@ def check_args(body: dict) -> str | None: return "Parameter --ip requires a value" if body.get("ball") and body.get("ip"): return "Parameters --ball and --ip are mutually exclusive" + if "target" in body: + if not isinstance(body["target"], str) or not body["target"].strip(): + return "Parameter --target requires a value" + if not body.get("ip"): + return "Parameter --target requires --ip" return None diff --git a/bbdev/api/steps/uvm/02_run_event.step.py b/bbdev/api/steps/uvm/02_run_event.step.py index a8ab800a5..a1e24a397 100644 --- a/bbdev/api/steps/uvm/02_run_event.step.py +++ b/bbdev/api/steps/uvm/02_run_event.step.py @@ -14,6 +14,10 @@ from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path from scripts.uvm_common import run_uvm +from steps.uvm.scripts.report import coverage_report, verification_targets +from utils.reports import source_context +from utils.path import log_dir +from datetime import datetime config = { "name": "uvm-run", @@ -27,6 +31,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() + report_context = source_context(bbdir, input_data["chip"]) try: info = await asyncio.to_thread( @@ -37,7 +42,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: input_data.get("ip"), ctx, True, + input_data.get("target"), ) + if "results" not in info: + info = {"chip": info["chip"], "results": [info], "failures": []} + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + directory = log_dir(bbdir, input_data["chip"], "verilog", stamp, "uvm", f"report-{origin_tid}") + expected = verification_targets( + bbdir, input_data["chip"], input_data.get("ball"), + input_data.get("ip"), input_data.get("target"), + ) + info["report_path"] = coverage_report(report_context, origin_tid, info, directory, expected) except Exception as e: ctx.logger.error(str(e)) await check_result( @@ -51,7 +66,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result( ctx, - 0, + 1 if info["failures"] else 0, continue_run=False, extra_fields={"task": "run", **info}, trace_id=origin_tid, diff --git a/bbdev/api/steps/uvm/scripts/report.py b/bbdev/api/steps/uvm/scripts/report.py new file mode 100644 index 000000000..764a21c7d --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/report.py @@ -0,0 +1,94 @@ +import tomllib +from pathlib import Path + +from utils.reports import new_report, attachment, finish_report +from .uvm_common import dashboard_summary, load_ip + + +def verification_targets(repo, chip, ball=None, ip=None, target=None): + if ip is not None: + return [ + f"ip/{ip}/{item['name']}" + for item in load_ip(str(repo), ip, target, chip)["targets"] + ] + if ball is not None: + return [f"ball/{ball}"] + config = tomllib.loads( + (Path(repo) / f"examples/chips/{chip}/configs/chip.toml").read_text() + )["uvm"] + targets = [f"ball/{ball}" for ball in config["balls"]] + for ip in config["ips"]: + targets.extend( + f"ip/{ip}/{target['name']}" for target in load_ip(str(repo), ip, chip=chip)["targets"] + ) + return targets + + +def coverage_report(context, trace_id, info, output, expected_targets): + report = new_report(context, "uvm.run", trace_id, "uvm") + for result in info["results"]: + if "balls" in result: + rows = [ + ( + f"ball/{name}", + Path(result["log"]) + / (f"{name}/coverage" if len(result["balls"]) > 1 else "coverage"), + ) + for name in result["balls"] + ] + else: + rows = [ + ( + f"ip/{result['ip']}/{target}", + Path(result["log"]) / target / "coverage", + ) + for target in result["targets"] + ] + for target, directory in rows: + report["verification_targets"].append(target) + if ( + result.get("expected_assertions") + and target.split("/")[-1] in result["expected_assertions"] + ): + report["verification"].append( + { + "target": target, + "status": "pass", + "coverage": None, + "report_url": None, + } + ) + continue + values = dashboard_summary(str(directory)) + url = attachment( + report, + output, + directory / "dashboard.txt", + f"coverage-{target.replace('/', '-')}.txt", + ) + report["verification"].append( + { + "target": target, + "status": "pass", + "coverage": { + name: float(value) + for name, value in values.items() + if value != "--" + }, + "report_url": url, + } + ) + for failure in info["failures"]: + target = failure["target"] + report["verification_targets"].append(target) + report["verification"].append( + { + "target": target, + "status": "fail", + "coverage": None, + "report_url": None, + "error": failure["error"], + } + ) + report["verification_targets"] = expected_targets + return finish_report(report, output) diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index b58bf3bd5..5dd37a7f9 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -13,7 +13,6 @@ from utils.path import get_buckyball_path, log_dir from utils.stream_run import stream_run_logger -from .waive import apply_waivers config_scripts = os.path.abspath( os.path.join(os.path.dirname(__file__), "..", "..", "config", "scripts") @@ -81,7 +80,7 @@ def selected_mappings(domain, ball: str | None): raise ValueError(f"ball {ball!r} not in chip.pb") -def vcs_defines(domain, mapping, core, tile): +def vcs_defines(domain, mapping, core, tile, parameters): shared_bank_num = ( tile.shared_mem.entries // core.mem.bank.entries if tile.shared_mem.enable @@ -103,26 +102,15 @@ def vcs_defines(domain, mapping, core, tile): for e in domain.isa: if e.bid == mapping.ball_id: defs.append(f"+define+{e.mnemonic}_FUNCT7={e.funct7}") - if mapping.ball_dir == "smatmul": - defs.append(f"+define+SMATMUL_TILE_ROWS={mapping.ball_params['tileRows']}") + for define, parameter in parameters.items(): + defs.append(f"+define+{define}={mapping.ball_params[parameter]}") return defs -def smatmul_accumulator_filename(rtl_dir: Path) -> str: - text = (rtl_dir / "SMatMulUnit.sv").read_text() - modules = re.findall(r"\b(accumulator_\d+x\d+)\s+accumulator_ext\s*\(", text) - if len(modules) != 1: - raise ValueError( - f"expected one SMatMul accumulator instance, found {modules!r}" - ) - return f"{modules[0]}.sv" - - def _filelist( verify_dir: Path, ball_dir: str, uvm_rel: str, - rtl_rel: str, rtl_dir: Path, sim_dir: Path, ) -> str: @@ -130,38 +118,26 @@ def _filelist( dst = sim_dir / f"{ball_dir}_ball.f" sim_dir.mkdir(parents=True, exist_ok=True) text = src.read_text() - if "@SMATMUL_ACCUMULATOR@" in text: - text = text.replace( - "@SMATMUL_ACCUMULATOR@", smatmul_accumulator_filename(rtl_dir) - ) - if "@IM2COL_BUFFER@" in text: - modules = set( - re.findall( - r"\b(buf_\d+x\d+)\s+buf(?:Lo|Hi)_ext\s*\(", - (rtl_dir / "LineBufferManager.sv").read_text(), - ) - ) - if len(modules) != 1: - raise ValueError(f"expected one Im2col buffer module, found {modules!r}") - text = text.replace("@IM2COL_BUFFER@", f"{modules.pop()}.sv") - dst.write_text(text.replace("@UVM@", uvm_rel).replace("@RTL@", rtl_rel)) + dst.write_text(text.replace("@UVM@", uvm_rel).replace("@RTL@", str(rtl_dir.resolve()))) return str(dst.relative_to(verify_dir)) -def load_ip(bbdir: str, name: str) -> dict: - registry = Path(bbdir) / "verify" / "uvm" / "ip.toml" - if name not in tomllib.loads(registry.read_text()).get("ips", []): +def load_ip(bbdir: str, name: str, target_name: str | None = None, chip: str | None = None) -> dict: + registry = Path(bbdir) / "arch" / "uvm.toml" + registered = tomllib.loads(registry.read_text()) + if name not in registered["ips"]: raise ValueError(f"IP {name!r} is not registered in {registry}") - root = ( - Path(bbdir) - / "arch" - / "src" - / "main" - / "scala" - / "framework" - / "mem-core" - / name - ) + root = Path(bbdir) / registered["paths"][name] + build = registered.get("build", {}).get(name, {}) + if set(build) - {"directory", "module", "arguments"}: + raise ValueError(f"Invalid IP build settings for {name!r}: {build}") + directory = build.get("directory", str(root.relative_to(bbdir))) + module = build.get("module", name) + arguments = build.get("arguments", []) + if not isinstance(arguments, list) or not all(isinstance(value, str) for value in arguments): + raise ValueError(f"IP build arguments must be a list of strings: {name!r}") + if not isinstance(directory, str) or not directory or not isinstance(module, str) or not module: + raise ValueError(f"IP build directory and module must be nonempty strings: {name!r}") resources = root / "src" / "main" / "resources" targets = [ { @@ -172,11 +148,46 @@ def load_ip(bbdir: str, name: str) -> dict: } for filelist in sorted(resources.glob("*.f")) ] + for target in targets: + metadata = resources / f"{target['name']}.toml" + if metadata.is_file(): + settings = tomllib.loads(metadata.read_text()) + if not settings or set(settings) - {"expected_assertion", "chips", "invalid_case"}: + raise ValueError(f"Invalid IP test settings: {metadata}") + if "invalid_case" in settings: + case = settings["invalid_case"] + if type(case) is not int or case < 0 or "expected_assertion" not in settings: + raise ValueError(f"invalid_case requires a nonnegative integer and expected_assertion: {metadata}") + target["invalid_case"] = case + if "expected_assertion" in settings: + assertion = settings["expected_assertion"] + if not isinstance(assertion, str) or not assertion or "\n" in assertion: + raise ValueError(f"Expected one assertion message in {metadata}") + target["expected_assertion"] = assertion + if "chips" in settings: + profiles = settings["chips"] + if (not isinstance(profiles, list) or not profiles or + not all(isinstance(value, str) and value.strip() and value == value.strip() for value in profiles)): + raise ValueError(f"chips must be a nonempty list of nonempty chip names: {metadata}") + if chip is None: + raise ValueError(f"Chip is required to select declared IP profiles: {metadata}") + target["chips"] = profiles if not targets: raise ValueError(f"registered IP {name!r} has no filelists under {resources}") + if target_name is not None: + targets = [target for target in targets if target["name"] == target_name] + if not targets: + raise ValueError(f"Unknown target {target_name!r} for IP {name!r}") + if "chips" in targets[0] and chip not in targets[0]["chips"]: + raise ValueError(f"Target {target_name!r} for IP {name!r} does not support chip {chip!r}; declared chips: {targets[0]['chips']}") + targets = [target for target in targets if "chips" not in target or chip in target["chips"]] + if not targets: + raise ValueError(f"IP {name!r} has no applicable targets for chip {chip!r}") return { "root": str(root.relative_to(bbdir)), - "mill_module": name, + "mill_module": module, + "mill_directory": directory, + "mill_arguments": arguments, "model": "src/csrc/Cargo.toml", "resources": "src/main/resources", "targets": targets, @@ -197,11 +208,24 @@ def _ip_filelist( return dst -def check_uvm_result(result, target: str) -> None: +def check_uvm_result(result, target: str, expected_assertion: str | None = None) -> None: + output = result.stdout + result.stderr + if expected_assertion is not None: + assertions = re.findall(r"Assertion failed: ([^\r\n]+)", output) + failure_counts = re.findall(r"UVM_(?:ERROR|FATAL)\s*:\s*(\d+)", output) + if ( + not assertions + or set(assertions) != {expected_assertion} + or "Fatal:" not in output + or any(int(count) for count in failure_counts) + or result.returncode < 0 + ): + raise RuntimeError(f"Expected RTL assertion did not terminate {target}: {expected_assertion}") + return if result.returncode != 0: raise RuntimeError(f"UVM failed for {target}") counts = dict( - re.findall(r"UVM_(ERROR|FATAL)\s*:\s*(\d+)", result.stdout + result.stderr) + re.findall(r"UVM_(ERROR|FATAL)\s*:\s*(\d+)", output) ) if set(counts) != {"ERROR", "FATAL"}: raise RuntimeError(f"UVM report summary missing for {target}") @@ -211,8 +235,8 @@ def check_uvm_result(result, target: str) -> None: ) -def build_ip(bbdir: str, chip: str, name: str, ctx) -> dict: - config = load_ip(bbdir, name) +def build_ip(bbdir: str, chip: str, name: str, ctx, target_name: str | None = None) -> dict: + config = load_ip(bbdir, name, target_name, chip) root = Path(bbdir) / config["root"] resources = root / config["resources"] manifest = root / config["model"] @@ -236,20 +260,20 @@ def build_ip(bbdir: str, chip: str, name: str, ctx) -> dict: if result.returncode != 0: raise RuntimeError(f"cargo build failed for IP {name}") - emit = ( - f"{shlex.quote(str(Path(bbdir) / 'result' / 'bin' / 'mill'))} " - f"{shlex.quote(config['mill_module'] + '.run')}" - ) + emit = shlex.join([ + str(Path(bbdir) / "result" / "bin" / "mill"), + config["mill_module"] + ".run", + *(argument.replace("{chip}", chip).replace("{targets}", ",".join(target["name"] for target in config["targets"])) for argument in config["mill_arguments"]), + ]) result = stream_run_logger( cmd=emit, logger=ctx.logger, - cwd=str(root), + cwd=str(Path(bbdir) / config["mill_directory"]), stdout_prefix="uvm rtl", stderr_prefix="uvm rtl", ) if result.returncode != 0: raise RuntimeError(f"RTL generation failed for IP {name}") - apply_waivers(rtl) for target in config["targets"]: sim_dir = sim_root / target["name"] @@ -257,16 +281,26 @@ def build_ip(bbdir: str, chip: str, name: str, ctx) -> dict: simv = sim_dir / "simv" csrc = sim_dir / "csrc" hier = sim_dir / "cm_hier.cfg" - hier.write_text( - f"+tree {target['top']}.dut\n" - f"+tree {target['top']}.source_if\n" - f"+tree {target['top']}.sink_if\n" - ) + declared_hier = resources / f"{target['name']}.cm_hier" + if declared_hier.is_file(): + scopes = [line.strip() for line in declared_hier.read_text().splitlines() if line.strip()] + if not scopes or any( + not re.fullmatch(r"\+tree @TOP@\.[A-Za-z_][A-Za-z0-9_$.]*", line) + for line in scopes + ): + raise ValueError(f"Invalid IP coverage hierarchy: {declared_hier}") + hier.write_text("\n".join(line.replace("@TOP@", target["top"]) for line in scopes) + "\n") + else: + hier.write_text( + f"+tree {target['top']}.dut\n" + f"+tree {target['top']}.source_if\n" + f"+tree {target['top']}.sink_if\n" + ) script = ( f"cd {shlex.quote(str(sim_dir))} && " f"rm -rf {shlex.quote(str(csrc))} {shlex.quote(str(simv))} {shlex.quote(str(simv))}.daidir && " f"mkdir -p {shlex.quote(str(csrc))} && " - "vcs -full64 -sverilog -timescale=1ns/1ps -debug_access+all " + f"vcs -full64 -sverilog -timescale=1ns/1ps -debug_access+all -top {shlex.quote(target['top'])} " "${=VCS_UVM_ARGS} " f"-cm line+cond+tgl+assert -cm_hier {shlex.quote(str(hier))} " f"-cm_assert_hier {shlex.quote(str(hier))} " @@ -317,21 +351,13 @@ def report_ip_coverage( ) -> None: cov_dir.mkdir(parents=True, exist_ok=True) exclusions = root / "src" / "main" / "resources" / f"{target['name']}.el" - reports = [(cov_dir, "")] + hierarchy = simv.parent / "rtl_hier.cfg" + hierarchy.write_text(f"+tree {target['top']}.dut\n") + scope = f" -hier {shlex.quote(str(hierarchy))} -metric line+cond+tgl" + reports = [(cov_dir, ""), (cov_dir / "rtl_raw", scope)] if exclusions.is_file(): reports[0] = (cov_dir, " -dump full_exclusions") - hierarchy = simv.parent / "rtl_hier.cfg" - hierarchy.write_text(f"+tree {target['top']}.dut\n") - scope = f" -hier {shlex.quote(str(hierarchy))}" - reports.extend( - [ - (cov_dir / "rtl_raw", scope), - ( - cov_dir / "rtl", - scope + f" -elfile {shlex.quote(str(exclusions))} -excl_strict", - ), - ] - ) + reports.append((cov_dir / "rtl", scope + f" -elfile {shlex.quote(str(exclusions))} -excl_strict")) for report, options in reports: if report.name == "rtl": validate_ip_exclusions(exclusions, simv.parent) @@ -357,8 +383,8 @@ def report_ip_coverage( ) -def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str) -> dict: - config = build_ip(bbdir, chip, name, ctx) +def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str, target_name: str | None = None) -> dict: + config = build_ip(bbdir, chip, name, ctx, target_name) root = Path(bbdir) / config["root"] manifest = root / config["model"] crate = tomllib.loads(manifest.read_text())["package"]["name"].replace("-", "_") @@ -366,15 +392,17 @@ def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str) -> dict: sim_root = root / "build" / "uvm" / chip coverage = [] + expected_assertions = [] for target in config["targets"]: simv = sim_root / target["name"] / "simv" simv_q = shlex.quote(str(simv)) cov_dir = Path(cov_root) / target["name"] / "coverage" + scenario = f"+invalid_case={target['invalid_case']} " if "invalid_case" in target else "" script = ( f'loader="$(patchelf --print-interpreter {simv_q})"; ' f'library_path="$(patchelf --print-rpath {simv_q})"; ' f'"$loader" --library-path "$library_path" {simv_q} -no_save ' - f"-sv_lib {shlex.quote(str(model))} +UVM_TESTNAME={shlex.quote(target['test'])} " + f"-sv_lib {shlex.quote(str(model))} +UVM_TESTNAME={shlex.quote(target['test'])} {scenario}" f"-cm line+cond+tgl+assert -cm_name {shlex.quote(target['test'])}" ) command = ( @@ -388,7 +416,11 @@ def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str) -> dict: stdout_prefix="uvm run", stderr_prefix="uvm run", ) - check_uvm_result(result, f"IP {name} target={target['name']}") + expected_assertion = target.get("expected_assertion") + check_uvm_result(result, f"IP {name} target={target['name']}", expected_assertion) + if expected_assertion is not None: + expected_assertions.append(target["name"]) + continue report_ip_coverage(bbdir, root, target, simv, cov_dir, ctx) coverage.append((target["name"], cov_dir)) @@ -402,17 +434,25 @@ def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str) -> dict: f"{target} {summary['SCORE']} {summary['LINE']} {summary['COND']} " f"{summary['TOGGLE']} {summary.get('ASSERT', '--')} {summary.get('GROUP', '--')} pass" ) + rows.extend(f"{target} -- -- -- -- -- -- expected-assertion" for target in expected_assertions) index = index_dir / "index.txt" index.write_text("\n".join(rows) + "\n") return { "chip": chip, "ip": name, "targets": [t["name"] for t in config["targets"]], + "expected_assertions": expected_assertions, "index": str(index), + "rtl_raw_coverage": { + target["name"]: str(Path(cov_root) / target["name"] / "coverage" / "rtl_raw") + for target in config["targets"] + if target["name"] not in expected_assertions + }, "rtl_coverage": { target["name"]: str(Path(cov_root) / target["name"] / "coverage" / "rtl") for target in config["targets"] - if (root / "src/main/resources" / f"{target['name']}.el").is_file() + if target["name"] not in expected_assertions + and (root / "src/main/resources" / f"{target['name']}.el").is_file() }, "log": cov_root, } @@ -428,8 +468,9 @@ def build_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx) tile = next(tile for tile in chip.tiles if core.index in tile.core_indices) sim_dir = verify_dir / "build" / chip_name uvm_rel = os.path.relpath(Path(bbdir) / "verify" / "uvm", verify_dir) - rtl_rel = os.path.relpath(rtl_dir, verify_dir) - flist = _filelist(verify_dir, ball, uvm_rel, rtl_rel, rtl_dir, sim_dir) + flist = _filelist(verify_dir, ball, uvm_rel, rtl_dir, sim_dir) + parameters_path = verify_dir / "parameters.toml" + parameters = tomllib.loads(parameters_path.read_text())["defines"] if parameters_path.is_file() else {} cargo = ( f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " f"cargo build --manifest-path {shlex.quote(str(casegen))}" @@ -455,9 +496,9 @@ def build_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx) f"cd {shlex.quote(str(verify_dir))} && " f"rm -rf {shlex.quote(str(csrc))} {shlex.quote(str(simv))} {shlex.quote(str(simv))}.daidir && " f"mkdir -p {shlex.quote(str(sim_dir))} {shlex.quote(str(csrc))} && " - "vcs -full64 -sverilog -timescale=1ns/1ps -debug_access+all " + "vcs -full64 -sverilog -timescale=1ns/1ps -debug_access+all -top tb_top " "${=VCS_UVM_ARGS} " - + " ".join(shlex.quote(d) for d in vcs_defines(domain, mapping, core, tile)) + + " ".join(shlex.quote(d) for d in vcs_defines(domain, mapping, core, tile, parameters)) + f" -cm line+cond+tgl+assert -cm_hier {shlex.quote(str(hier))} " f"-Mdir={shlex.quote(str(csrc))} -o {shlex.quote(str(simv))} " f"-f {shlex.quote(flist)}" @@ -552,6 +593,8 @@ def run_chip(bbdir: str, chip: str, ball: str | None, ctx, do_run: bool) -> dict msg = load_chip(bbdir, chip) domain = ball_domain(msg, ball) mill_cfg = msg.mill.verilator_config + if not mill_cfg: + raise ValueError(f"chip {chip!r} has no sims.verilator for UVM") maps = selected_mappings(domain, ball) ran = [] covs = [] @@ -595,16 +638,19 @@ def run_chip(bbdir: str, chip: str, ball: str | None, ctx, do_run: bool) -> dict def run_uvm( - bbdir: str, chip: str, ball: str | None, ip: str | None, ctx, do_run: bool + bbdir: str, chip: str, ball: str | None, ip: str | None, ctx, do_run: bool, target_name: str | None = None ) -> dict: + if target_name is not None and ip is None: + raise ValueError("Parameter --target requires --ip") if ball is not None: return run_chip(bbdir, chip, ball, ctx, do_run) if ip is not None: if do_run: - stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - run_root = log_dir(bbdir, chip, "verilog", stamp, "uvm", ip) - return run_ip(bbdir, chip, ip, ctx, run_root) - config = build_ip(bbdir, chip, ip, ctx) + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + run_name = f"{ip}-{target_name}" if target_name is not None else ip + run_root = log_dir(bbdir, chip, "verilog", stamp, "uvm", run_name) + return run_ip(bbdir, chip, ip, ctx, run_root, target_name) + config = build_ip(bbdir, chip, ip, ctx, target_name) return { "chip": chip, "ip": ip, @@ -612,21 +658,25 @@ def run_uvm( } targets = load_chip_uvm(bbdir, chip) - results = [ - run_chip(bbdir, chip, target, ctx, do_run) for target in targets["balls"] - ] - for target in targets["ips"]: - if do_run: - stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - run_root = log_dir(bbdir, chip, "verilog", stamp, "uvm", target) - results.append(run_ip(bbdir, chip, target, ctx, run_root)) - else: - config = build_ip(bbdir, chip, target, ctx) - results.append( - { - "chip": chip, - "ip": target, - "targets": [item["name"] for item in config["targets"]], - } - ) - return {"chip": chip, "results": results} + results, failures = [], [] + for ball in targets["balls"]: + try: + results.append(run_chip(bbdir, chip, ball, ctx, do_run)) + except Exception as error: + if not do_run: raise + ctx.logger.error(f"UVM ball/{ball} failed: {error}") + failures.append({"target": f"ball/{ball}", "error": str(error)}) + for ip in targets["ips"]: + try: + if do_run: + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + run_root = log_dir(bbdir, chip, "verilog", stamp, "uvm", ip) + results.append(run_ip(bbdir, chip, ip, ctx, run_root)) + else: + config = build_ip(bbdir, chip, ip, ctx) + results.append({"chip": chip, "ip": ip, "targets": [item["name"] for item in config["targets"]]}) + except Exception as error: + if not do_run: raise + ctx.logger.error(f"UVM ip/{ip} failed: {error}") + failures.append({"target": f"ip/{ip}", "error": str(error)}) + return {"chip": chip, "results": results, "failures": failures} diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index 3ef7ff3bb..d2b8510d9 100644 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -39,8 +39,6 @@ def main(): "test_api_bebop_p2e_verilog.py", "test_api_bebop_p2e_buildbitstream.py", "test_api_bebop_p2e_runworkload.py", - "test_api_regression_buildbitstream.py", - "test_api_regression_eval_area_power.py", ] for name in tests: path = test_dir / name diff --git a/bbdev/api/tests/test_api_regression_buildbitstream.py b/bbdev/api/tests/test_api_regression_buildbitstream.py deleted file mode 100644 index bc316bd2a..000000000 --- a/bbdev/api/tests/test_api_regression_buildbitstream.py +++ /dev/null @@ -1,3 +0,0 @@ -from _api_test_helper import run_bbdev_case - -run_bbdev_case("bbdev regression --buildbitstream '--chip pebble'") diff --git a/bbdev/api/tests/test_api_regression_eval_area_power.py b/bbdev/api/tests/test_api_regression_eval_area_power.py deleted file mode 100644 index e303dc2bd..000000000 --- a/bbdev/api/tests/test_api_regression_eval_area_power.py +++ /dev/null @@ -1,3 +0,0 @@ -from _api_test_helper import run_bbdev_case - -run_bbdev_case("bbdev regression --eval-area-power '--chip pebble'") diff --git a/bbdev/api/tests/test_bemu_binary.py b/bbdev/api/tests/test_bemu_binary.py new file mode 100644 index 000000000..b52734427 --- /dev/null +++ b/bbdev/api/tests/test_bemu_binary.py @@ -0,0 +1,50 @@ +import importlib.util +from pathlib import Path +import tempfile +import unittest + + +API = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location( + "bemu_sim_event", API / "steps/bebop/bemu/02_sim_event.step.py" +) +event = importlib.util.module_from_spec(spec) +spec.loader.exec_module(event) + + +class BemuBinaryResolutionTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + + def binary(self, relative): + path = self.root / "bb-tests/output" / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + return str(path) + + def test_same_model_image_on_two_chips(self): + for chip in ("toy", "pebble"): + self.binary(f"kernel/{chip}/fw_payload-alexnet.elf") + for chip in ("toy", "pebble"): + expected = self.root / f"bb-tests/output/kernel/{chip}/fw_payload-alexnet.elf" + self.assertEqual( + event.resolve_bemu_binary(str(self.root), chip, expected.name), + str(expected), + ) + + def test_workload_must_belong_to_requested_chip(self): + path = self.binary("toy/workloads/src/shared-test") + self.assertEqual(event.resolve_bemu_binary(str(self.root), "toy", "shared-test"), path) + self.assertIsNone(event.resolve_bemu_binary(str(self.root), "pebble", "shared-test")) + + def test_ambiguous_workload_within_chip_is_an_error(self): + self.binary("toy/workloads/src/one/shared-test") + self.binary("toy/workloads/src/two/shared-test") + with self.assertRaises(ValueError): + event.resolve_bemu_binary(str(self.root), "toy", "shared-test") + + +if __name__ == "__main__": + unittest.main() diff --git a/bbdev/api/tests/test_p2e_resume.py b/bbdev/api/tests/test_p2e_resume.py new file mode 100644 index 000000000..0fdd16109 --- /dev/null +++ b/bbdev/api/tests/test_p2e_resume.py @@ -0,0 +1,67 @@ +import importlib.util +from pathlib import Path +import tempfile +import sys +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch + + +API = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(API)) +spec = importlib.util.spec_from_file_location( + "p2e_build_api", API / "steps/bebop/p2e/02_buildbitstream_api.step.py" +) +event = importlib.util.module_from_spec(spec) +spec.loader.exec_module(event) + + +class P2eResumeTest(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.case = self.root / "bebop/build/toy-test" + self.routed = self.case / "fpgaCompDir/part_b0_f0/pnrDir" + self.routed.mkdir(parents=True) + self.artifacts = [self.routed / name for name in ( + "xepic_vvac_top_0_0_route.dcp", "xepic_vvac_top_0_0.bit" + )] + for path in self.artifacts: + path.write_bytes(b"existing artifact") + self.context = SimpleNamespace(enqueue=AsyncMock(), trace_id="test") + + async def request(self, **options): + body = {"chip": "toy", "diff": True, "resume-post-route": True, + "vsrc_dir": str(self.root / "rtl"), "output_dir": str(self.case)} + body.update(options) + with patch.object(event, "get_buckyball_path", return_value=str(self.root)): + return await event.handler(SimpleNamespace(body=body), self.context) + + async def test_routed_case_is_enqueued_without_mutation(self): + response = await self.request() + self.assertEqual(response.status, 202) + data = self.context.enqueue.await_args.args[0]["data"] + self.assertTrue(data["resume_post_route"]) + for path in self.artifacts: + self.assertEqual(path.read_bytes(), b"existing artifact") + + async def test_incomplete_case_is_rejected_before_enqueue(self): + self.artifacts[1].unlink() + response = await self.request() + self.assertEqual(response.status, 400) + self.context.enqueue.assert_not_awaited() + + async def test_resume_requires_explicit_case(self): + response = await self.request(output_dir=None) + self.assertEqual(response.status, 400) + self.context.enqueue.assert_not_awaited() + + async def test_case_outside_build_root_is_rejected(self): + response = await self.request(output_dir=str(self.root)) + self.assertEqual(response.status, 400) + self.context.enqueue.assert_not_awaited() + + +if __name__ == "__main__": + unittest.main() diff --git a/bbdev/api/tests/test_p2e_runtime_reuse.py b/bbdev/api/tests/test_p2e_runtime_reuse.py new file mode 100644 index 000000000..2657e0a6b --- /dev/null +++ b/bbdev/api/tests/test_p2e_runtime_reuse.py @@ -0,0 +1,54 @@ +from pathlib import Path +import tempfile +import sys +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from steps.bebop.p2e.scripts.runtime_case import validate_runtime_reuse + + +class RuntimeReuseTest(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.case = Path(self.directory.name) + self.bit = self.case / "fpgaCompDir/bitstream.bit" + for path in (self.bit, self.case / "bebop-p2e", + self.case / "vvacDir/runtimeDir/rtcfg", + self.case / "vvacDir/runtimeDir/lib/lib_arm/libvCtb.so"): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"existing artifact") + self.mode = self.case / "p2e_trace_mode" + self.mode.write_text("btrace") + (self.case / "bebop-p2e").chmod(0o755) + + def test_valid_case_remains_unchanged(self): + before = {p: p.read_bytes() for p in self.case.rglob("*") if p.is_file()} + self.assertEqual(validate_runtime_reuse(str(self.bit), True), + self.case / "bebop-p2e") + self.assertEqual(before, {p: p.read_bytes() for p in self.case.rglob("*") if p.is_file()}) + + def test_missing_library_is_rejected(self): + (self.case / "vvacDir/runtimeDir/lib/lib_arm/libvCtb.so").unlink() + with self.assertRaisesRegex(ValueError, "incomplete"): + validate_runtime_reuse(str(self.bit), True) + + def test_diff_requires_trace_capability(self): + self.mode.write_text("none") + with self.assertRaisesRegex(ValueError, "does not support diff"): + validate_runtime_reuse(str(self.bit), True) + validate_runtime_reuse(str(self.bit), False) + + def test_unknown_protocol_is_rejected(self): + self.mode.write_text("unknown") + with self.assertRaisesRegex(ValueError, "Unknown"): + validate_runtime_reuse(str(self.bit), False) + + def test_nonexecutable_runtime_is_rejected(self): + (self.case / "bebop-p2e").chmod(0o644) + with self.assertRaisesRegex(ValueError, "not executable"): + validate_runtime_reuse(str(self.bit), True) + + +if __name__ == "__main__": + unittest.main() diff --git a/bbdev/api/tests/test_uvm_ip_profiles.py b/bbdev/api/tests/test_uvm_ip_profiles.py new file mode 100644 index 000000000..dbfdf4f62 --- /dev/null +++ b/bbdev/api/tests/test_uvm_ip_profiles.py @@ -0,0 +1,51 @@ +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from steps.uvm.scripts.uvm_common import load_ip +from steps.uvm.scripts.report import verification_targets + + +class IpProfiles(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.repo = Path(self.temp.name) + registry = self.repo / "arch/uvm.toml" + registry.parent.mkdir(parents=True) + registry.write_text('ips=["unit"]\n[paths]\nunit="ip"\n') + self.resources = self.repo / "ip/src/main/resources" + self.resources.mkdir(parents=True) + (self.resources / "common.f").write_text("") + (self.resources / "goban.f").write_text("") + (self.resources / "goban.toml").write_text('chips=["goban"]\n') + + def tearDown(self): + self.temp.cleanup() + + def test_build_and_report_choose_same_profile(self): + for chip, names in [("toy", ["common"]), ("goban", ["common", "goban"])]: + self.assertEqual([x["name"] for x in load_ip(str(self.repo), "unit", chip=chip)["targets"]], names) + self.assertEqual(verification_targets(self.repo, chip, ip="unit"), [f"ip/unit/{n}" for n in names]) + with self.assertRaisesRegex(ValueError, "does not support chip"): + load_ip(str(self.repo), "unit", "goban", "toy") + + def test_profile_and_negative_assertion_can_combine(self): + (self.resources / "goban.toml").write_text('chips=["goban"]\nexpected_assertion="precise failure"\n') + target = load_ip(str(self.repo), "unit", "goban", "goban")["targets"][0] + self.assertEqual(target["expected_assertion"], "precise failure") + + def test_invalid_metadata_and_empty_selection_fail(self): + for value in ['[]', '[""]', '[" goban"]', '[7]', '"goban"']: + (self.resources / "goban.toml").write_text(f"chips={value}\n") + with self.assertRaisesRegex(ValueError, "nonempty list"): + load_ip(str(self.repo), "unit", chip="goban") + (self.resources / "goban.toml").write_text('chips=["goban"]\n') + (self.resources / "common.toml").write_text('chips=["goban"]\n') + with self.assertRaisesRegex(ValueError, "no applicable targets"): + load_ip(str(self.repo), "unit", chip="toy") + + +if __name__ == "__main__": + unittest.main() diff --git a/bbdev/api/utils/process.py b/bbdev/api/utils/process.py index f0039659f..7d5d45fdc 100644 --- a/bbdev/api/utils/process.py +++ b/bbdev/api/utils/process.py @@ -188,8 +188,10 @@ def stop_workspace_servers( port: int | None = None, worker_url: str | None = None, ) -> int: - pids = _workspace_pids(workflow_dir, worker_url=worker_url) - pids.update(_port_pids(workflow_dir, port, worker_url=worker_url)) + if port is None: + pids = _workspace_pids(workflow_dir, worker_url=worker_url) + else: + pids = _port_pids(workflow_dir, port, worker_url=worker_url) if not pids: return 0 diff --git a/bbdev/api/utils/report_schema.py b/bbdev/api/utils/report_schema.py new file mode 100644 index 000000000..f0740164f --- /dev/null +++ b/bbdev/api/utils/report_schema.py @@ -0,0 +1,183 @@ +import math +import re +from datetime import datetime, timezone +from pathlib import Path + +STATUSES = {"pass", "fail", "timeout", "infra_error"} +CATEGORIES = { + "Conv", + "MatMul", + "Transfer", + "Quantization", + "Activation", + "Pool", + "Other", +} + + +def report_directory(run): + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", run["chip"]): + raise ValueError("chip must be a safe directory name") + timestamp = datetime.fromisoformat(run["timestamp"].replace("Z", "+00:00")) + if timestamp.utcoffset() is None: + raise ValueError("timestamp must include a timezone") + folder = timestamp.astimezone(timezone.utc).strftime("%Y-%m-%d-%H-%M") + return Path(run["chip"]) / folder + + +def nonempty(value, label): + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{label} must be a non-empty string") + return value + + +def number(value, label, *, positive=False, integer=False): + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + ): + raise ValueError(f"{label} must be a finite number") + if ( + value < 0 + or (positive and value == 0) + or (integer and (value != int(value) or value > 2**53 - 1)) + ): + raise ValueError(f"invalid {label}: {value}") + return value + + +def validate_run(run): + if run["schema_version"] != 2: + raise ValueError("invalid schema_version") + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", run["id"]): + raise ValueError("run id must be a safe file name") + stamp = datetime.fromisoformat(run["timestamp"].replace("Z", "+00:00")) + if stamp.utcoffset() is None: + raise ValueError("timestamp must include a timezone") + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", run["repository"]): + raise ValueError("repository must be owner/name") + if not re.fullmatch(r"[0-9a-f]{40}", run["commit"]): + raise ValueError("commit must be a full 40-character SHA") + for key in ("branch", "chip", "backend", "config_hash", "producer"): + nonempty(run[key], key) + if run["clock_hz"] is not None: + number(run["clock_hz"], "clock_hz", positive=True) + if run["area"] is not None: + area = run["area"] + if area["status"] != "pass": + raise ValueError("area measurement requires a successful DC result") + number(area["cell_um2"], "cell_um2", positive=True) + if not isinstance(area["constraints_met"], bool): + raise ValueError("area constraints_met must be a boolean") + for key in ("scope", "conditions", "report_url", "constraints_report_url"): + nonempty(area[key], f"area {key}") + if len(set(run["verification_targets"])) != len(run["verification_targets"]): + raise ValueError("duplicate expected verification target") + for target in run["verification_targets"]: + nonempty(target, "expected verification target") + targets = set() + for result in run["verification"]: + target = nonempty(result["target"], "verification target") + if target in targets: + raise ValueError(f"duplicate verification target: {target}") + targets.add(target) + if result["status"] not in STATUSES | {"not_run"}: + raise ValueError("invalid verification status") + if result["coverage"] is not None: + if not result["coverage"]: + raise ValueError("empty coverage measurements") + for metric, value in result["coverage"].items(): + nonempty(metric, "coverage metric") + if number(value, "coverage percent") > 100: + raise ValueError("coverage must be in [0,100]") + nonempty(result["report_url"], "coverage report_url") + if targets - set(run["verification_targets"]): + raise ValueError("verification results contain undeclared targets") + names = set() + for model in run["models"]: + if model["latency_ms"] is not None: + number(model["latency_ms"], "latency_ms", positive=True) + if model["throughput"] is not None: + number(model["throughput"]["value"], "throughput", positive=True) + nonempty(model["throughput"]["unit"], "throughput unit") + name = nonempty(model["name"], "model name") + if name in names: + raise ValueError(f"duplicate model: {name}") + names.add(name) + for key in ("workload_hash", "dataset_hash"): + nonempty(model[key], key) + if model["status"] not in STATUSES: + raise ValueError(f"unknown status: {model['status']}") + if model["accuracy"] is not None: + accuracy = number(model["accuracy"], "accuracy") + if accuracy > 1: + raise ValueError("accuracy must be in [0,1]") + nonempty(model["accuracy_metric"], "accuracy_metric") + if model["status"] == "pass": + number(model["cycles"], "cycles", positive=True, integer=True) + if model["trace_url"] is not None and not model["operators"]: + raise ValueError("trace attachment requires operator measurements") + if model["error"] is not None: + raise ValueError("passing model cannot contain an error") + else: + nonempty(model["error"], "model error") + if model["cycles"] is not None or model["operators"]: + raise ValueError( + "failed models cannot contain performance measurements" + ) + ids = set() + for op in model["operators"]: + nonempty(op["id"], "operator id") + nonempty(op["name"], "operator name") + if op["id"] in ids: + raise ValueError("duplicate operator id") + ids.add(op["id"]) + if op["category"] not in CATEGORIES: + raise ValueError("unknown operator category") + for key in ("cycles", "start_cycle", "level"): + number(op[key], key, integer=True) + number(op["calls"], "calls", positive=True, integer=True) + if op["start_cycle"] + op["cycles"] > model["cycles"]: + raise ValueError("operator interval exceeds model trace span") + if run["submission_errors"] != submission_errors(run): + raise ValueError("submission status does not match the report measurements") + return run + + +def submission_errors(run): + missing = [] + if run["area"] is None: + missing.append("Area：缺少面积报告") + elif not run["area"]["constraints_met"]: + missing.append("Area：综合约束未满足") + if not run["models"]: + missing.append("Performance：没有模型结果") + for model in run["models"]: + if model["status"] != "pass": + missing.append(f"Performance：{model['name']} 未通过") + for key, label in ( + ("accuracy", "精度"), + ("latency_ms", "延迟"), + ("throughput", "吞吐"), + ): + if model[key] is None: + missing.append(f"Performance：{model['name']} 缺少{label}") + expected = set(run["verification_targets"]) + actual = {result["target"] for result in run["verification"]} + if not expected or expected != actual: + missing.append("Coverage：验证项不完整") + for result in run["verification"]: + if result["status"] != "pass" or result["coverage"] is None: + missing.append(f"Coverage：{result['target']} 未通过或缺少覆盖率报告") + if run.get("provenance", {}).get("source_dirty", False): + missing.append("源码包含未提交修改") + return missing + + +def validate_submission(run): + validate_run(run) + missing = submission_errors(run) + if missing: + raise ValueError(f"submission blocked for {run['id']}: {'; '.join(missing)}") + return run diff --git a/bbdev/api/utils/reports.py b/bbdev/api/utils/reports.py new file mode 100644 index 000000000..36e8539f8 --- /dev/null +++ b/bbdev/api/utils/reports.py @@ -0,0 +1,125 @@ +import hashlib +import json +import shutil +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +from utils.report_schema import ( + report_directory, + submission_errors, + validate_run, +) + + +def source_context(repo, chip): + repo = Path(repo) + + def plain(value): + if isinstance(value, dict): + return {key: plain(item) for key, item in value.items() if key != "_file"} + if isinstance(value, list): + return [plain(item) for item in value] + return value + + config = json.loads( + ( + repo / f"examples/chips/{chip}/configs/generated/config/config.json" + ).read_text() + ) + + def git(*args): + return subprocess.check_output(["git", *args], cwd=repo, text=True).strip() + + return { + "repository": "DangoSys/buckyball", + "branch": git("branch", "--show-current") or git("rev-parse", "HEAD"), + "commit": git("rev-parse", "HEAD"), + "chip": chip, + "config_hash": hashlib.sha256( + json.dumps(plain(config), sort_keys=True).encode() + ).hexdigest(), + "provenance": { + "source_dirty": bool( + git( + "status", + "--porcelain", + "--untracked-files=all", + "--ignore-submodules=none", + ) + ), + "workflow_url": None, + }, + } + + +def new_report(context, producer, trace_id, backend): + now = datetime.now(timezone.utc) + return { + "schema_version": 2, + "id": f"{context['chip']}-{producer.replace('.', '-')}-{trace_id}", + "timestamp": now.isoformat().replace("+00:00", "Z"), + **context, + "producer": producer, + "backend": backend, + "clock_hz": None, + "area": None, + "verification_targets": [], + "verification": [], + "models": [], + "artifacts": {}, + } + + +def attachment(report, directory, source, name): + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + source = Path(source).resolve(strict=True) + target = directory / name + if target.exists(): + raise FileExistsError(target) + shutil.copyfile(source, target) + report["artifacts"][name] = hashlib.sha256(target.read_bytes()).hexdigest() + return f"data/{report_directory(report).as_posix()}/{name}" + + +def finish_report(report, directory): + report["submission_errors"] = submission_errors(report) + validate_run(report) + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + path = directory / "report.json" + temporary = directory / ".report.json.tmp" + temporary.write_text(json.dumps(report, indent=2) + "\n") + temporary.replace(path) + return str(path) + + +def verify_artifacts(report, directory): + prefix = f"data/{report_directory(report).as_posix()}/" + references = [] + if report["area"] is not None: + references.extend( + [report["area"]["report_url"], report["area"]["constraints_report_url"]] + ) + references.extend( + row["report_url"] + for row in report["verification"] + if row["report_url"] is not None + ) + references.extend( + model["trace_url"] + for model in report["models"] + if model["trace_url"] is not None + ) + for url in references: + if not url.startswith(prefix): + raise ValueError("attachment URL does not belong to this report") + name = url[len(prefix) :] + if name not in report["artifacts"]: + raise ValueError(f"unregistered report attachment: {name}") + for name, digest in report["artifacts"].items(): + if Path(name).name != name: + raise ValueError("attachment name must be a file name") + if hashlib.sha256((Path(directory) / name).read_bytes()).hexdigest() != digest: + raise ValueError(f"attachment hash mismatch: {name}") diff --git a/bbdev/bbdev b/bbdev/bbdev index d87bebfe0..d306caf50 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -48,7 +48,7 @@ WORKFLOW_COMMANDS = { "build": 'Build model recipes (all chip models when --model is omitted). Args: "--chip [--model ] [--ctrace] [--dtrace]"', }, "kernel": { - "build": 'Build RISC-V kernel + rootfs. Args: "[--chip ] [--model ] [--interactive] [--visible-hart-count ] [--total-hart-count ]"; --model requires --chip', + "build": 'Build RISC-V kernel + rootfs. Args: "[--chip ] [--model ] [--interactive] [--visible-hart-count ] [--total-hart-count ] [--guest-memory-mib ] [--model-storage initramfs|pmem]"; --model requires --chip', }, "firesim": { "enumeratefpgas": 'Enumerate FPGAs. Args: "--chip "', @@ -60,7 +60,7 @@ WORKFLOW_COMMANDS = { "build": 'Build compiler. Args: "--chip "', }, "bebop-bemu": { - "sim": 'Run Bebop emulator simulation. Model args: "--chip --model ". ELF args: "--chip --binary [--pk] [--disasm] [--tool-profile] [--itrace] [--mtrace]"', + "sim": 'Run Bebop emulator simulation. Model args: "--chip --model [--reuse-simulator]". ELF args: "--chip --binary [--pk] [--disasm] [--tool-profile] [--itrace] [--mtrace]"; resolves files in the selected chip workload/kernel outputs', "analysis": 'Analyze itrace/mtrace in an existing log dir. Args: "--chip --log-dir --itrace|--mtrace"', "batch": 'Run Bebop BEMU batch regression. Args: "--chip --test [--clean-before]"', }, @@ -80,8 +80,8 @@ WORKFLOW_COMMANDS = { "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--chip [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--chip [--diff] [--output-dir ]"', - "buildbitstream": 'Build Bebop P2E runtime case. Args: "--chip [--diff] [--vsrc-dir ] [--output-dir ]"', - "runworkload": 'Run Bebop P2E workload. Args: "--chip --image --bitstream [--fpga-location ] [--multi-fpga] [--wave] [--wave-start ] [--diff] [--itrace] [--mtrace] [--pmctrace]"', + "buildbitstream": 'Build Bebop P2E runtime case. Args: "--chip [--diff] [--vsrc-dir ] [--output-dir ] [--resume-post-route | --stop-after vsyn|vcom]"; stop-after produces resource assessment only, not a runnable case; resume requires an existing routed case and preserves its completed synthesis/PNR', + "runworkload": 'Cold-load Bebop P2E workload (requires p2e-cold-load-v1 case). Args: "--chip (--image | --load-manifest ) --bitstream [--fpga-location ] [--multi-fpga] [--wave] [--wave-start ] [--diff (image only)] [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--reuse-runtime]"', "batch": 'Run Bebop P2E batch regression. Args: "--chip --bitstream --test [--diff]"', }, "dc": { @@ -100,18 +100,12 @@ WORKFLOW_COMMANDS = { }, "uvm": { "verilog": 'Generate isolated Ball RTL for UVM. Args: "--balltype --output-dir "', - "build": 'Build a Ball UVM simulation. Args: "--ball= [--filelist ]"', - "run": 'Build and run a Ball UVM simulation. Args: "--ball= [--filelist ] [--test ]"', + "build": 'Build a Ball or IP UVM simulation. Args: "--chip [--ball | --ip [--target ]]"', + "run": 'Build and run a Ball or IP UVM simulation. Args: "--chip [--ball | --ip [--target ]]"', }, "config": { "install": True, }, - "regression": { - "buildbitstream": 'Build pebble p2e bitstream. Args: "--chip [--output-dir ]"', - "check": 'P2E pk-tests. Args: "--chip --bitstream [--chip ]"', - "eval-performance": 'P2E model perfetto cycles. Args: "--chip --bitstream [--chip ]"', - "eval-area-power": 'DC area+freq. Args: "--chip [--top ]"', - }, } def make_iii_config( @@ -358,6 +352,11 @@ def _parse_sub_args(arg_string: str) -> dict: i += 1 else: raise ValueError(f"Unexpected argument: {token}") + if "arguments" in args_dict: + values = json.loads(args_dict["arguments"]) + if not isinstance(values, list) or not all(isinstance(value, str) for value in values): + raise ValueError("arguments must be a JSON list of strings") + args_dict["arguments"] = values return args_dict diff --git a/bbdev/mcp/common.py b/bbdev/mcp/common.py index 48cedb1de..1415eb725 100644 --- a/bbdev/mcp/common.py +++ b/bbdev/mcp/common.py @@ -4,6 +4,7 @@ import atexit import json +import importlib.util import os import shutil import socket @@ -369,14 +370,13 @@ def _balldomain_path(chip: str, balldomain: Optional[str]) -> Path: return (core_file.parent / include).resolve() -def _validate(path: Path) -> Dict[str, Any]: - cfg = _load_toml(path) +def _validate_domain(path: Path, cfg: Dict[str, Any]) -> Dict[str, Any]: mappings = cfg.get("ballIdMappings", []) if not isinstance(mappings, list): raise ValueError(f"{path}: ballIdMappings must be an array") isa = cfg.get("ballISA") - if not isinstance(isa, list) or not isa: - raise ValueError(f"{path}: missing or empty ballISA") + if not isinstance(isa, list): + raise ValueError(f"{path}: missing ballISA") for i, e in enumerate(isa): if not isinstance(e, dict): raise ValueError(f"{path}: ballISA[{i}] must be a table") @@ -394,6 +394,8 @@ def _validate(path: Path) -> Dict[str, Any]: missing_config = [] bad_bw = [] for m in mappings: + if m.get("builtin") == "kernel": + continue # The shared normalizer validates the complete builtin contract. name = m.get("ballName") if not m.get("ballClass"): bad_bw.append({"ballName": name, "error": "missing ballClass"}) @@ -405,18 +407,10 @@ def _validate(path: Path) -> Dict[str, Any]: or out_bw <= 0 ): bad_bw.append({"ballName": name, "inBW": in_bw, "outBW": out_bw}) - cfg_rel = m.get("config") - if not isinstance(cfg_rel, str) or not cfg_rel: - missing_config.append( - {"ballName": name, "config": cfg_rel, "error": "missing"} - ) - continue - cfg_path = (path.parent / cfg_rel).resolve() + config = m.get("config") + cfg_path = Path(config["_file"]) if not cfg_path.is_file(): - missing_config.append( - {"ballName": name, "config": cfg_rel, "resolved": str(cfg_path)} - ) - continue + missing_config.append({"ballName": name, "resolved": str(cfg_path)}) funct7s = [e.get("funct7") for e in isa] mnemonics = [e.get("mnemonic") for e in isa] bids = [e.get("bid") for e in isa] @@ -456,7 +450,7 @@ def dups(xs): "pass": not missing_config, "missing": missing_config, }, - "bandwidth_positive": {"pass": not bad_bw, "invalid": bad_bw}, + "bandwidth_contract": {"pass": not bad_bw, "invalid": bad_bw}, } id_to_isa: Dict[Any, list] = {} @@ -467,6 +461,7 @@ def dups(xs): "ballId": m.get("ballId"), "ballName": m.get("ballName"), "ballClass": m.get("ballClass"), + "builtin": m.get("builtin"), "inBW": m.get("inBW"), "outBW": m.get("outBW"), "config": m.get("config"), @@ -486,6 +481,37 @@ def dups(xs): } +def _validate_chip(chip: str, balldomain: Optional[str]) -> Dict[str, Any]: + scripts = repo_path() / "bbdev/api/steps/config/scripts" + sys.path.insert(0, str(scripts)) + from ball_normalize import normalize_ball_domain + spec = importlib.util.spec_from_file_location("ball_validation_topology", scripts / "2_parameter_derivation.py") + topology = importlib.util.module_from_spec(spec) + spec.loader.exec_module(topology) + expanded = repo_path() / "examples/chips" / chip / "configs/generated/config/config.json" + config = json.loads(expanded.read_text(encoding="utf-8")) + selected = _balldomain_path(chip, balldomain) if balldomain is not None else None + results = [] + for index, core in enumerate(topology.iter_cores(config["designs"])): + raw = core.get("balldomain") + if not isinstance(raw, dict) or not raw.get("_file"): + raise ValueError(f"Core {index}: missing expanded BallDomain") + path = Path(raw["_file"]).resolve() + if selected is not None and path != selected: + continue + try: + domain = normalize_ball_domain(core) + if domain is None: + raise ValueError("missing effective BallDomain") + result = _validate_domain(path, domain) + except ValueError as error: + result = {"passed": False, "error": str(error)} + results.append({"core_index": index, "core_config": core["_file"], **result}) + if not results: + raise ValueError(f"No expanded core matches BallDomain {selected}") + return {"chip": chip, "passed": all(result["passed"] for result in results), "cores": results} + + atexit.register(_stop) @@ -497,4 +523,4 @@ def dups(xs): need = _need opt = _opt balldomain_path = _balldomain_path -validate_toml = _validate +validate_balldomains = _validate_chip diff --git a/bbdev/mcp/tests/test_common.py b/bbdev/mcp/tests/test_common.py index 03afd25ed..f5e855ca0 100644 --- a/bbdev/mcp/tests/test_common.py +++ b/bbdev/mcp/tests/test_common.py @@ -140,7 +140,7 @@ def test_task_status_reports_nested_processing(self, read_state): read_state.return_value = { "processing": { "processing": True, - "task": "regression.buildbitstream", + "task": "bebop.p2e.buildbitstream", "chip": "pebble", } } @@ -149,7 +149,7 @@ def test_task_status_reports_nested_processing(self, read_state): common.task_status("trace-1"), { "processing": True, - "task": "regression.buildbitstream", + "task": "bebop.p2e.buildbitstream", "chip": "pebble", "accepted": True, "trace_id": "trace-1", diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index c532545ff..b43622d2e 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -44,10 +44,6 @@ from . import yosys_run from . import yosys_verilog from . import yosys_synth -from . import regression_buildbitstream -from . import regression_check -from . import regression_eval_performance -from . import regression_eval_area_power def modules(): return [ @@ -93,10 +89,6 @@ def modules(): yosys_run, yosys_verilog, yosys_synth, - regression_buildbitstream, - regression_check, - regression_eval_performance, - regression_eval_area_power, ] diff --git a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py index 52423bd16..2616341ca 100644 --- a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py +++ b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py @@ -14,6 +14,8 @@ def bbdev_bebop_p2e_buildbitstream( diff: bool = False, vsrc_dir: Optional[str] = None, output_dir: Optional[str] = None, + resume_post_route: bool = False, + stop_after: Optional[str] = None, ) -> str: """Build bebop-p2e bitstream. POST /bebop/p2e/buildbitstream.""" if e := need("chip", chip): @@ -22,9 +24,10 @@ def bbdev_bebop_p2e_buildbitstream( submit( "/bebop/p2e/buildbitstream", opt( - {"chip": chip, "diff": diff}, + {"chip": chip, "diff": diff, "resume_post_route": resume_post_route}, vsrc_dir=vsrc_dir, output_dir=output_dir, + stop_after=stop_after, ), ) ) diff --git a/bbdev/mcp/tools/bebop_p2e_runworkload.py b/bbdev/mcp/tools/bebop_p2e_runworkload.py index 41cd24826..59e52b296 100644 --- a/bbdev/mcp/tools/bebop_p2e_runworkload.py +++ b/bbdev/mcp/tools/bebop_p2e_runworkload.py @@ -11,24 +11,38 @@ def register(mcp): @mcp.tool() def bbdev_bebop_p2e_runworkload( chip: str, - image: str, - bitstream: str, + image: str = "", + bitstream: str = "", itrace: bool = False, mtrace: bool = False, pmctrace: bool = False, + ctrace: bool = False, + reuse_runtime: bool = False, diff: bool = False, + load_manifest: str = "", ) -> str: - """Run one image on bebop-p2e FPGA. POST /bebop/p2e/runworkload.""" - for n, v in (("chip", chip), ("image", image), ("bitstream", bitstream)): + """Cold-load one hex image or a validated binary load manifest on P2E.""" + if bool(image) == bool(load_manifest): + return err("exactly one of image and load_manifest is required") + if load_manifest and diff: + return err("load_manifest cannot use single-ELF diff") + for n, v in (("chip", chip), ("bitstream", bitstream)): if e := need(n, v): return err(e) params: Dict[str, Any] = {"chip": chip, "image": image, "bitstream": bitstream} + if load_manifest: + params.pop("image") + params["load-manifest"] = load_manifest if itrace: params["itrace"] = True if mtrace: params["mtrace"] = True + if reuse_runtime: + params["reuse-runtime"] = True if pmctrace: params["pmctrace"] = True + if ctrace: + params["ctrace"] = True if diff: params["diff"] = True return fmt(submit("/bebop/p2e/runworkload", params)) diff --git a/bbdev/mcp/tools/bebop_verilator_sim.py b/bbdev/mcp/tools/bebop_verilator_sim.py index 2c0b70cd8..15c93e522 100644 --- a/bbdev/mcp/tools/bebop_verilator_sim.py +++ b/bbdev/mcp/tools/bebop_verilator_sim.py @@ -18,6 +18,7 @@ def bbdev_bebop_verilator_sim( banktrace: bool = False, no_wave: bool = False, batch: bool = False, + diff: bool = False, ) -> str: """Run one workload on bebop-verilator. POST /bebop/verilator/sim.""" for n, v in (("binary", binary), ("chip", chip)): @@ -33,5 +34,6 @@ def bbdev_bebop_verilator_sim( "banktrace": banktrace, "no-wave": no_wave, "batch": batch, + "diff": diff, } return fmt(submit("/bebop/verilator/sim", params)) diff --git a/bbdev/mcp/tools/bemu_sim.py b/bbdev/mcp/tools/bemu_sim.py index 8ab005501..7269b2549 100644 --- a/bbdev/mcp/tools/bemu_sim.py +++ b/bbdev/mcp/tools/bemu_sim.py @@ -10,7 +10,7 @@ def register(mcp): @mcp.tool() def bbdev_bemu_sim( chip: str, - binary: str, + binary: str | None = None, pk: bool = False, disasm: bool = False, tool_profile: bool = False, @@ -18,8 +18,20 @@ def bbdev_bemu_sim( mtrace: bool = False, core_index: int | None = None, arguments: list[str] | None = None, + model: str | None = None, + reuse_simulator: bool = False, ) -> str: - """Run one workload on bebop-bemu. POST /bebop/bemu/sim.""" + """Run a logical ELF file name from the selected chip's workload/kernel outputs, or a built model recipe, on BEMU. Model runs use running-param.toml. POST /bebop/bemu/sim.""" + if model is not None: + for n, v in (("chip", chip), ("model", model)): + if e := need(n, v): + return err(e) + if binary is not None or pk or disasm or tool_profile or itrace or mtrace or core_index is not None or arguments: + return err("model runs use running-param.toml; ELF options cannot be supplied") + return fmt(submit("/bebop/bemu/sim", { + "chip": chip, "model": model, "reuse-simulator": reuse_simulator})) + if reuse_simulator: + return err("reuse-simulator is supported for model runs only") for n, v in (("chip", chip), ("binary", binary)): if e := need(n, v): return err(e) diff --git a/bbdev/mcp/tools/kernel_build.py b/bbdev/mcp/tools/kernel_build.py index 951a80d01..27e7f6bc9 100644 --- a/bbdev/mcp/tools/kernel_build.py +++ b/bbdev/mcp/tools/kernel_build.py @@ -13,6 +13,8 @@ def bbdev_kernel_build( chip: Optional[str] = None, model: Optional[str] = None, interactive: bool = False, + guest_memory_mib: int = 512, + model_storage: str = "initramfs", visible_hart_count: Optional[int] = None, total_hart_count: Optional[int] = None, ) -> str: @@ -24,7 +26,7 @@ def bbdev_kernel_build( stack/models/build///artifact into fw_payload-. With --interactive: keep shared /init shell and do not auto-run. """ - params: Dict[str, Any] = {} + params: Dict[str, Any] = {"guest-memory-mib": guest_memory_mib, "model-storage": model_storage} opt(params, chip=chip, model=model) if interactive: params["interactive"] = True diff --git a/bbdev/mcp/tools/regression_buildbitstream.py b/bbdev/mcp/tools/regression_buildbitstream.py deleted file mode 100644 index 091632c51..000000000 --- a/bbdev/mcp/tools/regression_buildbitstream.py +++ /dev/null @@ -1,24 +0,0 @@ -"""MCP tool: bbdev_regression_buildbitstream.""" - -from __future__ import annotations - -from typing import Optional - -from common import submit, err, fmt, need, opt - - -def register(mcp): - @mcp.tool() - def bbdev_regression_buildbitstream( - chip: str, - output_dir: Optional[str] = None, - ) -> str: - """Build p2e bitstream. POST /regression/buildbitstream.""" - if e := need("chip", chip): - return err(e) - return fmt( - submit( - "/regression/buildbitstream", - opt({"chip": chip}, output_dir=output_dir), - ) - ) diff --git a/bbdev/mcp/tools/regression_check.py b/bbdev/mcp/tools/regression_check.py deleted file mode 100644 index 9acee1625..000000000 --- a/bbdev/mcp/tools/regression_check.py +++ /dev/null @@ -1,24 +0,0 @@ -"""MCP tool: bbdev_regression_check.""" - -from __future__ import annotations - - -from common import submit, err, fmt, need - - -def register(mcp): - @mcp.tool() - def bbdev_regression_check( - chip: str, - bitstream: str, - ) -> str: - """P2E pk-tests. POST /regression/check.""" - for n, v in (("chip", chip), ("bitstream", bitstream)): - if e := need(n, v): - return err(e) - return fmt( - submit( - "/regression/check", - {"chip": chip, "bitstream": bitstream}, - ) - ) diff --git a/bbdev/mcp/tools/regression_eval_area_power.py b/bbdev/mcp/tools/regression_eval_area_power.py deleted file mode 100644 index a402b947b..000000000 --- a/bbdev/mcp/tools/regression_eval_area_power.py +++ /dev/null @@ -1,24 +0,0 @@ -"""MCP tool: bbdev_regression_eval_area_power.""" - -from __future__ import annotations - -from typing import Optional - -from common import submit, err, fmt, need, opt - - -def register(mcp): - @mcp.tool() - def bbdev_regression_eval_area_power( - chip: str, - top: Optional[str] = None, - ) -> str: - """DC area+freq. POST /regression/eval-area-power.""" - if e := need("chip", chip): - return err(e) - return fmt( - submit( - "/regression/eval-area-power", - opt({"chip": chip}, top=top), - ) - ) diff --git a/bbdev/mcp/tools/regression_eval_performance.py b/bbdev/mcp/tools/regression_eval_performance.py deleted file mode 100644 index 71bd1740c..000000000 --- a/bbdev/mcp/tools/regression_eval_performance.py +++ /dev/null @@ -1,24 +0,0 @@ -"""MCP tool: bbdev_regression_eval_performance.""" - -from __future__ import annotations - - -from common import submit, err, fmt, need - - -def register(mcp): - @mcp.tool() - def bbdev_regression_eval_performance( - chip: str, - bitstream: str, - ) -> str: - """P2E model perfetto latency. POST /regression/eval-performance.""" - for n, v in (("chip", chip), ("bitstream", bitstream)): - if e := need(n, v): - return err(e) - return fmt( - submit( - "/regression/eval-performance", - {"chip": chip, "bitstream": bitstream}, - ) - ) diff --git a/bbdev/mcp/tools/uvm_build.py b/bbdev/mcp/tools/uvm_build.py index 8132b7055..f5b85095f 100644 --- a/bbdev/mcp/tools/uvm_build.py +++ b/bbdev/mcp/tools/uvm_build.py @@ -10,7 +10,7 @@ def register(mcp): @mcp.tool() def bbdev_uvm_build( - chip: str, ball: Optional[str] = None, ip: Optional[str] = None + chip: str, ball: Optional[str] = None, ip: Optional[str] = None, target: Optional[str] = None ) -> str: """Build a Ball or IP UVM simulation. POST /uvm/build.""" if e := need("chip", chip): @@ -18,6 +18,10 @@ def bbdev_uvm_build( params = {"chip": chip} if ball and ip: return err("Parameters --ball and --ip are mutually exclusive") + if target is not None: + if not ip or not target.strip(): + return err("Parameter --target requires --ip and a target name") + params["target"] = target if ball: params["ball"] = ball if ip: diff --git a/bbdev/mcp/tools/uvm_run.py b/bbdev/mcp/tools/uvm_run.py index c285dbf80..5c40e49d8 100644 --- a/bbdev/mcp/tools/uvm_run.py +++ b/bbdev/mcp/tools/uvm_run.py @@ -10,7 +10,7 @@ def register(mcp): @mcp.tool() def bbdev_uvm_run( - chip: str, ball: Optional[str] = None, ip: Optional[str] = None + chip: str, ball: Optional[str] = None, ip: Optional[str] = None, target: Optional[str] = None ) -> str: """Build and run a Ball or IP UVM simulation. POST /uvm/run.""" if e := need("chip", chip): @@ -18,6 +18,10 @@ def bbdev_uvm_run( params = {"chip": chip} if ball and ip: return err("Parameters --ball and --ip are mutually exclusive") + if target is not None: + if not ip or not target.strip(): + return err("Parameter --target requires --ip and a target name") + params["target"] = target if ball: params["ball"] = ball if ip: diff --git a/bbdev/mcp/tools/validate.py b/bbdev/mcp/tools/validate.py index c1accb82a..c14bada76 100644 --- a/bbdev/mcp/tools/validate.py +++ b/bbdev/mcp/tools/validate.py @@ -4,7 +4,7 @@ from typing import Optional -from common import balldomain_path, err, fmt, need, validate_toml +from common import err, fmt, need, validate_balldomains def register(mcp): @@ -14,7 +14,7 @@ def validate(chip: str = "toy", balldomain: Optional[str] = None) -> str: if e := need("chip", chip): return err(e) try: - return fmt(validate_toml(balldomain_path(chip, balldomain))) + return fmt(validate_balldomains(chip, balldomain)) except Exception as ex: return fmt( {"passed": False, "success": False, "failure": True, "error": str(ex)} From 44b3ed8b3d5a65ae40dbf722375efbd56a4e6874 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 4 Oct 2026 20:19:31 +0800 Subject: [PATCH 176/196] refactor: remove cold load case validation from P2E workload steps --- bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py | 10 +++++----- .../api/steps/bebop/p2e/03_runworkload_event.step.py | 3 +-- bbdev/api/steps/bebop/p2e/scripts/runtime_case.py | 11 ----------- bbdev/bbdev | 2 +- 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py index b34465116..1dbe86471 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py @@ -1,5 +1,7 @@ +from pathlib import Path + from motia import ApiRequest, ApiResponse, FlowContext, api -from steps.bebop.p2e.scripts.runtime_case import validate_runtime_reuse, validate_cold_load_case +from steps.bebop.p2e.scripts.runtime_case import validate_runtime_reuse config = { "name": "bebop-p2e-runworkload-api", @@ -31,10 +33,8 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: ) if manifest and body.get("diff", False): return ApiResponse(status=400, body={"error": "load-manifest cannot use single-ELF diff"}) - try: - validate_cold_load_case(bitstream) - except (ValueError, OSError, TypeError) as error: - return ApiResponse(status=400, body={"error": str(error)}) + if not Path(bitstream).is_file(): + return ApiResponse(status=400, body={"error": f"P2E bitstream is not a file: {bitstream}"}) if body.get("reuse-runtime", False): try: validate_runtime_reuse(bitstream, bool(body.get("diff", False))) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index 48a9f8809..62ecbd5a0 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -27,7 +27,7 @@ from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from resolve_image import resolve_image -from steps.bebop.p2e.scripts.runtime_case import validate_runtime_reuse, validate_cold_load_case +from steps.bebop.p2e.scripts.runtime_case import validate_runtime_reuse from steps.bebop.performance_report import performance_report from utils.reports import source_context from pathlib import Path @@ -152,7 +152,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: elf_path = f"{image_base}.elf" if os.path.isfile(f"{image_base}.elf") else image_base if diff and not os.path.isfile(elf_path): raise ValueError(f"workload ELF for P2E DiffTest not found: {elf_path}") - validate_cold_load_case(bitstream) except (ValueError, OSError, TypeError) as error: ctx.logger.error(str(error)) await check_result(ctx, 1, continue_run=False, diff --git a/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py b/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py index 13d83b108..869429508 100644 --- a/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py +++ b/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py @@ -2,17 +2,6 @@ import os -def validate_cold_load_case(bitstream: str) -> Path: - bit = Path(bitstream).resolve(strict=True) - if not bit.is_file(): - raise ValueError(f"P2E bitstream is not a file: {bit}") - case = bit.parent.parent - marker = case / "p2e-cold-load.cap" - if marker.read_text() != "p2e-cold-load-v1\n": - raise ValueError(f"P2E case does not support cold loading: {case}") - return case - - def validate_runtime_reuse(bitstream: str, diff: bool) -> Path: bit = Path(bitstream).resolve() case = bit.parent.parent diff --git a/bbdev/bbdev b/bbdev/bbdev index d306caf50..760b78ea9 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -81,7 +81,7 @@ WORKFLOW_COMMANDS = { "clean": 'Clean Bebop P2E build directory. Args: "--chip [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--chip [--diff] [--output-dir ]"', "buildbitstream": 'Build Bebop P2E runtime case. Args: "--chip [--diff] [--vsrc-dir ] [--output-dir ] [--resume-post-route | --stop-after vsyn|vcom]"; stop-after produces resource assessment only, not a runnable case; resume requires an existing routed case and preserves its completed synthesis/PNR', - "runworkload": 'Cold-load Bebop P2E workload (requires p2e-cold-load-v1 case). Args: "--chip (--image | --load-manifest ) --bitstream [--fpga-location ] [--multi-fpga] [--wave] [--wave-start ] [--diff (image only)] [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--reuse-runtime]"', + "runworkload": 'Cold-load Bebop P2E workload. Args: "--chip (--image | --load-manifest ) --bitstream [--fpga-location ] [--multi-fpga] [--wave] [--wave-start ] [--diff (image only)] [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--reuse-runtime]"', "batch": 'Run Bebop P2E batch regression. Args: "--chip --bitstream --test [--diff]"', }, "dc": { From 789a335b385c72449dd63b334a7da2604e3371e6 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 4 Oct 2026 21:03:26 +0800 Subject: [PATCH 177/196] refactor: streamline BEMU workflow by removing chip emulation checks and updating entry point handling --- .../api/steps/bebop/bemu/02_sim_event.step.py | 62 ++++++------------- .../steps/bebop/bemu/03_batch_event.step.py | 7 +-- .../steps/bebop/bemu/scripts/bemu_common.py | 21 ++----- .../api/steps/bebop/bemu/scripts/model_sim.py | 6 +- .../config/scripts/2_parameter_derivation.py | 10 +-- bbdev/api/steps/config/scripts/4_install.py | 19 ++++-- 6 files changed, 45 insertions(+), 80 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 2d5634941..c278814d2 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -28,7 +28,7 @@ from steps.bebop.bemu.scripts.model_sim import model_run_commands from utils.event_common import check_result, get_origin_trace_id from utils.process_registry import cancellation_requested -from bemu_common import bemu_manifest, bemu_tile_index, chip_emu_manifest +from bemu_common import bemu_chip_binary, bemu_manifest, bemu_tile_index from steps.bebop.performance_report import performance_report from utils.reports import source_context @@ -179,50 +179,24 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: run_log = log_dir(bbdir, chip, "verilog", timestamp, "bemu", binary_name) os.makedirs(run_log, exist_ok=True) core_index = input_data.get("core_index") - tile_index = bemu_tile_index(chip, bbdir) - chip_emu = chip_emu_manifest(chip, bbdir) - if (tile_index is not None) != bool(chip_emu): - ctx.logger.error( - f"chip {chip}: bundle bemu.chipMain and emu/Cargo.toml must both exist or both be absent" - ) - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": "chip_emu_entry_mismatch", "chip": chip}, - trace_id=origin_tid, - ) - return - if chip_emu and core_index is None: - cargo_args = [ - "cargo", - "run", - "--release", - "--manifest-path", - str(chip_emu), - "--", - "--tile-index", - str(tile_index), - "--elf", - binary_path, - "--log-dir", - run_log, - ] + # The chip's tile runner by default; --core-index selects the single-core bebop-bemu. + if core_index is None: + entry = [bemu_chip_binary(chip), "--", "--tile-index", str(bemu_tile_index(chip, bbdir))] else: - cargo_args = [ - "cargo", - "run", - "--release", - "--manifest-path", - str(bemu_cargo_manifest), - "--bin", - "bebop-bemu", - "--", - "--elf", - binary_path, - "--log-dir", - run_log, - ] + entry = ["bebop-bemu", "--"] + cargo_args = [ + "cargo", + "run", + "--release", + "--manifest-path", + str(bemu_cargo_manifest), + "--bin", + *entry, + "--elf", + binary_path, + "--log-dir", + run_log, + ] cargo_args[1:1] = [ "--config", 'build.rustflags=["-C", "target-cpu=native"]', diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index d9ed4efa2..0cb759c82 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -24,7 +24,7 @@ from utils.path import bebop_cargo_env, get_buckyball_path, workloads_output_root from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id -from bemu_common import bemu_manifest, chip_emu_manifest +from bemu_common import bemu_manifest from utils.workload_manifest import resolve_workload_toml config = { @@ -52,11 +52,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return try: bemu_cargo_manifest = bemu_manifest(chip, bbdir) - chip_emu = chip_emu_manifest(chip, bbdir) - if chip_emu is not None and 'name = "test_bemu"' in chip_emu.read_text( - encoding="utf-8" - ): - bemu_cargo_manifest = chip_emu except ValueError as e: ctx.logger.error(str(e)) await check_result( diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py index 727bc32bd..80306dce2 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py @@ -35,21 +35,10 @@ def bemu_manifest(chip: str, bbdir: str | None = None) -> Path: return path -def bemu_core_manifest(chip: str, bbdir: str | None = None) -> Path: - return bemu_manifest(chip, bbdir) +def bemu_chip_binary(chip: str) -> str: + """The chip's tile-level BEMU entry in its generated crate.""" + return f"bebop-chip-{require_chip({'chip': chip})}" -def chip_emu_manifest(chip: str, bbdir: str | None = None) -> Path | None: - chip = require_chip({"chip": chip}) - root = _repo(bbdir) - main = _chip(chip, bbdir).bemu.chip_main - if not main: - return None - return root / "examples" / "chips" / chip / "emu" / "Cargo.toml" - - -def bemu_tile_index(chip: str, bbdir: str | None = None) -> int | None: - bemu = _chip(chip, bbdir).bemu - if not bemu.chip_main: - return None - return bemu.tile_index +def bemu_tile_index(chip: str, bbdir: str | None = None) -> int: + return _chip(chip, bbdir).bemu.tile_index diff --git a/bbdev/api/steps/bebop/bemu/scripts/model_sim.py b/bbdev/api/steps/bebop/bemu/scripts/model_sim.py index f24196bed..b580b958b 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/model_sim.py +++ b/bbdev/api/steps/bebop/bemu/scripts/model_sim.py @@ -30,14 +30,12 @@ def model_run_commands(repo: str, params: dict): layout = json.loads((build / "artifact/layout.json").read_text()) kind = layout["execution"]["kind"] settings = tomllib.loads(config.read_text()) + manifest = root / "examples/chips" / chip / "configs/generated/bemu/Cargo.toml" if kind == "native" and "tile_indices" not in settings: - manifest = root / "examples/chips" / chip / "configs/generated/bemu/Cargo.toml" binary = "bebop-bemu" python = root / "result/bin/python3" elif kind == "python" or (kind == "native" and "tile_indices" in settings): - manifest = root / "examples/chips" / chip / "emu/Cargo.toml" - with manifest.open("rb") as source: - binary = tomllib.load(source)["bin"][0]["name"] + binary = f"bebop-chip-{chip}" python = root / ("result/bin/python3" if kind == "native" else "stack/serving/.venv/bin/python") else: raise ValueError(f"Unknown model execution kind: {kind}") diff --git a/bbdev/api/steps/config/scripts/2_parameter_derivation.py b/bbdev/api/steps/config/scripts/2_parameter_derivation.py index d9ee14852..710a6a91a 100644 --- a/bbdev/api/steps/config/scripts/2_parameter_derivation.py +++ b/bbdev/api/steps/config/scripts/2_parameter_derivation.py @@ -424,16 +424,16 @@ def _bemu_balls(repo: Path, topo: dict[str, Any]) -> list[dict[str, str]]: def _bemu_paths(repo: Path, chip: str, topo: dict[str, Any]) -> tuple[str, int]: - main = repo / "examples" / "chips" / chip / "emu" / "src" / "main.rs" - if not main.is_file(): - return "", 0 + """The chip's own bebop-chip entry (empty for the default tile runner) and the tile it runs.""" tiles = iter_topology_tiles(topo) for kind in ("main", "compute"): files = {t.get("_file") for t in tiles if t["kind"] == kind} if len(files) > 1: - _die(f"chip {chip}: emu requires one {kind} tile file, got {sorted(files)}") + _die(f"chip {chip}: bemu requires one {kind} tile file, got {sorted(files)}") compute = [t["tile_id"] for t in tiles if t["kind"] == "compute"] - return f"examples/chips/{chip}/emu/src/main.rs", compute[0] if compute else 0 + main = repo / "examples" / "chips" / chip / "emu" / "src" / "main.rs" + entry = f"examples/chips/{chip}/emu/src/main.rs" if main.is_file() else "" + return entry, compute[0] if compute else 0 def _ball_ctest_dirs(repo: Path, core: dict[str, Any]) -> list[str]: diff --git a/bbdev/api/steps/config/scripts/4_install.py b/bbdev/api/steps/config/scripts/4_install.py index cfe205e3d..2b0556d81 100644 --- a/bbdev/api/steps/config/scripts/4_install.py +++ b/bbdev/api/steps/config/scripts/4_install.py @@ -95,11 +95,20 @@ def install_bemu(chip: pb.Chip, bbdir: Path, gen: Path) -> Path: bemu = gen / "bemu" bemu.mkdir(parents=True, exist_ok=True) _emit_dispatch(chip, bbdir, bemu / "dispatch.rs") - for source in (cargo, build_rs): - target = bemu / source.name - target.unlink(missing_ok=True) - target.symlink_to(os.path.relpath(source, bemu)) - return bemu / "Cargo.toml" + target = bemu / build_rs.name + target.unlink(missing_ok=True) + target.symlink_to(os.path.relpath(build_rs, bemu)) + # Each chip owns its crate: the single-core bebop-bemu plus its bebop-chip- entry, which + # is the chip's own emu/src/main.rs when present and the default tile runner otherwise. + entry = bbdir / chip.bemu.chip_main if chip.bemu.chip_main else src / "main.rs" + manifest = bemu / "Cargo.toml" + manifest.unlink(missing_ok=True) + manifest.write_text( + cargo.read_text(encoding="utf-8") + + f'\n[[bin]]\nname = "bebop-chip-{chip.name}"\npath = "{os.path.relpath(entry, bemu)}"\n', + encoding="utf-8", + ) + return manifest def install_bebop(gen: Path) -> Path: From 1d29975f2de1bb993d1ca2bf30fb265f542813bf Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sun, 4 Oct 2026 23:33:02 +0800 Subject: [PATCH 178/196] feat: add support for configurable job count in BEMU and Verilator steps --- bbdev/api/steps/bebop/bemu/03_batch_api.step.py | 1 + bbdev/api/steps/bebop/bemu/03_batch_event.step.py | 10 +++++++++- bbdev/api/steps/bebop/verilator/06_batch_api.step.py | 1 + bbdev/api/steps/bebop/verilator/06_batch_event.step.py | 2 ++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py index 0b4e0cb59..56e7d7422 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -47,6 +47,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "data": { "chip": chip, "test": test_type, + "jobs": body.get("jobs", 1), "clean-before": body.get( "clean-before", body.get("clean_before", False) ), diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index 0cb759c82..c536a1fe9 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -108,7 +108,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) ctx.logger.info("Cleaned previous bebop test artifacts") harness = shlex.join( - ["--", "--workload-toml", workload_toml, "--bb-tests-root", elf_root, *[]] + [ + "--", + "--workload-toml", + workload_toml, + "--bb-tests-root", + elf_root, + "--jobs", + str(input_data.get("jobs", 1)), + ] ) test_cmd = f"nix develop -c cargo test --manifest-path {shlex.quote(str(bemu_cargo_manifest))} --test test_bemu {harness}" ctx.logger.info(f"Running bebop bemu regression: {test_cmd}") diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py index 3e3443d92..a6975a9ff 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -37,6 +37,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "chip": chip, "vsrc_dir": vsrc_dir, "test": test_type, + "jobs": body.get("jobs", 1), "clean-before": body.get("clean-before", body.get("clean_before", False)), "diff": diff, } diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 3110aa1df..7fa0e21da 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -98,6 +98,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: elf_root, "--arch-config", chip, + "--jobs", + str(input_data.get("jobs", 1)), ] if diff: harness_args.append("--diff") From 9594839bae839d91b40dc951c670cc1f69f05b23 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 6 Oct 2026 02:34:01 +0800 Subject: [PATCH 179/196] feat: add CMake build type option to model build process --- bbdev/api/steps/bebop/bemu/02_sim_api.step.py | 32 ++++- .../api/steps/bebop/bemu/02_sim_event.step.py | 64 ++++++++-- .../api/steps/bebop/bemu/03_batch_api.step.py | 7 +- .../steps/bebop/bemu/03_batch_event.step.py | 56 ++++++++- .../steps/bebop/p2e/01_verilog_api.step.py | 4 +- .../bebop/p2e/02_buildbitstream_event.step.py | 2 +- .../api/steps/bebop/p2e/04_batch_api.step.py | 11 +- .../steps/bebop/p2e/04_batch_event.step.py | 4 +- .../api/steps/bebop/vcs/03_batch_api.step.py | 2 +- .../steps/bebop/vcs/03_batch_event.step.py | 2 +- .../bebop/verilator/06_batch_api.step.py | 6 +- .../bebop/verilator/06_batch_event.step.py | 2 +- bbdev/api/steps/kernel/01_build_event.step.py | 112 +++++++++++++----- bbdev/api/tests/test_api_bebop_bemu_batch.py | 2 +- bbdev/api/tests/test_api_bebop_bemu_sim.py | 2 +- .../tests/test_api_bebop_verilator_batch.py | 2 +- bbdev/api/utils/workload_manifest.py | 8 +- bbdev/bbdev | 10 +- bbdev/mcp/tools/bebop_p2e_batch.py | 6 +- bbdev/mcp/tools/bebop_verilator_batch.py | 4 +- bbdev/mcp/tools/bemu_batch.py | 22 +++- bbdev/mcp/tools/bemu_sim.py | 40 ++++++- bbdev/mcp/tools/kernel_build.py | 4 +- 23 files changed, 314 insertions(+), 90 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py index c9913bbd1..9f5a1fa3a 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py @@ -25,6 +25,23 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + if "pk" in body or "host-io" in body: + return ApiResponse( + status=400, body={"error": "Unknown parameter: pk or host-io"} + ) + if body.get("system") and body.get("core_index") is not None: + return ApiResponse( + status=400, + body={ + "error": "system boot runs all chip harts; core_index cannot be supplied" + }, + ) + if not body.get("system") and any( + key in body for key in ("dtb", "initrd", "memory-mib") + ): + return ApiResponse( + status=400, body={"error": "dtb, initrd and memory-mib require system boot"} + ) chip = body.get("chip", "") try: require_chip({"chip": chip}) @@ -44,8 +61,15 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: try: model_run_commands(get_buckyball_path(), body) except (ValueError, KeyError, OSError) as error: - return ApiResponse(status=400, body={"success": False, "failure": True, - "returncode": 400, "message": str(error)}) + return ApiResponse( + status=400, + body={ + "success": False, + "failure": True, + "returncode": 400, + "message": str(error), + }, + ) elif not binary: return ApiResponse( status=400, @@ -57,5 +81,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: }, ) - await ctx.enqueue({"topic": "bebop.bemu.sim", "data": {**body, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + {"topic": "bebop.bemu.sim", "data": {**body, "_trace_id": ctx.trace_id}} + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index c278814d2..b1881e5f2 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -108,7 +108,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: commands, run_log = model_run_commands(bbdir, params) run_log.mkdir(parents=True, exist_ok=True) ctx.logger.info(f"Model simulation logs: {run_log}") - filenames = ("run.log",) if input_data.get("reuse-simulator", False) else ("build.log", "run.log") + filenames = ( + ("run.log",) + if input_data.get("reuse-simulator", False) + else ("build.log", "run.log") + ) for command, filename in zip(commands, filenames, strict=True): command_line = shlex.join(["nix", "develop", "-c", *command]) command_line = f"set -o pipefail; {command_line} 2>&1 | tee {shlex.quote(str(run_log / filename))}" @@ -126,9 +130,26 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if result.returncode: break try: - report_path = performance_report(bbdir, report_context, origin_tid, "bemu", run_log, (run_log / "run.log").read_text(), result.returncode) + report_path = performance_report( + bbdir, + report_context, + origin_tid, + "bemu", + run_log, + (run_log / "run.log").read_text(), + result.returncode, + ) except (OSError, ValueError, KeyError) as error: - await check_result(ctx, 1, extra_fields={"task": "report", "error": str(error), "log_dir": str(run_log)}, trace_id=origin_tid) + await check_result( + ctx, + 1, + extra_fields={ + "task": "report", + "error": str(error), + "log_dir": str(run_log), + }, + trace_id=origin_tid, + ) return await check_result( ctx, @@ -179,9 +200,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: run_log = log_dir(bbdir, chip, "verilog", timestamp, "bemu", binary_name) os.makedirs(run_log, exist_ok=True) core_index = input_data.get("core_index") + system = bool(input_data.get("system", False)) + if system and core_index is not None: + raise ValueError( + "system boot runs all chip harts; core_index cannot be supplied" + ) # The chip's tile runner by default; --core-index selects the single-core bebop-bemu. if core_index is None: - entry = [bemu_chip_binary(chip), "--", "--tile-index", str(bemu_tile_index(chip, bbdir))] + entry = [bemu_chip_binary(chip), "--"] + if not system: + entry.extend(["--tile-index", str(bemu_tile_index(chip, bbdir))]) else: entry = ["bebop-bemu", "--"] cargo_args = [ @@ -205,12 +233,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--config", "profile.release.codegen-units=1", ] + if system: + cargo_args.append("--system") + for option in ("dtb", "initrd"): + if input_data.get(option): + cargo_args.extend([f"--{option}", input_data[option]]) + if input_data.get("memory-mib") is not None: + cargo_args.extend(["--memory-mib", str(input_data["memory-mib"])]) if core_index is not None: cargo_args.extend(["--core-index", str(core_index)]) - if input_data.get("pk"): - cargo_args.append("--pk") - if input_data.get("host-io"): - cargo_args.append("--host-io") if input_data.get("disasm"): cargo_args.append("--disasm") if input_data.get("tool-profile"): @@ -243,9 +274,22 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if cancellation_requested(origin_tid): return try: - report_path = performance_report(bbdir, report_context, origin_tid, "bemu", run_log, (Path(run_log) / "run.log").read_text(), run_result.returncode) + report_path = performance_report( + bbdir, + report_context, + origin_tid, + "bemu", + run_log, + (Path(run_log) / "run.log").read_text(), + run_result.returncode, + ) except (OSError, ValueError, KeyError) as error: - await check_result(ctx, 1, extra_fields={"task": "report", "error": str(error), "log_dir": run_log}, trace_id=origin_tid) + await check_result( + ctx, + 1, + extra_fields={"task": "report", "error": str(error), "log_dir": run_log}, + trace_id=origin_tid, + ) return if run_result.returncode != 0: await check_result( diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py index 56e7d7422..dd2bbaca5 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -31,14 +31,14 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: return ApiResponse( status=400, body={ - "error": "Missing required parameter: --test must be specified (elf-tests or pk-tests)" + "error": "Missing required parameter: --test must be specified (bare-tests or linux-tests)" }, ) - if test_type not in ["elf-tests", "pk-tests"]: + if test_type not in ("bare-tests", "linux-tests"): return ApiResponse( status=400, body={ - "error": f"Invalid test type: {test_type}. Must be 'elf-tests' or 'pk-tests'" + "error": f"Invalid test type: {test_type}. Must be 'bare-tests' or 'linux-tests'" }, ) await ctx.enqueue( @@ -48,6 +48,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "chip": chip, "test": test_type, "jobs": body.get("jobs", 1), + "cpu-tests": bool(body.get("cpu-tests", False)), "clean-before": body.get( "clean-before", body.get("clean_before", False) ), diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index c536a1fe9..513465866 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -7,9 +7,11 @@ """ import os +import json import shutil import shlex import sys +from pathlib import Path from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) @@ -65,7 +67,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: elf_root = workloads_output_root(bbdir) env = os.environ.copy() env.update(bebop_cargo_env(bbdir, chip)) - test_type = input_data.get("test", "elf-tests") + test_type = input_data.get("test", "bare-tests") try: workload_toml = resolve_workload_toml(chip, "bemu", test_type, bbdir) except ValueError as e: @@ -83,6 +85,27 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return ctx.logger.info(f"Running {test_type} with workload config: {workload_toml}") + if input_data.get("cpu-tests", False): + for stage, manifest in ( + ("CPU", Path(bbdir) / "bebop/src/nodes/bemu/rvcpu/Cargo.toml"), + ("clock", Path(bbdir) / "bebop/src/nodes/lib/clint/Cargo.toml"), + ): + unit_result = await stream_run_logger_async( + cmd=f"nix develop -c cargo test --manifest-path {shlex.quote(str(manifest))} --lib --jobs {int(input_data.get('jobs', 1))}", + logger=ctx.logger, + cwd=bbdir, + stdout_prefix=f"bemu {stage} tests", + stderr_prefix=f"bemu {stage} tests", + ) + if unit_result.returncode != 0: + await check_result( + ctx, + unit_result.returncode, + continue_run=False, + extra_fields={"task": f"{stage}-tests", "backend": "bemu"}, + trace_id=origin_tid, + ) + return build_cmd = f"nix develop -c cargo build --manifest-path {shlex.quote(str(bemu_cargo_manifest))} --tests" ctx.logger.info("Building bebop bemu (tests)...") build_result = await stream_run_logger_async( @@ -102,6 +125,37 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + if input_data.get("cpu-tests", False): + focused_cmd = f"nix develop -c cargo test --manifest-path {shlex.quote(str(bemu_cargo_manifest))} --lib" + placement = ( + Path(bbdir) + / "examples/chips" + / chip + / "configs/generated/config/derived.json" + ) + cores = json.loads(placement.read_text())["cores"] + if not any(core["ball_num"] > 0 for core in cores): + ctx.logger.info( + "CPU-only configuration: NPU DMA unit tests are not applicable" + ) + focused_cmd += " -- --skip accel::dma_preflight_tests" + focused_result = await stream_run_logger_async( + cmd=focused_cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="bemu memory tests", + stderr_prefix="bemu memory tests", + env=env, + ) + if focused_result.returncode != 0: + await check_result( + ctx, + focused_result.returncode, + continue_run=False, + extra_fields={"task": "memory-tests", "backend": "bemu"}, + trace_id=origin_tid, + ) + return if input_data.get("clean-before", input_data.get("clean_before", False)): shutil.rmtree( os.path.join(env["CARGO_TARGET_DIR"], "test-artifacts"), ignore_errors=True diff --git a/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py index 0c2c17169..d49eeaa5b 100644 --- a/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py +++ b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py @@ -19,7 +19,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) data = {"chip": chip, "diff": bool(body.get("diff", False))} - if body.get("output_dir"): - data["output_dir"] = body["output_dir"] + if body.get("output-dir"): + data["output_dir"] = body["output-dir"] await ctx.enqueue({"topic": "bebop.p2e.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index a02393289..3b9ac56bd 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -80,10 +80,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: **bebop_cargo_env(bbdir, chip), "VSRC_PATH": vsrc_dir, "OUT_PATH": str(build_path), + "CARGO_TARGET_DIR": os.path.join(bebop_dir, "target", f"{chip}-p2e{'-diff' if diff else ''}"), } if diff: features.append("bemu") - build_env["CARGO_TARGET_DIR"] = os.path.join(bebop_dir, "target", f"{chip}-p2e-diff") build_cmd = shlex.join([ "nix", "develop", "--ignore-env", "--keep-env-var", "HOME", diff --git a/bbdev/api/steps/bebop/p2e/04_batch_api.step.py b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py index 5f8265f72..940d813d6 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_api.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py @@ -35,21 +35,26 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: if not test_type: return ApiResponse( status=400, - body={"error": "Missing required parameter: --test must be specified (elf-tests or pk-tests)"} + body={"error": "Missing required parameter: --test must be specified (bare-tests or linux-tests)"} ) - if test_type not in ["elf-tests", "pk-tests"]: + if test_type not in ["bare-tests", "linux-tests"]: return ApiResponse( status=400, - body={"error": f"Invalid test type: {test_type}. Must be 'elf-tests' or 'pk-tests'"} + body={"error": f"Invalid test type: {test_type}. Must be 'bare-tests' or 'linux-tests'"} ) + fpga_location = body.get("fpga-location", "0.A") + if not isinstance(fpga_location, str) or not fpga_location: + return ApiResponse(status=400, body={"error": "--fpga-location must be a location such as 1.A"}) + diff = bool(body.get("diff", False)) data = { "chip": chip, "bitstream": bitstream, "test": test_type, "diff": diff, + "fpga-location": fpga_location, } await ctx.enqueue({"topic": "bebop.p2e.batch", "data": {**data, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py index ec2c71c2a..a4053ad34 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py @@ -82,7 +82,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return elf_root = workloads_output_root(bbdir) cargo_env = bebop_cargo_env(bbdir, chip) - test_type = input_data.get("test", "elf-tests") + test_type = input_data.get("test", "bare-tests") diff = bool(input_data.get("diff", False)) try: workload_toml = resolve_workload_toml(chip, "p2e", test_type, bbdir, diff=diff) @@ -173,6 +173,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: elf_root, "--p2e-bitstream", bitstream, + "--p2e-fpga-location", + input_data.get("fpga-location", "0.A"), ] if diff: harness_args.append("--diff") diff --git a/bbdev/api/steps/bebop/vcs/03_batch_api.step.py b/bbdev/api/steps/bebop/vcs/03_batch_api.step.py index ab9762d23..28e7aeb81 100644 --- a/bbdev/api/steps/bebop/vcs/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/vcs/03_batch_api.step.py @@ -7,5 +7,5 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - await ctx.enqueue({"topic": "bebop.vcs.batch", "data": {"chip": require_chip(body), "test": body.get("test", "elf-tests"), "_trace_id": ctx.trace_id}}) + await ctx.enqueue({"topic": "bebop.vcs.batch", "data": {"chip": require_chip(body), "test": body.get("test", "bare-tests"), "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/vcs/03_batch_event.step.py b/bbdev/api/steps/bebop/vcs/03_batch_event.step.py index d0e786b2e..acacf8d28 100644 --- a/bbdev/api/steps/bebop/vcs/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/vcs/03_batch_event.step.py @@ -29,7 +29,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: try: chip = require_chip(input_data) bbdir = get_buckyball_path() - test_type = input_data.get("test", "elf-tests") + test_type = input_data.get("test", "bare-tests") manifest = resolve_workload_toml(chip, "verilator", test_type, bbdir) except ValueError as error: await check_result(ctx, 1, extra_fields={"error": str(error)}, trace_id=origin_tid) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py index a6975a9ff..abfe23394 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -24,13 +24,13 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: return ApiResponse( status=400, body={ - "error": "Missing required parameter: --test must be specified (elf-tests)" + "error": "Missing required parameter: --test must be specified (bare-tests or linux-tests)" }, ) - if test_type != "elf-tests": + if test_type not in ("bare-tests", "linux-tests"): return ApiResponse( status=400, - body={"error": f"Invalid test type: {test_type}. Must be 'elf-tests'"}, + body={"error": f"Invalid test type: {test_type}. Must be 'bare-tests' or 'linux-tests'"}, ) vsrc_dir = rtl_dir(bbdir, chip, "verilog", body.get("vsrc_dir")) data = { diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 7fa0e21da..54cc72b39 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -53,7 +53,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return elf_root = workloads_output_root(bbdir) - test_type = input_data.get("test", "elf-tests") + test_type = input_data.get("test", "bare-tests") diff = bool(input_data.get("diff", False)) try: workload_toml = resolve_workload_toml( diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index 7760f9cb2..96c3733ab 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -1,4 +1,5 @@ import os +import json import re import shutil import sys @@ -44,10 +45,35 @@ def hart_count_params(input_data: dict) -> dict: raise ValueError(f"unknown kernel build parameter(s): {', '.join(unknown)}") if "hidden-hart-base" in input_data: - raise ValueError("hidden-hart-base is not supported; hidden harts must start at visible-hart-count") + raise ValueError( + "hidden-hart-base is not supported; hidden harts must start at visible-hart-count" + ) - visible = int(input_data.get("visible-hart-count", 64)) - total = int(input_data.get("total-hart-count", visible)) + chip = input_data.get("chip") + if chip: + derived = os.path.join( + get_buckyball_path(), + "examples", + "chips", + chip, + "configs", + "generated", + "config", + "derived.json", + ) + with open(derived) as stream: + harts = json.load(stream)["harts"] + visible = sum(hart["visible"] for hart in harts) + total = len(harts) + for key, expected in ( + ("visible-hart-count", visible), + ("total-hart-count", total), + ): + if key in input_data and int(input_data[key]) != expected: + raise ValueError(f"{key} must match chip topology: {expected}") + else: + visible = int(input_data.get("visible-hart-count", 64)) + total = int(input_data.get("total-hart-count", visible)) hidden_base = visible if visible < 1: @@ -81,7 +107,7 @@ def kernel_model(input_data: dict) -> str: return model -def kernel_chip(input_data: dict, bbdir: str, *, require_overlay: bool) -> str: +def kernel_chip(input_data: dict, bbdir: str) -> str: chip = input_data.get("chip", "") if chip in ("", None): return "" @@ -94,10 +120,6 @@ def kernel_chip(input_data: dict, bbdir: str, *, require_overlay: bool) -> str: if not os.path.isdir(chip_dir): raise ValueError(f"unknown chip: {chip}") - if require_overlay: - chip_init = os.path.join(chip_dir, "kernel", "overlay", "init") - if not os.path.isfile(chip_init): - raise ValueError(f"chip OS overlay init not found: {chip_init}") return chip @@ -123,10 +145,18 @@ def kernel_build_dir( suffix += "-pmem" if visible == 64 and total == 64: return os.path.join(bbdir, "bb-tests", "build", f"kernel{suffix}") - return os.path.join(bbdir, "bb-tests", "build", f"kernel-v{visible}-t{total}{suffix}") + return os.path.join( + bbdir, "bb-tests", "build", f"kernel-v{visible}-t{total}{suffix}" + ) -def fw_payload_name(hart_params: dict, model: str = "", chip: str = "", guest_memory_mib: int = 512, model_storage: str = "initramfs") -> str: +def fw_payload_name( + hart_params: dict, + model: str = "", + chip: str = "", + guest_memory_mib: int = 512, + model_storage: str = "initramfs", +) -> str: visible = hart_params["visible"] total = hart_params["total"] name = "fw_payload" @@ -135,7 +165,7 @@ def fw_payload_name(hart_params: dict, model: str = "", chip: str = "", guest_me if model: name = f"{name}-{model}" elif chip: - name = f"{name}-{chip}-pk" + name = f"{name}-{chip}-linux" if guest_memory_mib != 512: name += f"-mem{guest_memory_mib}M" if model_storage == "pmem": @@ -150,21 +180,29 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: kernel_src = os.path.join(bbdir, "bb-tests", "workloads", "lib", "kernel") try: memory_value = input_data.get("guest-memory-mib", 512) - if isinstance(memory_value, bool) or not str(memory_value).isdigit() or not 1 <= int(memory_value) <= 16384: - raise ValueError("guest-memory-mib must be an integer in 1..16384 (P2E DDR capacity)") + if ( + isinstance(memory_value, bool) + or not str(memory_value).isdigit() + or not 1 <= int(memory_value) <= 16384 + ): + raise ValueError( + "guest-memory-mib must be an integer in 1..16384 (P2E DDR capacity)" + ) guest_memory_mib = int(memory_value) model_storage = input_data.get("model-storage", "initramfs") if model_storage not in ("initramfs", "pmem"): raise ValueError("model-storage must be initramfs or pmem") - if model_storage == "pmem" and (not input_data.get("model") or guest_memory_mib >= 16384): + if model_storage == "pmem" and ( + not input_data.get("model") or guest_memory_mib >= 16384 + ): raise ValueError("pmem requires --model and guest-memory-mib < 16384") hart_params = hart_count_params(input_data) model = kernel_model(input_data) if model and not input_data.get("chip"): raise ValueError("--model requires --chip") interactive = kernel_interactive(input_data) - # pk mode needs OS overlay; model mode only needs chip for the model artifact path - chip = kernel_chip(input_data, bbdir, require_overlay=not bool(model)) + # Linux mode needs OS overlay; model mode only needs chip for the model artifact path + chip = kernel_chip(input_data, bbdir) except ValueError as e: ctx.logger.error(str(e)) await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) @@ -174,7 +212,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: output_dir = os.path.join(output_dir, chip) os.makedirs(output_dir, exist_ok=True) kernel_build = kernel_build_dir( - bbdir, hart_params, model, chip, interactive=interactive, guest_memory_mib=guest_memory_mib, model_storage=model_storage + bbdir, + hart_params, + model, + chip, + interactive=interactive, + guest_memory_mib=guest_memory_mib, + model_storage=model_storage, ) interactive_arg = "ON" if interactive else "OFF" cmake_model_name = model @@ -193,18 +237,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"-DBUCKYBALL_MODEL_DATASET=" ) if chip and not model: - chip_kernel = os.path.join(bbdir, "examples", "chips", chip, "kernel") - pk_toml = os.path.join( - bbdir, "examples", "chips", chip, "regression", "batch", "bemu", "workloads-pk.toml" + workload_toml = os.path.join( + bbdir, "examples", "chips", chip, "kernel", "workloads.toml" ) - if not os.path.isfile(pk_toml): - ctx.logger.error(f"chip pk workload toml not found: {pk_toml}") + if not os.path.isfile(workload_toml): + ctx.logger.error(f"chip workload toml not found: {workload_toml}") await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) return - configure_cmd += ( - f" -DBUCKYBALL_CHIP_KERNEL_DIR={chip_kernel}" - f" -DBUCKYBALL_PK_WORKLOAD_TOML={pk_toml}" - ) + configure_cmd += f" -DBUCKYBALL_WORKLOAD_TOML={workload_toml}" result = await stream_run_logger_async( cmd=configure_cmd, @@ -213,7 +253,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stderr_prefix="marshal build", ) if result.returncode != 0: - await check_result(ctx, result.returncode, continue_run=False, trace_id=origin_tid) + await check_result( + ctx, result.returncode, continue_run=False, trace_id=origin_tid + ) return # cmake build @@ -226,11 +268,19 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) if result.returncode != 0: - await check_result(ctx, result.returncode, continue_run=False, trace_id=origin_tid) + await check_result( + ctx, result.returncode, continue_run=False, trace_id=origin_tid + ) return # Convert fw_payload.bin to hex for P2E memory backdoor - payload_name = fw_payload_name(hart_params, cmake_model_name if model else "", chip, guest_memory_mib, model_storage) + payload_name = fw_payload_name( + hart_params, + cmake_model_name if model else "", + chip, + guest_memory_mib, + model_storage, + ) fw_payload_bin = os.path.join(output_dir, f"{payload_name}.bin") fw_payload_hex = os.path.join(output_dir, f"{payload_name}.hex") fw_payload_elf = os.path.join(output_dir, f"{payload_name}.elf") @@ -267,4 +317,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if not os.path.isfile(manifest): raise ValueError(f"Missing pmem load manifest: {manifest}") storage_fields["load_manifest"] = manifest - await check_result(ctx, 0, continue_run=False, extra_fields=storage_fields, trace_id=origin_tid) + await check_result( + ctx, 0, continue_run=False, extra_fields=storage_fields, trace_id=origin_tid + ) diff --git a/bbdev/api/tests/test_api_bebop_bemu_batch.py b/bbdev/api/tests/test_api_bebop_bemu_batch.py index 7eff4a36e..595ba40b6 100644 --- a/bbdev/api/tests/test_api_bebop_bemu_batch.py +++ b/bbdev/api/tests/test_api_bebop_bemu_batch.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-bemu --batch '--chip toy --test elf-tests'") +run_bbdev_case("bbdev bebop-bemu --batch '--chip toy --test bare-tests'") diff --git a/bbdev/api/tests/test_api_bebop_bemu_sim.py b/bbdev/api/tests/test_api_bebop_bemu_sim.py index 179a87d2a..f72013459 100644 --- a/bbdev/api/tests/test_api_bebop_bemu_sim.py +++ b/bbdev/api/tests/test_api_bebop_bemu_sim.py @@ -1,4 +1,4 @@ from _api_test_helper import run_bbdev_case run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_relu_test-singlecore-baremetal'") -run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_relu_test-linux --pk'") +run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_relu_test-linux'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_batch.py b/bbdev/api/tests/test_api_bebop_verilator_batch.py index 5546cc79d..dd4077030 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_batch.py +++ b/bbdev/api/tests/test_api_bebop_verilator_batch.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --batch '--chip toy --test elf-tests'") +run_bbdev_case("bbdev bebop-verilator --batch '--chip toy --test bare-tests'") diff --git a/bbdev/api/utils/workload_manifest.py b/bbdev/api/utils/workload_manifest.py index 27822d5ce..aa8e11516 100644 --- a/bbdev/api/utils/workload_manifest.py +++ b/bbdev/api/utils/workload_manifest.py @@ -26,10 +26,10 @@ def resolve_workload_toml( bbdir: str | None = None, diff: bool = False, ) -> str: - if test_type == "elf-tests": - suffix = "elf" - elif test_type == "pk-tests": - suffix = "pk" + if test_type == "bare-tests": + suffix = "bare" + elif test_type == "linux-tests": + suffix = "linux" else: raise ValueError(f"invalid test type: {test_type}") regression_dir = chip_regression_dir(chip, backend, bbdir) diff --git a/bbdev/bbdev b/bbdev/bbdev index 760b78ea9..e435f43f9 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -60,9 +60,9 @@ WORKFLOW_COMMANDS = { "build": 'Build compiler. Args: "--chip "', }, "bebop-bemu": { - "sim": 'Run Bebop emulator simulation. Model args: "--chip --model [--reuse-simulator]". ELF args: "--chip --binary [--pk] [--disasm] [--tool-profile] [--itrace] [--mtrace]"; resolves files in the selected chip workload/kernel outputs', + "sim": 'Run Bebop emulator simulation. Model args: "--chip --model [--reuse-simulator]". ELF args: "--chip --binary [--disasm] [--tool-profile] [--itrace] [--mtrace]"; resolves files in the selected chip workload/kernel outputs', "analysis": 'Analyze itrace/mtrace in an existing log dir. Args: "--chip --log-dir --itrace|--mtrace"', - "batch": 'Run Bebop BEMU batch regression. Args: "--chip --test [--clean-before]"', + "batch": 'Run Bebop BEMU batch regression. Args: "--chip --test [--cpu-tests] [--jobs ] [--clean-before]"', }, "bebop-verilator": { "clean": 'Clean Bebop Verilator build directory. Args: "--chip [--output-dir ]"', @@ -70,19 +70,19 @@ WORKFLOW_COMMANDS = { "build": 'Build Bebop Verilator simulation executable. Args: "--chip [--diff] [--jobs ] [--output-dir ]"', "sim": 'Run Bebop Verilator simulation. Args: "--chip --binary [--log-dir ] [--fst-dir ] [--no-wave]"', "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--jobs ] [--chip ] [--log-dir ] [--fst-dir ] [--no-wave]"', - "batch": 'Run Bebop Verilator batch regression. Args: "--chip --test [--diff] [--clean-before]"', + "batch": 'Run Bebop Verilator batch regression. Args: "--chip --test [--diff] [--clean-before]"', }, "bebop-vcs": { "verilog": 'Generate Bebop VCS RTL once. Args: "--chip "', "build": 'Build the reusable Bebop VCS simv. Args: "--chip [--jobs ]"', - "batch": 'Run Bebop VCS workload batch. Args: "--chip --test "', + "batch": 'Run Bebop VCS workload batch. Args: "--chip --test "', }, "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--chip [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--chip [--diff] [--output-dir ]"', "buildbitstream": 'Build Bebop P2E runtime case. Args: "--chip [--diff] [--vsrc-dir ] [--output-dir ] [--resume-post-route | --stop-after vsyn|vcom]"; stop-after produces resource assessment only, not a runnable case; resume requires an existing routed case and preserves its completed synthesis/PNR', "runworkload": 'Cold-load Bebop P2E workload. Args: "--chip (--image | --load-manifest ) --bitstream [--fpga-location ] [--multi-fpga] [--wave] [--wave-start ] [--diff (image only)] [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--reuse-runtime]"', - "batch": 'Run Bebop P2E batch regression. Args: "--chip --bitstream --test [--diff]"', + "batch": 'Run Bebop P2E batch regression. Args: "--chip --bitstream --test [--diff] [--fpga-location ]"', }, "dc": { "verilog": 'Generate top-scoped DC RTL under arch/build//. Args: "--chip [--top ]"', diff --git a/bbdev/mcp/tools/bebop_p2e_batch.py b/bbdev/mcp/tools/bebop_p2e_batch.py index ccbb71ea6..0224a8b88 100644 --- a/bbdev/mcp/tools/bebop_p2e_batch.py +++ b/bbdev/mcp/tools/bebop_p2e_batch.py @@ -14,12 +14,12 @@ def bbdev_bebop_p2e_batch( test: str, diff: bool = False, ) -> str: - """Batch bebop-p2e regression. test: elf-tests|pk-tests. POST /bebop/p2e/batch.""" + """Batch bebop-p2e regression. test: bare-tests|linux-tests. POST /bebop/p2e/batch.""" for n, v in (("chip", chip), ("bitstream", bitstream)): if e := need(n, v): return err(e) - if test not in ("elf-tests", "pk-tests"): - return err("test must be elf-tests or pk-tests") + if test not in ("bare-tests", "linux-tests"): + return err("test must be bare-tests or linux-tests") return fmt( submit( "/bebop/p2e/batch", diff --git a/bbdev/mcp/tools/bebop_verilator_batch.py b/bbdev/mcp/tools/bebop_verilator_batch.py index c1a53dba9..747fbb179 100644 --- a/bbdev/mcp/tools/bebop_verilator_batch.py +++ b/bbdev/mcp/tools/bebop_verilator_batch.py @@ -13,8 +13,8 @@ def bbdev_bebop_verilator_batch( """Batch bebop-verilator regression. POST /bebop/verilator/batch.""" if e := need("chip", chip): return err(e) - if test != "elf-tests": - return err("test must be elf-tests") + if test not in ("bare-tests", "linux-tests"): + return err("test must be bare-tests or linux-tests") return fmt( submit( "/bebop/verilator/batch", diff --git a/bbdev/mcp/tools/bemu_batch.py b/bbdev/mcp/tools/bemu_batch.py index cb4ac11cf..e8c01bef9 100644 --- a/bbdev/mcp/tools/bemu_batch.py +++ b/bbdev/mcp/tools/bemu_batch.py @@ -7,15 +7,27 @@ def register(mcp): @mcp.tool() - def bbdev_bemu_batch(chip: str, test: str, clean_before: bool = False) -> str: - """Batch bemu regression. test: elf-tests|pk-tests. POST /bebop/bemu/batch.""" + def bbdev_bemu_batch( + chip: str, + test: str, + clean_before: bool = False, + cpu_tests: bool = False, + jobs: int = 1, + ) -> str: + """Batch bemu regression. test: bare-tests|linux-tests. POST /bebop/bemu/batch.""" if e := need("chip", chip): return err(e) - if test not in ("elf-tests", "pk-tests"): - return err("test must be elf-tests or pk-tests") + if test not in ("bare-tests", "linux-tests"): + return err("test must be bare-tests or linux-tests") return fmt( submit( "/bebop/bemu/batch", - {"chip": chip, "test": test, "clean-before": clean_before}, + { + "chip": chip, + "test": test, + "clean-before": clean_before, + "cpu-tests": cpu_tests, + "jobs": jobs, + }, ) ) diff --git a/bbdev/mcp/tools/bemu_sim.py b/bbdev/mcp/tools/bemu_sim.py index 7269b2549..265748234 100644 --- a/bbdev/mcp/tools/bemu_sim.py +++ b/bbdev/mcp/tools/bemu_sim.py @@ -11,7 +11,10 @@ def register(mcp): def bbdev_bemu_sim( chip: str, binary: str | None = None, - pk: bool = False, + system: bool = False, + dtb: str | None = None, + initrd: str | None = None, + memory_mib: int | None = None, disasm: bool = False, tool_profile: bool = False, itrace: bool = False, @@ -26,10 +29,28 @@ def bbdev_bemu_sim( for n, v in (("chip", chip), ("model", model)): if e := need(n, v): return err(e) - if binary is not None or pk or disasm or tool_profile or itrace or mtrace or core_index is not None or arguments: - return err("model runs use running-param.toml; ELF options cannot be supplied") - return fmt(submit("/bebop/bemu/sim", { - "chip": chip, "model": model, "reuse-simulator": reuse_simulator})) + if ( + binary is not None + or system + or dtb + or initrd + or memory_mib is not None + or disasm + or tool_profile + or itrace + or mtrace + or core_index is not None + or arguments + ): + return err( + "model runs use running-param.toml; ELF options cannot be supplied" + ) + return fmt( + submit( + "/bebop/bemu/sim", + {"chip": chip, "model": model, "reuse-simulator": reuse_simulator}, + ) + ) if reuse_simulator: return err("reuse-simulator is supported for model runs only") for n, v in (("chip", chip), ("binary", binary)): @@ -38,7 +59,7 @@ def bbdev_bemu_sim( params: Dict[str, Any] = { "chip": chip, "binary": binary, - "pk": pk, + "system": system, "disasm": disasm, "tool-profile": tool_profile, "itrace": itrace, @@ -46,4 +67,11 @@ def bbdev_bemu_sim( "core_index": core_index, "arguments": arguments if arguments is not None else [], } + for key, value in ( + ("dtb", dtb), + ("initrd", initrd), + ("memory-mib", memory_mib), + ): + if value is not None: + params[key] = value return fmt(submit("/bebop/bemu/sim", params)) diff --git a/bbdev/mcp/tools/kernel_build.py b/bbdev/mcp/tools/kernel_build.py index 27e7f6bc9..f4d395b67 100644 --- a/bbdev/mcp/tools/kernel_build.py +++ b/bbdev/mcp/tools/kernel_build.py @@ -20,8 +20,8 @@ def bbdev_kernel_build( ) -> str: """Build RISC-V kernel + rootfs. POST /kernel/build. - With --chip only: uses examples/chips//kernel OS overlay and packs - that chip's bemu workloads-pk.toml into fw_payload--pk. + With --chip only: packs programs listed in examples/chips//kernel/workloads.toml + and runs them under Linux using the shared workload /init. With --model: requires --chip; packs a native model artifact from stack/models/build///artifact into fw_payload-. With --interactive: keep shared /init shell and do not auto-run. From 4028a4d4ae343cb735d9ab10b351b45bbd5fe974 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 6 Oct 2026 02:38:09 +0800 Subject: [PATCH 180/196] refactor: remove cpu-tests option from BEMU batch regression steps and update related documentation --- .../api/steps/bebop/bemu/03_batch_api.step.py | 1 - .../steps/bebop/bemu/03_batch_event.step.py | 54 ------------------- bbdev/bbdev | 2 +- bbdev/mcp/tools/bemu_batch.py | 2 - 4 files changed, 1 insertion(+), 58 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py index dd2bbaca5..dc987026e 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -48,7 +48,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "chip": chip, "test": test_type, "jobs": body.get("jobs", 1), - "cpu-tests": bool(body.get("cpu-tests", False)), "clean-before": body.get( "clean-before", body.get("clean_before", False) ), diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index 513465866..5621e5323 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -7,11 +7,9 @@ """ import os -import json import shutil import shlex import sys -from pathlib import Path from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) @@ -85,27 +83,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return ctx.logger.info(f"Running {test_type} with workload config: {workload_toml}") - if input_data.get("cpu-tests", False): - for stage, manifest in ( - ("CPU", Path(bbdir) / "bebop/src/nodes/bemu/rvcpu/Cargo.toml"), - ("clock", Path(bbdir) / "bebop/src/nodes/lib/clint/Cargo.toml"), - ): - unit_result = await stream_run_logger_async( - cmd=f"nix develop -c cargo test --manifest-path {shlex.quote(str(manifest))} --lib --jobs {int(input_data.get('jobs', 1))}", - logger=ctx.logger, - cwd=bbdir, - stdout_prefix=f"bemu {stage} tests", - stderr_prefix=f"bemu {stage} tests", - ) - if unit_result.returncode != 0: - await check_result( - ctx, - unit_result.returncode, - continue_run=False, - extra_fields={"task": f"{stage}-tests", "backend": "bemu"}, - trace_id=origin_tid, - ) - return build_cmd = f"nix develop -c cargo build --manifest-path {shlex.quote(str(bemu_cargo_manifest))} --tests" ctx.logger.info("Building bebop bemu (tests)...") build_result = await stream_run_logger_async( @@ -125,37 +102,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - if input_data.get("cpu-tests", False): - focused_cmd = f"nix develop -c cargo test --manifest-path {shlex.quote(str(bemu_cargo_manifest))} --lib" - placement = ( - Path(bbdir) - / "examples/chips" - / chip - / "configs/generated/config/derived.json" - ) - cores = json.loads(placement.read_text())["cores"] - if not any(core["ball_num"] > 0 for core in cores): - ctx.logger.info( - "CPU-only configuration: NPU DMA unit tests are not applicable" - ) - focused_cmd += " -- --skip accel::dma_preflight_tests" - focused_result = await stream_run_logger_async( - cmd=focused_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="bemu memory tests", - stderr_prefix="bemu memory tests", - env=env, - ) - if focused_result.returncode != 0: - await check_result( - ctx, - focused_result.returncode, - continue_run=False, - extra_fields={"task": "memory-tests", "backend": "bemu"}, - trace_id=origin_tid, - ) - return if input_data.get("clean-before", input_data.get("clean_before", False)): shutil.rmtree( os.path.join(env["CARGO_TARGET_DIR"], "test-artifacts"), ignore_errors=True diff --git a/bbdev/bbdev b/bbdev/bbdev index e435f43f9..86f7b83e6 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -62,7 +62,7 @@ WORKFLOW_COMMANDS = { "bebop-bemu": { "sim": 'Run Bebop emulator simulation. Model args: "--chip --model [--reuse-simulator]". ELF args: "--chip --binary [--disasm] [--tool-profile] [--itrace] [--mtrace]"; resolves files in the selected chip workload/kernel outputs', "analysis": 'Analyze itrace/mtrace in an existing log dir. Args: "--chip --log-dir --itrace|--mtrace"', - "batch": 'Run Bebop BEMU batch regression. Args: "--chip --test [--cpu-tests] [--jobs ] [--clean-before]"', + "batch": 'Run Bebop BEMU batch regression. Args: "--chip --test [--jobs ] [--clean-before]"', }, "bebop-verilator": { "clean": 'Clean Bebop Verilator build directory. Args: "--chip [--output-dir ]"', diff --git a/bbdev/mcp/tools/bemu_batch.py b/bbdev/mcp/tools/bemu_batch.py index e8c01bef9..eafcd3171 100644 --- a/bbdev/mcp/tools/bemu_batch.py +++ b/bbdev/mcp/tools/bemu_batch.py @@ -11,7 +11,6 @@ def bbdev_bemu_batch( chip: str, test: str, clean_before: bool = False, - cpu_tests: bool = False, jobs: int = 1, ) -> str: """Batch bemu regression. test: bare-tests|linux-tests. POST /bebop/bemu/batch.""" @@ -26,7 +25,6 @@ def bbdev_bemu_batch( "chip": chip, "test": test, "clean-before": clean_before, - "cpu-tests": cpu_tests, "jobs": jobs, }, ) From 266f014839f0d8522491f94c9fb5561910a94bbd Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 6 Oct 2026 10:57:34 +0800 Subject: [PATCH 181/196] feat: enhance Bebop P2E and BEMU workflows with itrace and mtrace support- --- .../api/steps/bebop/bemu/scripts/model_sim.py | 6 +++++- .../bebop/p2e/02_buildbitstream_api.step.py | 2 ++ .../bebop/p2e/02_buildbitstream_event.step.py | 5 +++++ .../bebop/p2e/03_runworkload_api.step.py | 3 ++- .../bebop/p2e/03_runworkload_event.step.py | 13 +++++++++--- .../steps/bebop/p2e/scripts/runtime_case.py | 20 ++++++++++++++----- bbdev/bbdev | 4 ++-- bbdev/mcp/tools/bebop_p2e_buildbitstream.py | 4 +++- 8 files changed, 44 insertions(+), 13 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/scripts/model_sim.py b/bbdev/api/steps/bebop/bemu/scripts/model_sim.py index b580b958b..34d89e986 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/model_sim.py +++ b/bbdev/api/steps/bebop/bemu/scripts/model_sim.py @@ -8,7 +8,7 @@ def model_run_commands(repo: str, params: dict): - unknown = params.keys() - {"chip", "model", "reuse-simulator"} + unknown = params.keys() - {"chip", "model", "reuse-simulator", "itrace", "mtrace"} if unknown: raise ValueError(f"Unknown model simulation parameters: {sorted(unknown)}") for key in ("chip", "model"): @@ -19,6 +19,9 @@ def model_run_commands(repo: str, params: dict): root = Path(repo) if not isinstance(params.get("reuse-simulator", False), bool): raise ValueError("reuse-simulator must be a boolean") + for flag in ("itrace", "mtrace"): + if not isinstance(params.get(flag, False), bool): + raise ValueError(f"{flag} must be a boolean") chip, model = params["chip"], params["model"] config = root / "examples/models" / chip / model / "configs/running-param.toml" build = root / "stack/models/build" / chip / model @@ -82,4 +85,5 @@ def model_run_commands(repo: str, params: dict): if not simulator.is_file(): raise ValueError(f"Reusable model simulator missing: {simulator}") commands = commands[1:] + commands[-1].extend(f"--{flag}" for flag in ("itrace", "mtrace") if params.get(flag, False)) return commands, log diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py index a1aa093e2..12e891a9e 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py @@ -50,6 +50,8 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "chip": chip, "diff": bool(body.get("diff", False)), + "itrace": bool(body.get("itrace", False)), + "mtrace": bool(body.get("mtrace", False)), "vsrc_dir": vsrc_dir, "output_dir": output_dir, "resume_post_route": resume, diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index 3b9ac56bd..e9767de4c 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -75,6 +75,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: manifest = Path(bbdir) / "examples" / "chips" / chip / "generated" / "bebop" / "Cargo.toml" features = ["p2e"] + for trace in ("itrace", "mtrace"): + if input_data.get(trace, False): + features.append(f"bebop-p2e/{trace}") build_env = { **os.environ, **bebop_cargo_env(bbdir, chip), @@ -99,6 +102,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--rtl-dir", vsrc_dir, "--out-dir", build_dir, *(["--diff"] if diff else []), + *(["--itrace"] if input_data.get("itrace", False) else []), + *(["--mtrace"] if input_data.get("mtrace", False) else []), *(["--resume-post-route"] if resume else []), *(["--stop-after", stop_after] if stop_after else []), ]) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py index 1dbe86471..b6d403566 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py @@ -37,7 +37,8 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: return ApiResponse(status=400, body={"error": f"P2E bitstream is not a file: {bitstream}"}) if body.get("reuse-runtime", False): try: - validate_runtime_reuse(bitstream, bool(body.get("diff", False))) + validate_runtime_reuse(bitstream, bool(body.get("diff", False)), + bool(body.get("itrace", False)), bool(body.get("mtrace", False))) except (ValueError, OSError) as error: return ApiResponse(status=400, body={"error": str(error)}) await ctx.enqueue({ diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index 62ecbd5a0..bd946d0d2 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -27,7 +27,7 @@ from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from resolve_image import resolve_image -from steps.bebop.p2e.scripts.runtime_case import validate_runtime_reuse +from steps.bebop.p2e.scripts.runtime_case import trace_modes, validate_runtime_reuse from steps.bebop.performance_report import performance_report from utils.reports import source_context from pathlib import Path @@ -219,9 +219,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) reuse_runtime = input_data.get("reuse-runtime", False) if reuse_runtime: - validate_runtime_reuse(bitstream, diff) + validate_runtime_reuse(bitstream, diff, bool(input_data.get("itrace", False)), bool(input_data.get("mtrace", False))) ctx.logger.info(f"Reusing selected P2E runtime without rebuilding: {bebop_p2e_path}") else: + modes = trace_modes(bitstream) + features = ["p2e"] + if modes & {"btrace", "btrace_nb_v1"}: + features.append("bemu") + for trace in ("itrace", "mtrace"): + if trace in modes: + features.append(f"bebop-p2e/{trace}") build_cmd = shlex.join( [ "nix", @@ -244,7 +251,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--bin", "bebop", "--features", - "p2e,bemu" if diff else "p2e", + ",".join(features), ] ) ctx.logger.info("Building bebop p2e runtime for the selected case...") diff --git a/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py b/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py index 869429508..57f23f255 100644 --- a/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py +++ b/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py @@ -2,7 +2,16 @@ import os -def validate_runtime_reuse(bitstream: str, diff: bool) -> Path: +def trace_modes(bitstream: str) -> set[str]: + case = Path(bitstream).resolve().parent.parent + modes = set((case / "p2e_trace_mode").read_text().strip().split("+")) + known = {"none", "btrace", "btrace_nb_v1", "itrace", "mtrace"} + if not modes <= known or ("none" in modes and len(modes) != 1): + raise ValueError(f"Unknown P2E trace mode: {modes}") + return modes + + +def validate_runtime_reuse(bitstream: str, diff: bool, itrace: bool = False, mtrace: bool = False) -> Path: bit = Path(bitstream).resolve() case = bit.parent.parent executable = case / "bebop-p2e" @@ -14,9 +23,10 @@ def validate_runtime_reuse(bitstream: str, diff: bool) -> Path: raise ValueError(f"Reusable P2E case is incomplete: {missing}") if not os.access(executable, os.X_OK): raise ValueError(f"Reusable P2E runtime is not executable: {executable}") - mode = (case / "p2e_trace_mode").read_text().strip() - if mode not in ("none", "btrace", "btrace_nb_v1"): - raise ValueError(f"Unknown reusable P2E trace mode: {mode}") - if diff and mode == "none": + modes = trace_modes(bitstream) + if diff and not modes & {"btrace", "btrace_nb_v1"}: raise ValueError("Reusable P2E case does not support diff") + for trace, requested in (("itrace", itrace), ("mtrace", mtrace)): + if requested and trace not in modes: + raise ValueError(f"Reusable P2E case does not support {trace}; rebuild with --{trace}") return executable diff --git a/bbdev/bbdev b/bbdev/bbdev index 86f7b83e6..480da42f8 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -60,7 +60,7 @@ WORKFLOW_COMMANDS = { "build": 'Build compiler. Args: "--chip "', }, "bebop-bemu": { - "sim": 'Run Bebop emulator simulation. Model args: "--chip --model [--reuse-simulator]". ELF args: "--chip --binary [--disasm] [--tool-profile] [--itrace] [--mtrace]"; resolves files in the selected chip workload/kernel outputs', + "sim": 'Run Bebop emulator simulation. Model args: "--chip --model [--reuse-simulator] [--itrace] [--mtrace]". ELF args: "--chip --binary [--disasm] [--tool-profile] [--itrace] [--mtrace]"; resolves files in the selected chip workload/kernel outputs', "analysis": 'Analyze itrace/mtrace in an existing log dir. Args: "--chip --log-dir --itrace|--mtrace"', "batch": 'Run Bebop BEMU batch regression. Args: "--chip --test [--jobs ] [--clean-before]"', }, @@ -80,7 +80,7 @@ WORKFLOW_COMMANDS = { "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--chip [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--chip [--diff] [--output-dir ]"', - "buildbitstream": 'Build Bebop P2E runtime case. Args: "--chip [--diff] [--vsrc-dir ] [--output-dir ] [--resume-post-route | --stop-after vsyn|vcom]"; stop-after produces resource assessment only, not a runnable case; resume requires an existing routed case and preserves its completed synthesis/PNR', + "buildbitstream": 'Build Bebop P2E runtime case. Args: "--chip [--diff] [--itrace] [--mtrace] [--vsrc-dir ] [--output-dir ] [--resume-post-route | --stop-after vsyn|vcom]"; stop-after produces resource assessment only, not a runnable case; resume requires an existing routed case and preserves its completed synthesis/PNR', "runworkload": 'Cold-load Bebop P2E workload. Args: "--chip (--image | --load-manifest ) --bitstream [--fpga-location ] [--multi-fpga] [--wave] [--wave-start ] [--diff (image only)] [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--reuse-runtime]"', "batch": 'Run Bebop P2E batch regression. Args: "--chip --bitstream --test [--diff] [--fpga-location ]"', }, diff --git a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py index 2616341ca..a536e42c5 100644 --- a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py +++ b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py @@ -12,6 +12,8 @@ def register(mcp): def bbdev_bebop_p2e_buildbitstream( chip: str, diff: bool = False, + itrace: bool = False, + mtrace: bool = False, vsrc_dir: Optional[str] = None, output_dir: Optional[str] = None, resume_post_route: bool = False, @@ -24,7 +26,7 @@ def bbdev_bebop_p2e_buildbitstream( submit( "/bebop/p2e/buildbitstream", opt( - {"chip": chip, "diff": diff, "resume_post_route": resume_post_route}, + {"chip": chip, "diff": diff, "itrace": itrace, "mtrace": mtrace, "resume_post_route": resume_post_route}, vsrc_dir=vsrc_dir, output_dir=output_dir, stop_after=stop_after, From cd7227c32176b15a466aaaebbf98cfc5d5a9f2d5 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 6 Oct 2026 18:18:05 +0800 Subject: [PATCH 182/196] feat: add support for SoC test workloads in build process and update related documentation --- bbdev/api/steps/workload/01_buidl_api.step.py | 14 +++++++++----- bbdev/api/steps/workload/01_build_event.step.py | 12 ++++++++---- bbdev/api/steps/workload/README.md | 4 ++++ bbdev/bbdev | 2 +- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/bbdev/api/steps/workload/01_buidl_api.step.py b/bbdev/api/steps/workload/01_buidl_api.step.py index 86164c3f5..12c31422b 100644 --- a/bbdev/api/steps/workload/01_buidl_api.step.py +++ b/bbdev/api/steps/workload/01_buidl_api.step.py @@ -20,7 +20,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - allowed = {"chip", "stable", "ctest", "mlirtest"} + allowed = {"chip", "stable", "ctest", "mlirtest", "soctest"} unknown = sorted((k for k in body if k not in allowed)) if unknown: return ApiResponse( @@ -50,19 +50,23 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: ) ctest = body.get("ctest", False) mlirtest = body.get("mlirtest", False) - if not isinstance(ctest, bool) or not isinstance(mlirtest, bool): + soctest = body.get("soctest", False) + if not all(isinstance(flag, bool) for flag in (ctest, mlirtest, soctest)): return ApiResponse( - status=400, body={"error": "--ctest and --mlirtest must be boolean flags"} + status=400, + body={"error": "--ctest, --mlirtest and --soctest must be boolean flags"}, ) - if ctest and mlirtest: + if sum((ctest, mlirtest, soctest)) > 1: return ApiResponse( - status=400, body={"error": "--ctest and --mlirtest cannot be used together"} + status=400, + body={"error": "--ctest, --mlirtest and --soctest are mutually exclusive"}, ) data = { "chip": chip, "stable": stable, "ctest": ctest, "mlirtest": mlirtest, + "soctest": soctest, } await ctx.enqueue( {"topic": "workload.build", "data": {**data, "_trace_id": ctx.trace_id}} diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index 51b2f4c0e..5c8929af2 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -38,6 +38,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "stable", "ctest", "mlirtest", + "soctest", "_trace_id", } unknown = sorted((k for k in input_data if k not in allowed)) @@ -96,8 +97,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return ctest = input_data.get("ctest", False) mlirtest = input_data.get("mlirtest", False) - if not isinstance(ctest, bool) or not isinstance(mlirtest, bool): - ctx.logger.error("--ctest and --mlirtest must be boolean flags") + soctest = input_data.get("soctest", False) + if not all(isinstance(flag, bool) for flag in (ctest, mlirtest, soctest)): + ctx.logger.error("--ctest, --mlirtest and --soctest must be boolean flags") await check_result( ctx, 1, @@ -106,8 +108,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - if ctest and mlirtest: - ctx.logger.error("--ctest and --mlirtest cannot be used together") + if sum((ctest, mlirtest, soctest)) > 1: + ctx.logger.error("--ctest, --mlirtest and --soctest are mutually exclusive") await check_result( ctx, 1, @@ -122,6 +124,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: chip, ctest=ctest, mlirtest=mlirtest, + soctest=soctest, stable=stable, logger=ctx.logger, task_scope=origin_tid, @@ -148,6 +151,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "chip": chip, "ctest": ctest, "mlirtest": mlirtest, + "soctest": soctest, }, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/workload/README.md b/bbdev/api/steps/workload/README.md index b852dc172..8b640d616 100644 --- a/bbdev/api/steps/workload/README.md +++ b/bbdev/api/steps/workload/README.md @@ -27,6 +27,7 @@ bbdev workload --clean "--chip toy" - **`stable`** - Optional boolean flag. If set, build with stable LLVM Buckyball extensions. - **`ctest`** - Build CTest workloads only. - **`mlirtest`** - Build MLIRTest workloads only. +- **`soctest`** - Build SoC test workloads only. For chip workloads under paths like `*/chips/`, only the directory selected by `chip` is synced to `bb-tests/output//workloads`. @@ -40,6 +41,9 @@ bbdev workload --build "--chip pebble --ctest" # Build only MLIRTest workloads bbdev workload --build "--chip pebble --mlirtest" + +# Build only SoC test workloads +bbdev workload --build "--chip goban --soctest" ``` **Response**: diff --git a/bbdev/bbdev b/bbdev/bbdev index 480da42f8..0b3aaafdc 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -41,7 +41,7 @@ WORKFLOW_COMMANDS = { }, "workload": { "clean": 'Clean workload output directory. Args: "--chip "', - "build": 'Build test workloads. Args: "--chip [--stable] [--ctest|--mlirtest]"', + "build": 'Build test workloads. Args: "--chip [--stable] [--ctest|--mlirtest|--soctest]"', "tohex": 'Convert chip workload ELF to hex. Args: "--chip "', }, "model": { From 4b1f87bebd936371efa78dc7b9328b092d6a75e4 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 6 Oct 2026 21:01:27 +0800 Subject: [PATCH 183/196] refactor: remove resume-post-route handling from P2E build steps and update related logic --- .../bebop/p2e/02_buildbitstream_api.step.py | 18 ----- .../bebop/p2e/02_buildbitstream_event.step.py | 10 +-- bbdev/api/tests/test_p2e_resume.py | 67 ------------------- bbdev/bbdev | 2 +- bbdev/mcp/tools/bebop_p2e_buildbitstream.py | 3 +- 5 files changed, 4 insertions(+), 96 deletions(-) delete mode 100644 bbdev/api/tests/test_p2e_resume.py diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py index 12e891a9e..a1bc91789 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py @@ -24,29 +24,12 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir, chip, "p2e", body.get("vsrc_dir") or body.get("vsrc-dir") ) - resume = body.get("resume_post_route", body.get("resume-post-route", False)) - if not isinstance(resume, bool): - return ApiResponse(status=400, body={"error": "resume-post-route must be a boolean flag"}) stop_after = body.get("stop_after", body.get("stop-after")) if stop_after is not None and stop_after not in ("vsyn", "vcom"): return ApiResponse(status=400, body={"error": "stop-after must be vsyn or vcom"}) - if resume and stop_after is not None: - return ApiResponse(status=400, body={"error": "stop-after conflicts with resume-post-route"}) output_dir = body.get("output_dir") or body.get("output-dir") if stop_after and output_dir and Path(output_dir).exists(): return ApiResponse(status=400, body={"error": "Resource assessment requires a fresh output_dir"}) - if resume: - if not isinstance(output_dir, str) or not output_dir: - return ApiResponse(status=400, body={"error": "Post-route resume requires output_dir"}) - case = Path(output_dir).resolve() - build_root = (Path(bbdir) / "bebop/build").resolve() - if build_root not in case.parents or not case.is_dir(): - return ApiResponse(status=400, body={"error": "P2E resume requires an existing case under bebop/build"}) - for artifact in ("xepic_vvac_top_0_0_route.dcp", "xepic_vvac_top_0_0.bit"): - path = case / "fpgaCompDir/part_b0_f0/pnrDir" / artifact - if not path.is_file(): - return ApiResponse(status=400, body={"error": f"Post-route resume requires {path}"}) - data = { "chip": chip, "diff": bool(body.get("diff", False)), @@ -54,7 +37,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "mtrace": bool(body.get("mtrace", False)), "vsrc_dir": vsrc_dir, "output_dir": output_dir, - "resume_post_route": resume, "stop_after": stop_after, } await ctx.enqueue({ diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index e9767de4c..e6087bacc 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -53,10 +53,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: normalize_p2e_timescale(vsrc_dir, ctx.logger) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") diff = bool(input_data.get("diff", False)) - resume = input_data.get("resume_post_route", False) stop_after = input_data.get("stop_after") - if resume and not input_data.get("output_dir"): - raise ValueError("Post-route resume requires an explicit output_dir") build_dir = ( input_data.get("output_dir") or f"{bebop_dir}/build/{chip}{'-diff' if diff else ''}-{timestamp}" @@ -65,13 +62,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: build_root = (Path(bebop_dir) / "build").resolve() if build_root not in build_path.parents: raise ValueError(f"P2E output_dir must be under {build_root}: {build_path}") - if resume and not build_path.is_dir(): - raise ValueError(f"P2E resume case does not exist: {build_path}") if stop_after and build_path.exists(): raise ValueError(f"Resource assessment requires a fresh output_dir: {build_path}") - if build_path.exists() and not resume: + if build_path.exists(): shutil.rmtree(build_path) - build_path.mkdir(parents=True, exist_ok=resume) + build_path.mkdir(parents=True) manifest = Path(bbdir) / "examples" / "chips" / chip / "generated" / "bebop" / "Cargo.toml" features = ["p2e"] @@ -104,7 +99,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: *(["--diff"] if diff else []), *(["--itrace"] if input_data.get("itrace", False) else []), *(["--mtrace"] if input_data.get("mtrace", False) else []), - *(["--resume-post-route"] if resume else []), *(["--stop-after", stop_after] if stop_after else []), ]) build_cmd = f"cd {shlex.quote(bbdir)} && {build_cmd}" diff --git a/bbdev/api/tests/test_p2e_resume.py b/bbdev/api/tests/test_p2e_resume.py deleted file mode 100644 index 0fdd16109..000000000 --- a/bbdev/api/tests/test_p2e_resume.py +++ /dev/null @@ -1,67 +0,0 @@ -import importlib.util -from pathlib import Path -import tempfile -import sys -from types import SimpleNamespace -import unittest -from unittest.mock import AsyncMock, patch - - -API = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(API)) -spec = importlib.util.spec_from_file_location( - "p2e_build_api", API / "steps/bebop/p2e/02_buildbitstream_api.step.py" -) -event = importlib.util.module_from_spec(spec) -spec.loader.exec_module(event) - - -class P2eResumeTest(unittest.IsolatedAsyncioTestCase): - def setUp(self): - self.directory = tempfile.TemporaryDirectory() - self.addCleanup(self.directory.cleanup) - self.root = Path(self.directory.name) - self.case = self.root / "bebop/build/toy-test" - self.routed = self.case / "fpgaCompDir/part_b0_f0/pnrDir" - self.routed.mkdir(parents=True) - self.artifacts = [self.routed / name for name in ( - "xepic_vvac_top_0_0_route.dcp", "xepic_vvac_top_0_0.bit" - )] - for path in self.artifacts: - path.write_bytes(b"existing artifact") - self.context = SimpleNamespace(enqueue=AsyncMock(), trace_id="test") - - async def request(self, **options): - body = {"chip": "toy", "diff": True, "resume-post-route": True, - "vsrc_dir": str(self.root / "rtl"), "output_dir": str(self.case)} - body.update(options) - with patch.object(event, "get_buckyball_path", return_value=str(self.root)): - return await event.handler(SimpleNamespace(body=body), self.context) - - async def test_routed_case_is_enqueued_without_mutation(self): - response = await self.request() - self.assertEqual(response.status, 202) - data = self.context.enqueue.await_args.args[0]["data"] - self.assertTrue(data["resume_post_route"]) - for path in self.artifacts: - self.assertEqual(path.read_bytes(), b"existing artifact") - - async def test_incomplete_case_is_rejected_before_enqueue(self): - self.artifacts[1].unlink() - response = await self.request() - self.assertEqual(response.status, 400) - self.context.enqueue.assert_not_awaited() - - async def test_resume_requires_explicit_case(self): - response = await self.request(output_dir=None) - self.assertEqual(response.status, 400) - self.context.enqueue.assert_not_awaited() - - async def test_case_outside_build_root_is_rejected(self): - response = await self.request(output_dir=str(self.root)) - self.assertEqual(response.status, 400) - self.context.enqueue.assert_not_awaited() - - -if __name__ == "__main__": - unittest.main() diff --git a/bbdev/bbdev b/bbdev/bbdev index 0b3aaafdc..bb902e18e 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -80,7 +80,7 @@ WORKFLOW_COMMANDS = { "bebop-p2e": { "clean": 'Clean Bebop P2E build directory. Args: "--chip [--output-dir ]"', "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--chip [--diff] [--output-dir ]"', - "buildbitstream": 'Build Bebop P2E runtime case. Args: "--chip [--diff] [--itrace] [--mtrace] [--vsrc-dir ] [--output-dir ] [--resume-post-route | --stop-after vsyn|vcom]"; stop-after produces resource assessment only, not a runnable case; resume requires an existing routed case and preserves its completed synthesis/PNR', + "buildbitstream": 'Build Bebop P2E runtime case. Args: "--chip [--diff] [--itrace] [--mtrace] [--vsrc-dir ] [--output-dir ] [--stop-after vsyn|vcom]"; stop-after produces resource assessment only, not a runnable case', "runworkload": 'Cold-load Bebop P2E workload. Args: "--chip (--image | --load-manifest ) --bitstream [--fpga-location ] [--multi-fpga] [--wave] [--wave-start ] [--diff (image only)] [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--reuse-runtime]"', "batch": 'Run Bebop P2E batch regression. Args: "--chip --bitstream --test [--diff] [--fpga-location ]"', }, diff --git a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py index a536e42c5..df6341896 100644 --- a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py +++ b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py @@ -16,7 +16,6 @@ def bbdev_bebop_p2e_buildbitstream( mtrace: bool = False, vsrc_dir: Optional[str] = None, output_dir: Optional[str] = None, - resume_post_route: bool = False, stop_after: Optional[str] = None, ) -> str: """Build bebop-p2e bitstream. POST /bebop/p2e/buildbitstream.""" @@ -26,7 +25,7 @@ def bbdev_bebop_p2e_buildbitstream( submit( "/bebop/p2e/buildbitstream", opt( - {"chip": chip, "diff": diff, "itrace": itrace, "mtrace": mtrace, "resume_post_route": resume_post_route}, + {"chip": chip, "diff": diff, "itrace": itrace, "mtrace": mtrace}, vsrc_dir=vsrc_dir, output_dir=output_dir, stop_after=stop_after, From ec925ff9057275719b4b7a2cd5a585eccca90a6c Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 6 Oct 2026 23:16:58 +0800 Subject: [PATCH 184/196] refactor: enhance parameter derivation logic and update hart visibility handling in tile configuration --- .../config/scripts/2_parameter_derivation.py | 12 ++++++++---- bbdev/api/steps/kernel/01_build_event.step.py | 16 +++++++++++----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/bbdev/api/steps/config/scripts/2_parameter_derivation.py b/bbdev/api/steps/config/scripts/2_parameter_derivation.py index 710a6a91a..ea2548dfb 100644 --- a/bbdev/api/steps/config/scripts/2_parameter_derivation.py +++ b/bbdev/api/steps/config/scripts/2_parameter_derivation.py @@ -363,7 +363,9 @@ def _derive_tiles( "mem_ball_channel_num": mem_ball_channel_num, } ) - if tile["kind"] == "compute": + if tile["kind"] == "compute" or ( + n > 1 and has_buckyball and cores[indices[0]]["ball_num"] == 0 + ): placements[-1]["controller_core_index"] = indices[0] offset += n return placements @@ -504,8 +506,8 @@ def _derive_targets( def _derive_harts(tiles: list[dict[str, Any]], cores: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Main tile cores are harts 0..m-1 and the only Linux-visible harts. Compute tile t - (1..N) has its controller at m+t-1; workers follow from m+N, tile by tile.""" + """A main tile with a task controller exposes only core 0 to Linux. + Physical hart IDs cover the main tile, then compute controllers and workers.""" main = tiles[0] compute = tiles[1:] m = main["cores_per_tile"] @@ -529,7 +531,9 @@ def _derive_harts(tiles: list[dict[str, Any]], cores: list[dict[str, Any]]) -> l "tile_id": tile_id, "core_id": local, "core_index": core_index, - "visible": tile["kind"] == "main", + "visible": tile["kind"] == "main" and ( + "controller_core_index" not in tile or local == 0 + ), "target": _target_name(inst["role"], inst["pkg"]), "pkg": inst["pkg"], "role": inst["role"], diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index 96c3733ab..7e399fa95 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -3,6 +3,7 @@ import re import shutil import sys +import tomllib from motia import FlowContext, queue @@ -179,7 +180,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: kernel_src = os.path.join(bbdir, "bb-tests", "workloads", "lib", "kernel") try: + model = kernel_model(input_data) + if model and not input_data.get("chip"): + raise ValueError("--model requires --chip") + chip = kernel_chip(input_data, bbdir) memory_value = input_data.get("guest-memory-mib", 512) + if model and "guest-memory-mib" not in input_data: + config_path = os.path.join( + bbdir, "examples", "models", chip, model, "configs", "running-param.toml" + ) + with open(config_path, "rb") as stream: + memory_value = tomllib.load(stream).get("memory_mib", 512) if ( isinstance(memory_value, bool) or not str(memory_value).isdigit() @@ -197,12 +208,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ): raise ValueError("pmem requires --model and guest-memory-mib < 16384") hart_params = hart_count_params(input_data) - model = kernel_model(input_data) - if model and not input_data.get("chip"): - raise ValueError("--model requires --chip") interactive = kernel_interactive(input_data) - # Linux mode needs OS overlay; model mode only needs chip for the model artifact path - chip = kernel_chip(input_data, bbdir) except ValueError as e: ctx.logger.error(str(e)) await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) From b5ac9823d7f066ca7d691efa6be466734708ed75 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 9 Oct 2026 19:15:35 +0800 Subject: [PATCH 185/196] refactor: update command arguments and validation in Bebop workflows for improved clarity and functionality --- bbdev/api/steps/bebop/bemu/02_sim_api.step.py | 16 +- .../api/steps/bebop/bemu/02_sim_event.step.py | 58 ++---- .../steps/bebop/bemu/scripts/bemu_common.py | 34 +-- .../api/steps/bebop/bemu/scripts/model_sim.py | 61 +++++- bbdev/api/steps/bebop/performance_report.py | 163 ++++++++++----- .../steps/bebop/vcs/03_batch_event.step.py | 2 +- .../bebop/verilator/02_verilog_api.step.py | 1 + .../steps/bebop/verilator/04_sim_api.step.py | 2 + .../bebop/verilator/04_sim_event.step.py | 12 +- .../steps/bebop/verilator/05_run_api.step.py | 2 + bbdev/api/steps/config/01_install_api.step.py | 14 +- .../api/steps/config/01_install_event.step.py | 5 +- .../config/scripts/2_parameter_derivation.py | 197 +++++++++--------- .../api/steps/config/scripts/3_json2proto.py | 59 ++++-- .../steps/config/scripts/ball_normalize.py | 46 +--- .../api/steps/config/scripts/proto/chip.proto | 24 ++- bbdev/api/steps/kernel/01_build_api.step.py | 8 +- bbdev/api/steps/kernel/01_build_event.step.py | 80 +++---- .../06_bebop_verilator_verilog_event.step.py | 2 + bbdev/api/steps/uvm/scripts/uvm_common.py | 17 +- bbdev/api/steps/vcs/04_sim_event.step.py | 1 + .../api/steps/verilator/04_sim_event.step.py | 33 +-- bbdev/api/tests/test_api_kernel_build.py | 2 +- bbdev/api/utils/report_schema.py | 24 ++- bbdev/bbdev | 6 +- bbdev/mcp/common.py | 3 - bbdev/mcp/tools/bebop_verilator_run.py | 2 - bbdev/mcp/tools/bebop_verilator_sim.py | 2 - bbdev/mcp/tools/bebop_verilator_verilog.py | 4 +- bbdev/mcp/tools/bemu_sim.py | 8 +- bbdev/mcp/tools/kernel_build.py | 9 +- 31 files changed, 509 insertions(+), 388 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py index 9f5a1fa3a..cdff96dae 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py @@ -29,18 +29,20 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: return ApiResponse( status=400, body={"error": "Unknown parameter: pk or host-io"} ) - if body.get("system") and body.get("core_index") is not None: + if "system" in body: + return ApiResponse(status=400, body={"error": "Unknown parameter: system"}) + if body.get("core_index") is not None and any( + key in body for key in ("dtb", "initrd", "memory-mib", "load-manifest") + ): return ApiResponse( - status=400, - body={ - "error": "system boot runs all chip harts; core_index cannot be supplied" - }, + status=400, body={"error": "chip boot options cannot select a single core"} ) - if not body.get("system") and any( + if body.get("load-manifest") and any( key in body for key in ("dtb", "initrd", "memory-mib") ): return ApiResponse( - status=400, body={"error": "dtb, initrd and memory-mib require system boot"} + status=400, + body={"error": "load-manifest conflicts with dtb, initrd and memory-mib"}, ) chip = body.get("chip", "") try: diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index b1881e5f2..894f517c8 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -6,6 +6,7 @@ import os import shlex +import shutil import sys from datetime import datetime from pathlib import Path @@ -28,7 +29,7 @@ from steps.bebop.bemu.scripts.model_sim import model_run_commands from utils.event_common import check_result, get_origin_trace_id from utils.process_registry import cancellation_requested -from bemu_common import bemu_chip_binary, bemu_manifest, bemu_tile_index +from bemu_common import bemu_chip_binary, bemu_manifest from steps.bebop.performance_report import performance_report from utils.reports import source_context @@ -43,26 +44,8 @@ def clean_model_trace(binary_dir: str) -> None: trace_dir = Path(binary_dir) / "trace" - for subdir in ("cycle", "tensor"): - target_dir = trace_dir / subdir - if not target_dir.exists(): - continue - if not target_dir.is_dir(): - raise NotADirectoryError(f"trace path is not a directory: {target_dir}") - for path in target_dir.glob("trace-*.txt"): - if not path.is_file(): - raise FileNotFoundError(f"trace path is not a file: {path}") - path.unlink() - summary = target_dir / "summary.txt" - if summary.exists(): - if not summary.is_file(): - raise FileNotFoundError(f"trace summary path is not a file: {summary}") - summary.unlink() - perfetto = trace_dir / "perfetto.json" - if perfetto.exists(): - if not perfetto.is_file(): - raise FileNotFoundError(f"perfetto path is not a file: {perfetto}") - perfetto.unlink() + if trace_dir.exists(): + shutil.rmtree(trace_dir) def resolve_bemu_binary(bbdir: str, chip: str, binary_name: str) -> str | None: @@ -200,18 +183,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: run_log = log_dir(bbdir, chip, "verilog", timestamp, "bemu", binary_name) os.makedirs(run_log, exist_ok=True) core_index = input_data.get("core_index") - system = bool(input_data.get("system", False)) - if system and core_index is not None: - raise ValueError( - "system boot runs all chip harts; core_index cannot be supplied" - ) - # The chip's tile runner by default; --core-index selects the single-core bebop-bemu. - if core_index is None: - entry = [bemu_chip_binary(chip), "--"] - if not system: - entry.extend(["--tile-index", str(bemu_tile_index(chip, bbdir))]) - else: - entry = ["bebop-bemu", "--"] + if "system" in input_data: + raise ValueError("Unknown parameter: system") + if core_index is not None and any( + key in input_data for key in ("dtb", "initrd", "memory-mib", "load-manifest") + ): + raise ValueError("chip boot options cannot select a single core") + entry = [bemu_chip_binary(chip) if core_index is None else "bebop-bemu", "--"] cargo_args = [ "cargo", "run", @@ -233,13 +211,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--config", "profile.release.codegen-units=1", ] - if system: - cargo_args.append("--system") - for option in ("dtb", "initrd"): - if input_data.get(option): - cargo_args.extend([f"--{option}", input_data[option]]) - if input_data.get("memory-mib") is not None: - cargo_args.extend(["--memory-mib", str(input_data["memory-mib"])]) + for option in ("dtb", "initrd", "load-manifest"): + if input_data.get(option): + cargo_args.extend([f"--{option}", input_data[option]]) + if input_data.get("memory-mib") is not None: + cargo_args.extend(["--memory-mib", str(input_data["memory-mib"])]) if core_index is not None: cargo_args.extend(["--core-index", str(core_index)]) if input_data.get("disasm"): @@ -250,6 +226,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if input_data.get(trace_name, False): cargo_args.append(f"--{trace_name}") arguments = input_data.get("arguments", []) + if arguments and core_index is None: + raise ValueError("chip boot does not accept process arguments") if arguments: if not isinstance(arguments, list) or not all( isinstance(arg, str) for arg in arguments diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py index 80306dce2..b0aafeece 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py @@ -1,4 +1,3 @@ -import sys from pathlib import Path from utils.event_common import require_chip @@ -10,35 +9,24 @@ def _repo(bbdir: str | None) -> Path: return Path(bbdir or get_buckyball_path()) -def _chip(chip: str, bbdir: str | None = None): - chip = require_chip({"chip": chip}) - root = _repo(bbdir) - scripts = root / "bbdev" / "api" / "steps" / "config" / "scripts" - if str(scripts) not in sys.path: - sys.path.insert(0, str(scripts)) - import chip_pb2 - - path = root / "examples" / "chips" / chip / "configs" / "generated" / "chip.pb" - if not path.is_file(): - raise FileNotFoundError(f"missing {path}; run bbdev config --install") - msg = chip_pb2.Chip() - msg.ParseFromString(path.read_bytes()) - return msg - - def bemu_manifest(chip: str, bbdir: str | None = None) -> Path: chip = require_chip({"chip": chip}) root = _repo(bbdir) - path = root / "examples" / "chips" / chip / "configs" / "generated" / "bemu" / "Cargo.toml" + path = ( + root + / "examples" + / "chips" + / chip + / "configs" + / "generated" + / "bemu" + / "Cargo.toml" + ) if not path.is_file(): raise FileNotFoundError(f"missing {path}; run bbdev config --install") return path def bemu_chip_binary(chip: str) -> str: - """The chip's tile-level BEMU entry in its generated crate.""" + """The chip's BEMU boot entry in its generated crate.""" return f"bebop-chip-{require_chip({'chip': chip})}" - - -def bemu_tile_index(chip: str, bbdir: str | None = None) -> int: - return _chip(chip, bbdir).bemu.tile_index diff --git a/bbdev/api/steps/bebop/bemu/scripts/model_sim.py b/bbdev/api/steps/bebop/bemu/scripts/model_sim.py index 34d89e986..5482bd2e3 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/model_sim.py +++ b/bbdev/api/steps/bebop/bemu/scripts/model_sim.py @@ -1,4 +1,5 @@ import json +import importlib.util import re import tomllib from datetime import datetime @@ -8,7 +9,16 @@ def model_run_commands(repo: str, params: dict): - unknown = params.keys() - {"chip", "model", "reuse-simulator", "itrace", "mtrace"} + unknown = params.keys() - { + "chip", + "model", + "reuse-simulator", + "itrace", + "mtrace", + "firmware", + "guest-memory-mib", + "load-manifest", + } if unknown: raise ValueError(f"Unknown model simulation parameters: {sorted(unknown)}") for key in ("chip", "model"): @@ -39,9 +49,34 @@ def model_run_commands(repo: str, params: dict): python = root / "result/bin/python3" elif kind == "python" or (kind == "native" and "tile_indices" in settings): binary = f"bebop-chip-{chip}" - python = root / ("result/bin/python3" if kind == "native" else "stack/serving/.venv/bin/python") + python = root / "result/bin/python3" else: raise ValueError(f"Unknown model execution kind: {kind}") + firmware = None + endpoint = layout["execution"].get("p2e", {}) + if endpoint.get("task_runtime") == "ant" or ( + kind == "native" and layout["execution"]["input"] == "resource" + ): + if kind != "native" or endpoint.get("kind") != "native": + raise ValueError("prepared model execution must run inside the Linux guest") + binary = f"bebop-chip-{chip}" + memory = params.get("guest-memory-mib", settings["memory_mib"]) + if memory is not None: + if isinstance(memory, bool) or not str(memory).isdigit() or int(memory) < 1: + raise ValueError("guest-memory-mib must be a positive integer") + memory = int(memory) + spec = importlib.util.spec_from_file_location( + "buckyball_model_guest", root / "stack/serving/guest.py" + ) + guest = importlib.util.module_from_spec(spec) + spec.loader.exec_module(guest) + firmware, guest_memory, _ = guest.firmware_profile( + root, chip, model, params.get("firmware"), memory + ) + elif any( + key in params for key in ("firmware", "guest-memory-mib", "load-manifest") + ): + raise ValueError("firmware options require a packaged Linux guest model") target = root / "bebop/target" / chip stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") log = Path(log_dir(repo, chip, "verilog", stamp, "bemu", model)) @@ -80,10 +115,30 @@ def model_run_commands(repo: str, params: dict): str(config), ], ] + if firmware is not None: + commands[-1].extend( + ["--firmware", str(firmware), "--guest-memory-mib", str(guest_memory)] + ) + if firmware.name.endswith("-ddr.elf"): + manifest = firmware.with_suffix(".load.json") + if ( + "load-manifest" in params + and Path(params["load-manifest"]).resolve() != manifest.resolve() + ): + raise ValueError( + "load-manifest does not belong to the requested firmware" + ) + if not manifest.is_file(): + raise ValueError(f"MODEL_DDR load manifest missing: {manifest}") + commands[-1].extend(["--load-manifest", str(manifest)]) + elif "load-manifest" in params: + raise ValueError("load-manifest requires a MODEL_DDR firmware") if params.get("reuse-simulator", False): simulator = target / "release" / binary if not simulator.is_file(): raise ValueError(f"Reusable model simulator missing: {simulator}") commands = commands[1:] - commands[-1].extend(f"--{flag}" for flag in ("itrace", "mtrace") if params.get(flag, False)) + commands[-1].extend( + f"--{flag}" for flag in ("itrace", "mtrace") if params.get(flag, False) + ) return commands, log diff --git a/bbdev/api/steps/bebop/performance_report.py b/bbdev/api/steps/bebop/performance_report.py index 402a4bce9..591462089 100644 --- a/bbdev/api/steps/bebop/performance_report.py +++ b/bbdev/api/steps/bebop/performance_report.py @@ -7,82 +7,118 @@ from utils.reports import new_report, attachment, finish_report from utils.report_schema import number, nonempty, CATEGORIES -from bisect import bisect_right def parse_trace(data, categories): - events = data["traceEvents"] groups = {} - for event in events: + for event in data["traceEvents"]: if event.get("ph") != "X" or event.get("cat") != "buddy.trace": continue args = event["args"] + counter = args["counter"] + if counter not in ("riscv-cycle", "x86-tsc"): + raise ValueError(f"invalid trace counter: {counter}") + unit = "cycle" if counter == "riscv-cycle" else "tick" path = args["id_path"] if not isinstance(path, list) or not path: raise ValueError("trace id_path must be a non-empty list") for part in path: number(part, "trace id", integer=True) - identity = "-".join(str(int(part)) for part in path) - start = number(args["start_cycle"], "start_cycle", integer=True) - end = number(args["end_cycle"], "end_cycle", integer=True) - elapsed = number(args["elapsed_cycle"], "elapsed_cycle", integer=True) + controller = int(number(args["controller"], "controller", integer=True)) + core = int(number(args["core"], "core", integer=True)) + platform = args["platform"] + if platform not in ("linux", "baremetal"): + raise ValueError(f"invalid trace platform: {platform}") + number(event["pid"], "pid", integer=True, positive=platform == "linux") + number(event["tid"], "tid", integer=True, positive=platform == "linux") + if platform == "baremetal" and event["pid"] != 0: + raise ValueError("baremetal trace requires pid=0") + key = (controller, core, tuple(path)) + identity = f"controller-{controller}/core-{core}/" + "-".join(map(str, path)) + start, end = args[f"start_{unit}"], args[f"end_{unit}"] + if any( + isinstance(value, bool) + or not isinstance(value, int) + or value < 0 + or value >= 2**64 + for value in (start, end) + ): + raise ValueError("counter boundaries must be unsigned 64-bit integers") + elapsed = number(args[f"elapsed_{unit}"], "elapsed_cycle", integer=True) if end < start or end - start != elapsed or event["dur"] != elapsed: raise ValueError(f"inconsistent cycle interval for {identity}") name = nonempty(event["name"], "operator name") category = categories.get(name, "Other") if category not in CATEGORIES: raise ValueError(f"unknown operator category: {category}") - groups.setdefault(identity, []).append( + groups.setdefault(key, []).append( { "id": identity, "name": name, "category": category, - "cycles": int(elapsed), - "start_cycle": int(start), + "counter": counter, + "duration": int(elapsed), + "start": int(start), + "end": int(end), "level": len(path) - 1, } ) if not groups: raise ValueError("no buddy.trace cycle intervals found") - starts = {} - for identity, records in groups.items(): - records.sort(key=lambda op: op["start_cycle"]) - starts[identity] = [op["start_cycle"] for op in records] + if ( + len({record["counter"] for records in groups.values() for record in records}) + != 1 + ): + raise ValueError("performance trace cannot mix counter sources") + for key, records in groups.items(): + records.sort(key=lambda op: op["start"]) for previous, current in zip(records, records[1:]): if previous["name"] != current["name"]: - raise ValueError(f"trace id has inconsistent names: {identity}") - if previous["start_cycle"] + previous["cycles"] > current["start_cycle"]: - raise ValueError(f"overlapping calls of trace {identity}") - for identity, records in groups.items(): - parts = identity.split("-") - if len(parts) > 1: - parent_id = "-".join(parts[:-1]) - if parent_id not in groups: - raise ValueError(f"missing parent for trace {identity}") - for op in records: - index = bisect_right(starts[parent_id], op["start_cycle"]) - 1 - if index < 0: - raise ValueError(f"nested trace outside parent: {identity}") - parent = groups[parent_id][index] - if ( - op["start_cycle"] + op["cycles"] - > parent["start_cycle"] + parent["cycles"] - ): - raise ValueError(f"nested trace outside parent: {identity}") - start = min(records[0]["start_cycle"] for records in groups.values()) + raise ValueError(f"trace scope has inconsistent names: {key}") + if previous["start"] + previous["duration"] > current["start"]: + raise ValueError(f"overlapping calls of trace scope {key}") + for (controller, core, path), records in groups.items(): + if len(path) == 1: + continue + for op in records: + parents = [ + parent + for (owner, _, parent_path), calls in groups.items() + if owner == controller and parent_path == path[:-1] + for parent in calls + if parent["start"] <= op["start"] + and op["start"] + op["duration"] <= parent["start"] + parent["duration"] + ] + if len(parents) != 1: + raise ValueError( + f"trace scope {controller}/{core}/{path} has {len(parents)} parents" + ) + start = min(record["start"] for records in groups.values() for record in records) end = max( - records[-1]["start_cycle"] + records[-1]["cycles"] + record["start"] + record["duration"] for records in groups.values() + for record in records ) if start == end: raise ValueError("trace span must be positive") operators = [] for records in groups.values(): - op = dict(records[0]) - op["cycles"] = sum(record["cycles"] for record in records) - op["calls"] = len(records) - op["start_cycle"] -= start - operators.append(op) + first = records[0] + unit = "cycle" if first["counter"] == "riscv-cycle" else "tick" + plural = "cycles" if unit == "cycle" else "ticks" + operators.append( + { + "id": first["id"], + "name": first["name"], + "category": first["category"], + "counter": first["counter"], + "level": first["level"], + "calls": len(records), + plural: sum(record["duration"] for record in records), + f"start_{unit}": first["start"] - start, + f"end_{unit}": records[-1]["end"] - start, + } + ) return int(end - start), operators @@ -115,14 +151,27 @@ def performance_report(repo, context, trace_id, backend, log, stdout, returncode output = log / "report" report["clock_hz"] = settings.get(backend, {}).get("clock_hz") expected = settings.get("expected_classes") - correct = samples = elapsed = cycles = 0 + counters = {record["counter"] for record in records} + if len(counters) != 1 or not counters <= {"riscv-cycle", "x86-tsc"}: + raise ValueError("benchmark records require one explicit counter source") + benchmark_counter = counters.pop() + benchmark_count_field = "cycles" if benchmark_counter == "riscv-cycle" else "ticks" + correct = samples = elapsed = measured = 0 for index, record in enumerate(records): number(record["samples"], "samples", positive=True, integer=True) number(record["elapsed_ns"], "elapsed_ns", positive=True, integer=True) - number(record["cycles"], "cycles", positive=True, integer=True) + number( + record[benchmark_count_field], + benchmark_count_field, + positive=True, + integer=True, + ) samples += record["samples"] elapsed += record["elapsed_ns"] - cycles += record["cycles"] + other = "ticks" if benchmark_count_field == "cycles" else "cycles" + if other in record: + raise ValueError("benchmark record mixes cycle and tick fields") + measured += record[benchmark_count_field] if "correct" in record: number(record["correct"], "correct samples", integer=True) if record["correct"] > record["samples"]: @@ -156,9 +205,10 @@ def performance_report(repo, context, trace_id, backend, log, stdout, returncode operators = {} all_events = [] trace_span = 0 + trace_counter = None trace_roots = sorted( { - file.parent.parent + file.parents[3] for file in log.rglob("trace-*.txt") if file.parent.name == "cycle" } @@ -170,25 +220,38 @@ def performance_report(repo, context, trace_id, backend, log, stdout, returncode perfetto = importlib.util.module_from_spec(spec) spec.loader.exec_module(perfetto) metadata = [build / f"models/{model}/subgraph0.trace.mlir"] - for index, root in enumerate(trace_roots): + for root in trace_roots: data = perfetto.build_perfetto(root, recipe / "trace/trace.toml", metadata) span, ops = parse_trace(data, settings.get("operator_categories", {})) + counter = ops[0]["counter"] + if trace_counter is not None and trace_counter != counter: + raise ValueError("performance roots cannot mix counter sources") + if counter != benchmark_counter: + raise ValueError("benchmark and trace counter sources differ") + trace_counter = counter + count_field = "cycles" if counter == "riscv-cycle" else "ticks" + offset = trace_span trace_span += span + unit = "cycle" if counter == "riscv-cycle" else "tick" for op in ops: + op[f"start_{unit}"] += offset + op[f"end_{unit}"] += offset if op["id"] in operators: - operators[op["id"]]["cycles"] += op["cycles"] + operators[op["id"]][count_field] += op[count_field] operators[op["id"]]["calls"] += op["calls"] + operators[op["id"]][f"end_{unit}"] = op[f"end_{unit}"] else: operators[op["id"]] = op - for event in data["traceEvents"]: - event["pid"] = index all_events.extend(data["traceEvents"]) + counter = trace_counter if trace_roots else benchmark_counter + count_field = "cycles" if counter == "riscv-cycle" else "ticks" result = { + "counter": counter, "name": model, "status": "pass" if returncode == 0 else "fail", "workload_hash": workload.hexdigest(), "dataset_hash": dataset.hexdigest(), - "cycles": trace_span if trace_roots else cycles, + count_field: trace_span if trace_roots else measured, "accuracy": accuracy, "accuracy_metric": records[0].get( "accuracy_metric", settings["accuracy_metric"] @@ -206,7 +269,7 @@ def performance_report(repo, context, trace_id, backend, log, stdout, returncode } if returncode != 0: result.update( - cycles=None, + **{count_field: None}, accuracy=None, latency_ms=None, throughput=None, diff --git a/bbdev/api/steps/bebop/vcs/03_batch_event.step.py b/bbdev/api/steps/bebop/vcs/03_batch_event.step.py index acacf8d28..0015e6920 100644 --- a/bbdev/api/steps/bebop/vcs/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/vcs/03_batch_event.step.py @@ -52,7 +52,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: binary = search_workload(workloads_output_root(bbdir), name) if binary is None: continue - command = " ".join(("env", f"LD_LIBRARY_PATH={shlex.quote(str(Path(libc).parent))}", shlex.quote(str(simv)), f"+elf={shlex.quote(binary)}", "+batch", "+trace=none", "+no-wave", f"+dramsim_ini_dir={shlex.quote(str(Path(bbdir) / 'result/share/dramsim3/configs'))}")) + command = " ".join(("env", f"LD_LIBRARY_PATH={shlex.quote(str(Path(libc).parent))}", shlex.quote(str(simv)), f"+elf={shlex.quote(binary)}", "+batch", "+trace=none", "+no-wave", f"+dramsim_ini_dir={shlex.quote(str(Path(bbdir) / 'result/share/dramsim3/configs'))}", f"+dramsim_outdir={shlex.quote(str(report / 'dramsim'))}")) result = await stream_run_logger_async(cmd=command, logger=ctx.logger, cwd=str(simv.parent), stdout_prefix="bebop vcs batch", stderr_prefix="bebop vcs batch") if result.returncode == 0: counts[ball][0] += 1 diff --git a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py index ce30cc603..d405b618e 100644 --- a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py +++ b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py @@ -29,6 +29,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: }, ) data = {"chip": chip, "diff": bool(body.get("diff", False))} + data["main_only"] = bool(body.get("main-only", False)) if body.get("output-dir"): data["output_dir"] = body["output-dir"] await ctx.enqueue( diff --git a/bbdev/api/steps/bebop/verilator/04_sim_api.step.py b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py index 989ac97fb..ccff49aa8 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_api.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py @@ -14,6 +14,8 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + if "batch" in body: + return ApiResponse(status=400, body={"error": "Unsupported parameter: batch"}) binary = body.get("binary", "") if not binary: return ApiResponse( diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index da26044cd..327446725 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -41,6 +41,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: + if "batch" in input_data: + raise ValueError("Unsupported parameter: batch") origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" @@ -139,7 +141,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return binary_name = input_data.get("binary", "") - binary_path = search_workload(workloads_output_root(bbdir), binary_name) + try: + binary_path = search_workload(workloads_output_root(bbdir), binary_name) + except ValueError as error: + await check_result( + ctx, 1, continue_run=False, + extra_fields={"error": "binary_resolution_error", "binary": binary_name, "message": str(error)}, + trace_id=origin_tid, + ) + return if binary_path is None: ctx.logger.error(f"binary not found: {binary_name}") await check_result( diff --git a/bbdev/api/steps/bebop/verilator/05_run_api.step.py b/bbdev/api/steps/bebop/verilator/05_run_api.step.py index 5e945fd19..25b138a8f 100644 --- a/bbdev/api/steps/bebop/verilator/05_run_api.step.py +++ b/bbdev/api/steps/bebop/verilator/05_run_api.step.py @@ -12,6 +12,8 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + if "batch" in body: + return ApiResponse(status=400, body={"error": "Unsupported parameter: batch"}) if "output_dir" in body: return ApiResponse( status=400, diff --git a/bbdev/api/steps/config/01_install_api.step.py b/bbdev/api/steps/config/01_install_api.step.py index 5b82ecdd5..244b95a58 100644 --- a/bbdev/api/steps/config/01_install_api.step.py +++ b/bbdev/api/steps/config/01_install_api.step.py @@ -11,12 +11,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - if body: - unknown = sorted(body.keys()) - return ApiResponse( - status=400, - body={"error": f"config install takes no parameters: {', '.join(unknown)}"}, - ) - - await ctx.enqueue({"topic": "config.install", "data": {"_trace_id": ctx.trace_id}}) + unknown = set(body) - {"chip"} + if unknown: + return ApiResponse(status=400, body={"error": f"Unexpected parameters: {sorted(unknown)}"}) + if "chip" in body and (not isinstance(body["chip"], str) or not body["chip"].replace("_", "").replace("-", "").isalnum()): + return ApiResponse(status=400, body={"error": "--chip requires a chip name"}) + await ctx.enqueue({"topic": "config.install", "data": {**body, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/config/01_install_event.step.py b/bbdev/api/steps/config/01_install_event.step.py index 0012a32ef..bd356595c 100644 --- a/bbdev/api/steps/config/01_install_event.step.py +++ b/bbdev/api/steps/config/01_install_event.step.py @@ -41,7 +41,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = Path(get_buckyball_path()) chips = bbdir / "examples" / "chips" - tops = sorted(chips.glob("*/configs/chip.toml")) + selected = input_data.get("chip") + tops = [chips / selected / "configs" / "chip.toml"] if selected else sorted(chips.glob("*/configs/chip.toml")) + if any(not top.is_file() for top in tops): + raise ValueError(f"unknown chip: {selected}") if not tops: raise ValueError(f"no configs/chip.toml under {chips}") diff --git a/bbdev/api/steps/config/scripts/2_parameter_derivation.py b/bbdev/api/steps/config/scripts/2_parameter_derivation.py index ea2548dfb..710dcbb4c 100644 --- a/bbdev/api/steps/config/scripts/2_parameter_derivation.py +++ b/bbdev/api/steps/config/scripts/2_parameter_derivation.py @@ -18,6 +18,15 @@ def _die(msg: str) -> None: raise ValueError(msg) +def _factory(config: dict[str, Any], context: str) -> str: + value = config["factory"] + if not isinstance(value, str) or len(value.split(".")) < 2 or not all( + part.isidentifier() for part in value.split(".") + ): + _die(f"{context}: factory must be a Scala companion FQCN") + return value + + def _repo_rel(repo: Path, path: str | Path) -> str: p = Path(path) if not p.is_absolute(): @@ -69,7 +78,7 @@ def _require_id_list(obj: dict[str, Any], key: str, count: int, ctx: str) -> lis def _tile_cores(tile: dict[str, Any]) -> list[dict[str, Any]]: cores = tile.get("cores") - if isinstance(cores, list) and cores: + if isinstance(cores, list): out: list[dict[str, Any]] = [] seen: set[int] = set() for i, core in enumerate(cores): @@ -110,23 +119,24 @@ def _tile_cores(tile: dict[str, Any]) -> list[dict[str, Any]]: def iter_topology_tiles(topo: dict[str, Any]) -> list[dict[str, Any]]: - """The main tile is tile 0; homogeneous compute tiles follow as tiles 1..count.""" - for legacy in ("tiles", "tileTemplate", "main_core"): - if legacy in topo: - _die(f"[{legacy}] is replaced by [main_tile] and [compute_tiles]") - main = topo.get("main_tile") + """The built-in main tile is tile 0; configured mounts follow in list order.""" + for key in ("compute_tiles", "tileTemplate", "main_core"): + if key in topo: + _die(f"invalid topology field: {key}") + main = topo["main_tile"] if not isinstance(main, dict): - _die("topology must define [main_tile]") + _die("main_tile must be a table") out = [dict(main, tile_id=0, kind="main")] - compute = topo.get("compute_tiles") - if compute is not None: - if not isinstance(compute, dict): - _die("[compute_tiles] must be a table") - count = compute.get("count") + groups = topo.get("tiles", []) + if not isinstance(groups, list): + _die("tiles must be an array of tables") + for group in groups: + count = group["count"] if type(count) is not int or count < 1: - _die("[compute_tiles].count must be a positive int") - base = {k: v for k, v in compute.items() if k != "count"} - out += [dict(base, tile_id=i + 1, kind="compute") for i in range(count)] + _die("tile count must be a positive int") + base = {k: v for k, v in group.items() if k != "count"} + for _ in range(count): + out.append(dict(base, tile_id=len(out), kind="tile")) return out @@ -135,33 +145,6 @@ def iter_cores(topo: dict[str, Any]) -> Iterator[dict[str, Any]]: yield from _tile_cores(tile) -def _tile_signature(tile: dict[str, Any]) -> tuple[tuple[int, str, str], ...]: - sig = [] - for core in _tile_cores(tile): - rel = core.get("_file") - if not isinstance(rel, str): - _die("core config missing _file") - pkg = core_pkg(rel) - if not pkg: - _die(f"unsupported core config path: {rel}") - role = core.get("name") - if role is not None and not isinstance(role, str): - _die("core name must be a string") - sig.append((core["core_id"], pkg, role or "")) - return tuple(sig) - - -def _assert_isomorphic_tiles(topo: dict[str, Any]) -> None: - tiles = iter_topology_tiles(topo) - for tile in tiles: - if not _tile_signature(tile): - _die(f"tile_id={tile['tile_id']} has no cores") - compute = [t for t in tiles if t["kind"] == "compute"] - for tile in compute[1:]: - if _tile_signature(tile) != _tile_signature(compute[0]): - _die(f"compute tiles must be isomorphic; tile_id={tile['tile_id']} differs from tile_id=1") - - def unique_cores(topo: dict[str, Any]) -> list[str]: seen: set[str] = set() out: list[str] = [] @@ -256,7 +239,7 @@ def _ball_num(core: dict) -> int: def _n_tiles(topo: dict[str, Any]) -> int: if "top" in topo: - _die("[top] is retired: tile counts and hart layout follow [main_tile] and [compute_tiles]") + _die("invalid topology field: top") return len(iter_topology_tiles(topo)) @@ -285,6 +268,8 @@ def _derive_cores(repo: Path, topo: dict[str, Any]) -> list[dict[str, Any]]: "core_id": core_id, "pkg": pkg, "role": role or "", + "cpu_kind": core["cpu"]["kind"], + "factory_class": _factory(core, pkg), "config_path": _repo_rel(repo, rel), "balldomain_base_dir": _balldomain_base_dir(repo, core, pkg), "bank_num": num, @@ -309,15 +294,25 @@ def _derive_cores(repo: Path, topo: dict[str, Any]) -> list[dict[str, Any]]: entry["mappings"].append( { "ball_id": mapping.get("ballId"), - "ball_dir": "" if mapping.get("builtin") else _ball_dir(ball_class), - "config_path": "" if mapping.get("builtin") else _config_path(repo, mapping.get("config")), - "builtin": mapping.get("builtin", ""), + "ball_dir": _ball_dir(ball_class), + "config_path": _config_path(repo, mapping.get("config")), } ) out.append(entry) return out +def _validate_spm(config: dict[str, Any]) -> None: + base, size, bits = (config[key] for key in ("base", "bytes", "dataBits")) + if any(type(value) is not int for value in (base, size, bits)): + _die("SPM base/bytes/dataBits must be integers") + if bits < 64 or bits & (bits - 1): + _die("SPM dataBits must be a power of two >= 64") + beat = bits // 8 + if size < 2 * beat or size & (size - 1) or base < 0 or base % beat or base + size > 1 << 64: + _die("SPM requires an aligned base and power-of-two capacity within the 64-bit address space") + + def _derive_tiles( repo: Path, topo: dict[str, Any], cores: list[dict[str, Any]] ) -> list[dict[str, Any]]: @@ -330,11 +325,42 @@ def _derive_tiles( _die("tile missing _file") tile_cores = _tile_cores(tile) n = len(tile_cores) - if "controllerCore" in tile: - _die(f"tile_id={tile_id}: controllerCore is retired; a compute tile's controller is core 0") - if tile["kind"] == "compute" and n < 2: - _die(f"tile_id={tile_id}: a compute tile needs its controller and at least one worker") + controller = tile.get("controllerCore") + if controller is not None and (type(controller) is not int or controller not in range(n)): + _die(f"tile_id={tile_id}: controllerCore must name a local core") + if tile["kind"] == "main" and controller is not None: + _die("main tile has no task controller") + ants = [core for core in tile_cores if core["cpu"]["kind"] == "ant"] + if ants: + shared_spm = tile.get("tss") + if not isinstance(shared_spm, dict): + _die("Ant compute tile requires explicit tss geometry") + _validate_spm(shared_spm) + for core in ants: + params = core["cpu"]["config"] + tls = params["tls"] + _validate_spm(tls) + code = params["codeBytes"] + if type(code) is not int or code < 32 or code & (code - 1): + _die("Ant codeBytes must be a power of two >= 32") + if not 1 <= params["taskBits"] <= 64: + _die("Ant taskBits must be in [1,64]") + if min(tls["base"], shared_spm["base"]) < code: + _die("Ant code overlaps TLS/TSS") + if not (tls["base"] + tls["bytes"] <= shared_spm["base"] or + shared_spm["base"] + shared_spm["bytes"] <= tls["base"]): + _die("Ant TLS/TSS windows overlap") + if tls["dataBits"] != shared_spm["dataBits"]: + _die("Ant TLS/TSS access widths must match") shared = tile.get("sharedMem") + if isinstance(shared, dict): + if shared["bankWidth"] != 128: + _die("sharedMem.bankWidth must be 128 bits") + rows = shared["bankEntries"] + if type(rows) is not int or rows < 2 or rows > 65536 or rows & (rows - 1): + _die("sharedMem.bankEntries must be a power of two in [2,65536]") + if shared["enable"] and (shared["entries"] <= 0 or shared["entries"] % rows): + _die("sharedMem.entries must be a positive multiple of bankEntries") vbc = 0 if isinstance(shared, dict): raw = shared.get("virtualBankCount") @@ -352,10 +378,17 @@ def _derive_tiles( if not isinstance(raw, int): _die("tile with Buckyball cores must define memBallChannelNum") mem_ball_channel_num = raw + if tile["kind"] == "main": + if "factory" in tile: + _die("main tile must not specify a factory") + factory = "" + else: + factory = _factory(tile, str(tile_path)) placements.append( { "tile_id": tile_id, "kind": tile["kind"], + "factory_class": factory, "path": _repo_rel(repo, tile_path), "core_indices": indices, "cores_per_tile": n, @@ -363,10 +396,8 @@ def _derive_tiles( "mem_ball_channel_num": mem_ball_channel_num, } ) - if tile["kind"] == "compute" or ( - n > 1 and has_buckyball and cores[indices[0]]["ball_num"] == 0 - ): - placements[-1]["controller_core_index"] = indices[0] + if controller is not None: + placements[-1]["controller_core_index"] = indices[controller] offset += n return placements @@ -389,7 +420,6 @@ def _bemu_balls(repo: Path, topo: dict[str, Any]) -> list[dict[str, str]]: and isinstance(m.get("ballId"), int) and isinstance(m.get("ballClass"), str) } - builtin_ids = {m["ballId"] for m in mappings if m.get("builtin")} pkg = core_pkg(core.get("_file", "")) or "core" for entry in isa: if not isinstance(entry, dict): @@ -400,8 +430,6 @@ def _bemu_balls(repo: Path, topo: dict[str, Any]) -> list[dict[str, str]]: _die(f"ballISA entry must have funct7 and bid: {entry!r}") if funct7 in {0, 1, 16, 32, 33, 34, 35}: continue - if bid in builtin_ids: - continue ball_class = bid_to_class.get(bid) if not ball_class: _die( @@ -428,14 +456,10 @@ def _bemu_balls(repo: Path, topo: dict[str, Any]) -> list[dict[str, str]]: def _bemu_paths(repo: Path, chip: str, topo: dict[str, Any]) -> tuple[str, int]: """The chip's own bebop-chip entry (empty for the default tile runner) and the tile it runs.""" tiles = iter_topology_tiles(topo) - for kind in ("main", "compute"): - files = {t.get("_file") for t in tiles if t["kind"] == kind} - if len(files) > 1: - _die(f"chip {chip}: bemu requires one {kind} tile file, got {sorted(files)}") - compute = [t["tile_id"] for t in tiles if t["kind"] == "compute"] + mounted = [t["tile_id"] for t in tiles if t["kind"] == "tile"] main = repo / "examples" / "chips" / chip / "emu" / "src" / "main.rs" entry = f"examples/chips/{chip}/emu/src/main.rs" if main.is_file() else "" - return entry, compute[0] if compute else 0 + return entry, mounted[0] if mounted else 0 def _ball_ctest_dirs(repo: Path, core: dict[str, Any]) -> list[str]: @@ -449,8 +473,6 @@ def _ball_ctest_dirs(repo: Path, core: dict[str, Any]) -> list[str]: for mapping in mappings: if not isinstance(mapping, dict): _die("ballIdMappings entry must be a table") - if mapping.get("builtin"): - continue ball_class = mapping.get("ballClass") if not isinstance(ball_class, str): _die("ballIdMappings entry missing ballClass") @@ -506,42 +528,24 @@ def _derive_targets( def _derive_harts(tiles: list[dict[str, Any]], cores: list[dict[str, Any]]) -> list[dict[str, Any]]: - """A main tile with a task controller exposes only core 0 to Linux. - Physical hart IDs cover the main tile, then compute controllers and workers.""" - main = tiles[0] - compute = tiles[1:] - m = main["cores_per_tile"] - workers_per_tile = compute[0]["cores_per_tile"] - 1 if compute else 0 + """Only system CPUs receive hart IDs; Ant contexts are tile-local resources.""" harts: list[dict[str, Any]] = [] for tile in tiles: - tile_id = tile["tile_id"] + context = 0 for local, core_index in enumerate(tile["core_indices"]): inst = cores[core_index] if inst["core_id"] != local: - _die(f"tile_id={tile_id}: core_id={inst['core_id']} != sorted slot {local}") - if tile["kind"] == "main": - hid = local - elif local == 0: - hid = m + tile_id - 1 - else: - hid = m + len(compute) + (tile_id - 1) * workers_per_tile + local - 1 - harts.append( - { - "hart_id": hid, - "tile_id": tile_id, - "core_id": local, - "core_index": core_index, - "visible": tile["kind"] == "main" and ( - "controller_core_index" not in tile or local == 0 - ), - "target": _target_name(inst["role"], inst["pkg"]), - "pkg": inst["pkg"], - "role": inst["role"], - } - ) - harts.sort(key=lambda h: h["hart_id"]) - if [h["hart_id"] for h in harts] != list(range(len(cores))): - _die("chip hart_ids must be exactly 0..total_harts-1") + _die(f"tile {tile['tile_id']}: core IDs must match ordered slots") + if inst["cpu_kind"] == "ant": + inst["ant_context_id"] = context + context += 1 + continue + harts.append({ + "hart_id": len(harts), "tile_id": tile["tile_id"], + "core_id": local, "core_index": core_index, "visible": True, + "target": _target_name(inst["role"], inst["pkg"]), + "pkg": inst["pkg"], "role": inst["role"], + }) return harts @@ -570,7 +574,6 @@ def derive(data: dict[str, Any], repo: Path, chip: str) -> dict[str, Any]: if not isinstance(includes_raw, list): _die(f"{chip}: includes must be a list") - _assert_isomorphic_tiles(topo) cores = _derive_cores(repo, topo) tiles = _derive_tiles(repo, topo, cores) targets = _derive_targets(repo, topo, cores) diff --git a/bbdev/api/steps/config/scripts/3_json2proto.py b/bbdev/api/steps/config/scripts/3_json2proto.py index 6fa41fc08..008975aa0 100644 --- a/bbdev/api/steps/config/scripts/3_json2proto.py +++ b/bbdev/api/steps/config/scripts/3_json2proto.py @@ -83,13 +83,9 @@ def _fill_ball(msg: pb.BallDomain, d: dict[str, Any], bbdir: Path) -> None: e.ball_id = m["ballId"] e.ball_name = m["ballName"] e.ball_class = m["ballClass"] - if m.get("builtin"): - e.builtin = m["builtin"] - params = m["ball_params"] - else: - e.ball_dir = _ball_dir(m["ballClass"]) - e.config_path = _rel(bbdir, m["config"]["_file"]) - params = m["config"]["ball"] + e.ball_dir = _ball_dir(m["ballClass"]) + e.config_path = _rel(bbdir, m["config"]["_file"]) + params = m["config"]["ball"] for key, value in params.items(): e.ball_params[key] = str(value) e.in_bw = m["inBW"] @@ -148,8 +144,8 @@ def _fill_boom(msg: pb.BoomCpuConfig, d: dict[str, Any]) -> None: def _check_cpu_kind(cpu: dict[str, Any], pkg: str) -> str: kind = cpu.get("kind") - if kind not in ("rocket", "boom"): - raise ValueError(f"{pkg}: kind must be 'rocket' or 'boom', got {kind!r}") + if kind not in ("rocket", "boom", "ant"): + raise ValueError(f"{pkg}: kind must be 'rocket', 'boom' or 'ant', got {kind!r}") if not isinstance(cpu.get("config"), dict): raise ValueError(f"{pkg}: cpu.config must reference a TOML file") return kind @@ -168,9 +164,12 @@ def _fill_frontend(msg: pb.FrontendConfig, d: dict[str, Any], bbdir: Path) -> No def _fill_rvv(msg: pb.RvvConfig, d: dict[str, Any], bbdir: Path) -> None: - if type(d.get("enable", False)) is not bool: - raise ValueError("rvv.enable must be a boolean") - msg.enable = d.get("enable", False) + if type(d["enable"]) is not bool: + raise ValueError("rvv.enable must be an explicit boolean") + msg.enable = d["enable"] + for key in ("laneNumber", "vLen", "eLen", "iBufWords", "memoryPorts"): + if type(d[key]) is not int or d[key] <= 0: + raise ValueError(f"rvv.{key} must be a positive integer") msg.source_path = _rel(bbdir, d["_file"]) msg.lane_number = d["laneNumber"] msg.v_len = d["vLen"] @@ -189,14 +188,25 @@ def _fill_tile_params(msg: pb.TileParamConfig, d: dict[str, Any]) -> None: msg.n_pmps = d["nPMPs"] +def _fill_spm(msg: pb.SpmConfig, config: dict[str, Any]) -> None: + msg.base = config["base"] + msg.bytes = config["bytes"] + msg.data_bits = config["dataBits"] + + def _fill_cpu(msg: pb.CpuConfig, cpu: dict[str, Any], context: str, bbdir: Path) -> str: kind = _check_cpu_kind(cpu, context) msg.kind = kind msg.source_path = _rel(bbdir, cpu["_file"]) if kind == "rocket": _fill_rocket(msg.rocket, cpu["config"]) - else: + elif kind == "boom": _fill_boom(msg.boom, cpu["config"]) + else: + config = cpu["config"] + msg.ant.code_bytes = config["codeBytes"] + msg.ant.task_bits = config["taskBits"] + _fill_spm(msg.ant.tls, config["tls"]) return kind @@ -205,6 +215,9 @@ def _fill_core(ci: pb.CoreInstance, raw: dict[str, Any], meta: dict[str, Any], b ci.role = meta["role"] ci.pkg = meta["pkg"] ci.config_path = meta["config_path"] + if raw["factory"] != meta["factory_class"]: + raise ValueError("core factory differs between config and derived metadata") + ci.factory_class = raw["factory"] ci.balldomain_base_dir = meta["balldomain_base_dir"] kind = _fill_cpu(ci.cpu, raw["cpu"], meta["pkg"], bbdir) domain = normalize_ball_domain(raw) @@ -218,15 +231,24 @@ def _fill_core(ci: pb.CoreInstance, raw: dict[str, Any], meta: dict[str, Any], b raise ValueError(f"{meta['pkg']}: kind=boom forbids balldomain.ballNum > 0") if "frontend" in raw: _fill_frontend(ci.frontend, raw["frontend"], bbdir) - if ci.balldomain.ball_num > 0: + if "rvv" in raw: _fill_rvv(ci.rvv, raw["rvv"], bbdir) def _fill_tile(tp: pb.TilePlacement, meta: dict[str, Any], proto: dict[str, Any]) -> None: tp.path = meta["path"] - tp.kind = pb.TILE_KIND_MAIN if meta["kind"] == "main" else pb.TILE_KIND_COMPUTE + if meta["kind"] == "main": + if "factory" in proto or meta["factory_class"]: + raise ValueError("main tile must not specify a factory") + else: + if proto["factory"] != meta["factory_class"]: + raise ValueError("tile factory differs between config and derived metadata") + tp.factory_class = proto["factory"] + tp.kind = pb.TILE_KIND_MAIN if meta["kind"] == "main" else pb.TILE_KIND_TILE tp.virtual_bank_count = meta["virtual_bank_count"] tp.core_indices.extend(meta["core_indices"]) + if "tss" in proto: + _fill_spm(tp.tss, proto["tss"]) if "controller_core_index" in meta: tp.controller_core_index = meta["controller_core_index"] tp.mem_ball_channel_num = meta["mem_ball_channel_num"] @@ -234,6 +256,8 @@ def _fill_tile(tp: pb.TilePlacement, meta: dict[str, Any], proto: dict[str, Any] sm = proto["sharedMem"] tp.shared_mem.enable = sm["enable"] tp.shared_mem.entries = sm["entries"] + tp.shared_mem.bank_entries = sm["bankEntries"] + tp.shared_mem.bank_width = sm["bankWidth"] tp.shared_mem.input_channels = sm["inputChannels"] tp.shared_mem.default_group_count = sm["defaultGroupCount"] if "virtualBankCount" in sm: @@ -276,7 +300,10 @@ def fill_chip(config: dict[str, Any], derived: dict[str, Any], bbdir: Path) -> p for index, (raw, meta) in enumerate(zip(raw_cores, meta_cores)): core = b.cores.add() _fill_core(core, raw, meta, bbdir) - core.hart_id = hart_ids[index] + if core.cpu.kind == "ant": + core.ant_context_id = meta["ant_context_id"] + else: + core.hart_id = hart_ids[index] raw_tiles = _derive.iter_topology_tiles(config["designs"]) for raw, meta in zip(raw_tiles, derived["tiles"], strict=True): diff --git a/bbdev/api/steps/config/scripts/ball_normalize.py b/bbdev/api/steps/config/scripts/ball_normalize.py index 3f665f3b2..1cdd80d3c 100644 --- a/bbdev/api/steps/config/scripts/ball_normalize.py +++ b/bbdev/api/steps/config/scripts/ball_normalize.py @@ -1,21 +1,15 @@ -"""Normalize external Ball mappings and the explicitly enabled RVV kernel.""" +"""Validate and copy external Ball mappings.""" from copy import deepcopy from typing import Any def normalize_ball_domain(core: dict[str, Any]) -> dict[str, Any] | None: - rvv = core.get("rvv") - enabled = False - if rvv is not None: - if not isinstance(rvv, dict) or ("enable" in rvv and type(rvv["enable"]) is not bool): - raise ValueError("rvv.enable must be an explicit boolean") - enabled = rvv.get("enable", False) raw = core.get("balldomain") - if raw is None and not enabled: + if raw is None: return None if not isinstance(raw, dict): - raise ValueError("RVV kernel requires a Ball domain") + raise ValueError("Ball domain must be a table") domain = deepcopy(raw) mappings = domain.get("ballIdMappings") isa = domain.get("ballISA") @@ -25,17 +19,11 @@ def normalize_ball_domain(core: dict[str, Any]) -> dict[str, Any] | None: raise ValueError("Ball ISA entries must be tables") if type(domain.get("ballNum")) is not int or domain["ballNum"] != len(mappings): raise ValueError("ballNum must equal the number of mappings") - kernel = None for index, mapping in enumerate(mappings): if not isinstance(mapping, dict) or type(mapping.get("ballId")) is not int or mapping["ballId"] != index: raise ValueError("Ball IDs must enumerate mappings without gaps or reordering") if "builtin" in mapping: - if mapping["builtin"] != "kernel": - raise ValueError("unknown builtin Ball") - if kernel is not None or index != len(mappings) - 1: - raise ValueError("builtin kernel must occur once at the final Ball ID") - kernel = mapping - continue + raise ValueError("Ball mappings must select external Balls") klass = mapping.get("ballClass") config = mapping.get("config") if not isinstance(klass, str) or not klass.startswith("examples.balls."): @@ -44,28 +32,6 @@ def normalize_ball_domain(core: dict[str, Any]) -> dict[str, Any] | None: raise ValueError("external Ball requires a resolved config with [ball] parameters") if any(type(mapping.get(key)) is not int or mapping[key] <= 0 for key in ("inBW", "outBW")): raise ValueError("external Ball inBW/outBW must be positive integers") - if not enabled: - if kernel is not None: - raise ValueError("builtin kernel requires rvv.enable=true") - return domain - params = {} - for key in ("laneNumber", "vLen", "eLen", "iBufWords", "memoryPorts"): - value = rvv.get(key) - if type(value) is not int or value <= 0: - raise ValueError(f"rvv.{key} must be a positive integer") - params[key] = str(value) - bid = len(mappings) if kernel is None else len(mappings) - 1 - expected = {"ballId": bid, "ballName": "kernel", "ballClass": "framework.balldomain.kernel.KernelBall", - "builtin": "kernel", "inBW": 0, "outBW": 0, "ball_params": params} - instruction = {"mnemonic": "RUN_KERNEL", "funct7": 15, "bid": bid} - conflicts = [entry for entry in isa if entry.get("funct7") == 15 or entry.get("mnemonic") == "RUN_KERNEL"] - if kernel is not None: - if kernel != expected or conflicts != [instruction]: - raise ValueError("builtin kernel mapping or RUN_KERNEL ISA differs from RVV configuration") - else: - if conflicts: - raise ValueError("RUN_KERNEL/funct7=15 conflicts with an existing ISA entry") - mappings.append(expected) - isa.append(instruction) - domain["ballNum"] += 1 + if any(entry.get("funct7") == 15 or entry.get("mnemonic") == "RUN_KERNEL" for entry in isa): + raise ValueError("RUN_KERNEL is a global control instruction, not a Ball instruction") return domain diff --git a/bbdev/api/steps/config/scripts/proto/chip.proto b/bbdev/api/steps/config/scripts/proto/chip.proto index 0c209b683..919a84542 100644 --- a/bbdev/api/steps/config/scripts/proto/chip.proto +++ b/bbdev/api/steps/config/scripts/proto/chip.proto @@ -69,7 +69,6 @@ message BallIdMapping { uint32 mmio_read_bw = 8; uint32 mmio_write_bw = 9; map ball_params = 10; - string builtin = 11; } message BallIsaEntry { @@ -179,11 +178,24 @@ message TileParamConfig { uint32 n_pmps = 7; } +message SpmConfig { + uint64 base = 1; + uint32 bytes = 2; + uint32 data_bits = 3; +} + +message AntConfig { + uint32 code_bytes = 1; + SpmConfig tls = 2; + uint32 task_bits = 3; +} + message CpuConfig { string kind = 1; oneof cpu { RocketCpuConfig rocket = 2; BoomCpuConfig boom = 3; + AntConfig ant = 5; } string source_path = 4; } @@ -200,6 +212,8 @@ message CoreInstance { FrontendConfig frontend = 9; optional uint32 hart_id = 10; RvvConfig rvv = 11; + optional uint32 ant_context_id = 12; + string factory_class = 13; } message BemuBall { @@ -221,13 +235,15 @@ message SharedMemConfig { uint32 default_group_count = 4; uint32 virtual_bank_count = 5; uint32 mem_cycles = 6; + uint32 bank_entries = 7; + uint32 bank_width = 8; } // Tile 0 is the main tile: every core is Linux visible. Compute tiles follow and are // homogeneous: core 0 controls the tile and the other cores are hidden task workers. enum TileKind { TILE_KIND_MAIN = 0; - TILE_KIND_COMPUTE = 1; + TILE_KIND_TILE = 1; } message TilePlacement { @@ -238,9 +254,11 @@ message TilePlacement { reserved 5; SharedMemConfig shared_mem = 6; TileParamConfig param = 7; - // Set for compute tiles only. + // Optional local task controller, expressed as a global core index. optional uint32 controller_core_index = 8; TileKind kind = 9; + SpmConfig tss = 10; + string factory_class = 11; } message WorkloadConfig { diff --git a/bbdev/api/steps/kernel/01_build_api.step.py b/bbdev/api/steps/kernel/01_build_api.step.py index bdec4672b..21c08ce97 100644 --- a/bbdev/api/steps/kernel/01_build_api.step.py +++ b/bbdev/api/steps/kernel/01_build_api.step.py @@ -15,9 +15,9 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: if isinstance(memory, bool) or not str(memory).isdigit() or not 1 <= int(memory) <= 16384: return ApiResponse(status=400, body={"error": "guest-memory-mib must be an integer in 1..16384 (P2E DDR capacity)"}) storage = body.get("model-storage", "initramfs") - if storage not in ("initramfs", "pmem"): - return ApiResponse(status=400, body={"error": "model-storage must be initramfs or pmem"}) - if storage == "pmem" and (not body.get("model") or int(memory) >= 16384): - return ApiResponse(status=400, body={"error": "pmem requires --model and guest-memory-mib < 16384"}) + if storage not in ("initramfs", "ddr"): + return ApiResponse(status=400, body={"error": "model-storage must be initramfs or ddr"}) + if storage == "ddr" and (not body.get("model") or int(memory) >= 16384): + return ApiResponse(status=400, body={"error": "ddr requires --model and guest-memory-mib < 16384"}) await ctx.enqueue({"topic": "kernel.build", "data": {**body, "_trace_id": ctx.trace_id}}) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index 7e399fa95..1a9c439f0 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -32,8 +32,7 @@ def hart_count_params(input_data: dict) -> dict: allowed = { - "visible-hart-count", - "total-hart-count", + "hart-count", "model", "chip", "interactive", @@ -45,10 +44,10 @@ def hart_count_params(input_data: dict) -> dict: if unknown: raise ValueError(f"unknown kernel build parameter(s): {', '.join(unknown)}") - if "hidden-hart-base" in input_data: - raise ValueError( - "hidden-hart-base is not supported; hidden harts must start at visible-hart-count" - ) + if "hart-count" in input_data: + value = input_data["hart-count"] + if isinstance(value, bool) or not str(value).isdigit() or int(value) < 1: + raise ValueError("hart-count must be a positive integer") chip = input_data.get("chip") if chip: @@ -64,29 +63,16 @@ def hart_count_params(input_data: dict) -> dict: ) with open(derived) as stream: harts = json.load(stream)["harts"] - visible = sum(hart["visible"] for hart in harts) - total = len(harts) - for key, expected in ( - ("visible-hart-count", visible), - ("total-hart-count", total), - ): - if key in input_data and int(input_data[key]) != expected: - raise ValueError(f"{key} must match chip topology: {expected}") + count = sum(hart["visible"] for hart in harts) + if "hart-count" in input_data and int(input_data["hart-count"]) != count: + raise ValueError(f"hart-count must match chip topology: {count}") else: - visible = int(input_data.get("visible-hart-count", 64)) - total = int(input_data.get("total-hart-count", visible)) - hidden_base = visible - - if visible < 1: - raise ValueError("visible-hart-count must be at least 1") - if total < visible: - raise ValueError("total-hart-count must cover visible harts") - - return { - "visible": visible, - "total": total, - "hidden_base": hidden_base, - } + count = int(input_data.get("hart-count", 64)) + + if count < 1: + raise ValueError("hart-count must be at least 1") + + return {"count": count} def kernel_interactive(input_data: dict) -> bool: @@ -133,8 +119,7 @@ def kernel_build_dir( guest_memory_mib: int = 512, model_storage: str = "initramfs", ) -> str: - visible = hart_params["visible"] - total = hart_params["total"] + count = hart_params["count"] suffix = "" if guest_memory_mib == 512 else f"-mem{guest_memory_mib}M" if chip: suffix += f"-chip-{chip}" @@ -142,12 +127,12 @@ def kernel_build_dir( suffix += f"-model-{model}" if interactive: suffix += "-interactive" - if model_storage == "pmem": - suffix += "-pmem" - if visible == 64 and total == 64: + if model_storage == "ddr": + suffix += "-ddr" + if count == 64: return os.path.join(bbdir, "bb-tests", "build", f"kernel{suffix}") return os.path.join( - bbdir, "bb-tests", "build", f"kernel-v{visible}-t{total}{suffix}" + bbdir, "bb-tests", "build", f"kernel-h{count}{suffix}" ) @@ -158,19 +143,18 @@ def fw_payload_name( guest_memory_mib: int = 512, model_storage: str = "initramfs", ) -> str: - visible = hart_params["visible"] - total = hart_params["total"] + count = hart_params["count"] name = "fw_payload" - if visible != 64 or total != 64: - name = f"{name}-v{visible}-t{total}" + if count != 64: + name = f"{name}-h{count}" if model: name = f"{name}-{model}" elif chip: name = f"{name}-{chip}-linux" if guest_memory_mib != 512: name += f"-mem{guest_memory_mib}M" - if model_storage == "pmem": - name += "-pmem" + if model_storage == "ddr": + name += "-ddr" return name @@ -201,12 +185,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) guest_memory_mib = int(memory_value) model_storage = input_data.get("model-storage", "initramfs") - if model_storage not in ("initramfs", "pmem"): - raise ValueError("model-storage must be initramfs or pmem") - if model_storage == "pmem" and ( + if model_storage not in ("initramfs", "ddr"): + raise ValueError("model-storage must be initramfs or ddr") + if model_storage == "ddr" and ( not input_data.get("model") or guest_memory_mib >= 16384 ): - raise ValueError("pmem requires --model and guest-memory-mib < 16384") + raise ValueError("ddr requires --model and guest-memory-mib < 16384") hart_params = hart_count_params(input_data) interactive = kernel_interactive(input_data) except ValueError as e: @@ -232,9 +216,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # cmake configure configure_cmd = ( f"cmake -B {kernel_build} -S {kernel_src} " - f"-DBUCKYBALL_VISIBLE_HART_COUNT={hart_params['visible']} " - f"-DBUCKYBALL_TOTAL_HART_COUNT={hart_params['total']} " - f"-DBUCKYBALL_HIDDEN_HART_BASE={hart_params['hidden_base']} " + f"-DBUCKYBALL_HART_COUNT={hart_params['count']} " f"-DBUCKYBALL_GUEST_MEMORY_MIB={guest_memory_mib} " f"-DBUCKYBALL_MODEL_STORAGE={model_storage} " f"-DBUCKYBALL_KERNEL_MODEL={cmake_model_name} " @@ -318,10 +300,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return storage_fields = {"model_storage": model_storage} - if model_storage == "pmem": + if model_storage == "ddr": manifest = os.path.join(output_dir, f"{payload_name}.load.json") if not os.path.isfile(manifest): - raise ValueError(f"Missing pmem load manifest: {manifest}") + raise ValueError(f"Missing ddr load manifest: {manifest}") storage_fields["load_manifest"] = manifest await check_result( ctx, 0, continue_run=False, extra_fields=storage_fields, trace_id=origin_tid diff --git a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py index 199a1b68d..a04f30288 100644 --- a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py @@ -67,6 +67,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) if input_data.get("diff"): command += " --difftest" + if input_data.get("main_only"): + command += " --main-only" returncode = ( await stream_run_logger_async( cmd=command, diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index 5dd37a7f9..aa73346b1 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -152,8 +152,17 @@ def load_ip(bbdir: str, name: str, target_name: str | None = None, chip: str | N metadata = resources / f"{target['name']}.toml" if metadata.is_file(): settings = tomllib.loads(metadata.read_text()) - if not settings or set(settings) - {"expected_assertion", "chips", "invalid_case"}: + if not settings or set(settings) - {"expected_assertion", "chips", "invalid_case", "rtl_scopes"}: raise ValueError(f"Invalid IP test settings: {metadata}") + if "rtl_scopes" in settings: + scopes = settings["rtl_scopes"] + if not isinstance(scopes, list) or not scopes or any( + not isinstance(scope, str) or not re.fullmatch( + r"[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?(?:\.[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?)*", scope + ) for scope in scopes + ): + raise ValueError(f"Invalid RTL instance scopes: {metadata}") + target["rtl_scopes"] = scopes if "invalid_case" in settings: case = settings["invalid_case"] if type(case) is not int or case < 0 or "expected_assertion" not in settings: @@ -285,7 +294,7 @@ def build_ip(bbdir: str, chip: str, name: str, ctx, target_name: str | None = No if declared_hier.is_file(): scopes = [line.strip() for line in declared_hier.read_text().splitlines() if line.strip()] if not scopes or any( - not re.fullmatch(r"\+tree @TOP@\.[A-Za-z_][A-Za-z0-9_$.]*", line) + not re.fullmatch(r"\+tree @TOP@\.[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?(?:\.[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?)*", line) for line in scopes ): raise ValueError(f"Invalid IP coverage hierarchy: {declared_hier}") @@ -352,7 +361,9 @@ def report_ip_coverage( cov_dir.mkdir(parents=True, exist_ok=True) exclusions = root / "src" / "main" / "resources" / f"{target['name']}.el" hierarchy = simv.parent / "rtl_hier.cfg" - hierarchy.write_text(f"+tree {target['top']}.dut\n") + hierarchy.write_text("".join( + f"+tree {target['top']}.{instance}\n" for instance in target.get("rtl_scopes", ["dut"]) + )) scope = f" -hier {shlex.quote(str(hierarchy))} -metric line+cond+tgl" reports = [(cov_dir, ""), (cov_dir / "rtl_raw", scope)] if exclusions.is_file(): diff --git a/bbdev/api/steps/vcs/04_sim_event.step.py b/bbdev/api/steps/vcs/04_sim_event.step.py index 0102b9eb0..45371496b 100644 --- a/bbdev/api/steps/vcs/04_sim_event.step.py +++ b/bbdev/api/steps/vcs/04_sim_event.step.py @@ -78,6 +78,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "+batch" if input_data.get("batch", False) else "", wave_arg, f"+dramsim_ini_dir={shlex.quote(str(Path(bbdir) / 'result/share/dramsim3/configs'))}", + f"+dramsim_outdir={shlex.quote(str(run_log / 'dramsim'))}", f"+timeout-ns={shlex.quote(str(timeout_ns))}", "+permissive-off", ] diff --git a/bbdev/api/steps/verilator/04_sim_event.step.py b/bbdev/api/steps/verilator/04_sim_event.step.py index b17cf15d5..51d893459 100644 --- a/bbdev/api/steps/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -184,7 +184,13 @@ def _gcc_lib_dir(soname: str) -> str: lib_dirs = [result_lib, _gcc_lib_dir("liblz4.so"), _gcc_lib_dir("libstdc++.so")] ld_lib_path = ":".join(dict.fromkeys(lib_dirs)) ctx.logger.info(f"LD_LIBRARY_PATH prefix: {ld_lib_path}") - sim_args = [bin_path, "+permissive", f"+elf={binary_path}"] + dramsim_outdir = os.path.join(run_log, "dramsim") + sim_args = [ + bin_path, + "+permissive", + f"+elf={binary_path}", + f"+dramsim_outdir={dramsim_outdir}", + ] if batch: sim_args.append("+batch") if coverage_flag: @@ -201,20 +207,17 @@ def _gcc_lib_dir(soname: str) -> str: "+permissive-off", ] ) - dasm_cmd = shlex.join( - [ - "cargo", - "run", - "--quiet", - "--release", - "--manifest-path", - os.path.join(bbdir, "bebop", "Cargo.toml"), - "--bin", - "bebop", - "--", - "dasm", - ] - ) + dasm_bin = os.path.join(bbdir, "bebop", "target", "release", "bebop") + if not os.path.isfile(dasm_bin): + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "dasm_not_found", "dasm": dasm_bin}, + trace_id=origin_tid, + ) + return + dasm_cmd = shlex.join([dasm_bin, "dasm"]) sim_cmd = f"export LD_LIBRARY_PATH={shlex.quote(ld_lib_path)}:$LD_LIBRARY_PATH; export BDB_SIM_META={shlex.quote(meta_path)}; {shlex.join(sim_args)} 2> >({dasm_cmd} > {shlex.quote(os.path.join(run_log, 'disasm.log'))})" script_dir = os.path.dirname(__file__) result = await stream_run_logger_async( diff --git a/bbdev/api/tests/test_api_kernel_build.py b/bbdev/api/tests/test_api_kernel_build.py index 46514658a..ffb41469b 100644 --- a/bbdev/api/tests/test_api_kernel_build.py +++ b/bbdev/api/tests/test_api_kernel_build.py @@ -1,4 +1,4 @@ from _api_test_helper import run_bbdev_case run_bbdev_case("bbdev kernel --build") -run_bbdev_case("bbdev kernel --build '--visible-hart-count 64 --total-hart-count 256'") +run_bbdev_case("bbdev kernel --build '--hart-count 64'") diff --git a/bbdev/api/utils/report_schema.py b/bbdev/api/utils/report_schema.py index f0740164f..dec86aae9 100644 --- a/bbdev/api/utils/report_schema.py +++ b/bbdev/api/utils/report_schema.py @@ -114,15 +114,22 @@ def validate_run(run): if accuracy > 1: raise ValueError("accuracy must be in [0,1]") nonempty(model["accuracy_metric"], "accuracy_metric") + counter = model["counter"] + if counter not in ("riscv-cycle", "x86-tsc"): + raise ValueError(f"unknown model counter: {counter}") + count_field = "cycles" if counter == "riscv-cycle" else "ticks" + other = "ticks" if counter == "riscv-cycle" else "cycles" + if other in model: + raise ValueError("model must not mix cycle and tick fields") if model["status"] == "pass": - number(model["cycles"], "cycles", positive=True, integer=True) + number(model[count_field], count_field, positive=True, integer=True) if model["trace_url"] is not None and not model["operators"]: raise ValueError("trace attachment requires operator measurements") if model["error"] is not None: raise ValueError("passing model cannot contain an error") else: nonempty(model["error"], "model error") - if model["cycles"] is not None or model["operators"]: + if model[count_field] is not None or model["operators"]: raise ValueError( "failed models cannot contain performance measurements" ) @@ -135,10 +142,19 @@ def validate_run(run): ids.add(op["id"]) if op["category"] not in CATEGORIES: raise ValueError("unknown operator category") - for key in ("cycles", "start_cycle", "level"): + if op["counter"] != counter or other in op: + raise ValueError("operator counter differs from its model") + unit = "cycle" if counter == "riscv-cycle" else "tick" + opposite = "tick" if unit == "cycle" else "cycle" + if f"start_{opposite}" in op or f"end_{opposite}" in op: + raise ValueError("operator mixes cycle and tick intervals") + for key in (count_field, f"start_{unit}", f"end_{unit}", "level"): number(op[key], key, integer=True) number(op["calls"], "calls", positive=True, integer=True) - if op["start_cycle"] + op["cycles"] > model["cycles"]: + if ( + op[f"start_{unit}"] + op[count_field] > op[f"end_{unit}"] + or op[f"end_{unit}"] > model[count_field] + ): raise ValueError("operator interval exceeds model trace span") if run["submission_errors"] != submission_errors(run): raise ValueError("submission status does not match the report measurements") diff --git a/bbdev/bbdev b/bbdev/bbdev index bb902e18e..2051e5e5d 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -48,7 +48,7 @@ WORKFLOW_COMMANDS = { "build": 'Build model recipes (all chip models when --model is omitted). Args: "--chip [--model ] [--ctrace] [--dtrace]"', }, "kernel": { - "build": 'Build RISC-V kernel + rootfs. Args: "[--chip ] [--model ] [--interactive] [--visible-hart-count ] [--total-hart-count ] [--guest-memory-mib ] [--model-storage initramfs|pmem]"; --model requires --chip', + "build": 'Build RISC-V kernel + rootfs. Args: "[--chip ] [--model ] [--interactive] [--hart-count ] [--guest-memory-mib ] [--model-storage initramfs|ddr]"; --model requires --chip', }, "firesim": { "enumeratefpgas": 'Enumerate FPGAs. Args: "--chip "', @@ -66,7 +66,7 @@ WORKFLOW_COMMANDS = { }, "bebop-verilator": { "clean": 'Clean Bebop Verilator build directory. Args: "--chip [--output-dir ]"', - "verilog": 'Generate Bebop Verilog files. Args: "--chip [--diff] [--output-dir ]"', + "verilog": 'Generate Bebop Verilog files. Args: "--chip [--diff] [--main-only] [--output-dir ]"', "build": 'Build Bebop Verilator simulation executable. Args: "--chip [--diff] [--jobs ] [--output-dir ]"', "sim": 'Run Bebop Verilator simulation. Args: "--chip --binary [--log-dir ] [--fst-dir ] [--no-wave]"', "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--jobs ] [--chip ] [--log-dir ] [--fst-dir ] [--no-wave]"', @@ -104,7 +104,7 @@ WORKFLOW_COMMANDS = { "run": 'Build and run a Ball or IP UVM simulation. Args: "--chip [--ball | --ip [--target ]]"', }, "config": { - "install": True, + "install": 'Install chip configurations. Args: "[--chip ]"', }, } diff --git a/bbdev/mcp/common.py b/bbdev/mcp/common.py index 1415eb725..f94e829b7 100644 --- a/bbdev/mcp/common.py +++ b/bbdev/mcp/common.py @@ -394,8 +394,6 @@ def _validate_domain(path: Path, cfg: Dict[str, Any]) -> Dict[str, Any]: missing_config = [] bad_bw = [] for m in mappings: - if m.get("builtin") == "kernel": - continue # The shared normalizer validates the complete builtin contract. name = m.get("ballName") if not m.get("ballClass"): bad_bw.append({"ballName": name, "error": "missing ballClass"}) @@ -461,7 +459,6 @@ def dups(xs): "ballId": m.get("ballId"), "ballName": m.get("ballName"), "ballClass": m.get("ballClass"), - "builtin": m.get("builtin"), "inBW": m.get("inBW"), "outBW": m.get("outBW"), "config": m.get("config"), diff --git a/bbdev/mcp/tools/bebop_verilator_run.py b/bbdev/mcp/tools/bebop_verilator_run.py index 4f3faa645..122bc3be2 100644 --- a/bbdev/mcp/tools/bebop_verilator_run.py +++ b/bbdev/mcp/tools/bebop_verilator_run.py @@ -18,7 +18,6 @@ def bbdev_bebop_verilator_run( banktrace: bool = False, no_wave: bool = False, diff: bool = False, - batch: bool = False, ) -> str: """Full bebop-verilator flow. POST /bebop/verilator/run.""" for n, v in (("binary", binary), ("chip", chip)): @@ -38,7 +37,6 @@ def bbdev_bebop_verilator_run( "banktrace": banktrace, "no-wave": no_wave, "diff": diff, - "batch": batch, }, ) ) diff --git a/bbdev/mcp/tools/bebop_verilator_sim.py b/bbdev/mcp/tools/bebop_verilator_sim.py index 15c93e522..3eed4cead 100644 --- a/bbdev/mcp/tools/bebop_verilator_sim.py +++ b/bbdev/mcp/tools/bebop_verilator_sim.py @@ -17,7 +17,6 @@ def bbdev_bebop_verilator_sim( ctrace: bool = False, banktrace: bool = False, no_wave: bool = False, - batch: bool = False, diff: bool = False, ) -> str: """Run one workload on bebop-verilator. POST /bebop/verilator/sim.""" @@ -33,7 +32,6 @@ def bbdev_bebop_verilator_sim( "ctrace": ctrace, "banktrace": banktrace, "no-wave": no_wave, - "batch": batch, "diff": diff, } return fmt(submit("/bebop/verilator/sim", params)) diff --git a/bbdev/mcp/tools/bebop_verilator_verilog.py b/bbdev/mcp/tools/bebop_verilator_verilog.py index c94457dd5..9358583d6 100644 --- a/bbdev/mcp/tools/bebop_verilator_verilog.py +++ b/bbdev/mcp/tools/bebop_verilator_verilog.py @@ -8,8 +8,8 @@ def register(mcp): @mcp.tool() - def bbdev_bebop_verilator_verilog(chip: str, diff: bool = False) -> str: + def bbdev_bebop_verilator_verilog(chip: str, diff: bool = False, main_only: bool = False) -> str: """Generate Verilog for bebop-verilator. POST /bebop/verilator/verilog.""" if e := need("chip", chip): return err(e) - return fmt(submit("/bebop/verilator/verilog", {"chip": chip, "diff": diff})) + return fmt(submit("/bebop/verilator/verilog", {"chip": chip, "diff": diff, "main-only": main_only})) diff --git a/bbdev/mcp/tools/bemu_sim.py b/bbdev/mcp/tools/bemu_sim.py index 265748234..898dd1338 100644 --- a/bbdev/mcp/tools/bemu_sim.py +++ b/bbdev/mcp/tools/bemu_sim.py @@ -11,7 +11,6 @@ def register(mcp): def bbdev_bemu_sim( chip: str, binary: str | None = None, - system: bool = False, dtb: str | None = None, initrd: str | None = None, memory_mib: int | None = None, @@ -24,14 +23,16 @@ def bbdev_bemu_sim( model: str | None = None, reuse_simulator: bool = False, ) -> str: - """Run a logical ELF file name from the selected chip's workload/kernel outputs, or a built model recipe, on BEMU. Model runs use running-param.toml. POST /bebop/bemu/sim.""" + """Run a workload, kernel ELF, or compiled model on BEMU. + + Model runs use running-param.toml. POST /bebop/bemu/sim. + """ if model is not None: for n, v in (("chip", chip), ("model", model)): if e := need(n, v): return err(e) if ( binary is not None - or system or dtb or initrd or memory_mib is not None @@ -59,7 +60,6 @@ def bbdev_bemu_sim( params: Dict[str, Any] = { "chip": chip, "binary": binary, - "system": system, "disasm": disasm, "tool-profile": tool_profile, "itrace": itrace, diff --git a/bbdev/mcp/tools/kernel_build.py b/bbdev/mcp/tools/kernel_build.py index f4d395b67..322e2d496 100644 --- a/bbdev/mcp/tools/kernel_build.py +++ b/bbdev/mcp/tools/kernel_build.py @@ -15,8 +15,7 @@ def bbdev_kernel_build( interactive: bool = False, guest_memory_mib: int = 512, model_storage: str = "initramfs", - visible_hart_count: Optional[int] = None, - total_hart_count: Optional[int] = None, + hart_count: Optional[int] = None, ) -> str: """Build RISC-V kernel + rootfs. POST /kernel/build. @@ -30,9 +29,7 @@ def bbdev_kernel_build( opt(params, chip=chip, model=model) if interactive: params["interactive"] = True - if visible_hart_count is not None: - params["visible-hart-count"] = visible_hart_count - if total_hart_count is not None: - params["total-hart-count"] = total_hart_count + if hart_count is not None: + params["hart-count"] = hart_count return fmt(submit("/kernel/build", params)) From 8dd7935627f4ad477631345905b1ac8eab8bd132 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Fri, 9 Oct 2026 21:13:34 +0800 Subject: [PATCH 186/196] feat: extend Rocket CPU configuration with additional parameters and validation checks for improved functionality --- .../api/steps/config/scripts/3_json2proto.py | 51 ++++++++++++++++++ .../api/steps/config/scripts/proto/chip.proto | 52 +++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/bbdev/api/steps/config/scripts/3_json2proto.py b/bbdev/api/steps/config/scripts/3_json2proto.py index 008975aa0..532100ff2 100644 --- a/bbdev/api/steps/config/scripts/3_json2proto.py +++ b/bbdev/api/steps/config/scripts/3_json2proto.py @@ -102,6 +102,57 @@ def _fill_ball(msg: pb.BallDomain, d: dict[str, Any], bbdir: Path) -> None: def _fill_rocket(msg: pb.RocketCpuConfig, d: dict[str, Any]) -> None: + msg.mtvec_init_enable = d["mtvecInitEnable"] + msg.use_user = d["useUser"] + msg.use_supervisor = d["useSupervisor"] + msg.use_hypervisor = d["useHypervisor"] + msg.use_debug = d["useDebug"] + msg.use_atomics = d["useAtomics"] + msg.use_atomics_only_for_i_o = d["useAtomicsOnlyForIO"] + msg.use_compressed = d["useCompressed"] + msg.use_r_v_e = d["useRVE"] + msg.use_conditional_zero = d["useConditionalZero"] + msg.n_local_interrupts = d["nLocalInterrupts"] + msg.use_n_m_i = d["useNMI"] + msg.n_breakpoints = d["nBreakpoints"] + msg.use_b_p_watch = d["useBPWatch"] + msg.mcontext_width = d["mcontextWidth"] + msg.scontext_width = d["scontextWidth"] + msg.n_perf_counters = d["nPerfCounters"] + msg.have_basic_counters = d["haveBasicCounters"] + msg.misa_writable = d["misaWritable"] + msg.mtvec_init = d["mtvecInit"] + msg.mtvec_writable = d["mtvecWritable"] + msg.fast_load_word = d["fastLoadWord"] + msg.fast_load_byte = d["fastLoadByte"] + msg.branch_prediction_mode_c_s_r = d["branchPredictionModeCSR"] + msg.clock_gate = d["clockGate"] + msg.mvendorid = d["mvendorid"] + msg.mimpid = d["mimpid"] + msg.have_cease = d["haveCease"] + msg.have_sim_timeout = d["haveSimTimeout"] + for field in ("useHypervisor", "useDebug", "useNMI"): + if d[field]: + raise ValueError(f"Rocket {field} has no platform interface") + if d["nLocalInterrupts"] != 0: + raise ValueError("Rocket local interrupts have no platform interface") + msg.mul_div.div_unroll = d["mulDiv"]["divUnroll"] + msg.mul_div.div_early_out_granularity = d["mulDiv"]["divEarlyOutGranularity"] + msg.fpu.div_sqrt = d["fpu"]["divSqrt"] + msg.fpu.sfma_latency = d["fpu"]["sfmaLatency"] + msg.fpu.dfma_latency = d["fpu"]["dfmaLatency"] + msg.fpu.fpmu_latency = d["fpu"]["fpmuLatency"] + msg.fpu.ifpu_latency = d["fpu"]["ifpuLatency"] + msg.dcache.clock_gate = d["dcache"]["clockGate"] + msg.icache.prefetch = d["icache"]["prefetch"] + msg.btb.n_match_bits = d["btb"]["nMatchBits"] + msg.btb.n_pages = d["btb"]["nPages"] + msg.btb.updates_out_of_order = d["btb"]["updatesOutOfOrder"] + msg.btb.bht_enable = d["btb"]["bhtEnable"] + msg.btb.bht_entries = d["btb"]["bhtEntries"] + msg.btb.bht_counter_length = d["btb"]["bhtCounterLength"] + msg.btb.bht_history_length = d["btb"]["bhtHistoryLength"] + msg.btb.bht_history_bits = d["btb"]["bhtHistoryBits"] msg.use_vm = d["useVM"] msg.use_zba = d["useZba"] msg.use_zbb = d["useZbb"] diff --git a/bbdev/api/steps/config/scripts/proto/chip.proto b/bbdev/api/steps/config/scripts/proto/chip.proto index 919a84542..3042253aa 100644 --- a/bbdev/api/steps/config/scripts/proto/chip.proto +++ b/bbdev/api/steps/config/scripts/proto/chip.proto @@ -85,6 +85,9 @@ message BallDomain { } message RocketMulDivConfig { + uint32 div_unroll = 5; + uint32 div_early_out_granularity = 6; + bool enable = 1; uint32 mul_unroll = 2; bool mul_early_out = 3; @@ -92,29 +95,78 @@ message RocketMulDivConfig { } message RocketFpuConfig { + bool div_sqrt = 4; + uint32 sfma_latency = 5; + uint32 dfma_latency = 6; + uint32 fpmu_latency = 7; + uint32 ifpu_latency = 8; + bool enable = 1; uint32 min_f_len = 2; uint32 f_len = 3; } message RocketDCacheConfig { + bool clock_gate = 4; + uint32 n_sets = 1; uint32 n_ways = 2; uint32 n_mshrs = 3; } message RocketICacheConfig { + bool prefetch = 3; + uint32 n_sets = 1; uint32 n_ways = 2; } message RocketBtbConfig { + uint32 n_match_bits = 4; + uint32 n_pages = 5; + bool updates_out_of_order = 6; + bool bht_enable = 7; + uint32 bht_entries = 8; + uint32 bht_counter_length = 9; + uint32 bht_history_length = 10; + uint32 bht_history_bits = 11; + bool enable = 1; uint32 n_entries = 2; uint32 n_ras = 3; } message RocketCpuConfig { + bool mtvec_init_enable = 41; + bool use_user = 13; + bool use_supervisor = 14; + bool use_hypervisor = 15; + bool use_debug = 16; + bool use_atomics = 17; + bool use_atomics_only_for_i_o = 18; + bool use_compressed = 19; + bool use_r_v_e = 20; + bool use_conditional_zero = 21; + uint32 n_local_interrupts = 22; + bool use_n_m_i = 23; + uint32 n_breakpoints = 24; + bool use_b_p_watch = 25; + uint32 mcontext_width = 26; + uint32 scontext_width = 27; + uint32 n_perf_counters = 28; + bool have_basic_counters = 29; + bool misa_writable = 30; + uint64 mtvec_init = 31; + bool mtvec_writable = 32; + bool fast_load_word = 33; + bool fast_load_byte = 34; + bool branch_prediction_mode_c_s_r = 35; + bool clock_gate = 36; + uint32 mvendorid = 37; + uint32 mimpid = 38; + bool have_cease = 39; + bool have_sim_timeout = 40; + bool use_vm = 3; bool use_zba = 4; bool use_zbb = 5; From e2264f9c08c2b8fe56d36390842b5328483091d7 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Sat, 10 Oct 2026 00:44:47 +0800 Subject: [PATCH 187/196] refactor: simplify artifact cleanup logic in Bebop batch regression steps and enhance command argument handling --- .../steps/bebop/bemu/03_batch_event.step.py | 27 ++++++++----------- .../bebop/verilator/06_batch_event.step.py | 7 ++--- .../config/scripts/2_parameter_derivation.py | 4 ++- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index 5621e5323..36bfed88a 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -7,7 +7,6 @@ """ import os -import shutil import shlex import sys from motia import FlowContext, queue @@ -102,22 +101,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + harness_args = [ + "--", + "--workload-toml", + workload_toml, + "--bb-tests-root", + elf_root, + "--jobs", + str(input_data.get("jobs", 1)), + ] if input_data.get("clean-before", input_data.get("clean_before", False)): - shutil.rmtree( - os.path.join(env["CARGO_TARGET_DIR"], "test-artifacts"), ignore_errors=True - ) - ctx.logger.info("Cleaned previous bebop test artifacts") - harness = shlex.join( - [ - "--", - "--workload-toml", - workload_toml, - "--bb-tests-root", - elf_root, - "--jobs", - str(input_data.get("jobs", 1)), - ] - ) + harness_args.append("--clean-before") + harness = shlex.join(harness_args) test_cmd = f"nix develop -c cargo test --manifest-path {shlex.quote(str(bemu_cargo_manifest))} --test test_bemu {harness}" ctx.logger.info(f"Running bebop bemu regression: {test_cmd}") run_result = await stream_run_logger_async( diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 54cc72b39..b72afeaf4 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -5,7 +5,6 @@ """ import os -import shutil import shlex import sys from motia import FlowContext, queue @@ -86,10 +85,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if diff: manifest = f"{bbdir}/examples/chips/{chip}/generated/bebop/Cargo.toml" features = "verilator,bemu" - if input_data.get("clean-before", input_data.get("clean_before", False)): - artifact_dir = os.path.join(env["CARGO_TARGET_DIR"], "test-artifacts") - shutil.rmtree(artifact_dir, ignore_errors=True) - ctx.logger.info(f"Cleaned previous bebop test artifacts: {artifact_dir}") harness_args = [ "--", "--workload-toml", @@ -101,6 +96,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--jobs", str(input_data.get("jobs", 1)), ] + if input_data.get("clean-before", input_data.get("clean_before", False)): + harness_args.append("--clean-before") if diff: harness_args.append("--diff") harness = shlex.join(harness_args) diff --git a/bbdev/api/steps/config/scripts/2_parameter_derivation.py b/bbdev/api/steps/config/scripts/2_parameter_derivation.py index 710dcbb4c..f63ff6bed 100644 --- a/bbdev/api/steps/config/scripts/2_parameter_derivation.py +++ b/bbdev/api/steps/config/scripts/2_parameter_derivation.py @@ -428,7 +428,9 @@ def _bemu_balls(repo: Path, topo: dict[str, Any]) -> list[dict[str, str]]: bid = entry.get("bid") if not isinstance(funct7, int) or not isinstance(bid, int): _die(f"ballISA entry must have funct7 and bid: {entry!r}") - if funct7 in {0, 1, 16, 32, 33, 34, 35}: + if funct7 == 0: + _die(f"core {pkg}: funct7 zero is not a Buckyball instruction") + if funct7 in {1, 16, 32, 33, 34, 35}: continue ball_class = bid_to_class.get(bid) if not ball_class: From 4b8e01b743e2e67442a6640456647a76cb297f0e Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 26 Oct 2026 16:39:55 +0800 Subject: [PATCH 188/196] feat: sync to buckyball 0.2.1-a --- .agents/mcps | 2 +- .agents/skills | 2 +- .github/scripts/ci_repo_lock.sh | 14 +- .github/workflows/README.zh.md | 2 +- .github/workflows/check.yaml | 49 +- .github/workflows/regression.yml | 106 +- .gitignore | 5 + .gitmodules | 53 +- .pre-commit-config.yaml | 2 +- README.md | 4 +- arch/.gitignore | 3 +- arch/build.sc | 466 +-- arch/settings.sc | 34 + arch/src/csrc/include/ioe/mmio.h | 2 +- arch/src/csrc/include/monitor/trace.h | 32 +- arch/src/csrc/src/monitor/ioe/BBSimDRAM.cc | 20 +- arch/src/csrc/src/monitor/ioe/mm_dramsim3.cc | 13 +- arch/src/csrc/src/monitor/trace/ctrace.cc | 24 + arch/src/csrc/src/monitor/trace/itrace.cc | 55 +- arch/src/csrc/src/monitor/trace/mtrace.cc | 59 +- arch/src/csrc/src/monitor/trace/pmctrace.cc | 4 +- arch/src/main/resources/bootrom/bootrom.S | 2 +- .../src/main/scala/configs/GlobalConfig.scala | 28 + .../scala/framework/acc/balldomain/build.sc | 33 + .../src/main/scala/BallDomain.scala | 198 + .../acc/balldomain/src/main/scala/Emit.scala | 39 + .../src/main/scala/ball/blink/baseball.scala | 27 + .../src/main/scala/ball/blink/blink.scala | 76 + .../src/main/scala/ball/blink/build.sc | 12 + .../balldomain/src/main/scala/ball/build.sc | 10 + .../balldomain/src/main/scala/bbus/bbus.scala | 192 + .../balldomain/src/main/scala/bbus/build.sc | 39 + .../main/scala/bbus/channels/Channels.scala | 202 + .../scala}/bbus/pmc/ballcyclepmc.spec.en.md | 0 .../scala}/bbus/pmc/ballcyclepmc.spec.zh.md | 0 .../main/scala/configs/BallDomainParam.scala | 67 + .../src/main/scala/decoder/DISA.scala | 7 + .../src/main/scala/decoder/Decoder.scala | 47 + .../src/main/scala/mmio_bank/MmioBank.scala | 131 + .../src/main/scala/sub_rob/Expansion.scala | 34 + .../src/main/scala/sub_rob/SubROB.scala | 277 ++ .../main/scala/sub_rob/configs/Params.scala | 7 + arch/src/main/scala/framework/acc/build.sc | 25 + .../framework/acc/configs/AccParam.scala | 63 + .../framework/acc/debug/BankCheckDPI.scala | 77 + .../scala/framework/acc/debug/Debug.scala | 179 + .../main/scala/framework/acc/debug/build.sc | 17 + .../acc/debug/trace/AntTraceDPI.scala | 47 + .../debug/trace}/ITraceDPI.scala | 21 +- .../framework/acc/debug/trace/MTraceDPI.scala | 118 + .../pmc => acc/debug/trace}/PMCTraceDPI.scala | 8 +- .../framework/acc/debug/trace/Trace.scala | 131 + .../scala/framework/acc/frontend/build.sc | 32 + .../acc/frontend/configs/default.toml | 9 + .../acc/frontend/src/csrc/decoder/Cargo.lock | 7 + .../acc/frontend/src/csrc/decoder/Cargo.toml | 6 + .../acc/frontend/src/csrc/decoder/src/lib.rs | 23 + .../main/resources/decoder/decoder.cm_hier} | 0 .../src/main/resources/decoder/decoder.f | 8 + .../src/main/resources/decoder/decoder_tb.sv | 75 + .../acc/frontend/src/main/scala/Emit.scala | 33 + .../frontend/src/main/scala/Frontend.scala | 97 + .../acc/frontend/src/main/scala/Ingress.scala | 56 + .../acc/frontend/src/main/scala/RoCC.scala | 30 + .../acc/frontend/src/main/scala/boot/build.sc | 20 + .../src/main/scala/boot/configs/uvm.toml | 21 + .../boot/src/main/resources/agent/driver.svh | 25 + .../boot/src/main/resources/agent/item.svh | 18 + .../boot/src/main/resources/agent/monitor.svh | 25 + .../scala/boot/src/main/resources/env/env.svh | 31 + .../boot/src/main/resources/env/model.svh | 100 + .../scala/boot/src/main/resources/filelist.f | 8 + .../boot/src/main/resources/interface.sv | 14 + .../scala/boot/src/main/resources/package.sv | 15 + .../boot/src/main/resources/seq/sequence.svh | 69 + .../scala/boot/src/main/resources/suite.toml | 20 + .../main/scala/boot/src/main/resources/tb.sv | 16 + .../boot/src/main/resources/tests/test.svh | 15 + .../main/scala/boot/src/main/scala/Boot.scala | 93 + .../main/scala/boot/src/main/scala/Emit.scala | 60 + .../main/scala/configs/FrontendParam.scala | 45 + .../src/main/scala/decoder/DISA.scala | 16 + .../main/scala/decoder/GlobalDecoder.scala | 178 + .../src/main/scala/scheduler/build.sc | 29 + .../scheduler/src/csrc/rename_retire.cpp | 140 + .../scala/scheduler/src/main/scala/Emit.scala | 35 + .../scheduler/src/main/scala/Scheduler.scala | 239 ++ .../scheduler/src/main/scala/bat/build.sc | 21 + .../src/main/scala/bat/configs/uvm.toml | 6 + .../bat/src/main/resources/agent/driver.svh | 25 + .../bat/src/main/resources/agent/item.svh | 18 + .../bat/src/main/resources/agent/monitor.svh | 25 + .../scala/bat/src/main/resources/env/env.svh | 25 + .../bat/src/main/resources/env/model.svh | 99 + .../scala/bat/src/main/resources/filelist.f | 8 + .../scala/bat/src/main/resources/interface.sv | 8 + .../scala/bat/src/main/resources/package.sv | 15 + .../bat/src/main/resources/seq/sequence.svh | 152 + .../scala/bat/src/main/resources/suite.toml | 122 + .../main/scala/bat/src/main/resources/tb.sv | 16 + .../bat/src/main/resources/tests/test.svh | 35 + .../bat/src/main/scala/BankAliasTable.scala | 106 + .../main/scala/bat/src/main/scala/Emit.scala | 54 + .../bat/src/main/scala/configs/Params.scala | 12 + .../src/main/scala/rob/global/build.sc | 23 + .../main/scala/rob/global/configs/uvm.toml | 57 + .../resources/global_rob/agent/driver.svh | 27 + .../main/resources/global_rob/agent/item.svh | 52 + .../resources/global_rob/agent/monitor.svh | 25 + .../main/resources/global_rob/agent/peer.svh | 61 + .../src/main/resources/global_rob/env/env.svh | 36 + .../main/resources/global_rob/env/model.svh | 475 +++ .../src/main/resources/global_rob/filelist.f | 8 + .../main/resources/global_rob/interface.sv | 14 + .../src/main/resources/global_rob/package.sv | 24 + .../resources/global_rob/seq/sequence.svh | 123 + .../src/main/resources/global_rob/tb.sv | 19 + .../main/resources/global_rob/tests/test.svh | 22 + .../src/main/resources/rob/agent/driver.svh | 25 + .../src/main/resources/rob/agent/item.svh | 18 + .../src/main/resources/rob/agent/monitor.svh | 25 + .../global/src/main/resources/rob/env/env.svh | 25 + .../src/main/resources/rob/env/model.svh | 117 + .../global/src/main/resources/rob/filelist.f | 8 + .../src/main/resources/rob/interface.sv | 8 + .../global/src/main/resources/rob/package.sv | 15 + .../src/main/resources/rob/seq/sequence.svh | 154 + .../rob/global/src/main/resources/rob/tb.sv | 16 + .../src/main/resources/rob/tests/test.svh | 35 + .../rob/global/src/main/resources/suite.toml | 71 + .../src/main/scala/DependencyResolver.scala | 124 + .../rob/global/src/main/scala/Emit.scala | 152 + .../rob/global/src/main/scala/GlobalROB.scala | 475 +++ .../scala/rob/global/src/main/scala/Rob.scala | 123 + .../src/main/scala/configs/RobParams.scala | 11 + .../src/main/scala/scoreboard/build.sc | 22 + .../main/scala/scoreboard/configs/uvm.toml | 6 + .../src/main/resources/agent/driver.svh | 25 + .../src/main/resources/agent/item.svh | 18 + .../src/main/resources/agent/monitor.svh | 25 + .../scoreboard/src/main/resources/env/env.svh | 25 + .../src/main/resources/env/model.svh | 57 + .../scoreboard/src/main/resources/filelist.f | 8 + .../src/main/resources/interface.sv | 8 + .../scoreboard/src/main/resources/package.sv | 15 + .../src/main/resources/seq/sequence.svh | 95 + .../scoreboard/src/main/resources/suite.toml | 20 + .../scala/scoreboard/src/main/resources/tb.sv | 16 + .../src/main/resources/tests/test.svh | 15 + .../src/main/scala}/BankAccessInfo.scala | 9 +- .../src/main/scala/BankScoreboard.scala | 113 + .../scoreboard/src/main/scala/Emit.scala | 61 + .../src/main/scala/configs/Params.scala | 10 + .../{rvv => acc/gpdomain}/.gitignore | 0 .../{mem-core => acc/gpdomain}/LICENSE | 0 .../scala/framework/acc/gpdomain/README.md | 1 + .../scala/framework/acc/gpdomain/build.sc | 33 + .../acc/gpdomain/configs}/default.toml | 0 .../acc/gpdomain/src/csrc/Cargo.lock | 128 + .../acc/gpdomain/src/csrc/Cargo.toml | 15 + .../framework/acc/gpdomain/src/csrc/build.rs | 64 + .../{rvv => acc/gpdomain}/src/csrc/src/ffi.rs | 2 +- .../acc/gpdomain/src/csrc/src/lib.rs | 4 + .../acc/gpdomain/src/csrc/src/model.rs | 903 +++++ .../gpdomain/src/main/resources}/README.md | 0 .../src/main/resources/gpdomain.json} | 0 .../gpdomain}/src/main/resources/interface.sv | 0 .../gpdomain/src/main/resources/package.sv | 97 + .../gpdomain/src/main/resources/suite.toml | 20 + .../acc/gpdomain/src/main/resources/test.svh | 265 ++ .../gpdomain/src/main/resources/topkernel.f | 8 + .../src/main/resources/topkernel_tb.sv | 2 +- .../acc/gpdomain/src/main/scala/Emit.scala | 26 + .../gpdomain/src/main/scala/GpDomain.scala | 76 + .../main/scala/configs/GpDomainParam.scala | 31 + .../src/main/scala/decoder/DISA.scala | 7 + .../src/main/scala/decoder/Decoder.scala | 48 + .../src/main/scala/execution/build.sc | 31 + .../scala/execution/src/main/scala/Emit.scala | 26 + .../execution/src/main/scala/Execution.scala | 441 +++ .../execution/src/main/scala}/IBuf.scala | 9 +- .../src/main/scala/arithmetic}/FALU.scala | 26 +- .../src/main/scala/arithmetic}/IALU.scala | 79 +- .../src/main/scala/configs/Params.scala | 26 + .../main/scala/scalar}/ScalarExecution.scala | 6 +- .../main/scala/scalar}/ScalarFloating.scala | 11 +- .../src/main/scala/scalar}/ScalarRF.scala | 10 +- .../src/main/scala/vector}/Operands.scala | 14 +- .../src/main/scala/vector}/VRF.scala | 9 +- .../src/main/scala/vector/VectorThread.scala | 941 +++++ .../gpdomain/src/main/scala/loader/build.sc | 23 + .../scala/loader/src/main/scala/Emit.scala | 26 + .../loader/src/main/scala}/ImageLoader.scala | 24 +- .../scala/loader/src/main/scala/Loader.scala | 237 ++ .../src/main/scala/configs/Params.scala | 8 + .../gpdomain/src/main/scala/memory/build.sc | 24 + .../memory/src/main/scala/BankPort.scala | 122 + .../scala/memory/src/main/scala/Emit.scala | 26 + .../memory/src/main/scala}/KernelMemory.scala | 16 +- .../scala/memory/src/main/scala/Memory.scala | 82 + .../src/main/scala/configs/Params.scala | 11 + .../scala/framework/acc/memdomain/build.sc | 36 + .../acc/memdomain/configs/default.toml | 21 + .../acc/memdomain/src/main/scala/Emit.scala | 39 + .../memdomain/src/main/scala/MemDomain.scala | 232 ++ .../src/main/scala/MemoryControl.scala | 242 ++ .../src/main/scala/bank_router/build.sc | 23 + .../main/scala/bank_router/configs/uvm.toml | 48 + .../src/main/resources/agent/driver.svh | 25 + .../src/main/resources/agent/item.svh | 22 + .../src/main/resources/agent/monitor.svh | 25 + .../src/main/resources/agent/peer.svh | 56 + .../src/main/resources/env/env.svh | 33 + .../src/main/resources/env/model.svh | 225 ++ .../bank_router/src/main/resources/filelist.f | 8 + .../src/main/resources/interface.sv | 9 + .../bank_router/src/main/resources/package.sv | 14 + .../src/main/resources/seq/sequence.svh | 108 + .../bank_router/src/main/resources/suite.toml | 19 + .../bank_router/src/main/resources/tb.sv | 15 + .../src/main/resources/tests/test.svh | 15 + .../src/main/scala/BankRouter.scala | 346 ++ .../bank_router/src/main/scala/Emit.scala | 104 + .../src/main/scala/blt/AccPipe.scala | 102 + .../bank_router/src/main/scala/blt/BLT.scala | 621 +++ .../main/scala/configs/MemDomainParam.scala | 91 + .../memdomain/src/main/scala/ctrl/Ctrl.scala | 102 + .../main/scala/ctrl/configer/Configer.scala | 138 + .../src/main/scala/ctrl/decoder/DISA.scala | 17 + .../src/main/scala/ctrl/decoder/Decoder.scala | 172 + .../acc/memdomain/src/main/scala/prm/build.sc | 23 + .../src/main/scala/prm/configs/uvm.toml | 27 + .../prm/src/main/resources/agent/driver.svh | 36 + .../prm/src/main/resources/agent/item.svh | 23 + .../prm/src/main/resources/agent/monitor.svh | 24 + .../scala/prm/src/main/resources/env/env.svh | 31 + .../prm/src/main/resources/env/model.svh | 108 + .../scala/prm/src/main/resources/filelist.f | 9 + .../scala/prm/src/main/resources/interface.sv | 21 + .../scala/prm/src/main/resources/package.sv | 13 + .../prm/src/main/resources/seq/sequence.svh | 140 + .../scala/prm/src/main/resources/suite.toml | 20 + .../main/scala/prm/src/main/resources/tb.sv | 19 + .../prm/src/main/resources/tests/test.svh | 16 + .../scala/prm/src/main/resources/types.svh | 13 + .../main/scala/prm/src/main/scala/Bank.scala | 101 + .../main/scala/prm/src/main/scala/Emit.scala | 55 + .../scala/prm/src/main/scala/MvoverIO.scala | 27 + .../main/scala/prm/src/main/scala/PRM.scala | 58 + .../memdomain/src/main/scala/tomem/build.sc | 28 + .../src/main/scala/tomem/configs/default.toml | 81 + .../src/main/scala/tomem/configs/uvm.toml | 82 + .../tomem/src/main/resources/agent/driver.svh | 25 + .../src/main/resources/agent/fixture.svh | 19 + .../tomem/src/main/resources/agent/item.svh | 18 + .../src/main/resources/agent/monitor.svh | 25 + .../tomem/src/main/resources/agent/peer.svh | 211 + .../tomem/src/main/resources/env/env.svh | 36 + .../tomem/src/main/resources/env/model.svh | 390 ++ .../scala/tomem/src/main/resources/filelist.f | 8 + .../tomem/src/main/resources/interface.sv | 9 + .../scala/tomem/src/main/resources/package.sv | 21 + .../tomem/src/main/resources/seq/sequence.svh | 118 + .../scala/tomem/src/main/resources/suite.toml | 19 + .../main/scala/tomem/src/main/resources/tb.sv | 15 + .../tomem/src/main/resources/tests/test.svh | 16 + .../scala/tomem/src/main/scala/Emit.scala | 57 + .../scala/tomem/src/main/scala/ToMem.scala | 434 ++ .../tomem/src/main/scala/configs/Params.scala | 42 + .../scala/tomem/src/main/scala/toddr/build.sc | 25 + .../src/main/scala/toddr/configs/uvm.toml | 32 + .../src/main/scala/toddr/src/csrc/Cargo.lock | 7 + .../src/main/scala/toddr/src/csrc/Cargo.toml | 7 + .../src/main/scala/toddr/src/csrc/src/lib.rs | 4 + .../src/main/resources/read_dma/env/env.svh | 44 + .../src/main/resources/read_dma/filelist.f | 8 + .../src/main/resources/read_dma/interface.sv | 10 + .../src/main/resources/read_dma/package.sv | 8 + .../main/resources/read_dma/seq/scenario.svh | 162 + .../toddr/src/main/resources/read_dma/tb.sv | 13 + .../main/resources/read_dma/tests/test.svh | 14 + .../scala/toddr/src/main/resources/suite.toml | 54 + .../src/main/resources/toddr/env/env.svh | 42 + .../toddr/src/main/resources/toddr/filelist.f | 8 + .../src/main/resources/toddr/interface.sv | 10 + .../toddr/src/main/resources/toddr/package.sv | 8 + .../src/main/resources/toddr/seq/scenario.svh | 186 + .../toddr/src/main/resources/toddr/tb.sv | 13 + .../src/main/resources/toddr/tests/test.svh | 14 + .../src/main/resources/write_dma/env/env.svh | 42 + .../src/main/resources/write_dma/filelist.f | 8 + .../src/main/resources/write_dma/interface.sv | 10 + .../src/main/resources/write_dma/package.sv | 8 + .../main/resources/write_dma/seq/scenario.svh | 123 + .../toddr/src/main/resources/write_dma/tb.sv | 13 + .../main/resources/write_dma/tests/test.svh | 14 + .../scala/toddr/src/main/scala/DmaPort.scala | 85 + .../toddr/src/main/scala}/DmaStatus.scala | 2 +- .../scala/toddr/src/main/scala/Emit.scala | 66 + .../scala/toddr/src/main/scala}/ReadDma.scala | 51 +- .../scala/toddr/src/main/scala/ToDDR.scala | 85 + .../toddr/src/main/scala}/WriteDma.scala | 41 +- .../scala/tomem/src/main/scala/tol1/build.sc | 20 + .../src/main/scala/tol1/configs/uvm.toml | 8 + .../scala/tol1/src/main/resources/filelist.f | 9 + .../scala/tol1/src/main/resources/suite.toml | 343 ++ .../main/scala/tol1/src/main/resources/tb.sv | 16 + .../main/scala/tol1/src/main/scala/Emit.scala | 72 + .../main/scala/tol1/src/main/scala/ToL1.scala | 52 + .../framework/balldomain/BallDomain.scala | 125 - .../framework/balldomain/bbus/bbus.scala | 255 -- .../balldomain/bbus/cmdrouter/CmdRouter.scala | 60 - .../bbus/cmdrouter/cmdrouter.spec.en.md | 33 - .../bbus/cmdrouter/cmdrouter.spec.zh.md | 33 - .../balldomain/bbus/pmc/BallCyclePMC.scala | 83 - .../balldomain/blink/SubRobRow.scala | 54 - .../framework/balldomain/blink/bank.scala | 34 - .../framework/balldomain/blink/baseball.scala | 14 - .../framework/balldomain/blink/blink.scala | 38 - .../balldomain/blink/mmio/MmioRead.scala | 50 - .../balldomain/blink/mmio/MmioWrite.scala | 34 - .../framework/balldomain/blink/status.scala | 9 - .../balldomain/configs/BallDomainParam.scala | 66 - .../balldomain/configs/BallParamLoader.scala | 66 - .../balldomain/decoder/BallDecodeCmd.scala | 35 - .../decoder/BallDomainDecoder.scala | 107 - .../framework/balldomain/isa/BallISA.scala | 14 - .../framework/balldomain/rs/interfaces.scala | 22 - .../balldomain/rs/reservationStation.scala | 121 - arch/src/main/scala/framework/build.sc | 236 ++ arch/src/main/scala/framework/core/build.sc | 31 + .../scala/framework/core/clink/basecore.scala | 5 + .../main/scala/framework/core/clink/build.sc | 13 + .../scala/framework/core/clink/clink.scala | 68 + .../framework/core/clink/configs/Params.scala | 23 + .../framework/core/configs/CoreParam.scala | 211 + .../core/controller/Controller.scala | 54 + .../core/controller/admission/build.sc | 23 + .../controller/admission/configs/uvm.toml | 15 + .../src/main/resources/command/env/env.svh | 25 + .../src/main/resources/command/env/model.svh | 148 + .../src/main/resources/command/filelist.f | 8 + .../src/main/resources/command/package.sv | 17 + .../main/resources/command/seq/sequence.svh | 88 + .../src/main/resources/command/tb.sv | 15 + .../src/main/resources/command/tests/test.svh | 15 + .../main/resources/common/agent/driver.svh | 25 + .../src/main/resources/common/agent/item.svh | 18 + .../main/resources/common/agent/monitor.svh | 25 + .../src/main/resources/common/interface.sv | 8 + .../src/main/resources/direct/env/env.svh | 25 + .../src/main/resources/direct/env/model.svh | 143 + .../src/main/resources/direct/filelist.f | 8 + .../src/main/resources/direct/package.sv | 17 + .../main/resources/direct/seq/sequence.svh | 92 + .../admission/src/main/resources/direct/tb.sv | 15 + .../src/main/resources/direct/tests/test.svh | 15 + .../src/main/resources/move/env/env.svh | 25 + .../src/main/resources/move/env/model.svh | 86 + .../src/main/resources/move/filelist.f | 8 + .../src/main/resources/move/package.sv | 17 + .../src/main/resources/move/seq/sequence.svh | 61 + .../admission/src/main/resources/move/tb.sv | 15 + .../src/main/resources/move/tests/test.svh | 15 + .../admission/src/main/resources/suite.toml | 156 + .../src/main/resources/task/env/env.svh | 25 + .../src/main/resources/task/env/model.svh | 126 + .../src/main/resources/task/filelist.f | 8 + .../src/main/resources/task/package.sv | 17 + .../src/main/resources/task/seq/sequence.svh | 68 + .../admission/src/main/resources/task/tb.sv | 15 + .../src/main/resources/task/tests/test.svh | 15 + .../src/main/scala/CommandAdmission.scala | 151 + .../src/main/scala/DirectAdmission.scala | 81 + .../admission/src/main/scala/Emit.scala | 134 + .../admission/src/main/scala/TaskRouter.scala | 85 + .../src/main/scala/configs/Params.scala | 7 + .../scala/framework/core/controller/build.sc | 16 + .../controller/configs/ControllerParam.scala | 26 + .../core/controller/configs/default.toml | 1 + .../core/controller/task/Control.scala | 165 + .../core/controller/task/Service.scala | 118 + .../framework/core/execution/Execution.scala | 60 + .../scala/framework/core/execution/build.sc | 33 + .../execution/configs/ExecutionParam.scala | 26 + .../framework/core/execution/fetch/build.sc | 22 + .../core/execution/fetch/configs/uvm.toml | 4 + .../execution}/fetch/src/csrc/Cargo.lock | 0 .../execution}/fetch/src/csrc/Cargo.toml | 0 .../fetch/src/csrc/src/instructions.rs | 0 .../execution}/fetch/src/csrc/src/lib.rs | 0 .../fetch/src/main/resources/agent/driver.svh | 62 + .../fetch/src/main/resources/agent/item.svh | 57 + .../src/main/resources/agent/monitor.svh | 25 + .../fetch/src/main/resources/agent/peer.svh | 95 + .../fetch/src/main/resources/env/env.svh | 40 + .../fetch/src/main/resources/env/model.svh | 224 ++ .../fetch/src/main/resources/filelist.f | 8 + .../fetch/src/main/resources/interface.sv | 20 + .../fetch/src/main/resources/package.sv | 26 + .../fetch/src/main/resources/seq/sequence.svh | 235 ++ .../fetch/src/main/resources/suite.toml | 105 + .../execution/fetch/src/main/resources/tb.sv | 19 + .../fetch/src/main/resources/tests/test.svh | 50 + .../execution/fetch/src/main/scala/Emit.scala | 69 + .../fetch/src/main/scala/Fetch.scala | 23 +- .../fetch/src/main/scala/configs/Params.scala | 11 + .../fetch/src/main/scala/configs/default.toml | 1 + .../framework/core/execution/fpu/build.sc | 21 + .../core/execution/fpu/configs/uvm.toml | 5 + .../core/execution/fpu/src/csrc/Cargo.lock | 7 + .../core/execution/fpu/src/csrc/Cargo.toml | 7 + .../core/execution/fpu/src/csrc/src/lib.rs | 58 + .../execution/fpu/src/csrc/src/vectors.rs | 3485 +++++++++++++++++ .../fpu/src/main/resources/agent/agent.svh | 19 + .../fpu/src/main/resources/agent/driver.svh | 55 + .../fpu/src/main/resources/agent/item.svh | 39 + .../fpu/src/main/resources/agent/monitor.svh | 57 + .../fpu/src/main/resources/env/coverage.svh | 47 + .../fpu/src/main/resources/env/env.svh | 27 + .../fpu/src/main/resources/env/model.svh | 20 + .../fpu/src/main/resources/filelist.f | 7 + .../fpu/src/main/resources/interface.sv | 29 + .../fpu/src/main/resources/package.sv | 38 + .../fpu/src/main/resources/seq/sequence.svh | 27 + .../fpu/src/main/resources/suite.toml | 20 + .../execution/fpu/src/main/resources/tb.sv | 34 + .../fpu/src/main/resources/tests/test.svh | 27 + .../execution/fpu/src/main/scala/Emit.scala | 15 + .../execution/fpu/src/main/scala/FPU.scala | 250 ++ .../fpu/src/main/scala/configs/Params.scala | 6 + .../fpu/src/main/scala/configs/default.toml | 1 + .../execution/instruction/LICENSE.Berkeley | 24 + .../core/execution/instruction/LICENSE.SiFive | 202 + .../core/execution/instruction/build.sc | 21 + .../execution/instruction/configs/uvm.toml | 4 + .../execution/instruction/src/csrc/Cargo.lock | 7 + .../execution/instruction/src/csrc/Cargo.toml | 7 + .../instruction/src/csrc/src/core/mod.rs | 1 + .../src/csrc/src/core/riscv/mod.rs | 6 + .../execution/instruction/src/csrc/src/lib.rs | 152 + .../src/main/resources/agent/agent.svh | 19 + .../src/main/resources/agent/driver.svh | 27 + .../src/main/resources/agent/item.svh | 47 + .../src/main/resources/agent/monitor.svh | 45 + .../src/main/resources/env/coverage.svh | 25 + .../src/main/resources/env/env.svh | 19 + .../src/main/resources/env/model.svh | 32 + .../instruction/src/main/resources/filelist.f | 7 + .../src/main/resources/interface.sv | 22 + .../instruction/src/main/resources/package.sv | 25 + .../src/main/resources/seq/sequence.svh | 40 + .../instruction/src/main/resources/suite.toml | 20 + .../instruction/src/main/resources/tb.sv | 36 + .../src/main/resources/tests/test.svh | 24 + .../instruction/src/main/scala/Emit.scala | 15 + .../src/main/scala/Instruction.scala | 315 ++ .../src/main/scala/configs/Params.scala | 6 + .../core/execution/integer/LICENSE.Berkeley | 24 + .../core/execution/integer/LICENSE.SiFive | 202 + .../framework/core/execution/integer/build.sc | 21 + .../core/execution/integer/configs/uvm.toml | 4 + .../src/main/resources/agent/agent.svh | 19 + .../src/main/resources/agent/driver.svh | 33 + .../integer/src/main/resources/agent/item.svh | 29 + .../src/main/resources/agent/monitor.svh | 36 + .../src/main/resources/env/coverage.svh | 30 + .../integer/src/main/resources/env/env.svh | 19 + .../integer/src/main/resources/env/model.svh | 65 + .../integer/src/main/resources/filelist.f | 7 + .../integer/src/main/resources/interface.sv | 13 + .../integer/src/main/resources/package.sv | 15 + .../src/main/resources/seq/sequence.svh | 66 + .../integer/src/main/resources/suite.toml | 20 + .../integer/src/main/resources/tb.sv | 27 + .../integer/src/main/resources/tests/test.svh | 25 + .../integer/src/main/scala/Emit.scala | 15 + .../integer/src/main/scala/Integer.scala | 93 + .../src/main/scala/configs/Params.scala | 6 + .../framework/core/execution/muldiv/build.sc | 21 + .../core/execution/muldiv/configs/uvm.toml | 4 + .../muldiv/src/main/resources/agent/agent.svh | 19 + .../src/main/resources/agent/driver.svh | 51 + .../muldiv/src/main/resources/agent/item.svh | 29 + .../src/main/resources/agent/monitor.svh | 52 + .../src/main/resources/env/coverage.svh | 34 + .../muldiv/src/main/resources/env/env.svh | 26 + .../muldiv/src/main/resources/env/model.svh | 35 + .../muldiv/src/main/resources/filelist.f | 7 + .../muldiv/src/main/resources/interface.sv | 23 + .../muldiv/src/main/resources/package.sv | 15 + .../src/main/resources/seq/sequence.svh | 46 + .../muldiv/src/main/resources/suite.toml | 20 + .../execution/muldiv/src/main/resources/tb.sv | 29 + .../muldiv/src/main/resources/tests/test.svh | 26 + .../muldiv/src/main/scala/Emit.scala | 15 + .../muldiv/src/main/scala/MulDiv.scala | 99 + .../src/main/scala/configs/Params.scala | 6 + .../core/execution/pipeline/Sequencer.scala | 453 +++ .../core/execution/privileged/build.sc | 21 + .../execution/privileged/configs/uvm.toml | 5 + .../execution/privileged/src/csrc/Cargo.lock | 7 + .../execution/privileged/src/csrc/Cargo.toml | 7 + .../privileged/src/csrc/src/core/mod.rs | 1 + .../privileged/src/csrc/src/core/riscv/mod.rs | 14 + .../execution/privileged/src/csrc/src/lib.rs | 326 ++ .../src/main/resources/agent/driver.svh | 25 + .../src/main/resources/agent/item.svh | 18 + .../src/main/resources/agent/monitor.svh | 25 + .../privileged/src/main/resources/env/env.svh | 25 + .../src/main/resources/env/model.svh | 141 + .../privileged/src/main/resources/filelist.f | 8 + .../src/main/resources/interface.sv | 8 + .../privileged/src/main/resources/package.sv | 56 + .../src/main/resources/seq/sequence.svh | 211 + .../privileged/src/main/resources/suite.toml | 20 + .../privileged/src/main/resources/tb.sv | 16 + .../src/main/resources/tests/test.svh | 15 + .../privileged/src/main/scala/Emit.scala | 57 + .../src/main/scala/Privileged.scala | 310 ++ .../src/main/scala/configs/Params.scala | 7 + .../core/execution/register_file/build.sc | 21 + .../execution/register_file/configs/uvm.toml | 4 + .../register_file/src/csrc/dual_read.cpp | 45 + .../src/main/resources/agent/agent.svh | 19 + .../src/main/resources/agent/driver.svh | 35 + .../src/main/resources/agent/item.svh | 29 + .../src/main/resources/agent/monitor.svh | 36 + .../src/main/resources/env/coverage.svh | 21 + .../src/main/resources/env/env.svh | 19 + .../src/main/resources/env/model.svh | 20 + .../src/main/resources/filelist.f | 7 + .../src/main/resources/interface.sv | 13 + .../src/main/resources/package.sv | 15 + .../src/main/resources/seq/sequence.svh | 47 + .../src/main/resources/suite.toml | 20 + .../register_file/src/main/resources/tb.sv | 27 + .../src/main/resources/tests/test.svh | 25 + .../register_file/src/main/scala/Emit.scala | 15 + .../src/main/scala/RegisterFile.scala | 40 + .../src/main/scala/configs/Params.scala | 6 + .../scala/framework/core/memory/Memory.scala | 65 + .../framework/core/memory/MemorySystem.scala | 177 + .../framework/core/memory/ThreadMemory.scala | 86 + .../framework/core/memory/access/build.sc | 32 + .../core/memory/access/configs/uvm.toml | 37 + .../core/memory/access/src/csrc/Cargo.lock | 7 + .../core/memory/access/src/csrc/Cargo.toml | 7 + .../memory/access}/src/csrc/src/access.rs | 4 +- .../core/memory/access/src/csrc/src/lib.rs | 5 + .../src/main/resources/physical/env/env.svh | 382 ++ .../src/main/resources/physical}/interface.sv | 0 .../src/main/resources/physical/mem_access.f | 10 + .../physical/mem_access_bad_atomic.f | 11 + .../physical/mem_access_bad_atomic_write.f | 11 + .../resources/physical/mem_access_bad_fence.f | 11 + .../physical/mem_access_bad_operand.f | 11 + .../resources/physical/mem_access_bad_size.f | 11 + .../resources/physical/mem_access_bad_tag.f | 11 + .../src/main/resources/physical/package.sv | 46 + .../access/src/main/resources/physical/tb.sv | 89 + .../main/resources/physical/tests/test.svh | 14 + .../access/src/main/resources/suite.toml | 139 + .../main/resources/virtual/agent/driver.svh | 25 + .../main/resources/virtual/agent/fixture.svh | 19 + .../src/main/resources/virtual/agent/item.svh | 26 + .../main/resources/virtual/agent/monitor.svh | 25 + .../src/main/resources/virtual/agent/peer.svh | 68 + .../src/main/resources/virtual/env/env.svh | 36 + .../src/main/resources/virtual/env/model.svh | 219 ++ .../src/main/resources/virtual/filelist.f | 8 + .../src/main/resources/virtual/interface.sv | 9 + .../src/main/resources/virtual/package.sv | 106 + .../main/resources/virtual/seq/sequence.svh | 110 + .../access/src/main/resources/virtual/tb.sv | 15 + .../src/main/resources/virtual/tests/test.svh | 16 + .../memory/access/src/main/scala/Emit.scala | 88 + .../src/main/scala/PhysicalAccess.scala | 151 + .../access/src/main/scala/VirtualAccess.scala | 349 ++ .../access/src/main/scala/cache/build.sc | 22 + .../src/main/scala/cache/configs/uvm.toml | 19 + .../cache/src/main/resources/agent/cmo_if.sv | 19 + .../cache/src/main/resources/env/model.svh | 116 + .../scala/cache/src/main/resources/filelist.f | 12 + .../scala/cache/src/main/resources/package.sv | 6 + .../cache/src/main/resources/seq/sequence.svh | 72 + .../scala/cache/src/main/resources/suite.toml | 105 + .../main/scala/cache/src/main/resources/tb.sv | 169 + .../cache/src/main/resources/tests/test.svh | 41 + .../scala/cache/src/main/scala/Cache.scala | 84 + .../scala/cache/src/main/scala/Emit.scala | 34 + .../cache/src/main/scala}/Maintenance.scala | 32 +- .../cache/src/main/scala/configs/Params.scala | 15 + .../src/main/scala/configs/Params.scala | 21 + .../memory/access/src/main/scala/mmu/build.sc | 22 + .../src/main/scala/mmu/configs/uvm.toml | 9 + .../src/main/scala}/mmu/src/csrc/Cargo.lock | 0 .../src/main/scala}/mmu/src/csrc/Cargo.toml | 0 .../src/main/scala}/mmu/src/csrc/src/lib.rs | 0 .../scala/mmu/src/csrc/src/translation.rs | 352 ++ .../mmu/src/main/resources/agent/driver.svh | 48 + .../mmu/src/main/resources/agent/item.svh | 43 + .../mmu/src/main/resources/agent/monitor.svh | 25 + .../mmu/src/main/resources/agent/peer.svh | 24 + .../scala/mmu/src/main/resources/env/env.svh | 36 + .../mmu/src/main/resources/env/model.svh | 158 + .../scala/mmu/src/main/resources/filelist.f | 8 + .../scala/mmu/src/main/resources/interface.sv | 20 + .../scala/mmu/src/main/resources/package.sv | 76 + .../mmu/src/main/resources/seq/sequence.svh | 331 ++ .../scala/mmu/src/main/resources/suite.toml | 71 + .../main/scala/mmu/src/main/resources/tb.sv | 19 + .../mmu/src/main/resources/tests/test.svh | 43 + .../main/scala/mmu/src/main/scala/Emit.scala | 57 + .../scala}/mmu/src/main/scala/Walker.scala | 68 +- .../mmu/src/main/scala/configs/Params.scala | 11 + .../access/src/main/scala/preflight/build.sc | 28 + .../src/main/scala/preflight/configs/uvm.toml | 22 + .../main/scala}/preflight/src/csrc/Cargo.lock | 0 .../main/scala}/preflight/src/csrc/Cargo.toml | 0 .../main/scala}/preflight/src/csrc/src/lib.rs | 0 .../preflight/src/csrc/src/preparation.rs | 2 +- .../preflight/src/csrc/src/prepared_map.rs | 0 .../resources/preflight/agent/service.svh | 113 + .../src/main/resources/preflight/env/env.svh | 170 + .../src/main/resources/preflight/filelist.f | 9 + .../main/resources/preflight/interface.sv} | 0 .../src/main/resources/preflight/package.sv | 96 + .../main/resources/preflight/seq/scenario.svh | 174 + .../src/main/resources/preflight/tb.sv | 52 + .../main/resources/preflight/tests/test.svh | 14 + .../resources/preparation/agent/driver.svh | 25 + .../resources/preparation/agent/fixture.svh | 8 + .../main/resources/preparation/agent/item.svh | 18 + .../resources/preparation/agent/monitor.svh | 25 + .../main/resources/preparation/agent/peer.svh | 81 + .../main/resources/preparation/env/env.svh | 36 + .../main/resources/preparation/env/model.svh | 205 + .../src/main/resources/preparation/filelist.f | 8 + .../main/resources/preparation/interface.sv | 9 + .../src/main/resources/preparation/package.sv | 97 + .../resources/preparation/seq/sequence.svh | 113 + .../src/main/resources/preparation/tb.sv | 15 + .../main/resources/preparation/tests/test.svh | 16 + .../resources/prepared_map/agent/service.svh | 47 + .../main/resources/prepared_map/env/env.svh | 89 + .../main/resources/prepared_map/filelist.f | 9 + .../main/resources/prepared_map/interface.sv | 12 + .../main/resources/prepared_map/package.sv | 52 + .../resources/prepared_map/seq/scenario.svh | 168 + .../src/main/resources/prepared_map/tb.sv | 42 + .../resources/prepared_map/tests/test.svh | 14 + .../preflight/src/main/resources/suite.toml | 54 + .../scala/preflight/src/main/scala/Emit.scala | 126 + .../preflight/src/main/scala/Permission.scala | 23 + .../src/main/scala/PermissionCheck.scala | 108 + .../preflight/src/main/scala/Preflight.scala | 93 +- .../src/main/scala/Preparation.scala | 88 + .../src/main/scala/PreparedMap.scala | 36 +- .../preflight/src/main/scala/Segmenter.scala | 7 +- .../src/main/scala/configs/Config.scala | 18 + .../src/main/scala/configs/Params.scala | 33 + .../main/scala/framework/core/memory/build.sc | 21 + .../core/memory/configs/MemoryParam.scala | 44 + .../scala/framework/core/memory/home/build.sc | 21 + .../core/memory/home/configs/uvm.toml | 14 + .../src/main/resources/agent/transport.svh | 57 + .../src/main/resources/agent/transport_if.sv | 9 + .../memory/home/src/main/resources/filelist.f | 11 + .../home}/src/main/resources/interface.sv | 0 .../memory/home/src/main/resources/package.sv | 8 + .../home/src/main/resources/seq/sequence.svh | 16 + .../memory/home/src/main/resources/suite.toml | 19 + .../core/memory/home/src/main/resources/tb.sv | 228 ++ .../home/src/main/resources/tests/test.svh | 17 + .../memory/home/src/main/scala/Emit.scala | 169 + .../memory/home/src/main/scala/Home.scala | 72 + .../home/src/main/scala/configs/Params.scala | 13 + .../core/memory/uncached/Uncached.scala | 77 + .../uncached/configs/UncachedParam.scala | 15 + .../core/memory/uncached/configs/default.toml | 2 + .../main/scala/framework/core/shm/SHM.scala | 19 + .../main/scala/framework/core/shm/build.sc | 11 + .../main/scala/framework/core/tbus/build.sc | 53 + .../core/tbus/src/main/scala/Emit.scala | 36 + .../tbus/src/main/scala/SharedNetwork.scala | 129 + .../core/tbus/src/main/scala/TBus.scala | 148 + .../tbus/src/main/scala/configs/Params.scala | 41 + .../core/tbus/src/main/scala/router/build.sc | 21 + .../src/main/scala/router/configs/uvm.toml | 57 + .../src/main/resources/agent/driver.svh | 25 + .../src/main/resources/agent/fixture.svh | 8 + .../router/src/main/resources/agent/item.svh | 23 + .../src/main/resources/agent/monitor.svh | 25 + .../router/src/main/resources/agent/peer.svh | 70 + .../router/src/main/resources/env/env.svh | 36 + .../router/src/main/resources/env/model.svh | 379 ++ .../router/src/main/resources/filelist.f | 9 + .../router/src/main/resources/interface.sv | 9 + .../router/src/main/resources/package.sv | 18 + .../src/main/resources/seq/sequence.svh | 281 ++ .../router/src/main/resources/suite.toml | 88 + .../scala/router/src/main/resources/tb.sv | 15 + .../src/main/resources/tests/invalid.svh | 63 + .../router/src/main/resources/tests/test.svh | 16 + .../scala/router/src/main/scala/Emit.scala | 142 + .../scala/router/src/main/scala/Router.scala | 645 +++ .../src/main/scala/configs/Params.scala | 45 + .../main/scala/framework/core/tss/build.sc | 22 + .../scala/framework/core/tss/configs/uvm.toml | 9 + .../tss/src/main/resources/agent/driver.svh | 25 + .../tss/src/main/resources/agent/item.svh | 23 + .../tss/src/main/resources/agent/monitor.svh | 25 + .../core/tss/src/main/resources/env/env.svh | 34 + .../core/tss/src/main/resources/env/model.svh | 155 + .../core/tss/src/main/resources/filelist.f | 8 + .../core/tss/src/main/resources/interface.sv | 17 + .../core/tss/src/main/resources/package.sv | 13 + .../tss/src/main/resources/seq/sequence.svh | 164 + .../core/tss/src/main/resources/suite.toml | 20 + .../core/tss/src/main/resources/tb.sv | 16 + .../tss/src/main/resources/tests/test.svh | 15 + .../core/tss/src/main/scala/Emit.scala | 88 + .../core/tss/src/main/scala/Store.scala | 94 + .../tss/src/main/scala/configs/Params.scala | 13 + arch/src/main/scala/framework/device/build.sc | 22 + .../scala/framework/device/cbus/C2C.scala | 134 + .../scala/framework/device/cbus/CBus.scala | 148 + .../scala/framework/device/cbus/Control.scala | 145 + .../scala/framework/device/cbus/Fabric.scala | 92 + .../cbus}/SharedAxiEndpoint.scala | 36 +- .../main/scala/framework/device/cbus/build.sc | 46 + .../device/configs/DeviceParam.scala | 45 + .../main/scala/framework/device/ddr/build.sc | 22 + .../framework/device/ddr/configs/default.toml | 13 + .../framework/device/ddr/configs/uvm.toml | 14 + .../ddr/src/csrc/Cargo.lock | 0 .../ddr/src/csrc/Cargo.toml | 0 .../bank => device/ddr}/src/csrc/src/lib.rs | 0 .../ddr/src/csrc/src/memory.rs | 0 .../ddr/src/main/resources/agent/item.svh | 17 + .../ddr/src/main/resources/agent/service.svh | 166 + .../device/ddr/src/main/resources/env/env.svh | 147 + .../device/ddr/src/main/resources/filelist.f | 9 + .../ddr/src/main/resources/interface.sv} | 0 .../device/ddr/src/main/resources/package.sv | 30 + .../ddr/src/main/resources/seq/scenario.svh | 258 ++ .../device/ddr/src/main/resources/suite.toml | 139 + .../device/ddr/src/main/resources/tb.sv | 75 + .../ddr/src/main/resources/tests/test.svh | 14 + .../ddr/src/main/scala/Bridge.scala | 44 +- .../device/ddr/src/main/scala/Emit.scala | 116 + .../device/ddr/src/main/scala/Params.scala | 47 + .../framework/device/dlink/basedevice.scala | 16 + .../scala/framework/device/dlink/build.sc | 12 + .../scala/framework/device/dlink/dlink.scala | 13 + .../device/peripherals/Controller.scala | 176 + .../device/peripherals/bootrom/BootRom.scala | 79 + .../bootrom/configs/Parameters.scala | 21 + .../peripherals/bootrom/configs/default.toml | 4 + .../framework/device/peripherals/build.sc | 20 + .../device/peripherals/clint/Clint.scala | 69 + .../clint/configs/Parameters.scala | 26 + .../peripherals/clint/configs/default.toml | 4 + .../peripherals/plic}/Plic.scala | 18 +- .../peripherals/plic/configs/Parameters.scala | 28 + .../peripherals/plic/configs/default.toml | 5 + .../device/peripherals/scu/SCU.scala | 116 + .../peripherals/scu/SystemControl.scala | 81 + .../peripherals/scu/configs/Parameters.scala | 64 + .../peripherals/scu/configs/default.toml | 4 + .../scala/framework/frontend/Frontend.scala | 119 - .../scala/framework/frontend/boot/Boot.scala | 72 - .../frontend/configs/FrontendParam.scala | 33 - .../framework/frontend/decoder/GISA.scala | 19 - .../frontend/decoder/GobalDecoder.scala | 139 - .../frontend/decoder/PostGDCmd.scala | 17 - .../frontend/globalrs/GlobalROB.scala | 592 --- .../frontend/globalrs/GlobalScheduler.scala | 258 -- .../framework/frontend/globalrs/SubROB.scala | 206 - .../frontend/scoreboard/BankAliasTable.scala | 126 - .../frontend/scoreboard/BankScoreboard.scala | 97 - .../framework/{mem-core => ip}/.gitignore | 0 .../main/scala/framework/{root => ip}/LICENSE | 0 .../framework/{mem-core => ip}/README.md | 0 .../src/main/scala/framework/ip/axi4/build.sc | 24 + .../framework/ip/axi4/configs/default.toml | 3 + .../scala/framework/ip/axi4/configs/uvm.toml | 17 + .../main/resources/decoder/agent/driver.svh | 41 + .../src/main/resources/decoder/agent/item.svh | 17 + .../main/resources/decoder/agent/monitor.svh | 24 + .../src/main/resources/decoder/env/env.svh | 25 + .../src/main/resources/decoder/env/model.svh | 112 + .../src/main/resources/decoder/filelist.f | 9 + .../src/main/resources/decoder/interface.sv | 7 + .../src/main/resources/decoder/package.sv | 14 + .../main/resources/decoder/seq/sequence.svh | 117 + .../ip/axi4/src/main/resources/decoder/tb.sv | 15 + .../src/main/resources/decoder/tests/test.svh | 15 + .../resources/interconnect/agent/driver.svh | 156 + .../resources/interconnect/agent/item.svh | 33 + .../resources/interconnect/agent/monitor.svh | 25 + .../main/resources/interconnect/env/env.svh | 27 + .../main/resources/interconnect/env/model.svh | 186 + .../main/resources/interconnect/filelist.f | 9 + .../main/resources/interconnect/interface.sv | 35 + .../main/resources/interconnect/package.sv | 13 + .../resources/interconnect/seq/sequence.svh | 22 + .../src/main/resources/interconnect/tb.sv | 22 + .../resources/interconnect/tests/test.svh | 17 + .../src/main/resources/interconnect/types.svh | 46 + .../ip/axi4/src/main/resources/suite.toml | 37 + .../ip/axi4/src/main/scala/Decoder.scala | 64 + .../ip/axi4/src/main/scala/Emit.scala | 72 + .../axi4/src/main/scala/Interconnect.scala | 22 +- .../ip/axi4/src/main/scala/Interface.scala | 44 + .../ip/axi4/src/main/scala/Params.scala | 17 + .../src/main/scala/framework/ip/axis/build.sc | 24 + .../scala/framework/ip/axis/configs/uvm.toml | 6 + .../{mem-core => ip}/axis/src/csrc/Cargo.lock | 0 .../{mem-core => ip}/axis/src/csrc/Cargo.toml | 0 .../resources => ip/axis/src/csrc}/README.md | 0 .../framework/ip/axis/src/csrc/src/arbiter.rs | 23 + .../{mem-core => ip}/axis/src/csrc/src/ffi.rs | 0 .../{mem-core => ip}/axis/src/csrc/src/lib.rs | 0 .../framework/ip/axis/src/csrc/src/model.rs | 23 + .../main/resources/arbiter/agent/agent.svh | 22 + .../main/resources/arbiter/agent/driver.svh | 37 + .../src/main/resources/arbiter/agent/item.svh | 40 + .../main/resources/arbiter/agent/monitor.svh | 50 + .../main/resources/arbiter/env/coverage.svh | 44 + .../src/main/resources/arbiter/env/env.svh | 22 + .../main/resources/arbiter/env/ref_model.svh | 36 + .../src/main/resources/arbiter/filelist.f | 7 + .../src/main/resources/arbiter/interface.sv | 36 + .../src/main/resources/arbiter/package.sv | 24 + .../main/resources/arbiter/seq/sequence.svh | 139 + .../ip/axis/src/main/resources/arbiter/tb.sv | 36 + .../src/main/resources/arbiter/tests/test.svh | 33 + .../src/main/resources/common/coverage.svh | 19 + .../src/main/resources/master/env/env.svh | 44 + .../main/resources/master/env/ref_model.svh | 39 + .../axis/src/main/resources/master/filelist.f | 9 + .../axis/src/main/resources/master/package.sv | 31 + .../main/resources/master/seq/sequence.svh | 19 + .../ip/axis/src/main/resources/master/tb.sv | 43 + .../src/main/resources/master/tests/test.svh | 23 + .../axis/src/main/resources/slave/env/env.svh | 44 + .../main/resources/slave/env/ref_model.svh | 39 + .../axis/src/main/resources/slave/filelist.f | 9 + .../axis/src/main/resources/slave/package.sv | 31 + .../src/main/resources/slave/seq/sequence.svh | 19 + .../ip/axis/src/main/resources/slave/tb.sv | 43 + .../src/main/resources/slave/tests/test.svh | 23 + .../ip/axis/src/main/resources/suite.toml | 54 + .../ip/axis/src/main/scala/Emit.scala | 22 + .../ip/axis/src/main/scala/Interface.scala | 51 + .../axis/src/main/scala/Master.scala | 2 +- .../axis/src/main/scala/PacketArbiter.scala | 6 +- .../axis/src/main/scala/Slave.scala | 2 +- .../src/main/scala/framework/ip/bank/build.sc | 23 + .../scala/framework/ip/bank/configs/uvm.toml | 7 + .../{mem-core => ip}/bank/src/csrc/Cargo.lock | 0 .../{mem-core => ip}/bank/src/csrc/Cargo.toml | 0 .../{mem-core => ip}/bank/src/csrc/README.md | 0 .../cache => ip/bank}/src/csrc/src/lib.rs | 0 .../framework/ip/bank/src/csrc/src/memory.rs | 50 + .../src/main/resources/bank/agent/agent.svh | 22 + .../src/main/resources/bank/agent/driver.svh | 65 + .../src/main/resources/bank/agent/item.svh | 37 + .../src/main/resources/bank/agent/monitor.svh | 74 + .../src/main/resources/bank/env/coverage.svh | 20 + .../bank/src/main/resources/bank/env/env.svh | 32 + .../src/main/resources/bank/env/ref_model.svh | 26 + .../bank/src/main/resources/bank/filelist.f | 8 + .../bank/src/main/resources/bank/interface.sv | 20 + .../bank/src/main/resources/bank/package.sv | 26 + .../src/main/resources/bank/seq/sequence.svh | 43 + .../ip/bank/src/main/resources/bank/tb.sv | 36 + .../src/main/resources/bank/tests/test.svh | 32 + .../main/resources/bankset/agent/agent.svh | 18 + .../main/resources/bankset/agent/command.svh | 10 + .../main/resources/bankset/agent/driver.svh | 41 + .../src/main/resources/bankset/agent/item.svh | 11 + .../main/resources/bankset/agent/monitor.svh | 83 + .../main/resources/bankset/env/coverage.svh | 25 + .../src/main/resources/bankset/env/env.svh | 27 + .../main/resources/bankset/env/ref_model.svh | 25 + .../src/main/resources/bankset/filelist.f | 11 + .../src/main/resources/bankset/interface.sv | 20 + .../src/main/resources/bankset/package.sv | 28 + .../main/resources/bankset/seq/sequence.svh | 77 + .../ip/bank/src/main/resources/bankset/tb.sv | 49 + .../src/main/resources/bankset/tests/test.svh | 20 + .../ip/bank/src/main/resources/suite.toml | 37 + .../ip/bank/src/main/scala/Bank.scala | 68 + .../bank/src/main/scala/BankSet.scala | 4 +- .../ip/bank/src/main/scala/Emit.scala | 39 + .../ip/bank/src/main/scala/SramIO.scala | 36 + arch/src/main/scala/framework/ip/build.sc | 34 + .../main/scala/framework/ip/cache/build.sc | 23 + .../ip/cache/configs/CacheParams.scala | 39 + .../cache/configs/default.toml | 0 .../scala/framework/ip/cache/configs/uvm.toml | 10 + .../cache/src/csrc/Cargo.lock | 0 .../cache/src/csrc/Cargo.toml | 0 .../cache}/src/csrc/src/lib.rs | 0 .../cache/src/csrc/src/memory.rs | 0 .../cache/src/main/resources/agent/agent.svh | 20 + .../cache/src/main/resources/agent/driver.svh | 57 + .../cache/src/main/resources/agent/item.svh | 40 + .../src/main/resources/agent/monitor.svh | 60 + .../cache/src/main/resources/agent/sink.svh | 19 + .../cache/src/main/resources/env/coverage.svh | 23 + .../ip/cache/src/main/resources/env/env.svh | 26 + .../src/main/resources/env}/ref_model.svh | 0 .../ip/cache/src/main/resources/filelist.f | 8 + .../cache/src/main/resources/interface.sv | 0 .../ip/cache/src/main/resources/package.sv | 41 + .../cache/src/main/resources/seq/sequence.svh | 122 + .../ip/cache/src/main/resources/suite.toml | 20 + .../cache/src/main/resources/tb.sv} | 0 .../cache/src/main/resources/tests/test.svh | 18 + .../ip/cache/src/main/scala/Cache.scala | 112 + .../ip/cache/src/main/scala/Emit.scala | 43 + .../ip/cache/src/main/scala/Interface.scala | 42 + arch/src/main/scala/framework/ip/chi/build.sc | 37 + .../scala/framework/ip/chi/configs/uvm.toml | 19 + .../{mem-core => ip}/chi/src/csrc/Cargo.lock | 0 .../{mem-core => ip}/chi/src/csrc/Cargo.toml | 0 .../{mem-core => ip}/chi/src/csrc/README.md | 0 .../framework/ip/chi/src/csrc/src/lib.rs | 4 + .../chi/src/csrc/src/memory.rs | 0 .../src/main/resources/common}/flit_types.svh | 16 +- .../chi/src/main/resources/common/profile.svh | 10 + .../src/main/resources/retry/agent/agent.svh | 28 + .../src/main/resources/retry/agent/driver.svh | 44 + .../main/resources/retry/agent/endpoint.svh | 20 + .../src/main/resources/retry/agent/item.svh | 37 + .../main/resources/retry/agent/monitor.svh | 39 + .../chi/src/main/resources/retry/env/env.svh | 28 + .../src/main/resources/retry/env/model.svh | 66 + .../chi/src/main/resources/retry/filelist.f | 10 + .../chi/src/main/resources/retry/interface.sv | 24 + .../chi/src/main/resources/retry/package.sv | 16 + .../src/main/resources/retry/seq/sequence.svh | 430 ++ .../ip/chi/src/main/resources/retry/tb.sv | 104 + .../src/main/resources/retry/tests/test.svh | 26 + .../ip/chi/src/main/resources/retry/types.svh | 14 + .../ip/chi/src/main/resources/rnf/filelist.f | 11 + .../ip/chi/src/main/resources/rnf/package.sv | 3 + .../ip/chi/src/main/resources/rnf/tb.sv | 155 + .../chi/src/main/resources/rx/agent/agent.svh | 18 + .../src/main/resources/rx/agent/driver.svh | 29 + .../chi/src/main/resources/rx/agent/item.svh | 17 + .../src/main/resources/rx/agent/monitor.svh | 24 + .../ip/chi/src/main/resources/rx/env/env.svh | 26 + .../chi/src/main/resources/rx/env/model.svh | 50 + .../ip/chi/src/main/resources/rx/filelist.f | 9 + .../ip/chi/src/main/resources/rx/interface.sv | 12 + .../ip/chi/src/main/resources/rx/package.sv | 14 + .../src/main/resources/rx/seq/sequence.svh | 89 + .../ip/chi/src/main/resources/rx/tb.sv | 23 + .../chi/src/main/resources/rx/tests/test.svh | 31 + .../ip/chi/src/main/resources/rx/types.svh | 9 + .../src/main/resources/sram/agent/agent.svh | 24 + .../src/main/resources/sram/agent/driver.svh | 39 + .../main/resources/sram/agent/endpoint.svh | 24 + .../src/main/resources/sram/agent/item.svh | 37 + .../src/main/resources/sram/agent/monitor.svh | 44 + .../chi/src/main/resources/sram/env/env.svh | 27 + .../chi/src/main/resources/sram/env/model.svh | 79 + .../ip/chi/src/main/resources/sram/filelist.f | 10 + .../chi/src/main/resources/sram/interface.sv | 17 + .../ip/chi/src/main/resources/sram/package.sv | 29 + .../src/main/resources/sram/seq/sequence.svh | 450 +++ .../ip/chi/src/main/resources/sram/tb.sv | 41 + .../src/main/resources/sram/tests/test.svh | 25 + .../ip/chi/src/main/resources/sram/types.svh | 16 + .../ip/chi/src/main/resources/suite.toml | 241 ++ .../chi/src/main/resources/tx/agent/agent.svh | 18 + .../src/main/resources/tx/agent/driver.svh | 28 + .../chi/src/main/resources/tx/agent/item.svh | 17 + .../src/main/resources/tx/agent/monitor.svh | 24 + .../ip/chi/src/main/resources/tx/env/env.svh | 26 + .../chi/src/main/resources/tx/env/model.svh | 55 + .../ip/chi/src/main/resources/tx/filelist.f | 9 + .../ip/chi/src/main/resources/tx/interface.sv | 7 + .../ip/chi/src/main/resources/tx/package.sv | 14 + .../src/main/resources/tx/seq/sequence.svh | 79 + .../ip/chi/src/main/resources/tx/tb.sv | 24 + .../chi/src/main/resources/tx/tests/test.svh | 31 + .../ip/chi/src/main/resources/tx/types.svh | 9 + .../chi/src/main/scala/Channel.scala | 2 +- .../chi/src/main/scala/CoherenceState.scala | 2 +- .../chi/src/main/scala/DirectedSnoop.scala | 2 +- .../ip/chi/src/main/scala/Emit.scala | 79 + .../ip/chi/src/main/scala/Fabric.scala | 104 + .../chi/src/main/scala/Flit.scala | 11 +- .../chi/src/main/scala/HomeMapping.scala | 2 +- .../chi/src/main/scala/RequestRetry.scala | 11 +- .../chi/src/main/scala/RequesterPort.scala | 2 +- .../chi/src/main/scala/Rx.scala | 2 +- .../chi/src/main/scala/Tx.scala | 2 +- .../chi/src/main/scala/hnf/ChiHome.scala | 6 +- .../ip/chi/src/main/scala/rnf/AxiCache.scala | 131 + .../src/main/scala/rnf/BankedChiCache.scala | 68 +- .../chi/src/main/scala/rnf/CachePort.scala | 4 +- .../chi/src/main/scala/rnf/ChiCache.scala | 4 +- .../chi/src/main/scala/snf/MemoryNode.scala | 37 +- .../chi/src/main/scala/snf/SramEndpoint.scala | 4 +- .../scala/framework/ip/coherence/build.sc | 25 + .../coherence/configs/CoherenceParams.scala | 34 + .../coherence/configs/default.toml | 0 .../ip/coherence/configs/requesters.toml | 3 + .../framework/ip/coherence/configs/uvm.toml | 14 + .../coherence/src/main/resources/filelist.f | 10 + .../coherence/src/main/resources/interface.sv | 12 + .../coherence/src/main/resources/package.sv | 5 + .../coherence/src/main/resources/suite.toml | 20 + .../coherence/src/main/resources/tb.sv} | 0 .../coherence/src/main/scala/Coherence.scala | 71 +- .../coherence/src/main/scala/Directory.scala | 4 +- .../ip/coherence/src/main/scala/Emit.scala | 127 + .../coherence/src/main/scala/Interface.scala | 19 + .../coherence/src/main/scala/Mshr.scala | 6 +- .../scala/framework/ip/interlock/build.sc | 23 + .../framework/ip/interlock/configs/uvm.toml | 8 + .../interlock/src/main/resources/filelist.f | 9 + .../interlock/src/main/resources/suite.toml | 343 ++ .../ip/interlock/src/main/resources/tb.sv | 16 + .../ip/interlock/src/main/scala/Emit.scala | 70 + .../interlock/src/main/scala/Interface.scala | 38 + .../interlock/src/main/scala/Interlock.scala | 17 +- .../ip/interlock/src/main/scala/Params.scala | 24 + .../main/scala/framework/ip/mesh_shm/build.sc | 32 + .../mesh_shm/src/csrc/Cargo.lock | 0 .../mesh_shm/src/csrc/Cargo.toml | 0 .../ip/mesh_shm/src/csrc/fixed_attachment.cpp | 106 + .../mesh_shm/src/csrc/src/lib.rs | 0 .../mesh_shm/src/csrc/src/transfer.rs | 0 .../ip/mesh_shm/src/main/resources/router.f | 7 + .../src/main/resources/router_item.svh | 0 .../src/main/resources/router_package.sv | 13 + .../mesh_shm/src/main/resources/router_tb.sv | 0 .../src/main/resources/router_test.svh | 6 +- .../ip/mesh_shm/src/main/resources/shared.f | 8 + .../src/main/resources/shared_item.svh | 0 .../src/main/resources/shared_package.sv | 2 +- .../mesh_shm/src/main/resources/shared_tb.sv | 4 +- .../src/main/resources/shared_test.svh | 30 +- .../ip/mesh_shm/src/main/resources/suite.toml | 37 + .../src/main/resources/transfer_if.sv | 2 +- .../ip/mesh_shm/src/main/scala/Emit.scala | 15 + .../src/main/scala/MeshBankNode.scala | 4 +- .../mesh_shm/src/main/scala/MeshChannel.scala | 10 + .../src/main/scala/MeshChannelOrdering.scala | 2 +- .../src/main/scala/MeshControlInterface.scala | 53 + .../src/main/scala/MeshCreditLink.scala | 7 +- .../mesh_shm/src/main/scala/MeshRouter.scala | 18 +- .../src/main/scala/MeshSharedMem.scala | 259 ++ .../main/scala/MeshSharedMemInterface.scala | 37 + .../src/main/scala/MeshSharedMemParams.scala | 79 + .../src/main/scala/MeshThreadEndpoint.scala | 59 + .../main/scala/MeshTransferController.scala | 28 +- .../src/main/scala/configs/Geometry.scala | 11 + arch/src/main/scala/framework/ip/spm/build.sc | 22 + .../scala/framework/ip/spm/configs/uvm.toml | 6 + .../{mem-core => ip}/spm/src/csrc/src/lib.rs | 0 .../ip/spm/src/main/resources/agent/agent.svh | 19 + .../spm/src/main/resources/agent/driver.svh | 55 + .../ip/spm/src/main/resources/agent/item.svh | 35 + .../spm/src/main/resources/agent/monitor.svh | 56 + .../spm/src/main/resources/env/coverage.svh | 27 + .../ip/spm/src/main/resources/env/env.svh | 26 + .../ip/spm/src/main/resources/env/model.svh | 37 + .../ip/spm/src/main/resources/filelist.f | 8 + .../ip/spm/src/main/resources/interface.sv | 27 + .../ip/spm/src/main/resources/package.sv | 16 + .../spm/src/main/resources/seq/sequence.svh | 53 + .../ip/spm/src/main/resources/suite.toml | 20 + .../framework/ip/spm/src/main/resources/tb.sv | 33 + .../ip/spm/src/main/resources/tests/test.svh | 27 + .../ip/spm/src/main/scala/Emit.scala | 24 + .../ip/spm/src/main/scala/Interfaces.scala | 35 + .../ip/spm/src/main/scala/Memory.scala | 60 + .../scala/framework/lib/arbiter/Arbiter.scala | 45 + .../main/scala/framework/lib/arbiter/build.sc | 11 + .../src/main/scala => lib/arith}/DivRem.scala | 2 +- .../main/scala/framework/lib/arith/Fma.scala | 89 + .../framework/lib/arith/IntToFloat.scala | 109 + .../framework/lib/arith/LICENSE.Berkeley | 24 + .../scala/framework/lib/arith/LICENSE.SiFive | 202 + .../main/scala => lib/arith}/Multiply.scala | 5 +- .../main/scala/framework/lib/arith/build.sc | 14 + arch/src/main/scala/framework/lib/build.sc | 22 + .../framework/{ => lib}/dpi/DpiGuard.scala | 2 +- .../src/main/scala/framework/lib/dpi/build.sc | 11 + .../scala/framework/lib/hash/BankHash.scala | 75 + .../main/scala/framework/lib/hash/build.sc | 11 + .../framework/lib/hash/src/csrc/monitor.cpp | 110 + .../src/main/scala => lib/queue}/Queue.scala | 2 +- .../main/scala/framework/lib/queue/build.sc | 11 + .../scala/framework/mem-core/axi4/build.sc | 20 - .../mem-core/axi4/src/main/scala/Emit.scala | 9 - .../axi4/src/main/scala/Interface.scala | 44 - .../mem-core/axi4/src/main/scala/Params.scala | 7 - .../scala/framework/mem-core/axis/build.sc | 11 - .../mem-core/axis/src/csrc/src/arbiter.rs | 49 - .../mem-core/axis/src/csrc/src/model.rs | 50 - .../axis/src/main/resources/arbiter.f | 5 - .../src/main/resources/arbiter_checker.svh | 47 - .../src/main/resources/arbiter_driver.svh | 80 - .../src/main/resources/arbiter_interface.sv | 25 - .../src/main/resources/arbiter_package.sv | 17 - .../axis/src/main/resources/arbiter_tb.sv | 42 - .../axis/src/main/resources/arbiter_test.svh | 42 - .../mem-core/axis/src/main/resources/env.svh | 37 - .../mem-core/axis/src/main/resources/master.f | 9 - .../axis/src/main/resources/master_tb.sv | 41 - .../axis/src/main/resources/package.sv | 29 - .../axis/src/main/resources/ref_model.svh | 39 - .../mem-core/axis/src/main/resources/slave.f | 9 - .../axis/src/main/resources/slave_tb.sv | 41 - .../mem-core/axis/src/main/resources/test.svh | 43 - .../mem-core/axis/src/main/scala/Emit.scala | 19 - .../axis/src/main/scala/Interface.scala | 51 - .../scala/framework/mem-core/bank/build.sc | 22 - .../mem-core/bank/src/csrc/src/memory.rs | 80 - .../mem-core/bank/src/main/resources/bank.el | 5 - .../mem-core/bank/src/main/resources/bank.f | 7 - .../bank/src/main/resources/bank_tb.sv | 29 - .../bank/src/main/resources/bankset.el | 79 - .../bank/src/main/resources/bankset.f | 10 - .../src/main/resources/bankset_interface.sv | 19 - .../src/main/resources/bankset_package.sv | 201 - .../bank/src/main/resources/bankset_tb.sv | 46 - .../mem-core/bank/src/main/resources/env.svh | 47 - .../bank/src/main/resources/interface.sv | 18 - .../mem-core/bank/src/main/resources/item.svh | 31 - .../bank/src/main/resources/package.sv | 20 - .../bank/src/main/resources/ref_model.svh | 26 - .../mem-core/bank/src/main/resources/test.svh | 106 - .../mem-core/bank/src/main/scala/Bank.scala | 68 - .../mem-core/bank/src/main/scala/Emit.scala | 15 - .../scala/framework/mem-core/cache/build.sc | 21 - .../mem-core/cache/configs/CacheParams.scala | 54 - .../cache/src/main/resources/cache.el | 113 - .../mem-core/cache/src/main/resources/cache.f | 8 - .../mem-core/cache/src/main/resources/env.svh | 62 - .../cache/src/main/resources/item.svh | 36 - .../cache/src/main/resources/package.sv | 35 - .../cache/src/main/resources/test.svh | 142 - .../mem-core/cache/src/main/scala/Cache.scala | 110 - .../mem-core/cache/src/main/scala/Emit.scala | 29 - .../cache/src/main/scala/Interface.scala | 42 - .../scala/framework/mem-core/chi/build.sc | 20 - .../mem-core/chi/src/csrc/src/lib.rs | 3 - .../mem-core/chi/src/csrc/src/rnf.rs | 110 - .../chi/src/main/resources/chi_profile.svh | 12 - .../src/main/resources/credit_interface.sv | 6 - .../chi/src/main/resources/credit_package.sv | 43 - .../chi/src/main/resources/credit_tb.sv | 39 - .../mem-core/chi/src/main/resources/link.el | 45 - .../mem-core/chi/src/main/resources/link.f | 8 - .../chi/src/main/resources/link_interface.sv | 9 - .../chi/src/main/resources/link_package.sv | 128 - .../chi/src/main/resources/link_tb.sv | 38 - .../mem-core/chi/src/main/resources/retry.el | 415 -- .../mem-core/chi/src/main/resources/retry.f | 7 - .../chi/src/main/resources/retry_bad_opcode.f | 9 - .../src/main/resources/retry_bad_opcode.toml | 1 - .../main/resources/retry_duplicate_txnid.f | 9 - .../main/resources/retry_duplicate_txnid.toml | 1 - .../chi/src/main/resources/retry_interface.sv | 21 - .../chi/src/main/resources/retry_package.sv | 480 --- .../chi/src/main/resources/retry_tb.sv | 104 - .../mem-core/chi/src/main/resources/rnf.f | 7 - .../chi/src/main/resources/rnf_bad_dbid.f | 9 - .../chi/src/main/resources/rnf_bad_dbid.toml | 1 - .../src/main/resources/rnf_bad_dbid_width.f | 9 - .../main/resources/rnf_bad_dbid_width.toml | 1 - .../chi/src/main/resources/rnf_bad_resp.f | 9 - .../chi/src/main/resources/rnf_bad_resp.toml | 1 - .../chi/src/main/resources/rnf_interface.sv | 58 - .../chi/src/main/resources/rnf_package.sv | 110 - .../mem-core/chi/src/main/resources/rnf_tb.sv | 155 - .../chi/src/main/resources/rnf_test.svh | 788 ---- .../mem-core/chi/src/main/resources/rx.el | 78 - .../mem-core/chi/src/main/resources/rx.f | 8 - .../chi/src/main/resources/rx_bad_credit.f | 9 - .../chi/src/main/resources/rx_bad_credit.toml | 1 - .../mem-core/chi/src/main/resources/rx_tb.sv | 38 - .../mem-core/chi/src/main/resources/sram.el | 971 ----- .../mem-core/chi/src/main/resources/sram.f | 7 - .../mem-core/chi/src/main/resources/sram128.f | 10 - .../mem-core/chi/src/main/resources/sram512.f | 10 - .../chi/src/main/resources/sram_bad_opcode.f | 9 - .../src/main/resources/sram_bad_opcode.toml | 1 - .../src/main/resources/sram_duplicate_data.f | 9 - .../main/resources/sram_duplicate_data.toml | 1 - .../chi/src/main/resources/sram_interface.sv | 16 - .../chi/src/main/resources/sram_package.sv | 20 - .../chi/src/main/resources/sram_tb.sv | 44 - .../chi/src/main/resources/sram_test.svh | 499 --- .../chi/src/main/resources/tx_bad_credit.f | 9 - .../chi/src/main/resources/tx_bad_credit.toml | 1 - .../mem-core/chi/src/main/scala/Emit.scala | 42 - .../mem-core/chi/src/main/scala/Fabric.scala | 110 - .../framework/mem-core/coherence/build.sc | 39 - .../coherence/configs/CoherenceParams.scala | 53 - .../mem-core/coherence/src/csrc/Cargo.lock | 7 - .../mem-core/coherence/src/csrc/Cargo.toml | 7 - .../mem-core/coherence/src/csrc/src/memory.rs | 36 - .../coherence/src/main/resources/coherence.el | 316 -- .../coherence/src/main/resources/coherence.f | 9 - .../coherence/src/main/resources/env.svh | 393 -- .../coherence/src/main/resources/package.sv | 54 - .../coherence/src/main/resources/test.svh | 641 --- .../coherence/src/main/scala/Emit.scala | 110 - .../coherence/src/main/scala/Interface.scala | 19 - .../scala/framework/mem-core/cpu_mem/build.sc | 35 - .../mem-core/cpu_mem/src/csrc/Cargo.lock | 7 - .../mem-core/cpu_mem/src/csrc/Cargo.toml | 7 - .../mem-core/cpu_mem/src/csrc/src/lib.rs | 2 - .../cpu_mem/src/main/resources/cpu_mem.f | 9 - .../src/main/resources/cpu_mem_bad_atomic.f | 11 - .../main/resources/cpu_mem_bad_atomic.toml | 1 - .../main/resources/cpu_mem_bad_atomic_write.f | 11 - .../resources/cpu_mem_bad_atomic_write.toml | 1 - .../src/main/resources/cpu_mem_bad_fence.f | 11 - .../src/main/resources/cpu_mem_bad_fence.toml | 1 - .../src/main/resources/cpu_mem_bad_operand.f | 11 - .../main/resources/cpu_mem_bad_operand.toml | 1 - .../src/main/resources/cpu_mem_bad_size.f | 11 - .../src/main/resources/cpu_mem_bad_size.toml | 1 - .../src/main/resources/cpu_mem_bad_tag.f | 11 - .../src/main/resources/cpu_mem_bad_tag.toml | 1 - .../cpu_mem/src/main/resources/cpu_mem_tb.sv | 89 - .../cpu_mem/src/main/resources/package.sv | 36 - .../cpu_mem/src/main/resources/test.svh | 350 -- .../cpu_mem/src/main/scala/CpuMem.scala | 135 - .../cpu_mem/src/main/scala/Emit.scala | 92 - .../src/main/scala/VirtualMemory.scala | 306 -- .../scala/framework/mem-core/ddr/build.sc | 35 - .../mem-core/ddr/src/csrc/src/lib.rs | 2 - .../ddr/src/main/resources/bad_alignment.f | 10 - .../ddr/src/main/resources/bad_alignment.toml | 2 - .../ddr/src/main/resources/bad_early_b.f | 10 - .../ddr/src/main/resources/bad_early_b.toml | 2 - .../ddr/src/main/resources/bad_early_rlast.f | 10 - .../src/main/resources/bad_early_rlast.toml | 2 - .../ddr/src/main/resources/bad_late_rlast.f | 10 - .../src/main/resources/bad_late_rlast.toml | 2 - .../ddr/src/main/resources/bad_read_exokay.f | 10 - .../src/main/resources/bad_read_exokay.toml | 2 - .../mem-core/ddr/src/main/resources/bad_rid.f | 10 - .../ddr/src/main/resources/bad_rid.toml | 2 - .../ddr/src/main/resources/bad_write_exokay.f | 10 - .../src/main/resources/bad_write_exokay.toml | 2 - .../mem-core/ddr/src/main/resources/bridge.f | 9 - .../ddr/src/main/resources/bridge256.f | 11 - .../ddr/src/main/resources/bridge512.f | 11 - .../ddr/src/main/resources/bridge64.f | 11 - .../ddr/src/main/resources/bridge_six.f | 11 - .../ddr/src/main/resources/bridge_tb.sv | 71 - .../ddr/src/main/resources/package.sv | 599 --- .../ddr/src/main/resources/profile.svh | 20 - .../mem-core/ddr/src/main/scala/Emit.scala | 68 - .../mem-core/ddr/src/main/scala/Params.scala | 20 - .../scala/framework/mem-core/fetch/build.sc | 10 - .../fetch/src/main/resources/fetch.el | 50 - .../mem-core/fetch/src/main/resources/fetch.f | 9 - .../fetch/src/main/resources/fetch_bad_ack.f | 11 - .../src/main/resources/fetch_bad_ack.toml | 1 - .../src/main/resources/fetch_bad_flush.f | 11 - .../src/main/resources/fetch_bad_flush.toml | 1 - .../src/main/resources/fetch_bad_privilege.f | 11 - .../main/resources/fetch_bad_privilege.toml | 1 - .../src/main/resources/fetch_bad_redirect.f | 11 - .../main/resources/fetch_bad_redirect.toml | 1 - .../src/main/resources/fetch_bad_reset.f | 11 - .../src/main/resources/fetch_bad_reset.toml | 1 - .../fetch/src/main/resources/fetch_tb.sv | 87 - .../fetch/src/main/resources/interface.sv | 14 - .../fetch/src/main/resources/package.sv | 20 - .../fetch/src/main/resources/test.svh | 410 -- .../mem-core/fetch/src/main/scala/Emit.scala | 72 - .../framework/mem-core/interlock/build.sc | 10 - .../mem-core/interlock/src/csrc/Cargo.lock | 7 - .../mem-core/interlock/src/csrc/Cargo.toml | 6 - .../mem-core/interlock/src/csrc/src/lib.rs | 2 - .../interlock/src/csrc/src/ownership.rs | 99 - .../src/main/resources/control_if.sv | 5 - .../src/main/resources/interlock.cm_hier | 9 - .../interlock/src/main/resources/interlock.f | 9 - .../interlock_bad_cancel_info.cm_hier | 9 - .../resources/interlock_bad_cancel_info.f | 9 - .../resources/interlock_bad_cancel_info.toml | 1 - .../interlock_bad_cancel_info_scenario.svh | 8 - .../resources/interlock_bad_cancel_info_tb.sv | 3 - .../interlock_bad_cancel_sealed.cm_hier | 9 - .../resources/interlock_bad_cancel_sealed.f | 9 - .../interlock_bad_cancel_sealed.toml | 1 - .../interlock_bad_cancel_sealed_scenario.svh | 6 - .../interlock_bad_cancel_sealed_tb.sv | 3 - .../interlock_bad_cancel_unknown.cm_hier | 9 - .../resources/interlock_bad_cancel_unknown.f | 9 - .../interlock_bad_cancel_unknown.toml | 1 - .../interlock_bad_cancel_unknown_scenario.svh | 4 - .../interlock_bad_cancel_unknown_tb.sv | 3 - .../resources/interlock_bad_capacity.cm_hier | 9 - .../main/resources/interlock_bad_capacity.f | 9 - .../resources/interlock_bad_capacity.toml | 1 - .../interlock_bad_capacity_scenario.svh | 3 - .../resources/interlock_bad_capacity_tb.sv | 3 - .../interlock_bad_cpu_alignment.cm_hier | 9 - .../resources/interlock_bad_cpu_alignment.f | 9 - .../interlock_bad_cpu_alignment.toml | 1 - .../interlock_bad_cpu_alignment_scenario.svh | 5 - .../interlock_bad_cpu_alignment_tb.sv | 3 - .../resources/interlock_bad_cpu_size.cm_hier | 9 - .../main/resources/interlock_bad_cpu_size.f | 9 - .../resources/interlock_bad_cpu_size.toml | 1 - .../interlock_bad_cpu_size_scenario.svh | 4 - .../resources/interlock_bad_cpu_size_tb.sv | 3 - .../resources/interlock_bad_direction.cm_hier | 9 - .../main/resources/interlock_bad_direction.f | 9 - .../resources/interlock_bad_direction.toml | 1 - .../interlock_bad_direction_scenario.svh | 4 - .../resources/interlock_bad_direction_tb.sv | 3 - .../interlock_bad_dma_failed.cm_hier | 9 - .../main/resources/interlock_bad_dma_failed.f | 9 - .../resources/interlock_bad_dma_failed.toml | 1 - .../interlock_bad_dma_failed_scenario.svh | 15 - .../resources/interlock_bad_dma_failed_tb.sv | 3 - .../interlock_bad_dma_ungranted.cm_hier | 9 - .../resources/interlock_bad_dma_ungranted.f | 9 - .../interlock_bad_dma_ungranted.toml | 1 - .../interlock_bad_dma_ungranted_scenario.svh | 7 - .../interlock_bad_dma_ungranted_tb.sv | 3 - .../interlock_bad_dma_unknown.cm_hier | 9 - .../resources/interlock_bad_dma_unknown.f | 9 - .../resources/interlock_bad_dma_unknown.toml | 1 - .../interlock_bad_dma_unknown_scenario.svh | 5 - .../resources/interlock_bad_dma_unknown_tb.sv | 3 - .../resources/interlock_bad_duplicate.cm_hier | 9 - .../main/resources/interlock_bad_duplicate.f | 9 - .../resources/interlock_bad_duplicate.toml | 1 - .../interlock_bad_duplicate_scenario.svh | 5 - .../resources/interlock_bad_duplicate_tb.sv | 3 - .../interlock_bad_empty_range.cm_hier | 9 - .../resources/interlock_bad_empty_range.f | 9 - .../resources/interlock_bad_empty_range.toml | 1 - .../interlock_bad_empty_range_scenario.svh | 3 - .../resources/interlock_bad_empty_range_tb.sv | 3 - .../interlock_bad_info_duplicate.cm_hier | 9 - .../resources/interlock_bad_info_duplicate.f | 9 - .../interlock_bad_info_duplicate.toml | 1 - .../interlock_bad_info_duplicate_scenario.svh | 5 - .../interlock_bad_info_duplicate_tb.sv | 3 - .../interlock_bad_info_unknown.cm_hier | 9 - .../resources/interlock_bad_info_unknown.f | 9 - .../resources/interlock_bad_info_unknown.toml | 1 - .../interlock_bad_info_unknown_scenario.svh | 4 - .../interlock_bad_info_unknown_tb.sv | 3 - .../interlock_bad_maint_failed.cm_hier | 9 - .../resources/interlock_bad_maint_failed.f | 9 - .../resources/interlock_bad_maint_failed.toml | 1 - .../interlock_bad_maint_failed_scenario.svh | 5 - .../interlock_bad_maint_failed_tb.sv | 3 - .../resources/interlock_bad_maint_tag.cm_hier | 9 - .../main/resources/interlock_bad_maint_tag.f | 9 - .../resources/interlock_bad_maint_tag.toml | 1 - .../interlock_bad_maint_tag_scenario.svh | 5 - .../resources/interlock_bad_maint_tag_tb.sv | 3 - .../interlock_bad_maint_unknown.cm_hier | 9 - .../resources/interlock_bad_maint_unknown.f | 9 - .../interlock_bad_maint_unknown.toml | 1 - .../interlock_bad_maint_unknown_scenario.svh | 2 - .../interlock_bad_maint_unknown_tb.sv | 3 - .../interlock_bad_no_memory_seal.cm_hier | 9 - .../resources/interlock_bad_no_memory_seal.f | 9 - .../interlock_bad_no_memory_seal.toml | 1 - .../interlock_bad_no_memory_seal_scenario.svh | 3 - .../interlock_bad_no_memory_seal_tb.sv | 3 - .../interlock_bad_overflow_range.cm_hier | 9 - .../resources/interlock_bad_overflow_range.f | 9 - .../interlock_bad_overflow_range.toml | 1 - .../interlock_bad_overflow_range_scenario.svh | 3 - .../interlock_bad_overflow_range_tb.sv | 3 - .../src/main/resources/interlock_tb.sv | 743 ---- .../src/main/resources/negative_tb.svh | 136 - .../interlock/src/main/resources/package.sv | 84 - .../interlock/src/main/scala/Emit.scala | 60 - .../interlock/src/main/scala/Interface.scala | 38 - .../interlock/src/main/scala/Params.scala | 17 - .../framework/mem-core/mesh_shm/build.sc | 28 - .../mesh_shm/src/main/resources/router.f | 7 - .../src/main/resources/router_package.sv | 13 - .../mesh_shm/src/main/resources/shared.f | 8 - .../mesh_shm/src/main/scala/Emit.scala | 15 - .../mesh_shm/src/main/scala/MeshChannel.scala | 10 - .../src/main/scala/MeshClientEndpoint.scala | 110 - .../src/main/scala/MeshControlInterface.scala | 53 - .../src/main/scala/MeshCoreEndpoint.scala | 59 - .../src/main/scala/MeshSharedMem.scala | 246 -- .../main/scala/MeshSharedMemInterface.scala | 37 - .../src/main/scala/MeshSharedMemParams.scala | 151 - .../scala/framework/mem-core/mmu/build.sc | 30 - .../mem-core/mmu/src/csrc/src/translation.rs | 209 - .../mem-core/mmu/src/main/resources/walker.f | 9 - .../src/main/resources/walker_bad_access.f | 9 - .../src/main/resources/walker_bad_access.toml | 1 - .../main/resources/walker_bad_access_tb.sv | 44 - .../mmu/src/main/resources/walker_bad_mode.f | 9 - .../src/main/resources/walker_bad_mode.toml | 1 - .../src/main/resources/walker_bad_mode_tb.sv | 44 - .../src/main/resources/walker_bad_privilege.f | 9 - .../main/resources/walker_bad_privilege.toml | 1 - .../main/resources/walker_bad_privilege_tb.sv | 44 - .../src/main/resources/walker_interface.sv | 7 - .../main/resources/walker_negative_package.sv | 64 - .../mmu/src/main/resources/walker_package.sv | 458 --- .../mmu/src/main/resources/walker_tb.sv | 43 - .../mem-core/mmu/src/main/scala/Emit.scala | 58 - .../framework/mem-core/preflight/build.sc | 35 - .../preflight/src/main/resources/package.sv | 543 --- .../preflight/src/main/resources/preflight.f | 9 - .../src/main/resources/preflight64.f | 10 - .../src/main/resources/preflight_tb.sv | 59 - .../src/main/resources/prepared_map.cm_hier | 5 - .../src/main/resources/prepared_map.f | 9 - .../main/resources/prepared_map_control_if.sv | 12 - .../main/resources/prepared_map_package.sv | 336 -- .../src/main/resources/prepared_map_tb.sv | 42 - .../preflight/src/main/scala/Emit.scala | 94 - .../preflight/src/main/scala/Interface.scala | 50 - .../preflight/src/main/scala/Params.scala | 15 - .../scala/framework/mem-core/queue/build.sc | 10 - .../spm/src/main/resources/interface.sv | 81 - .../spm/src/main/resources/package.sv | 33 - .../spm/src/main/scala/Interfaces.scala | 35 - .../mem-core/spm/src/main/scala/Memory.scala | 59 - .../mem-core/tls/src/csrc/Cargo.lock | 7 - .../mem-core/tls/src/csrc/Cargo.toml | 8 - .../mem-core/tls/src/main/resources/tls.f | 10 - .../mem-core/tls/src/main/resources/tls_tb.sv | 121 - .../mem-core/tls/src/main/scala/Emit.scala | 18 - .../mem-core/tls/src/main/scala/Store.scala | 48 - .../mem-core/tss/src/csrc/Cargo.lock | 7 - .../mem-core/tss/src/csrc/Cargo.toml | 8 - .../tss/src/main/resources/tss.cm_hier | 2 - .../mem-core/tss/src/main/resources/tss.f | 10 - .../mem-core/tss/src/main/resources/tss.toml | 1 - .../mem-core/tss/src/main/resources/tss_tb.sv | 151 - .../mem-core/tss/src/main/scala/Emit.scala | 18 - .../mem-core/tss/src/main/scala/Store.scala | 86 - .../scala/framework/memdomain/MemDomain.scala | 332 -- .../framework/memdomain/backend/MTrace.scala | 115 - .../memdomain/backend/MemBackend.scala | 292 -- .../memdomain/backend/accpipe/AccPipe.scala | 95 - .../memdomain/backend/banks/SramBank.scala | 91 - .../memdomain/backend/banks/SramIO.scala | 36 - .../backend/banks/btrace/BTraceDPI.scala | 91 - .../memdomain/backend/mmio/MmioBank.scala | 62 - .../memdomain/backend/mmio/MmioIO.scala | 51 - .../memdomain/backend/mmio/MmioPool.scala | 165 - .../memdomain/backend/mmio/MmioRouter.scala | 137 - .../privatepath/PrivateMemBackend.scala | 440 --- .../backend/shared/PhysicalPort.scala | 23 - .../backend/shared/SharedLease.scala | 17 - .../memdomain/backend/shared/SharedMem.scala | 39 - .../backend/shared/SharedMemBackend.scala | 588 --- .../framework/memdomain/boot/MemBoot.scala | 16 - .../memdomain/configs/MemDomainParam.scala | 89 - .../memdomain/frontend/MemFrontend.scala | 263 -- .../memdomain/frontend/cmd/decoder/DISA.scala | 17 - .../frontend/cmd/decoder/DomainDecoder.scala | 162 - .../memdomain/frontend/cmd/rs/README.md | 54 - .../frontend/cmd/rs/reservationStation.scala | 136 - .../memdomain/frontend/mem/Footprint.scala | 21 - .../memdomain/frontend/mem/KernelDma.scala | 145 - .../memdomain/frontend/mem/MemConfiger.scala | 141 - .../memdomain/frontend/mem/MemLoader.scala | 303 -- .../memdomain/frontend/mem/MemStorer.scala | 212 - .../memdomain/frontend/mem/dma/Bundles.scala | 38 - .../memdomain/frontend/mem/dma/Dma.scala | 69 - .../memdomain/frontend/mem/dma/DmaPort.scala | 21 - .../framework/memdomain/isa/MemISA.scala | 31 - .../framework/memdomain/isa/MvoverIO.scala | 27 - .../memdomain/midend/MemMidend.scala | 277 -- .../memdomain/utils/pmc/MemCyclePMC.scala | 79 - .../memdomain/utils/pmc/MemPMCTraceDPI.scala | 64 - .../memdomain/verification/.gitignore | 4 - .../verification/src/csrc/Cargo.lock | 268 -- .../verification/src/csrc/Cargo.toml | 11 - .../verification/src/csrc/src/lib.rs | 12 - .../verification/src/csrc/src/memory.rs | 18 - .../verification/src/csrc/src/tile_peer.rs | 147 - .../src/main/resources/ack_package.sv | 17 - .../main/resources/admission_system.cm_hier | 3 - .../src/main/resources/admission_system.f | 8 - .../src/main/resources/admission_system.toml | 1 - .../src/main/resources/admission_system_tb.sv | 479 --- .../main/resources/controller_system.cm_hier | 4 - .../src/main/resources/controller_system.f | 8 - .../src/main/resources/controller_system.toml | 1 - .../main/resources/controller_system_tb.sv | 305 -- .../src/main/resources/kernel_dma.cm_hier | 4 - .../src/main/resources/kernel_dma.f | 8 - .../src/main/resources/kernel_dma_tb.sv | 348 -- .../src/main/resources/loader_ack.f | 9 - .../src/main/resources/loader_ack_tb.sv | 412 -- .../src/main/resources/memory_system.f | 7 - .../src/main/resources/memory_system.toml | 1 - .../src/main/resources/memory_system_tb.sv | 355 -- .../src/main/resources/read_dma.cm_hier | 1 - .../src/main/resources/read_dma.f | 7 - .../src/main/resources/read_dma_tb.sv | 185 - .../src/main/resources/shared_lease.cm_hier | 1 - .../src/main/resources/shared_lease.f | 7 - .../src/main/resources/shared_lease_tb.sv | 163 - .../src/main/resources/storer_ack.f | 9 - .../src/main/resources/storer_ack_tb.sv | 366 -- .../src/main/resources/tile_axi_peer.svh | 103 - .../main/resources/tile_system_four.cm_hier | 1 - .../src/main/resources/tile_system_four.f | 7 - .../src/main/resources/tile_system_four.toml | 1 - .../resources/tile_system_four_config.svh | 2 - .../src/main/resources/tile_system_four_tb.sv | 4 - .../main/resources/tile_system_single.cm_hier | 1 - .../src/main/resources/tile_system_single.f | 7 - .../main/resources/tile_system_single.toml | 1 - .../resources/tile_system_single_config.svh | 2 - .../main/resources/tile_system_single_tb.sv | 4 - .../src/main/resources/tile_system_tb.svh | 318 -- .../src/main/scala/AdmissionSystem.scala | 117 - .../src/main/scala/ControllerSystem.scala | 107 - .../verification/src/main/scala/Emit.scala | 321 -- .../src/main/scala/EmitController.scala | 62 - .../src/main/scala/EmitMemory.scala | 54 - .../src/main/scala/EmitTile.scala | 88 - .../src/main/scala/TileSystem.scala | 34 - arch/src/main/scala/framework/root/.gitignore | 4 - arch/src/main/scala/framework/root/README.md | 2 - .../scala/framework/root/root-chip/build.sc | 43 - .../root/root-chip/src/csrc/Cargo.lock | 7 - .../root/root-chip/src/csrc/Cargo.toml | 6 - .../root/root-chip/src/csrc/src/lib.rs | 2 - .../root/root-chip/src/csrc/src/routing.rs | 4 - .../root-chip/src/main/resources/bad_node.f | 7 - .../src/main/resources/bad_node.toml | 1 - .../src/main/resources/bad_node_package.sv | 30 - .../src/main/resources/bad_node_tb.sv | 28 - .../src/main/resources/bad_requester_req.f | 7 - .../src/main/resources/bad_requester_req.toml | 1 - .../resources/bad_requester_req_package.sv | 57 - .../main/resources/bad_requester_req_tb.sv | 126 - .../root-chip/src/main/resources/codec.el | 180 - .../root/root-chip/src/main/resources/codec.f | 7 - .../src/main/resources/codec_interface.sv | 8 - .../src/main/resources/codec_package.sv | 124 - .../root-chip/src/main/resources/codec_tb.sv | 28 - .../src/main/resources/fabric_interface.sv | 11 - .../root-chip/src/main/resources/network.f | 7 - .../src/main/resources/network_package.sv | 386 -- .../src/main/resources/network_tb.sv | 288 -- .../root-chip/src/main/resources/router.el | 502 --- .../root-chip/src/main/resources/router.f | 7 - .../src/main/resources/router_package.sv | 211 - .../root-chip/src/main/resources/router_tb.sv | 355 -- .../src/main/scala/chip/mesh/ChiMesh.scala | 475 --- .../src/main/scala/chip/mesh/Emit.scala | 35 - .../src/main/scala/chip/mesh/Mesh.scala | 126 - .../src/main/scala/chip/mesh/MeshBulk.scala | 192 - .../src/main/scala/chip/mesh/MeshCredit.scala | 165 - .../chip/mesh/ScratchpadBulkWriteSink.scala | 50 - .../scala/chip/mesh/TileMeshEndpoint.scala | 95 - .../root/root-core/src/csrc/Cargo.lock | 7 - .../root/root-core/src/csrc/Cargo.toml | 6 - .../root/root-core/src/csrc/src/lib.rs | 2 - .../root/root-core/src/csrc/src/memory.rs | 43 - .../root-core/src/main/resources/admission.S | 84 - .../src/main/resources/admission.cm_hier | 12 - .../root-core/src/main/resources/admission.f | 10 - .../root-core/src/main/resources/admission.ld | 18 - .../src/main/resources/admission_bridge.f | 9 - .../admission_bridge_bad_late_fault.f | 11 - .../admission_bridge_bad_late_fault.toml | 1 - .../resources/admission_bridge_control.sv | 17 - .../resources/admission_bridge_package.sv | 354 -- .../src/main/resources/admission_bridge_tb.sv | 43 - .../src/main/resources/admission_control.sv | 13 - .../src/main/resources/admission_package.sv | 40 - .../src/main/resources/admission_tb.sv | 605 --- .../src/main/resources/build_fixture.py | 108 - .../root/root-core/src/main/resources/core.S | 203 - .../root-core/src/main/resources/core.cm_hier | 5 - .../root/root-core/src/main/resources/core.f | 9 - .../root/root-core/src/main/resources/core.ld | 14 - .../src/main/resources/core_interface.sv | 17 - .../src/main/resources/core_package.sv | 34 - .../root-core/src/main/resources/core_tb.sv | 85 - .../src/main/resources/core_test.svh | 321 -- .../root/root-core/src/main/resources/fpu.S | 148 - .../root/root-core/src/main/resources/fpu.f | 10 - .../root/root-core/src/main/resources/fpu.ld | 14 - .../root/root-core/src/main/resources/irq.S | 97 - .../root-core/src/main/resources/irq.cm_hier | 5 - .../root/root-core/src/main/resources/irq.f | 10 - .../root/root-core/src/main/resources/irq.ld | 14 - .../root/root-core/src/main/resources/lsu.f | 9 - .../src/main/resources/lsu_interface.sv | 14 - .../src/main/resources/lsu_package.sv | 8 - .../root-core/src/main/resources/lsu_tb.sv | 59 - .../root-core/src/main/resources/lsu_test.svh | 193 - .../src/main/resources/maintenance.cm_hier | 5 - .../src/main/resources/maintenance.f | 9 - .../maintenance_bad_response.cm_hier | 5 - .../main/resources/maintenance_bad_response.f | 11 - .../resources/maintenance_bad_response.toml | 1 - .../main/resources/maintenance_interface.sv | 10 - .../src/main/resources/maintenance_package.sv | 166 - .../src/main/resources/maintenance_tb.sv | 46 - .../root-core/src/main/resources/multicore.S | 108 - .../root-core/src/main/resources/multicore.ld | 14 - .../src/main/resources/permission.cm_hier | 1 - .../root-core/src/main/resources/permission.f | 5 - .../src/main/resources/permission_tb.sv | 216 - .../root-core/src/main/resources/supervisor.S | 221 -- .../src/main/resources/supervisor.cm_hier | 5 - .../root-core/src/main/resources/supervisor.f | 13 - .../src/main/resources/supervisor.ld | 20 - .../src/main/scala/rocket/Emit.scala | 481 --- .../scala/rocket/PermissionVerification.scala | 39 - .../src/main/scala/rocket/Verification.scala | 98 - .../root/root-tile/src/csrc/Cargo.lock | 7 - .../root/root-tile/src/csrc/Cargo.toml | 8 - .../root-tile/src/csrc/src/consistency.rs | 80 - .../root/root-tile/src/csrc/src/lib.rs | 15 - .../src/main/resources/cache_system.f | 9 - .../main/resources/cache_system_interface.sv | 35 - .../main/resources/cache_system_package.sv | 606 --- .../src/main/resources/cache_system_tb.sv | 32 - .../src/main/resources/consistency.cm_hier | 12 - .../src/main/resources/consistency.f | 9 - .../src/main/resources/consistency_control.sv | 13 - .../src/main/resources/consistency_package.sv | 66 - .../src/main/resources/consistency_tb.sv | 564 --- .../root/root-tile/src/main/resources/tile.f | 9 - .../src/main/resources/tile_interface.sv | 20 - .../src/main/resources/tile_package.sv | 415 -- .../root-tile/src/main/resources/tile_tb.sv | 58 - .../src/main/resources/virtual_cache.f | 9 - .../main/resources/virtual_cache_bad_atomic.f | 10 - .../resources/virtual_cache_bad_atomic.toml | 2 - .../virtual_cache_bad_atomic_write.f | 10 - .../virtual_cache_bad_atomic_write.toml | 2 - .../virtual_cache_bad_execute_atomic.f | 10 - .../virtual_cache_bad_execute_atomic.toml | 2 - .../virtual_cache_bad_execute_signed.f | 10 - .../virtual_cache_bad_execute_signed.toml | 2 - .../virtual_cache_bad_execute_size.f | 10 - .../virtual_cache_bad_execute_size.toml | 2 - .../virtual_cache_bad_execute_write.f | 10 - .../virtual_cache_bad_execute_write.toml | 2 - .../main/resources/virtual_cache_bad_mode.f | 10 - .../resources/virtual_cache_bad_mode.toml | 2 - .../resources/virtual_cache_bad_operand.f | 10 - .../resources/virtual_cache_bad_operand.toml | 2 - .../resources/virtual_cache_bad_privilege.f | 10 - .../virtual_cache_bad_privilege.toml | 2 - .../main/resources/virtual_cache_bad_size.f | 10 - .../resources/virtual_cache_bad_size.toml | 2 - .../main/resources/virtual_cache_interface.sv | 18 - .../main/resources/virtual_cache_package.sv | 1127 ------ .../src/main/resources/virtual_cache_tb.sv | 102 - .../src/main/scala/TaskAdmission.scala | 55 - .../src/main/scala/TaskController.scala | 108 - .../src/main/scala/memory/CacheSystem.scala | 59 - .../src/main/scala/memory/Composition.scala | 175 - .../src/main/scala/memory/Consistency.scala | 91 - .../src/main/scala/memory/Emit.scala | 468 --- .../src/main/scala/memory/Home.scala | 52 - .../src/main/scala/memory/Memory.scala | 30 - .../src/main/scala/memory/Tile.scala | 82 - .../scala/memory/VirtualCacheSystem.scala | 78 - arch/src/main/scala/framework/rvv/LICENSE | 201 - arch/src/main/scala/framework/rvv/README.md | 1 - arch/src/main/scala/framework/rvv/build.sc | 64 - .../scala/framework/rvv/src/csrc/Cargo.lock | 135 - .../scala/framework/rvv/src/csrc/Cargo.toml | 15 - .../scala/framework/rvv/src/csrc/build.rs | 62 - .../scala/framework/rvv/src/csrc/src/lib.rs | 2 - .../scala/framework/rvv/src/csrc/src/model.rs | 903 ----- .../rvv/src/main/resources/package.sv | 97 - .../framework/rvv/src/main/resources/test.svh | 265 -- .../rvv/src/main/resources/topkernel.f | 8 - .../framework/rvv/src/main/scala/Emit.scala | 35 - .../rvv/src/main/scala/configs/RvvParam.scala | 45 - .../rvv/src/main/scala/core/VectorCore.scala | 921 ----- .../main/scala/interfaces/KernelBlinkIO.scala | 41 - .../main/scala/interfaces/RvvInterface.scala | 88 - .../rvv/src/main/scala/memory/BankPort.scala | 109 - .../rvv/src/main/scala/top/Execution.scala | 354 -- .../rvv/src/main/scala/top/KernelEngine.scala | 241 -- .../main/scala/framework/system/Device.scala | 13 - .../framework/system/DeviceConnection.scala | 131 - .../framework/system/GlobalConfigOps.scala | 186 - .../main/scala/framework/system/System.scala | 31 - .../system/configloader/ChipLoader.scala | 499 --- .../system/configloader/Topology.scala | 50 - .../framework/system/core/AntEndpoint.scala | 19 - .../system/core/ControllerAdmission.scala | 104 - .../system/core/CoreConnection.scala | 80 - .../framework/system/core/CoreFactory.scala | 24 - .../framework/system/core/CoreParams.scala | 15 - .../system/core/RocketEndpoint.scala | 49 - .../system/core/accelerator/Admission.scala | 420 -- .../core/accelerator/AntAdmission.scala | 147 - .../system/core/accelerator/AntMemory.scala | 73 - .../accelerator/BuckyballAccelerator.scala | 219 -- .../system/core/accelerator/Rvv.scala | 120 - .../system/core/ant/src/csrc/src/lib.rs | 20 - .../system/core/ant/src/main/resources/ant.f | 10 - .../core/ant/src/main/resources/ant_tb.sv | 314 -- .../core/ant/src/main/resources/control.f | 9 - .../core/ant/src/main/resources/control_tb.sv | 141 - .../core/ant/src/main/resources/service.f | 9 - .../core/ant/src/main/resources/service_tb.sv | 160 - .../system/core/ant/src/main/scala/Ant.scala | 282 -- .../core/ant/src/main/scala/Control.scala | 143 - .../system/core/ant/src/main/scala/Emit.scala | 69 - .../core/ant/src/main/scala/Execution.scala | 55 - .../core/ant/src/main/scala/Interfaces.scala | 70 - .../ant/src/main/scala/LocalExecution.scala | 42 - .../core/ant/src/main/scala/Service.scala | 102 - .../framework/system/core/clink/Shm.scala | 30 - .../framework/system/core/clink/base.scala | 7 - .../framework/system/core/clink/clink.scala | 69 - .../system/core/rocket/BuckyballCommand.scala | 13 - .../system/core/rocket/CoreParameters.scala | 175 - .../system/core/rocket/CpuInterfaces.scala | 170 - .../system/core/rocket/CpuParams.scala | 15 - .../system/core/rocket/Parameters.scala | 24 - .../system/core/rocket/RoCCCommandBB.scala | 36 - .../system/core/rocket/RocketCoreBB.scala | 1486 ------- .../core/rocket/RocketCoreParameters.scala | 34 - .../system/core/rocket/TileInterrupts.scala | 21 - .../system/core/rocket/TileParameters.scala | 78 - .../system/core/rocket/VectorInterfaces.scala | 53 - .../core/rocket/configs/RocketCpuParam.scala | 378 -- .../src/main/scala/framework/system/core/seed | 1 - .../framework/system/device/BootRom.scala | 86 - .../scala/framework/system/device/Clint.scala | 86 - .../framework/system/device/Devices.scala | 101 - .../scala/framework/system/dlink/dlink.scala | 11 - .../framework/system/memory/Memory.scala | 33 - .../framework/system/memory/UncachedAxi.scala | 73 - .../framework/system/tile/BankNetwork.scala | 113 - .../framework/system/tile/BarrierUnit.scala | 35 - .../system/tile/ComputeTilePlatform.scala | 228 -- .../framework/system/tile/MainTile.scala | 15 - .../system/tile/MainTilePlatform.scala | 406 -- .../system/tile/TileConnection.scala | 30 - .../framework/system/tile/TileEndpoint.scala | 74 - .../framework/system/tile/TileFactory.scala | 18 - .../system/tile/TileParameters.scala | 34 - .../system/tile/configs/TileParam.scala | 27 - .../framework/system/tile/tlink/Control.scala | 129 - .../framework/system/tile/tlink/Fabric.scala | 80 - .../framework/system/tile/tlink/T2T.scala | 127 - .../framework/system/tile/tlink/tlink.scala | 54 - .../main/scala/framework/thread/ant/build.sc | 25 + .../thread/ant/configs/AntParam.scala | 35 + .../framework/thread/ant/configs/default.toml | 7 + .../core => thread}/ant/src/csrc/Cargo.lock | 0 .../core => thread}/ant/src/csrc/Cargo.toml | 0 .../framework/thread/ant/src/csrc/src/lib.rs | 20 + .../ant/src/main/resources/ant.cm_hier | 0 .../thread/ant/src/main/resources/ant.f | 10 + .../thread/ant/src/main/resources/ant_tb.sv | 315 ++ .../ant/src/main/resources/control.cm_hier | 0 .../thread/ant/src/main/resources/control.f | 9 + .../ant/src/main/resources/control_if.sv | 0 .../ant/src/main/resources/control_tb.sv | 176 + .../ant/src/main/resources/service.cm_hier | 0 .../thread/ant/src/main/resources/service.f | 9 + .../ant/src/main/resources/service_tb.sv | 153 + .../thread/ant/src/main/resources/suite.toml | 71 + .../ant}/src/main/resources/tls.cm_hier | 0 .../thread/ant/src/main/resources/tls.f | 10 + .../ant}/src/main/resources/tls.toml | 0 .../thread/ant/src/main/resources/tls_tb.sv | 121 + .../thread/ant/src/main/scala/Ant.scala | 163 + .../ant/src/main/scala/Decode.scala | 5 +- .../thread/ant/src/main/scala/Emit.scala | 148 + .../thread/ant/src/main/scala/Executor.scala | 329 ++ .../ant/src/main/scala/LocalStore.scala | 61 + arch/src/main/scala/framework/thread/build.sc | 16 + .../thread/configs/ThreadParam.scala | 39 + .../framework/thread/tlink/basethread.scala | 9 + .../scala/framework/thread/tlink/build.sc | 12 + .../scala/framework/thread/tlink/tlink.scala | 82 + .../scala/framework/top/GlobalConfig.scala | 40 - .../framework/top/configs/DesignConfig.scala | 130 - .../framework/top/configs/SimParam.scala | 9 - arch/src/main/scala/sims/build.sc | 17 + .../main/scala/sims/configs/SimParam.scala | 31 + .../src/main/scala/sims/core/CoreTarget.scala | 71 + arch/src/main/scala/sims/hash/BankHash.scala | 75 - arch/src/main/scala/sims/p2e/P2ETop.scala | 77 +- .../main/scala/sims/p2e/TargetConfig.scala | 54 +- arch/src/main/scala/sims/scu/SCU.scala | 162 - .../main/scala/sims/scu/SystemControl.scala | 114 - arch/src/main/scala/sims/soc/SimSoc.scala | 109 +- .../main/scala/sims/verilator/BBsimDRAM.scala | 43 +- .../main/scala/sims/verilator/BdbClkDPI.scala | 2 + .../scala/sims/verilator/SystemHarness.scala | 17 +- .../scala/sims/verilator/TargetConfig.scala | 62 +- arch/thirdparty/build.sc | 101 + arch/thirdparty/rocket-chip | 1 - arch/uvm.toml | 300 +- bb-tests/CMakeLists.txt | 8 +- bb-tests/thirdparty/busybox | 1 - bb-tests/thirdparty/linux | 1 - bb-tests/thirdparty/opensbi | 1 - bb-tests/workloads/CMakeLists.txt | 100 +- bb-tests/workloads/common/CMakeLists.txt | 233 -- bb-tests/workloads/lib/CMakeLists.txt | 6 - bb-tests/workloads/lib/bbhw/CMakeLists.txt | 3 - bb-tests/workloads/lib/bbhw/isa/01_barrier.c | 10 +- bb-tests/workloads/lib/bbhw/isa/13_mvover.c | 12 +- bb-tests/workloads/lib/bbhw/isa/14_task.c | 52 - bb-tests/workloads/lib/bbhw/isa/16_mvout.c | 2 + bb-tests/workloads/lib/bbhw/isa/32_mset.c | 2 + bb-tests/workloads/lib/bbhw/isa/34_mvin_2d.c | 8 +- bb-tests/workloads/lib/bbhw/isa/bb_func7.h | 1 + bb-tests/workloads/lib/bbhw/isa/isa.c | 1 - bb-tests/workloads/lib/bbhw/isa/isa.h | 18 +- .../lib/bbsw/CRunnerUtils/CRunnerUtils.cpp | 75 - .../workloads/lib/bbsw/baremetal/bbsim.ld | 63 - bb-tests/workloads/lib/bbsw/baremetal/crt0.S | 71 - .../lib/bbsw/baremetal/multicore/multicore.h | 17 - bb-tests/workloads/lib/bbsw/baremetal/start.S | 100 - .../workloads/lib/bbsw/baremetal/syscalls.c | 18 - bb-tests/workloads/lib/bbsw/baremetal/test.c | 12 - .../workloads/lib/bbsw/kernels/CMakeLists.txt | 6 - .../lib/bbsw/kernels/mxfp8/CMakeLists.txt | 13 - .../workloads/lib/bbsw/kernels/mxfp8/pack.cpp | 69 - .../lib/bbsw/kernels/mxfp8/quant.cpp | 46 - .../workloads/lib/bbsw/kernels/mxfp8/quant.h | 5 - .../lib/bbsw/kernels/mxquant/CMakeLists.txt | 16 - .../lib/bbsw/kernels/mxquant/dispatch.cpp | 15 - .../lib/bbsw/kernels/rvv/CMakeLists.txt | 57 - .../bbsw/kernels/rvv/activation_dispatch.cpp | 86 - .../lib/bbsw/kernels/rvv/build_images.py | 198 - .../lib/bbsw/kernels/rvv/cache_dispatch.cpp | 179 - .../lib/bbsw/kernels/rvv/dequant.cpp | 73 - .../lib/bbsw/kernels/rvv/dispatch.cpp | 442 --- .../lib/bbsw/kernels/rvv/flash_attention.cpp | 131 - .../lib/bbsw/kernels/rvv/flash_attention.h | 15 - .../kernels/rvv/flash_attention_dispatch.cpp | 153 - .../kernels/rvv/flash_attention_mxfp8.cpp | 144 - .../rvv/flash_attention_mxfp8_dispatch.cpp | 187 - .../workloads/lib/bbsw/kernels/rvv/gelu.cpp | 19 - .../workloads/lib/bbsw/kernels/rvv/kernel.ld | 13 - .../workloads/lib/bbsw/kernels/rvv/kernels.h | 40 - .../lib/bbsw/kernels/rvv/layernorm.cpp | 52 - .../lib/bbsw/kernels/rvv/math/constants.h | 39 - .../lib/bbsw/kernels/rvv/math/cos.cpp | 99 - .../lib/bbsw/kernels/rvv/math/erf.cpp | 248 -- .../lib/bbsw/kernels/rvv/math/exp.cpp | 66 - .../lib/bbsw/kernels/rvv/math/math.h | 6 - .../lib/bbsw/kernels/rvv/math/mxfp8.h | 64 - .../lib/bbsw/kernels/rvv/math/sin.cpp | 107 - .../workloads/lib/bbsw/kernels/rvv/matmul.cpp | 19 - .../lib/bbsw/kernels/rvv/matmul_dispatch.cpp | 212 - .../workloads/lib/bbsw/kernels/rvv/norm.cpp | 69 - .../kernels/rvv/normalization_dispatch.cpp | 178 - .../workloads/lib/bbsw/kernels/rvv/pack.cpp | 19 - .../lib/bbsw/kernels/rvv/pointwise.cpp | 52 - .../bbsw/kernels/rvv/pointwise_dispatch.cpp | 145 - .../workloads/lib/bbsw/kernels/rvv/quant.cpp | 131 - .../lib/bbsw/kernels/rvv/quant_dispatch.cpp | 41 - .../workloads/lib/bbsw/kernels/rvv/rope.cpp | 72 - .../workloads/lib/bbsw/kernels/rvv/silu.cpp | 50 - .../workloads/lib/bbsw/kernels/rvv/snake.cpp | 27 - .../lib/bbsw/kernels/rvv/softmax.cpp | 183 - .../workloads/lib/bbsw/kernels/rvv/tanh.cpp | 54 - .../workloads/lib/bbsw/kernels/rvv/trig.cpp | 26 - .../lib/bbsw/linux/multicore/multicore.h | 20 - bb-tests/workloads/lib/bbsw/linux/test.c | 6 - bb-tests/workloads/lib/bbsw/test.h | 15 - bb-tests/workloads/lib/buckyball.h | 16 +- bb-tests/workloads/lib/kernel | 1 - bb-tests/workloads/scripts/build.py | 243 +- bb-tests/workloads/src/CMakeLists.txt | 33 +- bb-tests/workloads/src/CTest/CMakeLists.txt | 465 ++- .../workloads/src/CTest/base/CMakeLists.txt | 62 +- .../workloads/src/CTest/base/mvin_2d_test.c | 29 +- .../src/CTest/coremark/CMakeLists.txt | 100 +- .../src/CTest/coremark/buckyball/main.c | 15 +- .../src/CTest/coremark/src/core_state.c | 4 +- .../src/CTest/coremark/src/coremark.h | 2 +- .../src/CTest/embench/CMakeLists.txt | 356 +- .../src/CTest/embench/support/chipsupport.c | 10 +- .../src/CTest/embench/support/chipsupport.h | 6 +- bb-tests/workloads/src/CTest/rvv/rvv_kernel.c | 1 + .../workloads/src/CTest/tlb/CMakeLists.txt | 18 +- bb-tests/workloads/src/CTest/tlb/tlb_common.h | 12 +- .../src/CTest/tutorial/CMakeLists.txt | 24 +- .../workloads/src/MLIRTest/CMakeLists.txt | 529 ++- .../src/MLIRTest/base/CMakeLists.txt | 68 +- .../base/base_bank_mvin_mvout_16x16_i8.mlir | 12 +- .../base/base_memref_copy_dynamic.mlir | 15 + .../base/base_memref_copy_dynamic.mlir.in | 15 - .../base/base_mvin_mvout_16x16_i8.mlir | 11 +- .../src/MLIRTest/compiler/CMakeLists.txt | 24 - bb-tests/workloads/src/SocTest/CMakeLists.txt | 116 +- bb-tests/workloads/src/SocTest/soc.h | 2 +- bbdev/api/services/workflow_tools.py | 9 +- bbdev/api/steps/bebop/bemu/02_sim_api.step.py | 72 +- .../api/steps/bebop/bemu/02_sim_event.step.py | 270 +- .../api/steps/bebop/bemu/03_batch_api.step.py | 20 +- .../steps/bebop/bemu/03_batch_event.step.py | 56 +- .../steps/bebop/bemu/04_analysis_api.step.py | 28 +- .../bebop/bemu/04_analysis_event.step.py | 38 +- .../steps/bebop/bemu/scripts/bank_replay.py | 402 ++ .../bebop/bemu/scripts/bank_replay_native.py | 459 +++ .../steps/bebop/bemu/scripts/bemu_analysis.py | 109 +- .../steps/bebop/bemu/scripts/bemu_common.py | 25 +- .../api/steps/bebop/bemu/scripts/model_sim.py | 144 - .../api/steps/bebop/p2e/00_clean_api.step.py | 25 +- .../steps/bebop/p2e/00_clean_event.step.py | 18 +- .../steps/bebop/p2e/01_verilog_api.step.py | 18 +- .../bebop/p2e/02_buildbitstream_api.step.py | 36 +- .../bebop/p2e/02_buildbitstream_event.step.py | 116 +- .../bebop/p2e/03_runworkload_api.step.py | 46 +- .../bebop/p2e/03_runworkload_event.step.py | 149 +- .../api/steps/bebop/p2e/04_batch_api.step.py | 32 +- .../steps/bebop/p2e/04_batch_event.step.py | 60 +- .../steps/bebop/p2e/scripts/resolve_image.py | 8 +- bbdev/api/steps/bebop/performance_report.py | 285 +- .../steps/bebop/vcs/01_verilog_api.step.py | 20 +- .../api/steps/bebop/vcs/02_build_api.step.py | 26 +- .../api/steps/bebop/vcs/03_batch_api.step.py | 35 +- .../steps/bebop/vcs/03_batch_event.step.py | 123 +- bbdev/api/steps/bebop/vcs/04_sim_api.step.py | 26 + .../bebop/verilator/01_clean_api.step.py | 19 +- .../bebop/verilator/01_clean_event.step.py | 26 +- .../bebop/verilator/02_verilog_api.step.py | 19 +- .../bebop/verilator/03_build_api.step.py | 29 +- .../bebop/verilator/03_build_event.step.py | 74 +- .../steps/bebop/verilator/04_sim_api.step.py | 26 +- .../bebop/verilator/04_sim_event.step.py | 89 +- .../steps/bebop/verilator/05_run_api.step.py | 24 +- .../bebop/verilator/06_batch_api.step.py | 31 +- .../bebop/verilator/06_batch_event.step.py | 61 +- bbdev/api/steps/compiler/01_build_api.step.py | 30 - .../api/steps/compiler/01_build_event.step.py | 60 - bbdev/api/steps/compiler/scripts/build.py | 231 -- bbdev/api/steps/config/.gitignore | 2 +- bbdev/api/steps/config/01_install_api.step.py | 27 +- .../api/steps/config/01_install_event.step.py | 149 +- bbdev/api/steps/config/scripts/1_toml2json.py | 107 +- .../config/scripts/2_parameter_derivation.py | 628 +-- .../api/steps/config/scripts/3_json2proto.py | 689 ++-- bbdev/api/steps/config/scripts/4_install.py | 136 - .../steps/config/scripts/ball_normalize.py | 64 +- .../steps/config/scripts/ball_parameters.py | 31 + .../steps/config/scripts/boom_params_pb2.py | 34 + .../steps/config/scripts/f32add_params_pb2.py | 34 + .../config/scripts/gemmini_params_pb2.py | 34 + .../api/steps/config/scripts/hardware_pb2.py | 84 + .../steps/config/scripts/im2col_params_pb2.py | 34 + .../steps/config/scripts/int2fp_params_pb2.py | 34 + .../config/scripts/int8add_params_pb2.py | 34 + .../config/scripts/int8mul_params_pb2.py | 34 + .../steps/config/scripts/lut_params_pb2.py | 34 + .../steps/config/scripts/matadd_params_pb2.py | 34 + .../config/scripts/maxpool_params_pb2.py | 34 + .../config/scripts/mxfp2int_params_pb2.py | 34 + .../steps/config/scripts/mxmm_params_pb2.py | 34 + .../config/scripts/mxquant_params_pb2.py | 34 + .../api/steps/config/scripts/proto/chip.proto | 346 -- .../steps/config/scripts/proto/design.proto | 335 ++ .../steps/config/scripts/proto/hardware.proto | 158 + .../config/scripts/redmule_params_pb2.py | 34 + .../steps/config/scripts/relu_params_pb2.py | 34 + .../steps/config/scripts/rocket_params_pb2.py | 36 + .../config/scripts/smatmul_params_pb2.py | 34 + .../steps/config/scripts/toint8_params_pb2.py | 34 + .../steps/config/scripts/trace_params_pb2.py | 34 + .../config/scripts/transpose_params_pb2.py | 34 + .../steps/config/scripts/vector_params_pb2.py | 34 + .../api/steps/config/scripts/verification.py | 99 + .../config/scripts/xiangshan_params_pb2.py | 38 + bbdev/api/steps/dc/01_area_api.step.py | 12 +- bbdev/api/steps/dc/01_area_event.step.py | 58 +- bbdev/api/steps/dc/02_verilog_api.step.py | 12 +- bbdev/api/steps/dc/03_sim_event.step.py | 172 +- bbdev/api/steps/dc/04_power_api.step.py | 12 +- .../steps/dc/scripts/emit_seq_mem_models.py | 42 +- bbdev/api/steps/dc/scripts/gate_dpi.cc | 35 +- bbdev/api/steps/dc/scripts/ptpx.tcl | 2 +- bbdev/api/steps/dc/scripts/report.py | 6 +- bbdev/api/steps/dc/scripts/tapeout.py | 53 +- .../firesim/00_enumeratefpgas_api.step.py | 11 +- .../firesim/00_enumeratefpgas_event.step.py | 16 +- .../firesim/01_buildbitstream_api.step.py | 11 +- .../firesim/01_buildbitstream_event.step.py | 22 +- .../steps/firesim/02_infrasetup_api.step.py | 11 +- .../steps/firesim/02_infrasetup_event.step.py | 24 +- .../steps/firesim/03_runworkload_api.step.py | 15 +- .../firesim/03_runworkload_event.step.py | 26 +- bbdev/api/steps/firesim/README.md | 16 +- bbdev/api/steps/firesim/scripts/chip_yaml.py | 168 - .../api/steps/firesim/scripts/design_yaml.py | 168 + .../scripts/makefrag/firesim/config.mk | 2 +- .../scripts/makefrag/firesim/driver.mk | 2 +- bbdev/api/steps/ip/01_generate_api.step.py | 26 +- bbdev/api/steps/ip/01_generate_event.step.py | 33 +- bbdev/api/steps/ip/02_replace_api.step.py | 26 +- bbdev/api/steps/ip/02_replace_event.step.py | 36 +- bbdev/api/steps/ip/scripts/generate.py | 111 +- bbdev/api/steps/ip/scripts/macro_compiler.py | 13 +- bbdev/api/steps/ip/scripts/replace.py | 69 +- bbdev/api/steps/kernel/01_build_api.step.py | 26 +- bbdev/api/steps/kernel/01_build_event.step.py | 269 +- bbdev/api/steps/kernel/scripts/bin_to_hex.py | 74 - .../mill/01_verilator_verilog_event.step.py | 52 +- .../steps/mill/02_vcs_verilog_event.step.py | 54 +- .../steps/mill/03_yosys_verilog_event.step.py | 87 +- .../steps/mill/04_dc_verilog_event.step.py | 62 +- .../06_bebop_verilator_verilog_event.step.py | 60 +- .../mill/07_bebop_p2e_verilog_event.step.py | 40 +- bbdev/api/steps/mill/scripts/mill.py | 105 +- bbdev/api/steps/model/01_build_api.step.py | 22 +- bbdev/api/steps/model/01_build_event.step.py | 33 +- bbdev/api/steps/model/scripts/build.py | 110 +- bbdev/api/steps/pt/01_ptpx_event.step.py | 35 +- bbdev/api/steps/stack/01_build_api.step.py | 37 + bbdev/api/steps/stack/01_build_event.step.py | 54 + bbdev/api/steps/stack/scripts/build.py | 244 ++ bbdev/api/steps/uvm/00_waive_event.step.py | 34 +- bbdev/api/steps/uvm/01_build_api.step.py | 25 +- bbdev/api/steps/uvm/01_build_event.step.py | 30 +- bbdev/api/steps/uvm/02_run_api.step.py | 25 +- bbdev/api/steps/uvm/02_run_event.step.py | 44 +- bbdev/api/steps/uvm/scripts/ball.py | 377 ++ bbdev/api/steps/uvm/scripts/build.py | 186 + bbdev/api/steps/uvm/scripts/config.py | 50 + bbdev/api/steps/uvm/scripts/coverage.py | 75 + bbdev/api/steps/uvm/scripts/report.py | 22 +- bbdev/api/steps/uvm/scripts/run.py | 210 + bbdev/api/steps/uvm/scripts/test_report.py | 23 +- bbdev/api/steps/uvm/scripts/uvm_common.py | 656 +++- bbdev/api/steps/vcs/01_clean_api.step.py | 18 +- bbdev/api/steps/vcs/01_clean_event.step.py | 18 +- bbdev/api/steps/vcs/02_verilog_api.step.py | 13 +- bbdev/api/steps/vcs/03_build_api.step.py | 30 +- bbdev/api/steps/vcs/03_build_event.step.py | 274 +- bbdev/api/steps/vcs/04_sim_api.step.py | 31 +- bbdev/api/steps/vcs/04_sim_event.step.py | 108 +- bbdev/api/steps/vcs/05_run_api.step.py | 35 +- bbdev/api/steps/vcs/README.md | 10 +- .../api/steps/verilator/01_clean_api.step.py | 26 +- .../steps/verilator/01_clean_event.step.py | 23 +- .../steps/verilator/02_verilog_api.step.py | 32 +- .../api/steps/verilator/03_build_api.step.py | 21 +- .../steps/verilator/03_build_event.step.py | 12 +- bbdev/api/steps/verilator/04_sim_api.step.py | 26 +- .../api/steps/verilator/04_sim_event.step.py | 42 +- bbdev/api/steps/verilator/05_run_api.step.py | 31 +- bbdev/api/steps/verilator/README.md | 4 +- bbdev/api/steps/workload/00_clean_api.step.py | 24 +- .../api/steps/workload/00_clean_event.step.py | 18 +- bbdev/api/steps/workload/01_buidl_api.step.py | 37 +- .../api/steps/workload/01_build_event.step.py | 62 +- bbdev/api/steps/workload/02_tohex_api.step.py | 25 +- .../api/steps/workload/02_tohex_event.step.py | 16 +- bbdev/api/steps/workload/README.md | 22 +- bbdev/api/steps/yosys/01_run_api.step.py | 35 +- bbdev/api/steps/yosys/02_verilog_api.step.py | 34 +- bbdev/api/steps/yosys/03_synth_api.step.py | 30 +- bbdev/api/steps/yosys/03_synth_event.step.py | 99 +- bbdev/api/steps/yosys/scripts/yosys_log.py | 10 - bbdev/api/tests/test_all_api_serial.py | 90 +- .../api/tests/test_api_bebop_bemu_analysis.py | 31 +- bbdev/api/tests/test_api_bebop_bemu_batch.py | 2 +- bbdev/api/tests/test_api_bebop_bemu_sim.py | 6 +- .../test_api_bebop_p2e_buildbitstream.py | 2 +- bbdev/api/tests/test_api_bebop_p2e_clean.py | 2 +- .../tests/test_api_bebop_p2e_runworkload.py | 6 +- bbdev/api/tests/test_api_bebop_p2e_verilog.py | 2 +- .../tests/test_api_bebop_verilator_batch.py | 2 +- .../tests/test_api_bebop_verilator_build.py | 2 +- .../tests/test_api_bebop_verilator_clean.py | 2 +- .../api/tests/test_api_bebop_verilator_run.py | 4 +- .../api/tests/test_api_bebop_verilator_sim.py | 5 +- .../tests/test_api_bebop_verilator_verilog.py | 2 +- bbdev/api/tests/test_api_compiler_build.py | 3 - bbdev/api/tests/test_api_dc_verilog.py | 2 +- .../tests/test_api_firesim_buildbitstream.py | 2 +- .../tests/test_api_firesim_enumeratefpgas.py | 2 +- .../api/tests/test_api_firesim_infrasetup.py | 2 +- .../api/tests/test_api_firesim_runworkload.py | 2 +- bbdev/api/tests/test_api_kernel_build.py | 3 +- bbdev/api/tests/test_api_stack_build.py | 3 + bbdev/api/tests/test_api_uvm_build.py | 2 +- bbdev/api/tests/test_api_uvm_run.py | 2 +- bbdev/api/tests/test_api_verilator_build.py | 2 +- bbdev/api/tests/test_api_verilator_clean.py | 2 +- bbdev/api/tests/test_api_verilator_run.py | 4 +- bbdev/api/tests/test_api_verilator_sim.py | 4 +- bbdev/api/tests/test_api_verilator_verilog.py | 2 +- bbdev/api/tests/test_api_workload_build.py | 2 +- bbdev/api/tests/test_api_workload_clean.py | 2 +- bbdev/api/tests/test_api_workload_tohex.py | 2 +- bbdev/api/tests/test_api_yosys_run.py | 2 +- bbdev/api/tests/test_api_yosys_synth.py | 2 +- bbdev/api/tests/test_api_yosys_verilog.py | 2 +- bbdev/api/tests/test_bemu_binary.py | 50 - bbdev/api/tests/test_p2e_runtime_reuse.py | 54 - bbdev/api/tests/test_uvm_ip_profiles.py | 51 - bbdev/api/unit_tests/test_process_cleanup.py | 72 - bbdev/api/utils/event_common.py | 46 +- bbdev/api/utils/path.py | 173 +- bbdev/api/utils/report_schema.py | 107 +- bbdev/api/utils/reports.py | 40 +- bbdev/api/utils/workload_manifest.py | 38 +- bbdev/bbdev | 390 +- bbdev/mcp/common.py | 203 +- bbdev/mcp/tests/test_common.py | 8 +- bbdev/mcp/tools/__init__.py | 100 +- bbdev/mcp/tools/bebop_p2e_batch.py | 22 +- bbdev/mcp/tools/bebop_p2e_buildbitstream.py | 23 +- bbdev/mcp/tools/bebop_p2e_clean.py | 13 +- bbdev/mcp/tools/bebop_p2e_runworkload.py | 26 +- bbdev/mcp/tools/bebop_p2e_verilog.py | 15 +- bbdev/mcp/tools/bebop_verilator_batch.py | 25 +- bbdev/mcp/tools/bebop_verilator_build.py | 15 +- bbdev/mcp/tools/bebop_verilator_clean.py | 13 +- bbdev/mcp/tools/bebop_verilator_run.py | 35 +- bbdev/mcp/tools/bebop_verilator_sim.py | 12 +- bbdev/mcp/tools/bebop_verilator_verilog.py | 13 +- bbdev/mcp/tools/bemu_analysis.py | 33 +- bbdev/mcp/tools/bemu_batch.py | 22 +- bbdev/mcp/tools/bemu_sim.py | 50 +- bbdev/mcp/tools/compiler_build.py | 13 - bbdev/mcp/tools/dc_area.py | 18 +- bbdev/mcp/tools/dc_power.py | 37 +- bbdev/mcp/tools/dc_verilog.py | 15 +- bbdev/mcp/tools/firesim_buildbitstream.py | 13 +- bbdev/mcp/tools/firesim_enumeratefpgas.py | 13 +- bbdev/mcp/tools/firesim_infrasetup.py | 13 +- bbdev/mcp/tools/firesim_runworkload.py | 13 +- bbdev/mcp/tools/ip_generate.py | 25 +- bbdev/mcp/tools/ip_replace.py | 25 +- bbdev/mcp/tools/kernel_build.py | 32 +- bbdev/mcp/tools/model_build.py | 8 +- bbdev/mcp/tools/stack_build.py | 20 + bbdev/mcp/tools/uvm_build.py | 19 +- bbdev/mcp/tools/uvm_run.py | 19 +- bbdev/mcp/tools/validate.py | 8 +- bbdev/mcp/tools/verilator_build.py | 12 +- bbdev/mcp/tools/verilator_clean.py | 13 +- bbdev/mcp/tools/verilator_run.py | 12 +- bbdev/mcp/tools/verilator_sim.py | 12 +- bbdev/mcp/tools/verilator_verilog.py | 13 +- bbdev/mcp/tools/workload_build.py | 14 +- bbdev/mcp/tools/workload_clean.py | 13 +- bbdev/mcp/tools/workload_tohex.py | 13 +- bbdev/mcp/tools/yosys_run.py | 25 +- bbdev/mcp/tools/yosys_synth.py | 25 +- bbdev/mcp/tools/yosys_verilog.py | 23 +- bebop | 2 +- docs | 2 +- examples/.gitignore | 1 + .../arch/src/main/scala/F32AddBall.scala | 124 - .../AssignPhysicalBankPatterns.cpp | 27 - .../Transforms/LegalizeForLLVMExport.cpp | 46 - examples/balls/f32add/configs/default.toml | 1 - examples/balls/f32add/emu/src/dispatch.rs | 26 - examples/balls/f32add/emu/src/lib.rs | 4 - .../f32add/workloads/ctests/CMakeLists.txt | 1 - .../mlir_tests/ball/f32add_groups.mlir | 17 - .../mlir_tests/ball/f32add_invalid.mlir | 36 - .../mlir_tests/bank/f32add_groups.mlir | 21 - .../mlir_tests/bank/f32add_groups_main.cpp | 55 - .../mlir_tests/bank/f32add_invalid.mlir | 27 - .../arch/src/main/scala/GemminiBall.scala | 295 -- .../arch/src/main/scala/GemminiExCtrl.scala | 26 - .../main/scala/GemminiExCtrlCmdStates.scala | 105 - .../main/scala/GemminiExCtrlDefaults.scala | 43 - .../src/main/scala/GemminiExCtrlDefs.scala | 138 - .../gemmini/arch/src/main/scala/LoopCmd.scala | 77 - .../arch/src/main/scala/LoopCmdEncoder.scala | 143 - .../src/main/scala/LoopConvUnroller.scala | 301 -- .../src/main/scala/LoopMatmulUnroller.scala | 321 -- .../balls/gemmini/compiler/python/quant.py | 197 - .../balls/gemmini/compiler/runtime/conv.cpp | 95 - .../AssignPhysicalBankPatterns.cpp | 92 - .../LowerBuckyballToBankSSAPatterns.cpp | 72 - .../Transforms/LegalizeForLLVMExport.cpp | 46 - .../gemmini/configs/GemminiBallParam.scala | 40 - examples/balls/gemmini/configs/default.toml | 12 - .../gemmini/emu/src/02_gemmini_config.rs | 21 - .../balls/gemmini/emu/src/03_gemmini_flush.rs | 15 - .../gemmini/emu/src/53_gemmini_preload.rs | 42 - .../gemmini/emu/src/80_gemmini_loop_ws.rs | 143 - examples/balls/gemmini/emu/src/dispatch.rs | 58 - examples/balls/gemmini/emu/src/lib.rs | 22 - examples/balls/gemmini/verify/src/tb_top.sv | 81 - .../gemmini/workloads/ctests/CMakeLists.txt | 16 - .../gemmini_accumulated_command_16x16.mlir | 68 - .../ball/gemmini_preloaded_command_16x16.mlir | 77 - .../workloads/mlir_tests/bank/CMakeLists.txt | 7 - ...ini_compute_accumulated_bank_tiles.mlir.in | 75 - ...mmini_compute_preloaded_bank_tiles.mlir.in | 74 - .../im2col/arch/src/main/scala/Im2col.scala | 223 -- .../arch/src/main/scala/Im2colBall.scala | 57 - .../src/main/scala/Im2colConfigRegs.scala | 95 - .../arch/src/main/scala/Im2colWindow.scala | 68 - .../src/main/scala/LineBufferManager.scala | 160 - .../arch/src/main/scala/StreamWriter.scala | 229 -- .../AssignPhysicalBankPatterns.cpp | 41 - .../Transforms/LegalizeForLLVMExport.cpp | 234 -- .../im2col/configs/Im2colBallParam.scala | 25 - examples/balls/im2col/configs/default.toml | 29 - examples/balls/im2col/emu/src/dispatch.rs | 31 - examples/balls/im2col/emu/src/lib.rs | 7 - examples/balls/im2col/verify/README.md | 102 - .../im2col/verify/casegen/src/casegen.rs | 312 -- .../balls/im2col/verify/casegen/src/lib.rs | 112 - .../balls/im2col/verify/casegen/src/model.rs | 103 - examples/balls/im2col/verify/src/tb_top.sv | 91 - .../im2col/workloads/ctests/CMakeLists.txt | 10 - .../mlir_tests/ball/im2col_6x6_k3.mlir | 59 - .../im2col_k3_padded_bank_windows.mlir.in | 41 - .../int2fp/arch/src/main/scala/Int2Fp.scala | 205 - .../arch/src/main/scala/Int2FpBall.scala | 34 - .../AssignPhysicalBankPatterns.cpp | 39 - .../Transforms/LegalizeForLLVMExport.cpp | 46 - .../int2fp/configs/Int2FpBallParam.scala | 21 - examples/balls/int2fp/configs/default.toml | 5 - examples/balls/int2fp/emu/src/dispatch.rs | 31 - examples/balls/int2fp/emu/src/lib.rs | 7 - examples/balls/int2fp/emu/src/model.rs | 117 - examples/balls/int2fp/verify/README.md | 77 - .../int2fp/verify/casegen/src/casegen.rs | 346 -- .../balls/int2fp/verify/casegen/src/lib.rs | 158 - examples/balls/int2fp/verify/src/tb_top.sv | 101 - .../int2fp/workloads/ctests/CMakeLists.txt | 8 - .../workloads/mlir_tests/ball/int2fp_4x4.mlir | 34 - .../mlir_tests/ball/int2fp_relu_4x4.mlir | 33 - .../workloads/mlir_tests/bank/CMakeLists.txt | 6 - .../workloads/mlir_tests/bank/int2fp_4x4.mlir | 35 - .../mlir_tests/bank/int2fp_relu_4x4.mlir | 34 - .../mlir_tests/bank/int2fp_unit_scale.mlir.in | 39 - .../arch/src/main/scala/Int8AddBall.scala | 278 -- .../src/main/scala/PipelinedAddRawFN.scala | 167 - .../AssignPhysicalBankPatterns.cpp | 35 - .../Transforms/LegalizeForLLVMExport.cpp | 56 - examples/balls/int8add/configs/default.toml | 3 - examples/balls/int8add/emu/src/dispatch.rs | 32 - examples/balls/int8add/emu/src/lib.rs | 7 - .../int8add/verify/casegen/src/casegen.rs | 339 -- .../balls/int8add/verify/casegen/src/lib.rs | 117 - examples/balls/int8add/verify/src/tb_top.sv | 101 - .../int8add/workloads/ctests/CMakeLists.txt | 1 - .../mlir_tests/ball/int8add_relu.mlir | 42 - .../bank/int8add_identity_scale.mlir.in | 44 - .../arch/src/main/scala/Int8MulBall.scala | 238 -- .../AssignPhysicalBankPatterns.cpp | 35 - .../Transforms/LegalizeForLLVMExport.cpp | 52 - examples/balls/int8mul/configs/default.toml | 3 - examples/balls/int8mul/emu/src/dispatch.rs | 31 - examples/balls/int8mul/emu/src/lib.rs | 7 - .../int8mul/verify/casegen/src/casegen.rs | 456 --- .../balls/int8mul/verify/casegen/src/lib.rs | 130 - examples/balls/int8mul/verify/src/tb_top.sv | 101 - .../int8mul/workloads/ctests/CMakeLists.txt | 4 - .../mlir_tests/ball/int8mul_gate.mlir | 41 - .../mlir_tests/bank/int8mul_unit_gate.mlir.in | 41 - .../lut/arch/src/main/scala/LutBall.scala | 165 - .../AssignPhysicalBankPatterns.cpp | 54 - .../Transforms/LegalizeForLLVMExport.cpp | 60 - examples/balls/lut/configs/default.toml | 3 - examples/balls/lut/emu/src/dispatch.rs | 36 - examples/balls/lut/emu/src/lib.rs | 7 - .../balls/lut/verify/casegen/src/casegen.rs | 353 -- examples/balls/lut/verify/casegen/src/lib.rs | 115 - examples/balls/lut/verify/src/tb_top.sv | 101 - .../balls/lut/workloads/ctests/CMakeLists.txt | 1 - .../mlir_tests/ball/lut_identity.mlir | 51 - .../bank/lut_identity_table.mlir.in | 53 - .../matadd/arch/src/main/scala/MatAdd.scala | 202 - .../arch/src/main/scala/MatAddBall.scala | 34 - .../AssignPhysicalBankPatterns.cpp | 34 - .../Transforms/LegalizeForLLVMExport.cpp | 44 - examples/balls/matadd/configs/default.toml | 1 - examples/balls/matadd/emu/src/dispatch.rs | 31 - examples/balls/matadd/emu/src/lib.rs | 8 - .../matadd/verify/src/common/matadd_defs.svh | 15 - examples/balls/matadd/verify/src/tb_top.sv | 96 - .../matadd/workloads/ctests/CMakeLists.txt | 1 - .../workloads/mlir_tests/ball/matadd_i32.mlir | 42 - .../bank/matadd_i32_elementwise.mlir.in | 43 - .../arch/src/main/scala/MaxPoolBall.scala | 258 -- .../AssignPhysicalBankPatterns.cpp | 38 - .../Transforms/LegalizeForLLVMExport.cpp | 109 - examples/balls/maxpool/configs/default.toml | 3 - examples/balls/maxpool/emu/src/dispatch.rs | 36 - examples/balls/maxpool/emu/src/lib.rs | 7 - .../maxpool/verify/casegen/src/casegen.rs | 565 --- .../balls/maxpool/verify/casegen/src/lib.rs | 160 - .../balls/maxpool/verify/casegen/src/model.rs | 104 - .../maxpool/workloads/ctests/CMakeLists.txt | 1 - .../mlir_tests/ball/maxpool_6x6_s2.mlir | 32 - .../bank/maxpool_k3_padded_spatial.mlir.in | 41 - .../arch/src/main/scala/Mxfp2Int.scala | 292 -- .../arch/src/main/scala/Mxfp2IntBall.scala | 57 - .../AssignPhysicalBankPatterns.cpp | 34 - .../Transforms/LegalizeForLLVMExport.cpp | 50 - .../mxfp2int/configs/Mxfp2IntBallParam.scala | 17 - examples/balls/mxfp2int/configs/default.toml | 4 - examples/balls/mxfp2int/emu/src/dispatch.rs | 31 - examples/balls/mxfp2int/emu/src/lib.rs | 8 - .../mxfp2int/workloads/ctests/CMakeLists.txt | 3 - .../workloads/mlir_tests/bank/CMakeLists.txt | 6 - .../mxmm/arch/src/main/scala/Array.scala | 51 - .../balls/mxmm/arch/src/main/scala/Emit.scala | 36 - .../arch/src/main/scala/InputWindow.scala | 82 - .../mxmm/arch/src/main/scala/MxmmBall.scala | 26 - .../balls/mxmm/arch/src/main/scala/PE.scala | 48 - .../balls/mxmm/arch/src/main/scala/Unit.scala | 377 -- .../AssignPhysicalBankPatterns.cpp | 58 - .../LowerBuckyball/FP32Patterns.inc | 337 -- .../LowerBuckyballToBankSSAPatterns.cpp | 28 - .../LowerBuckyball/MXFP8Patterns.inc | 222 -- .../LowerBuckyball/SharedPanelPatterns.inc | 63 - .../LowerBuckyball/SharedWindowPatterns.inc | 62 - .../Transforms/LegalizeForLLVMExport.cpp | 157 - .../mxmm/compiler/src/Transforms/Passes.cpp | 4 - .../balls/mxmm/configs/MxmmBallParam.scala | 19 - examples/balls/mxmm/configs/default.toml | 3 - examples/balls/mxmm/emu/src/accumulator.rs | 22 - examples/balls/mxmm/emu/src/dispatch.rs | 33 - examples/balls/mxmm/emu/src/fp32.rs | 121 - examples/balls/mxmm/emu/src/lib.rs | 8 - examples/balls/mxmm/emu/src/mxfp8.rs | 176 - .../mxmm/workloads/ctests/CMakeLists.txt | 11 - .../mxmm/workloads/kernels/CMakeLists.txt | 26 - .../mxmm/workloads/kernels/build_image.py | 93 - .../mxmm/workloads/mlir_tests/CMakeLists.txt | 3 - .../workloads/mlir_tests/ball/CMakeLists.txt | 21 - .../mlir_tests/ball/mxmm_window.mlir | 26 - .../mlir_tests/ball/mxmm_window_chain.mlir | 7 - .../mlir_tests/ball/mxmm_window_rows.mlir | 7 - .../workloads/mlir_tests/bank/CMakeLists.txt | 21 - .../mlir_tests/bank/mxmm_window.mlir | 28 - .../mlir_tests/bank/mxmm_window_chain.mlir | 9 - .../mlir_tests/bank/mxmm_window_rows.mlir | 9 - .../mlir_tests/tensor/CMakeLists.txt | 93 - .../mlir_tests/tensor/mxmm_fp32.mlir | 8 - .../mlir_tests/tensor/mxmm_fp32_main.cpp | 39 - .../tensor/mxmm_quant_branch.mlir.in | 26 - .../tensor/mxmm_quant_branch_main.cpp | 45 - .../tensor/mxmm_quant_input.mlir.in | 9 - .../tensor/mxmm_quant_input_main.cpp | 38 - .../tensor/mxmm_quant_layout.mlir.in | 11 - .../tensor/mxmm_quant_layout_main.cpp | 45 - .../tensor/mxmm_quant_shared.mlir.in | 16 - .../tensor/mxmm_quant_shared_main.cpp | 41 - .../arch/src/main/scala/MxquantBall.scala | 158 - .../AssignPhysicalBankPatterns.cpp | 30 - .../Transforms/LegalizeForLLVMExport.cpp | 56 - examples/balls/mxquant/configs/default.toml | 1 - examples/balls/mxquant/emu/src/dispatch.rs | 30 - examples/balls/mxquant/emu/src/lib.rs | 6 - examples/balls/mxquant/emu/src/quant.rs | 91 - .../mxquant/workloads/ctests/CMakeLists.txt | 14 - .../workloads/mlir_tests/ball/CMakeLists.txt | 7 - .../workloads/mlir_tests/ball/mxquant.mlir | 20 - .../workloads/mlir_tests/bank/CMakeLists.txt | 7 - .../workloads/mlir_tests/bank/mxquant.mlir | 25 - examples/balls/relu/README.md | 26 - .../balls/relu/arch/src/main/scala/Relu.scala | 164 - .../relu/arch/src/main/scala/ReluBall.scala | 33 - .../AssignPhysicalBankPatterns.cpp | 34 - .../LowerBuckyballToBankSSAPatterns.cpp | 112 - .../LowerTileToBuckyball/TileReluPatterns.cpp | 91 - .../Transforms/LegalizeForLLVMExport.cpp | 45 - examples/balls/relu/configs/default.toml | 2 - examples/balls/relu/emu/src/dispatch.rs | 31 - examples/balls/relu/emu/src/lib.rs | 8 - .../relu/workloads/ctests/CMakeLists.txt | 3 - examples/balls/relu/workloads/isa/relu.h | 12 - .../mlir_tests/ball/relu_i32_16row.mlir | 34 - .../bank/relu_i32_mixed_sign.mlir.in | 43 - .../smatmul/arch/src/main/scala/Array.scala | 83 - .../smatmul/arch/src/main/scala/PE.scala | 60 - .../arch/src/main/scala/SMatMulBall.scala | 50 - .../arch/src/main/scala/SMatMulUnit.scala | 431 -- .../balls/smatmul/compiler/runtime/fp32.cpp | 99 - .../AssignPhysicalBankPatterns.cpp | 53 - .../LowerBuckyballToBankSSAPatterns.cpp | 357 -- .../TileMatMulPatterns.cpp | 324 -- .../Transforms/LegalizeForLLVMExport.cpp | 263 -- .../smatmul/configs/SMatMulBallParam.scala | 21 - examples/balls/smatmul/configs/default.toml | 5 - examples/balls/smatmul/configs/lane16.toml | 5 - examples/balls/smatmul/emu/src/dispatch.rs | 38 - examples/balls/smatmul/emu/src/fp32.rs | 119 - examples/balls/smatmul/emu/src/lib.rs | 9 - examples/balls/smatmul/emu/src/mxfp8.rs | 165 - examples/balls/smatmul/verify/README.md | 81 - .../smatmul/verify/casegen/src/casegen.rs | 156 - .../balls/smatmul/verify/casegen/src/model.rs | 58 - examples/balls/smatmul/verify/src/tb_top.sv | 101 - .../smatmul/workloads/ctests/CMakeLists.txt | 13 - .../ball/smatmul_bias_accumulate.mlir | 86 - .../workloads/mlir_tests/bank/CMakeLists.txt | 7 - .../bank/smatmul_bias_two_terms.mlir.in | 80 - .../arch/src/main/scala/ToInt8Ball.scala | 325 -- .../AssignPhysicalBankPatterns.cpp | 56 - .../Transforms/LegalizeForLLVMExport.cpp | 122 - examples/balls/toint8/configs/default.toml | 3 - examples/balls/toint8/emu/src/dispatch.rs | 33 - examples/balls/toint8/emu/src/lib.rs | 12 - examples/balls/toint8/emu/src/quant.rs | 166 - .../toint8/emu/src/tests/activation_scale.rs | 11 - .../toint8/emu/src/tests/int32_rounding.rs | 21 - .../toint8/emu/src/tests/int8_saturation.rs | 9 - examples/balls/toint8/emu/src/tests/mod.rs | 1 - examples/balls/toint8/emu/src/tests/quant.rs | 10 - .../toint8/emu/src/tests/workload_vectors.rs | 19 - examples/balls/toint8/verify/README.md | 93 - .../toint8/verify/casegen/src/casegen.rs | 593 --- .../balls/toint8/verify/casegen/src/lib.rs | 165 - examples/balls/toint8/verify/src/tb_top.sv | 90 - .../toint8/workloads/ctests/CMakeLists.txt | 8 - .../mlir_tests/ball/quant_f32_to_i8.mlir | 30 - .../mlir_tests/ball/quant_i32_to_i8.mlir | 62 - .../workloads/mlir_tests/bank/CMakeLists.txt | 8 - .../mlir_tests/bank/quant_f32_pack16.mlir.in | 39 - .../mlir_tests/bank/quant_i32_pack16.mlir.in | 50 - .../trace/arch/src/main/scala/Trace.scala | 200 - .../trace/arch/src/main/scala/TraceBall.scala | 53 - .../Transforms/LegalizeForLLVMExport.cpp | 55 - examples/balls/trace/configs/default.toml | 3 - .../balls/trace/emu/src/04_bdb_counter.rs | 15 - examples/balls/trace/emu/src/dispatch.rs | 31 - examples/balls/trace/emu/src/lib.rs | 8 - .../trace/workloads/ctests/CMakeLists.txt | 6 - .../ball/bdb_counter_start_read_stop.mlir | 12 - .../transpose/arch/src/main/scala/README.md | 115 - .../arch/src/main/scala/Transpose.scala | 260 -- .../arch/src/main/scala/TransposeBall.scala | 49 - .../AssignPhysicalBankPatterns.cpp | 38 - .../LowerBuckyballToBankSSAPatterns.cpp | 52 - .../TileTransposePatterns.cpp | 50 - .../Transforms/LegalizeForLLVMExport.cpp | 115 - .../configs/TransposeBallParam.scala | 21 - examples/balls/transpose/configs/default.toml | 5 - examples/balls/transpose/emu/src/dispatch.rs | 31 - examples/balls/transpose/emu/src/lib.rs | 7 - examples/balls/transpose/verify/README.md | 91 - .../transpose/verify/casegen/src/casegen.rs | 303 -- .../balls/transpose/verify/casegen/src/lib.rs | 107 - examples/balls/transpose/verify/src/tb_top.sv | 90 - .../transpose/workloads/ctests/CMakeLists.txt | 5 - .../mlir_tests/ball/transpose_16x16_i8.mlir | 44 - .../bank/transpose_i8_packed_matrix.mlir.in | 35 - .../vector/arch/src/main/scala/VecBall.scala | 53 - .../arch/src/main/scala/VecCtrlUnit.scala | 131 - .../arch/src/main/scala/VecLoadUnit.scala | 178 - .../vector/arch/src/main/scala/VecMat16.scala | 190 - .../arch/src/main/scala/VecMat16Store.scala | 112 - .../arch/src/main/scala/VecStoreUnit.scala | 147 - .../vector/arch/src/main/scala/VecUnit.scala | 49 - .../vector/arch/src/main/scala/bond/vvv.scala | 33 - .../src/main/scala/thread/BaseThread.scala | 13 - .../src/main/scala/thread/CasThread.scala | 19 - .../src/main/scala/thread/MulThread.scala | 19 - .../AssignPhysicalBankPatterns.cpp | 38 - .../Transforms/LegalizeForLLVMExport.cpp | 47 - .../vector/configs/VectorBallParam.scala | 32 - examples/balls/vector/configs/default.toml | 9 - examples/balls/vector/emu/src/dispatch.rs | 31 - examples/balls/vector/emu/src/lib.rs | 8 - .../vector/workloads/ctests/CMakeLists.txt | 3 - examples/build.sc | 41 + .../main/scala/sims/p2e/TargetConfigs.scala | 10 - .../scala/sims/verilator/TargetConfigs.scala | 12 - examples/chips/goban/configs/chip.toml | 10 - .../chips/goban/configs/designs/goban.toml | 4 - .../goban/configs/designs/tiles/default.toml | 46 - examples/chips/goban/kernel/workloads.toml | 11 - .../regression/batch/bemu/workloads-bare.toml | 16 - .../batch/bemu/workloads-linux.toml | 4 - .../regression/batch/p2e/workloads-linux.toml | 4 - .../batch/verilator/workloads-bare.toml | 16 - examples/chips/goban/workloads/CMakeLists.txt | 17 - .../goban/workloads/ctests/CMakeLists.txt | 20 - .../MultiRocketVerilatorTarget.scala | 11 - examples/chips/multi-rocket/configs/chip.toml | 9 - .../configs/designs/multi-rocket.toml | 4 - .../configs/designs/tiles/32core.toml | 23 - .../chips/multi-rocket/kernel/workloads.toml | 4 - .../regression/batch/bemu/workloads-bare.toml | 24 - .../regression/batch/p2e/workloads-bare.toml | 21 - .../batch/verilator/workloads-bare.toml | 18 - .../multi-rocket/workloads/CMakeLists.txt | 12 - .../src/main/scala/omni_tile/OmniTile.scala | 21 - .../src/main/scala/poly_tile/PolyTile.scala | 21 - .../main/scala/sims/OmniVerilatorTarget.scala | 9 - examples/chips/omni/configs/chip.toml | 9 - examples/chips/omni/configs/designs/omni.toml | 11 - .../omni/configs/designs/tiles/main.toml | 23 - .../omni/configs/designs/tiles/omni.toml | 53 - .../omni/configs/designs/tiles/poly.toml | 83 - examples/chips/omni/configs/system.toml | 44 - examples/chips/omni/emu/src/main.rs | 41 - examples/chips/omni/kernel/workloads.toml | 8 - .../regression/batch/bemu/workloads-bare.toml | 6 - .../batch/bemu/workloads-linux.toml | 4 - .../regression/batch/p2e/workloads-linux.toml | 4 - examples/chips/omni/tools/weights.py | 200 - examples/chips/omni/workloads/CMakeLists.txt | 8 - .../omni/workloads/ctests/CMakeLists.txt | 52 - .../omni/workloads/ctests/backpressure.c | 37 - examples/chips/omni/workloads/ctests/bounds.c | 10 - .../chips/omni/workloads/ctests/host_io.c | 28 - .../omni/workloads/ctests/interconnect.c | 81 - .../chips/omni/workloads/ctests/mailbox.c | 34 - .../chips/omni/workloads/ctests/peripherals.c | 32 - .../chips/omni/workloads/ctests/process.c | 39 - .../chips/omni/workloads/ctests/reservation.c | 47 - .../omni/workloads/mlir_tests/CMakeLists.txt | 109 - .../workloads/mlir_tests/attention_main.cpp | 107 - .../workloads/mlir_tests/moe_batch_main.cpp | 61 - .../omni/workloads/mlir_tests/moe_main.cpp | 83 - .../main/scala/sims/p2e/TargetConfigs.scala | 10 - .../scala/sims/verilator/TargetConfigs.scala | 12 - examples/chips/pebble/configs/chip.toml | 10 - .../chips/pebble/configs/designs/pebble.toml | 4 - .../pebble/configs/designs/tiles/default.toml | 23 - examples/chips/pebble/kernel/workloads.toml | 36 - .../regression/batch/bemu/workloads-bare.toml | 91 - .../batch/bemu/workloads-linux.toml | 34 - .../regression/batch/p2e/workloads-bare.toml | 25 - .../batch/verilator/workloads-bare.toml | 80 - .../batch/verilator/workloads-linux.toml | 69 - .../chips/pebble/regression/soc/README.md | 0 examples/chips/pebble/tapeout/README.md | 10 - examples/chips/pebble/tapeout/area/dc.tcl | 65 - examples/chips/pebble/tapeout/config.toml | 117 - .../chips/pebble/workloads/CMakeLists.txt | 25 - .../pebble/workloads/ctests/CMakeLists.txt | 13 - .../ctests/dual_conv_add_int2fp_test.c | 80 - .../ctests/mega_conv_k5_pipeline_test.c | 70 - .../workloads/mlir_tests/CMakeLists.txt | 7 - .../bank/quant_im2col_k1_single_pixel.mlir | 46 - .../mlir_tests/linalg/CMakeLists.txt | 33 - .../linalg/linalg_matmul_16x16_i8.mlir | 44 - .../linalg/linalg_mega_conv2d_k3_c16.mlir | 84 - .../linalg/linalg_transpose_16x16_i8.mlir | 28 - .../workloads/mlir_tests/tile/CMakeLists.txt | 75 - .../mlir_tests/tile/mega_conv2d.mlir | 100 - .../mlir_tests/tile/mega_conv2d_k5_tail.mlir | 62 - .../tile/mega_conv2d_s2_depthwise_s2.mlir | 91 - .../mega_global_avg_7x7_c288_exact.mlir.in | 16 - .../tile/mega_global_avg_conv2_exact.mlir.in | 58 - .../tile/mega_global_avg_conv_exact.mlir.in | 24 - .../mlir_tests/tile/mega_matmul.mlir | 57 - .../tile/mega_matmul_m17_k65_n17.mlir | 61 - .../mlir_tests/tile/mega_se_gate_conv.mlir | 75 - .../mlir_tests/tile/tile_matmul_16x16_i8.mlir | 41 - .../tile/tile_matmul_16x16_i8_main.cpp | 29 - .../tile/tile_transpose_16x16_i8.mlir | 27 - .../tile/tile_transpose_16x16_i8_main.cpp | 25 - .../main/scala/compute_tile/ComputeTile.scala | 21 - .../main/scala/sims/p2e/TargetConfigs.scala | 10 - .../scala/sims/verilator/TargetConfigs.scala | 13 - examples/chips/poly/configs/chip.toml | 10 - examples/chips/poly/configs/designs/poly.toml | 9 - .../poly/configs/designs/tiles/compute.toml | 58 - .../poly/configs/designs/tiles/main.toml | 24 - examples/chips/poly/kernel/workloads.toml | 10 - .../regression/batch/bemu/workloads-bare.toml | 11 - .../batch/verilator/workloads-bare.toml | 11 - examples/chips/poly/workloads/CMakeLists.txt | 15 - .../poly/workloads/ctests/CMakeLists.txt | 151 - .../poly/workloads/ctests/ant_bank_exercise.c | 60 - .../poly/workloads/ctests/ant_bank_main.c | 49 - .../workloads/ctests/ant_bank_rows_exercise.c | 59 - .../workloads/ctests/ant_bank_rows_main.c | 49 - .../workloads/ctests/ant_coherence_main.c | 89 - .../poly/workloads/ctests/ant_highpa_main.c | 64 - .../workloads/ctests/ant_mesh_move_main.c | 89 - .../poly/workloads/ctests/ant_mxmm_main.cpp | 96 - .../workloads/ctests/ant_parallel_exercise.c | 68 - .../poly/workloads/ctests/ant_parallel_main.c | 49 - .../chips/poly/workloads/ctests/ant_shared.h | 6 - .../poly/workloads/ctests/ant_shared_main.c | 95 - .../poly/workloads/ctests/ant_stream_main.c | 57 - .../chips/poly/workloads/ctests/ant_tlink.c | 16 - .../chips/poly/workloads/ctests/ant_tlink.h | 6 - .../poly/workloads/ctests/ant_tlink_bank.c | 16 - .../poly/workloads/ctests/ant_tlink_bank.h | 6 - .../workloads/ctests/ant_tlink_bank_main.c | 92 - .../poly/workloads/ctests/ant_tlink_main.c | 88 - .../poly/workloads/ctests/ant_tlink_reduce.c | 21 - .../poly/workloads/ctests/ant_tlink_reduce.h | 8 - .../ctests/ant_tlink_reduce_control.c | 37 - .../workloads/ctests/ant_tlink_reduce_main.c | 97 - .../poly/workloads/ctests/controllers_start.S | 15 - .../workloads/ctests/mesh_shared_mem_test.c | 50 - .../scala/sims/verilator/TargetConfigs.scala | 11 - examples/chips/toy/configs/chip.toml | 9 - .../toy/configs/designs/tiles/default.toml | 23 - examples/chips/toy/configs/designs/toy.toml | 4 - examples/chips/toy/kernel/workloads.toml | 44 - .../regression/batch/bemu/workloads-bare.toml | 53 - .../regression/batch/p2e/workloads-bare.toml | 26 - .../batch/verilator/workloads-bare.toml | 39 - .../batch/verilator/workloads-linux.toml | 35 - examples/chips/toy/regression/soc/README.md | 0 examples/chips/toy/workloads/CMakeLists.txt | 18 - .../chips/toy/workloads/ctests/CMakeLists.txt | 14 - .../ctests/alexnet_gemmini_round_test.c | 22 - .../ctests/alexnet_gemmini_stride_test.c | 37 - .../ctests/alexnet_gemmini_tail_test.c | 41 - .../toy/workloads/mlir_tests/CMakeLists.txt | 8 - .../mlir_tests/linalg/CMakeLists.txt | 28 - examples/components/balls/build.sc | 12 + .../balls/f32add/arch/F32AddBall.scala | 139 + .../balls/f32add/arch}/F32AddRow.scala | 7 +- .../balls/f32add/configs/default.toml | 0 .../balls/f32add/configs/f32add_params.proto | 7 + .../balls/f32add/emu/src/add.rs | 4 +- .../balls/f32add/emu/src/dispatch.rs | 26 + .../components/balls/f32add/emu/src/lib.rs | 8 + .../balls/f32add/stack/src/Lowering.cpp | 55 + .../f32add/workloads/ctests/CMakeLists.txt | 1 + .../f32add/workloads/ctests/f32add_groups.c | 0 .../balls/f32add/workloads/isa/f32add.h | 0 .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/f32add_groups.mlir | 16 + .../mlir_tests/ball/f32add_groups_main.cpp | 0 .../mlir_tests/ball/f32add_invalid.mlir | 28 + .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../mlir_tests/bank/f32add_groups.mlir | 23 + .../mlir_tests/bank/f32add_groups_main.cpp | 55 + .../mlir_tests/bank/f32add_invalid.mlir | 40 + .../balls/gemmini/arch/GemminiBall.scala | 87 + .../balls/gemmini/arch/GemminiExCtrl.scala | 40 + .../arch}/GemminiExCtrlAccumulateState.scala | 6 +- .../gemmini/arch/GemminiExCtrlCmdStates.scala | 110 + .../arch}/GemminiExCtrlComputeFeedState.scala | 12 +- .../arch}/GemminiExCtrlComputeReadState.scala | 4 + .../gemmini/arch/GemminiExCtrlDefaults.scala | 45 + .../gemmini/arch/GemminiExCtrlDefs.scala | 150 + .../gemmini/arch}/GemminiExCtrlFsm.scala | 4 + .../arch}/GemminiExCtrlPreloadStates.scala | 10 +- .../gemmini/arch}/GemminiExCtrlStoreOps.scala | 8 +- .../balls/gemmini/arch}/LICENSE.gemmini | 0 .../components/balls/gemmini/arch/Loop.scala | 173 + .../balls/gemmini/arch/LoopCmd.scala | 86 + .../balls/gemmini/arch}/LoopConvAddrGen.scala | 19 +- .../balls/gemmini/arch/LoopConvUnroller.scala | 306 ++ .../gemmini/arch/LoopMatmulUnroller.scala | 340 ++ .../gemmini/configs/GemminiBallParam.scala | 41 + .../balls/gemmini/configs/default.toml | 11 + .../gemmini/configs/gemmini_params.proto | 16 + .../gemmini/emu/src/02_gemmini_config.rs | 22 + .../balls/gemmini/emu/src/03_gemmini_flush.rs | 15 + .../gemmini/emu/src/21_gemmini_preload.rs | 43 + .../emu/src/66_gemmini_compute_preloaded.rs | 7 +- .../emu/src/67_gemmini_compute_accumulated.rs | 7 +- .../gemmini/emu/src/80_gemmini_loop_ws.rs | 130 + .../emu/src/96_gemmini_loop_conv_ws.rs | 56 +- .../balls/gemmini/emu/src/dispatch.rs | 58 + .../balls/gemmini/emu/src/gemmini_state.rs | 13 +- .../components/balls/gemmini/emu/src/lib.rs | 26 + .../balls/gemmini/emu/src/loop_micro_ops.rs | 4 +- .../balls/gemmini/stack/src/Lowering.cpp | 101 + .../balls/gemmini/verify/casegen/Cargo.lock | 0 .../balls/gemmini/verify/casegen/Cargo.toml | 0 .../balls/gemmini/verify/casegen/src/lib.rs | 0 .../gemmini/verify/filelists/gemmini_ball.f | 2 +- .../verify/src/common/gemmini_defs.svh | 2 +- .../gemmini/verify/src/pkg/gemmini_pkg.sv | 0 .../balls/gemmini/verify/src/tb_top.sv | 81 + .../gemmini/verify/src/tests/gemmini_test.svh | 0 .../gemmini/workloads/ctests/CMakeLists.txt | 16 + .../ctests/gemmini_accumulated_16x16_test.c | 0 .../workloads/ctests/gemmini_loop_mmio.c | 56 + .../workloads/ctests/gemmini_loop_subrob.c | 51 + .../ctests/gemmini_os_cisc_atranspose_test.c | 4 +- .../ctests/gemmini_os_cisc_basic_test.c | 4 +- .../ctests/gemmini_os_cisc_btranspose_test.c | 4 +- .../ctests/gemmini_os_cisc_shift_test.c | 4 +- .../ctests/gemmini_os_risc_abtranspose_test.c | 4 +- .../ctests/gemmini_os_risc_atranspose_test.c | 4 +- .../ctests/gemmini_os_risc_basic_test.c | 4 +- .../ctests/gemmini_os_risc_btranspose_test.c | 4 +- .../ctests/gemmini_os_risc_shift_test.c | 4 +- .../ctests/gemmini_ws_cisc_conv_test.c | 4 +- .../ctests/gemmini_ws_risc_basic_test.c | 4 +- .../ctests/gemmini_ws_risc_btranspose_test.c | 4 +- .../ctests/gemmini_ws_risc_shift_test.c | 4 +- .../balls/gemmini/workloads/isa/gemmini.h | 0 .../isa/gemmini_compute_accumulated.h | 2 +- .../workloads/isa/gemmini_compute_preloaded.h | 0 .../gemmini/workloads/isa/gemmini_config.h | 0 .../gemmini/workloads/isa/gemmini_flush.h | 0 .../workloads/isa/gemmini_loop_conv_ws.h | 0 .../gemmini/workloads/isa/gemmini_loop_ws.h | 0 .../gemmini/workloads/isa/gemmini_preload.h | 0 .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../gemmini_accumulated_command_16x16.mlir | 52 + ...gemmini_accumulated_command_16x16_main.cpp | 0 .../ball/gemmini_preloaded_command_16x16.mlir | 51 + .../gemmini_preloaded_command_16x16_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 13 + ...ini_compute_accumulated_bank_tiles.mlir.in | 63 + ...ni_compute_accumulated_bank_tiles_main.cpp | 2 +- ...mmini_compute_preloaded_bank_tiles.mlir.in | 62 + ...mini_compute_preloaded_bank_tiles_main.cpp | 2 +- .../components/balls/im2col/arch/Im2col.scala | 267 ++ .../balls/im2col/arch/Im2colBall.scala | 74 + .../balls/im2col/arch/Im2colConfigRegs.scala | 108 + .../balls/im2col/arch/Im2colWindow.scala | 75 + .../balls/im2col/arch/LineBufferManager.scala | 175 + .../balls/im2col/arch}/README.md | 0 .../balls/im2col/arch}/RestoringDiv.scala | 9 +- .../balls/im2col/arch/StreamWriter.scala | 243 ++ .../im2col/configs/Im2colBallParam.scala | 22 + .../balls/im2col/configs/default.toml | 28 + .../balls/im2col/configs/im2col_params.proto | 11 + .../balls/im2col/emu/src/dispatch.rs | 31 + .../balls/im2col/emu/src/im2col.rs | 47 +- .../components/balls/im2col/emu/src/lib.rs | 11 + .../balls/im2col/stack/src/Lowering.cpp | 84 + .../components/balls/im2col/verify/README.md | 102 + .../balls/im2col/verify/casegen/Cargo.lock | 0 .../balls/im2col/verify/casegen/Cargo.toml | 0 .../im2col/verify/casegen/src/casegen.rs | 218 ++ .../balls/im2col/verify/casegen/src/lib.rs | 71 + .../balls/im2col/verify/casegen/src/model.rs | 103 + .../im2col/verify/filelists/im2col_ball.f | 0 .../im2col/verify/src/common/im2col_defs.svh | 4 +- .../im2col/verify/src/common/im2col_items.svh | 4 +- .../im2col/verify/src/cov/im2col_cov.svh | 1 + .../im2col/verify/src/env/im2col_env.svh | 0 .../verify/src/env/im2col_scoreboard.svh | 2 + .../balls/im2col/verify/src/pkg/im2col_pkg.sv | 0 .../verify/src/seq/im2col_sequences.svh | 0 .../balls/im2col/verify/src/tb_top.sv | 91 + .../im2col/verify/src/tests/im2col_test.svh | 0 .../im2col/workloads/ctests/CMakeLists.txt | 8 + .../im2col/workloads/ctests/im2col_k1_test.c | 4 +- .../ctests/im2col_k3_asym_pad_test.c | 4 +- .../workloads/ctests/im2col_k3_pad_test.c | 4 +- .../workloads/ctests/im2col_k3_partial_test.c | 6 +- .../im2col/workloads/ctests/im2col_k3_test.c | 6 +- .../im2col/workloads/ctests/im2col_k5_test.c | 4 +- .../im2col/workloads/ctests/im2col_k7_test.c | 4 +- .../workloads/ctests/im2col_window_test.c | 0 .../balls/im2col/workloads/isa/im2col.h | 0 .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/im2col_6x6_k3.mlir | 43 + .../mlir_tests/ball/im2col_6x6_k3_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../im2col_k3_padded_bank_windows.mlir.in | 32 + .../im2col_k3_padded_bank_windows_main.cpp | 0 .../balls/int2fp/arch}/Fp32Mul.scala | 21 +- .../components/balls/int2fp/arch/Int2Fp.scala | 239 ++ .../balls/int2fp/arch/Int2FpBall.scala | 52 + .../int2fp/configs/Int2FpBallParam.scala | 20 + .../balls/int2fp/configs/default.toml | 4 + .../balls/int2fp/configs/int2fp_params.proto | 9 + .../balls/int2fp/emu/src/dispatch.rs | 31 + .../balls/int2fp/emu/src/int2fp.rs | 2 +- .../components/balls/int2fp/emu/src/lib.rs | 11 + .../components/balls/int2fp/emu/src/model.rs | 88 + .../balls/int2fp/stack/src/Lowering.cpp | 49 + .../components/balls/int2fp/verify/README.md | 77 + .../balls/int2fp/verify/casegen/Cargo.lock | 0 .../balls/int2fp/verify/casegen/Cargo.toml | 0 .../int2fp/verify/casegen/src/casegen.rs | 259 ++ .../balls/int2fp/verify/casegen/src/lib.rs | 108 + .../int2fp/verify/filelists/int2fp_ball.f | 0 .../int2fp/verify/src/common/int2fp_defs.svh | 4 +- .../int2fp/verify/src/common/int2fp_items.svh | 6 +- .../int2fp/verify/src/cov/int2fp_cov.svh | 3 +- .../int2fp/verify/src/env/int2fp_env.svh | 0 .../verify/src/env/int2fp_scoreboard.svh | 2 + .../balls/int2fp/verify/src/pkg/int2fp_pkg.sv | 0 .../verify/src/seq/int2fp_sequences.svh | 0 .../balls/int2fp/verify/src/tb_top.sv | 101 + .../int2fp/verify/src/tests/int2fp_test.svh | 0 .../int2fp/workloads/ctests/CMakeLists.txt | 6 + .../ctests/int2fp_channel_base_test.c | 4 +- .../ctests/int2fp_channel_lanes_test.c | 4 +- .../ctests/int2fp_channel_two_rows_test.c | 4 +- .../ctests/int2fp_tensor_groups_test.c | 4 +- .../ctests/int2fp_tensor_rows_test.c | 4 +- .../ctests/int2fp_tensor_values_test.c | 4 +- .../balls/int2fp/workloads/isa/int2fp.h | 0 .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/int2fp_4x4.mlir | 28 + .../mlir_tests/ball/int2fp_4x4_main.cpp | 0 .../mlir_tests/ball/int2fp_relu_4x4.mlir | 27 + .../mlir_tests/ball/int2fp_relu_4x4_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 11 + .../workloads/mlir_tests/bank/int2fp_4x4.mlir | 28 + .../mlir_tests/bank/int2fp_4x4_main.cpp | 0 .../mlir_tests/bank/int2fp_relu_4x4.mlir | 27 + .../mlir_tests/bank/int2fp_relu_4x4_main.cpp | 0 .../mlir_tests/bank/int2fp_unit_scale.mlir.in | 36 + .../bank/int2fp_unit_scale_main.cpp | 0 .../balls/int8add/arch/Int8AddBall.scala | 291 ++ .../int8add/arch/PipelinedAddRawFN.scala | 171 + .../balls/int8add/configs/default.toml | 1 + .../int8add/configs/int8add_params.proto | 7 + .../balls/int8add/emu/src/dispatch.rs | 32 + .../balls/int8add/emu/src/int8add.rs | 2 +- .../components/balls/int8add/emu/src/lib.rs | 11 + .../balls/int8add/stack/src/Lowering.cpp | 66 + .../balls/int8add/verify/casegen/Cargo.lock | 0 .../balls/int8add/verify/casegen/Cargo.toml | 0 .../int8add/verify/casegen/src/casegen.rs | 257 ++ .../balls/int8add/verify/casegen/src/lib.rs | 70 + .../int8add/verify/filelists/int8add_ball.f | 0 .../verify/src/common/int8add_defs.svh | 8 +- .../verify/src/common/int8add_items.svh | 8 +- .../int8add/verify/src/cov/int8add_cov.svh | 5 +- .../int8add/verify/src/env/int8add_env.svh | 0 .../verify/src/env/int8add_scoreboard.svh | 2 + .../int8add/verify/src/pkg/int8add_pkg.sv | 0 .../verify/src/seq/int8add_sequences.svh | 0 .../balls/int8add/verify/src/tb_top.sv | 101 + .../int8add/verify/src/tests/int8add_test.svh | 0 .../int8add/workloads/ctests/CMakeLists.txt | 2 + .../workloads/ctests/int8add_relu_test.c | 0 .../workloads/ctests/int8add_scaled_test.c | 0 .../balls/int8add/workloads/isa/int8add.h | 0 .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/int8add_relu.mlir | 40 + .../mlir_tests/ball/int8add_relu_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../bank/int8add_identity_scale.mlir.in | 42 + .../bank/int8add_identity_scale_main.cpp | 0 .../balls/int8mul/arch/Int8MulBall.scala | 248 ++ .../balls/int8mul/configs/default.toml | 1 + .../int8mul/configs/int8mul_params.proto | 7 + .../balls/int8mul/emu/src/dispatch.rs | 31 + .../balls/int8mul/emu/src/int8mul.rs | 2 +- .../components/balls/int8mul/emu/src/lib.rs | 11 + .../balls/int8mul/stack/src/Lowering.cpp | 48 + .../balls/int8mul/verify/casegen/Cargo.lock | 0 .../balls/int8mul/verify/casegen/Cargo.toml | 0 .../int8mul/verify/casegen/src/casegen.rs | 266 ++ .../int8mul/verify/casegen/src/config.rs | 0 .../balls/int8mul/verify/casegen/src/lib.rs | 71 + .../int8mul/verify/filelists/int8mul_ball.f | 0 .../verify/src/common/int8mul_defs.svh | 4 +- .../verify/src/common/int8mul_items.svh | 6 +- .../int8mul/verify/src/cov/int8mul_cov.svh | 3 +- .../int8mul/verify/src/env/int8mul_env.svh | 0 .../verify/src/env/int8mul_scoreboard.svh | 2 + .../int8mul/verify/src/pkg/int8mul_pkg.sv | 0 .../verify/src/seq/int8mul_sequences.svh | 0 .../balls/int8mul/verify/src/tb_top.sv | 101 + .../int8mul/verify/src/tests/int8mul_test.svh | 0 .../int8mul/workloads/ctests/CMakeLists.txt | 2 + .../ctests/int8mul_large_gate_row_test.c | 0 .../ctests/int8mul_per_channel_gate_test.c | 0 .../balls/int8mul/workloads/isa/int8mul.h | 0 .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/int8mul_gate.mlir | 37 + .../mlir_tests/ball/int8mul_gate_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../mlir_tests/bank/int8mul_unit_gate.mlir.in | 37 + .../bank/int8mul_unit_gate_main.cpp | 0 .../components/balls/lut/arch/LutBall.scala | 169 + .../components/balls/lut/configs/default.toml | 1 + .../balls/lut/configs/lut_params.proto | 7 + .../components/balls/lut/emu/src/dispatch.rs | 36 + examples/components/balls/lut/emu/src/lib.rs | 11 + .../{ => components}/balls/lut/emu/src/lut.rs | 6 +- .../balls/lut/stack/src/Lowering.cpp | 38 + .../balls/lut/verify/casegen/Cargo.lock | 0 .../balls/lut/verify/casegen/Cargo.toml | 0 .../balls/lut/verify/casegen/src/casegen.rs | 260 ++ .../balls/lut/verify/casegen/src/lib.rs | 70 + .../balls/lut/verify/filelists/lut_ball.f | 0 .../balls/lut/verify/src/common/lut_defs.svh | 4 +- .../balls/lut/verify/src/common/lut_items.svh | 4 +- .../balls/lut/verify/src/cov/lut_cov.svh | 3 +- .../balls/lut/verify/src/env/lut_env.svh | 0 .../lut/verify/src/env/lut_scoreboard.svh | 2 + .../balls/lut/verify/src/pkg/lut_pkg.sv | 0 .../lut/verify/src/seq/lut_sequences.svh | 0 .../components/balls/lut/verify/src/tb_top.sv | 101 + .../balls/lut/verify/src/tests/lut_test.svh | 0 .../balls/lut/workloads/ctests/CMakeLists.txt | 2 + .../ctests/lut_channel_table_lookup_test.c | 0 .../ctests/lut_uniform_lookup_test.c | 0 .../balls/lut/workloads/isa/lut.h | 0 .../lut/workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/lut_identity.mlir | 48 + .../mlir_tests/ball/lut_identity_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../bank/lut_identity_table.mlir.in | 50 + .../bank/lut_identity_table_main.cpp | 0 .../{ => components}/balls/matadd/README.md | 0 .../components/balls/matadd/arch/MatAdd.scala | 219 ++ .../balls/matadd/arch/MatAddBall.scala | 49 + .../balls/matadd/configs/default.toml | 0 .../balls/matadd/configs/matadd_params.proto | 7 + .../balls/matadd/emu/src/64_matadd.rs | 6 +- .../balls/matadd/emu/src/dispatch.rs | 31 + .../components/balls/matadd/emu/src/lib.rs | 12 + .../balls/matadd/stack/src/Lowering.cpp | 49 + .../balls/matadd/verify/casegen/Cargo.lock | 0 .../balls/matadd/verify/casegen/Cargo.toml | 0 .../balls/matadd/verify/casegen/src/lib.rs | 0 .../matadd/verify/filelists/matadd_ball.f | 0 .../matadd/verify/src/common/matadd_defs.svh | 15 + .../matadd/verify/src/common/matadd_items.svh | 4 +- .../matadd/verify/src/env/matadd_env.svh | 0 .../verify/src/env/matadd_scoreboard.svh | 1 + .../balls/matadd/verify/src/pkg/matadd_pkg.sv | 0 .../verify/src/seq/matadd_sequences.svh | 0 .../balls/matadd/verify/src/tb_top.sv | 96 + .../matadd/verify/src/tests/matadd_test.svh | 0 .../matadd/workloads/ctests/CMakeLists.txt | 1 + .../workloads/ctests/matadd_i32_16row_test.c | 0 .../balls/matadd/workloads/isa/matadd.h | 0 .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/matadd_i32.mlir | 37 + .../mlir_tests/ball/matadd_i32_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../bank/matadd_i32_elementwise.mlir.in | 40 + .../bank/matadd_i32_elementwise_main.cpp | 0 .../balls/maxpool/arch/MaxPoolBall.scala | 256 ++ .../balls/maxpool/configs/default.toml | 1 + .../maxpool/configs/maxpool_params.proto | 7 + .../balls/maxpool/emu/src/dispatch.rs | 36 + .../components/balls/maxpool/emu/src/lib.rs | 11 + .../balls/maxpool/emu/src/maxpool.rs | 6 +- .../balls/maxpool/stack/src/Lowering.cpp | 87 + .../balls/maxpool/verify/casegen/Cargo.lock | 0 .../balls/maxpool/verify/casegen/Cargo.toml | 0 .../maxpool/verify/casegen/src/casegen.rs | 447 +++ .../balls/maxpool/verify/casegen/src/lib.rs | 107 + .../balls/maxpool/verify/casegen/src/model.rs | 104 + .../maxpool/verify/filelists/maxpool_ball.f | 0 .../verify/src/common/maxpool_defs.svh | 4 +- .../verify/src/common/maxpool_items.svh | 4 +- .../maxpool/verify/src/cov/maxpool_cov.svh | 3 +- .../maxpool/verify/src/env/maxpool_env.svh | 0 .../verify/src/env/maxpool_scoreboard.svh | 4 +- .../maxpool/verify/src/pkg/maxpool_pkg.sv | 0 .../verify/src/seq/maxpool_sequences.svh | 0 .../balls/maxpool/verify/src/tb_top.sv | 0 .../maxpool/verify/src/tests/maxpool_test.svh | 0 .../maxpool/workloads/ctests/CMakeLists.txt | 2 + .../workloads/ctests/maxpool_offset_test.c | 0 .../workloads/ctests/maxpool_window_test.c | 0 .../balls/maxpool/workloads/isa/maxpool.h | 2 +- .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/maxpool_6x6_s2.mlir | 27 + .../mlir_tests/ball/maxpool_6x6_s2_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../bank/maxpool_k3_padded_spatial.mlir.in | 37 + .../bank/maxpool_k3_padded_spatial_main.cpp | 0 .../balls/mxfp2int/arch/Mxfp2Int.scala | 309 ++ .../balls/mxfp2int/arch/Mxfp2IntBall.scala | 75 + .../balls/mxfp2int/arch}/Mxfp4Decoder.scala | 0 .../balls/mxfp2int/arch}/Mxfp6Decoder.scala | 0 .../balls/mxfp2int/arch}/Mxfp8Decoder.scala | 0 .../balls/mxfp2int/arch}/MxfpDecoder.scala | 0 .../mxfp2int/configs/Mxfp2IntBallParam.scala | 17 + .../balls/mxfp2int/configs/default.toml | 3 + .../mxfp2int/configs/mxfp2int_params.proto | 8 + .../balls/mxfp2int/emu/src/55_mxfp2int.rs | 6 +- .../balls/mxfp2int/emu/src/dispatch.rs | 31 + .../components/balls/mxfp2int/emu/src/lib.rs | 12 + .../mxfp2int/workloads/ctests/CMakeLists.txt | 1 + .../ctests/mxfp4_e8m0_two_block_test.c | 4 +- .../balls/mxfp2int/workloads/isa/mxfp2int.h | 0 .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../ball/mxfp4_unit_scale_2blocks.mlir | 0 .../ball/mxfp4_unit_scale_2blocks_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 16 + .../bank/mxfp4_unit_scale_bank.mlir.in | 0 .../bank/mxfp4_unit_scale_bank_main.cpp | 0 .../balls/mxmm/arch}/AddPipe.scala | 6 +- .../components/balls/mxmm/arch/Array.scala | 53 + .../balls/mxmm/arch/InputWindow.scala | 91 + .../balls/mxmm/arch}/Mxfp8Decode.scala | 0 .../components/balls/mxmm/arch/MxmmBall.scala | 49 + examples/components/balls/mxmm/arch/PE.scala | 52 + .../balls/mxmm/arch}/Panels.scala | 4 +- .../components/balls/mxmm/arch/Unit.scala | 411 ++ .../balls/mxmm/configs/MxmmBallParam.scala | 22 + .../balls/mxmm/configs/default.toml | 2 + .../balls/mxmm/configs/mxmm_params.proto | 9 + .../balls/mxmm/emu/src/accumulator.rs | 17 + .../components/balls/mxmm/emu/src/dispatch.rs | 33 + .../components/balls/mxmm/emu/src/fp32.rs | 123 + examples/components/balls/mxmm/emu/src/lib.rs | 12 + .../components/balls/mxmm/emu/src/mxfp8.rs | 178 + .../balls/mxmm/stack/kernels/CMakeLists.txt | 45 + .../balls/mxmm/stack}/kernels/dispatch.cpp | 0 .../balls/mxmm/stack}/kernels/norm_window.cpp | 2 +- .../mxmm/stack/kernels/norm_window_image.h | 15 + .../mxmm/stack}/kernels/pack_dispatch.cpp | 0 .../balls/mxmm/stack}/kernels/packmm.cpp | 2 +- .../balls/mxmm/stack}/kernels/packmm.h | 0 .../balls/mxmm/stack/kernels/packmm_image.h | 14 + .../balls/mxmm/stack}/kernels/window.h | 0 .../mxmm/workloads/ctests/CMakeLists.txt | 18 + .../ctests/mxmm_bank_capacity_test.c | 0 .../mxmm/workloads/ctests/mxmm_fp32_test.c | 0 .../mxmm/workloads/ctests/mxmm_mxfp8_test.c | 0 .../workloads/ctests/mxmm_window_chain_test.c | 0 .../workloads/ctests/mxmm_window_rows_test.c | 0 .../mxmm/workloads/ctests/mxmm_window_test.c | 0 .../balls/mxmm/workloads/isa/mxmm.h | 0 .../mxmm}/workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 32 + .../workloads/mlir_tests/ball/mxmm_f32.mlir | 0 .../mlir_tests/ball/mxmm_f32_main.cpp | 0 .../workloads/mlir_tests/ball/mxmm_fma32.mlir | 0 .../mlir_tests/ball/mxmm_fma32_main.cpp | 0 .../workloads/mlir_tests/ball/mxmm_mxfp8.mlir | 0 .../mlir_tests/ball/mxmm_mxfp8_main.cpp | 0 .../mlir_tests/ball/mxmm_window.mlir | 28 + .../mlir_tests/ball/mxmm_window_chain.mlir | 8 + .../ball/mxmm_window_chain_main.cpp | 0 .../mlir_tests/ball/mxmm_window_main.cpp | 0 .../mlir_tests/ball/mxmm_window_rows.mlir | 8 + .../mlir_tests/ball/mxmm_window_rows_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 32 + .../workloads/mlir_tests/bank/mxmm_f32.mlir | 0 .../mlir_tests/bank/mxmm_f32_main.cpp | 0 .../workloads/mlir_tests/bank/mxmm_fma32.mlir | 0 .../mlir_tests/bank/mxmm_fma32_main.cpp | 0 .../workloads/mlir_tests/bank/mxmm_mxfp8.mlir | 0 .../mlir_tests/bank/mxmm_mxfp8_main.cpp | 0 .../mlir_tests/bank/mxmm_window.mlir | 30 + .../mlir_tests/bank/mxmm_window_chain.mlir | 10 + .../bank/mxmm_window_chain_main.cpp | 0 .../mlir_tests/bank/mxmm_window_main.cpp | 0 .../mlir_tests/bank/mxmm_window_rows.mlir | 10 + .../mlir_tests/bank/mxmm_window_rows_main.cpp | 0 .../workloads/mlir_tests/mxmm_native_steps.h | 0 .../workloads/mlir_tests/mxmm_native_test.h | 0 .../balls/mxquant/arch}/Encode.scala | 4 + .../balls/mxquant/arch/MxquantBall.scala | 173 + .../balls/mxquant/configs/default.toml | 0 .../mxquant/configs/mxquant_params.proto | 7 + .../balls/mxquant/emu/src/dispatch.rs | 30 + .../components/balls/mxquant/emu/src/lib.rs | 10 + .../components/balls/mxquant/emu/src/quant.rs | 91 + .../mxquant/stack/kernels/CMakeLists.txt | 10 + .../balls/mxquant/stack/kernels/dispatch.cpp | 14 + .../balls/mxquant/stack/kernels/pack.cpp | 70 + .../mxquant/workloads/ctests/CMakeLists.txt | 25 + .../workloads/ctests/mxquant_capacity_test.c | 0 .../workloads/ctests/mxquant_values_test.c | 0 .../balls/mxquant/workloads/isa/mxquant.h | 0 .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 12 + .../workloads/mlir_tests/ball/mxquant.mlir | 22 + .../mlir_tests/ball/mxquant_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 12 + .../workloads/mlir_tests/bank/mxquant.mlir | 27 + .../mlir_tests/bank/mxquant_main.cpp | 0 .../balls/mxquant/workloads/reference.h | 2 +- .../balls/redmule/arch/RedmuleBall.scala | 234 ++ .../balls/redmule/arch/RedmuleNative.scala | 96 + .../balls/redmule/arch/RedmuleNative.sv | 117 + .../balls/redmule/configs/default.toml | 1 + .../redmule/configs/redmule_params.proto | 8 + .../components/balls/redmule/emu/src/fp16.rs | 71 + .../components/balls/redmule/emu/src/lib.rs | 36 + .../balls/redmule/emu/src/matmul.rs | 64 + .../balls/redmule/stack/src/Lowering.cpp | 68 + .../balls/redmule/verify/casegen/Cargo.lock | 7 + .../balls/redmule/verify/casegen/Cargo.toml | 10 + .../balls/redmule/verify/casegen/src/lib.rs | 118 + .../redmule/verify/filelists/redmule_ball.f | 13 + .../balls/redmule/verify/parameters.toml | 2 + .../redmule/verify/src/pkg/redmule_pkg.sv | 351 ++ .../verify/src/reference/redmule_reference.sv | 130 + .../balls/redmule/verify/src/tb_top.sv | 111 + .../redmule/workloads/ctests/CMakeLists.txt | 17 + .../workloads/ctests/redmule_f16_chain_test.c | 43 + .../workloads/ctests/redmule_f16_edges_test.c | 35 + .../workloads/ctests/redmule_f16_fused_test.c | 29 + .../workloads/ctests/redmule_f16_test.c | 47 + .../balls/redmule/workloads/isa/redmule.h | 22 + .../workloads/mlir_tests/CMakeLists.txt | 3 + .../workloads/mlir_tests/ball/CMakeLists.txt | 2 + .../mlir_tests/ball/redmule_f16_chain.mlir | 6 + .../ball/redmule_f16_chain_main.cpp | 28 + .../ball/redmule_f16_invalid_k.mlir | 7 + .../workloads/mlir_tests/bank/CMakeLists.txt | 2 + .../bank/redmule_f16_invalid_alias.mlir | 11 + .../mlir_tests/bank/redmule_f16_rows.mlir.in | 48 + .../mlir_tests/bank/redmule_f16_rows_main.cpp | 19 + examples/components/balls/relu/README.md | 28 + .../components/balls/relu/arch/Relu.scala | 184 + .../components/balls/relu/arch/ReluBall.scala | 46 + .../balls/relu/configs/default.toml | 1 + .../balls/relu/configs/relu_params.proto | 7 + .../balls/relu/emu/src/50_relu.rs | 10 +- .../components/balls/relu/emu/src/dispatch.rs | 31 + examples/components/balls/relu/emu/src/lib.rs | 12 + .../balls/relu/stack/src/Lowering.cpp | 55 + .../relu/workloads/ctests/CMakeLists.txt | 1 + .../workloads/ctests/relu_i32_16x16_test.c | 0 .../balls/relu/workloads/isa/relu.h | 13 + .../relu/workloads/mlir_tests/CMakeLists.txt | 2 + .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/relu_i32_16row.mlir | 31 + .../mlir_tests/ball/relu_i32_16row_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../bank/relu_i32_mixed_sign.mlir.in | 41 + .../bank/relu_i32_mixed_sign_main.cpp | 0 .../{ => components}/balls/smatmul/README.md | 0 .../components/balls/smatmul/arch/Array.scala | 82 + .../components/balls/smatmul/arch/PE.scala | 60 + .../balls/smatmul/arch/SMatMulBall.scala | 66 + .../balls/smatmul/arch/SMatMulUnit.scala | 461 +++ .../smatmul/configs/SMatMulBallParam.scala | 20 + .../balls/smatmul/configs/default.toml | 4 + .../balls/smatmul/configs/lane16.toml | 4 + .../smatmul/configs/smatmul_params.proto | 9 + .../balls/smatmul/emu/src/dispatch.rs | 38 + .../components/balls/smatmul/emu/src/fp32.rs | 118 + .../components/balls/smatmul/emu/src/lib.rs | 13 + .../components/balls/smatmul/emu/src/mxfp8.rs | 163 + .../balls/smatmul/emu/src/smatmul.rs | 49 +- .../balls/smatmul/stack/src/Lowering.cpp | 79 + .../components/balls/smatmul/verify/README.md | 81 + .../balls/smatmul/verify/casegen/Cargo.lock | 0 .../balls/smatmul/verify/casegen/Cargo.toml | 0 .../smatmul/verify/casegen/src/casegen.rs | 133 + .../balls/smatmul/verify/casegen/src/lib.rs | 0 .../balls/smatmul/verify/casegen/src/model.rs | 58 + .../smatmul/verify/filelists/smatmul_ball.f | 0 .../balls/smatmul/verify/parameters.toml | 0 .../verify/src/common/smatmul_defs.svh | 8 +- .../verify/src/common/smatmul_items.svh | 4 +- .../smatmul/verify/src/cov/smatmul_cov.svh | 3 +- .../smatmul/verify/src/env/smatmul_env.svh | 0 .../verify/src/env/smatmul_scoreboard.svh | 1 + .../smatmul/verify/src/pkg/smatmul_pkg.sv | 0 .../verify/src/seq/smatmul_sequences.svh | 0 .../balls/smatmul/verify/src/tb_top.sv | 101 + .../smatmul/verify/src/tests/smatmul_test.svh | 0 .../smatmul/workloads/ctests/CMakeLists.txt | 14 + .../workloads/ctests/smatmul_16x16_test.c | 0 .../workloads/ctests/smatmul_16x16x32_test.c | 0 .../workloads/ctests/smatmul_1x16x64_test.c | 0 .../workloads/ctests/smatmul_8x16_test.c | 0 .../ctests/smatmul_bias_accumulate_test.c | 2 + .../workloads/ctests/smatmul_f32_test.c | 0 .../ctests/smatmul_nondefault_banks_test.c | 0 .../balls/smatmul/workloads/isa/smatmul.h | 0 .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../ball/smatmul_bias_accumulate.mlir | 70 + .../ball/smatmul_bias_accumulate_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 19 + .../bank/smatmul_bias_two_terms.mlir.in | 69 + .../bank/smatmul_bias_two_terms_main.cpp | 0 .../balls/toint8/arch/ToInt8Ball.scala | 350 ++ .../balls/toint8/configs/default.toml | 1 + .../balls/toint8/configs/toint8_params.proto | 7 + .../balls/toint8/emu/src/dispatch.rs | 33 + .../components/balls/toint8/emu/src/lib.rs | 13 + .../balls/toint8/emu/src/model.rs | 0 .../components/balls/toint8/emu/src/quant.rs | 166 + .../balls/toint8/stack/src/Lowering.cpp | 81 + .../components/balls/toint8/verify/README.md | 93 + .../balls/toint8/verify/casegen/Cargo.lock | 0 .../balls/toint8/verify/casegen/Cargo.toml | 0 .../toint8/verify/casegen/src/casegen.rs | 504 +++ .../balls/toint8/verify/casegen/src/lib.rs | 119 + .../toint8/verify/filelists/toint8_ball.f | 0 .../toint8/verify/src/common/toint8_defs.svh | 8 +- .../toint8/verify/src/common/toint8_items.svh | 8 +- .../toint8/verify/src/cov/toint8_cov.svh | 4 +- .../toint8/verify/src/env/toint8_env.svh | 0 .../verify/src/env/toint8_scoreboard.svh | 2 + .../balls/toint8/verify/src/pkg/toint8_pkg.sv | 0 .../verify/src/seq/toint8_sequences.svh | 0 .../balls/toint8/verify/src/tb_top.sv | 90 + .../toint8/verify/src/tests/toint8_test.svh | 0 .../toint8/workloads/ctests/CMakeLists.txt | 6 + .../workloads/ctests/quant_f32_to_i8_test.c | 0 .../workloads/ctests/quant_i32_to_i8_test.c | 0 .../ctests/toint8_i8_rounding_test.c | 4 +- .../workloads/ctests/toint8_i8_rows_test.c | 4 +- .../ctests/toint8_i8_scale_rows_test.c | 4 +- .../workloads/ctests/toint8_i8_signed_test.c | 4 +- .../balls/toint8/workloads/isa/quant.h | 0 .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/quant_f32_to_i8.mlir | 25 + .../mlir_tests/ball/quant_f32_to_i8_main.cpp | 0 .../mlir_tests/ball/quant_i32_to_i8.mlir | 50 + .../mlir_tests/ball/quant_i32_to_i8_main.cpp | 8 +- .../workloads/mlir_tests/bank/CMakeLists.txt | 22 + .../mlir_tests/bank/quant_f32_pack16.mlir.in | 36 + .../mlir_tests/bank/quant_f32_pack16_main.cpp | 0 .../mlir_tests/bank/quant_i32_pack16.mlir.in | 41 + .../mlir_tests/bank/quant_i32_pack16_main.cpp | 0 .../balls/trace/arch}/README.md | 0 .../components/balls/trace/arch/Trace.scala | 219 ++ .../balls/trace/arch/TraceBall.scala | 68 + .../balls/trace/arch}/TraceDPI.scala | 6 +- .../balls/trace/configs/default.toml | 1 + .../balls/trace/configs/trace_params.proto | 7 + .../balls/trace/emu/src/04_bdb_counter.rs | 15 + .../balls/trace/emu/src/dispatch.rs | 31 + .../components/balls/trace/emu/src/lib.rs | 12 + .../balls/trace/stack/src/Lowering.cpp | 66 + .../trace/workloads/ctests/CMakeLists.txt | 4 + .../ctests/bdb_counter_multi_id_test.c | 0 .../ctests/bdb_counter_nested_pair_test.c | 0 .../ctests/bdb_counter_read_active_test.c | 0 .../ctests/bdb_counter_start_stop_test.c | 0 .../balls/trace/workloads/isa/bdb_counter.h | 0 .../trace/workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../ball/bdb_counter_start_read_stop.mlir | 8 + .../ball/bdb_counter_start_read_stop_main.cpp | 0 .../components/balls/transpose/arch/README.md | 115 + .../balls/transpose/arch/Transpose.scala | 278 ++ .../balls/transpose/arch/TransposeBall.scala | 67 + .../configs/TransposeBallParam.scala | 20 + .../balls/transpose/configs/default.toml | 4 + .../transpose/configs/transpose_params.proto | 9 + .../balls/transpose/emu/src/dispatch.rs | 31 + .../components/balls/transpose/emu/src/lib.rs | 11 + .../balls/transpose/emu/src/transpose.rs | 2 +- .../balls/transpose/stack/src/Lowering.cpp | 55 + .../balls/transpose/verify/README.md | 91 + .../balls/transpose/verify/casegen/Cargo.lock | 0 .../balls/transpose/verify/casegen/Cargo.toml | 0 .../transpose/verify/casegen/src/casegen.rs | 183 + .../balls/transpose/verify/casegen/src/lib.rs | 71 + .../transpose/verify/casegen/src/model.rs | 0 .../verify/filelists/transpose_ball.f | 0 .../verify/src/common/transpose_defs.svh | 4 +- .../verify/src/common/transpose_items.svh | 4 +- .../verify/src/cov/transpose_cov.svh | 1 + .../verify/src/env/transpose_env.svh | 0 .../verify/src/env/transpose_scoreboard.svh | 2 + .../transpose/verify/src/pkg/transpose_pkg.sv | 0 .../verify/src/seq/transpose_sequences.svh | 0 .../balls/transpose/verify/src/tb_top.sv | 90 + .../verify/src/tests/transpose_test.svh | 0 .../transpose/workloads/ctests/CMakeLists.txt | 3 + .../ctests/transpose_i32_8x16_test.c | 4 +- .../ctests/transpose_i8_16x16_test.c | 4 +- .../ctests/transpose_i8_16x32_test.c | 4 +- .../balls/transpose/workloads/isa/transpose.h | 0 .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/transpose_16x16_i8.mlir | 38 + .../ball/transpose_16x16_i8_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../bank/transpose_i8_packed_matrix.mlir.in | 32 + .../bank/transpose_i8_packed_matrix_main.cpp | 0 .../balls/vector/arch}/README.md | 0 .../balls/vector/arch/VecBall.scala | 70 + .../balls/vector/arch/VecCtrlUnit.scala | 131 + .../balls/vector/arch}/VecEXUnit.scala | 27 +- .../balls/vector/arch/VecLoadUnit.scala | 203 + .../balls/vector/arch/VecMat16.scala | 222 ++ .../balls/vector/arch}/VecMat16AccLoad.scala | 45 +- .../balls/vector/arch/VecMat16Store.scala | 135 + .../balls/vector/arch}/VecMat16Tie.scala | 19 +- .../balls/vector/arch/VecStoreUnit.scala | 161 + .../balls/vector/arch/VecUnit.scala | 79 + .../balls/vector/arch}/bond/BondWrapper.scala | 0 .../balls/vector/arch}/bond/README.md | 0 .../balls/vector/arch/bond/vvv.scala | 27 + .../balls/vector/arch}/op/README.md | 0 .../balls/vector/arch}/op/cascade.scala | 14 +- .../balls/vector/arch}/op/mul.scala | 15 +- .../balls/vector/arch/thread/BaseThread.scala | 19 + .../balls/vector/arch/thread/CasThread.scala | 35 + .../balls/vector/arch/thread/MulThread.scala | 35 + .../balls/vector/arch}/thread/README.md | 0 .../balls/vector/arch}/warp/MeshWarp.scala | 30 +- .../balls/vector/arch}/warp/README.md | 0 .../vector/configs/VectorBallParam.scala | 31 + .../balls/vector/configs/default.toml | 8 + .../balls/vector/configs/vector_params.proto | 13 + .../balls/vector/emu/src/64_vecmat16.rs | 4 +- .../balls/vector/emu/src/dispatch.rs | 31 + .../components/balls/vector/emu/src/lib.rs | 12 + .../vector/workloads/ctests/CMakeLists.txt | 1 + .../ctests/vecmat16_atranspose_16x16_test.c | 0 .../balls/vector/workloads/isa/vecmat16.h | 0 .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../ball/vecmat16_atranspose_16x16.mlir | 0 .../ball/vecmat16_atranspose_16x16_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../vecmat16_atranspose_bank_depth.mlir.in | 0 .../vecmat16_atranspose_bank_depth_main.cpp | 0 examples/components/cores/boom/LICENSE | 29 + .../components/cores/boom/arch/Boom.scala | 202 + .../components/cores/boom/arch/Native.scala | 75 + examples/components/cores/boom/arch/build.sc | 46 + .../cores/boom/configs/Params.scala | 64 + .../cores/boom/configs/boom_params.proto | 7 + .../cores/boom/stack/CMakeLists.txt | 25 + .../boom/workloads/ctests/CMakeLists.txt | 1 + .../boom/workloads/ctests/memory_matrix.c | 47 + .../cores/reduce/arch/ReduceCore.scala | 126 + .../components/cores/reduce/arch/acc.scala | 119 + .../components/cores/reduce/arch/build.sc | 14 + .../components/cores/reduce/arch/loom.scala | 38 + .../components/cores/reduce/emu/Cargo.lock | 822 ++++ .../components/cores/reduce/emu/Cargo.toml | 9 + .../components/cores/reduce/emu/src/lib.rs | 39 + .../cores/reduce/stack/CMakeLists.txt | 41 + .../components/cores/rocket/LICENSE.Berkeley | 24 + .../components/cores/rocket/LICENSE.SiFive | 202 + .../cores/rocket/arch/Execution.scala | 219 ++ .../components/cores/rocket/arch/Lsu.scala | 194 + .../components/cores/rocket/arch/Rocket.scala | 163 + .../components/cores/rocket/arch/build.sc | 42 + .../cores/rocket/arch/pipeline/Config.scala | 492 +++ .../cores/rocket/arch/pipeline/Pipeline.scala | 1810 +++++++++ .../cores/rocket/arch/pipeline}/Utility.scala | 0 .../cores/rocket/configs/Params.scala | 107 + .../cores/rocket/configs/rocket_params.proto | 17 + .../components/cores/rocket/emu/Cargo.lock | 822 ++++ .../components/cores/rocket/emu/Cargo.toml | 9 + .../components/cores/rocket/emu/src/lib.rs | 27 + .../cores/rocket/stack/CMakeLists.txt | 25 + examples/components/cores/xiangshan/LICENSE | 127 + .../components/cores/xiangshan/arch/Chi.scala | 192 + .../cores/xiangshan/arch/Native.scala | 239 ++ .../cores/xiangshan/arch/XiangShan.scala | 255 ++ .../components/cores/xiangshan/arch/build.sc | 41 + .../cores/xiangshan/configs/Params.scala | 73 + .../xiangshan/configs/xiangshan_params.proto | 10 + .../cores/xiangshan/stack/CMakeLists.txt | 25 + .../xiangshan/workloads/ctests/CMakeLists.txt | 1 + .../workloads/ctests/memory_matrix.c | 47 + .../attention/arch/AttentionThread.scala | 77 + .../threads/attention/arch/acc.scala | 119 + .../configs/balldomains/default.toml | 53 + .../attention/configs/cpu/default.toml | 2 + .../threads/attention/configs/default.toml | 12 + .../attention/configs/frontends/default.toml | 9 + .../attention/configs/gpdomain}/default.toml | 0 .../attention/configs/memdomains/default.toml | 23 + .../threads/attention/stack/CMakeLists.txt | 157 + examples/components/threads/build.sc | 10 + .../threads/ffn/arch/FfnThread.scala | 77 + .../components/threads/ffn/arch/acc.scala | 119 + .../ffn/configs/balldomains/default.toml | 53 + .../threads/ffn/configs/cpu/default.toml | 2 + .../threads/ffn/configs/default.toml | 12 + .../ffn/configs/frontends/default.toml | 9 + .../ffn/configs/gpdomain}/default.toml | 0 .../ffn/configs/memdomains/default.toml | 23 + .../threads/ffn/stack/CMakeLists.txt | 157 + .../threads/matmul/arch/MatMulThread.scala | 75 + .../components/threads/matmul/arch/acc.scala | 119 + .../threads/matmul/configs/ant.toml | 5 + .../matmul/configs/balldomains/default.toml | 41 + .../threads/matmul/configs/cpu/default.toml | 2 + .../threads/matmul/configs/default.toml | 13 + .../matmul/configs/frontends/default.toml | 9 + .../matmul/configs/gpdomain}/default.toml | 0 .../matmul/configs/memdomains/default.toml | 23 + .../threads/matmul/stack/CMakeLists.txt | 157 + .../components/threads/matmul/stack/matmul.h | 5 + .../components/threads/matmul/stack/prepare.c | 32 + .../components/threads/matmul/stack/program.c | 17 + .../matmul/workloads/ctests/CMakeLists.txt | 14 + .../workloads/ctests/rvv/CMakeLists.txt | 76 + .../matmul}/workloads/ctests/rvv/matmul.cpp | 0 .../workloads/ctests/rvv/mxfp8_quant.cpp | 4 +- .../matmul/workloads/ctests/rvv/norm.cpp | 81 + .../matmul}/workloads/ctests/rvv/silu.cpp | 0 .../matmul}/workloads/ctests/rvv/snake.cpp | 0 .../matmul}/workloads/ctests/rvv/swiglu.cpp | 0 .../CMakeLists.txt | 3 + .../transpose_smatmul_quant_pipeline/goban.c | 53 + .../transpose_smatmul_quant_pipeline-main.cpp | 22 + examples/cores/README.md | 24 - .../ant/arch/src/main/scala/AntCore.scala | 19 - examples/cores/ant/configs/cpu/ant.toml | 2 - examples/cores/ant/configs/cpu/default.toml | 7 - examples/cores/ant/configs/default.toml | 2 - .../arch/src/main/scala/AttentionCore.scala | 19 - .../cores/attention/compiler/CMakeLists.txt | 12 - .../attention/compiler/src/CMakeLists.txt | 51 - .../compiler/src/Conversion/CMakeLists.txt | 2 - .../Conversion/LowerBuckyball/CMakeLists.txt | 16 - .../LowerBuckyball/LowerBuckyballPass.cpp | 127 - .../LowerBuckyballToBankSSAPass.cpp | 108 - .../LowerTileToBuckyball/CMakeLists.txt | 10 - .../LowerTileToBuckyball.cpp | 104 - .../Dialect/Buckyball/AttentionBuckyball.td | 26 - .../src/Dialect/Buckyball/CMakeLists.txt | 17 - .../src/Dialect/Buckyball/IR/CMakeLists.txt | 19 - .../Buckyball/Transforms/CMakeLists.txt | 14 - .../Transforms/LegalizeForLLVMExport.cpp | 45 - .../compiler/src/Dialect/CMakeLists.txt | 1 - .../compiler/src/Target/CMakeLists.txt | 1 - .../compiler/src/Target/LLVMIR/CMakeLists.txt | 1 - .../LLVMIR/Dialect/Buckyball/CMakeLists.txt | 15 - .../src/Target/LLVMIR/Dialect/CMakeLists.txt | 1 - .../configs/balldomains/default.toml | 23 - .../cores/attention/configs/cpu/default.toml | 2 - examples/cores/attention/configs/default.toml | 8 - .../attention/configs/frontends/default.toml | 8 - .../attention/configs/memdomains/default.toml | 32 - .../src/main/scala/configs/BoomCpuParam.scala | 32 - .../boom/configs/balldomains/default.toml | 9 - .../cores/boom/configs/cpu/boom/default.toml | 12 - examples/cores/boom/configs/cpu/default.toml | 2 - examples/cores/boom/configs/default.toml | 5 - .../boom/configs/memdomains/default.toml | 31 - .../ffn/arch/src/main/scala/FfnCore.scala | 19 - examples/cores/ffn/compiler/CMakeLists.txt | 12 - .../cores/ffn/compiler/src/CMakeLists.txt | 51 - .../compiler/src/Conversion/CMakeLists.txt | 2 - .../Conversion/LowerBuckyball/CMakeLists.txt | 16 - .../LowerBuckyball/LowerBuckyballPass.cpp | 126 - .../LowerBuckyballToBankSSAPass.cpp | 108 - .../LowerTileToBuckyball/CMakeLists.txt | 10 - .../LowerTileToBuckyball.cpp | 104 - .../src/Dialect/Buckyball/CMakeLists.txt | 17 - .../src/Dialect/Buckyball/FFNBuckyball.td | 26 - .../src/Dialect/Buckyball/IR/CMakeLists.txt | 19 - .../Buckyball/Transforms/CMakeLists.txt | 14 - .../Transforms/LegalizeForLLVMExport.cpp | 45 - .../ffn/compiler/src/Dialect/CMakeLists.txt | 1 - .../ffn/compiler/src/Target/CMakeLists.txt | 1 - .../compiler/src/Target/LLVMIR/CMakeLists.txt | 1 - .../LLVMIR/Dialect/Buckyball/CMakeLists.txt | 15 - .../src/Target/LLVMIR/Dialect/CMakeLists.txt | 1 - .../ffn/configs/balldomains/default.toml | 23 - examples/cores/ffn/configs/cpu/default.toml | 2 - examples/cores/ffn/configs/default.toml | 8 - .../cores/ffn/configs/frontends/default.toml | 8 - .../cores/ffn/configs/memdomains/default.toml | 32 - .../arch/src/main/scala/AntGobanCore.scala | 19 - .../goban/arch/src/main/scala/GobanCore.scala | 20 - examples/cores/goban/compiler/CMakeLists.txt | 12 - .../cores/goban/compiler/src/CMakeLists.txt | 51 - .../compiler/src/Conversion/CMakeLists.txt | 2 - .../Conversion/LowerBuckyball/CMakeLists.txt | 17 - .../LowerBuckyball/LowerBuckyballPass.cpp | 126 - .../LowerBuckyballToBankSSAPass.cpp | 93 - .../LowerTileToBuckyball/CMakeLists.txt | 10 - .../LowerTileToBuckyball.cpp | 115 - .../src/Dialect/Buckyball/CMakeLists.txt | 17 - .../src/Dialect/Buckyball/GobanBuckyball.td | 26 - .../src/Dialect/Buckyball/IR/CMakeLists.txt | 19 - .../Buckyball/Transforms/CMakeLists.txt | 14 - .../Transforms/LegalizeForLLVMExport.cpp | 73 - .../goban/compiler/src/Dialect/CMakeLists.txt | 1 - .../goban/compiler/src/Target/CMakeLists.txt | 1 - .../compiler/src/Target/LLVMIR/CMakeLists.txt | 1 - .../LLVMIR/Dialect/Buckyball/CMakeLists.txt | 15 - .../src/Target/LLVMIR/Dialect/CMakeLists.txt | 1 - examples/cores/goban/configs/ant.toml | 6 - .../goban/configs/balldomains/default.toml | 19 - examples/cores/goban/configs/cpu/ant.toml | 2 - examples/cores/goban/configs/cpu/default.toml | 2 - examples/cores/goban/configs/default.toml | 8 - .../goban/configs/frontends/default.toml | 8 - .../goban/configs/memdomains/default.toml | 32 - .../goban/workloads/ctests/CMakeLists.txt | 8 - .../goban/workloads/ctests/rvv/CMakeLists.txt | 67 - .../CMakeLists.txt | 3 - .../transpose_smatmul_quant_pipeline/goban.c | 53 - .../transpose_smatmul_quant_pipeline-main.cpp | 23 - .../arch/src/main/scala/PebbleCore.scala | 20 - examples/cores/pebble/compiler/CMakeLists.txt | 12 - .../cores/pebble/compiler/src/CMakeLists.txt | 70 - .../compiler/src/Conversion/CMakeLists.txt | 2 - .../Conversion/LowerBuckyball/CMakeLists.txt | 27 - .../LowerBuckyball/CoreBankSSALowering.cpp | 19 - .../LowerBuckyball/LowerBuckyballPass.cpp | 127 - .../LowerBuckyballToBankSSAPass.cpp | 125 - .../LowerTileToBuckyball/CMakeLists.txt | 10 - .../LowerTileToBuckyball/CoreTileLowering.cpp | 88 - .../src/Dialect/Buckyball/CMakeLists.txt | 17 - .../src/Dialect/Buckyball/IR/CMakeLists.txt | 19 - .../src/Dialect/Buckyball/PebbleBuckyball.td | 31 - .../Buckyball/Transforms/CMakeLists.txt | 20 - .../Transforms/LegalizeForLLVMExport.cpp | 131 - .../compiler/src/Dialect/CMakeLists.txt | 1 - .../pebble/compiler/src/Target/CMakeLists.txt | 1 - .../compiler/src/Target/LLVMIR/CMakeLists.txt | 1 - .../LLVMIR/Dialect/Buckyball/CMakeLists.txt | 15 - .../src/Target/LLVMIR/Dialect/CMakeLists.txt | 1 - .../pebble/configs/balldomains/default.toml | 32 - .../cores/pebble/configs/cpu/default.toml | 2 - .../pebble/configs/cpu/rocket/default.toml | 76 - examples/cores/pebble/configs/default.toml | 8 - .../pebble/configs/frontends/default.toml | 8 - .../pebble/configs/memdomains/default.toml | 31 - .../pebble/workloads/ctests/CMakeLists.txt | 1 - .../CMakeLists.txt | 3 - .../im2col_smatmul_quant_pipeline-main.cpp | 23 - .../im2col_smatmul_quant_pipeline/pebble.c | 63 - .../arch/src/main/scala/ReduceCore.scala | 20 - examples/cores/reduce/compiler/CMakeLists.txt | 18 - .../reduce/configs/balldomains/default.toml | 9 - .../cores/reduce/configs/cpu/default.toml | 2 - examples/cores/reduce/configs/default.toml | 6 - .../reduce/configs/frontends/default.toml | 8 - .../reduce/configs/memdomains/default.toml | 29 - .../cores/reduce/configs/rvv/default.toml | 6 - .../reduce/workloads/ctests/CMakeLists.txt | 9 - .../reduce/workloads/ctests/tlink_lease.c | 36 - .../arch/src/main/scala/RocketCore.scala | 17 - .../src/main/scala/cpu/AdmissionBridge.scala | 164 - .../arch/src/main/scala/cpu/Cache.scala | 75 - .../rocket/arch/src/main/scala/cpu/Lsu.scala | 252 -- .../src/main/scala/cpu/PermissionCheck.scala | 86 - .../arch/src/main/scala/cpu/Preparation.scala | 77 - .../arch/src/main/scala/cpu/Rocket.scala | 655 ---- examples/cores/rocket/compiler/CMakeLists.txt | 12 - .../rocket/configs/balldomains/default.toml | 9 - .../cores/rocket/configs/cpu/default.toml | 2 - .../rocket/configs/cpu/rocket/default.toml | 76 - examples/cores/rocket/configs/default.toml | 6 - .../rocket/configs/memdomains/default.toml | 32 - .../toy/arch/src/main/scala/ToyCore.scala | 20 - examples/cores/toy/compiler/CMakeLists.txt | 12 - .../Patterns/ToyLowerBuckyballPatterns.h | 22 - .../include/IntrinsicsRISCVBuckyball.td | 3 - .../include/IntrinsicsRISCVBuckyballBase.td | 10 - .../cores/toy/compiler/src/CMakeLists.txt | 61 - .../compiler/src/Conversion/CMakeLists.txt | 4 - .../Conversion/LowerBuckyball/CMakeLists.txt | 20 - .../LowerBuckyball/LowerBuckyballPass.cpp | 128 - .../LowerBuckyballToBankSSAPass.cpp | 48 - .../ToyAssignPhysicalBankPatterns.cpp | 11 - .../ToyLowerBuckyballToBankSSAPatterns.cpp | 24 - .../ToyLowerBuckyballToLLVMPatterns.cpp | 172 - .../LowerTileToBuckyball/CMakeLists.txt | 8 - .../LowerTileToBuckyball.cpp | 226 -- .../src/Dialect/Buckyball/CMakeLists.txt | 20 - .../src/Dialect/Buckyball/IR/CMakeLists.txt | 19 - .../src/Dialect/Buckyball/ToyBuckyball.td | 27 - .../Buckyball/Transforms/CMakeLists.txt | 11 - .../Transforms/LegalizeForLLVMExport.cpp | 51 - .../toy/compiler/src/Dialect/CMakeLists.txt | 5 - .../toy/compiler/src/Target/CMakeLists.txt | 3 - .../compiler/src/Target/LLVMIR/CMakeLists.txt | 3 - .../LLVMIR/Dialect/Buckyball/CMakeLists.txt | 15 - .../src/Target/LLVMIR/Dialect/CMakeLists.txt | 3 - .../balldomains/balls/smatmul/lane16.toml | 8 - .../toy/configs/balldomains/default.toml | 39 - examples/cores/toy/configs/cpu/default.toml | 2 - examples/cores/toy/configs/default.toml | 8 - examples/cores/toy/configs/demo.toml | 8 - .../cores/toy/configs/frontends/default.toml | 8 - .../cores/toy/configs/frontends/demo.toml | 8 - .../cores/toy/configs/memdomains/default.toml | 32 - .../cores/toy/workloads/ctests/CMakeLists.txt | 11 - .../CMakeLists.txt | 3 - .../gemmini_relu_trace_pipeline-main.cpp | 23 - examples/designs/goban/arch/build.sc | 17 + .../designs/goban/arch/device/DeviceTop.scala | 77 + .../goban/arch/device/core/GobanCore.scala | 88 + .../designs/goban/arch/device/core/loom.scala | 60 + examples/designs/goban/configs/design.toml | 4 + .../designs/goban/configs/device/clint.toml | 4 + .../goban/configs/device/core/c2c/goban.toml | 3 + .../configs/device/core/controller/goban.toml | 1 + .../goban/configs/device/core/default.toml | 60 + .../device/core/interconnect/goban.toml | 2 + .../device/core/memory/access/goban.toml | 52 + .../device/core/memory/dcache/goban.toml | 7 + .../device/core/memory/dcache/shared.toml | 7 + .../device/core/memory/dma-cache/shared.toml | 7 + .../configs/device/core/memory/goban.toml | 5 + .../device/core/memory/home/goban.toml | 13 + .../device/core/memory/icache/shared.toml | 7 + .../goban/configs/device/core/rocket.toml | 12 + .../configs/device/core/tbus/default.toml | 1 + .../designs/goban/configs/device/ddr.toml | 13 + .../designs/goban/configs/device/default.toml | 9 + .../designs/goban/configs/device/plic.toml | 5 + .../designs/goban/configs/sims/default.toml | 14 + examples/designs/goban/emu/Cargo.lock | 821 ++++ examples/designs/goban/emu/Cargo.toml | 9 + .../designs/goban/emu/src/device/core/mod.rs | 37 + examples/designs/goban/emu/src/device/mod.rs | 2 + examples/designs/goban/emu/src/lib.rs | 1 + .../goban/sims/device/arch/DeviceConfig.scala | 18 + .../designs/goban/sims/device/arch/build.sc | 17 + .../designs/goban/sims/device/emu/Cargo.lock | 2008 ++++++++++ .../designs/goban/sims/device/emu/Cargo.toml | 83 + .../designs/goban/sims/device/emu/build.rs | 7 + .../designs/goban/sims/device/emu/src/lib.rs | 11 + .../regression/bemu/workloads-bare.toml | 7 + .../regression/bemu/workloads-linux.toml | 4 + .../regression/p2e/workloads-linux.toml | 4 + .../regression/verilator/workloads-bare.toml | 7 + .../goban/sims/device/stack/CMakeLists.txt | 48 + .../device/stack/core/goban/CMakeLists.txt | 6 + .../sims/device/stack/runtime/CMakeLists.txt | 37 + .../sims/device/workloads/CMakeLists.txt | 78 + .../device/workloads/kernel/CMakeLists.txt | 11 + .../designs/goban/sims/ip/uvm/registry.toml | 1 + .../designs/goban/stack/engine/CMakeLists.txt | 14 + .../designs/goban/stack/midend/CMakeLists.txt | 8 + .../designs/goban/workloads/CMakeLists.txt | 11 + .../goban/workloads/configs/tests.toml | 2 + .../goban/workloads/device/CMakeLists.txt | 1 + .../workloads/device/core/CMakeLists.txt | 1 + .../device/core/engine/CMakeLists.txt | 40 + .../device/core/engine/matmul_threads.c | 61 + .../device/core/engine}/mvover_rows.c | 19 +- examples/designs/multi-rocket/arch/build.sc | 17 + .../multi-rocket/arch/device/DeviceTop.scala | 80 + .../designs/multi-rocket/configs/design.toml | 4 + .../multi-rocket/configs/device/clint.toml | 4 + .../configs/device/core/c2c/core.toml | 3 + .../configs/device/core/default.toml | 30 + .../device/core/interconnect/core.toml | 2 + .../device/core/memory/access/core.toml | 52 + .../configs/device/core/memory/core.toml | 4 + .../device/core/memory/dcache/core.toml | 6 + .../device/core/memory/dcache/shared.toml | 6 + .../configs/device/core/memory/home/core.toml | 13 + .../device/core/memory/icache/shared.toml | 6 + .../configs/device/core/rocket.toml | 12 + .../multi-rocket/configs/device/ddr.toml | 13 + .../multi-rocket/configs/device/default.toml | 517 +++ .../multi-rocket/configs/device/plic.toml | 5 + .../multi-rocket/configs/sims/default.toml | 14 + .../sims/device/arch/DeviceConfig.scala | 18 + .../multi-rocket/sims/device/arch/build.sc | 17 + .../multi-rocket/sims/device/emu/Cargo.lock | 1971 ++++++++++ .../multi-rocket/sims/device/emu/Cargo.toml | 83 + .../multi-rocket/sims/device/emu/build.rs | 7 + .../multi-rocket/sims/device/emu/src/lib.rs | 11 + .../regression/bemu/workloads-bare.toml | 25 + .../regression}/bemu/workloads-linux.toml | 0 .../device/regression/p2e/workloads-bare.toml | 21 + .../regression}/p2e/workloads-linux.toml | 0 .../regression/verilator/workloads-bare.toml | 19 + .../sims/device/stack/CMakeLists.txt | 37 + .../sims/device/stack/runtime/CMakeLists.txt | 32 + .../sims/device/workloads/CMakeLists.txt | 78 + .../device/workloads/kernel/CMakeLists.txt | 4 + .../multi-rocket/sims/ip/uvm/registry.toml | 1 + .../multi-rocket/workloads/CMakeLists.txt | 5 + .../multi-rocket/workloads/configs/tests.toml | 2 + .../workloads/device/CMakeLists.txt | 2 + .../workloads/device/core/CMakeLists.txt | 1 + .../device/core/ctests/CMakeLists.txt | 21 + .../device/core/ctests}/cpu_arithmetic.c | 0 .../workloads/device/soc/CMakeLists.txt | 12 + .../workloads/device/soc/coherent_exchange.c | 30 + examples/designs/pebble/arch/build.sc | 18 + .../pebble/arch/device/DeviceTop.scala | 73 + .../pebble/arch/device/core/PebbleCore.scala | 46 + .../designs/pebble/arch/device/core/acc.scala | 113 + .../pebble/arch/device/core/loom.scala | 51 + examples/designs/pebble/configs/design.toml | 5 + .../device/core/acc/ablation-bat0-blt0.toml | 8 + .../device/core/acc/ablation-bat0-blt1.toml | 8 + .../device/core/acc/ablation-bat1-blt0.toml | 8 + .../acc/balldomain}/balls/smatmul/8x16.toml | 1 - .../device/core/acc/balldomain/default.toml | 102 + .../configs/device/core/acc/default.toml | 8 + .../core/acc/frontend/bat-off-rob24.toml | 9 + .../device/core/acc/frontend/rob24.toml | 9 + .../device/core/acc/memdomain/blt-off.toml | 22 + .../pebble/configs/device/core/acc/rob24.toml | 8 + .../configs/device/core/c2c/pebble.toml | 3 + .../pebble/configs/device/core/default.toml | 35 + .../device/core/interconnect/pebble.toml | 2 + .../device/core/memory/access/pebble.toml | 52 + .../device/core/memory/dcache/pebble.toml | 15 + .../device/core/memory/home/pebble.toml | 13 + .../device/core/memory/icache/pebble.toml | 7 + .../configs/device/core/memory/pebble.toml | 5 + .../pebble/configs/device/ddr/pebble.toml | 13 + .../pebble/configs/device/default.toml | 8 + .../designs/pebble/configs/sims/default.toml | 15 + examples/designs/pebble/emu/Cargo.lock | 821 ++++ examples/designs/pebble/emu/Cargo.toml | 9 + .../designs/pebble/emu/src/device/core/mod.rs | 39 + examples/designs/pebble/emu/src/device/mod.rs | 2 + examples/designs/pebble/emu/src/lib.rs | 1 + .../sims/device/arch/DeviceConfig.scala | 18 + .../designs/pebble/sims/device/arch/build.sc | 17 + .../designs/pebble/sims/device/emu/Cargo.lock | 2008 ++++++++++ .../designs/pebble/sims/device/emu/Cargo.toml | 83 + .../designs/pebble/sims/device/emu/build.rs | 7 + .../designs/pebble/sims/device/emu/src/lib.rs | 11 + .../regression/bemu/workloads-bare.toml | 112 + .../regression/bemu/workloads-linux.toml | 4 + .../device/regression/p2e/workloads-bare.toml | 33 + .../regression}/p2e/workloads-linux.toml | 0 .../device/regression/vcs/workloads-bare.toml | 11 + .../regression/verilator/workloads-bare.toml | 101 + .../pebble/sims/device/stack/CMakeLists.txt | 37 + .../device/stack/core/pebble/CMakeLists.txt | 6 + .../sims/device/stack/runtime/CMakeLists.txt | 37 + .../pebble/sims/device/tapeout/README.md | 10 + .../sims/device}/tapeout/area/.gitignore | 0 .../sims/device}/tapeout/area/constraints.sdc | 0 .../pebble/sims/device/tapeout/area/dc.tcl | 65 + .../pebble/sims/device/tapeout/config.toml | 117 + .../tapeout/ip/smic180_sram_library.mdf.json | 0 .../tapeout/ip/tsmc28_sram_library.mdf.json | 0 .../sims/device}/tapeout/power/power.tcl | 0 .../sims/device}/tapeout/power/power_sim.sh | 4 +- .../sims/device/workloads/CMakeLists.txt | 78 + .../device/workloads/kernel/CMakeLists.txt | 42 + .../designs/pebble/sims/ip/uvm/registry.toml | 1 + .../pebble/stack/engine/CMakeLists.txt | 13 + .../pebble/stack/midend/CMakeLists.txt | 32 + .../midend/src/ConvertLinalgToBuckyball.cpp | 328 ++ .../pebble/stack/midend/src/FormCustomOps.cpp | 295 ++ .../stack/midend/src/FormSpatialOps.cpp | 276 ++ .../stack/midend/src/LowerLinearTiles.cpp | 240 ++ .../stack/midend/src/LowerQuantizedOps.cpp | 326 ++ .../stack/midend/src/LowerSpatialOps.cpp | 525 +++ .../pebble/stack/midend/src/Lowering.h | 33 + .../stack/midend/src/QuantizedChains.cpp | 453 +++ .../stack/midend/tests/form-custom-ops.mlir | 83 + .../stack/midend/tests/form-depthwise.mlir | 33 + .../stack/midend/tests/form-int32-bias.mlir | 61 + .../stack/midend/tests/form-linear-chain.mlir | 83 + .../midend/tests/form-quantized-chain.mlir | 19 + .../midend/tests/lower-bank-updates.mlir | 21 + .../stack/midend/tests/lower-custom-ops.mlir | 26 + .../midend/tests/lower-frontend-types.mlir | 27 + .../stack/midend/tests/unknown-custom-op.mlir | 5 + .../designs/pebble/workloads/CMakeLists.txt | 24 + .../pebble/workloads/common/pebble.c | 0 .../pebble/workloads/common/pebble.h | 0 .../pebble/workloads/configs/tests.toml | 2 + .../pebble/workloads/device/CMakeLists.txt | 1 + .../workloads/device/core/CMakeLists.txt | 1 + .../workloads/device/core/acc/CMakeLists.txt | 10 + .../device/core/acc/ctests/CMakeLists.txt | 28 + .../acc/ctests/bank_capacity_rename_test.c | 36 + .../ctests/bank_dma_pingpong/CMakeLists.txt | 3 + .../bank_dma_pingpong-main.cpp | 22 + .../acc/ctests/bank_dma_pingpong/pebble.c | 55 + .../acc/ctests/bank_dma_reuse/CMakeLists.txt | 3 + .../bank_dma_reuse/bank_dma_reuse-main.cpp | 22 + .../core/acc/ctests/bank_dma_reuse/pebble.c | 62 + .../core/acc/ctests/bank_recycle_test.c | 25 + .../ctests/bank_rename_overlap/CMakeLists.txt | 3 + .../bank_rename_overlap-main.cpp | 22 + .../acc/ctests/bank_rename_overlap/pebble.c | 54 + .../acc/ctests/bank_transfer_recycle_test.c | 31 + .../acc/ctests/dual_conv_add_int2fp_test.c | 80 + .../acc}/ctests/dual_smatmul_matadd_test.c | 0 .../ctests/im2col_mvin_smatmul_hazard_test.c | 0 .../CMakeLists.txt | 3 + .../im2col_smatmul_quant_pipeline-main.cpp | 22 + .../im2col_smatmul_quant_pipeline/pebble.c | 61 + .../core/acc}/ctests/maxpool_im2col_test.c | 0 .../maxpool_partial_update/CMakeLists.txt | 3 + .../maxpool_partial_update-main.cpp | 22 + .../ctests/maxpool_partial_update/pebble.c | 40 + .../acc/ctests/mega_conv_k5_pipeline_test.c | 70 + .../acc}/ctests/mega_conv_pipeline_test.c | 0 .../acc}/ctests/mvin_bank78_stress_test.c | 0 .../acc}/ctests/quant_smatmul_int2fp_test.c | 0 .../ctests/read_bank_shared/CMakeLists.txt | 3 + .../core/acc/ctests/read_bank_shared/pebble.c | 52 + .../read_bank_shared-main.cpp | 22 + .../acc/ctests/read_bank_split/CMakeLists.txt | 3 + .../core/acc/ctests/read_bank_split/pebble.c | 52 + .../read_bank_split/read_bank_split-main.cpp | 22 + .../core/acc}/ctests/residual_add_lut_test.c | 0 .../ctests/rob_window_probe/CMakeLists.txt | 3 + .../core/acc/ctests/rob_window_probe/pebble.c | 57 + .../rob_window_probe-main.cpp | 22 + .../device/core/acc/ctests/runtime_job_test.c | 53 + .../same_ball_independent/CMakeLists.txt | 3 + .../acc/ctests/same_ball_independent/pebble.c | 56 + .../same_ball_independent-main.cpp | 22 + .../core/acc}/ctests/se_gate_lut_test.c | 0 .../shared_bank_versions/CMakeLists.txt | 3 + .../acc/ctests/shared_bank_versions/pebble.c | 47 + .../shared_bank_versions-main.cpp | 22 + .../ctests/smatmul_mvout_bank_reuse_test.c | 0 .../ctests/write_bank_versions/CMakeLists.txt | 3 + .../acc/ctests/write_bank_versions/pebble.c | 42 + .../write_bank_versions-main.cpp | 22 + .../device/core/acc/mlir_tests/CMakeLists.txt | 32 + .../core/acc}/mlir_tests/bank/CMakeLists.txt | 0 .../bank/quant_im2col_k1_single_pixel.mlir | 30 + .../quant_im2col_k1_single_pixel_main.cpp | 0 .../core/acc/mlir_tests/core/CMakeLists.txt | 63 + .../mlir_tests/core/core_matmul_16x16_i8.mlir | 40 + .../core/core_matmul_16x16_i8_main.cpp | 29 + .../core/core_transpose_16x16_i8.mlir | 27 + .../core/core_transpose_16x16_i8_main.cpp | 25 + .../data/mega_global_avg_7x7_c288_input.i8 | Bin .../mega_global_avg_conv2_exact_weight2.i8 | Bin .../mega_global_avg_conv2_exact_weight3.i8 | Bin .../data/mega_global_avg_conv_exact_input.i8 | Bin .../data/mega_global_avg_conv_exact_weight.i8 | Bin .../mlir_tests/core/generic_matmul_i8.mlir | 52 + .../core/generic_matmul_i8_main.cpp | 33 + .../acc/mlir_tests/core/linear_chain_i8.mlir | 93 + .../mlir_tests/core/linear_chain_i8_main.cpp | 60 + .../core/acc/mlir_tests/core/linear_int8.mlir | 73 + .../acc/mlir_tests/core/linear_int8_main.cpp | 58 + .../mlir_tests/core/lookup_channel_i8.mlir | 40 + .../core/lookup_channel_i8_main.cpp | 20 + .../core/acc/mlir_tests/core/mega_conv2d.mlir | 100 + .../mlir_tests/core/mega_conv2d_k5_tail.mlir | 59 + .../core}/mega_conv2d_k5_tail_main.cpp | 2 +- .../acc/mlir_tests/core}/mega_conv2d_main.cpp | 0 .../core/mega_conv2d_s2_depthwise_s2.mlir | 88 + .../mega_conv2d_s2_depthwise_s2_main.cpp | 10 +- .../mega_global_avg_7x7_c288_exact.mlir.in | 16 + .../mega_global_avg_7x7_c288_exact_main.cpp | 0 .../core/mega_global_avg_conv2_exact.mlir.in | 84 + .../mega_global_avg_conv2_exact_main.cpp | 0 .../core/mega_global_avg_conv_exact.mlir.in | 33 + .../core}/mega_global_avg_conv_exact_main.cpp | 0 .../core/acc/mlir_tests/core/mega_matmul.mlir | 45 + .../core/mega_matmul_m17_k65_n17.mlir | 50 + .../core}/mega_matmul_m17_k65_n17_main.cpp | 0 .../acc/mlir_tests/core}/mega_matmul_main.cpp | 0 .../mlir_tests/core/mega_se_gate_conv.mlir | 99 + .../core}/mega_se_gate_conv_main.cpp | 0 .../core/acc/mlir_tests/linalg/CMakeLists.txt | 15 + .../linalg/linalg_matmul_16x16_i8.mlir | 41 + .../linalg/linalg_matmul_16x16_i8_main.cpp | 0 .../linalg/linalg_mega_conv2d_k3_c16.mlir | 76 + .../linalg/linalg_mega_conv2d_k3_c16_main.cpp | 0 .../linalg/linalg_relu_513x17_i32.mlir | 2 - .../linalg/linalg_relu_513x17_i32_main.cpp | 0 .../linalg/linalg_transpose_16x16_i8.mlir | 27 + .../linalg/linalg_transpose_16x16_i8_main.cpp | 0 .../linalg_transpose_detect_1x84x100_f32.mlir | 2 - ...alg_transpose_detect_1x84x100_f32_main.cpp | 0 ...inalg_transpose_nchw_nhwc_1x3x8x8_f32.mlir | 2 - ...g_transpose_nchw_nhwc_1x3x8x8_f32_main.cpp | 0 ...inalg_transpose_oihw_ohwi_4x8x3x3_f32.mlir | 2 - ...g_transpose_oihw_ohwi_4x8x3x3_f32_main.cpp | 0 examples/designs/poly/arch/build.sc | 19 + .../designs/poly/arch/device/DeviceTop.scala | 90 + .../poly/arch/device/core/PolyCore.scala | 154 + examples/designs/poly/configs/design.toml | 5 + .../designs/poly/configs/device/clint.toml | 4 + .../cores/core0/acc/balldomain/default.toml | 9 + .../device/cores/core0/acc/default.toml | 6 + .../device/cores/core0/acc/frontend/poly.toml | 9 + .../cores/core0/acc/memdomain/poly.toml | 20 + .../configs/device/cores/core0/c2c/core.toml | 3 + .../device/cores/core0/controller/core.toml | 1 + .../configs/device/cores/core0/default.toml | 37 + .../device/cores/core0/dmaCache/core.toml | 7 + .../device/cores/core0/interconnect/core.toml | 2 + .../cores/core0/memory/access/core.toml | 52 + .../device/cores/core0/memory/core.toml | 5 + .../cores/core0/memory/dcache/core.toml | 15 + .../device/cores/core0/memory/home/core.toml | 13 + .../cores/core0/memory/icache/core.toml | 7 + .../configs/device/cores/core0/tbus/core.toml | 1 + .../configs/device/cores/core1/c2c/core.toml | 3 + .../device/cores/core1/controller/core.toml | 1 + .../configs/device/cores/core1/default.toml | 71 + .../device/cores/core1/dmaCache/core.toml | 7 + .../device/cores/core1/interconnect/core.toml | 2 + .../cores/core1/memory/access/core.toml | 52 + .../device/cores/core1/memory/core.toml | 5 + .../cores/core1/memory/dcache/core.toml | 15 + .../device/cores/core1/memory/home/core.toml | 13 + .../cores/core1/memory/icache/core.toml | 7 + .../configs/device/cores/core1/tbus/core.toml | 1 + .../configs/device/cores/core1/transport.toml | 82 + .../configs/device/cores/core2/c2c/core.toml | 3 + .../device/cores/core2/controller/core.toml | 1 + .../configs/device/cores/core2/default.toml | 71 + .../device/cores/core2/dmaCache/core.toml | 7 + .../device/cores/core2/interconnect/core.toml | 2 + .../cores/core2/memory/access/core.toml | 52 + .../device/cores/core2/memory/core.toml | 5 + .../cores/core2/memory/dcache/core.toml | 15 + .../device/cores/core2/memory/home/core.toml | 13 + .../cores/core2/memory/icache/core.toml | 7 + .../configs/device/cores/core2/tbus/core.toml | 1 + .../configs/device/cores/core3/c2c/core.toml | 3 + .../device/cores/core3/controller/core.toml | 1 + .../configs/device/cores/core3/default.toml | 71 + .../device/cores/core3/dmaCache/core.toml | 7 + .../device/cores/core3/interconnect/core.toml | 2 + .../cores/core3/memory/access/core.toml | 52 + .../device/cores/core3/memory/core.toml | 5 + .../cores/core3/memory/dcache/core.toml | 15 + .../device/cores/core3/memory/home/core.toml | 13 + .../cores/core3/memory/icache/core.toml | 7 + .../configs/device/cores/core3/tbus/core.toml | 1 + .../configs/device/cores/core4/c2c/core.toml | 3 + .../device/cores/core4/controller/core.toml | 1 + .../configs/device/cores/core4/default.toml | 71 + .../device/cores/core4/dmaCache/core.toml | 7 + .../device/cores/core4/interconnect/core.toml | 2 + .../cores/core4/memory/access/core.toml | 52 + .../device/cores/core4/memory/core.toml | 5 + .../cores/core4/memory/dcache/core.toml | 15 + .../device/cores/core4/memory/home/core.toml | 13 + .../cores/core4/memory/icache/core.toml | 7 + .../configs/device/cores/core4/tbus/core.toml | 1 + .../configs/device/cores/core5/c2c/core.toml | 3 + .../device/cores/core5/controller/core.toml | 1 + .../configs/device/cores/core5/default.toml | 71 + .../device/cores/core5/dmaCache/core.toml | 7 + .../device/cores/core5/interconnect/core.toml | 2 + .../cores/core5/memory/access/core.toml | 52 + .../device/cores/core5/memory/core.toml | 5 + .../cores/core5/memory/dcache/core.toml | 15 + .../device/cores/core5/memory/home/core.toml | 13 + .../cores/core5/memory/icache/core.toml | 7 + .../configs/device/cores/core5/tbus/core.toml | 1 + .../configs/device/cores/core6/c2c/core.toml | 3 + .../device/cores/core6/controller/core.toml | 1 + .../configs/device/cores/core6/default.toml | 71 + .../device/cores/core6/dmaCache/core.toml | 7 + .../device/cores/core6/interconnect/core.toml | 2 + .../cores/core6/memory/access/core.toml | 52 + .../device/cores/core6/memory/core.toml | 5 + .../cores/core6/memory/dcache/core.toml | 15 + .../device/cores/core6/memory/home/core.toml | 13 + .../cores/core6/memory/icache/core.toml | 7 + .../configs/device/cores/core6/tbus/core.toml | 1 + .../configs/device/cores/core7/c2c/core.toml | 3 + .../device/cores/core7/controller/core.toml | 1 + .../configs/device/cores/core7/default.toml | 71 + .../device/cores/core7/dmaCache/core.toml | 7 + .../device/cores/core7/interconnect/core.toml | 2 + .../cores/core7/memory/access/core.toml | 52 + .../device/cores/core7/memory/core.toml | 5 + .../cores/core7/memory/dcache/core.toml | 15 + .../device/cores/core7/memory/home/core.toml | 13 + .../cores/core7/memory/icache/core.toml | 7 + .../configs/device/cores/core7/tbus/core.toml | 1 + .../configs/device/cores/core8/c2c/core.toml | 3 + .../device/cores/core8/controller/core.toml | 1 + .../configs/device/cores/core8/default.toml | 71 + .../device/cores/core8/dmaCache/core.toml | 7 + .../device/cores/core8/interconnect/core.toml | 2 + .../cores/core8/memory/access/core.toml | 52 + .../device/cores/core8/memory/core.toml | 5 + .../cores/core8/memory/dcache/core.toml | 15 + .../device/cores/core8/memory/home/core.toml | 13 + .../cores/core8/memory/icache/core.toml | 7 + .../configs/device/cores/core8/tbus/core.toml | 1 + examples/designs/poly/configs/device/ddr.toml | 13 + .../designs/poly/configs/device/default.toml | 32 + .../designs/poly/configs/device/plic.toml | 5 + .../designs/poly/configs/sims/default.toml | 14 + examples/designs/poly/emu/Cargo.lock | 821 ++++ examples/designs/poly/emu/Cargo.toml | 9 + .../designs/poly/emu/src/device/core/mod.rs | 37 + examples/designs/poly/emu/src/device/mod.rs | 2 + examples/designs/poly/emu/src/lib.rs | 1 + .../poly/sims/device/arch/DeviceConfig.scala | 18 + .../designs/poly/sims/device/arch/build.sc | 17 + .../designs/poly/sims/device/emu/Cargo.lock | 1979 ++++++++++ .../designs/poly/sims/device/emu/Cargo.toml | 84 + .../designs/poly/sims/device/emu/build.rs | 7 + .../designs/poly/sims/device/emu/src/lib.rs | 12 + .../regression/bemu/workloads-bare.toml | 19 + .../regression}/bemu/workloads-linux.toml | 0 .../regression}/p2e/workloads-bare.toml | 0 .../regression}/p2e/workloads-linux.toml | 0 .../regression/verilator/workloads-bare.toml | 19 + .../poly/sims/device/stack/CMakeLists.txt | 67 + .../device/stack/core/poly/CMakeLists.txt | 6 + .../sims/device/stack/runtime/CMakeLists.txt | 38 + .../poly/sims/device/workloads/CMakeLists.txt | 78 + .../device/workloads/kernel/CMakeLists.txt | 10 + .../designs/poly/sims/ip/uvm/registry.toml | 1 + .../designs/poly/stack/engine/CMakeLists.txt | 14 + examples/designs/poly/stack/engine/execute.c | 23 + .../designs/poly/stack/midend/CMakeLists.txt | 8 + .../designs/poly/workloads/CMakeLists.txt | 17 + .../designs/poly/workloads/configs/tests.toml | 2 + .../poly/workloads/device/CMakeLists.txt | 3 + .../device/common/controllers_start.S | 1 + .../poly/workloads/device/core/CMakeLists.txt | 5 + .../workloads/device/core/ball/CMakeLists.txt | 5 + .../device/core/engine/CMakeLists.txt | 122 + .../device/core/engine/ant_bank_exercise.c | 67 + .../device/core/engine/ant_bank_main.c | 50 + .../core/engine/ant_bank_rows_exercise.c | 65 + .../device/core/engine/ant_bank_rows_main.c | 50 + .../device/core/engine/ant_highpa_main.c | 68 + .../device/core/engine/ant_mxmm_main.cpp | 100 + .../core/engine/ant_parallel_exercise.c | 82 + .../device/core/engine/ant_parallel_main.c | 50 + .../device/core/engine/ant_shared_exercise.c | 70 + .../device/core/engine/ant_shared_main.c | 45 + .../device/core/thread/CMakeLists.txt | 3 + .../device/core/thread/ctests/CMakeLists.txt | 94 + .../device/core/thread}/ctests/ant_bank.c | 3 +- .../device/core/thread}/ctests/ant_bank.h | 2 +- .../core/thread}/ctests/ant_bank_rows.c | 3 +- .../core/thread}/ctests/ant_bank_rows.h | 2 +- .../core/thread}/ctests/ant_coherence.c | 2 +- .../core/thread}/ctests/ant_coherence.h | 0 .../device/core/thread/ctests/ant_corelink.c | 17 + .../device/core/thread/ctests/ant_corelink.h | 9 + .../core/thread/ctests/ant_corelink_bank.c | 17 + .../core/thread/ctests/ant_corelink_bank.h | 9 + .../core/thread/ctests/ant_corelink_reduce.c | 22 + .../core/thread/ctests/ant_corelink_reduce.h | 8 + .../device/core/thread}/ctests/ant_highpa.c | 2 +- .../device/core/thread}/ctests/ant_highpa.h | 0 .../device/core/thread/ctests/ant_mesh.h | 5 + .../core/thread}/ctests/ant_mesh_move.c | 2 +- .../core/thread}/ctests/ant_mesh_move.h | 0 .../device/core/thread}/ctests/ant_mxmm.c | 3 +- .../device/core/thread}/ctests/ant_mxmm.h | 0 .../device/core/thread}/ctests/ant_parallel.c | 2 +- .../device/core/thread}/ctests/ant_parallel.h | 2 +- .../device/core/thread}/ctests/ant_shared.c | 2 +- .../device/core/thread/ctests/ant_shared.h | 9 + .../ctests/mesh_private_bank_move_test.c | 55 + .../core/thread/ctests/mesh_shared_mem_test.c | 22 + .../core/thread}/ctests/mxmm_fp32_small.c | 0 .../core/thread}/ctests/mxmm_mxfp8_small.c | 0 .../workloads/device/runtime/CMakeLists.txt | 147 + .../device/runtime/ant_coherence_main.c | 94 + .../device/runtime/ant_corelink_bank_main.c | 97 + .../device/runtime/ant_corelink_main.c | 93 + .../runtime/ant_corelink_reduce_control.c | 41 + .../device/runtime/ant_corelink_reduce_main.c | 99 + .../workloads/device/runtime/ant_mesh_main.c | 81 + .../device/runtime/ant_mesh_move_main.c | 94 + .../device/runtime/ctests/CMakeLists.txt | 7 + .../device/runtime/ctests/corelink_lease.c | 36 + .../workloads/device/runtime/stream_main.c | 59 + .../poly/workloads/device/soc/CMakeLists.txt | 37 + .../poly/workloads/device/soc}/controllers.c | 8 +- examples/designs/toy/arch/build.sc | 17 + .../designs/toy/arch/device/DeviceTop.scala | 72 + .../toy/arch/device/core/ToyCore.scala | 50 + .../designs/toy/arch/device/core/acc.scala | 114 + .../designs/toy/arch/device/core/loom.scala | 48 + examples/designs/toy/configs/design.toml | 5 + .../acc/balldomain/balls/smatmul/lane16.toml | 7 + .../device/core/acc/balldomain/default.toml | 61 + .../toy/configs/device/core/acc/default.toml | 6 + .../toy/configs/device/core/acc/demo.toml | 7 + .../device/core/acc/frontend/demo.toml | 9 + .../configs/device/core/acc/frontend/toy.toml | 9 + .../device/core/acc/gpdomain/toy.toml} | 0 .../device/core/acc/memdomain/paper.toml | 23 + .../device/core/acc/memdomain/toy.toml | 23 + .../toy/configs/device/core/acc/paper.toml | 7 + .../toy/configs/device/core/c2c/core.toml | 3 + .../toy/configs/device/core/default.toml | 39 + .../device/core/interconnect/core.toml | 2 + .../device/core/memory/access/core.toml | 52 + .../toy/configs/device/core/memory/core.toml | 5 + .../device/core/memory/dcache/core.toml | 15 + .../configs/device/core/memory/home/core.toml | 13 + .../device/core/memory/icache/core.toml | 7 + .../toy/configs/device/core/rocket.toml | 12 + examples/designs/toy/configs/device/ddr.toml | 13 + .../designs/toy/configs/device/default.toml | 8 + .../designs/toy/configs/sims/default.toml | 15 + examples/designs/toy/emu/Cargo.lock | 821 ++++ examples/designs/toy/emu/Cargo.toml | 9 + .../designs/toy/emu/src/device/core/mod.rs | 39 + examples/designs/toy/emu/src/device/mod.rs | 2 + examples/designs/toy/emu/src/lib.rs | 1 + .../toy/sims/device/arch/DeviceConfig.scala | 18 + .../designs/toy/sims/device/arch/build.sc | 17 + .../designs/toy/sims/device/emu/Cargo.lock | 1971 ++++++++++ .../designs/toy/sims/device/emu/Cargo.toml | 87 + examples/designs/toy/sims/device/emu/build.rs | 7 + .../designs/toy/sims/device/emu/src/lib.rs | 11 + .../device/emu/tests/instance_isolation.rs | 51 + .../regression/bemu/workloads-bare.toml | 56 + .../regression}/bemu/workloads-linux.toml | 0 .../device/regression/p2e/workloads-bare.toml | 24 + .../regression}/p2e/workloads-linux.toml | 0 .../toy/sims/device/regression/soc}/README.md | 0 .../regression/verilator/workloads-bare.toml | 42 + .../regression/verilator/workloads-linux.toml | 32 + .../regression/workloads-verilator.toml | 5 + .../toy/sims/device/stack/CMakeLists.txt | 37 + .../sims/device/stack/core/toy/CMakeLists.txt | 6 + .../sims/device/stack/runtime/CMakeLists.txt | 37 + .../sims/device/tapeout/area/constraints.sdc | 8 + .../toy/sims/device/tapeout/area/dc.tcl | 67 + .../toy/sims/device/tapeout/config.toml | 24 + .../tapeout/ip/smic180_sram_library.mdf.json | 1 + .../toy/sims/device/workloads/CMakeLists.txt | 75 + .../device/workloads/kernel/CMakeLists.txt | 46 + .../toy/sims/ip/uvm/configs/axis16.toml | 6 + .../designs/toy/sims/ip/uvm/registry.toml | 7 + .../designs/toy/stack/engine/CMakeLists.txt | 13 + .../designs/toy/stack/midend/CMakeLists.txt | 11 + examples/designs/toy/workloads/CMakeLists.txt | 17 + .../designs/toy/workloads/configs/tests.toml | 2 + .../toy/workloads/device/CMakeLists.txt | 2 + .../toy/workloads/device/core/CMakeLists.txt | 1 + .../workloads/device/core/acc/CMakeLists.txt | 10 + .../device/core/acc/ctests/CMakeLists.txt | 22 + .../CMakeLists.txt | 3 + .../gemmini_relu_trace_pipeline-main.cpp | 22 + .../ctests/gemmini_relu_trace_pipeline/toy.c | 8 +- .../core/acc}/ctests/mmio_minimal_test.c | 6 +- .../device/core/acc}/ctests/rvv_ball.c | 14 +- .../device/core/acc/mlir_tests/CMakeLists.txt | 8 + .../core/acc/mlir_tests/linalg/CMakeLists.txt | 27 + .../mlir_tests/linalg/conv2d_nchw_fchw.mlir | 0 .../linalg/conv2d_nchw_fchw_main.cpp | 0 .../mlir_tests/linalg/conv2d_nhwc_fhwc.mlir | 0 .../linalg/conv2d_nhwc_fhwc_main.cpp | 0 .../mlir_tests/linalg/conv2d_nhwc_hwcf.mlir | 0 .../linalg/conv2d_nhwc_hwcf_main.cpp | 0 .../toy/workloads/device/soc/CMakeLists.txt | 5 + .../device/soc}/scu_uart_rx_echo_test.c | 0 flake.nix | 23 +- scripts/claude/README.md | 12 +- scripts/nix/build-all.sh | 6 +- scripts/nix/build-env-compiler.nix | 3 + scripts/nix/build-env-cuda.nix | 21 - scripts/nix/build-env-python.nix | 1 + scripts/nix/build-env-riscv.nix | 6 +- scripts/nix/build-env-tools.nix | 3 + scripts/nix/download.sh | 15 +- scripts/nix/overlay.nix | 1 - sourceme.sh | 19 +- stack | 2 +- verify | 2 +- .../src/MLIRTest/base/base_dma_group.mlir | 0 .../src/MLIRTest/base/base_dma_group_main.cpp | 0 .../src/MLIRTest/base/base_mset_transfer.mlir | 2 +- .../MLIRTest/base/base_mset_transfer_main.cpp | 0 .../MLIRTest/compiler/bank_kernel_calls.mlir | 2 +- .../compiler/bank_kernel_invalid.mlir | 2 +- .../compiler/bank_transfer_cfg_invalid.mlir | 2 +- .../compiler/bank_transfer_invalid.mlir | 2 +- .../compiler/bank_transfer_ownership.mlir | 2 +- .../compiler/contiguous_scatter_rows.mlir | 0 .../MLIRTest/compiler/dma_group_invalid.mlir | 0 .../MLIRTest/compiler/dma_group_lowering.mlir | 10 +- .../MLIRTest/compiler/fp32_packmm_mixed.mlir | 4 +- .../compiler/mset_transfer_invalid.mlir | 0 .../compiler/mset_transfer_lowering.mlir | 6 +- .../compiler/norm_quant_window_mixed.mlir | 4 +- .../src/MLIRTest/compiler/rvv_rms_suffix.mlir | 0 .../src/MLIRTest/compiler/rvv_tanh.mlir | 0 .../LowerBuckyball/AssignPhysicalBanks.cpp | 16 + .../src/Dialect/Buckyball/F32AddBall.td | 2 +- .../src/Dialect/Buckyball/IR/F32Add.cpp | 2 +- .../Transforms/LegalizeForLLVMExport.cpp | 46 + .../balls/f32add/stack/src/Passes.cpp | 39 + .../LowerBuckyball/AssignPhysicalBanks.cpp | 76 + .../LowerBuckyball/LowerToBankSSA.cpp | 307 ++ .../Int8Contractions.cpp | 132 + .../src/Dialect/Buckyball/GemminiBall.td | 21 +- .../src/Dialect/Buckyball/IR/GemminiConv.cpp | 2 +- .../Transforms/LegalizeForLLVMExport.cpp | 46 + .../balls/gemmini/stack/src/Passes.cpp | 56 + .../include/IntrinsicsRISCVIm2colBall.td | 0 .../LowerBuckyball/AssignPhysicalBanks.cpp | 23 + .../src/Dialect/Buckyball/Im2colBall.td | 4 +- .../Transforms/LegalizeForLLVMExport.cpp | 234 ++ .../balls/im2col/stack/src/Passes.cpp | 39 + .../LowerBuckyball/AssignPhysicalBanks.cpp | 21 + .../src/Dialect/Buckyball/Int2FpBall.td | 2 +- .../Transforms/LegalizeForLLVMExport.cpp | 46 + .../balls/int2fp/stack/src/Passes.cpp | 39 + .../include/IntrinsicsRISCVInt8AddBall.td | 0 .../LowerBuckyball/AssignPhysicalBanks.cpp | 19 + .../src/Dialect/Buckyball/Int8AddBall.td | 2 +- .../Transforms/LegalizeForLLVMExport.cpp | 56 + .../balls/int8add/stack/src/Passes.cpp | 39 + .../include/IntrinsicsRISCVInt8MulBall.td | 0 .../LowerBuckyball/AssignPhysicalBanks.cpp | 19 + .../src/Dialect/Buckyball/Int8MulBall.td | 2 +- .../Transforms/LegalizeForLLVMExport.cpp | 52 + .../balls/int8mul/stack/src/Passes.cpp | 39 + .../LowerBuckyball/AssignPhysicalBanks.cpp | 35 + .../stack}/src/Dialect/Buckyball/LutBall.td | 2 +- .../Transforms/LegalizeForLLVMExport.cpp | 60 + .../components/balls/lut/stack/src/Passes.cpp | 39 + .../include/IntrinsicsRISCVMatAddBall.td | 0 .../LowerBuckyball/AssignPhysicalBanks.cpp | 18 + .../src/Dialect/Buckyball/MatAddBall.td | 2 +- .../Transforms/LegalizeForLLVMExport.cpp | 44 + .../balls/matadd/stack/src/Passes.cpp | 39 + .../LowerBuckyball/AssignPhysicalBanks.cpp | 22 + .../src/Dialect/Buckyball/MaxPoolBall.td | 5 +- .../Transforms/LegalizeForLLVMExport.cpp | 113 + .../balls/maxpool/stack/src/Passes.cpp | 39 + .../LowerBuckyball/AssignPhysicalBanks.cpp | 18 + .../src/Dialect/Buckyball/Mxfp2IntBall.td | 2 +- .../Transforms/LegalizeForLLVMExport.cpp | 50 + .../balls/mxfp2int/stack/src/Passes.cpp | 39 + .../balls/mxmm/stack}/python/fp32.py | 0 .../mxmm/stack}/python/grouped_attention.py | 0 .../balls/mxmm/stack}/python/layout.py | 21 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 39 + .../LowerBuckyball/LowerToBankSSA.cpp | 672 ++++ .../stack}/src/Dialect/Buckyball/IR/FP32.cpp | 0 .../stack}/src/Dialect/Buckyball/IR/MXFP8.cpp | 0 .../stack}/src/Dialect/Buckyball/MxmmBall.td | 14 +- .../Transforms/LegalizeForLLVMExport.cpp | 157 + .../Buckyball/Transforms/WindowPatterns.inc | 1 - .../balls/mxmm/stack/src/Passes.cpp | 41 + .../stack}/src/Transforms/NormQuantWindow.cpp | 4 +- .../LowerBuckyball/AssignPhysicalBanks.cpp | 16 + .../src/Dialect/Buckyball/MxquantBall.td | 2 +- .../Transforms/LegalizeForLLVMExport.cpp | 56 + .../balls/mxquant/stack/src/Passes.cpp | 39 + .../LowerBuckyball/AssignPhysicalBanks.cpp | 17 + .../src/Dialect/Buckyball/IR/Redmule.cpp | 40 + .../src/Dialect/Buckyball/RedmuleBall.td | 23 + .../Transforms/LegalizeForLLVMExport.cpp | 44 + .../balls/redmule/stack/src/Passes.cpp | 39 + .../include/IntrinsicsRISCVReluBall.td | 0 .../LowerBuckyball/AssignPhysicalBanks.cpp | 18 + .../LowerBuckyball/LowerToBankSSA.cpp | 102 + .../LowerLinalgToBuckyball/ReluPatterns.cpp | 118 + .../stack}/src/Dialect/Buckyball/ReluBall.td | 4 +- .../Transforms/LegalizeForLLVMExport.cpp | 50 + .../balls/relu/stack/src/Passes.cpp | 55 + .../include/IntrinsicsRISCVSMatMulBall.td | 0 .../LowerBuckyball/AssignPhysicalBanks.cpp | 31 + .../LowerBuckyballToBankSSAPatterns.cpp | 356 ++ .../MegaKernelToBankSSAPatterns.cpp | 114 +- .../LowerLinalgToBuckyball/MatMulPatterns.cpp | 416 ++ .../src/Dialect/Buckyball/SMatMulBall.td | 4 +- .../Transforms/LegalizeForLLVMExport.cpp | 263 ++ .../balls/smatmul/stack/src/Passes.cpp | 86 + .../LowerBuckyball/AssignPhysicalBanks.cpp | 33 + .../LowerBuckyball/LowerToBankSSA.cpp | 156 + .../QuantizePatterns.cpp | 64 + .../src/Dialect/Buckyball/ToInt8Ball.td | 14 +- .../Transforms/LegalizeForLLVMExport.cpp | 122 + .../balls/toint8/stack/src/Passes.cpp | 57 + .../stack}/src/Dialect/Buckyball/TraceBall.td | 0 .../Transforms/LegalizeForLLVMExport.cpp | 55 + .../balls/trace/stack/src/Passes.cpp | 39 + .../include/IntrinsicsRISCVTransposeBall.td | 0 .../LowerBuckyball/AssignPhysicalBanks.cpp | 20 + .../LowerBuckyball/LowerToBankSSA.cpp | 34 + .../TransposePatterns.cpp | 263 ++ .../Transforms/LegalizeForLLVMExport.cpp | 115 + .../src/Dialect/Buckyball/TransposeBall.td | 8 +- .../balls/transpose/stack/src/Passes.cpp | 59 + .../Backend/include/IntrinsicsRISCVVecBall.td | 0 .../LowerBuckyball/AssignPhysicalBanks.cpp | 20 + .../Transforms/LegalizeForLLVMExport.cpp | 47 + .../stack}/src/Dialect/Buckyball/VecBall.td | 2 +- .../balls/vector/stack/src/Passes.cpp | 39 + .../stack/midend/src/FormMegaKernels.cpp | 33 + .../designs/pebble/stack/midend/ops.py | 110 + .../MegaConv2dToBankSSAPatterns.cpp | 14 +- .../MemTransposeToBankSSAPatterns.cpp | 5 +- .../ResidentConvRegionToBankSSAPatterns.cpp | 178 +- .../stack/midend/src/LargeConvolution.cpp | 158 + .../pebble/stack/midend/src/Passes.cpp | 140 + .../stack/compiler/frontend/__init__.py | 0 .../compiler/frontend/layout/__init__.py | 0 .../compiler/frontend/layout/provenance.py | 62 + waste/stack/compiler/frontend/operation.py | 107 + .../stack/compiler/frontend/ops/__init__.py | 0 waste/stack/compiler/frontend/ops/expf.py | 83 + waste/stack/compiler/frontend/ops/int8.py | 250 ++ waste/stack/compiler/frontend/ops/linalg.py | 1599 ++++++++ .../compiler/frontend/ops/native_mxfp8.py | 187 + .../stack/compiler/frontend/ops/reductions.py | 308 ++ waste/stack/compiler/frontend/ops/tosa.py | 2271 +++++++++++ .../stack/compiler/frontend/quant/__init__.py | 0 waste/stack/compiler/frontend/quant/bwq.py | 163 + .../stack/compiler/frontend/quant/importer.py | 1187 ++++++ .../compiler/frontend/quant/mxfp8_cache.py | 70 + .../compiler/frontend/quant/native_mxfp8.py | 396 ++ .../frontend/quant/offline/__init__.py | 0 .../quant/offline/activations/__init__.py | 6 + .../quant/offline/activations/calibrate.py | 89 + .../quant/offline/weights/__init__.py | 6 + .../quant/offline/weights/quantize.py | 85 + .../offline/weights/strategy/__init__.py | 0 .../offline/weights/strategy/fp32toint8.py | 33 + waste/stack/compiler/frontend/quant/rax.py | 303 ++ .../frontend/quant/transforms/__init__.py | 0 .../frontend/quant/transforms/activation.py | 177 + .../frontend/quant/transforms/normalize.py | 20 + .../frontend/quant/transforms/symmetric.py | 28 + waste/stack/compiler/frontend/trace.py | 12 + waste/stack/compiler/frontend/transforms.py | 194 + .../compiler/src/Conversion/BallPasses.h | 131 + .../AssignPhysicalBanksPass.cpp | 97 + .../BaseAssignPhysicalBankPatterns.cpp | 212 + .../LowerBuckyball/LowerBuckyball.h | 70 + .../LowerBuckyball/LowerBuckyballPass.cpp | 109 + .../LowerBuckyballToBankSSAPass.cpp | 80 + .../LowerBuckyball/PhysicalBankState.cpp | 238 ++ .../FormMegaKernelRegions.cpp | 161 + .../LowerLinalgToBuckyball.h | 45 + .../LowerLinalgToBuckyballPass.cpp | 124 + .../MegaKernelPatterns.cpp | 346 ++ .../src/Dialect/Buckyball/Buckyball.td | 446 +++ .../src/Dialect/Buckyball/BuckyballDialect.h | 25 + .../Dialect/Buckyball/BuckyballInterfaces.td | 41 + .../src/Dialect/Buckyball/BuckyballOps.h | 38 + .../Dialect/Buckyball/IR/BuckyballDialect.cpp | 79 + .../Transforms/LegalizeForLLVMExportBase.cpp | 551 +++ .../Transforms/LegalizeForLLVMExportBase.h | 60 + waste/stack/compiler/src/Plugin.cpp | 30 + .../src/Target/BuckyballTargetRegistry.cpp | 74 + .../src/Target/BuckyballTargetRegistry.h | 47 + waste/stack/compiler/src/Utils/BankUtils.h | 131 + waste/stack/compiler/src/Utils/StageTrace.cpp | 66 + waste/stack/compiler/src/Utils/StageTrace.h | 11 + 4362 files changed, 143733 insertions(+), 101334 deletions(-) create mode 100644 arch/settings.sc create mode 100644 arch/src/main/scala/configs/GlobalConfig.scala create mode 100644 arch/src/main/scala/framework/acc/balldomain/build.sc create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/BallDomain.scala create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/baseball.scala create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/blink.scala create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/build.sc create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/build.sc create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/bbus.scala create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/build.sc create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/channels/Channels.scala rename arch/src/main/scala/framework/{balldomain => acc/balldomain/src/main/scala}/bbus/pmc/ballcyclepmc.spec.en.md (100%) rename arch/src/main/scala/framework/{balldomain => acc/balldomain/src/main/scala}/bbus/pmc/ballcyclepmc.spec.zh.md (100%) create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/configs/BallDomainParam.scala create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/decoder/DISA.scala create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/decoder/Decoder.scala create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/mmio_bank/MmioBank.scala create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/Expansion.scala create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/SubROB.scala create mode 100644 arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/configs/Params.scala create mode 100644 arch/src/main/scala/framework/acc/build.sc create mode 100644 arch/src/main/scala/framework/acc/configs/AccParam.scala create mode 100644 arch/src/main/scala/framework/acc/debug/BankCheckDPI.scala create mode 100644 arch/src/main/scala/framework/acc/debug/Debug.scala create mode 100644 arch/src/main/scala/framework/acc/debug/build.sc create mode 100644 arch/src/main/scala/framework/acc/debug/trace/AntTraceDPI.scala rename arch/src/main/scala/framework/{frontend/globalrs => acc/debug/trace}/ITraceDPI.scala (86%) create mode 100644 arch/src/main/scala/framework/acc/debug/trace/MTraceDPI.scala rename arch/src/main/scala/framework/{balldomain/bbus/pmc => acc/debug/trace}/PMCTraceDPI.scala (91%) create mode 100644 arch/src/main/scala/framework/acc/debug/trace/Trace.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/build.sc create mode 100644 arch/src/main/scala/framework/acc/frontend/configs/default.toml create mode 100644 arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/Cargo.lock create mode 100644 arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/Cargo.toml create mode 100644 arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/src/lib.rs rename arch/src/main/scala/framework/{memdomain/verification/src/main/resources/memory_system.cm_hier => acc/frontend/src/main/resources/decoder/decoder.cm_hier} (100%) create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder.f create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder_tb.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/Frontend.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/Ingress.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/RoCC.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/build.sc create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/scala/Boot.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/configs/FrontendParam.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/DISA.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/GlobalDecoder.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/build.sc create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/csrc/rename_retire.cpp create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Scheduler.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/build.sc create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/BankAliasTable.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/build.sc create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/driver.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/item.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/peer.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/env/env.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/env/model.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/filelist.f create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/interface.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/package.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/tb.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/tests/test.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/driver.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/item.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/env/env.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/env/model.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/filelist.f create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/interface.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/package.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/tb.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/tests/test.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/DependencyResolver.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/GlobalROB.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/Rob.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/configs/RobParams.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/build.sc create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/tests/test.svh rename arch/src/main/scala/framework/{frontend/scoreboard => acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala}/BankAccessInfo.scala (78%) create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/BankScoreboard.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/configs/Params.scala rename arch/src/main/scala/framework/{rvv => acc/gpdomain}/.gitignore (100%) rename arch/src/main/scala/framework/{mem-core => acc/gpdomain}/LICENSE (100%) create mode 100644 arch/src/main/scala/framework/acc/gpdomain/README.md create mode 100644 arch/src/main/scala/framework/acc/gpdomain/build.sc rename {examples/cores/pebble/configs/rvv => arch/src/main/scala/framework/acc/gpdomain/configs}/default.toml (100%) create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/csrc/Cargo.lock create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/csrc/Cargo.toml create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/csrc/build.rs rename arch/src/main/scala/framework/{rvv => acc/gpdomain}/src/csrc/src/ffi.rs (99%) create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/lib.rs create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/model.rs rename arch/src/main/scala/framework/{mem-core/axis/src/csrc => acc/gpdomain/src/main/resources}/README.md (100%) rename arch/src/main/scala/framework/{rvv/src/main/resources/rvv.json => acc/gpdomain/src/main/resources/gpdomain.json} (100%) rename arch/src/main/scala/framework/{rvv => acc/gpdomain}/src/main/resources/interface.sv (100%) create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/resources/test.svh create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/resources/topkernel.f rename arch/src/main/scala/framework/{rvv => acc/gpdomain}/src/main/resources/topkernel_tb.sv (98%) create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/GpDomain.scala create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/configs/GpDomainParam.scala create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/decoder/DISA.scala create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/decoder/Decoder.scala create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/build.sc create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/Execution.scala rename arch/src/main/scala/framework/{rvv/src/main/scala/frontend => acc/gpdomain/src/main/scala/execution/src/main/scala}/IBuf.scala (88%) rename arch/src/main/scala/framework/{rvv/src/main/scala/execution => acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic}/FALU.scala (94%) rename arch/src/main/scala/framework/{rvv/src/main/scala/execution => acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic}/IALU.scala (81%) create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/configs/Params.scala rename arch/src/main/scala/framework/{rvv/src/main/scala/execution => acc/gpdomain/src/main/scala/execution/src/main/scala/scalar}/ScalarExecution.scala (97%) rename arch/src/main/scala/framework/{rvv/src/main/scala/execution => acc/gpdomain/src/main/scala/execution/src/main/scala/scalar}/ScalarFloating.scala (93%) rename arch/src/main/scala/framework/{rvv/src/main/scala/register => acc/gpdomain/src/main/scala/execution/src/main/scala/scalar}/ScalarRF.scala (80%) rename arch/src/main/scala/framework/{rvv/src/main/scala/register => acc/gpdomain/src/main/scala/execution/src/main/scala/vector}/Operands.scala (96%) rename arch/src/main/scala/framework/{rvv/src/main/scala/register => acc/gpdomain/src/main/scala/execution/src/main/scala/vector}/VRF.scala (94%) create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/VectorThread.scala create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/build.sc create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/Emit.scala rename arch/src/main/scala/framework/{rvv/src/main/scala/frontend => acc/gpdomain/src/main/scala/loader/src/main/scala}/ImageLoader.scala (88%) create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/Loader.scala create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/build.sc create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/BankPort.scala create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/Emit.scala rename arch/src/main/scala/framework/{rvv/src/main/scala/memory => acc/gpdomain/src/main/scala/memory/src/main/scala}/KernelMemory.scala (92%) create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/Memory.scala create mode 100644 arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/build.sc create mode 100644 arch/src/main/scala/framework/acc/memdomain/configs/default.toml create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemDomain.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemoryControl.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/build.sc create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/peer.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/BankRouter.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/AccPipe.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/BLT.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/configs/MemDomainParam.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/Ctrl.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/configer/Configer.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/DISA.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/Decoder.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/build.sc create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/types.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/Bank.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/MvoverIO.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/PRM.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/build.sc create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/default.toml create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/fixture.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/peer.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/ToMem.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/build.sc create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/Cargo.lock create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/Cargo.toml create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/src/lib.rs create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/env/env.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/filelist.f create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/interface.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/package.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/seq/scenario.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/tb.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/tests/test.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/env/env.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/filelist.f create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/interface.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/package.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/seq/scenario.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/tb.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/tests/test.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/env/env.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/filelist.f create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/interface.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/package.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/seq/scenario.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/tb.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/tests/test.svh create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/DmaPort.scala rename arch/src/main/scala/framework/{memdomain/frontend/mem/dma => acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala}/DmaStatus.scala (89%) create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/Emit.scala rename arch/src/main/scala/framework/{memdomain/frontend/mem/dma => acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala}/ReadDma.scala (89%) create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ToDDR.scala rename arch/src/main/scala/framework/{memdomain/frontend/mem/dma => acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala}/WriteDma.scala (91%) create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/build.sc create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/scala/ToL1.scala delete mode 100644 arch/src/main/scala/framework/balldomain/BallDomain.scala delete mode 100644 arch/src/main/scala/framework/balldomain/bbus/bbus.scala delete mode 100644 arch/src/main/scala/framework/balldomain/bbus/cmdrouter/CmdRouter.scala delete mode 100644 arch/src/main/scala/framework/balldomain/bbus/cmdrouter/cmdrouter.spec.en.md delete mode 100644 arch/src/main/scala/framework/balldomain/bbus/cmdrouter/cmdrouter.spec.zh.md delete mode 100644 arch/src/main/scala/framework/balldomain/bbus/pmc/BallCyclePMC.scala delete mode 100644 arch/src/main/scala/framework/balldomain/blink/SubRobRow.scala delete mode 100644 arch/src/main/scala/framework/balldomain/blink/bank.scala delete mode 100644 arch/src/main/scala/framework/balldomain/blink/baseball.scala delete mode 100644 arch/src/main/scala/framework/balldomain/blink/blink.scala delete mode 100644 arch/src/main/scala/framework/balldomain/blink/mmio/MmioRead.scala delete mode 100644 arch/src/main/scala/framework/balldomain/blink/mmio/MmioWrite.scala delete mode 100644 arch/src/main/scala/framework/balldomain/blink/status.scala delete mode 100644 arch/src/main/scala/framework/balldomain/configs/BallDomainParam.scala delete mode 100644 arch/src/main/scala/framework/balldomain/configs/BallParamLoader.scala delete mode 100644 arch/src/main/scala/framework/balldomain/decoder/BallDecodeCmd.scala delete mode 100644 arch/src/main/scala/framework/balldomain/decoder/BallDomainDecoder.scala delete mode 100644 arch/src/main/scala/framework/balldomain/isa/BallISA.scala delete mode 100644 arch/src/main/scala/framework/balldomain/rs/interfaces.scala delete mode 100644 arch/src/main/scala/framework/balldomain/rs/reservationStation.scala create mode 100644 arch/src/main/scala/framework/build.sc create mode 100644 arch/src/main/scala/framework/core/build.sc create mode 100644 arch/src/main/scala/framework/core/clink/basecore.scala create mode 100644 arch/src/main/scala/framework/core/clink/build.sc create mode 100644 arch/src/main/scala/framework/core/clink/clink.scala create mode 100644 arch/src/main/scala/framework/core/clink/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/configs/CoreParam.scala create mode 100644 arch/src/main/scala/framework/core/controller/Controller.scala create mode 100644 arch/src/main/scala/framework/core/controller/admission/build.sc create mode 100644 arch/src/main/scala/framework/core/controller/admission/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/env/env.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/env/model.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/filelist.f create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/package.sv create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/tb.sv create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/driver.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/item.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/interface.sv create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/env/env.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/env/model.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/filelist.f create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/package.sv create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/tb.sv create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/env/env.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/env/model.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/filelist.f create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/package.sv create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/tb.sv create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/env/env.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/env/model.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/filelist.f create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/package.sv create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/tb.sv create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/scala/CommandAdmission.scala create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/scala/DirectAdmission.scala create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/scala/TaskRouter.scala create mode 100644 arch/src/main/scala/framework/core/controller/admission/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/controller/build.sc create mode 100644 arch/src/main/scala/framework/core/controller/configs/ControllerParam.scala create mode 100644 arch/src/main/scala/framework/core/controller/configs/default.toml create mode 100644 arch/src/main/scala/framework/core/controller/task/Control.scala create mode 100644 arch/src/main/scala/framework/core/controller/task/Service.scala create mode 100644 arch/src/main/scala/framework/core/execution/Execution.scala create mode 100644 arch/src/main/scala/framework/core/execution/build.sc create mode 100644 arch/src/main/scala/framework/core/execution/configs/ExecutionParam.scala create mode 100644 arch/src/main/scala/framework/core/execution/fetch/build.sc create mode 100644 arch/src/main/scala/framework/core/execution/fetch/configs/uvm.toml rename arch/src/main/scala/framework/{mem-core => core/execution}/fetch/src/csrc/Cargo.lock (100%) rename arch/src/main/scala/framework/{mem-core => core/execution}/fetch/src/csrc/Cargo.toml (100%) rename arch/src/main/scala/framework/{mem-core => core/execution}/fetch/src/csrc/src/instructions.rs (100%) rename arch/src/main/scala/framework/{mem-core => core/execution}/fetch/src/csrc/src/lib.rs (100%) create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/peer.svh create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/scala/Emit.scala rename arch/src/main/scala/framework/{mem-core => core/execution}/fetch/src/main/scala/Fetch.scala (93%) create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/default.toml create mode 100644 arch/src/main/scala/framework/core/execution/fpu/build.sc create mode 100644 arch/src/main/scala/framework/core/execution/fpu/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/csrc/Cargo.lock create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/csrc/Cargo.toml create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/csrc/src/lib.rs create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/csrc/src/vectors.rs create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/agent.svh create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/coverage.svh create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/scala/FPU.scala create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/default.toml create mode 100644 arch/src/main/scala/framework/core/execution/instruction/LICENSE.Berkeley create mode 100644 arch/src/main/scala/framework/core/execution/instruction/LICENSE.SiFive create mode 100644 arch/src/main/scala/framework/core/execution/instruction/build.sc create mode 100644 arch/src/main/scala/framework/core/execution/instruction/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/csrc/Cargo.lock create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/csrc/Cargo.toml create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/core/mod.rs create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/core/riscv/mod.rs create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/lib.rs create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/agent.svh create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/coverage.svh create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/scala/Instruction.scala create mode 100644 arch/src/main/scala/framework/core/execution/instruction/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/execution/integer/LICENSE.Berkeley create mode 100644 arch/src/main/scala/framework/core/execution/integer/LICENSE.SiFive create mode 100644 arch/src/main/scala/framework/core/execution/integer/build.sc create mode 100644 arch/src/main/scala/framework/core/execution/integer/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/agent.svh create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/coverage.svh create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/scala/Integer.scala create mode 100644 arch/src/main/scala/framework/core/execution/integer/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/build.sc create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/agent.svh create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/coverage.svh create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/MulDiv.scala create mode 100644 arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/execution/pipeline/Sequencer.scala create mode 100644 arch/src/main/scala/framework/core/execution/privileged/build.sc create mode 100644 arch/src/main/scala/framework/core/execution/privileged/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/csrc/Cargo.lock create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/csrc/Cargo.toml create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/core/mod.rs create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/core/riscv/mod.rs create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/lib.rs create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Privileged.scala create mode 100644 arch/src/main/scala/framework/core/execution/privileged/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/execution/register_file/build.sc create mode 100644 arch/src/main/scala/framework/core/execution/register_file/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/csrc/dual_read.cpp create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/agent.svh create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/coverage.svh create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/scala/RegisterFile.scala create mode 100644 arch/src/main/scala/framework/core/execution/register_file/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/memory/Memory.scala create mode 100644 arch/src/main/scala/framework/core/memory/MemorySystem.scala create mode 100644 arch/src/main/scala/framework/core/memory/ThreadMemory.scala create mode 100644 arch/src/main/scala/framework/core/memory/access/build.sc create mode 100644 arch/src/main/scala/framework/core/memory/access/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/core/memory/access/src/csrc/Cargo.lock create mode 100644 arch/src/main/scala/framework/core/memory/access/src/csrc/Cargo.toml rename arch/src/main/scala/framework/{mem-core/cpu_mem => core/memory/access}/src/csrc/src/access.rs (95%) create mode 100644 arch/src/main/scala/framework/core/memory/access/src/csrc/src/lib.rs create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/env/env.svh rename arch/src/main/scala/framework/{mem-core/cpu_mem/src/main/resources => core/memory/access/src/main/resources/physical}/interface.sv (100%) create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access.f create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_atomic.f create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_atomic_write.f create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_fence.f create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_operand.f create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_size.f create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_tag.f create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/package.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/tb.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/driver.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/fixture.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/item.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/peer.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/env/env.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/env/model.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/filelist.f create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/interface.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/package.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/tb.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/PhysicalAccess.scala create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/VirtualAccess.scala create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/build.sc create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/agent/cmo_if.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Cache.scala create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Emit.scala rename {examples/cores/rocket/arch/src/main/scala/cpu => arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala}/Maintenance.scala (81%) create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/build.sc create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/configs/uvm.toml rename arch/src/main/scala/framework/{mem-core => core/memory/access/src/main/scala}/mmu/src/csrc/Cargo.lock (100%) rename arch/src/main/scala/framework/{mem-core => core/memory/access/src/main/scala}/mmu/src/csrc/Cargo.toml (100%) rename arch/src/main/scala/framework/{mem-core => core/memory/access/src/main/scala}/mmu/src/csrc/src/lib.rs (100%) create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/src/translation.rs create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/peer.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/Emit.scala rename arch/src/main/scala/framework/{mem-core => core/memory/access/src/main/scala}/mmu/src/main/scala/Walker.scala (82%) create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/build.sc create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/configs/uvm.toml rename arch/src/main/scala/framework/{mem-core => core/memory/access/src/main/scala}/preflight/src/csrc/Cargo.lock (100%) rename arch/src/main/scala/framework/{mem-core => core/memory/access/src/main/scala}/preflight/src/csrc/Cargo.toml (100%) rename arch/src/main/scala/framework/{mem-core => core/memory/access/src/main/scala}/preflight/src/csrc/src/lib.rs (100%) rename arch/src/main/scala/framework/{mem-core => core/memory/access/src/main/scala}/preflight/src/csrc/src/preparation.rs (99%) rename arch/src/main/scala/framework/{mem-core => core/memory/access/src/main/scala}/preflight/src/csrc/src/prepared_map.rs (100%) create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/agent/service.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/env/env.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/filelist.f rename arch/src/main/scala/framework/{mem-core/preflight/src/main/resources/control_if.sv => core/memory/access/src/main/scala/preflight/src/main/resources/preflight/interface.sv} (100%) create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/package.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/seq/scenario.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/tb.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/driver.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/fixture.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/item.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/peer.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/env.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/model.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/filelist.f create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/interface.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/package.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/tb.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/agent/service.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/env/env.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/filelist.f create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/interface.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/package.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/seq/scenario.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/tb.sv create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Permission.scala create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PermissionCheck.scala rename arch/src/main/scala/framework/{mem-core => core/memory/access/src/main/scala}/preflight/src/main/scala/Preflight.scala (85%) create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preparation.scala rename arch/src/main/scala/framework/{mem-core => core/memory/access/src/main/scala}/preflight/src/main/scala/PreparedMap.scala (89%) rename arch/src/main/scala/framework/{mem-core => core/memory/access/src/main/scala}/preflight/src/main/scala/Segmenter.scala (95%) create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/configs/Config.scala create mode 100644 arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/memory/build.sc create mode 100644 arch/src/main/scala/framework/core/memory/configs/MemoryParam.scala create mode 100644 arch/src/main/scala/framework/core/memory/home/build.sc create mode 100644 arch/src/main/scala/framework/core/memory/home/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/core/memory/home/src/main/resources/agent/transport.svh create mode 100644 arch/src/main/scala/framework/core/memory/home/src/main/resources/agent/transport_if.sv create mode 100644 arch/src/main/scala/framework/core/memory/home/src/main/resources/filelist.f rename arch/src/main/scala/framework/{mem-core/coherence => core/memory/home}/src/main/resources/interface.sv (100%) create mode 100644 arch/src/main/scala/framework/core/memory/home/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/core/memory/home/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/memory/home/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/core/memory/home/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/core/memory/home/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/memory/home/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/core/memory/home/src/main/scala/Home.scala create mode 100644 arch/src/main/scala/framework/core/memory/home/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/memory/uncached/Uncached.scala create mode 100644 arch/src/main/scala/framework/core/memory/uncached/configs/UncachedParam.scala create mode 100644 arch/src/main/scala/framework/core/memory/uncached/configs/default.toml create mode 100644 arch/src/main/scala/framework/core/shm/SHM.scala create mode 100644 arch/src/main/scala/framework/core/shm/build.sc create mode 100644 arch/src/main/scala/framework/core/tbus/build.sc create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/SharedNetwork.scala create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/TBus.scala create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/build.sc create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/fixture.svh create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/peer.svh create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tests/invalid.svh create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/Router.scala create mode 100644 arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/core/tss/build.sc create mode 100644 arch/src/main/scala/framework/core/tss/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/core/tss/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/core/tss/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/core/tss/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/core/tss/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/core/tss/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/core/tss/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/core/tss/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/core/tss/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/core/tss/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/core/tss/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/core/tss/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/core/tss/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/core/tss/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/core/tss/src/main/scala/Store.scala create mode 100644 arch/src/main/scala/framework/core/tss/src/main/scala/configs/Params.scala create mode 100644 arch/src/main/scala/framework/device/build.sc create mode 100644 arch/src/main/scala/framework/device/cbus/C2C.scala create mode 100644 arch/src/main/scala/framework/device/cbus/CBus.scala create mode 100644 arch/src/main/scala/framework/device/cbus/Control.scala create mode 100644 arch/src/main/scala/framework/device/cbus/Fabric.scala rename arch/src/main/scala/framework/{system/tile/tlink => device/cbus}/SharedAxiEndpoint.scala (83%) create mode 100644 arch/src/main/scala/framework/device/cbus/build.sc create mode 100644 arch/src/main/scala/framework/device/configs/DeviceParam.scala create mode 100644 arch/src/main/scala/framework/device/ddr/build.sc create mode 100644 arch/src/main/scala/framework/device/ddr/configs/default.toml create mode 100644 arch/src/main/scala/framework/device/ddr/configs/uvm.toml rename arch/src/main/scala/framework/{mem-core => device}/ddr/src/csrc/Cargo.lock (100%) rename arch/src/main/scala/framework/{mem-core => device}/ddr/src/csrc/Cargo.toml (100%) rename arch/src/main/scala/framework/{mem-core/bank => device/ddr}/src/csrc/src/lib.rs (100%) rename arch/src/main/scala/framework/{mem-core => device}/ddr/src/csrc/src/memory.rs (100%) create mode 100644 arch/src/main/scala/framework/device/ddr/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/device/ddr/src/main/resources/agent/service.svh create mode 100644 arch/src/main/scala/framework/device/ddr/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/device/ddr/src/main/resources/filelist.f rename arch/src/main/scala/framework/{mem-core/ddr/src/main/resources/control_if.sv => device/ddr/src/main/resources/interface.sv} (100%) create mode 100644 arch/src/main/scala/framework/device/ddr/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/device/ddr/src/main/resources/seq/scenario.svh create mode 100644 arch/src/main/scala/framework/device/ddr/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/device/ddr/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/device/ddr/src/main/resources/tests/test.svh rename arch/src/main/scala/framework/{mem-core => device}/ddr/src/main/scala/Bridge.scala (89%) create mode 100644 arch/src/main/scala/framework/device/ddr/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/device/ddr/src/main/scala/Params.scala create mode 100644 arch/src/main/scala/framework/device/dlink/basedevice.scala create mode 100644 arch/src/main/scala/framework/device/dlink/build.sc create mode 100644 arch/src/main/scala/framework/device/dlink/dlink.scala create mode 100644 arch/src/main/scala/framework/device/peripherals/Controller.scala create mode 100644 arch/src/main/scala/framework/device/peripherals/bootrom/BootRom.scala create mode 100644 arch/src/main/scala/framework/device/peripherals/bootrom/configs/Parameters.scala create mode 100644 arch/src/main/scala/framework/device/peripherals/bootrom/configs/default.toml create mode 100644 arch/src/main/scala/framework/device/peripherals/build.sc create mode 100644 arch/src/main/scala/framework/device/peripherals/clint/Clint.scala create mode 100644 arch/src/main/scala/framework/device/peripherals/clint/configs/Parameters.scala create mode 100644 arch/src/main/scala/framework/device/peripherals/clint/configs/default.toml rename arch/src/main/scala/framework/{system/device => device/peripherals/plic}/Plic.scala (89%) create mode 100644 arch/src/main/scala/framework/device/peripherals/plic/configs/Parameters.scala create mode 100644 arch/src/main/scala/framework/device/peripherals/plic/configs/default.toml create mode 100644 arch/src/main/scala/framework/device/peripherals/scu/SCU.scala create mode 100644 arch/src/main/scala/framework/device/peripherals/scu/SystemControl.scala create mode 100644 arch/src/main/scala/framework/device/peripherals/scu/configs/Parameters.scala create mode 100644 arch/src/main/scala/framework/device/peripherals/scu/configs/default.toml delete mode 100644 arch/src/main/scala/framework/frontend/Frontend.scala delete mode 100644 arch/src/main/scala/framework/frontend/boot/Boot.scala delete mode 100644 arch/src/main/scala/framework/frontend/configs/FrontendParam.scala delete mode 100644 arch/src/main/scala/framework/frontend/decoder/GISA.scala delete mode 100644 arch/src/main/scala/framework/frontend/decoder/GobalDecoder.scala delete mode 100644 arch/src/main/scala/framework/frontend/decoder/PostGDCmd.scala delete mode 100644 arch/src/main/scala/framework/frontend/globalrs/GlobalROB.scala delete mode 100644 arch/src/main/scala/framework/frontend/globalrs/GlobalScheduler.scala delete mode 100644 arch/src/main/scala/framework/frontend/globalrs/SubROB.scala delete mode 100644 arch/src/main/scala/framework/frontend/scoreboard/BankAliasTable.scala delete mode 100644 arch/src/main/scala/framework/frontend/scoreboard/BankScoreboard.scala rename arch/src/main/scala/framework/{mem-core => ip}/.gitignore (100%) rename arch/src/main/scala/framework/{root => ip}/LICENSE (100%) rename arch/src/main/scala/framework/{mem-core => ip}/README.md (100%) create mode 100644 arch/src/main/scala/framework/ip/axi4/build.sc create mode 100644 arch/src/main/scala/framework/ip/axi4/configs/default.toml create mode 100644 arch/src/main/scala/framework/ip/axi4/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/driver.svh create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/item.svh create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/env/env.svh create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/env/model.svh create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/filelist.f create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/interface.sv create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/package.sv create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/tb.sv create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/tests/test.svh create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/driver.svh create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/item.svh create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/env/env.svh create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/env/model.svh create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/filelist.f create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/interface.sv create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/package.sv create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/tb.sv create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/tests/test.svh create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/types.svh create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/scala/Decoder.scala create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/scala/Emit.scala rename arch/src/main/scala/framework/{mem-core => ip}/axi4/src/main/scala/Interconnect.scala (94%) create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/scala/Interface.scala create mode 100644 arch/src/main/scala/framework/ip/axi4/src/main/scala/Params.scala create mode 100644 arch/src/main/scala/framework/ip/axis/build.sc create mode 100644 arch/src/main/scala/framework/ip/axis/configs/uvm.toml rename arch/src/main/scala/framework/{mem-core => ip}/axis/src/csrc/Cargo.lock (100%) rename arch/src/main/scala/framework/{mem-core => ip}/axis/src/csrc/Cargo.toml (100%) rename arch/src/main/scala/framework/{mem-core/axis/src/main/resources => ip/axis/src/csrc}/README.md (100%) create mode 100644 arch/src/main/scala/framework/ip/axis/src/csrc/src/arbiter.rs rename arch/src/main/scala/framework/{mem-core => ip}/axis/src/csrc/src/ffi.rs (100%) rename arch/src/main/scala/framework/{mem-core => ip}/axis/src/csrc/src/lib.rs (100%) create mode 100644 arch/src/main/scala/framework/ip/axis/src/csrc/src/model.rs create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/agent.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/driver.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/item.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/coverage.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/env.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/ref_model.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/filelist.f create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/interface.sv create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/package.sv create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/tb.sv create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/tests/test.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/common/coverage.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/master/env/env.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/master/env/ref_model.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/master/filelist.f create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/master/package.sv create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/master/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/master/tb.sv create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/master/tests/test.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/slave/env/env.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/slave/env/ref_model.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/slave/filelist.f create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/slave/package.sv create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/slave/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/slave/tb.sv create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/slave/tests/test.svh create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/ip/axis/src/main/scala/Interface.scala rename arch/src/main/scala/framework/{mem-core => ip}/axis/src/main/scala/Master.scala (96%) rename arch/src/main/scala/framework/{mem-core => ip}/axis/src/main/scala/PacketArbiter.scala (92%) rename arch/src/main/scala/framework/{mem-core => ip}/axis/src/main/scala/Slave.scala (96%) create mode 100644 arch/src/main/scala/framework/ip/bank/build.sc create mode 100644 arch/src/main/scala/framework/ip/bank/configs/uvm.toml rename arch/src/main/scala/framework/{mem-core => ip}/bank/src/csrc/Cargo.lock (100%) rename arch/src/main/scala/framework/{mem-core => ip}/bank/src/csrc/Cargo.toml (100%) rename arch/src/main/scala/framework/{mem-core => ip}/bank/src/csrc/README.md (100%) rename arch/src/main/scala/framework/{mem-core/cache => ip/bank}/src/csrc/src/lib.rs (100%) create mode 100644 arch/src/main/scala/framework/ip/bank/src/csrc/src/memory.rs create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/agent.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/driver.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/item.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/coverage.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/env.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/ref_model.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bank/filelist.f create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bank/interface.sv create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bank/package.sv create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bank/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bank/tb.sv create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bank/tests/test.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/agent.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/command.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/driver.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/item.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/coverage.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/env.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/ref_model.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/filelist.f create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/interface.sv create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/package.sv create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/tb.sv create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/tests/test.svh create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/scala/Bank.scala rename arch/src/main/scala/framework/{mem-core => ip}/bank/src/main/scala/BankSet.scala (98%) create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/ip/bank/src/main/scala/SramIO.scala create mode 100644 arch/src/main/scala/framework/ip/build.sc create mode 100644 arch/src/main/scala/framework/ip/cache/build.sc create mode 100644 arch/src/main/scala/framework/ip/cache/configs/CacheParams.scala rename arch/src/main/scala/framework/{mem-core => ip}/cache/configs/default.toml (100%) create mode 100644 arch/src/main/scala/framework/ip/cache/configs/uvm.toml rename arch/src/main/scala/framework/{mem-core => ip}/cache/src/csrc/Cargo.lock (100%) rename arch/src/main/scala/framework/{mem-core => ip}/cache/src/csrc/Cargo.toml (100%) rename arch/src/main/scala/framework/{mem-core/coherence => ip/cache}/src/csrc/src/lib.rs (100%) rename arch/src/main/scala/framework/{mem-core => ip}/cache/src/csrc/src/memory.rs (100%) create mode 100644 arch/src/main/scala/framework/ip/cache/src/main/resources/agent/agent.svh create mode 100644 arch/src/main/scala/framework/ip/cache/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/ip/cache/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/ip/cache/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/ip/cache/src/main/resources/agent/sink.svh create mode 100644 arch/src/main/scala/framework/ip/cache/src/main/resources/env/coverage.svh create mode 100644 arch/src/main/scala/framework/ip/cache/src/main/resources/env/env.svh rename arch/src/main/scala/framework/{mem-core/cache/src/main/resources => ip/cache/src/main/resources/env}/ref_model.svh (100%) create mode 100644 arch/src/main/scala/framework/ip/cache/src/main/resources/filelist.f rename arch/src/main/scala/framework/{mem-core => ip}/cache/src/main/resources/interface.sv (100%) create mode 100644 arch/src/main/scala/framework/ip/cache/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/ip/cache/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/ip/cache/src/main/resources/suite.toml rename arch/src/main/scala/framework/{mem-core/cache/src/main/resources/cache_tb.sv => ip/cache/src/main/resources/tb.sv} (100%) create mode 100644 arch/src/main/scala/framework/ip/cache/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/ip/cache/src/main/scala/Cache.scala create mode 100644 arch/src/main/scala/framework/ip/cache/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/ip/cache/src/main/scala/Interface.scala create mode 100644 arch/src/main/scala/framework/ip/chi/build.sc create mode 100644 arch/src/main/scala/framework/ip/chi/configs/uvm.toml rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/csrc/Cargo.lock (100%) rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/csrc/Cargo.toml (100%) rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/csrc/README.md (100%) create mode 100644 arch/src/main/scala/framework/ip/chi/src/csrc/src/lib.rs rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/csrc/src/memory.rs (100%) rename arch/src/main/scala/framework/{mem-core/chi/src/main/resources => ip/chi/src/main/resources/common}/flit_types.svh (78%) create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/common/profile.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/agent.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/driver.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/endpoint.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/item.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/retry/env/env.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/retry/env/model.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/retry/filelist.f create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/retry/interface.sv create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/retry/package.sv create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/retry/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/retry/tb.sv create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/retry/tests/test.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/retry/types.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/filelist.f create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/package.sv create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/tb.sv create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/agent.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/driver.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/item.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rx/env/env.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rx/env/model.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rx/filelist.f create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rx/interface.sv create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rx/package.sv create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rx/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rx/tb.sv create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rx/tests/test.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/rx/types.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/agent.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/driver.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/endpoint.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/item.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/sram/env/env.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/sram/env/model.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/sram/filelist.f create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/sram/interface.sv create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/sram/package.sv create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/sram/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/sram/tb.sv create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/sram/tests/test.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/sram/types.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/agent.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/driver.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/item.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/tx/env/env.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/tx/env/model.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/tx/filelist.f create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/tx/interface.sv create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/tx/package.sv create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/tx/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/tx/tb.sv create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/tx/tests/test.svh create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/resources/tx/types.svh rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/main/scala/Channel.scala (91%) rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/main/scala/CoherenceState.scala (78%) rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/main/scala/DirectedSnoop.scala (83%) create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/scala/Fabric.scala rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/main/scala/Flit.scala (97%) rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/main/scala/HomeMapping.scala (93%) rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/main/scala/RequestRetry.scala (95%) rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/main/scala/RequesterPort.scala (94%) rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/main/scala/Rx.scala (98%) rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/main/scala/Tx.scala (98%) rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/main/scala/hnf/ChiHome.scala (98%) create mode 100644 arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/AxiCache.scala rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/main/scala/rnf/BankedChiCache.scala (84%) rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/main/scala/rnf/CachePort.scala (95%) rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/main/scala/rnf/ChiCache.scala (99%) rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/main/scala/snf/MemoryNode.scala (92%) rename arch/src/main/scala/framework/{mem-core => ip}/chi/src/main/scala/snf/SramEndpoint.scala (97%) create mode 100644 arch/src/main/scala/framework/ip/coherence/build.sc create mode 100644 arch/src/main/scala/framework/ip/coherence/configs/CoherenceParams.scala rename arch/src/main/scala/framework/{mem-core => ip}/coherence/configs/default.toml (100%) create mode 100644 arch/src/main/scala/framework/ip/coherence/configs/requesters.toml create mode 100644 arch/src/main/scala/framework/ip/coherence/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/ip/coherence/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/ip/coherence/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/ip/coherence/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/ip/coherence/src/main/resources/suite.toml rename arch/src/main/scala/framework/{mem-core/coherence/src/main/resources/coherence_tb.sv => ip/coherence/src/main/resources/tb.sv} (100%) rename arch/src/main/scala/framework/{mem-core => ip}/coherence/src/main/scala/Coherence.scala (89%) rename arch/src/main/scala/framework/{mem-core => ip}/coherence/src/main/scala/Directory.scala (94%) create mode 100644 arch/src/main/scala/framework/ip/coherence/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/ip/coherence/src/main/scala/Interface.scala rename arch/src/main/scala/framework/{mem-core => ip}/coherence/src/main/scala/Mshr.scala (94%) create mode 100644 arch/src/main/scala/framework/ip/interlock/build.sc create mode 100644 arch/src/main/scala/framework/ip/interlock/configs/uvm.toml create mode 100644 arch/src/main/scala/framework/ip/interlock/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/ip/interlock/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/ip/interlock/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/ip/interlock/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/ip/interlock/src/main/scala/Interface.scala rename arch/src/main/scala/framework/{mem-core => ip}/interlock/src/main/scala/Interlock.scala (96%) create mode 100644 arch/src/main/scala/framework/ip/interlock/src/main/scala/Params.scala create mode 100644 arch/src/main/scala/framework/ip/mesh_shm/build.sc rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/csrc/Cargo.lock (100%) rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/csrc/Cargo.toml (100%) create mode 100644 arch/src/main/scala/framework/ip/mesh_shm/src/csrc/fixed_attachment.cpp rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/csrc/src/lib.rs (100%) rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/csrc/src/transfer.rs (100%) create mode 100644 arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router.f rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/main/resources/router_item.svh (100%) create mode 100644 arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_package.sv rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/main/resources/router_tb.sv (100%) rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/main/resources/router_test.svh (96%) create mode 100644 arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared.f rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/main/resources/shared_item.svh (100%) rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/main/resources/shared_package.sv (95%) rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/main/resources/shared_tb.sv (99%) rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/main/resources/shared_test.svh (89%) create mode 100644 arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/suite.toml rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/main/resources/transfer_if.sv (96%) create mode 100644 arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/Emit.scala rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/main/scala/MeshBankNode.scala (95%) create mode 100644 arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshChannel.scala rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/main/scala/MeshChannelOrdering.scala (98%) create mode 100644 arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshControlInterface.scala rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/main/scala/MeshCreditLink.scala (85%) rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/main/scala/MeshRouter.scala (88%) create mode 100644 arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMem.scala create mode 100644 arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMemInterface.scala create mode 100644 arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMemParams.scala create mode 100644 arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshThreadEndpoint.scala rename arch/src/main/scala/framework/{mem-core => ip}/mesh_shm/src/main/scala/MeshTransferController.scala (77%) create mode 100644 arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/configs/Geometry.scala create mode 100644 arch/src/main/scala/framework/ip/spm/build.sc create mode 100644 arch/src/main/scala/framework/ip/spm/configs/uvm.toml rename arch/src/main/scala/framework/{mem-core => ip}/spm/src/csrc/src/lib.rs (100%) create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/resources/agent/agent.svh create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/resources/agent/driver.svh create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/resources/agent/item.svh create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/resources/agent/monitor.svh create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/resources/env/coverage.svh create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/resources/env/env.svh create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/resources/env/model.svh create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/resources/filelist.f create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/resources/interface.sv create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/resources/package.sv create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/resources/seq/sequence.svh create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/resources/suite.toml create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/resources/tb.sv create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/resources/tests/test.svh create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/scala/Interfaces.scala create mode 100644 arch/src/main/scala/framework/ip/spm/src/main/scala/Memory.scala create mode 100644 arch/src/main/scala/framework/lib/arbiter/Arbiter.scala create mode 100644 arch/src/main/scala/framework/lib/arbiter/build.sc rename arch/src/main/scala/framework/{arith/src/main/scala => lib/arith}/DivRem.scala (99%) create mode 100644 arch/src/main/scala/framework/lib/arith/Fma.scala create mode 100644 arch/src/main/scala/framework/lib/arith/IntToFloat.scala create mode 100644 arch/src/main/scala/framework/lib/arith/LICENSE.Berkeley create mode 100644 arch/src/main/scala/framework/lib/arith/LICENSE.SiFive rename arch/src/main/scala/framework/{arith/src/main/scala => lib/arith}/Multiply.scala (96%) create mode 100644 arch/src/main/scala/framework/lib/arith/build.sc create mode 100644 arch/src/main/scala/framework/lib/build.sc rename arch/src/main/scala/framework/{ => lib}/dpi/DpiGuard.scala (96%) create mode 100644 arch/src/main/scala/framework/lib/dpi/build.sc create mode 100644 arch/src/main/scala/framework/lib/hash/BankHash.scala create mode 100644 arch/src/main/scala/framework/lib/hash/build.sc create mode 100644 arch/src/main/scala/framework/lib/hash/src/csrc/monitor.cpp rename arch/src/main/scala/framework/{mem-core/queue/src/main/scala => lib/queue}/Queue.scala (98%) create mode 100644 arch/src/main/scala/framework/lib/queue/build.sc delete mode 100644 arch/src/main/scala/framework/mem-core/axi4/build.sc delete mode 100644 arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Interface.scala delete mode 100644 arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Params.scala delete mode 100644 arch/src/main/scala/framework/mem-core/axis/build.sc delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/csrc/src/arbiter.rs delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/csrc/src/model.rs delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter.f delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_checker.svh delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_driver.svh delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_interface.sv delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_test.svh delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/resources/env.svh delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/resources/master.f delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/resources/master_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/resources/package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/resources/ref_model.svh delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/resources/slave.f delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/resources/slave_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/resources/test.svh delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/mem-core/axis/src/main/scala/Interface.scala delete mode 100644 arch/src/main/scala/framework/mem-core/bank/build.sc delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/csrc/src/memory.rs delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank.el delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank.f delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset.el delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset.f delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_interface.sv delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/resources/env.svh delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/resources/interface.sv delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/resources/item.svh delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/resources/package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/resources/ref_model.svh delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/resources/test.svh delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/scala/Bank.scala delete mode 100644 arch/src/main/scala/framework/mem-core/bank/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/mem-core/cache/build.sc delete mode 100644 arch/src/main/scala/framework/mem-core/cache/configs/CacheParams.scala delete mode 100644 arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache.el delete mode 100644 arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache.f delete mode 100644 arch/src/main/scala/framework/mem-core/cache/src/main/resources/env.svh delete mode 100644 arch/src/main/scala/framework/mem-core/cache/src/main/resources/item.svh delete mode 100644 arch/src/main/scala/framework/mem-core/cache/src/main/resources/package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/cache/src/main/resources/test.svh delete mode 100644 arch/src/main/scala/framework/mem-core/cache/src/main/scala/Cache.scala delete mode 100644 arch/src/main/scala/framework/mem-core/cache/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/mem-core/cache/src/main/scala/Interface.scala delete mode 100644 arch/src/main/scala/framework/mem-core/chi/build.sc delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/csrc/src/lib.rs delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/csrc/src/rnf.rs delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/chi_profile.svh delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_interface.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/link.el delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/link.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_interface.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry.el delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_bad_opcode.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_bad_opcode.toml delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_duplicate_txnid.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_duplicate_txnid.toml delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_interface.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid.toml delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid_width.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid_width.toml delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_resp.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_resp.toml delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_interface.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_test.svh delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx.el delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_bad_credit.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_bad_credit.toml delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram.el delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram128.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram512.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_bad_opcode.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_bad_opcode.toml delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_duplicate_data.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_duplicate_data.toml delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_interface.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_test.svh delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/tx_bad_credit.f delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/resources/tx_bad_credit.toml delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/mem-core/chi/src/main/scala/Fabric.scala delete mode 100644 arch/src/main/scala/framework/mem-core/coherence/build.sc delete mode 100644 arch/src/main/scala/framework/mem-core/coherence/configs/CoherenceParams.scala delete mode 100644 arch/src/main/scala/framework/mem-core/coherence/src/csrc/Cargo.lock delete mode 100644 arch/src/main/scala/framework/mem-core/coherence/src/csrc/Cargo.toml delete mode 100644 arch/src/main/scala/framework/mem-core/coherence/src/csrc/src/memory.rs delete mode 100644 arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence.el delete mode 100644 arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence.f delete mode 100644 arch/src/main/scala/framework/mem-core/coherence/src/main/resources/env.svh delete mode 100644 arch/src/main/scala/framework/mem-core/coherence/src/main/resources/package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/coherence/src/main/resources/test.svh delete mode 100644 arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Interface.scala delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/build.sc delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/Cargo.lock delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/Cargo.toml delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/src/lib.rs delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem.f delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic.f delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic.toml delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic_write.f delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic_write.toml delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_fence.f delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_fence.toml delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_operand.f delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_operand.toml delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_size.f delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_size.toml delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_tag.f delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_tag.toml delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/test.svh delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/CpuMem.scala delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/VirtualMemory.scala delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/build.sc delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/csrc/src/lib.rs delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_alignment.f delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_alignment.toml delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_b.f delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_b.toml delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_rlast.f delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_rlast.toml delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_late_rlast.f delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_late_rlast.toml delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_read_exokay.f delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_read_exokay.toml delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_rid.f delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_rid.toml delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_write_exokay.f delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_write_exokay.toml delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge.f delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge256.f delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge512.f delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge64.f delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge_six.f delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/resources/profile.svh delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Params.scala delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/build.sc delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch.el delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch.f delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_ack.f delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_ack.toml delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_flush.f delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_flush.toml delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_privilege.f delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_privilege.toml delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_redirect.f delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_redirect.toml delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_reset.f delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_reset.toml delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/interface.sv delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/resources/test.svh delete mode 100644 arch/src/main/scala/framework/mem-core/fetch/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/build.sc delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/csrc/Cargo.lock delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/csrc/Cargo.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/csrc/src/lib.rs delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/csrc/src/ownership.rs delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/control_if.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.f delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.toml delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range_scenario.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/negative_tb.svh delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/resources/package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Interface.scala delete mode 100644 arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Params.scala delete mode 100644 arch/src/main/scala/framework/mem-core/mesh_shm/build.sc delete mode 100644 arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router.f delete mode 100644 arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared.f delete mode 100644 arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshChannel.scala delete mode 100644 arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshClientEndpoint.scala delete mode 100644 arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshControlInterface.scala delete mode 100644 arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshCoreEndpoint.scala delete mode 100644 arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMem.scala delete mode 100644 arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMemInterface.scala delete mode 100644 arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMemParams.scala delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/build.sc delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/csrc/src/translation.rs delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker.f delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access.f delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access.toml delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode.f delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode.toml delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege.f delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege.toml delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_interface.sv delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_negative_package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/mmu/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/mem-core/preflight/build.sc delete mode 100644 arch/src/main/scala/framework/mem-core/preflight/src/main/resources/package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight.f delete mode 100644 arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight64.f delete mode 100644 arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map.f delete mode 100644 arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_control_if.sv delete mode 100644 arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Interface.scala delete mode 100644 arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Params.scala delete mode 100644 arch/src/main/scala/framework/mem-core/queue/build.sc delete mode 100644 arch/src/main/scala/framework/mem-core/spm/src/main/resources/interface.sv delete mode 100644 arch/src/main/scala/framework/mem-core/spm/src/main/resources/package.sv delete mode 100644 arch/src/main/scala/framework/mem-core/spm/src/main/scala/Interfaces.scala delete mode 100644 arch/src/main/scala/framework/mem-core/spm/src/main/scala/Memory.scala delete mode 100644 arch/src/main/scala/framework/mem-core/tls/src/csrc/Cargo.lock delete mode 100644 arch/src/main/scala/framework/mem-core/tls/src/csrc/Cargo.toml delete mode 100644 arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls.f delete mode 100644 arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/tls/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/mem-core/tls/src/main/scala/Store.scala delete mode 100644 arch/src/main/scala/framework/mem-core/tss/src/csrc/Cargo.lock delete mode 100644 arch/src/main/scala/framework/mem-core/tss/src/csrc/Cargo.toml delete mode 100644 arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.cm_hier delete mode 100644 arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.f delete mode 100644 arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.toml delete mode 100644 arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss_tb.sv delete mode 100644 arch/src/main/scala/framework/mem-core/tss/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/mem-core/tss/src/main/scala/Store.scala delete mode 100644 arch/src/main/scala/framework/memdomain/MemDomain.scala delete mode 100644 arch/src/main/scala/framework/memdomain/backend/MTrace.scala delete mode 100644 arch/src/main/scala/framework/memdomain/backend/MemBackend.scala delete mode 100644 arch/src/main/scala/framework/memdomain/backend/accpipe/AccPipe.scala delete mode 100644 arch/src/main/scala/framework/memdomain/backend/banks/SramBank.scala delete mode 100644 arch/src/main/scala/framework/memdomain/backend/banks/SramIO.scala delete mode 100644 arch/src/main/scala/framework/memdomain/backend/banks/btrace/BTraceDPI.scala delete mode 100644 arch/src/main/scala/framework/memdomain/backend/mmio/MmioBank.scala delete mode 100644 arch/src/main/scala/framework/memdomain/backend/mmio/MmioIO.scala delete mode 100644 arch/src/main/scala/framework/memdomain/backend/mmio/MmioPool.scala delete mode 100644 arch/src/main/scala/framework/memdomain/backend/mmio/MmioRouter.scala delete mode 100644 arch/src/main/scala/framework/memdomain/backend/privatepath/PrivateMemBackend.scala delete mode 100644 arch/src/main/scala/framework/memdomain/backend/shared/PhysicalPort.scala delete mode 100644 arch/src/main/scala/framework/memdomain/backend/shared/SharedLease.scala delete mode 100644 arch/src/main/scala/framework/memdomain/backend/shared/SharedMem.scala delete mode 100644 arch/src/main/scala/framework/memdomain/backend/shared/SharedMemBackend.scala delete mode 100644 arch/src/main/scala/framework/memdomain/boot/MemBoot.scala delete mode 100644 arch/src/main/scala/framework/memdomain/configs/MemDomainParam.scala delete mode 100644 arch/src/main/scala/framework/memdomain/frontend/MemFrontend.scala delete mode 100644 arch/src/main/scala/framework/memdomain/frontend/cmd/decoder/DISA.scala delete mode 100644 arch/src/main/scala/framework/memdomain/frontend/cmd/decoder/DomainDecoder.scala delete mode 100644 arch/src/main/scala/framework/memdomain/frontend/cmd/rs/README.md delete mode 100644 arch/src/main/scala/framework/memdomain/frontend/cmd/rs/reservationStation.scala delete mode 100644 arch/src/main/scala/framework/memdomain/frontend/mem/Footprint.scala delete mode 100644 arch/src/main/scala/framework/memdomain/frontend/mem/KernelDma.scala delete mode 100644 arch/src/main/scala/framework/memdomain/frontend/mem/MemConfiger.scala delete mode 100644 arch/src/main/scala/framework/memdomain/frontend/mem/MemLoader.scala delete mode 100644 arch/src/main/scala/framework/memdomain/frontend/mem/MemStorer.scala delete mode 100644 arch/src/main/scala/framework/memdomain/frontend/mem/dma/Bundles.scala delete mode 100644 arch/src/main/scala/framework/memdomain/frontend/mem/dma/Dma.scala delete mode 100644 arch/src/main/scala/framework/memdomain/frontend/mem/dma/DmaPort.scala delete mode 100644 arch/src/main/scala/framework/memdomain/isa/MemISA.scala delete mode 100644 arch/src/main/scala/framework/memdomain/isa/MvoverIO.scala delete mode 100644 arch/src/main/scala/framework/memdomain/midend/MemMidend.scala delete mode 100644 arch/src/main/scala/framework/memdomain/utils/pmc/MemCyclePMC.scala delete mode 100644 arch/src/main/scala/framework/memdomain/utils/pmc/MemPMCTraceDPI.scala delete mode 100644 arch/src/main/scala/framework/memdomain/verification/.gitignore delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/csrc/Cargo.lock delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/csrc/Cargo.toml delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/csrc/src/lib.rs delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/csrc/src/memory.rs delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/csrc/src/tile_peer.rs delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/ack_package.sv delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.cm_hier delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.f delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.toml delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system_tb.sv delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.cm_hier delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.f delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.toml delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system_tb.sv delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma.cm_hier delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma.f delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma_tb.sv delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/loader_ack.f delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/loader_ack_tb.sv delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.f delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.toml delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system_tb.sv delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma.cm_hier delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma.f delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma_tb.sv delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease.cm_hier delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease.f delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease_tb.sv delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/storer_ack.f delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/storer_ack_tb.sv delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_axi_peer.svh delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.cm_hier delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.f delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.toml delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four_config.svh delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four_tb.sv delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.cm_hier delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.f delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.toml delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single_config.svh delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single_tb.sv delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_tb.svh delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/scala/AdmissionSystem.scala delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/scala/ControllerSystem.scala delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitController.scala delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitMemory.scala delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitTile.scala delete mode 100644 arch/src/main/scala/framework/memdomain/verification/src/main/scala/TileSystem.scala delete mode 100644 arch/src/main/scala/framework/root/.gitignore delete mode 100644 arch/src/main/scala/framework/root/README.md delete mode 100644 arch/src/main/scala/framework/root/root-chip/build.sc delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/csrc/Cargo.lock delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/csrc/Cargo.toml delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/csrc/src/lib.rs delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/csrc/src/routing.rs delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node.f delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node.toml delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node_package.sv delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node_tb.sv delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req.f delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req.toml delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req_package.sv delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req_tb.sv delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/codec.el delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/codec.f delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_interface.sv delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_package.sv delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_tb.sv delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/fabric_interface.sv delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/network.f delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/network_package.sv delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/network_tb.sv delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/router.el delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/router.f delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/router_package.sv delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/resources/router_tb.sv delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/ChiMesh.scala delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/Emit.scala delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/Mesh.scala delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/MeshBulk.scala delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/MeshCredit.scala delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/ScratchpadBulkWriteSink.scala delete mode 100644 arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/TileMeshEndpoint.scala delete mode 100644 arch/src/main/scala/framework/root/root-core/src/csrc/Cargo.lock delete mode 100644 arch/src/main/scala/framework/root/root-core/src/csrc/Cargo.toml delete mode 100644 arch/src/main/scala/framework/root/root-core/src/csrc/src/lib.rs delete mode 100644 arch/src/main/scala/framework/root/root-core/src/csrc/src/memory.rs delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/admission.S delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/admission.cm_hier delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/admission.f delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/admission.ld delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge.f delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_bad_late_fault.f delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_bad_late_fault.toml delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_control.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_package.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_tb.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/admission_control.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/admission_package.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/admission_tb.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/build_fixture.py delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/core.S delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/core.cm_hier delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/core.f delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/core.ld delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/core_interface.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/core_package.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/core_tb.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/core_test.svh delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.S delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.f delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.ld delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/irq.S delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/irq.cm_hier delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/irq.f delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/irq.ld delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/lsu.f delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_interface.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_package.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_tb.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_test.svh delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance.cm_hier delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance.f delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.cm_hier delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.f delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.toml delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_interface.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_package.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_tb.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/multicore.S delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/multicore.ld delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/permission.cm_hier delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/permission.f delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/permission_tb.sv delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.S delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.cm_hier delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.f delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.ld delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/Emit.scala delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/PermissionVerification.scala delete mode 100644 arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/Verification.scala delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/csrc/Cargo.lock delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/csrc/Cargo.toml delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/csrc/src/consistency.rs delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/csrc/src/lib.rs delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system.f delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_interface.sv delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_package.sv delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_tb.sv delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency.cm_hier delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency.f delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_control.sv delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_package.sv delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_tb.sv delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/tile.f delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_interface.sv delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_package.sv delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_tb.sv delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache.f delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic.f delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic.toml delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic_write.f delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic_write.toml delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_atomic.f delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_atomic.toml delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_signed.f delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_signed.toml delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_size.f delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_size.toml delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_write.f delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_write.toml delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_mode.f delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_mode.toml delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_operand.f delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_operand.toml delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_privilege.f delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_privilege.toml delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_size.f delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_size.toml delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_interface.sv delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_package.sv delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_tb.sv delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/scala/TaskAdmission.scala delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/scala/TaskController.scala delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/CacheSystem.scala delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Composition.scala delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Consistency.scala delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Emit.scala delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Home.scala delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Memory.scala delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Tile.scala delete mode 100644 arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/VirtualCacheSystem.scala delete mode 100644 arch/src/main/scala/framework/rvv/LICENSE delete mode 100644 arch/src/main/scala/framework/rvv/README.md delete mode 100644 arch/src/main/scala/framework/rvv/build.sc delete mode 100644 arch/src/main/scala/framework/rvv/src/csrc/Cargo.lock delete mode 100644 arch/src/main/scala/framework/rvv/src/csrc/Cargo.toml delete mode 100644 arch/src/main/scala/framework/rvv/src/csrc/build.rs delete mode 100644 arch/src/main/scala/framework/rvv/src/csrc/src/lib.rs delete mode 100644 arch/src/main/scala/framework/rvv/src/csrc/src/model.rs delete mode 100644 arch/src/main/scala/framework/rvv/src/main/resources/package.sv delete mode 100644 arch/src/main/scala/framework/rvv/src/main/resources/test.svh delete mode 100644 arch/src/main/scala/framework/rvv/src/main/resources/topkernel.f delete mode 100644 arch/src/main/scala/framework/rvv/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/rvv/src/main/scala/configs/RvvParam.scala delete mode 100644 arch/src/main/scala/framework/rvv/src/main/scala/core/VectorCore.scala delete mode 100644 arch/src/main/scala/framework/rvv/src/main/scala/interfaces/KernelBlinkIO.scala delete mode 100644 arch/src/main/scala/framework/rvv/src/main/scala/interfaces/RvvInterface.scala delete mode 100644 arch/src/main/scala/framework/rvv/src/main/scala/memory/BankPort.scala delete mode 100644 arch/src/main/scala/framework/rvv/src/main/scala/top/Execution.scala delete mode 100644 arch/src/main/scala/framework/rvv/src/main/scala/top/KernelEngine.scala delete mode 100644 arch/src/main/scala/framework/system/Device.scala delete mode 100644 arch/src/main/scala/framework/system/DeviceConnection.scala delete mode 100644 arch/src/main/scala/framework/system/GlobalConfigOps.scala delete mode 100644 arch/src/main/scala/framework/system/System.scala delete mode 100644 arch/src/main/scala/framework/system/configloader/ChipLoader.scala delete mode 100644 arch/src/main/scala/framework/system/configloader/Topology.scala delete mode 100644 arch/src/main/scala/framework/system/core/AntEndpoint.scala delete mode 100644 arch/src/main/scala/framework/system/core/ControllerAdmission.scala delete mode 100644 arch/src/main/scala/framework/system/core/CoreConnection.scala delete mode 100644 arch/src/main/scala/framework/system/core/CoreFactory.scala delete mode 100644 arch/src/main/scala/framework/system/core/CoreParams.scala delete mode 100644 arch/src/main/scala/framework/system/core/RocketEndpoint.scala delete mode 100644 arch/src/main/scala/framework/system/core/accelerator/Admission.scala delete mode 100644 arch/src/main/scala/framework/system/core/accelerator/AntAdmission.scala delete mode 100644 arch/src/main/scala/framework/system/core/accelerator/AntMemory.scala delete mode 100644 arch/src/main/scala/framework/system/core/accelerator/BuckyballAccelerator.scala delete mode 100644 arch/src/main/scala/framework/system/core/accelerator/Rvv.scala delete mode 100644 arch/src/main/scala/framework/system/core/ant/src/csrc/src/lib.rs delete mode 100644 arch/src/main/scala/framework/system/core/ant/src/main/resources/ant.f delete mode 100644 arch/src/main/scala/framework/system/core/ant/src/main/resources/ant_tb.sv delete mode 100644 arch/src/main/scala/framework/system/core/ant/src/main/resources/control.f delete mode 100644 arch/src/main/scala/framework/system/core/ant/src/main/resources/control_tb.sv delete mode 100644 arch/src/main/scala/framework/system/core/ant/src/main/resources/service.f delete mode 100644 arch/src/main/scala/framework/system/core/ant/src/main/resources/service_tb.sv delete mode 100644 arch/src/main/scala/framework/system/core/ant/src/main/scala/Ant.scala delete mode 100644 arch/src/main/scala/framework/system/core/ant/src/main/scala/Control.scala delete mode 100644 arch/src/main/scala/framework/system/core/ant/src/main/scala/Emit.scala delete mode 100644 arch/src/main/scala/framework/system/core/ant/src/main/scala/Execution.scala delete mode 100644 arch/src/main/scala/framework/system/core/ant/src/main/scala/Interfaces.scala delete mode 100644 arch/src/main/scala/framework/system/core/ant/src/main/scala/LocalExecution.scala delete mode 100644 arch/src/main/scala/framework/system/core/ant/src/main/scala/Service.scala delete mode 100644 arch/src/main/scala/framework/system/core/clink/Shm.scala delete mode 100644 arch/src/main/scala/framework/system/core/clink/base.scala delete mode 100644 arch/src/main/scala/framework/system/core/clink/clink.scala delete mode 100644 arch/src/main/scala/framework/system/core/rocket/BuckyballCommand.scala delete mode 100644 arch/src/main/scala/framework/system/core/rocket/CoreParameters.scala delete mode 100644 arch/src/main/scala/framework/system/core/rocket/CpuInterfaces.scala delete mode 100644 arch/src/main/scala/framework/system/core/rocket/CpuParams.scala delete mode 100644 arch/src/main/scala/framework/system/core/rocket/Parameters.scala delete mode 100644 arch/src/main/scala/framework/system/core/rocket/RoCCCommandBB.scala delete mode 100644 arch/src/main/scala/framework/system/core/rocket/RocketCoreBB.scala delete mode 100644 arch/src/main/scala/framework/system/core/rocket/RocketCoreParameters.scala delete mode 100644 arch/src/main/scala/framework/system/core/rocket/TileInterrupts.scala delete mode 100644 arch/src/main/scala/framework/system/core/rocket/TileParameters.scala delete mode 100644 arch/src/main/scala/framework/system/core/rocket/VectorInterfaces.scala delete mode 100644 arch/src/main/scala/framework/system/core/rocket/configs/RocketCpuParam.scala delete mode 160000 arch/src/main/scala/framework/system/core/seed delete mode 100644 arch/src/main/scala/framework/system/device/BootRom.scala delete mode 100644 arch/src/main/scala/framework/system/device/Clint.scala delete mode 100644 arch/src/main/scala/framework/system/device/Devices.scala delete mode 100644 arch/src/main/scala/framework/system/dlink/dlink.scala delete mode 100644 arch/src/main/scala/framework/system/memory/Memory.scala delete mode 100644 arch/src/main/scala/framework/system/memory/UncachedAxi.scala delete mode 100644 arch/src/main/scala/framework/system/tile/BankNetwork.scala delete mode 100644 arch/src/main/scala/framework/system/tile/BarrierUnit.scala delete mode 100644 arch/src/main/scala/framework/system/tile/ComputeTilePlatform.scala delete mode 100644 arch/src/main/scala/framework/system/tile/MainTile.scala delete mode 100644 arch/src/main/scala/framework/system/tile/MainTilePlatform.scala delete mode 100644 arch/src/main/scala/framework/system/tile/TileConnection.scala delete mode 100644 arch/src/main/scala/framework/system/tile/TileEndpoint.scala delete mode 100644 arch/src/main/scala/framework/system/tile/TileFactory.scala delete mode 100644 arch/src/main/scala/framework/system/tile/TileParameters.scala delete mode 100644 arch/src/main/scala/framework/system/tile/configs/TileParam.scala delete mode 100644 arch/src/main/scala/framework/system/tile/tlink/Control.scala delete mode 100644 arch/src/main/scala/framework/system/tile/tlink/Fabric.scala delete mode 100644 arch/src/main/scala/framework/system/tile/tlink/T2T.scala delete mode 100644 arch/src/main/scala/framework/system/tile/tlink/tlink.scala create mode 100644 arch/src/main/scala/framework/thread/ant/build.sc create mode 100644 arch/src/main/scala/framework/thread/ant/configs/AntParam.scala create mode 100644 arch/src/main/scala/framework/thread/ant/configs/default.toml rename arch/src/main/scala/framework/{system/core => thread}/ant/src/csrc/Cargo.lock (100%) rename arch/src/main/scala/framework/{system/core => thread}/ant/src/csrc/Cargo.toml (100%) create mode 100644 arch/src/main/scala/framework/thread/ant/src/csrc/src/lib.rs rename arch/src/main/scala/framework/{system/core => thread}/ant/src/main/resources/ant.cm_hier (100%) create mode 100644 arch/src/main/scala/framework/thread/ant/src/main/resources/ant.f create mode 100644 arch/src/main/scala/framework/thread/ant/src/main/resources/ant_tb.sv rename arch/src/main/scala/framework/{system/core => thread}/ant/src/main/resources/control.cm_hier (100%) create mode 100644 arch/src/main/scala/framework/thread/ant/src/main/resources/control.f rename arch/src/main/scala/framework/{system/core => thread}/ant/src/main/resources/control_if.sv (100%) create mode 100644 arch/src/main/scala/framework/thread/ant/src/main/resources/control_tb.sv rename arch/src/main/scala/framework/{system/core => thread}/ant/src/main/resources/service.cm_hier (100%) create mode 100644 arch/src/main/scala/framework/thread/ant/src/main/resources/service.f create mode 100644 arch/src/main/scala/framework/thread/ant/src/main/resources/service_tb.sv create mode 100644 arch/src/main/scala/framework/thread/ant/src/main/resources/suite.toml rename arch/src/main/scala/framework/{mem-core/tls => thread/ant}/src/main/resources/tls.cm_hier (100%) create mode 100644 arch/src/main/scala/framework/thread/ant/src/main/resources/tls.f rename arch/src/main/scala/framework/{mem-core/tls => thread/ant}/src/main/resources/tls.toml (100%) create mode 100644 arch/src/main/scala/framework/thread/ant/src/main/resources/tls_tb.sv create mode 100644 arch/src/main/scala/framework/thread/ant/src/main/scala/Ant.scala rename arch/src/main/scala/framework/{system/core => thread}/ant/src/main/scala/Decode.scala (97%) create mode 100644 arch/src/main/scala/framework/thread/ant/src/main/scala/Emit.scala create mode 100644 arch/src/main/scala/framework/thread/ant/src/main/scala/Executor.scala create mode 100644 arch/src/main/scala/framework/thread/ant/src/main/scala/LocalStore.scala create mode 100644 arch/src/main/scala/framework/thread/build.sc create mode 100644 arch/src/main/scala/framework/thread/configs/ThreadParam.scala create mode 100644 arch/src/main/scala/framework/thread/tlink/basethread.scala create mode 100644 arch/src/main/scala/framework/thread/tlink/build.sc create mode 100644 arch/src/main/scala/framework/thread/tlink/tlink.scala delete mode 100644 arch/src/main/scala/framework/top/GlobalConfig.scala delete mode 100644 arch/src/main/scala/framework/top/configs/DesignConfig.scala delete mode 100644 arch/src/main/scala/framework/top/configs/SimParam.scala create mode 100644 arch/src/main/scala/sims/build.sc create mode 100644 arch/src/main/scala/sims/configs/SimParam.scala create mode 100644 arch/src/main/scala/sims/core/CoreTarget.scala delete mode 100644 arch/src/main/scala/sims/hash/BankHash.scala delete mode 100644 arch/src/main/scala/sims/scu/SCU.scala delete mode 100644 arch/src/main/scala/sims/scu/SystemControl.scala create mode 100644 arch/thirdparty/build.sc delete mode 160000 arch/thirdparty/rocket-chip delete mode 160000 bb-tests/thirdparty/busybox delete mode 160000 bb-tests/thirdparty/linux delete mode 160000 bb-tests/thirdparty/opensbi delete mode 100644 bb-tests/workloads/common/CMakeLists.txt delete mode 100644 bb-tests/workloads/lib/CMakeLists.txt delete mode 100644 bb-tests/workloads/lib/bbhw/CMakeLists.txt delete mode 100644 bb-tests/workloads/lib/bbhw/isa/14_task.c delete mode 100644 bb-tests/workloads/lib/bbhw/isa/isa.c delete mode 100644 bb-tests/workloads/lib/bbsw/CRunnerUtils/CRunnerUtils.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/baremetal/bbsim.ld delete mode 100644 bb-tests/workloads/lib/bbsw/baremetal/crt0.S delete mode 100644 bb-tests/workloads/lib/bbsw/baremetal/multicore/multicore.h delete mode 100644 bb-tests/workloads/lib/bbsw/baremetal/start.S delete mode 100644 bb-tests/workloads/lib/bbsw/baremetal/syscalls.c delete mode 100644 bb-tests/workloads/lib/bbsw/baremetal/test.c delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/CMakeLists.txt delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/mxfp8/CMakeLists.txt delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/mxfp8/pack.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/mxfp8/quant.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/mxfp8/quant.h delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/mxquant/CMakeLists.txt delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/mxquant/dispatch.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/CMakeLists.txt delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/activation_dispatch.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/build_images.py delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/cache_dispatch.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/dequant.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/dispatch.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention.h delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_dispatch.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_mxfp8.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_mxfp8_dispatch.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/gelu.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/kernel.ld delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/kernels.h delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/layernorm.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/math/constants.h delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/math/cos.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/math/erf.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/math/exp.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/math/math.h delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/math/mxfp8.h delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/math/sin.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/matmul.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/matmul_dispatch.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/norm.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/normalization_dispatch.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/pack.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/pointwise.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/pointwise_dispatch.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/quant.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/quant_dispatch.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/rope.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/silu.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/snake.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/softmax.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/tanh.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/kernels/rvv/trig.cpp delete mode 100644 bb-tests/workloads/lib/bbsw/linux/multicore/multicore.h delete mode 100644 bb-tests/workloads/lib/bbsw/linux/test.c delete mode 100644 bb-tests/workloads/lib/bbsw/test.h delete mode 160000 bb-tests/workloads/lib/kernel create mode 100644 bb-tests/workloads/src/MLIRTest/base/base_memref_copy_dynamic.mlir delete mode 100644 bb-tests/workloads/src/MLIRTest/base/base_memref_copy_dynamic.mlir.in delete mode 100644 bb-tests/workloads/src/MLIRTest/compiler/CMakeLists.txt create mode 100644 bbdev/api/steps/bebop/bemu/scripts/bank_replay.py create mode 100644 bbdev/api/steps/bebop/bemu/scripts/bank_replay_native.py delete mode 100644 bbdev/api/steps/bebop/bemu/scripts/model_sim.py create mode 100644 bbdev/api/steps/bebop/vcs/04_sim_api.step.py delete mode 100644 bbdev/api/steps/compiler/01_build_api.step.py delete mode 100644 bbdev/api/steps/compiler/01_build_event.step.py delete mode 100644 bbdev/api/steps/compiler/scripts/build.py delete mode 100644 bbdev/api/steps/config/scripts/4_install.py create mode 100644 bbdev/api/steps/config/scripts/ball_parameters.py create mode 100644 bbdev/api/steps/config/scripts/boom_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/f32add_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/gemmini_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/hardware_pb2.py create mode 100644 bbdev/api/steps/config/scripts/im2col_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/int2fp_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/int8add_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/int8mul_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/lut_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/matadd_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/maxpool_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/mxfp2int_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/mxmm_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/mxquant_params_pb2.py delete mode 100644 bbdev/api/steps/config/scripts/proto/chip.proto create mode 100644 bbdev/api/steps/config/scripts/proto/design.proto create mode 100644 bbdev/api/steps/config/scripts/proto/hardware.proto create mode 100644 bbdev/api/steps/config/scripts/redmule_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/relu_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/rocket_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/smatmul_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/toint8_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/trace_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/transpose_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/vector_params_pb2.py create mode 100644 bbdev/api/steps/config/scripts/verification.py create mode 100644 bbdev/api/steps/config/scripts/xiangshan_params_pb2.py delete mode 100644 bbdev/api/steps/firesim/scripts/chip_yaml.py create mode 100644 bbdev/api/steps/firesim/scripts/design_yaml.py delete mode 100644 bbdev/api/steps/kernel/scripts/bin_to_hex.py create mode 100644 bbdev/api/steps/stack/01_build_api.step.py create mode 100644 bbdev/api/steps/stack/01_build_event.step.py create mode 100644 bbdev/api/steps/stack/scripts/build.py create mode 100644 bbdev/api/steps/uvm/scripts/ball.py create mode 100644 bbdev/api/steps/uvm/scripts/build.py create mode 100644 bbdev/api/steps/uvm/scripts/config.py create mode 100644 bbdev/api/steps/uvm/scripts/coverage.py create mode 100644 bbdev/api/steps/uvm/scripts/run.py delete mode 100644 bbdev/api/tests/test_api_compiler_build.py create mode 100644 bbdev/api/tests/test_api_stack_build.py delete mode 100644 bbdev/api/tests/test_bemu_binary.py delete mode 100644 bbdev/api/tests/test_p2e_runtime_reuse.py delete mode 100644 bbdev/api/tests/test_uvm_ip_profiles.py delete mode 100644 bbdev/api/unit_tests/test_process_cleanup.py delete mode 100644 bbdev/mcp/tools/compiler_build.py create mode 100644 bbdev/mcp/tools/stack_build.py delete mode 100644 examples/balls/f32add/arch/src/main/scala/F32AddBall.scala delete mode 100644 examples/balls/f32add/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/f32add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/f32add/configs/default.toml delete mode 100644 examples/balls/f32add/emu/src/dispatch.rs delete mode 100644 examples/balls/f32add/emu/src/lib.rs delete mode 100644 examples/balls/f32add/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir delete mode 100644 examples/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir delete mode 100644 examples/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir delete mode 100644 examples/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp delete mode 100644 examples/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir delete mode 100644 examples/balls/gemmini/arch/src/main/scala/GemminiBall.scala delete mode 100644 examples/balls/gemmini/arch/src/main/scala/GemminiExCtrl.scala delete mode 100644 examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlCmdStates.scala delete mode 100644 examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlDefaults.scala delete mode 100644 examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlDefs.scala delete mode 100644 examples/balls/gemmini/arch/src/main/scala/LoopCmd.scala delete mode 100644 examples/balls/gemmini/arch/src/main/scala/LoopCmdEncoder.scala delete mode 100644 examples/balls/gemmini/arch/src/main/scala/LoopConvUnroller.scala delete mode 100644 examples/balls/gemmini/arch/src/main/scala/LoopMatmulUnroller.scala delete mode 100644 examples/balls/gemmini/compiler/python/quant.py delete mode 100644 examples/balls/gemmini/compiler/runtime/conv.cpp delete mode 100644 examples/balls/gemmini/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/gemmini/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp delete mode 100644 examples/balls/gemmini/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/gemmini/configs/GemminiBallParam.scala delete mode 100644 examples/balls/gemmini/configs/default.toml delete mode 100644 examples/balls/gemmini/emu/src/02_gemmini_config.rs delete mode 100644 examples/balls/gemmini/emu/src/03_gemmini_flush.rs delete mode 100644 examples/balls/gemmini/emu/src/53_gemmini_preload.rs delete mode 100644 examples/balls/gemmini/emu/src/80_gemmini_loop_ws.rs delete mode 100644 examples/balls/gemmini/emu/src/dispatch.rs delete mode 100644 examples/balls/gemmini/emu/src/lib.rs delete mode 100644 examples/balls/gemmini/verify/src/tb_top.sv delete mode 100644 examples/balls/gemmini/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir delete mode 100644 examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir delete mode 100644 examples/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt delete mode 100644 examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir.in delete mode 100644 examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir.in delete mode 100644 examples/balls/im2col/arch/src/main/scala/Im2col.scala delete mode 100644 examples/balls/im2col/arch/src/main/scala/Im2colBall.scala delete mode 100644 examples/balls/im2col/arch/src/main/scala/Im2colConfigRegs.scala delete mode 100644 examples/balls/im2col/arch/src/main/scala/Im2colWindow.scala delete mode 100644 examples/balls/im2col/arch/src/main/scala/LineBufferManager.scala delete mode 100644 examples/balls/im2col/arch/src/main/scala/StreamWriter.scala delete mode 100644 examples/balls/im2col/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/im2col/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/im2col/configs/Im2colBallParam.scala delete mode 100644 examples/balls/im2col/configs/default.toml delete mode 100644 examples/balls/im2col/emu/src/dispatch.rs delete mode 100644 examples/balls/im2col/emu/src/lib.rs delete mode 100644 examples/balls/im2col/verify/README.md delete mode 100644 examples/balls/im2col/verify/casegen/src/casegen.rs delete mode 100644 examples/balls/im2col/verify/casegen/src/lib.rs delete mode 100644 examples/balls/im2col/verify/casegen/src/model.rs delete mode 100644 examples/balls/im2col/verify/src/tb_top.sv delete mode 100644 examples/balls/im2col/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir delete mode 100644 examples/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows.mlir.in delete mode 100644 examples/balls/int2fp/arch/src/main/scala/Int2Fp.scala delete mode 100644 examples/balls/int2fp/arch/src/main/scala/Int2FpBall.scala delete mode 100644 examples/balls/int2fp/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/int2fp/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/int2fp/configs/Int2FpBallParam.scala delete mode 100644 examples/balls/int2fp/configs/default.toml delete mode 100644 examples/balls/int2fp/emu/src/dispatch.rs delete mode 100644 examples/balls/int2fp/emu/src/lib.rs delete mode 100644 examples/balls/int2fp/emu/src/model.rs delete mode 100644 examples/balls/int2fp/verify/README.md delete mode 100644 examples/balls/int2fp/verify/casegen/src/casegen.rs delete mode 100644 examples/balls/int2fp/verify/casegen/src/lib.rs delete mode 100644 examples/balls/int2fp/verify/src/tb_top.sv delete mode 100644 examples/balls/int2fp/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir delete mode 100644 examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir delete mode 100644 examples/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt delete mode 100644 examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir delete mode 100644 examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir delete mode 100644 examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in delete mode 100644 examples/balls/int8add/arch/src/main/scala/Int8AddBall.scala delete mode 100644 examples/balls/int8add/arch/src/main/scala/PipelinedAddRawFN.scala delete mode 100644 examples/balls/int8add/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/int8add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/int8add/configs/default.toml delete mode 100644 examples/balls/int8add/emu/src/dispatch.rs delete mode 100644 examples/balls/int8add/emu/src/lib.rs delete mode 100644 examples/balls/int8add/verify/casegen/src/casegen.rs delete mode 100644 examples/balls/int8add/verify/casegen/src/lib.rs delete mode 100644 examples/balls/int8add/verify/src/tb_top.sv delete mode 100644 examples/balls/int8add/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir delete mode 100644 examples/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in delete mode 100644 examples/balls/int8mul/arch/src/main/scala/Int8MulBall.scala delete mode 100644 examples/balls/int8mul/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/int8mul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/int8mul/configs/default.toml delete mode 100644 examples/balls/int8mul/emu/src/dispatch.rs delete mode 100644 examples/balls/int8mul/emu/src/lib.rs delete mode 100644 examples/balls/int8mul/verify/casegen/src/casegen.rs delete mode 100644 examples/balls/int8mul/verify/casegen/src/lib.rs delete mode 100644 examples/balls/int8mul/verify/src/tb_top.sv delete mode 100644 examples/balls/int8mul/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir delete mode 100644 examples/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in delete mode 100644 examples/balls/lut/arch/src/main/scala/LutBall.scala delete mode 100644 examples/balls/lut/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/lut/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/lut/configs/default.toml delete mode 100644 examples/balls/lut/emu/src/dispatch.rs delete mode 100644 examples/balls/lut/emu/src/lib.rs delete mode 100644 examples/balls/lut/verify/casegen/src/casegen.rs delete mode 100644 examples/balls/lut/verify/casegen/src/lib.rs delete mode 100644 examples/balls/lut/verify/src/tb_top.sv delete mode 100644 examples/balls/lut/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir delete mode 100644 examples/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in delete mode 100644 examples/balls/matadd/arch/src/main/scala/MatAdd.scala delete mode 100644 examples/balls/matadd/arch/src/main/scala/MatAddBall.scala delete mode 100644 examples/balls/matadd/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/matadd/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/matadd/configs/default.toml delete mode 100644 examples/balls/matadd/emu/src/dispatch.rs delete mode 100644 examples/balls/matadd/emu/src/lib.rs delete mode 100644 examples/balls/matadd/verify/src/common/matadd_defs.svh delete mode 100644 examples/balls/matadd/verify/src/tb_top.sv delete mode 100644 examples/balls/matadd/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir delete mode 100644 examples/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in delete mode 100644 examples/balls/maxpool/arch/src/main/scala/MaxPoolBall.scala delete mode 100644 examples/balls/maxpool/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/maxpool/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/maxpool/configs/default.toml delete mode 100644 examples/balls/maxpool/emu/src/dispatch.rs delete mode 100644 examples/balls/maxpool/emu/src/lib.rs delete mode 100644 examples/balls/maxpool/verify/casegen/src/casegen.rs delete mode 100644 examples/balls/maxpool/verify/casegen/src/lib.rs delete mode 100644 examples/balls/maxpool/verify/casegen/src/model.rs delete mode 100644 examples/balls/maxpool/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir delete mode 100644 examples/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in delete mode 100644 examples/balls/mxfp2int/arch/src/main/scala/Mxfp2Int.scala delete mode 100644 examples/balls/mxfp2int/arch/src/main/scala/Mxfp2IntBall.scala delete mode 100644 examples/balls/mxfp2int/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/mxfp2int/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/mxfp2int/configs/Mxfp2IntBallParam.scala delete mode 100644 examples/balls/mxfp2int/configs/default.toml delete mode 100644 examples/balls/mxfp2int/emu/src/dispatch.rs delete mode 100644 examples/balls/mxfp2int/emu/src/lib.rs delete mode 100644 examples/balls/mxfp2int/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt delete mode 100644 examples/balls/mxmm/arch/src/main/scala/Array.scala delete mode 100644 examples/balls/mxmm/arch/src/main/scala/Emit.scala delete mode 100644 examples/balls/mxmm/arch/src/main/scala/InputWindow.scala delete mode 100644 examples/balls/mxmm/arch/src/main/scala/MxmmBall.scala delete mode 100644 examples/balls/mxmm/arch/src/main/scala/PE.scala delete mode 100644 examples/balls/mxmm/arch/src/main/scala/Unit.scala delete mode 100644 examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/FP32Patterns.inc delete mode 100644 examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp delete mode 100644 examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/MXFP8Patterns.inc delete mode 100644 examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/SharedPanelPatterns.inc delete mode 100644 examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/SharedWindowPatterns.inc delete mode 100644 examples/balls/mxmm/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/mxmm/compiler/src/Transforms/Passes.cpp delete mode 100644 examples/balls/mxmm/configs/MxmmBallParam.scala delete mode 100644 examples/balls/mxmm/configs/default.toml delete mode 100644 examples/balls/mxmm/emu/src/accumulator.rs delete mode 100644 examples/balls/mxmm/emu/src/dispatch.rs delete mode 100644 examples/balls/mxmm/emu/src/fp32.rs delete mode 100644 examples/balls/mxmm/emu/src/lib.rs delete mode 100644 examples/balls/mxmm/emu/src/mxfp8.rs delete mode 100644 examples/balls/mxmm/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/mxmm/workloads/kernels/CMakeLists.txt delete mode 100644 examples/balls/mxmm/workloads/kernels/build_image.py delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/CMakeLists.txt delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/ball/CMakeLists.txt delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window.mlir delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain.mlir delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows.mlir delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/bank/CMakeLists.txt delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window.mlir delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain.mlir delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows.mlir delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/tensor/CMakeLists.txt delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_fp32.mlir delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_fp32_main.cpp delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_branch.mlir.in delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_branch_main.cpp delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_input.mlir.in delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_input_main.cpp delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_layout.mlir.in delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_layout_main.cpp delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_shared.mlir.in delete mode 100644 examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_shared_main.cpp delete mode 100644 examples/balls/mxquant/arch/src/main/scala/MxquantBall.scala delete mode 100644 examples/balls/mxquant/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/mxquant/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/mxquant/configs/default.toml delete mode 100644 examples/balls/mxquant/emu/src/dispatch.rs delete mode 100644 examples/balls/mxquant/emu/src/lib.rs delete mode 100644 examples/balls/mxquant/emu/src/quant.rs delete mode 100644 examples/balls/mxquant/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/mxquant/workloads/mlir_tests/ball/CMakeLists.txt delete mode 100644 examples/balls/mxquant/workloads/mlir_tests/ball/mxquant.mlir delete mode 100644 examples/balls/mxquant/workloads/mlir_tests/bank/CMakeLists.txt delete mode 100644 examples/balls/mxquant/workloads/mlir_tests/bank/mxquant.mlir delete mode 100644 examples/balls/relu/README.md delete mode 100644 examples/balls/relu/arch/src/main/scala/Relu.scala delete mode 100644 examples/balls/relu/arch/src/main/scala/ReluBall.scala delete mode 100644 examples/balls/relu/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/relu/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp delete mode 100644 examples/balls/relu/compiler/src/Conversion/LowerTileToBuckyball/TileReluPatterns.cpp delete mode 100644 examples/balls/relu/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/relu/configs/default.toml delete mode 100644 examples/balls/relu/emu/src/dispatch.rs delete mode 100644 examples/balls/relu/emu/src/lib.rs delete mode 100644 examples/balls/relu/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/relu/workloads/isa/relu.h delete mode 100644 examples/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir delete mode 100644 examples/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in delete mode 100644 examples/balls/smatmul/arch/src/main/scala/Array.scala delete mode 100644 examples/balls/smatmul/arch/src/main/scala/PE.scala delete mode 100644 examples/balls/smatmul/arch/src/main/scala/SMatMulBall.scala delete mode 100644 examples/balls/smatmul/arch/src/main/scala/SMatMulUnit.scala delete mode 100644 examples/balls/smatmul/compiler/runtime/fp32.cpp delete mode 100644 examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp delete mode 100644 examples/balls/smatmul/compiler/src/Conversion/LowerTileToBuckyball/TileMatMulPatterns.cpp delete mode 100644 examples/balls/smatmul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/smatmul/configs/SMatMulBallParam.scala delete mode 100644 examples/balls/smatmul/configs/default.toml delete mode 100644 examples/balls/smatmul/configs/lane16.toml delete mode 100644 examples/balls/smatmul/emu/src/dispatch.rs delete mode 100644 examples/balls/smatmul/emu/src/fp32.rs delete mode 100644 examples/balls/smatmul/emu/src/lib.rs delete mode 100644 examples/balls/smatmul/emu/src/mxfp8.rs delete mode 100644 examples/balls/smatmul/verify/README.md delete mode 100644 examples/balls/smatmul/verify/casegen/src/casegen.rs delete mode 100644 examples/balls/smatmul/verify/casegen/src/model.rs delete mode 100644 examples/balls/smatmul/verify/src/tb_top.sv delete mode 100644 examples/balls/smatmul/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir delete mode 100644 examples/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt delete mode 100644 examples/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in delete mode 100644 examples/balls/toint8/arch/src/main/scala/ToInt8Ball.scala delete mode 100644 examples/balls/toint8/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/toint8/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/toint8/configs/default.toml delete mode 100644 examples/balls/toint8/emu/src/dispatch.rs delete mode 100644 examples/balls/toint8/emu/src/lib.rs delete mode 100644 examples/balls/toint8/emu/src/quant.rs delete mode 100644 examples/balls/toint8/emu/src/tests/activation_scale.rs delete mode 100644 examples/balls/toint8/emu/src/tests/int32_rounding.rs delete mode 100644 examples/balls/toint8/emu/src/tests/int8_saturation.rs delete mode 100644 examples/balls/toint8/emu/src/tests/mod.rs delete mode 100644 examples/balls/toint8/emu/src/tests/quant.rs delete mode 100644 examples/balls/toint8/emu/src/tests/workload_vectors.rs delete mode 100644 examples/balls/toint8/verify/README.md delete mode 100644 examples/balls/toint8/verify/casegen/src/casegen.rs delete mode 100644 examples/balls/toint8/verify/casegen/src/lib.rs delete mode 100644 examples/balls/toint8/verify/src/tb_top.sv delete mode 100644 examples/balls/toint8/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir delete mode 100644 examples/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir delete mode 100644 examples/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt delete mode 100644 examples/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in delete mode 100644 examples/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in delete mode 100644 examples/balls/trace/arch/src/main/scala/Trace.scala delete mode 100644 examples/balls/trace/arch/src/main/scala/TraceBall.scala delete mode 100644 examples/balls/trace/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/trace/configs/default.toml delete mode 100644 examples/balls/trace/emu/src/04_bdb_counter.rs delete mode 100644 examples/balls/trace/emu/src/dispatch.rs delete mode 100644 examples/balls/trace/emu/src/lib.rs delete mode 100644 examples/balls/trace/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir delete mode 100644 examples/balls/transpose/arch/src/main/scala/README.md delete mode 100644 examples/balls/transpose/arch/src/main/scala/Transpose.scala delete mode 100644 examples/balls/transpose/arch/src/main/scala/TransposeBall.scala delete mode 100644 examples/balls/transpose/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/transpose/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp delete mode 100644 examples/balls/transpose/compiler/src/Conversion/LowerTileToBuckyball/TileTransposePatterns.cpp delete mode 100644 examples/balls/transpose/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/transpose/configs/TransposeBallParam.scala delete mode 100644 examples/balls/transpose/configs/default.toml delete mode 100644 examples/balls/transpose/emu/src/dispatch.rs delete mode 100644 examples/balls/transpose/emu/src/lib.rs delete mode 100644 examples/balls/transpose/verify/README.md delete mode 100644 examples/balls/transpose/verify/casegen/src/casegen.rs delete mode 100644 examples/balls/transpose/verify/casegen/src/lib.rs delete mode 100644 examples/balls/transpose/verify/src/tb_top.sv delete mode 100644 examples/balls/transpose/workloads/ctests/CMakeLists.txt delete mode 100644 examples/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir delete mode 100644 examples/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in delete mode 100644 examples/balls/vector/arch/src/main/scala/VecBall.scala delete mode 100644 examples/balls/vector/arch/src/main/scala/VecCtrlUnit.scala delete mode 100644 examples/balls/vector/arch/src/main/scala/VecLoadUnit.scala delete mode 100644 examples/balls/vector/arch/src/main/scala/VecMat16.scala delete mode 100644 examples/balls/vector/arch/src/main/scala/VecMat16Store.scala delete mode 100644 examples/balls/vector/arch/src/main/scala/VecStoreUnit.scala delete mode 100644 examples/balls/vector/arch/src/main/scala/VecUnit.scala delete mode 100644 examples/balls/vector/arch/src/main/scala/bond/vvv.scala delete mode 100644 examples/balls/vector/arch/src/main/scala/thread/BaseThread.scala delete mode 100644 examples/balls/vector/arch/src/main/scala/thread/CasThread.scala delete mode 100644 examples/balls/vector/arch/src/main/scala/thread/MulThread.scala delete mode 100644 examples/balls/vector/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp delete mode 100644 examples/balls/vector/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/balls/vector/configs/VectorBallParam.scala delete mode 100644 examples/balls/vector/configs/default.toml delete mode 100644 examples/balls/vector/emu/src/dispatch.rs delete mode 100644 examples/balls/vector/emu/src/lib.rs delete mode 100644 examples/balls/vector/workloads/ctests/CMakeLists.txt create mode 100644 examples/build.sc delete mode 100644 examples/chips/goban/arch/src/main/scala/sims/p2e/TargetConfigs.scala delete mode 100644 examples/chips/goban/arch/src/main/scala/sims/verilator/TargetConfigs.scala delete mode 100644 examples/chips/goban/configs/chip.toml delete mode 100644 examples/chips/goban/configs/designs/goban.toml delete mode 100644 examples/chips/goban/configs/designs/tiles/default.toml delete mode 100644 examples/chips/goban/kernel/workloads.toml delete mode 100644 examples/chips/goban/regression/batch/bemu/workloads-bare.toml delete mode 100644 examples/chips/goban/regression/batch/bemu/workloads-linux.toml delete mode 100644 examples/chips/goban/regression/batch/p2e/workloads-linux.toml delete mode 100644 examples/chips/goban/regression/batch/verilator/workloads-bare.toml delete mode 100644 examples/chips/goban/workloads/CMakeLists.txt delete mode 100644 examples/chips/goban/workloads/ctests/CMakeLists.txt delete mode 100644 examples/chips/multi-rocket/arch/src/main/scala/sims/verilator/MultiRocketVerilatorTarget.scala delete mode 100644 examples/chips/multi-rocket/configs/chip.toml delete mode 100644 examples/chips/multi-rocket/configs/designs/multi-rocket.toml delete mode 100644 examples/chips/multi-rocket/configs/designs/tiles/32core.toml delete mode 100644 examples/chips/multi-rocket/kernel/workloads.toml delete mode 100644 examples/chips/multi-rocket/regression/batch/bemu/workloads-bare.toml delete mode 100644 examples/chips/multi-rocket/regression/batch/p2e/workloads-bare.toml delete mode 100644 examples/chips/multi-rocket/regression/batch/verilator/workloads-bare.toml delete mode 100644 examples/chips/multi-rocket/workloads/CMakeLists.txt delete mode 100644 examples/chips/omni/arch/src/main/scala/omni_tile/OmniTile.scala delete mode 100644 examples/chips/omni/arch/src/main/scala/poly_tile/PolyTile.scala delete mode 100644 examples/chips/omni/arch/src/main/scala/sims/OmniVerilatorTarget.scala delete mode 100644 examples/chips/omni/configs/chip.toml delete mode 100644 examples/chips/omni/configs/designs/omni.toml delete mode 100644 examples/chips/omni/configs/designs/tiles/main.toml delete mode 100644 examples/chips/omni/configs/designs/tiles/omni.toml delete mode 100644 examples/chips/omni/configs/designs/tiles/poly.toml delete mode 100644 examples/chips/omni/configs/system.toml delete mode 100644 examples/chips/omni/emu/src/main.rs delete mode 100644 examples/chips/omni/kernel/workloads.toml delete mode 100644 examples/chips/omni/regression/batch/bemu/workloads-bare.toml delete mode 100644 examples/chips/omni/regression/batch/bemu/workloads-linux.toml delete mode 100644 examples/chips/omni/regression/batch/p2e/workloads-linux.toml delete mode 100644 examples/chips/omni/tools/weights.py delete mode 100644 examples/chips/omni/workloads/CMakeLists.txt delete mode 100644 examples/chips/omni/workloads/ctests/CMakeLists.txt delete mode 100644 examples/chips/omni/workloads/ctests/backpressure.c delete mode 100644 examples/chips/omni/workloads/ctests/bounds.c delete mode 100644 examples/chips/omni/workloads/ctests/host_io.c delete mode 100644 examples/chips/omni/workloads/ctests/interconnect.c delete mode 100644 examples/chips/omni/workloads/ctests/mailbox.c delete mode 100644 examples/chips/omni/workloads/ctests/peripherals.c delete mode 100644 examples/chips/omni/workloads/ctests/process.c delete mode 100644 examples/chips/omni/workloads/ctests/reservation.c delete mode 100644 examples/chips/omni/workloads/mlir_tests/CMakeLists.txt delete mode 100644 examples/chips/omni/workloads/mlir_tests/attention_main.cpp delete mode 100644 examples/chips/omni/workloads/mlir_tests/moe_batch_main.cpp delete mode 100644 examples/chips/omni/workloads/mlir_tests/moe_main.cpp delete mode 100644 examples/chips/pebble/arch/src/main/scala/sims/p2e/TargetConfigs.scala delete mode 100644 examples/chips/pebble/arch/src/main/scala/sims/verilator/TargetConfigs.scala delete mode 100644 examples/chips/pebble/configs/chip.toml delete mode 100644 examples/chips/pebble/configs/designs/pebble.toml delete mode 100644 examples/chips/pebble/configs/designs/tiles/default.toml delete mode 100644 examples/chips/pebble/kernel/workloads.toml delete mode 100644 examples/chips/pebble/regression/batch/bemu/workloads-bare.toml delete mode 100644 examples/chips/pebble/regression/batch/bemu/workloads-linux.toml delete mode 100644 examples/chips/pebble/regression/batch/p2e/workloads-bare.toml delete mode 100644 examples/chips/pebble/regression/batch/verilator/workloads-bare.toml delete mode 100644 examples/chips/pebble/regression/batch/verilator/workloads-linux.toml delete mode 100644 examples/chips/pebble/regression/soc/README.md delete mode 100644 examples/chips/pebble/tapeout/README.md delete mode 100755 examples/chips/pebble/tapeout/area/dc.tcl delete mode 100644 examples/chips/pebble/tapeout/config.toml delete mode 100644 examples/chips/pebble/workloads/CMakeLists.txt delete mode 100644 examples/chips/pebble/workloads/ctests/CMakeLists.txt delete mode 100644 examples/chips/pebble/workloads/ctests/dual_conv_add_int2fp_test.c delete mode 100644 examples/chips/pebble/workloads/ctests/mega_conv_k5_pipeline_test.c delete mode 100644 examples/chips/pebble/workloads/mlir_tests/CMakeLists.txt delete mode 100644 examples/chips/pebble/workloads/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir delete mode 100644 examples/chips/pebble/workloads/mlir_tests/linalg/CMakeLists.txt delete mode 100644 examples/chips/pebble/workloads/mlir_tests/linalg/linalg_matmul_16x16_i8.mlir delete mode 100644 examples/chips/pebble/workloads/mlir_tests/linalg/linalg_mega_conv2d_k3_c16.mlir delete mode 100644 examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_16x16_i8.mlir delete mode 100644 examples/chips/pebble/workloads/mlir_tests/tile/CMakeLists.txt delete mode 100644 examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d.mlir delete mode 100644 examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_k5_tail.mlir delete mode 100644 examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_s2_depthwise_s2.mlir delete mode 100644 examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_7x7_c288_exact.mlir.in delete mode 100644 examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv2_exact.mlir.in delete mode 100644 examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv_exact.mlir.in delete mode 100644 examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul.mlir delete mode 100644 examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul_m17_k65_n17.mlir delete mode 100644 examples/chips/pebble/workloads/mlir_tests/tile/mega_se_gate_conv.mlir delete mode 100644 examples/chips/pebble/workloads/mlir_tests/tile/tile_matmul_16x16_i8.mlir delete mode 100644 examples/chips/pebble/workloads/mlir_tests/tile/tile_matmul_16x16_i8_main.cpp delete mode 100644 examples/chips/pebble/workloads/mlir_tests/tile/tile_transpose_16x16_i8.mlir delete mode 100644 examples/chips/pebble/workloads/mlir_tests/tile/tile_transpose_16x16_i8_main.cpp delete mode 100644 examples/chips/poly/arch/src/main/scala/compute_tile/ComputeTile.scala delete mode 100644 examples/chips/poly/arch/src/main/scala/sims/p2e/TargetConfigs.scala delete mode 100644 examples/chips/poly/arch/src/main/scala/sims/verilator/TargetConfigs.scala delete mode 100644 examples/chips/poly/configs/chip.toml delete mode 100644 examples/chips/poly/configs/designs/poly.toml delete mode 100644 examples/chips/poly/configs/designs/tiles/compute.toml delete mode 100644 examples/chips/poly/configs/designs/tiles/main.toml delete mode 100644 examples/chips/poly/kernel/workloads.toml delete mode 100644 examples/chips/poly/regression/batch/bemu/workloads-bare.toml delete mode 100644 examples/chips/poly/regression/batch/verilator/workloads-bare.toml delete mode 100644 examples/chips/poly/workloads/CMakeLists.txt delete mode 100644 examples/chips/poly/workloads/ctests/CMakeLists.txt delete mode 100644 examples/chips/poly/workloads/ctests/ant_bank_exercise.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_bank_main.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_bank_rows_exercise.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_bank_rows_main.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_coherence_main.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_highpa_main.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_mesh_move_main.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_mxmm_main.cpp delete mode 100644 examples/chips/poly/workloads/ctests/ant_parallel_exercise.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_parallel_main.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_shared.h delete mode 100644 examples/chips/poly/workloads/ctests/ant_shared_main.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_stream_main.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_tlink.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_tlink.h delete mode 100644 examples/chips/poly/workloads/ctests/ant_tlink_bank.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_tlink_bank.h delete mode 100644 examples/chips/poly/workloads/ctests/ant_tlink_bank_main.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_tlink_main.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_tlink_reduce.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_tlink_reduce.h delete mode 100644 examples/chips/poly/workloads/ctests/ant_tlink_reduce_control.c delete mode 100644 examples/chips/poly/workloads/ctests/ant_tlink_reduce_main.c delete mode 100644 examples/chips/poly/workloads/ctests/controllers_start.S delete mode 100644 examples/chips/poly/workloads/ctests/mesh_shared_mem_test.c delete mode 100644 examples/chips/toy/arch/src/main/scala/sims/verilator/TargetConfigs.scala delete mode 100644 examples/chips/toy/configs/chip.toml delete mode 100644 examples/chips/toy/configs/designs/tiles/default.toml delete mode 100644 examples/chips/toy/configs/designs/toy.toml delete mode 100644 examples/chips/toy/kernel/workloads.toml delete mode 100644 examples/chips/toy/regression/batch/bemu/workloads-bare.toml delete mode 100644 examples/chips/toy/regression/batch/p2e/workloads-bare.toml delete mode 100644 examples/chips/toy/regression/batch/verilator/workloads-bare.toml delete mode 100644 examples/chips/toy/regression/batch/verilator/workloads-linux.toml delete mode 100644 examples/chips/toy/regression/soc/README.md delete mode 100644 examples/chips/toy/workloads/CMakeLists.txt delete mode 100644 examples/chips/toy/workloads/ctests/CMakeLists.txt delete mode 100644 examples/chips/toy/workloads/ctests/alexnet_gemmini_round_test.c delete mode 100644 examples/chips/toy/workloads/ctests/alexnet_gemmini_stride_test.c delete mode 100644 examples/chips/toy/workloads/ctests/alexnet_gemmini_tail_test.c delete mode 100644 examples/chips/toy/workloads/mlir_tests/CMakeLists.txt delete mode 100644 examples/chips/toy/workloads/mlir_tests/linalg/CMakeLists.txt create mode 100644 examples/components/balls/build.sc create mode 100644 examples/components/balls/f32add/arch/F32AddBall.scala rename examples/{balls/f32add/arch/src/main/scala => components/balls/f32add/arch}/F32AddRow.scala (87%) rename arch/src/main/scala/framework/mem-core/chi/src/main/resources/README.md => examples/components/balls/f32add/configs/default.toml (100%) create mode 100644 examples/components/balls/f32add/configs/f32add_params.proto rename examples/{ => components}/balls/f32add/emu/src/add.rs (92%) create mode 100644 examples/components/balls/f32add/emu/src/dispatch.rs create mode 100644 examples/components/balls/f32add/emu/src/lib.rs create mode 100644 examples/components/balls/f32add/stack/src/Lowering.cpp create mode 100644 examples/components/balls/f32add/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/f32add/workloads/ctests/f32add_groups.c (100%) rename examples/{ => components}/balls/f32add/workloads/isa/f32add.h (100%) rename examples/{ => components}/balls/f32add/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{ => components}/balls/f32add/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir rename examples/{ => components}/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp (100%) create mode 100644 examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir rename examples/{ => components}/balls/f32add/workloads/mlir_tests/bank/CMakeLists.txt (100%) create mode 100644 examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir create mode 100644 examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp create mode 100644 examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir create mode 100644 examples/components/balls/gemmini/arch/GemminiBall.scala create mode 100644 examples/components/balls/gemmini/arch/GemminiExCtrl.scala rename examples/{balls/gemmini/arch/src/main/scala => components/balls/gemmini/arch}/GemminiExCtrlAccumulateState.scala (82%) create mode 100644 examples/components/balls/gemmini/arch/GemminiExCtrlCmdStates.scala rename examples/{balls/gemmini/arch/src/main/scala => components/balls/gemmini/arch}/GemminiExCtrlComputeFeedState.scala (92%) rename examples/{balls/gemmini/arch/src/main/scala => components/balls/gemmini/arch}/GemminiExCtrlComputeReadState.scala (88%) create mode 100644 examples/components/balls/gemmini/arch/GemminiExCtrlDefaults.scala create mode 100644 examples/components/balls/gemmini/arch/GemminiExCtrlDefs.scala rename examples/{balls/gemmini/arch/src/main/scala => components/balls/gemmini/arch}/GemminiExCtrlFsm.scala (83%) rename examples/{balls/gemmini/arch/src/main/scala => components/balls/gemmini/arch}/GemminiExCtrlPreloadStates.scala (93%) rename examples/{balls/gemmini/arch/src/main/scala => components/balls/gemmini/arch}/GemminiExCtrlStoreOps.scala (94%) rename examples/{balls/gemmini/arch/src/main/scala => components/balls/gemmini/arch}/LICENSE.gemmini (100%) create mode 100644 examples/components/balls/gemmini/arch/Loop.scala create mode 100644 examples/components/balls/gemmini/arch/LoopCmd.scala rename examples/{balls/gemmini/arch/src/main/scala => components/balls/gemmini/arch}/LoopConvAddrGen.scala (80%) create mode 100644 examples/components/balls/gemmini/arch/LoopConvUnroller.scala create mode 100644 examples/components/balls/gemmini/arch/LoopMatmulUnroller.scala create mode 100644 examples/components/balls/gemmini/configs/GemminiBallParam.scala create mode 100644 examples/components/balls/gemmini/configs/default.toml create mode 100644 examples/components/balls/gemmini/configs/gemmini_params.proto create mode 100644 examples/components/balls/gemmini/emu/src/02_gemmini_config.rs create mode 100644 examples/components/balls/gemmini/emu/src/03_gemmini_flush.rs create mode 100644 examples/components/balls/gemmini/emu/src/21_gemmini_preload.rs rename examples/{ => components}/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs (95%) rename examples/{ => components}/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs (93%) create mode 100644 examples/components/balls/gemmini/emu/src/80_gemmini_loop_ws.rs rename examples/{ => components}/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs (80%) create mode 100644 examples/components/balls/gemmini/emu/src/dispatch.rs rename examples/{ => components}/balls/gemmini/emu/src/gemmini_state.rs (85%) create mode 100644 examples/components/balls/gemmini/emu/src/lib.rs rename examples/{ => components}/balls/gemmini/emu/src/loop_micro_ops.rs (96%) create mode 100644 examples/components/balls/gemmini/stack/src/Lowering.cpp rename examples/{ => components}/balls/gemmini/verify/casegen/Cargo.lock (100%) rename examples/{ => components}/balls/gemmini/verify/casegen/Cargo.toml (100%) rename examples/{ => components}/balls/gemmini/verify/casegen/src/lib.rs (100%) rename examples/{ => components}/balls/gemmini/verify/filelists/gemmini_ball.f (97%) rename examples/{ => components}/balls/gemmini/verify/src/common/gemmini_defs.svh (79%) rename examples/{ => components}/balls/gemmini/verify/src/pkg/gemmini_pkg.sv (100%) create mode 100644 examples/components/balls/gemmini/verify/src/tb_top.sv rename examples/{ => components}/balls/gemmini/verify/src/tests/gemmini_test.svh (100%) create mode 100644 examples/components/balls/gemmini/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/gemmini/workloads/ctests/gemmini_accumulated_16x16_test.c (100%) create mode 100644 examples/components/balls/gemmini/workloads/ctests/gemmini_loop_mmio.c create mode 100644 examples/components/balls/gemmini/workloads/ctests/gemmini_loop_subrob.c rename examples/{ => components}/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c (97%) rename examples/{ => components}/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c (96%) rename examples/{ => components}/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c (97%) rename examples/{ => components}/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c (96%) rename examples/{ => components}/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c (97%) rename examples/{ => components}/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c (97%) rename examples/{ => components}/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c (97%) rename examples/{ => components}/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c (97%) rename examples/{ => components}/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c (97%) rename examples/{ => components}/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c (98%) rename examples/{ => components}/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c (97%) rename examples/{ => components}/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c (97%) rename examples/{ => components}/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c (97%) rename examples/{ => components}/balls/gemmini/workloads/isa/gemmini.h (100%) rename examples/{ => components}/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h (93%) rename examples/{ => components}/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h (100%) rename examples/{ => components}/balls/gemmini/workloads/isa/gemmini_config.h (100%) rename examples/{ => components}/balls/gemmini/workloads/isa/gemmini_flush.h (100%) rename examples/{ => components}/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h (100%) rename examples/{ => components}/balls/gemmini/workloads/isa/gemmini_loop_ws.h (100%) rename examples/{ => components}/balls/gemmini/workloads/isa/gemmini_preload.h (100%) rename examples/{ => components}/balls/gemmini/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{ => components}/balls/gemmini/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir rename examples/{ => components}/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16_main.cpp (100%) create mode 100644 examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir rename examples/{ => components}/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16_main.cpp (100%) create mode 100644 examples/components/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt create mode 100644 examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir.in rename examples/{ => components}/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp (93%) create mode 100644 examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir.in rename examples/{ => components}/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp (93%) create mode 100644 examples/components/balls/im2col/arch/Im2col.scala create mode 100644 examples/components/balls/im2col/arch/Im2colBall.scala create mode 100644 examples/components/balls/im2col/arch/Im2colConfigRegs.scala create mode 100644 examples/components/balls/im2col/arch/Im2colWindow.scala create mode 100644 examples/components/balls/im2col/arch/LineBufferManager.scala rename examples/{balls/im2col/arch/src/main/scala => components/balls/im2col/arch}/README.md (100%) rename examples/{balls/im2col/arch/src/main/scala => components/balls/im2col/arch}/RestoringDiv.scala (83%) create mode 100644 examples/components/balls/im2col/arch/StreamWriter.scala create mode 100644 examples/components/balls/im2col/configs/Im2colBallParam.scala create mode 100644 examples/components/balls/im2col/configs/default.toml create mode 100644 examples/components/balls/im2col/configs/im2col_params.proto create mode 100644 examples/components/balls/im2col/emu/src/dispatch.rs rename examples/{ => components}/balls/im2col/emu/src/im2col.rs (76%) create mode 100644 examples/components/balls/im2col/emu/src/lib.rs create mode 100644 examples/components/balls/im2col/stack/src/Lowering.cpp create mode 100644 examples/components/balls/im2col/verify/README.md rename examples/{ => components}/balls/im2col/verify/casegen/Cargo.lock (100%) rename examples/{ => components}/balls/im2col/verify/casegen/Cargo.toml (100%) create mode 100644 examples/components/balls/im2col/verify/casegen/src/casegen.rs create mode 100644 examples/components/balls/im2col/verify/casegen/src/lib.rs create mode 100644 examples/components/balls/im2col/verify/casegen/src/model.rs rename examples/{ => components}/balls/im2col/verify/filelists/im2col_ball.f (100%) rename examples/{ => components}/balls/im2col/verify/src/common/im2col_defs.svh (91%) rename examples/{ => components}/balls/im2col/verify/src/common/im2col_items.svh (97%) rename examples/{ => components}/balls/im2col/verify/src/cov/im2col_cov.svh (97%) rename examples/{ => components}/balls/im2col/verify/src/env/im2col_env.svh (100%) rename examples/{ => components}/balls/im2col/verify/src/env/im2col_scoreboard.svh (98%) rename examples/{ => components}/balls/im2col/verify/src/pkg/im2col_pkg.sv (100%) rename examples/{ => components}/balls/im2col/verify/src/seq/im2col_sequences.svh (100%) create mode 100644 examples/components/balls/im2col/verify/src/tb_top.sv rename examples/{ => components}/balls/im2col/verify/src/tests/im2col_test.svh (100%) create mode 100644 examples/components/balls/im2col/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/im2col/workloads/ctests/im2col_k1_test.c (97%) rename examples/{ => components}/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c (97%) rename examples/{ => components}/balls/im2col/workloads/ctests/im2col_k3_pad_test.c (97%) rename examples/{ => components}/balls/im2col/workloads/ctests/im2col_k3_partial_test.c (95%) rename examples/{ => components}/balls/im2col/workloads/ctests/im2col_k3_test.c (96%) rename examples/{ => components}/balls/im2col/workloads/ctests/im2col_k5_test.c (97%) rename examples/{ => components}/balls/im2col/workloads/ctests/im2col_k7_test.c (97%) rename examples/{ => components}/balls/im2col/workloads/ctests/im2col_window_test.c (100%) rename examples/{ => components}/balls/im2col/workloads/isa/im2col.h (100%) rename examples/{ => components}/balls/im2col/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{ => components}/balls/im2col/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir rename examples/{ => components}/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp (100%) rename examples/{ => components}/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt (100%) create mode 100644 examples/components/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows.mlir.in rename examples/{ => components}/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows_main.cpp (100%) rename examples/{balls/int2fp/arch/src/main/scala => components/balls/int2fp/arch}/Fp32Mul.scala (89%) create mode 100644 examples/components/balls/int2fp/arch/Int2Fp.scala create mode 100644 examples/components/balls/int2fp/arch/Int2FpBall.scala create mode 100644 examples/components/balls/int2fp/configs/Int2FpBallParam.scala create mode 100644 examples/components/balls/int2fp/configs/default.toml create mode 100644 examples/components/balls/int2fp/configs/int2fp_params.proto create mode 100644 examples/components/balls/int2fp/emu/src/dispatch.rs rename examples/{ => components}/balls/int2fp/emu/src/int2fp.rs (98%) create mode 100644 examples/components/balls/int2fp/emu/src/lib.rs create mode 100644 examples/components/balls/int2fp/emu/src/model.rs create mode 100644 examples/components/balls/int2fp/stack/src/Lowering.cpp create mode 100644 examples/components/balls/int2fp/verify/README.md rename examples/{ => components}/balls/int2fp/verify/casegen/Cargo.lock (100%) rename examples/{ => components}/balls/int2fp/verify/casegen/Cargo.toml (100%) create mode 100644 examples/components/balls/int2fp/verify/casegen/src/casegen.rs create mode 100644 examples/components/balls/int2fp/verify/casegen/src/lib.rs rename examples/{ => components}/balls/int2fp/verify/filelists/int2fp_ball.f (100%) rename examples/{ => components}/balls/int2fp/verify/src/common/int2fp_defs.svh (91%) rename examples/{ => components}/balls/int2fp/verify/src/common/int2fp_items.svh (96%) rename examples/{ => components}/balls/int2fp/verify/src/cov/int2fp_cov.svh (91%) rename examples/{ => components}/balls/int2fp/verify/src/env/int2fp_env.svh (100%) rename examples/{ => components}/balls/int2fp/verify/src/env/int2fp_scoreboard.svh (98%) rename examples/{ => components}/balls/int2fp/verify/src/pkg/int2fp_pkg.sv (100%) rename examples/{ => components}/balls/int2fp/verify/src/seq/int2fp_sequences.svh (100%) create mode 100644 examples/components/balls/int2fp/verify/src/tb_top.sv rename examples/{ => components}/balls/int2fp/verify/src/tests/int2fp_test.svh (100%) create mode 100644 examples/components/balls/int2fp/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c (96%) rename examples/{ => components}/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c (96%) rename examples/{ => components}/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c (96%) rename examples/{ => components}/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c (96%) rename examples/{ => components}/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c (96%) rename examples/{ => components}/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c (96%) rename examples/{ => components}/balls/int2fp/workloads/isa/int2fp.h (100%) rename examples/{ => components}/balls/int2fp/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{ => components}/balls/int2fp/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir rename examples/{ => components}/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4_main.cpp (100%) create mode 100644 examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir rename examples/{ => components}/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4_main.cpp (100%) create mode 100644 examples/components/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt create mode 100644 examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir rename examples/{ => components}/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4_main.cpp (100%) create mode 100644 examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir rename examples/{ => components}/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4_main.cpp (100%) create mode 100644 examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in rename examples/{ => components}/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp (100%) create mode 100644 examples/components/balls/int8add/arch/Int8AddBall.scala create mode 100644 examples/components/balls/int8add/arch/PipelinedAddRawFN.scala create mode 100644 examples/components/balls/int8add/configs/default.toml create mode 100644 examples/components/balls/int8add/configs/int8add_params.proto create mode 100644 examples/components/balls/int8add/emu/src/dispatch.rs rename examples/{ => components}/balls/int8add/emu/src/int8add.rs (97%) create mode 100644 examples/components/balls/int8add/emu/src/lib.rs create mode 100644 examples/components/balls/int8add/stack/src/Lowering.cpp rename examples/{ => components}/balls/int8add/verify/casegen/Cargo.lock (100%) rename examples/{ => components}/balls/int8add/verify/casegen/Cargo.toml (100%) create mode 100644 examples/components/balls/int8add/verify/casegen/src/casegen.rs create mode 100644 examples/components/balls/int8add/verify/casegen/src/lib.rs rename examples/{ => components}/balls/int8add/verify/filelists/int8add_ball.f (100%) rename examples/{ => components}/balls/int8add/verify/src/common/int8add_defs.svh (85%) rename examples/{ => components}/balls/int8add/verify/src/common/int8add_items.svh (93%) rename examples/{ => components}/balls/int8add/verify/src/cov/int8add_cov.svh (79%) rename examples/{ => components}/balls/int8add/verify/src/env/int8add_env.svh (100%) rename examples/{ => components}/balls/int8add/verify/src/env/int8add_scoreboard.svh (98%) rename examples/{ => components}/balls/int8add/verify/src/pkg/int8add_pkg.sv (100%) rename examples/{ => components}/balls/int8add/verify/src/seq/int8add_sequences.svh (100%) create mode 100644 examples/components/balls/int8add/verify/src/tb_top.sv rename examples/{ => components}/balls/int8add/verify/src/tests/int8add_test.svh (100%) create mode 100644 examples/components/balls/int8add/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/int8add/workloads/ctests/int8add_relu_test.c (100%) rename examples/{ => components}/balls/int8add/workloads/ctests/int8add_scaled_test.c (100%) rename examples/{ => components}/balls/int8add/workloads/isa/int8add.h (100%) rename examples/{ => components}/balls/int8add/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{ => components}/balls/int8add/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/components/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir rename examples/{ => components}/balls/int8add/workloads/mlir_tests/ball/int8add_relu_main.cpp (100%) rename examples/{ => components}/balls/int8add/workloads/mlir_tests/bank/CMakeLists.txt (100%) create mode 100644 examples/components/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in rename examples/{ => components}/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp (100%) create mode 100644 examples/components/balls/int8mul/arch/Int8MulBall.scala create mode 100644 examples/components/balls/int8mul/configs/default.toml create mode 100644 examples/components/balls/int8mul/configs/int8mul_params.proto create mode 100644 examples/components/balls/int8mul/emu/src/dispatch.rs rename examples/{ => components}/balls/int8mul/emu/src/int8mul.rs (98%) create mode 100644 examples/components/balls/int8mul/emu/src/lib.rs create mode 100644 examples/components/balls/int8mul/stack/src/Lowering.cpp rename examples/{ => components}/balls/int8mul/verify/casegen/Cargo.lock (100%) rename examples/{ => components}/balls/int8mul/verify/casegen/Cargo.toml (100%) create mode 100644 examples/components/balls/int8mul/verify/casegen/src/casegen.rs rename examples/{ => components}/balls/int8mul/verify/casegen/src/config.rs (100%) create mode 100644 examples/components/balls/int8mul/verify/casegen/src/lib.rs rename examples/{ => components}/balls/int8mul/verify/filelists/int8mul_ball.f (100%) rename examples/{ => components}/balls/int8mul/verify/src/common/int8mul_defs.svh (92%) rename examples/{ => components}/balls/int8mul/verify/src/common/int8mul_items.svh (96%) rename examples/{ => components}/balls/int8mul/verify/src/cov/int8mul_cov.svh (90%) rename examples/{ => components}/balls/int8mul/verify/src/env/int8mul_env.svh (100%) rename examples/{ => components}/balls/int8mul/verify/src/env/int8mul_scoreboard.svh (98%) rename examples/{ => components}/balls/int8mul/verify/src/pkg/int8mul_pkg.sv (100%) rename examples/{ => components}/balls/int8mul/verify/src/seq/int8mul_sequences.svh (100%) create mode 100644 examples/components/balls/int8mul/verify/src/tb_top.sv rename examples/{ => components}/balls/int8mul/verify/src/tests/int8mul_test.svh (100%) create mode 100644 examples/components/balls/int8mul/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c (100%) rename examples/{ => components}/balls/int8mul/workloads/ctests/int8mul_per_channel_gate_test.c (100%) rename examples/{ => components}/balls/int8mul/workloads/isa/int8mul.h (100%) rename examples/{ => components}/balls/int8mul/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{ => components}/balls/int8mul/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/components/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir rename examples/{ => components}/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate_main.cpp (100%) rename examples/{ => components}/balls/int8mul/workloads/mlir_tests/bank/CMakeLists.txt (100%) create mode 100644 examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in rename examples/{ => components}/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp (100%) create mode 100644 examples/components/balls/lut/arch/LutBall.scala create mode 100644 examples/components/balls/lut/configs/default.toml create mode 100644 examples/components/balls/lut/configs/lut_params.proto create mode 100644 examples/components/balls/lut/emu/src/dispatch.rs create mode 100644 examples/components/balls/lut/emu/src/lib.rs rename examples/{ => components}/balls/lut/emu/src/lut.rs (94%) create mode 100644 examples/components/balls/lut/stack/src/Lowering.cpp rename examples/{ => components}/balls/lut/verify/casegen/Cargo.lock (100%) rename examples/{ => components}/balls/lut/verify/casegen/Cargo.toml (100%) create mode 100644 examples/components/balls/lut/verify/casegen/src/casegen.rs create mode 100644 examples/components/balls/lut/verify/casegen/src/lib.rs rename examples/{ => components}/balls/lut/verify/filelists/lut_ball.f (100%) rename examples/{ => components}/balls/lut/verify/src/common/lut_defs.svh (93%) rename examples/{ => components}/balls/lut/verify/src/common/lut_items.svh (97%) rename examples/{ => components}/balls/lut/verify/src/cov/lut_cov.svh (92%) rename examples/{ => components}/balls/lut/verify/src/env/lut_env.svh (100%) rename examples/{ => components}/balls/lut/verify/src/env/lut_scoreboard.svh (98%) rename examples/{ => components}/balls/lut/verify/src/pkg/lut_pkg.sv (100%) rename examples/{ => components}/balls/lut/verify/src/seq/lut_sequences.svh (100%) create mode 100644 examples/components/balls/lut/verify/src/tb_top.sv rename examples/{ => components}/balls/lut/verify/src/tests/lut_test.svh (100%) create mode 100644 examples/components/balls/lut/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c (100%) rename examples/{ => components}/balls/lut/workloads/ctests/lut_uniform_lookup_test.c (100%) rename examples/{ => components}/balls/lut/workloads/isa/lut.h (100%) rename examples/{ => components}/balls/lut/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{ => components}/balls/lut/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/components/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir rename examples/{ => components}/balls/lut/workloads/mlir_tests/ball/lut_identity_main.cpp (100%) rename examples/{ => components}/balls/lut/workloads/mlir_tests/bank/CMakeLists.txt (100%) create mode 100644 examples/components/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in rename examples/{ => components}/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp (100%) rename examples/{ => components}/balls/matadd/README.md (100%) create mode 100644 examples/components/balls/matadd/arch/MatAdd.scala create mode 100644 examples/components/balls/matadd/arch/MatAddBall.scala rename arch/src/main/scala/framework/mem-core/coherence/src/csrc/README.md => examples/components/balls/matadd/configs/default.toml (100%) create mode 100644 examples/components/balls/matadd/configs/matadd_params.proto rename examples/{ => components}/balls/matadd/emu/src/64_matadd.rs (94%) create mode 100644 examples/components/balls/matadd/emu/src/dispatch.rs create mode 100644 examples/components/balls/matadd/emu/src/lib.rs create mode 100644 examples/components/balls/matadd/stack/src/Lowering.cpp rename examples/{ => components}/balls/matadd/verify/casegen/Cargo.lock (100%) rename examples/{ => components}/balls/matadd/verify/casegen/Cargo.toml (100%) rename examples/{ => components}/balls/matadd/verify/casegen/src/lib.rs (100%) rename examples/{ => components}/balls/matadd/verify/filelists/matadd_ball.f (100%) create mode 100644 examples/components/balls/matadd/verify/src/common/matadd_defs.svh rename examples/{ => components}/balls/matadd/verify/src/common/matadd_items.svh (97%) rename examples/{ => components}/balls/matadd/verify/src/env/matadd_env.svh (100%) rename examples/{ => components}/balls/matadd/verify/src/env/matadd_scoreboard.svh (99%) rename examples/{ => components}/balls/matadd/verify/src/pkg/matadd_pkg.sv (100%) rename examples/{ => components}/balls/matadd/verify/src/seq/matadd_sequences.svh (100%) create mode 100644 examples/components/balls/matadd/verify/src/tb_top.sv rename examples/{ => components}/balls/matadd/verify/src/tests/matadd_test.svh (100%) create mode 100644 examples/components/balls/matadd/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/matadd/workloads/ctests/matadd_i32_16row_test.c (100%) rename examples/{ => components}/balls/matadd/workloads/isa/matadd.h (100%) rename examples/{ => components}/balls/matadd/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{ => components}/balls/matadd/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir rename examples/{ => components}/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp (100%) rename examples/{ => components}/balls/matadd/workloads/mlir_tests/bank/CMakeLists.txt (100%) create mode 100644 examples/components/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in rename examples/{ => components}/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp (100%) create mode 100644 examples/components/balls/maxpool/arch/MaxPoolBall.scala create mode 100644 examples/components/balls/maxpool/configs/default.toml create mode 100644 examples/components/balls/maxpool/configs/maxpool_params.proto create mode 100644 examples/components/balls/maxpool/emu/src/dispatch.rs create mode 100644 examples/components/balls/maxpool/emu/src/lib.rs rename examples/{ => components}/balls/maxpool/emu/src/maxpool.rs (97%) create mode 100644 examples/components/balls/maxpool/stack/src/Lowering.cpp rename examples/{ => components}/balls/maxpool/verify/casegen/Cargo.lock (100%) rename examples/{ => components}/balls/maxpool/verify/casegen/Cargo.toml (100%) create mode 100644 examples/components/balls/maxpool/verify/casegen/src/casegen.rs create mode 100644 examples/components/balls/maxpool/verify/casegen/src/lib.rs create mode 100644 examples/components/balls/maxpool/verify/casegen/src/model.rs rename examples/{ => components}/balls/maxpool/verify/filelists/maxpool_ball.f (100%) rename examples/{ => components}/balls/maxpool/verify/src/common/maxpool_defs.svh (93%) rename examples/{ => components}/balls/maxpool/verify/src/common/maxpool_items.svh (98%) rename examples/{ => components}/balls/maxpool/verify/src/cov/maxpool_cov.svh (93%) rename examples/{ => components}/balls/maxpool/verify/src/env/maxpool_env.svh (100%) rename examples/{ => components}/balls/maxpool/verify/src/env/maxpool_scoreboard.svh (97%) rename examples/{ => components}/balls/maxpool/verify/src/pkg/maxpool_pkg.sv (100%) rename examples/{ => components}/balls/maxpool/verify/src/seq/maxpool_sequences.svh (100%) rename examples/{ => components}/balls/maxpool/verify/src/tb_top.sv (100%) rename examples/{ => components}/balls/maxpool/verify/src/tests/maxpool_test.svh (100%) create mode 100644 examples/components/balls/maxpool/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/maxpool/workloads/ctests/maxpool_offset_test.c (100%) rename examples/{ => components}/balls/maxpool/workloads/ctests/maxpool_window_test.c (100%) rename examples/{ => components}/balls/maxpool/workloads/isa/maxpool.h (93%) rename examples/{ => components}/balls/maxpool/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{ => components}/balls/maxpool/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/components/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir rename examples/{ => components}/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2_main.cpp (100%) rename examples/{ => components}/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt (100%) create mode 100644 examples/components/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in rename examples/{ => components}/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp (100%) create mode 100644 examples/components/balls/mxfp2int/arch/Mxfp2Int.scala create mode 100644 examples/components/balls/mxfp2int/arch/Mxfp2IntBall.scala rename examples/{balls/mxfp2int/arch/src/main/scala => components/balls/mxfp2int/arch}/Mxfp4Decoder.scala (100%) rename examples/{balls/mxfp2int/arch/src/main/scala => components/balls/mxfp2int/arch}/Mxfp6Decoder.scala (100%) rename examples/{balls/mxfp2int/arch/src/main/scala => components/balls/mxfp2int/arch}/Mxfp8Decoder.scala (100%) rename examples/{balls/mxfp2int/arch/src/main/scala => components/balls/mxfp2int/arch}/MxfpDecoder.scala (100%) create mode 100644 examples/components/balls/mxfp2int/configs/Mxfp2IntBallParam.scala create mode 100644 examples/components/balls/mxfp2int/configs/default.toml create mode 100644 examples/components/balls/mxfp2int/configs/mxfp2int_params.proto rename examples/{ => components}/balls/mxfp2int/emu/src/55_mxfp2int.rs (96%) create mode 100644 examples/components/balls/mxfp2int/emu/src/dispatch.rs create mode 100644 examples/components/balls/mxfp2int/emu/src/lib.rs create mode 100644 examples/components/balls/mxfp2int/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c (98%) rename examples/{ => components}/balls/mxfp2int/workloads/isa/mxfp2int.h (100%) rename examples/{ => components}/balls/mxfp2int/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{ => components}/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt (100%) rename examples/{ => components}/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks.mlir (100%) rename examples/{ => components}/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks_main.cpp (100%) create mode 100644 examples/components/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt rename examples/{ => components}/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank.mlir.in (100%) rename examples/{ => components}/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank_main.cpp (100%) rename examples/{balls/mxmm/arch/src/main/scala => components/balls/mxmm/arch}/AddPipe.scala (87%) create mode 100644 examples/components/balls/mxmm/arch/Array.scala create mode 100644 examples/components/balls/mxmm/arch/InputWindow.scala rename examples/{balls/mxmm/arch/src/main/scala => components/balls/mxmm/arch}/Mxfp8Decode.scala (100%) create mode 100644 examples/components/balls/mxmm/arch/MxmmBall.scala create mode 100644 examples/components/balls/mxmm/arch/PE.scala rename examples/{balls/mxmm/arch/src/main/scala => components/balls/mxmm/arch}/Panels.scala (95%) create mode 100644 examples/components/balls/mxmm/arch/Unit.scala create mode 100644 examples/components/balls/mxmm/configs/MxmmBallParam.scala create mode 100644 examples/components/balls/mxmm/configs/default.toml create mode 100644 examples/components/balls/mxmm/configs/mxmm_params.proto create mode 100644 examples/components/balls/mxmm/emu/src/accumulator.rs create mode 100644 examples/components/balls/mxmm/emu/src/dispatch.rs create mode 100644 examples/components/balls/mxmm/emu/src/fp32.rs create mode 100644 examples/components/balls/mxmm/emu/src/lib.rs create mode 100644 examples/components/balls/mxmm/emu/src/mxfp8.rs create mode 100644 examples/components/balls/mxmm/stack/kernels/CMakeLists.txt rename examples/{balls/mxmm/workloads => components/balls/mxmm/stack}/kernels/dispatch.cpp (100%) rename examples/{balls/mxmm/workloads => components/balls/mxmm/stack}/kernels/norm_window.cpp (97%) create mode 100644 examples/components/balls/mxmm/stack/kernels/norm_window_image.h rename examples/{balls/mxmm/workloads => components/balls/mxmm/stack}/kernels/pack_dispatch.cpp (100%) rename examples/{balls/mxmm/workloads => components/balls/mxmm/stack}/kernels/packmm.cpp (97%) rename examples/{balls/mxmm/workloads => components/balls/mxmm/stack}/kernels/packmm.h (100%) create mode 100644 examples/components/balls/mxmm/stack/kernels/packmm_image.h rename examples/{balls/mxmm/workloads => components/balls/mxmm/stack}/kernels/window.h (100%) create mode 100644 examples/components/balls/mxmm/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c (100%) rename examples/{ => components}/balls/mxmm/workloads/ctests/mxmm_fp32_test.c (100%) rename examples/{ => components}/balls/mxmm/workloads/ctests/mxmm_mxfp8_test.c (100%) rename examples/{ => components}/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c (100%) rename examples/{ => components}/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c (100%) rename examples/{ => components}/balls/mxmm/workloads/ctests/mxmm_window_test.c (100%) rename examples/{ => components}/balls/mxmm/workloads/isa/mxmm.h (100%) rename examples/{balls/relu => components/balls/mxmm}/workloads/mlir_tests/CMakeLists.txt (100%) create mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/CMakeLists.txt rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir (100%) rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32_main.cpp (100%) rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir (100%) rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32_main.cpp (100%) rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8.mlir (100%) rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_main.cpp (100%) create mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window.mlir create mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain.mlir rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain_main.cpp (100%) rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_main.cpp (100%) create mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows.mlir rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows_main.cpp (100%) create mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/CMakeLists.txt rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir (100%) rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32_main.cpp (100%) rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir (100%) rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32_main.cpp (100%) rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8.mlir (100%) rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_main.cpp (100%) create mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window.mlir create mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain.mlir rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain_main.cpp (100%) rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_main.cpp (100%) create mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows.mlir rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows_main.cpp (100%) rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/mxmm_native_steps.h (100%) rename examples/{ => components}/balls/mxmm/workloads/mlir_tests/mxmm_native_test.h (100%) rename examples/{balls/mxquant/arch/src/main/scala => components/balls/mxquant/arch}/Encode.scala (94%) create mode 100644 examples/components/balls/mxquant/arch/MxquantBall.scala rename arch/src/main/scala/framework/mem-core/coherence/src/main/resources/README.md => examples/components/balls/mxquant/configs/default.toml (100%) create mode 100644 examples/components/balls/mxquant/configs/mxquant_params.proto create mode 100644 examples/components/balls/mxquant/emu/src/dispatch.rs create mode 100644 examples/components/balls/mxquant/emu/src/lib.rs create mode 100644 examples/components/balls/mxquant/emu/src/quant.rs create mode 100644 examples/components/balls/mxquant/stack/kernels/CMakeLists.txt create mode 100644 examples/components/balls/mxquant/stack/kernels/dispatch.cpp create mode 100644 examples/components/balls/mxquant/stack/kernels/pack.cpp create mode 100644 examples/components/balls/mxquant/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/mxquant/workloads/ctests/mxquant_capacity_test.c (100%) rename examples/{ => components}/balls/mxquant/workloads/ctests/mxquant_values_test.c (100%) rename examples/{ => components}/balls/mxquant/workloads/isa/mxquant.h (100%) rename examples/{ => components}/balls/mxquant/workloads/mlir_tests/CMakeLists.txt (100%) create mode 100644 examples/components/balls/mxquant/workloads/mlir_tests/ball/CMakeLists.txt create mode 100644 examples/components/balls/mxquant/workloads/mlir_tests/ball/mxquant.mlir rename examples/{ => components}/balls/mxquant/workloads/mlir_tests/ball/mxquant_main.cpp (100%) create mode 100644 examples/components/balls/mxquant/workloads/mlir_tests/bank/CMakeLists.txt create mode 100644 examples/components/balls/mxquant/workloads/mlir_tests/bank/mxquant.mlir rename examples/{ => components}/balls/mxquant/workloads/mlir_tests/bank/mxquant_main.cpp (100%) rename examples/{ => components}/balls/mxquant/workloads/reference.h (94%) create mode 100644 examples/components/balls/redmule/arch/RedmuleBall.scala create mode 100644 examples/components/balls/redmule/arch/RedmuleNative.scala create mode 100644 examples/components/balls/redmule/arch/RedmuleNative.sv create mode 100644 examples/components/balls/redmule/configs/default.toml create mode 100644 examples/components/balls/redmule/configs/redmule_params.proto create mode 100644 examples/components/balls/redmule/emu/src/fp16.rs create mode 100644 examples/components/balls/redmule/emu/src/lib.rs create mode 100644 examples/components/balls/redmule/emu/src/matmul.rs create mode 100644 examples/components/balls/redmule/stack/src/Lowering.cpp create mode 100644 examples/components/balls/redmule/verify/casegen/Cargo.lock create mode 100644 examples/components/balls/redmule/verify/casegen/Cargo.toml create mode 100644 examples/components/balls/redmule/verify/casegen/src/lib.rs create mode 100644 examples/components/balls/redmule/verify/filelists/redmule_ball.f create mode 100644 examples/components/balls/redmule/verify/parameters.toml create mode 100644 examples/components/balls/redmule/verify/src/pkg/redmule_pkg.sv create mode 100644 examples/components/balls/redmule/verify/src/reference/redmule_reference.sv create mode 100644 examples/components/balls/redmule/verify/src/tb_top.sv create mode 100644 examples/components/balls/redmule/workloads/ctests/CMakeLists.txt create mode 100644 examples/components/balls/redmule/workloads/ctests/redmule_f16_chain_test.c create mode 100644 examples/components/balls/redmule/workloads/ctests/redmule_f16_edges_test.c create mode 100644 examples/components/balls/redmule/workloads/ctests/redmule_f16_fused_test.c create mode 100644 examples/components/balls/redmule/workloads/ctests/redmule_f16_test.c create mode 100644 examples/components/balls/redmule/workloads/isa/redmule.h create mode 100644 examples/components/balls/redmule/workloads/mlir_tests/CMakeLists.txt create mode 100644 examples/components/balls/redmule/workloads/mlir_tests/ball/CMakeLists.txt create mode 100644 examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir create mode 100644 examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp create mode 100644 examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir create mode 100644 examples/components/balls/redmule/workloads/mlir_tests/bank/CMakeLists.txt create mode 100644 examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir create mode 100644 examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in create mode 100644 examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows_main.cpp create mode 100644 examples/components/balls/relu/README.md create mode 100644 examples/components/balls/relu/arch/Relu.scala create mode 100644 examples/components/balls/relu/arch/ReluBall.scala create mode 100644 examples/components/balls/relu/configs/default.toml create mode 100644 examples/components/balls/relu/configs/relu_params.proto rename examples/{ => components}/balls/relu/emu/src/50_relu.rs (89%) create mode 100644 examples/components/balls/relu/emu/src/dispatch.rs create mode 100644 examples/components/balls/relu/emu/src/lib.rs create mode 100644 examples/components/balls/relu/stack/src/Lowering.cpp create mode 100644 examples/components/balls/relu/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/relu/workloads/ctests/relu_i32_16x16_test.c (100%) create mode 100644 examples/components/balls/relu/workloads/isa/relu.h create mode 100644 examples/components/balls/relu/workloads/mlir_tests/CMakeLists.txt rename examples/{ => components}/balls/relu/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/components/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir rename examples/{ => components}/balls/relu/workloads/mlir_tests/ball/relu_i32_16row_main.cpp (100%) rename examples/{ => components}/balls/relu/workloads/mlir_tests/bank/CMakeLists.txt (100%) create mode 100644 examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in rename examples/{ => components}/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp (100%) rename examples/{ => components}/balls/smatmul/README.md (100%) create mode 100644 examples/components/balls/smatmul/arch/Array.scala create mode 100644 examples/components/balls/smatmul/arch/PE.scala create mode 100644 examples/components/balls/smatmul/arch/SMatMulBall.scala create mode 100644 examples/components/balls/smatmul/arch/SMatMulUnit.scala create mode 100644 examples/components/balls/smatmul/configs/SMatMulBallParam.scala create mode 100644 examples/components/balls/smatmul/configs/default.toml create mode 100644 examples/components/balls/smatmul/configs/lane16.toml create mode 100644 examples/components/balls/smatmul/configs/smatmul_params.proto create mode 100644 examples/components/balls/smatmul/emu/src/dispatch.rs create mode 100644 examples/components/balls/smatmul/emu/src/fp32.rs create mode 100644 examples/components/balls/smatmul/emu/src/lib.rs create mode 100644 examples/components/balls/smatmul/emu/src/mxfp8.rs rename examples/{ => components}/balls/smatmul/emu/src/smatmul.rs (84%) create mode 100644 examples/components/balls/smatmul/stack/src/Lowering.cpp create mode 100644 examples/components/balls/smatmul/verify/README.md rename examples/{ => components}/balls/smatmul/verify/casegen/Cargo.lock (100%) rename examples/{ => components}/balls/smatmul/verify/casegen/Cargo.toml (100%) create mode 100644 examples/components/balls/smatmul/verify/casegen/src/casegen.rs rename examples/{ => components}/balls/smatmul/verify/casegen/src/lib.rs (100%) create mode 100644 examples/components/balls/smatmul/verify/casegen/src/model.rs rename examples/{ => components}/balls/smatmul/verify/filelists/smatmul_ball.f (100%) rename examples/{ => components}/balls/smatmul/verify/parameters.toml (100%) rename examples/{ => components}/balls/smatmul/verify/src/common/smatmul_defs.svh (89%) rename examples/{ => components}/balls/smatmul/verify/src/common/smatmul_items.svh (95%) rename examples/{ => components}/balls/smatmul/verify/src/cov/smatmul_cov.svh (89%) rename examples/{ => components}/balls/smatmul/verify/src/env/smatmul_env.svh (100%) rename examples/{ => components}/balls/smatmul/verify/src/env/smatmul_scoreboard.svh (99%) rename examples/{ => components}/balls/smatmul/verify/src/pkg/smatmul_pkg.sv (100%) rename examples/{ => components}/balls/smatmul/verify/src/seq/smatmul_sequences.svh (100%) create mode 100644 examples/components/balls/smatmul/verify/src/tb_top.sv rename examples/{ => components}/balls/smatmul/verify/src/tests/smatmul_test.svh (100%) create mode 100644 examples/components/balls/smatmul/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/smatmul/workloads/ctests/smatmul_16x16_test.c (100%) rename examples/{ => components}/balls/smatmul/workloads/ctests/smatmul_16x16x32_test.c (100%) rename examples/{ => components}/balls/smatmul/workloads/ctests/smatmul_1x16x64_test.c (100%) rename examples/{ => components}/balls/smatmul/workloads/ctests/smatmul_8x16_test.c (100%) rename examples/{ => components}/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c (94%) rename examples/{ => components}/balls/smatmul/workloads/ctests/smatmul_f32_test.c (100%) rename examples/{ => components}/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c (100%) rename examples/{ => components}/balls/smatmul/workloads/isa/smatmul.h (100%) rename examples/{ => components}/balls/smatmul/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{ => components}/balls/smatmul/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir rename examples/{ => components}/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp (100%) create mode 100644 examples/components/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt create mode 100644 examples/components/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in rename examples/{ => components}/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp (100%) create mode 100644 examples/components/balls/toint8/arch/ToInt8Ball.scala create mode 100644 examples/components/balls/toint8/configs/default.toml create mode 100644 examples/components/balls/toint8/configs/toint8_params.proto create mode 100644 examples/components/balls/toint8/emu/src/dispatch.rs create mode 100644 examples/components/balls/toint8/emu/src/lib.rs rename examples/{ => components}/balls/toint8/emu/src/model.rs (100%) create mode 100644 examples/components/balls/toint8/emu/src/quant.rs create mode 100644 examples/components/balls/toint8/stack/src/Lowering.cpp create mode 100644 examples/components/balls/toint8/verify/README.md rename examples/{ => components}/balls/toint8/verify/casegen/Cargo.lock (100%) rename examples/{ => components}/balls/toint8/verify/casegen/Cargo.toml (100%) create mode 100644 examples/components/balls/toint8/verify/casegen/src/casegen.rs create mode 100644 examples/components/balls/toint8/verify/casegen/src/lib.rs rename examples/{ => components}/balls/toint8/verify/filelists/toint8_ball.f (100%) rename examples/{ => components}/balls/toint8/verify/src/common/toint8_defs.svh (92%) rename examples/{ => components}/balls/toint8/verify/src/common/toint8_items.svh (94%) rename examples/{ => components}/balls/toint8/verify/src/cov/toint8_cov.svh (84%) rename examples/{ => components}/balls/toint8/verify/src/env/toint8_env.svh (100%) rename examples/{ => components}/balls/toint8/verify/src/env/toint8_scoreboard.svh (98%) rename examples/{ => components}/balls/toint8/verify/src/pkg/toint8_pkg.sv (100%) rename examples/{ => components}/balls/toint8/verify/src/seq/toint8_sequences.svh (100%) create mode 100644 examples/components/balls/toint8/verify/src/tb_top.sv rename examples/{ => components}/balls/toint8/verify/src/tests/toint8_test.svh (100%) create mode 100644 examples/components/balls/toint8/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/toint8/workloads/ctests/quant_f32_to_i8_test.c (100%) rename examples/{ => components}/balls/toint8/workloads/ctests/quant_i32_to_i8_test.c (100%) rename examples/{ => components}/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c (95%) rename examples/{ => components}/balls/toint8/workloads/ctests/toint8_i8_rows_test.c (95%) rename examples/{ => components}/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c (95%) rename examples/{ => components}/balls/toint8/workloads/ctests/toint8_i8_signed_test.c (95%) rename examples/{ => components}/balls/toint8/workloads/isa/quant.h (100%) rename examples/{ => components}/balls/toint8/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{ => components}/balls/toint8/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/components/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir rename examples/{ => components}/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8_main.cpp (100%) create mode 100644 examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir rename examples/{ => components}/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp (80%) create mode 100644 examples/components/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt create mode 100644 examples/components/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in rename examples/{ => components}/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp (100%) create mode 100644 examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in rename examples/{ => components}/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp (100%) rename examples/{balls/trace/arch/src/main/scala => components/balls/trace/arch}/README.md (100%) create mode 100644 examples/components/balls/trace/arch/Trace.scala create mode 100644 examples/components/balls/trace/arch/TraceBall.scala rename examples/{balls/trace/arch/src/main/scala => components/balls/trace/arch}/TraceDPI.scala (96%) create mode 100644 examples/components/balls/trace/configs/default.toml create mode 100644 examples/components/balls/trace/configs/trace_params.proto create mode 100644 examples/components/balls/trace/emu/src/04_bdb_counter.rs create mode 100644 examples/components/balls/trace/emu/src/dispatch.rs create mode 100644 examples/components/balls/trace/emu/src/lib.rs create mode 100644 examples/components/balls/trace/stack/src/Lowering.cpp create mode 100644 examples/components/balls/trace/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/trace/workloads/ctests/bdb_counter_multi_id_test.c (100%) rename examples/{ => components}/balls/trace/workloads/ctests/bdb_counter_nested_pair_test.c (100%) rename examples/{ => components}/balls/trace/workloads/ctests/bdb_counter_read_active_test.c (100%) rename examples/{ => components}/balls/trace/workloads/ctests/bdb_counter_start_stop_test.c (100%) rename examples/{ => components}/balls/trace/workloads/isa/bdb_counter.h (100%) rename examples/{ => components}/balls/trace/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{ => components}/balls/trace/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/components/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir rename examples/{ => components}/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop_main.cpp (100%) create mode 100644 examples/components/balls/transpose/arch/README.md create mode 100644 examples/components/balls/transpose/arch/Transpose.scala create mode 100644 examples/components/balls/transpose/arch/TransposeBall.scala create mode 100644 examples/components/balls/transpose/configs/TransposeBallParam.scala create mode 100644 examples/components/balls/transpose/configs/default.toml create mode 100644 examples/components/balls/transpose/configs/transpose_params.proto create mode 100644 examples/components/balls/transpose/emu/src/dispatch.rs create mode 100644 examples/components/balls/transpose/emu/src/lib.rs rename examples/{ => components}/balls/transpose/emu/src/transpose.rs (97%) create mode 100644 examples/components/balls/transpose/stack/src/Lowering.cpp create mode 100644 examples/components/balls/transpose/verify/README.md rename examples/{ => components}/balls/transpose/verify/casegen/Cargo.lock (100%) rename examples/{ => components}/balls/transpose/verify/casegen/Cargo.toml (100%) create mode 100644 examples/components/balls/transpose/verify/casegen/src/casegen.rs create mode 100644 examples/components/balls/transpose/verify/casegen/src/lib.rs rename examples/{ => components}/balls/transpose/verify/casegen/src/model.rs (100%) rename examples/{ => components}/balls/transpose/verify/filelists/transpose_ball.f (100%) rename examples/{ => components}/balls/transpose/verify/src/common/transpose_defs.svh (90%) rename examples/{ => components}/balls/transpose/verify/src/common/transpose_items.svh (97%) rename examples/{ => components}/balls/transpose/verify/src/cov/transpose_cov.svh (96%) rename examples/{ => components}/balls/transpose/verify/src/env/transpose_env.svh (100%) rename examples/{ => components}/balls/transpose/verify/src/env/transpose_scoreboard.svh (98%) rename examples/{ => components}/balls/transpose/verify/src/pkg/transpose_pkg.sv (100%) rename examples/{ => components}/balls/transpose/verify/src/seq/transpose_sequences.svh (100%) create mode 100644 examples/components/balls/transpose/verify/src/tb_top.sv rename examples/{ => components}/balls/transpose/verify/src/tests/transpose_test.svh (100%) create mode 100644 examples/components/balls/transpose/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c (97%) rename examples/{ => components}/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c (98%) rename examples/{ => components}/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c (98%) rename examples/{ => components}/balls/transpose/workloads/isa/transpose.h (100%) rename examples/{ => components}/balls/transpose/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{ => components}/balls/transpose/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/components/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir rename examples/{ => components}/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8_main.cpp (100%) rename examples/{ => components}/balls/transpose/workloads/mlir_tests/bank/CMakeLists.txt (100%) create mode 100644 examples/components/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in rename examples/{ => components}/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp (100%) rename examples/{balls/vector/arch/src/main/scala => components/balls/vector/arch}/README.md (100%) create mode 100644 examples/components/balls/vector/arch/VecBall.scala create mode 100644 examples/components/balls/vector/arch/VecCtrlUnit.scala rename examples/{balls/vector/arch/src/main/scala => components/balls/vector/arch}/VecEXUnit.scala (76%) create mode 100644 examples/components/balls/vector/arch/VecLoadUnit.scala create mode 100644 examples/components/balls/vector/arch/VecMat16.scala rename examples/{balls/vector/arch/src/main/scala => components/balls/vector/arch}/VecMat16AccLoad.scala (75%) create mode 100644 examples/components/balls/vector/arch/VecMat16Store.scala rename examples/{balls/vector/arch/src/main/scala => components/balls/vector/arch}/VecMat16Tie.scala (77%) create mode 100644 examples/components/balls/vector/arch/VecStoreUnit.scala create mode 100644 examples/components/balls/vector/arch/VecUnit.scala rename examples/{balls/vector/arch/src/main/scala => components/balls/vector/arch}/bond/BondWrapper.scala (100%) rename examples/{balls/vector/arch/src/main/scala => components/balls/vector/arch}/bond/README.md (100%) create mode 100644 examples/components/balls/vector/arch/bond/vvv.scala rename examples/{balls/vector/arch/src/main/scala => components/balls/vector/arch}/op/README.md (100%) rename examples/{balls/vector/arch/src/main/scala => components/balls/vector/arch}/op/cascade.scala (79%) rename examples/{balls/vector/arch/src/main/scala => components/balls/vector/arch}/op/mul.scala (78%) create mode 100644 examples/components/balls/vector/arch/thread/BaseThread.scala create mode 100644 examples/components/balls/vector/arch/thread/CasThread.scala create mode 100644 examples/components/balls/vector/arch/thread/MulThread.scala rename examples/{balls/vector/arch/src/main/scala => components/balls/vector/arch}/thread/README.md (100%) rename examples/{balls/vector/arch/src/main/scala => components/balls/vector/arch}/warp/MeshWarp.scala (80%) rename examples/{balls/vector/arch/src/main/scala => components/balls/vector/arch}/warp/README.md (100%) create mode 100644 examples/components/balls/vector/configs/VectorBallParam.scala create mode 100644 examples/components/balls/vector/configs/default.toml create mode 100644 examples/components/balls/vector/configs/vector_params.proto rename examples/{ => components}/balls/vector/emu/src/64_vecmat16.rs (95%) create mode 100644 examples/components/balls/vector/emu/src/dispatch.rs create mode 100644 examples/components/balls/vector/emu/src/lib.rs create mode 100644 examples/components/balls/vector/workloads/ctests/CMakeLists.txt rename examples/{ => components}/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c (100%) rename examples/{ => components}/balls/vector/workloads/isa/vecmat16.h (100%) rename examples/{ => components}/balls/vector/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{ => components}/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt (100%) rename examples/{ => components}/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16.mlir (100%) rename examples/{ => components}/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16_main.cpp (100%) rename examples/{ => components}/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt (100%) rename examples/{ => components}/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth.mlir.in (100%) rename examples/{ => components}/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth_main.cpp (100%) create mode 100644 examples/components/cores/boom/LICENSE create mode 100644 examples/components/cores/boom/arch/Boom.scala create mode 100644 examples/components/cores/boom/arch/Native.scala create mode 100644 examples/components/cores/boom/arch/build.sc create mode 100644 examples/components/cores/boom/configs/Params.scala create mode 100644 examples/components/cores/boom/configs/boom_params.proto create mode 100644 examples/components/cores/boom/stack/CMakeLists.txt create mode 100644 examples/components/cores/boom/workloads/ctests/CMakeLists.txt create mode 100644 examples/components/cores/boom/workloads/ctests/memory_matrix.c create mode 100644 examples/components/cores/reduce/arch/ReduceCore.scala create mode 100644 examples/components/cores/reduce/arch/acc.scala create mode 100644 examples/components/cores/reduce/arch/build.sc create mode 100644 examples/components/cores/reduce/arch/loom.scala create mode 100644 examples/components/cores/reduce/emu/Cargo.lock create mode 100644 examples/components/cores/reduce/emu/Cargo.toml create mode 100644 examples/components/cores/reduce/emu/src/lib.rs create mode 100644 examples/components/cores/reduce/stack/CMakeLists.txt create mode 100644 examples/components/cores/rocket/LICENSE.Berkeley create mode 100644 examples/components/cores/rocket/LICENSE.SiFive create mode 100644 examples/components/cores/rocket/arch/Execution.scala create mode 100644 examples/components/cores/rocket/arch/Lsu.scala create mode 100644 examples/components/cores/rocket/arch/Rocket.scala create mode 100644 examples/components/cores/rocket/arch/build.sc create mode 100644 examples/components/cores/rocket/arch/pipeline/Config.scala create mode 100644 examples/components/cores/rocket/arch/pipeline/Pipeline.scala rename {arch/src/main/scala/framework/system/core/rocket => examples/components/cores/rocket/arch/pipeline}/Utility.scala (100%) create mode 100644 examples/components/cores/rocket/configs/Params.scala create mode 100644 examples/components/cores/rocket/configs/rocket_params.proto create mode 100644 examples/components/cores/rocket/emu/Cargo.lock create mode 100644 examples/components/cores/rocket/emu/Cargo.toml create mode 100644 examples/components/cores/rocket/emu/src/lib.rs create mode 100644 examples/components/cores/rocket/stack/CMakeLists.txt create mode 100644 examples/components/cores/xiangshan/LICENSE create mode 100644 examples/components/cores/xiangshan/arch/Chi.scala create mode 100644 examples/components/cores/xiangshan/arch/Native.scala create mode 100644 examples/components/cores/xiangshan/arch/XiangShan.scala create mode 100644 examples/components/cores/xiangshan/arch/build.sc create mode 100644 examples/components/cores/xiangshan/configs/Params.scala create mode 100644 examples/components/cores/xiangshan/configs/xiangshan_params.proto create mode 100644 examples/components/cores/xiangshan/stack/CMakeLists.txt create mode 100644 examples/components/cores/xiangshan/workloads/ctests/CMakeLists.txt create mode 100644 examples/components/cores/xiangshan/workloads/ctests/memory_matrix.c create mode 100644 examples/components/threads/attention/arch/AttentionThread.scala create mode 100644 examples/components/threads/attention/arch/acc.scala create mode 100644 examples/components/threads/attention/configs/balldomains/default.toml create mode 100644 examples/components/threads/attention/configs/cpu/default.toml create mode 100644 examples/components/threads/attention/configs/default.toml create mode 100644 examples/components/threads/attention/configs/frontends/default.toml rename examples/{cores/attention/configs/rvv => components/threads/attention/configs/gpdomain}/default.toml (100%) create mode 100644 examples/components/threads/attention/configs/memdomains/default.toml create mode 100644 examples/components/threads/attention/stack/CMakeLists.txt create mode 100644 examples/components/threads/build.sc create mode 100644 examples/components/threads/ffn/arch/FfnThread.scala create mode 100644 examples/components/threads/ffn/arch/acc.scala create mode 100644 examples/components/threads/ffn/configs/balldomains/default.toml create mode 100644 examples/components/threads/ffn/configs/cpu/default.toml create mode 100644 examples/components/threads/ffn/configs/default.toml create mode 100644 examples/components/threads/ffn/configs/frontends/default.toml rename examples/{cores/ffn/configs/rvv => components/threads/ffn/configs/gpdomain}/default.toml (100%) create mode 100644 examples/components/threads/ffn/configs/memdomains/default.toml create mode 100644 examples/components/threads/ffn/stack/CMakeLists.txt create mode 100644 examples/components/threads/matmul/arch/MatMulThread.scala create mode 100644 examples/components/threads/matmul/arch/acc.scala create mode 100644 examples/components/threads/matmul/configs/ant.toml create mode 100644 examples/components/threads/matmul/configs/balldomains/default.toml create mode 100644 examples/components/threads/matmul/configs/cpu/default.toml create mode 100644 examples/components/threads/matmul/configs/default.toml create mode 100644 examples/components/threads/matmul/configs/frontends/default.toml rename examples/{cores/goban/configs/rvv => components/threads/matmul/configs/gpdomain}/default.toml (100%) create mode 100644 examples/components/threads/matmul/configs/memdomains/default.toml create mode 100644 examples/components/threads/matmul/stack/CMakeLists.txt create mode 100644 examples/components/threads/matmul/stack/matmul.h create mode 100644 examples/components/threads/matmul/stack/prepare.c create mode 100644 examples/components/threads/matmul/stack/program.c create mode 100644 examples/components/threads/matmul/workloads/ctests/CMakeLists.txt create mode 100644 examples/components/threads/matmul/workloads/ctests/rvv/CMakeLists.txt rename examples/{cores/goban => components/threads/matmul}/workloads/ctests/rvv/matmul.cpp (100%) rename examples/{cores/goban => components/threads/matmul}/workloads/ctests/rvv/mxfp8_quant.cpp (95%) create mode 100644 examples/components/threads/matmul/workloads/ctests/rvv/norm.cpp rename examples/{cores/goban => components/threads/matmul}/workloads/ctests/rvv/silu.cpp (100%) rename examples/{cores/goban => components/threads/matmul}/workloads/ctests/rvv/snake.cpp (100%) rename examples/{cores/goban => components/threads/matmul}/workloads/ctests/rvv/swiglu.cpp (100%) create mode 100644 examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt create mode 100644 examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c create mode 100644 examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp delete mode 100644 examples/cores/README.md delete mode 100644 examples/cores/ant/arch/src/main/scala/AntCore.scala delete mode 100644 examples/cores/ant/configs/cpu/ant.toml delete mode 100644 examples/cores/ant/configs/cpu/default.toml delete mode 100644 examples/cores/ant/configs/default.toml delete mode 100644 examples/cores/attention/arch/src/main/scala/AttentionCore.scala delete mode 100644 examples/cores/attention/compiler/CMakeLists.txt delete mode 100644 examples/cores/attention/compiler/src/CMakeLists.txt delete mode 100644 examples/cores/attention/compiler/src/Conversion/CMakeLists.txt delete mode 100644 examples/cores/attention/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt delete mode 100644 examples/cores/attention/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp delete mode 100644 examples/cores/attention/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp delete mode 100644 examples/cores/attention/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt delete mode 100644 examples/cores/attention/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp delete mode 100644 examples/cores/attention/compiler/src/Dialect/Buckyball/AttentionBuckyball.td delete mode 100644 examples/cores/attention/compiler/src/Dialect/Buckyball/CMakeLists.txt delete mode 100644 examples/cores/attention/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt delete mode 100644 examples/cores/attention/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt delete mode 100644 examples/cores/attention/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/cores/attention/compiler/src/Dialect/CMakeLists.txt delete mode 100644 examples/cores/attention/compiler/src/Target/CMakeLists.txt delete mode 100644 examples/cores/attention/compiler/src/Target/LLVMIR/CMakeLists.txt delete mode 100644 examples/cores/attention/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt delete mode 100644 examples/cores/attention/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt delete mode 100644 examples/cores/attention/configs/balldomains/default.toml delete mode 100644 examples/cores/attention/configs/cpu/default.toml delete mode 100644 examples/cores/attention/configs/default.toml delete mode 100644 examples/cores/attention/configs/frontends/default.toml delete mode 100644 examples/cores/attention/configs/memdomains/default.toml delete mode 100644 examples/cores/boom/arch/src/main/scala/configs/BoomCpuParam.scala delete mode 100644 examples/cores/boom/configs/balldomains/default.toml delete mode 100644 examples/cores/boom/configs/cpu/boom/default.toml delete mode 100644 examples/cores/boom/configs/cpu/default.toml delete mode 100644 examples/cores/boom/configs/default.toml delete mode 100644 examples/cores/boom/configs/memdomains/default.toml delete mode 100644 examples/cores/ffn/arch/src/main/scala/FfnCore.scala delete mode 100644 examples/cores/ffn/compiler/CMakeLists.txt delete mode 100644 examples/cores/ffn/compiler/src/CMakeLists.txt delete mode 100644 examples/cores/ffn/compiler/src/Conversion/CMakeLists.txt delete mode 100644 examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt delete mode 100644 examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp delete mode 100644 examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp delete mode 100644 examples/cores/ffn/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt delete mode 100644 examples/cores/ffn/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp delete mode 100644 examples/cores/ffn/compiler/src/Dialect/Buckyball/CMakeLists.txt delete mode 100644 examples/cores/ffn/compiler/src/Dialect/Buckyball/FFNBuckyball.td delete mode 100644 examples/cores/ffn/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt delete mode 100644 examples/cores/ffn/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt delete mode 100644 examples/cores/ffn/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/cores/ffn/compiler/src/Dialect/CMakeLists.txt delete mode 100644 examples/cores/ffn/compiler/src/Target/CMakeLists.txt delete mode 100644 examples/cores/ffn/compiler/src/Target/LLVMIR/CMakeLists.txt delete mode 100644 examples/cores/ffn/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt delete mode 100644 examples/cores/ffn/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt delete mode 100644 examples/cores/ffn/configs/balldomains/default.toml delete mode 100644 examples/cores/ffn/configs/cpu/default.toml delete mode 100644 examples/cores/ffn/configs/default.toml delete mode 100644 examples/cores/ffn/configs/frontends/default.toml delete mode 100644 examples/cores/ffn/configs/memdomains/default.toml delete mode 100644 examples/cores/goban/arch/src/main/scala/AntGobanCore.scala delete mode 100644 examples/cores/goban/arch/src/main/scala/GobanCore.scala delete mode 100644 examples/cores/goban/compiler/CMakeLists.txt delete mode 100644 examples/cores/goban/compiler/src/CMakeLists.txt delete mode 100644 examples/cores/goban/compiler/src/Conversion/CMakeLists.txt delete mode 100644 examples/cores/goban/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt delete mode 100644 examples/cores/goban/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp delete mode 100644 examples/cores/goban/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp delete mode 100644 examples/cores/goban/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt delete mode 100644 examples/cores/goban/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp delete mode 100644 examples/cores/goban/compiler/src/Dialect/Buckyball/CMakeLists.txt delete mode 100644 examples/cores/goban/compiler/src/Dialect/Buckyball/GobanBuckyball.td delete mode 100644 examples/cores/goban/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt delete mode 100644 examples/cores/goban/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt delete mode 100644 examples/cores/goban/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/cores/goban/compiler/src/Dialect/CMakeLists.txt delete mode 100644 examples/cores/goban/compiler/src/Target/CMakeLists.txt delete mode 100644 examples/cores/goban/compiler/src/Target/LLVMIR/CMakeLists.txt delete mode 100644 examples/cores/goban/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt delete mode 100644 examples/cores/goban/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt delete mode 100644 examples/cores/goban/configs/ant.toml delete mode 100644 examples/cores/goban/configs/balldomains/default.toml delete mode 100644 examples/cores/goban/configs/cpu/ant.toml delete mode 100644 examples/cores/goban/configs/cpu/default.toml delete mode 100644 examples/cores/goban/configs/default.toml delete mode 100644 examples/cores/goban/configs/frontends/default.toml delete mode 100644 examples/cores/goban/configs/memdomains/default.toml delete mode 100644 examples/cores/goban/workloads/ctests/CMakeLists.txt delete mode 100644 examples/cores/goban/workloads/ctests/rvv/CMakeLists.txt delete mode 100644 examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt delete mode 100644 examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c delete mode 100644 examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp delete mode 100644 examples/cores/pebble/arch/src/main/scala/PebbleCore.scala delete mode 100644 examples/cores/pebble/compiler/CMakeLists.txt delete mode 100644 examples/cores/pebble/compiler/src/CMakeLists.txt delete mode 100644 examples/cores/pebble/compiler/src/Conversion/CMakeLists.txt delete mode 100644 examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt delete mode 100644 examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/CoreBankSSALowering.cpp delete mode 100644 examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp delete mode 100644 examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp delete mode 100644 examples/cores/pebble/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt delete mode 100644 examples/cores/pebble/compiler/src/Conversion/LowerTileToBuckyball/CoreTileLowering.cpp delete mode 100644 examples/cores/pebble/compiler/src/Dialect/Buckyball/CMakeLists.txt delete mode 100644 examples/cores/pebble/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt delete mode 100644 examples/cores/pebble/compiler/src/Dialect/Buckyball/PebbleBuckyball.td delete mode 100644 examples/cores/pebble/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt delete mode 100644 examples/cores/pebble/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/cores/pebble/compiler/src/Dialect/CMakeLists.txt delete mode 100644 examples/cores/pebble/compiler/src/Target/CMakeLists.txt delete mode 100644 examples/cores/pebble/compiler/src/Target/LLVMIR/CMakeLists.txt delete mode 100644 examples/cores/pebble/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt delete mode 100644 examples/cores/pebble/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt delete mode 100644 examples/cores/pebble/configs/balldomains/default.toml delete mode 100644 examples/cores/pebble/configs/cpu/default.toml delete mode 100644 examples/cores/pebble/configs/cpu/rocket/default.toml delete mode 100644 examples/cores/pebble/configs/default.toml delete mode 100644 examples/cores/pebble/configs/frontends/default.toml delete mode 100644 examples/cores/pebble/configs/memdomains/default.toml delete mode 100644 examples/cores/pebble/workloads/ctests/CMakeLists.txt delete mode 100644 examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/CMakeLists.txt delete mode 100644 examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp delete mode 100644 examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/pebble.c delete mode 100644 examples/cores/reduce/arch/src/main/scala/ReduceCore.scala delete mode 100644 examples/cores/reduce/compiler/CMakeLists.txt delete mode 100644 examples/cores/reduce/configs/balldomains/default.toml delete mode 100644 examples/cores/reduce/configs/cpu/default.toml delete mode 100644 examples/cores/reduce/configs/default.toml delete mode 100644 examples/cores/reduce/configs/frontends/default.toml delete mode 100644 examples/cores/reduce/configs/memdomains/default.toml delete mode 100644 examples/cores/reduce/configs/rvv/default.toml delete mode 100644 examples/cores/reduce/workloads/ctests/CMakeLists.txt delete mode 100644 examples/cores/reduce/workloads/ctests/tlink_lease.c delete mode 100644 examples/cores/rocket/arch/src/main/scala/RocketCore.scala delete mode 100644 examples/cores/rocket/arch/src/main/scala/cpu/AdmissionBridge.scala delete mode 100644 examples/cores/rocket/arch/src/main/scala/cpu/Cache.scala delete mode 100644 examples/cores/rocket/arch/src/main/scala/cpu/Lsu.scala delete mode 100644 examples/cores/rocket/arch/src/main/scala/cpu/PermissionCheck.scala delete mode 100644 examples/cores/rocket/arch/src/main/scala/cpu/Preparation.scala delete mode 100644 examples/cores/rocket/arch/src/main/scala/cpu/Rocket.scala delete mode 100644 examples/cores/rocket/compiler/CMakeLists.txt delete mode 100644 examples/cores/rocket/configs/balldomains/default.toml delete mode 100644 examples/cores/rocket/configs/cpu/default.toml delete mode 100644 examples/cores/rocket/configs/cpu/rocket/default.toml delete mode 100644 examples/cores/rocket/configs/default.toml delete mode 100644 examples/cores/rocket/configs/memdomains/default.toml delete mode 100644 examples/cores/toy/arch/src/main/scala/ToyCore.scala delete mode 100644 examples/cores/toy/compiler/CMakeLists.txt delete mode 100644 examples/cores/toy/compiler/include/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h delete mode 100644 examples/cores/toy/compiler/src/Backend/include/IntrinsicsRISCVBuckyball.td delete mode 100644 examples/cores/toy/compiler/src/Backend/include/IntrinsicsRISCVBuckyballBase.td delete mode 100644 examples/cores/toy/compiler/src/CMakeLists.txt delete mode 100644 examples/cores/toy/compiler/src/Conversion/CMakeLists.txt delete mode 100644 examples/cores/toy/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt delete mode 100644 examples/cores/toy/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp delete mode 100644 examples/cores/toy/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp delete mode 100644 examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyAssignPhysicalBankPatterns.cpp delete mode 100644 examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballToBankSSAPatterns.cpp delete mode 100644 examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballToLLVMPatterns.cpp delete mode 100644 examples/cores/toy/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt delete mode 100644 examples/cores/toy/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp delete mode 100644 examples/cores/toy/compiler/src/Dialect/Buckyball/CMakeLists.txt delete mode 100644 examples/cores/toy/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt delete mode 100644 examples/cores/toy/compiler/src/Dialect/Buckyball/ToyBuckyball.td delete mode 100644 examples/cores/toy/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt delete mode 100644 examples/cores/toy/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 examples/cores/toy/compiler/src/Dialect/CMakeLists.txt delete mode 100644 examples/cores/toy/compiler/src/Target/CMakeLists.txt delete mode 100644 examples/cores/toy/compiler/src/Target/LLVMIR/CMakeLists.txt delete mode 100644 examples/cores/toy/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt delete mode 100644 examples/cores/toy/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt delete mode 100644 examples/cores/toy/configs/balldomains/balls/smatmul/lane16.toml delete mode 100644 examples/cores/toy/configs/balldomains/default.toml delete mode 100644 examples/cores/toy/configs/cpu/default.toml delete mode 100644 examples/cores/toy/configs/default.toml delete mode 100644 examples/cores/toy/configs/demo.toml delete mode 100644 examples/cores/toy/configs/frontends/default.toml delete mode 100644 examples/cores/toy/configs/frontends/demo.toml delete mode 100644 examples/cores/toy/configs/memdomains/default.toml delete mode 100644 examples/cores/toy/workloads/ctests/CMakeLists.txt delete mode 100644 examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/CMakeLists.txt delete mode 100644 examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp create mode 100644 examples/designs/goban/arch/build.sc create mode 100644 examples/designs/goban/arch/device/DeviceTop.scala create mode 100644 examples/designs/goban/arch/device/core/GobanCore.scala create mode 100644 examples/designs/goban/arch/device/core/loom.scala create mode 100644 examples/designs/goban/configs/design.toml create mode 100644 examples/designs/goban/configs/device/clint.toml create mode 100644 examples/designs/goban/configs/device/core/c2c/goban.toml create mode 100644 examples/designs/goban/configs/device/core/controller/goban.toml create mode 100644 examples/designs/goban/configs/device/core/default.toml create mode 100644 examples/designs/goban/configs/device/core/interconnect/goban.toml create mode 100644 examples/designs/goban/configs/device/core/memory/access/goban.toml create mode 100644 examples/designs/goban/configs/device/core/memory/dcache/goban.toml create mode 100644 examples/designs/goban/configs/device/core/memory/dcache/shared.toml create mode 100644 examples/designs/goban/configs/device/core/memory/dma-cache/shared.toml create mode 100644 examples/designs/goban/configs/device/core/memory/goban.toml create mode 100644 examples/designs/goban/configs/device/core/memory/home/goban.toml create mode 100644 examples/designs/goban/configs/device/core/memory/icache/shared.toml create mode 100644 examples/designs/goban/configs/device/core/rocket.toml create mode 100644 examples/designs/goban/configs/device/core/tbus/default.toml create mode 100644 examples/designs/goban/configs/device/ddr.toml create mode 100644 examples/designs/goban/configs/device/default.toml create mode 100644 examples/designs/goban/configs/device/plic.toml create mode 100644 examples/designs/goban/configs/sims/default.toml create mode 100644 examples/designs/goban/emu/Cargo.lock create mode 100644 examples/designs/goban/emu/Cargo.toml create mode 100644 examples/designs/goban/emu/src/device/core/mod.rs create mode 100644 examples/designs/goban/emu/src/device/mod.rs create mode 100644 examples/designs/goban/emu/src/lib.rs create mode 100644 examples/designs/goban/sims/device/arch/DeviceConfig.scala create mode 100644 examples/designs/goban/sims/device/arch/build.sc create mode 100644 examples/designs/goban/sims/device/emu/Cargo.lock create mode 100644 examples/designs/goban/sims/device/emu/Cargo.toml create mode 100644 examples/designs/goban/sims/device/emu/build.rs create mode 100644 examples/designs/goban/sims/device/emu/src/lib.rs create mode 100644 examples/designs/goban/sims/device/regression/bemu/workloads-bare.toml create mode 100644 examples/designs/goban/sims/device/regression/bemu/workloads-linux.toml create mode 100644 examples/designs/goban/sims/device/regression/p2e/workloads-linux.toml create mode 100644 examples/designs/goban/sims/device/regression/verilator/workloads-bare.toml create mode 100644 examples/designs/goban/sims/device/stack/CMakeLists.txt create mode 100644 examples/designs/goban/sims/device/stack/core/goban/CMakeLists.txt create mode 100644 examples/designs/goban/sims/device/stack/runtime/CMakeLists.txt create mode 100644 examples/designs/goban/sims/device/workloads/CMakeLists.txt create mode 100644 examples/designs/goban/sims/device/workloads/kernel/CMakeLists.txt create mode 100644 examples/designs/goban/sims/ip/uvm/registry.toml create mode 100644 examples/designs/goban/stack/engine/CMakeLists.txt create mode 100644 examples/designs/goban/stack/midend/CMakeLists.txt create mode 100644 examples/designs/goban/workloads/CMakeLists.txt create mode 100644 examples/designs/goban/workloads/configs/tests.toml create mode 100644 examples/designs/goban/workloads/device/CMakeLists.txt create mode 100644 examples/designs/goban/workloads/device/core/CMakeLists.txt create mode 100644 examples/designs/goban/workloads/device/core/engine/CMakeLists.txt create mode 100644 examples/designs/goban/workloads/device/core/engine/matmul_threads.c rename examples/{chips/goban/workloads/ctests => designs/goban/workloads/device/core/engine}/mvover_rows.c (82%) create mode 100644 examples/designs/multi-rocket/arch/build.sc create mode 100644 examples/designs/multi-rocket/arch/device/DeviceTop.scala create mode 100644 examples/designs/multi-rocket/configs/design.toml create mode 100644 examples/designs/multi-rocket/configs/device/clint.toml create mode 100644 examples/designs/multi-rocket/configs/device/core/c2c/core.toml create mode 100644 examples/designs/multi-rocket/configs/device/core/default.toml create mode 100644 examples/designs/multi-rocket/configs/device/core/interconnect/core.toml create mode 100644 examples/designs/multi-rocket/configs/device/core/memory/access/core.toml create mode 100644 examples/designs/multi-rocket/configs/device/core/memory/core.toml create mode 100644 examples/designs/multi-rocket/configs/device/core/memory/dcache/core.toml create mode 100644 examples/designs/multi-rocket/configs/device/core/memory/dcache/shared.toml create mode 100644 examples/designs/multi-rocket/configs/device/core/memory/home/core.toml create mode 100644 examples/designs/multi-rocket/configs/device/core/memory/icache/shared.toml create mode 100644 examples/designs/multi-rocket/configs/device/core/rocket.toml create mode 100644 examples/designs/multi-rocket/configs/device/ddr.toml create mode 100644 examples/designs/multi-rocket/configs/device/default.toml create mode 100644 examples/designs/multi-rocket/configs/device/plic.toml create mode 100644 examples/designs/multi-rocket/configs/sims/default.toml create mode 100644 examples/designs/multi-rocket/sims/device/arch/DeviceConfig.scala create mode 100644 examples/designs/multi-rocket/sims/device/arch/build.sc create mode 100644 examples/designs/multi-rocket/sims/device/emu/Cargo.lock create mode 100644 examples/designs/multi-rocket/sims/device/emu/Cargo.toml create mode 100644 examples/designs/multi-rocket/sims/device/emu/build.rs create mode 100644 examples/designs/multi-rocket/sims/device/emu/src/lib.rs create mode 100644 examples/designs/multi-rocket/sims/device/regression/bemu/workloads-bare.toml rename examples/{chips/multi-rocket/regression/batch => designs/multi-rocket/sims/device/regression}/bemu/workloads-linux.toml (100%) create mode 100644 examples/designs/multi-rocket/sims/device/regression/p2e/workloads-bare.toml rename examples/{chips/multi-rocket/regression/batch => designs/multi-rocket/sims/device/regression}/p2e/workloads-linux.toml (100%) create mode 100644 examples/designs/multi-rocket/sims/device/regression/verilator/workloads-bare.toml create mode 100644 examples/designs/multi-rocket/sims/device/stack/CMakeLists.txt create mode 100644 examples/designs/multi-rocket/sims/device/stack/runtime/CMakeLists.txt create mode 100644 examples/designs/multi-rocket/sims/device/workloads/CMakeLists.txt create mode 100644 examples/designs/multi-rocket/sims/device/workloads/kernel/CMakeLists.txt create mode 100644 examples/designs/multi-rocket/sims/ip/uvm/registry.toml create mode 100644 examples/designs/multi-rocket/workloads/CMakeLists.txt create mode 100644 examples/designs/multi-rocket/workloads/configs/tests.toml create mode 100644 examples/designs/multi-rocket/workloads/device/CMakeLists.txt create mode 100644 examples/designs/multi-rocket/workloads/device/core/CMakeLists.txt create mode 100644 examples/designs/multi-rocket/workloads/device/core/ctests/CMakeLists.txt rename examples/{chips/multi-rocket/workloads => designs/multi-rocket/workloads/device/core/ctests}/cpu_arithmetic.c (100%) create mode 100644 examples/designs/multi-rocket/workloads/device/soc/CMakeLists.txt create mode 100644 examples/designs/multi-rocket/workloads/device/soc/coherent_exchange.c create mode 100644 examples/designs/pebble/arch/build.sc create mode 100644 examples/designs/pebble/arch/device/DeviceTop.scala create mode 100644 examples/designs/pebble/arch/device/core/PebbleCore.scala create mode 100644 examples/designs/pebble/arch/device/core/acc.scala create mode 100644 examples/designs/pebble/arch/device/core/loom.scala create mode 100644 examples/designs/pebble/configs/design.toml create mode 100644 examples/designs/pebble/configs/device/core/acc/ablation-bat0-blt0.toml create mode 100644 examples/designs/pebble/configs/device/core/acc/ablation-bat0-blt1.toml create mode 100644 examples/designs/pebble/configs/device/core/acc/ablation-bat1-blt0.toml rename examples/{cores/pebble/configs/balldomains => designs/pebble/configs/device/core/acc/balldomain}/balls/smatmul/8x16.toml (90%) create mode 100644 examples/designs/pebble/configs/device/core/acc/balldomain/default.toml create mode 100644 examples/designs/pebble/configs/device/core/acc/default.toml create mode 100644 examples/designs/pebble/configs/device/core/acc/frontend/bat-off-rob24.toml create mode 100644 examples/designs/pebble/configs/device/core/acc/frontend/rob24.toml create mode 100644 examples/designs/pebble/configs/device/core/acc/memdomain/blt-off.toml create mode 100644 examples/designs/pebble/configs/device/core/acc/rob24.toml create mode 100644 examples/designs/pebble/configs/device/core/c2c/pebble.toml create mode 100644 examples/designs/pebble/configs/device/core/default.toml create mode 100644 examples/designs/pebble/configs/device/core/interconnect/pebble.toml create mode 100644 examples/designs/pebble/configs/device/core/memory/access/pebble.toml create mode 100644 examples/designs/pebble/configs/device/core/memory/dcache/pebble.toml create mode 100644 examples/designs/pebble/configs/device/core/memory/home/pebble.toml create mode 100644 examples/designs/pebble/configs/device/core/memory/icache/pebble.toml create mode 100644 examples/designs/pebble/configs/device/core/memory/pebble.toml create mode 100644 examples/designs/pebble/configs/device/ddr/pebble.toml create mode 100644 examples/designs/pebble/configs/device/default.toml create mode 100644 examples/designs/pebble/configs/sims/default.toml create mode 100644 examples/designs/pebble/emu/Cargo.lock create mode 100644 examples/designs/pebble/emu/Cargo.toml create mode 100644 examples/designs/pebble/emu/src/device/core/mod.rs create mode 100644 examples/designs/pebble/emu/src/device/mod.rs create mode 100644 examples/designs/pebble/emu/src/lib.rs create mode 100644 examples/designs/pebble/sims/device/arch/DeviceConfig.scala create mode 100644 examples/designs/pebble/sims/device/arch/build.sc create mode 100644 examples/designs/pebble/sims/device/emu/Cargo.lock create mode 100644 examples/designs/pebble/sims/device/emu/Cargo.toml create mode 100644 examples/designs/pebble/sims/device/emu/build.rs create mode 100644 examples/designs/pebble/sims/device/emu/src/lib.rs create mode 100644 examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml create mode 100644 examples/designs/pebble/sims/device/regression/bemu/workloads-linux.toml create mode 100644 examples/designs/pebble/sims/device/regression/p2e/workloads-bare.toml rename examples/{chips/pebble/regression/batch => designs/pebble/sims/device/regression}/p2e/workloads-linux.toml (100%) create mode 100644 examples/designs/pebble/sims/device/regression/vcs/workloads-bare.toml create mode 100644 examples/designs/pebble/sims/device/regression/verilator/workloads-bare.toml create mode 100644 examples/designs/pebble/sims/device/stack/CMakeLists.txt create mode 100644 examples/designs/pebble/sims/device/stack/core/pebble/CMakeLists.txt create mode 100644 examples/designs/pebble/sims/device/stack/runtime/CMakeLists.txt create mode 100644 examples/designs/pebble/sims/device/tapeout/README.md rename examples/{chips/pebble => designs/pebble/sims/device}/tapeout/area/.gitignore (100%) rename examples/{chips/pebble => designs/pebble/sims/device}/tapeout/area/constraints.sdc (100%) create mode 100755 examples/designs/pebble/sims/device/tapeout/area/dc.tcl create mode 100644 examples/designs/pebble/sims/device/tapeout/config.toml rename examples/{chips/pebble => designs/pebble/sims/device}/tapeout/ip/smic180_sram_library.mdf.json (100%) rename examples/{chips/pebble => designs/pebble/sims/device}/tapeout/ip/tsmc28_sram_library.mdf.json (100%) rename examples/{chips/pebble => designs/pebble/sims/device}/tapeout/power/power.tcl (100%) rename examples/{chips/pebble => designs/pebble/sims/device}/tapeout/power/power_sim.sh (77%) create mode 100644 examples/designs/pebble/sims/device/workloads/CMakeLists.txt create mode 100644 examples/designs/pebble/sims/device/workloads/kernel/CMakeLists.txt create mode 100644 examples/designs/pebble/sims/ip/uvm/registry.toml create mode 100644 examples/designs/pebble/stack/engine/CMakeLists.txt create mode 100644 examples/designs/pebble/stack/midend/CMakeLists.txt create mode 100644 examples/designs/pebble/stack/midend/src/ConvertLinalgToBuckyball.cpp create mode 100644 examples/designs/pebble/stack/midend/src/FormCustomOps.cpp create mode 100644 examples/designs/pebble/stack/midend/src/FormSpatialOps.cpp create mode 100644 examples/designs/pebble/stack/midend/src/LowerLinearTiles.cpp create mode 100644 examples/designs/pebble/stack/midend/src/LowerQuantizedOps.cpp create mode 100644 examples/designs/pebble/stack/midend/src/LowerSpatialOps.cpp create mode 100644 examples/designs/pebble/stack/midend/src/Lowering.h create mode 100644 examples/designs/pebble/stack/midend/src/QuantizedChains.cpp create mode 100644 examples/designs/pebble/stack/midend/tests/form-custom-ops.mlir create mode 100644 examples/designs/pebble/stack/midend/tests/form-depthwise.mlir create mode 100644 examples/designs/pebble/stack/midend/tests/form-int32-bias.mlir create mode 100644 examples/designs/pebble/stack/midend/tests/form-linear-chain.mlir create mode 100644 examples/designs/pebble/stack/midend/tests/form-quantized-chain.mlir create mode 100644 examples/designs/pebble/stack/midend/tests/lower-bank-updates.mlir create mode 100644 examples/designs/pebble/stack/midend/tests/lower-custom-ops.mlir create mode 100644 examples/designs/pebble/stack/midend/tests/lower-frontend-types.mlir create mode 100644 examples/designs/pebble/stack/midend/tests/unknown-custom-op.mlir create mode 100644 examples/designs/pebble/workloads/CMakeLists.txt rename examples/{chips => designs}/pebble/workloads/common/pebble.c (100%) rename examples/{chips => designs}/pebble/workloads/common/pebble.h (100%) create mode 100644 examples/designs/pebble/workloads/configs/tests.toml create mode 100644 examples/designs/pebble/workloads/device/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/bank_capacity_rename_test.c create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/bank_dma_pingpong-main.cpp create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/pebble.c create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/bank_dma_reuse-main.cpp create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/pebble.c create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/bank_recycle_test.c create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/bank_rename_overlap-main.cpp create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/pebble.c create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/bank_transfer_recycle_test.c create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/dual_conv_add_int2fp_test.c rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/ctests/dual_smatmul_matadd_test.c (100%) rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/ctests/im2col_mvin_smatmul_hazard_test.c (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/pebble.c rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/ctests/maxpool_im2col_test.c (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/maxpool_partial_update-main.cpp create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/pebble.c create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_k5_pipeline_test.c rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/ctests/mega_conv_pipeline_test.c (100%) rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/ctests/mvin_bank78_stress_test.c (100%) rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/ctests/quant_smatmul_int2fp_test.c (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/pebble.c create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/read_bank_shared-main.cpp create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/pebble.c create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/read_bank_split-main.cpp rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/ctests/residual_add_lut_test.c (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/pebble.c create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/rob_window_probe-main.cpp create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/runtime_job_test.c create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/pebble.c create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/same_ball_independent-main.cpp rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/ctests/se_gate_lut_test.c (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/pebble.c create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/shared_bank_versions-main.cpp rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/ctests/smatmul_mvout_bank_reuse_test.c (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/pebble.c create mode 100644 examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/write_bank_versions-main.cpp create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/CMakeLists.txt rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/mlir_tests/bank/CMakeLists.txt (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/mlir_tests/bank/quant_im2col_k1_single_pixel_main.cpp (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_matmul_16x16_i8.mlir create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_matmul_16x16_i8_main.cpp create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_transpose_16x16_i8.mlir create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_transpose_16x16_i8_main.cpp rename examples/{chips/pebble/workloads/mlir_tests/tile => designs/pebble/workloads/device/core/acc/mlir_tests/core}/data/mega_global_avg_7x7_c288_input.i8 (100%) rename examples/{chips/pebble/workloads/mlir_tests/tile => designs/pebble/workloads/device/core/acc/mlir_tests/core}/data/mega_global_avg_conv2_exact_weight2.i8 (100%) rename examples/{chips/pebble/workloads/mlir_tests/tile => designs/pebble/workloads/device/core/acc/mlir_tests/core}/data/mega_global_avg_conv2_exact_weight3.i8 (100%) rename examples/{chips/pebble/workloads/mlir_tests/tile => designs/pebble/workloads/device/core/acc/mlir_tests/core}/data/mega_global_avg_conv_exact_input.i8 (100%) rename examples/{chips/pebble/workloads/mlir_tests/tile => designs/pebble/workloads/device/core/acc/mlir_tests/core}/data/mega_global_avg_conv_exact_weight.i8 (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/generic_matmul_i8.mlir create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/generic_matmul_i8_main.cpp create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8.mlir create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8_main.cpp create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_int8.mlir create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_int8_main.cpp create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/lookup_channel_i8.mlir create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/lookup_channel_i8_main.cpp create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_k5_tail.mlir rename examples/{chips/pebble/workloads/mlir_tests/tile => designs/pebble/workloads/device/core/acc/mlir_tests/core}/mega_conv2d_k5_tail_main.cpp (95%) rename examples/{chips/pebble/workloads/mlir_tests/tile => designs/pebble/workloads/device/core/acc/mlir_tests/core}/mega_conv2d_main.cpp (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_s2_depthwise_s2.mlir rename examples/{chips/pebble/workloads/mlir_tests/tile => designs/pebble/workloads/device/core/acc/mlir_tests/core}/mega_conv2d_s2_depthwise_s2_main.cpp (93%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact.mlir.in rename examples/{chips/pebble/workloads/mlir_tests/tile => designs/pebble/workloads/device/core/acc/mlir_tests/core}/mega_global_avg_7x7_c288_exact_main.cpp (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact.mlir.in rename examples/{chips/pebble/workloads/mlir_tests/tile => designs/pebble/workloads/device/core/acc/mlir_tests/core}/mega_global_avg_conv2_exact_main.cpp (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv_exact.mlir.in rename examples/{chips/pebble/workloads/mlir_tests/tile => designs/pebble/workloads/device/core/acc/mlir_tests/core}/mega_global_avg_conv_exact_main.cpp (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul.mlir create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul_m17_k65_n17.mlir rename examples/{chips/pebble/workloads/mlir_tests/tile => designs/pebble/workloads/device/core/acc/mlir_tests/core}/mega_matmul_m17_k65_n17_main.cpp (100%) rename examples/{chips/pebble/workloads/mlir_tests/tile => designs/pebble/workloads/device/core/acc/mlir_tests/core}/mega_matmul_main.cpp (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_se_gate_conv.mlir rename examples/{chips/pebble/workloads/mlir_tests/tile => designs/pebble/workloads/device/core/acc/mlir_tests/core}/mega_se_gate_conv_main.cpp (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_matmul_16x16_i8.mlir rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/mlir_tests/linalg/linalg_matmul_16x16_i8_main.cpp (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_mega_conv2d_k3_c16.mlir rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/mlir_tests/linalg/linalg_mega_conv2d_k3_c16_main.cpp (100%) rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/mlir_tests/linalg/linalg_relu_513x17_i32.mlir (94%) rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/mlir_tests/linalg/linalg_relu_513x17_i32_main.cpp (100%) create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_16x16_i8.mlir rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/mlir_tests/linalg/linalg_transpose_16x16_i8_main.cpp (100%) rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32.mlir (91%) rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32_main.cpp (100%) rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32.mlir (92%) rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32_main.cpp (100%) rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32.mlir (94%) rename examples/{chips/pebble/workloads => designs/pebble/workloads/device/core/acc}/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32_main.cpp (100%) create mode 100644 examples/designs/poly/arch/build.sc create mode 100644 examples/designs/poly/arch/device/DeviceTop.scala create mode 100644 examples/designs/poly/arch/device/core/PolyCore.scala create mode 100644 examples/designs/poly/configs/design.toml create mode 100644 examples/designs/poly/configs/device/clint.toml create mode 100644 examples/designs/poly/configs/device/cores/core0/acc/balldomain/default.toml create mode 100644 examples/designs/poly/configs/device/cores/core0/acc/default.toml create mode 100644 examples/designs/poly/configs/device/cores/core0/acc/frontend/poly.toml create mode 100644 examples/designs/poly/configs/device/cores/core0/acc/memdomain/poly.toml create mode 100644 examples/designs/poly/configs/device/cores/core0/c2c/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core0/controller/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core0/default.toml create mode 100644 examples/designs/poly/configs/device/cores/core0/dmaCache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core0/interconnect/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core0/memory/access/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core0/memory/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core0/memory/dcache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core0/memory/home/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core0/memory/icache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core0/tbus/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core1/c2c/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core1/controller/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core1/default.toml create mode 100644 examples/designs/poly/configs/device/cores/core1/dmaCache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core1/interconnect/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core1/memory/access/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core1/memory/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core1/memory/dcache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core1/memory/home/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core1/memory/icache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core1/tbus/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core1/transport.toml create mode 100644 examples/designs/poly/configs/device/cores/core2/c2c/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core2/controller/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core2/default.toml create mode 100644 examples/designs/poly/configs/device/cores/core2/dmaCache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core2/interconnect/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core2/memory/access/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core2/memory/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core2/memory/dcache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core2/memory/home/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core2/memory/icache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core2/tbus/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core3/c2c/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core3/controller/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core3/default.toml create mode 100644 examples/designs/poly/configs/device/cores/core3/dmaCache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core3/interconnect/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core3/memory/access/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core3/memory/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core3/memory/dcache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core3/memory/home/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core3/memory/icache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core3/tbus/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core4/c2c/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core4/controller/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core4/default.toml create mode 100644 examples/designs/poly/configs/device/cores/core4/dmaCache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core4/interconnect/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core4/memory/access/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core4/memory/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core4/memory/dcache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core4/memory/home/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core4/memory/icache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core4/tbus/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core5/c2c/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core5/controller/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core5/default.toml create mode 100644 examples/designs/poly/configs/device/cores/core5/dmaCache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core5/interconnect/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core5/memory/access/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core5/memory/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core5/memory/dcache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core5/memory/home/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core5/memory/icache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core5/tbus/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core6/c2c/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core6/controller/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core6/default.toml create mode 100644 examples/designs/poly/configs/device/cores/core6/dmaCache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core6/interconnect/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core6/memory/access/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core6/memory/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core6/memory/dcache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core6/memory/home/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core6/memory/icache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core6/tbus/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core7/c2c/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core7/controller/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core7/default.toml create mode 100644 examples/designs/poly/configs/device/cores/core7/dmaCache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core7/interconnect/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core7/memory/access/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core7/memory/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core7/memory/dcache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core7/memory/home/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core7/memory/icache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core7/tbus/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core8/c2c/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core8/controller/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core8/default.toml create mode 100644 examples/designs/poly/configs/device/cores/core8/dmaCache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core8/interconnect/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core8/memory/access/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core8/memory/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core8/memory/dcache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core8/memory/home/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core8/memory/icache/core.toml create mode 100644 examples/designs/poly/configs/device/cores/core8/tbus/core.toml create mode 100644 examples/designs/poly/configs/device/ddr.toml create mode 100644 examples/designs/poly/configs/device/default.toml create mode 100644 examples/designs/poly/configs/device/plic.toml create mode 100644 examples/designs/poly/configs/sims/default.toml create mode 100644 examples/designs/poly/emu/Cargo.lock create mode 100644 examples/designs/poly/emu/Cargo.toml create mode 100644 examples/designs/poly/emu/src/device/core/mod.rs create mode 100644 examples/designs/poly/emu/src/device/mod.rs create mode 100644 examples/designs/poly/emu/src/lib.rs create mode 100644 examples/designs/poly/sims/device/arch/DeviceConfig.scala create mode 100644 examples/designs/poly/sims/device/arch/build.sc create mode 100644 examples/designs/poly/sims/device/emu/Cargo.lock create mode 100644 examples/designs/poly/sims/device/emu/Cargo.toml create mode 100644 examples/designs/poly/sims/device/emu/build.rs create mode 100644 examples/designs/poly/sims/device/emu/src/lib.rs create mode 100644 examples/designs/poly/sims/device/regression/bemu/workloads-bare.toml rename examples/{chips/poly/regression/batch => designs/poly/sims/device/regression}/bemu/workloads-linux.toml (100%) rename examples/{chips/poly/regression/batch => designs/poly/sims/device/regression}/p2e/workloads-bare.toml (100%) rename examples/{chips/poly/regression/batch => designs/poly/sims/device/regression}/p2e/workloads-linux.toml (100%) create mode 100644 examples/designs/poly/sims/device/regression/verilator/workloads-bare.toml create mode 100644 examples/designs/poly/sims/device/stack/CMakeLists.txt create mode 100644 examples/designs/poly/sims/device/stack/core/poly/CMakeLists.txt create mode 100644 examples/designs/poly/sims/device/stack/runtime/CMakeLists.txt create mode 100644 examples/designs/poly/sims/device/workloads/CMakeLists.txt create mode 100644 examples/designs/poly/sims/device/workloads/kernel/CMakeLists.txt create mode 100644 examples/designs/poly/sims/ip/uvm/registry.toml create mode 100644 examples/designs/poly/stack/engine/CMakeLists.txt create mode 100644 examples/designs/poly/stack/engine/execute.c create mode 100644 examples/designs/poly/stack/midend/CMakeLists.txt create mode 100644 examples/designs/poly/workloads/CMakeLists.txt create mode 100644 examples/designs/poly/workloads/configs/tests.toml create mode 100644 examples/designs/poly/workloads/device/CMakeLists.txt create mode 100644 examples/designs/poly/workloads/device/common/controllers_start.S create mode 100644 examples/designs/poly/workloads/device/core/CMakeLists.txt create mode 100644 examples/designs/poly/workloads/device/core/ball/CMakeLists.txt create mode 100644 examples/designs/poly/workloads/device/core/engine/CMakeLists.txt create mode 100644 examples/designs/poly/workloads/device/core/engine/ant_bank_exercise.c create mode 100644 examples/designs/poly/workloads/device/core/engine/ant_bank_main.c create mode 100644 examples/designs/poly/workloads/device/core/engine/ant_bank_rows_exercise.c create mode 100644 examples/designs/poly/workloads/device/core/engine/ant_bank_rows_main.c create mode 100644 examples/designs/poly/workloads/device/core/engine/ant_highpa_main.c create mode 100644 examples/designs/poly/workloads/device/core/engine/ant_mxmm_main.cpp create mode 100644 examples/designs/poly/workloads/device/core/engine/ant_parallel_exercise.c create mode 100644 examples/designs/poly/workloads/device/core/engine/ant_parallel_main.c create mode 100644 examples/designs/poly/workloads/device/core/engine/ant_shared_exercise.c create mode 100644 examples/designs/poly/workloads/device/core/engine/ant_shared_main.c create mode 100644 examples/designs/poly/workloads/device/core/thread/CMakeLists.txt create mode 100644 examples/designs/poly/workloads/device/core/thread/ctests/CMakeLists.txt rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/ant_bank.c (88%) rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/ant_bank.h (81%) rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/ant_bank_rows.c (88%) rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/ant_bank_rows.h (82%) rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/ant_coherence.c (83%) rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/ant_coherence.h (100%) create mode 100644 examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.c create mode 100644 examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.h create mode 100644 examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.c create mode 100644 examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.h create mode 100644 examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.c create mode 100644 examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.h rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/ant_highpa.c (76%) rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/ant_highpa.h (100%) create mode 100644 examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh.h rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/ant_mesh_move.c (92%) rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/ant_mesh_move.h (100%) rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/ant_mxmm.c (87%) rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/ant_mxmm.h (100%) rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/ant_parallel.c (86%) rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/ant_parallel.h (78%) rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/ant_shared.c (87%) create mode 100644 examples/designs/poly/workloads/device/core/thread/ctests/ant_shared.h create mode 100644 examples/designs/poly/workloads/device/core/thread/ctests/mesh_private_bank_move_test.c create mode 100644 examples/designs/poly/workloads/device/core/thread/ctests/mesh_shared_mem_test.c rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/mxmm_fp32_small.c (100%) rename examples/{chips/poly/workloads => designs/poly/workloads/device/core/thread}/ctests/mxmm_mxfp8_small.c (100%) create mode 100644 examples/designs/poly/workloads/device/runtime/CMakeLists.txt create mode 100644 examples/designs/poly/workloads/device/runtime/ant_coherence_main.c create mode 100644 examples/designs/poly/workloads/device/runtime/ant_corelink_bank_main.c create mode 100644 examples/designs/poly/workloads/device/runtime/ant_corelink_main.c create mode 100644 examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_control.c create mode 100644 examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_main.c create mode 100644 examples/designs/poly/workloads/device/runtime/ant_mesh_main.c create mode 100644 examples/designs/poly/workloads/device/runtime/ant_mesh_move_main.c create mode 100644 examples/designs/poly/workloads/device/runtime/ctests/CMakeLists.txt create mode 100644 examples/designs/poly/workloads/device/runtime/ctests/corelink_lease.c create mode 100644 examples/designs/poly/workloads/device/runtime/stream_main.c create mode 100644 examples/designs/poly/workloads/device/soc/CMakeLists.txt rename examples/{chips/poly/workloads/ctests => designs/poly/workloads/device/soc}/controllers.c (92%) create mode 100644 examples/designs/toy/arch/build.sc create mode 100644 examples/designs/toy/arch/device/DeviceTop.scala create mode 100644 examples/designs/toy/arch/device/core/ToyCore.scala create mode 100644 examples/designs/toy/arch/device/core/acc.scala create mode 100644 examples/designs/toy/arch/device/core/loom.scala create mode 100644 examples/designs/toy/configs/design.toml create mode 100644 examples/designs/toy/configs/device/core/acc/balldomain/balls/smatmul/lane16.toml create mode 100644 examples/designs/toy/configs/device/core/acc/balldomain/default.toml create mode 100644 examples/designs/toy/configs/device/core/acc/default.toml create mode 100644 examples/designs/toy/configs/device/core/acc/demo.toml create mode 100644 examples/designs/toy/configs/device/core/acc/frontend/demo.toml create mode 100644 examples/designs/toy/configs/device/core/acc/frontend/toy.toml rename examples/{cores/toy/configs/rvv/default.toml => designs/toy/configs/device/core/acc/gpdomain/toy.toml} (100%) create mode 100644 examples/designs/toy/configs/device/core/acc/memdomain/paper.toml create mode 100644 examples/designs/toy/configs/device/core/acc/memdomain/toy.toml create mode 100644 examples/designs/toy/configs/device/core/acc/paper.toml create mode 100644 examples/designs/toy/configs/device/core/c2c/core.toml create mode 100644 examples/designs/toy/configs/device/core/default.toml create mode 100644 examples/designs/toy/configs/device/core/interconnect/core.toml create mode 100644 examples/designs/toy/configs/device/core/memory/access/core.toml create mode 100644 examples/designs/toy/configs/device/core/memory/core.toml create mode 100644 examples/designs/toy/configs/device/core/memory/dcache/core.toml create mode 100644 examples/designs/toy/configs/device/core/memory/home/core.toml create mode 100644 examples/designs/toy/configs/device/core/memory/icache/core.toml create mode 100644 examples/designs/toy/configs/device/core/rocket.toml create mode 100644 examples/designs/toy/configs/device/ddr.toml create mode 100644 examples/designs/toy/configs/device/default.toml create mode 100644 examples/designs/toy/configs/sims/default.toml create mode 100644 examples/designs/toy/emu/Cargo.lock create mode 100644 examples/designs/toy/emu/Cargo.toml create mode 100644 examples/designs/toy/emu/src/device/core/mod.rs create mode 100644 examples/designs/toy/emu/src/device/mod.rs create mode 100644 examples/designs/toy/emu/src/lib.rs create mode 100644 examples/designs/toy/sims/device/arch/DeviceConfig.scala create mode 100644 examples/designs/toy/sims/device/arch/build.sc create mode 100644 examples/designs/toy/sims/device/emu/Cargo.lock create mode 100644 examples/designs/toy/sims/device/emu/Cargo.toml create mode 100644 examples/designs/toy/sims/device/emu/build.rs create mode 100644 examples/designs/toy/sims/device/emu/src/lib.rs create mode 100644 examples/designs/toy/sims/device/emu/tests/instance_isolation.rs create mode 100644 examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml rename examples/{chips/toy/regression/batch => designs/toy/sims/device/regression}/bemu/workloads-linux.toml (100%) create mode 100644 examples/designs/toy/sims/device/regression/p2e/workloads-bare.toml rename examples/{chips/toy/regression/batch => designs/toy/sims/device/regression}/p2e/workloads-linux.toml (100%) rename {arch/src/main/scala/framework/root/root-chip/src/csrc => examples/designs/toy/sims/device/regression/soc}/README.md (100%) create mode 100644 examples/designs/toy/sims/device/regression/verilator/workloads-bare.toml create mode 100644 examples/designs/toy/sims/device/regression/verilator/workloads-linux.toml create mode 100644 examples/designs/toy/sims/device/regression/workloads-verilator.toml create mode 100644 examples/designs/toy/sims/device/stack/CMakeLists.txt create mode 100644 examples/designs/toy/sims/device/stack/core/toy/CMakeLists.txt create mode 100644 examples/designs/toy/sims/device/stack/runtime/CMakeLists.txt create mode 100644 examples/designs/toy/sims/device/tapeout/area/constraints.sdc create mode 100644 examples/designs/toy/sims/device/tapeout/area/dc.tcl create mode 100644 examples/designs/toy/sims/device/tapeout/config.toml create mode 100644 examples/designs/toy/sims/device/tapeout/ip/smic180_sram_library.mdf.json create mode 100644 examples/designs/toy/sims/device/workloads/CMakeLists.txt create mode 100644 examples/designs/toy/sims/device/workloads/kernel/CMakeLists.txt create mode 100644 examples/designs/toy/sims/ip/uvm/configs/axis16.toml create mode 100644 examples/designs/toy/sims/ip/uvm/registry.toml create mode 100644 examples/designs/toy/stack/engine/CMakeLists.txt create mode 100644 examples/designs/toy/stack/midend/CMakeLists.txt create mode 100644 examples/designs/toy/workloads/CMakeLists.txt create mode 100644 examples/designs/toy/workloads/configs/tests.toml create mode 100644 examples/designs/toy/workloads/device/CMakeLists.txt create mode 100644 examples/designs/toy/workloads/device/core/CMakeLists.txt create mode 100644 examples/designs/toy/workloads/device/core/acc/CMakeLists.txt create mode 100644 examples/designs/toy/workloads/device/core/acc/ctests/CMakeLists.txt create mode 100644 examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/CMakeLists.txt create mode 100644 examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp rename examples/{cores/toy/workloads => designs/toy/workloads/device/core/acc}/ctests/gemmini_relu_trace_pipeline/toy.c (90%) rename examples/{chips/toy/workloads => designs/toy/workloads/device/core/acc}/ctests/mmio_minimal_test.c (94%) rename examples/{cores/toy/workloads => designs/toy/workloads/device/core/acc}/ctests/rvv_ball.c (78%) create mode 100644 examples/designs/toy/workloads/device/core/acc/mlir_tests/CMakeLists.txt create mode 100644 examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt rename examples/{chips/toy/workloads => designs/toy/workloads/device/core/acc}/mlir_tests/linalg/conv2d_nchw_fchw.mlir (100%) rename examples/{chips/toy/workloads => designs/toy/workloads/device/core/acc}/mlir_tests/linalg/conv2d_nchw_fchw_main.cpp (100%) rename examples/{chips/toy/workloads => designs/toy/workloads/device/core/acc}/mlir_tests/linalg/conv2d_nhwc_fhwc.mlir (100%) rename examples/{chips/toy/workloads => designs/toy/workloads/device/core/acc}/mlir_tests/linalg/conv2d_nhwc_fhwc_main.cpp (100%) rename examples/{chips/toy/workloads => designs/toy/workloads/device/core/acc}/mlir_tests/linalg/conv2d_nhwc_hwcf.mlir (100%) rename examples/{chips/toy/workloads => designs/toy/workloads/device/core/acc}/mlir_tests/linalg/conv2d_nhwc_hwcf_main.cpp (100%) create mode 100644 examples/designs/toy/workloads/device/soc/CMakeLists.txt rename examples/{chips/toy/workloads/ctests => designs/toy/workloads/device/soc}/scu_uart_rx_echo_test.c (100%) delete mode 100644 scripts/nix/build-env-cuda.nix rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/base/base_dma_group.mlir (100%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/base/base_dma_group_main.cpp (100%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/base/base_mset_transfer.mlir (94%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/base/base_mset_transfer_main.cpp (100%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/compiler/bank_kernel_calls.mlir (89%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/compiler/bank_kernel_invalid.mlir (94%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/compiler/bank_transfer_cfg_invalid.mlir (93%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/compiler/bank_transfer_invalid.mlir (95%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/compiler/bank_transfer_ownership.mlir (97%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/compiler/contiguous_scatter_rows.mlir (100%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/compiler/dma_group_invalid.mlir (100%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/compiler/dma_group_lowering.mlir (78%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/compiler/fp32_packmm_mixed.mlir (81%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/compiler/mset_transfer_invalid.mlir (100%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/compiler/mset_transfer_lowering.mlir (77%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/compiler/norm_quant_window_mixed.mlir (92%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/compiler/rvv_rms_suffix.mlir (100%) rename {bb-tests => waste/bb-tests}/workloads/src/MLIRTest/compiler/rvv_tanh.mlir (100%) create mode 100644 waste/examples/components/balls/f32add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp rename {examples/balls/f32add/compiler => waste/examples/components/balls/f32add/stack}/src/Dialect/Buckyball/F32AddBall.td (88%) rename {examples/balls/f32add/compiler => waste/examples/components/balls/f32add/stack}/src/Dialect/Buckyball/IR/F32Add.cpp (96%) create mode 100644 waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/f32add/stack/src/Passes.cpp create mode 100644 waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp create mode 100644 waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp create mode 100644 waste/examples/components/balls/gemmini/stack/src/Conversion/LowerLinalgToBuckyball/Int8Contractions.cpp rename {examples/balls/gemmini/compiler => waste/examples/components/balls/gemmini/stack}/src/Dialect/Buckyball/GemminiBall.td (76%) rename {examples/balls/gemmini/compiler => waste/examples/components/balls/gemmini/stack}/src/Dialect/Buckyball/IR/GemminiConv.cpp (97%) create mode 100644 waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/gemmini/stack/src/Passes.cpp rename {examples/balls/im2col/compiler => waste/examples/components/balls/im2col/stack}/src/Backend/include/IntrinsicsRISCVIm2colBall.td (100%) create mode 100644 waste/examples/components/balls/im2col/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp rename {examples/balls/im2col/compiler => waste/examples/components/balls/im2col/stack}/src/Dialect/Buckyball/Im2colBall.td (97%) create mode 100644 waste/examples/components/balls/im2col/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/im2col/stack/src/Passes.cpp create mode 100644 waste/examples/components/balls/int2fp/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp rename {examples/balls/int2fp/compiler => waste/examples/components/balls/int2fp/stack}/src/Dialect/Buckyball/Int2FpBall.td (92%) create mode 100644 waste/examples/components/balls/int2fp/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/int2fp/stack/src/Passes.cpp rename {examples/balls/int8add/compiler => waste/examples/components/balls/int8add/stack}/src/Backend/include/IntrinsicsRISCVInt8AddBall.td (100%) create mode 100644 waste/examples/components/balls/int8add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp rename {examples/balls/int8add/compiler => waste/examples/components/balls/int8add/stack}/src/Dialect/Buckyball/Int8AddBall.td (90%) create mode 100644 waste/examples/components/balls/int8add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/int8add/stack/src/Passes.cpp rename {examples/balls/int8mul/compiler => waste/examples/components/balls/int8mul/stack}/src/Backend/include/IntrinsicsRISCVInt8MulBall.td (100%) create mode 100644 waste/examples/components/balls/int8mul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp rename {examples/balls/int8mul/compiler => waste/examples/components/balls/int8mul/stack}/src/Dialect/Buckyball/Int8MulBall.td (90%) create mode 100644 waste/examples/components/balls/int8mul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/int8mul/stack/src/Passes.cpp create mode 100644 waste/examples/components/balls/lut/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp rename {examples/balls/lut/compiler => waste/examples/components/balls/lut/stack}/src/Dialect/Buckyball/LutBall.td (88%) create mode 100644 waste/examples/components/balls/lut/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/lut/stack/src/Passes.cpp rename {examples/balls/matadd/compiler => waste/examples/components/balls/matadd/stack}/src/Backend/include/IntrinsicsRISCVMatAddBall.td (100%) create mode 100644 waste/examples/components/balls/matadd/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp rename {examples/balls/matadd/compiler => waste/examples/components/balls/matadd/stack}/src/Dialect/Buckyball/MatAddBall.td (86%) create mode 100644 waste/examples/components/balls/matadd/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/matadd/stack/src/Passes.cpp create mode 100644 waste/examples/components/balls/maxpool/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp rename {examples/balls/maxpool/compiler => waste/examples/components/balls/maxpool/stack}/src/Dialect/Buckyball/MaxPoolBall.td (89%) create mode 100644 waste/examples/components/balls/maxpool/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/maxpool/stack/src/Passes.cpp create mode 100644 waste/examples/components/balls/mxfp2int/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp rename {examples/balls/mxfp2int/compiler => waste/examples/components/balls/mxfp2int/stack}/src/Dialect/Buckyball/Mxfp2IntBall.td (89%) create mode 100644 waste/examples/components/balls/mxfp2int/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/mxfp2int/stack/src/Passes.cpp rename {examples/balls/mxmm/compiler => waste/examples/components/balls/mxmm/stack}/python/fp32.py (100%) rename {examples/balls/mxmm/compiler => waste/examples/components/balls/mxmm/stack}/python/grouped_attention.py (100%) rename {examples/balls/mxmm/compiler => waste/examples/components/balls/mxmm/stack}/python/layout.py (83%) create mode 100644 waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp create mode 100644 waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp rename {examples/balls/mxmm/compiler => waste/examples/components/balls/mxmm/stack}/src/Dialect/Buckyball/IR/FP32.cpp (100%) rename {examples/balls/mxmm/compiler => waste/examples/components/balls/mxmm/stack}/src/Dialect/Buckyball/IR/MXFP8.cpp (100%) rename {examples/balls/mxmm/compiler => waste/examples/components/balls/mxmm/stack}/src/Dialect/Buckyball/MxmmBall.td (88%) create mode 100644 waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp rename {examples/balls/mxmm/compiler => waste/examples/components/balls/mxmm/stack}/src/Dialect/Buckyball/Transforms/WindowPatterns.inc (98%) create mode 100644 waste/examples/components/balls/mxmm/stack/src/Passes.cpp rename {examples/balls/mxmm/compiler => waste/examples/components/balls/mxmm/stack}/src/Transforms/NormQuantWindow.cpp (99%) create mode 100644 waste/examples/components/balls/mxquant/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp rename {examples/balls/mxquant/compiler => waste/examples/components/balls/mxquant/stack}/src/Dialect/Buckyball/MxquantBall.td (82%) create mode 100644 waste/examples/components/balls/mxquant/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/mxquant/stack/src/Passes.cpp create mode 100644 waste/examples/components/balls/redmule/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp create mode 100644 waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/IR/Redmule.cpp create mode 100644 waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/RedmuleBall.td create mode 100644 waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/redmule/stack/src/Passes.cpp rename {examples/balls/relu/compiler => waste/examples/components/balls/relu/stack}/src/Backend/include/IntrinsicsRISCVReluBall.td (100%) create mode 100644 waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp create mode 100644 waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp create mode 100644 waste/examples/components/balls/relu/stack/src/Conversion/LowerLinalgToBuckyball/ReluPatterns.cpp rename {examples/balls/relu/compiler => waste/examples/components/balls/relu/stack}/src/Dialect/Buckyball/ReluBall.td (83%) create mode 100644 waste/examples/components/balls/relu/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/relu/stack/src/Passes.cpp rename {examples/balls/smatmul/compiler => waste/examples/components/balls/smatmul/stack}/src/Backend/include/IntrinsicsRISCVSMatMulBall.td (100%) create mode 100644 waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp create mode 100644 waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp rename {examples/balls/smatmul/compiler => waste/examples/components/balls/smatmul/stack}/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp (84%) create mode 100644 waste/examples/components/balls/smatmul/stack/src/Conversion/LowerLinalgToBuckyball/MatMulPatterns.cpp rename {examples/balls/smatmul/compiler => waste/examples/components/balls/smatmul/stack}/src/Dialect/Buckyball/SMatMulBall.td (92%) create mode 100644 waste/examples/components/balls/smatmul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/smatmul/stack/src/Passes.cpp create mode 100644 waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp create mode 100644 waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp create mode 100644 waste/examples/components/balls/toint8/stack/src/Conversion/LowerLinalgToBuckyball/QuantizePatterns.cpp rename {examples/balls/toint8/compiler => waste/examples/components/balls/toint8/stack}/src/Dialect/Buckyball/ToInt8Ball.td (77%) create mode 100644 waste/examples/components/balls/toint8/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/toint8/stack/src/Passes.cpp rename {examples/balls/trace/compiler => waste/examples/components/balls/trace/stack}/src/Dialect/Buckyball/TraceBall.td (100%) create mode 100644 waste/examples/components/balls/trace/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp create mode 100644 waste/examples/components/balls/trace/stack/src/Passes.cpp rename {examples/balls/transpose/compiler => waste/examples/components/balls/transpose/stack}/src/Backend/include/IntrinsicsRISCVTransposeBall.td (100%) create mode 100644 waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp create mode 100644 waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp create mode 100644 waste/examples/components/balls/transpose/stack/src/Conversion/LowerLinalgToBuckyball/TransposePatterns.cpp create mode 100644 waste/examples/components/balls/transpose/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp rename {examples/balls/transpose/compiler => waste/examples/components/balls/transpose/stack}/src/Dialect/Buckyball/TransposeBall.td (83%) create mode 100644 waste/examples/components/balls/transpose/stack/src/Passes.cpp rename {examples/balls/vector/compiler => waste/examples/components/balls/vector/stack}/src/Backend/include/IntrinsicsRISCVVecBall.td (100%) create mode 100644 waste/examples/components/balls/vector/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp create mode 100644 waste/examples/components/balls/vector/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp rename {examples/balls/vector/compiler => waste/examples/components/balls/vector/stack}/src/Dialect/Buckyball/VecBall.td (92%) create mode 100644 waste/examples/components/balls/vector/stack/src/Passes.cpp create mode 100644 waste/examples/designs/goban/stack/midend/src/FormMegaKernels.cpp create mode 100644 waste/examples/designs/pebble/stack/midend/ops.py rename {examples/cores/pebble/compiler => waste/examples/designs/pebble/stack/midend}/src/Conversion/LowerBuckyball/MegaConv2dToBankSSAPatterns.cpp (98%) rename {examples/cores/pebble/compiler => waste/examples/designs/pebble/stack/midend}/src/Conversion/LowerBuckyball/MemTransposeToBankSSAPatterns.cpp (98%) rename {examples/cores/pebble/compiler => waste/examples/designs/pebble/stack/midend}/src/Conversion/LowerBuckyball/ResidentConvRegionToBankSSAPatterns.cpp (96%) create mode 100644 waste/examples/designs/pebble/stack/midend/src/LargeConvolution.cpp create mode 100644 waste/examples/designs/pebble/stack/midend/src/Passes.cpp rename arch/src/main/scala/framework/root/root-chip/src/main/resources/README.md => waste/stack/compiler/frontend/__init__.py (100%) rename arch/src/main/scala/framework/root/root-core/src/csrc/README.md => waste/stack/compiler/frontend/layout/__init__.py (100%) create mode 100644 waste/stack/compiler/frontend/layout/provenance.py create mode 100644 waste/stack/compiler/frontend/operation.py rename arch/src/main/scala/framework/root/root-core/src/main/resources/README.md => waste/stack/compiler/frontend/ops/__init__.py (100%) create mode 100644 waste/stack/compiler/frontend/ops/expf.py create mode 100644 waste/stack/compiler/frontend/ops/int8.py create mode 100644 waste/stack/compiler/frontend/ops/linalg.py create mode 100644 waste/stack/compiler/frontend/ops/native_mxfp8.py create mode 100644 waste/stack/compiler/frontend/ops/reductions.py create mode 100644 waste/stack/compiler/frontend/ops/tosa.py rename arch/src/main/scala/framework/root/root-tile/src/csrc/README.md => waste/stack/compiler/frontend/quant/__init__.py (100%) create mode 100644 waste/stack/compiler/frontend/quant/bwq.py create mode 100644 waste/stack/compiler/frontend/quant/importer.py create mode 100644 waste/stack/compiler/frontend/quant/mxfp8_cache.py create mode 100644 waste/stack/compiler/frontend/quant/native_mxfp8.py rename arch/src/main/scala/framework/root/root-tile/src/main/resources/README.md => waste/stack/compiler/frontend/quant/offline/__init__.py (100%) create mode 100644 waste/stack/compiler/frontend/quant/offline/activations/__init__.py create mode 100644 waste/stack/compiler/frontend/quant/offline/activations/calibrate.py create mode 100644 waste/stack/compiler/frontend/quant/offline/weights/__init__.py create mode 100644 waste/stack/compiler/frontend/quant/offline/weights/quantize.py rename arch/src/main/scala/framework/rvv/src/csrc/README.md => waste/stack/compiler/frontend/quant/offline/weights/strategy/__init__.py (100%) create mode 100644 waste/stack/compiler/frontend/quant/offline/weights/strategy/fp32toint8.py create mode 100644 waste/stack/compiler/frontend/quant/rax.py rename arch/src/main/scala/framework/rvv/src/main/resources/README.md => waste/stack/compiler/frontend/quant/transforms/__init__.py (100%) create mode 100644 waste/stack/compiler/frontend/quant/transforms/activation.py create mode 100644 waste/stack/compiler/frontend/quant/transforms/normalize.py create mode 100644 waste/stack/compiler/frontend/quant/transforms/symmetric.py create mode 100644 waste/stack/compiler/frontend/trace.py create mode 100644 waste/stack/compiler/frontend/transforms.py create mode 100644 waste/stack/compiler/src/Conversion/BallPasses.h create mode 100644 waste/stack/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp create mode 100644 waste/stack/compiler/src/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp create mode 100644 waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyball.h create mode 100644 waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp create mode 100644 waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp create mode 100644 waste/stack/compiler/src/Conversion/LowerBuckyball/PhysicalBankState.cpp create mode 100644 waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/FormMegaKernelRegions.cpp create mode 100644 waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h create mode 100644 waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyballPass.cpp create mode 100644 waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/MegaKernelPatterns.cpp create mode 100644 waste/stack/compiler/src/Dialect/Buckyball/Buckyball.td create mode 100644 waste/stack/compiler/src/Dialect/Buckyball/BuckyballDialect.h create mode 100644 waste/stack/compiler/src/Dialect/Buckyball/BuckyballInterfaces.td create mode 100644 waste/stack/compiler/src/Dialect/Buckyball/BuckyballOps.h create mode 100644 waste/stack/compiler/src/Dialect/Buckyball/IR/BuckyballDialect.cpp create mode 100644 waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp create mode 100644 waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h create mode 100644 waste/stack/compiler/src/Plugin.cpp create mode 100644 waste/stack/compiler/src/Target/BuckyballTargetRegistry.cpp create mode 100644 waste/stack/compiler/src/Target/BuckyballTargetRegistry.h create mode 100644 waste/stack/compiler/src/Utils/BankUtils.h create mode 100644 waste/stack/compiler/src/Utils/StageTrace.cpp create mode 100644 waste/stack/compiler/src/Utils/StageTrace.h diff --git a/.agents/mcps b/.agents/mcps index a34c9b720..0fcca530b 160000 --- a/.agents/mcps +++ b/.agents/mcps @@ -1 +1 @@ -Subproject commit a34c9b7203df6b4ff78e9691da9e060f72228830 +Subproject commit 0fcca530b2c3f012aaa5e9589755e3bb0dd1206f diff --git a/.agents/skills b/.agents/skills index bc1b7ef14..9ca889df7 160000 --- a/.agents/skills +++ b/.agents/skills @@ -1 +1 @@ -Subproject commit bc1b7ef1446636b0e55cd0bef4c00bfee20d1160 +Subproject commit 9ca889df73af6c01634fe39cb18217ee205d6d3a diff --git a/.github/scripts/ci_repo_lock.sh b/.github/scripts/ci_repo_lock.sh index 5b47fc888..c3db1ba6a 100755 --- a/.github/scripts/ci_repo_lock.sh +++ b/.github/scripts/ci_repo_lock.sh @@ -31,20 +31,20 @@ repo_reset() { git clean -ffd rm -rf arch/out - rm -rf stack/compiler/thirdparty/buddy-mlir/llvm/build - rm -rf stack/compiler/thirdparty/buddy-mlir/build + rm -rf stack/thirdparty/buddy-mlir/llvm/build + rm -rf stack/thirdparty/buddy-mlir/build git submodule sync git submodule update --init --force # submodules in submodules we used need update by hands - git -C arch/thirdparty/rocket-chip submodule sync -- dependencies/cde - git -C arch/thirdparty/rocket-chip submodule update --init --force -- dependencies/cde + git -C examples/components/cores/rocket/thirdparty/rocket-chip submodule sync -- dependencies/cde + git -C examples/components/cores/rocket/thirdparty/rocket-chip submodule update --init --force -- dependencies/cde git -C stack submodule sync - git -C stack submodule update --init --force -- compiler/thirdparty/buddy-mlir - git -C stack/compiler/thirdparty/buddy-mlir submodule sync - git -C stack/compiler/thirdparty/buddy-mlir submodule update --init --force -- llvm + git -C stack submodule update --init --force -- thirdparty/buddy-mlir + git -C stack/thirdparty/buddy-mlir submodule sync + git -C stack/thirdparty/buddy-mlir submodule update --init --force -- llvm git submodule foreach 'git clean -ffd' have_sha="$(git rev-parse HEAD)" diff --git a/.github/workflows/README.zh.md b/.github/workflows/README.zh.md index 314dee3e5..a857fe524 100644 --- a/.github/workflows/README.zh.md +++ b/.github/workflows/README.zh.md @@ -3,7 +3,7 @@ push/PR 到 `main` 会自动触发 `check.yaml`： 1. **pre-commit**：拉代码, 并校验格式 -2. **chip-check**：通过后，matrix 并行跑各 chip（共享同一个 `~/Code/buckyball` 环境，基于 `ci_repo_lock.sh` 协调） +2. **design-check**：通过后，matrix 并行跑各 design（共享同一个 `~/Code/buckyball` 环境，基于 `ci_repo_lock.sh` 协调） `regression.yml` 是 reviewer 的可选测试。注意：Environment `regression` 必须在仓库 Settings → Environments 里打开 Required reviewers，否则会直接跑。 也可以在 Actions → regression → Run workflow 中选择任意分支手动启动，测试触发时该分支的最新 commit。 diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 23eb7b9a7..fb4de1314 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -59,6 +59,7 @@ jobs: - name: Check pre-commit shell: zsh {0} run: | + set -euo pipefail root=~/Code/buckyball source "${root}/.github/scripts/setproxy.sh" cd "${root}" @@ -72,27 +73,27 @@ jobs: cd "${root}" nix develop -c bbdev config --install - chip-check: - name: ${{ matrix.chip }} + design-check: + name: ${{ matrix.design }} needs: pre-commit runs-on: check strategy: fail-fast: false matrix: include: - - chip: toy + - design: toy run_bemu_tests: true run_verilator_tests: true - - chip: pebble + - design: pebble run_bemu_tests: true run_verilator_tests: true - - chip: goban + - design: goban run_bemu_tests: true run_verilator_tests: true - - chip: poly + - design: poly run_bemu_tests: true run_verilator_tests: true - - chip: multi-rocket + - design: multi-rocket run_bemu_tests: true run_verilator_tests: true env: @@ -100,7 +101,7 @@ jobs: steps: - name: Print information run: | - echo "chip=${{ matrix.chip }}" + echo "design=${{ matrix.design }}" echo "event=${{ github.event_name }}" echo "sha=${CHECKOUT_SHA}" echo "ref=${CHECKOUT_REF}" @@ -130,9 +131,9 @@ jobs: set -euo pipefail root="${BB_ROOT/#\~/$HOME}" cd "${root}" - nix develop -c bbdev compiler --build '--chip ${{ matrix.chip }}' - nix develop -c bbdev workload --clean '--chip ${{ matrix.chip }}' - nix develop -c bbdev workload --build '--chip ${{ matrix.chip }}' + nix develop -c bbdev stack --build '--design ${{ matrix.design }} --device' + nix develop -c bbdev workload --clean '--design ${{ matrix.design }} --device' + nix develop -c bbdev workload --build '--design ${{ matrix.design }} --device' - name: bemu tests if: matrix.run_bemu_tests @@ -141,9 +142,9 @@ jobs: set -euo pipefail root="${BB_ROOT/#\~/$HOME}" cd "${root}" - nix develop -c bbdev bebop-bemu --batch '--chip ${{ matrix.chip }} --test bare-tests --jobs 16 --clean-before' - nix develop -c bbdev kernel --build '--chip ${{ matrix.chip }}' - nix develop -c bbdev bebop-bemu --batch '--chip ${{ matrix.chip }} --test linux-tests --jobs 16' + nix develop -c bbdev bebop-bemu --batch "--design ${{ matrix.design }} --device --test bare-tests --jobs 16 --clean-before --junit ${root}/bebop/target/${{ matrix.design }}/test-reports/ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/bemu-bare.xml" + nix develop -c bbdev kernel --build '--design ${{ matrix.design }} --device' + nix develop -c bbdev bebop-bemu --batch "--design ${{ matrix.design }} --device --test linux-tests --jobs 16 --junit ${root}/bebop/target/${{ matrix.design }}/test-reports/ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/bemu-linux.xml" - name: verilator tests if: matrix.run_verilator_tests @@ -152,10 +153,10 @@ jobs: set -euo pipefail root="${BB_ROOT/#\~/$HOME}" cd "${root}" - nix develop -c bbdev bebop-verilator --clean '--chip ${{ matrix.chip }}' - nix develop -c bbdev bebop-verilator --verilog '--chip ${{ matrix.chip }} --diff' - nix develop -c bbdev bebop-verilator --build '--jobs 100 --chip ${{ matrix.chip }} --diff' - nix develop -c bbdev bebop-verilator --batch '--chip ${{ matrix.chip }} --test bare-tests --diff --jobs 16 --clean-before' + nix develop -c bbdev bebop-verilator --clean '--design ${{ matrix.design }} --device' + nix develop -c bbdev bebop-verilator --verilog '--design ${{ matrix.design }} --device --diff' + nix develop -c bbdev bebop-verilator --build '--jobs 100 --design ${{ matrix.design }} --device --diff' + nix develop -c bbdev bebop-verilator --batch "--design ${{ matrix.design }} --device --test bare-tests --diff --jobs 16 --clean-before --junit ${root}/bebop/target/${{ matrix.design }}/test-reports/ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/verilator-bare.xml" - name: Detect bebop JUnit files if: always() && (matrix.run_bemu_tests || matrix.run_verilator_tests) @@ -163,7 +164,9 @@ jobs: shell: zsh {0} run: | root="${BB_ROOT/#\~/$HOME}" - files=("${root}/bebop/target/${{ matrix.chip }}/nextest/junit-bbdev-"*.xml(N)) + reports="${root}/bebop/target/${{ matrix.design }}/test-reports/ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + echo "directory=${reports}" >> "$GITHUB_OUTPUT" + files=("${reports}/"*.xml(N)) if (( ${#files} == 0 )); then echo "found=false" >> "$GITHUB_OUTPUT" else @@ -174,8 +177,8 @@ jobs: if: always() && (matrix.run_bemu_tests || matrix.run_verilator_tests) && steps.junit.outputs.found == 'true' uses: mikepenz/action-junit-report@v4 with: - report_paths: "~/Code/buckyball/bebop/target/${{ matrix.chip }}/nextest/junit-bbdev-*.xml" - check_name: "bebop nextest (${{ matrix.chip }})" + report_paths: "${{ steps.junit.outputs.directory }}/*.xml" + check_name: "bebop regression (${{ matrix.design }})" detailed_summary: true include_passed: true @@ -188,8 +191,8 @@ jobs: HTTPS_PROXY: "" NO_PROXY: github.com,api.github.com,uploads.github.com,objects.githubusercontent.com,.blob.core.windows.net with: - name: bebop-nextest-junit-${{ matrix.chip }} - path: "~/Code/buckyball/bebop/target/${{ matrix.chip }}/nextest/junit-bbdev-*.xml" + name: bebop-junit-${{ matrix.design }} + path: "${{ steps.junit.outputs.directory }}/*.xml" - name: Exit repo lock if: always() diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 08d94738d..9e4b67eee 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -63,26 +63,26 @@ jobs: nix develop -c bbdev config --install eval-area-power: - name: eval-area-power-${{ matrix.chip }} + name: eval-area-power-${{ matrix.design }} needs: gate runs-on: check strategy: fail-fast: false matrix: include: - - chip: toy + - design: toy run_area: true run_power: false - - chip: pebble + - design: pebble run_area: true run_power: false - - chip: goban + - design: goban run_area: true run_power: false - - chip: poly + - design: poly run_area: true run_power: false - - chip: multi-rocket + - design: multi-rocket run_area: true run_power: false env: @@ -111,7 +111,7 @@ jobs: set +u source ~/.zshrc set -u - nix develop -c bbdev dc --area '--chip ${{ matrix.chip }}' + nix develop -c bbdev dc --area '--design ${{ matrix.design }} --device' - name: DC power if: matrix.run_power @@ -123,7 +123,7 @@ jobs: set +u source ~/.zshrc set -u - nix develop -c bbdev dc --power '--chip ${{ matrix.chip }}' + nix develop -c bbdev dc --power '--design ${{ matrix.design }} --device' - name: Exit repo lock if: always() && (matrix.run_area || matrix.run_power) @@ -135,22 +135,22 @@ jobs: exit "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" verification: - name: verification-${{ matrix.chip }} + name: verification-${{ matrix.design }} needs: gate runs-on: check strategy: fail-fast: false matrix: include: - - chip: toy + - design: toy run_uvm: true - - chip: pebble + - design: pebble run_uvm: true - - chip: goban + - design: goban run_uvm: true - - chip: poly + - design: poly run_uvm: true - - chip: multi-rocket + - design: multi-rocket run_uvm: true env: BB_ROOT: ~/Code/buckyball @@ -168,7 +168,7 @@ jobs: git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ enter "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" - - name: Chip UVM + - name: Design UVM if: matrix.run_uvm shell: zsh {0} run: | @@ -178,8 +178,8 @@ jobs: set +u source ~/.zshrc set -u - nix develop -c bbdev bebop-verilator --verilog '--chip ${{ matrix.chip }}' - nix develop -c bbdev uvm --run '--chip ${{ matrix.chip }}' + nix develop -c bbdev bebop-verilator --verilog '--design ${{ matrix.design }} --device' + nix develop -c bbdev uvm --run '--design ${{ matrix.design }} --device' - name: Exit repo lock if: always() && matrix.run_uvm @@ -191,7 +191,7 @@ jobs: exit "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" fpga-bitstream: - name: fpga-bitstream-${{ matrix.chip }} + name: fpga-bitstream-${{ matrix.design }} needs: gate runs-on: check timeout-minutes: 720 @@ -199,15 +199,15 @@ jobs: fail-fast: false matrix: include: - - chip: toy + - design: toy build_bitstream: true - - chip: pebble + - design: pebble build_bitstream: true - - chip: goban + - design: goban build_bitstream: true - - chip: poly + - design: poly build_bitstream: true - - chip: multi-rocket + - design: multi-rocket build_bitstream: true env: BB_ROOT: ~/Code/buckyball @@ -231,14 +231,14 @@ jobs: set -euo pipefail root="${BB_ROOT/#\~/$HOME}" cd "${root}" - for build_dir in "${root}"/bebop/build/"${{ matrix.chip }}"-*(N); do + for build_dir in "${root}"/bebop/build/"${{ matrix.design }}"-*(N); do rm -rf "${build_dir}" done - nix develop -c bbdev bebop-p2e --verilog '--chip ${{ matrix.chip }} --diff' - nix develop -c bbdev bebop-p2e --buildbitstream '--chip ${{ matrix.chip }} --diff' - bits=("${root}"/bebop/build/"${{ matrix.chip }}"-*/fpgaCompDir/bitstream.bit(N)) + nix develop -c bbdev bebop-p2e --verilog '--design ${{ matrix.design }} --device --diff' + nix develop -c bbdev bebop-p2e --buildbitstream '--design ${{ matrix.design }} --device --diff' + bits=("${root}"/bebop/build/"${{ matrix.design }}"-*/fpgaCompDir/bitstream.bit(N)) if (( ${#bits} != 1 )); then - echo "ERROR: expected exactly 1 bitstream.bit for ${{ matrix.chip }}, got ${#bits}" >&2 + echo "ERROR: expected exactly 1 bitstream.bit for ${{ matrix.design }}, got ${#bits}" >&2 printf '%s\n' "${bits[@]}" >&2 exit 1 fi @@ -254,26 +254,26 @@ jobs: exit "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" fpga-check: - name: fpga-check-${{ matrix.chip }} + name: fpga-check-${{ matrix.design }} needs: fpga-bitstream runs-on: check strategy: fail-fast: false matrix: include: - - chip: toy + - design: toy run_fpga_bare: true run_fpga_linux: true - - chip: pebble + - design: pebble run_fpga_bare: true run_fpga_linux: true - - chip: goban + - design: goban run_fpga_bare: true run_fpga_linux: false - - chip: poly + - design: poly run_fpga_bare: true run_fpga_linux: false - - chip: multi-rocket + - design: multi-rocket run_fpga_bare: true run_fpga_linux: false env: @@ -298,23 +298,23 @@ jobs: set -euo pipefail root="${BB_ROOT/#\~/$HOME}" cd "${root}" - bits=("${root}"/bebop/build/"${{ matrix.chip }}"-*/fpgaCompDir/bitstream.bit(N)) + bits=("${root}"/bebop/build/"${{ matrix.design }}"-*/fpgaCompDir/bitstream.bit(N)) if (( ${#bits} != 1 )); then - echo "ERROR: expected exactly 1 bitstream.bit for ${{ matrix.chip }}, got ${#bits}" >&2 + echo "ERROR: expected exactly 1 bitstream.bit for ${{ matrix.design }}, got ${#bits}" >&2 printf '%s\n' "${bits[@]}" >&2 exit 1 fi if [[ "${{ matrix.run_fpga_bare }}" == "true" ]]; then - nix develop -c bbdev compiler --build "--chip ${{ matrix.chip }}" - nix develop -c bbdev workload --build "--chip ${{ matrix.chip }}" - nix develop -c bbdev workload --tohex "--chip ${{ matrix.chip }}" - nix develop -c bbdev bebop-p2e --batch "--chip ${{ matrix.chip }} --bitstream ${bits[1]} --test bare-tests --diff" + nix develop -c bbdev stack --build "--design ${{ matrix.design }} --device" + nix develop -c bbdev workload --build "--design ${{ matrix.design }} --device" + nix develop -c bbdev workload --tohex "--design ${{ matrix.design }} --device" + nix develop -c bbdev bebop-p2e --batch "--design ${{ matrix.design }} --device --bitstream ${bits[1]} --test bare-tests --diff" fi if [[ "${{ matrix.run_fpga_linux }}" == "true" ]]; then - nix develop -c bbdev kernel --build "--chip ${{ matrix.chip }}" - nix develop -c bbdev bebop-p2e --batch "--chip ${{ matrix.chip }} --bitstream ${bits[1]} --test linux-tests --diff" + nix develop -c bbdev kernel --build "--design ${{ matrix.design }} --device" + nix develop -c bbdev bebop-p2e --batch "--design ${{ matrix.design }} --device --bitstream ${bits[1]} --test linux-tests --diff" fi - name: Exit repo lock @@ -327,20 +327,20 @@ jobs: exit "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" eval-performance: - name: eval-performance-${{ matrix.chip }}-${{ matrix.model }} + name: eval-performance-${{ matrix.design }}-${{ matrix.model }} needs: fpga-bitstream runs-on: check strategy: fail-fast: false matrix: include: - - chip: pebble + - design: pebble model: lenet - - chip: pebble + - design: pebble model: mobilenet - # - chip: pebble + # - design: pebble # model: resnet - # - chip: pebble + # - design: pebble # model: yolo env: BB_ROOT: ~/Code/buckyball @@ -366,9 +366,9 @@ jobs: echo "ERROR: eval-performance requires a real model, got model=none" >&2 exit 1 fi - nix develop -c bbdev compiler --build "--chip ${{ matrix.chip }}" - nix develop -c bbdev workload --build "--chip ${{ matrix.chip }} --model ${{ matrix.model }}" - nix develop -c bbdev kernel --build "--chip ${{ matrix.chip }} --model ${{ matrix.model }}" + nix develop -c bbdev stack --build "--design ${{ matrix.design }} --device" + nix develop -c bbdev workload --build "--design ${{ matrix.design }} --device --model ${{ matrix.model }}" + nix develop -c bbdev kernel --build "--design ${{ matrix.design }} --device --model ${{ matrix.model }}" - name: Model performance shell: zsh {0} @@ -376,13 +376,13 @@ jobs: set -euo pipefail root="${BB_ROOT/#\~/$HOME}" cd "${root}" - bits=("${root}"/bebop/build/"${{ matrix.chip }}"-*/fpgaCompDir/bitstream.bit(N)) + bits=("${root}"/bebop/build/"${{ matrix.design }}"-*/fpgaCompDir/bitstream.bit(N)) if (( ${#bits} != 1 )); then - echo "ERROR: expected exactly 1 bitstream.bit for ${{ matrix.chip }}, got ${#bits}" >&2 + echo "ERROR: expected exactly 1 bitstream.bit for ${{ matrix.design }}, got ${#bits}" >&2 printf '%s\n' "${bits[@]}" >&2 exit 1 fi - nix develop -c bbdev bebop-p2e --runworkload "--chip ${{ matrix.chip }} --bitstream ${bits[1]} --model ${{ matrix.model }}" + nix develop -c bbdev bebop-p2e --runworkload "--design ${{ matrix.design }} --device --bitstream ${bits[1]} --model ${{ matrix.model }}" - name: Exit repo lock if: always() diff --git a/.gitignore b/.gitignore index 96d7f8f59..e615cab8e 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,8 @@ skills-lock.json .mcp.json .tmp *.egg-info/ + +# VCS run artifacts +ucli.key +cm.log +tr_db.log diff --git a/.gitmodules b/.gitmodules index 069510370..1557a2a7f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,28 +4,12 @@ [submodule "bebop"] path = bebop url = https://github.com/DangoSys/bebop -[submodule "bb-tests/workloads/lib/kernel"] - path = bb-tests/workloads/lib/kernel - url = https://github.com/DangoSys/bbkernel.git -[submodule "bb-tests/thirdparty/busybox"] - path = bb-tests/thirdparty/busybox - url = https://github.com/mirror/busybox.git -[submodule "bb-tests/thirdparty/linux"] - path = bb-tests/thirdparty/linux - url = https://github.com/firesim/linux.git -[submodule "bb-tests/thirdparty/opensbi"] - path = bb-tests/thirdparty/opensbi - url = https://github.com/riscv-software-src/opensbi.git - branch = master [submodule "docs"] path = docs url = https://github.com/DangoSys/document.git [submodule "verify"] path = verify url = https://github.com/DangoSys/verify -[submodule "arch/thirdparty/rocket-chip"] - path = arch/thirdparty/rocket-chip - url = https://github.com/SEU-ACAL/rocket-chip.git [submodule "arch/thirdparty/berkeley-hardfloat"] path = arch/thirdparty/berkeley-hardfloat url = https://github.com/SEU-ACAL/berkeley-hardfloat.git @@ -38,9 +22,6 @@ [submodule ".agents/mcps"] path = .agents/mcps url = https://github.com/DangoSys/mcps.git -[submodule "arch/src/main/scala/framework/system/core/seed"] - path = arch/src/main/scala/framework/system/core/seed - url = https://github.com/DangoSys/seed.git [submodule "stack"] path = stack url = https://github.com/buddy-compiler/buddy-examples @@ -51,3 +32,37 @@ [submodule "thirdparty/soc-framework"] path = thirdparty/soc-framework url = https://github.com/DangoSys/soc-framework.git +[submodule "arch/thirdparty/redmule"] + path = arch/thirdparty/redmule + url = https://github.com/pulp-platform/redmule.git +[submodule "arch/thirdparty/cvfpu"] + path = arch/thirdparty/cvfpu + url = https://github.com/pulp-platform/cvfpu.git +[submodule "arch/thirdparty/common_cells"] + path = arch/thirdparty/common_cells + url = https://github.com/pulp-platform/common_cells.git +[submodule "arch/thirdparty/hci"] + path = arch/thirdparty/hci + url = https://github.com/pulp-platform/hci.git +[submodule "arch/thirdparty/hwpe-stream"] + path = arch/thirdparty/hwpe-stream + url = https://github.com/pulp-platform/hwpe-stream.git +[submodule "arch/thirdparty/hwpe-ctrl"] + path = arch/thirdparty/hwpe-ctrl + url = https://github.com/pulp-platform/hwpe-ctrl.git +[submodule "arch/thirdparty/tech_cells_generic"] + path = arch/thirdparty/tech_cells_generic + url = https://github.com/pulp-platform/tech_cells_generic.git +[submodule "examples/components/core/rocket/thirdparty/rocket-chip"] + path = examples/components/cores/rocket/thirdparty/rocket-chip + url = https://github.com/SEU-ACAL/rocket-chip.git + +[submodule "examples/components/core/xiangshan/thirdparty/xiangshan"] + path = examples/components/cores/xiangshan/thirdparty/xiangshan + url = https://github.com/OpenXiangShan/XiangShan.git +[submodule "examples/components/cores/boom/thirdparty/riscv-boom"] + path = examples/components/cores/boom/thirdparty/riscv-boom + url = https://github.com/riscv-boom/riscv-boom.git +[submodule "examples/components/cores/boom/thirdparty/rocket-chip"] + path = examples/components/cores/boom/thirdparty/rocket-chip + url = https://github.com/chipsalliance/rocket-chip.git diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8dc263385..49cc840dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: hooks: - id: flake8 language: system - exclude: (^|/)bbdev/api/steps/config/scripts/chip_pb2\.py$ + exclude: (^|/)bbdev/api/steps/config/scripts/[^/]+_pb2\.py$ args: - --max-line-length=120 # Adjust as per your style guide - --ignore=F821,F403,F405,F401,W503,E203,E402,E401,W605,E712,E711,F841 diff --git a/README.md b/README.md index 931972e49..34f4395e7 100644 --- a/README.md +++ b/README.md @@ -52,13 +52,13 @@ nix develop Run Verilator simulation test to verify installation: ```bash -bbdev verilator --run '--jobs 16 --chip toy --binary toy-toy-ctest-relu_i32_16x16_test-baremetal --batch' +bbdev verilator --run '--jobs 16 --design toy --device --binary toy-toy-ctest-relu_i32_16x16_test-baremetal --batch' ``` **4. Try faster simulation using bebop** ```bash -bbdev bebop-verilator --run '--chip toy --binary toy-toy-ctest-relu_i32_16x16_test-baremetal --itrace --mtrace --pmctrace --ctrace' +bbdev bebop-verilator --run '--design toy --device --binary toy-toy-ctest-relu_i32_16x16_test-baremetal --itrace --mtrace --pmctrace --ctrace' ``` ## Tutorial diff --git a/arch/.gitignore b/arch/.gitignore index 4dfc44f7a..e0edfb216 100644 --- a/arch/.gitignore +++ b/arch/.gitignore @@ -5,6 +5,7 @@ project/project/ .metals/ .bloop/ out/ +output/ build/ *.vcd @@ -16,5 +17,5 @@ build/ log/ *.sv -!src/main/scala/framework/**/src/main/resources/*.sv +!src/main/scala/framework/**/src/main/resources/**/*.sv gemmini_params.h diff --git a/arch/build.sc b/arch/build.sc index 7fc755c9d..22746c12c 100644 --- a/arch/build.sc +++ b/arch/build.sc @@ -1,458 +1,12 @@ import mill._ -import mill.define.Sources -import mill.modules.Util -import mill.scalalib.TestModule.ScalaTest -import scalalib._ -import mill.javalib.JavaModule -import mill.bsp._ - -object protoJava extends JavaModule { - def protoDir = T { - os.pwd / os.up / "bbdev" / "api" / "steps" / "config" / "scripts" / "proto" - } - - def protoSources = T.sources { - Seq(PathRef(protoDir() / "chip.proto")) - } - - def generatedSources = T { - val dir = protoDir() - val proto = protoSources().head.path - os.proc("protoc", s"-I$dir", s"--java_out=${T.dest}", proto).call() - Seq(PathRef(T.dest)) - } - - override def zincIncrementalCompilation = T { false } - - override def ivyDeps = Agg(ivy"com.google.protobuf:protobuf-java:4.35.1") -} - -trait FrameworkModule extends SbtModule { - def moduleRoot: os.Path - - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -val frameworkRoot = os.pwd / "src" / "main" / "scala" / "framework" - -object queue extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "queue" -} - -object axis extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "axis" - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") -} - -object axi4 extends FrameworkModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = frameworkRoot / "mem-core" / "axi4" - override def mainClass = Some("memcore.bus.axi4.Emit") -} - -object ddr extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "ddr" - override def moduleDeps = Seq(chi, axi4) - override def mainClass = Some("memcore.memory.ddr.Emit") -} - -object chi extends FrameworkModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = frameworkRoot / "mem-core" / "chi" - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") -} - -object cpu_mem extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "cpu_mem" - override def moduleDeps = Seq(chi, mmu) -} - -object mmu extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "mmu" - override def moduleDeps = Seq(chi) -} - -object fetch extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "fetch" -} - -object preflight extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "preflight" - override def moduleDeps = Seq(chi, mmu) - override def mainClass = Some("memcore.memory.preflight.Emit") -} - -object interlock extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "interlock" - override def mainClass = Some("memcore.memory.interlock.Emit") -} - -object bank extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "bank" - override def moduleDeps = Seq(axis) - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") -} - -object cache extends FrameworkModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = frameworkRoot / "mem-core" / "cache" - override def sources = T.sources { super.sources() ++ Seq(PathRef(moduleRoot / "configs")) } - override def ivyDeps = super.ivyDeps() ++ Agg(ivy"tech.sparse::toml-scala:0.2.2") - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") -} - -object mesh_shm extends FrameworkModule { - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") - override def moduleRoot = frameworkRoot / "mem-core" / "mesh_shm" - override def moduleDeps = Seq(bank) -} - -object coherence extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "coherence" - override def moduleDeps = Seq(chi, cache) - override def sources = T.sources { super.sources() ++ Seq(PathRef(moduleRoot / "configs")) } - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") -} - -object blink extends FrameworkModule { - override def moduleRoot = frameworkRoot / "balldomain" / "blink" - override def moduleDeps = Seq(rocket_bb, ant, coherence, cpu_mem, ddr, interlock) - override def ivyDeps = super.ivyDeps() ++ Agg(ivy"com.lihaoyi::upickle:3.3.1") - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") - override def sources = T.sources { - Seq( - PathRef(frameworkRoot / "top" / "GlobalConfig.scala"), - PathRef(frameworkRoot / "top" / "configs" / "DesignConfig.scala"), - PathRef(frameworkRoot / "top" / "configs" / "SimParam.scala"), - PathRef(frameworkRoot / "memdomain" / "configs" / "MemDomainParam.scala"), - PathRef(frameworkRoot / "frontend" / "configs" / "FrontendParam.scala"), - PathRef(frameworkRoot / "rvv" / "src" / "main" / "scala" / "configs" / "RvvParam.scala"), - PathRef(frameworkRoot / "balldomain" / "configs" / "BallDomainParam.scala"), - PathRef(frameworkRoot / "system" / "tile" / "configs" / "TileParam.scala"), - PathRef(frameworkRoot / "balldomain" / "blink" / "blink.scala"), - PathRef(frameworkRoot / "balldomain" / "blink" / "bank.scala"), - PathRef(frameworkRoot / "balldomain" / "blink" / "status.scala"), - PathRef(frameworkRoot / "balldomain" / "blink" / "baseball.scala"), - PathRef(frameworkRoot / "balldomain" / "blink" / "SubRobRow.scala"), - PathRef(frameworkRoot / "balldomain" / "blink" / "mmio" / "MmioRead.scala"), - PathRef(frameworkRoot / "balldomain" / "blink" / "mmio" / "MmioWrite.scala"), - PathRef(frameworkRoot / "memdomain" / "backend" / "banks" / "SramIO.scala"), - PathRef(frameworkRoot / "memdomain" / "backend" / "mmio" / "MmioIO.scala"), - PathRef(frameworkRoot / "balldomain" / "rs" / "interfaces.scala"), - PathRef(frameworkRoot / "balldomain" / "decoder" / "BallDecodeCmd.scala"), - PathRef(frameworkRoot / "frontend" / "decoder" / "PostGDCmd.scala"), - PathRef(frameworkRoot / "frontend" / "scoreboard" / "BankAccessInfo.scala") - ) - } -} - -object arith extends FrameworkModule { - override def moduleRoot = frameworkRoot / "arith" -} - -object spm extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "spm" - override def moduleDeps = Seq(bank) -} - -object tss extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "tss" - override def moduleDeps = Seq(spm) - override def mainClass = Some("memcore.memory.tss.Emit") -} - -object tls extends FrameworkModule { - override def moduleRoot = frameworkRoot / "mem-core" / "tls" - override def moduleDeps = Seq(spm) - override def mainClass = Some("memcore.memory.tls.Emit") -} - -object ant extends FrameworkModule { - override def moduleRoot = frameworkRoot / "system" / "core" / "ant" - override def moduleDeps = Seq(tls, tss, arith) - override def mainClass = Some("framework.ant.Emit") -} - -object rvv extends FrameworkModule { - override def moduleRoot = frameworkRoot / "rvv" - override def moduleDeps = Seq(hardfloat, blink, arith) - override def sources = T.sources { - os.walk(moduleRoot / "src" / "main" / "scala") - .filter(path => path.ext == "scala" && path.last != "RvvParam.scala") - .map(PathRef(_)) - } - override def scalacOptions = super.scalacOptions() ++ Seq("-Ymacro-annotations") -} - -object seed extends FrameworkModule { - override def moduleRoot = frameworkRoot / "system" / "core" / "seed" -} - -object rocket_bb extends FrameworkModule { - override def moduleRoot = frameworkRoot / "system" / "core" / "rocket" - override def moduleDeps = Seq(rocketchip) - override def ivyDeps = super.ivyDeps() ++ Agg(ivy"com.lihaoyi::upickle:3.3.1") - override def sources = T.sources { - os.walk(moduleRoot) - .filter(path => path.ext == "scala" && !Set("CoreParameters.scala", "CpuInterfaces.scala", "TileInterrupts.scala", "TileParameters.scala", "RocketCoreParameters.scala", "Utility.scala", "VectorInterfaces.scala", "RocketCpuParam.scala").contains(path.last)) - .map(PathRef(_)) - } -} - -object root_chip extends FrameworkModule { - override def moduleRoot = frameworkRoot / "root" / "root-chip" - override def moduleDeps = Seq(axis, chi, bank) -} - -object root_core extends FrameworkModule { - override def mainClass = Some("hier.core.rocket.Emit") - override def moduleRoot = frameworkRoot / "root" / "root-core" - override def moduleDeps = Seq(rocket_bb, fetch, cpu_mem, chi, coherence, interlock, preflight) - override def sources = T.sources { - super.sources() ++ Seq( - PathRef(os.pwd / os.up / "examples" / "cores" / "rocket" / "arch" / "src" / "main" / "scala" / "cpu"), - PathRef(frameworkRoot / "system" / "core" / "clink" / "base.scala") - ) - } -} - -object root_tile extends FrameworkModule { - override def mainClass = Some("hier.tile.memory.Emit") - override def moduleRoot = frameworkRoot / "root" / "root-tile" - override def moduleDeps = Seq(chi, coherence, cpu_mem, mmu, root_core, ddr) -} - -object buckyball extends SbtModule { m => - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - - override def scalacOptions = Seq( - "-language:reflectiveCalls", - "-deprecation", - "-feature", - "-Xcheckinit", - "-Ymacro-annotations" - ) - - override def moduleDeps = Seq( - rocketchip, - gemmini, - protoJava, - axis, - fetch, - chi, - bank, - mesh_shm, - cache, - coherence, - ddr, - rvv, - seed, - ant, - rocket_bb, - root_chip, - root_core, - root_tile - ) - - override def sources = T.sources { - val examples = os.pwd / os.up / "examples" - def archSrcs(kind: String) = - os.list(examples / kind) - .filter(os.isDir) - .map(_ / "arch" / "src" / "main" / "scala") - .filter(os.exists) - .flatMap(root => os.walk(root).filter(path => path.ext == "scala")) - .map(PathRef(_)) - - def configSrcs(kind: String) = - os.list(examples / kind) - .filter(os.isDir) - .map(_ / "configs") - .filter(os.exists) - .map(PathRef(_)) - - val sharedBlink = blink.sources().map(_.path).toSet - val localSources = os.walk(os.pwd / "src" / "main" / "scala") - .filter(path => path.ext == "scala") - .filterNot(sharedBlink.contains) - .filterNot(path => path.toString.contains("/framework/root/")) - .filterNot(path => path.toString.contains("/verification/")) - .filterNot(path => path.toString.contains("/framework/mem-core/")) - .filterNot(path => path.toString.contains("/framework/rvv/")) - .filterNot(path => path.toString.contains("/framework/system/core/seed/")) - .filterNot(path => path.toString.contains("/framework/system/core/ant/")) - .filterNot(path => path.toString.contains("/framework/arith/")) - .filterNot(path => path.toString.contains("/framework/system/core/rocket/")) - .filterNot(path => path == frameworkRoot / "system" / "core" / "clink" / "base.scala") - .map(PathRef(_)) - val coreSources = archSrcs("cores") - .filterNot(_.path.toString.contains("/cores/rocket/arch/src/main/scala/cpu/")) - localSources ++ archSrcs("balls") ++ coreSources ++ archSrcs("chips") ++ - configSrcs("balls") ++ configSrcs("chips") ++ configSrcs("cores") - } - - override def ivyDeps = Agg( - ivy"org.chipsalliance::chisel:6.7.0", - ivy"org.apache.commons:commons-lang3:3.12.0", - ivy"org.apache.commons:commons-text:1.9", - ivy"org.yaml:snakeyaml:2.0", - ivy"com.lihaoyi::sourcecode:0.3.0", - ivy"com.lihaoyi::upickle:3.3.1", - ivy"tech.sparse::toml-scala:0.2.2", - ivy"com.google.protobuf:protobuf-java:4.35.1" - ) - - override def scalacPluginIvyDeps = Agg( - ivy"org.chipsalliance:::chisel-plugin:6.7.0" - ) - - object test extends ScalaModule with TestModule.ScalaTest { - override def scalaVersion = T("2.13.16") - override def moduleDeps = Seq(m) - - override def ivyDeps = Agg( - ivy"org.scalatest::scalatest::3.2.19" - ) - - } - -} - -// Define cde module - must be compiled first -object cde extends SbtModule { - override def millSourcePath = - os.pwd / "thirdparty" / "rocket-chip" / "dependencies" / "cde" - override def scalaVersion = "2.13.16" - - override def sources = T.sources { - Seq(PathRef(millSourcePath / "cde" / "src")) - } - - override def ivyDeps = Agg( - ivy"org.chipsalliance::chisel:6.7.0" - ) - - override def scalacPluginIvyDeps = Agg( - ivy"org.chipsalliance:::chisel-plugin:6.7.0" - ) - -} - -// Define hardfloat module -object hardfloat extends SbtModule { - override def millSourcePath = - os.pwd / "thirdparty" / "berkeley-hardfloat" - override def scalaVersion = "2.13.16" - - // Override sources to match build.sbt behavior - override def sources = T.sources { - super.sources() ++ Seq( - PathRef(millSourcePath / "hardfloat" / "src" / "main" / "scala") - ) - } - - override def ivyDeps = Agg( - ivy"org.chipsalliance::chisel:6.7.0" - ) - - override def scalacPluginIvyDeps = Agg( - ivy"org.chipsalliance:::chisel-plugin:6.7.0" - ) - -} - -// Define midas_target_utils module -object midas_target_utils extends SbtModule { - override def millSourcePath = - os.pwd / os.up / "thirdparty" / "firesim" / "sim" / "midas" / "targetutils" - override def scalaVersion = "2.13.16" - - override def ivyDeps = Agg( - ivy"org.chipsalliance::chisel:6.7.0" - ) - - override def scalacPluginIvyDeps = Agg( - ivy"org.chipsalliance:::chisel-plugin:6.7.0" - ) - -} - -// Define rocket-chip module with proper dependencies -object rocketchip extends SbtModule { - override def millSourcePath = - os.pwd / "thirdparty" / "rocket-chip" - override def scalaVersion = "2.13.16" - - override def sources = T.sources { - val src = millSourcePath / "src" / "main" / "scala" - val files = Seq( - "rocket/ALU.scala", "rocket/AMOALU.scala", "rocket/BTB.scala", "rocket/Breakpoint.scala", - "rocket/CSR.scala", "rocket/Consts.scala", "rocket/CustomInstructions.scala", "rocket/DebugROB.scala", - "rocket/Decode.scala", "rocket/Events.scala", "rocket/IBuf.scala", "rocket/IDecode.scala", - "rocket/Instructions.scala", "rocket/Instructions32.scala", "rocket/Multiplier.scala", "rocket/PMP.scala", - "rocket/RVC.scala", "rocket/package.scala", "tile/FPU.scala", "tile/CustomCSRs.scala", - "util/Misc.scala", "util/ClockGate.scala", "util/CoreMonitor.scala", "util/Property.scala", - "util/Counters.scala", "util/ECC.scala", "util/Replacement.scala", "util/ShiftReg.scala", - "util/ShiftQueue.scala", "util/HellaQueue.scala", "util/LCG.scala", "util/PlusArg.scala", - "util/MuxLiteral.scala", "util/Arbiters.scala", "util/AsyncResetReg.scala", "util/Timer.scala", - "util/GeneratorUtils.scala", "unittest/UnitTest.scala" - ) - val adapter = frameworkRoot / "system" / "core" / "rocket" - files.map(f => PathRef(src / os.RelPath(f))) ++ Seq( - PathRef(adapter / "CoreParameters.scala"), PathRef(adapter / "CpuInterfaces.scala"), - PathRef(adapter / "TileInterrupts.scala"), PathRef(adapter / "TileParameters.scala"), - PathRef(adapter / "RocketCoreParameters.scala"), PathRef(adapter / "Utility.scala"), - PathRef(adapter / "VectorInterfaces.scala"), PathRef(adapter / "configs" / "RocketCpuParam.scala") - ) - } - - override def moduleDeps = Seq(cde, hardfloat, midas_target_utils) - - override def ivyDeps = Agg( - ivy"org.chipsalliance::chisel:6.7.0", - ivy"com.lihaoyi::upickle:3.3.1", - ivy"com.lihaoyi::mainargs:0.5.0", - ivy"org.json4s::json4s-jackson:4.0.5", - ivy"org.scala-graph::graph-core:1.13.5" - ) - - override def scalacPluginIvyDeps = Agg( - ivy"org.chipsalliance:::chisel-plugin:6.7.0" - ) - -} - - -// Define gemmini module -object gemmini extends SbtModule { - override def millSourcePath = - os.pwd / "thirdparty" / "gemmini" - override def scalaVersion = "2.13.16" - - override def sources = T.sources { - Seq("Arithmetic", "Dataflow", "MeshWithDelays", "Mesh", "Tile", "PE", "Util", "Transposer", "TagQueue", "Pipeline", "Shifter", "SyncMem") - .map(name => PathRef(millSourcePath / "src" / "main" / "scala" / "gemmini" / s"$name.scala")) - } - override def moduleDeps = Seq(hardfloat) - - override def ivyDeps = Agg( - ivy"org.chipsalliance::chisel:6.7.0" - ) - - override def scalacPluginIvyDeps = Agg( - ivy"org.chipsalliance:::chisel-plugin:6.7.0" - ) - -} - - -object memdomain_ack extends FrameworkModule { - override def moduleRoot = frameworkRoot / "memdomain" / "verification" - override def moduleDeps = Seq(buckyball, ddr, preflight) - override def mainClass = Some("framework.memdomain.verification.Emit") +import $file.thirdparty.{build => external} +import $file.src.main.scala.framework.{build => framework_build} +import $file.src.main.scala.sims.{build => simulation} +import $file.^.examples.{build => example_build} + +object architecture extends Module { + def thirdparty = external + def framework = framework_build + def examples = example_build + def sims = simulation } diff --git a/arch/settings.sc b/arch/settings.sc new file mode 100644 index 000000000..1688c733a --- /dev/null +++ b/arch/settings.sc @@ -0,0 +1,34 @@ +import mill._ +import mill.scalalib._ + +val archRoot: os.Path = millSourcePath + +trait FrameworkModule extends SbtModule { + def moduleRoot: os.Path + override def millSourcePath = moduleRoot + override def forkWorkingDir = T(moduleRoot) + override def scalaVersion = "2.13.16" + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0", ivy"com.lihaoyi::upickle:3.3.1") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") +} + +trait IpModule extends FrameworkModule { + def framework: ScalaModule + override def moduleDeps = Seq(framework) + override def sources = T.sources(Seq(PathRef(moduleRoot / "src" / "main" / "scala" / "Emit.scala"))) +} + +object lib extends FrameworkModule { + override def moduleRoot = archRoot / "src" / "main" / "scala" / "framework" / "lib" + + override def sources = T.sources { + Seq( + PathRef(moduleRoot / "queue"), + PathRef(moduleRoot / "arbiter"), + PathRef(moduleRoot / "arith"), + PathRef(moduleRoot / "dpi") + ) + } + +} diff --git a/arch/src/csrc/include/ioe/mmio.h b/arch/src/csrc/include/ioe/mmio.h index 311ae2823..35b252273 100644 --- a/arch/src/csrc/include/ioe/mmio.h +++ b/arch/src/csrc/include/ioe/mmio.h @@ -5,7 +5,7 @@ // These functions are called from the RTL via DPI-C when software writes to // the SCU registers (0x6000_0000 for sim_exit, 0x6002_0000 for UART). // -// The SCU (System Control Unit) is instantiated per-tile inside BBTile and +// The SCU (System Control Unit) is instantiated per-core inside BBCore and // intercepts MMIO writes before they reach the system bus. // // DPI-C functions (implemented in mmio.cc): diff --git a/arch/src/csrc/include/monitor/trace.h b/arch/src/csrc/include/monitor/trace.h index dab0b5f1c..44409f9fa 100644 --- a/arch/src/csrc/include/monitor/trace.h +++ b/arch/src/csrc/include/monitor/trace.h @@ -14,25 +14,35 @@ void dpi_bdb_set_clk(unsigned long long c); // DPI-C function for instruction trace (itrace) // Called from GlobalROB when instructions are allocated/issued/completed void dpi_itrace(unsigned int is_issue, // 2 = alloc, 1 = issue, 0 = complete - unsigned int hart_id_lo, unsigned int hart_id_hi, + unsigned int thread_id_lo, unsigned int thread_id_hi, unsigned int rob_id, unsigned int inst_id_lo, unsigned int inst_id_hi, unsigned int domain_id, unsigned int funct, unsigned int pc_lo, unsigned int pc_hi, unsigned int rs1_lo, unsigned int rs1_hi, unsigned int rs2_lo, unsigned int rs2_hi); +// Passive Ant retirement observation from the thread trace collector. +void dpi_ant_trace(unsigned int thread_lo, unsigned int thread_hi, + unsigned int pc_lo, unsigned int pc_hi, + unsigned int instruction, unsigned int rd, + unsigned int data_lo, unsigned int data_hi); + // DPI-C function for memory trace (mtrace) // Called from MemBackend when read/write requests are made -void dpi_mtrace(unsigned char is_write, // 1 = write, 0 = read - unsigned char is_shared, unsigned int channel, - unsigned long long hart_id, unsigned int vbank_id, - unsigned int pbank_id, unsigned int group_id, unsigned int addr, - unsigned long long data_lo, unsigned long long data_hi); +void dpi_mtrace(unsigned int is_write, unsigned int is_shared, + unsigned int channel, unsigned int thread_id_lo, + unsigned int thread_id_hi, unsigned int rob_id, + unsigned int inst_id_lo, unsigned int inst_id_hi, + unsigned int vbank_id, unsigned int pbank_id, + unsigned int group_id, unsigned int addr, + unsigned int write_mask, unsigned int data_lo_lo, + unsigned int data_lo_hi, unsigned int data_hi_lo, + unsigned int data_hi_hi); // DPI-C function for Ball PMC trace (pmctrace) // Called from BallCyclePMC when a Ball completes a task void dpi_pmctrace(unsigned int ball_id, unsigned int rob_id, - unsigned long long elapsed); + unsigned int elapsed_lo, unsigned int elapsed_hi); // DPI-C function for memory PMC trace (pmctrace) // Called from MemCyclePMC when a load/store completes @@ -45,6 +55,14 @@ void dpi_ctrace(unsigned char subcmd, // 0=START, 1=STOP, 2=READ unsigned int ctr_id, unsigned long long tag, unsigned long long elapsed, unsigned long long cycle); +// DPI-C function for bank hash trace (banktrace) +// Called when a Ball instruction writes a bank +void dpi_btrace(unsigned int inst_id_lo, unsigned int inst_id_hi, + unsigned int thread_id_lo, unsigned int thread_id_hi, + unsigned int owner_thread_id_lo, + unsigned int owner_thread_id_hi, unsigned int w0_vbank, + unsigned int w0_hash); + #ifdef __cplusplus } #endif diff --git a/arch/src/csrc/src/monitor/ioe/BBSimDRAM.cc b/arch/src/csrc/src/monitor/ioe/BBSimDRAM.cc index 1a450339d..f75182cee 100644 --- a/arch/src/csrc/src/monitor/ioe/BBSimDRAM.cc +++ b/arch/src/csrc/src/monitor/ioe/BBSimDRAM.cc @@ -97,10 +97,12 @@ static void load_elf_to_mem(const char *path, uint8_t *data, uint64_t mem_base, fflush(stdout); } -extern "C" void * -bbsim_memory_init(int chip_id, long long int mem_size, long long int word_size, - long long int line_size, long long int id_bits, - long long int clock_hz, long long int mem_base) { +extern "C" void *bbsim_memory_init(int device_id, long long int mem_size, + long long int word_size, + long long int line_size, + long long int id_bits, + long long int clock_hz, + long long int mem_base) { mm_t *mm; s_vpi_vlog_info info; @@ -125,11 +127,11 @@ bbsim_memory_init(int chip_id, long long int mem_size, long long int word_size, abort(); } - while (chip_id >= (int)mem_data.size()) + while (device_id >= (int)mem_data.size()) mem_data.push_back(std::map()); - if (mem_data[chip_id].find(mem_base) != mem_data[chip_id].end()) { - assert(mem_data[chip_id][mem_base].size == (size_t)mem_size); + if (mem_data[device_id].find(mem_base) != mem_data[device_id].end()) { + assert(mem_data[device_id][mem_base].size == (size_t)mem_size); } else { uint8_t *data = (uint8_t *)mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); @@ -143,13 +145,13 @@ bbsim_memory_init(int chip_id, long long int mem_size, long long int word_size, load_elf_to_mem(elf_file.c_str(), data, (uint64_t)mem_base, (uint64_t)mem_size); - mem_data[chip_id][mem_base] = {data, (size_t)mem_size}; + mem_data[device_id][mem_base] = {data, (size_t)mem_size}; } // External AXI memory is always modeled by DRAMSim3. Internal SRAMs are // modeled separately by the generated Verilog memory models. mm = (mm_t *)(new mm_dramsim3_t( - mem_base, mem_size, word_size, line_size, mem_data[chip_id][mem_base], + mem_base, mem_size, word_size, line_size, mem_data[device_id][mem_base], memory_ini, local_ini_dir, dramsim_outdir, 1 << id_bits, clock_hz)); channels.push_back(mm); diff --git a/arch/src/csrc/src/monitor/ioe/mm_dramsim3.cc b/arch/src/csrc/src/monitor/ioe/mm_dramsim3.cc index 06dccedd0..ce31e1f84 100644 --- a/arch/src/csrc/src/monitor/ioe/mm_dramsim3.cc +++ b/arch/src/csrc/src/monitor/ioe/mm_dramsim3.cc @@ -99,12 +99,9 @@ void mm_dramsim3_t::tick(bool reset, bool w_last, bool r_ready, bool b_ready) { - ar_ready_cache = !reset && mem->WillAcceptTransaction(ar_addr, false); - aw_ready_cache = - !reset && !store_inflight && mem->WillAcceptTransaction(aw_addr, true); - - bool ar_fire = ar_valid && ar_ready_cache; - bool aw_fire = aw_valid && aw_ready_cache; + // Consume only the READY values presented to RTL before this clock edge. + bool ar_fire = !reset && ar_valid && ar_ready(); + bool aw_fire = !reset && aw_valid && aw_ready(); bool w_fire = !reset && w_valid && w_ready(); bool r_fire = !reset && r_valid() && r_ready; bool b_fire = !reset && b_valid() && b_ready; @@ -154,6 +151,10 @@ void mm_dramsim3_t::tick(bool reset, mem->ClockTick(); cycle++; + ar_ready_cache = !reset && mem->WillAcceptTransaction(ar_addr, false); + aw_ready_cache = + !reset && !store_inflight && mem->WillAcceptTransaction(aw_addr, true); + if (reset) { while (!bresp.empty()) bresp.pop(); diff --git a/arch/src/csrc/src/monitor/trace/ctrace.cc b/arch/src/csrc/src/monitor/trace/ctrace.cc index 039411c8d..f0deac7ae 100644 --- a/arch/src/csrc/src/monitor/trace/ctrace.cc +++ b/arch/src/csrc/src/monitor/trace/ctrace.cc @@ -162,3 +162,27 @@ extern "C" void dpi_backdoor_put_write_done(unsigned int bank_id, fflush(ctrace_fp); } } + +// RTL calls this when a Ball instruction writes a bank: logs the bank hash +extern "C" void dpi_btrace(unsigned int inst_id_lo, unsigned int inst_id_hi, + unsigned int thread_id_lo, unsigned int thread_id_hi, + unsigned int owner_thread_id_lo, + unsigned int owner_thread_id_hi, + unsigned int w0_vbank, unsigned int w0_hash) { + if (!bdb_trace_on(BDB_TR_BANKTRACE)) { + return; + } + init_ctrace(); + if (ctrace_fp) { + fprintf(ctrace_fp, + "{\"type\":\"banktrace\",\"clk\":%llu,\"event\":\"btrace\"," + "\"inst_id\":%llu,\"thread_id\":%llu,\"owner_thread_id\":%llu," + "\"vbank\":%u,\"hash\":\"0x%08X\"}\n", + (unsigned long long)bdb_rtl_clk, + ((unsigned long long)inst_id_hi << 32) | inst_id_lo, + ((unsigned long long)thread_id_hi << 32) | thread_id_lo, + ((unsigned long long)owner_thread_id_hi << 32) | owner_thread_id_lo, + w0_vbank, w0_hash); + fflush(ctrace_fp); + } +} diff --git a/arch/src/csrc/src/monitor/trace/itrace.cc b/arch/src/csrc/src/monitor/trace/itrace.cc index 34f78a3b3..23d7fbaff 100644 --- a/arch/src/csrc/src/monitor/trace/itrace.cc +++ b/arch/src/csrc/src/monitor/trace/itrace.cc @@ -3,6 +3,7 @@ #include "utils/debug.h" #include #include +#include // Global log file pointer (shared with monitor.cc) extern const char *log_path; @@ -29,7 +30,7 @@ static void u64_hex(char *buf, size_t n, unsigned long long v) { // Called when an instruction is allocated/issued/completed in GlobalROB extern "C" void dpi_itrace(unsigned int is_issue, // 2 = alloc, 1 = issue, 0 = complete - unsigned int hart_id_lo, unsigned int hart_id_hi, + unsigned int thread_id_lo, unsigned int thread_id_hi, unsigned int rob_id, unsigned int inst_id_lo, unsigned int inst_id_hi, unsigned int domain_id, unsigned int funct, unsigned int pc_lo, unsigned int pc_hi, unsigned int rs1_lo, @@ -38,10 +39,11 @@ dpi_itrace(unsigned int is_issue, // 2 = alloc, 1 = issue, 0 = complete return; } init_itrace(); + const char *scope = svGetNameFromScope(svGetScope()); if (itrace_fp) { - unsigned long long hart_id = - ((unsigned long long)hart_id_hi << 32) | hart_id_lo; + unsigned long long thread_id = + ((unsigned long long)thread_id_hi << 32) | thread_id_lo; unsigned long long inst_id = ((unsigned long long)inst_id_hi << 32) | inst_id_lo; unsigned long long pc = ((unsigned long long)pc_hi << 32) | pc_lo; @@ -55,31 +57,58 @@ dpi_itrace(unsigned int is_issue, // 2 = alloc, 1 = issue, 0 = complete u64_hex(rs2_hex, sizeof(rs2_hex), rs2); if (is_issue == 2) { fprintf(itrace_fp, - "{\"type\":\"itrace\",\"clk\":%llu,\"event\":\"alloc\",\"hart_" + "{\"type\":\"itrace\",\"clk\":%llu,\"scope\":\"%s\",\"event\":" + "\"alloc\",\"thread_" "id\":%llu," "\"rob_id\":%u,\"inst_id\":%llu,\"domain_id\":%u,\"funct\":\"0x%" "02x\"," "\"pc\":\"%s\",\"rs1\":\"%s\",\"rs2\":\"%s\"}\n", - (unsigned long long)bdb_rtl_clk, hart_id, rob_id, inst_id, - domain_id, funct, pc_hex, rs1_hex, rs2_hex); + (unsigned long long)bdb_rtl_clk, scope, thread_id, rob_id, + inst_id, domain_id, funct, pc_hex, rs1_hex, rs2_hex); } else if (is_issue == 1) { fprintf(itrace_fp, - "{\"type\":\"itrace\",\"clk\":%llu,\"event\":\"issue\",\"hart_" + "{\"type\":\"itrace\",\"clk\":%llu,\"scope\":\"%s\",\"event\":" + "\"issue\",\"thread_" "id\":%llu," "\"rob_id\":%u,\"inst_id\":%llu,\"domain_id\":%u,\"funct\":\"0x%" "02x\"," "\"pc\":\"%s\",\"rs1\":\"%s\",\"rs2\":\"%s\"}\n", - (unsigned long long)bdb_rtl_clk, hart_id, rob_id, inst_id, - domain_id, funct, pc_hex, rs1_hex, rs2_hex); + (unsigned long long)bdb_rtl_clk, scope, thread_id, rob_id, + inst_id, domain_id, funct, pc_hex, rs1_hex, rs2_hex); } else { fprintf( itrace_fp, - "{\"type\":\"itrace\",\"clk\":%llu,\"event\":\"complete\"," - "\"hart_id\":%llu,\"rob_id\":%u,\"inst_id\":%llu,\"domain_id\":%u," + "{\"type\":\"itrace\",\"clk\":%llu,\"scope\":\"%s\",\"event\":" + "\"complete\"," + "\"thread_id\":%llu,\"rob_id\":%u,\"inst_id\":%llu,\"domain_id\":%u," "\"funct\":\"0x%02x\",\"pc\":\"%s\"}\n", - (unsigned long long)bdb_rtl_clk, hart_id, rob_id, inst_id, domain_id, - funct, pc_hex); + (unsigned long long)bdb_rtl_clk, scope, thread_id, rob_id, inst_id, + domain_id, funct, pc_hex); } fflush(itrace_fp); } } + +extern "C" void dpi_ant_trace(unsigned int thread_lo, unsigned int thread_hi, + unsigned int pc_lo, unsigned int pc_hi, + unsigned int instruction, unsigned int rd, + unsigned int data_lo, unsigned int data_hi) { + if (!bdb_trace_on(BDB_TR_ITRACE)) + return; + init_itrace(); + const char *scope = svGetNameFromScope(svGetScope()); + if (!itrace_fp) + return; + const auto thread = + (static_cast(thread_hi) << 32) | thread_lo; + const auto pc = (static_cast(pc_hi) << 32) | pc_lo; + const auto data = (static_cast(data_hi) << 32) | data_lo; + fprintf(itrace_fp, + "{\"type\":\"ant_trace\",\"clk\":%llu,\"scope\":\"%s\",\"thread_id\":" + "%llu," + "\"pc\":\"0x%016llx\",\"instruction\":\"0x%08x\"," + "\"rd\":%u,\"data\":\"0x%016llx\"}\n", + static_cast(bdb_rtl_clk), scope, thread, pc, + instruction, rd, data); + fflush(itrace_fp); +} diff --git a/arch/src/csrc/src/monitor/trace/mtrace.cc b/arch/src/csrc/src/monitor/trace/mtrace.cc index 319d3bb5d..63c6bc0b5 100644 --- a/arch/src/csrc/src/monitor/trace/mtrace.cc +++ b/arch/src/csrc/src/monitor/trace/mtrace.cc @@ -3,6 +3,7 @@ #include "utils/debug.h" #include #include +#include // Global log file pointer (shared with monitor.cc) extern const char *log_path; @@ -28,37 +29,53 @@ static void u128_hex(char *buf, size_t n, unsigned long long hi, // DPI-C function for memory trace (mtrace) // Called when MemBackend performs read/write operations -extern "C" void dpi_mtrace(unsigned char is_write, // 1 = write, 0 = read - unsigned char is_shared, unsigned int channel, - unsigned long long hart_id, unsigned int vbank_id, - unsigned int pbank_id, unsigned int group_id, - unsigned int addr, unsigned long long data_lo, - unsigned long long data_hi) { +extern "C" void dpi_mtrace(unsigned int is_write, unsigned int is_shared, + unsigned int channel, unsigned int thread_id_lo, + unsigned int thread_id_hi, unsigned int rob_id, + unsigned int inst_id_lo, unsigned int inst_id_hi, + unsigned int vbank_id, unsigned int pbank_id, + unsigned int group_id, unsigned int addr, + unsigned int write_mask, unsigned int data_lo_lo, + unsigned int data_lo_hi, unsigned int data_hi_lo, + unsigned int data_hi_hi) { if (!bdb_trace_on(BDB_TR_MTRACE)) { return; } init_mtrace(); + const char *scope = svGetNameFromScope(svGetScope()); if (mtrace_fp) { + const unsigned long long thread_id = + ((unsigned long long)thread_id_hi << 32) | thread_id_lo; + const unsigned long long inst_id = + ((unsigned long long)inst_id_hi << 32) | inst_id_lo; + const unsigned long long data_lo = + ((unsigned long long)data_lo_hi << 32) | data_lo_lo; + const unsigned long long data_hi = + ((unsigned long long)data_hi_hi << 32) | data_hi_lo; char data_hex[35]; if (is_write) { u128_hex(data_hex, sizeof(data_hex), data_hi, data_lo); - fprintf( - mtrace_fp, - "{\"type\":\"mtrace\",\"clk\":%llu,\"event\":\"write\"," - "\"channel\":%u," - "\"hart_id\":%llu,\"is_shared\":%u,\"vbank_id\":%u,\"pbank_id\":%u," - "\"group_id\":%u,\"addr\":\"0x%08x\",\"data\":\"%s\"}\n", - (unsigned long long)bdb_rtl_clk, channel, hart_id, is_shared, - vbank_id, pbank_id, group_id, addr, data_hex); + fprintf(mtrace_fp, + "{\"type\":\"mtrace\",\"clk\":%llu,\"scope\":\"%s\",\"event\":" + "\"write\"," + "\"channel\":%u," + "\"thread_id\":%llu,\"rob_id\":%u,\"inst_id\":%llu,\"is_shared\":" + "%u,\"vbank_id\":%u,\"pbank_id\":%u," + "\"group_id\":%u,\"addr\":\"0x%08x\",\"write_mask\":%u,\"data\":" + "\"%s\"}\n", + (unsigned long long)bdb_rtl_clk, scope, channel, thread_id, + rob_id, inst_id, is_shared, vbank_id, pbank_id, group_id, addr, + write_mask, data_hex); } else { - fprintf( - mtrace_fp, - "{\"type\":\"mtrace\",\"clk\":%llu,\"event\":\"read\",\"channel\":%u," - "\"hart_id\":%llu,\"is_shared\":%u,\"vbank_id\":%u,\"pbank_id\":%u," - "\"group_id\":%u,\"addr\":\"0x%08x\"}\n", - (unsigned long long)bdb_rtl_clk, channel, hart_id, is_shared, - vbank_id, pbank_id, group_id, addr); + fprintf(mtrace_fp, + "{\"type\":\"mtrace\",\"clk\":%llu,\"scope\":\"%s\",\"event\":" + "\"read\",\"channel\":%u," + "\"thread_id\":%llu,\"rob_id\":%u,\"inst_id\":%llu,\"is_shared\":" + "%u,\"vbank_id\":%u,\"pbank_id\":%u," + "\"group_id\":%u,\"addr\":\"0x%08x\"}\n", + (unsigned long long)bdb_rtl_clk, scope, channel, thread_id, + rob_id, inst_id, is_shared, vbank_id, pbank_id, group_id, addr); } fflush(mtrace_fp); } diff --git a/arch/src/csrc/src/monitor/trace/pmctrace.cc b/arch/src/csrc/src/monitor/trace/pmctrace.cc index 62fee5011..a3c973d20 100644 --- a/arch/src/csrc/src/monitor/trace/pmctrace.cc +++ b/arch/src/csrc/src/monitor/trace/pmctrace.cc @@ -21,10 +21,12 @@ static void init_pmctrace() { // DPI-C function for Ball PMC trace // Called when a Ball completes a task, reports elapsed cycles extern "C" void dpi_pmctrace(unsigned int ball_id, unsigned int rob_id, - unsigned long long elapsed) { + unsigned int elapsed_lo, unsigned int elapsed_hi) { if (!bdb_trace_on(BDB_TR_PMCTRACE)) { return; } + const unsigned long long elapsed = + ((unsigned long long)elapsed_hi << 32) | elapsed_lo; init_pmctrace(); if (pmctrace_fp) { diff --git a/arch/src/main/resources/bootrom/bootrom.S b/arch/src/main/resources/bootrom/bootrom.S index 453601beb..ddbb5cd8a 100644 --- a/arch/src/main/resources/bootrom/bootrom.S +++ b/arch/src/main/resources/bootrom/bootrom.S @@ -1,6 +1,6 @@ # Buckyball BootROM: validate mhartid and jump to the DRAM entry in M-mode. # The SoC fills the two dwords at _config (ROM offsets 0x8 and 0x10) when it -# builds the ROM, so this image is the same for every chip. OpenSBI takes the +# builds the ROM, so this image is the same for every design. OpenSBI takes the # DTB embedded in its fw_payload. .section .text diff --git a/arch/src/main/scala/configs/GlobalConfig.scala b/arch/src/main/scala/configs/GlobalConfig.scala new file mode 100644 index 000000000..b0ec9081c --- /dev/null +++ b/arch/src/main/scala/configs/GlobalConfig.scala @@ -0,0 +1,28 @@ +package configs + +import buckyball.config.Design +import framework.core.configs.CoreParam +import scala.jdk.CollectionConverters._ + +import chisel3.experimental.SerializableModuleParameter +import framework.device.configs.DeviceParam +import sims.configs.SimParam + +case class GlobalConfig( + device: DeviceParam, + sim: SimParam) + extends SerializableModuleParameter + +object GlobalConfig { + + def fromProto(design: Design, cores: Seq[CoreParam]): GlobalConfig = { + require(design.hasDevice && design.hasSims, "Design device and simulation configuration are required") + GlobalConfig(DeviceParam.fromProto(design.getDevice, cores), SimParam.fromProto(design.getSims)) + } + + def fromProto(design: Design): GlobalConfig = { + require(design.getThreadsCount == 0, "Thread implementations must supply their core parameters") + fromProto(design, design.getDevice.getCoresList.asScala.map(p => CoreParam.fromProto(p, Seq.empty)).toSeq) + } + +} diff --git a/arch/src/main/scala/framework/acc/balldomain/build.sc b/arch/src/main/scala/framework/acc/balldomain/build.sc new file mode 100644 index 000000000..87ff3a23b --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/build.sc @@ -0,0 +1,33 @@ +import mill._ +import $file.src.main.scala.ball.{build => ball_build} +import $file.src.main.scala.bbus.{build => bbus_build} +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/BallDomain.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/mmio_bank/MmioBank.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/sub_rob/SubROB.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/sub_rob/Expansion.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/sub_rob/configs/Params.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/BallDomainParam.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/decoder/DISA.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/decoder/Decoder.scala")) + ) ++ + ball_build.sources() ++ + bbus_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + ball_build.emitSources() ++ + bbus_build.emitSources() +) diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/BallDomain.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/BallDomain.scala new file mode 100644 index 000000000..fd945f6e5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/BallDomain.scala @@ -0,0 +1,198 @@ +package framework.acc.balldomain + +import framework.lib.arbiter.{Arbiter} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.acc.balldomain.ball.blink.BallResponse +import framework.acc.balldomain.bbus.{BBus, BallFactory, BallRequest} +import framework.acc.balldomain.decoder.Decoder +import framework.acc.balldomain.sub_rob.{SubROB, SubRobRow} +import framework.acc.balldomain.mmio_bank.{MetadataWrite, MmioBank} +import framework.acc.frontend.scheduler.{Complete, Issue, MemoryIssue} +import framework.acc.memdomain.{BankLayout, BankLayoutRequest, BankPort, Response => MemoryResponse} + +/** GP kernel Ball calls; the GP domain resolves its launch groups before issuing. */ +class KernelBallPort(f: FrontendParam, m: MemDomainParam) extends Bundle { + val request = Decoupled(new BallRequest(f, m)) + val response = Flipped(Decoupled(new BallResponse(f))) +} + +/** BallDomain owns child identities; MemDomain receives ordinary tagged memory operations. */ +class DomainMemory(f: FrontendParam, m: MemDomainParam, p: PmpParams) extends Bundle { + val request = Decoupled(new MemoryIssue(f, m, p)) + val response = Flipped(Decoupled(new MemoryResponse(f))) + val metadata = new MetadataWrite(m) +} + +@instantiable +class BallDomain( + fp: FrontendParam, + mp: MemDomainParam, + gp: GpDomainParam, + bp: BallDomainParam, + pp: PmpParams, + factories: Map[String, BallFactory]) + extends Module { + + @public + val io = IO(new Bundle { + val issue = Flipped(Decoupled(new MemoryIssue(fp, mp, pp))) + val complete = Decoupled(new MemoryResponse(fp)) + val memory = new DomainMemory(fp, mp, pp) + val banks = Vec(mp.bankRequestPorts, new BankPort(fp, mp)) + val layoutRequest = Decoupled(new BankLayoutRequest(fp)) + val layoutResponse = Flipped(Decoupled(new BankLayout(mp))) + val kernel = Option.when(gp.enable)(Flipped(new KernelBallPort(fp, mp))) + val traceStarted = Output(Vec(bp.balls.size, Valid(UInt(log2Ceil(fp.rob_entries).W)))) + val traceCompleted = Output(Vec(bp.balls.size, Valid(UInt(log2Ceil(fp.rob_entries).W)))) + }) + + val bbus: Instance[BBus] = Instantiate(new BBus(fp, mp, gp, bp, factories)) + val expander = bbus.expansion + if (expander.nonEmpty) require( + mp.mmioEnable && !fp.bank_alias_enable, + "Expanded instructions own architectural scratch banks until their parent completes" + ) + val decoder: Instance[Decoder] = Instantiate(new Decoder(fp, mp, bp)) + bbus.io.cmdReq <> decoder.io.command + + // Frontend and SubROB work enter as an Issue; the Ball sees only its BLink command. + val operation = Wire(new Issue(fp, mp)) + val access = operation.renamedBankAccess + val request = decoder.io.issue.bits + request.target := 0.U + request.instId := operation.instId + request.enabled(0) := access.rd_bank_0_valid + request.enabled(1) := access.rd_bank_1_valid + request.enabled(2) := access.wr_bank_valid + request.cmd.rob_id := operation.rob_id + request.cmd.funct := operation.cmd.cmd.raw_inst(31, 25) + request.cmd.rs2 := operation.cmd.cmd.rs2Data + request.cmd.iter := operation.cmd.iter + request.cmd.op1_bank := access.rd_bank_0_id + request.cmd.op2_bank := access.rd_bank_1_id + request.cmd.wr_bank := access.wr_bank_id + request.cmd.op1_group := 0.U + request.cmd.op2_group := 0.U + request.cmd.wr_group := 0.U + request.cmd.op1_col := operation.cmd.op1_col + request.cmd.op2_col := operation.cmd.op2_col + request.cmd.wr_col := operation.cmd.wr_col + request.cmd.wr_read_bank := access.rd_wr_id + request.cmd.wr_arch_bank := operation.cmd.architecturalDestination + io.banks <> bbus.io.banks + io.layoutRequest <> bbus.io.layoutRequest + bbus.io.layoutResponse <> io.layoutResponse + io.traceStarted := bbus.io.traceStarted + io.traceCompleted := bbus.io.traceCompleted + if (gp.enable) { + val kernelDecoder: Instance[Decoder] = Instantiate(new Decoder(fp, mp, bp)) + kernelDecoder.io.issue <> io.kernel.get.request + bbus.io.kernelReq.get <> kernelDecoder.io.command + io.kernel.get.response <> bbus.io.kernelResp.get + } + if (mp.mmioEnable) { + val reads = bp.balls.map(_.mmioReadBW).sum + val writes = bp.balls.map(_.mmioWriteBW).sum + val mmio: Instance[MmioBank] = Instantiate(new MmioBank(mp, reads + expander.size, writes + expander.size)) + mmio.io.upload <> io.memory.metadata + for (i <- 0 until reads) mmio.io.reads(i) <> bbus.io.mmioRead(i) + for (i <- 0 until writes) mmio.io.writes(i) <> bbus.io.mmioWrite(i) + expander.foreach { e => mmio.io.reads(reads) <> e.read; mmio.io.writes(writes) <> e.write } + } else { + require(bp.balls.forall(b => b.mmioReadBW == 0 && b.mmioWriteBW == 0)) + io.memory.metadata.request.ready := false.B + io.memory.metadata.response.valid := false.B + io.memory.metadata.response.bits := false.B + when(io.memory.metadata.request.valid)(assert(false.B, "MMIO metadata bank is disabled")) + } + + io.complete.bits := 0.U.asTypeOf(io.complete.bits) + io.memory.request.valid := false.B + io.memory.request.bits := 0.U.asTypeOf(io.memory.request.bits) + io.memory.response.ready := false.B + expander match { + case None => + decoder.io.issue.valid := io.issue.valid + operation := io.issue.bits.operation + io.issue.ready := decoder.io.issue.ready + io.complete.valid := bbus.io.cmdResp.valid + io.complete.bits.rob_id := bbus.io.cmdResp.bits.rob_id + bbus.io.cmdResp.ready := io.complete.ready + case Some(e) => + val sub: Instance[SubROB] = Instantiate(new SubROB(fp, mp, bp)) + val active = RegInit(false.B) + val parent = Reg(chiselTypeOf(io.issue.bits)) + val ballPending = RegInit(false.B) + val memoryPending = RegInit(false.B) + val ballTag = Reg(chiselTypeOf(sub.io.issueSubId(0))) + val memoryTag = Reg(chiselTypeOf(sub.io.issueSubId(1))) + val faulted = RegInit(false.B) + val faultReported = RegInit(false.B) + val fault = Reg(chiselTypeOf(io.complete.bits.fault)) + e.command.bits.rob_id := io.issue.bits.operation.rob_id + e.command.bits.funct := io.issue.bits.operation.cmd.cmd.raw_inst(31, 25) + e.command.bits.rs2 := io.issue.bits.operation.cmd.cmd.rs2Data + io.issue.ready := !active && !faulted && Mux(e.accepts, e.command.ready, decoder.io.issue.ready) + e.command.valid := io.issue.valid && e.accepts && !active && !faulted + when(io.issue.fire && e.starts) { active := true.B; parent := io.issue.bits } + sub.io.write <> e.rows + sub.io.write.valid := e.rows.valid && active && !faulted + e.rows.ready := sub.io.write.ready && active && !faulted + val child = Wire(new Issue(fp, mp)) + child := parent.operation + child.cmd := sub.io.issue(0).bits + child.cmd.cmd.pc := parent.operation.cmd.cmd.pc + child.renamedBankAccess := sub.io.issue(0).bits.bankAccess + decoder.io.issue.valid := !faulted && Mux( + active, + sub.io.issue(0).valid && !ballPending, + io.issue.valid && !e.accepts + ) + operation := Mux(active, child, io.issue.bits.operation) + sub.io.issue(0).ready := active && !faulted && !ballPending && decoder.io.issue.ready + when(sub.io.issue(0).fire) { ballPending := true.B; ballTag := sub.io.issueSubId(0) } + io.memory.request.valid := active && !faulted && sub.io.issue(1).valid && !memoryPending + io.memory.request.bits := parent + io.memory.request.bits.operation.cmd := sub.io.issue(1).bits + io.memory.request.bits.operation.cmd.cmd.pc := parent.operation.cmd.cmd.pc + io.memory.request.bits.operation.renamedBankAccess := sub.io.issue(1).bits.bankAccess + sub.io.issue(1).ready := active && !faulted && !memoryPending && io.memory.request.ready + when(io.memory.request.fire) { memoryPending := true.B; memoryTag := sub.io.issueSubId(1) } + val completed = Instantiate(new Arbiter(chiselTypeOf(ballTag), 2)) + completed.io.in(0).valid := active && ballPending && bbus.io.cmdResp.valid + completed.io.in(0).bits := ballTag + completed.io.in(1).valid := active && memoryPending && io.memory.response.valid + completed.io.in(1).bits := memoryTag + sub.io.subComplete <> completed.io.out + io.memory.response.ready := memoryPending && completed.io.in(1).ready + when(completed.io.in(0).fire)(ballPending := false.B) + when(io.memory.response.fire) { + memoryPending := false.B + when(io.memory.response.bits.fault.error =/= 0.U) { + faulted := true.B; fault := io.memory.response.bits.fault + } + } + val responses = Instantiate(new Arbiter(new Complete(fp), 3)) + responses.io.in(0).valid := !active && bbus.io.cmdResp.valid + responses.io.in(0).bits.rob_id := bbus.io.cmdResp.bits.rob_id + responses.io.in(1) <> e.complete + responses.io.in(2).valid := sub.io.masterComplete.valid && !faulted + responses.io.in(2).bits.rob_id := sub.io.masterComplete.bits + sub.io.masterComplete.ready := responses.io.in(2).ready && !faulted + when(sub.io.masterComplete.fire)(active := false.B) + bbus.io.cmdResp.ready := Mux(active, ballPending && completed.io.in(0).ready, responses.io.in(0).ready) + io.complete.valid := Mux(faulted, !faultReported, responses.io.out.valid) + io.complete.bits.rob_id := Mux(faulted, parent.operation.rob_id, responses.io.out.bits.rob_id) + io.complete.bits.fault := Mux(faulted, fault, 0.U.asTypeOf(fault)) + responses.io.out.ready := io.complete.ready && !faulted + when(io.complete.fire && faulted)(faultReported := true.B) + } +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/Emit.scala new file mode 100644 index 000000000..c7fdc1969 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/Emit.scala @@ -0,0 +1,39 @@ +package framework.acc.balldomain + +import framework.acc.balldomain.bbus.BallFactory +import framework.acc.balldomain.ball.blink.BlinkIO + +import chisel3.Module +import _root_.configs.GlobalConfig +import framework.acc.balldomain.ball.blink.HasBlink + +object Emit { + + def emit( + b: GlobalConfig, + factories: Map[String, BallFactory], + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new BallDomain( + thread.acc.get.frontend, + thread.acc.get.memDomain, + thread.acc.get.gpDomain, + thread.acc.get.ballDomain, + core.privileged.permission, + factories + ), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/baseball.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/baseball.scala new file mode 100644 index 000000000..341fce7b7 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/baseball.scala @@ -0,0 +1,27 @@ +//===- baseball.scala ------ Ball Interface Contract ----------------------===// +// +// Copyright 2026 The Buckyball Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +package framework.acc.balldomain.ball.blink + +import chisel3.experimental.hierarchy.{instantiable, public} + +// All Balls expose the same instance boundary. +@instantiable +trait HasBlink { + @public + val blink: BlinkIO +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/blink.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/blink.scala new file mode 100644 index 000000000..6567f678f --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/blink.scala @@ -0,0 +1,76 @@ +//===- blink.scala --------- Ball Link Interface --------------------------===// +// +// Copyright 2026 The Buckyball Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +package framework.acc.balldomain.ball.blink + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import framework.acc.memdomain.{BankRead, BankWrite} +import framework.acc.balldomain.mmio_bank.{MmioRead, MmioWriteReq} + +/** + * One Ball operation. Banks are already renamed; each operand covers `col` groups starting at `group`. + */ +class BallCommand(fp: FrontendParam, mp: MemDomainParam) extends Bundle { + val rob_id = UInt(log2Up(fp.rob_entries).W) + val funct = UInt(7.W) + val rs2 = UInt(fp.commandDataBits.W) + val iter = UInt(fp.iter_len.W) + val op1_bank = UInt(fp.bank_id_len.W) + val op2_bank = UInt(fp.bank_id_len.W) + val wr_bank = UInt(fp.bank_id_len.W) + val op1_group = UInt(mp.groupIdWidth.W) + val op2_group = UInt(mp.groupIdWidth.W) + val wr_group = UInt(mp.groupIdWidth.W) + val op1_col = UInt(mp.groupCountWidth.W) + val op2_col = UInt(mp.groupCountWidth.W) + val wr_col = UInt(mp.groupCountWidth.W) + // Renamed bank holding the previous destination value for accumulation. + val wr_read_bank = UInt(fp.bank_id_len.W) + // Destination bank before renaming; identifies chained writes to one architectural bank. + val wr_arch_bank = UInt(fp.bank_id_len.W) +} + +class BallResponse(fp: FrontendParam) extends Bundle { + val rob_id = UInt(log2Up(fp.rob_entries).W) +} + +class BallStatus extends Bundle { + val idle = Output(Bool()) + val running = Output(Bool()) +} + +class BlinkIO( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Bundle { + val ball = bp.balls.find(_.id == id).get + + val mmioRead = Vec(ball.mmioReadBW, Flipped(new MmioRead(mp))) + val mmioWrite = Vec(ball.mmioWriteBW, Decoupled(new MmioWriteReq(mp))) + val status = new BallStatus + val cmdReq = Flipped(Decoupled(new BallCommand(fp, mp))) + val cmdResp = Decoupled(new BallResponse(fp)) + val bankRead = Vec(ball.inBW, Flipped(new BankRead(fp, mp))) + val bankWrite = Vec(ball.outBW, Flipped(new BankWrite(fp, mp))) +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/build.sc b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/build.sc new file mode 100644 index 000000000..1b997830d --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/blink/build.sc @@ -0,0 +1,12 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("baseball.scala")), + PathRef(millSourcePath / os.RelPath("blink.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/build.sc b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/build.sc new file mode 100644 index 000000000..d208a889f --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/ball/build.sc @@ -0,0 +1,10 @@ +import mill._ +import $file.blink.{build => blink_build} + +def sources = T.sources( + blink_build.sources() +) + +def emitSources = T.sources( + blink_build.emitSources() +) diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/bbus.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/bbus.scala new file mode 100644 index 000000000..8d27bd9fc --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/bbus.scala @@ -0,0 +1,192 @@ +package framework.acc.balldomain.bbus + +import framework.lib.queue.Queue +import framework.lib.arbiter.{Arbiter, RRArbiter} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.balldomain.sub_rob.Expansion +import framework.acc.balldomain.ball.blink.BlinkIO +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.memdomain.{BankLayout, BankLayoutRequest, BankPort} +import framework.acc.balldomain.bbus.channels.Channels +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import framework.acc.balldomain.decoder.DISA.BALL_INIT_BITPAT +import framework.acc.balldomain.mmio_bank.{MmioRead, MmioWriteReq} + +/** A Ball command routed to `target`; `enabled` marks which of op1, op2 and wr name a bank. */ +class BallRequest(fp: FrontendParam, mp: MemDomainParam) extends Bundle { + val target = UInt(5.W) + val instId = UInt(64.W) // Observation identity; never controls scheduling. + val enabled = Vec(3, Bool()) + val cmd = new BallCommand(fp, mp) +} + +/** Interconnect for the Ball instances listed in the BallDomain configuration. */ +@instantiable +class BBus( + fp: FrontendParam, + mp: MemDomainParam, + gp: GpDomainParam, + bp: BallDomainParam, + factories: Map[String, BallFactory]) + extends Module { + val numBalls = bp.balls.size + + @public + val io = IO(new Bundle { + val mmioRead = Vec(bp.balls.map(_.mmioReadBW).sum, Flipped(new MmioRead(mp))) + val mmioWrite = Vec(bp.balls.map(_.mmioWriteBW).sum, Decoupled(new MmioWriteReq(mp))) + val traceStarted = Output(Vec(numBalls, Valid(UInt(log2Ceil(fp.rob_entries).W)))) + val traceCompleted = Output(Vec(numBalls, Valid(UInt(log2Ceil(fp.rob_entries).W)))) + // Frontend commands still need their group counts from the bank layout. + val cmdReq = Flipped(Decoupled(new BallRequest(fp, mp))) + val cmdResp = Decoupled(new BallResponse(fp)) + val layoutRequest = Decoupled(new BankLayoutRequest(fp)) + val layoutResponse = Flipped(Decoupled(new BankLayout(mp))) + val banks = Vec(mp.bankRequestPorts, new BankPort(fp, mp)) + // GP kernel commands arrive complete and complete back to the kernel. + val kernelReq = Option.when(gp.enable)(Flipped(Decoupled(new BallRequest(fp, mp)))) + val kernelResp = Option.when(gp.enable)(Decoupled(new BallResponse(fp))) + }) + + val channels: Instance[Channels] = Instantiate(new Channels(fp, mp, bp)) + val instructionIds = Reg(Vec(numBalls, UInt(64.W))) + channels.io.instructionIds := instructionIds + io.banks <> channels.io.mem_req + + // Apply BALL_INIT on the cycle after the command handshake. Driving reset + // directly from cmd.fire would feed the Ball's reset-dependent ready signal + // back into that same handshake. + val ballBootReset = RegInit(VecInit(Seq.fill(numBalls)(false.B))) + + val instances = bp.balls.zipWithIndex.map { case (ball, index) => + withReset(reset.asBool || ballBootReset(index)) { + factories(ball.component).instantiate(fp, mp, bp, ball.id) + } + } + + val balls = instances.map(_.blink) + val expanded = instances.flatMap(_.expansion) + require(expanded.size <= 1, "One instruction expander is supported per BallDomain") + require(expanded.nonEmpty == fp.sub_rob_enable) + + @public + val expansion: Option[Expansion] = expanded.headOption.map { source => + val port = IO(new Expansion(fp, mp, bp)) + port <> source + port + } + + balls.flatMap(_.mmioRead).zip(io.mmioRead).foreach { case (source, port) => port <> source } + balls.flatMap(_.mmioWrite).zip(io.mmioWrite).foreach { case (source, port) => port <> source } + + val pending = Instantiate(new Queue(new BallRequest(fp, mp), 4)) + io.layoutRequest.valid := io.cmdReq.valid && pending.io.enq.ready + pending.io.enq.valid := io.cmdReq.valid && io.layoutRequest.ready + pending.io.enq.bits := io.cmdReq.bits + io.cmdReq.ready := pending.io.enq.ready && io.layoutRequest.ready + io.layoutRequest.bits.banks(0) := io.cmdReq.bits.cmd.op1_bank + io.layoutRequest.bits.banks(1) := io.cmdReq.bits.cmd.op2_bank + io.layoutRequest.bits.banks(2) := io.cmdReq.bits.cmd.wr_bank + io.layoutRequest.bits.enabled := io.cmdReq.bits.enabled + val external = Wire(Decoupled(new BallRequest(fp, mp))) + external.valid := pending.io.deq.valid && io.layoutResponse.valid + external.bits := pending.io.deq.bits + external.bits.cmd.op1_col := io.layoutResponse.bits.groups(0) + external.bits.cmd.op2_col := io.layoutResponse.bits.groups(1) + external.bits.cmd.wr_col := io.layoutResponse.bits.groups(2) + pending.io.deq.ready := external.ready && io.layoutResponse.valid + io.layoutResponse.ready := external.ready && pending.io.deq.valid + + val command = Wire(Decoupled(new BallRequest(fp, mp))) + val fromKernel = WireDefault(false.B) + val kernelOwner = RegInit(VecInit(Seq.fill(numBalls)(false.B))) + val kernelPending = WireDefault(false.B) + if (gp.enable) { + val arbiter = Instantiate(new RRArbiter(new BallRequest(fp, mp), 2)) + arbiter.io.in(0) <> external + arbiter.io.in(1) <> io.kernelReq.get + command <> arbiter.io.out + fromKernel := arbiter.io.chosen === 1.U + assert(PopCount(kernelOwner) <= 1.U, "GP domain issues only one outstanding Ball operation") + } else { + command <> external + } + + val isBallInit = command.bits.cmd.funct === BALL_INIT_BITPAT + val initPending = RegInit(VecInit(Seq.fill(numBalls)(false.B))) + val initResp = Reg(Vec(numBalls, new BallResponse(fp))) + val targetMatches = VecInit(bp.balls.map(b => command.bits.target === b.id.U)) + val responses = Instantiate(new Arbiter(new BallResponse(fp), numBalls)) + val kernelDone = Wire(Vec(numBalls, Valid(new BallResponse(fp)))) + + for (i <- 0 until numBalls) { + channels.io.active(i) := balls(i).status.running + + val targetMatch = targetMatches(i) + balls(i).cmdReq.valid := command.valid && !isBallInit && !initPending(i) && targetMatch + balls(i).cmdReq.bits := command.bits.cmd + + val done = Mux(initPending(i), true.B, balls(i).cmdResp.valid) + val resp = Mux(initPending(i), initResp(i), balls(i).cmdResp.bits) + responses.io.in(i).valid := !kernelOwner(i) && done + responses.io.in(i).bits := resp + kernelDone(i).valid := kernelOwner(i) && done + kernelDone(i).bits := resp + + val accepted = Mux(kernelOwner(i), io.kernelResp.map(_.ready).getOrElse(false.B), responses.io.in(i).ready) + balls(i).cmdResp.ready := !initPending(i) && accepted + when(done && accepted) { + initPending(i) := false.B + kernelOwner(i) := false.B + } + + when(command.fire && targetMatch) { + instructionIds(i) := command.bits.instId + kernelOwner(i) := fromKernel + } + ballBootReset(i) := command.fire && isBallInit && targetMatch + when(command.fire && isBallInit && targetMatch) { + initPending(i) := true.B + initResp(i).rob_id := command.bits.cmd.rob_id + } + + io.traceStarted(i).valid := command.fire && targetMatch + io.traceStarted(i).bits := command.bits.cmd.rob_id + io.traceCompleted(i).valid := done && accepted + io.traceCompleted(i).bits := resp.rob_id + } + + io.kernelResp.foreach { port => + port.valid := kernelDone.map(_.valid).reduce(_ || _) + port.bits := Mux1H(kernelDone.map(_.valid), kernelDone.map(_.bits)) + } + + command.ready := + VecInit((0 until numBalls).map(i => balls(i).cmdReq.ready && !initPending(i) && targetMatches(i))).asUInt.orR + + io.cmdResp <> responses.io.out + + // Ball bank ports occupy consecutive Channel slots in configuration order. + channels.io.bankRead.zip(balls.flatMap(_.bankRead)).foreach { case (channel, port) => channel <> port } + channels.io.bankWrite.zip(balls.flatMap(_.bankWrite)).foreach { case (channel, port) => channel <> port } +} + +case class BallPorts(blink: BlinkIO, expansion: Option[Expansion]) + +trait BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts + +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/build.sc b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/build.sc new file mode 100644 index 000000000..f3aff3ac2 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/build.sc @@ -0,0 +1,39 @@ +import mill._ + +// The component library owns the generated table; framework only knows the contract. +def registry(sources: Seq[PathRef], componentRoot: os.Path, destination: os.Path): Seq[PathRef] = { + val declarations = sources.flatMap { source => + val text = os.read(source.path) + val names = "(?m)^object (\\w+) extends BallFactory".r.findAllMatchIn(text).map(_.group(1)).toSeq + if (names.isEmpty) Seq.empty + else { + val pkg = "(?m)^package ([\\w.]+)".r.findFirstMatchIn(text).get.group(1) + val component = source.path.relativeTo(componentRoot).segments.head + names.map(name => component -> s"$pkg.$name") + } + }.sortBy(_._1) + require(declarations.map(_._1).distinct.size == declarations.size, "Duplicate component factory") + val entries = declarations.map { case (component, factory) => s""""$component" -> $factory""" } + val output = destination / "BallComponents.scala" + os.write.over( + output, + s"""package framework.acc.balldomain.bbus.generated +import framework.acc.balldomain.bbus.BallFactory +object BallComponents { + val factories: Map[String, BallFactory] = Map(${entries.mkString(",\n")}) +} +""" + ) + Seq(PathRef(output)) +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("bbus.scala")), + PathRef(millSourcePath / os.RelPath("channels/Channels.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/channels/Channels.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/channels/Channels.scala new file mode 100644 index 000000000..1a0654093 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/channels/Channels.scala @@ -0,0 +1,202 @@ +package framework.acc.balldomain.bbus.channels + +import chisel3.experimental.hierarchy.Instantiate + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.lib.queue.Queue +import chisel3._ +import chisel3.util._ +import framework.acc.memdomain.{BankRead, BankWrite} +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.memdomain.BankPort + +/** + * Channel allocation and release + * Connects memory clients to the layout table + * + * Configured Ball read and write ports retain their channels until completion. + * The port ordering is determined by the Ball connection configuration. + * Requests retain their channel until the owning Ball completes and responses drain. + */ +@instantiable +class Channels(fp: FrontendParam, mp: MemDomainParam, bp: BallDomainParam) extends Module { + // Each configured Ball owns its declared read and write ports. + val ownerCount = bp.balls.size + val totalRead = bp.balls.map(_.inBW).sum + val totalWrite = bp.balls.map(_.outBW).sum + + @public + val io = IO(new Bundle { + // Ball read and write ports in configuration order. + val bankRead = Vec(totalRead, new BankRead(fp, mp)) + val bankWrite = Vec(totalWrite, new BankWrite(fp, mp)) + val active = Input(Vec(bp.balls.size, Bool())) + + val instructionIds = Input(Vec(ownerCount, UInt(64.W))) + + // Output to the layout table + val mem_req = Vec(mp.bankRequestPorts, new BankPort(fp, mp)) + }) + + val readPortOwner = bp.balls.zipWithIndex.flatMap { case (ball, owner) => Seq.fill(ball.inBW)(owner) } + val writePortOwner = bp.balls.zipWithIndex.flatMap { case (ball, owner) => Seq.fill(ball.outBW)(owner) } + + val readInstructionIds = Wire(Vec(totalRead, UInt(64.W))) + val readClients = Wire(Vec(totalRead, new BankRead(fp, mp))) + for (i <- 0 until totalRead) { + val source = io.bankRead(i) + val request = Wire(Decoupled(new Bundle { + val bank = chiselTypeOf(source.bank_id) + val rob = chiselTypeOf(source.rob_id) + val group = chiselTypeOf(source.group_id) + val address = chiselTypeOf(source.io.req.bits) + val instId = UInt(64.W) + })) + request.valid := source.io.req.valid + request.bits.bank := source.bank_id + request.bits.rob := source.rob_id + request.bits.group := source.group_id + request.bits.address := source.io.req.bits + request.bits.instId := io.instructionIds(readPortOwner(i)) + source.io.req.ready := request.ready + val queue = Instantiate(new Queue(chiselTypeOf(request.bits), 8, false, false)) + queue.io.enq <> request + readInstructionIds(i) := queue.io.deq.bits.instId + val target = readClients(i) + target.io.req.valid := queue.io.deq.valid + target.io.req.bits := queue.io.deq.bits.address + target.bank_id := queue.io.deq.bits.bank + target.rob_id := queue.io.deq.bits.rob + target.group_id := queue.io.deq.bits.group + queue.io.deq.ready := target.io.req.ready + source.io.resp <> target.io.resp + } + + // ----------------------------------------------------------------------------- + // Mapping table for configured Ball request ports + // ----------------------------------------------------------------------------- + class MappingTableEntry extends Bundle { + val valid = Bool() + val isRead = Bool() + val id = UInt(log2Ceil(math.max(totalRead, totalWrite)).W) + val owner = UInt(math.max(1, log2Ceil(ownerCount)).W) + } + + val mappingTable = RegInit(VecInit( + Seq.fill(mp.bankRequestPorts)(0.U.asTypeOf(new MappingTableEntry)) + )) + + // A route stays assigned until its Ball releases it, so maintain the reverse ownership at allocation time. This keeps + // readiness and pending detection from repeatedly scanning every backend + // request port for the same port. + val portSlots = math.max(totalRead, totalWrite) + val readPortAllocated = RegInit(VecInit(Seq.fill(portSlots)(false.B))) + val writePortAllocated = RegInit(VecInit(Seq.fill(portSlots)(false.B))) + val channelPending = + RegInit(VecInit(Seq.fill(mp.bankRequestPorts)(false.B))) + + // Allocate or release one channel per cycle, retaining its owner through response drain. + for (i <- 0 until totalRead) { + readClients(i).io.req.ready := false.B + readClients(i).io.resp.valid := false.B + readClients(i).io.resp.bits := DontCare + + } + + for (i <- 0 until totalWrite) { + io.bankWrite(i).io.req.ready := false.B + io.bankWrite(i).io.resp.valid := false.B + io.bankWrite(i).io.resp.bits := DontCare + } + + val pendingReads = VecInit((0 until totalRead).map(i => io.active(readPortOwner(i)) && !readPortAllocated(i))) + val pendingWrites = VecInit((0 until totalWrite).map(i => io.active(writePortOwner(i)) && !writePortAllocated(i))) + + val pending = VecInit(pendingReads ++ pendingWrites) + val freePorts = mappingTable.map(entry => !entry.valid) + + // Connect mapped entries to layout request ports + for (i <- 0 until mp.bankRequestPorts) { + io.mem_req(i).read.req.valid := false.B + io.mem_req(i).read.req.bits := DontCare + io.mem_req(i).read.resp.ready := false.B + io.mem_req(i).write.req.valid := false.B + io.mem_req(i).write.req.bits := DontCare + io.mem_req(i).write.resp.ready := false.B + io.mem_req(i).bank_id := 0.U + io.mem_req(i).group_id := 0.U + io.mem_req(i).is_shared := false.B + io.mem_req(i).thread_id := fp.ownerId.U(64.W) + io.mem_req(i).rob_id := 0.U + io.mem_req(i).inst_id := 0.U + + val isRead = mappingTable(i).isRead + val rid = mappingTable(i).id + val wid = mappingTable(i).id + val ballRead = readClients(rid).io + val ballWrite = io.bankWrite(wid).io + val rbank_id = readClients(rid).bank_id + val wbank_id = io.bankWrite(wid).bank_id + val rgroup_id = readClients(rid).group_id + val wgroup_id = io.bankWrite(wid).group_id + val rrob_id = readClients(rid).rob_id + val wrob_id = io.bankWrite(wid).rob_id + + when(mappingTable(i).valid) { + when(isRead) { + io.mem_req(i).read <> ballRead + io.mem_req(i).bank_id := rbank_id + io.mem_req(i).group_id := rgroup_id + io.mem_req(i).rob_id := rrob_id + io.mem_req(i).inst_id := readInstructionIds(rid) + }.otherwise { + io.mem_req(i).write <> ballWrite + io.mem_req(i).bank_id := wbank_id + io.mem_req(i).group_id := wgroup_id + io.mem_req(i).rob_id := wrob_id + io.mem_req(i).inst_id := io.instructionIds(mappingTable(i).owner) + } + } + } + + for (i <- 0 until mp.bankRequestPorts) { + when(io.mem_req(i).read.req.fire || io.mem_req(i).write.req.fire)(channelPending(i) := true.B) + when(io.mem_req(i).read.resp.fire || io.mem_req(i).write.resp.fire)(channelPending(i) := false.B) + } + + val release = VecInit((0 until mp.bankRequestPorts).map { i => + val ownerFinished = !io.active(mappingTable(i).owner) + val drained = !channelPending(i) && !(io.mem_req(i).read.resp.valid || io.mem_req(i).write.resp.valid || + io.mem_req(i).read.req.valid || io.mem_req(i).write.req.valid) + mappingTable(i).valid && ownerFinished && drained + }) + + when(release.asUInt.orR) { + val channel = PriorityEncoder(release) + when(mappingTable(channel).isRead) { + readPortAllocated(mappingTable(channel).id) := false.B + }.otherwise { + writePortAllocated(mappingTable(channel).id) := false.B + } + mappingTable(channel) := 0.U.asTypeOf(new MappingTableEntry) + }.elsewhen(pending.asUInt.orR && freePorts.reduce(_ || _)) { + val channel = PriorityEncoder(freePorts) + val port = PriorityEncoder(pending) + mappingTable(channel).valid := true.B + mappingTable(channel).isRead := port < totalRead.U + mappingTable(channel).id := Mux(port < totalRead.U, port, port - totalRead.U) + mappingTable(channel).owner := Mux( + port < totalRead.U, + MuxLookup(port, 0.U)(readPortOwner.zipWithIndex.map { case (ball, i) => i.U -> ball.U }.toSeq), + MuxLookup(port - totalRead.U, 0.U)(writePortOwner.zipWithIndex.map { case (ball, i) => i.U -> ball.U }.toSeq) + ) + when(port < totalRead.U) { + readPortAllocated(port) := true.B + }.otherwise { + writePortAllocated(port - totalRead.U) := true.B + } + } +} diff --git a/arch/src/main/scala/framework/balldomain/bbus/pmc/ballcyclepmc.spec.en.md b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/pmc/ballcyclepmc.spec.en.md similarity index 100% rename from arch/src/main/scala/framework/balldomain/bbus/pmc/ballcyclepmc.spec.en.md rename to arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/pmc/ballcyclepmc.spec.en.md diff --git a/arch/src/main/scala/framework/balldomain/bbus/pmc/ballcyclepmc.spec.zh.md b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/pmc/ballcyclepmc.spec.zh.md similarity index 100% rename from arch/src/main/scala/framework/balldomain/bbus/pmc/ballcyclepmc.spec.zh.md rename to arch/src/main/scala/framework/acc/balldomain/src/main/scala/bbus/pmc/ballcyclepmc.spec.zh.md diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/configs/BallDomainParam.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/configs/BallDomainParam.scala new file mode 100644 index 000000000..15ac4ee84 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/configs/BallDomainParam.scala @@ -0,0 +1,67 @@ +package framework.acc.balldomain.configs + +import buckyball.config.{BallDomain => BallDomainProto, BallInstance => BallInstanceProto} +import scala.jdk.CollectionConverters._ +import com.google.protobuf.{Any => BallParameters} + +import upickle.default.{macroRW, readwriter, ReadWriter} +import BallRouteParam.parametersRW + +case class InstructionParam(mnemonic: String, funct7: Int) { + require(funct7 >= 0 && funct7 < 128, "funct7 must fit the 7-bit instruction field") +} + +object InstructionParam { implicit val rw: ReadWriter[InstructionParam] = macroRW } + +case class BallRouteParam( + id: Int, + name: String, + inBW: Int, + outBW: Int, + mmioReadBW: Int, + mmioWriteBW: Int, + instructions: Seq[InstructionParam], + component: String, + parameters: BallParameters, + rtlClass: String) { + require(inBW > 0 && outBW > 0, "BBus widths must be positive") + require(id >= 0 && id < 32, "Ball ID must fit the 5-bit target field") + require( + instructions.map(_.mnemonic).distinct.size == instructions.size, + "Duplicate instruction mnemonic in Ball instance" + ) +} + +object BallRouteParam { + + def fromProto(source: BallInstanceProto): BallRouteParam = BallRouteParam( + source.getBallId, + source.getBallName, + source.getInBw, + source.getOutBw, + source.getMmioReadBw, + source.getMmioWriteBw, + source.getInstructionsList.asScala.map(i => InstructionParam(i.getMnemonic, i.getFunct7)).toSeq, + source.getBallDir, + source.getParameters, + source.getRtlClass + ) + + implicit val parametersRW: ReadWriter[BallParameters] = + readwriter[Array[Byte]].bimap[BallParameters](_.toByteArray, bytes => BallParameters.parseFrom(bytes)) + implicit val rw: ReadWriter[BallRouteParam] = macroRW +} + +case class BallDomainParam(balls: Seq[BallRouteParam]) { + require(balls.map(_.id).distinct.size == balls.size, "Duplicate Ball ID") + require(balls.map(_.name).distinct.size == balls.size, "Duplicate Ball instance name") + + val instructions = balls.flatMap(_.instructions) + require(instructions.map(_.funct7).distinct.size == instructions.size, "Duplicate instruction encoding") +} + +object BallDomainParam { + def fromProto(p: BallDomainProto): BallDomainParam = + BallDomainParam(p.getBallsList.asScala.map(BallRouteParam.fromProto).toSeq) + implicit val rw: ReadWriter[BallDomainParam] = macroRW +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/decoder/DISA.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/decoder/DISA.scala new file mode 100644 index 000000000..e9a32479a --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/decoder/DISA.scala @@ -0,0 +1,7 @@ +package framework.acc.balldomain.decoder + +import chisel3.util.BitPat + +object DISA { + val BALL_INIT_BITPAT = BitPat("b0000101") +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/decoder/Decoder.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/decoder/Decoder.scala new file mode 100644 index 000000000..48385b480 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/decoder/Decoder.scala @@ -0,0 +1,47 @@ +package framework.acc.balldomain.decoder + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.balldomain.bbus.BallRequest + +/** Selects the Ball that owns each funct7; BALL_INIT names its target in rs2[4:0]. */ +@instantiable +class Decoder(fp: FrontendParam, mp: MemDomainParam, bp: BallDomainParam) extends Module { + + @public + val io = IO(new Bundle { + val issue = Flipped(Decoupled(new BallRequest(fp, mp))) + val command = Decoupled(new BallRequest(fp, mp)) + }) + + val funct = io.issue.bits.cmd.funct + val init = funct === DISA.BALL_INIT_BITPAT + val target = WireDefault(0.U(5.W)) + val registered = WireDefault(false.B) + for { + ball <- bp.balls + entry <- ball.instructions + } { + when(funct === entry.funct7.U) { + target := ball.id.U + registered := true.B + } + } + when(init) { + target := io.issue.bits.cmd.rs2(4, 0) + registered := VecInit(bp.balls.map(b => target === b.id.U)).asUInt.orR + } + io.command.valid := io.issue.valid && registered + io.command.bits := io.issue.bits + io.command.bits.target := target + io.issue.ready := io.command.ready && registered + when(io.issue.valid) { + assert(registered, "Ball command is not registered") + when(init)(assert(io.issue.bits.cmd.rs2(63, 5) === 0.U, "BALL_INIT reserves rs2[63:5]")) + } +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/mmio_bank/MmioBank.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/mmio_bank/MmioBank.scala new file mode 100644 index 000000000..9dae1ef21 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/mmio_bank/MmioBank.scala @@ -0,0 +1,131 @@ +package framework.acc.balldomain.mmio_bank + +import framework.lib.arbiter.{Arbiter, RRArbiter} +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.memdomain.configs.MemDomainParam + +class MmioReadReq(p: MemDomainParam) extends Bundle { val addr = UInt(log2Ceil(p.mmioTotalBytes).W) } +class MmioReadResp(p: MemDomainParam) extends Bundle { val data = UInt(p.mmioReadWidth.W) } + +class MmioRead(p: MemDomainParam) extends Bundle { + val req = Flipped(Decoupled(new MmioReadReq(p))) + val resp = Decoupled(new MmioReadResp(p)) +} + +class MmioWriteReq(p: MemDomainParam) extends Bundle { + val addr = UInt(log2Ceil(p.mmioTotalBytes).W) + val data = UInt(p.mmioReadWidth.W) +} + +/** Metadata receiver: one addressed DMA beat, acknowledged after every byte is stored. */ +class MetadataWrite(p: MemDomainParam) extends Bundle { + + val request = Flipped(Decoupled(new Bundle { + val addr = UInt(log2Ceil(p.mmioTotalBytes).W) + val data = UInt(p.bankWidth.W) + val mask = UInt((p.bankWidth / 8).W) + })) + + val response = Decoupled(Bool()) +} + +/** Banked byte metadata SRAM; arbitration and response ownership remain local. */ +@instantiable +class MmioBank(p: MemDomainParam, readers: Int, writers: Int) extends Module { + require(p.mmioEnable && p.mmioBankWidth == 8 && p.mmioReadWidth == 8) + val bytes = p.bankWidth / 8 + val bankBits = math.max(1, log2Ceil(p.mmioBankNum)) + val rowBits = math.max(1, log2Ceil(p.mmioBankEntries)) + + @public + val io = IO(new Bundle { + val upload = new MetadataWrite(p) + val reads = Vec(readers, new MmioRead(p)) + val writes = Vec(writers, Flipped(Decoupled(new MmioWriteReq(p)))) + }) + + val idle :: storing :: respond :: Nil = Enum(3) + val state = RegInit(idle) + val beat = Reg(chiselTypeOf(io.upload.request.bits)) + val cursor = RegInit(0.U(math.max(1, log2Ceil(bytes)).W)) + io.upload.request.ready := state === idle + io.upload.response.valid := state === respond + io.upload.response.bits := true.B + when(io.upload.request.fire) { + assert(io.upload.request.bits.addr +& bytes.U <= p.mmioTotalBytes.U) + beat := io.upload.request.bits + cursor := 0.U + state := storing + } + when(state === storing) { + when(cursor +& p.mmioBankNum.U >= bytes.U)(state := respond) + .otherwise(cursor := cursor + p.mmioBankNum.U) + } + when(io.upload.response.fire)(state := idle) + + val readerBusy = RegInit(VecInit(Seq.fill(readers)(false.B))) + val readReady = Wire(Vec(readers, Vec(p.mmioBankNum, Bool()))) + val readValid = Wire(Vec(readers, Vec(p.mmioBankNum, Bool()))) + val readData = Wire(Vec(readers, Vec(p.mmioBankNum, UInt(8.W)))) + val writeReady = Wire(Vec(writers, Vec(p.mmioBankNum, Bool()))) + for (bank <- 0 until p.mmioBankNum) { + val memory = SyncReadMem(p.mmioBankEntries, UInt(8.W)) + val writes = Instantiate(new Arbiter(new MmioWriteReq(p), writers + 1)) + val offsets = (0 until p.mmioBankNum).map { offset => + val index = cursor +& offset.U + index < bytes.U && (beat.addr + index) % p.mmioBankNum.U === bank.U + } + val index = cursor +& PriorityEncoder(VecInit(offsets)) + writes.io.in(0).valid := state === storing && VecInit(offsets).asUInt.orR + writes.io.in(0).bits.addr := beat.addr + index + writes.io.in(0).bits.data := Mux(beat.mask(index(log2Ceil(bytes) - 1, 0)), (beat.data >> (index << 3))(7, 0), 0.U) + for (client <- 0 until writers) { + val selected = io.writes(client).bits.addr % p.mmioBankNum.U === bank.U + writes.io.in(client + 1).valid := io.writes(client).valid && selected + writes.io.in(client + 1).bits := io.writes(client).bits + writeReady(client)(bank) := selected && writes.io.in(client + 1).ready + } + writes.io.out.ready := true.B + val pending = RegInit(false.B) + val held = RegInit(false.B) + val owner = Reg(UInt(math.max(1, log2Ceil(readers)).W)) + val value = Reg(UInt(8.W)) + val reads = Instantiate(new RRArbiter(new MmioReadReq(p), math.max(1, readers))) + if (readers == 0) { + reads.io.in(0).valid := false.B + reads.io.in(0).bits := 0.U.asTypeOf(reads.io.in(0).bits) + } + for (client <- 0 until readers) { + val selected = io.reads(client).req.bits.addr % p.mmioBankNum.U === bank.U + reads.io.in(client).valid := io.reads(client).req.valid && selected && !readerBusy(client) + reads.io.in(client).bits := io.reads(client).req.bits + readReady(client)(bank) := selected && !readerBusy(client) && reads.io.in(client).ready + readValid(client)(bank) := held && owner === client.U + readData(client)(bank) := value + } + reads.io.out.ready := !writes.io.out.valid && !pending && !held + val raw = memory.readWrite( + (Mux(writes.io.out.valid, writes.io.out.bits.addr, reads.io.out.bits.addr) / p.mmioBankNum.U)(rowBits - 1, 0), + writes.io.out.bits.data, + writes.io.out.fire || reads.io.out.fire, + writes.io.out.fire + ) + pending := reads.io.out.fire + when(reads.io.out.fire)(owner := reads.io.chosen) + when(pending) { value := raw; held := true.B } + if (readers > 0) + when(held && (if (readers == 1) io.reads(0).resp.ready else io.reads(owner).resp.ready))(held := false.B) + } + for (client <- 0 until readers) { + io.reads(client).req.ready := readReady(client).asUInt.orR + io.reads(client).resp.valid := readValid(client).asUInt.orR + io.reads(client).resp.bits.data := Mux1H(readValid(client), readData(client)) + when(io.reads(client).req.fire)(readerBusy(client) := true.B) + when(io.reads(client).resp.fire)(readerBusy(client) := false.B) + } + for (client <- 0 until writers) io.writes(client).ready := writeReady(client).asUInt.orR +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/Expansion.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/Expansion.scala new file mode 100644 index 000000000..d21390d4c --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/Expansion.scala @@ -0,0 +1,34 @@ +package framework.acc.balldomain.sub_rob + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.frontend.scheduler.Complete +import framework.acc.balldomain.mmio_bank.{MmioRead, MmioWriteReq} + +/** CISC operands presented to a Ball-owned expander; address-space state stays in BallDomain. */ +class ExpansionCommand(f: FrontendParam) extends Bundle { + val rob_id = UInt(log2Up(f.rob_entries).W) + val funct = UInt(7.W) + val rs2 = UInt(f.commandDataBits.W) +} + +/** A Ball-owned expander supplies rows to the shared SubROB. */ +class Expansion(f: FrontendParam, m: MemDomainParam, b: BallDomainParam) extends Bundle { + val command = Flipped(Decoupled(new ExpansionCommand(f))) + val accepts = Output(Bool()) + val starts = Output(Bool()) + val rows = Decoupled(new SubRobRow(f, m, b)) + val complete = Decoupled(new Complete(f)) + val read = Flipped(new MmioRead(m)) + val write = Decoupled(new MmioWriteReq(m)) +} + +@instantiable +trait HasExpansion { + @public + val expansion: Expansion +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/SubROB.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/SubROB.scala new file mode 100644 index 000000000..3e2435176 --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/SubROB.scala @@ -0,0 +1,277 @@ +package framework.acc.balldomain.sub_rob + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.frontend.decoder.{DomainId, PostGDCmd} +import framework.acc.balldomain.sub_rob.configs.Params + +@instantiable +class SubROB(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: BallDomainParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + + val p = Params( + depth = frontendParam.sub_rob_depth, + masterEntries = frontendParam.rob_entries, + balls = ballParam.balls.size + ) + + val subRobDepth = p.depth + val subIdBits = log2Up(subRobDepth * 4) + val robIdBits = log2Up(p.masterEntries) + val ballIdBits = log2Up(p.balls) + val rowIdBits = log2Up(subRobDepth) + + @public + val io = IO(new Bundle { + // Ball → SubROB: write a row of sub-instructions + val write = Flipped(Decoupled(new SubRobRow(frontendParam, memParam, ballParam))) + // SubROB → GlobalRS: issue independent Ball and memory slots + val issue = Vec(2, Decoupled(new PostGDCmd(frontendParam, memParam))) + val issueSubId = Output(Vec(2, UInt(subIdBits.W))) + val issueMasterRobId = Output(UInt(robIdBits.W)) + // GlobalRS → SubROB: a sub-instruction completed (sub_rob_id) + val subComplete = Flipped(Decoupled(UInt(subIdBits.W))) + // SubROB → main ROB: all sub-instructions done + val masterComplete = Decoupled(UInt(robIdBits.W)) + // Status + val occupied = Output(Bool()) + val lockedBallId = Output(UInt(ballIdBits.W)) + }) + + // ------------------------------------------------------------------------- + // Storage + // ------------------------------------------------------------------------- + val sram = SyncReadMem(subRobDepth, new SubRobRow(frontendParam, memParam, ballParam)) + + val writePtr = RegInit(0.U(rowIdBits.W)) + val readPtr = RegInit(0.U(rowIdBits.W)) + val rowCount = RegInit(0.U((log2Up(subRobDepth) + 1).W)) + val lockedBallId = RegInit(0.U(ballIdBits.W)) + val masterRobId = RegInit(0.U(robIdBits.W)) + + def nextPtr(p: UInt): UInt = Mux(p === (subRobDepth - 1).U, 0.U, p + 1.U) + + val occupied = RegInit(false.B) + val lastQueued = RegInit(false.B) + val isFull = rowCount === subRobDepth.U + val ballMatch = + !occupied || (io.write.bits.ball_id === lockedBallId && io.write.bits.master_rob_id === masterRobId) + + // ------------------------------------------------------------------------- + // FSM + // ------------------------------------------------------------------------- + // States: + // sIdle - nothing to do (rowCount == 0) + // sReadReq - issue SRAM read request for readPtr row + // sReadWait - wait 1 cycle for sramRaw to be latched into sramData + // sReadResp - sramData valid; dispatch one slot per domain per cycle + // sWaitSlots - all slots issued; wait for all subCompletes + // sWaitMaster - all rows done; fire masterComplete then return to sIdle + val sIdle :: sReadReq :: sReadWait :: sReadResp :: sWaitSlots :: sWaitMaster :: Nil = + Enum(6) + val state = RegInit(sIdle) + + // Write path: accept if not full AND ball_id matches (or not occupied yet) + io.write.ready := !isFull && !lastQueued && ballMatch && state =/= sReadReq && state =/= sWaitMaster + + when(io.write.fire) { + occupied := true.B + when(io.write.bits.last)(lastQueued := true.B) + for (slot <- io.write.bits.slots) { + when(slot.valid) { + assert( + slot.cmd.domain_id === DomainId.BALL || slot.cmd.domain_id === DomainId.MEM, + "SubROB supports Ball and memory slots" + ) + } + } + when(occupied) { + assert( + io.write.bits.master_rob_id === masterRobId, + "SubROB row master_rob_id mismatch" + ) + } + writePtr := nextPtr(writePtr) + when(!occupied) { + lockedBallId := io.write.bits.ball_id + masterRobId := io.write.bits.master_rob_id + } + } + + // SyncReadMem: read issued in sReadReq (cycle N), data valid on cycle N+1. + // We latch it into sramData with RegEnable so it stays stable during sReadResp + // and sWaitSlots, even when the read port is not enabled. + val sramReadEn = state === sReadReq + + val sramRaw = sram.readWrite( + Mux(io.write.fire, writePtr, readPtr), + io.write.bits, + sramReadEn || io.write.fire, + io.write.fire + ) + + val dataFresh = + RegNext(sramReadEn, false.B) // pulse on first cycle of sReadResp + val sramData = RegEnable(sramRaw, dataFresh) + val readPtrReg = RegEnable(readPtr, sramReadEn) + + // Per-row tracking registers (reset when moving to next row) + // slotIssued: have we already sent this slot to a domain? + // slotDone: have we received subComplete for this slot? + val slotIssued = RegInit(VecInit(Seq.fill(4)(false.B))) + val slotDone = RegInit(VecInit(Seq.fill(4)(false.B))) + + // Combinational: which slots still need to be issued? + val slotNeedsIssue = VecInit( + (0 until 4).map(i => sramData.slots(i).valid && !slotIssued(i)) + ) + + val selected = Wire(Vec(2, UInt(2.W))) + for (port <- 0 until 2) { + val domain = if (port == 0) DomainId.BALL else DomainId.MEM + val candidates = + VecInit( + (0 until 4).map(slot => slotNeedsIssue(slot) && sramData.slots(slot).cmd.domain_id === domain) + ) + selected(port) := PriorityEncoder(candidates.asUInt) + io.issue(port).valid := state === sReadResp && candidates.asUInt.orR + io.issue(port).bits := sramData.slots(selected(port)).cmd + io.issueSubId(port) := readPtrReg * 4.U + selected(port) + } + + // Combinational: are all valid slots done? + val allSlotsDone = VecInit( + (0 until 4).map(i => !sramData.slots(i).valid || slotDone(i)) + ).asUInt.andR + + io.issueMasterRobId := masterRobId + + // subComplete: always ready, mark the corresponding slot done + io.subComplete.ready := true.B + when(io.subComplete.fire) { + val subId = io.subComplete.bits + val subRow = subId / 4.U + val slotIdx = subId(1, 0) + assert( + state === sReadResp || state === sWaitSlots || state === sWaitMaster, + "SubROB subComplete arrived in invalid state" + ) + assert(subRow === readPtrReg, "SubROB subComplete points to non-active row") + slotDone(slotIdx) := true.B + } + + // masterComplete + io.masterComplete.valid := state === sWaitMaster + io.masterComplete.bits := masterRobId + + // Status outputs + io.occupied := occupied + io.lockedBallId := lockedBallId + + // ------------------------------------------------------------------------- + // FSM transitions + // ------------------------------------------------------------------------- + // Helper: advance past current row (called when row is complete) + val rowAdvance = WireDefault(false.B) + + def advanceRow(): Unit = { + readPtr := nextPtr(readPtr) + rowAdvance := true.B + slotIssued.foreach(_ := false.B) + slotDone.foreach(_ := false.B) + } + + switch(state) { + is(sIdle) { + when(rowCount =/= 0.U)(state := sReadReq) + } + is(sReadReq) { + // SRAM read issued, data available on sramRaw next cycle + state := sReadWait + } + is(sReadWait) { + // sramRaw is valid; RegEnable latches it into sramData at this clock edge. + // sramData will be stable starting next cycle (sReadResp). + state := sReadResp + } + is(sReadResp) { + // Fire issued slot + for (port <- 0 until 2) { + when(io.issue(port).fire)(slotIssued(selected(port)) := true.B) + } + // Check if all slots have been issued + // After marking current slot issued, recalculate + val allIssued = VecInit( + (0 until 4).map(i => + !sramData.slots(i).valid || slotIssued(i) || + VecInit( + (0 until 2).map(port => io.issue(port).fire && selected(port) === i.U) + ).asUInt.orR + ) + ).asUInt.andR + + when(allIssued) { + // Check fast path: all already done (e.g., empty row or immediate completion) + val allDoneNow = VecInit( + (0 until 4).map(i => !sramData.slots(i).valid || slotDone(i)) + ).asUInt.andR + when(allDoneNow) { + // Fast path: skip sWaitSlots + advanceRow() + state := Mux( + sramData.last, + sWaitMaster, + Mux(rowCount === 1.U && !io.write.fire, sIdle, sReadReq) + ) + }.otherwise { + state := sWaitSlots + } + } + // else: stay in sReadResp to dispatch remaining slots + } + is(sWaitSlots) { + when(allSlotsDone) { + advanceRow() + state := Mux( + sramData.last, + sWaitMaster, + Mux(rowCount === 1.U && !io.write.fire, sIdle, sReadReq) + ) + } + } + is(sWaitMaster) { + when(io.masterComplete.fire) { + occupied := false.B + lastQueued := false.B + lockedBallId := 0.U + masterRobId := 0.U + state := sIdle + } + } + } + when(io.write.fire =/= rowAdvance) { + rowCount := Mux(io.write.fire, rowCount + 1.U, rowCount - 1.U) + } + +} + +class SubRobSlot(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Bundle { + val commandDataBits = frontendParam.commandDataBits + val valid = Bool() + val cmd = new PostGDCmd(frontendParam, memParam) +} + +class SubRobRow(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: BallDomainParam) + extends Bundle { + val commandDataBits = frontendParam.commandDataBits + val slots = Vec(4, new SubRobSlot(frontendParam, memParam)) + val ball_id = UInt(log2Up(ballParam.balls.size).W) + val master_rob_id = UInt(log2Up(frontendParam.rob_entries).W) + val last = Bool() +} diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/configs/Params.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/configs/Params.scala new file mode 100644 index 000000000..55893c34e --- /dev/null +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/sub_rob/configs/Params.scala @@ -0,0 +1,7 @@ +package framework.acc.balldomain.sub_rob.configs + +case class Params(depth: Int, masterEntries: Int, balls: Int) { + require(depth > 1) + require(masterEntries > 1) + require(balls > 0) +} diff --git a/arch/src/main/scala/framework/acc/build.sc b/arch/src/main/scala/framework/acc/build.sc new file mode 100644 index 000000000..962b2c0a2 --- /dev/null +++ b/arch/src/main/scala/framework/acc/build.sc @@ -0,0 +1,25 @@ +import mill._ +import $file.debug.{build => debug_build} +import $file.balldomain.{build => balldomain_build} +import $file.frontend.{build => frontend_build} +import $file.gpdomain.{build => gpdomain_build} +import $file.memdomain.{build => memdomain_build} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("configs/AccParam.scala")) + ) ++ + debug_build.sources() ++ + balldomain_build.sources() ++ + frontend_build.sources() ++ + gpdomain_build.sources() ++ + memdomain_build.sources() +) + +def emitSources = T.sources( + debug_build.emitSources() ++ + balldomain_build.emitSources() ++ + frontend_build.emitSources() ++ + gpdomain_build.emitSources() ++ + memdomain_build.emitSources() +) diff --git a/arch/src/main/scala/framework/acc/configs/AccParam.scala b/arch/src/main/scala/framework/acc/configs/AccParam.scala new file mode 100644 index 000000000..9b89d3a1c --- /dev/null +++ b/arch/src/main/scala/framework/acc/configs/AccParam.scala @@ -0,0 +1,63 @@ +package framework.acc.configs + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import buckyball.config.{CorePlacement, ThreadInstance} +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} +import upickle.default.{macroRW, ReadWriter} + +case class AccParam( + frontend: FrontendParam, + memDomain: MemDomainParam, + gpDomain: GpDomainParam, + ballDomain: BallDomainParam, + transport: TransportParam) { + require( + BigInt(memDomain.virtualBankCount) <= (BigInt(1) << frontend.bank_id_len), + "virtualBankCount must fit the frontend bank ID width" + ) + require( + frontend.vbank_id_upper_bound < memDomain.virtualBankCount, + "Private bank upper bound must be below virtualBankCount" + ) + if (memDomain.sharedEnable) { + require( + frontend.shared_bank_id_base > frontend.vbank_id_upper_bound, + "Shared bank base must be above the private bank range" + ) + require( + frontend.shared_bank_id_base <= memDomain.virtualBankCount, + "Shared bank base must not exceed virtualBankCount" + ) + } +} + +object AccParam { + + def fromProto(core: CorePlacement): AccParam = { + require(core.hasAccelerator, "Core has no local accelerator") + val acc = core.getAccelerator + AccParam( + FrontendParam.fromProto(acc.getFrontend, core.getParam.getXLen, core.getHartId), + MemDomainParam.fromProto(acc.getMem, core.getSharedMem, core.getVirtualBankCount, 1), + GpDomainParam.fromProto(acc.getGpdomain), + BallDomainParam.fromProto(acc.getBalldomain), + TransportParam.fromProto(acc.getTransport) + ) + } + + def fromProto(thread: ThreadInstance, core: CorePlacement): AccParam = { + require(thread.hasTransport) + AccParam( + FrontendParam.fromProto(thread.getFrontend, core.getParam.getXLen, thread.getLocalId), + MemDomainParam.fromProto(thread.getMem, core.getSharedMem, core.getVirtualBankCount, core.getThreadIndicesCount), + GpDomainParam.fromProto(thread.getGpdomain), + BallDomainParam.fromProto(thread.getBalldomain), + TransportParam.fromProto(thread.getTransport) + ) + } + + implicit val rw: ReadWriter[AccParam] = macroRW +} diff --git a/arch/src/main/scala/framework/acc/debug/BankCheckDPI.scala b/arch/src/main/scala/framework/acc/debug/BankCheckDPI.scala new file mode 100644 index 000000000..47c68f526 --- /dev/null +++ b/arch/src/main/scala/framework/acc/debug/BankCheckDPI.scala @@ -0,0 +1,77 @@ +package framework.acc.debug + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} +import chisel3.util._ +import framework.lib.dpi.DpiGuard + +@instantiable +class BankCheckDPI extends BlackBox with HasBlackBoxInline { + + @public + val io = IO(new Bundle { + val clock = Input(Clock()) + val reset = Input(Bool()) + val instId = Input(UInt(64.W)) + val threadId = Input(UInt(64.W)) + val ownerThreadId = Input(UInt(64.W)) + val w0Vbank = Input(UInt(32.W)) + val w0Hash = Input(UInt(32.W)) + val fire = Input(Bool()) + val produced = Input(UInt(64.W)) + val idle = Input(Bool()) + }) + + setInline( + "BankCheckDPI.v", + """ + |module BankCheckDPI( + | input clock, + | input reset, + | input [63:0] instId, + | input [63:0] threadId, + | input [63:0] ownerThreadId, + | input [31:0] w0Vbank, + | input [31:0] w0Hash, + | input fire, + | input [63:0] produced, + | input idle + |); + |""".stripMargin + DpiGuard.wrapBTrace( + """ + | export "DPI-C" function btrace_snapshot; + | function void btrace_snapshot( + | output int unsigned thread_lo, + | output int unsigned thread_hi, + | output int unsigned produced_lo, + | output int unsigned produced_hi, + | output int unsigned is_idle + | ); + | thread_lo = threadId[31:0]; + | thread_hi = threadId[63:32]; + | produced_lo = produced[31:0]; + | produced_hi = produced[63:32]; + | is_idle = {31'b0, idle}; + | endfunction + | import "DPI-C" context function void dpi_btrace( + | input int unsigned inst_id_lo, + | input int unsigned inst_id_hi, + | input int unsigned thread_id_lo, + | input int unsigned thread_id_hi, + | input int unsigned owner_thread_id_lo, + | input int unsigned owner_thread_id_hi, + | input int unsigned w0_vbank, + | input int unsigned w0_hash + | ); + | always @(posedge clock) begin + | if (!reset && fire) begin + | dpi_btrace(instId[31:0], instId[63:32], threadId[31:0], threadId[63:32], ownerThreadId[31:0], ownerThreadId[63:32], w0Vbank, w0Hash); + | end + | end + |""".stripMargin + ) + + """ + |endmodule + |""".stripMargin + ) +} diff --git a/arch/src/main/scala/framework/acc/debug/Debug.scala b/arch/src/main/scala/framework/acc/debug/Debug.scala new file mode 100644 index 000000000..0c7092f36 --- /dev/null +++ b/arch/src/main/scala/framework/acc/debug/Debug.scala @@ -0,0 +1,179 @@ +package framework.acc.debug + +import sims.configs.SimParam +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.memdomain.PhysicalBankHash +import framework.thread.ant.Retire +import framework.acc.debug.trace.{AcceleratorEvents, InstructionEvent, Trace} + +class DebugCommit(frontendParam: FrontendParam) extends Bundle { + val robId = UInt(log2Ceil(frontendParam.rob_entries).W) + val instId = UInt(64.W) + val threadId = UInt(64.W) + val bank = UInt(frontendParam.bank_id_len.W) +} + +/** Producer-facing debug handshake. Unlike trace, completion may wait for checked. */ +class DebugPort(frontendParam: FrontendParam) extends Bundle { + + val check = Decoupled(new Bundle { + val robId = UInt(log2Ceil(frontendParam.rob_entries).W) + val bank = UInt(frontendParam.bank_id_len.W) + }) + + val checked = Flipped(Decoupled(UInt(log2Ceil(frontendParam.rob_entries).W))) + val commit = Output(Valid(new DebugCommit(frontendParam))) + val idle = Output(Bool()) +} + +/** Frontend debug boundary: observations never share the differential-check handshake. */ +class FrontendDebugPort(frontendParam: FrontendParam, simp: SimParam) extends Bundle { + val diffTest = simp.diffTest + val trace = Output(Vec(4, new InstructionEvent)) + val control = Option.when(diffTest)(new DebugPort(frontendParam)) +} + +/** Producer-facing thread debug connection, assembled from the owning child IP ports. */ +class AcceleratorDebugPort( + frontendParam: FrontendParam, + memParam: MemDomainParam, + gpParam: GpDomainParam, + ballParam: BallDomainParam, + simp: SimParam) + extends Bundle { + val diffTest = simp.diffTest + + val frontend = new FrontendDebugPort(frontendParam, simp) + val trace = Output(new AcceleratorEvents(frontendParam, memParam, gpParam, ballParam)) + + val banks = Option.when(diffTest)( + Output(Vec( + memParam.bankNum + (if (memParam.sharedEnable) memParam.sharedBankNum else 0), + new PhysicalBankHash(frontendParam) + )) + ) + +} + +/** Differential bank checks may delay completion and serialize retirement. */ +@instantiable +class BankCheck( + frontendParam: FrontendParam, + memParam: MemDomainParam, + simp: SimParam, + traceId: Int, + sharedOwnerId: Int) + extends Module { + val diffTest = simp.diffTest + val ownerId = frontendParam.ownerId + require(diffTest) + + val bankCount = memParam.bankNum + (if (memParam.sharedEnable) memParam.sharedBankNum else 0) + + @public + val io = IO(new Bundle { + val pipeline = Flipped(new DebugPort(frontendParam)) + val banks = Input(Vec(bankCount, new PhysicalBankHash(frontendParam))) + }) + + val collecting = RegInit(false.B) + val ready = RegInit(false.B) + val robId = Reg(chiselTypeOf(io.pipeline.check.bits.robId)) + val bank = Reg(chiselTypeOf(io.pipeline.check.bits.bank)) + val group = Reg(UInt(32.W)) + val currentHash = Reg(UInt(32.W)) + val hashes = Reg(Vec(frontendParam.rob_entries, UInt(32.W))) + io.pipeline.check.ready := !collecting && !ready + when(io.pipeline.check.fire) { + robId := io.pipeline.check.bits.robId + bank := io.pipeline.check.bits.bank + group := 0.U + currentHash := 0.U + collecting := true.B + } + + val matches = VecInit(io.banks.map(entry => + entry.valid && entry.threadId === ownerId.U && entry.vbankId === bank && entry.groupId === group + )) + + val nextMatches = VecInit(io.banks.map(entry => + entry.valid && entry.threadId === ownerId.U && entry.vbankId === bank && entry.groupId === group + 1.U + )) + + val physical = Mux1H(matches, io.banks) + def rotateLeft(value: UInt, amount: Int): UInt = Cat(value(31 - amount, 0), value(31, 32 - amount)) + val mixed = physical.statusHash ^ rotateLeft(physical.groupId, 7) ^ "h9e3779b9".U + val nextHash = rotateLeft(currentHash, 5) ^ mixed ^ rotateLeft(mixed, 13) + when(collecting) { + assert(PopCount(matches) === 1.U, "Debug requires one physical bank per virtual bank group") + when(nextMatches.asUInt.orR) { + currentHash := nextHash + group := group + 1.U + }.otherwise { + hashes(robId) := nextHash + collecting := false.B + ready := true.B + } + } + io.pipeline.checked.valid := ready + io.pipeline.checked.bits := robId + when(io.pipeline.checked.fire)(ready := false.B) + + val produced = RegInit(0.U(64.W)) + when(io.pipeline.commit.valid)(produced := produced + 1.U) + val sink: Instance[BankCheckDPI] = Instantiate(new BankCheckDPI) + sink.io.clock := clock + sink.io.reset := reset.asBool + sink.io.instId := io.pipeline.commit.bits.instId + sink.io.threadId := traceId.U + sink.io.ownerThreadId := Mux( + memParam.sharedEnable.B && io.pipeline.commit.bits.bank >= frontendParam.shared_bank_id_base.U, + sharedOwnerId.U, + traceId.U + ) + sink.io.w0Vbank := io.pipeline.commit.bits.bank + sink.io.w0Hash := hashes(io.pipeline.commit.bits.robId) + sink.io.fire := io.pipeline.commit.valid + sink.io.produced := produced + sink.io.idle := io.pipeline.idle +} + +/** Thread-level observability: passive trace plus explicitly enabled timing-affecting checks. */ +@instantiable +class Debug( + frontendParam: FrontendParam, + memParam: MemDomainParam, + gpParam: GpDomainParam, + ballParam: BallDomainParam, + simp: SimParam, + traceId: Int, + sharedOwnerId: Int) + extends Module { + val diffTest = simp.diffTest + val ownerId = frontendParam.ownerId + + @public + val io = IO(new Bundle { + val ant = Input(Valid(new Retire)) + val acc = Flipped(new AcceleratorDebugPort(frontendParam, memParam, gpParam, ballParam, simp)) + + }) + + val trace: Instance[Trace] = Instantiate(new Trace(frontendParam, memParam, gpParam, ballParam, traceId)) + trace.io.ant := io.ant + trace.io.acc := io.acc.trace + trace.io.instructions := io.acc.frontend.trace + io.acc.frontend.control.foreach { port => + val check: Instance[BankCheck] = + Instantiate(new BankCheck(frontendParam, memParam, simp, traceId, sharedOwnerId)) + check.io.pipeline <> port + check.io.banks := io.acc.banks.get + } +} diff --git a/arch/src/main/scala/framework/acc/debug/build.sc b/arch/src/main/scala/framework/acc/debug/build.sc new file mode 100644 index 000000000..c35b4bc69 --- /dev/null +++ b/arch/src/main/scala/framework/acc/debug/build.sc @@ -0,0 +1,17 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("BankCheckDPI.scala")), + PathRef(millSourcePath / os.RelPath("Debug.scala")), + PathRef(millSourcePath / os.RelPath("trace/AntTraceDPI.scala")), + PathRef(millSourcePath / os.RelPath("trace/ITraceDPI.scala")), + PathRef(millSourcePath / os.RelPath("trace/MTraceDPI.scala")), + PathRef(millSourcePath / os.RelPath("trace/PMCTraceDPI.scala")), + PathRef(millSourcePath / os.RelPath("trace/Trace.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/acc/debug/trace/AntTraceDPI.scala b/arch/src/main/scala/framework/acc/debug/trace/AntTraceDPI.scala new file mode 100644 index 000000000..999fe189d --- /dev/null +++ b/arch/src/main/scala/framework/acc/debug/trace/AntTraceDPI.scala @@ -0,0 +1,47 @@ +package framework.acc.debug.trace + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.lib.dpi.DpiGuard + +@instantiable +class AntTraceDPI extends BlackBox with HasBlackBoxInline { + + @public + val io = IO(new Bundle { + val clock = Input(Clock()) + val reset = Input(Bool()) + val thread_id = Input(UInt(64.W)) + val pc = Input(UInt(64.W)) + val instruction = Input(UInt(32.W)) + val rd = Input(UInt(5.W)) + val data = Input(UInt(64.W)) + val enable = Input(Bool()) + }) + + setInline( + "AntTraceDPI.v", + """ + |module AntTraceDPI( + | input clock, input reset, input enable, + | input [63:0] thread_id, input [63:0] pc, + | input [31:0] instruction, input [4:0] rd, input [63:0] data + |); + |""".stripMargin + DpiGuard.wrapITrace( + """ + | import "DPI-C" context function void dpi_ant_trace( + | input int unsigned thread_lo, input int unsigned thread_hi, + | input int unsigned pc_lo, input int unsigned pc_hi, + | input int unsigned instruction, input int unsigned rd, + | input int unsigned data_lo, input int unsigned data_hi + | ); + | always @(posedge clock) begin + | if (!reset && enable) + | dpi_ant_trace(thread_id[31:0], thread_id[63:32], pc[31:0], pc[63:32], + | instruction, {27'b0, rd}, data[31:0], data[63:32]); + | end + |""".stripMargin + ) + "endmodule\n" + ) +} diff --git a/arch/src/main/scala/framework/frontend/globalrs/ITraceDPI.scala b/arch/src/main/scala/framework/acc/debug/trace/ITraceDPI.scala similarity index 86% rename from arch/src/main/scala/framework/frontend/globalrs/ITraceDPI.scala rename to arch/src/main/scala/framework/acc/debug/trace/ITraceDPI.scala index 08126e6c6..6e553ec43 100644 --- a/arch/src/main/scala/framework/frontend/globalrs/ITraceDPI.scala +++ b/arch/src/main/scala/framework/acc/debug/trace/ITraceDPI.scala @@ -1,16 +1,19 @@ -package framework.frontend.globalrs +package framework.acc.debug.trace import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} import chisel3.util._ -import framework.dpi.DpiGuard +import framework.lib.dpi.DpiGuard +@instantiable class ITraceDPI extends BlackBox with HasBlackBoxInline { + @public val io = IO(new Bundle { val clock = Input(Clock()) val reset = Input(Bool()) val is_issue = Input(UInt(8.W)) - val hart_id = Input(UInt(64.W)) + val thread_id = Input(UInt(64.W)) val rob_id = Input(UInt(32.W)) val inst_id = Input(UInt(64.W)) val domain_id = Input(UInt(32.W)) @@ -28,7 +31,7 @@ class ITraceDPI extends BlackBox with HasBlackBoxInline { | input clock, | input reset, | input [7:0] is_issue, - | input [63:0] hart_id, + | input [63:0] thread_id, | input [31:0] rob_id, | input [63:0] inst_id, | input [31:0] domain_id, @@ -41,8 +44,8 @@ class ITraceDPI extends BlackBox with HasBlackBoxInline { |""".stripMargin + DpiGuard.wrapITrace(""" | import "DPI-C" context function void dpi_itrace( | input int unsigned is_issue, - | input int unsigned hart_id_lo, - | input int unsigned hart_id_hi, + | input int unsigned thread_id_lo, + | input int unsigned thread_id_hi, | input int unsigned rob_id, | input int unsigned inst_id_lo, | input int unsigned inst_id_hi, @@ -56,7 +59,7 @@ class ITraceDPI extends BlackBox with HasBlackBoxInline { | input int unsigned rs2_data_hi | ); | reg [7:0] is_issue_reg; - | reg [63:0] hart_id_reg; + | reg [63:0] thread_id_reg; | reg [31:0] rob_id_reg; | reg [63:0] inst_id_reg; | reg [31:0] domain_id_reg; @@ -71,13 +74,13 @@ class ITraceDPI extends BlackBox with HasBlackBoxInline { | valid_reg <= 1'b0; | end else begin | if (valid_reg) begin - | dpi_itrace(is_issue_reg, hart_id_reg[31:0], hart_id_reg[63:32], rob_id_reg, inst_id_reg[31:0], inst_id_reg[63:32], domain_id_reg, funct_reg, pc_reg[31:0], pc_reg[63:32], rs1_data_reg[31:0], rs1_data_reg[63:32], rs2_data_reg[31:0], rs2_data_reg[63:32]); + | dpi_itrace({24'b0, is_issue_reg}, thread_id_reg[31:0], thread_id_reg[63:32], rob_id_reg, inst_id_reg[31:0], inst_id_reg[63:32], domain_id_reg, funct_reg, pc_reg[31:0], pc_reg[63:32], rs1_data_reg[31:0], rs1_data_reg[63:32], rs2_data_reg[31:0], rs2_data_reg[63:32]); | end | | valid_reg <= enable; | if (enable) begin | is_issue_reg <= is_issue; - | hart_id_reg <= hart_id; + | thread_id_reg <= thread_id; | rob_id_reg <= rob_id; | inst_id_reg <= inst_id; | domain_id_reg <= domain_id; diff --git a/arch/src/main/scala/framework/acc/debug/trace/MTraceDPI.scala b/arch/src/main/scala/framework/acc/debug/trace/MTraceDPI.scala new file mode 100644 index 000000000..a47177ad8 --- /dev/null +++ b/arch/src/main/scala/framework/acc/debug/trace/MTraceDPI.scala @@ -0,0 +1,118 @@ +package framework.acc.debug.trace + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.lib.dpi.DpiGuard + +@instantiable +class MTraceDPI extends BlackBox with HasBlackBoxInline { + + @public + val io = IO(new Bundle { + val clock = Input(Clock()) + val reset = Input(Bool()) + val is_write = Input(UInt(8.W)) + val is_shared = Input(UInt(8.W)) + val channel = Input(UInt(32.W)) + val thread_id = Input(UInt(64.W)) + val rob_id = Input(UInt(32.W)) + val inst_id = Input(UInt(64.W)) + val vbank_id = Input(UInt(32.W)) + val pbank_id = Input(UInt(32.W)) + val group_id = Input(UInt(32.W)) + val addr = Input(UInt(32.W)) + val write_mask = Input(UInt(32.W)) + val data_lo = Input(UInt(64.W)) + val data_hi = Input(UInt(64.W)) + val enable = Input(Bool()) + }) + + setInline( + "MTraceDPI.v", + """ + |module MTraceDPI( + | input clock, + | input reset, + | input [7:0] is_write, + | input [7:0] is_shared, + | input [31:0] channel, + | input [63:0] thread_id, + | input [31:0] rob_id, + | input [63:0] inst_id, + | input [31:0] vbank_id, + | input [31:0] pbank_id, + | input [31:0] group_id, + | input [31:0] addr, + | input [31:0] write_mask, + | input [63:0] data_lo, + | input [63:0] data_hi, + | input enable + |); + |""".stripMargin + DpiGuard.wrapMTrace(""" + | import "DPI-C" context function void dpi_mtrace( + | input int unsigned is_write, + | input int unsigned is_shared, + | input int unsigned channel, + | input int unsigned thread_id_lo, + | input int unsigned thread_id_hi, + | input int unsigned rob_id, + | input int unsigned inst_id_lo, + | input int unsigned inst_id_hi, + | input int unsigned vbank_id, + | input int unsigned pbank_id, + | input int unsigned group_id, + | input int unsigned addr, + | input int unsigned write_mask, + | input int unsigned data_lo_lo, + | input int unsigned data_lo_hi, + | input int unsigned data_hi_lo, + | input int unsigned data_hi_hi + | ); + | reg [7:0] is_write_reg; + | reg [7:0] is_shared_reg; + | reg [31:0] channel_reg; + | reg [63:0] thread_id_reg; + | reg [31:0] rob_id_reg; + | reg [63:0] inst_id_reg; + | reg [31:0] vbank_id_reg; + | reg [31:0] pbank_id_reg; + | reg [31:0] group_id_reg; + | reg [31:0] addr_reg; + | reg [31:0] write_mask_reg; + | reg [63:0] data_lo_reg; + | reg [63:0] data_hi_reg; + | reg valid_reg; + | + | always @(posedge clock) begin + | if (reset) begin + | valid_reg <= 1'b0; + | end else begin + | if (valid_reg) begin + | dpi_mtrace({24'b0, is_write_reg}, {24'b0, is_shared_reg}, channel_reg, thread_id_reg[31:0], thread_id_reg[63:32], rob_id_reg, inst_id_reg[31:0], inst_id_reg[63:32], vbank_id_reg, pbank_id_reg, group_id_reg, addr_reg, write_mask_reg, data_lo_reg[31:0], data_lo_reg[63:32], data_hi_reg[31:0], data_hi_reg[63:32]); + | end + | + | valid_reg <= enable; + | if (enable) begin + | is_write_reg <= is_write; + | is_shared_reg <= is_shared; + | channel_reg <= channel; + | thread_id_reg <= thread_id; + | rob_id_reg <= rob_id; + | inst_id_reg <= inst_id; + | vbank_id_reg <= vbank_id; + | pbank_id_reg <= pbank_id; + | group_id_reg <= group_id; + | addr_reg <= addr; + | write_mask_reg <= write_mask; + | data_lo_reg <= data_lo; + | data_hi_reg <= data_hi; + | end + | end + | end + |""".stripMargin) + + """ + |endmodule + """.stripMargin + ) +} diff --git a/arch/src/main/scala/framework/balldomain/bbus/pmc/PMCTraceDPI.scala b/arch/src/main/scala/framework/acc/debug/trace/PMCTraceDPI.scala similarity index 91% rename from arch/src/main/scala/framework/balldomain/bbus/pmc/PMCTraceDPI.scala rename to arch/src/main/scala/framework/acc/debug/trace/PMCTraceDPI.scala index 411dc25b0..43047b2e5 100644 --- a/arch/src/main/scala/framework/balldomain/bbus/pmc/PMCTraceDPI.scala +++ b/arch/src/main/scala/framework/acc/debug/trace/PMCTraceDPI.scala @@ -1,11 +1,15 @@ -package framework.balldomain.bbus.pmc +package framework.acc.debug.trace + +import chisel3.experimental.hierarchy.{instantiable, public} import chisel3._ import chisel3.util._ -import framework.dpi.DpiGuard +import framework.lib.dpi.DpiGuard +@instantiable class PMCTraceDPI extends BlackBox with HasBlackBoxInline { + @public val io = IO(new Bundle { val clock = Input(Clock()) val reset = Input(Bool()) diff --git a/arch/src/main/scala/framework/acc/debug/trace/Trace.scala b/arch/src/main/scala/framework/acc/debug/trace/Trace.scala new file mode 100644 index 000000000..d96d3ea6b --- /dev/null +++ b/arch/src/main/scala/framework/acc/debug/trace/Trace.scala @@ -0,0 +1,131 @@ +package framework.acc.debug.trace + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.thread.ant.Retire + +class InstructionEvent extends Bundle { + val is_issue = UInt(8.W) + val thread_id = UInt(64.W) + val rob_id = UInt(32.W) + val inst_id = UInt(64.W) + val domain_id = UInt(32.W) + val funct = UInt(32.W) + val pc = UInt(64.W) + val rs1_data = UInt(64.W) + val rs2_data = UInt(64.W) + val enable = Bool() +} + +class MemoryEvent extends Bundle { + val is_write = UInt(8.W) + val is_shared = UInt(8.W) + val channel = UInt(32.W) + val thread_id = UInt(64.W) + val rob_id = UInt(32.W) + val inst_id = UInt(64.W) + val vbank_id = UInt(32.W) + val pbank_id = UInt(32.W) + val group_id = UInt(32.W) + val addr = UInt(32.W) + val write_mask = UInt(32.W) + val data_lo = UInt(64.W) + val data_hi = UInt(64.W) + val enable = Bool() +} + +class AcceleratorEvents( + frontendParam: FrontendParam, + memParam: MemDomainParam, + gpParam: GpDomainParam, + ballParam: BallDomainParam) + extends Bundle { + + val memoryEvents = memParam.bankRequestPorts + + (if (gpParam.enable) gpParam.memory.memoryPorts else 0) + 2 + memParam.bankNum + val memory = Vec(memoryEvents, new MemoryEvent) + val ballStarted = Vec(ballParam.balls.size, Valid(UInt(log2Ceil(frontendParam.rob_entries).W))) + val ballCompleted = Vec(ballParam.balls.size, Valid(UInt(log2Ceil(frontendParam.rob_entries).W))) +} + +/** Owns trace sinks and performance counters for one hardware thread. */ +@instantiable +class Trace( + frontendParam: FrontendParam, + memParam: MemDomainParam, + gpParam: GpDomainParam, + ballParam: BallDomainParam, + threadId: Int) + extends Module { + + @public + val io = IO(new Bundle { + val instructions = Input(Vec(4, new InstructionEvent)) + val ant = Input(Valid(new Retire)) + val acc = Input(new AcceleratorEvents(frontendParam, memParam, gpParam, ballParam)) + }) + + val ant: Instance[AntTraceDPI] = Instantiate(new AntTraceDPI) + ant.io.clock := clock + ant.io.reset := reset.asBool + ant.io.thread_id := threadId.U(64.W) + ant.io.pc := io.ant.bits.pc + ant.io.instruction := io.ant.bits.instruction + ant.io.rd := io.ant.bits.rd + ant.io.data := io.ant.bits.data + ant.io.enable := io.ant.valid + + for (event <- io.instructions) { + val sink: Instance[ITraceDPI] = Instantiate(new ITraceDPI) + sink.io.clock := clock + sink.io.reset := reset.asBool + sink.io.is_issue := event.is_issue + sink.io.thread_id := threadId.U(64.W) + sink.io.rob_id := event.rob_id + sink.io.inst_id := event.inst_id + sink.io.domain_id := event.domain_id + sink.io.funct := event.funct + sink.io.pc := event.pc + sink.io.rs1_data := event.rs1_data + sink.io.rs2_data := event.rs2_data + sink.io.enable := event.enable + } + for (event <- io.acc.memory) { + val sink: Instance[MTraceDPI] = Instantiate(new MTraceDPI) + sink.io.clock := clock + sink.io.reset := reset.asBool + sink.io.is_write := event.is_write + sink.io.is_shared := event.is_shared + sink.io.channel := event.channel + sink.io.thread_id := threadId.U(64.W) + sink.io.rob_id := event.rob_id + sink.io.inst_id := event.inst_id + sink.io.vbank_id := event.vbank_id + sink.io.pbank_id := event.pbank_id + sink.io.group_id := event.group_id + sink.io.addr := event.addr + sink.io.write_mask := event.write_mask + sink.io.data_lo := event.data_lo + sink.io.data_hi := event.data_hi + sink.io.enable := event.enable + } + val cycle = RegInit(0.U(64.W)) + cycle := cycle + 1.U + val starts = Reg(Vec(io.acc.ballStarted.size, UInt(64.W))) + for (index <- io.acc.ballStarted.indices) { + when(io.acc.ballStarted(index).valid)(starts(index) := cycle) + val sink = Instantiate(new PMCTraceDPI) + sink.io.clock := clock + sink.io.reset := reset.asBool + sink.io.ball_id := index.U + sink.io.rob_id := io.acc.ballCompleted(index).bits + sink.io.elapsed := cycle - starts(index) + sink.io.enable := io.acc.ballCompleted(index).valid + } +} diff --git a/arch/src/main/scala/framework/acc/frontend/build.sc b/arch/src/main/scala/framework/acc/frontend/build.sc new file mode 100644 index 000000000..ccb8cf074 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/build.sc @@ -0,0 +1,32 @@ +import mill._ +import $file.src.main.scala.boot.{build => boot_build} +import $file.src.main.scala.scheduler.{build => scheduler_build} +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.frontend.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Frontend.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Ingress.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/RoCC.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/FrontendParam.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/decoder/DISA.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/decoder/GlobalDecoder.scala")) + ) ++ + boot_build.sources() ++ + scheduler_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + boot_build.emitSources() ++ + scheduler_build.emitSources() +) diff --git a/arch/src/main/scala/framework/acc/frontend/configs/default.toml b/arch/src/main/scala/framework/acc/frontend/configs/default.toml new file mode 100644 index 000000000..114197ff6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/configs/default.toml @@ -0,0 +1,9 @@ +bankAliasEnable = true +robEntries = 8 +rsOutOfOrderResponse = true +bankIdLen = 10 +vbankIdUpperBound = 23 +sharedBankIdBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/Cargo.lock b/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/Cargo.lock new file mode 100644 index 000000000..7ea1f5ec0 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "decoder-reference" +version = "0.1.0" diff --git a/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/Cargo.toml b/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/Cargo.toml new file mode 100644 index 000000000..201fb3abf --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "decoder-reference" +version = "0.1.0" +edition = "2021" +[lib] +crate-type = ["cdylib", "staticlib"] diff --git a/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/src/lib.rs b/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/src/lib.rs new file mode 100644 index 000000000..bace2123e --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/csrc/decoder/src/lib.rs @@ -0,0 +1,23 @@ +#[no_mangle] +pub extern "C" fn decoder_ref_domain( + funct: u32, + opcode: u32, + rs1: u64, + rvv: u32, + banks: u32, + bits: u32, +) -> u32 { + let bank = rs1 & ((1u64 << bits) - 1); + assert!(funct != 0 && ![0x57, 7, 0x27].contains(&opcode)); + if funct == 1 { + 0 + } else if [44, 79].contains(&funct) + || (rvv != 0 && funct == 32 && bank >= banks as u64 && bank < (banks + 2) as u64) + { + 2 + } else if [13, 16, 32, 33, 34, 35].contains(&funct) { + 1 + } else { + 3 + } +} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.cm_hier b/arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder.cm_hier similarity index 100% rename from arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.cm_hier rename to arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder.cm_hier diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder.f b/arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder.f new file mode 100644 index 000000000..bf67c3e35 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder.f @@ -0,0 +1,8 @@ ++incdir+@RESOURCES@ ++incdir+@RTL@ ++incdir+@VERIFY@/uvm/src/core ++define+BUCKYBALL_DISABLE_TRACE_DPI +@VERIFY@/uvm/src/core/scenario_if.sv +@VERIFY@/uvm/src/core/scenario_package.sv +-F @RTL@/GlobalDecoder/filelist.f +@RESOURCES@/decoder_tb.sv diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder_tb.sv b/arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder_tb.sv new file mode 100644 index 000000000..71e67f290 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/resources/decoder/decoder_tb.sv @@ -0,0 +1,75 @@ +module decoder_tb; + import uvm_pkg::*; + import scenario_pkg::*; + `include "uvm_macros.svh" + logic clock = 0, reset = 1; + always #5 clock = ~clock; + scenario_if ctl (clock); + `include "decoder_signals.svh" +GlobalDecoder dut ( + `include "decoder_ports.svh" + ); + `include "decoder_clocking.svh" + int checks = 0; + function automatic void check(bit good, string message); + verification_pkg::evidence::checks++; + checks++; + if (!good) `uvm_fatal("DECODER", message) + endfunction + `include "decoder_config.svh" + import "DPI-C" function int unsigned decoder_ref_domain( + int unsigned funct, + int unsigned opcode, + longint unsigned rs1, + int unsigned rvv, + int unsigned banks, + int unsigned bits + ); + task automatic decode(int funct, int expected_bid, longint unsigned rs1, longint unsigned rs2); + io_id_i_valid = 1; + io_id_i_bits_cmd_funct = funct; + io_id_i_bits_cmd_opcode = 'h7b; + io_id_i_bits_cmd_rs1Data = rs1; + io_id_i_bits_cmd_rs2Data = rs2; + io_id_o_ready = 0; + repeat (2) begin + @(sample); + check(sample.io_id_o_valid && !sample.io_id_i_ready, "decode backpressure mismatch"); + check(sample.io_id_o_bits_domain_id == decoder_ref_domain( + funct, 'h7b, rs1, `DECODER_RVV, `DECODER_VBANKS, `DECODER_BANK_BITS), + "domain decode mismatch"); + check( + sample.io_id_o_bits_ball_bid == expected_bid && sample.io_id_o_bits_cmd_rs1Data == rs1 && sample.io_id_o_bits_cmd_rs2Data == rs2, + "instruction payload mismatch"); + @(negedge clock); + end + io_id_o_ready = 1; + @(sample); + check(sample.io_id_i_ready, "decoder did not accept ready output"); + @(negedge clock); + io_id_i_valid = 0; + endtask + + initial begin + uvm_config_db#(virtual scenario_if)::set(null, "*", "vif", ctl); + uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 2ms); + run_test("protocol_test"); + end + initial begin + `include "decoder_init.svh" + wait (ctl.start); + repeat (3) @(sample); + @(negedge clock); + reset = 0; + decode(1, 0, 0, 0); + decode(32, 0, 0, 1056); + decode(33, 0, 2 << 20, 16); + decode(34, 0, (64'h1 << 63) | (2 << 20), 16); + decode(35, 0, 0, 16); + decode(16, 0, 0, 16); + decode(13, 0, 0, 0); + `include "decoder_cases.svh" +`uvm_info("DECODER", $sformatf("%0d checked outputs", checks), UVM_LOW) + ctl.done = 1; + end +endmodule diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/Emit.scala new file mode 100644 index 000000000..c3e49b0e9 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/Emit.scala @@ -0,0 +1,33 @@ +package framework.acc.frontend + +import sims.configs.SimParam +import _root_.configs.GlobalConfig + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Frontend( + thread.acc.get.frontend, + thread.acc.get.memDomain, + thread.acc.get.gpDomain, + thread.acc.get.ballDomain, + core.privileged.permission, + b.sim.copy(diffTest = false) + ), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/Frontend.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/Frontend.scala new file mode 100644 index 000000000..c6b2e18f0 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/Frontend.scala @@ -0,0 +1,97 @@ +package framework.acc.frontend + +import sims.configs.SimParam +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +import framework.acc.debug.FrontendDebugPort + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.frontend.boot.BootRom +import framework.acc.frontend.decoder.GlobalDecoder +import framework.acc.memdomain.bank_router.BankMapping +import framework.acc.frontend.scheduler.{Complete, Issue, MemoryIssue, Scheduler} +import framework.acc.memdomain.{Response => MemoryResponse} + +class Command(dataBits: Int) extends Bundle { + + val raw_inst = UInt(32.W) + val pc = UInt(dataBits.W) + val rs1Data = UInt(dataBits.W) + val rs2Data = UInt(dataBits.W) +} + +/** Frontend composition; boundary ports connect directly to their owning child IP. */ +@instantiable +class Frontend( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val ballParam: BallDomainParam, + val permissionParam: PmpParams, + val simp: SimParam) + extends Module { + val diffTest = simp.diffTest + val commandDataBits = frontendParam.commandDataBits + val ownerId = frontendParam.ownerId + + @public + val io = IO(new Bundle { + + val rocc = new RoCCIO(commandDataBits, permissionParam) + + // Request payloads leave Frontend; each ready signal returns from its domain. + val ballRequest = Decoupled(new MemoryIssue(frontendParam, memParam, permissionParam)) + // Completion payloads enter Frontend; Frontend drives their ready signals. + val ballCompletion = Flipped(Decoupled(new MemoryResponse(frontendParam))) + + // Bank-version update request to MemDomain; its ready input gates the update. + val mappingUpdate = Decoupled(new BankMapping(frontendParam)) + val memoryRequest = Decoupled(new MemoryIssue(frontendParam, memParam, permissionParam)) + val memoryCompletion = Flipped(Decoupled(new MemoryResponse(frontendParam))) + + val gpRequest = Option.when(gpParam.enable)(Decoupled(new Issue(frontendParam, memParam))) + val gpCompletion = Option.when(gpParam.enable)(Flipped(Decoupled(new Complete(frontendParam)))) + + val barrier = new Bundle { + val arrive = Output(Bool()) + val release = Input(Bool()) + } + + val debug = new FrontendDebugPort(frontendParam, simp) + }) + + val ingress: Instance[Ingress] = Instantiate(new Ingress(permissionParam, frontendParam)) + val boot: Instance[BootRom] = Instantiate(new BootRom(ballParam.balls.map(_.id), permissionParam, frontendParam)) + val gDecoder: Instance[GlobalDecoder] = + Instantiate(new GlobalDecoder(frontendParam, memParam, gpParam, permissionParam)) + val scheduler: Instance[Scheduler] = + Instantiate(new Scheduler(frontendParam, memParam, gpParam, permissionParam, simp)) + + io.rocc <> ingress.io.rocc + boot.io.command <> ingress.io.command + ingress.io.fault := scheduler.io.fault + ingress.io.idle := boot.io.idle + gDecoder.io.command <> boot.io.cmd + scheduler.io.admission.command <> gDecoder.io.decoded + boot.io.schedulerIdle := scheduler.io.admission.idle + + io.ballRequest <> scheduler.io.ballRequest + scheduler.io.ballCompletion <> io.ballCompletion + io.memoryRequest <> scheduler.io.memoryRequest + scheduler.io.memoryCompletion <> io.memoryCompletion + io.gpRequest.zip(scheduler.io.gpRequest).foreach { case (port, source) => port <> source } + io.gpCompletion.zip(scheduler.io.gpCompletion).foreach { case (port, sink) => sink <> port } + io.debug.trace := scheduler.io.traceInstructions + io.debug.control.zip(scheduler.io.debug).foreach { case (port, source) => port <> source } + io.mappingUpdate <> scheduler.io.mappingUpdate + + io.barrier.arrive := scheduler.io.barrier.arrive + scheduler.io.barrier.release := io.barrier.release + +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/Ingress.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/Ingress.scala new file mode 100644 index 000000000..b9335d7d1 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/Ingress.scala @@ -0,0 +1,56 @@ +package framework.acc.frontend + +import framework.core.memory.access.preflight.configs.PmpParams +import framework.acc.frontend.configs.FrontendParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.memdomain.MemoryContext + +class CommandPacket(permissionParam: PmpParams, fp: FrontendParam) extends Bundle { + val commandDataBits = fp.commandDataBits + val instruction = new Command(commandDataBits) + val context = new MemoryContext(permissionParam) + val runtime = Bool() +} + +/** Holds an accepted RoCC command until the frontend takes it. */ +@instantiable +class Ingress(permissionParam: PmpParams, fp: FrontendParam) extends Module { + val commandDataBits = fp.commandDataBits + + @public + val io = IO(new Bundle { + val rocc = new RoCCIO(commandDataBits, permissionParam) + val command = Decoupled(new CommandPacket(permissionParam, fp)) + val idle = Input(Bool()) + val fault = Input(Bool()) + }) + + val pending = RegInit(false.B) + val packet = Reg(new CommandPacket(permissionParam, fp)) + io.rocc.cmd.ready := !pending && !io.fault && !reset.asBool + when(io.rocc.cmd.fire) { + assert(io.rocc.cmd.bits.instruction.raw_inst(6, 0) === "h7b".U, "RoCC accepts accelerator commands only") + assert( + !io.rocc.cmd.bits.instruction.raw_inst(14), + "Accelerator commands report completion without register responses" + ) + packet.instruction := io.rocc.cmd.bits.instruction + packet.context.satp := io.rocc.cmd.bits.satp + packet.context.privilege := io.rocc.cmd.bits.effectivePrivilege + packet.context.sum := io.rocc.cmd.bits.sum + packet.context.mxr := io.rocc.cmd.bits.mxr + packet.context.pmp := io.rocc.cmd.bits.pmp + packet.runtime := true.B + pending := true.B + } + io.command.valid := pending && !io.fault + io.command.bits := packet + when(io.command.fire)(pending := false.B) + io.rocc.busy := pending || !io.idle || io.fault + io.rocc.interrupt := io.fault + io.rocc.resp.valid := false.B + io.rocc.resp.bits := 0.U.asTypeOf(io.rocc.resp.bits) +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/RoCC.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/RoCC.scala new file mode 100644 index 000000000..ad3a73d17 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/RoCC.scala @@ -0,0 +1,30 @@ +package framework.acc.frontend + +import chisel3._ +import chisel3.util._ +import framework.core.memory.access.preflight.PmpEntry +import framework.core.memory.access.preflight.configs.PmpParams + +class Response(dataBits: Int) extends Bundle { + + val rd = UInt(5.W) + val data = UInt(dataBits.W) +} + +/** RoCC instruction operands and the frozen submitting address-space context. */ +class RoCCCommand(dataBits: Int, permission: PmpParams) extends Bundle { + val instruction = new Command(dataBits) + val satp = UInt(64.W) + val effectivePrivilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + val pmp = Vec(permission.entries, new PmpEntry(permission)) +} + +/** Accelerator-facing RoCC direction; the host uses Flipped. */ +class RoCCIO(dataBits: Int, permission: PmpParams) extends Bundle { + val cmd = Flipped(Decoupled(new RoCCCommand(dataBits, permission))) + val resp = Decoupled(new Response(dataBits)) + val busy = Output(Bool()) + val interrupt = Output(Bool()) +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/build.sc b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/build.sc new file mode 100644 index 000000000..827b8bd4d --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/build.sc @@ -0,0 +1,20 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.settings +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.frontend.boot.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Boot.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/configs/uvm.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/configs/uvm.toml new file mode 100644 index 000000000..76af217fb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/configs/uvm.toml @@ -0,0 +1,21 @@ +[parameters] +ballIds = [0, 3, 31] + +[parameters.permission] +addressBits = 44 +entries = 16 + +[parameters.frontend] +ownerId = 0 +commandDataBits = 64 +rob_entries = 4 +bank_alias_enable = true +rs_out_of_order_response = true +bank_id_len = 4 +vbank_id_upper_bound = 3 +shared_bank_id_base = 16 +iter_len = 16 +sub_rob_enable = false +sub_rob_depth = 4 + +[defines] diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..87656e013 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual boot_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual boot_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Boot interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.inputs = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/item.svh new file mode 100644 index 000000000..8b1798268 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`BOOT_INPUT_BITS-1:0] inputs; + logic [`BOOT_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + logic [`BOOT_PACKET_BITS-1:0] payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..1959ba4bb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual boot_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual boot_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Boot monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/env/env.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/env/env.svh new file mode 100644 index 000000000..f1fc73e22 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/env/env.svh @@ -0,0 +1,31 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver drv; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + reference.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/env/model.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/env/model.svh new file mode 100644 index 000000000..8ac568d0f --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/env/model.svh @@ -0,0 +1,100 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + int ball_ids[`BOOT_BALL_COUNT] = `BOOT_BALL_IDS; + bit start_epoch = 1, waiting = 0, runtime_phase = 0; + int seen = 0; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + covergroup behavior with function sample ( + bit runtime_command, int target, bit input_stall, bit output_stall, bit scheduler_idle + ); + option.per_instance = 1; + command_kind: coverpoint runtime_command; + initialization_target: coverpoint target iff (!runtime_command) { + bins selected[] = `BOOT_ID_BINS; + } + source_backpressure: coverpoint input_stall; + sink_backpressure: coverpoint output_stall; + scheduler_drained: coverpoint scheduler_idle; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + behavior = new(); + endfunction + function void write(cycle_item t); + result_item expected, actual; + bit input_fire, output_fire, scheduler_idle; + if (t.reset) begin + reset_ap.write(1); + start_epoch = 1; + waiting = 0; + runtime_phase = 0; + seen = 0; + return; + end + if (start_epoch) begin + foreach (ball_ids[i]) begin + expected = result_item::type_id::create("initialization"); + expected.payload = '0; + expected.payload[`BOOT_OUT_CMD_BITS_INSTRUCTION_RAW_INST_OFFSET-`BOOT_OUT_PACKET_OFFSET+:32] = 32'h0a00007b; + expected.payload[`BOOT_OUT_CMD_BITS_INSTRUCTION_RS2DATA_OFFSET-`BOOT_OUT_PACKET_OFFSET+:`BOOT_OUT_CMD_BITS_INSTRUCTION_RS2DATA_WIDTH] = ball_ids[i]; + expected_ap.write(expected); + end + start_epoch = 0; + end + input_fire = `BI(t.inputs, COMMAND_VALID) && `BO(t.outputs, COMMAND_READY); + output_fire = `BO(t.outputs, CMD_VALID) && `BI(t.inputs, CMD_READY); + scheduler_idle = `BI(t.inputs, SCHEDULERIDLE); + if (seen < `BOOT_BALL_COUNT) + evidence::check(!`BO(t.outputs, COMMAND_READY), + "Boot accepted runtime input before initialization ended"); + if (waiting) begin + if (seen < `BOOT_BALL_COUNT || !scheduler_idle) + evidence::check(!`BO(t.outputs, CMD_VALID), + "Boot issued a command before prior work drained"); + if (scheduler_idle) waiting = 0; + end + if (seen == `BOOT_BALL_COUNT && !waiting && ( + `BO(t.outputs, COMMAND_READY) + || + `BO(t.outputs, IDLE) + )) + runtime_phase = 1; + if (runtime_phase) begin + evidence::check(`BO(t.outputs, COMMAND_READY) === `BI(t.inputs, CMD_READY), + "Boot runtime ready did not pass through"); + evidence::check(`BO(t.outputs, CMD_VALID) === `BI(t.inputs, COMMAND_VALID), + "Boot runtime valid did not pass through"); + end + if (input_fire) begin + evidence::check(seen == `BOOT_BALL_COUNT && !waiting, + "Boot forwarded a runtime command before initialization drained"); + expected = result_item::type_id::create("runtime"); + expected.payload = t.inputs[`BOOT_IN_PACKET_OFFSET+:`BOOT_PACKET_BITS]; + expected_ap.write(expected); + end + if (output_fire) begin + actual = result_item::type_id::create("actual"); + actual.payload = t.outputs[`BOOT_OUT_PACKET_OFFSET+:`BOOT_PACKET_BITS]; + if (seen < `BOOT_BALL_COUNT) begin + behavior.sample(0, ball_ids[seen], `BI(t.inputs, COMMAND_VALID) && ! + `BO(t.outputs, COMMAND_READY), 0, scheduler_idle); + seen++; + waiting = 1; + end else begin + evidence::check(input_fire, "Boot produced an unsolicited runtime output"); + behavior.sample(1, 0, 0, 0, scheduler_idle); + end + actual_ap.write(actual); + end + if (!output_fire) + behavior.sample(1, 0, `BI(t.inputs, COMMAND_VALID) && !`BO(t.outputs, COMMAND_READY), + `BO(t.outputs, CMD_VALID) && !`BI(t.inputs, CMD_READY), scheduler_idle); + if (`BO(t.outputs, IDLE)) + evidence::check(seen == `BOOT_BALL_COUNT && scheduler_idle && !`BI(t.inputs, COMMAND_VALID) + && !`BO(t.outputs, CMD_VALID), "Boot declared idle before all work drained"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/filelist.f b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/filelist.f new file mode 100644 index 000000000..c357eb6a5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/BootRom/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/interface.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/interface.sv new file mode 100644 index 000000000..71dc6d8a5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/interface.sv @@ -0,0 +1,14 @@ +`include "parameters.svh" +interface boot_if ( + input logic clock +); + logic reset = 1; + logic [`BOOT_INPUT_BITS-1:0] inputs; + logic [`BOOT_OUTPUT_BITS-1:0] outputs; + assert property (@(posedge clock) disable iff (reset) + outputs[`BOOT_OUT_CMD_VALID_OFFSET] && !inputs[`BOOT_IN_CMD_READY_OFFSET] |=> + outputs[`BOOT_OUT_CMD_VALID_OFFSET] && $stable( + outputs[`BOOT_OUT_PACKET_OFFSET+:`BOOT_PACKET_BITS] + )) + else $fatal(1, "Boot output changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/package.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/package.sv new file mode 100644 index 000000000..ed6ff959c --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/package.sv @@ -0,0 +1,15 @@ +`include "parameters.svh" +`define BI(P, N) P[`BOOT_IN_``N``_OFFSET+:`BOOT_IN_``N``_WIDTH] +`define BO(P, N) P[`BOOT_OUT_``N``_OFFSET+:`BOOT_OUT_``N``_WIDTH] +package boot_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..c7441adc5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/seq/sequence.svh @@ -0,0 +1,69 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + logic [`BOOT_INPUT_BITS-1:0] pins = '0; + logic [`BOOT_OUTPUT_BITS-1:0] observed; + bit reset = 1; + int pending_work = 0, cycle = 0; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task tick(int cycles = 1); + cycle_item req; + repeat (cycles) begin + `BI(pins, SCHEDULERIDLE) = pending_work == 0; + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.inputs = pins; + req.reset = reset; + finish_item(req); + observed = req.outputs; + if (pending_work > 0) pending_work--; + if (!reset && `BO(observed, CMD_VALID) && `BI(pins, CMD_READY)) pending_work = 3; + cycle++; + end + endtask + task restart(); + pins = '0; + reset = 1; + pending_work = 0; + tick(2); + reset = 0; + tick(); + endtask + task send(logic [`BOOT_PACKET_BITS-1:0] payload); + pins[`BOOT_IN_PACKET_OFFSET+:`BOOT_PACKET_BITS] = payload; + `BI(pins, COMMAND_VALID) = 1; + do begin + `BI(pins, CMD_READY) = cycle % 3 == 0; + tick(); + end while (! + `BO(observed, COMMAND_READY) + ); + `BI(pins, COMMAND_VALID) = 0; + endtask + task drain(); + `BI(pins, CMD_READY) = 1; + do tick(); while (!`BO(observed, IDLE)); + endtask + task body(); + logic [`BOOT_PACKET_BITS-1:0] packet; + restart(); + // A runtime producer may hold VALID while the ROM is still initializing balls. + packet = '1; + packet[`BOOT_IN_COMMAND_BITS_INSTRUCTION_RAW_INST_OFFSET-`BOOT_IN_PACKET_OFFSET+:32] = 32'h4a00007b; + send(packet); + drain(); + for (int bit_index = 0; bit_index < `BOOT_PACKET_BITS; bit_index++) begin + packet = (`BOOT_PACKET_BITS'(1) << bit_index) ^ ((bit_index % 2) ? '1 : '0); + packet[`BOOT_IN_COMMAND_BITS_INSTRUCTION_RAW_INST_OFFSET-`BOOT_IN_PACKET_OFFSET+:32] = (32'(bit_index%128)<<25) | 'h7b; + send(packet); + end + drain(); + // Restart after only one initialization command; the remaining expected commands are cancelled. + restart(); + `BI(pins, CMD_READY) = 1; + do tick(); while (!`BO(observed, CMD_VALID)); + restart(); + drain(); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/suite.toml new file mode 100644 index 000000000..54539e5fb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "boot" +top = "boot_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["boot_pkg::model::behavior"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/tb.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/tb.sv new file mode 100644 index 000000000..3cd0e2dfb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/tb.sv @@ -0,0 +1,16 @@ +module boot_tb; + import uvm_pkg::*; + import boot_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + boot_if bus (clock); + BootRom dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual boot_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/resources/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/scala/Boot.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/scala/Boot.scala new file mode 100644 index 000000000..0d6e27a31 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/scala/Boot.scala @@ -0,0 +1,93 @@ +package framework.acc.frontend.boot + +import framework.acc.frontend.configs.FrontendParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} +import chisel3.util._ +import framework.acc.balldomain.decoder.DISA.BALL_INIT_BITPAT +import framework.acc.frontend.CommandPacket + +@instantiable +class BootRom(val ballIds: Seq[Int], val permissionParam: PmpParams, val fp: FrontendParam) extends Module { + val commandDataBits = fp.commandDataBits + + val commandBits = commandDataBits + + @public + val io = IO(new Bundle { + + val command = Flipped(Decoupled(new CommandPacket(permissionParam, fp))) + val cmd = Decoupled(new CommandPacket(permissionParam, fp)) + val schedulerIdle = Input(Bool()) + val idle = Output(Bool()) + }) + + val ballInitRecords = ballIds.map { id => + require(id >= 0 && id < 32, s"BALL_INIT target ID out of range: $id") + BuckyballCommand(BALL_INIT_BITPAT.value.toInt, rs1 = 0, rs2 = BigInt(id)) + } + + val bootRecords = ballInitRecords + + val bootFuncts = VecInit(bootRecords.map(r => r.funct.U(7.W))) + val bootRs1Data = VecInit(bootRecords.map(r => r.rs1.U(commandBits.W))) + val bootRs2Data = VecInit(bootRecords.map(r => r.rs2.U(commandBits.W))) + val bootPcWidth = math.max(1, log2Ceil(bootRecords.length + 1)) + + val active = RegInit(true.B) + val drain = RegInit(false.B) + val waitIdle = RegInit(false.B) + val pc = RegInit(0.U(bootPcWidth.W)) + + val atEnd = pc === bootRecords.length.U(bootPcWidth.W) + + val current = Wire(new CommandPacket(permissionParam, fp)) + current := 0.U.asTypeOf(new CommandPacket(permissionParam, fp)) + current.instruction.raw_inst := Cat( + Mux(atEnd, 0.U(7.W), bootFuncts(pc)), + 0.U(18.W), + BuckyballCommand.Custom3Opcode.U(7.W) + ) + current.instruction.rs1Data := Mux(atEnd, 0.U, bootRs1Data(pc)) + current.instruction.rs2Data := Mux(atEnd, 0.U, bootRs2Data(pc)) + + val injectValid = active && !drain && !waitIdle && !atEnd + + when(active && !drain && atEnd) { + drain := true.B + } + when(waitIdle && io.schedulerIdle) { + waitIdle := false.B + } + when(drain && io.schedulerIdle) { + active := false.B + } + when(active && io.cmd.fire) { + pc := pc + 1.U + waitIdle := true.B + } + + // External commands enter only after all reset-time initialization has retired. + io.cmd.valid := Mux(active, injectValid, io.command.valid) + io.cmd.bits := Mux(active, current, io.command.bits) + io.command.ready := !active && io.cmd.ready + io.idle := !active && io.schedulerIdle && !io.command.valid + + when(io.command.fire) { + assert(!io.command.bits.instruction.raw_inst(14), "Frontend commands must not request a register response") + } +} + +/** + * Elaboration-time representation of one Buckyball custom instruction. + * Domain ISA builders own the funct7 and operand encodings; boot only streams + * these already-formed commands. + */ +case class BuckyballCommand(funct: Int, rs1: BigInt = 0, rs2: BigInt = 0) + +object BuckyballCommand { + val Custom3Opcode = 0x7b + val Custom3Funct3 = 3 +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/scala/Emit.scala new file mode 100644 index 000000000..4cd4a4b88 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/src/main/scala/Emit.scala @@ -0,0 +1,60 @@ +package framework.acc.frontend.boot + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.acc.frontend.configs.FrontendParam +import framework.core.memory.access.preflight.configs.PmpParams + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val frontend = upickle.default.read[FrontendParam](values("frontend")) + val permission = upickle.default.read[PmpParams](values("permission")) + val ballIds = values("ballIds").arr.toSeq.map(_.num.toInt) + Files.createDirectories(Paths.get(output)) + val ports = scala.collection.mutable.ArrayBuffer[String]() + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0 + var outputOffset = 0 + var packetBits = 0 + + def fields(data: Data, path: String): Seq[(String, Int)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(path -> leaf.getWidth) + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new BootRom(ballIds, permission, frontend) + packetBits = dut.io.command.bits.getWidth + for ((name, width) <- fields(dut.io, "")) { + val input = + name == "cmd_ready" || name == "schedulerIdle" || name == "command_valid" || name.startsWith("command_bits_") + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define BOOT_${direction}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define BOOT_${direction}_${name.toUpperCase}_WIDTH $width" + offsets(name) = offset + ports += s".io_$name(bus.$signal[$offset +: $width])" + if (input) inputOffset += width else outputOffset += width + } + dut + }, + args = Array("--target-dir", s"$output/BootRom", "--split-verilog") + ) + definitions += s"`define BOOT_INPUT_BITS $inputOffset" + definitions += s"`define BOOT_OUTPUT_BITS $outputOffset" + definitions += s"`define BOOT_PACKET_BITS $packetBits" + definitions += s"`define BOOT_IN_PACKET_OFFSET ${offsets.filter(_._1.startsWith("command_bits_")).values.min}" + definitions += s"`define BOOT_OUT_PACKET_OFFSET ${offsets.filter(_._1.startsWith("cmd_bits_")).values.min}" + definitions += s"`define BOOT_BALL_COUNT ${ballIds.size}" + definitions += s"`define BOOT_BALL_IDS ${ballIds.mkString("'{", ",", "}")}" + definitions += s"`define BOOT_ID_BINS ${ballIds.mkString("{", ",", "}")}" + Files.writeString(Paths.get(output, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/configs/FrontendParam.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/configs/FrontendParam.scala new file mode 100644 index 000000000..a13a88996 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/configs/FrontendParam.scala @@ -0,0 +1,45 @@ +package framework.acc.frontend.configs + +import buckyball.config.FrontendConfig + +import upickle.default._ + +/** + * Frontend parameters - includes all frontend configuration. + */ +case class FrontendParam( + ownerId: Int, + commandDataBits: Int, + rob_entries: Int, + bank_alias_enable: Boolean, + rs_out_of_order_response: Boolean, + bank_id_len: Int, + vbank_id_upper_bound: Int, + /** ISA bank-id namespace base for shared banks: raw id >= this means shared. */ + shared_bank_id_base: Int, + iter_len: Int, + sub_rob_enable: Boolean, + sub_rob_depth: Int) + +object FrontendParam { + + def fromProto(p: FrontendConfig, commandDataBits: Int, ownerId: Int): FrontendParam = { + require(p.hasBankAliasEnable, "Frontend bank alias policy is required") + FrontendParam( + ownerId, + commandDataBits, + p.getRobEntries, + p.getBankAliasEnable, + p.getRsOutOfOrderResponse, + p.getBankIdLen, + p.getVbankIdUpperBound, + p.getSharedBankIdBase, + p.getIterLen, + p.getSubRobEnable, + p.getSubRobDepth + ) + } + + implicit val rw: ReadWriter[FrontendParam] = macroRW + +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/DISA.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/DISA.scala new file mode 100644 index 000000000..a0258e9f3 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/DISA.scala @@ -0,0 +1,16 @@ +package framework.acc.frontend.decoder + +import chisel3._ +import chisel3.util._ + +object DISA { + val BARRIER_BITPAT = BitPat("b0000001") // 1 (0x01) — enable=000, opcode=1 +} + +// Domain ID constants +object DomainId { + val FRONTEND = 0.U(4.W) // Frontend (barrier), does not enter ROB queue + val GP = 2.U(4.W) // Private kernel execution and image control + val MEM = 1.U(4.W) // Memory domain + val BALL = 3.U(4.W) // Ball domain +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/GlobalDecoder.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/GlobalDecoder.scala new file mode 100644 index 000000000..3aef67780 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/GlobalDecoder.scala @@ -0,0 +1,178 @@ +package framework.acc.frontend.decoder + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.frontend.scheduler.scoreboard.BankAccessInfo + +import framework.acc.frontend.{Command, CommandPacket} +import framework.acc.memdomain.MemoryContext +import framework.acc.frontend.decoder.DISA._ +import framework.acc.gpdomain.decoder.DISA._ +import framework.acc.memdomain.DISA._ + +class PostGDCmd(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val commandDataBits = frontendParam.commandDataBits + + val runtime = Bool() + val domain_id = UInt(4.W) + val iter = UInt(frontendParam.iter_len.W) + val cmd = new Command(commandDataBits) + val bankAccess = new BankAccessInfo(frontendParam.bank_id_len) + val architecturalDestination = UInt(frontendParam.bank_id_len.W) + val op1_col = UInt(memParam.groupCountWidth.W) + val op2_col = UInt(memParam.groupCountWidth.W) + val wr_col = UInt(memParam.groupCountWidth.W) + val wr_read_bank = UInt(frontendParam.bank_id_len.W) + val isBarrier = Bool() +} + +/** Only the frontend pipeline carries address-space context for later memory dispatch. */ +class DecodedInstruction(frontendParam: FrontendParam, memParam: MemDomainParam, permissionParam: PmpParams) + extends Bundle { + val commandDataBits = frontendParam.commandDataBits + val operation = new PostGDCmd(frontendParam, memParam) + val context = new MemoryContext(permissionParam) +} + +@instantiable +class GlobalDecoder( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val permissionParam: PmpParams) + extends Module { + val commandDataBits = frontendParam.commandDataBits + + val bankIdLen = frontendParam.bank_id_len + + @public + val io = IO(new Bundle { + + val command = Flipped(Decoupled(new CommandPacket(permissionParam, frontendParam))) + + val decoded = Decoupled(new DecodedInstruction(frontendParam, memParam, permissionParam)) + }) + + // Decoding follows the consumer handshake. + io.command.ready := io.decoded.ready + + val func7 = io.command.bits.instruction.raw_inst(31, 25) + val opcode = io.command.bits.instruction.raw_inst(6, 0) + val rs1 = io.command.bits.instruction.rs1Data + + // Route memory and GP commands to their owning domains. + val is_mem_inst = (func7 === MVIN_MMIO_BITPAT) || (func7 === MVIN_BITPAT) || + (func7 === MVIN_2D_BITPAT) || + (func7 === MVOUT_BITPAT) || + (func7 === MSET_BITPAT) || + (func7 === MVOVER_BITPAT) || + (func7 === MVIN_KERNEL_BITPAT) + + val is_barrier_inst = func7 === BARRIER_BITPAT + + val is_kernel_inst = func7 === RUN_KERNEL_BITPAT + val is_bare_rvv = opcode === "h57".U || opcode === "h07".U || opcode === "h27".U + val is_ball_inst = !is_mem_inst && !is_barrier_inst && !is_bare_rvv && !is_kernel_inst + + when(io.command.fire) { + assert(func7 =/= 0.U, "GlobalDecoder: funct7 zero is not a Buckyball instruction") + assert(!is_bare_rvv, "GlobalDecoder: bare RVV instructions are not Buckyball commands") + when(is_kernel_inst) { + assert(gpParam.enable.B, "GlobalDecoder: kernel command requires gpDomain.enable=true") + } + } + + // Encode domain ID + val domain_id = MuxCase( + DomainId.BALL, + Seq( + is_barrier_inst -> DomainId.FRONTEND, + is_kernel_inst -> DomainId.GP, + is_mem_inst -> DomainId.MEM, + is_ball_inst -> DomainId.BALL + ) + ) + + // ------------------------------------------------------------------------- + // Bank access info extraction — enable flags from funct7[6:4] + // + // Unified rs1 layout (defined in isa.h): + // rs1[9:0] = bank_0 (rd_bank_0 or MSET config bank) + // rs1[19:10] = bank_1 (rd_bank_1, dual-operand only) + // rs1[29:20] = bank_2 (wr_bank for Ball instructions) + // rs1[62:30] = iter; rs1[63] declares an in-place update. + // + // funct7[6:4] enable encoding: + // 000 = no bank access + // 001 = 1 read (bank0) + // 010 = 1 write (bank2) + // 011 = 1 read + 1 write (bank0 read, bank2 write) + // 100 = 2 read + 1 write (bank0+bank1 read, bank2 write) + // 101,110,111 = no bank access (extended opcode space) + // ------------------------------------------------------------------------- + val bankAccess = Wire(new BankAccessInfo(frontendParam.bank_id_len)) + val enableBits = func7(6, 4) + + // Decode enable from funct7[6:4] + val msetTransfer = func7 === MSET_BITPAT && io.command.bits.instruction.rs2Data(12) + val hasRd0 = enableBits === 1.U || enableBits === 3.U || enableBits === 4.U || msetTransfer + val hasRd1 = enableBits === 4.U || msetTransfer + val hasWr = (enableBits === 2.U || enableBits === 3.U || enableBits === 4.U) && func7 =/= MVIN_MMIO_BITPAT + + bankAccess.rd_bank_0_valid := hasRd0 + bankAccess.rd_bank_0_id := rs1(bankIdLen - 1, 0) + bankAccess.rd_bank_1_valid := hasRd1 + bankAccess.rd_bank_1_id := Mux(msetTransfer, rs1(bankIdLen + 19, 20), rs1(bankIdLen + 9, 10)) + bankAccess.wr_bank_valid := hasWr + + val inplaceUpdate = hasWr && func7 =/= MSET_BITPAT && + (rs1( + 63 + ) || is_kernel_inst || func7 === MVIN_2D_BITPAT || (func7 === MVIN_BITPAT && io.command.bits.instruction.rs2Data(63))) + + val readOldWrite = hasWr && func7 =/= MSET_BITPAT && (rs1(29) || inplaceUpdate) + val writeBank = rs1(28, 20).pad(bankIdLen) + bankAccess.wr_bank_id := Mux( + func7 === MSET_BITPAT, + Mux(msetTransfer, rs1(bankIdLen + 19, 20), rs1(bankIdLen - 1, 0)), + writeBank + ) + bankAccess.rd_wr_valid := readOldWrite + bankAccess.rd_wr_id := writeBank + bankAccess.wr_update := inplaceUpdate + + def legalBank(raw: UInt): Bool = + raw <= frontendParam.vbank_id_upper_bound.U || + (gpParam.enable.B && raw >= memParam.virtualBankCount.U && raw < (memParam.virtualBankCount + 2).U) || + (memParam.sharedEnable.B && raw >= frontendParam.shared_bank_id_base.U && + raw < memParam.virtualBankCount.U) + + val usesArchitecturalBank = is_ball_inst || func7 === MSET_BITPAT || func7 === MVIN_BITPAT || + func7 === MVIN_2D_BITPAT || func7 === MVOUT_BITPAT + when(io.command.fire && usesArchitecturalBank) { + when(hasRd0)(assert(legalBank(bankAccess.rd_bank_0_id), "GlobalDecoder: bank0 is outside configured ranges")) + when(hasRd1)(assert(legalBank(bankAccess.rd_bank_1_id), "GlobalDecoder: bank1 is outside configured ranges")) + when(hasWr)(assert(legalBank(bankAccess.wr_bank_id), "GlobalDecoder: write bank is outside configured ranges")) + } + + // Output control + io.decoded.bits.context := io.command.bits.context + io.decoded.bits.operation.runtime := io.command.bits.runtime + io.decoded.valid := io.command.valid + io.decoded.bits.operation.domain_id := domain_id + io.decoded.bits.operation.iter := rs1(62, 30) + io.decoded.bits.operation.cmd := io.command.bits.instruction + io.decoded.bits.operation.bankAccess := bankAccess + io.decoded.bits.operation.architecturalDestination := bankAccess.wr_bank_id + io.decoded.bits.operation.op1_col := 0.U + io.decoded.bits.operation.op2_col := 0.U + io.decoded.bits.operation.wr_col := 0.U + io.decoded.bits.operation.wr_read_bank := 0.U + io.decoded.bits.operation.isBarrier := is_barrier_inst +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/build.sc b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/build.sc new file mode 100644 index 000000000..a98718ea5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/build.sc @@ -0,0 +1,29 @@ +import mill._ +import $file.src.main.scala.bat.{build => bat_build} +import $file.src.main.scala.rob.global.{build => rob_global_build} +import $file.src.main.scala.scoreboard.{build => scoreboard_build} +import $file.^.^.^.^.^.^.^.^.^.^.settings +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.frontend.scheduler.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Scheduler.scala")) + ) ++ + bat_build.sources() ++ + rob_global_build.sources() ++ + scoreboard_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + bat_build.emitSources() ++ + rob_global_build.emitSources() ++ + scoreboard_build.emitSources() +) diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/csrc/rename_retire.cpp b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/csrc/rename_retire.cpp new file mode 100644 index 000000000..b02854893 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/csrc/rename_retire.cpp @@ -0,0 +1,140 @@ +#include "VScheduler.h" +#include "verilated.h" +#include +#include +#include +#include + +struct Issued { + unsigned read, write; +}; +struct Reclaimed { + unsigned previous, current; +}; + +int main(int argc, char **argv) { + Verilated::commandArgs(argc, argv); + Verilated::randReset(0); + VScheduler dut; + std::map issued; + std::vector reclaimed; + unsigned cycle = 0; + auto check = [](bool condition, const char *message) { + if (!condition) + throw std::runtime_error(message); + }; + auto tick = [&]() { + dut.clock = 0; + dut.eval(); + if (dut.io_ball_issue_valid && dut.io_ball_issue_ready) { + const unsigned slot = dut.io_ball_issue_bits_rob_id; + issued[slot] = {dut.io_ball_issue_bits_renamedBankAccess_rd_bank_0_id, + dut.io_ball_issue_bits_renamedBankAccess_wr_bank_id}; + std::printf("cycle=%u issue slot=%u read=%u write=%u\n", cycle, slot, + issued[slot].read, issued[slot].write); + } + if (dut.io_mapping_valid && dut.io_mapping_ready && + dut.io_mapping_bits_retire) { + reclaimed.push_back( + {dut.io_mapping_bits_previous, dut.io_mapping_bits_vbank}); + std::printf("cycle=%u reclaim previous=%u current=%u\n", cycle, + dut.io_mapping_bits_previous, dut.io_mapping_bits_vbank); + } + dut.clock = 1; + dut.eval(); + dut.clock = 0; + dut.eval(); + ++cycle; + }; + auto wait = [&](auto condition) { + for (unsigned n = 0; n < 100; ++n) { + dut.eval(); + if (condition()) + return; + tick(); + } + throw std::runtime_error("Handshake timed out"); + }; + dut.reset = 1; + for (unsigned i = 0; i < 4; ++i) + tick(); + dut.reset = 0; + dut.io_ball_issue_ready = 1; + dut.io_memory_issue_ready = 1; + dut.io_mapping_ready = 1; + dut.io_thread_id = 1; + auto send = [&](bool read, bool write) { + dut.io_decode_cmd_i_bits_domain_id = 3; + dut.io_decode_cmd_i_bits_cmd_funct = 0x63; + dut.io_decode_cmd_i_bits_cmd_pc = 4 + cycle * 4; + dut.io_decode_cmd_i_bits_bankAccess_rd_bank_0_valid = read; + dut.io_decode_cmd_i_bits_bankAccess_rd_bank_0_id = 1; + dut.io_decode_cmd_i_bits_bankAccess_wr_bank_valid = write; + dut.io_decode_cmd_i_bits_bankAccess_wr_bank_id = 1; + dut.io_decode_cmd_i_valid = 1; + wait([&] { return dut.io_decode_cmd_i_ready; }); + const unsigned slot = dut.io_allocation_bits_rob_id; + tick(); + dut.io_decode_cmd_i_valid = 0; + return slot; + }; + auto complete = [&](unsigned slot) { + dut.io_ball_complete_bits_rob_id = slot; + dut.io_ball_complete_valid = 1; + wait([&] { return dut.io_ball_complete_ready; }); + tick(); + dut.io_ball_complete_valid = 0; + }; + try { + const auto reader = send(true, false); + const auto writer1 = send(false, true); + const auto writer2 = send(false, true); + wait([&] { + return issued.count(reader) && issued.count(writer1) && + issued.count(writer2); + }); + check(issued[reader].read == 1, "Old reader lost its binding"); + const auto first = issued[writer1].write, second = issued[writer2].write; + check(first != 1 && second != first, + "Consecutive writes must use distinct aliases"); + complete(writer2); + complete(writer1); + for (unsigned i = 0; i < 12; ++i) + tick(); + check(reclaimed.empty(), + "Out-of-order completion freed an alias before retirement"); + dut.io_mapping_ready = 0; + complete(reader); + wait([&] { return dut.io_mapping_valid && dut.io_mapping_bits_retire; }); + for (unsigned i = 0; i < 10; ++i) { + check(dut.io_mapping_bits_previous == 1 && + dut.io_mapping_bits_vbank == first, + "Reclaim payload changed under backpressure"); + check(!(dut.io_retired & ((1u << writer1) | (1u << writer2))), + "ROB retired a writer before reclaim was accepted"); + tick(); + } + dut.io_mapping_ready = 1; + wait([&] { return reclaimed.size() == 2; }); + check(reclaimed[0].previous == 1 && reclaimed[0].current == first, + "First stale binding mismatch"); + check(reclaimed[1].previous == first && reclaimed[1].current == second, + "Second stale binding mismatch"); + const auto writer3 = send(true, true); + wait([&] { return issued.count(writer3); }); + check(issued[writer3].read == second, + "Read-modify-write did not read the previous binding"); + check(issued[writer3].write == first, + "Retired alias was not returned to the free list"); + complete(writer3); + wait([&] { return reclaimed.size() == 3 && dut.io_idle; }); + check(reclaimed[2].previous == second && reclaimed[2].current == first, + "Reused alias retirement mismatch"); + std::puts("PASS: out-of-order completion, reclaim backpressure, repeated " + "rename and alias reuse"); + } catch (const std::exception &error) { + std::fprintf(stderr, "FAIL at cycle %u: %s\n", cycle, error.what()); + return 1; + } + dut.final(); +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Emit.scala new file mode 100644 index 000000000..aa1eab2aa --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Emit.scala @@ -0,0 +1,35 @@ +package framework.acc.frontend.scheduler + +import sims.configs.SimParam +import _root_.configs.GlobalConfig +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Scheduler( + thread.acc.get.frontend, + thread.acc.get.memDomain, + thread.acc.get.gpDomain, + core.privileged.permission, + b.sim + ), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Scheduler.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Scheduler.scala new file mode 100644 index 000000000..800ebb789 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Scheduler.scala @@ -0,0 +1,239 @@ +package framework.acc.frontend.scheduler + +import sims.configs.SimParam +import framework.lib.arbiter.{Arbiter} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +import framework.lib.queue.Queue + +import chisel3._ +import chisel3.util._ +import chisel3.experimental._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.frontend.scheduler.rob.global.GlobalROB +import framework.acc.frontend.decoder.{DecodedInstruction, DomainId, PostGDCmd} +import framework.acc.frontend.scheduler.bat.BankAliasTable +import framework.acc.frontend.scheduler.scoreboard.{BankAccessInfo, BankScoreboard} +import framework.acc.debug.trace.InstructionEvent +import framework.acc.debug.DebugPort +import framework.acc.memdomain.bank_router.BankMapping +import framework.acc.memdomain.{Response => MemoryResponse, MemoryContext} +import framework.acc.memdomain.toddr.DmaError + +/** Work dispatched to a domain, independent of the ROB's storage representation. */ +class Issue(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val commandDataBits = frontendParam.commandDataBits + val p = frontendParam + val cmd = new PostGDCmd(frontendParam, memParam) + val renamedBankAccess = new BankAccessInfo(frontendParam.bank_id_len) + val rob_id = UInt(log2Up(p.rob_entries).W) + val instId = UInt(64.W) // Observation identity; never controls scheduling. + val target = UInt(5.W) +} + +/** Memory dispatch alone carries the submitting address-space state. */ +class MemoryIssue(frontendParam: FrontendParam, memParam: MemDomainParam, permissionParam: PmpParams) extends Bundle { + val commandDataBits = frontendParam.commandDataBits + val operation = new Issue(frontendParam, memParam) + val context = new MemoryContext(permissionParam) +} + +/** Completion of an admitted operation; legality is an issue-time contract. */ +class Complete(frontendParam: FrontendParam) extends Bundle { + val rob_id = UInt(log2Up(frontendParam.rob_entries).W) +} + +@instantiable +class Scheduler( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val permissionParam: PmpParams, + val simp: SimParam) + extends Module { + val diffTest = simp.diffTest + val commandDataBits = frontendParam.commandDataBits + val ownerId = frontendParam.ownerId + + @public + val io = IO(new Bundle { + + val admission = new Bundle { + val command = Flipped(Decoupled(new DecodedInstruction(frontendParam, memParam, permissionParam))) + val idle = Output(Bool()) + } + + val ballRequest = Decoupled(new MemoryIssue(frontendParam, memParam, permissionParam)) + val ballCompletion = Flipped(Decoupled(new MemoryResponse(frontendParam))) + val memoryRequest = Decoupled(new MemoryIssue(frontendParam, memParam, permissionParam)) + val memoryCompletion = Flipped(Decoupled(new MemoryResponse(frontendParam))) + val gpRequest = Option.when(gpParam.enable)(Decoupled(new Issue(frontendParam, memParam))) + val gpCompletion = + Option.when(gpParam.enable)(Flipped(Decoupled(new Complete(frontendParam)))) + val mappingUpdate = Decoupled(new BankMapping(frontendParam)) + + val barrier = new Bundle { + val arrive = Output(Bool()) + val release = Input(Bool()) + } + + val fault = Output(Bool()) + val traceInstructions = Output(Vec(4, new InstructionEvent)) + val debug = Option.when(diffTest)(new DebugPort(frontendParam)) + }) + + val rob: Instance[GlobalROB] = + Instantiate(new GlobalROB(frontendParam, memParam, gpParam, permissionParam, simp.diffTest)) + + val resourceCount = memParam.virtualBankCount + (if (gpParam.enable) 2 else 0) + + val bat: Instance[BankAliasTable] = Instantiate( + new BankAliasTable(framework.acc.frontend.scheduler.bat.configs.Params( + frontendParam.bank_id_len, + resourceCount, + frontendParam.rob_entries + )) + ) + + val scoreboard: Instance[BankScoreboard] = Instantiate( + new BankScoreboard(framework.acc.frontend.scheduler.scoreboard.configs.Params( + frontendParam.bank_id_len, + 2 * resourceCount + frontendParam.rob_entries, + frontendParam.rob_entries + )) + ) + + scoreboard.io.alloc := rob.io.scoreboard.alloc + scoreboard.io.issue := rob.io.scoreboard.issue + scoreboard.io.complete := rob.io.scoreboard.complete + scoreboard.io.queryVec := rob.io.scoreboard.query + rob.io.scoreboard.hazard := scoreboard.io.hazardVec + bat.io.alloc := rob.io.bat.alloc + bat.io.reclaim := rob.io.bat.reclaim + rob.io.bat.stale := bat.io.stale + rob.io.bat.ready := bat.io.ready + rob.io.bat.renamed := bat.io.alloc_renamed + io.mappingUpdate <> rob.io.mapping + + io.traceInstructions := rob.io.traceInstructions + io.debug.zip(rob.io.debug).foreach { case (port, source) => + port <> source + port.idle := io.admission.idle && !io.admission.command.valid && !port.commit.valid + } + + val failed = RegInit(false.B) + val memoryFailed = io.memoryCompletion.bits.fault.error =/= DmaError.None.U + when( + (io.memoryCompletion.fire && memoryFailed) || (io.ballCompletion.fire && io.ballCompletion.bits.fault.error =/= 0.U) + )(failed := true.B) + io.fault := failed + + val isBarrierCmd = io.admission.command.valid && io.admission.command.bits.operation.isBarrier + val barrierWaitROB = RegInit(false.B) + val barrierWaitRelease = RegInit(false.B) + when(isBarrierCmd && !barrierWaitROB && !barrierWaitRelease) { + barrierWaitROB := true.B + } + when(barrierWaitROB && rob.io.empty) { + barrierWaitROB := false.B + barrierWaitRelease := true.B + } + when(barrierWaitRelease && io.barrier.release) { + barrierWaitRelease := false.B + } + io.barrier.arrive := barrierWaitRelease + + val isFrontendCmd = io.admission.command.bits.operation.isBarrier + // Extended control operations and metadata loads own the local scratch state until retirement. + val funct = io.admission.command.bits.operation.cmd.raw_inst(31, 25) + val exclusive = funct >= 80.U || funct === 35.U + val exclusiveActive = RegInit(false.B) + when(exclusiveActive && rob.io.empty)(exclusiveActive := false.B) + when(rob.io.alloc.fire && exclusive)(exclusiveActive := true.B) + val anyStall = barrierWaitROB || barrierWaitRelease || failed || exclusiveActive || (exclusive && !rob.io.empty) + rob.io.alloc.valid := io.admission.command.valid && !isFrontendCmd && !anyStall + rob.io.alloc.bits := io.admission.command.bits + io.admission.command.ready := Mux( + isFrontendCmd, + !anyStall, + rob.io.alloc.ready && !anyStall + ) + + val is_ball_domain = rob.io.issue.bits.cmd.domain_id === DomainId.BALL + val is_mem_domain = rob.io.issue.bits.cmd.domain_id === DomainId.MEM + val is_kernel_domain = rob.io.issue.bits.cmd.domain_id === DomainId.GP + + val mainIssueEntry = Wire(new Issue(frontendParam, memParam)) + mainIssueEntry.instId := rob.io.issue.bits.instId + mainIssueEntry.target := 0.U + mainIssueEntry.cmd := rob.io.issue.bits.cmd + mainIssueEntry.renamedBankAccess := rob.io.issue.bits.renamedBankAccess + mainIssueEntry.rob_id := rob.io.issue.bits.rob_id + mainIssueEntry.cmd.wr_read_bank := rob.io.issue.bits.renamedBankAccess.rd_wr_id + val rawOperands = rob.io.issue.bits.cmd.cmd.rs1Data + val access = rob.io.issue.bits.renamedBankAccess + when( + rob.io.issue.bits.cmd.cmd.raw_inst(31, 25) =/= framework.acc.memdomain.DISA.MSET_BITPAT + ) { + mainIssueEntry.cmd.cmd.rs1Data := Cat( + rawOperands(63, 30), + Mux(access.wr_bank_valid, access.wr_bank_id, rawOperands(29, 20)), + Mux(access.rd_bank_1_valid, access.rd_bank_1_id, rawOperands(19, 10)), + Mux(access.rd_bank_0_valid, access.rd_bank_0_id, rawOperands(9, 0)) + ) + } + + if (gpParam.enable) { + io.gpRequest.get.valid := rob.io.issue.valid && is_kernel_domain && !failed + io.gpRequest.get.bits := mainIssueEntry + } + val kernelReady = if (gpParam.enable) io.gpRequest.get.ready else false.B + + val completeArb = Instantiate(new Arbiter(new Complete(frontendParam), if (gpParam.enable) 3 else 2)) + + val completeQueue = Instantiate(new Queue( + UInt(log2Up(frontendParam.rob_entries).W), + frontendParam.rob_entries, + false, + false + )) + + rob.io.complete <> completeQueue.io.deq + completeArb.io.in(0).valid := io.ballCompletion.valid && io.ballCompletion.bits.fault.error === 0.U + completeArb.io.in(0).bits.rob_id := io.ballCompletion.bits.rob_id + io.ballCompletion.ready := completeArb.io.in(0).ready || io.ballCompletion.bits.fault.error =/= 0.U + completeArb.io.in(1).valid := io.memoryCompletion.valid && !memoryFailed + completeArb.io.in(1).bits.rob_id := io.memoryCompletion.bits.rob_id + io.memoryCompletion.ready := memoryFailed || completeArb.io.in(1).ready + + if (gpParam.enable) { completeArb.io.in(2) <> io.gpCompletion.get } + + val completeBits = completeArb.io.out.bits + + io.ballRequest.valid := rob.io.issue.valid && is_ball_domain && !failed + io.ballRequest.bits.operation := mainIssueEntry + io.ballRequest.bits.context := rob.io.issue.bits.context + io.memoryRequest.valid := rob.io.issue.valid && is_mem_domain && !failed + io.memoryRequest.bits.operation := mainIssueEntry + io.memoryRequest.bits.context := rob.io.issue.bits.context + rob.io.issue.ready := !failed && ((is_ball_domain && io.ballRequest.ready) || + (is_mem_domain && io.memoryRequest.ready) || (is_kernel_domain && kernelReady)) + + completeQueue.io.enq.valid := completeArb.io.out.valid + completeQueue.io.enq.bits := completeBits.rob_id + completeArb.io.out.ready := completeQueue.io.enq.ready + io.admission.idle := rob.io.empty && !barrierWaitROB && !barrierWaitRelease && !failed + + // Reset-time initialization is not a runtime command. Monitor runtime blocking locally. + val runtimeStarted = RegInit(false.B) + when(io.admission.command.fire && io.admission.command.bits.operation.runtime)(runtimeStarted := true.B) + val blocked = rob.io.full || barrierWaitROB || barrierWaitRelease + val blockedCycles = RegInit(0.U(32.W)) + blockedCycles := Mux(runtimeStarted && blocked, blockedCycles + 1.U, 0.U) + assert(blockedCycles < 10000000.U, "Accelerator: busy for too long!") + +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/build.sc b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/build.sc new file mode 100644 index 000000000..05a0fdf48 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.^.^.^.settings +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.frontend.scheduler.bat.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/BankAliasTable.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/configs/uvm.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/configs/uvm.toml new file mode 100644 index 000000000..968f67903 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/configs/uvm.toml @@ -0,0 +1,6 @@ +[parameters] +bankIdBits = 4 +virtualBanks = 4 +robEntries = 4 + +[defines] diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..d549e9643 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual bank_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bank_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Bank scheduling interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.inputs = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/item.svh new file mode 100644 index 000000000..111706948 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`BAT_INPUT_BITS-1:0] inputs; + logic [`BAT_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + logic [`BAT_OUTPUT_BITS-1:0] payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..dc93b2fb6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual bank_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bank_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Bank scheduling monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/env/env.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/env/env.svh new file mode 100644 index 000000000..ed95874fb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver drv; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/env/model.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/env/model.svh new file mode 100644 index 000000000..24b93f3f0 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/env/model.svh @@ -0,0 +1,99 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + localparam int V = `BAT_VIRTUALBANKS, A = 2 * V + `BAT_ROBENTRIES; + int current[V]; + bit [A-1:0] free_aliases; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + covergroup behavior with function sample ( + bit rename, bit reset_read, bit update, bit ready, int stale + ); + option.per_instance = 1; + renaming: coverpoint rename; + reset_mapping: coverpoint reset_read; + update_alias: coverpoint update; + free_capacity: coverpoint ready; + stale_aliases: coverpoint stale {bins combinations[] = {[0 : 3]};} + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + behavior = new(); + endfunction + function void write(cycle_item t); + result_item expected, actual; + int fresh, read0, read1, read_wr, wr, renamed_write, stale0, stale1; + bit rename, reset_read, update, resets_write, stale0_valid, stale1_valid; + if (t.reset) begin + foreach (current[b]) current[b] = b; + free_aliases = '1; + for (int b = 0; b < V; b++) free_aliases[b] = 0; + return; + end + expected = result_item::type_id::create("expected"); + actual = result_item::type_id::create("actual"); + expected.payload = '0; + actual.payload = t.outputs; + fresh = 0; + for (int slot = 0; slot < A; slot++) + if (free_aliases[slot]) begin + fresh = slot; + break; + end + read0 = `BI(t.inputs, ALLOC_RAW_RD_BANK_0_ID); + read1 = `BI(t.inputs, ALLOC_RAW_RD_BANK_1_ID); + read_wr = `BI(t.inputs, ALLOC_RAW_RD_WR_ID); + wr = `BI(t.inputs, ALLOC_RAW_WR_BANK_ID); + rename = `BI(t.inputs, ALLOC_RENAME); + reset_read = `BI(t.inputs, ALLOC_RESET_READ0); + update = `BI(t.inputs, ALLOC_RAW_WR_UPDATE); + renamed_write = rename ? fresh : update ? current[wr] : wr; + // The fresh-alias payload has no meaning when allocation is blocked. + if (rename && free_aliases == 0) begin + actual.payload[`BAT_OUT_ALLOC_RENAMED_WR_BANK_ID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_WR_BANK_ID_WIDTH] = 0; + end + resets_write = + `BI(t.inputs, ALLOC_RAW_WR_BANK_VALID) + && !rename && !update && current[wr] != wr; + stale0_valid = rename || (!reset_read && resets_write) || (reset_read && current[read0] != read0); + stale1_valid = reset_read && resets_write && read0 != wr; + stale0 = reset_read ? current[read0] : current[wr]; + stale1 = current[wr]; + expected.payload[`BAT_OUT_READY_OFFSET+:`BAT_OUT_READY_WIDTH] = !rename || free_aliases != 0; + expected.payload[`BAT_OUT_ALLOC_RENAMED_RD_BANK_0_VALID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_RD_BANK_0_VALID_WIDTH] = `BI( + t.inputs, ALLOC_RAW_RD_BANK_0_VALID); + expected.payload[`BAT_OUT_ALLOC_RENAMED_RD_BANK_1_VALID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_RD_BANK_1_VALID_WIDTH] = `BI( + t.inputs, ALLOC_RAW_RD_BANK_1_VALID); + expected.payload[`BAT_OUT_ALLOC_RENAMED_RD_WR_VALID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_RD_WR_VALID_WIDTH] = `BI( + t.inputs, ALLOC_RAW_RD_WR_VALID); + expected.payload[`BAT_OUT_ALLOC_RENAMED_WR_BANK_VALID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_WR_BANK_VALID_WIDTH] = `BI( + t.inputs, ALLOC_RAW_WR_BANK_VALID); + expected.payload[`BAT_OUT_ALLOC_RENAMED_WR_UPDATE_OFFSET +: `BAT_OUT_ALLOC_RENAMED_WR_UPDATE_WIDTH] = `BI( + t.inputs, ALLOC_RAW_WR_UPDATE); + expected.payload[`BAT_OUT_ALLOC_RENAMED_RD_BANK_0_ID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_RD_BANK_0_ID_WIDTH] = + `BI(t.inputs, ALLOC_RAW_RD_BANK_0_VALID) + ? current[read0] : 0; + expected.payload[`BAT_OUT_ALLOC_RENAMED_RD_BANK_1_ID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_RD_BANK_1_ID_WIDTH] = + `BI(t.inputs, ALLOC_RAW_RD_BANK_1_VALID) + ? current[read1] : 0; + expected.payload[`BAT_OUT_ALLOC_RENAMED_RD_WR_ID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_RD_WR_ID_WIDTH] = + `BI(t.inputs, ALLOC_RAW_RD_WR_VALID) + ? current[read_wr] : 0; + expected.payload[`BAT_OUT_ALLOC_RENAMED_WR_BANK_ID_OFFSET +: `BAT_OUT_ALLOC_RENAMED_WR_BANK_ID_WIDTH] = renamed_write; + expected.payload[`BAT_OUT_STALE_0_VALID_OFFSET+:`BAT_OUT_STALE_0_VALID_WIDTH] = stale0_valid; + expected.payload[`BAT_OUT_STALE_0_BITS_OFFSET+:`BAT_OUT_STALE_0_BITS_WIDTH] = stale0; + expected.payload[`BAT_OUT_STALE_1_VALID_OFFSET+:`BAT_OUT_STALE_1_VALID_WIDTH] = stale1_valid; + expected.payload[`BAT_OUT_STALE_1_BITS_OFFSET+:`BAT_OUT_STALE_1_BITS_WIDTH] = stale1; + behavior.sample(rename, reset_read, update, !rename || free_aliases != 0, + stale0_valid | (int'(stale1_valid) << 1)); + expected_ap.write(expected); + actual_ap.write(actual); + if (`BI(t.inputs, RECLAIM_VALID) && `BI(t.inputs, RECLAIM_BITS) >= V) + free_aliases[`BI(t.inputs, RECLAIM_BITS)] = 1; + if (`BI(t.inputs, ALLOC_VALID)) begin + if (rename) free_aliases[fresh] = 0; + if (`BI(t.inputs, ALLOC_RAW_WR_BANK_VALID)) current[wr] = renamed_write; + if (reset_read) current[read0] = read0; + end + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/filelist.f b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/filelist.f new file mode 100644 index 000000000..3eea691bb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/BankAliasTable/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/interface.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/interface.sv new file mode 100644 index 000000000..14c732776 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/interface.sv @@ -0,0 +1,8 @@ +`include "parameters.svh" +interface bank_if ( + input logic clock +); + logic reset = 1; + logic [`BAT_INPUT_BITS-1:0] inputs; + logic [`BAT_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/package.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/package.sv new file mode 100644 index 000000000..c173de906 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/package.sv @@ -0,0 +1,15 @@ +`include "parameters.svh" +`define BI(P, N) P[`BAT_IN_``N``_OFFSET +: `BAT_IN_``N``_WIDTH] +`define BO(P, N) P[`BAT_OUT_``N``_OFFSET +: `BAT_OUT_``N``_WIDTH] +package bat_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..42e4dec66 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/seq/sequence.svh @@ -0,0 +1,152 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + localparam int V = `BAT_VIRTUALBANKS, A = 2 * V + `BAT_ROBENTRIES; + logic [`BAT_INPUT_BITS-1:0] pins = '0; + logic [`BAT_OUTPUT_BITS-1:0] observed; + bit reset = 1; + int stale[$]; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task tick(int cycles = 1); + cycle_item req; + repeat (cycles) begin + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.inputs = pins; + req.reset = reset; + finish_item(req); + observed = req.outputs; + if (!reset && `BI(pins, ALLOC_VALID)) begin + if (`BO(observed, STALE_0_VALID)) stale.push_back(`BO(observed, STALE_0_BITS)); + if (`BO(observed, STALE_1_VALID)) stale.push_back(`BO(observed, STALE_1_BITS)); + end + end + endtask + task restart(); + pins = '0; + reset = 1; + stale.delete(); + tick(2); + reset = 0; + tick(); + endtask + task allocate(int bank, bit rename, bit reset_read, bit update, int read0, read1, read_wr, + int mask = 7); + `BI(pins, ALLOC_VALID) = 1; + `BI(pins, ALLOC_RENAME) = rename; + `BI(pins, ALLOC_RESET_READ0) = reset_read; + `BI(pins, ALLOC_RAW_WR_UPDATE) = update; + `BI(pins, ALLOC_RAW_WR_BANK_VALID) = 1; + `BI(pins, ALLOC_RAW_WR_BANK_ID) = bank; + `BI(pins, ALLOC_RAW_RD_BANK_0_VALID) = mask[0]; + `BI(pins, ALLOC_RAW_RD_BANK_0_ID) = read0; + `BI(pins, ALLOC_RAW_RD_BANK_1_VALID) = mask[1]; + `BI(pins, ALLOC_RAW_RD_BANK_1_ID) = read1; + `BI(pins, ALLOC_RAW_RD_WR_VALID) = mask[2]; + `BI(pins, ALLOC_RAW_RD_WR_ID) = read_wr; + tick(); + `BI(pins, ALLOC_VALID) = 0; + endtask + task reclaim_all(); + `BI(pins, ALLOC_VALID) = 0; + `BI(pins, ALLOC_RENAME) = 0; + `BI(pins, ALLOC_RESET_READ0) = 0; + while (stale.size() != 0) begin + `BI(pins, RECLAIM_VALID) = 1; + `BI(pins, RECLAIM_BITS) = stale.pop_front(); + tick(); + end + `BI(pins, RECLAIM_VALID) = 0; + endtask + task body(); + restart(); + for (int n = 0; n < A - V; n++) allocate(n % V, 1, 0, 0, n % V, (n + 1) % V, (n + 2) % V); + `BI(pins, ALLOC_RENAME) = 1; + tick(); + evidence::check(!`BO(observed, READY), "BAT free list did not exhaust"); + reclaim_all(); + for (int n = 0; n < 4 * V; n++) begin + allocate(n % V, 1, 0, 0, (n + 1) % V, (n + 2) % V, n % V); + reclaim_all(); + end + restart(); + for (int bank = 0; bank < V; bank++) allocate(bank, 1, 0, 0, bank, bank, bank); + reclaim_all(); + for (int mask = 0; mask < 8; mask++) + for (int read0 = 0; read0 < V; read0++) + for (int read1 = 0; read1 < V; read1++) + for (int read_wr = 0; read_wr < V; read_wr++) + allocate(read0, 0, 0, 1, read0, read1, read_wr, mask); + restart(); + allocate(0, 1, 0, 0, 0, 0, 0); + allocate(0, 0, 1, 0, 1 % V, 0, 0); + reclaim_all(); + restart(); + allocate(0, 1, 0, 0, 0, 0, 0); + allocate(1 % V, 1, 0, 0, 1 % V, 0, 0); + allocate(0, 0, 1, 0, 1 % V, 0, 0); + reclaim_all(); + restart(); + allocate(0, 1, 0, 0, 0, 0, 0); + allocate(0, 0, 1, 1, 0, 0, 0); + reclaim_all(); + // A retired alias becomes available on the next cycle, alongside an independent allocation. + restart(); + for (int n = 0; n < A - V - 1; n++) allocate(n % V, 1, 0, 0, n % V, n % V, n % V); + while (stale.size() && stale[0] < V) begin + `BI(pins, RECLAIM_VALID) = 1; + `BI(pins, RECLAIM_BITS) = stale.pop_front(); + tick(); + end + `BI(pins, RECLAIM_VALID) = 1; + `BI(pins, RECLAIM_BITS) = stale.pop_front(); + allocate(0, 1, 0, 0, 0, 0, 0); + `BI(pins, RECLAIM_VALID) = 0; + reclaim_all(); + restart(); + endtask +endclass +class invalid_sequence extends scenario_sequence; + `uvm_object_utils(invalid_sequence) + function new(string name = "invalid_sequence"); + super.new(name); + endfunction + task body(); + string scenario; + if (!$value$plusargs("CASE=%s", scenario)) + `uvm_fatal("CASE", "BAT assertion scenario is required") + restart(); + if (scenario == "full") begin + for (int n = 0; n <= A - V; n++) allocate(0, 1, 0, 0, 0, 0, 0); + end else if (scenario == "reclaim") begin + `BI(pins, RECLAIM_VALID) = 1; + `BI(pins, RECLAIM_BITS) = A; + tick(); + end else begin + `BI(pins, ALLOC_VALID) = 1; + case (scenario) + "read0": begin + `BI(pins, ALLOC_RAW_RD_BANK_0_VALID) = 1; + `BI(pins, ALLOC_RAW_RD_BANK_0_ID) = V; + end + "read1": begin + `BI(pins, ALLOC_RAW_RD_BANK_1_VALID) = 1; + `BI(pins, ALLOC_RAW_RD_BANK_1_ID) = V; + end + "update": begin + `BI(pins, ALLOC_RAW_RD_WR_VALID) = 1; + `BI(pins, ALLOC_RAW_RD_WR_ID) = V; + end + "write": begin + `BI(pins, ALLOC_RAW_WR_BANK_VALID) = 1; + `BI(pins, ALLOC_RAW_WR_BANK_ID) = V; + end + default: `uvm_fatal("CASE", "Unknown BAT assertion scenario") + endcase + tick(); + end + tick(3); + `uvm_fatal("EXPECTED_ASSERTION", "Invalid BAT input did not trigger its assertion") + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/suite.toml new file mode 100644 index 000000000..9e6f5dcd1 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/suite.toml @@ -0,0 +1,122 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "bat" +top = "bat_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["bat_pkg::model::behavior"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +name = "bad_full" +top = "bat_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=full"] +assertion = "BAT allocation has no free virtual bank" + +[[tests]] +name = "bad_reclaim" +top = "bat_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=reclaim"] +assertion = "BAT reclaim alias is outside the allocated namespace" + +[[tests]] +name = "bad_read0" +top = "bat_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=read0"] +assertion = "BAT read bank is outside the architectural namespace" + +[[tests]] +name = "bad_read1" +top = "bat_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=read1"] +assertion = "BAT second read bank is outside the architectural namespace" + +[[tests]] +name = "bad_update" +top = "bat_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=update"] +assertion = "BAT update source is outside the architectural namespace" + +[[tests]] +name = "bad_write" +top = "bat_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=write"] +assertion = "BAT write bank is outside the architectural namespace" diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/tb.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/tb.sv new file mode 100644 index 000000000..13242a5d5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/tb.sv @@ -0,0 +1,16 @@ +module bat_tb; + import uvm_pkg::*; + import bat_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + bank_if bus (clock); + BankAliasTable dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual bank_if)::set(null, "uvm_test_top*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/tests/test.svh new file mode 100644 index 000000000..0230681bb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/resources/tests/test.svh @@ -0,0 +1,35 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass +class invalid_test extends base_test; + `uvm_component_utils(invalid_test) + driver drv; + uvm_sequencer #(cycle_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction + task execute(); + invalid_sequence seq = invalid_sequence::type_id::create("scenario"); + seq.start(seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/BankAliasTable.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/BankAliasTable.scala new file mode 100644 index 000000000..3e2f9d741 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/BankAliasTable.scala @@ -0,0 +1,106 @@ +package framework.acc.frontend.scheduler.bat + +import framework.acc.frontend.scheduler.scoreboard.BankAccessInfo +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.frontend.scheduler.bat.configs.Params + +/** Current mappings and a free list; overwritten aliases return only at retirement. */ +@instantiable +class BankAliasTable(val p: Params) extends Module { + + val bankIdLen = p.bankIdBits + val vbankNum = p.virtualBanks + val robEntries = p.robEntries + val aliasCount = 2 * vbankNum + robEntries + + @public + val io = IO(new Bundle { + + val alloc = Input(new Bundle { + val valid = Bool() + val rename = Bool() + val reset_read0 = Bool() + val raw = new BankAccessInfo(p.bankIdBits) + }) + + val ready = Output(Bool()) + val alloc_renamed = Output(new BankAccessInfo(p.bankIdBits)) + val reclaim = Flipped(Valid(UInt(bankIdLen.W))) + val stale = Output(Vec(2, Valid(UInt(bankIdLen.W)))) + }) + + val current = RegInit(VecInit((0 until vbankNum).map(_.U(bankIdLen.W)))) + + val initialFree = ((BigInt(1) << aliasCount) - 1) ^ ((BigInt(1) << vbankNum) - 1) + val free = RegInit(initialFree.U(aliasCount.W)) + val available = free + + val fresh = PriorityEncoder(available) + val raw = io.alloc.raw + + // Architectural IDs are checked before allocation; the register array uses only its index bits. + def currentAlias(bank: UInt): UInt = + if (vbankNum == 1) current(0) else current(bank(log2Ceil(vbankNum) - 1, 0)) + + io.ready := !io.alloc.rename || available.orR + io.alloc_renamed := raw + io.alloc_renamed.rd_bank_0_id := Mux( + raw.rd_bank_0_valid, + currentAlias(raw.rd_bank_0_id), + 0.U + ) + io.alloc_renamed.rd_bank_1_id := Mux( + raw.rd_bank_1_valid, + currentAlias(raw.rd_bank_1_id), + 0.U + ) + io.alloc_renamed.rd_wr_id := Mux(raw.rd_wr_valid, currentAlias(raw.rd_wr_id), 0.U) + io.alloc_renamed.wr_bank_id := Mux( + io.alloc.rename, + fresh, + Mux(raw.wr_update, currentAlias(raw.wr_bank_id), raw.wr_bank_id) + ) + val resetBank = raw.rd_bank_0_id + val resetsWrite = raw.wr_bank_valid && !io.alloc.rename && !raw.wr_update && + currentAlias(raw.wr_bank_id) =/= raw.wr_bank_id + io.stale(0).valid := io.alloc.rename || (!io.alloc.reset_read0 && resetsWrite) || + (io.alloc.reset_read0 && currentAlias(resetBank) =/= resetBank) + io.stale(0).bits := Mux(io.alloc.reset_read0, currentAlias(resetBank), currentAlias(raw.wr_bank_id)) + io.stale(1).valid := io.alloc.reset_read0 && resetsWrite && resetBank =/= raw.wr_bank_id + io.stale(1).bits := currentAlias(raw.wr_bank_id) + + when(io.reclaim.valid)(assert(io.reclaim.bits < aliasCount.U, "BAT reclaim alias is outside the allocated namespace")) + + val returned = + Mux(io.reclaim.valid && io.reclaim.bits >= vbankNum.U, UIntToOH(io.reclaim.bits, aliasCount), 0.U(aliasCount.W)) + val allocated = Mux(io.alloc.valid && io.alloc.rename, UIntToOH(fresh, aliasCount), 0.U(aliasCount.W)) + free := (free | returned) & ~allocated + + when(io.alloc.valid) { + assert(io.ready, "BAT allocation has no free virtual bank") + when(raw.rd_bank_0_valid || io.alloc.reset_read0)(assert( + raw.rd_bank_0_id < vbankNum.U, + "BAT read bank is outside the architectural namespace" + )) + when(raw.rd_bank_1_valid)(assert( + raw.rd_bank_1_id < vbankNum.U, + "BAT second read bank is outside the architectural namespace" + )) + when(raw.rd_wr_valid)(assert(raw.rd_wr_id < vbankNum.U, "BAT update source is outside the architectural namespace")) + when(raw.wr_bank_valid || io.alloc.rename)(assert( + raw.wr_bank_id < vbankNum.U, + "BAT write bank is outside the architectural namespace" + )) + val access = io.alloc_renamed + when(raw.wr_bank_valid) { + if (vbankNum == 1) current(0) := access.wr_bank_id + else current(raw.wr_bank_id(log2Ceil(vbankNum) - 1, 0)) := access.wr_bank_id + } + when(io.alloc.reset_read0) { + if (vbankNum == 1) current(0) := raw.rd_bank_0_id + else current(raw.rd_bank_0_id(log2Ceil(vbankNum) - 1, 0)) := raw.rd_bank_0_id + } + } +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/Emit.scala new file mode 100644 index 000000000..d69019735 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/Emit.scala @@ -0,0 +1,54 @@ +package framework.acc.frontend.scheduler.bat + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.acc.frontend.scheduler.bat.configs.Params + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val p = upickle.default.read[Params](Files.readString(Paths.get(config))) + Files.createDirectories(Paths.get(output)) + + def fields(data: Data, path: String): Seq[(String, Int)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(path -> leaf.getWidth) + } + + val inputRoots = Set("alloc", "reclaim") + val ports = scala.collection.mutable.ArrayBuffer[String]() + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0 + var outputOffset = 0 + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new BankAliasTable(p) + for { + (root, bundle) <- dut.io.elements.toSeq + (name, width) <- fields(bundle, root) + } { + val input = inputRoots.contains(root) + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define BAT_${direction}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define BAT_${direction}_${name.toUpperCase}_WIDTH $width" + ports += s".io_$name(bus.$signal[$offset +: $width])" + offsets(name) = offset + if (input) inputOffset += width else outputOffset += width + } + dut + }, + args = Array("--target-dir", s"$output/BankAliasTable", "--split-verilog") + ) + definitions += s"`define BAT_INPUT_BITS $inputOffset" + definitions += s"`define BAT_OUTPUT_BITS $outputOffset" + definitions += s"`define BAT_BANKIDBITS ${p.bankIdBits}" + definitions += s"`define BAT_VIRTUALBANKS ${p.virtualBanks}" + definitions += s"`define BAT_ROBENTRIES ${p.robEntries}" + Files.writeString(Paths.get(output, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..43a00fd63 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat/src/main/scala/configs/Params.scala @@ -0,0 +1,12 @@ +package framework.acc.frontend.scheduler.bat.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Params(bankIdBits: Int, virtualBanks: Int, robEntries: Int) { + require(bankIdBits > 0) + require(virtualBanks > 0) + require(robEntries > 1) + require(2 * virtualBanks + robEntries <= (1 << bankIdBits)) +} + +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/build.sc b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/build.sc new file mode 100644 index 000000000..0846b1c10 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.^.^.^.^.settings +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.frontend.scheduler.rob.global.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/DependencyResolver.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/GlobalROB.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Rob.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/RobParams.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/configs/uvm.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/configs/uvm.toml new file mode 100644 index 000000000..8a3594f8d --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/configs/uvm.toml @@ -0,0 +1,57 @@ +[parameters] +diffTest = false + +[parameters.frontend] +ownerId = 7 +commandDataBits = 64 +rob_entries = 4 +bank_alias_enable = true +rs_out_of_order_response = true +bank_id_len = 4 +vbank_id_upper_bound = 3 +shared_bank_id_base = 16 +iter_len = 16 +sub_rob_enable = false +sub_rob_depth = 4 + +[parameters.memory] +sharedPorts = 0 +bankNum = 2 +bankLayoutEnable = true +bankWidth = 128 +bankEntries = 8 +bankMaskLen = 16 +virtualBankCount = 4 +sharedEnable = false +sharedEntries = 0 +sharedBankEntries = 0 +sharedBankNum = 0 +sharedInputChannels = 0 +sharedDefaultGroupCount = 0 +bankRequestPorts = 1 +dma_buswidth = 128 +memAddrLen = 44 +mmioEnable = false +mmioBankNum = 0 +mmioBankEntries = 0 +mmioBankWidth = 0 +mmioReadWidth = 0 + +[parameters.permission] +addressBits = 44 +entries = 16 + +[parameters.gp] +enable = false +laneNumber = 1 +vLen = 128 +eLen = 64 +iBufWords = 16 +memoryPorts = 1 + +[parameters.rob] +entries = 4 +dataBits = 64 +retireWidth = 4 + +[defines] diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/driver.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/driver.svh new file mode 100644 index 000000000..456ddc43c --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/driver.svh @@ -0,0 +1,27 @@ +class driver extends uvm_driver #(command_item); + `uvm_component_utils(driver) + virtual global_if vif; + command_item pending; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual global_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Global ROB input interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + if (pending == null) seq_item_port.try_next_item(pending); + vif.inputs[`GROB_IN_ALLOC_OFFSET+:`GROB_IN_ALLOC_BITS] = pending!=null ? pending.payload : '0; + `GI(vif.inputs, ALLOC_VALID) = pending != null; + @(posedge vif.clock); + if (!vif.reset && pending != null && `GO(vif.outputs, ALLOC_READY)) begin + #1; + seq_item_port.item_done(); + pending = null; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/item.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/item.svh new file mode 100644 index 000000000..a6787bc57 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/item.svh @@ -0,0 +1,52 @@ +typedef logic [`GROB_IN_ALLOC_BITS-1:0] command_t; +typedef logic [`GROB_IN_OPERATION_BITS-1:0] operation_t; +typedef logic [`GROB_IN_RENAMED_BITS-1:0] bank_access_t; +typedef logic [`GROB_ISSUE_BITS-1:0] issue_t; +class command_item extends uvm_sequence_item; + `uvm_object_utils(command_item) + command_t payload; + function new(string name = "command_item"); + super.new(name); + endfunction +endclass +class command_sequence extends uvm_sequence #(command_item); + `uvm_object_utils(command_sequence) + command_item item; + function new(string name = "command_sequence"); + super.new(name); + endfunction + task body(); + start_item(item); + finish_item(item); + endtask +endclass +class sample_item extends uvm_sequence_item; + `uvm_object_utils(sample_item) + logic [`GROB_INPUT_BITS-1:0] inputs; + logic [`GROB_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "sample_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + issue_t payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass +class transaction; + operation_t operation; + issue_t expected; + bank_access_t renamed; + int slot, serial, raw_write, new_alias, stale_raw, funct; + longint unsigned pc, inst_id; + bit [1:0] stale_valid; + int stale[2]; + bit selected = 0, delivered = 0, completed = 0, allocated = 0, config_op = 0, movement = 0; + bit [`GROB_VBANKS-1:0] reads, writes; + int due; +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/monitor.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/monitor.svh new file mode 100644 index 000000000..06034f023 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual global_if vif; + uvm_analysis_port #(sample_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual global_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Global ROB monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + sample_item t; + forever begin + @(posedge vif.clock); + t = sample_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/peer.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/peer.svh new file mode 100644 index 000000000..d90bb1085 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/agent/peer.svh @@ -0,0 +1,61 @@ +class peer_driver extends uvm_component; + `uvm_component_utils(peer_driver) + virtual global_if vif; + model reference; + int query_offsets[`GROB_ENTRIES][9] = `GROB_QUERY_OFFSETS; + int hazard_offsets[`GROB_ENTRIES] = `GROB_HAZARD_OFFSETS; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual global_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Global ROB peer interface is required") + endfunction + task run_phase(uvm_phase phase); + bank_access_t renamed; + bit ready, rename, reset_read; + bit [1:0] stale_valid; + int stale0, stale1, chosen; + forever begin + @(negedge vif.clock); + #1; + reference.predict_bat(vif.inputs, renamed, ready, rename, reset_read, stale_valid, stale0, + stale1); + vif.inputs[`GROB_IN_BAT_READY_OFFSET+:`GROB_IN_BAT_READY_WIDTH] = ready; + vif.inputs[`GROB_IN_RENAMED_OFFSET+:`GROB_IN_RENAMED_BITS] = renamed; + vif.inputs[`GROB_IN_BAT_STALE_0_VALID_OFFSET+:`GROB_IN_BAT_STALE_0_VALID_WIDTH]=stale_valid[0]; + vif.inputs[`GROB_IN_BAT_STALE_0_BITS_OFFSET+:`GROB_IN_BAT_STALE_0_BITS_WIDTH] = stale0; + vif.inputs[`GROB_IN_BAT_STALE_1_VALID_OFFSET+:`GROB_IN_BAT_STALE_1_VALID_WIDTH]=stale_valid[1]; + vif.inputs[`GROB_IN_BAT_STALE_1_BITS_OFFSET+:`GROB_IN_BAT_STALE_1_BITS_WIDTH] = stale1; + vif.inputs[`GROB_IN_ISSUE_READY_OFFSET+:`GROB_IN_ISSUE_READY_WIDTH]=!vif.reset&&!reference.hold_issue&&reference.cycle%5!=0; + vif.inputs[`GROB_IN_MAPPING_READY_OFFSET+:`GROB_IN_MAPPING_READY_WIDTH]=!vif.reset&&!reference.hold_mapping&&reference.cycle%4!=0; + for (int lane = 0; lane < `GROB_ENTRIES; lane++) begin + bit hazard = 0; + for (int port = 0; port < 3; port++) begin + if (vif.outputs[query_offsets[lane][port*2]]) + hazard |= reference.busy[vif.outputs[query_offsets[lane][port*2+1]+:`GROB_BANK_BITS]]; + end + vif.inputs[hazard_offsets[lane]] = hazard; + end + if (vif.reset) begin + vif.inputs[`GROB_IN_COMPLETE_VALID_OFFSET+:`GROB_IN_COMPLETE_VALID_WIDTH] = 0; + vif.inputs[`GROB_IN_COMPLETE_BITS_OFFSET+:`GROB_IN_COMPLETE_BITS_WIDTH] = 0; + end else begin + if (reference.active_completion == null && !reference.hold_complete) begin + chosen = -1; + foreach (reference.completion_queue[i]) begin + if(reference.completion_queue[i].due<=reference.cycle&&(chosen<0||reference.reverse_complete)) + chosen = i; + end + if (chosen >= 0) begin + reference.active_completion = reference.completion_queue[chosen]; + reference.completion_queue.delete(chosen); + end + end + vif.inputs[`GROB_IN_COMPLETE_VALID_OFFSET+:`GROB_IN_COMPLETE_VALID_WIDTH]=reference.active_completion!=null; + vif.inputs[`GROB_IN_COMPLETE_BITS_OFFSET+:`GROB_IN_COMPLETE_BITS_WIDTH]=reference.active_completion!=null?reference.active_completion.slot:0; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/env/env.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/env/env.svh new file mode 100644 index 000000000..387df80ca --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/env/env.svh @@ -0,0 +1,36 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + peer_driver peer; + model reference; + reset_controller reset; + uvm_sequencer #(command_item) seqr; + result_board #(result_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + peer = peer_driver::type_id::create("peer", this); + reference = model::type_id::create("model", this); + reset = reset_controller::type_id::create("reset", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + peer.reference = reference; + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + reference.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/env/model.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/env/model.svh new file mode 100644 index 000000000..588a8be0d --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/env/model.svh @@ -0,0 +1,475 @@ +class model extends uvm_subscriber #(sample_item); + `uvm_component_utils(model) + localparam int N = `GROB_ENTRIES, V = `GROB_VBANKS, A = `GROB_ALIASES; + transaction slots[N], order[$], completion_queue[$], active_completion; + int current[V], tail = 0, cycle = 0, serial = 0; + longint unsigned next_inst = 1; + bit [A-1:0] free_aliases, busy; + bit hold_issue = 0, hold_mapping = 0, hold_complete = 0, reverse_complete = 0; + int allocated = 0, selected = 0, delivered = 0, completed = 0, retired = 0, reordered = 0; + sample_item latest; + int cancelled_live = 0; + int trace_offsets[4][10] = `GROB_TRACE_OFFSETS; + int query_offsets[N][9] = `GROB_QUERY_OFFSETS; + int ba_offsets[9] = `GROB_BA_OFFSETS; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + covergroup behavior with function sample ( + int count, bit stalled, bit mapping_stall, bit reordered, bit config_op, bit movement + ); + option.per_instance = 1; + occupancy: coverpoint count {bins empty = {0}; bins partial = {[1 : N - 1]}; bins full = {N};} + issue_backpressure: coverpoint stalled; + mapping_backpressure: coverpoint mapping_stall; + out_of_order_completion: coverpoint reordered; + configuration: coverpoint config_op; + movement_command: coverpoint movement; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + behavior = new(); + endfunction + function void predict_bat(logic [`GROB_INPUT_BITS-1:0] pins, output bank_access_t renamed, + output bit ready, rename, reset_read, output bit [1:0] stale_valid, + output int stale0, stale1); + int fresh, wr, r0, r1, rw, funct; + bit update, resets_write; + renamed = '0; + fresh = 0; + for (int index = 0; index < A; index++) + if (free_aliases[index]) begin + fresh = index; + break; + end + wr = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_WR_BANK_ID); + r0 = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_RD_BANK_0_ID); + r1 = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_RD_BANK_1_ID); + rw = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_RD_WR_ID); + funct = `GI(pins, ALLOC_BITS_OPERATION_CMD_RAW_INST) >> 25; + update = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_WR_UPDATE); + rename = `GROB_RENAMING && + `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_WR_BANK_VALID) + && wr < V && funct != 32 && !update; + reset_read = funct == 32 && (`GI(pins, ALLOC_BITS_OPERATION_CMD_RS2DATA) & 64'h1000) != 0; + ready = !rename || free_aliases != 0; + `BA(renamed, RD_BANK_0_VALID) = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_RD_BANK_0_VALID); + `BA(renamed, RD_BANK_1_VALID) = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_RD_BANK_1_VALID); + `BA(renamed, RD_WR_VALID) = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_RD_WR_VALID); + `BA(renamed, WR_BANK_VALID) = `GI(pins, ALLOC_BITS_OPERATION_BANKACCESS_WR_BANK_VALID); + `BA(renamed, WR_UPDATE) = update; + `BA(renamed, RD_BANK_0_ID) = `BA(renamed, RD_BANK_0_VALID) ? current[r0] : 0; + `BA(renamed, RD_BANK_1_ID) = `BA(renamed, RD_BANK_1_VALID) ? current[r1] : 0; + `BA(renamed, RD_WR_ID) = `BA(renamed, RD_WR_VALID) ? current[rw] : 0; + `BA(renamed, WR_BANK_ID) = rename ? fresh : update ? current[wr] : wr; + resets_write = `BA(renamed, WR_BANK_VALID) && !rename && !update && current[wr] != wr; + stale_valid[0] = rename || (!reset_read && resets_write) || (reset_read && current[r0] != r0); + stale_valid[1] = reset_read && resets_write && r0 != wr; + stale0 = reset_read ? current[r0] : current[wr]; + stale1 = current[wr]; + endfunction + function bit hazard(bank_access_t access); + return ( + `BA(access, RD_BANK_0_VALID) + && busy[ + `BA(access, RD_BANK_0_ID) + ]) || ( + `BA(access, RD_BANK_1_VALID) + && busy[ + `BA(access, RD_BANK_1_ID) + ]) || ( + `BA(access, RD_WR_VALID) + && busy[ + `BA(access, RD_WR_ID) + ]); + endfunction + function bit eligible(transaction candidate); + bit conflict, overlap; + if (hazard(candidate.renamed)) return 0; + foreach (order[i]) begin + transaction older = order[i]; + if (older == candidate) break; + overlap=((candidate.reads&older.writes)|(candidate.writes&(older.reads|older.writes)))!=0; + conflict = (!`GROB_RENAMING || !`GROB_LAYOUT) ? overlap : 0; + conflict |= ( + `OP(candidate.operation, BANKACCESS_WR_UPDATE) + && overlap) || ( + `OP(older.operation, BANKACCESS_WR_UPDATE) + && (candidate.reads & older.writes) != 0); + if (!older.completed && conflict) return 0; + if((candidate.config_op&&older.config_op)||((candidate.config_op||older.config_op)&& + ((candidate.reads|candidate.writes)&(older.reads|older.writes))!=0)) + return 0; + if (candidate.movement && !older.completed) return 0; + end + return 1; + endfunction + function void verify_trace(logic [`GROB_OUTPUT_BITS-1:0] outputs, int index, transaction r, + int kind); + evidence::check( + outputs[trace_offsets[index][0]+:8]==kind&& + outputs[trace_offsets[index][1]+:64]==`GROB_OWNER&&outputs[trace_offsets[index][2]+:32]==r.slot&& + outputs[trace_offsets[index][3]+:64]==r.inst_id&&outputs[trace_offsets[index][4]+:32]== + `OP(r.operation, DOMAIN_ID) + && outputs[trace_offsets[index][5]+:32] == r.funct && + outputs[trace_offsets[index][6]+:64] == r.pc && outputs[trace_offsets[index][7]+:64] == + `OP(r.operation, CMD_RS1DATA) && outputs[trace_offsets[index][8]+:64] == + `OP(r.operation, CMD_RS2DATA), + "Global ROB public instruction observation changed its fields"); + endfunction + function void write(sample_item t); + transaction candidate, map_retire, r, ignored, oldest_incomplete; + result_item expected, actual; + bank_access_t renamed, query_access; + bit bat_ready, rename, reset_read, input_fire, select_fire, issue_fire, complete_fire, map_fire; + bit [1:0] stale_valid; + bit movement_live, has_retirement, needs_mapping, reads_previous, completion_reordered; + int stale0, stale1, head, map_index, retire_count, cid; + latest = t; + if (t.reset) begin + reset_ap.write(1); + cancelled_live += order.size(); + order.delete(); + completion_queue.delete(); + active_completion = null; + tail = 0; + next_inst = 1; + busy = 0; + free_aliases = '1; + foreach (slots[i]) slots[i] = null; + foreach (current[i]) begin + current[i] = i; + free_aliases[i] = 0; + end + return; + end + cycle++; + completion_reordered = 0; + predict_bat(t.inputs, renamed, bat_ready, rename, reset_read, stale_valid, stale0, stale1); + head = order.size() ? order[0].slot : tail; + input_fire = `GI(t.inputs, ALLOC_VALID) && `GO(t.outputs, ALLOC_READY); + select_fire = `GO(t.outputs, SCOREBOARD_ISSUE_VALID); + issue_fire = `GO(t.outputs, ISSUE_VALID) && `GI(t.inputs, ISSUE_READY); + complete_fire = `GI(t.inputs, COMPLETE_VALID) && `GO(t.outputs, COMPLETE_READY); + map_fire = `GO(t.outputs, MAPPING_VALID) && `GI(t.inputs, MAPPING_READY); + candidate = null; + map_retire = null; + map_index = 0; + retire_count = 0; + movement_live = 0; + foreach (order[i]) begin + if (candidate == null && !order[i].selected && !order[i].completed) candidate = order[i]; + if (order[i].movement && !order[i].completed) movement_live = 1; + end + // Retirement is a completed FIFO prefix. A final mapping acknowledgement releases its entry. + foreach (order[i]) begin + if (!order[i].completed) break; + if (order[i].stale_valid != 0) begin + map_retire = order[i]; + map_index = order[i].stale_valid[0] ? 0 : 1; + break; + end + end + if (map_fire && `GO(t.outputs, MAPPING_BITS_RETIRE)) begin + evidence::check(map_retire != null, "Global ROB retired an unsolicited mapping"); + evidence::check(`GO(t.outputs, MAPPING_BITS_BANK) + == (map_index == 0 ? map_retire.stale_raw : map_retire.raw_write) && + `GO(t.outputs, MAPPING_BITS_VBANK) + == (map_index == 0 ? map_retire.new_alias : map_retire.raw_write) && + `GO(t.outputs, MAPPING_BITS_PREVIOUS) == map_retire.stale[map_index] && + `GO(t.outputs, MAPPING_BITS_THREADID) == `GROB_OWNER && ! + `GO(t.outputs, MAPPING_BITS_ALLOCATE) && !`GO(t.outputs, MAPPING_BITS_REUSE) + && !`GO(t.outputs, MAPPING_BITS_WRITE), + "Global ROB mapping retirement metadata mismatch"); + end + foreach (order[i]) begin + if (!order[i].completed) break; + if (order[i].stale_valid != 0 && !(order[i] == map_retire && map_fire && + `GO(t.outputs, MAPPING_BITS_RETIRE) + && $countones( + order[i].stale_valid + ) == 1)) + break; + retire_count++; + end + has_retirement = retire_count != 0; + evidence::check(`GO(t.outputs, EMPTY) == (order.size() == 0) && `GO(t.outputs, FULL) + == (order.size() == N), "Global ROB occupancy flags mismatch"); + evidence::check(`GO(t.outputs, ALLOC_READY) + == (order.size() < N && !has_retirement && bat_ready && !movement_live), + "Global ROB admission rule mismatch"); + evidence::check(`GO(t.outputs, BAT_ALLOC_VALID) == input_fire, + "Global ROB BAT allocation handshake mismatch"); + evidence::check(`GO(t.outputs, TRACEINSTRUCTIONS_0_ENABLE) == input_fire && + `GO(t.outputs, TRACEINSTRUCTIONS_2_ENABLE) == complete_fire, + "Global ROB public observation handshakes mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_VALID) == (input_fire && rename), + "Global ROB scoreboard allocation handshake mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_VALID) == complete_fire, + "Global ROB completion handshake mismatch"); + evidence::check(`GO(t.outputs, TRACEINSTRUCTIONS_1_ENABLE) == select_fire, + "Global ROB issue observation mismatch"); + for (int age = 0; age < N; age++) begin + int slot = (head + age) % N; + if (slots[slot] != null) + for (int field = 0; field < 9; field++) begin + if (field % 2 == 0) + evidence::check( + t.outputs[query_offsets[age][field]] === slots[slot].renamed[ba_offsets[field]], + "Global ROB query flag lost its entry"); + else + evidence::check( + t.outputs[query_offsets[age][field]+:`GROB_BANK_BITS]===slots[slot].renamed[ba_offsets[field]+:`GROB_BANK_BITS], + "Global ROB query ID lost its alias"); + end + end + if (map_fire && !`GO(t.outputs, MAPPING_BITS_RETIRE)) begin + evidence::check(candidate != null && eligible(candidate) && select_fire, + "Global ROB mapping allocation bypassed a dependency"); + needs_mapping = candidate.allocated || ( + `BA(candidate.renamed, WR_BANK_VALID) + && ! + `BA(candidate.renamed, WR_UPDATE) + && candidate.raw_write < V && !candidate.config_op); + reads_previous = ( + `BA(candidate.renamed, RD_BANK_0_VALID) + && + `BA(candidate.renamed, RD_BANK_0_ID) + == candidate.stale[0]) || ( + `BA(candidate.renamed, RD_BANK_1_VALID) + && + `BA(candidate.renamed, RD_BANK_1_ID) + == candidate.stale[0]) || ( + `BA(candidate.renamed, RD_WR_VALID) + && + `BA(candidate.renamed, RD_WR_ID) + == candidate.stale[0]); + evidence::check(needs_mapping && `GO(t.outputs, MAPPING_BITS_BANK) == candidate.raw_write && + `GO(t.outputs, MAPPING_BITS_VBANK) == `BA(candidate.renamed, WR_BANK_ID) && + `GO(t.outputs, MAPPING_BITS_PREVIOUS) == candidate.stale[0] && + `GO(t.outputs, MAPPING_BITS_THREADID) == `GROB_OWNER && + `GO(t.outputs, MAPPING_BITS_ALLOCATE) == candidate.allocated && + `GO(t.outputs, MAPPING_BITS_REUSE) + == (candidate.allocated && candidate.slot == head && !reads_previous) && + `GO(t.outputs, MAPPING_BITS_WRITE) == (`BA(candidate.renamed, WR_BANK_VALID) + && !`BA(candidate.renamed, WR_UPDATE) + && candidate.raw_write < V && !candidate.config_op), + "Global ROB mapping issue metadata mismatch"); + end + if (select_fire) begin + evidence::check(candidate != null && eligible(candidate), + "Global ROB selected a blocked or younger instruction"); + needs_mapping = candidate.allocated || ( + `BA(candidate.renamed, WR_BANK_VALID) + && ! + `BA(candidate.renamed, WR_UPDATE) + && candidate.raw_write < V && !candidate.config_op); + if (needs_mapping) + evidence::check(map_fire && !`GO(t.outputs, MAPPING_BITS_RETIRE), + "Global ROB selected work before mapping acceptance"); + + evidence::check(`GO(t.outputs, TRACEINSTRUCTIONS_1_ROB_ID) == candidate.slot && + `GO(t.outputs, TRACEINSTRUCTIONS_1_PC) == candidate.pc && + `GO(t.outputs, TRACEINSTRUCTIONS_1_INST_ID) == candidate.inst_id && + `GO(t.outputs, TRACEINSTRUCTIONS_1_THREAD_ID) == `GROB_OWNER, + "Global ROB issue identity mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_RD_BANK_0_VALID) === + `BA(candidate.renamed, RD_BANK_0_VALID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_RD_BANK_0_ID) === + `BA(candidate.renamed, RD_BANK_0_ID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_RD_BANK_1_VALID) === + `BA(candidate.renamed, RD_BANK_1_VALID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_RD_BANK_1_ID) === + `BA(candidate.renamed, RD_BANK_1_ID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_RD_WR_VALID) === + `BA(candidate.renamed, RD_WR_VALID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_RD_WR_ID) === + `BA(candidate.renamed, RD_WR_ID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_WR_BANK_VALID) === + `BA(candidate.renamed, WR_BANK_VALID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_WR_BANK_ID) === + `BA(candidate.renamed, WR_BANK_ID), + "Global ROB scoreboard issue field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ISSUE_BITS_WR_UPDATE) === + `BA(candidate.renamed, WR_UPDATE), + "Global ROB scoreboard issue field mismatch"); + verify_trace(t.outputs, 1, candidate, 1); + candidate.selected = 1; + selected++; + end + if (issue_fire) begin + int slot = `GO(t.outputs, ISSUE_BITS_ROB_ID); + r = slots[slot]; + evidence::check(r != null && r.selected && !r.delivered, + "Global ROB delivered an unselected or duplicate instruction"); + actual = result_item::type_id::create("issued"); + actual.payload = t.outputs[`GROB_ISSUE_OFFSET+:`GROB_ISSUE_BITS]; + actual_ap.write(actual); + r.delivered = 1; + r.due = cycle + (r.serial % 4 == 0 ? 17 : 3); + completion_queue.push_back(r); + delivered++; + end + if (complete_fire) begin + cid = `GI(t.inputs, COMPLETE_BITS); + r = slots[cid]; + evidence::check(r != null && r.delivered && !r.completed && active_completion == r, + "Global ROB accepted an invalid completion"); + oldest_incomplete = null; + foreach (order[i]) + if (oldest_incomplete == null && !order[i].completed) oldest_incomplete = order[i]; + if (r != oldest_incomplete) begin + reordered++; + completion_reordered = 1; + end + evidence::check(`GO(t.outputs, TRACEINSTRUCTIONS_2_ROB_ID) == r.slot && + `GO(t.outputs, TRACEINSTRUCTIONS_2_PC) == r.pc && + `GO(t.outputs, TRACEINSTRUCTIONS_2_INST_ID) == r.inst_id, + "Global ROB completion identity mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_RD_BANK_0_VALID) === + `BA(r.renamed, RD_BANK_0_VALID), + "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_RD_BANK_0_ID) === + `BA(r.renamed, RD_BANK_0_ID), + "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_RD_BANK_1_VALID) === + `BA(r.renamed, RD_BANK_1_VALID), + "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_RD_BANK_1_ID) === + `BA(r.renamed, RD_BANK_1_ID), + "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_RD_WR_VALID) === + `BA(r.renamed, RD_WR_VALID), "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_RD_WR_ID) === + `BA(r.renamed, RD_WR_ID), "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_WR_BANK_VALID) === + `BA(r.renamed, WR_BANK_VALID), + "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_WR_BANK_ID) === + `BA(r.renamed, WR_BANK_ID), "Global ROB scoreboard complete field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_COMPLETE_BITS_WR_UPDATE) === + `BA(r.renamed, WR_UPDATE), "Global ROB scoreboard complete field mismatch"); + verify_trace(t.outputs, 2, r, 0); + r.completed = 1; + completed++; + active_completion = null; + end + if (input_fire) begin + evidence::check(t.inputs[`GROB_IN_RENAMED_OFFSET+:`GROB_IN_RENAMED_BITS] === renamed, + "Global ROB peer alias changed before allocation"); + evidence::check(`GO(t.outputs, BAT_ALLOC_RENAME) == rename && + `GO(t.outputs, BAT_ALLOC_RESET_READ0) == reset_read, + "Global ROB requested the wrong alias operation"); + r = new; + r.operation = t.inputs[`GROB_IN_OPERATION_OFFSET+:`GROB_IN_OPERATION_BITS]; + r.renamed = renamed; + r.slot = tail; + r.serial = serial++; + r.pc = `OP(r.operation, CMD_PC); + r.inst_id = `OP(r.operation, RUNTIME) ? next_inst++ : 0; + r.funct = `OP(r.operation, CMD_RAW_INST) >> 25; + r.config_op = r.funct == 32; + r.movement = r.funct == 13; + r.raw_write = `OP(r.operation, BANKACCESS_WR_BANK_ID); + r.new_alias = reset_read ? + `OP(r.operation, BANKACCESS_RD_BANK_0_ID) + : + `BA(renamed, WR_BANK_ID); + r.stale_raw = reset_read ? `OP(r.operation, BANKACCESS_RD_BANK_0_ID) : r.raw_write; + r.allocated = rename; + r.stale_valid = stale_valid; + r.stale[0] = stale0; + r.stale[1] = stale1; + r.reads = 0; + r.writes = 0; + if (`OP(r.operation, BANKACCESS_RD_BANK_0_VALID)) + r.reads[`OP(r.operation, BANKACCESS_RD_BANK_0_ID)] = 1; + if (`OP(r.operation, BANKACCESS_RD_BANK_1_VALID)) + r.reads[`OP(r.operation, BANKACCESS_RD_BANK_1_ID)] = 1; + if (`OP(r.operation, BANKACCESS_RD_WR_VALID)) + r.reads[`OP(r.operation, BANKACCESS_RD_WR_ID)] = 1; + if (`OP(r.operation, BANKACCESS_WR_BANK_VALID)) r.writes[r.raw_write] = 1; + r.expected = '0; + r.expected[`GROB_OUT_COMMAND_OFFSET-`GROB_ISSUE_OFFSET+:`GROB_OUT_COMMAND_BITS] = r.operation; + r.expected[`GROB_OUT_CONTEXT_OFFSET-`GROB_ISSUE_OFFSET+:`GROB_OUT_CONTEXT_BITS]=t.inputs[`GROB_IN_CONTEXT_OFFSET+:`GROB_IN_CONTEXT_BITS]; + r.expected[`GROB_OUT_RENAMED_OFFSET-`GROB_ISSUE_OFFSET+:`GROB_OUT_RENAMED_BITS] = renamed; + `ISS(r.expected, ROB_ID) = tail; + `ISS(r.expected, INSTID) = r.inst_id; + expected = result_item::type_id::create("allocated"); + expected.payload = r.expected; + expected_ap.write(expected); + verify_trace(t.outputs, 0, r, 2); + slots[tail] = r; + order.push_back(r); + tail = (tail + 1) % N; + allocated++; + if (rename) begin + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_RD_BANK_0_VALID) === + `BA(renamed, RD_BANK_0_VALID), + "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_RD_BANK_0_ID) === + `BA(renamed, RD_BANK_0_ID), "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_RD_BANK_1_VALID) === + `BA(renamed, RD_BANK_1_VALID), + "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_RD_BANK_1_ID) === + `BA(renamed, RD_BANK_1_ID), "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_RD_WR_VALID) === + `BA(renamed, RD_WR_VALID), "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_RD_WR_ID) === `BA(renamed, RD_WR_ID), + "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_WR_BANK_VALID) === + `BA(renamed, WR_BANK_VALID), "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_WR_BANK_ID) === + `BA(renamed, WR_BANK_ID), "Global ROB scoreboard alloc field mismatch"); + evidence::check(`GO(t.outputs, SCOREBOARD_ALLOC_BITS_WR_UPDATE) === `BA(renamed, WR_UPDATE), + "Global ROB scoreboard alloc field mismatch"); + end + if (rename) free_aliases[`BA(renamed, WR_BANK_ID)] = 0; + if (`BA(renamed, WR_BANK_VALID)) current[r.raw_write] = `BA(renamed, WR_BANK_ID); + if (reset_read) + current[ + `OP(r.operation, BANKACCESS_RD_BANK_0_ID) + ] = + `OP(r.operation, BANKACCESS_RD_BANK_0_ID); + end + if (select_fire && `BA(candidate.renamed, WR_BANK_VALID)) + busy[`BA(candidate.renamed, WR_BANK_ID)] = 1; + if (input_fire && rename) busy[`BA(renamed, WR_BANK_ID)] = 1; + if (complete_fire) begin + r = slots[cid]; + if (`BA(r.renamed, WR_BANK_VALID)) busy[`BA(r.renamed, WR_BANK_ID)] = 0; + end + if (map_fire && `GO(t.outputs, MAPPING_BITS_RETIRE)) begin + map_retire.stale_valid[map_index] = 0; + if (map_retire.stale[map_index] >= V) free_aliases[map_retire.stale[map_index]] = 1; + end + behavior.sample(order.size(), `GO(t.outputs, ISSUE_VALID) && !`GI(t.inputs, ISSUE_READY), + `GO(t.outputs, MAPPING_VALID) && !`GI(t.inputs, MAPPING_READY), + completion_reordered, input_fire && ( + `GI(t.inputs, ALLOC_BITS_OPERATION_CMD_RAW_INST) >> 25) == 32, input_fire && ( + `GI(t.inputs, ALLOC_BITS_OPERATION_CMD_RAW_INST) >> 25) == 13); + repeat (retire_count) begin + ignored = order.pop_front(); + slots[ignored.slot] = null; + retired++; + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + evidence::check(order.size() == 0 && completion_queue.size() == 0 && active_completion == null, + "Global ROB left live instructions"); + evidence::check(retired + cancelled_live == allocated, + "Global ROB allocation/retirement accounting mismatch"); + evidence::check(reordered != 0, "Global ROB did not exercise out-of-order completion"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/filelist.f b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/filelist.f new file mode 100644 index 000000000..ce6c74713 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@/GlobalROB ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/GlobalROB/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/interface.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/interface.sv new file mode 100644 index 000000000..303a1ff62 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/interface.sv @@ -0,0 +1,14 @@ +`include "parameters.svh" +interface global_if ( + input logic clock +); + logic reset; + logic [`GROB_INPUT_BITS-1:0] inputs; + logic [`GROB_OUTPUT_BITS-1:0] outputs; + assert property (@(posedge clock) disable iff (reset) + outputs[`GROB_OUT_ISSUE_VALID_OFFSET] && !inputs[`GROB_IN_ISSUE_READY_OFFSET] |=> + outputs[`GROB_OUT_ISSUE_VALID_OFFSET] && $stable( + outputs[`GROB_ISSUE_OFFSET+:`GROB_ISSUE_BITS] + )) + else $fatal(1, "Global ROB issue payload changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/package.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/package.sv new file mode 100644 index 000000000..6c5d3663e --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/package.sv @@ -0,0 +1,24 @@ +`include "parameters.svh" +`define GI(P, N) P[`GROB_IN_``N``_OFFSET+:`GROB_IN_``N``_WIDTH] +`define GO(P, N) P[`GROB_OUT_``N``_OFFSET+:`GROB_OUT_``N``_WIDTH] +`define OP(P, + N) P[`GROB_IN_ALLOC_BITS_OPERATION_``N``_OFFSET-`GROB_IN_OPERATION_OFFSET+:`GROB_IN_ALLOC_BITS_OPERATION_``N``_WIDTH] +`define CP( + P, N) P[`GROB_IN_ALLOC_BITS_``N``_OFFSET-`GROB_IN_ALLOC_OFFSET+:`GROB_IN_ALLOC_BITS_``N``_WIDTH] +`define BA(P, + N) P[`GROB_IN_BAT_RENAMED_``N``_OFFSET-`GROB_IN_RENAMED_OFFSET+:`GROB_IN_BAT_RENAMED_``N``_WIDTH] +`define ISS( + P, N) P[`GROB_OUT_ISSUE_BITS_``N``_OFFSET-`GROB_ISSUE_OFFSET+:`GROB_OUT_ISSUE_BITS_``N``_WIDTH] +package global_rob_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "env/model.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/peer.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/seq/sequence.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/seq/sequence.svh new file mode 100644 index 000000000..ff829cb8a --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/seq/sequence.svh @@ -0,0 +1,123 @@ +class scenario_sequence extends uvm_sequence; + `uvm_object_utils(scenario_sequence) + env test_env; + virtual global_if vif; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task send(int serial, int kind, int write_bank, int read0, int read1); + command_sequence seq = command_sequence::type_id::create("command"); + command_t packet = '0; + longint unsigned rs1, rs2; + int funct; + bit wr, rd0, rd1, update, config_op, movement; + operation_t operation; + wr = kind == 1 || kind == 3 || kind == 4 || kind == 5 || kind == 6 || kind == 7; + rd0 = kind == 2 || kind == 3 || kind == 4 || kind == 5 || kind == 7; + rd1 = kind == 4 || kind == 5 || kind == 7; + update = kind == 5; + config_op = kind == 6 || kind == 7; + movement = kind == 8; + if (kind == 6) read0 = write_bank; + if (kind == 7) read1 = write_bank; + case (kind) + 0: funct = 5; + 1: funct = 'h24; + 2: funct = 'h14; + 3: funct = 'h34; + 4, 5: funct = 'h44; + 6, 7: funct = 32; + 8: funct = 13; + default: `uvm_fatal("KIND", "Unknown global ROB scenario command") + endcase + rs1=(64'(serial%17+1)<<30)|(64'(write_bank)<<20)|(64'(read1)<<10)|read0|(update?64'h8000000000000000:0); + rs2 = config_op ? (kind == 7 ? 64'h1002 : 64'h2) : (64'h123456789abcdef0 ^ serial); + for (int bit_index = 0; bit_index < `GROB_IN_CONTEXT_BITS; bit_index++) + packet[`GROB_IN_CONTEXT_OFFSET-`GROB_IN_ALLOC_OFFSET+bit_index]=(serial>>(bit_index%16))^((bit_index+serial)%3==0); + operation = '0; + `OP(operation, RUNTIME) = serial % 5 != 0; + `OP(operation, DOMAIN_ID) = config_op || movement ? 1 : 3; + `OP(operation, ITER) = serial % 17 + 1; + `OP(operation, CMD_RAW_INST) = (funct << 25) | 'h7b; + `OP(operation, CMD_PC) = 64'hdead000010000000 + 64'(serial) * 4; + `OP(operation, CMD_RS1DATA) = rs1; + `OP(operation, CMD_RS2DATA) = rs2; + `OP(operation, BANKACCESS_WR_BANK_VALID) = wr; + `OP(operation, BANKACCESS_WR_BANK_ID) = write_bank; + `OP(operation, BANKACCESS_RD_BANK_0_VALID) = rd0; + `OP(operation, BANKACCESS_RD_BANK_0_ID) = read0; + `OP(operation, BANKACCESS_RD_BANK_1_VALID) = rd1; + `OP(operation, BANKACCESS_RD_BANK_1_ID) = read1; + `OP(operation, BANKACCESS_RD_WR_VALID) = update; + `OP(operation, BANKACCESS_RD_WR_ID) = write_bank; + `OP(operation, BANKACCESS_WR_UPDATE) = update; + `OP(operation, ARCHITECTURALDESTINATION) = write_bank; + packet[`GROB_IN_OPERATION_OFFSET-`GROB_IN_ALLOC_OFFSET+:`GROB_IN_OPERATION_BITS] = operation; + seq.item = command_item::type_id::create("item"); + seq.item.payload = packet; + seq.start(test_env.seqr); + endtask + task drain(); + do + @(negedge vif.clock); + while (test_env.reference.order.size() != 0 || ! + `GO(test_env.reference.latest.outputs, EMPTY) + ); + repeat (3) @(negedge vif.clock); + endtask + task body(); + int before_delivered; + test_env.reference.hold_complete = 1; + test_env.reference.hold_issue = 1; + test_env.reference.reverse_complete = 1; + test_env.reset.pulse(4); + for (int i = 0; i < `GROB_ENTRIES; i++) send(i + 1, 1, i % `GROB_VBANKS, 0, 0); + repeat (5) @(negedge vif.clock); + evidence::check(`GO(test_env.reference.latest.outputs, FULL), + "Global ROB did not reach full occupancy"); + fork + send(`GROB_ENTRIES + 1, 1, 0, 0, 0); + begin + repeat (5) @(negedge vif.clock); + evidence::check(!`GO(test_env.reference.latest.outputs, ALLOC_READY), + "Global ROB admitted work while full"); + test_env.reference.hold_issue = 0; + test_env.reference.hold_complete = 0; + end + join + drain(); + before_delivered = test_env.reference.delivered; + test_env.reference.hold_mapping = 1; + send(6, 1, 0, 0, 0); + send(7, 2, 0, 1 % `GROB_VBANKS, 0); + repeat (10) @(negedge vif.clock); + evidence::check(test_env.reference.delivered == before_delivered && + `GO(test_env.reference.latest.outputs, MAPPING_VALID), + "Global ROB bypassed a blocked oldest mapping request"); + test_env.reference.hold_mapping = 0; + drain(); + for (int n = 10; n < 510; n++) + send(n, n % 9, (n / 3) % `GROB_VBANKS, n % `GROB_VBANKS, (n + 1) % `GROB_VBANKS); + drain(); + // A partially selected issue pipeline and all its queued work cancel on reset. + test_env.reference.hold_issue = 1; + test_env.reference.hold_complete = 1; + for (int n = 0; n < `GROB_ENTRIES; n++) send(600 + n, 1, n % `GROB_VBANKS, 0, 0); + repeat (6) @(negedge vif.clock); + test_env.reset.pulse(4); + test_env.reference.hold_issue = 0; + test_env.reference.hold_complete = 0; + for (int n = 0; n < 32; n++) + send(700 + n, n % 6, n % `GROB_VBANKS, (n + 1) % `GROB_VBANKS, (n + 2) % `GROB_VBANKS); + drain(); + evidence::check(evidence::cancelled == `GROB_ENTRIES, + "Global ROB reset cancellation count mismatch"); + `uvm_info( + "GLOBAL_ROB", + $sformatf( + "allocated=%0d selected=%0d delivered=%0d completed=%0d retired=%0d reset_cancelled=%0d reordered=%0d", + test_env.reference.allocated, test_env.reference.selected, + test_env.reference.delivered, test_env.reference.completed, test_env.reference.retired, + test_env.reference.cancelled_live, test_env.reference.reordered), UVM_LOW) + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/tb.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/tb.sv new file mode 100644 index 000000000..119039c69 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/tb.sv @@ -0,0 +1,19 @@ +module global_rob_tb; + import uvm_pkg::*; + import global_rob_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + global_if bus (clock); + assign bus.reset = timing.reset; + GlobalROB dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual global_if)::set(null, "uvm_test_top*", "vif", bus); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top.env.reset", "timing", timing); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/tests/test.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/tests/test.svh new file mode 100644 index 000000000..8acdd1560 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/global_rob/tests/test.svh @@ -0,0 +1,22 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + virtual global_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + if (!uvm_config_db#(virtual global_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Global ROB timing interface is required") + if (`GROB_DIFF || `GROB_GP) + `uvm_fatal("PROFILE", "Global ROB scenarios require normal execution with private banks") + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.vif = vif; + seq.start(null); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/driver.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/driver.svh new file mode 100644 index 000000000..308166c78 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual rob_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual rob_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "ROB storage interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.inputs = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/item.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/item.svh new file mode 100644 index 000000000..6ecbd9a3f --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`ROB_INPUT_BITS-1:0] inputs; + logic [`ROB_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + logic [`ROB_OUTPUT_BITS-1:0] payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/monitor.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/monitor.svh new file mode 100644 index 000000000..81d33e587 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual rob_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual rob_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "ROB storage monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/env/env.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/env/env.svh new file mode 100644 index 000000000..ed95874fb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver drv; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/env/model.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/env/model.svh new file mode 100644 index 000000000..49284a2ed --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/env/model.svh @@ -0,0 +1,117 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + localparam int N = `ROB_ENTRIES; + logic [`ROB_DATA_BITS-1:0] data[N]; + bit valid[N], issued[N], done[N], barrier[N]; + bit [N-1:0] dependencies[N]; + int order[$], tail = 0; + int state_offsets[N][5] = `ROB_STATE_OFFSETS; + int ready_offsets[N] = `ROB_READY_OFFSETS; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + covergroup behavior with function sample ( + int count, int retired, bit blocked, bit simultaneous, bit commit_barrier + ); + option.per_instance = 1; + occupancy: coverpoint count {bins empty = {0}; bins partial = {[1 : N - 1]}; bins full = {N};} + retirement: coverpoint retired { + bins none = {0}; bins single = {1}; bins multiple = {[2 : `ROB_RETIRE_WIDTH]}; + } + source_backpressure: coverpoint blocked; + issue_complete: coverpoint simultaneous; + sticky_dependency: coverpoint commit_barrier; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + behavior = new(); + endfunction + function void write(cycle_item t); + result_item expected, actual; + bit [N-1:0] candidates, retiring, old_barriers; + int retire_count, in_flight, head, ignored; + bit admit; + if (t.reset) begin + foreach (data[i]) begin + data[i] = 0; + valid[i] = 0; + issued[i] = 0; + done[i] = 0; + barrier[i] = 0; + dependencies[i] = 0; + end + order.delete(); + tail = 0; + return; + end + candidates = 0; + retiring = 0; + retire_count = 0; + in_flight = 0; + old_barriers = 0; + foreach (barrier[i]) old_barriers[i] = barrier[i]; + for (int age = 0; age < `ROB_RETIRE_WIDTH && age < order.size(); age++) begin + if (!done[order[age]]) break; + candidates[order[age]] = 1; + end + for (int age = 0; age < `ROB_RETIRE_WIDTH && age < order.size(); age++) begin + if (!done[order[age]] || !t.inputs[ready_offsets[order[age]]]) break; + retiring[order[age]] = 1; + retire_count++; + end + head = order.size() ? order[0] : tail; + admit = order.size() < N && retiring == 0; + foreach (valid[i]) if (valid[i] && issued[i] && !done[i]) in_flight++; + expected = result_item::type_id::create("expected"); + actual = result_item::type_id::create("actual"); + expected.payload = '0; + actual.payload = t.outputs; + expected.payload[`ROB_OUT_ALLOC_READY_OFFSET+:`ROB_OUT_ALLOC_READY_WIDTH] = admit; + expected.payload[`ROB_OUT_RETIRECANDIDATES_OFFSET+:`ROB_OUT_RETIRECANDIDATES_WIDTH] = candidates; + expected.payload[`ROB_OUT_RETIRED_OFFSET+:`ROB_OUT_RETIRED_WIDTH] = retiring; + expected.payload[`ROB_OUT_HEAD_OFFSET+:`ROB_OUT_HEAD_WIDTH] = head; + expected.payload[`ROB_OUT_TAIL_OFFSET+:`ROB_OUT_TAIL_WIDTH] = tail; + expected.payload[`ROB_OUT_ISSUEDCOUNT_OFFSET+:`ROB_OUT_ISSUEDCOUNT_WIDTH] = in_flight; + expected.payload[`ROB_OUT_EMPTY_OFFSET+:`ROB_OUT_EMPTY_WIDTH] = order.size() == 0; + expected.payload[`ROB_OUT_FULL_OFFSET+:`ROB_OUT_FULL_WIDTH] = order.size() == N; + foreach (data[i]) begin + expected.payload[state_offsets[i][0]+:`ROB_DATA_BITS] = data[i]; + expected.payload[state_offsets[i][1]] = valid[i]; + expected.payload[state_offsets[i][2]] = issued[i]; + expected.payload[state_offsets[i][3]] = done[i]; + expected.payload[state_offsets[i][4]+:N] = dependencies[i]; + end + behavior.sample(order.size(), retire_count, `RI(t.inputs, ALLOC_VALID) && !admit, + `RI(t.inputs, ISSUE_VALID) && `RI(t.inputs, COMPLETE_VALID), + `RI(t.inputs, ALLOC_BITS_COMMITBARRIER)); + expected_ap.write(expected); + actual_ap.write(actual); + if (`RI(t.inputs, ALLOC_VALID) && admit) begin + data[tail] = `RI(t.inputs, ALLOC_BITS_DATA); + valid[tail] = 1; + issued[tail] = 0; + done[tail] = 0; + barrier[tail] = `RI(t.inputs, ALLOC_BITS_COMMITBARRIER); + dependencies[tail] = `RI(t.inputs, ALLOC_BITS_DEPENDENCIES); + order.push_back(tail); + tail = (tail + 1) % N; + end + if (`RI(t.inputs, ISSUE_VALID)) issued[`RI(t.inputs, ISSUE_BITS)] = 1; + if (`RI(t.inputs, COMPLETE_VALID)) done[`RI(t.inputs, COMPLETE_BITS)] = 1; + foreach (dependencies[i]) begin + dependencies[i] &= ~retiring; + if ( + `RI(t.inputs, COMPLETE_VALID) + && !barrier[i] && !old_barriers[ + `RI(t.inputs, COMPLETE_BITS) + ]) + dependencies[i][`RI(t.inputs, COMPLETE_BITS)] = 0; + end + repeat (retire_count) begin + ignored = order.pop_front(); + valid[ignored] = 0; + issued[ignored] = 0; + done[ignored] = 0; + end + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/filelist.f b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/filelist.f new file mode 100644 index 000000000..a90a3eb69 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@/Rob ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Rob/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/interface.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/interface.sv new file mode 100644 index 000000000..b95dcc654 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/interface.sv @@ -0,0 +1,8 @@ +`include "parameters.svh" +interface rob_if ( + input logic clock +); + logic reset = 1; + logic [`ROB_INPUT_BITS-1:0] inputs; + logic [`ROB_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/package.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/package.sv new file mode 100644 index 000000000..68bacc400 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/package.sv @@ -0,0 +1,15 @@ +`include "parameters.svh" +`define RI(P, N) P[`ROB_IN_``N``_OFFSET+:`ROB_IN_``N``_WIDTH] +`define RO(P, N) P[`ROB_OUT_``N``_OFFSET+:`ROB_OUT_``N``_WIDTH] +package rob_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/seq/sequence.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/seq/sequence.svh new file mode 100644 index 000000000..23e9d498c --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/seq/sequence.svh @@ -0,0 +1,154 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + localparam int N = `ROB_ENTRIES; + logic [`ROB_INPUT_BITS-1:0] pins = '0; + logic [`ROB_OUTPUT_BITS-1:0] observed; + bit reset = 1; + bit [N-1:0] live = 0, issued = 0, done = 0; + int ready_offsets[N] = `ROB_READY_OFFSETS; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task tick(int cycles = 1); + cycle_item req; + int slot; + repeat (cycles) begin + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.inputs = pins; + req.reset = reset; + finish_item(req); + observed = req.outputs; + if (reset) begin + live = 0; + issued = 0; + done = 0; + end else begin + if (`RI(pins, ALLOC_VALID) && `RO(observed, ALLOC_READY)) begin + slot = `RO(observed, TAIL); + live[slot] = 1; + issued[slot] = 0; + done[slot] = 0; + `RI(pins, ALLOC_VALID) = 0; + end + if (`RI(pins, ISSUE_VALID)) issued[`RI(pins, ISSUE_BITS)] = 1; + if (`RI(pins, COMPLETE_VALID)) done[`RI(pins, COMPLETE_BITS)] = 1; + live &= ~`RO(observed, RETIRED); + issued &= ~`RO(observed, RETIRED); + done &= ~`RO(observed, RETIRED); + end + end + endtask + task restart(); + pins = '0; + reset = 1; + tick(2); + reset = 0; + tick(); + endtask + task body(); + int selected; + restart(); + for (int n = 0; n < N; n++) begin + `RI(pins, ALLOC_VALID) = 1; + `RI(pins, ALLOC_BITS_DATA) = 64'hdeadbeef00000000 | n; + `RI(pins, ALLOC_BITS_DEPENDENCIES) = live; + `RI(pins, ALLOC_BITS_COMMITBARRIER) = n % 2; + do tick(); while (`RI(pins, ALLOC_VALID)); + end + tick(3); + evidence::check(`RO(observed, FULL), "ROB did not fill"); + for (int slot = 0; slot < N; slot++) begin + `RI(pins, ISSUE_VALID) = 1; + `RI(pins, ISSUE_BITS) = slot; + tick(); + end + `RI(pins, ISSUE_VALID) = 0; + for (int slot = N - 1; slot >= 0; slot--) begin + `RI(pins, COMPLETE_VALID) = 1; + `RI(pins, COMPLETE_BITS) = slot; + tick(); + end + `RI(pins, COMPLETE_VALID) = 0; + tick(2); + foreach (ready_offsets[i]) pins[ready_offsets[i]] = 1; + tick(2); + evidence::check(`RO(observed, EMPTY), "ROB did not retire its completed batch"); + for (int cycle = 0; cycle < 1000; cycle++) begin + if (!`RI(pins, ALLOC_VALID) && cycle % 3 != 0) begin + `RI(pins, ALLOC_VALID) = 1; + `RI(pins, ALLOC_BITS_DATA) = (64'h9e3779b97f4a7c15 * cycle) ^ 64'hcafe00001234beef; + `RI(pins, ALLOC_BITS_DEPENDENCIES) = live; + `RI(pins, ALLOC_BITS_COMMITBARRIER) = cycle % 4 == 0; + end + `RI(pins, ISSUE_VALID) = 0; + `RI(pins, COMPLETE_VALID) = 0; + for (int i = 0; i < N; i++) begin + selected = (cycle + i) % N; + if (live[selected] && !issued[selected] && !done[selected]) begin + `RI(pins, ISSUE_VALID) = 1; + `RI(pins, ISSUE_BITS) = selected; + break; + end + end + for (int i = 0; i < N; i++) begin + selected = (N - 1 - i + cycle) % N; + if (live[selected] && issued[selected] && !done[selected]) begin + `RI(pins, COMPLETE_VALID) = 1; + `RI(pins, COMPLETE_BITS) = selected; + break; + end + end + foreach (ready_offsets[i]) pins[ready_offsets[i]] = (cycle + i) % 5 != 0; + tick(); + end + `RI(pins, ALLOC_VALID) = 0; + `RI(pins, ISSUE_VALID) = 0; + foreach (ready_offsets[i]) pins[ready_offsets[i]] = 1; + while (live != 0) begin + `RI(pins, COMPLETE_VALID) = 0; + for (int i = 0; i < N; i++) + if (live[i] && !done[i]) begin + `RI(pins, COMPLETE_VALID) = 1; + `RI(pins, COMPLETE_BITS) = i; + break; + end + tick(); + end + `RI(pins, COMPLETE_VALID) = 0; + // A one-cycle unit may issue and complete the same live entry in one clock. + `RI(pins, ALLOC_VALID) = 1; + `RI(pins, ALLOC_BITS_DATA) = '1; + do tick(); while (`RI(pins, ALLOC_VALID)); + selected = `RO(observed, TAIL); + `RI(pins, ISSUE_VALID) = 1; + `RI(pins, ISSUE_BITS) = selected; + `RI(pins, COMPLETE_VALID) = 1; + `RI(pins, COMPLETE_BITS) = selected; + tick(); + `RI(pins, ISSUE_VALID) = 0; + `RI(pins, COMPLETE_VALID) = 0; + tick(3); + endtask +endclass +class invalid_sequence extends scenario_sequence; + `uvm_object_utils(invalid_sequence) + function new(string name = "invalid_sequence"); + super.new(name); + endfunction + task body(); + string scenario; + if (!$value$plusargs("CASE=%s", scenario)) + `uvm_fatal("CASE", "ROB assertion scenario is required") + restart(); + if (scenario == "issue") begin + `RI(pins, ISSUE_VALID) = 1; + `RI(pins, ISSUE_BITS) = 0; + end else begin + `RI(pins, COMPLETE_VALID) = 1; + `RI(pins, COMPLETE_BITS) = 0; + end + tick(3); + `uvm_fatal("EXPECTED_ASSERTION", "Invalid ROB command did not trigger its assertion") + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/tb.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/tb.sv new file mode 100644 index 000000000..cba4070f9 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/tb.sv @@ -0,0 +1,16 @@ +module rob_tb; + import uvm_pkg::*; + import rob_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + rob_if bus (clock); + Rob dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual rob_if)::set(null, "uvm_test_top*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/tests/test.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/tests/test.svh new file mode 100644 index 000000000..0230681bb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/rob/tests/test.svh @@ -0,0 +1,35 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass +class invalid_test extends base_test; + `uvm_component_utils(invalid_test) + driver drv; + uvm_sequencer #(cycle_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction + task execute(); + invalid_sequence seq = invalid_sequence::type_id::create("scenario"); + seq.start(seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/suite.toml new file mode 100644 index 000000000..3f1dbbb1d --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/resources/suite.toml @@ -0,0 +1,71 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "rob" +top = "rob_tb" +test = "protocol_test" +filelist = "rob/filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["rob_pkg::model::behavior"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +name = "rob_bad_issue" +top = "rob_tb" +test = "invalid_test" +filelist = "rob/filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=issue"] +assertion = "ROB issue must name a live unissued entry" + +[[tests]] +name = "rob_bad_complete" +top = "rob_tb" +test = "invalid_test" +filelist = "rob/filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = [] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=complete"] +assertion = "ROB completion must name a live incomplete entry" + +[[tests]] +name = "global_rob" +top = "global_rob_tb" +test = "protocol_test" +filelist = "global_rob/filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["global_rob_pkg::model::behavior"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/DependencyResolver.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/DependencyResolver.scala new file mode 100644 index 000000000..c7e7eede0 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/DependencyResolver.scala @@ -0,0 +1,124 @@ +package framework.acc.frontend.scheduler.rob.global + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} +import chisel3.util._ +import framework.acc.frontend.decoder.PostGDCmd +import framework.acc.memdomain.DISA.MSET_BITPAT +import framework.acc.memdomain.MvoverISA + +@instantiable +class DependencyResolver( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val permissionParam: PmpParams) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val robDepth = frontendParam.rob_entries + + val resourceCount = + memParam.virtualBankCount + (if (gpParam.enable) 2 + else 0) + + @public + val io = IO(new Bundle { + val alloc = Input(new PostGDCmd(frontendParam, memParam)) + val entries = Input(Vec(robDepth, new GlobalRobEntry(frontendParam, memParam, permissionParam))) + val valid = Input(Vec(robDepth, Bool())) + val complete = Input(Vec(robDepth, Bool())) + val dependencies = Output(UInt(robDepth.W)) + val movementLive = Output(Bool()) + }) + + val entryReads = Wire(Vec(robDepth, UInt(resourceCount.W))) + val entryWrites = Wire(Vec(robDepth, UInt(resourceCount.W))) + val entryIsConfig = Wire(Vec(robDepth, Bool())) + + for (slot <- 0 until robDepth) { + val access = io.entries(slot).cmd.bankAccess + val read0 = Mux( + access.rd_bank_0_valid && access.rd_bank_0_id < resourceCount.U, + UIntToOH(access.rd_bank_0_id, resourceCount), + 0.U(resourceCount.W) + ) + val read1 = Mux( + access.rd_bank_1_valid && access.rd_bank_1_id < resourceCount.U, + UIntToOH(access.rd_bank_1_id, resourceCount), + 0.U(resourceCount.W) + ) + val readWrite = Mux( + access.rd_wr_valid && access.rd_wr_id < resourceCount.U, + UIntToOH(access.rd_wr_id, resourceCount), + 0.U(resourceCount.W) + ) + entryReads(slot) := read0 | read1 | readWrite + entryWrites(slot) := Mux( + access.wr_bank_valid && access.wr_bank_id < resourceCount.U, + UIntToOH(access.wr_bank_id, resourceCount), + 0.U(resourceCount.W) + ) + entryIsConfig(slot) := io.entries(slot).cmd.cmd.raw_inst(31, 25) === MSET_BITPAT + } + + val allocAccess = io.alloc.bankAccess + + val allocRead0 = Mux( + allocAccess.rd_bank_0_valid && allocAccess.rd_bank_0_id < resourceCount.U, + UIntToOH(allocAccess.rd_bank_0_id, resourceCount), + 0.U(resourceCount.W) + ) + + val allocRead1 = Mux( + allocAccess.rd_bank_1_valid && allocAccess.rd_bank_1_id < resourceCount.U, + UIntToOH(allocAccess.rd_bank_1_id, resourceCount), + 0.U(resourceCount.W) + ) + + val allocReadWrite = Mux( + allocAccess.rd_wr_valid && allocAccess.rd_wr_id < resourceCount.U, + UIntToOH(allocAccess.rd_wr_id, resourceCount), + 0.U(resourceCount.W) + ) + + val allocReads = allocRead0 | allocRead1 | allocReadWrite + + val allocWrites = Mux( + allocAccess.wr_bank_valid && allocAccess.wr_bank_id < resourceCount.U, + UIntToOH(allocAccess.wr_bank_id, resourceCount), + 0.U(resourceCount.W) + ) + + val allocIsConfig = io.alloc.cmd.raw_inst(31, 25) === MSET_BITPAT + + val unrenamed = + if (frontendParam.bank_alias_enable && memParam.bankLayoutEnable) + ~((BigInt(1) << memParam.virtualBankCount) - 1).U(resourceCount.W) + else Fill(resourceCount, 1.U(1.W)) + + val dependency = Wire(Vec(robDepth, Bool())) + for (older <- 0 until robDepth) { + val olderAccess = io.entries(older).cmd.bankAccess + val olderUse = entryReads(older) | entryWrites(older) + val bankConflict = + (((allocReads & entryWrites(older)) | (allocWrites & olderUse)) & unrenamed).orR || + (allocAccess.wr_update && ((allocReads & entryWrites(older)) | (allocWrites & olderUse)).orR) || + (olderAccess.wr_update && (allocReads & entryWrites(older)).orR) + val configConflict = (allocIsConfig && entryIsConfig(older)) || + ((allocIsConfig || entryIsConfig(older)) && ((allocReads | allocWrites) & olderUse).orR) + dependency(older) := + (io.valid(older) && !io.complete(older) && bankConflict) || + (io.valid(older) && configConflict) || + (io.valid(older) && !io.complete(older) && io.alloc.cmd.raw_inst(31, 25) === MvoverISA.Funct.U) + } + + io.dependencies := dependency.asUInt + io.movementLive := VecInit((0 until robDepth).map(slot => + io.valid(slot) && !io.complete(slot) && io.entries(slot).cmd.cmd.raw_inst(31, 25) === MvoverISA.Funct.U + )).asUInt.orR +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/Emit.scala new file mode 100644 index 000000000..b20fafac6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/Emit.scala @@ -0,0 +1,152 @@ +package framework.acc.frontend.scheduler.rob.global + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.acc.frontend.scheduler.rob.global.configs.RobParams + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val frontend = upickle.default.read[FrontendParam](values("frontend")) + val memory = upickle.default.read[MemDomainParam](values("memory")) + val gp = upickle.default.read[GpDomainParam](values("gp")) + val permission = upickle.default.read[PmpParams](values("permission")) + val storage = upickle.default.read[RobParams](values("rob")) + val diffTest = values("diffTest").bool + Files.createDirectories(Paths.get(output)) + + def fields(data: Data, path: String): Seq[(String, Int)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(path -> leaf.getWidth) + } + + for (name <- Seq("Rob", "GlobalROB")) { + val prefix = if (name == "Rob") "ROB" else "GROB" + val ports = scala.collection.mutable.ArrayBuffer[String]() + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + val widths = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0 + var outputOffset = 0 + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val (module, io) = + if (name == "Rob") { + val dut = new Rob(storage); (dut, dut.io) + } else { + val dut = new GlobalROB(frontend, memory, gp, permission, diffTest); (dut, dut.io) + } + for { + (root, data) <- io.elements.toSeq + (field, width) <- fields(data, root) + } { + val input = + if (name == "Rob") Set("alloc", "issue", "complete", "retireReady").contains(root) && field != "alloc_ready" + else field == "alloc_valid" || field.startsWith( + "alloc_bits_" + ) || field == "issue_ready" || field == "mapping_ready" || + field == "complete_valid" || field == "complete_bits" || field == "bat_ready" || field.startsWith( + "bat_renamed_" + ) || + field.startsWith("bat_stale_") || field.startsWith("scoreboard_hazard_") + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define ${prefix}_${direction}_${field.toUpperCase}_OFFSET $offset" + definitions += s"`define ${prefix}_${direction}_${field.toUpperCase}_WIDTH $width" + ports += s".io_$field(bus.$signal[$offset +: $width])" + offsets(field) = offset; widths(field) = width + if (input) inputOffset += width else outputOffset += width + } + module + }, + args = Array("--target-dir", s"$output/$name", "--split-verilog") + ) + definitions += s"`define ${prefix}_INPUT_BITS $inputOffset" + definitions += s"`define ${prefix}_OUTPUT_BITS $outputOffset" + if (name == "Rob") { + definitions += s"`define ROB_ENTRIES ${storage.entries}" + definitions += s"`define ROB_DATA_BITS ${storage.dataBits}" + definitions += s"`define ROB_RETIRE_WIDTH ${storage.retireWidth}" + val state = (0 until storage.entries).map(i => + Seq("data", "valid", "issued", "completed", "dependencies").map(n => offsets(s"${n}_$i")).mkString( + "'{", + ",", + "}" + ) + ).mkString("'{", ",", "}") + val ready = (0 until storage.entries).map(i => offsets(s"retireReady_$i")).mkString("'{", ",", "}") + definitions += s"`define ROB_STATE_OFFSETS $state" + definitions += s"`define ROB_READY_OFFSETS $ready" + } else { + definitions += s"`define GROB_ENTRIES ${frontend.rob_entries}" + definitions += s"`define GROB_VBANKS ${memory.virtualBankCount}" + definitions += s"`define GROB_BANK_BITS ${frontend.bank_id_len}" + definitions += s"`define GROB_OWNER ${frontend.ownerId}" + definitions += s"`define GROB_ALIASES ${2 * (memory.virtualBankCount + (if (gp.enable) 2 else 0)) + frontend.rob_entries}" + definitions += s"`define GROB_RENAMING ${if (frontend.bank_alias_enable) 1 else 0}" + definitions += s"`define GROB_LAYOUT ${if (memory.bankLayoutEnable) 1 else 0}" + definitions += s"`define GROB_GP ${if (gp.enable) 1 else 0}" + definitions += s"`define GROB_DIFF ${if (diffTest) 1 else 0}" + for ( + (key, fieldPrefix) <- Seq( + "IN_OPERATION" -> "alloc_bits_operation_", + "OUT_COMMAND" -> "issue_bits_cmd_", + "IN_CONTEXT" -> "alloc_bits_context_", + "OUT_CONTEXT" -> "issue_bits_context_", + "IN_RENAMED" -> "bat_renamed_", + "OUT_RENAMED" -> "issue_bits_renamedBankAccess_" + ) + ) { + val fs = offsets.keys.filter(_.startsWith(fieldPrefix)).toSeq + definitions += s"`define GROB_${key}_OFFSET ${fs.map(offsets).min}" + definitions += s"`define GROB_${key}_BITS ${fs.map(widths).sum}" + } + val issueFields = offsets.keys.filter(_.startsWith("issue_bits_")).toSeq + definitions += s"`define GROB_ISSUE_OFFSET ${issueFields.map(offsets).min}" + definitions += s"`define GROB_ISSUE_BITS ${issueFields.map(widths).sum}" + val queryFields = Seq( + "rd_bank_0_valid", + "rd_bank_0_id", + "rd_bank_1_valid", + "rd_bank_1_id", + "rd_wr_valid", + "rd_wr_id", + "wr_bank_valid", + "wr_bank_id", + "wr_update" + ) + val queries = (0 until frontend.rob_entries).map(i => + queryFields.map(n => offsets(s"scoreboard_query_${i}_$n")).mkString("'{", ",", "}") + ).mkString("'{", ",", "}") + definitions += s"`define GROB_QUERY_OFFSETS $queries" + val baBase = offsets.keys.filter(_.startsWith("bat_renamed_")).map(offsets).min + val baOffsets = queryFields.map(n => offsets(s"bat_renamed_$n") - baBase).mkString("'{", ",", "}") + definitions += s"`define GROB_BA_OFFSETS $baOffsets" + val traceFields = + Seq("is_issue", "thread_id", "rob_id", "inst_id", "domain_id", "funct", "pc", "rs1_data", "rs2_data", "enable") + val traces = (0 until 4).map(i => + traceFields.map(n => offsets(s"traceInstructions_${i}_$n")).mkString("'{", ",", "}") + ).mkString("'{", ",", "}") + definitions += s"`define GROB_TRACE_OFFSETS $traces" + + for ((key, fieldPrefix) <- Seq("IN_ALLOC" -> "alloc_bits_", "MAPPING" -> "mapping_bits_")) { + val fs = offsets.keys.filter(_.startsWith(fieldPrefix)).toSeq + definitions += s"`define GROB_${key}_OFFSET ${fs.map(offsets).min}" + definitions += s"`define GROB_${key}_BITS ${fs.map(widths).sum}" + } + + val hazards = (0 until frontend.rob_entries).map(i => offsets(s"scoreboard_hazard_$i")).mkString("'{", ",", "}") + definitions += s"`define GROB_HAZARD_OFFSETS $hazards" + } + Files.writeString(Paths.get(output, name, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, name, "ports.svh"), ports.mkString(",\n") + "\n") + } +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/GlobalROB.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/GlobalROB.scala new file mode 100644 index 000000000..88ade204d --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/GlobalROB.scala @@ -0,0 +1,475 @@ +package framework.acc.frontend.scheduler.rob.global + +import framework.lib.arbiter.{Arbiter} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.frontend.decoder.{DecodedInstruction, DomainId, PostGDCmd} +import framework.acc.frontend.scheduler.scoreboard.BankAccessInfo +import framework.acc.memdomain.DISA.MSET_BITPAT +import framework.acc.memdomain.DISA.MVIN_KERNEL_BITPAT +import framework.acc.gpdomain.decoder.DISA.RUN_KERNEL_BITPAT +import framework.acc.memdomain.bank_router.BankMapping +import framework.acc.memdomain.{MemoryContext, PhysicalBankHash} +import framework.acc.debug.trace.InstructionEvent +import framework.acc.debug.DebugPort + +class GlobalRobEntry(val frontendParam: FrontendParam, val memParam: MemDomainParam, val permissionParam: PmpParams) + extends Bundle { + val commandDataBits = frontendParam.commandDataBits + val context = new MemoryContext(permissionParam) + val cmd = new PostGDCmd(frontendParam, memParam) + val renamedBankAccess = new BankAccessInfo(frontendParam.bank_id_len) + val rob_id = UInt(log2Up(frontendParam.rob_entries).W) + val instId = UInt(64.W) +} + +@instantiable +class GlobalROB( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val permissionParam: PmpParams, + val diffTest: Boolean) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val ownerId = frontendParam.ownerId + + require( + !memParam.sharedEnable || (frontendParam.bank_alias_enable && memParam.bankLayoutEnable), + "BAT/BLT ablation modes require private-bank workloads" + ) + + val robDepth = frontendParam.rob_entries + val idWidth = log2Up(robDepth) + + val resourceCount = + memParam.virtualBankCount + (if (gpParam.enable) 2 + else 0) + + val scoreBankNum = 2 * resourceCount + robDepth + require( + scoreBankNum <= (1 << frontendParam.bank_id_len), + "Bank resource IDs and ROB aliases exceed bank_id_len" + ) + val sharedHashCount = + if (memParam.sharedEnable) memParam.sharedBankNum else 0 + val physicalBankCount = memParam.bankNum + sharedHashCount + + require( + frontendParam.vbank_id_upper_bound < memParam.virtualBankCount, + s"vbank_id_upper_bound(${frontendParam.vbank_id_upper_bound}) must be < virtualBankCount(${memParam.virtualBankCount})" + ) + + @public + val io = IO(new Bundle { + val traceInstructions = Output(Vec(4, new InstructionEvent)) + val alloc = + Flipped(new DecoupledIO(new DecodedInstruction(frontendParam, memParam, permissionParam))) + val issue = new DecoupledIO(new GlobalRobEntry(frontendParam, memParam, permissionParam)) + val complete = Flipped(new DecoupledIO(UInt(idWidth.W))) + val mapping = Decoupled(new BankMapping(frontendParam)) + + val bat = new Bundle { + + val alloc = Output(new Bundle { + val valid = Bool() + val rename = Bool() + val reset_read0 = Bool() + val raw = new BankAccessInfo(frontendParam.bank_id_len) + }) + + val reclaim = Output(Valid(UInt(frontendParam.bank_id_len.W))) + val stale = Input(Vec(2, Valid(UInt(frontendParam.bank_id_len.W)))) + val ready = Input(Bool()) + val renamed = Input(new BankAccessInfo(frontendParam.bank_id_len)) + } + + val scoreboard = new Bundle { + val alloc = Valid(new BankAccessInfo(frontendParam.bank_id_len)) + val issue = Valid(new BankAccessInfo(frontendParam.bank_id_len)) + val complete = Valid(new BankAccessInfo(frontendParam.bank_id_len)) + + val query = Output(Vec( + robDepth, + new BankAccessInfo(frontendParam.bank_id_len) + )) + + val hazard = Input(Vec(robDepth, Bool())) + } + + val empty = Output(Bool()) + val full = Output(Bool()) + + val debug = Option.when(diffTest)(new DebugPort(frontendParam)) + + }) + + val mappingIssue = Wire(Decoupled(new BankMapping(frontendParam))) + val mappingRetire = Wire(Decoupled(new BankMapping(frontendParam))) + val mappings = Instantiate(new Arbiter(new BankMapping(frontendParam), 2)) + mappings.io.in(0) <> mappingRetire + mappings.io.in(1) <> mappingIssue + io.mapping <> mappings.io.out + + // --------------------------------------------------------------------------- + // Passive instruction observations consumed by the thread trace collector. + // --------------------------------------------------------------------------- + val itraceAlloc = io.traceInstructions(0) + val itraceIssue = io.traceInstructions(1) + val itraceComp = io.traceInstructions(2) + + for (t <- Seq(itraceAlloc, itraceIssue, itraceComp)) { + t.is_issue := 0.U + t.thread_id := ownerId.U(64.W) + t.rob_id := 0.U + t.inst_id := 0.U + t.domain_id := 0.U + t.funct := 0.U + t.pc := 0.U + t.rs1_data := 0.U + t.rs2_data := 0.U + t.enable := false.B + } + + // --------------------------------------------------------------------------- + // Storage + // --------------------------------------------------------------------------- + val queue: Instance[Rob] = Instantiate(new Rob(configs.RobParams( + robDepth, + (new GlobalRobEntry(frontendParam, memParam, permissionParam)).getWidth, + if (diffTest) 1 else robDepth + ))) + + val robEntries = + VecInit(queue.io.data.map(_.asTypeOf(new GlobalRobEntry(frontendParam, memParam, permissionParam)))) + val robValid = queue.io.valid + val robIssued = queue.io.issued + val robComplete = queue.io.completed + val allocatedBank = RegInit(VecInit(Seq.fill(robDepth)(false.B))) + val nextInstId = RegInit(1.U(64.W)) + + val headPtr = queue.io.head + val tailPtr = queue.io.tail + val issuedCount = queue.io.issuedCount + + val dependencies: Instance[DependencyResolver] = + Instantiate(new DependencyResolver(frontendParam, memParam, gpParam, permissionParam)) + dependencies.io.alloc := io.alloc.bits.operation + dependencies.io.entries := robEntries + dependencies.io.valid := robValid + dependencies.io.complete := robComplete + + // Dependency state is maintained incrementally instead of rebuilding all + // older-entry comparisons every cycle. A bit in slot k names the ROB slot + // that currently blocks k. Ordinary RAW dependencies disappear at complete; + // dependencies involving a mapping/config instruction disappear at commit. + // Keeping one union mask is sufficient because a config dependency is never + // allowed to clear earlier than commit, even when it also looks like a RAW. + val dependencyMask = queue.io.dependencies + + val isEmpty = queue.io.empty + val isFull = queue.io.full + val fullRequestBlocked = isFull && io.alloc.valid + val previousFullRequestBlocked = RegNext(fullRequestBlocked, false.B) + val itraceRobFull = io.traceInstructions(3) + itraceRobFull.is_issue := Mux(fullRequestBlocked, 3.U, 4.U) + itraceRobFull.thread_id := ownerId.U(64.W) + itraceRobFull.rob_id := tailPtr + itraceRobFull.inst_id := 0.U + itraceRobFull.domain_id := 0.U + itraceRobFull.funct := 0.U + itraceRobFull.pc := io.alloc.bits.operation.cmd.pc + itraceRobFull.rs1_data := 0.U + itraceRobFull.rs2_data := 0.U + itraceRobFull.enable := fullRequestBlocked =/= previousFullRequestBlocked + + // --------------------------------------------------------------------------- + // Allocate: enqueue decoded instruction into ROB + // rob_id == tailPtr at allocation time (no separate counter needed) + // --------------------------------------------------------------------------- + val commitMask = queue.io.retired.asBools + val staleValid = RegInit(VecInit(Seq.fill(robDepth)(0.U(2.W)))) + val staleBank = Reg(Vec(robDepth, Vec(2, UInt(frontendParam.bank_id_len.W)))) + val staleRaw = Reg(Vec(robDepth, UInt(frontendParam.bank_id_len.W))) + val newBank = Reg(Vec(robDepth, UInt(frontendParam.bank_id_len.W))) + + val retireSlots = VecInit((0 until robDepth).map { offset => + val sum = headPtr +& offset.U + Mux(sum >= robDepth.U, sum - robDepth.U, sum)(idWidth - 1, 0) + }) + + val pendingReclaim = VecInit(retireSlots.map(slot => queue.io.retireCandidates(slot) && staleValid(slot).orR)) + val reclaimSlot = retireSlots(PriorityEncoder(pendingReclaim)) + val reclaimIndex = PriorityEncoder(staleValid(reclaimSlot)) + val targetRaw = robEntries(reclaimSlot).cmd.bankAccess.wr_bank_id + mappingRetire.valid := pendingReclaim.asUInt.orR + mappingRetire.bits.bank := Mux(reclaimIndex === 0.U, staleRaw(reclaimSlot), targetRaw) + mappingRetire.bits.vbank := Mux(reclaimIndex === 0.U, newBank(reclaimSlot), targetRaw) + mappingRetire.bits.previous := staleBank(reclaimSlot)(reclaimIndex) + mappingRetire.bits.threadId := ownerId.U(64.W) + mappingRetire.bits.allocate := false.B + mappingRetire.bits.reuse := false.B + mappingRetire.bits.retire := true.B + mappingRetire.bits.write := false.B + when(mappingRetire.fire) { + staleValid(reclaimSlot) := staleValid(reclaimSlot) & ~UIntToOH(reclaimIndex, 2) + } + for (slot <- 0 until robDepth) { + queue.io.retireReady(slot) := !staleValid(slot).orR || + (slot.U === reclaimSlot && mappingRetire.ready && PopCount(staleValid(slot)) === 1.U) + when(commitMask(slot))(staleValid(slot) := 0.U) + } + io.bat.reclaim.valid := mappingRetire.fire + io.bat.reclaim.bits := mappingRetire.bits.previous + + val headNeedsDebug = + robEntries(headPtr).instId =/= 0.U && robEntries(headPtr).cmd.cmd.raw_inst(31, 25) =/= MSET_BITPAT && + robEntries(headPtr).cmd.bankAccess.wr_bank_valid && robEntries( + headPtr + ).cmd.bankAccess.wr_bank_id < memParam.virtualBankCount.U + + val hasCommit = queue.io.retired.orR + + val admit = io.bat.ready && !dependencies.io.movementLive + io.alloc.ready := queue.io.alloc.ready && admit + queue.io.alloc.valid := io.alloc.valid && admit + io.bat.alloc.valid := io.alloc.fire + io.bat.alloc.raw := io.alloc.bits.operation.bankAccess + io.bat.alloc.rename := frontendParam.bank_alias_enable.B && io.alloc.bits.operation.bankAccess.wr_bank_valid && + io.alloc.bits.operation.bankAccess.wr_bank_id < memParam.virtualBankCount.U && + io.alloc.bits.operation.cmd.raw_inst(31, 25) =/= MSET_BITPAT && !io.alloc.bits.operation.bankAccess.wr_update + io.bat.alloc.reset_read0 := io.alloc.bits.operation.cmd.raw_inst( + 31, + 25 + ) === MSET_BITPAT && io.alloc.bits.operation.cmd + .rs2Data(12) + + // Mark write alias as busy in scoreboard at alloc time (not issue time). + io.scoreboard.alloc.valid := io.alloc.fire && io.bat.alloc.rename + io.scoreboard.alloc.bits := io.bat.renamed + + val allocIsConfig = io.alloc.bits.operation.cmd.raw_inst(31, 25) === MSET_BITPAT + val allocUsesInstId = io.alloc.bits.operation.runtime + val allocation = Wire(new GlobalRobEntry(frontendParam, memParam, permissionParam)) + allocation.context := io.alloc.bits.context + allocation.cmd := io.alloc.bits.operation + allocation.renamedBankAccess := io.bat.renamed + allocation.rob_id := tailPtr + allocation.instId := Mux(allocUsesInstId, nextInstId, 0.U) + queue.io.alloc.bits.data := allocation.asUInt + queue.io.alloc.bits.dependencies := dependencies.io.dependencies + queue.io.alloc.bits.commitBarrier := allocIsConfig + + when(io.alloc.fire) { + itraceAlloc.is_issue := 2.U + itraceAlloc.rob_id := tailPtr + itraceAlloc.inst_id := allocation.instId + itraceAlloc.domain_id := io.alloc.bits.operation.domain_id + itraceAlloc.funct := io.alloc.bits.operation.cmd.raw_inst(31, 25) + itraceAlloc.pc := io.alloc.bits.operation.cmd.pc + itraceAlloc.rs1_data := io.alloc.bits.operation.cmd.rs1Data + itraceAlloc.rs2_data := io.alloc.bits.operation.cmd.rs2Data + itraceAlloc.enable := true.B + + allocatedBank(tailPtr) := io.bat.alloc.rename + staleValid(tailPtr) := VecInit(io.bat.stale.map(_.valid)).asUInt + staleBank(tailPtr) := VecInit(io.bat.stale.map(_.bits)) + staleRaw(tailPtr) := Mux( + io.bat.alloc.reset_read0, + io.alloc.bits.operation.bankAccess.rd_bank_0_id, + io.alloc.bits.operation.bankAccess.wr_bank_id + ) + newBank(tailPtr) := Mux( + io.bat.alloc.reset_read0, + io.alloc.bits.operation.bankAccess.rd_bank_0_id, + io.bat.renamed.wr_bank_id + ) + + when(allocUsesInstId) { + nextInstId := nextInstId + 1.U + } + } + + // --------------------------------------------------------------------------- + // Complete: mark entry as completed, release scoreboard resources + // --------------------------------------------------------------------------- + if (diffTest) { + val debug = io.debug.get + val incomingAccess = robEntries(io.complete.bits).cmd.bankAccess + val needsCheck = robEntries(io.complete.bits).instId =/= 0.U && + !(robEntries(io.complete.bits).cmd.domain_id === DomainId.MEM && + robEntries(io.complete.bits).cmd.cmd.raw_inst(31, 25) === MSET_BITPAT) && + incomingAccess.wr_bank_valid && incomingAccess.wr_bank_id < memParam.virtualBankCount.U + debug.check.valid := io.complete.valid && needsCheck + debug.check.bits.robId := io.complete.bits + debug.check.bits.bank := (if (memParam.bankLayoutEnable) + robEntries(io.complete.bits).renamedBankAccess.wr_bank_id + else incomingAccess.wr_bank_id) + io.complete.ready := !needsCheck || (debug.checked.valid && debug.checked.bits === io.complete.bits) + debug.checked.ready := io.complete.fire && needsCheck + } else { + io.complete.ready := true.B + } + + io.scoreboard.complete.valid := false.B + io.scoreboard.complete.bits := 0.U.asTypeOf(io.scoreboard.complete.bits) + + val issueFired = WireDefault(false.B) + queue.io.complete.valid := io.complete.fire + queue.io.complete.bits := io.complete.bits + + when(io.complete.fire) { + val cid = io.complete.bits + io.scoreboard.complete.valid := true.B + io.scoreboard.complete.bits := robEntries(cid).renamedBankAccess + + itraceComp.is_issue := 0.U + itraceComp.rob_id := cid + itraceComp.inst_id := robEntries(cid).instId + itraceComp.domain_id := robEntries(cid).cmd.domain_id + itraceComp.funct := robEntries(cid).cmd.cmd.raw_inst(31, 25) + itraceComp.pc := robEntries(cid).cmd.cmd.pc + itraceComp.rs1_data := robEntries(cid).cmd.cmd.rs1Data + itraceComp.rs2_data := robEntries(cid).cmd.cmd.rs2Data + itraceComp.enable := true.B + } + + // --------------------------------------------------------------------------- + // Issue: select the oldest unissued entry; a blocked entry stalls younger entries. + // --------------------------------------------------------------------------- + val scanValid = Wire(Vec(robDepth, Bool())) + val scanReady = Wire(Vec(robDepth, Bool())) + // Check each candidate against older ROB entries using mask intersections. + // The masks are precomputed once per slot, so this retains the original + // age-sensitive RAW/WAR/WAW semantics without replicating three equality + // comparators for every bank access pair. + for (i <- 0 until robDepth) { + val age = headPtr +& i.U + val ptr = Mux(age >= robDepth.U, age - robDepth.U, age)(idWidth - 1, 0) + scanValid(i) := robValid(ptr) && !robIssued(ptr) && !robComplete(ptr) + io.scoreboard.query(i) := robEntries(ptr).renamedBankAccess + scanReady(i) := scanValid(i) && !io.scoreboard.hazard(i) && !dependencyMask( + ptr + ).orR + } + + val firstReady = PriorityEncoder(scanValid.asUInt) + val hasReady = scanValid.asUInt.orR && scanReady(firstReady) + val issueAge = headPtr +& firstReady + val actualIssuePtr = Mux(issueAge >= robDepth.U, issueAge - robDepth.U, issueAge)(idWidth - 1, 0) + + val canIssue = hasReady + + val issueStageValid = RegInit(false.B) + val issueStageEntry = Reg(new GlobalRobEntry(frontendParam, memParam, permissionParam)) + val issueStageFire = issueStageValid && io.issue.ready + val issueStageReady = !issueStageValid || issueStageFire + + // Keep the wide ROB entry and bank-column mux one register boundary away + // from the age/scoreboard scan. The pointer stage can accept one candidate + // per cycle; the payload stage drains at one per cycle, so steady-state II + // remains one while the extra latency is hidden by in-flight commands. + val issuePtrValid = RegInit(false.B) + val issuePtr = Reg(UInt(idWidth.W)) + val issuePtrFire = issuePtrValid && issueStageReady + val issuePtrReady = !issuePtrValid || issuePtrFire + mappingIssue.bits.bank := robEntries(actualIssuePtr).cmd.bankAccess.wr_bank_id + mappingIssue.bits.vbank := robEntries(actualIssuePtr).renamedBankAccess.wr_bank_id + val issueAccess = robEntries(actualIssuePtr).renamedBankAccess + val previousAlias = staleBank(actualIssuePtr)(0) + + val readsPrevious = (issueAccess.rd_bank_0_valid && issueAccess.rd_bank_0_id === previousAlias) || + (issueAccess.rd_bank_1_valid && issueAccess.rd_bank_1_id === previousAlias) || + (issueAccess.rd_wr_valid && issueAccess.rd_wr_id === previousAlias) + + // At the ROB head all older users have retired; younger users see the new alias. + mappingIssue.bits.previous := previousAlias + mappingIssue.bits.reuse := allocatedBank(actualIssuePtr) && actualIssuePtr === headPtr && !readsPrevious + mappingIssue.bits.allocate := allocatedBank(actualIssuePtr) + mappingIssue.bits.retire := false.B + mappingIssue.bits.write := robEntries(actualIssuePtr).cmd.bankAccess.wr_bank_valid && + !robEntries(actualIssuePtr).cmd.bankAccess.wr_update && + robEntries( + actualIssuePtr + ).cmd.bankAccess.wr_bank_id < memParam.virtualBankCount.U && + robEntries(actualIssuePtr).cmd.cmd.raw_inst(31, 25) =/= MSET_BITPAT + mappingIssue.bits.threadId := ownerId.U(64.W) + val needsMapping = mappingIssue.bits.allocate || mappingIssue.bits.write + mappingIssue.valid := canIssue && issuePtrReady && needsMapping + val issueLoad = canIssue && issuePtrReady && (!needsMapping || mappingIssue.ready) + val issuePayload = Wire(new GlobalRobEntry(frontendParam, memParam, permissionParam)) + issuePayload := robEntries(issuePtr) + + io.issue.valid := issueStageValid + io.issue.bits := issueStageEntry + + when(issuePtrReady) { + issuePtrValid := issueLoad + when(issueLoad) { + issuePtr := actualIssuePtr + } + } + when(issueStageReady) { + issueStageValid := issuePtrValid + when(issuePtrFire) { + issueStageEntry := issuePayload + } + } + + io.scoreboard.issue.valid := false.B + io.scoreboard.issue.bits := 0.U.asTypeOf(io.scoreboard.issue.bits) + + when(issueLoad) { + issueFired := true.B + io.scoreboard.issue.valid := true.B + io.scoreboard.issue.bits := robEntries(actualIssuePtr).renamedBankAccess + + itraceIssue.is_issue := 1.U + itraceIssue.rob_id := robEntries(actualIssuePtr).rob_id + itraceIssue.inst_id := robEntries(actualIssuePtr).instId + itraceIssue.domain_id := robEntries(actualIssuePtr).cmd.domain_id + itraceIssue.funct := robEntries(actualIssuePtr).cmd.cmd.raw_inst(31, 25) + itraceIssue.pc := robEntries(actualIssuePtr).cmd.cmd.pc + itraceIssue.rs1_data := robEntries(actualIssuePtr).cmd.cmd.rs1Data + itraceIssue.rs2_data := robEntries(actualIssuePtr).cmd.cmd.rs2Data + itraceIssue.enable := true.B + } + + queue.io.issue.valid := issueFired + queue.io.issue.bits := actualIssuePtr + + io.debug.foreach { debug => + debug.commit.valid := hasCommit && headNeedsDebug + debug.commit.bits.robId := headPtr + debug.commit.bits.instId := robEntries(headPtr).instId + debug.commit.bits.threadId := ownerId.U(64.W) + debug.commit.bits.bank := robEntries(headPtr).cmd.bankAccess.wr_bank_id + debug.idle := io.empty && !io.alloc.valid && !debug.commit.valid + } + + val mappingCommits = VecInit( + (0 until robDepth).map(i => + commitMask(i) && robEntries(i).cmd.domain_id === DomainId.MEM && + robEntries(i).cmd.cmd.raw_inst(31, 25) === MSET_BITPAT + ) + ) + + assert( + PopCount(mappingCommits) <= 1.U, + "Mapping dependency permits one configuration commit" + ) + // --------------------------------------------------------------------------- + // Status outputs + // --------------------------------------------------------------------------- + io.empty := isEmpty + io.full := isFull + +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/Rob.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/Rob.scala new file mode 100644 index 000000000..d0768fcdb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/Rob.scala @@ -0,0 +1,123 @@ +package framework.acc.frontend.scheduler.rob.global + +import chisel3._ +import framework.acc.frontend.scheduler.rob.global.configs.RobParams +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class Rob(val p: RobParams) extends Module { + + @public + val io = IO(new Bundle { + + val alloc = Flipped(Decoupled(new Bundle { + val data = UInt(p.dataBits.W) + val dependencies = UInt(p.entries.W) + // Dependencies involving this entry clear only at retirement. + val commitBarrier = Bool() + })) + + val issue = Flipped(Valid(UInt(p.idBits.W))) + val complete = Flipped(Valid(UInt(p.idBits.W))) + val data = Output(Vec(p.entries, UInt(p.dataBits.W))) + val valid = Output(Vec(p.entries, Bool())) + val issued = Output(Vec(p.entries, Bool())) + val completed = Output(Vec(p.entries, Bool())) + val dependencies = Output(Vec(p.entries, UInt(p.entries.W))) + val retireReady = Input(Vec(p.entries, Bool())) + val retireCandidates = Output(UInt(p.entries.W)) + val retired = Output(UInt(p.entries.W)) + val head = Output(UInt(p.idBits.W)) + val tail = Output(UInt(p.idBits.W)) + val issuedCount = Output(UInt(log2Ceil(p.entries + 1).W)) + val empty = Output(Bool()) + val full = Output(Bool()) + }) + + val data = RegInit(VecInit(Seq.fill(p.entries)(0.U(p.dataBits.W)))) + val valid = RegInit(VecInit(Seq.fill(p.entries)(false.B))) + val issued = RegInit(VecInit(Seq.fill(p.entries)(false.B))) + val completed = RegInit(VecInit(Seq.fill(p.entries)(false.B))) + val dependencies = RegInit(VecInit(Seq.fill(p.entries)(0.U(p.entries.W)))) + val barriers = RegInit(VecInit(Seq.fill(p.entries)(false.B))) + val head = RegInit(0.U(p.idBits.W)) + val tail = RegInit(0.U(p.idBits.W)) + val issuedCount = RegInit(0.U(log2Ceil(p.entries + 1).W)) + def wrap(value: UInt): UInt = Mux(value >= p.entries.U, value - p.entries.U, value)(p.idBits - 1, 0) + + val candidate = Wire(Vec(p.retireWidth, Bool())) + val eligible = Wire(Vec(p.retireWidth + 1, Bool())) + val retire = Wire(Vec(p.retireWidth, Bool())) + val keep = Wire(Vec(p.retireWidth + 1, Bool())) + eligible(0) := true.B + keep(0) := true.B + for (offset <- 0 until p.retireWidth) { + val index = wrap(head +& offset.U) + candidate(offset) := eligible(offset) && valid(index) && completed(index) + eligible(offset + 1) := candidate(offset) + retire(offset) := keep(offset) && valid(index) && completed(index) && io.retireReady(index) + keep(offset + 1) := retire(offset) + } + io.retireCandidates := VecInit((0 until p.entries).map { slot => + (0 until p.retireWidth).map(offset => candidate(offset) && wrap(head +& offset.U) === slot.U).reduce(_ || _) + }).asUInt + + val retired = VecInit((0 until p.entries).map { slot => + (0 until p.retireWidth).map(offset => retire(offset) && wrap(head +& offset.U) === slot.U).reduce(_ || _) + }).asUInt + + val completeMask = Mux(io.complete.valid, UIntToOH(io.complete.bits, p.entries), 0.U(p.entries.W)) + io.empty := head === tail && !valid(head) + io.full := head === tail && valid(head) + // Preserve the integrated ROB's commit-cycle admission rule. + io.alloc.ready := !io.full && !retired.orR + + when(io.alloc.fire) { + data(tail) := io.alloc.bits.data + valid(tail) := true.B + issued(tail) := false.B + completed(tail) := false.B + barriers(tail) := io.alloc.bits.commitBarrier + tail := wrap(tail +& 1.U) + } + when(io.issue.valid) { + assert( + io.issue.bits < p.entries.U && valid(io.issue.bits) && !issued(io.issue.bits) && !completed(io.issue.bits), + "ROB issue must name a live unissued entry" + ) + issued(io.issue.bits) := true.B + } + when(io.complete.valid) { + assert( + io.complete.bits < p.entries.U && valid(io.complete.bits) && !completed(io.complete.bits), + "ROB completion must name a live incomplete entry" + ) + completed(io.complete.bits) := true.B + } + val completingIssued = io.complete.valid && + (issued(io.complete.bits) || (io.issue.valid && io.issue.bits === io.complete.bits)) + issuedCount := issuedCount + io.issue.valid.asUInt - completingIssued.asUInt + for (slot <- 0 until p.entries) { + when(retired(slot)) { + valid(slot) := false.B + issued(slot) := false.B + completed(slot) := false.B + } + val allocating = io.alloc.fire && tail === slot.U + val barrier = Mux(allocating, io.alloc.bits.commitBarrier, barriers(slot)) + val sticky = barriers.asUInt | Fill(p.entries, barrier) + val clear = retired | (completeMask & ~sticky) + dependencies(slot) := Mux(allocating, io.alloc.bits.dependencies, dependencies(slot)) & ~clear + } + head := wrap(head +& PopCount(retire)) + io.data := data + io.valid := valid + io.issued := issued + io.completed := completed + io.dependencies := dependencies + io.retired := retired + io.head := head + io.tail := tail + io.issuedCount := issuedCount +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/configs/RobParams.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/configs/RobParams.scala new file mode 100644 index 000000000..efefb7e22 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/configs/RobParams.scala @@ -0,0 +1,11 @@ +package framework.acc.frontend.scheduler.rob.global.configs + +import chisel3.util.log2Ceil +import upickle.default.{macroRW, ReadWriter} + +case class RobParams(entries: Int, dataBits: Int, retireWidth: Int) { + require(entries > 1 && dataBits > 0 && retireWidth > 0 && retireWidth <= entries) + val idBits = log2Ceil(entries) +} + +object RobParams { implicit val rw: ReadWriter[RobParams] = macroRW } diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/build.sc b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/build.sc new file mode 100644 index 000000000..0c9885b37 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.^.^.^.settings +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.frontend.scheduler.scoreboard.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/BankAccessInfo.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/BankScoreboard.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/configs/uvm.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/configs/uvm.toml new file mode 100644 index 000000000..9e69ca533 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/configs/uvm.toml @@ -0,0 +1,6 @@ +[parameters] +bankIdBits = 4 +banks = 12 +entries = 4 + +[defines] diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..d549e9643 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual bank_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bank_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Bank scheduling interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.inputs = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/item.svh new file mode 100644 index 000000000..0649002f2 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`SB_INPUT_BITS-1:0] inputs; + logic [`SB_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + logic [`SB_OUTPUT_BITS-1:0] payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..dc93b2fb6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual bank_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bank_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Bank scheduling monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/env/env.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/env/env.svh new file mode 100644 index 000000000..ed95874fb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver drv; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/env/model.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/env/model.svh new file mode 100644 index 000000000..55253e2b4 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/env/model.svh @@ -0,0 +1,57 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + bit busy[`SB_BANKS]; + int query_offsets[`SB_ENTRIES][6] = `SB_QUERY_OFFSETS; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + covergroup behavior with function sample ( + int bank, bit allocated, bit issued, bit hazard, int read_mask + ); + option.per_instance = 1; + allocated_bank: coverpoint bank iff (allocated) {bins aliases[] = {[0 : `SB_BANKS - 1]};} + issued_bank: coverpoint bank iff (issued) {bins aliases[] = {[0 : `SB_BANKS - 1]};} + read_ports: coverpoint read_mask {bins combinations[] = {[0 : 7]};} + blocked: coverpoint hazard; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + behavior = new(); + endfunction + function void write(cycle_item t); + result_item expected, actual; + if (t.reset) begin + foreach (busy[b]) busy[b] = 0; + return; + end + expected = result_item::type_id::create("expected"); + actual = result_item::type_id::create("actual"); + expected.payload = '0; + actual.payload = t.outputs; + for (int lane = 0; lane < `SB_ENTRIES; lane++) begin + bit hazard = 0; + int read_mask = 0; + for (int port = 0; port < 3; port++) begin + bit enabled = t.inputs[query_offsets[lane][port*2]]; + int bank = t.inputs[query_offsets[lane][port*2+1]+:`SB_BANKIDBITS]; + if (enabled) begin + hazard |= busy[bank]; + read_mask |= 1 << port; + end + end + expected.payload[lane] = hazard; + behavior.sample(`BI(t.inputs, ALLOC_BITS_WR_BANK_ID), `BI(t.inputs, ALLOC_VALID) && + `BI(t.inputs, ALLOC_BITS_WR_BANK_VALID), 0, hazard, read_mask); + behavior.sample(`BI(t.inputs, ISSUE_BITS_WR_BANK_ID), 0, `BI(t.inputs, ISSUE_VALID) && + `BI(t.inputs, ISSUE_BITS_WR_BANK_VALID), hazard, read_mask); + end + expected_ap.write(expected); + actual_ap.write(actual); + if (`BI(t.inputs, ALLOC_VALID) && `BI(t.inputs, ALLOC_BITS_WR_BANK_VALID)) + busy[`BI(t.inputs, ALLOC_BITS_WR_BANK_ID)] = 1; + if (`BI(t.inputs, ISSUE_VALID) && `BI(t.inputs, ISSUE_BITS_WR_BANK_VALID)) + busy[`BI(t.inputs, ISSUE_BITS_WR_BANK_ID)] = 1; + if (`BI(t.inputs, COMPLETE_VALID) && `BI(t.inputs, COMPLETE_BITS_WR_BANK_VALID)) + busy[`BI(t.inputs, COMPLETE_BITS_WR_BANK_ID)] = 0; + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/filelist.f b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/filelist.f new file mode 100644 index 000000000..4553acc1c --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/BankScoreboard/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/interface.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/interface.sv new file mode 100644 index 000000000..90a5a959a --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/interface.sv @@ -0,0 +1,8 @@ +`include "parameters.svh" +interface bank_if ( + input logic clock +); + logic reset = 1; + logic [`SB_INPUT_BITS-1:0] inputs; + logic [`SB_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/package.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/package.sv new file mode 100644 index 000000000..dd191ab1a --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/package.sv @@ -0,0 +1,15 @@ +`include "parameters.svh" +`define BI(P, N) P[`SB_IN_``N``_OFFSET +: `SB_IN_``N``_WIDTH] +`define BO(P, N) P[`SB_OUT_``N``_OFFSET +: `SB_OUT_``N``_WIDTH] +package scoreboard_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..9126bcf84 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/seq/sequence.svh @@ -0,0 +1,95 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + logic [`SB_INPUT_BITS-1:0] pins = '0; + bit reset = 1; + int query_offsets[`SB_ENTRIES][6] = `SB_QUERY_OFFSETS; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task tick(int cycles = 1); + cycle_item req; + repeat (cycles) begin + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.inputs = pins; + req.reset = reset; + finish_item(req); + end + endtask + task queries(int mask, int bank, int alternate); + for (int lane = 0; lane < `SB_ENTRIES; lane++) + for (int port = 0; port < 3; port++) begin + pins[query_offsets[lane][port*2]] = (mask >> port) & 1; + pins[query_offsets[lane][port*2+1]+:`SB_BANKIDBITS] = (lane+port)%2 == 0 ? bank : alternate; + end + endtask + task body(); + tick(2); + reset = 0; + for (int bank = 0; bank < `SB_BANKS; bank++) begin + `BI(pins, ALLOC_VALID) = 1; + `BI(pins, ALLOC_BITS_WR_BANK_VALID) = 1; + `BI(pins, ALLOC_BITS_WR_BANK_ID) = bank; + tick(); + `BI(pins, ALLOC_VALID) = 0; + for (int mask = 0; mask < 8; mask++) + for (int alternate = 0; alternate < `SB_BANKS; alternate++) begin + queries(mask, bank, alternate); + tick(); + end + `BI(pins, COMPLETE_VALID) = 1; + `BI(pins, COMPLETE_BITS_WR_BANK_VALID) = 1; + `BI(pins, COMPLETE_BITS_WR_BANK_ID) = bank; + tick(); + `BI(pins, COMPLETE_VALID) = 0; + tick(); + `BI(pins, ISSUE_VALID) = 1; + `BI(pins, ISSUE_BITS_WR_BANK_VALID) = 1; + `BI(pins, ISSUE_BITS_WR_BANK_ID) = bank; + tick(); + `BI(pins, ISSUE_VALID) = 0; + queries(7, bank, (bank + 1) % `SB_BANKS); + tick(); + // Retirement wins when the same alias is presented by issue and complete together. + `BI(pins, ISSUE_VALID) = 1; + `BI(pins, COMPLETE_VALID) = 1; + tick(); + `BI(pins, ISSUE_VALID) = 0; + `BI(pins, COMPLETE_VALID) = 0; + tick(); + end + // Read/read sharing never creates a write dependency. + `BI(pins, ISSUE_BITS_WR_BANK_VALID) = 0; + `BI(pins, COMPLETE_BITS_WR_BANK_VALID) = 0; + for (int bank = 0; bank < `SB_BANKS; bank++) begin + `BI(pins, ISSUE_VALID) = 1; + `BI(pins, ISSUE_BITS_RD_BANK_0_VALID) = 1; + `BI(pins, ISSUE_BITS_RD_BANK_0_ID) = bank; + `BI(pins, ISSUE_BITS_RD_BANK_1_VALID) = 1; + `BI(pins, ISSUE_BITS_RD_BANK_1_ID) = bank; + `BI(pins, ISSUE_BITS_RD_WR_VALID) = 1; + `BI(pins, ISSUE_BITS_RD_WR_ID) = bank; + queries(7, bank, bank); + tick(`SB_ENTRIES); + `BI(pins, ISSUE_VALID) = 0; + `BI(pins, COMPLETE_VALID) = 1; + `BI(pins, COMPLETE_BITS_RD_BANK_0_VALID) = 1; + `BI(pins, COMPLETE_BITS_RD_BANK_0_ID) = bank; + `BI(pins, COMPLETE_BITS_RD_BANK_1_VALID) = 1; + `BI(pins, COMPLETE_BITS_RD_BANK_1_ID) = bank; + `BI(pins, COMPLETE_BITS_RD_WR_VALID) = 1; + `BI(pins, COMPLETE_BITS_RD_WR_ID) = bank; + tick(`SB_ENTRIES); + `BI(pins, COMPLETE_VALID) = 0; + end + `BI(pins, ALLOC_VALID) = 1; + `BI(pins, ALLOC_BITS_WR_BANK_VALID) = 1; + tick(); + reset = 1; + tick(2); + pins = '0; + reset = 0; + queries(7, 0, `SB_BANKS - 1); + tick(2); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/suite.toml new file mode 100644 index 000000000..e022b231e --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "scoreboard" +top = "scoreboard_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["scoreboard_pkg::model::behavior"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/tb.sv b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/tb.sv new file mode 100644 index 000000000..a766519fb --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/tb.sv @@ -0,0 +1,16 @@ +module scoreboard_tb; + import uvm_pkg::*; + import scoreboard_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + bank_if bus (clock); + BankScoreboard dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual bank_if)::set(null, "uvm_test_top*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/resources/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/frontend/scoreboard/BankAccessInfo.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/BankAccessInfo.scala similarity index 78% rename from arch/src/main/scala/framework/frontend/scoreboard/BankAccessInfo.scala rename to arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/BankAccessInfo.scala index 8b3b2f2c1..f089ef508 100644 --- a/arch/src/main/scala/framework/frontend/scoreboard/BankAccessInfo.scala +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/BankAccessInfo.scala @@ -1,4 +1,4 @@ -package framework.frontend.scoreboard +package framework.acc.frontend.scheduler.scoreboard import chisel3._ @@ -7,12 +7,16 @@ import chisel3._ // It's instruction-agnostic: the scoreboard only sees read/write bank_id. //====----------------------------------------------------------====// class BankAccessInfo(val bankIdLen: Int) extends Bundle { + val rd_bank_0_valid = Bool() val rd_bank_0_id = UInt(bankIdLen.W) val rd_bank_1_valid = Bool() val rd_bank_1_id = UInt(bankIdLen.W) + val rd_wr_valid = Bool() + val rd_wr_id = UInt(bankIdLen.W) val wr_bank_valid = Bool() val wr_bank_id = UInt(bankIdLen.W) + val wr_update = Bool() } object BankAccessInfo { @@ -23,8 +27,11 @@ object BankAccessInfo { w.rd_bank_0_id := 0.U w.rd_bank_1_valid := false.B w.rd_bank_1_id := 0.U + w.rd_wr_valid := false.B + w.rd_wr_id := 0.U w.wr_bank_valid := false.B w.wr_bank_id := 0.U + w.wr_update := false.B w } diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/BankScoreboard.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/BankScoreboard.scala new file mode 100644 index 000000000..567350e8b --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/BankScoreboard.scala @@ -0,0 +1,113 @@ +package framework.acc.frontend.scheduler.scoreboard + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.frontend.scheduler.scoreboard.configs.Params + +//====----------------------------------------------------------====// +// Bank Scoreboard: tracks in-flight read/write operations per bank. +// +// Hazard rules: +// - Read bank X → requires bankWrBusy(X) == false (RAW) +// - Write bank X → requires bankRdCount(X) == 0 (WAR) +// AND bankWrBusy(X) == false (WAW) +// +// bankRdCount: multi-bit counter (multiple concurrent readers allowed, RR is OK) +// bankWrBusy: 1-bit flag (WAW rule guarantees at most 1 writer in-flight) +// +// Lifecycle: +// - alloc: sets bankWrBusy for the renamed write alias. This must happen +// at alloc time (not issue) because the BAT updates v2a at alloc, +// and subsequent reads of the same vbank will be renamed to the +// new alias immediately. If we only set bankWrBusy at issue, the +// window between alloc and issue lets dependent reads slip through +// the hazard check and issue out-of-order. +// - issue: increments bankRdCount for the renamed read aliases. +// - complete: decrements bankRdCount and clears bankWrBusy. +// +// Issue and complete may fire in the same cycle. Updates are computed +// combinationally so both increments and decrements take effect. +//====----------------------------------------------------------====// + +@instantiable +class BankScoreboard(val p: Params) extends Module { + + val bankIdLen = p.bankIdBits + val bankNum = p.banks + val robEntries = p.entries + + val cntWidth = log2Ceil(3 * robEntries + 1) + + @public + val io = IO(new Bundle { + val alloc = Flipped(Valid(new BankAccessInfo(p.bankIdBits))) + val issue = Flipped(Valid(new BankAccessInfo(p.bankIdBits))) + val complete = Flipped(Valid(new BankAccessInfo(p.bankIdBits))) + + val queryVec = Input(Vec( + robEntries, + new BankAccessInfo(p.bankIdBits) + )) + + val hazardVec = Output(Vec(robEntries, Bool())) + }) + + val bankRdCount = RegInit(VecInit(Seq.fill(bankNum)(0.U(cntWidth.W)))) + val bankWrBusy = RegInit(VecInit(Seq.fill(bankNum)(false.B))) + def bankIndex(id: UInt): UInt = id(log2Ceil(bankNum) - 1, 0) + + // --- Hazard detection (reads current register state) --- + def hazardOf(q: BankAccessInfo): Bool = { + val rd0_hazard = q.rd_bank_0_valid && bankWrBusy(bankIndex(q.rd_bank_0_id)) + val rd1_hazard = q.rd_bank_1_valid && bankWrBusy(bankIndex(q.rd_bank_1_id)) + // Write hazard is NOT checked here. BAT eliminates WAW/WAR by giving each + // write a unique alias that no other instruction uses. Checking wr_hazard + // would falsely block the writer itself, since io.alloc marks the writer's + // own alias as busy in the same cycle the entry enters the ROB. + val rdWrHazard = q.rd_wr_valid && bankWrBusy(bankIndex(q.rd_wr_id)) + rd0_hazard || rd1_hazard || rdWrHazard + } + + for (i <- 0 until robEntries) { + io.hazardVec(i) := hazardOf(io.queryVec(i)) + } + + // --- Compute per-bank deltas to handle simultaneous io.issue + io.complete --- + for (bank <- 0 until bankNum) { + val bankU = bank.U(bankIdLen.W) + + // Read counter: +1 per io.issue read, -1 per io.complete read + val issRd0 = + io.issue.valid && io.issue.bits.rd_bank_0_valid && (io.issue.bits.rd_bank_0_id === bankU) + val issRd1 = + io.issue.valid && io.issue.bits.rd_bank_1_valid && (io.issue.bits.rd_bank_1_id === bankU) + val cmpRd0 = + io.complete.valid && io.complete.bits.rd_bank_0_valid && (io.complete.bits.rd_bank_0_id === bankU) + val cmpRd1 = + io.complete.valid && io.complete.bits.rd_bank_1_valid && (io.complete.bits.rd_bank_1_id === bankU) + val issRdWr = + io.issue.valid && io.issue.bits.rd_wr_valid && io.issue.bits.rd_wr_id === bankU + val cmpRdWr = + io.complete.valid && io.complete.bits.rd_wr_valid && io.complete.bits.rd_wr_id === bankU + + val rdInc = issRd0.asUInt +& issRd1.asUInt +& issRdWr.asUInt + val rdDec = cmpRd0.asUInt +& cmpRd1.asUInt +& cmpRdWr.asUInt + bankRdCount(bank) := bankRdCount(bank) + rdInc - rdDec + + // Write flag: io.alloc sets (alias is reserved as soon as it is renamed), + // io.complete clears. If both happen to same bank, io.complete takes priority. + val allocWr = + io.alloc.valid && io.alloc.bits.wr_bank_valid && (io.alloc.bits.wr_bank_id === bankU) + val issueWr = + io.issue.valid && io.issue.bits.wr_bank_valid && (io.issue.bits.wr_bank_id === bankU) + val cmpWr = + io.complete.valid && io.complete.bits.wr_bank_valid && (io.complete.bits.wr_bank_id === bankU) + + when((allocWr || issueWr) && !cmpWr) { + bankWrBusy(bank) := true.B + }.elsewhen(cmpWr) { + bankWrBusy(bank) := false.B + } + } +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/Emit.scala new file mode 100644 index 000000000..2dfd516b2 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/Emit.scala @@ -0,0 +1,61 @@ +package framework.acc.frontend.scheduler.scoreboard + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.acc.frontend.scheduler.scoreboard.configs.Params + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val p = upickle.default.read[Params](Files.readString(Paths.get(config))) + Files.createDirectories(Paths.get(output)) + + def fields(data: Data, path: String): Seq[(String, Int)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(path -> leaf.getWidth) + } + + val inputRoots = Set("alloc", "issue", "complete", "queryVec") + val ports = scala.collection.mutable.ArrayBuffer[String]() + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0 + var outputOffset = 0 + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new BankScoreboard(p) + for { + (root, bundle) <- dut.io.elements.toSeq + (name, width) <- fields(bundle, root) + } { + val input = inputRoots.contains(root) + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define SB_${direction}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define SB_${direction}_${name.toUpperCase}_WIDTH $width" + ports += s".io_$name(bus.$signal[$offset +: $width])" + offsets(name) = offset + if (input) inputOffset += width else outputOffset += width + } + dut + }, + args = Array("--target-dir", s"$output/BankScoreboard", "--split-verilog") + ) + definitions += s"`define SB_INPUT_BITS $inputOffset" + definitions += s"`define SB_OUTPUT_BITS $outputOffset" + definitions += s"`define SB_BANKIDBITS ${p.bankIdBits}" + definitions += s"`define SB_BANKS ${p.banks}" + definitions += s"`define SB_ENTRIES ${p.entries}" + + val queryOffsets = (0 until p.entries).map { index => + Seq("rd_bank_0_valid", "rd_bank_0_id", "rd_bank_1_valid", "rd_bank_1_id", "rd_wr_valid", "rd_wr_id") + .map(name => offsets(s"queryVec_${index}_$name")).mkString("'{", ",", "}") + }.mkString("'{", ",", "}") + + definitions += s"`define SB_QUERY_OFFSETS $queryOffsets" + Files.writeString(Paths.get(output, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..e1115b533 --- /dev/null +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard/src/main/scala/configs/Params.scala @@ -0,0 +1,10 @@ +package framework.acc.frontend.scheduler.scoreboard.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Params(bankIdBits: Int, banks: Int, entries: Int) { + require(bankIdBits > 0 && banks > 1 && banks <= (BigInt(1) << bankIdBits)) + require(entries > 1) +} + +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/rvv/.gitignore b/arch/src/main/scala/framework/acc/gpdomain/.gitignore similarity index 100% rename from arch/src/main/scala/framework/rvv/.gitignore rename to arch/src/main/scala/framework/acc/gpdomain/.gitignore diff --git a/arch/src/main/scala/framework/mem-core/LICENSE b/arch/src/main/scala/framework/acc/gpdomain/LICENSE similarity index 100% rename from arch/src/main/scala/framework/mem-core/LICENSE rename to arch/src/main/scala/framework/acc/gpdomain/LICENSE diff --git a/arch/src/main/scala/framework/acc/gpdomain/README.md b/arch/src/main/scala/framework/acc/gpdomain/README.md new file mode 100644 index 000000000..17b8d47b0 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/README.md @@ -0,0 +1 @@ +# gpdomain diff --git a/arch/src/main/scala/framework/acc/gpdomain/build.sc b/arch/src/main/scala/framework/acc/gpdomain/build.sc new file mode 100644 index 000000000..88250a654 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/build.sc @@ -0,0 +1,33 @@ +import mill._ +import $file.src.main.scala.execution.{build => execution_build} +import $file.src.main.scala.loader.{build => loader_build} +import $file.src.main.scala.memory.{build => memory_build} +import $file.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.gpdomain.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/GpDomain.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/GpDomainParam.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/decoder/DISA.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/decoder/Decoder.scala")) + ) ++ + execution_build.sources() ++ + loader_build.sources() ++ + memory_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + execution_build.emitSources() ++ + loader_build.emitSources() ++ + memory_build.emitSources() +) diff --git a/examples/cores/pebble/configs/rvv/default.toml b/arch/src/main/scala/framework/acc/gpdomain/configs/default.toml similarity index 100% rename from examples/cores/pebble/configs/rvv/default.toml rename to arch/src/main/scala/framework/acc/gpdomain/configs/default.toml diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/csrc/Cargo.lock b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/Cargo.lock new file mode 100644 index 000000000..f7cc692c4 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/Cargo.lock @@ -0,0 +1,128 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + +[[package]] +name = "rvv-reference-model" +version = "0.1.0" +dependencies = [ + "rustc_apfloat", + "serde_json", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "smallvec" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/csrc/Cargo.toml b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/Cargo.toml new file mode 100644 index 000000000..dd0f78908 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "rvv-reference-model" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib", "staticlib"] + +[dependencies] +rustc_apfloat = "0.2.3" + +[build-dependencies] +serde_json = "1" + +[workspace] diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/csrc/build.rs b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/build.rs new file mode 100644 index 000000000..1d7d1e4ce --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/build.rs @@ -0,0 +1,64 @@ +use std::{env, fs, path::PathBuf, process::Command}; +fn main() { + let root = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()) + .ancestors() + .nth(12) + .unwrap() + .to_path_buf(); + let ip = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()).parent().unwrap().parent().unwrap().to_path_buf(); + let parameters = ip.join("src/main/resources/gpdomain.json"); + println!("cargo:rerun-if-changed={}", parameters.display()); + let config: serde_json::Value = serde_json::from_str(&fs::read_to_string(parameters).unwrap()).unwrap(); + let value = |key: &str| config[key].as_u64().unwrap() as usize; + let constants = format!("const VLEN: usize = {};\nconst ELEN: usize = {};\nconst IBUF_BYTES: usize = {};\n", + value("vLen"), value("eLen"), value("iBufWords") * 4); + fs::write(PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("params.rs"), constants).unwrap(); + let rtl = ip.join("build/KernelEngine"); + fs::create_dir_all(&rtl).unwrap(); + fs::write(rtl.join("rvv_config.svh"), format!("`ifndef RVV_MEMORY_PORTS\n`define RVV_MEMORY_PORTS {}\n`endif\n", value("memoryPorts"))).unwrap(); + let mut connections = String::new(); + for port in 0..value("memoryPorts") { + for (direction, fields) in [ + ("Read", vec![("bank_id", "read_bank"), ("group_id", "read_group"), ("io_req_valid", "read_valid"), ("io_req_ready", "read_ready"), ("io_req_bits_addr", "read_row"), ("io_resp_valid", "read_response_valid"), ("io_resp_ready", "read_response_ready"), ("io_resp_bits_data", "read_data")]), + ("Write", vec![("bank_id", "write_bank"), ("group_id", "write_group"), ("io_req_valid", "write_valid"), ("io_req_ready", "write_ready"), ("io_req_bits_addr", "write_row"), ("io_resp_valid", "write_response_valid"), ("io_resp_ready", "write_response_ready"), ("io_req_bits_data", "write_data"), ("io_resp_bits_ok", "write_ok")]), + ] { + for (field, signal) in fields { + connections += &format!(".io_bank{direction}_{port}_{field}(vif.{signal}[{port}]),\n"); + } + } + for bit in 0..16 { + connections += &format!(".io_bankWrite_{port}_io_req_bits_mask_{bit}(vif.write_mask[{port}][{bit}]),\n"); + } + } + connections.truncate(connections.trim_end().len() - 1); + fs::write(rtl.join("rvv_ports.svh"), connections).unwrap(); + let source = root.join("stack/kernels/rvv"); + for name in [ + "CMakeLists.txt", + "kernel.ld", + "kernels.h", + "silu.cpp", + "snake.cpp", + "quant.cpp", + "math/exp.cpp", + "math/sin.cpp", + "math/cos.cpp", + "math/erf.cpp", + "rope.cpp", + "trig.cpp", + "math/math.h", + "math/constants.h", + ] { + println!("cargo:rerun-if-changed={}", source.join(name).display()); + } + let output = PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("kernels"); + assert!(Command::new("cmake") + .args(["-G", "Ninja", "-S"]) + .arg(&source) + .arg("-B") + .arg(&output) + .status() + .unwrap() + .success()); + assert!(Command::new("cmake").arg("--build").arg(&output).status().unwrap().success()); +} diff --git a/arch/src/main/scala/framework/rvv/src/csrc/src/ffi.rs b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/ffi.rs similarity index 99% rename from arch/src/main/scala/framework/rvv/src/csrc/src/ffi.rs rename to arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/ffi.rs index 946c82f5f..37f556787 100644 --- a/arch/src/main/scala/framework/rvv/src/csrc/src/ffi.rs +++ b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/ffi.rs @@ -1,5 +1,5 @@ use crate::model::{Model, BANKS, BANK_BYTES}; -use rvv::Memory; +use crate::rvv::Memory; use std::{ path::Path, sync::{Mutex, OnceLock}, diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/lib.rs b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/lib.rs new file mode 100644 index 000000000..aaba4be55 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/lib.rs @@ -0,0 +1,4 @@ +#[path = "../../../../../../../../../../bebop/src/nodes/bemu/src/thread/kernels/mod.rs"] +mod rvv; +mod ffi; +mod model; diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/model.rs b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/model.rs new file mode 100644 index 000000000..699f809a5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/csrc/src/model.rs @@ -0,0 +1,903 @@ +include!(concat!(env!("OUT_DIR"), "/params.rs")); + +use crate::rvv::{Engine, Fault, Memory, MemoryError}; +use std::path::Path; + +pub const BANK_BYTES: usize = 4096; +pub const BANKS: usize = 6; +#[derive(Clone)] +pub struct Banks(pub Vec); +impl Banks { + fn range(&self, address: u64, bytes: usize) -> Result, MemoryError> { + let bank = (address >> 32) as usize; + let offset = (address & 0xffff_ffff) as usize; + if bank >= BANKS || offset + bytes > BANK_BYTES { + return Err(MemoryError); + } + Ok(bank * BANK_BYTES + offset..bank * BANK_BYTES + offset + bytes) + } + fn put(&mut self, bank: usize, offset: usize, data: &[u8]) { + self.0[bank * BANK_BYTES + offset..bank * BANK_BYTES + offset + data.len()] + .copy_from_slice(data); + } +} +impl Memory for Banks { + fn ball_command(&mut self, _: u32, _: u64, _: u64) -> Result { + Err(MemoryError) + } + fn read(&mut self, address: u64, bytes: usize) -> Result { + let mut value = [0u8; 8]; + value[..bytes].copy_from_slice(&self.0[self.range(address, bytes)?]); + Ok(u64::from_le_bytes(value)) + } + fn write(&mut self, address: u64, bytes: usize, value: u64) -> Result<(), MemoryError> { + let range = self.range(address, bytes)?; + self.0[range].copy_from_slice(&value.to_le_bytes()[..bytes]); + Ok(()) + } +} +pub struct Case { + pub name: String, + pub program: Vec, + pub constants: Vec, + pub entry: u32, + pub args: [u64; 8], + pub initial: Banks, + pub expected_cause: u32, + pub fixed_expected: Option, +} +pub struct Model { + pub cases: Vec, + pub current: usize, + pub actual: Banks, + pub expected: Banks, + pub engine: Engine, + pub fault: Option, +} +fn i(op: u32, rd: u32, f: u32, rs: u32, imm: i32) -> u32 { + ((imm as u32 & 4095) << 20) | rs << 15 | f << 12 | rd << 7 | op +} +fn s(op: u32, f: u32, rs1: u32, rs2: u32, imm: u32) -> u32 { + (imm >> 5) << 25 | rs2 << 20 | rs1 << 15 | f << 12 | (imm & 31) << 7 | op +} +fn r(op: u32, rd: u32, f: u32, rs1: u32, rs2: u32, f7: u32) -> u32 { + f7 << 25 | rs2 << 20 | rs1 << 15 | f << 12 | rd << 7 | op +} +fn initial() -> Banks { + let mut b = Banks(vec![0; BANKS * BANK_BYTES]); + for offset in 0..BANK_BYTES { + b.0[BANK_BYTES + offset] = (offset as u8).wrapping_mul(73).wrapping_add(19); + } + for n in 0..256 { + let x = (n as f32 - 123.0) / 31.0; + b.put(2, n * 4, &x.to_le_bytes()); + b.put(3, n * 4, &(0.25 + n as f32 / 128.0).to_le_bytes()); + } + b +} +fn micro(name: &str, words: &[u32], args: [u64; 8]) -> Case { + Case { + name: name.into(), + program: words.iter().flat_map(|w| w.to_le_bytes()).collect(), + constants: vec![0; 512], + entry: 0, + args, + initial: initial(), + expected_cause: 0, + fixed_expected: None, + } +} +fn elf(path: &Path, name: &str, args: [u64; 8]) -> Case { + let bytes = std::fs::read(path).unwrap(); + assert_eq!(&bytes[..7], b"\x7fELF\x02\x01\x01"); + let u64_at = |p| u64::from_le_bytes(bytes[p..p + 8].try_into().unwrap()); + let u32_at = |p| u32::from_le_bytes(bytes[p..p + 4].try_into().unwrap()); + let u16_at = |p| u16::from_le_bytes(bytes[p..p + 2].try_into().unwrap()); + assert_eq!(u16_at(18), 243); + let mut c = micro(name, &[], args); + c.entry = u32::try_from(u64_at(24)).expect("kernel entry exceeds local address width"); + assert_eq!(u16_at(58), 64, "invalid ELF64 section header size"); + let sections = usize::try_from(u64_at(40)).unwrap(); + for n in 0..u16_at(60) as usize { + let p = sections.checked_add(n.checked_mul(64).unwrap()).unwrap(); + let flags = u64_at(p + 8); + let address = u32::try_from(u64_at(p + 16)).expect("kernel section exceeds local address width"); + let size = usize::try_from(u64_at(p + 32)).unwrap(); + if flags & 2 == 0 || size == 0 { + continue; + } + let offset = usize::try_from(u64_at(p + 24)).unwrap(); + if flags & 4 != 0 { + assert_eq!(address, 0); + c.program = bytes[offset..offset.checked_add(size).unwrap()].to_vec(); + } else { + assert_ne!( + u32_at(p + 4), + 8, + "allocated NOBITS requires explicit initialization" + ); + assert_eq!(address, 0x40000000); + assert_eq!(flags & 1, 0, "kernel globals must be read-only"); + c.constants.resize(size.max(512), 0); + c.constants[..size].copy_from_slice(&bytes[offset..offset.checked_add(size).unwrap()]); + } + } + assert!(!c.program.is_empty() && c.program.len() <= 4096 && c.program.len() % 4 == 0); + c +} +// Independent integer arithmetic gold: the BEMU Engine does not implement these fixed-point opcodes. +fn fixed_gold(case: &Case, bits: usize, op: u32, mode: u32) -> Banks { + let mut gold = case.initial.clone(); + let bytes = bits / 8; + let mask = (1i128 << bits) - 1; + let minimum = -(1i128 << (bits - 1)); + let maximum = (1i128 << (bits - 1)) - 1; + let read = |offset: usize, size: usize| { + let mut value = [0; 8]; + value[..size].copy_from_slice(&case.initial.0[2 * BANK_BYTES + offset..2 * BANK_BYTES + offset + size]); + u64::from_le_bytes(value) as i128 + }; + let signed = |value: i128, width: usize| if value & (1i128 << (width - 1)) != 0 { value - (1i128 << width) } else { value }; + let mut saturated = false; + for lane in 0..5 { + let a = read(lane * bytes, bytes); + let b = read(128 + lane * bytes, bytes); + let mut result = a; + if mode != 3 || lane >= 2 { + let (value, shift) = match op { + 8 => (a + b, 1), 9 => (signed(a, bits) + signed(b, bits), 1), + 10 => (a - b, 1), 11 => (signed(a, bits) - signed(b, bits), 1), + 39 => (signed(a, bits) * signed(b, bits), bits - 1), + 42 => (a, b as usize & (bits - 1)), + 43 => (signed(a, bits), b as usize & (bits - 1)), + 46 => (read(lane * bytes * 2, bytes * 2), b as usize & (bits * 2 - 1)), + 47 => (signed(read(lane * bytes * 2, bytes * 2), bits * 2), b as usize & (bits * 2 - 1)), + _ => unreachable!(), + }; + let truncated = value >> shift; + let half = shift != 0 && value & (1i128 << (shift - 1)) != 0; + let low = shift != 0 && value & ((1i128 << (shift - 1)) - 1) != 0; + let increment = match mode { + 0 => half, 1 => half && (low || truncated & 1 != 0), 2 => false, + 3 => truncated & 1 == 0 && value & ((1i128 << shift) - 1) != 0, + _ => unreachable!(), + }; + result = truncated + i128::from(increment); + if op == 39 || op == 47 { + saturated |= result < minimum || result > maximum; + result = result.clamp(minimum, maximum); + } else if op == 46 { + saturated |= result > mask; + result = result.min(mask); + } + } + gold.put(1, lane * bytes, &((result & mask) as u64).to_le_bytes()[..bytes]); + } + gold.put(1, 128, &u32::from(saturated).to_le_bytes()); + gold +} + +// Additional coverage for balanced integer reductions and bounded rounding carry. +fn integer_depth_cases(args: [u64; 8]) -> Vec { + let mut cases = vec![]; + for sew in 0..3 { + let bits = 8usize << sew; + let mask = u64::MAX >> (64 - bits); + let values = [1u64 << (bits - 1), mask >> 1, 0, 1, mask]; + let width = [0, 5, 6, 7][sew as usize]; + for reduction in [true, false] { + let operations: Vec<(u32, u32)> = if reduction { + (0..8).map(|op| (op, 2)).collect() + } else { + let mut ops = vec![(8, 2), (9, 2), (10, 2), (11, 2), (39, 0), (42, 0), (43, 0)]; + if sew < 2 { ops.extend([(46, 0), (47, 0)]); } + ops + }; + for (op, kind) in operations { + for mode in 0..(if reduction { 5 } else { 4 }) { + let mut a = args; + a[2] = 5; + let lmul = if 5 * bits > VLEN { 1 } else { 0 }; + let mut words = vec![ + i(0x57, 5, 7, 12, ((sew << 3) | lmul) as i32), + (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, + i(0x13, 6, 0, 11, 128), + (1 << 25) | (6 << 15) | (width << 12) | (16 << 7) | 7, + (1 << 25) | (11 << 15) | (width << 12) | (24 << 7) | 7, + ]; + let mut vm = 1; + if reduction { + if mode == 1 { + words.push((0x19 << 26) | (1 << 25) | (8 << 20) | (3 << 12) | 0x57); + vm = 0; + } else if mode == 2 { + words.push((0x1b << 26) | (1 << 25) | (2 << 12) | 0x57); + vm = 0; + } else if mode == 3 { + words.push(i(0x57, 5, 7, 0, 0xc00 | (sew << 3) as i32)); + } else if mode == 4 { + words.push(i(0x73, 0, 5, 1, 8)); + } + } else { + words.push(i(0x73, 0, 5, mode, 0xa)); + words.push(i(0x73, 0, 5, 0, 9)); + if op == 46 || op == 47 { + words.push((1 << 25) | (11 << 15) | ([5, 6, 7][sew as usize] << 12) | (8 << 7) | 7); + } + } + if !reduction && mode == 3 { words.push(i(0x73, 0, 5, 2, 8)); } + words.push((op << 26) | (vm << 25) | (8 << 20) | (16 << 15) | (kind << 12) | (24 << 7) | 0x57); + words.push(i(0x57, 5, 7, 12, ((sew << 3) | lmul) as i32)); + words.push((1 << 25) | (10 << 15) | (width << 12) | (24 << 7) | 0x27); + if !reduction { + words.push(i(0x73, 6, 2, 0, 9)); + words.push(s(0x23, 2, 10, 6, 128)); + } + words.push(0x8067); + let mut case = micro(&format!("depth_{}_e{bits}_op{op}_mode{mode}", if reduction {"reduce"} else {"round"}), &words, a); + for (index, value) in values.into_iter().enumerate() { + let bytes = bits / 8; + let rhs = if reduction || op == 39 || kind == 2 { value } else { [0, 1, bits as u64 - 1, bits as u64 / 2, 3][index] }; + case.initial.put(2, index * bytes, &value.to_le_bytes()[..bytes]); + case.initial.put(2, 128 + index * bytes, &rhs.to_le_bytes()[..bytes]); + } + if !reduction { case.fixed_expected = Some(fixed_gold(&case, bits, op, mode)); } + if reduction && mode == 4 { case.expected_cause = 2; } + cases.push(case); + } + } + } + } + cases +} + +impl Model { + pub fn new(kernel_dir: &Path) -> Self { + let mut cases = vec![]; + let mut args = [0; 8]; + args[0] = 1 << 32; + args[1] = 2 << 32; + for kernel in ["silu", "swiglu", "snake", "quant"] { + let lengths: &[u32] = if kernel == "quant" { + &[32, 64] + } else { + &[1, 31, 33, 65] + }; + for &n in lengths { + let mut a = args; + match kernel { + "swiglu" => { + a[2] = 3 << 32; + a[3] = u64::from(n); + } + "snake" => { + a[2] = 2; + a[3] = u64::from(n); + a[4] = 48; + a[5] = 56; + } + "quant" => { + a[1] = args[0] + 1024; + a[2] = args[1]; + a[3] = u64::from(n); + a[4] = args[0] + 1536; + } + _ => a[2] = u64::from(n), + } + if kernel == "silu" { a[3] = args[0] + 1024; } + if kernel == "swiglu" { a[4] = args[0] + 1024; } + let mut c = elf( + &kernel_dir.join(format!("{kernel}.elf")), + &format!("{kernel}_{n}"), + a, + ); + if kernel == "snake" { + c.initial.put( + 0, + 48, + &[ + 0.0f32.to_le_bytes(), + 0.5f32.to_le_bytes(), + 0.25f32.to_le_bytes(), + (-0.25f32).to_le_bytes(), + ] + .concat(), + ); + } + cases.push(c); + } + } + // Unaligned loads/stores and every RV32M operation, including divide by zero and overflow. + for &(a, b) in &[ + (17u32, 3u32), + (0x80000000, 0xffffffff), + (0xffffffef, 0), + (0xffffffff, 7), + ] { + let mut aargs = args; + aargs[2] = u64::from(a); + aargs[3] = u64::from(b); + let mut words = vec![]; + for f in 0..8 { + words.push(r(0x33, 5, f, 12, 13, 1)); + words.push(s(0x23, 2, 10, 5, f * 4 + 1)); + } + for f in [0, 1, 2, 4, 5] { + words.push(i(3, 6, f, 10, 1)); + words.push(s(0x23, 2, 10, 6, 48 + f * 4)); + } + words.push(0x8067); + cases.push(micro(&format!("scalar_m_{a:x}_{b:x}"), &words, aargs)); + } + for funct in [0, 1, 4, 5, 6, 7] { + for &(left, right) in &[(0xffffffff, 1), (7, 7)] { + let mut a = args; + a[2] = left; + a[3] = right; + let words = [ + i(0x13, 5, 0, 0, 0), + (13 << 20) | (12 << 15) | (funct << 12) | (4 << 8) | 0x63, + i(0x13, 5, 0, 0, 1), + s(0x23, 2, 10, 5, 0), + 0x8067, + ]; + cases.push(micro( + &format!("branch_{funct}_{left:x}_{right:x}"), + &words, + a, + )); + } + } + for sew in [2, 3] { + for (vl, start) in [(0, 0), (4, 0), (4, 1), (4, 4)] { + let width = if sew == 2 { 6 } else { 7 }; + let words = [ + i(0x57, 5, 7, 4, 0xc00 | (sew << 3)), + (1 << 25) | (11 << 15) | (width << 12) | (9 << 7) | 7, + i(0x57, 5, 7, 12, (sew << 3) | 3), + i(0x73, 0, 5, start, 8), + r(0x57, 9, 6, 13, 0, 33), + i(0x57, 5, 7, 4, 0xc00 | (sew << 3)), + (1 << 25) | (10 << 15) | (width << 12) | (9 << 7) | 0x27, + i(0x73, 7, 2, 0, 1), + s(0x23, 2, 10, 7, 64), + 0x8067, + ]; + let mut a = args; + a[2] = vl; + a[3] = 0x80000001; + let mut c = micro(&format!("scalar_insert_e{}_vl{vl}_start{start}", 8 << sew), &words, a); + if sew == 3 && ELEN == 32 { c.expected_cause = 2; } + cases.push(c); + } + } + for scalar in [0x3f800000u64, 0x7f800123] { + for (vl, start) in [(0, 0), (4, 0), (4, 1), (4, 4)] { + let words = [ + i(0x57, 5, 7, 4, 0xc00 | (2 << 3)), + (1 << 25) | (11 << 15) | (6 << 12) | (9 << 7) | 7, + i(7, 3, 2, 13, 0), + i(0x57, 5, 7, 12, (2 << 3) | 3), + i(0x73, 0, 5, start, 8), + r(0x57, 9, 5, 3, 0, 33), + i(0x57, 5, 7, 4, 0xc00 | (2 << 3)), + (1 << 25) | (10 << 15) | (6 << 12) | (9 << 7) | 0x27, + i(0x73, 7, 2, 0, 1), + s(0x23, 2, 10, 7, 64), + 0x8067, + ]; + let mut a = args; + a[2] = vl; + a[3] = 3 << 32; + let mut c = micro(&format!("floating_insert_{scalar:x}_vl{vl}_start{start}"), &words, a); + c.initial.put(3, 0, &scalar.to_le_bytes()); + cases.push(c); + } + } + let mut a = args; + a[2] = 17; + cases.push(micro( + "vset_avl_register_rules", + &[ + i(0x57, 5, 7, 12, 16), + s(0x23, 2, 10, 5, 0), + i(0x57, 5, 7, 0, 16), + s(0x23, 2, 10, 5, 4), + i(0x57, 0, 7, 0, 16), + i(0x73, 5, 2, 0, 0xc20), + s(0x23, 2, 10, 5, 8), + i(0x57, 5, 7, 3, 0xc10u32 as i32), + s(0x23, 2, 10, 5, 12), + 0x8067, + ], + a, + )); + // SEW and integer/fractional LMUL, VL=0, mask/tail preservation, CSR visibility. + for sew in 0..4 { + for lmul in [0, 1, 2, 3, 5, 6, 7] { + for avl in [0, 1, 17, 63] { + let vt = (sew << 3) | lmul; + let mut a = args; + a[2] = avl; + let width = [0, 5, 6, 7][sew as usize]; + let words = [ + i(0x57, 5, 7, 12, vt), + (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, + r(0x57, 16, 0, 8, 8, 0) | (1 << 25), + (1 << 25) | (10 << 15) | (width << 12) | (16 << 7) | 0x27, + i(0x73, 6, 2, 0, 0xc20), + s(0x23, 2, 10, 6, 1536), + i(0x73, 6, 2, 0, 0xc21), + s(0x23, 2, 10, 6, 1540), + 0x8067, + ]; + let mut c = micro( + &format!("vector_sew{}_lmul{lmul}_avl{avl}", 8 << sew), + &words, + a, + ); + if (8 << sew) > ELEN as i32 || (lmul >= 5 && (8 << sew) > (ELEN as i32 >> (8 - lmul))) { + c.expected_cause = 2; + } + cases.push(c); + } + } + } + for sew in 0..4 { + let mut a = args; + a[2] = 17; + let width = [0, 5, 6, 7][sew as usize]; + let words = [ + i(0x57, 5, 7, 12, sew << 3), + (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, + (0x18 << 26) | (1 << 25) | (8 << 20) | (3 << 12) | 0x57, + (8 << 20) | (8 << 15) | (16 << 7) | 0x57, + (1 << 25) | (10 << 15) | (width << 12) | (16 << 7) | 0x27, + 0x8067, + ]; + let mut c = micro(&format!("masked_sew{}", 8 << sew), &words, a); + if sew == 3 && ELEN == 32 { c.expected_cause = 2; } + cases.push(c); + } + for sew in [2, 3] { + let mut a = args; + a[2] = 17; + let width = if sew == 2 { 6 } else { 7 }; + let mut c = micro( + &format!("vector_floating_{}", 8 << sew), + &[ + i(0x57, 5, 7, 12, sew << 3), + (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, + (1 << 25) | (8 << 20) | (8 << 15) | (1 << 12) | (16 << 7) | 0x57, + (0x2c << 26) | (1 << 25) | (8 << 20) | (8 << 15) | (1 << 12) | (16 << 7) | 0x57, + (1 << 25) | (10 << 15) | (width << 12) | (16 << 7) | 0x27, + (0x12 << 26) + | (1 << 25) + | (16 << 20) + | (1 << 15) + | (1 << 12) + | (24 << 7) + | 0x57, + i(0x13, 7, 0, 10, 1024), + (1 << 25) | (7 << 15) | (width << 12) | (24 << 7) | 0x27, + i(0x73, 6, 2, 0, 1), + s(0x23, 2, 10, 6, 1536), + 0x8067, + ], + a, + ); + if sew == 3 { + if ELEN == 32 { c.expected_cause = 2; } + for n in 0..17 { + c.initial + .put(2, n * 8, &(n as f64 / 4.0 - 2.0).to_le_bytes()); + } + } + cases.push(c); + } + // Scalar IEEE arithmetic/FMA/conversion flags are stored through CSR reads. + for fmt in 0..2 { + let load_f = if fmt == 0 { 2 } else { 3 }; + let step = if fmt == 0 { 4 } else { 8 }; + let mut c = micro( + &format!("floating_{}", 32 << fmt), + &[ + i(7, 1, load_f, 11, 0), + i(7, 2, load_f, 11, step), + r(0x53, 3, 0, 1, 2, fmt), + (2 << 27) | (fmt << 25) | (2 << 20) | (1 << 15) | (4 << 7) | 0x43, + s(0x27, load_f, 10, 3, 0), + s(0x27, load_f, 10, 4, 8), + r(0x53, 5, 0, 1, 0, 0x60 | fmt), + s(0x23, 2, 10, 5, 16), + i(0x73, 6, 2, 0, 1), + s(0x23, 2, 10, 6, 20), + 0x8067, + ], + args, + ); + if fmt == 0 { + c.initial.put( + 2, + 0, + &[1.5f32.to_le_bytes(), (-2.25f32).to_le_bytes()].concat(), + ); + } else { + if ELEN == 32 { c.expected_cause = 2; } + c.initial.put( + 2, + 0, + &[1.5f64.to_le_bytes(), (-2.25f64).to_le_bytes()].concat(), + ); + } + cases.push(c); + } + // Raw IEEE encodings preserve signaling NaNs and signed zero in the inputs. + for fmt in 0..2 { + let fixtures: &[(&str, [u64; 3])] = if fmt == 0 { + &[ + ("zeros", [0, 0x80000000, 0x3f800000]), + ("division_by_zero", [0x3f800000, 0, 0xbf800000]), + ("negative_zero", [0x80000000, 0x40000000, 0]), + ("qnan", [0x7fc12345, 0x3f800000, 0xbf800000]), + ("snan", [0x7f812345, 0x3f800000, 0]), + ("infinities", [0x7f800000, 0, 0xff800000]), + ("subnormal", [1, 0x3f000000, 0x00800000]), + ("overflow", [0x7f7fffff, 0x40000000, 0xff7fffff]), + ("conversion_ties", [0x3fc00000, 0xc0200000, 0x3f800000]), + ("division_inexact", [0x3f800000, 0x40400000, 0x80000000]), + ("fused_cancellation", [0x3f800001, 0x3f800001, 0xbf800000]), + ] + } else { + &[ + ("zeros", [0, 0x8000000000000000, 0x3ff0000000000000]), + ( + "division_by_zero", + [0x3ff0000000000000, 0, 0xbff0000000000000], + ), + ("negative_zero", [0x8000000000000000, 0x4000000000000000, 0]), + ( + "qnan", + [0x7ff8123456789abc, 0x3ff0000000000000, 0xbff0000000000000], + ), + ("snan", [0x7ff0123456789abc, 0x3ff0000000000000, 0]), + ("infinities", [0x7ff0000000000000, 0, 0xfff0000000000000]), + ("subnormal", [1, 0x3fe0000000000000, 0x0010000000000000]), + ( + "overflow", + [0x7fefffffffffffff, 0x4000000000000000, 0xffefffffffffffff], + ), + ( + "conversion_ties", + [0x3ff8000000000000, 0xc004000000000000, 0x3ff0000000000000], + ), + ( + "division_inexact", + [0x3ff0000000000000, 0x4008000000000000, 0x8000000000000000], + ), + ( + "fused_cancellation", + [0x3ff0000000000001, 0x3ff0000000000001, 0xbff0000000000000], + ), + ("narrowing_tie", [0x3ff0000010000000, 0x3ff0000000000000, 0]), + ] + }; + let width = if fmt == 0 { 2 } else { 3 }; + let bytes = 1usize << width; + for frm in 0..5 { + for &(name, values) in fixtures { + if frm != 0 + && !matches!( + name, + "conversion_ties" | "fused_cancellation" | "narrowing_tie" + ) + { + continue; + } + let mut a = args; + a[2] = 0x01000001; + a[3] = 0xffffffff; + let mut words = vec![i(0x73, 0, 5, frm, 2)]; + for f in 1..=3 { + words.push(i(7, f, width, 11, (f as usize - 1) as i32 * bytes as i32)); + } + // Clear and sample fflags for every operation; result and flags share a 32-byte record. + let mut operations = vec![]; + for op in [0, 4, 8, 12] { + operations.push((r(0x53, 4, 7, 1, 2, op | fmt), s(0x27, width, 10, 4, 0))); + } + for op in [0x43, 0x47, 0x4b, 0x4f] { + operations.push(( + (3 << 27) + | (fmt << 25) + | (2 << 20) + | (1 << 15) + | (7 << 12) + | (4 << 7) + | op, + s(0x27, width, 10, 4, 0), + )); + } + for source in [1, 2] { + for unsigned in 0..2 { + operations.push(( + r(0x53, 5, 7, source, unsigned, 0x60 | fmt), + s(0x23, 2, 10, 5, 0), + )); + } + } + for unsigned in 0..2 { + operations.push(( + r(0x53, 4, 7, 12 + unsigned, unsigned, 0x68 | fmt), + s(0x27, width, 10, 4, 0), + )); + } + let target = 1 - fmt; + if fmt == 1 { operations.push(( + r(0x53, 4, 7, 1, fmt, 0x20 | target), + s(0x27, if target == 0 { 2 } else { 3 }, 10, 4, 0), + )); } + for (n, (operation, store)) in operations.into_iter().enumerate() { + let offset = n as u32 * 32; + words.extend([ + i(0x73, 0, 5, 0, 1), + operation, + store | ((offset >> 5) << 25), + i(0x73, 6, 2, 0, 1), + s(0x23, 2, 10, 6, offset + 16), + ]); + } + words.extend([i(0x73, 6, 2, 0, 2), s(0x23, 2, 10, 6, 1536), 0x8067]); + let mut c = micro( + &format!("scalar_ieee_{}_frm{frm}_{name}", 32 << fmt), + &words, + a, + ); + if fmt == 1 && ELEN == 32 { c.expected_cause = 2; } + for (n, bits) in values.into_iter().enumerate() { + c.initial.put(2, n * bytes, &bits.to_le_bytes()[..bytes]); + } + cases.push(c); + } + // The vector case runs the same exceptional inputs lane by lane. + let mut a = args; + a[2] = fixtures.len() as u64; + a[3] = 3 << 32; + a[4] = 5 << 32; + let vwidth: u32 = if fmt == 0 { 6 } else { 7 }; + let load = + |vd: u32, rs: u32| (1 << 25) | (rs << 15) | (vwidth << 12) | (vd << 7) | 7; + let store = |vd: u32| (1 << 25) | (7 << 15) | (vwidth << 12) | (vd << 7) | 0x27; + let mut words = vec![ + i(0x73, 0, 5, frm, 2), + i(0x57, 5, 7, 12, (fmt as i32 + 2) << 3), + load(8, 11), + load(12, 13), + ]; + for (n, operation) in [0, 2, 0x24, 0x20, 0x2c].into_iter().enumerate() { + let offset = n as u32 * 128; + words.extend([ + i(0x73, 0, 5, 0, 1), + load(20, 14), + (operation << 26) + | (1 << 25) + | (12 << 20) + | (8 << 15) + | (1 << 12) + | (20 << 7) + | 0x57, + i(0x13, 7, 0, 10, offset as i32), + store(20), + i(0x73, 6, 2, 0, 1), + s(0x23, 2, 10, 6, 1024 + n as u32 * 4), + ]); + } + for unsigned in 0..2 { + words.extend([ + i(0x73, 0, 5, 0, 1), + (0x12 << 26) + | (1 << 25) + | (8 << 20) + | ((1 - unsigned) << 15) + | (1 << 12) + | (24 << 7) + | 0x57, + i(0x13, 7, 0, 10, 640 + unsigned as i32 * 128), + store(24), + i(0x73, 6, 2, 0, 1), + s(0x23, 2, 10, 6, 1044 + unsigned * 4), + ]); + } + words.push(0x8067); + let mut c = micro(&format!("vector_ieee_{}_frm{frm}", 32 << fmt), &words, a); + if fmt == 1 && ELEN == 32 { c.expected_cause = 2; } + for (n, (_, values)) in fixtures.iter().enumerate() { + for (bank, bits) in [(2, values[0]), (3, values[1]), (5, values[2])] { + c.initial.put(bank, n * bytes, &bits.to_le_bytes()[..bytes]); + } + } + cases.push(c); + } + } + cases.push(micro( + "bank_line_crossing_word", + &[i(3, 5, 2, 11, 15), s(0x23, 2, 10, 5, 15), 0x8067], + args, + )); + let mut double_load = micro( + "bank_line_crossing_double", + &[i(7, 5, 3, 11, 11), s(0x27, 3, 10, 5, 11), 0x8067], + args, + ); + if ELEN == 32 { double_load.expected_cause = 2; } + cases.push(double_load); + let mut a = args; + a[2] = 17; + let mut c = micro( + "illegal_group_alignment", + &[ + i(0x57, 5, 7, 12, 17), + (1 << 25) | (11 << 15) | (6 << 12) | (9 << 7) | 7, + ], + a, + ); + c.expected_cause = 2; + cases.push(c); + cases.push(micro("illegal_opcode", &[0xffffffff], args)); + cases.push(micro("load_fault", &[i(3, 5, 2, 11, 0)], { + let mut a = args; + a[1] = 6 << 32; + a + })); + cases.push(micro("store_fault", &[s(0x23, 2, 10, 11, 0)], { + let mut a = args; + a[0] = (1 << 32) + 4094; + a + })); + for (n, cause) in [ + (cases.len() - 3, 2), + (cases.len() - 2, 5), + (cases.len() - 1, 7), + ] { + cases[n].expected_cause = cause; + } + cases.extend(integer_depth_cases(args)); + let mut local = micro( + "local_stack_and_constants", + &[ + i(0x13, 2, 0, 2, -16), + s(0x23, 2, 2, 11, 0), + s(0x23, 2, 2, 12, 4), + i(3, 5, 2, 2, 0), + i(3, 6, 2, 2, 4), + s(0x23, 2, 10, 5, 0), + s(0x23, 2, 10, 6, 4), + 0x400003b7, + i(3, 5, 2, 7, 0), + s(0x23, 2, 10, 5, 8), + i(0x13, 2, 0, 2, 16), + 0x8067, + ], + args, + ); + local.constants[..4].copy_from_slice(&0x72cafe31u32.to_le_bytes()); + cases.push(local); + let mut readonly = micro( + "store_readonly_constants", + &[0x400002b7, s(0x23, 2, 5, 11, 0), 0x8067], + args, + ); + readonly.expected_cause = 7; + cases.push(readonly); + for kernel in ["sin", "cos"] { + let values: Vec = (0..65).map(|n| (n as f32 - 32.0) * 0.75).chain([ + 120.0, -120.0, 10000.0, -10000.0, 1e10, -1e10, 1e30, -1e30, + f32::MAX, -f32::MAX, f32::from_bits(1), -f32::from_bits(1), + 0.0, -0.0, f32::INFINITY, f32::NEG_INFINITY, f32::NAN, + ]).collect(); + let mut a = args; + a[2] = values.len() as u64; + a[3] = a[0] + 1024; + let mut c = elf(&kernel_dir.join(format!("{kernel}.elf")), &format!("{kernel}_fp32_range"), a); + for (n, value) in values.iter().enumerate() { c.initial.put(2, n * 4, &value.to_le_bytes()); } + cases.push(c); + } + for kernel in ["gelu", "tanh"] { + let mut a = args; + a[2] = 37; + a[3] = a[0] + 1024; + cases.push(elf(&kernel_dir.join(format!("{kernel}.elf")), &format!("{kernel}_fp32"), a)); + } + let b = initial(); + Self { + cases, + current: 0, + actual: b.clone(), + expected: b, + engine: Engine::new(VLEN, ELEN, IBUF_BYTES), + fault: None, + } + } + pub fn select(&mut self, n: usize) { + self.current = n; + let c = &self.cases[n]; + self.actual = c.initial.clone(); + let descriptor = [ + u64::from(c.entry), + c.program.len() as u64, + 0x40002000, + u64::from(c.args[0]), + u64::from(c.args[1]), + u64::from(c.args[2]), + u64::from(c.args[3]), + u64::from(c.args[4]), + u64::from(c.args[5]), + u64::from(c.args[6]), + u64::from(c.args[7]), + 0, + ]; + for (index, value) in descriptor.iter().enumerate() { + self.actual.put(1, 2048 + index * 8, &value.to_le_bytes()); + } + self.expected = match &c.fixed_expected { + Some(gold) => { + let mut expected = gold.clone(); + let descriptor = BANK_BYTES + 2048..BANK_BYTES + 2144; + expected.0[descriptor.clone()].copy_from_slice(&self.actual.0[descriptor]); + expected + } + None => self.actual.clone(), + }; + self.engine.load_constants(n % 2, &c.constants); + self.engine.load_program(n % 2, &c.program); + self.fault = if c.fixed_expected.is_some() { None } else { self + .engine + .run( + n % 2, + c.entry, + c.program.len() as u32, + c.args.map(u64::from), + 0x40002000, + &mut self.expected, + ) + .err() }; + assert_eq!( + self.fault.map(|f| f.cause).unwrap_or(0), + c.expected_cause, + "unexpected reference outcome for {}: {:?}", + c.name, self.fault + ); + let read_float = |banks: &Banks, address: u64| { + f32::from_le_bytes(banks.0[banks.range(address, 4).unwrap()].try_into().unwrap()) + }; + let operation = c.name.split('_').next().unwrap(); + if matches!(operation, "silu" | "swiglu" | "snake" | "sin" | "cos") { + let length = c.args[if matches!(operation, "swiglu" | "snake") { 3 } else { 2 }] as usize; + let channels = if operation == "snake" { c.args[2] as usize } else { 1 }; + for channel in 0..channels { + for index in 0..length { + let offset = ((channel * length + index) * 4) as u64; + let x = read_float(&c.initial, c.args[1] + offset); + let expected = match operation { + "silu" => x / (1.0 + (-x).exp()), + "swiglu" => x / (1.0 + (-x).exp()) * read_float(&c.initial, c.args[2] + offset), + "snake" => { + let alpha = read_float(&c.initial, c.args[4] + channel as u64 * 4).exp(); + let beta = read_float(&c.initial, c.args[5] + channel as u64 * 4).exp() + 1e-9; + let sine = (x * alpha).sin(); + x + sine * sine / beta + } + "sin" => x.sin(), + "cos" => x.cos(), + _ => unreachable!(), + }; + let actual = read_float(&self.expected, c.args[0] + offset); + assert!(if expected.is_nan() { actual.is_nan() } else { + actual == expected || (actual - expected).abs() <= 2e-6 * (1.0 + expected.abs()) + }, "{} element {index}: {actual} != libm {expected}", c.name); + } + } + } + eprintln!("RVV case {n}: {}", c.name); + } +} diff --git a/arch/src/main/scala/framework/mem-core/axis/src/csrc/README.md b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/README.md similarity index 100% rename from arch/src/main/scala/framework/mem-core/axis/src/csrc/README.md rename to arch/src/main/scala/framework/acc/gpdomain/src/main/resources/README.md diff --git a/arch/src/main/scala/framework/rvv/src/main/resources/rvv.json b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/gpdomain.json similarity index 100% rename from arch/src/main/scala/framework/rvv/src/main/resources/rvv.json rename to arch/src/main/scala/framework/acc/gpdomain/src/main/resources/gpdomain.json diff --git a/arch/src/main/scala/framework/rvv/src/main/resources/interface.sv b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/interface.sv similarity index 100% rename from arch/src/main/scala/framework/rvv/src/main/resources/interface.sv rename to arch/src/main/scala/framework/acc/gpdomain/src/main/resources/interface.sv diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/package.sv b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/package.sv new file mode 100644 index 000000000..12694ca7e --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/package.sv @@ -0,0 +1,97 @@ +package rvv_pkg; + timeunit 1ns; timeprecision 1ps; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function void rvv_model_init(); + import "DPI-C" function int unsigned rvv_case_count(); + import "DPI-C" function void rvv_case_select(int unsigned n); + import "DPI-C" function longint unsigned rvv_case_meta(int unsigned field); + import "DPI-C" function int unsigned rvv_image_bytes(); + import "DPI-C" function int unsigned rvv_program_word(int unsigned n); + import "DPI-C" function int unsigned rvv_image_word(int unsigned n); + import "DPI-C" function void rvv_memory_masked_write( + longint unsigned address, + longint unsigned data, + int unsigned mask + ); + import "DPI-C" function longint unsigned rvv_expected_status(int unsigned field); + import "DPI-C" function longint unsigned rvv_memory_read( + longint unsigned address, + int unsigned size + ); + import "DPI-C" function int unsigned rvv_memory_error( + longint unsigned address, + int unsigned size + ); + import "DPI-C" function void rvv_memory_write( + longint unsigned address, + int unsigned size, + longint unsigned data, + int unsigned mask + ); + import "DPI-C" function int unsigned rvv_compare_memory(); + import "DPI-C" function int unsigned rvv_compare_image(); + + class launch_item extends uvm_sequence_item; + bit execute; + bit protocol_fault; + bit [31:0] protocol_cause, protocol_pc, protocol_instruction; + bit [63:0] protocol_tval; + bit [3:0] rob; + `uvm_object_utils(launch_item) + function new(string name = "launch_item"); + super.new(name); + endfunction + endclass + class completion_item extends uvm_sequence_item; + bit fault; + bit [15:0] write_bank; + bit [3:0] rob; + bit [31:0] pc, instruction, cause; + bit [63:0] tval; + `uvm_object_utils_begin(completion_item) + `uvm_field_int(fault, UVM_DEFAULT) + `uvm_field_int(rob, UVM_DEFAULT) + `uvm_field_int(write_bank, UVM_DEFAULT) + `uvm_field_int(pc, UVM_DEFAULT) + `uvm_field_int(instruction, UVM_DEFAULT | UVM_NOCOMPARE) + `uvm_field_int(cause, UVM_DEFAULT) + `uvm_field_int(tval, UVM_DEFAULT) + `uvm_object_utils_end + function new(string name = "completion_item"); + super.new(name); + endfunction + function bit do_compare(uvm_object rhs, uvm_comparer comparer); + completion_item other; + if (!$cast(other, rhs)) return 0; + return super.do_compare(rhs, comparer) && (!fault || instruction == other.instruction); + endfunction + endclass + class kernel_model extends reference_model #(launch_item, completion_item); + `uvm_component_utils(kernel_model) + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void write(launch_item t); + completion_item expected = completion_item::type_id::create("expected"); + expected.rob = t.rob; + expected.fault = t.execute ? rvv_expected_status(0) : 0; + expected.pc = t.execute ? rvv_expected_status(1) : 0; + expected.instruction = t.execute ? rvv_expected_status(2) : 0; + expected.cause = t.execute ? rvv_expected_status(3) : 0; + expected.tval = t.execute ? rvv_expected_status(4) : 0; + if (t.protocol_fault) begin + expected.fault = 1; + expected.pc = t.protocol_pc; + expected.instruction = t.protocol_instruction; + expected.cause = t.protocol_cause; + expected.tval = t.protocol_tval; + end + expected.write_bank = t.execute && !expected.fault ? 1 : 0; + expected_ap.write(expected); + endfunction + endclass + typedef environment#(launch_item, completion_item, kernel_model) kernel_env; + `include "test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/suite.toml new file mode 100644 index 000000000..10972b204 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "librvv_reference_model" + +[[tests]] +functional_groups = [] +name = "topkernel" +top = "topkernel_tb" +test = "protocol_test" +filelist = "topkernel.f" +timeout_ns = 40000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/test.svh b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/test.svh new file mode 100644 index 000000000..dc916616c --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/test.svh @@ -0,0 +1,265 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + virtual rvv_if vif; + kernel_env test_env; + int unsigned completed = 0; + bit protocol_fault = 0; + bit reject_write = 0; + bit [31:0] protocol_cause, protocol_pc, protocol_instruction; + bit [63:0] protocol_tval; + int unsigned requests[`RVV_MEMORY_PORTS]; + function new(string name, uvm_component parent); + super.new(name, parent); + + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual rvv_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "RVV interface missing") + test_env = kernel_env::type_id::create("env", this); + endfunction + task memory_port(int p); + bit read_pending = 0, write_pending = 0; + int read_delay, write_delay; + bit [31:0] seed = 32'h629731a5 ^ (p * 32'h1234567); + longint unsigned address; + forever begin + @(negedge vif.clock); + seed = {seed[30:0], seed[31] ^ seed[21] ^ seed[1] ^ seed[0]}; + if (vif.reset) begin + read_pending = 0; + write_pending = 0; + vif.read_ready[p] = 0; + vif.write_ready[p] = 0; + vif.read_response_valid[p] = 0; + vif.write_response_valid[p] = 0; + end else begin + vif.read_ready[p] = !read_pending && seed[0]; + vif.write_ready[p] = !write_pending && seed[1]; + if (read_pending && read_delay != 0) read_delay--; + if (write_pending && write_delay != 0) write_delay--; + vif.read_response_valid[p] = read_pending && read_delay == 0; + vif.write_response_valid[p] = write_pending && write_delay == 0; + end + @(posedge vif.clock); + if (!vif.reset) begin + if (vif.read_valid[p] && vif.read_ready[p]) begin + requests[p]++; + read_pending = 1; + read_delay = 1 + seed[4:2]; + address = (longint'(vif.read_bank[p] + vif.read_group[p]) << 32) | (int'(vif.read_row[p]) << 4); + vif.read_data[p] = {rvv_memory_read(address + 8, 3), rvv_memory_read(address, 3)}; + end + if (vif.write_valid[p] && vif.write_ready[p]) begin + requests[p]++; + write_pending = 1; + write_delay = 1 + seed[7:5]; + address = (longint'(vif.write_bank[p] + vif.write_group[p]) << 32) | (int'(vif.write_row[p]) << 4); + if (!reject_write) begin + rvv_memory_masked_write(address, vif.write_data[p][63:0], vif.write_mask[p][7:0]); + rvv_memory_masked_write(address + 8, vif.write_data[p][127:64], + vif.write_mask[p][15:8]); + end + vif.write_ok[p] = !reject_write; + end + if (vif.read_response_valid[p] && vif.read_response_ready[p]) read_pending = 0; + if (vif.write_response_valid[p] && vif.write_response_ready[p]) write_pending = 0; + end + end + endtask + task monitor(); + forever begin + @(posedge vif.clock); + if (!vif.reset) begin + if (vif.command_valid && vif.command_ready) begin + launch_item item = launch_item::type_id::create("launch"); + item.execute = vif.command_funct7 == 79; + item.protocol_fault = protocol_fault; + item.protocol_cause = protocol_cause; + item.protocol_tval = protocol_tval; + item.protocol_pc = protocol_pc; + item.protocol_instruction = protocol_instruction; + item.rob = vif.command_rob; + test_env.input_export.write(item); + end + if (vif.done_valid && vif.done_ready) begin + completion_item item = completion_item::type_id::create("completion"); + item.fault = vif.done_fault; + item.pc = vif.done_pc; + item.instruction = vif.done_instruction; + item.cause = vif.done_cause; + item.tval = vif.done_tval; + item.rob = vif.done_rob; + item.write_bank = vif.done_write_bank; + test_env.output_export.write(item); + end + end + end + endtask + task command(int funct7, longint unsigned rs1, longint unsigned rs2); + @(negedge vif.clock); + vif.command_funct7 = funct7; + vif.command_rs1 = rs1; + vif.command_rs2 = rs2; + vif.command_rob = completed % 16; + vif.command_valid = 1; + do @(posedge vif.clock); while (!vif.command_ready); + @(negedge vif.clock); + vif.command_valid = 0; + endtask + task retire(); + bit [250:0] held; + while (!vif.done_valid) @(negedge vif.clock); + held = { + vif.done_rob, + vif.done_write_bank, + vif.done_fault, + vif.done_pc, + vif.done_instruction, + vif.done_cycles, + vif.done_cause, + vif.done_tval, + vif.done_fflags, + vif.done_vxsat + }; + repeat (5) begin + @(negedge vif.clock); + if(!vif.done_valid || held !== {vif.done_rob,vif.done_write_bank,vif.done_fault,vif.done_pc, + vif.done_instruction,vif.done_cycles,vif.done_cause,vif.done_tval,vif.done_fflags,vif.done_vxsat}) + `uvm_fatal("COMPLETION", "completion changed under backpressure") + if (vif.command_ready || !vif.busy) + `uvm_fatal("COMPLETION", "completion allowed command before retirement") + end + vif.done_ready = 1; + @(negedge vif.clock); + vif.done_ready = 0; + completed++; + test_env.scoreboard.wait_checked(completed); + endtask + task upload(); + command(44, (longint'(rvv_image_bytes()) << 30) | ((6 + rvv_case_meta(0)) << 20), 64'h10000000); + for (int word_index = 0; word_index < rvv_image_bytes() / 4; word_index++) begin + @(negedge vif.clock); + vif.image_valid = 1; + vif.image_data = rvv_image_word(word_index); + do @(posedge vif.clock); while (!vif.image_ready); + end + @(negedge vif.clock); + vif.image_valid = 0; + retire(); + if (!rvv_compare_image()) `uvm_fatal("IMAGE", "image load modified compute banks") + endtask + task execute(); + vif.reset = 1; + vif.image_valid = 0; + vif.command_valid = 0; + vif.done_ready = 0; + foreach (vif.read_ready[p]) begin + vif.read_ready[p] = 0; + vif.write_ready[p] = 0; + vif.read_response_valid[p] = 0; + vif.write_response_valid[p] = 0; + vif.read_data[p] = 0; + vif.write_ok[p] = 0; + end + rvv_model_init(); + fork + begin + for (int p = 0; p < `RVV_MEMORY_PORTS; p++) begin + automatic int port = p; + fork + memory_port(port); + join_none + end + wait fork; + end + monitor(); + join_none + repeat (4) @(negedge vif.clock); + vif.reset = 0; + for (int case_index = 0; case_index < rvv_case_count(); case_index++) begin + rvv_case_select(case_index); + upload(); + command(79, (64'd1 << 20) | ((6 + rvv_case_meta(0)) << 10), 2048); + while (!vif.done_valid) @(negedge vif.clock); + if (!rvv_compare_memory()) + `uvm_fatal("REFERENCE", $sformatf("case %0d bank contents differ", case_index)) + retire(); + `uvm_info("RVV", $sformatf("case %0d completed and compared", case_index), UVM_LOW) + end + foreach (requests[p]) + if (requests[p] == 0) `uvm_fatal("PORTS", $sformatf("bank port %0d was never exercised", p)) + // A malformed header reports a command fault before any code/data is written. + protocol_fault = 1; + protocol_cause = 2; + protocol_tval = 32'hdeadbeef; + command(44, (longint'(rvv_image_bytes()) << 30) | (64'd6 << 20), 0); + for (int index = 0; index < 6; index++) begin + @(negedge vif.clock); + vif.image_valid = 1; + vif.image_data = index == 0 ? 32'hdeadbeef : rvv_image_word(index); + do @(posedge vif.clock); while (!vif.image_ready); + end + @(negedge vif.clock); + vif.image_valid = 0; + retire(); + protocol_tval = 25; + command(44, (64'd25 << 30) | (64'd6 << 20), 0); + retire(); + protocol_cause = 1; + protocol_tval = (64'd1 << 20) | (64'd6 << 10); + command(79, protocol_tval, 2048); + retire(); + protocol_cause = 2; + protocol_tval = 127; + command(127, 0, 0); + retire(); + // Writable globals and BSS are forbidden in a kernel image. + for (int invalid_field = 3; invalid_field <= 5; invalid_field += 2) begin + protocol_cause = 2; + protocol_tval = invalid_field == 3 ? 32'h50000 : 4; + command(44, (longint'(rvv_image_bytes()) << 30) | (64'd6 << 20), 0); + for (int index = 0; index < 6; index++) begin + @(negedge vif.clock); + vif.image_valid = 1; + vif.image_data = index == invalid_field ? + (invalid_field == 3 ? 32'h50000 : 4) : rvv_image_word(index); + do @(posedge vif.clock); while (!vif.image_ready); + end + @(negedge vif.clock); + vif.image_valid = 0; + retire(); + end + // A failed compute-bank write acknowledgement is an execution fault. + protocol_fault = 0; + rvv_case_select(14); + upload(); + protocol_fault = 1; + protocol_cause = 7; + protocol_tval = (64'd1 << 32) + 1; + protocol_pc = 4; + protocol_instruction = rvv_program_word(1); + reject_write = 1; + command(79, (64'd1 << 20) | ((6 + rvv_case_meta(0)) << 10), 2048); + retire(); + reject_write = 0; + protocol_pc = 0; + protocol_instruction = 0; + protocol_fault = 0; + // Reset cancels a command while it waits for a descriptor bank response. + rvv_case_select(rvv_case_count() - 2); + upload(); + command(79, (64'd1 << 20) | ((6 + rvv_case_meta(0)) << 10), 2048); + do @(posedge vif.clock); while (!(vif.read_valid[0] && vif.read_ready[0])); + @(negedge vif.clock); + vif.reset = 1; + test_env.scoreboard.cancel_pending(); + repeat (3) @(negedge vif.clock); + vif.reset = 0; + repeat (5) begin + @(negedge vif.clock); + if (vif.done_valid || vif.busy) `uvm_fatal("RESET", "reset did not cancel pending kernel") + end + `uvm_info("RVV", $sformatf("checked %0d Blink commands", completed), UVM_LOW) + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/topkernel.f b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/topkernel.f new file mode 100644 index 000000000..bb71b8eb7 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/topkernel.f @@ -0,0 +1,8 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ ++incdir+@RTL@/KernelEngine +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/KernelEngine/filelist.f +@RESOURCES@/topkernel_tb.sv diff --git a/arch/src/main/scala/framework/rvv/src/main/resources/topkernel_tb.sv b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/topkernel_tb.sv similarity index 98% rename from arch/src/main/scala/framework/rvv/src/main/resources/topkernel_tb.sv rename to arch/src/main/scala/framework/acc/gpdomain/src/main/resources/topkernel_tb.sv index a81179167..b7b912972 100644 --- a/arch/src/main/scala/framework/rvv/src/main/resources/topkernel_tb.sv +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/resources/topkernel_tb.sv @@ -48,7 +48,7 @@ module topkernel_tb; KernelEngine dut ( .clock(clock), .reset(vif.reset), - .io_channelReady(1'b1), + .io_bankAccessReady(1'b1), .io_cmdReq_valid(vif.command_valid), .io_cmdReq_ready(vif.command_ready), .io_cmdReq_bits_cmd_funct7(vif.command_funct7), diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/Emit.scala new file mode 100644 index 000000000..841ae5377 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/Emit.scala @@ -0,0 +1,26 @@ +package framework.acc.gpdomain + +import _root_.configs.GlobalConfig + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new GpDomain(thread.acc.get.frontend, thread.acc.get.memDomain, thread.acc.get.gpDomain), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/GpDomain.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/GpDomain.scala new file mode 100644 index 000000000..c4bfaa995 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/GpDomain.scala @@ -0,0 +1,76 @@ +package framework.acc.gpdomain + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.frontend.scheduler.{Complete, Issue} +import framework.acc.memdomain.{BankLayout, BankLayoutRequest, BankPort} +import framework.acc.memdomain.toddr.DmaStatus +import framework.acc.gpdomain.execution.{Execution, KernelCompletion} +import framework.acc.gpdomain.loader.Loader +import framework.acc.gpdomain.memory.Memory +import framework.acc.balldomain.KernelBallPort +import framework.acc.gpdomain.decoder.Decoder +import framework.acc.gpdomain.loader.ImageLoad + +class ImagePort extends Bundle { + val start = Decoupled(new ImageLoad) + val data = Decoupled(UInt(32.W)) + val terminal = Decoupled(new DmaStatus) + val complete = Flipped(Decoupled(new KernelCompletion)) + val release = Valid(Bool()) +} + +@instantiable +class GpDomain(val frontendParam: FrontendParam, val memParam: MemDomainParam, val gpParam: GpDomainParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val ownerId = frontendParam.ownerId + require(gpParam.enable) + + @public + val io = IO(new Bundle { + val layoutRequest = Decoupled(new BankLayoutRequest(frontendParam)) + val layoutResponse = Flipped(Decoupled(new BankLayout(memParam))) + val command = Flipped(Decoupled(new Issue(frontendParam, memParam))) + val complete = Decoupled(new Complete(frontendParam)) + val image = Flipped(new ImagePort) + val banks = Vec(gpParam.memory.memoryPorts, new BankPort(frontendParam, memParam)) + val ball = new KernelBallPort(frontendParam, memParam) + }) + + val loader: Instance[Loader] = Instantiate(new Loader(frontendParam, memParam, gpParam)) + val execution: Instance[Execution] = Instantiate(new Execution(frontendParam, memParam, gpParam)) + val memory: Instance[Memory] = Instantiate(new Memory(frontendParam, memParam, gpParam)) + val decoder: Instance[Decoder] = Instantiate(new Decoder(frontendParam, memParam)) + + decoder.io.issue <> io.command + io.layoutRequest <> decoder.io.layoutRequest + decoder.io.layoutResponse <> io.layoutResponse + loader.io.cmdReq <> decoder.io.request + + execution.io.program <> loader.io.program + execution.io.launch <> loader.io.launch + loader.io.done <> execution.io.done + loader.io.memory <> memory.io.loader + + for (port <- 0 until gpParam.memory.memoryPorts) { + memory.io.request(port) <> execution.io.memoryRequest(port) + execution.io.memoryResponse(port) <> memory.io.response(port) + io.banks(port) <> memory.io.banks(port) + } + + io.ball <> execution.io.ball + + loader.io.uploadStart <> io.image.start + loader.io.image <> io.image.data + loader.io.imageTerminal <> io.image.terminal + io.image.complete <> loader.io.uploadComplete + loader.io.release := io.image.release + + io.complete <> loader.io.cmdResp +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/configs/GpDomainParam.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/configs/GpDomainParam.scala new file mode 100644 index 000000000..c6dd993f4 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/configs/GpDomainParam.scala @@ -0,0 +1,31 @@ +package framework.acc.gpdomain.configs + +import buckyball.config.GpDomainConfig + +import upickle.default._ +import framework.acc.gpdomain.execution.configs.{Params => ExecutionParams} +import framework.acc.gpdomain.loader.configs.{Params => LoaderParams} +import framework.acc.gpdomain.memory.configs.{Params => MemoryParams} + +case class GpDomainParam( + enable: Boolean, + laneNumber: Int, + vLen: Int, + eLen: Int, + iBufWords: Int, + memoryPorts: Int) { + val execution = ExecutionParams(laneNumber, vLen, eLen, iBufWords, memoryPorts) + val memory = MemoryParams(memoryPorts, eLen) + val loader = LoaderParams(iBufWords, memory.constBytes) +} + +object GpDomainParam { + + def fromProto(p: GpDomainConfig): GpDomainParam = { + require(p.hasEnable, "GP enable policy is required") + GpDomainParam(p.getEnable, p.getLaneNumber, p.getVLen, p.getELen, p.getIBufWords, p.getMemoryPorts) + } + + implicit val rw: ReadWriter[GpDomainParam] = macroRW[GpDomainParam] + +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/decoder/DISA.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/decoder/DISA.scala new file mode 100644 index 000000000..dd061f459 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/decoder/DISA.scala @@ -0,0 +1,7 @@ +package framework.acc.gpdomain.decoder + +import chisel3.util.BitPat + +object DISA { + val RUN_KERNEL_BITPAT = BitPat("b1001111") +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/decoder/Decoder.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/decoder/Decoder.scala new file mode 100644 index 000000000..799761e59 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/decoder/Decoder.scala @@ -0,0 +1,48 @@ +package framework.acc.gpdomain.decoder + +import framework.lib.queue.Queue +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.frontend.scheduler.Issue +import framework.acc.memdomain.{BankLayout, BankLayoutRequest} + +@instantiable +class Decoder(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Module { + val commandDataBits = frontendParam.commandDataBits + + @public + val io = IO(new Bundle { + val issue = Flipped(Decoupled(new Issue(frontendParam, memParam))) + val layoutRequest = Decoupled(new BankLayoutRequest(frontendParam)) + val layoutResponse = Flipped(Decoupled(new BankLayout(memParam))) + val request = Decoupled(new Issue(frontendParam, memParam)) + }) + + val run = io.issue.bits.cmd.cmd.raw_inst(31, 25) === DISA.RUN_KERNEL_BITPAT + + val pending = Instantiate(new Queue(new Issue(frontendParam, memParam), 4)) + io.layoutRequest.valid := io.issue.valid && run && pending.io.enq.ready + pending.io.enq.valid := io.issue.valid && run && io.layoutRequest.ready + pending.io.enq.bits := io.issue.bits + io.issue.ready := run && pending.io.enq.ready && io.layoutRequest.ready + io.layoutRequest.bits.banks(0) := io.issue.bits.renamedBankAccess.rd_bank_0_id + io.layoutRequest.bits.banks(1) := 0.U // Operand 1 names the program buffer, not a data bank. + io.layoutRequest.bits.banks(2) := io.issue.bits.renamedBankAccess.wr_bank_id + io.layoutRequest.bits.enabled(0) := true.B + io.layoutRequest.bits.enabled(1) := false.B + io.layoutRequest.bits.enabled(2) := true.B + io.request.valid := pending.io.deq.valid && io.layoutResponse.valid + io.request.bits := pending.io.deq.bits + io.request.bits.cmd.op1_col := io.layoutResponse.bits.groups(0) + io.request.bits.cmd.op2_col := 0.U + io.request.bits.cmd.wr_col := io.layoutResponse.bits.groups(2) + pending.io.deq.ready := io.request.ready && io.layoutResponse.valid + io.layoutResponse.ready := io.request.ready && pending.io.deq.valid + when(io.issue.valid)(assert(run, "GPDomain accepts only RUN_KERNEL")) +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/build.sc b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/build.sc new file mode 100644 index 000000000..a2cbc2e3e --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/build.sc @@ -0,0 +1,31 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.gpdomain.execution.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Execution.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/IBuf.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/arithmetic/FALU.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/arithmetic/IALU.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/scalar/ScalarExecution.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/scalar/ScalarFloating.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/scalar/ScalarRF.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/vector/Operands.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/vector/VRF.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/vector/VectorThread.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/Emit.scala new file mode 100644 index 000000000..7418ed1ae --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/Emit.scala @@ -0,0 +1,26 @@ +package framework.acc.gpdomain.execution + +import _root_.configs.GlobalConfig + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Execution(thread.acc.get.frontend, thread.acc.get.memDomain, thread.acc.get.gpDomain), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/Execution.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/Execution.scala new file mode 100644 index 000000000..bf9456913 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/Execution.scala @@ -0,0 +1,441 @@ +package framework.acc.gpdomain.execution + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.acc.gpdomain.loader.KernelContext +import framework.acc.balldomain.KernelBallPort +import framework.acc.gpdomain.execution.scalar.{ScalarExecution, ScalarFloating, ScalarRF} +import framework.acc.gpdomain.execution.vector.VectorThread +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} + +class ProgramWrite extends Bundle { + val buffer = Bool() + val first = Bool() + val address = UInt(32.W) + val data = UInt(32.W) +} + +class KernelLaunch(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val context = new KernelContext(frontendParam, memParam) + val iBuffer = Bool() + val entry = UInt(32.W) + val end = UInt(32.W) + val stack = UInt(64.W) + val args = Vec(8, UInt(64.W)) +} + +class KernelCompletion extends Bundle { + val fault = Bool() + val pc = UInt(32.W) + val instruction = UInt(32.W) + val cycles = UInt(64.W) + val cause = UInt(32.W) + val tval = UInt(64.W) + val fflags = UInt(5.W) + val vxsat = Bool() +} + +class VectorMemoryRequest extends Bundle { + val address = UInt(64.W) + val write = Bool() + val data = UInt(64.W) + val mask = UInt(8.W) + val size = UInt(2.W) +} + +class VectorMemoryResponse extends Bundle { + val data = UInt(64.W) + val error = Bool() +} + +@instantiable +class Execution(val frontendParam: FrontendParam, val memParam: MemDomainParam, val gpParam: GpDomainParam) + extends Module { + val p = gpParam.execution + + @public + val io = IO(new Bundle { + val program = Flipped(Decoupled(new ProgramWrite)) + val launch = Flipped(Decoupled(new KernelLaunch(frontendParam, memParam))) + val done = Decoupled(new KernelCompletion) + val ball = new KernelBallPort(frontendParam, memParam) + val memoryRequest = Vec(p.memoryPorts, Decoupled(new VectorMemoryRequest)) + val memoryResponse = Vec(p.memoryPorts, Flipped(Decoupled(new VectorMemoryResponse))) + }) + + val iBuf: Seq[Instance[IBuf]] = Seq.fill(2)(Instantiate(new IBuf(gpParam))) + val registers: Instance[ScalarRF] = Instantiate(new ScalarRF(frontendParam, memParam)) + val scalar: Instance[ScalarExecution] = Instantiate(new ScalarExecution()) + val floating: Instance[ScalarFloating] = Instantiate(new ScalarFloating(gpParam)) + val vector: Instance[VectorThread] = Instantiate(new VectorThread(gpParam)) + + val idle :: fetch :: fetchWait :: execute :: vectorWait :: scalarWait :: floatWait :: memorySend :: memoryWait :: ballSend :: ballWait :: completed :: Nil = + Enum(12) + + val state = RegInit(idle) + val activeBuffer = RegInit(false.B) + val end = Reg(UInt(32.W)) + val pc = Reg(UInt(32.W)) + val instruction = RegInit(0.U(32.W)) + val cycles = RegInit(0.U(64.W)) + val fault = RegInit(false.B) + val cause = RegInit(0.U(32.W)) + val tval = RegInit(0.U(64.W)) + val fflags = RegInit(0.U(5.W)) + val frm = RegInit(0.U(3.W)) + val vxrm = RegInit(0.U(2.W)) + val vxsat = RegInit(false.B) + val scalarTarget = Reg(UInt(5.W)) + val scalarNextPc = Reg(UInt(32.W)) + + val scalarDivide = (instruction(6, 0) === "h33".U || instruction(6, 0) === "h3b".U) && instruction( + 31, + 25 + ) === 1.U && instruction(14, 12) >= 4.U + + val memory = Reg(new VectorMemoryRequest) + val memoryFloating = Reg(Bool()) + val memoryUnsigned = Reg(Bool()) + + def finish(isFault: Bool, code: UInt, value: UInt): Unit = { + state := completed + fault := isFault + cause := code + tval := value + } + + val context = Reg(new KernelContext(frontendParam, memParam)) + + // A kernel names one launch group per local bank: the read groups first, then the write groups. + val ballRs1 = registers.io.xReadData1 + val ballFunct = instruction(31, 25) + val ballEnable = ballFunct(6, 4) + val ballLocal = VecInit(Seq(0, 10, 20).map(lsb => ballRs1(frontendParam.bank_id_len + lsb - 1, lsb))) + + val ballUsed = VecInit( + ballEnable === 1.U || ballEnable === 3.U || ballEnable === 4.U, + ballEnable === 4.U, + ballEnable === 2.U || ballEnable === 3.U || ballEnable === 4.U + ) + + val ballFromRead = ballLocal.map(_ < context.readGroups) + val ballGroups = context.readGroups +& context.writeGroups + val ballRequest = io.ball.request.bits + ballRequest.target := 0.U + ballRequest.instId := context.instId + ballRequest.enabled := ballUsed + ballRequest.cmd.rob_id := context.rob_id + ballRequest.cmd.funct := ballFunct + ballRequest.cmd.rs2 := registers.io.xReadData2 + ballRequest.cmd.iter := ballRs1(62, 30) + ballRequest.cmd.op1_bank := Mux(ballFromRead(0), context.readBank, context.writeBank) + ballRequest.cmd.op2_bank := Mux(ballFromRead(1), context.readBank, context.writeBank) + ballRequest.cmd.wr_bank := context.writeBank + ballRequest.cmd.op1_group := Mux(ballFromRead(0), ballLocal(0), ballLocal(0) - context.readGroups) + ballRequest.cmd.op2_group := Mux(ballFromRead(1), ballLocal(1), ballLocal(1) - context.readGroups) + ballRequest.cmd.wr_group := ballLocal(2) - context.readGroups + ballRequest.cmd.op1_col := 1.U + ballRequest.cmd.op2_col := 1.U + ballRequest.cmd.wr_col := 1.U + ballRequest.cmd.wr_read_bank := 0.U + ballRequest.cmd.wr_arch_bank := ballLocal(2) + io.ball.request.valid := state === ballSend + io.ball.response.ready := state === ballWait + when(io.ball.request.fire)(state := ballWait) + when(io.ball.request.valid) { + for (i <- 0 until 2) { + assert(!ballUsed(i) || ballLocal(i) < ballGroups, "GP Ball read bank outside launch groups") + } + assert( + !ballUsed(2) || !ballFromRead(2) && ballLocal(2) < ballGroups, + "GP Ball write outside write groups" + ) + } + + val running = state =/= idle && state =/= completed + io.launch.ready := state === idle + io.done.valid := state === completed + io.done.bits.fault := fault + io.done.bits.pc := pc + io.done.bits.instruction := instruction + io.done.bits.cycles := cycles + io.done.bits.cause := cause + io.done.bits.tval := tval + io.done.bits.fflags := fflags + io.done.bits.vxsat := vxsat + + for (i <- 0 until 2) { + iBuf(i).io.upload.valid := io.program.valid && io.program.bits.buffer === i.U + iBuf(i).io.upload.bits := io.program.bits + iBuf(i).io.uploadEnabled := !io.launch.valid && (!running || activeBuffer =/= i.U) + iBuf(i).io.fetchEnabled := state === fetch && activeBuffer === i.U && pc < end + iBuf(i).io.fetchAddress := pc + } + io.program.ready := Mux(io.program.bits.buffer, iBuf(1).io.upload.ready, iBuf(0).io.upload.ready) + val imageWords = Mux(io.launch.bits.iBuffer, iBuf(1).io.wordCount, iBuf(0).io.wordCount) + val fetchLoaded = Mux(activeBuffer, iBuf(1).io.loaded, iBuf(0).io.loaded) + val fetchedInstruction = Mux(activeBuffer, iBuf(1).io.instruction, iBuf(0).io.instruction) + + registers.io.initialize.valid := io.launch.fire + registers.io.initialize.bits := io.launch.bits + registers.io.xReadAddress1 := instruction(19, 15) + registers.io.xReadAddress2 := instruction(24, 20) + registers.io.fReadAddress1 := instruction(19, 15) + registers.io.fReadAddress2 := instruction(24, 20) + registers.io.fReadAddress3 := instruction(31, 27) + registers.io.xWrite.valid := false.B + registers.io.xWrite.bits.address := instruction(11, 7) + registers.io.xWrite.bits.data := 0.U + registers.io.fWrite.valid := false.B + registers.io.fWrite.bits.address := instruction(11, 7) + registers.io.fWrite.bits.data := 0.U + scalar.io.divRequest.valid := false.B + scalar.io.divRequest.bits.a := registers.io.xReadData1 + scalar.io.divRequest.bits.b := registers.io.xReadData2 + scalar.io.divRequest.bits.sew := Mux(instruction(6, 0) === "h3b".U, 2.U, 3.U) + scalar.io.divRequest.bits.signed := !instruction(12) + scalar.io.divRequest.bits.remainder := instruction(13) + scalar.io.divResponse.ready := state === scalarWait && !io.launch.fire + scalar.io.clear := io.launch.fire + scalar.io.instruction := instruction + scalar.io.pc := pc + scalar.io.source1 := registers.io.xReadData1 + scalar.io.source2 := registers.io.xReadData2 + floating.io.instruction := instruction + floating.io.source1 := registers.io.fReadData1 + floating.io.source2 := registers.io.fReadData2 + floating.io.source3 := registers.io.fReadData3 + floating.io.xSource := registers.io.xReadData1 + floating.io.frm := frm + floating.io.start := false.B + vector.io.initialize := false.B + vector.io.roundingMode := frm + vector.io.vxrm := vxrm + vector.io.vstartWrite.valid := false.B + vector.io.vstartWrite.bits := 0.U + vector.io.issue.valid := false.B + vector.io.issue.bits.instruction := instruction + vector.io.issue.bits.scalar1 := registers.io.xReadData1 + vector.io.issue.bits.scalar2 := registers.io.xReadData2 + vector.io.issue.bits.floating := registers.io.fReadData1 + vector.io.result.ready := state === vectorWait + + for (i <- 0 until p.memoryPorts) { + io.memoryRequest(i) <> vector.io.memoryRequest(i) + vector.io.memoryResponse(i) <> io.memoryResponse(i) + } + when(io.ball.response.fire) { + registers.io.xWrite.valid := instruction(11, 7) =/= 0.U + registers.io.xWrite.bits.data := 0.U + pc := pc + 4.U + state := fetch + } + when(state === scalarWait && scalar.io.divResponse.fire) { + registers.io.xWrite.valid := true.B + registers.io.xWrite.bits.address := scalarTarget + registers.io.xWrite.bits.data := Mux( + instruction(6, 0) === "h3b".U, + Cat(Fill(32, scalar.io.divResponse.bits(31)), scalar.io.divResponse.bits(31, 0)), + scalar.io.divResponse.bits + ) + pc := scalarNextPc + state := fetch + } + when(state === memorySend || state === memoryWait) { + io.memoryRequest(0).valid := state === memorySend + io.memoryRequest(0).bits := memory + vector.io.memoryRequest(0).ready := false.B + vector.io.memoryResponse(0).valid := false.B + io.memoryResponse(0).ready := state === memoryWait + } + + val opcode = instruction(6, 0) + val funct3 = instruction(14, 12) + val vectorInstruction = opcode === "h57".U || + (opcode === "h07".U || opcode === "h27".U) && (funct3 === 0.U || funct3 >= 5.U) + val floatingInstruction = opcode === "h53".U || opcode === "h43".U || opcode === "h47".U || + opcode === "h4b".U || opcode === "h4f".U + val load = opcode === "h03".U || opcode === "h07".U + val store = opcode === "h23".U || opcode === "h27".U + val csrAddress = instruction(31, 20) + val csrKnown = WireDefault(true.B) + val csrValue = WireDefault(0.U(64.W)) + switch(csrAddress) { + is("h001".U)(csrValue := fflags) + is("h002".U)(csrValue := frm) + is("h003".U)(csrValue := Cat(frm, fflags)) + is("h008".U)(csrValue := vector.io.vstart) + is("h009".U)(csrValue := vxsat) + is("h00a".U)(csrValue := vxrm) + is("h00f".U)(csrValue := Cat(vxrm, vxsat)) + is("hc20".U)(csrValue := vector.io.vl) + is("hc21".U)(csrValue := vector.io.vtype) + is("hc22".U)(csrValue := (p.vLen / 8).U) + } + csrKnown := Seq(0x001, 0x002, 0x003, 0x008, 0x009, 0x00a, 0x00f, 0xc20, 0xc21, 0xc22) + .map(a => csrAddress === a.U).reduce(_ || _) + val csrSource = Mux(funct3(2), instruction(19, 15), registers.io.xReadData1) + val csrWrite = funct3(1, 0) === 1.U || instruction(19, 15) =/= 0.U + + val csrNew = MuxLookup(funct3(1, 0), csrValue)(Seq( + 1.U -> csrSource, + 2.U -> (csrValue | csrSource), + 3.U -> (csrValue & ~csrSource) + )) + + when(running)(cycles := cycles + 1.U) + when(io.done.fire)(state := idle) + when(io.launch.fire) { + context := io.launch.bits.context + activeBuffer := io.launch.bits.iBuffer + pc := io.launch.bits.entry + end := io.launch.bits.end + instruction := 0.U + cycles := 0.U + fflags := 0.U + frm := 0.U + fault := false.B + cause := 0.U + tval := 0.U + state := fetch + when(io.launch.bits.entry(1, 0).orR || io.launch.bits.end(1, 0).orR) { + finish(true.B, 0.U, io.launch.bits.entry) + }.elsewhen(io.launch.bits.entry >= io.launch.bits.end || io.launch.bits.end > imageWords * 4.U) { + finish(true.B, 1.U, io.launch.bits.entry) + } + } + when(state === fetch) { + when(pc === end)(finish(false.B, 0.U, 0.U)) + .elsewhen(pc(1, 0).orR)(finish(true.B, 0.U, pc)) + .elsewhen(pc >= end || !fetchLoaded)(finish(true.B, 1.U, pc)) + .otherwise(state := fetchWait) + } + when(state === fetchWait) { instruction := fetchedInstruction; state := execute } + when(state === execute) { + when(opcode === "h7b".U && funct3 === 3.U) { + state := ballSend + }.elsewhen(vectorInstruction) { + vector.io.issue.valid := true.B + when(vector.io.issue.fire)(state := vectorWait) + }.elsewhen(floatingInstruction) { + when(!floating.io.legal)(finish(true.B, 2.U, instruction)) + .otherwise { + floating.io.start := floating.io.ready + when(floating.io.ready)(state := floatWait) + } + }.elsewhen(load || store) { + val fp = opcode === "h07".U || opcode === "h27".U + val size = Mux(fp, Mux(funct3 === 3.U, 3.U, 2.U), funct3(1, 0)) + val legal = Mux( + fp, + funct3 === 2.U || (p.eLen == 64).B && funct3 === 3.U, + Mux(store, funct3 <= 3.U, funct3 <= 6.U) + ) + val immediate = Mux( + store, + Cat(Fill(52, instruction(31)), instruction(31, 25), instruction(11, 7)), + Cat(Fill(52, instruction(31)), instruction(31, 20)) + ) + val address = registers.io.xReadData1 + immediate + memory.address := address + memory.write := store + memory.data := Mux(fp, registers.io.fReadData2, registers.io.xReadData2) + memory.size := size + memory.mask := MuxLookup(size, 1.U(8.W))(Seq(0.U -> 1.U, 1.U -> 3.U, 2.U -> 15.U, 3.U -> 255.U)) + memoryFloating := fp + memoryUnsigned := funct3(2) + when(!legal)(finish(true.B, 2.U, instruction)) + .otherwise(state := memorySend) + }.elsewhen(opcode === "h73".U) { + when(funct3 === 0.U) { + when(instruction === "h00100073".U)(finish(true.B, 3.U, pc)) + .elsewhen(instruction === "h00000073".U)(finish(true.B, 11.U, 0.U)) + .otherwise(finish(true.B, 2.U, instruction)) + }.elsewhen(funct3 === 4.U || !csrKnown || csrWrite && csrAddress(11, 10) === 3.U) { + finish(true.B, 2.U, instruction) + }.otherwise { + registers.io.xWrite.valid := true.B + registers.io.xWrite.bits.data := csrValue + when(csrWrite) { + switch(csrAddress) { + is("h001".U)(fflags := csrNew(4, 0)) + is("h002".U)(frm := csrNew(2, 0)) + is("h003".U) { frm := csrNew(7, 5); fflags := csrNew(4, 0) } + is("h008".U) { vector.io.vstartWrite.valid := true.B; vector.io.vstartWrite.bits := csrNew } + is("h009".U)(vxsat := csrNew(0)) + is("h00a".U)(vxrm := csrNew(1, 0)) + is("h00f".U) { vxrm := csrNew(2, 1); vxsat := csrNew(0) } + } + } + pc := pc + 4.U + state := fetch + } + }.elsewhen(!scalar.io.legal)(finish(true.B, 2.U, instruction)) + .elsewhen(scalar.io.nextPc(63, 32).orR)(finish(true.B, 1.U, scalar.io.nextPc)) + .elsewhen(scalar.io.nextPc(1, 0).orR)(finish(true.B, 0.U, scalar.io.nextPc)) + .elsewhen(scalarDivide) { + scalar.io.divRequest.valid := true.B + when(scalar.io.divRequest.fire) { + scalarTarget := scalar.io.destination + scalarNextPc := scalar.io.nextPc + state := scalarWait + } + }.otherwise { + registers.io.xWrite.valid := scalar.io.write + registers.io.xWrite.bits.data := scalar.io.result + pc := scalar.io.nextPc + state := fetch + } + } + when(state === memorySend && io.memoryRequest(0).fire)(state := memoryWait) + when(state === memoryWait && io.memoryResponse(0).fire) { + when(io.memoryResponse(0).bits.error)(finish(true.B, Mux(memory.write, 7.U, 5.U), memory.address)) + .otherwise { + when(!memory.write) { + val data = io.memoryResponse(0).bits.data + when(memoryFloating) { + registers.io.fWrite.valid := true.B + registers.io.fWrite.bits.data := Mux(memory.size === 2.U, Cat("hffffffff".U(32.W), data(31, 0)), data) + }.otherwise { + registers.io.xWrite.valid := true.B + registers.io.xWrite.bits.data := MuxLookup(memory.size, data)(Seq( + 0.U -> Cat(Fill(56, !memoryUnsigned && data(7)), data(7, 0)), + 1.U -> Cat(Fill(48, !memoryUnsigned && data(15)), data(15, 0)), + 2.U -> Cat(Fill(32, !memoryUnsigned && data(31)), data(31, 0)) + )) + } + } + pc := pc + 4.U + state := fetch + } + } + when((state === floatWait || state === execute && floatingInstruction) && floating.io.valid) { + when(!floating.io.legal)(finish(true.B, 2.U, instruction)) + .otherwise { + registers.io.fWrite.valid := floating.io.floatWrite + registers.io.fWrite.bits.data := floating.io.result + registers.io.xWrite.valid := floating.io.integerWrite + registers.io.xWrite.bits.data := floating.io.result + fflags := fflags | floating.io.flags + pc := pc + 4.U + state := fetch + } + } + when(vector.io.result.fire) { + when(vector.io.result.bits.fault)(finish(true.B, vector.io.result.bits.cause, vector.io.result.bits.tval)) + .otherwise { + registers.io.xWrite.valid := vector.io.result.bits.scalarWrite + registers.io.xWrite.bits.data := vector.io.result.bits.scalarData + registers.io.fWrite.valid := vector.io.result.bits.floatWrite + registers.io.fWrite.bits.data := vector.io.result.bits.floatData + fflags := fflags | vector.io.result.bits.flags + vxsat := vxsat || vector.io.result.bits.saturated + pc := pc + 4.U + state := fetch + } + } +} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/frontend/IBuf.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/IBuf.scala similarity index 88% rename from arch/src/main/scala/framework/rvv/src/main/scala/frontend/IBuf.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/IBuf.scala index 0bd92d18c..2c6750233 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/frontend/IBuf.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/IBuf.scala @@ -1,13 +1,14 @@ -package framework.rvv +package framework.acc.gpdomain.execution + +import framework.acc.gpdomain.configs.GpDomainParam import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig @instantiable -class IBuf(val b: GlobalConfig) extends Module { - val p = b.rvv +class IBuf(val gpParam: GpDomainParam) extends Module { + val p = gpParam.execution @public val io = IO(new Bundle { diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/execution/FALU.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/FALU.scala similarity index 94% rename from arch/src/main/scala/framework/rvv/src/main/scala/execution/FALU.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/FALU.scala index 5a964cc48..fd48ad896 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/execution/FALU.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/FALU.scala @@ -1,4 +1,8 @@ -package framework.rvv +package framework.acc.gpdomain.execution.arithmetic + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.gpdomain.configs.GpDomainParam import chisel3._ import chisel3.util._ @@ -6,7 +10,9 @@ import chisel3.experimental.hierarchy.{instantiable, public} import hardfloat._ @instantiable -class FALU(val maxWidth: Int) extends Module { +class FALU(gpParam: GpDomainParam) extends Module { + val maxWidth = gpParam.eLen + require(maxWidth == 32 || maxWidth == 64) @public @@ -45,29 +51,29 @@ class FALU(val maxWidth: Int) extends Module { val a = recFNFromFN(expWidth, sigWidth, av) val b = recFNFromFN(expWidth, sigWidth, bv) val c = recFNFromFN(expWidth, sigWidth, cv) - val add = Module(new AddRecFN(expWidth, sigWidth)) + val add = Instantiate(new AddRecFN(expWidth, sigWidth)) add.io.a := Mux(io.op === 39.U, b, a) add.io.b := Mux(io.op === 39.U, a, b) add.io.subOp := io.op === 2.U || io.op === 39.U add.io.roundingMode := io.roundingMode add.io.detectTininess := 1.U - val mul = Module(new MulRecFN(expWidth, sigWidth)) + val mul = Instantiate(new MulRecFN(expWidth, sigWidth)) mul.io.a := a mul.io.b := b mul.io.roundingMode := io.roundingMode mul.io.detectTininess := 1.U - val fma = Module(new MulAddRecFN(expWidth, sigWidth)) + val fma = Instantiate(new MulAddRecFN(expWidth, sigWidth)) fma.io.a := b fma.io.b := Mux(io.op(2), a, c) fma.io.c := Mux(io.op(2), c, a) fma.io.op := Cat(io.op(0), io.op(1) ^ io.op(0)) fma.io.roundingMode := io.roundingMode fma.io.detectTininess := 1.U - val cmp = Module(new CompareRecFN(expWidth, sigWidth)) + val cmp = Instantiate(new CompareRecFN(expWidth, sigWidth)) cmp.io.a := a cmp.io.b := b cmp.io.signaling := Seq(25, 27, 29, 31).map(n => io.op === n.U).reduce(_ || _) - val div = Module(new DivSqrtRecFN_small(expWidth, sigWidth, 0)) + val div = Instantiate(new DivSqrtRecFN_small(expWidth, sigWidth, 0)) div.io.a := Mux(io.op === 33.U, b, a) div.io.b := Mux(io.op === 33.U, a, b) div.io.sqrtOp := squareRoot @@ -224,11 +230,11 @@ class FALU(val maxWidth: Int) extends Module { val (se, ss) = if (srcWidth == 32) (8, 24) else (11, 53) val (de, ds) = if (dstWidth == 32) (8, 24) else (11, 53) val matchesWidth = io.sew === (if (baseSelector != 0 || srcWidth == 32) 2 else 3).U - val floatToInt = Module(new RecFNToIN(se, ss, dstWidth)) + val floatToInt = Instantiate(new RecFNToIN(se, ss, dstWidth)) floatToInt.io.in := recFNFromFN(se, ss, io.a(srcWidth - 1, 0)) floatToInt.io.signedOut := io.subop(0) floatToInt.io.roundingMode := Mux(io.subop(2, 0) >= 6.U, 1.U, io.roundingMode) - val intToFloat = Module(new INToRecFN(srcWidth, de, ds)) + val intToFloat = Instantiate(new INToRecFN(srcWidth, de, ds)) intToFloat.io.in := io.a(srcWidth - 1, 0) intToFloat.io.signedIn := io.subop(0) intToFloat.io.roundingMode := io.roundingMode @@ -245,7 +251,7 @@ class FALU(val maxWidth: Int) extends Module { io.flags := intToFloat.io.exceptionFlags } if (baseSelector != 0) { - val convert = Module(new RecFNToRecFN(se, ss, de, ds)) + val convert = Instantiate(new RecFNToRecFN(se, ss, de, ds)) convert.io.in := recFNFromFN(se, ss, io.a(srcWidth - 1, 0)) convert.io.roundingMode := Mux(io.subop === 21.U, 6.U, io.roundingMode) convert.io.detectTininess := 1.U diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/execution/IALU.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/IALU.scala similarity index 81% rename from arch/src/main/scala/framework/rvv/src/main/scala/execution/IALU.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/IALU.scala index dc053c515..96d875784 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/execution/IALU.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/IALU.scala @@ -1,6 +1,8 @@ -package framework.rvv +package framework.acc.gpdomain.execution.arithmetic -import framework.arith.{DivRem, DivRemRequest} +import framework.acc.gpdomain.configs.GpDomainParam + +import framework.lib.arith.{DivRem, DivRemRequest} import chisel3._ import chisel3.util._ @@ -8,7 +10,9 @@ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantia /** Element arithmetic shared by the e8/e16/e32/e64 integer vector instructions. */ @instantiable -class IALU(val maxWidth: Int) extends Module { +class IALU(gpParam: GpDomainParam) extends Module { + val maxWidth = gpParam.eLen + require(maxWidth == 32 || maxWidth == 64) val maxSew = log2Ceil(maxWidth / 8) val productWidth = 2 * maxWidth @@ -32,14 +36,15 @@ class IALU(val maxWidth: Int) extends Module { val legal = Output(Bool()) }) - val divider: Instance[DivRem] = Instantiate(new DivRem(maxWidth)) + val divider: Instance[DivRem] = + Instantiate(new DivRem(maxWidth)) divider.io.request <> io.divRequest io.divResponse <> divider.io.response divider.io.clear := io.clear val mask = MuxLookup(io.sew, Fill(maxWidth, 1.U(1.W)))(Seq(0.U -> "hff".U, 1.U -> "hffff".U, 2.U -> "hffffffff".U)) val a = io.a & mask - val b = io.b & mask + val rhs = io.b & mask val c = io.c & mask val sa = MuxLookup(io.sew, a)(Seq( @@ -48,19 +53,19 @@ class IALU(val maxWidth: Int) extends Module { 2.U -> a(31, 0).asSInt.pad(maxWidth).asUInt )).asSInt - val sb = MuxLookup(io.sew, b)(Seq( - 0.U -> b(7, 0).asSInt.pad(maxWidth).asUInt, - 1.U -> b(15, 0).asSInt.pad(maxWidth).asUInt, - 2.U -> b(31, 0).asSInt.pad(maxWidth).asUInt + val sb = MuxLookup(io.sew, rhs)(Seq( + 0.U -> rhs(7, 0).asSInt.pad(maxWidth).asUInt, + 1.U -> rhs(15, 0).asSInt.pad(maxWidth).asUInt, + 2.U -> rhs(31, 0).asSInt.pad(maxWidth).asUInt )).asSInt val width = 8.U(7.W) << io.sew - val shift = b(5, 0) & (width - 1.U) + val shift = rhs(5, 0) & (width - 1.U) val signedProduct = sa * sb - val unsignedProduct = a * b - val mixedProduct = sa * Cat(0.U(1.W), b).asSInt + val unsignedProduct = a * rhs + val mixedProduct = sa * Cat(0.U(1.W), rhs).asSInt val reverseMixedProduct = Cat(0.U(1.W), a).asSInt * sb - val unsignedSum = a +& b + val unsignedSum = a +& rhs val signedSum = sa +& sb val signedDifference = sa -& sb val maximum = Cat(0.U(1.W), mask(maxWidth - 1, 1)).asSInt @@ -68,7 +73,7 @@ class IALU(val maxWidth: Int) extends Module { val signedOverflow = signedSum > maximum || signedSum < minimum val differenceOverflow = signedDifference > maximum || signedDifference < minimum val carrySum = unsignedSum + io.carry - val borrow = a < (b +& io.carry) + val borrow = a < (rhs +& io.carry) val wideMask = MuxLookup(io.sew, Fill(maxWidth, 1.U(1.W)))(Seq(0.U -> "hffff".U, 1.U -> "hffffffff".U)) val wideA = io.a & wideMask @@ -104,7 +109,7 @@ class IALU(val maxWidth: Int) extends Module { switch(io.funct6) { is(8.U)(roundInput := unsignedSum.zext) is(9.U)(roundInput := signedSum) - is(10.U)(roundInput := a.zext - b.zext) + is(10.U)(roundInput := a.zext - rhs.zext) is(11.U)(roundInput := signedDifference) } }.otherwise { @@ -150,13 +155,13 @@ class IALU(val maxWidth: Int) extends Module { is(37.U)(io.result := unsignedProduct & mask) is(38.U)(io.result := (mixedProduct.asUInt >> width) & mask) is(39.U)(io.result := (signedProduct.asUInt >> width) & mask) - is(41.U)(io.result := (b * c + a) & mask) - is(43.U)(io.result := (a - b * c) & mask) + is(41.U)(io.result := (rhs * c + a) & mask) + is(43.U)(io.result := (a - rhs * c) & mask) is(45.U)(io.result := (unsignedProduct + c) & mask) is(47.U)(io.result := (c - unsignedProduct) & mask) - is(48.U, 52.U)(io.result := (wideFirst +& b) & wideMask) + is(48.U, 52.U)(io.result := (wideFirst +& rhs) & wideMask) is(49.U, 53.U)(io.result := (signedWideFirst +& sb).asUInt & wideMask) - is(50.U, 54.U)(io.result := (wideFirst - b) & wideMask) + is(50.U, 54.U)(io.result := (wideFirst - rhs) & wideMask) is(51.U, 55.U)(io.result := (signedWideFirst - sb).asUInt & wideMask) is(56.U)(io.result := unsignedProduct & wideMask) is(58.U)(io.result := mixedProduct.asUInt & wideMask) @@ -173,35 +178,35 @@ class IALU(val maxWidth: Int) extends Module { (io.sew < maxSew.U && io.funct6 >= 48.U && io.funct6 =/= 57.U)) }.otherwise { switch(io.funct6) { - is(0.U)(io.result := (a + b) & mask) - is(2.U)(io.result := (a - b) & mask) - is(3.U)(io.result := (b - a) & mask) - is(4.U)(io.result := Mux(a < b, a, b)) - is(5.U)(io.result := Mux(sa < sb, a, b)) - is(6.U)(io.result := Mux(a > b, a, b)) - is(7.U)(io.result := Mux(sa > sb, a, b)) - is(9.U)(io.result := a & b) - is(10.U)(io.result := a | b) - is(11.U)(io.result := a ^ b) + is(0.U)(io.result := (a + rhs) & mask) + is(2.U)(io.result := (a - rhs) & mask) + is(3.U)(io.result := (rhs - a) & mask) + is(4.U)(io.result := Mux(a < rhs, a, rhs)) + is(5.U)(io.result := Mux(sa < sb, a, rhs)) + is(6.U)(io.result := Mux(a > rhs, a, rhs)) + is(7.U)(io.result := Mux(sa > sb, a, rhs)) + is(9.U)(io.result := a & rhs) + is(10.U)(io.result := a | rhs) + is(11.U)(io.result := a ^ rhs) is(16.U)(io.result := carrySum & mask) is(17.U)(io.result := (carrySum >> width)(0)) - is(18.U)(io.result := (a - b - io.carry) & mask) + is(18.U)(io.result := (a - rhs - io.carry) & mask) is(19.U)(io.result := borrow) - is(23.U)(io.result := Mux(io.carry, b, a)) - is(24.U)(io.result := a === b) - is(25.U)(io.result := a =/= b) - is(26.U)(io.result := a < b) + is(23.U)(io.result := Mux(io.carry, rhs, a)) + is(24.U)(io.result := a === rhs) + is(25.U)(io.result := a =/= rhs) + is(26.U)(io.result := a < rhs) is(27.U)(io.result := sa < sb) - is(28.U)(io.result := a <= b) + is(28.U)(io.result := a <= rhs) is(29.U)(io.result := sa <= sb) - is(30.U)(io.result := a > b) + is(30.U)(io.result := a > rhs) is(31.U)(io.result := sa > sb) is(32.U) { io.result := Mux(unsignedSum > mask, mask, unsignedSum); io.saturated := unsignedSum > mask } is(33.U) { io.result := Mux(signedOverflow, Mux(sa < 0.S, minimum, maximum), signedSum).asUInt & mask io.saturated := signedOverflow } - is(34.U) { io.result := Mux(a < b, 0.U, a - b); io.saturated := a < b } + is(34.U) { io.result := Mux(a < rhs, 0.U, a - rhs); io.saturated := a < rhs } is(35.U) { io.result := Mux(differenceOverflow, Mux(sa < 0.S, minimum, maximum), signedDifference).asUInt & mask io.saturated := differenceOverflow diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..9634e9caa --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/configs/Params.scala @@ -0,0 +1,26 @@ +package framework.acc.gpdomain.execution.configs + +import chisel3.util.{isPow2, log2Ceil} + +case class Params( + laneNumber: Int, + vLen: Int, + eLen: Int, + iBufWords: Int, + memoryPorts: Int) { + require(laneNumber > 0) + require(vLen >= 128 && vLen >= eLen && isPow2(vLen)) + require(eLen == 32 || eLen == 64) + require(iBufWords > 0 && isPow2(iBufWords)) + require(memoryPorts > 0) + require(laneNumber <= memoryPorts) + + val wordBits: Int = eLen + val wordOffsetBits: Int = log2Ceil(wordBits) + val maxSew: Int = log2Ceil(eLen / 8) + val wordsPerRegister: Int = vLen / wordBits + val registerBanks: Int = math.min(laneNumber, wordsPerRegister) + val elementsPerRegister: Int = vLen / eLen + + require(isPow2(registerBanks), "RVV register bank count must be a power of two") +} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/execution/ScalarExecution.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarExecution.scala similarity index 97% rename from arch/src/main/scala/framework/rvv/src/main/scala/execution/ScalarExecution.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarExecution.scala index 1c5292e3c..337b8e6cd 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/execution/ScalarExecution.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarExecution.scala @@ -1,13 +1,13 @@ -package framework.rvv +package framework.acc.gpdomain.execution.scalar -import framework.arith.{DivRem, DivRemRequest} +import framework.lib.arith.{DivRem, DivRemRequest} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} @instantiable -class ScalarExecution extends Module { +class ScalarExecution() extends Module { @public val io = IO(new Bundle { diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/execution/ScalarFloating.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarFloating.scala similarity index 93% rename from arch/src/main/scala/framework/rvv/src/main/scala/execution/ScalarFloating.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarFloating.scala index 72bb0023d..ab978be64 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/execution/ScalarFloating.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarFloating.scala @@ -1,11 +1,15 @@ -package framework.rvv +package framework.acc.gpdomain.execution.scalar +import framework.acc.gpdomain.configs.GpDomainParam + +import framework.acc.gpdomain.execution.arithmetic.FALU import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} @instantiable -class ScalarFloating(val maxWidth: Int) extends Module { +class ScalarFloating(gpParam: GpDomainParam) extends Module { + val maxWidth = gpParam.eLen @public val io = IO(new Bundle { @@ -35,7 +39,8 @@ class ScalarFloating(val maxWidth: Int) extends Module { val usesRounding = fmaInstruction || Seq(0, 4, 8, 12, 0x2c, 0x20, 0x60, 0x68).map(n => operation === n.U).reduce(_ || _) val rounding = Mux(usesRounding, Mux(funct === 7.U, io.frm, funct), 0.U) - val falu: Instance[FALU] = Instantiate(new FALU(maxWidth)) + val falu: Instance[FALU] = + Instantiate(new FALU(gpParam)) def unbox(source: UInt): UInt = Mux(double, source, Mux(source(63, 32).andR, source(31, 0), "h7fc00000".U)) falu.io.a := unbox(io.source1) falu.io.b := unbox(io.source2) diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/register/ScalarRF.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarRF.scala similarity index 80% rename from arch/src/main/scala/framework/rvv/src/main/scala/register/ScalarRF.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarRF.scala index c237121fc..3baa5e393 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/register/ScalarRF.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/scalar/ScalarRF.scala @@ -1,15 +1,19 @@ -package framework.rvv +package framework.acc.gpdomain.execution.scalar +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.acc.gpdomain.execution.KernelLaunch import chisel3._ import chisel3.util.Valid import chisel3.experimental.hierarchy.{instantiable, public} @instantiable -class ScalarRF extends Module { +class ScalarRF(frontendParam: FrontendParam, memParam: MemDomainParam) extends Module { @public val io = IO(new Bundle { - val initialize = Flipped(Valid(new KernelLaunch)) + val initialize = Flipped(Valid(new KernelLaunch(frontendParam, memParam))) val xReadAddress1 = Input(UInt(5.W)) val xReadAddress2 = Input(UInt(5.W)) diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/register/Operands.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/Operands.scala similarity index 96% rename from arch/src/main/scala/framework/rvv/src/main/scala/register/Operands.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/Operands.scala index 84db6ecfb..9b7a1b807 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/register/Operands.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/Operands.scala @@ -1,13 +1,14 @@ -package framework.rvv +package framework.acc.gpdomain.execution.vector + +import framework.acc.gpdomain.configs.GpDomainParam import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import framework.top.GlobalConfig @instantiable -class Operands(val b: GlobalConfig) extends Module { - val p = b.rvv +class Operands(val gpParam: GpDomainParam) extends Module { + val p = gpParam.execution val wordsPerRegister = p.wordsPerRegister val bankCount = p.registerBanks val bankBits = log2Ceil(bankCount) @@ -16,7 +17,8 @@ class Operands(val b: GlobalConfig) extends Module { val snapshotWords = 8 * wordsPerRegister val snapshotBits = log2Ceil(snapshotWords) - @public val io = IO(new Bundle { + @public + val io = IO(new Bundle { val initialize = Input(Bool()) val busy = Output(Bool()) @@ -45,7 +47,7 @@ class Operands(val b: GlobalConfig) extends Module { val snapshotDone = Decoupled(Bool()) }) - val vrf = Instantiate(new VRF(b)) + val vrf = Instantiate(new VRF(gpParam)) vrf.io.initialize := io.initialize val idle :: reading :: readDone :: writing :: writeDone :: copying :: copyDone :: Nil = Enum(7) val state = RegInit(idle) diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/register/VRF.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/VRF.scala similarity index 94% rename from arch/src/main/scala/framework/rvv/src/main/scala/register/VRF.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/VRF.scala index da415034e..e708c3e0a 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/register/VRF.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/VRF.scala @@ -1,13 +1,14 @@ -package framework.rvv +package framework.acc.gpdomain.execution.vector + +import framework.acc.gpdomain.configs.GpDomainParam import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig @instantiable -class VRF(val b: GlobalConfig) extends Module { - val p = b.rvv +class VRF(val gpParam: GpDomainParam) extends Module { + val p = gpParam.execution val wordBits = p.wordBits val wordsPerRegister = p.vLen / wordBits val bankCount = p.registerBanks diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/VectorThread.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/VectorThread.scala new file mode 100644 index 000000000..8a01f93c5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/vector/VectorThread.scala @@ -0,0 +1,941 @@ +package framework.acc.gpdomain.execution.vector + +import framework.acc.gpdomain.configs.GpDomainParam + +import framework.acc.gpdomain.execution.{VectorMemoryRequest, VectorMemoryResponse} +import framework.acc.gpdomain.execution.arithmetic.{FALU, IALU} +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} + +class VectorIssue extends Bundle { + val instruction = UInt(32.W) + val scalar1 = UInt(64.W) + val scalar2 = UInt(64.W) + val floating = UInt(64.W) +} + +class VectorResult extends Bundle { + val fault = Bool() + val cause = UInt(32.W) + val tval = UInt(64.W) + val scalarWrite = Bool() + val scalarData = UInt(64.W) + val floatWrite = Bool() + val floatData = UInt(64.W) + val flags = UInt(5.W) + val saturated = Bool() +} + +@instantiable +class VectorThread(val gpParam: GpDomainParam) extends Module { + val p = gpParam.execution + + @public + val io = IO(new Bundle { + val initialize = Input(Bool()) + val roundingMode = Input(UInt(3.W)) + val vxrm = Input(UInt(2.W)) + val vstartWrite = Flipped(Valid(UInt(64.W))) + val vl = Output(UInt(32.W)) + val vtype = Output(UInt(64.W)) + val vstart = Output(UInt(64.W)) + val issue = Flipped(Decoupled(new VectorIssue)) + val result = Decoupled(new VectorResult) + val memoryRequest = Vec(p.memoryPorts, Decoupled(new VectorMemoryRequest)) + val memoryResponse = + Vec(p.memoryPorts, Flipped(Decoupled(new VectorMemoryResponse))) + }) + + val idle :: snapshot :: capture :: execute :: commit :: Nil = Enum(5) + val state = RegInit(idle) + val resultValid = RegInit(false.B) + val result = RegInit(0.U.asTypeOf(new VectorResult)) + val instruction = Reg(UInt(32.W)) + val scalar1 = Reg(UInt(64.W)) + val scalar2 = Reg(UInt(64.W)) + val floatScalar = Reg(UInt(64.W)) + val vl = RegInit(0.U(32.W)) + val vtype = RegInit("h8000000000000000".U(64.W)) + val vstart = RegInit(0.U(64.W)) + val base = Reg(UInt(32.W)) + val readPortCount = math.max(p.laneNumber, p.memoryPorts) + val wordsPerRegister = p.wordsPerRegister + val maskWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) + val operandWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) + val sourceWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) + val destinationWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) + val readPhase = RegInit(0.U(3.W)) + val readSent = RegInit(false.B) + val snapshotSent = RegInit(false.B) + val firstBatch = RegInit(true.B) + val writeSent = RegInit(false.B) + val resumeState = Reg(UInt(3.W)) + val finishRequested = WireDefault(false.B) + val accumulator = Reg(UInt(p.wordBits.W)) + val count = Reg(UInt(32.W)) + val offered = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) + val issued = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) + val received = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) + val memoryData = Reg(Vec(p.memoryPorts, UInt(p.wordBits.W))) + val memoryFailed = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) + val divBatchValid = RegInit(false.B) + val divActive = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) + val divStarted = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) + val divDone = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) + val divData = Reg(Vec(p.laneNumber, UInt(p.wordBits.W))) + val fpStarted = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) + val fpDone = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) + val fpData = Reg(Vec(p.laneNumber, UInt(p.wordBits.W))) + val fpFlags = Reg(Vec(p.laneNumber, UInt(5.W))) + val fpLegal = Reg(Vec(p.laneNumber, Bool())) + + val opcode = instruction(6, 0) + val rd = instruction(11, 7) + val kind = instruction(14, 12) + val rs1 = instruction(19, 15) + val rs2 = instruction(24, 20) + val vm = instruction(25) + val op = instruction(31, 26) + val sew = vtype(4, 3) + val lmul = Cat(vtype(2), vtype(2, 0)).asSInt + val configure = opcode === "h57".U && kind === 7.U + val load = opcode === "h07".U + val store = opcode === "h27".U + val memory = load || store + val fp = opcode === "h57".U && (kind === 1.U || kind === 5.U) + val integer = + opcode === "h57".U && (kind === 0.U || kind === 2.U || kind === 3.U || kind === 4.U || kind === 6.U) + val divide = integer && (kind === 2.U || kind === 6.U) && op >= 32.U && op <= 35.U + val multiply = kind === 2.U || kind === 6.U + val vectorOperand = kind === 0.U || kind === 1.U || kind === 2.U + val maskLogical = integer && kind === 2.U && op >= 24.U && op <= 31.U + + val compare = + (integer && !multiply && (op >= 24.U && op <= 31.U || op === 17.U || op === 19.U)) || + (fp && Seq(24, 25, 27, 28, 29, 31).map(n => op === n.U).reduce(_ || _)) + + val carryOperation = integer && !multiply && op >= 16.U && op <= 19.U + val carryMask = carryOperation && op(0) + val merge = op === 23.U && (fp || integer && !multiply) + val scalarMoveOut = + op === 16.U && rs1 === 0.U && vm && (kind === 2.U || kind === 1.U) + val scalarMoveIn = + op === 16.U && rs2 === 0.U && vm && (kind === 6.U || kind === 5.U) + val population = integer && kind === 2.U && op === 16.U && rs1 === 16.U + val first = integer && kind === 2.U && op === 16.U && rs1 === 17.U + val identity = + integer && kind === 2.U && op === 20.U && rs1 === 17.U && rs2 === 0.U + val intReduction = integer && kind === 2.U && op <= 7.U + val fpReduction = + fp && kind === 1.U && (op === 1.U || op === 3.U || op === 5.U || op === 7.U) + val reduction = intReduction || fpReduction + val wholeMove = integer && kind === 3.U && op === 39.U + val memMode = instruction(27, 26) + + val memWidth = MuxLookup(kind, 0.U(2.W))( + Seq(0.U -> 0.U, 5.U -> 1.U, 6.U -> 2.U, 7.U -> 3.U) + ) + + val wholeMemory = memory && memMode === 0.U && rs2 === 8.U + val indexed = memMode(0) + val memSew = Mux(indexed && !wholeMemory, sew, memWidth) + val intWide = integer && multiply && op >= 48.U + val intNarrow = integer && !multiply && op >= 44.U && op <= 47.U + val floatWide = fp && op === 18.U && rs1 >= 8.U && rs1 <= 15.U + val floatNarrow = fp && op === 18.U && rs1 >= 16.U && rs1 <= 23.U + val extension = integer && multiply && op === 18.U + + val sourceSew = Mux( + memory, + Mux(indexed, memWidth, memSew), + Mux( + intNarrow || floatNarrow || intWide && op >= 52.U && op <= 55.U, + sew +& 1.U, + Mux( + extension, + sew - Mux(rs1(2, 1) === 1.U, 3.U, Mux(rs1(2, 1) === 2.U, 2.U, 1.U)), + sew + ) + ) + ) + + val destinationSew = + Mux(memory, memSew, Mux(intWide || floatWide, sew +& 1.U, sew)) + + val limit = Mux( + wholeMemory, + ((instruction(31, 29) +& 1.U) * p.vLen.U) >> (memWidth +& 3.U), + Mux(wholeMove, ((rs1 +& 1.U) * p.vLen.U) >> (sew +& 3.U), vl) + ) + + def element(word: UInt, index: UInt, width: UInt): UInt = { + val shift = (index << (width.pad(3) + 3.U))(p.wordOffsetBits - 1, 0) + val mask = MuxLookup(width, Fill(p.wordBits, 1.U(1.W)))( + Seq(0.U -> 255.U, 1.U -> 65535.U, 2.U -> "hffffffff".U) + ) + ((word >> shift) & mask)(p.eLen - 1, 0) + } + + def maskElement(word: UInt, index: UInt): Bool = (word >> index(p.wordOffsetBits - 1, 0))(0) + + def wordAddress( + register: UInt, + index: UInt, + width: UInt, + maskBit: Bool = false.B + ): UInt = { + val shift = Mux(maskBit, index, index << (width.pad(3) + 3.U)) + val address = Mux(maskBit, register, register + (shift >> log2Ceil(p.vLen)))(4, 0) + val within = shift.pad(log2Ceil(p.vLen))(log2Ceil(p.vLen) - 1, 0) + address * wordsPerRegister.U + (within >> p.wordOffsetBits) + } + + def groupLegal(register: UInt, width: UInt): Bool = { + val exponent = lmul.pad(6) + width.zext - sew.zext + val registers = 1.U(6.W) << Mux(exponent > 0.S, exponent.asUInt, 0.U) + exponent >= -3.S && exponent <= 3.S && (register & (registers - 1.U)) === 0.U && register +& registers <= 32.U + } + + def overlapLegal( + destination: UInt, + destinationWidth: UInt, + operand: UInt, + operandWidth: UInt + ): Bool = { + val destinationExponent = lmul.pad(6) + destinationWidth.zext - sew.zext + val operandExponent = lmul.pad(6) + operandWidth.zext - sew.zext + val destinationCount = 1 + .U(6.W) << Mux(destinationExponent > 0.S, destinationExponent.asUInt, 0.U) + val operandCount = + 1.U(6.W) << Mux(operandExponent > 0.S, operandExponent.asUInt, 0.U) + val overlap = + destination < operand +& operandCount && operand < destination +& destinationCount + !overlap || destinationWidth === operandWidth || + (destinationWidth < operandWidth && destination === operand) || + (destinationWidth > operandWidth && operandExponent >= 0.S && destination +& destinationCount === operand +& operandCount) + } + + def maximum(width: UInt, multiplier: UInt): UInt = { + val elements = p.vLen.U(32.W) >> (width.pad(3) + 3.U) + Mux(multiplier(2), elements >> (8.U - multiplier), elements << multiplier) + } + + val vlmax = maximum(sew, vtype(2, 0)) + val registers: Instance[Operands] = Instantiate(new Operands(gpParam)) + registers.io.initialize := io.initialize + val writes = Wire(chiselTypeOf(registers.io.write.bits)) + val pendingWrites = Reg(chiselTypeOf(registers.io.write.bits)) + for (lane <- 0 until p.laneNumber) { + writes(lane) := 0.U.asTypeOf(writes(lane)) + } + registers.io.write.valid := state === commit && !writeSent && !io.initialize + registers.io.write.bits := pendingWrites + registers.io.writeDone.ready := state === commit && writeSent && !io.initialize + when(registers.io.write.fire)(writeSent := true.B) + when(registers.io.writeDone.fire) { + writeSent := false.B + state := resumeState + } + + def writeElement( + lane: Int, + index: UInt, + width: UInt, + data: UInt, + maskBit: Bool = false.B + ): Unit = { + val bitOffset = Mux(maskBit, index, index << (width.pad(3) + 3.U)) + val within = bitOffset.pad(log2Ceil(p.vLen))(log2Ceil(p.vLen) - 1, 0) + val mask = Mux( + maskBit, + 1.U(p.wordBits.W), + MuxLookup(width, Fill(p.wordBits, 1.U(1.W)))( + Seq(0.U -> 255.U, 1.U -> 65535.U, 2.U -> "hffffffff".U) + ) + ) + writes(lane).valid := true.B + writes(lane).address := wordAddress(rd, index, width, maskBit) + writes(lane).mask := (mask << within(p.wordOffsetBits - 1, 0))(p.wordBits - 1, 0) + writes(lane).data := (data << within(p.wordOffsetBits - 1, 0))(p.wordBits - 1, 0) + } + + def finish( + fault: Bool = false.B, + cause: UInt = 0.U, + value: UInt = 0.U + ): Unit = { + finishRequested := true.B + state := idle + resultValid := true.B + result.fault := fault + result.cause := cause + result.tval := value + when(!fault)(vstart := 0.U) + } + + def nextBatch(step: UInt): Unit = { + when(base + step >= limit)(finish()).otherwise { + base := base + step + state := capture + readPhase := 0.U + } + offered.foreach(_ := false.B) + issued.foreach(_ := false.B) + received.foreach(_ := false.B) + memoryFailed.foreach(_ := false.B) + divBatchValid := false.B + divStarted.foreach(_ := false.B) + divDone.foreach(_ := false.B) + fpStarted.foreach(_ := false.B) + fpDone.foreach(_ := false.B) + } + + io.vl := vl + io.vtype := vtype + io.vstart := vstart + io.issue.ready := state === idle && !resultValid && !registers.io.busy && !io.initialize + io.result.valid := resultValid && state === idle && !registers.io.busy + io.result.bits := result + when(io.result.fire)(resultValid := false.B) + when(io.vstartWrite.valid)(vstart := io.vstartWrite.bits) + when(io.issue.fire) { + instruction := io.issue.bits.instruction + scalar1 := io.issue.bits.scalar1 + scalar2 := io.issue.bits.scalar2 + floatScalar := io.issue.bits.floating + state := snapshot + readPhase := 0.U + readSent := false.B + snapshotSent := false.B + firstBatch := true.B + writeSent := false.B + result := 0.U.asTypeOf(new VectorResult) + base := vstart + count := 0.U + offered.foreach(_ := false.B) + issued.foreach(_ := false.B) + received.foreach(_ := false.B) + memoryFailed.foreach(_ := false.B) + divBatchValid := false.B + divStarted.foreach(_ := false.B) + divDone.foreach(_ := false.B) + fpStarted.foreach(_ := false.B) + fpDone.foreach(_ := false.B) + } + + val instructionLegal = Wire(Bool()) + val alu: Seq[Instance[IALU]] = Seq.fill(p.laneNumber)(Instantiate(new IALU(gpParam))) + val falu: Seq[Instance[FALU]] = Seq.fill(p.laneNumber)(Instantiate(new FALU(gpParam))) + val laneA = Wire(Vec(p.laneNumber, UInt(p.wordBits.W))) + val laneB = Wire(Vec(p.laneNumber, UInt(p.wordBits.W))) + val laneSelected = Wire(Vec(p.laneNumber, Bool())) + val laneValue = Wire(Vec(p.laneNumber, UInt(p.wordBits.W))) + val laneFlags = Wire(Vec(p.laneNumber, UInt(5.W))) + val laneLegal = Wire(Vec(p.laneNumber, Bool())) + val fpComplete = Wire(Vec(p.laneNumber, Bool())) + val gather = + integer && !multiply && (op === 12.U || op === 14.U && kind === 0.U) + val slide = + integer && !multiply && (op === 14.U || op === 15.U) && kind =/= 0.U + val slideOne = integer && multiply && (op === 14.U || op === 15.U) + + val operandReads = (0 until readPortCount).map { port => + val index = base + port.U + val scalar = Mux(kind === 3.U, Cat(Fill(59, rs1(4)), rs1), scalar1) + val floatingScalar = Mux(sew === 2.U && !floatScalar(63, 32).andR, "h7fc00000".U, floatScalar) + val b = + Mux( + vectorOperand, + element(operandWords(port), index, Mux(gather && op === 14.U, 1.U, sew)), + Mux(fp, floatingScalar, scalar) + ) + val slideOffset = Mux(slideOne, 1.U, Mux(kind === 3.U, rs1, scalar1)) + val sourceIndex = Mux(gather && !slide, b, Mux(op === 14.U, index - slideOffset, index + slideOffset)) + val permuteRead = gather || slide || slideOne + // element already keeps only the low group bit offset; upper index bits cannot affect its read. + val a = element( + sourceWords(port), + Mux(scalarMoveOut, 0.U, Mux(permuteRead, sourceIndex(31, 0), index)), + Mux(permuteRead, sew, sourceSew) + ) + val c = element(destinationWords(port), index, Mux(fp, sew, destinationSew)) + (a, b, c, sourceIndex) + } + + val permuteRead = gather || slide || slideOne + registers.io.snapshot.valid := state === snapshot && permuteRead && !snapshotSent && !io.initialize + registers.io.snapshot.bits := rs2 + registers.io.snapshotDone.ready := state === snapshot && snapshotSent && !io.initialize + when(registers.io.snapshot.fire)(snapshotSent := true.B) + when(state === snapshot && !permuteRead || registers.io.snapshotDone.fire) { + state := Mux(configure, execute, capture) + readPhase := 0.U + readSent := false.B + } + + registers.io.read.valid := state === capture && !readSent && !io.initialize + registers.io.readResult.ready := state === capture && readSent && !io.initialize + for (port <- 0 until readPortCount) { + val index = base + port.U + val maskSource = maskLogical || population || first + val sourceIndex = Mux(scalarMoveOut, 0.U, Mux(permuteRead, operandReads(port)._4(31, 0), index)) + val sourceWidth = Mux(scalarMoveOut || permuteRead, sew, sourceSew) + val shift = sourceIndex << (sourceWidth.pad(3) + 3.U) + registers.io.read.bits(port).address := MuxLookup(readPhase, wordAddress(rs1, 0.U, sew))( + Seq( + 0.U -> wordAddress(0.U, index, sew, true.B), + 1.U -> wordAddress(rs1, index, Mux(gather && op === 14.U, 1.U, sew), maskLogical), + 2.U -> Mux(permuteRead, shift >> p.wordOffsetBits, wordAddress(rs2, sourceIndex, sourceWidth, maskSource)), + 3.U -> wordAddress(rd, index, Mux(fp, sew, destinationSew)) + ) + ) + registers.io.read.bits(port).snapshot := readPhase === 2.U && permuteRead + } + when(registers.io.read.fire)(readSent := true.B) + when(registers.io.readResult.fire) { + readSent := false.B + switch(readPhase) { + is(0.U)(maskWords := registers.io.readResult.bits) + is(1.U)(operandWords := registers.io.readResult.bits) + is(2.U)(sourceWords := registers.io.readResult.bits) + is(3.U)(destinationWords := registers.io.readResult.bits) + is(4.U) { + accumulator := registers.io.readResult.bits(0) & MuxLookup(sew, Fill(p.eLen, 1.U(1.W)))( + Seq(0.U -> 255.U, 1.U -> 65535.U, 2.U -> "hffffffff".U) + ) + firstBatch := false.B + } + } + when(readPhase === 4.U || readPhase === 3.U && !firstBatch) { + state := execute + }.otherwise(readPhase := readPhase + 1.U) + } + + for (lane <- 0 until p.laneNumber) { + val index = base + lane.U + val selected = maskElement(maskWords(lane), index) + val scalar = Mux( + kind === 3.U, + Cat(Fill(59, rs1(4)), rs1), + scalar1 + ) + val floatingScalar = + Mux(sew === 2.U && !floatScalar(63, 32).andR, "h7fc00000".U, floatScalar) + val b = operandReads(lane)._2 + val slideOffset = Mux(slideOne, 1.U, Mux(kind === 3.U, rs1, scalar1)) + val up = op === 14.U + val sourceIndex = operandReads(lane)._4 + val a = operandReads(lane)._1 + laneA(lane) := a + laneB(lane) := b + laneSelected( + lane + ) := index < limit && (wholeMove || vm || selected || merge || carryOperation) && + (!slide || !up || index >= slideOffset) + alu(lane).io.divRequest.valid := state === execute && instructionLegal && divide && + divBatchValid && divActive(lane) && !divStarted(lane) && !io.initialize + alu(lane).io.divRequest.bits.a := a + alu(lane).io.divRequest.bits.b := b + alu(lane).io.divRequest.bits.sew := sew + alu(lane).io.divRequest.bits.signed := op(0) + alu(lane).io.divRequest.bits.remainder := op(1) + alu(lane).io.divResponse.ready := state === execute && divide && divBatchValid && + divStarted(lane) && !divDone(lane) && !io.initialize + alu(lane).io.clear := io.initialize + when(alu(lane).io.divRequest.fire)(divStarted(lane) := true.B) + when(alu(lane).io.divResponse.fire) { + divData(lane) := alu(lane).io.divResponse.bits + divDone(lane) := true.B + } + alu(lane).io.a := a + alu(lane).io.b := b + alu(lane).io.c := operandReads(lane)._3 + alu(lane).io.carry := (selected && (!carryMask || !vm)) || merge && vm + alu(lane).io.vxrm := io.vxrm + alu(lane).io.selector := rs1 + alu(lane).io.sew := sew + alu(lane).io.funct6 := op + alu(lane).io.multiplyClass := multiply + falu(lane).io.a := a + falu(lane).io.b := Mux(fpReduction, accumulator, b) + falu(lane).io.c := operandReads(lane)._3 + falu(lane).io.sew := sew + falu(lane).io.op := Mux( + fpReduction, + Mux(op === 5.U, 4.U, Mux(op === 7.U, 6.U, 0.U)), + op + ) + falu(lane).io.subop := rs1 + falu(lane).io.roundingMode := io.roundingMode + val fpRequired = + laneSelected(lane) && !merge && (!fpReduction || (lane == 0).B) + falu( + lane + ).io.start := state === execute && instructionLegal && fp && fpRequired && !fpStarted( + lane + ) && falu( + lane + ).io.ready + fpComplete(lane) := !fpRequired || fpDone(lane) || falu(lane).io.valid + laneFlags(lane) := Mux( + merge, + 0.U, + Mux(fpDone(lane), fpFlags(lane), falu(lane).io.flags) + ) + laneLegal(lane) := merge || Mux( + fpDone(lane), + fpLegal(lane), + falu(lane).io.legal + ) + val floatingValue = Mux(fpDone(lane), fpData(lane), falu(lane).io.result) + val gathered = Mux(sourceIndex < vlmax, a, 0.U) + val slid = Mux( + slideOne && Mux(up, index === 0.U, index === vl - 1.U), + scalar, + Mux(!up && sourceIndex >= vlmax, 0.U, a) + ) + val maskA = maskElement(sourceWords(lane), index) + val maskB = maskElement(operandWords(lane), index) + val maskResult = MuxLookup(op, false.B)( + Seq( + 24.U -> (maskA && !maskB), + 25.U -> (maskA && maskB), + 26.U -> (maskA || maskB), + 27.U -> (maskA ^ maskB), + 28.U -> (maskA || !maskB), + 29.U -> !(maskA && maskB), + 30.U -> !(maskA || maskB), + 31.U -> !(maskA ^ maskB) + ) + ) + laneValue(lane) := Mux( + fp, + Mux(merge, Mux(vm || selected, b, a), floatingValue), + Mux( + wholeMove, + a, + Mux( + maskLogical, + maskResult, + Mux( + identity, + index, + Mux( + slide || slideOne, + slid, + Mux(gather, gathered, alu(lane).io.result) + ) + ) + ) + ) + ) + when(falu(lane).io.start)(fpStarted(lane) := true.B) + when(state === execute && falu(lane).io.valid) { + fpDone(lane) := true.B + fpData(lane) := falu(lane).io.result + fpFlags(lane) := falu(lane).io.flags + fpLegal(lane) := falu(lane).io.legal + } + } + + val destinationGroupLegal = + compare || maskLogical || population || first || reduction || scalarMoveOut || scalarMoveIn || + groupLegal(rd, destinationSew) + + val sourceGroupLegal = + maskLogical || population || first || scalarMoveIn || scalarMoveOut || groupLegal( + rs2, + sourceSew + ) + + val operandGroupLegal = + !vectorOperand || op === 18.U || reduction || scalarMoveOut || maskLogical || population || first || + groupLegal(rs1, sew) + + val aluLegal = alu + .map(_.io.legal) + .reduce(_ && _) || maskLogical || identity || gather || slide || slideOne || + scalarMoveOut || scalarMoveIn || population || first || intReduction || wholeMove + + val fpOperationLegal = Seq(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16, 18, 19, 23, 24, 25, 27, 28, 29, 31, 32, 33, 36, 39, + 40, 41, 42, 43, 44, 45, 46, 47) + .map(n => op === n.U) + .reduce(_ || _) + + val ordinaryLegal = + !vtype( + 63 + ) && sourceSew <= p.maxSew.U && destinationSew <= p.maxSew.U && destinationGroupLegal && sourceGroupLegal && operandGroupLegal && + (!(intWide || floatWide || intNarrow || floatNarrow || extension) || + overlapLegal(rd, destinationSew, rs2, sourceSew) && + (!vectorOperand || op === 18.U || overlapLegal( + rd, + destinationSew, + rs1, + sew + ))) && + (vm || rd =/= 0.U || compare || maskLogical || population || first || reduction) && + (!merge || !vm || rs2 === 0.U) && (!maskLogical || vm) && + (!carryOperation || carryMask || !vm) && + (!(reduction || population || first) || vstart === 0.U) && + (!fp || sew >= 2.U && fpOperationLegal && io.roundingMode <= 4.U && + (scalarMoveOut || scalarMoveIn || merge || falu(0).io.legal) && + (op =/= 16.U || scalarMoveOut || scalarMoveIn) && + (op =/= 18.U && op =/= 19.U || kind === 1.U) && (!merge || kind === 5.U)) && + (!integer || aluLegal) + + val wholeMoveLegal = vm && vstart === 0.U && Seq(0, 1, 3, 7) + .map(n => rs1 === n.U) + .reduce(_ || _) && + (rd & rs1) === 0.U && (rs2 & rs1) === 0.U && rd +& rs1 < 32.U && rs2 +& rs1 < 32.U + + val memoryLegal = memWidth <= p.maxSew.U && memSew <= p.maxSew.U && Seq( + 0, + 5, + 6, + 7 + ).map(n => kind === n.U).reduce(_ || _) && !instruction(28) && + Mux( + wholeMemory, + vm && Seq(0, 1, 3, 7) + .map(n => instruction(31, 29) === n.U) + .reduce(_ || _) && + (rd & instruction(31, 29)) === 0.U && rd +& instruction( + 31, + 29 + ) < 32.U && (!store || memWidth === 0.U), + !vtype(63) && instruction( + 31, + 29 + ) === 0.U && (memMode =/= 0.U || rs2 === 0.U) && + groupLegal(rd, memSew) && (!indexed || groupLegal( + rs2, + memWidth + )) && (vm || store || rd =/= 0.U) + ) + + val legal = Mux( + memory, + memoryLegal, + Mux(wholeMove, wholeMoveLegal, (integer || fp) && ordinaryLegal) + ) + + instructionLegal := legal + + val required = Wire(Vec(p.memoryPorts, Bool())) + val addresses = Wire(Vec(p.memoryPorts, UInt(64.W))) + val canceled = Wire(Vec(p.memoryPorts, Bool())) + for (port <- 0 until p.memoryPorts) { + val index = base + port.U + required( + port + ) := (port < p.laneNumber).B && index < limit && (wholeMemory || vm || maskElement( + maskWords(port), + index + )) + addresses(port) := scalar1 + Mux( + indexed && !wholeMemory, + operandReads(port)._1, + Mux(memMode === 2.U, index * scalar2, index << memSew) + ) + val earlierFailure = (0 until port) + .map { prior => + required(prior) && (memoryFailed(prior) || io + .memoryResponse(prior) + .fire && io.memoryResponse(prior).bits.error) + } + .foldLeft(false.B)(_ || _) + val earlierPending = (0 until port) + .map { prior => + val distance = addresses(port) - addresses(prior) + val reverseDistance = addresses(prior) - addresses(port) + val collision = + store && (distance < (1.U << memSew) || reverseDistance < (1.U << memSew)) + required(prior) && (memMode === 3.U || collision) && !received(prior) + } + .foldLeft(false.B)(_ || _) + canceled(port) := !issued(port) && !offered(port) && earlierFailure + io.memoryRequest(port) + .valid := state === execute && memory && legal && required( + port + ) && !issued( + port + ) && !earlierPending && !canceled(port) + io.memoryRequest(port).bits.address := addresses(port) + io.memoryRequest(port).bits.write := store + io.memoryRequest(port).bits.data := operandReads(port)._3 + io.memoryRequest(port).bits.mask := MuxLookup(memSew, 255.U(8.W))( + Seq(0.U -> 1.U, 1.U -> 3.U, 2.U -> 15.U) + ) + io.memoryRequest(port).bits.size := memSew + io.memoryResponse(port).ready := state === execute && memory && issued( + port + ) && !received(port) + when(io.memoryRequest(port).valid)(offered(port) := true.B) + when(io.memoryRequest(port).fire)(issued(port) := true.B) + when(io.memoryResponse(port).fire) { + received(port) := true.B + memoryData(port) := io.memoryResponse(port).bits.data + memoryFailed(port) := io.memoryResponse(port).bits.error + } + } + + val memoryComplete = (0 until p.memoryPorts) + .map(i => !required(i) || canceled(i) || received(i) || io.memoryResponse(i).fire) + .reduce(_ && _) + + val failures = VecInit( + (0 until p.memoryPorts).map(i => + required(i) && (memoryFailed(i) || io + .memoryResponse(i) + .fire && io.memoryResponse(i).bits.error) + ) + ) + + val failureIndex = PriorityEncoder(failures.asUInt) + + when(state === execute) { + when(configure) { + val immediate = instruction(31, 30) === 3.U + val registerType = instruction(31, 25) === 64.U + val requested = Mux( + registerType, + scalar2, + Mux(immediate, instruction(29, 20), instruction(30, 20)) + ) + val requestedSew = requested(5, 3) + val requestedLmul = requested(2, 0) + val typeLegal = + requested(63, 8) === 0.U && requestedSew <= p.maxSew.U && requestedLmul =/= 4.U && + (!requestedLmul( + 2 + ) || (8.U << requestedSew) <= (p.eLen.U >> (8.U - requestedLmul))) + val newMaximum = maximum(requestedSew, requestedLmul) + val keepLength = !immediate && rs1 === 0.U && rd === 0.U + when( + !(immediate || registerType || !instruction( + 31 + )) || typeLegal && keepLength && newMaximum =/= vlmax + ) { + finish(true.B, 2.U, instruction) + }.otherwise { + val avl = Mux( + immediate, + rs1, + Mux(rs1 =/= 0.U, scalar1, Mux(rd =/= 0.U, newMaximum, vl)) + ) + val length = Mux(typeLegal, Mux(avl < newMaximum, avl, newMaximum), 0.U) + vl := length + vtype := Mux(typeLegal, requested, "h8000000000000000".U) + result.scalarWrite := rd =/= 0.U + result.scalarData := length + finish() + } + }.elsewhen(!legal) { + finish(true.B, 2.U, instruction) + }.elsewhen(scalarMoveOut) { + val value = element(sourceWords(0), 0.U, sew) + when(fp) { + result.floatWrite := true.B + result.floatData := Mux( + sew === 2.U, + Cat("hffffffff".U(32.W), value(31, 0)), + value + ) + }.otherwise { + result.scalarWrite := rd =/= 0.U + result.scalarData := MuxLookup(sew, value)( + Seq( + 0.U -> Cat(Fill(56, value(7)), value(7, 0)), + 1.U -> Cat(Fill(48, value(15)), value(15, 0)), + 2.U -> Cat(Fill(32, value(31)), value(31, 0)) + ) + ) + } + finish() + }.elsewhen(scalarMoveIn) { + when(vl =/= 0.U && vstart < vl) { + val scalar = Mux( + sew === 2.U && !floatScalar(63, 32).andR, + "h7fc00000".U, + floatScalar + ) + writeElement( + 0, + 0.U, + sew, + Mux(fp, scalar, scalar1) + ) + } + finish() + }.elsewhen(base >= limit) { + when(population || first) { + result.scalarWrite := rd =/= 0.U + result.scalarData := Mux(first, "hffffffffffffffff".U, count) + } + finish() + }.elsewhen(memory) { + when(memoryComplete) { + for (lane <- 0 until p.laneNumber) { + when( + load && required( + lane + ) && (!failures.asUInt.orR || lane.U < failureIndex) + ) { + writeElement( + lane, + base + lane.U, + memSew, + Mux( + io.memoryResponse(lane).fire, + io.memoryResponse(lane).bits.data, + memoryData(lane) + ) + ) + } + } + when(failures.asUInt.orR) { + vstart := base + failureIndex + finish(true.B, Mux(load, 5.U, 7.U), addresses(failureIndex)) + }.otherwise(nextBatch(p.laneNumber.U)) + } + }.elsewhen(population || first) { + val selected = VecInit((0 until p.laneNumber).map { lane => + val index = base + lane.U + index < vl && maskElement(sourceWords(lane), index) && + (vm || maskElement(maskWords(lane), index)) + }) + val total = count + PopCount(selected) + count := total + when(first && selected.asUInt.orR || base + p.laneNumber.U >= vl) { + result.scalarWrite := rd =/= 0.U + result.scalarData := Mux( + first, + Mux( + selected.asUInt.orR, + base + PriorityEncoder(selected.asUInt), + "hffffffff".U + ), + total + ) + finish() + }.otherwise { + base := base + p.laneNumber.U + readPhase := 0.U + state := capture + } + }.elsewhen(intReduction) { + // All integer reductions are associative at the destination width. Keep + // the original leaf order and ignore masked lanes instead of serializing them. + def combine(left: (Bool, UInt), right: (Bool, UInt)): (Bool, UInt) = { + val (leftValid, a) = left + val (rightValid, b) = right + val width = 8.U(7.W) << sew + val sa = (a << (p.eLen.U - width))(p.eLen - 1, 0).asSInt + val sb = (b << (p.eLen.U - width))(p.eLen - 1, 0).asSInt + val value = MuxLookup(op, a + b)(Seq( + 1.U -> (a & b), + 2.U -> (a | b), + 3.U -> (a ^ b), + 4.U -> Mux(b < a, b, a), + 5.U -> Mux(sb < sa, b, a), + 6.U -> Mux(b > a, b, a), + 7.U -> Mux(sb > sa, b, a) + )) + (leftValid || rightValid, Mux(!leftValid, b, Mux(!rightValid, a, value))) + } + def tree(values: Seq[(Bool, UInt)]): (Bool, UInt) = { + if (values.size == 1) values.head + else { + val (left, right) = values.splitAt(values.size / 2) + combine(tree(left), tree(right)) + } + } + val reduced = tree(Seq(true.B -> accumulator) ++ + (0 until p.laneNumber).map(lane => laneSelected(lane) -> laneA(lane)))._2 + accumulator := reduced + when(base + p.laneNumber.U >= vl)(writeElement(0, 0.U, sew, reduced)) + nextBatch(p.laneNumber.U) + }.elsewhen(fpReduction) { + when(fpComplete(0)) { + when(laneSelected(0) && !laneLegal(0))(finish(true.B, 2.U, instruction)) + .otherwise { + val value = Mux(laneSelected(0), laneValue(0), accumulator) + accumulator := value + when(laneSelected(0))(result.flags := result.flags | laneFlags(0)) + when(base + 1.U >= vl)(writeElement(0, 0.U, sew, value)) + nextBatch(1.U) + } + } + }.elsewhen(divide) { + when(!io.initialize) { + when(!divBatchValid) { + divActive := laneSelected + divBatchValid := true.B + }.elsewhen((0 until p.laneNumber).map(i => !divActive(i) || divDone(i)).reduce(_ && _)) { + for (lane <- 0 until p.laneNumber) { + when(divActive(lane)) { + writeElement(lane, base + lane.U, destinationSew, divData(lane)) + } + } + nextBatch(p.laneNumber.U) + } + } + }.elsewhen(!fp || fpComplete.reduce(_ && _)) { + val floatingLegal = (0 until p.laneNumber) + .map(i => !laneSelected(i) || laneLegal(i)) + .reduce(_ && _) + when(fp && !floatingLegal)(finish(true.B, 2.U, instruction)).otherwise { + for (lane <- 0 until p.laneNumber) { + when(laneSelected(lane)) { + writeElement( + lane, + base + lane.U, + destinationSew, + laneValue(lane), + compare || maskLogical + ) + } + } + when(fp) { + result.flags := result.flags | (0 until p.laneNumber) + .map(i => Mux(laneSelected(i), laneFlags(i), 0.U)) + .reduce(_ | _) + } + when(integer && !wholeMove) { + result.saturated := result.saturated || (0 until p.laneNumber) + .map(i => laneSelected(i) && alu(i).io.saturated) + .reduce(_ || _) + } + nextBatch(p.laneNumber.U) + } + } + } + when(state === execute && writes.map(_.valid).reduce(_ || _) && !io.initialize) { + pendingWrites := writes + resumeState := Mux(finishRequested, idle, capture) + state := commit + writeSent := false.B + } + when(io.initialize) { + state := idle + readSent := false.B + snapshotSent := false.B + writeSent := false.B + firstBatch := true.B + readPhase := 0.U + resultValid := false.B + vl := 0.U + vtype := "h8000000000000000".U + vstart := 0.U + divBatchValid := false.B + divStarted.foreach(_ := false.B) + divDone.foreach(_ := false.B) + fpStarted.foreach(_ := false.B) + fpDone.foreach(_ := false.B) + offered.foreach(_ := false.B) + issued.foreach(_ := false.B) + received.foreach(_ := false.B) + } +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/build.sc b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/build.sc new file mode 100644 index 000000000..6e754ceb7 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.gpdomain.loader.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/ImageLoader.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Loader.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/Emit.scala new file mode 100644 index 000000000..ceebc616d --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/Emit.scala @@ -0,0 +1,26 @@ +package framework.acc.gpdomain.loader + +import _root_.configs.GlobalConfig + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Loader(thread.acc.get.frontend, thread.acc.get.memDomain, thread.acc.get.gpDomain), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/frontend/ImageLoader.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/ImageLoader.scala similarity index 88% rename from arch/src/main/scala/framework/rvv/src/main/scala/frontend/ImageLoader.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/ImageLoader.scala index af49ca7f8..18ceb6bbc 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/frontend/ImageLoader.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/ImageLoader.scala @@ -1,13 +1,29 @@ -package framework.rvv +package framework.acc.gpdomain.loader +import framework.acc.gpdomain.configs.GpDomainParam + +import framework.acc.gpdomain.execution.{ProgramWrite, VectorMemoryRequest, VectorMemoryResponse} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig + +class ImageLoad extends Bundle { + val buffer = Bool() + val bytes = UInt(32.W) +} + +class ImageResult extends Bundle { + val fault = Bool() + val cause = UInt(32.W) + val tval = UInt(32.W) + val entry = UInt(32.W) + val textBytes = UInt(32.W) + val constBytes = UInt(32.W) +} @instantiable -class ImageLoader(val b: GlobalConfig) extends Module { - val p = b.rvv +class ImageLoader(val gpParam: GpDomainParam) extends Module { + val p = gpParam.loader @public val io = IO(new Bundle { diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/Loader.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/Loader.scala new file mode 100644 index 000000000..b7e768e55 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/Loader.scala @@ -0,0 +1,237 @@ +package framework.acc.gpdomain.loader + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.acc.frontend.scheduler.{Complete, Issue} +import framework.acc.memdomain.toddr.DmaStatus +import framework.acc.gpdomain.execution.{ + KernelCompletion, + KernelLaunch, + ProgramWrite, + VectorMemoryRequest, + VectorMemoryResponse +} +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} + +class KernelContext(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val instId = UInt(64.W) + val rob_id = UInt(log2Ceil(frontendParam.rob_entries).W) + val readBank = UInt(frontendParam.bank_id_len.W) + val writeBank = UInt(frontendParam.bank_id_len.W) + val readGroups = UInt(memParam.groupCountWidth.W) + val writeGroups = UInt(memParam.groupCountWidth.W) +} + +class LoaderMemoryPort(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val request = Decoupled(new VectorMemoryRequest) + val response = Flipped(Decoupled(new VectorMemoryResponse)) + val control = Output(Bool()) + val buffer = Output(Bool()) + val loading = Output(Bool()) + val constBytes = Output(UInt(32.W)) + val context = Output(new KernelContext(frontendParam, memParam)) +} + +@instantiable +class Loader(val frontendParam: FrontendParam, val memParam: MemDomainParam, val gpParam: GpDomainParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + require(gpParam.enable, "GP domain must be explicitly enabled") + + @public + val io = IO(new Bundle { + val running = Output(Bool()) + val cmdReq = Flipped(Decoupled(new Issue(frontendParam, memParam))) + val cmdResp = Decoupled(new Complete(frontendParam)) + val uploadStart = Flipped(Decoupled(new ImageLoad)) + val uploadComplete = Decoupled(new KernelCompletion) + val release = Flipped(Valid(Bool())) + val image = Flipped(Decoupled(UInt(32.W))) + val imageTerminal = Flipped(Decoupled(new DmaStatus)) + val program = Decoupled(new ProgramWrite) + val launch = Decoupled(new KernelLaunch(frontendParam, memParam)) + val done = Flipped(Decoupled(new KernelCompletion)) + val memory = new LoaderMemoryPort(frontendParam, memParam) + }) + + val request = io.cmdReq + val uploading = RegInit(false.B) + + val parser: Instance[ImageLoader] = Instantiate(new ImageLoader(gpParam)) + val idle :: loading :: descriptorSend :: descriptorWait :: launch :: executing :: completed :: Nil = Enum(7) + + val state = RegInit(idle) + val command = Reg(new Issue(frontendParam, memParam)) + val validImage = RegInit(VecInit(Seq.fill(2)(false.B))) + val imageEntry = Reg(Vec(2, UInt(32.W))) + val imageBytes = Reg(Vec(2, UInt(32.W))) + val imageConstants = Reg(Vec(2, UInt(32.W))) + val buffer = Reg(Bool()) + val descriptor = Reg(Vec(12, UInt(64.W))) + val word = Reg(UInt(4.W)) + val terminalSeen = RegInit(false.B) + val terminalFailed = RegInit(false.B) + val terminalCancelled = RegInit(false.B) + val result = RegInit(0.U.asTypeOf(new KernelCompletion)) + + request.ready := state === idle && !io.uploadStart.valid + io.uploadStart.ready := state === idle + val completedImage = state === completed && (!uploading || result.fault || terminalSeen) + io.cmdResp.valid := completedImage && !uploading + io.uploadComplete.valid := completedImage && uploading + io.uploadComplete.bits := result + io.cmdResp.bits.rob_id := command.rob_id + when(io.cmdResp.valid)(assert(!result.fault, "GP task violated its execution contract")) + io.running := state =/= idle + io.memory.context.instId := command.instId + io.memory.context.rob_id := command.rob_id + io.memory.context.readBank := command.cmd.cmd.rs1Data(9, 0) + io.memory.context.writeBank := command.cmd.cmd.rs1Data(29, 20) + io.memory.context.readGroups := command.cmd.op1_col + io.memory.context.writeGroups := command.cmd.wr_col + io.memory.control := state === loading || state === descriptorSend || state === descriptorWait + io.memory.buffer := buffer + io.memory.loading := state === loading + io.memory.constBytes := imageConstants(buffer) + + parser.io.start.valid := io.uploadStart.fire + parser.io.start.bits := io.uploadStart.bits + parser.io.image <> io.image + io.imageTerminal.ready := state =/= idle && uploading && !terminalSeen + val terminalError = io.imageTerminal.fire && io.imageTerminal.bits.error =/= 0.U + parser.io.abort := terminalError + when(io.imageTerminal.fire) { + terminalSeen := true.B + terminalFailed := io.imageTerminal.bits.error =/= 0.U + terminalCancelled := io.imageTerminal.bits.error === 10.U + when(terminalError) { + validImage(buffer) := false.B + when(!result.fault) { + result.fault := true.B + result.cause := 5.U + result.tval := io.imageTerminal.bits.address + } + } + } + parser.io.done.ready := state === loading + io.program <> parser.io.program + io.launch.valid := state === launch + io.launch.bits.context := io.memory.context + io.launch.bits.iBuffer := buffer + io.launch.bits.entry := descriptor(0)(31, 0) + io.launch.bits.end := descriptor(1)(31, 0) + io.launch.bits.stack := descriptor(2) + for (i <- 0 until 8) (io.launch.bits.args(i) := descriptor(i + 3)) + io.done.ready := state === executing + + io.memory.request.valid := state === loading && parser.io.memoryRequest.valid + io.memory.request.bits := parser.io.memoryRequest.bits + parser.io.memoryRequest.ready := state === loading && io.memory.request.ready + parser.io.memoryResponse.valid := state === loading && io.memory.response.valid + parser.io.memoryResponse.bits := io.memory.response.bits + io.memory.response.ready := state === loading && parser.io.memoryResponse.ready + when(state === descriptorSend) { + io.memory.request.valid := true.B + io.memory.request.bits.address := (command.cmd.op1_col.pad(64) << 32) + command.cmd.cmd.rs2Data + word * 8.U + io.memory.request.bits.write := false.B + io.memory.request.bits.data := 0.U + io.memory.request.bits.mask := 255.U + io.memory.request.bits.size := 3.U + } + when(state === descriptorWait)(io.memory.response.ready := true.B) + + when(io.release.valid)(validImage(io.release.bits) := false.B) + when(io.uploadStart.fire) { + uploading := true.B + command := 0.U.asTypeOf(command) + buffer := io.uploadStart.bits.buffer + validImage(io.uploadStart.bits.buffer) := false.B + result := 0.U.asTypeOf(result) + terminalSeen := false.B + terminalFailed := false.B + terminalCancelled := false.B + state := loading + } + when(request.fire) { + assert(request.bits.cmd.cmd.raw_inst(31, 25) === 79.U, "Loader launch requires RUN_KERNEL") + uploading := false.B + command := request.bits + result := 0.U.asTypeOf(result) + terminalSeen := false.B + terminalFailed := false.B + terminalCancelled := false.B + val program = request.bits.cmd.cmd.rs1Data(19, 10) + val programValid = + program >= memParam.virtualBankCount.U && program < (memParam.virtualBankCount + 2).U + val slot = program - memParam.virtualBankCount.U + buffer := slot(0) + word := 0.U + state := descriptorSend + when( + !programValid || !validImage(slot(0)) || request.bits.cmd.op1_col === 0.U || + request.bits.cmd.wr_col === 0.U || request.bits.cmd.cmd.rs1Data(63, 30).orR || + request.bits.cmd.cmd.rs1Data(9, 0) === request.bits.cmd.cmd.rs1Data(29, 20) || + request.bits.cmd.cmd.rs2Data(63, 32).orR || + (request.bits.cmd.cmd.rs2Data( + 31, + 0 + ) +& 96.U) > (memParam.bankEntries * (memParam.bankWidth / 8)).U + ) { + result.fault := true.B + result.cause := 1.U + result.tval := request.bits.cmd.cmd.rs1Data + state := completed + } + } + when(parser.io.done.fire) { + val nativeFault = parser.io.done.bits.fault && parser.io.done.bits.cause =/= 5.U + when((!terminalFailed && !terminalError) || + (nativeFault && (terminalCancelled || (io.imageTerminal.fire && io.imageTerminal.bits.error === 10.U)))) { + result.fault := parser.io.done.bits.fault + result.cause := parser.io.done.bits.cause + result.tval := parser.io.done.bits.tval + } + imageEntry(buffer) := parser.io.done.bits.entry + imageBytes(buffer) := parser.io.done.bits.textBytes + imageConstants(buffer) := parser.io.done.bits.constBytes + state := completed + } + when(state === descriptorSend && io.memory.request.fire)(state := descriptorWait) + when(state === descriptorWait && io.memory.response.fire) { + when(io.memory.response.bits.error) { + result.fault := true.B + result.cause := 5.U + result.tval := (command.cmd.op1_col.pad(64) << 32) + command.cmd.cmd.rs2Data + word * 8.U + state := completed + }.otherwise { + descriptor(word) := io.memory.response.bits.data + word := word + 1.U + state := Mux(word === 11.U, launch, descriptorSend) + } + } + when(state === launch) { + when(descriptor(0) =/= imageEntry(buffer) || descriptor(1) =/= imageBytes(buffer) || descriptor( + 11 + ) =/= 0.U || descriptor(2) =/= "h40002000".U) { + io.launch.valid := false.B + result.fault := true.B + result.cause := 1.U + result.tval := descriptor(0) + state := completed + }.elsewhen(io.launch.fire)(state := executing) + } + when(io.done.fire) { + result := io.done.bits + state := completed + } + when(io.cmdResp.fire || io.uploadComplete.fire) { + when(uploading && !result.fault && terminalSeen && !terminalFailed) { + validImage(buffer) := true.B + } + state := idle + } +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..d9c5c321b --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/loader/src/main/scala/configs/Params.scala @@ -0,0 +1,8 @@ +package framework.acc.gpdomain.loader.configs + +case class Params( + iBufWords: Int, + constBytes: Int) { + require(iBufWords > 0) + require(constBytes > 0) +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/build.sc b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/build.sc new file mode 100644 index 000000000..f4df32f36 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/build.sc @@ -0,0 +1,24 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.gpdomain.memory.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/BankPort.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/KernelMemory.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Memory.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/BankPort.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/BankPort.scala new file mode 100644 index 000000000..0fe1f588a --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/BankPort.scala @@ -0,0 +1,122 @@ +package framework.acc.gpdomain.memory + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.acc.gpdomain._ +import framework.acc.gpdomain.execution.{VectorMemoryRequest, VectorMemoryResponse} +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.memdomain.{BankPort => BankAccessPort} +import framework.acc.gpdomain.loader.KernelContext + +class BankRequest(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val operation = new VectorMemoryRequest + val context = new KernelContext(frontendParam, memParam) +} + +@instantiable +class BankPort(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Module { + val ownerId = frontendParam.ownerId + val lineBytes = memParam.bankWidth / 8 + val offsetBits = log2Ceil(lineBytes) + require(memParam.bankWidth >= 64 && isPow2( + memParam.bankWidth + )) + require(lineBytes <= 65536) + require(memParam.bankMaskLen == lineBytes) + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new BankRequest(frontendParam, memParam))) + val response = Decoupled(new VectorMemoryResponse) + val bank = new BankAccessPort(frontendParam, memParam) + }) + + val idle :: readRequest :: readResponse :: writeRequest :: writeResponse :: complete :: Nil = Enum(6) + val state = RegInit(idle) + val request = Reg(new BankRequest(frontendParam, memParam)) + val consumed = RegInit(0.U(4.W)) + val result = RegInit(0.U(64.W)) + val error = RegInit(false.B) + + val byteCount = (1.U(4.W) << request.operation.size)(3, 0) + val byteAddress = request.operation.address(31, 0) +& consumed + val lineOffset = byteAddress(offsetBits - 1, 0) + val remaining = byteCount - consumed + val lineRemaining = lineBytes.U - lineOffset + val chunk = Mux(remaining <= lineRemaining, remaining, lineRemaining)(3, 0) + val nextConsumed = consumed + chunk + val lineAddress = byteAddress >> offsetBits + val readShifted = io.bank.read.resp.bits.data >> (lineOffset << 3) + val chunkMask = ((1.U(65.W) << (chunk << 3)) - 1.U)(63, 0) + val readChunk = readShifted.pad(64)(63, 0) & chunkMask + + io.request.ready := state === idle + io.response.valid := state === complete + io.response.bits.data := result + io.response.bits.error := error + val localBank = request.operation.address(63, 32) + val fromRead = localBank < request.context.readGroups + val parentBank = Mux(fromRead, request.context.readBank, request.context.writeBank) + val parentGroup = Mux(fromRead, localBank, localBank - request.context.readGroups) + io.bank.bank_id := parentBank + io.bank.group_id := parentGroup + io.bank.rob_id := request.context.rob_id + io.bank.inst_id := request.context.instId + io.bank.thread_id := ownerId.U + io.bank.is_shared := false.B // BankRouter resolves private/shared ownership from the alias. + io.bank.read.req.valid := state === readRequest + io.bank.read.req.bits.addr := lineAddress + io.bank.read.resp.ready := state === readResponse + io.bank.write.req.valid := state === writeRequest + io.bank.write.req.bits.addr := lineAddress + io.bank.write.req.bits.data := (request.operation.data >> (consumed << 3)).pad( + memParam.bankWidth + ) << (lineOffset << 3) + io.bank.write.resp.ready := state === writeResponse + for (i <- 0 until lineBytes) { + val sourceByte = consumed + i.U - lineOffset + io.bank.write.req.bits.mask(i) := i.U >= lineOffset && i.U < lineOffset +& chunk && request.operation.mask( + sourceByte(2, 0) + ) + } + + when(io.request.fire) { + request := io.request.bits + consumed := 0.U + result := 0.U + val end = io.request.bits.operation.address(31, 0) +& (1.U(33.W) << io.request.bits.operation.size) + val invalid = io.request.bits.operation.address( + 63, + 32 + ) >= (io.request.bits.context.readGroups +& io.request.bits.context.writeGroups) || + (io.request.bits.operation.write && io.request.bits.operation.address( + 63, + 32 + ) < io.request.bits.context.readGroups) || + end > (BigInt(memParam.bankEntries) * lineBytes).U + error := invalid + state := Mux(invalid, complete, Mux(io.request.bits.operation.write, writeRequest, readRequest)) + } + when(io.bank.read.req.fire) { + state := readResponse + } + when(io.bank.read.resp.fire) { + result := result | (readChunk << (consumed << 3)) + consumed := nextConsumed + state := Mux(nextConsumed === byteCount, complete, readRequest) + } + when(io.bank.write.req.fire) { + state := writeResponse + } + when(io.bank.write.resp.fire) { + consumed := nextConsumed + error := !io.bank.write.resp.bits.ok + state := Mux(!io.bank.write.resp.bits.ok || nextConsumed === byteCount, complete, writeRequest) + } + when(io.response.fire) { + state := idle + } +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/Emit.scala new file mode 100644 index 000000000..a7aabb852 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/Emit.scala @@ -0,0 +1,26 @@ +package framework.acc.gpdomain.memory + +import _root_.configs.GlobalConfig + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Memory(thread.acc.get.frontend, thread.acc.get.memDomain, thread.acc.get.gpDomain), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/memory/KernelMemory.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/KernelMemory.scala similarity index 92% rename from arch/src/main/scala/framework/rvv/src/main/scala/memory/KernelMemory.scala rename to arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/KernelMemory.scala index 3ca93e5d4..575a5b816 100644 --- a/arch/src/main/scala/framework/rvv/src/main/scala/memory/KernelMemory.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/KernelMemory.scala @@ -1,13 +1,19 @@ -package framework.rvv +package framework.acc.gpdomain.memory +import framework.lib.arbiter.{RRArbiter} +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.gpdomain.configs.GpDomainParam + +import framework.acc.gpdomain._ +import framework.acc.gpdomain.execution.{VectorMemoryRequest, VectorMemoryResponse} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig @instantiable -class KernelMemory(val b: GlobalConfig) extends Module { - val p = b.rvv +class KernelMemory(val gpParam: GpDomainParam) extends Module { + val p = gpParam.memory @public val io = IO(new Bundle { @@ -38,7 +44,7 @@ class KernelMemory(val b: GlobalConfig) extends Module { VecInit((0 until p.memoryPorts).map(port => io.request(port).valid && !pending(port) && io.request(port).bits.write)) val hasWrite = writes.asUInt.orR val writePort = PriorityEncoder(writes) - val arbiter = Module(new RRArbiter(new VectorMemoryRequest, p.memoryPorts)) + val arbiter = Instantiate(new RRArbiter(new VectorMemoryRequest, p.memoryPorts)) arbiter.io.out.ready := state === idle for (port <- 0 until p.memoryPorts) { val eligible = !pending(port) && (!hasWrite || writePort === port.U) diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/Memory.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/Memory.scala new file mode 100644 index 000000000..1bb3635d4 --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/Memory.scala @@ -0,0 +1,82 @@ +package framework.acc.gpdomain.memory + +import framework.lib.queue.Queue + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.memdomain.{BankPort => BankAccessPort} +import framework.acc.gpdomain.execution.{VectorMemoryRequest, VectorMemoryResponse} +import framework.acc.gpdomain.loader.LoaderMemoryPort + +class ResponseOwner extends Bundle { + val local = Bool() + val loader = Bool() +} + +@instantiable +class Memory(val frontendParam: FrontendParam, val memParam: MemDomainParam, val gpParam: GpDomainParam) + extends Module { + val ownerId = frontendParam.ownerId + val p = gpParam.memory + + @public + val io = IO(new Bundle { + val loader = Flipped(new LoaderMemoryPort(frontendParam, memParam)) + val request = Vec(p.memoryPorts, Flipped(Decoupled(new VectorMemoryRequest))) + val response = Vec(p.memoryPorts, Decoupled(new VectorMemoryResponse)) + val banks = Vec(p.memoryPorts, new BankAccessPort(frontendParam, memParam)) + }) + + val local: Instance[KernelMemory] = Instantiate(new KernelMemory(gpParam)) + val ports: Seq[Instance[BankPort]] = + Seq.fill(p.memoryPorts)(Instantiate(new BankPort(frontendParam, memParam))) + + local.io.buffer := io.loader.buffer + local.io.loading := io.loader.loading + local.io.constBytes := io.loader.constBytes + + for (i <- 0 until p.memoryPorts) { + io.banks(i) <> ports(i).io.bank + val owners = Instantiate(new Queue( + new ResponseOwner, + 2 + )) + val fromLoader = if (i == 0) io.loader.control else false.B + val request = if (i == 0) Mux(fromLoader, io.loader.request.bits, io.request(i).bits) else io.request(i).bits + val offered = + if (i == 0) Mux(fromLoader, io.loader.request.valid, io.request(i).valid) + else !io.loader.control && io.request(i).valid + val internal = request.address >= "h40000000".U && request.address < "h40002000".U + val destinationReady = Mux(internal, local.io.request(i).ready, ports(i).io.request.ready) + + local.io.request(i).valid := offered && internal && owners.io.enq.ready + local.io.request(i).bits := request + ports(i).io.request.valid := offered && !internal && owners.io.enq.ready + ports(i).io.request.bits.operation := request + ports(i).io.request.bits.context := io.loader.context + owners.io.enq.valid := offered && destinationReady + owners.io.enq.bits.local := internal + owners.io.enq.bits.loader := fromLoader + io.request(i).ready := !io.loader.control && owners.io.enq.ready && destinationReady + if (i == 0) io.loader.request.ready := io.loader.control && owners.io.enq.ready && destinationReady + + // Hold response ownership until acceptance, even when the request source changes. + val responseValid = Mux(owners.io.deq.bits.local, local.io.response(i).valid, ports(i).io.response.valid) + val responseBits = Mux(owners.io.deq.bits.local, local.io.response(i).bits, ports(i).io.response.bits) + val responseReady = Mux(owners.io.deq.bits.loader, io.loader.response.ready, io.response(i).ready) + io.response(i).valid := owners.io.deq.valid && !owners.io.deq.bits.loader && responseValid + io.response(i).bits := responseBits + if (i == 0) { + io.loader.response.valid := owners.io.deq.valid && owners.io.deq.bits.loader && responseValid + io.loader.response.bits := responseBits + } + local.io.response(i).ready := owners.io.deq.valid && owners.io.deq.bits.local && responseReady + ports(i).io.response.ready := owners.io.deq.valid && !owners.io.deq.bits.local && responseReady + owners.io.deq.ready := responseValid && responseReady + } +} diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..76f3a3bce --- /dev/null +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/memory/src/main/scala/configs/Params.scala @@ -0,0 +1,11 @@ +package framework.acc.gpdomain.memory.configs + +case class Params( + memoryPorts: Int, + wordBits: Int) { + require(memoryPorts > 0) + require(wordBits == 32 || wordBits == 64) + + val constBytes: Int = 4096 + val stackBytes: Int = 4096 +} diff --git a/arch/src/main/scala/framework/acc/memdomain/build.sc b/arch/src/main/scala/framework/acc/memdomain/build.sc new file mode 100644 index 000000000..4803374bc --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/build.sc @@ -0,0 +1,36 @@ +import mill._ +import $file.src.main.scala.bank_router.{build => bank_router_build} +import $file.src.main.scala.prm.{build => prm_build} +import $file.src.main.scala.tomem.{build => tomem_build} +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.memdomain.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/MemDomain.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MemoryControl.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/MemDomainParam.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/ctrl/Ctrl.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/ctrl/configer/Configer.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/ctrl/decoder/DISA.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/ctrl/decoder/Decoder.scala")) + ) ++ + bank_router_build.sources() ++ + prm_build.sources() ++ + tomem_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + bank_router_build.emitSources() ++ + prm_build.emitSources() ++ + tomem_build.emitSources() +) diff --git a/arch/src/main/scala/framework/acc/memdomain/configs/default.toml b/arch/src/main/scala/framework/acc/memdomain/configs/default.toml new file mode 100644 index 000000000..a265701ad --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/configs/default.toml @@ -0,0 +1,21 @@ + +[bank] +layoutEnable = true +num = 24 +width = 128 +entries = 512 +maskLen = 16 +requestPorts = 13 + +[dma] +busWidth = 128 + +[mmio] +enable = false +bankNum = 5 +bankEntries = 1024 +bankWidth = 8 +readWidth = 8 + +[mem] +addrLen = 39 diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/Emit.scala new file mode 100644 index 000000000..fc189a982 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/Emit.scala @@ -0,0 +1,39 @@ +package framework.acc.memdomain + +import sims.configs.SimParam +import framework.core.tbus.configs.Layout + +import _root_.configs.GlobalConfig +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new MemDomain( + thread.acc.get.frontend, + thread.acc.get.memDomain, + thread.acc.get.gpDomain, + thread.acc.get.transport, + core.privileged.permission, + core.memory.dcache.tracking, + core.memory.memoryAccess.walker, + b.sim.copy(diffTest = false) + ), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemDomain.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemDomain.scala new file mode 100644 index 000000000..4cc467e7b --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemDomain.scala @@ -0,0 +1,232 @@ +package framework.acc.memdomain + +import sims.configs.SimParam +import framework.lib.arbiter.{RRArbiter} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.acc.debug.trace.MemoryEvent +import framework.ip.memory.bank.{SramReadIO, SramReadReq, SramReadResp, SramWriteIO, SramWriteReq, SramWriteResp} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.frontend.scheduler.MemoryIssue +import framework.acc.balldomain.DomainMemory +import framework.acc.memdomain.tomem.ToMem +import framework.core.memory.MemoryPort +import framework.core.memory.access.preflight.Preparation +import framework.ip.bus.axi4 +import framework.acc.memdomain.bank_router.{BankMapping, BankRouter} +import framework.acc.memdomain.prm.PRM +import framework.acc.memdomain.{MvoverISA, MvoverPort} +import framework.ip.memory.mesh_shm.MeshLocalBankPort +import framework.acc.gpdomain.ImagePort +import framework.thread.tlink.BankAccess + +class PhysicalBankHash(val frontendParam: FrontendParam) extends Bundle { + val valid = Bool() + val threadId = UInt(64.W) + val vbankId = UInt(frontendParam.bank_id_len.W) + val pbankId = UInt(32.W) + val groupId = UInt(32.W) + val statusHash = UInt(32.W) +} + +class BankRead(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Bundle { + val bank_id = Input(UInt(frontendParam.bank_id_len.W)) + val rob_id = Input(UInt(log2Up(frontendParam.rob_entries).W)) + val group_id = Input(UInt(memParam.groupIdWidth.W)) + + val io = new SramReadIO(memParam) +} + +class BankWrite(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Bundle { + val bank_id = Input(UInt(frontendParam.bank_id_len.W)) + val rob_id = Input(UInt(log2Up(frontendParam.rob_entries).W)) + val group_id = Input(UInt(memParam.groupIdWidth.W)) + + val io = new SramWriteIO(memParam) +} + +class BankPort(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val write = Flipped(new SramWriteIO(memParam)) + val read = Flipped(new SramReadIO(memParam)) + val bank_id = Output(UInt(frontendParam.bank_id_len.W)) + val group_id = Output(UInt(memParam.groupIdWidth.W)) + val is_shared = Output(Bool()) + val thread_id = Output(UInt(64.W)) + val rob_id = Output(UInt(log2Up(frontendParam.rob_entries).W)) + val inst_id = Output(UInt(64.W)) +} + +class LayoutQuery(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val bank = Output(UInt(frontendParam.bank_id_len.W)) + val groups = Input(UInt(memParam.groupCountWidth.W)) +} + +class BankLayoutRequest(frontendParam: FrontendParam) extends Bundle { + val banks = Vec(3, UInt(frontendParam.bank_id_len.W)) + val enabled = Vec(3, Bool()) +} + +class BankLayout(memParam: MemDomainParam) extends Bundle { + val groups = Vec(3, UInt(memParam.groupCountWidth.W)) +} + +@instantiable +class MemDomain( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val transportParam: TransportParam, + val permissionParam: PmpParams, + val trackingParam: TrackingParam, + val walkerParam: WalkerParam, + val simp: SimParam) + extends Module { + val sharedPorts = memParam.sharedPorts + val diffTest = simp.diffTest + val commandDataBits = frontendParam.commandDataBits + val ownerId = frontendParam.ownerId + + val kernelPorts = + if (gpParam.enable) + gpParam.memory.memoryPorts + else 0 + + val sharedHashCount = if (memParam.sharedEnable) memParam.sharedBankNum else 0 + + @public + val io = IO(new Bundle { + val traceMemory = Output(Vec(memParam.bankRequestPorts + kernelPorts + 2 + memParam.bankNum, new MemoryEvent)) +// Command Channel + val issue = Flipped(Decoupled(new MemoryIssue(frontendParam, memParam, permissionParam))) + val complete = Decoupled(new Response(frontendParam)) + val mapping = Flipped(Decoupled(new BankMapping(frontendParam))) +// Inside Channel + val banks = Vec(memParam.bankRequestPorts, Flipped(new BankPort(frontendParam, memParam))) + val ballLayoutRequest = Flipped(Decoupled(new BankLayoutRequest(frontendParam))) + val ballLayoutResponse = Decoupled(new BankLayout(memParam)) + val gpLayoutRequest = Option.when(gpParam.enable)(Flipped(Decoupled(new BankLayoutRequest(frontendParam)))) + val gpLayoutResponse = Option.when(gpParam.enable)(Decoupled(new BankLayout(memParam))) + val gpBanks = Option.when(gpParam.enable)(Vec(kernelPorts, Flipped(new BankPort(frontendParam, memParam)))) + val image = Option.when(gpParam.enable)(new ImagePort) +// Outside Channel + val ball = Flipped(new DomainMemory(frontendParam, memParam, permissionParam)) + val memory = new MemoryPort(trackingParam, walkerParam) + val ddr = new axi4.Port(transportParam.axi) + + val shared = new Bundle { + val bank = new BankAccess(frontendParam, memParam) + val move = new MvoverPort + } + + val bank_hashes = + if (diffTest) { + Some(Output(Vec(memParam.bankNum + sharedHashCount, new PhysicalBankHash(frontendParam)))) + } else { + None + } + + val shared_bank_hashes = + if (diffTest && memParam.sharedEnable) { + Some(Input(Vec(sharedHashCount, new PhysicalBankHash(frontendParam)))) + } else { + None + } + + }) + + val control: Instance[MemoryControl] = + Instantiate(new MemoryControl(gpParam, transportParam, permissionParam, trackingParam, walkerParam)) + + val preparation: Instance[Preparation] = Instantiate(new Preparation( + framework.core.memory.access.preflight.configs.Config( + permissionParam, + transportParam.mapping, + transportParam.regions + ), + walkerParam + )) + + val ctrl: Instance[Ctrl] = Instantiate(new Ctrl(frontendParam, memParam, permissionParam)) + val toMem: Instance[ToMem] = + Instantiate(new ToMem(frontendParam, memParam, gpParam, transportParam, permissionParam)) + val bankRouter: Instance[BankRouter] = + Instantiate(new BankRouter(frontendParam, memParam, gpParam, simp.diffTest)) + val prm: Instance[PRM] = Instantiate(new PRM(memParam, simp.diffTest)) + + io.traceMemory := bankRouter.io.traceMemory + prm.io.banks <> bankRouter.io.physical + bankRouter.io.mapping <> io.mapping + io.shared.bank.mapping <> bankRouter.io.shared.mapping + bankRouter.io.hashes.foreach(_ := prm.io.hashes.get) + io.bank_hashes.foreach { hashes => + for (i <- 0 until memParam.bankNum) + hashes(i) := bankRouter.io.bankHashes.get(i) + for (i <- 0 until sharedHashCount) + hashes(memParam.bankNum + i) := io.shared_bank_hashes.get(i) + } + + // Query the router layout. + bankRouter.io.queryVbank := toMem.io.queryVbank + bankRouter.io.queryShared := toMem.io.queryShared + toMem.io.queryGroups := bankRouter.io.queryGroups + ctrl.io.clearBusy := bankRouter.io.clearBusy + +//===----------------------------------------------------------------------===// +// Control and transport connections +//===----------------------------------------------------------------------===// + val commands = Instantiate(new RRArbiter( + new MemoryIssue(frontendParam, memParam, permissionParam), + 2 + )) + + commands.io.in(0) <> io.issue + commands.io.in(1) <> io.ball.request + ctrl.io.issue <> commands.io.out + val childOwner = RegInit(VecInit(Seq.fill(frontendParam.rob_entries)(false.B))) + when(ctrl.io.issue.fire)(childOwner(ctrl.io.issue.bits.operation.rob_id) := commands.io.chosen === 1.U) + val child = childOwner(ctrl.io.complete.bits.rob_id) + io.complete.valid := ctrl.io.complete.valid && !child + io.complete.bits := ctrl.io.complete.bits + io.ball.response.valid := ctrl.io.complete.valid && child + io.ball.response.bits := ctrl.io.complete.bits + ctrl.io.complete.ready := Mux(child, io.ball.response.ready, io.complete.ready) + io.ball.metadata <> toMem.io.metadata + io.shared.move <> ctrl.io.mvover + ctrl.io.request <> toMem.io.request + toMem.io.response <> ctrl.io.response + + control.io.memory <> toMem.io.memory.control + control.io.preparation <> preparation.io + io.memory <> control.io.service + io.ddr <> toMem.io.memory.axi + + bankRouter.io.requests <> io.banks + bankRouter.io.ballLayoutRequest <> io.ballLayoutRequest + io.ballLayoutResponse <> bankRouter.io.ballLayoutResponse + io.gpLayoutRequest.zip(bankRouter.io.gpLayoutRequest).foreach { case (source, sink) => sink <> source } + io.gpLayoutResponse.zip(bankRouter.io.gpLayoutResponse).foreach { case (sink, source) => sink <> source } + + io.image.zip(toMem.io.image).foreach { case (sink, source) => sink <> source } + io.gpBanks.foreach { ports => + for (port <- 0 until kernelPorts) bankRouter.io.directRequest(port) <> ports(port) + } + + bankRouter.io.directRequest(kernelPorts) <> toMem.io.bankRead + bankRouter.io.directRequest(kernelPorts + 1) <> toMem.io.bankWrite + bankRouter.io.shared.local <> io.shared.bank.local + bankRouter.io.config <> ctrl.io.config + + // Shared path passthrough + io.shared.bank.requests <> bankRouter.io.shared.requests + io.shared.bank.config <> bankRouter.io.shared.config +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemoryControl.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemoryControl.scala new file mode 100644 index 000000000..fa90ae0bb --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemoryControl.scala @@ -0,0 +1,242 @@ +package framework.acc.memdomain + +import framework.acc.gpdomain.configs.GpDomainParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} + +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.memory.MemoryPort +import framework.core.memory.access.preflight.{Command, Error, PmpEntry, PreparationPort} +import framework.acc.memdomain.toddr.{DmaError, DmaStatus} +import framework.acc.memdomain.tomem.ControlPort + +/** Address-space state frozen with the instruction and consumed only by memory requests. */ +class MemoryContext(permission: PmpParams) extends Bundle { + val satp = UInt(64.W) + val privilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + val pmp = + Vec(permission.entries, new PmpEntry(permission)) +} + +/** Tracks the three DMA producers, independent of frontend tags and ROB retirement. */ +@instantiable +class MemoryControl( + gpParam: GpDomainParam, + transportParam: TransportParam, + permissionParam: PmpParams, + trackingParam: TrackingParam, + walkerParam: WalkerParam) + extends Module { + val transport = transportParam + val producers = if (gpParam.enable) 3 else 2 + require(transport.tracking.entries >= producers && transport.prepared.idBits >= log2Ceil(producers)) + + @public + val io = IO(new Bundle { + val service = new MemoryPort(trackingParam, walkerParam) + val preparation = new PreparationPort(transportParam.mapping, permissionParam, walkerParam) + val memory = Flipped(new ControlPort(transportParam, permissionParam)) + }) + + val active = RegInit(VecInit(Seq.fill(3)(false.B))) + val dispatchPending = RegInit(VecInit(Seq.fill(3)(false.B))) + val preparePending = RegInit(VecInit(Seq.fill(3)(false.B))) + val mapReady = RegInit(VecInit(Seq.fill(3)(false.B))) + val granted = RegInit(VecInit(Seq.fill(3)(false.B))) + val doneIssued = RegInit(VecInit(Seq.fill(3)(false.B))) + val coherent = RegInit(VecInit(Seq.fill(3)(false.B))) + val released = RegInit(VecInit(Seq.fill(3)(false.B))) + val completionPending = RegInit(VecInit(Seq.fill(3)(false.B))) + val completionStatus = Reg(Vec(3, new DmaStatus)) + val contexts = Reg(Vec(3, new MemoryContext(permissionParam))) + val shapes = Reg(Vec(3, new Command(transportParam.mapping))) + val halted = RegInit(false.B) + val firstFault = RegInit(0.U.asTypeOf(new DmaStatus)) + + def stop(error: UInt, address: UInt): Unit = { + when(!halted) { + halted := true.B + firstFault.error := error + firstFault.address := address + } + } + + def dmaError(error: UInt): UInt = MuxLookup(error, DmaError.Context.U)(Seq( + Error.Ok.U -> DmaError.None.U, + Error.Shape.U -> DmaError.Shape.U, + Error.Overflow.U -> DmaError.Shape.U, + Error.PageFault.U -> DmaError.PageFault.U, + Error.AccessFault.U -> DmaError.AccessFault.U, + Error.Capacity.U -> DmaError.Capacity.U, + Error.Context.U -> DmaError.Context.U + )) + + // Producer 0 is the bank reader, 1 the writer, and 2 the image reader. + for (i <- 0 until 3) { + val access = if (i == 1) io.memory.write.bits.access else io.memory.read.bits.access + val offered = + if (i >= producers) false.B + else if (i == 1) io.memory.write.valid + else io.memory.read.valid && io.memory.read.bits.producer === i.U + when(offered && !active(i) && !halted) { + active(i) := true.B + dispatchPending(i) := true.B + preparePending(i) := false.B + mapReady(i) := false.B + granted(i) := false.B + doneIssued(i) := false.B + coherent(i) := false.B + released(i) := false.B + contexts(i) := access.context + shapes(i).id := i.U + shapes(i).baseVA := access.baseVA + shapes(i).rows := access.rows(31, 0) + shapes(i).columns := access.columns(31, 0) + shapes(i).spanBytes := access.spanBytes(31, 0) + shapes(i).columnStride := access.columnStride(31, 0) + shapes(i).rowStride := access.rowStride(31, 0) + shapes(i).write := (i == 1).B + shapes(i).mode := access.context.satp(63, 60) + shapes(i).rootPpn := access.context.satp(43, 0) + shapes(i).privilege := access.context.privilege + shapes(i).sum := access.context.sum + shapes(i).mxr := access.context.mxr + when(access.rows(63, 32).orR || access.columns(63, 32).orR || access.spanBytes(63, 32).orR || + access.columnStride(63, 32).orR || access.rowStride(63, 32).orR) { + stop(DmaError.Shape.U, access.baseVA) + } + } + io.memory.decisions(i).parentId := i.U + io.memory.decisions(i).valid := active(i) && ((mapReady(i) && granted(i)) || halted) + io.memory.decisions(i).fault := firstFault + val finished = io.memory.finished(i) + when(finished.valid && finished.bits.error =/= DmaError.None.U) { + stop(finished.bits.error, finished.bits.address) + } + // A rejected image may finish locally without ever starting DMA. + val localFailure = !active(i) && finished.bits.error =/= DmaError.None.U + when(!completionPending(i) && finished.valid && (localFailure || (active(i) && (released(i) || halted)))) { + completionPending(i) := true.B + completionStatus(i) := Mux(halted, firstFault, finished.bits) + } + io.memory.complete(i).valid := completionPending(i) + io.memory.complete(i).bits := completionStatus(i) + when(io.memory.complete(i).fire) { + completionPending(i) := false.B + active(i) := false.B + } + } + + val dispatchOffer = RegInit(false.B) + val dispatchSlot = Reg(UInt(2.W)) + when(!dispatchOffer && dispatchPending.asUInt.orR && !halted) { + dispatchOffer := true.B + dispatchSlot := PriorityEncoder(dispatchPending) + } + io.memory.coherence.dispatch.valid := dispatchOffer && !halted + io.memory.coherence.dispatch.bits.id := dispatchSlot + when(io.memory.coherence.dispatch.fire) { + dispatchOffer := false.B + dispatchPending(dispatchSlot) := false.B + preparePending(dispatchSlot) := true.B + } + + val prepareOffer = RegInit(false.B) + val prepareSlot = Reg(UInt(2.W)) + when(!prepareOffer && preparePending.asUInt.orR && !halted) { + prepareOffer := true.B + prepareSlot := PriorityEncoder(preparePending) + } + io.preparation.command.valid := prepareOffer && !halted + io.preparation.command.bits := shapes(prepareSlot) + when(io.preparation.command.fire) { + prepareOffer := false.B + preparePending(prepareSlot) := false.B + } + val contextId = io.preparation.contextTag + io.preparation.contextValid := contextId < 3.U && active(contextId) + io.preparation.contextId := contextId + io.preparation.contextPmp := contexts(contextId).pmp + + val range = io.preparation.ranges + val info = io.memory.coherence.accessInfo + info.valid := range.valid && range.bits.error === Error.Ok.U && !halted + info.bits.id := range.bits.id + info.bits.hasMemory := true.B + info.bits.base := range.bits.pa + info.bits.bytes := range.bits.bytes + info.bits.write := range.bits.write + info.bits.last := range.bits.last + range.ready := range.bits.error =/= Error.Ok.U || (info.ready && !halted) + when(range.fire && range.bits.error =/= Error.Ok.U) { + stop(dmaError(range.bits.error), range.bits.va) + } + io.preparation.ready.ready := true.B + when(io.preparation.ready.fire) { + val result = io.preparation.ready.bits + assert(result.id < 3.U && active(result.id), "Prepared result has no DMA owner") + mapReady(result.id) := true.B + when(result.error =/= Error.Ok.U)(stop(dmaError(result.error), result.va)) + } + io.memory.coherence.grant.ready := true.B + when(io.memory.coherence.grant.fire) { + val id = io.memory.coherence.grant.bits.tag + assert(id < 3.U && active(id), "Coherence grant has no DMA owner") + granted(id) := true.B + } + io.preparation.queries := io.memory.queries + io.memory.mappings := io.preparation.results + + // Finishing the bank stream precedes post-DMA cache maintenance and map release. + val doneCandidates = + VecInit((0 until 3).map(i => active(i) && io.memory.finished(i).valid && mapReady(i) && granted(i) && !doneIssued(i))) + val doneOffer = RegInit(false.B) + val doneSlot = Reg(UInt(2.W)) + when(!doneOffer && doneCandidates.asUInt.orR && !halted) { + doneOffer := true.B + doneSlot := PriorityEncoder(doneCandidates) + } + io.memory.coherence.done.valid := doneOffer && !halted + io.memory.coherence.done.bits.tag := doneSlot + io.memory.coherence.done.bits.ok := true.B + when(io.memory.coherence.done.fire) { + doneOffer := false.B + doneIssued(doneSlot) := true.B + } + io.memory.coherence.complete.ready := true.B + when(io.memory.coherence.complete.fire) { + val id = io.memory.coherence.complete.bits.tag + assert(id < 3.U && active(id), "Coherence completion has no DMA owner") + coherent(id) := true.B + } + val releaseCandidates = VecInit((0 until 3).map(i => active(i) && coherent(i) && !released(i))) + val releaseOffer = RegInit(false.B) + val releaseSlot = Reg(UInt(2.W)) + when(!releaseOffer && releaseCandidates.asUInt.orR && !halted) { + releaseOffer := true.B + releaseSlot := PriorityEncoder(releaseCandidates) + } + io.preparation.release.valid := releaseOffer && !halted + io.preparation.release.bits.id := releaseSlot + when(io.preparation.release.fire) { + releaseOffer := false.B + released(releaseSlot) := true.B + } + + io.memory.coherence.cpuQuery := io.service.cpuQuery + io.service.cpuAllow := io.memory.coherence.cpuAllow && !halted + io.service.maintenance <> io.memory.coherence.maintenance + io.memory.coherence.maintained <> io.service.maintained + io.service.pteRequest <> io.preparation.pteRequest + io.preparation.pteResponse <> io.service.pteResponse + io.memory.coherence.cancel.valid := false.B + io.memory.coherence.cancel.bits := 0.U.asTypeOf(io.memory.coherence.cancel.bits) +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/build.sc b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/build.sc new file mode 100644 index 000000000..4998fa422 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.memdomain.bank_router.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/BankRouter.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/blt/AccPipe.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/blt/BLT.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/configs/uvm.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/configs/uvm.toml new file mode 100644 index 000000000..f6d98beb5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/configs/uvm.toml @@ -0,0 +1,48 @@ +[parameters] +diffTest = false + +[parameters.frontend] +ownerId = 7 +commandDataBits = 64 +rob_entries = 4 +bank_alias_enable = true +rs_out_of_order_response = true +bank_id_len = 4 +vbank_id_upper_bound = 3 +shared_bank_id_base = 16 +iter_len = 16 +sub_rob_enable = false +sub_rob_depth = 4 + +[parameters.memory] +sharedPorts = 0 +bankNum = 4 +bankLayoutEnable = true +bankWidth = 128 +bankEntries = 8 +bankMaskLen = 16 +virtualBankCount = 4 +sharedEnable = false +sharedEntries = 0 +sharedBankEntries = 0 +sharedBankNum = 0 +sharedInputChannels = 0 +sharedDefaultGroupCount = 0 +bankRequestPorts = 2 +dma_buswidth = 128 +memAddrLen = 44 +mmioEnable = false +mmioBankNum = 0 +mmioBankEntries = 0 +mmioBankWidth = 0 +mmioReadWidth = 0 + +[parameters.gp] +enable = false +laneNumber = 1 +vLen = 128 +eLen = 64 +iBufWords = 16 +memoryPorts = 1 + +[defines] diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..6f9eeb3d9 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual router_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual router_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.source = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/item.svh new file mode 100644 index 000000000..0cbe365ce --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/item.svh @@ -0,0 +1,22 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`BR_INPUT_BITS-1:0] inputs; + logic [`BR_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(writing, UVM_ALL_ON) + `uvm_field_int(port, UVM_ALL_ON) + `uvm_object_utils_end + logic [`BR_DATA_BITS-1:0] data; + bit writing; + int port; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..d42bc55c3 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual router_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual router_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/peer.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/peer.svh new file mode 100644 index 000000000..a4bafede6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/agent/peer.svh @@ -0,0 +1,56 @@ +class peer extends uvm_component; + `uvm_component_utils(peer) + virtual router_if vif; + const int banks[`BR_BANKS][17] = `BR_BANK_OFFSETS; + logic [`BR_DATA_BITS-1:0] memory[`BR_BANKS][`BR_ROWS]; + bit initialized[`BR_BANKS][`BR_ROWS]; + int cycle; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual router_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Router interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + logic [`BR_INPUT_BITS-1:0] next_backend; + @(posedge vif.clock); + next_backend = '0; + cycle++; + if (!vif.reset) + for (int b = 0; b < `BR_BANKS; b++) begin + if (vif.outputs[banks[b][14]]) + for (int row = 0; row < `BR_ROWS; row++) begin + memory[b][row] = '0; + initialized[b][row] = 1; + end + if (vif.outputs[banks[b][0]] && vif.inputs[banks[b][1]]) begin + int row = int'(vif.outputs[banks[b][2]+:16]); + if (row >= `BR_ROWS || !initialized[b][row]) + `uvm_fatal("MEMORY", "Uninitialized physical bank read") + next_backend[banks[b][3]] = 1; + next_backend[banks[b][5]+:`BR_DATA_BITS] = memory[b][row]; + end + if (vif.outputs[banks[b][6]] && vif.inputs[banks[b][7]]) begin + int row = int'(vif.outputs[banks[b][8]+:16]); + if (row >= `BR_ROWS) `uvm_fatal("MEMORY", "Physical bank address overflow") + for (int byte_index = 0; byte_index < `BR_MASK_BITS; byte_index++) + if (vif.outputs[banks[b][10]+byte_index]) + memory[b][row][byte_index*8+:8] = vif.outputs[banks[b][9]+byte_index*8+:8]; + // Every row is explicitly initialized by a full write before partial writes. + initialized[b][row] = 1; + next_backend[banks[b][11]] = 1; + next_backend[banks[b][13]] = 1; + end + end + @(negedge vif.clock); + for (int b = 0; b < `BR_BANKS; b++) begin + next_backend[banks[b][1]] = cycle % 4 != b % 4; + next_backend[banks[b][7]] = cycle % 5 != b % 5; + end + vif.backend = next_backend; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/env/env.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/env/env.svh new file mode 100644 index 000000000..ea4857b7d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/env/env.svh @@ -0,0 +1,33 @@ +class env extends uvm_env; + `uvm_component_utils(env) + uvm_analysis_imp #(uvm_sequence_item, env) reset_events; + driver source; + monitor mon; + peer backend; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 1) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_events = new("reset_events", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + backend = peer::type_id::create("backend", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 1)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + reference.cancelled.connect(reset_events); + endfunction + function void write(uvm_sequence_item t); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/env/model.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/env/model.svh new file mode 100644 index 000000000..0fbb446ca --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/env/model.svh @@ -0,0 +1,225 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(result_item) expected, actual; + uvm_analysis_port #(uvm_sequence_item) cancelled; + const int clients[`BR_CLIENTS][17] = `BR_CLIENT_OFFSETS; + const int banks[`BR_BANKS][17] = `BR_BANK_OFFSETS; + bit live[`BR_BANKS]; + int architecture[`BR_BANKS], alias_id[`BR_BANKS], group_id[`BR_BANKS]; + int columns[`BR_ALIASES], layouts[`BR_ALIASES]; + logic [`BR_DATA_BITS-1:0] memory[`BR_BANKS][`BR_ROWS]; + bit initialized[`BR_BANKS][`BR_ROWS]; + bit pending[`BR_CLIENTS], writing[`BR_CLIENTS]; + bit accepted_now[`BR_CLIENTS]; + int selected_now[`BR_CLIENTS], row_now[`BR_CLIENTS]; + logic [`BR_DATA_BITS-1:0] data_now[`BR_CLIENTS]; + logic [`BR_MASK_BITS-1:0] mask_now[`BR_CLIENTS]; + int delayed_query[2]; + bit query_enabled[2]; + covergroup behavior with function sample ( + int port, bit write, int groups, bit stall, bit remapped, bit retired, bit transfer + ); + option.per_instance = 1; + ports: coverpoint port {bins all[] = {[0 : `BR_CLIENTS - 1]};} + direction: coverpoint write; + layout: coverpoint groups {bins single = {1}; bins multi = {[2 : `BR_BANKS]};} + backpressure: coverpoint stall; + renaming: coverpoint remapped; + retirement: coverpoint retired; + transfers: coverpoint transfer; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + cancelled = new("cancelled", this); + behavior = new(); + endfunction + function void verify(bit ok, string message); + evidence::checks++; + if (!ok) `uvm_fatal("ROUTER", message) + endfunction + function int route(int tag, int group_index); + int found = -1; + foreach (live[b]) + if (live[b] && alias_id[b] == tag && group_id[b] == group_index) begin + verify(found == -1, "Ambiguous logical bank binding"); + found = b; + end + verify(found >= 0, "Request has no allocated bank binding"); + return found; + endfunction + function void write(cycle_item t); + bit bound[`BR_BANKS]; + if (t.reset) begin + foreach (live[b]) live[b] = 0; + foreach (columns[a]) begin + columns[a] = 0; + layouts[a] = 0; + end + foreach (pending[c]) pending[c] = 0; + query_enabled[0] = 0; + query_enabled[1] = 0; + cancelled.write(null); + return; + end + if (query_enabled[1]) + verify(t.outputs[`BR_OUT_QUERYGROUPS_OFFSET+:`BR_OUT_QUERYGROUPS_WIDTH] == delayed_query[1], + "Group count query mismatch"); + delayed_query[1] = columns[delayed_query[0]]; + query_enabled[1] = query_enabled[0]; + delayed_query[0] = int'(t.inputs[`BR_IN_QUERYVBANK_OFFSET+:`BR_IN_QUERYVBANK_WIDTH]); + query_enabled[0] = 1; + foreach (pending[c]) begin + result_item result; + int tag, group_index, row, physical; + bit wr, rd; + accepted_now[c] = 0; + rd = t.inputs[clients[c][0]] && t.outputs[clients[c][1]]; + wr = t.inputs[clients[c][6]] && t.outputs[clients[c][7]]; + verify(!(rd && wr), "Read and write accepted simultaneously on one port"); + if (rd || wr) begin + verify(!pending[c], "Second request before the port response retired"); + tag = int'(t.inputs[clients[c][14]+:`BR_IN_REQUESTS_0_BANK_ID_WIDTH]); + group_index = int'(t.inputs[clients[c][15]+:`BR_IN_REQUESTS_0_GROUP_ID_WIDTH]); + physical = route(tag, group_index); + row = int'(t.inputs[clients[c][wr?8 : 2]+:16]); + verify(row < `BR_ROWS, "Logical bank row overflow"); + accepted_now[c] = 1; + selected_now[c] = physical; + row_now[c] = row; + data_now[c] = t.inputs[clients[c][9]+:`BR_DATA_BITS]; + mask_now[c] = t.inputs[clients[c][10]+:`BR_MASK_BITS]; + result = result_item::type_id::create("expected"); + result.port = c; + result.writing = wr; + result.set_transaction_id(c); + if (wr) begin + for (int b = 0; b < `BR_MASK_BITS; b++) + if (mask_now[c][b]) memory[physical][row][b*8+:8] = data_now[c][b*8+:8]; + initialized[physical][row] = 1; + result.data = 1; + end else begin + verify(initialized[physical][row], "Read of an uninitialized logical bank row"); + result.data = memory[physical][row]; + end + expected.write(result); + pending[c] = 1; + writing[c] = wr; + behavior.sample(c, wr, columns[tag], 0, tag >= 4, 0, 0); + end + if(t.outputs[clients[c][3]]&&t.inputs[clients[c][4]]||t.outputs[clients[c][11]]&&t.inputs[clients[c][12]])begin + verify(pending[c], "Unsolicited client response"); + result = result_item::type_id::create("actual"); + result.port = c; + result.writing = writing[c]; + result.set_transaction_id(c); + result.data=writing[c]?`BR_DATA_BITS'(t.outputs[clients[c][13]]):t.outputs[clients[c][5]+:`BR_DATA_BITS]; + actual.write(result); + pending[c] = 0; + end + if(t.outputs[clients[c][3]]&&!t.inputs[clients[c][4]]||t.outputs[clients[c][11]]&&!t.inputs[clients[c][12]]) + behavior.sample(c, writing[c], 1, 1, 0, 0, 0); + end + for (int b = 0; b < `BR_BANKS; b++) begin + bit rd, wr; + int owner; + rd = t.outputs[banks[b][0]] && t.inputs[banks[b][1]]; + wr = t.outputs[banks[b][6]] && t.inputs[banks[b][7]]; + if (rd || wr) begin + owner = -1; + foreach (accepted_now[c]) + if (accepted_now[c] && selected_now[c] == b) begin + verify(owner == -1, "Multiple clients accepted by one bank"); + owner = c; + end + verify(owner >= 0, "Physical request without an accepted client"); + verify(writing[owner] == wr, "Physical write/read direction mismatch"); + verify(t.outputs[banks[b][wr?8 : 2]+:16] == row_now[owner], + "Physical row changed during routing"); + if (wr) + verify( + t.outputs[banks[b][9]+:`BR_DATA_BITS]===data_now[owner]&&t.outputs[banks[b][10]+:`BR_MASK_BITS]===mask_now[owner], + "Write data or byte mask changed during routing"); + accepted_now[owner] = 0; + end + end + foreach (accepted_now[c]) + verify(!accepted_now[c], "Client accepted without a physical bank request"); + if (t.inputs[`BR_IN_CONFIG_VALID_OFFSET] && t.outputs[`BR_OUT_CONFIG_READY_OFFSET]) begin + int tag, group_index, chosen; + bit alloc, transfer; + tag = int'(t.inputs[`BR_IN_CONFIG_BITS_VBANK_ID_OFFSET+:`BR_IN_CONFIG_BITS_VBANK_ID_WIDTH]); + group_index=int'(t.inputs[`BR_IN_CONFIG_BITS_GROUP_ID_OFFSET+:`BR_IN_CONFIG_BITS_GROUP_ID_WIDTH]); + alloc = t.inputs[`BR_IN_CONFIG_BITS_ALLOC_OFFSET]; + transfer = t.inputs[`BR_IN_CONFIG_BITS_TRANSFER_OFFSET]; + if (transfer) begin + int source=int'(t.inputs[`BR_IN_CONFIG_BITS_SOURCE_BANK_ID_OFFSET+:`BR_IN_CONFIG_BITS_SOURCE_BANK_ID_WIDTH]); + foreach (live[b]) + if (live[b] && architecture[b] == source) begin + alias_id[b] = tag; + architecture[b] = tag; + group_id[b] += columns[tag]; + end + columns[tag] += columns[source]; + columns[source] = 0; + layouts[tag] += layouts[source]; + layouts[source] = 0; + behavior.sample(0, 0, columns[tag], 0, 0, 0, 1); + end else if (alloc) begin + if (group_index == 0) foreach (live[b]) if (live[b] && architecture[b] == tag) live[b] = 0; + chosen = -1; + foreach (live[b]) if (!live[b] && chosen == -1) chosen = b; + verify(chosen >= 0, "Allocation exceeded physical capacity"); + live[chosen] = 1; + architecture[chosen] = tag; + alias_id[chosen] = tag; + group_id[chosen] = group_index; + bound[chosen] = 1; + columns[tag] = t.inputs[`BR_IN_CONFIG_BITS_IS_MULTI_OFFSET] ? group_index + 1 : 1; + layouts[tag] = columns[tag]; + if (t.inputs[`BR_IN_CONFIG_BITS_CLEAR_OFFSET]) + for (int row = 0; row < `BR_ROWS; row++) begin + memory[chosen][row] = '0; + initialized[chosen][row] = 1; + end + end else begin + foreach (live[b]) if (live[b] && architecture[b] == tag) live[b] = 0; + columns[tag] = 0; + layouts[tag] = 0; + end + end + if (t.inputs[`BR_IN_MAPPING_VALID_OFFSET] && t.outputs[`BR_OUT_MAPPING_READY_OFFSET]) begin + int bank_id, tag, previous, count; + bank_id = int'(t.inputs[`BR_IN_MAPPING_BITS_BANK_OFFSET+:`BR_IN_MAPPING_BITS_BANK_WIDTH]); + tag = int'(t.inputs[`BR_IN_MAPPING_BITS_VBANK_OFFSET+:`BR_IN_MAPPING_BITS_VBANK_WIDTH]); + previous=int'(t.inputs[`BR_IN_MAPPING_BITS_PREVIOUS_OFFSET+:`BR_IN_MAPPING_BITS_PREVIOUS_WIDTH]); + count = layouts[bank_id]; + if (t.inputs[`BR_IN_MAPPING_BITS_ALLOCATE_OFFSET]) begin + if (t.inputs[`BR_IN_MAPPING_BITS_REUSE_OFFSET]) + foreach (live[b]) if (live[b] && alias_id[b] == previous) live[b] = 0; + for (int group_index = 0; group_index < count; group_index++) begin + int chosen = -1; + foreach (live[b]) if (!live[b] && chosen == -1) chosen = b; + verify(chosen >= 0, "Alias allocation exceeded physical capacity"); + live[chosen] = 1; + architecture[chosen] = bank_id; + alias_id[chosen] = tag; + group_id[chosen] = group_index; + bound[chosen] = 1; + end + columns[tag] = count; + behavior.sample(0, 0, count, 0, 1, 0, 0); + end + if (t.inputs[`BR_IN_MAPPING_BITS_RETIRE_OFFSET]) begin + foreach (live[b]) if (live[b] && alias_id[b] == previous) live[b] = 0; + columns[previous] = 0; + behavior.sample(0, 0, count, 0, tag >= 4, 1, 0); + end + if(t.inputs[`BR_IN_MAPPING_BITS_WRITE_OFFSET]&&!t.inputs[`BR_IN_MAPPING_BITS_ALLOCATE_OFFSET]) + foreach (live[b]) if (live[b] && alias_id[b] == tag) bound[b] = 1; + end + foreach (bound[b]) + verify(t.outputs[banks[b][16]] == bound[b], "Physical binding pulse mismatch"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/filelist.f b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/filelist.f new file mode 100644 index 000000000..fecc7d73e --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/BankRouter/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/interface.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/interface.sv new file mode 100644 index 000000000..38389c2fa --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/interface.sv @@ -0,0 +1,9 @@ +`include "parameters.svh" +interface router_if ( + input logic clock +); + logic reset = 1; + logic [`BR_INPUT_BITS-1:0] source = '0, backend = '0; + wire [ `BR_INPUT_BITS-1:0] inputs = source | backend; + wire [`BR_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/package.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/package.sv new file mode 100644 index 000000000..301cc988f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/package.sv @@ -0,0 +1,14 @@ +`include "parameters.svh" +package router_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/peer.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..0d0db7514 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/seq/sequence.svh @@ -0,0 +1,108 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + const int clients[`BR_CLIENTS][17] = `BR_CLIENT_OFFSETS; + logic [`BR_INPUT_BITS-1:0] pins; + logic [`BR_OUTPUT_BITS-1:0] observed; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task layout(int tag, int groups, bit clear = 0, bit transfer = 0, int source = 0); + for (int group_index = 0; group_index < (groups == 0 ? 1 : groups); group_index++) begin + pins[`BR_IN_CONFIG_VALID_OFFSET] = 1; + pins[`BR_IN_CONFIG_BITS_VBANK_ID_OFFSET+:`BR_IN_CONFIG_BITS_VBANK_ID_WIDTH] = tag; + pins[`BR_IN_CONFIG_BITS_GROUP_ID_OFFSET+:`BR_IN_CONFIG_BITS_GROUP_ID_WIDTH] = group_index; + pins[`BR_IN_CONFIG_BITS_IS_MULTI_OFFSET] = groups > 1; + pins[`BR_IN_CONFIG_BITS_ALLOC_OFFSET] = groups > 0; + pins[`BR_IN_CONFIG_BITS_CLEAR_OFFSET] = clear; + pins[`BR_IN_CONFIG_BITS_THREAD_ID_OFFSET+:64] = 7; + pins[`BR_IN_CONFIG_BITS_TRANSFER_OFFSET] = transfer; + pins[`BR_IN_CONFIG_BITS_SOURCE_BANK_ID_OFFSET+:`BR_IN_CONFIG_BITS_SOURCE_BANK_ID_WIDTH]=source; + do tick(); while (!observed[`BR_OUT_CONFIG_READY_OFFSET]); + pins[`BR_IN_CONFIG_VALID_OFFSET] = 0; + if (transfer) break; + end + repeat (3) tick(); + endtask + task mapping(int bank_id, int tag, int previous, bit allocate, bit retire, bit reuse = 0); + pins[`BR_IN_MAPPING_VALID_OFFSET] = 1; + pins[`BR_IN_MAPPING_BITS_BANK_OFFSET+:`BR_IN_MAPPING_BITS_BANK_WIDTH] = bank_id; + pins[`BR_IN_MAPPING_BITS_VBANK_OFFSET+:`BR_IN_MAPPING_BITS_VBANK_WIDTH] = tag; + pins[`BR_IN_MAPPING_BITS_PREVIOUS_OFFSET+:`BR_IN_MAPPING_BITS_PREVIOUS_WIDTH] = previous; + pins[`BR_IN_MAPPING_BITS_ALLOCATE_OFFSET] = allocate; + pins[`BR_IN_MAPPING_BITS_RETIRE_OFFSET] = retire; + pins[`BR_IN_MAPPING_BITS_REUSE_OFFSET] = reuse; + pins[`BR_IN_MAPPING_BITS_THREADID_OFFSET+:64] = 7; + do tick(); while (!observed[`BR_OUT_MAPPING_READY_OFFSET]); + pins[`BR_IN_MAPPING_VALID_OFFSET] = 0; + repeat (3) tick(); + endtask + task transfer(int port, int tag, int group_index, int row, bit writing, + logic [`BR_DATA_BITS-1:0] data = '0, logic [`BR_MASK_BITS-1:0] mask = '1, + int stall = 0); + pins[clients[port][14]+:`BR_IN_REQUESTS_0_BANK_ID_WIDTH] = tag; + pins[clients[port][15]+:`BR_IN_REQUESTS_0_GROUP_ID_WIDTH] = group_index; + pins[clients[port][16]+:64] = 7; + pins[clients[port][writing?6 : 0]] = 1; + pins[clients[port][writing?8 : 2]+:16] = row; + pins[clients[port][9]+:`BR_DATA_BITS] = data; + pins[clients[port][10]+:`BR_MASK_BITS] = mask; + pins[clients[port][writing?12 : 4]] = 0; + do tick(); while (!observed[clients[port][writing?7 : 1]]); + pins[clients[port][writing?6 : 0]] = 0; + // Change the live route payload after acceptance; the old response keeps its owner. + pins[clients[port][14]+:`BR_IN_REQUESTS_0_BANK_ID_WIDTH] = (tag + 1) % `BR_ALIASES; + do tick(); while (!observed[clients[port][writing?11 : 3]]); + repeat (stall) tick(); + pins[clients[port][writing?12 : 4]] = 1; + tick(); + endtask + task body(); + pins = '0; + repeat (3) tick(1); + tick(); + layout(0, 1, 1); + layout(1, 1, 1); + for (int c = 0; c < `BR_CLIENTS; c++) + for (int row = 0; row < `BR_ROWS; row++) begin + transfer(c, c % 2, 0, row, 1, + `BR_DATA_BITS'(64'h123456789abcdef0) ^ (128'hdeadbeef << 64) ^ row ^ (c << 8), '1, + c % 4); + transfer(c, c % 2, 0, row, 0, '0, '1, c % 4); + end + for (int bit_index = 0; bit_index < `BR_MASK_BITS; bit_index++) begin + transfer(bit_index % `BR_CLIENTS, 0, 0, 0, 1, '1, `BR_MASK_BITS'(1 << bit_index)); + transfer((bit_index + 1) % `BR_CLIENTS, 0, 0, 0, 0); + end + layout(0, 0); + layout(1, 0); + layout(2, 2, 1); + for (int c = 0; c < `BR_CLIENTS; c++) + for (int g = 0; g < 2; g++) transfer(c, 2, g, c, 0, '0, '1, 5); + mapping(2, 6, 2, 1, 0); + for (int g = 0; g < 2; g++) + for (int row = 0; row < `BR_ROWS; row++) + transfer(g, 6, g, row, 1, `BR_DATA_BITS'(row + g * 64), '1); + mapping(2, 6, 2, 0, 1); + for (int g = 0; g < 2; g++) transfer(3, 6, g, 0, 0); + mapping(2, 10, 6, 1, 0, 1); + for (int g = 0; g < 2; g++) begin + transfer(g, 10, g, 0, 1, '1); + transfer(g, 10, g, 0, 0); + end + mapping(2, 10, 6, 0, 1); + layout(2, 0); + layout(0, 1, 1); + layout(1, 1, 1); + layout(0, 1, 0, 1, 1); + for (int g = 0; g < 2; g++) transfer(3, 0, g, 0, 0); + repeat (6) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/suite.toml new file mode 100644 index 000000000..021179d07 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/suite.toml @@ -0,0 +1,19 @@ +[model] +manifest="" +library="" +[[tests]] +name="bank_router" +top="bank_router_tb" +test="protocol_test" +filelist="filelist.f" +timeout_ns=200000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["router_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/tb.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/tb.sv new file mode 100644 index 000000000..6c5e4142c --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/tb.sv @@ -0,0 +1,15 @@ +module bank_router_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + router_if bus (clock); + BankRouter dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual router_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/resources/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/BankRouter.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/BankRouter.scala new file mode 100644 index 000000000..c3e0f40d4 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/BankRouter.scala @@ -0,0 +1,346 @@ +package framework.acc.memdomain.bank_router + +import framework.lib.queue.Queue + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.acc.debug.trace.MemoryEvent + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.memdomain._ +import framework.acc.memdomain.bank_router.blt.BankLayoutTable +import framework.acc.memdomain.prm.PhysicalBankPort +import framework.thread.tlink.BankAccess + +/** Allocate a renamed binding, or commit a new binding and retire its previous one. */ +class BankMapping(frontendParam: FrontendParam) extends Bundle { + val bank = UInt(frontendParam.bank_id_len.W) + val vbank = UInt(frontendParam.bank_id_len.W) + val previous = UInt(frontendParam.bank_id_len.W) + val threadId = UInt(64.W) + val allocate = Bool() + val reuse = Bool() + val retire = Bool() + val write = Bool() +} + +@instantiable +class BankRouter( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val diffTest: Boolean) + extends Module { + val sharedPorts = memParam.sharedPorts + val ownerId = frontendParam.ownerId + + val gpRequests = + if (gpParam.enable) + gpParam.memory.memoryPorts + else 0 + + @public + val io = IO(new Bundle { + + val traceMemory = Output(Vec( + memParam.bankRequestPorts + gpRequests + 2 + memParam.bankNum, + new MemoryEvent + )) + + val requests = Vec(memParam.bankRequestPorts, Flipped(new BankPort(frontendParam, memParam))) + val directRequest = Vec(gpRequests + 2, Flipped(new BankPort(frontendParam, memParam))) + val config = Flipped(Decoupled(new LayoutConfig(frontendParam, memParam))) + val mapping = Flipped(Decoupled(new BankMapping(frontendParam))) + val queryVbank = Input(UInt(frontendParam.bank_id_len.W)) + val queryShared = Input(Bool()) + val queryGroups = Output(UInt(memParam.groupCountWidth.W)) + val ballLayoutRequest = Flipped(Decoupled(new BankLayoutRequest(frontendParam))) + val ballLayoutResponse = Decoupled(new BankLayout(memParam)) + val gpLayoutRequest = + Option.when(gpParam.enable)(Flipped(Decoupled(new BankLayoutRequest(frontendParam)))) + val gpLayoutResponse = + Option.when(gpParam.enable)(Decoupled(new BankLayout(memParam))) + val clearBusy = Output(Bool()) + val physical = Vec(memParam.bankNum, Flipped(new PhysicalBankPort(memParam))) + + val hashes = + if (diffTest) Some(Input(Vec(memParam.bankNum, UInt(32.W)))) + else None + + val bankHashes = + if (diffTest) + Some(Output(Vec(memParam.bankNum, new PhysicalBankHash(frontendParam)))) + else None + + val shared = new BankAccess(frontendParam, memParam) + }) + + val blt: Instance[BankLayoutTable] = Instantiate(new BankLayoutTable(frontendParam, memParam, gpParam, diffTest)) + blt.io.physical <> io.physical + blt.io.hashes.foreach(_ := io.hashes.get) + io.bankHashes.foreach(_ := blt.io.bank_hashes.get) + io.traceMemory := blt.io.traceMemory + io.clearBusy := blt.io.clearBusy + blt.io.query_vbank_id := io.queryVbank + + val aliasCount = + 2 * (memParam.virtualBankCount + (if (gpParam.enable) 2 + else + 0)) + frontendParam.rob_entries + + // Shared layout shape belongs to this thread; physical allocation stays in SHM. + val sharedColumns = RegInit(VecInit(Seq.fill(memParam.virtualBankCount)(0.U( + memParam.groupCountWidth.W + )))) + + val sharedGroups = + RegInit(VecInit(Seq.fill(aliasCount)(0.U(memParam.groupCountWidth.W)))) + when(io.shared.config.fire) { + val config = io.shared.config.bits + when(config.transfer) { + val groups = sharedColumns(config.source_bank_id) +& sharedColumns(config.vbank_id) + sharedColumns(config.source_bank_id) := 0.U + sharedGroups(config.source_bank_id) := 0.U + sharedColumns(config.vbank_id) := groups + sharedGroups(config.vbank_id) := groups + }.otherwise { + val groups = Mux(config.alloc, config.group_id +& 1.U, 0.U) + sharedColumns(config.vbank_id) := groups + sharedGroups(config.vbank_id) := groups + } + } + when(io.shared.mapping.fire) { + val change = io.shared.mapping.bits + when(change.allocate)(sharedGroups(change.vbank) := sharedColumns(change.bank)) + when(change.retire)(sharedGroups(change.previous) := 0.U) + } + val sharedQuery = RegNext(io.queryVbank, 0.U) + val sharedResult = RegNext(sharedGroups(sharedQuery), 0.U) + val queryShared = RegNext(RegNext(io.queryShared, false.B), false.B) + io.queryGroups := Mux(queryShared, sharedResult, blt.io.query_group_count) + + val owners = RegInit(VecInit((0 until aliasCount).map(i => + (i % (memParam.virtualBankCount + (if (gpParam.enable) 2 + else 0))).U( + frontendParam.bank_id_len.W + ) + ))) + + val committed = RegInit(VecInit((0 until memParam.virtualBankCount).map(_.U( + frontendParam.bank_id_len.W + )))) + + // Each client has independent lookup lanes and credits; responses hold under backpressure. + val layoutRequests = Seq(io.ballLayoutRequest) ++ io.gpLayoutRequest.toSeq + val layoutResponses = Seq(io.ballLayoutResponse) ++ io.gpLayoutResponse.toSeq + for (((request, response), client) <- layoutRequests.zip(layoutResponses).zipWithIndex) { + val replies = Instantiate(new Queue(new BankLayout(memParam), 4)) + val outstanding = RegInit(0.U(3.W)) + request.ready := outstanding < 4.U + when(request.fire =/= response.fire) { + outstanding := Mux(request.fire, outstanding + 1.U, outstanding - 1.U) + } + replies.io.enq.valid := RegNext(RegNext(request.fire, false.B), false.B) + for (operand <- 0 until 3) { + val query = blt.io.layout(3 * client + operand) + val bank = request.bits.banks(operand) + query.bank := bank + val delayedBank = RegNext(bank, 0.U) + val sharedCount = RegNext(sharedGroups(delayedBank), 0.U) + val owner = owners(bank) + val shared = memParam.sharedEnable.B && + owner >= frontendParam.shared_bank_id_base.U && + owner < memParam.virtualBankCount.U + val returnedShared = RegNext(RegNext(shared, false.B), false.B) + val enabled = RegNext(RegNext(request.bits.enabled(operand), false.B), false.B) + replies.io.enq.bits.groups(operand) := Mux(enabled, Mux(returnedShared, sharedCount, query.groups), 0.U) + when(request.fire && request.bits.enabled(operand)) { + assert(bank < aliasCount.U, "Layout query bank exceeds the configured namespace") + } + } + assert(!replies.io.enq.valid || replies.io.enq.ready, "Layout response credit was lost") + response <> replies.io.deq + } + + val localBusy = WireDefault(false.B) + + val sharedMapping = + memParam.sharedEnable.B && io.mapping.bits.bank >= frontendParam.shared_bank_id_base.U && + io.mapping.bits.bank < memParam.virtualBankCount.U && + (io.mapping.bits.allocate || io.mapping.bits.retire || io.mapping.bits.write) + + val retireAllowed = !io.mapping.bits.retire || !localBusy + blt.io.mapping.valid := io.mapping.valid && !sharedMapping && retireAllowed + blt.io.mapping.bits := io.mapping.bits + io.shared.mapping.valid := io.mapping.valid && sharedMapping + io.shared.mapping.bits := io.mapping.bits + io.mapping.ready := Mux(sharedMapping, io.shared.mapping.ready, blt.io.mapping.ready && retireAllowed) + when(io.mapping.fire && (io.mapping.bits.allocate || io.mapping.bits.write)) { + owners(io.mapping.bits.vbank) := io.mapping.bits.bank + } + when(io.mapping.fire && io.mapping.bits.retire) { + committed(io.mapping.bits.bank) := io.mapping.bits.vbank + } + + blt.io.config.valid := io.config.valid && !io.config.bits.is_shared + blt.io.config.bits := io.config.bits + io.shared.config.valid := io.config.valid && io.config.bits.is_shared + io.shared.config.bits := io.config.bits + io.config.ready := Mux(io.config.bits.is_shared, io.shared.config.ready, blt.io.config.ready) + + val logical = Wire(Vec(memParam.bankRequestPorts, new BankPort(frontendParam, memParam))) + for (i <- logical.indices) { + logical(i) <> io.requests(i) + } + + if (memParam.sharedEnable && memParam.bankWidth == 128) { + // Reserve channel 0 for Mesh-initiated private-Bank access only when its + // ordinary request/response pair has drained. This preserves the normal + // AccPipe route and prevents one-cycle SRAM responses from being lost. + val normal = io.requests(0) + val routed = logical(0) + val local = io.shared.local + val normalReadPending = RegInit(false.B) + val normalWritePending = RegInit(false.B) + val localPending = RegInit(false.B) + localBusy := localPending + val localIsWrite = RegInit(false.B) + val localTag = Reg(chiselTypeOf(local.request.bits.tag)) + val normalActive = normalReadPending || normalWritePending || + normal.read.req.valid || normal.write.req.valid + val localRequest = local.request.valid && !localPending && !normalActive + val localRead = localRequest && !local.request.bits.write + val localWrite = localRequest && local.request.bits.write + + routed.bank_id := Mux(localRequest, committed(local.request.bits.bank), normal.bank_id) + routed.group_id := Mux(localRequest, 0.U, normal.group_id) + routed.is_shared := Mux(localRequest, false.B, normal.is_shared) + routed.thread_id := Mux(localRequest, ownerId.U(64.W), normal.thread_id) + routed.rob_id := Mux(localRequest, 0.U, normal.rob_id) + routed.inst_id := Mux(localRequest, 0.U, normal.inst_id) + routed.read.req.valid := localRead || (normal.read.req.valid && !localPending) + routed.read.req.bits.addr := Mux(localRead, local.request.bits.addr, normal.read.req.bits.addr) + routed.write.req.valid := localWrite || (normal.write.req.valid && !localPending) + routed.write.req.bits.addr := Mux(localWrite, local.request.bits.addr, normal.write.req.bits.addr) + routed.write.req.bits.data := Mux(localWrite, local.request.bits.data, normal.write.req.bits.data) + routed.write.req.bits.mask := Mux(localWrite, VecInit(local.request.bits.mask.asBools), normal.write.req.bits.mask) + normal.read.req.ready := !localPending && routed.read.req.ready + normal.write.req.ready := !localPending && routed.write.req.ready + local.request.ready := !normalActive && !localPending && + Mux(local.request.bits.write, routed.write.req.ready, routed.read.req.ready) + + routed.read.resp.ready := Mux(localPending && !localIsWrite, local.response.ready, normal.read.resp.ready) + routed.write.resp.ready := Mux(localPending && localIsWrite, local.response.ready, normal.write.resp.ready) + normal.read.resp.valid := routed.read.resp.valid && !localPending + normal.read.resp.bits := routed.read.resp.bits + normal.write.resp.valid := routed.write.resp.valid && !localPending + normal.write.resp.bits := routed.write.resp.bits + local.response.valid := localPending && Mux(localIsWrite, routed.write.resp.valid, routed.read.resp.valid) + local.response.bits.data := routed.read.resp.bits.data + local.response.bits.tag := localTag + local.response.bits.error := false.B + + when(normal.read.req.fire)(normalReadPending := true.B) + when(normal.read.resp.fire)(normalReadPending := false.B) + when(normal.write.req.fire)(normalWritePending := true.B) + when(normal.write.resp.fire)(normalWritePending := false.B) + when(local.request.fire) { + assert(local.request.bits.bank <= frontendParam.vbank_id_upper_bound.U) + assert(local.request.bits.addr < memParam.bankEntries.U) + assert(!local.request.bits.write || local.request.bits.mask.andR) + localPending := true.B + localIsWrite := local.request.bits.write + localTag := local.request.bits.tag + } + when(local.response.fire)(localPending := false.B) + } else { + io.shared.local.request.ready := false.B + io.shared.local.response.valid := false.B + io.shared.local.response.bits := 0.U.asTypeOf(io.shared.local.response.bits) + } + + val routedRequests = logical.toSeq ++ io.directRequest.toSeq + require( + !memParam.sharedEnable || io.shared.requests.size >= routedRequests.size, + "Shared-bank channels must cover Ball, GP and DMA request ports" + ) + val privateRequests = blt.io.mem_req.toSeq ++ blt.io.kernel_req.toSeq + for (i <- routedRequests.indices) { + val request = routedRequests(i) + val memory = privateRequests(i) + val owner = owners(request.bank_id) + val shared = + memParam.sharedEnable.B && owner >= frontendParam.shared_bank_id_base.U && + owner < memParam.virtualBankCount.U + memory <> request + memory.is_shared := false.B + if (i < io.shared.requests.size) { + val remote = io.shared.requests(i) + // Private and shared responses may have different latency. Keep one route per accepted transfer. + val readOwners = Instantiate(new Queue(Bool(), 8)) + val writeOwners = Instantiate(new Queue(Bool(), 8)) + remote.bank_id := request.bank_id + remote.group_id := request.group_id + remote.thread_id := request.thread_id + remote.rob_id := request.rob_id + remote.inst_id := request.inst_id + remote.is_shared := true.B + + val readReady = Mux(shared, remote.read.req.ready, memory.read.req.ready) + memory.read.req.valid := request.read.req.valid && !shared && readOwners.io.enq.ready + remote.read.req.valid := request.read.req.valid && shared && readOwners.io.enq.ready + remote.read.req.bits := request.read.req.bits + request.read.req.ready := readReady && readOwners.io.enq.ready + readOwners.io.enq.valid := request.read.req.valid && readReady + readOwners.io.enq.bits := shared + val readShared = readOwners.io.deq.bits + val readValid = Mux(readShared, remote.read.resp.valid, memory.read.resp.valid) + request.read.resp.valid := readOwners.io.deq.valid && readValid + request.read.resp.bits := Mux(readShared, remote.read.resp.bits, memory.read.resp.bits) + memory.read.resp.ready := readOwners.io.deq.valid && !readShared && request.read.resp.ready + remote.read.resp.ready := readOwners.io.deq.valid && readShared && request.read.resp.ready + readOwners.io.deq.ready := readValid && request.read.resp.ready + + val writeReady = Mux(shared, remote.write.req.ready, memory.write.req.ready) + memory.write.req.valid := request.write.req.valid && !shared && writeOwners.io.enq.ready + remote.write.req.valid := request.write.req.valid && shared && writeOwners.io.enq.ready + remote.write.req.bits := request.write.req.bits + request.write.req.ready := writeReady && writeOwners.io.enq.ready + writeOwners.io.enq.valid := request.write.req.valid && writeReady + writeOwners.io.enq.bits := shared + val writeShared = writeOwners.io.deq.bits + val writeValid = Mux(writeShared, remote.write.resp.valid, memory.write.resp.valid) + request.write.resp.valid := writeOwners.io.deq.valid && writeValid + request.write.resp.bits := Mux(writeShared, remote.write.resp.bits, memory.write.resp.bits) + memory.write.resp.ready := writeOwners.io.deq.valid && !writeShared && request.write.resp.ready + remote.write.resp.ready := writeOwners.io.deq.valid && writeShared && request.write.resp.ready + writeOwners.io.deq.ready := writeValid && request.write.resp.ready + } else { + memory.read.req.valid := request.read.req.valid && !shared + memory.write.req.valid := request.write.req.valid && !shared + request.read.req.ready := memory.read.req.ready && !shared + request.write.req.ready := memory.write.req.ready && !shared + when(request.read.req.valid || request.write.req.valid)(assert(!shared, "Shared request has no core port")) + } + } + + for (i <- routedRequests.size until io.shared.requests.size) { + val remote = io.shared.requests(i) + remote.bank_id := 0.U + remote.group_id := 0.U + remote.thread_id := 0.U + remote.rob_id := 0.U + remote.inst_id := 0.U + remote.is_shared := true.B + remote.read.req.valid := false.B + remote.read.req.bits := 0.U.asTypeOf(remote.read.req.bits) + remote.read.resp.ready := false.B + remote.write.req.valid := false.B + remote.write.req.bits := 0.U.asTypeOf(remote.write.req.bits) + remote.write.resp.ready := false.B + } +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/Emit.scala new file mode 100644 index 000000000..b46540f0e --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/Emit.scala @@ -0,0 +1,104 @@ +package framework.acc.memdomain.bank_router + +import chisel3._ +import chisel3.reflect.DataMirror +import java.nio.file.{Files, Paths} +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.gpdomain.configs.GpDomainParam + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val frontend = upickle.default.read[FrontendParam](values("frontend")) + val memory = upickle.default.read[MemDomainParam](values("memory")) + val gp = upickle.default.read[GpDomainParam](values("gp")) + + val defs = scala.collection.mutable.ArrayBuffer[String]( + s"`define BR_BANKS ${memory.bankNum}", + s"`define BR_ROWS ${memory.bankEntries}", + s"`define BR_CLIENTS ${memory.bankRequestPorts + 2}", + s"`define BR_ALIASES ${2 * memory.virtualBankCount + frontend.rob_entries}", + s"`define BR_DATA_BITS ${memory.bankWidth}", + s"`define BR_MASK_BITS ${memory.bankMaskLen}" + ) + + val ports = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + var inOffset = 0; var outOffset = 0 + + def leaves(data: Data, path: String): Seq[(String, Data)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + leaves(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => leaves(field, s"${path}_$index") } + case field => if (field.getWidth > 0) Seq(path -> field) else Seq.empty + } + + Files.createDirectories(Paths.get(directory)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new BankRouter(frontend, memory, gp, values("diffTest").bool) + for ((name, field) <- leaves(dut.io, "")) { + val input = DataMirror.directionOf(field) == ActualDirection.Input + val direction = if (input) "IN" else "OUT"; val signal = if (input) "inputs" else "outputs"; + val offset = if (input) inOffset else outOffset; val width = field.getWidth + defs += s"`define BR_${direction}_${name.toUpperCase}_OFFSET $offset"; + defs += s"`define BR_${direction}_${name.toUpperCase}_WIDTH $width"; offsets(name) = offset + ports += s".io_$name(bus.$signal[$offset +: $width])" + if (input) inOffset += width else outOffset += width + }; dut + }, + args = Array("--target-dir", s"$directory/BankRouter", "--split-verilog") + ) + defs += s"`define BR_INPUT_BITS $inOffset"; defs += s"`define BR_OUTPUT_BITS $outOffset" + val clientNames = + (0 until memory.bankRequestPorts).map(i => s"requests_$i") ++ (0 until 2).map(i => s"directRequest_$i") + + val clientFields = Seq( + "read_req_valid", + "read_req_ready", + "read_req_bits_addr", + "read_resp_valid", + "read_resp_ready", + "read_resp_bits_data", + "write_req_valid", + "write_req_ready", + "write_req_bits_addr", + "write_req_bits_data", + "write_req_bits_mask_0", + "write_resp_valid", + "write_resp_ready", + "write_resp_bits_ok", + "bank_id", + "group_id", + "thread_id" + ) + + val bankFields = Seq( + "sramRead_req_valid", + "sramRead_req_ready", + "sramRead_req_bits_addr", + "sramRead_resp_valid", + "sramRead_resp_ready", + "sramRead_resp_bits_data", + "sramWrite_req_valid", + "sramWrite_req_ready", + "sramWrite_req_bits_addr", + "sramWrite_req_bits_data", + "sramWrite_req_bits_mask_0", + "sramWrite_resp_valid", + "sramWrite_resp_ready", + "sramWrite_resp_bits_ok", + "clear", + "clearing", + "bind" + ) + + def table(names: Seq[String], fields: Seq[String]) = + names.map(n => fields.map(f => offsets(s"${n}_$f")).mkString("'{", ",", "}")).mkString("'{", ",", "}") + defs += s"`define BR_CLIENT_OFFSETS ${table(clientNames, clientFields)}" + defs += s"`define BR_BANK_OFFSETS ${table((0 until memory.bankNum).map(i => s"physical_$i"), bankFields)}" + Files.writeString(Paths.get(directory, "parameters.svh"), defs.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/AccPipe.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/AccPipe.scala new file mode 100644 index 000000000..9919e91c1 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/AccPipe.scala @@ -0,0 +1,102 @@ +package framework.acc.memdomain.bank_router.blt + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.ip.memory.bank.{SramReadIO, SramReadReq, SramReadResp, SramWriteIO, SramWriteReq, SramWriteResp} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class AccPipe(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Module { + + @public + val io = IO(new Bundle { + val sramRead = Flipped(new SramReadIO(memParam)) + val sramWrite = Flipped(new SramWriteIO(memParam)) + + val mem_req = new Bundle { + val read = new SramReadIO(memParam) + val write = new SramWriteIO(memParam) + val bank_id = Input(UInt(frontendParam.bank_id_len.W)) + val group_id = Input(UInt(memParam.groupIdWidth.W)) + val thread_id = Input(UInt(64.W)) + } + + val busy = Output(Bool()) + val group_id = Output(UInt(memParam.groupIdWidth.W)) + val bank_id = Output(UInt(frontendParam.bank_id_len.W)) + val thread_id = Output(UInt(64.W)) + }) + + // Each group has its own physical bank, so no address shifting is needed. + + //group_id output + val group_id_reg = RegInit(0.U(memParam.groupIdWidth.W)) + io.group_id := group_id_reg + + //Bank_id output + val bank_id_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + io.bank_id := bank_id_reg + val thread_id_reg = RegInit(0.U(64.W)) + io.thread_id := thread_id_reg + + val rd_hold = RegInit(false.B) + val rd_data_reg = RegInit(0.U(memParam.bankWidth.W)) + val wr_hold = RegInit(false.B) + val wr_ok_reg = RegInit(false.B) + val busy_reg = RegInit(false.B) + + val canStart = !busy_reg && !rd_hold && !wr_hold + val hasWriteReq = io.mem_req.write.req.valid + val wrReq = canStart && hasWriteReq + val rdReq = canStart && !hasWriteReq && io.mem_req.read.req.valid + + io.sramRead.req.valid := rdReq + io.sramRead.req.bits.addr := io.mem_req.read.req.bits.addr + io.sramRead.resp.ready := !rd_hold + + io.sramWrite.req.valid := wrReq + io.sramWrite.req.bits.addr := io.mem_req.write.req.bits.addr + io.sramWrite.req.bits.data := io.mem_req.write.req.bits.data + io.sramWrite.req.bits.mask := io.mem_req.write.req.bits.mask + io.sramWrite.resp.ready := !wr_hold + + io.mem_req.read.req.ready := canStart && !hasWriteReq && io.sramRead.req.ready + io.mem_req.read.resp.valid := rd_hold + io.mem_req.read.resp.bits.data := rd_data_reg + + io.mem_req.write.req.ready := canStart && io.sramWrite.req.ready + io.mem_req.write.resp.valid := wr_hold + io.mem_req.write.resp.bits.ok := wr_ok_reg + + when(rd_hold && io.mem_req.read.resp.ready) { + rd_hold := false.B + }.elsewhen(io.sramRead.resp.fire) { + rd_hold := true.B + rd_data_reg := io.sramRead.resp.bits.data + } + + when(wr_hold && io.mem_req.write.resp.ready) { + wr_hold := false.B + }.elsewhen(io.sramWrite.resp.fire) { + wr_hold := true.B + wr_ok_reg := io.sramWrite.resp.bits.ok + } + + when(io.mem_req.read.req.fire || io.mem_req.write.req.fire) { + group_id_reg := io.mem_req.group_id + bank_id_reg := io.mem_req.bank_id + thread_id_reg := io.mem_req.thread_id + } + when(io.mem_req.read.req.fire || io.mem_req.write.req.fire) { + busy_reg := true.B + } + when(io.mem_req.read.resp.fire || io.mem_req.write.resp.fire) { + busy_reg := false.B + } + + io.busy := busy_reg +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/BLT.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/BLT.scala new file mode 100644 index 000000000..ca0f2d870 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/BLT.scala @@ -0,0 +1,621 @@ +package framework.acc.memdomain.bank_router.blt + +import framework.lib.arbiter.{RRArbiter} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.memdomain.{LayoutConfig, LayoutQuery} +import framework.acc.memdomain.BankPort +import framework.acc.debug.trace.MemoryEvent +import framework.acc.memdomain.bank_router.BankMapping +import framework.acc.memdomain.prm.PhysicalBankPort +import framework.acc.memdomain.PhysicalBankHash + +class BankRequest(memp: MemDomainParam) extends Bundle { + val write = Bool() + val addr = UInt(16.W) + val data = UInt(memp.bankWidth.W) + val mask = Vec(memp.bankMaskLen, Bool()) +} + +@instantiable +class BankLayoutTable( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val diffTest: Boolean) + extends Module { + + val kernelRequests = + if (gpParam.enable) + gpParam.memory.memoryPorts + else 0 + + val requestCount = memParam.bankRequestPorts + kernelRequests + 2 + + @public + val io = IO(new Bundle { + val traceMemory = + Output(Vec(requestCount + memParam.bankNum, new MemoryEvent)) + val mem_req = Vec(memParam.bankRequestPorts, Flipped(new BankPort(frontendParam, memParam))) + val kernel_req = Vec(kernelRequests + 2, Flipped(new BankPort(frontendParam, memParam))) + val config = Flipped(Decoupled(new LayoutConfig(frontendParam, memParam))) + val mapping = Flipped(Decoupled(new BankMapping(frontendParam))) + + // Query interface for frontend to get group count + val query_vbank_id = Input(UInt(frontendParam.bank_id_len.W)) + val query_group_count = Output(UInt(memParam.groupCountWidth.W)) + val layout = + Vec(3 * (if (gpParam.enable) 2 else 1), Flipped(new LayoutQuery(frontendParam, memParam))) + val clearBusy = Output(Bool()) + val physical = Vec(memParam.bankNum, Flipped(new PhysicalBankPort(memParam))) + + val hashes = + if (diffTest) Some(Input(Vec(memParam.bankNum, UInt(32.W)))) + else None + + val bank_hashes = + if (diffTest) + Some(Output(Vec(memParam.bankNum, new PhysicalBankHash(frontendParam)))) + else None + + }) + + val requests = io.mem_req.toSeq ++ io.kernel_req.toSeq + + val banks = io.physical.toSeq + val accPipes: Seq[Instance[AccPipe]] = Seq.fill(requestCount)(Instantiate(new AccPipe(frontendParam, memParam))) + + // Per-channel memory trace DPI-C modules to avoid losing simultaneous events + val mtraces = io.traceMemory.take(requestCount) + for (mt <- mtraces) { + mt.is_write := 0.U + mt.is_shared := 0.U + mt.channel := 0.U + mt.thread_id := 0.U + mt.rob_id := 0.U + mt.inst_id := 0.U + mt.vbank_id := 0.U + mt.pbank_id := 0.U + mt.group_id := 0.U + mt.addr := 0.U + mt.write_mask := 0.U + mt.data_lo := 0.U + mt.data_hi := 0.U + mt.enable := false.B + } + + // ----------------------------------------------------------------------------- + // Mapping table + // ----------------------------------------------------------------------------- + class MappingTableEntry extends Bundle { + val valid = Bool() + val thread_id = UInt(64.W) + val vbank_id = UInt(frontendParam.bank_id_len.W) + val architectural_id = UInt(frontendParam.bank_id_len.W) + val is_multi = Bool() + val group_id = UInt(memParam.groupIdWidth.W) + } + + val mappingTable = RegInit( + VecInit(Seq.fill(memParam.bankNum)(0.U.asTypeOf(new MappingTableEntry))) + ) + + // The frontend contract reserves [0, vbank_id_upper_bound] for private + // banks. Keep the direct route table at that architectural size. + val privateVbankCount = + 2 * (memParam.virtualBankCount + (if (gpParam.enable) 2 + else + 0)) + frontendParam.rob_entries + + require( + privateVbankCount > 0 && privateVbankCount <= (1 << frontendParam.bank_id_len), + "Private alias IDs exceed bank_id_len" + ) + val groupIndexWidth = log2Up(memParam.bankNum) + val pbankIndexWidth = log2Up(memParam.bankNum) + val privateVbankIdWidth = log2Up(privateVbankCount) + + // A non-multi vbank has exactly one route, so keep that common case as a + // small indexed table. Multi-bank mappings retain their per-group entries + // in mappingTable and use the fallback scan below. This avoids the much + // larger 32x24 valid+pbank register matrix while still removing the full + // mapping scan from the overwhelmingly common single-bank path. + val singleRouteValid = RegInit(VecInit(Seq.fill(privateVbankCount)(false.B))) + + val singleRoutePbank = RegInit( + VecInit(Seq.fill(privateVbankCount)(0.U(pbankIndexWidth.W))) + ) + + // The private namespace is bounded by the frontend contract, while + // multi-bank groups remain represented individually in mappingTable. + val groupCountByVbank = RegInit( + VecInit(Seq.fill(privateVbankCount)(0.U(memParam.groupCountWidth.W))) + ) + + val owners = RegInit(VecInit((0 until privateVbankCount).map(i => + (i % (memParam.virtualBankCount + (if (gpParam.enable) 2 + else 0))).U( + frontendParam.bank_id_len.W + ) + ))) + + val committed = RegInit(VecInit((0 until memParam.virtualBankCount).map(_.U( + frontendParam.bank_id_len.W + )))) + + // Record binding and lifetime changes, including aliases with no active accesses. + for ((entry, physical) <- mappingTable.zipWithIndex) { + val live = entry.valid && true.B + val current = entry.valid && committed(entry.architectural_id) === entry.vbank_id + val previous = RegNext(entry, 0.U.asTypeOf(entry)) + val previousLive = RegNext(live, false.B) + val previousCurrent = RegNext(current, false.B) + val trace = io.traceMemory(requestCount + physical) + trace.is_write := Mux(entry.valid, 2.U, 3.U) + trace.is_shared := 0.U + trace.channel := 0.U + trace.thread_id := entry.thread_id + trace.rob_id := 0.U + trace.inst_id := 0.U + trace.vbank_id := entry.vbank_id + trace.pbank_id := physical.U + trace.group_id := entry.group_id + trace.addr := entry.architectural_id + trace.write_mask := 0.U + trace.data_lo := live.asUInt + trace.data_hi := current.asUInt + trace.enable := (entry.valid || previous.valid) && + (entry.asUInt =/= previous.asUInt || live =/= previousLive || current =/= previousCurrent) + } + + val architecturalColumns = RegInit(VecInit(Seq.fill( + memParam.virtualBankCount + )(0.U(memParam.groupCountWidth.W)))) + + def addEntry( + thread_id: UInt, + vbank_id: UInt, + pbank_id: UInt, + is_multi: Bool, + group_id: UInt + ): Unit = { + val entry = mappingTable(pbank_id) + entry.valid := true.B + entry.thread_id := thread_id + entry.vbank_id := vbank_id + entry.architectural_id := vbank_id + entry.is_multi := is_multi + entry.group_id := group_id + } + + def deleteEntry(vbank_id: UInt): Unit = { + for (i <- 0 until memParam.bankNum) { + when(mappingTable(i).valid && mappingTable(i).architectural_id === vbank_id) { + mappingTable(i).valid := false.B + } + } + } + + // ----------------------------------------------------------------------------- + // Default Value + // ----------------------------------------------------------------------------- + + for (i <- 0 until requestCount) { + accPipes(i).io.mem_req.write <> requests(i).write + accPipes(i).io.mem_req.read <> requests(i).read + accPipes(i).io.mem_req.bank_id := requests(i).bank_id + accPipes(i).io.mem_req.group_id := requests(i).group_id + accPipes(i).io.mem_req.thread_id := requests(i).thread_id + + // Bank-side defaults (only driven when a bank is actually connected) + accPipes(i).io.sramRead.req.ready := false.B + accPipes(i).io.sramRead.resp.valid := false.B + accPipes(i).io.sramRead.resp.bits := DontCare + + accPipes(i).io.sramWrite.req.ready := false.B + accPipes(i).io.sramWrite.resp.valid := false.B + accPipes(i).io.sramWrite.resp.bits := DontCare + } + + banks.zipWithIndex.foreach { + case (bank, _) => + bank.sramRead.req.valid := false.B + bank.sramRead.req.bits := DontCare + bank.sramRead.resp.ready := true.B + + bank.sramWrite.req.valid := false.B + bank.sramWrite.req.bits := DontCare + bank.sramWrite.resp.ready := true.B + bank.clear := false.B + bank.bind := false.B + } + io.clearBusy := VecInit(banks.map(_.clearing)).asUInt.orR + + val realloc = io.config.bits.group_id === 0.U + + val freePbankMask = + VecInit(mappingTable.map(entry => + !entry.valid || + (realloc && entry.architectural_id === io.config.bits.vbank_id) + )) + + val hasFreePbank = freePbankMask.asUInt.orR + + val configPbank = + if (memParam.bankLayoutEnable) PriorityEncoder(freePbankMask) + else (io.config.bits.vbank_id +& io.config.bits.group_id)(pbankIndexWidth - 1, 0) + + val configBankAvailable = + if (memParam.bankLayoutEnable) hasFreePbank + else freePbankMask(configPbank) + + if (!memParam.bankLayoutEnable) { + when(io.config.valid) { + assert(!io.config.bits.transfer, "Fixed bank layout requires compiler-lowered transfers") + when(io.config.bits.alloc) { + assert( + io.config.bits.vbank_id +& io.config.bits.group_id < memParam.bankNum.U, + "Fixed bank layout allocation exceeds physical bank count" + ) + } + } + when(io.mapping.valid && io.mapping.bits.allocate) { + assert( + io.mapping.bits.bank +& architecturalColumns( + io.mapping.bits.bank + ) <= memParam.bankNum.U, + "Fixed bank alias exceeds physical bank count" + ) + } + } + io.config.ready := !io.config.bits.alloc || configBankAvailable + when(io.config.valid && io.config.bits.alloc && !configBankAvailable) { + assert(false.B, "BLT allocation failed: no free physical bank\n") + } + + // ----------------------------------------------------------------------------- + // Bank Alloc/Release + // ----------------------------------------------------------------------------- + + val change = io.mapping.bits + val columns = architecturalColumns(change.bank) + val freeBanks = + VecInit(mappingTable.map(entry => !entry.valid || (change.reuse && entry.vbank_id === change.previous))).asUInt + io.mapping.ready := !io.config.valid && (!change.allocate || + (if (memParam.bankLayoutEnable) PopCount(freeBanks) >= columns else true.B)) + when(io.mapping.fire && change.allocate) { + assert( + columns =/= 0.U && columns <= memParam.bankNum.U, + "Renamed bank requires an allocated layout" + ) + owners(change.vbank) := change.bank + } + when(io.mapping.fire && change.retire) { + committed(change.bank) := change.vbank + singleRouteValid(change.previous) := false.B + groupCountByVbank(change.previous) := 0.U + for (entry <- mappingTable) { + when(entry.valid && entry.vbank_id === change.previous)(entry.valid := false.B) + } + } + for (bank <- 0 until memParam.bankNum) { + when(io.mapping.fire && change.write && !change.allocate && + mappingTable(bank).valid && mappingTable(bank).vbank_id === change.vbank) { + io.physical(bank).bind := true.B + } + } + + val remaining = Wire(Vec( + memParam.bankNum + 1, + UInt(memParam.bankNum.W) + )) + + remaining(0) := freeBanks + for (group <- 0 until memParam.bankNum) { + val physical = + if (memParam.bankLayoutEnable) PriorityEncoder(remaining(group)) + else (change.bank + group.U)(pbankIndexWidth - 1, 0) + remaining(group + 1) := remaining(group) & ~UIntToOH( + physical, + memParam.bankNum + ) + when(io.mapping.fire && change.allocate && group.U < columns) { + if (memParam.bankLayoutEnable) { + addEntry(change.threadId, change.vbank, physical, columns > 1.U, group.U) + mappingTable(physical).architectural_id := change.bank + } else { + assert( + mappingTable(physical).valid && mappingTable(physical).architectural_id === change.bank, + "Fixed bank alias requires its allocated physical slot" + ) + mappingTable(physical).vbank_id := change.vbank + } + + for (bank <- 0 until memParam.bankNum) { + when(physical === bank.U)(io.physical(bank).bind := true.B) + } + + when(group.U === 0.U) { + singleRoutePbank(change.vbank) := physical + singleRouteValid(change.vbank) := columns === 1.U + groupCountByVbank(change.vbank) := columns + } + } + } + + when(io.config.fire) { + val vbank = io.config.bits.vbank_id + val vbankIdx = vbank(privateVbankIdWidth - 1, 0) + when(io.config.bits.transfer) { + val source = io.config.bits.source_bank_id + val sourceIdx = source(privateVbankIdWidth - 1, 0) + val sourceCount = architecturalColumns(source) + val targetCount = architecturalColumns(vbank) + val sourceRoutes = VecInit(mappingTable.map(entry => entry.valid && entry.architectural_id === source)) + val totalCount = targetCount +& sourceCount + assert(source =/= vbank && sourceCount =/= 0.U, "Private bank transfer requires a distinct allocated source") + assert( + totalCount <= memParam.bankNum.U, + "Private bank transfer exceeds physical bank count" + ) + for (entry <- mappingTable) { + when(entry.valid && entry.architectural_id === source) { + entry.vbank_id := vbank + entry.architectural_id := vbank + entry.group_id := targetCount + entry.group_id + entry.is_multi := totalCount > 1.U + } + when(entry.valid && entry.architectural_id === vbank) { + entry.vbank_id := vbank + entry.is_multi := totalCount > 1.U + } + } + singleRouteValid(sourceIdx) := false.B + singleRouteValid(vbankIdx) := totalCount === 1.U + when(totalCount === 1.U)(singleRoutePbank(vbankIdx) := PriorityEncoder(sourceRoutes.asUInt)) + groupCountByVbank(sourceIdx) := 0.U + groupCountByVbank(vbankIdx) := totalCount + architecturalColumns(source) := 0.U + architecturalColumns(vbank) := totalCount + }.elsewhen(io.config.bits.alloc) { + val freePbank = configPbank + + for (i <- 0 until memParam.bankNum) { + io.physical(i).bind := freePbank === i.U + } + + // The bank hash already treats a newly bound bank as zero, so the clear writes need no hash update. + for (i <- 0 until memParam.bankNum) { + when(io.config.bits.clear && freePbank === i.U)(banks(i).clear := true.B) + } + // Match bemu mset: realloc of the same vbank frees prior physical banks first. + // Configer emits one fire per group; only group 0 drops the old mapping. + when(io.config.bits.group_id === 0.U) { + deleteEntry(io.config.bits.vbank_id) + singleRouteValid(vbankIdx) := false.B + } + addEntry( + io.config.bits.thread_id, + io.config.bits.vbank_id, + freePbank, + io.config.bits.is_multi, + io.config.bits.group_id + ) + when(!io.config.bits.is_multi) { + singleRoutePbank(vbankIdx) := freePbank + singleRouteValid(vbankIdx) := true.B + } + groupCountByVbank(vbankIdx) := Mux(io.config.bits.is_multi, io.config.bits.group_id +& 1.U, 1.U) + architecturalColumns(vbank) := Mux(io.config.bits.is_multi, io.config.bits.group_id +& 1.U, 1.U) + }.otherwise { + deleteEntry(io.config.bits.vbank_id) + singleRouteValid(vbankIdx) := false.B + groupCountByVbank(vbankIdx) := 0.U + architecturalColumns(vbank) := 0.U + } + } + + // ----------------------------------------------------------------------------- + // Query interface: return group count for a given vbank_id + // ----------------------------------------------------------------------------- + val queryVbankId = RegNext(io.query_vbank_id, 0.U) + val queryVbank = queryVbankId(privateVbankIdWidth - 1, 0) + io.query_group_count := RegNext(groupCountByVbank(queryVbank), 0.U) + for (port <- io.layout) { + val bank = RegNext(port.bank(privateVbankIdWidth - 1, 0), 0.U) + port.groups := RegNext(groupCountByVbank(bank), 0.U) + } + + // ----------------------------------------------------------------------------- + // Connect AccPipe and Banks + // ----------------------------------------------------------------------------- + def emitTrace( + ch: Int, + isWrite: UInt, + pbankId: UInt, + addr: UInt, + writeMask: UInt, + dataLo: UInt, + dataHi: UInt, + en: Bool + ): Unit = { + mtraces(ch).is_write := isWrite + mtraces(ch).is_shared := requests(ch).is_shared.asUInt + mtraces(ch).channel := ch.U + mtraces(ch).thread_id := requests(ch).thread_id + mtraces(ch).rob_id := requests(ch).rob_id + mtraces(ch).inst_id := requests(ch).inst_id + mtraces(ch).vbank_id := requests(ch).bank_id + mtraces(ch).pbank_id := pbankId + mtraces(ch).group_id := requests(ch).group_id + mtraces(ch).addr := addr + mtraces(ch).write_mask := writeMask + mtraces(ch).data_lo := dataLo + mtraces(ch).data_hi := dataHi + mtraces(ch).enable := en + } + + // The route is held for the complete AccPipe transaction. Keep the route + // vectors in one place so the bank-side command mux and response demux share + // exactly the same ownership predicate. + val activeRouteOHs = Seq.fill(requestCount)(Wire(UInt(memParam.bankNum.W))) + + for (i <- 0 until requestCount) { + val routePbankReg = RegInit(0.U(pbankIndexWidth.W)) + val routeValidReg = RegInit(false.B) + val routePending = RegInit(false.B) + val requestVbank = requests(i).bank_id + val requestVbankIdx = requestVbank(privateVbankIdWidth - 1, 0) + val requestGroup = requests(i).group_id(groupIndexWidth - 1, 0) + val singleRoute = singleRoutePbank(requestVbankIdx) + val singleValid = singleRouteValid(requestVbankIdx) + val multiRouteMatch = VecInit(mappingTable.map(entry => + entry.valid && entry.is_multi && + entry.vbank_id === requestVbank && entry.group_id === requestGroup + )) + val multiRoute = PriorityEncoder(multiRouteMatch) + val multiValid = multiRouteMatch.asUInt.orR + val requestRoute = + if (memParam.bankLayoutEnable) Mux(singleValid, singleRoute, multiRoute) + else (owners(requestVbankIdx) + requestGroup)(pbankIndexWidth - 1, 0) + val requestRouteValid = + if (memParam.bankLayoutEnable) singleValid || multiValid + else mappingTable(requestRoute).valid && + mappingTable(requestRoute).architectural_id === owners(requestVbankIdx) && + mappingTable(requestRoute).group_id === requestGroup + // Route lookup is deliberately one cycle ahead of the SRAM request. The + // route is captured while the AccPipe is idle, then held through the + // entire request/response lifetime. This removes the live mapping-table + // lookup from every wide bank-side mux select while preserving one request + // per channel per cycle once the pipe is running. + val requestSeen = requests(i).read.req.valid || requests(i).write.req.valid + val activeRoute = routePbankReg + val requestActive = routePending || accPipes(i).io.busy + val activeRouteValid = routeValidReg && requestActive + val activeRouteOH = UIntToOH(activeRoute, memParam.bankNum) & + Fill(memParam.bankNum, activeRouteValid) + + activeRouteOHs(i) := activeRouteOH + + when(!routePending && !accPipes(i).io.busy && requestSeen && requestRouteValid) { + routePbankReg := requestRoute + routeValidReg := true.B + routePending := true.B + } + when(requests(i).read.req.fire || requests(i).write.req.fire) { + routePending := false.B + } + + // Requests are attributed once the selected SRAM accepts them. + when(accPipes(i).io.sramRead.req.fire) { + emitTrace(i, 0.U, activeRoute, accPipes(i).io.sramRead.req.bits.addr, 0.U, 0.U, 0.U, true.B) + } + + // Arrival trace: the write has crossed arbitration and is accepted by the + // selected SPM SRAM port. Keep attribution metadata from the same AccPipe + // transaction and data from the bank-side request. + when(accPipes(i).io.sramWrite.req.fire) { + emitTrace( + i, + 1.U, + activeRoute, + accPipes(i).io.sramWrite.req.bits.addr, + accPipes(i).io.sramWrite.req.bits.mask.asUInt, + accPipes(i).io.sramWrite.req.bits.data(63, 0), + accPipes(i).io.sramWrite.req.bits.data(127, 64), + true.B + ) + } + + } + + // Fixed GP request slots share the physical-bank arbitration with ordinary + // channels. Capture the accepted request's owner for the one-cycle SRAM + // response; a stalled requester must never receive another owner's response. + val responseRoutes = + Seq.fill(requestCount)(WireDefault(0.U(memParam.bankNum.W))) + val responseRouteBits = + Seq.fill(requestCount)(Wire(Vec(memParam.bankNum, Bool()))) + + for (j <- 0 until memParam.bankNum) { + val arb = Instantiate(new RRArbiter( + new BankRequest(memParam), + requestCount + )) + + for (i <- 0 until requestCount) { + val pipe = accPipes(i) + arb.io.in(i).valid := activeRouteOHs(i)(j) && + (pipe.io.sramRead.req.valid || pipe.io.sramWrite.req.valid) + arb.io.in(i).bits.write := pipe.io.sramWrite.req.valid + arb.io.in(i).bits.addr := Mux( + pipe.io.sramWrite.req.valid, + pipe.io.sramWrite.req.bits.addr, + pipe.io.sramRead.req.bits.addr + ) + arb.io.in(i).bits.data := pipe.io.sramWrite.req.bits.data + arb.io.in(i).bits.mask := pipe.io.sramWrite.req.bits.mask + when(activeRouteOHs(i)(j)) { + pipe.io.sramRead.req.ready := arb.io.in(i).ready && !arb.io.in(i).bits.write + pipe.io.sramWrite.req.ready := arb.io.in(i).ready && arb.io.in(i).bits.write + } + responseRouteBits(i)(j) := RegNext(arb.io.in(i).fire, false.B) + } + + val bank = banks(j) + bank.sramRead.req.valid := arb.io.out.valid && !arb.io.out.bits.write + bank.sramRead.req.bits.addr := arb.io.out.bits.addr + bank.sramWrite.req.valid := arb.io.out.valid && arb.io.out.bits.write + bank.sramWrite.req.bits.addr := arb.io.out.bits.addr + bank.sramWrite.req.bits.data := arb.io.out.bits.data + bank.sramWrite.req.bits.mask := arb.io.out.bits.mask + arb.io.out.ready := Mux(arb.io.out.bits.write, bank.sramWrite.req.ready, bank.sramRead.req.ready) + + bank.sramRead.resp.ready := VecInit((0 until requestCount).map { i => + responseRouteBits(i)(j) && accPipes(i).io.sramRead.resp.ready + }).asUInt.orR + bank.sramWrite.resp.ready := VecInit((0 until requestCount).map { i => + responseRouteBits(i)(j) && accPipes(i).io.sramWrite.resp.ready + }).asUInt.orR + when(bank.sramRead.resp.valid)(assert(bank.sramRead.resp.ready)) + when(bank.sramWrite.resp.valid)(assert(bank.sramWrite.resp.ready)) + } + + for (i <- 0 until requestCount) { + responseRoutes(i) := responseRouteBits(i).asUInt + val readRespSel = VecInit((0 until memParam.bankNum).map { j => + responseRoutes(i)(j) && banks(j).sramRead.resp.valid + }).asUInt + val writeRespSel = VecInit((0 until memParam.bankNum).map { j => + responseRoutes(i)(j) && banks(j).sramWrite.resp.valid + }).asUInt + accPipes(i).io.sramRead.resp.valid := readRespSel.orR + accPipes(i).io.sramRead.resp.bits.data := (0 until memParam.bankNum).map { + j => + Fill(memParam.bankWidth, responseRoutes(i)(j)) & banks( + j + ).sramRead.resp.bits.data + }.reduce(_ | _) + accPipes(i).io.sramWrite.resp.valid := writeRespSel.orR + accPipes(i).io.sramWrite.resp.bits.ok := writeRespSel.orR + } + + io.bank_hashes.foreach { states => + for (i <- 0 until memParam.bankNum) { + states(i).valid := mappingTable(i).valid + states(i).threadId := mappingTable(i).thread_id + states(i).vbankId := (if (memParam.bankLayoutEnable) + mappingTable(i).vbank_id + else mappingTable(i).architectural_id) + states(i).pbankId := i.U + states(i).groupId := mappingTable(i).group_id + states(i).statusHash := io.hashes.get(i) + } + } +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/configs/MemDomainParam.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/configs/MemDomainParam.scala new file mode 100644 index 000000000..511f834df --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/configs/MemDomainParam.scala @@ -0,0 +1,91 @@ +package framework.acc.memdomain.configs + +import buckyball.config.{MemDomainConfig, SharedMemConfig} + +import upickle.default._ + +/** + * MemDomain Parameter + */ +case class MemDomainParam( + sharedPorts: Int, + bankNum: Int, + bankLayoutEnable: Boolean, + bankWidth: Int, + bankEntries: Int, + bankMaskLen: Int, + virtualBankCount: Int, + sharedEnable: Boolean, + sharedEntries: Int, + sharedBankEntries: Int, + sharedBankNum: Int, + sharedInputChannels: Int, + sharedDefaultGroupCount: Int, + bankRequestPorts: Int, + dma_buswidth: Int, + memAddrLen: Int, + mmioEnable: Boolean, + mmioBankNum: Int, + mmioBankEntries: Int, + mmioBankWidth: Int, + mmioReadWidth: Int) { + + if (sharedEnable) { + require(sharedBankEntries >= 2 && sharedBankEntries <= 65536) + require(sharedEntries == sharedBankNum * sharedBankEntries, "Shared pool size must equal banks times rows per bank") + } + + // MMIO derived values + val mmioBankBytes: Int = mmioBankEntries * (mmioBankWidth / 8) + val mmioTotalBytes: Int = mmioBankNum * mmioBankBytes + val vbankIdWidth: Int = chisel3.util.log2Up(virtualBankCount) + val maxGroupCount: Int = math.max(bankNum, if (sharedEnable) sharedBankNum else bankNum) + val groupCountWidth: Int = chisel3.util.log2Up(maxGroupCount + 1) + val groupIdWidth: Int = chisel3.util.log2Up(maxGroupCount) +} + +object MemDomainParam { + + def fromProto( + p: MemDomainConfig, + shared: SharedMemConfig, + virtualBanks: Int, + participants: Int + ): MemDomainParam = { + require(p.hasBank && p.hasDma && p.hasMmio && p.hasMem, "Memory domain configuration is incomplete") + if (shared.getEnable) require(participants > 0 && shared.getInputChannels % participants == 0) + val bank = p.getBank + require(bank.hasLayoutEnable, "Bank layout policy is required") + if (shared.getEnable) { + require(shared.getBankEntries > 0 && shared.getEntries % shared.getBankEntries == 0) + require(shared.getBankWidth == bank.getWidth, "Private and shared bank interfaces must agree") + } + val mmio = p.getMmio + MemDomainParam( + if (shared.getEnable) shared.getInputChannels / participants else 0, + bank.getNum, + bank.getLayoutEnable, + bank.getWidth, + bank.getEntries, + bank.getMaskLen, + virtualBanks, + shared.getEnable, + shared.getEntries, + shared.getBankEntries, + if (shared.getEnable) shared.getEntries / shared.getBankEntries else 0, + shared.getInputChannels, + shared.getDefaultGroupCount, + bank.getRequestPorts, + p.getDma.getBusWidth, + p.getMem.getAddrLen, + mmio.getEnable, + mmio.getBankNum, + mmio.getBankEntries, + mmio.getBankWidth, + mmio.getReadWidth + ) + } + + implicit val rw: ReadWriter[MemDomainParam] = macroRW + +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/Ctrl.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/Ctrl.scala new file mode 100644 index 000000000..7b6e9775f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/Ctrl.scala @@ -0,0 +1,102 @@ +package framework.acc.memdomain + +import framework.lib.arbiter.{Arbiter} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.frontend.scheduler.MemoryIssue +import framework.acc.memdomain.toddr.DmaStatus + +class Request(val frontendParam: FrontendParam, val memParam: MemDomainParam, val permissionParam: PmpParams) + extends Bundle { + val context = new MemoryContext(permissionParam) + val instId = UInt(64.W) + val cmd = new DecodedCommand(frontendParam, memParam) + val rob_id = UInt(log2Up(frontendParam.rob_entries).W) +} + +class Response(val frontendParam: FrontendParam) extends Bundle { + val fault = new DmaStatus + val rob_id = UInt(log2Up(frontendParam.rob_entries).W) +} + +@instantiable +class Ctrl(val frontendParam: FrontendParam, val memParam: MemDomainParam, val permissionParam: PmpParams) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val ownerId = frontendParam.ownerId + + @public + val io = IO(new Bundle { + val issue = Flipped(Decoupled(new MemoryIssue(frontendParam, memParam, permissionParam))) + val complete = Decoupled(new Response(frontendParam)) + val request = Decoupled(new Request(frontendParam, memParam, permissionParam)) + val response = Flipped(Decoupled(new Response(frontendParam))) + val config = Decoupled(new LayoutConfig(frontendParam, memParam)) + val clearBusy = Input(Bool()) + val mvover = new MvoverPort + }) + + val decoder: Instance[Decoder] = Instantiate(new Decoder(frontendParam, memParam)) + val configer: Instance[Configer] = Instantiate(new Configer(frontendParam, memParam, permissionParam)) + + val operation = io.issue.bits.operation + val isMvover = operation.cmd.cmd.raw_inst(31, 25) === MvoverISA.Funct.U + val mvoverPending = RegInit(false.B) + val mvoverRobId = Reg(chiselTypeOf(operation.rob_id)) + + io.mvover.command.valid := io.issue.valid && isMvover && !mvoverPending + io.mvover.command.bits.sourceThread := operation.cmd.cmd.rs1Data(7, 0) + io.mvover.command.bits.targetThread := operation.cmd.cmd.rs1Data(15, 8) + io.mvover.command.bits.sourceBank := operation.cmd.cmd.rs1Data(25, 16) + io.mvover.command.bits.targetBank := operation.cmd.cmd.rs1Data(35, 26) + io.mvover.command.bits.sourceAddr := operation.cmd.cmd.rs2Data(15, 0) + io.mvover.command.bits.targetAddr := operation.cmd.cmd.rs2Data(31, 16) + io.mvover.command.bits.rows := operation.cmd.cmd.rs2Data(47, 32) +& 1.U + when(io.mvover.command.fire) { + mvoverPending := true.B + mvoverRobId := operation.rob_id + } + + decoder.io.request.valid := io.issue.valid && !isMvover && !mvoverPending + decoder.io.request.bits := operation.cmd + io.issue.ready := !mvoverPending && Mux(isMvover, io.mvover.command.ready, decoder.io.request.ready) + + val decoded = decoder.io.command + val control = + decoded.bits.is_config && decoded.bits.bank_id < memParam.virtualBankCount.U + configer.io.request.valid := decoded.valid && control + configer.io.request.bits.cmd := decoded.bits + configer.io.request.bits.rob_id := operation.rob_id + configer.io.request.bits.context := io.issue.bits.context + configer.io.request.bits.instId := operation.instId + io.request.valid := decoded.valid && !control + io.request.bits.cmd := decoded.bits + io.request.bits.rob_id := operation.rob_id + io.request.bits.context := io.issue.bits.context + io.request.bits.instId := operation.instId + decoded.ready := Mux(control, configer.io.request.ready, io.request.ready) + + configer.io.clearBusy := io.clearBusy + io.config <> configer.io.config + + val completion = Instantiate(new Arbiter(new Response(frontendParam), 2)) + completion.io.in(0) <> io.response + completion.io.in(1) <> configer.io.response + + val moved = mvoverPending && io.mvover.completion.valid + io.complete.valid := moved || completion.io.out.valid + io.complete.bits.fault := Mux(moved, 0.U.asTypeOf(new DmaStatus), completion.io.out.bits.fault) + io.complete.bits.rob_id := Mux(moved, mvoverRobId, completion.io.out.bits.rob_id) + completion.io.out.ready := io.complete.ready && !moved + io.mvover.completion.ready := io.complete.ready && mvoverPending + when(io.mvover.completion.fire) { + assert(!io.mvover.completion.bits, "Bank movement violated its admitted layout") + mvoverPending := false.B + } +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/configer/Configer.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/configer/Configer.scala new file mode 100644 index 000000000..29db4d710 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/configer/Configer.scala @@ -0,0 +1,138 @@ +package framework.acc.memdomain + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ + +class LayoutConfig(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Bundle { + val vbank_id = Output(UInt(frontendParam.bank_id_len.W)) + val is_shared = Output(Bool()) + val is_multi = Output(Bool()) + val alloc = Output(Bool()) + val transfer = Output(Bool()) + val source_bank_id = Output(UInt(frontendParam.bank_id_len.W)) + val clear = Output(Bool()) + val group_id = Output(UInt(memParam.groupIdWidth.W)) + val thread_id = Output(UInt(64.W)) +} + +import framework.acc.memdomain.{Request, Response} +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class Configer(val frontendParam: FrontendParam, val memParam: MemDomainParam, val permissionParam: PmpParams) + extends Module { + val ownerId = frontendParam.ownerId + + val rob_id_width = log2Up(frontendParam.rob_entries) + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new Request(frontendParam, memParam, permissionParam))) + val response = Decoupled(new Response(frontendParam)) + val config = Decoupled(new LayoutConfig(frontendParam, memParam)) + // A physical bank is still clearing; MSET.clear completes only after it drains. + val clearBusy = Input(Bool()) + }) + + val idle :: config :: clearWait :: resp :: Nil = Enum(4) + + val state = RegInit(idle) + val transfer_reg = RegInit(false.B) + val source_bank_id_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + val alloc_reg = RegInit(false.B) + val is_shared_reg = RegInit(false.B) + val col_reg = RegInit(0.U(memParam.groupCountWidth.W)) + val clear_reg = RegInit(false.B) + val vbank_id_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + val rob_id_reg = RegInit(0.U(rob_id_width.W)) + val counter = RegInit(0.U(memParam.groupCountWidth.W)) + + io.config.bits.is_multi := false.B + io.config.bits.is_shared := false.B + io.config.bits.transfer := false.B + io.config.bits.source_bank_id := 0.U + io.config.bits.alloc := false.B + io.config.bits.clear := false.B + io.config.bits.vbank_id := 0.U(frontendParam.bank_id_len.W) + io.config.bits.group_id := 0.U + io.config.bits.thread_id := ownerId.U(64.W) + io.config.valid := false.B + io.response.valid := false.B + io.response.bits := 0.U.asTypeOf(io.response.bits) + io.response.bits.fault := 0.U.asTypeOf(io.response.bits.fault) + io.response.bits.rob_id := 0.U(rob_id_width.W) + + io.request.ready := state === idle + + when(state === idle) { + when(io.request.valid) { + when(io.request.fire) { + val rawCol = io.request.bits.cmd.special(9, 5) + val alloc = io.request.bits.cmd.special(10) + val fullCol = + if (memParam.sharedEnable) { + Mux( + io.request.bits.cmd.is_shared, + memParam.sharedBankNum.U(col_reg.getWidth.W), + memParam.bankNum.U(col_reg.getWidth.W) + ) + } else { + memParam.bankNum.U(col_reg.getWidth.W) + } + + state := config + col_reg := Mux( + io.request.bits.cmd.transfer, + 1.U, + Mux(alloc && rawCol === 0.U, fullCol, Mux(rawCol > 1.U, rawCol, 1.U)) + ) + transfer_reg := io.request.bits.cmd.transfer + source_bank_id_reg := io.request.bits.cmd.source_bank_id + alloc_reg := alloc + clear_reg := io.request.bits.cmd.clear + is_shared_reg := io.request.bits.cmd.is_shared + vbank_id_reg := io.request.bits.cmd.bank_id + rob_id_reg := io.request.bits.rob_id + assert( + !(io.request.bits.cmd.clear && io.request.bits.cmd.is_shared), + "MSET clear is currently supported for private banks only" + ) + } + } + + }.elsewhen(state === config) { + io.config.bits.is_multi := col_reg > 1.U + io.config.bits.is_shared := is_shared_reg + io.config.bits.transfer := transfer_reg + io.config.bits.source_bank_id := source_bank_id_reg + io.config.bits.alloc := alloc_reg + io.config.bits.clear := clear_reg && alloc_reg + io.config.bits.vbank_id := vbank_id_reg + io.config.bits.group_id := counter(memParam.groupIdWidth - 1, 0) + io.config.valid := true.B + + when(io.config.fire) { + when(counter === col_reg - 1.U) { + counter := 0.U + state := Mux(clear_reg && alloc_reg, clearWait, resp) + }.otherwise { + counter := counter + 1.U + } + } + }.elsewhen(state === clearWait) { + // Each bound bank started clearing when its allocation fired. + when(!io.clearBusy)(state := resp) + }.elsewhen(state === resp) { + io.response.valid := true.B + io.response.bits.rob_id := rob_id_reg + + when(io.response.fire) { + state := idle + counter := 0.U + } + } +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/DISA.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/DISA.scala new file mode 100644 index 000000000..a7c19ce26 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/DISA.scala @@ -0,0 +1,17 @@ +package framework.acc.memdomain + +import chisel3._ +import chisel3.util._ + +object DISA { + val MVOVER_BITPAT = BitPat("b0001101") + val MVIN_MMIO_BITPAT = BitPat("b0100011") + val MSET_FUNCT = 0x20 + // enable=010 (1 write), opcode group + val MSET_BITPAT = BitPat("b0100000") // 32 (0x20) — enable=010, opcode=0 + val MVIN_BITPAT = BitPat("b0100001") // 33 (0x21) — enable=010, opcode=1 + val MVIN_2D_BITPAT = BitPat("b0100010") // 34 (0x22) — 2D DMA into bank rows + val MVIN_KERNEL_BITPAT = BitPat("b0101100") + // enable=001 (1 read), opcode group + val MVOUT_BITPAT = BitPat("b0010000") // 16 (0x10) — enable=001, opcode=0 +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/Decoder.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/Decoder.scala new file mode 100644 index 000000000..94e6f5aa6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/Decoder.scala @@ -0,0 +1,172 @@ +package framework.acc.memdomain + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import framework.acc.frontend.decoder.{DomainId, PostGDCmd} + +class DecodedCommand(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val is_shared = Bool() + val is_load = Bool() + val is_store = Bool() + val is_config = Bool() + val clear = Bool() + val transfer = Bool() + val source_bank_id = UInt(frontendParam.bank_id_len.W) + val is_mvin_2d = Bool() + val is_image = Bool() + val is_mmio = Bool() + val mem_addr = UInt(memParam.memAddrLen.W) + val iter = UInt(frontendParam.iter_len.W) + val bank_id = UInt(frontendParam.bank_id_len.W) + val special = UInt(64.W) +} + +import framework.acc.memdomain.DISA._ +import chisel3.experimental.hierarchy.{instantiable, public} + +// LS decode fields (iter removed from decode table — always from rs1[63:48]) +object LSDecodeFields extends Enumeration { + type Field = Value + val LD_EN, ST_EN, MEMADDR, BANK_ID, SPECIAL, VALID = Value +} + +// Default constants for Mem decoder +object DecodeConstants { + val Y = true.B + val N = false.B + val DADDR = 0.U(15.W) + val DSPECIAL = 0.U(64.W) +} + +@instantiable +class Decoder(val frontendParam: FrontendParam, val memParam: MemDomainParam) extends Module { + val commandDataBits = frontendParam.commandDataBits + import DecodeConstants._ + + val sharedBankIdBase = frontendParam.shared_bank_id_base + require(sharedBankIdBase > 0, s"sharedBankIdBase($sharedBankIdBase) must be > 0") + require( + sharedBankIdBase < (1 << frontendParam.bank_id_len), + s"sharedBankIdBase($sharedBankIdBase) must fit in bankIdLen(${frontendParam.bank_id_len})" + ) + require( + sharedBankIdBase > frontendParam.vbank_id_upper_bound, + s"sharedBankIdBase($sharedBankIdBase) must be > vbankIdUpperBound(${frontendParam.vbank_id_upper_bound})" + ) + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new PostGDCmd(frontendParam, memParam))) + val command = Decoupled(new DecodedCommand(frontendParam, memParam)) + }) + + val memAddrLen = memParam.memAddrLen + val bankIdLen = frontendParam.bank_id_len + val iterLen = frontendParam.iter_len + + // Only process Mem instructions + io.request.ready := io.command.ready + + val func7 = io.request.bits.cmd.raw_inst(31, 25) + val rs1 = io.request.bits.cmd.rs1Data + val rs2 = io.request.bits.cmd.rs2Data + + // Unified encoding: + // rs1[9:0] = read/config bank (BANK0) + // rs1[29:20] = write bank (BANK2) + // rs1[62:30] = iter; rs1[63] declares an in-place update. + // funct7[6:4] = enable (bank access flags, decoded by GlobalDecoder) + // rs2[38:0] = mem_addr (for MVIN/MVOUT, 39-bit) + // + import LSDecodeFields._ + // format: off + val ls_default_decode = List(N, N, DADDR, DADDR, DSPECIAL, N) + val ls_decode_list = ListLookup(func7, ls_default_decode, Array( + MSET_BITPAT -> List(N, N, 0.U(memAddrLen.W), rs1(bankIdLen - 1, 0), rs2, Y), + MVIN_MMIO_BITPAT -> List(Y, N, rs2(memAddrLen - 1, 0), 0.U, rs2, Y), + MVIN_BITPAT -> List(Y, N, rs2(memAddrLen - 1, 0), rs1(bankIdLen + 19, 20), rs2, Y), + MVIN_2D_BITPAT -> List(Y, N, Cat(rs2(35, 0), 0.U(3.W)), rs1(bankIdLen + 19, 20), rs2, Y), + MVIN_KERNEL_BITPAT -> List(Y, N, rs2(memAddrLen - 1, 0), rs1(bankIdLen + 19, 20), rs2, Y), + MVOUT_BITPAT -> List(N, Y, rs2(memAddrLen - 1, 0), rs1(bankIdLen - 1, 0), rs2, Y) + ) + ) + // format: on + + assert( + !(io.request.fire && !ls_decode_list(LSDecodeFields.VALID.id).asBool), + s"Decoder: Invalid command opcode, func7 = 0x%x\n", + func7 + ) + +// ----------------------------------------------------------------------------- +// Output assignment +// ----------------------------------------------------------------------------- + io.command.valid := io.request.valid && (io.request.bits.domain_id === DomainId.MEM) + + val transfer = func7 === MSET_BITPAT && rs2(12) + val raw_bank_id = Mux(func7 === MVIN_BITPAT || func7 === MVIN_2D_BITPAT || transfer, rs1(29, 20), rs1(9, 0)) + + val architectural_bank = Mux( + func7 === MVIN_BITPAT || func7 === MVIN_2D_BITPAT || transfer, + io.request.bits.bankAccess.wr_bank_id, + io.request.bits.bankAccess.rd_bank_0_id + ) + + val config_bank = Mux(func7 === MSET_BITPAT && !transfer, io.request.bits.bankAccess.wr_bank_id, architectural_bank) + val shared_bank_id = + memParam.sharedEnable.B && config_bank >= sharedBankIdBase.U + // format: off + // BB_BANK0 is encoded in rs1[9:0]. Keep the configured architectural ID; + // private and shared banks occupy disjoint ranges in that namespace. + io.command.bits.is_shared := io.command.valid && shared_bank_id + io.command.bits.is_load := Mux(io.command.valid, ls_decode_list(LSDecodeFields.LD_EN.id).asBool, false.B) + io.command.bits.is_store := Mux(io.command.valid, ls_decode_list(LSDecodeFields.ST_EN.id).asBool, false.B) + io.command.bits.is_config := Mux(io.command.valid, func7 === MSET_BITPAT, false.B) + io.command.bits.transfer := io.command.valid && transfer + io.command.bits.source_bank_id := rs1(bankIdLen - 1, 0) + io.command.bits.clear := io.command.valid && (func7 === MSET_BITPAT) && rs2(11) + io.command.bits.is_mvin_2d := Mux(io.command.valid, func7 === MVIN_2D_BITPAT, false.B) + io.command.bits.is_mmio := func7 === MVIN_MMIO_BITPAT + io.command.bits.is_image := func7 === MVIN_KERNEL_BITPAT + io.command.bits.mem_addr := Mux(io.command.valid, ls_decode_list(LSDecodeFields.MEMADDR.id).asUInt, 0.U(memParam.memAddrLen.W)) + io.command.bits.iter := Mux(io.command.valid, rs1(62, 30), 0.U(iterLen.W)) + + when(io.request.fire && func7 === MSET_BITPAT) { + when(transfer) { + assert(rs1(63, 30) === 0.U && rs1(19, 10) === 0.U, "MSET transfer reserves rs1 outside BANK0/BANK2") + assert(rs2 === (1.U << 12), "MSET transfer reserves other rs2 bits") + assert(rs1(9, 0) =/= rs1(29, 20), "MSET transfer requires distinct source and target") + assert((rs1(9, 0) >= sharedBankIdBase.U) === (rs1(29, 20) >= sharedBankIdBase.U), "MSET transfer requires one bank namespace") + }.otherwise { + assert(rs1(63, 10) === 0.U, "MSET reserves rs1[63:10]") + assert(rs2(63, 12) === 0.U, "MSET reserves rs2[63:12]") + } + assert(!(rs2(11) && !rs2(10)), "MSET clear requires alloc=1") + } + when(io.request.fire && (func7 === MVIN_BITPAT || func7 === MVOUT_BITPAT)) { + assert(rs2(63) || rs2(62, 58) === 0.U, "Whole-bank DMA reserves selected group bits") + } + when(io.request.fire && func7 === MVIN_2D_BITPAT) { + val height = rs1(62, 30) + val pixelBytes = rs2(42, 36) << 3 + val width = rs2(61, 59) +& 1.U + val validBytes = Mux(rs2(62), 8.U(5.W), 16.U(5.W)) + assert(height =/= 0.U, "MVIN_2D height must be non-zero") + assert(rs2(42, 36) =/= 0.U, "MVIN_2D pixel bytes must be non-zero") + assert(rs2(52, 43) =/= 0.U, "MVIN_2D source width must be non-zero") + assert(validBytes <= pixelBytes, "MVIN_2D valid bytes exceed pixel bytes") + assert((rs2(58, 53) +& (height * width)) <= Mux(shared_bank_id, memParam.sharedBankEntries.U, memParam.bankEntries.U), "MVIN_2D destination exceeds bank") + assert(!rs2(63), "MVIN_2D reserves rs2[63]") + } + when(io.request.fire && (func7 === MVIN_BITPAT || func7 === MVIN_2D_BITPAT)) { + assert(rs1(19, 0) === 0.U, "MVIN write-bank encoding reserves rs1[19:0]") + } + + val ls_bank_id = ls_decode_list(LSDecodeFields.BANK_ID.id).asUInt + io.command.bits.bank_id := Mux(io.command.valid, Mux(transfer, raw_bank_id, ls_bank_id), 0.U(frontendParam.bank_id_len.W)) + io.command.bits.special := Mux(io.command.valid, ls_decode_list(LSDecodeFields.SPECIAL.id).asUInt, 0.U(64.W)) + // format: on +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/build.sc b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/build.sc new file mode 100644 index 000000000..d1c452eda --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.memdomain.prm.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Bank.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MvoverIO.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/PRM.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/configs/uvm.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/configs/uvm.toml new file mode 100644 index 000000000..0b7df89d6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/configs/uvm.toml @@ -0,0 +1,27 @@ +[parameters] +diffTest = false + +[parameters.memory] +sharedPorts = 0 +bankNum = 2 +bankLayoutEnable = false +bankWidth = 128 +bankEntries = 8 +bankMaskLen = 16 +virtualBankCount = 4 +sharedEnable = false +sharedEntries = 0 +sharedBankEntries = 0 +sharedBankNum = 0 +sharedInputChannels = 0 +sharedDefaultGroupCount = 0 +bankRequestPorts = 1 +dma_buswidth = 128 +memAddrLen = 44 +mmioEnable = false +mmioBankNum = 0 +mmioBankEntries = 0 +mmioBankWidth = 0 +mmioReadWidth = 0 + +[defines] diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..e345f7d57 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/driver.svh @@ -0,0 +1,36 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual bus_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bus_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "PRM driver requires an interface") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + foreach (vif.ports[b]) begin + vif.ports[b].read_valid = req.pins[b].read_valid; + vif.ports[b].read_addr = req.pins[b].read_addr; + vif.ports[b].read_result_ready = req.pins[b].read_result_ready; + vif.ports[b].write_valid = req.pins[b].write_valid; + vif.ports[b].write_addr = req.pins[b].write_addr; + vif.ports[b].write_data = req.pins[b].write_data; + vif.ports[b].write_mask = req.pins[b].write_mask; + vif.ports[b].write_result_ready = req.pins[b].write_result_ready; + vif.ports[b].clear = req.pins[b].clear; + vif.ports[b].bind_generation = req.pins[b].bind_generation; + end + @(posedge vif.clock); + req.sampled = vif.ports; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/item.svh new file mode 100644 index 000000000..69fae1d35 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/item.svh @@ -0,0 +1,23 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + bank_signals pins[BANKS], sampled[BANKS]; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(ok, UVM_ALL_ON) + `uvm_field_int(bank, UVM_ALL_ON) + `uvm_field_int(write, UVM_ALL_ON) + `uvm_object_utils_end + logic [WIDTH-1:0] data; + logic ok; + int bank; + bit write; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..644e6ad42 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/agent/monitor.svh @@ -0,0 +1,24 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual bus_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bus_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "PRM monitor requires an interface") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.sampled = vif.ports; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/env/env.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/env/env.svh new file mode 100644 index 000000000..f6b4be38d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/env/env.svh @@ -0,0 +1,31 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver drv; + monitor mon; + memory_model model; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 1) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + model = memory_model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 1)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(model.analysis_export); + model.expected_ap.connect(scoreboard.expected_export); + model.actual_ap.connect(scoreboard.actual_export); + model.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/env/model.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/env/model.svh new file mode 100644 index 000000000..e53468236 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/env/model.svh @@ -0,0 +1,108 @@ +class memory_model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(memory_model) + logic [WIDTH-1:0] words[BANKS][ROWS]; + bit [MASKS-1:0] initialized[BANKS][ROWS]; + int expected_serial[BANKS][2], actual_serial[BANKS][2]; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + covergroup accesses with function sample (int bank, bit write, int row, logic [MASKS-1:0] mask); + option.per_instance = 1; + bank_index: coverpoint bank {bins banks[] = {[0 : BANKS - 1]};} + direction: coverpoint write; + address: coverpoint row { + bins first = {0}; bins interior = {[1 : ROWS - 2]}; bins last = {ROWS - 1}; + } + write_mask: coverpoint mask iff (write) { + bins none = {'0}; bins full = {'1}; bins partial = default; + } + bank_direction: cross bank_index, direction; + endgroup + covergroup protocol with function sample ( + bit clear, bit clearing, bit read_stall, bit write_stall, bit collision + ); + option.per_instance = 1; + clear_request: coverpoint clear; + clearing_active: coverpoint clearing; + read_backpressure: coverpoint read_stall; + write_backpressure: coverpoint write_stall; + read_write_collision: coverpoint collision; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + accesses = new(); + protocol = new(); + foreach (expected_serial[b, d]) begin + expected_serial[b][d] = 0; + actual_serial[b][d] = 0; + end + endfunction + function void write(cycle_item t); + result_item expected, actual; + if (t.reset) begin + reset_ap.write(1); + foreach (actual_serial[b, d]) actual_serial[b][d] = expected_serial[b][d]; + return; + end + foreach (t.sampled[b]) begin + bank_signals s = t.sampled[b]; + protocol.sample(s.clear, s.clearing, s.read_result_valid && !s.read_result_ready, + s.write_result_valid && !s.write_result_ready, s.read_valid && s.write_valid); + if (s.clear || s.clearing) + evidence::check(!s.read_ready && !s.write_ready, "PRM accepted an access during clear"); + if (s.clear) begin + for (int row = 0; row < ROWS; row++) begin + words[b][row] = '0; + initialized[b][row] = '1; + end + end + if (s.write_valid && s.write_ready) begin + evidence::check(!s.read_ready, "PRM read bypassed a same-cycle write"); + for (int m = 0; m < MASKS; m++) + if (s.write_mask[m]) begin + words[b][s.write_addr][m*(WIDTH/MASKS)+:(WIDTH/MASKS)] = s.write_data[m*(WIDTH/MASKS)+:(WIDTH/MASKS)]; + initialized[b][s.write_addr][m] = 1; + end + expected = result_item::type_id::create("write_ack"); + expected.bank = b; + expected.write = 1; + expected.data = 0; + expected.ok = 1; + expected.set_transaction_id((expected_serial[b][1]++ * BANKS + b) * 2 + 1); + expected_ap.write(expected); + accesses.sample(b, 1, s.write_addr, s.write_mask); + end + if (s.read_valid && s.read_ready) begin + evidence::check(initialized[b][s.read_addr] == '1, "PRM read of uninitialized storage"); + expected = result_item::type_id::create("read"); + expected.bank = b; + expected.write = 0; + expected.data = words[b][s.read_addr]; + expected.ok = 1; + expected.set_transaction_id((expected_serial[b][0]++ * BANKS + b) * 2); + expected_ap.write(expected); + accesses.sample(b, 0, s.read_addr, '0); + end + if (s.write_result_valid && s.write_result_ready) begin + actual = result_item::type_id::create("write_result"); + actual.bank = b; + actual.write = 1; + actual.data = 0; + actual.ok = s.write_ok; + actual.set_transaction_id((actual_serial[b][1]++ * BANKS + b) * 2 + 1); + actual_ap.write(actual); + end + if (s.read_result_valid && s.read_result_ready) begin + actual = result_item::type_id::create("read_result"); + actual.bank = b; + actual.write = 0; + actual.data = s.read_data; + actual.ok = 1; + actual.set_transaction_id((actual_serial[b][0]++ * BANKS + b) * 2); + actual_ap.write(actual); + end + end + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/filelist.f b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/filelist.f new file mode 100644 index 000000000..8c3e77d47 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/types.svh +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PRM/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/interface.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/interface.sv new file mode 100644 index 000000000..f78feb6a3 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/interface.sv @@ -0,0 +1,21 @@ +interface bus_if ( + input logic clock +); + import prm_types::*; + logic reset = 1; + bank_signals ports[BANKS]; + for (genvar b = 0; b < BANKS; b++) begin + assert property (@(posedge clock) disable iff (reset) + ports[b].read_result_valid && !ports[b].read_result_ready |=> + ports[b].read_result_valid && $stable( + ports[b].read_data + )) + else $fatal(1, "PRM read response changed under backpressure"); + assert property (@(posedge clock) disable iff (reset) + ports[b].write_result_valid && !ports[b].write_result_ready |=> + ports[b].write_result_valid && $stable( + ports[b].write_ok + )) + else $fatal(1, "PRM write response changed under backpressure"); + end +endinterface diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/package.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/package.sv new file mode 100644 index 000000000..8d916d3ca --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/package.sv @@ -0,0 +1,13 @@ +package prm_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import prm_types::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..c2f9f7a81 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/seq/sequence.svh @@ -0,0 +1,140 @@ +class access_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(access_sequence) + env test_env; + bank_signals pins[BANKS], observed[BANKS]; + bit reset = 1; + int checked = 0; + function new(string name = "access_sequence"); + super.new(name); + endfunction + function logic [WIDTH-1:0] pattern(int bank, int seed); + logic [WIDTH-1:0] value; + for (int bit_index = 0; bit_index < WIDTH; bit_index++) + value[bit_index] = ((seed * 32'h9e3779b9) >> (bit_index % 32)) ^ ((bit_index + bank) % 3 == 0); + return value; + endfunction + task tick(int cycles = 1); + cycle_item req; + repeat (cycles) begin + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.reset = reset; + req.pins = pins; + finish_item(req); + observed = req.sampled; + end + endtask + task drain(int count); + checked += count; + while (test_env.scoreboard.checked != checked) tick(); + endtask + task access (bit write, int row, logic [MASKS-1:0] mask = '1, int seed = 1, int stall = 0); + bit pending[BANKS]; + foreach (pins[b]) begin + pending[b] = 1; + pins[b].read_valid = !write; + pins[b].read_addr = row; + pins[b].write_valid = write; + pins[b].write_addr = row; + pins[b].write_data = pattern(b, seed); + pins[b].write_mask = mask; + pins[b].read_result_ready = stall == 0; + pins[b].write_result_ready = stall == 0; + end + do begin + tick(); + foreach (pins[b]) + if (pending[b] && (write ? observed[b].write_ready : observed[b].read_ready)) begin + pending[b] = 0; + pins[b].read_valid = 0; + pins[b].write_valid = 0; + end + end while (pending.or()); + tick(stall); + foreach (pins[b]) begin + pins[b].read_result_ready = 1; + pins[b].write_result_ready = 1; + end + drain(BANKS); + endtask + task reset_pending(bit write); + foreach (pins[b]) begin + pins[b].read_valid = !write; + pins[b].read_addr = ROWS - 1; + pins[b].write_valid = write; + pins[b].write_addr = ROWS - 1; + pins[b].write_data = pattern(b, 77); + pins[b].write_mask = '1; + pins[b].read_result_ready = 0; + pins[b].write_result_ready = 0; + end + tick(); + foreach (pins[b]) begin + evidence::check(write ? observed[b].write_ready : observed[b].read_ready, + "PRM cancellation request was not accepted"); + pins[b].read_valid = 0; + pins[b].write_valid = 0; + end + tick(3); + reset = 1; + tick(2); + reset = 0; + foreach (pins[b]) begin + pins[b].read_result_ready = 1; + pins[b].write_result_ready = 1; + end + tick(); + access (0, ROWS - 1); + endtask + task body(); + foreach (pins[b]) pins[b] = '0; + tick(4); + reset = 0; + foreach (pins[b]) begin + pins[b].clear = 1; + pins[b].read_result_ready = 1; + pins[b].write_result_ready = 1; + end + tick(); + foreach (pins[b]) pins[b].clear = 0; + tick(ROWS + 2); + for (int row = 0; row < ROWS; row++) access (0, row); + for (int row = 0; row < ROWS; row++) access (1, row, '1, row + 1); + for (int row = 0; row < ROWS; row++) access (0, row, '0, 0, 3); + for (int m = 0; m < MASKS; m++) begin + access (1, m % ROWS, MASKS'(1) << m, m + 100, 3); + access (0, m % ROWS); + end + access (1, ROWS - 1, '0, 0); + access (0, ROWS - 1); + // Concurrent read/write requests use the write port first, then read back its update. + foreach (pins[b]) begin + pins[b].read_valid = 1; + pins[b].read_addr = 0; + pins[b].write_valid = 1; + pins[b].write_addr = 0; + pins[b].write_data = pattern(b, 999); + pins[b].write_mask = '1; + end + tick(); + foreach (pins[b]) begin + evidence::check(observed[b].write_ready && !observed[b].read_ready, + "PRM did not prioritize a write"); + pins[b].write_valid = 0; + end + tick(); + foreach (pins[b]) begin + evidence::check(observed[b].read_ready, "PRM read did not resume"); + pins[b].read_valid = 0; + end + drain(2 * BANKS); + reset_pending(0); + reset_pending(1); + foreach (pins[b]) pins[b].clear = 1; + tick(); + foreach (pins[b]) pins[b].clear = 0; + tick(ROWS + 2); + for (int row = 0; row < ROWS; row++) access (0, row); + evidence::check(evidence::cancelled == 2 * BANKS, "PRM reset cancellation count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/suite.toml new file mode 100644 index 000000000..3837e4d46 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "prm" +top = "prm_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["prm_pkg::memory_model::accesses", "prm_pkg::memory_model::protocol"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/tb.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/tb.sv new file mode 100644 index 000000000..5efd2ddc9 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/tb.sv @@ -0,0 +1,19 @@ +module prm_tb; + import uvm_pkg::*; + import prm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + bus_if bus (clock); + PRM dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + if (!`PRM_DIFF_TEST) begin + for (genvar b = 0; b < `PRM_BANKS; b++) assign bus.ports[b].hash = 0; + end + initial begin + uvm_config_db#(virtual bus_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/tests/test.svh new file mode 100644 index 000000000..70eb6d956 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/tests/test.svh @@ -0,0 +1,16 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + access_sequence seq = access_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/types.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/types.svh new file mode 100644 index 000000000..e95a5e53a --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/resources/types.svh @@ -0,0 +1,13 @@ +`include "parameters.svh" +package prm_types; + localparam int BANKS = `PRM_BANKS, ROWS = `PRM_ROWS, WIDTH = `PRM_WIDTH, MASKS = `PRM_MASKS; + typedef struct packed { + logic read_valid, read_ready, read_result_valid, read_result_ready; + logic write_valid, write_ready, write_result_valid, write_result_ready, write_ok; + logic [15:0] read_addr, write_addr; + logic [WIDTH-1:0] read_data, write_data; + logic [MASKS-1:0] write_mask; + logic clear, clearing, bind_generation; + logic [31:0] hash; + } bank_signals; +endpackage diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/Bank.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/Bank.scala new file mode 100644 index 000000000..64772e43d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/Bank.scala @@ -0,0 +1,101 @@ +package framework.acc.memdomain.prm + +import framework.ip.memory.bank.{SramReadIO, SramReadReq, SramReadResp, SramWriteIO, SramWriteReq, SramWriteResp} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.memdomain.configs.MemDomainParam + +/** + * Bank: Pure SRAM bank + * Simple read/write memory without any accumulation logic + * Each bank is a single-port SRAM. `clear` zeroes every row locally, one row per cycle; + * the ports refuse requests while `clearing`, so banks clear in parallel. + */ +@instantiable +class Bank(val b: MemDomainParam) extends Module { + require( + b.bankEntries >= 2 && b.bankEntries <= 65536 + ) + val indexBits = log2Ceil(b.bankEntries) + val mask_len = b.bankMaskLen + val mask_elem = UInt((b.bankWidth / mask_len).W) + + @public + val io = IO(new Bundle { + val sramRead = new SramReadIO(b) + val sramWrite = new SramWriteIO(b) + val clear = Input(Bool()) + val clearing = Output(Bool()) + }) + + val mem = SyncReadMem(b.bankEntries, Vec(mask_len, mask_elem)) + + // ----------------------------------------------------------------------------- + // Local clear + // ----------------------------------------------------------------------------- + val clearing = RegInit(false.B) + val clearRow = RegInit(0.U(log2Ceil(b.bankEntries).W)) + io.clearing := clearing + when(io.clear) { + clearing := true.B + clearRow := 0.U + }.elsewhen(clearing) { + clearRow := clearRow + 1.U + when(clearRow === (b.bankEntries - 1).U)(clearing := false.B) + } + + val readPending = RegNext(io.sramRead.req.fire, false.B) + val readHeld = RegInit(false.B) + val readData = Reg(UInt(b.bankWidth.W)) + val writeValid = RegInit(false.B) + + io.sramRead.resp.valid := readPending || readHeld + io.sramWrite.resp.valid := writeValid + io.sramWrite.resp.bits.ok := true.B + io.sramWrite.req.ready := !io.clear && !clearing && (!writeValid || io.sramWrite.resp.ready) + io.sramRead.req.ready := !io.clear && !clearing && !io.sramWrite.req.fire && + (!io.sramRead.resp.valid || io.sramRead.resp.ready) + + val ren = io.sramRead.req.fire + val wen = clearing || io.sramWrite.req.fire + + when(io.sramRead.req.fire) { + assert( + io.sramRead.req.bits.addr < b.bankEntries.U, + "SRAM read exceeds bank depth" + ) + } + when(io.sramWrite.req.fire) { + assert( + io.sramWrite.req.bits.addr < b.bankEntries.U, + "SRAM write exceeds bank depth" + ) + } + + val rdata = mem.readWrite( + Mux( + clearing, + clearRow, + Mux(wen, io.sramWrite.req.bits.addr(indexBits - 1, 0), io.sramRead.req.bits.addr(indexBits - 1, 0)) + ), + Mux(clearing, 0.U.asTypeOf(Vec(mask_len, mask_elem)), io.sramWrite.req.bits.data.asTypeOf(Vec(mask_len, mask_elem))), + Mux(clearing, VecInit(Seq.fill(mask_len)(true.B)), io.sramWrite.req.bits.mask), + ren || wen, + wen + ) + + io.sramRead.resp.bits.data := Mux(readHeld, readData, rdata.asUInt) + when(readPending && !io.sramRead.resp.ready) { + readHeld := true.B + readData := rdata.asUInt + }.elsewhen(io.sramRead.resp.fire) { + readHeld := false.B + } + when(io.sramWrite.req.fire) { + writeValid := true.B + }.elsewhen(io.sramWrite.resp.fire) { + writeValid := false.B + } +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/Emit.scala new file mode 100644 index 000000000..0dce01275 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/Emit.scala @@ -0,0 +1,55 @@ +package framework.acc.memdomain.prm + +import java.nio.file.{Files, Paths} +import framework.acc.memdomain.configs.MemDomainParam + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val p = upickle.default.read[MemDomainParam](values("memory")) + val diffTest = values("diffTest").bool + Files.createDirectories(Paths.get(output)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new PRM(p, diffTest), + args = Array("--target-dir", s"$output/PRM", "--split-verilog") + ) + Files.writeString( + Paths.get(output, "parameters.svh"), + s"""`ifndef PRM_PARAMETERS +`define PRM_PARAMETERS +`define PRM_BANKS ${p.bankNum} +`define PRM_ROWS ${p.bankEntries} +`define PRM_WIDTH ${p.bankWidth} +`define PRM_MASKS ${p.bankMaskLen} +`define PRM_DIFF_TEST ${if (diffTest) 1 else 0} +`endif +""" + ) + + val ports = (0 until p.bankNum).flatMap { bank => + val prefix = s"io_banks_$bank" + val fields = Seq( + "sramRead_req_valid" -> "read_valid", + "sramRead_req_ready" -> "read_ready", + "sramRead_req_bits_addr" -> "read_addr", + "sramRead_resp_valid" -> "read_result_valid", + "sramRead_resp_ready" -> "read_result_ready", + "sramRead_resp_bits_data" -> "read_data", + "sramWrite_req_valid" -> "write_valid", + "sramWrite_req_ready" -> "write_ready", + "sramWrite_req_bits_addr" -> "write_addr", + "sramWrite_req_bits_data" -> "write_data", + "sramWrite_resp_valid" -> "write_result_valid", + "sramWrite_resp_ready" -> "write_result_ready", + "sramWrite_resp_bits_ok" -> "write_ok", + "clear" -> "clear", + "clearing" -> "clearing", + "bind" -> "bind_generation" + ).map { case (port, signal) => s".${prefix}_$port(bus.ports[$bank].$signal)" } + val masks = + (0 until p.bankMaskLen).map(mask => s".${prefix}_sramWrite_req_bits_mask_$mask(bus.ports[$bank].write_mask[$mask])") + fields ++ masks ++ (if (diffTest) Seq(s".io_hashes_$bank(bus.ports[$bank].hash)") else Seq.empty) + } + + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n")) +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/MvoverIO.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/MvoverIO.scala new file mode 100644 index 000000000..136e968f6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/MvoverIO.scala @@ -0,0 +1,27 @@ +package framework.acc.memdomain + +import chisel3._ +import chisel3.util._ + +object MvoverISA { + val Funct = 0x0d + val ThreadBits = 8 + val BankBits = 10 + val AddressBits = 16 +} + +/** Contiguous rows between logical bank endpoints attached to one fabric. */ +class MvoverCommand extends Bundle { + val sourceThread = UInt(MvoverISA.ThreadBits.W) + val sourceBank = UInt(MvoverISA.BankBits.W) + val sourceAddr = UInt(MvoverISA.AddressBits.W) + val targetThread = UInt(MvoverISA.ThreadBits.W) + val targetBank = UInt(MvoverISA.BankBits.W) + val targetAddr = UInt(MvoverISA.AddressBits.W) + val rows = UInt((MvoverISA.AddressBits + 1).W) +} + +class MvoverPort extends Bundle { + val command = Decoupled(new MvoverCommand) + val completion = Flipped(Decoupled(Bool())) +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/PRM.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/PRM.scala new file mode 100644 index 000000000..d949f187d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm/src/main/scala/PRM.scala @@ -0,0 +1,58 @@ +package framework.acc.memdomain.prm + +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.memdomain.prm.Bank +import framework.ip.memory.bank.{SramReadIO, SramWriteIO} +import framework.lib.hash.BankHashMonitor + +class PhysicalBankPort(memParam: MemDomainParam) extends Bundle { + val sramRead = new SramReadIO(memParam) + val sramWrite = new SramWriteIO(memParam) + val clear = Input(Bool()) + val clearing = Output(Bool()) + val bind = Input(Bool()) +} + +@instantiable +class PRM(val memParam: MemDomainParam, val diffTest: Boolean) extends Module { + + @public + val io = IO(new Bundle { + val banks = Vec(memParam.bankNum, new PhysicalBankPort(memParam)) + + val hashes = + if (diffTest) Some(Output(Vec(memParam.bankNum, UInt(32.W)))) + else None + + }) + + val banks: Seq[Instance[Bank]] = + Seq.fill(memParam.bankNum)(Instantiate( + new Bank(memParam) + )) + + for (i <- banks.indices) { + val bank = banks(i) + val port = io.banks(i) + bank.io.sramRead <> port.sramRead + bank.io.sramWrite <> port.sramWrite + bank.io.clear := port.clear + port.clearing := bank.io.clearing + if (diffTest) { + val monitor: Instance[BankHashMonitor] = + Instantiate(new BankHashMonitor(memParam)) + monitor.io.bind := port.bind + monitor.io.write.valid := port.sramWrite.req.valid && bank.io.sramWrite.req.ready + monitor.io.write.bits.addr := port.sramWrite.req.bits.addr + monitor.io.write.bits.mask := port.sramWrite.req.bits.mask + monitor.io.write.bits.data := port.sramWrite.req.bits.data + bank.io.sramWrite.req.valid := port.sramWrite.req.valid && monitor.io.write.ready + port.sramWrite.req.ready := bank.io.sramWrite.req.ready && monitor.io.write.ready + io.hashes.get(i) := monitor.io.statusHash + } + } +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/build.sc b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/build.sc new file mode 100644 index 000000000..c9cb32733 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/build.sc @@ -0,0 +1,28 @@ +import mill._ +import $file.src.main.scala.toddr.{build => toddr_build} +import $file.src.main.scala.tol1.{build => tol1_build} +import $file.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.memdomain.tomem.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/ToMem.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) ++ + toddr_build.sources() ++ + tol1_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + toddr_build.emitSources() ++ + tol1_build.emitSources() +) diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/default.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/default.toml new file mode 100644 index 000000000..c6c2b8674 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/default.toml @@ -0,0 +1,81 @@ + +[tracking] +entries = 24 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + + +[prepared] +contexts = 24 +maxRanges = 8 +idBits = 8 +beatBytes = 16 + +[prepared.bus] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 + + + +[axi] +addressBits = 44 +dataBits = 128 +idBits = 4 + + +[[regions]] +base = 0x80000000 +bytes = 0x20000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/uvm.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/uvm.toml new file mode 100644 index 000000000..fee3dc162 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/uvm.toml @@ -0,0 +1,82 @@ +[parameters] + +[parameters.frontend] +ownerId = 7 +commandDataBits = 64 +rob_entries = 4 +bank_alias_enable = true +rs_out_of_order_response = true +bank_id_len = 4 +vbank_id_upper_bound = 3 +shared_bank_id_base = 16 +iter_len = 16 +sub_rob_enable = false +sub_rob_depth = 4 + +[parameters.memory] +sharedPorts = 0 +bankNum = 2 +bankLayoutEnable = true +bankWidth = 128 +bankEntries = 8 +bankMaskLen = 16 +virtualBankCount = 4 +sharedEnable = false +sharedEntries = 0 +sharedBankEntries = 0 +sharedBankNum = 0 +sharedInputChannels = 0 +sharedDefaultGroupCount = 0 +bankRequestPorts = 1 +dma_buswidth = 128 +memAddrLen = 44 +mmioEnable = true +mmioBankNum = 1 +mmioBankEntries = 16 +mmioBankWidth = 128 +mmioReadWidth = 32 + +[parameters.permission] +addressBits = 44 +entries = 16 + +[parameters.gp] +enable = true +laneNumber = 1 +vLen = 128 +eLen = 64 +iBufWords = 16 +memoryPorts = 1 + + + +[parameters.transport.tracking] +entries=4 +addressBits=44 +idBits=8 +lineBytes=64 +maxRanges=8 +[parameters.transport.prepared] +contexts=4 +maxRanges=8 +idBits=8 +beatBytes=16 +[parameters.transport.prepared.bus] +nodeIdBits=7 +addressBits=44 +dataBits=256 +[parameters.transport.axi] +addressBits=44 +dataBits=128 +idBits=4 +[[parameters.transport.regions]] +base="0" +bytes="17592186044416" +normal=true +cacheable=true +atomic=true +readable=true +writable=true +executable=true + +[defines] diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..5a833dd30 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual tomem_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tomem_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.source = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/fixture.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/fixture.svh new file mode 100644 index 000000000..534a8094d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/fixture.svh @@ -0,0 +1,19 @@ +class fixture extends uvm_object; + `uvm_object_utils(fixture) + int groups[`TM_BANKS]; + bit read_error, write_error, bank_error, image_fault; + int admission_error, completion_error; + longint unsigned physical_offset = 'h100000; + function new(string name = "fixture"); + super.new(name); + foreach (groups[b]) groups[b] = 2; + endfunction + function byte unsigned initial_byte(longint unsigned address); + return 8'(address ^ (address >> 8) ^ (address >> 16)); + endfunction + function bit [127:0] initial_bank(int bank_id, int group_index, int row); + bit [127:0] value; + for (int b = 0; b < 16; b++) value[b*8+:8] = 8'(bank_id * 37 + group_index * 19 + row * 7 + b); + return value; + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/item.svh new file mode 100644 index 000000000..e30bf3bb5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`TM_INPUT_BITS-1:0] inputs; + logic [`TM_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_object_utils_end + bit [511:0] value; + function new(string name = "result"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..9a85039ff --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual tomem_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tomem_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/peer.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/peer.svh new file mode 100644 index 000000000..7251f9293 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/agent/peer.svh @@ -0,0 +1,211 @@ +class peer extends uvm_component; + `uvm_component_utils(peer) + virtual tomem_if vif; + fixture image; + bit [127:0] bank_memory[`TM_BANKS][`TM_GROUPS][`TM_ROWS]; + byte unsigned memory[longint unsigned]; + int cycle, read_beats, write_beats, rid, bid, bank_due, bank_ack_due, metadata_due, image_due; + longint unsigned read_address, write_address; + bit b_pending, bank_pending, bank_ack, metadata_pending, image_pending; + bit [127:0] bank_data; + bit complete_pending[3], complete_armed[3]; + int complete_due[3], complete_error[3]; + longint unsigned complete_address[3]; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tomem_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "ToMem interface is required") + endfunction + function void start_of_simulation_phase(uvm_phase phase); + foreach (bank_memory[b, g, row]) bank_memory[b][g][row] = image.initial_bank(b, g, row); + endfunction + function bit [127:0] word_at(longint unsigned address); + bit [127:0] value; + for (int b = 0; b < 16; b++) + value[b*8+:8] = memory.exists(address + b) ? memory[address+b] : + image.initial_byte(address + b); + return value; + endfunction + task run_phase(uvm_phase phase); + forever begin + @(posedge vif.clock); + cycle++; + if (vif.reset) begin + read_beats = 0; + write_beats = 0; + b_pending = 0; + bank_pending = 0; + bank_ack = 0; + metadata_pending = 0; + image_pending = 0; + foreach (complete_pending[p]) begin + complete_pending[p] = 0; + complete_armed[p] = 1; + end + end else begin + if (`TO(MEMORY_AXI_R_READY) && `TI(MEMORY_AXI_R_VALID)) begin + read_beats--; + read_address += 16; + end + if (`TO(MEMORY_AXI_B_READY) && `TI(MEMORY_AXI_B_VALID)) b_pending = 0; + if (`TO(MEMORY_AXI_AR_VALID) && `TI(MEMORY_AXI_AR_READY)) begin + read_address = `TO(MEMORY_AXI_AR_BITS_ADDR); + read_beats = `TO(MEMORY_AXI_AR_BITS_LEN) + 1; + rid = `TO(MEMORY_AXI_AR_BITS_ID); + end + if (`TO(MEMORY_AXI_AW_VALID) && `TI(MEMORY_AXI_AW_READY)) begin + write_address = `TO(MEMORY_AXI_AW_BITS_ADDR); + write_beats = `TO(MEMORY_AXI_AW_BITS_LEN) + 1; + bid = `TO(MEMORY_AXI_AW_BITS_ID); + end + if (`TO(MEMORY_AXI_W_VALID) && `TI(MEMORY_AXI_W_READY)) begin + for (int b = 0; b < 16; b++) + if (vif.outputs[`TM_OUT_MEMORY_AXI_W_BITS_STRB_OFFSET+b]) + memory[write_address+b] = vif.outputs[`TM_OUT_MEMORY_AXI_W_BITS_DATA_OFFSET+b*8+:8]; + write_address += 16; + write_beats--; + if (write_beats == 0) b_pending = 1; + end + if (`TO(BANKREAD_READ_RESP_READY) && `TI(BANKREAD_READ_RESP_VALID)) bank_pending = 0; + if (`TO(BANKREAD_READ_REQ_VALID) && `TI(BANKREAD_READ_REQ_READY)) begin + bank_data = bank_memory[ + `TO(BANKREAD_BANK_ID) + ][ + `TO(BANKREAD_GROUP_ID) + ][ + `TO(BANKREAD_READ_REQ_BITS_ADDR) + ]; + bank_pending = 1; + bank_due = cycle + 2; + end + if (`TO(BANKWRITE_WRITE_RESP_READY) && `TI(BANKWRITE_WRITE_RESP_VALID)) bank_ack = 0; + if (`TO(BANKWRITE_WRITE_REQ_VALID) && `TI(BANKWRITE_WRITE_REQ_READY)) begin + if (!image.bank_error) + bank_memory[ + `TO(BANKWRITE_BANK_ID) + ][ + `TO(BANKWRITE_GROUP_ID) + ][ + `TO(BANKWRITE_WRITE_REQ_BITS_ADDR) + ] = + `TO(BANKWRITE_WRITE_REQ_BITS_DATA); + bank_ack = 1; + bank_ack_due = cycle + 3; + end + if (`TO(METADATA_RESPONSE_READY) && `TI(METADATA_RESPONSE_VALID)) metadata_pending = 0; + if (`TO(METADATA_REQUEST_VALID) && `TI(METADATA_REQUEST_READY)) begin + metadata_pending = 1; + metadata_due = cycle + 2; + end + if (`TO(IMAGE_COMPLETE_READY) && `TI(IMAGE_COMPLETE_VALID)) image_pending = 0; + if (`TO(IMAGE_TERMINAL_VALID) && `TI(IMAGE_TERMINAL_READY)) begin + image_pending = 1; + image_due = cycle + 3; + end + if (`TO(MEMORY_CONTROL_COMPLETE_0_READY) && `TI(MEMORY_CONTROL_COMPLETE_0_VALID)) + complete_pending[0] = 0; + if (!`TO(MEMORY_CONTROL_FINISHED_0_VALID)) complete_armed[0] = 1; + if (`TO(MEMORY_CONTROL_FINISHED_0_VALID) && complete_armed[0] && !complete_pending[0]) begin + complete_armed[0] = 0; + complete_pending[0] = 1; + complete_due[0] = cycle + 6; + complete_error[0] = image.completion_error != 0 ? image.completion_error : `TO( + MEMORY_CONTROL_FINISHED_0_BITS_ERROR); + complete_address[0] = image.completion_error != 0 ? 64'hbeef : `TO( + MEMORY_CONTROL_FINISHED_0_BITS_ADDRESS); + end + if (`TO(MEMORY_CONTROL_COMPLETE_1_READY) && `TI(MEMORY_CONTROL_COMPLETE_1_VALID)) + complete_pending[1] = 0; + if (!`TO(MEMORY_CONTROL_FINISHED_1_VALID)) complete_armed[1] = 1; + if (`TO(MEMORY_CONTROL_FINISHED_1_VALID) && complete_armed[1] && !complete_pending[1]) begin + complete_armed[1] = 0; + complete_pending[1] = 1; + complete_due[1] = cycle + 6; + complete_error[1] = image.completion_error != 0 ? image.completion_error : `TO( + MEMORY_CONTROL_FINISHED_1_BITS_ERROR); + complete_address[1] = image.completion_error != 0 ? 64'hbeef : `TO( + MEMORY_CONTROL_FINISHED_1_BITS_ADDRESS); + end + if (`TO(MEMORY_CONTROL_COMPLETE_2_READY) && `TI(MEMORY_CONTROL_COMPLETE_2_VALID)) + complete_pending[2] = 0; + if (!`TO(MEMORY_CONTROL_FINISHED_2_VALID)) complete_armed[2] = 1; + if (`TO(MEMORY_CONTROL_FINISHED_2_VALID) && complete_armed[2] && !complete_pending[2]) begin + complete_armed[2] = 0; + complete_pending[2] = 1; + complete_due[2] = cycle + 6; + complete_error[2] = image.completion_error != 0 ? image.completion_error : `TO( + MEMORY_CONTROL_FINISHED_2_BITS_ERROR); + complete_address[2] = image.completion_error != 0 ? 64'hbeef : `TO( + MEMORY_CONTROL_FINISHED_2_BITS_ADDRESS); + end + end + @(negedge vif.clock); + vif.backend = '0; + vif.backend[`TM_IN_QUERYGROUPS_OFFSET+:`TM_IN_QUERYGROUPS_WIDTH] = image.groups[ + `TO(QUERYVBANK) + ]; + vif.backend[`TM_IN_MEMORY_AXI_AR_READY_OFFSET+:`TM_IN_MEMORY_AXI_AR_READY_WIDTH] = read_beats == 0 && cycle % 4 != 0; + vif.backend[`TM_IN_MEMORY_AXI_R_VALID_OFFSET+:`TM_IN_MEMORY_AXI_R_VALID_WIDTH] = read_beats > 0; + vif.backend[`TM_IN_MEMORY_AXI_R_BITS_DATA_OFFSET+:`TM_IN_MEMORY_AXI_R_BITS_DATA_WIDTH] = word_at( + read_address); + vif.backend[`TM_IN_MEMORY_AXI_R_BITS_ID_OFFSET+:`TM_IN_MEMORY_AXI_R_BITS_ID_WIDTH] = rid; + vif.backend[`TM_IN_MEMORY_AXI_R_BITS_RESP_OFFSET+:`TM_IN_MEMORY_AXI_R_BITS_RESP_WIDTH] = image.read_error ? 2 : 0; + vif.backend[`TM_IN_MEMORY_AXI_R_BITS_LAST_OFFSET+:`TM_IN_MEMORY_AXI_R_BITS_LAST_WIDTH] = read_beats == 1; + vif.backend[`TM_IN_MEMORY_AXI_AW_READY_OFFSET+:`TM_IN_MEMORY_AXI_AW_READY_WIDTH] = write_beats == 0 && !b_pending && cycle % 5 != 0; + vif.backend[`TM_IN_MEMORY_AXI_W_READY_OFFSET+:`TM_IN_MEMORY_AXI_W_READY_WIDTH] = write_beats > 0 && cycle % 3 != 0; + vif.backend[`TM_IN_MEMORY_AXI_B_VALID_OFFSET+:`TM_IN_MEMORY_AXI_B_VALID_WIDTH] = b_pending && cycle % 4 != 0; + vif.backend[`TM_IN_MEMORY_AXI_B_BITS_ID_OFFSET+:`TM_IN_MEMORY_AXI_B_BITS_ID_WIDTH] = bid; + vif.backend[`TM_IN_MEMORY_AXI_B_BITS_RESP_OFFSET+:`TM_IN_MEMORY_AXI_B_BITS_RESP_WIDTH] = image.write_error ? 2 : 0; + vif.backend[`TM_IN_BANKREAD_READ_REQ_READY_OFFSET+:`TM_IN_BANKREAD_READ_REQ_READY_WIDTH] = !bank_pending && cycle % 3 != 0; + vif.backend[`TM_IN_BANKREAD_READ_RESP_VALID_OFFSET+:`TM_IN_BANKREAD_READ_RESP_VALID_WIDTH] = bank_pending && cycle >= bank_due; + vif.backend[`TM_IN_BANKREAD_READ_RESP_BITS_DATA_OFFSET+:`TM_IN_BANKREAD_READ_RESP_BITS_DATA_WIDTH] = bank_data; + vif.backend[`TM_IN_BANKWRITE_WRITE_REQ_READY_OFFSET+:`TM_IN_BANKWRITE_WRITE_REQ_READY_WIDTH] = !bank_ack && cycle % 4 != 0; + vif.backend[`TM_IN_BANKWRITE_WRITE_RESP_VALID_OFFSET+:`TM_IN_BANKWRITE_WRITE_RESP_VALID_WIDTH] = bank_ack && cycle >= bank_ack_due; + vif.backend[`TM_IN_BANKWRITE_WRITE_RESP_BITS_OK_OFFSET+:`TM_IN_BANKWRITE_WRITE_RESP_BITS_OK_WIDTH] = !image.bank_error; + vif.backend[`TM_IN_METADATA_REQUEST_READY_OFFSET+:`TM_IN_METADATA_REQUEST_READY_WIDTH] = !metadata_pending && cycle % 3 != 0; + vif.backend[`TM_IN_METADATA_RESPONSE_VALID_OFFSET+:`TM_IN_METADATA_RESPONSE_VALID_WIDTH] = metadata_pending && cycle >= metadata_due; + vif.backend[`TM_IN_METADATA_RESPONSE_BITS_OFFSET+:`TM_IN_METADATA_RESPONSE_BITS_WIDTH] = !image.bank_error; + vif.backend[`TM_IN_IMAGE_START_READY_OFFSET+:`TM_IN_IMAGE_START_READY_WIDTH] = cycle % 4 != 0; + vif.backend[`TM_IN_IMAGE_DATA_READY_OFFSET+:`TM_IN_IMAGE_DATA_READY_WIDTH] = cycle % 5 != 0; + vif.backend[`TM_IN_IMAGE_TERMINAL_READY_OFFSET+:`TM_IN_IMAGE_TERMINAL_READY_WIDTH] = cycle % 3 != 0; + vif.backend[`TM_IN_IMAGE_COMPLETE_VALID_OFFSET+:`TM_IN_IMAGE_COMPLETE_VALID_WIDTH] = image_pending && cycle >= image_due; + vif.backend[`TM_IN_IMAGE_COMPLETE_BITS_FAULT_OFFSET+:`TM_IN_IMAGE_COMPLETE_BITS_FAULT_WIDTH] = image.image_fault; + vif.backend[`TM_IN_IMAGE_COMPLETE_BITS_CAUSE_OFFSET+:`TM_IN_IMAGE_COMPLETE_BITS_CAUSE_WIDTH] = 5; + vif.backend[`TM_IN_IMAGE_COMPLETE_BITS_TVAL_OFFSET+:`TM_IN_IMAGE_COMPLETE_BITS_TVAL_WIDTH] = 'hf00d; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_0_VALID_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_0_VALID_WIDTH] = 1; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_0_PARENTID_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_0_PARENTID_WIDTH] = 0; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_0_FAULT_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_0_FAULT_ERROR_WIDTH] = image.admission_error; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_0_FAULT_ADDRESS_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_0_FAULT_ADDRESS_WIDTH] = image.admission_error == 0 ? 0 : 64'hdead; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_0_VALID_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_0_VALID_WIDTH] = complete_pending[0] && cycle >= complete_due[0]; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_0_BITS_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_0_BITS_ERROR_WIDTH] = complete_error[0]; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_0_BITS_ADDRESS_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_0_BITS_ADDRESS_WIDTH] = complete_address[0]; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_1_VALID_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_1_VALID_WIDTH] = 1; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_1_PARENTID_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_1_PARENTID_WIDTH] = 1; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_1_FAULT_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_1_FAULT_ERROR_WIDTH] = image.admission_error; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_1_FAULT_ADDRESS_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_1_FAULT_ADDRESS_WIDTH] = image.admission_error == 0 ? 0 : 64'hdead; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_1_VALID_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_1_VALID_WIDTH] = complete_pending[1] && cycle >= complete_due[1]; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_1_BITS_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_1_BITS_ERROR_WIDTH] = complete_error[1]; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_1_BITS_ADDRESS_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_1_BITS_ADDRESS_WIDTH] = complete_address[1]; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_2_VALID_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_2_VALID_WIDTH] = 1; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_2_PARENTID_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_2_PARENTID_WIDTH] = 2; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_2_FAULT_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_2_FAULT_ERROR_WIDTH] = image.admission_error; + vif.backend[`TM_IN_MEMORY_CONTROL_DECISIONS_2_FAULT_ADDRESS_OFFSET+:`TM_IN_MEMORY_CONTROL_DECISIONS_2_FAULT_ADDRESS_WIDTH] = image.admission_error == 0 ? 0 : 64'hdead; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_2_VALID_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_2_VALID_WIDTH] = complete_pending[2] && cycle >= complete_due[2]; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_2_BITS_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_2_BITS_ERROR_WIDTH] = complete_error[2]; + vif.backend[`TM_IN_MEMORY_CONTROL_COMPLETE_2_BITS_ADDRESS_OFFSET+:`TM_IN_MEMORY_CONTROL_COMPLETE_2_BITS_ADDRESS_WIDTH] = complete_address[2]; + vif.backend[`TM_IN_MEMORY_CONTROL_MAPPINGS_0_HIT_OFFSET+:`TM_IN_MEMORY_CONTROL_MAPPINGS_0_HIT_WIDTH] = 1; + vif.backend[`TM_IN_MEMORY_CONTROL_MAPPINGS_0_PA_OFFSET+:`TM_IN_MEMORY_CONTROL_MAPPINGS_0_PA_WIDTH] = + `TO(MEMORY_CONTROL_QUERIES_0_VA) + + image.physical_offset; + vif.backend[`TM_IN_MEMORY_CONTROL_MAPPINGS_0_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_MAPPINGS_0_ERROR_WIDTH] = 0; + vif.backend[`TM_IN_MEMORY_CONTROL_MAPPINGS_1_HIT_OFFSET+:`TM_IN_MEMORY_CONTROL_MAPPINGS_1_HIT_WIDTH] = 1; + vif.backend[`TM_IN_MEMORY_CONTROL_MAPPINGS_1_PA_OFFSET+:`TM_IN_MEMORY_CONTROL_MAPPINGS_1_PA_WIDTH] = + `TO(MEMORY_CONTROL_QUERIES_1_VA) + + image.physical_offset; + vif.backend[`TM_IN_MEMORY_CONTROL_MAPPINGS_1_ERROR_OFFSET+:`TM_IN_MEMORY_CONTROL_MAPPINGS_1_ERROR_WIDTH] = 0; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/env/env.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/env/env.svh new file mode 100644 index 000000000..eeda0417e --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/env/env.svh @@ -0,0 +1,36 @@ +class env extends uvm_env; + `uvm_component_utils(env) + uvm_analysis_imp #(uvm_sequence_item, env) reset_events; + driver source; + monitor mon; + peer backend; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 1) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_events = new("reset_events", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + backend = peer::type_id::create("backend", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 1)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + fixture image = fixture::type_id::create("image"); + reference.image = image; + backend.image = image; + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + reference.cancelled.connect(reset_events); + endfunction + function void write(uvm_sequence_item t); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/env/model.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/env/model.svh new file mode 100644 index 000000000..90e175cbe --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/env/model.svh @@ -0,0 +1,390 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + fixture image; + uvm_analysis_port #(result_item) expected, actual; + uvm_analysis_port #(uvm_sequence_item) cancelled; + typedef struct { + int tag, bank_id, rows, groups, group_base, start_row, width, pixel, source_width; + longint unsigned base, special; + bit writing, is_2d, is_image, is_mmio, is_config; + int words, index, fault; + longint unsigned fault_address; + } operation; + operation load_op, store_op; + bit load_active, store_active, pending[int]; + int + load_serial, + store_serial, + read_beats, + write_beats, + image_words, + bank_acks, + bank_reads, + metadata_acks; + bit read_addresses[longint unsigned]; + longint unsigned write_address; + byte unsigned memory[longint unsigned], store_bytes[longint unsigned]; + bit [127:0] banks[`TM_BANKS][`TM_GROUPS][`TM_ROWS]; + result_item load_results[$]; + covergroup behavior with function sample ( + int kind, int fault, bit overlap, bit stall, bit reset_busy + ); + option.per_instance = 1; + operations: coverpoint kind {bins all[] = {[0 : 5]};} + faults: coverpoint fault { + bins success = {0}; + bins access_fault = {2}; + bins bank_fault = {5}; + bins shape = {7}; + bins completion = {10}; + } + concurrency: coverpoint overlap; + backpressure: coverpoint stall; + reset_cancellation: coverpoint reset_busy; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + cancelled = new("cancelled", this); + behavior = new(); + endfunction + function void start_of_simulation_phase(uvm_phase phase); + foreach (banks[b, g, row]) banks[b][g][row] = image.initial_bank(b, g, row); + endfunction + function void verify(bit ok, string message); + evidence::checks++; + if (!ok) `uvm_fatal("TOMEM", message) + endfunction + function byte unsigned byte_at(longint unsigned address); + return memory.exists(address) ? memory[address] : image.initial_byte(address); + endfunction + function bit [127:0] word_at(longint unsigned address); + bit [127:0] value; + for (int b = 0; b < 16; b++) value[b*8+:8] = byte_at(address + b); + return value; + endfunction + function operation decode(cycle_item t); + operation op; + op.tag = `MI(t, REQUEST_BITS_ROB_ID); + op.bank_id = `MI(t, REQUEST_BITS_CMD_BANK_ID); + op.rows = `MI(t, REQUEST_BITS_CMD_ITER); + op.base = `MI(t, REQUEST_BITS_CMD_MEM_ADDR); + op.special = `MI(t, REQUEST_BITS_CMD_SPECIAL); + op.writing = `MI(t, REQUEST_BITS_CMD_IS_STORE); + op.is_2d = `MI(t, REQUEST_BITS_CMD_IS_MVIN_2D); + op.is_image = `MI(t, REQUEST_BITS_CMD_IS_IMAGE); + op.is_mmio = `MI(t, REQUEST_BITS_CMD_IS_MMIO); + op.is_config = `MI(t, REQUEST_BITS_CMD_IS_CONFIG); + op.width = ((op.special >> 59) & 7) + 1; + op.pixel = ((op.special >> 36) & 127) * 8; + op.source_width = (op.special >> 43) & 1023; + op.group_base=op.special[63]&&!op.is_2d&&!op.is_image&&!op.is_mmio?int'((op.special>>58)&31):0; + op.groups=op.is_image||op.is_mmio||op.is_config?1:(op.special[63]&&!op.is_2d?1:image.groups[op.bank_id]); + op.start_row = op.is_2d ? int'((op.special >> 53) & 63) : 0; + op.words = op.is_image ? (op.rows + 15) / 16 : op.rows * (op.is_2d ? op.width : op.groups); + op.index = 0; + op.fault = 0; + op.fault_address = 0; + if (!op.is_config) begin + if (image.admission_error != 0) begin + op.fault = image.admission_error; + op.fault_address = 'hdead; + end else if (op.rows == 0) begin + op.fault = 7; + op.fault_address = op.base; + end else if (op.writing ? image.write_error : image.read_error) begin + op.fault = 2; + op.fault_address = ((op.base & ~64'd15) + image.physical_offset); + end else if (!op.writing && image.bank_error) begin + op.fault = 5; + op.fault_address = op.base; + end else if (op.is_image && image.image_fault) begin + op.fault = 5; + op.fault_address = 'hf00d; + end + end + return op; + endfunction + function longint unsigned load_address(operation op, int index); + if (op.is_image || op.is_mmio) return op.base + index * 16 + image.physical_offset; + if (op.is_2d) + return op.base+(index/op.width)*op.source_width*op.pixel+(index%op.width)*op.pixel+image.physical_offset; + return op.base+(index/op.groups)*op.groups*((op.special>>39)&19'h7ffff)*16+(index%op.groups)*16+image.physical_offset; + endfunction + function void predict(cycle_item t); + operation op = decode(t); + result_item result; + verify(!pending.exists(op.tag), "Duplicate live ROB tag"); + pending[op.tag] = 1; + if (op.writing) begin + verify(!store_active, "Overlapping stores"); + store_op = op; + store_active = 1; + store_serial++; + if (op.fault == 0 || image.write_error) + for (int row = 0; row < op.rows; row++) + for (int group_index = 0; group_index < op.groups; group_index++) begin + bit [127:0] data = banks[op.bank_id][op.group_base+group_index][row]; + longint unsigned address=op.base+row*op.groups*((op.special>>39)&19'h7ffff)*16+group_index*16+image.physical_offset; + for (int b = 0; b < 16; b++) store_bytes[address+b] = data[b*8+:8]; + end + end else begin + verify(!load_active, "Overlapping loads"); + load_op = op; + load_active = 1; + load_serial++; + image_words = 0; + read_addresses.delete(); + if (!op.is_config && image.admission_error == 0 && op.rows > 0) + for (int i = 0; i < op.words; i++) begin + longint unsigned address = load_address(op, i); + for (int b = 0; b < 16; b++) read_addresses[(address+b)&~64'd15] = 1; + end + if(!op.is_config&&(op.fault==0||image.bank_error||(op.is_image&&image.image_fault)))begin + int words = image.bank_error ? 1 : op.words; + if (op.is_image) begin + for (int i = 0; i < (op.rows + 3) / 4; i++) begin + result = result_item::type_id::create("image_expected"); + result.value = 32'({ + byte_at(op.base + image.physical_offset + i * 4 + 3), + byte_at(op.base + image.physical_offset + i * 4 + 2), + byte_at(op.base + image.physical_offset + i * 4 + 1), + byte_at(op.base + image.physical_offset + i * 4) + }); + result.set_transaction_id((2 << 20) | (load_serial << 10) | i); + expected.write(result); + end + end else + for (int i = 0; i < words; i++) begin + bit [127:0] data = word_at(load_address(op, i)); + int group_index = op.group_base + (op.is_2d ? 0 : i % op.groups); + int row = op.start_row + (op.is_2d ? i : i / op.groups); + if (op.is_2d && op.special[62]) data[127:64] = 0; + result = result_item::type_id::create("load_expected"); + if (op.is_mmio) + result.value = { + 64'(((op.special >> 39) & 17'h1ffff) + i * 16), + data, + 16'((17'd1 << ((op.special >> 56) & 255)) - 1) + }; + else result.value = {32'(op.bank_id), 32'(group_index), 32'(row), data, 16'hffff}; + result.set_transaction_id((1 << 20) | (load_serial << 10) | i); + expected.write(result); + load_results.push_back(result); + end + end + end + result = result_item::type_id::create("completion_expected"); + result.value = { + 32'(op.tag), + 32'(image.completion_error != 0 ? image.completion_error : op.fault), + image.completion_error != 0 ? 64'hbeef : op.fault_address + }; + result.set_transaction_id(op.tag); + expected.write(result); + behavior.sample( + op.is_config ? 5 : op.is_image ? 3 : op.is_mmio ? 4 : op.is_2d ? 2 : op.writing ? 1 : 0, + image.completion_error != 0 ? image.completion_error : op.fault, + load_active && store_active, 0, 0); + endfunction + function void write(cycle_item t); + if (t.reset) begin + behavior.sample(0, 0, 0, 0, load_active || store_active); + cancelled.write(null); + load_active = 0; + store_active = 0; + pending.delete(); + store_bytes.delete(); + load_results.delete(); + read_beats = 0; + write_beats = 0; + bank_acks = 0; + bank_reads = 0; + metadata_acks = 0; + read_addresses.delete(); + return; + end + if (`MI(t, REQUEST_VALID) && `MO(t, REQUEST_READY)) predict(t); + if (`MO(t, MEMORY_AXI_AR_VALID) && `MI(t, MEMORY_AXI_AR_READY)) begin + verify(load_active, "AXI read without a load owner"); + verify(`MO(t, MEMORY_AXI_AR_BITS_SIZE) == 4 && `MO(t, MEMORY_AXI_AR_BITS_BURST) == 1 && + `MO(t, MEMORY_AXI_AR_BITS_ID) == 0, "AXI read attributes mismatch"); + verify(read_beats == 0, "Overlapping AXI read bursts"); + read_beats = `MO(t, MEMORY_AXI_AR_BITS_LEN) + 1; + verify((`MO(t, MEMORY_AXI_AR_BITS_ADDR) & 4095) + read_beats * 16 <= 4096, + "AXI read crossed a page"); + verify((`MO(t, MEMORY_AXI_AR_BITS_ADDR) & 15) == 0, "Unaligned AXI read"); + for (int i = 0; i < read_beats; i++) + verify(read_addresses.exists(`MO(t, MEMORY_AXI_AR_BITS_ADDR) + i * 16), + "AXI read escaped the accepted command footprint"); + end + if (`MI(t, MEMORY_AXI_R_VALID) && `MO(t, MEMORY_AXI_R_READY)) begin + verify(read_beats > 0, "AXI response without a request"); + verify(`MI(t, MEMORY_AXI_R_BITS_LAST) == (read_beats == 1), "RLAST mismatch"); + read_beats--; + end + if (`MO(t, MEMORY_AXI_AW_VALID) && `MI(t, MEMORY_AXI_AW_READY)) begin + verify(store_active && write_beats == 0, "AXI write without a store owner"); + write_address = `MO(t, MEMORY_AXI_AW_BITS_ADDR); + write_beats = `MO(t, MEMORY_AXI_AW_BITS_LEN) + 1; + verify(`MO(t, MEMORY_AXI_AW_BITS_SIZE) == 4 && `MO(t, MEMORY_AXI_AW_BITS_BURST) == 1 && + `MO(t, MEMORY_AXI_AW_BITS_ID) == 1, "AXI write attributes mismatch"); + verify((write_address & 4095) + write_beats * 16 <= 4096, "AXI write crossed a page"); + end + if (`MO(t, MEMORY_AXI_W_VALID) && `MI(t, MEMORY_AXI_W_READY)) begin + verify(write_beats > 0, "Write data without AW"); + verify(`MO(t, MEMORY_AXI_W_BITS_LAST) == (write_beats == 1), "WLAST mismatch"); + for (int b = 0; b < 16; b++) begin + bit expected_strobe = store_bytes.exists(write_address + b); + verify(t.outputs[`TM_OUT_MEMORY_AXI_W_BITS_STRB_OFFSET+b] == expected_strobe, + "Store byte strobe mismatch"); + if (expected_strobe) begin + verify( + t.outputs[`TM_OUT_MEMORY_AXI_W_BITS_DATA_OFFSET+b*8+:8]===store_bytes[write_address+b], + "Store data differs from logical bank contents"); + memory[write_address+b] = store_bytes[write_address+b]; + store_bytes.delete(write_address + b); + end + end + write_address += 16; + write_beats--; + end + if (`MO(t, BANKREAD_READ_REQ_VALID) && `MI(t, BANKREAD_READ_REQ_READY)) begin + int i = store_op.index; + verify(store_active, "Bank read without a store owner"); + verify(`MO(t, BANKREAD_BANK_ID) == store_op.bank_id && `MO(t, BANKREAD_GROUP_ID) + == store_op.group_base + i % store_op.groups && `MO(t, BANKREAD_READ_REQ_BITS_ADDR) + == i / store_op.groups, "Store bank address/group mismatch"); + store_op.index++; + bank_reads++; + end + if (`MO(t, BANKWRITE_WRITE_REQ_VALID) && `MI(t, BANKWRITE_WRITE_REQ_READY)) begin + result_item result = result_item::type_id::create("load_actual"); + result_item plan; + verify(load_active && load_results.size() != 0, "Unexpected bank write"); + plan = load_results.pop_front(); + result.value = { + 32'(`MO(t, BANKWRITE_BANK_ID)), + 32'(`MO(t, BANKWRITE_GROUP_ID)), + 32'(`MO(t, BANKWRITE_WRITE_REQ_BITS_ADDR)), + 128'(`MO(t, BANKWRITE_WRITE_REQ_BITS_DATA)), + 16'hffff + }; + for (int b = 0; b < 16; b++) + verify(t.outputs[`TM_OUT_BANKWRITE_WRITE_REQ_BITS_MASK_0_OFFSET+b], + "Load did not replace the whole bank row"); + result.set_transaction_id(plan.get_transaction_id()); + actual.write(result); + if (!image.bank_error) + banks[ + `MO(t, BANKWRITE_BANK_ID) + ][ + `MO(t, BANKWRITE_GROUP_ID) + ][ + `MO(t, BANKWRITE_WRITE_REQ_BITS_ADDR) + ] = + `MO(t, BANKWRITE_WRITE_REQ_BITS_DATA); + load_op.index++; + bank_acks++; + end + if (`MO(t, METADATA_REQUEST_VALID) && `MI(t, METADATA_REQUEST_READY)) begin + result_item result = result_item::type_id::create("metadata_actual"); + result_item plan; + verify(load_active && load_op.is_mmio && load_results.size() != 0, + "Unexpected metadata write"); + plan = load_results.pop_front(); + result.value = { + 64'(`MO(t, METADATA_REQUEST_BITS_ADDR)), + 128'(`MO(t, METADATA_REQUEST_BITS_DATA)), + 16'(`MO(t, METADATA_REQUEST_BITS_MASK)) + }; + result.set_transaction_id(plan.get_transaction_id()); + actual.write(result); + load_op.index++; + metadata_acks++; + end + if (`MI(t, BANKREAD_READ_RESP_VALID) && `MO(t, BANKREAD_READ_RESP_READY)) begin + verify(bank_reads > 0, "Unexpected bank read response"); + bank_reads--; + end + if (`MI(t, BANKWRITE_WRITE_RESP_VALID) && `MO(t, BANKWRITE_WRITE_RESP_READY)) begin + verify(bank_acks > 0, "Unexpected bank write acknowledgment"); + bank_acks--; + end + if (`MI(t, METADATA_RESPONSE_VALID) && `MO(t, METADATA_RESPONSE_READY)) begin + verify(metadata_acks > 0, "Unexpected metadata acknowledgment"); + metadata_acks--; + end + if (`MO(t, IMAGE_START_VALID) && `MI(t, IMAGE_START_READY)) + verify(load_active && load_op.is_image && `MO(t, IMAGE_START_BITS_BYTES) == load_op.rows && + `MO(t, IMAGE_START_BITS_BUFFER) == load_op.bank_id - 4, + "Image start descriptor mismatch"); + if (`MO(t, IMAGE_TERMINAL_VALID) && `MI(t, IMAGE_TERMINAL_READY)) begin + int terminal_error = image.image_fault && load_op.fault == 5 ? 0 : load_op.fault; + verify(`MO(t, IMAGE_TERMINAL_BITS_ERROR) == terminal_error && + `MO(t, IMAGE_TERMINAL_BITS_ADDRESS) + == (terminal_error == 0 ? 64'd0 : load_op.fault_address), + "Image terminal status mismatch"); + end + if (`MO(t, IMAGE_DATA_VALID) && `MI(t, IMAGE_DATA_READY)) begin + result_item result = result_item::type_id::create("image_actual"); + verify(load_active && load_op.is_image, "Image data without an owner"); + result.value = `MO(t, IMAGE_DATA_BITS); + result.set_transaction_id((2 << 20) | (load_serial << 10) | image_words); + actual.write(result); + image_words++; + end + if (`MO(t, MEMORY_CONTROL_FINISHED_0_VALID)) begin + operation op = load_op; + verify(load_active, "Completion without an owner"); + verify(`MO(t, MEMORY_CONTROL_FINISHED_0_BITS_ERROR) == op.fault && + `MO(t, MEMORY_CONTROL_FINISHED_0_BITS_ADDRESS) == op.fault_address, + "Finished status differs from DMA/bank result"); + verify(load_results.size() == 0 && read_beats == 0 && bank_acks == 0 && metadata_acks == 0, + "Finished before data drained"); + end + if (`MO(t, MEMORY_CONTROL_FINISHED_1_VALID)) begin + operation op = store_op; + verify(store_active, "Completion without an owner"); + verify(`MO(t, MEMORY_CONTROL_FINISHED_1_BITS_ERROR) == op.fault && + `MO(t, MEMORY_CONTROL_FINISHED_1_BITS_ADDRESS) == op.fault_address, + "Finished status differs from DMA/bank result"); + verify(store_bytes.num() == 0 && write_beats == 0 && bank_reads == 0, + "Finished before data drained"); + end + if (`MO(t, MEMORY_CONTROL_FINISHED_2_VALID)) begin + operation op = load_op; + verify(load_active, "Completion without an owner"); + verify(`MO(t, MEMORY_CONTROL_FINISHED_2_BITS_ERROR) == op.fault && + `MO(t, MEMORY_CONTROL_FINISHED_2_BITS_ADDRESS) == op.fault_address, + "Finished status differs from DMA/bank result"); + verify(load_results.size() == 0 && read_beats == 0 && bank_acks == 0 && metadata_acks == 0, + "Finished before data drained"); + end + if (`MO(t, RESPONSE_VALID) && `MI(t, RESPONSE_READY)) begin + result_item result = result_item::type_id::create("completion_actual"); + int tag = `MO(t, RESPONSE_BITS_ROB_ID); + verify(pending.exists(tag), "Unsolicited command completion"); + result.value = { + 32'(tag), 32'(`MO(t, RESPONSE_BITS_FAULT_ERROR)), 64'(`MO(t, RESPONSE_BITS_FAULT_ADDRESS)) + }; + result.set_transaction_id(tag); + actual.write(result); + pending.delete(tag); + if (store_active && store_op.tag == tag) store_active = 0; + else begin + verify(load_active && load_op.tag == tag, "Completion owner mismatch"); + load_active = 0; + end + end + if (`MO(t, RESPONSE_VALID) && !`MI(t, RESPONSE_READY)) + behavior.sample(0, 0, load_active && store_active, 1, 0); + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + verify( + pending.num()==0&&!load_active&&!store_active&&load_results.size()==0&&store_bytes.num()==0, + "ToMem did not drain"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/filelist.f b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/filelist.f new file mode 100644 index 000000000..b8cb9ffa1 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/ToMem/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/interface.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/interface.sv new file mode 100644 index 000000000..b5639322d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/interface.sv @@ -0,0 +1,9 @@ +`include "parameters.svh" +interface tomem_if ( + input logic clock +); + logic reset = 1; + logic [`TM_INPUT_BITS-1:0] source = '0, backend = '0; + wire [ `TM_INPUT_BITS-1:0] inputs = source | backend; + wire [`TM_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/package.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/package.sv new file mode 100644 index 000000000..a6d32602f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/package.sv @@ -0,0 +1,21 @@ +`include "parameters.svh" +`define TI(F) vif.inputs[`TM_IN_``F``_OFFSET+:`TM_IN_``F``_WIDTH] +`define TO(F) vif.outputs[`TM_OUT_``F``_OFFSET+:`TM_OUT_``F``_WIDTH] +`define TB(F) vif.backend[`TM_IN_``F``_OFFSET+:`TM_IN_``F``_WIDTH] +`define TS(F) pins[`TM_IN_``F``_OFFSET+:`TM_IN_``F``_WIDTH] +`define MI(T, F) T.inputs[`TM_IN_``F``_OFFSET+:`TM_IN_``F``_WIDTH] +`define MO(T, F) T.outputs[`TM_OUT_``F``_OFFSET+:`TM_OUT_``F``_WIDTH] +package tomem_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/fixture.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/peer.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..ff6980172 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/seq/sequence.svh @@ -0,0 +1,118 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + model reference; + logic [`TM_INPUT_BITS-1:0] pins; + logic [`TM_OUTPUT_BITS-1:0] observed; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task send(int tag, int kind, longint unsigned address, int rows, int bank_id, + longint unsigned special = (64'd1 << 39)); + pins[`TM_IN_REQUEST_BITS_ROB_ID_OFFSET+:`TM_IN_REQUEST_BITS_ROB_ID_WIDTH] = tag; + pins[`TM_IN_REQUEST_BITS_INSTID_OFFSET+:`TM_IN_REQUEST_BITS_INSTID_WIDTH]=64'h123456789abcdef0+tag; + pins[`TM_IN_REQUEST_BITS_CONTEXT_PRIVILEGE_OFFSET+:`TM_IN_REQUEST_BITS_CONTEXT_PRIVILEGE_WIDTH]=3; + pins[`TM_IN_REQUEST_BITS_CONTEXT_SATP_OFFSET+:`TM_IN_REQUEST_BITS_CONTEXT_SATP_WIDTH]=64'h8000000000000010; + pins[`TM_IN_REQUEST_BITS_CMD_IS_STORE_OFFSET+:`TM_IN_REQUEST_BITS_CMD_IS_STORE_WIDTH]=kind==1; + pins[`TM_IN_REQUEST_BITS_CMD_IS_LOAD_OFFSET+:`TM_IN_REQUEST_BITS_CMD_IS_LOAD_WIDTH] = kind != 1; + pins[`TM_IN_REQUEST_BITS_CMD_IS_MVIN_2D_OFFSET+:`TM_IN_REQUEST_BITS_CMD_IS_MVIN_2D_WIDTH]=kind==2; + pins[`TM_IN_REQUEST_BITS_CMD_IS_IMAGE_OFFSET+:`TM_IN_REQUEST_BITS_CMD_IS_IMAGE_WIDTH]=kind==3; + pins[`TM_IN_REQUEST_BITS_CMD_IS_MMIO_OFFSET+:`TM_IN_REQUEST_BITS_CMD_IS_MMIO_WIDTH] = kind == 4; + pins[`TM_IN_REQUEST_BITS_CMD_IS_CONFIG_OFFSET+:`TM_IN_REQUEST_BITS_CMD_IS_CONFIG_WIDTH]=kind==5; + pins[`TM_IN_REQUEST_BITS_CMD_MEM_ADDR_OFFSET+:`TM_IN_REQUEST_BITS_CMD_MEM_ADDR_WIDTH] = address; + pins[`TM_IN_REQUEST_BITS_CMD_ITER_OFFSET+:`TM_IN_REQUEST_BITS_CMD_ITER_WIDTH] = rows; + pins[`TM_IN_REQUEST_BITS_CMD_BANK_ID_OFFSET+:`TM_IN_REQUEST_BITS_CMD_BANK_ID_WIDTH] = bank_id; + pins[`TM_IN_REQUEST_BITS_CMD_SPECIAL_OFFSET+:`TM_IN_REQUEST_BITS_CMD_SPECIAL_WIDTH] = special; + pins[`TM_IN_REQUEST_VALID_OFFSET+:`TM_IN_REQUEST_VALID_WIDTH] = 1; + do tick(); while (!observed[`TM_OUT_REQUEST_READY_OFFSET]); + pins[`TM_IN_REQUEST_VALID_OFFSET+:`TM_IN_REQUEST_VALID_WIDTH] = 0; + pins[`TM_IN_REQUEST_BITS_CMD_MEM_ADDR_OFFSET+:`TM_IN_REQUEST_BITS_CMD_MEM_ADDR_WIDTH] = '1; + endtask + task completed(int tag); + do tick(); while (reference.pending.exists(tag)); + repeat (3) tick(); + endtask + task body(); + pins = '0; + pins[`TM_IN_RESPONSE_READY_OFFSET+:`TM_IN_RESPONSE_READY_WIDTH] = 1; + repeat (3) tick(1); + tick(); + for (int offset = 0; offset < 16; offset++) begin + send(0, 0, 'h1000 + offset, 2, 0); + completed(0); + send(1, 1, 'h9000 + offset, 2, 1); + completed(1); + send(0, 0, 'h9000 + offset, 2, 0); + completed(0); + end + send(0, 0, 'h1ff3, 3, 0, 64'd3 << 39); + completed(0); + send(1, 1, 'h8ff5, 3, 1, 64'd3 << 39); + completed(1); + send(0, 0, 'h3000, 3, 0, (64'd1 << 63) | (64'd1 << 58) | (64'd1 << 39)); + completed(0); + send(1, 1, 'ha000, 3, 1, (64'd1 << 63) | (64'd1 << 58) | (64'd1 << 39)); + completed(1); + send(0, 2, 'h4003, 2, 0, + (64'd1 << 62) | (64'd2 << 59) | (64'd2 << 53) | (64'd7 << 43) | (64'd1 << 36)); + completed(0); + for (int length = 4; length <= 68; length += 32) begin + send(2, 3, 'h5000, length, 4); + completed(2); + end + send(3, 4, 'h6000, 3, 0, (64'd7 << 56) | (64'd32 << 39)); + completed(3); + send(2, 5, 0, 0, 4); + completed(2); + // Independent read and write producers retain their own ROB tags and data. + send(0, 0, 'h7000, 4, 0); + send(1, 1, 'hb000, 4, 1); + completed(0); + completed(1); + pins[`TM_IN_RESPONSE_READY_OFFSET+:`TM_IN_RESPONSE_READY_WIDTH] = 0; + send(0, 0, 'h7000, 1, 0); + repeat (80) tick(); + pins[`TM_IN_RESPONSE_READY_OFFSET+:`TM_IN_RESPONSE_READY_WIDTH] = 1; + completed(0); + reference.image.read_error = 1; + send(0, 0, 'h8000, 2, 0); + completed(0); + reference.image.read_error = 0; + reference.image.write_error = 1; + send(1, 1, 'hc000, 2, 1); + completed(1); + reference.image.write_error = 0; + reference.image.bank_error = 1; + send(0, 0, 'h8000, 2, 0); + completed(0); + reference.image.bank_error = 0; + reference.image.image_fault = 1; + send(2, 3, 'h5000, 12, 5); + completed(2); + reference.image.image_fault = 0; + reference.image.admission_error = 7; + send(0, 0, 'h8000, 2, 0); + completed(0); + reference.image.admission_error = 0; + reference.image.completion_error = 10; + send(0, 0, 'h8000, 1, 0); + completed(0); + reference.image.completion_error = 0; + send(0, 0, 'h8000, 0, 0); + completed(0); + send(0, 0, 'h8000, 4, 0); + repeat (8) tick(); + repeat (3) tick(1); + tick(); + send(0, 0, 'h8000, 1, 0); + completed(0); + repeat (8) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/suite.toml new file mode 100644 index 000000000..b3228fee5 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/suite.toml @@ -0,0 +1,19 @@ +[model] +manifest="" +library="" +[[tests]] +name="tomem" +top="tomem_tb" +test="protocol_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["tomem_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/tb.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/tb.sv new file mode 100644 index 000000000..6e6fd8e9f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/tb.sv @@ -0,0 +1,15 @@ +module tomem_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + tomem_if bus (clock); + ToMem dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual tomem_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/tests/test.svh new file mode 100644 index 000000000..b11c898c9 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/resources/tests/test.svh @@ -0,0 +1,16 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.reference = test_env.reference; + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/Emit.scala new file mode 100644 index 000000000..435d8501f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/Emit.scala @@ -0,0 +1,57 @@ +package framework.acc.memdomain.tomem + +import chisel3._ +import chisel3.reflect.DataMirror +import java.nio.file.{Files, Paths} +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.core.memory.access.preflight.configs.PmpParams + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val frontend = upickle.default.read[FrontendParam](values("frontend")) + val memory = upickle.default.read[MemDomainParam](values("memory")) + val gp = upickle.default.read[GpDomainParam](values("gp")) + val permission = upickle.default.read[PmpParams](values("permission")) + val transport = upickle.default.read[configs.Params](values("transport")) + + val definitions = scala.collection.mutable.ArrayBuffer[String]( + s"`define TM_BANKS ${memory.virtualBankCount + 2}", + s"`define TM_ROWS ${memory.bankEntries}", + s"`define TM_GROUPS ${memory.bankNum}", + s"`define TM_DATA_BITS ${memory.bankWidth}" + ) + + val binding = scala.collection.mutable.ArrayBuffer[String]() + var inputOffset = 0; var outputOffset = 0 + Files.createDirectories(Paths.get(directory)) + + def leaves(data: Data, path: String): Seq[(String, Data)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + leaves(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => leaves(field, s"${path}_$index") } + case field => if (field.getWidth > 0) Seq(path -> field) else Seq.empty + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new ToMem(frontend, memory, gp, transport, permission) + for ((name, field) <- leaves(dut.io, "")) { + val input = DataMirror.directionOf(field) == ActualDirection.Input + val direction = if (input) "IN" else "OUT"; val signal = if (input) "inputs" else "outputs"; + val offset = if (input) inputOffset else outputOffset; val width = field.getWidth + definitions += s"`define TM_${direction}_${name.toUpperCase}_OFFSET $offset"; + definitions += s"`define TM_${direction}_${name.toUpperCase}_WIDTH $width" + binding += s".io_$name(bus.$signal[$offset +: $width])"; + if (input) inputOffset += width else outputOffset += width + }; dut + }, + args = Array("--target-dir", s"$directory/ToMem", "--split-verilog") + ) + definitions += s"`define TM_INPUT_BITS $inputOffset"; definitions += s"`define TM_OUTPUT_BITS $outputOffset" + Files.writeString(Paths.get(directory, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, "ports.svh"), binding.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/ToMem.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/ToMem.scala new file mode 100644 index 000000000..864c9e8cf --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/ToMem.scala @@ -0,0 +1,434 @@ +package framework.acc.memdomain.tomem + +import framework.lib.arbiter.{Arbiter} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.gpdomain.ImagePort +import framework.acc.balldomain.mmio_bank.MetadataWrite +import framework.acc.memdomain.{BankPort, Request, Response} +import framework.core.memory.access.preflight.{MapQuery, MapResult} +import framework.ip.bus.axi4 +import framework.acc.memdomain.toddr._ +import framework.lib.queue.Queue + +class ControlPort(transportParam: TransportParam, permissionParam: PmpParams) extends Bundle { + + val read = Output(Valid(new DmaReadCommand( + transportParam, + permissionParam + ))) + + val write = Output(Valid(new WriteTransfer( + transportParam, + permissionParam + ))) + + val finished = Output(Vec(3, Valid(new DmaStatus))) + val complete = Flipped(Vec(3, Decoupled(new DmaStatus))) + + val decisions = Input(Vec( + 3, + new DmaDecision(transportParam) + )) + + val queries = Output(Vec( + 2, + new MapQuery(transportParam.mapping) + )) + + val mappings = Input(Vec( + 2, + new MapResult(transportParam.mapping) + )) + + val coherence = + new framework.acc.memdomain.tol1.Port(transportParam.interlock) + +} + +class Port(transportParam: TransportParam, permissionParam: PmpParams) extends Bundle { + val control = new ControlPort(transportParam, permissionParam) + + val axi = new axi4.Port(transportParam.axi) + +} + +@instantiable +class ToMem( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val gpParam: GpDomainParam, + val transportParam: TransportParam, + val permissionParam: PmpParams) + extends Module { + val ownerId = frontendParam.ownerId + require( + memParam.bankWidth == 128 && memParam.dma_buswidth == 128 + ) + val rowBytes = memParam.bankWidth / 8 + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new Request(frontendParam, memParam, permissionParam))) + val response = Decoupled(new Response(frontendParam)) + val bankRead = new BankPort(frontendParam, memParam) + val bankWrite = new BankPort(frontendParam, memParam) + val memory = new framework.acc.memdomain.tomem.Port(transportParam, permissionParam) + val metadata = Flipped(new MetadataWrite(memParam)) + val image = if (gpParam.enable) Some(new ImagePort) else None + val queryVbank = Output(UInt(frontendParam.bank_id_len.W)) + val queryShared = Output(Bool()) + val queryGroups = Input(UInt(memParam.groupCountWidth.W)) + }) + + io.bankRead.thread_id := ownerId.U + io.bankWrite.thread_id := ownerId.U + io.bankRead.write.req.valid := false.B + io.bankRead.write.req.bits := 0.U.asTypeOf(io.bankRead.write.req.bits) + io.bankRead.write.resp.ready := true.B + io.bankWrite.read.req.valid := false.B + io.bankWrite.read.req.bits := 0.U.asTypeOf(io.bankWrite.read.req.bits) + io.bankWrite.read.resp.ready := true.B + + val toDDR: Instance[ToDDR] = + Instantiate(new ToDDR(transportParam, permissionParam)) + + val toL1: Instance[framework.acc.memdomain.tol1.ToL1] = + Instantiate( + new framework.acc.memdomain.tol1.ToL1(transportParam.interlock) + ) + + val dma = + Wire(new DmaPort(transportParam, permissionParam)) + toDDR.io.dma <> dma + toDDR.io.decisions := io.memory.control.decisions + io.memory.control.queries := toDDR.io.queries + toDDR.io.mappings := io.memory.control.mappings + io.memory.axi <> toDDR.io.axi + io.memory.control.coherence <> toL1.io + io.memory.control.read.bits := dma.read.bits + io.memory.control.write.valid := dma.write.valid + io.memory.control.write.bits := dma.write.bits + + val idle :: query :: queryWait :: prepare :: Nil = Enum(4) + val setup = RegInit(idle) + val request = Reg(new Request(frontendParam, memParam, permissionParam)) + val readIdle :: readOffer :: readReceive :: readBank :: readBankWait :: readTerminal :: imageWait :: readComplete :: Nil = + Enum(8) + val readState = RegInit(readIdle) + val writeIdle :: writeOffer :: writeStream :: writeWait :: writeDrain :: writeComplete :: Nil = Enum(6) + val writeState = RegInit(writeIdle) + val read = Reg(new Request(frontendParam, memParam, permissionParam)) + val write = Reg(new Request(frontendParam, memParam, permissionParam)) + val readGroups = Reg(UInt(memParam.groupCountWidth.W)) + val writeGroups = Reg(UInt(memParam.groupCountWidth.W)) + val readGroupBase = Reg(UInt(memParam.groupIdWidth.W)) + val writeGroupBase = Reg(UInt(memParam.groupIdWidth.W)) + val readGroup = Reg(UInt(memParam.groupIdWidth.W)) + val writeGroup = Reg(UInt(memParam.groupIdWidth.W)) + val readRow = Reg(UInt(16.W)) + val writeRow = Reg(UInt(16.W)) + val readWords = Reg(UInt(32.W)) + val readData = Reg(UInt(128.W)) + val readLast = Reg(Bool()) + val imageWord = Reg(UInt(2.W)) + val imageBytes = Reg(UInt(32.W)) + val imageStarted = RegInit(false.B) + val writeFault = Reg(new DmaStatus) + val readFault = Reg(new DmaStatus) + val writeCursor = Reg(UInt(64.W)) + val writeFinal = Reg(Bool()) + val readsFinished = RegInit(false.B) + + val wantsWrite = io.request.bits.cmd.is_store + io.request.ready := setup === idle && Mux(wantsWrite, writeState === writeIdle, readState === readIdle) + when(io.request.fire) { request := io.request.bits; setup := query } + io.queryVbank := request.cmd.bank_id + io.queryShared := request.cmd.is_shared + when(setup === query)(setup := queryWait) + when(setup === queryWait)(setup := prepare) + + val selected = request.cmd.special(63) && !request.cmd.is_mvin_2d && !request.cmd.is_image && !request.cmd.is_mmio + val groups = + Mux(request.cmd.is_image || request.cmd.is_config || request.cmd.is_mmio, 1.U, Mux(selected, 1.U, io.queryGroups)) + val base = Mux(selected, request.cmd.special(62, 58), 0.U) + when(setup === prepare) { + setup := idle + when(request.cmd.is_image || request.cmd.is_config) { + assert( + gpParam.enable.B && request.cmd.bank_id >= memParam.virtualBankCount.U && + request.cmd.bank_id < (memParam.virtualBankCount + 2).U, + "Program bank is outside the image slots" + ) + } + when(request.cmd.is_store) { + writeFault := 0.U.asTypeOf(new DmaStatus) + write := request + writeGroups := groups + writeGroupBase := base + writeGroup := 0.U + writeRow := 0.U + writeCursor := request.cmd.mem_addr + + readsFinished := false.B + writeFinal := false.B + writeState := writeOffer + }.otherwise { + read := request + readGroups := groups + readGroupBase := base + readGroup := 0.U + readRow := Mux(request.cmd.is_mvin_2d, request.cmd.special(58, 53), 0.U) + readWords := 0.U + imageWord := 0.U + imageBytes := request.cmd.iter + imageStarted := false.B + readFault := 0.U.asTypeOf(readFault) + readState := Mux(request.cmd.is_config, readComplete, readOffer) + } + when(!request.cmd.is_image && !request.cmd.is_config && !request.cmd.is_mmio) { + assert(groups =/= 0.U, "Memory stream requires an allocated bank") + when(selected)(assert(base < io.queryGroups, "Selected group exceeds allocation")) + } + } + + val imageRead = read.cmd.is_image + + val imageRejected = + if (gpParam.enable) + io.image.get.complete.valid && io.image.get.complete.bits.fault + else false.B + + val width = read.cmd.special(61, 59) +& 1.U + val beats = + Mux(imageRead, (imageBytes.pad(33) + 15.U) >> 4, read.cmd.iter * Mux(read.cmd.is_mvin_2d, width, readGroups)) + val stride = Mux(imageRead || read.cmd.is_mvin_2d || read.cmd.is_mmio, 1.U, read.cmd.special(57, 39)) + + val readOfferReady = + if (gpParam.enable) !imageRead || imageStarted && !imageRejected + else true.B + + dma.read.valid := readState === readOffer && readOfferReady + dma.read.bits.producer := Mux(imageRead, 2.U, 0.U) + dma.read.bits.transfer.vaddr := read.cmd.mem_addr + dma.read.bits.transfer.len := Mux(imageRead, imageBytes, beats << 4) + dma.read.bits.transfer.groups := readGroups + dma.read.bits.transfer.stride := stride + dma.read.bits.transfer.is_2d := read.cmd.is_mvin_2d + dma.read.bits.transfer.pixel_bytes := read.cmd.special(42, 36) << 3 + dma.read.bits.transfer.source_width := read.cmd.special(52, 43) + dma.read.bits.transfer.core_width := width + when(dma.read.fire)(readState := readReceive) + dma.readResult.ready := readState === readReceive + when(dma.readResult.fire) { + readData := dma.readResult.bits.data + readLast := dma.readResult.bits.last + imageWord := 0.U + when(dma.readResult.bits.fault.error =/= DmaError.None.U || readFault.error =/= DmaError.None.U || imageRejected) { + when(dma.readResult.bits.fault.error =/= DmaError.None.U)(readFault := dma.readResult.bits.fault) + when(imageRejected && readFault.error === DmaError.None.U)(readFault.error := DmaError.Cancelled.U) + when(dma.readResult.bits.last)(readState := Mux(imageRead, readTerminal, readComplete)) + }.otherwise(readState := readBank) + } + + io.bankWrite.bank_id := read.cmd.bank_id + io.bankWrite.rob_id := read.rob_id + io.bankWrite.group_id := readGroupBase + readGroup + io.bankWrite.write.req.valid := readState === readBank && !imageRead && !read.cmd.is_mmio + io.bankWrite.write.req.bits.addr := readRow + // MVIN_2D replaces the entire row, including zero-padding unused bytes. + io.bankWrite.write.req.bits.data := + Mux(read.cmd.is_mvin_2d && read.cmd.special(62), Cat(0.U(64.W), readData(63, 0)), readData) + for (i <- 0 until 16) io.bankWrite.write.req.bits.mask(i) := true.B + io.bankWrite.write.resp.ready := readState === readBankWait && !read.cmd.is_mmio + io.metadata.request.valid := readState === readBank && read.cmd.is_mmio + io.metadata.request.bits.addr := read.cmd.special(55, 39) + (readRow << 4) + io.metadata.request.bits.data := readData + io.metadata.request.bits.mask := ((1.U(17.W) << read.cmd.special(63, 56)) - 1.U)(15, 0) + io.metadata.response.ready := readState === readBankWait && read.cmd.is_mmio + when(io.metadata.request.fire) { + assert(memParam.mmioEnable.B && read.cmd.special(63, 56) >= 1.U && read.cmd.special(63, 56) <= 16.U) + assert(read.cmd.special(55, 39) +& (read.cmd.iter << 4) <= memParam.mmioTotalBytes.U) + } + when(io.bankWrite.write.req.fire || io.metadata.request.fire)(readState := readBankWait) + when(io.bankWrite.write.resp.fire || io.metadata.response.fire) { + when(Mux(read.cmd.is_mmio, !io.metadata.response.bits, !io.bankWrite.write.resp.bits.ok)) { + readFault.error := DmaError.Bank.U; readFault.address := read.cmd.mem_addr + } + when(readGroup +& 1.U === readGroups || read.cmd.is_mvin_2d) { + readGroup := 0.U + readRow := readRow + 1.U + }.otherwise(readGroup := readGroup + 1.U) + readState := Mux(readLast, readComplete, readReceive) + } + + io.image.foreach { image => + image.start.valid := readState === readOffer && imageRead && !imageStarted + image.start.bits.buffer := (read.cmd.bank_id - memParam.virtualBankCount.U)( + 0 + ) + image.start.bits.bytes := imageBytes + when(image.start.fire)(imageStarted := true.B) + image.data.valid := readState === readBank && imageRead && !imageRejected + image.data.bits := (readData >> (imageWord << 5))(31, 0) + when(image.data.fire) { + readWords := readWords + 1.U + imageWord := imageWord + 1.U + when((readWords +& 1.U) * 4.U >= imageBytes || imageWord === 3.U) { + readState := Mux(readLast, readTerminal, readReceive) + } + } + when(imageRead && imageStarted && imageRejected) { + when(readState === readOffer) { + readFault.error := DmaError.Cancelled.U + readState := readTerminal + } + when(readState === readBank) { + readFault.error := DmaError.Cancelled.U + readState := Mux(readLast, readTerminal, readReceive) + } + } + image.terminal.valid := readState === readTerminal + image.terminal.bits := readFault + when(image.terminal.fire)(readState := imageWait) + image.complete.ready := readState === imageWait + when(image.complete.fire) { + when(image.complete.bits.fault && (readFault.error === DmaError.None.U || readFault.error === DmaError.Cancelled.U)) { + readFault.error := Mux( + image.complete.bits.cause === 5.U || image.complete.bits.cause === 7.U, + DmaError.Bank.U, + DmaError.Shape.U + ) + readFault.address := image.complete.bits.tval + } + readState := readComplete + } + image.release.valid := setup === prepare && request.cmd.is_config + image.release.bits := (request.cmd.bank_id - memParam.virtualBankCount.U)( + 0 + ) + } + + val tags = Instantiate(new Queue(Bool(), 2, false, true)) + + val data = Instantiate(new Queue( + new WriteData(transportParam), + 2, + true, + false + )) + + // Accepted SRAM reads reserve a slot until their data leaves the staging queue. + val reserved = tags.io.count +& data.io.count + + val totalBytes = write.cmd.iter * writeGroups * rowBytes.U + val writeStride = write.cmd.special(57, 39) + val descriptorBytes = Mux(writeStride === 1.U, totalBytes, writeGroups * rowBytes.U) + dma.write.valid := writeState === writeOffer + dma.write.bits.vaddr := writeCursor + dma.write.bits.len := descriptorBytes + when(dma.write.fire) { + writeFinal := writeStride === 1.U || writeRow +& 1.U === write.cmd.iter + writeState := writeStream + } + val lastGroup = writeGroup +& 1.U === writeGroups + val lastRow = writeRow +& 1.U === write.cmd.iter + val last = lastGroup && (writeStride =/= 1.U || lastRow) + io.bankRead.inst_id := write.instId + io.bankWrite.inst_id := read.instId + io.bankRead.bank_id := write.cmd.bank_id + io.bankRead.rob_id := write.rob_id + io.bankRead.group_id := writeGroupBase + writeGroup + io.bankRead.read.req.valid := writeState === writeStream && !readsFinished && reserved < 2.U && tags.io.enq.ready + io.bankRead.read.req.bits.addr := writeRow + tags.io.enq.valid := io.bankRead.read.req.fire + tags.io.enq.bits := last + io.bankRead.read.resp.ready := data.io.enq.ready && tags.io.deq.valid + data.io.enq.valid := io.bankRead.read.resp.valid && tags.io.deq.valid + data.io.enq.bits.data := io.bankRead.read.resp.bits.data + data.io.enq.bits.last := tags.io.deq.bits + tags.io.deq.ready := io.bankRead.read.resp.fire + when(io.bankRead.read.req.fire) { + when(lastGroup) { writeGroup := 0.U; writeRow := writeRow + 1.U } + .otherwise(writeGroup := writeGroup + 1.U) + when(last)(readsFinished := true.B) + } + dma.writeData.valid := writeState === writeStream && data.io.deq.valid + dma.writeData.bits := data.io.deq.bits + data.io.deq.ready := writeState === writeStream && dma.writeData.ready || writeState === writeDrain + when(dma.writeData.fire && dma.writeData.bits.last)(writeState := writeWait) + dma.writeResult.ready := writeState === writeWait + when(dma.writeResult.fire) { + when(dma.writeResult.bits.fault.error =/= DmaError.None.U) { + writeFault := dma.writeResult.bits.fault + + writeState := writeDrain + }.elsewhen(writeFinal)(writeState := writeComplete) + .otherwise { + writeCursor := writeCursor + writeGroups * writeStride * rowBytes.U + readsFinished := false.B + writeState := writeOffer + } + } + when(writeState === writeDrain && reserved === 0.U)(writeState := writeComplete) + + val readProducer = Mux(imageRead, 2.U, 0.U) + for (i <- 0 until 3) { + io.memory.control.finished(i).valid := (if (i == 1) writeState === writeComplete + else + readState === readComplete && !read.cmd.is_config && readProducer === i.U) + io.memory.control.finished(i).bits := (if (i == 1) writeFault else readFault) + io.memory.control.complete(i).ready := false.B + } + val completion = Instantiate(new Arbiter(new Response(frontendParam), 2)) + completion.io.in(0).valid := readState === readComplete && + (read.cmd.is_config || io.memory.control.complete(readProducer).valid) + completion.io.in(0).bits.rob_id := read.rob_id + completion.io.in(0).bits.fault := Mux(read.cmd.is_config, readFault, io.memory.control.complete(readProducer).bits) + completion.io.in(1).valid := writeState === writeComplete && io.memory.control.complete(1).valid + completion.io.in(1).bits.rob_id := write.rob_id + completion.io.in(1).bits.fault := io.memory.control.complete(1).bits + io.memory.control.complete( + 0 + ).ready := readState === readComplete && !imageRead && !read.cmd.is_config && completion.io.in(0).ready + io.memory.control.complete( + 2 + ).ready := readState === readComplete && imageRead && !read.cmd.is_config && completion.io.in(0).ready + io.memory.control.complete(1).ready := writeState === writeComplete && completion.io.in(1).ready + io.response <> completion.io.out + when(completion.io.in(0).fire)(readState := readIdle) + when(completion.io.in(1).fire)(writeState := writeIdle) + io.bankRead.is_shared := write.cmd.is_shared + io.bankWrite.is_shared := read.cmd.is_shared + + // Publish the read shape before image-loader backpressure can delay DMA. + // The setup stage accepts streams in order; coherence reservation follows that order. + io.memory.control.read.valid := readState === readOffer + val readAccess = dma.read.bits.access + readAccess.context := read.context + readAccess.baseVA := read.cmd.mem_addr + readAccess.rows := Mux(imageRead, 1.U, read.cmd.iter) + readAccess.columns := Mux(read.cmd.is_mvin_2d, width, 1.U) + readAccess.spanBytes := Mux(imageRead, (imageBytes.pad(64) + 15.U) & ~15.U(64.W), readGroups * rowBytes.U) + readAccess.rowStride := Mux( + read.cmd.is_mvin_2d, + read.cmd.special(52, 43) * (read.cmd.special(42, 36) << 3), + readGroups * stride * rowBytes.U + ) + readAccess.columnStride := Mux(read.cmd.is_mvin_2d, read.cmd.special(42, 36) << 3, 0.U) + dma.write.bits.access.context := write.context + dma.write.bits.access.baseVA := write.cmd.mem_addr + dma.write.bits.access.rows := write.cmd.iter + dma.write.bits.access.columnStride := 0.U + dma.write.bits.access.columns := 1.U + dma.write.bits.access.spanBytes := writeGroups * rowBytes.U + dma.write.bits.access.rowStride := writeGroups * writeStride * rowBytes.U +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..5f389790c --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/configs/Params.scala @@ -0,0 +1,42 @@ +package framework.acc.memdomain.tomem.configs + +import buckyball.config.TransportHardware +import scala.jdk.CollectionConverters._ + +import chisel3.util.log2Ceil +import framework.core.memory.access.PhysicalRegion +import framework.core.memory.access.preflight +import framework.ip.bus.{axi4, chi} +import framework.ip.memory.interlock +import upickle.default.{macroRW, ReadWriter} + +case class Params( + tracking: framework.ip.memory.interlock.Params, + prepared: preflight.Params, + axi: axi4.Params, + regions: Seq[PhysicalRegion]) { + val contexts = 1 << log2Ceil(tracking.entries) + val interlock = tracking.copy(entries = contexts) + val mapping = prepared.copy(contexts = contexts) + require(tracking.idBits >= log2Ceil(contexts)) + require(prepared.idBits == tracking.idBits && prepared.maxRanges == tracking.maxRanges) + require(tracking.addressBits == prepared.bus.addressBits && prepared.bus.addressBits == axi.addressBits) + require(axi.dataBits == 128 && prepared.beatBytes == axi.bytes) +} + +object Params { + + def fromProto(p: TransportHardware): Params = Params( + interlock.Params.fromProto(p.getTracking), + preflight.Params.fromProto(p.getPrepared), + axi4.Params.fromProto(p.getAxi), + p.getRegionsList.asScala.map(PhysicalRegion.fromProto).toSeq + ) + + implicit val chiRW: ReadWriter[chi.Params] = macroRW + implicit val axiRW: ReadWriter[axi4.Params] = macroRW + implicit val trackingRW: ReadWriter[interlock.Params] = macroRW + implicit val preparedRW: ReadWriter[preflight.Params] = macroRW + implicit val regionRW: ReadWriter[PhysicalRegion] = macroRW + implicit val rw: ReadWriter[Params] = macroRW +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/build.sc b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/build.sc new file mode 100644 index 000000000..8456bf352 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/build.sc @@ -0,0 +1,25 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.memdomain.toddr.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/DmaPort.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/DmaStatus.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/ReadDma.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/ToDDR.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/WriteDma.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/configs/uvm.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/configs/uvm.toml new file mode 100644 index 000000000..82d7fe90d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/configs/uvm.toml @@ -0,0 +1,32 @@ +[parameters.permission] +addressBits=44 +entries=16 +[parameters.transport.tracking] +entries=4 +addressBits=44 +idBits=8 +lineBytes=64 +maxRanges=8 +[parameters.transport.prepared] +contexts=4 +maxRanges=8 +idBits=8 +beatBytes=16 +[parameters.transport.prepared.bus] +nodeIdBits=7 +addressBits=44 +dataBits=256 +[parameters.transport.axi] +addressBits=44 +dataBits=128 +idBits=4 +[[parameters.transport.regions]] +base="0" +bytes="17592186044416" +normal=true +cacheable=true +atomic=true +readable=true +writable=true +executable=true +[defines] diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/Cargo.lock b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/Cargo.lock new file mode 100644 index 000000000..627bfe6b8 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "dma-reference" +version = "0.1.0" diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/Cargo.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/Cargo.toml new file mode 100644 index 000000000..2b1c44751 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "dma-reference" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib", "staticlib"] diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/src/lib.rs b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/src/lib.rs new file mode 100644 index 000000000..fbb816fbf --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/csrc/src/lib.rs @@ -0,0 +1,4 @@ +#[no_mangle] +pub extern "C" fn dma_ref_byte(address: u64) -> u32 { + ((address ^ (address >> 8)) & 255) as u32 +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/env/env.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/env/env.svh new file mode 100644 index 000000000..3227267e4 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/env/env.svh @@ -0,0 +1,44 @@ +class env extends uvm_env; + `uvm_component_utils(env) + virtual dma_if vif; + covergroup behavior with function sample ( + longint unsigned base, + int length, + int error, + bit bus_error, + bit two_dimensional, + bit cancellation + ); + option.per_instance = 1; + alignment: coverpoint (base % 16) {bins aligned = {0}; bins unaligned = {[1 : 15]};} + lengths: coverpoint length { + bins invalid = {0}; bins short_transfer = {[1 : 64]}; bins long_transfer = {[65 : $]}; + } + faults: coverpoint error {bins success = {0}; bins bus = {2}; bins shape = {7};} + backend_errors: coverpoint bus_error; + dimensionality: coverpoint two_dimensional; + reset_cancellation: coverpoint cancellation; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual dma_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "DMA interface is required") + endfunction + int cases = 0, beats = 0, bursts = 0, checks = 0; + function automatic bit [127:0] data_at(longint unsigned address); + bit [127:0] value; + for (int i = 0; i < 16; ++i) value[i*8+:8] = 8'(dma_ref_byte(address + i)); + return value; + endfunction + task automatic verify_contract(bit good, string reason); + verification_pkg::evidence::checks++; + checks++; + if (!good) `uvm_fatal("READ_DMA", reason) + endtask + + `include "seq/scenario.svh" +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/filelist.f b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/filelist.f new file mode 100644 index 000000000..75a203454 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/ReadDma/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/interface.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/interface.sv new file mode 100644 index 000000000..dd4db7cf6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/interface.sv @@ -0,0 +1,10 @@ +interface dma_if ( + input logic clock +); + logic reset = 1; + `include "read_dma_signals.svh" + `include "read_dma_clocking.svh" + task initialize(); + `include "read_dma_init.svh" + endtask +endinterface diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/package.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/package.sv new file mode 100644 index 000000000..57b04a4cf --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/package.sv @@ -0,0 +1,8 @@ +package dma_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function int unsigned dma_ref_byte(longint unsigned address); + `include "env/env.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/seq/scenario.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/seq/scenario.svh new file mode 100644 index 000000000..052ac0b1c --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/seq/scenario.svh @@ -0,0 +1,162 @@ +task automatic run_case(longint unsigned base, int length, int groups, int stride, bit mode2d = 0, + int pixel = 16, int width = 1, int source = 1, int expected_error = 0, + bit bus_error = 0, int cancel_after = 0); + int count, seen, pending, issued, tick, fault_seen; + longint unsigned read_address, expected_address, query_address; + int query_bytes; + bit [127:0] held_data; + bit held, held_last; + bit [ 3:0] held_error; + bit [63:0] held_address; + count = (length + 15) / 16; + seen = 0; + pending = 0; + issued = 0; + fault_seen = 0; + held = 0; + cases++; + behavior.sample(base, length, expected_error, bus_error, mode2d, cancel_after != 0); + @(negedge vif.clock); + vif.io_req_bits_vaddr = base; + vif.io_req_bits_len = length; + vif.io_req_bits_groups = groups; + vif.io_req_bits_stride = stride; + vif.io_req_bits_is_2d = mode2d; + vif.io_req_bits_pixel_bytes = pixel; + vif.io_req_bits_core_width = width; + vif.io_req_bits_source_width = source; + vif.io_req_valid = 1; + do @(vif.sample); while (!vif.sample.io_req_ready); + @(negedge vif.clock); + vif.io_req_valid = 0; + for (tick = 0; tick < 20000; ++tick) begin + vif.io_mapping_hit = 1; + vif.io_mapping_error = 0; + vif.io_mapping_pa = vif.io_query_va; + vif.io_axi_ar_ready = pending == 0 && tick % 3 != 0; + vif.io_axi_r_valid = pending != 0; + vif.io_axi_r_bits_data = data_at(read_address); + vif.io_axi_r_bits_last = pending == 1; + vif.io_axi_r_bits_resp = bus_error && issued == 1 ? 2 : 0; + vif.io_resp_ready = tick % 5 == 0; + @(vif.sample); + if (held) + verify_contract( + vif.sample.io_resp_valid && vif.sample.io_resp_bits_data == held_data && + vif.sample.io_resp_bits_last == held_last && vif.sample.io_resp_bits_fault_error == held_error && + vif.sample.io_resp_bits_fault_address == held_address, + "response changed under backpressure"); + held = vif.sample.io_resp_valid && !vif.sample.io_resp_ready; + held_data = vif.sample.io_resp_bits_data; + held_last = vif.sample.io_resp_bits_last; + held_error = vif.sample.io_resp_bits_fault_error; + held_address = vif.sample.io_resp_bits_fault_address; + if (vif.sample.io_query_valid) begin + verify_contract( + !vif.sample.io_query_write && vif.sample.io_query_va[3:0] == 0 && + vif.sample.io_query_bytes != 0 && vif.sample.io_query_bytes % 16 == 0, + "invalid prepared footprint"); + query_address = vif.sample.io_mapping_pa; + query_bytes = vif.sample.io_query_bytes; + end + if (vif.sample.io_axi_ar_valid && vif.sample.io_axi_ar_ready) begin + verify_contract( + vif.sample.io_axi_ar_bits_addr == query_address && + (int'(vif.sample.io_axi_ar_bits_len)+1)*16 == query_bytes, + "burst differs from prepared footprint"); + verify_contract(vif.sample.io_axi_ar_bits_size == 4 && vif.sample.io_axi_ar_bits_burst == 1, + "AXI request must be 128-bit INCR"); + verify_contract( + vif.sample.io_axi_ar_bits_addr[3:0] == 0 && + int'(vif.sample.io_axi_ar_bits_addr[11:0]) + (int'(vif.sample.io_axi_ar_bits_len)+1)*16 <= 4096, + "burst alignment/page boundary"); + pending = int'(vif.sample.io_axi_ar_bits_len) + 1; + read_address = vif.sample.io_axi_ar_bits_addr; + bursts++; + end + if (vif.sample.io_axi_r_valid && vif.sample.io_axi_r_ready) begin + pending--; + issued++; + read_address += 16; + end + if (vif.sample.io_resp_valid && vif.sample.io_resp_ready) begin + if (vif.sample.io_resp_bits_fault_error != 0) begin + verify_contract( + vif.sample.io_resp_bits_fault_error == expected_error && vif.sample.io_resp_bits_last, + "wrong terminal fault"); + verify_contract(pending == 0, "fault completed before accepted burst drained"); + fault_seen++; + end else begin + verify_contract(expected_error == 0 || bus_error, "shape failure dispatched data"); + expected_address = mode2d ? base + (seen/width)*source*pixel + (seen%width)*pixel : + base + (seen/groups)*groups*stride*16 + (seen%groups)*16; + verify_contract(vif.sample.io_resp_bits_data == data_at(expected_address), + "data/stride/unaligned assembly mismatch"); + verify_contract(vif.sample.io_resp_bits_addrcounter == seen, "beat index mismatch"); + if (!bus_error) + verify_contract(vif.sample.io_resp_bits_last == (seen + 1 == count), "last mismatch"); + seen++; + beats++; + end + if (vif.sample.io_resp_bits_last) begin + verify_contract(fault_seen == (expected_error != 0), "missing/unexpected terminal error"); + if (!expected_error) verify_contract(seen == count, "incomplete response"); + @(negedge vif.clock); + vif.io_resp_ready = 0; + vif.io_axi_r_valid = 0; + vif.io_axi_ar_ready = 0; + return; + end + end + @(negedge vif.clock); + if (cancel_after != 0 && tick == cancel_after) begin + verify_contract(issued == 0, "vif.reset fixture must precede AXI issue"); + vif.reset = 1; + vif.io_axi_r_valid = 0; + vif.io_axi_ar_ready = 0; + repeat (3) @(vif.sample); + @(negedge vif.clock); + vif.reset = 0; + repeat (4) begin + @(vif.sample); + verify_contract( + !vif.sample.io_resp_valid && !vif.sample.io_axi_ar_valid && !vif.sample.io_busy, + "vif.reset leaked old work"); + end + return; + end + end + `uvm_fatal("READ_DMA", "request timed out") +endtask + +task execute(); + vif.initialize(); + repeat (4) @(vif.sample); + @(negedge vif.clock); + vif.reset = 0; + vif.io_decisionValid = 1; + run_case('h1000, 16, 1, 1); + run_case('h1ff0, 16 * 257, 1, 1); + run_case('h3003, 53, 3, 5); + run_case('h4ff9, 17, 1, 1); + run_case('h6000, 16 * 19, 3, 7); + run_case('h8000, 16 * 64, 63, 524287); + run_case('h8000, 16 * 63, 63, 524287); + run_case('h9000, 16 * 20, 1, 1, 1, 7, 3, 7); + run_case('haff0, 16 * 20, 1, 1, 1, 16, 3, 7); + run_case('hb000, 16 * 17, 1, 1, 1, 1023, 15, 1023); + run_case(64'hfffffffffffffff0, 16, 1, 1, 0, 16, 1, 1, 0); + run_case(64'hfffffffffffffff1, 16, 1, 1, 0, 16, 1, 1, 7); + run_case(64'hfffffff000000000, 16 * 65536, 63, 524287, 0, 16, 1, 1, 7); + run_case(64'hfffffffffff00000, 16 * 65536, 1, 1, 1, 1023, 15, 1023, 7); + run_case('hc000, 0, 1, 1, 0, 16, 1, 1, 7); + run_case('hc000, 16, 0, 1, 0, 16, 1, 1, 7); + run_case('hc000, 16, 1, 1, 1, 0, 3, 3, 7); + run_case('hd000, 64, 1, 1, 0, 16, 1, 1, 2, 1); + run_case('he000, 256, 3, 1, 0, 16, 1, 1, 0, 0, 5); + run_case('hf000, 32, 1, 1); + `uvm_info("READ_DMA", $sformatf("%0d cases %0d checks %0d beats %0d bursts", cases, checks, + beats, bursts), UVM_LOW) + + +endtask diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/tb.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/tb.sv new file mode 100644 index 000000000..e79321835 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/tb.sv @@ -0,0 +1,13 @@ +module read_dma_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + dma_if bus (clock); + ReadDma dut ( + `include "read_dma_binding.svh" + ); + initial begin + uvm_config_db#(virtual dma_if)::set(null, "uvm_test_top.env", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/tests/test.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/tests/test.svh new file mode 100644 index 000000000..398043c00 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/read_dma/tests/test.svh @@ -0,0 +1,14 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + test_env.execute(); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/suite.toml new file mode 100644 index 000000000..8ada1f1c7 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/suite.toml @@ -0,0 +1,54 @@ +[model] +manifest="src/csrc/Cargo.toml" +library="libdma_reference" + +[[tests]] +functional_groups = ["dma_pkg::env::behavior"] +name = "dma" +top = "dma_tb" +test = "protocol_test" +filelist = "toddr/filelist.f" +timeout_ns = 2000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = ["dma_pkg::env::behavior"] +name = "read_dma" +top = "read_dma_tb" +test = "protocol_test" +filelist = "read_dma/filelist.f" +timeout_ns = 2000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = ["dma_pkg::env::behavior"] +name = "write_dma" +top = "write_dma_tb" +test = "protocol_test" +filelist = "write_dma/filelist.f" +timeout_ns = 2000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/env/env.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/env/env.svh new file mode 100644 index 000000000..b605189a6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/env/env.svh @@ -0,0 +1,42 @@ +class env extends uvm_env; + `uvm_component_utils(env) + virtual dma_if vif; + covergroup behavior with function sample ( + longint unsigned base, + int length, + int error, + bit bus_error, + bit two_dimensional, + bit cancellation + ); + option.per_instance = 1; + alignment: coverpoint (base % 16) {bins aligned = {0}; bins unaligned = {[1 : 15]};} + lengths: coverpoint length { + bins invalid = {0}; bins short_transfer = {[1 : 64]}; bins long_transfer = {[65 : $]}; + } + faults: coverpoint error {bins success = {0}; bins bus = {2}; bins shape = {7};} + backend_errors: coverpoint bus_error; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual dma_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "DMA interface is required") + endfunction + int checks = 0, cases = 0, bursts = 0; + function automatic void verify_contract(bit good, string reason); + verification_pkg::evidence::checks++; + checks++; + if (!good) `uvm_fatal("WRITE_DMA", reason) + endfunction + function automatic bit [127:0] data_at(longint unsigned address); + bit [127:0] value; + for (int i = 0; i < 16; i++) value[8*i+:8] = 8'(dma_ref_byte(address + i)); + return value; + endfunction + + `include "seq/scenario.svh" +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/filelist.f b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/filelist.f new file mode 100644 index 000000000..b38d44b6f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/ToDDR/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/interface.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/interface.sv new file mode 100644 index 000000000..a111b8902 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/interface.sv @@ -0,0 +1,10 @@ +interface dma_if ( + input logic clock +); + logic reset = 1; + `include "toddr_signals.svh" + `include "toddr_clocking.svh" + task initialize(); + `include "toddr_init.svh" + endtask +endinterface diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/package.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/package.sv new file mode 100644 index 000000000..57b04a4cf --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/package.sv @@ -0,0 +1,8 @@ +package dma_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function int unsigned dma_ref_byte(longint unsigned address); + `include "env/env.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/seq/scenario.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/seq/scenario.svh new file mode 100644 index 000000000..7677e756f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/seq/scenario.svh @@ -0,0 +1,186 @@ +task automatic run_case(longint unsigned base, int length, int expected_error = 0, + bit bus_error = 0); + int sent, remaining, accepted_bytes, tick; + longint unsigned write_address; + bit pending_b, held; + bit [127:0] held_data; + bit [15:0] held_mask; + bit held_last; + bit held_response, held_done; + bit [3:0] held_fault; + sent = 0; + remaining = 0; + accepted_bytes = 0; + pending_b = 0; + held = 0; + held_response = 0; + cases++; + behavior.sample(base, length, expected_error, bus_error, 0, 0); + vif.io_dma_write_bits_vaddr = base; + vif.io_dma_write_bits_len = length; + vif.io_dma_write_valid = 1; + do @(vif.sample); while (!vif.sample.io_dma_write_ready); + @(negedge vif.clock); + vif.io_dma_write_valid = 0; + for (tick = 0; tick < 30000; tick++) begin + vif.io_mappings_1_hit = 1; + vif.io_mappings_1_error = 0; + vif.io_mappings_1_pa = vif.io_queries_1_va; + vif.io_dma_writeData_valid = sent < length / 16 && tick % 3 != 0; + vif.io_dma_writeData_bits_data = data_at(base + sent * 16); + vif.io_dma_writeData_bits_last = sent + 1 == length / 16; + vif.io_axi_aw_ready = remaining == 0 && !pending_b && tick % 4 != 0; + vif.io_axi_w_ready = remaining > 0 && tick % 5 != 0; + vif.io_axi_b_valid = pending_b && tick % 3 == 0; + vif.io_axi_b_bits_id = 1; + vif.io_axi_b_bits_resp = bus_error ? 2 : 0; + vif.io_dma_writeResult_ready = tick % 7 == 0; + @(vif.sample); + if (held) + verify_contract( + vif.sample.io_axi_w_valid && vif.sample.io_axi_w_bits_data == held_data && vif.sample.io_axi_w_bits_strb == held_mask && vif.sample.io_axi_w_bits_last == held_last, + "W changed under backpressure"); + held = vif.sample.io_axi_w_valid && !vif.sample.io_axi_w_ready; + held_data = vif.sample.io_axi_w_bits_data; + held_mask = vif.sample.io_axi_w_bits_strb; + held_last = vif.sample.io_axi_w_bits_last; + if (held_response) + verify_contract( + vif.sample.io_dma_writeResult_valid && vif.sample.io_dma_writeResult_bits_done == held_done && vif.sample.io_dma_writeResult_bits_fault_error == held_fault, + "response changed under backpressure"); + held_response = vif.sample.io_dma_writeResult_valid && !vif.sample.io_dma_writeResult_ready; + held_done = vif.sample.io_dma_writeResult_bits_done; + held_fault = vif.sample.io_dma_writeResult_bits_fault_error; + if (vif.sample.io_dma_writeData_valid && vif.sample.io_dma_writeData_ready) sent++; + if (vif.sample.io_axi_aw_valid && vif.sample.io_axi_aw_ready) begin + remaining = int'(vif.sample.io_axi_aw_bits_len) + 1; + write_address = vif.sample.io_axi_aw_bits_addr; + verify_contract( + vif.sample.io_axi_aw_bits_id == 1 && vif.sample.io_axi_aw_bits_size == 4 && vif.sample.io_axi_aw_bits_burst == 1, + "AW format mismatch"); + verify_contract(write_address % 16 == 0 && (write_address % 4096) + remaining * 16 <= 4096, + "AW crosses a page"); + bursts++; + end + if (vif.sample.io_axi_w_valid && vif.sample.io_axi_w_ready) begin + verify_contract(remaining > 0 && vif.sample.io_axi_w_bits_last == (remaining == 1), + "WLAST mismatch"); + for (int i = 0; i < 16; i++) begin + verify_contract( + vif.sample.io_axi_w_bits_strb[i] == (write_address+i >= base && write_address+i < base+length), + "byte mask changed the transfer footprint"); + if (vif.sample.io_axi_w_bits_strb[i]) begin + verify_contract(vif.sample.io_axi_w_bits_data[8*i+:8] == 8'(dma_ref_byte(write_address + i + )), "unaligned write data mismatch"); + accepted_bytes++; + end + end + remaining--; + write_address += 16; + if (remaining == 0) pending_b = 1; + end + if (vif.sample.io_axi_b_valid && vif.sample.io_axi_b_ready) pending_b = 0; + if (vif.sample.io_dma_writeResult_valid && vif.sample.io_dma_writeResult_ready) begin + verify_contract( + vif.sample.io_dma_writeResult_bits_fault_error == expected_error && vif.sample.io_dma_writeResult_bits_done == (expected_error == 0), + "completion status mismatch"); + verify_contract(remaining == 0 && !pending_b, + "completed before accepted AXI traffic drained"); + if (expected_error == 0) + verify_contract(sent == length / 16 && accepted_bytes == length, + "incomplete write transfer"); + @(negedge vif.clock); + vif.io_dma_writeData_valid = 0; + vif.io_axi_aw_ready = 0; + vif.io_axi_w_ready = 0; + vif.io_axi_b_valid = 0; + vif.io_dma_writeResult_ready = 0; + return; + end + @(negedge vif.clock); + end + `uvm_fatal("WRITE_DMA", "request timed out") +endtask +int overlap_cycles = 0; +task automatic run_read(int producer = 2); + int pending, seen; + longint unsigned address; + pending = 0; + seen = 0; + vif.io_decisions_0_valid = 1; + vif.io_decisions_2_valid = 1; + vif.io_dma_read_bits_producer = producer; + vif.io_dma_read_bits_transfer_vaddr = 'hc000; + vif.io_dma_read_bits_transfer_len = 1024; + vif.io_dma_read_bits_transfer_groups = 1; + vif.io_dma_read_bits_transfer_stride = 1; + vif.io_dma_read_valid = 1; + do @(vif.sample); while (!vif.sample.io_dma_read_ready); + @(negedge vif.clock); + vif.io_dma_read_valid = 0; + for (int tick = 0; tick < 10000; tick++) begin + vif.io_mappings_0_hit = 1; + vif.io_mappings_0_error = 0; + vif.io_mappings_0_pa = vif.io_queries_0_va; + vif.io_axi_ar_ready = pending == 0; + vif.io_axi_r_valid = pending != 0; + vif.io_axi_r_bits_data = data_at(address); + vif.io_axi_r_bits_last = pending == 1; + vif.io_dma_readResult_ready = tick % 3 != 0; + @(vif.sample); + if (vif.sample.io_dma_readBusy && vif.sample.io_dma_writeBusy) overlap_cycles++; + if (vif.sample.io_axi_ar_valid && vif.sample.io_axi_ar_ready) begin + pending = int'(vif.sample.io_axi_ar_bits_len) + 1; + address = vif.sample.io_axi_ar_bits_addr; + end + if (vif.sample.io_axi_r_valid && vif.sample.io_axi_r_ready) begin + pending--; + address += 16; + end + if (vif.sample.io_dma_readResult_valid && vif.sample.io_dma_readResult_ready) begin + verify_contract(vif.sample.io_dma_readResult_bits_data == data_at('hc000 + seen * 16 + ) && vif.sample.io_dma_readResult_bits_fault_error == 0, + "concurrent read data mismatch"); + verify_contract( + vif.sample.io_dma_readResult_bits_addrcounter == seen && vif.sample.io_dma_readResult_bits_last == (seen == 63), + "concurrent read ordering mismatch"); + seen++; + if (seen == 64) begin + @(negedge vif.clock); + vif.io_axi_r_valid = 0; + vif.io_axi_ar_ready = 0; + vif.io_dma_readResult_ready = 0; + return; + end + end + @(negedge vif.clock); + end + `uvm_fatal("DMA", "concurrent read timed out") +endtask + +task execute(); + vif.initialize(); + repeat (3) @(vif.sample); + @(negedge vif.clock); + vif.reset = 0; + vif.io_decisions_1_valid = 1; + fork + run_read(); + begin + run_case('h1000, 16); + run_case('h1ff0, 16 * 257); + for (int offset = 1; offset < 16; offset++) run_case('h4ff0 + offset, 16 * 33); + run_case('h9000, 0, 7); + run_case('h9000, 17, 7); + run_case(64'hfffffffffffffff1, 16, 7); + run_case('ha003, 16 * 8, 2, 1); + run_case('hb000, 32); + end + join + run_read(0); + verify_contract(overlap_cycles > 0, "read and write engines did not overlap"); + `uvm_info("WRITE_DMA", $sformatf("%0d cases %0d checks %0d bursts", cases, checks, bursts), + UVM_LOW) + + +endtask diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/tb.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/tb.sv new file mode 100644 index 000000000..5a414bf4e --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/tb.sv @@ -0,0 +1,13 @@ +module dma_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + dma_if bus (clock); + ToDDR dut ( + `include "toddr_binding.svh" + ); + initial begin + uvm_config_db#(virtual dma_if)::set(null, "uvm_test_top.env", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/tests/test.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/tests/test.svh new file mode 100644 index 000000000..398043c00 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/toddr/tests/test.svh @@ -0,0 +1,14 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + test_env.execute(); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/env/env.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/env/env.svh new file mode 100644 index 000000000..b605189a6 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/env/env.svh @@ -0,0 +1,42 @@ +class env extends uvm_env; + `uvm_component_utils(env) + virtual dma_if vif; + covergroup behavior with function sample ( + longint unsigned base, + int length, + int error, + bit bus_error, + bit two_dimensional, + bit cancellation + ); + option.per_instance = 1; + alignment: coverpoint (base % 16) {bins aligned = {0}; bins unaligned = {[1 : 15]};} + lengths: coverpoint length { + bins invalid = {0}; bins short_transfer = {[1 : 64]}; bins long_transfer = {[65 : $]}; + } + faults: coverpoint error {bins success = {0}; bins bus = {2}; bins shape = {7};} + backend_errors: coverpoint bus_error; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual dma_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "DMA interface is required") + endfunction + int checks = 0, cases = 0, bursts = 0; + function automatic void verify_contract(bit good, string reason); + verification_pkg::evidence::checks++; + checks++; + if (!good) `uvm_fatal("WRITE_DMA", reason) + endfunction + function automatic bit [127:0] data_at(longint unsigned address); + bit [127:0] value; + for (int i = 0; i < 16; i++) value[8*i+:8] = 8'(dma_ref_byte(address + i)); + return value; + endfunction + + `include "seq/scenario.svh" +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/filelist.f b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/filelist.f new file mode 100644 index 000000000..60f89a58f --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/WriteDma/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/interface.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/interface.sv new file mode 100644 index 000000000..86ef990fc --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/interface.sv @@ -0,0 +1,10 @@ +interface dma_if ( + input logic clock +); + logic reset = 1; + `include "write_dma_signals.svh" + `include "write_dma_clocking.svh" + task initialize(); + `include "write_dma_init.svh" + endtask +endinterface diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/package.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/package.sv new file mode 100644 index 000000000..57b04a4cf --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/package.sv @@ -0,0 +1,8 @@ +package dma_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function int unsigned dma_ref_byte(longint unsigned address); + `include "env/env.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/seq/scenario.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/seq/scenario.svh new file mode 100644 index 000000000..97e1e61fc --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/seq/scenario.svh @@ -0,0 +1,123 @@ +task automatic run_case(longint unsigned base, int length, int expected_error = 0, + bit bus_error = 0); + int sent, remaining, accepted_bytes, tick; + longint unsigned write_address; + bit pending_b, held; + bit [127:0] held_data; + bit [15:0] held_mask; + bit held_last; + bit held_response, held_done; + bit [3:0] held_fault; + sent = 0; + remaining = 0; + accepted_bytes = 0; + pending_b = 0; + held = 0; + held_response = 0; + cases++; + behavior.sample(base, length, expected_error, bus_error, 0, 0); + vif.io_req_bits_vaddr = base; + vif.io_req_bits_len = length; + vif.io_req_valid = 1; + do @(vif.sample); while (!vif.sample.io_req_ready); + @(negedge vif.clock); + vif.io_req_valid = 0; + for (tick = 0; tick < 30000; tick++) begin + vif.io_mapping_hit = 1; + vif.io_mapping_error = 0; + vif.io_mapping_pa = vif.io_query_va; + vif.io_data_valid = sent < length / 16 && tick % 3 != 0; + vif.io_data_bits_data = data_at(base + sent * 16); + vif.io_data_bits_last = sent + 1 == length / 16; + vif.io_axi_aw_ready = remaining == 0 && !pending_b && tick % 4 != 0; + vif.io_axi_w_ready = remaining > 0 && tick % 5 != 0; + vif.io_axi_b_valid = pending_b && tick % 3 == 0; + vif.io_axi_b_bits_id = 1; + vif.io_axi_b_bits_resp = bus_error ? 2 : 0; + vif.io_resp_ready = tick % 7 == 0; + @(vif.sample); + if (held) + verify_contract( + vif.sample.io_axi_w_valid && vif.sample.io_axi_w_bits_data == held_data && vif.sample.io_axi_w_bits_strb == held_mask && vif.sample.io_axi_w_bits_last == held_last, + "W changed under backpressure"); + held = vif.sample.io_axi_w_valid && !vif.sample.io_axi_w_ready; + held_data = vif.sample.io_axi_w_bits_data; + held_mask = vif.sample.io_axi_w_bits_strb; + held_last = vif.sample.io_axi_w_bits_last; + if (held_response) + verify_contract( + vif.sample.io_resp_valid && vif.sample.io_resp_bits_done == held_done && vif.sample.io_resp_bits_fault_error == held_fault, + "response changed under backpressure"); + held_response = vif.sample.io_resp_valid && !vif.sample.io_resp_ready; + held_done = vif.sample.io_resp_bits_done; + held_fault = vif.sample.io_resp_bits_fault_error; + if (vif.sample.io_data_valid && vif.sample.io_data_ready) sent++; + if (vif.sample.io_axi_aw_valid && vif.sample.io_axi_aw_ready) begin + remaining = int'(vif.sample.io_axi_aw_bits_len) + 1; + write_address = vif.sample.io_axi_aw_bits_addr; + verify_contract( + vif.sample.io_axi_aw_bits_id == 1 && vif.sample.io_axi_aw_bits_size == 4 && vif.sample.io_axi_aw_bits_burst == 1, + "AW format mismatch"); + verify_contract(write_address % 16 == 0 && (write_address % 4096) + remaining * 16 <= 4096, + "AW crosses a page"); + bursts++; + end + if (vif.sample.io_axi_w_valid && vif.sample.io_axi_w_ready) begin + verify_contract(remaining > 0 && vif.sample.io_axi_w_bits_last == (remaining == 1), + "WLAST mismatch"); + for (int i = 0; i < 16; i++) begin + verify_contract( + vif.sample.io_axi_w_bits_strb[i] == (write_address+i >= base && write_address+i < base+length), + "byte mask changed the transfer footprint"); + if (vif.sample.io_axi_w_bits_strb[i]) begin + verify_contract(vif.sample.io_axi_w_bits_data[8*i+:8] == 8'(dma_ref_byte(write_address + i + )), "unaligned write data mismatch"); + accepted_bytes++; + end + end + remaining--; + write_address += 16; + if (remaining == 0) pending_b = 1; + end + if (vif.sample.io_axi_b_valid && vif.sample.io_axi_b_ready) pending_b = 0; + if (vif.sample.io_resp_valid && vif.sample.io_resp_ready) begin + verify_contract( + vif.sample.io_resp_bits_fault_error == expected_error && vif.sample.io_resp_bits_done == (expected_error == 0), + "completion status mismatch"); + verify_contract(remaining == 0 && !pending_b, + "completed before accepted AXI traffic drained"); + if (expected_error == 0) + verify_contract(sent == length / 16 && accepted_bytes == length, + "incomplete write transfer"); + @(negedge vif.clock); + vif.io_data_valid = 0; + vif.io_axi_aw_ready = 0; + vif.io_axi_w_ready = 0; + vif.io_axi_b_valid = 0; + vif.io_resp_ready = 0; + return; + end + @(negedge vif.clock); + end + `uvm_fatal("WRITE_DMA", "request timed out") +endtask + +task execute(); + vif.initialize(); + repeat (3) @(vif.sample); + @(negedge vif.clock); + vif.reset = 0; + vif.io_decisionValid = 1; + run_case('h1000, 16); + run_case('h1ff0, 16 * 257); + for (int offset = 1; offset < 16; offset++) run_case('h4ff0 + offset, 16 * 33); + run_case('h9000, 0, 7); + run_case('h9000, 17, 7); + run_case(64'hfffffffffffffff1, 16, 7); + run_case('ha003, 16 * 8, 2, 1); + run_case('hb000, 32); + `uvm_info("WRITE_DMA", $sformatf("%0d cases %0d checks %0d bursts", cases, checks, bursts), + UVM_LOW) + + +endtask diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/tb.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/tb.sv new file mode 100644 index 000000000..296921171 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/tb.sv @@ -0,0 +1,13 @@ +module write_dma_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + dma_if bus (clock); + WriteDma dut ( + `include "write_dma_binding.svh" + ); + initial begin + uvm_config_db#(virtual dma_if)::set(null, "uvm_test_top.env", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/tests/test.svh b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/tests/test.svh new file mode 100644 index 000000000..398043c00 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/resources/write_dma/tests/test.svh @@ -0,0 +1,14 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + test_env.execute(); + endtask +endclass diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/DmaPort.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/DmaPort.scala new file mode 100644 index 000000000..df0229ee3 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/DmaPort.scala @@ -0,0 +1,85 @@ +package framework.acc.memdomain.toddr + +import framework.core.memory.access.preflight.configs.PmpParams + +import framework.acc.memdomain.tomem.configs.{Params => ModuleParam} + +import chisel3._ +import framework.acc.memdomain.MemoryContext +import chisel3.util._ +import framework.core.memory.access.preflight.Params + +class ReadTransfer extends Bundle { + val vaddr = UInt(64.W) + val len = UInt(32.W) + val stride = UInt(19.W) + val groups = UInt(6.W) + val is_2d = Bool() + val pixel_bytes = UInt(10.W) + val source_width = UInt(10.W) + val core_width = UInt(4.W) +} + +class ReadResult(b: ModuleParam) extends Bundle { + val dataWidth = b.axi.dataBits + + val data = UInt(dataWidth.W) + val last = Bool() + val addrcounter = UInt(16.W) + val fault = new DmaStatus +} + +class WriteTransfer(b: ModuleParam, permission: PmpParams) extends Bundle { + val vaddr = UInt(64.W) + val len = UInt(32.W) + val access = new AccessRange(b, permission) +} + +class WriteData(b: ModuleParam) extends Bundle { + val dataWidth = b.axi.dataBits + + val data = UInt(dataWidth.W) + val last = Bool() +} + +class WriteResult extends Bundle { + val done = Bool() + val fault = new DmaStatus +} + +class DmaDecision(b: ModuleParam) extends Bundle { + val p = b.mapping + + val valid = Bool() + val parentId = UInt(p.idBits.W) + val fault = new DmaStatus +} + +class AccessRange(b: ModuleParam, permission: PmpParams) extends Bundle { + val context = new MemoryContext(permission) + val baseVA = UInt(64.W) + val rows = UInt(64.W) + val columns = UInt(64.W) + val spanBytes = UInt(64.W) + val columnStride = UInt(64.W) + val rowStride = UInt(64.W) +} + +class DmaReadCommand(b: ModuleParam, permission: PmpParams) extends Bundle { + val transfer = new ReadTransfer + val producer = UInt(2.W) + val access = new AccessRange(b, permission) +} + +/** Accelerator-side logical transfers. Translation and external-memory transport belong to the system. */ +class DmaPort(b: ModuleParam, permission: PmpParams) extends Bundle { + val dataBits = b.axi.dataBits + + val read = Decoupled(new DmaReadCommand(b, permission)) + val readResult = Flipped(Decoupled(new ReadResult(b))) + val write = Decoupled(new WriteTransfer(b, permission)) + val writeData = Decoupled(new WriteData(b)) + val writeResult = Flipped(Decoupled(new WriteResult)) + val readBusy = Input(Bool()) + val writeBusy = Input(Bool()) +} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/DmaStatus.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/DmaStatus.scala similarity index 89% rename from arch/src/main/scala/framework/memdomain/frontend/mem/dma/DmaStatus.scala rename to arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/DmaStatus.scala index d0e84e73d..128740835 100644 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/DmaStatus.scala +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/DmaStatus.scala @@ -1,4 +1,4 @@ -package framework.memdomain.frontend.mem.dma +package framework.acc.memdomain.toddr import chisel3._ diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/Emit.scala new file mode 100644 index 000000000..88967af02 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/Emit.scala @@ -0,0 +1,66 @@ +package framework.acc.memdomain.toddr + +import chisel3._ +import chisel3.reflect.DataMirror +import java.nio.file.{Files, Paths} +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} +import framework.core.memory.access.preflight.configs.PmpParams + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val transport = upickle.default.read[TransportParam](values("transport")) + val permission = upickle.default.read[PmpParams](values("permission")) + Files.createDirectories(Paths.get(directory)) + + def leaves(data: Data, prefix: String): Seq[(String, Data)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => leaves(field, s"${prefix}_$name") } + case vector: Vec[_] => vector.zipWithIndex.flatMap { case (field, index) => + leaves(field, s"${prefix}_$index") + }.toSeq + case field => if (field.getWidth > 0) Seq(prefix -> field) else Seq.empty + } + + for ((name, top) <- Seq("toddr" -> "ToDDR", "read_dma" -> "ReadDma", "write_dma" -> "WriteDma")) { + var interface: Data = null + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + name match { + case "toddr" => + val m = + new ToDDR(transport, permission); + interface = m.io; m + case "read_dma" => val m = new ReadDma(transport); interface = m.io; m + case "write_dma" => + val m = + new WriteDma(transport, permission); + interface = m.io; m + }, + args = Array("--target-dir", s"$directory/$top", "--split-verilog") + ) + val fields = leaves(interface, "io") + Files.writeString( + Paths.get(s"$directory/${name}_signals.svh"), + fields.map { case (n, d) => s"logic [${d.getWidth - 1}:0] $n;" }.mkString("\n") + "\n" + ) + Files.writeString( + Paths.get(s"$directory/${name}_ports.svh"), + (Seq(".clock(clock)", ".reset(reset)") ++ fields.map { case (n, _) => s".$n($n)" }).mkString(",\n") + "\n" + ) + Files.writeString( + Paths.get(s"$directory/${name}_binding.svh"), + (Seq(".clock(clock)", ".reset(bus.reset)") ++ fields.map { case (n, _) => s".$n(bus.$n)" }).mkString(",\n") + "\n" + ) + Files.writeString( + Paths.get(s"$directory/${name}_init.svh"), + fields.collect { case (n, d) if DataMirror.directionOf(d) == ActualDirection.Input => s"$n = '0;" }.mkString( + "\n" + ) + "\n" + ) + Files.writeString( + Paths.get(s"$directory/${name}_clocking.svh"), + "clocking sample @(posedge clock);\ndefault input #1step;\ninput reset;\n" + fields.map { case (n, _) => + s"input $n;" + }.mkString("\n") + "\nendclocking\n" + ) + } +} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/ReadDma.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ReadDma.scala similarity index 89% rename from arch/src/main/scala/framework/memdomain/frontend/mem/dma/ReadDma.scala rename to arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ReadDma.scala index 6974395ff..e759006db 100644 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/ReadDma.scala +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ReadDma.scala @@ -1,34 +1,45 @@ -package framework.memdomain.frontend.mem.dma +package framework.acc.memdomain.toddr + +import framework.acc.memdomain.tomem.configs.{Params => ModuleParam} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig -import memcore.memory.preflight.{Error, MapQuery, MapResult, Params} -import memcore.bus.axi4 +import framework.core.memory.access.preflight.{Error, MapQuery, MapResult} +import framework.ip.bus.axi4 @instantiable -class ReadDma(b: GlobalConfig, prepared: Params, axiParams: axi4.Params) extends Module { - require(b.memDomain.bankWidth == 128 && b.memDomain.dma_buswidth == 128 && axiParams.dataBits == 128) +class ReadDma(val b: ModuleParam) extends Module { + val p = b + + val prepared = p.mapping + val axiParams = p.axi + + require(axiParams.dataBits == 128) require(prepared.beatBytes == 16 && prepared.bus.addressBits == axiParams.addressBits) @public val io = IO(new Bundle { - val req = Flipped(Decoupled(new BBReadRequest)) - val resp = Decoupled(new BBReadResponse(128)) + val req = Flipped(Decoupled(new ReadTransfer)) + val resp = + Decoupled(new ReadResult(b)) val parentId = Input(UInt(prepared.idBits.W)) val decisionValid = Input(Bool()) val decisionFault = Input(new DmaStatus) val busy = Output(Bool()) - val query = Output(new MapQuery(prepared)) - val mapping = Input(new MapResult(prepared)) - val axi = new axi4.Port(axiParams) + + val query = Output(new MapQuery(b.mapping)) + + val mapping = Input(new MapResult(b.mapping)) + + val axi = new axi4.Port(axiParams) + }) val idle :: shape :: divide :: products :: offset :: check :: plan :: query :: offer :: receive :: respond :: drain :: failed :: Nil = Enum(13) val state = RegInit(idle) - val command = Reg(new BBReadRequest) + val command = Reg(new ReadTransfer) val parent = Reg(UInt(prepared.idBits.W)) val total = Reg(UInt(17.W)) val progress = RegInit(0.U(17.W)) @@ -51,7 +62,7 @@ class ReadDma(b: GlobalConfig, prepared: Params, axiParams: axi4.Params) extends io.resp.bits.last := state === failed || progress +& 1.U === total io.resp.bits.addrcounter := progress(15, 0) io.resp.bits.fault := Mux(state === failed, fault, 0.U.asTypeOf(new DmaStatus)) - io.query := 0.U.asTypeOf(new MapQuery(prepared)) + io.query := 0.U.asTypeOf(new MapQuery(b.mapping)) io.query.valid := state === query io.query.id := parent io.query.va := queryVA @@ -83,8 +94,8 @@ class ReadDma(b: GlobalConfig, prepared: Params, axiParams: axi4.Params) extends fault := 0.U.asTypeOf(new DmaStatus) val bad = io.req.bits.len === 0.U || count > 65536.U || io.req.bits.groups === 0.U || io.req.bits.stride === 0.U || - (io.req.bits.is_2d && (io.req.bits.pixel_bytes === 0.U || io.req.bits.tile_width === 0.U || - io.req.bits.source_width < io.req.bits.tile_width)) + (io.req.bits.is_2d && (io.req.bits.pixel_bytes === 0.U || io.req.bits.core_width === 0.U || + io.req.bits.source_width < io.req.bits.core_width)) when(io.decisionFault.error =/= DmaError.None.U) { fail(io.decisionFault.error, io.decisionFault.address) }.elsewhen(bad)(fail(DmaError.Shape.U, io.req.bits.vaddr)).otherwise { @@ -102,7 +113,7 @@ class ReadDma(b: GlobalConfig, prepared: Params, axiParams: axi4.Params) extends val rowOffset = Reg(UInt(45.W)) val columnOffset = Reg(UInt(16.W)) val lastOffset = Reg(UInt(46.W)) - val shapeDivisor = Mux(command.is_2d, command.tile_width.pad(6), command.groups) + val shapeDivisor = Mux(command.is_2d, command.core_width.pad(6), command.groups) when(state === shape) { quotient := (total - 1.U)(15, 0) remainder := 0.U @@ -139,7 +150,7 @@ class ReadDma(b: GlobalConfig, prepared: Params, axiParams: axi4.Params) extends Mux( command.pixel_bytes =/= 16.U, 1.U, - Mux(command.source_width === command.tile_width, total - progress, command.tile_width - column) + Mux(command.source_width === command.core_width, total - progress, command.core_width - column) ), Mux(command.stride === 1.U, total - progress, command.groups - group) ) @@ -201,8 +212,8 @@ class ReadDma(b: GlobalConfig, prepared: Params, axiParams: axi4.Params) extends val next = Mux( command.is_2d, Mux( - column +& 1.U === command.tile_width, - address + (command.source_width - command.tile_width +& 1.U) * command.pixel_bytes, + column +& 1.U === command.core_width, + address + (command.source_width - command.core_width +& 1.U) * command.pixel_bytes, address + command.pixel_bytes ), Mux( @@ -211,7 +222,7 @@ class ReadDma(b: GlobalConfig, prepared: Params, axiParams: axi4.Params) extends address + 16.U ) ) - when(command.is_2d)(column := Mux(column +& 1.U === command.tile_width, 0.U, column + 1.U)) + when(command.is_2d)(column := Mux(column +& 1.U === command.core_width, 0.U, column + 1.U)) .otherwise(group := Mux(group +& 1.U === command.groups, 0.U, group + 1.U)) address := next; progress := progress + 1.U; second := false.B state := Mux(remaining =/= 0.U, receive, plan) diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ToDDR.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ToDDR.scala new file mode 100644 index 000000000..4a4df2a9b --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ToDDR.scala @@ -0,0 +1,85 @@ +package framework.acc.memdomain.toddr + +import framework.core.memory.access.preflight.configs.PmpParams + +import framework.acc.memdomain.tomem.configs.{Params => ModuleParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.memory.access.preflight.{MapQuery, MapResult} +import framework.ip.bus.axi4 + +@instantiable +class ToDDR(val b: ModuleParam, permission: PmpParams) extends Module { + val p = b + + val prepared = p.mapping + val axiParams = p.axi + + @public + val io = IO(new Bundle { + val dma = + Flipped(new DmaPort(b, permission)) + + // One decision for each physical producer: loader, storer, kernel image. + val decisions = Input(Vec( + 3, + new DmaDecision(b) + )) + + val queries = Output(Vec( + 2, + new MapQuery(b.mapping) + )) + + val mappings = Input(Vec( + 2, + new MapResult(b.mapping) + )) + + val axi = new axi4.Port(axiParams) + + }) + + val reader: Instance[ReadDma] = Instantiate(new ReadDma(b)) + val writer: Instance[WriteDma] = Instantiate(new WriteDma(b, permission)) + val kernel = io.dma.read.bits.producer === 2.U + val readDecision = Mux(kernel, io.decisions(2), io.decisions(0)) + when(io.dma.read.valid) { + assert(io.dma.read.bits.producer === 0.U || kernel, "DMA read has an unknown producer") + } + reader.io.req.valid := io.dma.read.valid + reader.io.req.bits := io.dma.read.bits.transfer + io.dma.read.ready := reader.io.req.ready + io.dma.readResult <> reader.io.resp + reader.io.parentId := readDecision.parentId + reader.io.decisionValid := readDecision.valid + reader.io.decisionFault := readDecision.fault + writer.io.req <> io.dma.write + writer.io.data <> io.dma.writeData + io.dma.writeResult <> writer.io.resp + writer.io.parentId := io.decisions(1).parentId + writer.io.decisionValid := io.decisions(1).valid + writer.io.decisionFault := io.decisions(1).fault + io.dma.readBusy := reader.io.busy + io.dma.writeBusy := writer.io.busy + io.queries(0) := reader.io.query + io.queries(1) := writer.io.query + reader.io.mapping := io.mappings(0) + writer.io.mapping := io.mappings(1) + + // AXI read and write channels retain independent command owners. + io.axi.ar <> reader.io.axi.ar + reader.io.axi.r <> io.axi.r + io.axi.aw <> writer.io.axi.aw + io.axi.w <> writer.io.axi.w + writer.io.axi.b <> io.axi.b + reader.io.axi.aw.ready := false.B + reader.io.axi.w.ready := false.B + reader.io.axi.b.valid := false.B + reader.io.axi.b.bits := 0.U.asTypeOf(new axi4.WriteResponse(axiParams)) + writer.io.axi.ar.ready := false.B + writer.io.axi.r.valid := false.B + writer.io.axi.r.bits := 0.U.asTypeOf(new axi4.ReadData(axiParams)) +} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/WriteDma.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/WriteDma.scala similarity index 91% rename from arch/src/main/scala/framework/memdomain/frontend/mem/dma/WriteDma.scala rename to arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/WriteDma.scala index 159948db2..9af1a150d 100644 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/WriteDma.scala +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/WriteDma.scala @@ -1,34 +1,47 @@ -package framework.memdomain.frontend.mem.dma +package framework.acc.memdomain.toddr + +import framework.core.memory.access.preflight.configs.PmpParams + +import framework.acc.memdomain.tomem.configs.{Params => ModuleParam} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig -import memcore.memory.preflight.{Error, MapQuery, MapResult, Params} -import memcore.bus.axi4 +import framework.core.memory.access.preflight.{Error, MapQuery, MapResult} +import framework.ip.bus.axi4 @instantiable -class WriteDma(b: GlobalConfig, prepared: Params, axiParams: axi4.Params) extends Module { - require(b.memDomain.bankWidth == 128 && b.memDomain.dma_buswidth == 128 && axiParams.dataBits == 128) +class WriteDma(val b: ModuleParam, permission: PmpParams) extends Module { + val p = b + + val prepared = p.mapping + val axiParams = p.axi + + require(axiParams.dataBits == 128) require(prepared.beatBytes == 16 && prepared.bus.addressBits == axiParams.addressBits) @public val io = IO(new Bundle { - val req = Flipped(Decoupled(new BBWriteCommand)) - val data = Flipped(Decoupled(new BBWriteData(128))) - val resp = Decoupled(new BBWriteResponse) + val req = Flipped(Decoupled(new WriteTransfer(b, permission))) + val data = + Flipped(Decoupled(new WriteData(b))) + val resp = Decoupled(new WriteResult) val parentId = Input(UInt(prepared.idBits.W)) val decisionValid = Input(Bool()) val decisionFault = Input(new DmaStatus) val busy = Output(Bool()) - val query = Output(new MapQuery(prepared)) - val mapping = Input(new MapResult(prepared)) - val axi = new axi4.Port(axiParams) + + val query = Output(new MapQuery(b.mapping)) + + val mapping = Input(new MapResult(b.mapping)) + + val axi = new axi4.Port(axiParams) + }) val idle :: plan :: query :: stream :: waitB :: drain :: respond :: Nil = Enum(7) val state = RegInit(idle) - val command = Reg(new BBWriteCommand) + val command = Reg(new WriteTransfer(b, permission)) val parent = Reg(UInt(prepared.idBits.W)) val endVA = Reg(UInt(64.W)) val cursorVA = Reg(UInt(64.W)) @@ -64,7 +77,7 @@ class WriteDma(b: GlobalConfig, prepared: Params, axiParams: axi4.Params) extend io.resp.valid := state === respond io.resp.bits.done := fault.error === DmaError.None.U io.resp.bits.fault := fault - io.query := 0.U.asTypeOf(new MapQuery(prepared)) + io.query := 0.U.asTypeOf(new MapQuery(b.mapping)) io.query.valid := state === query io.query.id := parent io.query.va := queryVA diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/build.sc b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/build.sc new file mode 100644 index 000000000..18880442c --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/build.sc @@ -0,0 +1,20 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.^.^.^.settings +val ipRoot = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.acc.memdomain.tol1.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/ToL1.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/configs/uvm.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/configs/uvm.toml new file mode 100644 index 000000000..9896ccd3d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/configs/uvm.toml @@ -0,0 +1,8 @@ +[parameters] +entries = 4 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[defines] diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/filelist.f b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/filelist.f new file mode 100644 index 000000000..4d52f5094 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@VERIFY@/uvm/src/protocol/interlock ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/interlock/types.svh +@VERIFY@/uvm/src/protocol/interlock/interface.sv +@VERIFY@/uvm/src/protocol/interlock/package.sv +-F @RTL@/ToL1/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/suite.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/suite.toml new file mode 100644 index 000000000..cdf810147 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/suite.toml @@ -0,0 +1,343 @@ +[model] +manifest = "../../../../../../../../../../../../../../../verify/uvm/src/protocol/interlock/model/Cargo.toml" +library = "libinterlock_reference" + +[[tests]] +functional_groups = ["interlock_pkg::ownership_model::activity", "interlock_pkg::ownership_model::maintenance_kind", "interlock_pkg::ownership_model::probes"] +name = "tol1" +top = "tol1_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "tol1_bad_cancel_info" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cancel_info"] +assertion = "Interlock cancel conflicts with the same command" + +[[tests]] +functional_groups = [] +name = "tol1_bad_cancel_sealed" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cancel_sealed"] +assertion = "Interlock cancel requires an unsealed reservation" + +[[tests]] +functional_groups = [] +name = "tol1_bad_cancel_unknown" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cancel_unknown"] +assertion = "Interlock cancel ID has no reservation" + +[[tests]] +functional_groups = [] +name = "tol1_bad_capacity" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=capacity"] +assertion = "Interlock range list must seal within capacity" + +[[tests]] +functional_groups = [] +name = "tol1_bad_cpu_alignment" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cpu_alignment"] +assertion = "Interlock CPU access must be naturally aligned" + +[[tests]] +functional_groups = [] +name = "tol1_bad_cpu_size" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cpu_size"] +assertion = "Interlock CPU size must be 1, 2, 4 or 8 bytes" + +[[tests]] +functional_groups = [] +name = "tol1_bad_direction" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=direction"] +assertion = "Interlock ranges must have one access direction" + +[[tests]] +functional_groups = [] +name = "tol1_bad_dma_failed" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=dma_failed"] +assertion = "Interlock DMA failed" + +[[tests]] +functional_groups = [] +name = "tol1_bad_dma_ungranted" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=dma_ungranted"] +assertion = "Interlock DMA done requires a granted access" + +[[tests]] +functional_groups = [] +name = "tol1_bad_dma_unknown" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=dma_unknown"] +assertion = "Interlock DMA done tag has no reservation" + +[[tests]] +functional_groups = [] +name = "tol1_bad_duplicate" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=duplicate"] +assertion = "Interlock dispatch ID is already live" + +[[tests]] +functional_groups = [] +name = "tol1_bad_empty_range" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=empty_range"] +assertion = "Interlock memory range is empty" + +[[tests]] +functional_groups = [] +name = "tol1_bad_info_duplicate" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=info_duplicate"] +assertion = "Interlock access info is already declared" + +[[tests]] +functional_groups = [] +name = "tol1_bad_info_unknown" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=info_unknown"] +assertion = "Interlock access info ID has no reservation" + +[[tests]] +functional_groups = [] +name = "tol1_bad_maint_failed" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=maint_failed"] +assertion = "Interlock maintenance failed" + +[[tests]] +functional_groups = [] +name = "tol1_bad_maint_tag" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=maint_tag"] +assertion = "Interlock maintenance response tag mismatch" + +[[tests]] +functional_groups = [] +name = "tol1_bad_maint_unknown" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=maint_unknown"] +assertion = "Interlock maintenance response has no request" + +[[tests]] +functional_groups = [] +name = "tol1_bad_no_memory_seal" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=no_memory_seal"] +assertion = "Interlock no-memory declaration must seal an empty command" + +[[tests]] +functional_groups = [] +name = "tol1_bad_overflow_range" +top = "tol1_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=overflow_range"] +assertion = "Interlock memory range exceeds physical width" diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/tb.sv b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/tb.sv new file mode 100644 index 000000000..261fc3269 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/resources/tb.sv @@ -0,0 +1,16 @@ +module tol1_tb; + import uvm_pkg::*; + import interlock_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + interlock_if bus (clock); + ToL1 dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual interlock_if)::set(null, "uvm_test_top.*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/scala/Emit.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/scala/Emit.scala new file mode 100644 index 000000000..8fc571848 --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/scala/Emit.scala @@ -0,0 +1,72 @@ +package framework.acc.memdomain.tol1 + +import framework.ip.memory.interlock._ + +import chisel3._ +import java.nio.file.{Files, Paths} + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + + val p = Params( + values("entries").num.toInt, + values("addressBits").num.toInt, + values("idBits").num.toInt, + values("lineBytes").num.toInt, + values("maxRanges").num.toInt + ) + + Files.createDirectories(Paths.get(output)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new ToL1(p), + args = Array("--target-dir", s"$output/ToL1", "--split-verilog") + ) + + val definitions = scala.collection.mutable.ArrayBuffer[String]( + s"`define INTERLOCK_ENTRIES ${p.entries}", + s"`define INTERLOCK_ADDRESS_BITS ${p.addressBits}", + s"`define INTERLOCK_ID_BITS ${p.idBits}", + s"`define INTERLOCK_LINE_BYTES ${p.lineBytes}", + s"`define INTERLOCK_MAX_RANGES ${p.maxRanges}" + ) + + val ports = scala.collection.mutable.ArrayBuffer[String]() + + def channel( + kind: String, + prefix: String, + signal: String, + data: Bundle, + decoupled: Boolean = true + ): Unit = { + var offset = 0 + for ((name, field) <- data.elements.toSeq) { + val width = field.getWidth + definitions += s"`define INTERLOCK_${kind}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define INTERLOCK_${kind}_${name.toUpperCase}_WIDTH $width" + ports += s".$prefix$name(bus.signals.${signal}_bits[$offset +: $width])" + offset += width + } + definitions += s"`define INTERLOCK_${kind}_WIDTH $offset" + if (decoupled) { + val channelPrefix = prefix.stripSuffix("bits_") + ports += s".${channelPrefix}valid(bus.signals.${signal}_valid)" + ports += s".${channelPrefix}ready(bus.signals.${signal}_ready)" + } + } + + channel("DISPATCH", "io_dispatch_bits_", "dispatch", new Dispatch(p)) + channel("CANCEL", "io_cancel_bits_", "cancel", new Dispatch(p)) + channel("ACCESS_INFO", "io_accessInfo_bits_", "access_info", new AccessInfo(p)) + channel("MAINTENANCE", "io_maintenance_bits_", "maintenance", new Maintenance(p)) + channel("MAINTAINED", "io_maintained_bits_", "maintained", new Acknowledgement(p)) + channel("GRANT", "io_grant_bits_", "grant", new Tag(p)) + channel("DONE", "io_done_bits_", "done", new Acknowledgement(p)) + channel("COMPLETE", "io_complete_bits_", "complete", new Tag(p)) + channel("CPU_QUERY", "io_cpuQuery_", "cpu_query", new CpuQuery(p), decoupled = false) + ports += ".io_cpuAllow(bus.signals.cpu_allow)" + ports += ".io_cpuProbeAllow(bus.signals.cpu_probe_allow)" + Files.write(Paths.get(output, "parameters.svh"), (definitions.mkString("\n") + "\n").getBytes) + Files.write(Paths.get(output, "ports.svh"), ports.mkString(",\n").getBytes) +} diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/scala/ToL1.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/scala/ToL1.scala new file mode 100644 index 000000000..436d4687d --- /dev/null +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1/src/main/scala/ToL1.scala @@ -0,0 +1,52 @@ +package framework.acc.memdomain.tol1 + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.ip.memory.interlock.{ + AccessInfo, + Acknowledgement, + CpuQuery, + Dispatch, + Interlock, + Maintenance, + Params, + Tag +} + +class Port(val p: Params) extends Bundle { + + val dispatch = Flipped(Decoupled(new Dispatch(p))) + val cancel = Flipped(Decoupled(new Dispatch(p))) + val accessInfo = Flipped(Decoupled(new AccessInfo(p))) + val cpuQuery = Input(new CpuQuery(p)) + val cpuAllow = Output(Bool()) + val cpuProbeAllow = Output(Bool()) + val maintenance = Decoupled(new Maintenance(p)) + val maintained = + Flipped(Decoupled(new Acknowledgement(p))) + val grant = Decoupled(new Tag(p)) + val done = Flipped(Decoupled(new Acknowledgement(p))) + val complete = Decoupled(new Tag(p)) +} + +@instantiable +class ToL1(val p: Params) extends Module { + + @public + val io = IO(new Port(p)) + + val interlock: Instance[Interlock] = Instantiate(new Interlock(p)) + + interlock.io.dispatch <> io.dispatch + interlock.io.cancel <> io.cancel + interlock.io.accessInfo <> io.accessInfo + interlock.io.cpuQuery := io.cpuQuery + io.cpuAllow := interlock.io.cpuAllow + io.cpuProbeAllow := interlock.io.cpuProbeAllow + io.maintenance <> interlock.io.maintenance + interlock.io.maintained <> io.maintained + io.grant <> interlock.io.grant + interlock.io.done <> io.done + io.complete <> interlock.io.complete +} diff --git a/arch/src/main/scala/framework/balldomain/BallDomain.scala b/arch/src/main/scala/framework/balldomain/BallDomain.scala deleted file mode 100644 index 88f936054..000000000 --- a/arch/src/main/scala/framework/balldomain/BallDomain.scala +++ /dev/null @@ -1,125 +0,0 @@ -package framework.balldomain - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.balldomain.bbus.BBus -import framework.balldomain.decoder.BallDomainDecoder -import framework.balldomain.rs.BallReservationStation -import framework.balldomain.blink.{BankRead, BankWrite, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.frontend.globalrs.{GlobalSchedComplete, GlobalSchedIssue} -import framework.rvv.RvvBallCommand -import framework.frontend.decoder.{DomainId, PostGDCmd} - -/** - * Ball domain. - * - * BBus reflectively constructs ball instances from `b.ballDomain.ballIdMappings`; - * `BallDomainDecoder` reads `b.ballDomain.ballISA` to route funct7 → bid. - * Both are pure data-driven, so no example needs its own subclass. - */ -@instantiable -class BallDomain(val b: GlobalConfig) extends Module { - val totalBallRead = b.ballDomain.ballIdMappings.map(_.inBW).sum - val totalBallWrite = b.ballDomain.ballIdMappings.map(_.outBW).sum - val totalMmioRead = b.ballDomain.ballIdMappings.map(_.mmioReadBW).sum - val totalMmioWrite = b.ballDomain.ballIdMappings.map(_.mmioWriteBW).sum - - @public - val global_issue_i = IO(Flipped(Decoupled(new GlobalSchedIssue(b)))) - @public - val global_complete_o = IO(Decoupled(new GlobalSchedComplete(b))) - @public - val ballChannelActive = IO(Output(Vec(b.ballDomain.ballNum, Bool()))) - @public - val ballChannelReady = IO(Input(Vec(b.ballDomain.ballNum, Bool()))) - @public - val bankRead = IO(Vec(totalBallRead, Flipped(new BankRead(b)))) - @public - val bankWrite = IO(Vec(totalBallWrite, Flipped(new BankWrite(b)))) - @public - val mmioRead = IO(Vec(totalMmioRead, Flipped(new MmioRead(b)))) - @public - val mmioWrite = IO(Vec(totalMmioWrite, Flipped(new MmioWrite(b)))) - @public - val subRobReq = IO(Vec(b.ballDomain.ballNum, Decoupled(new SubRobRow(b)))) - @public val kernelBanks = if (b.rvv.enable) Some(IO(Input(new framework.rvv.BankLayout(b)))) else None - @public val internalCommand = if (b.rvv.enable) Some(IO(Flipped(Decoupled(new RvvBallCommand)))) else None - @public val internalComplete = if (b.rvv.enable) Some(IO(Decoupled(UInt(64.W)))) else None - @public val ownerRobId = if (b.rvv.enable) Some(IO(Input(UInt(log2Up(b.frontend.rob_entries).W)))) else None - - val bbus: Instance[BBus] = Instantiate(new BBus(b)) - val ballDecoder: Instance[BallDomainDecoder] = Instantiate(new BallDomainDecoder(b)) - val ballRs: Instance[BallReservationStation] = Instantiate(new BallReservationStation(b)) - - if (b.rvv.enable) { - bbus.kernelBanks.get := kernelBanks.get - val decoder = Instantiate(new BallDomainDecoder(b)) - decoder.cmd_i.valid := internalCommand.get.valid - decoder.cmd_i.bits := 0.U.asTypeOf(new PostGDCmd(b)) - decoder.cmd_i.bits.domain_id := DomainId.BALL - decoder.cmd_i.bits.op1_col := 1.U - decoder.cmd_i.bits.op2_col := 1.U - decoder.cmd_i.bits.wr_col := 1.U - decoder.cmd_i.bits.cmd.funct := internalCommand.get.bits.funct7 - decoder.cmd_i.bits.cmd.rs1Data := internalCommand.get.bits.rs1 - decoder.cmd_i.bits.cmd.rs2Data := internalCommand.get.bits.rs2 - internalCommand.get.ready := decoder.cmd_i.ready - bbus.internalCommand.get.valid := decoder.ball_decode_cmd_o.valid - bbus.internalCommand.get.bits.cmd := decoder.ball_decode_cmd_o.bits - bbus.internalCommand.get.bits.rob_id := ownerRobId.get - bbus.internalCommand.get.bits.is_sub := false.B - bbus.internalCommand.get.bits.sub_rob_id := 0.U - decoder.ball_decode_cmd_o.ready := bbus.internalCommand.get.ready - internalComplete.get <> bbus.internalComplete.get - } - -//===-------------------------------------------------------------------===// -// Global RS -> Decoder -//===-------------------------------------------------------------------===// - ballDecoder.cmd_i.valid := global_issue_i.valid - ballDecoder.cmd_i.bits := global_issue_i.bits.cmd - global_issue_i.ready := ballDecoder.cmd_i.ready - -//===-------------------------------------------------------------------===// -// Decoder -> Local BallRS -//===-------------------------------------------------------------------===// - ballRs.ball_decode_cmd_i.valid := ballDecoder.ball_decode_cmd_o.valid - ballRs.ball_decode_cmd_i.bits.cmd := ballDecoder.ball_decode_cmd_o.bits - ballRs.ball_decode_cmd_i.bits.rob_id := global_issue_i.bits.rob_id - ballRs.ball_decode_cmd_i.bits.is_sub := global_issue_i.bits.is_sub - ballRs.ball_decode_cmd_i.bits.sub_rob_id := global_issue_i.bits.sub_rob_id - ballDecoder.ball_decode_cmd_o.ready := ballRs.ball_decode_cmd_i.ready - -//===-------------------------------------------------------------------===// -// Local BallRS -> BBus -//===-------------------------------------------------------------------===// - bbus.cmdReq <> ballRs.issue_o.balls - ballRs.commit_i.balls <> bbus.cmdResp - -//===-------------------------------------------------------------------===// -// BBus -> Mem Domain -//===-------------------------------------------------------------------===// - bbus.bankRead <> bankRead - bbus.bankWrite <> bankWrite - ballChannelActive := bbus.ballChannelActive - bbus.ballChannelReady := ballChannelReady - bbus.mmioRead <> mmioRead - bbus.mmioWrite <> mmioWrite - - for (i <- 0 until b.ballDomain.ballNum) { - subRobReq(i) <> bbus.subRobReq(i) - } - -//===-------------------------------------------------------------------===// -// Local RS completion -> Global RS -//===-------------------------------------------------------------------===// - global_complete_o.valid := ballRs.complete_o.valid - global_complete_o.bits.rob_id := ballRs.complete_o.bits.rob_id - global_complete_o.bits.is_sub := ballRs.complete_o.bits.is_sub - global_complete_o.bits.sub_rob_id := ballRs.complete_o.bits.sub_rob_id - global_complete_o.bits.fault := 0.U.asTypeOf(global_complete_o.bits.fault) - ballRs.complete_o.ready := global_complete_o.ready -} diff --git a/arch/src/main/scala/framework/balldomain/bbus/bbus.scala b/arch/src/main/scala/framework/balldomain/bbus/bbus.scala deleted file mode 100644 index 15a5106f1..000000000 --- a/arch/src/main/scala/framework/balldomain/bbus/bbus.scala +++ /dev/null @@ -1,255 +0,0 @@ -package framework.balldomain.bbus - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.balldomain.blink.HasBlink -import framework.balldomain.bbus.pmc.BallCyclePMC -import framework.balldomain.bbus.cmdrouter.CmdRouter -import framework.balldomain.isa.BallISA -import framework.balldomain.blink.{BankRead, BankWrite, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import java.lang.reflect.InvocationTargetException - -/** - * BBus - Ball bus, manages connections and arbitration of multiple Ball devices. - * - * Ball generators are produced reflectively from `b.ballDomain.ballIdMappings`: - * each mapping carries a `ballClass` FQCN whose constructor `(GlobalConfig)` is - * invoked. Framework does not maintain or interpret the list of balls; the - * config layer is the single source of truth. - */ -@instantiable -class BBus(val b: GlobalConfig) extends Module { - val numBalls = b.ballDomain.ballNum - val totalBallRead = b.ballDomain.ballIdMappings.map(_.inBW).sum - val totalBallWrite = b.ballDomain.ballIdMappings.map(_.outBW).sum - val totalMmioRead = b.ballDomain.ballIdMappings.map(_.mmioReadBW).sum - val totalMmioWrite = b.ballDomain.ballIdMappings.map(_.mmioWriteBW).sum - - // Rs - bbus - balls - @public - val cmdReq = IO(Vec(numBalls, Flipped(Decoupled(new BallRsIssue(b))))) - @public - val cmdResp = IO(Vec(numBalls, Decoupled(new BallRsComplete(b)))) - @public - val ballChannelActive = IO(Output(Vec(numBalls, Bool()))) - @public - val ballChannelReady = IO(Input(Vec(numBalls, Bool()))) - // balls - bbus - @public - val bankRead = IO(Vec(totalBallRead, Flipped(new BankRead(b)))) - @public - val bankWrite = IO(Vec(totalBallWrite, Flipped(new BankWrite(b)))) - @public - val mmioRead = IO(Vec(totalMmioRead, Flipped(new MmioRead(b)))) - @public - val mmioWrite = IO(Vec(totalMmioWrite, Flipped(new MmioWrite(b)))) - // balls - bbus - SubROB - @public - val subRobReq = IO(Vec(numBalls, Decoupled(new SubRobRow(b)))) - @public val kernelBanks = if (b.rvv.enable) Some(IO(Input(new framework.rvv.BankLayout(b)))) else None - @public val internalCommand = if (b.rvv.enable) Some(IO(Flipped(Decoupled(new BallRsIssue(b))))) else None - @public val internalComplete = if (b.rvv.enable) Some(IO(Decoupled(UInt(64.W)))) else None - - require(b.ballDomain.ballIdMappings.length == numBalls, "ballNum must match ballIdMappings length") - - // Apply BALL_INIT on the cycle after the command handshake. Driving reset - // directly from cmd.fire would feed the Ball's reset-dependent ready signal - // back into that same handshake. - val ballBootReset = RegInit(VecInit(Seq.fill(numBalls)(false.B))) - - val balls = b.ballDomain.ballIdMappings.zipWithIndex.map { case (mapping, index) => - withReset(reset.asBool || ballBootReset(index)) { - Module { - try { - val cls = Class.forName(mapping.ballClass) - val ctor = cls.getConstructor(classOf[GlobalConfig]) - ctor.newInstance(b).asInstanceOf[HasBlink with Module] - } catch { - case e: InvocationTargetException => - val cause = Option(e.getCause).getOrElse(e) - throw new RuntimeException( - s"Failed to instantiate ball ${mapping.ballName} (${mapping.ballClass}): ${cause.getMessage}", - cause - ) - case e: Throwable => - throw new RuntimeException( - s"Failed to instantiate ball ${mapping.ballName} (${mapping.ballClass}): ${e.getMessage}", - e - ) - } - } - } - } - - val cmdRouter: Instance[CmdRouter] = Instantiate(new CmdRouter(b)) - val pmc: Instance[BallCyclePMC] = Instantiate(new BallCyclePMC(b)) - -// ----------------------------------------------------------------------------- -// cmd router -// ----------------------------------------------------------------------------- - - val idle_ball = VecInit(balls.map(_.blink.cmdReq.ready)) - - cmdRouter.io.cmdReq_i <> cmdReq - cmdRouter.io.ballIdle := idle_ball - - val command = Wire(Decoupled(new BallRsIssue(b))) - val internal = WireDefault(false.B) - val internalOwner = RegInit(VecInit(Seq.fill(numBalls)(false.B))) - if (b.rvv.enable) { - val arbiter = Module(new RRArbiter(new BallRsIssue(b), 2)) - arbiter.io.in(0) <> cmdRouter.io.cmdReq_o - arbiter.io.in(1) <> internalCommand.get - command <> arbiter.io.out - internal := arbiter.io.chosen === 1.U - internalComplete.get.valid := VecInit( - (0 until numBalls).map(i => internalOwner(i) && balls(i).blink.cmdResp.valid) - ).asUInt.orR - internalComplete.get.bits := 0.U - assert(PopCount(internalOwner) <= 1.U, "RVV issues only one outstanding Ball operation") - } else { - command <> cmdRouter.io.cmdReq_o - } - - val isBallInit = command.bits.cmd.funct7 === BallISA.InitFunct.U - val initPending = RegInit(VecInit(Seq.fill(numBalls)(false.B))) - val initResp = Reg(Vec(numBalls, new BallRsComplete(b))) - val targetMatches = VecInit(b.ballDomain.ballIdMappings.map(m => command.bits.cmd.bid === m.ballId.U)) - - for (i <- 0 until numBalls) { - val mapping = b.ballDomain.ballIdMappings(i) - ballChannelActive(i) := (if (mapping.inBW == 0 && mapping.outBW == 0) false.B else balls(i).blink.status.running) - balls(i).blink.channelReady := ballChannelReady(i) - - val targetMatch = targetMatches(i) - balls(i).blink.cmdReq.valid := command.valid && !isBallInit && !initPending(i) && targetMatch - balls(i).blink.cmdReq.bits := command.bits - - cmdRouter.io.cmdResp_i(i).valid := !internalOwner(i) && Mux(initPending(i), true.B, balls(i).blink.cmdResp.valid) - cmdRouter.io.cmdResp_i(i).bits := Mux(initPending(i), initResp(i), balls(i).blink.cmdResp.bits) - balls(i).blink.cmdResp.ready := !initPending(i) && Mux( - internalOwner(i), - (if (b.rvv.enable) internalComplete.get.ready else false.B), - cmdRouter.io.cmdResp_i(i).ready - ) - when(command.fire && targetMatch)(internalOwner(i) := internal) - when(balls(i).blink.cmdResp.fire || initPending(i) && cmdRouter.io.cmdResp_i(i).fire) { - internalOwner(i) := false.B - } - if (b.rvv.enable) { - when(internalOwner(i) && initPending(i)) { - internalComplete.get.valid := true.B - when(internalComplete.get.fire) { initPending(i) := false.B; internalOwner(i) := false.B } - } - } - - ballBootReset(i) := command.fire && isBallInit && targetMatch - when(command.fire && isBallInit && targetMatch) { - initPending(i) := true.B - initResp(i).rob_id := command.bits.rob_id - initResp(i).is_sub := command.bits.is_sub - initResp(i).sub_rob_id := command.bits.sub_rob_id - } - when(initPending(i) && cmdRouter.io.cmdResp_i(i).fire) { - initPending(i) := false.B - } - } - - val targetReady = - VecInit((0 until numBalls).map(i => balls(i).blink.cmdReq.ready && !initPending(i) && targetMatches(i))).asUInt.orR - command.ready := targetReady - - cmdResp <> cmdRouter.io.cmdResp_o - -// ----------------------------------------------------------------------------- -// PMC - Performance Monitor Counter -// ----------------------------------------------------------------------------- - for (i <- 0 until numBalls) { - pmc.io.cmdReq_i(i).valid := cmdRouter.io.cmdReq_i(i).fire || command.fire && internal && targetMatches(i) - pmc.io.cmdReq_i(i).bits := Mux(internal, command.bits, cmdRouter.io.cmdReq_i(i).bits) - pmc.io.cmdResp_o(i).valid := cmdRouter.io.cmdResp_o(i).valid || - internalOwner(i) && (balls(i).blink.cmdResp.fire || initPending(i) && (if (b.rvv.enable) internalComplete.get.fire - else false.B)) - pmc.io.cmdResp_o(i).bits := Mux( - internalOwner(i), - Mux(initPending(i), initResp(i), balls(i).blink.cmdResp.bits), - cmdRouter.io.cmdResp_o(i).bits - ) - } - -// Connect balls' bankRead and bankWrite to memrouter - var readChannelIdx = 0 - var writeChannelIdx = 0 - - for (((mapping, ball), index) <- b.ballDomain.ballIdMappings.zip(balls).zipWithIndex) { - val inBW = mapping.inBW - val outBW = mapping.outBW - - for (i <- 0 until inBW) { - bankRead(readChannelIdx) <> ball.blink.bankRead(i) - if (b.rvv.enable) { - val local = ball.blink.bankRead(i).bank_id - val layout = kernelBanks.get - when(internalOwner(index)) { - bankRead(readChannelIdx).bank_id := Mux(local < layout.readGroups, layout.readBank, layout.writeBank) - bankRead(readChannelIdx).group_id := Mux(local < layout.readGroups, local, local - layout.readGroups) - when(ball.blink.bankRead(i).io.req.valid) { - assert(local < (layout.readGroups +& layout.writeGroups), "RVV Ball read bank outside launch groups") - } - } - } - readChannelIdx = readChannelIdx + 1 - } - - for (i <- 0 until outBW) { - bankWrite(writeChannelIdx) <> ball.blink.bankWrite(i) - if (b.rvv.enable) { - val local = ball.blink.bankWrite(i).bank_id - val layout = kernelBanks.get - when(internalOwner(index)) { - bankWrite(writeChannelIdx).bank_id := layout.writeBank - bankWrite(writeChannelIdx).group_id := local - layout.readGroups - when(ball.blink.bankWrite(i).io.req.valid) { - assert( - local >= layout.readGroups && local < (layout.readGroups +& layout.writeGroups), - "RVV Ball write outside write groups" - ) - } - } - } - writeChannelIdx = writeChannelIdx + 1 - } - } - - var mmioWriteChannelIdx = 0 - for ((mapping, ball) <- b.ballDomain.ballIdMappings.zip(balls)) { - val mmioWriteBW = mapping.mmioWriteBW - for (i <- 0 until mmioWriteBW) { - mmioWrite(mmioWriteChannelIdx) <> ball.blink.mmioWrite(i) - mmioWriteChannelIdx = mmioWriteChannelIdx + 1 - } - } - - // Internal RVV operations complete directly; they cannot expand into SubROB. - for (i <- 0 until numBalls) { - subRobReq(i).valid := balls(i).blink.subRobReq.valid && !internalOwner(i) - subRobReq(i).bits := balls(i).blink.subRobReq.bits - balls(i).blink.subRobReq.ready := subRobReq(i).ready && !internalOwner(i) - assert(!(internalOwner(i) && balls(i).blink.subRobReq.valid), "RVV Ball operation cannot issue SubROB work") - } - - // Connect configurable MMIO metadata channels. - var mmioReadChannelIdx = 0 - for ((mapping, ball) <- b.ballDomain.ballIdMappings.zip(balls)) { - val mmioReadBW = mapping.mmioReadBW - for (i <- 0 until mmioReadBW) { - mmioRead(mmioReadChannelIdx) <> ball.blink.mmioRead(i) - mmioReadChannelIdx = mmioReadChannelIdx + 1 - } - } - -} diff --git a/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/CmdRouter.scala b/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/CmdRouter.scala deleted file mode 100644 index 97d804ea7..000000000 --- a/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/CmdRouter.scala +++ /dev/null @@ -1,60 +0,0 @@ -//===- CmdRouter.scala ------ Command Router of Ball Domain ---------------===// -// -// Copyright 2026 The Buckyball Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// -// -// Request channel (req): N inputs (cmdReq_i) -> 1 output (cmdReq_o) -// -// Response channel (resp): N inputs (cmdResp_i) -> N outputs (cmdResp_o), with -// direct point-to-point connections -// -//===----------------------------------------------------------------------===// -package framework.balldomain.bbus.cmdrouter - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import chisel3.experimental.hierarchy.{instantiable, public} - -@instantiable -class CmdRouter(val b: GlobalConfig) extends Module { - val numBalls = b.ballDomain.ballNum - - @public - val io = IO(new Bundle { - val cmdReq_i = Vec(numBalls, Flipped(Decoupled(new BallRsIssue(b)))) - val cmdResp_i = Vec(numBalls, Flipped(Decoupled(new BallRsComplete(b)))) - val cmdReq_o = Decoupled(new BallRsIssue(b)) - val cmdResp_o = Vec(numBalls, Decoupled(new BallRsComplete(b))) - val ballIdle = Input(Vec(numBalls, Bool())) - }) - - val arbiter = Module(new RRArbiter(new BallRsIssue(b), numBalls)) - val ballIdleR = RegNext(io.ballIdle, VecInit(Seq.fill(numBalls)(false.B))) - - for (i <- 0 until numBalls) { - arbiter.io.in(i).valid := io.cmdReq_i(i).valid && ballIdleR(i) - arbiter.io.in(i).bits := io.cmdReq_i(i).bits - io.cmdReq_i(i).ready := arbiter.io.in(i).ready && ballIdleR(i) - } - - io.cmdReq_o <> arbiter.io.out - - for (i <- 0 until numBalls) { - io.cmdResp_o(i) <> io.cmdResp_i(i) - } -} diff --git a/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/cmdrouter.spec.en.md b/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/cmdrouter.spec.en.md deleted file mode 100644 index 537b47c36..000000000 --- a/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/cmdrouter.spec.en.md +++ /dev/null @@ -1,33 +0,0 @@ -# CmdRouter - -## TL;DR -- 【Module Function】Arbitrates `numBalls` request lanes from `cmdReq_i` into one `cmdReq_o`, and forwards each `cmdResp_i` lane to the matching `cmdResp_o` lane. -- 【Module Placement】This module is `CmdRouter` on the `bbus` in `balldomain`, responsible for command routing. -- 【Module Inputs/Outputs】 - - Command dispatch: `cmdReq_i` from frontend to bbus; after round-robin arbitration, single `cmdReq_o` to one ball. - - Command completion response: `cmdResp_i` from each ball; forwarded lane-by-lane through `cmdResp_o` to frontend. - - BBus drives `ballIdle` to report whether each ball is idle. -- 【Key Points】 - - Request arbitration uses registered `ballIdleR` only, so `ballIdle` affects admission with a one-cycle delay. - -## Interface -| Direction | Signal | Type | Meaning | -| --- | --- | --- | --- | -| Input | `cmdReq_i` | `Vec(numBalls, Flipped(Decoupled(BallRsIssue)))` | Per-Ball request ingress channels using `valid/ready` handshake. | -| Input | `cmdResp_i` | `Vec(numBalls, Flipped(Decoupled(BallRsComplete)))` | Per-Ball response ingress channels, forwarded lane by lane. | -| Input | `ballIdle` | `Vec(numBalls, Bool)` | Per-Ball idle indicators, registered before request gating. | -| Output | `cmdReq_o` | `Decoupled(BallRsIssue)` | Single arbitrated request egress channel. | -| Output | `cmdResp_o` | `Vec(numBalls, Decoupled(BallRsComplete))` | Response egress channels directly mapped to `cmdResp_i` by index. | - -## Core Behavior -The module instantiates `RRArbiter(BallRsIssue, numBalls)` for round-robin request selection and computes `ballIdleR = RegNext(io.ballIdle, false)` as the lane-enable condition. For any index `i`, a request is presented to arbitration only when both `cmdReq_i(i).valid` and `ballIdleR(i)` are true. The same `ballIdleR(i)` condition gates `cmdReq_i(i).ready`, so upstream always sees `ready = false` when that lane is not idle in the registered view. The arbiter output is directly connected to `cmdReq_o` with `<>`, so `valid/bits/ready` semantics follow the standard `RRArbiter` plus Decoupled connection behavior. The response path performs no selection or transformation: each lane uses `cmdResp_o(i) <> cmdResp_i(i)` and keeps independent handshakes. - -## Verification Checklist -- With all `ballIdle` lanes held at `0` and all `cmdReq_i.valid` held at `1`, `cmdReq_o.valid` must remain `0`; any observed `1` is fail. -- Raise one `ballIdle(i)` from `0` to `1` at cycle `t` while keeping `cmdReq_i(i).valid=1`; that lane may be admitted no earlier than cycle `t+1`; admission at `t` is fail. -- With `ballIdle(i)=0` and `cmdReq_i(i).valid=1`, `cmdReq_i(i).ready` must stay `0`; any cycle with `ready=1` is fail. -- With two or more continuously valid lanes and their `ballIdleR` all `1`, `cmdReq_o.bits` source should rotate under round-robin instead of sticking forever to one lane; persistent starvation is fail. -- Inject a response transaction on any lane `i` while driving `cmdResp_o(i).ready=1`; `cmdResp_o(i).valid` and `bits` must match input in the same cycle; mismatch is fail. -- Drive different response streams on lanes `i` and `j (i!=j)` concurrently; outputs must remain lane-isolated with no cross-lane mixing; any mixing is fail. -- When downstream forces `cmdReq_o.ready=0`, request handshakes must stop and no fake completion can occur; any observed `fire` is fail. -- Toggle `ballIdle` across adjacent cycles and verify gating always follows one-cycle-late `RegNext` behavior; same-cycle pass-through is fail. diff --git a/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/cmdrouter.spec.zh.md b/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/cmdrouter.spec.zh.md deleted file mode 100644 index 7431fdbe9..000000000 --- a/arch/src/main/scala/framework/balldomain/bbus/cmdrouter/cmdrouter.spec.zh.md +++ /dev/null @@ -1,33 +0,0 @@ -# CmdRouter - -## TL;DR -- 【模块功能】将 `numBalls` 路 `cmdReq_i` 按轮询仲裁汇聚到单一路 `cmdReq_o`，并把每路 `cmdResp_i` 一一透传到对应的 `cmdResp_o`。 -- 【模块定位】该模块是 `balldomain`的`bbus`上的 `CmdRouter` 负责指令路由`。 -- 【模块输入/输出】 - - 指令分发：从前端到bbus输入 `cmdReq_i`，选择后输出单路 `cmdReq_o` 给某个ball。 - - 指令（完成后）响应：从每个ball输入`cmdResp_o` 的响应信号，选择后输出单路给前端。 - - bbus通过 `ballIdle` 获取每个ball是否空闲 -- 【关键点】 - - 请求仲裁只看寄存后的 `ballIdleR` - -## Interface -| 方向 | 信号名 | 类型 | 含义 | -| --- | --- | --- | --- | -| 输入 | `cmdReq_i` | `Vec(numBalls, Flipped(Decoupled(BallRsIssue)))` | 来自各 Ball 的请求输入通道，按 `valid/ready` 握手。 | -| 输入 | `cmdResp_i` | `Vec(numBalls, Flipped(Decoupled(BallRsComplete)))` | 来自各 Ball 的响应输入通道，逐路透传到输出。 | -| 输入 | `ballIdle` | `Vec(numBalls, Bool)` | 各 Ball 的空闲状态输入，先经 `RegNext` 后参与请求门控。 | -| 输出 | `cmdReq_o` | `Decoupled(BallRsIssue)` | 仲裁后的单一路请求输出通道。 | -| 输出 | `cmdResp_o` | `Vec(numBalls, Decoupled(BallRsComplete))` | 与 `cmdResp_i` 同索引直连的响应输出通道。 | - -## Core Behavior -模块内部实例化 `RRArbiter(BallRsIssue, numBalls)` 对请求进行轮询仲裁，并使用 `ballIdleR = RegNext(io.ballIdle, false)` 作为每路请求的使能条件。对任意索引 `i`，只有当 `cmdReq_i(i).valid` 与 `ballIdleR(i)` 同时为真时，该路才会向仲裁器宣告有效请求；同一条件也用于反向门控 `cmdReq_i(i).ready`，因此当 `ballIdleR(i)` 为假时，上游看到的 `ready` 必为假。仲裁器输出通过 `<>` 直接连接到 `cmdReq_o`，其 `valid/bits/ready` 握手语义由 `RRArbiter` 与该直连共同决定。响应路径不做选择与变换，每个索引 `i` 上 `cmdResp_o(i) <> cmdResp_i(i)`，保持逐路独立握手。 - -## Verification Checklist -- 在 `ballIdle` 全为 `0` 且各路 `cmdReq_i.valid` 为 `1` 的场景，期望 `cmdReq_o.valid` 始终为 `0`；若出现 `1` 判 fail。 -- 在第 `t` 周期将某一路 `ballIdle(i)` 从 `0` 拉到 `1`，并保持 `cmdReq_i(i).valid=1`，期望最早在 `t+1` 周期该路才可能被仲裁；若 `t` 周期已被放行判 fail。 -- 在某一路 `ballIdle(i)=0` 且 `cmdReq_i(i).valid=1` 时，检查 `cmdReq_i(i).ready` 必为 `0`；任一周期为 `1` 判 fail。 -- 在两路及以上持续有效且对应 `ballIdleR` 均为 `1` 的场景，观察 `cmdReq_o.bits` 来源应随轮询发生切换，不应永久固定在同一路；若长期饥饿某一路判 fail。 -- 对任意索引 `i` 注入 `cmdResp_i(i)` 事务并驱动 `cmdResp_o(i).ready=1`，期望同拍 `cmdResp_o(i).valid` 与 `bits` 等于输入；不一致判 fail。 -- 对任意索引 `i/j(i!=j)` 同时施加不同响应流量，期望两路 `cmdResp_o` 互不串扰；若出现跨索引数据混入判 fail。 -- 在下游将 `cmdReq_o.ready` 拉低时，期望仲裁输出握手暂停且不会伪造完成；若无 `ready` 仍出现 `fire` 判 fail。 -- 将 `ballIdle` 在相邻周期内反复翻转，验证门控行为始终按 `RegNext` 延后一拍；若出现当拍直通效果判 fail。 diff --git a/arch/src/main/scala/framework/balldomain/bbus/pmc/BallCyclePMC.scala b/arch/src/main/scala/framework/balldomain/bbus/pmc/BallCyclePMC.scala deleted file mode 100644 index 00f04b7dd..000000000 --- a/arch/src/main/scala/framework/balldomain/bbus/pmc/BallCyclePMC.scala +++ /dev/null @@ -1,83 +0,0 @@ -//===- BallCyclePMC.scala ------ Ball Performance Counter -----------------===// -// -// Copyright 2026 The Buckyball Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// -// -// This DPI-C module is used to collect the performance counter data from each -// ball mounted on the bbus. Each ball has its own PMCModule. -// PMCModule just triggers at two points: -// -// 1. When the ball issues a command. -// 2. When the ball completes a command. -// -//===----------------------------------------------------------------------===// - -package framework.balldomain.bbus.pmc - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.balldomain.rs.BallRsIssue -import framework.balldomain.rs.BallRsComplete -import chisel3.experimental.hierarchy.{instantiable, public} - -@instantiable -class BallCyclePMC(val b: GlobalConfig) extends Module { - val numBalls = b.ballDomain.ballNum - - @public - val io = IO(new Bundle { - val cmdReq_i = Input(Vec(numBalls, Valid(new BallRsIssue(b)))) - val cmdResp_o = Input(Vec(numBalls, Valid(new BallRsComplete(b)))) - }) - -// A global cycle counter after reset. - val cycleCounter = RegInit(0.U(64.W)) - val startTime = RegInit(VecInit(Seq.fill(b.frontend.rob_entries)(0.U(64.W)))) - val pmcTraces = Seq.fill(numBalls)(Module(new PMCTraceDPI)) - - cycleCounter := cycleCounter + 1.U - -//===---------- Initialize the PMC traces ----------===// - for (pt <- pmcTraces) { - pt.io.clock := clock - pt.io.reset := reset.asBool - pt.io.ball_id := 0.U - pt.io.rob_id := 0.U - pt.io.elapsed := 0.U - pt.io.enable := false.B - } - -//===---------- Collect the performance counter data ----------===// -// Trigger at the start of the command. - for (i <- 0 until numBalls) { - when(io.cmdReq_i(i).valid) { - startTime(io.cmdReq_i(i).bits.rob_id) := cycleCounter - } - } - -// Trigger at the end of the command. - for (i <- 0 until numBalls) { - when(io.cmdResp_o(i).valid) { - val robId = io.cmdResp_o(i).bits.rob_id - val elapsed = cycleCounter - startTime(robId) - pmcTraces(i).io.ball_id := i.U - pmcTraces(i).io.rob_id := robId - pmcTraces(i).io.elapsed := elapsed - pmcTraces(i).io.enable := true.B - } - } -} diff --git a/arch/src/main/scala/framework/balldomain/blink/SubRobRow.scala b/arch/src/main/scala/framework/balldomain/blink/SubRobRow.scala deleted file mode 100644 index aae0fef5a..000000000 --- a/arch/src/main/scala/framework/balldomain/blink/SubRobRow.scala +++ /dev/null @@ -1,54 +0,0 @@ -package framework.balldomain.blink - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.frontend.decoder.PostGDCmd -import framework.frontend.scoreboard.BankAccessInfo - -class SubRobSlot(val b: GlobalConfig) extends Bundle { - val valid = Bool() - val cmd = new PostGDCmd(b) -} - -class SubRobRow(val b: GlobalConfig) extends Bundle { - val slots = Vec(4, new SubRobSlot(b)) - val ball_id = UInt(log2Up(b.ballDomain.ballNum).W) - val master_rob_id = UInt(log2Up(b.frontend.rob_entries).W) -} - -/** Tie-off for subRobReq when a Ball does not use SubROB. */ -object SubRobRow { - - def tieOff(b: GlobalConfig): SubRobRow = { - val w = Wire(new SubRobRow(b)) - w.ball_id := 0.U - w.master_rob_id := 0.U - val bankIdLen = b.frontend.bank_id_len - for (i <- 0 until 4) { - w.slots(i).valid := false.B - w.slots(i).cmd.domain_id := 0.U - w.slots(i).cmd.ball_bid := 0.U - w.slots(i).cmd.cmd.raw_inst := 0.U - w.slots(i).cmd.cmd.pc := 0.U - w.slots(i).cmd.cmd.funct := 0.U - w.slots(i).cmd.cmd.funct3 := 0.U - w.slots(i).cmd.cmd.rs2 := 0.U - w.slots(i).cmd.cmd.rs1 := 0.U - w.slots(i).cmd.cmd.xd := false.B - w.slots(i).cmd.cmd.xs1 := false.B - w.slots(i).cmd.cmd.xs2 := false.B - w.slots(i).cmd.cmd.rd := 0.U - w.slots(i).cmd.cmd.opcode := 0.U - w.slots(i).cmd.cmd.rs1Data := 0.U - w.slots(i).cmd.cmd.rs2Data := 0.U - w.slots(i).cmd.bankAccess := BankAccessInfo.none(bankIdLen) - w.slots(i).cmd.op1_col := 0.U - w.slots(i).cmd.op2_col := 0.U - w.slots(i).cmd.wr_col := 0.U - w.slots(i).cmd.isBarrier := false.B - } - w - } - -} diff --git a/arch/src/main/scala/framework/balldomain/blink/bank.scala b/arch/src/main/scala/framework/balldomain/blink/bank.scala deleted file mode 100644 index 1c811e9fc..000000000 --- a/arch/src/main/scala/framework/balldomain/blink/bank.scala +++ /dev/null @@ -1,34 +0,0 @@ -package framework.balldomain.blink - -import chisel3._ -import chisel3.util._ -import framework.memdomain.backend.banks.{SramReadIO, SramWriteIO} -import framework.top.GlobalConfig - -trait HasBankId { - val b: GlobalConfig - val bank_id = Input(UInt(b.memDomain.vbankIdWidth.W)) -} - -trait HasRobId { - val b: GlobalConfig - val rob_id = Input(UInt(log2Up(b.frontend.rob_entries).W)) -} - -trait HasBallId { - val b: GlobalConfig - val ball_id = Input(UInt(log2Up(b.ballDomain.ballNum).W)) -} - -trait HasAccGroupId { - val b: GlobalConfig - val group_id = Input(UInt(b.memDomain.groupIdWidth.W)) -} - -class BankRead(val b: GlobalConfig) extends Bundle with HasBankId with HasRobId with HasBallId with HasAccGroupId { - val io = new SramReadIO(b) -} - -class BankWrite(val b: GlobalConfig) extends Bundle with HasBankId with HasRobId with HasBallId with HasAccGroupId { - val io = new SramWriteIO(b) -} diff --git a/arch/src/main/scala/framework/balldomain/blink/baseball.scala b/arch/src/main/scala/framework/balldomain/blink/baseball.scala deleted file mode 100644 index b00fc630f..000000000 --- a/arch/src/main/scala/framework/balldomain/blink/baseball.scala +++ /dev/null @@ -1,14 +0,0 @@ -package framework.balldomain.blink - -import chisel3._ -import chisel3.util._ - -// all balls must have a blink interface -trait HasBlink { - def blink: BlinkIO -} - -// can be customized by user to add additional status signals -trait HasBallStatus { - def status: BallStatus -} diff --git a/arch/src/main/scala/framework/balldomain/blink/blink.scala b/arch/src/main/scala/framework/balldomain/blink/blink.scala deleted file mode 100644 index 4cda0481c..000000000 --- a/arch/src/main/scala/framework/balldomain/blink/blink.scala +++ /dev/null @@ -1,38 +0,0 @@ -package framework.balldomain.blink - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} - -class BlinkPortIO[I <: Data, R <: Data]( - b: GlobalConfig, - inBW: Int, - outBW: Int, - issue: I, - complete: R) - extends Bundle - with HasBallStatus { - val status = new BallStatus() - val channelReady = Input(Bool()) - val cmdReq = Flipped(Decoupled(issue)) - val cmdResp = Decoupled(complete) - val bankRead = Vec(inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(outBW, Flipped(new BankWrite(b))) -} - -class BlinkIO( - b: GlobalConfig, - inBW: Int, - outBW: Int, - mmioReadBW: Int = 0, - mmioWriteBW: Int = 0) - extends BlinkPortIO(b, inBW, outBW, new BallRsIssue(b), new BallRsComplete(b)) { - val subRobReq = Decoupled(new SubRobRow(b)) - - // MMIO metadata channels. Like bankRead/bankWrite, their number is a Ball - // configuration property rather than a framework-wide fixed interface. - val mmioRead = Vec(mmioReadBW, Flipped(new MmioRead(b))) - val mmioWrite = Vec(mmioWriteBW, Flipped(new MmioWrite(b))) -} diff --git a/arch/src/main/scala/framework/balldomain/blink/mmio/MmioRead.scala b/arch/src/main/scala/framework/balldomain/blink/mmio/MmioRead.scala deleted file mode 100644 index 7b3ed17ff..000000000 --- a/arch/src/main/scala/framework/balldomain/blink/mmio/MmioRead.scala +++ /dev/null @@ -1,50 +0,0 @@ -package framework.balldomain.blink.mmio - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.balldomain.blink.{HasBallId, HasRobId} -import framework.memdomain.backend.mmio.{MmioReadReq, MmioReadResp} - -/** - * MmioRead: Ball-facing MMIO read port (part of BlinkIO). - * - * Defined from the CONSUMER (MmioRouter) perspective: - * - req is Flipped(Decoupled) so the consumer drives ready and reads valid/bits - * - resp is Decoupled so the consumer drives valid/bits and reads ready - * - ball_id, rob_id are Input (driven by the Ball when consumed) - * - * BlinkIO wraps this in Flipped(...), so from the Ball's perspective: - * - req becomes Decoupled (Ball drives valid/bits, reads ready) - * - resp becomes Flipped(Decoupled) (Ball reads valid/bits, drives ready) - * - ball_id, rob_id become Output (Ball drives them) - */ -class MmioRead(val b: GlobalConfig) extends Bundle with HasBallId with HasRobId { - val req = Flipped(Decoupled(new MmioReadReq(b))) - val resp = Decoupled(new MmioReadResp(b)) -} - -object MmioRead { - - /** - * Tie off an MmioRead port from the Ball's perspective (BlinkIO.mmioRead). - * Ball's perspective after Flipped: - * - req: Decoupled (Ball drives) - tie valid=0, bits=0 - * - resp: Flipped(Decoupled) (Ball reads) - tie ready=0 - * - ball_id, rob_id: Output (Ball drives) - tie to 0 - */ - def tieOff(port: MmioRead): Unit = { - port.req.valid := false.B - port.req.bits := 0.U.asTypeOf(port.req.bits) - port.resp.ready := false.B - port.ball_id := 0.U - port.rob_id := 0.U - } - - def tieOff(ports: Vec[MmioRead]): Unit = { - for (port <- ports) { - tieOff(port) - } - } - -} diff --git a/arch/src/main/scala/framework/balldomain/blink/mmio/MmioWrite.scala b/arch/src/main/scala/framework/balldomain/blink/mmio/MmioWrite.scala deleted file mode 100644 index 5e42c4d44..000000000 --- a/arch/src/main/scala/framework/balldomain/blink/mmio/MmioWrite.scala +++ /dev/null @@ -1,34 +0,0 @@ -package framework.balldomain.blink.mmio - -import chisel3._ -import chisel3.util.Decoupled -import framework.balldomain.blink.{HasBallId, HasRobId} -import framework.memdomain.backend.mmio.MmioWriteReq -import framework.top.GlobalConfig - -/** - * Ball-facing byte write port for the unified MMIO address space. - * - * BlinkIO wraps this in Flipped(...), so a Ball produces req and observes - * ready. A multi-byte value uses one configured line per byte. - */ -class MmioWrite(val b: GlobalConfig) extends Bundle with HasBallId with HasRobId { - val req = Flipped(Decoupled(new MmioWriteReq(b))) -} - -object MmioWrite { - - def tieOff(port: MmioWrite): Unit = { - port.req.valid := false.B - port.req.bits := 0.U.asTypeOf(port.req.bits) - port.ball_id := 0.U - port.rob_id := 0.U - } - - def tieOff(ports: Vec[MmioWrite]): Unit = { - for (port <- ports) { - tieOff(port) - } - } - -} diff --git a/arch/src/main/scala/framework/balldomain/blink/status.scala b/arch/src/main/scala/framework/balldomain/blink/status.scala deleted file mode 100644 index 843444a31..000000000 --- a/arch/src/main/scala/framework/balldomain/blink/status.scala +++ /dev/null @@ -1,9 +0,0 @@ -package framework.balldomain.blink - -import chisel3._ -import chisel3.util._ - -class BallStatus extends Bundle { - val idle = Output(Bool()) - val running = Output(Bool()) -} diff --git a/arch/src/main/scala/framework/balldomain/configs/BallDomainParam.scala b/arch/src/main/scala/framework/balldomain/configs/BallDomainParam.scala deleted file mode 100644 index cadf020f4..000000000 --- a/arch/src/main/scala/framework/balldomain/configs/BallDomainParam.scala +++ /dev/null @@ -1,66 +0,0 @@ -package framework.balldomain.configs - -import upickle.default._ - -case class BallIdMapping( - ballId: Int, - ballName: String, - ballClass: String, - config: Option[String] = None, - inBW: Int, - outBW: Int, - mmioReadBW: Int = 0, - mmioWriteBW: Int = 0, - ballParams: Map[String, String] = Map.empty) - -case class BallISAEntry( - mnemonic: String, - funct7: Int, - bid: Int) - -case class BallDomainParam( - ballNum: Int, - ballIdMappings: Seq[BallIdMapping], - ballISA: Seq[BallISAEntry]) { - - require(ballNum == ballIdMappings.length, "ballNum must match ballIdMappings length") - require( - ballIdMappings.map(_.ballId).distinct.length == ballIdMappings.length, - "ballIdMappings contains duplicate ballId" - ) - require( - ballISA.map(_.mnemonic).distinct.length == ballISA.length, - "ballISA contains duplicate mnemonic" - ) - require( - ballISA.map(_.funct7).distinct.length == ballISA.length, - "ballISA contains duplicate funct7" - ) - require( - ballISA.forall(entry => entry.funct7 >= 0 && entry.funct7 < 128), - "ballISA funct7 must fit the 7-bit instruction field" - ) - require( - ballISA.forall(entry => ballIdMappings.exists(_.ballId == entry.bid)), - "ballISA bid must reference a configured ballId" - ) - - def mapping(ballName: String): BallIdMapping = - ballIdMappings.find(_.ballName == ballName) match { - case Some(m) => m - case None => throw new RuntimeException(s"No ballIdMapping for ballName=$ballName") - } - -} - -object BallDomainParam { - implicit val ballIdMappingRW: ReadWriter[BallIdMapping] = macroRW - implicit val ballISAEntryRW: ReadWriter[BallISAEntry] = macroRW - implicit val rw: ReadWriter[BallDomainParam] = macroRW - - /** - * Empty default. Each example's GlobalConfig assembler is responsible for - * supplying its own `BallDomainParam` (typically via a layered JSON loader). - */ - def apply(): BallDomainParam = BallDomainParam(0, Seq.empty, Seq.empty) -} diff --git a/arch/src/main/scala/framework/balldomain/configs/BallParamLoader.scala b/arch/src/main/scala/framework/balldomain/configs/BallParamLoader.scala deleted file mode 100644 index 64283bda9..000000000 --- a/arch/src/main/scala/framework/balldomain/configs/BallParamLoader.scala +++ /dev/null @@ -1,66 +0,0 @@ -package framework.balldomain.configs - -import framework.top.GlobalConfig -import toml.{Toml, Value} -import java.nio.file.Paths - -object BallParamLoader { - - def load(path: String): Map[String, Value] = { - val content = scala.io.Source.fromFile(path, "UTF-8").mkString - Toml.parse(content) match { - case Right(Value.Tbl(root)) => root - case Right(_) => - throw new RuntimeException(s"TOML root must be a table in $path") - case Left((addr, msg)) => - throw new RuntimeException(s"TOML parse error at $addr: $msg in $path") - } - } - - def ball(root: Map[String, Value]): Map[String, Value] = - root.get("ball") match { - case Some(Value.Tbl(t)) => t - case None => throw new RuntimeException("Missing TOML section [ball]") - case _ => throw new RuntimeException("Expected TOML section [ball]") - } - - private def configPath(mapping: BallIdMapping): String = { - val config = mapping.config.getOrElse( - throw new RuntimeException(s"Ball ${mapping.ballName} has no config") - ) - if (!Paths.get(config).toFile.isFile) { - throw new RuntimeException(s"Ball ${mapping.ballName} config not found: $config") - } - config - } - - def ballTable(b: GlobalConfig, ballName: String): Map[String, Value] = { - val mapping = b.ballDomain.ballIdMappings.find(_.ballName == ballName) match { - case Some(m) => m - case None => throw new RuntimeException(s"No ballIdMapping for ballName=$ballName") - } - ball(load(configPath(mapping))) - } - - def int(table: Map[String, Value], key: String): Int = - table.get(key) match { - case Some(Value.Num(n)) => n.toInt - case None => throw new RuntimeException(s"Missing integer at key '$key'") - case _ => throw new RuntimeException(s"Expected integer at key '$key'") - } - - def str(table: Map[String, Value], key: String): String = - table.get(key) match { - case Some(Value.Str(s)) => s - case None => throw new RuntimeException(s"Missing string at key '$key'") - case _ => throw new RuntimeException(s"Expected string at key '$key'") - } - - def bool(table: Map[String, Value], key: String): Boolean = - table.get(key) match { - case Some(Value.Bool(b)) => b - case None => throw new RuntimeException(s"Missing boolean at key '$key'") - case _ => throw new RuntimeException(s"Expected boolean at key '$key'") - } - -} diff --git a/arch/src/main/scala/framework/balldomain/decoder/BallDecodeCmd.scala b/arch/src/main/scala/framework/balldomain/decoder/BallDecodeCmd.scala deleted file mode 100644 index e34e6f580..000000000 --- a/arch/src/main/scala/framework/balldomain/decoder/BallDecodeCmd.scala +++ /dev/null @@ -1,35 +0,0 @@ -package framework.balldomain.decoder - -import chisel3._ -import chisel3.util._ - -/** - * Wire-level command bundle exchanged between user-defined Ball decoders - * and framework infrastructure (BallReservationStation, BBus). The - * concrete decode logic (funct7 → BID/operands) lives in user code. - */ -class BallDecodeCmd(numBanks: Int, iterLen: Int, groupCountWidth: Int) extends Bundle { - val bid = UInt(5.W) - val funct7 = UInt(7.W) - val iter = UInt(iterLen.W) - - val op1_en = Bool() - val op2_en = Bool() - val wr_spad_en = Bool() - val op1_from_spad = Bool() - val op2_from_spad = Bool() - val special = UInt(64.W) - - val op1_bank = UInt(log2Up(numBanks).W) - val op2_bank = UInt(log2Up(numBanks).W) - val wr_bank = UInt(log2Up(numBanks).W) - val op1_col = UInt(groupCountWidth.W) - val op2_col = UInt(groupCountWidth.W) - val wr_col = UInt(groupCountWidth.W) - - // MMIO metadata bank: which main bank's MMIO region to access - val meta_bank = UInt(log2Up(numBanks).W) - - val rs1 = UInt(64.W) - val rs2 = UInt(64.W) -} diff --git a/arch/src/main/scala/framework/balldomain/decoder/BallDomainDecoder.scala b/arch/src/main/scala/framework/balldomain/decoder/BallDomainDecoder.scala deleted file mode 100644 index ca6db292d..000000000 --- a/arch/src/main/scala/framework/balldomain/decoder/BallDomainDecoder.scala +++ /dev/null @@ -1,107 +0,0 @@ -package framework.balldomain.decoder - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.balldomain.isa.BallISA -import framework.frontend.decoder.{DomainId, PostGDCmd} -import framework.top.GlobalConfig - -/** - * Ball-domain instruction decoder. - * - * - funct7 encoding contract → enable bits, operand/write banks, iter - * count, pass-through of rs1/rs2/funct7 (pure framework concern). - * - funct7 → bid routing, generated from `b.ballDomain.ballISA` - * (a `(mnemonic, funct7, bid)` table the user provides via - * TOML config); no per-example Scala decoder is needed. - */ -@instantiable -class BallDomainDecoder(val b: GlobalConfig) extends Module { - val bankIdLen = b.frontend.bank_id_len - val iterLen = b.frontend.iter_len - - @public - val cmd_i = IO(Flipped(Decoupled(new PostGDCmd(b)))) - - @public - val ball_decode_cmd_o = - IO(Decoupled(new BallDecodeCmd(b.memDomain.virtualBankCount, iterLen, b.memDomain.groupCountWidth))) - - cmd_i.ready := ball_decode_cmd_o.ready - - val func7 = cmd_i.bits.cmd.funct - val rs1 = cmd_i.bits.cmd.rs1Data - val rs2 = cmd_i.bits.cmd.rs2Data - - val op1_bank_raw = rs1(bankIdLen - 1, 0) - val op2_bank_raw = rs1(bankIdLen + 9, 10) - val wr_bank_raw = rs1(bankIdLen + 19, 20) - val iter_raw = rs1(63, 30) - - // funct7[6:4] enable encoding: - // 000 no bank 001 1rd 010 1wr 011 1rd+1wr - // 100 2rd+1wr 101/110/111 no bank (extended opcode space) - val enableBits = func7(6, 4) - val hasRd0 = enableBits === 1.U || enableBits === 3.U || enableBits === 4.U - val hasRd1 = enableBits === 4.U - val hasWr = enableBits === 2.U || enableBits === 3.U || enableBits === 4.U - - // funct7 -> bid lookup, generated from b.ballDomain.ballISA - require(!b.ballDomain.ballISA.exists(_.funct7 == BallISA.InitFunct), "BALL_INIT funct7 is framework-reserved") - - val bidWire = WireDefault(0.U(5.W)) - val isaValidWire = WireDefault(false.B) - b.ballDomain.ballISA.foreach { e => - when(func7 === e.funct7.U(7.W)) { - bidWire := e.bid.U - isaValidWire := true.B - } - } - - val isBallInit = func7 === BallISA.InitFunct.U - val decodedBid = Mux(isBallInit, rs2(4, 0), bidWire) - - ball_decode_cmd_o.valid := cmd_i.valid && (cmd_i.bits.domain_id === DomainId.BALL) - ball_decode_cmd_o.bits.bid := Mux(ball_decode_cmd_o.valid, decodedBid, 0.U) - ball_decode_cmd_o.bits.funct7 := Mux(ball_decode_cmd_o.valid, func7, 0.U) - ball_decode_cmd_o.bits.iter := Mux(ball_decode_cmd_o.valid, iter_raw, 0.U(iterLen.W)) - ball_decode_cmd_o.bits.special := Mux(ball_decode_cmd_o.valid, rs2, 0.U(64.W)) - ball_decode_cmd_o.bits.op1_bank := Mux(ball_decode_cmd_o.valid && hasRd0, op1_bank_raw, 0.U) - ball_decode_cmd_o.bits.op2_bank := Mux(ball_decode_cmd_o.valid && hasRd1, op2_bank_raw, 0.U) - ball_decode_cmd_o.bits.wr_bank := Mux(ball_decode_cmd_o.valid && hasWr, wr_bank_raw, 0.U) - ball_decode_cmd_o.bits.op1_col := Mux(ball_decode_cmd_o.valid && hasRd0, cmd_i.bits.op1_col, 0.U) - ball_decode_cmd_o.bits.op2_col := Mux(ball_decode_cmd_o.valid && hasRd1, cmd_i.bits.op2_col, 0.U) - ball_decode_cmd_o.bits.wr_col := Mux(ball_decode_cmd_o.valid && hasWr, cmd_i.bits.wr_col, 0.U) - ball_decode_cmd_o.bits.op1_en := ball_decode_cmd_o.valid && hasRd0 - ball_decode_cmd_o.bits.op2_en := ball_decode_cmd_o.valid && hasRd1 - ball_decode_cmd_o.bits.wr_spad_en := ball_decode_cmd_o.valid && hasWr - ball_decode_cmd_o.bits.op1_from_spad := ball_decode_cmd_o.valid && hasRd0 - ball_decode_cmd_o.bits.op2_from_spad := ball_decode_cmd_o.valid && hasRd1 - - // MMIO meta_bank: default to wr_bank (typical case: Ball's output bank - // is bound to a per-output-block MMIO scale region). - // Balls that need different binding can override via their decode logic. - ball_decode_cmd_o.bits.meta_bank := Mux(ball_decode_cmd_o.valid && hasWr, wr_bank_raw, 0.U) - - ball_decode_cmd_o.bits.rs1 := rs1 - ball_decode_cmd_o.bits.rs2 := rs2 - - assert( - !(cmd_i.fire && (cmd_i.bits.domain_id === DomainId.BALL) && !isBallInit && !isaValidWire), - "BallDomainDecoder: funct7 has no matching entry in ballISA" - ) - when(cmd_i.fire && isBallInit) { - assert(rs1 === 0.U, "BALL_INIT requires rs1=0") - assert(rs2(63, 5) === 0.U, "BALL_INIT reserves rs2[63:5]") - assert( - VecInit(b.ballDomain.ballIdMappings.map(m => rs2(4, 0) === m.ballId.U)).asUInt.orR, - "BALL_INIT target ID is not configured" - ) - } - assert( - !(ball_decode_cmd_o.valid && ball_decode_cmd_o.bits.op1_en && ball_decode_cmd_o.bits.op2_en && - ball_decode_cmd_o.bits.op1_bank === ball_decode_cmd_o.bits.op2_bank), - "BallDomainDecoder: Ball instruction OpA and OpB cannot access the same bank" - ) -} diff --git a/arch/src/main/scala/framework/balldomain/isa/BallISA.scala b/arch/src/main/scala/framework/balldomain/isa/BallISA.scala deleted file mode 100644 index ad7dbfac1..000000000 --- a/arch/src/main/scala/framework/balldomain/isa/BallISA.scala +++ /dev/null @@ -1,14 +0,0 @@ -package framework.balldomain.isa - -import framework.system.core.rocket.BuckyballCommand - -/** Framework-reserved BallDomain instructions. */ -object BallISA { - val InitFunct = 0x05 - - def init(ballId: Int): BuckyballCommand = { - require(ballId >= 0 && ballId < 32, s"BALL_INIT target ID out of range: $ballId") - BuckyballCommand(InitFunct, rs1 = 0, rs2 = BigInt(ballId)) - } - -} diff --git a/arch/src/main/scala/framework/balldomain/rs/interfaces.scala b/arch/src/main/scala/framework/balldomain/rs/interfaces.scala deleted file mode 100644 index a19dbe71b..000000000 --- a/arch/src/main/scala/framework/balldomain/rs/interfaces.scala +++ /dev/null @@ -1,22 +0,0 @@ -package framework.balldomain.rs - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.balldomain.decoder.BallDecodeCmd - -// Ball domain issue interface - includes global rob_id -class BallRsIssue(b: GlobalConfig) extends Bundle { - val cmd = new BallDecodeCmd(b.memDomain.virtualBankCount, b.frontend.iter_len, b.memDomain.groupCountWidth) - // Global ROB ID - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) -} - -// Ball domain completion interface -class BallRsComplete(b: GlobalConfig) extends Bundle { - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) -} diff --git a/arch/src/main/scala/framework/balldomain/rs/reservationStation.scala b/arch/src/main/scala/framework/balldomain/rs/reservationStation.scala deleted file mode 100644 index 0fac1316e..000000000 --- a/arch/src/main/scala/framework/balldomain/rs/reservationStation.scala +++ /dev/null @@ -1,121 +0,0 @@ -package framework.balldomain.rs - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig -import framework.balldomain.decoder.BallDecodeCmd - -// Generic Ball domain issue interface - supports dynamic number of Ball devices -class BallIssueInterface(b: GlobalConfig) extends Bundle { - val balls = Vec(b.ballDomain.ballNum, Decoupled(new BallRsIssue(b))) -} - -// Generic Ball domain completion interface - supports dynamic number of Ball devices -class BallCommitInterface(b: GlobalConfig) extends Bundle { - val balls = Vec(b.ballDomain.ballNum, Flipped(Decoupled(new BallRsComplete(b)))) -} - -// Local Ball reservation station - simple FIFO scheduler -@instantiable -class BallReservationStation(val b: GlobalConfig) extends Module { - - @public - val ball_decode_cmd_i = IO(Flipped(new DecoupledIO(new Bundle { - val cmd = new BallDecodeCmd(b.memDomain.virtualBankCount, b.frontend.iter_len, b.memDomain.groupCountWidth) - // Global ROB ID - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) - }))) - - // Rs -> BallController (multi-channel issue) - @public - val issue_o = IO(new BallIssueInterface(b)) - - @public - val commit_i = IO(new BallCommitInterface(b)) - - // Output completion signal (with global rob_id, single channel) - @public - val complete_o = IO(Decoupled(new BallRsComplete(b))) - - // Simple FIFO queue, only for buffering - val fifo = Module(new Queue( - new Bundle { - val cmd = new BallDecodeCmd(b.memDomain.virtualBankCount, b.frontend.iter_len, b.memDomain.groupCountWidth) - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) - }, - entries = 4 - )) // Small buffer is sufficient - -// ----------------------------------------------------------------------------- -// Inbound - FIFO enqueue -// ----------------------------------------------------------------------------- - fifo.io.enq <> ball_decode_cmd_i - -// ----------------------------------------------------------------------------- -// Outbound - instruction issue (dispatch to corresponding Ball device based on bid) -// ----------------------------------------------------------------------------- - val headEntry = fifo.io.deq.bits - - // Build ballId to index mapping from config - // Config order should match the order in busRegister.scala - // Each index in issue_o.balls corresponds to a ball registered in BBus - val numConfiguredBalls = b.ballDomain.ballIdMappings.length - - // Set issue signals for each Ball device - // Use configured ball id mappings: index i in issue_o.balls corresponds to ballIdMappings(i).ballId - for (i <- 0 until b.ballDomain.ballNum) { - if (i < numConfiguredBalls) { - val configuredBallId = b.ballDomain.ballIdMappings(i).ballId.U - issue_o.balls(i).valid := fifo.io.deq.valid && headEntry.cmd.bid === configuredBallId - issue_o.balls(i).bits.cmd := headEntry.cmd - issue_o.balls(i).bits.rob_id := headEntry.rob_id - issue_o.balls(i).bits.is_sub := headEntry.is_sub - issue_o.balls(i).bits.sub_rob_id := headEntry.sub_rob_id - } else { - // Unused slots - no valid signal - issue_o.balls(i).valid := false.B - issue_o.balls(i).bits.cmd := DontCare - issue_o.balls(i).bits.rob_id := DontCare - issue_o.balls(i).bits.is_sub := DontCare - issue_o.balls(i).bits.sub_rob_id := DontCare - } - } - - // FIFO deq.ready - can only dequeue when target Ball device is ready - // Find which index corresponds to the requested ballId - fifo.io.deq.ready := VecInit( - (0 until b.ballDomain.ballNum).map { idx => - if (idx < numConfiguredBalls) { - val configuredBallId = b.ballDomain.ballIdMappings(idx).ballId.U - (headEntry.cmd.bid === configuredBallId) && issue_o.balls(idx).ready - } else { - false.B - } - } - ).asUInt.orR - -// ----------------------------------------------------------------------------- -// Completion signal processing - directly forward to global RS -// ----------------------------------------------------------------------------- - val completeArb = Module(new Arbiter(new BallRsComplete(b), b.ballDomain.ballNum)) - - // Connect completion signals from all Ball devices to arbiter - for (i <- 0 until b.ballDomain.ballNum) { - completeArb.io.in(i).valid := commit_i.balls(i).valid - completeArb.io.in(i).bits := commit_i.balls(i).bits - commit_i.balls(i).ready := completeArb.io.in(i).ready - } - - // Forward completion signal (with global rob_id) - complete_o.valid := completeArb.io.out.valid - complete_o.bits := completeArb.io.out.bits - completeArb.io.out.ready := complete_o.ready - -} diff --git a/arch/src/main/scala/framework/build.sc b/arch/src/main/scala/framework/build.sc new file mode 100644 index 000000000..12de97e80 --- /dev/null +++ b/arch/src/main/scala/framework/build.sc @@ -0,0 +1,236 @@ +import mill._ +import mill.scalalib._ +import $file.device.{build => device_build} +import $file.core.{build => core_build} +import $file.thread.{build => thread_build} +import $file.acc.{build => acc_build} +import $file.ip.{build => ip_build} +import $file.lib.{build => lib_build} +import $file.acc.balldomain.{build => acc_balldomain_build} +import $file.acc.frontend.{build => acc_frontend_build} +import $file.acc.frontend.src.main.scala.boot.{build => acc_frontend_boot_build} +import $file.acc.frontend.src.main.scala.scheduler.{build => acc_frontend_scheduler_build} +import $file.acc.frontend.src.main.scala.scheduler.src.main.scala.bat.{build => acc_frontend_scheduler_bat_build} +import $file.acc.frontend.src.main.scala.scheduler.src.main.scala.rob.global.{build => acc_frontend_scheduler_rob_global_build} +import $file.acc.frontend.src.main.scala.scheduler.src.main.scala.scoreboard.{build => acc_frontend_scheduler_scoreboard_build} +import $file.acc.gpdomain.{build => acc_gpdomain_build} +import $file.acc.gpdomain.src.main.scala.execution.{build => acc_gpdomain_execution_build} +import $file.acc.gpdomain.src.main.scala.loader.{build => acc_gpdomain_loader_build} +import $file.acc.gpdomain.src.main.scala.memory.{build => acc_gpdomain_memory_build} +import $file.acc.memdomain.{build => acc_memdomain_build} +import $file.acc.memdomain.src.main.scala.bank_router.{build => acc_memdomain_bank_router_build} +import $file.acc.memdomain.src.main.scala.prm.{build => acc_memdomain_prm_build} +import $file.acc.memdomain.src.main.scala.tomem.{build => acc_memdomain_tomem_build} +import $file.acc.memdomain.src.main.scala.tomem.src.main.scala.toddr.{build => acc_memdomain_tomem_toddr_build} +import $file.acc.memdomain.src.main.scala.tomem.src.main.scala.tol1.{build => acc_memdomain_tomem_tol1_build} +import $file.core.controller.admission.{build => core_controller_admission_build} +import $file.core.execution.fetch.{build => core_execution_fetch_build} +import $file.core.execution.fpu.{build => core_execution_fpu_build} +import $file.core.execution.instruction.{build => core_execution_instruction_build} +import $file.core.execution.integer.{build => core_execution_integer_build} +import $file.core.execution.muldiv.{build => core_execution_muldiv_build} +import $file.core.execution.privileged.{build => core_execution_privileged_build} +import $file.core.execution.register_file.{build => core_execution_register_file_build} +import $file.core.memory.access.{build => core_memory_access_build} +import $file.core.memory.access.src.main.scala.cache.{build => core_memory_access_cache_build} +import $file.core.memory.access.src.main.scala.mmu.{build => core_memory_access_mmu_build} +import $file.core.memory.access.src.main.scala.preflight.{build => core_memory_access_preflight_build} +import $file.core.memory.home.{build => core_memory_home_build} +import $file.core.tbus.{build => core_tbus_build} +import $file.core.tbus.src.main.scala.router.{build => core_tbus_router_build} +import $file.core.tss.{build => core_tss_build} +import $file.device.ddr.{build => device_ddr_build} +import $file.ip.axi4.{build => ip_axi4_build} +import $file.ip.axis.{build => ip_axis_build} +import $file.ip.bank.{build => ip_bank_build} +import $file.ip.cache.{build => ip_cache_build} +import $file.ip.chi.{build => ip_chi_build} +import $file.ip.coherence.{build => ip_coherence_build} +import $file.ip.interlock.{build => ip_interlock_build} +import $file.ip.mesh_shm.{build => ip_mesh_shm_build} +import $file.ip.spm.{build => ip_spm_build} +import $file.thread.ant.{build => thread_ant_build} +import $file.^.^.^.^.thirdparty.{build => thirdparty_build} + +val frameworkRoot: os.Path = millSourcePath +val resourceRoot: os.Path = millSourcePath / os.up / os.up / "resources" + +object library extends SbtModule { + override def resources = T.sources(Seq(PathRef(resourceRoot))) + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(thirdparty_build.protoJava, thirdparty_build.hardfloat) + override def ivyDeps = + Agg(ivy"org.chipsalliance::chisel:6.7.0", ivy"com.lihaoyi::upickle:3.3.1") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") + + override def sources = T.sources( + device_build.sources() ++ core_build.sources() ++ thread_build.sources() ++ + acc_build.sources() ++ ip_build.sources() ++ lib_build.sources() ++ + Seq(PathRef(frameworkRoot / os.up / "configs"), PathRef(frameworkRoot / os.up / "sims" / "configs")) + ) + +} + +object emitters extends SbtModule { + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(library) + override def ivyDeps = library.ivyDeps() + override def scalacPluginIvyDeps = library.scalacPluginIvyDeps() + override def scalacOptions = library.scalacOptions() + override def sources = T.sources( + device_build.emitSources() ++ core_build.emitSources() ++ thread_build.emitSources() ++ + acc_build.emitSources() ++ ip_build.emitSources() ++ lib_build.emitSources() + ) +} + +object acc extends Module { + object balldomain extends acc_balldomain_build.Emitter { + override def framework = library + } + object frontend extends acc_frontend_build.Emitter { + override def framework = library + object boot extends acc_frontend_boot_build.Emitter { + override def framework = library + } + object scheduler extends acc_frontend_scheduler_build.Emitter { + override def framework = library + object bat extends acc_frontend_scheduler_bat_build.Emitter { + override def framework = library + } + object rob extends Module { + object global extends acc_frontend_scheduler_rob_global_build.Emitter { + override def framework = library + } + } + object scoreboard extends acc_frontend_scheduler_scoreboard_build.Emitter { + override def framework = library + } + } + } + object gpdomain extends acc_gpdomain_build.Emitter { + override def framework = library + object execution extends acc_gpdomain_execution_build.Emitter { + override def framework = library + } + object loader extends acc_gpdomain_loader_build.Emitter { + override def framework = library + } + object memory extends acc_gpdomain_memory_build.Emitter { + override def framework = library + } + } + object memdomain extends acc_memdomain_build.Emitter { + override def framework = library + object bank_router extends acc_memdomain_bank_router_build.Emitter { + override def framework = library + } + object prm extends acc_memdomain_prm_build.Emitter { + override def framework = library + } + object tomem extends acc_memdomain_tomem_build.Emitter { + override def framework = library + object toddr extends acc_memdomain_tomem_toddr_build.Emitter { + override def framework = library + } + object tol1 extends acc_memdomain_tomem_tol1_build.Emitter { + override def framework = library + } + } + } +} +object core extends Module { + object controller extends Module { + object admission extends core_controller_admission_build.Emitter { + override def framework = library + } + } + object execution extends Module { + object fetch extends core_execution_fetch_build.Emitter { + override def framework = library + } + object fpu extends core_execution_fpu_build.Emitter { + override def framework = library + } + object instruction extends core_execution_instruction_build.Emitter { + override def framework = library + } + object integer extends core_execution_integer_build.Emitter { + override def framework = library + } + object muldiv extends core_execution_muldiv_build.Emitter { + override def framework = library + } + object privileged extends core_execution_privileged_build.Emitter { + override def framework = library + } + object register_file extends core_execution_register_file_build.Emitter { + override def framework = library + } + } + object memory extends Module { + object access extends core_memory_access_build.Emitter { + override def framework = library + object cache extends core_memory_access_cache_build.Emitter { + override def framework = library + } + object mmu extends core_memory_access_mmu_build.Emitter { + override def framework = library + } + object preflight extends core_memory_access_preflight_build.Emitter { + override def framework = library + } + } + object home extends core_memory_home_build.Emitter { + override def framework = library + } + } + object tbus extends core_tbus_build.Emitter { + override def framework = library + object router extends core_tbus_router_build.Emitter { + override def framework = library + } + } + object tss extends core_tss_build.Emitter { + override def framework = library + } +} +object device extends Module { + object ddr extends device_ddr_build.Emitter { + override def framework = library + } +} +object ip extends Module { + object axi4 extends ip_axi4_build.Emitter { + override def framework = library + } + object axis extends ip_axis_build.Emitter { + override def framework = library + } + object bank extends ip_bank_build.Emitter { + override def framework = library + } + object cache extends ip_cache_build.Emitter { + override def framework = library + } + object chi extends ip_chi_build.Emitter { + override def framework = library + } + object coherence extends ip_coherence_build.Emitter { + override def framework = library + } + object interlock extends ip_interlock_build.Emitter { + override def framework = library + } + object mesh_shm extends ip_mesh_shm_build.Emitter { + override def framework = library + } + object spm extends ip_spm_build.Emitter { + override def framework = library + } +} +object thread extends Module { + object ant extends thread_ant_build.Emitter { + override def framework = library + } +} diff --git a/arch/src/main/scala/framework/core/build.sc b/arch/src/main/scala/framework/core/build.sc new file mode 100644 index 000000000..0685199c5 --- /dev/null +++ b/arch/src/main/scala/framework/core/build.sc @@ -0,0 +1,31 @@ +import mill._ +import $file.clink.{build => clink_build} +import $file.controller.{build => controller_build} +import $file.execution.{build => execution_build} +import $file.memory.{build => memory_build} +import $file.shm.{build => shm_build} +import $file.tbus.{build => tbus_build} +import $file.tss.{build => tss_build} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("configs/CoreParam.scala")) + ) ++ + clink_build.sources() ++ + controller_build.sources() ++ + execution_build.sources() ++ + memory_build.sources() ++ + shm_build.sources() ++ + tbus_build.sources() ++ + tss_build.sources() +) + +def emitSources = T.sources( + clink_build.emitSources() ++ + controller_build.emitSources() ++ + execution_build.emitSources() ++ + memory_build.emitSources() ++ + shm_build.emitSources() ++ + tbus_build.emitSources() ++ + tss_build.emitSources() +) diff --git a/arch/src/main/scala/framework/core/clink/basecore.scala b/arch/src/main/scala/framework/core/clink/basecore.scala new file mode 100644 index 000000000..fb2f9b82c --- /dev/null +++ b/arch/src/main/scala/framework/core/clink/basecore.scala @@ -0,0 +1,5 @@ +package framework.core.clink + +trait HasCLink { + def clink: CLinkIO +} diff --git a/arch/src/main/scala/framework/core/clink/build.sc b/arch/src/main/scala/framework/core/clink/build.sc new file mode 100644 index 000000000..956ae69c1 --- /dev/null +++ b/arch/src/main/scala/framework/core/clink/build.sc @@ -0,0 +1,13 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("basecore.scala")), + PathRef(millSourcePath / os.RelPath("clink.scala")), + PathRef(millSourcePath / os.RelPath("configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/core/clink/clink.scala b/arch/src/main/scala/framework/core/clink/clink.scala new file mode 100644 index 000000000..6fea650ff --- /dev/null +++ b/arch/src/main/scala/framework/core/clink/clink.scala @@ -0,0 +1,68 @@ +package framework.core.clink + +import framework.core.clink.configs.{Params => CLinkParam} +import framework.ip.bus.chi +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.memdomain.{MvoverPort, PhysicalBankHash} +import framework.thread.tlink.BankAccess + +import chisel3._ +import chisel3.util._ +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} + +class SharedPort(axi: axi4.Params) extends Bundle { + val tx = new axi4.Port(axi) + val rx = Flipped(new axi4.Port(axi)) +} + +/** An explicitly attached shared-bank fabric; it does not imply hardware threads. */ +class CoreBanks(geometry: Option[(FrontendParam, MemDomainParam, Int)], diff: Boolean) extends Bundle { + val access = geometry.map { case (frontend, memory, ports) => new BankAccess(frontend, memory) } + val move = new MvoverPort + + val barrier = Option.when(geometry.nonEmpty)(new Bundle { + val arrive = Output(Bool()) + val release = Input(Bool()) + }) + + val hashes = geometry.filter(_ => diff).map { case (frontend, memory, _) => + Input(Vec(memory.sharedBankNum, new PhysicalBankHash(frontend))) + } + +} + +/** Core memory protocols and hardware interrupt notifications behind CLink. */ +class CLinkIO( + val p: CLinkParam, + bankInterface: Option[CoreBanks] = None) + extends Bundle { + + val banks = bankInterface.map(_.cloneType) + + val interrupts = Input(Vec(p.harts, new CoreInterrupts)) + + val memory = new Bundle { + + val coherent = Vec(p.requesters, new chi.RequesterPort(p.line)) + + val backing = Option.when(p.backing)(new Bundle { + val request = Decoupled(new LineRequest(p.line)) + val response = Flipped(Decoupled(new LineResponse(p.line))) + }) + + val uncached = new axi4.Port(p.uncached) + + val ddr = new axi4.Port(p.ddr) + } + + val c2c = p.c2c.map(new SharedPort(_)) +} + +class CoreInterrupts extends Bundle { + val timer = Bool() + val software = Bool() + val external = Bool() + val supervisorExternal = Bool() +} diff --git a/arch/src/main/scala/framework/core/clink/configs/Params.scala b/arch/src/main/scala/framework/core/clink/configs/Params.scala new file mode 100644 index 000000000..aeed8283d --- /dev/null +++ b/arch/src/main/scala/framework/core/clink/configs/Params.scala @@ -0,0 +1,23 @@ +package framework.core.clink.configs + +import framework.ip.bus.{axi4, chi} +import upickle.default.{macroRW, ReadWriter} + +case class Params( + harts: Int, + backing: Boolean, + requesterIds: Seq[Int], + line: chi.Params, + uncached: axi4.Params, + ddr: axi4.Params, + c2c: Option[axi4.Params]) { + def requesters: Int = requesterIds.size + require( + requesterIds.distinct.size == requesterIds.size && requesterIds.forall(_ > 0), + "Invalid CHI requester node IDs" + ) + def shared: Boolean = c2c.nonEmpty + require(!(backing && requesters > 0), "A core exports either backing requests or a coherent requester") +} + +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/configs/CoreParam.scala b/arch/src/main/scala/framework/core/configs/CoreParam.scala new file mode 100644 index 000000000..ca364d8c1 --- /dev/null +++ b/arch/src/main/scala/framework/core/configs/CoreParam.scala @@ -0,0 +1,211 @@ +package framework.core.configs + +import buckyball.config.{CorePlacement, Design} +import framework.acc.configs.AccParam +import framework.core.tbus.router.configs.{Params => RouterParam} +import scala.jdk.CollectionConverters._ +import framework.ip.memory.spm +import framework.core.memory.access.preflight.configs.PmpParams + +import framework.core.execution.fetch.configs.{Params => FetchParams} +import framework.core.execution.instruction.configs.{Params => InstructionParams} +import framework.core.execution.integer.configs.{Params => IntegerParams} +import framework.core.execution.muldiv.configs.{Params => MulDivParams} +import framework.core.execution.fpu.configs.{Params => FpuParams} +import framework.core.execution.privileged.configs.{Params => PrivilegedParams} +import framework.core.execution.register_file.configs.{Params => RegisterFileParams} +import framework.core.controller.admission.configs.{Params => AdmissionParams} +import framework.thread.configs.ThreadParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.clink.configs.{Params => CLinkParams} +import framework.core.tbus.configs.{Params => TBusParams} +import framework.core.tss.configs.{Params => TaskStorageParams} +import framework.device.cbus.C2CParams +import framework.ip.bus.chi.rnf.RnfParams +import framework.ip.bus.axi4.{InterconnectParam, Params => AxiParam} +import framework.core.execution.configs.ExecutionParam +import framework.core.memory.configs.MemoryParam +import framework.core.controller.configs.{CommandsParam, ControllerParam, TasksParam} + +case class CoreParam( + hartId: Int, + resetVector: BigInt, + mtimeAddress: BigInt, + clink: CLinkParams, + tbus: Option[TBusParams], + tss: Option[TaskStorageParams], + c2c: C2CParams, + transferLocalAccess: Boolean, + transferLeaseAccess: Boolean, + dmaCache: Option[RnfParams], + interconnect: InterconnectParam, + threads: Seq[ThreadParam], + acc: Option[AccParam], + memory: MemoryParam, + fetch: FetchParams, + instruction: InstructionParams, + integer: IntegerParams, + muldiv: MulDivParams, + fpu: FpuParams, + privileged: PrivilegedParams, + registerFile: RegisterFileParams, + admission: AdmissionParams) { + + // TBus uses the bank/tag widths, not a thread's private scheduling policy. + def sharedFrontend: FrontendParam = threads.map(_.acc.get.frontend).reduce { (left, right) => + left.copy( + bank_id_len = math.max(left.bank_id_len, right.bank_id_len), + rob_entries = math.max(left.rob_entries, right.rob_entries) + ) + } + + // Physical shared SRAM geometry must agree. Private capacity and virtual namespaces may differ. + def sharedMemory: MemDomainParam = threads.map(_.acc.get.memDomain).reduce { (left, right) => + require( + ( + left.bankWidth, + left.bankMaskLen, + left.sharedEnable, + left.sharedBankNum, + left.sharedBankEntries, + left.sharedInputChannels + ) == + (right.bankWidth, right.bankMaskLen, right.sharedEnable, right.sharedBankNum, right.sharedBankEntries, right.sharedInputChannels), + "Threads attached to one TBus must agree on the shared SRAM interface" + ) + left.copy( + bankNum = math.max(left.bankNum, right.bankNum), + bankEntries = left.sharedBankEntries, + virtualBankCount = math.max(left.virtualBankCount, right.virtualBankCount) + ) + } + + def controller: ControllerParam = { + val taskParams = Option.when(threads.nonEmpty)( + TasksParam(threads.map(_.ant.get), threads.map(_.signature), tss.get, privileged.permission) + ) + ControllerParam( + CommandsParam( + memory.dcache.tracking, + integer.xLen, + privileged.permission, + admission.moves, + threads.nonEmpty, + clink.shared + ), + taskParams + ) + } + + def execution(directAccelerator: Boolean): ExecutionParam = ExecutionParam( + hartId, + resetVector, + threads.nonEmpty || clink.shared || directAccelerator, + directAccelerator, + admission.moves, + memory.dcache.tracking, + mtimeAddress, + memory.memoryAccess, + instruction, + registerFile, + integer, + muldiv, + fpu, + privileged + ) + +} + +object CoreParam { + + def fromProto( + design: Design, + core: CorePlacement, + externalHome: Boolean + ): CoreParam = { + val threads = core.getThreadIndicesList.asScala.map { index => + val thread = design.getThreads(index.intValue) + require(thread.getIndex == index.intValue) + ThreadParam.fromProto(thread, core) + }.toSeq + val corep = fromProto(core, threads) + corep.copy(clink = + corep.clink.copy( + backing = !externalHome, + requesterIds = + if (externalHome) + Seq(corep.memory.icache.nodeId, corep.memory.dcache.cache.nodeId) ++ corep.dmaCache.map(_.nodeId) + else Seq.empty + ) + ) + } + + def fromProto(p: CorePlacement, threads: Seq[ThreadParam]): CoreParam = { + require( + p.hasMemory && p.hasParam && p.hasSharedMem && (threads.isEmpty || p.hasTss), + "Core configuration is incomplete" + ) + require(p.getThreadIndicesCount == threads.size, "Core thread configuration is incomplete") + val accp = Option.when(p.hasAccelerator)(AccParam.fromProto(p)) + val memp = MemoryParam.fromProto(p.getMemory, threads.size) + val cp = p.getParam + val permission = PmpParams(cp.getPaddrBits, cp.getNPmps) + val shared = p.getSharedMem + val tss = p.getTss + val ids = threads.map(_.id) + val directShared = threads.isEmpty && accp.exists(_.memDomain.sharedEnable) + val participants = if (directShared) Seq(0) else ids + val crossbar = AxiParam.fromProto(p.getInterconnect.getAxi) + val c2c = p.getC2C + val c2cAxi = AxiParam.fromProto(c2c.getAxi) + require(ids == ids.indices, "TBus thread IDs must be ordered and contiguous") + CoreParam( + hartId = p.getHartId, + resetVector = BigInt(java.lang.Long.toUnsignedString(p.getResetVector)), + mtimeAddress = BigInt(java.lang.Long.toUnsignedString(p.getMtimeAddress)), + clink = CLinkParams( + 1, + true, + Seq.empty, + memp.memoryAccess.memory.chi, + memp.uncached.axi, + accp.map(_.transport.axi).getOrElse(crossbar), + Option.when(c2c.getEnabled)(c2cAxi) + ), + tbus = Option.when(p.hasTbus)(TBusParams( + RouterParam(if (c2c.getEnabled) 3 + (if (directShared) 1 else 0) else 0), + participants, + participants.size, + p.getTbus.getBankNodesList.asScala.map(_.intValue).toSeq, + !directShared && p.getController.getMoves + )), + tss = Option.when(threads.nonEmpty)(TaskStorageParams( + spm.Params(BigInt(java.lang.Long.toUnsignedString(tss.getBase)), tss.getBytes, tss.getDataBits), + ids.size + )), + c2c = C2CParams( + BigInt(shared.getEntries) * (shared.getBankWidth / 8), + shared.getBankEntries * (shared.getBankWidth / 8), + c2c.getPeerCores, + c2cAxi + ), + transferLocalAccess = directShared && c2c.getEnabled, + transferLeaseAccess = c2c.getEnabled, + dmaCache = Option.when(p.hasDmaCache)(RnfParams.fromProto(p.getDmaCache, memp.home.memory.chi)), + interconnect = InterconnectParam(crossbar, ids.size, Some(p.getInterconnect.getSlots)), + threads = threads, + acc = accp, + memory = memp, + fetch = FetchParams(p.getFetch.getInstructionBytes), + instruction = InstructionParams(cp.getXLen), + integer = IntegerParams(cp.getXLen), + muldiv = MulDivParams(cp.getXLen), + fpu = FpuParams(cp.getXLen, p.getFpu.getFLen), + privileged = PrivilegedParams(permission), + registerFile = RegisterFileParams(cp.getXLen), + admission = AdmissionParams(permission, p.getController.getMoves) + ) + } + +} diff --git a/arch/src/main/scala/framework/core/controller/Controller.scala b/arch/src/main/scala/framework/core/controller/Controller.scala new file mode 100644 index 000000000..d135653a1 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/Controller.scala @@ -0,0 +1,54 @@ +package framework.core.controller + +import framework.thread.ant.Task +import framework.ip.memory.spm +import chisel3._ +import chisel3.util.MixedVec +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.controller.configs.ControllerParam +import framework.core.controller.admission.{CommandAdmission, Commands, TaskEndpoint, TaskRouter} +import framework.core.controller.task.Service +import framework.acc.memdomain.MvoverPort +import framework.core.tss.TssControlPort + +/** Core command admission, task management and thread launch control. */ +@instantiable +class Controller(b: ControllerParam) extends Module { + val commands = b.commands + + @public + val io = IO(new Bundle { + + val execution = Flipped(new Commands(commands.tracking, commands.dataBits, commands.permission)) + val threads = b.tasks.map(p => MixedVec(p.locals.map(local => Flipped(new Task(local, p.permission))))) + val move = new MvoverPort + val remote = Option.when(commands.remote)(new TaskEndpoint) + val tss = b.tasks.map(p => new TssControlPort(p.tss)) + val memoryAllowed = Output(Bool()) + val drained = Output(Bool()) + }) + + val admission: Instance[CommandAdmission] = Instantiate(new CommandAdmission(commands)) + val router: Instance[TaskRouter] = Instantiate(new TaskRouter(commands)) + admission.io.execution <> io.execution + admission.io.move <> io.move + router.io.command <> admission.io.taskRequest + admission.io.taskResponse <> router.io.response + io.remote.zip(router.io.remote).foreach { case (port, endpoint) => port <> endpoint } + io.memoryAllowed := admission.io.memoryAllowed + io.drained := admission.io.idle && !router.io.busy + + if (b.tasks.nonEmpty) { + val service: Instance[Service] = Instantiate(new Service(b.tasks.get)) + service.io.request <> router.io.local.request + router.io.local.reply <> service.io.reply + router.io.local.busy := !service.io.idle + io.tss.get <> service.io.tss + service.io.tasks <> io.threads.get + } else { + router.io.local.request.ready := false.B + router.io.local.reply.valid := false.B + router.io.local.reply.bits := 0.U + router.io.local.busy := false.B + } +} diff --git a/arch/src/main/scala/framework/core/controller/admission/build.sc b/arch/src/main/scala/framework/core/controller/admission/build.sc new file mode 100644 index 000000000..43fc6a4c7 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.controller.admission.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/CommandAdmission.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/DirectAdmission.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/TaskRouter.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/controller/admission/configs/uvm.toml b/arch/src/main/scala/framework/core/controller/admission/configs/uvm.toml new file mode 100644 index 000000000..5669ed6cb --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/configs/uvm.toml @@ -0,0 +1,15 @@ +[parameters] +dataBits=64 +moves=true +taskEnabled=true +remote=true +[parameters.permission] +addressBits=44 +entries=4 +[parameters.tracking] +entries=4 +addressBits=44 +idBits=8 +lineBytes=64 +maxRanges=8 +[defines] diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/env/env.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/env/env.svh new file mode 100644 index 000000000..1f8748f8e --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/env/model.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/env/model.svh new file mode 100644 index 000000000..16c3c29b9 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/env/model.svh @@ -0,0 +1,148 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(result_item) expected, actual; + const int copies[`AD_PAYLOAD_FIELDS][4] = `AD_PAYLOAD_MAP; + bit pending, moving, has_result, work_done, response_done, move_error, remote_owner; + int saved_tag, destination, tags[$]; + covergroup behavior with function sample (int route, bit stalled, bit cancelled); + option.per_instance = 1; + routes: coverpoint route {bins all[] = {[0 : 1]};} + backpressure: coverpoint stalled; + reset_cancellation: coverpoint cancelled; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + behavior = new(); + endfunction + function void write(cycle_item t); + logic [`AD_OUTPUT_BITS-1:0] want, mask; + result_item predicted, received; + + if (t.reset) begin + behavior.sample(0, 0, pending || tags.size() != 0); + pending = 0; + tags.delete(); + return; + end + want = '0; + mask = '0; + begin + bit offered, task_command, move_command, write_result; + logic [31:0] instruction; + instruction = `AI(t, EXECUTION_COMMAND_BITS_INSTRUCTION_RAW_INST); + offered = `AI(t, EXECUTION_COMMAND_VALID); + task_command = instruction[6:0] == 'h2b; + move_command = instruction[6:0] == 'h7b && instruction[31:25] == 'h0d; + want[`AD_OUT_EXECUTION_COMMAND_READY_OFFSET+:`AD_OUT_EXECUTION_COMMAND_READY_WIDTH]=!pending&&(task_command? + `AI(t, TASKREQUEST_READY) + : (move_command && + `AI(t, MOVE_COMMAND_READY) + )); + mask[`AD_OUT_EXECUTION_COMMAND_READY_OFFSET+:`AD_OUT_EXECUTION_COMMAND_READY_WIDTH] = '1; + + want[`AD_OUT_TASKREQUEST_VALID_OFFSET+:`AD_OUT_TASKREQUEST_VALID_WIDTH]=!pending&&offered&&task_command; + mask[`AD_OUT_TASKREQUEST_VALID_OFFSET+:`AD_OUT_TASKREQUEST_VALID_WIDTH] = '1; + + want[`AD_OUT_MOVE_COMMAND_VALID_OFFSET+:`AD_OUT_MOVE_COMMAND_VALID_WIDTH]=!pending&&offered&&move_command; + mask[`AD_OUT_MOVE_COMMAND_VALID_OFFSET+:`AD_OUT_MOVE_COMMAND_VALID_WIDTH] = '1; + + want[`AD_OUT_MOVE_COMMAND_BITS_ROWS_OFFSET+:`AD_OUT_MOVE_COMMAND_BITS_ROWS_WIDTH]={1'b0,t.inputs[`AD_IN_EXECUTION_COMMAND_BITS_INSTRUCTION_RS2DATA_OFFSET+32+:16]}+17'd1; + mask[`AD_OUT_MOVE_COMMAND_BITS_ROWS_OFFSET+:`AD_OUT_MOVE_COMMAND_BITS_ROWS_WIDTH] = '1; + + want[`AD_OUT_MOVE_COMPLETION_READY_OFFSET+:`AD_OUT_MOVE_COMPLETION_READY_WIDTH]=pending&&moving&&!work_done; + mask[`AD_OUT_MOVE_COMPLETION_READY_OFFSET+:`AD_OUT_MOVE_COMPLETION_READY_WIDTH] = '1; + + want[`AD_OUT_TASKRESPONSE_READY_OFFSET+:`AD_OUT_TASKRESPONSE_READY_WIDTH]=pending&&!moving&&!work_done&&(!has_result|| + `AI(t, EXECUTION_RESPONSE_READY) + ); + mask[`AD_OUT_TASKRESPONSE_READY_OFFSET+:`AD_OUT_TASKRESPONSE_READY_WIDTH] = '1; + + want[`AD_OUT_EXECUTION_RESPONSE_VALID_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_VALID_WIDTH]=pending&&has_result&&!response_done&&(moving?work_done: + `AI(t, TASKRESPONSE_VALID) + ); + mask[`AD_OUT_EXECUTION_RESPONSE_VALID_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_VALID_WIDTH] = '1; + + if (want[`AD_OUT_EXECUTION_RESPONSE_VALID_OFFSET]) begin + want[`AD_OUT_EXECUTION_RESPONSE_BITS_RD_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_RD_WIDTH]=moving?destination:`AI( + t, TASKRESPONSE_BITS_RD); + mask[`AD_OUT_EXECUTION_RESPONSE_BITS_RD_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_RD_WIDTH]='1; + + want[`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_WIDTH]=moving?64'(move_error):`AI( + t, TASKRESPONSE_BITS_DATA); + mask[`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_WIDTH]='1; + + end + want[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_VALID_WIDTH]=pending&&work_done&&(!has_result||response_done); + mask[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_VALID_WIDTH] = '1; + + if (want[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET]) begin + want[`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_WIDTH]=saved_tag; + mask[`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_WIDTH]='1; + end + want[`AD_OUT_MEMORYALLOWED_OFFSET+:`AD_OUT_MEMORYALLOWED_WIDTH]=!(offered&&move_command||pending&&(moving||work_done)); + mask[`AD_OUT_MEMORYALLOWED_OFFSET+:`AD_OUT_MEMORYALLOWED_WIDTH] = '1; + + want[`AD_OUT_IDLE_OFFSET+:`AD_OUT_IDLE_WIDTH] = !pending; + mask[`AD_OUT_IDLE_OFFSET+:`AD_OUT_IDLE_WIDTH] = '1; + + + end + foreach (copies[field]) + for (int b = 0; b < copies[field][2]; b++) + if (want[copies[field][3]]) begin + want[copies[field][1]+b] = t.inputs[copies[field][0]+b]; + mask[copies[field][1]+b] = 1; + end + predicted = result_item::type_id::create("expected"); + received = result_item::type_id::create("actual"); + predicted.value = want & mask; + received.value = t.outputs & mask; + expected.write(predicted); + actual.write(received); + begin + logic [31:0] instruction; + bit task_command, move_command; + instruction = `AI(t, EXECUTION_COMMAND_BITS_INSTRUCTION_RAW_INST); + task_command = instruction[6:0] == 'h2b; + move_command = instruction[6:0] == 'h7b && instruction[31:25] == 'h0d; + if (`AI(t, EXECUTION_COMMAND_VALID) && `AO(t, EXECUTION_COMMAND_READY)) begin + pending = 1; + moving = move_command; + has_result = instruction[14]; + destination = instruction[11:7]; + saved_tag = `AI(t, EXECUTION_COMMAND_BITS_TAG); + work_done = 0; + response_done = 0; + behavior.sample(move_command, 0, 0); + end + if (`AI(t, TASKRESPONSE_VALID) && `AO(t, TASKRESPONSE_READY)) work_done = 1; + if (`AI(t, MOVE_COMPLETION_VALID) && `AO(t, MOVE_COMPLETION_READY)) begin + work_done = 1; + move_error = `AI(t, MOVE_COMPLETION_BITS); + end + if (`AO(t, EXECUTION_RESPONSE_VALID) && `AI(t, EXECUTION_RESPONSE_READY)) response_done = 1; + if (`AO(t, EXECUTION_COMPLETE_VALID) && `AI(t, EXECUTION_COMPLETE_READY)) pending = 0; + if ( + `AI(t, EXECUTION_COMMAND_VALID) + && ! + `AO(t, EXECUTION_COMMAND_READY) + || + `AO(t, EXECUTION_COMPLETE_VALID) + && ! + `AI(t, EXECUTION_COMPLETE_READY) + || + `AO(t, EXECUTION_RESPONSE_VALID) + && ! + `AI(t, EXECUTION_RESPONSE_READY) + ) + behavior.sample(moving, 1, 0); + + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + if (pending || tags.size() != 0) `uvm_fatal("PENDING", "Admission ownership did not drain") + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/filelist.f b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/filelist.f new file mode 100644 index 000000000..9b229b6fb --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@/CommandAdmission ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/../common/interface.sv +@RESOURCES@/package.sv +-F @RTL@/CommandAdmission/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/package.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/package.sv new file mode 100644 index 000000000..3793ede46 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/package.sv @@ -0,0 +1,17 @@ +`include "parameters.svh" +`define AI(T, F) T.inputs[`AD_IN_``F``_OFFSET+:`AD_IN_``F``_WIDTH] +`define AO(T, F) T.outputs[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH] +`define EXPECT(F, + V) want[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]=V;mask[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]='1; +package admission_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "../common/agent/item.svh" + `include "../common/agent/driver.svh" + `include "../common/agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/seq/sequence.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/seq/sequence.svh new file mode 100644 index 000000000..87ff49a49 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/seq/sequence.svh @@ -0,0 +1,88 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + logic [ `AD_INPUT_BITS-1:0] pins; + logic [`AD_OUTPUT_BITS-1:0] observed; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task command(int tag, logic [31:0] instruction, logic [63:0] rs1, logic [63:0] rs2); + for (int b = 0; b < `AD_COMMAND_WIDTH; b++) + pins[`AD_COMMAND_OFFSET+b] = ((b + tag) % 3 == 0) ^ ((b + tag) % 7 == 0); + pins[`AD_IN_EXECUTION_COMMAND_BITS_INSTRUCTION_RAW_INST_OFFSET+:32]=instruction|((tag%32)<<7); + pins[`AD_IN_EXECUTION_COMMAND_BITS_INSTRUCTION_RS1DATA_OFFSET+:64] = rs1; + pins[`AD_IN_EXECUTION_COMMAND_BITS_INSTRUCTION_RS2DATA_OFFSET+:64] = rs2; + pins[`AD_IN_EXECUTION_COMMAND_BITS_TAG_OFFSET+:`AD_IN_EXECUTION_COMMAND_BITS_TAG_WIDTH] = tag; + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 1; + endtask + task accepted(); + do tick(); while (!observed[`AD_OUT_EXECUTION_COMMAND_READY_OFFSET]); + endtask + task body(); + string negative; + pins = '0; + repeat (3) tick(1); + tick(); + if ($value$plusargs("CASE=%s", negative)) begin + command(0, 0, 0, 0); + repeat (8) tick(); + `uvm_fatal("MISSING_ASSERTION", "Expected admission assertion did not fire") + end + for (int i = 0; i < 48; i++) begin + bit moving = i % 2; + bit result = i % 3 != 0; + command(i, moving ? (32'h1a00007b | (result << 14)) : (32'h2b | (result << 14)), + i * 64'hfedcba987654321, ~(i * 64'h123456789abcdef)); + pins[`AD_IN_TASKREQUEST_READY_OFFSET] = 0; + pins[`AD_IN_MOVE_COMMAND_READY_OFFSET] = 0; + repeat (3) tick(); + pins[`AD_IN_TASKREQUEST_READY_OFFSET] = 1; + pins[`AD_IN_MOVE_COMMAND_READY_OFFSET] = 1; + accepted(); + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 0; + pins[`AD_IN_EXECUTION_RESPONSE_READY_OFFSET] = 0; + pins[`AD_IN_EXECUTION_COMPLETE_READY_OFFSET] = 0; + repeat (2) tick(); + if (moving) begin + pins[`AD_IN_MOVE_COMPLETION_BITS_OFFSET] = i % 4 >= 2; + pins[`AD_IN_MOVE_COMPLETION_VALID_OFFSET] = 1; + do tick(); while (!observed[`AD_OUT_MOVE_COMPLETION_READY_OFFSET]); + pins[`AD_IN_MOVE_COMPLETION_VALID_OFFSET] = 0; + if (result) begin + repeat (3) tick(); + pins[`AD_IN_EXECUTION_RESPONSE_READY_OFFSET] = 1; + tick(); + end + end else begin + pins[`AD_IN_TASKRESPONSE_BITS_RD_OFFSET+:5] = 5'(31 - i); + pins[`AD_IN_TASKRESPONSE_BITS_DATA_OFFSET+:64] = 64'hcafe000000000000 + i; + pins[`AD_IN_TASKRESPONSE_VALID_OFFSET] = 1; + if (result) begin + repeat (3) tick(); + pins[`AD_IN_EXECUTION_RESPONSE_READY_OFFSET] = 1; + end + do tick(); while (!observed[`AD_OUT_TASKRESPONSE_READY_OFFSET]); + pins[`AD_IN_TASKRESPONSE_VALID_OFFSET] = 0; + end + repeat (3) tick(); + pins[`AD_IN_EXECUTION_COMPLETE_READY_OFFSET] = 1; + do tick(); while (!observed[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET]); + tick(); + end + command(250, 32'h402b, '1, '1); + accepted(); + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 0; + repeat (3) tick(); + repeat (2) tick(1); + pins = '0; + tick(); + repeat (4) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/tb.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/tb.sv new file mode 100644 index 000000000..dbe564378 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/tb.sv @@ -0,0 +1,15 @@ +module admission_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + admission_if bus (clock); + CommandAdmission dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual admission_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/tests/test.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/command/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/driver.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/driver.svh new file mode 100644 index 000000000..ade23d264 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual admission_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual admission_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Admission interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.inputs = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/item.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/item.svh new file mode 100644 index 000000000..f287d532a --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`AD_INPUT_BITS-1:0] inputs; + logic [`AD_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_object_utils_end + logic [`AD_OUTPUT_BITS-1:0] value; + function new(string name = "result"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/monitor.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/monitor.svh new file mode 100644 index 000000000..57dedf7d8 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual admission_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual admission_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Admission monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/interface.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/interface.sv new file mode 100644 index 000000000..4933b0aab --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/common/interface.sv @@ -0,0 +1,8 @@ +`include "parameters.svh" +interface admission_if ( + input logic clock +); + logic reset = 1; + logic [`AD_INPUT_BITS-1:0] inputs = '0; + wire [`AD_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/env/env.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/env/env.svh new file mode 100644 index 000000000..1f8748f8e --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/env/model.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/env/model.svh new file mode 100644 index 000000000..de6209c06 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/env/model.svh @@ -0,0 +1,143 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(result_item) expected, actual; + const int copies[`AD_PAYLOAD_FIELDS][4] = `AD_PAYLOAD_MAP; + bit pending, moving, has_result, work_done, response_done, move_error, remote_owner; + int saved_tag, destination, tags[$]; + covergroup behavior with function sample (int route, bit stalled, bit cancelled); + option.per_instance = 1; + routes: coverpoint route {bins all[] = {[0 : 1]};} + backpressure: coverpoint stalled; + reset_cancellation: coverpoint cancelled; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + behavior = new(); + endfunction + function void write(cycle_item t); + logic [`AD_OUTPUT_BITS-1:0] want, mask; + result_item predicted, received; + + if (t.reset) begin + behavior.sample(0, 0, pending || tags.size() != 0); + pending = 0; + tags.delete(); + return; + end + want = '0; + mask = '0; + begin + bit management, available; + logic [31:0] instruction; + instruction = `AI(t, EXECUTION_COMMAND_BITS_INSTRUCTION_RAW_INST); + management = instruction[6:0] == 'h2b; + available = !pending && ! + `AI(t, REMOTE_BUSY) + && tags.size() < `AD_ENTRIES && (!management || (! + `AI(t, ACCELERATOR_BUSY) + && tags.size() == 0)); + want[`AD_OUT_EXECUTION_COMMAND_READY_OFFSET+:`AD_OUT_EXECUTION_COMMAND_READY_WIDTH]=available&&(management? + `AI(t, REMOTE_REQUEST_READY) + : + `AI(t, ACCELERATOR_CMD_READY) + ); + mask[`AD_OUT_EXECUTION_COMMAND_READY_OFFSET+:`AD_OUT_EXECUTION_COMMAND_READY_WIDTH] = '1; + + want[`AD_OUT_ACCELERATOR_CMD_VALID_OFFSET+:`AD_OUT_ACCELERATOR_CMD_VALID_WIDTH] = + `AI(t, EXECUTION_COMMAND_VALID) + && available && !management; + mask[`AD_OUT_ACCELERATOR_CMD_VALID_OFFSET+:`AD_OUT_ACCELERATOR_CMD_VALID_WIDTH] = '1; + + want[`AD_OUT_REMOTE_REQUEST_VALID_OFFSET+:`AD_OUT_REMOTE_REQUEST_VALID_WIDTH] = + `AI(t, EXECUTION_COMMAND_VALID) + && available && management; + mask[`AD_OUT_REMOTE_REQUEST_VALID_OFFSET+:`AD_OUT_REMOTE_REQUEST_VALID_WIDTH] = '1; + + want[`AD_OUT_ACCELERATOR_RESP_READY_OFFSET+:`AD_OUT_ACCELERATOR_RESP_READY_WIDTH] = 0; + mask[`AD_OUT_ACCELERATOR_RESP_READY_OFFSET+:`AD_OUT_ACCELERATOR_RESP_READY_WIDTH] = '1; + + want[`AD_OUT_EXECUTION_RESPONSE_VALID_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_VALID_WIDTH]=pending&&!work_done&&has_result&&`AI( + t, REMOTE_REPLY_VALID); + mask[`AD_OUT_EXECUTION_RESPONSE_VALID_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_VALID_WIDTH] = '1; + + if (want[`AD_OUT_EXECUTION_RESPONSE_VALID_OFFSET]) begin + want[`AD_OUT_EXECUTION_RESPONSE_BITS_RD_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_RD_WIDTH]=destination; + mask[`AD_OUT_EXECUTION_RESPONSE_BITS_RD_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_RD_WIDTH]='1; + want[`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_WIDTH]=`AI( + t, REMOTE_REPLY_BITS); + mask[`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_OFFSET+:`AD_OUT_EXECUTION_RESPONSE_BITS_DATA_WIDTH]='1; + end + want[`AD_OUT_REMOTE_REPLY_READY_OFFSET+:`AD_OUT_REMOTE_REPLY_READY_WIDTH]=pending&&!work_done&&(!has_result|| + `AI(t, EXECUTION_RESPONSE_READY) + ); + mask[`AD_OUT_REMOTE_REPLY_READY_OFFSET+:`AD_OUT_REMOTE_REPLY_READY_WIDTH] = '1; + + want[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_VALID_WIDTH]=tags.size()!=0&&(pending?work_done:! + `AI(t, ACCELERATOR_BUSY) + ); + mask[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_VALID_WIDTH] = '1; + + if (want[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET]) begin + want[`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_WIDTH]=tags[0]; + mask[`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_OFFSET+:`AD_OUT_EXECUTION_COMPLETE_BITS_TAG_WIDTH]='1; + end + want[`AD_OUT_DRAINED_OFFSET+:`AD_OUT_DRAINED_WIDTH] = !pending && tags.size() == 0 && ! + `AI(t, ACCELERATOR_BUSY) + && ! + `AI(t, REMOTE_BUSY); + mask[`AD_OUT_DRAINED_OFFSET+:`AD_OUT_DRAINED_WIDTH] = '1; + + + end + foreach (copies[field]) + for (int b = 0; b < copies[field][2]; b++) + if (want[copies[field][3]]) begin + want[copies[field][1]+b] = t.inputs[copies[field][0]+b]; + mask[copies[field][1]+b] = 1; + end + predicted = result_item::type_id::create("expected"); + received = result_item::type_id::create("actual"); + predicted.value = want & mask; + received.value = t.outputs & mask; + expected.write(predicted); + actual.write(received); + begin + logic [31:0] instruction; + bit management; + instruction = `AI(t, EXECUTION_COMMAND_BITS_INSTRUCTION_RAW_INST); + management = instruction[6:0] == 'h2b; + if (`AO(t, EXECUTION_COMPLETE_VALID) && `AI(t, EXECUTION_COMPLETE_READY)) begin + int tag = tags.pop_front(); + if (pending) pending = 0; + end + if (`AI(t, EXECUTION_COMMAND_VALID) && `AO(t, EXECUTION_COMMAND_READY)) begin + tags.push_back(`AI(t, EXECUTION_COMMAND_BITS_TAG)); + if (management) begin + pending = 1; + work_done = 0; + has_result = instruction[14]; + destination = instruction[11:7]; + end + behavior.sample(management, 0, 0); + end + if (`AI(t, REMOTE_REPLY_VALID) && `AO(t, REMOTE_REPLY_READY)) work_done = 1; + if ( + `AI(t, EXECUTION_COMMAND_VALID) + && ! + `AO(t, EXECUTION_COMMAND_READY) + || + `AO(t, EXECUTION_COMPLETE_VALID) + && ! + `AI(t, EXECUTION_COMPLETE_READY) + ) + behavior.sample(management, 1, 0); + + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + if (pending || tags.size() != 0) `uvm_fatal("PENDING", "Admission ownership did not drain") + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/filelist.f b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/filelist.f new file mode 100644 index 000000000..a5bedca99 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@/DirectAdmission ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/../common/interface.sv +@RESOURCES@/package.sv +-F @RTL@/DirectAdmission/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/package.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/package.sv new file mode 100644 index 000000000..3793ede46 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/package.sv @@ -0,0 +1,17 @@ +`include "parameters.svh" +`define AI(T, F) T.inputs[`AD_IN_``F``_OFFSET+:`AD_IN_``F``_WIDTH] +`define AO(T, F) T.outputs[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH] +`define EXPECT(F, + V) want[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]=V;mask[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]='1; +package admission_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "../common/agent/item.svh" + `include "../common/agent/driver.svh" + `include "../common/agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/seq/sequence.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/seq/sequence.svh new file mode 100644 index 000000000..5f04c9a41 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/seq/sequence.svh @@ -0,0 +1,92 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + logic [ `AD_INPUT_BITS-1:0] pins; + logic [`AD_OUTPUT_BITS-1:0] observed; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task command(int tag, logic [31:0] instruction, logic [63:0] rs1, logic [63:0] rs2); + for (int b = 0; b < `AD_COMMAND_WIDTH; b++) + pins[`AD_COMMAND_OFFSET+b] = ((b + tag) % 3 == 0) ^ ((b + tag) % 7 == 0); + pins[`AD_IN_EXECUTION_COMMAND_BITS_INSTRUCTION_RAW_INST_OFFSET+:32]=instruction|((tag%32)<<7); + pins[`AD_IN_EXECUTION_COMMAND_BITS_INSTRUCTION_RS1DATA_OFFSET+:64] = rs1; + pins[`AD_IN_EXECUTION_COMMAND_BITS_INSTRUCTION_RS2DATA_OFFSET+:64] = rs2; + pins[`AD_IN_EXECUTION_COMMAND_BITS_TAG_OFFSET+:`AD_IN_EXECUTION_COMMAND_BITS_TAG_WIDTH] = tag; + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 1; + endtask + task accepted(); + do tick(); while (!observed[`AD_OUT_EXECUTION_COMMAND_READY_OFFSET]); + endtask + task body(); + string negative; + pins = '0; + repeat (3) tick(1); + tick(); + if ($value$plusargs("CASE=%s", negative)) begin + pins[`AD_IN_ACCELERATOR_CMD_READY_OFFSET] = 1; + if (negative == "reply") pins[`AD_IN_ACCELERATOR_RESP_VALID_OFFSET] = 1; + else if (negative == "interrupt") pins[`AD_IN_ACCELERATOR_INTERRUPT_OFFSET] = 1; + else command(0, 32'h407b, 0, 0); + repeat (8) tick(); + `uvm_fatal("MISSING_ASSERTION", "Expected admission assertion did not fire") + end + pins[`AD_IN_ACCELERATOR_CMD_READY_OFFSET] = 1; + pins[`AD_IN_EXECUTION_COMPLETE_READY_OFFSET] = 0; + pins[`AD_IN_ACCELERATOR_BUSY_OFFSET] = 1; + for (int i = 0; i < `AD_ENTRIES; i++) begin + command(i, 32'h7b, i * 64'h123456789abcdef, ~64'(i)); + accepted(); + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 0; + end + command(100, 32'h7b, '1, '1); + repeat (5) tick(); + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 0; + pins[`AD_IN_ACCELERATOR_BUSY_OFFSET] = 0; + pins[`AD_IN_EXECUTION_COMPLETE_READY_OFFSET] = 1; + do tick(); while (!observed[`AD_OUT_DRAINED_OFFSET]); + for (int i = 0; i < 32; i++) begin + bit result = i % 2; + pins[`AD_IN_REMOTE_BUSY_OFFSET] = 1; + command(16 + i, (32'((13 + i % 20) << 25) | 32'h2b | (result << 14)), i * 64'h123456789abcdef, + ~64'(i)); + repeat (3) tick(); + pins[`AD_IN_REMOTE_BUSY_OFFSET] = 0; + pins[`AD_IN_REMOTE_REQUEST_READY_OFFSET] = 0; + repeat (3) tick(); + pins[`AD_IN_REMOTE_REQUEST_READY_OFFSET] = 1; + accepted(); + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 0; + pins[`AD_IN_EXECUTION_RESPONSE_READY_OFFSET] = 0; + pins[`AD_IN_EXECUTION_COMPLETE_READY_OFFSET] = 0; + pins[`AD_IN_REMOTE_REPLY_BITS_OFFSET+:64] = 64'h1234000056780000 + i; + pins[`AD_IN_REMOTE_REPLY_VALID_OFFSET] = 1; + if (result) begin + repeat (3) tick(); + pins[`AD_IN_EXECUTION_RESPONSE_READY_OFFSET] = 1; + end + do tick(); while (!observed[`AD_OUT_REMOTE_REPLY_READY_OFFSET]); + pins[`AD_IN_REMOTE_REPLY_VALID_OFFSET] = 0; + repeat (3) tick(); + pins[`AD_IN_EXECUTION_COMPLETE_READY_OFFSET] = 1; + do tick(); while (!observed[`AD_OUT_EXECUTION_COMPLETE_VALID_OFFSET]); + tick(); + end + pins[`AD_IN_EXECUTION_COMPLETE_READY_OFFSET] = 0; + command(200, 32'h7b, '1, '1); + accepted(); + pins[`AD_IN_EXECUTION_COMMAND_VALID_OFFSET] = 0; + repeat (3) tick(); + repeat (2) tick(1); + pins = '0; + tick(); + repeat (4) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/tb.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/tb.sv new file mode 100644 index 000000000..ecf85aee1 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/tb.sv @@ -0,0 +1,15 @@ +module admission_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + admission_if bus (clock); + DirectAdmission dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual admission_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/tests/test.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/direct/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/env/env.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/env/env.svh new file mode 100644 index 000000000..1f8748f8e --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/env/model.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/env/model.svh new file mode 100644 index 000000000..69aee0888 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/env/model.svh @@ -0,0 +1,86 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(result_item) expected, actual; + const int copies[`AD_PAYLOAD_FIELDS][4] = `AD_PAYLOAD_MAP; + bit pending, moving, has_result, work_done, response_done, move_error, remote_owner; + int saved_tag, destination, tags[$]; + covergroup behavior with function sample (int route, bit stalled, bit cancelled); + option.per_instance = 1; + routes: coverpoint route {bins all[] = {[0 : 0]};} + backpressure: coverpoint stalled; + reset_cancellation: coverpoint cancelled; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + behavior = new(); + endfunction + function void write(cycle_item t); + logic [`AD_OUTPUT_BITS-1:0] want, mask; + result_item predicted, received; + + if (t.reset) begin + behavior.sample(0, 0, pending || tags.size() != 0); + pending = 0; + tags.delete(); + return; + end + want = '0; + mask = '0; + begin + want[`AD_OUT_MOVE_COMMAND_VALID_OFFSET+:`AD_OUT_MOVE_COMMAND_VALID_WIDTH] = + `AI(t, ROCC_CMD_VALID) + && !pending; + mask[`AD_OUT_MOVE_COMMAND_VALID_OFFSET+:`AD_OUT_MOVE_COMMAND_VALID_WIDTH] = '1; + + want[`AD_OUT_ROCC_CMD_READY_OFFSET+:`AD_OUT_ROCC_CMD_READY_WIDTH] = + `AI(t, MOVE_COMMAND_READY) + && !pending; + mask[`AD_OUT_ROCC_CMD_READY_OFFSET+:`AD_OUT_ROCC_CMD_READY_WIDTH] = '1; + + want[`AD_OUT_MOVE_COMMAND_BITS_ROWS_OFFSET+:`AD_OUT_MOVE_COMMAND_BITS_ROWS_WIDTH]= + {1'b0, t.inputs[`AD_IN_ROCC_CMD_BITS_INSTRUCTION_RS2DATA_OFFSET+32+:16]} + 17'd1; + mask[`AD_OUT_MOVE_COMMAND_BITS_ROWS_OFFSET+:`AD_OUT_MOVE_COMMAND_BITS_ROWS_WIDTH] = '1; + + want[`AD_OUT_MOVE_COMPLETION_READY_OFFSET+:`AD_OUT_MOVE_COMPLETION_READY_WIDTH] = pending; + mask[`AD_OUT_MOVE_COMPLETION_READY_OFFSET+:`AD_OUT_MOVE_COMPLETION_READY_WIDTH] = '1; + + want[`AD_OUT_ROCC_BUSY_OFFSET+:`AD_OUT_ROCC_BUSY_WIDTH] = pending; + mask[`AD_OUT_ROCC_BUSY_OFFSET+:`AD_OUT_ROCC_BUSY_WIDTH] = '1; + + want[`AD_OUT_ROCC_INTERRUPT_OFFSET+:`AD_OUT_ROCC_INTERRUPT_WIDTH] = 0; + mask[`AD_OUT_ROCC_INTERRUPT_OFFSET+:`AD_OUT_ROCC_INTERRUPT_WIDTH] = '1; + + want[`AD_OUT_ROCC_RESP_VALID_OFFSET+:`AD_OUT_ROCC_RESP_VALID_WIDTH] = 0; + mask[`AD_OUT_ROCC_RESP_VALID_OFFSET+:`AD_OUT_ROCC_RESP_VALID_WIDTH] = '1; + + + end + foreach (copies[field]) + for (int b = 0; b < copies[field][2]; b++) + if (want[copies[field][3]]) begin + want[copies[field][1]+b] = t.inputs[copies[field][0]+b]; + mask[copies[field][1]+b] = 1; + end + predicted = result_item::type_id::create("expected"); + received = result_item::type_id::create("actual"); + predicted.value = want & mask; + received.value = t.outputs & mask; + expected.write(predicted); + actual.write(received); + begin + if (`AI(t, ROCC_CMD_VALID) && `AO(t, ROCC_CMD_READY)) begin + pending = 1; + behavior.sample(0, 0, 0); + end + if (`AI(t, MOVE_COMPLETION_VALID) && `AO(t, MOVE_COMPLETION_READY)) pending = 0; + if (`AI(t, ROCC_CMD_VALID) && !`AO(t, ROCC_CMD_READY)) behavior.sample(0, 1, 0); + + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + if (pending || tags.size() != 0) `uvm_fatal("PENDING", "Admission ownership did not drain") + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/filelist.f b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/filelist.f new file mode 100644 index 000000000..deff3ae10 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@/MoveAdmission ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/../common/interface.sv +@RESOURCES@/package.sv +-F @RTL@/MoveAdmission/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/package.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/package.sv new file mode 100644 index 000000000..3793ede46 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/package.sv @@ -0,0 +1,17 @@ +`include "parameters.svh" +`define AI(T, F) T.inputs[`AD_IN_``F``_OFFSET+:`AD_IN_``F``_WIDTH] +`define AO(T, F) T.outputs[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH] +`define EXPECT(F, + V) want[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]=V;mask[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]='1; +package admission_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "../common/agent/item.svh" + `include "../common/agent/driver.svh" + `include "../common/agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/seq/sequence.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/seq/sequence.svh new file mode 100644 index 000000000..e5976bf84 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/seq/sequence.svh @@ -0,0 +1,61 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + logic [ `AD_INPUT_BITS-1:0] pins; + logic [`AD_OUTPUT_BITS-1:0] observed; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task command(int tag, logic [31:0] instruction, logic [63:0] rs1, logic [63:0] rs2); + for (int b = 0; b < `AD_COMMAND_WIDTH; b++) + pins[`AD_COMMAND_OFFSET+b] = ((b + tag) % 3 == 0) ^ ((b + tag) % 7 == 0); + pins[`AD_IN_ROCC_CMD_BITS_INSTRUCTION_RAW_INST_OFFSET+:32] = instruction | ((tag % 32) << 7); + pins[`AD_IN_ROCC_CMD_BITS_INSTRUCTION_RS1DATA_OFFSET+:64] = rs1; + pins[`AD_IN_ROCC_CMD_BITS_INSTRUCTION_RS2DATA_OFFSET+:64] = rs2; + pins[`AD_IN_ROCC_CMD_VALID_OFFSET] = 1; + endtask + task accepted(); + do tick(); while (!observed[`AD_OUT_ROCC_CMD_READY_OFFSET]); + endtask + task body(); + string negative; + pins = '0; + repeat (3) tick(1); + tick(); + if ($value$plusargs("CASE=%s", negative)) begin + command(0, 32'h1a00707b, 0, 0); + repeat (8) tick(); + `uvm_fatal("MISSING_ASSERTION", "Expected admission assertion did not fire") + end + for (int i = 0; i < 48; i++) begin + command(i, 32'h1a00307b, i * 64'hfedcba987654321, + i == 47 ? 64'h0000ffff12345678 : ~(i * 64'h123456789abcdef)); + pins[`AD_IN_MOVE_COMMAND_READY_OFFSET] = 0; + repeat (3) tick(); + pins[`AD_IN_MOVE_COMMAND_READY_OFFSET] = 1; + accepted(); + pins[`AD_IN_ROCC_CMD_VALID_OFFSET] = 0; + repeat (5) tick(); + pins[`AD_IN_MOVE_COMPLETION_BITS_OFFSET] = i % 2; + pins[`AD_IN_MOVE_COMPLETION_VALID_OFFSET] = 1; + do tick(); while (!observed[`AD_OUT_MOVE_COMPLETION_READY_OFFSET]); + pins[`AD_IN_MOVE_COMPLETION_VALID_OFFSET] = 0; + tick(); + end + command(0, 32'h1a00307b, '1, '1); + accepted(); + pins[`AD_IN_ROCC_CMD_VALID_OFFSET] = 0; + repeat (3) tick(); + repeat (2) tick(1); + pins = '0; + tick(); + repeat (4) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/tb.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/tb.sv new file mode 100644 index 000000000..a5ff595c4 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/tb.sv @@ -0,0 +1,15 @@ +module admission_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + admission_if bus (clock); + MoveAdmission dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual admission_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/tests/test.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/move/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/suite.toml new file mode 100644 index 000000000..6225286ac --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/suite.toml @@ -0,0 +1,156 @@ +[model] +manifest="" +library="" + +[[tests]] +name="command" +top="admission_tb" +test="protocol_test" +filelist="command/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["admission_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" + +[[tests]] +name="move" +top="admission_tb" +test="protocol_test" +filelist="move/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["admission_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" + +[[tests]] +name="direct" +top="admission_tb" +test="protocol_test" +filelist="direct/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["admission_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" + +[[tests]] +name="task" +top="admission_tb" +test="protocol_test" +filelist="task/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["admission_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" + +[[tests]] +name="command_bad_unsupported" +top="admission_tb" +test="protocol_test" +filelist="command/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=unsupported"] +assertion="CPU-only controller received an unsupported compute instruction" + +[[tests]] +name="move_bad_posted" +top="admission_tb" +test="protocol_test" +filelist="move/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=posted"] +assertion="Move admission only accepts posted bank moves" + +[[tests]] +name="direct_bad_unsupported" +top="admission_tb" +test="protocol_test" +filelist="direct/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=unsupported"] +assertion="Direct core received an unsupported command" + +[[tests]] +name="direct_bad_reply" +top="admission_tb" +test="protocol_test" +filelist="direct/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=reply"] +assertion="Direct accelerator returned an unexpected register result" + +[[tests]] +name="direct_bad_interrupt" +top="admission_tb" +test="protocol_test" +filelist="direct/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=interrupt"] +assertion="Direct accelerator fault" diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/env/env.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/env/env.svh new file mode 100644 index 000000000..1f8748f8e --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/env/model.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/env/model.svh new file mode 100644 index 000000000..440cf8030 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/env/model.svh @@ -0,0 +1,126 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(result_item) expected, actual; + const int copies[`AD_PAYLOAD_FIELDS][4] = `AD_PAYLOAD_MAP; + bit pending, moving, has_result, work_done, response_done, move_error, remote_owner; + int saved_tag, destination, tags[$]; + covergroup behavior with function sample (int route, bit stalled, bit cancelled); + option.per_instance = 1; + routes: coverpoint route {bins all[] = {[0 : 1]};} + backpressure: coverpoint stalled; + reset_cancellation: coverpoint cancelled; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + behavior = new(); + endfunction + function void write(cycle_item t); + logic [`AD_OUTPUT_BITS-1:0] want, mask; + result_item predicted, received; + + if (t.reset) begin + behavior.sample(0, 0, pending || tags.size() != 0); + pending = 0; + tags.delete(); + return; + end + want = '0; + mask = '0; + begin + bit remote_command; + logic [31:0] instruction; + instruction = `AI(t, COMMAND_BITS_INSTRUCTION_RAW_INST); + remote_command = instruction[31:25] >= 13; + want[`AD_OUT_COMMAND_READY_OFFSET+:`AD_OUT_COMMAND_READY_WIDTH]=!pending&&(remote_command? + `AI(t, REMOTE_REQUEST_READY) + : + `AI(t, LOCAL_REQUEST_READY) + ); + mask[`AD_OUT_COMMAND_READY_OFFSET+:`AD_OUT_COMMAND_READY_WIDTH] = '1; + + want[`AD_OUT_LOCAL_REQUEST_VALID_OFFSET+:`AD_OUT_LOCAL_REQUEST_VALID_WIDTH] = + `AI(t, COMMAND_VALID) + && !pending && !remote_command; + mask[`AD_OUT_LOCAL_REQUEST_VALID_OFFSET+:`AD_OUT_LOCAL_REQUEST_VALID_WIDTH] = '1; + + want[`AD_OUT_REMOTE_REQUEST_VALID_OFFSET+:`AD_OUT_REMOTE_REQUEST_VALID_WIDTH] = + `AI(t, COMMAND_VALID) + && !pending && remote_command; + mask[`AD_OUT_REMOTE_REQUEST_VALID_OFFSET+:`AD_OUT_REMOTE_REQUEST_VALID_WIDTH] = '1; + + want[`AD_OUT_RESPONSE_VALID_OFFSET+:`AD_OUT_RESPONSE_VALID_WIDTH] = pending && (remote_owner ? + `AI(t, REMOTE_REPLY_VALID) + : + `AI(t, LOCAL_REPLY_VALID) + ); + mask[`AD_OUT_RESPONSE_VALID_OFFSET+:`AD_OUT_RESPONSE_VALID_WIDTH] = '1; + + if (want[`AD_OUT_RESPONSE_VALID_OFFSET]) begin + want[`AD_OUT_RESPONSE_BITS_RD_OFFSET+:`AD_OUT_RESPONSE_BITS_RD_WIDTH] = destination; + mask[`AD_OUT_RESPONSE_BITS_RD_OFFSET+:`AD_OUT_RESPONSE_BITS_RD_WIDTH] = '1; + want[`AD_OUT_RESPONSE_BITS_DATA_OFFSET+:`AD_OUT_RESPONSE_BITS_DATA_WIDTH] = remote_owner ? + `AI(t, REMOTE_REPLY_BITS) + : + `AI(t, LOCAL_REPLY_BITS); + mask[`AD_OUT_RESPONSE_BITS_DATA_OFFSET+:`AD_OUT_RESPONSE_BITS_DATA_WIDTH] = '1; + end + want[`AD_OUT_LOCAL_REPLY_READY_OFFSET+:`AD_OUT_LOCAL_REPLY_READY_WIDTH]=pending&&!remote_owner&&`AI( + t, RESPONSE_READY); + mask[`AD_OUT_LOCAL_REPLY_READY_OFFSET+:`AD_OUT_LOCAL_REPLY_READY_WIDTH] = '1; + + want[`AD_OUT_REMOTE_REPLY_READY_OFFSET+:`AD_OUT_REMOTE_REPLY_READY_WIDTH]=pending&&remote_owner&&`AI( + t, RESPONSE_READY); + mask[`AD_OUT_REMOTE_REPLY_READY_OFFSET+:`AD_OUT_REMOTE_REPLY_READY_WIDTH] = '1; + + want[`AD_OUT_BUSY_OFFSET+:`AD_OUT_BUSY_WIDTH] = pending || + `AI(t, LOCAL_BUSY) + || + `AI(t, REMOTE_BUSY); + mask[`AD_OUT_BUSY_OFFSET+:`AD_OUT_BUSY_WIDTH] = '1; + + + end + foreach (copies[field]) + for (int b = 0; b < copies[field][2]; b++) + if (want[copies[field][3]]) begin + want[copies[field][1]+b] = t.inputs[copies[field][0]+b]; + mask[copies[field][1]+b] = 1; + end + predicted = result_item::type_id::create("expected"); + received = result_item::type_id::create("actual"); + predicted.value = want & mask; + received.value = t.outputs & mask; + expected.write(predicted); + actual.write(received); + begin + logic [31:0] instruction; + bit remote_command; + instruction = `AI(t, COMMAND_BITS_INSTRUCTION_RAW_INST); + remote_command = instruction[31:25] >= 13; + if (`AI(t, COMMAND_VALID) && `AO(t, COMMAND_READY)) begin + pending = 1; + remote_owner = remote_command; + destination = instruction[11:7]; + behavior.sample(remote_command, 0, 0); + end + if (`AO(t, RESPONSE_VALID) && `AI(t, RESPONSE_READY)) pending = 0; + if ( + `AI(t, COMMAND_VALID) + && ! + `AO(t, COMMAND_READY) + || + `AO(t, RESPONSE_VALID) + && ! + `AI(t, RESPONSE_READY) + ) + behavior.sample(remote_owner, 1, 0); + + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + if (pending || tags.size() != 0) `uvm_fatal("PENDING", "Admission ownership did not drain") + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/filelist.f b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/filelist.f new file mode 100644 index 000000000..181f872a3 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@/TaskRouter ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/../common/interface.sv +@RESOURCES@/package.sv +-F @RTL@/TaskRouter/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/package.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/package.sv new file mode 100644 index 000000000..3793ede46 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/package.sv @@ -0,0 +1,17 @@ +`include "parameters.svh" +`define AI(T, F) T.inputs[`AD_IN_``F``_OFFSET+:`AD_IN_``F``_WIDTH] +`define AO(T, F) T.outputs[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH] +`define EXPECT(F, + V) want[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]=V;mask[`AD_OUT_``F``_OFFSET+:`AD_OUT_``F``_WIDTH]='1; +package admission_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "../common/agent/item.svh" + `include "../common/agent/driver.svh" + `include "../common/agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/seq/sequence.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/seq/sequence.svh new file mode 100644 index 000000000..5079a481e --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/seq/sequence.svh @@ -0,0 +1,68 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + logic [ `AD_INPUT_BITS-1:0] pins; + logic [`AD_OUTPUT_BITS-1:0] observed; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task command(int tag, logic [31:0] instruction, logic [63:0] rs1, logic [63:0] rs2); + for (int b = 0; b < `AD_COMMAND_WIDTH; b++) + pins[`AD_COMMAND_OFFSET+b] = ((b + tag) % 3 == 0) ^ ((b + tag) % 7 == 0); + pins[`AD_IN_COMMAND_BITS_INSTRUCTION_RAW_INST_OFFSET+:32] = instruction | ((tag % 32) << 7); + pins[`AD_IN_COMMAND_BITS_INSTRUCTION_RS1DATA_OFFSET+:64] = rs1; + pins[`AD_IN_COMMAND_BITS_INSTRUCTION_RS2DATA_OFFSET+:64] = rs2; + pins[`AD_IN_COMMAND_BITS_TAG_OFFSET+:`AD_IN_COMMAND_BITS_TAG_WIDTH] = tag; + pins[`AD_IN_COMMAND_VALID_OFFSET] = 1; + endtask + task accepted(); + do tick(); while (!observed[`AD_OUT_COMMAND_READY_OFFSET]); + endtask + task body(); + string negative; + pins = '0; + repeat (3) tick(1); + tick(); + for (int i = 0; i < 48; i++) begin + bit remote = i % 2; + command(i, (32'((remote ? 13 + i % 20 : i % 13) << 25) | 32'h2b), i * 64'h123456789abcdef, + ~64'(i)); + pins[`AD_IN_LOCAL_REQUEST_READY_OFFSET] = remote; + pins[`AD_IN_REMOTE_REQUEST_READY_OFFSET] = !remote; + repeat (3) tick(); + pins[`AD_IN_LOCAL_REQUEST_READY_OFFSET] = 1; + pins[`AD_IN_REMOTE_REQUEST_READY_OFFSET] = 1; + accepted(); + pins[`AD_IN_COMMAND_VALID_OFFSET] = 0; + pins[`AD_IN_RESPONSE_READY_OFFSET] = 0; + pins[`AD_IN_LOCAL_REPLY_BITS_OFFSET+:64] = 64'h9876543200000000 + i; + pins[`AD_IN_REMOTE_REPLY_BITS_OFFSET+:64] = 64'h1234567800000000 + i; + pins[`AD_IN_LOCAL_REPLY_VALID_OFFSET] = remote; + pins[`AD_IN_REMOTE_REPLY_VALID_OFFSET] = !remote; + repeat (3) tick(); + pins[`AD_IN_LOCAL_REPLY_VALID_OFFSET] = 1; + pins[`AD_IN_REMOTE_REPLY_VALID_OFFSET] = 1; + repeat (3) tick(); + pins[`AD_IN_RESPONSE_READY_OFFSET] = 1; + do tick(); while (!observed[`AD_OUT_RESPONSE_VALID_OFFSET]); + pins[`AD_IN_LOCAL_REPLY_VALID_OFFSET] = 0; + pins[`AD_IN_REMOTE_REPLY_VALID_OFFSET] = 0; + tick(); + end + command(0, 32'h2b, '1, '1); + accepted(); + pins[`AD_IN_COMMAND_VALID_OFFSET] = 0; + repeat (3) tick(); + repeat (2) tick(1); + pins = '0; + tick(); + repeat (4) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/tb.sv b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/tb.sv new file mode 100644 index 000000000..49c6210ad --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/tb.sv @@ -0,0 +1,15 @@ +module admission_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + admission_if bus (clock); + TaskRouter dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual admission_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/tests/test.svh b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/resources/task/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/scala/CommandAdmission.scala b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/CommandAdmission.scala new file mode 100644 index 000000000..8cde3f83b --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/CommandAdmission.scala @@ -0,0 +1,151 @@ +package framework.core.controller.admission + +import framework.acc.frontend.{Response, RoCCIO} +import framework.acc.frontend.Command +import framework.core.memory.access.preflight.PmpEntry +import framework.ip.memory.interlock.{Tag, Params => TrackingParam} +import framework.core.memory.access.preflight.configs.PmpParams +import framework.core.controller.configs.CommandsParam +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.memdomain.{MvoverISA, MvoverPort} + +class CommandSnapshot(tracking: TrackingParam, dataBits: Int, permission: PmpParams) extends Bundle { + + val tag = UInt(tracking.idBits.W) + val instruction = new Command(dataBits) + val satp = UInt(64.W) + val effectivePrivilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + val pmp = Vec(permission.entries, new PmpEntry(permission)) +} + +/** Execution-side task and bank-move admission, with completion ordering. */ +class Commands(tracking: TrackingParam, dataBits: Int, permission: PmpParams) extends Bundle { + val command = Decoupled(new CommandSnapshot(tracking, dataBits, permission)) + val response = Flipped(Decoupled(new Response(dataBits))) + val complete = Flipped(Decoupled(new Tag(tracking))) +} + +/** CPU-only controller command path: task RPC and bank moves have explicit completion owners. */ +@instantiable +class CommandAdmission(b: CommandsParam) extends Module { + val tracking = b.tracking + val moves = b.moves + + @public + val io = IO(new Bundle { + val execution = + Flipped(new Commands(tracking, b.dataBits, b.permission)) + + val taskRequest = Decoupled(new CommandSnapshot( + tracking, + b.dataBits, + b.permission + )) + + val taskResponse = Flipped(Decoupled(new Response(b.dataBits))) + val move = new MvoverPort + val memoryAllowed = Output(Bool()) + val idle = Output(Bool()) + }) + + val idle :: task :: moving :: moveResponse :: releasing :: Nil = Enum(5) + val state = RegInit(idle) + val tag = Reg(UInt(tracking.idBits.W)) + val rd = Reg(UInt(5.W)) + val failedMove = Reg(Bool()) + val instruction = io.execution.command.bits.instruction + val isTask = instruction.raw_inst(6, 0) === "h2b".U + val isMove = moves.B && instruction.raw_inst(6, 0) === "h7b".U && instruction.raw_inst(31, 25) === MvoverISA.Funct.U + + val writeResult = Reg(Bool()) + io.taskRequest.valid := state === idle && io.execution.command.valid && isTask && !reset.asBool + io.taskRequest.bits := io.execution.command.bits + + io.execution.command.ready := state === idle && !reset.asBool && + Mux(isTask, io.taskRequest.ready, isMove && io.move.command.ready) + when(io.execution.command.valid && state === idle && !reset.asBool) { + assert( + b.taskEnabled.B && (isTask || isMove), + "CPU-only controller received an unsupported compute instruction" + ) + } + io.move.command.valid := state === idle && io.execution.command.valid && isMove && !reset.asBool + io.move.command.bits.sourceThread := instruction.rs1Data(7, 0) + io.move.command.bits.targetThread := instruction.rs1Data(15, 8) + io.move.command.bits.sourceBank := instruction.rs1Data(25, 16) + io.move.command.bits.targetBank := instruction.rs1Data(35, 26) + io.move.command.bits.sourceAddr := instruction.rs2Data(15, 0) + io.move.command.bits.targetAddr := instruction.rs2Data(31, 16) + io.move.command.bits.rows := instruction.rs2Data(47, 32) +& 1.U + + when(io.execution.command.fire) { + writeResult := instruction.raw_inst(14) + tag := io.execution.command.bits.tag + rd := instruction.raw_inst(11, 7) + state := Mux(isTask, task, moving) + } + io.move.completion.ready := state === moving && !reset.asBool + when(io.move.completion.fire) { + failedMove := io.move.completion.bits + state := Mux(writeResult, moveResponse, releasing) + } + + io.execution.response.valid := ((state === task && io.taskResponse.valid && writeResult) || state === moveResponse) && !reset.asBool + io.execution.response.bits := io.taskResponse.bits + when(state === moveResponse) { + io.execution.response.bits.rd := rd + io.execution.response.bits.data := failedMove.asUInt + } + io.taskResponse.ready := state === task && (!writeResult || io.execution.response.ready) && !reset.asBool + when(io.taskResponse.fire || (state === moveResponse && io.execution.response.fire))(state := releasing) + io.execution.complete.valid := state === releasing && !reset.asBool + io.execution.complete.bits.tag := tag + when(io.execution.complete.fire)(state := idle) + // A move may have retired at CPU dispatch while its work is still pending here. + // Do not let a following CPU store publish a completion flag before the bank move ends. + val pendingMove = io.execution.command.valid && isMove + io.memoryAllowed := !(pendingMove || state === moving || state === moveResponse || state === releasing) + io.idle := state === idle +} + +/** One posted move command remains busy until the shared fabric reports completion. */ +@instantiable +class MoveAdmission(dataBits: Int, permission: PmpParams) extends Module { + + @public + val io = IO(new Bundle { + val rocc = new RoCCIO(dataBits, permission) + val move = new MvoverPort + }) + + val moving = RegInit(false.B) + val command = io.rocc.cmd + val instruction = command.bits.instruction + io.move.command.valid := command.valid && !moving + io.move.command.bits.sourceThread := instruction.rs1Data(7, 0) + io.move.command.bits.targetThread := instruction.rs1Data(15, 8) + io.move.command.bits.sourceBank := instruction.rs1Data(25, 16) + io.move.command.bits.targetBank := instruction.rs1Data(35, 26) + io.move.command.bits.sourceAddr := instruction.rs2Data(15, 0) + io.move.command.bits.targetAddr := instruction.rs2Data(31, 16) + io.move.command.bits.rows := instruction.rs2Data(47, 32) +& 1.U + command.ready := io.move.command.ready && !moving + when(command.valid) { + assert( + instruction.raw_inst(6, 0) === "h7b".U && + instruction.raw_inst(31, 25) === MvoverISA.Funct.U && instruction.raw_inst(14, 12) === 3.U, + "Move admission only accepts posted bank moves" + ) + } + when(io.move.command.fire)(moving := true.B) + io.move.completion.ready := moving + when(io.move.completion.fire)(moving := false.B) + io.rocc.busy := moving + io.rocc.interrupt := false.B + io.rocc.resp.valid := false.B + io.rocc.resp.bits := 0.U.asTypeOf(io.rocc.resp.bits) +} diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/scala/DirectAdmission.scala b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/DirectAdmission.scala new file mode 100644 index 000000000..c9d207aa5 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/DirectAdmission.scala @@ -0,0 +1,81 @@ +package framework.core.controller.admission + +import framework.lib.queue.Queue +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.controller.configs.CommandsParam +import framework.acc.frontend.RoCCIO + +/** Tracks posted accelerator commands and orders cross-core management after their drain. */ +@instantiable +class DirectAdmission(p: CommandsParam) extends Module { + require(p.remote) + + @public + val io = IO(new Bundle { + val execution = Flipped(new Commands(p.tracking, p.dataBits, p.permission)) + val accelerator = Flipped(new RoCCIO(p.dataBits, p.permission)) + val remote = new TaskEndpoint + val drained = Output(Bool()) + }) + + val idle :: managing :: releasing :: Nil = Enum(3) + val state = RegInit(idle) + val tags = Instantiate(new Queue(UInt(p.tracking.idBits.W), p.tracking.entries)) + val destination = Reg(UInt(5.W)) + val writeResult = Reg(Bool()) + val command = io.execution.command.bits + val instruction = command.instruction + val management = instruction.raw_inst(6, 0) === "h2b".U + val available = state === idle && !io.remote.busy && !reset.asBool && tags.io.enq.ready && + (!management || (!io.accelerator.busy && !tags.io.deq.valid)) + + io.accelerator.cmd.valid := io.execution.command.valid && available && !management + io.accelerator.cmd.bits.instruction := instruction + io.accelerator.cmd.bits.satp := command.satp + io.accelerator.cmd.bits.effectivePrivilege := command.effectivePrivilege + io.accelerator.cmd.bits.sum := command.sum + io.accelerator.cmd.bits.mxr := command.mxr + io.accelerator.cmd.bits.pmp := command.pmp + io.accelerator.resp.ready := false.B + + io.remote.request.valid := io.execution.command.valid && available && management + io.remote.request.bits.operation := instruction.raw_inst(31, 25) + io.remote.request.bits.context := instruction.rs1Data(63, 32) + io.remote.request.bits.field := instruction.rs1Data(31, 0) + io.remote.request.bits.data := instruction.rs2Data + io.execution.command.ready := available && Mux(management, io.remote.request.ready, io.accelerator.cmd.ready) + when(io.execution.command.fire) { + assert( + Mux( + management, + instruction.raw_inst(31, 25) >= 13.U, + instruction.raw_inst(6, 0) === "h7b".U && !instruction.raw_inst(14) + ), + "Direct core received an unsupported command" + ) + destination := instruction.raw_inst(11, 7) + writeResult := instruction.raw_inst(14) + when(management)(state := managing) + } + tags.io.enq.valid := io.execution.command.fire + tags.io.enq.bits := command.tag + io.execution.response.valid := state === managing && writeResult && io.remote.reply.valid + io.execution.response.bits.rd := destination + io.execution.response.bits.data := io.remote.reply.bits + io.remote.reply.ready := state === managing && (!writeResult || io.execution.response.ready) + when(io.remote.reply.fire)(state := releasing) + val completed = state === releasing || (state === idle && !io.accelerator.busy) + io.execution.complete.valid := tags.io.deq.valid && completed + io.execution.complete.bits.tag := tags.io.deq.bits + tags.io.deq.ready := io.execution.complete.ready && completed + when(io.execution.complete.fire && state === releasing)(state := idle) + io.drained := state === idle && !tags.io.deq.valid && !io.accelerator.busy && !io.remote.busy + when(!reset.asBool) { + assert(!io.accelerator.resp.valid, "Direct accelerator returned an unexpected register result") + assert(!io.accelerator.interrupt, "Direct accelerator fault") + } +} diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/Emit.scala new file mode 100644 index 000000000..207e94758 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/Emit.scala @@ -0,0 +1,134 @@ +package framework.core.controller.admission + +import chisel3._ +import chisel3.reflect.DataMirror +import java.nio.file.{Files, Paths} +import framework.core.controller.configs.CommandsParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + + implicit val trackingRW: upickle.default.ReadWriter[TrackingParam] = upickle.default.macroRW + + val p = CommandsParam( + upickle.default.read[TrackingParam](values("tracking")), + values("dataBits").num.toInt, + upickle.default.read[PmpParams](values("permission")), + values("moves").bool, + values("taskEnabled").bool, + values("remote").bool + ) + + Files.createDirectories(Paths.get(directory)) + + def leaves(data: Data, path: String): Seq[(String, Data)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + leaves(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => leaves(field, s"${path}_$index") } + case field => if (field.getWidth > 0) Seq(path -> field) else Seq.empty + } + + for ((top, kind) <- Seq("CommandAdmission", "MoveAdmission", "DirectAdmission", "TaskRouter").zipWithIndex) { + val definitions = scala.collection.mutable.ArrayBuffer[String]( + s"`define AD_KIND $kind", + s"`define AD_ENTRIES ${p.tracking.entries}", + s"`define AD_DATA_BITS ${p.dataBits}" + ) + val binding = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + val widths = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0; var outputOffset = 0 + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val (dut, io) = kind match { + case 0 => val m = new CommandAdmission(p); (m, m.io) + case 1 => val m = new MoveAdmission(p.dataBits, p.permission); (m, m.io) + case 2 => val m = new DirectAdmission(p); (m, m.io) + case 3 => val m = new TaskRouter(p); (m, m.io) + } + for ((name, field) <- leaves(io, "")) { + val input = DataMirror.directionOf(field) == ActualDirection.Input + val direction = if (input) "IN" else "OUT"; val signal = if (input) "inputs" else "outputs"; + val offset = if (input) inputOffset else outputOffset; val width = field.getWidth + definitions += s"`define AD_${direction}_${name.toUpperCase}_OFFSET $offset"; + definitions += s"`define AD_${direction}_${name.toUpperCase}_WIDTH $width" + offsets(name) = offset; widths(name) = width + binding += s".io_$name(bus.$signal[$offset +: $width])"; + if (input) inputOffset += width else outputOffset += width + }; dut + }, + args = Array("--target-dir", s"$directory/$top", "--split-verilog") + ) + val command = kind match { case 1 => "rocc_cmd"; case 3 => "command"; case _ => "execution_command" } + val source = s"${command}_bits_" + val instruction = s"${source}instruction_" + val maps = scala.collection.mutable.ArrayBuffer[(Int, Int, Int, Int)]() + def copyPayload(target: String): Unit = for (name <- offsets.keys.toSeq.sorted if name.startsWith(target)) { + val from = source + name.stripPrefix(target); + maps += ((offsets(from), offsets(name), widths(name), offsets(name.take(name.indexOf("_bits")) + "_valid"))) + } + def control(target: String): Unit = { + for ( + (name, from, lsb) <- + Seq(("operation", "raw_inst", 25), ("context", "rs1Data", 32), ("field", "rs1Data", 0), ("data", "rs2Data", 0)) + ) { + val to = s"${target}_$name"; + maps += ( + ( + offsets(instruction + from) + lsb, + offsets(to), + widths(to), + offsets(to.take(to.indexOf("_bits")) + "_valid") + ) + ) + } + } + kind match { + case 0 => copyPayload("taskRequest_bits_") + case 1 => () + case 2 => copyPayload("accelerator_cmd_bits_"); control("remote_request_bits") + case 3 => + control("local_request_bits_control"); control("remote_request_bits") + for (name <- offsets.keys.toSeq.sorted if name.startsWith("local_request_bits_context_")) { + val from = source + name.stripPrefix("local_request_bits_context_"); + maps += ((offsets(from), offsets(name), widths(name), offsets(name.take(name.indexOf("_bits")) + "_valid"))) + } + } + if (kind == 0 || kind == 1) { + for ( + (name, from, lsb) <- Seq( + ("sourceThread", "rs1Data", 0), + ("targetThread", "rs1Data", 8), + ("sourceBank", "rs1Data", 16), + ("targetBank", "rs1Data", 26), + ("sourceAddr", "rs2Data", 0), + ("targetAddr", "rs2Data", 16) + ) + ) { + val to = s"move_command_bits_$name"; + maps += ( + ( + offsets(instruction + from) + lsb, + offsets(to), + widths(to), + offsets(to.take(to.indexOf("_bits")) + "_valid") + ) + ) + } + } + definitions += s"`define AD_PAYLOAD_MAP ${maps.map { case (from, to, width, gate) => + s"'{$from,$to,$width,$gate}" + }.mkString("'{", ",", "}")}" + definitions += s"`define AD_PAYLOAD_FIELDS ${maps.size}" + val commandFields = offsets.keys.filter(_.startsWith(source)).toSeq + definitions += s"`define AD_COMMAND_OFFSET ${commandFields.map(offsets).min}" + definitions += s"`define AD_COMMAND_WIDTH ${commandFields.map(widths).sum}" + definitions += s"`define AD_INPUT_BITS $inputOffset"; definitions += s"`define AD_OUTPUT_BITS $outputOffset" + Files.writeString(Paths.get(directory, top, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, top, "ports.svh"), binding.mkString(",\n") + "\n") + } +} diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/scala/TaskRouter.scala b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/TaskRouter.scala new file mode 100644 index 000000000..699f99857 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/TaskRouter.scala @@ -0,0 +1,85 @@ +package framework.core.controller.admission + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.controller.configs.CommandsParam +import framework.device.cbus.ControlRequest +import framework.core.controller.task.TaskRequest +import framework.acc.frontend.Response + +class TaskEndpoint extends Bundle { + val request = Decoupled(new ControlRequest) + val reply = Flipped(Decoupled(UInt(64.W))) + val busy = Input(Bool()) +} + +/** Retains the reply destination and owner of each accepted management command. */ +@instantiable +class TaskRouter(b: CommandsParam) extends Module { + val remoteEnabled = b.remote + + @public + val io = IO(new Bundle { + + val command = Flipped(Decoupled(new CommandSnapshot( + b.tracking, + b.dataBits, + b.permission + ))) + + val response = Decoupled(new Response(b.dataBits)) + val busy = Output(Bool()) + + val local = new Bundle { + val request = Decoupled(new TaskRequest(b.permission)) + val reply = Flipped(Decoupled(UInt(64.W))) + val busy = Input(Bool()) + } + + val remote = Option.when(remoteEnabled)(new TaskEndpoint) + }) + + val pending = RegInit(false.B) + val remoteOwner = Reg(Bool()) + val destination = Reg(UInt(5.W)) + val command = io.command.bits + val instruction = command.instruction + val remote = instruction.raw_inst(31, 25) >= 13.U + val request = Wire(new ControlRequest) + request.operation := instruction.raw_inst(31, 25) + request.context := instruction.rs1Data(63, 32) + request.field := instruction.rs1Data(31, 0) + request.data := instruction.rs2Data + io.local.request.bits.control := request + io.local.request.bits.context.satp := command.satp + io.local.request.bits.context.effectivePrivilege := command.effectivePrivilege + io.local.request.bits.context.sum := command.sum + io.local.request.bits.context.mxr := command.mxr + io.local.request.bits.context.pmp := command.pmp + io.local.request.valid := io.command.valid && !pending && !remote + io.local.reply.ready := pending && !remoteOwner && io.response.ready + io.response.bits.rd := destination + if (remoteEnabled) { + val port = io.remote.get + port.request.bits := request + port.request.valid := io.command.valid && !pending && remote + port.reply.ready := pending && remoteOwner && io.response.ready + io.command.ready := !pending && Mux(remote, port.request.ready, io.local.request.ready) + io.response.valid := pending && Mux(remoteOwner, port.reply.valid, io.local.reply.valid) + io.response.bits.data := Mux(remoteOwner, port.reply.bits, io.local.reply.bits) + io.busy := pending || io.local.busy || port.busy + } else { + io.command.ready := !pending && !remote && io.local.request.ready + io.response.valid := pending && io.local.reply.valid + io.response.bits.data := io.local.reply.bits + io.busy := pending || io.local.busy + when(io.command.valid)(assert(!remote, "No cross-core task endpoint")) + } + when(io.command.fire) { + pending := true.B + remoteOwner := remote + destination := instruction.raw_inst(11, 7) + } + when(io.response.fire)(pending := false.B) +} diff --git a/arch/src/main/scala/framework/core/controller/admission/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..3ae1d8536 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/admission/src/main/scala/configs/Params.scala @@ -0,0 +1,7 @@ +package framework.core.controller.admission.configs + +import framework.core.memory.access.preflight.configs.PmpParams +import upickle.default.{macroRW, ReadWriter} + +case class Params(permission: PmpParams, moves: Boolean) +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/controller/build.sc b/arch/src/main/scala/framework/core/controller/build.sc new file mode 100644 index 000000000..6ae568944 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/build.sc @@ -0,0 +1,16 @@ +import mill._ +import $file.admission.{build => admission_build} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("Controller.scala")), + PathRef(millSourcePath / os.RelPath("configs/ControllerParam.scala")), + PathRef(millSourcePath / os.RelPath("task/Control.scala")), + PathRef(millSourcePath / os.RelPath("task/Service.scala")) + ) ++ + admission_build.sources() +) + +def emitSources = T.sources( + admission_build.emitSources() +) diff --git a/arch/src/main/scala/framework/core/controller/configs/ControllerParam.scala b/arch/src/main/scala/framework/core/controller/configs/ControllerParam.scala new file mode 100644 index 000000000..0c60f424b --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/configs/ControllerParam.scala @@ -0,0 +1,26 @@ +package framework.core.controller.configs + +import framework.thread.ant.configs.AntParam +import framework.core.tss.configs.{Params => TssParam} +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} + +case class CommandsParam( + tracking: TrackingParam, + dataBits: Int, + permission: PmpParams, + moves: Boolean, + taskEnabled: Boolean, + remote: Boolean) + +case class TasksParam( + locals: Seq[AntParam], + signatures: Seq[BigInt], + tss: TssParam, + permission: PmpParams) { + val count = signatures.size + require(count > 0 && locals.size == count && tss.ports == count) + require(locals.forall(p => p.shared == tss.memory && p.taskBits <= 64)) +} + +case class ControllerParam(commands: CommandsParam, tasks: Option[TasksParam]) diff --git a/arch/src/main/scala/framework/core/controller/configs/default.toml b/arch/src/main/scala/framework/core/controller/configs/default.toml new file mode 100644 index 000000000..37a096595 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/configs/default.toml @@ -0,0 +1 @@ +moves = false diff --git a/arch/src/main/scala/framework/core/controller/task/Control.scala b/arch/src/main/scala/framework/core/controller/task/Control.scala new file mode 100644 index 000000000..963f4def4 --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/task/Control.scala @@ -0,0 +1,165 @@ +package framework.core.controller.task + +import framework.core.controller.configs.TasksParam +import framework.core.memory.access.preflight.configs.PmpParams + +import framework.thread.ant.{Launch, TaskContext} +import framework.thread.ant.Completion +import framework.device.cbus.ControlRequest +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +object ControlOp { + val Query = 0 + val Stage = 1 + val Start = 2 + val Acknowledge = 3 + val Cancel = 4 + val Acquire = 5 + val Release = 6 +} + +/** A management request carries the submitter's context through every accepting endpoint. */ +class TaskRequest(permission: PmpParams) extends Bundle { + val control = new ControlRequest + val context = new TaskContext(permission) +} + +/** + * Task registers for one software owner. Invalid descriptors and resource reuse are + * programming errors. Execution contract violations assert at their source. + */ +@instantiable +class Control(val b: TasksParam) extends Module { + val contexts = b.count + val signatures = VecInit(b.signatures.map(_.U(64.W))) + require(contexts > 0) + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new TaskRequest(b.permission))) + val response = Decoupled(UInt(64.W)) + val storageBusy = Input(Bool()) + + val start = MixedVec(b.locals.map(p => Decoupled(new Launch(p, b.permission)))) + val result = MixedVec(b.locals.map(p => Flipped(Decoupled(new Completion(p))))) + + val cancel = Output(Vec(contexts, Bool())) + val inUse = Output(Bool()) + val available = Output(Vec(contexts, Bool())) + }) + + val group = RegInit(false.B) + val owned = RegInit(VecInit(Seq.fill(contexts)(false.B))) + val done = RegInit(VecInit(Seq.fill(contexts)(false.B))) + val staged = Reg(Vec(contexts, Vec(6, UInt(64.W)))) + val fields = RegInit(VecInit(Seq.fill(contexts)(0.U(6.W)))) + + val results = Reg(Vec( + contexts, + new Bundle { + val task = UInt(64.W) + val value = UInt(64.W) + val cancelled = Bool() + } + )) + + val pending = RegInit(false.B) + val answer = Reg(UInt(64.W)) + val req = io.request.bits.control + val index = req.context(math.max(1, log2Ceil(contexts)) - 1, 0) + val inRange = req.context < contexts.U + val free = !owned(index) && !done(index) + val canStart = inRange && free && group && + fields(index) === 63.U && staged(index)(5) === signatures(index) + val launching = req.operation === ControlOp.Start.U && canStart + io.request.ready := !pending && !reset.asBool && (!launching || VecInit(io.start.map(_.ready))(index)) + io.response.valid := pending && !reset.asBool + io.response.bits := answer + io.inUse := group + when(io.response.fire)(pending := false.B) + for (i <- 0 until contexts) { + io.available(i) := !owned(i) && !done(i) + io.start(i).valid := io.request.valid && !pending && !reset.asBool && launching && req.context === i.U + io.start(i).bits.descriptor.task := staged(i)(0) + io.start(i).bits.descriptor.entry := staged(i)(1) + io.start(i).bits.descriptor.codeEnd := staged(i)(2) + io.start(i).bits.descriptor.argument := staged(i)(3) + io.start(i).bits.descriptor.stack := staged(i)(4) + io.start(i).bits.context := io.request.bits.context + // Cancel accepts immediately, independently of the start-ready path. + io.cancel(i) := io.request.valid && !pending && !reset.asBool && + req.operation === ControlOp.Cancel.U && req.context === i.U && owned(i) + io.result(i).ready := owned(i) && !done(i) && !reset.asBool + when(io.result(i).valid && !reset.asBool) { + assert(owned(i) && !done(i), "Ant result arrived without a live task") + assert(io.result(i).bits.task === staged(i)(0), "Ant completion task does not match its submission") + } + when(io.result(i).fire) { + results(i).task := io.result(i).bits.task + results(i).value := io.result(i).bits.value + results(i).cancelled := io.result(i).bits.cancelled + owned(i) := false.B + done(i) := true.B + } + } + when(io.request.fire) { + pending := true.B + answer := 0.U + assert(req.operation <= ControlOp.Release.U, "Invalid Ant control operation") + when(req.operation <= ControlOp.Cancel.U && !(req.operation === ControlOp.Query.U && req.field === 7.U)) { + assert(inRange, "Invalid Ant context index") + } + switch(req.operation) { + is(ControlOp.Query.U) { + assert(req.field < 10.U, "Invalid Ant query field") + when(req.field >= 8.U)(assert(done(index), "Ant result read before completion")) + answer := MuxLookup(req.field, 0.U)(Seq( + 0.U -> signatures(index), + 1.U -> Cat(done(index) && results(index).cancelled, done(index), owned(index), true.B), + 2.U -> VecInit(b.locals.map(p => p.codeBytes.U(64.W)))(index), + 3.U -> VecInit(b.locals.map(p => p.data.base.U(64.W)))(index), + 4.U -> VecInit(b.locals.map(p => p.data.bytes.U(64.W)))(index), + 5.U -> b.tss.memory.base.U, + 6.U -> b.tss.memory.bytes.U, + 7.U -> contexts.U, + 8.U -> results(index).task, + 9.U -> results(index).value + )) + } + is(ControlOp.Stage.U) { + assert(free, "Ant descriptor overwritten before resource release") + assert(req.field < 6.U, "Invalid Ant descriptor field") + when(req.field === 0.U)(assert( + VecInit(b.locals.map(p => (req.data >> p.taskBits) === 0.U))(index), + "Ant task ID overflow" + )) + staged(index)(req.field(2, 0)) := req.data + fields(index) := fields(index) | UIntToOH(req.field, 6) + } + is(ControlOp.Start.U) { + assert(free, "Ant context reused before completion acknowledgement") + assert(group, "Ant submitted without TSS group ownership") + assert(fields(index) === 63.U, "Incomplete Ant descriptor") + assert(staged(index)(5) === signatures(index), "Ant program fingerprint mismatch") + when(canStart) { owned(index) := true.B; fields(index) := 0.U } + } + is(ControlOp.Acknowledge.U) { + assert(done(index), "Ant completion acknowledged before result") + done(index) := false.B + } + is(ControlOp.Acquire.U) { + assert(!group && !io.storageBusy, "Ant group acquired before loader completion or previous release") + group := true.B + } + is(ControlOp.Release.U) { + assert( + group && !owned.asUInt.orR && !done.asUInt.orR && !io.storageBusy, + "Ant group released before tasks and storage drained" + ) + group := false.B + } + } + } +} diff --git a/arch/src/main/scala/framework/core/controller/task/Service.scala b/arch/src/main/scala/framework/core/controller/task/Service.scala new file mode 100644 index 000000000..303a4850b --- /dev/null +++ b/arch/src/main/scala/framework/core/controller/task/Service.scala @@ -0,0 +1,118 @@ +package framework.core.controller.task + +import chisel3.experimental.hierarchy.Instance + +import framework.core.controller.configs.TasksParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.core.tss.TssControlPort +import framework.thread.ant.Task +import framework.ip.memory.spm + +/** + * Core execution service. The caller supplies the uncached management transport and + * task-bound NPU/DMA owner; neither CPU cache nor address translation lives here. + */ +@instantiable +class Service(val b: TasksParam) extends Module { + val contexts = b.count + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new TaskRequest(b.permission))) + val reply = Decoupled(UInt(64.W)) + + val tasks = MixedVec(b.locals.map(p => Flipped(new Task(p, b.permission)))) + + val idle = Output(Bool()) + val tss = new TssControlPort(b.tss) + }) + + val control: Instance[Control] = Instantiate(new Control(b)) + + require(b.tss.ports == contexts) + + val transferBusy = Wire(Bool()) + val running = VecInit(io.tasks.map(_.running)) + val storageBusy = io.tasks.map(_.storageBusy).reduce(_ || _) || + io.tss.busy.asUInt.orR || io.tss.hostBusy + control.io.storageBusy := storageBusy + io.tss.inUse := control.io.inUse || running.asUInt.orR || transferBusy + io.tss.cancel := control.io.cancel + for (i <- 0 until contexts) { + io.tasks(i).inUse := control.io.inUse + io.tasks(i).start.valid := control.io.start(i).valid && !io.tss.hostBusy && !io.tss.busy(i) + io.tasks(i).start.bits := control.io.start(i).bits + control.io.start(i).ready := io.tasks(i).start.ready && !io.tss.hostBusy && !io.tss.busy(i) + // A cancelled Ant may finish locally before its accepted TSS access drains. + control.io.result(i).valid := io.tasks(i).result.valid && !io.tss.busy(i) + control.io.result(i).bits := io.tasks(i).result.bits + io.tasks(i).result.ready := control.io.result(i).ready && !io.tss.busy(i) + io.tasks(i).cancel := control.io.cancel(i) + } + val idle :: controlReply :: transferring :: sharedResponse :: loadReply :: Nil = Enum(5) + val state = RegInit(idle) + val request = io.request.bits.control + val isTransfer = request.operation >= 7.U && request.operation <= 10.U + val isShared = request.operation === 11.U || request.operation === 12.U + val isLoad = isTransfer || isShared + val localIndex = request.context(math.max(1, log2Ceil(contexts)) - 1, 0) + val owner = Reg(UInt(math.max(1, log2Ceil(contexts)).W)) + val value = Reg(UInt(64.W)) + transferBusy := state === transferring + val transferAllowed = request.context < contexts.U && control.io.available(localIndex) && + !running(localIndex) && !VecInit(io.tasks.map(_.storageBusy))(localIndex) && !io.tss.busy(localIndex) + for (i <- 0 until contexts) { + val task = io.tasks(i) + task.transfer.valid := state === idle && io.request.valid && isTransfer && transferAllowed && request.context === i.U + task.transfer.bits.code := request.operation <= 8.U + task.transfer.bits.toLocal := !request.operation(0) + task.transfer.bits.localOffset := request.field + task.transfer.bits.sharedOffset := request.data(31, 0) + task.transfer.bits.bytes := request.data(63, 32) + task.transferred.ready := state === transferring && owner === i.U + when(task.transferred.fire) { + assert(task.transferred.bits, "Ant TSS transfer failed") + value := 0.U + state := loadReply + } + } + val sharedValid = request.context === 0.U && request.field(2, 0) === 0.U && + (request.field +& 8.U) <= b.tss.memory.bytes.U + val port = io.tss.host + port.request.valid := state === idle && io.request.valid && isShared && sharedValid && !reset.asBool + port.request.bits := 0.U.asTypeOf(port.request.bits) + port.request.bits.address := b.tss.memory.base.U + request.field + port.request.bits.size := 3.U + port.request.bits.write := request.operation === 12.U + port.request.bits.data := request.data + port.request.bits.mask := Mux(request.operation === 12.U, 255.U, 0.U) + port.response.ready := state === sharedResponse + when(port.response.fire) { + assert(!port.response.bits.error, "TSS management access failed") + value := port.response.bits.data(63, 0) + state := loadReply + } + when(state === idle && io.request.valid && isLoad && !reset.asBool) { + assert(Mux(isTransfer, transferAllowed, sharedValid), "Invalid TSS transfer or storage ownership") + } + control.io.request.valid := state === idle && io.request.valid && !isLoad && !reset.asBool + control.io.request.bits := io.request.bits + io.request.ready := state === idle && !reset.asBool && Mux( + isTransfer, + transferAllowed && VecInit(io.tasks.map(_.transfer.ready))(localIndex), + Mux(isShared, sharedValid && port.request.ready, control.io.request.ready) + ) + when(io.request.fire) { + owner := localIndex + state := Mux(isTransfer, transferring, Mux(isShared, sharedResponse, controlReply)) + } + io.reply.valid := state === loadReply || (state === controlReply && control.io.response.valid) + io.reply.bits := Mux(state === loadReply, value, control.io.response.bits) + control.io.response.ready := state === controlReply && io.reply.ready + when(io.reply.fire)(state := idle) + io.idle := state === idle && !running.asUInt.orR && !storageBusy +} diff --git a/arch/src/main/scala/framework/core/execution/Execution.scala b/arch/src/main/scala/framework/core/execution/Execution.scala new file mode 100644 index 000000000..62c5e6b83 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/Execution.scala @@ -0,0 +1,60 @@ +package framework.core.execution + +import chisel3.util._ +import framework.acc.frontend.RoCCIO +import framework.core.controller.admission.Commands +import framework.core.clink.CoreInterrupts +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.execution.configs.ExecutionParam +import framework.core.execution.pipeline.{Retirement, Sequencer} +import framework.core.execution.instruction.Instruction +import framework.core.execution.register_file.RegisterFile +import framework.core.execution.integer.Integer +import framework.core.execution.muldiv.MulDiv +import framework.core.execution.fpu.FPU +import framework.core.execution.privileged.Privileged +import framework.core.memory.access.{UncachedRequest, UncachedResponse, VirtualPort} + +/** Composes instruction execution; memory requests are served by the core memory hierarchy. */ +@instantiable +class Execution(b: ExecutionParam, localRocc: Boolean) extends Module { + + @public + val io = IO(new Bundle { + val interrupts = Input(new CoreInterrupts) + val fetch = Flipped(new VirtualPort(b.memoryAccess)) + val data = Flipped(new VirtualPort(b.memoryAccess)) + val timeRequest = Decoupled(new UncachedRequest(b.memoryAccess.memory)) + val timeResponse = Flipped(Decoupled(new UncachedResponse(b.memoryAccess.memory))) + val controller = Option.when(!localRocc)(new Commands(b.tracking, b.integer.xLen, b.privileged.permission)) + val rocc = Option.when(localRocc)(Flipped(new RoCCIO(b.integer.xLen, b.privileged.permission))) + val workDrained = Input(Bool()) + val retirement = Valid(new Retirement) + }) + + val sequencer: Instance[Sequencer] = Instantiate(new Sequencer(b, localRocc)) + val instruction: Instance[Instruction] = Instantiate(new Instruction(b.instruction)) + val registers: Instance[RegisterFile] = Instantiate(new RegisterFile(b.registerFile)) + val integer: Instance[Integer] = Instantiate(new Integer(b.integer)) + val multiply: Instance[MulDiv] = Instantiate(new MulDiv(b.muldiv)) + val floating: Instance[FPU] = Instantiate(new FPU(b.fpu)) + val privileged: Instance[Privileged] = Instantiate(new Privileged(b.privileged)) + + sequencer.io.instruction <> instruction.io + sequencer.io.registers <> registers.io + sequencer.io.integer <> integer.io + sequencer.io.multiply <> multiply.io + sequencer.io.floating <> floating.io + sequencer.io.privileged <> privileged.io + sequencer.io.interrupts := io.interrupts + sequencer.io.controller.zip(io.controller).foreach { case (child, parent) => child <> parent } + sequencer.io.rocc.zip(io.rocc).foreach { case (child, parent) => child <> parent } + io.retirement := sequencer.io.retirement + sequencer.io.workDrained := io.workDrained + + io.fetch <> sequencer.io.fetch + io.data <> sequencer.io.memory + io.timeRequest <> sequencer.io.timeRequest + sequencer.io.timeResponse <> io.timeResponse +} diff --git a/arch/src/main/scala/framework/core/execution/build.sc b/arch/src/main/scala/framework/core/execution/build.sc new file mode 100644 index 000000000..48dd6158f --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/build.sc @@ -0,0 +1,33 @@ +import mill._ +import $file.fetch.{build => fetch_build} +import $file.fpu.{build => fpu_build} +import $file.instruction.{build => instruction_build} +import $file.integer.{build => integer_build} +import $file.muldiv.{build => muldiv_build} +import $file.privileged.{build => privileged_build} +import $file.register_file.{build => register_file_build} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("Execution.scala")), + PathRef(millSourcePath / os.RelPath("configs/ExecutionParam.scala")), + PathRef(millSourcePath / os.RelPath("pipeline/Sequencer.scala")) + ) ++ + fetch_build.sources() ++ + fpu_build.sources() ++ + instruction_build.sources() ++ + integer_build.sources() ++ + muldiv_build.sources() ++ + privileged_build.sources() ++ + register_file_build.sources() +) + +def emitSources = T.sources( + fetch_build.emitSources() ++ + fpu_build.emitSources() ++ + instruction_build.emitSources() ++ + integer_build.emitSources() ++ + muldiv_build.emitSources() ++ + privileged_build.emitSources() ++ + register_file_build.emitSources() +) diff --git a/arch/src/main/scala/framework/core/execution/configs/ExecutionParam.scala b/arch/src/main/scala/framework/core/execution/configs/ExecutionParam.scala new file mode 100644 index 000000000..14eca1a71 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/configs/ExecutionParam.scala @@ -0,0 +1,26 @@ +package framework.core.execution.configs + +import framework.core.execution.instruction.configs.{Params => InstructionParam} +import framework.core.execution.register_file.configs.{Params => RegisterFileParam} +import framework.core.execution.integer.configs.{Params => IntegerParam} +import framework.core.execution.muldiv.configs.{Params => MulDivParam} +import framework.core.execution.fpu.configs.{Params => FpuParam} +import framework.core.execution.privileged.configs.{Params => PrivilegedParam} +import framework.core.memory.access.configs.{Params => AccessParam} +import framework.ip.memory.interlock.{Params => TrackingParam} + +case class ExecutionParam( + hartId: Int, + resetVector: BigInt, + commandsEnabled: Boolean, + directAccelerator: Boolean, + moves: Boolean, + tracking: TrackingParam, + mtimeAddress: BigInt, + memoryAccess: AccessParam, + instruction: InstructionParam, + registerFile: RegisterFileParam, + integer: IntegerParam, + muldiv: MulDivParam, + fpu: FpuParam, + privileged: PrivilegedParam) diff --git a/arch/src/main/scala/framework/core/execution/fetch/build.sc b/arch/src/main/scala/framework/core/execution/fetch/build.sc new file mode 100644 index 000000000..05531cd7e --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.execution.fetch.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Fetch.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/execution/fetch/configs/uvm.toml b/arch/src/main/scala/framework/core/execution/fetch/configs/uvm.toml new file mode 100644 index 000000000..50aab1014 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/configs/uvm.toml @@ -0,0 +1,4 @@ +[parameters] +instructionBytes = 2 + +[defines] diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/csrc/Cargo.lock b/arch/src/main/scala/framework/core/execution/fetch/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/fetch/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/core/execution/fetch/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/csrc/Cargo.toml b/arch/src/main/scala/framework/core/execution/fetch/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/fetch/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/core/execution/fetch/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/csrc/src/instructions.rs b/arch/src/main/scala/framework/core/execution/fetch/src/csrc/src/instructions.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/fetch/src/csrc/src/instructions.rs rename to arch/src/main/scala/framework/core/execution/fetch/src/csrc/src/instructions.rs diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/csrc/src/lib.rs b/arch/src/main/scala/framework/core/execution/fetch/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/fetch/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/core/execution/fetch/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..1efe4bd9f --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/driver.svh @@ -0,0 +1,62 @@ +class driver extends uvm_driver #(control_item); + `uvm_component_utils(driver) + virtual fetch_if vif; + program_image image; + control_item pending; + longint unsigned reset_vector = 0, satp = 0; + int privilege = 3; + bit sum = 0, mxr = 0; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual fetch_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Fetch control interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + if (pending == null) seq_item_port.try_next_item(pending); + vif.inputs[`FETCH_IN_REDIRECT_VALID_OFFSET+:`FETCH_IN_REDIRECT_VALID_WIDTH] = 0; + vif.inputs[`FETCH_IN_FLUSH_OFFSET+:`FETCH_IN_FLUSH_WIDTH] = 0; + vif.inputs[`FETCH_IN_INVALIDATE_OFFSET+:`FETCH_IN_INVALIDATE_WIDTH] = 0; + if (pending != null) begin + case (pending.kind) + REDIRECT: begin + vif.inputs[`FETCH_IN_REDIRECT_VALID_OFFSET+:`FETCH_IN_REDIRECT_VALID_WIDTH] = 1; + vif.inputs[`FETCH_IN_REDIRECT_BITS_OFFSET+:`FETCH_IN_REDIRECT_BITS_WIDTH]=pending.target; + end + FENCE: begin + vif.inputs[`FETCH_IN_REDIRECT_VALID_OFFSET+:`FETCH_IN_REDIRECT_VALID_WIDTH] = 1; + vif.inputs[`FETCH_IN_REDIRECT_BITS_OFFSET+:`FETCH_IN_REDIRECT_BITS_WIDTH]=pending.target; + vif.inputs[`FETCH_IN_FLUSH_OFFSET+:`FETCH_IN_FLUSH_WIDTH] = 1; + image.version = pending.version; + end + CONTEXT: begin + vif.inputs[`FETCH_IN_REDIRECT_VALID_OFFSET+:`FETCH_IN_REDIRECT_VALID_WIDTH] = 1; + vif.inputs[`FETCH_IN_REDIRECT_BITS_OFFSET+:`FETCH_IN_REDIRECT_BITS_WIDTH]=pending.target; + privilege = pending.privilege; + satp = pending.satp; + sum = pending.sum; + mxr = pending.mxr; + end + INVALIDATE: vif.inputs[`FETCH_IN_INVALIDATE_OFFSET+:`FETCH_IN_INVALIDATE_WIDTH] = 1; + RESET_VECTOR: reset_vector = pending.target; + BAD_FLUSH: vif.inputs[`FETCH_IN_FLUSH_OFFSET+:`FETCH_IN_FLUSH_WIDTH] = 1; + endcase + end + vif.inputs[`FETCH_IN_RESETVECTOR_OFFSET+:`FETCH_IN_RESETVECTOR_WIDTH] = reset_vector; + vif.inputs[`FETCH_IN_CONTEXT_PRIVILEGE_OFFSET+:`FETCH_IN_CONTEXT_PRIVILEGE_WIDTH] = privilege; + vif.inputs[`FETCH_IN_CONTEXT_SATP_OFFSET+:`FETCH_IN_CONTEXT_SATP_WIDTH] = satp; + vif.inputs[`FETCH_IN_CONTEXT_SUM_OFFSET+:`FETCH_IN_CONTEXT_SUM_WIDTH] = sum; + vif.inputs[`FETCH_IN_CONTEXT_MXR_OFFSET+:`FETCH_IN_CONTEXT_MXR_WIDTH] = mxr; + @(posedge vif.clock); + if (pending != null) begin + #1; + seq_item_port.item_done(); + pending = null; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/item.svh new file mode 100644 index 000000000..c5eca9906 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/item.svh @@ -0,0 +1,57 @@ +typedef enum int { + REDIRECT, + FENCE, + CONTEXT, + INVALIDATE, + RESET_VECTOR, + BAD_FLUSH +} control_kind; +class program_image; + int version = 0; +endclass +class control_item extends uvm_sequence_item; + `uvm_object_utils(control_item) + control_kind kind; + longint unsigned target, satp; + int privilege, version; + bit sum, mxr; + function new(string name = "control_item"); + super.new(name); + endfunction +endclass +class control_sequence extends uvm_sequence #(control_item); + `uvm_object_utils(control_sequence) + control_item item; + function new(string name = "control_sequence"); + super.new(name); + endfunction + task body(); + start_item(item); + finish_item(item); + endtask +endclass +class sample_item extends uvm_sequence_item; + `uvm_object_utils(sample_item) + logic [`FETCH_INPUT_BITS-1:0] inputs; + logic [`FETCH_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "sample_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + logic [`FETCH_PACKET_BITS-1:0] payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass +class word_transaction; + longint unsigned pc, address; + int epoch, version, due; + logic [63:0] data; + logic [511:0] line; + bit executable, pf, af; +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..3a5d4a910 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual fetch_if vif; + uvm_analysis_port #(sample_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual fetch_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Fetch monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + sample_item t; + forever begin + @(posedge vif.clock); + t = sample_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/peer.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/peer.svh new file mode 100644 index 000000000..f94d41949 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/agent/peer.svh @@ -0,0 +1,95 @@ +class peer_driver extends uvm_component; + `uvm_component_utils(peer_driver) + virtual fetch_if vif; + model reference; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual fetch_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Fetch backend interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + #1; + vif.inputs[`FETCH_IN_PACKET_READY_OFFSET+:`FETCH_IN_PACKET_READY_WIDTH]=!vif.reset&&reference.packet_ready&&reference.cycle%5>=2; + vif.inputs[`FETCH_IN_REQUEST_READY_OFFSET+:`FETCH_IN_REQUEST_READY_WIDTH]=!vif.reset&&!reference.hold_request&&reference.cycle%4!=0; + vif.inputs[`FETCH_IN_RESPONSE_VALID_OFFSET+:`FETCH_IN_RESPONSE_VALID_WIDTH]=!vif.reset&&reference.word!=null&&!reference.hold_word&&reference.cycle>=reference.word.due; + if (reference.word != null) begin + vif.inputs[`FETCH_IN_RESPONSE_BITS_DATA_OFFSET+:`FETCH_IN_RESPONSE_BITS_DATA_WIDTH]=reference.word.data; + vif.inputs[`FETCH_IN_RESPONSE_BITS_LINE_OFFSET+:`FETCH_IN_RESPONSE_BITS_LINE_WIDTH]=reference.word.line; + vif.inputs[`FETCH_IN_RESPONSE_BITS_LINEEXECUTABLE_OFFSET+:`FETCH_IN_RESPONSE_BITS_LINEEXECUTABLE_WIDTH]=reference.word.executable; + vif.inputs[`FETCH_IN_RESPONSE_BITS_PAGEFAULT_OFFSET+:`FETCH_IN_RESPONSE_BITS_PAGEFAULT_WIDTH]=reference.word.pf; + vif.inputs[`FETCH_IN_RESPONSE_BITS_ACCESSFAULT_OFFSET+:`FETCH_IN_RESPONSE_BITS_ACCESSFAULT_WIDTH]=reference.word.af; + end + vif.inputs[`FETCH_IN_MAINTENANCE_READY_OFFSET+:`FETCH_IN_MAINTENANCE_READY_WIDTH]=!vif.reset&&!reference.hold_maintenance; + vif.inputs[`FETCH_IN_MAINTAINED_VALID_OFFSET+:`FETCH_IN_MAINTAINED_VALID_WIDTH]=!vif.reset&&reference.maintenance_active&&!reference.hold_ack&&reference.cycle>=reference.ack_due; + vif.inputs[`FETCH_IN_MAINTAINED_BITS_OFFSET+:`FETCH_IN_MAINTAINED_BITS_WIDTH] = 1; + end + endtask +endclass +class invalid_peer extends uvm_component; + `uvm_component_utils(invalid_peer) + virtual fetch_if vif; + program_image image; + bit pending_word = 0, pending_ack = 0; + longint unsigned address; + string scenario; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual fetch_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Fetch assertion backend is required") + if (!$value$plusargs("CASE=%s", scenario)) + `uvm_fatal("CASE", "Fetch assertion scenario is required") + endfunction + task run_phase(uvm_phase phase); + logic [511:0] line; + forever begin + @(posedge vif.clock); + if (vif.reset) begin + pending_word = 0; + pending_ack = 0; + end else begin + if ( + `FO(vif.outputs, REQUEST_VALID) + && vif.inputs[`FETCH_IN_REQUEST_READY_OFFSET+:`FETCH_IN_REQUEST_READY_WIDTH]) begin + pending_word = 1; + address = `FO(vif.outputs, REQUEST_BITS_ADDR); + end + if (vif.inputs[`FETCH_IN_RESPONSE_VALID_OFFSET+:`FETCH_IN_RESPONSE_VALID_WIDTH] && + `FO(vif.outputs, RESPONSE_READY) + ) + pending_word = 0; + if ( + `FO(vif.outputs, MAINTENANCE_VALID) + && vif.inputs[`FETCH_IN_MAINTENANCE_READY_OFFSET+:`FETCH_IN_MAINTENANCE_READY_WIDTH]) + pending_ack = 1; + if (vif.inputs[`FETCH_IN_MAINTAINED_VALID_OFFSET+:`FETCH_IN_MAINTAINED_VALID_WIDTH] && + `FO(vif.outputs, MAINTAINED_READY) + ) + pending_ack = 0; + end + @(negedge vif.clock); + #1; + vif.inputs[`FETCH_IN_PACKET_READY_OFFSET+:`FETCH_IN_PACKET_READY_WIDTH] = 0; + vif.inputs[`FETCH_IN_REQUEST_READY_OFFSET+:`FETCH_IN_REQUEST_READY_WIDTH] = !vif.reset; + vif.inputs[`FETCH_IN_RESPONSE_VALID_OFFSET+:`FETCH_IN_RESPONSE_VALID_WIDTH]=pending_word&&!vif.reset; + vif.inputs[`FETCH_IN_RESPONSE_BITS_DATA_OFFSET+:`FETCH_IN_RESPONSE_BITS_DATA_WIDTH]=fetch_ref_word( + address, image.version); + for (int word = 0; word < 8; word++) + line[word*64+:64] = fetch_ref_word((address & ~64'h3f) + word * 8, image.version); + vif.inputs[`FETCH_IN_RESPONSE_BITS_LINE_OFFSET+:`FETCH_IN_RESPONSE_BITS_LINE_WIDTH] = line; + vif.inputs[`FETCH_IN_RESPONSE_BITS_LINEEXECUTABLE_OFFSET+:`FETCH_IN_RESPONSE_BITS_LINEEXECUTABLE_WIDTH]=0; + vif.inputs[`FETCH_IN_RESPONSE_BITS_PAGEFAULT_OFFSET+:`FETCH_IN_RESPONSE_BITS_PAGEFAULT_WIDTH]=0; + vif.inputs[`FETCH_IN_RESPONSE_BITS_ACCESSFAULT_OFFSET+:`FETCH_IN_RESPONSE_BITS_ACCESSFAULT_WIDTH]=0; + vif.inputs[`FETCH_IN_MAINTENANCE_READY_OFFSET+:`FETCH_IN_MAINTENANCE_READY_WIDTH]=!vif.reset; + vif.inputs[`FETCH_IN_MAINTAINED_VALID_OFFSET+:`FETCH_IN_MAINTAINED_VALID_WIDTH]=pending_ack&&!vif.reset; + vif.inputs[`FETCH_IN_MAINTAINED_BITS_OFFSET+:`FETCH_IN_MAINTAINED_BITS_WIDTH]=scenario!="ack"; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/env/env.svh new file mode 100644 index 000000000..86c211c1b --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/env/env.svh @@ -0,0 +1,40 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver control; + monitor mon; + peer_driver peer; + model reference; + program_image image; + reset_controller reset; + uvm_sequencer #(control_item) seqr; + result_board #(result_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + image = new; + control = driver::type_id::create("control", this); + mon = monitor::type_id::create("monitor", this); + peer = peer_driver::type_id::create("peer", this); + reference = model::type_id::create("model", this); + reset = reset_controller::type_id::create("reset", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + control.seq_item_port.connect(seqr.seq_item_export); + control.image = image; + reference.image = image; + mon.ap.connect(reference.analysis_export); + peer.reference = reference; + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + reference.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/env/model.svh new file mode 100644 index 000000000..0cd1259a8 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/env/model.svh @@ -0,0 +1,224 @@ +class model extends uvm_subscriber #(sample_item); + `uvm_component_utils(model) + program_image image; + sample_item latest; + logic [`FETCH_INPUT_BITS-1:0] previous_inputs = '0; + longint unsigned cursor = 0, last_pc = 0; + bit last_pf = 0, last_af = 0, last_valid = 0, offered_packet = 0; + int epoch = 0, cycle = 0, checked = 0, reads = 0, replies = 0, reset_words = 0, drained_words = 0; + int maintenance_requests = 0, maintenance_acks = 0, reset_maintenance = 0, ack_due = 0; + bit needs_maintenance = 0, maintenance_active = 0; + word_transaction offer, word; + bit + packet_ready = 0, + hold_request = 0, + hold_word = 0, + hold_maintenance = 0, + hold_ack = 0, + line_executable = 1; + bit pf_enabled = 0, af_enabled = 0; + longint unsigned pf_address = 0, af_address = 0; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + covergroup behavior with function sample ( + int privilege, + bit packet_stall, + bit word_stall, + bit redirect, + bit fence, + bit invalidate, + int mask, + bit pf, + bit af, + bit executable, + bit packet_valid, + bit response_valid + ); + option.per_instance = 1; + context_privilege: coverpoint privilege { + bins user_mode = {0}; bins supervisor = {1}; bins machine_mode = {3}; + } + cpu_backpressure: coverpoint packet_stall; + word_backpressure: coverpoint word_stall; + redirect_event: coverpoint redirect; + fence_event: coverpoint fence; + invalidate_event: coverpoint invalidate; + page_fault: coverpoint pf iff (packet_valid); + access_fault: coverpoint af iff (packet_valid); + executable_line: coverpoint executable iff (response_valid); + packet_mask: coverpoint mask iff (packet_valid) {bins legal[] = `FETCH_MASK_VALUES;} + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + behavior = new(); + endfunction + function void write(sample_item t); + result_item expected, actual; + bit cancel, pf, af, request_fire, response_fire, maintenance_fire, ack_fire, packet_fire; + logic [`FETCH_PACKET_BITS-1:0] actual_packet; + longint unsigned expected_npc; + latest = t; + if (t.reset) begin + reset_ap.write(1); + if (word != null) reset_words++; + if (maintenance_active) reset_maintenance++; + offer = null; + word = null; + maintenance_active = 0; + needs_maintenance = 0; + offered_packet = 0; + last_valid = 0; + cursor = `FI(t.inputs, RESETVECTOR); + epoch++; + previous_inputs = t.inputs; + return; + end + cycle++; + expected_npc = `FI(t.inputs, REDIRECT_VALID) ? `FI(t.inputs, REDIRECT_BITS) : cursor; + evidence::check(`FO(t.outputs, NPC) === expected_npc, + "Fetch next-PC output disagrees with accepted CPU packets and redirects"); + // The word descriptor is formed before its physical handshake and remains stable across redirects. + if (`FO(t.outputs, REQUEST_VALID) && offer == null) begin + offer = new; + offer.pc = cursor; + offer.address = cursor & ~64'h7; + offer.epoch = epoch; + evidence::check( + `FO(t.outputs, REQUEST_BITS_ADDR) === offer.address && + `FO(t.outputs, REQUEST_BITS_CONTEXT_PRIVILEGE) === `FI(previous_inputs, CONTEXT_PRIVILEGE) + && `FO(t.outputs, REQUEST_BITS_CONTEXT_SATP) === `FI(previous_inputs, CONTEXT_SATP) && + `FO(t.outputs, REQUEST_BITS_CONTEXT_SUM) === `FI(previous_inputs, CONTEXT_SUM) && + `FO(t.outputs, REQUEST_BITS_CONTEXT_MXR) === `FI(previous_inputs, CONTEXT_MXR) && + `FO(t.outputs, REQUEST_BITS_EXECUTE) === 1'b1, + "Fetch word request lost its PC or captured address-space context"); + end + cancel = `FI(t.inputs, REDIRECT_VALID) || `FI(t.inputs, FLUSH); + if (cancel) begin + reset_ap.write(1); + offered_packet = 0; + last_valid = 0; + epoch++; + if (`FI(t.inputs, REDIRECT_VALID)) cursor = `FI(t.inputs, REDIRECT_BITS); + evidence::check(!`FO(t.outputs, PACKET_VALID), "Fetch exposed a cancelled CPU packet"); + end + if (`FI(t.inputs, FLUSH)) needs_maintenance = 1; + request_fire = `FO(t.outputs, REQUEST_VALID) && `FI(t.inputs, REQUEST_READY); + response_fire = `FI(t.inputs, RESPONSE_VALID) && `FO(t.outputs, RESPONSE_READY); + maintenance_fire = `FO(t.outputs, MAINTENANCE_VALID) && `FI(t.inputs, MAINTENANCE_READY); + ack_fire = `FI(t.inputs, MAINTAINED_VALID) && `FO(t.outputs, MAINTAINED_READY); + packet_fire = `FO(t.outputs, PACKET_VALID) && `FI(t.inputs, PACKET_READY); + if (offer != null && needs_maintenance) + evidence::check(offer.epoch != epoch, "Fetch issued a new word before maintenance completed"); + if (needs_maintenance || maintenance_active) + evidence::check(!`FO(t.outputs, PACKET_VALID), + "Fetch delivered instructions before maintenance completed"); + if (request_fire) begin + evidence::check(offer != null && word == null && !maintenance_active, + "Fetch overlapped backend word transactions"); + word = offer; + offer = null; + word.version = image.version; + word.due = cycle + 2 + reads % 3; + word.pf = pf_enabled && word.address == pf_address; + word.af = af_enabled && word.address == af_address; + word.executable = line_executable && !word.pf && !word.af; + word.data = word.pf || word.af ? 0 : fetch_ref_word(word.address, word.version); + word.line = '0; + if (!word.pf && !word.af) + for (int index = 0; index < 8; index++) + word.line[index*64+:64] = + fetch_ref_word((word.address & ~64'h3f) + index * 8, word.version); + reads++; + end + if (response_fire) begin + if (word == null) `uvm_fatal("WORD_RESPONSE", "Fetch accepted an unsolicited word") + evidence::check(`FI(t.inputs, RESPONSE_BITS_DATA) === word.data && + `FI(t.inputs, RESPONSE_BITS_LINE) === word.line && + `FI(t.inputs, RESPONSE_BITS_PAGEFAULT) === word.pf && + `FI(t.inputs, RESPONSE_BITS_ACCESSFAULT) === word.af && + `FI(t.inputs, RESPONSE_BITS_LINEEXECUTABLE) === word.executable, + "Fetch backend response disagrees with its programmed image"); + if (word.epoch == epoch) begin + last_pc = word.pc; + last_pf = word.pf; + last_af = word.af; + last_valid = 1; + end else drained_words++; + word = null; + replies++; + end + if (maintenance_fire) begin + evidence::check(word == null && offer == null && !maintenance_active && needs_maintenance && + `FO(t.outputs, MAINTENANCE_BITS), + "Fetch maintenance did not drain its outstanding word"); + maintenance_active = 1; + maintenance_requests++; + ack_due = cycle + 4; + end + if (ack_fire) begin + evidence::check(maintenance_active && `FI(t.inputs, MAINTAINED_BITS), + "Fetch consumed an invalid maintenance acknowledgement"); + maintenance_active = 0; + needs_maintenance = 0; + maintenance_acks++; + end + if (`FI(t.inputs, FLUSH)) needs_maintenance = 1; + if (!cancel && `FO(t.outputs, PACKET_VALID)) begin + pf = last_valid && last_pc == cursor ? last_pf : 0; + af = last_valid && last_pc == cursor ? last_af : 0; + expected = result_item::type_id::create("expected"); + expected.payload = '0; + expected.payload[`FETCH_OUT_PACKET_BITS_PC_OFFSET-`FETCH_PACKET_OFFSET+:`FETCH_OUT_PACKET_BITS_PC_WIDTH]=cursor; + expected.payload[`FETCH_OUT_PACKET_BITS_DATA_OFFSET-`FETCH_PACKET_OFFSET+:`FETCH_OUT_PACKET_BITS_DATA_WIDTH]=pf||af?0:fetch_ref_group( + cursor, image.version); + expected.payload[`FETCH_OUT_PACKET_BITS_MASK_OFFSET-`FETCH_PACKET_OFFSET+:`FETCH_OUT_PACKET_BITS_MASK_WIDTH]=`FETCH_INSTRUCTION_BYTES==2?(cursor[1]?2:3):1; + expected.payload[`FETCH_OUT_PACKET_BITS_PAGEFAULT_OFFSET-`FETCH_PACKET_OFFSET+:`FETCH_OUT_PACKET_BITS_PAGEFAULT_WIDTH]=pf; + expected.payload[`FETCH_OUT_PACKET_BITS_ACCESSFAULT_OFFSET-`FETCH_PACKET_OFFSET+:`FETCH_OUT_PACKET_BITS_ACCESSFAULT_WIDTH]=af; + actual_packet = t.outputs[`FETCH_PACKET_OFFSET+:`FETCH_PACKET_BITS]; + if (`FETCH_INSTRUCTION_BYTES == 2 && cursor[1]) begin + expected.payload[`FETCH_OUT_PACKET_BITS_DATA_OFFSET-`FETCH_PACKET_OFFSET+:16] = 0; + actual_packet[`FETCH_OUT_PACKET_BITS_DATA_OFFSET-`FETCH_PACKET_OFFSET+:16] = 0; + end + if (!offered_packet) begin + evidence::check(actual_packet === expected.payload, $sformatf( + "Fetch offered wrong packet pc=%h expected=%h got=%h", + cursor, + expected.payload, + actual_packet + )); + expected_ap.write(expected); + offered_packet = 1; + end + if (packet_fire) begin + actual = result_item::type_id::create("actual"); + actual.payload = actual_packet; + actual_ap.write(actual); + offered_packet = 0; + last_valid = 0; + cursor = (cursor & ~64'h3) + 4; + checked++; + end + end + behavior.sample( + `FI(t.inputs, CONTEXT_PRIVILEGE), `FO(t.outputs, PACKET_VALID) && ! + `FI(t.inputs, PACKET_READY), `FO(t.outputs, REQUEST_VALID) && !`FI(t.inputs, REQUEST_READY), + `FI(t.inputs, REDIRECT_VALID), `FI(t.inputs, FLUSH), `FI(t.inputs, INVALIDATE), + `FO(t.outputs, PACKET_BITS_MASK), `FO(t.outputs, PACKET_BITS_PAGEFAULT), + `FO(t.outputs, PACKET_BITS_ACCESSFAULT), `FI(t.inputs, RESPONSE_BITS_LINEEXECUTABLE), + `FO(t.outputs, PACKET_VALID), response_fire); + previous_inputs = t.inputs; + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + evidence::check(word == null && offer == null && !maintenance_active && !needs_maintenance, + "Fetch backend did not drain or reset"); + evidence::check( + reads==replies+reset_words&&maintenance_requests==maintenance_acks+reset_maintenance, + "Fetch backend transaction accounting mismatch"); + evidence::check(drained_words != 0 && maintenance_acks != 0, + "Fetch did not exercise cancelled-word drain and maintenance acknowledgement"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/filelist.f new file mode 100644 index 000000000..f68ec10bc --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Fetch/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/interface.sv new file mode 100644 index 000000000..87dddda2d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/interface.sv @@ -0,0 +1,20 @@ +`include "parameters.svh" +interface fetch_if ( + input logic clock +); + logic reset; + logic [`FETCH_INPUT_BITS-1:0] inputs = '0; + logic [`FETCH_OUTPUT_BITS-1:0] outputs; + assert property (@(posedge clock) disable iff (reset) + outputs[`FETCH_OUT_REQUEST_VALID_OFFSET] && !inputs[`FETCH_IN_REQUEST_READY_OFFSET] |=> + outputs[`FETCH_OUT_REQUEST_VALID_OFFSET] && $stable( + outputs[`FETCH_REQUEST_OFFSET+:`FETCH_REQUEST_BITS] + )) + else $fatal(1, "Fetch word request changed under backpressure"); + assert property (@(posedge clock) disable iff (reset || inputs[`FETCH_IN_REDIRECT_VALID_OFFSET] || inputs[`FETCH_IN_FLUSH_OFFSET]) + outputs[`FETCH_OUT_PACKET_VALID_OFFSET] && !inputs[`FETCH_IN_PACKET_READY_OFFSET] |=> + outputs[`FETCH_OUT_PACKET_VALID_OFFSET] && $stable( + outputs[`FETCH_PACKET_OFFSET+:`FETCH_PACKET_BITS] + )) + else $fatal(1, "Fetch packet changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/package.sv b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/package.sv new file mode 100644 index 000000000..692ec5f77 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/package.sv @@ -0,0 +1,26 @@ +`include "parameters.svh" +`define FI(P, N) P[`FETCH_IN_``N``_OFFSET+:`FETCH_IN_``N``_WIDTH] +`define FO(P, N) P[`FETCH_OUT_``N``_OFFSET+:`FETCH_OUT_``N``_WIDTH] +`define PK(P, + N) P[`FETCH_OUT_PACKET_BITS_``N``_OFFSET-`FETCH_PACKET_OFFSET+:`FETCH_OUT_PACKET_BITS_``N``_WIDTH] +package fetch_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function longint unsigned fetch_ref_word( + input longint unsigned address, + input int unsigned version + ); + import "DPI-C" function int unsigned fetch_ref_group( + input longint unsigned address, + input int unsigned version + ); + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "agent/peer.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..9213cad3a --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/seq/sequence.svh @@ -0,0 +1,235 @@ +class scenario_sequence extends uvm_sequence; + `uvm_object_utils(scenario_sequence) + env test_env; + virtual fetch_if vif; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task operate(control_kind kind, longint unsigned target = 0, int version = 0, int privilege = 3, + longint unsigned satp = 0, bit sum = 0, bit mxr = 0); + control_sequence seq = control_sequence::type_id::create("control"); + seq.item = control_item::type_id::create("item"); + seq.item.kind = kind; + seq.item.target = target; + seq.item.version = version; + seq.item.privilege = privilege; + seq.item.satp = satp; + seq.item.sum = sum; + seq.item.mxr = mxr; + seq.start(test_env.seqr); + endtask + task packet_at(longint unsigned target); + do @(negedge vif.clock); while (!`FO(test_env.reference.latest.outputs, PACKET_VALID)); + evidence::check(`FO(test_env.reference.latest.outputs, PACKET_BITS_PC) === target, + "Fetch packet appeared at the wrong redirected PC"); + endtask + task consume(int count); + int target = test_env.reference.checked + count; + test_env.reference.packet_ready = 1; + while (test_env.reference.checked < target) @(negedge vif.clock); + test_env.reference.packet_ready = 0; + @(negedge vif.clock); + endtask + task body(); + int baseline, target; + longint unsigned high; + operate(RESET_VECTOR, 'h1000); + test_env.reset.pulse(4); + packet_at('h1000); + consume(64); + // Fill both line buffers, then alternate held packets without a backend read. + operate(FENCE, 'h1000); + packet_at('h1000); + operate(REDIRECT, 'h1040); + packet_at('h1040); + baseline = test_env.reference.reads; + for (int n = 0; n < 8; n++) begin + operate(REDIRECT, n % 2 ? 'h1040 : 'h1000); + packet_at(n % 2 ? 'h1040 : 'h1000); + evidence::check(test_env.reference.reads == baseline, + "Fetch did not retain two executable lines"); + end + operate(REDIRECT, 'h1080); + packet_at('h1080); + baseline = test_env.reference.reads; + operate(REDIRECT, 'h1040); + packet_at('h1040); + evidence::check(test_env.reference.reads == baseline, + "Fetch replaced its most recently used line"); + operate(REDIRECT, 'h1000); + packet_at('h1000); + evidence::check(test_env.reference.reads == baseline + 1, + "Fetch did not replace its least recently used line"); + // Every individual context field participates in the line-buffer key. + for (int field = 0; field < 4; field++) begin + operate(CONTEXT, 'h1000, 0, 3, 0, 0, 0); + operate(FENCE, 'h1000); + packet_at('h1000); + baseline = test_env.reference.reads; + operate(CONTEXT, 'h1000, 0, field == 0 ? 1 : 3, field == 1 ? 64'h8000000000001234 : 0, + field == 2, field == 3); + packet_at('h1000); + evidence::check(test_env.reference.reads == baseline + 1, + "Fetch ignored an address-space context field"); + operate(CONTEXT, 'h1000, 0, 3, 0, 0, 0); + packet_at('h1000); + evidence::check(test_env.reference.reads == baseline + 1, + "Fetch lost the retained prior context"); + end + baseline = test_env.reference.reads; + operate(INVALIDATE); + operate(REDIRECT, 'h1000); + packet_at('h1000); + evidence::check(test_env.reference.reads == baseline + 1, + "Fetch did not invalidate its buffered line"); + // Redirects drain stalled offers and accepted words instead of cancelling the backend port. + test_env.reference.hold_request = 1; + operate(REDIRECT, 'h2000); + wait (`FO(test_env.reference.latest.outputs, REQUEST_VALID)); + operate(REDIRECT, 'h3000); + repeat (4) @(negedge vif.clock); + test_env.reference.hold_request = 0; + packet_at('h3000); + test_env.reference.hold_word = 1; + operate(REDIRECT, 'h4000); + wait (test_env.reference.word != null); + operate(REDIRECT, 'h5000); + repeat (4) @(negedge vif.clock); + test_env.reference.hold_word = 0; + packet_at('h5000); + // An image replacement and fence occur atomically; the older response must not refill either buffer. + test_env.reference.hold_word = 1; + operate(REDIRECT, 'h6000); + wait (test_env.reference.word != null); + test_env.reference.hold_maintenance = 1; + test_env.reference.hold_ack = 1; + operate(FENCE, 'h7000, 1); + repeat (4) @(negedge vif.clock); + evidence::check(!`FO(test_env.reference.latest.outputs, MAINTENANCE_VALID), + "Fetch maintenance overtook the outstanding word"); + test_env.reference.hold_word = 0; + wait (`FO(test_env.reference.latest.outputs, MAINTENANCE_VALID)); + repeat (4) @(negedge vif.clock); + test_env.reference.hold_maintenance = 0; + wait (test_env.reference.maintenance_active); + repeat (4) @(negedge vif.clock); + test_env.reference.hold_ack = 0; + packet_at('h7000); + operate(REDIRECT, 'h6000); + packet_at('h6000); + consume(32); + // Fault packets carry no instruction data and are never buffered. + test_env.reference.line_executable = 0; + test_env.reference.pf_enabled = 1; + test_env.reference.af_enabled = 1; + test_env.reference.pf_address = 'ha000; + test_env.reference.af_address = 'ha008; + operate(FENCE, 'ha000, 1); + packet_at('ha000); + consume(6); + test_env.reference.pf_enabled = 0; + test_env.reference.af_enabled = 0; + if (`FETCH_INSTRUCTION_BYTES == 2) begin + operate(FENCE, 'hb002, 1); + packet_at('hb002); + consume(12); + end + // Exercise every virtual-address bit without interpreting instructions in this IP. + test_env.reference.line_executable = 1; + for (int bit_index = 2; bit_index < 64; bit_index++) begin + high = 64'(1) << bit_index; + operate(FENCE, high, bit_index + 2); + packet_at(high); + consume(1); + end + // SATP mode, PPN and ASID bits are independently part of the buffered context. + for (int bit_index = 0; bit_index < 60; bit_index++) begin + operate(CONTEXT, 'h1000, 0, 1, 64'h8000000000000000); + packet_at('h1000); + baseline = test_env.reference.reads; + operate(CONTEXT, 'h1000, 0, 1, 64'h8000000000000000 | (64'(1) << bit_index)); + packet_at('h1000); + evidence::check(test_env.reference.reads == baseline + 1, "Fetch ignored a SATP context bit"); + operate(CONTEXT, 'h1000, 0, 1, 64'h8000000000000000); + packet_at('h1000); + evidence::check(test_env.reference.reads == baseline + 1, + "Fetch did not retain its baseline SATP context"); + end + operate(CONTEXT, 'h1000, 0, 0, 0); + packet_at('h1000); + consume(2); + // Reset at offer, receive and held CPU-packet boundaries. + for (int phase_index = 0; phase_index < 3; phase_index++) begin + test_env.reference.hold_request = phase_index == 0; + test_env.reference.hold_word = phase_index == 1; + operate(REDIRECT, 'hc000 + phase_index * 'h1000); + if (phase_index == 0) wait (`FO(test_env.reference.latest.outputs, REQUEST_VALID)); + else if (phase_index == 1) wait (test_env.reference.word != null); + else packet_at('he000); + test_env.reset.pulse(3); + test_env.reference.hold_request = 0; + test_env.reference.hold_word = 0; + packet_at('h1000); + consume(2); + end + test_env.reference.hold_ack = 1; + operate(FENCE, 'hf000, test_env.image.version); + wait (test_env.reference.maintenance_active); + test_env.reset.pulse(3); + test_env.reference.hold_ack = 0; + packet_at('h1000); + consume(2); + test_env.reference.hold_maintenance = 1; + operate(FENCE, 'hf000, test_env.image.version); + wait (`FO(test_env.reference.latest.outputs, MAINTENANCE_VALID)); + test_env.reset.pulse(3); + test_env.reference.hold_maintenance = 0; + packet_at('h1000); + consume(2); + test_env.reference.packet_ready = 0; + test_env.reset.pulse(3); + `uvm_info("FETCH", + $sformatf( + "packets=%0d backend_reads=%0d replies=%0d redirected_drains=%0d maintenance=%0d", + test_env.reference.checked, test_env.reference.reads, test_env.reference.replies, + test_env.reference.drained_words, test_env.reference.maintenance_acks), UVM_LOW) + endtask +endclass +class invalid_sequence extends uvm_sequence; + `uvm_object_utils(invalid_sequence) + reset_controller reset; + uvm_sequencer #(control_item) seqr; + function new(string name = "invalid_sequence"); + super.new(name); + endfunction + task command(control_kind kind, longint unsigned target = 0, int privilege = 3); + control_sequence seq = control_sequence::type_id::create("command"); + seq.item = control_item::type_id::create("item"); + seq.item.kind = kind; + seq.item.target = target; + seq.item.privilege = privilege; + seq.item.version = 0; + seq.item.satp = 0; + seq.item.sum = 0; + seq.item.mxr = 0; + seq.start(seqr); + endtask + task body(); + string scenario; + if (!$value$plusargs("CASE=%s", scenario)) + `uvm_fatal("CASE", "Fetch assertion scenario is required") + if (scenario == "reset") command(RESET_VECTOR, 1); + reset.pulse(4); + case (scenario) + "reset": begin + end + "redirect": command(REDIRECT, 1); + "flush": command(BAD_FLUSH); + "privilege": command(CONTEXT, 'h1000, 2); + "ack": command(FENCE, 0); + default: `uvm_fatal("CASE", "Unknown fetch assertion scenario") + endcase + #1000ns; + `uvm_fatal("EXPECTED_ASSERTION", "Invalid fetch input did not trigger its assertion") + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/suite.toml new file mode 100644 index 000000000..87ca37d58 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/suite.toml @@ -0,0 +1,105 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libfetch_reference_model" + +[[tests]] +functional_groups = ["fetch_pkg::model::behavior"] +name = "fetch" +top = "fetch_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "fetch_bad_ack" +top = "fetch_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=ack"] +assertion = "Fetch maintenance response must acknowledge completion" + +[[tests]] +functional_groups = [] +name = "fetch_bad_flush" +top = "fetch_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=flush"] +assertion = "Fetch flush requires a restart redirect" + +[[tests]] +functional_groups = [] +name = "fetch_bad_privilege" +top = "fetch_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=privilege"] +assertion = "Fetch privilege must be U, S or M" + +[[tests]] +functional_groups = [] +name = "fetch_bad_redirect" +top = "fetch_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=redirect"] +assertion = "Fetch redirect must be instruction aligned" + +[[tests]] +functional_groups = [] +name = "fetch_bad_reset" +top = "fetch_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=reset"] +assertion = "Fetch reset vector must be instruction aligned" diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/tb.sv new file mode 100644 index 000000000..432235910 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/tb.sv @@ -0,0 +1,19 @@ +module fetch_tb; + import uvm_pkg::*; + import fetch_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + fetch_if bus (clock); + assign bus.reset = timing.reset; + Fetch dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual fetch_if)::set(null, "uvm_test_top*", "vif", bus); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/tests/test.svh new file mode 100644 index 000000000..bfc358fee --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/resources/tests/test.svh @@ -0,0 +1,50 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + virtual fetch_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + if (!uvm_config_db#(virtual fetch_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Fetch timing interface is required") + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.vif = vif; + seq.start(null); + endtask +endclass +class invalid_test extends base_test; + `uvm_component_utils(invalid_test) + driver control; + invalid_peer peer; + program_image image; + reset_controller reset; + uvm_sequencer #(control_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + image = new; + control = driver::type_id::create("control", this); + peer = invalid_peer::type_id::create("peer", this); + reset = reset_controller::type_id::create("reset", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + control.seq_item_port.connect(seqr.seq_item_export); + control.image = image; + peer.image = image; + endfunction + task execute(); + invalid_sequence seq = invalid_sequence::type_id::create("violation"); + seq.reset = reset; + seq.seqr = seqr; + seq.start(null); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/Emit.scala new file mode 100644 index 000000000..da795ae24 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/Emit.scala @@ -0,0 +1,69 @@ +package framework.core.execution.fetch + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.core.execution.fetch.configs.Params + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val p = upickle.default.read[Params](Files.readString(Paths.get(config))) + Files.createDirectories(Paths.get(output)) + val ports = scala.collection.mutable.ArrayBuffer[String]() + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + val widths = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0 + var outputOffset = 0 + + def fields(data: Data, path: String): Seq[(String, Int)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(path -> leaf.getWidth) + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new Fetch(p) + for ((name, width) <- fields(dut.io, "")) { + val input = Set( + "resetVector", + "flush", + "invalidate", + "redirect_valid", + "redirect_bits", + "packet_ready", + "request_ready", + "response_valid", + "maintenance_ready", + "maintained_valid", + "maintained_bits" + ).contains(name) || name.startsWith("context_") || name.startsWith("response_bits_") + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define FETCH_${direction}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define FETCH_${direction}_${name.toUpperCase}_WIDTH $width" + ports += s".io_$name(bus.$signal[$offset +: $width])" + offsets(name) = offset; widths(name) = width + if (input) inputOffset += width else outputOffset += width + } + dut + }, + args = Array("--target-dir", s"$output/Fetch", "--split-verilog") + ) + definitions += s"`define FETCH_INPUT_BITS $inputOffset" + definitions += s"`define FETCH_OUTPUT_BITS $outputOffset" + definitions += s"`define FETCH_INSTRUCTION_BYTES ${p.instructionBytes}" + definitions += s"`define FETCH_MASK_VALUES ${if (p.instructionBytes == 2) "{2,3}" else "{1}"}" + for ( + (name, prefix) <- Seq("REQUEST" -> "request_bits_", "RESPONSE" -> "response_bits_", "PACKET" -> "packet_bits_") + ) { + val names = offsets.keys.filter(_.startsWith(prefix)).toSeq + definitions += s"`define FETCH_${name}_OFFSET ${names.map(offsets).min}" + definitions += s"`define FETCH_${name}_BITS ${names.map(widths).sum}" + } + Files.writeString(Paths.get(output, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/scala/Fetch.scala b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/Fetch.scala similarity index 93% rename from arch/src/main/scala/framework/mem-core/fetch/src/main/scala/Fetch.scala rename to arch/src/main/scala/framework/core/execution/fetch/src/main/scala/Fetch.scala index bcabb3243..d8b85f370 100644 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/scala/Fetch.scala +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/Fetch.scala @@ -1,13 +1,10 @@ -package memcore.memory.fetch +package framework.core.execution.fetch import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -case class Params(instructionBytes: Int = 2) { - require(instructionBytes == 2 || instructionBytes == 4) - val lanes: Int = 4 / instructionBytes -} +import framework.core.execution.fetch.configs.{Params => ModuleParam} class Context extends Bundle { val privilege = UInt(2.W) @@ -31,7 +28,9 @@ class Response extends Bundle { val accessFault = Bool() } -class Packet(p: Params) extends Bundle { +class Packet(b: ModuleParam) extends Bundle { + val p = b + val pc = UInt(64.W) val data = UInt(32.W) val mask = UInt(p.lanes.W) @@ -45,8 +44,10 @@ class Packet(p: Params) extends Bundle { * a request, which replaces the less recently used line. Lines are keyed by virtual line and fetch * context, and are dropped by fence.i (`flush`) and sfence.vma (`invalidate`). */ + @instantiable -class Fetch(p: Params) extends Module { +class Fetch(b: ModuleParam) extends Module { + val p = b @public val io = IO(new Bundle { @@ -55,7 +56,7 @@ class Fetch(p: Params) extends Module { val redirect = Flipped(Valid(UInt(64.W))) val flush = Input(Bool()) val invalidate = Input(Bool()) - val packet = Decoupled(new Packet(p)) + val packet = Decoupled(new Packet(b)) val request = Decoupled(new Request) val response = Flipped(Decoupled(new Response)) val maintenance = Decoupled(Bool()) @@ -68,7 +69,7 @@ class Fetch(p: Params) extends Module { val pc = Reg(UInt(64.W)) val readPc = Reg(UInt(64.W)) val command = Reg(new Request) - val answer = Reg(new Packet(p)) + val answer = Reg(new Packet(b)) val discard = RegInit(false.B) val pendingMaintenance = RegInit(false.B) val cancelled = io.redirect.valid || io.flush @@ -89,7 +90,7 @@ class Fetch(p: Params) extends Module { def bufferedLine(addr: UInt): UInt = Mux(hits(addr)(1), bufferData(1), bufferData(0)) def packetAt(addr: UInt, data: UInt): Packet = { - val packet = Wire(new Packet(p)) + val packet = Wire(new Packet(b)) packet.pc := addr packet.data := (data >> (addr(5, 2) << 5))(31, 0) packet.mask := (if (p.instructionBytes == 2) (3.U(2.W) << addr(1))(1, 0) else 1.U) @@ -143,7 +144,7 @@ class Fetch(p: Params) extends Module { state := idle }.otherwise { answer := packetAt(readPc, io.response.bits.line) - answer.data := Mux(fault, 0.U, (io.response.bits.line >> (readPc(5, 2) << 5))(31, 0)) + answer.data := Mux(fault, 0.U, (io.response.bits.data >> (readPc(2) << 5))(31, 0)) answer.pageFault := io.response.bits.pageFault answer.accessFault := io.response.bits.accessFault state := deliver diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..2c00df7b1 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/Params.scala @@ -0,0 +1,11 @@ +package framework.core.execution.fetch.configs + +import chisel3.util._ +import upickle.default.{macroRW, ReadWriter} + +case class Params(instructionBytes: Int) { + require(instructionBytes == 2 || instructionBytes == 4) + val lanes: Int = 4 / instructionBytes +} + +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/default.toml b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/default.toml new file mode 100644 index 000000000..37359c823 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/default.toml @@ -0,0 +1 @@ +instructionBytes = 2 diff --git a/arch/src/main/scala/framework/core/execution/fpu/build.sc b/arch/src/main/scala/framework/core/execution/fpu/build.sc new file mode 100644 index 000000000..4bb2d1ef2 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.execution.fpu.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/FPU.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/execution/fpu/configs/uvm.toml b/arch/src/main/scala/framework/core/execution/fpu/configs/uvm.toml new file mode 100644 index 000000000..422e3e9f3 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/configs/uvm.toml @@ -0,0 +1,5 @@ +[parameters] +xLen = 64 +fLen = 64 + +[defines] diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/csrc/Cargo.lock b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/Cargo.lock new file mode 100644 index 000000000..90aa159e8 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "fpu-reference" +version = "0.1.0" diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/csrc/Cargo.toml b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/Cargo.toml new file mode 100644 index 000000000..20557768f --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "fpu-reference" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/csrc/src/lib.rs b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/src/lib.rs new file mode 100644 index 000000000..b78e3f544 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/src/lib.rs @@ -0,0 +1,58 @@ +mod vectors; +type Input = (u32, u64, u64, u64, u64, u32); +type Output = (u64, u32, bool, bool); + +#[no_mangle] +pub extern "C" fn fpu_case_count() -> u32 { + vectors::CASES.len() as u32 +} + +#[no_mangle] +pub extern "C" fn fpu_cancel_case() -> u32 { + vectors::CASES + .iter() + .position(|(input, _)| input.0 & 0x7f == 0x53 && input.0 >> 27 == 3) + .unwrap() as u32 +} + +#[no_mangle] +pub unsafe extern "C" fn fpu_case( + index: u32, + instruction: *mut u32, + integer: *mut u64, + a: *mut u64, + rhs: *mut u64, + c: *mut u64, + rounding: *mut u32, +) { + let input = vectors::CASES[index as usize].0; + *instruction = input.0; + *integer = input.1; + *a = input.2; + *rhs = input.3; + *c = input.4; + *rounding = input.5; +} + +#[no_mangle] +pub unsafe extern "C" fn fpu_expected( + instruction: u32, + integer: u64, + a: u64, + rhs: u64, + c: u64, + rounding: u32, + data: *mut u64, + flags: *mut u32, + is_integer: *mut u8, + illegal: *mut u8, +) { + let observed = (instruction, integer, a, rhs, c, rounding); + let Some((_, expected)) = vectors::CASES.iter().find(|(input, _)| *input == observed) else { + panic!("No reference for observed FPU request: {observed:?}"); + }; + *data = expected.0; + *flags = expected.1; + *is_integer = u8::from(expected.2); + *illegal = u8::from(expected.3); +} diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/csrc/src/vectors.rs b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/src/vectors.rs new file mode 100644 index 000000000..73dc800be --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/csrc/src/vectors.rs @@ -0,0 +1,3485 @@ +use super::{Input, Output}; + +// instruction, integer, a, rhs, c, rounding -> data, flags, integer, illegal +pub const CASES: &[(Input, Output)] = &[ + // S exact arithmetic + ( + ( + 0x00000053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40400000, 0, false, false), + ), + // S exact arithmetic + ( + ( + 0x08000053, + 0x0000000000000000, + 0xffffffff40400000, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 0, false, false), + ), + // S exact arithmetic + ( + ( + 0x10000053, + 0x0000000000000000, + 0xffffffff3fc00000, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40400000, 0, false, false), + ), + // S exact arithmetic + ( + ( + 0x18000053, + 0x0000000000000000, + 0xffffffff40c00000, + 0xffffffff40400000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40000000, 0, false, false), + ), + // S sqrt 4 + ( + ( + 0x58000053, + 0x0000000000000000, + 0xffffffff40800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40000000, 0, false, false), + ), + // S sqrt negative + ( + ( + 0x58000053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 16, false, false), + ), + // S divide by zero + ( + ( + 0x18000053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7f800000, 8, false, false), + ), + // S zero divided by zero + ( + ( + 0x18000053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 16, false, false), + ), + // S infinity divided by infinity + ( + ( + 0x18000053, + 0x0000000000000000, + 0xffffffff7f800000, + 0xffffffff7f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 16, false, false), + ), + // S opposite infinities + ( + ( + 0x00000053, + 0x0000000000000000, + 0xffffffff7f800000, + 0xffffffffff800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 16, false, false), + ), + // S quiet NaN arithmetic + ( + ( + 0x00000053, + 0x0000000000000000, + 0xffffffff7fc00011, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 0, false, false), + ), + // S signaling NaN arithmetic + ( + ( + 0x00000053, + 0x0000000000000000, + 0xffffffff7f800001, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 16, false, false), + ), + // S fused sign variants + ( + ( + 0x00000043, + 0x0000000000000000, + 0xffffffff40000000, + 0xffffffff40400000, + 0xffffffff40800000, + 0, + ), + (0xffffffff41200000, 0, false, false), + ), + // S fused sign variants + ( + ( + 0x00000047, + 0x0000000000000000, + 0xffffffff40000000, + 0xffffffff40400000, + 0xffffffff40800000, + 0, + ), + (0xffffffff40000000, 0, false, false), + ), + // S fused sign variants + ( + ( + 0x0000004b, + 0x0000000000000000, + 0xffffffff40000000, + 0xffffffff40400000, + 0xffffffff40800000, + 0, + ), + (0xffffffffc0000000, 0, false, false), + ), + // S fused sign variants + ( + ( + 0x0000004f, + 0x0000000000000000, + 0xffffffff40000000, + 0xffffffff40400000, + 0xffffffff40800000, + 0, + ), + (0xffffffffc1200000, 0, false, false), + ), + // S fused cancellation retains product precision + ( + ( + 0x00000043, + 0x0000000000000000, + 0xffffffff3f800001, + 0xffffffff3f7ffffe, + 0xffffffffbf800000, + 0, + ), + (0xffffffffa8800000, 0, false, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00000053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00000053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 1, false, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 1, false, false), + ), + // S positive overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff7f7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7f800000, 5, false, false), + ), + // S negative overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffffff7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffff800000, 5, false, false), + ), + // S positive tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff00000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff00000000, 3, false, false), + ), + // S negative tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff80000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff80000000, 3, false, false), + ), + // S cancellation zero sign + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff80000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff00000000, 0, false, false), + ), + // S float to signed integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffff3fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000002, 1, true, false), + ), + // S negative integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffffbfc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xfffffffffffffffe, 1, true, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00001053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00001053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 1, false, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 1, + ), + (0xffffffff3f800000, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 1, + ), + (0xffffffffbf800000, 1, false, false), + ), + // S positive overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff7f7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 1, + ), + (0xffffffff7f7fffff, 5, false, false), + ), + // S negative overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffffff7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 1, + ), + (0xffffffffff7fffff, 5, false, false), + ), + // S positive tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff00000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 1, + ), + (0xffffffff00000000, 3, false, false), + ), + // S negative tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff80000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 1, + ), + (0xffffffff80000000, 3, false, false), + ), + // S cancellation zero sign + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff80000000, + 0xffffffff00000000, + 1, + ), + (0xffffffff00000000, 0, false, false), + ), + // S float to signed integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffff3fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 1, + ), + (0x0000000000000001, 1, true, false), + ), + // S negative integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffffbfc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 1, + ), + (0xffffffffffffffff, 1, true, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00002053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00002053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800001, 1, false, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 2, + ), + (0xffffffff3f800000, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 2, + ), + (0xffffffffbf800001, 1, false, false), + ), + // S positive overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff7f7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 2, + ), + (0xffffffff7f7fffff, 5, false, false), + ), + // S negative overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffffff7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 2, + ), + (0xffffffffff800000, 5, false, false), + ), + // S positive tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff00000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 2, + ), + (0xffffffff00000000, 3, false, false), + ), + // S negative tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff80000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 2, + ), + (0xffffffff80000001, 3, false, false), + ), + // S cancellation zero sign + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff80000000, + 0xffffffff00000000, + 2, + ), + (0xffffffff80000000, 0, false, false), + ), + // S float to signed integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffff3fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 2, + ), + (0x0000000000000001, 1, true, false), + ), + // S negative integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffffbfc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 2, + ), + (0xfffffffffffffffe, 1, true, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00003053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800001, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00003053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 1, false, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 3, + ), + (0xffffffff3f800001, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 3, + ), + (0xffffffffbf800000, 1, false, false), + ), + // S positive overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff7f7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 3, + ), + (0xffffffff7f800000, 5, false, false), + ), + // S negative overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffffff7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 3, + ), + (0xffffffffff7fffff, 5, false, false), + ), + // S positive tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff00000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 3, + ), + (0xffffffff00000001, 3, false, false), + ), + // S negative tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff80000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 3, + ), + (0xffffffff80000000, 3, false, false), + ), + // S cancellation zero sign + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff80000000, + 0xffffffff00000000, + 3, + ), + (0xffffffff00000000, 0, false, false), + ), + // S float to signed integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffff3fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 3, + ), + (0x0000000000000002, 1, true, false), + ), + // S negative integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffffbfc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 3, + ), + (0xffffffffffffffff, 1, true, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00004053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800001, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00004053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800001, 1, false, false), + ), + // S positive half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff33800000, + 0xffffffff00000000, + 4, + ), + (0xffffffff3f800001, 1, false, false), + ), + // S negative half-ulp rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffffb3800000, + 0xffffffff00000000, + 4, + ), + (0xffffffffbf800001, 1, false, false), + ), + // S positive overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff7f7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 4, + ), + (0xffffffff7f800000, 5, false, false), + ), + // S negative overflow + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffffff7fffff, + 0xffffffff40000000, + 0xffffffff00000000, + 4, + ), + (0xffffffffff800000, 5, false, false), + ), + // S positive tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff00000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 4, + ), + (0xffffffff00000001, 3, false, false), + ), + // S negative tiny tie + ( + ( + 0x10007053, + 0x0000000000000000, + 0xffffffff80000001, + 0xffffffff3f000000, + 0xffffffff00000000, + 4, + ), + (0xffffffff80000001, 3, false, false), + ), + // S cancellation zero sign + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff80000000, + 0xffffffff00000000, + 4, + ), + (0xffffffff00000000, 0, false, false), + ), + // S float to signed integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffff3fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 4, + ), + (0x0000000000000002, 1, true, false), + ), + // S negative integer tie + ( + ( + 0xc0207053, + 0x0000000000000000, + 0xffffffffbfc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 4, + ), + (0xfffffffffffffffe, 1, true, false), + ), + // S reserved instruction rounding + ( + ( + 0x00005053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // S reserved instruction rounding + ( + ( + 0x00006053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // S reserved dynamic rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 5, + ), + (0x0000000000000000, 0, false, true), + ), + // S reserved dynamic rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 6, + ), + (0x0000000000000000, 0, false, true), + ), + // S reserved dynamic rounding + ( + ( + 0x00007053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 7, + ), + (0x0000000000000000, 0, false, true), + ), + // S sign injection + ( + ( + 0x20000053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffffc0000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 0, false, false), + ), + // S sign injection + ( + ( + 0x20001053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffffc0000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 0, false, false), + ), + // S sign injection + ( + ( + 0x20002053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffffc0000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 0, false, false), + ), + // S sign injection preserves payload + ( + ( + 0x20000053, + 0x0000000000000000, + 0xffffffff7fc00011, + 0xffffffffc0000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffffc00011, 0, false, false), + ), + // S min/max signed zero + ( + ( + 0x28000053, + 0x0000000000000000, + 0xffffffff80000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff80000000, 0, false, false), + ), + // S min/max finite + ( + ( + 0x28000053, + 0x0000000000000000, + 0xffffffffc0000000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffffc0000000, 0, false, false), + ), + // S min/max quiet NaN + ( + ( + 0x28000053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 0, false, false), + ), + // S min/max signaling NaN + ( + ( + 0x28000053, + 0x0000000000000000, + 0xffffffff7f800001, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 16, false, false), + ), + // S min/max two NaNs + ( + ( + 0x28000053, + 0x0000000000000000, + 0xffffffff7fc00003, + 0xffffffff7fc00007, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 0, false, false), + ), + // S min/max signed zero + ( + ( + 0x28001053, + 0x0000000000000000, + 0xffffffff80000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff00000000, 0, false, false), + ), + // S min/max finite + ( + ( + 0x28001053, + 0x0000000000000000, + 0xffffffffc0000000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 0, false, false), + ), + // S min/max quiet NaN + ( + ( + 0x28001053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 0, false, false), + ), + // S min/max signaling NaN + ( + ( + 0x28001053, + 0x0000000000000000, + 0xffffffff7f800001, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3f800000, 16, false, false), + ), + // S min/max two NaNs + ( + ( + 0x28001053, + 0x0000000000000000, + 0xffffffff7fc00003, + 0xffffffff7fc00007, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 0, false, false), + ), + // S comparison ordered + ( + ( + 0xa0000053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // S comparison signed zero + ( + ( + 0xa0000053, + 0x0000000000000000, + 0xffffffff80000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // S comparison quiet NaN + ( + ( + 0xa0000053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // S comparison signaling NaN + ( + ( + 0xa0000053, + 0x0000000000000000, + 0xffffffff7f800001, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // S comparison ordered + ( + ( + 0xa0001053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // S comparison signed zero + ( + ( + 0xa0001053, + 0x0000000000000000, + 0xffffffff80000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, true, false), + ), + // S comparison quiet NaN + ( + ( + 0xa0001053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // S comparison signaling NaN + ( + ( + 0xa0001053, + 0x0000000000000000, + 0xffffffff7f800001, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // S comparison ordered + ( + ( + 0xa0002053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, true, false), + ), + // S comparison signed zero + ( + ( + 0xa0002053, + 0x0000000000000000, + 0xffffffff80000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // S comparison quiet NaN + ( + ( + 0xa0002053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, true, false), + ), + // S comparison signaling NaN + ( + ( + 0xa0002053, + 0x0000000000000000, + 0xffffffff7f800001, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // S classification 0 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffffff800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // S classification 1 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffffbf800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000002, 0, true, false), + ), + // S classification 2 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff80000001, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000004, 0, true, false), + ), + // S classification 3 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff80000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000008, 0, true, false), + ), + // S classification 4 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000010, 0, true, false), + ), + // S classification 5 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff00000001, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000020, 0, true, false), + ), + // S classification 6 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000040, 0, true, false), + ), + // S classification 7 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff7f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000080, 0, true, false), + ), + // S classification 8 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff7f800001, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000100, 0, true, false), + ), + // S classification 9 + ( + ( + 0xe0001053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000200, 0, true, false), + ), + // S move float bits to integer + ( + ( + 0xe0000053, + 0x0000000000000000, + 0x0000000080000001, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff80000001, 0, true, false), + ), + // S move integer bits to float + ( + ( + 0xf0000053, + 0x0000000080000001, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff80000001, 0, false, false), + ), + // S integer conversion type 0 + ( + ( + 0xd0000053, + 0x0000000000000003, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40400000, 0, false, false), + ), + // S float conversion type 0 + ( + ( + 0xc0000053, + 0x0000000000000000, + 0xffffffff40400000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0000053, + 0x0000000000000000, + 0xffffffff7f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x000000007fffffff, 16, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0000053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x000000007fffffff, 16, true, false), + ), + // S invalid negative conversion + ( + ( + 0xc0000053, + 0x0000000000000000, + 0xffffffffff800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff80000000, 16, true, false), + ), + // S integer conversion boundary + ( + ( + 0xd0000053, + 0xffffffffffffffff, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 0, false, false), + ), + // S integer conversion type 1 + ( + ( + 0xd0100053, + 0x0000000000000003, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40400000, 0, false, false), + ), + // S float conversion type 1 + ( + ( + 0xc0100053, + 0x0000000000000000, + 0xffffffff40400000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0100053, + 0x0000000000000000, + 0xffffffff7f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0100053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // S invalid negative conversion + ( + ( + 0xc0100053, + 0x0000000000000000, + 0xffffffffff800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // S integer conversion boundary + ( + ( + 0xd0100053, + 0xffffffffffffffff, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff4f800000, 1, false, false), + ), + // S integer conversion type 2 + ( + ( + 0xd0200053, + 0x0000000000000003, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40400000, 0, false, false), + ), + // S float conversion type 2 + ( + ( + 0xc0200053, + 0x0000000000000000, + 0xffffffff40400000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0200053, + 0x0000000000000000, + 0xffffffff7f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x7fffffffffffffff, 16, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0200053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x7fffffffffffffff, 16, true, false), + ), + // S invalid negative conversion + ( + ( + 0xc0200053, + 0x0000000000000000, + 0xffffffffff800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x8000000000000000, 16, true, false), + ), + // S integer conversion boundary + ( + ( + 0xd0200053, + 0xffffffffffffffff, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffbf800000, 0, false, false), + ), + // S integer conversion type 3 + ( + ( + 0xd0300053, + 0x0000000000000003, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff40400000, 0, false, false), + ), + // S float conversion type 3 + ( + ( + 0xc0300053, + 0x0000000000000000, + 0xffffffff40400000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0300053, + 0x0000000000000000, + 0xffffffff7f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // S invalid positive conversion + ( + ( + 0xc0300053, + 0x0000000000000000, + 0xffffffff7fc00000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // S invalid negative conversion + ( + ( + 0xc0300053, + 0x0000000000000000, + 0xffffffffff800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // S integer conversion boundary + ( + ( + 0xd0300053, + 0xffffffffffffffff, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff5f800000, 1, false, false), + ), + // S cross-format exact + ( + ( + 0x40100053, + 0x0000000000000000, + 0x3ff8000000000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0xffffffff3fc00000, 0, false, false), + ), + // S unboxed single is quiet NaN + ( + ( + 0x00000053, + 0x0000000000000000, + 0x000000003f800000, + 0xffffffff3f800000, + 0xffffffff00000000, + 0, + ), + (0xffffffff7fc00000, 0, false, false), + ), + // S unboxed single classification + ( + ( + 0xe0001053, + 0x0000000000000000, + 0x000000003f800000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000200, 0, true, false), + ), + // S unsupported format + ( + ( + 0x04000053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // S unsupported format + ( + ( + 0x06000053, + 0x0000000000000000, + 0xffffffff3f800000, + 0xffffffff40000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // S unsupported operation + ( + ( + 0xf8000053, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // S unsupported opcode + ( + ( + 0x00000000, + 0x0000000000000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0xffffffff00000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // D exact arithmetic + ( + ( + 0x02000053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x4008000000000000, 0, false, false), + ), + // D exact arithmetic + ( + ( + 0x0a000053, + 0x0000000000000000, + 0x4008000000000000, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 0, false, false), + ), + // D exact arithmetic + ( + ( + 0x12000053, + 0x0000000000000000, + 0x3ff8000000000000, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x4008000000000000, 0, false, false), + ), + // D exact arithmetic + ( + ( + 0x1a000053, + 0x0000000000000000, + 0x4018000000000000, + 0x4008000000000000, + 0x0000000000000000, + 0, + ), + (0x4000000000000000, 0, false, false), + ), + // D sqrt 4 + ( + ( + 0x5a000053, + 0x0000000000000000, + 0x4010000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x4000000000000000, 0, false, false), + ), + // D sqrt negative + ( + ( + 0x5a000053, + 0x0000000000000000, + 0xbff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 16, false, false), + ), + // D divide by zero + ( + ( + 0x1a000053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff0000000000000, 8, false, false), + ), + // D zero divided by zero + ( + ( + 0x1a000053, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 16, false, false), + ), + // D infinity divided by infinity + ( + ( + 0x1a000053, + 0x0000000000000000, + 0x7ff0000000000000, + 0x7ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 16, false, false), + ), + // D opposite infinities + ( + ( + 0x02000053, + 0x0000000000000000, + 0x7ff0000000000000, + 0xfff0000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 16, false, false), + ), + // D quiet NaN arithmetic + ( + ( + 0x02000053, + 0x0000000000000000, + 0x7ff8000000000011, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 0, false, false), + ), + // D signaling NaN arithmetic + ( + ( + 0x02000053, + 0x0000000000000000, + 0x7ff0000000000001, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 16, false, false), + ), + // D fused sign variants + ( + ( + 0x02000043, + 0x0000000000000000, + 0x4000000000000000, + 0x4008000000000000, + 0x4010000000000000, + 0, + ), + (0x4024000000000000, 0, false, false), + ), + // D fused sign variants + ( + ( + 0x02000047, + 0x0000000000000000, + 0x4000000000000000, + 0x4008000000000000, + 0x4010000000000000, + 0, + ), + (0x4000000000000000, 0, false, false), + ), + // D fused sign variants + ( + ( + 0x0200004b, + 0x0000000000000000, + 0x4000000000000000, + 0x4008000000000000, + 0x4010000000000000, + 0, + ), + (0xc000000000000000, 0, false, false), + ), + // D fused sign variants + ( + ( + 0x0200004f, + 0x0000000000000000, + 0x4000000000000000, + 0x4008000000000000, + 0x4010000000000000, + 0, + ), + (0xc024000000000000, 0, false, false), + ), + // D fused cancellation retains product precision + ( + ( + 0x02000043, + 0x0000000000000000, + 0x3ff0000000000001, + 0x3feffffffffffffe, + 0xbff0000000000000, + 0, + ), + (0xb970000000000000, 0, false, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02000053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02000053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 1, false, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 1, false, false), + ), + // D positive overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0x7fefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x7ff0000000000000, 5, false, false), + ), + // D negative overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0xffefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0xfff0000000000000, 5, false, false), + ), + // D positive tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x0000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 3, false, false), + ), + // D negative tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x8000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 0, + ), + (0x8000000000000000, 3, false, false), + ), + // D cancellation zero sign + ( + ( + 0x02007053, + 0x0000000000000000, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, false), + ), + // D float to signed integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0x3ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000002, 1, true, false), + ), + // D negative integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0xbff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xfffffffffffffffe, 1, true, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02001053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02001053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 1, false, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 1, + ), + (0x3ff0000000000000, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 1, + ), + (0xbff0000000000000, 1, false, false), + ), + // D positive overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0x7fefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 1, + ), + (0x7fefffffffffffff, 5, false, false), + ), + // D negative overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0xffefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 1, + ), + (0xffefffffffffffff, 5, false, false), + ), + // D positive tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x0000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 1, + ), + (0x0000000000000000, 3, false, false), + ), + // D negative tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x8000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 1, + ), + (0x8000000000000000, 3, false, false), + ), + // D cancellation zero sign + ( + ( + 0x02007053, + 0x0000000000000000, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 1, + ), + (0x0000000000000000, 0, false, false), + ), + // D float to signed integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0x3ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 1, + ), + (0x0000000000000001, 1, true, false), + ), + // D negative integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0xbff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 1, + ), + (0xffffffffffffffff, 1, true, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02002053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02002053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000001, 1, false, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 2, + ), + (0x3ff0000000000000, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 2, + ), + (0xbff0000000000001, 1, false, false), + ), + // D positive overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0x7fefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 2, + ), + (0x7fefffffffffffff, 5, false, false), + ), + // D negative overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0xffefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 2, + ), + (0xfff0000000000000, 5, false, false), + ), + // D positive tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x0000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 2, + ), + (0x0000000000000000, 3, false, false), + ), + // D negative tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x8000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 2, + ), + (0x8000000000000001, 3, false, false), + ), + // D cancellation zero sign + ( + ( + 0x02007053, + 0x0000000000000000, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 2, + ), + (0x8000000000000000, 0, false, false), + ), + // D float to signed integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0x3ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 2, + ), + (0x0000000000000001, 1, true, false), + ), + // D negative integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0xbff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 2, + ), + (0xfffffffffffffffe, 1, true, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02003053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000001, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02003053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 1, false, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 3, + ), + (0x3ff0000000000001, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 3, + ), + (0xbff0000000000000, 1, false, false), + ), + // D positive overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0x7fefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 3, + ), + (0x7ff0000000000000, 5, false, false), + ), + // D negative overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0xffefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 3, + ), + (0xffefffffffffffff, 5, false, false), + ), + // D positive tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x0000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 3, + ), + (0x0000000000000001, 3, false, false), + ), + // D negative tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x8000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 3, + ), + (0x8000000000000000, 3, false, false), + ), + // D cancellation zero sign + ( + ( + 0x02007053, + 0x0000000000000000, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 3, + ), + (0x0000000000000000, 0, false, false), + ), + // D float to signed integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0x3ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 3, + ), + (0x0000000000000002, 1, true, false), + ), + // D negative integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0xbff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 3, + ), + (0xffffffffffffffff, 1, true, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02004053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000001, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02004053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000001, 1, false, false), + ), + // D positive half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x3ca0000000000000, + 0x0000000000000000, + 4, + ), + (0x3ff0000000000001, 1, false, false), + ), + // D negative half-ulp rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0xbff0000000000000, + 0xbca0000000000000, + 0x0000000000000000, + 4, + ), + (0xbff0000000000001, 1, false, false), + ), + // D positive overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0x7fefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 4, + ), + (0x7ff0000000000000, 5, false, false), + ), + // D negative overflow + ( + ( + 0x12007053, + 0x0000000000000000, + 0xffefffffffffffff, + 0x4000000000000000, + 0x0000000000000000, + 4, + ), + (0xfff0000000000000, 5, false, false), + ), + // D positive tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x0000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 4, + ), + (0x0000000000000001, 3, false, false), + ), + // D negative tiny tie + ( + ( + 0x12007053, + 0x0000000000000000, + 0x8000000000000001, + 0x3fe0000000000000, + 0x0000000000000000, + 4, + ), + (0x8000000000000001, 3, false, false), + ), + // D cancellation zero sign + ( + ( + 0x02007053, + 0x0000000000000000, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 4, + ), + (0x0000000000000000, 0, false, false), + ), + // D float to signed integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0x3ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 4, + ), + (0x0000000000000002, 1, true, false), + ), + // D negative integer tie + ( + ( + 0xc2207053, + 0x0000000000000000, + 0xbff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 4, + ), + (0xfffffffffffffffe, 1, true, false), + ), + // D reserved instruction rounding + ( + ( + 0x02005053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // D reserved instruction rounding + ( + ( + 0x02006053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // D reserved dynamic rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 5, + ), + (0x0000000000000000, 0, false, true), + ), + // D reserved dynamic rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 6, + ), + (0x0000000000000000, 0, false, true), + ), + // D reserved dynamic rounding + ( + ( + 0x02007053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 7, + ), + (0x0000000000000000, 0, false, true), + ), + // D sign injection + ( + ( + 0x22000053, + 0x0000000000000000, + 0x3ff0000000000000, + 0xc000000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 0, false, false), + ), + // D sign injection + ( + ( + 0x22001053, + 0x0000000000000000, + 0x3ff0000000000000, + 0xc000000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 0, false, false), + ), + // D sign injection + ( + ( + 0x22002053, + 0x0000000000000000, + 0x3ff0000000000000, + 0xc000000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 0, false, false), + ), + // D sign injection preserves payload + ( + ( + 0x22000053, + 0x0000000000000000, + 0x7ff8000000000011, + 0xc000000000000000, + 0x0000000000000000, + 0, + ), + (0xfff8000000000011, 0, false, false), + ), + // D min/max signed zero + ( + ( + 0x2a000053, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x8000000000000000, 0, false, false), + ), + // D min/max finite + ( + ( + 0x2a000053, + 0x0000000000000000, + 0xc000000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0xc000000000000000, 0, false, false), + ), + // D min/max quiet NaN + ( + ( + 0x2a000053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 0, false, false), + ), + // D min/max signaling NaN + ( + ( + 0x2a000053, + 0x0000000000000000, + 0x7ff0000000000001, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 16, false, false), + ), + // D min/max two NaNs + ( + ( + 0x2a000053, + 0x0000000000000000, + 0x7ff8000000000003, + 0x7ff8000000000007, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 0, false, false), + ), + // D min/max signed zero + ( + ( + 0x2a001053, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, false), + ), + // D min/max finite + ( + ( + 0x2a001053, + 0x0000000000000000, + 0xc000000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 0, false, false), + ), + // D min/max quiet NaN + ( + ( + 0x2a001053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 0, false, false), + ), + // D min/max signaling NaN + ( + ( + 0x2a001053, + 0x0000000000000000, + 0x7ff0000000000001, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff0000000000000, 16, false, false), + ), + // D min/max two NaNs + ( + ( + 0x2a001053, + 0x0000000000000000, + 0x7ff8000000000003, + 0x7ff8000000000007, + 0x0000000000000000, + 0, + ), + (0x7ff8000000000000, 0, false, false), + ), + // D comparison ordered + ( + ( + 0xa2000053, + 0x0000000000000000, + 0xbff0000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // D comparison signed zero + ( + ( + 0xa2000053, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // D comparison quiet NaN + ( + ( + 0xa2000053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // D comparison signaling NaN + ( + ( + 0xa2000053, + 0x0000000000000000, + 0x7ff0000000000001, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // D comparison ordered + ( + ( + 0xa2001053, + 0x0000000000000000, + 0xbff0000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // D comparison signed zero + ( + ( + 0xa2001053, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, true, false), + ), + // D comparison quiet NaN + ( + ( + 0xa2001053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // D comparison signaling NaN + ( + ( + 0xa2001053, + 0x0000000000000000, + 0x7ff0000000000001, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // D comparison ordered + ( + ( + 0xa2002053, + 0x0000000000000000, + 0xbff0000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, true, false), + ), + // D comparison signed zero + ( + ( + 0xa2002053, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // D comparison quiet NaN + ( + ( + 0xa2002053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, true, false), + ), + // D comparison signaling NaN + ( + ( + 0xa2002053, + 0x0000000000000000, + 0x7ff0000000000001, + 0x3ff0000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // D classification 0 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0xfff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000001, 0, true, false), + ), + // D classification 1 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0xbff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000002, 0, true, false), + ), + // D classification 2 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x8000000000000001, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000004, 0, true, false), + ), + // D classification 3 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x8000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000008, 0, true, false), + ), + // D classification 4 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000010, 0, true, false), + ), + // D classification 5 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x0000000000000001, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000020, 0, true, false), + ), + // D classification 6 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000040, 0, true, false), + ), + // D classification 7 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x7ff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000080, 0, true, false), + ), + // D classification 8 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x7ff0000000000001, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000100, 0, true, false), + ), + // D classification 9 + ( + ( + 0xe2001053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000200, 0, true, false), + ), + // D move float bits to integer + ( + ( + 0xe2000053, + 0x0000000000000000, + 0x0123456789abcdef, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0123456789abcdef, 0, true, false), + ), + // D move integer bits to float + ( + ( + 0xf2000053, + 0x0123456789abcdef, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0123456789abcdef, 0, false, false), + ), + // D integer conversion type 0 + ( + ( + 0xd2000053, + 0x0000000000000003, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x4008000000000000, 0, false, false), + ), + // D float conversion type 0 + ( + ( + 0xc2000053, + 0x0000000000000000, + 0x4008000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2000053, + 0x0000000000000000, + 0x7ff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x000000007fffffff, 16, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2000053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x000000007fffffff, 16, true, false), + ), + // D invalid negative conversion + ( + ( + 0xc2000053, + 0x0000000000000000, + 0xfff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xffffffff80000000, 16, true, false), + ), + // D integer conversion boundary + ( + ( + 0xd2000053, + 0xffffffffffffffff, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 0, false, false), + ), + // D integer conversion type 1 + ( + ( + 0xd2100053, + 0x0000000000000003, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x4008000000000000, 0, false, false), + ), + // D float conversion type 1 + ( + ( + 0xc2100053, + 0x0000000000000000, + 0x4008000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2100053, + 0x0000000000000000, + 0x7ff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2100053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // D invalid negative conversion + ( + ( + 0xc2100053, + 0x0000000000000000, + 0xfff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // D integer conversion boundary + ( + ( + 0xd2100053, + 0xffffffffffffffff, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x41efffffffe00000, 0, false, false), + ), + // D integer conversion type 2 + ( + ( + 0xd2200053, + 0x0000000000000003, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x4008000000000000, 0, false, false), + ), + // D float conversion type 2 + ( + ( + 0xc2200053, + 0x0000000000000000, + 0x4008000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2200053, + 0x0000000000000000, + 0x7ff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x7fffffffffffffff, 16, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2200053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x7fffffffffffffff, 16, true, false), + ), + // D invalid negative conversion + ( + ( + 0xc2200053, + 0x0000000000000000, + 0xfff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x8000000000000000, 16, true, false), + ), + // D integer conversion boundary + ( + ( + 0xd2200053, + 0xffffffffffffffff, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xbff0000000000000, 0, false, false), + ), + // D integer conversion type 3 + ( + ( + 0xd2300053, + 0x0000000000000003, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x4008000000000000, 0, false, false), + ), + // D float conversion type 3 + ( + ( + 0xc2300053, + 0x0000000000000000, + 0x4008000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000003, 0, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2300053, + 0x0000000000000000, + 0x7ff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // D invalid positive conversion + ( + ( + 0xc2300053, + 0x0000000000000000, + 0x7ff8000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0xffffffffffffffff, 16, true, false), + ), + // D invalid negative conversion + ( + ( + 0xc2300053, + 0x0000000000000000, + 0xfff0000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 16, true, false), + ), + // D integer conversion boundary + ( + ( + 0xd2300053, + 0xffffffffffffffff, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x43f0000000000000, 1, false, false), + ), + // D cross-format exact + ( + ( + 0x42000053, + 0x0000000000000000, + 0xffffffff3fc00000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x3ff8000000000000, 0, false, false), + ), + // D unsupported format + ( + ( + 0x04000053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // D unsupported format + ( + ( + 0x06000053, + 0x0000000000000000, + 0x3ff0000000000000, + 0x4000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // D unsupported operation + ( + ( + 0xfa000053, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), + // D unsupported opcode + ( + ( + 0x02000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + 0, + ), + (0x0000000000000000, 0, false, true), + ), +]; diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/agent.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/agent.svh new file mode 100644 index 000000000..6d2a96092 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/agent.svh @@ -0,0 +1,19 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(request_item) seqr; + driver drv; + monitor mon; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(request_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..a28f247d1 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/driver.svh @@ -0,0 +1,55 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual port_if vif; + reset_controller resetter; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get( + this, "", "vif", vif + ) || !uvm_config_db#(reset_controller)::get( + this, "", "resetter", resetter + )) + `uvm_fatal("INTERFACE", "FPU driver requires ports and reset controller") + endfunction + task run_phase(uvm_phase phase); + request_item req; + vif.request_valid = 0; + vif.response_ready = 0; + vif.instruction = 0; + vif.integer_value = 0; + vif.a = 0; + vif.rhs = 0; + vif.c = 0; + vif.rounding = 0; + forever begin + seq_item_port.get_next_item(req); + do @(negedge vif.clock); while (vif.reset); + vif.instruction = req.instruction; + vif.integer_value = req.integer_value; + vif.a = req.a; + vif.rhs = req.rhs; + vif.c = req.c; + vif.rounding = req.rounding; + vif.stall = req.stall; + vif.cancel = req.cancel; + vif.request_valid = 1; + do @(posedge vif.clock); while (!vif.request_ready); + @(negedge vif.clock); + vif.request_valid = 0; + if (req.cancel) begin + resetter.pulse(2); + end else begin + while (!vif.response_valid) @(negedge vif.clock); + repeat (req.stall) @(negedge vif.clock); + vif.response_ready = 1; + @(posedge vif.clock); + @(negedge vif.clock); + vif.response_ready = 0; + end + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/item.svh new file mode 100644 index 000000000..17846b089 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/item.svh @@ -0,0 +1,39 @@ +class request_item extends uvm_sequence_item; + logic [31:0] instruction; + logic [63:0] integer_value; + logic [63:0] a; + logic [63:0] rhs; + logic [63:0] c; + logic [2:0] rounding; + int unsigned stall; + logic cancel; + `uvm_object_utils_begin(request_item) + `uvm_field_int(instruction, UVM_ALL_ON) + `uvm_field_int(integer_value, UVM_ALL_ON) + `uvm_field_int(a, UVM_ALL_ON) + `uvm_field_int(rhs, UVM_ALL_ON) + `uvm_field_int(c, UVM_ALL_ON) + `uvm_field_int(rounding, UVM_ALL_ON) + `uvm_field_int(stall, UVM_ALL_ON) + `uvm_field_int(cancel, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [63:0] value; + logic [4:0] flags; + logic integer_result; + logic illegal; + `uvm_object_utils_begin(response_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_field_int(flags, UVM_ALL_ON) + `uvm_field_int(integer_result, UVM_ALL_ON) + `uvm_field_int(illegal, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..640970c5a --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/agent/monitor.svh @@ -0,0 +1,57 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual port_if vif; + uvm_analysis_port #(request_item) requests; + uvm_analysis_port #(response_item) responses; + uvm_analysis_port #(bit) reset_ap; + function new(string name, uvm_component parent); + super.new(name, parent); + requests = new("requests", this); + responses = new("responses", this); + reset_ap = new("reset_ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "FPU monitor requires ports") + endfunction + task run_phase(uvm_phase phase); + bit pending = 0; + request_item req; + response_item rsp; + forever begin + @(posedge vif.clock); + if (vif.reset) begin + pending = 0; + reset_ap.write(1); + end else begin + if (vif.request_valid && vif.request_ready) begin + if (pending) `uvm_fatal("OUTSTANDING", "FPU accepted a second request") + req = request_item::type_id::create("request"); + req.instruction = vif.instruction; + req.integer_value = vif.integer_value; + req.a = vif.a; + req.rhs = vif.rhs; + req.c = vif.c; + req.rounding = vif.rounding; + req.stall = vif.stall; + req.cancel = vif.cancel; + requests.write(req); + pending = 1; + end else if (pending && vif.request_ready) + `uvm_fatal("BUSY", "FPU reopened its request port before completion") + if (vif.response_valid && !pending) + `uvm_fatal("EXTRA_RESULT", "FPU produced an unsolicited result") + if (vif.response_valid && vif.response_ready) begin + rsp = response_item::type_id::create("response"); + rsp.value = vif.illegal ? 0 : vif.value; + rsp.flags = vif.flags; + rsp.integer_result = vif.illegal ? 0 : vif.integer_result; + rsp.illegal = vif.illegal; + responses.write(rsp); + pending = 0; + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/coverage.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/coverage.svh new file mode 100644 index 000000000..8d32b1064 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/coverage.svh @@ -0,0 +1,47 @@ +`uvm_analysis_imp_decl(_request) +`uvm_analysis_imp_decl(_response) +class coverage extends uvm_component; + `uvm_component_utils(coverage) + uvm_analysis_imp_request #(request_item, coverage) requests; + uvm_analysis_imp_response #(response_item, coverage) responses; + request_item active; + covergroup behavior with function sample (request_item req, response_item rsp); + option.per_instance = 1; + opcode: coverpoint req.instruction[6:0] { + bins arithmetic = {7'h53}; bins fused[] = {7'h43, 7'h47, 7'h4b, 7'h4f}; bins invalid = {0}; + } + operation: coverpoint req.instruction[31:27] iff (req.instruction[6:0] == 7'h53) { + bins supported[] = {0, 1, 2, 3, 4, 5, 8, 11, 20, 24, 26, 28, 30}; bins invalid = {31}; + } + format: coverpoint req.instruction[26:25] {bins formats[] = {0, 1}; bins invalid = {[2 : 3]};} + rounding: coverpoint (req.instruction[14:12] == 7 ? req.rounding : req.instruction[14:12]) + iff (req.instruction[6:0] inside {7'h43,7'h47,7'h4b,7'h4f} || + (req.instruction[6:0] == 7'h53 && req.instruction[31:27] inside {0,1,2,3,8,11,24,26})) { + bins modes[] = {[0 : 4]}; bins invalid = {[5 : 7]}; + } + integer_type: coverpoint req.instruction[21:20] iff (req.instruction[31:27] inside {24, 26}) { + bins types[] = {[0 : 3]}; + } + flags: coverpoint rsp.flags {bins exceptions[] = {0, 1, 3, 5, 8, 16};} + result_kind: coverpoint rsp.integer_result; + illegal: coverpoint rsp.illegal; + classification: coverpoint rsp.value[9:0] + iff (req.instruction[6:0] == 7'h53 && req.instruction[31:27] == 28 && + req.instruction[14:12] == 1 && !rsp.illegal) { + bins classes[] = {1, 2, 4, 8, 16, 32, 64, 128, 256, 512}; + } + backpressure: coverpoint req.stall {bins stalls[] = {0, 3};} + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + requests = new("requests", this); + responses = new("responses", this); + behavior = new(); + endfunction + function void write_request(request_item req); + active = req; + endfunction + function void write_response(response_item rsp); + behavior.sample(active, rsp); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/env.svh new file mode 100644 index 000000000..17f7af36e --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/env.svh @@ -0,0 +1,27 @@ +`uvm_analysis_imp_decl(_reset) +class env extends environment #(request_item, response_item, model); + `uvm_component_utils(env) + agent bus; + coverage cov; + uvm_analysis_imp_reset #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + cov = coverage::type_id::create("coverage", this); + bus = agent::type_id::create("agent", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.requests.connect(input_export); + bus.mon.requests.connect(cov.requests); + bus.mon.reset_ap.connect(reset_export); + bus.mon.responses.connect(output_export); + bus.mon.responses.connect(cov.responses); + endfunction + function void write_reset(bit asserted); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/model.svh new file mode 100644 index 000000000..eb00ef84d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/env/model.svh @@ -0,0 +1,20 @@ +class model extends reference_model #(request_item, response_item); + `uvm_component_utils(model) + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void write(request_item req); + response_item rsp = response_item::type_id::create("expected"); + longint unsigned value; + int unsigned flags; + byte unsigned integer_result, illegal; + fpu_expected(req.instruction, req.integer_value, req.a, req.rhs, req.c, req.rounding, value, + flags, integer_result, illegal); + rsp.value = illegal ? 0 : value; + rsp.flags = flags[4:0]; + rsp.integer_result = illegal ? 0 : integer_result[0]; + rsp.illegal = illegal[0]; + expected_ap.write(rsp); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/filelist.f new file mode 100644 index 000000000..ec2877c75 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/filelist.f @@ -0,0 +1,7 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/FPU/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/interface.sv new file mode 100644 index 000000000..57e3fd0e7 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/interface.sv @@ -0,0 +1,29 @@ +interface port_if ( + input logic clock, + input logic reset +); + logic request_valid, request_ready, response_valid, response_ready; + logic [31:0] instruction; + logic [63:0] integer_value; + logic [63:0] a; + logic [63:0] rhs; + logic [63:0] c; + logic [2:0] rounding; + int unsigned stall; + logic cancel; + logic [63:0] value; + logic [4:0] flags; + logic integer_result; + logic illegal; + assert property (@(posedge clock) disable iff (reset) + request_valid && !request_ready |=> request_valid && + $stable( + {instruction, integer_value, a, rhs, c, rounding} + )) + else $fatal(1, "FPU request changed under backpressure"); + assert property (@(posedge clock) disable iff (reset) + response_valid && !response_ready |=> response_valid && $stable( + {value, flags, integer_result, illegal} + )) + else $fatal(1, "FPU result changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/package.sv b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/package.sv new file mode 100644 index 000000000..34827bdbe --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/package.sv @@ -0,0 +1,38 @@ +package fpu_pkg; + timeunit 1ns; timeprecision 1ps; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function int unsigned fpu_case_count(); + import "DPI-C" function int unsigned fpu_cancel_case(); + import "DPI-C" function void fpu_case( + input int unsigned index, + output int unsigned instruction, + output longint unsigned integer_value, + a, + rhs, + c, + output int unsigned rounding + ); + import "DPI-C" function void fpu_expected( + input int unsigned instruction, + input longint unsigned integer_value, + a, + rhs, + c, + input int unsigned rounding, + output longint unsigned value, + output int unsigned flags, + output byte unsigned integer_result, + illegal + ); + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..a4470fe36 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/seq/sequence.svh @@ -0,0 +1,27 @@ +class stimulus extends uvm_sequence #(request_item); + `uvm_object_utils(stimulus) + int unsigned sent = 0; + function new(string name = "stimulus"); + super.new(name); + endfunction + task body(); + request_item req = request_item::type_id::create("request"); + int unsigned rounding; + start_item(req); + fpu_case(fpu_cancel_case(), req.instruction, req.integer_value, req.a, req.rhs, req.c, + rounding); + req.rounding = rounding[2:0]; + req.stall = 0; + req.cancel = 1; + finish_item(req); + for (int i = 0; i < fpu_case_count(); i++) begin + start_item(req); + fpu_case(i, req.instruction, req.integer_value, req.a, req.rhs, req.c, rounding); + req.rounding = rounding[2:0]; + req.stall = (i % 2) * 3; + req.cancel = 0; + finish_item(req); + sent++; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/suite.toml new file mode 100644 index 000000000..4b9957736 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libfpu_reference" + +[[tests]] +name = "fpu" +top = "fpu_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 2000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["fpu_pkg::coverage::behavior"] +sink_ready = 1 +sink_stall = 0 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/tb.sv new file mode 100644 index 000000000..42c3d96b9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/tb.sv @@ -0,0 +1,34 @@ +module fpu_tb; + import uvm_pkg::*; + import fpu_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + port_if bus ( + clock, + timing.reset + ); + FPU dut ( + .clock(clock), + .reset(bus.reset), + .io_request_valid(bus.request_valid), + .io_request_ready(bus.request_ready), + .io_request_bits_instruction(bus.instruction), + .io_request_bits_integer(bus.integer_value), + .io_request_bits_a(bus.a), + .io_request_bits_rhs(bus.rhs), + .io_request_bits_c(bus.c), + .io_request_bits_rounding(bus.rounding), + .io_response_valid(bus.response_valid), + .io_response_ready(bus.response_ready), + .io_response_bits_data(bus.value), + .io_response_bits_flags(bus.flags), + .io_response_bits_integer(bus.integer_result), + .io_response_bits_illegal(bus.illegal) + ); + initial begin + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/tests/test.svh new file mode 100644 index 000000000..ef256d9e1 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/resources/tests/test.svh @@ -0,0 +1,27 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env bench; + reset_controller resetter; + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + resetter = reset_controller::type_id::create("reset", this); + uvm_config_db#(reset_controller)::set(this, "env.agent.driver", "resetter", resetter); + bench = env::type_id::create("env", this); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Test interface is required") + endfunction + task execute(); + stimulus sequence_ = stimulus::type_id::create("sequence"); + resetter.pulse(4); + sequence_.start(bench.bus.seqr); + bench.scoreboard.wait_checked(sequence_.sent); + @(negedge vif.clock); + evidence::check(evidence::cancelled == 1, + "Reset did not cancel the outstanding floating divide"); + evidence::check(bench.scoreboard.checked == sequence_.sent, "Comparison count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/Emit.scala new file mode 100644 index 000000000..6518b056d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/Emit.scala @@ -0,0 +1,15 @@ +package framework.core.execution.fpu + +import framework.core.execution.fpu.configs.Params +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit extends App { + require(args.length >= 2 && args(0) == "--config") + val param = read[Params](Files.readString(Paths.get(args(1)))) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new FPU(param), + firtoolOpts = args.drop(2), + args = Array("--target-dir", "build/FPU", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/FPU.scala b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/FPU.scala new file mode 100644 index 000000000..827e90c4d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/FPU.scala @@ -0,0 +1,250 @@ +package framework.core.execution.fpu + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.lib.arith.Fma +import framework.core.execution.fpu.configs.{Params => ModuleParam} + +class Request(b: ModuleParam) extends Bundle { + val p = b + + val instruction = UInt(32.W) + val integer = UInt(p.xLen.W) + val a = UInt(p.fLen.W) + val rhs = UInt(p.fLen.W) + val c = UInt(p.fLen.W) + val rounding = UInt(3.W) +} + +class Response(b: ModuleParam) extends Bundle { + val p = b + + val data = UInt(p.xLen.W) + val flags = UInt(5.W) + val integer = Bool() + val illegal = Bool() +} + +/** IEEE binary32/binary64 arithmetic, conversions and register transfers. */ +class FPUIO(b: ModuleParam) extends Bundle { + val p = b + + val request = Flipped(Decoupled(new Request(b))) + val response = Decoupled(new Response(b)) + +} + +@instantiable +class FPU(b: ModuleParam) extends Module { + val p = b + + @public + val io = IO(new FPUIO(b)) + + val idle :: arithmetic :: divide :: respond :: Nil = Enum(4) + val state = RegInit(idle) + val selectedDouble = Reg(Bool()) + val answer = Reg(new Response(b)) + val request = io.request.bits + val instruction = request.instruction + val opcode = instruction(6, 0) + val operation = instruction(31, 27) + val funct3 = instruction(14, 12) + val rs2 = instruction(24, 20) + val fused = opcode === "h43".U || opcode === "h47".U || opcode === "h4b".U || opcode === "h4f".U + val double = instruction(25) + val rounding = Mux(funct3 === 7.U, request.rounding, funct3) + val rounded = fused || operation <= 3.U || operation === 8.U || operation === 11.U || + operation === 24.U || operation === 26.U + val formatLegal = instruction(26) === 0.U + val legal = WireDefault(false.B) + val toInteger = !fused && (operation === 20.U || operation === 24.U || operation === 28.U) + val isArithmetic = fused || operation <= 2.U + val isDivide = operation === 3.U || operation === 11.U + when(fused)(legal := true.B) + when(opcode === "h53".U) { + switch(operation) { + is(0.U, 1.U, 2.U, 3.U)(legal := true.B) + is(4.U)(legal := funct3 <= 2.U) + is(5.U)(legal := funct3 <= 1.U) + is(8.U)(legal := rs2 === Mux(double, 0.U, 1.U)) + is(11.U)(legal := rs2 === 0.U) + is(20.U)(legal := funct3 <= 2.U) + is(24.U, 26.U)(legal := rs2 <= 3.U) + is(28.U)(legal := rs2 === 0.U && funct3 <= 1.U) + is(30.U)(legal := rs2 === 0.U && funct3 === 0.U) + } + } + val illegal = !legal || !formatLegal || (rounded && rounding > 4.U) + io.request.ready := state === idle + io.response.valid := state === respond + io.response.bits := answer + when(io.response.fire)(state := idle) + + val simpleData = Wire(Vec(2, UInt(64.W))) + val simpleFlags = Wire(Vec(2, UInt(5.W))) + val arithmeticData = Wire(Vec(2, UInt(64.W))) + val arithmeticFlags = Wire(Vec(2, UInt(5.W))) + val arithmeticValid = Wire(Vec(2, Bool())) + val divideData = Wire(Vec(2, UInt(64.W))) + val divideFlags = Wire(Vec(2, UInt(5.W))) + val divideValid = Wire(Vec(2, Bool())) + val divideReady = Wire(Vec(2, Bool())) + + for (((exp, sig), index) <- Seq((8, 24), (11, 53)).zipWithIndex) { + val width = exp + sig + val single = index == 0 + val canonical = if (single) BigInt("7fc00000", 16) else BigInt("7ff8000000000000", 16) + val rawA = if (single) Mux(request.a(63, 32).andR, request.a(31, 0), canonical.U(32.W)) else request.a + val rawB = if (single) Mux(request.rhs(63, 32).andR, request.rhs(31, 0), canonical.U(32.W)) else request.rhs + val rawC = if (single) Mux(request.c(63, 32).andR, request.c(31, 0), canonical.U(32.W)) else request.c + val a = hardfloat.recFNFromFN(exp, sig, rawA) + val b = hardfloat.recFNFromFN(exp, sig, rawB) + val c = hardfloat.recFNFromFN(exp, sig, rawC) + val unpackedA = hardfloat.rawFloatFromRecFN(exp, sig, a) + val unpackedB = hardfloat.rawFloatFromRecFN(exp, sig, b) + val signNaN = hardfloat.isSigNaNRawFloat(unpackedA) || hardfloat.isSigNaNRawFloat(unpackedB) + val start = io.request.fire && !illegal && double === (index == 1).B + val fma: Instance[Fma] = Instantiate(new Fma(3, exp, sig)) + fma.io.validin := start && isArithmetic + fma.io.roundingMode := rounding + fma.io.detectTininess := 1.U + fma.io.op := Mux(fused, opcode(3, 2), Mux(operation === 1.U, 1.U, 0.U)) + fma.io.a := a + fma.io.b := Mux( + fused || operation === 2.U, + b, + hardfloat.recFNFromFN(exp, sig, (BigInt((1 << (exp - 1)) - 1) << (sig - 1)).U(width.W)) + ) + fma.io.c := Mux( + fused, + c, + Mux( + operation === 2.U, + hardfloat.recFNFromFN(exp, sig, Cat(rawA(width - 1) ^ rawB(width - 1), 0.U((width - 1).W))), + b + ) + ) + val fmaNan = hardfloat.rawFloatFromRecFN(exp, sig, fma.io.out).isNaN + val fmaRaw = Mux(fmaNan, canonical.U(width.W), hardfloat.fNFromRecFN(exp, sig, fma.io.out)) + arithmeticData(index) := (if (single) Cat(~0.U(32.W), fmaRaw) else fmaRaw) + arithmeticFlags(index) := fma.io.exceptionFlags + arithmeticValid(index) := fma.io.validout + + val div = Instantiate(new hardfloat.DivSqrtRecFN_small(exp, sig, 0)) + div.io.inValid := start && isDivide + div.io.sqrtOp := operation === 11.U + div.io.a := a + div.io.b := b + div.io.roundingMode := rounding + div.io.detectTininess := 1.U + val divRaw = Mux( + hardfloat.rawFloatFromRecFN(exp, sig, div.io.out).isNaN, + canonical.U(width.W), + hardfloat.fNFromRecFN(exp, sig, div.io.out) + ) + divideData(index) := (if (single) Cat(~0.U(32.W), divRaw) else divRaw) + divideFlags(index) := div.io.exceptionFlags + divideValid(index) := div.io.outValid_div || div.io.outValid_sqrt + divideReady(index) := div.io.inReady + + val compare = Instantiate(new hardfloat.CompareRecFN(exp, sig)) + compare.io.a := a + compare.io.b := b + compare.io.signaling := funct3 =/= 2.U + val bothZero = unpackedA.isZero && unpackedB.isZero + val chooseA = + Mux(funct3(0), compare.io.gt || (bothZero && !rawA(width - 1)), compare.io.lt || (bothZero && rawA(width - 1))) + val minimumMaximum = Mux( + unpackedA.isNaN && unpackedB.isNaN, + canonical.U(width.W), + Mux(unpackedA.isNaN, rawB, Mux(unpackedB.isNaN, rawA, Mux(chooseA, rawA, rawB))) + ) + val injectedSign = MuxLookup(funct3, rawB(width - 1))(Seq( + 1.U -> !rawB(width - 1), + 2.U -> (rawA(width - 1) ^ rawB(width - 1)) + )) + val injected = Cat(injectedSign, rawA(width - 2, 0)) + + val intInput = Mux(rs2(1), request.integer, Cat(Fill(32, !rs2(0) && request.integer(31)), request.integer(31, 0))) + val fromInt = Instantiate(new hardfloat.INToRecFN(64, exp, sig)) + fromInt.io.in := intInput + fromInt.io.signedIn := !rs2(0) + fromInt.io.roundingMode := rounding + fromInt.io.detectTininess := 1.U + val fromIntRaw = hardfloat.fNFromRecFN(exp, sig, fromInt.io.out) + val convertedIntegers = Wire(Vec(2, UInt(64.W))) + val convertedFlags = Wire(Vec(2, UInt(5.W))) + for ((intWidth, intIndex) <- Seq(32, 64).zipWithIndex) { + val toInt = Instantiate(new hardfloat.RecFNToIN(exp, sig, intWidth)) + toInt.io.in := a + toInt.io.signedOut := !rs2(0) + toInt.io.roundingMode := rounding + convertedIntegers(intIndex) := (if (intWidth == 32) Cat(Fill(32, toInt.io.out(31)), toInt.io.out) + else toInt.io.out) + convertedFlags(intIndex) := Cat(toInt.io.intExceptionFlags(2, 1).orR, 0.U(3.W), toInt.io.intExceptionFlags(0)) + } + val (sourceExp, sourceSig) = if (single) (11, 53) else (8, 24) + val source = + if (single) request.a + else + Mux(request.a(63, 32).andR, request.a(31, 0), "h7fc00000".U(32.W)) + val convert = Instantiate(new hardfloat.RecFNToRecFN(sourceExp, sourceSig, exp, sig)) + convert.io.in := hardfloat.recFNFromFN(sourceExp, sourceSig, source) + convert.io.roundingMode := rounding + convert.io.detectTininess := 1.U + val convertedFloat = Mux( + hardfloat.rawFloatFromRecFN(exp, sig, convert.io.out).isNaN, + canonical.U(width.W), + hardfloat.fNFromRecFN(exp, sig, convert.io.out) + ) + val floatResult = MuxLookup(operation, 0.U(width.W))(Seq( + 4.U -> injected, + 5.U -> minimumMaximum, + 8.U -> convertedFloat, + 26.U -> fromIntRaw, + 30.U -> request.integer(width - 1, 0) + )) + val integerResult = MuxLookup(operation, 0.U(64.W))(Seq( + 20.U -> Cat( + 0.U(63.W), + Mux(funct3 === 0.U, compare.io.lt || compare.io.eq, Mux(funct3 === 1.U, compare.io.lt, compare.io.eq)) + ), + 24.U -> convertedIntegers(rs2(1)), + 28.U -> Mux( + funct3(0), + hardfloat.classifyRecFN(exp, sig, a).pad(64), + if (single) Cat(Fill(32, request.a(31)), request.a(31, 0)) else request.a + ) + )) + simpleData(index) := Mux(toInteger, integerResult, if (single) Cat(~0.U(32.W), floatResult) else floatResult) + simpleFlags(index) := MuxLookup(operation, 0.U(5.W))(Seq( + 5.U -> Cat(signNaN, 0.U(4.W)), + 8.U -> convert.io.exceptionFlags, + 20.U -> compare.io.exceptionFlags, + 24.U -> convertedFlags(rs2(1)), + 26.U -> fromInt.io.exceptionFlags + )) + } + + when(io.request.fire) { + selectedDouble := double + answer.data := simpleData(double) + answer.flags := Mux(illegal, 0.U, simpleFlags(double)) + answer.integer := toInteger + answer.illegal := illegal + state := Mux(illegal, respond, Mux(isArithmetic, arithmetic, Mux(isDivide, divide, respond))) + when(!illegal && isDivide)(assert(divideReady(double), "Floating divider must be idle on accepted command")) + } + when(state === arithmetic && arithmeticValid(selectedDouble)) { + answer.data := arithmeticData(selectedDouble) + answer.flags := arithmeticFlags(selectedDouble) + state := respond + } + when(state === divide && divideValid(selectedDouble)) { + answer.data := divideData(selectedDouble) + answer.flags := divideFlags(selectedDouble) + state := respond + } +} diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..e3fb405e9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/Params.scala @@ -0,0 +1,6 @@ +package framework.core.execution.fpu.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Params(xLen: Int, fLen: Int) { require(xLen == 64 && fLen == 64) } +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/default.toml b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/default.toml new file mode 100644 index 000000000..180fa9af7 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/default.toml @@ -0,0 +1 @@ +fLen = 64 diff --git a/arch/src/main/scala/framework/core/execution/instruction/LICENSE.Berkeley b/arch/src/main/scala/framework/core/execution/instruction/LICENSE.Berkeley new file mode 100644 index 000000000..7cff15e40 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/LICENSE.Berkeley @@ -0,0 +1,24 @@ +Copyright (c) 2012-2014, The Regents of the University of California +(Regents). All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the Regents nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING +OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED +HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. diff --git a/arch/src/main/scala/framework/core/execution/instruction/LICENSE.SiFive b/arch/src/main/scala/framework/core/execution/instruction/LICENSE.SiFive new file mode 100644 index 000000000..7e7093375 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/LICENSE.SiFive @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2017 SiFive, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/arch/src/main/scala/framework/core/execution/instruction/build.sc b/arch/src/main/scala/framework/core/execution/instruction/build.sc new file mode 100644 index 000000000..2d6650403 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.execution.instruction.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Instruction.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/execution/instruction/configs/uvm.toml b/arch/src/main/scala/framework/core/execution/instruction/configs/uvm.toml new file mode 100644 index 000000000..3da02cd69 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/configs/uvm.toml @@ -0,0 +1,4 @@ +[parameters] +xLen = 64 + +[defines] diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/csrc/Cargo.lock b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/Cargo.lock new file mode 100644 index 000000000..6935dd56e --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "instruction-reference" +version = "0.1.0" diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/csrc/Cargo.toml b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/Cargo.toml new file mode 100644 index 000000000..f94657703 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "instruction-reference" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/core/mod.rs b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/core/mod.rs new file mode 100644 index 000000000..fb27202ca --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/core/mod.rs @@ -0,0 +1 @@ +pub mod riscv; diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/core/riscv/mod.rs b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/core/riscv/mod.rs new file mode 100644 index 000000000..97ffb8d9c --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/core/riscv/mod.rs @@ -0,0 +1,6 @@ +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/trap.rs"] +mod trap; +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/compressed.rs"] +mod compressed; +pub use trap::Trap; +pub use compressed::decode as expand_compressed; diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/lib.rs b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/lib.rs new file mode 100644 index 000000000..be67a8efd --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/csrc/src/lib.rs @@ -0,0 +1,152 @@ +mod core; + +fn signed(value: u32, width: u32) -> u64 { + ((value << (32 - width)) as i32 >> (32 - width)) as i64 as u64 +} + +#[no_mangle] +pub unsafe extern "C" fn instruction_expected( + bits: u32, + expanded: *mut u32, + illegal: *mut u8, + immediate: *mut u64, + atomic: *mut u32, + function: *mut u32, + word: *mut u8, + immediate_mode: *mut u8, +) { + let insn = if bits & 3 != 3 { + match core::riscv::expand_compressed(bits as u16) { + Ok(value) => value, + Err(_) => { + *expanded = 0; + *illegal = 1; + *immediate = 0; + *atomic = 0; + *function = 0; + *word = 0; + *immediate_mode = 0; + return; + } + } + } else { + bits + }; + let opcode = insn & 127; + let funct3 = (insn >> 12) & 7; + let funct7 = insn >> 25; + let rd = (insn >> 7) & 31; + let rs2 = (insn >> 20) & 31; + let is_word = matches!(opcode, 0x1b | 0x3b); + let atomic_code = match funct7 >> 2 { + 0 => 2, + 1 => 1, + 2 => 10, + 3 => 11, + 4 => 3, + 8 => 5, + 12 => 4, + 16 => 6, + 20 => 7, + 24 => 8, + 28 => 9, + _ => 0, + }; + let legal = match opcode { + 0x37 | 0x17 | 0x6f => true, + 0x67 => funct3 == 0, + 0x63 => matches!(funct3, 0 | 1 | 4 | 5 | 6 | 7), + 0x13 | 0x1b => match funct3 { + 0 => true, + 1 => { + if is_word { + funct7 == 0 + } else { + funct7 >> 1 == 0 + } + } + 5 => { + if is_word { + matches!(funct7, 0 | 32) + } else { + matches!(funct7 >> 1, 0 | 16) + } + } + _ => !is_word, + }, + 0x33 | 0x3b => { + if funct7 == 1 { + !is_word || funct3 == 0 || funct3 >= 4 + } else { + (funct7 == 0 || (funct7 == 32 && matches!(funct3, 0 | 5))) + && (!is_word || matches!(funct3, 0 | 1 | 5)) + } + } + 0x03 => funct3 <= 6, + 0x23 => funct3 <= 3, + 0x07 | 0x27 => matches!(funct3, 2 | 3), + 0x2f => matches!(funct3, 2 | 3) && atomic_code != 0 && (funct7 >> 2 != 2 || rs2 == 0), + 0x43 | 0x47 | 0x4b | 0x4f | 0x53 => true, + 0x0f => matches!(funct3, 0 | 1), + 0x73 => { + if funct3 == 0 { + matches!( + insn, + 0x00000073 | 0x00100073 | 0x30200073 | 0x10200073 | 0x10500073 + ) || (funct7 == 9 && rd == 0) + } else { + funct3 != 4 + } + } + 0x2b => funct3 == 7, + 0x7b => true, + _ => false, + }; + *expanded = insn; + *illegal = u8::from(!legal); + *immediate = match opcode { + 0x23 | 0x27 => signed(((insn >> 25) << 5) | ((insn >> 7) & 31), 12), + 0x63 => signed( + ((insn >> 31) << 12) + | (((insn >> 7) & 1) << 11) + | (((insn >> 25) & 63) << 5) + | (((insn >> 8) & 15) << 1), + 13, + ), + 0x6f => signed( + ((insn >> 31) << 20) + | (((insn >> 12) & 255) << 12) + | (((insn >> 20) & 1) << 11) + | (((insn >> 21) & 1023) << 1), + 21, + ), + 0x17 | 0x37 => signed(insn & 0xfffff000, 32), + _ => signed(insn >> 20, 12), + }; + *atomic = atomic_code; + *function = match funct3 { + 0 => { + if matches!(opcode, 0x33 | 0x3b) && funct7 & 32 != 0 { + 10 + } else { + 0 + } + } + 1 => 1, + 2 => 12, + 3 => 14, + 4 => 4, + 5 => { + if funct7 & 32 != 0 { + 11 + } else { + 5 + } + } + 6 => 6, + 7 => 7, + _ => unreachable!(), + }; + *word = u8::from(is_word); + *immediate_mode = u8::from(matches!(opcode, 0x13 | 0x1b)); +} diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/agent.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/agent.svh new file mode 100644 index 000000000..6d2a96092 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/agent.svh @@ -0,0 +1,19 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(request_item) seqr; + driver drv; + monitor mon; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(request_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..343374875 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/driver.svh @@ -0,0 +1,27 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Driver interface is required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + vif.sample = 0; + vif.bits = 0; + forever begin + seq_item_port.get_next_item(req); + do @(negedge vif.clock); while (vif.reset); + vif.bits = req.bits; + vif.sample = 1; + @(posedge vif.clock); + seq_item_port.item_done(); + @(negedge vif.clock); + vif.sample = 0; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/item.svh new file mode 100644 index 000000000..e0f3bdade --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/item.svh @@ -0,0 +1,47 @@ +class request_item extends uvm_sequence_item; + logic [31:0] bits; + `uvm_object_utils_begin(request_item) + `uvm_field_int(bits, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [31:0] expanded; + logic compressed; + logic illegal; + logic [6:0] opcode; + logic [2:0] funct3; + logic [6:0] funct7; + logic [4:0] rs1; + logic [4:0] rs2; + logic [4:0] rd; + logic [11:0] csr_address; + logic [63:0] immediate; + logic [3:0] atomic; + logic [4:0] function_code; + logic word; + logic immediate_mode; + `uvm_object_utils_begin(response_item) + `uvm_field_int(expanded, UVM_ALL_ON) + `uvm_field_int(compressed, UVM_ALL_ON) + `uvm_field_int(illegal, UVM_ALL_ON) + `uvm_field_int(opcode, UVM_ALL_ON) + `uvm_field_int(funct3, UVM_ALL_ON) + `uvm_field_int(funct7, UVM_ALL_ON) + `uvm_field_int(rs1, UVM_ALL_ON) + `uvm_field_int(rs2, UVM_ALL_ON) + `uvm_field_int(rd, UVM_ALL_ON) + `uvm_field_int(csr_address, UVM_ALL_ON) + `uvm_field_int(immediate, UVM_ALL_ON) + `uvm_field_int(atomic, UVM_ALL_ON) + `uvm_field_int(function_code, UVM_ALL_ON) + `uvm_field_int(word, UVM_ALL_ON) + `uvm_field_int(immediate_mode, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..0bb1954a4 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/agent/monitor.svh @@ -0,0 +1,45 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual port_if vif; + uvm_analysis_port #(request_item) requests; + uvm_analysis_port #(response_item) responses; + function new(string name, uvm_component parent); + super.new(name, parent); + requests = new("requests", this); + responses = new("responses", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + response_item rsp; + forever begin + @(posedge vif.clock); + if (!vif.reset && vif.sample) begin + req = request_item::type_id::create("request"); + rsp = response_item::type_id::create("response"); + req.bits = vif.bits; + rsp.expanded = vif.illegal ? 0 : vif.expanded; + rsp.compressed = vif.compressed; + rsp.illegal = vif.illegal; + rsp.opcode = vif.illegal ? 0 : vif.opcode; + rsp.funct3 = vif.illegal ? 0 : vif.funct3; + rsp.funct7 = vif.illegal ? 0 : vif.funct7; + rsp.rs1 = vif.illegal ? 0 : vif.rs1; + rsp.rs2 = vif.illegal ? 0 : vif.rs2; + rsp.rd = vif.illegal ? 0 : vif.rd; + rsp.csr_address = vif.illegal ? 0 : vif.csr_address; + rsp.immediate = vif.illegal ? 0 : vif.immediate; + rsp.atomic = vif.illegal ? 0 : vif.atomic; + rsp.function_code = vif.illegal ? 0 : vif.function_code; + rsp.word = vif.illegal ? 0 : vif.word; + rsp.immediate_mode = vif.illegal ? 0 : vif.immediate_mode; + requests.write(req); + responses.write(rsp); + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/coverage.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/coverage.svh new file mode 100644 index 000000000..c71181aa1 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/coverage.svh @@ -0,0 +1,25 @@ +class coverage extends uvm_subscriber #(request_item); + `uvm_component_utils(coverage) + covergroup behavior with function sample (request_item t); + option.per_instance = 1; + quadrant: coverpoint t.bits[1:0] {bins compressed[] = {[0 : 2]};} + compressed_opcode: coverpoint t.bits[15:13] iff (t.bits[1:0] != 3) { + bins functions[] = {[0 : 7]}; + } + compressed_encodings: cross quadrant, compressed_opcode; + opcode: coverpoint t.bits[6:0] iff (t.bits[1:0] == 3) { + bins opcodes[]={7'h03,7'h07,7'h0f,7'h13,7'h17,7'h1b,7'h23,7'h27,7'h2b,7'h2f,7'h33,7'h37,7'h3b,7'h43,7'h47,7'h4b,7'h4f,7'h53,7'h63,7'h67,7'h6f,7'h73,7'h7b}; + } + rd: coverpoint t.bits[11:7] {bins registers[] = {[0 : 31]};} + rs1: coverpoint t.bits[19:15] iff (t.bits[1:0] == 3) {bins registers[] = {[0 : 31]};} + rs2: coverpoint t.bits[24:20] iff (t.bits[1:0] == 3) {bins registers[] = {[0 : 31]};} + immediate_sign: coverpoint t.bits[31] iff (t.bits[1:0] == 3); + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void write(request_item t); + behavior.sample(t); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/env.svh new file mode 100644 index 000000000..5a8082f7a --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/env.svh @@ -0,0 +1,19 @@ +class env extends environment #(request_item, response_item, model); + `uvm_component_utils(env) + agent bus; + coverage cov; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + cov = coverage::type_id::create("coverage", this); + bus = agent::type_id::create("agent", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.requests.connect(input_export); + bus.mon.requests.connect(cov.analysis_export); + bus.mon.responses.connect(output_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/model.svh new file mode 100644 index 000000000..c3369607b --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/env/model.svh @@ -0,0 +1,32 @@ +class model extends reference_model #(request_item, response_item); + `uvm_component_utils(model) + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void write(request_item req); + response_item rsp = response_item::type_id::create("expected"); + int unsigned expanded, atomic, function_code; + longint unsigned immediate; + byte unsigned illegal, word, immediate_mode; + instruction_expected(req.bits, expanded, illegal, immediate, atomic, function_code, word, + immediate_mode); + rsp.set_name($sformatf("expected_%08h", req.bits)); + rsp.compressed = req.bits[1:0] != 3; + rsp.illegal = illegal[0]; + rsp.expanded = illegal ? 0 : expanded; + rsp.opcode = illegal ? 0 : expanded[6:0]; + rsp.funct3 = illegal ? 0 : expanded[14:12]; + rsp.funct7 = illegal ? 0 : expanded[31:25]; + rsp.rs1 = illegal ? 0 : expanded[19:15]; + rsp.rs2 = illegal ? 0 : expanded[24:20]; + rsp.rd = illegal ? 0 : expanded[11:7]; + rsp.csr_address = illegal ? 0 : expanded[31:20]; + rsp.immediate = illegal ? 0 : immediate; + rsp.atomic = illegal ? 0 : atomic[3:0]; + rsp.function_code = illegal ? 0 : function_code[4:0]; + rsp.word = illegal ? 0 : word[0]; + rsp.immediate_mode = illegal ? 0 : immediate_mode[0]; + expected_ap.write(rsp); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/filelist.f new file mode 100644 index 000000000..fd2376e6c --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/filelist.f @@ -0,0 +1,7 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Instruction/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/interface.sv new file mode 100644 index 000000000..342d5d325 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/interface.sv @@ -0,0 +1,22 @@ +interface port_if ( + input logic clock, + input logic reset +); + logic sample; + logic [31:0] bits; + logic [31:0] expanded; + logic compressed; + logic illegal; + logic [6:0] opcode; + logic [2:0] funct3; + logic [6:0] funct7; + logic [4:0] rs1; + logic [4:0] rs2; + logic [4:0] rd; + logic [11:0] csr_address; + logic [63:0] immediate; + logic [3:0] atomic; + logic [4:0] function_code; + logic word; + logic immediate_mode; +endinterface diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/package.sv b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/package.sv new file mode 100644 index 000000000..88d268d71 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/package.sv @@ -0,0 +1,25 @@ +package instruction_pkg; + timeunit 1ns; timeprecision 1ps; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function void instruction_expected( + input int unsigned bits, + output int unsigned expanded, + output byte unsigned illegal, + output longint unsigned immediate, + output int unsigned atomic, + function_code, + output byte unsigned word, + immediate_mode + ); + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..aeda933df --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/seq/sequence.svh @@ -0,0 +1,40 @@ +class stimulus extends uvm_sequence #(request_item); + `uvm_object_utils(stimulus) + int unsigned sent = 0; + function new(string name = "stimulus"); + super.new(name); + endfunction + task body(); + request_item req = request_item::type_id::create("request"); + int funct7s[15] = '{0, 1, 4, 8, 9, 12, 16, 32, 33, 48, 64, 80, 96, 112, 127}; + int system_words[6] = '{'h00000073, 'h00100073, 'h30200073, 'h10200073, 'h10500073, 'h12000073}; + for (int bits = 0; bits < 65536; bits++) begin + if ((bits & 3) == 3) continue; + start_item(req); + req.bits = bits; + finish_item(req); + sent++; + if ((bits & 'hfc) == 0) begin + start_item(req); + req.bits = 'hffff0000 | bits; + finish_item(req); + sent++; + end + end + for (int opcode = 3; opcode < 128; opcode += 4) + for (int funct3 = 0; funct3 < 8; funct3++) + foreach (funct7s[index]) + for (int rd = 0; rd < 32; rd++) begin + start_item(req); + req.bits=(funct7s[index]<<25)|(((rd+13)&31)<<20)|(((rd+7)&31)<<15)|(funct3<<12)|(rd<<7)|opcode; + finish_item(req); + sent++; + end + foreach (system_words[index]) begin + start_item(req); + req.bits = system_words[index]; + finish_item(req); + sent++; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/suite.toml new file mode 100644 index 000000000..c72329195 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libinstruction_reference" + +[[tests]] +name = "instruction" +top = "instruction_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 10000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["cond", "tgl"] +functional_groups = ["instruction_pkg::coverage::behavior"] +sink_ready = 1 +sink_stall = 0 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/tb.sv new file mode 100644 index 000000000..26e28ebc1 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/tb.sv @@ -0,0 +1,36 @@ +module instruction_tb; + import uvm_pkg::*; + import instruction_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + port_if bus ( + clock, + timing.reset + ); + Instruction dut ( + .clock(clock), + .reset(bus.reset), + .io_bits(bus.bits), + .io_expanded(bus.expanded), + .io_compressed(bus.compressed), + .io_illegal(bus.illegal), + .io_opcode(bus.opcode), + .io_funct3(bus.funct3), + .io_funct7(bus.funct7), + .io_rs1(bus.rs1), + .io_rs2(bus.rs2), + .io_rd(bus.rd), + .io_csrAddress(bus.csr_address), + .io_immediate(bus.immediate), + .io_atomicCode(bus.atomic), + .io_integerFunction(bus.function_code), + .io_integerWord(bus.word), + .io_integerImmediate(bus.immediate_mode) + ); + initial begin + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/tests/test.svh new file mode 100644 index 000000000..f29ec05f4 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/resources/tests/test.svh @@ -0,0 +1,24 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env bench; + reset_controller resetter; + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + resetter = reset_controller::type_id::create("reset", this); + bench = env::type_id::create("env", this); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Test interface is required") + endfunction + task execute(); + stimulus sequence_ = stimulus::type_id::create("sequence"); + resetter.pulse(4); + sequence_.start(bench.bus.seqr); + bench.scoreboard.wait_checked(sequence_.sent); + @(negedge vif.clock); + evidence::check(bench.scoreboard.checked == sequence_.sent, "Comparison count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/Emit.scala new file mode 100644 index 000000000..967f33d54 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/Emit.scala @@ -0,0 +1,15 @@ +package framework.core.execution.instruction + +import framework.core.execution.instruction.configs.Params +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit extends App { + require(args.length >= 2 && args(0) == "--config") + val param = read[Params](Files.readString(Paths.get(args(1)))) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Instruction(param), + firtoolOpts = args.drop(2), + args = Array("--target-dir", "build/Instruction", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/Instruction.scala b/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/Instruction.scala new file mode 100644 index 000000000..943a0bc2a --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/Instruction.scala @@ -0,0 +1,315 @@ +package framework.core.execution.instruction + +import chisel3._ +import chisel3.util._ +import framework.ip.bus.chi.rnf.CacheAtomic +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.execution.instruction.configs.{Params => ModuleParam} + +/** Expands RV64C instructions and decodes their fields and immediate operand. */ +class InstructionIO(b: ModuleParam) extends Bundle { + val p = b + + val bits = Input(UInt(32.W)) + val expanded = Output(UInt(32.W)) + val compressed = Output(Bool()) + val illegal = Output(Bool()) + val opcode = Output(UInt(7.W)) + val funct3 = Output(UInt(3.W)) + val funct7 = Output(UInt(7.W)) + val rs1 = Output(UInt(5.W)) + val rs2 = Output(UInt(5.W)) + val rd = Output(UInt(5.W)) + val csrAddress = Output(UInt(12.W)) + val immediate = Output(UInt(p.xLen.W)) + val atomicCode = Output(UInt(4.W)) + val integerFunction = Output(UInt(5.W)) + val integerWord = Output(Bool()) + val integerImmediate = Output(Bool()) + +} + +@instantiable +class Instruction(b: ModuleParam) extends Module { + val p = b + + @public + val io = IO(new InstructionIO(b)) + + val compressedIllegal = WireDefault(false.B) + val c = io.bits(15, 0) + val rd = c(11, 7) + val rs2 = c(6, 2) + val compactRd = Cat(1.U(2.W), c(4, 2)) + val compactRs1 = Cat(1.U(2.W), c(9, 7)) + val signedImmediate = Cat(Fill(7, c(12)), c(6, 2)) + val loadWordOffset = Cat(0.U(5.W), c(5), c(12, 10), c(6), 0.U(2.W)) + val loadDoubleOffset = Cat(0.U(4.W), c(6, 5), c(12, 10), 0.U(3.W)) + val jumpOffset = Cat(Fill(10, c(12)), c(12), c(8), c(10, 9), c(6), c(7), c(2), c(11), c(5, 3), 0.U(1.W)) + val branchOffset = Cat(Fill(5, c(12)), c(6, 5), c(2), c(11, 10), c(4, 3), 0.U(1.W)) + val addSpOffset = Cat(Fill(3, c(12)), c(4, 3), c(5), c(2), c(6), 0.U(4.W)) + val addSpUnsigned = Cat(0.U(2.W), c(10, 7), c(12, 11), c(5), c(6), 0.U(2.W)) + val stackWordLoad = Cat(0.U(4.W), c(3, 2), c(12), c(6, 4), 0.U(2.W)) + val stackDoubleLoad = Cat(0.U(3.W), c(4, 2), c(12), c(6, 5), 0.U(3.W)) + val stackWordStore = Cat(0.U(4.W), c(8, 7), c(12, 9), 0.U(2.W)) + val stackDoubleStore = Cat(0.U(3.W), c(9, 7), c(12, 10), 0.U(3.W)) + + io.compressed := c(1, 0) =/= 3.U + io.expanded := io.bits + compressedIllegal := false.B + when(io.compressed) { + io.expanded := "h00000013".U + compressedIllegal := true.B + switch(c(1, 0)) { + is(0.U) { + switch(c(15, 13)) { + is(0.U) { // C.ADDI4SPN + io.expanded := Cat(addSpUnsigned, 2.U(5.W), 0.U(3.W), compactRd, "h13".U(7.W)) + compressedIllegal := !addSpUnsigned.orR + } + is(1.U, 2.U, 3.U) { // C.FLD, C.LW, C.LD + val word = c(15, 13) === 2.U + io.expanded := Cat( + Mux(word, loadWordOffset, loadDoubleOffset), + compactRs1, + Mux(word, 2.U(3.W), 3.U(3.W)), + compactRd, + Mux(c(15, 13) === 1.U, "h07".U(7.W), "h03".U(7.W)) + ) + compressedIllegal := false.B + } + is(5.U, 6.U, 7.U) { // C.FSD, C.SW, C.SD + val word = c(15, 13) === 6.U + val offset = Mux(word, loadWordOffset, loadDoubleOffset) + io.expanded := Cat( + offset(11, 5), + compactRd, + compactRs1, + Mux(word, 2.U(3.W), 3.U(3.W)), + offset(4, 0), + Mux(c(15, 13) === 5.U, "h27".U(7.W), "h23".U(7.W)) + ) + compressedIllegal := false.B + } + } + } + is(1.U) { + switch(c(15, 13)) { + is(0.U, 1.U) { // C.ADDI/NOP and C.ADDIW + io.expanded := Cat(signedImmediate, rd, 0.U(3.W), rd, Mux(c(13), "h1b".U(7.W), "h13".U(7.W))) + compressedIllegal := c(13) && rd === 0.U + } + is(2.U) { // C.LI, including hints + io.expanded := Cat(signedImmediate, 0.U(5.W), 0.U(3.W), rd, "h13".U(7.W)) + compressedIllegal := false.B + } + is(3.U) { // C.ADDI16SP / C.LUI + io.expanded := Mux( + rd === 2.U, + Cat(addSpOffset, 2.U(5.W), 0.U(3.W), 2.U(5.W), "h13".U(7.W)), + Cat(Fill(15, c(12)), c(6, 2), rd, "h37".U(7.W)) + ) + compressedIllegal := !Cat(c(12), c(6, 2)).orR + } + is(4.U) { + compressedIllegal := false.B + switch(c(11, 10)) { + is(0.U, 1.U) { // C.SRLI / C.SRAI + io.expanded := Cat( + 0.U(1.W), + c(10), + 0.U(4.W), + c(12), + c(6, 2), + compactRs1, + 5.U(3.W), + compactRs1, + "h13".U(7.W) + ) + } + is(2.U) { // C.ANDI + io.expanded := Cat(signedImmediate, compactRs1, 7.U(3.W), compactRs1, "h13".U(7.W)) + } + is(3.U) { + val sub = c(6, 5) === 0.U + val funct = MuxLookup(c(6, 5), 0.U(3.W))(Seq(0.U -> 0.U, 1.U -> 4.U, 2.U -> 6.U, 3.U -> 7.U)) + io.expanded := Cat( + Mux(sub, 32.U(7.W), 0.U(7.W)), + compactRd, + compactRs1, + Mux(c(12), 0.U(3.W), funct), + compactRs1, + Mux(c(12), "h3b".U(7.W), "h33".U(7.W)) + ) + compressedIllegal := c(12) && c(6, 5) >= 2.U + } + } + } + is(5.U) { // C.J + io.expanded := Cat( + jumpOffset(20), + jumpOffset(10, 1), + jumpOffset(11), + jumpOffset(19, 12), + 0.U(5.W), + "h6f".U(7.W) + ) + compressedIllegal := false.B + } + is(6.U, 7.U) { // C.BEQZ / C.BNEZ + io.expanded := Cat( + branchOffset(12), + branchOffset(10, 5), + 0.U(5.W), + compactRs1, + Cat(0.U(2.W), c(13)), + branchOffset(4, 1), + branchOffset(11), + "h63".U(7.W) + ) + compressedIllegal := false.B + } + } + } + is(2.U) { + switch(c(15, 13)) { + is(0.U) { // C.SLLI, including hints + io.expanded := Cat(0.U(6.W), c(12), c(6, 2), rd, 1.U(3.W), rd, "h13".U(7.W)) + compressedIllegal := false.B + } + is(1.U, 2.U, 3.U) { // C.FLDSP / C.LWSP / C.LDSP + val word = c(15, 13) === 2.U + io.expanded := Cat( + Mux(word, stackWordLoad, stackDoubleLoad), + 2.U(5.W), + Mux(word, 2.U(3.W), 3.U(3.W)), + rd, + Mux(c(15, 13) === 1.U, "h07".U(7.W), "h03".U(7.W)) + ) + compressedIllegal := c(15, 13) =/= 1.U && rd === 0.U + } + is(4.U) { + compressedIllegal := false.B + when(rs2 =/= 0.U) { // C.MV / C.ADD, including hints + io.expanded := Cat(0.U(7.W), rs2, Mux(c(12), rd, 0.U(5.W)), 0.U(3.W), rd, "h33".U(7.W)) + }.elsewhen(c(12) && rd === 0.U) { + io.expanded := "h00100073".U // C.EBREAK + }.otherwise { // C.JR / C.JALR + io.expanded := Cat(0.U(12.W), rd, 0.U(3.W), Mux(c(12), 1.U(5.W), 0.U(5.W)), "h67".U(7.W)) + compressedIllegal := rd === 0.U + } + } + is(5.U, 6.U, 7.U) { // C.FSDSP / C.SWSP / C.SDSP + val word = c(15, 13) === 6.U + val offset = Mux(word, stackWordStore, stackDoubleStore) + io.expanded := Cat( + offset(11, 5), + rs2, + 2.U(5.W), + Mux(word, 2.U(3.W), 3.U(3.W)), + offset(4, 0), + Mux(c(15, 13) === 5.U, "h27".U(7.W), "h23".U(7.W)) + ) + compressedIllegal := false.B + } + } + } + } + } + + val insn = io.expanded + io.opcode := insn(6, 0) + io.funct3 := insn(14, 12) + io.funct7 := insn(31, 25) + io.rs1 := insn(19, 15) + io.rs2 := insn(24, 20) + io.rd := insn(11, 7) + io.csrAddress := insn(31, 20) + + val immediateI = Cat(Fill(p.xLen - 12, insn(31)), insn(31, 20)) + val immediateS = Cat(Fill(p.xLen - 12, insn(31)), insn(31, 25), insn(11, 7)) + val immediateB = Cat(Fill(p.xLen - 12, insn(31)), insn(7), insn(30, 25), insn(11, 8), 0.U(1.W)) + val immediateJ = Cat(Fill(p.xLen - 20, insn(31)), insn(19, 12), insn(20), insn(30, 21), 0.U(1.W)) + val immediateU = Cat(Fill(p.xLen - 32, insn(31)), insn(31, 12), 0.U(12.W)) + io.immediate := MuxLookup(io.opcode, immediateI)(Seq( + "h23".U -> immediateS, + "h27".U -> immediateS, + "h63".U -> immediateB, + "h6f".U -> immediateJ, + "h17".U -> immediateU, + "h37".U -> immediateU + )) + + io.atomicCode := MuxLookup(io.funct7(6, 2), CacheAtomic.None.U(4.W))(Seq( + 0.U -> CacheAtomic.Add.U, + 1.U -> CacheAtomic.Swap.U, + 2.U -> CacheAtomic.LR.U, + 3.U -> CacheAtomic.SC.U, + 4.U -> CacheAtomic.Xor.U, + 8.U -> CacheAtomic.Or.U, + 12.U -> CacheAtomic.And.U, + 16.U -> CacheAtomic.Min.U, + 20.U -> CacheAtomic.Max.U, + 24.U -> CacheAtomic.MinU.U, + 28.U -> CacheAtomic.MaxU.U + )) + io.integerWord := io.opcode === "h1b".U || io.opcode === "h3b".U + io.integerImmediate := io.opcode === "h13".U || io.opcode === "h1b".U + val subtract = (io.opcode === "h33".U || io.opcode === "h3b".U) && io.funct7(5) && io.funct3 === 0.U + io.integerFunction := MuxLookup(io.funct3, 0.U(5.W))(Seq( + 0.U -> Mux(subtract, 10.U, 0.U), + 1.U -> 1.U, + 2.U -> 12.U, + 3.U -> 14.U, + 4.U -> 4.U, + 5.U -> Mux(io.funct7(5), 11.U, 5.U), + 6.U -> 6.U, + 7.U -> 7.U + )) + + val shift = io.funct3 === 1.U || io.funct3 === 5.U + + val immediateLegal = Mux(io.integerWord, io.funct3 === 0.U || shift, true.B) && + (!shift || Mux(io.integerWord, io.funct7 === 0.U, io.funct7(6, 1) === 0.U) || + (io.funct3 === 5.U && Mux(io.integerWord, io.funct7 === 32.U, io.funct7(6, 1) === 16.U))) + + val registerLegal = Mux( + io.funct7 === 1.U, + !io.integerWord || io.funct3 === 0.U || io.funct3 >= 4.U, + (io.funct7 === 0.U || (io.funct7 === 32.U && (io.funct3 === 0.U || io.funct3 === 5.U))) && + (!io.integerWord || io.funct3 === 0.U || io.funct3 === 1.U || io.funct3 === 5.U) + ) + + val systemLegal = Seq("h00000073", "h00100073", "h30200073", "h10200073", "h10500073") + .map(code => insn === code.U(32.W)).reduce(_ || _) || (io.funct7 === "h09".U && io.rd === 0.U) + + val encodingLegal = MuxLookup(io.opcode, false.B)(Seq( + "h37".U -> true.B, + "h17".U -> true.B, + "h6f".U -> true.B, + "h67".U -> (io.funct3 === 0.U), + "h63".U -> (io.funct3 =/= 2.U && io.funct3 =/= 3.U), + "h13".U -> immediateLegal, + "h1b".U -> immediateLegal, + "h33".U -> registerLegal, + "h3b".U -> registerLegal, + "h03".U -> (io.funct3 =/= 7.U), + "h23".U -> (io.funct3 <= 3.U), + "h07".U -> (io.funct3 === 2.U || io.funct3 === 3.U), + "h27".U -> (io.funct3 === 2.U || io.funct3 === 3.U), + "h2f".U -> ((io.funct3 === 2.U || io.funct3 === 3.U) && io.atomicCode =/= CacheAtomic.None.U && + (io.funct7(6, 2) =/= 2.U || io.rs2 === 0.U)), + "h43".U -> true.B, + "h47".U -> true.B, + "h4b".U -> true.B, + "h4f".U -> true.B, + "h53".U -> true.B, + "h0f".U -> (io.funct3 === 0.U || io.funct3 === 1.U), + "h73".U -> Mux(io.funct3 =/= 0.U, io.funct3 =/= 4.U, systemLegal), + "h2b".U -> (io.funct3 === 7.U), + "h7b".U -> true.B + )) + + io.illegal := compressedIllegal || !encodingLegal + +} diff --git a/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..ceca7b03d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/instruction/src/main/scala/configs/Params.scala @@ -0,0 +1,6 @@ +package framework.core.execution.instruction.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Params(xLen: Int) { require(xLen == 64) } +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/execution/integer/LICENSE.Berkeley b/arch/src/main/scala/framework/core/execution/integer/LICENSE.Berkeley new file mode 100644 index 000000000..7cff15e40 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/LICENSE.Berkeley @@ -0,0 +1,24 @@ +Copyright (c) 2012-2014, The Regents of the University of California +(Regents). All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the Regents nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING +OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED +HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. diff --git a/arch/src/main/scala/framework/core/execution/integer/LICENSE.SiFive b/arch/src/main/scala/framework/core/execution/integer/LICENSE.SiFive new file mode 100644 index 000000000..7e7093375 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/LICENSE.SiFive @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2017 SiFive, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/arch/src/main/scala/framework/core/execution/integer/build.sc b/arch/src/main/scala/framework/core/execution/integer/build.sc new file mode 100644 index 000000000..4647dd8d9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.execution.integer.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Integer.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/execution/integer/configs/uvm.toml b/arch/src/main/scala/framework/core/execution/integer/configs/uvm.toml new file mode 100644 index 000000000..3da02cd69 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/configs/uvm.toml @@ -0,0 +1,4 @@ +[parameters] +xLen = 64 + +[defines] diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/agent.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/agent.svh new file mode 100644 index 000000000..6d2a96092 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/agent.svh @@ -0,0 +1,19 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(request_item) seqr; + driver drv; + monitor mon; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(request_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..7bf3af2be --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/driver.svh @@ -0,0 +1,33 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Driver interface is required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + vif.sample = 0; + vif.a = 0; + vif.rhs = 0; + vif.fn = 0; + vif.dw = 0; + forever begin + seq_item_port.get_next_item(req); + do @(negedge vif.clock); while (vif.reset); + vif.a = req.a; + vif.rhs = req.rhs; + vif.fn = req.fn; + vif.dw = req.dw; + vif.sample = 1; + @(posedge vif.clock); + seq_item_port.item_done(); + @(negedge vif.clock); + vif.sample = 0; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/item.svh new file mode 100644 index 000000000..4fdc50670 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/item.svh @@ -0,0 +1,29 @@ +class request_item extends uvm_sequence_item; + logic [63:0] a; + logic [63:0] rhs; + logic [4:0] fn; + logic dw; + `uvm_object_utils_begin(request_item) + `uvm_field_int(a, UVM_ALL_ON) + `uvm_field_int(rhs, UVM_ALL_ON) + `uvm_field_int(fn, UVM_ALL_ON) + `uvm_field_int(dw, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [63:0] value; + logic [63:0] adder; + logic comparison; + `uvm_object_utils_begin(response_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_field_int(adder, UVM_ALL_ON) + `uvm_field_int(comparison, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..89371d505 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/agent/monitor.svh @@ -0,0 +1,36 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual port_if vif; + uvm_analysis_port #(request_item) requests; + uvm_analysis_port #(response_item) responses; + function new(string name, uvm_component parent); + super.new(name, parent); + requests = new("requests", this); + responses = new("responses", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + response_item rsp; + forever begin + @(posedge vif.clock); + if (!vif.reset && vif.sample) begin + req = request_item::type_id::create("request"); + rsp = response_item::type_id::create("response"); + req.a = vif.a; + req.rhs = vif.rhs; + req.fn = vif.fn; + req.dw = vif.dw; + rsp.value = vif.value; + rsp.adder = vif.adder; + rsp.comparison = vif.comparison; + requests.write(req); + responses.write(rsp); + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/coverage.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/coverage.svh new file mode 100644 index 000000000..f6f8178f8 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/coverage.svh @@ -0,0 +1,30 @@ +class coverage extends uvm_subscriber #(request_item); + `uvm_component_utils(coverage) + covergroup behavior with function sample (request_item t); + option.per_instance = 1; + operation: coverpoint t.fn { + bins arithmetic[] = {0, 10}; + bins shift[] = {1, 5, 11, 17, 18, 19}; + bins logic_ops[] = {4, 6, 7, 8, 9, 24, 25, 26}; + bins compare_ops[] = {12, 13, 14, 15, 28, 29, 30, 31}; + bins unary = {16}; + } + width: coverpoint t.dw; + unary: coverpoint t.rhs[11:0] iff (t.fn == 16) { + bins selectors[] = {0, 1, 2, 'h287, 'h6b8, 'h080, 'h604, 'h605}; + } + shift_amount: coverpoint (t.rhs & (t.dw ? 63 : 31)) iff (t.fn inside {1, 5, 11, 17, 18, 19}) { + bins boundaries[] = {0, 1, 31, 32, 63}; + } + sign_a: coverpoint t.a[63]; + sign_rhs: coverpoint t.rhs[63]; + operand_signs: cross sign_a, sign_rhs; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void write(request_item t); + behavior.sample(t); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/env.svh new file mode 100644 index 000000000..5a8082f7a --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/env.svh @@ -0,0 +1,19 @@ +class env extends environment #(request_item, response_item, model); + `uvm_component_utils(env) + agent bus; + coverage cov; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + cov = coverage::type_id::create("coverage", this); + bus = agent::type_id::create("agent", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.requests.connect(input_export); + bus.mon.requests.connect(cov.analysis_export); + bus.mon.responses.connect(output_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/model.svh new file mode 100644 index 000000000..92e9996ae --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/env/model.svh @@ -0,0 +1,65 @@ +class model extends reference_model #(request_item, response_item); + `uvm_component_utils(model) + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void write(request_item req); + response_item rsp = response_item::type_id::create("expected"); + logic [63:0] value; + int width = req.dw ? 64 : 32; + int shift = req.rhs & (width - 1); + logic [63:0] operand = req.dw ? req.a : {32'b0, req.a[31:0]}; + int count; + rsp.adder = req.fn[3] ? req.a - req.rhs : req.a + req.rhs; + rsp.comparison = req.fn[0] ^ (!req.fn[3] ? req.a == req.rhs : + req.fn[1] ? req.a < req.rhs : $signed(req.a) < $signed(req.rhs)); + case (req.fn) + 0: value = req.a + req.rhs; + 10: value = req.a - req.rhs; + 1: value = operand << shift; + 5: value = operand >> shift; + 11: value = req.dw ? $signed(req.a) >>> shift : $signed(req.a[31:0]) >>> shift; + 4: value = req.a ^ req.rhs; + 6: value = req.a | req.rhs; + 7: value = req.a & req.rhs; + 8: value = req.rhs != 0 ? req.a : 0; + 9: value = req.rhs == 0 ? req.a : 0; + 12, 13: value = $signed(req.a) < $signed(req.rhs); + 14, 15: value = req.a < req.rhs; + 17: value = (operand << shift) | (operand >> (width - shift)); + 18: value = (operand >> shift) | (operand << (width - shift)); + 19: value = (operand >> shift) & 1; + 24: value = req.a & ~req.rhs; + 25: value = req.a | ~req.rhs; + 26: value = req.a ^ ~req.rhs; + 28: value = $signed(req.a) > $signed(req.rhs) ? req.a : req.rhs; + 29: value = $signed(req.a) < $signed(req.rhs) ? req.a : req.rhs; + 30: value = req.a > req.rhs ? req.a : req.rhs; + 31: value = req.a < req.rhs ? req.a : req.rhs; + 16: begin + value = 0; + case (req.rhs[11:0]) + 12'h287: for (int i = 0; i < 8; i++) value[i*8+:8] = |req.a[i*8+:8] ? 8'hff : 0; + 12'h6b8: for (int i = 0; i < 8; i++) value[i*8+:8] = req.a[(7-i)*8+:8]; + 12'h080: value = {48'b0, req.a[15:0]}; + 12'h604: value = {{56{req.a[7]}}, req.a[7:0]}; + 12'h605: value = {{48{req.a[15]}}, req.a[15:0]}; + 0, 1, 2: begin + count = 0; + if (req.rhs == 2) begin + for (int i = 0; i < width; i++) count += operand[i]; + end else begin + while (count < width && !operand[req.rhs==0?width-1-count : count]) count++; + end + value = count; + end + default: `uvm_fatal("OPERATION", "Unknown unary instruction") + endcase + end + default: `uvm_fatal("OPERATION", "Unknown integer operation") + endcase + rsp.value = req.dw ? value : {{32{value[31]}}, value[31:0]}; + expected_ap.write(rsp); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/filelist.f new file mode 100644 index 000000000..1f4d31b52 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/filelist.f @@ -0,0 +1,7 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Integer/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/interface.sv new file mode 100644 index 000000000..10b689346 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/interface.sv @@ -0,0 +1,13 @@ +interface port_if ( + input logic clock, + input logic reset +); + logic sample; + logic [63:0] a; + logic [63:0] rhs; + logic [4:0] fn; + logic dw; + logic [63:0] value; + logic [63:0] adder; + logic comparison; +endinterface diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/package.sv b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/package.sv new file mode 100644 index 000000000..e21dddad9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/package.sv @@ -0,0 +1,15 @@ +package integer_pkg; + timeunit 1ns; timeprecision 1ps; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..b400a82e9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/seq/sequence.svh @@ -0,0 +1,66 @@ +class stimulus extends uvm_sequence #(request_item); + `uvm_object_utils(stimulus) + int unsigned sent = 0; + reset_controller resetter; + function new(string name = "stimulus"); + super.new(name); + endfunction + task body(); + request_item req = request_item::type_id::create("request"); + logic [63:0] operands[8] = '{0, 1, 31, 32, 63, '1, 64'h8000000000000000, 64'h55555555aaaaaaaa}; + int operations[24] = '{ + 0, + 10, + 1, + 5, + 11, + 4, + 6, + 7, + 8, + 9, + 12, + 13, + 14, + 15, + 17, + 18, + 19, + 24, + 25, + 26, + 28, + 29, + 30, + 31 + }; + int unary[8] = '{0, 1, 2, 'h287, 'h6b8, 'h080, 'h604, 'h605}; + for (int word = 0; word < 2; word++) begin + foreach (operations[op]) begin + if (word && !(operations[op] inside {0, 10, 1, 5, 11, 17, 18})) continue; + foreach (operands[a]) + foreach (operands[b]) begin + start_item(req); + req.a = operands[a]; + req.rhs = operands[b]; + req.fn = operations[op]; + req.dw = !word; + finish_item(req); + sent++; + end + end + foreach (unary[op]) begin + if (word && op >= 3) continue; + foreach (operands[a]) begin + start_item(req); + req.a = operands[a]; + req.rhs = unary[op]; + req.fn = 16; + req.dw = !word; + finish_item(req); + sent++; + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/suite.toml new file mode 100644 index 000000000..1fd36e0df --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +functional_groups = ["integer_pkg::coverage::behavior"] +name = "integer" +top = "integer_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["cond", "tgl"] +sink_ready = 1 +sink_stall = 0 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/tb.sv new file mode 100644 index 000000000..f6df99158 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/tb.sv @@ -0,0 +1,27 @@ +module integer_tb; + import uvm_pkg::*; + import integer_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + port_if bus ( + clock, + timing.reset + ); + Integer dut ( + .clock(clock), + .reset(bus.reset), + .io_in1(bus.a), + .io_in2(bus.rhs), + .io_fn(bus.fn), + .io_dw(bus.dw), + .io_out(bus.value), + .io_adder_out(bus.adder), + .io_cmp_out(bus.comparison) + ); + initial begin + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/tests/test.svh new file mode 100644 index 000000000..d5f2d6ed9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/resources/tests/test.svh @@ -0,0 +1,25 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env bench; + reset_controller resetter; + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + resetter = reset_controller::type_id::create("reset", this); + bench = env::type_id::create("env", this); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Test interface is required") + endfunction + task execute(); + stimulus sequence_ = stimulus::type_id::create("sequence"); + resetter.pulse(4); + sequence_.resetter = resetter; + sequence_.start(bench.bus.seqr); + bench.scoreboard.wait_checked(sequence_.sent); + @(negedge vif.clock); + evidence::check(bench.scoreboard.checked == sequence_.sent, "Comparison count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/execution/integer/src/main/scala/Emit.scala new file mode 100644 index 000000000..c77d7b22b --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/scala/Emit.scala @@ -0,0 +1,15 @@ +package framework.core.execution.integer + +import framework.core.execution.integer.configs.Params +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit extends App { + require(args.length >= 2 && args(0) == "--config") + val param = read[Params](Files.readString(Paths.get(args(1)))) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Integer(param), + firtoolOpts = args.drop(2), + args = Array("--target-dir", "build/Integer", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/scala/Integer.scala b/arch/src/main/scala/framework/core/execution/integer/src/main/scala/Integer.scala new file mode 100644 index 000000000..2d02fda4b --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/scala/Integer.scala @@ -0,0 +1,93 @@ +// See LICENSE.Berkeley and LICENSE.SiFive for license details. +package framework.core.execution.integer + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.execution.integer.configs.{Params => ModuleParam} + +class IntegerIO(b: ModuleParam) extends Bundle { + val p = b + + val in1 = Input(UInt(p.xLen.W)) + val in2 = Input(UInt(p.xLen.W)) + val fn = Input(UInt(5.W)) + val dw = Input(Bool()) + val out = Output(UInt(p.xLen.W)) + val adder_out = Output(UInt(p.xLen.W)) + val cmp_out = Output(Bool()) + +} + +@instantiable +class Integer(b: ModuleParam) extends Module { + val p = b + + @public + val io = IO(new IntegerIO(b)) + + val a = io.in1 + val rhs = io.in2 + val fn = io.fn + val subtract = fn(3) + val inverted = Mux(subtract, ~rhs, rhs) + val difference = a ^ inverted + val conjunction = a & inverted + val sum = a + inverted + subtract + val less = Mux(a(63) === rhs(63), sum(63), Mux(fn(1), rhs(63), a(63))) + io.adder_out := sum + io.cmp_out := fn(0) ^ Mux(!subtract, difference === 0.U, less) + val shift = Cat(rhs(5) && io.dw, rhs(4, 0)) + val high = Mux(io.dw, a(63, 32), Fill(32, subtract && a(31))) + val input = Cat(high, a(31, 0)) + val reverse = !(fn === 5.U || fn === 11.U || fn === 18.U || fn === 19.U) + val shiftedInput = Mux(reverse, Reverse(input), input) + val right = (Cat(subtract && shiftedInput(63), shiftedInput).asSInt >> shift)(63, 0) + val left = Reverse(right) + val shifts = Mux(fn === 5.U || fn === 11.U || fn === 19.U, right, 0.U) | Mux(fn === 1.U, left, 0.U) + val logic = Mux(fn === 4.U || fn === 6.U || fn === 25.U || fn === 26.U, difference, 0.U) | + Mux(fn === 6.U || fn === 7.U || fn === 25.U || fn === 24.U, conjunction, 0.U) + val comparison = (fn >= 12.U && fn <= 15.U) && less + val conditional = Mux((fn === 8.U && rhs.orR) || (fn === 9.U && !rhs.orR), a, 0.U) + val shiftLogic = comparison.asUInt | logic | (shifts & Mux(fn === 19.U, 1.U, ~0.U(64.W))) | conditional + + val countInput = MuxLookup(Cat(!io.dw, !rhs(0)), 0.U)(Seq( + 0.U -> a, + 1.U -> Reverse(a), + 2.U -> Cat(1.U(1.W), a(31, 0)), + 3.U -> Cat(1.U(1.W), Reverse(a(31, 0))) + )) + + val populationInput = Mux(rhs(1), Mux(io.dw, a, a(31, 0)), PriorityEncoderOH(Cat(1.U(1.W), countInput)) - 1.U)(63, 0) + val count = PopCount(populationInput) + val bytes = a.asTypeOf(Vec(8, UInt(8.W))) + + val unary = MuxLookup(rhs(11, 0), count)(Seq( + "h287".U -> VecInit(bytes.map(byte => Fill(8, byte.orR))).asUInt, + "h6b8".U -> VecInit(bytes.reverse).asUInt, + "h080".U -> a(15, 0), + "h604".U -> Cat(Fill(56, a(7)), a(7, 0)), + "h605".U -> Cat(Fill(48, a(15)), a(15, 0)) + )) + + val rotationShift = Mux(io.dw, 64.U, 32.U) - shift + val rotationInput = Mux(fn(0), input, Reverse(input)) + val rotateRight = (rotationInput >> rotationShift)(63, 0) + val rotateLeft = Reverse(rotateRight) + val rotation = Mux(fn(0), rotateRight, rotateLeft) | Mux(fn(0), left, right) + val minimumMaximum = Mux(io.cmp_out, rhs, a) + + val result = MuxLookup(fn, shiftLogic)(Seq( + 0.U -> sum, + 10.U -> sum, + 16.U -> unary, + 28.U -> minimumMaximum, + 29.U -> minimumMaximum, + 30.U -> minimumMaximum, + 31.U -> minimumMaximum, + 17.U -> rotation, + 18.U -> rotation + )) + + io.out := Mux(io.dw, result, Cat(Fill(32, result(31)), result(31, 0))) +} diff --git a/arch/src/main/scala/framework/core/execution/integer/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/execution/integer/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..aabdb1464 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/integer/src/main/scala/configs/Params.scala @@ -0,0 +1,6 @@ +package framework.core.execution.integer.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Params(xLen: Int) { require(xLen == 64) } +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/execution/muldiv/build.sc b/arch/src/main/scala/framework/core/execution/muldiv/build.sc new file mode 100644 index 000000000..754f56a30 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.execution.muldiv.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/MulDiv.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/execution/muldiv/configs/uvm.toml b/arch/src/main/scala/framework/core/execution/muldiv/configs/uvm.toml new file mode 100644 index 000000000..3da02cd69 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/configs/uvm.toml @@ -0,0 +1,4 @@ +[parameters] +xLen = 64 + +[defines] diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/agent.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/agent.svh new file mode 100644 index 000000000..6d2a96092 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/agent.svh @@ -0,0 +1,19 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(request_item) seqr; + driver drv; + monitor mon; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(request_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..885c3e2b9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/driver.svh @@ -0,0 +1,51 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual port_if vif; + reset_controller resetter; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get( + this, "", "vif", vif + ) || !uvm_config_db#(reset_controller)::get( + this, "", "resetter", resetter + )) + `uvm_fatal("INTERFACE", "MulDiv driver requires ports and reset controller") + endfunction + task run_phase(uvm_phase phase); + request_item req; + vif.request_valid = 0; + vif.response_ready = 0; + vif.a = 0; + vif.rhs = 0; + vif.operation = 0; + vif.word = 0; + forever begin + seq_item_port.get_next_item(req); + do @(negedge vif.clock); while (vif.reset); + vif.a = req.a; + vif.rhs = req.rhs; + vif.operation = req.operation; + vif.word = req.word; + vif.stall = req.stall; + vif.cancel = req.cancel; + vif.request_valid = 1; + do @(posedge vif.clock); while (!vif.request_ready); + @(negedge vif.clock); + vif.request_valid = 0; + if (req.cancel) begin + resetter.pulse(2); + end else begin + while (!vif.response_valid) @(negedge vif.clock); + repeat (req.stall) @(negedge vif.clock); + vif.response_ready = 1; + @(posedge vif.clock); + @(negedge vif.clock); + vif.response_ready = 0; + end + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/item.svh new file mode 100644 index 000000000..4184086de --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/item.svh @@ -0,0 +1,29 @@ +class request_item extends uvm_sequence_item; + logic [63:0] a; + logic [63:0] rhs; + logic [2:0] operation; + logic word; + int unsigned stall; + logic cancel; + `uvm_object_utils_begin(request_item) + `uvm_field_int(a, UVM_ALL_ON) + `uvm_field_int(rhs, UVM_ALL_ON) + `uvm_field_int(operation, UVM_ALL_ON) + `uvm_field_int(word, UVM_ALL_ON) + `uvm_field_int(stall, UVM_ALL_ON) + `uvm_field_int(cancel, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [63:0] value; + `uvm_object_utils_begin(response_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..1febc4201 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/agent/monitor.svh @@ -0,0 +1,52 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual port_if vif; + uvm_analysis_port #(request_item) requests; + uvm_analysis_port #(response_item) responses; + uvm_analysis_port #(bit) reset_ap; + function new(string name, uvm_component parent); + super.new(name, parent); + requests = new("requests", this); + responses = new("responses", this); + reset_ap = new("reset_ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "MulDiv monitor requires ports") + endfunction + task run_phase(uvm_phase phase); + bit pending = 0; + request_item req; + response_item rsp; + forever begin + @(posedge vif.clock); + if (vif.reset) begin + pending = 0; + reset_ap.write(1); + end else begin + if (vif.request_valid && vif.request_ready) begin + if (pending) `uvm_fatal("OUTSTANDING", "MulDiv accepted a second request") + req = request_item::type_id::create("request"); + req.a = vif.a; + req.rhs = vif.rhs; + req.operation = vif.operation; + req.word = vif.word; + req.stall = vif.stall; + req.cancel = vif.cancel; + requests.write(req); + pending = 1; + end else if (pending && vif.request_ready) + `uvm_fatal("BUSY", "MulDiv reopened its request port before completion") + if (vif.response_valid && !pending) + `uvm_fatal("EXTRA_RESULT", "MulDiv produced an unsolicited result") + if (vif.response_valid && vif.response_ready) begin + rsp = response_item::type_id::create("response"); + rsp.value = vif.value; + responses.write(rsp); + pending = 0; + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/coverage.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/coverage.svh new file mode 100644 index 000000000..cbc2cb93d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/coverage.svh @@ -0,0 +1,34 @@ +class coverage extends uvm_subscriber #(request_item); + `uvm_component_utils(coverage) + covergroup behavior with function sample (request_item t); + option.per_instance = 1; + operation: coverpoint t.operation {bins operations[] = {[0 : 7]};} + word: coverpoint t.word; + operation_width: cross operation, word{ + ignore_bins invalid_word = binsof (operation) intersect {[1 : 3]} && binsof (word) intersect { + 1 + }; + } + zero_divisor: coverpoint (t.rhs == 0) iff (t.operation >= 4); + signed_overflow: coverpoint (t.a == 64'h8000000000000000 && t.rhs == '1) + iff (!t.word && t.operation inside {4,6}) { + bins seen = {1}; + } + word_overflow: coverpoint (t.a[31:0] == 32'h80000000 && t.rhs[31:0] == '1) + iff (t.word && t.operation inside {4,6}) { + bins seen = {1}; + } + sign_a: coverpoint t.a[63]; + sign_rhs: coverpoint t.rhs[63]; + signs: cross sign_a, sign_rhs; + backpressure: coverpoint t.stall {bins stalls[] = {0, 3};} + cancelled: coverpoint t.cancel; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void write(request_item t); + behavior.sample(t); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/env.svh new file mode 100644 index 000000000..8bd17acae --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/env.svh @@ -0,0 +1,26 @@ +`uvm_analysis_imp_decl(_reset) +class env extends environment #(request_item, response_item, model); + `uvm_component_utils(env) + agent bus; + coverage cov; + uvm_analysis_imp_reset #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + cov = coverage::type_id::create("coverage", this); + bus = agent::type_id::create("agent", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.requests.connect(input_export); + bus.mon.requests.connect(cov.analysis_export); + bus.mon.reset_ap.connect(reset_export); + bus.mon.responses.connect(output_export); + endfunction + function void write_reset(bit asserted); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/model.svh new file mode 100644 index 000000000..08bc0404c --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/env/model.svh @@ -0,0 +1,35 @@ +class model extends reference_model #(request_item, response_item); + `uvm_component_utils(model) + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void write(request_item req); + response_item rsp = response_item::type_id::create("expected"); + bit signed_a = req.operation < 4 ? req.operation inside {1, 2} : !req.operation[0]; + bit signed_b = req.operation < 4 ? req.operation == 1 : !req.operation[0]; + logic [63:0] a = req.word ? (signed_a ? {{32{req.a[31]}},req.a[31:0]} : {32'b0,req.a[31:0]}) : req.a; + logic [63:0] b = req.word ? (signed_b ? {{32{req.rhs[31]}},req.rhs[31:0]} : {32'b0,req.rhs[31:0]}) : req.rhs; + logic signed [128:0] wide_a = signed_a ? $signed({{65{a[63]}}, a}) : $signed({65'b0, a}); + logic signed [128:0] wide_b = signed_b ? $signed({{65{b[63]}}, b}) : $signed({65'b0, b}); + logic signed [257:0] product = wide_a * wide_b; + logic [63:0] answer; + if (req.operation < 4) begin + answer = req.operation == 0 ? product[63:0] : product[127:64]; + end else if (b == 0) begin + answer = req.operation[1] ? a : '1; + end else if (!req.operation[0] && a == 64'h8000000000000000 && b == '1) begin + answer = req.operation[1] ? 0 : a; + end else begin + case (req.operation) + 4: answer = $signed(a) / $signed(b); + 5: answer = a / b; + 6: answer = $signed(a) % $signed(b); + 7: answer = a % b; + default: `uvm_fatal("OPERATION", "Invalid RV64M operation") + endcase + end + rsp.value = req.word ? {{32{answer[31]}}, answer[31:0]} : answer; + expected_ap.write(rsp); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/filelist.f new file mode 100644 index 000000000..cab59cc74 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/filelist.f @@ -0,0 +1,7 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/MulDiv/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/interface.sv new file mode 100644 index 000000000..d78bd4d68 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/interface.sv @@ -0,0 +1,23 @@ +interface port_if ( + input logic clock, + input logic reset +); + logic request_valid, request_ready, response_valid, response_ready; + logic [63:0] a; + logic [63:0] rhs; + logic [2:0] operation; + logic word; + int unsigned stall; + logic cancel; + logic [63:0] value; + assert property (@(posedge clock) disable iff (reset) + request_valid && !request_ready |=> request_valid && $stable( + {a, rhs, operation, word} + )) + else $fatal(1, "MulDiv request changed under backpressure"); + assert property (@(posedge clock) disable iff (reset) + response_valid && !response_ready |=> response_valid && $stable( + value + )) + else $fatal(1, "MulDiv result changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/package.sv b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/package.sv new file mode 100644 index 000000000..da90543ec --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/package.sv @@ -0,0 +1,15 @@ +package muldiv_pkg; + timeunit 1ns; timeprecision 1ps; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..8b5acb2c7 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/seq/sequence.svh @@ -0,0 +1,46 @@ +class stimulus extends uvm_sequence #(request_item); + `uvm_object_utils(stimulus) + int unsigned sent = 0; + function new(string name = "stimulus"); + super.new(name); + endfunction + task body(); + request_item req = request_item::type_id::create("request"); + logic [63:0] operands[10] = '{ + 0, + 1, + 3, + '1, + 64'h8000000000000000, + 64'h7fffffffffffffff, + 64'h80000000, + 64'h7fffffff, + 64'hffffffff, + 64'h123456789abcdef0 + }; + start_item(req); + req.a = 123456789; + req.rhs = 7; + req.operation = 4; + req.word = 0; + req.stall = 0; + req.cancel = 1; + finish_item(req); + for (int word = 0; word < 2; word++) + for (int operation = 0; operation < 8; operation++) begin + if (word && operation inside {1, 2, 3}) continue; + foreach (operands[a]) + foreach (operands[b]) begin + start_item(req); + req.a = operands[a]; + req.rhs = operands[b]; + req.operation = operation; + req.word = word; + req.stall = ((a + b) % 2) * 3; + req.cancel = 0; + finish_item(req); + sent++; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/suite.toml new file mode 100644 index 000000000..6d7c8d91d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "muldiv" +top = "muldiv_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 2000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["muldiv_pkg::coverage::behavior"] +sink_ready = 1 +sink_stall = 0 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/tb.sv new file mode 100644 index 000000000..9c8f8f43e --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/tb.sv @@ -0,0 +1,29 @@ +module muldiv_tb; + import uvm_pkg::*; + import muldiv_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + port_if bus ( + clock, + timing.reset + ); + MulDiv dut ( + .clock(clock), + .reset(bus.reset), + .io_request_valid(bus.request_valid), + .io_request_ready(bus.request_ready), + .io_request_bits_a(bus.a), + .io_request_bits_rhs(bus.rhs), + .io_request_bits_operation(bus.operation), + .io_request_bits_word(bus.word), + .io_response_valid(bus.response_valid), + .io_response_ready(bus.response_ready), + .io_response_bits(bus.value) + ); + initial begin + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/tests/test.svh new file mode 100644 index 000000000..7013953b6 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/resources/tests/test.svh @@ -0,0 +1,26 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env bench; + reset_controller resetter; + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + resetter = reset_controller::type_id::create("reset", this); + uvm_config_db#(reset_controller)::set(this, "env.agent.driver", "resetter", resetter); + bench = env::type_id::create("env", this); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Test interface is required") + endfunction + task execute(); + stimulus sequence_ = stimulus::type_id::create("sequence"); + resetter.pulse(4); + sequence_.start(bench.bus.seqr); + bench.scoreboard.wait_checked(sequence_.sent); + @(negedge vif.clock); + evidence::check(evidence::cancelled == 1, "Reset did not cancel the outstanding divide"); + evidence::check(bench.scoreboard.checked == sequence_.sent, "Comparison count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/Emit.scala new file mode 100644 index 000000000..d6833741d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/Emit.scala @@ -0,0 +1,15 @@ +package framework.core.execution.muldiv + +import framework.core.execution.muldiv.configs.Params +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit extends App { + require(args.length >= 2 && args(0) == "--config") + val param = read[Params](Files.readString(Paths.get(args(1)))) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new MulDiv(param), + firtoolOpts = args.drop(2), + args = Array("--target-dir", "build/MulDiv", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/MulDiv.scala b/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/MulDiv.scala new file mode 100644 index 000000000..ce072d196 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/MulDiv.scala @@ -0,0 +1,99 @@ +package framework.core.execution.muldiv + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.execution.muldiv.configs.{Params => ModuleParam} + +class Request(b: ModuleParam) extends Bundle { + val p = b + + val a = UInt(p.xLen.W) + val rhs = UInt(p.xLen.W) + val operation = UInt(3.W) + val word = Bool() +} + +/** RV64M multiplier and restoring divider; one operation owns the circuit until response. */ +class MulDivIO(b: ModuleParam) extends Bundle { + val p = b + + val request = Flipped(Decoupled(new Request(b))) + val response = Decoupled(UInt(p.xLen.W)) + +} + +@instantiable +class MulDiv(b: ModuleParam) extends Module { + val p = b + + @public + val io = IO(new MulDivIO(b)) + + val idle :: divide :: respond :: Nil = Enum(3) + val state = RegInit(idle) + val quotient = Reg(UInt(64.W)) + val remainder = Reg(UInt(65.W)) + val divisor = Reg(UInt(65.W)) + val count = Reg(UInt(7.W)) + val negateQuotient = Reg(Bool()) + val negateRemainder = Reg(Bool()) + val returnRemainder = Reg(Bool()) + val word = Reg(Bool()) + val result = Reg(UInt(64.W)) + + io.request.ready := state === idle + io.response.valid := state === respond + io.response.bits := Mux(word, Cat(Fill(32, result(31)), result(31, 0)), result) + when(io.response.fire)(state := idle) + + val r = io.request.bits + val signedDivision = !r.operation(0) + val signedA = Mux(r.operation(2), signedDivision, r.operation === 1.U || r.operation === 2.U) + val signedB = Mux(r.operation(2), signedDivision, r.operation === 1.U) + val a = Mux(r.word, Cat(Fill(32, signedA && r.a(31)), r.a(31, 0)), r.a) + val rhs = Mux(r.word, Cat(Fill(32, signedB && r.rhs(31)), r.rhs(31, 0)), r.rhs) + val negativeA = signedA && a(63) + val negativeB = signedB && rhs(63) + val magnitudeA = Mux(negativeA, -a, a) + val magnitudeB = Mux(negativeB, -rhs, rhs) + val product = (Cat(negativeA, a).asSInt * Cat(negativeB, rhs).asSInt).asUInt + + when(io.request.fire) { + word := r.word + when(!r.operation(2)) { + result := Mux(r.operation === 0.U, product(63, 0), product(127, 64)) + state := respond + }.otherwise { + quotient := magnitudeA + remainder := 0.U + divisor := magnitudeB + count := 64.U + negateQuotient := negativeA ^ negativeB + negateRemainder := negativeA + returnRemainder := r.operation(1) + when(rhs === 0.U) { + result := Mux(r.operation(1), a, ~0.U(64.W)) + state := respond + }.otherwise(state := divide) + } + } + + val shifted = Cat(remainder(63, 0), quotient(63)) + val subtract = shifted >= divisor + val nextRemainder = Mux(subtract, shifted - divisor, shifted) + val nextQuotient = Cat(quotient(62, 0), subtract) + when(state === divide) { + remainder := nextRemainder + quotient := nextQuotient + count := count - 1.U + when(count === 1.U) { + result := Mux( + returnRemainder, + Mux(negateRemainder, -nextRemainder(63, 0), nextRemainder(63, 0)), + Mux(negateQuotient, -nextQuotient, nextQuotient) + ) + state := respond + } + } +} diff --git a/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..4e2a371ac --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/muldiv/src/main/scala/configs/Params.scala @@ -0,0 +1,6 @@ +package framework.core.execution.muldiv.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Params(xLen: Int) { require(xLen == 64) } +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/execution/pipeline/Sequencer.scala b/arch/src/main/scala/framework/core/execution/pipeline/Sequencer.scala new file mode 100644 index 000000000..305f15853 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/pipeline/Sequencer.scala @@ -0,0 +1,453 @@ +package framework.core.execution.pipeline + +import chisel3._ +import chisel3.util._ +import framework.acc.frontend.RoCCIO +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.execution.configs.ExecutionParam +import framework.core.execution.instruction.InstructionIO +import framework.core.execution.register_file.RegisterFileIO +import framework.core.execution.integer.IntegerIO +import framework.core.execution.muldiv.MulDivIO +import framework.core.execution.fpu.FPUIO +import framework.core.execution.privileged.{PrivilegedIO, Trap} +import framework.core.memory.access.{UncachedRequest, UncachedResponse, VirtualPort, VirtualRequest} +import framework.core.controller.admission.Commands +import framework.acc.memdomain.MvoverISA +import framework.core.clink.CoreInterrupts +import framework.ip.bus.chi.rnf.CacheAtomic + +class Retirement extends Bundle { + val pc = UInt(64.W) + val instruction = UInt(32.W) + val destination = UInt(5.W) + val data = UInt(64.W) + val integer = Bool() + val floating = Bool() +} + +/** Architectural sequencing and retirement; functional units are connected by Execution. */ +@instantiable +class Sequencer(b: ExecutionParam, localRocc: Boolean) extends Module { + val core = b + val resetVector = core.resetVector + val hartId = core.hartId + val commandsEnabled = core.commandsEnabled + val tracking = core.tracking + + @public + val io = IO(new Bundle { + val interrupts = Input(new CoreInterrupts) + val controller = Option.when(!localRocc)(new Commands(b.tracking, b.integer.xLen, b.privileged.permission)) + val rocc = Option.when(localRocc)(Flipped(new RoCCIO(b.integer.xLen, b.privileged.permission))) + val workDrained = Input(Bool()) + val retirement = Valid(new Retirement) + val fetch = Flipped(new VirtualPort(b.memoryAccess)) + val memory = Flipped(new VirtualPort(b.memoryAccess)) + val timeRequest = Decoupled(new UncachedRequest(b.memoryAccess.memory)) + val timeResponse = Flipped(Decoupled(new UncachedResponse(b.memoryAccess.memory))) + val instruction = Flipped(new InstructionIO(b.instruction)) + val registers = Flipped(new RegisterFileIO(b.registerFile)) + val integer = Flipped(new IntegerIO(b.integer)) + val multiply = Flipped(new MulDivIO(b.muldiv)) + val floating = Flipped(new FPUIO(b.fpu)) + val privileged = Flipped(new PrivilegedIO(b.privileged)) + }) + + val retirement = io.retirement + val starting :: fetchLow :: waitLow :: fetchHigh :: waitHigh :: operands :: execute :: sendMemory :: waitMemory :: sendTime :: waitTime :: waitMultiply :: waitFloating :: reserveCommand :: sendCommand :: waitCommand :: fence :: sleeping :: trapped :: Nil = + Enum(19) + val state = RegInit(starting) + val pc = Reg(UInt(64.W)) + val encoded = Reg(UInt(32.W)) + val half = Reg(UInt(16.W)) + val compressed = Reg(Bool()) + + val memoryRequest = Reg(new VirtualRequest(b.memoryAccess)) + + val trap = Reg(new Trap) + val fpRegisters = Reg(Vec(32, UInt(64.W))) + val commandResponse = RegInit(false.B) + + io.instruction.bits := encoded + val insn = io.instruction.expanded + val opcode = io.instruction.opcode + val funct3 = io.instruction.funct3 + val funct7 = io.instruction.funct7 + val rs1 = io.instruction.rs1 + val rs2 = io.instruction.rs2 + val rd = io.instruction.rd + io.registers.readAddress(0) := rs1 + io.registers.readAddress(1) := rs2 + val a = Reg(UInt(64.W)) + val operandB = Reg(UInt(64.W)) + when(state === operands) { + a := io.registers.readData(0) + operandB := io.registers.readData(1) + state := execute + } + val immediate = io.instruction.immediate + val sequentialPc = pc + Mux(compressed, 2.U, 4.U) + val writeInteger = WireDefault(false.B) + val writeFloating = WireDefault(false.B) + val writeData = WireDefault(0.U(64.W)) + val commit = WireDefault(false.B) + val nextPc = WireDefault(sequentialPc) + val exception = WireDefault(false.B) + val cause = WireDefault(2.U(64.W)) + val value = WireDefault(Mux(compressed, encoded(15, 0).pad(64), encoded.pad(64))) + val legal = WireDefault(false.B) + val startMemory = WireDefault(false.B) + val store = opcode === "h23".U || opcode === "h27".U + val fpMemory = opcode === "h07".U || opcode === "h27".U + val atomic = opcode === "h2f".U + val atomicOperation = funct7(6, 2) + val writesMemory = store || (atomic && atomicOperation =/= 2.U) + val effectiveAddress = a + Mux(atomic, 0.U, immediate) + val memorySize = Mux(atomic, funct3, funct3(1, 0)) + val memoryData = Mux(fpMemory, fpRegisters(rs2), operandB) + + val atomicCode = io.instruction.atomicCode + + io.privileged.hartId := hartId.U(64.W) + io.privileged.interrupts := io.interrupts + io.privileged.retired := commit + io.privileged.floating.valid := writeFloating || (state === waitFloating && io.floating.response.fire && !io.floating.response.bits.illegal) + io.privileged.floating.bits := Mux(state === waitFloating, io.floating.response.bits.flags, 0.U) + io.privileged.trap.valid := state === trapped + io.privileged.trap.bits := trap + io.privileged.returning.valid := false.B + io.privileged.returning.bits := 0.U + io.privileged.csr.address := io.instruction.csrAddress + io.privileged.csr.operation := funct3(1, 0) + io.privileged.csr.operand := Mux(funct3(2), rs1.pad(64), a) + io.privileged.csr.write := funct3(1, 0) === 1.U || rs1 =/= 0.U + io.privileged.csr.commit := false.B + + io.registers.write.valid := writeInteger && commit + io.registers.write.bits.address := rd + io.registers.write.bits.data := writeData + when(writeFloating && commit)(fpRegisters(rd) := writeData) + retirement.valid := commit + retirement.bits.pc := pc + retirement.bits.instruction := encoded + retirement.bits.destination := rd + retirement.bits.data := writeData + retirement.bits.integer := writeInteger && rd =/= 0.U + retirement.bits.floating := writeFloating + + io.integer.in1 := a + io.integer.in2 := Mux(io.instruction.integerImmediate, immediate, operandB) + io.integer.dw := !io.instruction.integerWord + io.integer.fn := io.instruction.integerFunction + io.multiply.request.valid := false.B + io.multiply.request.bits.a := a + io.multiply.request.bits.rhs := operandB + io.multiply.request.bits.operation := funct3 + io.multiply.request.bits.word := opcode === "h3b".U + io.multiply.response.ready := state === waitMultiply + io.floating.request.valid := false.B + io.floating.request.bits.instruction := insn + io.floating.request.bits.integer := a + io.floating.request.bits.a := fpRegisters(rs1) + io.floating.request.bits.rhs := fpRegisters(rs2) + io.floating.request.bits.c := fpRegisters(funct7(6, 2)) + io.floating.request.bits.rounding := io.privileged.rounding + io.floating.response.ready := state === waitFloating + + val flushTranslation = WireDefault(false.B) + io.fetch.flushTranslation := flushTranslation + io.memory.flushTranslation := flushTranslation + io.fetch.request.valid := (state === fetchLow && !io.privileged.interrupt.valid) || state === fetchHigh + io.fetch.request.bits := 0.U.asTypeOf(io.fetch.request.bits) + io.fetch.request.bits.vaddr := Mux(state === fetchHigh, pc + 2.U, pc) + io.fetch.request.bits.size := 1.U + io.fetch.request.bits.execute := true.B + io.fetch.request.bits.privilege := io.privileged.privilege + io.fetch.request.bits.sum := io.privileged.sum + io.fetch.request.bits.mxr := io.privileged.mxr + io.fetch.request.bits.satpMode := io.privileged.satp(63, 60) + io.fetch.request.bits.rootPpn := io.privileged.satp(43, 0) + io.fetch.response.ready := state === waitLow || state === waitHigh + io.memory.request.valid := state === sendMemory + io.memory.request.bits := memoryRequest + io.memory.response.ready := state === waitMemory + + when(state === starting) { pc := resetVector.U(64.W); state := fetchLow } + when(state === fetchLow && io.privileged.interrupt.valid) { + trap.pc := pc; trap.cause := io.privileged.interrupt.bits; trap.value := 0.U; state := trapped + } + when(io.fetch.request.fire)(state := Mux(state === fetchHigh, waitHigh, waitLow)) + when(io.fetch.response.fire) { + val response = io.fetch.response.bits + when(response.accessFault || response.pageFault || response.misaligned) { + trap.pc := pc + trap.cause := Mux(response.misaligned, 0.U, Mux(response.pageFault, 12.U, 1.U)) + trap.value := Mux(state === waitHigh, pc + 2.U, pc) + state := trapped + }.elsewhen(state === waitHigh) { + encoded := Cat(response.data(15, 0), half) + compressed := false.B + state := operands + }.elsewhen(response.data(1, 0) =/= 3.U) { + encoded := response.data(15, 0) + compressed := true.B + state := operands + }.otherwise { + half := response.data(15, 0) + state := fetchHigh + } + } + when(state === trapped) { + printf(p"TRAP pc=0x${Hexadecimal(trap.pc)} cause=0x${Hexadecimal(trap.cause)} value=0x${Hexadecimal(trap.value)}\n") + pc := io.privileged.redirect.bits; state := fetchLow + } + + when(state === execute) { + switch(opcode) { + is("h37".U) { legal := true.B; writeData := immediate; writeInteger := true.B; commit := true.B } + is("h17".U) { legal := true.B; writeData := pc + immediate; writeInteger := true.B; commit := true.B } + is("h6f".U) { + legal := true.B; nextPc := pc + immediate; writeData := sequentialPc; writeInteger := true.B; commit := true.B + } + is("h67".U) { + legal := true.B; nextPc := (a + immediate) & ~1.U(64.W) + writeData := sequentialPc; writeInteger := true.B; commit := true.B + } + is("h63".U) { + legal := true.B + val taken = MuxLookup(funct3, false.B)(Seq( + 0.U -> (a === operandB), + 1.U -> (a =/= operandB), + 4.U -> (a.asSInt < operandB.asSInt), + 5.U -> (a.asSInt >= operandB.asSInt), + 6.U -> (a < operandB), + 7.U -> (a >= operandB) + )) + nextPc := Mux(taken, pc + immediate, sequentialPc); commit := true.B + } + is("h13".U, "h1b".U) { + legal := true.B + writeData := io.integer.out; writeInteger := true.B; commit := true.B + } + is("h33".U, "h3b".U) { + when(funct7 === 1.U) { + legal := true.B + io.multiply.request.valid := legal && !io.instruction.illegal + when(io.multiply.request.fire)(state := waitMultiply) + }.otherwise { + legal := true.B + writeData := io.integer.out; writeInteger := true.B; commit := true.B + } + } + is("h03".U, "h23".U, "h07".U, "h27".U) { + legal := !fpMemory || io.privileged.floatingEnabled + startMemory := true.B + } + is("h2f".U) { + legal := true.B + startMemory := true.B + } + is("h43".U, "h47".U, "h4b".U, "h4f".U, "h53".U) { + legal := io.privileged.floatingEnabled + io.floating.request.valid := legal && !io.instruction.illegal + when(io.floating.request.fire)(state := waitFloating) + } + is("h0f".U) { + legal := true.B + state := fence + } + is("h73".U) { + when(funct3 =/= 0.U) { + legal := !io.privileged.csr.illegal + when(io.privileged.csr.memoryRead && legal && !io.instruction.illegal) { + state := sendTime + }.otherwise { + io.privileged.csr.commit := legal && !io.instruction.illegal + writeData := io.privileged.csr.data; writeInteger := true.B; commit := true.B + } + }.otherwise { + switch(insn) { + is("h00000073".U) { + legal := true.B; exception := true.B; cause := 8.U + io.privileged.privilege; value := 0.U + } + is("h00100073".U) { legal := true.B; exception := true.B; cause := 3.U; value := pc } + is("h30200073".U) { + legal := io.privileged.privilege === 3.U + io.privileged.returning.valid := legal; io.privileged.returning.bits := 3.U + nextPc := io.privileged.redirect.bits; commit := true.B + } + is("h10200073".U) { + legal := io.privileged.privilege >= 1.U && (io.privileged.privilege === 3.U || !io.privileged.tsr) + io.privileged.returning.valid := legal; io.privileged.returning.bits := 1.U + nextPc := io.privileged.redirect.bits; commit := true.B + } + is("h10500073".U) { + legal := io.privileged.privilege >= 1.U && (io.privileged.privilege === 3.U || !io.privileged.tw) + commit := true.B; state := sleeping + } + } + when(funct7 === "h09".U && rd === 0.U) { + legal := io.privileged.privilege >= 1.U && (io.privileged.privilege === 3.U || !io.privileged.tvm) + flushTranslation := legal; commit := true.B + } + } + } + is("h2b".U) { + legal := commandsEnabled.B && !localRocc.B + when(legal)(state := (if (localRocc) sendCommand else reserveCommand)) + } + is("h7b".U) { + legal := commandsEnabled.B && (core.directAccelerator.B || (core.moves.B && funct7 === MvoverISA.Funct.U)) + when(legal)(state := (if (localRocc) sendCommand else reserveCommand)) + } + } + when(startMemory && legal && !io.instruction.illegal) { + memoryRequest := 0.U.asTypeOf(memoryRequest) + memoryRequest.vaddr := effectiveAddress + memoryRequest.size := memorySize + memoryRequest.write := store + memoryRequest.signed := !funct3(2) && !fpMemory + memoryRequest.data := memoryData + memoryRequest.atomic := Mux(atomic, atomicCode, CacheAtomic.None.U) + memoryRequest.privilege := io.privileged.dataPrivilege + memoryRequest.sum := io.privileged.sum + memoryRequest.mxr := io.privileged.mxr + memoryRequest.satpMode := io.privileged.satp(63, 60) + memoryRequest.rootPpn := io.privileged.satp(43, 0) + state := sendMemory + } + when(!legal || io.instruction.illegal) { + exception := true.B; cause := 2.U + } + } + + when(io.memory.request.fire)(state := waitMemory) + when(io.memory.response.fire) { + val response = io.memory.response.bits + when(response.misaligned || response.pageFault || response.accessFault) { + exception := true.B + cause := Mux( + response.misaligned, + Mux(writesMemory, 6.U, 4.U), + Mux(response.pageFault, Mux(writesMemory, 15.U, 13.U), Mux(writesMemory, 7.U, 5.U)) + ) + value := memoryRequest.vaddr + }.otherwise { + commit := true.B + writeInteger := !store && !fpMemory + writeFloating := !store && fpMemory + writeData := Mux(fpMemory && funct3 === 2.U, Cat(~0.U(32.W), response.data(31, 0)), response.data) + } + } + when(io.multiply.response.fire) { writeInteger := true.B; writeData := io.multiply.response.bits; commit := true.B } + when(io.floating.response.fire) { + when(io.floating.response.bits.illegal)(exception := true.B) + .otherwise { + writeInteger := io.floating.response.bits.integer; writeFloating := !io.floating.response.bits.integer + writeData := io.floating.response.bits.data; commit := true.B + } + } + when(state === fence && io.workDrained)(commit := true.B) + when(state === sleeping && io.privileged.wake)(state := fetchLow) + when(commit) { + pc := nextPc + when(!(state === execute && insn === "h10500073".U))(state := fetchLow) + } + when(exception) { + commit := false.B + io.privileged.csr.commit := false.B + trap.pc := pc; trap.cause := cause; trap.value := value; state := trapped + } + + if (localRocc) { + val rocc = io.rocc.get + rocc.cmd.valid := state === sendCommand + rocc.cmd.bits.instruction.raw_inst := insn + rocc.cmd.bits.instruction.pc := pc + rocc.cmd.bits.instruction.rs1Data := a + rocc.cmd.bits.instruction.rs2Data := operandB + rocc.cmd.bits.satp := io.privileged.satp + rocc.cmd.bits.effectivePrivilege := io.privileged.dataPrivilege + rocc.cmd.bits.sum := io.privileged.sum + rocc.cmd.bits.mxr := io.privileged.mxr + rocc.cmd.bits.pmp := io.privileged.pmp + when(rocc.cmd.fire) { + commandResponse := funct3(2) + when(funct3(2))(state := waitCommand).otherwise(commit := true.B) + } + rocc.resp.ready := state === waitCommand + when(rocc.resp.fire) { + assert( + commandResponse && rocc.resp.bits.rd === rd, + "NPU response destination does not match the accepted command" + ) + writeInteger := true.B; writeData := rocc.resp.bits.data; commit := true.B + } + } else { + val commandsLive = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) + val commandTag = Reg(UInt(tracking.idBits.W)) + val host = io.controller.get + val availableTag = PriorityEncoder(~commandsLive.asUInt) + when(state === reserveCommand && !commandsLive.asUInt.andR && !reset.asBool) { + commandTag := availableTag + commandsLive(availableTag) := true.B + state := sendCommand + } + host.command.valid := state === sendCommand + host.command.bits.tag := commandTag + host.command.bits.instruction.raw_inst := insn + host.command.bits.instruction.pc := pc + host.command.bits.instruction.rs1Data := a + host.command.bits.instruction.rs2Data := operandB + host.command.bits.satp := io.privileged.satp + host.command.bits.effectivePrivilege := io.privileged.dataPrivilege + host.command.bits.sum := io.privileged.sum + host.command.bits.mxr := io.privileged.mxr + host.command.bits.pmp := io.privileged.pmp + when(host.command.fire) { + commandResponse := funct3(2) + when(funct3(2))(state := waitCommand).otherwise(commit := true.B) + } + host.response.ready := state === waitCommand + when(host.response.fire) { + assert( + commandResponse && host.response.bits.rd === rd, + "NPU response destination does not match the accepted command" + ) + writeInteger := true.B; writeData := host.response.bits.data; commit := true.B + } + host.complete.ready := true.B + when(host.complete.fire)(commandsLive(host.complete.bits.tag) := false.B) + } + // PMP replies retain the accepted decision through backpressure. + for ((virtual, index) <- Seq(io.fetch, io.memory).zipWithIndex) { + val pending = RegInit(false.B) + val allowed = Reg(Bool()) + io.privileged.authorization(index) := virtual.authorizationRequest.bits + virtual.authorizationRequest.ready := !pending + virtual.authorizationResponse.valid := pending + virtual.authorizationResponse.bits := allowed + when(virtual.authorizationRequest.fire) { pending := true.B; allowed := io.privileged.permitted(index) } + when(virtual.authorizationResponse.fire)(pending := false.B) + } + + val timeRequest = io.timeRequest + timeRequest.valid := state === sendTime + timeRequest.bits.addr := b.mtimeAddress.U + timeRequest.bits.tag := 0.U + timeRequest.bits.size := 3.U + timeRequest.bits.write := false.B + timeRequest.bits.data := 0.U + when(timeRequest.fire)(state := waitTime) + val timeResponse = io.timeResponse + timeResponse.ready := state === waitTime + when(timeResponse.fire) { + when(timeResponse.bits.error) { + exception := true.B; cause := 2.U; value := encoded + }.otherwise { + writeInteger := true.B; writeData := timeResponse.bits.data; commit := true.B + } + } + +} diff --git a/arch/src/main/scala/framework/core/execution/privileged/build.sc b/arch/src/main/scala/framework/core/execution/privileged/build.sc new file mode 100644 index 000000000..56a117440 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.execution.privileged.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Privileged.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/execution/privileged/configs/uvm.toml b/arch/src/main/scala/framework/core/execution/privileged/configs/uvm.toml new file mode 100644 index 000000000..b6f3b4b96 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/configs/uvm.toml @@ -0,0 +1,5 @@ +[parameters.permission] +addressBits = 44 +entries = 16 + +[defines] diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/csrc/Cargo.lock b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/Cargo.lock new file mode 100644 index 000000000..c84e4c329 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "privileged-reference" +version = "0.1.0" diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/csrc/Cargo.toml b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/Cargo.toml new file mode 100644 index 000000000..86130c2a7 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "privileged-reference" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/core/mod.rs b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/core/mod.rs new file mode 100644 index 000000000..fb27202ca --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/core/mod.rs @@ -0,0 +1 @@ +pub mod riscv; diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/core/riscv/mod.rs b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/core/riscv/mod.rs new file mode 100644 index 000000000..ef3108d2b --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/core/riscv/mod.rs @@ -0,0 +1,14 @@ +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/bus.rs"] +pub mod bus; +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/csr.rs"] +pub mod csr; +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/mmu.rs"] +pub mod mmu; +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/pmp.rs"] +pub mod pmp; +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/privilege.rs"] +pub mod privilege; +#[path = "../../../../../../../../../../../../../bebop/src/nodes/bemu/src/core/riscv/trap.rs"] +pub mod trap; +pub use privilege::Privilege; +pub use trap::{Access, Trap}; diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/lib.rs b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/lib.rs new file mode 100644 index 000000000..3e013d40b --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/csrc/src/lib.rs @@ -0,0 +1,326 @@ +mod core; +use core::riscv::{ + csr::{Csrs, FS, MIE, SIE}, + Privilege, +}; +use std::ffi::c_void; + +#[derive(Default, Clone, Copy)] +struct Inputs { + address: u16, + operand: u64, + operation: u32, + write: bool, + commit: bool, + trap: bool, + pc: u64, + cause: u64, + value: u64, + returning: bool, + return_mode: u32, + retired: bool, + floating: bool, + flags: u8, +} +struct Model { + csrs: Csrs, + privilege: Privilege, + satp: u64, + address_bits: u32, + inputs: Inputs, + values: [u64; 18], + read_value: u64, + illegal: bool, +} +fn mode(value: u32) -> Privilege { + match value { + 0 => Privilege::User, + 1 => Privilege::Supervisor, + 3 => Privilege::Machine, + _ => panic!("Reserved privilege encoding"), + } +} +unsafe fn model<'a>(p: *mut c_void) -> &'a mut Model { + &mut *p.cast::() +} +#[no_mangle] +pub extern "C" fn privileged_create(address_bits: u32) -> *mut c_void { + assert!((32..=56).contains(&address_bits)); + Box::into_raw(Box::new(Model { + csrs: Csrs::default(), + privilege: Privilege::Machine, + satp: 0, + address_bits, + inputs: Inputs::default(), + values: [0; 18], + read_value: 0, + illegal: false, + })) + .cast() +} +#[no_mangle] +pub unsafe extern "C" fn privileged_destroy(p: *mut c_void) { + drop(Box::from_raw(p.cast::())); +} +#[no_mangle] +pub unsafe extern "C" fn privileged_reset(p: *mut c_void) { + let m = model(p); + m.csrs = Csrs::default(); + m.privilege = Privilege::Machine; + m.satp = 0; +} +#[no_mangle] +pub unsafe extern "C" fn privileged_evaluate( + p: *mut c_void, + address: u32, + operand: u64, + operation: u32, + write: u8, + commit: u8, + trap: u8, + pc: u64, + cause: u64, + value: u64, + returning: u8, + return_mode: u32, + hart: u64, + interrupts: u64, + retired: u8, + floating: u8, + flags: u32, +) { + let m = model(p); + m.inputs = Inputs { + address: address as u16, + operand, + operation, + write: write != 0, + commit: commit != 0, + trap: trap != 0, + pc, + cause, + value, + returning: returning != 0, + return_mode, + retired: retired != 0, + floating: floating != 0, + flags: flags as u8, + }; + let read = match address { + // The hardware profile exposes RV64 mstatush as zero and omits environment CSRs. + 0x310 if m.privilege == Privilege::Machine => Ok(0), + 0x10a | 0x30a => Err(core::riscv::csr::CsrError), + _ => m + .csrs + .read(address as u16, m.privilege, hart, 0, interrupts), + }; + m.illegal = read.is_err() || (write != 0 && address >> 10 == 3); + m.read_value = read.unwrap_or(0); + if address == 0x180 && !m.illegal { + m.read_value = m.satp; + } + let pending = m.csrs.software_pending | interrupts; + let enabled = pending & m.csrs.ie; + let machine = m.privilege != Privilege::Machine || m.csrs.status & MIE != 0; + let supervisor = m.privilege == Privilege::User + || (m.privilege == Privilege::Supervisor && m.csrs.status & SIE != 0); + let eligible = enabled + & ((if machine { !m.csrs.ideleg } else { 0 }) + | (if supervisor { m.csrs.ideleg } else { 0 })); + let interrupt = [11, 3, 7, 9, 1, 5] + .into_iter() + .find(|bit| eligible & (1 << bit) != 0); + let delegated = m.privilege != Privilege::Machine + && (if cause >> 63 != 0 { + m.csrs.ideleg + } else { + m.csrs.edeleg + }) & (1 << (cause & 63)) + != 0; + let vector = m.csrs.tvec[usize::from(!delegated)]; + let redirect = if trap != 0 { + (vector & !3).wrapping_add(if cause >> 63 != 0 && vector & 1 != 0 { + (cause & 63) * 4 + } else { + 0 + }) + } else if return_mode == 3 { + m.csrs.epc[1] + } else { + m.csrs.epc[0] + }; + let status = m.csrs.status; + m.values = [ + if m.illegal { 0 } else { m.read_value }, + u64::from(m.illegal), + u64::from(address == 0xc01), + u64::from(trap != 0 || returning != 0), + if trap != 0 || returning != 0 { + redirect + } else { + 0 + }, + u64::from(interrupt.is_some()), + interrupt.map(|bit| 1u64 << 63 | bit).unwrap_or(0), + u64::from(enabled != 0), + m.privilege as u64, + if status & (1 << 17) != 0 { + (status >> 11) & 3 + } else { + m.privilege as u64 + }, + m.satp, + (status >> 18) & 1, + (status >> 19) & 1, + u64::from(status & FS != 0), + u64::from(m.csrs.fcsr >> 5), + (status >> 20) & 1, + (status >> 21) & 1, + (status >> 22) & 1, + ]; +} +#[no_mangle] +pub unsafe extern "C" fn privileged_value(p: *mut c_void, index: u32) -> u64 { + model(p).values[index as usize] +} +#[no_mangle] +pub unsafe extern "C" fn privileged_pmp(p: *mut c_void, index: u32, field: u32) -> u64 { + let m = model(p); + let address = m.csrs.pmp.address(index as usize); + match field { + 0 => u64::from(m.csrs.pmp.config(index as usize)), + 1 => address, + 2 => (((address ^ address.wrapping_add(1)) << 2) | 3) & ((1 << m.address_bits) - 1), + _ => panic!("Unknown PMP field"), + } +} +#[no_mangle] +pub unsafe extern "C" fn privileged_permitted( + p: *mut c_void, + address: u64, + size: u32, + read: u8, + write: u8, + execute: u8, + privilege: u32, +) -> u8 { + let m = model(p); + let end = u128::from(address) + (1u128 << size); + for index in 0..16 { + let cfg = m.csrs.pmp.config(index); + let encoded = u128::from(m.csrs.pmp.address(index)); + let (low, high) = match (cfg >> 3) & 3 { + 0 => continue, + 1 => ( + if index == 0 { + 0 + } else { + u128::from(m.csrs.pmp.address(index - 1)) * 4 + }, + encoded * 4, + ), + 2 => (encoded * 4, encoded * 4 + 4), + 3 => { + let bytes = (1u128 << (m.csrs.pmp.address(index).trailing_ones() + 3)) + .min(1u128 << m.address_bits); + let low = encoded * 4 & !(bytes - 1); + (low, low + bytes) + } + _ => unreachable!(), + }; + if low >= high || u128::from(address) >= high || end <= low { + continue; + } + let access = (read == 0 || cfg & 1 != 0) + && (write == 0 || cfg & 2 != 0) + && (execute == 0 || cfg & 4 != 0); + return u8::from( + u128::from(address) >= low + && end <= high + && ((privilege == 3 && cfg & 0x80 == 0) || access), + ); + } + u8::from(privilege == 3) +} +#[no_mangle] +pub unsafe extern "C" fn privileged_advance(p: *mut c_void) { + let m = model(p); + let i = m.inputs; + let old_status = m.csrs.status; + let old_privilege = m.privilege; + if m.csrs.inhibit & 1 == 0 { + m.csrs.cycle = m.csrs.cycle.wrapping_add(1); + } + if i.retired && m.csrs.inhibit & 4 == 0 { + m.csrs.instret = m.csrs.instret.wrapping_add(1); + } + if i.floating { + m.csrs.fcsr |= i.flags & 31; + m.csrs.status |= FS; + } + if i.commit && i.write && !m.illegal { + let value = match i.operation { + 2 => m.read_value | i.operand, + 3 => m.read_value & !i.operand, + _ => i.operand, + }; + match i.address { + 0x180 => { + if value >> 60 == 0 || value >> 60 == 8 { + m.satp = value; + m.csrs.satp.write(value); + } + } + 0x310 => (), + address => { + let value = if (0x3b0..=0x3bf).contains(&address) { + value & ((1u64 << (m.address_bits - 2)) - 1) + } else { + value + }; + m.csrs + .write(address, value, old_privilege) + .expect("Legal CSR write rejected by the reference"); + } + } + } + if i.trap { + let delegated = old_privilege != Privilege::Machine + && (if i.cause >> 63 != 0 { + m.csrs.ideleg + } else { + m.csrs.edeleg + }) & (1 << (i.cause & 63)) + != 0; + let slot = usize::from(!delegated); + m.csrs.epc[slot] = i.pc & !1; + m.csrs.cause[slot] = i.cause; + m.csrs.tval[slot] = i.value; + if delegated { + m.csrs.status = old_status & !0x122 + | ((old_status >> 1) & 1) << 5 + | u64::from(old_privilege == Privilege::Supervisor) << 8; + m.privilege = Privilege::Supervisor; + } else { + m.csrs.status = + old_status & !0x1888 | ((old_status >> 3) & 1) << 7 | (old_privilege as u64) << 11; + m.privilege = Privilege::Machine; + } + } else if i.returning { + if i.return_mode == 3 { + m.privilege = mode(((old_status >> 11) & 3) as u32); + m.csrs.status = old_status & !0x1888 | ((old_status >> 7) & 1) << 3 | 0x80; + if m.privilege != Privilege::Machine { + m.csrs.status &= !(1 << 17); + } + } else { + assert_eq!(i.return_mode, 1); + m.privilege = if old_status & 0x100 != 0 { + Privilege::Supervisor + } else { + Privilege::User + }; + m.csrs.status = old_status & !0x20122 | ((old_status >> 5) & 1) << 1 | 0x20; + } + } +} diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..cb38e4c51 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual privileged_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual privileged_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Privileged interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.inputs = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/item.svh new file mode 100644 index 000000000..b50179a31 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`PRIV_INPUT_BITS-1:0] inputs; + logic [`PRIV_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + logic [`PRIV_OUTPUT_BITS-1:0] payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..e4e167708 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual privileged_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual privileged_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Privileged monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/env.svh new file mode 100644 index 000000000..416baa90b --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver drv; + monitor mon; + reference_model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + reference = reference_model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/model.svh new file mode 100644 index 000000000..69874dbc9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/model.svh @@ -0,0 +1,141 @@ +class reference_model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(reference_model) + chandle reference; + int pmp_offsets[16][8] = `PRIV_PMP_OFFSETS; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + covergroup behavior with function sample ( + int address, int operation, bit write, int privilege, bit trap, bit returning, bit interrupt + ); + option.per_instance = 1; + csr_operation: coverpoint operation {bins kinds[] = {[0 : 3]};} + access_direction: coverpoint write; + current_privilege: coverpoint privilege { + bins user_mode = {0}; bins supervisor = {1}; bins machine_mode = {3}; + } + trap_event: coverpoint trap; + return_event: coverpoint returning; + interrupt_event: coverpoint interrupt; + pmp_address: coverpoint (address - 'h3b0) iff (address >= 'h3b0 && address <= 'h3bf && write) { + bins entries[] = {[0 : 15]}; + } + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + behavior = new(); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + reference = privileged_create(`PRIV_ADDRESS_BITS); + endfunction + function void write(cycle_item t); + result_item expected, actual; + longint unsigned interrupts, cfg; + if (t.reset) begin + privileged_reset(reference); + return; + end + expected = result_item::type_id::create("expected"); + actual = result_item::type_id::create("actual"); + expected.payload = '0; + actual.payload = t.outputs; + interrupts = (longint'( + `PIN(t.inputs, INTERRUPTS_SOFTWARE) + ) << 3) | (longint'( + `PIN(t.inputs, INTERRUPTS_TIMER) + ) << 7) | (longint'( + `PIN(t.inputs, INTERRUPTS_EXTERNAL) + ) << 11) | (longint'( + `PIN(t.inputs, INTERRUPTS_SUPERVISOREXTERNAL) + ) << 9); + privileged_evaluate( + reference, `PIN(t.inputs, CSR_ADDRESS), `PIN(t.inputs, CSR_OPERAND), + `PIN(t.inputs, CSR_OPERATION), `PIN(t.inputs, CSR_WRITE), `PIN(t.inputs, CSR_COMMIT), + `PIN(t.inputs, TRAP_VALID), `PIN(t.inputs, TRAP_BITS_PC), `PIN(t.inputs, TRAP_BITS_CAUSE), + `PIN(t.inputs, TRAP_BITS_VALUE), `PIN(t.inputs, RETURNING_VALID), + `PIN(t.inputs, RETURNING_BITS), `PIN(t.inputs, HARTID), interrupts, `PIN(t.inputs, RETIRED), + `PIN(t.inputs, FLOATING_VALID), `PIN(t.inputs, FLOATING_BITS)); + expected.payload[`PRIV_OUT_CSR_DATA_OFFSET+:`PRIV_OUT_CSR_DATA_WIDTH] = + privileged_value(reference, 0); + expected.payload[`PRIV_OUT_CSR_ILLEGAL_OFFSET+:`PRIV_OUT_CSR_ILLEGAL_WIDTH] = + privileged_value(reference, 1); + expected.payload[`PRIV_OUT_CSR_MEMORYREAD_OFFSET +: `PRIV_OUT_CSR_MEMORYREAD_WIDTH] = privileged_value( + reference, 2); + expected.payload[`PRIV_OUT_REDIRECT_VALID_OFFSET +: `PRIV_OUT_REDIRECT_VALID_WIDTH] = privileged_value( + reference, 3); + expected.payload[`PRIV_OUT_REDIRECT_BITS_OFFSET +: `PRIV_OUT_REDIRECT_BITS_WIDTH] = privileged_value( + reference, 4); + expected.payload[`PRIV_OUT_INTERRUPT_VALID_OFFSET +: `PRIV_OUT_INTERRUPT_VALID_WIDTH] = privileged_value( + reference, 5); + expected.payload[`PRIV_OUT_INTERRUPT_BITS_OFFSET +: `PRIV_OUT_INTERRUPT_BITS_WIDTH] = privileged_value( + reference, 6); + expected.payload[`PRIV_OUT_WAKE_OFFSET+:`PRIV_OUT_WAKE_WIDTH] = privileged_value(reference, 7); + expected.payload[`PRIV_OUT_PRIVILEGE_OFFSET+:`PRIV_OUT_PRIVILEGE_WIDTH] = + privileged_value(reference, 8); + expected.payload[`PRIV_OUT_DATAPRIVILEGE_OFFSET +: `PRIV_OUT_DATAPRIVILEGE_WIDTH] = privileged_value( + reference, 9); + expected.payload[`PRIV_OUT_SATP_OFFSET+:`PRIV_OUT_SATP_WIDTH] = privileged_value(reference, 10); + expected.payload[`PRIV_OUT_SUM_OFFSET+:`PRIV_OUT_SUM_WIDTH] = privileged_value(reference, 11); + expected.payload[`PRIV_OUT_MXR_OFFSET+:`PRIV_OUT_MXR_WIDTH] = privileged_value(reference, 12); + expected.payload[`PRIV_OUT_FLOATINGENABLED_OFFSET +: `PRIV_OUT_FLOATINGENABLED_WIDTH] = privileged_value( + reference, 13); + expected.payload[`PRIV_OUT_ROUNDING_OFFSET+:`PRIV_OUT_ROUNDING_WIDTH] = + privileged_value(reference, 14); + expected.payload[`PRIV_OUT_TVM_OFFSET+:`PRIV_OUT_TVM_WIDTH] = privileged_value(reference, 15); + expected.payload[`PRIV_OUT_TW_OFFSET+:`PRIV_OUT_TW_WIDTH] = privileged_value(reference, 16); + expected.payload[`PRIV_OUT_TSR_OFFSET+:`PRIV_OUT_TSR_WIDTH] = privileged_value(reference, 17); + expected.payload[`PRIV_OUT_PERMITTED_0_OFFSET +: `PRIV_OUT_PERMITTED_0_WIDTH] = privileged_permitted( + reference, + `PIN(t.inputs, AUTHORIZATION_0_PADDR), + `PIN(t.inputs, AUTHORIZATION_0_SIZE), + `PIN(t.inputs, AUTHORIZATION_0_READ), + `PIN(t.inputs, AUTHORIZATION_0_WRITE), + `PIN(t.inputs, AUTHORIZATION_0_EXECUTE), + `PIN(t.inputs, AUTHORIZATION_0_PRIVILEGE) + ); + expected.payload[`PRIV_OUT_PERMITTED_1_OFFSET +: `PRIV_OUT_PERMITTED_1_WIDTH] = privileged_permitted( + reference, + `PIN(t.inputs, AUTHORIZATION_1_PADDR), + `PIN(t.inputs, AUTHORIZATION_1_SIZE), + `PIN(t.inputs, AUTHORIZATION_1_READ), + `PIN(t.inputs, AUTHORIZATION_1_WRITE), + `PIN(t.inputs, AUTHORIZATION_1_EXECUTE), + `PIN(t.inputs, AUTHORIZATION_1_PRIVILEGE) + ); + for (int index = 0; index < 16; index++) begin + cfg = privileged_pmp(reference, index, 0); + expected.payload[pmp_offsets[index][0]] = cfg[7]; + expected.payload[pmp_offsets[index][1]+:2] = cfg[6:5]; + expected.payload[pmp_offsets[index][2]+:2] = cfg[4:3]; + expected.payload[pmp_offsets[index][3]] = cfg[2]; + expected.payload[pmp_offsets[index][4]] = cfg[1]; + expected.payload[pmp_offsets[index][5]] = cfg[0]; + expected.payload[pmp_offsets[index][6]+:(`PRIV_ADDRESS_BITS-2)] = + privileged_pmp(reference, index, 1); + expected.payload[pmp_offsets[index][7]+:`PRIV_ADDRESS_BITS] = + privileged_pmp(reference, index, 2); + end + if (!actual.payload[`PRIV_OUT_REDIRECT_VALID_OFFSET+:`PRIV_OUT_REDIRECT_VALID_WIDTH]) begin + actual.payload[`PRIV_OUT_REDIRECT_BITS_OFFSET+:`PRIV_OUT_REDIRECT_BITS_WIDTH] = 0; + end + if (!actual.payload[`PRIV_OUT_INTERRUPT_VALID_OFFSET+:`PRIV_OUT_INTERRUPT_VALID_WIDTH]) begin + actual.payload[`PRIV_OUT_INTERRUPT_BITS_OFFSET+:`PRIV_OUT_INTERRUPT_BITS_WIDTH] = 0; + end + if (actual.payload[`PRIV_OUT_CSR_ILLEGAL_OFFSET+:`PRIV_OUT_CSR_ILLEGAL_WIDTH]) begin + actual.payload[`PRIV_OUT_CSR_DATA_OFFSET+:`PRIV_OUT_CSR_DATA_WIDTH] = 0; + end + behavior.sample( + `PIN(t.inputs, CSR_ADDRESS), `PIN(t.inputs, CSR_OPERATION), `PIN(t.inputs, CSR_WRITE), + actual.payload[`PRIV_OUT_PRIVILEGE_OFFSET+:`PRIV_OUT_PRIVILEGE_WIDTH], + `PIN(t.inputs, TRAP_VALID), `PIN(t.inputs, RETURNING_VALID), + actual.payload[`PRIV_OUT_INTERRUPT_VALID_OFFSET+:`PRIV_OUT_INTERRUPT_VALID_WIDTH]); + expected_ap.write(expected); + actual_ap.write(actual); + privileged_advance(reference); + endfunction + function void final_phase(uvm_phase phase); + privileged_destroy(reference); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/filelist.f new file mode 100644 index 000000000..deed612e5 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Privileged/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/interface.sv new file mode 100644 index 000000000..b0c238888 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/interface.sv @@ -0,0 +1,8 @@ +`include "parameters.svh" +interface privileged_if ( + input logic clock +); + logic reset = 1; + logic [`PRIV_INPUT_BITS-1:0] inputs; + logic [`PRIV_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/package.sv b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/package.sv new file mode 100644 index 000000000..a1159b0d0 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/package.sv @@ -0,0 +1,56 @@ +`include "parameters.svh" +`define PIN(P, N) P[`PRIV_IN_``N``_OFFSET +: `PRIV_IN_``N``_WIDTH] +`define POUT(P, N) P[`PRIV_OUT_``N``_OFFSET +: `PRIV_OUT_``N``_WIDTH] +package privileged_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function chandle privileged_create(input int unsigned address_bits); + import "DPI-C" function void privileged_destroy(input chandle model); + import "DPI-C" function void privileged_reset(input chandle model); + import "DPI-C" function void privileged_evaluate( + input chandle model, + input int unsigned address, + input longint unsigned operand, + input int unsigned operation, + input byte unsigned wr, + commit, + trap, + input longint unsigned pc, + cause, + value, + input byte unsigned returning, + input int unsigned return_mode, + input longint unsigned hart, + interrupts, + input byte unsigned retired, + floating, + input int unsigned flags + ); + import "DPI-C" function longint unsigned privileged_value( + input chandle model, + input int unsigned index + ); + import "DPI-C" function longint unsigned privileged_pmp( + input chandle model, + input int unsigned index, + field + ); + import "DPI-C" function byte unsigned privileged_permitted( + input chandle model, + input longint unsigned address, + input int unsigned size, + input byte unsigned rd, + wr, + execute, + input int unsigned privilege + ); + import "DPI-C" function void privileged_advance(input chandle model); + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..fe8c9dfba --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/seq/sequence.svh @@ -0,0 +1,211 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + logic [`PRIV_INPUT_BITS-1:0] pins = '0; + bit reset = 1; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task tick(int cycles = 1); + cycle_item req; + repeat (cycles) begin + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.inputs = pins; + req.reset = reset; + finish_item(req); + end + endtask + task restart(); + pins = '0; + reset = 1; + tick(2); + reset = 0; + `PIN(pins, HARTID) = 1; + tick(); + endtask + task csr(int address, bit write = 0, longint unsigned value = 0, int operation = 1, + bit commit = 1); + `PIN(pins, CSR_ADDRESS) = address; + `PIN(pins, CSR_WRITE) = write; + `PIN(pins, CSR_OPERAND) = value; + `PIN(pins, CSR_OPERATION) = operation; + `PIN(pins, CSR_COMMIT) = commit; + tick(); + `PIN(pins, CSR_WRITE) = 0; + `PIN(pins, CSR_COMMIT) = 0; + endtask + task trap_event(longint unsigned pc, cause, value); + `PIN(pins, TRAP_VALID) = 1; + `PIN(pins, TRAP_BITS_PC) = pc; + `PIN(pins, TRAP_BITS_CAUSE) = cause; + `PIN(pins, TRAP_BITS_VALUE) = value; + tick(); + `PIN(pins, TRAP_VALID) = 0; + endtask + task return_event(int mode); + `PIN(pins, RETURNING_VALID) = 1; + `PIN(pins, RETURNING_BITS) = mode; + tick(); + `PIN(pins, RETURNING_VALID) = 0; + endtask + task authorize(longint unsigned address, int size, int kind, int privilege); + `PIN(pins, AUTHORIZATION_0_PADDR) = address; + `PIN(pins, AUTHORIZATION_1_PADDR) = address; + `PIN(pins, AUTHORIZATION_0_SIZE) = size; + `PIN(pins, AUTHORIZATION_1_SIZE) = size; + `PIN(pins, AUTHORIZATION_0_READ) = kind == 0; + `PIN(pins, AUTHORIZATION_1_READ) = kind == 0; + `PIN(pins, AUTHORIZATION_0_WRITE) = kind == 1; + `PIN(pins, AUTHORIZATION_1_WRITE) = kind == 1; + `PIN(pins, AUTHORIZATION_0_EXECUTE) = kind == 2; + `PIN(pins, AUTHORIZATION_1_EXECUTE) = kind == 2; + `PIN(pins, AUTHORIZATION_0_PRIVILEGE) = privilege; + `PIN(pins, AUTHORIZATION_1_PRIVILEGE) = 3; + `PIN(pins, AUTHORIZATION_0_ISPTE) = size % 2; + `PIN(pins, AUTHORIZATION_1_ISPTE) = size % 2; + tick(); + endtask + task body(); + int registers[22] = '{ + 'h105, + 'h140, + 'h141, + 'h142, + 'h143, + 'h180, + 'h300, + 'h302, + 'h303, + 'h304, + 'h305, + 'h306, + 'h310, + 'h320, + 'h340, + 'h341, + 'h342, + 'h343, + 'h344, + 'hb00, + 'hb02, + 'h106 + }; + int offsets[8] = '{-1, 0, 1, 3, 4, 63, 64, 128}; + restart(); + for (int address = 0; address < 4096; address++) csr(address); + foreach (registers[index]) begin + csr(registers[index], 1, 0); + csr(registers[index], 1, '1); + csr(registers[index], 1, 64'h0123456789abcdef, 1, 0); + for (int bit_index = 0; bit_index < 64; bit_index++) begin + csr(registers[index], 1, 64'(1) << bit_index, 2); + csr(registers[index]); + csr(registers[index], 1, 64'(1) << bit_index, 3); + end + end + restart(); + csr('h300, 1, 'h2000); + for (int rounding = 0; rounding < 8; rounding++) begin + csr(2, 1, rounding); + for (int flags = 0; flags < 32; flags++) begin + csr(1, 1, 0); + `PIN(pins, FLOATING_VALID) = 1; + `PIN(pins, FLOATING_BITS) = flags; + tick(); + `PIN(pins, FLOATING_VALID) = 0; + csr(3); + end + end + // Every implemented PMP entry, addressing mode, access permission and lock state. + for (int index = 0; index < 16; index++) + for (int mode = 0; mode < 4; mode++) + for (int permissions = 0; permissions < 8; permissions++) + for (int locked = 0; locked < 2; locked++) begin + longint unsigned base = 'h1000 + index * 'h100; + longint unsigned encoded = (base >> 2) | (mode == 3 ? 7 : 0); + int pmp_config = (locked << 7) | (mode << 3) | permissions; + restart(); + if (mode == 1) begin + if (index > 0) csr('h3b0 + index - 1, 1, base >> 2); + encoded = (base + 64) >> 2; + end + csr('h3b0 + index, 1, encoded); + csr('h3a0 + (index / 8) * 2, 1, 64'(pmp_config) << ((index % 8) * 8)); + for (int sample = 0; sample < 8; sample ++) + for (int kind = 0; kind < 3; kind++) + authorize(base + offsets[sample], sample, kind, sample % 2); + csr('h3b0 + index, 1, '1); + if (index > 0) csr('h3b0 + index - 1, 1, '1); + csr('h3a0 + (index / 8) * 2, 1, 0); + end + // An empty or reversed TOR range must not claim an overlapping access. + restart(); + csr('h3b0, 1, 'h410); + csr('h3b1, 1, 'h400); + csr('h3a0, 1, 64'h8f00); + authorize('hfe0, 7, 0, 0); + // CSR privilege checks cover the entire address space in S and U modes. + for (int target = 0; target < 2; target++) begin + restart(); + csr('h300, 1, target << 11); + csr('h341, 1, 'h1000); + return_event(3); + for (int address = 0; address < 4096; address++) csr(address); + trap_event('h12345, 64'h800000000000000b, 'hdeadbeef); + csr('h341); + csr('h342); + csr('h343); + end + for (int target = 0; target < 2; target++) + for (int vector_mode = 0; vector_mode < 2; vector_mode++) begin + restart(); + csr('h305, 1, 'h1000 | vector_mode); + csr('h105, 1, 'h2000 | vector_mode); + csr('h302, 1, 'hb3ff); + csr('h303, 1, 'h222); + csr('h304, 1, 'haaa); + csr('h300, 1, (target << 11) | 'h20); + csr('h341, 1, 'h1000); + return_event(3); + trap_event('h12345, 8, 'hbad0); + csr('h141); + csr('h142); + csr('h143); + return_event(1); + trap_event('h34567, 64'h800000000000000b, 'hbad1); + csr('h341); + return_event(3); + end + // Interrupt masking, delegation and priority across all external pending combinations. + for (int target = 0; target < 4; target++) begin + if (target == 2) continue; + for (int enable = 0; enable < 4; enable++) begin + restart(); + csr('h303, 1, 'h222); + csr('h304, 1, 'haaa); + csr('h344, 1, 'h222); + csr('h300, 1, (target << 11) | ((enable & 1) << 3) | ((enable >> 1) << 1)); + if (target != 3) return_event(3); + for (int pending = 0; pending < 16; pending++) begin + `PIN(pins, INTERRUPTS_TIMER) = pending[0]; + `PIN(pins, INTERRUPTS_SOFTWARE) = pending[1]; + `PIN(pins, INTERRUPTS_EXTERNAL) = pending[2]; + `PIN(pins, INTERRUPTS_SUPERVISOREXTERNAL) = pending[3]; + tick(); + end + end + end + restart(); + for (int inhibit = 0; inhibit < 8; inhibit++) begin + csr('h320, 1, inhibit); + for (int n = 0; n < 16; n++) begin + `PIN(pins, RETIRED) = n % 2; + csr('hb00); + csr('hb02); + end + end + `PIN(pins, RETIRED) = 0; + csr('h320, 1, 0); + tick(2); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/suite.toml new file mode 100644 index 000000000..c2c21da36 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libprivileged_reference" + +[[tests]] +name = "privileged" +top = "privileged_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 2000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["privileged_pkg::reference_model::behavior"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/tb.sv new file mode 100644 index 000000000..e122383fa --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/tb.sv @@ -0,0 +1,16 @@ +module privileged_tb; + import uvm_pkg::*; + import privileged_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + privileged_if bus (clock); + Privileged dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual privileged_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Emit.scala new file mode 100644 index 000000000..89b705a1f --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Emit.scala @@ -0,0 +1,57 @@ +package framework.core.execution.privileged + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.core.execution.privileged.configs.Params + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val p = upickle.default.read[Params](Files.readString(Paths.get(config))) + Files.createDirectories(Paths.get(output)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Privileged(p), + args = Array("--target-dir", s"$output/Privileged", "--split-verilog") + ) + val inputs = Set("hartId", "interrupts", "trap", "returning", "retired", "floating", "authorization") + + def fields(data: Data, path: String): Seq[(String, Int, Boolean)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(( + path, + leaf.getWidth, + inputs.contains(path.takeWhile(_ != '_')) || + (path.startsWith("csr_") && !Set("csr_data", "csr_illegal", "csr_memoryRead").contains(path)) + )) + } + + var inputOffset = 0 + var outputOffset = 0 + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val ports = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + for ((name, width, input) <- fields(new PrivilegedIO(p), "")) { + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define PRIV_${direction}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define PRIV_${direction}_${name.toUpperCase}_WIDTH $width" + ports += s".io_$name(bus.$signal[$offset +: $width])" + if (input) inputOffset += width else outputOffset += width + offsets(name) = offset + } + + val pmpOffsets = (0 until p.permission.entries).map { index => + val names = Seq("cfg_l", "cfg_res", "cfg_a", "cfg_x", "cfg_w", "cfg_r", "addr", "mask") + names.map(name => offsets(s"pmp_${index}_$name")).mkString("'{", ",", "}") + }.mkString("'{", ",", "}") + + definitions += s"`define PRIV_INPUT_BITS $inputOffset" + definitions += s"`define PRIV_OUTPUT_BITS $outputOffset" + definitions += s"`define PRIV_ADDRESS_BITS ${p.permission.addressBits}" + definitions += s"`define PRIV_PMP_OFFSETS $pmpOffsets" + Files.writeString(Paths.get(output, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Privileged.scala b/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Privileged.scala new file mode 100644 index 000000000..6e27c0126 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Privileged.scala @@ -0,0 +1,310 @@ +package framework.core.execution.privileged + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.clink.CoreInterrupts +import framework.core.memory.access.PhysicalAuthorization +import framework.core.memory.access.preflight.PmpEntry +import framework.core.memory.access.preflight.configs.PmpParams +import framework.core.execution.privileged.configs.{Params => ModuleParam} +import framework.core.execution.privileged.configs.Params + +class Trap extends Bundle { + val pc = UInt(64.W) + val cause = UInt(64.W) + val value = UInt(64.W) +} + +/** M/S/U architectural state, precise traps and physical memory permissions. */ +class PrivilegedIO(b: ModuleParam) extends Bundle { + val p = b + implicit val permission: PmpParams = p.permission + + val hartId = Input(UInt(64.W)) + val interrupts = Input(new CoreInterrupts) + + val csr = new Bundle { + val address = Input(UInt(12.W)) + val operand = Input(UInt(64.W)) + val operation = Input(UInt(2.W)) + val write = Input(Bool()) + val commit = Input(Bool()) + val data = Output(UInt(64.W)) + val illegal = Output(Bool()) + val memoryRead = Output(Bool()) + } + + val trap = Flipped(Valid(new Trap)) + val returning = Flipped(Valid(UInt(2.W))) + val redirect = Valid(UInt(64.W)) + val interrupt = Valid(UInt(64.W)) + val wake = Output(Bool()) + val retired = Input(Bool()) + val floating = Flipped(Valid(UInt(5.W))) + val privilege = Output(UInt(2.W)) + val dataPrivilege = Output(UInt(2.W)) + val satp = Output(UInt(64.W)) + val sum = Output(Bool()) + val mxr = Output(Bool()) + val floatingEnabled = Output(Bool()) + val rounding = Output(UInt(3.W)) + val tvm = Output(Bool()) + val tw = Output(Bool()) + val tsr = Output(Bool()) + val authorization = Input(Vec(2, new PhysicalAuthorization)) + val permitted = Output(Vec(2, Bool())) + + val pmp = Output(Vec( + permission.entries, + new PmpEntry(b.permission) + )) + +} + +@instantiable +class Privileged(b: ModuleParam) extends Module { + val p = b + implicit val permission: PmpParams = p.permission + require(permission.entries == 16) + + @public + val io = IO(new PrivilegedIO(b)) + + val privilege = RegInit(3.U(2.W)) + val status = RegInit(0.U(64.W)) + val supervisorMask = BigInt("80000003000de122", 16) + val statusMask = BigInt("00000000007e79aa", 16) + val statusRead = status | (BigInt(2).U(64.W) << 32) | (BigInt(2).U(64.W) << 34) | + Mux(status(14, 13) === 3.U, BigInt("8000000000000000", 16).U, 0.U) + val mie = RegInit(0.U(64.W)) + val softwarePending = RegInit(0.U(64.W)) + val medeleg = RegInit(0.U(64.W)) + val mideleg = RegInit(0.U(64.W)) + val mtvec = RegInit(0.U(64.W)) + val stvec = RegInit(0.U(64.W)) + val mscratch = RegInit(0.U(64.W)) + val sscratch = RegInit(0.U(64.W)) + val mepc = RegInit(0.U(64.W)) + val sepc = RegInit(0.U(64.W)) + val mcause = RegInit(0.U(64.W)) + val scause = RegInit(0.U(64.W)) + val mtval = RegInit(0.U(64.W)) + val stval = RegInit(0.U(64.W)) + val satp = RegInit(0.U(64.W)) + val mcounteren = RegInit(0.U(3.W)) + val scounteren = RegInit(0.U(3.W)) + val inhibit = RegInit(0.U(3.W)) + val cycles = RegInit(0.U(64.W)) + val instructions = RegInit(0.U(64.W)) + val fflags = RegInit(0.U(5.W)) + val frm = RegInit(0.U(3.W)) + val pmpConfig = RegInit(VecInit(Seq.fill(16)(0.U(8.W)))) + val pmpAddress = RegInit(VecInit(Seq.fill(16)(0.U((permission.addressBits - 2).W)))) + + when(!inhibit(0))(cycles := cycles + 1.U) + when(io.retired && !inhibit(2))(instructions := instructions + 1.U) + when(io.floating.valid) { + fflags := fflags | io.floating.bits + status := status | (3.U(64.W) << 13) + } + + val pending = softwarePending | + (io.interrupts.software.asUInt << 3) | (io.interrupts.timer.asUInt << 7) | + (io.interrupts.external.asUInt << 11) | (io.interrupts.supervisorExternal.asUInt << 9) + + val enabled = pending & mie + val machineEnabled = privilege < 3.U || status(3) + val supervisorEnabled = privilege === 0.U || (privilege === 1.U && status(1)) + val eligible = enabled & (Mux(machineEnabled, ~mideleg, 0.U(64.W)) | + Mux(supervisorEnabled, mideleg, 0.U(64.W))) + val interruptPriority = Seq(11, 3, 7, 9, 1, 5) + io.interrupt.valid := eligible.orR + io.interrupt.bits := BigInt("8000000000000000", 16).U | + PriorityMux(interruptPriority.map(i => eligible(i) -> i.U(64.W)) :+ (true.B -> 0.U(64.W))) + io.wake := enabled.orR + io.privilege := privilege + io.dataPrivilege := Mux(status(17), status(12, 11), privilege) + io.satp := satp + io.sum := status(18) + io.mxr := status(19) + io.floatingEnabled := status(14, 13).orR + io.rounding := frm + io.tvm := status(20) + io.tw := status(21) + io.tsr := status(22) + + val csr = io.csr + val floatingCsr = csr.address >= 1.U && csr.address <= 3.U + val counterCsr = csr.address >= "hc00".U && csr.address <= "hc02".U + val counterBit = csr.address(1, 0) + val counterAllowed = privilege === 3.U || (mcounteren(counterBit) && + (privilege === 1.U || scounteren(counterBit))) + val known = WireDefault(false.B) + csr.data := 0.U + + val read = Seq( + 0x001 -> fflags.pad(64), + 0x002 -> frm.pad(64), + 0x003 -> Cat(frm, fflags).pad(64), + 0x100 -> (statusRead & supervisorMask.U(64.W)), + 0x104 -> (mie & mideleg), + 0x105 -> stvec, + 0x106 -> scounteren.pad(64), + 0x140 -> sscratch, + 0x141 -> sepc, + 0x142 -> scause, + 0x143 -> stval, + 0x144 -> (pending & mideleg), + 0x180 -> satp, + 0x300 -> statusRead, + 0x301 -> BigInt("800000000014112d", 16).U(64.W), + 0x302 -> medeleg, + 0x303 -> mideleg, + 0x304 -> mie, + 0x305 -> mtvec, + 0x306 -> mcounteren.pad(64), + 0x310 -> 0.U(64.W), + 0x320 -> inhibit.pad(64), + 0x340 -> mscratch, + 0x341 -> mepc, + 0x342 -> mcause, + 0x343 -> mtval, + 0x344 -> pending, + 0x3a0 -> pmpConfig.take(8).reverse.reduce(Cat(_, _)), + 0x3a2 -> pmpConfig.drop(8).reverse.reduce(Cat(_, _)), + 0xb00 -> cycles, + 0xb02 -> instructions, + 0xc00 -> cycles, + 0xc02 -> instructions, + 0xf11 -> 0.U(64.W), + 0xf12 -> 0.U(64.W), + 0xf13 -> 0.U(64.W), + 0xf14 -> io.hartId, + 0xf15 -> 0.U(64.W) + ) ++ (0 until 16).map(i => (0x3b0 + i) -> pmpAddress(i).pad(64)) + + for ((address, data) <- read) { + when(csr.address === address.U) { known := true.B; csr.data := data } + } + csr.memoryRead := csr.address === "hc01".U + when(csr.memoryRead)(known := true.B) + csr.illegal := !known || privilege < csr.address(9, 8) || + (csr.write && csr.address(11, 10) === 3.U) || + (floatingCsr && !io.floatingEnabled) || (counterCsr && !counterAllowed) || + (csr.address === "h180".U && privilege === 1.U && status(20)) + + val writeData = MuxLookup(csr.operation, csr.operand)(Seq( + 2.U -> (csr.data | csr.operand), + 3.U -> (csr.data & ~csr.operand) + )) + + when(csr.commit && csr.write && !csr.illegal) { + switch(csr.address) { + is("h001".U) { fflags := writeData(4, 0); status := status | (3.U(64.W) << 13) } + is("h002".U) { frm := writeData(2, 0); status := status | (3.U(64.W) << 13) } + is("h003".U) { fflags := writeData(4, 0); frm := writeData(7, 5); status := status | (3.U(64.W) << 13) } + is("h100".U) { + status := (status & ~supervisorMask.U(64.W)) | (writeData & supervisorMask.U(64.W) & statusMask.U(64.W)) + } + is("h104".U)(mie := (mie & ~mideleg) | (writeData & mideleg)) + is("h105".U)(stvec := Cat(writeData(63, 2), 0.U(1.W), writeData(1, 0) === 1.U)) + is("h106".U)(scounteren := writeData(2, 0)) + is("h140".U)(sscratch := writeData) + is("h141".U)(sepc := writeData & ~1.U(64.W)) + is("h142".U)(scause := writeData) + is("h143".U)(stval := writeData) + is("h144".U)(softwarePending := (softwarePending & ~2.U(64.W)) | (writeData & mideleg & 2.U(64.W))) + is("h180".U)(when(writeData(63, 60) === 0.U || writeData(63, 60) === 8.U)(satp := writeData)) + is("h300".U) { + status := (writeData & statusMask.U(64.W) & ~(3.U(64.W) << 11)) | + (Mux(writeData(12, 11) === 2.U, 0.U(2.W), writeData(12, 11)).pad(64) << 11) + } + is("h302".U)(medeleg := writeData & "hb3ff".U) + is("h303".U)(mideleg := writeData & "h222".U) + is("h304".U)(mie := writeData & "haaa".U) + is("h305".U)(mtvec := Cat(writeData(63, 2), 0.U(1.W), writeData(1, 0) === 1.U)) + is("h306".U)(mcounteren := writeData(2, 0)) + is("h320".U)(inhibit := writeData(2, 0) & 5.U) + is("h340".U)(mscratch := writeData) + is("h341".U)(mepc := writeData & ~1.U(64.W)) + is("h342".U)(mcause := writeData) + is("h343".U)(mtval := writeData) + is("h344".U)(softwarePending := writeData & "h222".U) + is("hb00".U)(cycles := writeData) + is("hb02".U)(instructions := writeData) + } + for (i <- 0 until 16) { + when(csr.address === (0x3a0 + (i / 8) * 2).U && !pmpConfig(i)(7)) { + val cfg = writeData(8 * (i % 8) + 7, 8 * (i % 8)) + pmpConfig(i) := cfg & "h9d".U | Mux(cfg(0) && cfg(1), 2.U, 0.U) + } + val lockedByTor = if (i == 15) false.B else pmpConfig(i + 1)(7) && pmpConfig(i + 1)(4, 3) === 1.U + when(csr.address === (0x3b0 + i).U && !pmpConfig(i)(7) && !lockedByTor) { + pmpAddress(i) := writeData(permission.addressBits - 3, 0) + } + } + } + + val supervisorTrap = + privilege =/= 3.U && Mux(io.trap.bits.cause(63), mideleg(io.trap.bits.cause(5, 0)), medeleg(io.trap.bits.cause(5, 0))) + val trapVector = Mux(supervisorTrap, stvec, mtvec) + io.redirect.valid := io.trap.valid || io.returning.valid + io.redirect.bits := Mux( + io.trap.valid, + (trapVector & ~3.U(64.W)) + Mux(io.trap.bits.cause(63) && trapVector(0), io.trap.bits.cause(5, 0) << 2, 0.U), + Mux(io.returning.bits === 3.U, mepc, sepc) + ) + when(io.trap.valid) { + when(supervisorTrap) { + sepc := io.trap.bits.pc & ~1.U(64.W) + scause := io.trap.bits.cause + stval := io.trap.bits.value + status := (status & ~"h122".U(64.W)) | (status(1).asUInt << 5) | ((privilege === 1.U).asUInt << 8) + privilege := 1.U + }.otherwise { + mepc := io.trap.bits.pc & ~1.U(64.W) + mcause := io.trap.bits.cause + mtval := io.trap.bits.value + status := (status & ~"h1888".U(64.W)) | (status(3).asUInt << 7) | (privilege.pad(64) << 11) + privilege := 3.U + } + }.elsewhen(io.returning.valid) { + when(io.returning.bits === 3.U) { + privilege := status(12, 11) + status := (status & ~"h1888".U(64.W) & Mux(status(12, 11) === 3.U, ~0.U(64.W), ~"h20000".U(64.W))) | + (status(7).asUInt << 3) | "h80".U + }.otherwise { + privilege := Mux(status(8), 1.U, 0.U) + status := (status & ~"h20122".U(64.W)) | (status(5).asUInt << 1) | "h20".U + } + } + + for (i <- 0 until 16) { + io.pmp(i).cfg := pmpConfig(i).asTypeOf(io.pmp(i).cfg) + io.pmp(i).addr := pmpAddress(i) + io.pmp(i).mask := ((pmpAddress(i) ^ (pmpAddress(i) + 1.U)) << 2) | 3.U + } + for (port <- 0 until 2) { + val request = io.authorization(port) + val end = request.paddr +& (1.U(65.W) << request.size) + val matches = (0 until 16).map { i => + val encoded = (pmpAddress(i) << 2).pad(64) + val mask = io.pmp(i).mask.pad(64) + val mode = pmpConfig(i)(4, 3) + val low = Mux( + mode === 1.U, + if (i == 0) 0.U(64.W) else (pmpAddress(i - 1) << 2).pad(64), + Mux(mode === 3.U, encoded & ~mask, encoded) + ) + val high = Mux(mode === 1.U, encoded.pad(65), Mux(mode === 3.U, low +& mask + 1.U, encoded +& 4.U)) + val hit = mode =/= 0.U && low < high && request.paddr < high && end > low + val allowed = (request.privilege === 3.U && !pmpConfig(i)(7)) || + ((!request.read || pmpConfig(i)(0)) && (!request.write || pmpConfig(i)(1)) && (!request.execute || pmpConfig(i)( + 2 + ))) + hit -> (allowed && request.paddr >= low && end <= high) + } + io.permitted(port) := PriorityMux(matches :+ (true.B -> (request.privilege === 3.U))) + } +} diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..e90a22e1d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/configs/Params.scala @@ -0,0 +1,7 @@ +package framework.core.execution.privileged.configs + +import framework.core.memory.access.preflight.configs.PmpParams +import upickle.default.{macroRW, ReadWriter} + +case class Params(permission: PmpParams) +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/execution/register_file/build.sc b/arch/src/main/scala/framework/core/execution/register_file/build.sc new file mode 100644 index 000000000..e47500e02 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.execution.register_file.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/RegisterFile.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/execution/register_file/configs/uvm.toml b/arch/src/main/scala/framework/core/execution/register_file/configs/uvm.toml new file mode 100644 index 000000000..3da02cd69 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/configs/uvm.toml @@ -0,0 +1,4 @@ +[parameters] +xLen = 64 + +[defines] diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/csrc/dual_read.cpp b/arch/src/main/scala/framework/core/execution/register_file/src/csrc/dual_read.cpp new file mode 100644 index 000000000..0a33b028d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/csrc/dual_read.cpp @@ -0,0 +1,45 @@ +#include "VRegisterFile.h" +#include "verilated.h" +#include +#include +#include +#include +int main() { + VerilatedContext context; + VRegisterFile dut(&context); + std::array reference{}; + for (unsigned i = 1; i < 32; ++i) { + dut.io_write_valid = 1; + dut.io_write_bits_address = i; + dut.io_write_bits_data = reference[i] = 0x1234000000000000ULL + i; + dut.clock = 0; + dut.eval(); + dut.clock = 1; + dut.eval(); + } + std::mt19937_64 random(0x6447); + for (unsigned i = 0; i < 60000; ++i) { + unsigned a = random() % 32, b = random() % 32, w = random() % 32; + bool valid = random() % 2; + uint64_t data = random(); + dut.io_readAddress_0 = a; + dut.io_readAddress_1 = b; + dut.io_write_valid = valid; + dut.io_write_bits_address = w; + dut.io_write_bits_data = data; + dut.clock = 0; + dut.eval(); + assert(dut.io_readData_0 == (a == 0 ? 0 + : valid && a == w ? data + : reference[a])); + assert(dut.io_readData_1 == (b == 0 ? 0 + : valid && b == w ? data + : reference[b])); + dut.clock = 1; + dut.eval(); + if (valid && w) + reference[w] = data; + } + puts("PASS: 60000 dual-read register-file checks, write bypass and x0 " + "suppression"); +} diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/agent.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/agent.svh new file mode 100644 index 000000000..6d2a96092 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/agent.svh @@ -0,0 +1,19 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(request_item) seqr; + driver drv; + monitor mon; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(request_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..38a8b5c0d --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/driver.svh @@ -0,0 +1,35 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Driver interface is required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + vif.sample = 0; + vif.read0 = 0; + vif.read1 = 0; + vif.write = 0; + vif.address = 0; + vif.data = 0; + forever begin + seq_item_port.get_next_item(req); + do @(negedge vif.clock); while (vif.reset); + vif.read0 = req.read0; + vif.read1 = req.read1; + vif.write = req.write; + vif.address = req.address; + vif.data = req.data; + vif.sample = 1; + @(posedge vif.clock); + seq_item_port.item_done(); + @(negedge vif.clock); + vif.sample = 0; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/item.svh new file mode 100644 index 000000000..5cfddf213 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/item.svh @@ -0,0 +1,29 @@ +class request_item extends uvm_sequence_item; + logic [4:0] read0; + logic [4:0] read1; + logic write; + logic [4:0] address; + logic [63:0] data; + `uvm_object_utils_begin(request_item) + `uvm_field_int(read0, UVM_ALL_ON) + `uvm_field_int(read1, UVM_ALL_ON) + `uvm_field_int(write, UVM_ALL_ON) + `uvm_field_int(address, UVM_ALL_ON) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [63:0] data0; + logic [63:0] data1; + `uvm_object_utils_begin(response_item) + `uvm_field_int(data0, UVM_ALL_ON) + `uvm_field_int(data1, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..a741412f7 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/agent/monitor.svh @@ -0,0 +1,36 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual port_if vif; + uvm_analysis_port #(request_item) requests; + uvm_analysis_port #(response_item) responses; + function new(string name, uvm_component parent); + super.new(name, parent); + requests = new("requests", this); + responses = new("responses", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + response_item rsp; + forever begin + @(posedge vif.clock); + if (!vif.reset && vif.sample) begin + req = request_item::type_id::create("request"); + rsp = response_item::type_id::create("response"); + req.read0 = vif.read0; + req.read1 = vif.read1; + req.write = vif.write; + req.address = vif.address; + req.data = vif.data; + rsp.data0 = vif.data0; + rsp.data1 = vif.data1; + requests.write(req); + responses.write(rsp); + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/coverage.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/coverage.svh new file mode 100644 index 000000000..d88990b0c --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/coverage.svh @@ -0,0 +1,21 @@ +class coverage extends uvm_subscriber #(request_item); + `uvm_component_utils(coverage) + covergroup behavior with function sample (request_item t); + option.per_instance = 1; + read0: coverpoint t.read0 {bins addresses[] = {[0 : 31]};} + read1: coverpoint t.read1 {bins addresses[] = {[0 : 31]};} + read_pairs: cross read0, read1; + write: coverpoint t.write; + destination: coverpoint t.address iff (t.write) {bins addresses[] = {[0 : 31]};} + bypass0: coverpoint (t.write && t.address != 0 && t.address == t.read0); + bypass1: coverpoint (t.write && t.address != 0 && t.address == t.read1); + write_zero: coverpoint (t.write && t.address == 0) {bins seen = {1};} + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void write(request_item t); + behavior.sample(t); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/env.svh new file mode 100644 index 000000000..5a8082f7a --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/env.svh @@ -0,0 +1,19 @@ +class env extends environment #(request_item, response_item, model); + `uvm_component_utils(env) + agent bus; + coverage cov; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + cov = coverage::type_id::create("coverage", this); + bus = agent::type_id::create("agent", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.requests.connect(input_export); + bus.mon.requests.connect(cov.analysis_export); + bus.mon.responses.connect(output_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/model.svh new file mode 100644 index 000000000..385609f57 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/env/model.svh @@ -0,0 +1,20 @@ +class model extends reference_model #(request_item, response_item); + `uvm_component_utils(model) + logic [63:0] registers[32]; + bit initialized[32]; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void write(request_item req); + response_item rsp = response_item::type_id::create("expected"); + if (req.write && req.address != 0) begin + registers[req.address] = req.data; + initialized[req.address] = 1; + end + if ((req.read0 != 0 && !initialized[req.read0]) || (req.read1 != 0 && !initialized[req.read1])) + `uvm_fatal("UNINITIALIZED", "Reading a register before its first write") + rsp.data0 = req.read0 == 0 ? 0 : registers[req.read0]; + rsp.data1 = req.read1 == 0 ? 0 : registers[req.read1]; + expected_ap.write(rsp); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/filelist.f new file mode 100644 index 000000000..ca0929461 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/filelist.f @@ -0,0 +1,7 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/RegisterFile/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/interface.sv new file mode 100644 index 000000000..8ef3bb674 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/interface.sv @@ -0,0 +1,13 @@ +interface port_if ( + input logic clock, + input logic reset +); + logic sample; + logic [4:0] read0; + logic [4:0] read1; + logic write; + logic [4:0] address; + logic [63:0] data; + logic [63:0] data0; + logic [63:0] data1; +endinterface diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/package.sv b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/package.sv new file mode 100644 index 000000000..e8dc35920 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/package.sv @@ -0,0 +1,15 @@ +package register_file_pkg; + timeunit 1ns; timeprecision 1ps; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..6fca06d4a --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/seq/sequence.svh @@ -0,0 +1,47 @@ +class stimulus extends uvm_sequence #(request_item); + `uvm_object_utils(stimulus) + int unsigned sent = 0; + reset_controller resetter; + function new(string name = "stimulus"); + super.new(name); + endfunction + task body(); + request_item req = request_item::type_id::create("request"); + for (int i = 1; i < 32; i++) begin + start_item(req); + req.read0 = 0; + req.read1 = 0; + req.write = 1; + req.address = i; + req.data = 64'h0102040810204081 * i; + finish_item(req); + sent++; + end + resetter.pulse(2); + for (int i = 0; i < 32; i++) + for (int j = 0; j < 32; j++) begin + start_item(req); + req.read0 = i; + req.read1 = j; + req.write = 0; + req.address = 0; + req.data = 0; + finish_item(req); + sent++; + end + for (int i = 0; i < 32; i++) begin + start_item(req); + req.read0 = i; + req.read1 = i; + req.write = 1; + req.address = i; + req.data = '1; + finish_item(req); + sent++; + start_item(req); + req.write = 0; + finish_item(req); + sent++; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/suite.toml new file mode 100644 index 000000000..514f36aec --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +functional_groups = ["register_file_pkg::coverage::behavior"] +name = "register_file" +top = "register_file_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 0 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/tb.sv new file mode 100644 index 000000000..c9e5e3497 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/tb.sv @@ -0,0 +1,27 @@ +module register_file_tb; + import uvm_pkg::*; + import register_file_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + port_if bus ( + clock, + timing.reset + ); + RegisterFile dut ( + .clock(clock), + .reset(bus.reset), + .io_readAddress_0(bus.read0), + .io_readAddress_1(bus.read1), + .io_readData_0(bus.data0), + .io_readData_1(bus.data1), + .io_write_valid(bus.sample && bus.write), + .io_write_bits_address(bus.address), + .io_write_bits_data(bus.data) + ); + initial begin + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/tests/test.svh new file mode 100644 index 000000000..d5f2d6ed9 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/resources/tests/test.svh @@ -0,0 +1,25 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env bench; + reset_controller resetter; + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + resetter = reset_controller::type_id::create("reset", this); + bench = env::type_id::create("env", this); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Test interface is required") + endfunction + task execute(); + stimulus sequence_ = stimulus::type_id::create("sequence"); + resetter.pulse(4); + sequence_.resetter = resetter; + sequence_.start(bench.bus.seqr); + bench.scoreboard.wait_checked(sequence_.sent); + @(negedge vif.clock); + evidence::check(bench.scoreboard.checked == sequence_.sent, "Comparison count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/Emit.scala new file mode 100644 index 000000000..915b42ec6 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/Emit.scala @@ -0,0 +1,15 @@ +package framework.core.execution.register_file + +import framework.core.execution.register_file.configs.Params +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit extends App { + require(args.length >= 2 && args(0) == "--config") + val param = read[Params](Files.readString(Paths.get(args(1)))) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new RegisterFile(param), + firtoolOpts = args.drop(2), + args = Array("--target-dir", "build/RegisterFile", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/RegisterFile.scala b/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/RegisterFile.scala new file mode 100644 index 000000000..045511a99 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/RegisterFile.scala @@ -0,0 +1,40 @@ +package framework.core.execution.register_file + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.execution.register_file.configs.{Params => ModuleParam} + +class RegisterFileIO(b: ModuleParam) extends Bundle { + val p = b + + val readAddress = Input(Vec(2, UInt(5.W))) + val readData = Output(Vec(2, UInt(p.xLen.W))) + + val write = Flipped(Valid(new Bundle { + val address = UInt(5.W) + val data = UInt(p.xLen.W) + })) + +} + +@instantiable +class RegisterFile(b: ModuleParam) extends Module { + val p = b + + @public + val io = IO(new RegisterFileIO(b)) + + val registers = Reg(Vec(32, UInt(p.xLen.W))) + when(io.write.valid && io.write.bits.address =/= 0.U) { + registers(io.write.bits.address) := io.write.bits.data + } + for (port <- 0 until 2) { + val address = io.readAddress(port) + io.readData(port) := Mux( + address === 0.U, + 0.U, + Mux(io.write.valid && io.write.bits.address === address, io.write.bits.data, registers(address)) + ) + } +} diff --git a/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..7c3717f46 --- /dev/null +++ b/arch/src/main/scala/framework/core/execution/register_file/src/main/scala/configs/Params.scala @@ -0,0 +1,6 @@ +package framework.core.execution.register_file.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Params(xLen: Int) { require(xLen == 64) } +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/memory/Memory.scala b/arch/src/main/scala/framework/core/memory/Memory.scala new file mode 100644 index 000000000..a387b680f --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/Memory.scala @@ -0,0 +1,65 @@ +package framework.core.memory + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.memory.configs.MemoryParam +import framework.core.memory.access.{UncachedRequest, UncachedResponse, VirtualPort} +import framework.ip.bus.axi4 +import framework.ip.bus.chi.RequesterPort +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} + +/** Core memory hierarchy and shared cache services for attached threads. */ +@instantiable +class Memory(b: MemoryParam, externalHome: Boolean) extends Module { + val core = b + + @public + val io = IO(new Bundle { + val fetch = new VirtualPort(b.memoryAccess) + val data = new VirtualPort(b.memoryAccess) + val timeRequest = Flipped(Decoupled(new UncachedRequest(b.memoryAccess.memory))) + val timeResponse = Decoupled(new UncachedResponse(b.memoryAccess.memory)) + + val threads = Option.when(core.threadPorts > 0)(Vec( + core.threadPorts, + Flipped(new MemoryPort(b.dcache.tracking, b.memoryAccess.walker)) + )) + + val commandAllowed = Input(Bool()) + val uncached = new axi4.Port(b.uncached.axi) + + val coherent = Vec(if (externalHome) 2 else 0, new RequesterPort(core.memoryAccess.memory.chi)) + + val backing = Option.when(!externalHome)(new Bundle { + val request = Decoupled(new LineRequest(core.home.memory.chi)) + val response = Flipped(Decoupled(new LineResponse(core.home.memory.chi))) + }) + + }) + + val system: Instance[MemorySystem] = Instantiate(new MemorySystem(b, externalHome)) + system.io.fetch <> io.fetch + system.io.memory <> io.data + system.io.timeRequest <> io.timeRequest + io.timeResponse <> system.io.timeResponse + system.io.commandAllowed := io.commandAllowed + io.uncached <> system.io.uncached + io.backing.zip(system.io.backing).foreach { case (port, backing) => port <> backing } + io.coherent <> system.io.coherent + + if (core.threadPorts > 0) { + val threads: Instance[ThreadMemory] = + Instantiate(new ThreadMemory(b.threadPorts, b.dcache.tracking, b.memoryAccess.walker)) + threads.io.clients <> io.threads.get + system.io.services <> threads.io.system + } else { + system.io.services.cpuAllow := true.B + system.io.services.maintenance.valid := false.B + system.io.services.maintenance.bits := 0.U.asTypeOf(system.io.services.maintenance.bits) + system.io.services.pteRequest.valid := false.B + system.io.services.pteRequest.bits := 0.U.asTypeOf(system.io.services.pteRequest.bits) + system.io.services.maintained.ready := true.B + system.io.services.pteResponse.ready := true.B + } +} diff --git a/arch/src/main/scala/framework/core/memory/MemorySystem.scala b/arch/src/main/scala/framework/core/memory/MemorySystem.scala new file mode 100644 index 000000000..eaba50a62 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/MemorySystem.scala @@ -0,0 +1,177 @@ +package framework.core.memory + +import framework.lib.arbiter.{Arbiter} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.memory.configs.MemoryParam +import framework.core.memory.uncached.Uncached +import framework.core.memory.access.{UncachedRequest, UncachedResponse, VirtualAccess, VirtualPort} +import framework.core.memory.access.cache.Cache +import framework.core.memory.home.Home +import framework.ip.bus.axi4 +import framework.ip.bus.chi.RequesterPort +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} +import framework.ip.bus.chi.rnf.{BankedChiCache, CacheAccess, CacheAtomic} +import framework.lib.queue.Queue + +/** Translation, cache services and ordered cached/uncached reply routing. */ +@instantiable +class MemorySystem(b: MemoryParam, externalHome: Boolean) extends Module { + require(b.uncached.ports == 1, "MemorySystem arbitrates requests into one uncached adapter input") + val core = b + val access = core.memoryAccess.memory + require(b.icache.chi == access.chi, "Instruction cache and memory access CHI must match") + require(b.dcache.cache.chi == access.chi, "Data cache and memory access CHI must match") + if (!externalHome) require(b.home.memory.chi == access.chi, "Local Home and cache fabric CHI must match") + + @public + val io = IO(new Bundle { + val fetch = new VirtualPort(b.memoryAccess) + val memory = new VirtualPort(b.memoryAccess) + val timeRequest = Flipped(Decoupled(new UncachedRequest(b.memoryAccess.memory))) + val timeResponse = Decoupled(new UncachedResponse(b.memoryAccess.memory)) + val services = Flipped(new MemoryPort(b.dcache.tracking, b.memoryAccess.walker)) + val commandAllowed = Input(Bool()) + val uncached = new axi4.Port(b.uncached.axi) + + val coherent = Vec(if (externalHome) 2 else 0, new RequesterPort(access.chi)) + + val backing = Option.when(!externalHome)(new Bundle { + val request = Decoupled(new LineRequest(core.home.memory.chi)) + val response = Flipped(Decoupled(new LineResponse(core.home.memory.chi))) + }) + + }) + + val fetch: Instance[VirtualAccess] = + Instantiate(new VirtualAccess(b.memoryAccess)) + + val memory: Instance[VirtualAccess] = + Instantiate(new VirtualAccess(b.memoryAccess)) + + val icache: Instance[BankedChiCache] = Instantiate(new BankedChiCache(core.icache)) + + val dcache: Instance[Cache] = + Instantiate(new Cache(b.dcache)) + + fetch.io.active := true.B + fetch.io.lookup.config := 0.U.asTypeOf(fetch.io.lookup.config) + fetch.io.lookup.req := 0.U.asTypeOf(fetch.io.lookup.req) + fetch.io.flushTranslation := io.fetch.flushTranslation + fetch.io.request <> io.fetch.request + io.fetch.response <> fetch.io.response + io.fetch.authorizationRequest <> fetch.io.authorizationRequest + fetch.io.authorizationResponse <> io.fetch.authorizationResponse + memory.io.active := true.B + memory.io.lookup.config := 0.U.asTypeOf(memory.io.lookup.config) + memory.io.lookup.req := 0.U.asTypeOf(memory.io.lookup.req) + memory.io.flushTranslation := io.memory.flushTranslation + memory.io.request <> io.memory.request + memory.io.request.valid := io.memory.request.valid && io.commandAllowed + io.memory.request.ready := memory.io.request.ready && io.commandAllowed + io.memory.response <> memory.io.response + io.memory.authorizationRequest <> memory.io.authorizationRequest + memory.io.authorizationResponse <> io.memory.authorizationResponse + val services = io.services + val memoryAllowed = services.cpuAllow + services.cpuQuery := 0.U.asTypeOf(services.cpuQuery) + services.cpuQuery.valid := memory.io.cacheRequest.valid || fetch.io.cacheRequest.valid + services.cpuQuery.paddr := Mux( + memory.io.cacheRequest.valid, + memory.io.cacheRequest.bits.addr, + fetch.io.cacheRequest.bits.addr + ) + services.cpuQuery.sizeLog2 := Mux( + memory.io.cacheRequest.valid && + memory.io.cacheRequest.bits.atomic =/= CacheAtomic.None.U && memory.io.cacheRequest.bits.atomicWord, + 2.U, + 3.U + ) + services.cpuQuery.write := memory.io.cacheRequest.valid && (memory.io.cacheRequest.bits.write || + (memory.io.cacheRequest.bits.atomic =/= CacheAtomic.None.U && memory.io.cacheRequest.bits.atomic =/= CacheAtomic.LR.U)) + + icache.io.access.valid := fetch.io.cacheRequest.valid && memoryAllowed + icache.io.access.bits := fetch.io.cacheRequest.bits + fetch.io.cacheRequest.ready := icache.io.access.ready && memoryAllowed + fetch.io.cacheResponse <> icache.io.result + + val cacheRequests = Instantiate(new Arbiter( + new CacheAccess(access.chi), + 2 + )) + + cacheRequests.io.in(0).valid := memory.io.cacheRequest.valid && memoryAllowed + cacheRequests.io.in(0).bits := memory.io.cacheRequest.bits + memory.io.cacheRequest.ready := cacheRequests.io.in(0).ready && memoryAllowed + cacheRequests.io.in(1).valid := services.pteRequest.valid + cacheRequests.io.in(1).bits := 0.U.asTypeOf(cacheRequests.io.in(1).bits) + cacheRequests.io.in(1).bits.addr := services.pteRequest.bits.addr + services.pteRequest.ready := cacheRequests.io.in(1).ready + + val cacheOwners = Instantiate(new Queue(Bool(), 2, false, false)) + + dcache.io.access.valid := cacheRequests.io.out.valid && cacheOwners.io.enq.ready + dcache.io.access.bits := cacheRequests.io.out.bits + cacheRequests.io.out.ready := dcache.io.access.ready && cacheOwners.io.enq.ready + cacheOwners.io.enq.valid := dcache.io.access.fire + cacheOwners.io.enq.bits := cacheRequests.io.chosen === 1.U + memory.io.cacheResponse.valid := dcache.io.result.valid && cacheOwners.io.deq.valid && !cacheOwners.io.deq.bits + memory.io.cacheResponse.bits := dcache.io.result.bits + services.pteResponse.valid := dcache.io.result.valid && cacheOwners.io.deq.valid && cacheOwners.io.deq.bits + services.pteResponse.bits.data := dcache.io.result.bits.data + services.pteResponse.bits.error := dcache.io.result.bits.error + dcache.io.result.ready := cacheOwners.io.deq.valid && Mux( + cacheOwners.io.deq.bits, + services.pteResponse.ready, + memory.io.cacheResponse.ready + ) + cacheOwners.io.deq.ready := dcache.io.result.fire + dcache.io.maintenance <> services.maintenance + services.maintained <> dcache.io.maintained + dcache.io.olderRequestsDrained := cacheOwners.io.count === 0.U + + val uncachedRequests = Instantiate(new Arbiter( + new UncachedRequest(b.memoryAccess.memory), + 3 + )) + + uncachedRequests.io.in(0) <> fetch.io.uncachedRequest + uncachedRequests.io.in(1) <> memory.io.uncachedRequest + + val uncached: Instance[Uncached] = Instantiate(new Uncached(b.memoryAccess.memory, b.uncached)) + + uncached.io.request(0) <> uncachedRequests.io.out + io.uncached <> uncached.io.axi + uncachedRequests.io.in(2) <> io.timeRequest + val uncachedOwner = Reg(UInt(2.W)) + when(uncachedRequests.io.out.fire)(uncachedOwner := uncachedRequests.io.chosen) + fetch.io.uncachedResponse.valid := uncached.io.response(0).valid && uncachedOwner === 0.U + memory.io.uncachedResponse.valid := uncached.io.response(0).valid && uncachedOwner === 1.U + fetch.io.uncachedResponse.bits := uncached.io.response(0).bits + memory.io.uncachedResponse.bits := uncached.io.response(0).bits + + val timeResponse = io.timeResponse + + timeResponse.valid := uncached.io.response(0).valid && uncachedOwner === 2.U + timeResponse.bits := uncached.io.response(0).bits + uncached.io.response(0).ready := VecInit(Seq( + fetch.io.uncachedResponse.ready, + memory.io.uncachedResponse.ready, + timeResponse.ready + ))(uncachedOwner) + if (externalHome) { + io.coherent(0) <> icache.io.chi + io.coherent(1) <> dcache.io.chi + } else { + val home: Instance[Home] = Instantiate(new Home(b.home)) + home.io.active := true.B + home.io.blockRequesterRsp := false.B + home.io.requesters(0) <> icache.io.chi + home.io.requesters(1) <> dcache.io.chi + home.io.memoryReq <> io.backing.get.request + home.io.memoryResp <> io.backing.get.response + } + +} diff --git a/arch/src/main/scala/framework/core/memory/ThreadMemory.scala b/arch/src/main/scala/framework/core/memory/ThreadMemory.scala new file mode 100644 index 000000000..d8112bef5 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/ThreadMemory.scala @@ -0,0 +1,86 @@ +package framework.core.memory + +import framework.core.memory.access.mmu.{PteRequest, PteResponse} +import framework.ip.memory.interlock.{Acknowledgement, CpuQuery, Maintenance} +import chisel3._ +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +/** Thread-side cache services; requests retain their original admission tag. */ +class MemoryPort(tracking: TrackingParam, walker: WalkerParam) extends Bundle { + + val cpuQuery = Input(new CpuQuery(tracking)) + val cpuAllow = Output(Bool()) + val maintenance = Decoupled(new Maintenance(tracking)) + val maintained = Flipped(Decoupled(new Acknowledgement(tracking))) + val pteRequest = Decoupled(new PteRequest(walker)) + val pteResponse = Flipped(Decoupled(new PteResponse)) +} + +/** Independent fair, single-owner cache-maintenance and page-table service arbiters. */ +@instantiable +class ThreadMemory(ports: Int, tracking: TrackingParam, walker: WalkerParam) extends Module { + require(ports > 0) + + @public + val io = IO(new Bundle { + val clients = Vec(ports, Flipped(new MemoryPort(tracking, walker))) + val system = new MemoryPort(tracking, walker) + }) + + io.system.cpuAllow := io.clients.map(_.cpuAllow).reduce(_ && _) + io.clients.foreach(_.cpuQuery := io.system.cpuQuery) + + def arbitrate[T <: Data, R <: Data]( + requests: Seq[DecoupledIO[T]], + responses: Seq[DecoupledIO[R]], + request: DecoupledIO[T], + response: DecoupledIO[R] + ): Unit = { + val ownerBits = math.max(1, log2Ceil(ports)) + val next = RegInit(0.U(ownerBits.W)) + val owner = Reg(UInt(ownerBits.W)) + val occupied = RegInit(false.B) + val sent = RegInit(false.B) + val packet = Reg(chiselTypeOf(request.bits)) + val candidates = VecInit(requests.map(_.valid)).asUInt + val later = VecInit(requests.zipWithIndex.map { case (r, i) => r.valid && i.U >= next }).asUInt + val selected = PriorityEncoder(Mux(later.orR, later, candidates)) + for (i <- 0 until ports) { + requests(i).ready := !occupied && selected === i.U && !reset.asBool + responses(i).valid := occupied && sent && owner === i.U && response.valid + responses(i).bits := response.bits + when(requests(i).fire) { + packet := requests(i).bits + owner := i.U + occupied := true.B + sent := false.B + next := (if (i + 1 == ports) 0 else i + 1).U + } + } + request.valid := occupied && !sent && !reset.asBool + request.bits := packet + when(request.fire)(sent := true.B) + response.ready := occupied && sent && VecInit(responses.map(_.ready))(owner) && !reset.asBool + when(response.valid)(assert(occupied && sent, "Ant cache service returned a response without a sent owner")) + when(response.fire) { + occupied := false.B + sent := false.B + } + } + + arbitrate( + io.clients.map(_.maintenance), + io.clients.map(_.maintained), + io.system.maintenance, + io.system.maintained + ) + arbitrate( + io.clients.map(_.pteRequest), + io.clients.map(_.pteResponse), + io.system.pteRequest, + io.system.pteResponse + ) +} diff --git a/arch/src/main/scala/framework/core/memory/access/build.sc b/arch/src/main/scala/framework/core/memory/access/build.sc new file mode 100644 index 000000000..d50a3fe29 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/build.sc @@ -0,0 +1,32 @@ +import mill._ +import $file.src.main.scala.cache.{build => cache_build} +import $file.src.main.scala.mmu.{build => mmu_build} +import $file.src.main.scala.preflight.{build => preflight_build} +import $file.^.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.memory.access.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/PhysicalAccess.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/VirtualAccess.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) ++ + cache_build.sources() ++ + mmu_build.sources() ++ + preflight_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + cache_build.emitSources() ++ + mmu_build.emitSources() ++ + preflight_build.emitSources() +) diff --git a/arch/src/main/scala/framework/core/memory/access/configs/uvm.toml b/arch/src/main/scala/framework/core/memory/access/configs/uvm.toml new file mode 100644 index 000000000..1df39ea51 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/configs/uvm.toml @@ -0,0 +1,37 @@ +[parameters] +translationEntries=4 +lineBits=512 +[parameters.memory] +tagBits=6 +[parameters.memory.chi] +nodeIdBits=7 +addressBits=44 +dataBits=256 +[[parameters.regions]] +base="4096" +bytes="1048576" +cacheable=true +executable=true +readable=true +writable=true +atomic=true +normal=true +[[parameters.regions]] +base="2097152" +bytes="4096" +cacheable=false +executable=false +readable=true +writable=true +atomic=false +normal=false +[[parameters.regions]] +base="3145728" +bytes="4096" +cacheable=true +executable=false +readable=true +writable=false +atomic=false +normal=true +[defines] diff --git a/arch/src/main/scala/framework/core/memory/access/src/csrc/Cargo.lock b/arch/src/main/scala/framework/core/memory/access/src/csrc/Cargo.lock new file mode 100644 index 000000000..50eddeffd --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/csrc/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "mem-access-reference-model" +version = "0.1.0" diff --git a/arch/src/main/scala/framework/core/memory/access/src/csrc/Cargo.toml b/arch/src/main/scala/framework/core/memory/access/src/csrc/Cargo.toml new file mode 100644 index 000000000..9cd2edfde --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/csrc/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "mem-access-reference-model" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/src/access.rs b/arch/src/main/scala/framework/core/memory/access/src/csrc/src/access.rs similarity index 95% rename from arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/src/access.rs rename to arch/src/main/scala/framework/core/memory/access/src/csrc/src/access.rs index 553bfe964..bd840030c 100644 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/src/access.rs +++ b/arch/src/main/scala/framework/core/memory/access/src/csrc/src/access.rs @@ -1,5 +1,5 @@ #[no_mangle] -pub unsafe extern "C" fn cpu_mem_ref_prepare( +pub unsafe extern "C" fn mem_access_ref_prepare( addr: u64, size: u32, write: u32, data: u64, atomic: u32, cacheable: u32, normal: u32, address_bits: u32, target: *mut u32, misaligned: *mut u32, access_fault: *mut u32, bus_addr: *mut u64, bus_data: *mut u64, bus_mask: *mut u32, atomic_word: *mut u32, @@ -30,7 +30,7 @@ pub unsafe extern "C" fn cpu_mem_ref_prepare( } #[no_mangle] -pub extern "C" fn cpu_mem_ref_result( +pub extern "C" fn mem_access_ref_result( addr: u64, size: u32, write: u32, signed: u32, atomic: u32, cacheable: u32, raw: u64, error: u32, ) -> u64 { diff --git a/arch/src/main/scala/framework/core/memory/access/src/csrc/src/lib.rs b/arch/src/main/scala/framework/core/memory/access/src/csrc/src/lib.rs new file mode 100644 index 000000000..d91305c8d --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/csrc/src/lib.rs @@ -0,0 +1,5 @@ +mod access; +pub use access::*; + +#[path = "../../main/scala/mmu/src/csrc/src/translation.rs"] +mod translation; diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/env/env.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/env/env.svh new file mode 100644 index 000000000..6654b5ae5 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/env/env.svh @@ -0,0 +1,382 @@ +class env extends uvm_env; + `uvm_component_utils(env) + virtual cpu_control_if control; + virtual stream_if #(`ACCESS_REQ_WIDTH) request; + virtual stream_if #(`ACCESS_RESP_WIDTH) response; + virtual stream_if #(`ACCESS_CACHE_WIDTH) cache_request; + virtual stream_if #(`ACCESS_CRESULT_WIDTH) cache_response; + virtual stream_if #(`ACCESS_UNCACHED_WIDTH) uncached_request; + virtual stream_if #(`ACCESS_URESULT_WIDTH) uncached_response; + result_board #(result_item, 0) scoreboard; + int checked = 0, cache_checked = 0, uncached_checked = 0, faults_checked = 0, cancelled = 0; + covergroup behavior with function sample ( + int size, int atomic_op, int target, bit writing, bit fault, bit error + ); + option.per_instance = 1; + sizes: coverpoint size {bins all[] = {[0 : 3]};} + atomics: coverpoint atomic_op {bins all[] = {[0 : 11]};} + targets: coverpoint target {bins all[] = {[0 : 2]};} + direction: coverpoint writing; + faults: coverpoint fault; + errors: coverpoint error; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + if (!uvm_config_db#(virtual cpu_control_if)::get( + this, "", "control", control + ) || !uvm_config_db#(virtual stream_if #(`ACCESS_REQ_WIDTH))::get( + this, "", "request", request + ) || !uvm_config_db#(virtual stream_if #(`ACCESS_RESP_WIDTH))::get( + this, "", "response", response + ) || !uvm_config_db#(virtual stream_if #(`ACCESS_CACHE_WIDTH))::get( + this, "", "cache_request", cache_request + ) || !uvm_config_db#(virtual stream_if #(`ACCESS_CRESULT_WIDTH))::get( + this, "", "cache_response", cache_response + ) || !uvm_config_db#(virtual stream_if #(`ACCESS_UNCACHED_WIDTH))::get( + this, "", "uncached_request", uncached_request + ) || !uvm_config_db#(virtual stream_if #(`ACCESS_URESULT_WIDTH))::get( + this, "", "uncached_response", uncached_response + )) + `uvm_fatal("VIF", "Memory access interfaces missing") + endfunction + task transact(longint unsigned addr, int size, bit write, bit signed_load, longint unsigned data, + int atomic_op, bit cacheable, longint unsigned raw = 64'h80ff7f0181fe8000, + bit error = 0, bit normal_ram = 0, bit device = 0); + req_t packet = '0; + result_item predicted = result_item::type_id::create("predicted"); + result_item received = result_item::type_id::create("received"); + int unsigned target, misaligned, access_fault, bus_mask, atomic_word; + longint unsigned bus_addr, bus_data, expected_data; + bit [`ACCESS_RESP_WIDTH-1:0] expected_response; + int tag = checked & ((1 << `CPU_TAG_BITS) - 1); + mem_access_ref_prepare(addr, size, write, data, atomic_op, cacheable, + !device && (cacheable || normal_ram), `CPU_PHYSICAL_BITS, target, + misaligned, access_fault, bus_addr, bus_data, bus_mask, atomic_word); + expected_data = target == 0 ? 0 : + mem_access_ref_result(addr, size, write, signed_load, atomic_op, cacheable, raw, error); + expected_response = '0; + if (target == 1 && !error) + expected_response[`CPUF(RESP, LINE)] = `CPU_LINE_BITS'(raw) ^ (`CPU_LINE_BITS'(1) << 511); + expected_response[`CPUF(RESP, TAG)] = tag; + expected_response[`CPUF(RESP, DATA)] = expected_data; + expected_response[`CPUF(RESP, MISALIGNED)] = misaligned; + expected_response[`CPUF(RESP, ACCESSFAULT)] = access_fault || (target != 0 && error); + packet[`CPUF(REQ, ADDR)] = addr; + packet[`CPUF(REQ, TAG)] = tag; + packet[`CPUF(REQ, SIZE)] = size; + packet[`CPUF(REQ, WRITE)] = write; + packet[`CPUF(REQ, SIGNED)] = signed_load; + packet[`CPUF(REQ, DATA)] = data; + packet[`CPUF(REQ, ATOMIC)] = atomic_op; + packet[`CPUF(REQ, CACHEABLE)] = cacheable; + packet[`CPUF(REQ, NORMAL)] = !device && (cacheable || normal_ram); + @(control.cb); + response.ready <= 0; + cache_request.ready <= 0; + uncached_request.ready <= 0; + request.bits <= packet; + request.valid <= 1; + do @(request.sample); while (!request.sample.ready); + request.valid <= 0; + predicted.packet = expected_response; + scoreboard.expected_export.write(predicted); + if (target == 1) begin + do @(cache_request.sample); while (!cache_request.sample.valid); + if (cache_request.sample.bits[ + `CPUF(CACHE, ADDR) + ] !== bus_addr || cache_request.sample.bits[ + `CPUF(CACHE, DATA) + ] !== bus_data || cache_request.sample.bits[ + `CPUF(CACHE, MASK) + ] !== bus_mask || cache_request.sample.bits[ + `CPUF(CACHE, WRITE) + ] !== write || cache_request.sample.bits[ + `CPUF(CACHE, ATOMIC) + ] !== atomic_op || cache_request.sample.bits[ + `CPUF(CACHE, ATOMICWORD) + ] !== atomic_word || uncached_request.valid) + `uvm_fatal("CACHE", $sformatf( + "Cache translation mismatch addr=%h size=%0d atomic=%0d got=%h", + addr, + size, + atomic_op, + cache_request.sample.bits + )) + repeat (3) @(control.cb); + cache_request.ready <= 1; + @(cache_request.sample); + cache_request.ready <= 0; + repeat (2) @(control.cb); + cache_response.bits <= '0; + cache_response.bits[ + `CPUF(CRESULT, LINE) + ] <= `CPU_LINE_BITS'(raw) ^ (`CPU_LINE_BITS'(1) << 511); + cache_response.bits[`CPUF(CRESULT, DATA)] <= raw; + cache_response.bits[`CPUF(CRESULT, ERROR)] <= error; + cache_response.valid <= 1; + do @(cache_response.sample); while (!cache_response.sample.ready); + cache_response.valid <= 0; + cache_checked++; + end else if (target == 2) begin + cache_response.bits[`CPUF(CRESULT, DATA)] <= ~raw; + do @(uncached_request.sample); while (!uncached_request.sample.valid); + if (uncached_request.sample.bits[ + `CPUF(UNCACHED, ADDR) + ] !== addr || uncached_request.sample.bits[ + `CPUF(UNCACHED, SIZE) + ] !== size || uncached_request.sample.bits[ + `CPUF(UNCACHED, WRITE) + ] !== write || uncached_request.sample.bits[ + `CPUF(UNCACHED, DATA) + ] !== data || uncached_request.sample.bits[ + `CPUF(UNCACHED, TAG) + ] !== tag || cache_request.valid) + `uvm_fatal("MMIO", "Uncached access expanded or changed its address/size/data/tag") + repeat (3) @(control.cb); + uncached_request.ready <= 1; + @(uncached_request.sample); + uncached_request.ready <= 0; + repeat (2) @(control.cb); + uncached_response.bits <= '0; +`ifdef CPU_BAD_TAG + uncached_response.bits[`CPUF(URESULT, TAG)] <= tag ^ 1; +`else + uncached_response.bits[`CPUF(URESULT, TAG)] <= tag; +`endif + uncached_response.bits[`CPUF(URESULT, DATA)] <= raw; + uncached_response.bits[`CPUF(URESULT, ERROR)] <= error; + uncached_response.valid <= 1; + do @(uncached_response.sample); while (!uncached_response.sample.ready); + uncached_response.valid <= 0; + uncached_checked++; + end else begin + repeat (3) begin + @(control.cb); + if (cache_request.valid || uncached_request.valid) + `uvm_fatal("FAULT", "Faulting request reached an external memory port") + end + faults_checked++; + end + do @(response.sample); while (!response.sample.valid); + if (response.sample.bits !== expected_response) + `uvm_fatal("RESULT", $sformatf( + "CPU result mismatch addr=%h size=%0d atomic=%0d got=%h expected=%h", + addr, + size, + atomic_op, + response.sample.bits, + expected_response + )) + repeat (3) begin + @(control.cb); + if (request.ready || response.bits !== expected_response || !response.valid || + cache_request.valid || uncached_request.valid) + `uvm_fatal("SINGLE", "Outstanding CPU response changed or accepted another operation") + end + response.ready <= 1; + @(response.sample); + response.ready <= 0; + received.packet = response.sample.bits; + scoreboard.actual_export.write(received); + behavior.sample(size, atomic_op, target, write, misaligned || access_fault, error); + checked++; + endtask + // Reset jointly cancels the CPU operation and its memory response producer; stores are not rolled back. + task cancel_at(int phase_index, bit cacheable); + req_t packet = '0; + result_item predicted = result_item::type_id::create("predicted"); + result_item received = result_item::type_id::create("received"); + packet[`CPUF(REQ, ADDR)] = 64'h5000; + packet[`CPUF(REQ, SIZE)] = 3; + packet[`CPUF(REQ, TAG)] = 63; + packet[`CPUF(REQ, CACHEABLE)] = cacheable; + packet[`CPUF(REQ, NORMAL)] = cacheable; + @(control.cb); + request.bits <= packet; + request.valid <= 1; + cache_request.ready <= 0; + uncached_request.ready <= 0; + response.ready <= 0; + do @(request.sample); while (!request.sample.ready); + request.valid <= 0; + predicted.packet = '0; + scoreboard.expected_export.write(predicted); + if (cacheable) begin + do @(cache_request.sample); while (!cache_request.sample.valid); + if (phase_index != 0) begin + cache_request.ready <= 1; + @(cache_request.sample); + cache_request.ready <= 0; + if (phase_index == 2) begin + cache_response.bits <= '0; + cache_response.valid <= 1; + do @(cache_response.sample); while (!cache_response.sample.ready); + cache_response.valid <= 0; + do @(response.sample); while (!response.sample.valid); + end + end + end else begin + do @(uncached_request.sample); while (!uncached_request.sample.valid); + if (phase_index != 0) begin + uncached_request.ready <= 1; + @(uncached_request.sample); + uncached_request.ready <= 0; + if (phase_index == 2) begin + uncached_response.bits <= '0; + uncached_response.bits[`CPUF(URESULT, TAG)] <= 63; + uncached_response.valid <= 1; + do @(uncached_response.sample); while (!uncached_response.sample.ready); + uncached_response.valid <= 0; + do @(response.sample); while (!response.sample.valid); + end + end + end + control.cb.reset <= 1; + scoreboard.cancel_pending(); + cache_response.valid <= 0; + // Payload and VALID are ignored during joint reset, including an unrelated tag. + uncached_response.bits <= '0; + uncached_response.valid <= !cacheable && phase_index == 1; + repeat (2) @(control.cb); + uncached_response.valid <= 0; + control.cb.reset <= 0; + repeat (2) @(control.cb); + if (response.valid || cache_request.valid || uncached_request.valid || !request.ready) + `uvm_fatal("RESET", "Reset retained a cancelled transaction") + cancelled++; + endtask + task execute(); + control.reset = 1; + request.valid = 1; + request.bits = '1; + response.ready = 0; + cache_request.ready = 0; + uncached_request.ready = 0; + cache_response.valid = 0; + cache_response.bits = '0; + uncached_response.valid = 0; + uncached_response.bits = '0; + repeat (4) @(control.cb); + request.valid <= 0; + control.cb.reset <= 0; +`ifdef ACCESS_BAD_CASE + begin + req_t bad = '0; + bad[`CPUF(REQ, SIZE)] = `ACCESS_BAD_CASE == 1 ? 4 : (`ACCESS_BAD_CASE == 4 ? 1 : 3); + bad[`CPUF(REQ, ATOMIC)] = `ACCESS_BAD_CASE == 2 ? 12 : (`ACCESS_BAD_CASE == 3 ? 15 : 1); + bad[`CPUF(REQ, WRITE)] = `ACCESS_BAD_CASE == 5; + bad[`CPUF(REQ, CACHEABLE)] = 1; + @(control.cb); + request.bits <= bad; + request.valid <= 1; + repeat (10) @(control.cb); + `uvm_fatal("NO_ASSERT", "Invalid Memory access contract was accepted") + end +`elsif CPU_BAD_TAG + transact(64'h60020000, 0, 0, 0, 0, 0, 0); + `uvm_fatal("NO_ASSERT", "Incorrect uncached response tag was accepted") +`else + for (int phase_index = 0; phase_index < 3; phase_index++) + for (int cacheable = 0; cacheable < 2; cacheable++) cancel_at(phase_index, cacheable); + for (int cacheable = 0; cacheable < 2; cacheable++) + for (int size = 0; size < 4; size++) + for (int offset = 0; offset < 8; offset++) + for (int sign_bit = 0; sign_bit < 2; sign_bit++) begin + transact(64'h60020000 + offset, size, 0, sign_bit, 0, 0, cacheable); + transact(64'h60020000 + offset, size, 1, 0, 64'hfedcba9876543210, 0, cacheable); + end + // Every aligned upper/lower Word atomic lane, including sign-expanded results and SC status. + for (int op = 1; op <= 11; op++) + for (int size = 2; size < 4; size++) + for (int offset = 0; offset < 8; offset += 4) begin + longint unsigned raw = op == 11 ? 1 : (size == 2 ? 64'hffffffff80000081 : 64'h8123456789abcdef); + transact(64'h1000 + offset, size, 0, 0, 64'h8877665544332211, op, 1, raw); + transact(64'h1000 + offset, size, 0, 1, 64'h1122334455667788, op, 1, raw, 1); + transact(64'h1000 + offset, size, 0, 0, 1, op, 0, raw); + end + for (int size = 2; size < 4; size++) + for (int offset = 1; offset < 8; offset++) + if (offset % (1 << size) != 0) transact(64'h1000 + offset, size, 0, 0, 1, 1, 1); + for (int size = 0; size < 4; size++) begin + transact(64'h2000, size, 0, 1, 0, 0, 1, '1); + transact(64'h2000, size, 0, 0, 0, 0, 1, 0); + transact(64'h2000, size, 0, 0, 0, 0, 1, '1, 1); + transact(64'h60020004, size, 0, 1, 0, 0, 0, '1, 1); + transact(64'h60020004, size, 1, 0, '1, 0, 0, 0, 1); + transact(64'h60020000, size, 0, 1, 0, 0, 0, '1); + transact(64'h60020000, size, 0, 0, 0, 0, 0, '1); + transact(64'h60020000, size, 1, 0, '1, 0, 0, 0, 1); + end + for (int bit_index = 3; bit_index < 64; bit_index++) begin + transact(64'h1 << bit_index, 3, 0, 0, 0, 0, 1); + transact(64'h1 << bit_index, 3, 1, 0, '1, 0, 0); + end + transact(64'h1004, 2, 0, 1, 0, 0, 1, 0); + transact(64'h60020004, 2, 0, 1, 0, 0, 0, 0); + transact(64'h1000, 3, 1, 0, 0, 0, 1, 0, 1); + transact(64'h1000, 2, 0, 0, 1, 11, 1, 0); + transact(64'h1004, 2, 0, 0, 1, 11, 1, 0); + transact(64'h1000, 3, 0, 0, 1, 11, 1, 0); + transact(64'h1000, 3, 1, 0, 0, 0, 1); + transact(64'h1000, 3, 1, 0, '1, 0, 1); + transact((64'h1 << `CPU_PHYSICAL_BITS) - 8, 3, 0, 0, 0, 0, 1); + transact((64'h1 << `CPU_PHYSICAL_BITS) - 1, 0, 1, 0, '1, 0, 1); + // Noncacheable normal memory, including every atomic, faults without issuing traffic. + for (int size = 0; size < 4; size++) begin + transact(64'h3000, size, 0, 0, 0, 0, 0, 0, 0, 1); + transact(64'h3000, size, 1, 0, 1, 0, 0, 0, 0, 1); + transact(64'h3001, size, 0, 0, 0, 0, 0, 0, 0, 1); + transact(64'h60020000, size, 0, 0, 0, 0, 1, 0, 0, 0, 1); + transact(64'h60020000, size, 1, 0, 1, 0, 1, 0, 0, 0, 1); + transact(64'h60020001, size, 0, 0, 0, 0, 1, 0, 0, 0, 1); + end + for (int op = 1; op <= 11; op++) + for (int size = 2; size <= 3; size++) begin + transact(64'h3000 + (size == 2 ? 4 : 0), size, 0, 0, 64'h76543210, op, 0, + op == 11 ? 0 : 64'hffffffff81234567, 0, 1); + transact(64'h3000, size, 0, 0, 1, op, 0, 64'hdeadbeef12345678, 1, 1); + end + transact(64'h3004, 2, 0, 0, 1, 11, 0, 1, 0, 1); + // Reset ignores even VALID=1 garbage payload; no transaction may escape reset. + control.cb.reset <= 1; + request.valid <= 1; + request.bits <= '1; + repeat (2) @(control.cb); + request.valid <= 0; + request.bits <= '0; + control.cb.reset <= 0; + repeat (2) @(control.cb); + if (response.valid || cache_request.valid || uncached_request.valid || !request.ready) + `uvm_fatal("RESET_PAYLOAD", "Reset payload escaped as a transaction") + // VALID=0 payload is unconstrained; it must not create an operation. + response.ready <= 1; + cache_request.ready <= 1; + uncached_request.ready <= 1; + request.bits <= '1; + cache_response.bits <= '1; + uncached_response.bits <= '1; + repeat (2) @(control.cb); + request.bits <= '0; + cache_response.bits <= '0; + + uncached_response.bits <= '0; + repeat (2) @(control.cb); + if (response.valid || cache_request.valid || uncached_request.valid || !request.ready) + `uvm_fatal("IDLE", "Idle payload generated an operation") + `uvm_info("MEM_ACCESS", $sformatf( + "Checked %0d CPU results, %0d cache requests, %0d exact uncached requests, %0d suppressed faults, %0d joint-reset cancellations", + checked, + cache_checked, + uncached_checked, + faults_checked, + cancelled + ), UVM_LOW) +`endif + endtask +endclass diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/interface.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/interface.sv rename to arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/interface.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access.f new file mode 100644 index 000000000..a8de5d1cb --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access.f @@ -0,0 +1,10 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core + +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PhysicalAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_atomic.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_atomic.f new file mode 100644 index 000000000..2d2363053 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_atomic.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++define+MEM_ACCESS_TOP=mem_access_bad_atomic_tb ++define+ACCESS_BAD_CASE=3 +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PhysicalAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_atomic_write.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_atomic_write.f new file mode 100644 index 000000000..03c2ec8a5 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_atomic_write.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++define+MEM_ACCESS_TOP=mem_access_bad_atomic_write_tb ++define+ACCESS_BAD_CASE=5 +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PhysicalAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_fence.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_fence.f new file mode 100644 index 000000000..5a8b53b8e --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_fence.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++define+MEM_ACCESS_TOP=mem_access_bad_fence_tb ++define+ACCESS_BAD_CASE=2 +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PhysicalAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_operand.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_operand.f new file mode 100644 index 000000000..4f160b666 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_operand.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++define+MEM_ACCESS_TOP=mem_access_bad_operand_tb ++define+ACCESS_BAD_CASE=4 +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PhysicalAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_size.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_size.f new file mode 100644 index 000000000..45365a4d6 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_size.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++define+MEM_ACCESS_TOP=mem_access_bad_size_tb ++define+ACCESS_BAD_CASE=1 +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PhysicalAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_tag.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_tag.f new file mode 100644 index 000000000..4514b69b2 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/mem_access_bad_tag.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++define+MEM_ACCESS_TOP=mem_access_bad_tag_tb ++define+CPU_BAD_TAG +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PhysicalAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/package.sv b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/package.sv new file mode 100644 index 000000000..ad7579ba0 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/package.sv @@ -0,0 +1,46 @@ +`include "physical_parameters.svh" +package mem_access_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `define CPUF(kind, field) `CPU_``kind``_``field``_OFFSET +: `CPU_``kind``_``field``_WIDTH + typedef bit [`ACCESS_REQ_WIDTH-1:0] req_t; + import "DPI-C" function void mem_access_ref_prepare( + input longint unsigned addr, + input int unsigned size, + write, + input longint unsigned data, + input int unsigned atomic_op, + cacheable, + normal, + address_bits, + output int unsigned target, + misaligned, + access_fault, + output longint unsigned bus_addr, + bus_data, + output int unsigned bus_mask, + atomic_word + ); + import "DPI-C" function longint unsigned mem_access_ref_result( + input longint unsigned addr, + input int unsigned size, + write, + signed_load, + atomic_op, + cacheable, + input longint unsigned raw, + input int unsigned error + ); + class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(packet, UVM_ALL_ON) + `uvm_object_utils_end + logic [`ACCESS_RESP_WIDTH-1:0] packet; + function new(string name = "result"); + super.new(name); + endfunction + endclass + `include "env/env.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/tb.sv b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/tb.sv new file mode 100644 index 000000000..90e705daf --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/tb.sv @@ -0,0 +1,89 @@ +`include "physical_parameters.svh" +`ifndef MEM_ACCESS_TOP +`define MEM_ACCESS_TOP mem_access_tb +`endif +module `MEM_ACCESS_TOP; + import uvm_pkg::*; + import mem_access_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + cpu_control_if control (clock); + stream_if #(`ACCESS_REQ_WIDTH) request ( + clock, + control.reset + ); + stream_if #(`ACCESS_RESP_WIDTH) response ( + clock, + control.reset + ); + stream_if #(`ACCESS_CACHE_WIDTH) cache_request ( + clock, + control.reset + ); + stream_if #(`ACCESS_CRESULT_WIDTH) cache_response ( + clock, + control.reset + ); + stream_if #(`ACCESS_UNCACHED_WIDTH) uncached_request ( + clock, + control.reset + ); + stream_if #(`ACCESS_URESULT_WIDTH) uncached_response ( + clock, + control.reset + ); + int unsigned in_flight = 0; + bit issued = 0, returned = 0, uncached = 0; + always @(posedge clock) begin + if (control.reset) begin + in_flight = 0; + issued = 0; + returned = 0; + end else begin + if (in_flight != 0 && request.ready) + $fatal(1, "CPU port became ready before consuming its response"); + if (request.valid && request.ready) begin + if (in_flight != 0) $fatal(1, "Memory access accepted overlapping requests"); + in_flight = 1; + issued = 0; + returned = 0; + end + if ((cache_request.valid && cache_request.ready) || (uncached_request.valid && uncached_request.ready)) begin + if (in_flight != 1 || issued) $fatal(1, "Unexpected or duplicate backend request"); + issued = 1; + uncached = uncached_request.valid && uncached_request.ready; + end + if ((cache_response.valid && cache_response.ready) || (uncached_response.valid && uncached_response.ready)) begin + if (in_flight != 1 || !issued || returned || + uncached != (uncached_response.valid && uncached_response.ready)) + $fatal(1, "Unexpected or duplicate backend response"); + returned = 1; + end + if (response.valid && response.ready) begin + if (in_flight != 1 || (issued && !returned)) $fatal(1, "Unexpected CPU result"); + in_flight = 0; + end + end + end + PhysicalAccess dut ( + .clock(clock), + .reset(control.reset), + `include "physical_ports.svh" + ); + initial begin + uvm_config_db#(virtual cpu_control_if)::set(null, "uvm_test_top*", "control", control); + uvm_config_db#(virtual stream_if #(`ACCESS_REQ_WIDTH))::set(null, "uvm_test_top*", "request", + request); + uvm_config_db#(virtual stream_if #(`ACCESS_RESP_WIDTH))::set(null, "uvm_test_top*", "response", + response); + uvm_config_db#(virtual stream_if #(`ACCESS_CACHE_WIDTH))::set(null, "uvm_test_top*", + "cache_request", cache_request); + uvm_config_db#(virtual stream_if #(`ACCESS_CRESULT_WIDTH))::set( + null, "uvm_test_top*", "cache_response", cache_response); + uvm_config_db#(virtual stream_if #(`ACCESS_UNCACHED_WIDTH))::set( + null, "uvm_test_top*", "uncached_request", uncached_request); + uvm_config_db#(virtual stream_if #(`ACCESS_URESULT_WIDTH))::set( + null, "uvm_test_top*", "uncached_response", uncached_response); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/tests/test.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/tests/test.svh new file mode 100644 index 000000000..398043c00 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/physical/tests/test.svh @@ -0,0 +1,14 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + test_env.execute(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/memory/access/src/main/resources/suite.toml new file mode 100644 index 000000000..2807cc85a --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/suite.toml @@ -0,0 +1,139 @@ +[model] +manifest="src/csrc/Cargo.toml" +library="libmem_access_reference_model" + +[[tests]] +name="virtual_access" +top="virtual_access_tb" +test="protocol_test" +filelist="virtual/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["virtual_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" + +[[tests]] +functional_groups = ["mem_access_pkg::env::behavior"] +name = "mem_access" +top = "mem_access_tb" +test = "protocol_test" +filelist = "physical/mem_access.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "mem_access_bad_atomic" +top = "mem_access_bad_atomic_tb" +test = "protocol_test" +filelist = "physical/mem_access_bad_atomic.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "Memory access port does not accept Fence or unknown atomics" + +[[tests]] +functional_groups = [] +name = "mem_access_bad_atomic_write" +top = "mem_access_bad_atomic_write_tb" +test = "protocol_test" +filelist = "physical/mem_access_bad_atomic_write.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "Memory atomic requires a 32 or 64 bit operand" + +[[tests]] +functional_groups = [] +name = "mem_access_bad_fence" +top = "mem_access_bad_fence_tb" +test = "protocol_test" +filelist = "physical/mem_access_bad_fence.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "Memory access port does not accept Fence or unknown atomics" + +[[tests]] +functional_groups = [] +name = "mem_access_bad_operand" +top = "mem_access_bad_operand_tb" +test = "protocol_test" +filelist = "physical/mem_access_bad_operand.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "Memory atomic requires a 32 or 64 bit operand" + +[[tests]] +functional_groups = [] +name = "mem_access_bad_size" +top = "mem_access_bad_size_tb" +test = "protocol_test" +filelist = "physical/mem_access_bad_size.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "Memory access size must be 1, 2, 4 or 8 bytes" + +[[tests]] +functional_groups = [] +name = "mem_access_bad_tag" +top = "mem_access_bad_tag_tb" +test = "protocol_test" +filelist = "physical/mem_access_bad_tag.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "Uncached response tag mismatch" diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/driver.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/driver.svh new file mode 100644 index 000000000..e0c289e6b --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual virtual_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual virtual_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.source = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/fixture.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/fixture.svh new file mode 100644 index 000000000..c38fe090e --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/fixture.svh @@ -0,0 +1,19 @@ +class fixture extends uvm_object; + `uvm_object_utils(fixture) + logic [63:0] ptes[longint unsigned]; + bit errors[longint unsigned]; + bit deny_pte, deny_data, backend_error; + longint unsigned authorized_pte; + function new(string name = "fixture"); + super.new(name); + endfunction + function logic [63:0] word(longint unsigned address); + return 64'h80ff7f0181fe8000 ^ (address * 64'h0101010101010101); + endfunction + function logic [`VA_LINE_BITS-1:0] line(longint unsigned address); + logic [`VA_LINE_BITS-1:0] value; + for (int i = 0; i < `VA_LINE_BITS / 64; i++) + value[i*64+:64] = word((address & ~64'd63) + i * 8); + return value; + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/item.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/item.svh new file mode 100644 index 000000000..1179c85f6 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/item.svh @@ -0,0 +1,26 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`VA_INPUT_BITS-1:0] inputs; + logic [`VA_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(tag, UVM_ALL_ON) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(line, UVM_ALL_ON) + `uvm_field_int(misaligned, UVM_ALL_ON) + `uvm_field_int(page_fault, UVM_ALL_ON) + `uvm_field_int(access_fault, UVM_ALL_ON) + `uvm_object_utils_end + int tag; + bit [63:0] data; + bit [`VA_LINE_BITS-1:0] line; + bit misaligned, page_fault, access_fault; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/monitor.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/monitor.svh new file mode 100644 index 000000000..46c8093ac --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual virtual_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual virtual_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/peer.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/peer.svh new file mode 100644 index 000000000..bf81965cf --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/agent/peer.svh @@ -0,0 +1,68 @@ +class peer extends uvm_component; + `uvm_component_utils(peer) + virtual virtual_if vif; + fixture image; + bit auth_pending, cache_pending, uncached_pending; + bit auth_allow, cache_error; + logic [63:0] cache_data, uncached_data; + logic [`VA_LINE_BITS-1:0] cache_line; + int tag, cycle, auth_due, cache_due, uncached_due; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual virtual_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Virtual access interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(posedge vif.clock); + cycle++; + if (vif.reset) begin + auth_pending = 0; + cache_pending = 0; + uncached_pending = 0; + end else begin + if (`VO(AUTHORIZATIONRESPONSE_READY) && `VI(AUTHORIZATIONRESPONSE_VALID)) auth_pending = 0; + if (`VO(CACHERESPONSE_READY) && `VI(CACHERESPONSE_VALID)) cache_pending = 0; + if (`VO(UNCACHEDRESPONSE_READY) && `VI(UNCACHEDRESPONSE_VALID)) uncached_pending = 0; + if (`VO(AUTHORIZATIONREQUEST_VALID) && `VI(AUTHORIZATIONREQUEST_READY)) begin + bit pte = `VO(AUTHORIZATIONREQUEST_BITS_ISPTE); + auth_pending = 1; + auth_due = cycle + 3; + auth_allow = pte ? !image.deny_pte : !image.deny_data; + end + if (`VO(CACHEREQUEST_VALID) && `VI(CACHEREQUEST_READY)) begin + longint unsigned address = `VO(CACHEREQUEST_BITS_ADDR); + cache_pending = 1; + cache_due = cycle + 4; + cache_data = image.ptes.exists(address) ? image.ptes[address] : image.word(address); + cache_error = image.ptes.exists(address) ? image.errors[address] : image.backend_error; + cache_line = image.line(address); + end + if (`VO(UNCACHEDREQUEST_VALID) && `VI(UNCACHEDREQUEST_READY)) begin + uncached_pending = 1; + uncached_due = cycle + 4; + uncached_data = image.word(`VO(UNCACHEDREQUEST_BITS_ADDR)); + tag = `VO(UNCACHEDREQUEST_BITS_TAG); + end + end + @(negedge vif.clock); + vif.backend = '0; + vif.backend[`VA_IN_AUTHORIZATIONREQUEST_READY_OFFSET+:`VA_IN_AUTHORIZATIONREQUEST_READY_WIDTH]=cycle%4!=0; + vif.backend[`VA_IN_AUTHORIZATIONRESPONSE_VALID_OFFSET+:`VA_IN_AUTHORIZATIONRESPONSE_VALID_WIDTH]=auth_pending&&cycle>=auth_due; + vif.backend[`VA_IN_AUTHORIZATIONRESPONSE_BITS_OFFSET+:`VA_IN_AUTHORIZATIONRESPONSE_BITS_WIDTH]=auth_allow; + vif.backend[`VA_IN_CACHEREQUEST_READY_OFFSET+:`VA_IN_CACHEREQUEST_READY_WIDTH]=cycle%5!=0; + vif.backend[`VA_IN_UNCACHEDREQUEST_READY_OFFSET+:`VA_IN_UNCACHEDREQUEST_READY_WIDTH]=cycle%3!=0; + vif.backend[`VA_IN_CACHERESPONSE_VALID_OFFSET+:`VA_IN_CACHERESPONSE_VALID_WIDTH]=cache_pending&&cycle>=cache_due; + vif.backend[`VA_IN_CACHERESPONSE_BITS_DATA_OFFSET+:`VA_IN_CACHERESPONSE_BITS_DATA_WIDTH]=cache_data; + vif.backend[`VA_IN_CACHERESPONSE_BITS_ERROR_OFFSET+:`VA_IN_CACHERESPONSE_BITS_ERROR_WIDTH]=cache_error; + vif.backend[`VA_IN_CACHERESPONSE_BITS_LINE_OFFSET+:`VA_IN_CACHERESPONSE_BITS_LINE_WIDTH]=cache_line; + vif.backend[`VA_IN_UNCACHEDRESPONSE_VALID_OFFSET+:`VA_IN_UNCACHEDRESPONSE_VALID_WIDTH]=uncached_pending&&cycle>=uncached_due; + vif.backend[`VA_IN_UNCACHEDRESPONSE_BITS_DATA_OFFSET+:`VA_IN_UNCACHEDRESPONSE_BITS_DATA_WIDTH]=uncached_data; + vif.backend[`VA_IN_UNCACHEDRESPONSE_BITS_TAG_OFFSET+:`VA_IN_UNCACHEDRESPONSE_BITS_TAG_WIDTH]=tag; + vif.backend[`VA_IN_UNCACHEDRESPONSE_BITS_ERROR_OFFSET+:`VA_IN_UNCACHEDRESPONSE_BITS_ERROR_WIDTH]=image.backend_error; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/env/env.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/env/env.svh new file mode 100644 index 000000000..26a25d996 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/env/env.svh @@ -0,0 +1,36 @@ +class env extends uvm_env; + `uvm_component_utils(env) + uvm_analysis_imp #(uvm_sequence_item, env) reset_events; + driver source; + monitor mon; + peer backend; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_events = new("reset_events", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + backend = peer::type_id::create("backend", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + fixture image = fixture::type_id::create("image"); + reference.image = image; + backend.image = image; + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + reference.cancelled.connect(reset_events); + endfunction + function void write(uvm_sequence_item t); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/env/model.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/env/model.svh new file mode 100644 index 000000000..8d9c3283d --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/env/model.svh @@ -0,0 +1,219 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + fixture image; + chandle walker; + uvm_analysis_port #(result_item) expected, actual; + uvm_analysis_port #(uvm_sequence_item) cancelled; + typedef struct { + longint unsigned pa; + int size; + bit reading, writing, executing, pte; + int privilege; + } authorization; + typedef struct { + longint unsigned address, data; + int mask, atomic_op, word; + bit writing, pte, uncached; + int tag, size; + } transfer; + authorization authorizations[$]; + transfer transfers[$]; + bit pending; + covergroup behavior with function sample ( + int size, + int privilege, + int mode, + int target, + bit write, + int atomic_op, + bit misaligned, + bit pf, + bit af, + bit denied, + bit error + ); + option.per_instance = 1; + sizes: coverpoint size {bins all[] = {[0 : 3]};} + privileges: coverpoint privilege {bins all[] = {0, 1, 3};} + modes: coverpoint mode {bins all[] = {0, 8};} + targets: coverpoint target {bins none = {0}; bins cached = {1}; bins uncached = {2};} + direction: coverpoint write; + atomics: coverpoint atomic_op { + bins ordinary = {0}; bins lr = {10}; bins sc = {11}; bins other = {[1 : 9]}; + } + alignment: coverpoint misaligned; + page_faults: coverpoint pf; + access_faults: coverpoint af; + permission: coverpoint denied; + backend_errors: coverpoint error; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + cancelled = new("cancelled", this); + behavior = new(); + walker = mmu_ref_create(`VA_ADDRESS_BITS, 0); + endfunction + function void verify(bit ok, string message); + evidence::checks++; + if (!ok) `uvm_fatal("VIRTUAL_ACCESS", message) + endfunction + function void set_pte(longint unsigned address, logic [63:0] value, bit error = 0); + image.ptes[address] = value; + image.errors[address] = error; + mmu_ref_program(walker, address, value, error); + endfunction + function void predict(cycle_item t); + longint unsigned va, pa, root, data, bus_address, bus_data, raw, pte_address; + int size, tag, privilege, mode, atomic_op, target, misaligned, af, pf, level, mask, atomic_word; + bit writing, executing, signed_load, read_data, write_data, region_ok, cacheable, normal; + result_item result = result_item::type_id::create("expected"); + va = `MI(t, REQUEST_BITS_VADDR); + size = `MI(t, REQUEST_BITS_SIZE); + tag = `MI(t, REQUEST_BITS_TAG); + privilege = `MI(t, REQUEST_BITS_PRIVILEGE); + mode = `MI(t, REQUEST_BITS_SATPMODE); + root = `MI(t, REQUEST_BITS_ROOTPPN); + data = `MI(t, REQUEST_BITS_DATA); + writing = `MI(t, REQUEST_BITS_WRITE); + executing = `MI(t, REQUEST_BITS_EXECUTE); + signed_load = `MI(t, REQUEST_BITS_SIGNED); + atomic_op = `MI(t, REQUEST_BITS_ATOMIC); + result.tag = tag; + misaligned = (va & ((64'd1 << size) - 1)) != 0; + result.misaligned = misaligned; + target = 0; + pf = 0; + af = 0; + verify(!pending && authorizations.size() == 0 && transfers.size() == 0, + "Overlapping virtual requests"); + if (!misaligned) begin + // The fixture denies a PTE through the authorization boundary, before cache access. + foreach (image.ptes[address]) + mmu_ref_program(walker, address, image.ptes[address], + image.errors[address] || image.deny_pte); + verify(mmu_ref_translate( + walker, + va, + mode, + root, + privilege, + writing || (atomic_op != 0 && atomic_op != 10), + executing, + `MI(t, REQUEST_BITS_SUM), + `MI(t, REQUEST_BITS_MXR), + pa, + pf, + af, + level + ) == 0, "Missing page table fixture"); + while (mmu_ref_peek( + walker, pte_address + ) != 0) begin + authorizations.push_back('{pte_address, 3, 1, 0, 0, 1, 1}); + if (!image.deny_pte) transfers.push_back('{pte_address, 0, 0, 0, 0, 0, 1, 0, 0, 3}); + verify(mmu_ref_consume(walker, pte_address) != 0, "PTE trace consumption failed"); + end + result.page_fault = pf; + result.access_fault = af; + if (!pf && !af) begin + longint unsigned last = pa + ((64'd1 << size) - 1); + read_data = !executing && !writing && atomic_op != 11; + write_data = writing || (atomic_op != 0 && atomic_op != 10); + cacheable = pa >= 4096 && last < 1052672 || pa >= 3145728 && last < 3149824; + normal = cacheable; + region_ok=(pa>=4096&&last<1052672)|| +(pa>=2097152&&last<2101248&&!executing&&atomic_op==0)|| +(pa>=3145728&&last<3149824&&!executing&&!write_data&&atomic_op==0); + authorizations.push_back('{pa, size, read_data, write_data, executing, 0, privilege}); + if (region_ok && !image.deny_data) begin + mem_access_ref_prepare(pa, size, writing, data, atomic_op, cacheable, normal, + `VA_ADDRESS_BITS, target, misaligned, af, bus_address, bus_data, + mask, atomic_word); + verify(target != 0 && !af && !misaligned, "Invalid physical reference plan"); + transfers.push_back('{bus_address, bus_data, mask, atomic_op, atomic_word, writing, 0, + target == 2, tag, size}); + raw = image.word(bus_address); + result.data = mem_access_ref_result(pa, size, writing, signed_load, atomic_op, cacheable, + raw, image.backend_error); + result.access_fault = image.backend_error; + if (target == 1 && !image.backend_error) result.line = image.line(bus_address); + end else result.access_fault = 1; + end + end + expected.write(result); + pending = 1; + behavior.sample(size, privilege, mode, target, writing, atomic_op, misaligned, pf, + result.access_fault, image.deny_pte || image.deny_data, image.backend_error); + endfunction + function void write(cycle_item t); + if (t.reset) begin + cancelled.write(null); + pending = 0; + authorizations.delete(); + transfers.delete(); + mmu_ref_reset(walker); + return; + end + if (`MI(t, REQUEST_VALID) && `MO(t, REQUEST_READY)) predict(t); + if (`MO(t, AUTHORIZATIONREQUEST_VALID)) begin + authorization a; + verify(authorizations.size() != 0, "Unexpected authorization request"); + a = authorizations[0]; + verify(`MO(t, AUTHORIZATIONREQUEST_BITS_PADDR) == a.pa && + `MO(t, AUTHORIZATIONREQUEST_BITS_SIZE) == a.size && + `MO(t, AUTHORIZATIONREQUEST_BITS_READ) == a.reading && + `MO(t, AUTHORIZATIONREQUEST_BITS_WRITE) == a.writing && + `MO(t, AUTHORIZATIONREQUEST_BITS_EXECUTE) == a.executing && + `MO(t, AUTHORIZATIONREQUEST_BITS_ISPTE) == a.pte && + `MO(t, AUTHORIZATIONREQUEST_BITS_PRIVILEGE) == a.privilege, + "Authorization footprint mismatch"); + if (`MI(t, AUTHORIZATIONREQUEST_READY)) void'(authorizations.pop_front()); + end + if (`MO(t, CACHEREQUEST_VALID) || `MO(t, UNCACHEDREQUEST_VALID)) begin + transfer a; + bit uncached; + verify(transfers.size() != 0, "Speculative backend request without authorization"); + a = transfers[0]; + uncached = `MO(t, UNCACHEDREQUEST_VALID); + verify(uncached == a.uncached, "Cache/MMIO target mismatch"); + if (uncached) begin + verify(`MO(t, UNCACHEDREQUEST_BITS_ADDR) == a.address && `MO(t, UNCACHEDREQUEST_BITS_SIZE) + == a.size && `MO(t, UNCACHEDREQUEST_BITS_TAG) == a.tag && + `MO(t, UNCACHEDREQUEST_BITS_WRITE) == a.writing && `MO(t, UNCACHEDREQUEST_BITS_DATA) + == a.data, "Uncached request mismatch"); + if (`MI(t, UNCACHEDREQUEST_READY)) void'(transfers.pop_front()); + end else begin + verify(`MO(t, CACHEREQUEST_BITS_ADDR) == a.address && `MO(t, CACHEREQUEST_BITS_WRITE) + == a.writing && `MO(t, CACHEREQUEST_BITS_DATA) == a.data && + `MO(t, CACHEREQUEST_BITS_MASK) == a.mask && `MO(t, CACHEREQUEST_BITS_ATOMIC) + == a.atomic_op && `MO(t, CACHEREQUEST_BITS_ATOMICWORD) == a.word, + "Cache/PTE request mismatch"); + if (`MI(t, CACHEREQUEST_READY)) void'(transfers.pop_front()); + end + end + if (`MO(t, RESPONSE_VALID) && `MI(t, RESPONSE_READY)) begin + result_item result = result_item::type_id::create("actual"); + verify(pending && authorizations.size() == 0 && transfers.size() == 0, + "Early result before the planned accesses drained"); + result.tag = `MO(t, RESPONSE_BITS_TAG); + result.data = `MO(t, RESPONSE_BITS_DATA); + result.line = `MO(t, RESPONSE_BITS_LINE); + result.misaligned = `MO(t, RESPONSE_BITS_MISALIGNED); + result.page_fault = `MO(t, RESPONSE_BITS_PAGEFAULT); + result.access_fault = `MO(t, RESPONSE_BITS_ACCESSFAULT); + actual.write(result); + pending = 0; + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + verify(!pending && authorizations.size() == 0 && transfers.size() == 0, + "Virtual access did not drain"); + endfunction + function void final_phase(uvm_phase phase); + mmu_ref_destroy(walker); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/filelist.f b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/filelist.f new file mode 100644 index 000000000..d2b648311 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/VirtualAccess/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/interface.sv b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/interface.sv new file mode 100644 index 000000000..4cb146951 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/interface.sv @@ -0,0 +1,9 @@ +`include "virtual_parameters.svh" +interface virtual_if ( + input logic clock +); + logic reset = 1; + logic [`VA_INPUT_BITS-1:0] source = '0, backend = '0; + wire [ `VA_INPUT_BITS-1:0] inputs = source | backend; + wire [`VA_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/package.sv b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/package.sv new file mode 100644 index 000000000..cf65ee932 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/package.sv @@ -0,0 +1,106 @@ +`include "virtual_parameters.svh" +`define VI(F) vif.inputs[`VA_IN_``F``_OFFSET+:`VA_IN_``F``_WIDTH] +`define VO(F) vif.outputs[`VA_OUT_``F``_OFFSET+:`VA_OUT_``F``_WIDTH] +`define VB(F) vif.backend[`VA_IN_``F``_OFFSET+:`VA_IN_``F``_WIDTH] +`define VS(F) pins[`VA_IN_``F``_OFFSET+:`VA_IN_``F``_WIDTH] +`define MI(T, F) T.inputs[`VA_IN_``F``_OFFSET+:`VA_IN_``F``_WIDTH] +`define MO(T, F) T.outputs[`VA_OUT_``F``_OFFSET+:`VA_OUT_``F``_WIDTH] +package virtual_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function chandle mmu_ref_create( + input int unsigned address_bits, + entries + ); + import "DPI-C" function void mmu_ref_destroy(input chandle model); + import "DPI-C" function void mmu_ref_clear(input chandle model); + import "DPI-C" function void mmu_ref_program( + input chandle model, + input longint unsigned address, + value, + input int unsigned error + ); + import "DPI-C" function int unsigned mmu_ref_read( + input chandle model, + input longint unsigned address, + output longint unsigned value, + output int unsigned error + ); + import "DPI-C" function int unsigned mmu_ref_pending(input chandle model); + import "DPI-C" function int unsigned mmu_ref_peek( + input chandle model, + output longint unsigned address + ); + import "DPI-C" function int unsigned mmu_ref_consume( + input chandle model, + input longint unsigned address + ); + import "DPI-C" function int unsigned mmu_ref_translate( + input chandle model, + input longint unsigned address, + input int unsigned mode, + input longint unsigned root, + input int unsigned privilege, + wr, + execute, + sum, + mxr, + output longint unsigned paddr, + output int unsigned pf, + af, + level + ); + import "DPI-C" function int unsigned mmu_ref_lookup( + input chandle model, + input longint unsigned address, + input int unsigned mode, + input longint unsigned root, + input int unsigned privilege, + wr, + execute, + sum, + mxr, + output longint unsigned paddr + ); + import "DPI-C" function int unsigned mmu_ref_cached(input chandle model); + import "DPI-C" function void mmu_ref_result(input chandle model); + import "DPI-C" function void mmu_ref_reset(input chandle model); + import "DPI-C" function void mmu_ref_flush(input chandle model); + import "DPI-C" function void mem_access_ref_prepare( + input longint unsigned addr, + input int unsigned size, + write, + input longint unsigned data, + input int unsigned atomic_op, + cacheable, + normal, + address_bits, + output int unsigned target, + misaligned, + access_fault, + output longint unsigned bus_addr, + bus_data, + output int unsigned bus_mask, + atomic_word + ); + import "DPI-C" function longint unsigned mem_access_ref_result( + input longint unsigned addr, + input int unsigned size, + write, + signed_load, + atomic_op, + cacheable, + input longint unsigned raw, + input int unsigned error + ); + `include "agent/item.svh" + `include "agent/fixture.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/peer.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/seq/sequence.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/seq/sequence.svh new file mode 100644 index 000000000..c639ccce3 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/seq/sequence.svh @@ -0,0 +1,110 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + model reference; + logic [`VA_INPUT_BITS-1:0] pins; + logic [`VA_OUTPUT_BITS-1:0] observed; + int tag; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task transact(longint unsigned address, int size, bit writing = 0, bit signed_load = 0, + int atomic_op = 0, bit executing = 0, int privilege = 3, int mode = 0, bit sum = 0, + bit mxr = 0, int cancel_after = 0); + pins[`VA_IN_REQUEST_BITS_VADDR_OFFSET+:`VA_IN_REQUEST_BITS_VADDR_WIDTH] = address; + pins[`VA_IN_REQUEST_BITS_SIZE_OFFSET+:`VA_IN_REQUEST_BITS_SIZE_WIDTH] = size; + pins[`VA_IN_REQUEST_BITS_TAG_OFFSET+:`VA_IN_REQUEST_BITS_TAG_WIDTH] = tag++; + pins[`VA_IN_REQUEST_BITS_WRITE_OFFSET+:`VA_IN_REQUEST_BITS_WRITE_WIDTH] = writing; + pins[`VA_IN_REQUEST_BITS_SIGNED_OFFSET+:`VA_IN_REQUEST_BITS_SIGNED_WIDTH] = signed_load; + pins[`VA_IN_REQUEST_BITS_ATOMIC_OFFSET+:`VA_IN_REQUEST_BITS_ATOMIC_WIDTH] = atomic_op; + pins[`VA_IN_REQUEST_BITS_EXECUTE_OFFSET+:`VA_IN_REQUEST_BITS_EXECUTE_WIDTH] = executing; + pins[`VA_IN_REQUEST_BITS_DATA_OFFSET+:`VA_IN_REQUEST_BITS_DATA_WIDTH] = 64'hf1e2d3c4b5a69788; + pins[`VA_IN_REQUEST_BITS_PRIVILEGE_OFFSET+:`VA_IN_REQUEST_BITS_PRIVILEGE_WIDTH] = privilege; + pins[`VA_IN_REQUEST_BITS_SATPMODE_OFFSET+:`VA_IN_REQUEST_BITS_SATPMODE_WIDTH] = mode; + pins[`VA_IN_REQUEST_BITS_ROOTPPN_OFFSET+:`VA_IN_REQUEST_BITS_ROOTPPN_WIDTH] = 16; + pins[`VA_IN_REQUEST_BITS_SUM_OFFSET+:`VA_IN_REQUEST_BITS_SUM_WIDTH] = sum; + pins[`VA_IN_REQUEST_BITS_MXR_OFFSET+:`VA_IN_REQUEST_BITS_MXR_WIDTH] = mxr; + pins[`VA_IN_FLUSHTRANSLATION_OFFSET+:`VA_IN_FLUSHTRANSLATION_WIDTH] = 1; + tick(); + pins[`VA_IN_FLUSHTRANSLATION_OFFSET+:`VA_IN_FLUSHTRANSLATION_WIDTH] = 0; + pins[`VA_IN_REQUEST_VALID_OFFSET+:`VA_IN_REQUEST_VALID_WIDTH] = 1; + pins[`VA_IN_RESPONSE_READY_OFFSET+:`VA_IN_RESPONSE_READY_WIDTH] = 0; + do tick(); while (!observed[`VA_OUT_REQUEST_READY_OFFSET]); + pins[`VA_IN_REQUEST_VALID_OFFSET+:`VA_IN_REQUEST_VALID_WIDTH] = 0; + if (cancel_after > 0) begin + repeat (cancel_after) tick(); + repeat (2) tick(1); + tick(); + return; + end + do tick(); while (!observed[`VA_OUT_RESPONSE_VALID_OFFSET]); + repeat (4) tick(); + pins[`VA_IN_RESPONSE_READY_OFFSET+:`VA_IN_RESPONSE_READY_WIDTH] = 1; + tick(); + tick(); + endtask + task body(); + pins = '0; + pins[`VA_IN_ACTIVE_OFFSET+:`VA_IN_ACTIVE_WIDTH] = 1; + repeat (3) tick(1); + tick(); + for (int size = 0; size < 4; size++) + for (int offset = 0; offset < 16; offset++) + for (int direction = 0; direction < 2; direction++) begin + transact('h2000 + offset, size, direction, 0); + transact('h200000 + offset, size, direction, 1); + end + for (int atomic_op = 1; atomic_op <= 11; atomic_op++) begin + transact('h2000, 2, 0, 0, atomic_op); + transact('h2008, 3, 0, 0, atomic_op); + end + transact('h2000, 1, 0, 0, 0, 1); + transact('h200000, 1, 0, 0, 0, 1); + transact('h300000, 3); + transact('h300000, 3, 1); + transact('h500000, 3); + for (int kind = 0; kind < 2; kind++) begin + reference.image.deny_data = kind == 0; + reference.image.backend_error = kind == 1; + transact('h2000, 3); + transact('h200000, 3); + reference.image.deny_data = 0; + reference.image.backend_error = 0; + end + reference.set_pte('h10000, ('h11 << 10) | 1); + reference.set_pte('h11000, ('h12 << 10) | 1); + reference.set_pte('h12008, ('h2 << 10) | 'hcf); + for (int privilege = 0; privilege < 4; privilege++) + if (privilege != 2) begin + transact('h1000, 3, 0, 0, 0, 0, privilege, 8); + transact('h1000, 3, 1, 0, 0, 0, privilege, 8); + end + transact('h1002, 1, 0, 0, 0, 1, 1, 8); + transact(64'h4000000000, 3, 0, 0, 0, 0, 1, 8); + reference.set_pte('h12008, ('h2 << 10) | 'hdf); + transact('h1000, 3, 0, 0, 0, 0, 0, 8); + transact('h1000, 3, 0, 0, 0, 0, 1, 8, 1); + reference.set_pte('h12008, ('h2 << 10) | 'hc9); + transact('h1000, 3, 0, 0, 0, 0, 1, 8, 0, 1); + transact('h1000, 3, 0, 0, 0, 0, 1, 8); + reference.set_pte('h12008, 0); + transact('h1000, 3, 0, 0, 0, 0, 1, 8); + reference.set_pte('h12008, ('h2 << 10) | 'hcf, 1); + transact('h1000, 3, 0, 0, 0, 0, 1, 8); + reference.set_pte('h12008, ('h2 << 10) | 'hcf); + reference.image.deny_pte = 1; + transact('h1000, 3, 0, 0, 0, 0, 1, 8); + reference.image.deny_pte = 0; + transact('h2000, 3, 0, 0, 0, 0, 3, 0, 0, 0, 2); + transact('h2000, 3, 0, 0, 0, 0, 3, 0, 0, 0, 7); + transact('h2000, 3); + repeat (8) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/tb.sv b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/tb.sv new file mode 100644 index 000000000..cab5cca4f --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/tb.sv @@ -0,0 +1,15 @@ +module virtual_access_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + virtual_if bus (clock); + VirtualAccess dut ( + .clock(clock), + .reset(bus.reset), + `include "virtual_ports.svh" + ); + initial begin + uvm_config_db#(virtual virtual_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/tests/test.svh b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/tests/test.svh new file mode 100644 index 000000000..b11c898c9 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/resources/virtual/tests/test.svh @@ -0,0 +1,16 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.reference = test_env.reference; + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/Emit.scala new file mode 100644 index 000000000..06b563d60 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/Emit.scala @@ -0,0 +1,88 @@ +package framework.core.memory.access + +import chisel3._ +import chisel3.reflect.DataMirror +import java.nio.file.{Files, Paths} +import framework.core.memory.access.configs.Params +import framework.ip.bus.chi.rnf.{CacheAccess, CacheResult} + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val p = upickle.default.read[Params](Files.readString(Paths.get(config))) + Files.createDirectories(Paths.get(directory)) + + val defs = scala.collection.mutable.ArrayBuffer[String]( + s"`define CPU_PHYSICAL_BITS ${p.memory.chi.addressBits}", + s"`define CPU_TAG_BITS ${p.memory.tagBits}", + s"`define CPU_LINE_BITS ${p.lineBits}" + ) + + val ports = scala.collection.mutable.ArrayBuffer[String]() + + def channel( + kind: String, + prefix: String, + signal: String, + bundle: Bundle + ): Unit = { + var offset = 0 + for ((name, field) <- bundle.elements.toSeq) { + val width = field.getWidth + if (width > 0) { + defs += s"`define CPU_${kind}_${name.toUpperCase}_OFFSET $offset"; + defs += s"`define CPU_${kind}_${name.toUpperCase}_WIDTH $width"; + ports += s".${prefix}_bits_$name($signal.bits[$offset +: $width])"; offset += width + } + } + defs += s"`define ACCESS_${kind}_WIDTH $offset" + ports += s".${prefix}_valid($signal.valid)"; ports += s".${prefix}_ready($signal.ready)" + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new PhysicalAccess(p), + args = Array("--target-dir", s"$directory/PhysicalAccess", "--split-verilog") + ) + channel("REQ", "io_request", "request", new AccessRequest(p)) + channel("RESP", "io_response", "response", new AccessResponse(p)) + channel("CACHE", "io_cacheRequest", "cache_request", new CacheAccess(p.memory.chi)) + channel("CRESULT", "io_cacheResponse", "cache_response", new CacheResult(p.lineBits)) + channel("UNCACHED", "io_uncachedRequest", "uncached_request", new UncachedRequest(p.memory)) + channel("URESULT", "io_uncachedResponse", "uncached_response", new UncachedResponse(p.memory)) + Files.writeString(Paths.get(directory, "physical_parameters.svh"), defs.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, "physical_ports.svh"), ports.mkString(",\n") + "\n") + + val definitions = scala.collection.mutable.ArrayBuffer[String]( + s"`define VA_LINE_BITS ${p.lineBits}", + s"`define VA_ADDRESS_BITS ${p.memory.chi.addressBits}" + ) + + val binding = scala.collection.mutable.ArrayBuffer[String]() + var inputOffset = 0; var outputOffset = 0 + + def fields(data: Data, path: String): Seq[(String, Data)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => if (leaf.getWidth > 0) Seq(path -> leaf) else Seq.empty + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new VirtualAccess(p) + for ((name, field) <- fields(dut.io, "")) { + val input = DataMirror.directionOf(field) == ActualDirection.Input + val direction = if (input) "IN" else "OUT"; val signal = if (input) "inputs" else "outputs"; + val offset = if (input) inputOffset else outputOffset; val width = field.getWidth + definitions += s"`define VA_${direction}_${name.toUpperCase}_OFFSET $offset"; + definitions += s"`define VA_${direction}_${name.toUpperCase}_WIDTH $width" + binding += s".io_$name(bus.$signal[$offset +: $width])" + if (input) inputOffset += width else outputOffset += width + }; dut + }, + args = Array("--target-dir", s"$directory/VirtualAccess", "--split-verilog") + ) + definitions += s"`define VA_INPUT_BITS $inputOffset"; definitions += s"`define VA_OUTPUT_BITS $outputOffset" + Files.writeString(Paths.get(directory, "virtual_parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, "virtual_ports.svh"), binding.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/PhysicalAccess.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/PhysicalAccess.scala new file mode 100644 index 000000000..abb86362d --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/PhysicalAccess.scala @@ -0,0 +1,151 @@ +package framework.core.memory.access + +import framework.core.memory.access.configs.{Params => ModuleParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.ip.bus.chi.Params +import framework.ip.bus.chi.rnf.{CacheAccess, CacheAtomic, CacheResult} + +case class AccessParams(chi: Params = Params(), tagBits: Int = 6) { + require(chi.addressBits < 64 && tagBits > 0) +} + +class AccessRequest(b: ModuleParam) extends Bundle { + val p = b.memory + + val addr = UInt(64.W) + val tag = UInt(p.tagBits.W) + val size = UInt(3.W) + val write = Bool() + val signed = Bool() + val data = UInt(64.W) + val atomic = UInt(4.W) + val cacheable = Bool() + val normal = Bool() +} + +/** `line` is the whole cached line for full-line clients; it is absent by default. */ +class AccessResponse(b: ModuleParam) extends Bundle { + val p = b.memory + val lineBits = b.lineBits + + val tag = UInt(p.tagBits.W) + val data = UInt(64.W) + val misaligned = Bool() + val accessFault = Bool() + val line = UInt(lineBits.W) +} + +class UncachedRequest(p: AccessParams) extends Bundle { + + val addr = UInt(64.W) + val tag = UInt(p.tagBits.W) + val size = UInt(3.W) + val write = Bool() + val data = UInt(64.W) +} + +class UncachedResponse(p: AccessParams) extends Bundle { + + val tag = UInt(p.tagBits.W) + val data = UInt(64.W) + val error = Bool() +} + +@instantiable +class PhysicalAccess(b: ModuleParam) extends Module { + val p = b.memory + val lineBits = b.lineBits + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new AccessRequest(b))) + val response = Decoupled(new AccessResponse(b)) + + val cacheRequest = Decoupled(new CacheAccess(p.chi)) + + val cacheResponse = Flipped(Decoupled(new CacheResult(lineBits))) + val uncachedRequest = Decoupled(new UncachedRequest(b.memory)) + val uncachedResponse = Flipped(Decoupled(new UncachedResponse(b.memory))) + }) + + val idle :: issueCache :: waitCache :: issueUncached :: waitUncached :: respond :: Nil = Enum(6) + val state = RegInit(idle) + val command = Reg(new AccessRequest(b)) + val answer = Reg(new AccessResponse(b)) + + val mask = MuxLookup(command.size, 255.U(8.W))(Seq( + 0.U -> 1.U(8.W), + 1.U -> 3.U(8.W), + 2.U -> 15.U(8.W) + )) + + io.request.ready := state === idle + when(io.request.fire) { + val r = io.request.bits + assert(r.size <= 3.U, "Memory access size must be 1, 2, 4 or 8 bytes") + assert(r.atomic <= CacheAtomic.SC.U, "Memory access port does not accept Fence or unknown atomics") + assert( + r.atomic === CacheAtomic.None.U || (!r.write && r.size >= 2.U), + "Memory atomic requires a 32 or 64 bit operand" + ) + val misaligned = (r.addr & ((1.U(64.W) << r.size) - 1.U)).orR + val accessFault = r.addr(63, p.chi.addressBits).orR || (r.normal && !r.cacheable) || + (!r.normal && (r.cacheable || r.atomic =/= CacheAtomic.None.U)) + command := r + answer.tag := r.tag + answer.data := 0.U + if (lineBits > 0) answer.line := 0.U + answer.misaligned := misaligned + answer.accessFault := !misaligned && accessFault + state := Mux(misaligned || accessFault, respond, Mux(r.cacheable, issueCache, issueUncached)) + } + + val atomic = command.atomic =/= CacheAtomic.None.U + io.cacheRequest.valid := state === issueCache + io.cacheRequest.bits.addr := Mux(atomic, command.addr, command.addr & ~7.U(64.W))(p.chi.addressBits - 1, 0) + io.cacheRequest.bits.write := command.write + io.cacheRequest.bits.data := Mux(atomic, command.data, command.data << (command.addr(2, 0) << 3)) + io.cacheRequest.bits.mask := Mux(atomic, 255.U, Mux(command.write, mask << command.addr(2, 0), 0.U)) + io.cacheRequest.bits.atomic := command.atomic + io.cacheRequest.bits.atomicWord := atomic && command.size === 2.U + when(io.cacheRequest.fire)(state := waitCache) + io.cacheResponse.ready := state === waitCache + io.uncachedRequest.valid := state === issueUncached + io.uncachedRequest.bits.addr := command.addr + io.uncachedRequest.bits.tag := command.tag + io.uncachedRequest.bits.size := command.size + io.uncachedRequest.bits.write := command.write + io.uncachedRequest.bits.data := command.data + when(io.uncachedRequest.fire)(state := waitUncached) + io.uncachedResponse.ready := state === waitUncached + + val raw = + Mux(state === waitUncached, io.uncachedResponse.bits.data, io.cacheResponse.bits.data >> (command.addr(2, 0) << 3)) + + val load = MuxLookup(command.size, raw)(Seq( + 0.U -> Cat(Fill(56, command.signed && raw(7)), raw(7, 0)), + 1.U -> Cat(Fill(48, command.signed && raw(15)), raw(15, 0)), + 2.U -> Cat(Fill(32, command.signed && raw(31)), raw(31, 0)) + )) + + when(io.cacheResponse.fire || io.uncachedResponse.fire) { + val error = Mux(state === waitUncached, io.uncachedResponse.bits.error, io.cacheResponse.bits.error) + when(io.uncachedResponse.fire) { + assert(io.uncachedResponse.bits.tag === command.tag, "Uncached response tag mismatch") + } + answer.data := Mux( + error || command.write, + 0.U, + Mux(atomic, io.cacheResponse.bits.data, load) + ) + answer.accessFault := error + if (lineBits > 0) answer.line := Mux(state === waitCache && !error, io.cacheResponse.bits.line, 0.U) + state := respond + } + io.response.valid := state === respond + io.response.bits := answer + when(io.response.fire)(state := idle) +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/VirtualAccess.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/VirtualAccess.scala new file mode 100644 index 000000000..daed79033 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/VirtualAccess.scala @@ -0,0 +1,349 @@ +package framework.core.memory.access + +import buckyball.config.RegionHardware + +import framework.core.memory.access.configs.{Params => ModuleParam} +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.ip.bus.chi.rnf.{CacheAccess, CacheAtomic, CacheResult} +import framework.core.memory.access.mmu.Walker + +class VirtualPort(b: ModuleParam) extends Bundle { + val flushTranslation = Input(Bool()) + val request = Flipped(Decoupled(new VirtualRequest(b))) + val response = Decoupled(new VirtualResponse(b)) + val authorizationRequest = Decoupled(new PhysicalAuthorization) + val authorizationResponse = Flipped(Decoupled(Bool())) +} + +case class PhysicalRegion( + base: BigInt, + bytes: BigInt, + cacheable: Boolean, + executable: Boolean, + readable: Boolean, + writable: Boolean, + atomic: Boolean, + normal: Boolean) + +object PhysicalRegion { + + def fromProto(p: RegionHardware): PhysicalRegion = PhysicalRegion( + BigInt(java.lang.Long.toUnsignedString(p.getBase)), + BigInt(java.lang.Long.toUnsignedString(p.getBytes)), + p.getCacheable, + p.getExecutable, + p.getReadable, + p.getWritable, + p.getAtomic, + p.getNormal + ) + +} + +class PhysicalAuthorization extends Bundle { + val paddr = UInt(64.W) + val size = UInt(3.W) + val read = Bool() + val write = Bool() + val execute = Bool() + val isPte = Bool() + val privilege = UInt(2.W) +} + +class VirtualRequest(b: ModuleParam) extends Bundle { + val p = b.memory + + val vaddr = UInt(64.W) + val tag = UInt(p.tagBits.W) + val size = UInt(3.W) + val write = Bool() + val execute = Bool() + val signed = Bool() + val data = UInt(64.W) + val atomic = UInt(4.W) + val privilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + val satpMode = UInt(4.W) + val rootPpn = UInt(44.W) +} + +class VirtualResponse(b: ModuleParam) extends Bundle { + val p = b.memory + val lineBits = b.lineBits + + val tag = UInt(p.tagBits.W) + val data = UInt(64.W) + val misaligned = Bool() + val pageFault = Bool() + val accessFault = Bool() + // Whole cached line of the translated word, for full-line clients only. + val line = UInt(lineBits.W) +} + +/** Single-outstanding virtual data/instruction-word frontend; cache ownership is external. */ +@instantiable +class VirtualAccess(b: ModuleParam) extends Module { + val cp = b.memory + val regions = b.regions + val lineBits = b.lineBits + val translationEntries = b.translationEntries + + require(regions.nonEmpty) + for ((region, i) <- regions.zipWithIndex) { + require(region.base >= 0 && region.bytes > 0 && region.base + region.bytes <= (BigInt(1) << cp.chi.addressBits)) + require(!region.cacheable || (region.base % 64 == 0 && region.bytes % 64 == 0)) + require( + region.normal || (!region.cacheable && !region.executable && !region.atomic), + "Device regions cannot be cached, executable or atomic" + ) + for (other <- regions.drop(i + 1)) { + require(region.base + region.bytes <= other.base || other.base + other.bytes <= region.base) + } + } + + @public + val io = IO(new Bundle { + val active = Input(Bool()) + // SFENCE.VMA: drop every cached translation. + val flushTranslation = Input(Bool()) + + // Combinational translation through the TLB only, for a caller's fast path. + val lookup = new Bundle { + val config = Input(new framework.core.memory.access.mmu.Config) + val req = Input(new framework.core.memory.access.mmu.Request) + val hit = Output(Bool()) + val paddr = Output(UInt(cp.chi.addressBits.W)) + } + + val request = Flipped(Decoupled(new VirtualRequest(b))) + val response = Decoupled(new VirtualResponse(b)) + val authorizationRequest = Decoupled(new PhysicalAuthorization) + val authorizationResponse = Flipped(Decoupled(Bool())) + + val cacheRequest = Decoupled(new CacheAccess(cp.chi)) + + val cacheResponse = Flipped(Decoupled(new CacheResult(lineBits))) + val uncachedRequest = Decoupled(new UncachedRequest(b.memory)) + val uncachedResponse = Flipped(Decoupled(new UncachedResponse(b.memory))) + + val observedTranslation = + Output(Valid(new framework.core.memory.access.mmu.Response(b.walker))) + + val observedPhysical = Output(Valid(new AccessRequest(b))) + + val observedCache = Output(Valid(new Bundle { + val pte = Bool() + + val access = new CacheAccess(cp.chi) + + })) + + }) + + val walker: Instance[Walker] = Instantiate(new Walker(b.walker)) + + val physical: Instance[PhysicalAccess] = Instantiate(new PhysicalAccess(b)) + + val idle :: startWalk :: waitWalk :: authorize :: waitAuthorization :: startAccess :: waitAccess :: respond :: Nil = + Enum(8) + val state = RegInit(idle) + val command = Reg(new VirtualRequest(b)) + val answer = Reg(new VirtualResponse(b)) + val paddr = Reg(UInt(64.W)) + val cacheable = Reg(Bool()) + val normal = Reg(Bool()) + val pteInFlight = RegInit(false.B) + val pteError = RegInit(false.B) + val pteAuthorized = RegInit(false.B) + val pteAuthorizationPending = RegInit(false.B) + val regionAllowed = Reg(Bool()) + def matches(first: UInt, last: UInt): Seq[Bool] = + regions.map(region => first >= region.base.U(65.W) && last < (region.base + region.bytes).U(65.W)) + io.request.ready := state === idle && io.active + when(io.request.fire) { + val r = io.request.bits + assert(r.size <= 3.U, "Virtual LSU size must be 1, 2, 4 or 8 bytes") + assert(r.atomic <= CacheAtomic.SC.U, "Virtual LSU does not accept Fence or unknown atomics") + assert( + r.atomic === CacheAtomic.None.U || (!r.write && r.size >= 2.U), + "Virtual LSU atomic requires a 32 or 64 bit operand" + ) + assert(r.satpMode === 0.U || r.satpMode === 8.U, "Virtual LSU requires Bare or Sv39 mode") + assert( + r.privilege === 0.U || r.privilege === 1.U || r.privilege === 3.U, + "Virtual LSU effective privilege must be U, S or M" + ) + assert( + !r.execute || (!r.write && r.atomic === CacheAtomic.None.U && r.size >= 1.U && !r.signed), + "Virtual instruction fetch requires unsigned 16, 32 or 64 bit read without atomic" + ) + command := r + answer := 0.U.asTypeOf(answer) + answer.tag := r.tag + val misaligned = (r.vaddr & ((1.U(64.W) << r.size) - 1.U)).orR + answer.misaligned := misaligned + state := Mux(misaligned, respond, startWalk) + } + + walker.io.flush := io.flushTranslation + walker.io.lookup.config := io.lookup.config + walker.io.lookup.req := io.lookup.req + io.lookup.hit := walker.io.lookup.hit + io.lookup.paddr := walker.io.lookup.paddr + walker.io.config.mode := command.satpMode + walker.io.config.rootPpn := command.rootPpn + walker.io.req.valid := state === startWalk + walker.io.req.bits.vaddr := command.vaddr + walker.io.req.bits.write := command.write || + (command.atomic =/= CacheAtomic.None.U && command.atomic =/= CacheAtomic.LR.U) + walker.io.req.bits.execute := command.execute + walker.io.req.bits.privilege := command.privilege + walker.io.req.bits.sum := command.sum + walker.io.req.bits.mxr := command.mxr + when(walker.io.req.fire)(state := waitWalk) + walker.io.resp.ready := state === waitWalk + + val lastByte = walker.io.resp.bits.paddr.pad(65) + MuxLookup(command.size, 0.U(4.W))( + Seq(1.U -> 1.U(4.W), 2.U -> 3.U(4.W), 3.U -> 7.U(4.W)) + ) + + val dataRegions = matches(walker.io.resp.bits.paddr.pad(65), lastByte) + val isAtomic = command.atomic =/= CacheAtomic.None.U + val readsData = !command.execute && !command.write && command.atomic =/= CacheAtomic.SC.U + val writesData = command.write || (isAtomic && command.atomic =/= CacheAtomic.LR.U) + + val permittedRegion = dataRegions.zip(regions).map { case (hit, region) => + hit && (!region.normal || region.cacheable).B && (!command.execute || (region.normal && region.executable).B) && + (!readsData || region.readable.B) && (!writesData || region.writable.B) && + (!isAtomic || region.atomic.B) + }.reduce(_ || _) + + when(walker.io.resp.fire) { + when(walker.io.resp.bits.pageFault || walker.io.resp.bits.accessFault) { + answer.pageFault := walker.io.resp.bits.pageFault + answer.accessFault := walker.io.resp.bits.accessFault + state := respond + }.otherwise { + paddr := walker.io.resp.bits.paddr + cacheable := dataRegions.zip(regions).map { case (hit, region) => hit && region.cacheable.B }.reduce(_ || _) + normal := dataRegions.zip(regions).map { case (hit, region) => hit && region.normal.B }.reduce(_ || _) + regionAllowed := permittedRegion + state := authorize + } + } + + physical.io.request.valid := state === startAccess + physical.io.request.bits.addr := paddr + physical.io.request.bits.tag := command.tag + physical.io.request.bits.size := command.size + physical.io.request.bits.write := command.write + physical.io.request.bits.signed := command.signed + physical.io.request.bits.data := command.data + physical.io.request.bits.atomic := command.atomic + physical.io.request.bits.cacheable := cacheable + physical.io.request.bits.normal := normal + when(physical.io.request.fire)(state := waitAccess) + physical.io.response.ready := state === waitAccess + when(physical.io.response.fire) { + assert(physical.io.response.bits.tag === command.tag, "Virtual LSU physical response tag mismatch") + answer.data := physical.io.response.bits.data + answer.misaligned := physical.io.response.bits.misaligned + answer.accessFault := physical.io.response.bits.accessFault + if (lineBits > 0) answer.line := physical.io.response.bits.line + state := respond + } + + val walking = state === startWalk || state === waitWalk + val accessing = state === startAccess || state === waitAccess + val pteFirst = walker.io.access.bits.addr.pad(65) + val pteRegions = matches(pteFirst, pteFirst + 7.U) + + val pteAllowed = pteRegions.zip(regions).map { case (hit, region) => + hit && (region.normal && region.readable && region.cacheable).B + }.reduce(_ || _) + + // The Thread owns a per-command PMP snapshot. Each permission decision is a + // single-outstanding handshake; neither a stalled query nor its response + // allows speculative cache or MMIO traffic. PTE accesses use S privilege. + io.authorizationRequest.valid := state === authorize || + (walking && walker.io.access.valid && !pteAuthorized && !pteAuthorizationPending && !pteError) + io.authorizationRequest.bits.paddr := Mux(walking, walker.io.access.bits.addr.pad(64), paddr) + io.authorizationRequest.bits.size := Mux(walking, 3.U, command.size) + io.authorizationRequest.bits.read := walking || readsData + io.authorizationRequest.bits.write := !walking && writesData + io.authorizationRequest.bits.execute := !walking && command.execute + io.authorizationRequest.bits.isPte := walking + io.authorizationRequest.bits.privilege := Mux(walking, 1.U, command.privilege) + when(io.authorizationRequest.fire) { + when(walking) { + pteAuthorizationPending := true.B + } + .otherwise(state := waitAuthorization) + } + io.authorizationResponse.ready := pteAuthorizationPending || state === waitAuthorization + when(io.authorizationResponse.fire) { + when(pteAuthorizationPending) { + pteAuthorizationPending := false.B + pteAuthorized := true.B + pteError := !io.authorizationResponse.bits || !pteAllowed + }.otherwise { + when(io.authorizationResponse.bits && regionAllowed)(state := startAccess) + .otherwise { answer.accessFault := true.B; state := respond } + } + } + io.cacheRequest.valid := Mux( + walking, + walker.io.access.valid && pteAuthorized && !pteError, + accessing && physical.io.cacheRequest.valid + ) + io.cacheRequest.bits := physical.io.cacheRequest.bits + when(walking) { + io.cacheRequest.bits := 0.U.asTypeOf(io.cacheRequest.bits) + io.cacheRequest.bits.addr := walker.io.access.bits.addr + } + io.uncachedRequest.valid := accessing && physical.io.uncachedRequest.valid + io.uncachedRequest.bits := physical.io.uncachedRequest.bits + walker.io.access.ready := walking && pteAuthorized && + (pteError || io.cacheRequest.ready) + when(walker.io.access.fire) { + pteAuthorized := false.B + pteInFlight := !pteError + } + walker.io.result.valid := walking && ((!pteAuthorized && pteError) || (pteInFlight && io.cacheResponse.valid)) + walker.io.result.bits := 0.U.asTypeOf(walker.io.result.bits) + walker.io.result.bits.data := Mux(pteError || io.cacheResponse.bits.error, 0.U, io.cacheResponse.bits.data) + walker.io.result.bits.error := pteError || io.cacheResponse.bits.error + when(walker.io.result.fire) { pteError := false.B; pteInFlight := false.B } + physical.io.cacheRequest.ready := accessing && io.cacheRequest.ready + physical.io.cacheResponse.valid := accessing && io.cacheResponse.valid + physical.io.cacheResponse.bits := io.cacheResponse.bits + physical.io.uncachedRequest.ready := accessing && io.uncachedRequest.ready + physical.io.uncachedResponse.valid := accessing && io.uncachedResponse.valid + physical.io.uncachedResponse.bits := io.uncachedResponse.bits + io.cacheResponse.ready := Mux( + walking, + pteInFlight && walker.io.result.ready, + accessing && physical.io.cacheResponse.ready + ) + io.uncachedResponse.ready := accessing && physical.io.uncachedResponse.ready + when(io.cacheResponse.valid) { + assert((walking && pteInFlight) || accessing, "Virtual LSU cache result has no owner") + } + when(io.uncachedResponse.valid) { + assert(accessing, "Virtual LSU uncached result has no owner") + } + + io.response.valid := state === respond + io.response.bits := answer + when(io.response.fire)(state := idle) + io.observedTranslation.valid := walker.io.resp.fire + io.observedTranslation.bits := walker.io.resp.bits + io.observedPhysical.valid := physical.io.request.fire + io.observedPhysical.bits := physical.io.request.bits + io.observedCache.valid := io.cacheRequest.fire + io.observedCache.bits.pte := walking + io.observedCache.bits.access := io.cacheRequest.bits +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/build.sc b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/build.sc new file mode 100644 index 000000000..d4bc46a7a --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.memory.access.cache.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Cache.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Maintenance.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/configs/uvm.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/configs/uvm.toml new file mode 100644 index 000000000..87cf84120 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/configs/uvm.toml @@ -0,0 +1,19 @@ +[parameters.cache] +nodeId=1 +cacheLines=8 +homeId=64 +homeCount=1 +banks=2 +lineResult=false +probe=false +[parameters.cache.chi] +nodeIdBits=7 +addressBits=44 +dataBits=256 +[parameters.tracking] +entries=4 +addressBits=44 +idBits=8 +lineBytes=64 +maxRanges=8 +[defines] diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/agent/cmo_if.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/agent/cmo_if.sv new file mode 100644 index 000000000..b0d144c6d --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/agent/cmo_if.sv @@ -0,0 +1,19 @@ +`include "parameters.svh" +interface cmo_if ( + input logic clock, + input logic reset +); + logic valid = 0, ready, response_valid, response_ready = 1, ok; + logic [`CMO_ID_BITS-1:0] tag = 0, response_tag; + logic [`CHI_ADDRESS_BITS-1:0] first_line = 0, last_line = 0; + logic [1:0] operation = 0; + logic older_drained = 1; + assert property (@(posedge clock) disable iff (reset) valid && !ready |=> valid && $stable( + {tag, first_line, last_line, operation} + )) + else $fatal(1, "Maintenance request changed under backpressure"); + assert property(@(posedge clock) disable iff(reset)response_valid&&!response_ready|=>response_valid&&$stable( + {response_tag, ok} + )) + else $fatal(1, "Maintenance response changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/env/model.svh new file mode 100644 index 000000000..4e50ae22f --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/env/model.svh @@ -0,0 +1,116 @@ +class cmo_result extends uvm_sequence_item; + `uvm_object_utils_begin(cmo_result) + `uvm_field_int(tag, UVM_ALL_ON) + `uvm_field_int(ok, UVM_ALL_ON) + `uvm_object_utils_end + int tag; + bit ok; + function new(string name = "result"); + super.new(name); + endfunction +endclass +class cache_model extends protocol_model; + `uvm_component_utils(cache_model) + virtual cmo_if cmo; + result_board #(cmo_result, 0) acknowledgments; + bit maintenance_pending; + longint unsigned next_line, last_line; + int operation, maintenance_tag, issued, returned, fail_index = -1; + covergroup maintenance with function sample (int op, bit ok, bit stalled, bit overlap); + option.per_instance = 1; + operations: coverpoint op {bins all[] = {[0 : 2]};} + errors: coverpoint ok; + backpressure: coverpoint stalled; + concurrency: coverpoint overlap; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + maintenance = new(); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual cmo_if)::get(this, "", "cmo", cmo)) + `uvm_fatal("VIF", "Cache maintenance interface is required") + acknowledgments = result_board#(cmo_result, 0)::type_id::create("maintenance_results", this); + endfunction + function void write(sample_item t); + sample_item demand; + rsp_t response, received_rsp; + req_t request; + demand = sample_item::type_id::create("demand"); + demand.sampled = t.sampled; + if (t.sampled.reset) begin + acknowledgments.cancel_pending(); + maintenance_pending = 0; + issued = 0; + returned = 0; + end else begin + if (cmo.valid && cmo.ready) begin + cmo_result result = cmo_result::type_id::create("expected"); + if (maintenance_pending) `uvm_fatal("CMO", "Overlapping maintenance admissions") + maintenance_pending = 1; + next_line = cmo.first_line; + last_line = cmo.last_line; + operation = cmo.operation; + maintenance_tag = cmo.tag; + issued = 0; + returned = 0; + result.tag = maintenance_tag; + result.ok = fail_index < 0 || fail_index > int'((last_line - next_line) / 64); + acknowledgments.write_expected(result); + evidence::check(cmo.older_drained && t.sampled.outstanding == 0 && !t.sampled.access_ready, + "Maintenance admitted before drain or alongside a CPU request"); + end + if (t.sampled.req_valid && t.sampled.req_ready) begin + request = req_t'(t.sampled.req); + if (request.txn == `CHI_BANKS) begin + int opcode=operation==0?`CMO_CLEAN:operation==1?`CMO_CLEAN_INVALID:`CMO_INVALID; + evidence::check( + maintenance_pending&&request.addr==next_line&&request.opcode==opcode&&request.src==1&&request.tgt==64&&request.size==6&&request.snp_attr&&request.mem_attr==12&&request.allow_retry, + "Incorrect maintenance request footprint or attributes"); + response = '0; + response.src = 64; + response.tgt = 1; + response.txn = `CHI_BANKS; + response.opcode = `CMO_COMP; + response.error = issued == fail_index ? 2 : 0; + if (scenario == "bad_maintenance_identity") response.src = 65; + if (scenario == "bad_maintenance_opcode") response.opcode = 1; + rx_rsp_queue.push_back(response); + issued++; + demand.sampled.req_valid = 0; + end + end + received_rsp = rsp_t'(t.sampled.rx_rsp); + if (t.sampled.rx_rsp_valid && received_rsp.txn == `CHI_BANKS) begin + response = rsp_t'(t.sampled.rx_rsp); + if (t.sampled.rx_rsp_ready) begin + rsp_active = 0; + returned++; + if (response.error == 0) next_line += 64; + end + demand.sampled.rx_rsp_valid = 0; + demand.sampled.rx_rsp_ready = 0; + end + if (cmo.response_valid && cmo.response_ready) begin + cmo_result result = cmo_result::type_id::create("actual"); + evidence::check( + maintenance_pending && issued == returned && (next_line > last_line || fail_index >= 0), + "Maintenance acknowledged before all completions"); + result.tag = cmo.response_tag; + result.ok = cmo.ok; + acknowledgments.write_actual(result); + maintenance_pending = 0; + maintenance.sample(operation, cmo.ok, 0, t.sampled.outstanding > 0); + end + if (cmo.valid && !cmo.ready || cmo.response_valid && !cmo.response_ready) + maintenance.sample(cmo.operation, 1, 1, t.sampled.outstanding > 0); + if (maintenance_pending && t.sampled.access_valid) maintenance.sample(operation, 1, 0, 1); + end + super.write(demand); + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + if (maintenance_pending) `uvm_fatal("CMO", "Maintenance did not drain") + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/filelist.f new file mode 100644 index 000000000..1591e158c --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/filelist.f @@ -0,0 +1,12 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/protocol/chi ++incdir+@VERIFY@/uvm/src/protocol/chi/rnf +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/chi/rnf/types.svh +@VERIFY@/uvm/src/protocol/chi/rnf/interface.sv +@RESOURCES@/agent/cmo_if.sv +@RESOURCES@/package.sv +-F @RTL@/Cache/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/package.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/package.sv new file mode 100644 index 000000000..c47e402be --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/package.sv @@ -0,0 +1,6 @@ +package core_cache_pkg; + `include "rnf/body.svh" + `include "env/model.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..3039f1e0f --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/seq/sequence.svh @@ -0,0 +1,72 @@ +class cache_sequence extends scenario_sequence; + `uvm_object_utils(cache_sequence) + virtual cmo_if cmo; + cache_model checked_state; + function new(string name = "scenario"); + super.new(name); + endfunction + task operation(int op, int lines, int tag, int failure = -1); + longint unsigned checked_before = checked_state.acknowledgments.checked; + `uvm_info("CACHE_CMO", + $sformatf("op=%0d lines=%0d tag=%0d error=%0d", op, lines, tag, failure), UVM_HIGH) + checked_state.fail_index = failure; + @(negedge vif.clock); + cmo.valid = 1; + cmo.first_line = 'h40000; + cmo.last_line = 'h40000 + (lines - 1) * 64; + cmo.operation = op; + cmo.tag = tag; + cmo.response_ready = 0; + do @(posedge vif.clock); while (!cmo.ready); + @(negedge vif.clock); + cmo.valid = 0; + wait (cmo.response_valid); + repeat (4) @(negedge vif.clock); + cmo.response_ready = 1; + wait (checked_state.acknowledgments.checked == checked_before + 1); + @(negedge vif.clock); + endtask + task body(); + `uvm_info("CACHE_STAGE", "Starting demand regression", UVM_HIGH) + super.body(); + `uvm_info("CACHE_STAGE", "Starting maintenance regression", UVM_HIGH) + for (int op = 0; op < 3; op++) begin + cmo.older_drained = 0; + fork + operation(op, 1 + op, 20 + op); + begin + repeat (8) @(negedge vif.clock); + evidence::check(!cmo.ready, "Maintenance ignored an older undrained request"); + cmo.older_drained = 1; + end + join + operation(op, 3, 40 + op, op); + end + // An independent demand request progresses while a serialized CMO is outstanding. + state.hold_req = 1; + fork + operation(0, 4, 60); + begin + repeat (6) @(negedge vif.clock); + issue(16320); + state.hold_req = 0; + settle(); + end + join + state.hold_req = 1; + @(negedge vif.clock); + cmo.valid = 1; + cmo.first_line = 'h50000; + cmo.last_line = 'h50100; + cmo.operation = 2; + cmo.tag = 200; + do @(posedge vif.clock); while (!cmo.ready); + @(negedge vif.clock); + cmo.valid = 0; + repeat (3) @(negedge vif.clock); + test_env.reset.pulse(3); + state.hold_req = 0; + repeat (4) @(negedge vif.clock); + operation(2, 2, 201); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/suite.toml new file mode 100644 index 000000000..23b2331f7 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/suite.toml @@ -0,0 +1,105 @@ +[model] +manifest="../../../../../../../../../../../../verify/uvm/src/protocol/chi/rnf/model/Cargo.toml" +library="librnf_reference_model" + +[[tests]] +name="normal" +top="core_cache_tb" +test="cache_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["core_cache_pkg::protocol_model::behavior","core_cache_pkg::cache_model::maintenance"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=normal"] +assertion="" + +[[tests]] +name="bad_maintenance_alignment" +top="core_cache_tb" +test="cache_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=bad_maintenance_alignment"] +assertion="Maintenance endpoints must be cache-line aligned" + +[[tests]] +name="bad_maintenance_range" +top="core_cache_tb" +test="cache_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=bad_maintenance_range"] +assertion="Maintenance range is reversed" + +[[tests]] +name="bad_maintenance_operation" +top="core_cache_tb" +test="cache_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=bad_maintenance_operation"] +assertion="Unknown maintenance operation" + +[[tests]] +name="bad_maintenance_identity" +top="core_cache_tb" +test="cache_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=bad_maintenance_identity"] +assertion="Maintenance completion identity mismatch" + +[[tests]] +name="bad_maintenance_opcode" +top="core_cache_tb" +test="cache_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=bad_maintenance_opcode"] +assertion="Maintenance requires a completion response" diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/tb.sv new file mode 100644 index 000000000..7a81925a3 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/tb.sv @@ -0,0 +1,169 @@ +module core_cache_tb; + import uvm_pkg::*; + import core_cache_pkg::*; + logic clock = 0; + clock_if timing (clock); + always #5 clock = ~clock; + chi_rnf_if vif (clock); + cmo_if cmo ( + clock, + vif.reset + ); + Cache dut ( + .clock(clock), + .reset(vif.reset), + .io_chi_req_bits_qos(vif.req[0+:4]), + .io_chi_req_bits_tgtId(vif.req[4+:7]), + .io_chi_req_bits_srcId(vif.req[11+:7]), + .io_chi_req_bits_txnId(vif.req[18+:12]), + .io_chi_req_bits_returnNid(vif.req[30+:7]), + .io_chi_req_bits_stashNidValidEndian(vif.req[37+:1]), + .io_chi_req_bits_returnTxnId(vif.req[38+:12]), + .io_chi_req_bits_opcode(vif.req[50+:7]), + .io_chi_req_bits_multiReq(vif.req[57+:1]), + .io_chi_req_bits_size(vif.req[58+:6]), + .io_chi_req_bits_addr(vif.req[64+:44]), + .io_chi_req_bits_pas(vif.req[108+:3]), + .io_chi_req_bits_likelyShared(vif.req[111+:1]), + .io_chi_req_bits_allowRetry(vif.req[112+:1]), + .io_chi_req_bits_order(vif.req[113+:2]), + .io_chi_req_bits_pCrdType(vif.req[115+:4]), + .io_chi_req_bits_memAttr(vif.req[119+:4]), + .io_chi_req_bits_snpAttr(vif.req[123+:1]), + .io_chi_req_bits_lpid(vif.req[124+:8]), + .io_chi_req_bits_exclSnoopMe(vif.req[132+:1]), + .io_chi_req_bits_expCompAck(vif.req[133+:1]), + .io_chi_req_bits_tagOp(vif.req[134+:2]), + .io_chi_req_bits_traceTag(vif.req[136+:1]), + .io_chi_req_valid(vif.req_valid), + .io_chi_req_ready(vif.req_ready), + .io_chi_txRsp_bits_qos(vif.rsp[0+:4]), + .io_chi_txRsp_bits_tgtId(vif.rsp[4+:7]), + .io_chi_txRsp_bits_srcId(vif.rsp[11+:7]), + .io_chi_txRsp_bits_txnId(vif.rsp[18+:12]), + .io_chi_txRsp_bits_opcode(vif.rsp[30+:5]), + .io_chi_txRsp_bits_respErr(vif.rsp[35+:2]), + .io_chi_txRsp_bits_resp(vif.rsp[37+:3]), + .io_chi_txRsp_bits_fwdState(vif.rsp[40+:3]), + .io_chi_txRsp_bits_cBusy(vif.rsp[43+:3]), + .io_chi_txRsp_bits_dbid(vif.rsp[46+:12]), + .io_chi_txRsp_bits_pCrdType(vif.rsp[58+:4]), + .io_chi_txRsp_bits_tagOp(vif.rsp[62+:2]), + .io_chi_txRsp_bits_traceTag(vif.rsp[64+:1]), + .io_chi_txRsp_bits_cacheLineId(vif.rsp[65+:6]), + .io_chi_txRsp_valid(vif.rsp_valid), + .io_chi_txRsp_ready(vif.rsp_ready), + .io_chi_txDat_bits_qos(vif.dat[0+:4]), + .io_chi_txDat_bits_tgtId(vif.dat[4+:7]), + .io_chi_txDat_bits_srcId(vif.dat[11+:7]), + .io_chi_txDat_bits_txnId(vif.dat[18+:12]), + .io_chi_txDat_bits_homeNid(vif.dat[30+:7]), + .io_chi_txDat_bits_opcode(vif.dat[37+:4]), + .io_chi_txDat_bits_respErr(vif.dat[41+:2]), + .io_chi_txDat_bits_resp(vif.dat[43+:3]), + .io_chi_txDat_bits_dataSource(vif.dat[46+:8]), + .io_chi_txDat_bits_dataPull(vif.dat[54+:1]), + .io_chi_txDat_bits_cBusy(vif.dat[55+:3]), + .io_chi_txDat_bits_dbid(vif.dat[58+:16]), + .io_chi_txDat_bits_ccid(vif.dat[74+:2]), + .io_chi_txDat_bits_dataId(vif.dat[76+:2]), + .io_chi_txDat_bits_cacheLineId(vif.dat[78+:6]), + .io_chi_txDat_bits_tagOp(vif.dat[84+:2]), + .io_chi_txDat_bits_tag(vif.dat[86+:8]), + .io_chi_txDat_bits_tagUpdate(vif.dat[94+:2]), + .io_chi_txDat_bits_traceTag(vif.dat[96+:1]), + .io_chi_txDat_bits_copyAtHome(vif.dat[97+:1]), + .io_chi_txDat_bits_numDat(vif.dat[98+:2]), + .io_chi_txDat_bits_replicate(vif.dat[100+:1]), + .io_chi_txDat_bits_be(vif.dat[101+:32]), + .io_chi_txDat_bits_data(vif.dat[133+:256]), + .io_chi_txDat_valid(vif.dat_valid), + .io_chi_txDat_ready(vif.dat_ready), + .io_chi_rxRsp_bits_qos(vif.rx_rsp[0+:4]), + .io_chi_rxRsp_bits_tgtId(vif.rx_rsp[4+:7]), + .io_chi_rxRsp_bits_srcId(vif.rx_rsp[11+:7]), + .io_chi_rxRsp_bits_txnId(vif.rx_rsp[18+:12]), + .io_chi_rxRsp_bits_opcode(vif.rx_rsp[30+:5]), + .io_chi_rxRsp_bits_respErr(vif.rx_rsp[35+:2]), + .io_chi_rxRsp_bits_resp(vif.rx_rsp[37+:3]), + .io_chi_rxRsp_bits_fwdState(vif.rx_rsp[40+:3]), + .io_chi_rxRsp_bits_cBusy(vif.rx_rsp[43+:3]), + .io_chi_rxRsp_bits_dbid(vif.rx_rsp[46+:12]), + .io_chi_rxRsp_bits_pCrdType(vif.rx_rsp[58+:4]), + .io_chi_rxRsp_bits_tagOp(vif.rx_rsp[62+:2]), + .io_chi_rxRsp_bits_traceTag(vif.rx_rsp[64+:1]), + .io_chi_rxRsp_bits_cacheLineId(vif.rx_rsp[65+:6]), + .io_chi_rxRsp_valid(vif.rx_rsp_valid), + .io_chi_rxRsp_ready(vif.rx_rsp_ready), + .io_chi_rxDat_bits_qos(vif.rx_dat[0+:4]), + .io_chi_rxDat_bits_tgtId(vif.rx_dat[4+:7]), + .io_chi_rxDat_bits_srcId(vif.rx_dat[11+:7]), + .io_chi_rxDat_bits_txnId(vif.rx_dat[18+:12]), + .io_chi_rxDat_bits_homeNid(vif.rx_dat[30+:7]), + .io_chi_rxDat_bits_opcode(vif.rx_dat[37+:4]), + .io_chi_rxDat_bits_respErr(vif.rx_dat[41+:2]), + .io_chi_rxDat_bits_resp(vif.rx_dat[43+:3]), + .io_chi_rxDat_bits_dataSource(vif.rx_dat[46+:8]), + .io_chi_rxDat_bits_dataPull(vif.rx_dat[54+:1]), + .io_chi_rxDat_bits_cBusy(vif.rx_dat[55+:3]), + .io_chi_rxDat_bits_dbid(vif.rx_dat[58+:16]), + .io_chi_rxDat_bits_ccid(vif.rx_dat[74+:2]), + .io_chi_rxDat_bits_dataId(vif.rx_dat[76+:2]), + .io_chi_rxDat_bits_cacheLineId(vif.rx_dat[78+:6]), + .io_chi_rxDat_bits_tagOp(vif.rx_dat[84+:2]), + .io_chi_rxDat_bits_tag(vif.rx_dat[86+:8]), + .io_chi_rxDat_bits_tagUpdate(vif.rx_dat[94+:2]), + .io_chi_rxDat_bits_traceTag(vif.rx_dat[96+:1]), + .io_chi_rxDat_bits_copyAtHome(vif.rx_dat[97+:1]), + .io_chi_rxDat_bits_numDat(vif.rx_dat[98+:2]), + .io_chi_rxDat_bits_replicate(vif.rx_dat[100+:1]), + .io_chi_rxDat_bits_be(vif.rx_dat[101+:32]), + .io_chi_rxDat_bits_data(vif.rx_dat[133+:256]), + .io_chi_rxDat_valid(vif.rx_dat_valid), + .io_chi_rxDat_ready(vif.rx_dat_ready), + .io_chi_snp_bits_qos(vif.snp[0+:4]), + .io_chi_snp_bits_srcId(vif.snp[4+:7]), + .io_chi_snp_bits_txnId(vif.snp[11+:12]), + .io_chi_snp_bits_fwdNid(vif.snp[23+:7]), + .io_chi_snp_bits_fwdTxnId(vif.snp[30+:12]), + .io_chi_snp_bits_opcode(vif.snp[42+:5]), + .io_chi_snp_bits_addr(vif.snp[47+:41]), + .io_chi_snp_bits_pas(vif.snp[88+:3]), + .io_chi_snp_bits_doNotGoToSd(vif.snp[91+:1]), + .io_chi_snp_bits_retToSrc(vif.snp[92+:1]), + .io_chi_snp_bits_traceTag(vif.snp[93+:1]), + .io_chi_snp_valid(vif.snp_valid), + .io_chi_snp_ready(vif.snp_ready), + .io_access_valid(vif.access_valid), + .io_access_ready(vif.access_ready), + .io_access_bits_write(vif.access_write), + .io_access_bits_addr(vif.access_addr), + .io_access_bits_data(vif.access_data), + .io_access_bits_mask(vif.access_mask), + .io_access_bits_atomic(vif.access_atomic), + .io_access_bits_atomicWord(vif.access_atomic_word), + .io_result_valid(vif.result_valid), + .io_result_ready(vif.result_ready), + .io_result_bits_error(vif.result_error), + .io_result_bits_data(vif.result_data), + .io_outstanding(vif.outstanding), + .io_maintenance_valid(cmo.valid), + .io_maintenance_ready(cmo.ready), + .io_maintenance_bits_tag(cmo.tag), + .io_maintenance_bits_firstLine(cmo.first_line), + .io_maintenance_bits_lastLine(cmo.last_line), + .io_maintenance_bits_op(cmo.operation), + .io_maintained_valid(cmo.response_valid), + .io_maintained_ready(cmo.response_ready), + .io_maintained_bits_tag(cmo.response_tag), + .io_maintained_bits_ok(cmo.ok), + .io_olderRequestsDrained(cmo.older_drained) + ); + assign vif.reset = timing.reset; + initial begin + uvm_config_db#(virtual chi_rnf_if)::set(null, "uvm_test_top*", "vif", vif); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top.env.reset", "timing", timing); + uvm_config_db#(virtual cmo_if)::set(null, "uvm_test_top*", "cmo", cmo); + run_test("cache_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/tests/test.svh new file mode 100644 index 000000000..4768cb593 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/resources/tests/test.svh @@ -0,0 +1,41 @@ +class cache_test extends protocol_test; + `uvm_component_utils(cache_test) + virtual cmo_if cmo; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + protocol_model::type_id::set_type_override(cache_model::get_type()); + super.build_phase(phase); + if (!uvm_config_db#(virtual cmo_if)::get(this, "", "cmo", cmo)) + `uvm_fatal("VIF", "Cache maintenance interface is required") + endfunction + task execute(); + cache_sequence seq = cache_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.state = test_env.state; + seq.vif = vif; + seq.scenario = test_env.state.scenario; + seq.cmo = cmo; + if (!$cast(seq.checked_state, test_env.state)) + `uvm_fatal("TYPE", "Maintenance model is required") + if (seq.scenario == "normal") seq.start(null); + else begin + repeat (3) @(negedge vif.clock); + test_env.reset.pulse(4); + cmo.first_line = 'h40000; + cmo.last_line = 'h40000; + cmo.operation = 0; + cmo.tag = 1; + if (seq.scenario == "bad_maintenance_alignment") cmo.first_line = 'h40001; + if (seq.scenario == "bad_maintenance_range") cmo.last_line = 'h3ffc0; + if (seq.scenario == "bad_maintenance_operation") cmo.operation = 3; + cmo.valid = 1; + do @(posedge vif.clock); while (!cmo.ready); + @(negedge vif.clock); + cmo.valid = 0; + repeat (40) @(negedge vif.clock); + `uvm_fatal("MISSING_ASSERTION", "Expected maintenance assertion did not fire") + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Cache.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Cache.scala new file mode 100644 index 000000000..408375823 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Cache.scala @@ -0,0 +1,84 @@ +package framework.core.memory.access.cache + +import framework.lib.arbiter.{RRArbiter} + +import framework.core.memory.access.cache.configs.{Params => ModuleParam} + +import framework.lib.queue.Queue + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.ip.bus.chi.{RequestFlit, RequesterPort} +import framework.ip.bus.chi.rnf.{BankedChiCache, CacheAccess, CacheProbe, CacheResult, RnfParams} +import framework.ip.memory.interlock.{Acknowledgement, Maintenance => Range, Params} + +/** A thread's own L1 and range maintenance client share one explicit CHI requester. */ +@instantiable +class Cache(b: ModuleParam) extends Module { + val config = b.cache + val tracking = b.tracking + + val c = config.chi + + @public + val io = IO(new Bundle { + + val access = Flipped(Decoupled(new CacheAccess(c))) + + val result = + Decoupled(new CacheResult(0)) + + val probe = Option.when(config.probe)(new CacheProbe(c)) + + val maintenance = Flipped(Decoupled(new Range(b.tracking))) + val maintained = Decoupled(new Acknowledgement(b.tracking)) + // Includes previously accepted virtual accesses, page walks and fetches outside this L1. + val olderRequestsDrained = Input(Bool()) + + val chi = new RequesterPort(c) + + val outstanding = Output(UInt(32.W)) + }) + + val cache: Instance[BankedChiCache] = Instantiate(new BankedChiCache(b.cache)) + val maintenance: Instance[Maintenance] = Instantiate(new Maintenance(b)) + val drained = io.olderRequestsDrained && cache.io.outstanding === 0.U + maintenance.io.request.valid := io.maintenance.valid && drained + maintenance.io.request.bits := io.maintenance.bits + io.maintenance.ready := maintenance.io.request.ready && drained + // Existing owners may keep progressing until drained. At admission, a new CPU + // request cannot race the CMO; subsequent conflicts belong to the external interlock. + val admitMaintenance = io.maintenance.valid && io.maintenance.ready + cache.io.access.valid := io.access.valid && !admitMaintenance + cache.io.access.bits := io.access.bits + io.access.ready := cache.io.access.ready && !admitMaintenance + io.result <> cache.io.result + io.probe.foreach(_ <> cache.io.probe.get) + maintenance.io.drained := true.B + io.maintained <> maintenance.io.response + io.outstanding := cache.io.outstanding + + val requests = Instantiate(new RRArbiter( + new RequestFlit(c), + 2 + )) + + requests.io.in(0) <> cache.io.chi.req + requests.io.in(1) <> maintenance.io.req + + val requestQueue = Instantiate(new Queue(new RequestFlit(c), 2, false, false)) + requestQueue.io.enq <> requests.io.out + io.chi.req <> requestQueue.io.deq + io.chi.txRsp <> cache.io.chi.txRsp + io.chi.txDat <> cache.io.chi.txDat + cache.io.chi.rxDat <> io.chi.rxDat + cache.io.chi.snp <> io.chi.snp + + val maintenanceResponse = io.chi.rxRsp.bits.txnId === config.banks.U + maintenance.io.rsp.valid := io.chi.rxRsp.valid && maintenanceResponse + maintenance.io.rsp.bits := io.chi.rxRsp.bits + cache.io.chi.rxRsp.valid := io.chi.rxRsp.valid && !maintenanceResponse + cache.io.chi.rxRsp.bits := io.chi.rxRsp.bits + io.chi.rxRsp.ready := Mux(maintenanceResponse, maintenance.io.rsp.ready, cache.io.chi.rxRsp.ready) +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Emit.scala new file mode 100644 index 000000000..1d2d2e8e9 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Emit.scala @@ -0,0 +1,34 @@ +package framework.core.memory.access.cache + +import java.nio.file.{Files, Paths} +import framework.ip.bus.chi.{Opcode, RequestFlit, ResponseFlit, SnoopFlit} + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val p = upickle.default.read[configs.Params](Files.readString(Paths.get(config))) + val c = p.cache.chi + Files.createDirectories(Paths.get(directory)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Cache(p), + args = Array("--target-dir", s"$directory/Cache", "--split-verilog") + ) + Files.writeString( + Paths.get(directory, "parameters.svh"), + s"""`define CHI_NODE_BITS ${c.nodeIdBits} +`define CHI_ADDRESS_BITS ${c.addressBits} +`define CHI_DATA_BITS ${c.dataBits} +`define CHI_REQ_BITS ${(new RequestFlit(c)).flitWidth} +`define CHI_RSP_BITS ${(new ResponseFlit(c)).flitWidth} +`define CHI_SNP_BITS ${(new SnoopFlit(c)).flitWidth} +`define CHI_NODE_ID ${p.cache.nodeId} +`define CHI_HOME_ID ${p.cache.homeId} +`define CHI_BANKS ${p.cache.banks} +`define CHI_CACHE_LINES ${p.cache.cacheLines} +`define CMO_ID_BITS ${p.tracking.idBits} +`define CMO_CLEAN ${Opcode.CleanShared} +`define CMO_CLEAN_INVALID ${Opcode.CleanInvalid} +`define CMO_INVALID ${Opcode.MakeInvalid} +`define CMO_COMP ${Opcode.Comp} +""" + ) +} diff --git a/examples/cores/rocket/arch/src/main/scala/cpu/Maintenance.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Maintenance.scala similarity index 81% rename from examples/cores/rocket/arch/src/main/scala/cpu/Maintenance.scala rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Maintenance.scala index bd2e3af40..dd5f23b84 100644 --- a/examples/cores/rocket/arch/src/main/scala/cpu/Maintenance.scala +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/Maintenance.scala @@ -1,15 +1,20 @@ -package hier.core.rocket +package framework.core.memory.access.cache + +import framework.core.memory.access.cache.configs.{Params => ModuleParam} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.chi.{HomeMapping, Opcode, RequestFlit, ResponseFlit} -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.interlock.{Acknowledgement, Maintenance => Range, MaintenanceOp, Params} +import framework.ip.bus.chi.{HomeMapping, Opcode, RequestFlit, ResponseFlit} +import framework.ip.bus.chi.rnf.RnfParams +import framework.ip.memory.interlock.{Acknowledgement, Maintenance => Range, MaintenanceOp, Params} -/** Serial range CMO client sharing its core's CHI node, with a distinct transaction ID. */ +/** Serial range CMO client sharing its thread's CHI node, with a distinct transaction ID. */ @instantiable -class Maintenance(config: RnfParams, tracking: Params) extends Module { +class Maintenance(b: ModuleParam) extends Module { + val config = b.cache + val tracking = b.tracking + val c = config.chi val mapping = HomeMapping(config.homeCount, config.homeId) val transaction = config.banks @@ -18,18 +23,21 @@ class Maintenance(config: RnfParams, tracking: Params) extends Module { @public val io = IO(new Bundle { - val request = Flipped(Decoupled(new Range(tracking))) - val response = Decoupled(new Acknowledgement(tracking)) + val request = Flipped(Decoupled(new Range(b.tracking))) + val response = Decoupled(new Acknowledgement(b.tracking)) // The caller drains older accepted CPU accesses and fills before asserting this. val drained = Input(Bool()) - val req = Decoupled(new RequestFlit(c)) - val rsp = Flipped(Decoupled(new ResponseFlit(c))) - val idle = Output(Bool()) + + val req = Decoupled(new RequestFlit(c)) + + val rsp = Flipped(Decoupled(new ResponseFlit(c))) + + val idle = Output(Bool()) }) val idle :: drain :: send :: waitResponse :: respond :: Nil = Enum(5) val state = RegInit(idle) - val command = Reg(new Range(tracking)) + val command = Reg(new Range(b.tracking)) val address = Reg(UInt(c.addressBits.W)) val succeeded = RegInit(false.B) io.idle := state === idle diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..277b2ba56 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/cache/src/main/scala/configs/Params.scala @@ -0,0 +1,15 @@ +package framework.core.memory.access.cache.configs + +import framework.ip.bus.chi.rnf.RnfParams +import framework.ip.memory.interlock.{Params => TrackingParams} +import upickle.default.{macroRW, ReadWriter} + +case class Params(cache: RnfParams, tracking: TrackingParams) + +object Params { + implicit val chiRW: ReadWriter[framework.ip.bus.chi.Params] = macroRW + implicit val cacheRW: ReadWriter[framework.ip.memory.cache.configs.CacheParams] = macroRW + implicit val rnfRW: ReadWriter[RnfParams] = macroRW + implicit val trackingRW: ReadWriter[TrackingParams] = macroRW + implicit val rw: ReadWriter[Params] = macroRW +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..8aa5f85e8 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/configs/Params.scala @@ -0,0 +1,21 @@ +package framework.core.memory.access.configs + +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} + +import framework.core.memory.access.{AccessParams, PhysicalRegion} +import upickle.default.{macroRW, ReadWriter} + +case class Params( + memory: AccessParams, + regions: Seq[PhysicalRegion], + translationEntries: Int, + lineBits: Int) { + val walker = WalkerParam(memory.chi, translationEntries) +} + +object Params { + implicit val chiRW: ReadWriter[framework.ip.bus.chi.Params] = macroRW + implicit val regionRW: ReadWriter[PhysicalRegion] = macroRW + implicit val memoryRW: ReadWriter[AccessParams] = macroRW + implicit val rw: ReadWriter[Params] = macroRW +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/build.sc b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/build.sc new file mode 100644 index 000000000..3c130a3fe --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.memory.access.mmu.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Walker.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/configs/uvm.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/configs/uvm.toml new file mode 100644 index 000000000..b961665cb --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/configs/uvm.toml @@ -0,0 +1,9 @@ +[parameters] +entries = 4 + +[parameters.bus] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 + +[defines] diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/csrc/Cargo.lock b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/mmu/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/csrc/Cargo.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/mmu/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/csrc/src/lib.rs b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/mmu/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/src/translation.rs b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/src/translation.rs new file mode 100644 index 000000000..9ba36d3f1 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/csrc/src/translation.rs @@ -0,0 +1,352 @@ +use std::collections::{HashMap, VecDeque}; +use std::ffi::c_void; +#[derive(Clone, Copy)] +struct Pte { + value: u64, + error: bool, +} +#[derive(Clone, Copy)] +struct Leaf { + mode: u32, + root: u64, + vpn: u64, + level: u32, + pte: u64, +} +struct Model { + address_bits: u32, + memory: HashMap, + expected_reads: VecDeque, + tlb: Vec>, + victim: usize, + epoch: u64, + candidate: Option<(Leaf, u64)>, + cached: bool, +} +#[derive(Default)] +struct Translation { + address: u64, + page_fault: bool, + access_fault: bool, + level: u32, +} +impl Translation { + fn fault(page: bool, level: u32) -> Self { + Self { + page_fault: page, + access_fault: !page, + level, + ..Self::default() + } + } +} +struct Request { + address: u64, + mode: u32, + root: u64, + privilege: u32, + write: bool, + execute: bool, + sum: bool, + mxr: bool, +} +fn canonical(address: u64) -> bool { + address <= 0x0000_003f_ffff_ffff || address >= 0xffff_ffc0_0000_0000 +} +impl Model { + fn leaf(&self, r: &Request, level: u32, value: u64) -> Translation { + let ppn = (value >> 10) & ((1u64 << 44) - 1); + let user = value & 16 != 0; + let privilege = if r.privilege == 0 { + user + } else { + !user || (r.sum && !r.execute) + }; + let operation = if r.execute { + value & 8 != 0 + } else if r.write { + value & 4 != 0 + } else { + value & 2 != 0 || (r.mxr && value & 8 != 0) + }; + if !privilege + || !operation + || value & 64 == 0 + || (r.write && value & 128 == 0) + || ppn & ((1u64 << (9 * level)) - 1) != 0 + { + return Translation::fault(true, level); + } + let bytes = 4096u64 << (9 * level); + let address = ppn * 4096 + (r.address & (bytes - 1)); + if address >= (1u64 << self.address_bits) { + Translation::fault(false, level) + } else { + Translation { + address, + level, + ..Translation::default() + } + } + } + fn cached(&self, r: &Request) -> Option { + if r.privilege == 3 || r.mode == 0 || !canonical(r.address) { + return None; + } + self.tlb + .iter() + .flatten() + .find(|entry| { + entry.mode == r.mode + && entry.root == r.root + && entry.vpn == r.address >> (12 + 9 * entry.level) + }) + .map(|entry| self.leaf(r, entry.level, entry.pte)) + } + fn walk(&self, r: &Request, reads: &mut Vec) -> Result { + let limit = 1u64 << self.address_bits; + if r.privilege == 3 || r.mode == 0 { + return Ok(if r.address >= limit { + Translation::fault(false, 0) + } else { + Translation { + address: r.address, + ..Translation::default() + } + }); + } + if !canonical(r.address) { + return Ok(Translation::fault(true, 2)); + } + let mut ppn = r.root; + for level in (0..=2u32).rev() { + let address = ppn * 4096 + ((r.address >> (12 + 9 * level)) & 511) * 8; + if address >= limit { + return Ok(Translation::fault(false, level)); + } + reads.push(address); + let pte = *self.memory.get(&address).ok_or(address)?; + if pte.error { + return Ok(Translation::fault(false, level)); + } + let value = pte.value; + if value & 1 == 0 || value & 6 == 4 || value >> 54 != 0 { + return Ok(Translation::fault(true, level)); + } + ppn = (value >> 10) & ((1u64 << 44) - 1); + if value & 10 != 0 { + return Ok(self.leaf(r, level, value)); + } + if level == 0 || value & 0xd0 != 0 { + return Ok(Translation::fault(true, level)); + } + } + unreachable!() + } +} +#[no_mangle] +pub extern "C" fn mmu_ref_create(address_bits: u32, entries: u32) -> *mut c_void { + assert!((44..=52).contains(&address_bits)); + Box::into_raw(Box::new(Model { + address_bits, + memory: HashMap::new(), + expected_reads: VecDeque::new(), + tlb: vec![None; entries as usize], + victim: 0, + epoch: 0, + candidate: None, + cached: false, + })) + .cast() +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_destroy(p: *mut c_void) { + drop(Box::from_raw(p.cast::())); +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_clear(p: *mut c_void) { + let m = &mut *p.cast::(); + assert!(m.expected_reads.is_empty()); + m.memory.clear(); +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_program(p: *mut c_void, address: u64, value: u64, error: u32) { + assert_eq!(address % 8, 0); + (*p.cast::()).memory.insert( + address, + Pte { + value, + error: error != 0, + }, + ); +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_read( + p: *mut c_void, + address: u64, + value: *mut u64, + error: *mut u32, +) -> u32 { + if let Some(pte) = (*p.cast::()).memory.get(&address) { + *value = pte.value; + *error = u32::from(pte.error); + 1 + } else { + 0 + } +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_pending(p: *mut c_void) -> u32 { + (*p.cast::()).expected_reads.len() as u32 +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_peek(p: *mut c_void, address: *mut u64) -> u32 { + if let Some(next) = (*p.cast::()).expected_reads.front() { + *address = *next; + 1 + } else { + 0 + } +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_consume(p: *mut c_void, address: u64) -> u32 { + let queue = &mut (*p.cast::()).expected_reads; + if queue.front() != Some(&address) { + return 0; + } + queue.pop_front(); + 1 +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_translate( + p: *mut c_void, + address: u64, + mode: u32, + root: u64, + privilege: u32, + write: u32, + execute: u32, + sum: u32, + mxr: u32, + paddr: *mut u64, + page_fault: *mut u32, + access_fault: *mut u32, + level: *mut u32, +) -> u32 { + assert!(mode == 0 || mode == 8); + assert!(matches!(privilege, 0 | 1 | 3)); + assert!(write == 0 || execute == 0); + assert!(root < 1u64 << 44); + let m = &mut *p.cast::(); + let r = Request { + address, + mode, + root, + privilege, + write: write != 0, + execute: execute != 0, + sum: sum != 0, + mxr: mxr != 0, + }; + let mut reads = Vec::new(); + m.candidate = None; + let cached = m.cached(&r); + m.cached = cached.is_some(); + let result = match cached { + Some(answer) => Ok(answer), + None => m.walk(&r, &mut reads), + }; + match result { + Ok(answer) => { + if !answer.page_fault && !answer.access_fault && !reads.is_empty() { + let pte = m.memory[reads.last().unwrap()].value; + m.candidate = Some(( + Leaf { + mode, + root, + vpn: address >> (12 + 9 * answer.level), + level: answer.level, + pte, + }, + m.epoch, + )); + } + m.expected_reads.extend(reads); + *paddr = answer.address; + *page_fault = u32::from(answer.page_fault); + *access_fault = u32::from(answer.access_fault); + *level = answer.level; + 0 + } + Err(address) => { + *paddr = address; + 1 + } + } +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_cached(p: *mut c_void) -> u32 { + u32::from((*p.cast::()).cached) +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_result(p: *mut c_void) { + let m = &mut *p.cast::(); + if m.expected_reads.is_empty() { + if let Some((leaf, epoch)) = m.candidate.take() { + if epoch == m.epoch && !m.tlb.is_empty() { + m.tlb[m.victim] = Some(leaf); + m.victim = (m.victim + 1) % m.tlb.len(); + } + } + } +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_flush(p: *mut c_void) { + let m = &mut *p.cast::(); + m.tlb.fill(None); + m.epoch += 1; +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_reset(p: *mut c_void) { + let m = &mut *p.cast::(); + m.tlb.fill(None); + m.epoch += 1; + m.victim = 0; + m.candidate = None; + m.expected_reads.clear(); +} +#[no_mangle] +pub unsafe extern "C" fn mmu_ref_lookup( + p: *mut c_void, + address: u64, + mode: u32, + root: u64, + privilege: u32, + write: u32, + execute: u32, + sum: u32, + mxr: u32, + paddr: *mut u64, +) -> u32 { + let m = &*p.cast::(); + let r = Request { + address, + mode, + root, + privilege, + write: write != 0, + execute: execute != 0, + sum: sum != 0, + mxr: mxr != 0, + }; + if privilege == 3 || mode == 0 { + *paddr = address; + return u32::from(address < 1u64 << m.address_bits); + } + if let Some(answer) = m.cached(&r) { + *paddr = answer.address; + u32::from(!answer.page_fault && !answer.access_fault) + } else { + 0 + } +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..6fc15df50 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/driver.svh @@ -0,0 +1,48 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual mmu_if vif; + request_item pending; + int mode = 0; + longint unsigned root = 0; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual mmu_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Walker input interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + if (pending == null) seq_item_port.try_next_item(pending); + vif.inputs[`MMU_IN_FLUSH_OFFSET+:`MMU_IN_FLUSH_WIDTH] = 0; + vif.inputs[`MMU_IN_REQ_VALID_OFFSET+:`MMU_IN_REQ_VALID_WIDTH] = 0; + if (pending != null) begin + if (pending.kind == 0) begin + mode = pending.mode; + root = pending.root; + vif.inputs[`MMU_REQ_OFFSET+:`MMU_REQ_BITS] = pending.payload; + vif.inputs[`MMU_IN_REQ_VALID_OFFSET+:`MMU_IN_REQ_VALID_WIDTH] = 1; + end else if (pending.kind == 1) vif.inputs[`MMU_IN_FLUSH_OFFSET+:`MMU_IN_FLUSH_WIDTH] = 1; + else begin + vif.inputs[`MMU_LOOKUP_OFFSET+:`MMU_LOOKUP_BITS] = pending.payload; + vif.inputs[`MMU_IN_LOOKUP_CONFIG_MODE_OFFSET+:`MMU_IN_LOOKUP_CONFIG_MODE_WIDTH]=pending.mode; + vif.inputs[`MMU_IN_LOOKUP_CONFIG_ROOTPPN_OFFSET+:`MMU_IN_LOOKUP_CONFIG_ROOTPPN_WIDTH]=pending.root; + end + end + vif.inputs[`MMU_IN_CONFIG_MODE_OFFSET+:`MMU_IN_CONFIG_MODE_WIDTH] = mode; + vif.inputs[`MMU_IN_CONFIG_ROOTPPN_OFFSET+:`MMU_IN_CONFIG_ROOTPPN_WIDTH] = root; + @(posedge vif.clock); + if (!vif.reset && pending != null && (pending.kind != 0 || `MO(vif.outputs, REQ_READY))) begin + if (pending.kind == 0 && pending.change_inputs) begin + mode = pending.next_mode; + root = pending.next_root; + end + #1; + seq_item_port.item_done(); + pending = null; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/item.svh new file mode 100644 index 000000000..9c3f6c649 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/item.svh @@ -0,0 +1,43 @@ +typedef logic [`MMU_REQ_BITS-1:0] request_t; +class request_item extends uvm_sequence_item; + `uvm_object_utils(request_item) + request_t payload; + int kind = 0; + int mode; + longint unsigned root; + bit change_inputs = 0; + int next_mode; + longint unsigned next_root; + function new(string name = "request_item"); + super.new(name); + endfunction +endclass +class request_sequence extends uvm_sequence #(request_item); + `uvm_object_utils(request_sequence) + request_item item; + function new(string name = "request_sequence"); + super.new(name); + endfunction + task body(); + start_item(item); + finish_item(item); + endtask +endclass +class sample_item extends uvm_sequence_item; + `uvm_object_utils(sample_item) + logic [`MMU_INPUT_BITS-1:0] inputs; + logic [`MMU_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "sample_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_object_utils_end + logic [`MMU_RESP_BITS-1:0] payload; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..bc3f0cc43 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual mmu_if vif; + uvm_analysis_port #(sample_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual mmu_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Walker monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + sample_item t; + forever begin + @(posedge vif.clock); + t = sample_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/peer.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/peer.svh new file mode 100644 index 000000000..2330591b8 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/agent/peer.svh @@ -0,0 +1,24 @@ +class peer_driver extends uvm_component; + `uvm_component_utils(peer_driver) + virtual mmu_if vif; + model reference; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual mmu_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Walker PTE interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + #1; + vif.inputs[`MMU_IN_ACCESS_READY_OFFSET+:`MMU_IN_ACCESS_READY_WIDTH]=!vif.reset&&!reference.hold_access&&reference.cycle%5>=2; + vif.inputs[`MMU_IN_RESP_READY_OFFSET+:`MMU_IN_RESP_READY_WIDTH]=!vif.reset&&!reference.hold_response&&reference.cycle%7>=3; + vif.inputs[`MMU_IN_RESULT_VALID_OFFSET+:`MMU_IN_RESULT_VALID_WIDTH]=!vif.reset&&reference.pending_result&&!reference.hold_result&&reference.cycle>=reference.due; + vif.inputs[`MMU_IN_RESULT_BITS_DATA_OFFSET+:`MMU_IN_RESULT_BITS_DATA_WIDTH]=reference.result_data; + vif.inputs[`MMU_IN_RESULT_BITS_ERROR_OFFSET+:`MMU_IN_RESULT_BITS_ERROR_WIDTH]=reference.result_error; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/env/env.svh new file mode 100644 index 000000000..ae371c302 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/env/env.svh @@ -0,0 +1,36 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + peer_driver peer; + model reference; + reset_controller reset; + uvm_sequencer #(request_item) seqr; + result_board #(result_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + peer = peer_driver::type_id::create("peer", this); + reference = model::type_id::create("model", this); + reset = reset_controller::type_id::create("reset", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + peer.reference = reference; + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + reference.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/env/model.svh new file mode 100644 index 000000000..05a77fbcc --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/env/model.svh @@ -0,0 +1,158 @@ +class model extends uvm_subscriber #(sample_item); + `uvm_component_utils(model) + chandle reference; + sample_item latest; + bit active = 0, pending_result = 0, hold_access = 0, hold_response = 0, hold_result = 0; + longint unsigned result_data; + bit result_error; + int due = 0, cycle = 0, accepted = 0, returned = 0, reads = 0, cancelled = 0; + string current_case; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + covergroup behavior with function sample ( + int privilege, bit write, bit execute, bit hit, int level, bit pf, bit af + ); + option.per_instance = 1; + privilege_mode: coverpoint privilege { + bins user_mode = {0}; bins supervisor = {1}; bins machine_mode = {3}; + } + write_access: coverpoint write; + execute_access: coverpoint execute; + cached_translation: coverpoint hit; + page_size: coverpoint level iff (!pf && !af) {bins sizes[] = {[0 : 2]};} + page_fault: coverpoint pf; + access_fault: coverpoint af; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + behavior = new(); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + reference = mmu_ref_create(`MMU_ADDRESS_BITS, `MMU_TLB_ENTRIES); + endfunction + function void write(sample_item t); + result_item expected, actual; + longint unsigned paddr, next_address, value, probe_paddr, address; + int unsigned pf, af, level, error, probe_hit; + bit req_fire, resp_fire, pte_result; + latest = t; + if (t.reset) begin + reset_ap.write(1); + if (active) cancelled++; + active = 0; + pending_result = 0; + mmu_ref_reset(reference); + return; + end + cycle++; + probe_hit = mmu_ref_lookup( + reference, + t.inputs[`MMU_IN_LOOKUP_REQ_VADDR_OFFSET+:`MMU_IN_LOOKUP_REQ_VADDR_WIDTH], + t.inputs[`MMU_IN_LOOKUP_CONFIG_MODE_OFFSET+:`MMU_IN_LOOKUP_CONFIG_MODE_WIDTH], + t.inputs[`MMU_IN_LOOKUP_CONFIG_ROOTPPN_OFFSET+:`MMU_IN_LOOKUP_CONFIG_ROOTPPN_WIDTH], + t.inputs[`MMU_IN_LOOKUP_REQ_PRIVILEGE_OFFSET+:`MMU_IN_LOOKUP_REQ_PRIVILEGE_WIDTH], + t.inputs[`MMU_IN_LOOKUP_REQ_WRITE_OFFSET+:`MMU_IN_LOOKUP_REQ_WRITE_WIDTH], + t.inputs[`MMU_IN_LOOKUP_REQ_EXECUTE_OFFSET+:`MMU_IN_LOOKUP_REQ_EXECUTE_WIDTH], + t.inputs[`MMU_IN_LOOKUP_REQ_SUM_OFFSET+:`MMU_IN_LOOKUP_REQ_SUM_WIDTH], + t.inputs[`MMU_IN_LOOKUP_REQ_MXR_OFFSET+:`MMU_IN_LOOKUP_REQ_MXR_WIDTH], + probe_paddr + ); + evidence::check(`MO(t.outputs, LOOKUP_HIT) === 1'(probe_hit), $sformatf( + "%s lookup hit mismatch va=%h mode=%0d root=%h expected=%0d got=%0d", + current_case, + t.inputs[`MMU_IN_LOOKUP_REQ_VADDR_OFFSET+:`MMU_IN_LOOKUP_REQ_VADDR_WIDTH], + t.inputs[`MMU_IN_LOOKUP_CONFIG_MODE_OFFSET+:`MMU_IN_LOOKUP_CONFIG_MODE_WIDTH], + t.inputs[`MMU_IN_LOOKUP_CONFIG_ROOTPPN_OFFSET+:`MMU_IN_LOOKUP_CONFIG_ROOTPPN_WIDTH], + probe_hit, + `MO(t.outputs, LOOKUP_HIT) + )); + if (probe_hit) + evidence::check(`MO(t.outputs, LOOKUP_PADDR) === `MMU_ADDRESS_BITS'(probe_paddr), + "Walker lookup returned the wrong physical address"); + req_fire = t.inputs[`MMU_IN_REQ_VALID_OFFSET+:`MMU_IN_REQ_VALID_WIDTH] && `MO(t.outputs, + REQ_READY); + resp_fire = + `MO(t.outputs, RESP_VALID) + && t.inputs[`MMU_IN_RESP_READY_OFFSET+:`MMU_IN_RESP_READY_WIDTH]; + pte_result = t.inputs[`MMU_IN_RESULT_VALID_OFFSET+:`MMU_IN_RESULT_VALID_WIDTH] && `MO( + t.outputs, RESULT_READY); + if (resp_fire) begin + evidence::check(active, "Walker returned an unsolicited translation"); + actual = result_item::type_id::create("actual"); + actual.payload = t.outputs[`MMU_RESP_OFFSET+:`MMU_RESP_BITS]; + actual_ap.write(actual); + active = 0; + returned++; + end + if (req_fire) begin + evidence::check(!active, "Walker overlapped address translations"); + if (mmu_ref_translate( + reference, + t.inputs[`MMU_IN_REQ_BITS_VADDR_OFFSET+:`MMU_IN_REQ_BITS_VADDR_WIDTH], + t.inputs[`MMU_IN_CONFIG_MODE_OFFSET+:`MMU_IN_CONFIG_MODE_WIDTH], + t.inputs[`MMU_IN_CONFIG_ROOTPPN_OFFSET+:`MMU_IN_CONFIG_ROOTPPN_WIDTH], + t.inputs[`MMU_IN_REQ_BITS_PRIVILEGE_OFFSET+:`MMU_IN_REQ_BITS_PRIVILEGE_WIDTH], + t.inputs[`MMU_IN_REQ_BITS_WRITE_OFFSET+:`MMU_IN_REQ_BITS_WRITE_WIDTH], + t.inputs[`MMU_IN_REQ_BITS_EXECUTE_OFFSET+:`MMU_IN_REQ_BITS_EXECUTE_WIDTH], + t.inputs[`MMU_IN_REQ_BITS_SUM_OFFSET+:`MMU_IN_REQ_BITS_SUM_WIDTH], + t.inputs[`MMU_IN_REQ_BITS_MXR_OFFSET+:`MMU_IN_REQ_BITS_MXR_WIDTH], + paddr, + pf, + af, + level + ) != 0) + `uvm_fatal("PTE_FIXTURE", $sformatf( + "%s requires an unprogrammed PTE at %h", current_case, paddr)) + expected = result_item::type_id::create("expected"); + expected.payload = '0; + expected.payload[`MMU_OUT_RESP_BITS_PADDR_OFFSET-`MMU_RESP_OFFSET+:`MMU_OUT_RESP_BITS_PADDR_WIDTH]=paddr; + expected.payload[`MMU_OUT_RESP_BITS_PAGEFAULT_OFFSET-`MMU_RESP_OFFSET] = pf; + expected.payload[`MMU_OUT_RESP_BITS_ACCESSFAULT_OFFSET-`MMU_RESP_OFFSET] = af; + expected.payload[`MMU_OUT_RESP_BITS_LEVEL_OFFSET-`MMU_RESP_OFFSET+:2] = level; + expected_ap.write(expected); + active = 1; + accepted++; + behavior.sample(t.inputs[`MMU_IN_REQ_BITS_PRIVILEGE_OFFSET+:`MMU_IN_REQ_BITS_PRIVILEGE_WIDTH], + t.inputs[`MMU_IN_REQ_BITS_WRITE_OFFSET+:`MMU_IN_REQ_BITS_WRITE_WIDTH], + t.inputs[`MMU_IN_REQ_BITS_EXECUTE_OFFSET+:`MMU_IN_REQ_BITS_EXECUTE_WIDTH], + mmu_ref_cached(reference), level, pf, af); + end + if (pte_result) begin + evidence::check(pending_result, "Walker accepted an unsolicited PTE result"); + mmu_ref_result(reference); + pending_result = 0; + end + if (`MO(t.outputs, ACCESS_VALID)) begin + address = `MO(t.outputs, ACCESS_BITS_ADDR); + evidence::check(mmu_ref_peek(reference, next_address + ) && address === next_address && address[2:0] == 0, $sformatf( + "%s unexpected PTE access %h", current_case, address)); + if (t.inputs[`MMU_IN_ACCESS_READY_OFFSET+:`MMU_IN_ACCESS_READY_WIDTH]) begin + evidence::check(!pending_result, "Walker overlapped PTE accesses"); + if (!mmu_ref_read(reference, address, value, error)) + `uvm_fatal("PTE_FIXTURE", "Walker read an unprogrammed PTE") + evidence::check(mmu_ref_consume(reference, address), "Walker skipped an expected PTE read"); + result_data = value; + result_error = error; + reads++; + due = cycle + 2 + reads % 4; + pending_result = 1; + end + end + if (t.inputs[`MMU_IN_FLUSH_OFFSET+:`MMU_IN_FLUSH_WIDTH]) mmu_ref_flush(reference); + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + evidence::check(!active && !pending_result && mmu_ref_pending(reference + ) == 0 && accepted == returned + cancelled, + "Walker transactions did not drain"); + endfunction + function void final_phase(uvm_phase phase); + mmu_ref_destroy(reference); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/filelist.f new file mode 100644 index 000000000..b9e867ad7 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Walker/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/interface.sv new file mode 100644 index 000000000..3bfcf2600 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/interface.sv @@ -0,0 +1,20 @@ +`include "parameters.svh" +interface mmu_if ( + input logic clock +); + logic reset; + logic [`MMU_INPUT_BITS-1:0] inputs = '0; + logic [`MMU_OUTPUT_BITS-1:0] outputs; + assert property (@(posedge clock) disable iff (reset) + outputs[`MMU_OUT_ACCESS_VALID_OFFSET] && !inputs[`MMU_IN_ACCESS_READY_OFFSET] |=> + outputs[`MMU_OUT_ACCESS_VALID_OFFSET] && $stable( + outputs[`MMU_OUT_ACCESS_BITS_ADDR_OFFSET+:`MMU_OUT_ACCESS_BITS_ADDR_WIDTH] + )) + else $fatal(1, "Walker PTE address changed under backpressure"); + assert property (@(posedge clock) disable iff (reset) + outputs[`MMU_OUT_RESP_VALID_OFFSET] && !inputs[`MMU_IN_RESP_READY_OFFSET] |=> + outputs[`MMU_OUT_RESP_VALID_OFFSET] && $stable( + outputs[`MMU_RESP_OFFSET+:`MMU_RESP_BITS] + )) + else $fatal(1, "Walker translation changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/package.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/package.sv new file mode 100644 index 000000000..33cded8b5 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/package.sv @@ -0,0 +1,76 @@ +`include "parameters.svh" +`define MI(P, N) P[`MMU_IN_``N``_OFFSET+:`MMU_IN_``N``_WIDTH] +`define MO(P, N) P[`MMU_OUT_``N``_OFFSET+:`MMU_OUT_``N``_WIDTH] +`define RQ(N) `MMU_IN_REQ_BITS_``N``_OFFSET-`MMU_REQ_OFFSET+:`MMU_IN_REQ_BITS_``N``_WIDTH +package mmu_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + typedef bit [63:0] u64; + import "DPI-C" function chandle mmu_ref_create( + input int unsigned address_bits, + entries + ); + import "DPI-C" function void mmu_ref_destroy(input chandle model); + import "DPI-C" function void mmu_ref_clear(input chandle model); + import "DPI-C" function void mmu_ref_program( + input chandle model, + input longint unsigned address, + value, + input int unsigned error + ); + import "DPI-C" function int unsigned mmu_ref_read( + input chandle model, + input longint unsigned address, + output longint unsigned value, + output int unsigned error + ); + import "DPI-C" function int unsigned mmu_ref_pending(input chandle model); + import "DPI-C" function int unsigned mmu_ref_peek( + input chandle model, + output longint unsigned address + ); + import "DPI-C" function int unsigned mmu_ref_consume( + input chandle model, + input longint unsigned address + ); + import "DPI-C" function int unsigned mmu_ref_translate( + input chandle model, + input longint unsigned address, + input int unsigned mode, + input longint unsigned root, + input int unsigned privilege, + wr, + execute, + sum, + mxr, + output longint unsigned paddr, + output int unsigned pf, + af, + level + ); + import "DPI-C" function int unsigned mmu_ref_lookup( + input chandle model, + input longint unsigned address, + input int unsigned mode, + input longint unsigned root, + input int unsigned privilege, + wr, + execute, + sum, + mxr, + output longint unsigned paddr + ); + import "DPI-C" function int unsigned mmu_ref_cached(input chandle model); + import "DPI-C" function void mmu_ref_result(input chandle model); + import "DPI-C" function void mmu_ref_reset(input chandle model); + import "DPI-C" function void mmu_ref_flush(input chandle model); + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "agent/peer.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..e8c91cc61 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/seq/sequence.svh @@ -0,0 +1,331 @@ +class scenario_sequence extends uvm_sequence; + `uvm_object_utils(scenario_sequence) + env test_env; + virtual mmu_if vif; + localparam u64 MAX_PA = (64'h1 << `MMU_ADDRESS_BITS) - 1; + localparam u64 HIGH_PPN = 64'h1 << (`MMU_ADDRESS_BITS - 12); + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task operation(int kind, request_t payload = '0, int mode = 8, u64 root = 16, bit change = 0, + int next_mode = 0, u64 next_root = 0); + request_sequence seq = request_sequence::type_id::create("operation"); + seq.item = request_item::type_id::create("item"); + seq.item.kind = kind; + seq.item.payload = payload; + seq.item.mode = mode; + seq.item.root = root; + seq.item.change_inputs = change; + seq.item.next_mode = next_mode; + seq.item.next_root = next_root; + seq.start(test_env.seqr); + endtask + task flush(); + operation(1); + endtask + task probe(request_t r, int mode = 8, u64 root = 16); + operation(2, r, mode, root); + endtask + task send(request_t r, int mode = 8, u64 root = 16, bit change_inputs = 0, int next_mode = 0, + u64 next_root = 0); + operation(0, r, mode, root, change_inputs, next_mode, next_root); + endtask + task run_case(string name, request_t r, int mode = 8, u64 root = 16, bit change_inputs = 0); + int target = test_env.scoreboard.checked + 1; + test_env.reference.current_case = name; + send(r, mode, root, change_inputs); + test_env.scoreboard.wait_checked(target); + @(negedge vif.clock); + evidence::check(mmu_ref_pending(test_env.reference.reference + ) == 0 && !test_env.reference.pending_result, + "Walker skipped expected PTE responses"); + endtask + function request_t request(u64 va, int operation = 0, int privilege = 1, bit sum = 0, + bit mxr = 0); + request_t r = '0; + r[`RQ(VADDR)] = va; + r[`RQ(WRITE)] = operation == 1; + r[`RQ(EXECUTE)] = operation == 2; + r[`RQ(PRIVILEGE)] = privilege; + r[`RQ(SUM)] = sum; + r[`RQ(MXR)] = mxr; + return r; + endfunction + function void program_pte(u64 address, u64 value, bit error = 0); + mmu_ref_program(test_env.reference.reference, address, value, error); + endfunction + function u64 pte(u64 address); + longint unsigned value; + int unsigned error; + if (!mmu_ref_read(test_env.reference.reference, address, value, error)) + `uvm_fatal("FIXTURE", "Attempt to modify an unprogrammed PTE") + return value; + endfunction + function u64 pte_address(u64 va, int level); + return (64'h12 - level) * 4096 + ((va >> (12 + 9 * level)) & 511) * 8; + endfunction + task prepare(u64 va, int leaf_level, u64 flags = 64'hcf, u64 mapped = 64'h80000000); + flush(); + mmu_ref_clear(test_env.reference.reference); + for (int level = 2; level >= leaf_level; level--) begin + if (level == leaf_level) program_pte(pte_address(va, level), ((mapped >> 12) << 10) | flags); + else program_pte(pte_address(va, level), ((64'h13 - level) << 10) | 1); + end + endtask + task table(u64 address, u64 mapped, u64 flags = 64'hcf, u64 root = 16); + for (int level = 2; level >= 0; level--) begin + u64 word_address = (root + 2 - level) * 4096 + ((address >> (12 + 9 * level)) & 511) * 8; + program_pte(word_address, + level == 0 ? ((mapped >> 12) << 10) | flags : ((root + 3 - level) << 10) | 1); + end + endtask + task body(); + u64 va = 64'h123456789, flags, mapped; + int permissions[5] = '{1, 3, 4, 5, 7}; + int target, before_reads; + test_env.reset.pulse(4); + prepare(va, 0, 64'h43, 64'h81234000); + target = test_env.scoreboard.checked + 2; + test_env.reference.current_case = "back_to_back_cached_permission_recheck"; + test_env.reference.hold_access = 1; + test_env.reference.hold_response = 1; + send(request(va)); + fork + send(request(va, 2)); + begin + wait (`MO(test_env.reference.latest.outputs, ACCESS_VALID)); + repeat (8) @(negedge vif.clock); + test_env.reference.hold_access = 0; + wait (`MO(test_env.reference.latest.outputs, RESP_VALID)); + repeat (8) @(negedge vif.clock); + test_env.reference.hold_response = 0; + end + join + test_env.scoreboard.wait_checked(target); + // Three page sizes and both canonical sign regions, including page edges. + for (int level = 0; level < 3; level++) begin + for (int position = 0; position < 3; position++) begin + u64 size = 64'h1000 << (9 * level); + u64 address = (va & ~(size - 1)) + (position == 0 ? 0 : position == 1 ? 123 : size - 1); + prepare(address, level); + run_case($sformatf("page_size_%0d_edge_%0d", level, position), request(address)); + address |= 64'hffffffc000000000; + prepare(address, level); + run_case($sformatf("negative_canonical_%0d_%0d", level, position), request(address)); + end + end + // Permissions: all valid RWX forms, U/S, U-page, SUM, MXR, load/store/fetch. + foreach (permissions[i]) + for (int privilege = 0; privilege < 2; privilege++) + for (int user_page = 0; user_page < 2; user_page++) + for (int sum = 0; sum < 2; sum++) + for (int mxr = 0; mxr < 2; mxr++) + for (int operation = 0; operation < 3; operation++) begin + flags = 64'hc1 | (u64'(permissions[i]) << 1) | (u64'(user_page) << 4); + prepare(va, 0, flags, 64'h81234000); + run_case($sformatf( + "perm_%0d_prv%0d_u%0d_sum%0d_mxr%0d_op%0d", + permissions[i], + privilege, + user_page, + sum, + mxr, + operation + ), request(va, operation, privilege, sum, mxr)); + end + for (int level = 0; level < 3; level++) + for (int ad = 0; ad < 4; ad++) + for (int operation = 0; operation < 3; operation++) begin + flags = 64'hf | (u64'(ad) << 6); + prepare(va, level, flags); + run_case($sformatf("svade_level%0d_ad%0d_op%0d", level, ad, operation), request( + va, operation)); + end + for (int level = 0; level < 3; level++) begin + for (int reserved_bit = 54; reserved_bit < 64; reserved_bit++) begin + prepare(va, level); + program_pte(pte_address(va, level), pte(pte_address(va, level)) | (64'h1 << reserved_bit)); + run_case($sformatf("reserved_leaf_l%0d_bit%0d", level, reserved_bit), request(va)); + end + for (int encoding = 0; encoding < 3; encoding++) begin + prepare(va, level, encoding == 0 ? 64'hce : encoding == 1 ? 64'hc5 : 64'hcd); + run_case($sformatf("invalid_pte_l%0d_encoding%0d", level, encoding), request(va)); + end + prepare(va, level); + program_pte(pte_address(va, level), '1, 1); + run_case($sformatf("backend_error_l%0d", level), request(va)); + prepare(va, level, 64'hcf, HIGH_PPN * 4096); + run_case($sformatf("physical_leaf_overflow_l%0d", level), request(va)); + prepare(va, level, 64'hf, HIGH_PPN * 4096); + run_case($sformatf("pte_fault_precedes_final_pa_overflow_l%0d", level), request(va)); + end + for (int level = 1; level <= 2; level++) begin + for (int bit_index = 0; bit_index < 9 * level; bit_index++) begin + prepare(va, level); + program_pte(pte_address(va, level), pte(pte_address(va, level) + ) | (64'h1 << (10 + bit_index))); + run_case($sformatf("superpage_alignment_l%0d_bit%0d", level, bit_index), request(va)); + end + for (int bad = 0; bad < 3; bad++) begin + prepare(va, 0); + program_pte(pte_address(va, level), pte(pte_address(va, level) + ) | (64'h1 << (bad == 0 ? 4 : bad == 1 ? 6 : 7))); + run_case($sformatf("nonleaf_reserved_l%0d_%0d", level, bad), request(va)); + end + for (int bit_index = 54; bit_index < 64; bit_index++) begin + prepare(va, 0); + program_pte(pte_address(va, level), pte(pte_address(va, level)) | (64'h1 << bit_index)); + run_case($sformatf("nonleaf_upper_reserved_l%0d_%0d", level, bit_index), request(va)); + end + prepare(va, 0); + program_pte(pte_address(va, level), (HIGH_PPN << 10) | 1); + run_case($sformatf("next_table_address_overflow_l%0d", level), request(va)); + end + prepare(va, 0); + program_pte(pte_address(va, 0), (64'h13 << 10) | 1); + run_case("nonleaf_at_bottom", request(va)); + prepare(va, 0, 64'h3ef); // G and both RSW bits are permitted and ignored for translation. + for (int level = 1; level <= 2; level++) + program_pte(pte_address(va, level), pte(pte_address(va, level)) | 64'h320); + run_case("software_and_global_bits", request(va)); + prepare(va, 0, 64'h59); + run_case("latched_root_sum_mxr_privilege", request(va, 0, 1, 1, 1), 8, 16, 1); + mmu_ref_clear(test_env.reference.reference); + run_case("root_physical_overflow", request(va), 8, HIGH_PPN); + run_case("positive_noncanonical", request(64'h0000004000000000)); + run_case("upper_noncanonical", request(64'hffffffbfffffffff)); + for (int operation = 0; operation < 3; operation++) begin + for (int privilege = 0; privilege < 2; privilege++) begin + run_case("bare_noncanonical_physical", request(64'h8000000000, operation, privilege), 0, 0); + run_case("bare_last_physical_byte", request(MAX_PA, operation, privilege), 0, 0); + run_case("bare_physical_overflow", request(MAX_PA + 1, operation, privilege), 0, 0); + end + run_case("machine_ignores_sv39", request(64'h8000000000, operation, 3), 8, HIGH_PPN); + run_case("machine_physical_overflow", request(MAX_PA + 1, operation, 3), 8, 16); + end + // Single-address translations on opposite sides of a base-page boundary. + prepare(64'hfff, 0, 64'hcf, MAX_PA & ~64'hfff); + run_case("last_byte_before_physical_limit", request(64'hfff)); + prepare(64'h1000, 0, 64'hcf, MAX_PA + 1); + run_case("next_page_physical_overflow", request(64'h1000)); + // The last PTE word of the highest implemented physical table page is legal. + mmu_ref_clear(test_env.reference.reference); + program_pte(MAX_PA - 7, 64'hcf); + run_case("last_physical_pte_word", request(64'hffffffffc0000000), 8, MAX_PA >> 12); + // Warm translations re-check permissions without re-reading page tables. + prepare(va, 0, 64'h4f, 64'h81234000); + run_case("cached_read", request(va)); + before_reads = test_env.reference.reads; + run_case("cached_store_dirty_fault", request(va, 1)); + run_case("cached_user_fault", request(va, 0, 0)); + evidence::check(test_env.reference.reads == before_reads, + "Walker did not reuse its cached leaf"); + probe(request(va)); + probe(request(va, 0, 0)); + // Updating memory without SFENCE does not change an already cached leaf. + program_pte(pte_address(va, 0), ((64'h84567000 >> 12) << 10) | 64'hcf); + run_case("cached_old_pte", request(va)); + flush(); + run_case("after_sfence_new_pte", request(va)); + // Every configured entry and round-robin eviction, checked through the public lookup port. + flush(); + mmu_ref_clear(test_env.reference.reference); + for (int index = 0; index <= `MMU_TLB_ENTRIES; index++) + table('h400000 + index * 4096, 'h90000000 + index * 4096); + for (int index = 0; index < `MMU_TLB_ENTRIES; index++) + run_case("fill_tlb", request('h400000 + index * 4096)); + for (int index = 0; index < `MMU_TLB_ENTRIES; index++) probe(request('h400000 + index * 4096)); + run_case("evict_oldest_tlb_entry", request('h400000 + `MMU_TLB_ENTRIES * 4096)); + for (int index = 0; index <= `MMU_TLB_ENTRIES; index++) probe(request('h400000 + index * 4096)); + // A changed live config cannot relabel a walk that was accepted under another root. + prepare(va, 0, 64'hcf, 64'h81234000); + target = test_env.scoreboard.checked + 1; + test_env.reference.current_case = "latched_tlb_fill_root"; + send(request(va), 8, 16, 1, 8, 32); + test_env.scoreboard.wait_checked(target); + probe(request(va), 8, 16); + probe(request(va), 8, 32); + table(va, 64'h84567000, 64'hcf, 32); + run_case("different_root_same_vpn", request(va), 8, 32); + probe(request(va), 8, 16); + probe(request(va), 8, 32); + // SFENCE during an outstanding PTE response must prevent a later stale TLB refill. + prepare(va, 0, 64'hcf, 64'h81234000); + target = test_env.scoreboard.checked + 1; + test_env.reference.current_case = "flush_during_walk"; + test_env.reference.hold_result = 1; + send(request(va)); + wait (test_env.reference.pending_result); + flush(); + test_env.reference.hold_result = 0; + test_env.scoreboard.wait_checked(target); + probe(request(va)); + // SFENCE wins when the final leaf response and flush share a clock edge. + prepare(va, 0, 64'hcf, 64'h81234000); + target = test_env.scoreboard.checked + 1; + test_env.reference.current_case = "flush_on_fill"; + send(request(va)); + wait (test_env.reference.pending_result && mmu_ref_pending(test_env.reference.reference) == 0); + test_env.reference.hold_result = 1; + repeat (5) @(negedge vif.clock); + fork + flush(); + begin + wait (`MI(vif.inputs, FLUSH)); + test_env.reference.hold_result = 0; + end + join + test_env.scoreboard.wait_checked(target); + probe(request(va)); + // Reset cancellation at the offer, memory wait and held translation response boundaries. + for (int phase_index = 0; phase_index < 3; phase_index++) begin + prepare(va, 0, 64'hcf, 64'h81234000); + test_env.reference.current_case = "reset_cancel_translation"; + test_env.reference.hold_access = phase_index == 0; + test_env.reference.hold_result = phase_index == 1; + test_env.reference.hold_response = phase_index == 2; + send(request(va)); + if (phase_index == 0) wait (`MO(test_env.reference.latest.outputs, ACCESS_VALID)); + else if (phase_index == 1) wait (test_env.reference.pending_result); + else wait (`MO(test_env.reference.latest.outputs, RESP_VALID)); + test_env.reset.pulse(3); + test_env.reference.hold_access = 0; + test_env.reference.hold_result = 0; + test_env.reference.hold_response = 0; + run_case("after_reset_rewalk", request(va)); + end + evidence::check(evidence::cancelled == 3, "Walker reset cancellation count mismatch"); + `uvm_info("WALKER", $sformatf("accepted=%0d returned=%0d PTEreads=%0d reset_cancelled=%0d", + test_env.reference.accepted, test_env.reference.returned, + test_env.reference.reads, test_env.reference.cancelled), UVM_LOW) + endtask +endclass +class invalid_sequence extends uvm_sequence #(request_item); + `uvm_object_utils(invalid_sequence) + function new(string name = "invalid_sequence"); + super.new(name); + endfunction + task body(); + request_item req = request_item::type_id::create("invalid"); + string scenario; + if (!$value$plusargs("CASE=%s", scenario)) + `uvm_fatal("CASE", "Walker assertion scenario is required") + start_item(req); + req.payload = '0; + req.mode = 8; + req.root = 16; + req.payload[`RQ(PRIVILEGE)] = 1; + case (scenario) + "mode": req.mode = 9; + "privilege": req.payload[`RQ(PRIVILEGE)] = 2; + "access": begin + req.payload[`RQ(WRITE)] = 1; + req.payload[`RQ(EXECUTE)] = 1; + end + default: `uvm_fatal("CASE", "Unknown walker assertion scenario") + endcase + finish_item(req); + #100ns; + `uvm_fatal("EXPECTED_ASSERTION", "Invalid walker input did not trigger its assertion") + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/suite.toml new file mode 100644 index 000000000..21bc2e14f --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/suite.toml @@ -0,0 +1,71 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libmmu_reference_model" + +[[tests]] +functional_groups = ["mmu_pkg::model::behavior"] +name = "walker" +top = "walker_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "walker_bad_access" +top = "walker_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=access"] +assertion = "Walker access cannot both write and execute" + +[[tests]] +functional_groups = [] +name = "walker_bad_mode" +top = "walker_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=mode"] +assertion = "Walker requires Bare or Sv39 mode" + +[[tests]] +functional_groups = [] +name = "walker_bad_privilege" +top = "walker_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=privilege"] +assertion = "Walker effective privilege must be U, S or M" diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/tb.sv new file mode 100644 index 000000000..398baf810 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/tb.sv @@ -0,0 +1,19 @@ +module walker_tb; + import uvm_pkg::*; + import mmu_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + mmu_if bus (clock); + assign bus.reset = timing.reset; + Walker dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual mmu_if)::set(null, "uvm_test_top*", "vif", bus); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/tests/test.svh new file mode 100644 index 000000000..49cb108dd --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/resources/tests/test.svh @@ -0,0 +1,43 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + virtual mmu_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + if (!uvm_config_db#(virtual mmu_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Walker timing interface is required") + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.vif = vif; + seq.start(null); + endtask +endclass +class invalid_test extends base_test; + `uvm_component_utils(invalid_test) + driver source; + reset_controller reset; + uvm_sequencer #(request_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + reset = reset_controller::type_id::create("reset", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + endfunction + task execute(); + invalid_sequence seq = invalid_sequence::type_id::create("scenario"); + reset.pulse(4); + seq.start(seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/Emit.scala new file mode 100644 index 000000000..7e1ccda87 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/Emit.scala @@ -0,0 +1,57 @@ +package framework.core.memory.access.mmu + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.core.memory.access.mmu.configs.Params + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val p = upickle.default.read[Params](Files.readString(Paths.get(config))) + Files.createDirectories(Paths.get(output)) + val ports = scala.collection.mutable.ArrayBuffer[String]() + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + val widths = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0 + var outputOffset = 0 + + def fields(data: Data, path: String): Seq[(String, Int)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(path -> leaf.getWidth) + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new Walker(p) + for ((name, width) <- fields(dut.io, "")) { + val input = name.startsWith("config_") || name == "flush" || name.startsWith("lookup_config_") || + name.startsWith("lookup_req_") || name == "req_valid" || name.startsWith("req_bits_") || + name == "resp_ready" || name == "access_ready" || name == "result_valid" || name.startsWith("result_bits_") + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define MMU_${direction}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define MMU_${direction}_${name.toUpperCase}_WIDTH $width" + ports += s".io_$name(bus.$signal[$offset +: $width])" + offsets(name) = offset; widths(name) = width + if (input) inputOffset += width else outputOffset += width + } + dut + }, + args = Array("--target-dir", s"$output/Walker", "--split-verilog") + ) + definitions += s"`define MMU_INPUT_BITS $inputOffset" + definitions += s"`define MMU_OUTPUT_BITS $outputOffset" + definitions += s"`define MMU_TLB_ENTRIES ${p.entries}" + definitions += s"`define MMU_ADDRESS_BITS ${p.bus.addressBits}" + for ((name, prefix) <- Seq("REQ" -> "req_bits_", "RESP" -> "resp_bits_", "LOOKUP" -> "lookup_req_")) { + val names = offsets.keys.filter(_.startsWith(prefix)).toSeq + definitions += s"`define MMU_${name}_OFFSET ${names.map(offsets).min}" + definitions += s"`define MMU_${name}_BITS ${names.map(widths).sum}" + } + Files.writeString(Paths.get(output, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/scala/Walker.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/Walker.scala similarity index 82% rename from arch/src/main/scala/framework/mem-core/mmu/src/main/scala/Walker.scala rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/Walker.scala index 2a0765a76..af52bbcd8 100644 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/scala/Walker.scala +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/Walker.scala @@ -1,10 +1,20 @@ -package memcore.memory.mmu +package framework.core.memory.access.mmu + +import framework.core.memory.access.mmu.configs.{Params => ModuleParam} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.chi.Params -import memcore.bus.chi.rnf.{CacheAccess, CacheAtomic, CacheResult} + +/** One naturally aligned, read-only 64-bit page-table entry access. */ +class PteRequest(b: ModuleParam) extends Bundle { + val addr = UInt(b.bus.addressBits.W) +} + +class PteResponse extends Bundle { + val data = UInt(64.W) + val error = Bool() +} /** One address translation; the caller resolves effective privilege (including MPRV). */ class Request extends Bundle { @@ -16,7 +26,9 @@ class Request extends Bundle { val mxr = Bool() } -class Response(p: Params) extends Bundle { +class Response(b: ModuleParam) extends Bundle { + val p = b.bus + val paddr = UInt(p.addressBits.W) val pageFault = Bool() val accessFault = Bool() @@ -38,7 +50,9 @@ class Config extends Bundle { * walk, so a caller can take a fast path on a hit. */ @instantiable -class Walker(p: Params = Params(), entries: Int = 0) extends Module { +class Walker(b: ModuleParam) extends Module { + val p = b.bus + val entries = b.entries @public val io = IO(new Bundle { @@ -53,23 +67,25 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { } val req = Flipped(Decoupled(new Request)) - val resp = Decoupled(new Response(p)) - val access = Decoupled(new CacheAccess(p)) - val result = Flipped(Decoupled(new CacheResult)) + val resp = Decoupled(new Response(b)) + val access = Decoupled(new PteRequest(b)) + val result = Flipped(Decoupled(new PteResponse)) }) val idle :: issue :: waitResult :: respond :: Nil = Enum(4) val state = RegInit(idle) val request = Reg(new Request) + val requestConfig = Reg(new Config) + val staleFill = RegInit(false.B) val ppn = Reg(UInt(44.W)) val level = RegInit(0.U(2.W)) - val response = Reg(new Response(p)) + val response = Reg(new Response(b)) - private def vpn(address: UInt, walkLevel: UInt): UInt = + def vpn(address: UInt, walkLevel: UInt): UInt = MuxLookup(walkLevel, address(20, 12))(Seq(2.U -> address(38, 30), 1.U -> address(29, 21))) /** Leaf address, permission/A/D fault and physical-width overflow for one request. */ - private def leafOf(pte: UInt, walkLevel: UInt, r: Request): (UInt, Bool, Bool) = { + def leafOf(pte: UInt, walkLevel: UInt, r: Request): (UInt, Bool, Bool) = { val ptePpn = pte(53, 10) val supervisor = r.privilege === 1.U val userPermission = Mux(supervisor, !pte(4) || (r.sum && !r.execute), pte(4)) @@ -99,7 +115,7 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { val tlb = Reg(Vec(entries max 1, new Entry)) val tlbVictim = RegInit(0.U(log2Ceil(entries max 2).W)) - private def tlbMatches(vaddr: UInt, config: Config): Seq[Bool] = tlb.zip(tlbValid).map { case (e, valid) => + def tlbMatches(vaddr: UInt, config: Config): Seq[Bool] = tlb.zip(tlbValid).map { case (e, valid) => val vpn = vaddr(38, 12) (entries > 0).B && valid && e.mode === config.mode && e.root === config.rootPpn && MuxLookup(e.level, vpn === e.vpn)(Seq( @@ -109,7 +125,7 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { } /** (hit, address, page fault, access fault, level) for a request served without a walk. */ - private def translate(r: Request, config: Config): (Bool, UInt, Bool, Bool, UInt) = { + def translate(r: Request, config: Config): (Bool, UInt, Bool, Bool, UInt) = { val identity = r.privilege === 3.U || config.mode === 0.U val canonical = r.vaddr(63, 39) === Fill(25, r.vaddr(38)) val matches = tlbMatches(r.vaddr, config) @@ -128,7 +144,7 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { io.lookup.paddr := paddr } - private def finish( + def finish( address: UInt, pageFault: Bool, accessFault: Bool, @@ -150,7 +166,9 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { "Walker effective privilege must be U, S or M" ) assert(!(r.write && r.execute), "Walker access cannot both write and execute") - request := r + request := r + requestConfig := io.config + staleFill := false.B val (hit, paddr, pageFault, accessFault, walkLevel) = translate(r, io.config) when(hit) { finish(paddr, pageFault, accessFault, walkLevel) @@ -165,13 +183,8 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { // A PTE index occupies exactly the low 12 bits of its 4-KiB table page. val pteAddress = Cat(ppn, vpn(request.vaddr, level), 0.U(3.W)) val pteAddressOverflow = pteAddress(55, p.addressBits).orR - io.access.valid := state === issue && !pteAddressOverflow - io.access.bits.addr := pteAddress(p.addressBits - 1, 0) - io.access.bits.write := false.B - io.access.bits.data := 0.U - io.access.bits.mask := 0.U - io.access.bits.atomic := CacheAtomic.None.U - io.access.bits.atomicWord := false.B + io.access.valid := state === issue && !pteAddressOverflow + io.access.bits.addr := pteAddress(p.addressBits - 1, 0) when(state === issue && pteAddressOverflow) { finish(0.U, false.B, true.B, level) }.elsewhen(io.access.fire) { @@ -191,10 +204,10 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { finish(0.U, true.B, false.B, level) }.elsewhen(leaf) { finish(leafAddress(p.addressBits - 1, 0), leafFault, !leafFault && leafOverflow, level) - if (entries > 0) when(!leafFault && !leafOverflow) { + if (entries > 0) when(!leafFault && !leafOverflow && !staleFill && !io.flush) { tlbValid(tlbVictim) := true.B - tlb(tlbVictim).mode := io.config.mode - tlb(tlbVictim).root := io.config.rootPpn + tlb(tlbVictim).mode := requestConfig.mode + tlb(tlbVictim).root := requestConfig.rootPpn tlb(tlbVictim).vpn := request.vaddr(38, 12) tlb(tlbVictim).level := level tlb(tlbVictim).pte := pte @@ -210,7 +223,10 @@ class Walker(p: Params = Params(), entries: Int = 0) extends Module { } // SFENCE.VMA wins over a same-cycle fill. - when(io.flush)(tlbValid.foreach(_ := false.B)) + when(io.flush) { + tlbValid.foreach(_ := false.B) + staleFill := true.B + } io.resp.valid := state === respond io.resp.bits := response diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..842d96885 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu/src/main/scala/configs/Params.scala @@ -0,0 +1,11 @@ +package framework.core.memory.access.mmu.configs + +import framework.ip.bus.chi +import upickle.default.{macroRW, ReadWriter} + +case class Params(bus: chi.Params, entries: Int) + +object Params { + implicit val chiRW: ReadWriter[chi.Params] = macroRW + implicit val rw: ReadWriter[Params] = macroRW +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/build.sc b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/build.sc new file mode 100644 index 000000000..2f0176451 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/build.sc @@ -0,0 +1,28 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.memory.access.preflight.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Permission.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/PermissionCheck.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Preflight.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Preparation.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/PreparedMap.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Segmenter.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Config.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/configs/uvm.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/configs/uvm.toml new file mode 100644 index 000000000..ad95fb0a7 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/configs/uvm.toml @@ -0,0 +1,22 @@ +[parameters] +contexts=4 +maxRanges=8 +idBits=8 +beatBytes=16 +[parameters.bus] +nodeIdBits=7 +addressBits=44 +dataBits=256 +[parameters.permission] +addressBits=44 +entries=4 +[[parameters.regions]] +base="0" +bytes="17592186044416" +cacheable=true +executable=true +readable=true +writable=true +atomic=true +normal=true +[defines] diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/csrc/Cargo.lock b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/preflight/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/csrc/Cargo.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/preflight/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/csrc/src/lib.rs b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/preflight/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/csrc/src/preparation.rs b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/src/preparation.rs similarity index 99% rename from arch/src/main/scala/framework/mem-core/preflight/src/csrc/src/preparation.rs rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/src/preparation.rs index e7be0416c..8e7afff13 100644 --- a/arch/src/main/scala/framework/mem-core/preflight/src/csrc/src/preparation.rs +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/src/preparation.rs @@ -12,7 +12,7 @@ struct Model { } #[no_mangle] pub extern "C" fn preflight_ref_create(address_bits:u32, beat:u32)->*mut c_void { - Box::into_raw(Box::new(Model{mmu:mmu::mmu_ref_create(address_bits),address_bits,beat:beat as u64, + Box::into_raw(Box::new(Model{mmu:mmu::mmu_ref_create(address_bits, 0),address_bits,beat:beat as u64, plans:HashMap::new(),deny_pte:None,deny_range:None})).cast() } #[no_mangle] diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/csrc/src/prepared_map.rs b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/src/prepared_map.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/preflight/src/csrc/src/prepared_map.rs rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/csrc/src/prepared_map.rs diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/agent/service.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/agent/service.svh new file mode 100644 index 000000000..ca4024b56 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/agent/service.svh @@ -0,0 +1,113 @@ +task service(); + forever begin + @(ctl.sample); + if (!ctl.sample.reset) begin + if (authorization.sample.valid) + behavior.sample(0, authorization.sample.bits[`PF(AUTH, WRITE)], 0, + authorization.sample.bits[`PF(AUTH, ISPTE)], !authorization.sample.ready); + if (prepared.sample.valid) + behavior.sample(prepared.sample.bits[`PF(OUT, ERROR)], prepared.sample.bits[`PF(OUT, WRITE) + ], prepared.sample.bits[`PF(OUT, LAST)], 0, !prepared.sample.ready); + cycle++; + if (cmd.sample.valid && !cmd.sample.ready) command_stalls++; + if (prepared.sample.valid && !prepared.sample.ready) output_stalls++; + if (authorization.sample.valid && !authorization.sample.ready) auth_stalls++; + if (cmd.sample.valid && cmd.sample.ready) predict(cmd.sample.bits); + if (permission.sample.valid && permission.sample.ready) begin + verify_contract(auth_pending, "Unsolicited permission consumed"); + if (auth_pte && auth_allow) begin + pte_grant = 1; + grant_id = auth_id; + end + auth_pending = 0; + end + if (authorization.sample.valid && authorization.sample.ready) begin + int unsigned allow; + int id = authorization.sample.bits[`PF(AUTH, ID)]; + verify_contract(!$isunknown(authorization.sample.bits) && !auth_pending, + "Overlapping/unknown authorization"); + verify_contract(preflight_ref_authorize( + model, + id, + authorization.sample.bits[ + `PF(AUTH, PA) + ], + authorization.sample.bits[ + `PF(AUTH, BYTES) + ], + authorization.sample.bits[ + `PF(AUTH, WRITE) + ], + authorization.sample.bits[ + `PF(AUTH, ISPTE) + ], + authorization.sample.bits[ + `PF(AUTH, PRIVILEGE) + ], + allow + ) == 1, "Authorization lost snapshot, range or PTE S privilege"); + auth_pending = 1; + auth_id = id; + auth_allow = allow; + auth_pte = authorization.sample.bits[`PF(AUTH, ISPTE)]; + grant_pa = authorization.sample.bits[`PF(AUTH, PA)]; + auth_due = cycle + 3; + permit_packet = '0; + permit_packet[`PF(PERMIT, ID)] = id; + permit_packet[`PF(PERMIT, ALLOW)] = allow; + auth_count++; + end + if (pte_resp.sample.valid && pte_resp.sample.ready) pte_pending = 0; + if (pte_req.sample.valid && pte_req.sample.ready) begin + u64 value; + int unsigned error; + verify_contract(pte_grant && !pte_pending && !$isunknown(pte_req.sample.bits), + "PTE accessed before authorization"); + verify_contract(pte_req.sample.bits[`PF(PTE, ADDR)] == grant_pa && grant_pa[2:0] == 0, + "PTE request changed address or is not 8-byte aligned"); + verify_contract(preflight_ref_pte(model, grant_id, grant_pa, value, error) == 1, + "PTE address chain differs from architectural walker"); + pte_packet = '0; + pte_packet[`PF(PTERESP, DATA)] = value; + pte_packet[`PF(PTERESP, ERROR)] = error; + pte_id = grant_id; + pte_pending = 1; + pte_due = cycle + 2 + (pte_count % 4); + pte_grant = 0; + pte_count++; + end + if (prepared.sample.valid) begin + int id = prepared.sample.bits[`PF(OUT, ID)]; + verify_contract(live.exists(id) && !$isunknown(prepared.sample.bits), + "Unsolicited/unknown prepared plan"); + verify_contract( + preflight_ref_pending(model, id + ) == 0 && !(auth_pending && auth_id == id) && !(pte_pending && pte_id == id), + "Partial plan exposed before all translations/permissions completed"); + if (prepared.sample.ready) begin + result_item actual = new; + actual.packet = prepared.sample.bits; + actual.set_transaction_id((id << 4) | output_index[id]); + scoreboard.actual_export.write(actual); + output_index[id]++; + segments++; + if (prepared.sample.bits[`PF(OUT, LAST)]) begin + failures[int'(prepared.sample.bits[`PF(OUT, ERROR)])]++; + retired++; + live.delete(id); + output_index.delete(id); + preflight_ref_retire(model, id); + end + end + end + end + @(negedge ctl.clock); + authorization.ready = !ctl.reset && !auth_pending && cycle % 5 >= 2; + permission.valid = !ctl.reset && auth_pending && !hold_permission && cycle >= auth_due; + permission.bits = permit_packet; + pte_req.ready = !ctl.reset && !pte_pending && cycle % 4 != 0; + pte_resp.valid = !ctl.reset && pte_pending && cycle >= pte_due; + pte_resp.bits = pte_packet; + prepared.ready = !ctl.reset && !hold_output && cycle % 7 >= 3; + end +endtask diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/env/env.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/env/env.svh new file mode 100644 index 000000000..120889ac5 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/env/env.svh @@ -0,0 +1,170 @@ +class env extends uvm_env; + `uvm_component_utils(env) + virtual preflight_control_if ctl; + virtual stream_if #(`PF_CMD_WIDTH) cmd; + virtual stream_if #(`PF_OUT_WIDTH) prepared; + virtual stream_if #(`PF_AUTH_WIDTH) authorization; + virtual stream_if #(`PF_PERMIT_WIDTH) permission; + virtual stream_if #(`PF_PTE_WIDTH) pte_req; + virtual stream_if #(`PF_PTERESP_WIDTH) pte_resp; + result_board #(result_item, 1) scoreboard; + chandle model; + bit live[int]; + int output_index[int]; + int cycle = 0, submitted = 0, retired = 0, segments = 0, failures[7] = '{default: 0}, peak = 0; + int + auth_count = 0, + pte_count = 0, + command_stalls = 0, + output_stalls = 0, + auth_stalls = 0, + cancelled = 0; + bit hold_output = 0, hold_permission = 0, auth_pending = 0, pte_pending = 0, pte_grant = 0; + int auth_id, pte_id, grant_id, auth_due, pte_due; + bit auth_allow, auth_pte; + u64 grant_pa; + logic [`PF_PERMIT_WIDTH-1:0] permit_packet; + logic [`PF_PTERESP_WIDTH-1:0] pte_packet; + u64 next_table = 'h10000; + covergroup behavior with function sample (int error, bit writing, bit last, bit pte, bit stall); + option.per_instance = 1; + outcomes: coverpoint error {bins all[] = {[0 : 6]};} + direction: coverpoint writing; + records: coverpoint last; + authorization_kind: coverpoint pte; + backpressure: coverpoint stall; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + + endfunction + function void verify_contract(bit ok, string message); + verification_pkg::evidence::checks++; + if (!ok) `uvm_fatal("PREFLIGHT", message) + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + model = preflight_ref_create(`PF_ADDRESS_BITS, `PF_BEAT_BYTES); + scoreboard = result_board#(result_item, 1)::type_id::create("scoreboard", this); + verify_contract(uvm_config_db#(virtual preflight_control_if)::get(this, "", "ctl", ctl), + "Missing control"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_CMD_WIDTH))::get(this, "", "cmd", cmd), + "Missing command"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::get( + this, "", "prepared", prepared), "Missing output"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_AUTH_WIDTH))::get( + this, "", "authorization", authorization), "Missing authorization"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_PERMIT_WIDTH))::get( + this, "", "permission", permission), "Missing permission"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_PTE_WIDTH))::get( + this, "", "pte_req", pte_req), "Missing PTE request"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_PTERESP_WIDTH))::get( + this, "", "pte_resp", pte_resp), "Missing PTE response"); + endfunction + function command_t request(int id, u64 base, int rows, int span, int row_stride, int cols = 1, + int col_stride = 0, bit write = 0, int mode = 0, u64 root = 0, + int privilege = 1, bit sum = 0, bit mxr = 0); + command_t q = '0; + q[`PF(CMD, ID)] = id; + q[`PF(CMD, BASEVA)] = base; + q[`PF(CMD, ROWS)] = rows; + q[`PF(CMD, COLUMNS)] = cols; + q[`PF(CMD, SPANBYTES)] = span; + q[`PF(CMD, ROWSTRIDE)] = row_stride; + q[`PF(CMD, COLUMNSTRIDE)] = col_stride; + q[`PF(CMD, WRITE)] = write; + q[`PF(CMD, MODE)] = mode; + q[`PF(CMD, ROOTPPN)] = root; + q[`PF(CMD, PRIVILEGE)] = privilege; + q[`PF(CMD, SUM)] = sum; + q[`PF(CMD, MXR)] = mxr; + return q; + endfunction + function u64 mapping(u64 va, int pages, u64 flags = 'hc7, int backend_error_page = -1); + u64 root = next_table; + next_table += 'h3000; + preflight_ref_program(model, root + ((va >> 30) & 511) * 8, ((root + 4096) >> 12) << 10 | 1, 0); + preflight_ref_program(model, root + 4096 + ((va >> 21) & 511) * 8, + ((root + 8192) >> 12) << 10 | 1, 0); + for (int i = 0; i < pages; i++) begin + u64 a = (va & ~64'hfff) + i * 4096; + // Deliberately noncontiguous 4KiB physical pages; no identity-map shortcut. + preflight_ref_program(model, root + 8192 + ((a >> 12) & 511) * 8, + ((64'h800000 + i * 8192) >> 12) << 10 | flags, i == backend_error_page); + end + return root >> 12; + endfunction + function void predict(command_t q); + int id = q[`PF(CMD, ID)]; + int count; + verify_contract(!live.exists(id), "Duplicate admitted command ID"); + verify_contract(preflight_ref_submit( + model, + id, + q[ + `PF(CMD, BASEVA) + ], + q[ + `PF(CMD, ROWS) + ], + q[ + `PF(CMD, COLUMNS) + ], + q[ + `PF(CMD, SPANBYTES) + ], + q[ + `PF(CMD, COLUMNSTRIDE) + ], + q[ + `PF(CMD, ROWSTRIDE) + ], + q[ + `PF(CMD, WRITE) + ], + q[ + `PF(CMD, MODE) + ], + q[ + `PF(CMD, ROOTPPN) + ], + q[ + `PF(CMD, PRIVILEGE) + ], + q[ + `PF(CMD, SUM) + ], + q[ + `PF(CMD, MXR) + ] + ) == 0, "Uninitialized PTE in reference fixture"); + live[id] = 1; + output_index[id] = 0; + submitted++; + if (live.num() > peak) peak = live.num(); + count = preflight_ref_count(model, id); + for (int i = 0; i < count; i++) begin + result_item expected = new; + u64 va, pa; + int unsigned bytes, write, last, error; + preflight_ref_expected(model, id, i, va, pa, bytes, write, last, error); + expected.packet = '0; + expected.packet[`PF(OUT, ID)] = id; + expected.packet[`PF(OUT, VA)] = va; + expected.packet[`PF(OUT, PA)] = pa; + expected.packet[`PF(OUT, BYTES)] = bytes; + expected.packet[`PF(OUT, WRITE)] = write; + expected.packet[`PF(OUT, LAST)] = last; + expected.packet[`PF(OUT, ERROR)] = error; + expected.set_transaction_id((id << 4) | i); + scoreboard.expected_export.write(expected); + end + endfunction + function void final_phase(uvm_phase phase); + preflight_ref_destroy(model); + super.final_phase(phase); + endfunction + `include "agent/service.svh" + `include "seq/scenario.svh" +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/filelist.f b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/filelist.f new file mode 100644 index 000000000..481c37ef5 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Preflight/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/control_if.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/interface.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/preflight/src/main/resources/control_if.sv rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/interface.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/package.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/package.sv new file mode 100644 index 000000000..f42b8d59c --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/package.sv @@ -0,0 +1,96 @@ +`include "parameters.svh" +`define PF(K, F) `PF_``K``_``F``_OFFSET +: `PF_``K``_``F``_WIDTH +package preflight_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + typedef longint unsigned u64; + typedef bit [`PF_CMD_WIDTH-1:0] command_t; + import "DPI-C" function chandle preflight_ref_create( + input int unsigned address_bits, + beat + ); + import "DPI-C" function void preflight_ref_destroy(input chandle model); + import "DPI-C" function void preflight_ref_program( + input chandle model, + input longint unsigned address, + value, + input int unsigned error + ); + import "DPI-C" function void preflight_ref_policy( + input chandle model, + input int unsigned pte_valid, + input longint unsigned pte, + input int unsigned range_valid, + input longint unsigned first, + last + ); + import "DPI-C" function int unsigned preflight_ref_submit( + input chandle model, + input int unsigned id, + input longint unsigned base, + input int unsigned rows, + cols, + span, + col_stride, + row_stride, + write, + mode, + input longint unsigned root, + input int unsigned privilege, + sum, + mxr + ); + import "DPI-C" function int unsigned preflight_ref_count( + input chandle model, + input int unsigned id + ); + import "DPI-C" function void preflight_ref_expected( + input chandle model, + input int unsigned id, + index, + output longint unsigned va, + pa, + output int unsigned bytes, + write, + last, + error + ); + import "DPI-C" function int unsigned preflight_ref_authorize( + input chandle model, + input int unsigned id, + input longint unsigned pa, + input int unsigned bytes, + write, + pte, + privilege, + output int unsigned allow + ); + import "DPI-C" function int unsigned preflight_ref_pte( + input chandle model, + input int unsigned id, + input longint unsigned address, + output longint unsigned value, + output int unsigned error + ); + import "DPI-C" function int unsigned preflight_ref_pending( + input chandle model, + input int unsigned id + ); + import "DPI-C" function void preflight_ref_retire( + input chandle model, + input int unsigned id + ); + import "DPI-C" function void preflight_ref_reset(input chandle model); + class result_item extends uvm_sequence_item; + bit [`PF_OUT_WIDTH-1:0] packet; + `uvm_object_utils_begin(result_item) + `uvm_field_int(packet, UVM_DEFAULT) + `uvm_object_utils_end + function new(string name = "result_item"); + super.new(name); + endfunction + endclass + `include "env/env.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/seq/scenario.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/seq/scenario.svh new file mode 100644 index 000000000..966fe31d4 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/seq/scenario.svh @@ -0,0 +1,174 @@ +task submit(command_t q); + @(negedge ctl.clock); + cmd.bits = q; + cmd.valid = 1; + do @(cmd.sample); while (!cmd.sample.ready); + @(negedge ctl.clock); + cmd.valid = 0; + cmd.bits = ~q; +endtask +task drain(); + wait (live.num() == 0); + @(ctl.sample); + verify_contract(!auth_pending && !pte_pending && !pte_grant, + "Preparation endpoints did not drain"); +endtask +task execute(); + u64 root, root2; + int prior, gap_stride; + command_t q; + ctl.reset = 1; + cmd.valid = 0; + cmd.bits = '0; + prepared.ready = 0; + authorization.ready = 0; + permission.valid = 0; + permission.bits = '0; + pte_req.ready = 0; + pte_resp.valid = 0; + pte_resp.bits = '0; + repeat (5) @(negedge ctl.clock); + ctl.reset = 0; + fork + service(); + join_none + // Current BERT maximum: 1024 dense 16B rows normalize to five pages. + root = mapping('h400000f0, 9); + root2 = mapping('h50000000, 9); + hold_output = 1; + submit(request(1, 'h400000f0, 1024, 16, 16, 1, 0, 0, 8, root)); + submit(request(2, 'h50000000, 2048, 16, 16, 1, 0, 1, 8, root2)); // exactly eight pages + submit(request(3, 'h60000ff8, 1, 16, 16)); // alignment-expanded beat crosses page + submit(request(4, 'h70000000, 1, 16, 16)); + wait (prepared.sample.valid); + repeat (8) @(ctl.sample); + fork + submit(request(5, 'h70002000, 1, 16, 16)); + begin + repeat (12) @(ctl.sample); + hold_output = 0; + end + join + drain(); + submit(request(6, 'h50000000, 2304, 16, 16, 1, 0, 0, 8, root2)); + drain(); // ninth page: one error, zero successful output + submit(request(7, 'h60000000, 2, 16, 8192, 2, 4096)); + drain(); // sparse 2D, four extents + submit(request(8, 'h60000000, 1, 16, 16)); + drain(); // progress immediately after capacity error + root = mapping('h40010000, 1, 'h43); + submit(request(9, 'h40010000, 1, 16, 16, 1, 0, 1, 8, root)); + drain(); + root = mapping('h40020000, 1, 'h7); + submit(request(10, 'h40020000, 1, 16, 16, 1, 0, 0, 8, root)); + drain(); + root = mapping('h40030000, 1, 'hc7, 0); + submit(request(11, 'h40030000, 1, 16, 16, 1, 0, 0, 8, root)); + drain(); + root = mapping('h40040000, 1); + preflight_ref_policy(model, 1, (root << 12) + ((64'h40040000 >> 30) & 511) * 8, 0, 0, 0); + submit(request(12, 'h40040000, 1, 16, 16, 1, 0, 0, 8, root)); + drain(); + preflight_ref_policy(model, 0, 0, 1, 'h800000, 'h80000f); + submit(request(13, 'h40040000, 1, 16, 16, 1, 0, 0, 8, root)); + drain(); + preflight_ref_policy(model, 0, 0, 1, 'h71000000, 'h71000003); + // Denied bytes precede the user span: the aligned transport range is still authorized. + submit(request(14, 'h71000004, 1, 4, 4)); + drain(); + preflight_ref_policy(model, 0, 0, 0, 0, 0); + submit(request(15, (64'h1 << `PF_ADDRESS_BITS) - 16, 1, 32, 32)); + drain(); + submit(request(16, 64'hfffffffffffffff8, 1, 16, 16)); + drain(); + submit(request(17, 'h80000000, 0, 16, 16)); + drain(); + q = request(18, 'h80000000, 1, 16, 16); + q[`PF(CMD, MODE)] = 9; + submit(q); + drain(); + root = mapping('h40050000, 1, 'h59); + submit(request(19, 'h40050000, 1, 16, 16, 1, 0, 0, 8, root, 1, 1, 1)); + drain(); + // External snapshot service captures policy at command acceptance, not from later live settings. + hold_permission = 1; + submit(request(20, 'h72000000, 1, 16, 16)); + preflight_ref_policy(model, 0, 0, 1, 'h72000000, 'h7200000f); + wait (auth_pending); + repeat (8) @(ctl.sample); + hold_permission = 0; + drain(); + preflight_ref_policy(model, 0, 0, 0, 0, 0); + submit(request(21, 'h73000000, 1, 16, 256, 16, 16)); + drain(); // columns retains all 32 input bits + q = request(22, 64'hfffffffffffff000, -1, -1, -1, -1, -1); + submit(q); + drain(); + // Write authorization covers actual bytes; read authorization covers transport padding. + prior = failures[4]; + preflight_ref_policy(model, 0, 0, 1, 'h74000000, 'h74000003); + submit(request(23, 'h74000004, 1, 4, 4, 1, 0, 1)); + drain(); + verify_contract(failures[4] == prior, "write was denied for untouched prefix padding"); + submit(request(24, 'h74000004, 1, 4, 4)); + drain(); + verify_contract(failures[4] == prior + 1, "read failed to authorize actual prefix overread"); + prior = failures[4]; + preflight_ref_policy(model, 0, 0, 1, 'h74000108, 'h7400013f); + submit(request(25, 'h74000104, 1, 4, 4, 1, 0, 1)); + drain(); + verify_contract(failures[4] == prior, "write was denied for untouched tail padding"); + submit(request(26, 'h74000104, 1, 4, 4)); + drain(); + verify_contract(failures[4] == prior + 1, "read failed to authorize actual tail overread"); + prior = failures[4]; + preflight_ref_policy(model, 0, 0, 1, 'h74002ff0, 'h74002ffb); + submit(request(27, 'h74002ffc, 1, 8, 8, 1, 0, 1)); + drain(); + verify_contract(failures[4] == prior, "cross-page write included untouched bytes"); + submit(request(28, 'h74002ffc, 1, 8, 8)); + drain(); + verify_contract(failures[4] == prior + 1, "cross-page read omitted padded transport bytes"); + preflight_ref_policy(model, 0, 0, 0, 0, 0); + // MobileNet's gather has nine aligned segments on two noncontiguous PA pages. + // Every same-page gap must be explicitly authorized before the plan retains a union. + root = mapping('h74003e00, 2); + prior = failures[5]; + submit(request(29, 'h74003e00, 1, 16, 4032, 8, 72, 0, 8, root)); + drain(); + verify_contract(failures[5] == prior, "Authorized MobileNet gather exceeded range capacity"); + // Splitting preserves the exact accessed bytes and keeps each list within eight records. + prior = failures[5]; + submit(request(30, 'h74003e00, 1, 16, 4032, 4, 72, 0, 8, root)); + drain(); + submit(request(31, 'h74003f20, 1, 16, 4032, 4, 72, 0, 8, root)); + drain(); + verify_contract(failures[5] == prior, "Split MobileNet gather exceeded range capacity"); + // A denied hole must keep both individually legal spans, rather than poison the command. + gap_stride = `PF_BEAT_BYTES == 16 ? 72 : 2 * `PF_BEAT_BYTES + 8; + preflight_ref_policy(model, 0, 0, 1, 'h800e00 + `PF_BEAT_BYTES, + 'h800e00 + (gap_stride / `PF_BEAT_BYTES) * `PF_BEAT_BYTES - 1); + prior = failures[4]; + submit(request(32, 'h74003e00, 1, 16, 4032, 2, gap_stride, 0, 8, root)); + drain(); + verify_contract(failures[4] == prior, "Denied union rejected individually authorized pixels"); + // Denial of an actual transport byte must still yield one terminal fault, no partial plan. + preflight_ref_policy(model, 0, 0, 1, 'h800e00 + gap_stride, 'h800e00 + gap_stride + 7); + submit(request(33, 'h74003e00, 1, 16, 4032, 2, gap_stride, 0, 8, root)); + drain(); + verify_contract(failures[4] == prior + 1, "Actual MobileNet pixel bytes escaped authorization"); + preflight_ref_policy(model, 0, 0, 0, 0, 0); + verify_contract( + peak==4&&submitted==retired&&command_stalls>0&&output_stalls>0&&auth_stalls>0, + "Missing concurrency/backpressure/drain coverage"); + for (int error = 1; error < 7; error++) + verify_contract(failures[error] > 0, "Missing typed error scenario"); + verify_contract(scoreboard.checked == segments, "Not every output descriptor was compared"); + `uvm_info( + "PREFLIGHT_PASS", + $sformatf( + "commands=%0d descriptors=%0d peak=%0d PTE=%0d auth=%0d errors shape/overflow/PF/AF/capacity/context=%0d/%0d/%0d/%0d/%0d/%0d stalls cmd/out/auth=%0d/%0d/%0d; all plans/permissions drained", + retired, segments, peak, pte_count, auth_count, failures[1], failures[2], failures[3], + failures[4], failures[5], failures[6], command_stalls, output_stalls, auth_stalls), + UVM_LOW) +endtask diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/tb.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/tb.sv new file mode 100644 index 000000000..ba54c8e3b --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/tb.sv @@ -0,0 +1,52 @@ +`include "parameters.svh" +module preflight_tb; + import uvm_pkg::*; + import preflight_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + preflight_control_if ctl (clock); + stream_if #(`PF_CMD_WIDTH) cmd ( + clock, + ctl.reset + ); + stream_if #(`PF_OUT_WIDTH) prepared ( + clock, + ctl.reset + ); + stream_if #(`PF_AUTH_WIDTH) authorization ( + clock, + ctl.reset + ); + stream_if #(`PF_PERMIT_WIDTH) permission ( + clock, + ctl.reset + ); + stream_if #(`PF_PTE_WIDTH) pte_req ( + clock, + ctl.reset + ); + stream_if #(`PF_PTERESP_WIDTH) pte_resp ( + clock, + ctl.reset + ); + Preflight dut ( + .clock(clock), + .reset(ctl.reset), + `include "preflight_ports.svh" + ); + initial begin + uvm_config_db#(virtual preflight_control_if)::set(null, "uvm_test_top.env", "ctl", ctl); + uvm_config_db#(virtual stream_if #(`PF_CMD_WIDTH))::set(null, "uvm_test_top.env", "cmd", cmd); + uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::set(null, "uvm_test_top.env", "prepared", + prepared); + uvm_config_db#(virtual stream_if #(`PF_AUTH_WIDTH))::set(null, "uvm_test_top.env", + "authorization", authorization); + uvm_config_db#(virtual stream_if #(`PF_PERMIT_WIDTH))::set(null, "uvm_test_top.env", + "permission", permission); + uvm_config_db#(virtual stream_if #(`PF_PTE_WIDTH))::set(null, "uvm_test_top.env", "pte_req", + pte_req); + uvm_config_db#(virtual stream_if #(`PF_PTERESP_WIDTH))::set(null, "uvm_test_top.env", + "pte_resp", pte_resp); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/tests/test.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/tests/test.svh new file mode 100644 index 000000000..398043c00 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preflight/tests/test.svh @@ -0,0 +1,14 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + test_env.execute(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/driver.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/driver.svh new file mode 100644 index 000000000..d57501589 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual preparation_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual preparation_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.source = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/fixture.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/fixture.svh new file mode 100644 index 000000000..80199dac4 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/fixture.svh @@ -0,0 +1,8 @@ +class fixture extends uvm_object; + `uvm_object_utils(fixture) + int policy[1<<`PREP_ID_BITS]; + bit pte_error; + function new(string name = "fixture"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/item.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/item.svh new file mode 100644 index 000000000..75524d2ad --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/item.svh @@ -0,0 +1,18 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`PREP_INPUT_BITS-1:0] inputs; + logic [`PREP_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_object_utils_end + bit [511:0] value; + function new(string name = "result"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/monitor.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/monitor.svh new file mode 100644 index 000000000..8a27911bb --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual preparation_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual preparation_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/peer.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/peer.svh new file mode 100644 index 000000000..3408512a7 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/agent/peer.svh @@ -0,0 +1,81 @@ +class peer extends uvm_component; + `uvm_component_utils(peer) + virtual preparation_if vif; + fixture image; + bit pending; + int cycle, due; + bit error; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual preparation_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Preparation interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(posedge vif.clock); + cycle++; + if (vif.reset) pending = 0; + else begin + if (`PO(PTERESPONSE_READY) && `PI(PTERESPONSE_VALID)) pending = 0; + if (`PO(PTEREQUEST_VALID) && `PI(PTEREQUEST_READY)) begin + pending = 1; + due = cycle + 3; + error = image.pte_error; + end + end + @(negedge vif.clock); + vif.backend = '0; + begin + int id = `PO(CONTEXTTAG); + int policy = image.policy[id]; + vif.backend[`PREP_IN_CONTEXTVALID_OFFSET+:`PREP_IN_CONTEXTVALID_WIDTH] = policy != 4; + vif.backend[`PREP_IN_CONTEXTID_OFFSET+:`PREP_IN_CONTEXTID_WIDTH] = id; + vif.backend[`PREP_IN_CONTEXTPMP_0_ADDR_OFFSET+:`PREP_IN_CONTEXTPMP_0_ADDR_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_0_MASK_OFFSET+:`PREP_IN_CONTEXTPMP_0_MASK_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_A_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_A_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_L_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_L_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_R_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_R_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_W_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_W_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_1_ADDR_OFFSET+:`PREP_IN_CONTEXTPMP_1_ADDR_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_1_MASK_OFFSET+:`PREP_IN_CONTEXTPMP_1_MASK_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_1_CFG_A_OFFSET+:`PREP_IN_CONTEXTPMP_1_CFG_A_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_1_CFG_L_OFFSET+:`PREP_IN_CONTEXTPMP_1_CFG_L_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_1_CFG_R_OFFSET+:`PREP_IN_CONTEXTPMP_1_CFG_R_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_1_CFG_W_OFFSET+:`PREP_IN_CONTEXTPMP_1_CFG_W_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_2_ADDR_OFFSET+:`PREP_IN_CONTEXTPMP_2_ADDR_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_2_MASK_OFFSET+:`PREP_IN_CONTEXTPMP_2_MASK_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_2_CFG_A_OFFSET+:`PREP_IN_CONTEXTPMP_2_CFG_A_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_2_CFG_L_OFFSET+:`PREP_IN_CONTEXTPMP_2_CFG_L_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_2_CFG_R_OFFSET+:`PREP_IN_CONTEXTPMP_2_CFG_R_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_2_CFG_W_OFFSET+:`PREP_IN_CONTEXTPMP_2_CFG_W_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_3_ADDR_OFFSET+:`PREP_IN_CONTEXTPMP_3_ADDR_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_3_MASK_OFFSET+:`PREP_IN_CONTEXTPMP_3_MASK_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_3_CFG_A_OFFSET+:`PREP_IN_CONTEXTPMP_3_CFG_A_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_3_CFG_L_OFFSET+:`PREP_IN_CONTEXTPMP_3_CFG_L_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_3_CFG_R_OFFSET+:`PREP_IN_CONTEXTPMP_3_CFG_R_WIDTH] = 0; + vif.backend[`PREP_IN_CONTEXTPMP_3_CFG_W_OFFSET+:`PREP_IN_CONTEXTPMP_3_CFG_W_WIDTH] = 0; + if (policy == 1 || policy == 2 || policy == 6) begin + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_A_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_A_WIDTH] = 3; + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_L_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_L_WIDTH] = 1; + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_R_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_R_WIDTH]=policy!=1; + vif.backend[`PREP_IN_CONTEXTPMP_0_CFG_W_OFFSET+:`PREP_IN_CONTEXTPMP_0_CFG_W_WIDTH]=policy==6; + vif.backend[`PREP_IN_CONTEXTPMP_0_ADDR_OFFSET+:`PREP_IN_CONTEXTPMP_0_ADDR_WIDTH] = '1; + vif.backend[`PREP_IN_CONTEXTPMP_0_MASK_OFFSET+:`PREP_IN_CONTEXTPMP_0_MASK_WIDTH] = '1; + end + if (policy == 3) begin + vif.backend[`PREP_IN_CONTEXTPMP_0_ADDR_OFFSET+:`PREP_IN_CONTEXTPMP_0_ADDR_WIDTH]='h2008>>2; + vif.backend[`PREP_IN_CONTEXTPMP_1_ADDR_OFFSET+:`PREP_IN_CONTEXTPMP_1_ADDR_WIDTH]='h2004>>2; + vif.backend[`PREP_IN_CONTEXTPMP_1_CFG_A_OFFSET+:`PREP_IN_CONTEXTPMP_1_CFG_A_WIDTH] = 1; + vif.backend[`PREP_IN_CONTEXTPMP_1_CFG_L_OFFSET+:`PREP_IN_CONTEXTPMP_1_CFG_L_WIDTH] = 1; + end + end + vif.backend[`PREP_IN_PTEREQUEST_READY_OFFSET+:`PREP_IN_PTEREQUEST_READY_WIDTH]=cycle%3!=0; + vif.backend[`PREP_IN_PTERESPONSE_VALID_OFFSET+:`PREP_IN_PTERESPONSE_VALID_WIDTH]=pending&&cycle>=due; + vif.backend[`PREP_IN_PTERESPONSE_BITS_DATA_OFFSET+:`PREP_IN_PTERESPONSE_BITS_DATA_WIDTH]=64'hcf; + vif.backend[`PREP_IN_PTERESPONSE_BITS_ERROR_OFFSET+:`PREP_IN_PTERESPONSE_BITS_ERROR_WIDTH]=error; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/env.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/env.svh new file mode 100644 index 000000000..eeda0417e --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/env.svh @@ -0,0 +1,36 @@ +class env extends uvm_env; + `uvm_component_utils(env) + uvm_analysis_imp #(uvm_sequence_item, env) reset_events; + driver source; + monitor mon; + peer backend; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 1) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_events = new("reset_events", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + backend = peer::type_id::create("backend", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 1)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + fixture image = fixture::type_id::create("image"); + reference.image = image; + backend.image = image; + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + reference.cancelled.connect(reset_events); + endfunction + function void write(uvm_sequence_item t); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/model.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/model.svh new file mode 100644 index 000000000..36d8309d5 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/model.svh @@ -0,0 +1,205 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + fixture image; + chandle reference; + uvm_analysis_port #(result_item) expected, actual; + uvm_analysis_port #(uvm_sequence_item) cancelled; + int index[int], errors[int], counts[int]; + longint unsigned addresses[int][int], physical[int][int]; + int lengths[int][int]; + bit direction[int], notified[int], live[int]; + longint query_key; + covergroup behavior with function sample ( + int error, bit writing, int mode, int policy, bit hit, bit released + ); + option.per_instance = 1; + outcomes: coverpoint error { + bins success = {0}; + bins shape = {1}; + bins page_fault = {3}; + bins access_fault = {4}; + bins capacity = {5}; + } + directions: coverpoint writing; + modes: coverpoint mode {bins all[] = {0, 8};} + pmp: coverpoint policy {bins all[] = {0, 1, 2, 3, 4, 6};} + queries: coverpoint hit; + release_events: coverpoint released; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + cancelled = new("cancelled", this); + reference = preflight_ref_create(`PREP_ADDRESS_BITS, 16); + behavior = new(); + endfunction + function void verify(bit ok, string message); + evidence::checks++; + if (!ok) `uvm_fatal("PREPARATION", message) + endfunction + function void predict(cycle_item t); + int id, policy, count; + bit deny; + int mode; + longint unsigned base; + id = `AI(t, COMMAND_BITS_ID); + policy = image.policy[id]; + mode = `AI(t, COMMAND_BITS_MODE); + base = `AI(t, COMMAND_BITS_BASEVA); + deny = policy == 1 || policy == 4 || (policy == 2 && `AI(t, COMMAND_BITS_WRITE)); + preflight_ref_program(reference, 'h10000, 'hcf, image.pte_error); + preflight_ref_policy(reference, deny && mode == 8, 'h10000, deny, 0, + (64'd1 << `PREP_ADDRESS_BITS) - 1); + verify(preflight_ref_submit( + reference, + id, + base, + `AI(t, COMMAND_BITS_ROWS), + `AI(t, COMMAND_BITS_COLUMNS), + `AI(t, COMMAND_BITS_SPANBYTES), + `AI(t, COMMAND_BITS_COLUMNSTRIDE), + `AI(t, COMMAND_BITS_ROWSTRIDE), + `AI(t, COMMAND_BITS_WRITE), + mode, + `AI(t, COMMAND_BITS_ROOTPPN), + `AI(t, COMMAND_BITS_PRIVILEGE), + `AI(t, COMMAND_BITS_SUM), + `AI(t, COMMAND_BITS_MXR) + ) == 0, "Missing preparation fixture"); + verify(!live.exists(id), "Duplicate live command ID"); + live[id] = 1; + direction[id] = `AI(t, COMMAND_BITS_WRITE); + index[id] = 0; + count = preflight_ref_count(reference, id); + counts[id] = count; + for (int i = 0; i < count; i++) begin + result_item result = result_item::type_id::create("range_expected"); + longint unsigned va, pa; + int bytes, writing, last, error; + preflight_ref_expected(reference, id, i, va, pa, bytes, writing, last, error); + result.value = {32'(id), va, pa, 32'(bytes), 32'(writing), 32'(last), 32'(error)}; + result.set_transaction_id((id << 4) | i); + expected.write(result); + addresses[id][i] = va; + physical[id][i] = pa; + lengths[id][i] = bytes; + if (last) begin + result = result_item::type_id::create("ready_expected"); + result.value = {32'(id), 32'(error), error == 0 ? 64'd0 : va}; + result.set_transaction_id((1 << 20) | id); + expected.write(result); + errors[id] = error; + behavior.sample(error, writing, mode, policy, 0, 0); + end + end + endfunction + function void query(cycle_item t, int port, int valid, int id, longint unsigned va, int bytes, + bit writing, bit hit, longint unsigned pa, int error); + bit expected_hit, shape; + longint unsigned expected_pa; + int expected_error; + logic [64:0] finish_address = {1'b0, va} + bytes; + bit releasing = `AI(t, RELEASE_VALID) && `AO(t, RELEASE_READY) && `AI(t, RELEASE_BITS_ID) == id; + shape = bytes > 0 && !finish_address[64] && int'(va & 4095) + bytes <= 4096; + expected_hit = 0; + expected_pa = 0; + if (valid && shape && live.exists( + id + ) && notified.exists( + id + ) && errors[id] == 0 && !releasing && direction[id] == writing) + for (int i = 0; i < counts[id]; i++) + if (va >= addresses[id][i] && finish_address <= addresses[id][i] + lengths[id][i]) begin + expected_hit = 1; + expected_pa = physical[id][i] + va - addresses[id][i]; + end + expected_error = !valid || expected_hit ? 0 : (!shape ? 1 : 4); + verify(hit == expected_hit && pa == expected_pa && error == expected_error, $sformatf( + "Query port%0d id%0d VA%h expected hit%0d PA%h error%0d got hit%0d PA%h error%0d", + port, + id, + va, + expected_hit, + expected_pa, + expected_error, + hit, + pa, + error + )); + if (valid) behavior.sample(expected_error, writing, 0, 0, hit, releasing); + endfunction + function void write(cycle_item t); + if (t.reset) begin + cancelled.write(null); + preflight_ref_reset(reference); + index.delete(); + errors.delete(); + counts.delete(); + addresses.delete(); + physical.delete(); + lengths.delete(); + direction.delete(); + notified.delete(); + live.delete(); + return; + end + query(t, 0, `AI(t, QUERIES_0_VALID), `AI(t, QUERIES_0_ID), `AI(t, QUERIES_0_VA), + `AI(t, QUERIES_0_BYTES), `AI(t, QUERIES_0_WRITE), `AO(t, RESULTS_0_HIT), + `AO(t, RESULTS_0_PA), `AO(t, RESULTS_0_ERROR)); + query(t, 1, `AI(t, QUERIES_1_VALID), `AI(t, QUERIES_1_ID), `AI(t, QUERIES_1_VA), + `AI(t, QUERIES_1_BYTES), `AI(t, QUERIES_1_WRITE), `AO(t, RESULTS_1_HIT), + `AO(t, RESULTS_1_PA), `AO(t, RESULTS_1_ERROR)); + if (`AI(t, COMMAND_VALID) && `AO(t, COMMAND_READY)) predict(t); + if (`AO(t, PTEREQUEST_VALID) && `AI(t, PTEREQUEST_READY)) begin + longint unsigned value; + int error, id; + id = `AO(t, CONTEXTTAG); + verify(preflight_ref_pte(reference, id, `AO(t, PTEREQUEST_BITS_ADDR), value, error) == 1, + "PTE request differs from preparation plan"); + end + if (`AO(t, RANGES_VALID) && `AI(t, RANGES_READY)) begin + int id = `AO(t, RANGES_BITS_ID); + result_item result = result_item::type_id::create("range_actual"); + verify(live.exists(id), "Prepared range has no live command"); + result.value = { + 32'(id), + 64'(`AO(t, RANGES_BITS_VA)), + 64'(`AO(t, RANGES_BITS_PA)), + 32'(`AO(t, RANGES_BITS_BYTES)), + 32'(`AO(t, RANGES_BITS_WRITE)), + 32'(`AO(t, RANGES_BITS_LAST)), + 32'(`AO(t, RANGES_BITS_ERROR)) + }; + result.set_transaction_id((id << 4) | index[id]); + actual.write(result); + index[id]++; + end + if (`AO(t, READY_VALID) && `AI(t, READY_READY)) begin + int id = `AO(t, READY_BITS_ID); + result_item result = result_item::type_id::create("ready_actual"); + verify(live.exists(id) && index[id] == counts[id], + "Notification before all ranges were retained"); + result.value = {32'(id), 32'(`AO(t, READY_BITS_ERROR)), 64'(`AO(t, READY_BITS_VA))}; + result.set_transaction_id((1 << 20) | id); + actual.write(result); + notified[id] = 1; + end + if (`AI(t, RELEASE_VALID) && `AO(t, RELEASE_READY)) begin + int id = `AI(t, RELEASE_BITS_ID); + verify(notified.exists(id), "Released a mapping before notification"); + preflight_ref_retire(reference, id); + live.delete(id); + notified.delete(id); + behavior.sample(errors[id], direction[id], 0, 0, 0, 1); + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + verify(live.num() == 0, "Mappings did not release"); + endfunction + function void final_phase(uvm_phase phase); + preflight_ref_destroy(reference); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/filelist.f b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/filelist.f new file mode 100644 index 000000000..3cda6a201 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Preparation/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/interface.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/interface.sv new file mode 100644 index 000000000..92bb989a4 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/interface.sv @@ -0,0 +1,9 @@ +`include "preparation_parameters.svh" +interface preparation_if ( + input logic clock +); + logic reset = 1; + logic [`PREP_INPUT_BITS-1:0] source = '0, backend = '0; + wire [ `PREP_INPUT_BITS-1:0] inputs = source | backend; + wire [`PREP_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/package.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/package.sv new file mode 100644 index 000000000..348bcb50b --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/package.sv @@ -0,0 +1,97 @@ +`include "preparation_parameters.svh" +`define PI(F) vif.inputs[`PREP_IN_``F``_OFFSET+:`PREP_IN_``F``_WIDTH] +`define PO(F) vif.outputs[`PREP_OUT_``F``_OFFSET+:`PREP_OUT_``F``_WIDTH] +`define PB(F) vif.backend[`PREP_IN_``F``_OFFSET+:`PREP_IN_``F``_WIDTH] +`define PS(F) pins[`PREP_IN_``F``_OFFSET+:`PREP_IN_``F``_WIDTH] +`define AI(T, F) T.inputs[`PREP_IN_``F``_OFFSET+:`PREP_IN_``F``_WIDTH] +`define AO(T, F) T.outputs[`PREP_OUT_``F``_OFFSET+:`PREP_OUT_``F``_WIDTH] +package preparation_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function chandle preflight_ref_create( + input int unsigned address_bits, + beat + ); + import "DPI-C" function void preflight_ref_destroy(input chandle model); + import "DPI-C" function void preflight_ref_program( + input chandle model, + input longint unsigned address, + value, + input int unsigned error + ); + import "DPI-C" function void preflight_ref_policy( + input chandle model, + input int unsigned pte_valid, + input longint unsigned pte, + input int unsigned range_valid, + input longint unsigned first, + last + ); + import "DPI-C" function int unsigned preflight_ref_submit( + input chandle model, + input int unsigned id, + input longint unsigned base, + input int unsigned rows, + cols, + span, + col_stride, + row_stride, + write, + mode, + input longint unsigned root, + input int unsigned privilege, + sum, + mxr + ); + import "DPI-C" function int unsigned preflight_ref_count( + input chandle model, + input int unsigned id + ); + import "DPI-C" function void preflight_ref_expected( + input chandle model, + input int unsigned id, + index, + output longint unsigned va, + pa, + output int unsigned bytes, + write, + last, + error + ); + import "DPI-C" function int unsigned preflight_ref_authorize( + input chandle model, + input int unsigned id, + input longint unsigned pa, + input int unsigned bytes, + write, + pte, + privilege, + output int unsigned allow + ); + import "DPI-C" function int unsigned preflight_ref_pte( + input chandle model, + input int unsigned id, + input longint unsigned address, + output longint unsigned value, + output int unsigned error + ); + import "DPI-C" function int unsigned preflight_ref_pending( + input chandle model, + input int unsigned id + ); + import "DPI-C" function void preflight_ref_retire( + input chandle model, + input int unsigned id + ); + import "DPI-C" function void preflight_ref_reset(input chandle model); + `include "agent/item.svh" + `include "agent/fixture.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/peer.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/seq/sequence.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/seq/sequence.svh new file mode 100644 index 000000000..18dd3141d --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/seq/sequence.svh @@ -0,0 +1,113 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + model reference; + logic [`PREP_INPUT_BITS-1:0] pins; + logic [`PREP_OUTPUT_BITS-1:0] observed; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task submit(int id, longint unsigned base, int rows, int columns, int span, int column_stride, + int row_stride, bit writing = 0, int policy = 0, int mode = 0, int privilege = 3); + reference.image.policy[id] = policy; + pins[`PREP_IN_COMMAND_BITS_ID_OFFSET+:`PREP_IN_COMMAND_BITS_ID_WIDTH] = id; + pins[`PREP_IN_COMMAND_BITS_BASEVA_OFFSET+:`PREP_IN_COMMAND_BITS_BASEVA_WIDTH] = base; + pins[`PREP_IN_COMMAND_BITS_ROWS_OFFSET+:`PREP_IN_COMMAND_BITS_ROWS_WIDTH] = rows; + pins[`PREP_IN_COMMAND_BITS_COLUMNS_OFFSET+:`PREP_IN_COMMAND_BITS_COLUMNS_WIDTH] = columns; + pins[`PREP_IN_COMMAND_BITS_SPANBYTES_OFFSET+:`PREP_IN_COMMAND_BITS_SPANBYTES_WIDTH] = span; + pins[`PREP_IN_COMMAND_BITS_COLUMNSTRIDE_OFFSET+:`PREP_IN_COMMAND_BITS_COLUMNSTRIDE_WIDTH]=column_stride; + pins[`PREP_IN_COMMAND_BITS_ROWSTRIDE_OFFSET+:`PREP_IN_COMMAND_BITS_ROWSTRIDE_WIDTH]=row_stride; + pins[`PREP_IN_COMMAND_BITS_WRITE_OFFSET+:`PREP_IN_COMMAND_BITS_WRITE_WIDTH] = writing; + pins[`PREP_IN_COMMAND_BITS_MODE_OFFSET+:`PREP_IN_COMMAND_BITS_MODE_WIDTH] = mode; + pins[`PREP_IN_COMMAND_BITS_PRIVILEGE_OFFSET+:`PREP_IN_COMMAND_BITS_PRIVILEGE_WIDTH] = privilege; + pins[`PREP_IN_COMMAND_BITS_ROOTPPN_OFFSET+:`PREP_IN_COMMAND_BITS_ROOTPPN_WIDTH] = 16; + pins[`PREP_IN_COMMAND_VALID_OFFSET+:`PREP_IN_COMMAND_VALID_WIDTH] = 1; + do tick(); while (!observed[`PREP_OUT_COMMAND_READY_OFFSET]); + pins[`PREP_IN_COMMAND_VALID_OFFSET+:`PREP_IN_COMMAND_VALID_WIDTH] = 0; + endtask + task notified(int id); + do tick(); while (!reference.notified.exists(id)); + endtask + task release_tag(int id); + pins[`PREP_IN_RELEASE_BITS_ID_OFFSET+:`PREP_IN_RELEASE_BITS_ID_WIDTH] = id; + pins[`PREP_IN_RELEASE_VALID_OFFSET+:`PREP_IN_RELEASE_VALID_WIDTH] = 1; + do tick(); while (!observed[`PREP_OUT_RELEASE_READY_OFFSET]); + pins[`PREP_IN_RELEASE_VALID_OFFSET+:`PREP_IN_RELEASE_VALID_WIDTH] = 0; + tick(); + endtask + task probe(int id, longint unsigned address, int bytes, bit writing = 0); + pins[`PREP_IN_QUERIES_0_VALID_OFFSET+:`PREP_IN_QUERIES_0_VALID_WIDTH] = 1; + pins[`PREP_IN_QUERIES_0_ID_OFFSET+:`PREP_IN_QUERIES_0_ID_WIDTH] = id; + pins[`PREP_IN_QUERIES_0_VA_OFFSET+:`PREP_IN_QUERIES_0_VA_WIDTH] = address; + pins[`PREP_IN_QUERIES_0_BYTES_OFFSET+:`PREP_IN_QUERIES_0_BYTES_WIDTH] = bytes; + pins[`PREP_IN_QUERIES_0_WRITE_OFFSET+:`PREP_IN_QUERIES_0_WRITE_WIDTH] = writing ^ 0; + pins[`PREP_IN_QUERIES_1_VALID_OFFSET+:`PREP_IN_QUERIES_1_VALID_WIDTH] = 1; + pins[`PREP_IN_QUERIES_1_ID_OFFSET+:`PREP_IN_QUERIES_1_ID_WIDTH] = id; + pins[`PREP_IN_QUERIES_1_VA_OFFSET+:`PREP_IN_QUERIES_1_VA_WIDTH] = address; + pins[`PREP_IN_QUERIES_1_BYTES_OFFSET+:`PREP_IN_QUERIES_1_BYTES_WIDTH] = bytes; + pins[`PREP_IN_QUERIES_1_WRITE_OFFSET+:`PREP_IN_QUERIES_1_WRITE_WIDTH] = writing ^ 1; + repeat (3) tick(); + endtask + task check_one(int id, longint unsigned base, int rows, int columns, int span, int column_stride, + int row_stride, bit writing = 0, int policy = 0, int mode = 0, int privilege = 3); + submit(id, base, rows, columns, span, column_stride, row_stride, writing, policy, mode, + privilege); + notified(id); + probe(id, base, 1, writing); + probe(id, base, 0, writing); + probe(id, base + 8192, 16, writing); + probe(id, base, 16, writing); + release_tag(id); + probe(id, base, 1, writing); + endtask + task body(); + pins = '0; + pins[`PREP_IN_RANGES_READY_OFFSET+:`PREP_IN_RANGES_READY_WIDTH] = 1; + pins[`PREP_IN_READY_READY_OFFSET+:`PREP_IN_READY_READY_WIDTH] = 1; + repeat (3) tick(1); + tick(); + check_one(0, 'h2000, 1, 1, 16, 16, 16); + check_one(1, 'h2ff7, 2, 1, 16, 16, 32); + check_one(2, 'h2000, 2, 3, 8, 24, 128, 1); + check_one(3, 'h4000, 9, 1, 16, 16, 4096, 1); + check_one(4, 'h4000, 0, 1, 16, 16, 16); + check_one(5, 'h2000, 1, 1, 16, 16, 16, 1, 1); + check_one(6, 'h2000, 1, 1, 16, 16, 16, 0, 2); + check_one(7, 'h2000, 1, 1, 16, 16, 16, 1, 2); + check_one(8, 'h2000, 1, 1, 16, 16, 16, 1, 3); + check_one(9, 'h2000, 1, 1, 16, 16, 16, 0, 4); + check_one(10, 'h5000, 2, 1, 16, 16, 32, 0, 6, 8, 1); + reference.image.pte_error = 1; + check_one(11, 'h5000, 1, 1, 16, 16, 16, 0, 6, 8, 1); + reference.image.pte_error = 0; + // An invalid root entry produces a page fault independently of permission failures. + check_one(12, 64'h4000000000, 1, 1, 16, 16, 16, 0, 6, 8, 1); + pins[`PREP_IN_READY_READY_OFFSET+:`PREP_IN_READY_READY_WIDTH] = 0; + pins[`PREP_IN_RANGES_READY_OFFSET+:`PREP_IN_RANGES_READY_WIDTH] = 0; + for (int id = 20; id < 24; id++) submit(id, 'h6000 + (id - 20) * 4096, 1, 1, 16, 16, 16); + repeat (8) tick(); + pins[`PREP_IN_RANGES_READY_OFFSET+:`PREP_IN_RANGES_READY_WIDTH] = 1; + repeat (40) tick(); + pins[`PREP_IN_READY_READY_OFFSET+:`PREP_IN_READY_READY_WIDTH] = 1; + for (int id = 20; id < 24; id++) begin + notified(id); + probe(id, 'h6000 + (id - 20) * 4096, 16); + end + for (int id = 20; id < 24; id++) release_tag(id); + pins[`PREP_IN_RANGES_READY_OFFSET+:`PREP_IN_RANGES_READY_WIDTH] = 0; + submit(30, 'h8000, 1, 1, 16, 16, 16); + repeat (10) tick(); + repeat (2) tick(1); + pins[`PREP_IN_RANGES_READY_OFFSET+:`PREP_IN_RANGES_READY_WIDTH] = 1; + tick(); + check_one(30, 'h8000, 1, 1, 16, 16, 16); + repeat (8) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/tb.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/tb.sv new file mode 100644 index 000000000..64475ed3b --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/tb.sv @@ -0,0 +1,15 @@ +module preparation_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + preparation_if bus (clock); + Preparation dut ( + .clock(clock), + .reset(bus.reset), + `include "preparation_ports.svh" + ); + initial begin + uvm_config_db#(virtual preparation_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/tests/test.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/tests/test.svh new file mode 100644 index 000000000..b11c898c9 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/tests/test.svh @@ -0,0 +1,16 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.reference = test_env.reference; + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/agent/service.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/agent/service.svh new file mode 100644 index 000000000..2f6400362 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/agent/service.svh @@ -0,0 +1,47 @@ +task monitor(); + forever begin + @(ctl.sample); + if (ctl.sample.reset) begin + pmap_ref_reset(model); + notified.delete(); + status.delete(); + end else begin + query_check(ctl.sample.query_0, ctl.sample.result_0); + query_check(ctl.sample.query_1, ctl.sample.result_1); + if (ctl.sample.query_0[`PF(QUERY, VALID)] && ctl.sample.query_1[`PF(QUERY, VALID)]) dual++; + if (mapped.sample.valid) begin + int error, id; + u64 va; + id = mapped.sample.bits[`PF(MAP_READY, ID)]; + verify_contract(pmap_ref_ready(model, id, mapped.sample.ready, error, va) == 1, + "unexpected notification"); + verify_contract(mapped.sample.bits[`PF(MAP_READY, ERROR) + ] === 3'(error) && mapped.sample.bits[`PF(MAP_READY, VA)] === va, + "notification mismatch"); + behavior.sample(0, 0, 1, error); + if (mapped.sample.ready) begin + notified[id] = 1; + status[id] = error; + notifications++; + end else stalls++; + end + if (retire.sample.valid && retire.sample.ready) begin + int id; + id = retire.sample.bits[`PF(RELEASE, ID)]; + pmap_ref_release(model, id); + notified.delete(id); + retired++; + end + if (reserve.sample.valid && reserve.sample.ready) begin + pmap_ref_reserve(model, reserve.sample.bits[`PF(RESERVE, ID)]); + allocations++; + end + if (prepared.sample.valid && prepared.sample.ready) + pmap_ref_prepared(model, prepared.sample.bits[`PF(OUT, ID)], prepared.sample.bits[ + `PF(OUT, VA)], prepared.sample.bits[`PF(OUT, PA)], prepared.sample.bits[ + `PF(OUT, BYTES)], prepared.sample.bits[`PF(OUT, WRITE)], + prepared.sample.bits[`PF(OUT, LAST)], prepared.sample.bits[`PF(OUT, ERROR) + ]); + end + end +endtask diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/env/env.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/env/env.svh new file mode 100644 index 000000000..14ab56009 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/env/env.svh @@ -0,0 +1,89 @@ +class env extends uvm_env; + `uvm_component_utils(env) + virtual prepared_map_control_if ctl; + virtual stream_if #(`PF_RESERVE_WIDTH) reserve; + virtual stream_if #(`PF_OUT_WIDTH) prepared; + virtual stream_if #(`PF_MAP_READY_WIDTH) mapped; + virtual stream_if #(`PF_RELEASE_WIDTH) retire; + chandle model; + bit notified[int]; + int status[int]; + int + queries = 0, + hits = 0, + misses = 0, + allocations = 0, + notifications = 0, + retired = 0, + stalls = 0, + dual = 0, + cancelled = 0; + covergroup behavior with function sample (bit hit, bit writing, bit valid, int error); + option.per_instance = 1; + hits: coverpoint hit; + direction: coverpoint writing; + enabled: coverpoint valid; + outcomes: coverpoint error {bins all[] = {[0 : 6]};} + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + + endfunction + function void verify_contract(bit ok, string message); + verification_pkg::evidence::checks++; + if (!ok) `uvm_fatal("PREPARED_MAP", message) + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + verify_contract(uvm_config_db#(virtual prepared_map_control_if)::get(this, "", "ctl", ctl), + "ctl missing"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_RESERVE_WIDTH))::get( + this, "", "reserve", reserve), "reserve missing"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::get( + this, "", "prepared", prepared), "prepared missing"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_MAP_READY_WIDTH))::get( + this, "", "mapped", mapped), "mapped missing"); + verify_contract(uvm_config_db#(virtual stream_if #(`PF_RELEASE_WIDTH))::get( + this, "", "retire", retire), "retire missing"); + model = pmap_ref_create(`PF_ADDRESS_BITS); + endfunction + function void query_check(logic [`PF_QUERY_WIDTH-1:0] q, logic [`PF_RESULT_WIDTH-1:0] actual); + int hit, error; + u64 pa; + pmap_ref_query(model, q[`PF(QUERY, VALID)], q[`PF(QUERY, ID)], q[`PF(QUERY, VA)], q[ + `PF(QUERY, BYTES)], q[`PF(QUERY, WRITE)], + retire.sample.valid && retire.sample.ready, retire.sample.bits[`PF(RELEASE, ID)], + hit, pa, error); + verify_contract(actual[`PF(RESULT, HIT)] === 1'(hit) && actual[`PF(RESULT, PA) + ] === `PF_ADDRESS_BITS'(pa) && actual[`PF(RESULT, ERROR)] === 3'(error), + $sformatf( + "query id%0d VA%h bytes%0d expected hit%0d PA%h error%0d got %h", + q[ + `PF(QUERY, ID) + ], + q[ + `PF(QUERY, VA) + ], + q[ + `PF(QUERY, BYTES) + ], + hit, + pa, + error, + actual + )); + behavior.sample(hit, q[`PF(QUERY, WRITE)], q[`PF(QUERY, VALID)], error); + if (q[`PF(QUERY, VALID)]) begin + queries++; + if (hit) hits++; + else misses++; + end + endfunction + function void final_phase(uvm_phase phase); + pmap_ref_destroy(model); + super.final_phase(phase); + endfunction + `include "agent/service.svh" + `include "seq/scenario.svh" +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/filelist.f b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/filelist.f new file mode 100644 index 000000000..ee3c1a197 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PreparedMap/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/interface.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/interface.sv new file mode 100644 index 000000000..7b84c0369 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/interface.sv @@ -0,0 +1,12 @@ +`include "parameters.svh" +interface prepared_map_control_if ( + input logic clock +); + logic reset, hold_ready; + logic [`PF_QUERY_WIDTH-1:0] query_0, query_1; + logic [`PF_RESULT_WIDTH-1:0] result_0, result_1; + clocking sample @(posedge clock); + default input #1step; + input reset, query_0, query_1, result_0, result_1; + endclocking +endinterface diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/package.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/package.sv new file mode 100644 index 000000000..114541285 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/package.sv @@ -0,0 +1,52 @@ +`include "parameters.svh" +`define PF(K, F) `PF_``K``_``F``_OFFSET +: `PF_``K``_``F``_WIDTH +package prepared_map_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + typedef longint unsigned u64; + import "DPI-C" function chandle pmap_ref_create(input int unsigned address_bits); + import "DPI-C" function void pmap_ref_destroy(input chandle model); + import "DPI-C" function void pmap_ref_reset(input chandle model); + import "DPI-C" function void pmap_ref_reserve( + input chandle model, + input int unsigned id + ); + import "DPI-C" function void pmap_ref_prepared( + input chandle model, + input int unsigned id, + input u64 va, + pa, + input int unsigned bytes, + write, + last, + error + ); + import "DPI-C" function int unsigned pmap_ref_ready( + input chandle model, + input int unsigned id, + fire, + output int unsigned error, + output u64 va + ); + import "DPI-C" function void pmap_ref_query( + input chandle model, + input int unsigned valid, + id, + input u64 va, + input int unsigned bytes, + write, + retiring, + retiring_id, + output int unsigned hit, + output u64 pa, + output int unsigned error + ); + import "DPI-C" function void pmap_ref_release( + input chandle model, + input int unsigned id + ); + import "DPI-C" function int unsigned pmap_ref_pending(input chandle model); + `include "env/env.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/seq/scenario.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/seq/scenario.svh new file mode 100644 index 000000000..1a07f81fd --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/seq/scenario.svh @@ -0,0 +1,168 @@ +task reserve_tag(int id); + @(negedge ctl.clock); + reserve.bits = '0; + reserve.bits[`PF(RESERVE, ID)] = id; + reserve.valid = 1; + do @(ctl.sample); while (!reserve.sample.ready); + @(negedge ctl.clock); + reserve.valid = 0; +endtask +task record(int id, u64 va, u64 pa, int bytes, bit write, bit last, int error = 0); + @(negedge ctl.clock); + prepared.bits = '0; + prepared.bits[`PF(OUT, ID)] = id; + prepared.bits[`PF(OUT, VA)] = va; + prepared.bits[`PF(OUT, PA)] = pa; + prepared.bits[`PF(OUT, BYTES)] = bytes; + prepared.bits[`PF(OUT, WRITE)] = write; + prepared.bits[`PF(OUT, LAST)] = last; + prepared.bits[`PF(OUT, ERROR)] = error; + prepared.valid = 1; + do @(ctl.sample); while (!prepared.sample.ready); + @(negedge ctl.clock); + prepared.valid = 0; +endtask +function logic [`PF_QUERY_WIDTH-1:0] query(int id, u64 va, int bytes, bit write); + logic [`PF_QUERY_WIDTH-1:0] q = '0; + q[`PF(QUERY, VALID)] = 1; + q[`PF(QUERY, ID)] = id; + q[`PF(QUERY, VA)] = va; + q[`PF(QUERY, BYTES)] = bytes; + q[`PF(QUERY, WRITE)] = write; + return q; +endfunction +task probes(logic [`PF_QUERY_WIDTH-1:0] a, logic [`PF_QUERY_WIDTH-1:0] b, int clocks = 3); + @(negedge ctl.clock); + ctl.query_0 = a; + ctl.query_1 = b; + repeat (clocks) @(ctl.sample); + @(negedge ctl.clock); + ctl.query_0 = '0; + ctl.query_1 = '0; +endtask +task wait_ready(int id, int error = 0); + while (!notified.exists(id)) @(negedge ctl.clock); + verify_contract(status[id] == error, "wrong terminal status"); +endtask +task retire_tag(int id, bit probe_release = 0); + @(negedge ctl.clock); + retire.bits = '0; + retire.bits[`PF(RELEASE, ID)] = id; + retire.valid = 1; + if (probe_release) begin + ctl.query_0 = query(id, 'h1010, 8, 0); + ctl.query_1 = query(2, 'h4004, 8, 1); + end + do @(ctl.sample); while (!retire.sample.ready); + @(negedge ctl.clock); + retire.valid = 0; + ctl.query_0 = '0; + ctl.query_1 = '0; +endtask +task execute(); + ctl.reset = 1; + ctl.hold_ready = 0; + ctl.query_0 = '0; + ctl.query_1 = '0; + reserve.valid = 0; + reserve.bits = '0; + prepared.valid = 0; + prepared.bits = '0; + retire.valid = 0; + retire.bits = '0; + fork + monitor(); + join_none + repeat (3) @(negedge ctl.clock); + ctl.reset = 0; + reserve_tag(1); + record(1, 'h1000, 'h8000, 4096, 0, 0); + probes(query(1, 'h1010, 16, 0), query(1, 'h1010, 16, 1)); + @(negedge ctl.clock); + ctl.hold_ready = 1; + record(1, 'h2000, 'hc000, 4096, 0, 1); + probes(query(1, 'h1010, 16, 0), query(1, 'h2010, 16, 0), 8); + @(negedge ctl.clock); + ctl.hold_ready = 0; + wait_ready(1); + probes(query(1, 'h1ffe, 2, 0), query(1, 'h2ffe, 2, 0)); + probes(query(1, 'h1fff, 2, 0), query(1, 'h3000, 4, 0)); + probes(query(1, 'h1000, 16, 1), query(1, 64'hffffffffffffffff, 2, 0)); + probes(query(1, 'h1000, 0, 0), query(99, 'h1000, 8, 0)); + reserve_tag(2); + record(2, 'h4000, 'h9000, 64, 1, 0); + record(2, 64'hffffffffffffffe0, (64'd1 << `PF_ADDRESS_BITS) - 32, 32, 1, 1); + wait_ready(2); + probes(query(1, 'h1f00, 16, 0), query(2, 'h4004, 16, 1)); + probes(query(2, 64'hfffffffffffffff8, 8, 1), query(2, 64'hfffffffffffffff8, 9, 1)); + retire_tag(1, 1); + probes(query(1, 'h1010, 16, 0), query(2, 'h4004, 16, 1)); + reserve_tag(1); + record(1, 'h6000, 'ha000, 128, 0, 1); + wait_ready(1); + probes(query(1, 'h1010, 16, 0), query(1, 'h6040, 64, 0)); + reserve_tag(3); + record(3, 'h7000, 'hb000, 64, 0, 0); + record(3, 'h8000, 0, 0, 0, 1, 3); + wait_ready(3, 3); + probes(query(3, 'h7004, 8, 0), query(3, 'h8000, 8, 0)); + reserve_tag(4); + fork + reserve_tag(5); + begin + repeat (6) begin + @(negedge ctl.clock); + verify_contract(!reserve.ready, "full table accepted fifth reserve"); + end + record(4, 'h9000, 0, 0, 0, 1, 4); + wait_ready(4, 4); + retire_tag(4); + end + join + for (int i = 0; i < 8; i++) begin + record(5, 'h10000 + i * 4096, 'h30000 + (7 - i) * 4096, 32, 0, i == 7); + end + wait_ready(5); + probes(query(5, 'h17018, 8, 0), query(5, 'h10008, 8, 0)); + retire_tag(1); + retire_tag(2); + retire_tag(3); + retire_tag(5); + for (int code = 1; code <= 6; code++) begin + reserve_tag(20 + code); + record(20 + code, 'h9000 + code * 16, 0, 0, code % 2, 1, code); + wait_ready(20 + code, code); + retire_tag(20 + code); + end + // Retain an explicitly authorized same-page union and a discontiguous next PA page. + reserve_tag(10); + record(10, 'h74003e00, 'h800e00, 512, 0, 0); + record(10, 'h74004000, 'h802000, 16, 0, 1); + wait_ready(10); + probes(query(10, 'h74003e10, 16, 0), query(10, 'h74004000, 16, 0)); + probes(query(10, 'h74003df0, 16, 0), query(10, 'h74004010, 8, 0)); + probes(query(10, 'h74003ff8, 16, 0), query(10, 'h74003e48, 16, 1)); + retire_tag(10); + reserve_tag(9); + record(9, 'h9000, 'h19000, 64, 0, 1); + // Reset cancels a terminal notification still under backpressure. + @(negedge ctl.clock); + ctl.hold_ready = 1; + repeat (2) @(negedge ctl.clock); + cancelled += pmap_ref_pending(model); + ctl.reset = 1; + repeat (3) @(negedge ctl.clock); + ctl.reset = 0; + ctl.hold_ready = 0; + probes(query(9, 'h9000, 8, 0), query(1, 'h6000, 8, 0)); + repeat (3) @(negedge ctl.clock); + verify_contract(pmap_ref_pending(model) == 0, "mapping remained after retirement/reset"); + verify_contract(hits > 0 && misses > 0 && dual > 0 && stalls >= 4 && cancelled == 1, + "required checks did not execute"); + `uvm_info( + "PREPARED_MAP", + $sformatf( + "Checked %0d queries, %0d hits, %0d misses; reserve=%0d ready=%0d retire=%0d stalls=%0d dual=%0d reset_cancel=%0d", + queries, hits, misses, allocations, notifications, retired, stalls, dual, cancelled), + UVM_LOW) +endtask diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/tb.sv b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/tb.sv new file mode 100644 index 000000000..d0725b78c --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/tb.sv @@ -0,0 +1,42 @@ +`include "parameters.svh" +module prepared_map_tb; + import uvm_pkg::*; + import prepared_map_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + prepared_map_control_if ctl (clock); + stream_if #(`PF_RESERVE_WIDTH) reserve ( + clock, + ctl.reset + ); + stream_if #(`PF_OUT_WIDTH) prepared ( + clock, + ctl.reset + ); + stream_if #(`PF_MAP_READY_WIDTH) mapped ( + clock, + ctl.reset + ); + stream_if #(`PF_RELEASE_WIDTH) retire ( + clock, + ctl.reset + ); + assign mapped.ready = !ctl.reset && !ctl.hold_ready; + PreparedMap dut ( + .clock(clock), + .reset(ctl.reset), + `include "prepared_map_ports.svh" + ); + initial begin + uvm_config_db#(virtual prepared_map_control_if)::set(null, "uvm_test_top.env", "ctl", ctl); + uvm_config_db#(virtual stream_if #(`PF_RESERVE_WIDTH))::set(null, "uvm_test_top.env", "reserve", + reserve); + uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::set(null, "uvm_test_top.env", "prepared", + prepared); + uvm_config_db#(virtual stream_if #(`PF_MAP_READY_WIDTH))::set(null, "uvm_test_top.env", + "mapped", mapped); + uvm_config_db#(virtual stream_if #(`PF_RELEASE_WIDTH))::set(null, "uvm_test_top.env", "retire", + retire); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/tests/test.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/tests/test.svh new file mode 100644 index 000000000..398043c00 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/tests/test.svh @@ -0,0 +1,14 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + test_env.execute(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/suite.toml new file mode 100644 index 000000000..ec50d41b1 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/suite.toml @@ -0,0 +1,54 @@ +[model] +manifest="src/csrc/Cargo.toml" +library="libpreflight_reference" + +[[tests]] +functional_groups = ["preflight_pkg::env::behavior"] +name = "preflight" +top = "preflight_tb" +test = "protocol_test" +filelist = "preflight/filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = ["prepared_map_pkg::env::behavior"] +name = "prepared_map" +top = "prepared_map_tb" +test = "protocol_test" +filelist = "prepared_map/filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +name="preparation" +top="preparation_tb" +test="protocol_test" +filelist="preparation/filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["preparation_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Emit.scala new file mode 100644 index 000000000..0ad00a3da --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Emit.scala @@ -0,0 +1,126 @@ +package framework.core.memory.access.preflight + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.ip.bus.chi +import framework.core.memory.access.mmu.{PteRequest, PteResponse} + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val bus = upickle.default.read[chi.Params](values("bus"))(upickle.default.macroRW[chi.Params]) + + val p = Params( + bus, + values("contexts").num.toInt, + values("maxRanges").num.toInt, + values("idBits").num.toInt, + values("beatBytes").num.toInt + ) + + Files.createDirectories(Paths.get(directory)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Preflight(p), + args = Array("--target-dir", s"$directory/Preflight", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new PreparedMap(p), + args = Array("--target-dir", s"$directory/PreparedMap", "--split-verilog") + ) + + val defs = scala.collection.mutable.ArrayBuffer[String]( + s"`define PF_ADDRESS_BITS ${bus.addressBits}", + s"`define PF_BEAT_BYTES ${p.beatBytes}", + s"`define PF_CONTEXTS ${p.contexts}" + ) + + val ports = scala.collection.mutable.ArrayBuffer[String]() + val mapPorts = scala.collection.mutable.ArrayBuffer[String]() + + def channel( + kind: String, + prefix: String, + signal: String, + bundle: Bundle, + target: scala.collection.mutable.ArrayBuffer[String], + stream: Boolean = true, + define: Boolean = true + ): Unit = { + var offset = 0 + for ((name, field) <- bundle.elements.toSeq) { + val width = field.getWidth + if (define) { + defs += s"`define PF_${kind}_${name.toUpperCase}_OFFSET $offset"; + defs += s"`define PF_${kind}_${name.toUpperCase}_WIDTH $width" + } + val bits = if (stream) "_bits" else "" + val payload = if (stream) s"$signal.bits" else signal + target += s".$prefix${bits}_$name($payload[$offset +: $width])"; offset += width + } + if (define) defs += s"`define PF_${kind}_WIDTH $offset" + if (stream) { target += s".${prefix}_valid($signal.valid)"; target += s".${prefix}_ready($signal.ready)" } + } + + channel("CMD", "io_command", "cmd", new Command(p), ports) + channel("OUT", "io_prepared", "prepared", new PreparedSegment(p), ports) + channel("AUTH", "io_authorization", "authorization", new Authorization(p), ports) + channel("PERMIT", "io_permission", "permission", new Permission(p), ports) + channel("PTE", "io_pteRequest", "pte_req", new PteRequest(p.walker), ports) + channel("PTERESP", "io_pteResponse", "pte_resp", new PteResponse, ports) + channel("RESERVE", "io_reserve", "reserve", new MapTag(p), mapPorts) + channel("OUT", "io_prepared", "prepared", new PreparedSegment(p), mapPorts, define = false) + channel("MAP_READY", "io_ready", "mapped", new MapReady(p), mapPorts) + channel("RELEASE", "io_release", "retire", new MapTag(p), mapPorts) + for (i <- 0 until 2) { + channel("QUERY", s"io_queries_$i", s"ctl.query_$i", new MapQuery(p), mapPorts, stream = false, define = i == 0) + channel("RESULT", s"io_results_$i", s"ctl.result_$i", new MapResult(p), mapPorts, stream = false, define = i == 0) + } + Files.writeString(Paths.get(directory, "parameters.svh"), defs.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, "preflight_ports.svh"), ports.mkString(",\n") + "\n") + Files.writeString(Paths.get(directory, "prepared_map_ports.svh"), mapPorts.mkString(",\n") + "\n") + val permission = upickle.default.read[configs.PmpParams](values("permission")) + + val regions = values("regions").arr.toSeq.map(v => + upickle.default.read[framework.core.memory.access.PhysicalRegion](v)(configs.Config.regionRW) + ) + + val spec = configs.Config(permission, p, regions) + + val definitions = scala.collection.mutable.ArrayBuffer[String]( + s"`define PREP_CONTEXTS ${p.contexts}", + s"`define PREP_ID_BITS ${p.idBits}", + s"`define PREP_ADDRESS_BITS ${bus.addressBits}", + s"`define PREP_PMP_ENTRIES ${permission.entries}" + ) + + val binding = scala.collection.mutable.ArrayBuffer[String]() + var inputOffset = 0; var outputOffset = 0 + + def leaves(data: Data, path: String): Seq[(String, Data)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + leaves(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => leaves(field, s"${path}_$index") } + case field => Seq(path -> field) + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new Preparation(spec, p.walker) + for ((name, field) <- leaves(dut.io, "")) { + val input = chisel3.reflect.DataMirror.directionOf(field) == ActualDirection.Input + val direction = if (input) "IN" else "OUT"; val signal = if (input) "inputs" else "outputs"; + val offset = if (input) inputOffset else outputOffset; val width = field.getWidth + definitions += s"`define PREP_${direction}_${name.toUpperCase}_OFFSET $offset"; + definitions += s"`define PREP_${direction}_${name.toUpperCase}_WIDTH $width" + binding += s".io_$name(bus.$signal[$offset +: $width])"; + if (input) inputOffset += width else outputOffset += width + }; dut + }, + args = Array("--target-dir", s"$directory/Preparation", "--split-verilog") + ) + definitions += s"`define PREP_INPUT_BITS $inputOffset"; definitions += s"`define PREP_OUTPUT_BITS $outputOffset" + Files.writeString(Paths.get(directory, "preparation_parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, "preparation_ports.svh"), binding.mkString(",\n") + "\n") + +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Permission.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Permission.scala new file mode 100644 index 000000000..35017ac11 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Permission.scala @@ -0,0 +1,23 @@ +package framework.core.memory.access.preflight + +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ + +/** Frozen RISC-V physical-memory protection state at the architectural boundary. */ +class PmpConfig extends Bundle { + val l = Bool() + val res = UInt(2.W) + val a = UInt(2.W) + val x = Bool() + val w = Bool() + val r = Bool() +} + +class PmpEntry(b: PmpParams) extends Bundle { + implicit val permission = b + + val cfg = new PmpConfig + val addr = UInt((permission.addressBits - 2).W) + val mask = UInt(permission.addressBits.W) +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PermissionCheck.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PermissionCheck.scala new file mode 100644 index 000000000..f167ae673 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PermissionCheck.scala @@ -0,0 +1,108 @@ +package framework.core.memory.access.preflight + +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.memory.access.preflight.configs.PmpParams +import framework.core.memory.access.preflight.PmpEntry +import framework.core.memory.access.PhysicalRegion +import framework.core.memory.access.preflight.{Authorization, Permission, Params => PreparationParams} + +/** Checks the complete transport range against the submitting command's frozen PMP image. */ +@instantiable +class PermissionCheck(val specification: configs.Config) extends Module { + val config = specification.parameters + val regions = specification.regions + val permissionParam = specification.permission + implicit val permission = permissionParam + + val physicalBits = math.min(config.bus.addressBits, permission.addressBits) + require(regions.nonEmpty) + regions.foreach(r => require(r.base >= 0 && r.bytes > 0 && r.base + r.bytes <= (BigInt(1) << physicalBits))) + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new Authorization(config))) + // Selected from the captured admission context using request.bits.id, never the live CPU CSRs. + val contextValid = Input(Bool()) + val contextId = Input(UInt(config.idBits.W)) + val pmp = Input(Vec(permission.entries, new PmpEntry(permissionParam))) + val response = Decoupled(new Permission(config)) + }) + + val idle :: checking :: reporting :: Nil = Enum(3) + val state = RegInit(idle) + val request = Reg(new Authorization(config)) + val capturedPmp = Reg(Vec(permission.entries, new PmpEntry(permissionParam))) + val address = Reg(UInt(config.bus.addressBits.W)) + val remaining = Reg(UInt(13.W)) + val allowed = Reg(Bool()) + io.request.ready := state === idle && !reset.asBool + io.response.valid := state === reporting && !reset.asBool + io.response.bits.id := request.id + io.response.bits.allow := allowed + + when(io.request.fire) { + val r = io.request.bits + val end = r.pa +& r.bytes + val shape = io.contextValid && io.contextId === r.id && r.bytes =/= 0.U && r.bytes <= 4096.U && + end <= (BigInt(1) << physicalBits).U && + (r.privilege === 0.U || r.privilege === 1.U || r.privilege === 3.U) && + (!r.isPte || (!r.write && r.bytes === 8.U && r.privilege === 1.U)) + val regionAllowed = regions.map { region => + val accessAllowed = Mux(r.write, region.writable.B, region.readable.B) + region.normal.B && r.pa >= region.base.U && end <= (region.base + region.bytes).U && accessAllowed + }.reduce(_ || _) + request := r + capturedPmp := io.pmp + address := r.pa + remaining := r.bytes + allowed := shape && regionAllowed + state := Mux(shape && regionAllowed, checking, reporting) + } + + // A page segment may cover different PMP entries. Check each actual transport + // transaction (at most one 64-byte line), never a widened page-sized access. + val size = (1 to 6).foldLeft(0.U(3.W)) { (selected, logBytes) => + Mux(remaining >= (1 << logBytes).U && address(logBytes - 1, 0) === 0.U, logBytes.U, selected) + } + + val bytes = (1.U(13.W) << size)(12, 0) + val accessEnd = address +& bytes + val machine = request.privilege === 3.U + + val permitted = + (0 until permission.entries).reverse.foldLeft(if (permission.entries == 0) true.B else machine) { (previous, i) => + val entry = capturedPmp(i) + val encoded = (entry.addr << 2).pad(permission.addressBits) + val low = Mux( + entry.cfg.a === 1.U, + if (i == 0) 0.U(permission.addressBits.W) else (capturedPmp(i - 1).addr << 2), + Mux(entry.cfg.a === 3.U, encoded & ~entry.mask, encoded) + ) + val high = Mux( + entry.cfg.a === 1.U, + encoded.pad(permission.addressBits + 1), + Mux(entry.cfg.a === 3.U, low +& entry.mask + 1.U, encoded +& 4.U) + ) + val hit = entry.cfg.a =/= 0.U && low < high && address < high && accessEnd > low + val contained = address >= low && accessEnd <= high + val accessAllowed = Mux(request.write, entry.cfg.w, entry.cfg.r) || (machine && !entry.cfg.l) + Mux(hit, contained && accessAllowed, previous) + } + + when(state === checking) { + when(!permitted) { + allowed := false.B + state := reporting + }.elsewhen(remaining === bytes) { + state := reporting + }.otherwise { + address := address + bytes + remaining := remaining - bytes + } + } + when(io.response.fire)(state := idle) +} diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Preflight.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preflight.scala similarity index 85% rename from arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Preflight.scala rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preflight.scala index 162e94943..d266db412 100644 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Preflight.scala +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preflight.scala @@ -1,29 +1,34 @@ -package memcore.memory.preflight +package framework.core.memory.access.preflight + +import framework.core.memory.access.preflight.{Params => ModuleParam} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi.rnf.{CacheAccess, CacheResult} -import memcore.memory.mmu.Walker +import framework.core.memory.access.mmu.{PteRequest, PteResponse, Walker} /** Preparation only. A consumer retains the entire successful mapping until DMA retires. */ @instantiable -class Preflight(p: Params) extends Module { +class Preflight(b: ModuleParam) extends Module { + val p = b + val contextBits = log2Ceil(p.contexts) @public val io = IO(new Bundle { - val command = Flipped(Decoupled(new Command(p))) - val prepared = Decoupled(new PreparedSegment(p)) - val authorization = Decoupled(new Authorization(p)) - val permission = Flipped(Decoupled(new Permission(p))) - val pteRequest = Decoupled(new CacheAccess(p.bus)) - val pteResponse = Flipped(Decoupled(new CacheResult)) + val command = Flipped(Decoupled(new Command(b))) + val prepared = Decoupled(new PreparedSegment(b)) + val authorization = Decoupled(new Authorization(b)) + val permission = Flipped(Decoupled(new Permission(b))) + + val pteRequest = Decoupled(new PteRequest(b.walker)) + + val pteResponse = Flipped(Decoupled(new PteResponse)) }) val free :: queued :: working :: ready :: Nil = Enum(4) val phase = RegInit(VecInit(Seq.fill(p.contexts)(free))) - val commands = Reg(Vec(p.contexts, new Command(p))) + val commands = Reg(Vec(p.contexts, new Command(b))) val counts = RegInit(VecInit(Seq.fill(p.contexts)(0.U(4.W)))) val errors = Reg(Vec(p.contexts, UInt(3.W))) val faultVA = Reg(Vec(p.contexts, UInt(64.W))) @@ -43,7 +48,8 @@ class Preflight(p: Params) extends Module { MuxCase( 0.U(contextBits.W), (0 until p.contexts).map { off => - val index = (cursor + off.U)(contextBits - 1, 0) + val sum = cursor +& off.U + val index = Mux(sum >= p.contexts.U, sum - p.contexts.U, sum)(contextBits - 1, 0) candidates(index) -> index } ) @@ -67,9 +73,13 @@ class Preflight(p: Params) extends Module { val previousEnd = previousVA +& sizes(worker)(previous) val currentEnd = currentVA +& currentBytes val cancelSegmenter = WireDefault(false.B) - val splitter: Instance[Segmenter] = Instantiate(new Segmenter(4096, 4096)) + + val splitter: Instance[Segmenter] = Instantiate(new Segmenter) + splitter.io.cancel := cancelSegmenter - val walker: Instance[Walker] = Instantiate(new Walker(p.bus)) + + val walker: Instance[Walker] = Instantiate(new Walker(p.walker)) + val cmd = commands(worker) def finish(code: UInt, address: UInt): Unit = { @@ -234,3 +244,58 @@ class Preflight(p: Params) extends Module { .otherwise(outputIndex := outputIndex + 1.U) } } + +/** Shape describes transport bytes, including any read beyond the user's useful payload. */ +class Command(b: ModuleParam) extends Bundle { + val p = b + + val id = UInt(p.idBits.W) + val baseVA = UInt(64.W) + val rows = UInt(32.W) + val columns = UInt(32.W) + val spanBytes = UInt(32.W) + val columnStride = UInt(32.W) + val rowStride = UInt(32.W) + val write = Bool() + val mode = UInt(4.W) + val rootPpn = UInt(44.W) + val privilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() +} + +object Error { + val Ok = 0; val Shape = 1; val Overflow = 2; val PageFault = 3 + val AccessFault = 4; val Capacity = 5; val Context = 6 +} + +/** Failure returns one terminal zero-PA record. The caller retains successful maps and IDs until DMA retirement. */ +class PreparedSegment(b: ModuleParam) extends Bundle { + val p = b + + val id = UInt(p.idBits.W) + val va = UInt(64.W) + val pa = UInt(p.bus.addressBits.W) + val bytes = UInt(13.W) + val write = Bool() + val last = Bool() + val error = UInt(3.W) +} + +class Authorization(b: ModuleParam) extends Bundle { + val p = b + + val id = UInt(p.idBits.W) + val pa = UInt(p.bus.addressBits.W) + val bytes = UInt(13.W) + val write = Bool() + val isPte = Bool() + val privilege = UInt(2.W) +} + +class Permission(b: ModuleParam) extends Bundle { + val p = b + + val id = UInt(p.idBits.W) + val allow = Bool() +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preparation.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preparation.scala new file mode 100644 index 000000000..25fbf4c16 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preparation.scala @@ -0,0 +1,88 @@ +package framework.core.memory.access.preflight + +import framework.core.memory.access.preflight.configs.PmpParams +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.memory.access.preflight.configs.PmpParams +import framework.core.memory.access.preflight.PmpEntry +import framework.core.memory.access.mmu.{PteRequest, PteResponse} +import framework.core.memory.access.PhysicalRegion +import framework.core.memory.access.preflight.{ + Command, + PreparedSegment, + Preflight, + PreparedMap, + MapQuery, + MapResult, + MapReady, + MapTag, + Params => PreparationParams +} + +/** Client boundary for an externally selected DMA address-preparation implementation. */ +class PreparationPort(config: PreparationParams, permissionParam: PmpParams, walkerParam: WalkerParam) extends Bundle { + implicit val permission: PmpParams = permissionParam + + val command = Decoupled(new Command(config)) + val contextTag = Input(UInt(config.idBits.W)) + val contextValid = Output(Bool()) + val contextId = Output(UInt(config.idBits.W)) + val contextPmp = Output(Vec(permission.entries, new PmpEntry(permissionParam))) + val ranges = Flipped(Decoupled(new PreparedSegment(config))) + val ready = Flipped(Decoupled(new MapReady(config))) + val release = Decoupled(new MapTag(config)) + val queries = Output(Vec(2, new MapQuery(config))) + val results = Input(Vec(2, new MapResult(config))) + val pteRequest = Flipped(Decoupled(new PteRequest(walkerParam))) + val pteResponse = Decoupled(new PteResponse) +} + +/** Uses the admission snapshot and retains translations until DMA and cache completion. */ +@instantiable +class Preparation(val specification: configs.Config, walkerParam: WalkerParam) extends Module { + val config = specification.parameters + val permissionParam = specification.permission + implicit val permission: PmpParams = permissionParam + + @public + val io = IO(Flipped(new PreparationPort(config, permissionParam, walkerParam))) + + val preflight: Instance[Preflight] = + Instantiate(new Preflight(config)) + val maps: Instance[PreparedMap] = + Instantiate(new PreparedMap(config)) + + val check: Instance[PermissionCheck] = + Instantiate(new PermissionCheck(specification)) + + preflight.io.command.valid := io.command.valid && maps.io.reserve.ready && !reset.asBool + preflight.io.command.bits := io.command.bits + maps.io.reserve.valid := io.command.valid && preflight.io.command.ready && !reset.asBool + maps.io.reserve.bits.id := io.command.bits.id + io.command.ready := preflight.io.command.ready && maps.io.reserve.ready && !reset.asBool + assert(preflight.io.command.fire === io.command.fire, "Preparation command reservation is not atomic") + assert(maps.io.reserve.fire === io.command.fire, "Preparation map reservation is not atomic") + + val authorization = preflight.io.authorization + io.contextTag := authorization.bits.id + check.io.request <> authorization + check.io.contextValid := io.contextValid + check.io.contextId := io.contextId + check.io.pmp := io.contextPmp + preflight.io.permission <> check.io.response + io.pteRequest <> preflight.io.pteRequest + preflight.io.pteResponse <> io.pteResponse + + maps.io.prepared.valid := preflight.io.prepared.valid && io.ranges.ready + maps.io.prepared.bits := preflight.io.prepared.bits + io.ranges.valid := preflight.io.prepared.valid && maps.io.prepared.ready + io.ranges.bits := preflight.io.prepared.bits + preflight.io.prepared.ready := maps.io.prepared.ready && io.ranges.ready + io.ready <> maps.io.ready + maps.io.queries := io.queries + io.results := maps.io.results + maps.io.release <> io.release +} diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/PreparedMap.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PreparedMap.scala similarity index 89% rename from arch/src/main/scala/framework/mem-core/preflight/src/main/scala/PreparedMap.scala rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PreparedMap.scala index 33e243798..a47b2277a 100644 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/PreparedMap.scala +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PreparedMap.scala @@ -1,18 +1,27 @@ -package memcore.memory.preflight +package framework.core.memory.access.preflight + +import framework.core.memory.access.preflight.{Params => ModuleParam} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -class MapTag(p: Params) extends Bundle { val id = UInt(p.idBits.W) } +class MapTag(b: ModuleParam) extends Bundle { + val p = b + val id = UInt(p.idBits.W) +} + +class MapReady(b: ModuleParam) extends Bundle { + val p = b -class MapReady(p: Params) extends Bundle { val id = UInt(p.idBits.W) val error = UInt(3.W) val va = UInt(64.W) } -class MapQuery(p: Params) extends Bundle { +class MapQuery(b: ModuleParam) extends Bundle { + val p = b + val valid = Bool() val id = UInt(p.idBits.W) val va = UInt(64.W) @@ -20,23 +29,26 @@ class MapQuery(p: Params) extends Bundle { val write = Bool() } -class MapResult(p: Params) extends Bundle { +class MapResult(b: ModuleParam) extends Bundle { + val p = b + val hit = Bool() val pa = UInt(p.bus.addressBits.W) val error = UInt(3.W) } @instantiable -class PreparedMap(p: Params) extends Module { +class PreparedMap(b: ModuleParam) extends Module { + val p = b @public val io = IO(new Bundle { - val reserve = Flipped(Decoupled(new MapTag(p))) - val prepared = Flipped(Decoupled(new PreparedSegment(p))) - val ready = Decoupled(new MapReady(p)) - val release = Flipped(Decoupled(new MapTag(p))) - val queries = Input(Vec(2, new MapQuery(p))) - val results = Output(Vec(2, new MapResult(p))) + val reserve = Flipped(Decoupled(new MapTag(b))) + val prepared = Flipped(Decoupled(new PreparedSegment(b))) + val ready = Decoupled(new MapReady(b)) + val release = Flipped(Decoupled(new MapTag(b))) + val queries = Input(Vec(2, new MapQuery(b))) + val results = Output(Vec(2, new MapResult(b))) }) val slotBits = log2Ceil(p.contexts) diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Segmenter.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Segmenter.scala similarity index 95% rename from arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Segmenter.scala rename to arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Segmenter.scala index 68d3b1e3a..eb5423dc5 100644 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Segmenter.scala +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Segmenter.scala @@ -1,4 +1,4 @@ -package memcore.memory.preflight +package framework.core.memory.access.preflight import chisel3._ import chisel3.util._ @@ -27,7 +27,10 @@ class Segment extends Bundle { * row. No output segment crosses a row, a page, or maxSegmentBytes. */ @instantiable -class Segmenter(pageBytes: Int = 4096, maxSegmentBytes: Int = 16384) extends Module { +class Segmenter extends Module { + val pageBytes = 4096 + val maxSegmentBytes = pageBytes + require(isPow2(pageBytes) && pageBytes >= 8) require(isPow2(maxSegmentBytes) && maxSegmentBytes >= 8) diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/configs/Config.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/configs/Config.scala new file mode 100644 index 000000000..424be2d59 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/configs/Config.scala @@ -0,0 +1,18 @@ +package framework.core.memory.access.preflight.configs + +import framework.core.memory.access.PhysicalRegion +import framework.core.memory.access.preflight.Params +import framework.ip.bus.chi +import upickle.default.{macroRW, ReadWriter} + +case class PmpParams(addressBits: Int, entries: Int) +object PmpParams { implicit val rw: ReadWriter[PmpParams] = macroRW } + +case class Config(permission: PmpParams, parameters: Params, regions: Seq[PhysicalRegion]) + +object Config { + implicit val busRW: ReadWriter[chi.Params] = macroRW + implicit val parametersRW: ReadWriter[Params] = macroRW + implicit val regionRW: ReadWriter[PhysicalRegion] = macroRW + implicit val rw: ReadWriter[Config] = macroRW +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..0cecb126c --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/configs/Params.scala @@ -0,0 +1,33 @@ +package framework.core.memory.access.preflight + +import buckyball.config.PreparationHardware + +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} + +import framework.ip.bus.chi + +case class Params( + bus: chi.Params = chi.Params(), + contexts: Int = 4, + maxRanges: Int = 8, + idBits: Int = 8, + beatBytes: Int = 16) { + val walker = WalkerParam(bus, 0) + + require(contexts >= 2) + require(maxRanges == 8) + require(idBits > 0 && bus.addressBits >= 12 && bus.addressBits < 64) + require(beatBytes >= 8 && beatBytes <= 64 && (beatBytes & (beatBytes - 1)) == 0) +} + +object Params { + + def fromProto(p: PreparationHardware): Params = Params( + chi.Params.fromProto(p.getBus), + p.getContexts, + p.getMaxRanges, + p.getIdBits, + p.getBeatBytes + ) + +} diff --git a/arch/src/main/scala/framework/core/memory/build.sc b/arch/src/main/scala/framework/core/memory/build.sc new file mode 100644 index 000000000..0bd309691 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.access.{build => access_build} +import $file.home.{build => home_build} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("Memory.scala")), + PathRef(millSourcePath / os.RelPath("MemorySystem.scala")), + PathRef(millSourcePath / os.RelPath("ThreadMemory.scala")), + PathRef(millSourcePath / os.RelPath("configs/MemoryParam.scala")), + PathRef(millSourcePath / os.RelPath("uncached/Uncached.scala")), + PathRef(millSourcePath / os.RelPath("uncached/configs/UncachedParam.scala")) + ) ++ + access_build.sources() ++ + home_build.sources() +) + +def emitSources = T.sources( + access_build.emitSources() ++ + home_build.emitSources() +) diff --git a/arch/src/main/scala/framework/core/memory/configs/MemoryParam.scala b/arch/src/main/scala/framework/core/memory/configs/MemoryParam.scala new file mode 100644 index 000000000..8c91fd5c4 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/configs/MemoryParam.scala @@ -0,0 +1,44 @@ +package framework.core.memory.configs + +import framework.core.memory.access.configs.{Params => AccessParam} +import framework.core.memory.access.cache.configs.{Params => CacheParam} +import framework.core.memory.home.configs.{Params => HomeParam} +import framework.ip.bus.chi.rnf.RnfParams +import framework.core.memory.uncached.configs.UncachedParam +import buckyball.config.MemoryConfig +import scala.jdk.CollectionConverters._ +import framework.core.memory.access.{AccessParams, PhysicalRegion} +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.ip.memory.coherence.configs.CoherenceParams + +case class MemoryParam( + memoryAccess: AccessParam, + icache: RnfParams, + dcache: CacheParam, + home: HomeParam, + threadPorts: Int, + uncached: UncachedParam) + +object MemoryParam { + + def fromProto(p: MemoryConfig, threadPorts: Int): MemoryParam = { + require(p.isInitialized, "Memory IP configuration is incomplete") + val a = p.getAccess + val homep = HomeParam(CoherenceParams.fromProto(p.getHome)) + val requester = homep.memory.chi + MemoryParam( + AccessParam( + AccessParams(requester, a.getAccessTagBits), + a.getRegionsList.asScala.map(PhysicalRegion.fromProto).toSeq, + a.getTranslationEntries, + a.getResultLineBits + ), + RnfParams.fromProto(p.getIcache, requester), + CacheParam(RnfParams.fromProto(p.getDcache.getCache, requester), TrackingParam.fromProto(p.getDcache.getTracking)), + homep, + threadPorts, + UncachedParam.fromProto(p.getUncached) + ) + } + +} diff --git a/arch/src/main/scala/framework/core/memory/home/build.sc b/arch/src/main/scala/framework/core/memory/home/build.sc new file mode 100644 index 000000000..8c1539ece --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.memory.home.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Home.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/memory/home/configs/uvm.toml b/arch/src/main/scala/framework/core/memory/home/configs/uvm.toml new file mode 100644 index 000000000..a0b6176d7 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/configs/uvm.toml @@ -0,0 +1,14 @@ +[parameters] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 +agents = 2 +mshrEntries = 4 +homeId = 64 +cacheSets = 4 +cacheWays = 2 +cacheIdBits = 8 +metadataBits = 4 +responseDepth = 2 + +[defines] diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/agent/transport.svh b/arch/src/main/scala/framework/core/memory/home/src/main/resources/agent/transport.svh new file mode 100644 index 000000000..1f2d2af3d --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/agent/transport.svh @@ -0,0 +1,57 @@ +class transport_item extends uvm_sequence_item; + `uvm_object_utils_begin(transport_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_object_utils_end + logic [`COH_DAT_WIDTH-1:0] value; + function new(string name = "flit"); + super.new(name); + endfunction +endclass +class transport_monitor extends uvm_component; + `uvm_component_utils(transport_monitor) + virtual home_transport_if vif; + result_board #(transport_item, 0) boards[6][`COH_AGENTS]; + covergroup behavior with function sample (int channel, int node); + option.per_instance = 1; + channels: coverpoint channel {bins all[] = {[0 : 5]};} + nodes: coverpoint node {bins all[] = {[1 : `COH_AGENTS]};} + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual home_transport_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Home transport interface is required") + foreach (boards[c, n]) + boards[c][n] = result_board#(transport_item, 0)::type_id::create( + $sformatf("channel%0d_node%0d", c, n + 1), this); + endfunction + task run_phase(uvm_phase phase); + forever begin + @(posedge vif.clock); + foreach (boards[c, n]) if (vif.reset) boards[c][n].cancel_pending(); + if (!vif.reset) + for (int c = 0; c < 6; c++) begin + if (vif.expected_valid[c]) begin + transport_item t = transport_item::type_id::create("expected"); + int node = vif.expected_node[c]; + if (node < 1 || node > `COH_AGENTS) + `uvm_fatal("NODE", "Unexpected source or target node at Home") + t.value = vif.expected_bits[c]; + boards[c][node-1].write_expected(t); + behavior.sample(c, node); + end + if (vif.actual_valid[c]) begin + transport_item t = transport_item::type_id::create("actual"); + int node = vif.actual_node[c]; + if (node < 1 || node > `COH_AGENTS) + `uvm_fatal("NODE", "Unexpected delivered node at Home") + t.value = vif.actual_bits[c]; + boards[c][node-1].write_actual(t); + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/agent/transport_if.sv b/arch/src/main/scala/framework/core/memory/home/src/main/resources/agent/transport_if.sv new file mode 100644 index 000000000..e0291b49b --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/agent/transport_if.sv @@ -0,0 +1,9 @@ +`include "coherence_config.svh" +interface home_transport_if ( + input logic clock, + input logic reset +); + logic [5:0] expected_valid, actual_valid; + logic [`COH_DAT_WIDTH-1:0] expected_bits[6], actual_bits[6]; + int expected_node[6], actual_node[6]; +endinterface diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/memory/home/src/main/resources/filelist.f new file mode 100644 index 000000000..822f9033f --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/filelist.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/protocol/chi +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/agent/transport_if.sv +@RESOURCES@/package.sv +-F @RTL@/Home/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/memory/home/src/main/resources/interface.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/coherence/src/main/resources/interface.sv rename to arch/src/main/scala/framework/core/memory/home/src/main/resources/interface.sv diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/package.sv b/arch/src/main/scala/framework/core/memory/home/src/main/resources/package.sv new file mode 100644 index 000000000..05db26d39 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/package.sv @@ -0,0 +1,8 @@ +`include "coherence_config.svh" +`define CF(K, F) `COH_``K``_``F``_OFFSET+:`COH_``K``_``F``_WIDTH +package home_pkg; + `include "coherence/body.svh" + `include "agent/transport.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/memory/home/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..e3ddc8ea2 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/seq/sequence.svh @@ -0,0 +1,16 @@ +class home_sequence extends access_sequence; + `uvm_object_utils(home_sequence) + function new(string name = "scenario"); + super.new(name); + endfunction + virtual task drive_idle(req_t payload, int cycles = 1, bit blocked = 0); + // The requester boundary may accept into its credit FIFO while the Home is busy. + super.drive_idle(payload, cycles, 0); + endtask + virtual task send(int node, int opcode, longint unsigned address, bit error = 0); + int key = (node << 12) | (serial & 4095); + // Requester-owned transaction IDs are reusable after the final response arrives. + while (test_env.model.requests.exists(key)) @(control.cb); + super.send(node, opcode, address, error); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/memory/home/src/main/resources/suite.toml new file mode 100644 index 000000000..a98cd21a0 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/suite.toml @@ -0,0 +1,19 @@ +[model] +manifest="../../../../../../../../verify/uvm/src/protocol/chi/coherence/model/Cargo.toml" +library="libcoherence_reference_model" +[[tests]] +name="home" +top="home_tb" +test="home_test" +filelist="filelist.f" +timeout_ns=1000000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["home_pkg::system_model::operations","home_pkg::system_model::occupancy","home_pkg::transport_monitor::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/memory/home/src/main/resources/tb.sv new file mode 100644 index 000000000..f9d871339 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/tb.sv @@ -0,0 +1,228 @@ +`include "coherence_config.svh" +module home_tb; + import uvm_pkg::*; + import home_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + coherence_control_if control (clock); + stream_if #(`COH_REQ_WIDTH) source_if ( + clock, + control.reset + ); + stream_if #(`COH_RSP_WIDTH) rx_rsp ( + clock, + control.reset + ); + stream_if #(`COH_DAT_WIDTH) rx_dat ( + clock, + control.reset + ); + stream_if #(`COH_TXRSP_WIDTH) tx_rsp ( + clock, + control.reset + ); + stream_if #(`COH_TXDAT_WIDTH) sink_if ( + clock, + control.reset + ); + stream_if #(`COH_SNP_CHANNEL_WIDTH) tx_snp ( + clock, + control.reset + ); + stream_if #(`COH_MEMREQ_WIDTH) mem_req ( + clock, + control.reset + ); + stream_if #(`COH_MEMRESP_WIDTH) mem_resp ( + clock, + control.reset + ); + stream_if #(`COH_REQ_WIDTH) observedReq ( + clock, + control.reset + ); + stream_if #(`COH_RSP_WIDTH) observedRxRsp ( + clock, + control.reset + ); + stream_if #(`COH_DAT_WIDTH) observedRxDat ( + clock, + control.reset + ); + logic [`COH_AGENTS-1:0] req_valid, req_ready; + logic [`COH_REQ_WIDTH-1:0] req_bits[`COH_AGENTS]; + logic [`COH_AGENTS-1:0] txRsp_valid, txRsp_ready; + logic [`COH_RSP_WIDTH-1:0] txRsp_bits[`COH_AGENTS]; + logic [`COH_AGENTS-1:0] txDat_valid, txDat_ready; + logic [`COH_DAT_WIDTH-1:0] txDat_bits[`COH_AGENTS]; + logic [`COH_AGENTS-1:0] rxRsp_valid, rxRsp_ready; + logic [`COH_TXRSP_WIDTH-1:0] rxRsp_bits[`COH_AGENTS]; + logic [`COH_AGENTS-1:0] rxDat_valid, rxDat_ready; + logic [`COH_TXDAT_WIDTH-1:0] rxDat_bits[`COH_AGENTS]; + logic [`COH_AGENTS-1:0] snp_valid, snp_ready; + logic [`COH_SNP_WIDTH-1:0] snp_bits[`COH_AGENTS]; + wire observedReq_valid; + wire [`COH_REQ_WIDTH-1:0] observedReq_bits; + wire observedRxRsp_valid; + wire [`COH_RSP_WIDTH-1:0] observedRxRsp_bits; + wire observedRxDat_valid; + wire [`COH_DAT_WIDTH-1:0] observedRxDat_bits; + wire observedRsp_valid; + wire [`COH_TXRSP_WIDTH-1:0] observedRsp_bits; + wire observedDat_valid; + wire [`COH_TXDAT_WIDTH-1:0] observedDat_bits; + wire observedSnp_valid; + wire [`COH_SNP_CHANNEL_WIDTH-1:0] observedSnp_bits; + assign observedReq.valid = observedReq_valid; + assign observedReq.bits = observedReq_bits; + assign observedReq.ready = 1; + assign observedRxRsp.valid = observedRxRsp_valid; + assign observedRxRsp.bits = observedRxRsp_bits; + assign observedRxRsp.ready = 1; + assign observedRxDat.valid = observedRxDat_valid; + assign observedRxDat.bits = observedRxDat_bits; + assign observedRxDat.ready = 1; + Home dut ( + .clock(clock), + .reset(control.reset), + .io_active(!control.reset), + .io_blockRequesterRsp(1'b0), + .io_outstanding(control.outstanding), + `include "home_ports.svh" + ); + for (genvar i = 0; i < `COH_AGENTS; i++) begin + assign req_valid[i] = source_if.valid && source_if.bits[`CF(REQ, SRCID)] == i + 1; + assign req_bits[i] = source_if.bits; + assign txRsp_valid[i] = rx_rsp.valid && rx_rsp.bits[`CF(RSP, SRCID)] == i + 1; + assign txRsp_bits[i] = rx_rsp.bits; + assign txDat_valid[i] = rx_dat.valid && rx_dat.bits[`CF(DAT, SRCID)] == i + 1; + assign txDat_bits[i] = rx_dat.bits; + end + always_comb begin + source_if.ready = 0; + rx_rsp.ready = 0; + rx_dat.ready = 0; + for (int i = 0; i < `COH_AGENTS; i++) begin + source_if.ready |= source_if.bits[`CF(REQ, SRCID)] == i + 1 && req_ready[i]; + rx_rsp.ready |= rx_rsp.bits[`CF(RSP, SRCID)] == i + 1 && txRsp_ready[i]; + rx_dat.ready |= rx_dat.bits[`CF(DAT, SRCID)] == i + 1 && txDat_ready[i]; + end + end + int selected_rxRsp = -1; + always @(posedge clock) if (control.reset || (tx_rsp.valid && tx_rsp.ready)) selected_rxRsp <= -1; + always @(negedge clock) + if (!control.reset && selected_rxRsp < 0) + for (int i = 0; i < `COH_AGENTS; i++) + if (selected_rxRsp < 0 && rxRsp_valid[i]) selected_rxRsp = i; + always_comb begin + tx_rsp.valid = 0; + tx_rsp.bits = '0; + rxRsp_ready = '0; + if (selected_rxRsp >= 0) begin + tx_rsp.valid = rxRsp_valid[selected_rxRsp]; + tx_rsp.bits = rxRsp_bits[selected_rxRsp]; + rxRsp_ready[selected_rxRsp] = tx_rsp.ready; + end + end + int selected_rxDat = -1; + always @(posedge clock) + if (control.reset || (sink_if.valid && sink_if.ready)) + selected_rxDat <= -1; + always @(negedge clock) + if (!control.reset && selected_rxDat < 0) + for (int i = 0; i < `COH_AGENTS; i++) + if (selected_rxDat < 0 && rxDat_valid[i]) selected_rxDat = i; + always_comb begin + sink_if.valid = 0; + sink_if.bits = '0; + rxDat_ready = '0; + if (selected_rxDat >= 0) begin + sink_if.valid = rxDat_valid[selected_rxDat]; + sink_if.bits = rxDat_bits[selected_rxDat]; + rxDat_ready[selected_rxDat] = sink_if.ready; + end + end + int selected_snp = -1; + always @(posedge clock) if (control.reset || (tx_snp.valid && tx_snp.ready)) selected_snp <= -1; + always @(negedge clock) + if (!control.reset && selected_snp < 0) + for (int i = 0; i < `COH_AGENTS; i++) if (selected_snp < 0 && snp_valid[i]) selected_snp = i; + always_comb begin + tx_snp.valid = 0; + tx_snp.bits = '0; + snp_ready = '0; + if (selected_snp >= 0) begin + tx_snp.valid = snp_valid[selected_snp]; + tx_snp.bits = {`COH_SNP_TARGET_WIDTH'(selected_snp + 1), snp_bits[selected_snp]}; + snp_ready[selected_snp] = tx_snp.ready; + end + end + home_transport_if transport ( + clock, + control.reset + ); + assign transport.expected_valid[0] = source_if.valid && source_if.ready; + assign transport.expected_bits[0] = `COH_DAT_WIDTH'(source_if.bits); + assign transport.expected_node[0] = source_if.bits[`CF(REQ, SRCID)]; + assign transport.actual_valid[0] = observedReq.valid; + assign transport.actual_bits[0] = `COH_DAT_WIDTH'(observedReq.bits); + assign transport.actual_node[0] = observedReq.bits[`CF(REQ, SRCID)]; + assign transport.expected_valid[1] = rx_rsp.valid && rx_rsp.ready; + assign transport.expected_bits[1] = `COH_DAT_WIDTH'(rx_rsp.bits); + assign transport.expected_node[1] = rx_rsp.bits[`CF(RSP, SRCID)]; + assign transport.actual_valid[1] = observedRxRsp.valid; + assign transport.actual_bits[1] = `COH_DAT_WIDTH'(observedRxRsp.bits); + assign transport.actual_node[1] = observedRxRsp.bits[`CF(RSP, SRCID)]; + assign transport.expected_valid[2] = rx_dat.valid && rx_dat.ready; + assign transport.expected_bits[2] = `COH_DAT_WIDTH'(rx_dat.bits); + assign transport.expected_node[2] = rx_dat.bits[`CF(DAT, SRCID)]; + assign transport.actual_valid[2] = observedRxDat.valid; + assign transport.actual_bits[2] = `COH_DAT_WIDTH'(observedRxDat.bits); + assign transport.actual_node[2] = observedRxDat.bits[`CF(DAT, SRCID)]; + assign transport.expected_valid[3] = observedRsp_valid; + assign transport.expected_bits[3] = `COH_DAT_WIDTH'(observedRsp_bits); + assign transport.expected_node[3] = observedRsp_bits[`CF(TXRSP, TGTID)]; + assign transport.actual_valid[3] = tx_rsp.valid && tx_rsp.ready; + assign transport.actual_bits[3] = `COH_DAT_WIDTH'(tx_rsp.bits); + assign transport.actual_node[3] = tx_rsp.bits[`CF(TXRSP, TGTID)]; + assign transport.expected_valid[4] = observedDat_valid; + assign transport.expected_bits[4] = `COH_DAT_WIDTH'(observedDat_bits); + assign transport.expected_node[4] = observedDat_bits[`CF(TXDAT, TGTID)]; + assign transport.actual_valid[4] = sink_if.valid && sink_if.ready; + assign transport.actual_bits[4] = `COH_DAT_WIDTH'(sink_if.bits); + assign transport.actual_node[4] = sink_if.bits[`CF(TXDAT, TGTID)]; + assign transport.expected_valid[5] = observedSnp_valid; + assign transport.expected_bits[5] = `COH_DAT_WIDTH'(observedSnp_bits); + assign transport.expected_node[5] = observedSnp_bits[`CF(SNP, TARGET)]; + assign transport.actual_valid[5] = tx_snp.valid && tx_snp.ready; + assign transport.actual_bits[5] = `COH_DAT_WIDTH'(tx_snp.bits); + assign transport.actual_node[5] = tx_snp.bits[`CF(SNP, TARGET)]; + initial begin + uvm_config_db#(virtual coherence_control_if)::set(null, "uvm_test_top*", "control", control); + uvm_config_db#(virtual stream_if #(`COH_REQ_WIDTH))::set(null, "uvm_test_top*", "req", + source_if); + uvm_config_db#(virtual stream_if #(`COH_RSP_WIDTH))::set(null, "uvm_test_top*", "rx_rsp", + rx_rsp); + uvm_config_db#(virtual stream_if #(`COH_DAT_WIDTH))::set(null, "uvm_test_top*", "rx_dat", + rx_dat); + uvm_config_db#(virtual stream_if #(`COH_TXRSP_WIDTH))::set(null, "uvm_test_top*", "tx_rsp", + tx_rsp); + uvm_config_db#(virtual stream_if #(`COH_TXDAT_WIDTH))::set(null, "uvm_test_top*", "tx_dat", + sink_if); + uvm_config_db#(virtual stream_if #(`COH_SNP_CHANNEL_WIDTH))::set(null, "uvm_test_top*", "snp", + tx_snp); + uvm_config_db#(virtual stream_if #(`COH_MEMREQ_WIDTH))::set(null, "uvm_test_top*", "mem_req", + mem_req); + uvm_config_db#(virtual stream_if #(`COH_MEMRESP_WIDTH))::set(null, "uvm_test_top*", "mem_resp", + mem_resp); + uvm_config_db#(virtual stream_if #(`COH_REQ_WIDTH))::set(null, "uvm_test_top.env.bus.mon", + "req", observedReq); + uvm_config_db#(virtual stream_if #(`COH_RSP_WIDTH))::set(null, "uvm_test_top.env.bus.mon", + "rx_rsp", observedRxRsp); + uvm_config_db#(virtual stream_if #(`COH_DAT_WIDTH))::set(null, "uvm_test_top.env.bus.mon", + "rx_dat", observedRxDat); + uvm_config_db#(virtual home_transport_if)::set(null, "uvm_test_top.transport", "vif", + transport); + run_test("home_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/memory/home/src/main/resources/tests/test.svh new file mode 100644 index 000000000..181e07297 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/resources/tests/test.svh @@ -0,0 +1,17 @@ +class home_test extends protocol_test; + `uvm_component_utils(home_test) + transport_monitor transport; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + transport = transport_monitor::type_id::create("transport", this); + endfunction + task execute(); + home_sequence seq = home_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.control = control; + seq.start(test_env.bus.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/memory/home/src/main/scala/Emit.scala new file mode 100644 index 000000000..3a7900e4d --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/scala/Emit.scala @@ -0,0 +1,169 @@ +package framework.core.memory.home + +import java.nio.file.{Files, Paths} +import framework.ip.bus.chi._ +import framework.ip.memory.coherence.configs.CoherenceParams + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val output = Paths.get(directory) + Files.createDirectories(output) + val chi = Params(values("nodeIdBits").num.toInt, values("addressBits").num.toInt, values("dataBits").num.toInt) + + val cache = framework.ip.memory.cache.configs.CacheParams( + chi.addressBits, + 64, + values("cacheSets").num.toInt, + values("cacheWays").num.toInt, + values("cacheIdBits").num.toInt, + values("metadataBits").num.toInt, + values("responseDepth").num.toInt + ) + + val p = + CoherenceParams(chi, cache, values("agents").num.toInt, values("mshrEntries").num.toInt, values("homeId").num.toInt) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Home(configs.Params(p)), + args = Array("--target-dir", output.resolve("Home").toString, "--split-verilog") + ) + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val ports = scala.collection.mutable.ArrayBuffer[String]() + + def fields(flit: Flit): Seq[(String, Int)] = flit.fieldsLSB.map { field => + flit.elements.find(_._2 eq field).get._1 -> field.getWidth + } + + def channel( + kind: String, + prefix: String, + instance: String, + fs: Seq[(String, Int)] + ): Unit = { + var offset = 0 + for ((name, width) <- fs if width > 0) { + definitions += s"`define COH_${kind}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define COH_${kind}_${name.toUpperCase}_WIDTH $width" + ports += s".$prefix$name($instance.bits[$offset +: $width])" + offset += width + } + definitions += s"`define COH_${kind}_WIDTH $offset" + } + + channel("REQ", "io_req_bits_", "source_if", fields(new RequestFlit(p.chi))) + channel("RSP", "io_rxRsp_bits_", "rx_rsp", fields(new ResponseFlit(p.chi))) + channel("DAT", "io_rxDat_bits_", "rx_dat", fields(new DataFlit(p.chi))) + channel("TXRSP", "io_rsp_bits_", "tx_rsp", fields(new ResponseFlit(p.chi))) + channel("TXDAT", "io_dat_bits_", "sink_if", fields(new DataFlit(p.chi))) + channel("SNP", "io_snp_bits_flit_", "tx_snp", fields(new SnoopFlit(p.chi))) + val snpWidth = (new SnoopFlit(p.chi)).flitWidth + ports += s".io_snp_bits_targetNode(tx_snp.bits[$snpWidth +: ${p.chi.nodeIdBits}])" + definitions += s"`define COH_SNP_TARGET_OFFSET $snpWidth" + definitions += s"`define COH_SNP_TARGET_WIDTH ${p.chi.nodeIdBits}" + definitions += s"`define COH_SNP_CHANNEL_WIDTH ${snpWidth + p.chi.nodeIdBits}" + channel( + "MEMREQ", + "io_memoryReq_bits_", + "mem_req", + Seq("id" -> p.chi.txnIdBits, "addr" -> p.chi.addressBits, "write" -> 1, "data" -> 512, "mask" -> 64) + ) + channel("MEMRESP", "io_memoryResp_bits_", "mem_resp", Seq("id" -> p.chi.txnIdBits, "data" -> 512, "error" -> 1)) + for ( + (port, instance) <- Seq( + "req" -> "source_if", + "rxRsp" -> "rx_rsp", + "rxDat" -> "rx_dat", + "rsp" -> "tx_rsp", + "dat" -> "sink_if", + "snp" -> "tx_snp", + "memoryReq" -> "mem_req", + "memoryResp" -> "mem_resp" + ) + ) { + ports += s".io_${port}_valid($instance.valid)" + ports += s".io_${port}_ready($instance.ready)" + } + for ( + (name, value) <- Seq( + "AGENTS" -> p.agents, + "MSHRS" -> p.mshrEntries, + "HOME" -> p.homeId, + "SETS" -> p.cache.sets, + "WAYS" -> p.cache.ways, + "BEATS" -> p.chi.beatsPerLine, + "DATA_BITS" -> p.chi.dataBits, + "OUTSTANDING_BITS" -> chisel3.util.log2Ceil(p.mshrEntries + 1), + "READ_SHARED" -> Opcode.ReadShared, + "READ_NSD" -> Opcode.ReadNotSharedDirty, + "READ_UNIQUE" -> Opcode.ReadUnique, + "MAKE_UNIQUE" -> Opcode.MakeUnique, + "EVICT" -> Opcode.Evict, + "WRITEBACK" -> Opcode.WriteBackFull, + "CLEAN_SHARED" -> Opcode.CleanShared, + "CLEAN_INVALID" -> Opcode.CleanInvalid, + "MAKE_INVALID" -> Opcode.MakeInvalid, + "COMP" -> Opcode.Comp, + "COMP_DBID" -> Opcode.CompDBIDResp, + "COMP_ACK" -> Opcode.CompAck, + "COMP_DATA" -> Opcode.CompData, + "SNP_RESP" -> Opcode.SnpResp, + "SNP_DATA" -> Opcode.SnpRespData, + "COPYBACK_DATA" -> Opcode.CopyBackWriteData, + "SNP_UNIQUE" -> Opcode.SnpUnique, + "SNP_INVALID" -> Opcode.SnpCleanInvalid, + "SNP_CLEAN_SHARED" -> Opcode.SnpCleanShared, + "SNP_MAKE_INVALID" -> Opcode.SnpMakeInvalid, + "SNP_SHARED" -> Opcode.SnpNotSharedDirty + ) + ) { + definitions += s"`define COH_$name $value" + } + Files.writeString( + output.resolve("coherence_config.svh"), + "`ifndef COHERENCE_CONFIG_SVH\n`define COHERENCE_CONFIG_SVH\n" + definitions.mkString("\n") + "\n`endif\n" + ) + + val homePorts = scala.collection.mutable.ArrayBuffer[String]() + + def bindFlit(prefix: String, signal: String, flit: Flit): Unit = { + var offset = 0 + for ((name, width) <- fields(flit) if width > 0) { + homePorts += s".${prefix}_$name($signal[$offset +: $width])"; offset += width + } + } + + for (i <- 0 until p.agents) { + for ( + (channel, flit) <- Seq( + "req" -> new RequestFlit(p.chi), + "txRsp" -> new ResponseFlit(p.chi), + "txDat" -> new DataFlit(p.chi), + "rxRsp" -> new ResponseFlit(p.chi), + "rxDat" -> new DataFlit(p.chi), + "snp" -> new SnoopFlit(p.chi) + ) + ) { + homePorts += s".io_requesters_${i}_${channel}_valid(${channel}_valid[$i])" + homePorts += s".io_requesters_${i}_${channel}_ready(${channel}_ready[$i])" + bindFlit(s"io_requesters_${i}_${channel}_bits", s"${channel}_bits[$i]", flit) + } + } + for ( + (name, flit) <- Seq( + "Req" -> new RequestFlit(p.chi), + "RxRsp" -> new ResponseFlit(p.chi), + "RxDat" -> new DataFlit(p.chi), + "Rsp" -> new ResponseFlit(p.chi), + "Dat" -> new DataFlit(p.chi), + "Snp" -> new SnoopFlit(p.chi) + ) + ) { + val signal = s"observed$name" + homePorts += s".io_${signal}_valid(${signal}_valid)" + bindFlit(s"io_${signal}_bits" + (if (name == "Snp") "_flit" else ""), s"${signal}_bits", flit) + if (name == "Snp") + homePorts += s".io_observedSnp_bits_targetNode(observedSnp_bits[$snpWidth +: ${p.chi.nodeIdBits}])" + } + homePorts ++= ports.filter(p => p.startsWith(".io_memory")) + Files.writeString(output.resolve("home_ports.svh"), homePorts.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/scala/Home.scala b/arch/src/main/scala/framework/core/memory/home/src/main/scala/Home.scala new file mode 100644 index 000000000..b03ed63db --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/scala/Home.scala @@ -0,0 +1,72 @@ +package framework.core.memory.home + +import framework.core.memory.home.configs.{Params => ModuleParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.ip.bus.chi._ +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} +import framework.ip.memory.coherence.Coherence +import framework.ip.memory.coherence.configs.CoherenceParams + +/** One Core-owned shared L2/Home and its explicit requester fabric. */ +@instantiable +class Home(b: ModuleParam) extends Module { + val p = b.memory + + val c = p.chi + + @public + val io = IO(new Bundle { + val active = Input(Bool()) + val blockRequesterRsp = Input(Bool()) + + val requesters = Vec( + p.agents, + Flipped(new RequesterPort(c)) + ) + + val memoryReq = Decoupled(new LineRequest(c)) + + val memoryResp = Flipped(Decoupled(new LineResponse(c))) + + val outstanding = Output(UInt(log2Ceil(p.mshrEntries + 1).W)) + + val observedReq = Output(Valid(new RequestFlit(c))) + + val observedRsp = Output(Valid(new ResponseFlit(c))) + + val observedDat = Output(Valid(new DataFlit(c))) + + val observedSnp = Output(Valid(new DirectedSnoop(c))) + + val observedRxRsp = Output(Valid(new ResponseFlit(c))) + + val observedRxDat = Output(Valid(new DataFlit(c))) + + }) + + val home: Instance[Coherence] = Instantiate(new Coherence(b.memory)) + + val fabric: Instance[Fabric] = Instantiate(new Fabric(c, p.agents, p.homeId)) + + fabric.io.active := io.active + fabric.io.blockRequesterRsp := io.blockRequesterRsp + fabric.io.requesters <> io.requesters + home.io.req <> fabric.io.req + home.io.rxRsp <> fabric.io.rxRsp + home.io.rxDat <> fabric.io.rxDat + fabric.io.rsp <> home.io.rsp + fabric.io.dat <> home.io.dat + fabric.io.snp <> home.io.snp + io.memoryReq <> home.io.memoryReq + home.io.memoryResp <> io.memoryResp + io.outstanding := home.io.outstanding + io.observedReq.valid := home.io.req.fire; io.observedReq.bits := home.io.req.bits + io.observedRsp.valid := home.io.rsp.fire; io.observedRsp.bits := home.io.rsp.bits + io.observedDat.valid := home.io.dat.fire; io.observedDat.bits := home.io.dat.bits + io.observedSnp.valid := home.io.snp.fire; io.observedSnp.bits := home.io.snp.bits + io.observedRxRsp.valid := home.io.rxRsp.fire; io.observedRxRsp.bits := home.io.rxRsp.bits + io.observedRxDat.valid := home.io.rxDat.fire; io.observedRxDat.bits := home.io.rxDat.bits +} diff --git a/arch/src/main/scala/framework/core/memory/home/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/memory/home/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..13767bbbf --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/home/src/main/scala/configs/Params.scala @@ -0,0 +1,13 @@ +package framework.core.memory.home.configs + +import framework.ip.memory.coherence.configs.CoherenceParams +import upickle.default.{macroRW, ReadWriter} + +case class Params(memory: CoherenceParams) + +object Params { + implicit val chiRW: ReadWriter[framework.ip.bus.chi.Params] = macroRW + implicit val cacheRW: ReadWriter[framework.ip.memory.cache.configs.CacheParams] = macroRW + implicit val coherenceRW: ReadWriter[CoherenceParams] = macroRW + implicit val rw: ReadWriter[Params] = macroRW +} diff --git a/arch/src/main/scala/framework/core/memory/uncached/Uncached.scala b/arch/src/main/scala/framework/core/memory/uncached/Uncached.scala new file mode 100644 index 000000000..baff80aa0 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/uncached/Uncached.scala @@ -0,0 +1,77 @@ +package framework.core.memory.uncached + +import framework.core.memory.uncached.configs.UncachedParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.ip.bus.axi4 +import framework.core.memory.access.{AccessParams, UncachedRequest, UncachedResponse} + +/** Uncached CPU accesses share one AXI master; responses retain the CPU port and tag. */ +@instantiable +class Uncached(requestp: AccessParams, uncachedp: UncachedParam) extends Module { + val ports = uncachedp.ports + val axip = uncachedp.axi + + @public + val io = IO(new Bundle { + val request = Vec(ports, Flipped(Decoupled(new UncachedRequest(requestp)))) + val response = Vec(ports, Decoupled(new UncachedResponse(requestp))) + val axi = new axi4.Port(axip) + }) + + val indexBits = math.max(1, log2Ceil(ports)) + val idle :: issue :: waitResponse :: respond :: Nil = Enum(4) + val state = RegInit(idle) + val cursor = RegInit(0.U(indexBits.W)) + val owner = Reg(UInt(indexBits.W)) + val command = Reg(new UncachedRequest(requestp)) + val answer = Reg(new UncachedResponse(requestp)) + val addressSent = RegInit(false.B) + val dataSent = RegInit(false.B) + val ordered = (0 until ports).map(i => (cursor +& i.U) % ports.U) + val selected = if (ports == 1) 0.U else PriorityMux(ordered.map(i => io.request(i).valid), ordered) + val present = io.request.map(_.valid).reduce(_ || _) + for (i <- 0 until ports) { + io.request(i).ready := state === idle && present && selected === i.U + io.response(i).valid := state === respond && owner === i.U + io.response(i).bits := answer + when(io.response(i).fire)(state := idle) + } + when(state === idle && present) { + owner := selected + command := (if (ports == 1) io.request.head.bits else io.request(selected).bits) + addressSent := false.B + dataSent := false.B + cursor := Mux(selected === (ports - 1).U, 0.U, selected + 1.U) + state := issue + } + for (address <- Seq(io.axi.aw, io.axi.ar)) { + address.bits := 0.U.asTypeOf(address.bits) + address.bits.addr := command.addr + address.bits.size := command.size + address.bits.burst := 1.U + } + io.axi.aw.valid := state === issue && command.write && !addressSent + io.axi.ar.valid := state === issue && !command.write + val shift = command.addr(log2Ceil(axip.bytes) - 1, 0) + io.axi.w.valid := state === issue && command.write && !dataSent + io.axi.w.bits.data := command.data << (shift << 3) + io.axi.w.bits.strb := (((1.U((axip.bytes + 1).W) << (1.U << command.size)) - 1.U) << shift) + io.axi.w.bits.last := true.B + when(io.axi.aw.fire)(addressSent := true.B) + when(io.axi.w.fire)(dataSent := true.B) + when(io.axi.ar.fire || (command.write && (addressSent || io.axi.aw.fire) && (dataSent || io.axi.w.fire))) { + when(state === issue)(state := waitResponse) + } + io.axi.r.ready := state === waitResponse && !command.write + io.axi.b.ready := state === waitResponse && command.write + when(io.axi.r.fire || io.axi.b.fire) { + when(io.axi.r.fire)(assert(io.axi.r.bits.last)) + answer.tag := command.tag + answer.data := Mux(command.write, 0.U, io.axi.r.bits.data >> (shift << 3)) + answer.error := Mux(command.write, io.axi.b.bits.resp =/= 0.U, io.axi.r.bits.resp =/= 0.U) + state := respond + } +} diff --git a/arch/src/main/scala/framework/core/memory/uncached/configs/UncachedParam.scala b/arch/src/main/scala/framework/core/memory/uncached/configs/UncachedParam.scala new file mode 100644 index 000000000..870d4eea7 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/uncached/configs/UncachedParam.scala @@ -0,0 +1,15 @@ +package framework.core.memory.uncached.configs + +import buckyball.config.UncachedConfig +import framework.ip.bus.axi4 +import upickle.default.{macroRW, ReadWriter} + +case class UncachedParam(ports: Int, axi: axi4.Params) { + require(ports > 0) +} + +object UncachedParam { + def fromProto(p: UncachedConfig): UncachedParam = + UncachedParam(p.getPorts, axi4.Params.fromProto(p.getAxi)) + implicit val rw: ReadWriter[UncachedParam] = macroRW +} diff --git a/arch/src/main/scala/framework/core/memory/uncached/configs/default.toml b/arch/src/main/scala/framework/core/memory/uncached/configs/default.toml new file mode 100644 index 000000000..90c728ed1 --- /dev/null +++ b/arch/src/main/scala/framework/core/memory/uncached/configs/default.toml @@ -0,0 +1,2 @@ +ports = 1 +axi = "default" diff --git a/arch/src/main/scala/framework/core/shm/SHM.scala b/arch/src/main/scala/framework/core/shm/SHM.scala new file mode 100644 index 000000000..33e6e6d5d --- /dev/null +++ b/arch/src/main/scala/framework/core/shm/SHM.scala @@ -0,0 +1,19 @@ +package framework.core.shm + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.tbus.configs.{Params => TBusParam} + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.ip.memory.mesh_shm.{MeshSharedMem, MeshSharedMemIO} + +@instantiable +class SHM(frontendParam: FrontendParam, memParam: MemDomainParam, tbusParam: TBusParam) extends Module { + val p = tbusParam.router.mesh(frontendParam, memParam, tbusParam) + @public + val io = IO(new MeshSharedMemIO(p)) + + val memory: Instance[MeshSharedMem] = Instantiate(new MeshSharedMem(p)) + io <> memory.io +} diff --git a/arch/src/main/scala/framework/core/shm/build.sc b/arch/src/main/scala/framework/core/shm/build.sc new file mode 100644 index 000000000..6ac2f75a6 --- /dev/null +++ b/arch/src/main/scala/framework/core/shm/build.sc @@ -0,0 +1,11 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("SHM.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/core/tbus/build.sc b/arch/src/main/scala/framework/core/tbus/build.sc new file mode 100644 index 000000000..0a875595e --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/build.sc @@ -0,0 +1,53 @@ +import mill._ +import $file.src.main.scala.router.{build => router_build} +import $file.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.tbus.Emit") +} + +// The component library owns the generated table; framework only knows the contract. +def registry(sources: Seq[PathRef], componentRoot: os.Path, destination: os.Path): Seq[PathRef] = { + val declarations = sources.flatMap { source => + val text = os.read(source.path) + val names = "(?m)^object (\\w+) extends ThreadFactory".r.findAllMatchIn(text).map(_.group(1)).toSeq + if (names.isEmpty) Seq.empty + else { + val pkg = "(?m)^package ([\\w.]+)".r.findFirstMatchIn(text).get.group(1) + val component = source.path.relativeTo(componentRoot).segments.head + names.map(name => component -> s"$pkg.$name") + } + }.sortBy(_._1) + require(declarations.map(_._1).distinct.size == declarations.size, "Duplicate component factory") + val entries = declarations.map { case (component, factory) => s""""$component" -> $factory""" } + val output = destination / "ThreadComponents.scala" + os.write.over( + output, + s"""package framework.core.tbus.generated +import framework.core.tbus.ThreadFactory +object ThreadComponents { + val factories: Map[String, ThreadFactory] = Map(${entries.mkString(",\n")}) +} +""" + ) + Seq(PathRef(output)) +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/SharedNetwork.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/TBus.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) ++ + router_build.sources() +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) ++ + router_build.emitSources() +) diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/tbus/src/main/scala/Emit.scala new file mode 100644 index 000000000..51a2ed754 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/Emit.scala @@ -0,0 +1,36 @@ +package framework.core.tbus + +import sims.configs.SimParam +import _root_.configs.GlobalConfig +import java.nio.file.{Files, Paths} +import upickle.default.read + +object Emit { + + def emit( + b: GlobalConfig, + hartId: Int, + threadId: Int, + directory: String, + firtoolOpts: Array[String] + ): Unit = { + val core = + b.device.cores.find(_.hartId == hartId).getOrElse(throw new IllegalArgumentException(s"Unknown hart ID: $hartId")) + val thread = + core.threads.find(_.id == threadId).getOrElse(throw new IllegalArgumentException(s"Unknown thread ID: $threadId")) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new SharedNetwork( + thread.acc.get.frontend, + thread.acc.get.memDomain, + core.tbus.get, + core.threads.map(_.id), + core.threads.map(_.id), + b.sim + ), + firtoolOpts = firtoolOpts, + args = Array("--target-dir", directory, "--split-verilog") + ) + } + +} diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/SharedNetwork.scala b/arch/src/main/scala/framework/core/tbus/src/main/scala/SharedNetwork.scala new file mode 100644 index 000000000..4603165d2 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/SharedNetwork.scala @@ -0,0 +1,129 @@ +package framework.core.tbus + +import sims.configs.SimParam +import framework.lib.arbiter.{Arbiter} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.tbus.configs.{Params => TBusParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.tbus.configs.Layout +import framework.thread.tlink.{ExternalLink, SharedLink} +import framework.core.tbus.router.Router +import framework.acc.memdomain.{LayoutConfig, MvoverCommand, PhysicalBankHash} +import framework.ip.memory.mesh_shm.MeshSharedMemIO + +/** Shared-bank interconnect: endpoint arbitration, mapping and synchronization. */ +@instantiable +class SharedNetwork( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val tbusParam: TBusParam, + val endpointIds: Seq[Int], + val traceIds: Seq[Int], + val simp: SimParam) + extends Module { + val diffTest = simp.diffTest + val p = tbusParam + val participants = p.participants + require(participants.nonEmpty && participants.distinct.size == participants.size) + require(participants.forall(slot => slot >= 0 && slot < p.slots)) + val ports = Layout.portsPerThread(memParam, tbusParam) + + @public + val io = IO(new Bundle { + val shm = Flipped(new MeshSharedMemIO(p.router.mesh(frontendParam, memParam, tbusParam))) + val threads = Vec(participants.size, Flipped(new SharedLink(frontendParam, memParam))) + val external = Option.when(p.router.physicalPorts > 0 || p.externalMove)(new ExternalLink(memParam, tbusParam)) + val bankHashes = Option.when(diffTest)(Output(Vec(Layout.totalBank(memParam), new PhysicalBankHash(frontendParam)))) + }) + + val router: Instance[Router] = + Instantiate(new Router(frontendParam, memParam, tbusParam, endpointIds, traceIds, simp.diffTest)) + val barrier: Instance[Barrier] = Instantiate(new Barrier(tbusParam)) + + router.io.memory <> io.shm + io.bankHashes.foreach(_ := router.io.bank_hashes.get) + + val endpointSlots = VecInit(participants.map(_.U(8.W))) + val endpointOwners = VecInit(participants.map(slot => endpointIds(slot).U(64.W))) + io.external.foreach { port => + router.io.physical <> port.memory + port.lease.foreach { lease => + router.io.lease.get <> lease + router.io.leaseOwnerThreadId.get := endpointOwners(lease.request.bits.endpoint) + when(lease.request.valid && !reset.asBool) { + assert(lease.request.bits.endpoint < participants.size.U, "Shared lease endpoint is outside TBus") + } + } + } + + for ((slot, endpoint) <- participants.zipWithIndex) { + val link = io.threads(endpoint) + for (port <- 0 until ports) { + router.io.mem_req(endpoint * ports + port) <> link.bank.requests(port) + } + router.io.localBanks(slot) <> link.bank.local + router.io.mapping(slot) <> link.bank.mapping + barrier.io.arrive(endpoint) := link.barrierArrive + link.barrierRelease := barrier.io.release(endpoint) + } + + for (slot <- 0 until p.slots if !participants.contains(slot)) { + val local = router.io.localBanks(slot) + local.request.ready := false.B + local.response.valid := false.B + local.response.bits := 0.U.asTypeOf(local.response.bits) + router.io.mapping(slot).valid := false.B + router.io.mapping(slot).bits := 0.U.asTypeOf(router.io.mapping(slot).bits) + } + + val moves = io.external.toSeq.flatMap(_.move.toSeq) ++ io.threads.map(_.move) + val moveArb = Instantiate(new Arbiter(new MvoverCommand, moves.size)) + val moveOwner = Reg(UInt(math.max(1, log2Ceil(moves.size)).W)) + + for ((port, endpoint) <- moves.zipWithIndex) { + moveArb.io.in(endpoint) <> port.command + port.completion.valid := router.io.mvover.completion.valid && moveOwner === endpoint.U + port.completion.bits := router.io.mvover.completion.bits + } + router.io.mvover.command <> moveArb.io.out + router.io.mvover.command.bits.sourceThread := endpointSlots(moveArb.io.out.bits.sourceThread) + router.io.mvover.command.bits.targetThread := endpointSlots(moveArb.io.out.bits.targetThread) + router.io.mvover.completion.ready := VecInit(moves.map(_.completion.ready))(moveOwner) + when(moveArb.io.out.fire) { + assert( + moveArb.io.out.bits.sourceThread < participants.size.U && + moveArb.io.out.bits.targetThread < participants.size.U, + "Bank movement endpoint is outside TBus" + ) + moveOwner := moveArb.io.chosen + } + + val layouts = Instantiate(new Arbiter(new LayoutConfig(frontendParam, memParam), participants.size)) + for (endpoint <- participants.indices) { layouts.io.in(endpoint) <> io.threads(endpoint).bank.config } + router.io.config <> layouts.io.out +} + +/** Core-local barrier; synchronization releases all participating endpoints together. */ +@instantiable +class Barrier(val tbusParam: TBusParam) extends Module { + val count = tbusParam.participants.size + + @public + val io = IO(new Bundle { + val arrive = Input(Vec(count, Bool())) + val release = Output(Vec(count, Bool())) + }) + + val arrived = RegInit(VecInit(Seq.fill(count)(false.B))) + val release = arrived.asUInt.andR + for (endpoint <- 0 until count) { + when(io.arrive(endpoint))(arrived(endpoint) := true.B) + io.release(endpoint) := release + } + when(release)(arrived.foreach(_ := false.B)) +} diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/TBus.scala b/arch/src/main/scala/framework/core/tbus/src/main/scala/TBus.scala new file mode 100644 index 000000000..1a2517406 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/TBus.scala @@ -0,0 +1,148 @@ +package framework.core.tbus + +import sims.configs.SimParam +import framework.core.tbus.configs.Layout + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.thread.tlink.TLinkIO +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.thread.configs.ThreadParam +import framework.ip.memory.mesh_shm.MeshSharedMemIO +import framework.acc.memdomain.MvoverPort +import framework.core.controller.admission.TaskEndpoint +import framework.thread.ant.Task +import framework.core.memory.MemoryPort +import framework.ip.memory.spm +import framework.core.tbus.SharedNetwork +import framework.device.cbus.{Control => TransferControl, C2CTransfer, SharedAxiEndpoint} +import framework.core.clink.SharedPort +import framework.ip.bus.axi4 +import framework.ip.bus.axi4.Interconnect + +@instantiable +class TBus( + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int], + generators: Seq[() => TLinkIO]) + extends Module { + val diffTest = simp.diffTest + val core = corep + require(traceIds.size == core.threads.size) + + @public + val io = IO(new Bundle { + + val control = Option.when(core.threads.nonEmpty)(MixedVec(core.threads.map(thread => + new Task(thread.ant.get, core.privileged.permission) + ))) + + val memory = Option.when(core.threads.nonEmpty)(Vec( + core.threads.size, + new MemoryPort(core.memory.dcache.tracking, core.memory.memoryAccess.walker) + )) + + val remote = Option.when(core.clink.shared)(Flipped(new TaskEndpoint)) + val move = Flipped(new MvoverPort) + val ddr = new axi4.Port(core.interconnect.bus) + val c2c = Option.when(core.clink.shared)(new SharedPort(core.c2c.axi)) + + val shm = Option.when(core.threads.nonEmpty)(Flipped(new MeshSharedMemIO(core.tbus.get.router.mesh( + core.sharedFrontend, + core.sharedMemory, + core.tbus.get + )))) + + val tss = Option.when(core.threads.nonEmpty)(Vec(core.threads.size, new spm.Port(core.tss.get.memory))) + }) + + require(generators.size == core.threads.size) + val endpoints = generators.map(_()) + + require(core.tss.get.ports == endpoints.size) + require(core.interconnect.masters == endpoints.size) + require(core.tbus.get.participants == core.threads.map(_.id) && core.tbus.get.slots == endpoints.size) + + val banks: Instance[SharedNetwork] = Instantiate(new SharedNetwork( + core.sharedFrontend, + core.sharedMemory, + core.tbus.get, + core.threads.map(_.id), + traceIds, + simp + )) + + val ddr: Instance[Interconnect] = Instantiate(new Interconnect(core.interconnect)) + io.ddr <> ddr.io.out + io.shm.get <> banks.io.shm + + if (core.clink.shared) { + require(core.tbus.get.router.physicalPorts == 3 && !core.transferLocalAccess && core.transferLeaseAccess) + val transferControl: Instance[TransferControl] = Instantiate(new TransferControl( + core.sharedMemory, + core.c2c, + core.transferLeaseAccess, + core.transferLocalAccess + )) + val transfer: Instance[C2CTransfer] = Instantiate(new C2CTransfer(core.c2c)) + val endpoint: Instance[SharedAxiEndpoint] = Instantiate(new SharedAxiEndpoint(core.c2c)) + transferControl.io.coreId := core.hartId.U + transfer.io.command <> transferControl.io.command + transferControl.io.completion <> transfer.io.completion + banks.io.external.get.memory(0) <> transfer.io.source + banks.io.external.get.memory(1) <> endpoint.io.read + banks.io.external.get.memory(2) <> endpoint.io.write + banks.io.external.get.lease.get <> transferControl.io.lease.get + io.c2c.get.tx <> transfer.io.mem + endpoint.io.mem <> io.c2c.get.rx + transferControl.io.request <> io.remote.get.request + io.remote.get.reply <> transferControl.io.reply + io.remote.get.busy := !transferControl.io.idle || endpoint.io.busy + } else { + require(core.tbus.get.router.physicalPorts == 0) + + } + require(core.tbus.get.externalMove) + banks.io.external.get.move.get <> io.move + + for ((link, index) <- endpoints.zipWithIndex) { + io.control.get(index) <> link.task + io.tss.get(index) <> link.tss + io.memory.get(index) <> link.memory + ddr.io.in(index) <> link.ddr + banks.io.threads(index) <> link.shm + link.sharedHashes.foreach(_ := banks.io.bankHashes.get) + } +} + +object TBus { + + def generators( + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int], + factories: Map[String, ThreadFactory] + ): Seq[() => TLinkIO] = { + require(corep.threads.size == traceIds.size) + corep.threads.zip(traceIds).map { case (threadp, traceId) => + () => factories(threadp.component).instantiate(corep, threadp, simp, traceId) + } + } + +} + +trait ThreadFactory { + + def instantiate( + corep: CoreParam, + threadp: ThreadParam, + simp: SimParam, + traceId: Int + ): TLinkIO + +} diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/tbus/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..994858c98 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/configs/Params.scala @@ -0,0 +1,41 @@ +package framework.core.tbus.configs + +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.tbus.configs.{Params => TBusParam} + +import upickle.default.{macroRW, ReadWriter} + +case class Params( + router: framework.core.tbus.router.configs.Params, + participants: Seq[Int], + slots: Int, + bankNodes: Seq[Int], + externalMove: Boolean) + +object Params { implicit val rw: ReadWriter[Params] = macroRW } + +object Layout { + + def totalBank(memParam: MemDomainParam): Int = { + require(memParam.sharedEnable) + require(memParam.sharedBankNum > 0) + memParam.sharedBankNum + } + + def portsPerThread(memParam: MemDomainParam, tbusParam: TBusParam): Int = { + if (!memParam.sharedEnable) return 0 + val threads = tbusParam.participants.size + require(threads > 0) + require(memParam.sharedInputChannels > 0) + require(memParam.sharedInputChannels % threads == 0) + val channels = memParam.sharedInputChannels / threads + require(channels > 0) + channels + } + + def totalChannel(memParam: MemDomainParam): Int = + if (memParam.sharedEnable) + memParam.sharedInputChannels + else 0 + +} diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/build.sc b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/build.sc new file mode 100644 index 000000000..11436c15a --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/build.sc @@ -0,0 +1,21 @@ +import mill._ +import $file.^.^.^.^.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.tbus.router.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Router.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/configs/uvm.toml b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/configs/uvm.toml new file mode 100644 index 000000000..cb0c04b6e --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/configs/uvm.toml @@ -0,0 +1,57 @@ +[parameters] +diffTest = false +endpointIds=[10,20] +traceIds=[100,101] + +[parameters.frontend] +ownerId = 7 +commandDataBits = 64 +rob_entries = 4 +bank_alias_enable = true +rs_out_of_order_response = true +bank_id_len = 4 +vbank_id_upper_bound = 3 +shared_bank_id_base = 16 +iter_len = 16 +sub_rob_enable = false +sub_rob_depth = 4 + +[parameters.memory] +sharedPorts = 2 +bankNum = 4 +bankLayoutEnable = true +bankWidth = 128 +bankEntries = 8 +bankMaskLen = 16 +virtualBankCount = 4 +sharedEnable = true +sharedEntries = 32 +sharedBankEntries = 8 +sharedBankNum = 4 +sharedInputChannels = 4 +sharedDefaultGroupCount = 1 +bankRequestPorts = 2 +dma_buswidth = 128 +memAddrLen = 44 +mmioEnable = false +mmioBankNum = 0 +mmioBankEntries = 0 +mmioBankWidth = 0 +mmioReadWidth = 0 + +[parameters.gp] +enable = false +laneNumber = 1 +vLen = 128 +eLen = 64 +iBufWords = 16 +memoryPorts = 1 + +[parameters.tbus] +participants=[0,1] +slots=2 +bankNodes=[0,3] +externalMove=true +[parameters.tbus.router] +physicalPorts=1 +[defines] diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..e24058a6c --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual tbus_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tbus_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.source = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/fixture.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/fixture.svh new file mode 100644 index 000000000..e54717dad --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/fixture.svh @@ -0,0 +1,8 @@ +class fixture extends uvm_object; + `uvm_object_utils(fixture) + bit error[`TR_CLIENTS+`TR_EXTERNAL]; + bit move_error; + function new(string name = "fixture"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/item.svh new file mode 100644 index 000000000..5b652e1be --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/item.svh @@ -0,0 +1,23 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`TR_INPUT_BITS-1:0] inputs; + logic [`TR_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(error, UVM_ALL_ON) + `uvm_field_int(kind, UVM_ALL_ON) + `uvm_field_int(port, UVM_ALL_ON) + `uvm_object_utils_end + logic [127:0] data; + bit error; + int kind, port; + function new(string name = "result"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..17e208527 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual tbus_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tbus_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/peer.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/peer.svh new file mode 100644 index 000000000..bf82043ea --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/agent/peer.svh @@ -0,0 +1,70 @@ +class peer extends uvm_component; + `uvm_component_utils(peer) + virtual tbus_if vif; + fixture image; + const int offsets[`TR_CLIENTS+`TR_EXTERNAL][15] = `TR_MESH_OFFSETS; + bit [127:0] memory[`TR_BANKS][`TR_ROWS]; + bit [15:0] initialized[`TR_BANKS][`TR_ROWS]; + bit + pending[`TR_CLIENTS+`TR_EXTERNAL], + writing[`TR_CLIENTS+`TR_EXTERNAL], + error[`TR_CLIENTS+`TR_EXTERNAL]; + bit [127:0] response[`TR_CLIENTS+`TR_EXTERNAL]; + int tag[`TR_CLIENTS+`TR_EXTERNAL], due[`TR_CLIENTS+`TR_EXTERNAL], cycle; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tbus_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TBus interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(posedge vif.clock); + cycle++; + for (int c = 0; c < `TR_CLIENTS + `TR_EXTERNAL; c++) begin + if (vif.reset) pending[c] = 0; + else begin + if (pending[c] && vif.inputs[offsets[c][8]] && vif.outputs[offsets[c][9]]) pending[c] = 0; + if (vif.outputs[offsets[c][0]] && vif.inputs[offsets[c][1]]) begin + int bank_id = int'(vif.outputs[offsets[c][2]+:$clog2(`TR_BANKS)]); + int row = int'(vif.outputs[offsets[c][3]+1+:16]); + bit write_request = vif.outputs[offsets[c][3]]; + bit [15:0] mask = vif.outputs[offsets[c][5]+:16]; + if (pending[c] || bank_id >= `TR_BANKS || row >= `TR_ROWS) + `uvm_fatal("BACKEND", "Invalid or overlapping Mesh request") + pending[c] = 1; + writing[c] = write_request; + error[c] = image.error[c]; + tag[c] = int'(vif.outputs[offsets[c][6]+:`TR_TAG_BITS]); + due[c] = cycle + 2 + c % 4; + response[c] = 0; + if (write_request && !error[c]) begin + for (int b = 0; b < 16; b++) + if (mask[b]) begin + memory[bank_id][row][b*8+:8] = vif.outputs[offsets[c][4]+b*8+:8]; + initialized[bank_id][row][b] = 1; + end + end else if (!write_request) begin + if (initialized[bank_id][row] != '1) + `uvm_fatal("MEMORY", "Uninitialized Mesh bank read") + response[c] = memory[bank_id][row]; + end + end + end + end + @(negedge vif.clock); + vif.backend = '0; + for (int c = 0; c < `TR_CLIENTS + `TR_EXTERNAL; c++) begin + vif.backend[offsets[c][1]] = !pending[c] && cycle % 5 != c % 5; + vif.backend[offsets[c][8]] = pending[c] && cycle >= due[c]; + vif.backend[offsets[c][10]+:128] = response[c]; + vif.backend[offsets[c][11]+:16] = '1; + vif.backend[offsets[c][12]+:`TR_TAG_BITS] = tag[c]; + vif.backend[offsets[c][13]+:2] = {writing[c], error[c]}; + vif.backend[offsets[c][14]] = 1; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/env/env.svh new file mode 100644 index 000000000..eeda0417e --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/env/env.svh @@ -0,0 +1,36 @@ +class env extends uvm_env; + `uvm_component_utils(env) + uvm_analysis_imp #(uvm_sequence_item, env) reset_events; + driver source; + monitor mon; + peer backend; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 1) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_events = new("reset_events", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + backend = peer::type_id::create("backend", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 1)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + fixture image = fixture::type_id::create("image"); + reference.image = image; + backend.image = image; + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + reference.cancelled.connect(reset_events); + endfunction + function void write(uvm_sequence_item t); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/env/model.svh new file mode 100644 index 000000000..96650b9d3 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/env/model.svh @@ -0,0 +1,379 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + localparam int BANKS = `TR_BANKS; + fixture image; + uvm_analysis_port #(result_item) expected, actual; + uvm_analysis_port #(uvm_sequence_item) cancelled; + const int clients[`TR_CLIENTS][17] = `TR_CLIENT_OFFSETS; + const int mesh[`TR_CLIENTS+`TR_EXTERNAL][15] = `TR_MESH_OFFSETS; + const int mappings[`TR_THREADS][10] = `TR_MAPPING_OFFSETS; + const int owners[`TR_THREADS] = `TR_OWNERS; + const int copies[`TR_LOCAL_FIELDS][4] = `TR_LOCAL_COPIES; + bit live[BANKS]; + int owner[BANKS], architecture[BANKS], alias_id[BANKS], group_id[BANKS], leases[BANKS]; + int layouts[`TR_THREADS][`TR_ALIASES], committed[`TR_THREADS][`TR_ALIASES]; + logic [127:0] memory[BANKS][`TR_ROWS]; + bit [15:0] initialized[BANKS][`TR_ROWS]; + bit pending[`TR_CLIENTS+`TR_EXTERNAL], writing[`TR_CLIENTS+`TR_EXTERNAL]; + int + selected[`TR_CLIENTS+`TR_EXTERNAL], + rows[`TR_CLIENTS+`TR_EXTERNAL], + tags[`TR_CLIENTS+`TR_EXTERNAL]; + bit accepted_now[`TR_CLIENTS+`TR_EXTERNAL]; + logic [127:0] data_now[`TR_CLIENTS+`TR_EXTERNAL]; + bit [15:0] masks_now[`TR_CLIENTS+`TR_EXTERNAL]; + bit lease_pending, move_pending; + covergroup behavior with function sample ( + int port, bit writing, int owner, int event_kind, bit error, bit blocked + ); + option.per_instance = 1; + ports: coverpoint port {bins all[] = {[0 : `TR_CLIENTS + `TR_EXTERNAL - 1]};} + direction: coverpoint writing; + tenants: coverpoint owner {bins all[] = {[0 : `TR_THREADS - 1]};} + events: coverpoint event_kind { + bins access = {0}; + bins configuration = {1}; + bins renamed = {2}; + bins retirement = {3}; + bins export_lease = {4}; + bins release_lease = {5}; + bins move = {6}; + bins transfer = {7}; + } + errors: coverpoint error; + backpressure: coverpoint blocked; + endgroup + covergroup epochs with function sample (bit pending); + option.per_instance = 1; + cancellation: coverpoint pending; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + cancelled = new("cancelled", this); + behavior = new(); + epochs = new(); + endfunction + function void verify(bit ok, string reason); + evidence::checks++; + if (!ok) `uvm_fatal("TBUS_ROUTER", reason) + endfunction + function int tenant(longint unsigned id); + foreach (owners[t]) if (owners[t] == id) return t; + `uvm_fatal("OWNER", "Unknown configured owner") + return -1; + endfunction + function int route(int thread, int tag, int group_index); + int found = -1; + foreach (live[b]) + if (live[b] && owner[b] == thread && alias_id[b] == tag && group_id[b] == group_index) begin + verify(found == -1, "Ambiguous shared binding"); + found = b; + end + verify(found >= 0, "Missing shared binding"); + return found; + endfunction + function void request(int port, int bank_id, int row, bit wr, logic [127:0] data, + bit [15:0] mask); + result_item result = result_item::type_id::create("expected"); + verify(!pending[port] && row < `TR_ROWS, "Overlapping or out-of-range shared access"); + result.port = port; + result.kind = port < `TR_CLIENTS ? 0 : 1; + result.error = image.error[port]; + result.set_transaction_id(port); + selected[port] = bank_id; + rows[port] = row; + data_now[port] = data; + masks_now[port] = mask; + writing[port] = wr; + accepted_now[port] = 1; + pending[port] = 1; + if (wr) begin + if (!image.error[port]) + for (int b = 0; b < 16; b++) + if (mask[b]) begin + memory[bank_id][row][b*8+:8] = data[b*8+:8]; + initialized[bank_id][row][b] = 1; + end + result.data = 0; + end else begin + verify(initialized[bank_id][row] == '1, "Read of uninitialized shared memory"); + result.data = memory[bank_id][row]; + end + expected.write(result); + behavior.sample(port, wr, owner[bank_id], 0, image.error[port], 0); + endfunction + function void write(cycle_item t); + if (t.reset) begin + bit unfinished = lease_pending || move_pending; + foreach (pending[c]) unfinished |= pending[c]; + epochs.sample(unfinished); + cancelled.write(null); + foreach (live[b]) begin + live[b] = 0; + leases[b] = 0; + end + foreach (layouts[o, a]) begin + layouts[o][a] = 0; + committed[o][a] = a; + end + foreach (pending[c]) begin + pending[c] = 0; + tags[c] = 0; + end + lease_pending = 0; + move_pending = 0; + return; + end + foreach (accepted_now[c]) accepted_now[c] = 0; + for (int c = 0; c < `TR_CLIENTS; c++) begin + bit rd = t.inputs[clients[c][0]] && t.outputs[clients[c][1]]; + bit wr = t.inputs[clients[c][6]] && t.outputs[clients[c][7]]; + verify(!(rd && wr), "Client accepted both read and write"); + if (rd || wr) begin + int thread = tenant(t.inputs[clients[c][16]+:64]); + int alias_value = int'(t.inputs[clients[c][14]+:`TR_IN_MEM_REQ_0_BANK_ID_WIDTH]); + int group_index = int'(t.inputs[clients[c][15]+:`TR_IN_MEM_REQ_0_GROUP_ID_WIDTH]); + request(c, route(thread, alias_value, group_index), + int'(t.inputs[clients[c][wr?8 : 2]+:16]), wr, t.inputs[clients[c][9]+:128], + t.inputs[clients[c][10]+:16]); + end + if(t.outputs[clients[c][3]]&&t.inputs[clients[c][4]]||t.outputs[clients[c][11]]&&t.inputs[clients[c][12]])begin + result_item result = result_item::type_id::create("actual"); + verify(pending[c], "Client response has no owner"); + result.port = c; + result.kind = 0; + result.data = writing[c] ? 128'd0 : t.outputs[clients[c][5]+:128]; + result.error = writing[c] ? !t.outputs[clients[c][13]] : 0; + result.set_transaction_id(c); + actual.write(result); + pending[c] = 0; + end + if(t.outputs[clients[c][3]]&&!t.inputs[clients[c][4]]||t.outputs[clients[c][11]]&&!t.inputs[clients[c][12]]) + behavior.sample(c, writing[c], 0, 0, 0, 1); + end + if (`RI(t, PHYSICAL_0_REQUEST_VALID) && `RO(t, PHYSICAL_0_REQUEST_READY)) begin + longint unsigned address = `RI(t, PHYSICAL_0_REQUEST_BITS_ADDRESS); + int bank_id = int'(address / (`TR_ROWS * 16)); + int row = int'(address % (16 * `TR_ROWS)) / 16; + verify(address % 16 == 0 && bank_id < BANKS && live[bank_id], + "Invalid physical shared access"); + request(`TR_CLIENTS, bank_id, row, `RI(t, PHYSICAL_0_REQUEST_BITS_WRITE), + `RI(t, PHYSICAL_0_REQUEST_BITS_DATA), `RI(t, PHYSICAL_0_REQUEST_BITS_MASK)); + end + if (`RO(t, PHYSICAL_0_RESPONSE_VALID) && `RI(t, PHYSICAL_0_RESPONSE_READY)) begin + result_item result = result_item::type_id::create("physical_actual"); + verify(pending[`TR_CLIENTS], "Physical response has no owner"); + result.port = `TR_CLIENTS; + result.kind = 1; + result.data = `RO(t, PHYSICAL_0_RESPONSE_BITS_DATA); + result.error = `RO(t, PHYSICAL_0_RESPONSE_BITS_ERROR); + result.set_transaction_id(`TR_CLIENTS); + actual.write(result); + pending[`TR_CLIENTS] = 0; + end + for (int c = 0; c < `TR_CLIENTS + `TR_EXTERNAL; c++) begin + if (t.outputs[mesh[c][0]] && t.inputs[mesh[c][1]]) begin + verify(accepted_now[c], "Mesh request has no accepted client"); + verify(t.outputs[mesh[c][2]+:$clog2(BANKS + )] == selected[c] && t.outputs[mesh[c][3]+:17] == ((rows[c] << 1) | writing[c]), + "Shared bank, row or direction routing mismatch"); + verify(t.outputs[mesh[c][7]], "Shared row request did not terminate its packet"); + if (c < `TR_CLIENTS) begin + verify(t.outputs[mesh[c][6]+:`TR_TAG_BITS] == tags[c], + "Shared channel tag did not advance"); + tags[c] = (tags[c] + 1) % (1 << `TR_TAG_BITS); + end else verify(t.outputs[mesh[c][6]+:`TR_TAG_BITS] == 0, "Physical port tag must be zero"); + if (writing[c]) + verify( + t.outputs[mesh[c][4]+:128]===data_now[c]&&t.outputs[mesh[c][5]+:16]==masks_now[c], + "Shared write data or mask changed"); + accepted_now[c] = 0; + end + verify(!accepted_now[c], "Client accepted without forwarding its Mesh request"); + end + for (int thread = 0; thread < `TR_THREADS; thread++) begin + if (t.inputs[mappings[thread][0]]) begin + int previous = int'(t.inputs[mappings[thread][4]+:`TR_IN_MAPPING_0_BITS_PREVIOUS_WIDTH]); + bit held; + held = 0; + foreach (live[b]) + if (live[b] && owner[b] == thread && alias_id[b] == previous && leases[b] != 0) held = 1; + if (held && (t.inputs[mappings[thread][7]] || t.inputs[mappings[thread][8]])) begin + verify(!t.outputs[mappings[thread][1]], "Mapping reclaimed a live export"); + behavior.sample(0, 0, thread, 3, 0, 1); + end + end + if (t.inputs[mappings[thread][0]] && t.outputs[mappings[thread][1]]) begin + int bank_id = int'(t.inputs[mappings[thread][2]+:`TR_IN_MAPPING_0_BITS_BANK_WIDTH]); + int tag = int'(t.inputs[mappings[thread][3]+:`TR_IN_MAPPING_0_BITS_VBANK_WIDTH]); + int previous = int'(t.inputs[mappings[thread][4]+:`TR_IN_MAPPING_0_BITS_PREVIOUS_WIDTH]); + verify(t.inputs[mappings[thread][5]+:64] == owners[thread], + "Mapping port changed its owner"); + if (t.inputs[mappings[thread][6]]) begin + bit free_bank[BANKS]; + foreach (live[b]) + free_bank[b]=!live[b]||(t.inputs[mappings[thread][7]]&&owner[b]==thread&&alias_id[b]==previous); + for (int group_index = 0; group_index < layouts[thread][bank_id]; group_index++) begin + int chosen = -1; + foreach (free_bank[b]) if (free_bank[b] && chosen < 0) chosen = b; + verify(chosen >= 0, "Alias allocation exceeded capacity"); + free_bank[chosen] = 0; + verify(leases[chosen] == 0, "Replaced a leased shared bank"); + live[chosen] = 1; + owner[chosen] = thread; + architecture[chosen] = bank_id; + alias_id[chosen] = tag; + group_id[chosen] = group_index; + end + behavior.sample(0, 0, thread, 2, 0, 0); + end + if (t.inputs[mappings[thread][8]]) begin + committed[thread][bank_id] = tag; + foreach (live[b]) + if (live[b] && owner[b] == thread && alias_id[b] == previous) live[b] = 0; + behavior.sample(0, 0, thread, 3, 0, 0); + end + end + end + if (`RI(t, CONFIG_VALID) && `RO(t, CONFIG_READY)) begin + int thread = tenant(`RI(t, CONFIG_BITS_THREAD_ID)); + int bank_id = `RI(t, CONFIG_BITS_VBANK_ID); + int group_index = `RI(t, CONFIG_BITS_GROUP_ID); + if (`RI(t, CONFIG_BITS_TRANSFER)) begin + int source = `RI(t, CONFIG_BITS_SOURCE_BANK_ID); + int prior = layouts[thread][bank_id]; + foreach (live[b]) + if (live[b] && owner[b] == thread && architecture[b] == source) begin + verify(leases[b] == 0, "Transferred a leased bank"); + alias_id[b] = bank_id; + architecture[b] = bank_id; + group_id[b] += prior; + end + layouts[thread][bank_id] += layouts[thread][source]; + layouts[thread][source] = 0; + behavior.sample(0, 0, thread, 7, 0, 0); + end else begin + if (group_index == 0 || !`RI(t, CONFIG_BITS_ALLOC)) + foreach (live[b]) + if (live[b] && owner[b] == thread && architecture[b] == bank_id) begin + verify(leases[b] == 0, "Changed a leased bank mapping"); + live[b] = 0; + end + if (`RI(t, CONFIG_BITS_ALLOC)) begin + int chosen = -1; + foreach (live[b]) if (!live[b] && chosen < 0) chosen = b; + verify(chosen >= 0, "Configuration exceeded shared capacity"); + live[chosen] = 1; + owner[chosen] = thread; + architecture[chosen] = bank_id; + alias_id[chosen] = bank_id; + group_id[chosen] = group_index; + layouts[thread][bank_id] = group_index + 1; + end else layouts[thread][bank_id] = 0; + behavior.sample(0, 0, thread, 1, 0, 0); + end + end + if (`RI(t, LEASE_REQUEST_VALID) && `RO(t, LEASE_REQUEST_READY)) begin + int thread = tenant(`RI(t, LEASEOWNERTHREADID)); + int bank_id = `RI(t, LEASE_REQUEST_BITS_VBANK); + int group_index = `RI(t, LEASE_REQUEST_BITS_GROUP); + int chosen = -1; + bit release_lease = `RI(t, LEASE_REQUEST_BITS_RELEASE); + result_item result = result_item::type_id::create("lease_expected"); + verify(!lease_pending, "Overlapping lease requests"); + foreach (live[b]) + if(live[b]&&owner[b]==thread&&architecture[b]==bank_id&&group_id[b]==group_index&&(release_lease?leases[b]!=0:alias_id[b]==committed[thread][bank_id]))begin + verify(chosen < 0, "Ambiguous lease binding"); + chosen = b; + end + verify(chosen >= 0, "Lease has no current shared binding"); + if (release_lease) begin + leases[chosen]--; + result.data = 0; + end else begin + leases[chosen]++; + result.data = chosen * `TR_ROWS * 16; + end + result.port = -1; + result.kind = 2; + result.set_transaction_id(100); + expected.write(result); + lease_pending = 1; + behavior.sample(0, 0, thread, release_lease ? 5 : 4, 0, 0); + end + if (`RO(t, LEASE_RESPONSE_VALID) && `RI(t, LEASE_RESPONSE_READY)) begin + result_item result = result_item::type_id::create("lease_actual"); + verify(lease_pending, "Lease response without request"); + result.port = -1; + result.kind = 2; + result.data = `RO(t, LEASE_RESPONSE_BITS); + result.set_transaction_id(100); + actual.write(result); + lease_pending = 0; + end + if (`RI(t, MVOVER_COMMAND_VALID) && `RO(t, MVOVER_COMMAND_READY)) begin + result_item result = result_item::type_id::create("move_expected"); + verify(!move_pending, "Overlapping move requests"); + verify(`RO(t, MEMORY_TRANSFERCOMMAND_VALID) && `RO(t, MEMORY_TRANSFERCOMMAND_BITS_TAG) == 0, + "Move was not forwarded with its reserved tag"); + verify(`RI(t, MVOVER_COMMAND_BITS_SOURCETHREAD) == + `RO(t, MEMORY_TRANSFERCOMMAND_BITS_SOURCETHREAD), + "Move command field SOURCETHREAD changed"); + verify(`RI(t, MVOVER_COMMAND_BITS_TARGETTHREAD) == + `RO(t, MEMORY_TRANSFERCOMMAND_BITS_TARGETTHREAD), + "Move command field TARGETTHREAD changed"); + verify(`RI(t, MVOVER_COMMAND_BITS_SOURCEBANK) == + `RO(t, MEMORY_TRANSFERCOMMAND_BITS_SOURCEBANK), + "Move command field SOURCEBANK changed"); + verify(`RI(t, MVOVER_COMMAND_BITS_TARGETBANK) == + `RO(t, MEMORY_TRANSFERCOMMAND_BITS_TARGETBANK), + "Move command field TARGETBANK changed"); + verify(`RI(t, MVOVER_COMMAND_BITS_SOURCEADDR) == + `RO(t, MEMORY_TRANSFERCOMMAND_BITS_SOURCEADDR), + "Move command field SOURCEADDR changed"); + verify(`RI(t, MVOVER_COMMAND_BITS_TARGETADDR) == + `RO(t, MEMORY_TRANSFERCOMMAND_BITS_TARGETADDR), + "Move command field TARGETADDR changed"); + verify(`RI(t, MVOVER_COMMAND_BITS_ROWS) == `RO(t, MEMORY_TRANSFERCOMMAND_BITS_ROWS), + "Move command field ROWS changed"); + + result.port = -2; + result.kind = 3; + result.data = 0; + result.error = image.move_error; + result.set_transaction_id(101); + expected.write(result); + move_pending = 1; + behavior.sample(0, 0, 0, 6, image.move_error, 0); + end + verify(`RO(t, MVOVER_COMMAND_READY) == `RI(t, MEMORY_TRANSFERCOMMAND_READY), + "Move readiness changed"); + verify(`RO(t, MVOVER_COMPLETION_VALID) == `RI(t, MEMORY_TRANSFERCOMPLETION_VALID) && + `RO(t, MEMORY_TRANSFERCOMPLETION_READY) == `RI(t, MVOVER_COMPLETION_READY), + "Move completion handshake changed"); + if (`RO(t, MVOVER_COMPLETION_VALID) && `RI(t, MVOVER_COMPLETION_READY)) begin + result_item result = result_item::type_id::create("move_actual"); + verify(move_pending, "Unsolicited move completion"); + result.port = -2; + result.kind = 3; + result.data = 0; + result.error = `RO(t, MVOVER_COMPLETION_BITS); + result.set_transaction_id(101); + actual.write(result); + move_pending = 0; + end + foreach (copies[f]) + if (copies[f][3] < 0 || t.inputs[copies[f][3]]) + for (int b = 0; b < copies[f][2]; b++) + verify(t.outputs[copies[f][1]+b] === t.inputs[copies[f][0]+b], + "Thread-local bank passthrough changed"); + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + foreach (pending[c]) verify(!pending[c], "Shared access did not drain"); + verify(!move_pending && !lease_pending, "Control transaction did not drain"); + foreach (leases[b]) verify(leases[b] == 0, "Outstanding shared export"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/filelist.f new file mode 100644 index 000000000..4c29c10a6 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/filelist.f @@ -0,0 +1,9 @@ ++define+BUCKYBALL_DISABLE_MTRACE_DPI ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Router/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/interface.sv new file mode 100644 index 000000000..656dc02b5 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/interface.sv @@ -0,0 +1,9 @@ +`include "parameters.svh" +interface tbus_if ( + input logic clock +); + logic reset = 1; + logic [`TR_INPUT_BITS-1:0] source = '0, backend = '0; + wire [ `TR_INPUT_BITS-1:0] inputs = source | backend; + wire [`TR_OUTPUT_BITS-1:0] outputs; +endinterface diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/package.sv b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/package.sv new file mode 100644 index 000000000..98fb5ea9e --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/package.sv @@ -0,0 +1,18 @@ +`include "parameters.svh" +`define RI(T, F) T.inputs[`TR_IN_``F``_OFFSET+:`TR_IN_``F``_WIDTH] +`define RO(T, F) T.outputs[`TR_OUT_``F``_OFFSET+:`TR_OUT_``F``_WIDTH] +package tbus_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/fixture.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/peer.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" + `include "tests/invalid.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..414a104be --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/seq/sequence.svh @@ -0,0 +1,281 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + model reference; + const int clients[`TR_CLIENTS][17] = `TR_CLIENT_OFFSETS; + const int mappings[`TR_THREADS][10] = `TR_MAPPING_OFFSETS; + const int owners[`TR_THREADS] = `TR_OWNERS; + const int copies[`TR_LOCAL_FIELDS][4] = `TR_LOCAL_COPIES; + logic [`TR_INPUT_BITS-1:0] pins; + logic [`TR_OUTPUT_BITS-1:0] observed; + int mapping_fired[`TR_THREADS]; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + for (int thread = 0; thread < `TR_THREADS; thread++) + if (pins[mappings[thread][0]] && observed[mappings[thread][1]]) begin + pins[mappings[thread][0]] = 0; + mapping_fired[thread]++; + end + endtask + task layout(int thread, int bank_id, int groups, bit transfer = 0, int source = 0); + for (int group_index = 0; group_index < (groups == 0 ? 1 : groups); group_index++) begin + pins[`TR_IN_CONFIG_VALID_OFFSET+:`TR_IN_CONFIG_VALID_WIDTH] = 1; + pins[`TR_IN_CONFIG_BITS_THREAD_ID_OFFSET+:`TR_IN_CONFIG_BITS_THREAD_ID_WIDTH]=owners[thread]; + pins[`TR_IN_CONFIG_BITS_VBANK_ID_OFFSET+:`TR_IN_CONFIG_BITS_VBANK_ID_WIDTH] = bank_id; + pins[`TR_IN_CONFIG_BITS_GROUP_ID_OFFSET+:`TR_IN_CONFIG_BITS_GROUP_ID_WIDTH] = group_index; + pins[`TR_IN_CONFIG_BITS_ALLOC_OFFSET+:`TR_IN_CONFIG_BITS_ALLOC_WIDTH] = groups > 0; + pins[`TR_IN_CONFIG_BITS_IS_MULTI_OFFSET+:`TR_IN_CONFIG_BITS_IS_MULTI_WIDTH] = groups > 1; + pins[`TR_IN_CONFIG_BITS_TRANSFER_OFFSET+:`TR_IN_CONFIG_BITS_TRANSFER_WIDTH] = transfer; + pins[`TR_IN_CONFIG_BITS_SOURCE_BANK_ID_OFFSET+:`TR_IN_CONFIG_BITS_SOURCE_BANK_ID_WIDTH]=source; + pins[`TR_IN_CONFIG_BITS_IS_SHARED_OFFSET+:`TR_IN_CONFIG_BITS_IS_SHARED_WIDTH] = 1; + do tick(); while (!observed[`TR_OUT_CONFIG_READY_OFFSET]); + pins[`TR_IN_CONFIG_VALID_OFFSET+:`TR_IN_CONFIG_VALID_WIDTH] = 0; + if (transfer) break; + end + repeat (2) tick(); + endtask + task map_offer(int thread, int bank_id, int tag, int previous, bit allocate, bit retire, + bit reuse = 0); + pins[mappings[thread][0]] = 1; + pins[mappings[thread][2]+:`TR_IN_MAPPING_0_BITS_BANK_WIDTH] = bank_id; + pins[mappings[thread][3]+:`TR_IN_MAPPING_0_BITS_VBANK_WIDTH] = tag; + pins[mappings[thread][4]+:`TR_IN_MAPPING_0_BITS_PREVIOUS_WIDTH] = previous; + pins[mappings[thread][5]+:64] = owners[thread]; + pins[mappings[thread][6]] = allocate; + pins[mappings[thread][7]] = reuse; + pins[mappings[thread][8]] = retire; + pins[mappings[thread][9]] = allocate; + endtask + task mapping(int thread, int bank_id, int tag, int previous, bit allocate, bit retire, + bit reuse = 0); + map_offer(thread, bank_id, tag, previous, allocate, retire, reuse); + do tick(); while (pins[mappings[thread][0]]); + repeat (2) tick(); + endtask + task access (int port, int bank_id, int group_index, int row, bit write_request, + logic [127:0] data = 0, bit [15:0] mask = '1, int stall = 2); + pins[clients[port][14]+:`TR_IN_MEM_REQ_0_BANK_ID_WIDTH] = bank_id; + pins[clients[port][15]+:`TR_IN_MEM_REQ_0_GROUP_ID_WIDTH] = group_index; + pins[clients[port][16]+:64] = owners[port/(`TR_CLIENTS/`TR_THREADS)]; + pins[clients[port][write_request?6 : 0]] = 1; + pins[clients[port][write_request?8 : 2]+:16] = row; + pins[clients[port][9]+:128] = data; + pins[clients[port][10]+:16] = mask; + pins[clients[port][write_request?12 : 4]] = 0; + do tick(); while (!observed[clients[port][write_request?7 : 1]]); + pins[clients[port][write_request?6 : 0]] = 0; + // Ownership follows the accepted command while live inputs move to a different bank. + pins[clients[port][14]+:`TR_IN_MEM_REQ_0_BANK_ID_WIDTH] = (bank_id + 1) % `TR_ALIASES; + do tick(); while (!observed[clients[port][write_request?11 : 3]]); + repeat (stall) tick(); + pins[clients[port][write_request?12 : 4]] = 1; + tick(); + endtask + task lease(int thread, int bank_id, int group_index, bit release_lease, + output longint unsigned address); + pins[`TR_IN_LEASEOWNERTHREADID_OFFSET+:`TR_IN_LEASEOWNERTHREADID_WIDTH] = owners[thread]; + pins[`TR_IN_LEASE_REQUEST_BITS_ENDPOINT_OFFSET+:`TR_IN_LEASE_REQUEST_BITS_ENDPOINT_WIDTH]=thread; + pins[`TR_IN_LEASE_REQUEST_BITS_VBANK_OFFSET+:`TR_IN_LEASE_REQUEST_BITS_VBANK_WIDTH] = bank_id; + pins[`TR_IN_LEASE_REQUEST_BITS_GROUP_OFFSET+:`TR_IN_LEASE_REQUEST_BITS_GROUP_WIDTH]=group_index; + pins[`TR_IN_LEASE_REQUEST_BITS_RELEASE_OFFSET+:`TR_IN_LEASE_REQUEST_BITS_RELEASE_WIDTH]=release_lease; + pins[`TR_IN_LEASE_REQUEST_VALID_OFFSET+:`TR_IN_LEASE_REQUEST_VALID_WIDTH] = 1; + pins[`TR_IN_LEASE_RESPONSE_READY_OFFSET+:`TR_IN_LEASE_RESPONSE_READY_WIDTH] = 0; + do tick(); while (!observed[`TR_OUT_LEASE_REQUEST_READY_OFFSET]); + pins[`TR_IN_LEASE_REQUEST_VALID_OFFSET+:`TR_IN_LEASE_REQUEST_VALID_WIDTH] = 0; + do tick(); while (!observed[`TR_OUT_LEASE_RESPONSE_VALID_OFFSET]); + address = observed[`TR_OUT_LEASE_RESPONSE_BITS_OFFSET+:64]; + repeat (3) tick(); + pins[`TR_IN_LEASE_RESPONSE_READY_OFFSET+:`TR_IN_LEASE_RESPONSE_READY_WIDTH] = 1; + tick(); + endtask + task physical(longint unsigned address, bit writing, logic [127:0] data = 0, + bit [15:0] mask = '1); + pins[`TR_IN_PHYSICAL_0_REQUEST_BITS_ADDRESS_OFFSET+:`TR_IN_PHYSICAL_0_REQUEST_BITS_ADDRESS_WIDTH]=address; + pins[`TR_IN_PHYSICAL_0_REQUEST_BITS_WRITE_OFFSET+:`TR_IN_PHYSICAL_0_REQUEST_BITS_WRITE_WIDTH]=writing; + pins[`TR_IN_PHYSICAL_0_REQUEST_BITS_DATA_OFFSET+:`TR_IN_PHYSICAL_0_REQUEST_BITS_DATA_WIDTH]=data; + pins[`TR_IN_PHYSICAL_0_REQUEST_BITS_MASK_OFFSET+:`TR_IN_PHYSICAL_0_REQUEST_BITS_MASK_WIDTH]=mask; + pins[`TR_IN_PHYSICAL_0_REQUEST_VALID_OFFSET+:`TR_IN_PHYSICAL_0_REQUEST_VALID_WIDTH] = 1; + pins[`TR_IN_PHYSICAL_0_RESPONSE_READY_OFFSET+:`TR_IN_PHYSICAL_0_RESPONSE_READY_WIDTH] = 0; + do tick(); while (!observed[`TR_OUT_PHYSICAL_0_REQUEST_READY_OFFSET]); + pins[`TR_IN_PHYSICAL_0_REQUEST_VALID_OFFSET+:`TR_IN_PHYSICAL_0_REQUEST_VALID_WIDTH] = 0; + do tick(); while (!observed[`TR_OUT_PHYSICAL_0_RESPONSE_VALID_OFFSET]); + repeat (4) tick(); + pins[`TR_IN_PHYSICAL_0_RESPONSE_READY_OFFSET+:`TR_IN_PHYSICAL_0_RESPONSE_READY_WIDTH] = 1; + tick(); + endtask + task move(int serial); + reference.image.move_error = serial % 2; + pins[`TR_IN_MVOVER_COMMAND_BITS_SOURCETHREAD_OFFSET+:`TR_IN_MVOVER_COMMAND_BITS_SOURCETHREAD_WIDTH]=serial; + pins[`TR_IN_MVOVER_COMMAND_BITS_TARGETTHREAD_OFFSET+:`TR_IN_MVOVER_COMMAND_BITS_TARGETTHREAD_WIDTH]=255-serial; + pins[`TR_IN_MVOVER_COMMAND_BITS_SOURCEBANK_OFFSET+:`TR_IN_MVOVER_COMMAND_BITS_SOURCEBANK_WIDTH]=serial*7; + pins[`TR_IN_MVOVER_COMMAND_BITS_TARGETBANK_OFFSET+:`TR_IN_MVOVER_COMMAND_BITS_TARGETBANK_WIDTH]=1023-serial; + pins[`TR_IN_MVOVER_COMMAND_BITS_SOURCEADDR_OFFSET+:`TR_IN_MVOVER_COMMAND_BITS_SOURCEADDR_WIDTH]=serial*131; + pins[`TR_IN_MVOVER_COMMAND_BITS_TARGETADDR_OFFSET+:`TR_IN_MVOVER_COMMAND_BITS_TARGETADDR_WIDTH]=65535-serial; + pins[`TR_IN_MVOVER_COMMAND_BITS_ROWS_OFFSET+:`TR_IN_MVOVER_COMMAND_BITS_ROWS_WIDTH]=serial==15?65536:serial+1; + pins[`TR_IN_MVOVER_COMMAND_VALID_OFFSET+:`TR_IN_MVOVER_COMMAND_VALID_WIDTH] = 1; + pins[`TR_IN_MEMORY_TRANSFERCOMMAND_READY_OFFSET+:`TR_IN_MEMORY_TRANSFERCOMMAND_READY_WIDTH] = 0; + repeat (3) tick(); + pins[`TR_IN_MEMORY_TRANSFERCOMMAND_READY_OFFSET+:`TR_IN_MEMORY_TRANSFERCOMMAND_READY_WIDTH] = 1; + tick(); + pins[`TR_IN_MVOVER_COMMAND_VALID_OFFSET+:`TR_IN_MVOVER_COMMAND_VALID_WIDTH] = 0; + pins[`TR_IN_MEMORY_TRANSFERCOMMAND_READY_OFFSET+:`TR_IN_MEMORY_TRANSFERCOMMAND_READY_WIDTH] = 0; + repeat (2) tick(); + pins[`TR_IN_MEMORY_TRANSFERCOMPLETION_VALID_OFFSET+:`TR_IN_MEMORY_TRANSFERCOMPLETION_VALID_WIDTH]=1; + pins[`TR_IN_MEMORY_TRANSFERCOMPLETION_BITS_TAG_OFFSET+:`TR_IN_MEMORY_TRANSFERCOMPLETION_BITS_TAG_WIDTH]='1; + pins[`TR_IN_MEMORY_TRANSFERCOMPLETION_BITS_ERROR_OFFSET+:`TR_IN_MEMORY_TRANSFERCOMPLETION_BITS_ERROR_WIDTH]=reference.image.move_error; + pins[`TR_IN_MVOVER_COMPLETION_READY_OFFSET+:`TR_IN_MVOVER_COMPLETION_READY_WIDTH] = 0; + repeat (4) tick(); + pins[`TR_IN_MVOVER_COMPLETION_READY_OFFSET+:`TR_IN_MVOVER_COMPLETION_READY_WIDTH] = 1; + tick(); + pins[`TR_IN_MEMORY_TRANSFERCOMPLETION_VALID_OFFSET+:`TR_IN_MEMORY_TRANSFERCOMPLETION_VALID_WIDTH]=0; + tick(); + endtask + task body(); + longint unsigned address, discard; + pins = '0; + repeat (3) tick(1); + tick(); + layout(0, 0, 1); + layout(1, 0, 1); + for (int c = 0; c < `TR_CLIENTS; c++) + for (int row = 0; row < `TR_ROWS; row++) begin + access (c, 0, 0, row, 1, + 128'h123456789abcdef0fedcba9876543210 ^ (128'(c * 37 + row) << 32)); + access (c, 0, 0, row, 0); + end + for (int bit_index = 0; bit_index < 16; bit_index++) begin + access (bit_index % `TR_CLIENTS, 0, 0, 0, 1, '1, 16'(1 << bit_index)); + access ((bit_index + 1) % `TR_CLIENTS, 0, 0, 0, 0); + end + access (0, 0, 0, 0, 1, 0, 0); + access (0, 0, 0, 0, 0); + reference.image.error[1] = 1; + access (1, 0, 0, 0, 1, '1); + reference.image.error[1] = 0; + access (1, 0, 0, 0, 0); + for (int thread = 0; thread < 2; thread++) begin + lease(thread, 0, 0, 0, address); + physical(address, 0); + physical(address + 16, 1, 128'hcafef00d0123456789abcdefdeadbeef); + physical(address + 16, 0); + reference.image.error[`TR_CLIENTS] = 1; + physical(address, 0); + reference.image.error[`TR_CLIENTS] = 0; + lease(thread, 0, 0, 1, discard); + end + // Four independently stalled clients retain their response ownership. + begin + bit waiting[`TR_CLIENTS], any_waiting, all_returned; + for (int c = 0; c < `TR_CLIENTS; c++) begin + pins[clients[c][14]+:`TR_IN_MEM_REQ_0_BANK_ID_WIDTH] = 0; + pins[clients[c][15]+:`TR_IN_MEM_REQ_0_GROUP_ID_WIDTH] = 0; + pins[clients[c][16]+:64] = owners[c/2]; + pins[clients[c][2]+:16] = c; + pins[clients[c][0]] = 1; + pins[clients[c][4]] = 0; + waiting[c] = 1; + end + do begin + tick(); + any_waiting = 0; + foreach (waiting[c]) begin + if (waiting[c] && observed[clients[c][1]]) begin + waiting[c] = 0; + pins[clients[c][0]] = 0; + end + any_waiting |= waiting[c]; + end + end while (any_waiting); + do begin + tick(); + all_returned = 1; + for (int c = 0; c < `TR_CLIENTS; c++) all_returned &= observed[clients[c][3]]; + end while (!all_returned); + repeat (5) tick(); + for (int c = 0; c < `TR_CLIENTS; c++) pins[clients[c][4]] = 1; + tick(); + end + layout(0, 0, 0); + layout(1, 0, 0); + layout(0, 0, 1); + layout(0, 1, 1); + access (0, 0, 0, 0, 1, 128'h11112222333344445555666677778888); + access (1, 1, 0, 0, 1, 128'h9999aaaabbbbccccddddeeeeffff0000); + layout(0, 0, 1, 1, 1); + access (0, 0, 0, 0, 0); + access (1, 0, 1, 0, 0); + mapping(0, 0, 6, 0, 1, 0); + for (int group_index = 0; group_index < 2; group_index++) + for (int row = 0; row < `TR_ROWS; row++) + access (group_index, 6, group_index, row, 1, 128'(row + 100 * group_index)); + lease(0, 0, 0, 0, address); + map_offer(0, 0, 6, 0, 0, 1); + repeat (8) tick(); + evidence::check(pins[mappings[0][0]], "Lease did not block retirement"); + lease(0, 0, 0, 1, discard); + do tick(); while (pins[mappings[0][0]]); + lease(0, 0, 0, 0, address); + physical(address, 0); + physical(address + 16, 0); + lease(0, 0, 0, 1, discard); + mapping(0, 0, 10, 6, 1, 0, 1); + mapping(0, 0, 10, 6, 0, 1); + access (0, 10, 0, 0, 1, '1); + access (0, 10, 0, 0, 0); + for (int i = 0; i < 16; i++) move(i); + // Request and response payloads cross the Thread-local endpoint unchanged. + for (int pattern = 0; pattern < 32; pattern++) begin + foreach (copies[f]) + for (int b = 0; b < copies[f][2]; b++) + pins[copies[f][0]+b] = ((b + pattern) % 3 == 0) ^ ((b + pattern) % 7 == 0); + tick(); + end + foreach (copies[f]) for (int b = 0; b < copies[f][2]; b++) pins[copies[f][0]+b] = 0; + + // Joint reset cancels held logical, physical, lease and move transactions. + pins[clients[0][14]+:`TR_IN_MEM_REQ_0_BANK_ID_WIDTH] = 10; + pins[clients[0][15]+:`TR_IN_MEM_REQ_0_GROUP_ID_WIDTH] = 0; + pins[clients[0][16]+:64] = owners[0]; + pins[clients[0][2]+:16] = 0; + pins[clients[0][0]] = 1; + pins[clients[0][4]] = 0; + do tick(); while (!observed[clients[0][1]]); + pins[clients[0][0]] = 0; + pins[`TR_IN_LEASEOWNERTHREADID_OFFSET+:64] = owners[0]; + pins[`TR_IN_LEASE_REQUEST_BITS_ENDPOINT_OFFSET+:`TR_IN_LEASE_REQUEST_BITS_ENDPOINT_WIDTH] = 0; + pins[`TR_IN_LEASE_REQUEST_BITS_VBANK_OFFSET+:`TR_IN_LEASE_REQUEST_BITS_VBANK_WIDTH] = 0; + pins[`TR_IN_LEASE_REQUEST_BITS_GROUP_OFFSET+:`TR_IN_LEASE_REQUEST_BITS_GROUP_WIDTH] = 0; + pins[`TR_IN_LEASE_REQUEST_BITS_RELEASE_OFFSET] = 0; + pins[`TR_IN_LEASE_REQUEST_VALID_OFFSET] = 1; + pins[`TR_IN_LEASE_RESPONSE_READY_OFFSET] = 0; + do tick(); while (!observed[`TR_OUT_LEASE_REQUEST_READY_OFFSET]); + pins[`TR_IN_LEASE_REQUEST_VALID_OFFSET] = 0; + do tick(); while (!observed[`TR_OUT_LEASE_RESPONSE_VALID_OFFSET]); + address = observed[`TR_OUT_LEASE_RESPONSE_BITS_OFFSET+:64]; + pins[`TR_IN_PHYSICAL_0_REQUEST_BITS_ADDRESS_OFFSET+:64] = address; + pins[`TR_IN_PHYSICAL_0_REQUEST_BITS_WRITE_OFFSET] = 0; + pins[`TR_IN_PHYSICAL_0_REQUEST_VALID_OFFSET] = 1; + pins[`TR_IN_PHYSICAL_0_RESPONSE_READY_OFFSET] = 0; + do tick(); while (!observed[`TR_OUT_PHYSICAL_0_REQUEST_READY_OFFSET]); + pins[`TR_IN_PHYSICAL_0_REQUEST_VALID_OFFSET] = 0; + pins[`TR_IN_MVOVER_COMMAND_VALID_OFFSET] = 1; + pins[`TR_IN_MEMORY_TRANSFERCOMMAND_READY_OFFSET] = 1; + tick(); + pins[`TR_IN_MVOVER_COMMAND_VALID_OFFSET] = 0; + pins[`TR_IN_MEMORY_TRANSFERCOMMAND_READY_OFFSET] = 0; + repeat (8) tick(); + pins = '0; + repeat (3) tick(1); + tick(); + layout(0, 0, 1); + access (0, 0, 0, 0, 0); + repeat (8) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/suite.toml new file mode 100644 index 000000000..0df07d94a --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/suite.toml @@ -0,0 +1,88 @@ +[model] +manifest="" +library="" + +[[tests]] +name="normal" +top="tbus_router_tb" +test="protocol_test" +filelist="filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["tbus_pkg::model::behavior","tbus_pkg::model::epochs"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" + +[[tests]] +name="physical" +top="tbus_router_tb" +test="invalid_test" +filelist="filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=physical"] +assertion="C2C physical access requires an allocated shared bank and aligned address" + +[[tests]] +name="lease" +top="tbus_router_tb" +test="invalid_test" +filelist="filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=lease"] +assertion="Shared lease requires one allocated owner/bank/group" + +[[tests]] +name="transfer" +top="tbus_router_tb" +test="invalid_test" +filelist="filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=transfer"] +assertion="Shared bank transfer requires a distinct allocated source" + +[[tests]] +name="alias" +top="tbus_router_tb" +test="invalid_test" +filelist="filelist.f" +timeout_ns=500000 +wall_timeout_s=120 +kind="assertion" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=[] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=["+CASE=alias"] +assertion="Renamed shared bank requires an allocated layout" diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tb.sv new file mode 100644 index 000000000..fe7c9d8ee --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tb.sv @@ -0,0 +1,15 @@ +module tbus_router_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + tbus_if bus (clock); + Router dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual tbus_if)::set(null, "uvm_test_top*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tests/invalid.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tests/invalid.svh new file mode 100644 index 000000000..b5fb026be --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tests/invalid.svh @@ -0,0 +1,63 @@ +class invalid_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(invalid_sequence) + logic [`TR_INPUT_BITS-1:0] pins; + function new(string name = "invalid"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + endtask + task body(); + string scenario; + if (!$value$plusargs("CASE=%s", scenario)) `uvm_fatal("CASE", "Invalid test requires a case") + pins = '0; + repeat (3) tick(1); + tick(); + if (scenario == "physical") begin + pins[`TR_IN_PHYSICAL_0_REQUEST_VALID_OFFSET] = 1; + pins[`TR_IN_PHYSICAL_0_REQUEST_BITS_ADDRESS_OFFSET+:64] = 1; + end + if (scenario == "lease") begin + pins[`TR_IN_LEASE_REQUEST_VALID_OFFSET] = 1; + pins[`TR_IN_LEASEOWNERTHREADID_OFFSET+:64] = 10; + end + if (scenario == "transfer") begin + pins[`TR_IN_CONFIG_VALID_OFFSET] = 1; + pins[`TR_IN_CONFIG_BITS_TRANSFER_OFFSET] = 1; + pins[`TR_IN_CONFIG_BITS_THREAD_ID_OFFSET+:64] = 10; + pins[`TR_IN_CONFIG_BITS_SOURCE_BANK_ID_OFFSET+:`TR_IN_CONFIG_BITS_SOURCE_BANK_ID_WIDTH] = 1; + end + if (scenario == "alias") begin + pins[`TR_IN_MAPPING_0_VALID_OFFSET] = 1; + pins[`TR_IN_MAPPING_0_BITS_ALLOCATE_OFFSET] = 1; + pins[`TR_IN_MAPPING_0_BITS_THREADID_OFFSET+:64] = 10; + pins[`TR_IN_MAPPING_0_BITS_VBANK_OFFSET+:`TR_IN_MAPPING_0_BITS_VBANK_WIDTH] = 6; + end + repeat (10) tick(); + `uvm_fatal("MISSING_ASSERTION", "Expected Router assertion did not fire") + endtask +endclass +class invalid_test extends base_test; + `uvm_component_utils(invalid_test) + driver source; + uvm_sequencer #(cycle_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + endfunction + task execute(); + invalid_sequence seq = invalid_sequence::type_id::create("invalid"); + seq.start(seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tests/test.svh new file mode 100644 index 000000000..b11c898c9 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/resources/tests/test.svh @@ -0,0 +1,16 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.reference = test_env.reference; + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/Emit.scala new file mode 100644 index 000000000..5f012fc26 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/Emit.scala @@ -0,0 +1,142 @@ +package framework.core.tbus.router + +import chisel3._ +import chisel3.reflect.DataMirror +import java.nio.file.{Files, Paths} +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.gpdomain.configs.GpDomainParam + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val frontend = upickle.default.read[FrontendParam](values("frontend")) + val memory = upickle.default.read[MemDomainParam](values("memory")) + val bus = upickle.default.read[framework.core.tbus.configs.Params](values("tbus")) + val endpointIds = values("endpointIds").arr.toSeq.map(_.num.toInt) + val traceIds = values("traceIds").arr.toSeq.map(_.num.toInt) + + val defs = scala.collection.mutable.ArrayBuffer[String]( + s"`define TR_BANKS ${memory.sharedBankNum}", + s"`define TR_ROWS ${memory.sharedBankEntries}", + s"`define TR_CLIENTS ${memory.sharedInputChannels}", + s"`define TR_THREADS ${bus.slots}", + s"`define TR_EXTERNAL ${bus.router.physicalPorts}", + s"`define TR_DATA_BITS ${memory.bankWidth}", + s"`define TR_MASK_BITS ${memory.bankMaskLen}", + s"`define TR_TAG_BITS ${math.max(1, chisel3.util.log2Ceil(frontend.rob_entries))}", + s"`define TR_ALIASES ${2 * memory.virtualBankCount + frontend.rob_entries}", + s"`define TR_OWNERS ${endpointIds.mkString("'{", ",", "}")}" + ) + + val ports = scala.collection.mutable.ArrayBuffer[String]() + val leavesWidth = scala.collection.mutable.Map[String, Int]() + val offsets = scala.collection.mutable.Map[String, Int]() + var inOffset = 0; var outOffset = 0 + + def leaves(data: Data, path: String): Seq[(String, Data)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + leaves(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => leaves(field, s"${path}_$index") } + case field => if (field.getWidth > 0) Seq(path -> field) else Seq.empty + } + + Files.createDirectories(Paths.get(directory)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new Router(frontend, memory, bus, endpointIds, traceIds, values("diffTest").bool) + for ((name, field) <- leaves(dut.io, "")) { + val input = DataMirror.directionOf(field) == ActualDirection.Input + val direction = if (input) "IN" else "OUT"; val signal = if (input) "inputs" else "outputs"; + val offset = if (input) inOffset else outOffset; val width = field.getWidth + defs += s"`define TR_${direction}_${name.toUpperCase}_OFFSET $offset"; + defs += s"`define TR_${direction}_${name.toUpperCase}_WIDTH $width"; offsets(name) = offset; + leavesWidth(name) = width + ports += s".io_$name(bus.$signal[$offset +: $width])" + if (input) inOffset += width else outOffset += width + }; dut + }, + args = Array("--target-dir", s"$directory/Router", "--split-verilog") + ) + defs += s"`define TR_INPUT_BITS $inOffset"; defs += s"`define TR_OUTPUT_BITS $outOffset" + val clientNames = (0 until memory.sharedInputChannels).map(i => s"mem_req_$i") + + val clientFields = Seq( + "read_req_valid", + "read_req_ready", + "read_req_bits_addr", + "read_resp_valid", + "read_resp_ready", + "read_resp_bits_data", + "write_req_valid", + "write_req_ready", + "write_req_bits_addr", + "write_req_bits_data", + "write_req_bits_mask_0", + "write_resp_valid", + "write_resp_ready", + "write_resp_bits_ok", + "bank_id", + "group_id", + "thread_id" + ) + + val meshNames = (0 until memory.sharedInputChannels).map(i => + s"memory_channels_$i" + ) ++ (0 until bus.router.physicalPorts).map(i => s"memory_external_$i") + + val meshFields = Seq( + "request_valid", + "request_ready", + "request_bits_tdest", + "request_bits_tuser", + "request_bits_tdata", + "request_bits_tkeep", + "request_bits_tid", + "request_bits_tlast", + "response_valid", + "response_ready", + "response_bits_tdata", + "response_bits_tkeep", + "response_bits_tid", + "response_bits_tuser", + "response_bits_tlast" + ) + + val mappingNames = (0 until bus.slots).map(i => s"mapping_$i") + + val mappingFields = Seq( + "valid", + "ready", + "bits_bank", + "bits_vbank", + "bits_previous", + "bits_threadId", + "bits_allocate", + "bits_reuse", + "bits_retire", + "bits_write" + ) + + def table(names: Seq[String], fields: Seq[String]) = + names.map(n => fields.map(f => offsets(s"${n}_$f")).mkString("'{", ",", "}")).mkString("'{", ",", "}") + defs += s"`define TR_CLIENT_OFFSETS ${table(clientNames, clientFields)}" + defs += s"`define TR_MESH_OFFSETS ${table(meshNames, meshFields)}" + defs += s"`define TR_MAPPING_OFFSETS ${table(mappingNames, mappingFields)}" + val copies = scala.collection.mutable.ArrayBuffer[String]() + for ((name, offset) <- offsets.toSeq.sortBy(_._1) if name.startsWith("memory_localBanks_")) { + val other = "localBanks_" + name.stripPrefix("memory_localBanks_") + val memoryInput = name.contains("_request_") && !name.endsWith("_ready") || name.contains("_response_ready") + val from = if (memoryInput) name else other; val to = if (memoryInput) other else name + val gate = + if (name.endsWith("_ready") || name.endsWith("_valid")) -1 + else offsets(from.take(from.indexOf("_bits")) + "_valid") + val width = leavesWidth(name) + copies += s"'{${offsets(from)},${offsets(to)},$width,$gate}" + } + defs += s"`define TR_LOCAL_COPIES ${copies.mkString("'{", ",", "}")}" + defs += s"`define TR_LOCAL_FIELDS ${copies.size}" + Files.writeString(Paths.get(directory, "parameters.svh"), defs.mkString("\n") + "\n") + Files.writeString(Paths.get(directory, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/Router.scala b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/Router.scala new file mode 100644 index 000000000..de26dc0c3 --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/Router.scala @@ -0,0 +1,645 @@ +package framework.core.tbus.router + +import framework.lib.arbiter.{Arbiter} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.tbus.configs.{Params => TBusParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.memdomain.BankPort +import framework.acc.debug.trace.MTraceDPI +import framework.lib.hash.BankHashMonitor +import framework.acc.memdomain.bank_router.blt.AccPipe +import framework.acc.memdomain.PhysicalBankHash +import framework.acc.memdomain.LayoutConfig +import framework.core.tbus.configs.Layout +import framework.acc.memdomain.bank_router.BankMapping +import framework.ip.memory.mesh_shm.{MeshLocalBankPort, MeshSharedMemIO} +import framework.acc.memdomain.{MvoverISA, MvoverPort} + +@instantiable +class Router( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val tbusParam: TBusParam, + val endpointIds: Seq[Int], + val traceIds: Seq[Int], + val diffTest: Boolean) + extends Module { + val externalPhysicalPorts = tbusParam.router.physicalPorts + require(externalPhysicalPorts >= 0) + val nThreads = tbusParam.slots + val totalBanks = Layout.totalBank(memParam) + val totalChannel = Layout.totalChannel(memParam) + + val sharedMemory = memParam.copy( + bankEntries = memParam.sharedBankEntries + ) + + val tagBits = math.max(1, log2Ceil(frontendParam.rob_entries)) + + val threadIds = VecInit(endpointIds.map(_.U(64.W))) + require(traceIds.size == endpointIds.size) + def traceId(owner: UInt): UInt = Mux1H(endpointIds.map(id => owner === id.U), traceIds.map(_.U(64.W))) + + val memoryParams = tbusParam.router.mesh(frontendParam, memParam, tbusParam) + + @public + val io = IO(new Bundle { + + val memory = Flipped(new MeshSharedMemIO(memoryParams)) + + val mem_req = Vec(totalChannel, Flipped(new BankPort(frontendParam, memParam))) + val lease = Option.when(externalPhysicalPorts > 0)(Flipped(new LeasePort)) + val leaseOwnerThreadId = Option.when(externalPhysicalPorts > 0)(Input(UInt(64.W))) + + val physical = Vec( + externalPhysicalPorts, + Flipped(new PhysicalPort(memParam.bankWidth)) + ) + + val mvover = Flipped(new MvoverPort) + + val localBanks = + Vec( + nThreads, + new MeshLocalBankPort( + MvoverISA.AddressBits, + MvoverISA.BankBits, + memParam.bankWidth, + tagBits + ) + ) + + val config = Flipped(Decoupled(new LayoutConfig(frontendParam, memParam))) + val mapping = Vec(nThreads, Flipped(Decoupled(new BankMapping(frontendParam)))) + + val bank_hashes = if (diffTest) Some(Output(Vec(totalBanks, new PhysicalBankHash(frontendParam)))) else None + }) + + require(io.memory.transferCommand.bits.sourceAddr.getWidth <= 16) + require(io.memory.transferCommand.bits.sourceThread.getWidth <= 8) + io.memory.transferCommand.valid := io.mvover.command.valid + io.memory.transferCommand.bits.sourceThread := io.mvover.command.bits.sourceThread + io.memory.transferCommand.bits.targetThread := io.mvover.command.bits.targetThread + io.memory.transferCommand.bits.sourceBank := io.mvover.command.bits.sourceBank + io.memory.transferCommand.bits.targetBank := io.mvover.command.bits.targetBank + io.memory.transferCommand.bits.sourceAddr := io.mvover.command.bits.sourceAddr + io.memory.transferCommand.bits.targetAddr := io.mvover.command.bits.targetAddr + io.memory.transferCommand.bits.rows := io.mvover.command.bits.rows + io.memory.transferCommand.bits.tag := 0.U + io.mvover.command.ready := io.memory.transferCommand.ready + io.mvover.completion.valid := io.memory.transferCompletion.valid + io.mvover.completion.bits := io.memory.transferCompletion.bits.error + io.memory.transferCompletion.ready := io.mvover.completion.ready + for ((local, index) <- io.memory.localBanks.zipWithIndex) { + io.localBanks(index).request.valid := local.request.valid + io.localBanks(index).request.bits := local.request.bits + local.request.ready := io.localBanks(index).request.ready + local.response.valid := io.localBanks(index).response.valid + local.response.bits := io.localBanks(index).response.bits + io.localBanks(index).response.ready := local.response.ready + } + + val accPipes: Seq[Instance[AccPipe]] = Seq.fill(totalChannel)(Instantiate(new AccPipe(frontendParam, memParam))) + + val hashMonitors: Option[Seq[Instance[BankHashMonitor]]] = + if (diffTest) { + Some(Seq.fill(totalBanks)(Instantiate(new BankHashMonitor(sharedMemory)))) + } else { + None + } + + // Per-channel memory trace DPI-C modules to avoid losing simultaneous events + val mtraces: Seq[Instance[MTraceDPI]] = Seq.fill(totalChannel + externalPhysicalPorts)(Instantiate(new MTraceDPI)) + for (mt <- mtraces) { + mt.io.clock := clock + mt.io.reset := reset.asBool + mt.io.is_write := 0.U + mt.io.is_shared := 0.U + mt.io.channel := 0.U + mt.io.thread_id := 0.U + mt.io.rob_id := 0.U + mt.io.inst_id := 0.U + mt.io.vbank_id := 0.U + mt.io.pbank_id := 0.U + mt.io.group_id := 0.U + mt.io.addr := 0.U + mt.io.write_mask := 0.U + mt.io.data_lo := 0.U + mt.io.data_hi := 0.U + mt.io.enable := false.B + } + + // ----------------------------------------------------------------------------- + // Mapping table + // ----------------------------------------------------------------------------- + class MappingTableEntry extends Bundle { + val valid = Bool() + val thread_id = UInt(64.W) + val vbank_id = UInt(frontendParam.bank_id_len.W) + val architectural_id = UInt(frontendParam.bank_id_len.W) + val is_multi = Bool() + val group_id = UInt(memParam.groupIdWidth.W) + } + + val mappingTable = RegInit(VecInit(Seq.fill(totalBanks)(0.U.asTypeOf(new MappingTableEntry)))) + + val architecturalColumns = RegInit(VecInit(Seq.fill(nThreads)(VecInit( + Seq.fill(memParam.virtualBankCount)(0.U( + memParam.groupCountWidth.W + )) + )))) + + val committed = RegInit(VecInit(Seq.fill(nThreads)(VecInit( + (0 until memParam.virtualBankCount).map(_.U( + frontendParam.bank_id_len.W + )) + )))) + + val leases = Option.when(externalPhysicalPorts > 0)(RegInit(VecInit(Seq.fill(totalBanks)(0.U(32.W))))) + + // Record binding and lifetime changes, including aliases with no active accesses. + for ((entry, physical) <- mappingTable.zipWithIndex) { + val ownerThread = PriorityEncoder(VecInit(threadIds.map(_ === entry.thread_id)).asUInt) + val live = entry.valid + val current = entry.valid && committed(ownerThread)(entry.architectural_id) === entry.vbank_id + val previous = RegNext(entry, 0.U.asTypeOf(entry)) + val previousLive = RegNext(live, false.B) + val previousCurrent = RegNext(current, false.B) + val trace = Instantiate(new MTraceDPI) + trace.io.clock := clock + trace.io.reset := reset.asBool + trace.io.is_write := Mux(entry.valid, 2.U, 3.U) + trace.io.is_shared := 1.U + trace.io.channel := 0.U + trace.io.thread_id := traceId(entry.thread_id) + trace.io.rob_id := 0.U + trace.io.inst_id := 0.U + trace.io.vbank_id := entry.vbank_id + trace.io.pbank_id := physical.U + trace.io.group_id := entry.group_id + trace.io.addr := entry.architectural_id + trace.io.write_mask := 0.U + trace.io.data_lo := live.asUInt + trace.io.data_hi := current.asUInt + trace.io.enable := (entry.valid || previous.valid) && + (entry.asUInt =/= previous.asUInt || live =/= previousLive || current =/= previousCurrent) + } + + io.lease.foreach { port => + val pending = RegInit(false.B) + val result = Reg(UInt(64.W)) + val ownerThread = PriorityEncoder(VecInit(threadIds.map(_ === io.leaseOwnerThreadId.get)).asUInt) + val currentAlias = committed(ownerThread)(port.request.bits.vbank) + val matches = VecInit(mappingTable.zipWithIndex.map { case (entry, physical) => + entry.valid && + entry.thread_id === io.leaseOwnerThreadId.get && entry.architectural_id === port.request.bits.vbank && + entry.group_id === port.request.bits.group && + Mux(port.request.bits.release, leases.get(physical) =/= 0.U, entry.vbank_id === currentAlias) + }) + val index = PriorityEncoder(matches) + port.request.ready := !pending && !io.config.valid && !reset.asBool + port.response.valid := pending && !reset.asBool + port.response.bits := result + when(port.request.fire) { + assert(PopCount(matches) === 1.U, "Shared lease requires one allocated owner/bank/group") + val count = leases.get(index) + when(port.request.bits.release) { + assert(count =/= 0.U, "Shared lease release has no export") + count := count - 1.U + result := 0.U + }.otherwise { + assert(count =/= "hffffffff".U, "Shared lease count overflow") + count := count + 1.U + result := index * (memParam.sharedBankEntries * (memParam.bankWidth / 8)).U( + 64.W + ) + } + pending := true.B + } + when(port.response.fire)(pending := false.B) + } + + val rowBits = log2Ceil(memParam.sharedBankEntries) + val bytesPerRow = memParam.bankWidth / 8 + val byteBits = log2Ceil(bytesPerRow) + for (i <- 0 until externalPhysicalPorts) { + val physical = io.physical(i) + val channel = io.memory.external(i) + val bank = physical.request.bits.address >> (rowBits + byteBits) + val legal = bank < totalBanks.U && physical.request.bits.address(byteBits - 1, 0) === 0.U + val bankIndex = bank(math.max(1, log2Ceil(totalBanks)) - 1, 0) + val allocated = VecInit(mappingTable.map(_.valid))(bankIndex) + when(physical.request.valid && !reset.asBool) { + assert(legal && allocated, "C2C physical access requires an allocated shared bank and aligned address") + } + channel.request.valid := physical.request.valid && legal && allocated + channel.request.bits := 0.U.asTypeOf(channel.request.bits) + channel.request.bits.bank := bankIndex + channel.request.bits.tuser := Cat( + physical.request.bits.address(rowBits + byteBits - 1, byteBits).pad(16), + physical.request.bits.write + ) + channel.request.bits.data := physical.request.bits.data + channel.request.bits.mask := physical.request.bits.mask + channel.request.bits.tag := 0.U + channel.request.bits.tlast := true.B + physical.request.ready := channel.request.ready && legal && allocated + physical.response.valid := channel.response.valid + physical.response.bits.data := channel.response.bits.data + physical.response.bits.error := channel.response.bits.error + channel.response.ready := physical.response.ready + } + + for (i <- 0 until externalPhysicalPorts) { + val physical = io.physical(i) + val held = Reg(new PhysicalRequest(memParam.bankWidth)) + val entry = Reg(new MappingTableEntry) + val pending = RegInit(false.B) + val bankBits = log2Ceil( + memParam.sharedBankEntries * (memParam.bankWidth / 8) + ) + when(physical.request.fire) { + assert(!pending, "Physical shared port accepts one outstanding request") + held := physical.request.bits + entry := mappingTable((physical.request.bits.address >> bankBits)(math.max(1, log2Ceil(totalBanks)) - 1, 0)) + pending := true.B + } + when(physical.response.valid && !reset.asBool)(assert(pending)) + when(physical.response.fire)(pending := false.B) + val trace = mtraces(totalChannel + i) + val value = Mux(held.write, held.data, physical.response.bits.data) + trace.io.is_write := held.write.asUInt + trace.io.is_shared := 1.U + trace.io.channel := (totalChannel + i).U + trace.io.thread_id := traceId(entry.thread_id) + trace.io.vbank_id := entry.vbank_id + trace.io.pbank_id := held.address >> bankBits + trace.io.group_id := entry.group_id + trace.io.addr := (held.address >> 4) & (memParam.sharedBankEntries - 1).U + trace.io.write_mask := Mux(held.write, held.mask, 0.U) + trace.io.data_lo := value(63, 0) + trace.io.data_hi := value(127, 64) + trace.io.enable := physical.response.fire && !physical.response.bits.error + } + + def addEntry( + thread_id: UInt, + vbank_id: UInt, + pbank_id: UInt, + is_multi: Bool, + group_id: UInt + ): Unit = { + val duplicate = mappingTable.map(entry => + entry.valid && + !(group_id === 0.U && entry.thread_id === thread_id && entry.vbank_id === vbank_id) && + (entry.thread_id === thread_id) && + (entry.vbank_id === vbank_id) && + (entry.group_id === group_id) + ).reduce(_ || _) + when(duplicate) { + assert(false.B, "Router duplicate allocation: thread=%d vbank=%d group=%d\n", thread_id, vbank_id, group_id) + } + + val entry = mappingTable(pbank_id) + leases.foreach(counts => assert(counts(pbank_id) === 0.U, "Shared bank allocation replaces live exports")) + entry.valid := true.B + entry.thread_id := thread_id + entry.vbank_id := vbank_id + entry.architectural_id := vbank_id + entry.is_multi := is_multi + entry.group_id := group_id + } + + def deleteEntry(thread_id: UInt, vbank_id: UInt): Unit = { + val thread = PriorityEncoder(VecInit(threadIds.map(_ === thread_id)).asUInt) + val found = architecturalColumns(thread)(vbank_id) =/= 0.U + when(!found) { + assert(false.B, "Router release missing allocation: thread=%d vbank=%d\n", thread_id, vbank_id) + } + clearVbank(thread_id, vbank_id) + } + + def clearVbank(thread_id: UInt, vbank_id: UInt): Unit = { + for (i <- 0 until totalBanks) { + when(mappingTable(i).valid && mappingTable(i).architectural_id === vbank_id && mappingTable( + i + ).thread_id === thread_id) { + leases.foreach(counts => assert(counts(i) === 0.U, "Shared bank mapping changed with live exports")) + mappingTable(i).valid := false.B + } + } + } + + // ----------------------------------------------------------------------------- + // Default Value + // ----------------------------------------------------------------------------- + + for (i <- 0 until totalChannel) { + accPipes(i).io.mem_req.write <> io.mem_req(i).write + accPipes(i).io.mem_req.read <> io.mem_req(i).read + accPipes(i).io.mem_req.bank_id := io.mem_req(i).bank_id + accPipes(i).io.mem_req.group_id := io.mem_req(i).group_id + accPipes(i).io.mem_req.thread_id := io.mem_req(i).thread_id + + // Bank-side defaults (only driven when a bank is actually connected) + accPipes(i).io.sramRead.req.ready := false.B + accPipes(i).io.sramRead.resp.valid := false.B + accPipes(i).io.sramRead.resp.bits := DontCare + + accPipes(i).io.sramWrite.req.ready := false.B + accPipes(i).io.sramWrite.resp.valid := false.B + accPipes(i).io.sramWrite.resp.bits := DontCare + } + + val realloc = io.config.bits.group_id === 0.U + + val freeBanks = VecInit(mappingTable.map(entry => !entry.valid)).asUInt + + val freePbankMask = VecInit(mappingTable.zipWithIndex.map { case (entry, physical) => + freeBanks(physical) || + (realloc && entry.thread_id === io.config.bits.thread_id && entry.architectural_id === io.config.bits.vbank_id) + }) + + val hasFreePbank = freePbankMask.asUInt.orR + io.config.ready := !io.config.bits.alloc || hasFreePbank + when(io.config.valid && io.config.bits.alloc && !hasFreePbank) { + assert(false.B, "Router allocation failed: no free physical shared bank\n") + } + + // ----------------------------------------------------------------------------- + // Bank Alloc/Release + // ----------------------------------------------------------------------------- + + hashMonitors.foreach(_.foreach(_.io.bind := false.B)) + + val mappings = Instantiate(new Arbiter(new BankMapping(frontendParam), nThreads)) + for (thread <- 0 until nThreads) { mappings.io.in(thread) <> io.mapping(thread) } + val change = mappings.io.out.bits + val owner = mappings.io.chosen + val columns = architecturalColumns(owner)(change.bank) + + val leaseBusy = VecInit(mappingTable.zipWithIndex.map { case (entry, physical) => + val held = if (externalPhysicalPorts > 0) leases.get(physical) =/= 0.U else false.B + entry.valid && entry.thread_id === change.threadId && entry.vbank_id === change.previous && held + }).asUInt.orR + + val reusableBanks = VecInit(mappingTable.map(entry => + !entry.valid || (change.reuse && entry.thread_id === change.threadId && entry.vbank_id === change.previous) + )).asUInt + + mappings.io.out.ready := !io.config.valid && (!change.allocate || PopCount(reusableBanks) >= columns) && + (!(change.retire || change.reuse) || !leaseBusy) + when(mappings.io.out.valid && change.allocate) { + assert(columns =/= 0.U && columns <= totalBanks.U, "Renamed shared bank requires an allocated layout") + } + when(mappings.io.out.fire && change.retire) { + committed(owner)(change.bank) := change.vbank + for (entry <- mappingTable) { + when(entry.valid && entry.thread_id === change.threadId && entry.vbank_id === change.previous) { + entry.valid := false.B + } + } + } + val remaining = Wire(Vec(totalBanks + 1, UInt(totalBanks.W))) + remaining(0) := reusableBanks + for (group <- 0 until totalBanks) { + val physical = PriorityEncoder(remaining(group)) + remaining(group + 1) := remaining(group) & ~UIntToOH(physical, totalBanks) + when(mappings.io.out.fire && change.allocate && group.U < columns) { + assert(change.allocate && columns =/= 0.U, "Shared alias requires an allocated layout") + addEntry(change.threadId, change.vbank, physical, columns > 1.U, group.U) + mappingTable(physical).architectural_id := change.bank + hashMonitors.foreach { hashes => + for (bank <- 0 until totalBanks) { when(physical === bank.U)(hashes(bank).io.bind := true.B) } + } + } + } + + when(io.config.fire) { + val thread = PriorityEncoder(VecInit(threadIds.map(_ === io.config.bits.thread_id)).asUInt) + when(io.config.bits.transfer) { + val source = io.config.bits.source_bank_id + val target = io.config.bits.vbank_id + val owner = io.config.bits.thread_id + val sourceCount = architecturalColumns(thread)(source) + val targetCount = architecturalColumns(thread)(target) + val totalCount = sourceCount +& targetCount + architecturalColumns(thread)(source) := 0.U + architecturalColumns(thread)(target) := totalCount + assert(source =/= target && sourceCount =/= 0.U, "Shared bank transfer requires a distinct allocated source") + assert(totalCount <= totalBanks.U, "Shared bank transfer exceeds physical bank count") + for ((entry, physical) <- mappingTable.zipWithIndex) { + when(entry.valid && entry.thread_id === owner && entry.architectural_id === source) { + leases.foreach(counts => assert(counts(physical) === 0.U, "Shared bank transfer source has live exports")) + entry.vbank_id := target + entry.architectural_id := target + entry.group_id := targetCount + entry.group_id + entry.is_multi := totalCount > 1.U + } + when(entry.valid && entry.thread_id === owner && entry.architectural_id === target) { + entry.vbank_id := target + entry.is_multi := totalCount > 1.U + } + } + }.elsewhen(io.config.bits.alloc) { + architecturalColumns(thread)(io.config.bits.vbank_id) := io.config.bits.group_id +& 1.U + when(io.config.bits.group_id === 0.U) { + clearVbank(io.config.bits.thread_id, io.config.bits.vbank_id) + } + val pbankId = PriorityEncoder(freePbankMask) + hashMonitors.foreach { hashes => + for (i <- 0 until totalBanks) { + hashes(i).io.bind := pbankId === i.U + } + } + printf( + p"[Router][ALLOC] thread=${io.config.bits.thread_id} vbank=0x${Hexadecimal(io.config.bits.vbank_id)} " + + p"group=${io.config.bits.group_id} pbank=$pbankId is_multi=${io.config.bits.is_multi}\n" + ) + addEntry( + io.config.bits.thread_id, + io.config.bits.vbank_id, + pbankId, + io.config.bits.is_multi, + io.config.bits.group_id + ) + }.otherwise { + architecturalColumns(thread)(io.config.bits.vbank_id) := 0.U + printf( + p"[Router][RELEASE] thread=${io.config.bits.thread_id} vbank=0x${Hexadecimal(io.config.bits.vbank_id)}\n" + ) + deleteEntry(io.config.bits.thread_id, io.config.bits.vbank_id) + } + } + + // ----------------------------------------------------------------------------- + // Connect AccPipe and Banks + // ----------------------------------------------------------------------------- + def emitTrace( + ch: Int, + isWrite: UInt, + pbankId: UInt, + addr: UInt, + writeMask: UInt, + dataLo: UInt, + dataHi: UInt, + en: Bool + ): Unit = { + mtraces(ch).io.is_write := isWrite + mtraces(ch).io.is_shared := io.mem_req(ch).is_shared.asUInt + mtraces(ch).io.channel := ch.U + mtraces(ch).io.thread_id := traceId(io.mem_req(ch).thread_id) + mtraces(ch).io.rob_id := io.mem_req(ch).rob_id + mtraces(ch).io.inst_id := io.mem_req(ch).inst_id + mtraces(ch).io.vbank_id := io.mem_req(ch).bank_id + mtraces(ch).io.pbank_id := pbankId + mtraces(ch).io.group_id := io.mem_req(ch).group_id + mtraces(ch).io.addr := addr + mtraces(ch).io.write_mask := writeMask + mtraces(ch).io.data_lo := dataLo + mtraces(ch).io.data_hi := dataHi + mtraces(ch).io.enable := en + } + + for (i <- 0 until totalChannel) { + val activeHart = Mux(accPipes(i).io.busy, accPipes(i).io.thread_id, io.mem_req(i).thread_id) + val activeBank = Mux(accPipes(i).io.busy, accPipes(i).io.bank_id, io.mem_req(i).bank_id) + val activeGroup = Mux(accPipes(i).io.busy, accPipes(i).io.group_id, io.mem_req(i).group_id) + val req_valid = io.mem_req(i).read.req.valid || io.mem_req(i).write.req.valid || accPipes(i).io.busy + + val tracePbankId = Wire(UInt(32.W)) + tracePbankId := 0.U + for (j <- 0 until totalBanks) { + val trace_hit_bank = mappingTable(j).valid && + (mappingTable(j).thread_id === activeHart) && + (mappingTable(j).vbank_id === activeBank) && + (!mappingTable(j).is_multi || + (mappingTable(j).is_multi && (mappingTable(j).group_id === activeGroup))) + when(trace_hit_bank) { + tracePbankId := j.U + } + } + + // Memory trace: read request + when(io.mem_req(i).read.req.fire) { + emitTrace(i, 0.U, tracePbankId, io.mem_req(i).read.req.bits.addr, 0.U, 0.U, 0.U, true.B) + } + + // Arrival trace: observe acceptance at the selected shared SPM SRAM port. + when(accPipes(i).io.sramWrite.req.fire) { + emitTrace( + i, + 1.U, + tracePbankId, + accPipes(i).io.sramWrite.req.bits.addr, + accPipes(i).io.sramWrite.req.bits.mask.asUInt, + accPipes(i).io.sramWrite.req.bits.data(63, 0), + accPipes(i).io.sramWrite.req.bits.data(127, 64), + true.B + ) + } + + val channel = io.memory.channels(i) + val mapped = mappingTable.map(entry => + entry.valid && + entry.thread_id === activeHart && + entry.vbank_id === activeBank && + (!entry.is_multi || entry.group_id === activeGroup) + ).reduce(_ || _) + val writeReq = accPipes(i).io.sramWrite.req.valid + val nextTag = RegInit(0.U(tagBits.W)) + + channel.request.valid := mapped && (writeReq || accPipes(i).io.sramRead.req.valid) + channel.request.bits.bank := tracePbankId + channel.request.bits.tuser := Cat( + Mux(writeReq, accPipes(i).io.sramWrite.req.bits.addr, accPipes(i).io.sramRead.req.bits.addr), + writeReq + ) + channel.request.bits.tlast := true.B + channel.request.bits.data := accPipes(i).io.sramWrite.req.bits.data + channel.request.bits.mask := accPipes(i).io.sramWrite.req.bits.mask.asUInt + channel.request.bits.tag := nextTag + accPipes(i).io.sramRead.req.ready := mapped && !writeReq && channel.request.ready + accPipes(i).io.sramWrite.req.ready := mapped && channel.request.ready + accPipes(i).io.sramRead.resp.valid := channel.response.valid && !channel.response.bits.write + accPipes(i).io.sramRead.resp.bits.data := channel.response.bits.data + accPipes(i).io.sramWrite.resp.valid := channel.response.valid && channel.response.bits.write + accPipes(i).io.sramWrite.resp.bits.ok := !channel.response.bits.error + channel.response.ready := Mux( + channel.response.bits.write, + accPipes(i).io.sramWrite.resp.ready, + accPipes(i).io.sramRead.resp.ready + ) + when(channel.request.fire) { + nextTag := nextTag + 1.U + } + + } + + io.memory.bankWrites.foreach(_.ready := true.B) + + hashMonitors.foreach { hashes => + for (j <- 0 until totalBanks) { + val monitor = hashes(j) + + val write = io.memory.bankWrites(j) + write.ready := monitor.io.write.ready + monitor.io.write.valid := write.valid + monitor.io.write.bits.addr := write.bits.addr + monitor.io.write.bits.mask := write.bits.mask.asBools + monitor.io.write.bits.data := write.bits.data + + } + } + + io.bank_hashes.foreach { states => + for (i <- 0 until totalBanks) { + states(i).valid := mappingTable(i).valid + states(i).threadId := mappingTable(i).thread_id + states(i).vbankId := mappingTable(i).vbank_id + states(i).pbankId := i.U + states(i).groupId := mappingTable(i).group_id + states(i).statusHash := hashMonitors.get(i).io.statusHash + } + } +} + +/** Core shared SRAM byte space; not a Thread virtual-bank or private-bank address. */ +class PhysicalRequest(dataBits: Int) extends Bundle { + + val address = UInt(64.W) + val write = Bool() + val data = UInt(dataBits.W) + val mask = UInt((dataBits / 8).W) +} + +class PhysicalResponse(dataBits: Int) extends Bundle { + + val data = UInt(dataBits.W) + val error = Bool() +} + +/** One outstanding transaction per port; a write response acknowledges actual SRAM completion. */ +class PhysicalPort(dataBits: Int) extends Bundle { + + val request = Decoupled(new PhysicalRequest(dataBits)) + val response = Flipped(Decoupled(new PhysicalResponse(dataBits))) +} + +class LeaseRequest extends Bundle { + val endpoint = UInt(32.W) + val vbank = UInt(16.W) + val group = UInt(16.W) + val release = Bool() +} + +/** One outstanding mapping lease operation; export returns its physical byte base. */ +class LeasePort extends Bundle { + val request = Decoupled(new LeaseRequest) + val response = Flipped(Decoupled(UInt(64.W))) +} diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..897b127dd --- /dev/null +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/src/main/scala/configs/Params.scala @@ -0,0 +1,45 @@ +package framework.core.tbus.router.configs + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.tbus.configs.{Params => TBusParam} + +import chisel3.util.log2Ceil +import framework.core.tbus.configs.Layout +import framework.acc.memdomain.MvoverISA +import framework.ip.memory.mesh_shm.{MeshSharedMemParams, MeshThreadAttachment} +import upickle.default.{macroRW, ReadWriter} + +case class Params(physicalPorts: Int) { + + def mesh(frontendParam: FrontendParam, memParam: MemDomainParam, tbusParam: TBusParam): MeshSharedMemParams = { + require( + memParam.bankWidth == 128 && memParam.bankMaskLen == 16 + ) + val totalBanks = Layout.totalBank(memParam) + val nThreads = tbusParam.slots + val tagBits = math.max(1, log2Ceil(frontendParam.rob_entries)) + val externalPhysicalPorts = physicalPorts + val physicalBanks = totalBanks + val rows = math.ceil(math.sqrt(physicalBanks.toDouble)).toInt + val columns = (physicalBanks + rows - 1) / rows + val perThread = Layout.portsPerThread(memParam, tbusParam) + MeshSharedMemParams( + rows = rows, + cols = columns, + entriesPerBank = memParam.sharedBankEntries, + dataBits = memParam.bankWidth, + tagBits = tagBits, + localBankBits = MvoverISA.BankBits, + threads = (0 until nThreads).map { thread => + val ports = if (tbusParam.participants.contains(thread)) perThread else 0 + MeshThreadAttachment(tbusParam.bankNodes(thread), ports) + }, + externalChannels = externalPhysicalPorts, + visibleBanks = totalBanks + ) + } + +} + +object Params { implicit val rw: ReadWriter[Params] = macroRW } diff --git a/arch/src/main/scala/framework/core/tss/build.sc b/arch/src/main/scala/framework/core/tss/build.sc new file mode 100644 index 000000000..4a781e221 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.core.tss.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Store.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/core/tss/configs/uvm.toml b/arch/src/main/scala/framework/core/tss/configs/uvm.toml new file mode 100644 index 000000000..38cb7675d --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/configs/uvm.toml @@ -0,0 +1,9 @@ +[parameters] +ports = 2 + +[parameters.memory] +base = "4096" +bytes = 256 +dataBits = 128 + +[defines] diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/core/tss/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..be52642be --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/agent/driver.svh @@ -0,0 +1,25 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual tss_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tss_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.inputs = req.inputs; + @(posedge vif.clock); + req.outputs = vif.outputs; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/core/tss/src/main/resources/agent/item.svh new file mode 100644 index 000000000..78ff84d64 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/agent/item.svh @@ -0,0 +1,23 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + logic [`TSS_INPUT_BITS-1:0] inputs; + logic [`TSS_OUTPUT_BITS-1:0] outputs; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_field_int(error, UVM_ALL_ON) + `uvm_field_int(port, UVM_ALL_ON) + `uvm_object_utils_end + logic [`TSS_DATA_BITS-1:0] value; + bit error; + int port; + int message_kind; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/core/tss/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..57bfe2544 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual tss_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual tss_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "TSS monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.inputs = vif.inputs; + t.outputs = vif.outputs; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/env/env.svh b/arch/src/main/scala/framework/core/tss/src/main/resources/env/env.svh new file mode 100644 index 000000000..26a84020c --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/env/env.svh @@ -0,0 +1,34 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) boards[`TSS_PORTS]; + uvm_analysis_imp #(result_item, env) results; + function new(string name, uvm_component parent); + super.new(name, parent); + results = new("results", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + foreach (boards[p]) + boards[p] = result_board#(result_item, 0)::type_id::create($sformatf("port%0d", p), this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.results.connect(results); + endfunction + function void write(result_item t); + case (t.message_kind) + 0: boards[t.port].write_expected(t); + 1: boards[t.port].write_actual(t); + 2: boards[t.port].cancel_pending(); + endcase + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/tss/src/main/resources/env/model.svh new file mode 100644 index 000000000..e30ea3b2a --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/env/model.svh @@ -0,0 +1,155 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(result_item) results; + byte unsigned memory[`TSS_BYTES]; + bit initialized[`TSS_BYTES]; + bit pending[`TSS_PORTS], draining[`TSS_PORTS]; + int overtakes[`TSS_PORTS]; + const int offsets[`TSS_PORTS][13] = `TSS_PORT_OFFSETS; + longint unsigned accepted, completed, cancelled; + covergroup behavior with function sample ( + int port, + bit writing, + int size, + int masking, + bit error, + bit cancelling, + bit in_use, + bit holding, + bit contending + ); + option.per_instance = 1; + ports: coverpoint port {bins all[] = {[0 : `TSS_PORTS - 1]};} + direction: coverpoint writing; + sizes: coverpoint size { + bins all[] = {[0 : $clog2(`TSS_MASK_BITS)]}; + ignore_bins invalid = {[$clog2(`TSS_MASK_BITS) + 1 : (1 << `TSS_SIZE_BITS) - 1]}; + } + masks: coverpoint masking {bins all[] = {[0 : 2]};} + errors: coverpoint error; + cancels: coverpoint cancelling; + usage: coverpoint in_use; + stalls: coverpoint holding; + contention: coverpoint contending; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + results = new("results", this); + behavior = new(); + endfunction + function void event_result(int port, int kind, logic [`TSS_DATA_BITS-1:0] value = '0, + bit error = 0); + result_item t = result_item::type_id::create("result"); + t.port = port; + t.message_kind = kind; + t.value = value; + t.error = error; + results.write(t); + endfunction + function void write(cycle_item t); + int fires, offers; + if (t.reset) begin + foreach (pending[p]) begin + if (pending[p]) cancelled++; + event_result(p, 2); + pending[p] = 0; + draining[p] = 0; + overtakes[p] = 0; + end + return; + end + offers = 0; + fires = 0; + for (int p = 0; p < `TSS_PORTS; p++) begin + if (t.inputs[offsets[p][0]]) offers++; + if (t.inputs[offsets[p][0]] && t.outputs[offsets[p][6]]) fires++; + end + if (fires > 1) `uvm_fatal("ARBITRATION", "Multiple memory requests accepted in one cycle") + for (int p = 0; p < `TSS_PORTS; p++) begin + bit cancel_port, valid, ready, rvalid, rready, busy, in_use, eligible; + cancel_port = p < `TSS_CLIENTS && t.inputs[offsets[p][12]]; + in_use = t.inputs[`TSS_IN_INUSE_OFFSET]; + valid = t.inputs[offsets[p][0]]; + ready = t.outputs[offsets[p][6]]; + rvalid = t.outputs[offsets[p][7]]; + rready = t.inputs[offsets[p][10]]; + busy = t.outputs[offsets[p][11]]; + verification_pkg::evidence::checks += 6; + if (pending[p] && !busy) `uvm_fatal("BUSY", "An outstanding request lost its busy indication") + if (!pending[p] && !draining[p] && busy) `uvm_fatal("BUSY", "An idle port is marked busy") + if (draining[p] && !busy) draining[p] = 0; + if ((cancel_port || (p < `TSS_CLIENTS && !in_use)) && ready) + `uvm_fatal("GATING", "Disabled client accepted a request") + if (draining[p] && (ready || rvalid)) + `uvm_fatal("CANCEL", "A cancelled request became visible") + if (cancel_port && pending[p]) begin + event_result(p, 2); + pending[p] = 0; + draining[p] = 1; + cancelled++; + behavior.sample(p, 0, 0, 0, 0, 1, in_use, 0, offers > 1); + end + if (rvalid && !pending[p]) `uvm_fatal("RESPONSE", "Response without an outstanding request") + if (rvalid && rready) begin + event_result(p, 1, t.outputs[offsets[p][8]+:`TSS_DATA_BITS], t.outputs[offsets[p][9]]); + pending[p] = 0; + completed++; + end + eligible = valid && !busy && !cancel_port && (p == `TSS_CLIENTS || in_use); + if (!eligible || ready) overtakes[p] = 0; + else if (fires != 0) begin + overtakes[p]++; + if (overtakes[p] >= `TSS_PORTS) + `uvm_fatal("FAIRNESS", + "Continuously eligible client was overtaken by a full arbitration round") + end + if (valid && ready) begin + logic [63:0] address; + logic [64:0] finish_address, limit_address; + logic [`TSS_DATA_BITS-1:0] value, data; + logic [`TSS_MASK_BITS-1:0] mask, allowed; + int size, count, start; + bit writing, legal; + if (pending[p] || draining[p]) + `uvm_fatal("OUTSTANDING", "Port accepted a second outstanding request") + address = t.inputs[offsets[p][1]+:64]; + size = t.inputs[offsets[p][2]+:`TSS_SIZE_BITS]; + count = 1 << size; + writing = t.inputs[offsets[p][3]]; + data = t.inputs[offsets[p][4]+:`TSS_DATA_BITS]; + mask = t.inputs[offsets[p][5]+:`TSS_MASK_BITS]; + allowed = `TSS_MASK_BITS'((64'd1 << count) - 1); + finish_address = {1'b0, address} + count; + limit_address = {1'b0, `TSS_BASE} + `TSS_BYTES; + legal=count<=`TSS_MASK_BITS && (address&(count-1))==0 && address>=`TSS_BASE && finish_address<=limit_address && !(writing&&((mask&~allowed)!=0)); + value = '0; + if (legal) begin + start = int'(address - `TSS_BASE); + for (int b = 0; b < count; b++) begin + if (writing) begin + if (mask[b]) begin + memory[start+b] = data[b*8+:8]; + initialized[start+b] = 1; + end + end else begin + if (!initialized[start+b]) `uvm_fatal("REFERENCE", "Read of an uninitialized byte") + value[b*8+:8] = memory[start+b]; + end + end + end + event_result(p, 0, value, !legal); + pending[p] = 1; + accepted++; + behavior.sample(p, writing, size, mask == 0 ? 0 : (mask == allowed ? 1 : 2), !legal, 0, + in_use, rvalid && !rready, offers > 1); + end else if (rvalid && !rready) behavior.sample(p, 0, 0, 0, 0, 0, in_use, 1, offers > 1); + end + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + if (accepted != completed + cancelled) + `uvm_fatal("CONSERVATION", "Accepted requests did not retire or cancel") + foreach (pending[p]) + if (pending[p] || draining[p]) `uvm_fatal("PENDING", "Port has not drained") + endfunction +endclass diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/filelist.f b/arch/src/main/scala/framework/core/tss/src/main/resources/filelist.f new file mode 100644 index 000000000..ffef9daea --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Store/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/interface.sv b/arch/src/main/scala/framework/core/tss/src/main/resources/interface.sv new file mode 100644 index 000000000..70c5fbe60 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/interface.sv @@ -0,0 +1,17 @@ +`include "parameters.svh" +interface tss_if ( + input logic clock +); + localparam int PORT_OFFSETS[`TSS_PORTS][13] = `TSS_PORT_OFFSETS; + logic reset = 1; + logic [`TSS_INPUT_BITS-1:0] inputs = '0; + logic [`TSS_OUTPUT_BITS-1:0] outputs; + for (genvar p = 0; p < `TSS_PORTS; p++) begin + assert property (@(posedge clock) disable iff(reset || (p<`TSS_CLIENTS && inputs[PORT_OFFSETS[p][12]])) + outputs[PORT_OFFSETS[p][7]] && !inputs[PORT_OFFSETS[p][10]] |=> outputs[PORT_OFFSETS[p][7]] && + $stable( + {outputs[PORT_OFFSETS[p][8]+:`TSS_DATA_BITS], outputs[PORT_OFFSETS[p][9]]} + )) + else $fatal(1, "TSS port response changed under backpressure"); + end +endinterface diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/package.sv b/arch/src/main/scala/framework/core/tss/src/main/resources/package.sv new file mode 100644 index 000000000..a3c5a5347 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/package.sv @@ -0,0 +1,13 @@ +`include "parameters.svh" +package tss_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/core/tss/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..10ea4f77a --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/seq/sequence.svh @@ -0,0 +1,164 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + const int offsets[`TSS_PORTS][13] = `TSS_PORT_OFFSETS; + logic [`TSS_INPUT_BITS-1:0] pins; + logic [`TSS_OUTPUT_BITS-1:0] observed; + int serial; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.inputs = pins; + t.reset = reset; + finish_item(t); + observed = t.outputs; + endtask + task request(int p, logic [63:0] address, int size, bit writing, logic [`TSS_DATA_BITS-1:0] data, + logic [`TSS_MASK_BITS-1:0] mask); + pins[offsets[p][0]] = 1; + pins[offsets[p][1]+:64] = address; + pins[offsets[p][2]+:`TSS_SIZE_BITS] = size; + pins[offsets[p][3]] = writing; + pins[offsets[p][4]+:`TSS_DATA_BITS] = data; + pins[offsets[p][5]+:`TSS_MASK_BITS] = mask; + endtask + task accept(int p); + do tick(); while (!observed[offsets[p][6]]); + pins[offsets[p][0]] = 0; + endtask + task access (int p, logic [63:0] address, int size, bit writing = 0, + logic [`TSS_DATA_BITS-1:0] data = '0, logic [`TSS_MASK_BITS-1:0] mask = '0); + request(p, address, size, writing, data, mask); + accept(p); + do tick(); while (!observed[offsets[p][7]]); + endtask + task together(bit writing, bit hold_responses); + bit waiting[`TSS_PORTS]; + bit any_waiting; + for (int p = 0; p < `TSS_PORTS; p++) begin + request(p, `TSS_BASE + p * `TSS_MASK_BITS, $clog2(`TSS_MASK_BITS), writing, + `TSS_DATA_BITS'(serial * 1234567 + p), '1); + pins[offsets[p][10]] = !hold_responses; + waiting[p] = 1; + end + serial++; + do begin + tick(); + any_waiting = 0; + for (int p = 0; p < `TSS_PORTS; p++) begin + if (waiting[p] && observed[offsets[p][6]]) begin + waiting[p] = 0; + pins[offsets[p][0]] = 0; + end + any_waiting |= waiting[p]; + end + end while (any_waiting); + if (hold_responses) begin + bit all_valid; + do begin + tick(); + all_valid = 1; + for (int p = 0; p < `TSS_PORTS; p++) all_valid &= observed[offsets[p][7]]; + end while (!all_valid); + end else repeat (8) tick(); + endtask + task body(); + pins = '0; + for (int p = 0; p < `TSS_PORTS; p++) pins[offsets[p][10]] = 1; + repeat (2) tick(1); + tick(); + for (int p = 0; p < `TSS_CLIENTS; p++) + request(p, `TSS_BASE + p * `TSS_MASK_BITS, $clog2(`TSS_MASK_BITS), 0, '0, '0); + for (int row = 0; row < `TSS_BYTES / `TSS_MASK_BITS; row++) begin + logic [`TSS_DATA_BITS-1:0] pattern; + for (int b = 0; b < `TSS_MASK_BITS; b++) pattern[b*8+:8] = 8'(row * 17 + b); + access (`TSS_CLIENTS, `TSS_BASE + row * `TSS_MASK_BITS, $clog2(`TSS_MASK_BITS), 1, pattern, + '1); + end + pins[`TSS_IN_INUSE_OFFSET] = 1; + begin + bit waiting[`TSS_CLIENTS]; + bit any_waiting; + foreach (waiting[p]) waiting[p] = 1; + do begin + tick(); + any_waiting = 0; + foreach (waiting[p]) begin + if (waiting[p] && observed[offsets[p][6]]) begin + waiting[p] = 0; + pins[offsets[p][0]] = 0; + end + any_waiting |= waiting[p]; + end + end while (any_waiting); + repeat (8) tick(); + end + for (int size = 0; size <= $clog2(`TSS_MASK_BITS); size++) + for (int address = 0; address < `TSS_BYTES; address += 1 << size) + access ((address + size) % `TSS_PORTS, `TSS_BASE + address, size); + for (int b = 0; b < `TSS_MASK_BITS; b++) begin + access (b % `TSS_PORTS, `TSS_BASE, $clog2(`TSS_MASK_BITS), 1, ~`TSS_DATA_BITS'(b), + `TSS_MASK_BITS'(1 << b)); + access ((b + 1) % `TSS_PORTS, `TSS_BASE, $clog2(`TSS_MASK_BITS)); + end + access (0, `TSS_BASE, $clog2(`TSS_MASK_BITS), 1, '1, '0); + access (0, `TSS_BASE, $clog2(`TSS_MASK_BITS)); + repeat (32) together(serial % 2, 0); + pins[offsets[0][10]] = 0; + request(0, `TSS_BASE, 0, 0, '0, '0); + accept(0); + do tick(); while (!observed[offsets[0][7]]); + for (int n = 0; n < 10; n++) access (1 + n % (`TSS_PORTS - 1), `TSS_BASE + 64 + n, 0); + pins[offsets[0][10]] = 1; + tick(); + together(1, 1); + pins[offsets[0][12]] = 1; + tick(); + pins[offsets[0][12]] = 0; + for (int p = 1; p < `TSS_PORTS; p++) pins[offsets[p][10]] = 1; + repeat (8) tick(); + pins[offsets[0][10]] = 1; + access (0, `TSS_BASE, $clog2(`TSS_MASK_BITS)); + for (int p = 0; p < `TSS_CLIENTS; p++) begin + pins[offsets[p][10]] = 0; + request(p, `TSS_BASE + 32 + p * `TSS_MASK_BITS, $clog2(`TSS_MASK_BITS), 1, + `TSS_DATA_BITS'(64'hcafef00d + p), '1); + accept(p); + pins[offsets[p][12]] = 1; + tick(); + pins[offsets[p][12]] = 0; + do tick(); while (observed[offsets[p][11]]); + pins[offsets[p][10]] = 1; + access (p, `TSS_BASE + 32 + p * `TSS_MASK_BITS, $clog2(`TSS_MASK_BITS)); + end + together(1, 1); + repeat (2) tick(1); + pins = '0; + pins[`TSS_IN_INUSE_OFFSET] = 1; + for (int p = 0; p < `TSS_PORTS; p++) pins[offsets[p][10]] = 1; + tick(); + for (int p = 0; p < `TSS_PORTS; p++) + access (p, `TSS_BASE + p * `TSS_MASK_BITS, $clog2(`TSS_MASK_BITS)); + pins[offsets[0][10]] = 0; + request(0, `TSS_BASE, 0, 0, '0, '0); + accept(0); + pins[`TSS_IN_INUSE_OFFSET] = 0; + pins[offsets[0][10]] = 1; + repeat (6) tick(); + request(0, `TSS_BASE, 0, 0, '0, '0); + access (`TSS_CLIENTS, `TSS_BASE + 128, 0); + pins[offsets[0][0]] = 0; + pins[`TSS_IN_INUSE_OFFSET] = 1; + for (int p = 0; p < `TSS_PORTS; p++) begin + access (p, `TSS_BASE - 1, 0); + access (p, `TSS_BASE + `TSS_BYTES, 0); + access (p, 64'hffffffffffffffff, 0); + access (p, `TSS_BASE + 1, 1); + access (p, `TSS_BASE, $clog2(`TSS_MASK_BITS) + 1); + access (p, `TSS_BASE, 0, 1, '1, `TSS_MASK_BITS'(2)); + end + repeat (12) tick(); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/suite.toml b/arch/src/main/scala/framework/core/tss/src/main/resources/suite.toml new file mode 100644 index 000000000..6b77c4d9a --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "tss" +top = "tss_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 200000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line","cond","tgl"] +functional_groups = ["tss_pkg::model::behavior"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/tb.sv b/arch/src/main/scala/framework/core/tss/src/main/resources/tb.sv new file mode 100644 index 000000000..ac79ca1a7 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/tb.sv @@ -0,0 +1,16 @@ +module tss_tb; + import uvm_pkg::*; + import tss_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + tss_if bus (clock); + Store dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual tss_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/core/tss/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/core/tss/src/main/resources/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/resources/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/core/tss/src/main/scala/Emit.scala b/arch/src/main/scala/framework/core/tss/src/main/scala/Emit.scala new file mode 100644 index 000000000..c492cad35 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/scala/Emit.scala @@ -0,0 +1,88 @@ +package framework.core.tss + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.ip.memory.spm +import framework.core.tss.configs.Params + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val mem = values("memory") + + val p = Params( + spm.Params(BigInt(mem("base").str), mem("bytes").num.toInt, mem("dataBits").num.toInt), + values("ports").num.toInt + ) + + Files.createDirectories(Paths.get(output)) + val ports = scala.collection.mutable.ArrayBuffer[String]() + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val offsets = scala.collection.mutable.Map[String, Int]() + var inputOffset = 0 + var outputOffset = 0 + + def fields(data: Data, path: String): Seq[(String, Int)] = data match { + case record: Record => record.elements.toSeq.flatMap { case (name, field) => + fields(field, if (path.isEmpty) name else s"${path}_$name") + } + case vector: Vec[_] => vector.toSeq.zipWithIndex.flatMap { case (field, index) => fields(field, s"${path}_$index") } + case leaf => Seq(path -> leaf.getWidth) + } + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + { + val dut = new Store(p) + for ((name, width) <- fields(dut.io, "")) { + val input = name == "inUse" || name.startsWith("cancel_") || name.endsWith("request_valid") || name.contains( + "request_bits_" + ) || name.endsWith("response_ready") + val direction = if (input) "IN" else "OUT" + val signal = if (input) "inputs" else "outputs" + val offset = if (input) inputOffset else outputOffset + definitions += s"`define TSS_${direction}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define TSS_${direction}_${name.toUpperCase}_WIDTH $width" + ports += s".io_$name(bus.$signal[$offset +: $width])" + offsets(name) = offset + if (input) inputOffset += width else outputOffset += width + } + dut + }, + args = Array("--target-dir", s"$output/Store", "--split-verilog") + ) + definitions += s"`define TSS_INPUT_BITS $inputOffset" + definitions += s"`define TSS_OUTPUT_BITS $outputOffset" + definitions += s"`define TSS_CLIENTS ${p.ports}" + definitions += s"`define TSS_PORTS ${p.ports + 1}" + definitions += s"`define TSS_DATA_BITS ${p.memory.dataBits}" + definitions += s"`define TSS_MASK_BITS ${p.memory.beatBytes}" + definitions += s"`define TSS_SIZE_BITS ${p.memory.sizeBits}" + definitions += s"`define TSS_BYTES ${p.memory.bytes}" + definitions += s"`define TSS_BASE 64'h${p.memory.base.toString(16)}" + + val portOffsets = (0 to p.ports).map { index => + val name = if (index == p.ports) "host" else s"clients_$index" + val fields = Seq( + "request_valid", + "request_bits_address", + "request_bits_size", + "request_bits_write", + "request_bits_data", + "request_bits_mask", + "request_ready", + "response_valid", + "response_bits_data", + "response_bits_error", + "response_ready" + ).map(field => offsets(s"${name}_$field")) + (fields ++ Seq( + if (index == p.ports) offsets("hostBusy") else offsets(s"busy_$index"), + if (index == p.ports) 0 + else offsets(s"cancel_$index") + )).mkString("'{", ",", "}") + }.mkString("'{", ",", "}") + + definitions += s"`define TSS_PORT_OFFSETS $portOffsets" + Files.writeString(Paths.get(output, "parameters.svh"), definitions.mkString("\n") + "\n") + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/core/tss/src/main/scala/Store.scala b/arch/src/main/scala/framework/core/tss/src/main/scala/Store.scala new file mode 100644 index 000000000..e88e4be99 --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/scala/Store.scala @@ -0,0 +1,94 @@ +package framework.core.tss + +import framework.core.tss.configs.{Params => ModuleParam} +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.ip.memory.spm.{Memory, Params => MemoryParams, Port, Response} +import framework.core.tss.configs.Params + +class TssControlPort(b: ModuleParam) extends Bundle { + val p = b + val host = new Port(p.memory) + val inUse = Output(Bool()) + val cancel = Output(Vec(p.ports, Bool())) + val busy = Input(Vec(p.ports, Bool())) + val hostBusy = Input(Bool()) +} + +class TssPort(b: ModuleParam) extends TssControlPort(b) { + val clients = Vec(p.ports, new Port(p.memory)) +} + +/** + * One shared SRAM, fair arbitration, one outstanding request and one response slot per port. + * SRAM service is serialized; a held response does not block other clients. inUse covers the + * lifetime of the cooperating group or a private-store transfer. The host may stage disjoint + * ranges while threads run; software owns those ranges. Cancel drains only its owning port. + */ +@instantiable +class Store(b: ModuleParam) extends Module { + val p = b + + val count = p.ports + 1 + val host = p.ports + val indexBits = log2Ceil(count) + + @public + val io = IO(Flipped(new TssPort(b))) + + val memory = Instantiate(new Memory(p.memory)) + val ports = io.clients.toSeq :+ io.host + val occupied = RegInit(VecInit(Seq.fill(count)(false.B))) + val available = RegInit(VecInit(Seq.fill(count)(false.B))) + val cancelled = RegInit(VecInit(Seq.fill(count)(false.B))) + val answers = Reg(Vec(count, new Response(p.memory))) + val active = RegInit(false.B) + val owner = Reg(UInt(indexBits.W)) + val next = RegInit(0.U(indexBits.W)) + val cancel = VecInit(io.cancel.toSeq :+ false.B) + + val eligible = VecInit(ports.zipWithIndex.map { case (port, i) => + val allowed = + if (i == host) true.B + else io.inUse + port.request.valid && !occupied(i) && !cancel(i) && allowed && !reset.asBool + }) + + val after = VecInit((0 until count).map(i => eligible(i) && i.U >= next)).asUInt + val chosen = Mux(after.orR, PriorityEncoder(after), PriorityEncoder(eligible.asUInt)) + memory.io.readOnly := false.B + memory.io.port.request.valid := !active && eligible.asUInt.orR + memory.io.port.request.bits := VecInit(ports.map(_.request.bits))(chosen) + memory.io.port.response.ready := active + when(memory.io.port.request.fire) { + active := true.B + owner := chosen + occupied(chosen) := true.B + cancelled(chosen) := false.B + next := Mux(chosen === (count - 1).U, 0.U, chosen + 1.U) + } + for ((port, i) <- ports.zipWithIndex) { + port.request.ready := !active && eligible(i) && chosen === i.U && memory.io.port.request.ready + port.response.valid := available(i) && !cancelled(i) && !cancel(i) && !reset.asBool + port.response.bits := answers(i) + when(cancel(i) && occupied(i))(cancelled(i) := true.B) + when(port.response.fire || (cancel(i) && available(i))) { + available(i) := false.B + occupied(i) := false.B + cancelled(i) := false.B + } + } + when(memory.io.port.response.fire) { + active := false.B + when(cancelled(owner) || cancel(owner)) { + occupied(owner) := false.B + cancelled(owner) := false.B + }.otherwise { + answers(owner) := memory.io.port.response.bits + available(owner) := true.B + } + } + io.busy := VecInit(occupied.take(p.ports)) + io.hostBusy := occupied(host) +} diff --git a/arch/src/main/scala/framework/core/tss/src/main/scala/configs/Params.scala b/arch/src/main/scala/framework/core/tss/src/main/scala/configs/Params.scala new file mode 100644 index 000000000..f19b5823c --- /dev/null +++ b/arch/src/main/scala/framework/core/tss/src/main/scala/configs/Params.scala @@ -0,0 +1,13 @@ +package framework.core.tss.configs + +import framework.ip.memory.spm.{Params => MemoryParams} +import upickle.default.{macroRW, ReadWriter} + +case class Params(memory: MemoryParams, ports: Int) { + require(ports > 0) +} + +object Params { + implicit val memoryRW: ReadWriter[MemoryParams] = macroRW + implicit val rw: ReadWriter[Params] = macroRW +} diff --git a/arch/src/main/scala/framework/device/build.sc b/arch/src/main/scala/framework/device/build.sc new file mode 100644 index 000000000..2ae5a0909 --- /dev/null +++ b/arch/src/main/scala/framework/device/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.cbus.{build => cbus_build} +import $file.ddr.{build => ddr_build} +import $file.dlink.{build => dlink_build} +import $file.peripherals.{build => peripherals_build} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("configs/DeviceParam.scala")) + ) ++ + cbus_build.sources() ++ + ddr_build.sources() ++ + dlink_build.sources() ++ + peripherals_build.sources() +) + +def emitSources = T.sources( + cbus_build.emitSources() ++ + ddr_build.emitSources() ++ + dlink_build.emitSources() ++ + peripherals_build.emitSources() +) diff --git a/arch/src/main/scala/framework/device/cbus/C2C.scala b/arch/src/main/scala/framework/device/cbus/C2C.scala new file mode 100644 index 000000000..15bf97ef3 --- /dev/null +++ b/arch/src/main/scala/framework/device/cbus/C2C.scala @@ -0,0 +1,134 @@ +package framework.device.cbus + +import framework.device.cbus.C2CParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.core.tbus.router.PhysicalPort +import framework.ip.bus.axi4 +import framework.lib.queue.Queue + +case class C2CParams( + sharedBytes: BigInt, + bankBytes: Int, + cores: Int, + axi: axi4.Params) { + require(sharedBytes > 0 && bankBytes >= 16 && isPow2(bankBytes) && sharedBytes % bankBytes == 0) + require(cores > 0 && axi.dataBits == 128) + val sharedAddressBits: Int = math.max(1, log2Ceil(sharedBytes)) + val coreBits: Int = math.max(1, log2Ceil(cores)) + require(sharedAddressBits + coreBits <= axi.addressBits && sharedAddressBits <= 32) +} + +class C2CCommand(b: C2CParams) extends Bundle { + val p = b + + val sourceByteAddress = UInt(64.W) + val targetCore = UInt(p.coreBits.W) + val targetByteAddress = UInt(64.W) + val bytes = UInt(32.W) +} + +/** + * Push shared SRAM bytes to another Core. Starts are 16-byte aligned; the final beat may be partial. + * One command is active. Bank mappings remain owned by the caller through completion. + * Reset is coordinated with the fabric and SRAM ports, not an independent transaction cancellation. + */ +@instantiable +class C2CTransfer(b: C2CParams) extends Module { + val p = b + + @public + val io = IO(new Bundle { + val command = Flipped(Decoupled(new C2CCommand(b))) + val completion = Decoupled(Bool()) // false = no error, after the final destination write response + val source = + new PhysicalPort(b.axi.dataBits) + val mem = new axi4.Port(p.axi) + }) + + val idle :: plan :: stream :: reply :: complete :: Nil = Enum(5) + val state = RegInit(idle) + val source = Reg(UInt(p.sharedAddressBits.W)) + val target = Reg(UInt(p.sharedAddressBits.W)) + val core = Reg(UInt(p.coreBits.W)) + val remaining = Reg(UInt((p.sharedAddressBits + 1).W)) + val burstBytes = Reg(UInt(13.W)) + val beats = Reg(UInt(9.W)) + val issued = RegInit(0.U(9.W)) + val sent = RegInit(0.U(9.W)) + val pending = RegInit(false.B) + val awSent = RegInit(false.B) + val data = Instantiate(new Queue(UInt(128.W), 2, false, false)) + val live = !reset.asBool + io.command.ready := state === idle && live + io.completion.valid := state === complete && live + io.completion.bits := false.B + when(io.completion.fire)(state := idle) + when(io.command.fire) { + val cmd = io.command.bits + assert(cmd.bytes =/= 0.U && cmd.sourceByteAddress(3, 0) === 0.U && cmd.targetByteAddress(3, 0) === 0.U) + assert(cmd.targetCore < p.cores.U, "C2C target must be a Core system ID") + assert( + cmd.sourceByteAddress +& cmd.bytes <= p.sharedBytes.U && cmd.targetByteAddress +& cmd.bytes <= p.sharedBytes.U, + "C2C command exceeds Core shared SRAM" + ) + source := cmd.sourceByteAddress; target := cmd.targetByteAddress + core := cmd.targetCore; remaining := cmd.bytes; state := plan + } + def minimum(a: UInt, b: UInt): UInt = Mux(a < b, a, b) + val sourcePage = 4096.U(13.W) - source.pad(12)(11, 0) + val targetPage = 4096.U(13.W) - target.pad(12)(11, 0) + val sourceBank = p.bankBytes.U - (source & (p.bankBytes - 1).U) + val targetBank = p.bankBytes.U - (target & (p.bankBytes - 1).U) + val nextBytes = Seq(remaining, sourcePage, targetPage, sourceBank, targetBank, 4096.U).reduce(minimum) + when(state === plan) { + burstBytes := nextBytes; beats := (nextBytes +& 15.U) >> 4 + issued := 0.U; sent := 0.U; pending := false.B; awSent := false.B; state := stream + assert(data.io.count === 0.U && nextBytes =/= 0.U) + } + io.mem.aw.valid := state === stream && !awSent && live + io.mem.aw.bits := 0.U.asTypeOf(new axi4.Address(p.axi)) + io.mem.aw.bits.addr := Cat(core, target(p.sharedAddressBits - 1, 0)) + io.mem.aw.bits.len := beats - 1.U + io.mem.aw.bits.size := 4.U + io.mem.aw.bits.burst := 1.U + when(io.mem.aw.fire)(awSent := true.B) + io.source.request.valid := state === stream && issued < beats && !pending && data.io.count < 2.U && live + io.source.request.bits.address := source + (issued << 4) + io.source.request.bits.write := false.B + io.source.request.bits.data := 0.U + io.source.request.bits.mask := 0.U + when(io.source.request.fire) { pending := true.B; issued := issued + 1.U } + data.io.enq.valid := state === stream && pending && io.source.response.valid && !io.source.response.bits.error && live + data.io.enq.bits := io.source.response.bits.data + io.source.response.ready := state === stream && pending && data.io.enq.ready && live + when(io.source.response.valid && live) { + assert(pending && !io.source.response.bits.error, "C2C shared source read failed") + } + when(io.source.response.fire)(pending := false.B) + val last = sent +& 1.U === beats + val tail = burstBytes(3, 0) + io.mem.w.valid := state === stream && awSent && data.io.deq.valid && live + io.mem.w.bits.data := data.io.deq.bits + io.mem.w.bits.strb := Mux(last && tail =/= 0.U, (1.U(17.W) << tail) - 1.U, "hffff".U) + io.mem.w.bits.last := last + data.io.deq.ready := state === stream && awSent && io.mem.w.ready && live + when(io.mem.w.fire) { sent := sent + 1.U; when(last)(state := reply) } + io.mem.b.ready := state === reply && live + when(io.mem.b.fire) { + assert(io.mem.b.bits.id === 0.U && io.mem.b.bits.resp === 0.U, "C2C destination write failed") + assert(!pending && data.io.count === 0.U && issued === beats && sent === beats) + when(io.mem.b.bits.id === 0.U && io.mem.b.bits.resp === 0.U) { + when(remaining === burstBytes)(state := complete).otherwise { + source := source + burstBytes; target := target + burstBytes + remaining := remaining - burstBytes; state := plan + } + } + } + io.mem.ar.valid := false.B + io.mem.ar.bits := 0.U.asTypeOf(new axi4.Address(p.axi)) + io.mem.r.ready := false.B + when(live)(assert(!io.mem.r.valid, "C2C push engine received an unsolicited AXI read response")) +} diff --git a/arch/src/main/scala/framework/device/cbus/CBus.scala b/arch/src/main/scala/framework/device/cbus/CBus.scala new file mode 100644 index 000000000..d2326c7c2 --- /dev/null +++ b/arch/src/main/scala/framework/device/cbus/CBus.scala @@ -0,0 +1,148 @@ +package framework.device.cbus + +import sims.configs.SimParam +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.device.configs.DeviceParam +import framework.core.configs.CoreParam +import buckyball.config.{CorePlacement, Design} +import scala.jdk.CollectionConverters._ +import framework.core.clink.{CLinkIO, CoreInterrupts} +import framework.device.ddr.Bridge +import framework.ip.bus.axi4 +import framework.ip.bus.chi.RequesterPort +import framework.ip.bus.chi.snf.LineSramIO +import framework.device.dlink.BaseDevice + +/** Routes core memory requests, shared-storage transfers and hardware interrupts. */ +@instantiable +class CBus(b: DeviceParam, generators: Seq[() => CLinkIO]) extends BaseDevice(b.ddr.axi) { + require(generators.size == b.cores.size) + val corePorts = generators.map(_()) + val ports = corePorts + require( + corePorts.zip(b.cores).forall { case (port, corep) => port.p == corep.clink }, + "Core implementation must expose its configured CLink" + ) + require( + ports.forall(_.p.ddr == b.ddr.axi), + "CBus DDR connections require matching AXI protocols or an explicit adapter" + ) + require( + ports.forall(_.p.uncached == b.ddr.axi), + "CBus uncached connections require matching AXI protocols or an explicit adapter" + ) + require( + ports.filter(port => port.p.backing || port.p.requesters > 0).forall(_.p.line == b.ddr.line), + "CBus line connections require matching CHI protocols or an explicit adapter" + ) + val backingPorts = corePorts.flatMap(_.memory.backing.toSeq) + val coherentNodes = corePorts.flatMap(port => port.p.requesterIds.zip(port.memory.coherent)) + require( + coherentNodes.map(_._1).sorted == (1 to coherentNodes.size), + "CHI Home requires one connection for every requester node ID" + ) + val coherentPorts = coherentNodes.sortBy(_._1).map(_._2) + val bankPorts = corePorts.flatMap(_.banks.toSeq) + @public + val banks = IO(MixedVec(ports.flatMap(_.banks.toSeq).map(_.cloneType))) + bankPorts.zip(banks).foreach { case (core, fabric) => fabric <> core } + val hartCounts = corePorts.map(_.interrupts.size) + val hartBase = hartCounts.scanLeft(0)(_ + _) + + @public + val backing = IO(Vec(backingPorts.size, Flipped(new LineSramIO(b.ddr.line)))) + @public + val coherent = IO(Vec(coherentPorts.size, new RequesterPort(b.ddr.line))) + @public + val memory = IO(Vec(backingPorts.size, new LineSramIO(b.ddr.line))) + @public + val uncached = IO(new axi4.Port(b.ddr.axi)) + @public + val external = IO(Flipped(new axi4.Port(b.ddr.axi))) + @public + val interrupts = IO(Input(Vec(hartCounts.sum, new CoreInterrupts))) + + for ((port, index) <- backingPorts.zipWithIndex) { + backing(index).req <> port.request + port.response <> backing(index).resp + } + for ((port, index) <- coherentPorts.zipWithIndex) coherent(index) <> port + for { + (core, index) <- corePorts.zipWithIndex + hart <- core.interrupts.indices + } { + core.interrupts(hart) := interrupts(hartBase(index) + hart) + } + + val linked = corePorts.indices.filter(i => corePorts(i).c2c.isDefined) + if (linked.nonEmpty) { + val fabric = Instantiate(new Fabric(b)) + for ((core, index) <- linked.zipWithIndex) { + fabric.io.source(index) <> corePorts(core).c2c.get.tx + corePorts(core).c2c.get.rx <> fabric.io.target(index) + } + } + + val bridgeMasters = if (backingPorts.nonEmpty) 1 else 0 + + val memoryBus = + Instantiate(new axi4.Interconnect(axi4.InterconnectParam(b.ddr.axi, corePorts.size + bridgeMasters + 1, None))) + + if (backingPorts.nonEmpty) { + require(b.ddr.clients == backingPorts.size) + val bridge = Instantiate(new Bridge(b.ddr)) + memoryBus.io.in(0) <> bridge.io.axi + for (index <- backingPorts.indices) { + bridge.io.request(index) <> memory(index).req + memory(index).resp <> bridge.io.response(index) + } + } + for ((core, index) <- corePorts.zipWithIndex) memoryBus.io.in(bridgeMasters + index) <> core.memory.ddr + memoryBus.io.in(corePorts.size + bridgeMasters) <> external + dlink.axi <> memoryBus.io.out + + val cpuMemory = Instantiate(new axi4.Interconnect(axi4.InterconnectParam(b.ddr.axi, corePorts.size, None))) + + for ((core, index) <- corePorts.zipWithIndex) cpuMemory.io.in(index) <> core.memory.uncached + uncached <> cpuMemory.io.out +} + +trait CoreFactory { + def component: String + def parameters(design: Design, source: CorePlacement): CoreParam + + def instantiate( + source: CorePlacement, + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int] + ): CLinkIO + +} + +object CBus { + + def parameters(design: Design, components: Map[String, CoreFactory]): Seq[CoreParam] = + design.getDevice.getCoresList.asScala.map { source => + components(source.getComponent).parameters(design, source) + }.toSeq + + def generators( + design: Design, + devicep: DeviceParam, + simp: SimParam, + components: Map[String, CoreFactory] + ): Seq[() => CLinkIO] = { + val sources = design.getDevice.getCoresList.asScala.toSeq + require(sources.size == devicep.cores.size) + sources.zip(devicep.cores).map { case (source, corep) => + require(source.getHartId == corep.hartId, "Core configuration order does not match design.pb") + val factory = components(source.getComponent) + val traceIds = source.getThreadIndicesList.asScala.map(_.intValue + sources.size).toSeq + () => factory.instantiate(source, corep, simp, traceIds) + } + } + +} diff --git a/arch/src/main/scala/framework/device/cbus/Control.scala b/arch/src/main/scala/framework/device/cbus/Control.scala new file mode 100644 index 000000000..95d8717d3 --- /dev/null +++ b/arch/src/main/scala/framework/device/cbus/Control.scala @@ -0,0 +1,145 @@ +package framework.device.cbus + +import framework.device.cbus.C2CParams +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.core.tbus.router.{LeasePort, PhysicalPort} + +/** The controller stages one transfer; the start instruction returns after destination commit. */ +class ControlRequest extends Bundle { + val operation = UInt(7.W) + val context = UInt(32.W) + val field = UInt(32.W) + val data = UInt(64.W) +} + +@instantiable +class Control( + memParam: MemDomainParam, + transferParam: C2CParams, + leaseAccess: Boolean, + localAccess: Boolean) + extends Module { + val p = transferParam + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new ControlRequest)) + val reply = Decoupled(UInt(64.W)) + val coreId = Input(UInt(p.coreBits.W)) + val command = Decoupled(new C2CCommand(transferParam)) + val completion = Flipped(Decoupled(Bool())) + val idle = Output(Bool()) + val local = Option.when(localAccess)(new PhysicalPort(memParam.bankWidth)) + val lease = Option.when(leaseAccess)(new LeasePort) + }) + + val idle :: running :: reading :: leasing :: responding :: Nil = Enum(5) + val state = RegInit(idle) + val fields = RegInit(0.U(4.W)) + val staged = Reg(Vec(4, UInt(64.W))) + val answer = Reg(UInt(64.W)) + val request = io.request.bits + val start = request.operation === 14.U + val access = request.operation === 16.U || request.operation === 17.U + val lease = request.operation === 18.U || request.operation === 19.U + val upperHalf = Reg(Bool()) + val localWrite = Reg(Bool()) + val localReady = io.local.map(_.request.ready).getOrElse(false.B) + val leaseReady = io.lease.map(_.request.ready).getOrElse(false.B) + io.lease.foreach { port => + port.request.valid := state === idle && io.request.valid && lease && !reset.asBool + port.request.bits.endpoint := request.context + port.request.bits.vbank := request.field(31, 16) + port.request.bits.group := request.field(15, 0) + port.request.bits.release := request.operation === 19.U + port.response.ready := state === leasing && !reset.asBool + when(port.response.fire) { + answer := port.response.bits + state := responding + } + } + io.local.foreach { port => + val address = Cat(request.context, request.field) + port.request.valid := state === idle && io.request.valid && access && !reset.asBool + port.request.bits.address := address & ~15.U(64.W) + port.request.bits.write := request.operation === 17.U + port.request.bits.data := Cat(request.data, request.data) + port.request.bits.mask := Mux(address(3), "hff00".U, "h00ff".U) + port.response.ready := state === reading && !reset.asBool + when(port.response.fire) { + assert(!port.response.bits.error, "Main shared-storage access failed") + answer := Mux(localWrite, 0.U, Mux(upperHalf, port.response.bits.data(127, 64), port.response.bits.data(63, 0))) + state := responding + } + } + io.command.valid := state === idle && io.request.valid && start && !reset.asBool + io.command.bits.sourceByteAddress := staged(0) + io.command.bits.targetCore := staged(1) + io.command.bits.targetByteAddress := staged(2) + io.command.bits.bytes := staged(3) + io.request.ready := state === idle && !reset.asBool && + Mux(lease, leaseReady, Mux(access, localReady, !start || io.command.ready)) + io.reply.valid := state === responding && !reset.asBool + io.reply.bits := answer + io.completion.ready := state === running && !reset.asBool + io.idle := state === idle + when(io.request.valid && state === idle && !reset.asBool) { + assert( + (request.operation >= 13.U && request.operation <= 15.U) || + (localAccess.B && access) || (leaseAccess.B && lease), + "Invalid CLink management operation" + ) + } + when(io.request.fire) { + when(!access && !lease)(assert(request.context === 0.U, "CLink management has one core-owned transfer context")) + answer := 0.U + state := responding + when(lease) { + assert(request.data === 0.U, "CLink shared lease reserves rs2") + state := leasing + } + when(access) { + val address = Cat(request.context, request.field) + assert(address(2, 0) === 0.U && (address +& 8.U) <= p.sharedBytes.U, "Invalid main shared-storage access") + upperHalf := address(3) + localWrite := request.operation === 17.U + state := reading + } + switch(request.operation) { + is(13.U) { + assert(request.field < 4.U, "Invalid CLink transfer descriptor field") + staged(request.field(1, 0)) := request.data + fields := fields | UIntToOH(request.field, 4) + } + is(14.U) { + assert(request.field === 0.U, "CLink start reserves the field selector") + assert(fields.andR, "Incomplete CLink transfer descriptor") + assert( + staged(1) < p.cores.U && staged(1) =/= io.coreId, + "CLink transfer must target another core" + ) + assert(staged(3) > 0.U && (staged(3) >> 32) === 0.U, "Invalid CLink transfer size") + fields := 0.U + state := running + } + is(15.U) { + assert(request.field < 4.U, "Invalid CLink geometry query") + answer := MuxLookup(request.field, 0.U)(Seq( + 0.U -> p.sharedBytes.U(64.W), + 1.U -> p.bankBytes.U(64.W), + 2.U -> io.coreId.pad(64), + 3.U -> p.cores.U(64.W) + )) + } + } + } + when(io.completion.fire) { + assert(!io.completion.bits, "CLink transfer failed") + state := responding + } + when(io.reply.fire)(state := idle) +} diff --git a/arch/src/main/scala/framework/device/cbus/Fabric.scala b/arch/src/main/scala/framework/device/cbus/Fabric.scala new file mode 100644 index 000000000..a0c9306bb --- /dev/null +++ b/arch/src/main/scala/framework/device/cbus/Fabric.scala @@ -0,0 +1,92 @@ +package framework.device.cbus + +import framework.device.configs.DeviceParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.ip.bus.axi4 + +/** Core-addressed shared-storage fabric, separate from the DDR and CPU coherence paths. */ +@instantiable +class Fabric(b: DeviceParam) extends Module { + val cores = b.cores.filter(_.clink.shared) + val p = cores.head.clink.c2c.get + require(cores.forall(_.clink.c2c.contains(p)), "C2C endpoints must use the fabric AXI protocol") + val hartIds = cores.map(_.hartId) + val sharedBytes = cores.head.c2c.sharedBytes + require(cores.forall(_.c2c.sharedBytes == sharedBytes)) + + require(hartIds.nonEmpty && hartIds.distinct.size == hartIds.size && hartIds.forall(_ >= 0)) + require(p.dataBits == 128 && sharedBytes >= p.bytes) + val localBits = log2Ceil(sharedBytes) + val indexBits = math.max(1, log2Ceil(hartIds.size)) + require(p.addressBits >= localBits + log2Ceil(hartIds.max + 1)) + + @public + val io = IO(new Bundle { + val source = Vec(hartIds.size, Flipped(new axi4.Port(p))) + val target = Vec(hartIds.size, new axi4.Port(p)) + }) + + val targets = + hartIds.indices.map(_ => Instantiate(new axi4.Interconnect(axi4.InterconnectParam(p, hartIds.size, None)))) + + for (i <- hartIds.indices) { io.target(i) <> targets(i).io.out } + + for (source <- hartIds.indices) { + val port = io.source(source) + val reading = RegInit(false.B) + val writing = RegInit(false.B) + val readTarget = Reg(UInt(indexBits.W)) + val writeTarget = Reg(UInt(indexBits.W)) + val readMatches = VecInit(hartIds.map(id => (port.ar.bits.addr >> localBits) === id.U)) + val writeMatches = VecInit(hartIds.map(id => (port.aw.bits.addr >> localBits) === id.U)) + val readSelection = PriorityEncoder(readMatches) + val writeSelection = PriorityEncoder(writeMatches) + + def check(address: axi4.Address, matched: Bool): Unit = { + val offset = address.addr(localBits - 1, 0) + val bytes = (address.len +& 1.U) << log2Ceil(p.bytes) + assert(matched, "CLink address does not select a shared-storage core") + assert( + address.size === log2Ceil(p.bytes).U && address.burst === 1.U && !address.lock, + "CLink requires full-width INCR bursts" + ) + assert( + offset(log2Ceil(p.bytes) - 1, 0) === 0.U && (offset +& bytes) <= sharedBytes.U, + "CLink burst exceeds shared storage or is unaligned" + ) + assert((offset.pad(12)(11, 0) +& bytes) <= 4096.U, "CLink burst crosses a 4KiB boundary") + } + when(port.ar.valid && !reading)(check(port.ar.bits, readMatches.asUInt.orR)) + when(port.aw.valid && !writing)(check(port.aw.bits, writeMatches.asUInt.orR)) + when(port.ar.fire) { reading := true.B; readTarget := readSelection } + when(port.r.fire && port.r.bits.last)(reading := false.B) + when(port.aw.fire) { writing := true.B; writeTarget := writeSelection } + when(port.b.fire)(writing := false.B) + + port.ar.ready := !reading && readMatches.asUInt.orR && VecInit(targets.map(_.io.in(source).ar.ready))(readSelection) + port.aw.ready := !writing && writeMatches.asUInt.orR && VecInit(targets.map(_.io.in(source).aw.ready))( + writeSelection + ) + port.w.ready := writing && VecInit(targets.map(_.io.in(source).w.ready))(writeTarget) + port.r.valid := reading && VecInit(targets.map(_.io.in(source).r.valid))(readTarget) + port.r.bits := VecInit(targets.map(_.io.in(source).r.bits))(readTarget) + port.b.valid := writing && VecInit(targets.map(_.io.in(source).b.valid))(writeTarget) + port.b.bits := VecInit(targets.map(_.io.in(source).b.bits))(writeTarget) + for (target <- hartIds.indices) { + val route = targets(target).io.in(source) + route.ar.valid := port.ar.valid && !reading && readMatches(target) + route.ar.bits := port.ar.bits + route.ar.bits.addr := port.ar.bits.addr(localBits - 1, 0) + route.aw.valid := port.aw.valid && !writing && writeMatches(target) + route.aw.bits := port.aw.bits + route.aw.bits.addr := port.aw.bits.addr(localBits - 1, 0) + route.w.valid := port.w.valid && writing && writeTarget === target.U + route.w.bits := port.w.bits + route.r.ready := port.r.ready && reading && readTarget === target.U + route.b.ready := port.b.ready && writing && writeTarget === target.U + } + } +} diff --git a/arch/src/main/scala/framework/system/tile/tlink/SharedAxiEndpoint.scala b/arch/src/main/scala/framework/device/cbus/SharedAxiEndpoint.scala similarity index 83% rename from arch/src/main/scala/framework/system/tile/tlink/SharedAxiEndpoint.scala rename to arch/src/main/scala/framework/device/cbus/SharedAxiEndpoint.scala index e0b24ea05..da03bdd2e 100644 --- a/arch/src/main/scala/framework/system/tile/tlink/SharedAxiEndpoint.scala +++ b/arch/src/main/scala/framework/device/cbus/SharedAxiEndpoint.scala @@ -1,24 +1,30 @@ -package framework.system.tile.tlink +package framework.device.cbus + +import framework.device.cbus.C2CParams import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.memdomain.backend.shared.SharedPhysicalPort -import memcore.bus.axi4 +import framework.core.tbus.router.PhysicalPort +import framework.ip.bus.axi4 /** - * AXI access to the existing Tile shared SRAM. No storage is instantiated here. + * AXI access to the existing Core shared SRAM. No storage is instantiated here. * Read and write bursts each have one outstanding transaction and independent ports. * W acceptance is not completion: B follows the actual final shared-bank write acknowledgement. */ @instantiable -class SharedAxiEndpoint(p: T2TParams) extends Module { +class SharedAxiEndpoint(b: C2CParams) extends Module { + val p = b - @public val io = IO(new Bundle { + @public + val io = IO(new Bundle { val busy = Output(Bool()) val mem = Flipped(new axi4.Port(p.axi)) - val read = new SharedPhysicalPort(128) - val write = new SharedPhysicalPort(128) + val read = + new PhysicalPort(b.axi.dataBits) + val write = + new PhysicalPort(b.axi.dataBits) }) val rIdle :: rIssue :: rWait :: rReply :: Nil = Enum(4) @@ -41,11 +47,11 @@ class SharedAxiEndpoint(p: T2TParams) extends Module { val bytes = (address.len +& 1.U) << 4 assert( address.size === 4.U && address.burst === 1.U && !address.lock && address.addr(3, 0) === 0.U, - "T2T shared SRAM requires unlocked aligned 128-bit INCR bursts" + "C2C shared SRAM requires unlocked aligned 128-bit INCR bursts" ) - assert(address.addr +& bytes <= p.sharedBytes.U, "T2T AXI address exceeds Tile shared SRAM") - assert(address.addr.pad(12)(11, 0) +& bytes <= 4096.U, "T2T AXI burst crosses 4KiB") - assert((address.addr & (p.bankBytes - 1).U) +& bytes <= p.bankBytes.U, "T2T AXI burst crosses a shared bank") + assert(address.addr +& bytes <= p.sharedBytes.U, "C2C AXI address exceeds Core shared SRAM") + assert(address.addr.pad(12)(11, 0) +& bytes <= 4096.U, "C2C AXI burst crosses 4KiB") + assert((address.addr & (p.bankBytes - 1).U) +& bytes <= p.bankBytes.U, "C2C AXI burst crosses a shared bank") } io.mem.ar.ready := readState === rIdle && live @@ -62,7 +68,7 @@ class SharedAxiEndpoint(p: T2TParams) extends Module { when(io.read.request.fire)(readState := rWait) io.read.response.ready := readState === rWait && live when(io.read.response.fire) { - assert(!io.read.response.bits.error, "T2T shared SRAM read failed") + assert(!io.read.response.bits.error, "C2C shared SRAM read failed") when(!io.read.response.bits.error) { readData := io.read.response.bits.data; readState := rReply } } io.mem.r.valid := readState === rReply && live @@ -84,7 +90,7 @@ class SharedAxiEndpoint(p: T2TParams) extends Module { } io.mem.w.ready := writeState === wData && live when(io.mem.w.fire) { - assert(io.mem.w.bits.last === (writeRemaining === 1.U), "T2T AXI WLAST does not match AWLEN") + assert(io.mem.w.bits.last === (writeRemaining === 1.U), "C2C AXI WLAST does not match AWLEN") writeData := io.mem.w.bits.data; writeMask := io.mem.w.bits.strb; writeState := wIssue } io.write.request.valid := writeState === wIssue && live @@ -95,7 +101,7 @@ class SharedAxiEndpoint(p: T2TParams) extends Module { when(io.write.request.fire)(writeState := wWait) io.write.response.ready := writeState === wWait && live when(io.write.response.fire) { - assert(!io.write.response.bits.error, "T2T shared SRAM write failed") + assert(!io.write.response.bits.error, "C2C shared SRAM write failed") when(!io.write.response.bits.error) { when(writeRemaining === 1.U)(writeState := wReply).otherwise { writeAddress := writeAddress + 16.U; writeRemaining := writeRemaining - 1.U; writeState := wData diff --git a/arch/src/main/scala/framework/device/cbus/build.sc b/arch/src/main/scala/framework/device/cbus/build.sc new file mode 100644 index 000000000..d30e2cfd7 --- /dev/null +++ b/arch/src/main/scala/framework/device/cbus/build.sc @@ -0,0 +1,46 @@ +import mill._ + +// The registry is compiled with its consumers; framework never imports component implementations. +def registry(sources: Seq[PathRef], destination: os.Path): Seq[PathRef] = { + val files = sources.flatMap { source => + if (os.isDir(source.path)) + os.walk(source.path).filter(p => p.ext == "scala" && !p.segments.exists(Set("out", "generated", "target", "build"))) + else Seq(source.path) + } + val factories = files.flatMap { source => + val text = os.read(source) + val names = "(?m)^object (\\w+) extends CoreFactory".r.findAllMatchIn(text).map(_.group(1)).toSeq + if (names.isEmpty) Seq.empty + else { + val pkg = "(?m)^package ([\\w.]+)".r.findFirstMatchIn(text).get.group(1) + names.map(name => s"$pkg.$name") + } + }.distinct.sorted + val output = destination / "CoreComponents.scala" + os.write.over( + output, + s"""package framework.device.cbus.generated +import framework.device.cbus.CoreFactory +object CoreComponents { + val components: Seq[CoreFactory] = Seq(${factories.mkString(",\n")}) + require(components.map(_.component).distinct.size == components.size, "Duplicate Core component registration") + val factories: Map[String, CoreFactory] = components.map(f => f.component -> f).toMap +} +""" + ) + Seq(PathRef(output)) +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("C2C.scala")), + PathRef(millSourcePath / os.RelPath("CBus.scala")), + PathRef(millSourcePath / os.RelPath("Control.scala")), + PathRef(millSourcePath / os.RelPath("Fabric.scala")), + PathRef(millSourcePath / os.RelPath("SharedAxiEndpoint.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/device/configs/DeviceParam.scala b/arch/src/main/scala/framework/device/configs/DeviceParam.scala new file mode 100644 index 000000000..470942a13 --- /dev/null +++ b/arch/src/main/scala/framework/device/configs/DeviceParam.scala @@ -0,0 +1,45 @@ +package framework.device.configs + +import buckyball.config.DeviceConfig + +import framework.core.configs.CoreParam +import framework.device.ddr.{Params => DdrParams} +import framework.device.peripherals.bootrom.configs.BootRomParams +import framework.device.peripherals.clint.configs.ClintParams +import framework.device.peripherals.plic.configs.PlicParams +import framework.device.peripherals.scu.configs.SCUParams + +case class DeviceParam( + cores: Seq[CoreParam], + ddr: DdrParams, + bootrom: BootRomParams, + clint: ClintParams, + plic: PlicParams, + scu: SCUParams) { + val hartIds = cores.map(_.hartId) + require(hartIds.distinct.size == hartIds.size, "Duplicate core hart ID") + require( + clint.hartIds == hartIds && plic.hartIds == hartIds, + "Peripheral interrupt ordering must match the configured core instances" + ) +} + +object DeviceParam { + + def fromProto(p: DeviceConfig, cores: Seq[CoreParam]): DeviceParam = { + require( + p.hasDdr && p.hasBootrom && p.hasClint && p.hasPlic && p.hasScu && p.isInitialized, + "Device configuration is incomplete" + ) + require(p.getCoresCount == cores.size, "Device core configuration is incomplete") + DeviceParam( + cores, + DdrParams.fromProto(p.getDdr), + BootRomParams.fromProto(p.getBootrom), + ClintParams.fromProto(p.getClint), + PlicParams.fromProto(p.getPlic), + SCUParams.fromProto(p.getScu) + ) + } + +} diff --git a/arch/src/main/scala/framework/device/ddr/build.sc b/arch/src/main/scala/framework/device/ddr/build.sc new file mode 100644 index 000000000..5e5f532e5 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.device.ddr.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Bridge.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/device/ddr/configs/default.toml b/arch/src/main/scala/framework/device/ddr/configs/default.toml new file mode 100644 index 000000000..c68d70845 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/configs/default.toml @@ -0,0 +1,13 @@ +clients = 2 +slotsPerClient = 4 +dataBits = 128 +idBits = 4 +idBase = 0 +base = 0x80000000 +bytes = 0x20000000 +model = 0 + +[line] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 diff --git a/arch/src/main/scala/framework/device/ddr/configs/uvm.toml b/arch/src/main/scala/framework/device/ddr/configs/uvm.toml new file mode 100644 index 000000000..b6bdc76f5 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/configs/uvm.toml @@ -0,0 +1,14 @@ +[parameters] +clients=2 +slotsPerClient=4 +dataBits=128 +idBits=4 +idBase=0 +base="2147483648" +bytes="536870912" +model="0" +[parameters.line] +nodeIdBits=7 +addressBits=44 +dataBits=256 +[defines] diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/csrc/Cargo.lock b/arch/src/main/scala/framework/device/ddr/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/ddr/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/device/ddr/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/csrc/Cargo.toml b/arch/src/main/scala/framework/device/ddr/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/ddr/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/device/ddr/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/bank/src/csrc/src/lib.rs b/arch/src/main/scala/framework/device/ddr/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/bank/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/device/ddr/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/csrc/src/memory.rs b/arch/src/main/scala/framework/device/ddr/src/csrc/src/memory.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/ddr/src/csrc/src/memory.rs rename to arch/src/main/scala/framework/device/ddr/src/csrc/src/memory.rs diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/device/ddr/src/main/resources/agent/item.svh new file mode 100644 index 000000000..5430e1893 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/agent/item.svh @@ -0,0 +1,17 @@ +class result_item extends uvm_sequence_item; + bit [511:0] data; + bit error; + int tag, client; + `uvm_object_utils_begin(result_item) + + `uvm_field_int(data, UVM_DEFAULT) + `uvm_field_int(error, UVM_DEFAULT) + + `uvm_field_int(tag, UVM_DEFAULT) + `uvm_field_int(client, UVM_DEFAULT) + + `uvm_object_utils_end + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/agent/service.svh b/arch/src/main/scala/framework/device/ddr/src/main/resources/agent/service.svh new file mode 100644 index 000000000..869a823f6 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/agent/service.svh @@ -0,0 +1,166 @@ +task service(); + forever begin + @(ctl.sample); + if (!ctl.sample.reset) begin + cycle++; + for (int c = 0; c < 2; c++) begin + if (req[c].sample.valid && !req[c].sample.ready) request_stalls++; + if (resp[c].sample.valid && !resp[c].sample.ready) response_stalls++; + if (req[c].sample.valid && req[c].sample.ready) begin + result_item expected = new; + req_t q = req[c].sample.bits; + int key = (c << 12) | int'(q[`DF(REQ, ID)]); + bit [511:0] line; + verify_contract(!commands.exists(key), "Accepted duplicate live client tag"); + verify_contract(ddr_ref_read(model, q[`DF(REQ, ADDR)], line), + "Uninitialized fixture line"); + commands[key] = '{q, c, 0}; + expected.set_transaction_id(key); + expected.client = c; + expected.tag = q[`DF(REQ, ID)]; + expected.error = q[`DF(REQ, WRITE)] ? write_error.exists(q[`DF(REQ, ADDR)]) : + read_error.exists(q[`DF(REQ, ADDR)]); + expected.data = (q[`DF(REQ, WRITE)] || expected.error) ? '0 : line; + scoreboard.expected_export.write(expected); + accepted++; + behavior.sample(c, q[`DF(REQ, WRITE)], q[`DF(REQ, MASK)] == 0 ? 0 : (q[`DF(REQ, MASK) + ] == 64'hffffffffffffffff ? 1 : 2), expected.error, commands.num(), 0); + if (commands.num() > peak) peak = commands.num(); + end + end + if (aw.sample.valid && !aw.sample.ready) aw_stalls++; + if (w.sample.valid && !w.sample.ready) w_stalls++; + if (aw.sample.valid && aw.sample.ready) begin + int key = observe_address(aw.sample.bits, 1); + int id = aw.sample.bits[`DF(A, ID)]; + aw_ids.push_back(id); + aw_keys.push_back(key); + aw_serials.push_back(aw_count); + aw_count++; + end + if (w.sample.valid && w.sample.ready) begin + verify_contract(!$isunknown(w.sample.bits), "Unknown W payload"); + verify_contract(w.sample.bits[`DF(W, LAST)] == (assembling_beat == `DDR_BEATS - 1), + "Wrong WLAST"); + assembling.data[assembling_beat*`DDR_DATA_BITS+:`DDR_DATA_BITS] = w.sample.bits[ + `DF(W, DATA) + ]; + assembling.mask[assembling_beat*(`DDR_BEAT_BYTES)+:(`DDR_BEAT_BYTES)] = w.sample.bits[ + `DF(W, STRB) + ]; + if (assembling_beat == `DDR_BEATS - 1) begin + w_lines.push_back(assembling); + assembling = '0; + assembling_beat = 0; + w_bursts++; + end else assembling_beat++; + end + if (aw_ids.size() && w_lines.size()) begin + int id = aw_ids.pop_front(); + int key = aw_keys.pop_front(); + int serial = aw_serials.pop_front(); + wline line = w_lines.pop_front(); + req_t q = commands[key].packet; + verify_contract(!writing_ids.exists(id) && !reading_ids.exists(id), + "AXI ID reused while active"); + verify_contract(line.data === q[`DF(REQ, DATA)] && line.mask === q[`DF(REQ, MASK)], + "AW/W order, payload or byte mask mismatch"); + verify_contract(ddr_ref_write(model, q[`DF(REQ, ADDR)], line.data, line.mask), + "Write outside initialized DDR"); + writing_ids[id] = '{key, cycle + 7, serial}; + end + if (ar.sample.valid && ar.sample.ready) begin + int key = observe_address(ar.sample.bits, 0); + int id = ar.sample.bits[`DF(A, ID)]; + bit [511:0] line; + verify_contract(!reading_ids.exists(id) && !writing_ids.exists(id), + "AXI ID reused while active"); + verify_contract(ddr_ref_read(model, ar.sample.bits[`DF(A, ADDR)], line), + "Read outside initialized DDR"); + reading_ids[id] = '{key, 0, ar_count, line}; + ar_count++; + end + if (r.sample.valid && r.sample.ready) begin + int key = reading_ids[r_id].key; + r_beats++; + if (last_r_id >= 0 && last_r_id != r_id && reading_ids.exists( + last_r_id + ) && reading_ids[last_r_id].serial == last_r_serial) + interleaves++; + last_r_id = r_id; + last_r_serial = reading_ids[r_id].serial; + if (reading_ids[r_id].beat == `DDR_BEATS - 1) begin + if (reading_ids[r_id].serial < max_r_serial) r_reorders++; + else max_r_serial = reading_ids[r_id].serial; + commands[key].done = 1; + reading_ids.delete(r_id); + end else reading_ids[r_id].beat++; + r_active = 0; + r_cursor = (r_id + 1) % 16; + end + if (b.sample.valid && b.sample.ready) begin + if (writing_ids[b_id].serial < max_b_serial) b_reorders++; + else max_b_serial = writing_ids[b_id].serial; + commands[writing_ids[b_id].key].done = 1; + writing_ids.delete(b_id); + b_active = 0; + end + for (int c = 0; c < 2; c++) + if (resp[c].sample.valid && resp[c].sample.ready) begin + result_item actual = new; + int key = (c << 12) | int'(resp[c].sample.bits[`DF(RESP, ID)]); + verify_contract(!$isunknown(resp[c].sample.bits) && commands.exists(key + ) && commands[key].done, "Early/unknown line response before final R or B"); + actual.set_transaction_id(key); + actual.client = c; + actual.tag = resp[c].sample.bits[`DF(RESP, ID)]; + actual.data = resp[c].sample.bits[`DF(RESP, DATA)]; + actual.error = resp[c].sample.bits[`DF(RESP, ERROR)]; + scoreboard.actual_export.write(actual); + if (actual.error) error_results++; + returned[c]++; + commands.delete(key); + end + end + @(negedge ctl.clock); + aw.ready = !ctl.reset && !hold_aw && cycle % 5 != 0; + w.ready = !ctl.reset && !hold_w && cycle % 7 >= 2; + ar.ready = !ctl.reset && !hold_ar && cycle % 4 != 0; + for (int c = 0; c < 2; c++) resp[c].ready = !ctl.reset && !hold_resp[c] && cycle % 5 != c; + if (!r_active && !hold_r) + for (int off = 0; off < 16; off++) begin + int id = (r_cursor + off) % 16; + if (!r_active && reading_ids.exists(id)) begin + int key = reading_ids[id].key; + u64 address = commands[key].packet[`DF(REQ, ADDR)]; + int beat = reading_ids[id].beat; + r_active = 1; + r_id = id; + r_packet = '0; + r_packet[`DF(R, ID)] = id; + r_packet[`DF(R, DATA)] = reading_ids[id].line[beat*`DDR_DATA_BITS+:`DDR_DATA_BITS]; + r_packet[`DF(R, LAST)] = beat == `DDR_BEATS - 1; + if (read_error.exists(address) && beat == read_error[address]) + r_packet[`DF(R, RESP)] = beat % 2 ? 3 : 2; + end + end + if (!b_active && !hold_b) + for (int id = 15; id >= 0; id--) + if (!b_active && writing_ids.exists(id) && writing_ids[id].due <= cycle) begin + b_active = 1; + b_id = id; + b_packet = '0; + b_packet[`DF(B, ID)] = id; + if (write_error.exists(commands[writing_ids[id].key].packet[`DF(REQ, ADDR)])) + b_packet[ + `DF(B, RESP) + ] = write_error[commands[writing_ids[id].key].packet[ + `DF(REQ, ADDR) + ]]; + end + r.valid = !ctl.reset && r_active; + r.bits = r_packet; + b.valid = !ctl.reset && b_active; + b.bits = b_packet; + end +endtask diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/env/env.svh b/arch/src/main/scala/framework/device/ddr/src/main/resources/env/env.svh new file mode 100644 index 000000000..38de213b8 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/env/env.svh @@ -0,0 +1,147 @@ +class env extends uvm_env; + `uvm_component_utils(env) + virtual ddr_control_if ctl; + virtual stream_if #(`DDR_REQ_WIDTH) req[2]; + virtual stream_if #(`DDR_RESP_WIDTH) resp[2]; + virtual stream_if #(`DDR_A_WIDTH) aw, ar; + virtual stream_if #(`DDR_W_WIDTH) w; + virtual stream_if #(`DDR_B_WIDTH) b; + virtual stream_if #(`DDR_R_WIDTH) r; + result_board #(result_item, 1) scoreboard; + chandle model; + typedef struct { + req_t packet; + int client; + bit done; + } command; + typedef struct { + int key, beat, serial; + bit [511:0] line; + } reading; + typedef struct {int key, due, serial;} writing; + typedef struct packed { + bit [511:0] data; + bit [63:0] mask; + } wline; + command commands[int]; + reading reading_ids[int]; + writing writing_ids[int]; + int aw_ids[$], aw_keys[$], aw_serials[$]; + wline w_lines[$]; + wline assembling; + int assembling_beat = 0; + int read_error[u64], write_error[u64]; + bit hold_aw = 0, hold_w = 0, hold_ar = 0, hold_b = 0, hold_r = 0, hold_resp[2] = '{0, 0}; + bit r_active = 0, b_active = 0; + int r_id = 0, b_id = 0, last_r_id = -1, last_r_serial = -1, r_cursor = 0, r_beats = 0; + logic [`DDR_R_WIDTH-1:0] r_packet; + logic [`DDR_B_WIDTH-1:0] b_packet; + int + cycle = 0, + accepted = 0, + returned[2] = '{0, 0}, + cancelled = 0, + peak = 0, + aw_count = 0, + w_bursts = 0, + ar_count = 0; + int + request_stalls = 0, + response_stalls = 0, + aw_stalls = 0, + w_stalls = 0, + interleaves = 0, + error_results = 0; + int invalid_case = -1; + int b_reorders = 0, r_reorders = 0, max_b_serial = -1, max_r_serial = -1; + covergroup behavior with function sample ( + int client, bit writing, int mask_kind, bit error, int occupancy, bit reset_busy + ); + option.per_instance = 1; + clients: coverpoint client {bins all[] = {0, 1};} + direction: coverpoint writing; + masks: coverpoint mask_kind {bins all[] = {[0 : 2]};} + errors: coverpoint error; + occupancy_level: coverpoint occupancy { + bins one = {1}; bins full = {`DDR_SLOTS}; ignore_bins other = {[2 : `DDR_SLOTS - 1]}; + } + reset_cancellation: coverpoint reset_busy; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + + endfunction + function void verify_contract(bit ok, string message); + verification_pkg::evidence::checks++; + if (!ok) `uvm_fatal("DDR_CONTRACT", message) + endfunction + function bit [511:0] pattern(int seed); + bit [511:0] data; + for (int i = 0; i < 8; i++) + data[i*64+:64]=64'h0123456789abcdef^(64'h0101010101010101*(seed+i))^(64'h1<<((seed+i*7)%64)); + return data; + endfunction + function void initialize(u64 address, int seed); + ddr_ref_program(model, address, pattern(seed)); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + model = ddr_ref_create(); + scoreboard = result_board#(result_item, 1)::type_id::create("scoreboard", this); + verify_contract(uvm_config_db#(virtual ddr_control_if)::get(this, "", "ctl", ctl), + "Missing control"); + for (int c = 0; c < 2; c++) begin + verify_contract(uvm_config_db#(virtual stream_if #(`DDR_REQ_WIDTH))::get( + this, "", $sformatf("req%0d", c), req[c]), "Missing request"); + verify_contract(uvm_config_db#(virtual stream_if #(`DDR_RESP_WIDTH))::get( + this, "", $sformatf("resp%0d", c), resp[c]), "Missing response"); + end + verify_contract(uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::get(this, "", "aw", aw), + "Missing AW"); + verify_contract(uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::get(this, "", "ar", ar), + "Missing AR"); + verify_contract(uvm_config_db#(virtual stream_if #(`DDR_W_WIDTH))::get(this, "", "w", w), + "Missing W"); + verify_contract(uvm_config_db#(virtual stream_if #(`DDR_B_WIDTH))::get(this, "", "b", b), + "Missing B"); + verify_contract(uvm_config_db#(virtual stream_if #(`DDR_R_WIDTH))::get(this, "", "r", r), + "Missing R"); + void'(uvm_config_db#(int)::get(this, "", "invalid_case", invalid_case)); + endfunction + function int address_owner(u64 address, bit write); + int found = -1; + foreach (commands[key]) + if (commands[key].packet[ + `DF(REQ, ADDR) + ] == address && commands[key].packet[ + `DF(REQ, WRITE) + ] == write) begin + verify_contract(found == -1, "Ambiguous fixture address"); + found = key; + end + verify_contract(found != -1, "AXI address without accepted line command"); + return found; + endfunction + function int observe_address(logic [`DDR_A_WIDTH-1:0] a, bit write); + int key, id; + verify_contract(!$isunknown(a), "Unknown AXI address payload"); + key = address_owner(a[`DF(A, ADDR)], write); + id = a[`DF(A, ID)]; + verify_contract(id >= `DDR_ID_BASE && id < `DDR_ID_BASE + `DDR_SLOTS, + "AXI ID outside configured namespace"); + verify_contract((id - `DDR_ID_BASE) / `DDR_SLOTS_PER_CLIENT == commands[key].client, + "AXI ID crossed client namespace"); + verify_contract(a[`DF(A, LEN)] == `DDR_BEATS - 1 && a[`DF(A, SIZE)] == $clog2((`DDR_BEAT_BYTES) + ) && a[`DF(A, BURST)] == 1, "Wrong AXI burst geometry"); + verify_contract(a[`DF(A, LOCK)] == 0 && a[`DF(A, CACHE)] == 0 && a[`DF(A, PROT)] == 0 && a[ + `DF(A, QOS)] == 0 && a[`DF(A, REGION)] == 0, "Unexpected AXI attributes"); + return key; + endfunction + function void final_phase(uvm_phase phase); + ddr_ref_destroy(model); + super.final_phase(phase); + endfunction + `include "agent/service.svh" + `include "seq/scenario.svh" +endclass diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/filelist.f b/arch/src/main/scala/framework/device/ddr/src/main/resources/filelist.f new file mode 100644 index 000000000..88f237394 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Bridge/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/control_if.sv b/arch/src/main/scala/framework/device/ddr/src/main/resources/interface.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/ddr/src/main/resources/control_if.sv rename to arch/src/main/scala/framework/device/ddr/src/main/resources/interface.sv diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/package.sv b/arch/src/main/scala/framework/device/ddr/src/main/resources/package.sv new file mode 100644 index 000000000..8ab853857 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/package.sv @@ -0,0 +1,30 @@ +`include "parameters.svh" +`define DF(K, F) `DDR_``K``_``F``_OFFSET +: `DDR_``K``_``F``_WIDTH +package ddr_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function chandle ddr_ref_create(); + import "DPI-C" function void ddr_ref_destroy(input chandle model); + import "DPI-C" function void ddr_ref_program( + input chandle model, + input longint unsigned address, + input bit [511:0] data + ); + import "DPI-C" function int unsigned ddr_ref_read( + input chandle model, + input longint unsigned address, + output bit [511:0] data + ); + import "DPI-C" function int unsigned ddr_ref_write( + input chandle model, + input longint unsigned address, + input bit [511:0] data, + input longint unsigned mask + ); + typedef longint unsigned u64; + typedef bit [`DDR_REQ_WIDTH-1:0] req_t; + `include "agent/item.svh" + `include "env/env.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/seq/scenario.svh b/arch/src/main/scala/framework/device/ddr/src/main/resources/seq/scenario.svh new file mode 100644 index 000000000..9f5d9ef24 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/seq/scenario.svh @@ -0,0 +1,258 @@ +task submit(int client, int id, u64 address, bit write, bit [63:0] mask = '1); + req_t q = '0; + q[`DF(REQ, ID)] = id; + q[`DF(REQ, ADDR)] = address; + q[`DF(REQ, WRITE)] = write; + q[`DF(REQ, DATA)] = pattern(id + client * 17 + int'(address >> 6)); + q[`DF(REQ, MASK)] = mask; + @(negedge ctl.clock); + req[client].bits = q; + req[client].valid = 1; + do @(req[client].sample); while (!req[client].sample.ready); + @(negedge ctl.clock); + req[client].valid = 0; +endtask +task batch(u64 base, bit write); + for (int c = 0; c < 2; c++) + for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) + initialize(base + c * 4096 + i * 64, i + c * 7 + int'(base >> 6)); + fork + begin + for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) + submit(0, 12'h800 + i, base + i * 64, write, + i == 0 ? 0 : i == 1 ? '1 : i == 2 ? 64'h5555555555555555 : 64'haaaaaaaaaaaaaaaa); + end + begin + for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) + submit(1, 12'h800 + i, base + 4096 + i * 64, write); + end + join +endtask +task drain(); + wait (commands.num() == 0); + @(ctl.sample); + verify_contract( + !reading_ids.num()&&!writing_ids.num()&&!aw_ids.size()&&!w_lines.size()&&!assembling_beat&&!r_active&&!b_active, + "Backend did not drain"); +endtask +task joint_reset(); + @(negedge ctl.clock); + ctl.reset = 1; + req[0].valid = 0; + req[1].valid = 0; + repeat (3) @(ctl.sample); + behavior.sample(0, 0, 0, 0, commands.num(), commands.num() != 0); + cancelled += commands.num(); + commands.delete(); + reading_ids.delete(); + writing_ids.delete(); + aw_ids.delete(); + aw_keys.delete(); + aw_serials.delete(); + w_lines.delete(); + scoreboard.cancel_pending(); + assembling = '0; + assembling_beat = 0; + r_active = 0; + b_active = 0; + @(negedge ctl.clock); + ctl.reset = 0; +endtask +function int finished_commands(); + int count = 0; + foreach (commands[key]) if (commands[key].done) count++; + return count; +endfunction +task invalid_scenario(); + int id, beats; + bit aw_seen; + if (invalid_case == 0) begin + submit(0, 0, 'h10001, 0); + end else if (invalid_case == 1) begin + @(negedge ctl.clock); + r.bits = '0; + r.bits[`DF(R, ID)] = 0; + r.valid = 1; + end else if (invalid_case == 2 || invalid_case == 6) begin + aw.ready = 1; + w.ready = invalid_case == 6; + submit(0, 0, 'h10000, 1); + beats = 0; + aw_seen = 0; + do begin + @(ctl.sample); + if (aw.sample.valid && aw.sample.ready) begin + id = aw.sample.bits[`DF(A, ID)]; + aw_seen = 1; + end + if (w.sample.valid && w.sample.ready) beats++; + end while (!aw_seen || (invalid_case == 6 && beats != `DDR_BEATS)); + @(negedge ctl.clock); + b.bits = '0; + b.bits[`DF(B, ID)] = id; + b.bits[`DF(B, RESP)] = invalid_case == 6 ? 1 : 0; + b.valid = 1; + end else begin + ar.ready = 1; + submit(0, 0, 'h10000, 0); + do @(ctl.sample); while (!ar.sample.valid || !ar.sample.ready); + id = ar.sample.bits[`DF(A, ID)]; + for (int beat = 0; beat < (invalid_case == 4 ? `DDR_BEATS : 1); beat++) begin + @(negedge ctl.clock); + r.bits = '0; + r.bits[`DF(R, ID)] = id; + r.bits[`DF(R, LAST)] = invalid_case == 3; + r.bits[`DF(R, RESP)] = invalid_case == 5 ? 1 : 0; + r.valid = 1; + do @(r.sample); while (!r.sample.ready); + end + end + repeat (8) @(ctl.sample); + `uvm_fatal("MISSING_ASSERTION", "Invalid protocol did not trigger expected DUT assertion") +endtask +task execute(); + int before_aw, before_w, before_ar, before_r; + int initial_returned; + ctl.reset = 1; + req[0].valid = 0; + req[1].valid = 0; + req[0].bits = '0; + req[1].bits = '0; + resp[0].ready = 0; + resp[1].ready = 0; + aw.ready = 0; + ar.ready = 0; + w.ready = 0; + b.valid = 0; + b.bits = '0; + r.valid = 0; + r.bits = '0; + repeat (5) @(negedge ctl.clock); + ctl.reset = 0; + if (invalid_case >= 0) begin + invalid_scenario(); + return; + end + fork + service(); + join_none + // W must progress even when AW has never been accepted; all eight slots stay live until B. + hold_aw = 1; + hold_b = 1; + hold_resp[0] = 1; + batch('h10000, 1); + wait (w_bursts == `DDR_SLOTS); + verify_contract(aw_count == 0 && returned[0] == 0 && returned[1] == 0, + "W-before-AW or B-delayed behavior failed"); + hold_aw = 0; + wait (aw_count == `DDR_SLOTS); + initialize('h18000, 99); + fork + submit(0, 12'h800, 'h18000, 1); + begin + hold_b = 0; + wait (returned[1] == `DDR_SLOTS_PER_CLIENT); + verify_contract(returned[0] == 0, "Client0 backpressure leaked"); + hold_resp[0] = 0; + end + join + drain(); + for (int c = 0; c < 2; c++) + for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) + submit(c, 12'h900 + i, 'h10000 + c * 4096 + i * 64, 0); + drain(); + // Conversely allow addresses to run ahead of every W beat. + before_aw = aw_count; + before_w = w_bursts; + hold_w = 1; + batch('h20000, 1); + wait (aw_count == before_aw + `DDR_SLOTS); + verify_contract(w_bursts == before_w, "AW-before-W stalled ordering failed"); + hold_w = 0; + drain(); + // Concurrent reads and writes: hold B until all independent reads return. + hold_b = 1; + initial_returned = returned[0] + returned[1]; + for (int c = 0; c < 2; c++) + for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) initialize('h30000 + c * 4096 + i * 64, 30 + i); + fork + begin + for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) + submit(0, 12'ha00 + i, 'h30000 + i * 64, i[0]); + end + begin + for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) + submit(1, 12'ha00 + i, 'h31000 + i * 64, i[0]); + end + join + wait (returned[0] + returned[1] == initial_returned + 2 * ((`DDR_SLOTS_PER_CLIENT + 1) / 2)); + hold_b = 0; + drain(); + for (int i = 0; i < 4; i++) begin + initialize('h40000 + i * 64, 40 + i); + read_error['h40000+i*64] = i % `DDR_BEATS; + submit(i % 2, 12'hb00 + i, 'h40000 + i * 64, 0); + end + initialize('h41000, 44); + write_error['h41000] = 2; + submit(1, 12'hb10, 'h41000, 1); + initialize('h41040, 45); + write_error['h41040] = 3; + submit(0, 12'hb10, 'h41040, 1); + drain(); + before_ar = ar_count; + hold_r = 1; + batch('h48000, 0); + wait (ar_count == before_ar + `DDR_SLOTS); + r_cursor = `DDR_ID_BASE; + hold_r = 0; + drain(); + // Each byte strobe is independently checked, then read back through the reference. + for (int i = 0; i < 64; i++) begin + initialize('h50000 + i * 64, 50 + i); + submit(i % 2, 12'hc00 + i, 'h50000 + i * 64, 1, 64'h1 << i); + drain(); + submit(i % 2, 12'hd00 + i, 'h50000 + i * 64, 0); + drain(); + end + // Coordinated reset cancels queued AW-only, W-only and stalled line completions. + before_aw = aw_count; + hold_w = 1; + batch('h60000, 1); + wait (aw_count == before_aw + `DDR_SLOTS); + joint_reset(); + hold_w = 0; + before_w = w_bursts; + hold_aw = 1; + batch('h70000, 1); + wait (w_bursts == before_w + `DDR_SLOTS); + joint_reset(); + hold_aw = 0; + before_ar = ar_count; + before_r = r_beats; + hold_r = 1; + batch('h78000, 0); + wait (ar_count == before_ar + `DDR_SLOTS); + hold_r = 0; + wait (r_beats >= before_r + 1); + joint_reset(); + hold_resp[0] = 1; + hold_resp[1] = 1; + batch('h80000, 0); + do @(ctl.sample); while (finished_commands() != `DDR_SLOTS); + joint_reset(); + hold_resp[0] = 0; + hold_resp[1] = 0; + batch('h90000, 0); + drain(); + verify_contract( + peak==`DDR_SLOTS&&(`DDR_BEATS==1?interleaves==0:interleaves>0)&&b_reorders>0&&r_reorders>0&&request_stalls>0&&response_stalls>0&&aw_stalls>0&&w_stalls>0&&error_results==6&&cancelled==4*`DDR_SLOTS, + "Missing concurrency, stalls, errors or reset coverage"); + verify_contract(accepted == returned[0] + returned[1] + cancelled, "Lost line response"); + `uvm_info( + "DDR_PASS", + $sformatf( + "accepted=%0d returned=%0d/%0d cancelled=%0d peak=%0d errors=%0d readInterleave=%0d R/Breorder=%0d/%0d stalls req/resp/AW/W=%0d/%0d/%0d/%0d; full64B payload/mask/tag and all queues drained", + accepted, returned[0], returned[1], cancelled, peak, error_results, interleaves, + r_reorders, b_reorders, request_stalls, response_stalls, aw_stalls, w_stalls), UVM_LOW) +endtask diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/suite.toml b/arch/src/main/scala/framework/device/ddr/src/main/resources/suite.toml new file mode 100644 index 000000000..1af7a2050 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/suite.toml @@ -0,0 +1,139 @@ +[model] +manifest="src/csrc/Cargo.toml" +library="libddr_reference" + +[[tests]] +functional_groups = ["ddr_pkg::env::behavior"] +name = "bridge" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "bad_alignment" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+invalid_case=0"] +assertion = "DDR line request must be 64 byte aligned" + +[[tests]] +functional_groups = [] +name = "bad_early_b" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+invalid_case=2"] +assertion = "DDR received B before AW and complete W or for inactive ID" + +[[tests]] +functional_groups = [] +name = "bad_early_rlast" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+invalid_case=3"] +assertion = "DDR AXI RLAST does not match line beat count" + +[[tests]] +functional_groups = [] +name = "bad_late_rlast" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+invalid_case=4"] +assertion = "DDR AXI RLAST does not match line beat count" + +[[tests]] +functional_groups = [] +name = "bad_read_exokay" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+invalid_case=5"] +assertion = "DDR nonexclusive read received EXOKAY" + +[[tests]] +functional_groups = [] +name = "bad_rid" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+invalid_case=1"] +assertion = "DDR received R for an inactive AXI ID" + +[[tests]] +functional_groups = [] +name = "bad_write_exokay" +top = "bridge_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+invalid_case=6"] +assertion = "DDR nonexclusive write received EXOKAY" diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/tb.sv b/arch/src/main/scala/framework/device/ddr/src/main/resources/tb.sv new file mode 100644 index 000000000..d026fb891 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/tb.sv @@ -0,0 +1,75 @@ +`include "parameters.svh" +`ifndef DDR_TB +`define DDR_TB bridge_tb +`endif +module `DDR_TB; + import uvm_pkg::*; + import ddr_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + ddr_control_if ctl (clock); + stream_if #(`DDR_REQ_WIDTH) + req0 ( + clock, + ctl.reset + ), + req1 ( + clock, + ctl.reset + ); + stream_if #(`DDR_RESP_WIDTH) + resp0 ( + clock, + ctl.reset + ), + resp1 ( + clock, + ctl.reset + ); + stream_if #(`DDR_A_WIDTH) + aw ( + clock, + ctl.reset + ), + ar ( + clock, + ctl.reset + ); + stream_if #(`DDR_W_WIDTH) w ( + clock, + ctl.reset + ); + stream_if #(`DDR_B_WIDTH) b ( + clock, + ctl.reset + ); + stream_if #(`DDR_R_WIDTH) r ( + clock, + ctl.reset + ); + Bridge dut ( + .clock(clock), + .reset(ctl.reset), + `include "ports.svh" + ); + initial begin + int invalid_case; + uvm_config_db#(virtual ddr_control_if)::set(null, "uvm_test_top.env", "ctl", ctl); + uvm_config_db#(virtual stream_if #(`DDR_REQ_WIDTH))::set(null, "uvm_test_top.env", "req0", + req0); + uvm_config_db#(virtual stream_if #(`DDR_REQ_WIDTH))::set(null, "uvm_test_top.env", "req1", + req1); + uvm_config_db#(virtual stream_if #(`DDR_RESP_WIDTH))::set(null, "uvm_test_top.env", "resp0", + resp0); + uvm_config_db#(virtual stream_if #(`DDR_RESP_WIDTH))::set(null, "uvm_test_top.env", "resp1", + resp1); + uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::set(null, "uvm_test_top.env", "aw", aw); + uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::set(null, "uvm_test_top.env", "ar", ar); + uvm_config_db#(virtual stream_if #(`DDR_W_WIDTH))::set(null, "uvm_test_top.env", "w", w); + uvm_config_db#(virtual stream_if #(`DDR_B_WIDTH))::set(null, "uvm_test_top.env", "b", b); + uvm_config_db#(virtual stream_if #(`DDR_R_WIDTH))::set(null, "uvm_test_top.env", "r", r); + if ($value$plusargs("invalid_case=%d", invalid_case)) + uvm_config_db#(int)::set(null, "uvm_test_top.env", "invalid_case", invalid_case); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/device/ddr/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/device/ddr/src/main/resources/tests/test.svh new file mode 100644 index 000000000..398043c00 --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/resources/tests/test.svh @@ -0,0 +1,14 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + test_env.execute(); + endtask +endclass diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Bridge.scala b/arch/src/main/scala/framework/device/ddr/src/main/scala/Bridge.scala similarity index 89% rename from arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Bridge.scala rename to arch/src/main/scala/framework/device/ddr/src/main/scala/Bridge.scala index eca9f127b..4fa2b031c 100644 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Bridge.scala +++ b/arch/src/main/scala/framework/device/ddr/src/main/scala/Bridge.scala @@ -1,22 +1,38 @@ -package memcore.memory.ddr +package framework.device.ddr -import memcore.memory.queue.Queue +import framework.lib.arbiter.{RRArbiter} +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.device.ddr.{Params => ModuleParam} + +import framework.lib.queue.Queue import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.axi4 -import memcore.bus.chi.snf.{LineRequest, LineResponse} +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} /** Joint-reset transport: AXI slave and bridge must discard old transactions together. */ @instantiable -class Bridge(p: Params) extends Module { +class Bridge(b: ModuleParam) extends Module { + val p = b @public val io = IO(new Bundle { - val request = Vec(p.clients, Flipped(Decoupled(new LineRequest(p.line)))) - val response = Vec(p.clients, Decoupled(new LineResponse(p.line))) - val axi = new axi4.Port(p.axi) + + val request = Vec( + p.clients, + Flipped(Decoupled(new LineRequest(p.line))) + ) + + val response = Vec( + p.clients, + Decoupled(new LineResponse(p.line)) + ) + + val axi = new axi4.Port(p.axi) + val outstanding = Output(UInt(log2Ceil(p.slots + 1).W)) }) @@ -35,18 +51,12 @@ class Bridge(p: Params) extends Module { val complete = RegInit(VecInit(Seq.fill(p.slots)(false.B))) val errors = RegInit(VecInit(Seq.fill(p.slots)(false.B))) val readBeat = RegInit(VecInit(Seq.fill(p.slots)(0.U(beatBits.W)))) - val awQueue = Module(new Queue(UInt(indexBits.W), p.slots)) - val wQueue = Module(new Queue(UInt(indexBits.W), p.slots)) - val arb = Module(new RRArbiter(UInt(indexBits.W), p.clients) { + val awQueue = Instantiate(new Queue(UInt(indexBits.W), p.slots, false, false)) - override lazy val lastGrant = { - val last = RegInit(0.U(math.max(1, log2Ceil(p.clients)).W)) - when(io.out.fire)(last := io.chosen) - last - } + val wQueue = Instantiate(new Queue(UInt(indexBits.W), p.slots, false, false)) - }) + val arb = Instantiate(new RRArbiter(UInt(indexBits.W), p.clients)) for (c <- 0 until p.clients) { val indices = c * p.slotsPerClient until (c + 1) * p.slotsPerClient diff --git a/arch/src/main/scala/framework/device/ddr/src/main/scala/Emit.scala b/arch/src/main/scala/framework/device/ddr/src/main/scala/Emit.scala new file mode 100644 index 000000000..928845fcd --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/scala/Emit.scala @@ -0,0 +1,116 @@ +package framework.device.ddr + +import chisel3._ +import java.nio.file.{Files, Paths} +import framework.ip.bus.axi4 +import framework.ip.bus.chi +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + + val p = Params( + upickle.default.read[chi.Params](values("line")), + values("clients").num.toInt, + values("slotsPerClient").num.toInt, + values("dataBits").num.toInt, + values("idBits").num.toInt, + values("idBase").num.toInt, + BigInt(values("base").str), + BigInt(values("bytes").str), + BigInt(values("model").str) + ) + + Files.createDirectories(Paths.get(directory)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Bridge(p), + args = Array("--target-dir", s"$directory/Bridge", "--split-verilog") + ) + + val defs = scala.collection.mutable.ArrayBuffer[String]( + s"`define DDR_DATA_BITS ${p.dataBits}", + s"`define DDR_BEAT_BYTES ${p.axi.bytes}", + s"`define DDR_BEATS ${p.beats}", + s"`define DDR_SLOTS ${p.slots}", + s"`define DDR_SLOTS_PER_CLIENT ${p.slotsPerClient}", + s"`define DDR_ID_BASE ${p.idBase}" + ) + + val ports = scala.collection.mutable.ArrayBuffer[String]() + + def channel( + kind: String, + prefix: String, + signal: String, + bundle: Bundle, + define: Boolean = true + ): Unit = { + var offset = 0 + for ((name, field) <- bundle.elements.toSeq) { + val width = field.getWidth + if (define) { + defs += s"`define DDR_${kind}_${name.toUpperCase}_OFFSET $offset" + defs += s"`define DDR_${kind}_${name.toUpperCase}_WIDTH $width" + } + ports += s".${prefix}_bits_$name($signal.bits[$offset +: $width])" + offset += width + } + if (define) defs += s"`define DDR_${kind}_WIDTH $offset" + ports += s".${prefix}_valid($signal.valid)" + ports += s".${prefix}_ready($signal.ready)" + } + + for (c <- 0 until p.clients) { + channel( + "REQ", + s"io_request_$c", + s"req$c", + new LineRequest(p.line), + c == 0 + ) + channel( + "RESP", + s"io_response_$c", + s"resp$c", + new LineResponse(p.line), + c == 0 + ) + } + channel( + "A", + "io_axi_aw", + "aw", + new axi4.Address(p.axi) + ) + channel( + "A", + "io_axi_ar", + "ar", + new axi4.Address(p.axi), + false + ) + channel( + "W", + "io_axi_w", + "w", + new axi4.WriteData(p.axi) + ) + channel( + "B", + "io_axi_b", + "b", + new axi4.WriteResponse(p.axi) + ) + channel( + "R", + "io_axi_r", + "r", + new axi4.ReadData(p.axi) + ) + Files.writeString( + Paths.get(s"$directory/parameters.svh"), + "`ifndef DDR_CONFIG_SVH\n`define DDR_CONFIG_SVH\n" + defs.mkString("\n") + "\n`endif\n" + ) + Files.writeString(Paths.get(s"$directory/ports.svh"), ports.mkString(",\n") + "\n") +} diff --git a/arch/src/main/scala/framework/device/ddr/src/main/scala/Params.scala b/arch/src/main/scala/framework/device/ddr/src/main/scala/Params.scala new file mode 100644 index 000000000..e061af72e --- /dev/null +++ b/arch/src/main/scala/framework/device/ddr/src/main/scala/Params.scala @@ -0,0 +1,47 @@ +package framework.device.ddr + +import buckyball.config.DdrHardware +import scala.jdk.CollectionConverters._ + +import framework.ip.bus.chi +import framework.ip.bus.axi4 +import upickle.default.{macroRW, ReadWriter} + +case class Params( + line: chi.Params = chi.Params(), + clients: Int = 2, + slotsPerClient: Int = 4, + dataBits: Int = 128, + idBits: Int = 4, + idBase: Int = 0, + base: BigInt = BigInt("80000000", 16), + bytes: BigInt = BigInt("20000000", 16), + model: BigInt = 0) { + require(line.addressBits >= 6) + require(clients > 0 && slotsPerClient > 0) + require(Set(64, 128, 256, 512).contains(dataBits)) + val slots = clients * slotsPerClient + require(slots > 1 && idBase >= 0 && BigInt(idBase) + slots <= (BigInt(1) << idBits)) + require(model <= bytes) + val axi = axi4.Params(line.addressBits, dataBits, idBits) + val beats = 512 / dataBits +} + +object Params { + + def fromProto(p: DdrHardware): Params = + Params( + chi.Params.fromProto(p.getLine), + p.getClients, + p.getSlotsPerClient, + p.getDataBits, + p.getIdBits, + p.getIdBase, + BigInt(java.lang.Long.toUnsignedString(p.getBase)), + BigInt(java.lang.Long.toUnsignedString(p.getBytes)), + BigInt(java.lang.Long.toUnsignedString(p.getModel)) + ) + + implicit val chiRW: ReadWriter[chi.Params] = macroRW + implicit val rw: ReadWriter[Params] = macroRW +} diff --git a/arch/src/main/scala/framework/device/dlink/basedevice.scala b/arch/src/main/scala/framework/device/dlink/basedevice.scala new file mode 100644 index 000000000..f072c5eaf --- /dev/null +++ b/arch/src/main/scala/framework/device/dlink/basedevice.scala @@ -0,0 +1,16 @@ +package framework.device.dlink + +import framework.ip.bus.axi4.{Params => ModuleParam} + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} + +trait HasDLink { def dlink: DLinkIO } + +@instantiable +abstract class BaseDevice(b: ModuleParam) extends Module with HasDLink { + val linkParams = b + + @public + val dlink = IO(new DLinkIO(b)) +} diff --git a/arch/src/main/scala/framework/device/dlink/build.sc b/arch/src/main/scala/framework/device/dlink/build.sc new file mode 100644 index 000000000..600bf9b00 --- /dev/null +++ b/arch/src/main/scala/framework/device/dlink/build.sc @@ -0,0 +1,12 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("basedevice.scala")), + PathRef(millSourcePath / os.RelPath("dlink.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/device/dlink/dlink.scala b/arch/src/main/scala/framework/device/dlink/dlink.scala new file mode 100644 index 000000000..8bb1af6bc --- /dev/null +++ b/arch/src/main/scala/framework/device/dlink/dlink.scala @@ -0,0 +1,13 @@ +package framework.device.dlink + +import framework.ip.bus.axi4.{Params => ModuleParam} + +import chisel3._ +import framework.ip.bus.axi4 + +/** Unified memory access; address decoding belongs to the device. */ +class DLinkIO(b: ModuleParam) extends Bundle { + val p = b + + val axi = new framework.ip.bus.axi4.Port(b) +} diff --git a/arch/src/main/scala/framework/device/peripherals/Controller.scala b/arch/src/main/scala/framework/device/peripherals/Controller.scala new file mode 100644 index 000000000..74b7faba8 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/Controller.scala @@ -0,0 +1,176 @@ +package framework.device.peripherals + +import chisel3.experimental.hierarchy.Instance + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.device.peripherals.clint.Clint +import framework.device.peripherals.plic.Plic +import framework.device.peripherals.scu.SystemControl +import framework.ip.bus.axi4 +import framework.device.configs.DeviceParam +import framework.device.peripherals.plic.configs.PlicParams +import framework.core.clink.CoreInterrupts +import framework.core.memory.access.{AccessParams, UncachedRequest, UncachedResponse} + +/** One register access presented for a single cycle; data is right-aligned at `addr`. */ +class DeviceAccess extends Bundle { + val addr = UInt(64.W) + val write = Bool() + val size = UInt(3.W) + val data = UInt(64.W) +} + +/** Register-block interface: `read` and `error` answer the access presented in the same cycle. */ +class DevicePort extends Bundle { + val access = Flipped(Valid(new DeviceAccess)) + val read = Output(UInt(64.W)) + val error = Output(Bool()) +} + +/** Device interrupt events and the resulting per-hart interrupt notifications. */ +class HardwareInterrupt(p: PlicParams) extends Bundle { + val sourceCount = p.sources + val hartCount = p.hartIds.size + val sources = Input(UInt(sourceCount.W)) + val harts = Output(Vec(hartCount, new CoreInterrupts)) +} + +/** AXI4 register endpoint: single transfers up to 64 bits, with full transfer strobes or a zero-strobe no-op. */ +@instantiable +class Controller(device: DeviceParam, ap: axi4.Params, cp: AccessParams) extends Module { + val hartIds = device.plic.hartIds + + @public + val io = IO(new Bundle { + val axi = Flipped(new axi4.Port(ap)) + val hardwareInterrupt = new HardwareInterrupt(device.plic) + }) + + require(ap.dataBits >= 64 && cp.tagBits >= ap.idBits) + val request = Wire(Decoupled(new UncachedRequest(cp))) + val response = Wire(Decoupled(new UncachedResponse(cp))) + val aw = Reg(new axi4.Address(ap)) + val ar = Reg(new axi4.Address(ap)) + val wd = Reg(new axi4.WriteData(ap)) + val awHeld = RegInit(false.B) + val arHeld = RegInit(false.B) + val wHeld = RegInit(false.B) + io.axi.aw.ready := !awHeld + io.axi.ar.ready := !arHeld + io.axi.w.ready := !wHeld + when(io.axi.aw.fire) { + assert(io.axi.aw.bits.len === 0.U && !io.axi.aw.bits.lock, "Peripheral AXI accepts single nonexclusive transfers") + aw := io.axi.aw.bits; awHeld := true.B + } + when(io.axi.ar.fire) { + assert(io.axi.ar.bits.len === 0.U && !io.axi.ar.bits.lock, "Peripheral AXI accepts single nonexclusive transfers") + ar := io.axi.ar.bits; arHeld := true.B + } + when(io.axi.w.fire) { + assert(io.axi.w.bits.last, "Peripheral AXI write must contain one beat") + wd := io.axi.w.bits; wHeld := true.B + } + + val bPending = RegInit(false.B) + val rPending = RegInit(false.B) + val bAnswer = Reg(new axi4.WriteResponse(ap)) + val rAnswer = Reg(new axi4.ReadData(ap)) + io.axi.b.valid := bPending + io.axi.b.bits := bAnswer + io.axi.r.valid := rPending + io.axi.r.bits := rAnswer + when(io.axi.b.fire)(bPending := false.B) + when(io.axi.r.fire)(rPending := false.B) + + val active = RegInit(false.B) + val preferWrite = RegInit(false.B) + val canWrite = awHeld && wHeld && !bPending + val canRead = arHeld && !rPending + val writing = canWrite && (!canRead || preferWrite) + val address = Mux(writing, aw, ar) + val lane = address.addr(log2Ceil(ap.bytes) - 1, 0) + val expectedMask = (((1.U((ap.bytes + 1).W) << (1.U << address.size)) - 1.U) << lane)(ap.bytes - 1, 0) + val noWrite = writing && wd.strb === 0.U + val invalidAccess = address.size > 3.U || (address.addr & ((1.U(ap.addressBits.W) << address.size) - 1.U)).orR || + (writing && !noWrite && wd.strb =/= expectedMask) + request.valid := !active && (canWrite || canRead) + request.bits.addr := address.addr + request.bits.tag := address.id + request.bits.size := address.size + request.bits.write := writing + request.bits.data := (wd.data >> (lane << 3))(63, 0) + val responseWrite = Reg(Bool()) + val responseId = Reg(UInt(ap.idBits.W)) + val responseLane = Reg(UInt(log2Ceil(ap.bytes).W)) + when(request.fire) { + active := true.B + preferWrite := !writing + responseWrite := writing + responseId := address.id + responseLane := lane + when(writing) { awHeld := false.B; wHeld := false.B }.otherwise(arHeld := false.B) + } + response.ready := active && Mux(responseWrite, !bPending, !rPending) + when(response.fire) { + active := false.B + when(responseWrite) { + bPending := true.B + bAnswer.id := responseId + bAnswer.resp := Mux(response.bits.error, 2.U, 0.U) + }.otherwise { + rPending := true.B + rAnswer.id := responseId + rAnswer.data := response.bits.data << (responseLane << 3) + rAnswer.resp := Mux(response.bits.error, 2.U, 0.U) + rAnswer.last := true.B + } + } + + val clint: Instance[Clint] = Instantiate(new Clint(device.clint)) + val plic: Instance[Plic] = Instantiate(new Plic(device.plic)) + val scu: Instance[SystemControl] = Instantiate(new SystemControl(device.scu, cp)) + plic.io.sources := io.hardwareInterrupt.sources + for (i <- hartIds.indices) { + io.hardwareInterrupt.harts(i).timer := clint.io.mtip(i) + io.hardwareInterrupt.harts(i).software := clint.io.msip(i) + io.hardwareInterrupt.harts(i).external := plic.io.meip(i) + io.hardwareInterrupt.harts(i).supervisorExternal := plic.io.seip(i) + } + + val addr = request.bits.addr + val inClint = addr >= device.clint.base.U && addr < (device.clint.base + device.clint.bytes).U + val inPlic = addr >= device.plic.base.U && addr < (device.plic.base + device.plic.bytes).U + val inScu = addr >= device.scu.baseAddress.U && addr < (device.scu.baseAddress + device.scu.totalSizeBytes).U + val pending = RegInit(false.B) + val selectedScu = Reg(Bool()) + val answer = Reg(new UncachedResponse(cp)) + + request.ready := !pending && Mux(inScu && !invalidAccess && !noWrite, scu.io.request.ready, true.B) + scu.io.request.valid := request.valid && !pending && inScu && !invalidAccess && !noWrite + scu.io.request.bits := request.bits + for (port <- Seq(clint.io.port, plic.io.port)) { + port.access.bits.addr := addr + port.access.bits.write := request.bits.write + port.access.bits.size := request.bits.size + port.access.bits.data := request.bits.data + } + clint.io.port.access.valid := request.fire && inClint && !invalidAccess && !noWrite + plic.io.port.access.valid := request.fire && inPlic && !invalidAccess && !noWrite + when(request.fire) { + pending := true.B + selectedScu := inScu && !invalidAccess && !noWrite + answer.tag := request.bits.tag + answer.data := Mux(request.bits.write, 0.U, Mux(inClint, clint.io.port.read, plic.io.port.read)) + answer.error := invalidAccess || (!noWrite && Mux( + inClint, + clint.io.port.error, + Mux(inPlic, plic.io.port.error, !inScu) + )) + } + response.valid := pending && Mux(selectedScu, scu.io.response.valid, true.B) + response.bits := Mux(selectedScu, scu.io.response.bits, answer) + scu.io.response.ready := pending && selectedScu && response.ready + when(response.fire)(pending := false.B) +} diff --git a/arch/src/main/scala/framework/device/peripherals/bootrom/BootRom.scala b/arch/src/main/scala/framework/device/peripherals/bootrom/BootRom.scala new file mode 100644 index 000000000..d999749c9 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/bootrom/BootRom.scala @@ -0,0 +1,79 @@ +package framework.device.peripherals.bootrom + +import chisel3._ +import framework.device.peripherals.bootrom.configs.BootRomParams +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import chisel3.util._ +import framework.ip.bus.chi.{Params => ChiParams} +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} + +/** + * The BootROM image (`bootrom/bootrom.rv64.img`, built from `bootrom.S`) with its two config dwords + * filled: the hart count at offset 0x8 and the DRAM entry at offset 0x10. + */ +object BootRom { + + def apply( + base: BigInt, + bytes: BigInt, + hartCount: Int, + entry: BigInt + ): BootRomParams = { + val stream = getClass.getResourceAsStream("/bootrom/bootrom.rv64.img") + require(stream != null, "missing BootROM resource bootrom/bootrom.rv64.img") + val image = + try stream.readAllBytes().toSeq + finally stream.close() + require(image.size >= 24 && image.slice(8, 24).forall(_ == 0), "BootROM image lacks its zeroed config dwords") + def dword(value: BigInt): Seq[Byte] = (0 until 8).map(i => ((value >> (8 * i)) & 0xff).toByte) + BootRomParams(base, bytes, image.take(8) ++ dword(hartCount) ++ dword(entry) ++ image.drop(24)) + } + +} + +/** + * Serves read-only BootROM cache lines. Address routing belongs to the enclosing design. The + * ROM is a cacheable, executable, read-only region, so fetches and loads reach it as line reads; + * a write to a ROM line can only come from a faulty requester and answers with an error. + */ +@instantiable +class BootRomLines(p: BootRomParams, chi: ChiParams) extends Module { + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new LineRequest(chi))) + val response = Decoupled(new LineResponse(chi)) + }) + + val lines = p.image.padTo(((p.image.size + 63) / 64) * 64, 0.toByte).grouped(64).map { line => + line.zipWithIndex.map { case (byte, i) => BigInt(byte & 0xff) << (8 * i) }.sum + }.toSeq + + val rom = VecInit(lines.map(_.U(512.W))) + + val offset = io.request.bits.addr - p.base.U + val inRom = io.request.bits.addr >= p.base.U && io.request.bits.addr < (p.base + p.bytes).U + val line = offset >> 6 + val pending = RegInit(false.B) + val id = Reg(UInt(chi.txnIdBits.W)) + val data = Reg(UInt(512.W)) + val error = Reg(Bool()) + + io.request.ready := !pending + when(io.request.fire) { + pending := true.B + id := io.request.bits.id + error := !inRom || io.request.bits.write || io.request.bits.addr(5, 0).orR + data := Mux( + inRom && !io.request.bits.write && line < lines.size.U, + rom(line(log2Ceil(lines.size.max(2)) - 1, 0)), + 0.U + ) + } + + io.response.valid := pending + io.response.bits.id := id + io.response.bits.data := Mux(error, 0.U, data) + io.response.bits.error := error + when(io.response.fire)(pending := false.B) +} diff --git a/arch/src/main/scala/framework/device/peripherals/bootrom/configs/Parameters.scala b/arch/src/main/scala/framework/device/peripherals/bootrom/configs/Parameters.scala new file mode 100644 index 000000000..dd222a127 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/bootrom/configs/Parameters.scala @@ -0,0 +1,21 @@ +package framework.device.peripherals.bootrom.configs + +import buckyball.config.BootRomHardware +import scala.jdk.CollectionConverters._ + +import upickle.default.{macroRW, ReadWriter} + +case class BootRomParams(base: BigInt, bytes: BigInt, image: Seq[Byte]) { + require(base % 64 == 0 && bytes % 64 == 0 && image.size <= bytes, "BootROM must be whole cache lines") +} + +object BootRomParams { + + def fromProto(p: BootRomHardware): BootRomParams = BootRomParams( + BigInt(java.lang.Long.toUnsignedString(p.getBase)), + BigInt(java.lang.Long.toUnsignedString(p.getBytes)), + p.getImageList.asScala.map(_.toByte).toSeq + ) + + implicit val rw: ReadWriter[BootRomParams] = macroRW +} diff --git a/arch/src/main/scala/framework/device/peripherals/bootrom/configs/default.toml b/arch/src/main/scala/framework/device/peripherals/bootrom/configs/default.toml new file mode 100644 index 000000000..fa2e61cc6 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/bootrom/configs/default.toml @@ -0,0 +1,4 @@ +base = 0x10000 +bytes = 0x10000 +image = "arch/src/main/resources/bootrom/bootrom.rv64.img" +entry = 0x80000000 diff --git a/arch/src/main/scala/framework/device/peripherals/build.sc b/arch/src/main/scala/framework/device/peripherals/build.sc new file mode 100644 index 000000000..7292c6014 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/build.sc @@ -0,0 +1,20 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("Controller.scala")), + PathRef(millSourcePath / os.RelPath("bootrom/BootRom.scala")), + PathRef(millSourcePath / os.RelPath("bootrom/configs/Parameters.scala")), + PathRef(millSourcePath / os.RelPath("clint/Clint.scala")), + PathRef(millSourcePath / os.RelPath("clint/configs/Parameters.scala")), + PathRef(millSourcePath / os.RelPath("plic/Plic.scala")), + PathRef(millSourcePath / os.RelPath("plic/configs/Parameters.scala")), + PathRef(millSourcePath / os.RelPath("scu/SCU.scala")), + PathRef(millSourcePath / os.RelPath("scu/SystemControl.scala")), + PathRef(millSourcePath / os.RelPath("scu/configs/Parameters.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/device/peripherals/clint/Clint.scala b/arch/src/main/scala/framework/device/peripherals/clint/Clint.scala new file mode 100644 index 000000000..016b2c174 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/clint/Clint.scala @@ -0,0 +1,69 @@ +package framework.device.peripherals.clint + +import chisel3._ +import framework.device.peripherals.clint.configs.ClintParams +import framework.device.peripherals.DevicePort +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import chisel3.util._ + +/** `mtime` advances once every `tickCycles` clock cycles; the device tree timebase must match. */ +/** Standard CLINT layout: msip at 4*hart, mtimecmp at 0x4000 + 8*hart, mtime at 0xbff8. */ +@instantiable +class Clint(p: ClintParams) extends Module { + val hartIds = p.hartIds + require(hartIds.distinct.size == hartIds.size && hartIds.max < 4095) + val n = hartIds.size + + @public + val io = IO(new Bundle { + val port = new DevicePort + val msip = Output(Vec(n, Bool())) + val mtip = Output(Vec(n, Bool())) + }) + + val msip = RegInit(VecInit(Seq.fill(n)(false.B))) + val mtimecmp = RegInit(VecInit(Seq.fill(n)(~0.U(64.W)))) + val mtime = RegInit(0.U(64.W)) + val ticks = RegInit(0.U(log2Ceil(p.tickCycles + 1).W)) + + val access = io.port.access.bits + val offset = access.addr - p.base.U + val write = io.port.access.valid && access.write + val word = access.size === 2.U + val double = access.size === 3.U + + // Each register answers a 32-bit access to either half, and a 64-bit access to its base. + val hits = Wire(Vec(2 * n + 1, Bool())) + val reads = Wire(Vec(2 * n + 1, UInt(64.W))) + + def register( + slot: Int, + at: BigInt, + state: UInt, + update: UInt => Unit, + wide: Boolean + ): Unit = { + val value = WireDefault(UInt(64.W), state) + val low = word && offset === at.U + val high = wide.B && word && offset === (at + 4).U + val full = wide.B && double && offset === at.U + hits(slot) := low || high || full + reads(slot) := Mux(high, value(63, 32), Mux(low, value(31, 0), value)) + when(write && low)(update(if (wide) Cat(value(63, 32), access.data(31, 0)) else access.data(31, 0))) + when(write && high)(update(Cat(access.data(31, 0), value(31, 0)))) + when(write && full)(update(access.data)) + } + + ticks := Mux(ticks === (p.tickCycles - 1).U, 0.U, ticks + 1.U) + when(ticks === (p.tickCycles - 1).U)(mtime := mtime + 1.U) + for ((hart, i) <- hartIds.zipWithIndex) { + register(i, 4 * hart, msip(i).asUInt, v => msip(i) := v(0), wide = false) + register(n + i, 0x4000 + 8 * BigInt(hart), mtimecmp(i), v => mtimecmp(i) := v, wide = true) + io.msip(i) := msip(i) + io.mtip(i) := mtime >= mtimecmp(i) + } + register(2 * n, 0xbff8, mtime, v => mtime := v, wide = true) + + io.port.read := Mux1H(hits, reads) + io.port.error := !hits.asUInt.orR +} diff --git a/arch/src/main/scala/framework/device/peripherals/clint/configs/Parameters.scala b/arch/src/main/scala/framework/device/peripherals/clint/configs/Parameters.scala new file mode 100644 index 000000000..eeb10f6fc --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/clint/configs/Parameters.scala @@ -0,0 +1,26 @@ +package framework.device.peripherals.clint.configs + +import buckyball.config.ClintHardware +import scala.jdk.CollectionConverters._ + +import upickle.default.{macroRW, ReadWriter} + +case class ClintParams( + hartIds: Seq[Int], + base: BigInt = BigInt("2000000", 16), + bytes: BigInt = BigInt("10000", 16), + tickCycles: Int = 1000) { + require(tickCycles >= 1) +} + +object ClintParams { + + def fromProto(p: ClintHardware): ClintParams = ClintParams( + p.getHartIdsList.asScala.map(_.intValue).toSeq, + BigInt(java.lang.Long.toUnsignedString(p.getBase)), + BigInt(java.lang.Long.toUnsignedString(p.getBytes)), + p.getTickCycles + ) + + implicit val rw: ReadWriter[ClintParams] = macroRW +} diff --git a/arch/src/main/scala/framework/device/peripherals/clint/configs/default.toml b/arch/src/main/scala/framework/device/peripherals/clint/configs/default.toml new file mode 100644 index 000000000..417a2788c --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/clint/configs/default.toml @@ -0,0 +1,4 @@ +base = 0x2000000 +bytes = 0x10000 +tickCycles = 1000 +hartIds = [0] diff --git a/arch/src/main/scala/framework/system/device/Plic.scala b/arch/src/main/scala/framework/device/peripherals/plic/Plic.scala similarity index 89% rename from arch/src/main/scala/framework/system/device/Plic.scala rename to arch/src/main/scala/framework/device/peripherals/plic/Plic.scala index a36517d70..04daa7222 100644 --- a/arch/src/main/scala/framework/system/device/Plic.scala +++ b/arch/src/main/scala/framework/device/peripherals/plic/Plic.scala @@ -1,28 +1,24 @@ -package framework.system.device +package framework.device.peripherals.plic import chisel3._ +import framework.device.peripherals.plic.configs.PlicParams +import framework.device.peripherals.DevicePort import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} import chisel3.util._ -case class PlicParams( - base: BigInt = BigInt("c000000", 16), - bytes: BigInt = BigInt("4000000", 16), - sources: Int = 1, - priorityBits: Int = 3) { - require(sources >= 1 && sources <= 31 && priorityBits >= 1 && priorityBits <= 32) -} - /** * RISC-V PLIC with level-triggered gateways and 32-bit registers. Hart h owns context 2h (M) and * 2h+1 (S). A claim read clears the source's pending bit and holds it in service until completed. */ @instantiable -class Plic(p: PlicParams, hartIds: Seq[Int]) extends Module { +class Plic(p: PlicParams) extends Module { + val hartIds = p.hartIds require(hartIds.distinct.size == hartIds.size) val n = hartIds.size val contexts = 2 * (hartIds.max + 1) - @public val io = IO(new Bundle { + @public + val io = IO(new Bundle { val port = new DevicePort val sources = Input(UInt(p.sources.W)) val meip = Output(Vec(n, Bool())) diff --git a/arch/src/main/scala/framework/device/peripherals/plic/configs/Parameters.scala b/arch/src/main/scala/framework/device/peripherals/plic/configs/Parameters.scala new file mode 100644 index 000000000..be11ef0d7 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/plic/configs/Parameters.scala @@ -0,0 +1,28 @@ +package framework.device.peripherals.plic.configs + +import buckyball.config.PlicHardware +import scala.jdk.CollectionConverters._ + +import upickle.default.{macroRW, ReadWriter} + +case class PlicParams( + hartIds: Seq[Int], + base: BigInt = BigInt("c000000", 16), + bytes: BigInt = BigInt("4000000", 16), + sources: Int = 1, + priorityBits: Int = 3) { + require(sources >= 1 && sources <= 31 && priorityBits >= 1 && priorityBits <= 32) +} + +object PlicParams { + + def fromProto(p: PlicHardware): PlicParams = PlicParams( + p.getHartIdsList.asScala.map(_.intValue).toSeq, + BigInt(java.lang.Long.toUnsignedString(p.getBase)), + BigInt(java.lang.Long.toUnsignedString(p.getBytes)), + p.getSources, + p.getPriorityBits + ) + + implicit val rw: ReadWriter[PlicParams] = macroRW +} diff --git a/arch/src/main/scala/framework/device/peripherals/plic/configs/default.toml b/arch/src/main/scala/framework/device/peripherals/plic/configs/default.toml new file mode 100644 index 000000000..dea72f94e --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/plic/configs/default.toml @@ -0,0 +1,5 @@ +base = 0xc000000 +bytes = 0x4000000 +sources = 1 +priorityBits = 3 +hartIds = [0] diff --git a/arch/src/main/scala/framework/device/peripherals/scu/SCU.scala b/arch/src/main/scala/framework/device/peripherals/scu/SCU.scala new file mode 100644 index 000000000..fa858becc --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/scu/SCU.scala @@ -0,0 +1,116 @@ +package framework.device.peripherals.scu + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} +import chisel3.util.HasBlackBoxInline + +/** + * Single DPI-C bridge module for all harts. The hart_id is supplied as an + * input signal rather than being baked into the module name, so only one + * BlackBox is generated and DPI-C imports are not duplicated. + * + * Has separate hart_id inputs for uart and exit operations because different + * harts may write uart and exit simultaneously. + */ +@instantiable +class SCUWriteDPI extends BlackBox with HasBlackBoxInline { + override def desiredName = "SCUWriteDPI" + + @public + val io = IO(new Bundle { + val clock = Input(Clock()) + val reset = Input(Bool()) + val uart_hart_id = Input(UInt(32.W)) + val uart_valid = Input(Bool()) + val uart_data = Input(UInt(8.W)) + val exit_hart_id = Input(UInt(32.W)) + val exit_valid = Input(Bool()) + val exit_code = Input(UInt(32.W)) + }) + + setInline( + "SCUWriteDPI.v", + s""" + |import "DPI-C" context function void scu_uart_write(input int unsigned hart_id, input int unsigned ch); + |import "DPI-C" context function void scu_sim_exit(input int unsigned hart_id, input int unsigned code); + | + |module SCUWriteDPI( + | input clock, + | input reset, + | input [31:0] uart_hart_id, + | input uart_valid, + | input [7:0] uart_data, + | input [31:0] exit_hart_id, + | input exit_valid, + | input [31:0] exit_code + |); + | always @(posedge clock) begin + | if (!reset) begin + | if (uart_valid) begin + | scu_uart_write(uart_hart_id, {24'h0, uart_data}); + | end + | + | if (exit_valid) begin + | scu_sim_exit(exit_hart_id, exit_code); + | end + | end + | end + |endmodule + """.stripMargin + ) +} + +@instantiable +class SCUReadDPI extends BlackBox with HasBlackBoxInline { + override def desiredName = "SCUReadDPI" + + @public + val io = IO(new Bundle { + val clock = Input(Clock()) + val reset = Input(Bool()) + val hart_id = Input(UInt(32.W)) + val enable = Input(Bool()) + val pop = Input(Bool()) + val rx_valid = Output(Bool()) + val rx_data = Output(UInt(8.W)) + }) + + setInline( + "SCUReadDPI.v", + s""" + |import "DPI-C" context function void scu_uart_rx_sample( + | input int unsigned hart_id, + | input int unsigned pop, + | output int unsigned valid, + | output int unsigned data + |); + | + |module SCUReadDPI( + | input clock, + | input reset, + | input [31:0] hart_id, + | input enable, + | input pop, + | output reg rx_valid, + | output reg [7:0] rx_data + |); + | integer valid; + | integer data; + | always @(posedge clock) begin + | if (reset) begin + | rx_valid <= 1'b0; + | rx_data <= 8'h0; + | end else if (enable) begin + | scu_uart_rx_sample(hart_id, {31'h0, pop}, valid, data); + | if (pop) begin + | rx_valid <= 1'b0; + | end else begin + | rx_valid <= (valid != 0); + | rx_data <= data[7:0]; + | end + | end + | end + |endmodule + """.stripMargin + ) +} diff --git a/arch/src/main/scala/framework/device/peripherals/scu/SystemControl.scala b/arch/src/main/scala/framework/device/peripherals/scu/SystemControl.scala new file mode 100644 index 000000000..7a04954c5 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/scu/SystemControl.scala @@ -0,0 +1,81 @@ +package framework.device.peripherals.scu + +import chisel3._ +import framework.device.peripherals.scu.configs.SCUParams +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import chisel3.util._ +import framework.core.memory.access.{AccessParams, UncachedRequest, UncachedResponse} + +/** + * SCU on the System device ports, with the register layout documented on SCUParams. The hart sub-region is + * selected by address, so software keeps using `base + hartId * stride` regardless of port. + * Data is right-aligned at `addr`, as on every uncached CPU port. + */ +@instantiable +class SystemControl(params: SCUParams, p: AccessParams) extends Module { + val strideBits = log2Ceil(params.strideBytes) + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new UncachedRequest(p))) + val response = Decoupled(new UncachedResponse(p)) + }) + + val request = io.request + val response = io.response + val write: Instance[SCUWriteDPI] = Instantiate(new SCUWriteDPI) + val read: Instance[SCUReadDPI] = Instantiate(new SCUReadDPI) + val ready = RegInit(VecInit(Seq.fill(params.maxHarts)(0.U(8.W)))) + + val pending = RegInit(false.B) + val tag = Reg(UInt(p.tagBits.W)) + val error = Reg(Bool()) + val data = Reg(UInt(64.W)) + val status = Reg(Bool()) + + val offset = request.bits.addr - params.baseAddress.U + val hart = offset(31, strideBits) + val readyIndex = if (params.maxHarts == 1) 0.U(0.W) else hart(log2Ceil(params.maxHarts) - 1, 0) + val register = offset(strideBits - 1, 0) + val inside = request.bits.addr >= params.baseAddress.U && + request.bits.addr < (params.baseAddress + params.totalSizeBytes).U && hart < params.maxHarts.U + val fire = request.fire + val wr = request.bits.write + val exit = inside && register === 0x00000.U && wr + val uartTx = inside && register === 0x20000.U && wr + val uartRx = inside && register === 0x20004.U && !wr + val uartSt = inside && register === 0x20005.U && !wr + val readyReg = inside && register === 0x20006.U + + write.io.clock := clock + write.io.reset := reset.asBool + write.io.exit_hart_id := hart + write.io.exit_valid := fire && exit + write.io.exit_code := request.bits.data(31, 0) + write.io.uart_hart_id := hart + write.io.uart_valid := fire && uartTx + write.io.uart_data := request.bits.data(7, 0) + + // The DPI samples on this edge; status reads answer with the fresh sample next cycle. + read.io.clock := clock + read.io.reset := reset.asBool + read.io.hart_id := hart + read.io.enable := fire && (uartRx || uartSt) + read.io.pop := fire && uartRx + + request.ready := !pending + when(fire) { + pending := true.B + tag := request.bits.tag + error := !(exit || uartTx || uartRx || uartSt || readyReg) + status := uartSt + data := Mux(uartRx, read.io.rx_data, Mux(readyReg && !wr, ready(readyIndex), 0.U)) + when(readyReg && wr)(ready(readyIndex) := request.bits.data(7, 0)) + } + + response.valid := pending + response.bits.tag := tag + response.bits.error := error + response.bits.data := Mux(status, "h60".U | read.io.rx_valid.asUInt, data) + when(response.fire)(pending := false.B) +} diff --git a/arch/src/main/scala/framework/device/peripherals/scu/configs/Parameters.scala b/arch/src/main/scala/framework/device/peripherals/scu/configs/Parameters.scala new file mode 100644 index 000000000..634564a53 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/scu/configs/Parameters.scala @@ -0,0 +1,64 @@ +package framework.device.peripherals.scu.configs + +import buckyball.config.ScuHardware +import scala.jdk.CollectionConverters._ + +import upickle.default.{macroRW, ReadWriter} + +/** + * SCU is a global multi-hart System Control Unit (see SystemControl). It + * provides a per-hart sub-region for UART output and simulation exit. Each hart accesses its own sub-region via + * `baseAddress + hartId * strideBytes`. + * + * Address layout (default values): + * baseAddress = 0x60000000 + * strideBytes = 0x40000 (256 KiB per hart) + * totalSizeBytes = 0x10000000 (256 MiB total, supports up to 1024 harts) + * maxHarts = 64 + * + * Within each hart's sub-region: + * +0x00000: simExit register (write triggers $finish with exit code) + * +0x20000: uartTx register (write outputs a byte via DPI-C) + * +0x20004: uartRx register (read pops a byte from host input) + * +0x20005: status register (read-only, bit0 rx-valid, bit5/6 tx-ready) + * +0x20006: ready register (read/write software-visible byte) + * + * @param baseAddress physical base address (must be aligned to totalSizeBytes) + * @param strideBytes bytes per hart (must be power of two) + * @param totalSizeBytes total address window (must be power of two, + * >= maxHarts * strideBytes) + * @param maxHarts number of hart sub-regions. Accesses targeting + * hartId >= maxHarts return an access error. + */ +case class SCUParams( + baseAddress: BigInt = BigInt("60000000", 16), + strideBytes: BigInt = BigInt("40000", 16), + totalSizeBytes: BigInt = BigInt("10000000", 16), + maxHarts: Int = 64) { + require( + strideBytes > 0 && (strideBytes & (strideBytes - 1)) == 0, + s"SCU strideBytes ($strideBytes) must be a positive power of two" + ) + require( + totalSizeBytes > 0 && (totalSizeBytes & (totalSizeBytes - 1)) == 0, + s"SCU totalSizeBytes ($totalSizeBytes) must be a positive power of two" + ) + require(maxHarts > 0, s"SCU maxHarts ($maxHarts) must be positive") + require( + BigInt(maxHarts) * strideBytes <= totalSizeBytes, + s"SCU maxHarts ($maxHarts) * strideBytes ($strideBytes) = " + + s"${BigInt(maxHarts) * strideBytes} exceeds totalSizeBytes ($totalSizeBytes)" + ) +} + +object SCUParams { + + def fromProto(p: ScuHardware): SCUParams = SCUParams( + BigInt(java.lang.Long.toUnsignedString(p.getBaseAddress)), + BigInt(java.lang.Long.toUnsignedString(p.getStrideBytes)), + BigInt(java.lang.Long.toUnsignedString(p.getTotalSizeBytes)), + p.getMaxHarts + ) + + implicit val rw: ReadWriter[SCUParams] = macroRW +} diff --git a/arch/src/main/scala/framework/device/peripherals/scu/configs/default.toml b/arch/src/main/scala/framework/device/peripherals/scu/configs/default.toml new file mode 100644 index 000000000..9c3640f38 --- /dev/null +++ b/arch/src/main/scala/framework/device/peripherals/scu/configs/default.toml @@ -0,0 +1,4 @@ +baseAddress = 0x60000000 +strideBytes = 0x40000 +totalSizeBytes = 0x10000000 +maxHarts = 64 diff --git a/arch/src/main/scala/framework/frontend/Frontend.scala b/arch/src/main/scala/framework/frontend/Frontend.scala deleted file mode 100644 index 004ca1513..000000000 --- a/arch/src/main/scala/framework/frontend/Frontend.scala +++ /dev/null @@ -1,119 +0,0 @@ -package framework.frontend - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.frontend.boot.BootRom -import framework.frontend.decoder.GlobalDecoder -import framework.frontend.globalrs.{GlobalSchedComplete, GlobalSchedIssue, GlobalScheduler, RobAllocation} -import framework.top.GlobalConfig -import framework.system.core.rocket.{RoCCCommandBB, RoCCResponseBB} -import framework.balldomain.blink.SubRobRow -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.backend.shared.SharedMemLayout - -/** - * Frontend Module - * Encapsulates GlobalDecoder and global scheduler - */ -@instantiable -class Frontend(val b: GlobalConfig) extends Module { - - val sharedHashCount = if (b.memDomain.sharedEnable) SharedMemLayout.totalBank(b) else 0 - - @public - val io = IO(new Bundle { - val hartid = Input(UInt(b.tile.xLen.W)) - val sharedBankOwnerHartId = Input(UInt(b.tile.xLen.W)) - - // RoCC command input - val cmd = Flipped(Decoupled(new Bundle { - val cmd = new RoCCCommandBB(b.tile.xLen) - })) - - // Issue to domains - val ball_issue_o = Decoupled(new GlobalSchedIssue(b)) - val mem_issue_o = Decoupled(new GlobalSchedIssue(b)) - // Complete from domains - val ball_complete_i = Flipped(Decoupled(new GlobalSchedComplete(b))) - val mem_complete_i = Flipped(Decoupled(new GlobalSchedComplete(b))) - val kernel_issue_o = if (b.rvv.enable) Some(Decoupled(new GlobalSchedIssue(b))) else None - val kernel_complete_i = if (b.rvv.enable) Some(Flipped(Decoupled(new GlobalSchedComplete(b)))) else None - - // Ball -> SubROB request passthrough - val ball_subrob_req_i = Flipped(Vec(b.ballDomain.ballNum, Decoupled(new SubRobRow(b)))) - val inst_ids = Output(Vec(b.frontend.rob_entries, UInt(64.W))) - val allocation = Valid(new RobAllocation(b)) - val retired = Output(UInt(b.frontend.rob_entries.W)) - - val bank_hashes = - if (b.sim.diffTest) { - Some(Input(Vec(b.memDomain.bankNum + sharedHashCount, new PhysicalBankHash(b)))) - } else { - None - } - - // RoCC response - val resp = Decoupled(new RoCCResponseBB(b.tile.xLen)) - val busy = Output(Bool()) - val idle = Output(Bool()) - // Propagates the Global ROB retirement pulse to the host bridge. - - // Barrier interface — passthrough to GlobalRS - val barrier_arrive = Output(Bool()) - val barrier_release = Input(Bool()) - }) - - val gDecoder: Instance[GlobalDecoder] = Instantiate(new GlobalDecoder(b)) - val scheduler: Instance[GlobalScheduler] = Instantiate(new GlobalScheduler(b)) - - val boot: Instance[BootRom] = Instantiate(new BootRom(b)) - - boot.io.schedulerIdle := scheduler.io.idle - boot.io.cmd.ready := boot.io.active && gDecoder.io.id_i.ready - - gDecoder.io.id_i.valid := Mux(boot.io.active, boot.io.cmd.valid, io.cmd.valid) - gDecoder.io.id_i.bits.cmd := Mux(boot.io.active, boot.io.cmd.bits.cmd, io.cmd.bits.cmd) - // RoCC can't accept new instructions when boot is active - io.cmd.ready := !boot.io.active && gDecoder.io.id_i.ready - - scheduler.io.decode_cmd_i <> gDecoder.io.id_o - scheduler.io.hart_id := io.hartid - scheduler.io.sharedBankOwnerHartId := io.sharedBankOwnerHartId - scheduler.io.bank_hashes.foreach(_ := io.bank_hashes.get) - - io.ball_issue_o <> scheduler.io.ball_issue_o - io.mem_issue_o <> scheduler.io.mem_issue_o - if (b.rvv.enable) { - io.kernel_issue_o.get <> scheduler.io.kernel_issue_o.get - scheduler.io.kernel_complete_i.get <> io.kernel_complete_i.get - } - io.inst_ids := scheduler.io.inst_ids - io.allocation := scheduler.io.allocation - io.retired := scheduler.io.retired - io.idle := !boot.io.active && scheduler.io.idle && !gDecoder.io.id_o.valid - - scheduler.io.ball_complete_i <> io.ball_complete_i - scheduler.io.mem_complete_i <> io.mem_complete_i - - // Wire SubROB request from BallDomain through to scheduler - for (i <- 0 until b.ballDomain.ballNum) { - scheduler.io.ball_subrob_req_i(i) <> io.ball_subrob_req_i(i) - } - - io.resp <> scheduler.io.scheduler_rocc_o.resp - - // io.busy = 1 means NPU will block CPU - // This is the only time when NPU will block CPU: RoB can't accept new - // instructions (ROB full or barrier) - // - // Why we add boot.io.active here is because when boot is active, RoB is - // typically full with mset instructions. This situation is not need to - // block CPU. - io.busy := !boot.io.active && scheduler.io.scheduler_rocc_o.busy - - // Barrier passthrough - io.barrier_arrive := scheduler.io.barrier_arrive - scheduler.io.barrier_release := io.barrier_release - -} diff --git a/arch/src/main/scala/framework/frontend/boot/Boot.scala b/arch/src/main/scala/framework/frontend/boot/Boot.scala deleted file mode 100644 index 00bb950ab..000000000 --- a/arch/src/main/scala/framework/frontend/boot/Boot.scala +++ /dev/null @@ -1,72 +0,0 @@ -package framework.frontend.boot - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import framework.balldomain.isa.BallISA -import framework.memdomain.boot.MemBoot -import framework.system.core.rocket.{BuckyballCommand, RoCCCommandBB} -import framework.top.GlobalConfig - -@instantiable -class BootRom(val b: GlobalConfig) extends Module { - - @public - val io = IO(new Bundle { - - val cmd = Decoupled(new Bundle { - val cmd = new RoCCCommandBB(b.tile.xLen) - }) - - val schedulerIdle = Input(Bool()) - val active = Output(Bool()) - }) - - val ballInitRecords = b.ballDomain.ballIdMappings.map { mapping => - BallISA.init(mapping.ballId) - } - - val bootRecords = - MemBoot.initializationCommands(b) ++ ballInitRecords ++ MemBoot.releaseCommands(b) ++ - Seq.empty[BuckyballCommand] - - val bootFuncts = VecInit(bootRecords.map(r => r.funct.U(7.W))) - val bootRs1Data = VecInit(bootRecords.map(r => r.rs1.U(b.tile.xLen.W))) - val bootRs2Data = VecInit(bootRecords.map(r => r.rs2.U(b.tile.xLen.W))) - val bootPcWidth = math.max(1, log2Ceil(bootRecords.length + 1)) - - val active = RegInit(true.B) - val drain = RegInit(false.B) - val waitIdle = RegInit(false.B) - val pc = RegInit(0.U(bootPcWidth.W)) - - val atEnd = pc === bootRecords.length.U(bootPcWidth.W) - - val current = Wire(new RoCCCommandBB(b.tile.xLen)) - current := 0.U.asTypeOf(new RoCCCommandBB(b.tile.xLen)) - current.funct := Mux(atEnd, 0.U, bootFuncts(pc)) - current.funct3 := BuckyballCommand.Custom3Funct3.U - current.opcode := BuckyballCommand.Custom3Opcode.U - current.rs1Data := Mux(atEnd, 0.U, bootRs1Data(pc)) - current.rs2Data := Mux(atEnd, 0.U, bootRs2Data(pc)) - - val injectValid = active && !drain && !waitIdle && !atEnd - - when(active && !drain && atEnd) { - drain := true.B - } - when(waitIdle && io.schedulerIdle) { - waitIdle := false.B - } - when(drain && io.schedulerIdle) { - active := false.B - } - when(io.cmd.fire) { - pc := pc + 1.U - waitIdle := true.B - } - - io.cmd.valid := injectValid - io.cmd.bits.cmd := current - io.active := active -} diff --git a/arch/src/main/scala/framework/frontend/configs/FrontendParam.scala b/arch/src/main/scala/framework/frontend/configs/FrontendParam.scala deleted file mode 100644 index dedf00289..000000000 --- a/arch/src/main/scala/framework/frontend/configs/FrontendParam.scala +++ /dev/null @@ -1,33 +0,0 @@ -package framework.frontend.configs - -import upickle.default._ - -/** - * Frontend parameters - includes all frontend configuration. - */ -case class FrontendParam( - rob_entries: Int, - rs_out_of_order_response: Boolean, - bank_id_len: Int, - vbank_id_upper_bound: Int, - /** ISA bank-id namespace base for shared banks: raw id >= this means shared. */ - shared_bank_id_base: Int, - iter_len: Int, - sub_rob_enable: Boolean, - sub_rob_depth: Int) - -object FrontendParam { - implicit val rw: ReadWriter[FrontendParam] = macroRW - - def apply(): FrontendParam = FrontendParam( - rob_entries = 0, - rs_out_of_order_response = false, - bank_id_len = 0, - vbank_id_upper_bound = 0, - shared_bank_id_base = 0, - iter_len = 0, - sub_rob_enable = false, - sub_rob_depth = 0 - ) - -} diff --git a/arch/src/main/scala/framework/frontend/decoder/GISA.scala b/arch/src/main/scala/framework/frontend/decoder/GISA.scala deleted file mode 100644 index cd66dbd12..000000000 --- a/arch/src/main/scala/framework/frontend/decoder/GISA.scala +++ /dev/null @@ -1,19 +0,0 @@ -package framework.frontend.decoder - -import chisel3._ -import chisel3.util._ - -object GISA { - // enable=000, opcode group for no-bank-access instructions - val MVIN_KERNEL_BITPAT = BitPat("b0101100") // 44: writes program bank - val RUN_KERNEL_BITPAT = BitPat("b1001111") // 79: read data/program, write data - val BARRIER_BITPAT = BitPat("b0000001") // 1 (0x01) — enable=000, opcode=1 -} - -// Domain ID constants -object DomainId { - val FRONTEND = 0.U(4.W) // Frontend (barrier), does not enter ROB queue - val RVV = 2.U(4.W) // Private kernel execution and image control - val MEM = 1.U(4.W) // Memory domain - val BALL = 3.U(4.W) // Ball domain -} diff --git a/arch/src/main/scala/framework/frontend/decoder/GobalDecoder.scala b/arch/src/main/scala/framework/frontend/decoder/GobalDecoder.scala deleted file mode 100644 index 4d5ada3ca..000000000 --- a/arch/src/main/scala/framework/frontend/decoder/GobalDecoder.scala +++ /dev/null @@ -1,139 +0,0 @@ -package framework.frontend.decoder - -import chisel3._ -import chisel3.util._ -import chisel3.stage._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig -import framework.frontend.scoreboard.BankAccessInfo -import freechips.rocketchip.tile._ - -import framework.frontend.decoder.GISA._ -import framework.memdomain.frontend.cmd.decoder.DISA._ - -import framework.system.core.rocket.RoCCCommandBB - -@instantiable -class GlobalDecoder(val b: GlobalConfig) extends Module { - - val bankIdLen = b.frontend.bank_id_len - - @public - val io = IO(new Bundle { - - val id_i = Flipped(Decoupled(new Bundle { - val cmd = new RoCCCommandBB(b.tile.xLen) - })) - - val id_o = Decoupled(new PostGDCmd(b)) - }) - - // If reservation station is blocked, id_i is also blocked - io.id_i.ready := io.id_o.ready - - val func7 = io.id_i.bits.cmd.funct - val funct3 = io.id_i.bits.cmd.funct3 - val opcode = io.id_i.bits.cmd.opcode - val rs1 = io.id_i.bits.cmd.rs1Data - - // Private kernel commands are issued by the RVV controller. - val is_mem_inst = (func7 === MVIN_BITPAT) || - (func7 === MVIN_2D_BITPAT) || - (func7 === MVOUT_BITPAT) || - (func7 === MSET_BITPAT) || - (func7 === MVIN_MMIO_BITPAT) || - (func7 === MVOVER_BITPAT) - - val is_barrier_inst = func7 === BARRIER_BITPAT - - val programBank = rs1(bankIdLen - 1, 0) >= b.memDomain.virtualBankCount.U && - rs1(bankIdLen - 1, 0) < (b.memDomain.virtualBankCount + 2).U - val isProgramRelease = b.rvv.enable.B && func7 === MSET_BITPAT && programBank - val is_kernel_inst = func7 === MVIN_KERNEL_BITPAT || func7 === RUN_KERNEL_BITPAT || isProgramRelease - val is_bare_rvv = opcode === "h57".U || opcode === "h07".U || opcode === "h27".U - val is_ball_inst = !is_mem_inst && !is_barrier_inst && !is_bare_rvv && !is_kernel_inst - - when(io.id_i.fire) { - assert(func7 =/= 0.U, "GlobalDecoder: funct7 zero is not a Buckyball instruction") - assert(!is_bare_rvv, "GlobalDecoder: bare RVV instructions are not Buckyball commands") - when(is_kernel_inst) { - assert(b.rvv.enable.B, "GlobalDecoder: kernel command requires rvv.enable=true") - } - } - - // Encode domain ID - val domain_id = MuxCase( - DomainId.BALL, - Seq( - is_barrier_inst -> DomainId.FRONTEND, - is_kernel_inst -> DomainId.RVV, - is_mem_inst -> DomainId.MEM, - is_ball_inst -> DomainId.BALL - ) - ) - - // ------------------------------------------------------------------------- - // Bank access info extraction — enable flags from funct7[6:4] - // - // Unified rs1 layout (defined in isa.h): - // rs1[9:0] = bank_0 (rd_bank_0 or MSET config bank) - // rs1[19:10] = bank_1 (rd_bank_1, dual-operand only) - // rs1[29:20] = bank_2 (wr_bank for Ball instructions) - // rs1[63:30] = iter (34-bit) - // - // funct7[6:4] enable encoding: - // 000 = no bank access - // 001 = 1 read (bank0) - // 010 = 1 write (bank2) - // 011 = 1 read + 1 write (bank0 read, bank2 write) - // 100 = 2 read + 1 write (bank0+bank1 read, bank2 write) - // 101,110,111 = no bank access (extended opcode space) - // ------------------------------------------------------------------------- - val bankAccess = Wire(new BankAccessInfo(bankIdLen)) - val enableBits = func7(6, 4) - - // Decode enable from funct7[6:4] - val msetTransfer = func7 === MSET_BITPAT && io.id_i.bits.cmd.rs2Data(12) - val hasRd0 = enableBits === 1.U || enableBits === 3.U || enableBits === 4.U || msetTransfer - val hasRd1 = enableBits === 4.U - val hasWr = (enableBits === 2.U || enableBits === 3.U || enableBits === 4.U) && func7 =/= MVIN_MMIO_BITPAT - - val ballBid = WireDefault(0.U(5.W)) - b.ballDomain.ballISA.foreach { entry => - when(func7 === entry.funct7.U) { - ballBid := entry.bid.U - } - } - - bankAccess.rd_bank_0_valid := hasRd0 - bankAccess.rd_bank_0_id := rs1(bankIdLen - 1, 0) - bankAccess.rd_bank_1_valid := hasRd1 - bankAccess.rd_bank_1_id := rs1(bankIdLen + 9, 10) - bankAccess.wr_bank_valid := hasWr - bankAccess.wr_bank_id := Mux(func7 === MSET_BITPAT && !msetTransfer, rs1(bankIdLen - 1, 0), rs1(bankIdLen + 19, 20)) - - private def legalBank(raw: UInt): Bool = - raw <= b.frontend.vbank_id_upper_bound.U || - (b.rvv.enable.B && raw >= b.memDomain.virtualBankCount.U && raw < (b.memDomain.virtualBankCount + 2).U) || - (b.memDomain.sharedEnable.B && raw >= b.frontend.shared_bank_id_base.U && - raw < b.memDomain.virtualBankCount.U) - - val usesArchitecturalBank = is_ball_inst || func7 === MSET_BITPAT || func7 === MVIN_BITPAT || - func7 === MVIN_2D_BITPAT || func7 === MVOUT_BITPAT - when(io.id_i.fire && usesArchitecturalBank) { - when(hasRd0)(assert(legalBank(bankAccess.rd_bank_0_id), "GlobalDecoder: bank0 is outside configured ranges")) - when(hasRd1)(assert(legalBank(bankAccess.rd_bank_1_id), "GlobalDecoder: bank1 is outside configured ranges")) - when(hasWr)(assert(legalBank(bankAccess.wr_bank_id), "GlobalDecoder: write bank is outside configured ranges")) - } - - // Output control - io.id_o.valid := io.id_i.valid - io.id_o.bits.domain_id := domain_id - io.id_o.bits.ball_bid := ballBid - io.id_o.bits.cmd := io.id_i.bits.cmd - io.id_o.bits.bankAccess := bankAccess - io.id_o.bits.op1_col := 0.U - io.id_o.bits.op2_col := 0.U - io.id_o.bits.wr_col := 0.U - io.id_o.bits.isBarrier := is_barrier_inst -} diff --git a/arch/src/main/scala/framework/frontend/decoder/PostGDCmd.scala b/arch/src/main/scala/framework/frontend/decoder/PostGDCmd.scala deleted file mode 100644 index ae5d68448..000000000 --- a/arch/src/main/scala/framework/frontend/decoder/PostGDCmd.scala +++ /dev/null @@ -1,17 +0,0 @@ -package framework.frontend.decoder - -import chisel3._ -import framework.top.GlobalConfig -import framework.frontend.scoreboard.BankAccessInfo -import framework.system.core.rocket.RoCCCommandBB - -class PostGDCmd(val b: GlobalConfig) extends Bundle { - val domain_id = UInt(4.W) - val ball_bid = UInt(5.W) - val cmd = new RoCCCommandBB(b.tile.xLen) - val bankAccess = new BankAccessInfo(b.frontend.bank_id_len) - val op1_col = UInt(b.memDomain.groupCountWidth.W) - val op2_col = UInt(b.memDomain.groupCountWidth.W) - val wr_col = UInt(b.memDomain.groupCountWidth.W) - val isBarrier = Bool() -} diff --git a/arch/src/main/scala/framework/frontend/globalrs/GlobalROB.scala b/arch/src/main/scala/framework/frontend/globalrs/GlobalROB.scala deleted file mode 100644 index 2df0a27fa..000000000 --- a/arch/src/main/scala/framework/frontend/globalrs/GlobalROB.scala +++ /dev/null @@ -1,592 +0,0 @@ -package framework.frontend.globalrs - -import chisel3._ -import chisel3.util._ -import chisel3.experimental._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.frontend.decoder.{DomainId, PostGDCmd} -import framework.frontend.scoreboard.{BankAccessInfo, BankAliasTable, BankScoreboard} -import framework.memdomain.frontend.cmd.decoder.DISA.{MSET_BITPAT, MVIN_MMIO_BITPAT} -import framework.frontend.decoder.GISA.{MVIN_KERNEL_BITPAT, RUN_KERNEL_BITPAT} -import framework.memdomain.backend.shared.SharedMemLayout -import framework.memdomain.backend.banks.btrace.{BTraceRecord, PhysicalBankHash} - -@instantiable -class GlobalROB(val b: GlobalConfig) extends Module { - - val robDepth = b.frontend.rob_entries - val idWidth = log2Up(robDepth) - val resourceCount = b.memDomain.virtualBankCount + (if (b.rvv.enable) 2 else 0) - val scoreBankNum = resourceCount + robDepth - require(scoreBankNum <= (1 << b.frontend.bank_id_len), "Bank resource IDs and ROB aliases exceed bank_id_len") - val sharedHashCount = if (b.memDomain.sharedEnable) SharedMemLayout.totalBank(b) else 0 - val physicalBankCount = b.memDomain.bankNum + sharedHashCount - - require( - b.frontend.vbank_id_upper_bound < b.memDomain.virtualBankCount, - s"vbank_id_upper_bound(${b.frontend.vbank_id_upper_bound}) must be < virtualBankCount(${b.memDomain.virtualBankCount})" - ) - - @public - val io = IO(new Bundle { - val hart_id = Input(UInt(b.tile.xLen.W)) - val alloc = Flipped(new DecoupledIO(new PostGDCmd(b))) - val issue = new DecoupledIO(new GlobalRobEntry(b)) - val complete = Flipped(new DecoupledIO(UInt(idWidth.W))) - val allocation = Valid(new RobAllocation(b)) - val retired = Output(UInt(robDepth.W)) - - val empty = Output(Bool()) - val full = Output(Bool()) - val head_ptr = Output(UInt(idWidth.W)) - val issued_count = Output(UInt(log2Up(robDepth + 1).W)) - val entry_valid = Output(Vec(robDepth, Bool())) - val entry_complete = Output(Vec(robDepth, Bool())) - - val subRobActive = Input(Bool()) - val inst_ids = Output(Vec(robDepth, UInt(64.W))) - - val trace = if (b.sim.diffTest) Some(Output(Valid(new BTraceRecord))) else None - val traceCount = if (b.sim.diffTest) Some(Output(UInt(64.W))) else None - - val bank_hashes = - if (b.sim.diffTest) { - Some(Input(Vec(physicalBankCount, new PhysicalBankHash(b)))) - } else { - None - } - - }) - - // --------------------------------------------------------------------------- - // BAT + Bank Scoreboard - // --------------------------------------------------------------------------- - val bat: Instance[BankAliasTable] = Instantiate( - new BankAliasTable( - bankIdLen = b.frontend.bank_id_len, - vbankUpper = resourceCount - 1, - robEntries = robDepth - ) - ) - - val scoreboard: Instance[BankScoreboard] = - Instantiate(new BankScoreboard(b.frontend.bank_id_len, scoreBankNum, robDepth)) - - // --------------------------------------------------------------------------- - // Instruction trace (DPI-C, defined in ITraceDPI.scala) - // --------------------------------------------------------------------------- - val itraceAlloc = Module(new ITraceDPI) - val itraceIssue = Module(new ITraceDPI) - val itraceComp = Module(new ITraceDPI) - - for (t <- Seq(itraceAlloc, itraceIssue, itraceComp)) { - t.io.clock := clock - t.io.reset := reset.asBool - t.io.is_issue := 0.U - t.io.hart_id := io.hart_id - t.io.rob_id := 0.U - t.io.inst_id := 0.U - t.io.domain_id := 0.U - t.io.funct := 0.U - t.io.pc := 0.U - t.io.rs1_data := 0.U - t.io.rs2_data := 0.U - t.io.enable := false.B - } - - // --------------------------------------------------------------------------- - // Storage - // --------------------------------------------------------------------------- - val robEntries = RegInit(VecInit(Seq.fill(robDepth)(0.U.asTypeOf(new GlobalRobEntry(b))))) - val robValid = RegInit(VecInit(Seq.fill(robDepth)(false.B))) - val robIssued = RegInit(VecInit(Seq.fill(robDepth)(false.B))) - val robComplete = RegInit(VecInit(Seq.fill(robDepth)(false.B))) - val instIds = RegInit(VecInit(Seq.fill(robDepth)(0.U(64.W)))) - val nextInstId = RegInit(1.U(64.W)) - - val statusHashes = - if (b.sim.diffTest) { - Some(RegInit(VecInit(Seq.fill(robDepth)(0.U(32.W))))) - } else { - None - } - - io.inst_ids := instIds - - val headPtr = RegInit(0.U(idWidth.W)) - val tailPtr = RegInit(0.U(idWidth.W)) - val issuedCount = RegInit(0.U(log2Up(robDepth + 1).W)) - val bankColWidth = b.memDomain.groupCountWidth - val bankCols = RegInit(VecInit(Seq.fill(b.memDomain.virtualBankCount)(0.U(bankColWidth.W)))) - // In-flight ownership is tracked in the architectural vbank namespace. - val vbankMaskWidth = resourceCount - val vbankBusy = RegInit(0.U(vbankMaskWidth.W)) - - // Dependency state is maintained incrementally instead of rebuilding all - // older-entry comparisons every cycle. A bit in slot k names the ROB slot - // that currently blocks k. Ordinary RAW dependencies disappear at complete; - // dependencies involving a mapping/config instruction disappear at commit. - // Keeping one union mask is sufficient because a config dependency is never - // allowed to clear earlier than commit, even when it also looks like a RAW. - val dependencyMask = RegInit(VecInit(Seq.fill(robDepth)(0.U(robDepth.W)))) - - val isEmpty = headPtr === tailPtr && !robValid(headPtr) - val isFull = headPtr === tailPtr && robValid(headPtr) - - def nextPtr(p: UInt): UInt = Mux(p === (robDepth - 1).U, 0.U, p + 1.U) - def wrapPtr(v: UInt): UInt = Mux(v >= robDepth.U, v - robDepth.U, v) - def robIdx(v: UInt): UInt = v(idWidth - 1, 0) - - def isMappingConfig(entry: GlobalRobEntry): Bool = - entry.cmd.cmd.funct === MSET_BITPAT - - def accessUsesBank(access: BankAccessInfo, bank: UInt): Bool = - (access.rd_bank_0_valid && access.rd_bank_0_id === bank) || - (access.rd_bank_1_valid && access.rd_bank_1_id === bank) || - (access.wr_bank_valid && access.wr_bank_id === bank) - - // Build one architectural-vbank mask per ROB slot once, then hazard checks - // become mask intersections instead of repeating ID comparisons. - val hazardBankCount = resourceCount - - def rawReadMask(access: BankAccessInfo): UInt = { - val rd0 = Mux( - access.rd_bank_0_valid && access.rd_bank_0_id < hazardBankCount.U, - UIntToOH(access.rd_bank_0_id, hazardBankCount), - 0.U(hazardBankCount.W) - ) - val rd1 = Mux( - access.rd_bank_1_valid && access.rd_bank_1_id < hazardBankCount.U, - UIntToOH(access.rd_bank_1_id, hazardBankCount), - 0.U(hazardBankCount.W) - ) - rd0 | rd1 - } - - def rawWriteMask(access: BankAccessInfo): UInt = Mux( - access.wr_bank_valid && access.wr_bank_id < hazardBankCount.U, - UIntToOH(access.wr_bank_id, hazardBankCount), - 0.U(hazardBankCount.W) - ) - - def rawUseMask(access: BankAccessInfo): UInt = - rawReadMask(access) | rawWriteMask(access) - - // --------------------------------------------------------------------------- - // Allocate: enqueue decoded instruction into ROB - // rob_id == tailPtr at allocation time (no separate counter needed) - // --------------------------------------------------------------------------- - val commitMask = Wire(Vec(robDepth, Bool())) - for (i <- 0 until robDepth) { - commitMask(i) := false.B - } - bat.io.free.valid := commitMask.asUInt.orR - bat.io.free.mask := commitMask - - val commitScan = Wire(Vec(robDepth, Bool())) - val commitKeep = Wire(Vec(robDepth + 1, Bool())) - - val headNeedsTrace = instIds(headPtr) =/= 0.U && !isMappingConfig(robEntries(headPtr)) && - robEntries(headPtr).cmd.bankAccess.wr_bank_valid && robEntries( - headPtr - ).cmd.bankAccess.wr_bank_id < b.memDomain.virtualBankCount.U - - commitKeep(0) := true.B - for (i <- 0 until robDepth) { - val ptr = robIdx(wrapPtr(headPtr + i.U)) - commitScan(i) := commitKeep(i) && robValid(ptr) && robComplete(ptr) - commitKeep(i + 1) := (if (b.sim.diffTest) false.B else commitScan(i)) - } - val hasCommit = commitScan.asUInt.orR - val tailAlias = Wire(UInt(b.frontend.bank_id_len.W)) - tailAlias := - resourceCount.U(b.frontend.bank_id_len.W) + tailPtr - - val tailAliasLive = WireDefault(false.B) - for (i <- 0 until robDepth) { - when(robValid(i) && !robComplete(i) && accessUsesBank(robEntries(i).renamedBankAccess, tailAlias)) { - tailAliasLive := true.B - } - } - - io.alloc.ready := !isFull && !hasCommit && !tailAliasLive - io.allocation.valid := io.alloc.fire - io.allocation.bits.rob_id := tailPtr - io.retired := commitMask.asUInt - bat.io.alloc.valid := io.alloc.fire - bat.io.alloc.rob_id := tailPtr - bat.io.alloc.raw := io.alloc.bits.bankAccess - - // Mark write alias as busy in scoreboard at alloc time (not issue time). - scoreboard.alloc.valid := io.alloc.fire && io.alloc.bits.bankAccess.wr_bank_valid - scoreboard.alloc.bits := bat.io.alloc_renamed - - // Bank masks are shared by allocation and issue. This avoids rebuilding the - // UIntToOH decoders once for every older-entry comparison at allocation. - val entryRawReads = Wire(Vec(robDepth, UInt(hazardBankCount.W))) - val entryRawWrites = Wire(Vec(robDepth, UInt(hazardBankCount.W))) - val entryIsConfig = Wire(Vec(robDepth, Bool())) - for (slot <- 0 until robDepth) { - entryRawReads(slot) := rawReadMask(robEntries(slot).cmd.bankAccess) - entryRawWrites(slot) := rawWriteMask(robEntries(slot).cmd.bankAccess) - entryIsConfig(slot) := isMappingConfig(robEntries(slot)) - } - - // A newly allocated entry is always younger than all currently valid ROB - // entries (tailPtr points at the first free slot). Capture its older-entry - // dependencies once here; issue no longer repeats pairwise comparisons. - val allocReadMask = rawReadMask(io.alloc.bits.bankAccess) - val allocWriteMask = rawWriteMask(io.alloc.bits.bankAccess) - val allocIsConfig = io.alloc.bits.cmd.funct === MSET_BITPAT - val allocIsBall = io.alloc.bits.domain_id === DomainId.BALL - // MMIO is a separate shared storage resource, independent of tensor banks. - val mmioReaders = b.ballDomain.ballIdMappings.filter(_.mmioReadBW > 0) - val mmioWriters = b.ballDomain.ballIdMappings.filter(_.mmioWriteBW > 0) - val allocMmioRead = mmioReaders.map(m => allocIsBall && io.alloc.bits.ball_bid === m.ballId.U) - .foldLeft(false.B)(_ || _) - val allocMmioWrite = io.alloc.bits.cmd.funct === MVIN_MMIO_BITPAT || - mmioWriters.map(m => allocIsBall && io.alloc.bits.ball_bid === m.ballId.U).foldLeft(false.B)(_ || _) - val allocDependencyBits = Wire(Vec(robDepth, Bool())) - for (older <- 0 until robDepth) { - val olderUseMask = entryRawReads(older) | entryRawWrites(older) - val rawConflict = ((allocReadMask & entryRawWrites(older)) | - (allocWriteMask & olderUseMask)).orR - val configConflict = (allocIsConfig && entryIsConfig(older)) || - ((allocIsConfig || entryIsConfig(older)) && ((allocReadMask | allocWriteMask) & olderUseMask).orR) - val sameBall = allocIsBall && - robEntries(older).cmd.domain_id === DomainId.BALL && - robEntries(older).cmd.ball_bid === io.alloc.bits.ball_bid - val olderIsBall = robEntries(older).cmd.domain_id === DomainId.BALL - val olderMmioRead = mmioReaders - .map(m => olderIsBall && robEntries(older).cmd.ball_bid === m.ballId.U).foldLeft(false.B)(_ || _) - val olderMmioWrite = robEntries(older).cmd.cmd.funct === MVIN_MMIO_BITPAT || - mmioWriters.map(m => olderIsBall && robEntries(older).cmd.ball_bid === m.ballId.U).foldLeft(false.B)(_ || _) - val mmioConflict = (allocMmioRead && olderMmioWrite) || - (allocMmioWrite && (olderMmioRead || olderMmioWrite)) - // Config conflicts include entries that have completed but are waiting to - // commit. RAW conflicts and same-Ball commands release at completion. - allocDependencyBits(older) := - (robValid(older) && !robComplete(older) && (rawConflict || sameBall || mmioConflict)) || - (robValid(older) && configConflict) - } - val allocDependencies = allocDependencyBits.asUInt - val allocUsesInstId = io.alloc.bits.cmd.pc =/= 0.U - - when(io.alloc.fire) { - itraceAlloc.io.is_issue := 2.U - itraceAlloc.io.rob_id := tailPtr - itraceAlloc.io.inst_id := Mux(allocUsesInstId, nextInstId, 0.U) - itraceAlloc.io.domain_id := io.alloc.bits.domain_id - itraceAlloc.io.funct := io.alloc.bits.cmd.funct - itraceAlloc.io.pc := io.alloc.bits.cmd.pc - itraceAlloc.io.rs1_data := io.alloc.bits.cmd.rs1Data - itraceAlloc.io.rs2_data := io.alloc.bits.cmd.rs2Data - itraceAlloc.io.enable := true.B - - robEntries(tailPtr).cmd := io.alloc.bits - robEntries(tailPtr).renamedBankAccess := bat.io.alloc_renamed - robEntries(tailPtr).rob_id := tailPtr - instIds(tailPtr) := Mux(allocUsesInstId, nextInstId, 0.U) - robValid(tailPtr) := true.B - robIssued(tailPtr) := false.B - robComplete(tailPtr) := false.B - statusHashes.foreach { hashes => - hashes(tailPtr) := 0.U - } - tailPtr := nextPtr(tailPtr) - when(allocUsesInstId) { - nextInstId := nextInstId + 1.U - } - } - - // --------------------------------------------------------------------------- - // Complete: mark entry as completed, release scoreboard resources - // --------------------------------------------------------------------------- - if (b.sim.diffTest) { - val collecting = RegInit(false.B) - val collectCid = Reg(UInt(idWidth.W)) - val collectGroup = Reg(UInt(32.W)) - val currentHash = Reg(UInt(32.W)) - val hashReady = RegInit(false.B) - val hashCid = Reg(UInt(idWidth.W)) - - val incomingAccess = robEntries(io.complete.bits).cmd.bankAccess - val incomingNeedsHash = instIds(io.complete.bits) =/= 0.U && !isMappingConfig(robEntries(io.complete.bits)) && - incomingAccess.wr_bank_valid && incomingAccess.wr_bank_id < b.memDomain.virtualBankCount.U - io.complete.ready := !incomingNeedsHash || (hashReady && hashCid === io.complete.bits) - - when(!collecting && !hashReady && io.complete.valid && incomingNeedsHash) { - collecting := true.B - collectCid := io.complete.bits - collectGroup := 0.U - currentHash := 0.U - } - - val vbank = robEntries(collectCid).cmd.bankAccess.wr_bank_id - val matches = VecInit(io.bank_hashes.get.map { state => - state.valid && state.hartId === io.hart_id && state.vbankId === vbank && state.groupId === collectGroup - }) - val nextMatches = VecInit(io.bank_hashes.get.map { state => - state.valid && state.hartId === io.hart_id && state.vbankId === vbank && state.groupId === collectGroup + 1.U - }) - val physical = Mux1H(matches, io.bank_hashes.get) - def rotateLeft(value: UInt, amount: Int): UInt = Cat(value(31 - amount, 0), value(31, 32 - amount)) - - val mixed = physical.statusHash ^ rotateLeft(physical.groupId, 7) ^ "h9e3779b9".U - val nextHash = rotateLeft(currentHash, 5) ^ mixed ^ rotateLeft(mixed, 13) - - when(collecting) { - assert(PopCount(matches) === 1.U, "BTrace requires exactly one physical bank per vbank group") - when(nextMatches.asUInt.orR) { - currentHash := nextHash - collectGroup := collectGroup + 1.U - }.otherwise { - statusHashes.get(collectCid) := nextHash - collecting := false.B - hashReady := true.B - hashCid := collectCid - } - } - - when(io.complete.fire) { - hashReady := false.B - } - } else { - io.complete.ready := true.B - } - - scoreboard.complete.valid := false.B - scoreboard.complete.bits := 0.U.asTypeOf(scoreboard.complete.bits) - - val issueFired = WireDefault(false.B) - val completeIssuedEntry = io.complete.fire && robIssued(io.complete.bits) - - when(io.complete.fire) { - val cid = io.complete.bits - robComplete(cid) := true.B - scoreboard.complete.valid := true.B - scoreboard.complete.bits := robEntries(cid).renamedBankAccess - - itraceComp.io.is_issue := 0.U - itraceComp.io.rob_id := cid - itraceComp.io.inst_id := instIds(cid) - itraceComp.io.domain_id := robEntries(cid).cmd.domain_id - itraceComp.io.funct := robEntries(cid).cmd.cmd.funct - itraceComp.io.pc := robEntries(cid).cmd.cmd.pc - itraceComp.io.rs1_data := robEntries(cid).cmd.cmd.rs1Data - itraceComp.io.rs2_data := robEntries(cid).cmd.cmd.rs2Data - itraceComp.io.enable := true.B - } - - val completedBitMask = Mux( - io.complete.fire, - UIntToOH(io.complete.bits, robDepth), - 0.U(robDepth.W) - ) - - // --------------------------------------------------------------------------- - // Issue: scan from head for first issuable entry (valid && !issued && !complete) - // --------------------------------------------------------------------------- - val scanValid = Wire(Vec(robDepth, Bool())) - val scanReady = Wire(Vec(robDepth, Bool())) - val vbankMasks = Wire(Vec(robDepth, UInt(vbankMaskWidth.W))) - for (slot <- 0 until robDepth) { - // Reuse the raw hazard mask for active-command ownership. - vbankMasks(slot) := entryRawReads(slot) | entryRawWrites(slot) - } - - // Check each candidate against older ROB entries using mask intersections. - // The masks are precomputed once per slot, so this retains the original - // age-sensitive RAW/WAR/WAW semantics without replicating three equality - // comparators for every bank access pair. - for (i <- 0 until robDepth) { - val ptr = robIdx(wrapPtr(headPtr + i.U)) - scanValid(i) := robValid(ptr) && !robIssued(ptr) && !robComplete(ptr) - scoreboard.queryVec(i) := robEntries(ptr).renamedBankAccess - scanReady(i) := scanValid(i) && !scoreboard.hazardVec(i) && - !dependencyMask(ptr).orR && - !(vbankMasks(ptr) & vbankBusy).orR - } - - val hasReady = scanReady.asUInt.orR - val firstReady = PriorityEncoder(scanReady.asUInt) - val actualIssuePtr = robIdx(wrapPtr(headPtr + firstReady)) - - scoreboard.query := robEntries(actualIssuePtr).renamedBankAccess - val canIssue = hasReady - - val issueStageValid = RegInit(false.B) - val issueStageEntry = Reg(new GlobalRobEntry(b)) - val issueStageFire = issueStageValid && !io.subRobActive && io.issue.ready - val issueStageReady = !issueStageValid || issueStageFire - - // Keep the wide ROB entry and bank-column mux one register boundary away - // from the age/scoreboard scan. The pointer stage can accept one candidate - // per cycle; the payload stage drains at one per cycle, so steady-state II - // remains one while the extra latency is hidden by in-flight commands. - val issuePtrValid = RegInit(false.B) - val issuePtr = Reg(UInt(idWidth.W)) - val issuePtrFire = issuePtrValid && issueStageReady - val issuePtrReady = !issuePtrValid || issuePtrFire - val issueLoad = canIssue && !io.subRobActive && issuePtrReady - val issuePayload = Wire(new GlobalRobEntry(b)) - issuePayload := robEntries(issuePtr) - issuePayload.cmd.op1_col := Mux( - issuePayload.cmd.bankAccess.rd_bank_0_valid && issuePayload.cmd.bankAccess.rd_bank_0_id < b.memDomain.virtualBankCount.U, - bankCols(issuePayload.cmd.bankAccess.rd_bank_0_id(b.memDomain.vbankIdWidth - 1, 0)), - 0.U - ) - issuePayload.cmd.op2_col := Mux( - issuePayload.cmd.bankAccess.rd_bank_1_valid && issuePayload.cmd.bankAccess.rd_bank_1_id < b.memDomain.virtualBankCount.U, - bankCols(issuePayload.cmd.bankAccess.rd_bank_1_id(b.memDomain.vbankIdWidth - 1, 0)), - 0.U - ) - issuePayload.cmd.wr_col := Mux( - issuePayload.cmd.bankAccess.wr_bank_valid && issuePayload.cmd.bankAccess.wr_bank_id < b.memDomain.virtualBankCount.U, - bankCols(issuePayload.cmd.bankAccess.wr_bank_id(b.memDomain.vbankIdWidth - 1, 0)), - 0.U - ) - - io.issue.valid := issueStageValid && !io.subRobActive - io.issue.bits := issueStageEntry - - when(issuePtrReady) { - issuePtrValid := issueLoad - when(issueLoad) { - issuePtr := actualIssuePtr - } - } - when(issueStageReady) { - issueStageValid := issuePtrValid - when(issuePtrFire) { - issueStageEntry := issuePayload - } - } - - scoreboard.issue.valid := false.B - scoreboard.issue.bits := 0.U.asTypeOf(scoreboard.issue.bits) - - when(issueLoad) { - robIssued(actualIssuePtr) := true.B - issueFired := true.B - scoreboard.issue.valid := true.B - scoreboard.issue.bits := robEntries(actualIssuePtr).renamedBankAccess - - itraceIssue.io.is_issue := 1.U - itraceIssue.io.rob_id := robEntries(actualIssuePtr).rob_id - itraceIssue.io.inst_id := instIds(actualIssuePtr) - itraceIssue.io.domain_id := robEntries(actualIssuePtr).cmd.domain_id - itraceIssue.io.funct := robEntries(actualIssuePtr).cmd.cmd.funct - itraceIssue.io.pc := robEntries(actualIssuePtr).cmd.cmd.pc - itraceIssue.io.rs1_data := robEntries(actualIssuePtr).cmd.cmd.rs1Data - itraceIssue.io.rs2_data := robEntries(actualIssuePtr).cmd.cmd.rs2Data - itraceIssue.io.enable := true.B - } - - // Preserve the original update ordering when issue and complete coincide: - // a completion clears ownership after the newly issued transaction marks - // its banks busy. - val issuedVbankMask = Mux(issueLoad, vbankMasks(actualIssuePtr), 0.U) - val completedVbankMask = Mux(io.complete.fire, vbankMasks(io.complete.bits), 0.U) - when(issueLoad || io.complete.fire) { - vbankBusy := - (vbankBusy & ~completedVbankMask) | issuedVbankMask - } - - issuedCount := issuedCount + issueFired.asUInt - completeIssuedEntry.asUInt - - // --------------------------------------------------------------------------- - // Commit: clear completed entries. - // Explicitly skip entries being allocated or completed this cycle. - // --------------------------------------------------------------------------- - for (i <- 0 until robDepth) { - val hits = (0 until robDepth).map { off => - val ptr = robIdx(wrapPtr(headPtr + off.U)) - commitScan(off) && ptr === i.U - } - commitMask(i) := hits.reduce(_ || _) - when(commitMask(i)) { - robValid(i) := false.B - robIssued(i) := false.B - robComplete(i) := false.B - } - } - - if (b.sim.diffTest) { - val trace = io.trace.get - trace.valid := hasCommit && headNeedsTrace - trace.bits.instId := instIds(headPtr) - trace.bits.hartId := io.hart_id - trace.bits.w0Vbank := robEntries(headPtr).cmd.bankAccess.wr_bank_id - trace.bits.w0Hash := statusHashes.get(headPtr) - - val produced = RegInit(0.U(64.W)) - when(trace.valid)(produced := produced + 1.U) - io.traceCount.get := produced - } - - // Maintain dependency masks at the same architectural events that gate the - // issue scan. A dependency involving a config entry (on either side) clears - // only at commit; all other dependencies clear at complete. This is the - // union-mask form of the former separate RAW/config masks. - val configEntryMask = entryIsConfig.asUInt - for (slot <- 0 until robDepth) { - val isAllocatedSlot = io.alloc.fire && tailPtr === slot.U - val slotIsConfig = Mux(isAllocatedSlot, allocIsConfig, entryIsConfig(slot)) - // Each dependency bit names the older entry that blocks this slot. Keep a - // config-related edge through completion when either endpoint is config. - val configRelated = configEntryMask | Fill(robDepth, slotIsConfig) - val clearMask = commitMask.asUInt | (completedBitMask & ~configRelated) - when(isAllocatedSlot) { - dependencyMask(slot) := allocDependencies & ~clearMask - }.otherwise { - dependencyMask(slot) := dependencyMask(slot) & ~clearMask - } - } - - val mappingCommits = VecInit((0 until robDepth).map(i => - commitMask(i) && robEntries(i).cmd.domain_id === DomainId.MEM && - robEntries(i).cmd.cmd.funct === MSET_BITPAT - )) - - assert(PopCount(mappingCommits) <= 1.U, "Mapping dependency permits one configuration commit") - val mapping = robEntries(PriorityEncoder(mappingCommits.asUInt)).cmd - val destination = mapping.bankAccess.wr_bank_id - val source = mapping.bankAccess.rd_bank_0_id - val transferColumns = bankCols(destination(b.memDomain.vbankIdWidth - 1, 0)) + - bankCols(source(b.memDomain.vbankIdWidth - 1, 0)) - - val fullColumns = - if (b.memDomain.sharedEnable) { - Mux(destination >= b.frontend.shared_bank_id_base.U, SharedMemLayout.totalBank(b).U, b.memDomain.bankNum.U) - } else b.memDomain.bankNum.U - - val allocatedColumns = Mux(mapping.cmd.rs2Data(9, 5) === 0.U, fullColumns, mapping.cmd.rs2Data(9, 5)) - for (bank <- 0 until b.memDomain.virtualBankCount) { - when(mappingCommits.asUInt.orR) { - when(destination === bank.U) { - bankCols(bank) := Mux( - mapping.cmd.rs2Data(12), - transferColumns, - Mux(mapping.cmd.rs2Data(10), allocatedColumns, 0.U) - ) - } - when(mapping.cmd.rs2Data(12) && source === bank.U)(bankCols(bank) := 0.U) - } - } - - val commitCount = PopCount(commitScan) - headPtr := wrapPtr(headPtr + commitCount) - - // --------------------------------------------------------------------------- - // Status outputs - // --------------------------------------------------------------------------- - io.empty := isEmpty - io.full := isFull - io.head_ptr := headPtr - io.issued_count := issuedCount - io.entry_valid := robValid - io.entry_complete := robComplete - -} diff --git a/arch/src/main/scala/framework/frontend/globalrs/GlobalScheduler.scala b/arch/src/main/scala/framework/frontend/globalrs/GlobalScheduler.scala deleted file mode 100644 index c09e27c1c..000000000 --- a/arch/src/main/scala/framework/frontend/globalrs/GlobalScheduler.scala +++ /dev/null @@ -1,258 +0,0 @@ -package framework.frontend.globalrs - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.frontend.decoder.{DomainId, PostGDCmd} -import framework.frontend.decoder.GISA._ -import framework.frontend.scoreboard.BankAccessInfo -import framework.system.core.rocket.RoCCResponseBB -import framework.balldomain.blink.SubRobRow -import framework.memdomain.backend.banks.btrace.{BTraceDPI, PhysicalBankHash} -import framework.memdomain.backend.shared.SharedMemLayout -import framework.memdomain.frontend.mem.dma.DmaStatus - -class GlobalRobEntry(val b: GlobalConfig) extends Bundle { - val cmd = new PostGDCmd(b) - val renamedBankAccess = new BankAccessInfo(b.frontend.bank_id_len) - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) -} - -class GlobalSchedIssue(b: GlobalConfig) extends GlobalRobEntry(b) { - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) -} - -class GlobalSchedComplete(b: GlobalConfig) extends Bundle { - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) - val fault = new DmaStatus -} - -class RobAllocation(b: GlobalConfig) extends Bundle { - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) -} - -@instantiable -class GlobalScheduler(val b: GlobalConfig) extends Module { - - val sharedHashCount = if (b.memDomain.sharedEnable) SharedMemLayout.totalBank(b) else 0 - - @public - val io = IO(new Bundle { - val hart_id = Input(UInt(b.tile.xLen.W)) - val sharedBankOwnerHartId = Input(UInt(b.tile.xLen.W)) - val decode_cmd_i = Flipped(new DecoupledIO(new PostGDCmd(b))) - val ball_issue_o = Decoupled(new GlobalSchedIssue(b)) - val mem_issue_o = Decoupled(new GlobalSchedIssue(b)) - val ball_complete_i = Flipped(Decoupled(new GlobalSchedComplete(b))) - val mem_complete_i = Flipped(Decoupled(new GlobalSchedComplete(b))) - val kernel_issue_o = if (b.rvv.enable) Some(Decoupled(new GlobalSchedIssue(b))) else None - val kernel_complete_i = if (b.rvv.enable) Some(Flipped(Decoupled(new GlobalSchedComplete(b)))) else None - val ball_subrob_req_i = Flipped(Vec(b.ballDomain.ballNum, Decoupled(new SubRobRow(b)))) - val inst_ids = Output(Vec(b.frontend.rob_entries, UInt(64.W))) - val allocation = Valid(new RobAllocation(b)) - val retired = Output(UInt(b.frontend.rob_entries.W)) - - val bank_hashes = - if (b.sim.diffTest) { - Some(Input(Vec(b.memDomain.bankNum + sharedHashCount, new PhysicalBankHash(b)))) - } else { - None - } - - val scheduler_rocc_o = new Bundle { - val resp = new DecoupledIO(new RoCCResponseBB(b.tile.xLen)) - val busy = Output(Bool()) - } - - val idle = Output(Bool()) - - val barrier_arrive = Output(Bool()) - val barrier_release = Input(Bool()) - }) - - val rob: Instance[GlobalROB] = Instantiate(new GlobalROB(b)) - rob.io.hart_id := io.hart_id - rob.io.bank_hashes.foreach(_ := io.bank_hashes.get) - io.inst_ids := rob.io.inst_ids - io.allocation := rob.io.allocation - io.retired := rob.io.retired - - if (b.sim.diffTest) { - val btrace = Module(new BTraceDPI) - val trace = rob.io.trace.get - // VVAC stalls logical clocks when its nonblocking channel is full. - btrace.io.clock := clock - btrace.io.reset := reset.asBool - btrace.io.instId := trace.bits.instId - btrace.io.hartId := io.hart_id - val sharedBank = trace.bits.w0Vbank > b.frontend.vbank_id_upper_bound.U && - trace.bits.w0Vbank >= b.frontend.shared_bank_id_base.U && - trace.bits.w0Vbank < b.memDomain.virtualBankCount.U - btrace.io.ownerHartId := Mux(sharedBank, io.sharedBankOwnerHartId, io.hart_id) - btrace.io.w0Vbank := trace.bits.w0Vbank - btrace.io.w0Hash := trace.bits.w0Hash - btrace.io.fire := trace.valid - btrace.io.produced := rob.io.traceCount.get - btrace.io.idle := io.idle && !io.decode_cmd_i.valid && !trace.valid - } - - val isBarrierCmd = io.decode_cmd_i.valid && io.decode_cmd_i.bits.isBarrier - val barrierWaitROB = RegInit(false.B) - val barrierWaitRelease = RegInit(false.B) - when(isBarrierCmd && !barrierWaitROB && !barrierWaitRelease) { - barrierWaitROB := true.B - } - when(barrierWaitROB && rob.io.empty) { - barrierWaitROB := false.B - barrierWaitRelease := true.B - } - when(barrierWaitRelease && io.barrier_release) { - barrierWaitRelease := false.B - } - io.barrier_arrive := barrierWaitRelease - - val isFrontendCmd = io.decode_cmd_i.bits.isBarrier - val anyStall = barrierWaitROB || barrierWaitRelease - rob.io.alloc.valid := io.decode_cmd_i.valid && !isFrontendCmd && !anyStall - rob.io.alloc.bits := io.decode_cmd_i.bits - io.decode_cmd_i.ready := Mux( - isFrontendCmd, - !anyStall, - rob.io.alloc.ready && !anyStall - ) - - val is_ball_domain = rob.io.issue.bits.cmd.domain_id === DomainId.BALL - val is_mem_domain = rob.io.issue.bits.cmd.domain_id === DomainId.MEM - val is_kernel_domain = rob.io.issue.bits.cmd.domain_id === DomainId.RVV - - val mainIssueEntry = Wire(new GlobalSchedIssue(b)) - mainIssueEntry.cmd := rob.io.issue.bits.cmd - mainIssueEntry.renamedBankAccess := rob.io.issue.bits.renamedBankAccess - mainIssueEntry.rob_id := rob.io.issue.bits.rob_id - mainIssueEntry.is_sub := false.B - mainIssueEntry.sub_rob_id := 0.U - - if (b.rvv.enable) { - io.kernel_issue_o.get.valid := rob.io.issue.valid && is_kernel_domain - io.kernel_issue_o.get.bits := mainIssueEntry - } - val kernelReady = if (b.rvv.enable) io.kernel_issue_o.get.ready else false.B - - val completeArb = Module(new Arbiter(new GlobalSchedComplete(b), if (b.rvv.enable) 3 else 2)) - val completeQueue = Module(new Queue(UInt(log2Up(b.frontend.rob_entries).W), b.frontend.rob_entries)) - rob.io.complete <> completeQueue.io.deq - completeArb.io.in(0).valid := io.ball_complete_i.valid - completeArb.io.in(0).bits := io.ball_complete_i.bits - io.ball_complete_i.ready := completeArb.io.in(0).ready - completeArb.io.in(1).valid := io.mem_complete_i.valid - completeArb.io.in(1).bits := io.mem_complete_i.bits - io.mem_complete_i.ready := completeArb.io.in(1).ready - - if (b.rvv.enable) { completeArb.io.in(2) <> io.kernel_complete_i.get } - - val completeBits = completeArb.io.out.bits - - if (b.frontend.sub_rob_enable) { - val subRob: Instance[SubROB] = Instantiate(new SubROB(b)) - - val subRobWriteArb = Module(new Arbiter(new SubRobRow(b), b.ballDomain.ballNum)) - for (i <- 0 until b.ballDomain.ballNum) { - subRobWriteArb.io.in(i) <> io.ball_subrob_req_i(i) - } - subRob.io.write <> subRobWriteArb.io.out - - val subRobIssueValid = subRob.io.issue.valid - val subRobCmd = subRob.io.issue.bits - - val subRobIssueEntry = Wire(new GlobalSchedIssue(b)) - subRobIssueEntry.cmd := subRobCmd - subRobIssueEntry.renamedBankAccess := 0.U.asTypeOf(subRobIssueEntry.renamedBankAccess) - subRobIssueEntry.rob_id := subRob.io.issueMasterRobId - subRobIssueEntry.is_sub := true.B - subRobIssueEntry.sub_rob_id := subRob.io.issueSubId - - val subRobIssBall = subRobCmd.domain_id === DomainId.BALL - val subRobIssMem = subRobCmd.domain_id === DomainId.MEM - when(subRobIssueValid) { - assert( - (subRobCmd.cmd.funct =/= MVIN_KERNEL_BITPAT) && (subRobCmd.cmd.funct =/= RUN_KERNEL_BITPAT), - "RVV kernel commands cannot originate from SubROB" - ) - } - - io.ball_issue_o.valid := Mux( - subRobIssueValid && subRobIssBall, - true.B, - rob.io.issue.valid && is_ball_domain && !subRobIssueValid - ) - io.ball_issue_o.bits := Mux(subRobIssueValid && subRobIssBall, subRobIssueEntry, mainIssueEntry) - - io.mem_issue_o.valid := Mux( - subRobIssueValid && subRobIssMem, - true.B, - rob.io.issue.valid && is_mem_domain && !subRobIssueValid - ) - io.mem_issue_o.bits := Mux(subRobIssueValid && subRobIssMem, subRobIssueEntry, mainIssueEntry) - - subRob.io.issue.ready := - (subRobIssBall && io.ball_issue_o.ready) || - (subRobIssMem && io.mem_issue_o.ready) - - if (b.rvv.enable) { io.kernel_issue_o.get.valid := rob.io.issue.valid && is_kernel_domain && !subRobIssueValid } - rob.io.issue.ready := !subRobIssueValid && ( - (is_ball_domain && io.ball_issue_o.ready) || - (is_mem_domain && io.mem_issue_o.ready) || (is_kernel_domain && kernelReady) - ) - rob.io.subRobActive := subRobIssueValid - - subRob.io.subComplete.valid := completeArb.io.out.valid && completeBits.is_sub - subRob.io.subComplete.bits := completeBits.sub_rob_id - - val normalComplete = completeArb.io.out.valid && !completeBits.is_sub - val masterComplete = subRob.io.masterComplete.valid - val completeRobId = Mux(masterComplete, subRob.io.masterComplete.bits, completeBits.rob_id) - - completeQueue.io.enq.valid := masterComplete || normalComplete - completeQueue.io.enq.bits := completeRobId - subRob.io.masterComplete.ready := masterComplete && completeQueue.io.enq.ready - completeArb.io.out.ready := Mux( - completeBits.is_sub, - subRob.io.subComplete.ready, - !masterComplete && completeQueue.io.enq.ready - ) - io.idle := rob.io.empty && !barrierWaitROB && !barrierWaitRelease && !subRob.io.occupied - } else { - for (i <- 0 until b.ballDomain.ballNum) { - io.ball_subrob_req_i(i).ready := false.B - } - - io.ball_issue_o.valid := rob.io.issue.valid && is_ball_domain - io.ball_issue_o.bits := mainIssueEntry - io.mem_issue_o.valid := rob.io.issue.valid && is_mem_domain - io.mem_issue_o.bits := mainIssueEntry - rob.io.issue.ready := (is_ball_domain && io.ball_issue_o.ready) || - (is_mem_domain && io.mem_issue_o.ready) || (is_kernel_domain && kernelReady) - rob.io.subRobActive := false.B - - when(completeArb.io.out.valid) { - assert(!completeBits.is_sub, "SubROB completion observed when frontend.sub_rob_enable=false") - } - completeQueue.io.enq.valid := completeArb.io.out.valid - completeQueue.io.enq.bits := completeBits.rob_id - completeArb.io.out.ready := completeQueue.io.enq.ready - io.idle := rob.io.empty && !barrierWaitROB && !barrierWaitRelease - } - - io.scheduler_rocc_o.resp.valid := false.B - io.scheduler_rocc_o.resp.bits.rd := 0.U - io.scheduler_rocc_o.resp.bits.data := 0.U - io.scheduler_rocc_o.busy := rob.io.full || barrierWaitROB || barrierWaitRelease - -} diff --git a/arch/src/main/scala/framework/frontend/globalrs/SubROB.scala b/arch/src/main/scala/framework/frontend/globalrs/SubROB.scala deleted file mode 100644 index ce5091d5a..000000000 --- a/arch/src/main/scala/framework/frontend/globalrs/SubROB.scala +++ /dev/null @@ -1,206 +0,0 @@ -package framework.frontend.globalrs - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig -import framework.balldomain.blink.SubRobRow -import framework.frontend.decoder.PostGDCmd - -@instantiable -class SubROB(val b: GlobalConfig) extends Module { - val subRobDepth = b.frontend.sub_rob_depth - val subIdBits = log2Up(subRobDepth * 4) - val robIdBits = log2Up(b.frontend.rob_entries) - val ballIdBits = log2Up(b.ballDomain.ballNum) - val rowIdBits = log2Up(subRobDepth) - - @public - val io = IO(new Bundle { - // Ball → SubROB: write a row of sub-instructions - val write = Flipped(Decoupled(new SubRobRow(b))) - // SubROB → GlobalRS: issue one slot at a time (PostGDCmd, domain_id determines target) - val issue = Decoupled(new PostGDCmd(b)) - val issueSubId = Output(UInt(subIdBits.W)) - val issueMasterRobId = Output(UInt(robIdBits.W)) - // GlobalRS → SubROB: a sub-instruction completed (sub_rob_id) - val subComplete = Flipped(Decoupled(UInt(subIdBits.W))) - // SubROB → main ROB: all sub-instructions done - val masterComplete = Decoupled(UInt(robIdBits.W)) - // Status - val occupied = Output(Bool()) - val lockedBallId = Output(UInt(ballIdBits.W)) - }) - - // ------------------------------------------------------------------------- - // Storage - // ------------------------------------------------------------------------- - val sram = SyncReadMem(subRobDepth, new SubRobRow(b)) - - val writePtr = RegInit(0.U(rowIdBits.W)) - val readPtr = RegInit(0.U(rowIdBits.W)) - val rowCount = RegInit(0.U((log2Up(subRobDepth) + 1).W)) - val lockedBallId = RegInit(0.U(ballIdBits.W)) - val masterRobId = RegInit(0.U(robIdBits.W)) - - def nextPtr(p: UInt): UInt = Mux(p === (subRobDepth - 1).U, 0.U, p + 1.U) - - val occupied = rowCount > 0.U - val isFull = rowCount === subRobDepth.U - val ballMatch = !occupied || (io.write.bits.ball_id === lockedBallId) - - // ------------------------------------------------------------------------- - // FSM - // ------------------------------------------------------------------------- - // States: - // sIdle - nothing to do (rowCount == 0) - // sReadReq - issue SRAM read request for readPtr row - // sReadWait - wait 1 cycle for sramRaw to be latched into sramData - // sReadResp - sramData valid; dispatch valid slots one per cycle - // sWaitSlots - all slots issued; wait for all subCompletes - // sWaitMaster - all rows done; fire masterComplete then return to sIdle - val sIdle :: sReadReq :: sReadWait :: sReadResp :: sWaitSlots :: sWaitMaster :: Nil = Enum(6) - val state = RegInit(sIdle) - - // Write path: accept if not full AND ball_id matches (or not occupied yet) - io.write.ready := !isFull && ballMatch && state =/= sReadReq && state =/= sWaitMaster - - when(io.write.fire) { - when(occupied) { - assert(io.write.bits.master_rob_id === masterRobId, "SubROB row master_rob_id mismatch") - } - writePtr := nextPtr(writePtr) - when(!occupied) { - lockedBallId := io.write.bits.ball_id - masterRobId := io.write.bits.master_rob_id - } - } - - // SyncReadMem: read issued in sReadReq (cycle N), data valid on cycle N+1. - // We latch it into sramData with RegEnable so it stays stable during sReadResp - // and sWaitSlots, even when the read port is not enabled. - val sramReadEn = state === sReadReq - - val sramRaw = sram.readWrite( - Mux(io.write.fire, writePtr, readPtr), - io.write.bits, - sramReadEn || io.write.fire, - io.write.fire - ) - - val dataFresh = RegNext(sramReadEn, false.B) // pulse on first cycle of sReadResp - val sramData = RegEnable(sramRaw, dataFresh) - val readPtrReg = RegEnable(readPtr, sramReadEn) - - // Per-row tracking registers (reset when moving to next row) - // slotIssued: have we already sent this slot to a domain? - // slotDone: have we received subComplete for this slot? - val slotIssued = RegInit(VecInit(Seq.fill(4)(false.B))) - val slotDone = RegInit(VecInit(Seq.fill(4)(false.B))) - - // Combinational: which slots still need to be issued? - val slotNeedsIssue = VecInit((0 until 4).map(i => sramData.slots(i).valid && !slotIssued(i))) - val hasSlotToIssue = slotNeedsIssue.asUInt.orR - val firstSlotIdx = PriorityEncoder(slotNeedsIssue.asUInt) - - // Combinational: are all valid slots done? - val allSlotsDone = VecInit((0 until 4).map(i => !sramData.slots(i).valid || slotDone(i))).asUInt.andR - - // Issue output (valid only in sReadResp when there's a slot to issue) - io.issue.valid := (state === sReadResp) && hasSlotToIssue - io.issue.bits := sramData.slots(firstSlotIdx).cmd - io.issueSubId := readPtrReg * 4.U + firstSlotIdx - io.issueMasterRobId := masterRobId - - // subComplete: always ready, mark the corresponding slot done - io.subComplete.ready := true.B - when(io.subComplete.fire) { - val subId = io.subComplete.bits - val subRow = subId / 4.U - val slotIdx = subId(1, 0) - assert( - state === sReadResp || state === sWaitSlots || state === sWaitMaster, - "SubROB subComplete arrived in invalid state" - ) - assert(subRow === readPtrReg, "SubROB subComplete points to non-active row") - slotDone(slotIdx) := true.B - } - - // masterComplete - io.masterComplete.valid := state === sWaitMaster - io.masterComplete.bits := masterRobId - - // Status outputs - io.occupied := occupied - io.lockedBallId := lockedBallId - - // ------------------------------------------------------------------------- - // FSM transitions - // ------------------------------------------------------------------------- - // Helper: advance past current row (called when row is complete) - val rowAdvance = WireDefault(false.B) - - def advanceRow(): Unit = { - readPtr := nextPtr(readPtr) - rowAdvance := true.B - slotIssued.foreach(_ := false.B) - slotDone.foreach(_ := false.B) - } - - switch(state) { - is(sIdle) { - when(occupied)(state := sReadReq) - } - is(sReadReq) { - // SRAM read issued, data available on sramRaw next cycle - state := sReadWait - } - is(sReadWait) { - // sramRaw is valid; RegEnable latches it into sramData at this clock edge. - // sramData will be stable starting next cycle (sReadResp). - state := sReadResp - } - is(sReadResp) { - // Fire issued slot - when(io.issue.fire) { - slotIssued(firstSlotIdx) := true.B - } - // Check if all slots have been issued - // After marking current slot issued, recalculate - val allIssued = VecInit((0 until 4).map(i => - !sramData.slots(i).valid || slotIssued(i) || - (io.issue.fire && firstSlotIdx === i.U) - )).asUInt.andR - - when(allIssued) { - // Check fast path: all already done (e.g., empty row or immediate completion) - val allDoneNow = VecInit((0 until 4).map(i => !sramData.slots(i).valid || slotDone(i))).asUInt.andR - when(allDoneNow) { - // Fast path: skip sWaitSlots - advanceRow() - state := Mux(rowCount === 1.U && !io.write.fire, sWaitMaster, sReadReq) - }.otherwise { - state := sWaitSlots - } - } - // else: stay in sReadResp to dispatch remaining slots - } - is(sWaitSlots) { - when(allSlotsDone) { - advanceRow() - state := Mux(rowCount === 1.U && !io.write.fire, sWaitMaster, sReadReq) - } - } - is(sWaitMaster) { - when(io.masterComplete.fire) { - lockedBallId := 0.U - masterRobId := 0.U - state := sIdle - } - } - } - when(io.write.fire =/= rowAdvance) { - rowCount := Mux(io.write.fire, rowCount + 1.U, rowCount - 1.U) - } - -} diff --git a/arch/src/main/scala/framework/frontend/scoreboard/BankAliasTable.scala b/arch/src/main/scala/framework/frontend/scoreboard/BankAliasTable.scala deleted file mode 100644 index 545b06deb..000000000 --- a/arch/src/main/scala/framework/frontend/scoreboard/BankAliasTable.scala +++ /dev/null @@ -1,126 +0,0 @@ -package framework.frontend.scoreboard - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -/** - * BAT (Bank Alias Table) - * - * Rename virtual bank IDs into the high-ID alias namespace for scoreboard. - * Lifetime policy: per ROB entry. - * - * ID partition: - * [0, vbankUpper] : virtual/architected bank IDs - * [vbankUpper + 1, maxBankId] : renamed alias IDs (one alias per ROB entry) - */ -@instantiable -class BankAliasTable(val bankIdLen: Int, val vbankUpper: Int, val robEntries: Int) extends Module { - val aliasIdLen = bankIdLen - val robIdLen = log2Up(robEntries) - val vbankNum = vbankUpper + 1 - val vbankIdLen = log2Up(vbankNum) - val maxBankId = (1 << bankIdLen) - 1 - val aliasBase = vbankNum - - @public - val io = IO(new Bundle { - - val alloc = Input(new Bundle { - val valid = Bool() - val rob_id = UInt(robIdLen.W) - val raw = new BankAccessInfo(aliasIdLen) - }) - - val alloc_renamed = Output(new BankAccessInfo(aliasIdLen)) - - val free = Input(new Bundle { - val valid = Bool() - val mask = Vec(robEntries, Bool()) - }) - - }) - - // Current architectural alias for each virtual bank. - val v2a = RegInit(VecInit((0 until vbankNum).map(_.U(aliasIdLen.W)))) - - // Extra aliases [aliasBase, aliasBase + robEntries - 1] are one-per-ROB. - val aliasInUse = RegInit(VecInit(Seq.fill(robEntries)(false.B))) - - // Per-entry metadata for commit-time free. - val entHasWrite = RegInit(VecInit(Seq.fill(robEntries)(false.B))) - val entNewAlias = RegInit(VecInit(Seq.fill(robEntries)(0.U(aliasIdLen.W)))) - val entWrVbank = RegInit(VecInit(Seq.fill(robEntries)(0.U(vbankIdLen.W)))) - - require(vbankUpper >= 0, s"BAT vbankUpper must be non-negative, got $vbankUpper") - require(vbankUpper < maxBankId, s"BAT vbankUpper must be < maxBankId($maxBankId), got $vbankUpper") - require( - aliasBase + robEntries - 1 <= maxBankId, - s"BAT alias range exceeds bank_id_len space: base=$aliasBase entries=$robEntries max=$maxBankId" - ) - - private def extraAlias(robId: UInt): UInt = aliasBase.U(aliasIdLen.W) + robId - private def toVbankIdx(v: UInt): UInt = v(vbankIdLen - 1, 0) - - private def mapVbank(v: UInt): UInt = { - val idx = toVbankIdx(v) - val out = WireDefault(0.U(aliasIdLen.W)) - for (i <- 0 until vbankNum) { - when(idx === i.U(vbankIdLen.W)) { - out := v2a(i) - } - } - out - } - - val q = io.alloc.raw - io.alloc_renamed.rd_bank_0_valid := q.rd_bank_0_valid - io.alloc_renamed.rd_bank_1_valid := q.rd_bank_1_valid - io.alloc_renamed.wr_bank_valid := q.wr_bank_valid - io.alloc_renamed.rd_bank_0_id := Mux(q.rd_bank_0_valid, mapVbank(q.rd_bank_0_id), 0.U) - io.alloc_renamed.rd_bank_1_id := Mux(q.rd_bank_1_valid, mapVbank(q.rd_bank_1_id), 0.U) - io.alloc_renamed.wr_bank_id := Mux(q.wr_bank_valid, extraAlias(io.alloc.rob_id), 0.U) - - when(io.alloc.valid) { - val rid = io.alloc.rob_id - when(q.rd_bank_0_valid) { - // assert(q.rd_bank_0_id <= vbankUpper.U, "BAT rd_bank_0_id exceeds virtual bank upper bound") - } - when(q.rd_bank_1_valid) { - // assert(q.rd_bank_1_id <= vbankUpper.U, "BAT rd_bank_1_id exceeds virtual bank upper bound") - } - when(q.wr_bank_valid) { - // assert(q.wr_bank_id <= vbankUpper.U, "BAT wr_bank_id exceeds virtual bank upper bound") - } - - entHasWrite(rid) := q.wr_bank_valid - entNewAlias(rid) := Mux(q.wr_bank_valid, extraAlias(rid), 0.U) - entWrVbank(rid) := toVbankIdx(q.wr_bank_id) - - when(q.wr_bank_valid) { - assert(!aliasInUse(rid), "BAT alias reused before free") - aliasInUse(rid) := true.B - v2a(toVbankIdx(q.wr_bank_id)) := extraAlias(rid) - } - } - - when(io.free.valid) { - for (i <- 0 until robEntries) { - when(io.free.mask(i)) { - when(entHasWrite(i)) { - assert(aliasInUse(i), "BAT free on non-allocated alias") - aliasInUse(i) := false.B - - // Commit is in order. If this is still the current alias, no younger - // write exists and every older write has already committed. - when(v2a(entWrVbank(i)) === entNewAlias(i)) { - v2a(entWrVbank(i)) := entWrVbank(i) - } - } - entHasWrite(i) := false.B - entNewAlias(i) := 0.U - entWrVbank(i) := 0.U - } - } - } -} diff --git a/arch/src/main/scala/framework/frontend/scoreboard/BankScoreboard.scala b/arch/src/main/scala/framework/frontend/scoreboard/BankScoreboard.scala deleted file mode 100644 index a793d3699..000000000 --- a/arch/src/main/scala/framework/frontend/scoreboard/BankScoreboard.scala +++ /dev/null @@ -1,97 +0,0 @@ -package framework.frontend.scoreboard - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -//====----------------------------------------------------------====// -// Bank Scoreboard: tracks in-flight read/write operations per bank. -// -// Hazard rules: -// - Read bank X → requires bankWrBusy(X) == false (RAW) -// - Write bank X → requires bankRdCount(X) == 0 (WAR) -// AND bankWrBusy(X) == false (WAW) -// -// bankRdCount: multi-bit counter (multiple concurrent readers allowed, RR is OK) -// bankWrBusy: 1-bit flag (WAW rule guarantees at most 1 writer in-flight) -// -// Lifecycle: -// - alloc: sets bankWrBusy for the renamed write alias. This must happen -// at alloc time (not issue) because the BAT updates v2a at alloc, -// and subsequent reads of the same vbank will be renamed to the -// new alias immediately. If we only set bankWrBusy at issue, the -// window between alloc and issue lets dependent reads slip through -// the hazard check and issue out-of-order. -// - issue: increments bankRdCount for the renamed read aliases. -// - complete: decrements bankRdCount and clears bankWrBusy. -// -// Issue and complete may fire in the same cycle. Updates are computed -// combinationally so both increments and decrements take effect. -//====----------------------------------------------------------====// - -@instantiable -class BankScoreboard(val bankIdLen: Int, val bankNum: Int, val robEntries: Int) extends Module { - - val cntWidth = log2Ceil(robEntries + 1) - - @public - val alloc = IO(Flipped(Valid(new BankAccessInfo(bankIdLen)))) - @public - val issue = IO(Flipped(Valid(new BankAccessInfo(bankIdLen)))) - @public - val complete = IO(Flipped(Valid(new BankAccessInfo(bankIdLen)))) - @public - val query = IO(Input(new BankAccessInfo(bankIdLen))) - @public - val hasHazard = IO(Output(Bool())) - @public - val queryVec = IO(Input(Vec(robEntries, new BankAccessInfo(bankIdLen)))) - @public - val hazardVec = IO(Output(Vec(robEntries, Bool()))) - - val bankRdCount = RegInit(VecInit(Seq.fill(bankNum)(0.U(cntWidth.W)))) - val bankWrBusy = RegInit(VecInit(Seq.fill(bankNum)(false.B))) - private def bankIndex(id: UInt): UInt = id(log2Ceil(bankNum) - 1, 0) - - // --- Hazard detection (reads current register state) --- - private def hazardOf(q: BankAccessInfo): Bool = { - val rd0_hazard = q.rd_bank_0_valid && bankWrBusy(bankIndex(q.rd_bank_0_id)) - val rd1_hazard = q.rd_bank_1_valid && bankWrBusy(bankIndex(q.rd_bank_1_id)) - // Write hazard is NOT checked here. BAT eliminates WAW/WAR by giving each - // write a unique alias that no other instruction uses. Checking wr_hazard - // would falsely block the writer itself, since alloc marks the writer's - // own alias as busy in the same cycle the entry enters the ROB. - rd0_hazard || rd1_hazard - } - - hasHazard := hazardOf(query) - for (i <- 0 until robEntries) { - hazardVec(i) := hazardOf(queryVec(i)) - } - - // --- Compute per-bank deltas to handle simultaneous issue + complete --- - for (bank <- 0 until bankNum) { - val bankU = bank.U(bankIdLen.W) - - // Read counter: +1 per issue read, -1 per complete read - val issRd0 = issue.valid && issue.bits.rd_bank_0_valid && (issue.bits.rd_bank_0_id === bankU) - val issRd1 = issue.valid && issue.bits.rd_bank_1_valid && (issue.bits.rd_bank_1_id === bankU) - val cmpRd0 = complete.valid && complete.bits.rd_bank_0_valid && (complete.bits.rd_bank_0_id === bankU) - val cmpRd1 = complete.valid && complete.bits.rd_bank_1_valid && (complete.bits.rd_bank_1_id === bankU) - - val rdInc = issRd0.asUInt +& issRd1.asUInt - val rdDec = cmpRd0.asUInt +& cmpRd1.asUInt - bankRdCount(bank) := bankRdCount(bank) + rdInc - rdDec - - // Write flag: alloc sets (alias is reserved as soon as it is renamed), - // complete clears. If both happen to same bank, complete takes priority. - val allocWr = alloc.valid && alloc.bits.wr_bank_valid && (alloc.bits.wr_bank_id === bankU) - val cmpWr = complete.valid && complete.bits.wr_bank_valid && (complete.bits.wr_bank_id === bankU) - - when(allocWr && !cmpWr) { - bankWrBusy(bank) := true.B - }.elsewhen(cmpWr) { - bankWrBusy(bank) := false.B - } - } -} diff --git a/arch/src/main/scala/framework/mem-core/.gitignore b/arch/src/main/scala/framework/ip/.gitignore similarity index 100% rename from arch/src/main/scala/framework/mem-core/.gitignore rename to arch/src/main/scala/framework/ip/.gitignore diff --git a/arch/src/main/scala/framework/root/LICENSE b/arch/src/main/scala/framework/ip/LICENSE similarity index 100% rename from arch/src/main/scala/framework/root/LICENSE rename to arch/src/main/scala/framework/ip/LICENSE diff --git a/arch/src/main/scala/framework/mem-core/README.md b/arch/src/main/scala/framework/ip/README.md similarity index 100% rename from arch/src/main/scala/framework/mem-core/README.md rename to arch/src/main/scala/framework/ip/README.md diff --git a/arch/src/main/scala/framework/ip/axi4/build.sc b/arch/src/main/scala/framework/ip/axi4/build.sc new file mode 100644 index 000000000..11322f523 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/build.sc @@ -0,0 +1,24 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.bus.axi4.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Decoder.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Interconnect.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Interface.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/axi4/configs/default.toml b/arch/src/main/scala/framework/ip/axi4/configs/default.toml new file mode 100644 index 000000000..f2c3f04c6 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/configs/default.toml @@ -0,0 +1,3 @@ +addressBits = 44 +dataBits = 128 +idBits = 4 diff --git a/arch/src/main/scala/framework/ip/axi4/configs/uvm.toml b/arch/src/main/scala/framework/ip/axi4/configs/uvm.toml new file mode 100644 index 000000000..2ec323bbd --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/configs/uvm.toml @@ -0,0 +1,17 @@ +[parameters] +addressBits = 32 +dataBits = 64 +idBits = 3 +masters = 3 +slots = 2 + +[[parameters.regions]] +base="4096" +bytes="256" +[[parameters.regions]] +base="8192" +bytes="512" +[[parameters.regions]] +base="12288" +bytes="1024" +[defines] diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/driver.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/driver.svh new file mode 100644 index 000000000..21b5cf438 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/driver.svh @@ -0,0 +1,41 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual decoder_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual decoder_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Decoder interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.ports[0].aw_valid = req.ports[0].aw_valid; + vif.ports[0].ar_valid = req.ports[0].ar_valid; + vif.ports[0].w_valid = req.ports[0].w_valid; + vif.ports[0].r_ready = req.ports[0].r_ready; + vif.ports[0].b_ready = req.ports[0].b_ready; + vif.ports[0].aw = req.ports[0].aw; + vif.ports[0].ar = req.ports[0].ar; + vif.ports[0].w = req.ports[0].w; + for (int p = 1; p < 3; p++) begin + vif.ports[p].aw_ready = req.ports[p].aw_ready; + vif.ports[p].ar_ready = req.ports[p].ar_ready; + vif.ports[p].w_ready = req.ports[p].w_ready; + vif.ports[p].r_valid = req.ports[p].r_valid; + vif.ports[p].b_valid = req.ports[p].b_valid; + vif.ports[p].r = req.ports[p].r; + vif.ports[p].b = req.ports[p].b; + end + @(posedge vif.clock); + req.ports = vif.ports; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/item.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/item.svh new file mode 100644 index 000000000..cf44232c4 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/item.svh @@ -0,0 +1,17 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + port_signals ports[3]; + bit reset; + function new(string name = "cycle"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_object_utils_end + logic [3*$bits(port_signals)-1:0] value; + function new(string name = "result"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/monitor.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/monitor.svh new file mode 100644 index 000000000..48f06fcb0 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/agent/monitor.svh @@ -0,0 +1,24 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual decoder_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual decoder_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Decoder interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + cycle_item t; + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.ports = vif.ports; + t.reset = vif.reset; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/env/env.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/env/env.svh new file mode 100644 index 000000000..1f8748f8e --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/env/env.svh @@ -0,0 +1,25 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver source; + monitor mon; + model reference; + uvm_sequencer #(cycle_item) seqr; + result_board #(result_item, 0) scoreboard; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + source = driver::type_id::create("source", this); + mon = monitor::type_id::create("monitor", this); + reference = model::type_id::create("model", this); + seqr = new("seqr", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + source.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(reference.analysis_export); + reference.expected.connect(scoreboard.expected_export); + reference.actual.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/env/model.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/env/model.svh new file mode 100644 index 000000000..9fb588c29 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/env/model.svh @@ -0,0 +1,112 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(result_item) expected, actual; + const logic [`AXI_ADDRESS_BITS-1:0] bases[`DECODER_REGIONS] = `DECODER_BASES; + const logic [`AXI_ADDRESS_BITS-1:0] sizes[`DECODER_REGIONS] = `DECODER_SIZES; + bit read_busy, write_busy, data_pending; + int read_target, write_target; + covergroup behavior with function sample ( + int read_target, int write_target, bit concurrent, bit held, bit leading_data, bit reset_busy + ); + option.per_instance = 1; + read_routes: coverpoint read_target {bins all[] = {1, 2};} + write_routes: coverpoint write_target {bins all[] = {1, 2};} + overlap: coverpoint concurrent; + backpressure: coverpoint held; + data_before_address: coverpoint leading_data; + reset_cancellation: coverpoint reset_busy; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected = new("expected", this); + actual = new("actual", this); + behavior = new(); + endfunction + function int branch(logic [`AXI_ADDRESS_BITS-1:0] address); + foreach (bases[i]) if (address >= bases[i] && address - bases[i] < sizes[i]) return 1; + return 2; + endfunction + function logic [3*$bits(port_signals)-1:0] outputs(port_signals ports[3]); + port_signals result[3] = '{default: '0}; + result[0].aw_ready = ports[0].aw_ready; + result[0].ar_ready = ports[0].ar_ready; + result[0].w_ready = ports[0].w_ready; + result[0].r_valid = ports[0].r_valid; + result[0].b_valid = ports[0].b_valid; + if (ports[0].r_valid) result[0].r = ports[0].r; + if (ports[0].b_valid) result[0].b = ports[0].b; + for (int p = 1; p < 3; p++) begin + result[p].aw_valid = ports[p].aw_valid; + result[p].ar_valid = ports[p].ar_valid; + result[p].w_valid = ports[p].w_valid; + result[p].r_ready = ports[p].r_ready; + result[p].b_ready = ports[p].b_ready; + if (ports[p].aw_valid) result[p].aw = ports[p].aw; + if (ports[p].ar_valid) result[p].ar = ports[p].ar; + if (ports[p].w_valid) result[p].w = ports[p].w; + end + return {result[2], result[1], result[0]}; + endfunction + function void write(cycle_item t); + port_signals want[3] = '{default: '0}; + int read_branch, write_branch; + result_item predicted, received; + if (t.reset) begin + behavior.sample(1, 1, 0, 0, 0, read_busy || write_busy); + read_busy = 0; + write_busy = 0; + data_pending = 0; + return; + end + predicted = result_item::type_id::create("expected"); + received = result_item::type_id::create("actual"); + read_branch = branch(t.ports[0].ar.addr); + write_branch = branch(t.ports[0].aw.addr); + want[read_branch].ar_valid = t.ports[0].ar_valid && !read_busy; + want[read_branch].ar = t.ports[0].ar; + want[0].ar_ready = !read_busy && t.ports[read_branch].ar_ready; + want[write_branch].aw_valid = t.ports[0].aw_valid && !write_busy; + want[write_branch].aw = t.ports[0].aw; + want[0].aw_ready = !write_busy && t.ports[write_branch].aw_ready; + if (read_busy) begin + want[0].r_valid = t.ports[read_target].r_valid; + want[0].r = t.ports[read_target].r; + want[read_target].r_ready = t.ports[0].r_ready; + end + if (write_busy) begin + want[0].b_valid = t.ports[write_target].b_valid; + want[0].b = t.ports[write_target].b; + want[write_target].b_ready = t.ports[0].b_ready; + if (data_pending) begin + want[write_target].w_valid = t.ports[0].w_valid; + want[write_target].w = t.ports[0].w; + want[0].w_ready = t.ports[write_target].w_ready; + end + end + predicted.value = outputs(want); + received.value = outputs(t.ports); + expected.write(predicted); + actual.write(received); + behavior.sample( + read_busy ? read_target : read_branch, write_busy ? write_target : write_branch, + read_busy && write_busy, + (t.ports[0].r_valid && !t.ports[0].r_ready) || (t.ports[0].w_valid && !t.ports[0].w_ready), + t.ports[0].w_valid && !write_busy, 0); + if (t.ports[0].ar_valid && t.ports[0].ar_ready) begin + read_busy = 1; + read_target = read_branch; + end + if (t.ports[0].r_valid && t.ports[0].r_ready && t.ports[0].r.last) read_busy = 0; + if (t.ports[0].aw_valid && t.ports[0].aw_ready) begin + write_busy = 1; + data_pending = 1; + write_target = write_branch; + end + if (t.ports[0].w_valid && t.ports[0].w_ready && t.ports[0].w.last) data_pending = 0; + if (t.ports[0].b_valid && t.ports[0].b_ready) write_busy = 0; + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + if (read_busy || write_busy) `uvm_fatal("PENDING", "Decoder transaction did not drain") + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/filelist.f b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/filelist.f new file mode 100644 index 000000000..b71763cde --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/../interconnect/types.svh +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Decoder/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/interface.sv b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/interface.sv new file mode 100644 index 000000000..14a1c9077 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/interface.sv @@ -0,0 +1,7 @@ +interface decoder_if ( + input logic clock +); + import axi_types::*; + logic reset = 1; + port_signals ports[3]; +endinterface diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/package.sv b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/package.sv new file mode 100644 index 000000000..ba6f3829b --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/package.sv @@ -0,0 +1,14 @@ +`include "decoder_regions.svh" +package decoder_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import axi_types::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/seq/sequence.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/seq/sequence.svh new file mode 100644 index 000000000..3f4e481e8 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/seq/sequence.svh @@ -0,0 +1,117 @@ +class scenario_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(scenario_sequence) + port_signals pins[3] = '{default: '0}, observed[3]; + const logic [`AXI_ADDRESS_BITS-1:0] bases[`DECODER_REGIONS] = `DECODER_BASES; + const logic [`AXI_ADDRESS_BITS-1:0] sizes[`DECODER_REGIONS] = `DECODER_SIZES; + int serial; + function new(string name = "scenario"); + super.new(name); + endfunction + task tick(bit reset = 0); + cycle_item t = cycle_item::type_id::create("cycle"); + start_item(t); + t.ports = pins; + t.reset = reset; + finish_item(t); + observed = t.ports; + endtask + task burst(logic [`AXI_ADDRESS_BITS-1:0] read_address, + logic [`AXI_ADDRESS_BITS-1:0] write_address, int beats, bit cancel = 0); + bit read_sent, write_sent; + int rtarget, wtarget; + pins = '{default: '0}; + pins[0].ar = '1; + pins[0].ar.addr = read_address; + pins[0].ar.len = beats - 1; + pins[0].ar.id = I'(serial); + pins[0].ar.burst = 1; + pins[0].aw = '0; + pins[0].aw.addr = write_address; + pins[0].aw.len = beats - 1; + pins[0].aw.id = I'(~serial); + pins[0].aw.burst = 1; + pins[0].w = '1; + pins[0].w.last = beats == 1; + pins[0].w_valid = 1; + repeat (3) tick(); + pins[0].ar_valid = 1; + pins[0].aw_valid = 1; + for (int cycle = 0; !read_sent || !write_sent; cycle++) begin + pins[1].ar_ready = cycle % 3 != 0; + pins[2].ar_ready = cycle % 3 == 0; + pins[1].aw_ready = cycle % 4 == 0; + pins[2].aw_ready = cycle % 4 != 0; + tick(); + if (!read_sent && observed[0].ar_ready) begin + read_sent = 1; + pins[0].ar_valid = 0; + rtarget = observed[1].ar_valid ? 1 : 2; + end + if (!write_sent && observed[0].aw_ready) begin + write_sent = 1; + pins[0].aw_valid = 0; + wtarget = observed[1].aw_valid ? 1 : 2; + end + end + // Offered addresses keep changing after acceptance; response owners stay captured. + pins[0].ar.addr = ~read_address; + pins[0].aw.addr = ~write_address; + if (cancel) begin + repeat (2) tick(1); + pins = '{default: '0}; + tick(); + return; + end + for (int beat = 0; beat < beats; beat++) begin + pins[rtarget].r_valid = 1; + pins[rtarget].r.id = pins[0].ar.id; + pins[rtarget].r.data = D'(64'h123456789abcdef0) ^ beat ^ serial; + pins[rtarget].r.resp = 2'(serial); + pins[rtarget].r.last = beat == beats - 1; + pins[0].w.data = ~pins[rtarget].r.data; + pins[0].w.strb = (beat % 2) ? '1 : 'h55; + pins[0].w.last = beat == beats - 1; + pins[0].w_valid = 1; + // Idle-branch traffic must not leak onto the selected response path. + pins[3-rtarget].r_valid = 1; + pins[3-rtarget].r = '1; + pins[0].r_ready = 0; + pins[wtarget].w_ready = 0; + repeat (3) tick(); + pins[0].r_ready = 1; + pins[wtarget].w_ready = 1; + tick(); + pins[rtarget].r_valid = 0; + pins[3-rtarget].r_valid = 0; + pins[0].w_valid = 0; + end + pins[wtarget].b_valid = 1; + pins[wtarget].b.id = pins[0].aw.id; + pins[wtarget].b.resp = 2'(~serial); + pins[3-wtarget].b_valid = 1; + pins[3-wtarget].b = '1; + pins[0].b_ready = 0; + repeat (3) tick(); + pins[0].b_ready = 1; + tick(); + pins = '{default: '0}; + tick(); + serial++; + endtask + task body(); + repeat (3) tick(1); + tick(); + foreach (bases[i]) begin + burst(bases[i] - 1, bases[i], 1); + burst(bases[i], bases[i] + sizes[i] - 1, 2); + burst(bases[i] + sizes[i] - 1, bases[i] + sizes[i], 3); + burst(bases[i] + sizes[i], bases[i] - 1, 4); + end + for (int i = 0; i < 32; i++) + burst(i % 2 ? bases[i%`DECODER_REGIONS] : 32'h80000000 + i * 64, + i % 3 ? 32'h90000000 + i * 64 : bases[i%`DECODER_REGIONS], 1 + i % 8); + burst(bases[0], 32'h80000000, 4, 1); + burst(32'h80000000, bases[0], 4, 1); + burst(bases[1], bases[2], 4); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/tb.sv b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/tb.sv new file mode 100644 index 000000000..72d32c46d --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/tb.sv @@ -0,0 +1,15 @@ +module decoder_tb; + import uvm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + decoder_if bus (clock); + Decoder dut ( + .clock(clock), + .reset(bus.reset), + `include "decoder_ports.svh" + ); + initial begin + uvm_config_db#(virtual decoder_if)::set(null, "uvm_test_top.env.*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/tests/test.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/tests/test.svh new file mode 100644 index 000000000..54d137a6f --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/decoder/tests/test.svh @@ -0,0 +1,15 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.start(test_env.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/driver.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/driver.svh new file mode 100644 index 000000000..8ec1f6e3e --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/driver.svh @@ -0,0 +1,156 @@ +class driver extends uvm_driver #(batch); + `uvm_component_utils(driver) + virtual bus_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bus_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "AXI driver requires an interface") + endfunction + task run_phase(uvm_phase phase); + batch req; + address posted_reads[$], posted_writes[$]; + int read_beat[$]; + int w_sent[M]; + int w_completed, r_completed, b_completed, cycles; + int read_selected, write_selected; + int ar_sent[M], aw_sent[M]; + bit write_responded[$]; + address a; + for (int m = 0; m < M; m++) begin + vif.ports[m].ar_valid = 0; + vif.ports[m].aw_valid = 0; + vif.ports[m].w_valid = 0; + vif.ports[m].r_ready = 0; + vif.ports[m].b_ready = 0; + vif.ports[m].ar = '0; + vif.ports[m].aw = '0; + vif.ports[m].w = '0; + end + vif.ports[M].ar_ready = 0; + vif.ports[M].aw_ready = 0; + vif.ports[M].w_ready = 0; + vif.ports[M].r_valid = 0; + vif.ports[M].b_valid = 0; + vif.ports[M].r = '0; + vif.ports[M].b = '0; + forever begin + seq_item_port.get_next_item(req); + posted_reads.delete(); + posted_writes.delete(); + read_beat.delete(); + write_responded.delete(); + w_completed = 0; + r_completed = 0; + b_completed = 0; + cycles = 0; + read_selected = -1; + write_selected = -1; + foreach (w_sent[m]) begin + w_sent[m] = 0; + ar_sent[m] = 0; + aw_sent[m] = 0; + end + while (r_completed < 2 * M || b_completed < 2 * M) begin + do @(negedge vif.clock); while (vif.reset); + for (int m = 0; m < M; m++) begin + a = '0; + a.id = req.serial; + a.addr = 'h1000 + req.serial * 'h100000 + m * 'h10000 + ar_sent[m] * 'h1000; + a.len = req.beats - 1; + a.size = $clog2(D / 8); + a.burst = (req.beats == 256 || req.beats == 1) ? 1 : req.serial % 3; + a.cache = req.serial; + a.prot = req.serial; + a.qos = req.serial ^ m; + a.region = ~req.serial; + vif.ports[m].ar = a; + a.addr = 'h1800 + req.serial * 'h100000 + m * 'h10000 + aw_sent[m] * 'h1000; + vif.ports[m].aw = a; + vif.ports[m].ar_valid = ar_sent[m] < 2; + vif.ports[m].aw_valid = aw_sent[m] < 2; + vif.ports[m].w_valid = w_sent[m] < 2 * req.beats; + a.addr = 'h1800 + req.serial * 'h100000 + m * 'h10000 + (w_sent[m] / req.beats) * 'h1000; + vif.ports[m].w.data = read_pattern(a, w_sent[m] % req.beats) ^ 'h55aa; + vif.ports[m].w.strb = (req.serial % 2) ? (w_sent[m] ^ m) : '1; + vif.ports[m].w.last = w_sent[m] % req.beats == req.beats - 1; + vif.ports[m].r_ready = !req.stalls || (cycles + m) % 5 == 0; + vif.ports[m].b_ready = !req.stalls || (cycles + m) % 4 == 0; + end + vif.ports[M].ar_ready = !req.stalls || cycles % 4 == 0; + vif.ports[M].aw_ready = !req.stalls || cycles % 9 == 0; + vif.ports[M].w_ready = !req.stalls || cycles % 3 == 0; + if (read_selected < 0 && cycles > 12) begin + for (int n = posted_reads.size() - 1; n >= 0; n--) begin + if (read_beat[n] <= posted_reads[n].len) begin + read_selected = n; + break; + end + end + end + vif.ports[M].r_valid = read_selected >= 0; + if (read_selected >= 0) begin + a = posted_reads[read_selected]; + vif.ports[M].r.id = a.id; + vif.ports[M].r.data = read_pattern(a, read_beat[read_selected]); + vif.ports[M].r.resp = (req.serial % 3 == 0) ? 0 : (req.serial % 3) + 1; + vif.ports[M].r.last = read_beat[read_selected] == a.len; + end + if (write_selected < 0 && cycles > 15) begin + for (int n = posted_writes.size() - 1; n >= 0; n--) begin + if (n < w_completed && !write_responded[n]) begin + write_selected = n; + break; + end + end + end + vif.ports[M].b_valid = write_selected >= 0; + if (write_selected >= 0) begin + vif.ports[M].b.id = posted_writes[write_selected].id; + vif.ports[M].b.resp = (req.serial % 3 == 0) ? 0 : (req.serial % 3) + 1; + end + @(posedge vif.clock); + for (int m = 0; m < M; m++) begin + if (vif.ports[m].aw_valid && vif.ports[m].aw_ready) aw_sent[m]++; + if (vif.ports[m].ar_valid && vif.ports[m].ar_ready) ar_sent[m]++; + if (vif.ports[m].w_valid && vif.ports[m].w_ready) w_sent[m]++; + end + if (vif.ports[M].ar_valid && vif.ports[M].ar_ready) begin + posted_reads.push_back(vif.ports[M].ar); + read_beat.push_back(0); + end + if (vif.ports[M].aw_valid && vif.ports[M].aw_ready) begin + posted_writes.push_back(vif.ports[M].aw); + write_responded.push_back(0); + end + if (vif.ports[M].w_valid && vif.ports[M].w_ready) begin + if (vif.ports[M].w.last) begin + w_completed++; + end + end + if (vif.ports[M].r_valid && vif.ports[M].r_ready) begin + read_beat[read_selected]++; + if (vif.ports[M].r.last) r_completed++; + read_selected = -1; + end + if (vif.ports[M].b_valid && vif.ports[M].b_ready) begin + write_responded[write_selected] = 1; + b_completed++; + write_selected = -1; + end + cycles++; + end + @(negedge vif.clock); + for (int m = 0; m < M; m++) begin + vif.ports[m].ar_valid = 0; + vif.ports[m].aw_valid = 0; + vif.ports[m].w_valid = 0; + end + vif.ports[M].r_valid = 0; + vif.ports[M].b_valid = 0; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/item.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/item.svh new file mode 100644 index 000000000..b44e5bf61 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/item.svh @@ -0,0 +1,33 @@ +class batch extends uvm_sequence_item; + `uvm_object_utils(batch) + int unsigned serial; + int unsigned beats; + bit stalls; + function new(string name = "batch"); + super.new(name); + endfunction +endclass + +class snapshot extends uvm_sequence_item; + `uvm_object_utils(snapshot) + port_signals ports[M+1]; + int unsigned outstanding; + bit reset; + function new(string name = "sample"); + super.new(name); + endfunction +endclass + +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(payload, UVM_ALL_ON) + `uvm_field_int(channel, UVM_ALL_ON) + `uvm_field_int(owner, UVM_ALL_ON) + `uvm_object_utils_end + logic [D+128:0] payload; + int channel; + int owner; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/monitor.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/monitor.svh new file mode 100644 index 000000000..fe3b2ec5d --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/agent/monitor.svh @@ -0,0 +1,25 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual bus_if vif; + uvm_analysis_port #(snapshot) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual bus_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "AXI monitor requires an interface") + endfunction + task run_phase(uvm_phase phase); + snapshot observed; + forever begin + @(posedge vif.clock); + observed = snapshot::type_id::create("observed"); + observed.ports = vif.ports; + observed.reset = vif.reset; + observed.outstanding = vif.outstanding; + ap.write(observed); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/env/env.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/env/env.svh new file mode 100644 index 000000000..1c167dfec --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/env/env.svh @@ -0,0 +1,27 @@ +class env extends uvm_env; + `uvm_component_utils(env) + driver drv; + monitor mon; + uvm_sequencer #(batch) seqr; + routing_model model; + result_board #(result_item, 0) scoreboard; + reset_controller reset; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + seqr = new("seqr", this); + model = routing_model::type_id::create("model", this); + scoreboard = result_board#(result_item, 0)::type_id::create("scoreboard", this); + reset = reset_controller::type_id::create("reset", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + mon.ap.connect(model.analysis_export); + model.expected_ap.connect(scoreboard.expected_export); + model.actual_ap.connect(scoreboard.actual_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/env/model.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/env/model.svh new file mode 100644 index 000000000..f30a78105 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/env/model.svh @@ -0,0 +1,186 @@ +class routing_model extends uvm_subscriber #(snapshot); + `uvm_component_utils(routing_model) + typedef struct { + address addr; + int master; + int beats; + } transaction; + transaction reads[$], writes[$], owners[$]; + transaction read_slots[int], write_slots[int]; + transaction read_order[longint][$], write_order[longint][$]; + int live_reads = 0, live_writes = 0; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + covergroup behavior with function sample (int length, bit read, int owner, int response_code); + option.per_instance = 1; + burst_length: coverpoint length { + bins one = {1}; bins short_burst = {[2 : 16]}; bins maximum = {256}; + } + direction: coverpoint read; + master: coverpoint owner {bins each_master[] = {[0 : M - 1]};} + response: coverpoint response_code {bins codes[] = {0, 2, 3};} + endgroup + covergroup load with function sample (int active, int live, bit stalled); + option.per_instance = 1; + concurrency: coverpoint active {bins single = {1}; bins competing = {[2 : M]};} + occupancy: coverpoint live { + bins full = {2 * `AXI_SLOTS}; bins active = {[1 : 2 * `AXI_SLOTS - 1]}; + } + backpressure: coverpoint stalled; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + behavior = new(); + load = new(); + endfunction + function void compare_value(logic [D+128:0] expected, logic [D+128:0] actual, int channel, + int owner); + result_item e = result_item::type_id::create("expected"); + result_item a = result_item::type_id::create("actual"); + e.payload = expected; + a.payload = actual; + e.channel = channel; + a.channel = channel; + e.owner = owner; + a.owner = owner; + expected_ap.write(e); + actual_ap.write(a); + endfunction + function void write(snapshot s); + transaction t; + address original; + int active, routed, slot; + longint key; + transaction retired; + read_data response_data; + response response_status; + if (s.reset) begin + evidence::check(live_reads == 0 && live_writes == 0, "Reset with unretired AXI transactions"); + return; + end + evidence::check(s.outstanding == live_reads + live_writes, "AXI outstanding count mismatch"); + active = 0; + for (int m = 0; m < M; m++) begin + if (s.ports[m].ar_valid || s.ports[m].aw_valid) active++; + if (s.ports[m].ar_valid && s.ports[m].ar_ready) begin + t.addr = s.ports[m].ar; + t.master = m; + t.beats = 0; + key = (longint'(m) << I) | t.addr.id; + if (!read_order.exists(key)) read_order[key] = {}; + read_order[key].push_back(t); + reads.push_back(t); + live_reads++; + end + if (s.ports[m].aw_valid && s.ports[m].aw_ready) begin + t.addr = s.ports[m].aw; + t.master = m; + t.beats = 0; + key = (longint'(m) << I) | t.addr.id; + if (!write_order.exists(key)) write_order[key] = {}; + write_order[key].push_back(t); + writes.push_back(t); + owners.push_back(t); + live_writes++; + end + end + load.sample(active, s.outstanding, + (s.ports[M].ar_valid && !s.ports[M].ar_ready) || + (s.ports[M].aw_valid && !s.ports[M].aw_ready) || + (s.ports[M].r_valid && !s.ports[M].r_ready) || + (s.ports[M].b_valid && !s.ports[M].b_ready)); + if (s.ports[M].ar_valid && s.ports[M].ar_ready) begin + if (reads.size() == 0) `uvm_fatal("AR", "Unrequested downstream read") + t = reads.pop_front(); + original = s.ports[M].ar; + slot = original.id; + original.id = t.addr.id; + evidence::check(slot < `AXI_SLOTS && !read_slots.exists(slot), + "Read slot reused before retirement"); + compare_value(t.addr, original, 0, t.master); + read_slots[slot] = t; + end + if (s.ports[M].aw_valid && s.ports[M].aw_ready) begin + if (writes.size() == 0) `uvm_fatal("AW", "Unrequested downstream write") + t = writes.pop_front(); + original = s.ports[M].aw; + slot = original.id; + original.id = t.addr.id; + evidence::check(slot < `AXI_SLOTS && !write_slots.exists(slot), + "Write slot reused before retirement"); + compare_value(t.addr, original, 1, t.master); + write_slots[slot] = t; + end + routed = 0; + for (int m = 0; m < M; m++) + if (s.ports[m].w_valid && s.ports[m].w_ready) begin + if (owners.size() == 0) `uvm_fatal("W", "Write data without an accepted address") + evidence::check(owners[0].master == m && s.ports[M].w_valid && s.ports[M].w_ready, + "Write data routed to the wrong owner"); + compare_value(s.ports[m].w, s.ports[M].w, 2, m); + evidence::check(s.ports[m].w.last == (owners[0].beats == owners[0].addr.len), + "Write burst length mismatch"); + owners[0].beats++; + if (s.ports[m].w.last) t = owners.pop_front(); + routed++; + end + evidence::check(routed == int'(s.ports[M].w_valid && s.ports[M].w_ready), + "Lost or duplicated W beat"); + routed = 0; + for (int m = 0; m < M; m++) + if (s.ports[m].r_valid && s.ports[m].r_ready) begin + slot = s.ports[M].r.id; + if (!read_slots.exists(slot)) `uvm_fatal("R", "Response for an unissued read") + t = read_slots[slot]; + key = (longint'(t.master) << I) | t.addr.id; + evidence::check(read_order[key][0].addr == t.addr, + "Read responses reordered within an original ID"); + evidence::check(t.master == m && s.ports[M].r_valid && s.ports[M].r_ready, + "Read response routed to the wrong master"); + response_data = s.ports[M].r; + response_data.id = t.addr.id; + compare_value(response_data, s.ports[m].r, 3, m); + evidence::check(response_data.last == (t.beats == t.addr.len), "Read burst length mismatch"); + behavior.sample(t.addr.len + 1, 1, m, response_data.resp); + read_slots[slot].beats++; + if (response_data.last) begin + retired = read_order[key].pop_front(); + if (read_order[key].size() == 0) read_order.delete(key); + read_slots.delete(slot); + live_reads--; + end + routed++; + end + evidence::check(routed == int'(s.ports[M].r_valid && s.ports[M].r_ready), + "Lost or duplicated R beat"); + routed = 0; + for (int m = 0; m < M; m++) + if (s.ports[m].b_valid && s.ports[m].b_ready) begin + slot = s.ports[M].b.id; + if (!write_slots.exists(slot)) `uvm_fatal("B", "Response for an unissued write") + t = write_slots[slot]; + key = (longint'(t.master) << I) | t.addr.id; + evidence::check(write_order[key][0].addr == t.addr, + "Write responses reordered within an original ID"); + evidence::check(t.master == m && s.ports[M].b_valid && s.ports[M].b_ready, + "Write response routed to the wrong master"); + response_status = s.ports[M].b; + response_status.id = t.addr.id; + compare_value(response_status, s.ports[m].b, 4, m); + behavior.sample(t.addr.len + 1, 0, m, response_status.resp); + retired = write_order[key].pop_front(); + if (write_order[key].size() == 0) write_order.delete(key); + write_slots.delete(slot); + live_writes--; + routed++; + end + evidence::check(routed == int'(s.ports[M].b_valid && s.ports[M].b_ready), + "Lost or duplicated B response"); + endfunction + function void check_phase(uvm_phase phase); + super.check_phase(phase); + evidence::check(live_reads == 0 && live_writes == 0 && owners.size() == 0, + "AXI transactions remain outstanding"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/filelist.f b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/filelist.f new file mode 100644 index 000000000..92d911b88 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/types.svh +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Interconnect/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/interface.sv b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/interface.sv new file mode 100644 index 000000000..de9d57756 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/interface.sv @@ -0,0 +1,35 @@ +interface bus_if ( + input logic clock, + input logic reset +); + import axi_types::*; + port_signals ports[M+1]; + logic [$clog2(2*`AXI_SLOTS+1)-1:0] outstanding; + for (genvar p = 0; p <= M; p++) begin + assert property (@(posedge clock) disable iff (reset) + ports[p].aw_valid && !ports[p].aw_ready |=> ports[p].aw_valid && $stable( + ports[p].aw + )) + else $fatal(1, "AXI AW changed while stalled"); + assert property (@(posedge clock) disable iff (reset) + ports[p].ar_valid && !ports[p].ar_ready |=> ports[p].ar_valid && $stable( + ports[p].ar + )) + else $fatal(1, "AXI AR changed while stalled"); + assert property (@(posedge clock) disable iff (reset) + ports[p].w_valid && !ports[p].w_ready |=> ports[p].w_valid && $stable( + ports[p].w + )) + else $fatal(1, "AXI W changed while stalled"); + assert property (@(posedge clock) disable iff (reset) + ports[p].r_valid && !ports[p].r_ready |=> ports[p].r_valid && $stable( + ports[p].r + )) + else $fatal(1, "AXI R changed while stalled"); + assert property (@(posedge clock) disable iff (reset) + ports[p].b_valid && !ports[p].b_ready |=> ports[p].b_valid && $stable( + ports[p].b + )) + else $fatal(1, "AXI B changed while stalled"); + end +endinterface diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/package.sv b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/package.sv new file mode 100644 index 000000000..fb40ddb20 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/package.sv @@ -0,0 +1,13 @@ +package interconnect_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import axi_types::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/seq/sequence.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/seq/sequence.svh new file mode 100644 index 000000000..f0e0c7076 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/seq/sequence.svh @@ -0,0 +1,22 @@ +class traffic extends uvm_sequence #(batch); + `uvm_object_utils(traffic) + function new(string name = "traffic"); + super.new(name); + endfunction + task body(); + batch req; + for (int n = 0; n < 16; n++) begin + req = batch::type_id::create("batch"); + start_item(req); + req.serial = n; + case (n % 4) + 0: req.beats = 1; + 1: req.beats = 2; + 2: req.beats = 16; + 3: req.beats = 256; + endcase + req.stalls = n >= 8; + finish_item(req); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/tb.sv b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/tb.sv new file mode 100644 index 000000000..a1a0bbb1a --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/tb.sv @@ -0,0 +1,22 @@ +module interconnect_tb; + import uvm_pkg::*; + import interconnect_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + bus_if bus ( + clock, + timing.reset + ); + Interconnect dut ( + .clock(clock), + .reset(timing.reset), + .io_outstanding(bus.outstanding), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual bus_if)::set(null, "uvm_test_top.env.*", "vif", bus); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top.env.reset", "timing", timing); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/tests/test.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/tests/test.svh new file mode 100644 index 000000000..b3eb6a6e9 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/tests/test.svh @@ -0,0 +1,17 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + traffic seq = traffic::type_id::create("traffic"); + test_env.reset.pulse(4); + seq.start(test_env.seqr); + test_env.reset.pulse(2); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/types.svh b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/types.svh new file mode 100644 index 000000000..19f28a2ce --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/interconnect/types.svh @@ -0,0 +1,46 @@ +`include "parameters.svh" +package axi_types; + localparam int M = `AXI_MASTERS; + localparam int D = `AXI_DATA_BITS; + localparam int I = `AXI_ID_BITS; + typedef struct packed { + logic [I-1:0] id; + logic [`AXI_ADDRESS_BITS-1:0] addr; + logic [7:0] len; + logic [2:0] size; + logic [1:0] burst; + logic lock; + logic [3:0] cache; + logic [2:0] prot; + logic [3:0] qos; + logic [3:0] region; + } address; + typedef struct packed { + logic [D-1:0] data; + logic [D/8-1:0] strb; + logic last; + } write_data; + typedef struct packed { + logic [I-1:0] id; + logic [D-1:0] data; + logic [1:0] resp; + logic last; + } read_data; + typedef struct packed { + logic [I-1:0] id; + logic [1:0] resp; + } response; + typedef struct packed { + logic aw_valid, aw_ready, ar_valid, ar_ready, w_valid, w_ready; + logic r_valid, r_ready, b_valid, b_ready; + address aw, ar; + write_data w; + read_data r; + response b; + } port_signals; + function automatic logic [D-1:0] read_pattern(address addr, int beat); + logic [D-1:0] value; + for (int b = 0; b < D / 8; b++) value[b*8+:8] = (addr.addr >> 3) ^ (beat * 17) ^ b; + return value; + endfunction +endpackage diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/axi4/src/main/resources/suite.toml new file mode 100644 index 000000000..862416c63 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/resources/suite.toml @@ -0,0 +1,37 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "interconnect" +top = "interconnect_tb" +test = "protocol_test" +filelist = "interconnect/filelist.f" +timeout_ns = 2000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["interconnect_pkg::routing_model::behavior", "interconnect_pkg::routing_model::load"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +name="decoder" +top="decoder_tb" +test="protocol_test" +filelist="decoder/filelist.f" +timeout_ns=200000 +wall_timeout_s=120 +kind="functional" +rtl_scopes=["dut"] +metrics=["line","cond","tgl"] +functional_groups=["decoder_pkg::model::behavior"] +sink_ready=1 +sink_stall=3 +designs=[] +plusargs=[] +assertion="" diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/scala/Decoder.scala b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Decoder.scala new file mode 100644 index 000000000..14caed4af --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Decoder.scala @@ -0,0 +1,64 @@ +package framework.ip.bus.axi4 + +import framework.ip.bus.axi4.{Params => AxiParam} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +/** Address-selected AXI branch. One read burst and one write burst may be live independently. */ +@instantiable +class Decoder(val regions: Seq[(BigInt, BigInt)], axiParam: AxiParam) extends Module { + val p = axiParam + + @public + val io = IO(new Bundle { + val in = Flipped(new Port(axiParam)) + val matched = new Port(axiParam) + val other = new Port(axiParam) + }) + + val readBusy = RegInit(false.B) + val writeDataPending = RegInit(false.B) + val writeBusy = RegInit(false.B) + val readMatched = Reg(Bool()) + val writeMatched = Reg(Bool()) + + val arMatched = regions.map { case (base, bytes) => + io.in.ar.bits.addr >= base.U && io.in.ar.bits.addr < (base + bytes).U + }.reduce(_ || _) + + val awMatched = regions.map { case (base, bytes) => + io.in.aw.bits.addr >= base.U && io.in.aw.bits.addr < (base + bytes).U + }.reduce(_ || _) + + io.matched.ar.valid := io.in.ar.valid && !readBusy && arMatched + io.other.ar.valid := io.in.ar.valid && !readBusy && !arMatched + io.matched.ar.bits := io.in.ar.bits + io.other.ar.bits := io.in.ar.bits + io.in.ar.ready := !readBusy && Mux(arMatched, io.matched.ar.ready, io.other.ar.ready) + when(io.in.ar.fire) { readBusy := true.B; readMatched := arMatched } + io.in.r.valid := readBusy && Mux(readMatched, io.matched.r.valid, io.other.r.valid) + io.in.r.bits := Mux(readMatched, io.matched.r.bits, io.other.r.bits) + io.matched.r.ready := readBusy && readMatched && io.in.r.ready + io.other.r.ready := readBusy && !readMatched && io.in.r.ready + when(io.in.r.fire && io.in.r.bits.last)(readBusy := false.B) + + io.matched.aw.valid := io.in.aw.valid && !writeBusy && awMatched + io.other.aw.valid := io.in.aw.valid && !writeBusy && !awMatched + io.matched.aw.bits := io.in.aw.bits + io.other.aw.bits := io.in.aw.bits + io.in.aw.ready := !writeBusy && Mux(awMatched, io.matched.aw.ready, io.other.aw.ready) + when(io.in.aw.fire) { writeBusy := true.B; writeDataPending := true.B; writeMatched := awMatched } + io.matched.w.valid := io.in.w.valid && writeBusy && writeDataPending && writeMatched + io.other.w.valid := io.in.w.valid && writeBusy && writeDataPending && !writeMatched + io.matched.w.bits := io.in.w.bits + io.other.w.bits := io.in.w.bits + io.in.w.ready := writeBusy && writeDataPending && Mux(writeMatched, io.matched.w.ready, io.other.w.ready) + when(io.in.w.fire && io.in.w.bits.last)(writeDataPending := false.B) + io.in.b.valid := writeBusy && Mux(writeMatched, io.matched.b.valid, io.other.b.valid) + io.in.b.bits := Mux(writeMatched, io.matched.b.bits, io.other.b.bits) + io.matched.b.ready := writeBusy && writeMatched && io.in.b.ready + io.other.b.ready := writeBusy && !writeMatched && io.in.b.ready + when(io.in.b.fire)(writeBusy := false.B) +} diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Emit.scala new file mode 100644 index 000000000..0bd41da82 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Emit.scala @@ -0,0 +1,72 @@ +package framework.ip.bus.axi4 + +import java.nio.file.{Files, Paths} + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val p = Params(values("addressBits").num.toInt, values("dataBits").num.toInt, values("idBits").num.toInt) + val masters = values("masters").num.toInt + val slots = values("slots").num.toInt + Files.createDirectories(Paths.get(output)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Interconnect(InterconnectParam(p, masters, Some(slots))), + args = Array("--target-dir", s"$output/Interconnect", "--split-verilog") + ) + Files.writeString( + Paths.get(output, "parameters.svh"), + s"""`ifndef AXI_PARAMETERS +`define AXI_PARAMETERS +`define AXI_ADDRESS_BITS ${p.addressBits} +`define AXI_DATA_BITS ${p.dataBits} +`define AXI_ID_BITS ${p.idBits} +`define AXI_MASTERS $masters +`define AXI_SLOTS $slots +`endif +""" + ) + + val fields = Seq( + "aw" -> Seq("id", "addr", "len", "size", "burst", "lock", "cache", "prot", "qos", "region"), + "ar" -> Seq("id", "addr", "len", "size", "burst", "lock", "cache", "prot", "qos", "region"), + "w" -> Seq("data", "strb", "last"), + "r" -> Seq("id", "data", "resp", "last"), + "b" -> Seq("id", "resp") + ) + + val ports = (0 to masters).flatMap { index => + val prefix = if (index == masters) "io_out" else s"io_in_$index" + fields.flatMap { case (channel, members) => + Seq( + s".${prefix}_${channel}_valid(bus.ports[$index].${channel}_valid)", + s".${prefix}_${channel}_ready(bus.ports[$index].${channel}_ready)" + ) ++ + members.map(field => s".${prefix}_${channel}_bits_$field(bus.ports[$index].$channel.$field)") + } + } + + Files.writeString(Paths.get(output, "ports.svh"), ports.mkString(",\n")) + val regions = values("regions").arr.toSeq.map(v => (BigInt(v("base").str), BigInt(v("bytes").str))) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Decoder(regions, p), + args = Array("--target-dir", s"$output/Decoder", "--split-verilog") + ) + + val decoderPorts = Seq("in", "matched", "other").zipWithIndex.flatMap { case (prefix, index) => + fields.flatMap { case (channel, members) => + Seq( + s".io_${prefix}_${channel}_valid(bus.ports[$index].${channel}_valid)", + s".io_${prefix}_${channel}_ready(bus.ports[$index].${channel}_ready)" + ) ++ members.map(field => s".io_${prefix}_${channel}_bits_$field(bus.ports[$index].$channel.$field)") + } + } + + Files.writeString(Paths.get(output, "decoder_ports.svh"), decoderPorts.mkString(",\n")) + Files.writeString( + Paths.get(output, "decoder_regions.svh"), + s"`define DECODER_REGIONS ${regions.size}\n" + + s"`define DECODER_BASES ${regions.map { case (base, _) => s"${p.addressBits}'h${base.toString(16)}" }.mkString("'{", ",", "}")}\n" + + s"`define DECODER_SIZES ${regions.map { case (_, size) => s"${p.addressBits}'h${size.toString(16)}" }.mkString("'{", ",", "}")}\n" + ) + +} diff --git a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Interconnect.scala b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Interconnect.scala similarity index 94% rename from arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Interconnect.scala rename to arch/src/main/scala/framework/ip/axi4/src/main/scala/Interconnect.scala index f16874d9f..adb987dd4 100644 --- a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Interconnect.scala +++ b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Interconnect.scala @@ -1,6 +1,8 @@ -package memcore.bus.axi4 +package framework.ip.bus.axi4 -import memcore.memory.queue.Queue +import framework.ip.bus.axi4.InterconnectParam + +import framework.lib.queue.Queue import chisel3._ import chisel3.util._ @@ -18,7 +20,8 @@ class AddressSelector(p: Params, masters: Int) extends Module { require(masters > 0) val masterBits = math.max(1, log2Ceil(masters)) - @public val io = IO(new Bundle { + @public + val io = IO(new Bundle { val in = Vec(masters, Flipped(Decoupled(new Address(p)))) val enabled = Input(Vec(masters, Bool())) val slot = Input(UInt(p.idBits.W)) @@ -59,9 +62,15 @@ class WriteOwner(p: Params, masters: Int) extends Bundle { val slot = UInt(p.idBits.W) } +case class InterconnectParam(bus: Params, masters: Int, slots: Option[Int]) + /** Normal AXI4 transfers, dynamically mapped onto a bounded DDR ID space. */ @instantiable -class Interconnect(p: Params, masters: Int, slots: Option[Int] = None) extends Module { +class Interconnect(b: InterconnectParam) extends Module { + val config = b + val p = config.bus + val masters = config.masters + val slots = config.slots require(masters > 0) val capacity = slots.getOrElse { @@ -73,7 +82,8 @@ class Interconnect(p: Params, masters: Int, slots: Option[Int] = None) extends M require(capacity > 1 && BigInt(capacity) <= (BigInt(1) << p.idBits)) val masterBits = math.max(1, log2Ceil(masters)) - @public val io = IO(new Bundle { + @public + val io = IO(new Bundle { val in = Vec(masters, Flipped(new Port(p))) val out = new Port(p) val outstanding = Output(UInt(log2Ceil(2 * capacity + 1).W)) @@ -95,7 +105,7 @@ class Interconnect(p: Params, masters: Int, slots: Option[Int] = None) extends M val reads: Instance[AddressSelector] = Instantiate(new AddressSelector(p, masters)) val writes: Instance[AddressSelector] = Instantiate(new AddressSelector(p, masters)) - val owners = Module(new Queue(new WriteOwner(p, masters), capacity)) + val owners = Instantiate(new Queue(new WriteOwner(p, masters), capacity)) val readFree = VecInit(readLive.map(live => !live)) val writeFree = VecInit(writeLive.map(live => !live)) reads.io.slot := PriorityEncoder(readFree) diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/scala/Interface.scala b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Interface.scala new file mode 100644 index 000000000..b39ba1fe7 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Interface.scala @@ -0,0 +1,44 @@ +package framework.ip.bus.axi4 + +import chisel3._ +import chisel3.util._ + +class Address(p: Params) extends Bundle { + val id = UInt(p.idBits.W) + val addr = UInt(p.addressBits.W) + val len = UInt(8.W) + val size = UInt(3.W) + val burst = UInt(2.W) + val lock = Bool() + val cache = UInt(4.W) + val prot = UInt(3.W) + val qos = UInt(4.W) + val region = UInt(4.W) +} + +class WriteData(p: Params) extends Bundle { + val data = UInt(p.dataBits.W) + val strb = UInt(p.bytes.W) + val last = Bool() +} + +class ReadData(p: Params) extends Bundle { + val id = UInt(p.idBits.W) + val data = UInt(p.dataBits.W) + val resp = UInt(2.W) + val last = Bool() +} + +class WriteResponse(p: Params) extends Bundle { + val id = UInt(p.idBits.W) + val resp = UInt(2.W) +} + +/** AXI4 memory-mapped master. Optional user sidebands are absent. */ +class Port(p: Params) extends Bundle { + val aw = Decoupled(new Address(p)) + val w = Decoupled(new WriteData(p)) + val b = Flipped(Decoupled(new WriteResponse(p))) + val ar = Decoupled(new Address(p)) + val r = Flipped(Decoupled(new ReadData(p))) +} diff --git a/arch/src/main/scala/framework/ip/axi4/src/main/scala/Params.scala b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Params.scala new file mode 100644 index 000000000..3d3a32ede --- /dev/null +++ b/arch/src/main/scala/framework/ip/axi4/src/main/scala/Params.scala @@ -0,0 +1,17 @@ +package framework.ip.bus.axi4 + +import upickle.default.{macroRW, ReadWriter} + +import buckyball.config.AxiHardware + +case class Params(addressBits: Int = 44, dataBits: Int = 128, idBits: Int = 4) { + require(addressBits > 0 && idBits > 0) + require(dataBits >= 8 && dataBits <= 1024 && (dataBits & (dataBits - 1)) == 0) + val bytes = dataBits / 8 +} + +object Params { + implicit val rw: ReadWriter[Params] = macroRW + + def fromProto(p: AxiHardware): Params = Params(p.getAddressBits, p.getDataBits, p.getIdBits) +} diff --git a/arch/src/main/scala/framework/ip/axis/build.sc b/arch/src/main/scala/framework/ip/axis/build.sc new file mode 100644 index 000000000..33bf0bcf7 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/build.sc @@ -0,0 +1,24 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.bus.axi.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Interface.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Master.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/PacketArbiter.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Slave.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/axis/configs/uvm.toml b/arch/src/main/scala/framework/ip/axis/configs/uvm.toml new file mode 100644 index 000000000..6d370b490 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/configs/uvm.toml @@ -0,0 +1,6 @@ +[parameters] +dataBits = 32 +inputs = 2 + +[defines] +AXIS_DATA_BITS = "dataBits" diff --git a/arch/src/main/scala/framework/mem-core/axis/src/csrc/Cargo.lock b/arch/src/main/scala/framework/ip/axis/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/axis/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/ip/axis/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/axis/src/csrc/Cargo.toml b/arch/src/main/scala/framework/ip/axis/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/axis/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/ip/axis/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/README.md b/arch/src/main/scala/framework/ip/axis/src/csrc/README.md similarity index 100% rename from arch/src/main/scala/framework/mem-core/axis/src/main/resources/README.md rename to arch/src/main/scala/framework/ip/axis/src/csrc/README.md diff --git a/arch/src/main/scala/framework/ip/axis/src/csrc/src/arbiter.rs b/arch/src/main/scala/framework/ip/axis/src/csrc/src/arbiter.rs new file mode 100644 index 000000000..853d9cacc --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/csrc/src/arbiter.rs @@ -0,0 +1,23 @@ +#[derive(Default)] +pub(crate) struct ArbiterModel { + selected: Option, +} + +impl ArbiterModel { + pub(crate) fn step(&mut self, valid: u32, ready: bool, last: u32) -> Option { + let selected = self + .selected + .or_else(|| (valid != 0).then(|| valid.trailing_zeros())); + let selected = selected?; + if valid & (1 << selected) == 0 { + return None; + } + + self.selected = if ready && last & (1 << selected) != 0 { + None + } else { + Some(selected) + }; + Some(selected) + } +} diff --git a/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/ffi.rs b/arch/src/main/scala/framework/ip/axis/src/csrc/src/ffi.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/axis/src/csrc/src/ffi.rs rename to arch/src/main/scala/framework/ip/axis/src/csrc/src/ffi.rs diff --git a/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/lib.rs b/arch/src/main/scala/framework/ip/axis/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/axis/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/ip/axis/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/ip/axis/src/csrc/src/model.rs b/arch/src/main/scala/framework/ip/axis/src/csrc/src/model.rs new file mode 100644 index 000000000..3b9bd7f47 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/csrc/src/model.rs @@ -0,0 +1,23 @@ +use std::collections::VecDeque; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Beat { + pub(crate) data: u32, + pub(crate) keep: u8, + pub(crate) last: bool, +} + +#[derive(Default)] +pub(crate) struct AxiSModel { + beats: VecDeque, +} + +impl AxiSModel { + pub(crate) fn push(&mut self, beat: Beat) { + self.beats.push_back(beat); + } + + pub(crate) fn pop(&mut self) -> Option { + self.beats.pop_front() + } +} diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/agent.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/agent.svh new file mode 100644 index 000000000..bb76bdeab --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/agent.svh @@ -0,0 +1,22 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(cycle_item) seqr; + driver drv; + monitor mon; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(cycle_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/driver.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/driver.svh new file mode 100644 index 000000000..9322d5f01 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/driver.svh @@ -0,0 +1,37 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual arbiter_if vif; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual arbiter_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Arbiter driver requires vif") + endfunction + + task run_phase(uvm_phase phase); + cycle_item req; + vif.reset = 1; + vif.in_valid = 0; + vif.in_last = 0; + vif.in_data = '{0, 0}; + vif.in_keep = '{0, 0}; + vif.out_ready = 0; + forever begin + seq_item_port.get_next_item(req); + // Drive on falling edges; monitor and DUT sample on rising edges. + @(negedge vif.clock); + vif.reset = req.reset; + vif.in_valid = req.valid; + vif.in_data = req.data; + vif.in_keep = req.keep; + vif.in_last = req.last; + vif.out_ready = req.ready; + @(posedge vif.clock); + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/item.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/item.svh new file mode 100644 index 000000000..d73806502 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/item.svh @@ -0,0 +1,40 @@ +class cycle_item extends uvm_sequence_item; + bit reset; + logic [1:0] valid; + logic [`AXIS_DATA_BITS-1:0] data[2]; + logic [`AXIS_DATA_BITS/8-1:0] keep[2]; + logic [1:0] last; + logic ready; + logic [1:0] accepted; + logic output_valid; + logic output_last; + `uvm_object_utils_begin(cycle_item) + `uvm_field_int(reset, UVM_ALL_ON) + `uvm_field_int(valid, UVM_ALL_ON) + `uvm_field_sarray_int(data, UVM_ALL_ON) + `uvm_field_sarray_int(keep, UVM_ALL_ON) + `uvm_field_int(last, UVM_ALL_ON) + `uvm_field_int(ready, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass + +class result_item extends uvm_sequence_item; + logic valid; + logic [1:0] ready; + logic [`AXIS_DATA_BITS-1:0] data; + logic [`AXIS_DATA_BITS/8-1:0] keep; + logic last; + `uvm_object_utils_begin(result_item) + `uvm_field_int(valid, UVM_ALL_ON) + `uvm_field_int(ready, UVM_ALL_ON) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(keep, UVM_ALL_ON) + `uvm_field_int(last, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "result_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/monitor.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/monitor.svh new file mode 100644 index 000000000..5f5341fda --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/agent/monitor.svh @@ -0,0 +1,50 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual arbiter_if vif; + uvm_analysis_port #(cycle_item) input_ap; + uvm_analysis_port #(result_item) output_ap; + int unsigned accepted = 0; + int unsigned transferred = 0; + + function new(string name, uvm_component parent); + super.new(name, parent); + input_ap = new("input_ap", this); + output_ap = new("output_ap", this); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual arbiter_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Arbiter monitor requires vif") + endfunction + + task run_phase(uvm_phase phase); + cycle_item observed; + result_item actual; + forever begin + @(posedge vif.clock); + observed = cycle_item::type_id::create("observed"); + observed.reset = vif.reset; + observed.valid = vif.in_valid; + observed.data = vif.in_data; + observed.keep = vif.in_keep; + observed.last = vif.in_last; + observed.ready = vif.out_ready; + observed.accepted = vif.in_valid & vif.in_ready; + observed.output_valid = vif.out_valid; + observed.output_last = vif.out_last; + input_ap.write(observed); + if (!vif.reset) begin + accepted += $countones(observed.accepted); + if (vif.out_valid && vif.out_ready) transferred++; + actual = result_item::type_id::create("actual"); + actual.valid = vif.out_valid; + actual.ready = vif.in_ready; + actual.data = vif.out_valid ? vif.out_data : 0; + actual.keep = vif.out_valid ? vif.out_keep : 0; + actual.last = vif.out_valid ? vif.out_last : 0; + output_ap.write(actual); + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/coverage.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/coverage.svh new file mode 100644 index 000000000..94dd8bb3f --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/coverage.svh @@ -0,0 +1,44 @@ +class coverage extends uvm_subscriber #(cycle_item); + `uvm_component_utils(coverage) + bit port1_stalled; + bit packet_open; + bit reset_pending; + + covergroup behavior with function sample ( + cycle_item t, bit late_competitor, bit locked_bubble, bit reset_held + ); + option.per_instance = 1; + contenders: coverpoint t.valid iff (!t.reset) {bins masks[] = {[0 : 3]};} + winner: coverpoint t.accepted iff (!t.reset && t.ready) { + bins idle = {0}; bins port0 = {1}; bins port1 = {2}; + } + output_state: coverpoint {t.output_valid, t.ready} iff (!t.reset) {bins states[] = {[0 : 3]};} + late_arrival: coverpoint late_competitor {bins seen = {1};} + owner_gap: coverpoint locked_bubble {bins seen = {1};} + reset_cancellation: coverpoint reset_held {bins seen = {1};} + last_stall: coverpoint (t.output_valid && t.output_last && !t.ready) iff (!t.reset) { + bins seen = {1}; + } + keep0: coverpoint t.keep[0] iff (!t.reset && t.accepted[0]) {bins masks[] = {[0 : 15]};} + endgroup + + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + + function void write(cycle_item t); + behavior.sample(t, port1_stalled && t.valid == 3 && !t.reset, + packet_open && t.valid != 0 && !t.output_valid && !t.reset, + t.reset && reset_pending); + if (t.reset) begin + port1_stalled = 0; + packet_open = 0; + reset_pending = 0; + end else begin + port1_stalled = t.valid == 2 && t.output_valid && !t.ready; + if (t.output_valid) packet_open = !(t.ready && t.output_last); + reset_pending = packet_open; + end + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/env.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/env.svh new file mode 100644 index 000000000..0d2fc4aff --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/env.svh @@ -0,0 +1,22 @@ +class env extends environment #(cycle_item, result_item, ref_model); + `uvm_component_utils(env) + agent bus; + coverage cov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + bus = agent::type_id::create("agent", this); + cov = coverage::type_id::create("coverage", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.input_ap.connect(input_export); + bus.mon.input_ap.connect(cov.analysis_export); + bus.mon.output_ap.connect(output_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/ref_model.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/ref_model.svh new file mode 100644 index 000000000..ab629ae3e --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/env/ref_model.svh @@ -0,0 +1,36 @@ +class ref_model extends reference_model #(cycle_item, result_item); + `uvm_component_utils(ref_model) + chandle model; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + model = axis_arbiter_create(); + endfunction + + function void write(cycle_item t); + int unsigned selected; + result_item expected; + if (t.reset) begin + axis_arbiter_destroy(model); + model = axis_arbiter_create(); + return; + end + selected = axis_arbiter_step(model, t.valid, t.ready, t.last); + expected = result_item::type_id::create("expected"); + expected.valid = selected != 32'hffffffff; + expected.ready = expected.valid && t.ready ? 2'b01 << selected : 0; + expected.data = expected.valid ? t.data[selected] : 0; + expected.keep = expected.valid ? t.keep[selected] : 0; + expected.last = expected.valid ? t.last[selected] : 0; + expected_ap.write(expected); + endfunction + + function void final_phase(uvm_phase phase); + axis_arbiter_destroy(model); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/filelist.f b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/filelist.f new file mode 100644 index 000000000..8af0617ea --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/filelist.f @@ -0,0 +1,7 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/PacketArbiter/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/interface.sv b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/interface.sv new file mode 100644 index 000000000..8b6bb7f0a --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/interface.sv @@ -0,0 +1,36 @@ +interface arbiter_if ( + input logic clock +); + logic reset; + logic [ 1:0] in_valid; + logic [ 1:0] in_ready; + logic [ `AXIS_DATA_BITS-1:0] in_data [2]; + logic [`AXIS_DATA_BITS/8-1:0] in_keep [2]; + logic [ 1:0] in_last; + logic out_valid; + logic out_ready; + logic [ `AXIS_DATA_BITS-1:0] out_data; + logic [`AXIS_DATA_BITS/8-1:0] out_keep; + logic out_last; + + assert property (@(posedge clock) disable iff (reset) $onehot0(in_ready)) + else $fatal(1, "Arbiter accepted more than one input"); + assert property (@(posedge clock) disable iff (reset) + out_valid && !out_ready |=> out_valid && $stable( + {out_data, out_keep, out_last} + )) + else $fatal(1, "Arbiter output changed under backpressure"); + + for (genvar i = 0; i < 2; i++) begin + assert property (@(posedge clock) disable iff (reset) + in_valid[i] && !in_ready[i] |=> in_valid[i] && + $stable( + {in_data[i], in_keep[i], in_last[i]} + )) + else $fatal(1, "Source changed an unaccepted input beat"); + end + + cover property (@(posedge clock) disable iff (reset) &in_valid); + cover property (@(posedge clock) disable iff (reset) in_valid[0] && in_ready[0]); + cover property (@(posedge clock) disable iff (reset) in_valid[1] && in_ready[1]); +endinterface diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/package.sv b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/package.sv new file mode 100644 index 000000000..13592d413 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/package.sv @@ -0,0 +1,24 @@ +package arbiter_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + + import "DPI-C" function chandle axis_arbiter_create(); + import "DPI-C" function void axis_arbiter_destroy(input chandle model); + import "DPI-C" function int unsigned axis_arbiter_step( + input chandle model, + input int unsigned valid, + input byte unsigned ready, + input int unsigned last + ); + + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/ref_model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/seq/sequence.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/seq/sequence.svh new file mode 100644 index 000000000..52d168957 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/seq/sequence.svh @@ -0,0 +1,139 @@ +class protocol_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(protocol_sequence) + cycle_item signals; + int unsigned cycles = 0; + int unsigned transfers = 0; + + function new(string name = "protocol_sequence"); + super.new(name); + endfunction + + task tick(int unsigned count, int unsigned beats = 0); + cycle_item req; + repeat (count) begin + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.copy(signals); + finish_item(req); + if (!signals.reset) cycles++; + transfers += beats; + end + endtask + + task body(); + signals = cycle_item::type_id::create("signals"); + signals.reset = 1; + signals.valid = 0; + signals.data = '{0, 32'h55555555}; + signals.keep = '{4'h1, 4'hf}; + signals.last = 0; + signals.ready = 0; + tick(4); + signals.reset = 0; + tick(2); + + `uvm_info("SCENARIO", "Simultaneous packets use fixed priority and preserve packet ownership", + UVM_LOW) + signals.valid = 3; + signals.ready = 0; + tick(2); + signals.ready = 1; + tick(1, 1); + signals.data[0] = '1; + signals.keep[0] = 15; + signals.last = 1; + tick(1, 1); + signals.valid = 2; + tick(1, 1); + signals.data[1] = '1; + signals.keep[1] = 10; + signals.last = 2; + tick(1, 1); + signals.valid = 0; + tick(1); + + `uvm_info("SCENARIO", "A late higher-priority input cannot replace a stalled first beat", + UVM_LOW) + signals.valid = 2; + signals.data[1] = 32'hbbbbbbbb; + signals.keep[1] = 5; + signals.last = 2; + signals.ready = 0; + tick(1); + signals.valid = 3; + signals.data[0] = 32'haaaaaaaa; + signals.keep[0] = 10; + signals.last = 3; + tick(2); + signals.ready = 1; + tick(1, 1); + signals.valid = 1; + tick(1, 1); + signals.valid = 0; + tick(1); + + `uvm_info("SCENARIO", "A locked source may pause; stalled TLAST retains ownership", UVM_LOW) + signals.valid = 2; + signals.data[1] = 32'h12345678; + signals.last = 0; + tick(1, 1); + signals.valid = 1; + signals.last = 1; + tick(2); + signals.valid = 3; + signals.data[1] = 32'h87654321; + signals.last = 3; + signals.ready = 0; + tick(2); + signals.ready = 1; + tick(1, 1); + signals.valid = 1; + tick(1, 1); + signals.valid = 0; + tick(1); + + `uvm_info("SCENARIO", "Single-beat packets preserve every TKEEP mask", UVM_LOW) + signals.valid = 1; + signals.last = 1; + for (int mask = 0; mask < (1 << (`AXIS_DATA_BITS / 8)); mask++) begin + signals.keep[0] = mask; + signals.data[0] = 32'h9e3779b9 * (mask + 1); + tick(1, 1); + end + signals.valid = 0; + tick(1); + + `uvm_info("SCENARIO", "Reset releases packet ownership", UVM_LOW) + signals.valid = 2; + signals.last = 0; + tick(1, 1); + signals.valid = 1; + signals.last = 1; + tick(1); + signals.reset = 1; + signals.valid = 0; + tick(2); + signals.reset = 0; + signals.valid = 1; + tick(1, 1); + signals.valid = 0; + tick(1); + + `uvm_info("SCENARIO", "Reset cancels a stalled first beat", UVM_LOW) + signals.data[1] = 0; + signals.valid = 2; + signals.last = 2; + signals.ready = 0; + tick(1); + signals.reset = 1; + signals.valid = 0; + tick(2); + signals.reset = 0; + signals.valid = 1; + signals.last = 1; + signals.ready = 1; + tick(1, 1); + signals.valid = 0; + tick(2); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/tb.sv b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/tb.sv new file mode 100644 index 000000000..ea40c226e --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/tb.sv @@ -0,0 +1,36 @@ +module arbiter_tb; + import uvm_pkg::*; + import arbiter_pkg::*; + + logic clock = 1'b0; + wire reset; + always #5 clock = ~clock; + + arbiter_if source_if (clock); + assign reset = source_if.reset; + + PacketArbiter dut ( + .clock(clock), + .reset(reset), + .io_in_0_ready(source_if.in_ready[0]), + .io_in_0_valid(source_if.in_valid[0]), + .io_in_0_bits_tdata(source_if.in_data[0]), + .io_in_0_bits_tkeep(source_if.in_keep[0]), + .io_in_0_bits_tlast(source_if.in_last[0]), + .io_in_1_ready(source_if.in_ready[1]), + .io_in_1_valid(source_if.in_valid[1]), + .io_in_1_bits_tdata(source_if.in_data[1]), + .io_in_1_bits_tkeep(source_if.in_keep[1]), + .io_in_1_bits_tlast(source_if.in_last[1]), + .io_out_ready(source_if.out_ready), + .io_out_valid(source_if.out_valid), + .io_out_bits_tdata(source_if.out_data), + .io_out_bits_tkeep(source_if.out_keep), + .io_out_bits_tlast(source_if.out_last) + ); + + initial begin + uvm_config_db#(virtual arbiter_if)::set(null, "*", "vif", source_if); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/tests/test.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/tests/test.svh new file mode 100644 index 000000000..df1c61638 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/arbiter/tests/test.svh @@ -0,0 +1,33 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + virtual arbiter_if vif; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual arbiter_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "Arbiter test requires vif") + test_env = env::type_id::create("env", this); + endfunction + + task execute(); + protocol_sequence seq = protocol_sequence::type_id::create("seq"); + seq.start(test_env.bus.seqr); + test_env.scoreboard.wait_checked(seq.cycles); + // The monitor finishes the last rising-edge sample before accounting is read. + @(negedge vif.clock); + if (test_env.scoreboard.checked != seq.cycles || + test_env.bus.mon.accepted != seq.transfers || + test_env.bus.mon.transferred != seq.transfers || + test_env.scoreboard.pending() != 0 || + test_env.scoreboard.pending() != 0) + `uvm_fatal("ARBITER_COMPLETE", "Arbiter transaction accounting mismatch") + `uvm_info("ARBITER_CHECKED", $sformatf("cycles=%0d accepted=%0d transferred=%0d pending=0", + test_env.scoreboard.checked, test_env.bus.mon.accepted, + test_env.bus.mon.transferred), UVM_LOW) + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/common/coverage.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/common/coverage.svh new file mode 100644 index 000000000..e475a6d8f --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/common/coverage.svh @@ -0,0 +1,19 @@ +class coverage extends uvm_subscriber #(item); + `uvm_component_utils(coverage) + covergroup behavior with function sample (logic [`AXIS_DATA_BITS/8-1:0] keep, bit last); + option.per_instance = 1; + masks: coverpoint keep { + bins none = {0}; + bins full = {(1 << (`AXIS_DATA_BITS / 8)) - 1}; + bins partial = {[1 : (1 << (`AXIS_DATA_BITS / 8)) - 2]}; + } + packet_boundary: coverpoint last; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void write(item t); + behavior.sample(t.keep, t.last); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/master/env/env.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/env/env.svh new file mode 100644 index 000000000..301af8cce --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/env/env.svh @@ -0,0 +1,44 @@ +class env extends environment #(item, item, ref_model); + `uvm_component_utils(env) + + coverage coverage_monitor; + source_agent source; + sink sink_driver; + monitor sink_monitor; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + virtual axis_if #(`AXIS_DATA_BITS) source_vif; + virtual axis_if #(`AXIS_DATA_BITS) sink_vif; + + super.build_phase(phase); + if (!uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::get( + this, "", "source_vif", source_vif + )) begin + `uvm_fatal("AXIS_VIF", "environment requires source_vif") + end + if (!uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::get( + this, "", "sink_vif", sink_vif + )) begin + `uvm_fatal("AXIS_VIF", "environment requires sink_vif") + end + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(this, "source.*", "vif", source_vif); + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(this, "sink_driver", "vif", sink_vif); + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(this, "sink_monitor", "vif", sink_vif); + + coverage_monitor = coverage::type_id::create("coverage", this); + source = source_agent::type_id::create("source", this); + sink_driver = sink::type_id::create("sink_driver", this); + sink_monitor = monitor::type_id::create("sink_monitor", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + source.mon.ap.connect(input_export); + sink_monitor.ap.connect(output_export); + sink_monitor.ap.connect(coverage_monitor.analysis_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/master/env/ref_model.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/env/ref_model.svh new file mode 100644 index 000000000..6606e1a50 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/env/ref_model.svh @@ -0,0 +1,39 @@ +class ref_model extends reference_model #(item, item); + `uvm_component_utils(ref_model) + + chandle model; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + model = axis_ref_create(); + if (model == null) begin + `uvm_fatal("AXIS_MODEL", "failed to create Rust reference model") + end + endfunction + + function void write(item t); + item expected; + int unsigned data; + byte unsigned keep; + byte unsigned last; + + axis_ref_push(model, t.data, t.keep, t.last); + if (!axis_ref_pop(model, data, keep, last)) begin + `uvm_fatal("AXIS_MODEL", "reference model produced no output") + end + expected = item::type_id::create("expected"); + expected.data = data; + expected.keep = keep[3:0]; + expected.last = last[0]; + expected_ap.write(expected); + endfunction + + function void final_phase(uvm_phase phase); + axis_ref_destroy(model); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/master/filelist.f b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/filelist.f new file mode 100644 index 000000000..1af63e578 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/filelist.f @@ -0,0 +1,9 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/protocol/axis ++incdir+@RESOURCES@ +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/axis/interface.sv +@VERIFY@/uvm/src/protocol/axis/package.sv +@RESOURCES@/package.sv +-F @RTL@/Master/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/master/package.sv b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/package.sv new file mode 100644 index 000000000..6684fd6e3 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/package.sv @@ -0,0 +1,31 @@ +package axis_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import axis_vip_pkg::*; + typedef axis_item#(`AXIS_DATA_BITS) item; + typedef axis_source_agent#(`AXIS_DATA_BITS) source_agent; + typedef axis_monitor#(`AXIS_DATA_BITS) monitor; + typedef axis_sink#(`AXIS_DATA_BITS) sink; + `include "uvm_macros.svh" + + import "DPI-C" function chandle axis_ref_create(); + import "DPI-C" function void axis_ref_destroy(input chandle model); + import "DPI-C" function void axis_ref_push( + input chandle model, + input int unsigned data, + input byte unsigned keep, + input byte unsigned last + ); + import "DPI-C" function byte unsigned axis_ref_pop( + input chandle model, + output int unsigned data, + output byte unsigned keep, + output byte unsigned last + ); + + `include "env/ref_model.svh" + `include "../common/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/master/seq/sequence.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/seq/sequence.svh new file mode 100644 index 000000000..2f9270f30 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/seq/sequence.svh @@ -0,0 +1,19 @@ +class protocol_sequence extends uvm_sequence #(item); + `uvm_object_utils(protocol_sequence) + + function new(string name = "protocol_sequence"); + super.new(name); + endfunction + + task body(); + item req; + for (int i = 0; i < 32; i++) begin + req = item::type_id::create($sformatf("item_%0d", i)); + start_item(req); + req.data = $urandom(); + req.keep = i % (1 << (`AXIS_DATA_BITS / 8)); + req.last = (i % 4) == 3; + finish_item(req); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/master/tb.sv b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/tb.sv new file mode 100644 index 000000000..1c06a459c --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/tb.sv @@ -0,0 +1,43 @@ +module master_tb; + import uvm_pkg::*; + import axis_pkg::*; + + logic clock = 1'b0; + logic reset = 1'b0; + always #5 clock = ~clock; + + axis_if #(`AXIS_DATA_BITS) source_if (clock); + axis_if #(`AXIS_DATA_BITS) sink_if (clock); + + assign source_if.reset = reset; + assign sink_if.reset = reset; + + Master dut ( + .clock(clock), + .reset(reset), + .io_in_ready(source_if.tready), + .io_in_valid(source_if.tvalid), + .io_in_bits_tdata(source_if.tdata), + .io_in_bits_tkeep(source_if.tkeep), + .io_in_bits_tlast(source_if.tlast), + .io_axis_tvalid(sink_if.tvalid), + .io_axis_tready(sink_if.tready), + .io_axis_tdata(sink_if.tdata), + .io_axis_tkeep(sink_if.tkeep), + .io_axis_tlast(sink_if.tlast) + ); + + initial begin + reset = 1'b1; + repeat (4) @(negedge clock); + reset = 1'b0; + end + + initial begin + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(null, "uvm_test_top.env", "source_vif", + source_if); + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(null, "uvm_test_top.env", "sink_vif", + sink_if); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/master/tests/test.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/tests/test.svh new file mode 100644 index 000000000..644a639ca --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/master/tests/test.svh @@ -0,0 +1,23 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + + env test_env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + + task execute(); + protocol_sequence seq; + seq = protocol_sequence::type_id::create("seq"); + seq.start(test_env.source.seqr); + test_env.scoreboard.wait_checked(32); + `uvm_info("AXIS_CHECKED", $sformatf("Checked %0d transfers", test_env.scoreboard.checked), + UVM_LOW) + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/env/env.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/env/env.svh new file mode 100644 index 000000000..301af8cce --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/env/env.svh @@ -0,0 +1,44 @@ +class env extends environment #(item, item, ref_model); + `uvm_component_utils(env) + + coverage coverage_monitor; + source_agent source; + sink sink_driver; + monitor sink_monitor; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + virtual axis_if #(`AXIS_DATA_BITS) source_vif; + virtual axis_if #(`AXIS_DATA_BITS) sink_vif; + + super.build_phase(phase); + if (!uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::get( + this, "", "source_vif", source_vif + )) begin + `uvm_fatal("AXIS_VIF", "environment requires source_vif") + end + if (!uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::get( + this, "", "sink_vif", sink_vif + )) begin + `uvm_fatal("AXIS_VIF", "environment requires sink_vif") + end + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(this, "source.*", "vif", source_vif); + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(this, "sink_driver", "vif", sink_vif); + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(this, "sink_monitor", "vif", sink_vif); + + coverage_monitor = coverage::type_id::create("coverage", this); + source = source_agent::type_id::create("source", this); + sink_driver = sink::type_id::create("sink_driver", this); + sink_monitor = monitor::type_id::create("sink_monitor", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + source.mon.ap.connect(input_export); + sink_monitor.ap.connect(output_export); + sink_monitor.ap.connect(coverage_monitor.analysis_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/env/ref_model.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/env/ref_model.svh new file mode 100644 index 000000000..6606e1a50 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/env/ref_model.svh @@ -0,0 +1,39 @@ +class ref_model extends reference_model #(item, item); + `uvm_component_utils(ref_model) + + chandle model; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + model = axis_ref_create(); + if (model == null) begin + `uvm_fatal("AXIS_MODEL", "failed to create Rust reference model") + end + endfunction + + function void write(item t); + item expected; + int unsigned data; + byte unsigned keep; + byte unsigned last; + + axis_ref_push(model, t.data, t.keep, t.last); + if (!axis_ref_pop(model, data, keep, last)) begin + `uvm_fatal("AXIS_MODEL", "reference model produced no output") + end + expected = item::type_id::create("expected"); + expected.data = data; + expected.keep = keep[3:0]; + expected.last = last[0]; + expected_ap.write(expected); + endfunction + + function void final_phase(uvm_phase phase); + axis_ref_destroy(model); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/filelist.f b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/filelist.f new file mode 100644 index 000000000..8bde69336 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/filelist.f @@ -0,0 +1,9 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/protocol/axis ++incdir+@RESOURCES@ +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/axis/interface.sv +@VERIFY@/uvm/src/protocol/axis/package.sv +@RESOURCES@/package.sv +-F @RTL@/Slave/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/package.sv b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/package.sv new file mode 100644 index 000000000..6684fd6e3 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/package.sv @@ -0,0 +1,31 @@ +package axis_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import axis_vip_pkg::*; + typedef axis_item#(`AXIS_DATA_BITS) item; + typedef axis_source_agent#(`AXIS_DATA_BITS) source_agent; + typedef axis_monitor#(`AXIS_DATA_BITS) monitor; + typedef axis_sink#(`AXIS_DATA_BITS) sink; + `include "uvm_macros.svh" + + import "DPI-C" function chandle axis_ref_create(); + import "DPI-C" function void axis_ref_destroy(input chandle model); + import "DPI-C" function void axis_ref_push( + input chandle model, + input int unsigned data, + input byte unsigned keep, + input byte unsigned last + ); + import "DPI-C" function byte unsigned axis_ref_pop( + input chandle model, + output int unsigned data, + output byte unsigned keep, + output byte unsigned last + ); + + `include "env/ref_model.svh" + `include "../common/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/seq/sequence.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/seq/sequence.svh new file mode 100644 index 000000000..2f9270f30 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/seq/sequence.svh @@ -0,0 +1,19 @@ +class protocol_sequence extends uvm_sequence #(item); + `uvm_object_utils(protocol_sequence) + + function new(string name = "protocol_sequence"); + super.new(name); + endfunction + + task body(); + item req; + for (int i = 0; i < 32; i++) begin + req = item::type_id::create($sformatf("item_%0d", i)); + start_item(req); + req.data = $urandom(); + req.keep = i % (1 << (`AXIS_DATA_BITS / 8)); + req.last = (i % 4) == 3; + finish_item(req); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/tb.sv b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/tb.sv new file mode 100644 index 000000000..ac2cf0f9e --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/tb.sv @@ -0,0 +1,43 @@ +module slave_tb; + import uvm_pkg::*; + import axis_pkg::*; + + logic clock = 1'b0; + logic reset = 1'b0; + always #5 clock = ~clock; + + axis_if #(`AXIS_DATA_BITS) source_if (clock); + axis_if #(`AXIS_DATA_BITS) sink_if (clock); + + assign source_if.reset = reset; + assign sink_if.reset = reset; + + Slave dut ( + .clock(clock), + .reset(reset), + .io_axis_tvalid(source_if.tvalid), + .io_axis_tready(source_if.tready), + .io_axis_tdata(source_if.tdata), + .io_axis_tkeep(source_if.tkeep), + .io_axis_tlast(source_if.tlast), + .io_out_ready(sink_if.tready), + .io_out_valid(sink_if.tvalid), + .io_out_bits_tdata(sink_if.tdata), + .io_out_bits_tkeep(sink_if.tkeep), + .io_out_bits_tlast(sink_if.tlast) + ); + + initial begin + reset = 1'b1; + repeat (4) @(negedge clock); + reset = 1'b0; + end + + initial begin + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(null, "uvm_test_top.env", "source_vif", + source_if); + uvm_config_db#(virtual axis_if #(`AXIS_DATA_BITS))::set(null, "uvm_test_top.env", "sink_vif", + sink_if); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/tests/test.svh b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/tests/test.svh new file mode 100644 index 000000000..644a639ca --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/slave/tests/test.svh @@ -0,0 +1,23 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + + env test_env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + + task execute(); + protocol_sequence seq; + seq = protocol_sequence::type_id::create("seq"); + seq.start(test_env.source.seqr); + test_env.scoreboard.wait_checked(32); + `uvm_info("AXIS_CHECKED", $sformatf("Checked %0d transfers", test_env.scoreboard.checked), + UVM_LOW) + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/axis/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/axis/src/main/resources/suite.toml new file mode 100644 index 000000000..82ea43ede --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/resources/suite.toml @@ -0,0 +1,54 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libaxis_reference_model" + +[[tests]] +name = "arbiter" +functional_groups = ["arbiter_pkg::coverage::behavior"] +top = "arbiter_tb" +test = "protocol_test" +filelist = "arbiter/filelist.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +name = "master" +functional_groups = ["axis_pkg::coverage::behavior"] +top = "master_tb" +test = "protocol_test" +filelist = "master/filelist.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +name = "slave" +functional_groups = ["axis_pkg::coverage::behavior"] +top = "slave_tb" +test = "protocol_test" +filelist = "slave/filelist.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/ip/axis/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/axis/src/main/scala/Emit.scala new file mode 100644 index 000000000..eef624205 --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/scala/Emit.scala @@ -0,0 +1,22 @@ +package framework.ip.bus.axi + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val parameters = ujson.read(os.read(os.Path(config))) + val dataBits = parameters("dataBits").num.toInt + val inputs = parameters("inputs").num.toInt + require(inputs == 2, "The current UVM arbiter interface requires two inputs") + require(dataBits == 16 || dataBits == 32, "The current UVM reference supports 16 or 32 bits") + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Master(dataBits = dataBits), + args = Array("--target-dir", s"$output/Master", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Slave(dataBits = dataBits), + args = Array("--target-dir", s"$output/Slave", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new PacketArbiter(inputs = inputs, dataBits = dataBits), + args = Array("--target-dir", s"$output/PacketArbiter", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/ip/axis/src/main/scala/Interface.scala b/arch/src/main/scala/framework/ip/axis/src/main/scala/Interface.scala new file mode 100644 index 000000000..c2412be0e --- /dev/null +++ b/arch/src/main/scala/framework/ip/axis/src/main/scala/Interface.scala @@ -0,0 +1,51 @@ +package framework.ip.bus.axi + +import chisel3._ + +/** Physical AXI4-Stream signals, directed from the master's perspective. */ +class Port( + dataBits: Int, + idBits: Int = 0, + destBits: Int = 0, + userBits: Int = 0) + extends Bundle { + require(dataBits > 0 && dataBits % 8 == 0) + val tvalid = Output(Bool()) + val tready = Input(Bool()) + val tdata = Output(UInt(dataBits.W)) + val tkeep = Output(UInt((dataBits / 8).W)) + val tlast = Output(Bool()) + val tid = Output(UInt(idBits.W)) + val tdest = Output(UInt(destBits.W)) + val tuser = Output(UInt(userBits.W)) +} + +/** + * AXI4-Stream payload signals. + * + * A `Decoupled[Beat]` is the full AXI-S channel: `valid` and `ready` + * are TVALID and TREADY. This bundle exposes the remaining standard signal + * names directly in generated RTL. + */ +class Beat( + dataBits: Int, + idBits: Int = 0, + destBits: Int = 0, + userBits: Int = 0) + extends Bundle { + require(dataBits > 0 && dataBits % 8 == 0) + val tdata = UInt(dataBits.W) + val tkeep = UInt((dataBits / 8).W) + val tlast = Bool() + val tid = UInt(idBits.W) + val tdest = UInt(destBits.W) + val tuser = UInt(userBits.W) + + // Internal aliases preserve concise datapath code while retaining AXI-S names. + def data: UInt = tdata + def keep: UInt = tkeep + def last: Bool = tlast + def id: UInt = tid + def dest: UInt = tdest + def user: UInt = tuser +} diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Master.scala b/arch/src/main/scala/framework/ip/axis/src/main/scala/Master.scala similarity index 96% rename from arch/src/main/scala/framework/mem-core/axis/src/main/scala/Master.scala rename to arch/src/main/scala/framework/ip/axis/src/main/scala/Master.scala index 529fe4b55..523f054d4 100644 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Master.scala +++ b/arch/src/main/scala/framework/ip/axis/src/main/scala/Master.scala @@ -1,4 +1,4 @@ -package memcore.bus.axi +package framework.ip.bus.axi import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/PacketArbiter.scala b/arch/src/main/scala/framework/ip/axis/src/main/scala/PacketArbiter.scala similarity index 92% rename from arch/src/main/scala/framework/mem-core/axis/src/main/scala/PacketArbiter.scala rename to arch/src/main/scala/framework/ip/axis/src/main/scala/PacketArbiter.scala index 7db666c76..c51ead948 100644 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/PacketArbiter.scala +++ b/arch/src/main/scala/framework/ip/axis/src/main/scala/PacketArbiter.scala @@ -1,4 +1,4 @@ -package memcore.bus.axi +package framework.ip.bus.axi import chisel3._ import chisel3.util._ @@ -42,8 +42,8 @@ class PacketArbiter( io.in(i).ready := io.out.ready && io.out.valid && active === i.U } - when(io.out.fire) { - locked := !io.out.bits.last + when(io.out.valid) { + locked := !io.out.ready || !io.out.bits.last selected := active } } diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Slave.scala b/arch/src/main/scala/framework/ip/axis/src/main/scala/Slave.scala similarity index 96% rename from arch/src/main/scala/framework/mem-core/axis/src/main/scala/Slave.scala rename to arch/src/main/scala/framework/ip/axis/src/main/scala/Slave.scala index 32cad3250..1cc331b5f 100644 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Slave.scala +++ b/arch/src/main/scala/framework/ip/axis/src/main/scala/Slave.scala @@ -1,4 +1,4 @@ -package memcore.bus.axi +package framework.ip.bus.axi import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/ip/bank/build.sc b/arch/src/main/scala/framework/ip/bank/build.sc new file mode 100644 index 000000000..2278ff7f9 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.memory.bank.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Bank.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/BankSet.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/SramIO.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/bank/configs/uvm.toml b/arch/src/main/scala/framework/ip/bank/configs/uvm.toml new file mode 100644 index 000000000..eab9824ac --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/configs/uvm.toml @@ -0,0 +1,7 @@ +[parameters] +dataBits = 32 +banks = 4 +entriesPerBank = 16 +tagBits = 8 + +[defines] diff --git a/arch/src/main/scala/framework/mem-core/bank/src/csrc/Cargo.lock b/arch/src/main/scala/framework/ip/bank/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/bank/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/ip/bank/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/bank/src/csrc/Cargo.toml b/arch/src/main/scala/framework/ip/bank/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/bank/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/ip/bank/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/bank/src/csrc/README.md b/arch/src/main/scala/framework/ip/bank/src/csrc/README.md similarity index 100% rename from arch/src/main/scala/framework/mem-core/bank/src/csrc/README.md rename to arch/src/main/scala/framework/ip/bank/src/csrc/README.md diff --git a/arch/src/main/scala/framework/mem-core/cache/src/csrc/src/lib.rs b/arch/src/main/scala/framework/ip/bank/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/ip/bank/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/ip/bank/src/csrc/src/memory.rs b/arch/src/main/scala/framework/ip/bank/src/csrc/src/memory.rs new file mode 100644 index 000000000..f20568ab4 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/csrc/src/memory.rs @@ -0,0 +1,50 @@ +struct Bank { + words: Vec, + initialized: Vec, +} + +impl Bank { + fn access(&mut self, addr: usize, write: bool, data: u32, mask: u8) -> u32 { + if write { + for byte in 0..4 { + if mask & (1 << byte) != 0 { + let bits = 0xff << (byte * 8); + self.words[addr] = (self.words[addr] & !bits) | (data & bits); + } + } + self.initialized[addr] |= mask; + 0 + } else { + assert_eq!( + self.initialized[addr], 0xf, + "read of uninitialized bank word" + ); + self.words[addr] + } + } +} + +#[no_mangle] +pub extern "C" fn bank_ref_create(entries: u32) -> *mut std::ffi::c_void { + Box::into_raw(Box::new(Bank { + words: vec![0; entries as usize], + initialized: vec![0; entries as usize], + })) + .cast() +} + +#[no_mangle] +pub unsafe extern "C" fn bank_ref_destroy(model: *mut std::ffi::c_void) { + drop(Box::from_raw(model.cast::())); +} + +#[no_mangle] +pub unsafe extern "C" fn bank_ref_access( + model: *mut std::ffi::c_void, + addr: u32, + write: u8, + data: u32, + mask: u8, +) -> u32 { + (*model.cast::()).access(addr as usize, write != 0, data, mask) +} diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/agent.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/agent.svh new file mode 100644 index 000000000..3520a6583 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/agent.svh @@ -0,0 +1,22 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(request_item) seqr; + driver drv; + monitor mon; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(request_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/driver.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/driver.svh new file mode 100644 index 000000000..c8a88163a --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/driver.svh @@ -0,0 +1,65 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual port_if request; + virtual port_if response; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get( + this, "", "request", request + ) || !uvm_config_db#(virtual port_if)::get( + this, "", "response", response + )) + `uvm_fatal("VIF", "Bank driver interfaces missing") + endfunction + + task run_phase(uvm_phase phase); + request_item req; + request.valid = 0; + request.write = 0; + request.addr = 0; + request.data = 0; + request.mask = 0; + request.tag = 0; + request.reset = 1; + response.ready = 0; + forever begin + seq_item_port.get_next_item(req); + if (req.reset_command) begin + @(negedge request.clock); + request.reset = 1; + repeat (4) @(negedge request.clock); + request.reset = 0; + end else begin + @(negedge request.clock); + response.ready = 0; + request.valid = 1; + request.write = req.write; + request.addr = req.addr; + request.data = req.data; + request.mask = req.mask; + request.tag = req.tag; + do @(posedge request.clock); while (!request.ready); + @(negedge request.clock); + request.valid = 0; + if (req.cancel_response) begin + request.reset = 1; + repeat (2) @(negedge request.clock); + request.reset = 0; + repeat (3) @(negedge request.clock); + end else begin + while (!response.valid) @(negedge request.clock); + repeat (req.stall_cycles + 1) @(negedge request.clock); + response.ready = 1; + @(negedge request.clock); + // Keep the sink ready while idle; the next transfer selects its stall. + end + end + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/item.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/item.svh new file mode 100644 index 000000000..c4abdb058 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/item.svh @@ -0,0 +1,37 @@ +class request_item extends uvm_sequence_item; + bit reset_command; + bit cancel_response; + int unsigned stall_cycles; + logic [`BANK_ROW_BITS-1:0] addr; + logic write; + logic [31:0] data; + logic [3:0] mask; + logic [`BANK_TAG_BITS-1:0] tag; + `uvm_object_utils_begin(request_item) + `uvm_field_int(reset_command, UVM_ALL_ON) + `uvm_field_int(cancel_response, UVM_ALL_ON) + `uvm_field_int(stall_cycles, UVM_ALL_ON) + `uvm_field_int(addr, UVM_ALL_ON) + `uvm_field_int(write, UVM_ALL_ON) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(mask, UVM_ALL_ON) + `uvm_field_int(tag, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [31:0] data; + logic [`BANK_TAG_BITS-1:0] tag; + logic error; + `uvm_object_utils_begin(response_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(tag, UVM_ALL_ON) + `uvm_field_int(error, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/monitor.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/monitor.svh new file mode 100644 index 000000000..d65c67821 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/agent/monitor.svh @@ -0,0 +1,74 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual port_if request; + virtual port_if response; + uvm_analysis_port #(request_item) request_ap; + uvm_analysis_port #(response_item) response_ap; + uvm_analysis_port #(bit) reset_ap; + int unsigned accepted = 0; + int unsigned retired = 0; + int unsigned cancelled = 0; + + function new(string name, uvm_component parent); + super.new(name, parent); + request_ap = new("request_ap", this); + response_ap = new("response_ap", this); + reset_ap = new("reset_ap", this); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get( + this, "", "request", request + ) || !uvm_config_db#(virtual port_if)::get( + this, "", "response", response + )) + `uvm_fatal("VIF", "Bank monitor interfaces missing") + endfunction + + task run_phase(uvm_phase phase); + request_item req; + response_item rsp; + bit pending = 0; + forever begin + @(posedge request.clock); + if (request.reset) begin + if (pending) cancelled++; + pending = 0; + reset_ap.write(1); + end else begin + if ($isunknown({request.valid, request.ready, response.valid, response.ready})) + `uvm_fatal("BANK_UNKNOWN", "unknown Bank handshake") + if (request.valid && request.ready) begin + if (pending) `uvm_fatal("BANK_PENDING", "accepted a second outstanding request") + if ($isunknown({request.addr, request.write, request.data, request.mask, request.tag})) + `uvm_fatal("BANK_UNKNOWN", "unknown accepted request") + req = request_item::type_id::create("request"); + req.addr = request.addr; + req.write = request.write; + req.data = request.data; + req.mask = request.mask; + req.tag = request.tag; + pending = 1; + accepted++; + request_ap.write(req); + end + if (response.valid && !pending) + `uvm_fatal("BANK_EXTRA", "response without an outstanding request") + if (pending && !(request.valid && request.ready) && request.ready !== 1'b0) + `uvm_fatal("BANK_PENDING", "request ready with an outstanding response") + if (response.valid && response.ready) begin + if ($isunknown({response.data, response.tag, response.error})) + `uvm_fatal("BANK_UNKNOWN", "unknown accepted response") + rsp = response_item::type_id::create("response"); + rsp.data = response.data; + rsp.tag = response.tag; + rsp.error = response.error; + response_ap.write(rsp); + pending = 0; + retired++; + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/coverage.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/coverage.svh new file mode 100644 index 000000000..2c19ddeaf --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/coverage.svh @@ -0,0 +1,20 @@ +class coverage extends uvm_subscriber #(request_item); + `uvm_component_utils(coverage) + + covergroup accesses with function sample (request_item req); + option.per_instance = 1; + direction: coverpoint req.write; + address: coverpoint req.addr {bins rows[] = {[0 : `BANK_ROWS - 1]};} + write_mask: coverpoint req.mask iff (req.write) {bins masks[] = {[0 : 15]};} + direction_address: cross direction, address; + endgroup + + function new(string name, uvm_component parent); + super.new(name, parent); + accesses = new(); + endfunction + + function void write(request_item req); + accesses.sample(req); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/env.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/env.svh new file mode 100644 index 000000000..551c59c41 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/env.svh @@ -0,0 +1,32 @@ +`uvm_analysis_imp_decl(_reset) + +class env extends environment #(request_item, response_item, ref_model); + `uvm_component_utils(env) + agent bus; + coverage cov; + uvm_analysis_imp_reset #(bit, env) reset_export; + + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + bus = agent::type_id::create("agent", this); + cov = coverage::type_id::create("coverage", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.request_ap.connect(input_export); + bus.mon.request_ap.connect(cov.analysis_export); + bus.mon.response_ap.connect(output_export); + bus.mon.reset_ap.connect(reset_export); + endfunction + + function void write_reset(bit asserted); + // Reset cancels responses; accepted writes remain in the reference memory. + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/ref_model.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/ref_model.svh new file mode 100644 index 000000000..2a173fd5d --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/env/ref_model.svh @@ -0,0 +1,26 @@ +class ref_model extends reference_model #(request_item, response_item); + `uvm_component_utils(ref_model) + chandle model; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + model = bank_ref_create(`BANK_ROWS); + endfunction + + function void write(request_item t); + response_item expected = response_item::type_id::create("expected"); + expected.data = bank_ref_access(model, t.addr, t.write, t.data, t.mask); + expected.tag = t.tag; + expected.error = 0; + expected_ap.write(expected); + endfunction + + function void final_phase(uvm_phase phase); + bank_ref_destroy(model); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/filelist.f b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/filelist.f new file mode 100644 index 000000000..c6d59af04 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Bank/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/interface.sv b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/interface.sv new file mode 100644 index 000000000..963a9bc06 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/interface.sv @@ -0,0 +1,20 @@ +`include "parameters.svh" +interface port_if ( + input logic clock +); + logic reset; + logic valid; + logic ready; + logic [`BANK_ROW_BITS-1:0] addr; + logic write; + logic [31:0] data; + logic [3:0] mask; + logic [`BANK_TAG_BITS-1:0] tag; + logic error; + + assert property (@(posedge clock) disable iff (reset) valid && !ready |=> valid && $stable( + {addr, write, data, mask, tag, error} + )) + else $fatal(1, "Bank payload changed under backpressure"); + cover property (@(posedge clock) disable iff (reset) valid && !ready ##1 valid && ready); +endinterface diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/package.sv b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/package.sv new file mode 100644 index 000000000..e06e390b1 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/package.sv @@ -0,0 +1,26 @@ +`include "parameters.svh" +package bank_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + + import "DPI-C" function chandle bank_ref_create(input int unsigned entries); + import "DPI-C" function void bank_ref_destroy(input chandle model); + import "DPI-C" function int unsigned bank_ref_access( + input chandle model, + input int unsigned addr, + input byte unsigned write, + input int unsigned data, + input byte unsigned mask + ); + + `include "agent/item.svh" + `include "env/ref_model.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/seq/sequence.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/seq/sequence.svh new file mode 100644 index 000000000..ae11e60a0 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/seq/sequence.svh @@ -0,0 +1,43 @@ +class access_sequence extends uvm_sequence #(request_item); + `uvm_object_utils(access_sequence) + int unsigned expected_responses = 0; + int unsigned expected_cancellations = 0; + + function new(string name = "access_sequence"); + super.new(name); + endfunction + + task access (bit wr, bit [`BANK_ROW_BITS-1:0] addr, bit [31:0] data, bit [3:0] mask, + bit [`BANK_TAG_BITS-1:0] tag, int unsigned stall, bit cancel_response = 0); + request_item req = request_item::type_id::create("access"); + start_item(req); + req.write = wr; + req.addr = addr; + req.data = data; + req.mask = mask; + req.tag = tag; + req.stall_cycles = stall; + req.cancel_response = cancel_response; + finish_item(req); + if (cancel_response) expected_cancellations++; + else expected_responses++; + endtask + + task body(); + request_item req = request_item::type_id::create("reset"); + start_item(req); + req.reset_command = 1; + finish_item(req); + for (int a = 0; a < `BANK_ROWS; a++) access (1, a, 32'h9e3779b9 * (a + 1), 15, a, 0); + for (int a = 0; a < `BANK_ROWS; a++) access (0, a, 0, 0, 255 - a, 2); + for (int m = 0; m < 16; m++) begin + access (1, m % `BANK_ROWS, ~(32'h9e3779b9 * (m + 1)), m, m, 5); + access (0, m % `BANK_ROWS, 0, 0, m, 0); + end + access (1, `BANK_ROWS - 1, 'hface1234, 15, 255, 0, 1); + access (0, `BANK_ROWS - 1, 0, 0, 42, 3); + access (0, `BANK_ROWS - 1, 0, 15, 255, 0, 1); + access (0, `BANK_ROWS - 1, 0, 0, 42, 3); + for (int a = 0; a < `BANK_ROWS; a++) access (0, a, 0, 0, a, 0); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/tb.sv b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/tb.sv new file mode 100644 index 000000000..fd8330982 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/tb.sv @@ -0,0 +1,36 @@ +module bank_tb; + import uvm_pkg::*; + import bank_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + port_if source_if (clock); + port_if sink_if (clock); + Bank dut ( + .clock(clock), + .reset(source_if.reset), + .io_request_valid(source_if.valid), + .io_request_ready(source_if.ready), + .io_request_bits_addr(source_if.addr), + .io_request_bits_write(source_if.write), + .io_request_bits_data(source_if.data), + .io_request_bits_mask(source_if.mask), + .io_request_bits_tag(source_if.tag), + .io_response_valid(sink_if.valid), + .io_response_ready(sink_if.ready), + .io_response_bits_data(sink_if.data), + .io_response_bits_tag(sink_if.tag), + .io_response_bits_error(sink_if.error) + ); + assign sink_if.reset = source_if.reset; + assign source_if.error = 0; + assign sink_if.addr = 0; + assign sink_if.write = 0; + assign sink_if.mask = 0; + assert property (@(posedge clock) source_if.reset |=> !sink_if.valid && source_if.ready) + else $fatal(1, "Bank reset did not discard the pending response"); + initial begin + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "request", source_if); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "response", sink_if); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/tests/test.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/tests/test.svh new file mode 100644 index 000000000..1a31b9a72 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bank/tests/test.svh @@ -0,0 +1,32 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + virtual port_if request; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + if (!uvm_config_db#(virtual port_if)::get(this, "", "request", request)) + `uvm_fatal("VIF", "Bank test clock interface missing") + endfunction + + task execute(); + access_sequence seq = access_sequence::type_id::create("seq"); + seq.start(test_env.bus.seqr); + test_env.scoreboard.wait_checked(seq.expected_responses); + // Observe the next idle cycles before ending the test. + repeat (2) @(negedge request.clock); + if (test_env.scoreboard.checked != seq.expected_responses || + test_env.bus.mon.retired != seq.expected_responses || + test_env.bus.mon.cancelled != seq.expected_cancellations || + test_env.bus.mon.accepted != seq.expected_responses + seq.expected_cancellations) + `uvm_fatal("BANK_COMPLETE", "Bank transaction accounting mismatch") + `uvm_info("BANK", $sformatf("accepted=%0d checked=%0d cancelled=%0d pending=%0d", + test_env.bus.mon.accepted, test_env.scoreboard.checked, + test_env.bus.mon.cancelled, test_env.scoreboard.pending()), UVM_LOW) + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/agent.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/agent.svh new file mode 100644 index 000000000..82cbf1792 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/agent.svh @@ -0,0 +1,18 @@ +class command_agent extends uvm_agent; + `uvm_component_utils(command_agent) + uvm_sequencer #(command) seqr; + command_driver driver; + command_monitor monitor; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = new("seqr", this); + driver = command_driver::type_id::create("driver", this); + monitor = command_monitor::type_id::create("monitor", this); + endfunction + function void connect_phase(uvm_phase phase); + driver.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/command.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/command.svh new file mode 100644 index 000000000..54ce90a1f --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/command.svh @@ -0,0 +1,10 @@ +class command extends uvm_sequence_item; + `uvm_object_utils(command) + bit write; + int unsigned word_addr; + int unsigned beats; + int unsigned completion_stall; + function new(string name = "command"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/driver.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/driver.svh new file mode 100644 index 000000000..cd26758a4 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/driver.svh @@ -0,0 +1,41 @@ +class command_driver extends uvm_driver #(command); + `uvm_component_utils(command_driver) + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "control", vif)) + `uvm_fatal("VIF", "BankSet control interface is required") + endfunction + task run_phase(uvm_phase phase); + command req; + vif.valid = 0; + vif.write = 0; + vif.addr = 0; + vif.beats = 0; + vif.done_ready = 0; + forever begin + seq_item_port.get_next_item(req); + do @(negedge vif.clock); while (vif.reset); + vif.valid = 1; + vif.write = req.write; + vif.addr = req.word_addr * 4; + vif.beats = req.beats; + do @(posedge vif.clock); while (!vif.ready); + @(negedge vif.clock); + vif.valid = 0; + wait (vif.done_valid); + repeat (req.completion_stall) @(negedge vif.clock); + @(negedge vif.clock); + vif.done_ready = 1; + @(posedge vif.clock); + @(negedge vif.clock); + vif.done_ready = 0; + evidence::check(vif.ready === 1'b1 && vif.done_valid === 1'b0, + "BankSet did not return to idle after completion"); + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/item.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/item.svh new file mode 100644 index 000000000..c8f6c65cf --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/item.svh @@ -0,0 +1,11 @@ +class operation extends uvm_sequence_item; + bit write; + int unsigned word_addr; + int unsigned data; + byte unsigned mask; + bit last; + `uvm_object_utils(operation) + function new(string name = "operation"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/monitor.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/monitor.svh new file mode 100644 index 000000000..47a86b7ac --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/agent/monitor.svh @@ -0,0 +1,83 @@ +class command_monitor extends uvm_monitor; + `uvm_component_utils(command_monitor) + virtual port_if control; + virtual axis_if write_vif; + virtual axis_if read_vif; + uvm_analysis_port #(operation) ap; + uvm_analysis_port #(command) commands; + int unsigned completed = 0; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + commands = new("commands", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get( + this, "", "control", control + ) || !uvm_config_db#(virtual axis_if)::get( + this, "", "write_vif", write_vif + ) || !uvm_config_db#(virtual axis_if)::get( + this, "", "read_vif", read_vif + )) + `uvm_fatal("VIF", "BankSet monitor interfaces are required") + endfunction + task run_phase(uvm_phase phase); + command active; + operation op; + int unsigned transferred = 0; + forever begin + @(posedge control.clock); + if (control.reset) begin + active = null; + transferred = 0; + end else begin + if (control.valid && control.ready) begin + evidence::check(active == null, "BankSet accepted overlapping commands"); + active = command::type_id::create("observed"); + active.write = control.write; + active.word_addr = control.addr / 4; + active.beats = control.beats; + transferred = 0; + if (!active.write) begin + for (int i = 0; i < active.beats; i++) begin + op = operation::type_id::create("read"); + op.word_addr = active.word_addr + i; + op.last = i == active.beats - 1; + ap.write(op); + end + end + end + if (write_vif.tvalid && write_vif.tready) begin + if (active == null) `uvm_fatal("COMMAND", "Write beat without a command") + evidence::check(active.write && transferred < active.beats, + "Unexpected BankSet write beat"); + op = operation::type_id::create("write"); + op.write = 1; + op.word_addr = active.word_addr + transferred; + op.data = write_vif.tdata; + op.mask = write_vif.tkeep; + ap.write(op); + transferred++; + end + if (read_vif.tvalid && read_vif.tready) begin + if (active == null) `uvm_fatal("COMMAND", "Read beat without a command") + evidence::check(!active.write && transferred < active.beats, + "Unexpected BankSet read beat"); + transferred++; + end + if (control.done_valid) begin + if (active == null) `uvm_fatal("COMMAND", "Completion without a command") + evidence::check( + transferred == active.beats && control.error === 1'b0 && control.ready === 1'b0, + "Invalid BankSet completion"); + if (control.done_ready) begin + commands.write(active); + active = null; + completed++; + end else active.completion_stall++; + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/coverage.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/coverage.svh new file mode 100644 index 000000000..102dce146 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/coverage.svh @@ -0,0 +1,25 @@ +class coverage extends uvm_subscriber #(command); + `uvm_component_utils(coverage) + command observed; + covergroup behavior; + option.per_instance = 1; + direction: coverpoint observed.write; + start_bank: coverpoint (observed.word_addr % `BANK_COUNT) { + bins banks[] = {[0 : `BANK_COUNT - 1]}; + } + length: coverpoint observed.beats { + bins single = {1}; bins partial = {[2 : `BANK_WORDS - 1]}; bins full = {`BANK_WORDS}; + } + crossed_row: coverpoint ((observed.word_addr % `BANK_COUNT) + observed.beats > `BANK_COUNT); + stalled: coverpoint (observed.completion_stall > 1); + direction_bank: cross direction, start_bank; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void write(command t); + observed = t; + behavior.sample(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/env.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/env.svh new file mode 100644 index 000000000..df09ca487 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/env.svh @@ -0,0 +1,27 @@ +class env extends environment #(operation, item, ref_model); + `uvm_component_utils(env) + command_agent commands; + axis_source_agent #(32) source; + axis_sink #(32) sink; + axis_monitor #(32) output_monitor; + reset_controller reset; + coverage cov; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + commands = command_agent::type_id::create("commands", this); + source = axis_source_agent#(32)::type_id::create("source", this); + sink = axis_sink#(32)::type_id::create("sink", this); + output_monitor = axis_monitor#(32)::type_id::create("output_monitor", this); + reset = reset_controller::type_id::create("reset", this); + cov = coverage::type_id::create("coverage", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + commands.monitor.ap.connect(input_export); + commands.monitor.commands.connect(cov.analysis_export); + output_monitor.ap.connect(output_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/ref_model.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/ref_model.svh new file mode 100644 index 000000000..1f70d9e21 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/env/ref_model.svh @@ -0,0 +1,25 @@ +class ref_model extends reference_model #(operation, item); + `uvm_component_utils(ref_model) + chandle model; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + model = bank_ref_create(`BANK_WORDS); + endfunction + function void write(operation t); + int unsigned result = bank_ref_access(model, t.word_addr, t.write, t.data, t.mask); + if (!t.write) begin + item expected = item::type_id::create("expected"); + expected.data = result; + expected.keep = 15; + expected.last = t.last; + expected_ap.write(expected); + end + endfunction + function void final_phase(uvm_phase phase); + bank_ref_destroy(model); + super.final_phase(phase); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/filelist.f b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/filelist.f new file mode 100644 index 000000000..073bd96cb --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/filelist.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/protocol/axis ++incdir+@RESOURCES@ +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/axis/interface.sv +@VERIFY@/uvm/src/protocol/axis/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/BankSet/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/interface.sv b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/interface.sv new file mode 100644 index 000000000..6a88956be --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/interface.sv @@ -0,0 +1,20 @@ +`include "parameters.svh" +interface port_if ( + input logic clock, + input logic reset +); + logic valid; + logic ready; + logic write; + logic [`BANK_ADDRESS_BITS-1:0] addr; + logic [31:0] beats; + logic done_valid; + logic done_ready; + logic error; + + assert property (@(posedge clock) disable iff (reset) + done_valid && !done_ready |=> done_valid && $stable( + error + )) + else $fatal(1, "BankSet completion changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/package.sv b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/package.sv new file mode 100644 index 000000000..52eb64096 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/package.sv @@ -0,0 +1,28 @@ +`include "parameters.svh" +package bankset_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import axis_vip_pkg::*; + `include "uvm_macros.svh" + typedef axis_item#(32) item; + + import "DPI-C" function chandle bank_ref_create(input int unsigned entries); + import "DPI-C" function void bank_ref_destroy(input chandle model); + import "DPI-C" function int unsigned bank_ref_access( + input chandle model, + input int unsigned addr, + input byte unsigned write, + input int unsigned data, + input byte unsigned mask + ); + `include "agent/item.svh" + `include "agent/command.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/coverage.svh" + `include "env/ref_model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/seq/sequence.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/seq/sequence.svh new file mode 100644 index 000000000..bae1c213d --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/seq/sequence.svh @@ -0,0 +1,77 @@ +class write_sequence extends uvm_sequence #(item); + `uvm_object_utils(write_sequence) + int unsigned beats; + bit masked; + function new(string name = "write_sequence"); + super.new(name); + endfunction + task body(); + item req; + for (int i = 0; i < beats; i++) begin + req = item::type_id::create("req"); + start_item(req); + req.data = 32'h9e3779b9 * (i + 1); + req.keep = masked ? (i % 16) : 15; + req.last = i == beats - 1; + finish_item(req); + end + endtask +endclass + +class command_sequence extends uvm_sequence #(command); + `uvm_object_utils(command_sequence) + command req; + function new(string name = "command_sequence"); + super.new(name); + endfunction + task body(); + start_item(req); + finish_item(req); + endtask +endclass + +class access_sequence extends uvm_sequence; + `uvm_object_utils(access_sequence) + env test_env; + int unsigned reads = 0; + int unsigned commands = 0; + function new(string name = "access_sequence"); + super.new(name); + endfunction + task transfer(bit write, int unsigned word_addr, int unsigned beats, bit masked = 0, + int unsigned stall = 0); + command_sequence cmd = command_sequence::type_id::create("command"); + write_sequence data = write_sequence::type_id::create("data"); + cmd.req = command::type_id::create("req"); + cmd.req.write = write; + cmd.req.word_addr = word_addr; + cmd.req.beats = beats; + cmd.req.completion_stall = stall; + data.beats = beats; + data.masked = masked; + fork + cmd.start(test_env.commands.seqr); + if (write) data.start(test_env.source.seqr); + join + commands++; + if (!write) begin + reads += beats; + test_env.scoreboard.wait_checked(reads); + end + endtask + task body(); + test_env.reset.pulse(4); + transfer(1, 0, `BANK_WORDS); + test_env.sink.stall_cycles = 0; + transfer(0, 0, `BANK_WORDS); + test_env.sink.stall_cycles = 3; + for (int b = 0; b < `BANK_COUNT; b++) begin + transfer(1, b, `BANK_COUNT + 1, 1, 3); + transfer(0, b, `BANK_COUNT + 1, 0, 3); + end + transfer(1, `BANK_WORDS - 1, 1); + transfer(0, `BANK_WORDS - 1, 1); + test_env.reset.pulse(2); + transfer(0, 0, `BANK_WORDS, 0, 3); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/tb.sv b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/tb.sv new file mode 100644 index 000000000..888283893 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/tb.sv @@ -0,0 +1,49 @@ +module bankset_tb; + import uvm_pkg::*; + import bankset_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + port_if control ( + clock, + timing.reset + ); + axis_if source_if (clock); + axis_if sink_if (clock); + assign source_if.reset = control.reset; + assign sink_if.reset = control.reset; + BankSet dut ( + .clock(clock), + .reset(control.reset), + .io_command_valid(control.valid), + .io_command_ready(control.ready), + .io_command_bits_write(control.write), + .io_command_bits_addr(control.addr), + .io_command_bits_beats(control.beats), + .io_write_valid(source_if.tvalid), + .io_write_ready(source_if.tready), + .io_write_bits_tdata(source_if.tdata), + .io_write_bits_tkeep(source_if.tkeep), + .io_write_bits_tlast(source_if.tlast), + .io_read_valid(sink_if.tvalid), + .io_read_ready(sink_if.tready), + .io_read_bits_tdata(sink_if.tdata), + .io_read_bits_tkeep(sink_if.tkeep), + .io_read_bits_tlast(sink_if.tlast), + .io_done_valid(control.done_valid), + .io_done_ready(control.done_ready), + .io_done_bits(control.error) + ); + initial begin + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top.env.reset", "timing", timing); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "control", control); + uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.commands.monitor", "write_vif", + source_if); + uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.source.*", "vif", source_if); + uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.sink", "vif", sink_if); + uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.output_monitor", "vif", sink_if); + uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.commands.monitor", "read_vif", + sink_if); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/tests/test.svh b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/tests/test.svh new file mode 100644 index 000000000..956039dc5 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/bankset/tests/test.svh @@ -0,0 +1,20 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + access_sequence seq = access_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.start(null); + evidence::check(test_env.commands.monitor.completed == seq.commands, + "BankSet command completion count mismatch"); + evidence::check(test_env.scoreboard.checked == seq.reads, + "BankSet read comparison count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/bank/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/bank/src/main/resources/suite.toml new file mode 100644 index 000000000..009338044 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/resources/suite.toml @@ -0,0 +1,37 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libbank_reference_model" + +[[tests]] +functional_groups = ["bank_pkg::coverage::accesses"] +name = "bank" +top = "bank_tb" +test = "protocol_test" +filelist = "bank/filelist.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = ["bankset_pkg::coverage::behavior"] +name = "bankset" +top = "bankset_tb" +test = "protocol_test" +filelist = "bankset/filelist.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/ip/bank/src/main/scala/Bank.scala b/arch/src/main/scala/framework/ip/bank/src/main/scala/Bank.scala new file mode 100644 index 000000000..8f00153a8 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/scala/Bank.scala @@ -0,0 +1,68 @@ +package framework.ip.memory.bank + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +class BankRequest(p: BankSetParams) extends Bundle { + val addr = UInt(p.rowBits.W) + val write = Bool() + val data = UInt(p.dataBits.W) + val mask = UInt(p.bytes.W) + val tag = UInt(p.tagBits.W) +} + +class BankResponse(p: BankSetParams) extends Bundle { + val data = UInt(p.dataBits.W) + val tag = UInt(p.tagBits.W) + val error = Bool() +} + +/** One physical SRAM bank. Cache and NoC policy remain outside this module. */ +@instantiable +class Bank(p: BankSetParams) extends Module { + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new BankRequest(p))) + val response = Decoupled(new BankResponse(p)) + }) + + val memory = SyncReadMem(p.entriesPerBank, Vec(p.bytes, UInt(8.W))) + val request = Reg(new BankRequest(p)) + val responseValid = RegInit(false.B) + val readPending = RegInit(false.B) + val responseData = Reg(UInt(p.dataBits.W)) + + val readData = memory.readWrite( + io.request.bits.addr, + io.request.bits.data.asTypeOf(Vec(p.bytes, UInt(8.W))), + io.request.bits.mask.asBools, + io.request.fire, + io.request.bits.write + ) + + io.request.ready := !responseValid && !readPending + when(io.request.fire) { + request := io.request.bits + when(io.request.bits.write) { + responseData := 0.U + responseValid := true.B + }.otherwise { + readPending := true.B + } + } + when(readPending) { + responseData := readData.asUInt + responseValid := true.B + readPending := false.B + } + + io.response.valid := responseValid + io.response.bits.data := responseData + io.response.bits.tag := request.tag + io.response.bits.error := false.B + when(io.response.fire) { + responseValid := false.B + } +} diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/scala/BankSet.scala b/arch/src/main/scala/framework/ip/bank/src/main/scala/BankSet.scala similarity index 98% rename from arch/src/main/scala/framework/mem-core/bank/src/main/scala/BankSet.scala rename to arch/src/main/scala/framework/ip/bank/src/main/scala/BankSet.scala index 01da682d6..649d13dfc 100644 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/scala/BankSet.scala +++ b/arch/src/main/scala/framework/ip/bank/src/main/scala/BankSet.scala @@ -1,9 +1,9 @@ -package memcore.memory.bank +package framework.ip.memory.bank import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.axi.Beat +import framework.ip.bus.axi.Beat case class BankSetParams( dataBits: Int, diff --git a/arch/src/main/scala/framework/ip/bank/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/bank/src/main/scala/Emit.scala new file mode 100644 index 000000000..9722320e2 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/scala/Emit.scala @@ -0,0 +1,39 @@ +package framework.ip.memory.bank + +import java.nio.file.{Files, Paths} + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + + val p = BankSetParams( + values("dataBits").num.toInt, + values("banks").num.toInt, + values("entriesPerBank").num.toInt, + values("tagBits").num.toInt + ) + + require(p.dataBits == 32, "The Bank reference model uses 32-bit words") + Files.createDirectories(Paths.get(output)) + Files.writeString( + Paths.get(output, "parameters.svh"), + s"""`ifndef BANK_PARAMETERS +`define BANK_PARAMETERS +`define BANK_ROWS ${p.entriesPerBank} +`define BANK_COUNT ${p.banks} +`define BANK_WORDS ${p.entriesPerBank * p.banks} +`define BANK_ROW_BITS ${p.rowBits} +`define BANK_ADDRESS_BITS ${p.addressBits} +`define BANK_TAG_BITS ${p.tagBits} +`endif +""" + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Bank(p), + args = Array("--target-dir", s"$output/Bank", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new BankSet(p), + args = Array("--target-dir", s"$output/BankSet", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/ip/bank/src/main/scala/SramIO.scala b/arch/src/main/scala/framework/ip/bank/src/main/scala/SramIO.scala new file mode 100644 index 000000000..1a12e62e2 --- /dev/null +++ b/arch/src/main/scala/framework/ip/bank/src/main/scala/SramIO.scala @@ -0,0 +1,36 @@ +package framework.ip.memory.bank + +import chisel3._ +import chisel3.util._ +import framework.acc.memdomain.configs.MemDomainParam + +/** + * Generic SRAM interface definitions + */ +class SramReadReq(val b: MemDomainParam) extends Bundle { + val addr = UInt(16.W) +} + +class SramReadResp(val b: MemDomainParam) extends Bundle { + val data = UInt(b.bankWidth.W) +} + +class SramReadIO(val b: MemDomainParam) extends Bundle { + val req = Flipped(Decoupled(new SramReadReq(b))) + val resp = Decoupled(new SramReadResp(b)) +} + +class SramWriteReq(val b: MemDomainParam) extends Bundle { + val addr = UInt(16.W) + val mask = Vec(b.bankMaskLen, Bool()) + val data = UInt(b.bankWidth.W) +} + +class SramWriteIO(val b: MemDomainParam) extends Bundle { + val req = Flipped(Decoupled(new SramWriteReq(b))) + val resp = Decoupled(new SramWriteResp(b)) +} + +class SramWriteResp(val b: MemDomainParam) extends Bundle { + val ok = Bool() +} diff --git a/arch/src/main/scala/framework/ip/build.sc b/arch/src/main/scala/framework/ip/build.sc new file mode 100644 index 000000000..f24913710 --- /dev/null +++ b/arch/src/main/scala/framework/ip/build.sc @@ -0,0 +1,34 @@ +import mill._ +import $file.axi4.{build => axi4_build} +import $file.axis.{build => axis_build} +import $file.bank.{build => bank_build} +import $file.cache.{build => cache_build} +import $file.chi.{build => chi_build} +import $file.coherence.{build => coherence_build} +import $file.interlock.{build => interlock_build} +import $file.mesh_shm.{build => mesh_shm_build} +import $file.spm.{build => spm_build} + +def sources = T.sources( + axi4_build.sources() ++ + axis_build.sources() ++ + bank_build.sources() ++ + cache_build.sources() ++ + chi_build.sources() ++ + coherence_build.sources() ++ + interlock_build.sources() ++ + mesh_shm_build.sources() ++ + spm_build.sources() +) + +def emitSources = T.sources( + axi4_build.emitSources() ++ + axis_build.emitSources() ++ + bank_build.emitSources() ++ + cache_build.emitSources() ++ + chi_build.emitSources() ++ + coherence_build.emitSources() ++ + interlock_build.emitSources() ++ + mesh_shm_build.emitSources() ++ + spm_build.emitSources() +) diff --git a/arch/src/main/scala/framework/ip/cache/build.sc b/arch/src/main/scala/framework/ip/cache/build.sc new file mode 100644 index 000000000..c377d90c7 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.memory.cache.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("configs/CacheParams.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Cache.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Interface.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/cache/configs/CacheParams.scala b/arch/src/main/scala/framework/ip/cache/configs/CacheParams.scala new file mode 100644 index 000000000..18f4e7a10 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/configs/CacheParams.scala @@ -0,0 +1,39 @@ +package framework.ip.memory.cache.configs + +import buckyball.config.CacheHardware + +import chisel3.util.{isPow2, log2Ceil} + +case class CacheParams( + addressBits: Int, + lineBytes: Int, + sets: Int, + ways: Int, + idBits: Int, + metadataBits: Int, + responseDepth: Int) { + require(lineBytes >= 8 && isPow2(lineBytes)) + require(sets >= 2 && isPow2(sets)) + require(ways >= 1 && isPow2(ways)) + require(idBits > 0 && metadataBits > 0 && responseDepth >= 2) + val offsetBits = log2Ceil(lineBytes) + val setBits = log2Ceil(sets) + val wayBits = math.max(1, log2Ceil(ways)) + val tagBits = addressBits - offsetBits - setBits + val lineBits = lineBytes * 8 + require(tagBits > 0) +} + +object CacheParams { + + def fromProto(p: CacheHardware): CacheParams = CacheParams( + p.getAddressBits, + p.getLineBytes, + p.getSets, + p.getWays, + p.getIdBits, + p.getMetadataBits, + p.getResponseDepth + ) + +} diff --git a/arch/src/main/scala/framework/mem-core/cache/configs/default.toml b/arch/src/main/scala/framework/ip/cache/configs/default.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/configs/default.toml rename to arch/src/main/scala/framework/ip/cache/configs/default.toml diff --git a/arch/src/main/scala/framework/ip/cache/configs/uvm.toml b/arch/src/main/scala/framework/ip/cache/configs/uvm.toml new file mode 100644 index 000000000..e47fe594a --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/configs/uvm.toml @@ -0,0 +1,10 @@ +[parameters] +addressBits = 44 +lineBytes = 64 +sets = 4 +ways = 2 +idBits = 8 +metadataBits = 4 +responseDepth = 2 + +[defines] diff --git a/arch/src/main/scala/framework/mem-core/cache/src/csrc/Cargo.lock b/arch/src/main/scala/framework/ip/cache/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/ip/cache/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/cache/src/csrc/Cargo.toml b/arch/src/main/scala/framework/ip/cache/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/ip/cache/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/src/lib.rs b/arch/src/main/scala/framework/ip/cache/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/coherence/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/ip/cache/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/mem-core/cache/src/csrc/src/memory.rs b/arch/src/main/scala/framework/ip/cache/src/csrc/src/memory.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/src/csrc/src/memory.rs rename to arch/src/main/scala/framework/ip/cache/src/csrc/src/memory.rs diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/agent.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/agent.svh new file mode 100644 index 000000000..ee01d47d4 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/agent.svh @@ -0,0 +1,20 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + driver drv; + monitor mon; + sink receiver; + uvm_sequencer #(request_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + receiver = sink::type_id::create("sink", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..d90b97f2e --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/driver.svh @@ -0,0 +1,57 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual cache_request_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual cache_request_if)::get(this, "", "request", vif)) + `uvm_fatal("VIF", "Cache request interface is required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + int cycles; + vif.reset = 1; + vif.valid = 0; + vif.id = 0; + vif.op = 0; + vif.addr = 0; + vif.way = 0; + vif.data = 0; + vif.mask = 0; + vif.metadata = 0; + vif.eligible = 0; + forever begin + seq_item_port.get_next_item(req); + if (req.reset_command) begin + vif.cb.valid <= 0; + vif.cb.reset <= 1; + repeat (req.reset_cycles) @(vif.cb); + vif.cb.reset <= 0; + @(vif.cb); + end else begin + vif.cb.valid <= 1; + vif.cb.id <= req.id; + vif.cb.op <= req.op; + vif.cb.addr <= req.addr; + vif.cb.way <= req.way; + vif.cb.data <= req.data; + vif.cb.mask <= req.mask; + vif.cb.metadata <= req.metadata; + vif.cb.eligible <= req.eligible; + cycles = 0; + do begin + @(vif.cb); + cycles++; + end while (!vif.cb.ready); + if (req.check_rate) + evidence::check(cycles == 1, + "Cache inserted a request bubble without response backpressure"); + vif.cb.valid <= 0; + if (req.idle_between) @(vif.cb); + end + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/item.svh new file mode 100644 index 000000000..6a24ad548 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/item.svh @@ -0,0 +1,40 @@ +class request_item extends uvm_sequence_item; + bit reset_command; + int unsigned reset_cycles; + bit check_rate; + bit idle_between; + logic [`CACHE_ID_BITS-1:0] id; + logic [2:0] op; + logic [`CACHE_ADDR_BITS-1:0] addr; + logic [`CACHE_WAY_BITS-1:0] way; + bit [`CACHE_LINE_BITS-1:0] data; + bit [`CACHE_LINE_BYTES-1:0] mask; + logic [`CACHE_META_BITS-1:0] metadata; + bit [`CACHE_WAYS-1:0] eligible; + `uvm_object_utils(request_item) + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [`CACHE_ID_BITS-1:0] id; + logic hit, available, entry_valid; + logic [`CACHE_WAY_BITS-1:0] way; + logic [`CACHE_ADDR_BITS-1:0] addr; + logic [`CACHE_LINE_BITS-1:0] data; + logic [`CACHE_META_BITS-1:0] metadata; + `uvm_object_utils_begin(response_item) + `uvm_field_int(id, UVM_ALL_ON) + `uvm_field_int(hit, UVM_ALL_ON) + `uvm_field_int(available, UVM_ALL_ON) + `uvm_field_int(entry_valid, UVM_ALL_ON) + `uvm_field_int(way, UVM_ALL_ON) + `uvm_field_int(addr, UVM_ALL_ON) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(metadata, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..6f7e158eb --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/monitor.svh @@ -0,0 +1,60 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual cache_request_if request; + virtual cache_response_if response; + uvm_analysis_port #(request_item) request_ap; + uvm_analysis_port #(response_item) response_ap; + uvm_analysis_port #(bit) reset_ap; + int unsigned accepted = 0, retired = 0; + function new(string name, uvm_component parent); + super.new(name, parent); + request_ap = new("request_ap", this); + response_ap = new("response_ap", this); + reset_ap = new("reset_ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual cache_request_if)::get( + this, "", "request", request + ) || !uvm_config_db#(virtual cache_response_if)::get( + this, "", "response", response + )) + `uvm_fatal("VIF", "Cache monitor interfaces are required") + endfunction + task run_phase(uvm_phase phase); + request_item req; + response_item rsp; + forever begin + @(posedge request.clock); + if (request.reset) reset_ap.write(1); + else begin + if (request.valid && request.ready) begin + req = request_item::type_id::create("request"); + req.id = request.id; + req.op = request.op; + req.addr = request.addr; + req.way = request.way; + req.data = request.data; + req.mask = request.mask; + req.metadata = request.metadata; + req.eligible = request.eligible; + request_ap.write(req); + accepted++; + end + if (response.valid && response.ready) begin + rsp = response_item::type_id::create("response"); + rsp.id = response.id; + rsp.hit = response.hit; + rsp.available = response.available; + rsp.entry_valid = response.entry_valid; + rsp.way = response.way; + rsp.addr = response.addr; + rsp.data = response.data; + rsp.metadata = response.metadata; + response_ap.write(rsp); + retired++; + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/sink.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/sink.svh new file mode 100644 index 000000000..e44a8a52b --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/agent/sink.svh @@ -0,0 +1,19 @@ +class sink extends uvm_component; + `uvm_component_utils(sink) + virtual cache_response_if vif; + bit blocked = 0; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual cache_response_if)::get(this, "", "response", vif)) + `uvm_fatal("VIF", "Cache response interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(vif.cb); + vif.cb.ready <= !vif.reset && !blocked; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/env/coverage.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/env/coverage.svh new file mode 100644 index 000000000..605384f05 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/env/coverage.svh @@ -0,0 +1,23 @@ +class coverage extends uvm_subscriber #(request_item); + `uvm_component_utils(coverage) + covergroup operations with function sample (request_item req); + option.per_instance = 1; + operation: coverpoint req.op {bins supported[] = {[0 : 4]};} + set_index: coverpoint ((req.addr / `CACHE_LINE_BYTES) % `CACHE_SETS) { + bins sets[] = {[0 : `CACHE_SETS - 1]}; + } + way: coverpoint req.way {bins ways[] = {[0 : `CACHE_WAYS - 1]};} + mask: coverpoint req.mask iff (req.op == WRITE) { + bins none = {'0}; bins all = {'1}; bins partial = default; + } + eligibility: coverpoint (req.eligible != 0); + operation_set: cross operation, set_index; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + operations = new(); + endfunction + function void write(request_item t); + operations.sample(t); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/env/env.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/env/env.svh new file mode 100644 index 000000000..5ba1235cd --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/env/env.svh @@ -0,0 +1,26 @@ +class env extends environment #(request_item, response_item, ref_model); + `uvm_component_utils(env) + agent bus; + coverage cov; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + bus = agent::type_id::create("bus", this); + cov = coverage::type_id::create("coverage", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.request_ap.connect(input_export); + bus.mon.request_ap.connect(cov.analysis_export); + bus.mon.response_ap.connect(output_export); + bus.mon.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + cache_ref_reset(model.model); + endfunction +endclass diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/ref_model.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/env/ref_model.svh similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/src/main/resources/ref_model.svh rename to arch/src/main/scala/framework/ip/cache/src/main/resources/env/ref_model.svh diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/filelist.f b/arch/src/main/scala/framework/ip/cache/src/main/resources/filelist.f new file mode 100644 index 000000000..00e97f384 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@RTL@ ++incdir+@RESOURCES@ +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Cache/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/interface.sv b/arch/src/main/scala/framework/ip/cache/src/main/resources/interface.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/src/main/resources/interface.sv rename to arch/src/main/scala/framework/ip/cache/src/main/resources/interface.sv diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/package.sv b/arch/src/main/scala/framework/ip/cache/src/main/resources/package.sv new file mode 100644 index 000000000..b5cccf94f --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/package.sv @@ -0,0 +1,41 @@ +`include "cache_config.svh" +package cache_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + localparam LOOKUP = 0, READ = 1, WRITE = 2, FILL = 3, INVALIDATE = 4; + import "DPI-C" function chandle cache_ref_create( + input int unsigned sets, + ways, + line_bytes + ); + import "DPI-C" function void cache_ref_destroy(input chandle model); + import "DPI-C" function void cache_ref_reset(input chandle model); + import "DPI-C" function void cache_ref_access( + input chandle model, + input int unsigned op, + input longint unsigned addr, + input int unsigned way, + input bit [`CACHE_LINE_BITS-1:0] data, + input bit [`CACHE_LINE_BYTES-1:0] mask, + input int unsigned metadata, + input bit [`CACHE_WAYS-1:0] eligible, + output byte unsigned hit, + available, + output int unsigned out_way, + output byte unsigned entry_valid, + output longint unsigned out_addr, + output bit [`CACHE_LINE_BITS-1:0] out_data, + output int unsigned out_metadata + ); + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/sink.svh" + `include "agent/agent.svh" + `include "env/ref_model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..02ca880e5 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/seq/sequence.svh @@ -0,0 +1,122 @@ +class access_sequence extends uvm_sequence #(request_item); + `uvm_object_utils(access_sequence) + env test_env; + int unsigned expected = 0; + bit [`CACHE_ID_BITS-1:0] serial = 0; + bit check_rate = 1, idle_between = 0; + function new(string name = "access_sequence"); + super.new(name); + endfunction + task pulse_reset(int unsigned cycles); + request_item req = request_item::type_id::create("reset"); + start_item(req); + req.reset_command = 1; + req.reset_cycles = cycles; + finish_item(req); + endtask + function longint unsigned address(int tag, int set_index); + return (longint'(tag) * `CACHE_SETS + set_index) * `CACHE_LINE_BYTES; + endfunction + task send(int op, longint unsigned addr, int way = 0, bit [`CACHE_LINE_BYTES-1:0] mask = '1, + bit [`CACHE_WAYS-1:0] eligible = '1, bit [`CACHE_LINE_BITS-1:0] data = '0, + int metadata = 0); + request_item req = request_item::type_id::create("access"); + start_item(req); + req.id = serial; + req.op = op; + req.addr = addr; + req.way = way; + req.data = data; + req.mask = mask; + req.metadata = metadata; + req.eligible = eligible; + req.check_rate = check_rate; + req.idle_between = idle_between; + finish_item(req); + expected++; + serial++; + endtask + task drain(); + test_env.scoreboard.wait_checked(expected); + @(test_env.bus.mon.request.cb); + endtask + task body(); + pulse_reset(4); + send(LOOKUP, 0, 0, '0, '0); + send(LOOKUP, 0); + for (int s = 0; s < `CACHE_SETS; s++) begin + for (int w = 0; w < `CACHE_WAYS; w++) begin + send(READ, address(w + 1, s), w); + send(FILL, address(w + 1, s), w, '1, '1, w[0] ? '1 : '0, w); + send(LOOKUP, address(w + 1, s)); + send(WRITE, address(w + 1, s), w, {(`CACHE_LINE_BYTES / 2) {2'b01}}, '1, w[0] ? '0 : '1, + '1); + send(READ, address(w + 1, s), w); + end + end + send(LOOKUP, address(`CACHE_WAYS + 1, 0)); + send(LOOKUP, address(`CACHE_WAYS + 1, 0), 0, '0, '0); + for (int w = 0; w < `CACHE_WAYS; w++) send(LOOKUP, address(`CACHE_WAYS + 1, 0), 0, '0, 1 << w); + send(INVALIDATE, address(1, 0), 0); + send(LOOKUP, address(1, 0)); + send(FILL, address(`CACHE_WAYS + 1, 0), 0, '1, '1, '1, '1); + send(LOOKUP, address(`CACHE_WAYS + 2, 0)); + send(WRITE, address(`CACHE_WAYS + 1, 0), 0, '0, '1, '0, 0); + send(READ, address(`CACHE_WAYS + 1, 0), 0); + send(INVALIDATE, address(`CACHE_WAYS + 1, 0), 0); + send(INVALIDATE, address(`CACHE_WAYS + 1, 0), 0); + drain(); + + idle_between = 1; + for (int s = 0; s < `CACHE_SETS; s++) begin + for (int w = 0; w < `CACHE_WAYS; w++) send(INVALIDATE, address(0, s), w); + for (int w = 0; w < `CACHE_WAYS; w++) begin + longint unsigned high_addr = (((64'h1 << `CACHE_ADDR_BITS) - 1) & + ~(longint'(`CACHE_SETS * `CACHE_LINE_BYTES) - 1)) | (s * `CACHE_LINE_BYTES); + send(FILL, address(0, s), w, '1, '1, '0, 0); + send(LOOKUP, address(0, s), 0, '0, '0); + send(FILL, high_addr, w, '1, '1, '1, '1); + send(LOOKUP, high_addr); + send(WRITE, high_addr, w, '1, '1, '0, 0); + send(READ, high_addr, w); + send(FILL, address(0, s), w, '1, '1, '0, 0); + send(READ, address(0, s), w); + send(INVALIDATE, address(0, s), w); + end + end + drain(); + + idle_between = 0; + serial = '1; + send(READ, address(0, 0), 0, '0, '0); + drain(); + serial = 0; + send(READ, address(0, 0), 0, '0, '0); + drain(); + check_rate = 0; + test_env.bus.receiver.blocked = 1; + repeat (2) @(test_env.bus.mon.request.cb); + fork + begin + for (int i = 0; i < `CACHE_RESPONSE_DEPTH + 2; i++) send(LOOKUP, address(2, 0)); + end + begin + repeat (`CACHE_RESPONSE_DEPTH + 5) @(test_env.bus.mon.request.cb); + test_env.bus.receiver.blocked = 0; + end + join + drain(); + + test_env.bus.receiver.blocked = 1; + repeat (2) @(test_env.bus.mon.request.cb); + for (int i = 0; i < `CACHE_RESPONSE_DEPTH; i++) send(READ, address(1, 1), 0); + repeat (2) @(test_env.bus.mon.request.cb); + pulse_reset(2); + expected = test_env.scoreboard.checked; + test_env.bus.receiver.blocked = 0; + @(test_env.bus.mon.request.cb); + for (int s = 0; s < `CACHE_SETS; s++) send(LOOKUP, address(1, s)); + drain(); + evidence::check(test_env.bus.mon.retired == expected, "Cache response count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/cache/src/main/resources/suite.toml new file mode 100644 index 000000000..9aa170f00 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libcache_reference_model" + +[[tests]] +functional_groups = ["cache_pkg::coverage::operations"] +name = "cache" +top = "cache_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache_tb.sv b/arch/src/main/scala/framework/ip/cache/src/main/resources/tb.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache_tb.sv rename to arch/src/main/scala/framework/ip/cache/src/main/resources/tb.sv diff --git a/arch/src/main/scala/framework/ip/cache/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/ip/cache/src/main/resources/tests/test.svh new file mode 100644 index 000000000..f24dba40a --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/resources/tests/test.svh @@ -0,0 +1,18 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + access_sequence seq = access_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.start(test_env.bus.seqr); + evidence::check(test_env.bus.mon.accepted == test_env.scoreboard.checked + evidence::cancelled, + "Cache transaction accounting mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/cache/src/main/scala/Cache.scala b/arch/src/main/scala/framework/ip/cache/src/main/scala/Cache.scala new file mode 100644 index 000000000..eeb087eec --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/scala/Cache.scala @@ -0,0 +1,112 @@ +package framework.ip.memory.cache + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.lib.queue.Queue + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.ip.memory.cache.configs.CacheParams + +@instantiable +class Cache(p: CacheParams) extends Module { + @public + val io = IO(new CacheIO(p)) + + val valid = RegInit(VecInit(Seq.fill(p.ways)(VecInit(Seq.fill(p.sets)(false.B))))) + val tags = Reg(Vec(p.ways, Vec(p.sets, UInt(p.tagBits.W)))) + val metadata = Reg(Vec(p.ways, Vec(p.sets, UInt(p.metadataBits.W)))) + val data = Seq.fill(p.ways)(SyncReadMem(p.sets, Vec(p.lineBytes, UInt(8.W)))) + val nextWay = RegInit(VecInit(Seq.fill(p.sets)(0.U(p.wayBits.W)))) + val responses = Instantiate(new Queue(new CacheResponse(p), p.responseDepth, true)) + val pending = RegInit(false.B) + val pendingRead = Reg(Bool()) + val pendingWay = Reg(UInt(p.wayBits.W)) + val result = Reg(new CacheResponse(p)) + + val req = io.request.bits + val set = req.addr(p.offsetBits + p.setBits - 1, p.offsetBits) + val tag = req.addr(p.addressBits - 1, p.offsetBits + p.setBits) + val hits = VecInit((0 until p.ways).map(w => valid(w)(set) && tags(w)(set) === tag)).asUInt + val invalid = VecInit((0 until p.ways).map(w => !valid(w)(set) && req.eligible(w))).asUInt + + val order = VecInit((0 until p.ways).map { offset => + val way = if (p.ways == 1) 0.U else (nextWay(set) + offset.U)(p.wayBits - 1, 0) + req.eligible(way) + }) + + val replacement = + if (p.ways == 1) 0.U + else + (nextWay(set) + PriorityEncoder(order.asUInt))(p.wayBits - 1, 0) + + val lookupWay = Mux(hits.orR, PriorityEncoder(hits), Mux(invalid.orR, PriorityEncoder(invalid), replacement)) + val lookup = req.op === CacheOp.Lookup.U + val selected = Mux(lookup, lookupWay, req.way) + val available = !lookup || hits.orR || req.eligible.orR + val entryValid = available && valid(selected)(set) + val read = (lookup || req.op === CacheOp.Read.U) && entryValid + + io.request.ready := (responses.io.count +& pending.asUInt) < p.responseDepth.U || responses.io.deq.fire + responses.io.enq.valid := pending + responses.io.enq.bits := result + io.response <> responses.io.deq + pending := io.request.fire + assert(!pending || responses.io.enq.ready, "Cache response capacity reservation failed") + + val readData = Wire(Vec(p.ways, UInt(p.lineBits.W))) + for (way <- 0 until p.ways) { + val write = req.op === CacheOp.Write.U || req.op === CacheOp.Fill.U + val mask = Mux(req.op === CacheOp.Fill.U, Fill(p.lineBytes, 1.U(1.W)), req.mask) + readData(way) := data(way).readWrite( + set, + req.data.asTypeOf(Vec(p.lineBytes, UInt(8.W))), + mask.asBools, + io.request.fire && selected === way.U && (read || write), + write + ).asUInt + } + when(pendingRead) { + responses.io.enq.bits.data := readData(pendingWay) + } + + when(io.request.fire) { + assert(req.op <= CacheOp.Invalidate.U, "Unknown Cache operation") + assert(req.addr(p.offsetBits - 1, 0) === 0.U, "Cache operations require a line-aligned address") + assert(lookup || req.way < p.ways.U, "Cache way exceeds associativity") + assert(PopCount(hits) <= 1.U, "Duplicate Cache tag") + pendingRead := read + pendingWay := selected + result.id := req.id + result.hit := lookup && hits.orR + result.available := available + result.way := Mux(available, selected, 0.U) + result.entryValid := entryValid + result.addr := Mux(entryValid, Cat(tags(selected)(set), set, 0.U(p.offsetBits.W)), 0.U) + result.data := 0.U + result.metadata := Mux(entryValid, metadata(selected)(set), 0.U) + + when(req.op === CacheOp.Fill.U) { + assert(!hits.orR || hits(req.way), "Cache fill would create a duplicate tag") + valid(req.way)(set) := true.B + tags(req.way)(set) := tag + metadata(req.way)(set) := req.metadata + nextWay(set) := (if (p.ways == 1) 0.U else (req.way + 1.U)(p.wayBits - 1, 0)) + result.entryValid := true.B + result.addr := req.addr + result.metadata := req.metadata + } + when(req.op === CacheOp.Write.U) { + assert(entryValid && hits(req.way), "Cache write requires the matching resident line") + metadata(req.way)(set) := req.metadata + result.metadata := req.metadata + } + when(req.op === CacheOp.Invalidate.U) { + valid(req.way)(set) := false.B + result.entryValid := false.B + result.addr := 0.U + result.metadata := 0.U + } + } +} diff --git a/arch/src/main/scala/framework/ip/cache/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/cache/src/main/scala/Emit.scala new file mode 100644 index 000000000..6892a1966 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/scala/Emit.scala @@ -0,0 +1,43 @@ +package framework.ip.memory.cache + +import java.nio.file.{Files, Paths} +import framework.ip.memory.cache.configs.CacheParams + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val output = Paths.get(directory) + Files.createDirectories(output) + + val p = CacheParams( + values("addressBits").num.toInt, + values("lineBytes").num.toInt, + values("sets").num.toInt, + values("ways").num.toInt, + values("idBits").num.toInt, + values("metadataBits").num.toInt, + values("responseDepth").num.toInt + ) + + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Cache(p), + args = Array("--target-dir", output.resolve("Cache").toString, "--split-verilog") + ) + + val constants = Seq( + "ADDR_BITS" -> p.addressBits, + "LINE_BYTES" -> p.lineBytes, + "LINE_BITS" -> p.lineBits, + "SETS" -> p.sets, + "WAYS" -> p.ways, + "WAY_BITS" -> p.wayBits, + "ID_BITS" -> p.idBits, + "META_BITS" -> p.metadataBits, + "RESPONSE_DEPTH" -> p.responseDepth + ).map { case (name, value) => s"`define CACHE_$name $value" }.mkString("\n") + + Files.writeString( + output.resolve("cache_config.svh"), + s"`ifndef CACHE_CONFIG_SVH\n`define CACHE_CONFIG_SVH\n$constants\n`endif\n" + ) +} diff --git a/arch/src/main/scala/framework/ip/cache/src/main/scala/Interface.scala b/arch/src/main/scala/framework/ip/cache/src/main/scala/Interface.scala new file mode 100644 index 000000000..9fa53bc07 --- /dev/null +++ b/arch/src/main/scala/framework/ip/cache/src/main/scala/Interface.scala @@ -0,0 +1,42 @@ +package framework.ip.memory.cache + +import chisel3._ +import chisel3.util._ +import framework.ip.memory.cache.configs.CacheParams + +object CacheOp { + val Lookup = 0 + val Read = 1 + val Write = 2 + val Fill = 3 + val Invalidate = 4 +} + +class CacheRequest(p: CacheParams) extends Bundle { + val id = UInt(p.idBits.W) + val op = UInt(3.W) + // Read and Invalidate address a physical set/way; all operations are line aligned. + val addr = UInt(p.addressBits.W) + val way = UInt(p.wayBits.W) + val data = UInt(p.lineBits.W) + val mask = UInt(p.lineBytes.W) + val metadata = UInt(p.metadataBits.W) + // Eligibility constrains miss replacement candidates, not tag hits. + val eligible = UInt(p.ways.W) +} + +class CacheResponse(p: CacheParams) extends Bundle { + val id = UInt(p.idBits.W) + val hit = Bool() + val available = Bool() + val way = UInt(p.wayBits.W) + val entryValid = Bool() + val addr = UInt(p.addressBits.W) + val data = UInt(p.lineBits.W) + val metadata = UInt(p.metadataBits.W) +} + +class CacheIO(p: CacheParams) extends Bundle { + val request = Flipped(Decoupled(new CacheRequest(p))) + val response = Decoupled(new CacheResponse(p)) +} diff --git a/arch/src/main/scala/framework/ip/chi/build.sc b/arch/src/main/scala/framework/ip/chi/build.sc new file mode 100644 index 000000000..8847edb58 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/build.sc @@ -0,0 +1,37 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.bus.chi.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Channel.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/CoherenceState.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/DirectedSnoop.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Fabric.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Flit.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/HomeMapping.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/RequestRetry.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/RequesterPort.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Rx.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Tx.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/hnf/ChiHome.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/rnf/AxiCache.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/rnf/BankedChiCache.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/rnf/CachePort.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/rnf/ChiCache.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/snf/MemoryNode.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/snf/SramEndpoint.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/chi/configs/uvm.toml b/arch/src/main/scala/framework/ip/chi/configs/uvm.toml new file mode 100644 index 000000000..1ae6959d9 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/configs/uvm.toml @@ -0,0 +1,19 @@ +[parameters] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 +linkFlitBits = 32 +credits = 4 +nodeId = 1 +homeId = 64 +records = 4 +banks = 2 +cacheLines = 8 +homeCount = 1 +lineResult = false +probe = false +sramSlots = 8 +sramLines = 256 +sramRxDepth = 4 + +[defines] diff --git a/arch/src/main/scala/framework/mem-core/chi/src/csrc/Cargo.lock b/arch/src/main/scala/framework/ip/chi/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/chi/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/ip/chi/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/chi/src/csrc/Cargo.toml b/arch/src/main/scala/framework/ip/chi/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/chi/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/ip/chi/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/mem-core/chi/src/csrc/README.md b/arch/src/main/scala/framework/ip/chi/src/csrc/README.md similarity index 100% rename from arch/src/main/scala/framework/mem-core/chi/src/csrc/README.md rename to arch/src/main/scala/framework/ip/chi/src/csrc/README.md diff --git a/arch/src/main/scala/framework/ip/chi/src/csrc/src/lib.rs b/arch/src/main/scala/framework/ip/chi/src/csrc/src/lib.rs new file mode 100644 index 000000000..d6e5c6d4b --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/csrc/src/lib.rs @@ -0,0 +1,4 @@ +#[path = "../../../../../../../../../../verify/uvm/src/protocol/chi/rnf/model/src/reference.rs"] +mod rnf; +mod memory; +pub use memory::*; diff --git a/arch/src/main/scala/framework/mem-core/chi/src/csrc/src/memory.rs b/arch/src/main/scala/framework/ip/chi/src/csrc/src/memory.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/chi/src/csrc/src/memory.rs rename to arch/src/main/scala/framework/ip/chi/src/csrc/src/memory.rs diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/flit_types.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/common/flit_types.svh similarity index 78% rename from arch/src/main/scala/framework/mem-core/chi/src/main/resources/flit_types.svh rename to arch/src/main/scala/framework/ip/chi/src/main/resources/common/flit_types.svh index cf3bb675a..3b3ea0c21 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/flit_types.svh +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/common/flit_types.svh @@ -1,5 +1,5 @@ -`include "chi_profile.svh" -// Issue H, NodeID=7, address=44, data=256, all optional buses absent. +`include "profile.svh" +// Issue H base profile with optional buses absent. typedef struct packed { bit trace_tag; bit [1:0] tag_op; @@ -10,15 +10,15 @@ typedef struct packed { bit [1:0] order; bit allow_retry, likely_shared; bit [2:0] pas; - bit [43:0] addr; + bit [`CHI_ADDRESS_BITS-1:0] addr; bit [5:0] size; bit multi_req; bit [6:0] opcode; bit [11:0] return_txn; bit stash_valid; - bit [6:0] return_nid; + bit [`CHI_NODE_BITS-1:0] return_nid; bit [11:0] txn; - bit [6:0] src, tgt; + bit [`CHI_NODE_BITS-1:0] src, tgt; bit [3:0] qos; } req_t; typedef struct packed { @@ -31,7 +31,7 @@ typedef struct packed { bit [1:0] error; bit [4:0] opcode; bit [11:0] txn; - bit [6:0] src, tgt; + bit [`CHI_NODE_BITS-1:0] src, tgt; bit [3:0] qos; } rsp_t; typedef struct packed { @@ -52,8 +52,8 @@ typedef struct packed { bit [2:0] resp; bit [1:0] error; bit [3:0] opcode; - bit [6:0] home; + bit [`CHI_NODE_BITS-1:0] home; bit [11:0] txn; - bit [6:0] src, tgt; + bit [`CHI_NODE_BITS-1:0] src, tgt; bit [3:0] qos; } dat_t; diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/common/profile.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/common/profile.svh new file mode 100644 index 000000000..7e723f930 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/common/profile.svh @@ -0,0 +1,10 @@ +`ifndef CHI_PROFILE_SVH +`define CHI_PROFILE_SVH +`include "parameters.svh" +`define CHI_BE_BITS (`CHI_DATA_BITS / 8) +`define CHI_TAG_BITS (`CHI_DATA_BITS / 32) +`define CHI_TAG_UPDATE_BITS (`CHI_DATA_BITS / 128) +`define CHI_DAT_BITS (91 + `CHI_DATA_BITS + `CHI_BE_BITS + `CHI_TAG_BITS + `CHI_TAG_UPDATE_BITS) +`define CHI_BEATS (512 / `CHI_DATA_BITS) +`define CHI_DATA_ID_STEP (`CHI_DATA_BITS / 128) +`endif diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/agent.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/agent.svh new file mode 100644 index 000000000..d59bb88c0 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/agent.svh @@ -0,0 +1,28 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + monitor mon; + endpoint_driver endpoint; + uvm_sequencer #(packet_item) seqr[3]; + source_driver #(0) source0; + source_driver #(1) source1; + source_driver #(2) source2; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + mon = monitor::type_id::create("monitor", this); + endpoint = endpoint_driver::type_id::create("endpoint", this); + seqr[0] = new("seqr0", this); + source0 = source_driver#(0)::type_id::create("source0", this); + seqr[1] = new("seqr1", this); + source1 = source_driver#(1)::type_id::create("source1", this); + seqr[2] = new("seqr2", this); + source2 = source_driver#(2)::type_id::create("source2", this); + endfunction + function void connect_phase(uvm_phase phase); + source0.seq_item_port.connect(seqr[0].seq_item_export); + source1.seq_item_port.connect(seqr[1].seq_item_export); + source2.seq_item_port.connect(seqr[2].seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/driver.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/driver.svh new file mode 100644 index 000000000..d8b9a4483 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/driver.svh @@ -0,0 +1,44 @@ +class source_driver #( + int PORT = 0 +) extends uvm_driver #(packet_item); + `uvm_component_param_utils(source_driver#(PORT)) + virtual chi_retry_if vif; + protocol_model state; + packet_item pending; + bit driven = 0; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual chi_retry_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI source interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + if (pending == null) seq_item_port.try_next_item(pending); + driven = 0; + if (PORT == 0) begin + vif.req_in = pending != null ? pending.payload : state.idle.req_in; + vif.req_in_valid = pending != null; + driven = pending != null; + end else if (PORT == 1) begin + vif.rsp_in = pending != null ? pending.payload : state.idle.rsp_in; + vif.rsp_in_valid = pending != null; + driven = pending != null; + end else begin + vif.accepted_src = pending != null ? pending.payload[18:12] : 0; + vif.accepted_txn = pending != null ? pending.payload[11:0] : 0; + vif.accepted_valid = pending != null; + driven = pending != null; + end + @(posedge vif.clock); + if (!vif.reset && driven && (PORT == 2 || (PORT == 0 ? vif.req_in_ready : vif.rsp_in_ready))) begin + #1; + seq_item_port.item_done(); + pending = null; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/endpoint.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/endpoint.svh new file mode 100644 index 000000000..24019f9ff --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/endpoint.svh @@ -0,0 +1,20 @@ +class endpoint_driver extends uvm_component; + `uvm_component_utils(endpoint_driver) + virtual chi_retry_if vif; + protocol_model state; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual chi_retry_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI endpoint interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + vif.req_out_ready = !vif.reset && state.idle.req_out_ready; + vif.rsp_out_ready = !vif.reset && state.idle.rsp_out_ready; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/item.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/item.svh new file mode 100644 index 000000000..a1c674ee2 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/item.svh @@ -0,0 +1,37 @@ +class packet_item extends uvm_sequence_item; + `uvm_object_utils(packet_item) + logic [`CHI_DAT_BITS-1:0] payload; + bit sc_success, failed; + function new(string name = "packet_item"); + super.new(name); + endfunction +endclass +class sample_item extends uvm_sequence_item; + `uvm_object_utils(sample_item) + pins_t sampled; + function new(string name = "sample_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(error, UVM_ALL_ON) + `uvm_object_utils_end + logic [63:0] data; + logic error; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass +class packet_sequence extends uvm_sequence #(packet_item); + `uvm_object_utils(packet_sequence) + packet_item item; + function new(string name = "packet_sequence"); + super.new(name); + endfunction + task body(); + start_item(item); + finish_item(item); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/monitor.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/monitor.svh new file mode 100644 index 000000000..9723936b5 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/agent/monitor.svh @@ -0,0 +1,39 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual chi_retry_if vif; + uvm_analysis_port #(sample_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual chi_retry_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + sample_item t; + forever begin + @(posedge vif.clock); + t = sample_item::type_id::create("observed"); + t.sampled.reset = vif.reset; + t.sampled.req_in_valid = vif.req_in_valid; + t.sampled.req_in_ready = vif.req_in_ready; + t.sampled.req_out_valid = vif.req_out_valid; + t.sampled.req_out_ready = vif.req_out_ready; + t.sampled.req_in = vif.req_in; + t.sampled.req_out = vif.req_out; + t.sampled.rsp_in_valid = vif.rsp_in_valid; + t.sampled.rsp_in_ready = vif.rsp_in_ready; + t.sampled.rsp_out_valid = vif.rsp_out_valid; + t.sampled.rsp_out_ready = vif.rsp_out_ready; + t.sampled.rsp_in = vif.rsp_in; + t.sampled.rsp_out = vif.rsp_out; + t.sampled.accepted_valid = vif.accepted_valid; + t.sampled.accepted_src = vif.accepted_src; + t.sampled.accepted_txn = vif.accepted_txn; + t.sampled.pending = vif.pending; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/env/env.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/env/env.svh new file mode 100644 index 000000000..6c6bc7b0b --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/env/env.svh @@ -0,0 +1,28 @@ +class env extends uvm_env; + `uvm_component_utils(env) + agent bus; + protocol_model state; + reset_controller reset; + result_board #(result_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + state = protocol_model::type_id::create("model", this); + bus = agent::type_id::create("bus", this); + reset = reset_controller::type_id::create("reset", this); + endfunction + function void connect_phase(uvm_phase phase); + bus.mon.ap.connect(state.analysis_export); + bus.endpoint.state = state; + bus.source0.state = state; + bus.source1.state = state; + bus.source2.state = state; + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/env/model.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/env/model.svh new file mode 100644 index 000000000..d63819b33 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/env/model.svh @@ -0,0 +1,66 @@ +class protocol_model extends uvm_subscriber #(sample_item); + `uvm_component_utils(protocol_model) + req_t requests[$]; + rsp_t responses[$], original_responses[$]; + int checks = 0; + req_t originals[int]; + int retry_types[int]; + string scenario; + pins_t sampled; + pins_t idle; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + covergroup behavior with function sample (int count, bit stalled, bit reissued); + option.per_instance = 1; + live: coverpoint count { + bins empty = {0}; bins partial = {[1 : `CHI_RECORDS - 1]}; bins full = {`CHI_RECORDS}; + } + backpressure: coverpoint stalled; + retry_request: coverpoint reissued; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!$value$plusargs("CASE=%s", scenario)) `uvm_fatal("CASE", "CHI scenario is required") + idle = '0; + + endfunction + function void write(sample_item t); + pins_t s = t.sampled; + sampled = s; + begin + req_t out_req = req_t'(s.req_out); + behavior.sample(s.pending, s.req_out_valid && !s.req_out_ready, + s.req_out_valid && !out_req.allow_retry); + end + + if (s.reset) begin + requests.delete(); + responses.delete(); + original_responses.delete(); + end else begin + if (s.req_in_valid && s.req_in_ready) begin + req_t r = req_t'(s.req_in); + originals[r.txn] = r; + end + if (s.rsp_in_valid && s.rsp_in_ready) begin + rsp_t r = rsp_t'(s.rsp_in); + if (r.opcode == 3) retry_types[r.txn] = r.pcrd_type; + if (r.opcode != 3 && r.opcode != 7) original_responses.push_back(r); + end + if (s.req_out_valid && s.req_out_ready) requests.push_back(req_t'(s.req_out)); + if (s.rsp_out_valid && s.rsp_out_ready) responses.push_back(rsp_t'(s.rsp_out)); + end + + endfunction + function void check_phase(uvm_phase phase); + evidence::check(requests.size() == 0 && responses.size() == 0 && original_responses.size() == 0, + "CHI retry queues did not drain"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/filelist.f b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/filelist.f new file mode 100644 index 000000000..60c7127cf --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/filelist.f @@ -0,0 +1,10 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@RESOURCES@/../common ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/types.svh +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/RequestRetry/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/interface.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/interface.sv new file mode 100644 index 000000000..0bfe6b217 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/interface.sv @@ -0,0 +1,24 @@ +`include "parameters.svh" +interface chi_retry_if ( + input logic clock +); + logic reset; + logic req_in_valid, req_in_ready, req_out_valid, req_out_ready; + logic [`CHI_REQ_BITS-1:0] req_in, req_out; + logic rsp_in_valid, rsp_in_ready, rsp_out_valid, rsp_out_ready; + logic [`CHI_RSP_BITS-1:0] rsp_in, rsp_out; + logic accepted_valid; + logic [`CHI_NODE_BITS-1:0] accepted_src; + logic [11:0] accepted_txn; + logic [`CHI_PENDING_BITS-1:0] pending; + assert property (@(posedge clock) disable iff (reset) + req_out_valid && !req_out_ready |=> req_out_valid && $stable( + req_out + )) + else $fatal(1, "CHI channel changed under backpressure"); + assert property (@(posedge clock) disable iff (reset) + rsp_out_valid && !rsp_out_ready |=> rsp_out_valid && $stable( + rsp_out + )) + else $fatal(1, "CHI channel changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/package.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/package.sv new file mode 100644 index 000000000..8a95541c1 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/package.sv @@ -0,0 +1,16 @@ +package chi_retry_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import observed_types::*; + `include "uvm_macros.svh" + `include "flit_types.svh" + `include "agent/item.svh" + `include "env/model.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/endpoint.svh" + `include "agent/agent.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/seq/sequence.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/seq/sequence.svh new file mode 100644 index 000000000..afb95a3dc --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/seq/sequence.svh @@ -0,0 +1,430 @@ +class scenario_sequence extends uvm_sequence; + `uvm_object_utils(scenario_sequence) + env test_env; + protocol_model state; + virtual chi_retry_if vif; + string scenario; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task packet(int port, logic [`CHI_DAT_BITS-1:0] payload, bit sc_success = 0, bit failed = 0); + packet_sequence seq = packet_sequence::type_id::create("packet"); + seq.item = packet_item::type_id::create("item"); + seq.item.payload = payload; + seq.item.sc_success = sc_success; + seq.item.failed = failed; + `uvm_info("SEND", $sformatf("port=%0d payload=%h", port, payload), UVM_HIGH) + seq.start(test_env.bus.seqr[port]); + `uvm_info("ACCEPTED", $sformatf("port=%0d", port), UVM_HIGH) + endtask + task drive_request(req_t r); + packet(0, r); + endtask + + function bit coherent_read(int opcode); + return opcode == 1 || opcode == 'h26 || opcode == 7; + endfunction + function bit coherent_request(int opcode); + return coherent_read(opcode) || opcode == 'hd || opcode == 'h1b || opcode == 8 || opcode == 9 || + opcode == 10; + endfunction + task request(int txn, int opcode = 'hd); + req_t r = '0; + r.src = 1; + r.tgt = 64; + r.txn = txn; + r.opcode = opcode; + r.size = 6; + r.addr = 64 * txn; + r.allow_retry = 1; + if (coherent_request(opcode)) begin + r.mem_attr = 12; + r.snp_attr = 1; + end else begin + r.mem_attr = 4; + r.return_nid = 1; + r.return_txn = txn; + end + r.exp_comp_ack = coherent_read(opcode); + drive_request(r); + endtask + task response(int opcode, int txn, int src = 65, int credit_type = 3, bit contiguous = 0); + rsp_t r = '0; + r.opcode = opcode; + r.txn = txn; + r.src = src; + r.tgt = 1; + r.pcrd_type = (opcode == 3 || opcode == 7) ? credit_type : 0; + r.qos = (txn ^ src) % 16; + r.trace_tag = txn % 2; + if (opcode == 6 || opcode == 5) r.dbid = (txn ^ src ^ 'h555) & 'hfff; + if (opcode == 4 || opcode == 6 || opcode == 5) r.busy = (txn + src) % 8; + if (opcode == 4 && txn % 2) r.error = 3; // Dataless completion may report NDERR. + packet(1, r); + endtask + task expect_request(int txn, bit retry); + req_t r, golden; + `uvm_info("EXPECT_REQ", $sformatf("txn=%0d retry=%0d", txn, retry), UVM_HIGH) + while (!state.requests.size()) @(negedge vif.clock); + r = state.requests.pop_front(); + if (!state.originals.exists(txn)) `uvm_fatal("MODEL", "Missing observed original request") + golden = state.originals[txn]; + if (retry) begin + golden.allow_retry = 0; + golden.pcrd_type = state.retry_types[txn]; + end + if (r !== golden) + `uvm_fatal("RETRY", $sformatf( + "Complete request changed txn=%0h got=%h expected=%h", txn, r, golden)) + state.checks++; + evidence::checks++; + endtask + task receive_response(output rsp_t r); + rsp_t golden; + while (!state.responses.size()) @(negedge vif.clock); + if (!state.original_responses.size()) `uvm_fatal("MODEL", "Missing response/original") + r = state.responses.pop_front(); + golden = state.original_responses.pop_front(); + if (r !== golden) `uvm_fatal("RESPONSE", "Complete response metadata changed") + endtask + task accept_data(int txn, int source); + packet(2, {7'(source), 12'(txn)}); + endtask + + task body(); + rsp_t completed; + state.idle.req_out_ready = 1; + state.idle.rsp_out_ready = 1; + test_env.reset.pulse(4); + if (scenario == "duplicate_txnid") begin + request(12'hf01); + expect_request(12'hf01, 0); + request(12'hf01); + `uvm_fatal("NEGATIVE", "Duplicate live TxnID did not terminate RTL") + end else if (scenario == "bad_opcode") begin + request(0, 'h11); // Separate response/data acceptance is outside this profile. + `uvm_fatal("NEGATIVE", "Unsupported retry opcode did not terminate RTL") + end else begin + // A normal completion passes through and releases the request record. + request(0); + expect_request(0, 0); + state.idle.rsp_out_ready = 0; + fork + response(4, 0); + begin + repeat (12) @(negedge vif.clock); + state.idle.rsp_out_ready = 1; + end + join + receive_response(completed); + if (completed.opcode != 4) `uvm_fatal("PASS", "Completion was swallowed") + for (int i = 0; i < 4; i++) begin + request(i); + expect_request(i, 0); + end + for (int i = 0; i < 4; i++) response(3, i); + repeat (10) @(negedge vif.clock); + if (state.requests.size() || state.responses.size() || state.original_responses.size() || state.sampled.pending != 4) + `uvm_fatal("WAIT", "Retry issued before P-Credit or internal response leaked") + // Home remapped from TgtID 64 to RetryAck SrcID 65; match that credit source. + response(7, 0, 66, 3); + response(7, 0, 65, 2); + repeat (10) @(negedge vif.clock); + if (state.requests.size()) `uvm_fatal("MATCH", "Mismatched protocol credit consumed") + for (int i = 0; i < 4; i++) begin + response(7, 0); + expect_request(i, 1); + response(4, i); + receive_response(completed); + if (completed.txn != i) `uvm_fatal("COMP", "Bad completion"); + end + // An output stall must preserve the queued request and its record. + state.idle.req_out_ready = 0; + request(0); + repeat (12) @(negedge vif.clock); + if (!state.sampled.req_out_valid || state.requests.size()) + `uvm_fatal("STALL", "Queued request lost under stall") + state.idle.req_out_ready = 1; + expect_request(0, 0); + response(4, 0); + receive_response(completed); + if (completed.txn != 0) `uvm_fatal("COMP", "Bad stalled completion"); + // Grant can precede RetryAck and must be retained for its matching retry. + request(0, 4); + expect_request(0, 0); + response(7, 0); + response(3, 0); + expect_request(0, 1); + accept_data(0, 9); + repeat (4) @(negedge vif.clock); + if (state.sampled.pending != 0 || state.requests.size() || state.responses.size() || state.original_responses.size()) + `uvm_fatal("DRAIN", "Retry records did not drain") + // Reset cancels retry-wait records and retained protocol credits. + request(0); + expect_request(0, 0); + response(3, 0); + @(negedge vif.clock); + test_env.reset.pulse(4); + request(0); + expect_request(0, 0); + response(4, 0); + receive_response(completed); + if (completed.txn != 0) `uvm_fatal("RESET", "Completion after reset corrupted"); + repeat (4) @(negedge vif.clock); + if (state.sampled.pending != 0 || state.requests.size() || state.responses.size() || state.original_responses.size()) + `uvm_fatal("DRAIN", "Retry reset did not drain") + // Full protocol IDs are independent of the four local acceptance slots. + // Exercise both polarities of every aligned address, TxnID and routing-ID bit. + for (int phase_index = 0; phase_index < 3; phase_index++) begin + int polarity = phase_index % 2; + for (int address_bit = 6; address_bit < 44; address_bit++) begin + int ids[4], homes[4], types[4], opcodes[4]; + bit [3:0] seen_reissues = '0; + int supported_ops[11] = '{1, 'h26, 7, 'hd, 'h1b, 8, 9, 10, 4, 'h1d, 'h1c}; + for (int slot = 0; slot < 4; slot++) begin + req_t r = '0; + int node_bits = (1 << ((address_bit + slot) % 7)) | 2; + ids[slot] = (1 << ((address_bit + slot) % 12)) ^ (polarity ? 'hfff : 0); + homes[slot] = (address_bit % 3 == 0 ? 65 : node_bits ^ (polarity ? 'h7f : 0)); + types[slot] = (address_bit % 3 == 1 ? address_bit % 16 : (address_bit + slot) % 16); + r.src = 1; + r.tgt = homes[slot]; + r.txn = ids[slot]; + opcodes[slot] = supported_ops[(address_bit+slot+polarity)%11]; + r.opcode = opcodes[slot]; + r.size = 6; + r.lpid = (address_bit + slot) % 32; + r.likely_shared = (address_bit + slot) % 2; + if (coherent_request(opcodes[slot])) begin + r.mem_attr = 12; + r.snp_attr = 1; + end else begin + case ((address_bit + slot) % 3) + 0: r.mem_attr = 0; + 1: r.mem_attr = 4; + 2: r.mem_attr = 12; + endcase + r.mem_attr |= (address_bit + slot + polarity) % 2; // EWA is legal for Normal memory. + r.return_nid = homes[slot]; + r.return_txn = ids[slot] ^ 'hfff; + end + r.exp_comp_ack = coherent_read(opcodes[slot]); + r.addr = ((64'h1 << address_bit) ^ (polarity ? 64'hfffffffffc0 : 0)); + r.qos = (address_bit + 3 * slot) % 16; + r.trace_tag = polarity; + r.allow_retry = 1; + drive_request(r); + expect_request(ids[slot], 0); + end + if (state.sampled.pending != 4) + `uvm_fatal("SLOTS", "Arbitrary IDs did not fill four acceptance slots") + state.idle.rsp_out_ready = 0; // Internal retry traffic must bypass endpoint backpressure. + for (int position = 0; position < 4; position++) begin + int slot = (position + address_bit / 4) % 4; + response(7, 0, homes[slot], types[slot]); + end + if (address_bit % 5 == 0) state.idle.req_out_ready = 0; + for (int position = 0; position < 4; position++) begin + int slot = (address_bit + (address_bit % 2 ? 3 - position : position)) % 4; + response(3, ids[slot], homes[slot], types[slot], position != 0); + end + repeat (3) @(negedge vif.clock); + state.idle.req_out_ready = 1; + state.idle.rsp_out_ready = 1; + for (int position = 0; position < 4; position++) begin + int slot = -1; + while (!state.requests.size()) @(negedge vif.clock); + for (int candidate = 0; candidate < 4; candidate++) + if (state.requests[0].txn == ids[candidate]) slot = candidate; + if (slot < 0 || seen_reissues[slot]) + `uvm_fatal("RETRY_SET", "Unexpected or duplicate reissued transaction") + seen_reissues[slot] = 1; + expect_request(ids[slot], 1); + if (coherent_read(opcodes[slot]) || opcodes[slot] == 4) begin + accept_data(ids[slot], homes[slot]); + end else begin + int acceptance=(opcodes[slot]=='h1b || opcodes[slot]=='h1d || opcodes[slot]=='h1c) ? 6 : 4; + if (acceptance == 6 && (address_bit + slot) % 2) acceptance = 5; + response(acceptance, ids[slot], homes[slot], 0); + receive_response(completed); + if (completed.txn != ids[slot] || completed.opcode != acceptance) + `uvm_fatal("PASS", "Acceptance response corrupted") + end + end + repeat (3) @(negedge vif.clock); + if (state.sampled.pending != 0 || state.requests.size() || state.responses.size() || state.original_responses.size()) + `uvm_fatal("DRAIN", "Full-width request batch did not drain") + end + end + // Fill the two-entry output queue and hold a third request under backpressure. + state.idle.req_out_ready = 0; + fork + begin + for (int slot = 0; slot < 4; slot++) request(slot); + end + begin + repeat (20) @(negedge vif.clock); + state.idle.req_out_ready = 1; + end + join + for (int slot = 0; slot < 4; slot++) expect_request(slot, 0); + for (int slot = 0; slot < 4; slot++) begin + response(4, slot); + receive_response(completed); + end + // Initial allocation overlaps independent RetryAck and completion handshakes. + for (int context_kind = 0; context_kind < 2; context_kind++) begin + for (int slot = 0; slot < 4; slot++) begin + int other = (slot + 1) % 4; + for (int i = 0; i < 4; i++) begin + request(100 + i); + expect_request(100 + i, 0); + end + response(4, 100 + slot); + receive_response(completed); + fork + request(200 + slot); + response(context_kind ? 4 : 3, 100 + other); + join + expect_request(200 + slot, 0); + if (!context_kind) begin + response(7, 0); + expect_request(100 + other, 1); + end else begin + receive_response(completed); + end + for (int i = 0; i < 4; i++) begin + if (context_kind && i == other) continue; + response(4, i == slot ? 200 + slot : 100 + i); + receive_response(completed); + end + end + end + // A new P-Credit may arrive in the cycle a different credit is consumed. + for (int slot = 0; slot < 4; slot++) begin + int next_slot = (slot + 1) % 4; + for (int i = 0; i < 4; i++) begin + request(300 + i); + expect_request(300 + i, 0); + end + response(7, 0); + response(3, 300 + slot); + response(7, 0, 65, 3, 1); + expect_request(300 + slot, 1); + response(3, 300 + next_slot); + expect_request(300 + next_slot, 1); + for (int i = 0; i < 4; i++) begin + response(4, 300 + i); + receive_response(completed); + end + end + // Later split completion/data acceptance may follow record release. + accept_data(3, 65); + repeat (3) @(negedge vif.clock); + if (state.sampled.pending || state.requests.size() || state.responses.size() || state.original_responses.size()) + `uvm_fatal("DRAIN", "Saturated output queue did not drain") + // Acceptance tracking can retire before later data beats or split completion. + request(500, 4); + expect_request(500, 0); + accept_data(500, 9); + request(501, 'h1d); + expect_request(501, 0); + response(6, 501); + receive_response(completed); + for (int i = 0; i < 4; i++) begin + request(600 + i); + expect_request(600 + i, 0); + end + for (int i = 0; i < 4; i++) response(3, 600 + i); + for (int i = 0; i < 4; i++) begin + rsp_t idle = '0; + idle.opcode = 4; + idle.tgt = 1; + idle.txn = 600 + i; + @(negedge vif.clock); + state.idle.rsp_in = idle; + state.idle.accepted_txn = 600 + i; + @(negedge vif.clock); + if (state.sampled.pending != 4 || state.requests.size() || state.responses.size()) + `uvm_fatal("INVALID_CHANNEL", "Inactive matching payload changed a retry-wait record") + end + response(4, 501); + receive_response(completed); + accept_data(500, 9); + if (state.sampled.pending != 4) + `uvm_fatal("LATE_ACCEPT", "Late acceptance retired an unrelated retry") + for (int i = 0; i < 4; i++) begin + response(7, 0); + expect_request(600 + i, 1); + response(4, 600 + i); + receive_response(completed); + end + // VALID held through coordinated reset is accepted once after release. + begin + req_t held = '0; + held.src = 1; + held.tgt = 64; + held.txn = 'ha55; + held.opcode = 'hd; + held.size = 6; + held.addr = 64; + held.allow_retry = 1; + held.mem_attr = 12; + held.snp_attr = 1; + @(negedge vif.clock); + fork + test_env.reset.pulse(4); + begin + wait (vif.reset); + drive_request(held); + end + join + expect_request(held.txn, 0); + response(4, held.txn); + receive_response(completed); + repeat (3) @(negedge vif.clock); + if (state.sampled.pending || state.requests.size() || state.responses.size() || state.original_responses.size()) + `uvm_fatal("RESET", "Held VALID did not complete exactly once") + end + // Invalid channels may carry arbitrary payload; no transaction may be consumed. + begin + req_t idle_req = '0; + rsp_t idle_rsp = '0; + idle_req.src = 1; + idle_req.opcode = 'hd; + idle_req.size = 6; + idle_req.mem_attr = 12; + idle_req.snp_attr = 1; + idle_req.allow_retry = 1; + idle_rsp.tgt = 1; + idle_rsp.opcode = 4; + for (int b = 0; b < $bits(req_t); b++) begin + @(negedge vif.clock); + state.idle.req_in = idle_req ^ (137'b1 << b); + @(negedge vif.clock); + state.idle.req_in = idle_req; + if (state.sampled.pending || state.sampled.req_out_valid || state.sampled.rsp_out_valid) + `uvm_fatal("INVALID_CHANNEL", "Invalid REQ payload changed live transactions") + end + @(negedge vif.clock); + idle_req.snp_attr = 0; + idle_req.mem_attr = 2; + state.idle.req_in = idle_req; + @(negedge vif.clock); + if (state.sampled.pending || state.sampled.req_out_valid) + `uvm_fatal("INVALID_CHANNEL", "Inactive Device payload was consumed") + for (int b = 0; b < $bits(rsp_t); b++) begin + @(negedge vif.clock); + state.idle.rsp_in = idle_rsp ^ (71'b1 << b); + @(negedge vif.clock); + state.idle.rsp_in = idle_rsp; + if (state.sampled.pending || state.sampled.req_out_valid || state.sampled.rsp_out_valid) + `uvm_fatal("INVALID_CHANNEL", "Invalid RSP payload changed live transactions") + end + end + repeat (2) @(negedge vif.clock); + `uvm_info("CHECKS", $sformatf("CHI retry: %0d request/reissue fields checked", state.checks), + UVM_LOW) + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/tb.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/tb.sv new file mode 100644 index 000000000..150c47833 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/tb.sv @@ -0,0 +1,104 @@ +module retry_tb; + import uvm_pkg::*; + import chi_retry_pkg::*; + logic clock = 0; + clock_if timing (clock); + always #5 clock = ~clock; + chi_retry_if source_if (clock); + RequestRetry dut ( + .clock(clock), + .reset(source_if.reset), + .io_reqIn_valid(source_if.req_in_valid), + .io_reqIn_ready(source_if.req_in_ready), + .io_reqIn_bits_qos(source_if.req_in[3:0]), + .io_reqIn_bits_tgtId(source_if.req_in[10:4]), + .io_reqIn_bits_srcId(source_if.req_in[17:11]), + .io_reqIn_bits_txnId(source_if.req_in[29:18]), + .io_reqIn_bits_returnNid(source_if.req_in[36:30]), + .io_reqIn_bits_stashNidValidEndian(source_if.req_in[37:37]), + .io_reqIn_bits_returnTxnId(source_if.req_in[49:38]), + .io_reqIn_bits_opcode(source_if.req_in[56:50]), + .io_reqIn_bits_multiReq(source_if.req_in[57:57]), + .io_reqIn_bits_size(source_if.req_in[63:58]), + .io_reqIn_bits_addr(source_if.req_in[107:64]), + .io_reqIn_bits_pas(source_if.req_in[110:108]), + .io_reqIn_bits_likelyShared(source_if.req_in[111:111]), + .io_reqIn_bits_allowRetry(source_if.req_in[112:112]), + .io_reqIn_bits_order(source_if.req_in[114:113]), + .io_reqIn_bits_pCrdType(source_if.req_in[118:115]), + .io_reqIn_bits_memAttr(source_if.req_in[122:119]), + .io_reqIn_bits_snpAttr(source_if.req_in[123:123]), + .io_reqIn_bits_lpid(source_if.req_in[131:124]), + .io_reqIn_bits_exclSnoopMe(source_if.req_in[132:132]), + .io_reqIn_bits_expCompAck(source_if.req_in[133:133]), + .io_reqIn_bits_tagOp(source_if.req_in[135:134]), + .io_reqIn_bits_traceTag(source_if.req_in[136:136]), + .io_reqOut_valid(source_if.req_out_valid), + .io_reqOut_ready(source_if.req_out_ready), + .io_reqOut_bits_qos(source_if.req_out[3:0]), + .io_reqOut_bits_tgtId(source_if.req_out[10:4]), + .io_reqOut_bits_srcId(source_if.req_out[17:11]), + .io_reqOut_bits_txnId(source_if.req_out[29:18]), + .io_reqOut_bits_returnNid(source_if.req_out[36:30]), + .io_reqOut_bits_stashNidValidEndian(source_if.req_out[37:37]), + .io_reqOut_bits_returnTxnId(source_if.req_out[49:38]), + .io_reqOut_bits_opcode(source_if.req_out[56:50]), + .io_reqOut_bits_multiReq(source_if.req_out[57:57]), + .io_reqOut_bits_size(source_if.req_out[63:58]), + .io_reqOut_bits_addr(source_if.req_out[107:64]), + .io_reqOut_bits_pas(source_if.req_out[110:108]), + .io_reqOut_bits_likelyShared(source_if.req_out[111:111]), + .io_reqOut_bits_allowRetry(source_if.req_out[112:112]), + .io_reqOut_bits_order(source_if.req_out[114:113]), + .io_reqOut_bits_pCrdType(source_if.req_out[118:115]), + .io_reqOut_bits_memAttr(source_if.req_out[122:119]), + .io_reqOut_bits_snpAttr(source_if.req_out[123:123]), + .io_reqOut_bits_lpid(source_if.req_out[131:124]), + .io_reqOut_bits_exclSnoopMe(source_if.req_out[132:132]), + .io_reqOut_bits_expCompAck(source_if.req_out[133:133]), + .io_reqOut_bits_tagOp(source_if.req_out[135:134]), + .io_reqOut_bits_traceTag(source_if.req_out[136:136]), + .io_rspIn_valid(source_if.rsp_in_valid), + .io_rspIn_ready(source_if.rsp_in_ready), + .io_rspIn_bits_qos(source_if.rsp_in[3:0]), + .io_rspIn_bits_tgtId(source_if.rsp_in[10:4]), + .io_rspIn_bits_srcId(source_if.rsp_in[17:11]), + .io_rspIn_bits_txnId(source_if.rsp_in[29:18]), + .io_rspIn_bits_opcode(source_if.rsp_in[34:30]), + .io_rspIn_bits_respErr(source_if.rsp_in[36:35]), + .io_rspIn_bits_resp(source_if.rsp_in[39:37]), + .io_rspIn_bits_fwdState(source_if.rsp_in[42:40]), + .io_rspIn_bits_cBusy(source_if.rsp_in[45:43]), + .io_rspIn_bits_dbid(source_if.rsp_in[57:46]), + .io_rspIn_bits_pCrdType(source_if.rsp_in[61:58]), + .io_rspIn_bits_tagOp(source_if.rsp_in[63:62]), + .io_rspIn_bits_traceTag(source_if.rsp_in[64:64]), + .io_rspIn_bits_cacheLineId(source_if.rsp_in[70:65]), + .io_rspOut_valid(source_if.rsp_out_valid), + .io_rspOut_ready(source_if.rsp_out_ready), + .io_rspOut_bits_qos(source_if.rsp_out[3:0]), + .io_rspOut_bits_tgtId(source_if.rsp_out[10:4]), + .io_rspOut_bits_srcId(source_if.rsp_out[17:11]), + .io_rspOut_bits_txnId(source_if.rsp_out[29:18]), + .io_rspOut_bits_opcode(source_if.rsp_out[34:30]), + .io_rspOut_bits_respErr(source_if.rsp_out[36:35]), + .io_rspOut_bits_resp(source_if.rsp_out[39:37]), + .io_rspOut_bits_fwdState(source_if.rsp_out[42:40]), + .io_rspOut_bits_cBusy(source_if.rsp_out[45:43]), + .io_rspOut_bits_dbid(source_if.rsp_out[57:46]), + .io_rspOut_bits_pCrdType(source_if.rsp_out[61:58]), + .io_rspOut_bits_tagOp(source_if.rsp_out[63:62]), + .io_rspOut_bits_traceTag(source_if.rsp_out[64:64]), + .io_rspOut_bits_cacheLineId(source_if.rsp_out[70:65]), + .io_acceptedData_valid(source_if.accepted_valid), + .io_acceptedData_bits_srcId(source_if.accepted_src), + .io_acceptedData_bits_txnId(source_if.accepted_txn), + .io_pending(source_if.pending) + ); + assign source_if.reset = timing.reset; + initial begin + uvm_config_db#(virtual chi_retry_if)::set(null, "uvm_test_top*", "vif", source_if); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top.env.reset", "timing", timing); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/tests/test.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/tests/test.svh new file mode 100644 index 000000000..45c2ab054 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/tests/test.svh @@ -0,0 +1,26 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + virtual chi_retry_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + if (`CHI_NODE_BITS != 7 || `CHI_ADDRESS_BITS != 44 || `CHI_NODE_ID != 1) + `uvm_fatal("PROFILE", + "CHI packet scenarios require 7-bit nodes, 44-bit addresses and requester 1") + if (`CHI_RECORDS != 4) `uvm_fatal("PROFILE", "Retry saturation scenarios require four records") + super.build_phase(phase); + test_env = env::type_id::create("env", this); + if (!uvm_config_db#(virtual chi_retry_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI timing interface is required") + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.state = test_env.state; + seq.vif = vif; + seq.scenario = test_env.state.scenario; + seq.start(null); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/types.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/types.svh new file mode 100644 index 000000000..0156fee1f --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/retry/types.svh @@ -0,0 +1,14 @@ +`include "profile.svh" +package observed_types; + typedef struct packed { + logic reset; + logic req_in_valid, req_in_ready, req_out_valid, req_out_ready; + logic [`CHI_REQ_BITS-1:0] req_in, req_out; + logic rsp_in_valid, rsp_in_ready, rsp_out_valid, rsp_out_ready; + logic [`CHI_RSP_BITS-1:0] rsp_in, rsp_out; + logic accepted_valid; + logic [`CHI_NODE_BITS-1:0] accepted_src; + logic [11:0] accepted_txn; + logic [`CHI_PENDING_BITS-1:0] pending; + } pins_t; +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/filelist.f b/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/filelist.f new file mode 100644 index 000000000..47528a5bc --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/filelist.f @@ -0,0 +1,11 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@RESOURCES@/../common ++incdir+@VERIFY@/uvm/src/protocol/chi ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/chi/rnf/types.svh +@VERIFY@/uvm/src/protocol/chi/rnf/interface.sv +@RESOURCES@/package.sv +-F @RTL@/BankedChiCache/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/package.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/package.sv new file mode 100644 index 000000000..8d63f8d23 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/package.sv @@ -0,0 +1,3 @@ +package chi_rnf_pkg; + `include "rnf/body.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/tb.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/tb.sv new file mode 100644 index 000000000..aeb1bee78 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rnf/tb.sv @@ -0,0 +1,155 @@ +module rnf_tb; + import uvm_pkg::*; + import chi_rnf_pkg::*; + logic clock = 0; + clock_if timing (clock); + always #5 clock = ~clock; + chi_rnf_if vif (clock); + BankedChiCache dut ( + .clock(clock), + .reset(vif.reset), + .io_chi_req_bits_qos(vif.req[0+:4]), + .io_chi_req_bits_tgtId(vif.req[4+:7]), + .io_chi_req_bits_srcId(vif.req[11+:7]), + .io_chi_req_bits_txnId(vif.req[18+:12]), + .io_chi_req_bits_returnNid(vif.req[30+:7]), + .io_chi_req_bits_stashNidValidEndian(vif.req[37+:1]), + .io_chi_req_bits_returnTxnId(vif.req[38+:12]), + .io_chi_req_bits_opcode(vif.req[50+:7]), + .io_chi_req_bits_multiReq(vif.req[57+:1]), + .io_chi_req_bits_size(vif.req[58+:6]), + .io_chi_req_bits_addr(vif.req[64+:44]), + .io_chi_req_bits_pas(vif.req[108+:3]), + .io_chi_req_bits_likelyShared(vif.req[111+:1]), + .io_chi_req_bits_allowRetry(vif.req[112+:1]), + .io_chi_req_bits_order(vif.req[113+:2]), + .io_chi_req_bits_pCrdType(vif.req[115+:4]), + .io_chi_req_bits_memAttr(vif.req[119+:4]), + .io_chi_req_bits_snpAttr(vif.req[123+:1]), + .io_chi_req_bits_lpid(vif.req[124+:8]), + .io_chi_req_bits_exclSnoopMe(vif.req[132+:1]), + .io_chi_req_bits_expCompAck(vif.req[133+:1]), + .io_chi_req_bits_tagOp(vif.req[134+:2]), + .io_chi_req_bits_traceTag(vif.req[136+:1]), + .io_chi_req_valid(vif.req_valid), + .io_chi_req_ready(vif.req_ready), + .io_chi_txRsp_bits_qos(vif.rsp[0+:4]), + .io_chi_txRsp_bits_tgtId(vif.rsp[4+:7]), + .io_chi_txRsp_bits_srcId(vif.rsp[11+:7]), + .io_chi_txRsp_bits_txnId(vif.rsp[18+:12]), + .io_chi_txRsp_bits_opcode(vif.rsp[30+:5]), + .io_chi_txRsp_bits_respErr(vif.rsp[35+:2]), + .io_chi_txRsp_bits_resp(vif.rsp[37+:3]), + .io_chi_txRsp_bits_fwdState(vif.rsp[40+:3]), + .io_chi_txRsp_bits_cBusy(vif.rsp[43+:3]), + .io_chi_txRsp_bits_dbid(vif.rsp[46+:12]), + .io_chi_txRsp_bits_pCrdType(vif.rsp[58+:4]), + .io_chi_txRsp_bits_tagOp(vif.rsp[62+:2]), + .io_chi_txRsp_bits_traceTag(vif.rsp[64+:1]), + .io_chi_txRsp_bits_cacheLineId(vif.rsp[65+:6]), + .io_chi_txRsp_valid(vif.rsp_valid), + .io_chi_txRsp_ready(vif.rsp_ready), + .io_chi_txDat_bits_qos(vif.dat[0+:4]), + .io_chi_txDat_bits_tgtId(vif.dat[4+:7]), + .io_chi_txDat_bits_srcId(vif.dat[11+:7]), + .io_chi_txDat_bits_txnId(vif.dat[18+:12]), + .io_chi_txDat_bits_homeNid(vif.dat[30+:7]), + .io_chi_txDat_bits_opcode(vif.dat[37+:4]), + .io_chi_txDat_bits_respErr(vif.dat[41+:2]), + .io_chi_txDat_bits_resp(vif.dat[43+:3]), + .io_chi_txDat_bits_dataSource(vif.dat[46+:8]), + .io_chi_txDat_bits_dataPull(vif.dat[54+:1]), + .io_chi_txDat_bits_cBusy(vif.dat[55+:3]), + .io_chi_txDat_bits_dbid(vif.dat[58+:16]), + .io_chi_txDat_bits_ccid(vif.dat[74+:2]), + .io_chi_txDat_bits_dataId(vif.dat[76+:2]), + .io_chi_txDat_bits_cacheLineId(vif.dat[78+:6]), + .io_chi_txDat_bits_tagOp(vif.dat[84+:2]), + .io_chi_txDat_bits_tag(vif.dat[86+:8]), + .io_chi_txDat_bits_tagUpdate(vif.dat[94+:2]), + .io_chi_txDat_bits_traceTag(vif.dat[96+:1]), + .io_chi_txDat_bits_copyAtHome(vif.dat[97+:1]), + .io_chi_txDat_bits_numDat(vif.dat[98+:2]), + .io_chi_txDat_bits_replicate(vif.dat[100+:1]), + .io_chi_txDat_bits_be(vif.dat[101+:32]), + .io_chi_txDat_bits_data(vif.dat[133+:256]), + .io_chi_txDat_valid(vif.dat_valid), + .io_chi_txDat_ready(vif.dat_ready), + .io_chi_rxRsp_bits_qos(vif.rx_rsp[0+:4]), + .io_chi_rxRsp_bits_tgtId(vif.rx_rsp[4+:7]), + .io_chi_rxRsp_bits_srcId(vif.rx_rsp[11+:7]), + .io_chi_rxRsp_bits_txnId(vif.rx_rsp[18+:12]), + .io_chi_rxRsp_bits_opcode(vif.rx_rsp[30+:5]), + .io_chi_rxRsp_bits_respErr(vif.rx_rsp[35+:2]), + .io_chi_rxRsp_bits_resp(vif.rx_rsp[37+:3]), + .io_chi_rxRsp_bits_fwdState(vif.rx_rsp[40+:3]), + .io_chi_rxRsp_bits_cBusy(vif.rx_rsp[43+:3]), + .io_chi_rxRsp_bits_dbid(vif.rx_rsp[46+:12]), + .io_chi_rxRsp_bits_pCrdType(vif.rx_rsp[58+:4]), + .io_chi_rxRsp_bits_tagOp(vif.rx_rsp[62+:2]), + .io_chi_rxRsp_bits_traceTag(vif.rx_rsp[64+:1]), + .io_chi_rxRsp_bits_cacheLineId(vif.rx_rsp[65+:6]), + .io_chi_rxRsp_valid(vif.rx_rsp_valid), + .io_chi_rxRsp_ready(vif.rx_rsp_ready), + .io_chi_rxDat_bits_qos(vif.rx_dat[0+:4]), + .io_chi_rxDat_bits_tgtId(vif.rx_dat[4+:7]), + .io_chi_rxDat_bits_srcId(vif.rx_dat[11+:7]), + .io_chi_rxDat_bits_txnId(vif.rx_dat[18+:12]), + .io_chi_rxDat_bits_homeNid(vif.rx_dat[30+:7]), + .io_chi_rxDat_bits_opcode(vif.rx_dat[37+:4]), + .io_chi_rxDat_bits_respErr(vif.rx_dat[41+:2]), + .io_chi_rxDat_bits_resp(vif.rx_dat[43+:3]), + .io_chi_rxDat_bits_dataSource(vif.rx_dat[46+:8]), + .io_chi_rxDat_bits_dataPull(vif.rx_dat[54+:1]), + .io_chi_rxDat_bits_cBusy(vif.rx_dat[55+:3]), + .io_chi_rxDat_bits_dbid(vif.rx_dat[58+:16]), + .io_chi_rxDat_bits_ccid(vif.rx_dat[74+:2]), + .io_chi_rxDat_bits_dataId(vif.rx_dat[76+:2]), + .io_chi_rxDat_bits_cacheLineId(vif.rx_dat[78+:6]), + .io_chi_rxDat_bits_tagOp(vif.rx_dat[84+:2]), + .io_chi_rxDat_bits_tag(vif.rx_dat[86+:8]), + .io_chi_rxDat_bits_tagUpdate(vif.rx_dat[94+:2]), + .io_chi_rxDat_bits_traceTag(vif.rx_dat[96+:1]), + .io_chi_rxDat_bits_copyAtHome(vif.rx_dat[97+:1]), + .io_chi_rxDat_bits_numDat(vif.rx_dat[98+:2]), + .io_chi_rxDat_bits_replicate(vif.rx_dat[100+:1]), + .io_chi_rxDat_bits_be(vif.rx_dat[101+:32]), + .io_chi_rxDat_bits_data(vif.rx_dat[133+:256]), + .io_chi_rxDat_valid(vif.rx_dat_valid), + .io_chi_rxDat_ready(vif.rx_dat_ready), + .io_chi_snp_bits_qos(vif.snp[0+:4]), + .io_chi_snp_bits_srcId(vif.snp[4+:7]), + .io_chi_snp_bits_txnId(vif.snp[11+:12]), + .io_chi_snp_bits_fwdNid(vif.snp[23+:7]), + .io_chi_snp_bits_fwdTxnId(vif.snp[30+:12]), + .io_chi_snp_bits_opcode(vif.snp[42+:5]), + .io_chi_snp_bits_addr(vif.snp[47+:41]), + .io_chi_snp_bits_pas(vif.snp[88+:3]), + .io_chi_snp_bits_doNotGoToSd(vif.snp[91+:1]), + .io_chi_snp_bits_retToSrc(vif.snp[92+:1]), + .io_chi_snp_bits_traceTag(vif.snp[93+:1]), + .io_chi_snp_valid(vif.snp_valid), + .io_chi_snp_ready(vif.snp_ready), + .io_access_valid(vif.access_valid), + .io_access_ready(vif.access_ready), + .io_access_bits_write(vif.access_write), + .io_access_bits_addr(vif.access_addr), + .io_access_bits_data(vif.access_data), + .io_access_bits_mask(vif.access_mask), + .io_access_bits_atomic(vif.access_atomic), + .io_access_bits_atomicWord(vif.access_atomic_word), + .io_result_valid(vif.result_valid), + .io_result_ready(vif.result_ready), + .io_result_bits_error(vif.result_error), + .io_result_bits_data(vif.result_data), + .io_outstanding(vif.outstanding), + .io_hits(), + .io_misses() + ); + assign vif.reset = timing.reset; + initial begin + uvm_config_db#(virtual chi_rnf_if)::set(null, "uvm_test_top*", "vif", vif); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top.env.reset", "timing", timing); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/agent.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/agent.svh new file mode 100644 index 000000000..6c18e56b9 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/agent.svh @@ -0,0 +1,18 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + driver drv; + monitor mon; + uvm_sequencer #(cycle_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/driver.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/driver.svh new file mode 100644 index 000000000..52f749096 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/driver.svh @@ -0,0 +1,29 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual link_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual link_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI link interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.pins.active = req.pins.active; + vif.pins.input_valid = req.pins.input_valid; + vif.pins.input_data = req.pins.input_data; + vif.pins.pend = req.pins.pend; + vif.pins.output_ready = req.pins.output_ready; + @(posedge vif.clock); + req.sampled = vif.pins; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/item.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/item.svh new file mode 100644 index 000000000..0147091c1 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/item.svh @@ -0,0 +1,17 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + link_signals pins, sampled; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class flit_item extends uvm_sequence_item; + `uvm_object_utils_begin(flit_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_object_utils_end + logic [WIDTH-1:0] data; + function new(string name = "flit_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/monitor.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/monitor.svh new file mode 100644 index 000000000..5ace5103b --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/agent/monitor.svh @@ -0,0 +1,24 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual link_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual link_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI link monitor requires an interface") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.reset = vif.reset; + t.sampled = vif.pins; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/env/env.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/env/env.svh new file mode 100644 index 000000000..8f7141d8c --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/env/env.svh @@ -0,0 +1,26 @@ +class env extends uvm_env; + `uvm_component_utils(env) + agent bus; + model reference; + result_board #(flit_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + bus = agent::type_id::create("bus", this); + reference = model::type_id::create("model", this); + scoreboard = result_board#(flit_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + bus.mon.ap.connect(reference.analysis_export); + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + reference.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/env/model.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/env/model.svh new file mode 100644 index 000000000..e572b250a --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/env/model.svh @@ -0,0 +1,50 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(flit_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + int granted = 0, occupied = 0; + covergroup behavior with function sample (int count, bit stalled, bit simultaneous); + option.per_instance = 1; + occupancy: coverpoint count { + bins empty = {0}; bins partial = {[1 : DEPTH - 1]}; bins full = {DEPTH}; + } + sink_stall: coverpoint stalled; + enqueue_and_dequeue: coverpoint simultaneous; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + behavior = new(); + endfunction + function void write(cycle_item t); + link_signals s = t.sampled; + flit_item item; + bit retire = s.output_valid && s.output_ready; + if (t.reset) begin + reset_ap.write(1); + granted = 0; + occupied = 0; + return; + end + evidence::check(s.output_valid === (occupied != 0), + "CHI RX valid disagrees with buffered flits"); + if (s.input_valid) evidence::check(granted > 0, "CHI RX consumed an ungranted credit"); + behavior.sample(occupied, s.output_valid && !s.output_ready, s.input_valid && retire); + if (s.input_valid) begin + item = flit_item::type_id::create("expected"); + item.data = s.input_data; + expected_ap.write(item); + end + if (retire) begin + item = flit_item::type_id::create("actual"); + item.data = s.output_data; + actual_ap.write(item); + end + granted += int'(s.credit) - int'(s.input_valid); + occupied += int'(s.input_valid) - int'(retire); + evidence::check(granted >= 0 && occupied >= 0 && granted + occupied <= DEPTH, + "CHI RX credit/storage conservation failed"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/filelist.f b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/filelist.f new file mode 100644 index 000000000..e545b857d --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/types.svh +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Rx/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/interface.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/interface.sv new file mode 100644 index 000000000..1d93de013 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/interface.sv @@ -0,0 +1,12 @@ +interface link_if ( + input logic clock +); + import link_types::*; + logic reset = 1; + link_signals pins; + assert property (@(posedge clock) disable iff (reset) + pins.output_valid && !pins.output_ready |=> pins.output_valid && $stable( + pins.output_data + )) + else $fatal(1, "CHI RX output changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/package.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/package.sv new file mode 100644 index 000000000..824e2712f --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/package.sv @@ -0,0 +1,14 @@ +package link_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import link_types::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/seq/sequence.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/seq/sequence.svh new file mode 100644 index 000000000..0442df222 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/seq/sequence.svh @@ -0,0 +1,89 @@ +class traffic extends uvm_sequence #(cycle_item); + `uvm_object_utils(traffic) + env test_env; + link_signals pins = '0, observed; + bit reset = 1, vary_ready = 0; + int budget = 0, cycle = 0, sent = 0; + function new(string name = "traffic"); + super.new(name); + endfunction + task tick(int cycles = 1); + cycle_item req; + repeat (cycles) begin + if (vary_ready) pins.output_ready = cycle % 7 < 3; + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.pins = pins; + req.reset = reset; + finish_item(req); + observed = req.sampled; + if (!reset) budget += int'(observed.credit) - int'(pins.input_valid); + cycle++; + end + endtask + task restart(); + pins = '0; + reset = 1; + budget = 0; + vary_ready = 0; + tick(4); + reset = 0; + pins.active = 1; + pins.pend = 1; + endtask + task send(logic [WIDTH-1:0] data); + while (budget == 0) tick(); + pins.input_valid = 1; + pins.input_data = data; + tick(); + pins.input_valid = 0; + sent++; + endtask + task drain(); + while (test_env.scoreboard.checked < sent) tick(); + endtask + task body(); + restart(); + for (int n = 0; n < DEPTH; n++) send(WIDTH'(n)); + tick(8); + evidence::check(budget == 0 && observed.output_valid, "CHI RX did not retain a full buffer"); + pins.output_ready = 1; + drain(); + vary_ready = 1; + for (int n = 0; n < 256; n++) send(WIDTH'(32'h9e3779b9 * (n + 1)) ^ ((n % 2) ? '1 : '0)); + pins.output_ready = 1; + vary_ready = 0; + drain(); + tick(DEPTH + 2); + pins.output_ready = 0; + for (int n = 0; n < DEPTH; n++) send('1); + tick(3); + restart(); + sent -= DEPTH; + pins.output_ready = 1; + for (int n = 0; n < WIDTH; n++) send(WIDTH'(1) << n); + drain(); + tick(2); + evidence::check(evidence::cancelled == DEPTH, "CHI RX reset cancellation count mismatch"); + endtask +endclass +class invalid_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(invalid_sequence) + function new(string name = "invalid_sequence"); + super.new(name); + endfunction + task body(); + cycle_item req; + for (int n = 0; n < 2 * DEPTH + 8; n++) begin + req = cycle_item::type_id::create("overflow"); + start_item(req); + req.pins = '0; + req.reset = n < 4; + req.pins.active = n >= 4; + req.pins.pend = n >= 4; + req.pins.input_valid = n >= DEPTH + 6; + finish_item(req); + end + `uvm_fatal("EXPECTED_ASSERTION", "CHI RX accepted an ungranted flit") + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/tb.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/tb.sv new file mode 100644 index 000000000..6d87af894 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/tb.sv @@ -0,0 +1,23 @@ +module rx_tb; + import uvm_pkg::*; + import link_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + link_if bus (clock); + Rx dut ( + .clock(clock), + .reset(bus.reset), + .io_active(bus.pins.active), + .io_link_flitpend(bus.pins.pend), + .io_link_flitv(bus.pins.input_valid), + .io_link_flit(bus.pins.input_data), + .io_link_lcrdv(bus.pins.credit), + .io_out_valid(bus.pins.output_valid), + .io_out_ready(bus.pins.output_ready), + .io_out_bits(bus.pins.output_data) + ); + initial begin + uvm_config_db#(virtual link_if)::set(null, "uvm_test_top*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/tests/test.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/tests/test.svh new file mode 100644 index 000000000..38c98c87f --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/tests/test.svh @@ -0,0 +1,31 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + traffic seq = traffic::type_id::create("traffic"); + seq.test_env = test_env; + seq.start(test_env.bus.seqr); + endtask +endclass +class invalid_test extends base_test; + `uvm_component_utils(invalid_test) + agent bus; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + bus = agent::type_id::create("bus", this); + endfunction + task execute(); + invalid_sequence seq = invalid_sequence::type_id::create("violation"); + seq.start(bus.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/types.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/types.svh new file mode 100644 index 000000000..0f5428c6d --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/rx/types.svh @@ -0,0 +1,9 @@ +`include "parameters.svh" +package link_types; + localparam int WIDTH = `CHI_LINK_BITS, DEPTH = `CHI_CREDITS; + typedef struct packed { + logic active, input_valid, input_ready, output_valid, output_ready, pend, credit; + logic [WIDTH-1:0] input_data, output_data; + logic [`CHI_CREDIT_BITS-1:0] credits; + } link_signals; +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/agent.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/agent.svh new file mode 100644 index 000000000..dc28eea7a --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/agent.svh @@ -0,0 +1,24 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + monitor mon; + endpoint_driver endpoint; + uvm_sequencer #(packet_item) seqr[2]; + source_driver #(0) source0; + source_driver #(1) source1; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + mon = monitor::type_id::create("monitor", this); + endpoint = endpoint_driver::type_id::create("endpoint", this); + seqr[0] = new("seqr0", this); + source0 = source_driver#(0)::type_id::create("source0", this); + seqr[1] = new("seqr1", this); + source1 = source_driver#(1)::type_id::create("source1", this); + endfunction + function void connect_phase(uvm_phase phase); + source0.seq_item_port.connect(seqr[0].seq_item_export); + source1.seq_item_port.connect(seqr[1].seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/driver.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/driver.svh new file mode 100644 index 000000000..1b3297f7a --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/driver.svh @@ -0,0 +1,39 @@ +class source_driver #( + int PORT = 0 +) extends uvm_driver #(packet_item); + `uvm_component_param_utils(source_driver#(PORT)) + virtual chi_sram_if vif; + protocol_model state; + packet_item pending; + bit driven = 0; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual chi_sram_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI source interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + if (pending == null) seq_item_port.try_next_item(pending); + driven = 0; + if (PORT == 0) begin + vif.req_flit = pending != null ? pending.payload : state.idle.req_flit; + driven = pending != null && !vif.reset && state.req_credits > 0; + vif.req_valid = driven; + end else begin + vif.dat_flit = pending != null ? pending.payload : state.idle.dat_flit; + driven = pending != null && !vif.reset && state.dat_credits > 0; + vif.dat_valid = driven; + end + @(posedge vif.clock); + if (driven) begin + #1; + seq_item_port.item_done(); + pending = null; + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/endpoint.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/endpoint.svh new file mode 100644 index 000000000..2d600813b --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/endpoint.svh @@ -0,0 +1,24 @@ +class endpoint_driver extends uvm_component; + `uvm_component_utils(endpoint_driver) + virtual chi_sram_if vif; + protocol_model state; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual chi_sram_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI endpoint interface is required") + endfunction + task run_phase(uvm_phase phase); + forever begin + @(negedge vif.clock); + vif.rx_active_req = state.idle.rx_active_req; + vif.tx_active_ack = state.idle.tx_active_ack; + vif.req_pend = state.idle.req_pend; + vif.dat_pend = state.idle.dat_pend; + vif.rsp_credit = !vif.reset && vif.tx_active_req && vif.tx_active_ack && state.grant_enabled && state.rsp_granted < 15 && state.cycle % 5 != 0; + vif.out_dat_credit = !vif.reset && vif.tx_active_req && vif.tx_active_ack && state.grant_enabled && state.out_dat_granted < 15 && state.cycle % 7 < 3; + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/item.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/item.svh new file mode 100644 index 000000000..a1c674ee2 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/item.svh @@ -0,0 +1,37 @@ +class packet_item extends uvm_sequence_item; + `uvm_object_utils(packet_item) + logic [`CHI_DAT_BITS-1:0] payload; + bit sc_success, failed; + function new(string name = "packet_item"); + super.new(name); + endfunction +endclass +class sample_item extends uvm_sequence_item; + `uvm_object_utils(sample_item) + pins_t sampled; + function new(string name = "sample_item"); + super.new(name); + endfunction +endclass +class result_item extends uvm_sequence_item; + `uvm_object_utils_begin(result_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(error, UVM_ALL_ON) + `uvm_object_utils_end + logic [63:0] data; + logic error; + function new(string name = "result_item"); + super.new(name); + endfunction +endclass +class packet_sequence extends uvm_sequence #(packet_item); + `uvm_object_utils(packet_sequence) + packet_item item; + function new(string name = "packet_sequence"); + super.new(name); + endfunction + task body(); + start_item(item); + finish_item(item); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/monitor.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/monitor.svh new file mode 100644 index 000000000..cf177eaed --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/agent/monitor.svh @@ -0,0 +1,44 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual chi_sram_if vif; + uvm_analysis_port #(sample_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual chi_sram_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI monitor interface is required") + endfunction + task run_phase(uvm_phase phase); + sample_item t; + forever begin + @(posedge vif.clock); + t = sample_item::type_id::create("observed"); + t.sampled.reset = vif.reset; + t.sampled.req_pend = vif.req_pend; + t.sampled.req_valid = vif.req_valid; + t.sampled.req_credit = vif.req_credit; + t.sampled.req_flit = vif.req_flit; + t.sampled.dat_pend = vif.dat_pend; + t.sampled.dat_valid = vif.dat_valid; + t.sampled.dat_credit = vif.dat_credit; + t.sampled.dat_flit = vif.dat_flit; + t.sampled.rsp_pend = vif.rsp_pend; + t.sampled.rsp_valid = vif.rsp_valid; + t.sampled.rsp_credit = vif.rsp_credit; + t.sampled.rsp_flit = vif.rsp_flit; + t.sampled.out_dat_pend = vif.out_dat_pend; + t.sampled.out_dat_valid = vif.out_dat_valid; + t.sampled.out_dat_credit = vif.out_dat_credit; + t.sampled.out_dat_flit = vif.out_dat_flit; + t.sampled.rx_active_req = vif.rx_active_req; + t.sampled.rx_active_ack = vif.rx_active_ack; + t.sampled.tx_active_req = vif.tx_active_req; + t.sampled.tx_active_ack = vif.tx_active_ack; + t.sampled.outstanding = vif.outstanding; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/env/env.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/env/env.svh new file mode 100644 index 000000000..5ec2c87e7 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/env/env.svh @@ -0,0 +1,27 @@ +class env extends uvm_env; + `uvm_component_utils(env) + agent bus; + protocol_model state; + reset_controller reset; + result_board #(result_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + state = protocol_model::type_id::create("model", this); + bus = agent::type_id::create("bus", this); + reset = reset_controller::type_id::create("reset", this); + endfunction + function void connect_phase(uvm_phase phase); + bus.mon.ap.connect(state.analysis_export); + bus.endpoint.state = state; + bus.source0.state = state; + bus.source1.state = state; + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/env/model.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/env/model.svh new file mode 100644 index 000000000..4d8f0c030 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/env/model.svh @@ -0,0 +1,79 @@ +class protocol_model extends uvm_subscriber #(sample_item); + `uvm_component_utils(protocol_model) + chandle reference; + int req_credits = 0, dat_credits = 0, rsp_granted = 0, out_dat_granted = 0; + int cycle = 0, checks = 0; + localparam int STALL_READS = 15 / `CHI_BEATS + 8; + bit grant_enabled = 1; + rsp_t responses[$]; + dat_t data_responses[$]; + string scenario; + pins_t sampled; + pins_t idle; + uvm_analysis_port #(result_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + covergroup behavior with function sample (int op, bit error); + option.per_instance = 1; + request_opcode: coverpoint op { + bins read = {4}; bins write_full = {'h1d}; bins write_partial = {'h1c}; + } + response_error: coverpoint error; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!$value$plusargs("CASE=%s", scenario)) `uvm_fatal("CASE", "CHI scenario is required") + idle = '0; + reference = chi_ref_create(256); + endfunction + function void write(sample_item t); + pins_t s = t.sampled; + sampled = s; + if (s.req_valid) begin + req_t req = req_t'(s.req_flit); + behavior.sample(req.opcode, 0); + end + if (s.rsp_valid) begin + rsp_t rsp = rsp_t'(s.rsp_flit); + behavior.sample(4, rsp.error != 0); + end + if (s.out_dat_valid) begin + dat_t dat = dat_t'(s.out_dat_flit); + behavior.sample(4, dat.error != 0); + end + + if (s.reset) begin + req_credits = 0; + dat_credits = 0; + rsp_granted = 0; + out_dat_granted = 0; + responses.delete(); + data_responses.delete(); + end else begin + cycle++; + req_credits += int'(s.req_credit) - int'(s.req_valid); + dat_credits += int'(s.dat_credit) - int'(s.dat_valid); + rsp_granted += int'(s.rsp_credit) - int'(s.rsp_valid); + out_dat_granted += int'(s.out_dat_credit) - int'(s.out_dat_valid); + if (req_credits < 0 || dat_credits < 0 || rsp_granted < 0 || out_dat_granted < 0) + `uvm_fatal("CREDIT", "Flit without credit") + if (s.rsp_valid) responses.push_back(rsp_t'(s.rsp_flit)); + if (s.out_dat_valid) data_responses.push_back(dat_t'(s.out_dat_flit)); + end + + endfunction + function void final_phase(uvm_phase phase); + chi_ref_destroy(reference); + super.final_phase(phase); + endfunction + function void check_phase(uvm_phase phase); + evidence::check(responses.size() == 0 && data_responses.size() == 0, + "CHI SRAM responses remain unchecked"); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/filelist.f b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/filelist.f new file mode 100644 index 000000000..b48d1618b --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/filelist.f @@ -0,0 +1,10 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@RESOURCES@/../common ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/types.svh +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/SramEndpoint/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/interface.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/interface.sv new file mode 100644 index 000000000..66748e026 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/interface.sv @@ -0,0 +1,17 @@ +`include "parameters.svh" +`include "profile.svh" +interface chi_sram_if ( + input logic clock +); + logic reset; + logic req_pend, req_valid, req_credit; + logic [`CHI_REQ_BITS-1:0] req_flit; + logic dat_pend, dat_valid, dat_credit; + logic [`CHI_DAT_BITS-1:0] dat_flit; + logic rsp_pend, rsp_valid, rsp_credit; + logic [`CHI_RSP_BITS-1:0] rsp_flit; + logic out_dat_pend, out_dat_valid, out_dat_credit; + logic [`CHI_DAT_BITS-1:0] out_dat_flit; + logic rx_active_req, rx_active_ack, tx_active_req, tx_active_ack; + logic [3:0] outstanding; +endinterface diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/package.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/package.sv new file mode 100644 index 000000000..68a20b18a --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/package.sv @@ -0,0 +1,29 @@ +package chi_sram_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import observed_types::*; + `include "uvm_macros.svh" + `include "flit_types.svh" + import "DPI-C" function chandle chi_ref_create(input int unsigned lines); + import "DPI-C" function void chi_ref_destroy(input chandle model); + import "DPI-C" function byte unsigned chi_ref_write( + input chandle model, + input longint unsigned addr, + input bit [511:0] data, + input bit [63:0] mask + ); + import "DPI-C" function byte unsigned chi_ref_read( + input chandle model, + input longint unsigned addr, + output bit [511:0] data + ); + `include "agent/item.svh" + `include "env/model.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/endpoint.svh" + `include "agent/agent.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/seq/sequence.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/seq/sequence.svh new file mode 100644 index 000000000..3a0e27f9d --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/seq/sequence.svh @@ -0,0 +1,450 @@ +class scenario_sequence extends uvm_sequence; + `uvm_object_utils(scenario_sequence) + localparam int STALL_READS = 15 / `CHI_BEATS + 8; + env test_env; + protocol_model state; + virtual chi_sram_if vif; + string scenario; + function new(string name = "scenario_sequence"); + super.new(name); + endfunction + task packet(int port, logic [`CHI_DAT_BITS-1:0] payload, bit sc_success = 0, bit failed = 0); + packet_sequence seq = packet_sequence::type_id::create("packet"); + seq.item = packet_item::type_id::create("item"); + seq.item.payload = payload; + seq.item.sc_success = sc_success; + seq.item.failed = failed; + seq.start(test_env.bus.seqr[port]); + endtask + task reset_link(); + state.idle.rx_active_req = 0; + state.idle.tx_active_ack = 0; + state.idle.req_pend = 0; + state.idle.dat_pend = 0; + test_env.reset.pulse(4); + repeat (3) @(negedge vif.clock); + state.idle.rx_active_req = 1; + wait (state.sampled.tx_active_req); + repeat (3) @(negedge vif.clock); + state.idle.tx_active_ack = 1; + state.idle.req_pend = 1; + state.idle.dat_pend = 1; + repeat (4) @(negedge vif.clock); + evidence::check(state.sampled.rx_active_ack && state.sampled.tx_active_req, + "CHI SRAM activation failed"); + endtask + + task send_req(req_t value); + packet(0, value); + endtask + + task send_dat(dat_t value); + packet(1, value); + endtask + + function req_t request(int opcode, int txn, longint unsigned addr, int source = 3); + req_t r = '0; + r.opcode = opcode; + r.txn = txn; + r.addr = addr; + r.size = 6; + r.src = (txn ^ source) & 127; + r.tgt = 1; + r.return_nid = (txn ^ 85) & 127; + r.return_txn = txn + 100; + r.allow_retry = txn % 2; + r.mem_attr = (txn % 3 == 0 ? 0 : txn % 3 == 1 ? 4 : 12) | (txn % 2); + r.likely_shared = txn % 2; + r.lpid = txn % 32; + r.qos = txn % 16; + r.trace_tag = txn % 2; + return r; + endfunction + task response(req_t r, int opcode, output rsp_t out_rsp); + bit found = 0; + while (!found) begin + @(negedge vif.clock); + foreach (state.responses[i]) begin + if (state.responses[i].txn == r.txn && state.responses[i].tgt == r.src) begin + out_rsp = state.responses[i]; + state.responses.delete(i); + found = 1; + break; + end + end + end + if (out_rsp.opcode != opcode || out_rsp.src != 1 || out_rsp.qos != r.qos || + out_rsp.trace_tag != r.trace_tag) + `uvm_fatal("RESPONSE", $sformatf("Bad response txn=%0d opcode=%0h", r.txn, out_rsp.opcode)) + state.checks++; + evidence::checks++; + endtask + function dat_t write_beat(req_t r, rsp_t dbid_rsp, int beat, bit [`CHI_DATA_BITS-1:0] data, + bit [`CHI_BE_BITS-1:0] mask = '1); + dat_t d = '0; + d.tgt = 1; + d.src = r.src; + d.txn = dbid_rsp.dbid; + d.qos = r.qos; + d.dbid = {4'b0, r.txn}; + d.line_id = r.addr[11:6]; + d.trace_tag = r.trace_tag; + d.opcode = 3; + d.data_id = beat * `CHI_DATA_ID_STEP; + d.data = data; + d.be = mask; + return d; + endfunction + task write_data(req_t r, rsp_t dbid_rsp, bit [511:0] data, bit [63:0] mask); + for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) + send_dat(write_beat( + r, + dbid_rsp, + beat, + data[beat*`CHI_DATA_BITS+:`CHI_DATA_BITS], + mask[beat*`CHI_BE_BITS+:`CHI_BE_BITS] + )); + endtask + task write_line(int txn, longint unsigned addr, bit [511:0] data, bit [63:0] mask = '1); + req_t r = request(mask == '1 ? 'h1d : 'h1c, txn, addr); + rsp_t s; + byte unsigned expected_error; + send_req(r); + response(r, 6, s); + write_data(r, s, data, mask); + response(r, 4, s); + expected_error = chi_ref_write(state.reference, addr, data, mask); + if (s.error != (expected_error ? 3 : 0)) `uvm_fatal("ERROR", "Wrong write error") + state.checks++; + evidence::checks++; + endtask + task read_line(int txn, longint unsigned addr); + req_t r = request(4, txn, addr); + dat_t d; + bit [511:0] golden; + bit [`CHI_BEATS-1:0] seen = 0; + byte unsigned error = chi_ref_read(state.reference, addr, golden); + send_req(r); + while (!(&seen)) begin + bit found = 0; + @(negedge vif.clock); + foreach (state.data_responses[i]) begin + if (state.data_responses[i].txn == r.return_txn) begin + d = state.data_responses[i]; + state.data_responses.delete(i); + found = 1; + break; + end + end + if (found) begin + int b = d.data_id / `CHI_DATA_ID_STEP; + if ((d.data_id % `CHI_DATA_ID_STEP != 0 || b >= `CHI_BEATS) || seen[b] || d.opcode != 4 || + d.tgt != r.return_nid || d.src != 1 || d.home != r.src || d.dbid != r.txn || + d.error != (error ? 3 : 0) || d.qos != r.qos || d.trace_tag != r.trace_tag || + d.data !== golden[b*`CHI_DATA_BITS +: `CHI_DATA_BITS]) + `uvm_fatal("READ", $sformatf("Bad read data txn=%0d DataID=%0d", txn, d.data_id)) + seen[b] = 1; + state.checks++; + evidence::checks++; + end + end + endtask + task scenarios(); + req_t r[8]; + rsp_t dbid[8], s; + bit [511:0] words[8]; + reset_link(); + if (scenario == "bad_opcode") begin + send_req(request('h07, 0, 0)); + repeat (20) @(negedge vif.clock); + `uvm_fatal("NEGATIVE", "Unsupported opcode did not terminate RTL") + end else if (scenario == "duplicate_data") begin + r[0] = request('h1d, 0, 0); + send_req(r[0]); + response(r[0], 6, s); + begin + dat_t d = '0; + d.tgt = 1; + d.src = r[0].src; + d.txn = s.dbid; + d.opcode = 3; + d.data_id = 0; + d.be = '1; + send_dat(d); + send_dat(d); + end + repeat (20) @(negedge vif.clock); + `uvm_fatal("NEGATIVE", "Duplicate DataID did not terminate RTL") + end else begin + // Fill all eight transaction slots, then interleave reverse-order DataIDs. + for (int i = 0; i < 8; i++) begin + for (int b = 0; b < 64; b++) words[i][b*8+:8] = (i * 37 + b); + r[i] = request('h1d, i, i * 64, i % 2 ? 5 : 3); + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + if (state.sampled.outstanding != 8) `uvm_fatal("MSHR", "Failed to occupy all slots") + for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) begin + for (int i = 7; i >= 0; i--) begin + send_dat(write_beat(r[i], dbid[i], beat, words[i][beat*`CHI_DATA_BITS+:`CHI_DATA_BITS], '1 + )); + end + end + for (int i = 0; i < 8; i++) begin + response(r[i], 4, s); + if (s.error != 0 || chi_ref_write(state.reference, i * 64, words[i], '1) != 0) + `uvm_fatal("WRITE", "Initial full write failed") + end + for (int i = 0; i < 8; i++) read_line(32 + i, i * 64); + // Every live slot must merge byte enables independently, including interleaved beats. + for (int phase_index = 0; phase_index < 2; phase_index++) begin + bit [63:0] mask = phase_index ? 64'haaaaaaaaaaaaaaaa : 64'h5555555555555555; + for (int i = 0; i < 8; i++) begin + r[i] = request('h1c, 140 + phase_index * 8 + i, i * 64, i % 2 ? 5 : 3); + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) begin + for (int i = 7; i >= 0; i--) begin + send_dat(write_beat( + r[i], + dbid[i], + beat, + phase_index ? words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS] : ~words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS], + mask[beat*`CHI_BE_BITS+:`CHI_BE_BITS] + )); + end + end + for (int i = 0; i < 8; i++) begin + bit [511:0] data = phase_index ? words[i] : ~words[i]; + response(r[i], 4, s); + if (s.error || chi_ref_write(state.reference, i * 64, data, mask)) + `uvm_fatal("MASK", "Concurrent partial write failed") + end + for (int i = 0; i < 8; i++) read_line(240 + phase_index * 8 + i, i * 64); + end + `uvm_info("PHASE", "Full-width slot vectors", UVM_LOW) + // Exercise each real transaction slot's address, opaque IDs and error status. + // Writes are concurrent, addresses distinct; then verify valid writes by readback. + for (int polarity = 0; polarity < 2; polarity++) begin + for (int address_bit = 9; address_bit < 44; address_bit++) begin + for (int i = 0; i < 8; i++) begin + longint unsigned high = (64'h1 << address_bit) ^ (polarity ? 64'hffffffffe00 : 0); + int tag = (1 << ((address_bit + i) % 12)) ^ (polarity ? 'hfff : 0); + int source = ((1 << ((address_bit + i) % 7)) | 2) ^ (polarity ? 'h7f : 0); + r[i] = request('h1d, tag, high | ((i ^ (1 << ((address_bit + polarity) % 3))) << 6)); + r[i].src = source; + r[i].return_nid = source ^ 'h7f; + r[i].return_txn = tag ^ 'hfff; + r[i].qos = (address_bit + i) % 16; + r[i].trace_tag = polarity; + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) begin + for (int i = 7; i >= 0; i--) begin + send_dat(write_beat( + r[i], + dbid[i], + beat, + polarity ? words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS] : ~words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS], + '1 + )); + end + end + for (int i = 0; i < 8; i++) begin + bit [511:0] data = polarity ? words[i] : ~words[i]; + byte unsigned error; + response(r[i], 4, s); + error = chi_ref_write(state.reference, r[i].addr, data, '1); + if (s.error != (error ? 3 : 0)) + `uvm_fatal("ERROR_SLOT", "Wrong error or opaque transaction attribution") + state.checks++; + evidence::checks++; + end + if (!polarity && address_bit < 14) + for (int i = 0; i < 8; i++) read_line(600 + address_bit * 8 + i, r[i].addr); + end + end + write_line(50, 0, '1, 64'h5555555555555555); + write_line(51, 0, '0, 64'haaaaaaaaaaaaaaaa); + write_line(52, 0, '1, '0); + read_line(53, 0); + write_line(54, 255 * 64, '1); + read_line(55, 255 * 64); + write_line(56, 256 * 64, '1); + read_line(57, 256 * 64); + // Full-width addresses and TxnIDs must not alias into SRAM or DBID slots. + write_line(4095, 0, '1); + read_line(3950, 0); + write_line(58, 64'h80000000000, '1); + read_line(59, 64'h80000000000); + `uvm_info("PHASE", "Data credit stall", UVM_LOW) + // Block output credit return; requests remain live, then resume safely. + state.grant_enabled = 0; + for (int i = 0; i < STALL_READS; i++) send_req(request(4, 80 + i, (i % 8) * 64)); + repeat (25) @(negedge vif.clock); + if (state.sampled.outstanding == 0) + `uvm_fatal("STALL", "No live request under data credit stall") + state.grant_enabled = 1; + // These reads were issued already: compare every returned beat before reset. + for (int i = 0; i < STALL_READS; i++) begin + bit [511:0] golden; + bit [`CHI_BEATS-1:0] seen = 0; + if (chi_ref_read(state.reference, (i % 8) * 64, golden)) + `uvm_fatal("MODEL", "Unexpected error") + while (!(&seen)) begin + @(negedge vif.clock); + foreach (state.data_responses[j]) begin + if (state.data_responses[j].txn == 180 + i) begin + dat_t d = state.data_responses[j]; + int b = d.data_id / `CHI_DATA_ID_STEP; + if (seen[b] || d.data !== golden[b*`CHI_DATA_BITS+:`CHI_DATA_BITS]) + `uvm_fatal("STALL", "Stalled read corrupted") + seen[b] = 1; + state.checks++; + evidence::checks++; + state.data_responses.delete(j); + break; + end + end + end + end + `uvm_info("PHASE", "Read-to-write slot reuse", UVM_LOW) + // Reallocate every slot to a write after the all-slot read drain. + for (int i = 0; i < 8; i++) begin + r[i] = request('h1d, 2800 + i, i * 64); + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + for (int i = 0; i < 8; i++) write_data(r[i], dbid[i], words[i], '1); + for (int i = 0; i < 8; i++) begin + response(r[i], 4, s); + if (s.error || chi_ref_write(state.reference, i * 64, words[i], '1)) + `uvm_fatal("SLOT_REUSE", "Read-to-write slot reuse failed") + end + for (int i = 0; i < 8; i++) read_line(2900 + i, i * 64); + `uvm_info("PHASE", "Response and request queue saturation", UVM_LOW) + // Exhaust all15 RSP credits, then fill all8 node slots and all4 RX entries. + // Distinct TxnIDs from one source and equal TxnIDs from distinct sources are legal. + begin + req_t blocked[11]; + int data_order[8] = '{0, 2, 1, 3, 4, 6, 5, 7}; + while (state.rsp_granted != 15) @(negedge vif.clock); + state.grant_enabled = 0; + for (int i = 0; i < 8; i++) begin + r[i] = request('h1d, 3200 + i, i * 64); + r[i].src = 3; + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + for (int i = 0; i < 8; i++) begin + int slot = data_order[i]; + write_data(r[slot], dbid[slot], words[slot], '1); + end + // Drain seven initial Comps before new DBID state.responses compete for credits. + // One-beat writes can otherwise finish their data input before SRAM replies. + while (state.rsp_granted != 0 || state.responses.size() != 7) @(negedge vif.clock); + for (int i = 0; i < 11; i++) begin + blocked[i] = request('h1d, 3400, i * 64); + blocked[i].src = 10 + i; + send_req(blocked[i]); + end + repeat (30) @(negedge vif.clock); + if (state.sampled.outstanding != 8 || state.req_credits != 0) + `uvm_fatal("RSP_STALL", "RSP credit exhaustion did not saturate node and RX queue") + state.grant_enabled = 1; + for (int i = 0; i < 8; i++) begin + response(r[i], 4, s); + if (s.error || chi_ref_write(state.reference, i * 64, words[i], '1)) + `uvm_fatal("RSP_STALL", "Initial write failed") + end + for (int i = 0; i < 11; i++) begin + response(blocked[i], 6, s); + write_data(blocked[i], s, words[i%8], '1); + response(blocked[i], 4, s); + if (s.error || chi_ref_write(state.reference, i * 64, words[i%8], '1)) + `uvm_fatal("RSP_STALL", "Queued same-TxnID write attributed to wrong source") + end + for (int i = 0; i < 11; i++) read_line(3600 + i, i * 64); + end + `uvm_info("PHASE", "Isolated upper read slots", UVM_LOW) + // Hold lower slots in write-data collection to isolate each upper read slot. + // A second read in that slot exercises arbitration after its own last grant. + for (int target_slot = 4; target_slot < 8; target_slot++) begin + for (int i = 0; i < target_slot; i++) begin + r[i] = request('h1d, 3700 + i, i * 64); + r[i].src = 3; + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + read_line(3800 + target_slot * 2, 0); + read_line(3801 + target_slot * 2, 0); + for (int i = 0; i < target_slot; i++) begin + write_data(r[i], dbid[i], words[i], '1); + response(r[i], 4, s); + if (s.error || chi_ref_write(state.reference, i * 64, words[i], '1)) + `uvm_fatal("ARBITRATION", "Isolated slot damaged held writes") + end + end + `uvm_info("PHASE", "Source and transaction identity", UVM_LOW) + // Slot7 stays live while a lower free slot accepts another transaction from its source. + for (int i = 0; i < 8; i++) begin + r[i] = request('h1d, 3900 + i, i * 64); + r[i].src = 3; + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + for (int i = 0; i < 2; i++) begin + write_data(r[i], dbid[i], words[i], '1); + response(r[i], 4, s); + if (s.error || chi_ref_write(state.reference, i * 64, words[i], '1)) + `uvm_fatal("SOURCE_IDS", "Initial source transaction failed") + end + for (int i = 0; i < 2; i++) begin + r[i] = request('h1d, 3990 + i, i * 64); + r[i].src = 3; + send_req(r[i]); + response(r[i], 6, dbid[i]); + end + repeat (4) @(negedge vif.clock); + for (int i = 0; i < 8; i++) begin + write_data(r[i], dbid[i], words[i], '1); + response(r[i], 4, s); + if (s.error || chi_ref_write(state.reference, i * 64, words[i], '1)) + `uvm_fatal("SOURCE_IDS", "Concurrent source transaction failed") + end + `uvm_info("PHASE", "Reset cancellation", UVM_LOW) + // Cancel a write waiting for payload. SRAM data from completed writes survives. + send_req(request('h1d, 110, 64)); + response(request('h1d, 110, 64), 6, s); + reset_link(); + read_line(111, 64); + write_line(110, 64, '1); + write_line(110, 64, '1); // Reuse is legal only after the first completion. + read_line(112, 64); + repeat (15) @(negedge vif.clock); + if (state.sampled.outstanding != 0 || state.responses.size() || state.data_responses.size()) + `uvm_fatal("DRAIN", "Unmatched transactions remain") + // Payload is unconstrained while FLITV=0 and must not create a transaction. + @(negedge vif.clock); + state.idle.req_flit = '0; + state.idle.dat_flit = '0; + repeat (2) @(negedge vif.clock); + state.idle.req_flit = '1; + state.idle.dat_flit = '1; + repeat (2) @(negedge vif.clock); + state.idle.req_flit = '0; + state.idle.dat_flit = '0; + repeat (2) @(negedge vif.clock); + if (state.sampled.outstanding || state.responses.size() || state.data_responses.size()) + `uvm_fatal("INVALID_CHANNEL", "Inactive physical payload changed transactions") + `uvm_info("CHECKS", $sformatf("CHI SN-F: %0d checked responses/data beats", state.checks), + UVM_LOW) + + end + endtask + task body(); + scenarios(); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/tb.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/tb.sv new file mode 100644 index 000000000..c46a36c7b --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/tb.sv @@ -0,0 +1,41 @@ +module sram_tb; + import uvm_pkg::*; + import chi_sram_pkg::*; + logic clock = 0; + clock_if timing (clock); + always #5 clock = ~clock; + chi_sram_if source_if (clock); + `CHI_SRAM_MODULE dut ( + .clock(clock), + .reset(source_if.reset), + .io_chi_rxReq_flitpend(source_if.req_pend), + .io_chi_rxReq_flitv(source_if.req_valid), + .io_chi_rxReq_flit(source_if.req_flit), + .io_chi_rxReq_lcrdv(source_if.req_credit), + .io_chi_rxDat_flitpend(source_if.dat_pend), + .io_chi_rxDat_flitv(source_if.dat_valid), + .io_chi_rxDat_flit(source_if.dat_flit), + .io_chi_rxDat_lcrdv(source_if.dat_credit), + .io_chi_txRsp_flitpend(source_if.rsp_pend), + .io_chi_txRsp_flitv(source_if.rsp_valid), + .io_chi_txRsp_flit(source_if.rsp_flit), + .io_chi_txRsp_lcrdv(source_if.rsp_credit), + .io_chi_txDat_flitpend(source_if.out_dat_pend), + .io_chi_txDat_flitv(source_if.out_dat_valid), + .io_chi_txDat_flit(source_if.out_dat_flit), + .io_chi_txDat_lcrdv(source_if.out_dat_credit), + .io_chi_rxLinkActiveReq(source_if.rx_active_req), + .io_chi_rxLinkActiveAck(source_if.rx_active_ack), + .io_chi_txLinkActiveReq(source_if.tx_active_req), + .io_chi_txLinkActiveAck(source_if.tx_active_ack), + .io_chi_rxSActive(source_if.rx_active_req), + .io_chi_txSActive(), + .io_outstanding(source_if.outstanding) + ); + assign source_if.reset = timing.reset; + initial begin + uvm_config_db#(virtual chi_sram_if)::set(null, "uvm_test_top*", "vif", source_if); + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top.env.reset", "timing", timing); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/tests/test.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/tests/test.svh new file mode 100644 index 000000000..4c5de65de --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/tests/test.svh @@ -0,0 +1,25 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + virtual chi_sram_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + if (`CHI_NODE_BITS != 7 || `CHI_ADDRESS_BITS != 44 || `CHI_NODE_ID != 1) + `uvm_fatal("PROFILE", + "CHI packet scenarios require 7-bit nodes, 44-bit addresses and requester 1") + super.build_phase(phase); + test_env = env::type_id::create("env", this); + if (!uvm_config_db#(virtual chi_sram_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI timing interface is required") + endfunction + task execute(); + scenario_sequence seq = scenario_sequence::type_id::create("scenario"); + seq.test_env = test_env; + seq.state = test_env.state; + seq.vif = vif; + seq.scenario = test_env.state.scenario; + seq.start(null); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/types.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/types.svh new file mode 100644 index 000000000..95197dd2a --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/sram/types.svh @@ -0,0 +1,16 @@ +`include "profile.svh" +package observed_types; + typedef struct packed { + logic reset; + logic req_pend, req_valid, req_credit; + logic [`CHI_REQ_BITS-1:0] req_flit; + logic dat_pend, dat_valid, dat_credit; + logic [`CHI_DAT_BITS-1:0] dat_flit; + logic rsp_pend, rsp_valid, rsp_credit; + logic [`CHI_RSP_BITS-1:0] rsp_flit; + logic out_dat_pend, out_dat_valid, out_dat_credit; + logic [`CHI_DAT_BITS-1:0] out_dat_flit; + logic rx_active_req, rx_active_ack, tx_active_req, tx_active_ack; + logic [3:0] outstanding; + } pins_t; +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/chi/src/main/resources/suite.toml new file mode 100644 index 000000000..832f73ad9 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/suite.toml @@ -0,0 +1,241 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libchi_reference_model" + +[[tests]] +functional_groups = ["chi_retry_pkg::protocol_model::behavior"] +name = "retry" +top = "retry_tb" +test = "protocol_test" +filelist = "retry/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=normal"] +assertion = "" + +[[tests]] +functional_groups = [] +name = "retry_bad_opcode" +top = "retry_tb" +test = "protocol_test" +filelist = "retry/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=bad_opcode"] +assertion = "Unsupported CHI retry request opcode" + +[[tests]] +functional_groups = [] +name = "retry_duplicate_txnid" +top = "retry_tb" +test = "protocol_test" +filelist = "retry/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=duplicate_txnid"] +assertion = "CHI retry requester reused a live TxnID" + +[[tests]] +functional_groups = ["chi_rnf_pkg::protocol_model::behavior"] +name = "rnf" +top = "rnf_tb" +test = "protocol_test" +filelist = "rnf/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=normal"] +assertion = "" + +[[tests]] +functional_groups = [] +name = "rnf_bad_dbid" +top = "rnf_tb" +test = "protocol_test" +filelist = "rnf/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=bad_dbid"] +assertion = "Inconsistent multi-beat fill metadata" + +[[tests]] +functional_groups = [] +name = "rnf_bad_dbid_width" +top = "rnf_tb" +test = "protocol_test" +filelist = "rnf/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=bad_dbid_width"] +assertion = "Fill DBID exceeds completion ID width" + +[[tests]] +functional_groups = [] +name = "rnf_bad_resp" +top = "rnf_tb" +test = "protocol_test" +filelist = "rnf/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=bad_resp"] +assertion = "Inconsistent multi-beat fill metadata" + +[[tests]] +functional_groups = ["chi_sram_pkg::protocol_model::behavior"] +name = "sram" +top = "sram_tb" +test = "protocol_test" +filelist = "sram/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=normal"] +assertion = "" + +[[tests]] +functional_groups = [] +name = "sram_bad_opcode" +top = "sram_tb" +test = "protocol_test" +filelist = "sram/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=bad_opcode"] +assertion = "Unsupported CHI request opcode" + +[[tests]] +functional_groups = [] +name = "sram_duplicate_data" +top = "sram_tb" +test = "protocol_test" +filelist = "sram/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=duplicate_data"] +assertion = "Duplicate CHI DataID" + +[[tests]] +functional_groups = ["link_pkg::model::behavior"] +name = "tx" +top = "tx_tb" +test = "protocol_test" +filelist = "tx/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "tx_bad_credit" +top = "tx_tb" +test = "invalid_test" +filelist = "tx/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "CHI TX credit overflow" + +[[tests]] +functional_groups = ["link_pkg::model::behavior"] +name = "rx" +top = "rx_tb" +test = "protocol_test" +filelist = "rx/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "rx_bad_credit" +top = "rx_tb" +test = "invalid_test" +filelist = "rx/filelist.f" +timeout_ns = 450000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "CHI RX flit without granted credit" diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/agent.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/agent.svh new file mode 100644 index 000000000..6c18e56b9 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/agent.svh @@ -0,0 +1,18 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + driver drv; + monitor mon; + uvm_sequencer #(cycle_item) seqr; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + seqr = new("seqr", this); + endfunction + function void connect_phase(uvm_phase phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/driver.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/driver.svh new file mode 100644 index 000000000..1468e5b41 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/driver.svh @@ -0,0 +1,28 @@ +class driver extends uvm_driver #(cycle_item); + `uvm_component_utils(driver) + virtual link_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual link_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI link interface is required") + endfunction + task run_phase(uvm_phase phase); + cycle_item req; + forever begin + seq_item_port.get_next_item(req); + @(negedge vif.clock); + vif.reset = req.reset; + vif.pins.active = req.pins.active; + vif.pins.input_valid = req.pins.input_valid; + vif.pins.input_data = req.pins.input_data; + vif.pins.credit = req.pins.credit; + @(posedge vif.clock); + req.sampled = vif.pins; + #1; + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/item.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/item.svh new file mode 100644 index 000000000..0147091c1 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/item.svh @@ -0,0 +1,17 @@ +class cycle_item extends uvm_sequence_item; + `uvm_object_utils(cycle_item) + link_signals pins, sampled; + bit reset; + function new(string name = "cycle_item"); + super.new(name); + endfunction +endclass +class flit_item extends uvm_sequence_item; + `uvm_object_utils_begin(flit_item) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_object_utils_end + logic [WIDTH-1:0] data; + function new(string name = "flit_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/monitor.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/monitor.svh new file mode 100644 index 000000000..5ace5103b --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/agent/monitor.svh @@ -0,0 +1,24 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual link_if vif; + uvm_analysis_port #(cycle_item) ap; + function new(string name, uvm_component parent); + super.new(name, parent); + ap = new("ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual link_if)::get(this, "", "vif", vif)) + `uvm_fatal("VIF", "CHI link monitor requires an interface") + endfunction + task run_phase(uvm_phase phase); + cycle_item t; + forever begin + @(posedge vif.clock); + t = cycle_item::type_id::create("observed"); + t.reset = vif.reset; + t.sampled = vif.pins; + ap.write(t); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/env/env.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/env/env.svh new file mode 100644 index 000000000..8f7141d8c --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/env/env.svh @@ -0,0 +1,26 @@ +class env extends uvm_env; + `uvm_component_utils(env) + agent bus; + model reference; + result_board #(flit_item, 0) scoreboard; + uvm_analysis_imp #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + bus = agent::type_id::create("bus", this); + reference = model::type_id::create("model", this); + scoreboard = result_board#(flit_item, 0)::type_id::create("scoreboard", this); + endfunction + function void connect_phase(uvm_phase phase); + bus.mon.ap.connect(reference.analysis_export); + reference.expected_ap.connect(scoreboard.expected_export); + reference.actual_ap.connect(scoreboard.actual_export); + reference.reset_ap.connect(reset_export); + endfunction + function void write(bit value); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/env/model.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/env/model.svh new file mode 100644 index 000000000..26b63b133 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/env/model.svh @@ -0,0 +1,55 @@ +class model extends uvm_subscriber #(cycle_item); + `uvm_component_utils(model) + uvm_analysis_port #(flit_item) expected_ap, actual_ap; + uvm_analysis_port #(bit) reset_ap; + int balance = 0; + bit delayed_credit = 0, delayed_send = 0; + covergroup behavior with function sample (int credits, bit stalled, bit simultaneous); + option.per_instance = 1; + available: coverpoint credits { + bins empty = {0}; bins partial = {[1 : DEPTH - 1]}; bins full = {DEPTH}; + } + source_stall: coverpoint stalled; + credit_and_send: coverpoint simultaneous; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + expected_ap = new("expected_ap", this); + actual_ap = new("actual_ap", this); + reset_ap = new("reset_ap", this); + behavior = new(); + endfunction + function void write(cycle_item t); + link_signals s = t.sampled; + flit_item item; + bit fire = s.input_valid && s.input_ready; + if (t.reset) begin + reset_ap.write(1); + balance = 0; + delayed_credit = 0; + delayed_send = 0; + return; + end + evidence::check(s.credits == balance, "CHI TX credit count mismatch"); + evidence::check(s.input_ready === (s.active && balance != 0), + "CHI TX ready disagrees with available credit"); + evidence::check(s.pend === s.active, "CHI TX FLITPEND disagrees with activation"); + evidence::check(s.output_valid === delayed_send, + "CHI TX FLITV did not follow the accepted flit"); + behavior.sample(balance, s.input_valid && !s.input_ready, delayed_credit && fire); + if (fire) begin + item = flit_item::type_id::create("expected"); + item.data = s.input_data; + expected_ap.write(item); + end + if (s.output_valid) begin + item = flit_item::type_id::create("actual"); + item.data = s.output_data; + actual_ap.write(item); + end + balance += int'(delayed_credit) - int'(fire); + evidence::check(balance >= 0 && balance <= DEPTH, "CHI TX credit conservation failed"); + delayed_credit = s.credit; + delayed_send = fire; + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/filelist.f b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/filelist.f new file mode 100644 index 000000000..17aa13b13 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@RESOURCES@/types.svh +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Tx/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/interface.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/interface.sv new file mode 100644 index 000000000..e74a8c197 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/interface.sv @@ -0,0 +1,7 @@ +interface link_if ( + input logic clock +); + import link_types::*; + logic reset = 1; + link_signals pins; +endinterface diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/package.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/package.sv new file mode 100644 index 000000000..824e2712f --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/package.sv @@ -0,0 +1,14 @@ +package link_pkg; + import uvm_pkg::*; + import verification_pkg::*; + import link_types::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/seq/sequence.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/seq/sequence.svh new file mode 100644 index 000000000..a9b7ecbf5 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/seq/sequence.svh @@ -0,0 +1,79 @@ +class traffic extends uvm_sequence #(cycle_item); + `uvm_object_utils(traffic) + env test_env; + link_signals pins = '0, observed; + bit reset = 1; + int grants = 0, cycle = 0, sent = 0; + function new(string name = "traffic"); + super.new(name); + endfunction + task tick(int cycles = 1); + cycle_item req; + repeat (cycles) begin + pins.credit = !reset && pins.active && grants > 0 && cycle % 4 != 0; + if (pins.credit) grants--; + req = cycle_item::type_id::create("cycle"); + start_item(req); + req.pins = pins; + req.reset = reset; + finish_item(req); + observed = req.sampled; + if (!reset && observed.output_valid) grants++; + cycle++; + end + endtask + task restart(); + pins = '0; + reset = 1; + grants = 0; + tick(4); + reset = 0; + pins.active = 1; + grants = DEPTH; + do tick(); while (observed.credits != DEPTH); + endtask + task send(logic [WIDTH-1:0] data); + pins.input_valid = 1; + pins.input_data = data; + do tick(); while (!observed.input_ready); + pins.input_valid = 0; + sent++; + endtask + task drain(); + while (test_env.scoreboard.checked < sent) tick(); + endtask + task body(); + restart(); + for (int n = 0; n < 256; n++) send(WIDTH'(32'h9e3779b9 * (n + 1)) ^ ((n % 2) ? '1 : '0)); + drain(); + // Reset cancels the accepted flit before its registered physical output is observed. + pins.input_valid = 1; + pins.input_data = '1; + do tick(); while (!observed.input_ready); + restart(); + for (int n = 0; n < WIDTH; n++) send(WIDTH'(1) << n); + drain(); + tick(3); + evidence::check(evidence::cancelled == 1, + "CHI TX reset did not cancel exactly one in-flight flit"); + endtask +endclass +class invalid_sequence extends uvm_sequence #(cycle_item); + `uvm_object_utils(invalid_sequence) + function new(string name = "invalid_sequence"); + super.new(name); + endfunction + task body(); + cycle_item req; + for (int n = 0; n < DEPTH + 8; n++) begin + req = cycle_item::type_id::create("overflow"); + start_item(req); + req.pins = '0; + req.reset = n < 4; + req.pins.active = n >= 4; + req.pins.credit = n >= 4; + finish_item(req); + end + `uvm_fatal("EXPECTED_ASSERTION", "CHI TX accepted excess credits") + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/tb.sv b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/tb.sv new file mode 100644 index 000000000..4aa0907ba --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/tb.sv @@ -0,0 +1,24 @@ +module tx_tb; + import uvm_pkg::*; + import link_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + link_if bus (clock); + Tx dut ( + .clock(clock), + .reset(bus.reset), + .io_active(bus.pins.active), + .io_in_valid(bus.pins.input_valid), + .io_in_ready(bus.pins.input_ready), + .io_in_bits(bus.pins.input_data), + .io_link_flitpend(bus.pins.pend), + .io_link_flitv(bus.pins.output_valid), + .io_link_flit(bus.pins.output_data), + .io_link_lcrdv(bus.pins.credit), + .io_credits(bus.pins.credits) + ); + initial begin + uvm_config_db#(virtual link_if)::set(null, "uvm_test_top*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/tests/test.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/tests/test.svh new file mode 100644 index 000000000..38c98c87f --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/tests/test.svh @@ -0,0 +1,31 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env test_env; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + test_env = env::type_id::create("env", this); + endfunction + task execute(); + traffic seq = traffic::type_id::create("traffic"); + seq.test_env = test_env; + seq.start(test_env.bus.seqr); + endtask +endclass +class invalid_test extends base_test; + `uvm_component_utils(invalid_test) + agent bus; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + bus = agent::type_id::create("bus", this); + endfunction + task execute(); + invalid_sequence seq = invalid_sequence::type_id::create("violation"); + seq.start(bus.seqr); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/types.svh b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/types.svh new file mode 100644 index 000000000..0f5428c6d --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/resources/tx/types.svh @@ -0,0 +1,9 @@ +`include "parameters.svh" +package link_types; + localparam int WIDTH = `CHI_LINK_BITS, DEPTH = `CHI_CREDITS; + typedef struct packed { + logic active, input_valid, input_ready, output_valid, output_ready, pend, credit; + logic [WIDTH-1:0] input_data, output_data; + logic [`CHI_CREDIT_BITS-1:0] credits; + } link_signals; +endpackage diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Channel.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/Channel.scala similarity index 91% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/Channel.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/Channel.scala index afb6d5fb3..1b5d8703c 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Channel.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/Channel.scala @@ -1,4 +1,4 @@ -package memcore.bus.chi +package framework.ip.bus.chi import chisel3._ diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/CoherenceState.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/CoherenceState.scala similarity index 78% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/CoherenceState.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/CoherenceState.scala index 282bc251c..1e074f59f 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/CoherenceState.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/CoherenceState.scala @@ -1,4 +1,4 @@ -package memcore.bus.chi +package framework.ip.bus.chi object CoherenceState { val I = 0 diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/DirectedSnoop.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/DirectedSnoop.scala similarity index 83% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/DirectedSnoop.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/DirectedSnoop.scala index 9c75e8d82..dc33d375c 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/DirectedSnoop.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/DirectedSnoop.scala @@ -1,4 +1,4 @@ -package memcore.bus.chi +package framework.ip.bus.chi import chisel3._ diff --git a/arch/src/main/scala/framework/ip/chi/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/Emit.scala new file mode 100644 index 000000000..34a551407 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/Emit.scala @@ -0,0 +1,79 @@ +package framework.ip.bus.chi + +import java.nio.file.{Files, Paths} +import framework.ip.bus.chi.snf.SramEndpoint +import framework.ip.bus.chi.rnf.{BankedChiCache, RnfParams} + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val p = Params(values("nodeIdBits").num.toInt, values("addressBits").num.toInt, values("dataBits").num.toInt) + val flitBits = values("linkFlitBits").num.toInt + val credits = values("credits").num.toInt + val node = values("nodeId").num.toInt + val home = values("homeId").num.toInt + val records = values("records").num.toInt + val banks = values("banks").num.toInt + val cacheLines = values("cacheLines").num.toInt + Files.createDirectories(Paths.get(output)) + Files.writeString( + Paths.get(output, "parameters.svh"), + s"""`ifndef CHI_PARAMETERS +`define CHI_PARAMETERS +`define CHI_NODE_BITS ${p.nodeIdBits} +`define CHI_ADDRESS_BITS ${p.addressBits} +`define CHI_DATA_BITS ${p.dataBits} +`define CHI_SRAM_MODULE SramEndpoint${p.dataBits} +`define CHI_REQ_BITS ${(new RequestFlit(p)).flitWidth} +`define CHI_RSP_BITS ${(new ResponseFlit(p)).flitWidth} +`define CHI_SNP_BITS ${(new SnoopFlit(p)).flitWidth} +`define CHI_LINK_BITS $flitBits +`define CHI_CREDITS $credits +`define CHI_CREDIT_BITS ${chisel3.util.log2Ceil(credits + 1)} +`define CHI_NODE_ID $node +`define CHI_HOME_ID $home +`define CHI_RECORDS $records +`define CHI_PENDING_BITS ${chisel3.util.log2Ceil(records + 1)} +`define CHI_BANKS $banks +`define CHI_CACHE_LINES $cacheLines +`define CHI_SRAM_SLOTS ${values("sramSlots").num.toInt} +`define CHI_SRAM_LINES ${values("sramLines").num.toInt} +`endif +""" + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Tx(flitBits, credits), + args = Array("--target-dir", s"$output/Tx", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Rx(flitBits, credits), + args = Array("--target-dir", s"$output/Rx", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new SramEndpoint( + p, + node, + values("sramSlots").num.toInt, + values("sramLines").num.toInt, + values("sramRxDepth").num.toInt + ), + args = Array("--target-dir", s"$output/SramEndpoint", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new RequestRetry(p, node, records), + args = Array("--target-dir", s"$output/RequestRetry", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new BankedChiCache(RnfParams( + p, + node, + cacheLines, + home, + values("homeCount").num.toInt, + banks, + values("lineResult").bool, + values("probe").bool + )), + args = Array("--target-dir", s"$output/BankedChiCache", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/ip/chi/src/main/scala/Fabric.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/Fabric.scala new file mode 100644 index 000000000..97f6b35cc --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/Fabric.scala @@ -0,0 +1,104 @@ +package framework.ip.bus.chi + +import framework.lib.arbiter.{RRArbiter} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} + +/** Explicit requester-to-Home CHI fabric; no caches or backing-memory ownership. */ +@instantiable +class Fabric(p: Params, requesterCount: Int, homeId: Int) extends Module { + require(requesterCount >= 1 && requesterCount < homeId && BigInt(homeId) < (BigInt(1) << p.nodeIdBits)) + val c = p + + @public + val io = IO(new Bundle { + val active = Input(Bool()) + val blockRequesterRsp = Input(Bool()) + val requesters = Vec(requesterCount, Flipped(new RequesterPort(c))) + val req = Decoupled(new RequestFlit(c)) + val rxRsp = Decoupled(new ResponseFlit(c)) + val rxDat = Decoupled(new DataFlit(c)) + val rsp = Flipped(Decoupled(new ResponseFlit(c))) + val dat = Flipped(Decoupled(new DataFlit(c))) + val snp = Flipped(Decoupled(new DirectedSnoop(c))) + }) + + def creditLink[T <: Flit](source: DecoupledIO[T], sink: DecoupledIO[T], blockRx: Bool = false.B): Unit = { + // Backpressure retains complete flits in the real credit-accounted Rx FIFO. + // Active and FLITV retain their channel semantics throughout the stall. + val tx: Instance[Tx] = Instantiate(new Tx(source.bits.flitWidth, 4)) + val rx: Instance[Rx] = Instantiate(new Rx(source.bits.flitWidth, 4)) + tx.io.active := io.active + rx.io.active := io.active + tx.io.in.valid := source.valid + tx.io.in.bits := source.bits.packed + source.ready := tx.io.in.ready + rx.io.link <> tx.io.link + sink.valid := rx.io.out.valid && !blockRx + sink.bits.unpack(rx.io.out.bits) + rx.io.out.ready := sink.ready && !blockRx + } + + // All three arbiters use initialized history: the default RR lastGrant is not + // reset, so simultaneous first requests can otherwise select unknown data. + def requesterArbiter[T <: Data](flit: T): Instance[RRArbiter[T]] = Instantiate(new RRArbiter(flit, requesterCount)) + + val requests = requesterArbiter(new RequestFlit(c)) + val responses = requesterArbiter(new ResponseFlit(c)) + val data = requesterArbiter(new DataFlit(c)) + io.req <> requests.io.out + io.rxRsp <> responses.io.out + io.rxDat <> data.io.out + for (i <- 0 until requesterCount) { + creditLink(io.requesters(i).req, requests.io.in(i)) + creditLink(io.requesters(i).txRsp, responses.io.in(i), io.blockRequesterRsp) + creditLink(io.requesters(i).txDat, data.io.in(i)) + when(requests.io.in(i).valid) { + assert( + requests.io.in(i).bits.srcId === (i + 1).U && requests.io.in(i).bits.tgtId === homeId.U, + "CacheSystem request crossed a wrong node or role" + ) + } + when(responses.io.in(i).valid)(assert( + responses.io.in(i).bits.srcId === (i + 1).U && + responses.io.in(i).bits.tgtId === homeId.U, + "CacheSystem response crossed a wrong node or role" + )) + when(data.io.in(i).valid)(assert( + data.io.in(i).bits.srcId === (i + 1).U && + data.io.in(i).bits.tgtId === homeId.U, + "CacheSystem data crossed a wrong node or role" + )) + } + val rsp = Wire(Vec(requesterCount, Decoupled(new ResponseFlit(c)))) + val dat = Wire(Vec(requesterCount, Decoupled(new DataFlit(c)))) + val snp = Wire(Vec(requesterCount, Decoupled(new SnoopFlit(c)))) + for (i <- 0 until requesterCount) { + rsp(i).valid := io.rsp.valid && io.rsp.bits.tgtId === (i + 1).U + rsp(i).bits := io.rsp.bits + dat(i).valid := io.dat.valid && io.dat.bits.tgtId === (i + 1).U + dat(i).bits := io.dat.bits + snp(i).valid := io.snp.valid && io.snp.bits.targetNode === (i + 1).U + snp(i).bits := io.snp.bits.flit + creditLink(rsp(i), io.requesters(i).rxRsp) + creditLink(dat(i), io.requesters(i).rxDat) + creditLink(snp(i), io.requesters(i).snp) + } + io.rsp.ready := (0 until requesterCount).map(i => rsp(i).ready && io.rsp.bits.tgtId === (i + 1).U).reduce(_ || _) + io.dat.ready := (0 until requesterCount).map(i => dat(i).ready && io.dat.bits.tgtId === (i + 1).U).reduce(_ || _) + io.snp.ready := (0 until requesterCount).map(i => snp(i).ready && io.snp.bits.targetNode === (i + 1).U).reduce(_ || _) + when(io.rsp.valid)(assert( + io.rsp.bits.tgtId >= 1.U && io.rsp.bits.tgtId <= requesterCount.U, + "CacheSystem response target is not a requester" + )) + when(io.dat.valid)(assert( + io.dat.bits.tgtId >= 1.U && io.dat.bits.tgtId <= requesterCount.U, + "CacheSystem data target is not a requester" + )) + when(io.snp.valid)(assert( + io.snp.bits.targetNode >= 1.U && io.snp.bits.targetNode <= requesterCount.U, + "CacheSystem snoop target is not a requester" + )) +} diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Flit.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/Flit.scala similarity index 97% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/Flit.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/Flit.scala index c01f5f3a0..fcbe67a58 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Flit.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/Flit.scala @@ -1,4 +1,7 @@ -package memcore.bus.chi +package framework.ip.bus.chi + +import buckyball.config.ChiHardware +import upickle.default.{macroRW, ReadWriter} import chisel3._ import chisel3.util.Cat @@ -30,6 +33,12 @@ case class Params( val commonSecIdBits = 0 } +object Params { + implicit val rw: ReadWriter[Params] = macroRW + + def fromProto(p: ChiHardware): Params = Params(p.getNodeIdBits, p.getAddressBits, p.getDataBits) +} + abstract class Flit extends Bundle { def fieldsLSB: Seq[UInt] diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/HomeMapping.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/HomeMapping.scala similarity index 93% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/HomeMapping.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/HomeMapping.scala index fc6c8b382..a12c79628 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/HomeMapping.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/HomeMapping.scala @@ -1,4 +1,4 @@ -package memcore.bus.chi +package framework.ip.bus.chi import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/RequestRetry.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/RequestRetry.scala similarity index 95% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/RequestRetry.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/RequestRetry.scala index 72462ec87..d98795e8c 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/RequestRetry.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/RequestRetry.scala @@ -1,6 +1,9 @@ -package memcore.bus.chi +package framework.ip.bus.chi -import memcore.memory.queue.Queue +import framework.lib.arbiter.{RRArbiter} +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.lib.queue.Queue import chisel3._ import chisel3.util._ @@ -41,8 +44,8 @@ class RequestRetry(p: Params, nodeId: Int, records: Int = 4) extends Module { def liveTxn(id: UInt) = VecInit((0 until records).map(i => state(i) =/= free && requests(i).txnId === id)) io.pending := PopCount(state.map(_ =/= free)) - val arb = Module(new RRArbiter(new RequestFlit(p), records + 1)) - val output = Module(new Queue(new RequestFlit(p), 2, pipe = true)) + val arb = Instantiate(new RRArbiter(new RequestFlit(p), records + 1)) + val output = Instantiate(new Queue(new RequestFlit(p), 2, true)) output.io.enq <> arb.io.out io.reqOut <> output.io.deq val initial = arb.io.in(0) diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/RequesterPort.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/RequesterPort.scala similarity index 94% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/RequesterPort.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/RequesterPort.scala index 1e425f5d3..2cf160e75 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/RequesterPort.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/RequesterPort.scala @@ -1,4 +1,4 @@ -package memcore.bus.chi +package framework.ip.bus.chi import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Rx.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/Rx.scala similarity index 98% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/Rx.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/Rx.scala index d67b5cf2a..2ae50ebaa 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Rx.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/Rx.scala @@ -1,4 +1,4 @@ -package memcore.bus.chi +package framework.ip.bus.chi import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Tx.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/Tx.scala similarity index 98% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/Tx.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/Tx.scala index 9b1074a1e..f29e6d972 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Tx.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/Tx.scala @@ -1,4 +1,4 @@ -package memcore.bus.chi +package framework.ip.bus.chi import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/hnf/ChiHome.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/hnf/ChiHome.scala similarity index 98% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/hnf/ChiHome.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/hnf/ChiHome.scala index c8d321537..e4820d140 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/hnf/ChiHome.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/hnf/ChiHome.scala @@ -1,10 +1,10 @@ -package memcore.bus.chi.hnf +package framework.ip.bus.chi.hnf import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.chi._ -import memcore.bus.chi.snf.{LineRequest, LineResponse} +import framework.ip.bus.chi._ +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} // A finite-address, precise-directory Home. One serialized transaction per Home; // CompAck closes a read grant before another request can observe its directory state. diff --git a/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/AxiCache.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/AxiCache.scala new file mode 100644 index 000000000..d7eaffe66 --- /dev/null +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/AxiCache.scala @@ -0,0 +1,131 @@ +package framework.ip.bus.chi.rnf + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.ip.bus.axi4 +import framework.ip.bus.chi.RequesterPort + +/** + * One independent AXI read and write burst, served through a coherent cache. + * AXI beats are ordinary memory accesses; 128-bit writes are not atomic. + */ +@instantiable +class AxiCache(p: axi4.Params, cacheParam: RnfParams) extends Module { + require(p.dataBits == 128 && p.addressBits == cacheParam.chi.addressBits) + require(cacheParam.lineResult && !cacheParam.probe) + + @public + val io = IO(new Bundle { + val axi = Flipped(new axi4.Port(p)) + val chi = new RequesterPort(cacheParam.chi) + }) + + val cache = Instantiate(new BankedChiCache(cacheParam)) + io.chi <> cache.io.chi + + val reading = RegInit(false.B) + val read = Reg(new axi4.Address(p)) + val readBeat = RegInit(0.U(8.W)) + val readValid = RegInit(false.B) + val readData = Reg(UInt(128.W)) + val readError = Reg(Bool()) + io.axi.ar.ready := !reading + when(io.axi.ar.fire) { + assert(io.axi.ar.bits.size <= 4.U && (io.axi.ar.bits.burst === 1.U || io.axi.ar.bits.len === 0.U)) + assert((io.axi.ar.bits.addr & ((1.U(17.W) << io.axi.ar.bits.size) - 1.U)) === 0.U) + assert(!io.axi.ar.bits.lock && io.axi.ar.bits.region === 0.U && io.axi.ar.bits.cache === 15.U) + read := io.axi.ar.bits + reading := true.B + readBeat := 0.U + } + io.axi.r.valid := readValid + io.axi.r.bits.id := read.id + io.axi.r.bits.data := readData + io.axi.r.bits.resp := Mux(readError, 2.U, 0.U) + io.axi.r.bits.last := readBeat === read.len + when(io.axi.r.fire) { + readValid := false.B + read.addr := read.addr + (1.U << read.size) + readBeat := readBeat + 1.U + when(io.axi.r.bits.last)(reading := false.B) + } + + val writing = RegInit(false.B) + val write = Reg(new axi4.Address(p)) + val writeBeat = RegInit(0.U(8.W)) + val writeValid = RegInit(false.B) + val writeData = Reg(UInt(128.W)) + val writeMask = Reg(UInt(16.W)) + val writeHalf = RegInit(false.B) + val writeError = RegInit(false.B) + val responseValid = RegInit(false.B) + io.axi.aw.ready := !writing && !responseValid + when(io.axi.aw.fire) { + assert(io.axi.aw.bits.size <= 4.U && (io.axi.aw.bits.burst === 1.U || io.axi.aw.bits.len === 0.U)) + assert((io.axi.aw.bits.addr & ((1.U(17.W) << io.axi.aw.bits.size) - 1.U)) === 0.U) + assert(!io.axi.aw.bits.lock && io.axi.aw.bits.region === 0.U && io.axi.aw.bits.cache === 15.U) + write := io.axi.aw.bits + writing := true.B + writeBeat := 0.U + writeError := false.B + } + io.axi.w.ready := writing && !writeValid + when(io.axi.w.fire) { + assert(io.axi.w.bits.last === (writeBeat === write.len)) + val allowed = (("hffff".U(16.W) >> (16.U - (1.U << write.size))) << write.addr(3, 0))(15, 0) + assert((io.axi.w.bits.strb & ~allowed) === 0.U) + writeData := io.axi.w.bits.data + writeMask := io.axi.w.bits.strb + writeValid := true.B + writeHalf := false.B + } + io.axi.b.valid := responseValid + io.axi.b.bits.id := write.id + io.axi.b.bits.resp := Mux(writeError, 2.U, 0.U) + when(io.axi.b.fire)(responseValid := false.B) + + val occupied = RegInit(false.B) + val ownerWrite = Reg(Bool()) + val preferWrite = RegInit(false.B) + val wantsRead = reading && !readValid + val wantsWrite = writeValid + val selectWrite = wantsWrite && (!wantsRead || preferWrite) + val mask = Mux(writeHalf, writeMask(15, 8), writeMask(7, 0)) + val skip = !occupied && selectWrite && !mask.orR + cache.io.access.valid := !occupied && (wantsRead || wantsWrite) && !skip + cache.io.access.bits := 0.U.asTypeOf(cache.io.access.bits) + cache.io.access.bits.addr := Mux( + selectWrite, + Cat(write.addr(p.addressBits - 1, 4), writeHalf, 0.U(3.W)), + Cat(read.addr(p.addressBits - 1, 3), 0.U(3.W)) + ) + cache.io.access.bits.write := selectWrite + cache.io.access.bits.data := Mux(writeHalf, writeData(127, 64), writeData(63, 0)) + cache.io.access.bits.mask := Mux(selectWrite, mask, "hff".U) + when(cache.io.access.fire) { + occupied := true.B + ownerWrite := selectWrite + preferWrite := !selectWrite + } + cache.io.result.ready := occupied + when(cache.io.result.fire) { + occupied := false.B + when(ownerWrite) { + writeError := writeError || cache.io.result.bits.error + }.otherwise { + readData := (cache.io.result.bits.line >> (read.addr(5, 4) << 7))(127, 0) + readError := cache.io.result.bits.error + readValid := true.B + } + } + when(skip || (cache.io.result.fire && ownerWrite)) { + when(writeHalf) { + writeValid := false.B + write.addr := write.addr + (1.U << write.size) + writeBeat := writeBeat + 1.U + when(writeBeat === write.len) { writing := false.B; responseValid := true.B } + }.otherwise(writeHalf := true.B) + when(skip)(preferWrite := false.B) + } +} diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/BankedChiCache.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/BankedChiCache.scala similarity index 84% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/BankedChiCache.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/BankedChiCache.scala index b390bf954..a99cbaefc 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/BankedChiCache.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/BankedChiCache.scala @@ -1,9 +1,15 @@ -package memcore.bus.chi.rnf +package framework.ip.bus.chi.rnf + +import framework.lib.arbiter.{RRArbiter} + +import buckyball.config.RnfHardware + +import framework.ip.bus.chi.rnf.RnfParams import chisel3._ import chisel3.util._ -import memcore.bus.chi._ -import memcore.memory.queue.Queue +import framework.ip.bus.chi._ +import framework.lib.queue.Queue import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} // Each bank executes one demand operation while serving snoops independently. @@ -20,8 +26,24 @@ case class RnfParams( val resultLineBits: Int = if (lineResult) 512 else 0 } +object RnfParams { + + def fromProto(p: RnfHardware, requester: Params): RnfParams = RnfParams( + requester, + p.getNodeId, + p.getCacheLines, + p.getHomeId, + p.getHomeCount, + p.getBanks, + p.getLineResult, + p.getProbe + ) + +} + @instantiable -class BankedChiCache(config: RnfParams) extends Module { +class BankedChiCache(b: RnfParams) extends Module { + val config = b val p = config.chi val nodeId = config.nodeId val cacheLines = config.cacheLines @@ -50,7 +72,7 @@ class BankedChiCache(config: RnfParams) extends Module { def bankOf(address: UInt): UInt = if (banks == 1) 0.U(0.W) else address(log2Ceil(banks) + 5, 6) val selected = bankOf(io.access.bits.addr) - val retirement = Module(new Queue(UInt(bankBits.W), banks, pipe = true)) + val retirement = Instantiate(new Queue(UInt(bankBits.W), banks, true)) val serial = io.access.bits.atomic =/= CacheAtomic.None.U val serialActive = RegInit(false.B) val serialSC = RegInit(false.B) @@ -108,39 +130,15 @@ class BankedChiCache(config: RnfParams) extends Module { val completedSC = io.result.fire && serialActive && serialSC for (cache <- caches) { cache.io.dropReservation := newLR || completedSC } - val reqArb = Module(new RRArbiter(new RequestFlit(p), banks) { + val reqArb = Instantiate(new RRArbiter(new RequestFlit(p), banks)) - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(banks)).W)) - when(io.out.fire)(pointer := io.chosen) - pointer - } - - }) + val rspArb = Instantiate(new RRArbiter(new ResponseFlit(p), banks)) - val rspArb = Module(new RRArbiter(new ResponseFlit(p), banks) { - - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(banks)).W)) - when(io.out.fire)(pointer := io.chosen) - pointer - } - - }) - - val datArb = Module(new RRArbiter(new DataFlit(p), banks) { - - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(banks)).W)) - when(io.out.fire)(pointer := io.chosen) - pointer - } - - }) + val datArb = Instantiate(new RRArbiter(new DataFlit(p), banks)) - val reqQueue = Module(new Queue(new RequestFlit(p), 2, pipe = true)) - val rspQueue = Module(new Queue(new ResponseFlit(p), 2, pipe = true)) - val datQueue = Module(new Queue(new DataFlit(p), 2, pipe = true)) + val reqQueue = Instantiate(new Queue(new RequestFlit(p), 2, true)) + val rspQueue = Instantiate(new Queue(new ResponseFlit(p), 2, true)) + val datQueue = Instantiate(new Queue(new DataFlit(p), 2, true)) for (i <- 0 until banks) { reqArb.io.in(i) <> caches(i).io.chi.req rspArb.io.in(i) <> caches(i).io.chi.txRsp diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/CachePort.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/CachePort.scala similarity index 95% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/CachePort.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/CachePort.scala index c1300502a..431694453 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/CachePort.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/CachePort.scala @@ -1,7 +1,7 @@ -package memcore.bus.chi.rnf +package framework.ip.bus.chi.rnf import chisel3._ -import memcore.bus.chi.Params +import framework.ip.bus.chi.Params class CacheAccess(p: Params) extends Bundle { val addr = UInt(p.addressBits.W) diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/ChiCache.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/ChiCache.scala similarity index 99% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/ChiCache.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/ChiCache.scala index e29de59e7..430f49219 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/rnf/ChiCache.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/ChiCache.scala @@ -1,8 +1,8 @@ -package memcore.bus.chi.rnf +package framework.ip.bus.chi.rnf import chisel3._ import chisel3.util._ -import memcore.bus.chi._ +import framework.ip.bus.chi._ import chisel3.experimental.hierarchy.{instantiable, public} // Direct-mapped write-back coherent cache agent. The snoop engine is independent diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/snf/MemoryNode.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/snf/MemoryNode.scala similarity index 92% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/snf/MemoryNode.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/snf/MemoryNode.scala index 831fa8086..3520be259 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/snf/MemoryNode.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/snf/MemoryNode.scala @@ -1,9 +1,12 @@ -package memcore.bus.chi.snf +package framework.ip.bus.chi.snf + +import framework.lib.arbiter.{RRArbiter} +import chisel3.experimental.hierarchy.{Instance, Instantiate} import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.chi._ +import framework.ip.bus.chi._ class LineRequest(p: Params) extends Bundle { val id = UInt(p.txnIdBits.W) @@ -121,15 +124,7 @@ class MemoryNode(p: Params, nodeId: Int = 1, slots: Int = 8) extends Module { } } - val memArb = Module(new RRArbiter(new LineRequest(p), slots) { - - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(slots)).W)) - when(this.io.out.fire)(pointer := this.io.chosen) - pointer - } - - }) + val memArb = Instantiate(new RRArbiter(new LineRequest(p), slots)) for (i <- 0 until slots) { memArb.io.in(i).valid := state(i) === issue @@ -154,15 +149,7 @@ class MemoryNode(p: Params, nodeId: Int = 1, slots: Int = 8) extends Module { } } - val rspArb = Module(new RRArbiter(new ResponseFlit(p), slots) { - - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(slots)).W)) - when(this.io.out.fire)(pointer := this.io.chosen) - pointer - } - - }) + val rspArb = Instantiate(new RRArbiter(new ResponseFlit(p), slots)) for (i <- 0 until slots) { val r = rspArb.io.in(i) @@ -180,15 +167,7 @@ class MemoryNode(p: Params, nodeId: Int = 1, slots: Int = 8) extends Module { } io.rsp <> rspArb.io.out - val datArb = Module(new RRArbiter(new DataFlit(p), slots) { - - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(slots)).W)) - when(this.io.out.fire)(pointer := this.io.chosen) - pointer - } - - }) + val datArb = Instantiate(new RRArbiter(new DataFlit(p), slots)) for (i <- 0 until slots) { val d = datArb.io.in(i) diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/snf/SramEndpoint.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/snf/SramEndpoint.scala similarity index 97% rename from arch/src/main/scala/framework/mem-core/chi/src/main/scala/snf/SramEndpoint.scala rename to arch/src/main/scala/framework/ip/chi/src/main/scala/snf/SramEndpoint.scala index 20fcc4568..1db3c8811 100644 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/snf/SramEndpoint.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/snf/SramEndpoint.scala @@ -1,9 +1,9 @@ -package memcore.bus.chi.snf +package framework.ip.bus.chi.snf import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi._ +import framework.ip.bus.chi._ class SnPort(p: Params) extends Bundle { val rxReq = Flipped(new Channel((new RequestFlit(p)).flitWidth)) diff --git a/arch/src/main/scala/framework/ip/coherence/build.sc b/arch/src/main/scala/framework/ip/coherence/build.sc new file mode 100644 index 000000000..4d02e63b0 --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/build.sc @@ -0,0 +1,25 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.memory.coherence.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("configs/CoherenceParams.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Coherence.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Directory.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Interface.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Mshr.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/coherence/configs/CoherenceParams.scala b/arch/src/main/scala/framework/ip/coherence/configs/CoherenceParams.scala new file mode 100644 index 000000000..8ea86be8a --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/configs/CoherenceParams.scala @@ -0,0 +1,34 @@ +package framework.ip.memory.coherence.configs + +import buckyball.config.HomeHardware + +import chisel3.util.log2Ceil +import framework.ip.bus.chi.Params +import framework.ip.memory.cache.configs.CacheParams + +case class CoherenceParams( + chi: Params, + cache: CacheParams, + agents: Int, + mshrEntries: Int, + homeId: Int) { + require(cache.lineBytes == 64 && cache.addressBits == chi.addressBits) + require(agents >= 1 && agents < homeId && BigInt(homeId) < (BigInt(1) << chi.nodeIdBits)) + require(mshrEntries >= 1 && BigInt(mshrEntries) <= (BigInt(1) << chi.txnIdBits)) + require(BigInt(mshrEntries) <= (BigInt(1) << cache.idBits)) + val slotBits = math.max(1, log2Ceil(mshrEntries)) + val lineAddressBits = chi.addressBits - cache.offsetBits +} + +object CoherenceParams { + + def fromProto(p: HomeHardware): CoherenceParams = + CoherenceParams( + Params.fromProto(p.getRequesters), + CacheParams.fromProto(p.getCache), + p.getAgents, + p.getMshrEntries, + p.getHomeId + ) + +} diff --git a/arch/src/main/scala/framework/mem-core/coherence/configs/default.toml b/arch/src/main/scala/framework/ip/coherence/configs/default.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/coherence/configs/default.toml rename to arch/src/main/scala/framework/ip/coherence/configs/default.toml diff --git a/arch/src/main/scala/framework/ip/coherence/configs/requesters.toml b/arch/src/main/scala/framework/ip/coherence/configs/requesters.toml new file mode 100644 index 000000000..2c6ca35c4 --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/configs/requesters.toml @@ -0,0 +1,3 @@ +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 diff --git a/arch/src/main/scala/framework/ip/coherence/configs/uvm.toml b/arch/src/main/scala/framework/ip/coherence/configs/uvm.toml new file mode 100644 index 000000000..a0b6176d7 --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/configs/uvm.toml @@ -0,0 +1,14 @@ +[parameters] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 +agents = 2 +mshrEntries = 4 +homeId = 64 +cacheSets = 4 +cacheWays = 2 +cacheIdBits = 8 +metadataBits = 4 +responseDepth = 2 + +[defines] diff --git a/arch/src/main/scala/framework/ip/coherence/src/main/resources/filelist.f b/arch/src/main/scala/framework/ip/coherence/src/main/resources/filelist.f new file mode 100644 index 000000000..e3b3935b9 --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/src/main/resources/filelist.f @@ -0,0 +1,10 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@RTL@ ++incdir+@RESOURCES@ ++incdir+@VERIFY@/uvm/src/protocol/chi +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/stream_if.sv +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Coherence/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/coherence/src/main/resources/interface.sv b/arch/src/main/scala/framework/ip/coherence/src/main/resources/interface.sv new file mode 100644 index 000000000..113cb1b8f --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/src/main/resources/interface.sv @@ -0,0 +1,12 @@ +`include "coherence_config.svh" +interface coherence_control_if ( + input logic clock +); + logic reset; + logic [`COH_OUTSTANDING_BITS-1:0] outstanding; + clocking cb @(posedge clock); + default input #1step output #0; + input outstanding; + output reset; + endclocking +endinterface diff --git a/arch/src/main/scala/framework/ip/coherence/src/main/resources/package.sv b/arch/src/main/scala/framework/ip/coherence/src/main/resources/package.sv new file mode 100644 index 000000000..e18a425a7 --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/src/main/resources/package.sv @@ -0,0 +1,5 @@ +`include "coherence_config.svh" +`define CF(K, F) `COH_``K``_``F``_OFFSET+:`COH_``K``_``F``_WIDTH +package coherence_pkg; + `include "coherence/body.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/coherence/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/coherence/src/main/resources/suite.toml new file mode 100644 index 000000000..206e15813 --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "../../../../../../../verify/uvm/src/protocol/chi/coherence/model/Cargo.toml" +library = "libcoherence_reference_model" + +[[tests]] +functional_groups = ["coherence_pkg::system_model::operations", "coherence_pkg::system_model::occupancy"] +name = "coherence" +top = "coherence_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 300000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence_tb.sv b/arch/src/main/scala/framework/ip/coherence/src/main/resources/tb.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence_tb.sv rename to arch/src/main/scala/framework/ip/coherence/src/main/resources/tb.sv diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Coherence.scala b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Coherence.scala similarity index 89% rename from arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Coherence.scala rename to arch/src/main/scala/framework/ip/coherence/src/main/scala/Coherence.scala index af313240e..7fd78048f 100644 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Coherence.scala +++ b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Coherence.scala @@ -1,14 +1,16 @@ -package memcore.memory.coherence +package framework.ip.memory.coherence -import memcore.memory.queue.Queue +import framework.lib.arbiter.{RRArbiter} + +import framework.lib.queue.Queue import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi._ -import memcore.bus.chi.snf.LineRequest -import memcore.memory.cache.{Cache, CacheOp, CacheRequest} -import memcore.memory.coherence.configs.CoherenceParams +import framework.ip.bus.chi._ +import framework.ip.bus.chi.snf.LineRequest +import framework.ip.memory.cache.{Cache, CacheOp, CacheRequest} +import framework.ip.memory.coherence.configs.CoherenceParams @instantiable class Coherence(p: CoherenceParams) extends Module { @@ -71,64 +73,59 @@ class Coherence(p: CoherenceParams) extends Module { io.req.ready := mshr.io.allocate.ready io.outstanding := mshr.io.outstanding when(io.req.fire) { - val r = io.req.bits - val read = + val r = io.req.bits + val read = r.opcode === Opcode.ReadShared.U || r.opcode === Opcode.ReadNotSharedDirty.U || r.opcode === Opcode.ReadUnique.U + val permission = r.opcode === Opcode.MakeUnique.U assert(r.tgtId === p.homeId.U && r.srcId >= 1.U && r.srcId <= p.agents.U, "Unknown CPU or Home") assert( - read || r.opcode === Opcode.Evict.U || r.opcode === Opcode.WriteBackFull.U || + read || permission || r.opcode === Opcode.Evict.U || r.opcode === Opcode.WriteBackFull.U || r.opcode === Opcode.CleanShared.U || r.opcode === Opcode.CleanInvalid.U || r.opcode === Opcode.MakeInvalid.U, "Unsupported coherence operation" ) assert( r.size === 6.U && r.addr(5, 0) === 0.U && r.order === 0.U && r.pas === 0.U && - r.returnNid === 0.U && r.returnTxnId === 0.U && r.memAttr === "b1100".U && r.snpAttr === 1.U && + r.returnNid === 0.U && r.returnTxnId === 0.U && r.memAttr(2) && !r.memAttr(1) && r.snpAttr === 1.U && r.allowRetry === 1.U && r.exclSnoopMe === 0.U && r.pCrdType === 0.U && r.stashNidValidEndian === 0.U && r.multiReq === 0.U && r.tagOp === 0.U, "Unsupported coherent access attributes" ) - assert(r.expCompAck === read.asUInt, "Coherent read requires CompAck") - val i = mshr.io.allocatedId + assert(r.expCompAck === (read || permission).asUInt, "Coherent grants require CompAck") + val i = mshr.io.allocatedId state(i) := lookupReq failed(i) := false.B sent(i) := 0.U seen(i) := 0.U } - private def arbitrate[T <: Data](gen: T): RRArbiter[T] = Module(new RRArbiter(gen, n) { - - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(this.n)).W)) - when(this.io.out.fire)(pointer := this.io.chosen) - pointer - } - - }) + private def arbitrate[T <: Data](gen: T): Instance[RRArbiter[T]] = Instantiate(new RRArbiter(gen, n)) val cacheArb = arbitrate(new CacheRequest(p.cache)) - val cacheQueue = Module(new Queue(new CacheRequest(p.cache), 2, pipe = true)) + val cacheQueue = Instantiate(new Queue(new CacheRequest(p.cache), 2, true)) cacheQueue.io.enq <> cacheArb.io.out cache.io.request <> cacheQueue.io.deq cache.io.response.ready := true.B val memArb = arbitrate(new LineRequest(c)) - val memQueue = Module(new Queue(new LineRequest(c), 2, pipe = true)) + val memQueue = Instantiate(new Queue(new LineRequest(c), 2, true)) memQueue.io.enq <> memArb.io.out io.memoryReq <> memQueue.io.deq io.memoryResp.ready := true.B val snpArb = arbitrate(new DirectedSnoop(c)) - val snpQueue = Module(new Queue(new DirectedSnoop(c), 2, pipe = true)) + val snpQueue = Instantiate(new Queue(new DirectedSnoop(c), 2, true)) snpQueue.io.enq <> snpArb.io.out io.snp <> snpQueue.io.deq val rspArb = arbitrate(new ResponseFlit(c)) - val rspQueue = Module(new Queue(new ResponseFlit(c), 2, pipe = true)) + val rspQueue = Instantiate(new Queue(new ResponseFlit(c), 2, true)) rspQueue.io.enq <> rspArb.io.out io.rsp <> rspQueue.io.deq val datArb = arbitrate(new DataFlit(c)) - val datQueue = Module(new Queue(new DataFlit(c), 2, pipe = true)) + val datQueue = Instantiate(new Queue(new DataFlit(c), 2, true)) datQueue.io.enq <> datArb.io.out io.dat <> datQueue.io.deq // Caller TxnID ends at its final response. The slot/DBID/set stays owned until full protocol completion. mshr.io.releaseCaller := VecInit((0 until n).map { i => (io.rsp.fire && io.rsp.bits.opcode === Opcode.CompDBIDResp.U && io.rsp.bits.dbid === i.U) || + (io.rsp.fire && io.rsp.bits.opcode === Opcode.Comp.U && io.rsp.bits.dbid === i.U && + entries(i).request.opcode === Opcode.MakeUnique.U) || (io.dat.fire && io.dat.bits.opcode === Opcode.CompData.U && io.dat.bits.dbid === i.U && io.dat.bits.dataId === ((c.beatsPerLine - 1) * c.dataBits / 128).U) }).asUInt @@ -158,7 +155,9 @@ class Coherence(p: CoherenceParams) extends Module { update.bits.address := directory.io.read(i) update.bits.entry := directory.io.entries(i) val mask = UIntToOH(r.srcId - 1.U, p.agents) - val exclusive = r.opcode === Opcode.ReadUnique.U + val permission = r.opcode === Opcode.MakeUnique.U + val exclusive = r.opcode === Opcode.ReadUnique.U || permission + val grant = Mux(permission, complete, grantData) val cleanShared = r.opcode === Opcode.CleanShared.U val cleanInvalid = r.opcode === Opcode.CleanInvalid.U val makeInvalid = r.opcode === Opcode.MakeInvalid.U @@ -218,7 +217,7 @@ class Coherence(p: CoherenceParams) extends Module { .otherwise { pending(i) := Mux( !result.hit || maintenance || exclusive || owned, - holders, + holders & Mux(permission && result.hit, ~mask, Fill(p.agents, 1.U(1.W))), 0.U ) state(i) := chooseSnoop @@ -233,14 +232,14 @@ class Coherence(p: CoherenceParams) extends Module { update.bits.entry.sharers := 0.U update.bits.entry.unique := false.B sent(i) := 0.U - state(i) := grantData + state(i) := grant }.elsewhen(state(i) === restoreWait) { sent(i) := 0.U - state(i) := Mux(maintenance, complete, grantData) + state(i) := Mux(maintenance, complete, grant) }.otherwise { assert(state(i) === updateWait, "Unexpected Cache response phase") sent(i) := 0.U - state(i) := Mux(writeback, finished, Mux(maintenance, complete, grantData)) + state(i) := Mux(writeback, finished, Mux(maintenance, complete, grant)) } } @@ -377,7 +376,7 @@ class Coherence(p: CoherenceParams) extends Module { failed(i) := io.memoryResp.bits.error payload(i) := io.memoryResp.bits.data.asTypeOf(payload(i)) sent(i) := 0.U - state(i) := Mux(io.memoryResp.bits.error, grantData, fillReq) + state(i) := Mux(io.memoryResp.bits.error, grant, fillReq) }.otherwise { when(io.memoryResp.bits.error) { failed(i) := true.B @@ -398,10 +397,16 @@ class Coherence(p: CoherenceParams) extends Module { rsp.bits.txnId := r.txnId rsp.bits.dbid := i.U rsp.bits.respErr := Mux(failed(i), 3.U, 0.U) + rsp.bits.resp := Mux(permission && !failed(i), CoherenceState.UC.U, CoherenceState.I.U) rsp.bits.opcode := Mux(state(i) === wbGrant, Opcode.CompDBIDResp.U, Opcode.Comp.U) when(rsp.fire) { seen(i) := 0.U - state(i) := Mux(state(i) === wbGrant, wbData, finished) + state(i) := Mux(state(i) === wbGrant, wbData, Mux(permission, waitAck, finished)) + when(permission && !failed(i)) { + update.valid := true.B + update.bits.entry.sharers := mask + update.bits.entry.unique := true.B + } } val dat = datArb.io.in(i) dat.valid := active && state(i) === grantData diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Directory.scala b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Directory.scala similarity index 94% rename from arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Directory.scala rename to arch/src/main/scala/framework/ip/coherence/src/main/scala/Directory.scala index 7b2b94d86..3540e2e40 100644 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Directory.scala +++ b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Directory.scala @@ -1,9 +1,9 @@ -package memcore.memory.coherence +package framework.ip.memory.coherence import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.memory.coherence.configs.CoherenceParams +import framework.ip.memory.coherence.configs.CoherenceParams class DirectoryEntry(p: CoherenceParams) extends Bundle { val sharers = UInt(p.agents.W) diff --git a/arch/src/main/scala/framework/ip/coherence/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Emit.scala new file mode 100644 index 000000000..f0b0bcb8c --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Emit.scala @@ -0,0 +1,127 @@ +package framework.ip.memory.coherence + +import java.nio.file.{Files, Paths} +import framework.ip.bus.chi._ +import framework.ip.memory.coherence.configs.CoherenceParams + +object Emit extends App { + val Array("--config", config, "--output-dir", directory) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val output = Paths.get(directory) + Files.createDirectories(output) + val chi = Params(values("nodeIdBits").num.toInt, values("addressBits").num.toInt, values("dataBits").num.toInt) + + val cache = framework.ip.memory.cache.configs.CacheParams( + chi.addressBits, + 64, + values("cacheSets").num.toInt, + values("cacheWays").num.toInt, + values("cacheIdBits").num.toInt, + values("metadataBits").num.toInt, + values("responseDepth").num.toInt + ) + + val p = + CoherenceParams(chi, cache, values("agents").num.toInt, values("mshrEntries").num.toInt, values("homeId").num.toInt) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Coherence(p), + args = Array("--target-dir", output.resolve("Coherence").toString, "--split-verilog") + ) + val definitions = scala.collection.mutable.ArrayBuffer[String]() + val ports = scala.collection.mutable.ArrayBuffer[String]() + + def fields(flit: Flit): Seq[(String, Int)] = flit.fieldsLSB.map { field => + flit.elements.find(_._2 eq field).get._1 -> field.getWidth + } + + def channel( + kind: String, + prefix: String, + instance: String, + fs: Seq[(String, Int)] + ): Unit = { + var offset = 0 + for ((name, width) <- fs if width > 0) { + definitions += s"`define COH_${kind}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define COH_${kind}_${name.toUpperCase}_WIDTH $width" + ports += s".$prefix$name($instance.bits[$offset +: $width])" + offset += width + } + definitions += s"`define COH_${kind}_WIDTH $offset" + } + + channel("REQ", "io_req_bits_", "source_if", fields(new RequestFlit(p.chi))) + channel("RSP", "io_rxRsp_bits_", "rx_rsp", fields(new ResponseFlit(p.chi))) + channel("DAT", "io_rxDat_bits_", "rx_dat", fields(new DataFlit(p.chi))) + channel("TXRSP", "io_rsp_bits_", "tx_rsp", fields(new ResponseFlit(p.chi))) + channel("TXDAT", "io_dat_bits_", "sink_if", fields(new DataFlit(p.chi))) + channel("SNP", "io_snp_bits_flit_", "tx_snp", fields(new SnoopFlit(p.chi))) + val snpWidth = (new SnoopFlit(p.chi)).flitWidth + ports += s".io_snp_bits_targetNode(tx_snp.bits[$snpWidth +: ${p.chi.nodeIdBits}])" + definitions += s"`define COH_SNP_TARGET_OFFSET $snpWidth" + definitions += s"`define COH_SNP_TARGET_WIDTH ${p.chi.nodeIdBits}" + definitions += s"`define COH_SNP_CHANNEL_WIDTH ${snpWidth + p.chi.nodeIdBits}" + channel( + "MEMREQ", + "io_memoryReq_bits_", + "mem_req", + Seq("id" -> p.chi.txnIdBits, "addr" -> p.chi.addressBits, "write" -> 1, "data" -> 512, "mask" -> 64) + ) + channel("MEMRESP", "io_memoryResp_bits_", "mem_resp", Seq("id" -> p.chi.txnIdBits, "data" -> 512, "error" -> 1)) + for ( + (port, instance) <- Seq( + "req" -> "source_if", + "rxRsp" -> "rx_rsp", + "rxDat" -> "rx_dat", + "rsp" -> "tx_rsp", + "dat" -> "sink_if", + "snp" -> "tx_snp", + "memoryReq" -> "mem_req", + "memoryResp" -> "mem_resp" + ) + ) { + ports += s".io_${port}_valid($instance.valid)" + ports += s".io_${port}_ready($instance.ready)" + } + for ( + (name, value) <- Seq( + "AGENTS" -> p.agents, + "MSHRS" -> p.mshrEntries, + "HOME" -> p.homeId, + "SETS" -> p.cache.sets, + "WAYS" -> p.cache.ways, + "BEATS" -> p.chi.beatsPerLine, + "DATA_BITS" -> p.chi.dataBits, + "OUTSTANDING_BITS" -> chisel3.util.log2Ceil(p.mshrEntries + 1), + "READ_SHARED" -> Opcode.ReadShared, + "READ_NSD" -> Opcode.ReadNotSharedDirty, + "READ_UNIQUE" -> Opcode.ReadUnique, + "MAKE_UNIQUE" -> Opcode.MakeUnique, + "EVICT" -> Opcode.Evict, + "WRITEBACK" -> Opcode.WriteBackFull, + "CLEAN_SHARED" -> Opcode.CleanShared, + "CLEAN_INVALID" -> Opcode.CleanInvalid, + "MAKE_INVALID" -> Opcode.MakeInvalid, + "COMP" -> Opcode.Comp, + "COMP_DBID" -> Opcode.CompDBIDResp, + "COMP_ACK" -> Opcode.CompAck, + "COMP_DATA" -> Opcode.CompData, + "SNP_RESP" -> Opcode.SnpResp, + "SNP_DATA" -> Opcode.SnpRespData, + "COPYBACK_DATA" -> Opcode.CopyBackWriteData, + "SNP_UNIQUE" -> Opcode.SnpUnique, + "SNP_INVALID" -> Opcode.SnpCleanInvalid, + "SNP_CLEAN_SHARED" -> Opcode.SnpCleanShared, + "SNP_MAKE_INVALID" -> Opcode.SnpMakeInvalid, + "SNP_SHARED" -> Opcode.SnpNotSharedDirty + ) + ) { + definitions += s"`define COH_$name $value" + } + Files.writeString( + output.resolve("coherence_config.svh"), + "`ifndef COHERENCE_CONFIG_SVH\n`define COHERENCE_CONFIG_SVH\n" + definitions.mkString("\n") + "\n`endif\n" + ) + Files.writeString(output.resolve("coherence_ports.svh"), ports.mkString(",\n") + "\n") + +} diff --git a/arch/src/main/scala/framework/ip/coherence/src/main/scala/Interface.scala b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Interface.scala new file mode 100644 index 000000000..2c850bb55 --- /dev/null +++ b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Interface.scala @@ -0,0 +1,19 @@ +package framework.ip.memory.coherence + +import chisel3._ +import chisel3.util._ +import framework.ip.bus.chi._ +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} +import framework.ip.memory.coherence.configs.CoherenceParams + +class CoherenceIO(p: CoherenceParams) extends Bundle { + val req = Flipped(Decoupled(new RequestFlit(p.chi))) + val rxRsp = Flipped(Decoupled(new ResponseFlit(p.chi))) + val rxDat = Flipped(Decoupled(new DataFlit(p.chi))) + val rsp = Decoupled(new ResponseFlit(p.chi)) + val dat = Decoupled(new DataFlit(p.chi)) + val snp = Decoupled(new DirectedSnoop(p.chi)) + val memoryReq = Decoupled(new LineRequest(p.chi)) + val memoryResp = Flipped(Decoupled(new LineResponse(p.chi))) + val outstanding = Output(UInt(log2Ceil(p.mshrEntries + 1).W)) +} diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Mshr.scala b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Mshr.scala similarity index 94% rename from arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Mshr.scala rename to arch/src/main/scala/framework/ip/coherence/src/main/scala/Mshr.scala index e55193839..fca25a7ed 100644 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Mshr.scala +++ b/arch/src/main/scala/framework/ip/coherence/src/main/scala/Mshr.scala @@ -1,10 +1,10 @@ -package memcore.memory.coherence +package framework.ip.memory.coherence import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.chi.RequestFlit -import memcore.memory.coherence.configs.CoherenceParams +import framework.ip.bus.chi.RequestFlit +import framework.ip.memory.coherence.configs.CoherenceParams class MshrAllocation(p: CoherenceParams) extends Bundle { val request = new RequestFlit(p.chi) diff --git a/arch/src/main/scala/framework/ip/interlock/build.sc b/arch/src/main/scala/framework/ip/interlock/build.sc new file mode 100644 index 000000000..07f0a08c6 --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/build.sc @@ -0,0 +1,23 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.memory.interlock.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Interface.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Interlock.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Params.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/interlock/configs/uvm.toml b/arch/src/main/scala/framework/ip/interlock/configs/uvm.toml new file mode 100644 index 000000000..9896ccd3d --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/configs/uvm.toml @@ -0,0 +1,8 @@ +[parameters] +entries = 4 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[defines] diff --git a/arch/src/main/scala/framework/ip/interlock/src/main/resources/filelist.f b/arch/src/main/scala/framework/ip/interlock/src/main/resources/filelist.f new file mode 100644 index 000000000..643c739e2 --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/src/main/resources/filelist.f @@ -0,0 +1,9 @@ ++incdir+@RTL@ ++incdir+@VERIFY@/uvm/src/protocol/interlock ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/interlock/types.svh +@VERIFY@/uvm/src/protocol/interlock/interface.sv +@VERIFY@/uvm/src/protocol/interlock/package.sv +-F @RTL@/Interlock/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/interlock/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/interlock/src/main/resources/suite.toml new file mode 100644 index 000000000..5864a21c0 --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/src/main/resources/suite.toml @@ -0,0 +1,343 @@ +[model] +manifest = "../../../../../../../verify/uvm/src/protocol/interlock/model/Cargo.toml" +library = "libinterlock_reference" + +[[tests]] +functional_groups = ["interlock_pkg::ownership_model::activity", "interlock_pkg::ownership_model::maintenance_kind", "interlock_pkg::ownership_model::probes"] +name = "interlock" +top = "interlock_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "interlock_bad_cancel_info" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cancel_info"] +assertion = "Interlock cancel conflicts with the same command" + +[[tests]] +functional_groups = [] +name = "interlock_bad_cancel_sealed" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cancel_sealed"] +assertion = "Interlock cancel requires an unsealed reservation" + +[[tests]] +functional_groups = [] +name = "interlock_bad_cancel_unknown" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cancel_unknown"] +assertion = "Interlock cancel ID has no reservation" + +[[tests]] +functional_groups = [] +name = "interlock_bad_capacity" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=capacity"] +assertion = "Interlock range list must seal within capacity" + +[[tests]] +functional_groups = [] +name = "interlock_bad_cpu_alignment" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cpu_alignment"] +assertion = "Interlock CPU access must be naturally aligned" + +[[tests]] +functional_groups = [] +name = "interlock_bad_cpu_size" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=cpu_size"] +assertion = "Interlock CPU size must be 1, 2, 4 or 8 bytes" + +[[tests]] +functional_groups = [] +name = "interlock_bad_direction" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=direction"] +assertion = "Interlock ranges must have one access direction" + +[[tests]] +functional_groups = [] +name = "interlock_bad_dma_failed" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=dma_failed"] +assertion = "Interlock DMA failed" + +[[tests]] +functional_groups = [] +name = "interlock_bad_dma_ungranted" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=dma_ungranted"] +assertion = "Interlock DMA done requires a granted access" + +[[tests]] +functional_groups = [] +name = "interlock_bad_dma_unknown" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=dma_unknown"] +assertion = "Interlock DMA done tag has no reservation" + +[[tests]] +functional_groups = [] +name = "interlock_bad_duplicate" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=duplicate"] +assertion = "Interlock dispatch ID is already live" + +[[tests]] +functional_groups = [] +name = "interlock_bad_empty_range" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=empty_range"] +assertion = "Interlock memory range is empty" + +[[tests]] +functional_groups = [] +name = "interlock_bad_info_duplicate" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=info_duplicate"] +assertion = "Interlock access info is already declared" + +[[tests]] +functional_groups = [] +name = "interlock_bad_info_unknown" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=info_unknown"] +assertion = "Interlock access info ID has no reservation" + +[[tests]] +functional_groups = [] +name = "interlock_bad_maint_failed" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=maint_failed"] +assertion = "Interlock maintenance failed" + +[[tests]] +functional_groups = [] +name = "interlock_bad_maint_tag" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=maint_tag"] +assertion = "Interlock maintenance response tag mismatch" + +[[tests]] +functional_groups = [] +name = "interlock_bad_maint_unknown" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=maint_unknown"] +assertion = "Interlock maintenance response has no request" + +[[tests]] +functional_groups = [] +name = "interlock_bad_no_memory_seal" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=no_memory_seal"] +assertion = "Interlock no-memory declaration must seal an empty command" + +[[tests]] +functional_groups = [] +name = "interlock_bad_overflow_range" +top = "interlock_tb" +test = "invalid_test" +filelist = "filelist.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "assertion" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = ["+CASE=overflow_range"] +assertion = "Interlock memory range exceeds physical width" diff --git a/arch/src/main/scala/framework/ip/interlock/src/main/resources/tb.sv b/arch/src/main/scala/framework/ip/interlock/src/main/resources/tb.sv new file mode 100644 index 000000000..33d8a66c0 --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/src/main/resources/tb.sv @@ -0,0 +1,16 @@ +module interlock_tb; + import uvm_pkg::*; + import interlock_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + interlock_if bus (clock); + Interlock dut ( + .clock(clock), + .reset(bus.reset), + `include "ports.svh" + ); + initial begin + uvm_config_db#(virtual interlock_if)::set(null, "uvm_test_top.*", "vif", bus); + run_test(); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/interlock/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Emit.scala new file mode 100644 index 000000000..7a0554b66 --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Emit.scala @@ -0,0 +1,70 @@ +package framework.ip.memory.interlock + +import chisel3._ +import java.nio.file.{Files, Paths} + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + + val p = Params( + values("entries").num.toInt, + values("addressBits").num.toInt, + values("idBits").num.toInt, + values("lineBytes").num.toInt, + values("maxRanges").num.toInt + ) + + Files.createDirectories(Paths.get(output)) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Interlock(p), + args = Array("--target-dir", s"$output/Interlock", "--split-verilog") + ) + + val definitions = scala.collection.mutable.ArrayBuffer[String]( + s"`define INTERLOCK_ENTRIES ${p.entries}", + s"`define INTERLOCK_ADDRESS_BITS ${p.addressBits}", + s"`define INTERLOCK_ID_BITS ${p.idBits}", + s"`define INTERLOCK_LINE_BYTES ${p.lineBytes}", + s"`define INTERLOCK_MAX_RANGES ${p.maxRanges}" + ) + + val ports = scala.collection.mutable.ArrayBuffer[String]() + + def channel( + kind: String, + prefix: String, + signal: String, + data: Bundle, + decoupled: Boolean = true + ): Unit = { + var offset = 0 + for ((name, field) <- data.elements.toSeq) { + val width = field.getWidth + definitions += s"`define INTERLOCK_${kind}_${name.toUpperCase}_OFFSET $offset" + definitions += s"`define INTERLOCK_${kind}_${name.toUpperCase}_WIDTH $width" + ports += s".$prefix$name(bus.signals.${signal}_bits[$offset +: $width])" + offset += width + } + definitions += s"`define INTERLOCK_${kind}_WIDTH $offset" + if (decoupled) { + val channelPrefix = prefix.stripSuffix("bits_") + ports += s".${channelPrefix}valid(bus.signals.${signal}_valid)" + ports += s".${channelPrefix}ready(bus.signals.${signal}_ready)" + } + } + + channel("DISPATCH", "io_dispatch_bits_", "dispatch", new Dispatch(p)) + channel("CANCEL", "io_cancel_bits_", "cancel", new Dispatch(p)) + channel("ACCESS_INFO", "io_accessInfo_bits_", "access_info", new AccessInfo(p)) + channel("MAINTENANCE", "io_maintenance_bits_", "maintenance", new Maintenance(p)) + channel("MAINTAINED", "io_maintained_bits_", "maintained", new Acknowledgement(p)) + channel("GRANT", "io_grant_bits_", "grant", new Tag(p)) + channel("DONE", "io_done_bits_", "done", new Acknowledgement(p)) + channel("COMPLETE", "io_complete_bits_", "complete", new Tag(p)) + channel("CPU_QUERY", "io_cpuQuery_", "cpu_query", new CpuQuery(p), decoupled = false) + ports += ".io_cpuAllow(bus.signals.cpu_allow)" + ports += ".io_cpuProbeAllow(bus.signals.cpu_probe_allow)" + Files.write(Paths.get(output, "parameters.svh"), (definitions.mkString("\n") + "\n").getBytes) + Files.write(Paths.get(output, "ports.svh"), ports.mkString(",\n").getBytes) +} diff --git a/arch/src/main/scala/framework/ip/interlock/src/main/scala/Interface.scala b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Interface.scala new file mode 100644 index 000000000..9d3c8fc0c --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Interface.scala @@ -0,0 +1,38 @@ +package framework.ip.memory.interlock + +import chisel3._ + +class Dispatch(p: Params) extends Bundle { val id = UInt(p.idBits.W) } + +class AccessInfo(p: Params) extends Bundle { + val id = UInt(p.idBits.W) + val hasMemory = Bool() + val base = UInt(p.addressBits.W) + val bytes = UInt((p.addressBits + 1).W) + val write = Bool() + val last = Bool() +} + +class CpuQuery(p: Params) extends Bundle { + val valid = Bool() + val paddr = UInt(p.addressBits.W) + val sizeLog2 = UInt(3.W) + val write = Bool() + val olderDispatchPending = Bool() +} + +object MaintenanceOp { val Clean = 0; val CleanInvalidate = 1; val Invalidate = 2 } + +class Maintenance(p: Params) extends Bundle { + val tag = UInt(p.idBits.W) + val firstLine = UInt(p.addressBits.W) + val lastLine = UInt(p.addressBits.W) + val op = UInt(2.W) +} + +class Tag(p: Params) extends Bundle { val tag = UInt(p.idBits.W) } + +class Acknowledgement(p: Params) extends Bundle { + val tag = UInt(p.idBits.W) + val ok = Bool() +} diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Interlock.scala b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Interlock.scala similarity index 96% rename from arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Interlock.scala rename to arch/src/main/scala/framework/ip/interlock/src/main/scala/Interlock.scala index 2dad4115c..d8722c9ec 100644 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Interlock.scala +++ b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Interlock.scala @@ -1,4 +1,4 @@ -package memcore.memory.interlock +package framework.ip.memory.interlock import chisel3._ import chisel3.util._ @@ -16,7 +16,7 @@ class Interlock(p: Params) extends Module { val cpuQuery = Input(new CpuQuery(p)) val cpuAllow = Output(Bool()) // cpuAllow without the same-cycle dispatch term, for a CPU access that cannot share its cycle - // with a dispatch (the access and the dispatching command occupy the same core stage). + // with a dispatch (the access and the dispatching command occupy the same thread stage). val cpuProbeAllow = Output(Bool()) val maintenance = Decoupled(new Maintenance(p)) val maintained = Flipped(Decoupled(new Acknowledgement(p))) @@ -47,7 +47,8 @@ class Interlock(p: Params) extends Module { MuxCase( 0.U(indexBits.W), (0 until p.entries).map { offset => - val index = (cursor + offset.U)(indexBits - 1, 0) + val sum = cursor +& offset.U + val index = Mux(sum >= p.entries.U, sum - p.entries.U, sum)(indexBits - 1, 0) candidates(index) -> index } ) @@ -179,11 +180,9 @@ class Interlock(p: Params) extends Module { maintenancePayload.tag := ids(index) maintenancePayload.firstLine := rangeFirst(index)(rangeCursor(index)) maintenancePayload.lastLine := rangeLast(index)(rangeCursor(index)) - maintenancePayload.op := Mux( - phase(index) === postReady, - MaintenanceOp.Invalidate.U, - Mux(writes(index), MaintenanceOp.CleanInvalidate.U, MaintenanceOp.Clean.U) - ) + // DMA may leave the newest bytes dirty in a coherent requester. Preserve them + // before releasing the range; discard-only invalidation would lose the write. + maintenancePayload.op := Mux(writes(index), MaintenanceOp.CleanInvalidate.U, MaintenanceOp.Clean.U) } io.maintenance.valid := maintenanceOffer && !protocolError io.maintenance.bits := maintenancePayload @@ -192,7 +191,7 @@ class Interlock(p: Params) extends Module { maintenanceBusy := true.B maintenanceOwner := maintenanceIndex maintenanceCursor := maintenanceIndex + 1.U - phase(maintenanceIndex) := Mux(maintenancePayload.op === MaintenanceOp.Invalidate.U, postWait, preWait) + phase(maintenanceIndex) := Mux(phase(maintenanceIndex) === postReady, postWait, preWait) } io.maintained.ready := maintenanceBusy && !protocolError when(io.maintained.valid && !maintenanceBusy && !io.maintenance.fire) { diff --git a/arch/src/main/scala/framework/ip/interlock/src/main/scala/Params.scala b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Params.scala new file mode 100644 index 000000000..fd5f1529c --- /dev/null +++ b/arch/src/main/scala/framework/ip/interlock/src/main/scala/Params.scala @@ -0,0 +1,24 @@ +package framework.ip.memory.interlock + +import buckyball.config.TrackingHardware + +case class Params( + entries: Int = 4, + addressBits: Int = 44, + idBits: Int = 8, + lineBytes: Int = 64, + maxRanges: Int = 8) { + require(entries >= 2) + require(addressBits >= 4 && idBits >= 1) + require(lineBytes >= 8 && (lineBytes & (lineBytes - 1)) == 0) + require(maxRanges >= 1 && (maxRanges & (maxRanges - 1)) == 0) + val rangeIndexBits: Int = math.max(1, Integer.numberOfTrailingZeros(maxRanges)) + val rangeCountBits: Int = Integer.SIZE - Integer.numberOfLeadingZeros(maxRanges) + val lineBits: Int = Integer.numberOfTrailingZeros(lineBytes) + require(addressBits > lineBits) +} + +object Params { + def fromProto(p: TrackingHardware): Params = + Params(p.getEntries, p.getAddressBits, p.getIdBits, p.getLineBytes, p.getMaxRanges) +} diff --git a/arch/src/main/scala/framework/ip/mesh_shm/build.sc b/arch/src/main/scala/framework/ip/mesh_shm/build.sc new file mode 100644 index 000000000..80489a76e --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/build.sc @@ -0,0 +1,32 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.memory.mesh_shm.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshBankNode.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshChannel.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshChannelOrdering.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshControlInterface.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshCreditLink.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshRouter.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshSharedMem.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshSharedMemInterface.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshSharedMemParams.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshThreadEndpoint.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/MeshTransferController.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/configs/Geometry.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/Cargo.lock b/arch/src/main/scala/framework/ip/mesh_shm/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/ip/mesh_shm/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/Cargo.toml b/arch/src/main/scala/framework/ip/mesh_shm/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/ip/mesh_shm/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/csrc/fixed_attachment.cpp b/arch/src/main/scala/framework/ip/mesh_shm/src/csrc/fixed_attachment.cpp new file mode 100644 index 000000000..8e8cb5269 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/csrc/fixed_attachment.cpp @@ -0,0 +1,106 @@ +#include "VMeshSharedMem.h" +#include "verilated.h" +#include +#include + +int main(int argc, char **argv) { + Verilated::commandArgs(argc, argv); + Verilated::randReset(0); + VMeshSharedMem dut; + unsigned cycle = 0; + auto check = [](bool condition, const char *message) { + if (!condition) + throw std::runtime_error(message); + }; + auto tick = [&]() { + dut.clock = 0; + dut.eval(); + dut.clock = 1; + dut.eval(); + dut.clock = 0; + dut.eval(); + ++cycle; + }; + auto wait = [&](auto condition) { + for (unsigned n = 0; n < 300; ++n) { + dut.eval(); + if (condition()) + return; + tick(); + } + throw std::runtime_error("Mesh response timed out"); + }; + dut.io_bankWrites_0_ready = 1; + dut.io_bankWrites_1_ready = 1; + dut.io_bankWrites_2_ready = 1; + dut.io_bankWrites_3_ready = 1; + dut.io_bankWrites_4_ready = 1; + dut.io_bankWrites_5_ready = 1; + dut.io_bankWrites_6_ready = 1; + dut.io_bankWrites_7_ready = 1; + dut.io_bankWrites_8_ready = 1; + dut.io_bankWrites_9_ready = 1; + dut.io_bankWrites_10_ready = 1; + dut.io_bankWrites_11_ready = 1; + dut.reset = 1; + for (unsigned i = 0; i < 4; ++i) + tick(); + dut.reset = 0; + try { + // Prototype thread 2 has two ports, both fixed at bank node 4. + // Write through its second port to remote bank 11, then read through its + // first. + dut.io_channels_3_request_bits_tdest = 11; + dut.io_channels_3_request_bits_tid = 9; + dut.io_channels_3_request_bits_tuser = (8u << 1) | 1u; + dut.io_channels_3_request_bits_tkeep = 0xffff; + dut.io_channels_3_request_bits_tlast = 1; + for (unsigned word = 0; word < 4; ++word) + dut.io_channels_3_request_bits_tdata[word] = 0x12340000u + word; + dut.io_channels_3_request_valid = 1; + wait([&] { return dut.io_channels_3_request_ready; }); + std::printf("cycle=%u write remote bank=11 row=8 through thread=2 port=1\n", + cycle); + tick(); + dut.io_channels_3_request_valid = 0; + wait([&] { return dut.io_channels_3_response_valid; }); + check(dut.io_channels_3_response_bits_tid == 9 && + dut.io_channels_3_response_bits_tuser == 2, + "Write response returned to the wrong endpoint"); + for (unsigned i = 0; i < 5; ++i) { + check(dut.io_channels_3_response_valid && + dut.io_channels_3_response_bits_tid == 9, + "Write response changed under backpressure"); + tick(); + } + dut.io_channels_3_response_ready = 1; + tick(); + dut.io_channels_2_request_bits_tdest = 11; + dut.io_channels_2_request_bits_tid = 10; + dut.io_channels_2_request_bits_tuser = 8u << 1; + dut.io_channels_2_request_bits_tkeep = 0xffff; + dut.io_channels_2_request_bits_tlast = 1; + dut.io_channels_2_request_valid = 1; + wait([&] { return dut.io_channels_2_request_ready; }); + std::printf("cycle=%u read remote bank=11 row=8 through thread=2 port=0\n", + cycle); + tick(); + dut.io_channels_2_request_valid = 0; + wait([&] { return dut.io_channels_2_response_valid; }); + check(dut.io_channels_2_response_bits_tid == 10 && + dut.io_channels_2_response_bits_tuser == 0, + "Read response returned to the wrong endpoint"); + for (unsigned word = 0; word < 4; ++word) { + check(dut.io_channels_2_response_bits_tdata[word] == 0x12340000u + word, + "Remote bank read data mismatch"); + } + dut.io_channels_2_response_ready = 1; + tick(); + std::puts("PASS: two ports at one fixed mesh node access a remote bank and " + "preserve response ownership"); + } catch (const std::exception &error) { + std::fprintf(stderr, "FAIL at cycle %u: %s\n", cycle, error.what()); + return 1; + } + dut.final(); +} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/src/lib.rs b/arch/src/main/scala/framework/ip/mesh_shm/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/ip/mesh_shm/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/src/transfer.rs b/arch/src/main/scala/framework/ip/mesh_shm/src/csrc/src/transfer.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/csrc/src/transfer.rs rename to arch/src/main/scala/framework/ip/mesh_shm/src/csrc/src/transfer.rs diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router.f b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router.f new file mode 100644 index 000000000..cf27953bd --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router.f @@ -0,0 +1,7 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/axis/interface.sv ++incdir+@RESOURCES@ +@RESOURCES@/router_package.sv +-F @RTL@/MeshRouter/filelist.f +@RESOURCES@/router_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_item.svh b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_item.svh similarity index 100% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_item.svh rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_item.svh diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_package.sv b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_package.sv new file mode 100644 index 000000000..b949e8a86 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_package.sv @@ -0,0 +1,13 @@ +package router_pkg; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + typedef virtual axis_if #(128, 8, 4, 45) router_vif; + import "DPI-C" function int unsigned mesh_route( + input int unsigned destination, + input int unsigned row, + input int unsigned col + ); + `include "router_item.svh" + `include "router_test.svh" +endpackage diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_tb.sv b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_tb.sv similarity index 100% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_tb.sv rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_test.svh b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_test.svh similarity index 96% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_test.svh rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_test.svh index ff62b3a96..5b6541d78 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_test.svh +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/router_test.svh @@ -1,13 +1,13 @@ -class protocol_test extends ip_test; +class protocol_test extends base_test; `uvm_component_utils(protocol_test) router_vif source[5], sink[5]; - keyed_scoreboard #(packet) scoreboard; + result_board #(packet, 1) scoreboard; function new(string name, uvm_component parent); super.new(name, parent); endfunction function void build_phase(uvm_phase phase); super.build_phase(phase); - scoreboard = keyed_scoreboard#(packet)::type_id::create("scoreboard", this); + scoreboard = result_board#(packet, 1)::type_id::create("scoreboard", this); for (int i = 0; i < 5; i++) begin if (!uvm_config_db#(router_vif)::get( this, "", $sformatf("source%0d", i), source[i] diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared.f b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared.f new file mode 100644 index 000000000..342a30920 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared.f @@ -0,0 +1,8 @@ ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/protocol/axis/interface.sv ++incdir+@RESOURCES@ +@RESOURCES@/transfer_if.sv +@RESOURCES@/shared_package.sv +-F @RTL@/MeshSharedMem/filelist.f +@RESOURCES@/shared_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_item.svh b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_item.svh similarity index 100% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_item.svh rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_item.svh diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_package.sv b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_package.sv similarity index 95% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_package.sv rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_package.sv index 7d7197ef8..eb329ee24 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_package.sv +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_package.sv @@ -1,6 +1,6 @@ package shared_pkg; import uvm_pkg::*; - import ip_pkg::*; + import verification_pkg::*; `include "uvm_macros.svh" typedef virtual axis_if #(128, 8, 4, 17) request_vif; typedef virtual axis_if #(128, 8, 0, 2) response_vif; diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_tb.sv b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_tb.sv similarity index 99% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_tb.sv rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_tb.sv index 62806b4cf..f0e31a0f9 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_tb.sv +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_tb.sv @@ -30,10 +30,10 @@ module shared_tb; .io_transferCompletion_ready(transfer.complete_ready), .io_transferCompletion_bits_error(transfer.error), .io_transferCompletion_bits_tag(transfer.complete_tag), - .io_transferCommand_bits_sourceCore(transfer.source_core), + .io_transferCommand_bits_sourceThread(transfer.source_thread), .io_transferCommand_bits_sourceBank(transfer.source_bank), .io_transferCommand_bits_sourceAddr(transfer.source_row), - .io_transferCommand_bits_targetCore(transfer.target_core), + .io_transferCommand_bits_targetThread(transfer.target_thread), .io_transferCommand_bits_targetBank(transfer.target_bank), .io_transferCommand_bits_targetAddr(transfer.target_row), .io_transferCommand_bits_rows(transfer.rows), diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_test.svh b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_test.svh similarity index 89% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_test.svh rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_test.svh index bdf3230b4..07d9960ef 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared_test.svh +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/shared_test.svh @@ -1,17 +1,17 @@ -class protocol_test extends ip_test; +class protocol_test extends base_test; `uvm_component_utils(protocol_test) request_vif source[10]; response_vif sink[10]; - keyed_scoreboard #(response) scoreboard; + result_board #(response, 1) scoreboard; chandle model; virtual transfer_if transfer; function new(string name, uvm_component parent); super.new(name, parent); - timeout = 500us; + endfunction function void build_phase(uvm_phase phase); super.build_phase(phase); - scoreboard = keyed_scoreboard#(response)::type_id::create("scoreboard", this); + scoreboard = result_board#(response, 1)::type_id::create("scoreboard", this); model = mesh_banks_create(); if (!uvm_config_db#(virtual transfer_if)::get(this, "", "transfer", transfer)) `uvm_fatal("VIF", "transfer interface missing") @@ -159,15 +159,15 @@ class protocol_test extends ip_test; end access (0, 0, 3, 0, 0, 253, 1); endtask - task move_rows(int source_core, int source_bank, int source_row, int target_core, int target_bank, - int target_row, int rows, bit invalid); + task move_rows(int source_thread, int source_bank, int source_row, int target_thread, + int target_bank, int target_row, int rows, bit invalid); bit [7:0] token = $urandom(); @(negedge transfer.clock); transfer.valid = 1; - transfer.source_core = source_core; + transfer.source_thread = source_thread; transfer.source_bank = source_bank; transfer.source_row = source_row; - transfer.target_core = target_core; + transfer.target_thread = target_thread; transfer.target_bank = target_bank; transfer.target_row = target_row; transfer.rows = rows; @@ -175,10 +175,10 @@ class protocol_test extends ip_test; transfer.complete_ready = 0; if (!invalid) for (int i = 0; i < rows; i++) begin - transfer.memory[source_core][source_bank][source_row+i] = { + transfer.memory[source_thread][source_bank][source_row+i] = { $urandom(), $urandom(), $urandom(), $urandom() }; - transfer.initialized[source_core][source_bank][source_row+i] = 1; + transfer.initialized[source_thread][source_bank][source_row+i] = 1; end do @(posedge transfer.clock); while (!transfer.ready); @(negedge transfer.clock); @@ -191,8 +191,8 @@ class protocol_test extends ip_test; end if (!invalid) for (int i = 0; i < rows; i++) - if(!transfer.initialized[target_core][target_bank][target_row+i] || - transfer.memory[target_core][target_bank][target_row+i]!==transfer.memory[source_core][source_bank][source_row+i]) + if(!transfer.initialized[target_thread][target_bank][target_row+i] || + transfer.memory[target_thread][target_bank][target_row+i]!==transfer.memory[source_thread][source_bank][source_row+i]) `uvm_fatal("MVOVER", "target rows differ from source") transfer.complete_ready = 1; @(posedge transfer.clock); @@ -240,10 +240,10 @@ class protocol_test extends ip_test; move_rows(1, 16, 0, 2, 0, 0, 1, 1); move_rows(1, 1, 3, 2, 16, 0, 1, 1); move_rows(1, 1023, 0, 2, 0, 0, 1, 1); - for (int core = 0; core < 5; core++) - move_rows(core, core, 16, (core + 1) % 5, 15 - core, 32, 8, 0); + for (int thread = 0; thread < 5; thread++) + move_rows(thread, thread, 16, (thread + 1) % 5, 15 - thread, 32, 8, 0); move_rows(1, 1, 0, 1, 1, 1, 8, 1); - `uvm_info("MVOVER", "checked core1.bank1 to core2.bank8 and range/error cases", UVM_LOW) + `uvm_info("MVOVER", "checked thread1.bank1 to thread2.bank8 and range/error cases", UVM_LOW) `uvm_info("MESH", $sformatf("checked %0d memory responses", scoreboard.checked), UVM_LOW) endtask function void final_phase(uvm_phase phase); diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/suite.toml new file mode 100644 index 000000000..17d2625b1 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/suite.toml @@ -0,0 +1,37 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libmesh_reference_model" + +[[tests]] +functional_groups = [] +name = "router" +top = "router_tb" +test = "protocol_test" +filelist = "router.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "shared" +top = "shared_tb" +test = "protocol_test" +filelist = "shared.f" +timeout_ns = 500000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/transfer_if.sv b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/transfer_if.sv similarity index 96% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/transfer_if.sv rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/transfer_if.sv index 3aae89b7e..2eba72f6f 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/transfer_if.sv +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/resources/transfer_if.sv @@ -3,7 +3,7 @@ interface transfer_if ( ); logic reset; logic valid, ready, complete_valid, complete_ready, error; - logic [7:0] source_core, target_core, tag, complete_tag; + logic [7:0] source_thread, target_thread, tag, complete_tag; logic [9:0] source_bank, target_bank; logic [15:0] source_row, target_row; logic [16:0] rows; diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/Emit.scala new file mode 100644 index 000000000..578c52eaa --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/Emit.scala @@ -0,0 +1,15 @@ +package framework.ip.memory.mesh_shm + +object Emit extends App { + val p = MeshSharedMemParams.prototype + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new MeshRouter(p, 1, 1), + firtoolOpts = args, + args = Array("--target-dir", "build/MeshRouter", "--split-verilog") + ) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new MeshSharedMem(p), + firtoolOpts = args, + args = Array("--target-dir", "build/MeshSharedMem", "--split-verilog") + ) +} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshBankNode.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshBankNode.scala similarity index 95% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshBankNode.scala rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshBankNode.scala index 5fe267592..2a1f8466b 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshBankNode.scala +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshBankNode.scala @@ -1,9 +1,9 @@ -package memcore.memory.mesh_shm +package framework.ip.memory.mesh_shm import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.memory.bank.{Bank, BankSetParams} +import framework.ip.memory.bank.{Bank, BankSetParams} @instantiable class MeshBankNode(p: MeshSharedMemParams, row: Int, col: Int) extends Module { diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshChannel.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshChannel.scala new file mode 100644 index 000000000..bd9a26e49 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshChannel.scala @@ -0,0 +1,10 @@ +package framework.ip.memory.mesh_shm + +import chisel3._ +import chisel3.util._ + +/** Thread-facing shared-bank port. Tagged requests support bounded outstanding transactions per channel. */ +class MeshChannel(p: MeshSharedMemParams) extends Bundle { + val request = Flipped(Decoupled(new MeshClientRequest(p))) + val response = Decoupled(new MeshClientResponse(p)) +} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshChannelOrdering.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshChannelOrdering.scala similarity index 98% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshChannelOrdering.scala rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshChannelOrdering.scala index ef07fb4cd..e27515a75 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshChannelOrdering.scala +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshChannelOrdering.scala @@ -1,4 +1,4 @@ -package memcore.memory.mesh_shm +package framework.ip.memory.mesh_shm import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshControlInterface.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshControlInterface.scala new file mode 100644 index 000000000..a72b8ea4e --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshControlInterface.scala @@ -0,0 +1,53 @@ +package framework.ip.memory.mesh_shm + +import chisel3._ +import chisel3.util._ + +/** Contiguous rows from one Thread-local Bank to another Thread-local Bank. */ +class MeshTransferCommand(p: MeshSharedMemParams) extends Bundle { + val sourceThread = UInt(8.W) + val sourceBank = UInt(p.localBankBits.W) + val sourceAddr = UInt(16.W) + val targetThread = UInt(8.W) + val targetBank = UInt(p.localBankBits.W) + val targetAddr = UInt(16.W) + val rows = UInt(17.W) + val tag = UInt(p.tagBits.W) +} + +class MeshTransferCompletion(p: MeshSharedMemParams) extends Bundle { + val tag = UInt(p.tagBits.W) + val error = Bool() +} + +class MeshLocalBankRequest( + addressBits: Int, + bankBits: Int, + dataBits: Int, + tagBits: Int) + extends Bundle { + val addr = UInt(addressBits.W) + val write = Bool() + val data = UInt(dataBits.W) + val mask = UInt((dataBits / 8).W) + val tag = UInt(tagBits.W) + val bank = UInt(bankBits.W) +} + +class MeshLocalBankResponse(dataBits: Int, tagBits: Int) extends Bundle { + val data = UInt(dataBits.W) + val tag = UInt(tagBits.W) + val error = Bool() +} + +/** SharedMem initiates both the source read and target write. */ +class MeshLocalBankPort( + addressBits: Int, + bankBits: Int, + dataBits: Int, + tagBits: Int) + extends Bundle { + + val request = Decoupled(new MeshLocalBankRequest(addressBits, bankBits, dataBits, tagBits)) + val response = Flipped(Decoupled(new MeshLocalBankResponse(dataBits, tagBits))) +} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshCreditLink.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshCreditLink.scala similarity index 85% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshCreditLink.scala rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshCreditLink.scala index 2c7bf499c..c19d8fa1f 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshCreditLink.scala +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshCreditLink.scala @@ -1,4 +1,7 @@ -package memcore.memory.mesh_shm +package framework.ip.memory.mesh_shm + +import framework.lib.queue.Queue +import chisel3.experimental.hierarchy.{Instance, Instantiate} import chisel3._ import chisel3.util._ @@ -44,7 +47,7 @@ class MeshCreditToDecoupled(p: MeshSharedMemParams) extends Module { val out = Decoupled(new MeshPacket(p)) }) - val buffer = Module(new Queue(new MeshPacket(p), p.linkBufferDepth, pipe = false, flow = false)) + val buffer = Instantiate(new Queue(new MeshPacket(p), p.linkBufferDepth, false, false)) buffer.io.enq.valid := io.in.valid buffer.io.enq.bits := io.in.bits io.out <> buffer.io.deq diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshRouter.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshRouter.scala similarity index 88% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshRouter.scala rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshRouter.scala index 3f086ea73..7f5499c78 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshRouter.scala +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshRouter.scala @@ -1,4 +1,7 @@ -package memcore.memory.mesh_shm +package framework.ip.memory.mesh_shm + +import framework.lib.queue.Queue +import framework.lib.arbiter.{RRArbiter} import chisel3._ import chisel3.util._ @@ -40,7 +43,7 @@ class MeshCreditRouter(p: MeshSharedMemParams, row: Int, col: Int) extends Modul } val inputs = Seq.fill(MeshDirection.portCount) { - Module(new Queue(new MeshPacket(p), p.linkBufferDepth, pipe = false, flow = false)) + Instantiate(new Queue(new MeshPacket(p), p.linkBufferDepth, false, false)) } for ((input, port) <- inputs.zip(io.in)) { @@ -54,15 +57,8 @@ class MeshCreditRouter(p: MeshSharedMemParams, row: Int, col: Int) extends Modul val choices = inputs.map(input => direction(input.io.deq.bits)) - val arbiters = Seq.fill(MeshDirection.portCount)(Module(new RRArbiter(new MeshPacket(p), MeshDirection.portCount) { - - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(MeshDirection.portCount)).W)) - when(io.out.fire)(pointer := io.chosen) - pointer - } - - })) + val arbiters = + Seq.fill(MeshDirection.portCount)(Instantiate(new RRArbiter(new MeshPacket(p), MeshDirection.portCount))) for (output <- 0 until MeshDirection.portCount) { for (input <- 0 until MeshDirection.portCount) { diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMem.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMem.scala new file mode 100644 index 000000000..0c9399886 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMem.scala @@ -0,0 +1,259 @@ +package framework.ip.memory.mesh_shm + +import framework.lib.arbiter.{RRArbiter} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} + +class MeshSharedMemIO(p: MeshSharedMemParams) extends Bundle { + val channels = Vec(p.totalChannels, new MeshChannel(p)) + val external = Vec(p.externalChannels, new MeshChannel(p)) + val transferCommand = Flipped(Decoupled(new MeshTransferCommand(p))) + val transferCompletion = Decoupled(new MeshTransferCompletion(p)) + val localBanks = Vec(p.threads.size, new MeshLocalBankPort(p.addressBits, p.localBankBits, p.dataBits, p.tagBits)) + val bankWrites = Vec(p.bankCount, Decoupled(new MeshClientRequest(p))) +} + +@instantiable +class MeshSharedMem(p: MeshSharedMemParams) extends Module { + + @public + val io = IO(new MeshSharedMemIO(p)) + + val transfer: Instance[MeshTransferController] = Instantiate(new MeshTransferController(p)) + transfer.io.command.valid := io.transferCommand.valid + transfer.io.command.bits := io.transferCommand.bits + io.transferCommand.ready := transfer.io.command.ready + io.transferCompletion.valid := transfer.io.completion.valid + io.transferCompletion.bits := transfer.io.completion.bits + transfer.io.completion.ready := io.transferCompletion.ready + + val ordering = Instantiate(new MeshChannelOrdering(p, p.totalChannels + p.externalChannels)) + for (channel <- 0 until p.totalChannels) { + ordering.io.client(channel) <> io.channels(channel) + } + for (channel <- 0 until p.externalChannels) { + ordering.io.client(p.totalChannels + channel) <> io.external(channel) + } + + for (thread <- p.threads.indices if p.threads(thread).ports == 0) { + io.localBanks(thread).request.valid := false.B + io.localBanks(thread).request.bits := 0.U.asTypeOf(io.localBanks(thread).request.bits) + io.localBanks(thread).response.ready := false.B + } + + val requests: Seq[Seq[Instance[MeshCreditRouter]]] = + Seq.tabulate(p.rows, p.cols)((row, col) => Instantiate(new MeshCreditRouter(p, row, col))) + val responses: Seq[Seq[Instance[MeshCreditRouter]]] = + Seq.tabulate(p.rows, p.cols)((row, col) => Instantiate(new MeshCreditRouter(p, row, col))) + val banks: Seq[Seq[Instance[MeshBankNode]]] = + Seq.tabulate(p.rows, p.cols)((row, col) => Instantiate(new MeshBankNode(p, row, col))) + val bankRows = VecInit((0 until p.bankCount).map(i => (i / p.cols).U(p.rowBits.W))) + val bankCols = VecInit((0 until p.bankCount).map(i => (i % p.cols).U(p.colBits.W))) + + for { + row <- 0 until p.rows + col <- 0 until p.cols + } { + val requestRouter = requests(row)(col) + val responseRouter = responses(row)(col) + val bank = banks(row)(col) + val bankIndex = row * p.cols + col + val requestEgress = Instantiate(new MeshCreditToDecoupled(p)) + requestEgress.io.in <> requestRouter.io.out(MeshDirection.endpoint) + val localRequest = requestEgress.io.out + io.bankWrites( + bankIndex + ).valid := localRequest.valid && !localRequest.bits.privateRequest && localRequest.bits.write && bank.io.request.ready + io.bankWrites(bankIndex).bits.bank := bankIndex.U + io.bankWrites(bankIndex).bits.tuser := Cat(bank.io.request.bits.addr, bank.io.request.bits.write) + io.bankWrites(bankIndex).bits.tlast := true.B + io.bankWrites(bankIndex).bits.data := bank.io.request.bits.data + io.bankWrites(bankIndex).bits.mask := bank.io.request.bits.mask + io.bankWrites(bankIndex).bits.tag := bank.io.request.bits.tag + bank.io.request.valid := localRequest.valid && !localRequest.bits.privateRequest && + (!localRequest.bits.write || io.bankWrites(bankIndex).ready) + bank.io.request.bits := localRequest.bits + val privateThreads = + p.threads.indices.filter(thread => p.threads(thread).ports > 0 && p.threadLocations(thread) == (row, col)) + val privateReady = Wire(Vec(privateThreads.size, Bool())) + val replies = Instantiate(new RRArbiter(new MeshPacket(p), privateThreads.size + 1)) + replies.io.in(0) <> bank.io.response + for ((thread, index) <- privateThreads.zipWithIndex) { + val endpoint: Instance[MeshThreadEndpoint] = Instantiate(new MeshThreadEndpoint(p, thread, row, col)) + endpoint.io.request.valid := localRequest.valid && localRequest.bits.privateRequest && localRequest.bits.thread === thread.U + endpoint.io.request.bits := localRequest.bits + privateReady(index) := localRequest.bits.thread === thread.U && endpoint.io.request.ready + io.localBanks(thread) <> endpoint.io.bank + replies.io.in(index + 1) <> endpoint.io.response + } + localRequest.ready := Mux( + localRequest.bits.privateRequest, + (if (privateThreads.nonEmpty) privateReady.asUInt.orR else false.B), + bank.io.request.ready && (!localRequest.bits.write || io.bankWrites(bankIndex).ready) + ) + when(localRequest.valid && localRequest.bits.privateRequest) { + assert( + privateThreads.map(thread => localRequest.bits.thread === thread.U).reduceOption(_ || _).getOrElse(false.B), + "private request reached a node without its Thread" + ) + } + val responseIngress = Instantiate(new MeshDecoupledToCredit(p)) + responseIngress.io.in <> replies.io.out + responseRouter.io.in(MeshDirection.endpoint) <> responseIngress.io.out + val requestIngress = Instantiate(new MeshDecoupledToCredit(p)) + requestRouter.io.in(MeshDirection.endpoint) <> requestIngress.io.out + val responseEgress = Instantiate(new MeshCreditToDecoupled(p)) + responseEgress.io.in <> responseRouter.io.out(MeshDirection.endpoint) + val response = responseEgress.io.out + + val attached = p.channelLocations.zipWithIndex.collect { + case ((r, c), index) if r == row && c == col => index + } ++ (if (row == 0 && col == 0) (p.totalChannels until p.totalChannels + 1 + p.externalChannels) else Seq.empty) + if (attached.nonEmpty) { + val arbiter = Instantiate(new RRArbiter(new MeshPacket(p), attached.size)) + val localResponseReady = Wire(Vec(attached.size, Bool())) + for ((channel, port) <- attached.zipWithIndex) { + if (channel == p.totalChannels) { + arbiter.io.in(port) <> transfer.io.mesh.request + val network = response + transfer.io.mesh.response.valid := network.valid && network.bits.channel === channel.U + transfer.io.mesh.response.bits := network.bits + localResponseReady(port) := network.bits.channel === channel.U && transfer.io.mesh.response.ready + } else { + val client = + if (channel < p.totalChannels) ordering.io.ordered(channel) else ordering.io.ordered(channel - 1) + val requestPort = client.request + val responsePort = client.response + val inFlight = RegInit(0.U(log2Ceil(p.maxInFlight + 1).W)) + val tags = RegInit(0.U((1 << p.tagBits).W)) + val available = inFlight < p.maxInFlight.U && !tags(requestPort.bits.tag) + val errorPending = RegInit(false.B) + val errorTag = Reg(UInt(p.tagBits.W)) + val errorWrite = Reg(Bool()) + val bankValid = requestPort.bits.bank < p.visibleBankCount.U + val packet = Wire(new MeshPacket(p)) + packet := 0.U.asTypeOf(new MeshPacket(p)) + packet.tdest := Cat(bankRows(requestPort.bits.bank), bankCols(requestPort.bits.bank)) + packet.tdata := requestPort.bits.data + packet.tkeep := requestPort.bits.mask + packet.tid := requestPort.bits.tag + packet.tlast := true.B + packet.tuser := Cat( + row.U(p.rowBits.W), + col.U(p.colBits.W), + channel.U(p.channelBits.W), + requestPort.bits.addr, + requestPort.bits.write, + false.B + ) + arbiter.io.in(port).valid := requestPort.valid && available && !errorPending && bankValid + arbiter.io.in(port).bits := packet + requestPort.ready := available && !errorPending && Mux(bankValid, arbiter.io.in(port).ready, true.B) + when(requestPort.fire) { + when(!bankValid) { + errorPending := true.B + errorTag := requestPort.bits.tag + errorWrite := requestPort.bits.write + } + } + + val responseValid = RegInit(false.B) + val responseBits = Reg(new MeshClientResponse(p)) + val responseNext = Wire(Decoupled(new MeshClientResponse(p))) + responsePort.valid := responseValid + responsePort.bits := responseBits + responseNext.ready := !responseValid || responsePort.ready + when(responseNext.ready) { + responseValid := responseNext.valid + when(responseNext.valid)(responseBits := responseNext.bits) + } + responseNext.valid := + errorPending || (response.valid && + response.bits.channel === channel.U) + responseNext.bits := 0.U.asTypeOf(responseNext.bits) + responseNext.bits.data := + Mux(errorPending, 0.U, response.bits.data) + responseNext.bits.tag := + Mux(errorPending, errorTag, response.bits.tag) + responseNext.bits.tkeep := Fill(p.maskBits, 1.U(1.W)) + responseNext.bits.tlast := true.B + responseNext.bits.tuser := Cat( + Mux(errorPending, errorWrite, response.bits.write), + errorPending || response.bits.error + ) + localResponseReady(port) := + response.bits.channel === channel.U && + responseNext.ready && !errorPending + when(response.valid && + response.bits.channel === channel.U) { + assert(inFlight =/= 0.U && tags(response.bits.tag)) + } + when(responsePort.fire) { + assert(inFlight =/= 0.U && tags(responsePort.bits.tag)) + } + when(responseNext.fire && errorPending)(errorPending := false.B) + val issued = Mux(requestPort.fire, 1.U((1 << p.tagBits).W) << requestPort.bits.tag, 0.U) + val completed = Mux(responsePort.fire, 1.U((1 << p.tagBits).W) << responsePort.bits.tag, 0.U) + tags := (tags | issued) & ~completed + when(requestPort.fire =/= responsePort.fire) { + inFlight := Mux(requestPort.fire, inFlight + 1.U, inFlight - 1.U) + } + } + } + requestIngress.io.in <> arbiter.io.out + response.ready := localResponseReady.asUInt.orR + when(response.valid) { + assert( + attached.map(channel => response.bits.channel === channel.U).reduce( + _ || _ + ) + ) + } + } else { + requestIngress.io.in.valid := false.B + requestIngress.io.in.bits := 0.U.asTypeOf(new MeshPacket(p)) + response.ready := false.B + when(response.valid) { + assert(false.B, "response reached a node without an attached channel") + } + } + + if (col + 1 < p.cols) { + requests(row)(col + 1).io.in(MeshDirection.west) <> requestRouter.io.out(MeshDirection.east) + requestRouter.io.in(MeshDirection.east) <> requests(row)(col + 1).io.out(MeshDirection.west) + responses(row)(col + 1).io.in(MeshDirection.west) <> responseRouter.io.out(MeshDirection.east) + responseRouter.io.in(MeshDirection.east) <> responses(row)(col + 1).io.out(MeshDirection.west) + } + if (row + 1 < p.rows) { + requests(row + 1)(col).io.in(MeshDirection.north) <> requestRouter.io.out(MeshDirection.south) + requestRouter.io.in(MeshDirection.south) <> requests(row + 1)(col).io.out(MeshDirection.north) + responses(row + 1)(col).io.in(MeshDirection.north) <> responseRouter.io.out(MeshDirection.south) + responseRouter.io.in(MeshDirection.south) <> responses(row + 1)(col).io.out(MeshDirection.north) + } + + for (router <- Seq(requestRouter, responseRouter)) { + if (col == 0) { + router.io.in(MeshDirection.west).valid := false.B + router.io.in(MeshDirection.west).bits := 0.U.asTypeOf(new MeshPacket(p)) + router.io.out(MeshDirection.west).credit := false.B + } + if (col == p.cols - 1) { + router.io.in(MeshDirection.east).valid := false.B + router.io.in(MeshDirection.east).bits := 0.U.asTypeOf(new MeshPacket(p)) + router.io.out(MeshDirection.east).credit := false.B + } + if (row == 0) { + router.io.in(MeshDirection.north).valid := false.B + router.io.in(MeshDirection.north).bits := 0.U.asTypeOf(new MeshPacket(p)) + router.io.out(MeshDirection.north).credit := false.B + } + if (row == p.rows - 1) { + router.io.in(MeshDirection.south).valid := false.B + router.io.in(MeshDirection.south).bits := 0.U.asTypeOf(new MeshPacket(p)) + router.io.out(MeshDirection.south).credit := false.B + } + } + } +} diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMemInterface.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMemInterface.scala new file mode 100644 index 000000000..e0306d241 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMemInterface.scala @@ -0,0 +1,37 @@ +package framework.ip.memory.mesh_shm + +import chisel3._ + +class MeshClientRequest(p: MeshSharedMemParams) + extends framework.ip.bus.axi.Beat(p.dataBits, p.tagBits, p.bankBits, p.addressBits + 1) { + def bank = tdest + def addr = tuser(p.addressBits, 1) + def write = tuser(0) + def mask = tkeep + def tag = tid +} + +class MeshClientResponse(p: MeshSharedMemParams) extends framework.ip.bus.axi.Beat(p.dataBits, p.tagBits, 0, 2) { + def tag = tid + def error = tuser(0) + def write = tuser(1) +} + +/** One row transaction: AXI-S payload plus routed event metadata in TUSER. */ +class MeshPacket(p: MeshSharedMemParams) + extends framework.ip.bus.axi.Beat(p.dataBits, p.tagBits, p.rowBits + p.colBits, p.packetUserBits) { + def privateRequest = tuser(p.eventBits) + def localBank = tuser(p.eventBits + p.localBankBits, p.eventBits + 1) + def thread = tuser(p.packetUserBits - 1, p.eventBits + p.localBankBits + 1) + def destRow = tdest(p.rowBits + p.colBits - 1, p.colBits) + def destCol = tdest(p.colBits - 1, 0) + def error = tuser(0) + def write = tuser(1) + def addr = tuser(p.addressBits + 1, 2) + def channel = tuser(p.addressBits + p.channelBits + 1, p.addressBits + 2) + def sourceCol = tuser(p.addressBits + p.channelBits + p.colBits + 1, p.addressBits + p.channelBits + 2) + def sourceRow = + tuser(p.addressBits + p.channelBits + p.colBits + p.rowBits + 1, p.addressBits + p.channelBits + p.colBits + 2) + def mask = tkeep + def tag = tid +} diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMemParams.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMemParams.scala new file mode 100644 index 000000000..fc0e5430b --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshSharedMemParams.scala @@ -0,0 +1,79 @@ +package framework.ip.memory.mesh_shm + +import chisel3.util.log2Ceil +import framework.ip.memory.bank.BankSetParams + +case class MeshThreadAttachment(bank: Int, ports: Int) + +case class MeshSharedMemParams( + rows: Int, + cols: Int, + entriesPerBank: Int, + dataBits: Int, + tagBits: Int, + threads: Seq[MeshThreadAttachment], + maxInFlight: Int = 4, + localBankBits: Int = 10, + visibleBanks: Int = 0, + externalChannels: Int = 0, + linkBufferDepth: Int = 2) { + require(rows > 0 && cols > 0) + require(entriesPerBank <= 65536) + require(entriesPerBank >= 2 && (entriesPerBank & (entriesPerBank - 1)) == 0) + require(dataBits > 0 && dataBits % 8 == 0) + require(tagBits > 0 && localBankBits > 0 && threads.nonEmpty) + require(threads.forall(t => t.bank >= 0 && t.bank < rows * cols && t.ports >= 0)) + + require(externalChannels >= 0) + require(linkBufferDepth > 0) + val linkCreditBits: Int = math.max(1, log2Ceil(linkBufferDepth + 1)) + require(maxInFlight > 0 && maxInFlight <= (1 << tagBits)) + val bankParams = BankSetParams(dataBits, 1, entriesPerBank, tagBits) + val bankCount: Int = rows * cols + val visibleBankCount: Int = if (visibleBanks == 0) bankCount else visibleBanks + require(visibleBankCount > 0 && visibleBankCount <= bankCount) + require(threads.exists(_.ports > 0), "mesh requires a data endpoint") + val totalChannels: Int = threads.map(_.ports).sum + val channelLocations: Seq[(Int, Int)] = + threads.flatMap(t => Seq.fill(t.ports)((t.bank / cols, t.bank % cols))) + val rowBits: Int = math.max(1, log2Ceil(rows)) + val colBits: Int = math.max(1, log2Ceil(cols)) + val bankBits: Int = math.max(1, log2Ceil(bankCount)) + val addressBits: Int = 16 + val channelBits: Int = math.max(1, log2Ceil(totalChannels + 1 + externalChannels)) + val threadBits: Int = math.max(1, log2Ceil(threads.size)) + + val threadLocations: Seq[(Int, Int)] = threads.map(t => (t.bank / cols, t.bank % cols)) + + val eventBits: Int = rowBits + colBits + channelBits + addressBits + 2 + val packetUserBits: Int = eventBits + localBankBits + 9 + val maskBits: Int = dataBits / 8 + + // Thread order in this sequence defines the flattened io.channels layout. + def channelsForThread(threadIndex: Int): Range = { + require(threadIndex >= 0 && threadIndex < threads.size) + val start = threads.take(threadIndex).map(_.ports).sum + start until (start + threads(threadIndex).ports) + } + +} + +object MeshSharedMemParams { + + // Five fixed thread attachments on a 3x4 mesh, with 1/1/2/2/4 data ports. + val prototype: MeshSharedMemParams = MeshSharedMemParams( + rows = 3, + cols = 4, + entriesPerBank = 1024, + dataBits = 128, + tagBits = 8, + threads = Seq( + MeshThreadAttachment(0, 1), + MeshThreadAttachment(3, 1), + MeshThreadAttachment(4, 2), + MeshThreadAttachment(7, 2), + MeshThreadAttachment(5, 4) + ) + ) + +} diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshThreadEndpoint.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshThreadEndpoint.scala new file mode 100644 index 000000000..1d81a5b3a --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshThreadEndpoint.scala @@ -0,0 +1,59 @@ +package framework.ip.memory.mesh_shm + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +/** The private Bank port stays beside its Thread; only packets cross the Mesh. */ +@instantiable +class MeshThreadEndpoint( + p: MeshSharedMemParams, + thread: Int, + row: Int, + col: Int) + extends Module { + + @public + val io = IO(new Bundle { + val request = Flipped(Decoupled(new MeshPacket(p))) + val response = Decoupled(new MeshPacket(p)) + val bank = new MeshLocalBankPort(p.addressBits, p.localBankBits, p.dataBits, p.tagBits) + }) + + val pending = RegInit(false.B) + val request = Reg(new MeshPacket(p)) + io.bank.request.valid := io.request.valid && !pending + io.bank.request.bits.bank := io.request.bits.localBank + io.bank.request.bits.addr := io.request.bits.addr + io.bank.request.bits.write := io.request.bits.write + io.bank.request.bits.data := io.request.bits.data + io.bank.request.bits.mask := io.request.bits.mask + io.bank.request.bits.tag := io.request.bits.tag + io.request.ready := io.bank.request.ready && !pending + when(io.request.fire) { + assert(io.request.bits.privateRequest && io.request.bits.thread === thread.U) + assert(io.request.bits.destRow === row.U && io.request.bits.destCol === col.U) + request := io.request.bits + pending := true.B + } + io.response.valid := pending && io.bank.response.valid + io.response.bits := request + io.response.bits.tdest := Cat(request.sourceRow, request.sourceCol) + io.response.bits.tdata := Mux(request.write, 0.U, io.bank.response.bits.data) + io.response.bits.tuser := Cat( + request.thread, + request.localBank, + true.B, + row.U(p.rowBits.W), + col.U(p.colBits.W), + request.channel, + request.addr, + request.write, + io.bank.response.bits.error + ) + io.bank.response.ready := pending && io.response.ready + when(io.bank.response.valid) { + assert(pending && io.bank.response.bits.tag === request.tag) + } + when(io.response.fire)(pending := false.B) +} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshTransferController.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshTransferController.scala similarity index 77% rename from arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshTransferController.scala rename to arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshTransferController.scala index a459a5dd5..6e2d11890 100644 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshTransferController.scala +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/MeshTransferController.scala @@ -1,10 +1,10 @@ -package memcore.memory.mesh_shm +package framework.ip.memory.mesh_shm import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -/** A row-range command uses routed private-Core endpoints, without a staging Bank. */ +/** A row-range command uses routed private-Thread endpoints, without a staging Bank. */ @instantiable class MeshTransferController(p: MeshSharedMemParams) extends Module { @@ -26,9 +26,9 @@ class MeshTransferController(p: MeshSharedMemParams) extends Module { val data = Reg(UInt(p.dataBits.W)) val remaining = Reg(UInt(17.W)) val failed = RegInit(false.B) - val endpoints = VecInit(p.cores.map(_.localBankNode.nonEmpty.B)) - val coreRows = VecInit(p.localBankLocations.map(_.map(_._1).getOrElse(0).U(p.rowBits.W))) - val coreCols = VecInit(p.localBankLocations.map(_.map(_._2).getOrElse(0).U(p.colBits.W))) + val endpoints = VecInit(p.threads.map(thread => (thread.ports > 0).B)) + val threadRows = VecInit(p.threadLocations.map(_._1.U(p.rowBits.W))) + val threadCols = VecInit(p.threadLocations.map(_._2.U(p.colBits.W))) io.command.ready := state === idle io.completion.valid := state === complete io.completion.bits.tag := command.tag @@ -38,13 +38,13 @@ class MeshTransferController(p: MeshSharedMemParams) extends Module { remaining := io.command.bits.rows failed := false.B val input = io.command.bits - val source = input.sourceCore(p.coreBits - 1, 0) - val target = input.targetCore(p.coreBits - 1, 0) - when(input.sourceCore >= p.cores.size.U || input.targetCore >= p.cores.size.U || + val source = input.sourceThread(p.threadBits - 1, 0) + val target = input.targetThread(p.threadBits - 1, 0) + when(input.sourceThread >= p.threads.size.U || input.targetThread >= p.threads.size.U || !endpoints(source) || !endpoints(target) || input.rows === 0.U || (input.sourceAddr +& input.rows) > 65536.U || (input.targetAddr +& input.rows) > 65536.U || - (input.sourceCore === input.targetCore && input.sourceBank === input.targetBank && + (input.sourceThread === input.targetThread && input.sourceBank === input.targetBank && input.sourceAddr =/= input.targetAddr && input.sourceAddr < (input.targetAddr +& input.rows) && input.targetAddr < (input.sourceAddr +& input.rows))) { @@ -53,20 +53,20 @@ class MeshTransferController(p: MeshSharedMemParams) extends Module { }.otherwise(state := sourceRequest) } val write = state === targetRequest - val core = Mux(write, command.targetCore, command.sourceCore) - val slot = core(p.coreBits - 1, 0) + val thread = Mux(write, command.targetThread, command.sourceThread) + val slot = thread(p.threadBits - 1, 0) val bank = Mux(write, command.targetBank, command.sourceBank) val address = Mux(write, command.targetAddr, command.sourceAddr) io.mesh.request.valid := state === sourceRequest || state === targetRequest io.mesh.request.bits := 0.U.asTypeOf(new MeshPacket(p)) - io.mesh.request.bits.tdest := Cat(coreRows(slot), coreCols(slot)) + io.mesh.request.bits.tdest := Cat(threadRows(slot), threadCols(slot)) io.mesh.request.bits.tuser := Cat( - core, + thread, bank, true.B, 0.U(p.rowBits.W), 0.U(p.colBits.W), - p.transferChannel.U(p.channelBits.W), + p.totalChannels.U(p.channelBits.W), address(p.addressBits - 1, 0), write, false.B diff --git a/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/configs/Geometry.scala b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/configs/Geometry.scala new file mode 100644 index 000000000..58aa8e360 --- /dev/null +++ b/arch/src/main/scala/framework/ip/mesh_shm/src/main/scala/configs/Geometry.scala @@ -0,0 +1,11 @@ +package framework.ip.memory.mesh_shm.configs + +import upickle.default.{macroRW, ReadWriter} + +case class Geometry(dataBits: Int, entriesPerBank: Int, banks: Int) { + require(dataBits == 128 && entriesPerBank > 0 && banks > 0) + val bankBytes: Int = entriesPerBank * (dataBits / 8) + val bytes: BigInt = BigInt(banks) * bankBytes +} + +object Geometry { implicit val rw: ReadWriter[Geometry] = macroRW } diff --git a/arch/src/main/scala/framework/ip/spm/build.sc b/arch/src/main/scala/framework/ip/spm/build.sc new file mode 100644 index 000000000..26bc47687 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + +val ipRoot: os.Path = millSourcePath + +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.ip.memory.spm.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Interfaces.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Memory.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/ip/spm/configs/uvm.toml b/arch/src/main/scala/framework/ip/spm/configs/uvm.toml new file mode 100644 index 000000000..669b7cf8c --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/configs/uvm.toml @@ -0,0 +1,6 @@ +[parameters] +base = "4096" +bytes = 256 +dataBits = 128 + +[defines] diff --git a/arch/src/main/scala/framework/mem-core/spm/src/csrc/src/lib.rs b/arch/src/main/scala/framework/ip/spm/src/csrc/src/lib.rs similarity index 100% rename from arch/src/main/scala/framework/mem-core/spm/src/csrc/src/lib.rs rename to arch/src/main/scala/framework/ip/spm/src/csrc/src/lib.rs diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/agent.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/agent.svh new file mode 100644 index 000000000..6d2a96092 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/agent.svh @@ -0,0 +1,19 @@ +class agent extends uvm_agent; + `uvm_component_utils(agent) + uvm_sequencer #(request_item) seqr; + driver drv; + monitor mon; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + seqr = uvm_sequencer#(request_item)::type_id::create("seqr", this); + drv = driver::type_id::create("driver", this); + mon = monitor::type_id::create("monitor", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + drv.seq_item_port.connect(seqr.seq_item_export); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/driver.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/driver.svh new file mode 100644 index 000000000..409427d6a --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/driver.svh @@ -0,0 +1,55 @@ +class driver extends uvm_driver #(request_item); + `uvm_component_utils(driver) + virtual port_if vif; + reset_controller resetter; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get( + this, "", "vif", vif + ) || !uvm_config_db#(reset_controller)::get( + this, "", "resetter", resetter + )) + `uvm_fatal("INTERFACE", "SPM driver requires ports and reset controller") + endfunction + task run_phase(uvm_phase phase); + request_item req; + vif.request_valid = 0; + vif.response_ready = 0; + vif.address = 0; + vif.size = 0; + vif.write = 0; + vif.data = 0; + vif.mask = 0; + vif.read_only = 0; + forever begin + seq_item_port.get_next_item(req); + do @(negedge vif.clock); while (vif.reset); + vif.address = req.address; + vif.size = req.size; + vif.write = req.write; + vif.data = req.data; + vif.mask = req.mask; + vif.read_only = req.read_only; + vif.stall = req.stall; + vif.cancel = req.cancel; + vif.request_valid = 1; + do @(posedge vif.clock); while (!vif.request_ready); + @(negedge vif.clock); + vif.request_valid = 0; + if (req.cancel) begin + resetter.pulse(2); + end else begin + while (!vif.response_valid) @(negedge vif.clock); + repeat (req.stall) @(negedge vif.clock); + vif.response_ready = 1; + @(posedge vif.clock); + @(negedge vif.clock); + vif.response_ready = 0; + end + seq_item_port.item_done(); + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/item.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/item.svh new file mode 100644 index 000000000..ba4693dd3 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/item.svh @@ -0,0 +1,35 @@ +class request_item extends uvm_sequence_item; + logic [63:0] address; + logic [$clog2($clog2(`SPM_BITS/8)+2)-1:0] size; + logic write; + logic [`SPM_BITS-1:0] data; + logic [`SPM_BITS/8-1:0] mask; + logic read_only; + int unsigned stall; + logic cancel; + `uvm_object_utils_begin(request_item) + `uvm_field_int(address, UVM_ALL_ON) + `uvm_field_int(size, UVM_ALL_ON) + `uvm_field_int(write, UVM_ALL_ON) + `uvm_field_int(data, UVM_ALL_ON) + `uvm_field_int(mask, UVM_ALL_ON) + `uvm_field_int(read_only, UVM_ALL_ON) + `uvm_field_int(stall, UVM_ALL_ON) + `uvm_field_int(cancel, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "request_item"); + super.new(name); + endfunction +endclass + +class response_item extends uvm_sequence_item; + logic [`SPM_BITS-1:0] value; + logic error; + `uvm_object_utils_begin(response_item) + `uvm_field_int(value, UVM_ALL_ON) + `uvm_field_int(error, UVM_ALL_ON) + `uvm_object_utils_end + function new(string name = "response_item"); + super.new(name); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/monitor.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/monitor.svh new file mode 100644 index 000000000..b8a8c2317 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/agent/monitor.svh @@ -0,0 +1,56 @@ +class monitor extends uvm_monitor; + `uvm_component_utils(monitor) + virtual port_if vif; + uvm_analysis_port #(request_item) requests; + uvm_analysis_port #(response_item) responses; + uvm_analysis_port #(bit) reset_ap; + function new(string name, uvm_component parent); + super.new(name, parent); + requests = new("requests", this); + responses = new("responses", this); + reset_ap = new("reset_ap", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "SPM monitor requires ports") + endfunction + task run_phase(uvm_phase phase); + bit pending = 0; + request_item req; + response_item rsp; + forever begin + @(posedge vif.clock); + if (vif.reset) begin + pending = 0; + reset_ap.write(1); + end else begin + if (!pending && vif.busy) `uvm_fatal("BUSY", "SPM is busy without an accepted request") + if (vif.request_valid && vif.request_ready) begin + if (pending) `uvm_fatal("OUTSTANDING", "SPM accepted a second request") + req = request_item::type_id::create("request"); + req.address = vif.address; + req.size = vif.size; + req.write = vif.write; + req.data = vif.data; + req.mask = vif.mask; + req.read_only = vif.read_only; + req.stall = vif.stall; + req.cancel = vif.cancel; + requests.write(req); + pending = 1; + end else if (pending && (vif.request_ready || !vif.busy)) + `uvm_fatal("BUSY", "SPM reopened its request port before completion") + if (vif.response_valid && !pending) + `uvm_fatal("EXTRA_RESULT", "SPM produced an unsolicited result") + if (vif.response_valid && vif.response_ready) begin + rsp = response_item::type_id::create("response"); + rsp.value = vif.value; + rsp.error = vif.error; + responses.write(rsp); + pending = 0; + end + end + end + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/env/coverage.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/env/coverage.svh new file mode 100644 index 000000000..c487cca6d --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/env/coverage.svh @@ -0,0 +1,27 @@ +class coverage extends uvm_subscriber #(request_item); + `uvm_component_utils(coverage) + covergroup behavior with function sample (request_item t); + option.per_instance = 1; + operation: coverpoint t.write; + size: coverpoint t.size { + bins legal[] = {[0 : $clog2(`SPM_BITS / 8)]}; + bins oversized = {[$clog2(`SPM_BITS / 8) + 1 : $]}; + } + lower_bound: coverpoint (t.address < `SPM_BASE) {bins seen = {1};} + upper_bound: coverpoint (t.address >= `SPM_BASE + `SPM_BYTES) {bins seen = {1};} + unaligned: coverpoint ((t.address & ((64'd1 << t.size) - 1)) != 0); + read_only: coverpoint t.read_only; + protected_write: coverpoint (t.read_only && t.write) {bins seen = {1};} + empty_mask: coverpoint (t.mask == 0) iff (t.write); + full_mask: coverpoint (&t.mask) iff (t.write); + backpressure: coverpoint t.stall {bins stalls[] = {0, 3};} + cancellation: coverpoint t.cancel; + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + behavior = new(); + endfunction + function void write(request_item t); + behavior.sample(t); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/env/env.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/env/env.svh new file mode 100644 index 000000000..8bd17acae --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/env/env.svh @@ -0,0 +1,26 @@ +`uvm_analysis_imp_decl(_reset) +class env extends environment #(request_item, response_item, model); + `uvm_component_utils(env) + agent bus; + coverage cov; + uvm_analysis_imp_reset #(bit, env) reset_export; + function new(string name, uvm_component parent); + super.new(name, parent); + reset_export = new("reset_export", this); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + cov = coverage::type_id::create("coverage", this); + bus = agent::type_id::create("agent", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + bus.mon.requests.connect(input_export); + bus.mon.requests.connect(cov.analysis_export); + bus.mon.reset_ap.connect(reset_export); + bus.mon.responses.connect(output_export); + endfunction + function void write_reset(bit asserted); + scoreboard.cancel_pending(); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/env/model.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/env/model.svh new file mode 100644 index 000000000..74975e1e2 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/env/model.svh @@ -0,0 +1,37 @@ +class model extends reference_model #(request_item, response_item); + `uvm_component_utils(model) + byte unsigned memory[`SPM_BYTES]; + bit initialized[`SPM_BYTES]; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void write(request_item req); + response_item rsp = response_item::type_id::create("expected"); + longint unsigned base = `SPM_BASE; + longint unsigned count = 64'd1 << req.size; + logic [64:0] finish = {1'b0, req.address} + count; + logic [64:0] limit = {1'b0, base} + `SPM_BYTES; + logic [`SPM_BITS/8-1:0] allowed = (`SPM_BITS / 8)'((64'd1 << count) - 1); + bit legal = count <= `SPM_BITS/8 && (req.address & (count-1)) == 0 && + req.address >= base && finish <= limit && + !(req.write && (req.read_only || (req.mask & ~allowed) != 0)); + int offset = req.address - base; + rsp.value = 0; + rsp.error = !legal; + if (legal) begin + for (int i = 0; i < count; i++) begin + if (req.write) begin + if (req.mask[i]) begin + memory[offset+i] = req.data[8*i+:8]; + initialized[offset+i] = 1; + end + end else begin + if (!initialized[offset+i]) + `uvm_fatal("UNINITIALIZED", "SPM read before byte initialization") + rsp.value[8*i+:8] = memory[offset+i]; + end + end + end + expected_ap.write(rsp); + endfunction +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/filelist.f b/arch/src/main/scala/framework/ip/spm/src/main/resources/filelist.f new file mode 100644 index 000000000..aa324abc6 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/filelist.f @@ -0,0 +1,8 @@ ++incdir+@RTL@/Memory ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv ++incdir+@RESOURCES@ +@RESOURCES@/interface.sv +@RESOURCES@/package.sv +-F @RTL@/Memory/filelist.f +@RESOURCES@/tb.sv diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/interface.sv b/arch/src/main/scala/framework/ip/spm/src/main/resources/interface.sv new file mode 100644 index 000000000..63a052ef2 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/interface.sv @@ -0,0 +1,27 @@ +`include "parameters.svh" +interface port_if ( + input logic clock, + input logic reset +); + logic request_valid, request_ready, response_valid, response_ready, busy; + logic [63:0] address; + logic [$clog2($clog2(`SPM_BITS/8)+2)-1:0] size; + logic write; + logic [`SPM_BITS-1:0] data; + logic [`SPM_BITS/8-1:0] mask; + logic read_only; + int unsigned stall; + logic cancel; + logic [`SPM_BITS-1:0] value; + logic error; + assert property (@(posedge clock) disable iff (reset) + request_valid && !request_ready |=> request_valid && $stable( + {address, size, write, data, mask, read_only} + )) + else $fatal(1, "SPM request changed under backpressure"); + assert property (@(posedge clock) disable iff (reset) + response_valid && !response_ready |=> response_valid && $stable( + {value, error} + )) + else $fatal(1, "SPM response changed under backpressure"); +endinterface diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/package.sv b/arch/src/main/scala/framework/ip/spm/src/main/resources/package.sv new file mode 100644 index 000000000..d7c844acb --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/package.sv @@ -0,0 +1,16 @@ +`include "parameters.svh" +package spm_pkg; + timeunit 1ns; timeprecision 1ps; + import uvm_pkg::*; + import verification_pkg::*; + `include "uvm_macros.svh" + `include "agent/item.svh" + `include "agent/driver.svh" + `include "agent/monitor.svh" + `include "agent/agent.svh" + `include "env/model.svh" + `include "env/coverage.svh" + `include "env/env.svh" + `include "seq/sequence.svh" + `include "tests/test.svh" +endpackage diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/seq/sequence.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/seq/sequence.svh new file mode 100644 index 000000000..61e840506 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/seq/sequence.svh @@ -0,0 +1,53 @@ +class stimulus extends uvm_sequence #(request_item); + `uvm_object_utils(stimulus) + int unsigned sent = 0; + function new(string name = "stimulus"); + super.new(name); + endfunction + task access (longint unsigned address, int size, bit write, logic [`SPM_BITS-1:0] data, + logic [`SPM_BITS/8-1:0] mask, bit read_only, int stall, bit cancel); + request_item req = request_item::type_id::create("request"); + start_item(req); + req.address = address; + req.size = size; + req.write = write; + req.data = data; + req.mask = mask; + req.read_only = read_only; + req.stall = stall; + req.cancel = cancel; + finish_item(req); + if (!cancel) sent++; + endtask + task body(); + int beat_bytes = `SPM_BITS / 8; + int maximum = $clog2(beat_bytes); + logic [`SPM_BITS-1:0] pattern; + for (int offset = 0; offset < `SPM_BYTES; offset += beat_bytes) begin + for (int byte_ = 0; byte_ < beat_bytes; byte_++) pattern[8*byte_+:8] = offset + byte_; + access (`SPM_BASE + offset, maximum, 1, pattern, '1, 0, 0, 0); + end + for (int size = 0; size <= maximum; size++) begin + for (int offset = 0; offset < `SPM_BYTES; offset += (1 << size)) + access (`SPM_BASE + offset, size, 0, 0, 0, 0, 3, 0); + for (int bit_ = 0; bit_ < (1 << size); bit_++) begin + access (`SPM_BASE + beat_bytes - (1 << size), size, 1, '1, 1 << bit_, 0, 3, 0); + access (`SPM_BASE + beat_bytes - (1 << size), size, 0, 0, 0, 0, 0, 0); + end + end + access (`SPM_BASE, maximum, 1, 0, 0, 0, 0, 0); + access (`SPM_BASE, maximum, 0, 0, 0, 0, 0, 0); + access (`SPM_BASE - 1, 0, 0, 0, 0, 0, 0, 0); + access (`SPM_BASE + `SPM_BYTES, 0, 0, 0, 0, 0, 3, 0); + access (64'hffffffffffffffff, 1, 0, 0, 0, 0, 0, 0); + access (`SPM_BASE + 1, 1, 0, 0, 0, 0, 3, 0); + access (`SPM_BASE, maximum + 1, 0, 0, 0, 0, 0, 0); + access (`SPM_BASE, 0, 1, 0, 2, 0, 0, 0); + access (`SPM_BASE, maximum, 1, 0, '1, 1, 3, 0); + access (`SPM_BASE, maximum, 0, 0, 0, 1, 0, 0); + access (`SPM_BASE, maximum, 1, '1, '1, 0, 0, 1); + access (`SPM_BASE, maximum, 0, 0, 0, 0, 0, 0); + access (`SPM_BASE, maximum, 0, 0, 0, 0, 0, 1); + access (`SPM_BASE, maximum, 0, 0, 0, 0, 3, 0); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/suite.toml b/arch/src/main/scala/framework/ip/spm/src/main/resources/suite.toml new file mode 100644 index 000000000..d513c6055 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/suite.toml @@ -0,0 +1,20 @@ +[model] +manifest = "" +library = "" + +[[tests]] +name = "spm" +top = "spm_tb" +test = "protocol_test" +filelist = "filelist.f" +timeout_ns = 1000000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +functional_groups = ["spm_pkg::coverage::behavior"] +sink_ready = 1 +sink_stall = 0 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/tb.sv b/arch/src/main/scala/framework/ip/spm/src/main/resources/tb.sv new file mode 100644 index 000000000..f4dcabc18 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/tb.sv @@ -0,0 +1,33 @@ +module spm_tb; + import uvm_pkg::*; + import spm_pkg::*; + logic clock = 0; + always #5 clock = ~clock; + clock_if timing (clock); + port_if bus ( + clock, + timing.reset + ); + Memory dut ( + .clock(clock), + .reset(bus.reset), + .io_port_request_valid(bus.request_valid), + .io_port_request_ready(bus.request_ready), + .io_port_request_bits_address(bus.address), + .io_port_request_bits_size(bus.size), + .io_port_request_bits_write(bus.write), + .io_port_request_bits_data(bus.data), + .io_port_request_bits_mask(bus.mask), + .io_readOnly(bus.read_only), + .io_busy(bus.busy), + .io_port_response_valid(bus.response_valid), + .io_port_response_ready(bus.response_ready), + .io_port_response_bits_data(bus.value), + .io_port_response_bits_error(bus.error) + ); + initial begin + uvm_config_db#(virtual clock_if)::set(null, "uvm_test_top*", "timing", timing); + uvm_config_db#(virtual port_if)::set(null, "uvm_test_top*", "vif", bus); + run_test("protocol_test"); + end +endmodule diff --git a/arch/src/main/scala/framework/ip/spm/src/main/resources/tests/test.svh b/arch/src/main/scala/framework/ip/spm/src/main/resources/tests/test.svh new file mode 100644 index 000000000..81a4001e5 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/resources/tests/test.svh @@ -0,0 +1,27 @@ +class protocol_test extends base_test; + `uvm_component_utils(protocol_test) + env bench; + reset_controller resetter; + virtual port_if vif; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + resetter = reset_controller::type_id::create("reset", this); + uvm_config_db#(reset_controller)::set(this, "env.agent.driver", "resetter", resetter); + bench = env::type_id::create("env", this); + if (!uvm_config_db#(virtual port_if)::get(this, "", "vif", vif)) + `uvm_fatal("INTERFACE", "Test interface is required") + endfunction + task execute(); + stimulus sequence_ = stimulus::type_id::create("sequence"); + resetter.pulse(4); + sequence_.start(bench.bus.seqr); + bench.scoreboard.wait_checked(sequence_.sent); + @(negedge vif.clock); + evidence::check(evidence::cancelled == 2, + "Reset did not cancel both outstanding SPM responses"); + evidence::check(bench.scoreboard.checked == sequence_.sent, "Comparison count mismatch"); + endtask +endclass diff --git a/arch/src/main/scala/framework/ip/spm/src/main/scala/Emit.scala b/arch/src/main/scala/framework/ip/spm/src/main/scala/Emit.scala new file mode 100644 index 000000000..377c846d0 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/scala/Emit.scala @@ -0,0 +1,24 @@ +package framework.ip.memory.spm + +import java.nio.file.{Files, Paths} + +object Emit extends App { + val Array("--config", config, "--output-dir", output) = args + val values = ujson.read(Files.readString(Paths.get(config))) + val p = Params(BigInt(values("base").str), values("bytes").num.toInt, values("dataBits").num.toInt) + val directory = Paths.get(output, "Memory") + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + new Memory(p), + args = Array("--target-dir", directory.toString, "--split-verilog") + ) + Files.writeString( + directory.resolve("parameters.svh"), + s"""`ifndef SPM_PARAMETERS +`define SPM_PARAMETERS +`define SPM_BASE 64'h${p.base.toString(16)} +`define SPM_BYTES ${p.bytes} +`define SPM_BITS ${p.dataBits} +`endif +""" + ) +} diff --git a/arch/src/main/scala/framework/ip/spm/src/main/scala/Interfaces.scala b/arch/src/main/scala/framework/ip/spm/src/main/scala/Interfaces.scala new file mode 100644 index 000000000..9a360f405 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/scala/Interfaces.scala @@ -0,0 +1,35 @@ +package framework.ip.memory.spm + +import chisel3._ +import chisel3.util._ +import framework.ip.memory.bank.BankSetParams + +/** One local byte-address window. This does not describe a DDR mapping. */ +case class Params(base: BigInt, bytes: Int, dataBits: Int = 128) { + require(dataBits >= 64 && dataBits % 8 == 0 && isPow2(dataBits / 8)) + val beatBytes = dataBits / 8 + require(base >= 0 && base % beatBytes == 0 && base + bytes <= (BigInt(1) << 64)) + require(bytes >= 2 * beatBytes && isPow2(bytes)) + val laneBits = log2Ceil(beatBytes) + val sizeBits = log2Ceil(laneBits + 2) + val bank = BankSetParams(dataBits, banks = 1, entriesPerBank = bytes / beatBytes) +} + +/** Naturally aligned 1..beatBytes access. Data and mask start at address, not the SRAM row. */ +class Request(p: Params) extends Bundle { + val address = UInt(64.W) + val size = UInt(p.sizeBits.W) + val write = Bool() + val data = UInt(p.dataBits.W) + val mask = UInt(p.beatBytes.W) +} + +class Response(p: Params) extends Bundle { + val data = UInt(p.dataBits.W) + val error = Bool() +} + +class Port(p: Params) extends Bundle { + val request = Decoupled(new Request(p)) + val response = Flipped(Decoupled(new Response(p))) +} diff --git a/arch/src/main/scala/framework/ip/spm/src/main/scala/Memory.scala b/arch/src/main/scala/framework/ip/spm/src/main/scala/Memory.scala new file mode 100644 index 000000000..e28c51887 --- /dev/null +++ b/arch/src/main/scala/framework/ip/spm/src/main/scala/Memory.scala @@ -0,0 +1,60 @@ +package framework.ip.memory.spm + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.ip.memory.bank.Bank + +/** + * One outstanding transaction on one physical read/write port. Writes acknowledge after + * SRAM acceptance; reads return after the synchronous read. Responses hold until accepted. + * Reset cancels protocol state; SRAM contents, including already accepted writes, remain. + */ +@instantiable +class Memory(p: Params) extends Module { + + @public + val io = IO(new Bundle { + val port = Flipped(new Port(p)) + val readOnly = Input(Bool()) + val busy = Output(Bool()) + }) + + val bank = Instantiate(new Bank(p.bank)) + val idle :: memory :: rejected :: Nil = Enum(3) + val state = RegInit(idle) + val offset = Reg(UInt(p.laneBits.W)) + val savedMask = Reg(UInt(p.beatBytes.W)) + val request = io.port.request.bits + val count = 1.U(64.W) << request.size + val end = request.address +& count + + val widthMask = MuxLookup(request.size, 0.U(p.beatBytes.W))( + (0 to p.laneBits).map(size => size.U -> ((BigInt(1) << (1 << size)) - 1).U(p.beatBytes.W)) + ) + + val legal = request.size <= p.laneBits.U && (request.address & (count - 1.U)) === 0.U && + request.address >= p.base.U && end <= (p.base + p.bytes).U && + !(request.write && (io.readOnly || (request.mask & ~widthMask) =/= 0.U)) + + val lane = request.address(p.laneBits - 1, 0) + io.port.request.ready := state === idle && !reset.asBool && Mux(legal, bank.io.request.ready, true.B) + io.busy := state =/= idle + bank.io.request.valid := state === idle && !reset.asBool && io.port.request.valid && legal + bank.io.request.bits.addr := (request.address - p.base.U) >> p.laneBits + bank.io.request.bits.write := request.write + bank.io.request.bits.data := request.data << (lane << 3) + bank.io.request.bits.mask := request.mask << lane + bank.io.request.bits.tag := 0.U + when(io.port.request.fire) { + offset := lane + savedMask := widthMask + state := Mux(legal, memory, rejected) + } + val dataMask = Cat((0 until p.beatBytes).reverse.map(i => Fill(8, savedMask(i)))) + io.port.response.valid := !reset.asBool && (state === rejected || (state === memory && bank.io.response.valid)) + io.port.response.bits.data := Mux(state === rejected, 0.U, (bank.io.response.bits.data >> (offset << 3)) & dataMask) + io.port.response.bits.error := state === rejected || bank.io.response.bits.error + bank.io.response.ready := state === memory && io.port.response.ready && !reset.asBool + when(io.port.response.fire)(state := idle) +} diff --git a/arch/src/main/scala/framework/lib/arbiter/Arbiter.scala b/arch/src/main/scala/framework/lib/arbiter/Arbiter.scala new file mode 100644 index 000000000..50c17eb75 --- /dev/null +++ b/arch/src/main/scala/framework/lib/arbiter/Arbiter.scala @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 +// Arbitration follows chisel3.util.Arbiter and RRArbiter. +package framework.lib.arbiter + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class Arbiter[T <: Data](gen: T, n: Int) extends Module { + @public + val io = IO(new ArbiterIO(gen, n)) + + io.chosen := (n - 1).U + io.out.bits := io.in(n - 1).bits + for (i <- n - 2 to 0 by -1) { + when(io.in(i).valid) { + io.chosen := i.U + io.out.bits := io.in(i).bits + } + } + val preceding = io.in.map(_.valid).scanLeft(false.B)(_ || _).dropRight(1) + for ((input, busy) <- io.in.zip(preceding)) input.ready := !busy && io.out.ready + io.out.valid := io.in.map(_.valid).reduce(_ || _) +} + +@instantiable +class RRArbiter[T <: Data](gen: T, n: Int) extends Module { + @public + val io = IO(new ArbiterIO(gen, n)) + + val lastGrant = RegInit(0.U(math.max(1, log2Ceil(n)).W)) + val mask = (0 until n).map(_.U > lastGrant) + val masked = io.in.zip(mask).map { case (input, eligible) => input.valid && eligible } + val requests = masked ++ io.in.map(_.valid) + val grant = requests.scanLeft(false.B)(_ || _).dropRight(1).map(!_) + + io.chosen := (n - 1).U + for (i <- n - 2 to 0 by -1) when(io.in(i).valid)(io.chosen := i.U) + for (i <- n - 1 to 1 by -1) when(masked(i))(io.chosen := i.U) + for (i <- 0 until n) io.in(i).ready := (grant(i) && mask(i) || grant(i + n)) && io.out.ready + io.out.valid := io.in(io.chosen).valid + io.out.bits := io.in(io.chosen).bits + when(io.out.fire)(lastGrant := io.chosen) +} diff --git a/arch/src/main/scala/framework/lib/arbiter/build.sc b/arch/src/main/scala/framework/lib/arbiter/build.sc new file mode 100644 index 000000000..c17f7200e --- /dev/null +++ b/arch/src/main/scala/framework/lib/arbiter/build.sc @@ -0,0 +1,11 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("Arbiter.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/arith/src/main/scala/DivRem.scala b/arch/src/main/scala/framework/lib/arith/DivRem.scala similarity index 99% rename from arch/src/main/scala/framework/arith/src/main/scala/DivRem.scala rename to arch/src/main/scala/framework/lib/arith/DivRem.scala index 494378649..015963f6a 100644 --- a/arch/src/main/scala/framework/arith/src/main/scala/DivRem.scala +++ b/arch/src/main/scala/framework/lib/arith/DivRem.scala @@ -1,4 +1,4 @@ -package framework.arith +package framework.lib.arith import chisel3._ import chisel3.util._ diff --git a/arch/src/main/scala/framework/lib/arith/Fma.scala b/arch/src/main/scala/framework/lib/arith/Fma.scala new file mode 100644 index 000000000..8c1947c0e --- /dev/null +++ b/arch/src/main/scala/framework/lib/arith/Fma.scala @@ -0,0 +1,89 @@ +// See LICENSE.Berkeley and LICENSE.SiFive for license details. +package framework.lib.arith + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class Fma(latency: Int, expWidth: Int, sigWidth: Int) extends Module { + override def desiredName = s"Fma_l${latency}_e${expWidth}_s$sigWidth" + require(latency <= 4) + + @public + val io = IO(new Bundle { + val validin = Input(Bool()) + val op = Input(Bits(2.W)) + val a = Input(Bits((expWidth + sigWidth + 1).W)) + val b = Input(Bits((expWidth + sigWidth + 1).W)) + val c = Input(Bits((expWidth + sigWidth + 1).W)) + val roundingMode = Input(UInt(3.W)) + val detectTininess = Input(UInt(1.W)) + val out = Output(Bits((expWidth + sigWidth + 1).W)) + val exceptionFlags = Output(Bits(5.W)) + val validout = Output(Bool()) + }) + + //------------------------------------------------------------------------ + //------------------------------------------------------------------------ + + val mulAddRecFNToRaw_preMul = Instantiate(new hardfloat.MulAddRecFNToRaw_preMul(expWidth, sigWidth)) + val mulAddRecFNToRaw_postMul = Instantiate(new hardfloat.MulAddRecFNToRaw_postMul(expWidth, sigWidth)) + + mulAddRecFNToRaw_preMul.io.op := io.op + mulAddRecFNToRaw_preMul.io.a := io.a + mulAddRecFNToRaw_preMul.io.b := io.b + mulAddRecFNToRaw_preMul.io.c := io.c + + // Isolate preMul's unpack/alignment logic from the multiplier-add stage. + val preMulRegs = if (latency >= 3) 1 else 0 + val preMulMulAddA = Pipe(io.validin, mulAddRecFNToRaw_preMul.io.mulAddA, preMulRegs) + val preMulMulAddB = Pipe(io.validin, mulAddRecFNToRaw_preMul.io.mulAddB, preMulRegs) + val preMulMulAddC = Pipe(io.validin, mulAddRecFNToRaw_preMul.io.mulAddC, preMulRegs) + val preMulToPostMul = Pipe(io.validin, mulAddRecFNToRaw_preMul.io.toPostMul, preMulRegs) + + // Keep the full-precision product unrounded while separating multiplication + // from the final addition. The aligned addend and post-processing metadata + // must cross this stage with the product to preserve fused FMA semantics. + val mulRegs = if (latency == 4) 1 else 0 + val mulResult = preMulMulAddA.bits * preMulMulAddB.bits + val mulResultPipe = Pipe(preMulToPostMul.valid, mulResult, mulRegs) + val mulAddCPipe = Pipe(preMulToPostMul.valid, preMulMulAddC.bits, mulRegs) + val mulToPostMul = Pipe(preMulToPostMul.valid, preMulToPostMul.bits, mulRegs) + val mulAddResult = mulResultPipe.bits +& mulAddCPipe.bits + + val valid_stage0 = Wire(Bool()) + val roundingMode_stage0 = Wire(UInt(3.W)) + val detectTininess_stage0 = Wire(UInt(1.W)) + + val postmul_regs = if (latency > 0) 1 else 0 + mulAddRecFNToRaw_postMul.io.fromPreMul := Pipe(mulToPostMul.valid, mulToPostMul.bits, postmul_regs).bits + mulAddRecFNToRaw_postMul.io.mulAddResult := Pipe(mulToPostMul.valid, mulAddResult, postmul_regs).bits + mulAddRecFNToRaw_postMul.io.roundingMode := Pipe( + io.validin, + io.roundingMode, + preMulRegs + mulRegs + postmul_regs + ).bits + roundingMode_stage0 := Pipe(io.validin, io.roundingMode, preMulRegs + mulRegs + postmul_regs).bits + detectTininess_stage0 := Pipe(io.validin, io.detectTininess, preMulRegs + mulRegs + postmul_regs).bits + valid_stage0 := Pipe(io.validin, false.B, preMulRegs + mulRegs + postmul_regs).valid + + //------------------------------------------------------------------------ + //------------------------------------------------------------------------ + + val roundRawFNToRecFN = Instantiate(new hardfloat.RoundRawFNToRecFN(expWidth, sigWidth, 0)) + + val round_regs = if (latency >= 2) 1 else 0 + roundRawFNToRecFN.io.invalidExc := Pipe(valid_stage0, mulAddRecFNToRaw_postMul.io.invalidExc, round_regs).bits + roundRawFNToRecFN.io.in := Pipe(valid_stage0, mulAddRecFNToRaw_postMul.io.rawOut, round_regs).bits + roundRawFNToRecFN.io.roundingMode := Pipe(valid_stage0, roundingMode_stage0, round_regs).bits + roundRawFNToRecFN.io.detectTininess := Pipe(valid_stage0, detectTininess_stage0, round_regs).bits + io.validout := Pipe(valid_stage0, false.B, round_regs).valid + + roundRawFNToRecFN.io.infiniteExc := false.B + + io.out := roundRawFNToRecFN.io.out + io.exceptionFlags := roundRawFNToRecFN.io.exceptionFlags +} diff --git a/arch/src/main/scala/framework/lib/arith/IntToFloat.scala b/arch/src/main/scala/framework/lib/arith/IntToFloat.scala new file mode 100644 index 000000000..8a9f08dbf --- /dev/null +++ b/arch/src/main/scala/framework/lib/arith/IntToFloat.scala @@ -0,0 +1,109 @@ +// See LICENSE.Berkeley and LICENSE.SiFive for license details. +package framework.lib.arith + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class IntToFloat(intWidth: Int, expWidth: Int, sigWidth: Int) extends Module { + + @public + val io = IO(new Bundle { + val validin = Input(Bool()) + val signedIn = Input(Bool()) + val in = Input(Bits(intWidth.W)) + val roundingMode = Input(UInt(3.W)) + val detectTininess = Input(UInt(1.W)) + val out = Output(Bits((expWidth + sigWidth + 1).W)) + val exceptionFlags = Output(Bits(5.W)) + val validout = Output(Bool()) + }) + + val rawExpWidth = log2Up(intWidth) + 1 + val extIntWidth = 1 << (rawExpWidth - 1) + val normalizationLevels = rawExpWidth - 1 + val firstStageLevels = 2 + val secondStageLevels = normalizationLevels - firstStageLevels + require(secondStageLevels > 0) + + val sign = io.signedIn && io.in(intWidth - 1) + val absIn = Mux(sign, -io.in.asUInt, io.in.asUInt) + val extAbsIn = (0.U(extIntWidth.W) ## absIn)(extIntWidth - 1, 0) + + // Complete the 32- and 16-bit normalization decisions before the cut. + val firstNormStages = Wire(Vec(firstStageLevels + 1, UInt(extIntWidth.W))) + val firstNormDist = Wire(Vec(firstStageLevels, Bool())) + firstNormStages(0) := extAbsIn + for (level <- 0 until firstStageLevels) { + val shiftAmt = extIntWidth >> (level + 1) + val upperHalfIsZero = + !firstNormStages(level)(extIntWidth - 1, extIntWidth - shiftAmt).orR + firstNormStages(level + 1) := Mux( + upperHalfIsZero, + (firstNormStages(level) << shiftAmt)(extIntWidth - 1, 0), + firstNormStages(level) + ) + firstNormDist(firstStageLevels - 1 - level) := upperHalfIsZero + } + + val midNormStage = RegEnable(firstNormStages(firstStageLevels), io.validin) + val midNormDist = RegEnable(firstNormDist.asUInt, io.validin) + val midSign = RegEnable(sign, io.validin) + val midRoundingMode = RegEnable(io.roundingMode, io.validin) + val midDetectTininess = RegEnable(io.detectTininess, io.validin) + val midValid = RegNext(io.validin, false.B) + + val secondNormStages = Wire(Vec(secondStageLevels + 1, UInt(extIntWidth.W))) + val secondNormDist = Wire(Vec(secondStageLevels, Bool())) + secondNormStages(0) := midNormStage + for (offset <- 0 until secondStageLevels) { + val level = firstStageLevels + offset + val shiftAmt = extIntWidth >> (level + 1) + val upperHalfIsZero = + !secondNormStages(offset)(extIntWidth - 1, extIntWidth - shiftAmt).orR + secondNormStages(offset + 1) := Mux( + upperHalfIsZero, + (secondNormStages(offset) << shiftAmt)(extIntWidth - 1, 0), + secondNormStages(offset) + ) + secondNormDist(secondStageLevels - 1 - offset) := upperHalfIsZero + } + + val adjustedNormDist = Cat(midNormDist, secondNormDist.asUInt) + val sig = secondNormStages(secondStageLevels)(extIntWidth - 1, extIntWidth - intWidth) + val intAsRawFloat = Wire(new hardfloat.RawFloat(rawExpWidth, intWidth)) + intAsRawFloat.isNaN := false.B + intAsRawFloat.isInf := false.B + intAsRawFloat.isZero := !sig(intWidth - 1) + intAsRawFloat.sign := midSign + intAsRawFloat.sExp := (2.U(2.W) ## ~adjustedNormDist).zext + intAsRawFloat.sig := sig + + // Isolate the remaining normalization and raw-float construction from the + // target-format rounding and packing logic. + val roundInput = RegEnable(intAsRawFloat, midValid) + val roundRoundingMode = RegEnable(midRoundingMode, midValid) + val roundDetectTininess = RegEnable(midDetectTininess, midValid) + val roundValid = RegNext(midValid, false.B) + + val roundAnyRawFNToRecFN = Instantiate(new hardfloat.RoundAnyRawFNToRecFN( + rawExpWidth, + intWidth, + expWidth, + sigWidth, + hardfloat.consts.flRoundOpt_sigMSBitAlwaysZero | hardfloat.consts.flRoundOpt_neverUnderflows + )) + + roundAnyRawFNToRecFN.io.invalidExc := false.B + roundAnyRawFNToRecFN.io.infiniteExc := false.B + roundAnyRawFNToRecFN.io.in := roundInput + roundAnyRawFNToRecFN.io.roundingMode := roundRoundingMode + roundAnyRawFNToRecFN.io.detectTininess := roundDetectTininess + + io.out := roundAnyRawFNToRecFN.io.out + io.exceptionFlags := roundAnyRawFNToRecFN.io.exceptionFlags + io.validout := roundValid +} diff --git a/arch/src/main/scala/framework/lib/arith/LICENSE.Berkeley b/arch/src/main/scala/framework/lib/arith/LICENSE.Berkeley new file mode 100644 index 000000000..7cff15e40 --- /dev/null +++ b/arch/src/main/scala/framework/lib/arith/LICENSE.Berkeley @@ -0,0 +1,24 @@ +Copyright (c) 2012-2014, The Regents of the University of California +(Regents). All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the Regents nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING +OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED +HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. diff --git a/arch/src/main/scala/framework/lib/arith/LICENSE.SiFive b/arch/src/main/scala/framework/lib/arith/LICENSE.SiFive new file mode 100644 index 000000000..7e7093375 --- /dev/null +++ b/arch/src/main/scala/framework/lib/arith/LICENSE.SiFive @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2017 SiFive, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/arch/src/main/scala/framework/arith/src/main/scala/Multiply.scala b/arch/src/main/scala/framework/lib/arith/Multiply.scala similarity index 96% rename from arch/src/main/scala/framework/arith/src/main/scala/Multiply.scala rename to arch/src/main/scala/framework/lib/arith/Multiply.scala index 9e025a715..bac5e2960 100644 --- a/arch/src/main/scala/framework/arith/src/main/scala/Multiply.scala +++ b/arch/src/main/scala/framework/lib/arith/Multiply.scala @@ -1,4 +1,4 @@ -package framework.arith +package framework.lib.arith import chisel3._ import chisel3.util._ @@ -16,7 +16,8 @@ class MultiplyRequest extends Bundle { @instantiable class Multiply extends Module { - @public val io = IO(new Bundle { + @public + val io = IO(new Bundle { val request = Flipped(Decoupled(new MultiplyRequest)) val response = Decoupled(UInt(64.W)) }) diff --git a/arch/src/main/scala/framework/lib/arith/build.sc b/arch/src/main/scala/framework/lib/arith/build.sc new file mode 100644 index 000000000..6046890ef --- /dev/null +++ b/arch/src/main/scala/framework/lib/arith/build.sc @@ -0,0 +1,14 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("DivRem.scala")), + PathRef(millSourcePath / os.RelPath("Fma.scala")), + PathRef(millSourcePath / os.RelPath("IntToFloat.scala")), + PathRef(millSourcePath / os.RelPath("Multiply.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/lib/build.sc b/arch/src/main/scala/framework/lib/build.sc new file mode 100644 index 000000000..b9aee3969 --- /dev/null +++ b/arch/src/main/scala/framework/lib/build.sc @@ -0,0 +1,22 @@ +import mill._ +import $file.arbiter.{build => arbiter_build} +import $file.arith.{build => arith_build} +import $file.dpi.{build => dpi_build} +import $file.hash.{build => hash_build} +import $file.queue.{build => queue_build} + +def sources = T.sources( + arbiter_build.sources() ++ + arith_build.sources() ++ + dpi_build.sources() ++ + hash_build.sources() ++ + queue_build.sources() +) + +def emitSources = T.sources( + arbiter_build.emitSources() ++ + arith_build.emitSources() ++ + dpi_build.emitSources() ++ + hash_build.emitSources() ++ + queue_build.emitSources() +) diff --git a/arch/src/main/scala/framework/dpi/DpiGuard.scala b/arch/src/main/scala/framework/lib/dpi/DpiGuard.scala similarity index 96% rename from arch/src/main/scala/framework/dpi/DpiGuard.scala rename to arch/src/main/scala/framework/lib/dpi/DpiGuard.scala index f85a18c32..7e28200d2 100644 --- a/arch/src/main/scala/framework/dpi/DpiGuard.scala +++ b/arch/src/main/scala/framework/lib/dpi/DpiGuard.scala @@ -1,4 +1,4 @@ -package framework.dpi +package framework.lib.dpi object DpiGuard { private def wrapUnless(macroName: String, body: String): String = diff --git a/arch/src/main/scala/framework/lib/dpi/build.sc b/arch/src/main/scala/framework/lib/dpi/build.sc new file mode 100644 index 000000000..c54afe76c --- /dev/null +++ b/arch/src/main/scala/framework/lib/dpi/build.sc @@ -0,0 +1,11 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("DpiGuard.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/lib/hash/BankHash.scala b/arch/src/main/scala/framework/lib/hash/BankHash.scala new file mode 100644 index 000000000..21f26b0ad --- /dev/null +++ b/arch/src/main/scala/framework/lib/hash/BankHash.scala @@ -0,0 +1,75 @@ +package framework.lib.hash + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} +import chisel3.util._ +import framework.acc.memdomain.configs.MemDomainParam + +class BankHashWrite(val b: MemDomainParam) extends Bundle { + val addr = UInt(log2Ceil(b.bankEntries).W) + val mask = Vec(b.bankMaskLen, Bool()) + val data = UInt(b.bankWidth.W) +} + +@instantiable +class BankHashMonitor(val b: MemDomainParam) extends Module { + require(b.bankWidth == 128, "bank hash requires 128-bit bank rows") + require(b.bankMaskLen == 16, "bank hash requires byte write masks") + + @public + val io = IO(new Bundle { + val bind = Input(Bool()) + val write = Flipped(Decoupled(new BankHashWrite(b))) + val statusHash = Output(UInt(32.W)) + }) + + val shadow = SyncReadMem(b.bankEntries, UInt(128.W)) + val validRows = RegInit(0.U(b.bankEntries.W)) + val generation = RegInit(0.U(8.W)) + val statusHash = RegInit(0.U(32.W)) + + def rotateLeft(value: UInt, amount: Int): UInt = + Cat(value(31 - amount, 0), value(31, 32 - amount)) + + def rowHash(addr: UInt, row: UInt): UInt = { + val address = addr.pad(32) + val mixed = row(31, 0) ^ rotateLeft(row(63, 32), 7) ^ rotateLeft(row(95, 64), 13) ^ + rotateLeft(row(127, 96), 21) ^ rotateLeft(address, 11) + Mux(row.orR, mixed, 0.U) + } + + val writeValid = RegInit(false.B) + val writeBits = Reg(new BankHashWrite(b)) + val writeGeneration = Reg(UInt(generation.getWidth.W)) + val commit = writeValid && writeGeneration === generation && !io.bind + io.write.ready := !writeValid && !io.bind + val oldRow = Wire(UInt(128.W)) + val oldBytes = oldRow.asTypeOf(Vec(16, UInt(8.W))) + val writeBytes = writeBits.data.asTypeOf(Vec(16, UInt(8.W))) + val newBytes = VecInit((0 until 16).map(i => Mux(writeBits.mask(i), writeBytes(i), oldBytes(i)))) + val newRow = newBytes.asUInt + val nextHash = statusHash - rowHash(writeBits.addr, oldRow) + rowHash(writeBits.addr, newRow) + + val readData = + shadow.readWrite(Mux(commit, writeBits.addr, io.write.bits.addr), newRow, io.write.fire || commit, commit) + oldRow := Mux(validRows(writeBits.addr), readData, 0.U) + + when(io.write.fire) { + writeValid := true.B + writeBits := io.write.bits + writeGeneration := generation + } + + io.statusHash := statusHash + + when(io.bind) { + generation := generation + 1.U + statusHash := 0.U + writeValid := false.B + validRows := 0.U + }.elsewhen(commit) { + validRows := validRows | UIntToOH(writeBits.addr, b.bankEntries) + statusHash := nextHash + writeValid := false.B + } +} diff --git a/arch/src/main/scala/framework/lib/hash/build.sc b/arch/src/main/scala/framework/lib/hash/build.sc new file mode 100644 index 000000000..ee1b17b54 --- /dev/null +++ b/arch/src/main/scala/framework/lib/hash/build.sc @@ -0,0 +1,11 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("BankHash.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/lib/hash/src/csrc/monitor.cpp b/arch/src/main/scala/framework/lib/hash/src/csrc/monitor.cpp new file mode 100644 index 000000000..6332142b9 --- /dev/null +++ b/arch/src/main/scala/framework/lib/hash/src/csrc/monitor.cpp @@ -0,0 +1,110 @@ +#include "VBankHashMonitor.h" +#include "verilated.h" +#include +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + Verilated::commandArgs(argc, argv); + if (argc != 2) + throw std::runtime_error("Expected bank row count"); + const unsigned rows = std::stoul(argv[1]); + VBankHashMonitor dut; + std::vector> contents(rows); + auto tick = [&] { + dut.clock = 0; + dut.eval(); + dut.clock = 1; + dut.eval(); + dut.clock = 0; + dut.eval(); + }; + auto check = [&](bool value, const char *message) { + if (!value) + throw std::runtime_error(message); + }; + auto rotate = [](uint32_t value, unsigned n) { + return (value << n) | (value >> (32 - n)); + }; + auto expected = [&] { + uint32_t hash = 0; + for (unsigned row = 0; row < rows; ++row) { + const auto &v = contents[row]; + if (v[0] | v[1] | v[2] | v[3]) + hash += v[0] ^ rotate(v[1], 7) ^ rotate(v[2], 13) ^ rotate(v[3], 21) ^ + rotate(row, 11); + } + return hash; + }; + auto mask = [&](uint16_t bits) { +#define BYTE(i) dut.io_write_bits_mask_##i = (bits >> i) & 1 + BYTE(0); + BYTE(1); + BYTE(2); + BYTE(3); + BYTE(4); + BYTE(5); + BYTE(6); + BYTE(7); + BYTE(8); + BYTE(9); + BYTE(10); + BYTE(11); + BYTE(12); + BYTE(13); + BYTE(14); + BYTE(15); +#undef BYTE + }; + dut.reset = 1; + dut.io_bind = 0; + dut.io_write_valid = 0; + tick(); + tick(); + dut.reset = 0; + tick(); + uint32_t random = 0x12345678; + for (unsigned round = 0; round < 3; ++round) { + for (unsigned row = 0; row < rows; ++row) { + check(dut.io_write_ready, "Write did not become ready"); + random = random * 1664525 + 1013904223; + const uint16_t bytes = round == 0 ? 0xffff : random; + dut.io_write_bits_addr = row; + for (unsigned word = 0; word < 4; ++word) { + random = random * 1664525 + 1013904223; + dut.io_write_bits_data[word] = random; + for (unsigned byte = 0; byte < 4; ++byte) { + uint32_t m = 0xffu << (byte * 8); + if ((bytes >> (word * 4 + byte)) & 1) + contents[row][word] = (contents[row][word] & ~m) | (random & m); + } + } + mask(bytes); + dut.io_write_valid = 1; + tick(); + check(!dut.io_write_ready, "Pending write did not apply backpressure"); + dut.io_write_valid = 0; + tick(); + check(dut.io_statusHash == expected(), "Masked write hash mismatch"); + } + if (round == 0) + continue; + dut.io_write_valid = 1; + tick(); + dut.io_bind = 1; + dut.eval(); + check(!dut.io_write_ready, "Bind accepted a concurrent write"); + tick(); + dut.io_write_valid = 0; + dut.io_bind = 0; + contents.assign(rows, {}); + tick(); + check(dut.io_statusHash == 0, "Bind did not cancel the pending write"); + } + std::puts("PASS: all rows, masked overwrite, rebind, cancellation and " + "backpressure"); + dut.final(); +} diff --git a/arch/src/main/scala/framework/mem-core/queue/src/main/scala/Queue.scala b/arch/src/main/scala/framework/lib/queue/Queue.scala similarity index 98% rename from arch/src/main/scala/framework/mem-core/queue/src/main/scala/Queue.scala rename to arch/src/main/scala/framework/lib/queue/Queue.scala index 7d39d32da..edf19148c 100644 --- a/arch/src/main/scala/framework/mem-core/queue/src/main/scala/Queue.scala +++ b/arch/src/main/scala/framework/lib/queue/Queue.scala @@ -1,4 +1,4 @@ -package memcore.memory.queue +package framework.lib.queue import chisel3._ import chisel3.experimental.hierarchy.{instantiable, public} diff --git a/arch/src/main/scala/framework/lib/queue/build.sc b/arch/src/main/scala/framework/lib/queue/build.sc new file mode 100644 index 000000000..16493b39a --- /dev/null +++ b/arch/src/main/scala/framework/lib/queue/build.sc @@ -0,0 +1,11 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("Queue.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/mem-core/axi4/build.sc b/arch/src/main/scala/framework/mem-core/axi4/build.sc deleted file mode 100644 index 54655ecf4..000000000 --- a/arch/src/main/scala/framework/mem-core/axi4/build.sc +++ /dev/null @@ -1,20 +0,0 @@ -import mill._ -import mill.scalalib._ - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object axi4 extends SbtModule { - override def moduleDeps = Seq(queue) - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") - override def mainClass = Some("memcore.bus.axi4.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Emit.scala deleted file mode 100644 index c3031c191..000000000 --- a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Emit.scala +++ /dev/null @@ -1,9 +0,0 @@ -package memcore.bus.axi4 - -object Emit extends App { - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Interconnect(Params(), masters = 2), - firtoolOpts = args, - args = Array("--target-dir", "build/Interconnect", "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Interface.scala b/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Interface.scala deleted file mode 100644 index e259e89bd..000000000 --- a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Interface.scala +++ /dev/null @@ -1,44 +0,0 @@ -package memcore.bus.axi4 - -import chisel3._ -import chisel3.util._ - -class Address(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val addr = UInt(p.addressBits.W) - val len = UInt(8.W) - val size = UInt(3.W) - val burst = UInt(2.W) - val lock = Bool() - val cache = UInt(4.W) - val prot = UInt(3.W) - val qos = UInt(4.W) - val region = UInt(4.W) -} - -class WriteData(p: Params) extends Bundle { - val data = UInt(p.dataBits.W) - val strb = UInt(p.bytes.W) - val last = Bool() -} - -class ReadData(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val data = UInt(p.dataBits.W) - val resp = UInt(2.W) - val last = Bool() -} - -class WriteResponse(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val resp = UInt(2.W) -} - -/** AXI4 memory-mapped master. Optional user sidebands are absent. */ -class Port(p: Params) extends Bundle { - val aw = Decoupled(new Address(p)) - val w = Decoupled(new WriteData(p)) - val b = Flipped(Decoupled(new WriteResponse(p))) - val ar = Decoupled(new Address(p)) - val r = Flipped(Decoupled(new ReadData(p))) -} diff --git a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Params.scala b/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Params.scala deleted file mode 100644 index 775a7a17f..000000000 --- a/arch/src/main/scala/framework/mem-core/axi4/src/main/scala/Params.scala +++ /dev/null @@ -1,7 +0,0 @@ -package memcore.bus.axi4 - -case class Params(addressBits: Int = 44, dataBits: Int = 128, idBits: Int = 4) { - require(addressBits > 0 && idBits > 0) - require(dataBits >= 8 && dataBits <= 1024 && (dataBits & (dataBits - 1)) == 0) - val bytes = dataBits / 8 -} diff --git a/arch/src/main/scala/framework/mem-core/axis/build.sc b/arch/src/main/scala/framework/mem-core/axis/build.sc deleted file mode 100644 index c49295076..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/build.sc +++ /dev/null @@ -1,11 +0,0 @@ -import mill._ -import mill.scalalib._ - -object axis extends SbtModule { - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") - override def mainClass = Some("memcore.bus.axi.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/arbiter.rs b/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/arbiter.rs deleted file mode 100644 index bb72e1a9e..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/arbiter.rs +++ /dev/null @@ -1,49 +0,0 @@ -#[derive(Default)] -pub(crate) struct ArbiterModel { - selected: Option, -} - -impl ArbiterModel { - pub(crate) fn step(&mut self, valid: u32, ready: bool, last: u32) -> Option { - let selected = self - .selected - .or_else(|| (valid != 0).then(|| valid.trailing_zeros())); - let selected = selected?; - if valid & (1 << selected) == 0 { - return None; - } - - if ready { - if last & (1 << selected) != 0 { - self.selected = None; - } else { - self.selected = Some(selected); - } - } - Some(selected) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn locks_until_last_then_rearbitrates() { - let mut model = ArbiterModel::default(); - - assert_eq!(model.step(0b11, true, 0b00), Some(0)); - assert_eq!(model.step(0b11, true, 0b10), Some(0)); - assert_eq!(model.step(0b11, true, 0b01), Some(0)); - assert_eq!(model.step(0b10, true, 0b10), Some(1)); - assert_eq!(model.step(0b00, true, 0b00), None); - } - - #[test] - fn preserves_selection_during_backpressure() { - let mut model = ArbiterModel::default(); - - assert_eq!(model.step(0b10, false, 0b00), Some(1)); - assert_eq!(model.step(0b11, true, 0b01), Some(0)); - } -} diff --git a/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/model.rs b/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/model.rs deleted file mode 100644 index 9faf1edb3..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/csrc/src/model.rs +++ /dev/null @@ -1,50 +0,0 @@ -use std::collections::VecDeque; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Beat { - pub(crate) data: u32, - pub(crate) keep: u8, - pub(crate) last: bool, -} - -#[derive(Default)] -pub(crate) struct AxiSModel { - beats: VecDeque, -} - -impl AxiSModel { - pub(crate) fn push(&mut self, beat: Beat) { - self.beats.push_back(beat); - } - - pub(crate) fn pop(&mut self) -> Option { - self.beats.pop_front() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn preserves_beat_order_and_packet_boundaries() { - let mut model = AxiSModel::default(); - let first = Beat { - data: 0x1234_5678, - keep: 0xf, - last: false, - }; - let second = Beat { - data: 0x9abc_def0, - keep: 0x3, - last: true, - }; - - model.push(first); - model.push(second); - - assert_eq!(model.pop(), Some(first)); - assert_eq!(model.pop(), Some(second)); - assert_eq!(model.pop(), None); - } -} diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter.f b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter.f deleted file mode 100644 index bad3c5bf4..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter.f +++ /dev/null @@ -1,5 +0,0 @@ -+incdir+@RESOURCES@ -@RESOURCES@/arbiter_interface.sv -@RESOURCES@/arbiter_package.sv --F @RTL@/PacketArbiter/filelist.f -@RESOURCES@/arbiter_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_checker.svh b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_checker.svh deleted file mode 100644 index a11f577cd..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_checker.svh +++ /dev/null @@ -1,47 +0,0 @@ -class arbiter_checker extends uvm_component; - `uvm_component_utils(arbiter_checker) - - virtual arbiter_if vif; - chandle model; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual arbiter_if)::get(this, "", "vif", vif)) begin - `uvm_fatal("VIF", "checker requires vif") - end - model = axis_arbiter_create(); - if (model == null) `uvm_fatal("MODEL", "failed to create arbiter model") - endfunction - - task run_phase(uvm_phase phase); - int unsigned selected; - bit expected_valid; - bit [1:0] expected_ready; - - forever begin - @(posedge vif.clock); - if (!vif.reset) begin - selected = axis_arbiter_step(model, vif.in_valid, vif.out_ready, vif.in_last); - expected_valid = selected != 32'hffffffff; - expected_ready = expected_valid && vif.out_ready ? 2'b01 << selected : 2'b00; - - if (vif.out_valid !== expected_valid) `uvm_fatal("VALID", "output valid mismatch") - if (vif.in_ready !== expected_ready) `uvm_fatal("READY", "input ready mismatch") - if (expected_valid) begin - if (vif.out_data !== vif.in_data[selected]) `uvm_fatal("DATA", "selected data mismatch") - if (vif.out_keep !== vif.in_keep[selected]) `uvm_fatal("KEEP", "selected keep mismatch") - if (vif.out_last !== vif.in_last[selected]) `uvm_fatal("LAST", "selected last mismatch") - end - end - end - endtask - - function void final_phase(uvm_phase phase); - axis_arbiter_destroy(model); - super.final_phase(phase); - endfunction -endclass diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_driver.svh b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_driver.svh deleted file mode 100644 index 9b8240145..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_driver.svh +++ /dev/null @@ -1,80 +0,0 @@ -class driver extends uvm_component; - `uvm_component_utils(driver) - - virtual arbiter_if vif; - bit done; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual arbiter_if)::get(this, "", "vif", vif)) begin - `uvm_fatal("VIF", "driver requires vif") - end - endfunction - - task run_phase(uvm_phase phase); - bit [31:0] data0[3] = '{32'h00000000, 32'hffffffff, 32'haaaaaaaa}; - bit [31:0] data1[3] = '{32'hffffffff, 32'h00000000, 32'h55555555}; - bit [3:0] keep0[3] = '{4'h1, 4'hf, 4'h5}; - bit [3:0] keep1[3] = '{4'hf, 4'h1, 4'ha}; - int unsigned index0 = 0; - int unsigned index1 = 0; - int unsigned cycle = 0; - - vif.in_valid <= 2'b00; - vif.in_data[0] <= 32'h00000000; - vif.in_data[1] <= 32'h00000000; - vif.in_keep[0] <= 4'h0; - vif.in_keep[1] <= 4'h0; - vif.in_last <= 2'b00; - vif.out_ready <= 1'b0; - do @(negedge vif.clock); while (vif.reset); - - while (index0 < 3 || index1 < 3) begin - vif.in_valid[0] <= index0 < 3; - vif.in_valid[1] <= index1 < 3; - vif.in_data[0] <= data0[index0<3?index0 : 2]; - vif.in_data[1] <= data1[index1<3?index1 : 2]; - vif.in_keep[0] <= keep0[index0<3?index0 : 2]; - vif.in_keep[1] <= keep1[index1<3?index1 : 2]; - vif.in_last[0] <= index0 == 2; - vif.in_last[1] <= index1 == 2; - vif.out_ready <= cycle % 3 != 0; - @(posedge vif.clock); - if (vif.in_valid[0] && vif.in_ready[0]) index0++; - if (vif.in_valid[1] && vif.in_ready[1]) index1++; - cycle++; - @(negedge vif.clock); - end - - vif.in_valid <= 2'b10; - vif.in_data[1] <= 32'hffffffff; - vif.in_keep[1] <= 4'hf; - vif.in_last <= 2'b10; - vif.out_ready <= 1'b1; - do @(posedge vif.clock); while (!vif.in_ready[1]); - @(negedge vif.clock); - - index0 = 0; - while (index0 < 2) begin - vif.in_valid <= 2'b01; - vif.in_data[0] <= index0 == 0 ? 32'hffffffff : 32'h00000000; - vif.in_keep[0] <= index0 == 0 ? 4'hf : 4'h1; - vif.in_last <= index0 == 1; - @(posedge vif.clock); - if (vif.in_ready[0]) index0++; - @(negedge vif.clock); - end - - vif.in_valid <= 2'b00; - vif.in_data[0] <= 32'h00000000; - vif.in_data[1] <= 32'h00000000; - vif.in_keep[0] <= 4'h0; - vif.in_keep[1] <= 4'h0; - vif.in_last <= 2'b00; - done = 1'b1; - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_interface.sv b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_interface.sv deleted file mode 100644 index 45f3ad70b..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_interface.sv +++ /dev/null @@ -1,25 +0,0 @@ -interface arbiter_if ( - input logic clock -); - logic reset; - logic [ 1:0] in_valid; - logic [ 1:0] in_ready; - logic [31:0] in_data [2]; - logic [ 3:0] in_keep [2]; - logic [ 1:0] in_last; - logic out_valid; - logic out_ready; - logic [31:0] out_data; - logic [ 3:0] out_keep; - logic out_last; - - assert property (@(posedge clock) disable iff (reset) $onehot0(in_ready)); - assert property (@(posedge clock) disable iff (reset) - out_valid && !out_ready |=> out_valid && $stable( - {out_data, out_keep, out_last} - )); - - cover property (@(posedge clock) disable iff (reset) &in_valid); - cover property (@(posedge clock) disable iff (reset) in_valid[0] && in_ready[0]); - cover property (@(posedge clock) disable iff (reset) in_valid[1] && in_ready[1]); -endinterface diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_package.sv b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_package.sv deleted file mode 100644 index 0d3a3de07..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_package.sv +++ /dev/null @@ -1,17 +0,0 @@ -package arbiter_pkg; - import uvm_pkg::*; - `include "uvm_macros.svh" - - import "DPI-C" function chandle axis_arbiter_create(); - import "DPI-C" function void axis_arbiter_destroy(input chandle model); - import "DPI-C" function int unsigned axis_arbiter_step( - input chandle model, - input int unsigned valid, - input byte unsigned ready, - input int unsigned last - ); - - `include "arbiter_driver.svh" - `include "arbiter_checker.svh" - `include "arbiter_test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_tb.sv b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_tb.sv deleted file mode 100644 index 9d8516136..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_tb.sv +++ /dev/null @@ -1,42 +0,0 @@ -module arbiter_tb; - import uvm_pkg::*; - import arbiter_pkg::*; - - logic clock = 1'b0; - logic reset = 1'b0; - always #5 clock = ~clock; - - arbiter_if source_if (clock); - assign source_if.reset = reset; - - PacketArbiter dut ( - .clock(clock), - .reset(reset), - .io_in_0_ready(source_if.in_ready[0]), - .io_in_0_valid(source_if.in_valid[0]), - .io_in_0_bits_tdata(source_if.in_data[0]), - .io_in_0_bits_tkeep(source_if.in_keep[0]), - .io_in_0_bits_tlast(source_if.in_last[0]), - .io_in_1_ready(source_if.in_ready[1]), - .io_in_1_valid(source_if.in_valid[1]), - .io_in_1_bits_tdata(source_if.in_data[1]), - .io_in_1_bits_tkeep(source_if.in_keep[1]), - .io_in_1_bits_tlast(source_if.in_last[1]), - .io_out_ready(source_if.out_ready), - .io_out_valid(source_if.out_valid), - .io_out_bits_tdata(source_if.out_data), - .io_out_bits_tkeep(source_if.out_keep), - .io_out_bits_tlast(source_if.out_last) - ); - - initial begin - reset = 1'b1; - repeat (4) @(posedge clock); - reset = 1'b0; - end - - initial begin - uvm_config_db#(virtual arbiter_if)::set(null, "*", "vif", source_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_test.svh b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_test.svh deleted file mode 100644 index 76f8ec6a7..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/arbiter_test.svh +++ /dev/null @@ -1,42 +0,0 @@ -class env extends uvm_env; - `uvm_component_utils(env) - - driver source; - arbiter_checker monitor; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - source = driver::type_id::create("source", this); - monitor = arbiter_checker::type_id::create("monitor", this); - endfunction -endclass - -class protocol_test extends uvm_test; - `uvm_component_utils(protocol_test) - - env test_env; - virtual arbiter_if vif; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual arbiter_if)::get(this, "", "vif", vif)) begin - `uvm_fatal("VIF", "test requires vif") - end - test_env = env::type_id::create("env", this); - endfunction - - task run_phase(uvm_phase phase); - phase.raise_objection(this); - wait (test_env.source.done); - repeat (2) @(posedge vif.clock); - phase.drop_objection(this); - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/env.svh b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/env.svh deleted file mode 100644 index 868d407c9..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/env.svh +++ /dev/null @@ -1,37 +0,0 @@ -class env extends checked_env #(item, item, ref_model); - `uvm_component_utils(env) - - source_agent source; - sink sink_driver; - monitor sink_monitor; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - virtual axis_if source_vif; - virtual axis_if sink_vif; - - super.build_phase(phase); - if (!uvm_config_db#(virtual axis_if)::get(this, "", "source_vif", source_vif)) begin - `uvm_fatal("AXIS_VIF", "environment requires source_vif") - end - if (!uvm_config_db#(virtual axis_if)::get(this, "", "sink_vif", sink_vif)) begin - `uvm_fatal("AXIS_VIF", "environment requires sink_vif") - end - uvm_config_db#(virtual axis_if)::set(this, "source.*", "vif", source_vif); - uvm_config_db#(virtual axis_if)::set(this, "sink_driver", "vif", sink_vif); - uvm_config_db#(virtual axis_if)::set(this, "sink_monitor", "vif", sink_vif); - - source = source_agent::type_id::create("source", this); - sink_driver = sink::type_id::create("sink_driver", this); - sink_monitor = monitor::type_id::create("sink_monitor", this); - endfunction - - function void connect_phase(uvm_phase phase); - super.connect_phase(phase); - source.mon.ap.connect(input_export); - sink_monitor.ap.connect(output_export); - endfunction -endclass diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/master.f b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/master.f deleted file mode 100644 index f5a1226b1..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/master.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/protocol/axis -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/protocol/axis/interface.sv -@VERIFY@/uvm/src/protocol/axis/package.sv -@RESOURCES@/package.sv --F @RTL@/Master/filelist.f -@RESOURCES@/master_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/master_tb.sv b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/master_tb.sv deleted file mode 100644 index 3ac27bcde..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/master_tb.sv +++ /dev/null @@ -1,41 +0,0 @@ -module master_tb; - import uvm_pkg::*; - import axis_pkg::*; - - logic clock = 1'b0; - logic reset = 1'b0; - always #5 clock = ~clock; - - axis_if source_if (clock); - axis_if sink_if (clock); - - assign source_if.reset = reset; - assign sink_if.reset = reset; - - Master dut ( - .clock(clock), - .reset(reset), - .io_in_ready(source_if.tready), - .io_in_valid(source_if.tvalid), - .io_in_bits_tdata(source_if.tdata), - .io_in_bits_tkeep(source_if.tkeep), - .io_in_bits_tlast(source_if.tlast), - .io_axis_tvalid(sink_if.tvalid), - .io_axis_tready(sink_if.tready), - .io_axis_tdata(sink_if.tdata), - .io_axis_tkeep(sink_if.tkeep), - .io_axis_tlast(sink_if.tlast) - ); - - initial begin - reset = 1'b1; - repeat (4) @(negedge clock); - reset = 1'b0; - end - - initial begin - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env", "source_vif", source_if); - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env", "sink_vif", sink_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/package.sv deleted file mode 100644 index cd19f48c7..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/package.sv +++ /dev/null @@ -1,29 +0,0 @@ -package axis_pkg; - import uvm_pkg::*; - import ip_pkg::*; - import axis_vip_pkg::*; - typedef axis_item#(32) item; - typedef axis_source_agent#(32) source_agent; - typedef axis_monitor#(32) monitor; - typedef axis_sink#(32) sink; - `include "uvm_macros.svh" - - import "DPI-C" function chandle axis_ref_create(); - import "DPI-C" function void axis_ref_destroy(input chandle model); - import "DPI-C" function void axis_ref_push( - input chandle model, - input int unsigned data, - input byte unsigned keep, - input byte unsigned last - ); - import "DPI-C" function byte unsigned axis_ref_pop( - input chandle model, - output int unsigned data, - output byte unsigned keep, - output byte unsigned last - ); - - `include "ref_model.svh" - `include "env.svh" - `include "test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/ref_model.svh b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/ref_model.svh deleted file mode 100644 index 89ce490c6..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/ref_model.svh +++ /dev/null @@ -1,39 +0,0 @@ -class ref_model extends reference_model #(item); - `uvm_component_utils(ref_model) - - chandle model; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - model = axis_ref_create(); - if (model == null) begin - `uvm_fatal("AXIS_MODEL", "failed to create Rust reference model") - end - endfunction - - function void write(item t); - item expected; - int unsigned data; - byte unsigned keep; - byte unsigned last; - - axis_ref_push(model, t.data, t.keep, t.last); - if (!axis_ref_pop(model, data, keep, last)) begin - `uvm_fatal("AXIS_MODEL", "reference model produced no output") - end - expected = item::type_id::create("expected"); - expected.data = data; - expected.keep = keep[3:0]; - expected.last = last[0]; - expected_ap.write(expected); - endfunction - - function void final_phase(uvm_phase phase); - axis_ref_destroy(model); - super.final_phase(phase); - endfunction -endclass diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/slave.f b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/slave.f deleted file mode 100644 index 75928f05e..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/slave.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/protocol/axis -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/protocol/axis/interface.sv -@VERIFY@/uvm/src/protocol/axis/package.sv -@RESOURCES@/package.sv --F @RTL@/Slave/filelist.f -@RESOURCES@/slave_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/slave_tb.sv b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/slave_tb.sv deleted file mode 100644 index e95331d67..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/slave_tb.sv +++ /dev/null @@ -1,41 +0,0 @@ -module slave_tb; - import uvm_pkg::*; - import axis_pkg::*; - - logic clock = 1'b0; - logic reset = 1'b0; - always #5 clock = ~clock; - - axis_if source_if (clock); - axis_if sink_if (clock); - - assign source_if.reset = reset; - assign sink_if.reset = reset; - - Slave dut ( - .clock(clock), - .reset(reset), - .io_axis_tvalid(source_if.tvalid), - .io_axis_tready(source_if.tready), - .io_axis_tdata(source_if.tdata), - .io_axis_tkeep(source_if.tkeep), - .io_axis_tlast(source_if.tlast), - .io_out_ready(sink_if.tready), - .io_out_valid(sink_if.tvalid), - .io_out_bits_tdata(sink_if.tdata), - .io_out_bits_tkeep(sink_if.tkeep), - .io_out_bits_tlast(sink_if.tlast) - ); - - initial begin - reset = 1'b1; - repeat (4) @(negedge clock); - reset = 1'b0; - end - - initial begin - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env", "source_vif", source_if); - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env", "sink_vif", sink_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/test.svh b/arch/src/main/scala/framework/mem-core/axis/src/main/resources/test.svh deleted file mode 100644 index 0d7389911..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/resources/test.svh +++ /dev/null @@ -1,43 +0,0 @@ -class protocol_sequence extends uvm_sequence #(item); - `uvm_object_utils(protocol_sequence) - - function new(string name = "protocol_sequence"); - super.new(name); - endfunction - - task body(); - item req; - for (int i = 0; i < 32; i++) begin - req = item::type_id::create($sformatf("item_%0d", i)); - start_item(req); - req.data = $urandom(); - req.keep = (i % 15) + 1; - req.last = (i % 4) == 3; - finish_item(req); - end - endtask -endclass - -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - - env test_env; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - test_env = env::type_id::create("env", this); - endfunction - - task execute(); - protocol_sequence seq; - seq = protocol_sequence::type_id::create("seq"); - seq.start(test_env.source.seqr); - test_env.scoreboard.wait_checked(32); - `uvm_info("AXIS_CHECKED", $sformatf("Checked %0d transfers", test_env.scoreboard.checked), - UVM_LOW) - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Emit.scala deleted file mode 100644 index 543e470bd..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Emit.scala +++ /dev/null @@ -1,19 +0,0 @@ -package memcore.bus.axi - -object Emit extends App { - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Master(dataBits = 32), - firtoolOpts = args, - args = Array("--target-dir", "build/Master", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Slave(dataBits = 32), - firtoolOpts = args, - args = Array("--target-dir", "build/Slave", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new PacketArbiter(inputs = 2, dataBits = 32), - firtoolOpts = args, - args = Array("--target-dir", "build/PacketArbiter", "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Interface.scala b/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Interface.scala deleted file mode 100644 index eba9d6037..000000000 --- a/arch/src/main/scala/framework/mem-core/axis/src/main/scala/Interface.scala +++ /dev/null @@ -1,51 +0,0 @@ -package memcore.bus.axi - -import chisel3._ - -/** Physical AXI4-Stream signals, directed from the master's perspective. */ -class Port( - dataBits: Int, - idBits: Int = 0, - destBits: Int = 0, - userBits: Int = 0) - extends Bundle { - require(dataBits > 0 && dataBits % 8 == 0) - val tvalid = Output(Bool()) - val tready = Input(Bool()) - val tdata = Output(UInt(dataBits.W)) - val tkeep = Output(UInt((dataBits / 8).W)) - val tlast = Output(Bool()) - val tid = Output(UInt(idBits.W)) - val tdest = Output(UInt(destBits.W)) - val tuser = Output(UInt(userBits.W)) -} - -/** - * AXI4-Stream payload signals. - * - * A `Decoupled[Beat]` is the full AXI-S channel: `valid` and `ready` - * are TVALID and TREADY. This bundle exposes the remaining standard signal - * names directly in generated RTL. - */ -class Beat( - dataBits: Int, - idBits: Int = 0, - destBits: Int = 0, - userBits: Int = 0) - extends Bundle { - require(dataBits > 0 && dataBits % 8 == 0) - val tdata = UInt(dataBits.W) - val tkeep = UInt((dataBits / 8).W) - val tlast = Bool() - val tid = UInt(idBits.W) - val tdest = UInt(destBits.W) - val tuser = UInt(userBits.W) - - // Internal aliases preserve concise datapath code while retaining AXI-S names. - def data: UInt = tdata - def keep: UInt = tkeep - def last: Bool = tlast - def id: UInt = tid - def dest: UInt = tdest - def user: UInt = tuser -} diff --git a/arch/src/main/scala/framework/mem-core/bank/build.sc b/arch/src/main/scala/framework/mem-core/bank/build.sc deleted file mode 100644 index 381e9d95e..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/build.sc +++ /dev/null @@ -1,22 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -object axis extends ChiselModule { - override def moduleRoot = os.pwd / os.up / "axis" -} - -object bank extends ChiselModule { - override def moduleRoot = os.pwd - override def moduleDeps = Seq(axis) - override def mainClass = Some("memcore.memory.bank.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/bank/src/csrc/src/memory.rs b/arch/src/main/scala/framework/mem-core/bank/src/csrc/src/memory.rs deleted file mode 100644 index 4968ad3ae..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/csrc/src/memory.rs +++ /dev/null @@ -1,80 +0,0 @@ -struct Bank { - words: Vec, - initialized: Vec, -} - -impl Bank { - fn access(&mut self, addr: usize, write: bool, data: u32, mask: u8) -> u32 { - if write { - for byte in 0..4 { - if mask & (1 << byte) != 0 { - let bits = 0xff << (byte * 8); - self.words[addr] = (self.words[addr] & !bits) | (data & bits); - } - } - self.initialized[addr] |= mask; - 0 - } else { - assert_eq!( - self.initialized[addr], 0xf, - "read of uninitialized bank word" - ); - self.words[addr] - } - } -} - -#[no_mangle] -pub extern "C" fn bank_ref_create(entries: u32) -> *mut std::ffi::c_void { - Box::into_raw(Box::new(Bank { - words: vec![0; entries as usize], - initialized: vec![0; entries as usize], - })) - .cast() -} - -#[no_mangle] -pub unsafe extern "C" fn bank_ref_destroy(model: *mut std::ffi::c_void) { - drop(Box::from_raw(model.cast::())); -} - -#[no_mangle] -pub unsafe extern "C" fn bank_ref_access( - model: *mut std::ffi::c_void, - addr: u32, - write: u8, - data: u32, - mask: u8, -) -> u32 { - (*model.cast::()).access(addr as usize, write != 0, data, mask) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn masked_writes_preserve_other_bytes_and_addresses() { - let mut bank = Bank { - words: vec![0; 16], - initialized: vec![0; 16], - }; - bank.access(0, true, 0x12345678, 15); - bank.access(15, true, 0xabcdef01, 15); - bank.access(0, true, 0xaabbccdd, 5); - assert_eq!(bank.access(0, false, 0, 0), 0x12bb56dd); - bank.access(0, true, 0, 0); - assert_eq!(bank.access(0, false, 0, 0), 0x12bb56dd); - assert_eq!(bank.access(15, false, 0, 0), 0xabcdef01); - } - - #[test] - #[should_panic(expected = "uninitialized")] - fn rejects_uninitialized_reads() { - let mut bank = Bank { - words: vec![0; 16], - initialized: vec![0; 16], - }; - bank.access(0, false, 0, 0); - } -} diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank.el b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank.el deleted file mode 100644 index a7c43228d..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank.el +++ /dev/null @@ -1,5 +0,0 @@ -// Format Version: 2 -CHECKSUM: "366479506 2693907090" -INSTANCE: bank_tb.dut -ANNOTATION: "Bank has no error response path; invalid accesses are outside the request contract and error is tied to zero." -Toggle io_response_bits_error "net io_response_bits_error" diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank.f b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank.f deleted file mode 100644 index c12d863fb..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -+incdir+@RESOURCES@ -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Bank/filelist.f -@RESOURCES@/bank_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank_tb.sv b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank_tb.sv deleted file mode 100644 index b247c06bd..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bank_tb.sv +++ /dev/null @@ -1,29 +0,0 @@ -module bank_tb; - import uvm_pkg::*; - import bank_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - bank_if source_if (clock); - bank_if sink_if (clock); - Bank dut ( - .clock(clock), - .reset(source_if.reset), - .io_request_valid(source_if.valid), - .io_request_ready(source_if.ready), - .io_request_bits_addr(source_if.addr), - .io_request_bits_write(source_if.write), - .io_request_bits_data(source_if.data), - .io_request_bits_mask(source_if.mask), - .io_request_bits_tag(source_if.tag), - .io_response_valid(sink_if.valid), - .io_response_ready(sink_if.ready), - .io_response_bits_data(sink_if.data), - .io_response_bits_tag(sink_if.tag), - .io_response_bits_error(sink_if.error) - ); - initial begin - uvm_config_db#(virtual bank_if)::set(null, "uvm_test_top*", "request", source_if); - uvm_config_db#(virtual bank_if)::set(null, "uvm_test_top*", "response", sink_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset.el b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset.el deleted file mode 100644 index 56c580493..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset.el +++ /dev/null @@ -1,79 +0,0 @@ -// Format Version: 2 -// Reviewed for Emit: dataBits=32, banks=4, entriesPerBank=16, tagBits=8. - -CHECKSUM: "1481319037 3871254824" -INSTANCE: bankset_tb.dut -ANNOTATION: "Nonempty-command assertion failure diagnostics; legal commands have beats > 0." -Block 2 "555555334" "if (1)" -ANNOTATION: "Nonempty-command assertion failure diagnostics; legal commands have beats > 0." -Block 3 "3006158740" "$error(\"Assertion failed: BankSet command must be nonempty\n at BankSet.scala:61 assert(io.command.bits.beats =/= 0.U, \\"BankSet command must be nonempty\\")\n\");" -ANNOTATION: "Nonempty-command assertion failure diagnostics; legal commands have beats > 0." -Block 5 "555555334" "if (1)" -ANNOTATION: "Nonempty-command assertion failure diagnostics; legal commands have beats > 0." -Block 6 "1956737205" "$fatal;" -ANNOTATION: "Alignment assertion failure diagnostics; commands are 4-byte aligned." -Block 10 "530868893" "if (1)" -ANNOTATION: "Alignment assertion failure diagnostics; commands are 4-byte aligned." -Block 11 "181752484" "$error(\"Assertion failed: BankSet command must be beat aligned\n at BankSet.scala:62 assert(io.command.bits.addr %%%% p.bytes.U === 0.U, \\"BankSet command must be beat aligned\\")\n\");" -ANNOTATION: "Alignment assertion failure diagnostics; commands are 4-byte aligned." -Block 13 "530868893" "if (1)" -ANNOTATION: "Alignment assertion failure diagnostics; commands are 4-byte aligned." -Block 14 "3298909189" "$fatal;" -ANNOTATION: "Capacity assertion failure diagnostics; command end is at most 256 bytes." -Block 18 "2870240055" "if (1)" -ANNOTATION: "Capacity assertion failure diagnostics; command end is at most 256 bytes." -Block 19 "3417018178" "$error(\"Assertion failed: BankSet command exceeds capacity\n at BankSet.scala:63 assert(\n\");" -ANNOTATION: "Capacity assertion failure diagnostics; command end is at most 256 bytes." -Block 21 "2870240055" "if (1)" -ANNOTATION: "Capacity assertion failure diagnostics; command end is at most 256 bytes." -Block 22 "2272232831" "$fatal;" -ANNOTATION: "TLAST assertion failure diagnostics; last is asserted on exactly the final write beat." -Block 26 "1487852324" "if (1)" -ANNOTATION: "TLAST assertion failure diagnostics; last is asserted on exactly the final write beat." -Block 27 "1783058220" "$error(\"Assertion failed: BankSet write TLAST does not match command length\n at BankSet.scala:87 assert(io.write.bits.last === last, \\"BankSet write TLAST does not match command length\\")\n\");" -ANNOTATION: "TLAST assertion failure diagnostics; last is asserted on exactly the final write beat." -Block 29 "1487852324" "if (1)" -ANNOTATION: "TLAST assertion failure diagnostics; last is asserted on exactly the final write beat." -Block 30 "605788389" "$fatal;" - -CHECKSUM: "1481319037 1542845018" -INSTANCE: bankset_tb.dut -ANNOTATION: "Assertion enable is disabled during reset; no command is issued in reset." -Condition 1 "3864235482" "(_GEN & ((~reset))) 1 -1" (2 "10") -ANNOTATION: "A valid command cannot have zero beats." -Condition 2 "4106865980" "(_GEN_10 & (io_command_bits_beats == 32'b0)) 1 -1" (3 "11") -ANNOTATION: "All driven addresses are 4-byte aligned." -Condition 4 "3972072533" "(_GEN_10 & ((|_GEN_11[2:0]))) 1 -1" (1 "01") -ANNOTATION: "All driven addresses are 4-byte aligned." -Condition 4 "3972072533" "(_GEN_10 & ((|_GEN_11[2:0]))) 1 -1" (3 "11") -ANNOTATION: "All driven commands end within the 256-byte capacity." -Condition 5 "3649435992" "(_GEN_10 & (({28'b0, io_command_bits_addr} + {2'b0, io_command_bits_beats, 2'b0}) > 36'h000000100)) 1 -1" (1 "01") -ANNOTATION: "All driven commands end within the 256-byte capacity." -Condition 5 "3649435992" "(_GEN_10 & (({28'b0, io_command_bits_addr} + {2'b0, io_command_bits_beats, 2'b0}) > 36'h000000100)) 1 -1" (3 "11") -ANNOTATION: "No write handshake in reset; accepted TLAST matches the command length." -Condition 6 "3432548480" "(_GEN_9 & ((~reset)) & (io_write_bits_tlast != last)) 1 -1" (2 "101") -ANNOTATION: "No write handshake in reset; accepted TLAST matches the command length." -Condition 6 "3432548480" "(_GEN_9 & ((~reset)) & (io_write_bits_tlast != last)) 1 -1" (4 "111") -ANNOTATION: "writeResponse follows an accepted Bank write; that Bank holds response.valid until ready." -Condition 9 "618862985" "(_GEN_3 & _GEN_13) 1 -1" (2 "10") -ANNOTATION: "Only one request is outstanding; a read is issued only after the preceding response retires." -Condition 13 "3211095629" "(_GEN_1 & _GEN_8) 1 -1" (2 "10") -ANNOTATION: "Only one request is outstanding; writing selects an idle Bank after the preceding response retires." -Condition 36 "1292920334" "(_io_write_ready_T & _GEN_8) 1 -1" (2 "10") - -CHECKSUM: "1481319037 745494789" -INSTANCE: bankset_tb.dut -ANNOTATION: "Four-byte aligned addresses have low two bits zero." -Toggle io_command_bits_addr [1:0] "net io_command_bits_addr[7:0]" -ANNOTATION: "Four-byte aligned addresses have low two bits zero." -Toggle command_addr [1:0] "reg command_addr[7:0]" -ANNOTATION: "Capacity is 64 words; legal lengths and completed-beat count cannot exceed 64." -Toggle io_command_bits_beats [31:7] "net io_command_bits_beats[31:0]" -ANNOTATION: "Capacity is 64 words; legal lengths and completed-beat count cannot exceed 64." -Toggle command_beats [31:7] "reg command_beats[31:0]" -ANNOTATION: "Capacity is 64 words; legal lengths and completed-beat count cannot exceed 64." -Toggle beat [31:7] "reg beat[31:0]" -ANNOTATION: "Read responses contain a full word; TKEEP is always 4'b1111." -Toggle io_read_bits_tkeep "net io_read_bits_tkeep[3:0]" -ANNOTATION: "Completion error is tied to zero; invalid commands are assertions, not error responses." -Toggle io_done_bits "net io_done_bits" diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset.f b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset.f deleted file mode 100644 index 442fb41c4..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset.f +++ /dev/null @@ -1,10 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/protocol/axis -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/protocol/axis/interface.sv -@VERIFY@/uvm/src/protocol/axis/package.sv -@RESOURCES@/bankset_interface.sv -@RESOURCES@/bankset_package.sv --F @RTL@/BankSet/filelist.f -@RESOURCES@/bankset_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_interface.sv b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_interface.sv deleted file mode 100644 index e380df75d..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_interface.sv +++ /dev/null @@ -1,19 +0,0 @@ -interface bankset_if ( - input logic clock -); - logic reset; - logic valid; - logic ready; - logic write; - logic [7:0] addr; - logic [31:0] beats; - logic done_valid; - logic done_ready; - logic error; - - assert property (@(posedge clock) disable iff (reset) - done_valid && !done_ready |=> done_valid && $stable( - error - )) - else $fatal(1, "BankSet completion changed under backpressure"); -endinterface diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_package.sv b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_package.sv deleted file mode 100644 index 99508d625..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_package.sv +++ /dev/null @@ -1,201 +0,0 @@ -package bankset_pkg; - import uvm_pkg::*; - import ip_pkg::*; - import axis_vip_pkg::*; - `include "uvm_macros.svh" - typedef axis_item#(32) item; - - import "DPI-C" function chandle bank_ref_create(input int unsigned entries); - import "DPI-C" function void bank_ref_destroy(input chandle model); - import "DPI-C" function int unsigned bank_ref_access( - input chandle model, - input int unsigned addr, - input byte unsigned write, - input int unsigned data, - input byte unsigned mask - ); - - class operation extends uvm_sequence_item; - bit write; - int unsigned word_addr; - int unsigned data; - byte unsigned mask; - bit last; - `uvm_object_utils(operation) - function new(string name = "operation"); - super.new(name); - endfunction - endclass - - class ref_model extends reference_model #(operation, item); - `uvm_component_utils(ref_model) - chandle model; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - model = bank_ref_create(64); - endfunction - function void write(operation t); - int unsigned result = bank_ref_access(model, t.word_addr, t.write, t.data, t.mask); - if (!t.write) begin - item expected = item::type_id::create("expected"); - expected.data = result; - expected.keep = 15; - expected.last = t.last; - expected_ap.write(expected); - end - endfunction - function void final_phase(uvm_phase phase); - bank_ref_destroy(model); - super.final_phase(phase); - endfunction - endclass - - class env extends checked_env #(operation, item, ref_model); - `uvm_component_utils(env) - virtual bankset_if control; - virtual axis_if write_vif; - axis_source_agent #(32) source; - axis_sink #(32) sink; - axis_monitor #(32) output_monitor; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual bankset_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual axis_if)::get( - this, "", "write_vif", write_vif - )) - `uvm_fatal("VIF", "BankSet interfaces missing") - source = axis_source_agent#(32)::type_id::create("source", this); - sink = axis_sink#(32)::type_id::create("sink", this); - output_monitor = axis_monitor#(32)::type_id::create("output_monitor", this); - endfunction - function void connect_phase(uvm_phase phase); - super.connect_phase(phase); - output_monitor.ap.connect(output_export); - endfunction - task run_phase(uvm_phase phase); - int unsigned word_addr; - operation op; - forever begin - @(posedge control.clock); - if (!control.reset) begin - if (control.valid && control.ready) begin - word_addr = control.addr / 4; - if (!control.write) begin - for (int i = 0; i < control.beats; i++) begin - op = operation::type_id::create("read"); - op.write = 0; - op.word_addr = word_addr + i; - op.last = i == control.beats - 1; - input_export.write(op); - end - end - end - if (write_vif.tvalid && write_vif.tready) begin - op = operation::type_id::create("write"); - op.write = 1; - op.word_addr = word_addr; - op.data = write_vif.tdata; - op.mask = write_vif.tkeep; - input_export.write(op); - word_addr++; - end - end - end - endtask - endclass - - class write_sequence extends uvm_sequence #(item); - `uvm_object_utils(write_sequence) - int unsigned beats; - bit masked; - function new(string name = "write_sequence"); - super.new(name); - endfunction - task body(); - item req; - for (int i = 0; i < beats; i++) begin - req = item::type_id::create("req"); - start_item(req); - req.data = $urandom(); - req.keep = masked ? (i % 16) : 15; - req.last = i == beats - 1; - finish_item(req); - end - endtask - endclass - - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - env test_env; - virtual bankset_if control; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - test_env = env::type_id::create("env", this); - if (!uvm_config_db#(virtual bankset_if)::get(this, "", "control", control)) - `uvm_fatal("VIF", "BankSet control interface missing") - endfunction - task transfer(bit write, int unsigned word_addr, int unsigned beats, bit masked = 0); - write_sequence seq; - int unsigned target = test_env.scoreboard.checked + (write ? 0 : beats); - @(negedge control.clock); - control.valid = 1; - control.write = write; - control.addr = word_addr * 4; - control.beats = beats; - control.done_ready = 0; - do @(posedge control.clock); while (!control.ready); - @(negedge control.clock); - control.valid = 0; - if (write) begin - seq = write_sequence::type_id::create("seq"); - seq.beats = beats; - seq.masked = masked; - seq.start(test_env.source.seqr); - end else begin - test_env.scoreboard.wait_checked(target); - end - do @(negedge control.clock); while (!control.done_valid); - repeat (3) begin - if (control.done_valid !== 1'b1 || control.error !== 1'b0 || control.ready !== 1'b0) - `uvm_fatal("DONE", "invalid BankSet completion under backpressure") - @(negedge control.clock); - end - control.done_ready = 1; - repeat (2) @(negedge control.clock); - control.done_ready = 0; - if (control.done_valid !== 1'b0 || control.ready !== 1'b1) - `uvm_fatal("DONE", "BankSet failed to retire completion") - endtask - task execute(); - control.valid = 0; - control.write = 0; - control.addr = 0; - control.beats = 0; - control.done_ready = 0; - wait (!control.reset); - transfer(1, 0, 64); - test_env.sink.stall_cycles = 0; - transfer(0, 0, 63); - test_env.sink.stall_cycles = 3; - transfer(1, 63, 1); - transfer(0, 63, 1); - transfer(1, 3, 17, 1); - control.reset = 1; - repeat (2) @(negedge control.clock); - control.reset = 0; - transfer(0, 0, 64); - `uvm_info("BANKSET", $sformatf("Checked %0d read beats", test_env.scoreboard.checked), - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_tb.sv b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_tb.sv deleted file mode 100644 index c61cef2e9..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/bankset_tb.sv +++ /dev/null @@ -1,46 +0,0 @@ -module bankset_tb; - import uvm_pkg::*; - import bankset_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - bankset_if control (clock); - axis_if source_if (clock); - axis_if sink_if (clock); - assign source_if.reset = control.reset; - assign sink_if.reset = control.reset; - BankSet dut ( - .clock(clock), - .reset(control.reset), - .io_command_valid(control.valid), - .io_command_ready(control.ready), - .io_command_bits_write(control.write), - .io_command_bits_addr(control.addr), - .io_command_bits_beats(control.beats), - .io_write_valid(source_if.tvalid), - .io_write_ready(source_if.tready), - .io_write_bits_tdata(source_if.tdata), - .io_write_bits_tkeep(source_if.tkeep), - .io_write_bits_tlast(source_if.tlast), - .io_read_valid(sink_if.tvalid), - .io_read_ready(sink_if.tready), - .io_read_bits_tdata(sink_if.tdata), - .io_read_bits_tkeep(sink_if.tkeep), - .io_read_bits_tlast(sink_if.tlast), - .io_done_valid(control.done_valid), - .io_done_ready(control.done_ready), - .io_done_bits(control.error) - ); - initial begin - control.reset = 1; - repeat (4) @(negedge clock); - control.reset = 0; - end - initial begin - uvm_config_db#(virtual bankset_if)::set(null, "uvm_test_top*", "control", control); - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env", "write_vif", source_if); - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.source.*", "vif", source_if); - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.sink", "vif", sink_if); - uvm_config_db#(virtual axis_if)::set(null, "uvm_test_top.env.output_monitor", "vif", sink_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/env.svh b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/env.svh deleted file mode 100644 index fae0f642d..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/env.svh +++ /dev/null @@ -1,47 +0,0 @@ -class env extends checked_env #(request_item, response_item, ref_model); - `uvm_component_utils(env) - virtual bank_if request; - virtual bank_if response; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual bank_if)::get( - this, "", "request", request - ) || !uvm_config_db#(virtual bank_if)::get( - this, "", "response", response - )) - `uvm_fatal("VIF", "bank interfaces missing") - endfunction - - task run_phase(uvm_phase phase); - request_item req; - response_item rsp; - forever begin - @(posedge request.clock); - if (request.reset) begin - scoreboard.cancel_pending(); - end else begin - if (request.valid && request.ready) begin - req = request_item::type_id::create("request"); - req.addr = request.addr; - req.write = request.write; - req.data = request.data; - req.mask = request.mask; - req.tag = request.tag; - input_export.write(req); - end - if (response.valid && response.ready) begin - rsp = response_item::type_id::create("response"); - rsp.data = response.data; - rsp.tag = response.tag; - rsp.error = response.error; - output_export.write(rsp); - end - end - end - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/interface.sv b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/interface.sv deleted file mode 100644 index 8f1a99a93..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/interface.sv +++ /dev/null @@ -1,18 +0,0 @@ -interface bank_if ( - input logic clock -); - logic reset; - logic valid; - logic ready; - logic [3:0] addr; - logic write; - logic [31:0] data; - logic [3:0] mask; - logic [7:0] tag; - logic error; - - assert property (@(posedge clock) disable iff (reset) valid && !ready |=> valid && $stable( - {data, tag, error} - )); - cover property (@(posedge clock) disable iff (reset) valid && !ready ##1 valid && ready); -endinterface diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/item.svh b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/item.svh deleted file mode 100644 index 2c5da1fb2..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/item.svh +++ /dev/null @@ -1,31 +0,0 @@ -class request_item extends uvm_sequence_item; - logic [3:0] addr; - logic write; - logic [31:0] data; - logic [3:0] mask; - logic [7:0] tag; - `uvm_object_utils_begin(request_item) - `uvm_field_int(addr, UVM_ALL_ON) - `uvm_field_int(write, UVM_ALL_ON) - `uvm_field_int(data, UVM_ALL_ON) - `uvm_field_int(mask, UVM_ALL_ON) - `uvm_field_int(tag, UVM_ALL_ON) - `uvm_object_utils_end - function new(string name = "request_item"); - super.new(name); - endfunction -endclass - -class response_item extends uvm_sequence_item; - logic [31:0] data; - logic [7:0] tag; - logic error; - `uvm_object_utils_begin(response_item) - `uvm_field_int(data, UVM_ALL_ON) - `uvm_field_int(tag, UVM_ALL_ON) - `uvm_field_int(error, UVM_ALL_ON) - `uvm_object_utils_end - function new(string name = "response_item"); - super.new(name); - endfunction -endclass diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/package.sv deleted file mode 100644 index 7b19fb001..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/package.sv +++ /dev/null @@ -1,20 +0,0 @@ -package bank_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - - import "DPI-C" function chandle bank_ref_create(input int unsigned entries); - import "DPI-C" function void bank_ref_destroy(input chandle model); - import "DPI-C" function int unsigned bank_ref_access( - input chandle model, - input int unsigned addr, - input byte unsigned write, - input int unsigned data, - input byte unsigned mask - ); - - `include "item.svh" - `include "ref_model.svh" - `include "env.svh" - `include "test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/ref_model.svh b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/ref_model.svh deleted file mode 100644 index 8458a6539..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/ref_model.svh +++ /dev/null @@ -1,26 +0,0 @@ -class ref_model extends reference_model #(request_item, response_item); - `uvm_component_utils(ref_model) - chandle model; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - model = bank_ref_create(16); - endfunction - - function void write(request_item t); - response_item expected = response_item::type_id::create("expected"); - expected.data = bank_ref_access(model, t.addr, t.write, t.data, t.mask); - expected.tag = t.tag; - expected.error = 0; - expected_ap.write(expected); - endfunction - - function void final_phase(uvm_phase phase); - bank_ref_destroy(model); - super.final_phase(phase); - endfunction -endclass diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/test.svh b/arch/src/main/scala/framework/mem-core/bank/src/main/resources/test.svh deleted file mode 100644 index 8a0abae63..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/resources/test.svh +++ /dev/null @@ -1,106 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual bank_if request; - virtual bank_if response; - env test_env; - int checked = 0; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual bank_if)::get( - this, "", "request", request - ) || !uvm_config_db#(virtual bank_if)::get( - this, "", "response", response - )) - `uvm_fatal("VIF", "bank interfaces missing") - test_env = env::type_id::create("env", this); - endfunction - - task access (bit wr, bit [3:0] addr, bit [31:0] data, bit [3:0] mask, bit [7:0] tag, int stall); - int unsigned target = test_env.scoreboard.checked + 1; - @(negedge request.clock); - request.valid = 1; - request.write = wr; - request.addr = addr; - request.data = data; - request.mask = mask; - request.tag = tag; - response.ready = 0; - do @(posedge request.clock); while (!request.ready); - @(negedge request.clock); - request.valid = 0; - while (!response.valid) @(negedge request.clock); - repeat (stall + 1) begin - if (response.valid !== 1'b1) `uvm_fatal("BANK", "response valid dropped under backpressure") - if (request.ready !== 1'b0) - `uvm_fatal("BANK", "accepted another request with an outstanding response") - @(negedge request.clock); - end - response.ready = 1; - @(negedge request.clock); - if (response.valid !== 1'b0) `uvm_fatal("BANK", "response was not retired") - response.ready = 0; - test_env.scoreboard.wait_checked(target); - checked++; - endtask - - task reset_pending(bit wr); - @(negedge request.clock); - request.valid = 1; - request.write = wr; - request.addr = 15; - request.data = 'hface1234; - request.mask = 15; - request.tag = 255; - response.ready = 0; - do @(posedge request.clock); while (!request.ready); - @(negedge request.clock); - request.valid = 0; - request.reset = 1; - response.reset = 1; - repeat (2) @(negedge request.clock); - request.reset = 0; - response.reset = 0; - repeat (3) begin - @(negedge request.clock); - if (response.valid !== 1'b0 || request.ready !== 1'b1) - `uvm_fatal("RESET", "reset did not discard outstanding transaction") - end - access (0, 15, 0, 0, 42, 3); - endtask - - task execute(); - request.valid = 0; - request.write = 0; - request.addr = 0; - request.data = 0; - request.mask = 0; - request.tag = 0; - request.error = 0; - request.reset = 1; - response.reset = 1; - response.ready = 0; - repeat (4) @(negedge request.clock); - request.reset = 0; - response.reset = 0; - for (int a = 0; a < 16; a++) access (1, a, $urandom(), 15, a, 0); - for (int a = 0; a < 16; a++) access (0, a, 0, 0, 255 - a, 2); - for (int m = 0; m < 16; m++) begin - access (1, m, $urandom(), m, m, 5); - access (0, m, 0, 0, m, 0); - end - reset_pending(1); - reset_pending(0); - for (int a = 0; a < 16; a++) access (0, a, 0, 0, a, 0); - response.ready = 1; - repeat (2) begin - @(negedge request.clock); - if (response.valid !== 1'b0) `uvm_fatal("BANK", "unexpected response while idle") - end - `uvm_info("BANK", $sformatf("Checked %0d responses", checked), UVM_LOW) - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/scala/Bank.scala b/arch/src/main/scala/framework/mem-core/bank/src/main/scala/Bank.scala deleted file mode 100644 index cf7e48059..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/scala/Bank.scala +++ /dev/null @@ -1,68 +0,0 @@ -package memcore.memory.bank - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -class BankRequest(p: BankSetParams) extends Bundle { - val addr = UInt(p.rowBits.W) - val write = Bool() - val data = UInt(p.dataBits.W) - val mask = UInt(p.bytes.W) - val tag = UInt(p.tagBits.W) -} - -class BankResponse(p: BankSetParams) extends Bundle { - val data = UInt(p.dataBits.W) - val tag = UInt(p.tagBits.W) - val error = Bool() -} - -/** One physical SRAM bank. Cache and NoC policy remain outside this module. */ -@instantiable -class Bank(p: BankSetParams) extends Module { - - @public - val io = IO(new Bundle { - val request = Flipped(Decoupled(new BankRequest(p))) - val response = Decoupled(new BankResponse(p)) - }) - - val memory = SyncReadMem(p.entriesPerBank, Vec(p.bytes, UInt(8.W))) - val request = Reg(new BankRequest(p)) - val responseValid = RegInit(false.B) - val readPending = RegInit(false.B) - val responseData = Reg(UInt(p.dataBits.W)) - - val readData = memory.readWrite( - io.request.bits.addr, - io.request.bits.data.asTypeOf(Vec(p.bytes, UInt(8.W))), - io.request.bits.mask.asBools, - io.request.fire, - io.request.bits.write - ) - - io.request.ready := !responseValid && !readPending - when(io.request.fire) { - request := io.request.bits - when(io.request.bits.write) { - responseData := 0.U - responseValid := true.B - }.otherwise { - readPending := true.B - } - } - when(readPending) { - responseData := readData.asUInt - responseValid := true.B - readPending := false.B - } - - io.response.valid := responseValid - io.response.bits.data := responseData - io.response.bits.tag := request.tag - io.response.bits.error := false.B - when(io.response.fire) { - responseValid := false.B - } -} diff --git a/arch/src/main/scala/framework/mem-core/bank/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/bank/src/main/scala/Emit.scala deleted file mode 100644 index 9d56eab79..000000000 --- a/arch/src/main/scala/framework/mem-core/bank/src/main/scala/Emit.scala +++ /dev/null @@ -1,15 +0,0 @@ -package memcore.memory.bank - -object Emit extends App { - val p = BankSetParams(dataBits = 32, banks = 4, entriesPerBank = 16) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Bank(p), - firtoolOpts = args, - args = Array("--target-dir", "build/Bank", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new BankSet(p), - firtoolOpts = args, - args = Array("--target-dir", "build/BankSet", "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/mem-core/cache/build.sc b/arch/src/main/scala/framework/mem-core/cache/build.sc deleted file mode 100644 index b9e3f0dc3..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/build.sc +++ /dev/null @@ -1,21 +0,0 @@ -import mill._ -import mill.scalalib._ - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object cache extends SbtModule { - override def moduleDeps = Seq(queue) - override def millSourcePath = os.pwd - override def sources = T.sources(super.sources() ++ Seq(PathRef(millSourcePath / "configs"))) - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0", ivy"tech.sparse::toml-scala:0.2.2") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") - override def mainClass = Some("memcore.memory.cache.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/cache/configs/CacheParams.scala b/arch/src/main/scala/framework/mem-core/cache/configs/CacheParams.scala deleted file mode 100644 index ba14f4da2..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/configs/CacheParams.scala +++ /dev/null @@ -1,54 +0,0 @@ -package memcore.memory.cache.configs - -import chisel3.util.{isPow2, log2Ceil} -import scala.io.Source -import scala.util.Using -import toml.{Toml, Value} - -case class CacheParams( - addressBits: Int, - lineBytes: Int, - sets: Int, - ways: Int, - idBits: Int, - metadataBits: Int, - responseDepth: Int) { - require(lineBytes >= 8 && isPow2(lineBytes)) - require(sets >= 2 && isPow2(sets)) - require(ways >= 1 && isPow2(ways)) - require(idBits > 0 && metadataBits > 0 && responseDepth >= 2) - val offsetBits = log2Ceil(lineBytes) - val setBits = log2Ceil(sets) - val wayBits = math.max(1, log2Ceil(ways)) - val tagBits = addressBits - offsetBits - setBits - val lineBits = lineBytes * 8 - require(tagBits > 0) -} - -object CacheParams { - - def load(path: String): CacheParams = { - val text = Using.resource(Source.fromFile(path, "UTF-8"))(_.mkString) - val table = Toml.parse(text) match { - case Right(Value.Tbl(root)) => root("cache") match { - case Value.Tbl(table) => table - case value => throw new IllegalArgumentException(s"Expected [cache] table, got $value") - } - case other => throw new IllegalArgumentException(s"Invalid cache configuration $path: $other") - } - def int(name: String): Int = table(name) match { - case Value.Num(n) if n == n.toInt => n.toInt - case value => throw new IllegalArgumentException(s"Expected integer cache.$name, got $value") - } - CacheParams( - int("addressBits"), - int("lineBytes"), - int("sets"), - int("ways"), - int("idBits"), - int("metadataBits"), - int("responseDepth") - ) - } - -} diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache.el b/arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache.el deleted file mode 100644 index 7df1fa991..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache.el +++ /dev/null @@ -1,113 +0,0 @@ -// Format Version: 2 -// Reviewed for configs/default.toml: 44-bit addresses, 64-byte lines, 4 sets, 2 ways, responseDepth=2. - -CHECKSUM: "1581756980 3097157900" -INSTANCE: cache_tb.dut -ANNOTATION: "Capacity assertion diagnostics: admission reserves count plus pending, so an outstanding pipeline response always has queue space." -Block 2 "3078567325" "if (1)" -ANNOTATION: "Capacity assertion diagnostics: admission reserves count plus pending, so an outstanding pipeline response always has queue space." -Block 3 "1895567494" "$error(\"Assertion failed: Cache response capacity reservation failed\n at Cache.scala:51 assert(!pending || responses.io.enq.ready, \\"Cache response capacity reservation failed\\")\n\");" -ANNOTATION: "Capacity assertion diagnostics: admission reserves count plus pending, so an outstanding pipeline response always has queue space." -Block 5 "3078567325" "if (1)" -ANNOTATION: "Capacity assertion diagnostics: admission reserves count plus pending, so an outstanding pipeline response always has queue space." -Block 6 "2299949409" "$fatal;" -ANNOTATION: "Opcode assertion diagnostics: the legal interface operations are Lookup, Read, Write, Fill, and Invalidate." -Block 10 "422554957" "if (1)" -ANNOTATION: "Opcode assertion diagnostics: the legal interface operations are Lookup, Read, Write, Fill, and Invalidate." -Block 11 "3963963684" "$error(\"Assertion failed: Unknown Cache operation\n at Cache.scala:67 assert(req.op <= CacheOp.Invalidate.U, \\"Unknown Cache operation\\")\n\");" -ANNOTATION: "Opcode assertion diagnostics: the legal interface operations are Lookup, Read, Write, Fill, and Invalidate." -Block 13 "422554957" "if (1)" -ANNOTATION: "Opcode assertion diagnostics: the legal interface operations are Lookup, Read, Write, Fill, and Invalidate." -Block 14 "2766194732" "$fatal;" -ANNOTATION: "Alignment assertion diagnostics: requests use cache-line-aligned addresses." -Block 18 "874237168" "if (1)" -ANNOTATION: "Alignment assertion diagnostics: requests use cache-line-aligned addresses." -Block 19 "1020309457" "$error(\"Assertion failed: Cache operations require a line-aligned address\n at Cache.scala:68 assert(req.addr(p.offsetBits - 1, 0) === 0.U, \\"Cache operations require a line-aligned address\\")\n\");" -ANNOTATION: "Alignment assertion diagnostics: requests use cache-line-aligned addresses." -Block 21 "874237168" "if (1)" -ANNOTATION: "Alignment assertion diagnostics: requests use cache-line-aligned addresses." -Block 22 "1069368929" "$fatal;" -ANNOTATION: "Duplicate-tag assertion diagnostics: reset clears valid, Fill requires an absent tag or its existing way, and Invalidate only clears valid." -Block 26 "1759021865" "if (1)" -ANNOTATION: "Duplicate-tag assertion diagnostics: reset clears valid, Fill requires an absent tag or its existing way, and Invalidate only clears valid." -Block 27 "155016256" "$error(\"Assertion failed: Duplicate Cache tag\n at Cache.scala:70 assert(PopCount(hits) <= 1.U, \\"Duplicate Cache tag\\")\n\");" -ANNOTATION: "Duplicate-tag assertion diagnostics: reset clears valid, Fill requires an absent tag or its existing way, and Invalidate only clears valid." -Block 29 "1759021865" "if (1)" -ANNOTATION: "Duplicate-tag assertion diagnostics: reset clears valid, Fill requires an absent tag or its existing way, and Invalidate only clears valid." -Block 30 "3067504005" "$fatal;" -ANNOTATION: "Fill assertion diagnostics: the coherence controller must not install the same line in two ways of a set." -Block 34 "1166789632" "if (1)" -ANNOTATION: "Fill assertion diagnostics: the coherence controller must not install the same line in two ways of a set." -Block 35 "1252711441" "$error(\"Assertion failed: Cache fill would create a duplicate tag\n at Cache.scala:83 assert(!hits.orR || hits(req.way), \\"Cache fill would create a duplicate tag\\")\n\");" -ANNOTATION: "Fill assertion diagnostics: the coherence controller must not install the same line in two ways of a set." -Block 37 "1166789632" "if (1)" -ANNOTATION: "Fill assertion diagnostics: the coherence controller must not install the same line in two ways of a set." -Block 38 "3120397658" "$fatal;" -ANNOTATION: "Write assertion diagnostics: masked writes require a matching valid resident line in the selected way." -Block 42 "2426821494" "if (1)" -ANNOTATION: "Write assertion diagnostics: masked writes require a matching valid resident line in the selected way." -Block 43 "1714232348" "$error(\"Assertion failed: Cache write requires the matching resident line\n at Cache.scala:93 assert(entryValid && hits(req.way), \\"Cache write requires the matching resident line\\")\n\");" -ANNOTATION: "Write assertion diagnostics: masked writes require a matching valid resident line in the selected way." -Block 45 "2426821494" "if (1)" -ANNOTATION: "Write assertion diagnostics: masked writes require a matching valid resident line in the selected way." -Block 46 "3284370139" "$fatal;" - -CHECKSUM: "1581756980 4074018439" -INSTANCE: cache_tb.dut -ANNOTATION: "Requests are inactive during reset; this condition only gates assertion diagnostics." -Condition 1 "2427994462" "(_readData_1_T & ((~reset))) 1 -1" (2 "10") -ANNOTATION: "Reserved capacity invariant: count plus pending never exceeds responseDepth, so pending implies enq.ready." -Condition 2 "1313259708" "(((~reset)) & ( ~ (((~pending)) | _responses_io_enq_ready) )) 1 -1" (1 "01") -ANNOTATION: "Reserved capacity invariant: count plus pending never exceeds responseDepth, so pending implies enq.ready." -Condition 2 "1313259708" "(((~reset)) & ( ~ (((~pending)) | _responses_io_enq_ready) )) 1 -1" (3 "11") -ANNOTATION: "Reserved capacity invariant: count plus pending never exceeds responseDepth, so pending implies enq.ready." -Condition 4 "1917659308" "(((~pending)) | _responses_io_enq_ready) 1 -1" (1 "00") -ANNOTATION: "Only the five defined operation encodings are driven; invalid-opcode diagnostics are outside legal-operation coverage." -Condition 5 "521367662" "(_GEN_10 & (io_request_bits_op > 3'h4)) 1 -1" (1 "01") -ANNOTATION: "Only the five defined operation encodings are driven; invalid-opcode diagnostics are outside legal-operation coverage." -Condition 5 "521367662" "(_GEN_10 & (io_request_bits_op > 3'h4)) 1 -1" (3 "11") -ANNOTATION: "Addresses are line aligned; misalignment diagnostics are outside legal-operation coverage." -Condition 6 "3741468578" "(_GEN_10 & ((|io_request_bits_addr[5:0]))) 1 -1" (1 "01") -ANNOTATION: "Addresses are line aligned; misalignment diagnostics are outside legal-operation coverage." -Condition 6 "3741468578" "(_GEN_10 & ((|io_request_bits_addr[5:0]))) 1 -1" (3 "11") -ANNOTATION: "Valid tags remain unique: reset clears validity and legal fills cannot duplicate another valid way." -Condition 7 "3021235683" "(_GEN_10 & _GEN_12[1]) 1 -1" (1 "01") -ANNOTATION: "Valid tags remain unique: reset clears validity and legal fills cannot duplicate another valid way." -Condition 7 "3021235683" "(_GEN_10 & _GEN_12[1]) 1 -1" (3 "11") -ANNOTATION: "Legal fills target an absent line or its existing way; failed and reset-gated fill diagnostics are excluded." -Condition 8 "2233729839" "(_readData_1_T & _mask_T_2 & ((~reset)) & ( ~ (((~(|hits))) | _GEN_11[0]) )) 1 -1" (1 "0111") -ANNOTATION: "Legal fills target an absent line or its existing way; failed and reset-gated fill diagnostics are excluded." -Condition 8 "2233729839" "(_readData_1_T & _mask_T_2 & ((~reset)) & ( ~ (((~(|hits))) | _GEN_11[0]) )) 1 -1" (3 "1101") -ANNOTATION: "Legal fills target an absent line or its existing way; failed and reset-gated fill diagnostics are excluded." -Condition 8 "2233729839" "(_readData_1_T & _mask_T_2 & ((~reset)) & ( ~ (((~(|hits))) | _GEN_11[0]) )) 1 -1" (5 "1111") -ANNOTATION: "Legal writes match a valid line; failed and reset-gated write diagnostics are excluded." -Condition 11 "3664488518" "(_readData_1_T & _GEN_8 & ((~reset)) & ( ~ (entryValid & _GEN_11[0]) )) 1 -1" (1 "0111") -ANNOTATION: "Legal writes match a valid line; failed and reset-gated write diagnostics are excluded." -Condition 11 "3664488518" "(_readData_1_T & _GEN_8 & ((~reset)) & ( ~ (entryValid & _GEN_11[0]) )) 1 -1" (3 "1101") -ANNOTATION: "Legal writes match a valid line; failed and reset-gated write diagnostics are excluded." -Condition 11 "3664488518" "(_readData_1_T & _GEN_8 & ((~reset)) & ( ~ (entryValid & _GEN_11[0]) )) 1 -1" (5 "1111") -ANNOTATION: "A tag hit includes its valid bit. Selecting a hit cannot produce entryValid=0." -Condition 13 "2913363430" "(entryValid & _GEN_11[0]) 1 -1" (1 "01") - -CHECKSUM: "784137509 320702925" -INSTANCE: cache_tb.dut.responses -ANNOTATION: "The Cache reserves queue capacity before accepting requests; enq.valid cannot occur with enq.ready=0." -Condition 6 "2766317690" "(io_enq_ready_0 & io_enq_valid) 1 -1" (1 "01") - -CHECKSUM: "1581756980 2991418269" -INSTANCE: cache_tb.dut -ANNOTATION: "Line addresses are 64-byte aligned; low six address bits are zero." -Toggle io_request_bits_addr [5:0] "net io_request_bits_addr[43:0]" -ANNOTATION: "Line addresses are 64-byte aligned; low six address bits are zero." -Toggle io_response_bits_addr [5:0] "net io_response_bits_addr[43:0]" -ANNOTATION: "Line addresses are 64-byte aligned; low six address bits are zero." -Toggle result_addr [5:0] "reg result_addr[43:0]" - -CHECKSUM: "784137509 896502883" -INSTANCE: cache_tb.dut.responses -ANNOTATION: "Line addresses are 64-byte aligned; low six address bits are zero." -Toggle io_enq_bits_addr [5:0] "net io_enq_bits_addr[43:0]" -ANNOTATION: "Line addresses are 64-byte aligned; low six address bits are zero." -Toggle io_deq_bits_addr [5:0] "net io_deq_bits_addr[43:0]" -ANNOTATION: "Line addresses are 64-byte aligned; low six address bits are zero. Queue packing maps address[5:0] to RAM output[17:12]." -Toggle _ram_ext_R0_data [17:12] "net _ram_ext_R0_data[571:0]" diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache.f b/arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache.f deleted file mode 100644 index 3a32dc7dc..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/cache.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Cache/filelist.f -@RESOURCES@/cache_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/env.svh b/arch/src/main/scala/framework/mem-core/cache/src/main/resources/env.svh deleted file mode 100644 index cf8d93fd0..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/env.svh +++ /dev/null @@ -1,62 +0,0 @@ -class env extends checked_env #(request_item, response_item, ref_model); - `uvm_component_utils(env) - virtual cache_request_if request; - virtual cache_response_if response; - bit block_responses = 0; - int unsigned accepted = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual cache_request_if)::get( - this, "", "request", request - ) || !uvm_config_db#(virtual cache_response_if)::get( - this, "", "response", response - )) - `uvm_fatal("VIF", "Cache interfaces missing") - endfunction - task run_phase(uvm_phase phase); - request_item req; - response_item rsp; - fork - forever begin - @(response.cb); - response.cb.ready <= !response.reset && !block_responses; - end - forever begin - @(posedge request.clock); - if (request.reset) begin - scoreboard.cancel_pending(); - cache_ref_reset(model.model); - end else begin - if (request.valid && request.ready) begin - req = request_item::type_id::create("req"); - req.id = request.id; - req.op = request.op; - req.addr = request.addr; - req.way = request.way; - req.data = request.data; - req.mask = request.mask; - req.metadata = request.metadata; - req.eligible = request.eligible; - input_export.write(req); - accepted++; - end - if (response.valid && response.ready) begin - rsp = response_item::type_id::create("rsp"); - rsp.id = response.id; - rsp.hit = response.hit; - rsp.available = response.available; - rsp.entry_valid = response.entry_valid; - rsp.way = response.way; - rsp.addr = response.addr; - rsp.data = response.data; - rsp.metadata = response.metadata; - output_export.write(rsp); - end - end - end - join - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/item.svh b/arch/src/main/scala/framework/mem-core/cache/src/main/resources/item.svh deleted file mode 100644 index a073b9265..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/item.svh +++ /dev/null @@ -1,36 +0,0 @@ -class request_item extends uvm_sequence_item; - logic [`CACHE_ID_BITS-1:0] id; - logic [2:0] op; - logic [`CACHE_ADDR_BITS-1:0] addr; - logic [`CACHE_WAY_BITS-1:0] way; - bit [`CACHE_LINE_BITS-1:0] data; - bit [`CACHE_LINE_BYTES-1:0] mask; - logic [`CACHE_META_BITS-1:0] metadata; - bit [`CACHE_WAYS-1:0] eligible; - `uvm_object_utils(request_item) - function new(string name = "request_item"); - super.new(name); - endfunction -endclass - -class response_item extends uvm_sequence_item; - logic [`CACHE_ID_BITS-1:0] id; - logic hit, available, entry_valid; - logic [`CACHE_WAY_BITS-1:0] way; - logic [`CACHE_ADDR_BITS-1:0] addr; - logic [`CACHE_LINE_BITS-1:0] data; - logic [`CACHE_META_BITS-1:0] metadata; - `uvm_object_utils_begin(response_item) - `uvm_field_int(id, UVM_ALL_ON) - `uvm_field_int(hit, UVM_ALL_ON) - `uvm_field_int(available, UVM_ALL_ON) - `uvm_field_int(entry_valid, UVM_ALL_ON) - `uvm_field_int(way, UVM_ALL_ON) - `uvm_field_int(addr, UVM_ALL_ON) - `uvm_field_int(data, UVM_ALL_ON) - `uvm_field_int(metadata, UVM_ALL_ON) - `uvm_object_utils_end - function new(string name = "response_item"); - super.new(name); - endfunction -endclass diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/cache/src/main/resources/package.sv deleted file mode 100644 index c90d270f7..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/package.sv +++ /dev/null @@ -1,35 +0,0 @@ -`include "cache_config.svh" -package cache_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - localparam LOOKUP = 0, READ = 1, WRITE = 2, FILL = 3, INVALIDATE = 4; - import "DPI-C" function chandle cache_ref_create( - input int unsigned sets, - ways, - line_bytes - ); - import "DPI-C" function void cache_ref_destroy(input chandle model); - import "DPI-C" function void cache_ref_reset(input chandle model); - import "DPI-C" function void cache_ref_access( - input chandle model, - input int unsigned op, - input longint unsigned addr, - input int unsigned way, - input bit [`CACHE_LINE_BITS-1:0] data, - input bit [`CACHE_LINE_BYTES-1:0] mask, - input int unsigned metadata, - input bit [`CACHE_WAYS-1:0] eligible, - output byte unsigned hit, - available, - output int unsigned out_way, - output byte unsigned entry_valid, - output longint unsigned out_addr, - output bit [`CACHE_LINE_BITS-1:0] out_data, - output int unsigned out_metadata - ); - `include "item.svh" - `include "ref_model.svh" - `include "env.svh" - `include "test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/test.svh b/arch/src/main/scala/framework/mem-core/cache/src/main/resources/test.svh deleted file mode 100644 index 6ce2bbb9e..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/resources/test.svh +++ /dev/null @@ -1,142 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual cache_request_if request; - env test_env; - int unsigned expected = 0; - bit [`CACHE_ID_BITS-1:0] serial = 0; - bit check_rate = 1; - bit idle_between = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - test_env = env::type_id::create("env", this); - if (!uvm_config_db#(virtual cache_request_if)::get(this, "", "request", request)) - `uvm_fatal("VIF", "Cache request interface missing") - endfunction - function longint unsigned address(int tag, int set_index); - return (longint'(tag) * `CACHE_SETS + set_index) * `CACHE_LINE_BYTES; - endfunction - task send(int op, longint unsigned addr, int way = 0, bit [`CACHE_LINE_BYTES-1:0] mask = '1, - bit [`CACHE_WAYS-1:0] eligible = '1, bit [`CACHE_LINE_BITS-1:0] data = '0, - int metadata = 0); - int cycles = 0; - request.cb.valid <= 1; - request.cb.id <= serial; - request.cb.op <= op; - request.cb.addr <= addr; - request.cb.way <= way; - request.cb.data <= data; - request.cb.mask <= mask; - request.cb.metadata <= metadata; - request.cb.eligible <= eligible; - do begin - @(request.cb); - cycles++; - end while (!request.cb.ready); - if (check_rate && cycles != 1) - `uvm_fatal("THROUGHPUT", "Cache inserted a request bubble without response backpressure") - request.cb.valid <= 0; - expected++; - serial++; - if (idle_between) @(request.cb); - endtask - task drain(); - test_env.scoreboard.wait_checked(expected); - @(request.cb); - endtask - task execute(); - request.reset = 1; - request.valid = 0; - request.id = 0; - request.op = 0; - request.addr = 0; - request.way = 0; - request.data = 0; - request.mask = 0; - request.metadata = 0; - request.eligible = 0; - repeat (4) @(request.cb); - request.cb.reset <= 0; - @(request.cb); - send(LOOKUP, 0, 0, '0, '0); - send(LOOKUP, 0); - for (int s = 0; s < `CACHE_SETS; s++) begin - for (int w = 0; w < `CACHE_WAYS; w++) begin - send(READ, address(w + 1, s), w); - send(FILL, address(w + 1, s), w, '1, '1, w[0] ? '1 : '0, w); - send(LOOKUP, address(w + 1, s)); - send(WRITE, address(w + 1, s), w, {(`CACHE_LINE_BYTES / 2) {2'b01}}, '1, w[0] ? '0 : '1, - '1); - send(READ, address(w + 1, s), w); - end - end - send(LOOKUP, address(`CACHE_WAYS + 1, 0)); - send(LOOKUP, address(`CACHE_WAYS + 1, 0), 0, '0, '0); - for (int w = 0; w < `CACHE_WAYS; w++) send(LOOKUP, address(`CACHE_WAYS + 1, 0), 0, '0, 1 << w); - send(INVALIDATE, address(1, 0), 0); - send(LOOKUP, address(1, 0)); - send(FILL, address(`CACHE_WAYS + 1, 0), 0, '1, '1, '1, '1); - send(LOOKUP, address(`CACHE_WAYS + 2, 0)); - send(WRITE, address(`CACHE_WAYS + 1, 0), 0, '0, '1, '0, 0); - send(READ, address(`CACHE_WAYS + 1, 0), 0); - send(INVALIDATE, address(`CACHE_WAYS + 1, 0), 0); - send(INVALIDATE, address(`CACHE_WAYS + 1, 0), 0); - drain(); - - idle_between = 1; - for (int s = 0; s < `CACHE_SETS; s++) begin - for (int w = 0; w < `CACHE_WAYS; w++) send(INVALIDATE, address(0, s), w); - for (int w = 0; w < `CACHE_WAYS; w++) begin - longint unsigned high_addr = (((64'h1 << `CACHE_ADDR_BITS) - 1) & - ~(longint'(`CACHE_SETS * `CACHE_LINE_BYTES) - 1)) | (s * `CACHE_LINE_BYTES); - send(FILL, address(0, s), w, '1, '1, '0, 0); - send(LOOKUP, address(0, s), 0, '0, '0); - send(FILL, high_addr, w, '1, '1, '1, '1); - send(LOOKUP, high_addr); - send(WRITE, high_addr, w, '1, '1, '0, 0); - send(READ, high_addr, w); - send(FILL, address(0, s), w, '1, '1, '0, 0); - send(READ, address(0, s), w); - send(INVALIDATE, address(0, s), w); - end - end - drain(); - - idle_between = 0; - serial = '1; - send(READ, address(0, 0), 0, '0, '0); - drain(); - serial = 0; - send(READ, address(0, 0), 0, '0, '0); - drain(); - check_rate = 0; - test_env.block_responses = 1; - repeat (2) @(request.cb); - fork - begin - for (int i = 0; i < `CACHE_RESPONSE_DEPTH + 2; i++) send(LOOKUP, address(2, 0)); - end - begin - repeat (`CACHE_RESPONSE_DEPTH + 5) @(request.cb); - test_env.block_responses = 0; - end - join - drain(); - - test_env.block_responses = 1; - repeat (2) @(request.cb); - for (int i = 0; i < `CACHE_RESPONSE_DEPTH; i++) send(READ, address(1, 1), 0); - repeat (2) @(request.cb); - request.cb.reset <= 1; - repeat (2) @(request.cb); - expected = test_env.scoreboard.checked; - request.cb.reset <= 0; - test_env.block_responses = 0; - @(request.cb); - for (int s = 0; s < `CACHE_SETS; s++) send(LOOKUP, address(1, s)); - drain(); - `uvm_info("CACHE", $sformatf("Checked %0d responses", test_env.scoreboard.checked), UVM_LOW) - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Cache.scala b/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Cache.scala deleted file mode 100644 index adaac8976..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Cache.scala +++ /dev/null @@ -1,110 +0,0 @@ -package memcore.memory.cache - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.memory.cache.configs.CacheParams - -@instantiable -class Cache(p: CacheParams) extends Module { - @public - val io = IO(new CacheIO(p)) - - val valid = RegInit(VecInit(Seq.fill(p.ways)(VecInit(Seq.fill(p.sets)(false.B))))) - val tags = Reg(Vec(p.ways, Vec(p.sets, UInt(p.tagBits.W)))) - val metadata = Reg(Vec(p.ways, Vec(p.sets, UInt(p.metadataBits.W)))) - val data = Seq.fill(p.ways)(SyncReadMem(p.sets, Vec(p.lineBytes, UInt(8.W)))) - val nextWay = RegInit(VecInit(Seq.fill(p.sets)(0.U(p.wayBits.W)))) - val responses = Module(new Queue(new CacheResponse(p), p.responseDepth, pipe = true)) - val pending = RegInit(false.B) - val pendingRead = Reg(Bool()) - val pendingWay = Reg(UInt(p.wayBits.W)) - val result = Reg(new CacheResponse(p)) - - val req = io.request.bits - val set = req.addr(p.offsetBits + p.setBits - 1, p.offsetBits) - val tag = req.addr(p.addressBits - 1, p.offsetBits + p.setBits) - val hits = VecInit((0 until p.ways).map(w => valid(w)(set) && tags(w)(set) === tag)).asUInt - val invalid = VecInit((0 until p.ways).map(w => !valid(w)(set) && req.eligible(w))).asUInt - - val order = VecInit((0 until p.ways).map { offset => - val way = if (p.ways == 1) 0.U else (nextWay(set) + offset.U)(p.wayBits - 1, 0) - req.eligible(way) - }) - - val replacement = - if (p.ways == 1) 0.U - else - (nextWay(set) + PriorityEncoder(order.asUInt))(p.wayBits - 1, 0) - - val lookupWay = Mux(hits.orR, PriorityEncoder(hits), Mux(invalid.orR, PriorityEncoder(invalid), replacement)) - val lookup = req.op === CacheOp.Lookup.U - val selected = Mux(lookup, lookupWay, req.way) - val available = !lookup || hits.orR || req.eligible.orR - val entryValid = available && valid(selected)(set) - val read = (lookup || req.op === CacheOp.Read.U) && entryValid - - io.request.ready := (responses.io.count +& pending.asUInt) < p.responseDepth.U || responses.io.deq.fire - responses.io.enq.valid := pending - responses.io.enq.bits := result - io.response <> responses.io.deq - pending := io.request.fire - assert(!pending || responses.io.enq.ready, "Cache response capacity reservation failed") - - val readData = Wire(Vec(p.ways, UInt(p.lineBits.W))) - for (way <- 0 until p.ways) { - val write = req.op === CacheOp.Write.U || req.op === CacheOp.Fill.U - val mask = Mux(req.op === CacheOp.Fill.U, Fill(p.lineBytes, 1.U(1.W)), req.mask) - readData(way) := data(way).readWrite( - set, - req.data.asTypeOf(Vec(p.lineBytes, UInt(8.W))), - mask.asBools, - io.request.fire && selected === way.U && (read || write), - write - ).asUInt - } - when(pendingRead) { - responses.io.enq.bits.data := readData(pendingWay) - } - - when(io.request.fire) { - assert(req.op <= CacheOp.Invalidate.U, "Unknown Cache operation") - assert(req.addr(p.offsetBits - 1, 0) === 0.U, "Cache operations require a line-aligned address") - assert(lookup || req.way < p.ways.U, "Cache way exceeds associativity") - assert(PopCount(hits) <= 1.U, "Duplicate Cache tag") - pendingRead := read - pendingWay := selected - result.id := req.id - result.hit := lookup && hits.orR - result.available := available - result.way := Mux(available, selected, 0.U) - result.entryValid := entryValid - result.addr := Mux(entryValid, Cat(tags(selected)(set), set, 0.U(p.offsetBits.W)), 0.U) - result.data := 0.U - result.metadata := Mux(entryValid, metadata(selected)(set), 0.U) - - when(req.op === CacheOp.Fill.U) { - assert(!hits.orR || hits(req.way), "Cache fill would create a duplicate tag") - valid(req.way)(set) := true.B - tags(req.way)(set) := tag - metadata(req.way)(set) := req.metadata - nextWay(set) := (if (p.ways == 1) 0.U else (req.way + 1.U)(p.wayBits - 1, 0)) - result.entryValid := true.B - result.addr := req.addr - result.metadata := req.metadata - } - when(req.op === CacheOp.Write.U) { - assert(entryValid && hits(req.way), "Cache write requires the matching resident line") - metadata(req.way)(set) := req.metadata - result.metadata := req.metadata - } - when(req.op === CacheOp.Invalidate.U) { - valid(req.way)(set) := false.B - result.entryValid := false.B - result.addr := 0.U - result.metadata := 0.U - } - } -} diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Emit.scala deleted file mode 100644 index b0139da49..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Emit.scala +++ /dev/null @@ -1,29 +0,0 @@ -package memcore.memory.cache - -import memcore.memory.cache.configs.CacheParams - -object Emit extends App { - val p = CacheParams.load("configs/default.toml") - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Cache(p), - firtoolOpts = args, - args = Array("--target-dir", "build/Cache", "--split-verilog") - ) - - val constants = Seq( - "ADDR_BITS" -> p.addressBits, - "LINE_BYTES" -> p.lineBytes, - "LINE_BITS" -> p.lineBits, - "SETS" -> p.sets, - "WAYS" -> p.ways, - "WAY_BITS" -> p.wayBits, - "ID_BITS" -> p.idBits, - "META_BITS" -> p.metadataBits, - "RESPONSE_DEPTH" -> p.responseDepth - ).map { case (name, value) => s"`define CACHE_$name $value" }.mkString("\n") - - java.nio.file.Files.writeString( - java.nio.file.Paths.get("build/cache_config.svh"), - s"`ifndef CACHE_CONFIG_SVH\n`define CACHE_CONFIG_SVH\n$constants\n`endif\n" - ) -} diff --git a/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Interface.scala b/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Interface.scala deleted file mode 100644 index 78abd4b65..000000000 --- a/arch/src/main/scala/framework/mem-core/cache/src/main/scala/Interface.scala +++ /dev/null @@ -1,42 +0,0 @@ -package memcore.memory.cache - -import chisel3._ -import chisel3.util._ -import memcore.memory.cache.configs.CacheParams - -object CacheOp { - val Lookup = 0 - val Read = 1 - val Write = 2 - val Fill = 3 - val Invalidate = 4 -} - -class CacheRequest(p: CacheParams) extends Bundle { - val id = UInt(p.idBits.W) - val op = UInt(3.W) - // Read and Invalidate address a physical set/way; all operations are line aligned. - val addr = UInt(p.addressBits.W) - val way = UInt(p.wayBits.W) - val data = UInt(p.lineBits.W) - val mask = UInt(p.lineBytes.W) - val metadata = UInt(p.metadataBits.W) - // Eligibility constrains miss replacement candidates, not tag hits. - val eligible = UInt(p.ways.W) -} - -class CacheResponse(p: CacheParams) extends Bundle { - val id = UInt(p.idBits.W) - val hit = Bool() - val available = Bool() - val way = UInt(p.wayBits.W) - val entryValid = Bool() - val addr = UInt(p.addressBits.W) - val data = UInt(p.lineBits.W) - val metadata = UInt(p.metadataBits.W) -} - -class CacheIO(p: CacheParams) extends Bundle { - val request = Flipped(Decoupled(new CacheRequest(p))) - val response = Decoupled(new CacheResponse(p)) -} diff --git a/arch/src/main/scala/framework/mem-core/chi/build.sc b/arch/src/main/scala/framework/mem-core/chi/build.sc deleted file mode 100644 index 17404bc1b..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/build.sc +++ /dev/null @@ -1,20 +0,0 @@ -import mill._ -import mill.scalalib._ - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object chi extends SbtModule { - override def moduleDeps = Seq(queue) - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") - override def mainClass = Some("memcore.bus.chi.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/chi/src/csrc/src/lib.rs b/arch/src/main/scala/framework/mem-core/chi/src/csrc/src/lib.rs deleted file mode 100644 index 8845fc066..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/csrc/src/lib.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod rnf; -mod memory; -pub use memory::*; diff --git a/arch/src/main/scala/framework/mem-core/chi/src/csrc/src/rnf.rs b/arch/src/main/scala/framework/mem-core/chi/src/csrc/src/rnf.rs deleted file mode 100644 index 9d99db67e..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/csrc/src/rnf.rs +++ /dev/null @@ -1,110 +0,0 @@ -use std::{collections::BTreeMap, ffi::c_void}; - -struct Memory { - limit: u64, - lines: BTreeMap, -} - -#[no_mangle] -pub extern "C" fn rnf_ref_create(address_bits: u32) -> *mut c_void { - assert!((44..=52).contains(&address_bits)); - Box::into_raw(Box::new(Memory { - limit: 1u64 << address_bits, - lines: BTreeMap::new(), - })).cast() -} - -#[no_mangle] -pub unsafe extern "C" fn rnf_ref_destroy(model: *mut c_void) { - drop(Box::from_raw(model.cast::())); -} - -#[no_mangle] -pub unsafe extern "C" fn rnf_ref_write( - model: *mut c_void, addr: u64, data: *const u32, mask: *const u32, -) -> u8 { - let memory = &mut *model.cast::(); - assert_eq!(addr % 64, 0); - if addr >= memory.limit { return 1; } - let data = std::slice::from_raw_parts(data, 16); - let mask = std::slice::from_raw_parts(mask, 2); - let full = mask.iter().all(|&word| word == u32::MAX); - assert!(memory.lines.contains_key(&addr) || full, "Partial write before initialization"); - let line = memory.lines.entry(addr).or_insert([0; 64]); - for b in 0..64 { - if mask[b / 32] & (1 << (b % 32)) != 0 { - line[b] = (data[b / 4] >> (8 * (b % 4))) as u8; - } - } - 0 -} - -#[no_mangle] -pub unsafe extern "C" fn rnf_ref_read( - model: *mut c_void, addr: u64, data: *mut u32, -) -> u8 { - let memory = &*model.cast::(); - assert_eq!(addr % 64, 0); - let output = std::slice::from_raw_parts_mut(data, 16); - output.fill(0); - if addr >= memory.limit { return 1; } - let line = memory.lines.get(&addr).expect("Read of uninitialized line"); - for (word, bytes) in output.iter_mut().zip(line.chunks_exact(4)) { - *word = u32::from_le_bytes(bytes.try_into().unwrap()); - } - 0 -} - -#[no_mangle] -pub unsafe extern "C" fn rnf_ref_access( - model: *mut c_void, addr: u64, write: u8, operand: u64, - atomic: u32, word: u8, mask: u8, sc_success: u8, failed: u8, -) -> u64 { - let memory = &mut *model.cast::(); - assert!(atomic <= 12); - assert!(word == 0 || (1..=11).contains(&atomic)); - assert_eq!(addr % if word != 0 { 4 } else { 8 }, 0); - assert!(atomic == 0 || (write == 0 && mask == 255)); - if failed != 0 { return 0; } - let line = memory.lines.get_mut(&(addr & !63)).expect("CPU access to uninitialized line"); - let offset = (addr & 63) as usize; - let (old, update) = if word != 0 { - let lhs = u32::from_le_bytes(line[offset..offset + 4].try_into().unwrap()); - let rhs = operand as u32; - let result = match atomic { - 2 => lhs.wrapping_add(rhs), - 3 => lhs ^ rhs, - 4 => lhs & rhs, - 5 => lhs | rhs, - 6 => if (lhs as i32) < (rhs as i32) { lhs } else { rhs }, - 7 => if (lhs as i32) > (rhs as i32) { lhs } else { rhs }, - 8 => lhs.min(rhs), - 9 => lhs.max(rhs), - _ => rhs, - }; - ((lhs as i32 as i64) as u64, result as u64) - } else { - let lhs = u64::from_le_bytes(line[offset..offset + 8].try_into().unwrap()); - let rhs = operand; - let result = match atomic { - 2 => lhs.wrapping_add(rhs), - 3 => lhs ^ rhs, - 4 => lhs & rhs, - 5 => lhs | rhs, - 6 => if (lhs as i64) < (rhs as i64) { lhs } else { rhs }, - 7 => if (lhs as i64) > (rhs as i64) { lhs } else { rhs }, - 8 => lhs.min(rhs), - 9 => lhs.max(rhs), - _ => rhs, - }; - (lhs, result) - }; - if write != 0 || (1..=9).contains(&atomic) || (atomic == 11 && sc_success != 0) { - for b in 0..if word != 0 { 4 } else { 8 } { - if word != 0 || mask & (1 << b) != 0 { - line[offset + b] = (update >> (8 * b)) as u8; - } - } - } - match atomic { 11 => u64::from(sc_success == 0), 12 => 0, _ => old } -} diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/chi_profile.svh b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/chi_profile.svh deleted file mode 100644 index fbb04f396..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/chi_profile.svh +++ /dev/null @@ -1,12 +0,0 @@ -`ifndef CHI_PROFILE_SVH -`define CHI_PROFILE_SVH -`ifndef CHI_DATA_BITS -`define CHI_DATA_BITS 256 -`endif -`define CHI_BE_BITS (`CHI_DATA_BITS / 8) -`define CHI_TAG_BITS (`CHI_DATA_BITS / 32) -`define CHI_TAG_UPDATE_BITS (`CHI_DATA_BITS / 128) -`define CHI_DAT_BITS (91 + `CHI_DATA_BITS + `CHI_BE_BITS + `CHI_TAG_BITS + `CHI_TAG_UPDATE_BITS) -`define CHI_BEATS (512 / `CHI_DATA_BITS) -`define CHI_DATA_ID_STEP (`CHI_DATA_BITS / 128) -`endif diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_interface.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_interface.sv deleted file mode 100644 index d9ff4688a..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_interface.sv +++ /dev/null @@ -1,6 +0,0 @@ -interface chi_credit_if ( - input logic clock -); - logic reset, active, pend, valid, lcrdv; - logic [31:0] flit; -endinterface diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_package.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_package.sv deleted file mode 100644 index a1b42639e..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_package.sv +++ /dev/null @@ -1,43 +0,0 @@ -package chi_credit_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual chi_credit_if vif; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual chi_credit_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "Credit interface missing") - endfunction - task execute(); - vif.reset = 1; - vif.active = 0; -`ifdef CHI_TX_OVERFLOW - vif.lcrdv = 0; -`else - vif.pend = 0; - vif.valid = 0; - vif.flit = 0; -`endif - repeat (4) @(negedge vif.clock); - vif.reset = 0; - vif.active = 1; -`ifdef CHI_TX_OVERFLOW - // Peer returns five credits to a four-credit transmitter with no traffic. - vif.lcrdv = 1; - repeat (8) @(negedge vif.clock); -`else - // Peer uses a fifth flit after exhausting four credits; receiver cannot drain. - vif.pend = 1; - repeat (8) @(negedge vif.clock); - vif.valid = 1; - repeat (6) @(negedge vif.clock); -`endif - `uvm_fatal("NEGATIVE", "Credit violation did not terminate RTL") - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_tb.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_tb.sv deleted file mode 100644 index 2c4a48591..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/credit_tb.sv +++ /dev/null @@ -1,39 +0,0 @@ -module `CHI_CREDIT_TB; - import uvm_pkg::*; - import chi_credit_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_credit_if source_if (clock); -`ifdef CHI_TX_OVERFLOW - Tx dut ( - .clock(clock), - .reset(source_if.reset), - .io_active(source_if.active), - .io_in_valid(1'b0), - .io_in_ready(), - .io_in_bits(32'b0), - .io_credits(), - .io_link_flitpend(), - .io_link_flitv(), - .io_link_flit(), - .io_link_lcrdv(source_if.lcrdv) - ); -`else - Rx dut ( - .clock(clock), - .reset(source_if.reset), - .io_active(source_if.active), - .io_link_flitpend(source_if.pend), - .io_link_flitv(source_if.valid), - .io_link_flit(source_if.flit), - .io_link_lcrdv(source_if.lcrdv), - .io_out_valid(), - .io_out_ready(1'b0), - .io_out_bits() - ); -`endif - initial begin - uvm_config_db#(virtual chi_credit_if)::set(null, "uvm_test_top", "vif", source_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link.el b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link.el deleted file mode 100644 index 28e83891b..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link.el +++ /dev/null @@ -1,45 +0,0 @@ -// Format Version: 2 -// Reviewed against the exact exported object signatures and the declared depth4 always-on profile. - -CHECKSUM: "1668538250 4134186348" -INSTANCE: link_tb.dut -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit overflow" -Block 2 "239387013" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit overflow" -Block 3 "4289838947" "$error(\"Assertion failed: CHI TX credit overflow\n at Tx.scala:43 when(returned && !send)(assert(credits < maxCredits.U, \\"CHI TX credit overflow\\"))\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit overflow" -Block 5 "239387013" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit overflow" -Block 6 "1750606003" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit underflow" -Block 10 "1503763571" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit underflow" -Block 11 "556320376" "$error(\"Assertion failed: CHI TX credit underflow\n at Tx.scala:44 when(send)(assert(credits =/= 0.U, \\"CHI TX credit underflow\\"))\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit underflow" -Block 13 "1503763571" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit underflow" -Block 14 "2166023300" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX requires coordinated reset to stop" -Block 18 "2653430091" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX requires coordinated reset to stop" -Block 19 "597017625" "$error(\"Assertion failed: CHI TX requires coordinated reset to stop\n at Tx.scala:45 when(wasActive)(assert(io.active, \\"CHI TX requires coordinated reset to stop\\"))\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX requires coordinated reset to stop" -Block 21 "2653430091" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX requires coordinated reset to stop" -Block 22 "2212642507" "$fatal;" - - -CHECKSUM: "1668538250 4069976122" -INSTANCE: link_tb.dut -ANNOTATION: "Tx32/depth4 peer: credit=4 means all four credits are available; another returned credit cannot simultaneously be pending. This vector would violate peer credit conservation." -Condition 1 "284300870" "(returned & ((~send)) & ((~reset)) & credits[2]) 1 -1" (2 "1011") -ANNOTATION: "Tx32/depth4 peer: no returned credit can coexist with all four credits available, including the coordinated reset boundary." -Condition 1 "284300870" "(returned & ((~send)) & ((~reset)) & credits[2]) 1 -1" (3 "1101") -ANNOTATION: "Assertion-failure vector: overgrant violates the declared four-credit peer contract. Independently checked by tx_bad_credit." -Condition 1 "284300870" "(returned & ((~send)) & ((~reset)) & credits[2]) 1 -1" (5 "1111") -ANNOTATION: "Structural invariant: send requires ready, and ready requires nonzero credits; send with zero credits is impossible even on reset." -Condition 2 "3481575346" "(send & ((~reset)) & ((~(|credits)))) 1 -1" (2 "101") -ANNOTATION: "Structural invariant: send implies credits != 0 through io.in.ready." -Condition 2 "3481575346" "(send & ((~reset)) & ((~(|credits)))) 1 -1" (4 "111") -ANNOTATION: "Always-on profile forbids deactivation without coordinated reset; this is the assertion-failure vector, not normal shutdown support." -Condition 3 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (4 "111") diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link.f deleted file mode 100644 index e7175b587..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/link_interface.sv -@RESOURCES@/link_package.sv --F @RTL@/Tx/filelist.f --F @RTL@/Rx/filelist.f -@RESOURCES@/link_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_interface.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_interface.sv deleted file mode 100644 index 6fbdfe9e8..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_interface.sv +++ /dev/null @@ -1,9 +0,0 @@ -interface chi_link_if ( - input logic clock -); - logic reset, active, valid, ready; - logic [31:0] data; - logic out_valid, out_ready; - logic [31:0] out_data; - logic [ 2:0] credits; -endinterface diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_package.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_package.sv deleted file mode 100644 index 7b2a0c97f..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_package.sv +++ /dev/null @@ -1,128 +0,0 @@ -package chi_link_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual chi_link_if vif; - bit [31:0] pending[$]; - int accepted = 0, checked = 0, cycle = 0; - bit stalled = 0; - bit [31:0] held; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual chi_link_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "CHI credit link missing") - endfunction - task monitor(); - forever begin - @(posedge vif.clock); - if (vif.reset) begin - pending.delete(); - stalled = 0; - end else begin - cycle++; - if (vif.valid && vif.ready) begin - pending.push_back(vif.data); - accepted++; - end - if (stalled && (!vif.out_valid || vif.out_data !== held)) - `uvm_fatal("STABLE", "Output changed while stalled") - if (vif.out_valid && vif.out_ready) begin - bit [31:0] golden; - if (!pending.size()) `uvm_fatal("UNEXPECTED", "Output without request") - golden = pending.pop_front(); - if (vif.out_data !== golden) `uvm_fatal("DATA", "Credit link data/order mismatch") - checked++; - end - stalled = vif.out_valid && !vif.out_ready; - held = vif.out_data; - if (vif.credits > 4) `uvm_fatal("CREDIT", "Credit count exceeds depth") - end - end - endtask - task restart(); - @(negedge vif.clock); - vif.reset = 1; - vif.active = 0; - vif.valid = 0; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - repeat (2) @(negedge vif.clock); - vif.active = 1; - repeat (8) @(negedge vif.clock); - endtask - task execute(); - vif.reset = 1; - vif.active = 0; - vif.valid = 0; - vif.data = 0; - vif.out_ready = 0; - fork - monitor(); - join_none - restart(); - for (int i = 0; i < 256; i++) begin - vif.valid = 1; - vif.data = (32'h9e3779b9 * (i + 1)) ^ ((i & 4) ? 32'hffffffff : 32'h0); - do begin - @(posedge vif.clock); - if (vif.ready) begin - @(negedge vif.clock); - break; - end - @(negedge vif.clock); - vif.out_ready = cycle % 7 < 3; - end while (1); - vif.out_ready = cycle % 7 < 3; - end - vif.valid = 0; - vif.out_ready = 1; - while (pending.size()) @(negedge vif.clock); - // Coordinated reset cancels buffered traffic and restores initial credits. - vif.out_ready = 0; - repeat (4) begin - vif.valid = 1; - vif.data = 32'hfedcba98; - do @(posedge vif.clock); while (!vif.ready); - @(negedge vif.clock); - end - vif.valid = 0; - repeat (8) @(negedge vif.clock); - restart(); - vif.out_ready = 1; - for (int i = 0; i < 32; i++) begin - vif.valid = 1; - vif.data = i; - do @(posedge vif.clock); while (!vif.ready); - @(negedge vif.clock); - end - vif.valid = 0; - while (pending.size()) @(negedge vif.clock); - repeat (8) @(negedge vif.clock); - // A coordinated reset can interrupt a physical flit before RX captures it. - vif.valid = 1; - vif.data = 32'h01234567; - do @(posedge vif.clock); while (!vif.ready); - @(negedge vif.clock); - vif.reset = 1; - vif.active = 0; - vif.valid = 0; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - repeat (2) @(negedge vif.clock); - vif.active = 1; - repeat (8) @(negedge vif.clock); - if (checked != 288 || accepted != 293 || vif.credits != 4) - `uvm_fatal("COUNT", $sformatf( - "accepted=%0d checked=%0d credits=%0d", accepted, checked, vif.credits)) - `uvm_info( - "CHECKS", - "CHI credit link: 288 ordered outputs checked, 5 reset-cancelled (4 buffered, 1 in-flight)", - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_tb.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_tb.sv deleted file mode 100644 index b1c59c0d8..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/link_tb.sv +++ /dev/null @@ -1,38 +0,0 @@ -module link_tb; - import uvm_pkg::*; - import chi_link_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_link_if source_if (clock); - wire flitpend, flitv, lcrdv; - wire [31:0] flit; - Tx dut ( - .clock(clock), - .reset(source_if.reset), - .io_active(source_if.active), - .io_in_valid(source_if.valid), - .io_in_ready(source_if.ready), - .io_in_bits(source_if.data), - .io_link_flitpend(flitpend), - .io_link_flitv(flitv), - .io_link_flit(flit), - .io_link_lcrdv(lcrdv), - .io_credits(source_if.credits) - ); - Rx receiver ( - .clock(clock), - .reset(source_if.reset), - .io_active(source_if.active), - .io_link_flitpend(flitpend), - .io_link_flitv(flitv), - .io_link_flit(flit), - .io_link_lcrdv(lcrdv), - .io_out_valid(source_if.out_valid), - .io_out_ready(source_if.out_ready), - .io_out_bits(source_if.out_data) - ); - initial begin - uvm_config_db#(virtual chi_link_if)::set(null, "uvm_test_top", "vif", source_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry.el b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry.el deleted file mode 100644 index 1b7ca84fc..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry.el +++ /dev/null @@ -1,415 +0,0 @@ -// Format Version: 2 -// Exact reviewed objects; grouped scopes and proofs, with equivalent adjacent bit/transition ranges. - -CHECKSUM: "1565409055 1106117561" -INSTANCE: retry_tb.dut -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unsupported CHI retry request opcode" -Block 2 "1744103448" "if (1)" -Block 3 "675298159" "$error(\"Assertion failed: Unsupported CHI retry request opcode\n at RequestRetry.scala:64 assert(coherent || noSnoop, \\"Unsupported CHI retry request opcode\\")\n\");" -Block 5 "1744103448" "if (1)" -Block 6 "3310318889" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unsupported CHI retry request attributes" -Block 10 "3534321110" "if (1)" -Block 11 "3076932440" "$error(\"Assertion failed: Unsupported CHI retry request attributes\n at RequestRetry.scala:65 assert(\n\");" -Block 13 "3534321110" "if (1)" -Block 14 "882429221" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Invalid retry completion-ack profile" -Block 18 "3921772647" "if (1)" -Block 19 "3527252594" "$error(\"Assertion failed: Invalid retry completion-ack profile\n at RequestRetry.scala:71 assert(r.expCompAck === coherentRead.asUInt, \\"Invalid retry completion-ack profile\\")\n\");" -Block 21 "3921772647" "if (1)" -Block 22 "2211796926" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Invalid retry coherent request attributes" -Block 26 "568216364" "if (1)" -Block 27 "439395600" "$error(\"Assertion failed: Invalid retry coherent request attributes\n at RequestRetry.scala:73 assert(r.returnNid === 0.U && r.returnTxnId === 0.U && r.memAttr === 12.U && r.snpAttr === 1.U,\n\");" -Block 29 "568216364" "if (1)" -Block 30 "1967310581" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Invalid retry non-snoop request attributes" -Block 34 "2827715915" "if (1)" -Block 35 "4275880477" "$error(\"Assertion failed: Invalid retry non-snoop request attributes\n at RequestRetry.scala:76 assert(!r.memAttr(1) && r.snpAttr === 0.U, \\"Invalid retry non-snoop request attributes\\")\n\");" -Block 37 "2827715915" "if (1)" -Block 38 "246045839" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Invalid retry requester" -Block 42 "1709660459" "if (1)" -Block 43 "4204837901" "$error(\"Assertion failed: Invalid retry requester\n at RequestRetry.scala:78 assert(io.reqIn.bits.srcId === nodeId.U, \\"Invalid retry requester\\")\n\");" -Block 45 "1709660459" "if (1)" -Block 46 "2514228879" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI retry requester reused a live TxnID" -Block 50 "1112493139" "if (1)" -Block 51 "2040280706" "$error(\"Assertion failed: CHI retry requester reused a live TxnID\n at RequestRetry.scala:79 assert(!duplicate, \\"CHI retry requester reused a live TxnID\\")\n\");" -Block 53 "1112493139" "if (1)" -Block 54 "1089463653" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Initial CHI request must allow retry without a protocol credit" -Block 58 "130518244" "if (1)" -Block 59 "4023718724" "$error(\"Assertion failed: Initial CHI request must allow retry without a protocol credit\n at RequestRetry.scala:80 assert(\n\");" -Block 61 "130518244" "if (1)" -Block 62 "422011914" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Response to wrong retry requester" -Block 66 "297091616" "if (1)" -Block 67 "3803880083" "$error(\"Assertion failed: Response to wrong retry requester\n at RequestRetry.scala:131 assert(r.tgtId === nodeId.U, \\"Response to wrong retry requester\\")\n\");" -Block 69 "297091616" "if (1)" -Block 70 "3411174966" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unexpected or repeated RetryAck" -Block 74 "2784317146" "if (1)" -Block 75 "3648346299" "$error(\"Assertion failed: Unexpected or repeated RetryAck\n at RequestRetry.scala:135 assert(matches.asUInt.orR && state(i) === firstAttempt && r.respErr === 0.U, \\"Unexpected or repeated RetryAck\\")\n\");" -Block 77 "2784317146" "if (1)" -Block 78 "74728005" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Excess or errored protocol credit" -Block 82 "2447225933" "if (1)" -Block 83 "3962228737" "$error(\"Assertion failed: Excess or errored protocol credit\n at RequestRetry.scala:142 assert(space.asUInt.orR && r.respErr === 0.U, \\"Excess or errored protocol credit\\")\n\");" -Block 85 "2447225933" "if (1)" -Block 86 "891196801" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Response accepted before request reissue" -Block 90 "3397368675" "if (1)" -Block 91 "2039620193" "$error(\"Assertion failed: Response accepted before request reissue\n at RequestRetry.scala:124 assert(state(i) =/= retryWait, \\"Response accepted before request reissue\\")\n\");" -Block 93 "3397368675" "if (1)" -Block 94 "1797854015" "$fatal;" -Block 98 "397495833" "if (1)" -Block 99 "587374098" "$error(\"Assertion failed: Response accepted before request reissue\n at RequestRetry.scala:124 assert(state(i) =/= retryWait, \\"Response accepted before request reissue\\")\n\");" -Block 101 "397495833" "if (1)" -Block 102 "834252620" "$fatal;" - -CHECKSUM: "1565409055 2680414210" -INSTANCE: retry_tb.dut -ANNOTATION: "RequestRetry asserted legal request contract: NodeID1, 64-byte aligned access, supported opcodes below64, normal memory (Device=0), PAS/Order/exclusive/tag/multi-request/stash=0, LPID lower5 only. All higher address and arbitrary TxnID/routing bits stay measured." -Toggle io_reqOut_bits_srcId [6:0] "net io_reqOut_bits_srcId[6:0]" -Toggle io_reqOut_bits_stashNidValidEndian "net io_reqOut_bits_stashNidValidEndian" -Toggle io_reqOut_bits_opcode [6] "net io_reqOut_bits_opcode[6:0]" -Toggle io_reqOut_bits_multiReq "net io_reqOut_bits_multiReq" -Toggle io_reqOut_bits_size [5:0] "net io_reqOut_bits_size[5:0]" -Toggle io_reqOut_bits_addr [5:0] "net io_reqOut_bits_addr[43:0]" -Toggle io_reqOut_bits_pas [2:0] "net io_reqOut_bits_pas[2:0]" -Toggle io_reqOut_bits_order [1:0] "net io_reqOut_bits_order[1:0]" -Toggle io_reqOut_bits_memAttr [1] "net io_reqOut_bits_memAttr[3:0]" -Toggle io_reqOut_bits_lpid [7:5] "net io_reqOut_bits_lpid[7:0]" -Toggle io_reqOut_bits_exclSnoopMe "net io_reqOut_bits_exclSnoopMe" -Toggle io_reqOut_bits_tagOp [1:0] "net io_reqOut_bits_tagOp[1:0]" -Toggle _arb_io_out_bits_srcId [6:0] "net _arb_io_out_bits_srcId[6:0]" -Toggle _arb_io_out_bits_stashNidValidEndian "net _arb_io_out_bits_stashNidValidEndian" -Toggle _arb_io_out_bits_opcode [6] "net _arb_io_out_bits_opcode[6:0]" -Toggle _arb_io_out_bits_multiReq "net _arb_io_out_bits_multiReq" -Toggle _arb_io_out_bits_size [5:0] "net _arb_io_out_bits_size[5:0]" -Toggle _arb_io_out_bits_addr [5:0] "net _arb_io_out_bits_addr[43:0]" -Toggle _arb_io_out_bits_pas [2:0] "net _arb_io_out_bits_pas[2:0]" -Toggle _arb_io_out_bits_order [1:0] "net _arb_io_out_bits_order[1:0]" -Toggle _arb_io_out_bits_memAttr [1] "net _arb_io_out_bits_memAttr[3:0]" -Toggle _arb_io_out_bits_lpid [7:5] "net _arb_io_out_bits_lpid[7:0]" -Toggle _arb_io_out_bits_exclSnoopMe "net _arb_io_out_bits_exclSnoopMe" -Toggle _arb_io_out_bits_tagOp [1:0] "net _arb_io_out_bits_tagOp[1:0]" -Toggle requests_0_srcId [6:0] "reg requests_0_srcId[6:0]" -Toggle requests_0_stashNidValidEndian "reg requests_0_stashNidValidEndian" -Toggle requests_0_opcode [6] "reg requests_0_opcode[6:0]" -Toggle requests_0_multiReq "reg requests_0_multiReq" -Toggle requests_0_size [5:0] "reg requests_0_size[5:0]" -Toggle requests_0_addr [5:0] "reg requests_0_addr[43:0]" -Toggle requests_0_pas [2:0] "reg requests_0_pas[2:0]" -Toggle requests_0_order [1:0] "reg requests_0_order[1:0]" -Toggle requests_0_memAttr [1] "reg requests_0_memAttr[3:0]" -Toggle requests_0_lpid [7:5] "reg requests_0_lpid[7:0]" -Toggle requests_0_exclSnoopMe "reg requests_0_exclSnoopMe" -Toggle requests_0_tagOp [1:0] "reg requests_0_tagOp[1:0]" -Toggle requests_1_srcId [6:0] "reg requests_1_srcId[6:0]" -Toggle requests_1_stashNidValidEndian "reg requests_1_stashNidValidEndian" -Toggle requests_1_opcode [6] "reg requests_1_opcode[6:0]" -Toggle requests_1_multiReq "reg requests_1_multiReq" -Toggle requests_1_size [5:0] "reg requests_1_size[5:0]" -Toggle requests_1_addr [5:0] "reg requests_1_addr[43:0]" -Toggle requests_1_pas [2:0] "reg requests_1_pas[2:0]" -Toggle requests_1_order [1:0] "reg requests_1_order[1:0]" -Toggle requests_1_memAttr [1] "reg requests_1_memAttr[3:0]" -Toggle requests_1_lpid [7:5] "reg requests_1_lpid[7:0]" -Toggle requests_1_exclSnoopMe "reg requests_1_exclSnoopMe" -Toggle requests_1_tagOp [1:0] "reg requests_1_tagOp[1:0]" -Toggle requests_2_srcId [6:0] "reg requests_2_srcId[6:0]" -Toggle requests_2_stashNidValidEndian "reg requests_2_stashNidValidEndian" -Toggle requests_2_opcode [6] "reg requests_2_opcode[6:0]" -Toggle requests_2_multiReq "reg requests_2_multiReq" -Toggle requests_2_size [5:0] "reg requests_2_size[5:0]" -Toggle requests_2_addr [5:0] "reg requests_2_addr[43:0]" -Toggle requests_2_pas [2:0] "reg requests_2_pas[2:0]" -Toggle requests_2_order [1:0] "reg requests_2_order[1:0]" -Toggle requests_2_memAttr [1] "reg requests_2_memAttr[3:0]" -Toggle requests_2_lpid [7:5] "reg requests_2_lpid[7:0]" -Toggle requests_2_exclSnoopMe "reg requests_2_exclSnoopMe" -Toggle requests_2_tagOp [1:0] "reg requests_2_tagOp[1:0]" -Toggle requests_3_srcId [6:0] "reg requests_3_srcId[6:0]" -Toggle requests_3_stashNidValidEndian "reg requests_3_stashNidValidEndian" -Toggle requests_3_opcode [6] "reg requests_3_opcode[6:0]" -Toggle requests_3_multiReq "reg requests_3_multiReq" -Toggle requests_3_size [5:0] "reg requests_3_size[5:0]" -Toggle requests_3_addr [5:0] "reg requests_3_addr[43:0]" -Toggle requests_3_pas [2:0] "reg requests_3_pas[2:0]" -Toggle requests_3_order [1:0] "reg requests_3_order[1:0]" -Toggle requests_3_memAttr [1] "reg requests_3_memAttr[3:0]" -Toggle requests_3_lpid [7:5] "reg requests_3_lpid[7:0]" -Toggle requests_3_exclSnoopMe "reg requests_3_exclSnoopMe" -Toggle requests_3_tagOp [1:0] "reg requests_3_tagOp[1:0]" - -CHECKSUM: "917436931 2269459522" -INSTANCE: retry_tb.dut.arb -ANNOTATION: "RequestRetry asserted legal request contract: NodeID1, 64-byte aligned access, supported opcodes below64, normal memory (Device=0), PAS/Order/exclusive/tag/multi-request/stash=0, LPID lower5 only. All higher address and arbitrary TxnID/routing bits stay measured." -Toggle io_in_1_bits_srcId [6:0] "net io_in_1_bits_srcId[6:0]" -Toggle io_in_1_bits_stashNidValidEndian "net io_in_1_bits_stashNidValidEndian" -Toggle io_in_1_bits_opcode [6] "net io_in_1_bits_opcode[6:0]" -Toggle io_in_1_bits_multiReq "net io_in_1_bits_multiReq" -Toggle io_in_1_bits_size [5:0] "net io_in_1_bits_size[5:0]" -Toggle io_in_1_bits_addr [5:0] "net io_in_1_bits_addr[43:0]" -Toggle io_in_1_bits_pas [2:0] "net io_in_1_bits_pas[2:0]" -Toggle io_in_1_bits_order [1:0] "net io_in_1_bits_order[1:0]" -Toggle io_in_1_bits_memAttr [1] "net io_in_1_bits_memAttr[3:0]" -Toggle io_in_1_bits_lpid [7:5] "net io_in_1_bits_lpid[7:0]" -Toggle io_in_1_bits_exclSnoopMe "net io_in_1_bits_exclSnoopMe" -Toggle io_in_1_bits_tagOp [1:0] "net io_in_1_bits_tagOp[1:0]" -Toggle io_in_2_bits_srcId [6:0] "net io_in_2_bits_srcId[6:0]" -Toggle io_in_2_bits_stashNidValidEndian "net io_in_2_bits_stashNidValidEndian" -Toggle io_in_2_bits_opcode [6] "net io_in_2_bits_opcode[6:0]" -Toggle io_in_2_bits_multiReq "net io_in_2_bits_multiReq" -Toggle io_in_2_bits_size [5:0] "net io_in_2_bits_size[5:0]" -Toggle io_in_2_bits_addr [5:0] "net io_in_2_bits_addr[43:0]" -Toggle io_in_2_bits_pas [2:0] "net io_in_2_bits_pas[2:0]" -Toggle io_in_2_bits_order [1:0] "net io_in_2_bits_order[1:0]" -Toggle io_in_2_bits_memAttr [1] "net io_in_2_bits_memAttr[3:0]" -Toggle io_in_2_bits_lpid [7:5] "net io_in_2_bits_lpid[7:0]" -Toggle io_in_2_bits_exclSnoopMe "net io_in_2_bits_exclSnoopMe" -Toggle io_in_2_bits_tagOp [1:0] "net io_in_2_bits_tagOp[1:0]" -Toggle io_in_3_bits_srcId [6:0] "net io_in_3_bits_srcId[6:0]" -Toggle io_in_3_bits_stashNidValidEndian "net io_in_3_bits_stashNidValidEndian" -Toggle io_in_3_bits_opcode [6] "net io_in_3_bits_opcode[6:0]" -Toggle io_in_3_bits_multiReq "net io_in_3_bits_multiReq" -Toggle io_in_3_bits_size [5:0] "net io_in_3_bits_size[5:0]" -Toggle io_in_3_bits_addr [5:0] "net io_in_3_bits_addr[43:0]" -Toggle io_in_3_bits_pas [2:0] "net io_in_3_bits_pas[2:0]" -Toggle io_in_3_bits_order [1:0] "net io_in_3_bits_order[1:0]" -Toggle io_in_3_bits_memAttr [1] "net io_in_3_bits_memAttr[3:0]" -Toggle io_in_3_bits_lpid [7:5] "net io_in_3_bits_lpid[7:0]" -Toggle io_in_3_bits_exclSnoopMe "net io_in_3_bits_exclSnoopMe" -Toggle io_in_3_bits_tagOp [1:0] "net io_in_3_bits_tagOp[1:0]" -Toggle io_in_4_bits_srcId [6:0] "net io_in_4_bits_srcId[6:0]" -Toggle io_in_4_bits_stashNidValidEndian "net io_in_4_bits_stashNidValidEndian" -Toggle io_in_4_bits_opcode [6] "net io_in_4_bits_opcode[6:0]" -Toggle io_in_4_bits_multiReq "net io_in_4_bits_multiReq" -Toggle io_in_4_bits_size [5:0] "net io_in_4_bits_size[5:0]" -Toggle io_in_4_bits_addr [5:0] "net io_in_4_bits_addr[43:0]" -Toggle io_in_4_bits_pas [2:0] "net io_in_4_bits_pas[2:0]" -Toggle io_in_4_bits_order [1:0] "net io_in_4_bits_order[1:0]" -Toggle io_in_4_bits_memAttr [1] "net io_in_4_bits_memAttr[3:0]" -Toggle io_in_4_bits_lpid [7:5] "net io_in_4_bits_lpid[7:0]" -Toggle io_in_4_bits_exclSnoopMe "net io_in_4_bits_exclSnoopMe" -Toggle io_in_4_bits_tagOp [1:0] "net io_in_4_bits_tagOp[1:0]" -Toggle io_out_bits_srcId [6:0] "net io_out_bits_srcId[6:0]" -Toggle io_out_bits_stashNidValidEndian "net io_out_bits_stashNidValidEndian" -Toggle io_out_bits_opcode [6] "net io_out_bits_opcode[6:0]" -Toggle io_out_bits_multiReq "net io_out_bits_multiReq" -Toggle io_out_bits_size [5:0] "net io_out_bits_size[5:0]" -Toggle io_out_bits_addr [5:0] "net io_out_bits_addr[43:0]" -Toggle io_out_bits_pas [2:0] "net io_out_bits_pas[2:0]" -Toggle io_out_bits_order [1:0] "net io_out_bits_order[1:0]" -Toggle io_out_bits_memAttr [1] "net io_out_bits_memAttr[3:0]" -Toggle io_out_bits_lpid [7:5] "net io_out_bits_lpid[7:0]" -Toggle io_out_bits_exclSnoopMe "net io_out_bits_exclSnoopMe" -Toggle io_out_bits_tagOp [1:0] "net io_out_bits_tagOp[1:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured srcId of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_2 [1][6:0] "net [7:0][6:0]_GEN_2" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured srcId of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_2 [2][6:0] "net [7:0][6:0]_GEN_2" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured srcId of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_2 [3][6:0] "net [7:0][6:0]_GEN_2" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured srcId of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_2 [4][6:0] "net [7:0][6:0]_GEN_2" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured stashNidValidEndian of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_5 [1] "net _GEN_5[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured stashNidValidEndian of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_5 [2] "net _GEN_5[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured stashNidValidEndian of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_5 [3] "net _GEN_5[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured stashNidValidEndian of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_5 [4] "net _GEN_5[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured opcode of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_7 [1][6] "net [7:0][6:0]_GEN_7" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured opcode of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_7 [2][6] "net [7:0][6:0]_GEN_7" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured opcode of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_7 [3][6] "net [7:0][6:0]_GEN_7" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured opcode of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_7 [4][6] "net [7:0][6:0]_GEN_7" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured multiReq of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_8 [1] "net _GEN_8[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured multiReq of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_8 [2] "net _GEN_8[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured multiReq of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_8 [3] "net _GEN_8[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured multiReq of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_8 [4] "net _GEN_8[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured size of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_9 [1][5:0] "net [7:0][5:0]_GEN_9" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured size of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_9 [2][5:0] "net [7:0][5:0]_GEN_9" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured size of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_9 [3][5:0] "net [7:0][5:0]_GEN_9" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured size of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_9 [4][5:0] "net [7:0][5:0]_GEN_9" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured addr of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_10 [1][5:0] "net [7:0][43:0]_GEN_10" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured addr of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_10 [2][5:0] "net [7:0][43:0]_GEN_10" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured addr of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_10 [3][5:0] "net [7:0][43:0]_GEN_10" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured addr of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_10 [4][5:0] "net [7:0][43:0]_GEN_10" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured pas of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_11 [1][2:0] "net [7:0][2:0]_GEN_11" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured pas of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_11 [2][2:0] "net [7:0][2:0]_GEN_11" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured pas of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_11 [3][2:0] "net [7:0][2:0]_GEN_11" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured pas of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_11 [4][2:0] "net [7:0][2:0]_GEN_11" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is literal 1'h0. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_13 [4:1] "net _GEN_13[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured order of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_14 [1][1:0] "net [7:0][1:0]_GEN_14" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured order of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_14 [2][1:0] "net [7:0][1:0]_GEN_14" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured order of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_14 [3][1:0] "net [7:0][1:0]_GEN_14" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured order of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_14 [4][1:0] "net [7:0][1:0]_GEN_14" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured memAttr of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_16 [1][1] "net [7:0][3:0]_GEN_16" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured memAttr of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_16 [2][1] "net [7:0][3:0]_GEN_16" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured memAttr of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_16 [3][1] "net [7:0][3:0]_GEN_16" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured memAttr of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_16 [4][1] "net [7:0][3:0]_GEN_16" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured lpid of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_18 [1][7:5] "net [7:0][7:0]_GEN_18" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured lpid of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_18 [2][7:5] "net [7:0][7:0]_GEN_18" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured lpid of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_18 [3][7:5] "net [7:0][7:0]_GEN_18" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured lpid of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_18 [4][7:5] "net [7:0][7:0]_GEN_18" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured exclSnoopMe of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_19 [1] "net _GEN_19[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured exclSnoopMe of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_19 [2] "net _GEN_19[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured exclSnoopMe of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_19 [3] "net _GEN_19[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured exclSnoopMe of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_19 [4] "net _GEN_19[7:0]" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured tagOp of replay port1. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_21 [1][1:0] "net [7:0][1:0]_GEN_21" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured tagOp of replay port2. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_21 [2][1:0] "net [7:0][1:0]_GEN_21" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured tagOp of replay port3. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_21 [3][1:0] "net [7:0][1:0]_GEN_21" -ANNOTATION: "RRArbiter mux source mapping proves this exact element is captured tagOp of replay port4. Replay request registers capture only asserted valid initial requests (NodeID1,Size6,aligned64B,normal memory,zero optional profile fields); replay AllowRetry is literal0. Initial-input and its padded aliases remain measured because invalid payload is arbitrary. No routing/TxnID/payload field is excluded." -Toggle _GEN_21 [4][1:0] "net [7:0][1:0]_GEN_21" - -CHECKSUM: "3478893755 3492225878" -INSTANCE: retry_tb.dut.output_0 -ANNOTATION: "RequestRetry asserted legal request contract: NodeID1, 64-byte aligned access, supported opcodes below64, normal memory (Device=0), PAS/Order/exclusive/tag/multi-request/stash=0, LPID lower5 only. All higher address and arbitrary TxnID/routing bits stay measured." -Toggle io_enq_bits_srcId [6:0] "net io_enq_bits_srcId[6:0]" -Toggle io_enq_bits_stashNidValidEndian "net io_enq_bits_stashNidValidEndian" -Toggle io_enq_bits_opcode [6] "net io_enq_bits_opcode[6:0]" -Toggle io_enq_bits_multiReq "net io_enq_bits_multiReq" -Toggle io_enq_bits_size [5:0] "net io_enq_bits_size[5:0]" -Toggle io_enq_bits_addr [5:0] "net io_enq_bits_addr[43:0]" -Toggle io_enq_bits_pas [2:0] "net io_enq_bits_pas[2:0]" -Toggle io_enq_bits_order [1:0] "net io_enq_bits_order[1:0]" -Toggle io_enq_bits_memAttr [1] "net io_enq_bits_memAttr[3:0]" -Toggle io_enq_bits_lpid [7:5] "net io_enq_bits_lpid[7:0]" -Toggle io_enq_bits_exclSnoopMe "net io_enq_bits_exclSnoopMe" -Toggle io_enq_bits_tagOp [1:0] "net io_enq_bits_tagOp[1:0]" -Toggle io_deq_bits_srcId [6:0] "net io_deq_bits_srcId[6:0]" -Toggle io_deq_bits_stashNidValidEndian "net io_deq_bits_stashNidValidEndian" -Toggle io_deq_bits_opcode [6] "net io_deq_bits_opcode[6:0]" -Toggle io_deq_bits_multiReq "net io_deq_bits_multiReq" -Toggle io_deq_bits_size [5:0] "net io_deq_bits_size[5:0]" -Toggle io_deq_bits_addr [5:0] "net io_deq_bits_addr[43:0]" -Toggle io_deq_bits_pas [2:0] "net io_deq_bits_pas[2:0]" -Toggle io_deq_bits_order [1:0] "net io_deq_bits_order[1:0]" -Toggle io_deq_bits_memAttr [1] "net io_deq_bits_memAttr[3:0]" -Toggle io_deq_bits_lpid [7:5] "net io_deq_bits_lpid[7:0]" -Toggle io_deq_bits_exclSnoopMe "net io_deq_bits_exclSnoopMe" -Toggle io_deq_bits_tagOp [1:0] "net io_deq_bits_tagOp[1:0]" -ANNOTATION: "Output Queue stores the asserted legal RequestRetry request profile. Exact packed positions are read from generated per-field assignments, preserving all full-width address/routing/TxnID and variable attribute bits." -Toggle _ram_ext_R0_data [17:11] "net _ram_ext_R0_data[136:0]" -Toggle _ram_ext_R0_data [37] "net _ram_ext_R0_data[136:0]" -Toggle _ram_ext_R0_data [69:56] "net _ram_ext_R0_data[136:0]" -Toggle _ram_ext_R0_data [110:108] "net _ram_ext_R0_data[136:0]" -Toggle _ram_ext_R0_data [114:113] "net _ram_ext_R0_data[136:0]" -Toggle _ram_ext_R0_data [120] "net _ram_ext_R0_data[136:0]" -Toggle _ram_ext_R0_data [132:129] "net _ram_ext_R0_data[136:0]" -Toggle _ram_ext_R0_data [135:134] "net _ram_ext_R0_data[136:0]" - -CHECKSUM: "1565409055 2845470659" -INSTANCE: retry_tb.dut -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Invalid retry completion-ack profile. Functional gates and assertion remain enabled." -Condition 29 "4177145288" "(_GEN_25 & (io_reqIn_bits_expCompAck != coherentRead)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Invalid retry non-snoop request attributes. Functional gates and assertion remain enabled." -Condition 37 "2137277555" "(io_reqIn_valid & ((~coherent)) & ((~reset)) & ( ~ (((~io_reqIn_bits_memAttr[1])) & ((~io_reqIn_bits_snpAttr))) )) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Invalid retry requester. Functional gates and assertion remain enabled." -Condition 40 "745923699" "(_GEN_25 & (io_reqIn_bits_srcId != 7'b1)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI retry requester reused a live TxnID. Functional gates and assertion remain enabled." -Condition 42 "3108172069" "(_GEN_25 & ((|_duplicate_T_12))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Initial CHI request must allow retry without a protocol credit. Functional gates and assertion remain enabled." -Condition 43 "108006917" "(_GEN_25 & ( ~ (io_reqIn_bits_allowRetry & (io_reqIn_bits_pCrdType == 4'b0)) )) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Response to wrong retry requester. Functional gates and assertion remain enabled." -Condition 47 "1101780613" "(_GEN_21 & ((~reset)) & (io_rspIn_bits_tgtId != 7'b1)) 1 -1" (4 "111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Response accepted before request reissue. Functional gates and assertion remain enabled." -Condition 56 "3214681341" "(_GEN_30 & ((~grant)) & _GEN_22 & ((|_GEN_23)) & ((~reset)) & (_GEN_28[i_1] == 2'h2)) 1 -1" (7 "111111") -Condition 58 "4231516992" "(io_acceptedData_valid & ((|_GEN_24)) & ((~reset)) & (_GEN_28[i_2] == 2'h2)) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unsupported CHI retry request opcode. Functional gates and assertion remain enabled." -Condition 14 "2551668066" "(_GEN_25 & ( ~ (coherent | (io_reqIn_bits_opcode == 7'h04) | (io_reqIn_bits_opcode == 7'h1d) | (io_reqIn_bits_opcode == 7'h1c)) )) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unsupported CHI retry request attributes. Functional gates and assertion remain enabled." -Condition 20 "1109361100" "(_GEN_25 & ( ~ (((~io_reqIn_bits_multiReq)) & (io_reqIn_bits_order == 2'b0) & (io_reqIn_bits_size == 6'h06) & (io_reqIn_bits_addr[5:0] == 6'b0) & (io_reqIn_bits_pas == 3'b0) & ((~io_reqIn_bits_exclSnoopMe)) & (io_reqIn_bits_tagOp == 2'b0) & ((~io_reqIn_bits_stashNidValidEndian)) & (io_reqIn_bits_lpid[7:5] == 3'b0)) )) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Invalid retry coherent request attributes. Functional gates and assertion remain enabled." -Condition 31 "517468456" "(io_reqIn_valid & coherent & ((~reset)) & ( ~ ((io_reqIn_bits_returnNid == 7'b0) & (io_reqIn_bits_returnTxnId == 12'b0) & (io_reqIn_bits_memAttr == 4'hc) & io_reqIn_bits_snpAttr) )) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unexpected or repeated RetryAck. Functional gates and assertion remain enabled." -Condition 49 "3899675501" "(_GEN_21 & retryAck & ((~reset)) & ( ~ (((|{((|state_3) & (requests_3_txnId == io_rspIn_bits_txnId)), matches_4_2, matches_4_1, matches_4_0})) & (_GEN_28[i] == 2'b1) & _GEN_29) )) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Excess or errored protocol credit. Functional gates and assertion remain enabled." -Condition 53 "873833217" "(_GEN_30 & grant & ((~reset)) & ( ~ (((|{(((~creditValid_3) | (_GEN_17 & (&credit_3))) | (_GEN_13 ? (((&credit_2) | _GEN_27) | _GEN_26) : (_GEN_27 | _GEN_26))), space_2, space_1, space_0})) & _GEN_29) )) 1 -1" (5 "1111") -ANNOTATION: "RetryAck decodes RSP opcode3, while each selected credit allocation term decodes PCrdGrant opcode7. One opcode cannot be both; exact vector would require both decodes." -Condition 82 "73012043" "(((~_GEN_21)) | retryAck | ((~_GEN_39))) 1 -1" (3 "010") -Condition 83 "379528980" "(((~_GEN_21)) | retryAck | ((~_GEN_40))) 1 -1" (3 "010") -Condition 84 "2400437582" "(((~_GEN_21)) | retryAck | ((~_GEN_41))) 1 -1" (3 "010") -Condition 85 "3656520365" "(((~_GEN_21)) | retryAck | ((~_GEN_42))) 1 -1" (3 "010") -ANNOTATION: "This subexpression is evaluated only in the ternary branch where a later replay port fires. The second term consumes credit through an earlier replay port. RRArbiter grants exactly one output per cycle, so the two replay fires cannot coexist; exact second-only vector is unreachable in this branch." -Condition 97 "3162203369" "(_GEN_7 | _GEN_47) 1 -1" (2 "01") -Condition 102 "2336271217" "(_GEN_9 | _GEN_48) 1 -1" (2 "01") -Condition 107 "1864735692" "(_GEN_11 | _GEN_49) 1 -1" (2 "01") -Condition 112 "167182624" "(((&credit_1)) | _GEN_50) 1 -1" (2 "01") -Condition 141 "1112285567" "(_GEN_18 | _GEN_55) 1 -1" (2 "01") -Condition 148 "3255526067" "(_GEN_19 | _GEN_56) 1 -1" (2 "01") -Condition 155 "2170964985" "(_GEN_20 | _GEN_57) 1 -1" (2 "01") -Condition 162 "2904700612" "(((&credit_3)) | _GEN_58) 1 -1" (2 "01") -ANNOTATION: "Four-entry PriorityEncoder returns fallback index3 when all live-TxnID matches are0. No-match with selected index0,1 or2 is structurally impossible, independent of payload or initialization." -Condition 114 "91711063" "(io_acceptedData_valid & ((|_GEN_24)) & (i_2 == 2'b0)) 1 -1" (2 "101") -Condition 118 "3535129296" "(_GEN_22 & ((|_GEN_23)) & (i_1 == 2'b0)) 1 -1" (2 "101") -Condition 120 "1021694279" "(io_acceptedData_valid & ((|_GEN_24)) & (i_2 == 2'b1)) 1 -1" (2 "101") -Condition 124 "3944017961" "(_GEN_22 & ((|_GEN_23)) & (i_1 == 2'b1)) 1 -1" (2 "101") -Condition 126 "3486384718" "(io_acceptedData_valid & ((|_GEN_24)) & (i_2 == 2'h2)) 1 -1" (2 "101") -Condition 130 "1247087977" "(_GEN_22 & ((|_GEN_23)) & (i_1 == 2'h2)) 1 -1" (2 "101") -ANNOTATION: "Acceptance opcode decode is DBIDResp6/Comp4/CompDBIDResp5; grant is PCrdGrant7. The exact vector requires grant and acceptance simultaneously, impossible for one RSP opcode." -Condition 116 "1348440713" "(grant | ( ~ (_GEN_22 & ((|_GEN_23)) & (i_1 == 2'b0)) )) 1 -1" (3 "10") -Condition 122 "559201839" "(grant | ( ~ (_GEN_22 & ((|_GEN_23)) & (i_1 == 2'b1)) )) 1 -1" (3 "10") -Condition 128 "2154737347" "(grant | ( ~ (_GEN_22 & ((|_GEN_23)) & (i_1 == 2'h2)) )) 1 -1" (3 "10") -Condition 133 "1731676477" "(grant | ( ~ (_GEN_22 & ((|_GEN_23)) & ((&i_1))) )) 1 -1" (3 "10") -ANNOTATION: "Selected credit allocation implies PCrdGrant opcode7, mutually exclusive with RetryAck opcode3. Exact vector requires both opcodes." -Condition 137 "3997230361" "(_GEN_21 & ((~retryAck)) & _GEN_39) 1 -1" (2 "101") -Condition 144 "312808281" "(_GEN_21 & ((~retryAck)) & _GEN_40) 1 -1" (2 "101") -Condition 151 "2618281630" "(_GEN_21 & ((~retryAck)) & _GEN_41) 1 -1" (2 "101") -Condition 158 "3436279767" "(_GEN_21 & ((~retryAck)) & _GEN_42) 1 -1" (2 "101") -ANNOTATION: "This ternary branch is selected by replay port3 fire. Its earlier-port consumption terms require another replay fire in the same cycle, forbidden by the single-output RRArbiter. Exact earlier-only vectors are unreachable; current-port selection remains measured." -Condition 272 "1429792695" "(_GEN_14 | _GEN_8 | _GEN_1) 1 -1" (2 "001") -Condition 272 "1429792695" "(_GEN_14 | _GEN_8 | _GEN_1) 1 -1" (3 "010") -Condition 277 "3581322673" "(_GEN_15 | _GEN_10 | _GEN_3) 1 -1" (2 "001") -Condition 277 "3581322673" "(_GEN_15 | _GEN_10 | _GEN_3) 1 -1" (3 "010") -Condition 282 "3852876295" "(_GEN_16 | _GEN_12 | _GEN_5) 1 -1" (2 "001") -Condition 282 "3852876295" "(_GEN_16 | _GEN_12 | _GEN_5) 1 -1" (3 "010") - -CHECKSUM: "917436931 1648997668" -INSTANCE: retry_tb.dut.arb -ANNOTATION: "Replay port4 becomes live only after its slot was accepted through initial port0. A port4 grant changes that slot to reissued and removes its valid immediately. Before port4 can become valid again, an initial port0 grant is mandatory and sets lastGrant0. Other intervening grants are ports1..3. Thus live port4 cannot coexist with lastGrant4; values5..7 cannot survive the required initial grant either." -Condition 6 "1359597920" "(io_in_4_valid & ((~ctrl_validMask_grantMask_lastGrant[2]))) 1 -1" (2 "10") diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry.f deleted file mode 100644 index 86c8eb77b..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/retry_interface.sv -@RESOURCES@/retry_package.sv --F @RTL@/RequestRetry/filelist.f -@RESOURCES@/retry_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_bad_opcode.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_bad_opcode.f deleted file mode 100644 index 71d35c4f9..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_bad_opcode.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+CHI_RETRY_BAD_OPCODE -+define+CHI_RETRY_TB=retry_bad_opcode_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/retry_interface.sv -@RESOURCES@/retry_package.sv --F @RTL@/RequestRetry/filelist.f -@RESOURCES@/retry_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_bad_opcode.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_bad_opcode.toml deleted file mode 100644 index e0e04bd9f..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_bad_opcode.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Unsupported CHI retry request opcode" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_duplicate_txnid.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_duplicate_txnid.f deleted file mode 100644 index 81e0cd39c..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_duplicate_txnid.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+CHI_RETRY_DUPLICATE -+define+CHI_RETRY_TB=retry_duplicate_txnid_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/retry_interface.sv -@RESOURCES@/retry_package.sv --F @RTL@/RequestRetry/filelist.f -@RESOURCES@/retry_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_duplicate_txnid.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_duplicate_txnid.toml deleted file mode 100644 index cca6bc008..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_duplicate_txnid.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CHI retry requester reused a live TxnID" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_interface.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_interface.sv deleted file mode 100644 index c1059d20b..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_interface.sv +++ /dev/null @@ -1,21 +0,0 @@ -interface chi_retry_if ( - input logic clock -); - logic reset; - logic req_in_valid, req_in_ready, req_out_valid, req_out_ready; - logic [136:0] req_in, req_out; - logic rsp_in_valid, rsp_in_ready, rsp_out_valid, rsp_out_ready; - logic [70:0] rsp_in, rsp_out; - logic accepted_valid; - logic [6:0] accepted_src; - logic [11:0] accepted_txn; - logic [2:0] pending; - assert property (@(posedge clock) disable iff (reset) - req_out_valid && !req_out_ready |=> req_out_valid && $stable( - req_out - )); - assert property (@(posedge clock) disable iff (reset) - rsp_out_valid && !rsp_out_ready |=> rsp_out_valid && $stable( - rsp_out - )); -endinterface diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_package.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_package.sv deleted file mode 100644 index 7e1405f15..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_package.sv +++ /dev/null @@ -1,480 +0,0 @@ -package chi_retry_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `include "flit_types.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual chi_retry_if vif; - req_t requests[$]; - rsp_t responses[$], original_responses[$]; - int checks = 0; - req_t originals[int]; - int retry_types[int]; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual chi_retry_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "Retry interface missing") - endfunction - task monitor(); - forever begin - @(posedge vif.clock); - if (vif.reset) begin - requests.delete(); - responses.delete(); - original_responses.delete(); - end else begin - if (vif.req_out_valid && vif.req_out_ready) requests.push_back(req_t'(vif.req_out)); - if (vif.rsp_out_valid && vif.rsp_out_ready) responses.push_back(rsp_t'(vif.rsp_out)); - end - end - endtask - task send_request(req_t r); - @(negedge vif.clock); - vif.req_in = r; - vif.req_in_valid = 1; - do @(posedge vif.clock); while (!vif.req_in_ready); - originals[r.txn] = r; - @(negedge vif.clock); - vif.req_in_valid = 0; - endtask - function bit coherent_read(int opcode); - return opcode == 1 || opcode == 'h26 || opcode == 7; - endfunction - function bit coherent_request(int opcode); - return coherent_read(opcode) || opcode == 'hd || opcode == 'h1b || opcode == 8 || - opcode == 9 || opcode == 10; - endfunction - task request(int txn, int opcode = 'hd); - req_t r = '0; - r.src = 1; - r.tgt = 64; - r.txn = txn; - r.opcode = opcode; - r.size = 6; - r.addr = 64 * txn; - r.allow_retry = 1; - if (coherent_request(opcode)) begin - r.mem_attr = 12; - r.snp_attr = 1; - end else begin - r.mem_attr = 4; - r.return_nid = 1; - r.return_txn = txn; - end - r.exp_comp_ack = coherent_read(opcode); - send_request(r); - endtask - task response(int opcode, int txn, int src = 65, int credit_type = 3, bit contiguous = 0); - rsp_t r = '0; - r.opcode = opcode; - r.txn = txn; - r.src = src; - r.tgt = 1; - r.pcrd_type = (opcode == 3 || opcode == 7) ? credit_type : 0; - r.qos = (txn ^ src) % 16; - r.trace_tag = txn % 2; - if (opcode == 6 || opcode == 5) r.dbid = (txn ^ src ^ 'h555) & 'hfff; - if (opcode == 4 || opcode == 6 || opcode == 5) r.busy = (txn + src) % 8; - if (opcode == 4 && txn % 2) r.error = 3; // Dataless completion may report NDERR. - if (!contiguous) @(negedge vif.clock); - vif.rsp_in = r; - vif.rsp_in_valid = 1; - do @(posedge vif.clock); while (!vif.rsp_in_ready); - if (opcode == 3) retry_types[txn] = credit_type; - if (opcode != 3 && opcode != 7) original_responses.push_back(r); - @(negedge vif.clock); - vif.rsp_in_valid = 0; - endtask - task expect_request(int txn, bit retry); - req_t r, golden; - while (!requests.size()) @(negedge vif.clock); - r = requests.pop_front(); - if (!originals.exists(txn)) `uvm_fatal("MODEL", "Missing observed original request") - golden = originals[txn]; - if (retry) begin - golden.allow_retry = 0; - golden.pcrd_type = retry_types[txn]; - end - if (r !== golden) - `uvm_fatal("RETRY", $sformatf( - "Complete request changed txn=%0h got=%h expected=%h", txn, r, golden)) - checks++; - endtask - task receive_response(output rsp_t r); - rsp_t golden; - while (!responses.size()) @(negedge vif.clock); - if (!original_responses.size()) `uvm_fatal("MODEL", "Missing response/original") - r = responses.pop_front(); - golden = original_responses.pop_front(); - if (r !== golden) `uvm_fatal("RESPONSE", "Complete response metadata changed") - endtask - task accept_data(int txn, int source); - @(negedge vif.clock); - vif.accepted_valid = 1; - vif.accepted_src = source; - vif.accepted_txn = txn; - @(negedge vif.clock); - vif.accepted_valid = 0; - endtask - task execute(); - rsp_t completed; - vif.reset = 1; - vif.req_in_valid = 0; - vif.rsp_in_valid = 0; - vif.accepted_valid = 0; - vif.req_in = 0; - vif.rsp_in = 0; - vif.accepted_src = 0; - vif.accepted_txn = 0; - vif.req_out_ready = 1; - vif.rsp_out_ready = 1; - fork - monitor(); - join_none - repeat (4) @(negedge vif.clock); - vif.reset = 0; -`ifdef CHI_RETRY_DUPLICATE - request(12'hf01); - expect_request(12'hf01, 0); - request(12'hf01); - `uvm_fatal("NEGATIVE", "Duplicate live TxnID did not terminate RTL") -`elsif CHI_RETRY_BAD_OPCODE - request(0, 'h11); // Separate response/data acceptance is outside this profile. - `uvm_fatal("NEGATIVE", "Unsupported retry opcode did not terminate RTL") -`else - // A normal completion passes through and releases the request record. - request(0); - expect_request(0, 0); - vif.rsp_out_ready = 0; - fork - response(4, 0); - begin - repeat (12) @(negedge vif.clock); - vif.rsp_out_ready = 1; - end - join - receive_response(completed); - if (completed.opcode != 4) `uvm_fatal("PASS", "Completion was swallowed") - for (int i = 0; i < 4; i++) begin - request(i); - expect_request(i, 0); - end - for (int i = 0; i < 4; i++) response(3, i); - repeat (10) @(negedge vif.clock); - if (requests.size() || responses.size() || original_responses.size() || vif.pending != 4) - `uvm_fatal("WAIT", "Retry issued before P-Credit or internal response leaked") - // Home remapped from TgtID 64 to RetryAck SrcID 65; match that credit source. - response(7, 0, 66, 3); - response(7, 0, 65, 2); - repeat (10) @(negedge vif.clock); - if (requests.size()) `uvm_fatal("MATCH", "Mismatched protocol credit consumed") - for (int i = 0; i < 4; i++) begin - response(7, 0); - expect_request(i, 1); - response(4, i); - receive_response(completed); - if (completed.txn != i) `uvm_fatal("COMP", "Bad completion"); - end - // An output stall must preserve the queued request and its record. - vif.req_out_ready = 0; - request(0); - repeat (12) @(negedge vif.clock); - if (!vif.req_out_valid || requests.size()) - `uvm_fatal("STALL", "Queued request lost under stall") - vif.req_out_ready = 1; - expect_request(0, 0); - response(4, 0); - receive_response(completed); - if (completed.txn != 0) `uvm_fatal("COMP", "Bad stalled completion"); - // Grant can precede RetryAck and must be retained for its matching retry. - request(0, 4); - expect_request(0, 0); - response(7, 0); - response(3, 0); - expect_request(0, 1); - @(negedge vif.clock); - vif.accepted_valid = 1; - vif.accepted_src = 9; - vif.accepted_txn = 0; - @(negedge vif.clock); - vif.accepted_valid = 0; - repeat (4) @(negedge vif.clock); - if (vif.pending != 0 || requests.size() || responses.size() || original_responses.size()) - `uvm_fatal("DRAIN", "Retry records did not drain") - // Reset cancels retry-wait records and retained protocol credits. - request(0); - expect_request(0, 0); - response(3, 0); - @(negedge vif.clock); - vif.reset = 1; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - request(0); - expect_request(0, 0); - response(4, 0); - receive_response(completed); - if (completed.txn != 0) `uvm_fatal("RESET", "Completion after reset corrupted"); - repeat (4) @(negedge vif.clock); - if (vif.pending != 0 || requests.size() || responses.size() || original_responses.size()) - `uvm_fatal("DRAIN", "Retry reset did not drain") - // Full protocol IDs are independent of the four local acceptance slots. - // Exercise both polarities of every aligned address, TxnID and routing-ID bit. - for (int phase_index = 0; phase_index < 3; phase_index++) begin - int polarity = phase_index % 2; - for (int address_bit = 6; address_bit < 44; address_bit++) begin - int ids[4], homes[4], types[4], opcodes[4]; - bit [3:0] seen_reissues = '0; - int supported_ops[11] = '{1, 'h26, 7, 'hd, 'h1b, 8, 9, 10, 4, 'h1d, 'h1c}; - for (int slot = 0; slot < 4; slot++) begin - req_t r = '0; - int node_bits = (1 << ((address_bit + slot) % 7)) | 2; - ids[slot] = (1 << ((address_bit + slot) % 12)) ^ (polarity ? 'hfff : 0); - homes[slot] = (address_bit % 3 == 0 ? 65 : node_bits ^ (polarity ? 'h7f : 0)); - types[slot] = (address_bit % 3 == 1 ? address_bit % 16 : (address_bit + slot) % 16); - r.src = 1; - r.tgt = homes[slot]; - r.txn = ids[slot]; - opcodes[slot] = supported_ops[(address_bit+slot+polarity)%11]; - r.opcode = opcodes[slot]; - r.size = 6; - r.lpid = (address_bit + slot) % 32; - r.likely_shared = (address_bit + slot) % 2; - if (coherent_request(opcodes[slot])) begin - r.mem_attr = 12; - r.snp_attr = 1; - end else begin - case ((address_bit + slot) % 3) - 0: r.mem_attr = 0; - 1: r.mem_attr = 4; - 2: r.mem_attr = 12; - endcase - r.mem_attr |= (address_bit + slot + polarity) % 2; // EWA is legal for Normal memory. - r.return_nid = homes[slot]; - r.return_txn = ids[slot] ^ 'hfff; - end - r.exp_comp_ack = coherent_read(opcodes[slot]); - r.addr = ((64'h1 << address_bit) ^ (polarity ? 64'hfffffffffc0 : 0)); - r.qos = (address_bit + 3 * slot) % 16; - r.trace_tag = polarity; - r.allow_retry = 1; - send_request(r); - expect_request(ids[slot], 0); - end - if (vif.pending != 4) - `uvm_fatal("SLOTS", "Arbitrary IDs did not fill four acceptance slots") - vif.rsp_out_ready = 0; // Internal retry traffic must bypass endpoint backpressure. - for (int position = 0; position < 4; position++) begin - int slot = (position + address_bit / 4) % 4; - response(7, 0, homes[slot], types[slot]); - end - if (address_bit % 5 == 0) vif.req_out_ready = 0; - for (int position = 0; position < 4; position++) begin - int slot = (address_bit + (address_bit % 2 ? 3 - position : position)) % 4; - response(3, ids[slot], homes[slot], types[slot], position != 0); - end - repeat (3) @(negedge vif.clock); - vif.req_out_ready = 1; - vif.rsp_out_ready = 1; - for (int position = 0; position < 4; position++) begin - int slot = -1; - while (!requests.size()) @(negedge vif.clock); - for (int candidate = 0; candidate < 4; candidate++) - if (requests[0].txn == ids[candidate]) slot = candidate; - if (slot < 0 || seen_reissues[slot]) - `uvm_fatal("RETRY_SET", "Unexpected or duplicate reissued transaction") - seen_reissues[slot] = 1; - expect_request(ids[slot], 1); - if (coherent_read(opcodes[slot]) || opcodes[slot] == 4) begin - accept_data(ids[slot], homes[slot]); - end else begin - int acceptance=(opcodes[slot]=='h1b || opcodes[slot]=='h1d || opcodes[slot]=='h1c) ? 6 : 4; - if (acceptance == 6 && (address_bit + slot) % 2) acceptance = 5; - response(acceptance, ids[slot], homes[slot], 0); - receive_response(completed); - if (completed.txn != ids[slot] || completed.opcode != acceptance) - `uvm_fatal("PASS", "Acceptance response corrupted") - end - end - repeat (3) @(negedge vif.clock); - if (vif.pending != 0 || requests.size() || responses.size() || original_responses.size()) - `uvm_fatal("DRAIN", "Full-width request batch did not drain") - end - end - // Fill the two-entry output queue and hold a third request under backpressure. - vif.req_out_ready = 0; - fork - begin - for (int slot = 0; slot < 4; slot++) request(slot); - end - begin - repeat (20) @(negedge vif.clock); - vif.req_out_ready = 1; - end - join - for (int slot = 0; slot < 4; slot++) expect_request(slot, 0); - for (int slot = 0; slot < 4; slot++) begin - response(4, slot); - receive_response(completed); - end - // Initial allocation overlaps independent RetryAck and completion handshakes. - for (int context_kind = 0; context_kind < 2; context_kind++) begin - for (int slot = 0; slot < 4; slot++) begin - int other = (slot + 1) % 4; - for (int i = 0; i < 4; i++) begin - request(100 + i); - expect_request(100 + i, 0); - end - response(4, 100 + slot); - receive_response(completed); - fork - request(200 + slot); - response(context_kind ? 4 : 3, 100 + other); - join - expect_request(200 + slot, 0); - if (!context_kind) begin - response(7, 0); - expect_request(100 + other, 1); - end else begin - receive_response(completed); - end - for (int i = 0; i < 4; i++) begin - if (context_kind && i == other) continue; - response(4, i == slot ? 200 + slot : 100 + i); - receive_response(completed); - end - end - end - // A new P-Credit may arrive in the cycle a different credit is consumed. - for (int slot = 0; slot < 4; slot++) begin - int next_slot = (slot + 1) % 4; - for (int i = 0; i < 4; i++) begin - request(300 + i); - expect_request(300 + i, 0); - end - response(7, 0); - response(3, 300 + slot); - response(7, 0, 65, 3, 1); - expect_request(300 + slot, 1); - response(3, 300 + next_slot); - expect_request(300 + next_slot, 1); - for (int i = 0; i < 4; i++) begin - response(4, 300 + i); - receive_response(completed); - end - end - // Later split completion/data acceptance may follow record release. - accept_data(3, 65); - repeat (3) @(negedge vif.clock); - if (vif.pending || requests.size() || responses.size() || original_responses.size()) - `uvm_fatal("DRAIN", "Saturated output queue did not drain") - // Acceptance tracking can retire before later data beats or split completion. - request(500, 4); - expect_request(500, 0); - accept_data(500, 9); - request(501, 'h1d); - expect_request(501, 0); - response(6, 501); - receive_response(completed); - for (int i = 0; i < 4; i++) begin - request(600 + i); - expect_request(600 + i, 0); - end - for (int i = 0; i < 4; i++) response(3, 600 + i); - for (int i = 0; i < 4; i++) begin - rsp_t idle = '0; - idle.opcode = 4; - idle.tgt = 1; - idle.txn = 600 + i; - @(negedge vif.clock); - vif.rsp_in = idle; - vif.accepted_txn = 600 + i; - @(negedge vif.clock); - if (vif.pending != 4 || requests.size() || responses.size()) - `uvm_fatal("INVALID_CHANNEL", "Inactive matching payload changed a retry-wait record") - end - response(4, 501); - receive_response(completed); - accept_data(500, 9); - if (vif.pending != 4) `uvm_fatal("LATE_ACCEPT", "Late acceptance retired an unrelated retry") - for (int i = 0; i < 4; i++) begin - response(7, 0); - expect_request(600 + i, 1); - response(4, 600 + i); - receive_response(completed); - end - // VALID held through coordinated reset is accepted once after release. - begin - req_t held = '0; - held.src = 1; - held.tgt = 64; - held.txn = 'ha55; - held.opcode = 'hd; - held.size = 6; - held.addr = 64; - held.allow_retry = 1; - held.mem_attr = 12; - held.snp_attr = 1; - @(negedge vif.clock); - vif.reset = 1; - vif.req_in = held; - vif.req_in_valid = 1; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - do @(posedge vif.clock); while (!vif.req_in_ready); - originals[held.txn] = held; - @(negedge vif.clock); - vif.req_in_valid = 0; - expect_request(held.txn, 0); - response(4, held.txn); - receive_response(completed); - repeat (3) @(negedge vif.clock); - if (vif.pending || requests.size() || responses.size() || original_responses.size()) - `uvm_fatal("RESET", "Held VALID did not complete exactly once") - end - // Invalid channels may carry arbitrary payload; no transaction may be consumed. - begin - req_t idle_req = '0; - rsp_t idle_rsp = '0; - idle_req.src = 1; - idle_req.opcode = 'hd; - idle_req.size = 6; - idle_req.mem_attr = 12; - idle_req.snp_attr = 1; - idle_req.allow_retry = 1; - idle_rsp.tgt = 1; - idle_rsp.opcode = 4; - for (int b = 0; b < $bits(req_t); b++) begin - @(negedge vif.clock); - vif.req_in = idle_req ^ (137'b1 << b); - @(negedge vif.clock); - vif.req_in = idle_req; - if (vif.pending || vif.req_out_valid || vif.rsp_out_valid) - `uvm_fatal("INVALID_CHANNEL", "Invalid REQ payload changed live transactions") - end - @(negedge vif.clock); - idle_req.snp_attr = 0; - idle_req.mem_attr = 2; - vif.req_in = idle_req; - @(negedge vif.clock); - if (vif.pending || vif.req_out_valid) - `uvm_fatal("INVALID_CHANNEL", "Inactive Device payload was consumed") - for (int b = 0; b < $bits(rsp_t); b++) begin - @(negedge vif.clock); - vif.rsp_in = idle_rsp ^ (71'b1 << b); - @(negedge vif.clock); - vif.rsp_in = idle_rsp; - if (vif.pending || vif.req_out_valid || vif.rsp_out_valid) - `uvm_fatal("INVALID_CHANNEL", "Invalid RSP payload changed live transactions") - end - end - repeat (2) @(negedge vif.clock); - `uvm_info("CHECKS", $sformatf("CHI retry: %0d request/reissue fields checked", checks), - UVM_LOW) -`endif - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_tb.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_tb.sv deleted file mode 100644 index 2ed3402ed..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/retry_tb.sv +++ /dev/null @@ -1,104 +0,0 @@ -`ifndef CHI_RETRY_TB -`define CHI_RETRY_TB retry_tb -`endif -module `CHI_RETRY_TB; - import uvm_pkg::*; - import chi_retry_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_retry_if source_if (clock); - RequestRetry dut ( - .clock(clock), - .reset(source_if.reset), - .io_reqIn_valid(source_if.req_in_valid), - .io_reqIn_ready(source_if.req_in_ready), - .io_reqIn_bits_qos(source_if.req_in[3:0]), - .io_reqIn_bits_tgtId(source_if.req_in[10:4]), - .io_reqIn_bits_srcId(source_if.req_in[17:11]), - .io_reqIn_bits_txnId(source_if.req_in[29:18]), - .io_reqIn_bits_returnNid(source_if.req_in[36:30]), - .io_reqIn_bits_stashNidValidEndian(source_if.req_in[37:37]), - .io_reqIn_bits_returnTxnId(source_if.req_in[49:38]), - .io_reqIn_bits_opcode(source_if.req_in[56:50]), - .io_reqIn_bits_multiReq(source_if.req_in[57:57]), - .io_reqIn_bits_size(source_if.req_in[63:58]), - .io_reqIn_bits_addr(source_if.req_in[107:64]), - .io_reqIn_bits_pas(source_if.req_in[110:108]), - .io_reqIn_bits_likelyShared(source_if.req_in[111:111]), - .io_reqIn_bits_allowRetry(source_if.req_in[112:112]), - .io_reqIn_bits_order(source_if.req_in[114:113]), - .io_reqIn_bits_pCrdType(source_if.req_in[118:115]), - .io_reqIn_bits_memAttr(source_if.req_in[122:119]), - .io_reqIn_bits_snpAttr(source_if.req_in[123:123]), - .io_reqIn_bits_lpid(source_if.req_in[131:124]), - .io_reqIn_bits_exclSnoopMe(source_if.req_in[132:132]), - .io_reqIn_bits_expCompAck(source_if.req_in[133:133]), - .io_reqIn_bits_tagOp(source_if.req_in[135:134]), - .io_reqIn_bits_traceTag(source_if.req_in[136:136]), - .io_reqOut_valid(source_if.req_out_valid), - .io_reqOut_ready(source_if.req_out_ready), - .io_reqOut_bits_qos(source_if.req_out[3:0]), - .io_reqOut_bits_tgtId(source_if.req_out[10:4]), - .io_reqOut_bits_srcId(source_if.req_out[17:11]), - .io_reqOut_bits_txnId(source_if.req_out[29:18]), - .io_reqOut_bits_returnNid(source_if.req_out[36:30]), - .io_reqOut_bits_stashNidValidEndian(source_if.req_out[37:37]), - .io_reqOut_bits_returnTxnId(source_if.req_out[49:38]), - .io_reqOut_bits_opcode(source_if.req_out[56:50]), - .io_reqOut_bits_multiReq(source_if.req_out[57:57]), - .io_reqOut_bits_size(source_if.req_out[63:58]), - .io_reqOut_bits_addr(source_if.req_out[107:64]), - .io_reqOut_bits_pas(source_if.req_out[110:108]), - .io_reqOut_bits_likelyShared(source_if.req_out[111:111]), - .io_reqOut_bits_allowRetry(source_if.req_out[112:112]), - .io_reqOut_bits_order(source_if.req_out[114:113]), - .io_reqOut_bits_pCrdType(source_if.req_out[118:115]), - .io_reqOut_bits_memAttr(source_if.req_out[122:119]), - .io_reqOut_bits_snpAttr(source_if.req_out[123:123]), - .io_reqOut_bits_lpid(source_if.req_out[131:124]), - .io_reqOut_bits_exclSnoopMe(source_if.req_out[132:132]), - .io_reqOut_bits_expCompAck(source_if.req_out[133:133]), - .io_reqOut_bits_tagOp(source_if.req_out[135:134]), - .io_reqOut_bits_traceTag(source_if.req_out[136:136]), - .io_rspIn_valid(source_if.rsp_in_valid), - .io_rspIn_ready(source_if.rsp_in_ready), - .io_rspIn_bits_qos(source_if.rsp_in[3:0]), - .io_rspIn_bits_tgtId(source_if.rsp_in[10:4]), - .io_rspIn_bits_srcId(source_if.rsp_in[17:11]), - .io_rspIn_bits_txnId(source_if.rsp_in[29:18]), - .io_rspIn_bits_opcode(source_if.rsp_in[34:30]), - .io_rspIn_bits_respErr(source_if.rsp_in[36:35]), - .io_rspIn_bits_resp(source_if.rsp_in[39:37]), - .io_rspIn_bits_fwdState(source_if.rsp_in[42:40]), - .io_rspIn_bits_cBusy(source_if.rsp_in[45:43]), - .io_rspIn_bits_dbid(source_if.rsp_in[57:46]), - .io_rspIn_bits_pCrdType(source_if.rsp_in[61:58]), - .io_rspIn_bits_tagOp(source_if.rsp_in[63:62]), - .io_rspIn_bits_traceTag(source_if.rsp_in[64:64]), - .io_rspIn_bits_cacheLineId(source_if.rsp_in[70:65]), - .io_rspOut_valid(source_if.rsp_out_valid), - .io_rspOut_ready(source_if.rsp_out_ready), - .io_rspOut_bits_qos(source_if.rsp_out[3:0]), - .io_rspOut_bits_tgtId(source_if.rsp_out[10:4]), - .io_rspOut_bits_srcId(source_if.rsp_out[17:11]), - .io_rspOut_bits_txnId(source_if.rsp_out[29:18]), - .io_rspOut_bits_opcode(source_if.rsp_out[34:30]), - .io_rspOut_bits_respErr(source_if.rsp_out[36:35]), - .io_rspOut_bits_resp(source_if.rsp_out[39:37]), - .io_rspOut_bits_fwdState(source_if.rsp_out[42:40]), - .io_rspOut_bits_cBusy(source_if.rsp_out[45:43]), - .io_rspOut_bits_dbid(source_if.rsp_out[57:46]), - .io_rspOut_bits_pCrdType(source_if.rsp_out[61:58]), - .io_rspOut_bits_tagOp(source_if.rsp_out[63:62]), - .io_rspOut_bits_traceTag(source_if.rsp_out[64:64]), - .io_rspOut_bits_cacheLineId(source_if.rsp_out[70:65]), - .io_acceptedData_valid(source_if.accepted_valid), - .io_acceptedData_bits_srcId(source_if.accepted_src), - .io_acceptedData_bits_txnId(source_if.accepted_txn), - .io_pending(source_if.pending) - ); - initial begin - uvm_config_db#(virtual chi_retry_if)::set(null, "uvm_test_top", "vif", source_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf.f deleted file mode 100644 index c6baa2651..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/rnf_interface.sv -@RESOURCES@/rnf_package.sv --F @RTL@/BankedChiCache/filelist.f -@RESOURCES@/rnf_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid.f deleted file mode 100644 index ec1749034..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+RNF_TB=rnf_bad_dbid_tb -+define+RNF_BAD_DBID -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/rnf_interface.sv -@RESOURCES@/rnf_package.sv --F @RTL@/BankedChiCache/filelist.f -@RESOURCES@/rnf_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid.toml deleted file mode 100644 index f84f14e9d..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Inconsistent multi-beat fill metadata" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid_width.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid_width.f deleted file mode 100644 index 72df73136..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid_width.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+RNF_TB=rnf_bad_dbid_width_tb -+define+RNF_BAD_DBID_WIDTH -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/rnf_interface.sv -@RESOURCES@/rnf_package.sv --F @RTL@/BankedChiCache/filelist.f -@RESOURCES@/rnf_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid_width.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid_width.toml deleted file mode 100644 index d9d50be48..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_dbid_width.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Fill DBID exceeds completion ID width" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_resp.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_resp.f deleted file mode 100644 index 2719bf861..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_resp.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+RNF_TB=rnf_bad_resp_tb -+define+RNF_BAD_RESP -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/rnf_interface.sv -@RESOURCES@/rnf_package.sv --F @RTL@/BankedChiCache/filelist.f -@RESOURCES@/rnf_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_resp.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_resp.toml deleted file mode 100644 index f84f14e9d..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_bad_resp.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Inconsistent multi-beat fill metadata" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_interface.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_interface.sv deleted file mode 100644 index 0870c06d6..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_interface.sv +++ /dev/null @@ -1,58 +0,0 @@ -`ifndef RNF_TB -`define RNF_TB rnf_tb -`endif -interface chi_rnf_if ( - input logic clock -); - logic reset; - logic access_valid, access_ready, access_write; - logic [43:0] access_addr; - logic [63:0] access_data; - logic [7:0] access_mask; - logic [3:0] access_atomic; - logic access_atomic_word; - logic result_valid, result_ready, result_error; - logic [63:0] result_data; - logic req_valid, req_ready, rsp_valid, rsp_ready, dat_valid, dat_ready; - logic rx_rsp_valid, rx_rsp_ready, rx_dat_valid, rx_dat_ready, snp_valid, snp_ready; - logic [136:0] req; - logic [70:0] rsp, rx_rsp; - logic [388:0] dat, rx_dat; - logic [93:0] snp; - logic [31:0] outstanding, hits, misses; - // Statistics only: compress a prefix of repeated legal hits/misses. - // Cache contents, coherence state, transactions, and credits are never seeded. - task seed_counter(bit bank, bit miss, int unsigned value); - if (bank) begin - if (miss) $deposit($root.`RNF_TB.dut.caches_1.misses, value); - else $deposit($root.`RNF_TB.dut.caches_1.hits, value); - end else begin - if (miss) $deposit($root.`RNF_TB.dut.caches_0.misses, value); - else $deposit($root.`RNF_TB.dut.caches_0.hits, value); - end - endtask - function int unsigned counter_value(bit bank, bit miss); - if (bank) return miss ? $root.`RNF_TB.dut.caches_1.misses : $root.`RNF_TB.dut.caches_1.hits; - return miss ? $root.`RNF_TB.dut.caches_0.misses : $root.`RNF_TB.dut.caches_0.hits; - endfunction - assert property (@(posedge clock) disable iff (reset) - access_valid && !access_ready |=> access_valid && $stable( - {access_addr, access_write, access_data, access_mask, access_atomic, access_atomic_word} - )); - assert property (@(posedge clock) disable iff (reset) - req_valid && !req_ready |=> req_valid && $stable( - req - )); - assert property (@(posedge clock) disable iff (reset) - rsp_valid && !rsp_ready |=> rsp_valid && $stable( - rsp - )); - assert property (@(posedge clock) disable iff (reset) - dat_valid && !dat_ready |=> dat_valid && $stable( - dat - )); - assert property (@(posedge clock) disable iff (reset) - result_valid && !result_ready |=> result_valid && $stable( - {result_error, result_data} - )); -endinterface diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_package.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_package.sv deleted file mode 100644 index 60d2d9fb8..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_package.sv +++ /dev/null @@ -1,110 +0,0 @@ -package chi_rnf_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - // Issue H, NodeID=7, address=44, data=256, all optional buses absent. - import "DPI-C" function longint unsigned rnf_ref_access( - input chandle model, - input longint unsigned addr, - input byte unsigned write, - input longint unsigned operand, - input int unsigned atomic, - input byte unsigned word, - input byte unsigned mask, - input byte unsigned sc_success, - input byte unsigned failed - ); - typedef struct packed { - bit trace_tag; - bit [1:0] tag_op; - bit exp_comp_ack, excl_snoop_me; - bit [7:0] lpid; - bit snp_attr; - bit [3:0] mem_attr, pcrd_type; - bit [1:0] order; - bit allow_retry, likely_shared; - bit [2:0] pas; - bit [43:0] addr; - bit [5:0] size; - bit multi_req; - bit [6:0] opcode; - bit [11:0] return_txn; - bit stash_valid; - bit [6:0] return_nid; - bit [11:0] txn; - bit [6:0] src, tgt; - bit [3:0] qos; - } req_t; - typedef struct packed { - bit [5:0] line_id; - bit trace_tag; - bit [1:0] tag_op; - bit [3:0] pcrd_type; - bit [11:0] dbid; - bit [2:0] busy, fwd_state, resp; - bit [1:0] error; - bit [4:0] opcode; - bit [11:0] txn; - bit [6:0] src, tgt; - bit [3:0] qos; - } rsp_t; - typedef struct packed { - bit [255:0] data; - bit [31:0] be; - bit replicate; - bit [1:0] num_dat; - bit copy_at_home, trace_tag; - bit [1:0] tag_update; - bit [7:0] tag; - bit [1:0] tag_op; - bit [5:0] line_id; - bit [1:0] data_id, ccid; - bit [15:0] dbid; - bit [2:0] busy; - bit data_pull; - bit [7:0] data_source; - bit [2:0] resp; - bit [1:0] error; - bit [3:0] opcode; - bit [6:0] home; - bit [11:0] txn; - bit [6:0] src, tgt; - bit [3:0] qos; - } dat_t; - import "DPI-C" function chandle rnf_ref_create(input int unsigned address_bits); - import "DPI-C" function void rnf_ref_destroy(input chandle model); - import "DPI-C" function byte unsigned rnf_ref_write( - input chandle model, - input longint unsigned addr, - input bit [511:0] data, - input bit [63:0] mask - ); - import "DPI-C" function byte unsigned rnf_ref_read( - input chandle model, - input longint unsigned addr, - output bit [511:0] data - ); - import "DPI-C" function longint unsigned rnf_ref_access( - input chandle model, - input longint unsigned addr, - input byte unsigned write, - input longint unsigned operand, - input int unsigned atomic, - input byte unsigned word, - input byte unsigned mask, - input byte unsigned sc_success, - input byte unsigned failed - ); - typedef struct packed { - bit trace_tag, ret_to_src, do_not_go_sd; - bit [2:0] pas; - bit [40:0] addr; - bit [4:0] opcode; - bit [11:0] fwd_txn; - bit [6:0] fwd_nid; - bit [11:0] txn; - bit [6:0] src; - bit [3:0] qos; - } snp_t; - `include "rnf_test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_tb.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_tb.sv deleted file mode 100644 index 11a5ae6f2..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_tb.sv +++ /dev/null @@ -1,155 +0,0 @@ -`ifndef RNF_TB -`define RNF_TB rnf_tb -`endif -module `RNF_TB; - import uvm_pkg::*; - import chi_rnf_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_rnf_if vif (clock); - BankedChiCache dut ( - .clock(clock), - .reset(vif.reset), - .io_chi_req_bits_qos(vif.req[0+:4]), - .io_chi_req_bits_tgtId(vif.req[4+:7]), - .io_chi_req_bits_srcId(vif.req[11+:7]), - .io_chi_req_bits_txnId(vif.req[18+:12]), - .io_chi_req_bits_returnNid(vif.req[30+:7]), - .io_chi_req_bits_stashNidValidEndian(vif.req[37+:1]), - .io_chi_req_bits_returnTxnId(vif.req[38+:12]), - .io_chi_req_bits_opcode(vif.req[50+:7]), - .io_chi_req_bits_multiReq(vif.req[57+:1]), - .io_chi_req_bits_size(vif.req[58+:6]), - .io_chi_req_bits_addr(vif.req[64+:44]), - .io_chi_req_bits_pas(vif.req[108+:3]), - .io_chi_req_bits_likelyShared(vif.req[111+:1]), - .io_chi_req_bits_allowRetry(vif.req[112+:1]), - .io_chi_req_bits_order(vif.req[113+:2]), - .io_chi_req_bits_pCrdType(vif.req[115+:4]), - .io_chi_req_bits_memAttr(vif.req[119+:4]), - .io_chi_req_bits_snpAttr(vif.req[123+:1]), - .io_chi_req_bits_lpid(vif.req[124+:8]), - .io_chi_req_bits_exclSnoopMe(vif.req[132+:1]), - .io_chi_req_bits_expCompAck(vif.req[133+:1]), - .io_chi_req_bits_tagOp(vif.req[134+:2]), - .io_chi_req_bits_traceTag(vif.req[136+:1]), - .io_chi_req_valid(vif.req_valid), - .io_chi_req_ready(vif.req_ready), - .io_chi_txRsp_bits_qos(vif.rsp[0+:4]), - .io_chi_txRsp_bits_tgtId(vif.rsp[4+:7]), - .io_chi_txRsp_bits_srcId(vif.rsp[11+:7]), - .io_chi_txRsp_bits_txnId(vif.rsp[18+:12]), - .io_chi_txRsp_bits_opcode(vif.rsp[30+:5]), - .io_chi_txRsp_bits_respErr(vif.rsp[35+:2]), - .io_chi_txRsp_bits_resp(vif.rsp[37+:3]), - .io_chi_txRsp_bits_fwdState(vif.rsp[40+:3]), - .io_chi_txRsp_bits_cBusy(vif.rsp[43+:3]), - .io_chi_txRsp_bits_dbid(vif.rsp[46+:12]), - .io_chi_txRsp_bits_pCrdType(vif.rsp[58+:4]), - .io_chi_txRsp_bits_tagOp(vif.rsp[62+:2]), - .io_chi_txRsp_bits_traceTag(vif.rsp[64+:1]), - .io_chi_txRsp_bits_cacheLineId(vif.rsp[65+:6]), - .io_chi_txRsp_valid(vif.rsp_valid), - .io_chi_txRsp_ready(vif.rsp_ready), - .io_chi_txDat_bits_qos(vif.dat[0+:4]), - .io_chi_txDat_bits_tgtId(vif.dat[4+:7]), - .io_chi_txDat_bits_srcId(vif.dat[11+:7]), - .io_chi_txDat_bits_txnId(vif.dat[18+:12]), - .io_chi_txDat_bits_homeNid(vif.dat[30+:7]), - .io_chi_txDat_bits_opcode(vif.dat[37+:4]), - .io_chi_txDat_bits_respErr(vif.dat[41+:2]), - .io_chi_txDat_bits_resp(vif.dat[43+:3]), - .io_chi_txDat_bits_dataSource(vif.dat[46+:8]), - .io_chi_txDat_bits_dataPull(vif.dat[54+:1]), - .io_chi_txDat_bits_cBusy(vif.dat[55+:3]), - .io_chi_txDat_bits_dbid(vif.dat[58+:16]), - .io_chi_txDat_bits_ccid(vif.dat[74+:2]), - .io_chi_txDat_bits_dataId(vif.dat[76+:2]), - .io_chi_txDat_bits_cacheLineId(vif.dat[78+:6]), - .io_chi_txDat_bits_tagOp(vif.dat[84+:2]), - .io_chi_txDat_bits_tag(vif.dat[86+:8]), - .io_chi_txDat_bits_tagUpdate(vif.dat[94+:2]), - .io_chi_txDat_bits_traceTag(vif.dat[96+:1]), - .io_chi_txDat_bits_copyAtHome(vif.dat[97+:1]), - .io_chi_txDat_bits_numDat(vif.dat[98+:2]), - .io_chi_txDat_bits_replicate(vif.dat[100+:1]), - .io_chi_txDat_bits_be(vif.dat[101+:32]), - .io_chi_txDat_bits_data(vif.dat[133+:256]), - .io_chi_txDat_valid(vif.dat_valid), - .io_chi_txDat_ready(vif.dat_ready), - .io_chi_rxRsp_bits_qos(vif.rx_rsp[0+:4]), - .io_chi_rxRsp_bits_tgtId(vif.rx_rsp[4+:7]), - .io_chi_rxRsp_bits_srcId(vif.rx_rsp[11+:7]), - .io_chi_rxRsp_bits_txnId(vif.rx_rsp[18+:12]), - .io_chi_rxRsp_bits_opcode(vif.rx_rsp[30+:5]), - .io_chi_rxRsp_bits_respErr(vif.rx_rsp[35+:2]), - .io_chi_rxRsp_bits_resp(vif.rx_rsp[37+:3]), - .io_chi_rxRsp_bits_fwdState(vif.rx_rsp[40+:3]), - .io_chi_rxRsp_bits_cBusy(vif.rx_rsp[43+:3]), - .io_chi_rxRsp_bits_dbid(vif.rx_rsp[46+:12]), - .io_chi_rxRsp_bits_pCrdType(vif.rx_rsp[58+:4]), - .io_chi_rxRsp_bits_tagOp(vif.rx_rsp[62+:2]), - .io_chi_rxRsp_bits_traceTag(vif.rx_rsp[64+:1]), - .io_chi_rxRsp_bits_cacheLineId(vif.rx_rsp[65+:6]), - .io_chi_rxRsp_valid(vif.rx_rsp_valid), - .io_chi_rxRsp_ready(vif.rx_rsp_ready), - .io_chi_rxDat_bits_qos(vif.rx_dat[0+:4]), - .io_chi_rxDat_bits_tgtId(vif.rx_dat[4+:7]), - .io_chi_rxDat_bits_srcId(vif.rx_dat[11+:7]), - .io_chi_rxDat_bits_txnId(vif.rx_dat[18+:12]), - .io_chi_rxDat_bits_homeNid(vif.rx_dat[30+:7]), - .io_chi_rxDat_bits_opcode(vif.rx_dat[37+:4]), - .io_chi_rxDat_bits_respErr(vif.rx_dat[41+:2]), - .io_chi_rxDat_bits_resp(vif.rx_dat[43+:3]), - .io_chi_rxDat_bits_dataSource(vif.rx_dat[46+:8]), - .io_chi_rxDat_bits_dataPull(vif.rx_dat[54+:1]), - .io_chi_rxDat_bits_cBusy(vif.rx_dat[55+:3]), - .io_chi_rxDat_bits_dbid(vif.rx_dat[58+:16]), - .io_chi_rxDat_bits_ccid(vif.rx_dat[74+:2]), - .io_chi_rxDat_bits_dataId(vif.rx_dat[76+:2]), - .io_chi_rxDat_bits_cacheLineId(vif.rx_dat[78+:6]), - .io_chi_rxDat_bits_tagOp(vif.rx_dat[84+:2]), - .io_chi_rxDat_bits_tag(vif.rx_dat[86+:8]), - .io_chi_rxDat_bits_tagUpdate(vif.rx_dat[94+:2]), - .io_chi_rxDat_bits_traceTag(vif.rx_dat[96+:1]), - .io_chi_rxDat_bits_copyAtHome(vif.rx_dat[97+:1]), - .io_chi_rxDat_bits_numDat(vif.rx_dat[98+:2]), - .io_chi_rxDat_bits_replicate(vif.rx_dat[100+:1]), - .io_chi_rxDat_bits_be(vif.rx_dat[101+:32]), - .io_chi_rxDat_bits_data(vif.rx_dat[133+:256]), - .io_chi_rxDat_valid(vif.rx_dat_valid), - .io_chi_rxDat_ready(vif.rx_dat_ready), - .io_chi_snp_bits_qos(vif.snp[0+:4]), - .io_chi_snp_bits_srcId(vif.snp[4+:7]), - .io_chi_snp_bits_txnId(vif.snp[11+:12]), - .io_chi_snp_bits_fwdNid(vif.snp[23+:7]), - .io_chi_snp_bits_fwdTxnId(vif.snp[30+:12]), - .io_chi_snp_bits_opcode(vif.snp[42+:5]), - .io_chi_snp_bits_addr(vif.snp[47+:41]), - .io_chi_snp_bits_pas(vif.snp[88+:3]), - .io_chi_snp_bits_doNotGoToSd(vif.snp[91+:1]), - .io_chi_snp_bits_retToSrc(vif.snp[92+:1]), - .io_chi_snp_bits_traceTag(vif.snp[93+:1]), - .io_chi_snp_valid(vif.snp_valid), - .io_chi_snp_ready(vif.snp_ready), - .io_access_valid(vif.access_valid), - .io_access_ready(vif.access_ready), - .io_access_bits_write(vif.access_write), - .io_access_bits_addr(vif.access_addr), - .io_access_bits_data(vif.access_data), - .io_access_bits_mask(vif.access_mask), - .io_access_bits_atomic(vif.access_atomic), - .io_access_bits_atomicWord(vif.access_atomic_word), - .io_result_valid(vif.result_valid), - .io_result_ready(vif.result_ready), - .io_result_bits_error(vif.result_error), - .io_result_bits_data(vif.result_data), - .io_outstanding(vif.outstanding), - .io_hits(vif.hits), - .io_misses(vif.misses) - ); - initial begin - uvm_config_db#(virtual chi_rnf_if)::set(null, "uvm_test_top", "vif", vif); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_test.svh b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_test.svh deleted file mode 100644 index 614383c3b..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rnf_test.svh +++ /dev/null @@ -1,788 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual chi_rnf_if vif; - chandle model; - bit [511:0] memory[longint unsigned]; - longint unsigned wb_address[int unsigned], snoop_address; - bit [511:0] snoop_data; - bit [1:0] snoop_seen; - bit [511:0] wb_data[int unsigned]; - bit [1:0] wb_seen[int unsigned]; - bit ack_pending[int unsigned]; - int home_allocations = 0, snoop_allocations = 0; - int home_ids[$]; - int snoop_id; - int fill_packets = 0, partial_goal = 0; - bit hold_dat = 0, forbid_early_snoop = 0, expect_snoop_data = 0; - bit [63:0] expected[$]; - bit expected_errors[$]; - bit [1:0] fail_read = 0; - bit mixed_data_error = 0; - rsp_t rx_rsp_queue[$]; - dat_t rx_dat_queue[$]; - int cycle = 0, submitted = 0, checked = 0, writebacks = 0, snoops = 0, peak = 0; - bit hold_fill = 0, reverse_transactions = 0, hold_req = 0, hold_rsp = 0, hold_output_dat = 0; - int cancelled_writebacks = 0, shared_writebacks = 0, cancelled_cpu = 0; - bit rsp_active = 0, dat_active = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - timeout = 450us; - if (!uvm_config_db#(virtual chi_rnf_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "RN-F interface missing") - model = rnf_ref_create(44); - home_ids.push_back(0); - home_ids.push_back(4095); - // Odd multiplication and Gray coding form a permutation of all 12-bit IDs. - // This avoids correlating a cache bank or response kind with fixed ID bits. - for (int i = 0; i < 4096; i++) begin - int encoded = (i * 4051) & 4095; - int id = encoded ^ (encoded >> 1); - if (id != 0 && id != 4095) home_ids.push_back(id); - end - endfunction - function int allocate_home_id(); - int id; - if (home_allocations >= 4096) `uvm_fatal("HOME_ID", "Test exhausted the 12-bit Home ID pool") - home_allocations++; - return home_ids.pop_front(); - endfunction - function void consume_req(req_t r); - rsp_t response = '0; - dat_t d = '0; - if (r.src != 1 || r.tgt != 64 || r.txn >= 2 || r.size != 6 || r.addr[5:0] != 0 || !r.allow_retry) - `uvm_fatal("REQ", "RN-F request outside declared no-retry Home contract") - response.src = 64; - response.tgt = 1; - response.txn = r.txn; - if (r.opcode == 'h26 || r.opcode == 'h07) begin - if (!r.exp_comp_ack || !memory.exists(r.addr)) - `uvm_fatal("READ", "Uninitialized read or missing CompAck") - d.src = 64; - d.tgt = 1; - d.home = 64; - d.txn = r.txn; - d.dbid = allocate_home_id(); - ack_pending[d.dbid] = 1; -`ifdef RNF_BAD_DBID_WIDTH - d.dbid = 'h1000; -`endif - // B9.1.3 permits any Resp for NDERR; both packets must retain it. - d.opcode = 4; - d.error = fail_read; - d.resp = fail_read == 3 ? 7 : r.opcode == 'h07 ? 2 : 1; - d.be = '1; - // Return the upper 256b first; DataID units are 128b. - d.data_id = 2; - d.data = fail_read ? 0 : memory[r.addr][511:256]; - rx_dat_queue.push_back(d); - d.data_id = 0; - d.data = fail_read ? 0 : memory[r.addr][255:0]; - if (mixed_data_error) d.error = 0; // first packet error must survive a clean final packet -`ifdef RNF_BAD_DBID - d.dbid = d.dbid ^ 16'h0001; -`endif -`ifdef RNF_BAD_RESP - d.resp = 2; -`endif - rx_dat_queue.push_back(d); - end else if (r.opcode == 'h1b) begin - response.dbid = allocate_home_id(); - response.opcode = 5; - rx_rsp_queue.push_back(response); - wb_address[response.dbid] = r.addr; - wb_seen[response.dbid] = 0; - end else if (r.opcode == 'h0d) begin - response.opcode = 4; - rx_rsp_queue.push_back(response); - end else `uvm_fatal("REQ", "Unexpected request opcode") - endfunction - function void consume_data(dat_t d); - bit [511:0] golden; - int index; - if (d.tgt != 64 || d.src != 1 || !(d.data_id == 0 || d.data_id == 2)) - `uvm_fatal("DAT", "Unexpected RN-F data endpoint/DataID") - index = d.data_id / 2; - if (d.opcode == 2) begin - if (d.resp == 0 && (d.be != 0 || d.data != 0)) - `uvm_fatal("WB_ZERO_DATA", "CopyBack_I must carry zero data and zero byte enables") - if (!wb_address.exists(d.txn) || wb_seen[d.txn][index]) - `uvm_fatal("WB", "Unknown/duplicate writeback") - wb_seen[d.txn][index] = 1; - wb_data[d.txn][index*256+:256] = d.data; - if (!(&d.be) && d.resp != 0) `uvm_fatal("WB", "Valid writeback has empty BE") - if (&wb_seen[d.txn]) begin - if (d.resp == 0) begin - if (d.be != 0) `uvm_fatal("WB", "Cancelled writeback carries valid bytes") - cancelled_writebacks++; - end else begin - if (rnf_ref_read(model, wb_address[d.txn], golden) || wb_data[d.txn] !== golden) - `uvm_fatal("WB", "Dirty writeback differs from independent model") - memory[wb_address[d.txn]] = wb_data[d.txn]; - if (d.resp == 1) shared_writebacks++; - end - writebacks++; - wb_address.delete(d.txn); - wb_data.delete(d.txn); - wb_seen.delete(d.txn); - end - end else if (d.opcode == 1) begin - if (forbid_early_snoop && fill_packets < partial_goal) - `uvm_fatal("SNOOP_EARLY", "Same-line snoop responded before the final fill packet") - if (d.txn != snoop_id || !(&d.be)) `uvm_fatal("SNOOP", "Unexpected snoop data") - if (snoop_seen[index]) `uvm_fatal("SNOOP", "Duplicate snoop DataID") - snoop_data[index*256+:256] = d.data; - snoop_seen[index] = 1; - if (&snoop_seen) begin - if (rnf_ref_read(model, snoop_address, golden) || snoop_data !== golden) - `uvm_fatal("SNOOP", "Snoop differs from golden CPU data") - memory[snoop_address] = snoop_data; - snoops++; - end - end else `uvm_fatal("DAT", "Unexpected RN-F DAT opcode") - endfunction - task monitor(); - forever begin - @(posedge vif.clock); - if (!vif.reset) begin - cycle++; - if (vif.outstanding > peak) peak = vif.outstanding; - if (vif.rx_dat_valid && vif.rx_dat_ready) fill_packets++; - if (vif.req_valid && vif.req_ready) consume_req(req_t'(vif.req)); - if (vif.dat_valid && vif.dat_ready) consume_data(dat_t'(vif.dat)); - if (vif.rsp_valid && vif.rsp_ready) begin - rsp_t r = rsp_t'(vif.rsp); - if (r.opcode != 2 && r.opcode != 1) `uvm_fatal("RSP", "Unexpected RN-F RSP opcode") - if (r.src != 1 || r.tgt != 64) `uvm_fatal("RSP", "RN-F response endpoint mismatch") - if (r.opcode == 2) begin - if (!ack_pending.exists(r.txn)) - `uvm_fatal("ACK", "CompAck did not retain the independent Home DBID") - ack_pending.delete(r.txn); - end else begin - if (forbid_early_snoop && fill_packets < partial_goal) - `uvm_fatal("SNOOP_EARLY", "Same-line snoop responded before the final fill packet") - if (expect_snoop_data) - `uvm_fatal("SNOOP_DATA", "Cached RetToSrc snoop did not return data") - if (r.txn != snoop_id) - `uvm_fatal("SNOOP", "Dataless snoop response has wrong transaction") - snoops++; - end - end - if (vif.result_valid && vif.result_ready) begin - bit [63:0] golden; - bit error; - if (!expected.size()) `uvm_fatal("RESULT", "Unexpected completion") - golden = expected.pop_front(); - error = expected_errors.pop_front(); - if (vif.result_error != error || vif.result_data !== golden) - `uvm_fatal("RESULT", $sformatf( - "completion %0d got=%h expected=%h", checked, vif.result_data, golden)) - checked++; - end - if (rsp_active && vif.rx_rsp_ready) rsp_active = 0; - if (dat_active && vif.rx_dat_ready) dat_active = 0; - end - @(negedge vif.clock); - vif.req_ready = !vif.reset && !hold_req && cycle % 7 >= 2; - vif.rsp_ready = !vif.reset && !hold_rsp && cycle % 11 >= 5; - vif.dat_ready = !vif.reset && !hold_output_dat && cycle % 13 >= 6; - vif.result_ready = !vif.reset && cycle % 9 >= 3; - if (!rsp_active && rx_rsp_queue.size()) begin - vif.rx_rsp = rx_rsp_queue.pop_front(); - rsp_active = 1; - end - if (!dat_active && !hold_fill && !hold_dat && rx_dat_queue.size()) begin - if (reverse_transactions) vif.rx_dat = rx_dat_queue.pop_back(); - else vif.rx_dat = rx_dat_queue.pop_front(); - dat_active = 1; - end - vif.rx_rsp_valid = rsp_active && !vif.reset; - vif.rx_dat_valid = dat_active && !vif.reset; - end - endtask - task issue(longint unsigned address, bit write = 0, bit [63:0] value = 0, int atomic = 0, - bit [7:0] mask = '1, bit sc_success = 0, bit expect_error = 0, bit atomic_word = 0); - bit [63:0] answer; - @(negedge vif.clock); - vif.access_addr = address; - vif.access_write = write; - vif.access_data = value; - vif.access_atomic = atomic; - vif.access_atomic_word = atomic_word; - vif.access_mask = mask; - vif.access_valid = 1; - do @(posedge vif.clock); while (!vif.access_ready); - answer = rnf_ref_access(model, address, write, value, atomic, atomic_word, mask, sc_success, - expect_error); - expected.push_back(answer); - expected_errors.push_back(expect_error); - submitted++; - @(negedge vif.clock); - vif.access_valid = 0; - endtask - task settle(); - wait (checked + cancelled_cpu == submitted); - while (ack_pending.num() != 0 || wb_address.num() != 0) @(negedge vif.clock); - repeat (4) @(negedge vif.clock); - endtask - task access (longint unsigned address, bit write = 0, bit [63:0] value = 0, int atomic = 0, - bit [7:0] mask = '1, bit sc_success = 0, bit expect_error = 0, bit atomic_word = 0); - issue(address, write, value, atomic, mask, sc_success, expect_error, atomic_word); - settle(); - endtask - task snoop_line(longint unsigned address, int opcode, bit ret_source = 0); - snp_t s = '0; - int before_count = snoops; - expect_snoop_data = ret_source; - snoop_id = snoop_allocations % 2 ? snoop_allocations / 2 : 4095 - snoop_allocations / 2; - snoop_allocations++; - s.src = 64; - s.txn = snoop_id; - s.addr = address >> 3; - s.opcode = opcode; - s.do_not_go_sd = 1; - s.ret_to_src = ret_source; - snoop_address = address; - snoop_seen = 0; - @(negedge vif.clock); - vif.snp = s; - vif.snp_valid = 1; - do @(posedge vif.clock); while (!vif.snp_ready); - @(negedge vif.clock); - vif.snp_valid = 0; - wait (snoops > before_count); - repeat (4) @(negedge vif.clock); - endtask - task partial_fill_snoop(longint unsigned target, longint unsigned probe, bit reverse); - int before_packets = fill_packets; - int before_snoops = snoops; - reverse_transactions = reverse; - issue(target); - wait (fill_packets == before_packets + 1); - hold_dat = 1; - partial_goal = before_packets + 2; - forbid_early_snoop = target == probe; - fork - snoop_line(probe, 4, 1); - begin - repeat (20) @(negedge vif.clock); - if (target != probe && snoops == before_snoops) - `uvm_fatal("SNOOP_INDEPENDENT", - "Different-line snoop waited for an unrelated final fill packet") - hold_dat = 0; - end - join - forbid_early_snoop = 0; - reverse_transactions = 0; - settle(); - endtask - task reset_pending_reads(); - bit [511:0] line; - hold_fill = 1; - issue(1024); - issue(1088); - wait (ack_pending.num() == 2); - if (vif.outstanding != 2 || expected.size() != 2) - `uvm_fatal("RESET_PENDING", "Reset did not start with two pending reads") - @(negedge vif.clock); - vif.reset = 1; - // Coordinated reset cancels unretired CPU work and both endpoints' CHI work. - cancelled_cpu += expected.size(); - expected.delete(); - expected_errors.delete(); - ack_pending.delete(); - rx_dat_queue.delete(); - rx_rsp_queue.delete(); - rsp_active = 0; - dat_active = 0; - vif.access_addr = 2048; - vif.access_write = 0; - vif.access_data = 0; - vif.access_atomic = 0; - vif.access_atomic_word = 0; - vif.access_mask = '1; - vif.access_valid = 1; - repeat (4) @(negedge vif.clock); - if (vif.outstanding != 0) `uvm_fatal("RESET_PENDING", "Reset retained CPU retirement entries") - // VALID may remain asserted through reset. Transfers count only outside reset; - // the held command must produce exactly one post-reset completion. - vif.reset = 0; - hold_fill = 0; - do @(posedge vif.clock); while (!vif.access_ready); - if (rnf_ref_read(model, 2048, line)) `uvm_fatal("MODEL", "Reset probe is uninitialized") - expected.push_back(line[63:0]); - expected_errors.push_back(0); - submitted++; - @(negedge vif.clock); - vif.access_valid = 0; - settle(); - endtask - task idle_metadata(); - // Invalid payload is unconstrained; changing it must not create a transaction. - for (int bank = 0; bank < 2; bank++) begin - for (int field = 0; field < 6; field++) begin - rsp_t r = '0; - dat_t d = '0; - snp_t probe = '0; - r.src = 64; - r.tgt = 1; - r.txn = bank; - d.opcode = 4; - d.src = 64; - d.tgt = 1; - d.txn = bank; - d.home = 64; - probe.src = 64; - probe.opcode = 4; - case (field) - 0: begin - r.tgt = 0; - d.opcode = 0; - probe.src = 0; - end - 1: begin - r.src = 0; - d.src = 0; - probe.pas = 1; - end - 2: begin - r.txn = bank ^ 1; - d.tgt = 0; - probe.fwd_nid = 1; - end - 3: begin - r.error = 3; - d.home = 0; - probe.fwd_txn = 1; - end - 4: begin - r.txn = 4095; - d.txn = 4095; - probe.opcode = 0; - end - 5: begin - d.data_id = 3; - d.dbid = 16'hf000; - end - endcase - @(negedge vif.clock); - vif.rx_rsp = r; - vif.rx_dat = d; - vif.snp = probe; - vif.access_addr = bank * 64 + 1; - vif.access_atomic = 15; - vif.access_atomic_word = 0; - vif.access_write = 1; - vif.access_mask = 0; - repeat (2) @(negedge vif.clock); - end - end - for (int bit_index = 0; bit_index < $bits(dat_t); bit_index++) begin - @(negedge vif.clock); - vif.rx_rsp = rsp_t'(rsp_t'(1) << bit_index); - vif.rx_dat = dat_t'(dat_t'(1) << bit_index); - vif.snp = snp_t'(snp_t'(1) << bit_index); - vif.access_addr = 44'(1) << bit_index; - vif.access_data = 64'(1) << bit_index; - vif.access_mask = 8'(1) << bit_index; - vif.access_atomic = 4'(1) << bit_index; - vif.access_atomic_word = bit_index % 2; - vif.access_write = bit_index % 2; - repeat (2) @(negedge vif.clock); - if (vif.req_valid || vif.rsp_valid || vif.dat_valid || vif.result_valid || vif.outstanding) - `uvm_fatal("IDLE_PAYLOAD", "Invalid walking-bit payload created CPU or CHI work") - end - if (vif.req_valid || vif.rsp_valid || vif.dat_valid || vif.result_valid || vif.outstanding) - `uvm_fatal("IDLE_PAYLOAD", "Invalid payload created CPU or CHI work") - vif.rx_rsp = '0; - vif.rx_dat = '0; - vif.snp = '0; - vif.access_addr = 0; - vif.access_atomic = 0; - vif.access_atomic_word = 0; - vif.access_write = 0; - vif.access_mask = '1; - endtask - task execute(); - vif.reset = 1; - vif.access_valid = 0; - vif.access_atomic_word = 0; - vif.snp_valid = 0; - vif.rx_rsp_valid = 0; - vif.rx_dat_valid = 0; - vif.req_ready = 0; - vif.rsp_ready = 0; - vif.dat_ready = 0; - vif.result_ready = 0; - for (int i = 0; i < 256; i++) begin - bit [511:0] initial_line; - for (int j = 0; j < 8; j++) - initial_line[j*64 +: 64] = (i%2 ? 64'h0f0f0f0f0f0f0f0f : 64'hf0f0f0f0f0f0f0f0) ^ 64'(i*8+j); - memory[i*64] = initial_line; - if (rnf_ref_write(model, i * 64, initial_line, '1)) - `uvm_fatal("MODEL", "Initialization failed") - end - repeat (4) @(negedge vif.clock); - vif.reset = 0; - idle_metadata(); - fork - monitor(); - join_none - hold_fill = 1; - issue(0); - issue(64); - repeat (10) @(negedge vif.clock); - if (vif.outstanding != 2) `uvm_fatal("PARALLEL", "Both cache banks did not remain in flight") - reverse_transactions = 1; - hold_fill = 0; - settle(); - reverse_transactions = 0; - for (int line_index = 2; line_index < 8; line_index++) begin - access (line_index * 64); - end - reset_pending_reads(); - for (int word = 1; word < 8; word++) begin - access (word * 8); - access (word * 8, 1, 64'h55aa55aa55aa55aa ^ 64'(word), 0, 8'hff >> word); - end - access (0, 1, 'h1122334455667788, 0, 'h0f); - snoop_line(0, 4); // downgrade dirty to shared - access (0); - access (0, 1, 'h8877665544332211); - access (512); // conflicting dirty victim - access (0); - for (int bank = 0; bank < 2; bank++) begin - for (int op = 1; op <= 9; op++) begin - // Exercise the independent hit and fill AMO datapaths with both comparisons. - access (bank * 64, 1, 0); - access (bank * 64, 0, 3, op); - access (bank * 64, 0, 64'hfffffffffffffffe, op); - snoop_line(bank * 64, 4); - access (bank * 64, 0, 3, op); - snoop_line(bank * 64, 4); - access (bank * 64, 0, 64'hfffffffffffffffe, op); - if (op >= 6) begin - for (int refill = 0; refill < 2; refill++) begin - access (bank * 64, 1, '1); - if (refill) snoop_line(bank * 64, 4); - access (bank * 64, 0, 0, op); - access (bank * 64, 1, 0); - if (refill) snoop_line(bank * 64, 4); - access (bank * 64, 0, '1, op); - end - end - end - access (bank * 64, 0, 0, 10); - access (bank * 64, 0, 99, 11, '1, 1); // successful SC hit - snoop_line(bank * 64, 4); - access (bank * 64, 0, 0, 10); - access (bank * 64, 0, 101, 11, '1, 1); // successful SC fill - access (bank * 64, 0, 0, 12); - end - // AMO.W uses an independent u32/i32 reference and must preserve the other lane. - for (int bank = 0; bank < 2; bank++) begin - for (int lane = 0; lane < 2; lane++) begin - longint unsigned address = bank * 64 + lane * 4; - bit [31:0] lhs_values[4] = '{32'hffffffff, 32'h80000000, 32'h7fffffff, 32'h0}; - bit [31:0] rhs_values[4] = '{32'h1, 32'h7fffffff, 32'h80000000, 32'hffffffff}; - for (int op = 1; op <= 9; op++) begin - for (int boundary = 0; boundary < 4; boundary++) begin - bit [63:0] initial_value = 64'h5aa55aa5a55aa55a; - initial_value[lane*32+:32] = lhs_values[boundary]; - access (bank * 64, 1, initial_value); - if (boundary % 2) snoop_line(bank * 64, 4); // exercise refill permission upgrade - // High operand bits are intentionally unrelated to the 32-bit operation. - access (address, 0, {32'hdeadbeef, rhs_values[boundary]}, op, '1, 0, 0, 1); - access (bank * 64); // independently check both lane update and neighbor preservation - end - end - access (address, 0, 0, 10, '1, 0, 0, 1); - access (address, 0, 64'habcdef0180000001, 11, '1, 1, 0, 1); - access (bank * 64); - snoop_line(bank * 64, 4); - access (address, 0, 0, 10, '1, 0, 0, 1); - access (address, 0, 64'hffffffffffffffff, 11, '1, 1, 0, 1); - access (bank * 64); - access (address, 0, 0, 10, '1, 0, 0, 1); - snoop_line(bank * 64, 7); - access (address, 0, 99, 11, '1, 0, 0, 1); - access (address, 0, 0, 10, '1, 0, 0, 1); - access (bank * 64 + (1 - lane) * 4, 0, 77, 11, '1, 0, 0, 1); - access (address, 0, 88, 11, '1, 0, 0, 1); // failed other-lane SC consumes LR - access (address, 0, 0, 10, '1, 0, 0, 1); - access ((1 - bank) * 64, 0, 77, 11, '1, 0, 0, 1); - access (address, 0, 88, 11, '1, 0, 0, 1); // failed other-bank SC consumes LR - access (address, 0, 0, 10, '1, 0, 0, 1); - access ((1 - bank) * 64, 0, 0, 10); // later LR.D replaces LR.W across banks - access (address, 0, 88, 11, '1, 0, 0, 1); - end - access (bank * 64, 0, 0, 10, '1, 0, 0, 1); - access (bank * 64, 0, 88, 11); // same address, mismatched reservation width - access (bank * 64, 0, 0, 10); - access (bank * 64, 0, 88, 11, '1, 0, 0, 1); - access (bank * 64, 0, 0, 10, '1, 0, 0, 1); - snoop_line((1 - bank) * 64, 7); // unrelated bank invalidation preserves reservation - access (bank * 64, 0, 99, 11, '1, 1, 0, 1); - access (bank * 64); - end - // Every word position uses both lane selectors in the hit and refill merge datapaths. - for (int bank = 0; bank < 2; bank++) begin - for (int word_index = 0; word_index < 8; word_index++) begin - longint unsigned word_address = bank * 64 + word_index * 8; - access (word_address, 1, 64'h800000007fffffff); - for (int lane = 0; lane < 2; lane++) begin - access (word_address + lane * 4, 0, 64'hdeadbeef00000001, 2, '1, 0, 0, 1); - access (word_address); - snoop_line(bank * 64, 4); - access (word_address + lane * 4, 0, 64'hdeadbeefffffffff, 2, '1, 0, 0, 1); - access (word_address); - end - end - end - access (576); // dirty eviction through the second bank - access (64); - access (64, 0, 0, 10); - access (64, 0, 99, 11, '1, 1); - access (64, 0, 0, 10); - snoop_line(64, 7); // invalidate consumes LR - access (64, 0, 123, 11); - access (0, 0, 0, 10); - access (64, 0, 123, 11); // SC to other bank fails and consumes reservation - access (0, 0, 123, 11); - access (0, 0, 0, 12); - snoop_line(448, 4); // clean cached Shared line: dataless response - snoop_line(1536, 9); // absent line: dataless Invalid response - fail_read = 2; - access (768, 0, 0, 0, '1, 0, 1); - fail_read = 0; - access (768); // errored fill must not leave a valid cache line - access (512); // write back the bank0 AMO/SC result before reset invalidates L1 - // Coordinated, quiescent reset invalidates L1; backing contents persist. - @(negedge vif.clock); - vif.reset = 1; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - issue(0); - access (64); - // Every legal high address bit must reach the tag compare and eviction path. - for (int bit_index = 9; bit_index < 44; bit_index++) begin - for (int cache_index = 0; cache_index < 8; cache_index++) begin - longint unsigned high = (64'(1) << bit_index) | (64'(cache_index) << 6); - bit [511:0] high_line = {8{(bit_index % 2 ? 64'h0123456789abcdef : ~64'h0123456789abcdef) ^ high}}; - memory[high] = high_line; - if (rnf_ref_write(model, high, high_line, '1)) `uvm_fatal("MODEL", "High-tag init failed") - access (high, 0, 0, 10); // full-address LR reservation in each bank - snoop_line(high, 4, 1); // all entry indices and high Snoop addresses, full clean data - access (high + 56, 1, - (bit_index % 2 ? 64'h0123456789abcdef : ~64'h0123456789abcdef) ^ high); - access (high + 8, 1, - (bit_index % 2 ? ~64'h0123456789abcdef : 64'h0123456789abcdef) ^ high); // hit-write R3 read port - access (high); // R0 hit-read port must observe the complementary full line - access (cache_index * 64); // dirty high-tag copyback retains the full address - end - end - for (int bank = 0; bank < 2; bank++) begin - access (bank * 64, 1, 64'h1234567890abcdef); - hold_req = 1; - issue(512 + bank * 64); - begin - req_t pending_request; - do begin - @(negedge vif.clock); - pending_request = req_t'(vif.req); - end while (!vif.req_valid || pending_request.opcode != 'h1b); - end - snoop_line(bank * 64, 7); // revoke dirty victim while its WriteBackFull is queued - begin - bit [511:0] replacement = {8{64'hfedcba9876543210}}; - memory[bank*64] = replacement; - if (rnf_ref_write(model, bank * 64, replacement, '1)) - `uvm_fatal("MODEL", "Remote write failed") - end - hold_req = 0; - settle(); - access (bank * 64); // cancelled copyback must not overwrite the newer line - end - if (cancelled_writebacks == 0) - `uvm_fatal("CANCEL", "Queued writeback cancellation did not execute") - fail_read = 2; - access (832, 0, 0, 0, '1, 0, 1); // second bank error path - fail_read = 0; - access (832); - access (448, 0, 0, 10); - snoop_line(448, 4, 1); // clean RetToSrc data must preserve LR reservation - access (448, 0, 44, 11, '1, 1); - for (int bank = 0; bank < 2; bank++) begin - access (bank * 64, 1, 64'h0123456789abcdef); - hold_req = 1; - issue(512 + bank * 64); - begin - req_t pending_request; - do begin - @(negedge vif.clock); - pending_request = req_t'(vif.req); - end while (!vif.req_valid || pending_request.opcode != 'h1b); - end - snoop_line(bank * 64, 4); // queued dirty victim is downgraded to Shared before copyback - hold_req = 0; - settle(); - access (bank * 64); - for (int entry = 0; entry < 4; entry++) begin - longint unsigned address = entry * 128 + bank * 64; - access (address); - snoop_line(address, 4); - access (address, 0, 0, 10); - hold_req = 1; - issue(address, 0, 64'habcdef, - 11); // initially valid LR, SC must fail after the intervening snoop - begin - req_t pending_request; - do begin - @(negedge vif.clock); - pending_request = req_t'(vif.req); - end while (!vif.req_valid || pending_request.opcode != 'h07); - end - snoop_line(address, 7); - hold_req = 0; - settle(); - access (address); // failed SC fill acquired UC but must not modify the word - access (address, 1, 64'h123456789abcdef0); // clean UC hit becomes dirty - end - end - for (int order = 0; order < 2; order++) begin - access (0, 1, 64'h55aa55aa55aa55aa); - access (64, 1, 64'haa55aa55aa55aa55); - hold_output_dat = 1; - issue(3072 + order * 64); - issue(3136 - order * 64); // both bank arrival orders - repeat (60) @(negedge vif.clock); - hold_output_dat = 0; - settle(); - end - // Fill both output request and response queues under sustained legal backpressure. - hold_req = 1; - hold_rsp = 1; - issue(2560); - issue(2624); - repeat (30) @(negedge vif.clock); - hold_req = 0; - repeat (60) @(negedge vif.clock); - fork - snoop_line(16000, 9); // third response meets two queued CompAcks - begin - repeat (20) @(negedge vif.clock); - hold_rsp = 0; - end - join - settle(); - for (int bank = 0; bank < 2; bank++) begin - for (int entry = 0; entry < 4; entry++) begin - longint unsigned address = entry * 128 + bank * 64; - snoop_line(address, 7); - snoop_line(address, 4); // absent Shared snoop at each directory entry - fail_read = entry % 2 ? 3 : 2; - mixed_data_error = entry == 2; - access (address, 0, 0, entry % 2 ? 0 : 10, '1, 0, 1); - fail_read = 0; - mixed_data_error = 0; - access (((entry + 1) % 4) * 128 + bank * 64); // a hit clears the previous response error - access (address); - access (address, 0, 0, 10); - access (address); // ordinary hit must retain LR - access (address, 0, 11, 11, '1, 1); - end - end - for (int bank = 0; bank < 2; bank++) begin - access (bank * 64, 0, 0, 10); - snoop_line(128 + bank * 64, 7); // unrelated-line invalidation must preserve LR - access (bank * 64, 0, 77, 11, '1, 1); - access (bank * 64, 0, 0, 0, 0); // a normal read does not consume byte enables - fork - issue(bank * 64); - begin - repeat (2) @(posedge vif.clock); - snoop_line(256 + bank * 64, 4); // snoop overlaps lookup and temporarily stops it - end - join - settle(); - end - access (128); // independent-line probe must be resident after invalidation scenarios - partial_fill_snoop(1024, 1024, 0); - partial_fill_snoop(1536, 1536, 1); // opposite DataID arrival order - partial_fill_snoop(2048, 128, 0); // same bank, independent cache line - for (int bank = 0; bank < 2; bank++) begin - access (bank * 64 + 56, 1, '1); - access (bank * 64, 1, '1); // full-width hit merge input includes the high word - access (bank * 64 + 56, 1, 0); - access (bank * 64, 1, 0); - for (int word = 0; word < 8; word++) begin - access (bank * 64 + word * 8, 0, 0, 10); - access (bank * 64 + word * 8, 1, '1, 0, 8'hfe); - access (bank * 64 + word * 8, 1, 0, 0, 8'h00); - access (bank * 64 + word * 8); - end - access (bank * 64, 0, 0, 10); // release high word-offset reservation bits - end - for (int bank = 0; bank < 2; bank++) begin - for (int miss = 0; miss < 2; miss++) begin - access (bank * 64); - for (int bit_index = 0; bit_index <= 32; bit_index++) begin - int unsigned seed = bit_index == 32 ? '1 : (32'(1) << bit_index) - 1; - int unsigned other, expected_counter = seed + 1; - longint unsigned address = miss ? (bit_index % 2 ? 1024 : 512) + bank * 64 : bank * 64; - @(negedge vif.clock); - other = vif.counter_value(!bank, miss); - vif.seed_counter(bank, miss, seed); - access (address); - if (vif.counter_value(bank, miss) !== expected_counter) - `uvm_fatal("COUNTER", "Real hit/miss did not increment the seeded statistical boundary") - if ((miss ? vif.misses : vif.hits) !== 32'(other + expected_counter)) - `uvm_fatal("COUNTER_SUM", "Bank sum did not retain modulo-32bit statistics") - end - end - end - begin - longint unsigned a = 24576; - bit [511:0] fresh = '0; - int before_alloc; - memory[a] = fresh; - if (rnf_ref_write(model, a, fresh, '1)) - `uvm_fatal("MODEL", "CMO fixture initialization failed") - access (a, 1, 64'h1122334455667788); - snoop_line(a, 8); // SnpCleanShared: pass dirty data, retain a clean shared copy. - before_alloc = home_allocations; - access (a); - if (home_allocations != before_alloc) - `uvm_fatal("CLEAN_SHARED", "SnpCleanShared invalidated a resident clean copy") - access (a, 1, 64'hdeadbeef01234567); - fresh[63:0] = 64'hcafebabe87654321; - memory[a] = fresh; - if (rnf_ref_write(model, a, fresh, '1)) - `uvm_fatal("GOLDEN", "DMA test write was outside physical memory") - snoop_line(a, 10); // SnpMakeInvalid must not return stale dirty data. - if (memory[a] !== fresh) - `uvm_fatal("MAKE_INVALID", "Stale dirty data overwrote newer DMA memory") - before_alloc = home_allocations; - access (a); - if (home_allocations != before_alloc + 1) - `uvm_fatal("INVALID_REFILL", "SnpMakeInvalid did not invalidate the resident line") - snoop_line(a, 8); // Clean shared copy: dataless retained response. - snoop_line(a, 10); - snoop_line(a, 10); // Absent line: dataless invalid response. - `uvm_info("CMO", "SnpCleanShared retention and SnpMakeInvalid dirty discard checked", UVM_LOW) - end - if (shared_writebacks == 0) - `uvm_fatal("SHARED_WB", "Downgraded victim did not return Shared copyback") - if (peak < 2 || writebacks == 0 || snoops < 2 || expected.size()) - `uvm_fatal("COVERAGE", "Required parallel, dirty eviction or snoop case not exercised") - `uvm_info( - "RNF", - $sformatf( - "Checked %0d CPU completions, peak=%0d writebacks=%0d snoops=%0d cancelled=%0d shared=%0d reset_cancelled=%0d", - checked, peak, writebacks, snoops, cancelled_writebacks, shared_writebacks, - cancelled_cpu), UVM_LOW) - rnf_ref_destroy(model); - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx.el b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx.el deleted file mode 100644 index cfecac53d..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx.el +++ /dev/null @@ -1,78 +0,0 @@ -// Format Version: 2 -// Reviewed against the exact exported object signatures and the declared depth4 always-on profile. - -CHECKSUM: "2163818788 95260501" -INSTANCE: rx_tb.dut -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit on inactive link" -Block 2 "1857595103" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit on inactive link" -Block 3 "971991619" "$error(\"Assertion failed: CHI RX flit on inactive link\n at Rx.scala:64 assert(io.active, \\"CHI RX flit on inactive link\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit on inactive link" -Block 5 "1857595103" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit on inactive link" -Block 6 "3528662196" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI FLITPEND must precede FLITV" -Block 10 "3823759045" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI FLITPEND must precede FLITV" -Block 11 "2663934422" "$error(\"Assertion failed: CHI FLITPEND must precede FLITV\n at Rx.scala:65 assert(previousPend, \\"CHI FLITPEND must precede FLITV\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI FLITPEND must precede FLITV" -Block 13 "3823759045" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI FLITPEND must precede FLITV" -Block 14 "2045170107" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit without granted credit" -Block 18 "798018255" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit without granted credit" -Block 19 "74350349" "$error(\"Assertion failed: CHI RX flit without granted credit\n at Rx.scala:66 assert(advertised =/= 0.U, \\"CHI RX flit without granted credit\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit without granted credit" -Block 21 "798018255" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit without granted credit" -Block 22 "2469915260" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX buffer overflow" -Block 26 "2435529851" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX buffer overflow" -Block 27 "2348576011" "$error(\"Assertion failed: CHI RX buffer overflow\n at Rx.scala:67 assert(enqueueReady, \\"CHI RX buffer overflow\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX buffer overflow" -Block 29 "2435529851" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX buffer overflow" -Block 30 "773203258" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX credit conservation" -Block 34 "3002688943" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX credit conservation" -Block 35 "2461347743" "$error(\"Assertion failed: CHI RX credit conservation\n at Rx.scala:69 assert(reserved <= depth.U, \\"CHI RX credit conservation\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX credit conservation" -Block 37 "3002688943" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX credit conservation" -Block 38 "1033059400" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX requires coordinated reset to stop" -Block 42 "2653430091" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX requires coordinated reset to stop" -Block 43 "1798953395" "$error(\"Assertion failed: CHI RX requires coordinated reset to stop\n at Rx.scala:70 when(wasActive)(assert(io.active, \\"CHI RX requires coordinated reset to stop\\"))\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX requires coordinated reset to stop" -Block 45 "2653430091" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX requires coordinated reset to stop" -Block 46 "2212642507" "$fatal;" - - -CHECKSUM: "2163818788 2138553741" -INSTANCE: rx_tb.dut -ANNOTATION: "Assertion-failure vector: physical FLITV requires the activated link." -Condition 2 "3562691098" "(_GEN_0 & ((~io_active))) 1 -1" (3 "11") -ANNOTATION: "Assertion-failure vector: FLITPEND must precede FLITV by a cycle." -Condition 3 "2541789932" "(_GEN_0 & ((~previousPend))) 1 -1" (3 "11") -ANNOTATION: "Assertion-failure vector: sender must hold a granted credit; independently checked by rx_bad_credit." -Condition 4 "959243910" "(_GEN_0 & (advertised == 3'b0)) 1 -1" (3 "11") -ANNOTATION: "Credit conservation proves any legal arriving flit has a reserved FIFO entry; overflow is outside legal peer behavior." -Condition 6 "183470145" "(_GEN_0 & ((~enqueueReady))) 1 -1" (3 "11") -ANNOTATION: "Depth4 invariant reserved=advertised+occupancy<=4, including coordinated reset boundary." -Condition 7 "311107905" "(((~reset)) & (reserved > 4'h4)) 1 -1" (1 "01") -ANNOTATION: "Depth4 invariant reserved<=4: grant is issued only below depth and legal enqueue replaces one advertised credit with one occupied entry." -Condition 7 "311107905" "(((~reset)) & (reserved > 4'h4)) 1 -1" (3 "11") -ANNOTATION: "Always-on profile forbids runtime deactivation except under coordinated reset; assertion-failure vector only." -Condition 8 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (4 "111") -ANNOTATION: "A legal arriving flit owns advertised credit>0; advertised+occupancy<=4 implies occupancy<4 and enqueueReady. This functional vector is unreachable." -Condition 9 "241394234" "(io_link_flitv & enqueueReady) 1 -1" (2 "10") - -CHECKSUM: "2163818788 1283625097" -INSTANCE: rx_tb.dut -ANNOTATION: "Only reserved[3]: reset starts at zero; legal FLITV consumes an advertised entry, enqueue moves it to occupancy, and grant cannot exceed depth4. Hence reserved<=4 and bit3 is invariant zero. Bits2:0 remain measured." -Toggle reserved [3] "net reserved[3:0]" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx.f deleted file mode 100644 index 75884eb40..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/link_interface.sv -@RESOURCES@/link_package.sv --F @RTL@/Tx/filelist.f --F @RTL@/Rx/filelist.f -@RESOURCES@/rx_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_bad_credit.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_bad_credit.f deleted file mode 100644 index b54ec2380..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_bad_credit.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+CHI_RX_OVERFLOW -+define+CHI_CREDIT_TB=rx_bad_credit_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/credit_interface.sv -@RESOURCES@/credit_package.sv --F @RTL@/Rx/filelist.f -@RESOURCES@/credit_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_bad_credit.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_bad_credit.toml deleted file mode 100644 index f05649ac4..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_bad_credit.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CHI RX flit without granted credit" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_tb.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_tb.sv deleted file mode 100644 index e52638c74..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/rx_tb.sv +++ /dev/null @@ -1,38 +0,0 @@ -module rx_tb; - import uvm_pkg::*; - import chi_link_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_link_if source_if (clock); - wire flitpend, flitv, lcrdv; - wire [31:0] flit; - Tx transmitter ( - .clock(clock), - .reset(source_if.reset), - .io_active(source_if.active), - .io_in_valid(source_if.valid), - .io_in_ready(source_if.ready), - .io_in_bits(source_if.data), - .io_link_flitpend(flitpend), - .io_link_flitv(flitv), - .io_link_flit(flit), - .io_link_lcrdv(lcrdv), - .io_credits(source_if.credits) - ); - Rx dut ( - .clock(clock), - .reset(source_if.reset), - .io_active(source_if.active), - .io_link_flitpend(flitpend), - .io_link_flitv(flitv), - .io_link_flit(flit), - .io_link_lcrdv(lcrdv), - .io_out_valid(source_if.out_valid), - .io_out_ready(source_if.out_ready), - .io_out_bits(source_if.out_data) - ); - initial begin - uvm_config_db#(virtual chi_link_if)::set(null, "uvm_test_top", "vif", source_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram.el b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram.el deleted file mode 100644 index 3fd3dfb7a..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram.el +++ /dev/null @@ -1,971 +0,0 @@ -// Format Version: 2 -// Reviewed SN256 instance-specific diagnostics, legal-input constraints and source invariants. -// Preserve full functional payload, arbitrary valid IDs, idle physical payload and reachable scheduling. - -CHECKSUM: "3483611062 3911945341" -INSTANCE: sram_tb.dut.node -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI request to wrong node" -Block 2 "823506465" "if (1)" -Block 3 "1562418237" "$error(\"Assertion failed: CHI request to wrong node\n at MemoryNode.scala:64 assert(r.tgtId === nodeId.U, \\"CHI request to wrong node\\")\n\");" -Block 5 "823506465" "if (1)" -Block 6 "4007593502" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unsupported CHI request opcode" -Block 10 "3060840224" "if (1)" -Block 11 "1238176658" "$error(\"Assertion failed: Unsupported CHI request opcode\n at MemoryNode.scala:65 assert(\n\");" -Block 13 "3060840224" "if (1)" -Block 14 "3791457536" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI node requires aligned 64-byte access" -Block 18 "3302013798" "if (1)" -Block 19 "3011646935" "$error(\"Assertion failed: CHI node requires aligned 64-byte access\n at MemoryNode.scala:70 assert(r.size === 6.U && r.addr(5, 0) === 0.U, \\"CHI node requires aligned 64-byte access\\")\n\");" -Block 21 "3302013798" "if (1)" -Block 22 "3453092984" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unsupported CHI request attributes" -Block 26 "3756387644" "if (1)" -Block 27 "1839675237" "$error(\"Assertion failed: Unsupported CHI request attributes\n at MemoryNode.scala:71 assert(\n\");" -Block 29 "3756387644" "if (1)" -Block 30 "2962064657" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Device memory is outside this CHI node profile" -Block 34 "3061807544" "if (1)" -Block 35 "2656390042" "$error(\"Assertion failed: Device memory is outside this CHI node profile\n at MemoryNode.scala:78 assert(!r.memAttr(1), \\"Device memory is outside this CHI node profile\\")\n\");" -Block 37 "3061807544" "if (1)" -Block 38 "388673725" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI source reused a live TxnID" -Block 42 "2059430060" "if (1)" -Block 43 "1492632624" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 45 "2059430060" "if (1)" -Block 46 "2414433873" "$fatal;" -Block 50 "974596391" "if (1)" -Block 51 "1474737040" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 53 "974596391" "if (1)" -Block 54 "2163745265" "$fatal;" -Block 58 "4218350522" "if (1)" -Block 59 "1188375408" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 61 "4218350522" "if (1)" -Block 62 "2446047505" "$fatal;" -Block 66 "3149507121" "if (1)" -Block 67 "1237605584" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 69 "3149507121" "if (1)" -Block 70 "2665139889" "$fatal;" -Block 74 "3221991744" "if (1)" -Block 75 "1864251192" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 77 "3221991744" "if (1)" -Block 78 "3087050073" "$fatal;" -Block 82 "2161963211" "if (1)" -Block 83 "1611612312" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 85 "2161963211" "if (1)" -Block 86 "3071398649" "$fatal;" -Block 90 "1101345366" "if (1)" -Block 91 "1899812984" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 93 "1101345366" "if (1)" -Block 94 "2787265049" "$fatal;" -Block 98 "24277981" "if (1)" -Block 99 "2116923352" "$error(\"Assertion failed: CHI source reused a live TxnID\n at MemoryNode.scala:80 assert(\n\");" -Block 101 "24277981" "if (1)" -Block 102 "2838707641" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unsupported CHI data opcode" -Block 106 "1895873786" "if (1)" -Block 107 "1082696673" "$error(\"Assertion failed: Unsupported CHI data opcode\n at MemoryNode.scala:97 assert(d.opcode === Opcode.NonCopyBackWriteData.U, \\"Unsupported CHI data opcode\\")\n\");" -Block 109 "1895873786" "if (1)" -Block 110 "4271299145" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Invalid CHI write destination/DBID" -Block 114 "2750264503" "if (1)" -Block 115 "3419488649" "$error(\"Assertion failed: Invalid CHI write destination/DBID\n at MemoryNode.scala:98 assert(d.tgtId === nodeId.U && d.txnId < slots.U, \\"Invalid CHI write destination/DBID\\")\n\");" -Block 117 "2750264503" "if (1)" -Block 118 "1909902123" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Errored write data is outside this CHI node profile" -Block 122 "3516907868" "if (1)" -Block 123 "3490684787" "$error(\"Assertion failed: Errored write data is outside this CHI node profile\n at MemoryNode.scala:99 assert(d.respErr === 0.U, \\"Errored write data is outside this CHI node profile\\")\n\");" -Block 125 "3516907868" "if (1)" -Block 126 "1011725012" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unsupported CHI data attributes" -Block 130 "2818579679" "if (1)" -Block 131 "2858884155" "$error(\"Assertion failed: Unsupported CHI data attributes\n at MemoryNode.scala:100 assert(\n\");" -Block 133 "2818579679" "if (1)" -Block 134 "3581507328" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Invalid CHI DataID for data width" -Block 138 "3427410875" "if (1)" -Block 139 "401734482" "$error(\"Assertion failed: Invalid CHI DataID for data width\n at MemoryNode.scala:108 assert(legalId, \\"Invalid CHI DataID for data width\\")\n\");" -Block 141 "3427410875" "if (1)" -Block 142 "3587222114" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI write data before DBID or after completion" -Block 146 "974379480" "if (1)" -Block 147 "534939643" "$error(\"Assertion failed: CHI write data before DBID or after completion\n at MemoryNode.scala:110 assert(state(index) === collect, \\"CHI write data before DBID or after completion\\")\n\");" -Block 149 "974379480" "if (1)" -Block 150 "1661163309" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI write data from wrong requester" -Block 154 "8984338" "if (1)" -Block 155 "548569855" "$error(\"Assertion failed: CHI write data from wrong requester\n at MemoryNode.scala:111 assert(d.srcId === requests(index).srcId, \\"CHI write data from wrong requester\\")\n\");" -Block 157 "8984338" "if (1)" -Block 158 "3172387341" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Duplicate CHI DataID" -Block 162 "1067614932" "if (1)" -Block 163 "3638277198" "$error(\"Assertion failed: Duplicate CHI DataID\n at MemoryNode.scala:112 assert(!(received(index) & UIntToOH(beat, p.beatsPerLine)).orR, \\"Duplicate CHI DataID\\")\n\");" -Block 165 "1067614932" "if (1)" -Block 166 "382543761" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: WriteNoSnpFull requires all byte enables" -Block 170 "1678828384" "if (1)" -Block 171 "848044923" "$error(\"Assertion failed: WriteNoSnpFull requires all byte enables\n at MemoryNode.scala:114 assert(d.be.andR, \\"WriteNoSnpFull requires all byte enables\\")\n\");" -Block 173 "1678828384" "if (1)" -Block 174 "1785821791" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Invalid memory response ID" -Block 178 "841885701" "if (1)" -Block 179 "1786497890" "$error(\"Assertion failed: Invalid memory response ID\n at MemoryNode.scala:145 assert(r.id < slots.U, \\"Invalid memory response ID\\")\n\");" -Block 181 "841885701" "if (1)" -Block 182 "4251582268" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unexpected memory completion" -Block 186 "2406142010" "if (1)" -Block 187 "1700436678" "$error(\"Assertion failed: Unexpected memory completion\n at MemoryNode.scala:147 assert(state(index) === waitMemory, \\"Unexpected memory completion\\")\n\");" -Block 189 "2406142010" "if (1)" -Block 190 "2380214045" "$fatal;" - -CHECKSUM: "1542839461 95260501" -INSTANCE: sram_tb.dut.rxReq -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit on inactive link" -Block 2 "1857595103" "if (1)" -Block 3 "971991619" "$error(\"Assertion failed: CHI RX flit on inactive link\n at Rx.scala:64 assert(io.active, \\"CHI RX flit on inactive link\\")\n\");" -Block 5 "1857595103" "if (1)" -Block 6 "3528662196" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI FLITPEND must precede FLITV" -Block 10 "3823759045" "if (1)" -Block 11 "2663934422" "$error(\"Assertion failed: CHI FLITPEND must precede FLITV\n at Rx.scala:65 assert(previousPend, \\"CHI FLITPEND must precede FLITV\\")\n\");" -Block 13 "3823759045" "if (1)" -Block 14 "2045170107" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit without granted credit" -Block 18 "798018255" "if (1)" -Block 19 "74350349" "$error(\"Assertion failed: CHI RX flit without granted credit\n at Rx.scala:66 assert(advertised =/= 0.U, \\"CHI RX flit without granted credit\\")\n\");" -Block 21 "798018255" "if (1)" -Block 22 "2469915260" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX buffer overflow" -Block 26 "2435529851" "if (1)" -Block 27 "2348576011" "$error(\"Assertion failed: CHI RX buffer overflow\n at Rx.scala:67 assert(enqueueReady, \\"CHI RX buffer overflow\\")\n\");" -Block 29 "2435529851" "if (1)" -Block 30 "773203258" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX credit conservation" -Block 34 "3002688943" "if (1)" -Block 35 "2461347743" "$error(\"Assertion failed: CHI RX credit conservation\n at Rx.scala:69 assert(reserved <= depth.U, \\"CHI RX credit conservation\\")\n\");" -Block 37 "3002688943" "if (1)" -Block 38 "1033059400" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX requires coordinated reset to stop" -Block 42 "2653430091" "if (1)" -Block 43 "1798953395" "$error(\"Assertion failed: CHI RX requires coordinated reset to stop\n at Rx.scala:70 when(wasActive)(assert(io.active, \\"CHI RX requires coordinated reset to stop\\"))\n\");" -Block 45 "2653430091" "if (1)" -Block 46 "2212642507" "$fatal;" - -CHECKSUM: "1028417357 1379207481" -INSTANCE: sram_tb.dut.txRsp -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit overflow" -Block 2 "3115401253" "if (1)" -Block 3 "1708991190" "$error(\"Assertion failed: CHI TX credit overflow\n at Tx.scala:43 when(returned && !send)(assert(credits < maxCredits.U, \\"CHI TX credit overflow\\"))\n\");" -Block 5 "3115401253" "if (1)" -Block 6 "4063542534" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit underflow" -Block 10 "1503763571" "if (1)" -Block 11 "556320376" "$error(\"Assertion failed: CHI TX credit underflow\n at Tx.scala:44 when(send)(assert(credits =/= 0.U, \\"CHI TX credit underflow\\"))\n\");" -Block 13 "1503763571" "if (1)" -Block 14 "2166023300" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX requires coordinated reset to stop" -Block 18 "2653430091" "if (1)" -Block 19 "597017625" "$error(\"Assertion failed: CHI TX requires coordinated reset to stop\n at Tx.scala:45 when(wasActive)(assert(io.active, \\"CHI TX requires coordinated reset to stop\\"))\n\");" -Block 21 "2653430091" "if (1)" -Block 22 "2212642507" "$fatal;" - -CHECKSUM: "3712248894 3505880431" -INSTANCE: sram_tb.dut.rxDat -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit on inactive link" -Block 2 "1857595103" "if (1)" -Block 3 "971991619" "$error(\"Assertion failed: CHI RX flit on inactive link\n at Rx.scala:64 assert(io.active, \\"CHI RX flit on inactive link\\")\n\");" -Block 5 "1857595103" "if (1)" -Block 6 "3528662196" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI FLITPEND must precede FLITV" -Block 10 "3823759045" "if (1)" -Block 11 "2663934422" "$error(\"Assertion failed: CHI FLITPEND must precede FLITV\n at Rx.scala:65 assert(previousPend, \\"CHI FLITPEND must precede FLITV\\")\n\");" -Block 13 "3823759045" "if (1)" -Block 14 "2045170107" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX flit without granted credit" -Block 18 "798018255" "if (1)" -Block 19 "74350349" "$error(\"Assertion failed: CHI RX flit without granted credit\n at Rx.scala:66 assert(advertised =/= 0.U, \\"CHI RX flit without granted credit\\")\n\");" -Block 21 "798018255" "if (1)" -Block 22 "2469915260" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX buffer overflow" -Block 26 "2435529851" "if (1)" -Block 27 "2348576011" "$error(\"Assertion failed: CHI RX buffer overflow\n at Rx.scala:67 assert(enqueueReady, \\"CHI RX buffer overflow\\")\n\");" -Block 29 "2435529851" "if (1)" -Block 30 "773203258" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX credit conservation" -Block 34 "3002688943" "if (1)" -Block 35 "2461347743" "$error(\"Assertion failed: CHI RX credit conservation\n at Rx.scala:69 assert(reserved <= depth.U, \\"CHI RX credit conservation\\")\n\");" -Block 37 "3002688943" "if (1)" -Block 38 "1033059400" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI RX requires coordinated reset to stop" -Block 42 "2653430091" "if (1)" -Block 43 "1798953395" "$error(\"Assertion failed: CHI RX requires coordinated reset to stop\n at Rx.scala:70 when(wasActive)(assert(io.active, \\"CHI RX requires coordinated reset to stop\\"))\n\");" -Block 45 "2653430091" "if (1)" -Block 46 "2212642507" "$fatal;" - -CHECKSUM: "2770640837 940463411" -INSTANCE: sram_tb.dut.txDat -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit overflow" -Block 2 "3115401253" "if (1)" -Block 3 "1708991190" "$error(\"Assertion failed: CHI TX credit overflow\n at Tx.scala:43 when(returned && !send)(assert(credits < maxCredits.U, \\"CHI TX credit overflow\\"))\n\");" -Block 5 "3115401253" "if (1)" -Block 6 "4063542534" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX credit underflow" -Block 10 "1503763571" "if (1)" -Block 11 "556320376" "$error(\"Assertion failed: CHI TX credit underflow\n at Tx.scala:44 when(send)(assert(credits =/= 0.U, \\"CHI TX credit underflow\\"))\n\");" -Block 13 "1503763571" "if (1)" -Block 14 "2166023300" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI TX requires coordinated reset to stop" -Block 18 "2653430091" "if (1)" -Block 19 "597017625" "$error(\"Assertion failed: CHI TX requires coordinated reset to stop\n at Tx.scala:45 when(wasActive)(assert(io.active, \\"CHI TX requires coordinated reset to stop\\"))\n\");" -Block 21 "2653430091" "if (1)" -Block 22 "2212642507" "$fatal;" - -CHECKSUM: "2383780593 637638803" -INSTANCE: sram_tb.dut -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Runtime RX link deactivation is unsupported" -Block 2 "337229911" "if (1)" -Block 3 "2663757680" "$error(\"Assertion failed: Runtime RX link deactivation is unsupported\n at SramEndpoint.scala:47 when(rxAck)(assert(io.chi.rxLinkActiveReq, \\"Runtime RX link deactivation is unsupported\\"))\n\");" -Block 5 "337229911" "if (1)" -Block 6 "1648053033" "$fatal;" - -CHECKSUM: "3483611062 1539539244" -INSTANCE: sram_tb.dut.node -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI request to wrong node. Functional gates and assertion remain enabled." -Condition 5 "148920103" "(_GEN_6 & (io_req_bits_tgtId != 7'b1)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unsupported CHI request opcode. Functional gates and assertion remain enabled." -Condition 7 "2559978673" "(_GEN_6 & ( ~ (_state_T | (io_req_bits_opcode == 7'h1d) | (io_req_bits_opcode == 7'h1c)) )) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI node requires aligned 64-byte access. Functional gates and assertion remain enabled." -Condition 12 "3277521579" "(_GEN_6 & ( ~ ((io_req_bits_size == 6'h06) & (io_req_bits_addr[5:0] == 6'b0)) )) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Device memory is outside this CHI node profile. Functional gates and assertion remain enabled." -Condition 24 "1834020855" "(_GEN_6 & io_req_bits_memAttr[1]) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI source reused a live TxnID. Functional gates and assertion remain enabled." -Condition 25 "4152540323" "(_GEN_6 & ((|state_0)) & (requests_0_srcId == io_req_bits_srcId) & (requests_0_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -Condition 28 "2098488880" "(_GEN_6 & ((|state_1)) & (requests_1_srcId == io_req_bits_srcId) & (requests_1_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -Condition 31 "3865973810" "(_GEN_6 & ((|state_2)) & (requests_2_srcId == io_req_bits_srcId) & (requests_2_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -Condition 34 "1828245153" "(_GEN_6 & ((|state_3)) & (requests_3_srcId == io_req_bits_srcId) & (requests_3_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -Condition 37 "3510787249" "(_GEN_6 & ((|state_4)) & (requests_4_srcId == io_req_bits_srcId) & (requests_4_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -Condition 40 "1540654626" "(_GEN_6 & ((|state_5)) & (requests_5_srcId == io_req_bits_srcId) & (requests_5_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -Condition 43 "3232691232" "(_GEN_6 & ((|state_6)) & (requests_6_srcId == io_req_bits_srcId) & (requests_6_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -Condition 46 "1245195955" "(_GEN_6 & ((|state_7)) & (requests_7_srcId == io_req_bits_srcId) & (requests_7_txnId == io_req_bits_txnId)) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unsupported CHI data opcode. Functional gates and assertion remain enabled." -Condition 49 "1423762789" "(_GEN_7 & (io_rxDat_bits_opcode != 4'h3)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Invalid CHI write destination/DBID. Functional gates and assertion remain enabled." -Condition 51 "3130223411" "(_GEN_7 & ( ~ ((io_rxDat_bits_tgtId == 7'b1) & _GEN_0) )) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Errored write data is outside this CHI node profile. Functional gates and assertion remain enabled." -Condition 55 "4258216982" "(_GEN_7 & ((|io_rxDat_bits_respErr))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Invalid CHI DataID for data width. Functional gates and assertion remain enabled." -Condition 63 "1066065420" "(_GEN_7 & ({(io_rxDat_bits_dataId == 2'h2), (io_rxDat_bits_dataId == 2'b0)} == 2'b0)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI write data before DBID or after completion. Functional gates and assertion remain enabled." -Condition 65 "1983754605" "(_GEN_10 & (_GEN_9[io_rxDat_bits_txnId[2:0]] != 3'h2)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI write data from wrong requester. Functional gates and assertion remain enabled." -Condition 67 "1394082298" "(_GEN_10 & (io_rxDat_bits_srcId != _GEN_11[io_rxDat_bits_txnId[2:0]])) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Duplicate CHI DataID. Functional gates and assertion remain enabled." -Condition 69 "3572934112" "(_GEN_10 & ((|(_GEN_4 & (2'b1 << _GEN_2))))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: WriteNoSnpFull requires all byte enables. Functional gates and assertion remain enabled." -Condition 70 "3806849419" "(_GEN_8 & (_GEN_1[io_rxDat_bits_txnId[2:0]] == 7'h1d) & ((~reset)) & (io_rxDat_bits_be != 32'hffffffff)) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Invalid memory response ID. Functional gates and assertion remain enabled." -Condition 73 "3254488011" "(io_memoryResp_valid & ((~reset)) & ((~_GEN_5))) 1 -1" (4 "111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unexpected memory completion. Functional gates and assertion remain enabled." -Condition 74 "1495455793" "(io_memoryResp_valid & _GEN_5 & ((~reset)) & (_GEN_9[io_memoryResp_bits_id[2:0]] != 3'h4)) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unsupported CHI request attributes. Functional gates and assertion remain enabled." -Condition 17 "815973446" "(_GEN_6 & ( ~ ((io_req_bits_order == 2'b0) & ((~io_req_bits_expCompAck)) & ((~io_req_bits_exclSnoopMe)) & ((~io_req_bits_snpAttr)) & ((~io_req_bits_stashNidValidEndian)) & (io_req_bits_pCrdType == 4'b0) & ((~io_req_bits_multiReq)) & (io_req_bits_pas == 3'b0) & (io_req_bits_tagOp == 2'b0)) )) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Unsupported CHI data attributes. Functional gates and assertion remain enabled." -Condition 56 "3320427110" "(_GEN_7 & ( ~ ((io_rxDat_bits_tagOp == 2'b0) & (io_rxDat_bits_tag == 8'b0) & (io_rxDat_bits_tagUpdate == 2'b0) & ((~io_rxDat_bits_dataPull)) & (io_rxDat_bits_numDat == 2'b0) & ((~io_rxDat_bits_replicate))) )) 1 -1" (3 "11") -ANNOTATION: "MemoryNode.scala lines98/109: accepted NonCopyBackWriteData must use an allocated local DBID0..7. The full-width id<8 check is false only for an illegal valid transfer; other vectors remain measured." -Condition 3 "3124372553" "(io_rxDat_valid & _GEN_0) 1 -1" (2 "10") -Condition 98 "1826432495" "(io_rxDat_valid & _GEN_0 & _GEN_20 & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 99 "2095748577" "(io_rxDat_valid & _GEN_0 & _GEN_20 & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 101 "2689989377" "(io_rxDat_valid & _GEN_0 & _GEN_23 & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 102 "2539887920" "(io_rxDat_valid & _GEN_0 & _GEN_23 & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 104 "26785311" "(io_rxDat_valid & _GEN_0 & _GEN_26 & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 105 "1935181793" "(io_rxDat_valid & _GEN_0 & _GEN_26 & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 107 "3885985187" "(io_rxDat_valid & _GEN_0 & _GEN_29 & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 108 "4186588712" "(io_rxDat_valid & _GEN_0 & _GEN_29 & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 110 "2718711594" "(io_rxDat_valid & _GEN_0 & _GEN_32 & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 111 "769774406" "(io_rxDat_valid & _GEN_0 & _GEN_32 & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 113 "3872915812" "(io_rxDat_valid & _GEN_0 & _GEN_35 & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 114 "1054611865" "(io_rxDat_valid & _GEN_0 & _GEN_35 & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 116 "3899578032" "(io_rxDat_valid & _GEN_0 & _GEN_38 & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 117 "3633996258" "(io_rxDat_valid & _GEN_0 & _GEN_38 & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 118 "265306532" "(io_rxDat_valid & _GEN_0 & ((&io_rxDat_bits_txnId[2:0])) & ((~io_rxDat_bits_dataId[1]))) 1 -1" (2 "1011") -Condition 119 "2039696548" "(io_rxDat_valid & _GEN_0 & ((&io_rxDat_bits_txnId[2:0])) & io_rxDat_bits_dataId[1]) 1 -1" (2 "1011") -Condition 149 "3967989931" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & _GEN_20) 1 -1" (2 "1011") -Condition 154 "298828490" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & _GEN_23) 1 -1" (2 "1011") -Condition 159 "400676024" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & _GEN_26) 1 -1" (2 "1011") -Condition 164 "2421754122" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & _GEN_29) 1 -1" (2 "1011") -Condition 169 "563353181" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & _GEN_32) 1 -1" (2 "1011") -Condition 174 "3818989856" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & _GEN_35) 1 -1" (2 "1011") -Condition 179 "2253940498" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & _GEN_38) 1 -1" (2 "1011") -Condition 184 "2397255552" "(io_rxDat_valid & _GEN_0 & ((&nextReceived)) & ((&io_rxDat_bits_txnId[2:0]))) 1 -1" (2 "1011") -Condition 185 "1143475756" "(io_rxDat_valid & _GEN_0 & _GEN_20) 1 -1" (2 "101") -Condition 186 "908938830" "(io_rxDat_valid & _GEN_0 & _GEN_23) 1 -1" (2 "101") -Condition 187 "3991860434" "(io_rxDat_valid & _GEN_0 & _GEN_26) 1 -1" (2 "101") -Condition 188 "3921519957" "(io_rxDat_valid & _GEN_0 & _GEN_29) 1 -1" (2 "101") -Condition 189 "3254585959" "(io_rxDat_valid & _GEN_0 & _GEN_32) 1 -1" (2 "101") -Condition 190 "2792948640" "(io_rxDat_valid & _GEN_0 & _GEN_35) 1 -1" (2 "101") -Condition 191 "510517628" "(io_rxDat_valid & _GEN_0 & _GEN_38) 1 -1" (2 "101") -Condition 192 "2930983340" "(io_rxDat_valid & _GEN_0 & ((&io_rxDat_bits_txnId[2:0]))) 1 -1" (2 "101") -ANNOTATION: "LineSram asserts response valid only after capturing a memArb request with literal local ID0..7; a valid response therefore has id<8, including a coincident reset. Uninitialized invalid response ID remains measured." -Condition 73 "3254488011" "(io_memoryResp_valid & ((~reset)) & ((~_GEN_5))) 1 -1" (2 "101") -ANNOTATION: "LineSram echoes its request ID; MemoryNode memArb supplies only literal local IDs0..7. Therefore a valid backend completion with id>=8 is structurally unreachable." -Condition 74 "1495455793" "(io_memoryResp_valid & _GEN_5 & ((~reset)) & (_GEN_9[io_memoryResp_bits_id[2:0]] != 3'h4)) 1 -1" (2 "1011") -Condition 120 "205294699" "(io_memoryResp_valid & _GEN_5 & (io_memoryResp_bits_id[2:0] == 3'b0)) 1 -1" (2 "101") -Condition 122 "2537288514" "(io_memoryResp_valid & _GEN_5 & (io_memoryResp_bits_id[2:0] == 3'b1)) 1 -1" (2 "101") -Condition 124 "3289281793" "(io_memoryResp_valid & _GEN_5 & (io_memoryResp_bits_id[2:0] == 3'h2)) 1 -1" (2 "101") -Condition 126 "2725440915" "(io_memoryResp_valid & _GEN_5 & (io_memoryResp_bits_id[2:0] == 3'h3)) 1 -1" (2 "101") -Condition 128 "1866493802" "(io_memoryResp_valid & _GEN_5 & (io_memoryResp_bits_id[2:0] == 3'h4)) 1 -1" (2 "101") -Condition 130 "3072721141" "(io_memoryResp_valid & _GEN_5 & (io_memoryResp_bits_id[2:0] == 3'h5)) 1 -1" (2 "101") -Condition 132 "506965470" "(io_memoryResp_valid & _GEN_5 & (io_memoryResp_bits_id[2:0] == 3'h6)) 1 -1" (2 "101") -Condition 134 "3797793362" "(io_memoryResp_valid & _GEN_5 & ((&io_memoryResp_bits_id[2:0]))) 1 -1" (2 "101") - -CHECKSUM: "1542839461 2138553741" -INSTANCE: sram_tb.dut.rxReq -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX flit on inactive link. Functional gates and assertion remain enabled." -Condition 2 "3562691098" "(_GEN_0 & ((~io_active))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI FLITPEND must precede FLITV. Functional gates and assertion remain enabled." -Condition 3 "2541789932" "(_GEN_0 & ((~previousPend))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX flit without granted credit. Functional gates and assertion remain enabled." -Condition 4 "959243910" "(_GEN_0 & (advertised == 3'b0)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX buffer overflow. Functional gates and assertion remain enabled." -Condition 6 "183470145" "(_GEN_0 & ((~enqueueReady))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX credit conservation. Functional gates and assertion remain enabled." -Condition 7 "311107905" "(((~reset)) & (reserved > 4'h4)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX requires coordinated reset to stop. Functional gates and assertion remain enabled." -Condition 8 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (4 "111") -ANNOTATION: "Depth4 invariant reserved=advertised+occupancy<=4, including coordinated reset boundary." -Condition 7 "311107905" "(((~reset)) & (reserved > 4'h4)) 1 -1" (1 "01") -ANNOTATION: "A legal arriving flit owns advertised credit>0; advertised+occupancy<=4 implies occupancy<4 and enqueueReady. This functional vector is unreachable." -Condition 9 "241394234" "(io_link_flitv & enqueueReady) 1 -1" (2 "10") -ANNOTATION: "SramEndpoint rxAck/txActive and child wasActive all synchronously reset to0. In this always-on profile active remains1 after activation until reset; before the first reset edge both are1, after it both are0. Therefore wasActive1 with active0 cannot coincide even with reset asserted." -Condition 8 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (2 "101") - -CHECKSUM: "1028417357 1909474316" -INSTANCE: sram_tb.dut.txRsp -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI TX credit overflow. Functional gates and assertion remain enabled." -Condition 1 "3175300005" "(returned & ((~send)) & ((~reset)) & ((&credits))) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI TX credit underflow. Functional gates and assertion remain enabled." -Condition 2 "3481575346" "(send & ((~reset)) & ((~(|credits)))) 1 -1" (4 "111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI TX requires coordinated reset to stop. Functional gates and assertion remain enabled." -Condition 3 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (4 "111") -ANNOTATION: "SN Tx/15-credit peer: credit=15 means all 15 credits are available; another returned credit cannot simultaneously be pending. This vector would violate peer credit conservation." -Condition 1 "3175300005" "(returned & ((~send)) & ((~reset)) & ((&credits))) 1 -1" (2 "1011") -ANNOTATION: "SN Tx/15-credit peer: no returned credit can coexist with all 15 credits available, including the coordinated reset boundary." -Condition 1 "3175300005" "(returned & ((~send)) & ((~reset)) & ((&credits))) 1 -1" (3 "1101") -ANNOTATION: "Structural invariant: send requires ready, and ready requires nonzero credits; send with zero credits is impossible even on reset." -Condition 2 "3481575346" "(send & ((~reset)) & ((~(|credits)))) 1 -1" (2 "101") -ANNOTATION: "Always-on SramEndpoint resets txActive and Tx credits together; peer credits start only after link activation. With no runtime deactivation, inactive implies credits0." -Condition 7 "1112456050" "(io_active & ((|credits))) 1 -1" (1 "01") -ANNOTATION: "SramEndpoint rxAck/txActive and child wasActive all synchronously reset to0. In this always-on profile active remains1 after activation until reset; before the first reset edge both are1, after it both are0. Therefore wasActive1 with active0 cannot coincide even with reset asserted." -Condition 3 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (2 "101") - -CHECKSUM: "3712248894 2941452375" -INSTANCE: sram_tb.dut.rxDat -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX flit on inactive link. Functional gates and assertion remain enabled." -Condition 2 "3562691098" "(_GEN_0 & ((~io_active))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI FLITPEND must precede FLITV. Functional gates and assertion remain enabled." -Condition 3 "2541789932" "(_GEN_0 & ((~previousPend))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX flit without granted credit. Functional gates and assertion remain enabled." -Condition 4 "959243910" "(_GEN_0 & (advertised == 3'b0)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX buffer overflow. Functional gates and assertion remain enabled." -Condition 6 "183470145" "(_GEN_0 & ((~enqueueReady))) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX credit conservation. Functional gates and assertion remain enabled." -Condition 7 "311107905" "(((~reset)) & (reserved > 4'h4)) 1 -1" (3 "11") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI RX requires coordinated reset to stop. Functional gates and assertion remain enabled." -Condition 8 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (4 "111") -ANNOTATION: "Depth4 invariant reserved=advertised+occupancy<=4, including coordinated reset boundary." -Condition 7 "311107905" "(((~reset)) & (reserved > 4'h4)) 1 -1" (1 "01") -ANNOTATION: "A legal arriving flit owns advertised credit>0; advertised+occupancy<=4 implies occupancy<4 and enqueueReady. This functional vector is unreachable." -Condition 9 "241394234" "(io_link_flitv & enqueueReady) 1 -1" (2 "10") -ANNOTATION: "MemoryNode ties rxDat.ready=true. From reset occupancy0, every occupied entry drains and at most one flit arrives per cycle, so occupancy<=1, occupancy[2]=0, enqueueReady=true. This exact condition vector contradicts that invariant." -Condition 6 "183470145" "(_GEN_0 & ((~enqueueReady))) 1 -1" (1 "01") -Condition 21 "1968069542" "(((~occupancy[2])) | ((|occupancy))) 1 -1" (1 "00") -Condition 21 "1968069542" "(((~occupancy[2])) | ((|occupancy))) 1 -1" (2 "01") -ANNOTATION: "SramEndpoint rxAck/txActive and child wasActive all synchronously reset to0. In this always-on profile active remains1 after activation until reset; before the first reset edge both are1, after it both are0. Therefore wasActive1 with active0 cannot coincide even with reset asserted." -Condition 8 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (2 "101") - -CHECKSUM: "2770640837 1909474316" -INSTANCE: sram_tb.dut.txDat -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI TX credit overflow. Functional gates and assertion remain enabled." -Condition 1 "3175300005" "(returned & ((~send)) & ((~reset)) & ((&credits))) 1 -1" (5 "1111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI TX credit underflow. Functional gates and assertion remain enabled." -Condition 2 "3481575346" "(send & ((~reset)) & ((~(|credits)))) 1 -1" (4 "111") -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: CHI TX requires coordinated reset to stop. Functional gates and assertion remain enabled." -Condition 3 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (4 "111") -ANNOTATION: "SN Tx/15-credit peer: credit=15 means all 15 credits are available; another returned credit cannot simultaneously be pending. This vector would violate peer credit conservation." -Condition 1 "3175300005" "(returned & ((~send)) & ((~reset)) & ((&credits))) 1 -1" (2 "1011") -ANNOTATION: "SN Tx/15-credit peer: no returned credit can coexist with all 15 credits available, including the coordinated reset boundary." -Condition 1 "3175300005" "(returned & ((~send)) & ((~reset)) & ((&credits))) 1 -1" (3 "1101") -ANNOTATION: "Structural invariant: send requires ready, and ready requires nonzero credits; send with zero credits is impossible even on reset." -Condition 2 "3481575346" "(send & ((~reset)) & ((~(|credits)))) 1 -1" (2 "101") -ANNOTATION: "Always-on SramEndpoint resets txActive and Tx credits together; peer credits start only after link activation. With no runtime deactivation, inactive implies credits0." -Condition 7 "1112456050" "(io_active & ((|credits))) 1 -1" (1 "01") -ANNOTATION: "SramEndpoint rxAck/txActive and child wasActive all synchronously reset to0. In this always-on profile active remains1 after activation until reset; before the first reset edge both are1, after it both are0. Therefore wasActive1 with active0 cannot coincide even with reset asserted." -Condition 3 "3706255149" "(wasActive & ((~reset)) & ((~io_active))) 1 -1" (2 "101") - -CHECKSUM: "2383780593 3054282342" -INSTANCE: sram_tb.dut -ANNOTATION: "Only the all-true assertion failure guard is excluded in normal legal protocol traffic: Runtime RX link deactivation is unsupported. Functional gates and assertion remain enabled." -Condition 1 "929193424" "(rxAck & ((~reset)) & ((~io_chi_rxLinkActiveReq))) 1 -1" (4 "111") - -CHECKSUM: "3483611062 488357000" -INSTANCE: sram_tb.dut.node -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured." -Toggle io_req_bits_tgtId [6:0] "net io_req_bits_tgtId[6:0]" -Toggle io_req_bits_stashNidValidEndian "net io_req_bits_stashNidValidEndian" -Toggle io_req_bits_opcode [2:1] "net io_req_bits_opcode[6:0]" -Toggle io_req_bits_opcode [6:5] "net io_req_bits_opcode[6:0]" -Toggle io_req_bits_multiReq "net io_req_bits_multiReq" -Toggle io_req_bits_size [5:0] "net io_req_bits_size[5:0]" -Toggle io_req_bits_addr [5:0] "net io_req_bits_addr[43:0]" -Toggle io_req_bits_pas [2:0] "net io_req_bits_pas[2:0]" -Toggle io_req_bits_order [1:0] "net io_req_bits_order[1:0]" -Toggle io_req_bits_pCrdType [3:0] "net io_req_bits_pCrdType[3:0]" -Toggle io_req_bits_memAttr [1] "net io_req_bits_memAttr[3:0]" -Toggle io_req_bits_snpAttr "net io_req_bits_snpAttr" -Toggle io_req_bits_exclSnoopMe "net io_req_bits_exclSnoopMe" -Toggle io_req_bits_expCompAck "net io_req_bits_expCompAck" -Toggle io_req_bits_tagOp [1:0] "net io_req_bits_tagOp[1:0]" -Toggle io_rxDat_bits_tgtId [6:0] "net io_rxDat_bits_tgtId[6:0]" -Toggle io_rxDat_bits_txnId [11:3] "net io_rxDat_bits_txnId[11:0]" -Toggle io_rxDat_bits_opcode [3:0] "net io_rxDat_bits_opcode[3:0]" -Toggle io_rxDat_bits_respErr [1:0] "net io_rxDat_bits_respErr[1:0]" -Toggle io_rxDat_bits_dataPull "net io_rxDat_bits_dataPull" -Toggle io_rxDat_bits_dataId [0] "net io_rxDat_bits_dataId[1:0]" -Toggle io_rxDat_bits_tagOp [1:0] "net io_rxDat_bits_tagOp[1:0]" -Toggle io_rxDat_bits_tag [7:0] "net io_rxDat_bits_tag[7:0]" -Toggle io_rxDat_bits_tagUpdate [1:0] "net io_rxDat_bits_tagUpdate[1:0]" -Toggle io_rxDat_bits_numDat [1:0] "net io_rxDat_bits_numDat[1:0]" -Toggle io_rxDat_bits_replicate "net io_rxDat_bits_replicate" -Toggle io_rsp_bits_opcode [0] "net io_rsp_bits_opcode[4:0]" -Toggle io_rsp_bits_opcode [4:2] "net io_rsp_bits_opcode[4:0]" -Toggle io_rsp_bits_dbid [11:3] "net io_rsp_bits_dbid[11:0]" -Toggle io_txDat_bits_dbid [15:12] "net io_txDat_bits_dbid[15:0]" -Toggle io_txDat_bits_dataId [0] "net io_txDat_bits_dataId[1:0]" -Toggle io_memoryReq_bits_id [11:3] "net io_memoryReq_bits_id[11:0]" -Toggle io_memoryReq_bits_addr [5:0] "net io_memoryReq_bits_addr[43:0]" -Toggle io_memoryResp_bits_id [11:3] "net io_memoryResp_bits_id[11:0]" -Toggle requests_0_opcode [2:1] "reg requests_0_opcode[6:0]" -Toggle requests_0_opcode [6:5] "reg requests_0_opcode[6:0]" -Toggle requests_0_addr [5:0] "reg requests_0_addr[43:0]" -Toggle requests_1_opcode [2:1] "reg requests_1_opcode[6:0]" -Toggle requests_1_opcode [6:5] "reg requests_1_opcode[6:0]" -Toggle requests_1_addr [5:0] "reg requests_1_addr[43:0]" -Toggle requests_2_opcode [2:1] "reg requests_2_opcode[6:0]" -Toggle requests_2_opcode [6:5] "reg requests_2_opcode[6:0]" -Toggle requests_2_addr [5:0] "reg requests_2_addr[43:0]" -Toggle requests_3_opcode [2:1] "reg requests_3_opcode[6:0]" -Toggle requests_3_opcode [6:5] "reg requests_3_opcode[6:0]" -Toggle requests_3_addr [5:0] "reg requests_3_addr[43:0]" -Toggle requests_4_opcode [2:1] "reg requests_4_opcode[6:0]" -Toggle requests_4_opcode [6:5] "reg requests_4_opcode[6:0]" -Toggle requests_4_addr [5:0] "reg requests_4_addr[43:0]" -Toggle requests_5_opcode [2:1] "reg requests_5_opcode[6:0]" -Toggle requests_5_opcode [6:5] "reg requests_5_opcode[6:0]" -Toggle requests_5_addr [5:0] "reg requests_5_addr[43:0]" -Toggle requests_6_opcode [2:1] "reg requests_6_opcode[6:0]" -Toggle requests_6_opcode [6:5] "reg requests_6_opcode[6:0]" -Toggle requests_6_addr [5:0] "reg requests_6_addr[43:0]" -Toggle requests_7_opcode [2:1] "reg requests_7_opcode[6:0]" -Toggle requests_7_opcode [6:5] "reg requests_7_opcode[6:0]" -Toggle requests_7_addr [5:0] "reg requests_7_addr[43:0]" -ANNOTATION: "256-bit line return has2beats; sent increments0->1->2 and clears on allocation, so bit2 is unreachable." -Toggle sent_0 [2] "reg sent_0[2:0]" -Toggle sent_1 [2] "reg sent_1[2:0]" -Toggle sent_2 [2] "reg sent_2[2:0]" -Toggle sent_3 [2] "reg sent_3[2:0]" -Toggle sent_4 [2] "reg sent_4[2:0]" -Toggle sent_5 [2] "reg sent_5[2:0]" -Toggle sent_6 [2] "reg sent_6[2:0]" -Toggle sent_7 [2] "reg sent_7[2:0]" -ANNOTATION: "MemoryNode.scala: DAT local DBID check is true for legal stored write DAT; backend response ID is below8 because LineSram echoes literal local slot IDs. Beat index is zero-extended DataID[1]. Stored request opcodes are4/28/29, whose bits2:1 and6:5 are fixed." -Toggle _GEN_1 [7:0][2:1] "net [7:0][6:0]_GEN_1" -Toggle _GEN_1 [7:0][6:5] "net [7:0][6:0]_GEN_1" -Toggle _GEN_2 [1] "net _GEN_2[1:0]" - -CHECKSUM: "1049582285 2762367578" -INSTANCE: sram_tb.dut.memory -ANNOTATION: "Backend receives only eight MemoryNode local slot IDs0..7 and returns the same ID; opaque original CHI TxnIDs remain full width in MemoryNode." -Toggle io_req_bits_id [11:3] "net io_req_bits_id[11:0]" -Toggle io_resp_bits_id [11:3] "net io_resp_bits_id[11:0]" -Toggle request_id [11:3] "reg request_id[11:0]" -ANNOTATION: "The connected MemoryNode asserts64B alignment before issuing backend requests." -Toggle io_req_bits_addr [5:0] "net io_req_bits_addr[43:0]" - -CHECKSUM: "4028718411 2205085749" -INSTANCE: sram_tb.dut.node.memArb -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured." -Toggle io_in_0_bits_addr [5:0] "net io_in_0_bits_addr[43:0]" -Toggle io_in_1_bits_addr [5:0] "net io_in_1_bits_addr[43:0]" -Toggle io_in_2_bits_addr [5:0] "net io_in_2_bits_addr[43:0]" -Toggle io_in_3_bits_addr [5:0] "net io_in_3_bits_addr[43:0]" -Toggle io_in_4_bits_addr [5:0] "net io_in_4_bits_addr[43:0]" -Toggle io_in_5_bits_addr [5:0] "net io_in_5_bits_addr[43:0]" -Toggle io_in_6_bits_addr [5:0] "net io_in_6_bits_addr[43:0]" -Toggle io_in_7_bits_addr [5:0] "net io_in_7_bits_addr[43:0]" -Toggle io_out_bits_id [11:3] "net io_out_bits_id[11:0]" -Toggle io_out_bits_addr [5:0] "net io_out_bits_addr[43:0]" -ANNOTATION: "MemoryNode.scala memArb input(i).id=i for i0..7: every bit of each array element is a compile-time literal, while selected output low3 ID bits remain measured. Address low6 bits inherit the accepted aligned64B request." -Toggle _GEN [7:0][11:0] "net [7:0][11:0]_GEN" -Toggle _GEN_1 [7:0][5:0] "net [7:0][43:0]_GEN_1" - -CHECKSUM: "4063230465 2261056641" -INSTANCE: sram_tb.dut.node.rspArb -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured." -Toggle io_in_0_bits_opcode [0] "net io_in_0_bits_opcode[4:0]" -Toggle io_in_0_bits_opcode [4:2] "net io_in_0_bits_opcode[4:0]" -Toggle io_in_1_bits_opcode [0] "net io_in_1_bits_opcode[4:0]" -Toggle io_in_1_bits_opcode [4:2] "net io_in_1_bits_opcode[4:0]" -Toggle io_in_2_bits_opcode [0] "net io_in_2_bits_opcode[4:0]" -Toggle io_in_2_bits_opcode [4:2] "net io_in_2_bits_opcode[4:0]" -Toggle io_in_3_bits_opcode [0] "net io_in_3_bits_opcode[4:0]" -Toggle io_in_3_bits_opcode [4:2] "net io_in_3_bits_opcode[4:0]" -Toggle io_in_4_bits_opcode [0] "net io_in_4_bits_opcode[4:0]" -Toggle io_in_4_bits_opcode [4:2] "net io_in_4_bits_opcode[4:0]" -Toggle io_in_5_bits_opcode [0] "net io_in_5_bits_opcode[4:0]" -Toggle io_in_5_bits_opcode [4:2] "net io_in_5_bits_opcode[4:0]" -Toggle io_in_6_bits_opcode [0] "net io_in_6_bits_opcode[4:0]" -Toggle io_in_6_bits_opcode [4:2] "net io_in_6_bits_opcode[4:0]" -Toggle io_in_7_bits_opcode [0] "net io_in_7_bits_opcode[4:0]" -Toggle io_in_7_bits_opcode [4:2] "net io_in_7_bits_opcode[4:0]" -Toggle io_out_bits_opcode [0] "net io_out_bits_opcode[4:0]" -Toggle io_out_bits_opcode [4:2] "net io_out_bits_opcode[4:0]" -Toggle io_out_bits_dbid [11:3] "net io_out_bits_dbid[11:0]" -ANNOTATION: "MemoryNode.scala rspArb input(i).dbid=i for i0..7: each array element is a compile-time literal. Input opcode is DBIDResp6 or Comp4, so only bit1 changes; selected output DBID low3bits remain measured." -Toggle _GEN [7:0][11:0] "net [7:0][11:0]_GEN" -Toggle _GEN_4 [7:0][0] "net [7:0][4:0]_GEN_4" -Toggle _GEN_4 [7:0][4:2] "net [7:0][4:0]_GEN_4" - -CHECKSUM: "1047165719 4281785704" -INSTANCE: sram_tb.dut.node.datArb -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured." -Toggle io_in_0_bits_dbid [15:12] "net io_in_0_bits_dbid[15:0]" -Toggle io_in_0_bits_dataId [0] "net io_in_0_bits_dataId[1:0]" -Toggle io_in_1_bits_dbid [15:12] "net io_in_1_bits_dbid[15:0]" -Toggle io_in_1_bits_dataId [0] "net io_in_1_bits_dataId[1:0]" -Toggle io_in_2_bits_dbid [15:12] "net io_in_2_bits_dbid[15:0]" -Toggle io_in_2_bits_dataId [0] "net io_in_2_bits_dataId[1:0]" -Toggle io_in_3_bits_dbid [15:12] "net io_in_3_bits_dbid[15:0]" -Toggle io_in_3_bits_dataId [0] "net io_in_3_bits_dataId[1:0]" -Toggle io_in_4_bits_dbid [15:12] "net io_in_4_bits_dbid[15:0]" -Toggle io_in_4_bits_dataId [0] "net io_in_4_bits_dataId[1:0]" -Toggle io_in_5_bits_dbid [15:12] "net io_in_5_bits_dbid[15:0]" -Toggle io_in_5_bits_dataId [0] "net io_in_5_bits_dataId[1:0]" -Toggle io_in_6_bits_dbid [15:12] "net io_in_6_bits_dbid[15:0]" -Toggle io_in_6_bits_dataId [0] "net io_in_6_bits_dataId[1:0]" -Toggle io_in_7_bits_dbid [15:12] "net io_in_7_bits_dbid[15:0]" -Toggle io_in_7_bits_dataId [0] "net io_in_7_bits_dataId[1:0]" -Toggle io_out_bits_dbid [15:12] "net io_out_bits_dbid[15:0]" -Toggle io_out_bits_dataId [0] "net io_out_bits_dataId[1:0]" -ANNOTATION: "MemoryNode.scala DAT dbid zero-extends the12-bit returnTxnId to16bits, and DataID is beat index shifted left1 at dataBits256. Only the extension and low DataID bit are excluded." -Toggle _GEN_5 [7:0][15:12] "net [7:0][15:0]_GEN_5" -Toggle _GEN_6 [7:0][0] "net [7:0][1:0]_GEN_6" - -CHECKSUM: "1542839461 3642783442" -INSTANCE: sram_tb.dut.rxReq -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured. Exact serialized field slices follow Flit.fieldsLSB; only missing transitions excluded." -Toggle 1to0 io_out_bits [10:4] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [37] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [52:51] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [69:55] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [110:108] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [118:113] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [120] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [123] "net io_out_bits[136:0]" -Toggle 1to0 io_out_bits [135:132] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [10:4] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [37] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [52:51] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [69:55] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [110:108] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [118:113] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [120] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [123] "net io_out_bits[136:0]" -Toggle 0to1 io_out_bits [135:132] "net io_out_bits[136:0]" -Toggle 1to0 storage_0 [10:4] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [37] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [52:51] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [69:55] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [110:108] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [118:113] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [120] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [123] "reg storage_0[136:0]" -Toggle 1to0 storage_0 [135:132] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [10:4] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [37] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [52:51] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [69:55] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [110:108] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [118:113] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [120] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [123] "reg storage_0[136:0]" -Toggle 0to1 storage_0 [135:132] "reg storage_0[136:0]" -Toggle 1to0 storage_1 [10:4] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [37] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [52:51] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [69:55] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [110:108] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [118:113] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [120] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [123] "reg storage_1[136:0]" -Toggle 1to0 storage_1 [135:132] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [10:4] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [37] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [52:51] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [69:55] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [110:108] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [118:113] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [120] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [123] "reg storage_1[136:0]" -Toggle 0to1 storage_1 [135:132] "reg storage_1[136:0]" -Toggle 1to0 storage_2 [10:4] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [37] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [52:51] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [69:55] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [110:108] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [118:113] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [120] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [123] "reg storage_2[136:0]" -Toggle 1to0 storage_2 [135:132] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [10:4] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [37] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [52:51] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [69:55] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [110:108] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [118:113] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [120] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [123] "reg storage_2[136:0]" -Toggle 0to1 storage_2 [135:132] "reg storage_2[136:0]" -Toggle 1to0 storage_3 [10:4] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [37] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [52:51] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [69:55] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [110:108] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [118:113] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [120] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [123] "reg storage_3[136:0]" -Toggle 1to0 storage_3 [135:132] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [10:4] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [37] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [52:51] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [69:55] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [110:108] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [118:113] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [120] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [123] "reg storage_3[136:0]" -Toggle 0to1 storage_3 [135:132] "reg storage_3[136:0]" -ANNOTATION: "Depth4 credit conservation proves advertised+occupancy<=4, so reserved[3] is always0." -Toggle reserved [3] "net reserved[3:0]" -ANNOTATION: "Rx.scala storage(writePointer) captures only on FLITV/enqueue; its output mux array is the exact concatenation of storage0..3. This precise source-field slice matches the reviewed storage slot exclusion. Invalid physical channel payload is never captured; legal stored REQ/DAT profile attributes remain fixed." -Toggle 1to0 _GEN [3:0][10:4] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][37] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][52:51] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][69:55] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][110:108] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][118:113] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][120] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][123] "net [3:0][136:0]_GEN" -Toggle 1to0 _GEN [3:0][135:132] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][10:4] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][37] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][52:51] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][69:55] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][110:108] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][118:113] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][120] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][123] "net [3:0][136:0]_GEN" -Toggle 0to1 _GEN [3:0][135:132] "net [3:0][136:0]_GEN" -ANNOTATION: "Arm IHI0050H: B13.9.1 LPID uses low5bits with upper3 MBZ; C1.13 NonCopyBackWriteData Resp=0 and CBusy=0a; C1.15 HomeNID/DataSource=0a; C1.1 defines0a MUST zero (not recommended). B13.9.4 DBID upper4 padding MBZ, while low12 X remains measured. B13.10.28 CAH is zero for noncopyback writes. B2.9 CCID=original Addr[5:4]=0 for this64B aligned profile. Only legally sampled FIFO storage and internal copies are excluded; invalid physical channel payload remains measured." -Toggle io_out_bits [131:129] "net io_out_bits[136:0]" -Toggle storage_0 [131:129] "reg storage_0[136:0]" -Toggle storage_1 [131:129] "reg storage_1[136:0]" -Toggle storage_2 [131:129] "reg storage_2[136:0]" -Toggle storage_3 [131:129] "reg storage_3[136:0]" -Toggle _GEN [3:0][131:129] "net [3:0][136:0]_GEN" - -CHECKSUM: "1028417357 3487977871" -INSTANCE: sram_tb.dut.txRsp -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured. Exact serialized field slices follow Flit.fieldsLSB; only missing transitions excluded." -Toggle 1to0 io_in_bits [17:11] "net io_in_bits[70:0]" -Toggle 1to0 io_in_bits [30] "net io_in_bits[70:0]" -Toggle 1to0 io_in_bits [34:32] "net io_in_bits[70:0]" -Toggle 1to0 io_in_bits [45:37] "net io_in_bits[70:0]" -Toggle 1to0 io_in_bits [63:49] "net io_in_bits[70:0]" -Toggle 1to0 io_in_bits [70:65] "net io_in_bits[70:0]" -Toggle 0to1 io_in_bits [17:11] "net io_in_bits[70:0]" -Toggle 0to1 io_in_bits [30] "net io_in_bits[70:0]" -Toggle 0to1 io_in_bits [34:32] "net io_in_bits[70:0]" -Toggle 0to1 io_in_bits [45:37] "net io_in_bits[70:0]" -Toggle 0to1 io_in_bits [63:49] "net io_in_bits[70:0]" -Toggle 0to1 io_in_bits [70:65] "net io_in_bits[70:0]" -Toggle 1to0 io_link_flit [17:12] "net io_link_flit[70:0]" -Toggle 1to0 io_link_flit [30] "net io_link_flit[70:0]" -Toggle 1to0 io_link_flit [34:33] "net io_link_flit[70:0]" -Toggle 1to0 io_link_flit [45:37] "net io_link_flit[70:0]" -Toggle 1to0 io_link_flit [63:49] "net io_link_flit[70:0]" -Toggle 1to0 io_link_flit [70:65] "net io_link_flit[70:0]" -Toggle 0to1 io_link_flit [17:12] "net io_link_flit[70:0]" -Toggle 0to1 io_link_flit [30] "net io_link_flit[70:0]" -Toggle 0to1 io_link_flit [34:33] "net io_link_flit[70:0]" -Toggle 0to1 io_link_flit [45:37] "net io_link_flit[70:0]" -Toggle 0to1 io_link_flit [63:49] "net io_link_flit[70:0]" -Toggle 0to1 io_link_flit [70:65] "net io_link_flit[70:0]" -Toggle 1to0 io_link_flit_r [17:12] "reg io_link_flit_r[70:0]" -Toggle 1to0 io_link_flit_r [30] "reg io_link_flit_r[70:0]" -Toggle 1to0 io_link_flit_r [34:33] "reg io_link_flit_r[70:0]" -Toggle 1to0 io_link_flit_r [45:37] "reg io_link_flit_r[70:0]" -Toggle 1to0 io_link_flit_r [63:49] "reg io_link_flit_r[70:0]" -Toggle 1to0 io_link_flit_r [70:65] "reg io_link_flit_r[70:0]" -Toggle 0to1 io_link_flit_r [17:12] "reg io_link_flit_r[70:0]" -Toggle 0to1 io_link_flit_r [30] "reg io_link_flit_r[70:0]" -Toggle 0to1 io_link_flit_r [34:33] "reg io_link_flit_r[70:0]" -Toggle 0to1 io_link_flit_r [45:37] "reg io_link_flit_r[70:0]" -Toggle 0to1 io_link_flit_r [63:49] "reg io_link_flit_r[70:0]" -Toggle 0to1 io_link_flit_r [70:65] "reg io_link_flit_r[70:0]" - -CHECKSUM: "3712248894 2138691777" -INSTANCE: sram_tb.dut.rxDat -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured. Exact serialized field slices follow Flit.fieldsLSB; only missing transitions excluded." -Toggle 1to0 io_out_bits [10:4] "net io_out_bits[388:0]" -Toggle 1to0 io_out_bits [29:21] "net io_out_bits[388:0]" -Toggle 1to0 io_out_bits [42:37] "net io_out_bits[388:0]" -Toggle 1to0 io_out_bits [54] "net io_out_bits[388:0]" -Toggle 1to0 io_out_bits [76] "net io_out_bits[388:0]" -Toggle 1to0 io_out_bits [95:84] "net io_out_bits[388:0]" -Toggle 1to0 io_out_bits [100:98] "net io_out_bits[388:0]" -Toggle 0to1 io_out_bits [10:4] "net io_out_bits[388:0]" -Toggle 0to1 io_out_bits [29:21] "net io_out_bits[388:0]" -Toggle 0to1 io_out_bits [42:37] "net io_out_bits[388:0]" -Toggle 0to1 io_out_bits [54] "net io_out_bits[388:0]" -Toggle 0to1 io_out_bits [76] "net io_out_bits[388:0]" -Toggle 0to1 io_out_bits [95:84] "net io_out_bits[388:0]" -Toggle 0to1 io_out_bits [100:98] "net io_out_bits[388:0]" -Toggle 1to0 storage_0 [10:4] "reg storage_0[388:0]" -Toggle 1to0 storage_0 [29:21] "reg storage_0[388:0]" -Toggle 1to0 storage_0 [42:37] "reg storage_0[388:0]" -Toggle 1to0 storage_0 [54] "reg storage_0[388:0]" -Toggle 1to0 storage_0 [76] "reg storage_0[388:0]" -Toggle 1to0 storage_0 [95:84] "reg storage_0[388:0]" -Toggle 1to0 storage_0 [100:98] "reg storage_0[388:0]" -Toggle 0to1 storage_0 [10:4] "reg storage_0[388:0]" -Toggle 0to1 storage_0 [29:21] "reg storage_0[388:0]" -Toggle 0to1 storage_0 [42:37] "reg storage_0[388:0]" -Toggle 0to1 storage_0 [54] "reg storage_0[388:0]" -Toggle 0to1 storage_0 [76] "reg storage_0[388:0]" -Toggle 0to1 storage_0 [95:84] "reg storage_0[388:0]" -Toggle 0to1 storage_0 [100:98] "reg storage_0[388:0]" -Toggle 1to0 storage_1 [10:4] "reg storage_1[388:0]" -Toggle 1to0 storage_1 [29:21] "reg storage_1[388:0]" -Toggle 1to0 storage_1 [42:37] "reg storage_1[388:0]" -Toggle 1to0 storage_1 [54] "reg storage_1[388:0]" -Toggle 1to0 storage_1 [76] "reg storage_1[388:0]" -Toggle 1to0 storage_1 [95:84] "reg storage_1[388:0]" -Toggle 1to0 storage_1 [100:98] "reg storage_1[388:0]" -Toggle 0to1 storage_1 [10:4] "reg storage_1[388:0]" -Toggle 0to1 storage_1 [29:21] "reg storage_1[388:0]" -Toggle 0to1 storage_1 [42:37] "reg storage_1[388:0]" -Toggle 0to1 storage_1 [54] "reg storage_1[388:0]" -Toggle 0to1 storage_1 [76] "reg storage_1[388:0]" -Toggle 0to1 storage_1 [95:84] "reg storage_1[388:0]" -Toggle 0to1 storage_1 [100:98] "reg storage_1[388:0]" -Toggle 1to0 storage_2 [10:4] "reg storage_2[388:0]" -Toggle 1to0 storage_2 [29:21] "reg storage_2[388:0]" -Toggle 1to0 storage_2 [42:37] "reg storage_2[388:0]" -Toggle 1to0 storage_2 [54] "reg storage_2[388:0]" -Toggle 1to0 storage_2 [76] "reg storage_2[388:0]" -Toggle 1to0 storage_2 [95:84] "reg storage_2[388:0]" -Toggle 1to0 storage_2 [100:98] "reg storage_2[388:0]" -Toggle 0to1 storage_2 [10:4] "reg storage_2[388:0]" -Toggle 0to1 storage_2 [29:21] "reg storage_2[388:0]" -Toggle 0to1 storage_2 [42:37] "reg storage_2[388:0]" -Toggle 0to1 storage_2 [54] "reg storage_2[388:0]" -Toggle 0to1 storage_2 [76] "reg storage_2[388:0]" -Toggle 0to1 storage_2 [95:84] "reg storage_2[388:0]" -Toggle 0to1 storage_2 [100:98] "reg storage_2[388:0]" -Toggle 1to0 storage_3 [10:4] "reg storage_3[388:0]" -Toggle 1to0 storage_3 [29:21] "reg storage_3[388:0]" -Toggle 1to0 storage_3 [42:37] "reg storage_3[388:0]" -Toggle 1to0 storage_3 [54] "reg storage_3[388:0]" -Toggle 1to0 storage_3 [76] "reg storage_3[388:0]" -Toggle 1to0 storage_3 [95:84] "reg storage_3[388:0]" -Toggle 1to0 storage_3 [100:98] "reg storage_3[388:0]" -Toggle 0to1 storage_3 [10:4] "reg storage_3[388:0]" -Toggle 0to1 storage_3 [29:21] "reg storage_3[388:0]" -Toggle 0to1 storage_3 [42:37] "reg storage_3[388:0]" -Toggle 0to1 storage_3 [54] "reg storage_3[388:0]" -Toggle 0to1 storage_3 [76] "reg storage_3[388:0]" -Toggle 0to1 storage_3 [95:84] "reg storage_3[388:0]" -Toggle 0to1 storage_3 [100:98] "reg storage_3[388:0]" -ANNOTATION: "MemoryNode.io.rxDat.ready is tied true, so every occupied entry drains next cycle; at most one incoming flit per cycle gives occupancy<=1 from reset." -Toggle occupancy [2:1] "reg occupancy[2:0]" -ANNOTATION: "Depth4 credit conservation proves advertised+occupancy<=4, so reserved[3] is always0." -Toggle reserved [3] "net reserved[3:0]" -ANNOTATION: "Rx.scala storage(writePointer) captures only on FLITV/enqueue; its output mux array is the exact concatenation of storage0..3. This precise source-field slice matches the reviewed storage slot exclusion. Invalid physical channel payload is never captured; legal stored REQ/DAT profile attributes remain fixed." -Toggle 1to0 _GEN [3:0][10:4] "net [3:0][388:0]_GEN" -Toggle 1to0 _GEN [3:0][29:21] "net [3:0][388:0]_GEN" -Toggle 1to0 _GEN [3:0][42:37] "net [3:0][388:0]_GEN" -Toggle 1to0 _GEN [3:0][54] "net [3:0][388:0]_GEN" -Toggle 1to0 _GEN [3:0][76] "net [3:0][388:0]_GEN" -Toggle 1to0 _GEN [3:0][95:84] "net [3:0][388:0]_GEN" -Toggle 1to0 _GEN [3:0][100:98] "net [3:0][388:0]_GEN" -Toggle 0to1 _GEN [3:0][10:4] "net [3:0][388:0]_GEN" -Toggle 0to1 _GEN [3:0][29:21] "net [3:0][388:0]_GEN" -Toggle 0to1 _GEN [3:0][42:37] "net [3:0][388:0]_GEN" -Toggle 0to1 _GEN [3:0][54] "net [3:0][388:0]_GEN" -Toggle 0to1 _GEN [3:0][76] "net [3:0][388:0]_GEN" -Toggle 0to1 _GEN [3:0][95:84] "net [3:0][388:0]_GEN" -Toggle 0to1 _GEN [3:0][100:98] "net [3:0][388:0]_GEN" -ANNOTATION: "MemoryNode rxDat.ready=true implies occupancy<=1 and therefore enqueueReady=true for depth4." -Toggle enqueueReady "net enqueueReady" -ANNOTATION: "Arm IHI0050H: B13.9.1 LPID uses low5bits with upper3 MBZ; C1.13 NonCopyBackWriteData Resp=0 and CBusy=0a; C1.15 HomeNID/DataSource=0a; C1.1 defines0a MUST zero (not recommended). B13.9.4 DBID upper4 padding MBZ, while low12 X remains measured. B13.10.28 CAH is zero for noncopyback writes. B2.9 CCID=original Addr[5:4]=0 for this64B aligned profile. Only legally sampled FIFO storage and internal copies are excluded; invalid physical channel payload remains measured." -Toggle io_out_bits [36:30] "net io_out_bits[388:0]" -Toggle io_out_bits [53:43] "net io_out_bits[388:0]" -Toggle io_out_bits [57:55] "net io_out_bits[388:0]" -Toggle io_out_bits [75:70] "net io_out_bits[388:0]" -Toggle io_out_bits [97] "net io_out_bits[388:0]" -Toggle storage_0 [36:30] "reg storage_0[388:0]" -Toggle storage_0 [53:43] "reg storage_0[388:0]" -Toggle storage_0 [57:55] "reg storage_0[388:0]" -Toggle storage_0 [75:70] "reg storage_0[388:0]" -Toggle storage_0 [97] "reg storage_0[388:0]" -Toggle storage_1 [36:30] "reg storage_1[388:0]" -Toggle storage_1 [53:43] "reg storage_1[388:0]" -Toggle storage_1 [57:55] "reg storage_1[388:0]" -Toggle storage_1 [75:70] "reg storage_1[388:0]" -Toggle storage_1 [97] "reg storage_1[388:0]" -Toggle storage_2 [36:30] "reg storage_2[388:0]" -Toggle storage_2 [53:43] "reg storage_2[388:0]" -Toggle storage_2 [57:55] "reg storage_2[388:0]" -Toggle storage_2 [75:70] "reg storage_2[388:0]" -Toggle storage_2 [97] "reg storage_2[388:0]" -Toggle storage_3 [36:30] "reg storage_3[388:0]" -Toggle storage_3 [53:43] "reg storage_3[388:0]" -Toggle storage_3 [57:55] "reg storage_3[388:0]" -Toggle storage_3 [75:70] "reg storage_3[388:0]" -Toggle storage_3 [97] "reg storage_3[388:0]" -Toggle _GEN [3:0][36:30] "net [3:0][388:0]_GEN" -Toggle _GEN [3:0][53:43] "net [3:0][388:0]_GEN" -Toggle _GEN [3:0][57:55] "net [3:0][388:0]_GEN" -Toggle _GEN [3:0][75:70] "net [3:0][388:0]_GEN" -Toggle _GEN [3:0][97] "net [3:0][388:0]_GEN" - -CHECKSUM: "2770640837 504484466" -INSTANCE: sram_tb.dut.txDat -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured. Exact serialized field slices follow Flit.fieldsLSB; only missing transitions excluded." -Toggle 1to0 io_in_bits [17:11] "net io_in_bits[388:0]" -Toggle 1to0 io_in_bits [40:37] "net io_in_bits[388:0]" -Toggle 1to0 io_in_bits [57:43] "net io_in_bits[388:0]" -Toggle 1to0 io_in_bits [76:70] "net io_in_bits[388:0]" -Toggle 1to0 io_in_bits [95:78] "net io_in_bits[388:0]" -Toggle 1to0 io_in_bits [132:97] "net io_in_bits[388:0]" -Toggle 0to1 io_in_bits [17:11] "net io_in_bits[388:0]" -Toggle 0to1 io_in_bits [40:37] "net io_in_bits[388:0]" -Toggle 0to1 io_in_bits [57:43] "net io_in_bits[388:0]" -Toggle 0to1 io_in_bits [76:70] "net io_in_bits[388:0]" -Toggle 0to1 io_in_bits [95:78] "net io_in_bits[388:0]" -Toggle 0to1 io_in_bits [132:97] "net io_in_bits[388:0]" -Toggle 1to0 io_link_flit [17:12] "net io_link_flit[388:0]" -Toggle 1to0 io_link_flit [38:37] "net io_link_flit[388:0]" -Toggle 1to0 io_link_flit [40] "net io_link_flit[388:0]" -Toggle 1to0 io_link_flit [57:43] "net io_link_flit[388:0]" -Toggle 1to0 io_link_flit [76:70] "net io_link_flit[388:0]" -Toggle 1to0 io_link_flit [95:78] "net io_link_flit[388:0]" -Toggle 1to0 io_link_flit [132:97] "net io_link_flit[388:0]" -Toggle 0to1 io_link_flit [17:12] "net io_link_flit[388:0]" -Toggle 0to1 io_link_flit [38:37] "net io_link_flit[388:0]" -Toggle 0to1 io_link_flit [40] "net io_link_flit[388:0]" -Toggle 0to1 io_link_flit [57:43] "net io_link_flit[388:0]" -Toggle 0to1 io_link_flit [76:70] "net io_link_flit[388:0]" -Toggle 0to1 io_link_flit [95:78] "net io_link_flit[388:0]" -Toggle 0to1 io_link_flit [132:97] "net io_link_flit[388:0]" -Toggle 1to0 io_link_flit_r [17:12] "reg io_link_flit_r[388:0]" -Toggle 1to0 io_link_flit_r [38:37] "reg io_link_flit_r[388:0]" -Toggle 1to0 io_link_flit_r [40] "reg io_link_flit_r[388:0]" -Toggle 1to0 io_link_flit_r [57:43] "reg io_link_flit_r[388:0]" -Toggle 1to0 io_link_flit_r [76:70] "reg io_link_flit_r[388:0]" -Toggle 1to0 io_link_flit_r [95:78] "reg io_link_flit_r[388:0]" -Toggle 1to0 io_link_flit_r [132:97] "reg io_link_flit_r[388:0]" -Toggle 0to1 io_link_flit_r [17:12] "reg io_link_flit_r[388:0]" -Toggle 0to1 io_link_flit_r [38:37] "reg io_link_flit_r[388:0]" -Toggle 0to1 io_link_flit_r [40] "reg io_link_flit_r[388:0]" -Toggle 0to1 io_link_flit_r [57:43] "reg io_link_flit_r[388:0]" -Toggle 0to1 io_link_flit_r [76:70] "reg io_link_flit_r[388:0]" -Toggle 0to1 io_link_flit_r [95:78] "reg io_link_flit_r[388:0]" -Toggle 0to1 io_link_flit_r [132:97] "reg io_link_flit_r[388:0]" - -CHECKSUM: "2383780593 1281742771" -INSTANCE: sram_tb.dut -ANNOTATION: "MemoryNode.scala source assignments and asserted SN profile: NodeID1; eight local DBIDs; aligned64B ReadNoSnp/WriteNoSnpFull/Ptl; no ordering, exclusive, snoop, tags, PAS or retry credit; incoming write DAT opcode3, clean error, local slot TxnID0..7; outgoing Comp/DBIDResp and CompData with zero optional attributes. Read BE is inapplicable (IssueH B13.9) and emitted zero. RespErr is OK0/NDERR3: neither error bit is excluded. Arbitrary routing, source, return and request TxnIDs and all payload remain measured. Exact serialized field slices follow Flit.fieldsLSB; only missing transitions excluded." -Toggle 1to0 io_chi_txRsp_flit [17:12] "net io_chi_txRsp_flit[70:0]" -Toggle 1to0 io_chi_txRsp_flit [30] "net io_chi_txRsp_flit[70:0]" -Toggle 1to0 io_chi_txRsp_flit [34:33] "net io_chi_txRsp_flit[70:0]" -Toggle 1to0 io_chi_txRsp_flit [45:37] "net io_chi_txRsp_flit[70:0]" -Toggle 1to0 io_chi_txRsp_flit [63:49] "net io_chi_txRsp_flit[70:0]" -Toggle 1to0 io_chi_txRsp_flit [70:65] "net io_chi_txRsp_flit[70:0]" -Toggle 0to1 io_chi_txRsp_flit [17:12] "net io_chi_txRsp_flit[70:0]" -Toggle 0to1 io_chi_txRsp_flit [30] "net io_chi_txRsp_flit[70:0]" -Toggle 0to1 io_chi_txRsp_flit [34:33] "net io_chi_txRsp_flit[70:0]" -Toggle 0to1 io_chi_txRsp_flit [45:37] "net io_chi_txRsp_flit[70:0]" -Toggle 0to1 io_chi_txRsp_flit [63:49] "net io_chi_txRsp_flit[70:0]" -Toggle 0to1 io_chi_txRsp_flit [70:65] "net io_chi_txRsp_flit[70:0]" -Toggle 1to0 io_chi_txDat_flit [17:12] "net io_chi_txDat_flit[388:0]" -Toggle 1to0 io_chi_txDat_flit [38:37] "net io_chi_txDat_flit[388:0]" -Toggle 1to0 io_chi_txDat_flit [40] "net io_chi_txDat_flit[388:0]" -Toggle 1to0 io_chi_txDat_flit [57:43] "net io_chi_txDat_flit[388:0]" -Toggle 1to0 io_chi_txDat_flit [76:70] "net io_chi_txDat_flit[388:0]" -Toggle 1to0 io_chi_txDat_flit [95:78] "net io_chi_txDat_flit[388:0]" -Toggle 1to0 io_chi_txDat_flit [132:97] "net io_chi_txDat_flit[388:0]" -Toggle 0to1 io_chi_txDat_flit [17:12] "net io_chi_txDat_flit[388:0]" -Toggle 0to1 io_chi_txDat_flit [38:37] "net io_chi_txDat_flit[388:0]" -Toggle 0to1 io_chi_txDat_flit [40] "net io_chi_txDat_flit[388:0]" -Toggle 0to1 io_chi_txDat_flit [57:43] "net io_chi_txDat_flit[388:0]" -Toggle 0to1 io_chi_txDat_flit [76:70] "net io_chi_txDat_flit[388:0]" -Toggle 0to1 io_chi_txDat_flit [95:78] "net io_chi_txDat_flit[388:0]" -Toggle 0to1 io_chi_txDat_flit [132:97] "net io_chi_txDat_flit[388:0]" -Toggle 1to0 _rxDat_io_out_bits [10:4] "net _rxDat_io_out_bits[388:0]" -Toggle 1to0 _rxDat_io_out_bits [29:21] "net _rxDat_io_out_bits[388:0]" -Toggle 1to0 _rxDat_io_out_bits [42:37] "net _rxDat_io_out_bits[388:0]" -Toggle 1to0 _rxDat_io_out_bits [54] "net _rxDat_io_out_bits[388:0]" -Toggle 1to0 _rxDat_io_out_bits [76] "net _rxDat_io_out_bits[388:0]" -Toggle 1to0 _rxDat_io_out_bits [95:84] "net _rxDat_io_out_bits[388:0]" -Toggle 1to0 _rxDat_io_out_bits [100:98] "net _rxDat_io_out_bits[388:0]" -Toggle 0to1 _rxDat_io_out_bits [10:4] "net _rxDat_io_out_bits[388:0]" -Toggle 0to1 _rxDat_io_out_bits [29:21] "net _rxDat_io_out_bits[388:0]" -Toggle 0to1 _rxDat_io_out_bits [42:37] "net _rxDat_io_out_bits[388:0]" -Toggle 0to1 _rxDat_io_out_bits [54] "net _rxDat_io_out_bits[388:0]" -Toggle 0to1 _rxDat_io_out_bits [76] "net _rxDat_io_out_bits[388:0]" -Toggle 0to1 _rxDat_io_out_bits [95:84] "net _rxDat_io_out_bits[388:0]" -Toggle 0to1 _rxDat_io_out_bits [100:98] "net _rxDat_io_out_bits[388:0]" -Toggle 1to0 _rxReq_io_out_bits [10:4] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [37] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [52:51] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [69:55] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [110:108] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [118:113] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [120] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [123] "net _rxReq_io_out_bits[136:0]" -Toggle 1to0 _rxReq_io_out_bits [135:132] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [10:4] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [37] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [52:51] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [69:55] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [110:108] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [118:113] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [120] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [123] "net _rxReq_io_out_bits[136:0]" -Toggle 0to1 _rxReq_io_out_bits [135:132] "net _rxReq_io_out_bits[136:0]" -ANNOTATION: "SramEndpoint direct child connections preserve source field invariants: local memory/response IDs0..7, RSP opcode4/6, DAT dbid zero-extension12to16, DataID0/2, backend64B address alignment. Only the listed constant bits are excluded." -Toggle _memory_io_resp_bits_id [11:3] "net _memory_io_resp_bits_id[11:0]" -Toggle _node_io_rsp_bits_opcode [0] "net _node_io_rsp_bits_opcode[4:0]" -Toggle _node_io_rsp_bits_opcode [4:2] "net _node_io_rsp_bits_opcode[4:0]" -Toggle _node_io_rsp_bits_dbid [11:3] "net _node_io_rsp_bits_dbid[11:0]" -Toggle _node_io_txDat_bits_dbid [15:12] "net _node_io_txDat_bits_dbid[15:0]" -Toggle _node_io_txDat_bits_dataId [0] "net _node_io_txDat_bits_dataId[1:0]" -Toggle _node_io_memoryReq_bits_id [11:3] "net _node_io_memoryReq_bits_id[11:0]" -Toggle _node_io_memoryReq_bits_addr [5:0] "net _node_io_memoryReq_bits_addr[43:0]" -ANNOTATION: "Arm IHI0050H: B13.9.1 LPID uses low5bits with upper3 MBZ; C1.13 NonCopyBackWriteData Resp=0 and CBusy=0a; C1.15 HomeNID/DataSource=0a; C1.1 defines0a MUST zero (not recommended). B13.9.4 DBID upper4 padding MBZ, while low12 X remains measured. B13.10.28 CAH is zero for noncopyback writes. B2.9 CCID=original Addr[5:4]=0 for this64B aligned profile. Only legally sampled FIFO storage and internal copies are excluded; invalid physical channel payload remains measured." -Toggle _rxDat_io_out_bits [36:30] "net _rxDat_io_out_bits[388:0]" -Toggle _rxDat_io_out_bits [53:43] "net _rxDat_io_out_bits[388:0]" -Toggle _rxDat_io_out_bits [57:55] "net _rxDat_io_out_bits[388:0]" -Toggle _rxDat_io_out_bits [75:70] "net _rxDat_io_out_bits[388:0]" -Toggle _rxDat_io_out_bits [97] "net _rxDat_io_out_bits[388:0]" -Toggle _rxReq_io_out_bits [131:129] "net _rxReq_io_out_bits[136:0]" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram.f deleted file mode 100644 index b9a0f6e99..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/sram_interface.sv -@RESOURCES@/sram_package.sv --F @RTL@/SramEndpoint256/filelist.f -@RESOURCES@/sram_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram128.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram128.f deleted file mode 100644 index 641720ca0..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram128.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+CHI_DATA_BITS=128 -+define+CHI_SRAM_DUT=SramEndpoint128 -+define+CHI_SRAM_TB=sram128_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/sram_interface.sv -@RESOURCES@/sram_package.sv --F @RTL@/SramEndpoint128/filelist.f -@RESOURCES@/sram_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram512.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram512.f deleted file mode 100644 index c7dca7651..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram512.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+CHI_DATA_BITS=512 -+define+CHI_SRAM_DUT=SramEndpoint512 -+define+CHI_SRAM_TB=sram512_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/sram_interface.sv -@RESOURCES@/sram_package.sv --F @RTL@/SramEndpoint512/filelist.f -@RESOURCES@/sram_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_bad_opcode.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_bad_opcode.f deleted file mode 100644 index af75c4f42..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_bad_opcode.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+CHI_BAD_OPCODE -+define+CHI_SRAM_TB=sram_bad_opcode_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/sram_interface.sv -@RESOURCES@/sram_package.sv --F @RTL@/SramEndpoint256/filelist.f -@RESOURCES@/sram_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_bad_opcode.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_bad_opcode.toml deleted file mode 100644 index dd6e02ff9..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_bad_opcode.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Unsupported CHI request opcode" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_duplicate_data.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_duplicate_data.f deleted file mode 100644 index 654192d64..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_duplicate_data.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+CHI_DUPLICATE_DATA -+define+CHI_SRAM_TB=sram_duplicate_data_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/sram_interface.sv -@RESOURCES@/sram_package.sv --F @RTL@/SramEndpoint256/filelist.f -@RESOURCES@/sram_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_duplicate_data.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_duplicate_data.toml deleted file mode 100644 index a4faa37ef..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_duplicate_data.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Duplicate CHI DataID" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_interface.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_interface.sv deleted file mode 100644 index 6fb9a0df9..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_interface.sv +++ /dev/null @@ -1,16 +0,0 @@ -`include "chi_profile.svh" -interface chi_sram_if ( - input logic clock -); - logic reset; - logic req_pend, req_valid, req_credit; - logic [136:0] req_flit; - logic dat_pend, dat_valid, dat_credit; - logic [`CHI_DAT_BITS-1:0] dat_flit; - logic rsp_pend, rsp_valid, rsp_credit; - logic [70:0] rsp_flit; - logic out_dat_pend, out_dat_valid, out_dat_credit; - logic [`CHI_DAT_BITS-1:0] out_dat_flit; - logic rx_active_req, rx_active_ack, tx_active_req, tx_active_ack; - logic [3:0] outstanding; -endinterface diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_package.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_package.sv deleted file mode 100644 index 7ccad3f88..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_package.sv +++ /dev/null @@ -1,20 +0,0 @@ -package chi_sram_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `include "flit_types.svh" - import "DPI-C" function chandle chi_ref_create(input int unsigned lines); - import "DPI-C" function void chi_ref_destroy(input chandle model); - import "DPI-C" function byte unsigned chi_ref_write( - input chandle model, - input longint unsigned addr, - input bit [511:0] data, - input bit [63:0] mask - ); - import "DPI-C" function byte unsigned chi_ref_read( - input chandle model, - input longint unsigned addr, - output bit [511:0] data - ); - `include "sram_test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_tb.sv b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_tb.sv deleted file mode 100644 index 2526b29b8..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_tb.sv +++ /dev/null @@ -1,44 +0,0 @@ -`ifndef CHI_SRAM_TB -`define CHI_SRAM_TB sram_tb -`endif -`ifndef CHI_SRAM_DUT -`define CHI_SRAM_DUT SramEndpoint256 -`endif -module `CHI_SRAM_TB; - import uvm_pkg::*; - import chi_sram_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_sram_if source_if (clock); - `CHI_SRAM_DUT dut ( - .clock(clock), - .reset(source_if.reset), - .io_chi_rxReq_flitpend(source_if.req_pend), - .io_chi_rxReq_flitv(source_if.req_valid), - .io_chi_rxReq_flit(source_if.req_flit), - .io_chi_rxReq_lcrdv(source_if.req_credit), - .io_chi_rxDat_flitpend(source_if.dat_pend), - .io_chi_rxDat_flitv(source_if.dat_valid), - .io_chi_rxDat_flit(source_if.dat_flit), - .io_chi_rxDat_lcrdv(source_if.dat_credit), - .io_chi_txRsp_flitpend(source_if.rsp_pend), - .io_chi_txRsp_flitv(source_if.rsp_valid), - .io_chi_txRsp_flit(source_if.rsp_flit), - .io_chi_txRsp_lcrdv(source_if.rsp_credit), - .io_chi_txDat_flitpend(source_if.out_dat_pend), - .io_chi_txDat_flitv(source_if.out_dat_valid), - .io_chi_txDat_flit(source_if.out_dat_flit), - .io_chi_txDat_lcrdv(source_if.out_dat_credit), - .io_chi_rxLinkActiveReq(source_if.rx_active_req), - .io_chi_rxLinkActiveAck(source_if.rx_active_ack), - .io_chi_txLinkActiveReq(source_if.tx_active_req), - .io_chi_txLinkActiveAck(source_if.tx_active_ack), - .io_chi_rxSActive(source_if.rx_active_req), - .io_chi_txSActive(), - .io_outstanding(source_if.outstanding) - ); - initial begin - uvm_config_db#(virtual chi_sram_if)::set(null, "uvm_test_top", "vif", source_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_test.svh b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_test.svh deleted file mode 100644 index d364ca91b..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/sram_test.svh +++ /dev/null @@ -1,499 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual chi_sram_if vif; - chandle model; - int req_credits = 0, dat_credits = 0, rsp_granted = 0, out_dat_granted = 0; - int cycle = 0, checks = 0; - localparam int STALL_READS = 15 / `CHI_BEATS + 8; - bit grant_enabled = 1; - rsp_t responses[$]; - dat_t data_responses[$]; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual chi_sram_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "CHI interface missing") - if ($bits(req_t) != 137 || $bits(rsp_t) != 71 || $bits(dat_t) != `CHI_DAT_BITS) - `uvm_fatal("FLIT", "Verification field packing does not match declared profile") - timeout = 300us; - model = chi_ref_create(256); - endfunction - task monitor(); - forever begin - @(posedge vif.clock); - if (vif.reset) begin - req_credits = 0; - dat_credits = 0; - rsp_granted = 0; - out_dat_granted = 0; - responses.delete(); - data_responses.delete(); - end else begin - cycle++; - req_credits += int'(vif.req_credit) - int'(vif.req_valid); - dat_credits += int'(vif.dat_credit) - int'(vif.dat_valid); - rsp_granted += int'(vif.rsp_credit) - int'(vif.rsp_valid); - out_dat_granted += int'(vif.out_dat_credit) - int'(vif.out_dat_valid); - if (req_credits < 0 || dat_credits < 0 || rsp_granted < 0 || out_dat_granted < 0) - `uvm_fatal("CREDIT", "Flit without credit") - if (vif.rsp_valid) responses.push_back(rsp_t'(vif.rsp_flit)); - if (vif.out_dat_valid) data_responses.push_back(dat_t'(vif.out_dat_flit)); - end - @(negedge vif.clock); - vif.rsp_credit = !vif.reset && vif.tx_active_req && vif.tx_active_ack && grant_enabled && rsp_granted < 15 && cycle % 5 != 0; - vif.out_dat_credit = !vif.reset && vif.tx_active_req && vif.tx_active_ack && grant_enabled && out_dat_granted < 15 && cycle % 7 < 3; - end - endtask - task reset_link(); - @(negedge vif.clock); - vif.reset = 1; - vif.req_pend = 0; - vif.dat_pend = 0; - vif.req_valid = 0; - vif.dat_valid = 0; - // Peer levels may settle after the coordinated reset edge; FLITV is already low. - repeat (2) @(negedge vif.clock); - vif.rx_active_req = 0; - vif.tx_active_ack = 0; - repeat (2) @(negedge vif.clock); - vif.reset = 0; - repeat (3) @(negedge vif.clock); - vif.rx_active_req = 1; - while (!vif.tx_active_req) @(negedge vif.clock); - repeat (3) @(negedge vif.clock); - vif.tx_active_ack = 1; - vif.req_pend = 1; - vif.dat_pend = 1; - repeat (4) @(negedge vif.clock); - if (!vif.rx_active_ack || !vif.tx_active_req) `uvm_fatal("LINK", "Activation did not complete") - endtask - task send_req(req_t r); - @(negedge vif.clock); - while (req_credits == 0) @(negedge vif.clock); - vif.req_flit = r; - vif.req_valid = 1; - @(negedge vif.clock); - vif.req_valid = 0; - endtask - task send_dat(dat_t d); - @(negedge vif.clock); - while (dat_credits == 0) @(negedge vif.clock); - vif.dat_flit = d; - vif.dat_valid = 1; - @(negedge vif.clock); - vif.dat_valid = 0; - endtask - function req_t request(int opcode, int txn, longint unsigned addr, int source = 3); - req_t r = '0; - r.opcode = opcode; - r.txn = txn; - r.addr = addr; - r.size = 6; - r.src = (txn ^ source) & 127; - r.tgt = 1; - r.return_nid = (txn ^ 85) & 127; - r.return_txn = txn + 100; - r.allow_retry = txn % 2; - r.mem_attr = (txn % 3 == 0 ? 0 : txn % 3 == 1 ? 4 : 12) | (txn % 2); - r.likely_shared = txn % 2; - r.lpid = txn % 32; - r.qos = txn % 16; - r.trace_tag = txn % 2; - return r; - endfunction - task response(req_t r, int opcode, output rsp_t out_rsp); - bit found = 0; - while (!found) begin - @(negedge vif.clock); - foreach (responses[i]) begin - if (responses[i].txn == r.txn && responses[i].tgt == r.src) begin - out_rsp = responses[i]; - responses.delete(i); - found = 1; - break; - end - end - end - if (out_rsp.opcode != opcode || out_rsp.src != 1 || out_rsp.qos != r.qos || - out_rsp.trace_tag != r.trace_tag) - `uvm_fatal("RESPONSE", $sformatf("Bad response txn=%0d opcode=%0h", r.txn, out_rsp.opcode)) - checks++; - endtask - function dat_t write_beat(req_t r, rsp_t dbid_rsp, int beat, bit [`CHI_DATA_BITS-1:0] data, - bit [`CHI_BE_BITS-1:0] mask = '1); - dat_t d = '0; - d.tgt = 1; - d.src = r.src; - d.txn = dbid_rsp.dbid; - d.qos = r.qos; - d.dbid = {4'b0, r.txn}; - d.line_id = r.addr[11:6]; - d.trace_tag = r.trace_tag; - d.opcode = 3; - d.data_id = beat * `CHI_DATA_ID_STEP; - d.data = data; - d.be = mask; - return d; - endfunction - task write_data(req_t r, rsp_t dbid_rsp, bit [511:0] data, bit [63:0] mask); - for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) - send_dat(write_beat( - r, - dbid_rsp, - beat, - data[beat*`CHI_DATA_BITS+:`CHI_DATA_BITS], - mask[beat*`CHI_BE_BITS+:`CHI_BE_BITS] - )); - endtask - task write_line(int txn, longint unsigned addr, bit [511:0] data, bit [63:0] mask = '1); - req_t r = request(mask == '1 ? 'h1d : 'h1c, txn, addr); - rsp_t s; - byte unsigned expected_error; - send_req(r); - response(r, 6, s); - write_data(r, s, data, mask); - response(r, 4, s); - expected_error = chi_ref_write(model, addr, data, mask); - if (s.error != (expected_error ? 3 : 0)) `uvm_fatal("ERROR", "Wrong write error") - checks++; - endtask - task read_line(int txn, longint unsigned addr); - req_t r = request(4, txn, addr); - dat_t d; - bit [511:0] golden; - bit [`CHI_BEATS-1:0] seen = 0; - byte unsigned error = chi_ref_read(model, addr, golden); - send_req(r); - while (!(&seen)) begin - bit found = 0; - @(negedge vif.clock); - foreach (data_responses[i]) begin - if (data_responses[i].txn == r.return_txn) begin - d = data_responses[i]; - data_responses.delete(i); - found = 1; - break; - end - end - if (found) begin - int b = d.data_id / `CHI_DATA_ID_STEP; - if ((d.data_id % `CHI_DATA_ID_STEP != 0 || b >= `CHI_BEATS) || seen[b] || d.opcode != 4 || - d.tgt != r.return_nid || d.src != 1 || d.home != r.src || d.dbid != r.txn || - d.error != (error ? 3 : 0) || d.qos != r.qos || d.trace_tag != r.trace_tag || - d.data !== golden[b*`CHI_DATA_BITS +: `CHI_DATA_BITS]) - `uvm_fatal("READ", $sformatf("Bad read data txn=%0d DataID=%0d", txn, d.data_id)) - seen[b] = 1; - checks++; - end - end - endtask - task scenarios(); - req_t r[8]; - rsp_t dbid[8], s; - bit [511:0] words[8]; - reset_link(); -`ifdef CHI_BAD_OPCODE - send_req(request('h07, 0, 0)); - repeat (20) @(negedge vif.clock); - `uvm_fatal("NEGATIVE", "Unsupported opcode did not terminate RTL") -`elsif CHI_DUPLICATE_DATA - r[0] = request('h1d, 0, 0); - send_req(r[0]); - response(r[0], 6, s); - begin - dat_t d = '0; - d.tgt = 1; - d.src = r[0].src; - d.txn = s.dbid; - d.opcode = 3; - d.data_id = 0; - d.be = '1; - send_dat(d); - send_dat(d); - end - repeat (20) @(negedge vif.clock); - `uvm_fatal("NEGATIVE", "Duplicate DataID did not terminate RTL") -`else - // Fill all eight transaction slots, then interleave reverse-order DataIDs. - for (int i = 0; i < 8; i++) begin - for (int b = 0; b < 64; b++) words[i][b*8+:8] = (i * 37 + b); - r[i] = request('h1d, i, i * 64, i % 2 ? 5 : 3); - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - if (vif.outstanding != 8) `uvm_fatal("MSHR", "Failed to occupy all slots") - for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) begin - for (int i = 7; i >= 0; i--) begin - send_dat(write_beat(r[i], dbid[i], beat, words[i][beat*`CHI_DATA_BITS+:`CHI_DATA_BITS], '1 - )); - end - end - for (int i = 0; i < 8; i++) begin - response(r[i], 4, s); - if (s.error != 0 || chi_ref_write(model, i * 64, words[i], '1) != 0) - `uvm_fatal("WRITE", "Initial full write failed") - end - for (int i = 0; i < 8; i++) read_line(32 + i, i * 64); - // Every live slot must merge byte enables independently, including interleaved beats. - for (int phase_index = 0; phase_index < 2; phase_index++) begin - bit [63:0] mask = phase_index ? 64'haaaaaaaaaaaaaaaa : 64'h5555555555555555; - for (int i = 0; i < 8; i++) begin - r[i] = request('h1c, 140 + phase_index * 8 + i, i * 64, i % 2 ? 5 : 3); - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) begin - for (int i = 7; i >= 0; i--) begin - send_dat(write_beat( - r[i], - dbid[i], - beat, - phase_index ? words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS] : ~words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS], - mask[beat*`CHI_BE_BITS+:`CHI_BE_BITS] - )); - end - end - for (int i = 0; i < 8; i++) begin - bit [511:0] data = phase_index ? words[i] : ~words[i]; - response(r[i], 4, s); - if (s.error || chi_ref_write(model, i * 64, data, mask)) - `uvm_fatal("MASK", "Concurrent partial write failed") - end - for (int i = 0; i < 8; i++) read_line(240 + phase_index * 8 + i, i * 64); - end - `uvm_info("PHASE", "Full-width slot vectors", UVM_LOW) - // Exercise each real transaction slot's address, opaque IDs and error status. - // Writes are concurrent, addresses distinct; then verify valid writes by readback. - for (int polarity = 0; polarity < 2; polarity++) begin - for (int address_bit = 9; address_bit < 44; address_bit++) begin - for (int i = 0; i < 8; i++) begin - longint unsigned high = (64'h1 << address_bit) ^ (polarity ? 64'hffffffffe00 : 0); - int tag = (1 << ((address_bit + i) % 12)) ^ (polarity ? 'hfff : 0); - int source = ((1 << ((address_bit + i) % 7)) | 2) ^ (polarity ? 'h7f : 0); - r[i] = request('h1d, tag, high | ((i ^ (1 << ((address_bit + polarity) % 3))) << 6)); - r[i].src = source; - r[i].return_nid = source ^ 'h7f; - r[i].return_txn = tag ^ 'hfff; - r[i].qos = (address_bit + i) % 16; - r[i].trace_tag = polarity; - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - for (int beat = `CHI_BEATS - 1; beat >= 0; beat--) begin - for (int i = 7; i >= 0; i--) begin - send_dat(write_beat( - r[i], - dbid[i], - beat, - polarity ? words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS] : ~words[i][beat*`CHI_DATA_BITS +: `CHI_DATA_BITS], - '1 - )); - end - end - for (int i = 0; i < 8; i++) begin - bit [511:0] data = polarity ? words[i] : ~words[i]; - byte unsigned error; - response(r[i], 4, s); - error = chi_ref_write(model, r[i].addr, data, '1); - if (s.error != (error ? 3 : 0)) - `uvm_fatal("ERROR_SLOT", "Wrong error or opaque transaction attribution") - checks++; - end - if (!polarity && address_bit < 14) - for (int i = 0; i < 8; i++) read_line(600 + address_bit * 8 + i, r[i].addr); - end - end - write_line(50, 0, '1, 64'h5555555555555555); - write_line(51, 0, '0, 64'haaaaaaaaaaaaaaaa); - write_line(52, 0, '1, '0); - read_line(53, 0); - write_line(54, 255 * 64, '1); - read_line(55, 255 * 64); - write_line(56, 256 * 64, '1); - read_line(57, 256 * 64); - // Full-width addresses and TxnIDs must not alias into SRAM or DBID slots. - write_line(4095, 0, '1); - read_line(3950, 0); - write_line(58, 64'h80000000000, '1); - read_line(59, 64'h80000000000); - `uvm_info("PHASE", "Data credit stall", UVM_LOW) - // Block output credit return; requests remain live, then resume safely. - grant_enabled = 0; - for (int i = 0; i < STALL_READS; i++) send_req(request(4, 80 + i, (i % 8) * 64)); - repeat (25) @(negedge vif.clock); - if (vif.outstanding == 0) `uvm_fatal("STALL", "No live request under data credit stall") - grant_enabled = 1; - // These reads were issued already: compare every returned beat before reset. - for (int i = 0; i < STALL_READS; i++) begin - bit [511:0] golden; - bit [`CHI_BEATS-1:0] seen = 0; - if (chi_ref_read(model, (i % 8) * 64, golden)) `uvm_fatal("MODEL", "Unexpected error") - while (!(&seen)) begin - @(negedge vif.clock); - foreach (data_responses[j]) begin - if (data_responses[j].txn == 180 + i) begin - dat_t d = data_responses[j]; - int b = d.data_id / `CHI_DATA_ID_STEP; - if (seen[b] || d.data !== golden[b*`CHI_DATA_BITS+:`CHI_DATA_BITS]) - `uvm_fatal("STALL", "Stalled read corrupted") - seen[b] = 1; - checks++; - data_responses.delete(j); - break; - end - end - end - end - `uvm_info("PHASE", "Read-to-write slot reuse", UVM_LOW) - // Reallocate every slot to a write after the all-slot read drain. - for (int i = 0; i < 8; i++) begin - r[i] = request('h1d, 2800 + i, i * 64); - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - for (int i = 0; i < 8; i++) write_data(r[i], dbid[i], words[i], '1); - for (int i = 0; i < 8; i++) begin - response(r[i], 4, s); - if (s.error || chi_ref_write(model, i * 64, words[i], '1)) - `uvm_fatal("SLOT_REUSE", "Read-to-write slot reuse failed") - end - for (int i = 0; i < 8; i++) read_line(2900 + i, i * 64); - `uvm_info("PHASE", "Response and request queue saturation", UVM_LOW) - // Exhaust all15 RSP credits, then fill all8 node slots and all4 RX entries. - // Distinct TxnIDs from one source and equal TxnIDs from distinct sources are legal. - begin - req_t blocked[11]; - int data_order[8] = '{0, 2, 1, 3, 4, 6, 5, 7}; - while (rsp_granted != 15) @(negedge vif.clock); - grant_enabled = 0; - for (int i = 0; i < 8; i++) begin - r[i] = request('h1d, 3200 + i, i * 64); - r[i].src = 3; - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - for (int i = 0; i < 8; i++) begin - int slot = data_order[i]; - write_data(r[slot], dbid[slot], words[slot], '1); - end - // Drain seven initial Comps before new DBID responses compete for credits. - // One-beat writes can otherwise finish their data input before SRAM replies. - while (rsp_granted != 0 || responses.size() != 7) @(negedge vif.clock); - for (int i = 0; i < 11; i++) begin - blocked[i] = request('h1d, 3400, i * 64); - blocked[i].src = 10 + i; - send_req(blocked[i]); - end - repeat (30) @(negedge vif.clock); - if (vif.outstanding != 8 || req_credits != 0) - `uvm_fatal("RSP_STALL", "RSP credit exhaustion did not saturate node and RX queue") - grant_enabled = 1; - for (int i = 0; i < 8; i++) begin - response(r[i], 4, s); - if (s.error || chi_ref_write(model, i * 64, words[i], '1)) - `uvm_fatal("RSP_STALL", "Initial write failed") - end - for (int i = 0; i < 11; i++) begin - response(blocked[i], 6, s); - write_data(blocked[i], s, words[i%8], '1); - response(blocked[i], 4, s); - if (s.error || chi_ref_write(model, i * 64, words[i%8], '1)) - `uvm_fatal("RSP_STALL", "Queued same-TxnID write attributed to wrong source") - end - for (int i = 0; i < 11; i++) read_line(3600 + i, i * 64); - end - `uvm_info("PHASE", "Isolated upper read slots", UVM_LOW) - // Hold lower slots in write-data collection to isolate each upper read slot. - // A second read in that slot exercises arbitration after its own last grant. - for (int target_slot = 4; target_slot < 8; target_slot++) begin - for (int i = 0; i < target_slot; i++) begin - r[i] = request('h1d, 3700 + i, i * 64); - r[i].src = 3; - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - read_line(3800 + target_slot * 2, 0); - read_line(3801 + target_slot * 2, 0); - for (int i = 0; i < target_slot; i++) begin - write_data(r[i], dbid[i], words[i], '1); - response(r[i], 4, s); - if (s.error || chi_ref_write(model, i * 64, words[i], '1)) - `uvm_fatal("ARBITRATION", "Isolated slot damaged held writes") - end - end - `uvm_info("PHASE", "Source and transaction identity", UVM_LOW) - // Slot7 stays live while a lower free slot accepts another transaction from its source. - for (int i = 0; i < 8; i++) begin - r[i] = request('h1d, 3900 + i, i * 64); - r[i].src = 3; - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - for (int i = 0; i < 2; i++) begin - write_data(r[i], dbid[i], words[i], '1); - response(r[i], 4, s); - if (s.error || chi_ref_write(model, i * 64, words[i], '1)) - `uvm_fatal("SOURCE_IDS", "Initial source transaction failed") - end - for (int i = 0; i < 2; i++) begin - r[i] = request('h1d, 3990 + i, i * 64); - r[i].src = 3; - send_req(r[i]); - response(r[i], 6, dbid[i]); - end - repeat (4) @(negedge vif.clock); - for (int i = 0; i < 8; i++) begin - write_data(r[i], dbid[i], words[i], '1); - response(r[i], 4, s); - if (s.error || chi_ref_write(model, i * 64, words[i], '1)) - `uvm_fatal("SOURCE_IDS", "Concurrent source transaction failed") - end - `uvm_info("PHASE", "Reset cancellation", UVM_LOW) - // Cancel a write waiting for payload. SRAM data from completed writes survives. - send_req(request('h1d, 110, 64)); - response(request('h1d, 110, 64), 6, s); - reset_link(); - read_line(111, 64); - write_line(110, 64, '1); - write_line(110, 64, '1); // Reuse is legal only after the first completion. - read_line(112, 64); - repeat (15) @(negedge vif.clock); - if (vif.outstanding != 0 || responses.size() || data_responses.size()) - `uvm_fatal("DRAIN", "Unmatched transactions remain") - // Payload is unconstrained while FLITV=0 and must not create a transaction. - @(negedge vif.clock); - vif.req_flit = '0; - vif.dat_flit = '0; - repeat (2) @(negedge vif.clock); - vif.req_flit = '1; - vif.dat_flit = '1; - repeat (2) @(negedge vif.clock); - vif.req_flit = '0; - vif.dat_flit = '0; - repeat (2) @(negedge vif.clock); - if (vif.outstanding || responses.size() || data_responses.size()) - `uvm_fatal("INVALID_CHANNEL", "Inactive physical payload changed transactions") - `uvm_info("CHECKS", $sformatf("CHI SN-F: %0d checked responses/data beats", checks), UVM_LOW) - chi_ref_destroy(model); -`endif - endtask - task execute(); - vif.reset = 1; - vif.req_pend = 0; - vif.dat_pend = 0; - vif.req_valid = 0; - vif.dat_valid = 0; - vif.req_flit = 0; - vif.dat_flit = 0; - vif.rx_active_req = 0; - vif.tx_active_ack = 0; - vif.rsp_credit = 0; - vif.out_dat_credit = 0; - fork - monitor(); - join_none - scenarios(); - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/tx_bad_credit.f b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/tx_bad_credit.f deleted file mode 100644 index 6772bfaeb..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/tx_bad_credit.f +++ /dev/null @@ -1,9 +0,0 @@ -+define+CHI_TX_OVERFLOW -+define+CHI_CREDIT_TB=tx_bad_credit_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/credit_interface.sv -@RESOURCES@/credit_package.sv --F @RTL@/Tx/filelist.f -@RESOURCES@/credit_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/tx_bad_credit.toml b/arch/src/main/scala/framework/mem-core/chi/src/main/resources/tx_bad_credit.toml deleted file mode 100644 index 763078b72..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/tx_bad_credit.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CHI TX credit overflow" diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Emit.scala deleted file mode 100644 index 726790415..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Emit.scala +++ /dev/null @@ -1,42 +0,0 @@ -package memcore.bus.chi - -import memcore.bus.chi.snf.SramEndpoint -import memcore.bus.chi.rnf.{BankedChiCache, RnfParams} - -object Emit extends App { - val p = Params() - require((new RequestFlit(p)).flitWidth == 137) - require((new ResponseFlit(p)).flitWidth == 71) - require((new SnoopFlit(p)).flitWidth == 94) - require((new DataFlit(p)).flitWidth == 389) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Tx(flitBits = 32, maxCredits = 4), - firtoolOpts = args, - args = Array("--target-dir", "build/Tx", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Rx(flitBits = 32, depth = 4), - firtoolOpts = args, - args = Array("--target-dir", "build/Rx", "--split-verilog") - ) - for ((dataBits, flitBits) <- Seq(128 -> 240, 256 -> 389, 512 -> 687)) { - val snParams = p.copy(dataBits = dataBits) - require((new DataFlit(snParams)).flitWidth == flitBits) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new SramEndpoint(snParams), - firtoolOpts = args, - args = Array("--target-dir", s"build/SramEndpoint$dataBits", "--split-verilog") - ) - } - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new RequestRetry(p, nodeId = 1), - firtoolOpts = args, - args = Array("--target-dir", "build/RequestRetry", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new BankedChiCache(RnfParams()), - firtoolOpts = args, - args = Array("--target-dir", "build/BankedChiCache", "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Fabric.scala b/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Fabric.scala deleted file mode 100644 index ebcbf6a85..000000000 --- a/arch/src/main/scala/framework/mem-core/chi/src/main/scala/Fabric.scala +++ /dev/null @@ -1,110 +0,0 @@ -package memcore.bus.chi - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} - -/** Explicit requester-to-Home CHI fabric; no caches or backing-memory ownership. */ -@instantiable -class Fabric(p: Params, requesterCount: Int, homeId: Int) extends Module { - require(requesterCount >= 1 && requesterCount < homeId && BigInt(homeId) < (BigInt(1) << p.nodeIdBits)) - val c = p - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - val blockRequesterRsp = Input(Bool()) - val requesters = Vec(requesterCount, Flipped(new RequesterPort(c))) - val req = Decoupled(new RequestFlit(c)) - val rxRsp = Decoupled(new ResponseFlit(c)) - val rxDat = Decoupled(new DataFlit(c)) - val rsp = Flipped(Decoupled(new ResponseFlit(c))) - val dat = Flipped(Decoupled(new DataFlit(c))) - val snp = Flipped(Decoupled(new DirectedSnoop(c))) - }) - - def creditLink[T <: Flit](source: DecoupledIO[T], sink: DecoupledIO[T], blockRx: Bool = false.B): Unit = { - // Backpressure retains complete flits in the real credit-accounted Rx FIFO. - // Active and FLITV retain their channel semantics throughout the stall. - val tx: Instance[Tx] = Instantiate(new Tx(source.bits.flitWidth, maxCredits = 4)) - val rx: Instance[Rx] = Instantiate(new Rx(source.bits.flitWidth, depth = 4)) - tx.io.active := io.active - rx.io.active := io.active - tx.io.in.valid := source.valid - tx.io.in.bits := source.bits.packed - source.ready := tx.io.in.ready - rx.io.link <> tx.io.link - sink.valid := rx.io.out.valid && !blockRx - sink.bits.unpack(rx.io.out.bits) - rx.io.out.ready := sink.ready && !blockRx - } - - // All three arbiters use initialized history: the default RR lastGrant is not - // reset, so simultaneous first requests can otherwise select unknown data. - def requesterArbiter[T <: Data](flit: T): RRArbiter[T] = Module(new RRArbiter(flit, requesterCount) { - - override lazy val lastGrant = { - val cursor = RegInit(0.U(math.max(1, log2Ceil(requesterCount)).W)) - when(io.out.fire)(cursor := io.chosen) - cursor - } - - }) - - val requests = requesterArbiter(new RequestFlit(c)) - val responses = requesterArbiter(new ResponseFlit(c)) - val data = requesterArbiter(new DataFlit(c)) - io.req <> requests.io.out - io.rxRsp <> responses.io.out - io.rxDat <> data.io.out - for (i <- 0 until requesterCount) { - creditLink(io.requesters(i).req, requests.io.in(i)) - creditLink(io.requesters(i).txRsp, responses.io.in(i), io.blockRequesterRsp) - creditLink(io.requesters(i).txDat, data.io.in(i)) - when(requests.io.in(i).valid) { - assert( - requests.io.in(i).bits.srcId === (i + 1).U && requests.io.in(i).bits.tgtId === homeId.U, - "CacheSystem request crossed a wrong node or role" - ) - } - when(responses.io.in(i).valid)(assert( - responses.io.in(i).bits.srcId === (i + 1).U && - responses.io.in(i).bits.tgtId === homeId.U, - "CacheSystem response crossed a wrong node or role" - )) - when(data.io.in(i).valid)(assert( - data.io.in(i).bits.srcId === (i + 1).U && - data.io.in(i).bits.tgtId === homeId.U, - "CacheSystem data crossed a wrong node or role" - )) - } - val rsp = Wire(Vec(requesterCount, Decoupled(new ResponseFlit(c)))) - val dat = Wire(Vec(requesterCount, Decoupled(new DataFlit(c)))) - val snp = Wire(Vec(requesterCount, Decoupled(new SnoopFlit(c)))) - for (i <- 0 until requesterCount) { - rsp(i).valid := io.rsp.valid && io.rsp.bits.tgtId === (i + 1).U - rsp(i).bits := io.rsp.bits - dat(i).valid := io.dat.valid && io.dat.bits.tgtId === (i + 1).U - dat(i).bits := io.dat.bits - snp(i).valid := io.snp.valid && io.snp.bits.targetNode === (i + 1).U - snp(i).bits := io.snp.bits.flit - creditLink(rsp(i), io.requesters(i).rxRsp) - creditLink(dat(i), io.requesters(i).rxDat) - creditLink(snp(i), io.requesters(i).snp) - } - io.rsp.ready := (0 until requesterCount).map(i => rsp(i).ready && io.rsp.bits.tgtId === (i + 1).U).reduce(_ || _) - io.dat.ready := (0 until requesterCount).map(i => dat(i).ready && io.dat.bits.tgtId === (i + 1).U).reduce(_ || _) - io.snp.ready := (0 until requesterCount).map(i => snp(i).ready && io.snp.bits.targetNode === (i + 1).U).reduce(_ || _) - when(io.rsp.valid)(assert( - io.rsp.bits.tgtId >= 1.U && io.rsp.bits.tgtId <= requesterCount.U, - "CacheSystem response target is not a requester" - )) - when(io.dat.valid)(assert( - io.dat.bits.tgtId >= 1.U && io.dat.bits.tgtId <= requesterCount.U, - "CacheSystem data target is not a requester" - )) - when(io.snp.valid)(assert( - io.snp.bits.targetNode >= 1.U && io.snp.bits.targetNode <= requesterCount.U, - "CacheSystem snoop target is not a requester" - )) -} diff --git a/arch/src/main/scala/framework/mem-core/coherence/build.sc b/arch/src/main/scala/framework/mem-core/coherence/build.sc deleted file mode 100644 index 6d1b6e2c6..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/build.sc +++ /dev/null @@ -1,39 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object chi extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = os.pwd / os.up / "chi" -} - -object cache extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = os.pwd / os.up / "cache" - override def sources = T.sources(super.sources() ++ Seq(PathRef(moduleRoot / "configs"))) - override def ivyDeps = super.ivyDeps() ++ Agg(ivy"tech.sparse::toml-scala:0.2.2") -} - -object coherence extends ChiselModule { - override def mainClass = Some("memcore.memory.coherence.Emit") - override def moduleRoot = os.pwd - override def moduleDeps = Seq(chi, cache) - override def sources = T.sources(super.sources() ++ Seq(PathRef(moduleRoot / "configs"))) -} diff --git a/arch/src/main/scala/framework/mem-core/coherence/configs/CoherenceParams.scala b/arch/src/main/scala/framework/mem-core/coherence/configs/CoherenceParams.scala deleted file mode 100644 index 17615701e..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/configs/CoherenceParams.scala +++ /dev/null @@ -1,53 +0,0 @@ -package memcore.memory.coherence.configs - -import chisel3.util.log2Ceil -import java.nio.file.Paths -import scala.io.Source -import scala.util.Using -import toml.{Toml, Value} -import memcore.bus.chi.Params -import memcore.memory.cache.configs.CacheParams - -case class CoherenceParams( - chi: Params, - cache: CacheParams, - agents: Int, - mshrEntries: Int, - homeId: Int) { - require(cache.lineBytes == 64 && cache.addressBits == chi.addressBits) - require(agents >= 1 && agents < homeId && BigInt(homeId) < (BigInt(1) << chi.nodeIdBits)) - require(mshrEntries >= 1 && BigInt(mshrEntries) <= (BigInt(1) << chi.txnIdBits)) - require(BigInt(mshrEntries) <= (BigInt(1) << cache.idBits)) - val slotBits = math.max(1, log2Ceil(mshrEntries)) - val lineAddressBits = chi.addressBits - cache.offsetBits -} - -object CoherenceParams { - - def load(path: String): CoherenceParams = { - val text = Using.resource(Source.fromFile(path, "UTF-8"))(_.mkString) - val table = Toml.parse(text) match { - case Right(Value.Tbl(root)) => root("coherence") match { - case Value.Tbl(t) => t - case value => throw new IllegalArgumentException(s"Expected [coherence], got $value") - } - case value => throw new IllegalArgumentException(s"Invalid coherence config: $value") - } - def int(name: String): Int = table(name) match { - case Value.Num(n) if n == n.toInt => n.toInt - case value => throw new IllegalArgumentException(s"Expected integer $name, got $value") - } - val cachePath = table("cacheConfig") match { - case Value.Str(value) => Paths.get(path).toAbsolutePath.getParent.resolve(value).normalize.toString - case value => throw new IllegalArgumentException(s"Expected cacheConfig path, got $value") - } - CoherenceParams( - Params(int("nodeIdBits"), int("addressBits"), int("dataBits")), - CacheParams.load(cachePath), - int("agents"), - int("mshrEntries"), - int("homeId") - ) - } - -} diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/Cargo.lock b/arch/src/main/scala/framework/mem-core/coherence/src/csrc/Cargo.lock deleted file mode 100644 index af0855ac8..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "coherence-reference-model" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/Cargo.toml b/arch/src/main/scala/framework/mem-core/coherence/src/csrc/Cargo.toml deleted file mode 100644 index d48bbdc57..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/Cargo.toml +++ /dev/null @@ -1,7 +0,0 @@ -[package] -name = "coherence-reference-model" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib"] diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/src/memory.rs b/arch/src/main/scala/framework/mem-core/coherence/src/csrc/src/memory.rs deleted file mode 100644 index 87260303f..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/src/memory.rs +++ /dev/null @@ -1,36 +0,0 @@ -use std::collections::HashMap; -use std::ffi::c_void; - -#[no_mangle] -pub extern "C" fn coherence_ref_create() -> *mut c_void { - Box::into_raw(Box::new(HashMap::::new())).cast() -} -#[no_mangle] -pub unsafe extern "C" fn coherence_ref_destroy(model: *mut c_void) { - drop(Box::from_raw(model.cast::>())); -} -#[no_mangle] -pub unsafe extern "C" fn coherence_ref_initial(addr: u64, out: *mut u32) { - let out = std::slice::from_raw_parts_mut(out, 16); - for (i, word) in out.iter_mut().enumerate() { - *word = ((addr as u32) ^ ((addr >> 32) as u32)).rotate_left(i as u32) - ^ (0x10203040u32.wrapping_mul(i as u32 + 1)); - } -} -#[no_mangle] -pub unsafe extern "C" fn coherence_ref_read(model: *mut c_void, addr: u64, out: *mut u32) { - let memory = &mut *model.cast::>(); - if let Some(data) = memory.get(&addr) { - std::slice::from_raw_parts_mut(out, 16).copy_from_slice(data); - } else { - coherence_ref_initial(addr, out); - } -} -#[no_mangle] -pub unsafe extern "C" fn coherence_ref_write(model: *mut c_void, addr: u64, data: *const u32) { - let memory = &mut *model.cast::>(); - memory.insert( - addr, - std::slice::from_raw_parts(data, 16).try_into().unwrap(), - ); -} diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence.el b/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence.el deleted file mode 100644 index aa66a6577..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence.el +++ /dev/null @@ -1,316 +0,0 @@ -// Format Version: 2 -// Reviewed for 2 CPUs, 4 MSHRs, 4 sets, 2 ways, 64-byte lines, 256-bit CHI data. - -CHECKSUM: "848348206 1645746301" -INSTANCE: coherence_tb.dut.cache -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unknown Cache operation" -Block 10 "1190204013" "if (1)" -Block 11 "3885794289" "$error(\"Assertion failed: Unknown Cache operation\n at Cache.scala:73 assert(req.op <= CacheOp.Invalidate.U, \\"Unknown Cache operation\\")\n\");" -Block 13 "1190204013" "if (1)" -Block 14 "3662014904" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Cache operations require a line-aligned address" -Block 18 "1317833989" "if (1)" -Block 19 "1722205720" "$error(\"Assertion failed: Cache operations require a line-aligned address\n at Cache.scala:74 assert(req.addr(p.offsetBits - 1, 0) === 0.U, \\"Cache operations require a line-aligned address\\")\n\");" -Block 21 "1317833989" "if (1)" -Block 22 "505164254" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Duplicate Cache tag" -Block 26 "3036749987" "if (1)" -Block 27 "2991635828" "$error(\"Assertion failed: Duplicate Cache tag\n at Cache.scala:76 assert(PopCount(hits) <= 1.U, \\"Duplicate Cache tag\\")\n\");" -Block 29 "3036749987" "if (1)" -Block 30 "3772797571" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Cache fill would create a duplicate tag" -Block 34 "3744269243" "if (1)" -Block 35 "987059305" "$error(\"Assertion failed: Cache fill would create a duplicate tag\n at Cache.scala:89 assert(!hits.orR || hits(req.way), \\"Cache fill would create a duplicate tag\\")\n\");" -Block 37 "3744269243" "if (1)" -Block 38 "629733424" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Cache write requires the matching resident line" -Block 42 "4266678685" "if (1)" -Block 43 "4034066179" "$error(\"Assertion failed: Cache write requires the matching resident line\n at Cache.scala:99 assert(entryValid && hits(req.way), \\"Cache write requires the matching resident line\\")\n\");" -Block 45 "4266678685" "if (1)" -Block 46 "1550685438" "$fatal;" - -CHECKSUM: "25866875 2457151256" -INSTANCE: coherence_tb.dut.directory -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 2 "2664146107" "if (1)" -Block 3 "3533917897" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 5 "2664146107" "if (1)" -Block 6 "3472375403" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 10 "2881929055" "if (1)" -Block 11 "995660494" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 13 "2881929055" "if (1)" -Block 14 "1126102520" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 18 "1723217575" "if (1)" -Block 19 "3904278350" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 21 "1723217575" "if (1)" -Block 22 "4109238252" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 26 "3405939620" "if (1)" -Block 27 "684207841" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 29 "3405939620" "if (1)" -Block 30 "1351502295" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 34 "3919299312" "if (1)" -Block 35 "2315541197" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 37 "3919299312" "if (1)" -Block 38 "2522819183" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 42 "539207809" "if (1)" -Block 43 "2227510213" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 45 "539207809" "if (1)" -Block 46 "4236475635" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 50 "4085285969" "if (1)" -Block 51 "2434010258" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 53 "4085285969" "if (1)" -Block 54 "2370796592" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 58 "3010328178" "if (1)" -Block 59 "839440352" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 61 "3010328178" "if (1)" -Block 62 "1246671062" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 66 "2091741190" "if (1)" -Block 67 "4087798033" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 69 "2091741190" "if (1)" -Block 70 "4026378675" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 74 "2992832868" "if (1)" -Block 75 "730445366" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 77 "2992832868" "if (1)" -Block 78 "1406128384" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 82 "2228281882" "if (1)" -Block 83 "3383957654" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 85 "2228281882" "if (1)" -Block 86 "3588778036" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 90 "3031124909" "if (1)" -Block 91 "4279530673" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 93 "3031124909" "if (1)" -Block 94 "2270303111" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 98 "201135693" "if (1)" -Block 99 "2869021973" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 101 "201135693" "if (1)" -Block 102 "3076177335" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 106 "3770691673" "if (1)" -Block 107 "389918641" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 109 "3770691673" "if (1)" -Block 110 "1870383239" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Multiple transactions update one directory entry" -Block 114 "3662072233" "if (1)" -Block 115 "3809633553" "$error(\"Assertion failed: Multiple transactions update one directory entry\n at Directory.scala:43 assert(PopCount(writers) <= 1.U, \\"Multiple transactions update one directory entry\\")\n\");" -Block 117 "3662072233" "if (1)" -Block 118 "4282979763" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Unique ownership requires exactly one CPU" -Block 122 "1829712518" "if (1)" -Block 123 "4024367402" "$error(\"Assertion failed: Unique ownership requires exactly one CPU\n at Directory.scala:46 assert(!next.unique || PopCount(next.sharers) === 1.U, \\"Unique ownership requires exactly one CPU\\")\n\");" -Block 125 "1829712518" "if (1)" -Block 126 "2543361564" "$fatal;" - -CHECKSUM: "3226703875 2930150559" -INSTANCE: coherence_tb.dut.cache.responses -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_enq_bits_id [7:2] "net io_enq_bits_id[7:0]" -Toggle io_enq_bits_available "net io_enq_bits_available" -Toggle io_enq_bits_addr [5:0] "net io_enq_bits_addr[43:0]" -Toggle io_enq_bits_metadata [3:1] "net io_enq_bits_metadata[3:0]" -Toggle io_deq_bits_id [7:2] "net io_deq_bits_id[7:0]" -Toggle io_deq_bits_available "net io_deq_bits_available" -Toggle io_deq_bits_addr [5:0] "net io_deq_bits_addr[43:0]" -Toggle io_deq_bits_metadata [3:1] "net io_deq_bits_metadata[3:0]" - -CHECKSUM: "1696518855 210682840" -INSTANCE: coherence_tb.dut.datQueue -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_enq_bits_tgtId [6:2] "net io_enq_bits_tgtId[6:0]" -Toggle io_enq_bits_resp [2] "net io_enq_bits_resp[2:0]" -Toggle io_enq_bits_dbid [15:2] "net io_enq_bits_dbid[15:0]" -Toggle io_enq_bits_dataId [0] "net io_enq_bits_dataId[1:0]" -Toggle io_deq_bits_qos [3:0] "net io_deq_bits_qos[3:0]" -Toggle io_deq_bits_tgtId [6:2] "net io_deq_bits_tgtId[6:0]" -Toggle io_deq_bits_srcId [6:0] "net io_deq_bits_srcId[6:0]" -Toggle io_deq_bits_homeNid [6:0] "net io_deq_bits_homeNid[6:0]" -Toggle io_deq_bits_opcode [3:0] "net io_deq_bits_opcode[3:0]" -Toggle io_deq_bits_resp [2] "net io_deq_bits_resp[2:0]" -Toggle io_deq_bits_dataSource [7:0] "net io_deq_bits_dataSource[7:0]" -Toggle io_deq_bits_dataPull "net io_deq_bits_dataPull" -Toggle io_deq_bits_cBusy [2:0] "net io_deq_bits_cBusy[2:0]" -Toggle io_deq_bits_dbid [15:2] "net io_deq_bits_dbid[15:0]" -Toggle io_deq_bits_ccid [1:0] "net io_deq_bits_ccid[1:0]" -Toggle io_deq_bits_dataId [0] "net io_deq_bits_dataId[1:0]" -Toggle io_deq_bits_cacheLineId [5:0] "net io_deq_bits_cacheLineId[5:0]" -Toggle io_deq_bits_tagOp [1:0] "net io_deq_bits_tagOp[1:0]" -Toggle io_deq_bits_tag [7:0] "net io_deq_bits_tag[7:0]" -Toggle io_deq_bits_tagUpdate [1:0] "net io_deq_bits_tagUpdate[1:0]" -Toggle io_deq_bits_traceTag "net io_deq_bits_traceTag" -Toggle io_deq_bits_copyAtHome "net io_deq_bits_copyAtHome" -Toggle io_deq_bits_numDat [1:0] "net io_deq_bits_numDat[1:0]" -Toggle io_deq_bits_replicate "net io_deq_bits_replicate" - -CHECKSUM: "1414150944 1331432527" -INSTANCE: coherence_tb.dut.cacheQueue -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_enq_bits_id [7:2] "net io_enq_bits_id[7:0]" -Toggle io_enq_bits_addr [5:0] "net io_enq_bits_addr[43:0]" -Toggle io_enq_bits_metadata [3:1] "net io_enq_bits_metadata[3:0]" -Toggle io_deq_bits_id [7:2] "net io_deq_bits_id[7:0]" -Toggle io_deq_bits_addr [5:0] "net io_deq_bits_addr[43:0]" -Toggle io_deq_bits_mask [63:0] "net io_deq_bits_mask[63:0]" -Toggle io_deq_bits_metadata [3:1] "net io_deq_bits_metadata[3:0]" -Toggle io_deq_bits_eligible [1:0] "net io_deq_bits_eligible[1:0]" - -CHECKSUM: "3112608224 3447389693" -INSTANCE: coherence_tb.dut.memQueue -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_enq_bits_id [11:2] "net io_enq_bits_id[11:0]" -Toggle io_enq_bits_addr [5:0] "net io_enq_bits_addr[43:0]" -Toggle io_deq_bits_id [11:2] "net io_deq_bits_id[11:0]" -Toggle io_deq_bits_addr [5:0] "net io_deq_bits_addr[43:0]" -Toggle io_deq_bits_mask [63:0] "net io_deq_bits_mask[63:0]" - -CHECKSUM: "769480049 1064374571" -INSTANCE: coherence_tb.dut.rspQueue -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_enq_bits_tgtId [6:2] "net io_enq_bits_tgtId[6:0]" -Toggle io_enq_bits_opcode [4:1] "net io_enq_bits_opcode[4:0]" -Toggle io_enq_bits_dbid [11:2] "net io_enq_bits_dbid[11:0]" -Toggle io_deq_bits_qos [3:0] "net io_deq_bits_qos[3:0]" -Toggle io_deq_bits_tgtId [6:2] "net io_deq_bits_tgtId[6:0]" -Toggle io_deq_bits_srcId [6:0] "net io_deq_bits_srcId[6:0]" -Toggle io_deq_bits_opcode [4:1] "net io_deq_bits_opcode[4:0]" -Toggle io_deq_bits_resp [2:0] "net io_deq_bits_resp[2:0]" -Toggle io_deq_bits_fwdState [2:0] "net io_deq_bits_fwdState[2:0]" -Toggle io_deq_bits_cBusy [2:0] "net io_deq_bits_cBusy[2:0]" -Toggle io_deq_bits_dbid [11:2] "net io_deq_bits_dbid[11:0]" -Toggle io_deq_bits_pCrdType [3:0] "net io_deq_bits_pCrdType[3:0]" -Toggle io_deq_bits_tagOp [1:0] "net io_deq_bits_tagOp[1:0]" -Toggle io_deq_bits_traceTag "net io_deq_bits_traceTag" -Toggle io_deq_bits_cacheLineId [5:0] "net io_deq_bits_cacheLineId[5:0]" - -CHECKSUM: "2391098903 4025403226" -INSTANCE: coherence_tb.dut.snpQueue -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_enq_bits_targetNode [6:2] "net io_enq_bits_targetNode[6:0]" -Toggle io_enq_bits_flit_txnId [11:2] "net io_enq_bits_flit_txnId[11:0]" -Toggle io_enq_bits_flit_opcode [4] "net io_enq_bits_flit_opcode[4:0]" -Toggle io_enq_bits_flit_addr [2:0] "net io_enq_bits_flit_addr[40:0]" -Toggle io_deq_bits_targetNode [6:2] "net io_deq_bits_targetNode[6:0]" -Toggle io_deq_bits_flit_qos [3:0] "net io_deq_bits_flit_qos[3:0]" -Toggle io_deq_bits_flit_srcId [6:0] "net io_deq_bits_flit_srcId[6:0]" -Toggle io_deq_bits_flit_txnId [11:2] "net io_deq_bits_flit_txnId[11:0]" -Toggle io_deq_bits_flit_fwdNid [6:0] "net io_deq_bits_flit_fwdNid[6:0]" -Toggle io_deq_bits_flit_fwdTxnId [11:0] "net io_deq_bits_flit_fwdTxnId[11:0]" -Toggle io_deq_bits_flit_opcode [4] "net io_deq_bits_flit_opcode[4:0]" -Toggle io_deq_bits_flit_addr [2:0] "net io_deq_bits_flit_addr[40:0]" -Toggle io_deq_bits_flit_pas [2:0] "net io_deq_bits_flit_pas[2:0]" -Toggle io_deq_bits_flit_doNotGoToSd "net io_deq_bits_flit_doNotGoToSd" -Toggle io_deq_bits_flit_traceTag "net io_deq_bits_flit_traceTag" - -CHECKSUM: "1853676028 3335661445" -INSTANCE: coherence_tb.dut.cacheArb -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_in_0_bits_addr [5:0] "net io_in_0_bits_addr[43:0]" -Toggle io_in_0_bits_metadata [3:1] "net io_in_0_bits_metadata[3:0]" -Toggle io_in_1_bits_addr [5:0] "net io_in_1_bits_addr[43:0]" -Toggle io_in_1_bits_metadata [3:1] "net io_in_1_bits_metadata[3:0]" -Toggle io_in_2_bits_addr [5:0] "net io_in_2_bits_addr[43:0]" -Toggle io_in_2_bits_metadata [3:1] "net io_in_2_bits_metadata[3:0]" -Toggle io_in_3_bits_addr [5:0] "net io_in_3_bits_addr[43:0]" -Toggle io_in_3_bits_metadata [3:1] "net io_in_3_bits_metadata[3:0]" -Toggle io_out_bits_id [7:2] "net io_out_bits_id[7:0]" -Toggle io_out_bits_addr [5:0] "net io_out_bits_addr[43:0]" -Toggle io_out_bits_metadata [3:1] "net io_out_bits_metadata[3:0]" - -CHECKSUM: "3130163852 2143613990" -INSTANCE: coherence_tb.dut.memArb -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_in_0_bits_addr [5:0] "net io_in_0_bits_addr[43:0]" -Toggle io_in_1_bits_addr [5:0] "net io_in_1_bits_addr[43:0]" -Toggle io_in_2_bits_addr [5:0] "net io_in_2_bits_addr[43:0]" -Toggle io_in_3_bits_addr [5:0] "net io_in_3_bits_addr[43:0]" -Toggle io_out_bits_id [11:2] "net io_out_bits_id[11:0]" -Toggle io_out_bits_addr [5:0] "net io_out_bits_addr[43:0]" - -CHECKSUM: "3716118738 1796417120" -INSTANCE: coherence_tb.dut.snpArb -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_in_0_bits_targetNode [6:2] "net io_in_0_bits_targetNode[6:0]" -Toggle io_in_0_bits_flit_opcode [4] "net io_in_0_bits_flit_opcode[4:0]" -Toggle io_in_0_bits_flit_addr [2:0] "net io_in_0_bits_flit_addr[40:0]" -Toggle io_in_1_bits_targetNode [6:2] "net io_in_1_bits_targetNode[6:0]" -Toggle io_in_1_bits_flit_opcode [4] "net io_in_1_bits_flit_opcode[4:0]" -Toggle io_in_1_bits_flit_addr [2:0] "net io_in_1_bits_flit_addr[40:0]" -Toggle io_in_2_bits_targetNode [6:2] "net io_in_2_bits_targetNode[6:0]" -Toggle io_in_2_bits_flit_opcode [4] "net io_in_2_bits_flit_opcode[4:0]" -Toggle io_in_2_bits_flit_addr [2:0] "net io_in_2_bits_flit_addr[40:0]" -Toggle io_in_3_bits_targetNode [6:2] "net io_in_3_bits_targetNode[6:0]" -Toggle io_in_3_bits_flit_opcode [4] "net io_in_3_bits_flit_opcode[4:0]" -Toggle io_in_3_bits_flit_addr [2:0] "net io_in_3_bits_flit_addr[40:0]" -Toggle io_out_bits_targetNode [6:2] "net io_out_bits_targetNode[6:0]" -Toggle io_out_bits_flit_txnId [11:2] "net io_out_bits_flit_txnId[11:0]" -Toggle io_out_bits_flit_opcode [4] "net io_out_bits_flit_opcode[4:0]" -Toggle io_out_bits_flit_addr [2:0] "net io_out_bits_flit_addr[40:0]" - -CHECKSUM: "2004716196 3529883256" -INSTANCE: coherence_tb.dut.rspArb -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_in_0_bits_tgtId [6:2] "net io_in_0_bits_tgtId[6:0]" -Toggle io_in_0_bits_opcode [4:1] "net io_in_0_bits_opcode[4:0]" -Toggle io_in_1_bits_tgtId [6:2] "net io_in_1_bits_tgtId[6:0]" -Toggle io_in_1_bits_opcode [4:1] "net io_in_1_bits_opcode[4:0]" -Toggle io_in_2_bits_tgtId [6:2] "net io_in_2_bits_tgtId[6:0]" -Toggle io_in_2_bits_opcode [4:1] "net io_in_2_bits_opcode[4:0]" -Toggle io_in_3_bits_tgtId [6:2] "net io_in_3_bits_tgtId[6:0]" -Toggle io_in_3_bits_opcode [4:1] "net io_in_3_bits_opcode[4:0]" -Toggle io_out_bits_tgtId [6:2] "net io_out_bits_tgtId[6:0]" -Toggle io_out_bits_opcode [4:1] "net io_out_bits_opcode[4:0]" -Toggle io_out_bits_dbid [11:2] "net io_out_bits_dbid[11:0]" - -CHECKSUM: "3496838959 3135397854" -INSTANCE: coherence_tb.dut.datArb -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_in_0_bits_tgtId [6:2] "net io_in_0_bits_tgtId[6:0]" -Toggle io_in_0_bits_resp [2] "net io_in_0_bits_resp[2:0]" -Toggle io_in_0_bits_dataId [0] "net io_in_0_bits_dataId[1:0]" -Toggle io_in_1_bits_tgtId [6:2] "net io_in_1_bits_tgtId[6:0]" -Toggle io_in_1_bits_resp [2] "net io_in_1_bits_resp[2:0]" -Toggle io_in_1_bits_dataId [0] "net io_in_1_bits_dataId[1:0]" -Toggle io_in_2_bits_tgtId [6:2] "net io_in_2_bits_tgtId[6:0]" -Toggle io_in_2_bits_resp [2] "net io_in_2_bits_resp[2:0]" -Toggle io_in_2_bits_dataId [0] "net io_in_2_bits_dataId[1:0]" -Toggle io_in_3_bits_tgtId [6:2] "net io_in_3_bits_tgtId[6:0]" -Toggle io_in_3_bits_resp [2] "net io_in_3_bits_resp[2:0]" -Toggle io_in_3_bits_dataId [0] "net io_in_3_bits_dataId[1:0]" -Toggle io_out_bits_tgtId [6:2] "net io_out_bits_tgtId[6:0]" -Toggle io_out_bits_resp [2] "net io_out_bits_resp[2:0]" -Toggle io_out_bits_dbid [15:2] "net io_out_bits_dbid[15:0]" -Toggle io_out_bits_dataId [0] "net io_out_bits_dataId[1:0]" - -CHECKSUM: "848348206 843507997" -INSTANCE: coherence_tb.dut.cache -ANNOTATION: "Source-proven Home profile: MSHR IDs 0..3, requester IDs 1..2, aligned 64-byte addresses, full-line masks, all ways eligible, dirty metadata bit 0; CHI optional fields are assigned constants. Only the specified constant bits are excluded." -Toggle io_request_bits_id [7:2] "net io_request_bits_id[7:0]" -Toggle io_request_bits_addr [5:0] "net io_request_bits_addr[43:0]" -Toggle io_request_bits_mask [63:0] "net io_request_bits_mask[63:0]" -Toggle io_request_bits_metadata [3:1] "net io_request_bits_metadata[3:0]" -Toggle io_request_bits_eligible [1:0] "net io_request_bits_eligible[1:0]" -Toggle io_response_bits_id [7:2] "net io_response_bits_id[7:0]" -Toggle io_response_bits_available "net io_response_bits_available" -Toggle io_response_bits_addr [5:0] "net io_response_bits_addr[43:0]" -Toggle io_response_bits_metadata [3:1] "net io_response_bits_metadata[3:0]" -Toggle metadata_0_0 [3:1] "reg metadata_0_0[3:0]" -Toggle metadata_0_1 [3:1] "reg metadata_0_1[3:0]" -Toggle metadata_0_2 [3:1] "reg metadata_0_2[3:0]" -Toggle metadata_0_3 [3:1] "reg metadata_0_3[3:0]" -Toggle metadata_1_0 [3:1] "reg metadata_1_0[3:0]" -Toggle metadata_1_1 [3:1] "reg metadata_1_1[3:0]" -Toggle metadata_1_2 [3:1] "reg metadata_1_2[3:0]" -Toggle metadata_1_3 [3:1] "reg metadata_1_3[3:0]" -Toggle result_id [7:2] "reg result_id[7:0]" -Toggle result_addr [5:0] "reg result_addr[43:0]" -Toggle result_metadata [3:1] "reg result_metadata[3:0]" diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence.f b/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence.f deleted file mode 100644 index 126bf5261..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/coherence.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Coherence/filelist.f -@RESOURCES@/coherence_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/env.svh b/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/env.svh deleted file mode 100644 index 1c4c2b110..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/env.svh +++ /dev/null @@ -1,393 +0,0 @@ -class env extends uvm_env; - `uvm_component_utils(env) - virtual coherence_control_if control; - virtual stream_if #(`COH_REQ_WIDTH) req; - virtual stream_if #(`COH_RSP_WIDTH) rx_rsp; - virtual stream_if #(`COH_DAT_WIDTH) rx_dat; - virtual stream_if #(`COH_TXRSP_WIDTH) tx_rsp; - virtual stream_if #(`COH_TXDAT_WIDTH) tx_dat; - virtual stream_if #(`COH_SNP_CHANNEL_WIDTH) snp; - virtual stream_if #(`COH_MEMREQ_WIDTH) mem_req; - virtual stream_if #(`COH_MEMRESP_WIDTH) mem_resp; - keyed_scoreboard #(completion) scoreboard; - chandle model; - req_t requests[int]; - logic [511:0] assembled[int]; - bit [`COH_BEATS-1:0] received[int]; - int data_dbid[int]; - logic [2:0] data_permission[int]; - logic [1:0] data_error[int]; - int expected_order[$]; - bit expected_error[int]; - private_line_t cpus[`COH_AGENTS][longint unsigned]; - bit [511:0] memory[longint unsigned]; - bit dirty_addresses[longint unsigned]; - int clean_shared_requests[longint unsigned]; - rsp_t rsp_queue[$], ack_queue[$]; - dat_t dat_queue[$]; - mem_t mem_queue[$]; - bit [`COH_MSHRS-1:0] hold_ack = 0; - bit vary_idle_payload = 0; - bit idle_payload_one = 0; - bit hold_memory = 0, reverse_memory = 0, saw_reorder = 0; - bit rsp_active = 0, dat_active = 0, mem_active = 0; - rsp_t rsp_packet; - dat_t dat_packet; - mem_t mem_packet; - int memory_writes = 0, memory_reads = 0, peak = 0; - int cycle = 0; - int snoops = 0; - bit hold_rsp = 0, hold_snp = 0, hold_mem_req = 0; - bit hold_copyback = 0; - bit fault_enabled = 0, write_fault_enabled = 0; - longint unsigned fault_addr; - - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - scoreboard = keyed_scoreboard#(completion)::type_id::create("scoreboard", this); - model = coherence_ref_create(); - if (!uvm_config_db#(virtual coherence_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`COH_REQ_WIDTH))::get( - this, "", "req", req - ) || !uvm_config_db#(virtual stream_if #(`COH_RSP_WIDTH))::get( - this, "", "rx_rsp", rx_rsp - ) || !uvm_config_db#(virtual stream_if #(`COH_DAT_WIDTH))::get( - this, "", "rx_dat", rx_dat - ) || !uvm_config_db#(virtual stream_if #(`COH_TXRSP_WIDTH))::get( - this, "", "tx_rsp", tx_rsp - ) || !uvm_config_db#(virtual stream_if #(`COH_TXDAT_WIDTH))::get( - this, "", "tx_dat", tx_dat - ) || !uvm_config_db#(virtual stream_if #(`COH_SNP_CHANNEL_WIDTH))::get( - this, "", "snp", snp - ) || !uvm_config_db#(virtual stream_if #(`COH_MEMREQ_WIDTH))::get( - this, "", "mem_req", mem_req - ) || !uvm_config_db#(virtual stream_if #(`COH_MEMRESP_WIDTH))::get( - this, "", "mem_resp", mem_resp - )) - `uvm_fatal("VIF", "Coherence interfaces missing") - endfunction - function void cpu_write(int node, longint unsigned addr, bit [511:0] value); - if (!cpus[node-1].exists(addr) || !cpus[node-1][addr].valid || !cpus[node-1][addr].unique_owner) - `uvm_fatal("CPU", "write without unique permission") - cpus[node-1][addr].data = value; - cpus[node-1][addr].dirty = 1; - coherence_ref_write(model, addr, value); - dirty_addresses[addr] = 1; - endfunction - function void finish(completion item); - int key = (item.node << 12) | item.txn; - int position = -1; - item.set_transaction_id(key); - foreach (expected_order[i]) if (expected_order[i] == key) position = i; - if (position < 0) `uvm_fatal("ORDER", "unexpected completion") - if (position != 0) saw_reorder = 1; - expected_order.delete(position); - scoreboard.actual_export.write(item); - if (requests[key][`CF(REQ, OPCODE)] == `COH_CLEAN_SHARED) begin - longint unsigned address = requests[key][`CF(REQ, ADDR)]; - clean_shared_requests[address]--; - if (clean_shared_requests[address] == 0) clean_shared_requests.delete(address); - end - requests.delete(key); - endfunction - - task run_phase(uvm_phase phase); - forever begin - @(posedge control.clock); - if (control.reset) begin - if (dirty_addresses.num() != 0) - `uvm_fatal("RESET", "test reset before dirty data reached memory") - rx_rsp.valid <= 0; - rx_rsp.bits <= '0; - rx_dat.valid <= 0; - rx_dat.bits <= '0; - mem_resp.valid <= 0; - mem_resp.bits <= '0; - tx_rsp.ready <= 0; - tx_dat.ready <= 0; - snp.ready <= 0; - mem_req.ready <= 0; - rsp_active = 0; - dat_active = 0; - mem_active = 0; - rsp_queue.delete(); - ack_queue.delete(); - dat_queue.delete(); - mem_queue.delete(); - foreach (cpus[i]) cpus[i].delete(); - end else begin - cycle++; - tx_rsp.ready <= !hold_rsp && (cycle % 8) >= 4; - tx_dat.ready <= (cycle % 8) >= 4; - snp.ready <= !hold_snp && (cycle % 8) >= 4; - mem_req.ready <= !hold_mem_req && (cycle % 8) >= 4; - if (control.outstanding > peak) peak = control.outstanding; - if (rsp_active && rx_rsp.ready) rsp_active = 0; - if (dat_active && rx_dat.ready) dat_active = 0; - if (mem_active && mem_resp.ready) mem_active = 0; - if (req.valid && req.ready) begin - completion item = completion::type_id::create("expected"); - int node = req.bits[`CF(REQ, SRCID)]; - int txn = req.bits[`CF(REQ, TXNID)]; - int key = (node << 12) | txn; - int opcode = req.bits[`CF(REQ, OPCODE)]; - longint unsigned addr = req.bits[`CF(REQ, ADDR)]; - bit [511:0] value; - requests[key] = req.bits; - if (opcode == `COH_CLEAN_SHARED) begin - if (!clean_shared_requests.exists(addr)) clean_shared_requests[addr] = 0; - clean_shared_requests[addr]++; - end - expected_order.push_back(key); - item.node = node; - item.txn = txn; - item.is_data=opcode==`COH_READ_SHARED || opcode==`COH_READ_NSD || opcode==`COH_READ_UNIQUE; - item.opcode=item.is_data ? `COH_COMP_DATA : (opcode==`COH_WRITEBACK ? `COH_COMP_DBID : `COH_COMP); - item.error = expected_error.exists(key) && expected_error[key] ? 3 : 0; - item.permission=item.is_data && item.error==0 ? (opcode==`COH_READ_UNIQUE ? 2 : 1) : 0; - if (item.is_data && !item.error) begin - coherence_ref_read(model, addr, value); - item.data = value; - end - item.set_transaction_id(key); - scoreboard.expected_export.write(item); - if (opcode == `COH_EVICT && cpus[node-1].exists(addr)) begin - if (cpus[node-1][addr].valid && cpus[node-1][addr].dirty) - `uvm_fatal("CPU", "dirty line sent as Evict") - cpus[node-1][addr].valid = 0; - end - end - if (snp.valid && snp.ready) begin - int node = snp.bits[`CF(SNP, TARGET)]; - int txn = snp.bits[`CF(SNP, TXNID)]; - int opcode = snp.bits[`CF(SNP, OPCODE)]; - longint unsigned addr = longint'(snp.bits[`CF(SNP, ADDR)]) << 3; - private_line_t line; - bit invalidate = opcode == `COH_SNP_UNIQUE || opcode == `COH_SNP_INVALID || opcode == `COH_SNP_MAKE_INVALID; - bit discard = opcode == `COH_SNP_MAKE_INVALID; - if (node < 1 || node > `COH_AGENTS || snp.bits[`CF(SNP, SRCID)] != `COH_HOME) - `uvm_fatal("SNOOP", "invalid destination/source") - snoops++; - line = cpus[node-1].exists(addr) ? cpus[node-1][addr] : '0; - if ((opcode == `COH_SNP_INVALID || opcode == `COH_SNP_CLEAN_SHARED || discard) && snp.bits[ - `CF(SNP, RETTOSRC) - ] !== 0) - `uvm_fatal("SNOOP", "Maintenance snoop must clear RetToSrc") - if (line.valid && !discard && (line.dirty || (!line.unique_owner && snp.bits[ - `CF(SNP, RETTOSRC) - ]))) begin - for (int b = 0; b < `COH_BEATS; b++) begin - dat_t packet = '0; - packet[`CF(DAT, TGTID)] = `COH_HOME; - packet[`CF(DAT, SRCID)] = node; - packet[`CF(DAT, TXNID)] = txn; - packet[`CF(DAT, OPCODE)] = `COH_SNP_DATA; - packet[`CF(DAT, RESP)] = (invalidate ? 0 : 1) | (line.dirty ? 4 : 0); - packet[`CF(DAT, DATAID)] = b * (`COH_DATA_BITS / 128); - packet[`CF(DAT, BE)] = '1; - packet[`CF(DAT, DATA)] = line.data[b*`COH_DATA_BITS+:`COH_DATA_BITS]; - dat_queue.push_back(packet); - end - end else begin - rsp_t packet = '0; - packet[`CF(RSP, TGTID)] = `COH_HOME; - packet[`CF(RSP, SRCID)] = node; - packet[`CF(RSP, TXNID)] = txn; - packet[`CF(RSP, OPCODE)] = `COH_SNP_RESP; - packet[`CF(RSP, RESP)] = line.valid && !invalidate ? 1 : 0; - rsp_queue.push_back(packet); - end - if (line.valid) begin - line.valid = !invalidate; - line.unique_owner = 0; - line.dirty = 0; - cpus[node-1][addr] = line; - end - end - if (tx_rsp.valid && tx_rsp.ready) begin - completion item = completion::type_id::create("rsp"); - int node = tx_rsp.bits[`CF(TXRSP, TGTID)]; - int txn = tx_rsp.bits[`CF(TXRSP, TXNID)]; - int key = (node << 12) | txn; - int dbid = tx_rsp.bits[`CF(TXRSP, DBID)]; - if (!requests.exists( - key - ) || tx_rsp.bits[ - `CF(TXRSP, SRCID) - ] != `COH_HOME || dbid >= `COH_MSHRS) - `uvm_fatal("RSP", "unexpected response routing") - item.node = node; - item.txn = txn; - item.opcode = tx_rsp.bits[`CF(TXRSP, OPCODE)]; - item.error = tx_rsp.bits[`CF(TXRSP, RESPERR)]; - item.permission = tx_rsp.bits[`CF(TXRSP, RESP)]; - if (item.opcode == `COH_COMP_DBID) begin - longint unsigned addr = requests[key][`CF(REQ, ADDR)]; - private_line_t line = cpus[node-1].exists(addr) ? cpus[node-1][addr] : '0; - for (int b = 0; b < `COH_BEATS; b++) begin - dat_t packet = '0; - packet[`CF(DAT, TGTID)] = `COH_HOME; - packet[`CF(DAT, SRCID)] = node; - packet[`CF(DAT, TXNID)] = dbid; - packet[`CF(DAT, OPCODE)] = `COH_COPYBACK_DATA; - packet[ - `CF(DAT, RESP) - ] = line.valid ? (line.unique_owner ? 2 : 1) | (line.dirty ? 4 : 0) : 0; - packet[`CF(DAT, DATAID)] = b * (`COH_DATA_BITS / 128); - packet[`CF(DAT, BE)] = line.valid ? '1 : '0; - packet[`CF(DAT, DATA)] = line.data[b*`COH_DATA_BITS+:`COH_DATA_BITS]; - dat_queue.push_back(packet); - end - if (line.valid) begin - line.valid = 0; - cpus[node-1][addr] = line; - end - end - finish(item); - end - if (tx_dat.valid && tx_dat.ready) begin - int node = tx_dat.bits[`CF(TXDAT, TGTID)]; - int txn = tx_dat.bits[`CF(TXDAT, TXNID)]; - int key = (node << 12) | txn; - int b = tx_dat.bits[`CF(TXDAT, DATAID)] / (`COH_DATA_BITS / 128); - int dbid = tx_dat.bits[`CF(TXDAT, DBID)]; - if (!requests.exists( - key - ) || tx_dat.bits[ - `CF(TXDAT, SRCID) - ] != `COH_HOME || tx_dat.bits[ - `CF(TXDAT, HOMENID) - ] != `COH_HOME || dbid >= `COH_MSHRS || b >= `COH_BEATS) - `uvm_fatal("DATA", "unexpected response routing") - if (!received.exists(key)) begin - received[key] = '0; - assembled[key] = '0; - data_dbid[key] = dbid; - data_permission[key] = tx_dat.bits[`CF(TXDAT, RESP)]; - data_error[key] = tx_dat.bits[`CF(TXDAT, RESPERR)]; - end - if (tx_dat.bits[ - `CF(TXDAT, OPCODE) - ] !== `COH_COMP_DATA || tx_dat.bits[ - `CF(TXDAT, DATAID) - ] % (`COH_DATA_BITS / 128) != 0 || tx_dat.bits[ - `CF(TXDAT, RESP) - ] !== data_permission[key] || tx_dat.bits[ - `CF(TXDAT, RESPERR) - ] !== data_error[key] || tx_dat.bits[ - `CF(TXDAT, BE) - ] !== (data_error[key] != 0 ? `COH_TXDAT_BE_WIDTH'(0) : {`COH_TXDAT_BE_WIDTH{1'b1}})) - `uvm_fatal("DATA", "inconsistent completion data header") - if (received[key][b] || dbid != data_dbid[key]) - `uvm_fatal("DATA", "duplicate beat or inconsistent DBID") - assembled[key][b*`COH_DATA_BITS+:`COH_DATA_BITS] = tx_dat.bits[`CF(TXDAT, DATA)]; - received[key][b] = 1; - if (&received[key]) begin - completion item = completion::type_id::create("data"); - rsp_t ack = '0; - longint unsigned addr = requests[key][`CF(REQ, ADDR)]; - item.node = node; - item.txn = txn; - item.is_data = 1; - item.opcode = tx_dat.bits[`CF(TXDAT, OPCODE)]; - item.error = tx_dat.bits[`CF(TXDAT, RESPERR)]; - item.permission = tx_dat.bits[`CF(TXDAT, RESP)]; - item.data = assembled[key]; - if (!item.error) - cpus[node-1][addr] = '{ - valid: 1, - unique_owner: (item.permission == 2), - dirty: 0, - data: item.data - }; - ack[`CF(RSP, TGTID)] = `COH_HOME; - ack[`CF(RSP, SRCID)] = node; - ack[`CF(RSP, TXNID)] = dbid; - ack[`CF(RSP, OPCODE)] = `COH_COMP_ACK; - ack_queue.push_back(ack); - finish(item); - received.delete(key); - assembled.delete(key); - data_dbid.delete(key); - data_permission.delete(key); - data_error.delete(key); - end - end - if (mem_req.valid && mem_req.ready) begin - mem_t packet = '0; - longint unsigned addr = mem_req.bits[`CF(MEMREQ, ADDR)]; - bit [511:0] value; - packet[`CF(MEMRESP, ID)] = mem_req.bits[`CF(MEMREQ, ID)]; - if (mem_req.bits[`CF(MEMREQ, WRITE)]) begin - coherence_ref_read(model, addr, value); - if (mem_req.bits[`CF(MEMREQ, DATA)] !== value || mem_req.bits[`CF(MEMREQ, MASK)] !== '1) - `uvm_fatal("MEMORY", "dirty victim writeback lost data") - foreach (cpus[i]) - if (cpus[i].exists( - addr - ) && cpus[i][addr].valid && (!clean_shared_requests.exists( - addr - ) || cpus[i][addr].dirty || cpus[i][addr].unique_owner || - cpus[i][addr].data !== value)) - `uvm_fatal( - "INCLUSION", - "DDR clean completed while a private copy was dirty, unique or inconsistent") - if (write_fault_enabled && addr == fault_addr) begin - packet[`CF(MEMRESP, ERROR)] = 1; - end else begin - memory[addr] = value; - dirty_addresses.delete(addr); - end - memory_writes++; - end else begin - if (memory.exists(addr)) value = memory[addr]; - else coherence_ref_initial(addr, value); - packet[`CF(MEMRESP, DATA)] = value; - packet[`CF(MEMRESP, ERROR)] = fault_enabled && addr == fault_addr; - memory_reads++; - end - mem_queue.push_back(packet); - end - if (!rsp_active) begin - if (rsp_queue.size() != 0) begin - rsp_packet = rsp_queue.pop_front(); - rsp_active = 1; - end else begin - int chosen = -1; - foreach (ack_queue[j]) - if (chosen < 0 && !hold_ack[ack_queue[j][`CF(RSP, TXNID)]]) chosen = j; - if (chosen >= 0) begin - rsp_packet = ack_queue[chosen]; - ack_queue.delete(chosen); - rsp_active = 1; - end - end - end - if (!dat_active && dat_queue.size() != 0 && (!hold_copyback || dat_queue[0][ - `CF(DAT, OPCODE) - ] != `COH_COPYBACK_DATA)) begin - dat_packet = dat_queue.pop_front(); - dat_active = 1; - end - if (!mem_active && !hold_memory && mem_queue.size() != 0) begin - mem_packet = reverse_memory ? mem_queue.pop_back() : mem_queue.pop_front(); - mem_active = 1; - end - rx_rsp.valid <= rsp_active; - rx_rsp.bits <= rsp_active || !vary_idle_payload ? rsp_packet : {`COH_RSP_WIDTH{idle_payload_one}}; - rx_dat.valid <= dat_active; - rx_dat.bits <= dat_active || !vary_idle_payload ? dat_packet : {`COH_DAT_WIDTH{idle_payload_one}}; - mem_resp.valid <= mem_active; - mem_resp.bits <= mem_active || !vary_idle_payload ? mem_packet : {`COH_MEMRESP_WIDTH{idle_payload_one}}; - end - end - endtask - function void final_phase(uvm_phase phase); - coherence_ref_destroy(model); - super.final_phase(phase); - endfunction -endclass diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/package.sv deleted file mode 100644 index 73f77bf30..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/package.sv +++ /dev/null @@ -1,54 +0,0 @@ -`include "coherence_config.svh" -`define CF(K, F) `COH_``K``_``F``_OFFSET +: `COH_``K``_``F``_WIDTH -package coherence_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - typedef logic [`COH_REQ_WIDTH-1:0] req_t; - typedef logic [`COH_RSP_WIDTH-1:0] rsp_t; - typedef logic [`COH_DAT_WIDTH-1:0] dat_t; - typedef logic [`COH_MEMRESP_WIDTH-1:0] mem_t; - typedef struct packed { - bit valid, unique_owner, dirty; - bit [511:0] data; - } private_line_t; - import "DPI-C" function chandle coherence_ref_create(); - import "DPI-C" function void coherence_ref_destroy(input chandle model); - import "DPI-C" function void coherence_ref_initial( - input longint unsigned addr, - output bit [511:0] data - ); - import "DPI-C" function void coherence_ref_read( - input chandle model, - input longint unsigned addr, - output bit [511:0] data - ); - import "DPI-C" function void coherence_ref_write( - input chandle model, - input longint unsigned addr, - input bit [511:0] data - ); - - class completion extends uvm_sequence_item; - int node, txn, opcode; - bit is_data; - logic [2:0] permission = 0; - logic [1:0] error = 0; - logic [511:0] data = 0; - `uvm_object_utils_begin(completion) - `uvm_field_int(node, UVM_ALL_ON) - `uvm_field_int(txn, UVM_ALL_ON) - `uvm_field_int(opcode, UVM_ALL_ON) - `uvm_field_int(is_data, UVM_ALL_ON) - `uvm_field_int(permission, UVM_ALL_ON) - `uvm_field_int(error, UVM_ALL_ON) - `uvm_field_int(data, UVM_ALL_ON) - `uvm_object_utils_end - function new(string name = "completion"); - super.new(name); - endfunction - endclass - - `include "env.svh" - `include "test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/test.svh b/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/test.svh deleted file mode 100644 index 321d4415b..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/test.svh +++ /dev/null @@ -1,641 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - env test_env; - virtual coherence_control_if control; - virtual stream_if #(`COH_REQ_WIDTH) req; - int serial = 1; - int reused_txn; - int expected = 0; - longint unsigned eviction_addresses[`COH_MSHRS]; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 300us; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - test_env = env::type_id::create("env", this); - if (!uvm_config_db#(virtual coherence_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`COH_REQ_WIDTH))::get( - this, "", "req", req - )) - `uvm_fatal("VIF", "Coherence test interfaces missing") - if (`COH_AGENTS < 2 || `COH_SETS < `COH_MSHRS || `COH_WAYS != 2) - `uvm_fatal("CONFIG", - "This regression requires two CPUs, two ways, and at least one set per MSHR") - endfunction - function longint unsigned addr(int tag, int set_index); - return (longint'(tag) * `COH_SETS + set_index) * 64; - endfunction - function bit [511:0] pattern(int seed); - bit [511:0] value; - for (int word = 0; word < 16; word++) value[word*32+:32] = seed ^ (32'h01010101 * word); - return value; - endfunction - task send(int node, int opcode, longint unsigned address, bit error = 0); - req_t packet = '0; - int txn = serial & 4095; - bit read = opcode == `COH_READ_SHARED || opcode == `COH_READ_NSD || opcode == `COH_READ_UNIQUE; - packet[`CF(REQ, QOS)] = serial & 15; - packet[`CF(REQ, LPID)] = serial & 31; - packet[`CF(REQ, SRCID)] = node; - packet[`CF(REQ, TGTID)] = `COH_HOME; - packet[`CF(REQ, TXNID)] = txn; - packet[`CF(REQ, OPCODE)] = opcode; - packet[`CF(REQ, ADDR)] = address; - packet[`CF(REQ, SIZE)] = 6; - packet[`CF(REQ, EXPCOMPACK)] = read; - packet[`CF(REQ, SNPATTR)] = 1; - packet[`CF(REQ, MEMATTR)] = 12; - packet[`CF(REQ, ALLOWRETRY)] = 1; - test_env.expected_error[(node<<12)|txn] = error; - req.bits <= packet; - req.valid <= 1; - do @(req.sample); while (!req.sample.ready); - req.valid <= 0; - serial++; - expected++; - endtask - task settle(int occupied = 0); - test_env.scoreboard.wait_checked(expected); - do @(control.cb); while (control.cb.outstanding != occupied); - endtask - task execute(); - control.reset = 1; - req.valid = 0; - req.bits = '0; - repeat (4) @(control.cb); - control.cb.reset <= 0; - @(control.cb); - // Idle payload is unconstrained: toggling it must not allocate a transaction or consume a response. - test_env.vary_idle_payload = 1; - test_env.idle_payload_one = 1; - req.bits <= '1; - repeat (3) @(control.cb); - test_env.idle_payload_one = 0; - req.bits <= '0; - repeat (3) @(control.cb); - test_env.vary_idle_payload = 0; - begin - req_t idle = '0; - idle[`CF(REQ, TGTID)] = `COH_HOME; - idle[`CF(REQ, SRCID)] = 1; - idle[`CF(REQ, OPCODE)] = `COH_READ_SHARED; - idle[`CF(REQ, SIZE)] = 6; - idle[`CF(REQ, MEMATTR)] = 12; - idle[`CF(REQ, SNPATTR)] = 1; - idle[`CF(REQ, ALLOWRETRY)] = 1; - idle[`CF(REQ, EXPCOMPACK)] = 1; - // Alter one unconstrained idle bit at a time, independently of the other access attributes. - for (int bit_index = 0; bit_index < `COH_REQ_WIDTH; bit_index++) begin - req.bits <= idle ^ (req_t'(1) << bit_index); - @(control.cb); - end - req.bits <= '0; - @(control.cb); - end - if (control.cb.outstanding != 0 || test_env.scoreboard.checked != 0) - `uvm_fatal("IDLE", "Invalid channel payload changed transaction state") - for (int slot = 0; slot < `COH_MSHRS; slot++) begin - longint unsigned a = addr(8 * slot, slot); - longint unsigned b = addr(8 * slot + 1, slot); - longint unsigned c = addr(8 * slot + 2, slot); - int saved; - test_env.hold_ack = (1 << slot) - 1; - for (int h = 0; h < slot; h++) send(1, `COH_READ_SHARED, addr(100 + slot, h)); - settle(slot); - send(1, `COH_READ_SHARED, a); - settle(slot); - send(2, `COH_READ_NSD, a); - settle(slot); - test_env.cpus[1][a].valid = 0; - send(1, `COH_READ_UNIQUE, a); - settle(slot); - send(2, `COH_READ_SHARED, a); - settle(slot); - send(1, `COH_READ_UNIQUE, a); - settle(slot); - test_env.cpu_write(1, a, '1); - send(2, `COH_READ_SHARED, a); - send(1, `COH_WRITEBACK, a); - settle(slot); - send(1, `COH_READ_UNIQUE, a); - send(2, `COH_EVICT, a); - settle(slot); - test_env.cpu_write(1, a, pattern(32'h12345678)); - send(2, `COH_READ_SHARED, a); - send(1, `COH_WRITEBACK, a); - settle(slot); - send(1, `COH_READ_SHARED, b); - settle(slot); - send(2, `COH_READ_SHARED, c); - settle(slot); - if (test_env.cpus[1][a].valid) - `uvm_fatal("INCLUSION", "inclusive victim survived replacement") - send(1, `COH_READ_SHARED, a); - settle(slot); - send(2, `COH_CLEAN_INVALID, a); - settle(slot); - send(1, `COH_CLEAN_INVALID, a); - settle(slot); - send(1, `COH_EVICT, a); - settle(slot); - send(1, `COH_WRITEBACK, a); - settle(slot); - send(1, `COH_READ_UNIQUE, a); - settle(slot); - test_env.cpu_write(1, a, pattern(32'h87654321)); - send(2, `COH_CLEAN_INVALID, a); - settle(slot); - send(2, `COH_READ_SHARED, a); - settle(slot); - test_env.fault_addr = addr(8 * slot + 3, slot); - test_env.fault_enabled = 1; - send(1, `COH_READ_SHARED, test_env.fault_addr, 1); - settle(slot); - test_env.fault_enabled = 0; - send(1, `COH_READ_SHARED, test_env.fault_addr); - settle(slot); - begin - longint unsigned high = (((64'h1 << `COH_REQ_ADDR_WIDTH) - 1) & - ~(longint'(`COH_SETS * 64) - 1)) | (slot * 64); - send(1, `COH_READ_UNIQUE, high); - settle(slot); - send(1, `COH_EVICT, high); - settle(slot); - send(1, `COH_READ_UNIQUE, high); - settle(slot); - test_env.cpu_write(1, high, '0); - send(1, `COH_WRITEBACK, high); - settle(slot); - send(2, `COH_CLEAN_INVALID, high); - settle(slot); - send(2, `COH_READ_SHARED, high); - settle(slot); - send(1, `COH_CLEAN_INVALID, high); - settle(slot); - send(1, `COH_READ_SHARED, addr(8 * slot + 7, slot)); - settle(slot); - end - send(1, `COH_READ_SHARED, addr(1750 + slot, slot)); - settle(slot); - send(2, `COH_READ_SHARED, addr(1750 + slot, slot)); - settle(slot); - send(1, `COH_EVICT, addr(1750 + slot, slot)); - settle(slot); - send(2, `COH_CLEAN_INVALID, addr(1750 + slot, slot)); - settle(slot); - saved = serial; - serial = 4095; - send(2, `COH_READ_SHARED, a); - settle(slot); - send(2, `COH_READ_SHARED, a); - settle(slot); - serial = 4095; - send(2, `COH_CLEAN_INVALID, a); - settle(slot); - serial = 4095; - send(2, `COH_WRITEBACK, a); - settle(slot); - serial = 4095; - send(2, `COH_EVICT, a); - settle(slot); - serial = saved; - test_env.hold_ack = 0; - settle(); - end - - test_env.hold_ack = '1; - send(1, `COH_READ_SHARED, addr(200, 0)); - test_env.scoreboard.wait_checked(expected); - fork - begin - send(2, `COH_READ_SHARED, addr(201, 0)); - end - begin - repeat (4) @(control.cb); - if (req.ready !== 0 || control.cb.outstanding != 1) - `uvm_fatal("LOCK", "same-set request entered before CompAck") - test_env.hold_ack = 0; - end - join - settle(); - test_env.hold_ack = '1; - test_env.hold_memory = 1; - reused_txn = serial; - send(1, `COH_READ_SHARED, addr(202, 0)); - repeat (8) @(control.cb); - serial = reused_txn; - fork - begin - send(1, `COH_READ_SHARED, addr(203, 1)); - end - begin - repeat (4) @(control.cb); - if (req.ready !== 0 || control.cb.outstanding != 1) - `uvm_fatal("ID_LIFETIME", "Requester TxnID was reused before its final response") - test_env.hold_memory = 0; - end - join - test_env.scoreboard.wait_checked(expected); - do @(control.cb); while (control.cb.outstanding != 2); - test_env.hold_ack = 0; - settle(); - for (int rotation = 0; rotation < `COH_SETS; rotation++) begin - for (int pass = 0; pass < `COH_WAYS; pass++) begin - test_env.hold_memory = 1; - test_env.hold_mem_req = 1; - test_env.reverse_memory = 1; - test_env.saw_reorder = 0; - for (int slot = 0; slot < `COH_MSHRS; slot++) begin - eviction_addresses[slot] = - addr(300 + rotation * `COH_WAYS + pass, (slot + rotation) % `COH_SETS); - send((slot % 2) + 1, `COH_READ_UNIQUE, eviction_addresses[slot]); - end - repeat (20) @(control.cb); - if (control.cb.outstanding != `COH_MSHRS) - `uvm_fatal("PARALLEL", "different sets did not occupy independent MSHRs") - test_env.hold_mem_req = 0; - repeat (16) @(control.cb); - test_env.hold_memory = 0; - settle(); - if (`COH_MSHRS > 1 && !test_env.saw_reorder) - `uvm_fatal("ORDER", "out-of-order response scenario did not execute") - end - end - for (int slot = 0; slot < `COH_MSHRS; slot++) - send((slot % 2) + 1, `COH_EVICT, eviction_addresses[slot]); - settle(); - // Delayed eviction notifications must not remove a newer owner's directory state. - for (int slot = 0; slot < `COH_MSHRS; slot++) - send((slot % 2) + 1, `COH_EVICT, addr( - 8 * ((slot + `COH_SETS - 1) % `COH_SETS), (slot + `COH_SETS - 1) % `COH_SETS)); - settle(); - for (int slot = 0; slot < `COH_MSHRS; slot++) begin - longint unsigned previous_line = eviction_addresses[slot] - `COH_SETS * 64; - test_env.cpu_write((slot % 2) + 1, previous_line, {16{32'h01020300 + slot}}); - send(((slot + 1) % 2) + 1, `COH_READ_SHARED, previous_line); - end - settle(); - for (int slot = 0; slot < `COH_MSHRS; slot++) - send(1, `COH_CLEAN_INVALID, eviction_addresses[slot] - `COH_SETS * 64); - settle(); - - // Fill the completion queue before each selected worker reaches completion. - for (int slot = 0; slot < `COH_MSHRS - 1; slot++) begin - longint unsigned slow = addr(900 + slot, slot); - send(1, `COH_READ_UNIQUE, slow); - settle(); - test_env.cpu_write(1, slow, pattern(32'h76543210)); - test_env.hold_ack = (1 << slot) - 1; - for (int h = 0; h < slot; h++) send(1, `COH_READ_SHARED, addr(950 + slot, h)); - settle(slot); - test_env.hold_memory = 1; - test_env.hold_rsp = 1; - send(2, `COH_CLEAN_INVALID, slow); - send(2, `COH_EVICT, addr(8 * (`COH_SETS - 1), `COH_SETS - 1)); - send(2, `COH_EVICT, addr(8 * (`COH_SETS - 1), `COH_SETS - 1)); - repeat (20) @(control.cb); - test_env.hold_memory = 0; - repeat (20) @(control.cb); - test_env.hold_rsp = 0; - settle(slot); - test_env.hold_ack = 0; - settle(); - end - - // Each victim has two sharers, so a worker can wait to issue its second snoop. - for (int s = 0; s < `COH_MSHRS; s++) begin - send(1, `COH_READ_SHARED, addr(700, s)); - settle(); - send(2, `COH_READ_SHARED, addr(700, s)); - settle(); - send(1, `COH_READ_SHARED, addr(701, s)); - settle(); - end - begin - int before_snoops = test_env.snoops; - fork - begin - for (int s = 0; s < `COH_MSHRS; s++) send(1, `COH_READ_SHARED, addr(702, s)); - end - begin - wait (test_env.snoops > before_snoops); - test_env.hold_snp = 1; - repeat (12) @(control.cb); - test_env.hold_snp = 0; - end - join - end - settle(); - // Align a returned memory operation with a new lookup to exercise every Cache arbiter input. - for (int slot = 0; slot < `COH_MSHRS; slot++) begin - int other; - test_env.hold_ack = (1 << slot) - 1; - for (int h = 0; h < slot; h++) send(1, `COH_READ_SHARED, addr(960 + slot, h)); - settle(slot); - test_env.hold_memory = 1; - send(1, `COH_READ_SHARED, addr(980 + slot, slot)); - wait (test_env.mem_queue.size() != 0); - if (slot == `COH_MSHRS - 1) begin - test_env.hold_ack[0] = 0; - do @(control.cb); while (control.cb.outstanding != slot); - other = 0; - end else other = slot + 1; - test_env.hold_memory = 0; - @(control.cb); - send(2, `COH_READ_SHARED, addr(990 + slot, other)); - settle($countones(test_env.hold_ack)); - test_env.hold_ack = 0; - settle(); - end - - // A recalled dirty line reaches memory after two independent reads fill the request queue. - for (int slot = 0; slot < 2; slot++) begin - longint unsigned slow = addr(1100 + slot, slot); - control.cb.reset <= 1; - repeat (2) @(control.cb); - control.cb.reset <= 0; - @(control.cb); - send(1, `COH_READ_UNIQUE, slow); - settle(); - test_env.cpu_write(1, slow, pattern(32'hFEDCBA98)); - test_env.hold_ack = (1 << slot) - 1; - for (int h = 0; h < slot; h++) send(1, `COH_READ_SHARED, addr(1150 + slot, h)); - settle(slot); - test_env.hold_snp = 1; - test_env.hold_mem_req = 1; - send(2, `COH_CLEAN_INVALID, slow); - send(1, `COH_READ_SHARED, addr(1200 + slot, slot + 1)); - send(2, `COH_READ_SHARED, addr(1250 + slot, slot + 2)); - repeat (16) @(control.cb); - test_env.hold_snp = 0; - repeat (16) @(control.cb); - test_env.hold_mem_req = 0; - settle(slot); - test_env.hold_ack = 0; - settle(); - end - for (int slot = 0; slot < `COH_MSHRS; slot++) begin - longint unsigned a = addr(1500 + 3 * slot, slot); - longint unsigned b = addr(1501 + 3 * slot, slot); - longint unsigned c = addr(1502 + 3 * slot, slot); - test_env.hold_ack = (1 << slot) - 1; - for (int h = 0; h < slot; h++) send(1, `COH_READ_SHARED, addr(1600 + slot, h)); - settle(slot); - send(1, `COH_READ_UNIQUE, a); - settle(slot); - test_env.cpu_write(1, a, pattern(32'hDEADBEEF)); - send(2, `COH_READ_SHARED, b); - settle(slot); - test_env.fault_addr = a; - test_env.write_fault_enabled = 1; - send(2, `COH_READ_SHARED, c, 1); - settle(slot); - test_env.write_fault_enabled = 0; - send(1, `COH_READ_SHARED, a); - settle(slot); - send(2, `COH_READ_SHARED, c); - settle(slot); - send(2, `COH_CLEAN_INVALID, a); - settle(slot); - send(1, `COH_READ_UNIQUE, a); - settle(slot); - test_env.cpu_write(1, a, pattern(32'h01234567)); - test_env.write_fault_enabled = 1; - send(1, `COH_CLEAN_INVALID, a, 1); - settle(slot); - test_env.write_fault_enabled = 0; - send(2, `COH_READ_SHARED, a); - settle(slot); - send(1, `COH_CLEAN_INVALID, a); - settle(slot); - test_env.hold_ack = 0; - settle(); - end - for (int slot = 0; slot < `COH_MSHRS; slot++) begin - test_env.hold_ack = (1 << slot) - 1; - for (int h = 0; h < slot; h++) send(1, `COH_READ_SHARED, addr(1700 + slot, h)); - settle(slot); - `uvm_info("ADDR_SWEEP", $sformatf( - "begin slot=%0d checked=%0d", slot, test_env.scoreboard.checked), UVM_LOW) - for (int bit_index = 8; bit_index < `COH_REQ_ADDR_WIDTH; bit_index++) begin - longint unsigned a = (64'h1 << bit_index) | (longint'(slot) << 6); - bit [511:0] value = {8{64'h1 << ((bit_index - 8) % 64)}}; - send(1, `COH_READ_UNIQUE, a); - settle(slot); - test_env.cpu_write(1, a, value); - send(2, `COH_READ_SHARED, a); - settle(slot); - send(1, `COH_CLEAN_INVALID, a); - settle(slot); - send(2, `COH_READ_SHARED, a); - settle(slot); - end - test_env.hold_ack = 0; - settle(); - end - // Both resident ways must see the high address bits, independently of replacement parity. - for (int set_index = 0; set_index < `COH_SETS; set_index++) begin - longint unsigned high = (((64'h1 << `COH_REQ_ADDR_WIDTH) - 1) & - ~(longint'(`COH_SETS * 64) - 1)) | (longint'(set_index) << 6); - send(1, `COH_READ_SHARED, high); - settle(); - send(2, `COH_READ_SHARED, high ^ (`COH_SETS * 64)); - settle(); - send(1, `COH_READ_SHARED, addr(2000, set_index)); - settle(); - send(2, `COH_READ_SHARED, addr(2001, set_index)); - settle(); - send(1, `COH_READ_SHARED, addr(2001, set_index)); - settle(); - send(1, `COH_EVICT, addr(2001, set_index)); - settle(); - send(2, `COH_CLEAN_INVALID, addr(2001, set_index)); - settle(); - end - // True CMO: clean retains cache copies; invalidate-after-DMA must discard stale dirty bytes. - begin - longint unsigned a = addr(2300, 0); - bit [511:0] dirty_value = pattern(32'habbacd01), dma_value = pattern(32'hd00dbeef); - int before_write, before_read; - send(1, `COH_READ_UNIQUE, a); - settle(); - test_env.cpu_write(1, a, dirty_value); - before_write = test_env.memory_writes; - send(2, `COH_CLEAN_SHARED, a); - settle(); - if (test_env.memory_writes != before_write + 1 || test_env.memory[a] !== dirty_value || - !test_env.cpus[0][a].valid || test_env.cpus[0][a].dirty || test_env.cpus[0][a].unique_owner) - `uvm_fatal("CLEAN_SHARED", "Dirty clean did not retain SC data and make DDR visible") - before_read = test_env.memory_reads; - before_write = test_env.memory_writes; - send(1, `COH_CLEAN_SHARED, a); - settle(); - send(2, `COH_READ_SHARED, a); - settle(); - if (test_env.memory_reads != before_read || test_env.memory_writes != before_write) - `uvm_fatal("CLEAN_HIT", "CleanShared evicted a valid line or wrote an already-clean line") - send(1, `COH_READ_UNIQUE, a); - settle(); - test_env.cpu_write(1, a, pattern(32'hfa11ed00)); - test_env.fault_addr = a; - test_env.write_fault_enabled = 1; - send(2, `COH_CLEAN_SHARED, a, 1); - settle(); - test_env.write_fault_enabled = 0; - if (!test_env.dirty_addresses.exists(a) || test_env.memory[a] !== dirty_value) - `uvm_fatal("CLEAN_ERROR", "Failed clean lost dirty data or falsely changed DDR") - send(2, `COH_READ_SHARED, a); - settle(); - send(2, `COH_CLEAN_SHARED, a); - settle(); - if (test_env.memory[a] !== pattern(32'hfa11ed00)) - `uvm_fatal("CLEAN_RETRY", "Restored dirty line did not reach DDR on later clean") - send(1, `COH_READ_UNIQUE, a); - settle(); - test_env.cpu_write(1, a, dirty_value); - test_env.memory[a] = dma_value; - coherence_ref_write(test_env.model, a, dma_value); - test_env.dirty_addresses.delete(a); - before_write = test_env.memory_writes; - send(2, `COH_MAKE_INVALID, a); - settle(); - if (test_env.memory_writes != before_write || test_env.memory[a] !== dma_value || - test_env.cpus[0][a].valid || test_env.cpus[1][a].valid) - `uvm_fatal("MAKE_INVALID", - "Post-DMA invalidation wrote stale cache data or retained a copy") - before_read = test_env.memory_reads; - send(1, `COH_READ_SHARED, a); - settle(); - if (test_env.memory_reads != before_read + 1) - `uvm_fatal("INVALID_REFILL", "Invalidated line did not refill fresh DDR data") - send(1, `COH_MAKE_INVALID, a); - settle(); - send(1, `COH_CLEAN_SHARED, a); - settle(); - send(1, `COH_MAKE_INVALID, a); - settle(); - `uvm_info( - "CMO", - "CleanShared DDR visibility/retention/error restoration and MakeInvalid discard checked", - UVM_LOW) - end - // Exercise the new maintenance paths in every real Home transaction slot. - begin - longint unsigned addresses[`COH_MSHRS]; - bit [511:0] values[`COH_MSHRS]; - int before_write; - for (int i = 0; i < `COH_MSHRS; i++) begin - addresses[i] = addr(2400 + i, i); - values[i] = pattern(32'h13572468 ^ i); - send(1, `COH_READ_UNIQUE, addresses[i]); - settle(); - test_env.cpu_write(1, addresses[i], values[i]); - end - before_write = test_env.memory_writes; - test_env.hold_memory = 1; - for (int i = 0; i < `COH_MSHRS; i++) send(2, `COH_CLEAN_SHARED, addresses[i]); - do @(control.cb); while (control.cb.outstanding != `COH_MSHRS); - repeat (8) @(control.cb); - test_env.hold_memory = 0; - settle(); - if (test_env.memory_writes != before_write + `COH_MSHRS) - `uvm_fatal("CMO_PARALLEL", "Every dirty clean did not receive a real DDR write") - for (int i = 0; i < `COH_MSHRS; i++) begin - if (test_env.memory[addresses[i]] !== values[i] || !test_env.cpus[0][addresses[i]].valid) - `uvm_fatal("CMO_PARALLEL", - "Parallel clean lost dirty data or invalidated a retained copy") - send(1, `COH_READ_UNIQUE, addresses[i]); - settle(); - test_env.cpu_write(1, addresses[i], pattern(32'hdeadcafe ^ i)); - values[i] = pattern(32'h02468ace ^ i); - test_env.memory[addresses[i]] = values[i]; - coherence_ref_write(test_env.model, addresses[i], values[i]); - test_env.dirty_addresses.delete(addresses[i]); - end - before_write = test_env.memory_writes; - test_env.hold_snp = 1; - for (int i = 0; i < `COH_MSHRS; i++) send(2, `COH_MAKE_INVALID, addresses[i]); - do @(control.cb); while (control.cb.outstanding != `COH_MSHRS); - repeat (8) @(control.cb); - test_env.hold_snp = 0; - settle(); - if (test_env.memory_writes != before_write) - `uvm_fatal("CMO_DISCARD", "Parallel MakeInvalid overwrote new DDR with stale dirty data") - for (int i = 0; i < `COH_MSHRS; i++) begin - send(1, `COH_READ_SHARED, addresses[i]); - settle(); - send(2, `COH_MAKE_INVALID, addresses[i]); - settle(); - end - end - // Initial caller IDs can be reused after their final response while old DBID/set ownership survives. - begin - longint unsigned a = addr(2600, 0), b = addr(2601, 1); - longint unsigned c = addr(2602, 2), d = addr(2603, 3); - int caller; - req_t probe = '0; - send(1, `COH_READ_UNIQUE, a); - settle(); - test_env.cpu_write(1, a, pattern(32'hc011ab1e)); - test_env.hold_copyback = 1; - caller = serial; - send(1, `COH_WRITEBACK, a); - settle(1); - serial = caller; - send(1, `COH_READ_SHARED, b); - settle(1); - // The old DBID is still live: even an unrelated caller cannot allocate the old set. - probe[`CF(REQ, ADDR)] = addr(2700, 0); - probe[`CF(REQ, SRCID)] = 2; - probe[`CF(REQ, TXNID)] = serial & 4095; - req.bits <= probe; - repeat (4) begin - @(req.sample); - if (req.sample.ready) - `uvm_fatal("MSHR_SET", "Caller retirement released the old DBID/set too early") - end - test_env.hold_copyback = 0; - do @(control.cb); while (control.cb.outstanding != 0); - send(1, `COH_READ_SHARED, a); - settle(); - send(1, `COH_CLEAN_INVALID, a); - settle(); - test_env.hold_ack = '1; - caller = serial; - send(1, `COH_READ_SHARED, c); - settle(1); - serial = caller; - send(1, `COH_READ_SHARED, d); - settle(2); - test_env.hold_ack = 0; - do @(control.cb); while (control.cb.outstanding != 0); - `uvm_info( - "CALLER_TXN", - "WB caller reuse before late DBID data and read caller reuse before late CompAck checked", - UVM_LOW) - end - if (test_env.memory_writes == 0) `uvm_fatal("WRITEBACK", "no dirty victim reached memory") - // Reset at retirement after all returned data was checked; the clean read has no dirty state. - test_env.hold_ack = '1; - send(1, `COH_READ_SHARED, addr(2100, 0)); - settle(1); - test_env.hold_ack = 0; - do - @(control.cb); - while (!test_env.rx_rsp.valid || test_env.rx_rsp.bits[ - `CF(RSP, OPCODE) - ] != `COH_COMP_ACK); - @(negedge control.clock); - control.reset <= 1; - repeat (2) @(control.cb); - control.cb.reset <= 0; - @(control.cb); - send(1, `COH_READ_SHARED, addr(0, 0)); - settle(); - `uvm_info("COHERENCE", $sformatf("Checked %0d completions, peak MSHRs=%0d, dirty writes=%0d", - test_env.scoreboard.checked, test_env.peak, - test_env.memory_writes), UVM_LOW) - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Emit.scala deleted file mode 100644 index 6dc07f0ed..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Emit.scala +++ /dev/null @@ -1,110 +0,0 @@ -package memcore.memory.coherence - -import java.nio.file.{Files, Paths} -import memcore.bus.chi._ -import memcore.memory.coherence.configs.CoherenceParams - -object Emit extends App { - val p = CoherenceParams.load("configs/default.toml") - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Coherence(p), - firtoolOpts = args, - args = Array("--target-dir", "build/Coherence", "--split-verilog") - ) - val definitions = scala.collection.mutable.ArrayBuffer[String]() - val ports = scala.collection.mutable.ArrayBuffer[String]() - - def fields(flit: Flit): Seq[(String, Int)] = flit.fieldsLSB.map { field => - flit.elements.find(_._2 eq field).get._1 -> field.getWidth - } - - def channel( - kind: String, - prefix: String, - instance: String, - fs: Seq[(String, Int)] - ): Unit = { - var offset = 0 - for ((name, width) <- fs if width > 0) { - definitions += s"`define COH_${kind}_${name.toUpperCase}_OFFSET $offset" - definitions += s"`define COH_${kind}_${name.toUpperCase}_WIDTH $width" - ports += s".$prefix$name($instance.bits[$offset +: $width])" - offset += width - } - definitions += s"`define COH_${kind}_WIDTH $offset" - } - - channel("REQ", "io_req_bits_", "source_if", fields(new RequestFlit(p.chi))) - channel("RSP", "io_rxRsp_bits_", "rx_rsp", fields(new ResponseFlit(p.chi))) - channel("DAT", "io_rxDat_bits_", "rx_dat", fields(new DataFlit(p.chi))) - channel("TXRSP", "io_rsp_bits_", "tx_rsp", fields(new ResponseFlit(p.chi))) - channel("TXDAT", "io_dat_bits_", "sink_if", fields(new DataFlit(p.chi))) - channel("SNP", "io_snp_bits_flit_", "tx_snp", fields(new SnoopFlit(p.chi))) - val snpWidth = (new SnoopFlit(p.chi)).flitWidth - ports += s".io_snp_bits_targetNode(tx_snp.bits[$snpWidth +: ${p.chi.nodeIdBits}])" - definitions += s"`define COH_SNP_TARGET_OFFSET $snpWidth" - definitions += s"`define COH_SNP_TARGET_WIDTH ${p.chi.nodeIdBits}" - definitions += s"`define COH_SNP_CHANNEL_WIDTH ${snpWidth + p.chi.nodeIdBits}" - channel( - "MEMREQ", - "io_memoryReq_bits_", - "mem_req", - Seq("id" -> p.chi.txnIdBits, "addr" -> p.chi.addressBits, "write" -> 1, "data" -> 512, "mask" -> 64) - ) - channel("MEMRESP", "io_memoryResp_bits_", "mem_resp", Seq("id" -> p.chi.txnIdBits, "data" -> 512, "error" -> 1)) - for ( - (port, instance) <- Seq( - "req" -> "source_if", - "rxRsp" -> "rx_rsp", - "rxDat" -> "rx_dat", - "rsp" -> "tx_rsp", - "dat" -> "sink_if", - "snp" -> "tx_snp", - "memoryReq" -> "mem_req", - "memoryResp" -> "mem_resp" - ) - ) { - ports += s".io_${port}_valid($instance.valid)" - ports += s".io_${port}_ready($instance.ready)" - } - for ( - (name, value) <- Seq( - "AGENTS" -> p.agents, - "MSHRS" -> p.mshrEntries, - "HOME" -> p.homeId, - "SETS" -> p.cache.sets, - "WAYS" -> p.cache.ways, - "BEATS" -> p.chi.beatsPerLine, - "DATA_BITS" -> p.chi.dataBits, - "OUTSTANDING_BITS" -> chisel3.util.log2Ceil(p.mshrEntries + 1), - "READ_SHARED" -> Opcode.ReadShared, - "READ_NSD" -> Opcode.ReadNotSharedDirty, - "READ_UNIQUE" -> Opcode.ReadUnique, - "EVICT" -> Opcode.Evict, - "WRITEBACK" -> Opcode.WriteBackFull, - "CLEAN_SHARED" -> Opcode.CleanShared, - "CLEAN_INVALID" -> Opcode.CleanInvalid, - "MAKE_INVALID" -> Opcode.MakeInvalid, - "COMP" -> Opcode.Comp, - "COMP_DBID" -> Opcode.CompDBIDResp, - "COMP_ACK" -> Opcode.CompAck, - "COMP_DATA" -> Opcode.CompData, - "SNP_RESP" -> Opcode.SnpResp, - "SNP_DATA" -> Opcode.SnpRespData, - "COPYBACK_DATA" -> Opcode.CopyBackWriteData, - "SNP_UNIQUE" -> Opcode.SnpUnique, - "SNP_INVALID" -> Opcode.SnpCleanInvalid, - "SNP_CLEAN_SHARED" -> Opcode.SnpCleanShared, - "SNP_MAKE_INVALID" -> Opcode.SnpMakeInvalid, - "SNP_SHARED" -> Opcode.SnpNotSharedDirty - ) - ) { - definitions += s"`define COH_$name $value" - } - Files.writeString( - Paths.get("build/coherence_config.svh"), - "`ifndef COHERENCE_CONFIG_SVH\n`define COHERENCE_CONFIG_SVH\n" + definitions.mkString("\n") + "\n`endif\n" - ) - Files.writeString(Paths.get("build/coherence_ports.svh"), ports.mkString(",\n") + "\n") - -} diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Interface.scala b/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Interface.scala deleted file mode 100644 index 02373d235..000000000 --- a/arch/src/main/scala/framework/mem-core/coherence/src/main/scala/Interface.scala +++ /dev/null @@ -1,19 +0,0 @@ -package memcore.memory.coherence - -import chisel3._ -import chisel3.util._ -import memcore.bus.chi._ -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.coherence.configs.CoherenceParams - -class CoherenceIO(p: CoherenceParams) extends Bundle { - val req = Flipped(Decoupled(new RequestFlit(p.chi))) - val rxRsp = Flipped(Decoupled(new ResponseFlit(p.chi))) - val rxDat = Flipped(Decoupled(new DataFlit(p.chi))) - val rsp = Decoupled(new ResponseFlit(p.chi)) - val dat = Decoupled(new DataFlit(p.chi)) - val snp = Decoupled(new DirectedSnoop(p.chi)) - val memoryReq = Decoupled(new LineRequest(p.chi)) - val memoryResp = Flipped(Decoupled(new LineResponse(p.chi))) - val outstanding = Output(UInt(log2Ceil(p.mshrEntries + 1).W)) -} diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/build.sc b/arch/src/main/scala/framework/mem-core/cpu_mem/build.sc deleted file mode 100644 index d78b5cebf..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/build.sc +++ /dev/null @@ -1,35 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object chi extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = os.pwd / os.up / "chi" -} - -object mmu extends ChiselModule { - override def moduleRoot = os.pwd / os.up / "mmu" - override def moduleDeps = Seq(chi) -} - -object cpu_mem extends ChiselModule { - override def mainClass = Some("memcore.memory.cpu.Emit") - override def moduleRoot = os.pwd - override def moduleDeps = Seq(chi, mmu) -} diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/Cargo.lock b/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/Cargo.lock deleted file mode 100644 index 33d3bc9f5..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "cpu-mem-reference-model" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/Cargo.toml b/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/Cargo.toml deleted file mode 100644 index 870e514dd..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/Cargo.toml +++ /dev/null @@ -1,7 +0,0 @@ -[package] -name = "cpu-mem-reference-model" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib"] diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/src/lib.rs b/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/src/lib.rs deleted file mode 100644 index 34957a641..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/csrc/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod access; -pub use access::*; diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem.f b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem.f deleted file mode 100644 index 037da65d2..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/CpuMem/filelist.f -@RESOURCES@/cpu_mem_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic.f b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic.f deleted file mode 100644 index 587362b3e..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+CPU_MEM_TOP=cpu_mem_bad_atomic_tb -+define+CPU_BAD_CASE=3 -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/CpuMem/filelist.f -@RESOURCES@/cpu_mem_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic.toml b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic.toml deleted file mode 100644 index 3e461fd78..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CPU memory port does not accept Fence or unknown atomics" diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic_write.f b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic_write.f deleted file mode 100644 index 9c3eff1c4..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic_write.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+CPU_MEM_TOP=cpu_mem_bad_atomic_write_tb -+define+CPU_BAD_CASE=5 -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/CpuMem/filelist.f -@RESOURCES@/cpu_mem_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic_write.toml b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic_write.toml deleted file mode 100644 index 72c087483..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_atomic_write.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CPU atomic requires a 32 or 64 bit operand" diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_fence.f b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_fence.f deleted file mode 100644 index 5e7cdeba1..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_fence.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+CPU_MEM_TOP=cpu_mem_bad_fence_tb -+define+CPU_BAD_CASE=2 -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/CpuMem/filelist.f -@RESOURCES@/cpu_mem_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_fence.toml b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_fence.toml deleted file mode 100644 index 3e461fd78..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_fence.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CPU memory port does not accept Fence or unknown atomics" diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_operand.f b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_operand.f deleted file mode 100644 index 44169ca32..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_operand.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+CPU_MEM_TOP=cpu_mem_bad_operand_tb -+define+CPU_BAD_CASE=4 -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/CpuMem/filelist.f -@RESOURCES@/cpu_mem_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_operand.toml b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_operand.toml deleted file mode 100644 index 72c087483..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_operand.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CPU atomic requires a 32 or 64 bit operand" diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_size.f b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_size.f deleted file mode 100644 index 2511563fd..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_size.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+CPU_MEM_TOP=cpu_mem_bad_size_tb -+define+CPU_BAD_CASE=1 -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/CpuMem/filelist.f -@RESOURCES@/cpu_mem_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_size.toml b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_size.toml deleted file mode 100644 index d3ba2a56f..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_size.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CPU memory size must be 1, 2, 4 or 8 bytes" diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_tag.f b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_tag.f deleted file mode 100644 index c793b9f0f..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_tag.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+CPU_MEM_TOP=cpu_mem_bad_tag_tb -+define+CPU_BAD_TAG -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/CpuMem/filelist.f -@RESOURCES@/cpu_mem_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_tag.toml b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_tag.toml deleted file mode 100644 index 090ec048b..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_bad_tag.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Uncached response tag mismatch" diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_tb.sv b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_tb.sv deleted file mode 100644 index d04d9df99..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/cpu_mem_tb.sv +++ /dev/null @@ -1,89 +0,0 @@ -`include "cpu_mem_config.svh" -`ifndef CPU_MEM_TOP -`define CPU_MEM_TOP cpu_mem_tb -`endif -module `CPU_MEM_TOP; - import uvm_pkg::*; - import cpu_mem_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - cpu_control_if control (clock); - stream_if #(`CPU_REQ_WIDTH) request ( - clock, - control.reset - ); - stream_if #(`CPU_RESP_WIDTH) response ( - clock, - control.reset - ); - stream_if #(`CPU_CACHE_WIDTH) cache_request ( - clock, - control.reset - ); - stream_if #(`CPU_CRESULT_WIDTH) cache_response ( - clock, - control.reset - ); - stream_if #(`CPU_UNCACHED_WIDTH) uncached_request ( - clock, - control.reset - ); - stream_if #(`CPU_URESULT_WIDTH) uncached_response ( - clock, - control.reset - ); - int unsigned in_flight = 0; - bit issued = 0, returned = 0, uncached = 0; - always @(posedge clock) begin - if (control.reset) begin - in_flight = 0; - issued = 0; - returned = 0; - end else begin - if (in_flight != 0 && request.ready) - $fatal(1, "CPU port became ready before consuming its response"); - if (request.valid && request.ready) begin - if (in_flight != 0) $fatal(1, "CPU memory accepted overlapping requests"); - in_flight = 1; - issued = 0; - returned = 0; - end - if ((cache_request.valid && cache_request.ready) || (uncached_request.valid && uncached_request.ready)) begin - if (in_flight != 1 || issued) $fatal(1, "Unexpected or duplicate backend request"); - issued = 1; - uncached = uncached_request.valid && uncached_request.ready; - end - if ((cache_response.valid && cache_response.ready) || (uncached_response.valid && uncached_response.ready)) begin - if (in_flight != 1 || !issued || returned || - uncached != (uncached_response.valid && uncached_response.ready)) - $fatal(1, "Unexpected or duplicate backend response"); - returned = 1; - end - if (response.valid && response.ready) begin - if (in_flight != 1 || (issued && !returned)) $fatal(1, "Unexpected CPU result"); - in_flight = 0; - end - end - end - CpuMem dut ( - .clock(clock), - .reset(control.reset), - `include "cpu_mem_ports.svh" - ); - initial begin - uvm_config_db#(virtual cpu_control_if)::set(null, "uvm_test_top*", "control", control); - uvm_config_db#(virtual stream_if #(`CPU_REQ_WIDTH))::set(null, "uvm_test_top*", "request", - request); - uvm_config_db#(virtual stream_if #(`CPU_RESP_WIDTH))::set(null, "uvm_test_top*", "response", - response); - uvm_config_db#(virtual stream_if #(`CPU_CACHE_WIDTH))::set(null, "uvm_test_top*", - "cache_request", cache_request); - uvm_config_db#(virtual stream_if #(`CPU_CRESULT_WIDTH))::set(null, "uvm_test_top*", - "cache_response", cache_response); - uvm_config_db#(virtual stream_if #(`CPU_UNCACHED_WIDTH))::set( - null, "uvm_test_top*", "uncached_request", uncached_request); - uvm_config_db#(virtual stream_if #(`CPU_URESULT_WIDTH))::set( - null, "uvm_test_top*", "uncached_response", uncached_response); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/package.sv deleted file mode 100644 index 8aa10cff2..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/package.sv +++ /dev/null @@ -1,36 +0,0 @@ -`include "cpu_mem_config.svh" -package cpu_mem_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `define CPUF(kind, field) `CPU_``kind``_``field``_OFFSET +: `CPU_``kind``_``field``_WIDTH - typedef bit [`CPU_REQ_WIDTH-1:0] req_t; - import "DPI-C" function void cpu_mem_ref_prepare( - input longint unsigned addr, - input int unsigned size, - write, - input longint unsigned data, - input int unsigned atomic_op, - cacheable, - normal, - address_bits, - output int unsigned target, - misaligned, - access_fault, - output longint unsigned bus_addr, - bus_data, - output int unsigned bus_mask, - atomic_word - ); - import "DPI-C" function longint unsigned cpu_mem_ref_result( - input longint unsigned addr, - input int unsigned size, - write, - signed_load, - atomic_op, - cacheable, - input longint unsigned raw, - input int unsigned error - ); - `include "test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/test.svh b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/test.svh deleted file mode 100644 index e1aea4692..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/resources/test.svh +++ /dev/null @@ -1,350 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual cpu_control_if control; - virtual stream_if #(`CPU_REQ_WIDTH) request; - virtual stream_if #(`CPU_RESP_WIDTH) response; - virtual stream_if #(`CPU_CACHE_WIDTH) cache_request; - virtual stream_if #(`CPU_CRESULT_WIDTH) cache_response; - virtual stream_if #(`CPU_UNCACHED_WIDTH) uncached_request; - virtual stream_if #(`CPU_URESULT_WIDTH) uncached_response; - int checked = 0, cache_checked = 0, uncached_checked = 0, faults_checked = 0, cancelled = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 200us; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual cpu_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`CPU_REQ_WIDTH))::get( - this, "", "request", request - ) || !uvm_config_db#(virtual stream_if #(`CPU_RESP_WIDTH))::get( - this, "", "response", response - ) || !uvm_config_db#(virtual stream_if #(`CPU_CACHE_WIDTH))::get( - this, "", "cache_request", cache_request - ) || !uvm_config_db#(virtual stream_if #(`CPU_CRESULT_WIDTH))::get( - this, "", "cache_response", cache_response - ) || !uvm_config_db#(virtual stream_if #(`CPU_UNCACHED_WIDTH))::get( - this, "", "uncached_request", uncached_request - ) || !uvm_config_db#(virtual stream_if #(`CPU_URESULT_WIDTH))::get( - this, "", "uncached_response", uncached_response - )) - `uvm_fatal("VIF", "CPU memory interfaces missing") - endfunction - task transact(longint unsigned addr, int size, bit write, bit signed_load, longint unsigned data, - int atomic_op, bit cacheable, longint unsigned raw = 64'h80ff7f0181fe8000, - bit error = 0, bit normal_ram = 0, bit device = 0); - req_t packet = '0; - int unsigned target, misaligned, access_fault, bus_mask, atomic_word; - longint unsigned bus_addr, bus_data, expected_data; - bit [`CPU_RESP_WIDTH-1:0] expected_response; - int tag = checked & ((1 << `CPU_TAG_BITS) - 1); - cpu_mem_ref_prepare(addr, size, write, data, atomic_op, cacheable, - !device && (cacheable || normal_ram), `CPU_PHYSICAL_BITS, target, - misaligned, access_fault, bus_addr, bus_data, bus_mask, atomic_word); - expected_data = target == 0 ? 0 : - cpu_mem_ref_result(addr, size, write, signed_load, atomic_op, cacheable, raw, error); - expected_response = '0; - expected_response[`CPUF(RESP, TAG)] = tag; - expected_response[`CPUF(RESP, DATA)] = expected_data; - expected_response[`CPUF(RESP, MISALIGNED)] = misaligned; - expected_response[`CPUF(RESP, ACCESSFAULT)] = access_fault || (target != 0 && error); - packet[`CPUF(REQ, ADDR)] = addr; - packet[`CPUF(REQ, TAG)] = tag; - packet[`CPUF(REQ, SIZE)] = size; - packet[`CPUF(REQ, WRITE)] = write; - packet[`CPUF(REQ, SIGNED)] = signed_load; - packet[`CPUF(REQ, DATA)] = data; - packet[`CPUF(REQ, ATOMIC)] = atomic_op; - packet[`CPUF(REQ, CACHEABLE)] = cacheable; - packet[`CPUF(REQ, NORMAL)] = !device && (cacheable || normal_ram); - @(control.cb); - response.ready <= 0; - cache_request.ready <= 0; - uncached_request.ready <= 0; - request.bits <= packet; - request.valid <= 1; - do @(request.sample); while (!request.sample.ready); - request.valid <= 0; - if (target == 1) begin - do @(cache_request.sample); while (!cache_request.sample.valid); - if (cache_request.sample.bits[ - `CPUF(CACHE, ADDR) - ] !== bus_addr || cache_request.sample.bits[ - `CPUF(CACHE, DATA) - ] !== bus_data || cache_request.sample.bits[ - `CPUF(CACHE, MASK) - ] !== bus_mask || cache_request.sample.bits[ - `CPUF(CACHE, WRITE) - ] !== write || cache_request.sample.bits[ - `CPUF(CACHE, ATOMIC) - ] !== atomic_op || cache_request.sample.bits[ - `CPUF(CACHE, ATOMICWORD) - ] !== atomic_word || uncached_request.valid) - `uvm_fatal("CACHE", $sformatf( - "Cache translation mismatch addr=%h size=%0d atomic=%0d got=%h", - addr, - size, - atomic_op, - cache_request.sample.bits - )) - repeat (3) @(control.cb); - cache_request.ready <= 1; - @(cache_request.sample); - cache_request.ready <= 0; - repeat (2) @(control.cb); - cache_response.bits <= '0; - cache_response.bits[`CPUF(CRESULT, DATA)] <= raw; - cache_response.bits[`CPUF(CRESULT, ERROR)] <= error; - cache_response.valid <= 1; - do @(cache_response.sample); while (!cache_response.sample.ready); - cache_response.valid <= 0; - cache_checked++; - end else if (target == 2) begin - cache_response.bits[`CPUF(CRESULT, DATA)] <= ~raw; - do @(uncached_request.sample); while (!uncached_request.sample.valid); - if (uncached_request.sample.bits[ - `CPUF(UNCACHED, ADDR) - ] !== addr || uncached_request.sample.bits[ - `CPUF(UNCACHED, SIZE) - ] !== size || uncached_request.sample.bits[ - `CPUF(UNCACHED, WRITE) - ] !== write || uncached_request.sample.bits[ - `CPUF(UNCACHED, DATA) - ] !== data || uncached_request.sample.bits[ - `CPUF(UNCACHED, TAG) - ] !== tag || cache_request.valid) - `uvm_fatal("MMIO", "Uncached access expanded or changed its address/size/data/tag") - repeat (3) @(control.cb); - uncached_request.ready <= 1; - @(uncached_request.sample); - uncached_request.ready <= 0; - repeat (2) @(control.cb); - uncached_response.bits <= '0; -`ifdef CPU_BAD_TAG - uncached_response.bits[`CPUF(URESULT, TAG)] <= tag ^ 1; -`else - uncached_response.bits[`CPUF(URESULT, TAG)] <= tag; -`endif - uncached_response.bits[`CPUF(URESULT, DATA)] <= raw; - uncached_response.bits[`CPUF(URESULT, ERROR)] <= error; - uncached_response.valid <= 1; - do @(uncached_response.sample); while (!uncached_response.sample.ready); - uncached_response.valid <= 0; - uncached_checked++; - end else begin - repeat (3) begin - @(control.cb); - if (cache_request.valid || uncached_request.valid) - `uvm_fatal("FAULT", "Faulting request reached an external memory port") - end - faults_checked++; - end - do @(response.sample); while (!response.sample.valid); - if (response.sample.bits !== expected_response) - `uvm_fatal("RESULT", $sformatf( - "CPU result mismatch addr=%h size=%0d atomic=%0d got=%h expected=%h", - addr, - size, - atomic_op, - response.sample.bits, - expected_response - )) - repeat (3) begin - @(control.cb); - if (request.ready || response.bits !== expected_response || !response.valid || - cache_request.valid || uncached_request.valid) - `uvm_fatal("SINGLE", "Outstanding CPU response changed or accepted another operation") - end - response.ready <= 1; - @(response.sample); - response.ready <= 0; - checked++; - endtask - // Reset jointly cancels the CPU operation and its memory response producer; stores are not rolled back. - task cancel_at(int phase_index, bit cacheable); - req_t packet = '0; - packet[`CPUF(REQ, ADDR)] = 64'h5000; - packet[`CPUF(REQ, SIZE)] = 3; - packet[`CPUF(REQ, TAG)] = 63; - packet[`CPUF(REQ, CACHEABLE)] = cacheable; - packet[`CPUF(REQ, NORMAL)] = cacheable; - @(control.cb); - request.bits <= packet; - request.valid <= 1; - cache_request.ready <= 0; - uncached_request.ready <= 0; - response.ready <= 0; - do @(request.sample); while (!request.sample.ready); - request.valid <= 0; - if (cacheable) begin - do @(cache_request.sample); while (!cache_request.sample.valid); - if (phase_index != 0) begin - cache_request.ready <= 1; - @(cache_request.sample); - cache_request.ready <= 0; - if (phase_index == 2) begin - cache_response.bits <= '0; - cache_response.valid <= 1; - do @(cache_response.sample); while (!cache_response.sample.ready); - cache_response.valid <= 0; - do @(response.sample); while (!response.sample.valid); - end - end - end else begin - do @(uncached_request.sample); while (!uncached_request.sample.valid); - if (phase_index != 0) begin - uncached_request.ready <= 1; - @(uncached_request.sample); - uncached_request.ready <= 0; - if (phase_index == 2) begin - uncached_response.bits <= '0; - uncached_response.bits[`CPUF(URESULT, TAG)] <= 63; - uncached_response.valid <= 1; - do @(uncached_response.sample); while (!uncached_response.sample.ready); - uncached_response.valid <= 0; - do @(response.sample); while (!response.sample.valid); - end - end - end - control.cb.reset <= 1; - cache_response.valid <= 0; - // Payload and VALID are ignored during joint reset, including an unrelated tag. - uncached_response.bits <= '0; - uncached_response.valid <= !cacheable && phase_index == 1; - repeat (2) @(control.cb); - uncached_response.valid <= 0; - control.cb.reset <= 0; - repeat (2) @(control.cb); - if (response.valid || cache_request.valid || uncached_request.valid || !request.ready) - `uvm_fatal("RESET", "Reset retained a cancelled transaction") - cancelled++; - endtask - task execute(); - control.reset = 1; - request.valid = 1; - request.bits = '1; - response.ready = 0; - cache_request.ready = 0; - uncached_request.ready = 0; - cache_response.valid = 0; - cache_response.bits = '0; - uncached_response.valid = 0; - uncached_response.bits = '0; - repeat (4) @(control.cb); - request.valid <= 0; - control.cb.reset <= 0; -`ifdef CPU_BAD_CASE - begin - req_t bad = '0; - bad[`CPUF(REQ, SIZE)] = `CPU_BAD_CASE == 1 ? 4 : (`CPU_BAD_CASE == 4 ? 1 : 3); - bad[`CPUF(REQ, ATOMIC)] = `CPU_BAD_CASE == 2 ? 12 : (`CPU_BAD_CASE == 3 ? 15 : 1); - bad[`CPUF(REQ, WRITE)] = `CPU_BAD_CASE == 5; - bad[`CPUF(REQ, CACHEABLE)] = 1; - @(control.cb); - request.bits <= bad; - request.valid <= 1; - repeat (10) @(control.cb); - `uvm_fatal("NO_ASSERT", "Invalid CPU memory contract was accepted") - end -`elsif CPU_BAD_TAG - transact(64'h60020000, 0, 0, 0, 0, 0, 0); - `uvm_fatal("NO_ASSERT", "Incorrect uncached response tag was accepted") -`else - for (int phase_index = 0; phase_index < 3; phase_index++) - for (int cacheable = 0; cacheable < 2; cacheable++) cancel_at(phase_index, cacheable); - for (int cacheable = 0; cacheable < 2; cacheable++) - for (int size = 0; size < 4; size++) - for (int offset = 0; offset < 8; offset++) - for (int sign_bit = 0; sign_bit < 2; sign_bit++) begin - transact(64'h60020000 + offset, size, 0, sign_bit, 0, 0, cacheable); - transact(64'h60020000 + offset, size, 1, 0, 64'hfedcba9876543210, 0, cacheable); - end - // Every aligned upper/lower Word atomic lane, including sign-expanded results and SC status. - for (int op = 1; op <= 11; op++) - for (int size = 2; size < 4; size++) - for (int offset = 0; offset < 8; offset += 4) begin - longint unsigned raw = op == 11 ? 1 : (size == 2 ? 64'hffffffff80000081 : 64'h8123456789abcdef); - transact(64'h1000 + offset, size, 0, 0, 64'h8877665544332211, op, 1, raw); - transact(64'h1000 + offset, size, 0, 1, 64'h1122334455667788, op, 1, raw, 1); - transact(64'h1000 + offset, size, 0, 0, 1, op, 0, raw); - end - for (int size = 2; size < 4; size++) - for (int offset = 1; offset < 8; offset++) - if (offset % (1 << size) != 0) transact(64'h1000 + offset, size, 0, 0, 1, 1, 1); - for (int size = 0; size < 4; size++) begin - transact(64'h2000, size, 0, 1, 0, 0, 1, '1); - transact(64'h2000, size, 0, 0, 0, 0, 1, 0); - transact(64'h2000, size, 0, 0, 0, 0, 1, '1, 1); - transact(64'h60020004, size, 0, 1, 0, 0, 0, '1, 1); - transact(64'h60020004, size, 1, 0, '1, 0, 0, 0, 1); - transact(64'h60020000, size, 0, 1, 0, 0, 0, '1); - transact(64'h60020000, size, 0, 0, 0, 0, 0, '1); - transact(64'h60020000, size, 1, 0, '1, 0, 0, 0, 1); - end - for (int bit_index = 3; bit_index < 64; bit_index++) begin - transact(64'h1 << bit_index, 3, 0, 0, 0, 0, 1); - transact(64'h1 << bit_index, 3, 1, 0, '1, 0, 0); - end - transact(64'h1004, 2, 0, 1, 0, 0, 1, 0); - transact(64'h60020004, 2, 0, 1, 0, 0, 0, 0); - transact(64'h1000, 3, 1, 0, 0, 0, 1, 0, 1); - transact(64'h1000, 2, 0, 0, 1, 11, 1, 0); - transact(64'h1004, 2, 0, 0, 1, 11, 1, 0); - transact(64'h1000, 3, 0, 0, 1, 11, 1, 0); - transact(64'h1000, 3, 1, 0, 0, 0, 1); - transact(64'h1000, 3, 1, 0, '1, 0, 1); - transact((64'h1 << `CPU_PHYSICAL_BITS) - 8, 3, 0, 0, 0, 0, 1); - transact((64'h1 << `CPU_PHYSICAL_BITS) - 1, 0, 1, 0, '1, 0, 1); - // Noncacheable normal memory, including every atomic, faults without issuing traffic. - for (int size = 0; size < 4; size++) begin - transact(64'h3000, size, 0, 0, 0, 0, 0, 0, 0, 1); - transact(64'h3000, size, 1, 0, 1, 0, 0, 0, 0, 1); - transact(64'h3001, size, 0, 0, 0, 0, 0, 0, 0, 1); - transact(64'h60020000, size, 0, 0, 0, 0, 1, 0, 0, 0, 1); - transact(64'h60020000, size, 1, 0, 1, 0, 1, 0, 0, 0, 1); - transact(64'h60020001, size, 0, 0, 0, 0, 1, 0, 0, 0, 1); - end - for (int op = 1; op <= 11; op++) - for (int size = 2; size <= 3; size++) begin - transact(64'h3000 + (size == 2 ? 4 : 0), size, 0, 0, 64'h76543210, op, 0, - op == 11 ? 0 : 64'hffffffff81234567, 0, 1); - transact(64'h3000, size, 0, 0, 1, op, 0, 64'hdeadbeef12345678, 1, 1); - end - transact(64'h3004, 2, 0, 0, 1, 11, 0, 1, 0, 1); - // Reset ignores even VALID=1 garbage payload; no transaction may escape reset. - control.cb.reset <= 1; - request.valid <= 1; - request.bits <= '1; - repeat (2) @(control.cb); - request.valid <= 0; - request.bits <= '0; - control.cb.reset <= 0; - repeat (2) @(control.cb); - if (response.valid || cache_request.valid || uncached_request.valid || !request.ready) - `uvm_fatal("RESET_PAYLOAD", "Reset payload escaped as a transaction") - // VALID=0 payload is unconstrained; it must not create an operation. - response.ready <= 1; - cache_request.ready <= 1; - uncached_request.ready <= 1; - request.bits <= '1; - cache_response.bits <= '1; - uncached_response.bits <= '1; - repeat (2) @(control.cb); - request.bits <= '0; - cache_response.bits <= '0; - uncached_response.bits <= '0; - repeat (2) @(control.cb); - if (response.valid || cache_request.valid || uncached_request.valid || !request.ready) - `uvm_fatal("IDLE", "Idle payload generated an operation") - `uvm_info("CPU_MEM", $sformatf( - "Checked %0d CPU results, %0d cache requests, %0d exact uncached requests, %0d suppressed faults, %0d joint-reset cancellations", - checked, - cache_checked, - uncached_checked, - faults_checked, - cancelled - ), UVM_LOW) -`endif - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/CpuMem.scala b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/CpuMem.scala deleted file mode 100644 index acf22eee6..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/CpuMem.scala +++ /dev/null @@ -1,135 +0,0 @@ -package memcore.memory.cpu - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.chi.Params -import memcore.bus.chi.rnf.{CacheAccess, CacheAtomic, CacheResult} - -case class CpuMemParams(chi: Params = Params(), tagBits: Int = 6) { - require(chi.addressBits < 64 && tagBits > 0) -} - -class CpuMemoryRequest(p: CpuMemParams) extends Bundle { - val addr = UInt(64.W) - val tag = UInt(p.tagBits.W) - val size = UInt(3.W) - val write = Bool() - val signed = Bool() - val data = UInt(64.W) - val atomic = UInt(4.W) - val cacheable = Bool() - val normal = Bool() -} - -/** `line` is the whole cached line for full-line clients; it is absent by default. */ -class CpuMemoryResponse(p: CpuMemParams, lineBits: Int = 0) extends Bundle { - val tag = UInt(p.tagBits.W) - val data = UInt(64.W) - val misaligned = Bool() - val accessFault = Bool() - val line = UInt(lineBits.W) -} - -class UncachedRequest(p: CpuMemParams) extends Bundle { - val addr = UInt(64.W) - val tag = UInt(p.tagBits.W) - val size = UInt(3.W) - val write = Bool() - val data = UInt(64.W) -} - -class UncachedResponse(p: CpuMemParams) extends Bundle { - val tag = UInt(p.tagBits.W) - val data = UInt(64.W) - val error = Bool() -} - -@instantiable -class CpuMem(p: CpuMemParams, lineBits: Int = 0) extends Module { - - @public - val io = IO(new Bundle { - val request = Flipped(Decoupled(new CpuMemoryRequest(p))) - val response = Decoupled(new CpuMemoryResponse(p, lineBits)) - val cacheRequest = Decoupled(new CacheAccess(p.chi)) - val cacheResponse = Flipped(Decoupled(new CacheResult(lineBits))) - val uncachedRequest = Decoupled(new UncachedRequest(p)) - val uncachedResponse = Flipped(Decoupled(new UncachedResponse(p))) - }) - - val idle :: issueCache :: waitCache :: issueUncached :: waitUncached :: respond :: Nil = Enum(6) - val state = RegInit(idle) - val command = Reg(new CpuMemoryRequest(p)) - val answer = Reg(new CpuMemoryResponse(p, lineBits)) - - val mask = MuxLookup(command.size, 255.U(8.W))(Seq( - 0.U -> 1.U(8.W), - 1.U -> 3.U(8.W), - 2.U -> 15.U(8.W) - )) - - io.request.ready := state === idle - when(io.request.fire) { - val r = io.request.bits - assert(r.size <= 3.U, "CPU memory size must be 1, 2, 4 or 8 bytes") - assert(r.atomic <= CacheAtomic.SC.U, "CPU memory port does not accept Fence or unknown atomics") - assert(r.atomic === CacheAtomic.None.U || (!r.write && r.size >= 2.U), "CPU atomic requires a 32 or 64 bit operand") - val misaligned = (r.addr & ((1.U(64.W) << r.size) - 1.U)).orR - val accessFault = r.addr(63, p.chi.addressBits).orR || (r.normal && !r.cacheable) || - (!r.normal && (r.cacheable || r.atomic =/= CacheAtomic.None.U)) - command := r - answer.tag := r.tag - answer.data := 0.U - if (lineBits > 0) answer.line := 0.U - answer.misaligned := misaligned - answer.accessFault := !misaligned && accessFault - state := Mux(misaligned || accessFault, respond, Mux(r.cacheable, issueCache, issueUncached)) - } - - val atomic = command.atomic =/= CacheAtomic.None.U - io.cacheRequest.valid := state === issueCache - io.cacheRequest.bits.addr := Mux(atomic, command.addr, command.addr & ~7.U(64.W))(p.chi.addressBits - 1, 0) - io.cacheRequest.bits.write := command.write - io.cacheRequest.bits.data := Mux(atomic, command.data, command.data << (command.addr(2, 0) << 3)) - io.cacheRequest.bits.mask := Mux(atomic, 255.U, Mux(command.write, mask << command.addr(2, 0), 0.U)) - io.cacheRequest.bits.atomic := command.atomic - io.cacheRequest.bits.atomicWord := atomic && command.size === 2.U - when(io.cacheRequest.fire)(state := waitCache) - io.cacheResponse.ready := state === waitCache - io.uncachedRequest.valid := state === issueUncached - io.uncachedRequest.bits.addr := command.addr - io.uncachedRequest.bits.tag := command.tag - io.uncachedRequest.bits.size := command.size - io.uncachedRequest.bits.write := command.write - io.uncachedRequest.bits.data := command.data - when(io.uncachedRequest.fire)(state := waitUncached) - io.uncachedResponse.ready := state === waitUncached - - val raw = - Mux(state === waitUncached, io.uncachedResponse.bits.data, io.cacheResponse.bits.data >> (command.addr(2, 0) << 3)) - - val load = MuxLookup(command.size, raw)(Seq( - 0.U -> Cat(Fill(56, command.signed && raw(7)), raw(7, 0)), - 1.U -> Cat(Fill(48, command.signed && raw(15)), raw(15, 0)), - 2.U -> Cat(Fill(32, command.signed && raw(31)), raw(31, 0)) - )) - - when(io.cacheResponse.fire || io.uncachedResponse.fire) { - val error = Mux(state === waitUncached, io.uncachedResponse.bits.error, io.cacheResponse.bits.error) - when(io.uncachedResponse.fire) { - assert(io.uncachedResponse.bits.tag === command.tag, "Uncached response tag mismatch") - } - answer.data := Mux( - error || command.write, - 0.U, - Mux(atomic, io.cacheResponse.bits.data, load) - ) - answer.accessFault := error - if (lineBits > 0) answer.line := Mux(state === waitCache && !error, io.cacheResponse.bits.line, 0.U) - state := respond - } - io.response.valid := state === respond - io.response.bits := answer - when(io.response.fire)(state := idle) -} diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/Emit.scala deleted file mode 100644 index a55ff637a..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/Emit.scala +++ /dev/null @@ -1,92 +0,0 @@ -package memcore.memory.cpu - -import java.nio.file.{Files, Paths} - -object Emit extends App { - val p = CpuMemParams() - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new CpuMem(p), - firtoolOpts = args, - args = Array("--target-dir", "build/CpuMem", "--split-verilog") - ) - - val definitions = scala.collection.mutable.ArrayBuffer[String]( - s"`define CPU_PHYSICAL_BITS ${p.chi.addressBits}", - s"`define CPU_TAG_BITS ${p.tagBits}" - ) - - val ports = scala.collection.mutable.ArrayBuffer[String]() - - def channel( - kind: String, - prefix: String, - instance: String, - fields: Seq[(String, Int)] - ): Unit = { - var offset = 0 - for ((field, width) <- fields) { - definitions += s"`define CPU_${kind}_${field.toUpperCase}_OFFSET $offset" - definitions += s"`define CPU_${kind}_${field.toUpperCase}_WIDTH $width" - ports += s".$prefix$field($instance.bits[$offset +: $width])" - offset += width - } - definitions += s"`define CPU_${kind}_WIDTH $offset" - } - - channel( - "REQ", - "io_request_bits_", - "request", - Seq( - "addr" -> 64, - "tag" -> p.tagBits, - "size" -> 3, - "write" -> 1, - "signed" -> 1, - "data" -> 64, - "atomic" -> 4, - "cacheable" -> 1, - "normal" -> 1 - ) - ) - channel( - "RESP", - "io_response_bits_", - "response", - Seq("tag" -> p.tagBits, "data" -> 64, "misaligned" -> 1, "accessFault" -> 1) - ) - channel( - "CACHE", - "io_cacheRequest_bits_", - "cache_request", - Seq("addr" -> p.chi.addressBits, "write" -> 1, "data" -> 64, "mask" -> 8, "atomic" -> 4, "atomicWord" -> 1) - ) - channel("CRESULT", "io_cacheResponse_bits_", "cache_response", Seq("data" -> 64, "error" -> 1)) - channel( - "UNCACHED", - "io_uncachedRequest_bits_", - "uncached_request", - Seq("addr" -> 64, "tag" -> p.tagBits, "size" -> 3, "write" -> 1, "data" -> 64) - ) - channel( - "URESULT", - "io_uncachedResponse_bits_", - "uncached_response", - Seq("tag" -> p.tagBits, "data" -> 64, "error" -> 1) - ) - for ( - (prefix, instance) <- Seq( - "request" -> "request", - "response" -> "response", - "cacheRequest" -> "cache_request", - "cacheResponse" -> "cache_response", - "uncachedRequest" -> "uncached_request", - "uncachedResponse" -> "uncached_response" - ) - ) { - ports += s".io_${prefix}_valid($instance.valid)" - ports += s".io_${prefix}_ready($instance.ready)" - } - Files.write(Paths.get("build/cpu_mem_config.svh"), (definitions.mkString("\n") + "\n").getBytes) - Files.write(Paths.get("build/cpu_mem_ports.svh"), ports.mkString(",\n").getBytes) -} diff --git a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/VirtualMemory.scala b/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/VirtualMemory.scala deleted file mode 100644 index 9cf9a104a..000000000 --- a/arch/src/main/scala/framework/mem-core/cpu_mem/src/main/scala/VirtualMemory.scala +++ /dev/null @@ -1,306 +0,0 @@ -package memcore.memory.cpu - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi.rnf.{CacheAccess, CacheAtomic, CacheResult} -import memcore.memory.mmu.Walker - -case class PhysicalRegion( - base: BigInt, - bytes: BigInt, - cacheable: Boolean, - executable: Boolean, - readable: Boolean, - writable: Boolean, - atomic: Boolean, - normal: Boolean) - -class PhysicalAuthorization extends Bundle { - val paddr = UInt(64.W) - val size = UInt(3.W) - val read = Bool() - val write = Bool() - val execute = Bool() - val isPte = Bool() - val privilege = UInt(2.W) -} - -class VirtualMemoryRequest(p: CpuMemParams) extends Bundle { - val vaddr = UInt(64.W) - val tag = UInt(p.tagBits.W) - val size = UInt(3.W) - val write = Bool() - val execute = Bool() - val signed = Bool() - val data = UInt(64.W) - val atomic = UInt(4.W) - val privilege = UInt(2.W) - val sum = Bool() - val mxr = Bool() - val satpMode = UInt(4.W) - val rootPpn = UInt(44.W) -} - -class VirtualMemoryResponse(p: CpuMemParams, lineBits: Int = 0) extends Bundle { - val tag = UInt(p.tagBits.W) - val data = UInt(64.W) - val misaligned = Bool() - val pageFault = Bool() - val accessFault = Bool() - // Whole cached line of the translated word, for full-line clients only. - val line = UInt(lineBits.W) -} - -/** Single-outstanding virtual data/instruction-word frontend; cache ownership is external. */ -@instantiable -class VirtualMemory( - cp: CpuMemParams, - regions: Seq[PhysicalRegion], - lineBits: Int = 0, - translationEntries: Int = 0) - extends Module { - require(regions.nonEmpty) - for ((region, i) <- regions.zipWithIndex) { - require(region.base >= 0 && region.bytes > 0 && region.base + region.bytes <= (BigInt(1) << cp.chi.addressBits)) - require(!region.cacheable || (region.base % 64 == 0 && region.bytes % 64 == 0)) - require( - region.normal || (!region.cacheable && !region.executable && !region.atomic), - "Device regions cannot be cached, executable or atomic" - ) - for (other <- regions.drop(i + 1)) { - require(region.base + region.bytes <= other.base || other.base + other.bytes <= region.base) - } - } - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - // SFENCE.VMA: drop every cached translation. - val flushTranslation = Input(Bool()) - - // Combinational translation through the TLB only, for a caller's fast path. - val lookup = new Bundle { - val config = Input(new memcore.memory.mmu.Config) - val req = Input(new memcore.memory.mmu.Request) - val hit = Output(Bool()) - val paddr = Output(UInt(cp.chi.addressBits.W)) - } - - val request = Flipped(Decoupled(new VirtualMemoryRequest(cp))) - val response = Decoupled(new VirtualMemoryResponse(cp, lineBits)) - val authorizationRequest = Decoupled(new PhysicalAuthorization) - val authorizationResponse = Flipped(Decoupled(Bool())) - val cacheRequest = Decoupled(new CacheAccess(cp.chi)) - val cacheResponse = Flipped(Decoupled(new CacheResult(lineBits))) - val uncachedRequest = Decoupled(new UncachedRequest(cp)) - val uncachedResponse = Flipped(Decoupled(new UncachedResponse(cp))) - val observedTranslation = Output(Valid(new memcore.memory.mmu.Response(cp.chi))) - val observedPhysical = Output(Valid(new CpuMemoryRequest(cp))) - - val observedCache = Output(Valid(new Bundle { - val pte = Bool() - val access = new CacheAccess(cp.chi) - })) - - }) - - val walker: Instance[Walker] = Instantiate(new Walker(cp.chi, translationEntries)) - val physical: Instance[CpuMem] = Instantiate(new CpuMem(cp, lineBits)) - - val idle :: startWalk :: waitWalk :: authorize :: waitAuthorization :: startAccess :: waitAccess :: respond :: Nil = - Enum(8) - val state = RegInit(idle) - val command = Reg(new VirtualMemoryRequest(cp)) - val answer = Reg(new VirtualMemoryResponse(cp, lineBits)) - val paddr = Reg(UInt(64.W)) - val cacheable = Reg(Bool()) - val normal = Reg(Bool()) - val pteInFlight = RegInit(false.B) - val pteError = RegInit(false.B) - val pteAuthorized = RegInit(false.B) - val pteAuthorizationPending = RegInit(false.B) - val regionAllowed = Reg(Bool()) - def matches(first: UInt, last: UInt): Seq[Bool] = - regions.map(region => first >= region.base.U(65.W) && last < (region.base + region.bytes).U(65.W)) - io.request.ready := state === idle && io.active - when(io.request.fire) { - val r = io.request.bits - assert(r.size <= 3.U, "Virtual LSU size must be 1, 2, 4 or 8 bytes") - assert(r.atomic <= CacheAtomic.SC.U, "Virtual LSU does not accept Fence or unknown atomics") - assert( - r.atomic === CacheAtomic.None.U || (!r.write && r.size >= 2.U), - "Virtual LSU atomic requires a 32 or 64 bit operand" - ) - assert(r.satpMode === 0.U || r.satpMode === 8.U, "Virtual LSU requires Bare or Sv39 mode") - assert( - r.privilege === 0.U || r.privilege === 1.U || r.privilege === 3.U, - "Virtual LSU effective privilege must be U, S or M" - ) - assert( - !r.execute || (!r.write && r.atomic === CacheAtomic.None.U && r.size === 3.U && !r.signed), - "Virtual instruction word requires unsigned 64 bit read without atomic" - ) - command := r - answer := 0.U.asTypeOf(answer) - answer.tag := r.tag - val misaligned = (r.vaddr & ((1.U(64.W) << r.size) - 1.U)).orR - answer.misaligned := misaligned - state := Mux(misaligned, respond, startWalk) - } - - walker.io.flush := io.flushTranslation - walker.io.lookup.config := io.lookup.config - walker.io.lookup.req := io.lookup.req - io.lookup.hit := walker.io.lookup.hit - io.lookup.paddr := walker.io.lookup.paddr - walker.io.config.mode := command.satpMode - walker.io.config.rootPpn := command.rootPpn - walker.io.req.valid := state === startWalk - walker.io.req.bits.vaddr := command.vaddr - walker.io.req.bits.write := command.write || - (command.atomic =/= CacheAtomic.None.U && command.atomic =/= CacheAtomic.LR.U) - walker.io.req.bits.execute := command.execute - walker.io.req.bits.privilege := command.privilege - walker.io.req.bits.sum := command.sum - walker.io.req.bits.mxr := command.mxr - when(walker.io.req.fire)(state := waitWalk) - walker.io.resp.ready := state === waitWalk - - val lastByte = walker.io.resp.bits.paddr.pad(65) + MuxLookup(command.size, 0.U(4.W))( - Seq(1.U -> 1.U(4.W), 2.U -> 3.U(4.W), 3.U -> 7.U(4.W)) - ) - - val dataRegions = matches(walker.io.resp.bits.paddr.pad(65), lastByte) - val isAtomic = command.atomic =/= CacheAtomic.None.U - val readsData = !command.execute && !command.write && command.atomic =/= CacheAtomic.SC.U - val writesData = command.write || (isAtomic && command.atomic =/= CacheAtomic.LR.U) - - val permittedRegion = dataRegions.zip(regions).map { case (hit, region) => - hit && (!region.normal || region.cacheable).B && (!command.execute || (region.normal && region.executable).B) && - (!readsData || region.readable.B) && (!writesData || region.writable.B) && - (!isAtomic || region.atomic.B) - }.reduce(_ || _) - - when(walker.io.resp.fire) { - when(walker.io.resp.bits.pageFault || walker.io.resp.bits.accessFault) { - answer.pageFault := walker.io.resp.bits.pageFault - answer.accessFault := walker.io.resp.bits.accessFault - state := respond - }.otherwise { - paddr := walker.io.resp.bits.paddr - cacheable := dataRegions.zip(regions).map { case (hit, region) => hit && region.cacheable.B }.reduce(_ || _) - normal := dataRegions.zip(regions).map { case (hit, region) => hit && region.normal.B }.reduce(_ || _) - regionAllowed := permittedRegion - state := authorize - } - } - - physical.io.request.valid := state === startAccess - physical.io.request.bits.addr := paddr - physical.io.request.bits.tag := command.tag - physical.io.request.bits.size := command.size - physical.io.request.bits.write := command.write - physical.io.request.bits.signed := command.signed - physical.io.request.bits.data := command.data - physical.io.request.bits.atomic := command.atomic - physical.io.request.bits.cacheable := cacheable - physical.io.request.bits.normal := normal - when(physical.io.request.fire)(state := waitAccess) - physical.io.response.ready := state === waitAccess - when(physical.io.response.fire) { - assert(physical.io.response.bits.tag === command.tag, "Virtual LSU physical response tag mismatch") - answer.data := physical.io.response.bits.data - answer.misaligned := physical.io.response.bits.misaligned - answer.accessFault := physical.io.response.bits.accessFault - if (lineBits > 0) answer.line := physical.io.response.bits.line - state := respond - } - - val walking = state === startWalk || state === waitWalk - val accessing = state === startAccess || state === waitAccess - val pteFirst = walker.io.access.bits.addr.pad(65) - val pteRegions = matches(pteFirst, pteFirst + 7.U) - - val pteAllowed = pteRegions.zip(regions).map { case (hit, region) => - hit && (region.normal && region.readable && region.cacheable).B - }.reduce(_ || _) - - // The Core owns a per-command PMP snapshot. Each permission decision is a - // single-outstanding handshake; neither a stalled query nor its response - // allows speculative cache or MMIO traffic. PTE accesses use S privilege. - io.authorizationRequest.valid := state === authorize || - (walking && walker.io.access.valid && !pteAuthorized && !pteAuthorizationPending && !pteError) - io.authorizationRequest.bits.paddr := Mux(walking, walker.io.access.bits.addr.pad(64), paddr) - io.authorizationRequest.bits.size := Mux(walking, 3.U, command.size) - io.authorizationRequest.bits.read := walking || readsData - io.authorizationRequest.bits.write := !walking && writesData - io.authorizationRequest.bits.execute := !walking && command.execute - io.authorizationRequest.bits.isPte := walking - io.authorizationRequest.bits.privilege := Mux(walking, 1.U, command.privilege) - when(io.authorizationRequest.fire) { - when(walking) { - pteAuthorizationPending := true.B - } - .otherwise(state := waitAuthorization) - } - io.authorizationResponse.ready := pteAuthorizationPending || state === waitAuthorization - when(io.authorizationResponse.fire) { - when(pteAuthorizationPending) { - pteAuthorizationPending := false.B - pteAuthorized := true.B - pteError := !io.authorizationResponse.bits || !pteAllowed - }.otherwise { - when(io.authorizationResponse.bits && regionAllowed)(state := startAccess) - .otherwise { answer.accessFault := true.B; state := respond } - } - } - io.cacheRequest.valid := Mux( - walking, - walker.io.access.valid && pteAuthorized && !pteError, - accessing && physical.io.cacheRequest.valid - ) - io.cacheRequest.bits := Mux(walking, walker.io.access.bits, physical.io.cacheRequest.bits) - io.uncachedRequest.valid := accessing && physical.io.uncachedRequest.valid - io.uncachedRequest.bits := physical.io.uncachedRequest.bits - walker.io.access.ready := walking && pteAuthorized && - (pteError || io.cacheRequest.ready) - when(walker.io.access.fire) { - pteAuthorized := false.B - pteInFlight := !pteError - } - walker.io.result.valid := walking && ((!pteAuthorized && pteError) || (pteInFlight && io.cacheResponse.valid)) - walker.io.result.bits := 0.U.asTypeOf(walker.io.result.bits) - walker.io.result.bits.data := Mux(pteError || io.cacheResponse.bits.error, 0.U, io.cacheResponse.bits.data) - walker.io.result.bits.error := pteError || io.cacheResponse.bits.error - when(walker.io.result.fire) { pteError := false.B; pteInFlight := false.B } - physical.io.cacheRequest.ready := accessing && io.cacheRequest.ready - physical.io.cacheResponse.valid := accessing && io.cacheResponse.valid - physical.io.cacheResponse.bits := io.cacheResponse.bits - physical.io.uncachedRequest.ready := accessing && io.uncachedRequest.ready - physical.io.uncachedResponse.valid := accessing && io.uncachedResponse.valid - physical.io.uncachedResponse.bits := io.uncachedResponse.bits - io.cacheResponse.ready := Mux( - walking, - pteInFlight && walker.io.result.ready, - accessing && physical.io.cacheResponse.ready - ) - io.uncachedResponse.ready := accessing && physical.io.uncachedResponse.ready - when(io.cacheResponse.valid) { - assert((walking && pteInFlight) || accessing, "Virtual LSU cache result has no owner") - } - when(io.uncachedResponse.valid) { - assert(accessing, "Virtual LSU uncached result has no owner") - } - - io.response.valid := state === respond - io.response.bits := answer - when(io.response.fire)(state := idle) - io.observedTranslation.valid := walker.io.resp.fire - io.observedTranslation.bits := walker.io.resp.bits - io.observedPhysical.valid := physical.io.request.fire - io.observedPhysical.bits := physical.io.request.bits - io.observedCache.valid := io.cacheRequest.fire - io.observedCache.bits.pte := walking - io.observedCache.bits.access := io.cacheRequest.bits -} diff --git a/arch/src/main/scala/framework/mem-core/ddr/build.sc b/arch/src/main/scala/framework/mem-core/ddr/build.sc deleted file mode 100644 index 60f1b35d0..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/build.sc +++ /dev/null @@ -1,35 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object chi extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = os.pwd / os.up / "chi" -} - -object axi4 extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = os.pwd / os.up / "axi4" -} - -object ddr extends ChiselModule { - override def moduleRoot = os.pwd - override def moduleDeps = Seq(chi, axi4) - override def mainClass = Some("memcore.memory.ddr.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/csrc/src/lib.rs b/arch/src/main/scala/framework/mem-core/ddr/src/csrc/src/lib.rs deleted file mode 100644 index 879e3a09c..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/csrc/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod memory; -pub use memory::*; diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_alignment.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_alignment.f deleted file mode 100644 index a0b44121a..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_alignment.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+DDR_TB=bad_alignment_tb -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_alignment.toml b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_alignment.toml deleted file mode 100644 index 75bd23a9d..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_alignment.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "DDR line request must be 64 byte aligned" -invalid_case = 0 diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_b.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_b.f deleted file mode 100644 index bb2e41872..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_b.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+DDR_TB=bad_early_b_tb -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_b.toml b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_b.toml deleted file mode 100644 index cfe95574d..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_b.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "DDR received B before AW and complete W or for inactive ID" -invalid_case = 2 diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_rlast.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_rlast.f deleted file mode 100644 index 38ced94f3..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_rlast.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+DDR_TB=bad_early_rlast_tb -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_rlast.toml b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_rlast.toml deleted file mode 100644 index 25e224c8a..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_early_rlast.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "DDR AXI RLAST does not match line beat count" -invalid_case = 3 diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_late_rlast.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_late_rlast.f deleted file mode 100644 index 4e063c528..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_late_rlast.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+DDR_TB=bad_late_rlast_tb -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_late_rlast.toml b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_late_rlast.toml deleted file mode 100644 index 5a24795e0..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_late_rlast.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "DDR AXI RLAST does not match line beat count" -invalid_case = 4 diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_read_exokay.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_read_exokay.f deleted file mode 100644 index 44dd5c1e8..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_read_exokay.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+DDR_TB=bad_read_exokay_tb -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_read_exokay.toml b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_read_exokay.toml deleted file mode 100644 index e2994b234..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_read_exokay.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "DDR nonexclusive read received EXOKAY" -invalid_case = 5 diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_rid.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_rid.f deleted file mode 100644 index ed36259a9..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_rid.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+DDR_TB=bad_rid_tb -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_rid.toml b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_rid.toml deleted file mode 100644 index 13b50d759..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_rid.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "DDR received R for an inactive AXI ID" -invalid_case = 1 diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_write_exokay.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_write_exokay.f deleted file mode 100644 index aba96f4a5..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_write_exokay.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+DDR_TB=bad_write_exokay_tb -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_write_exokay.toml b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_write_exokay.toml deleted file mode 100644 index 266196316..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bad_write_exokay.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "DDR nonexclusive write received EXOKAY" -invalid_case = 6 diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge.f deleted file mode 100644 index 58c2ae4cc..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge256.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge256.f deleted file mode 100644 index 08805a0d1..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge256.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+DDR_TB=bridge256_tb -+define+DDR_PROFILE256 -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge256/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge512.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge512.f deleted file mode 100644 index 92cd77dcf..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge512.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+DDR_TB=bridge512_tb -+define+DDR_PROFILE512 -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge512/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge64.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge64.f deleted file mode 100644 index 31a72de29..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge64.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+DDR_TB=bridge64_tb -+define+DDR_PROFILE64 -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Bridge64/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge_six.f b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge_six.f deleted file mode 100644 index 85fa55b93..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge_six.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+DDR_TB=bridge_six_tb -+define+DDR_PROFILESIX -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/BridgeSix/filelist.f -@RESOURCES@/bridge_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge_tb.sv b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge_tb.sv deleted file mode 100644 index 85dd485bf..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/bridge_tb.sv +++ /dev/null @@ -1,71 +0,0 @@ -`include "profile.svh" -`ifndef DDR_TB -`define DDR_TB bridge_tb -`endif -module `DDR_TB; - import uvm_pkg::*; - import ddr_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - ddr_control_if ctl (clock); - stream_if #(`DDR_REQ_WIDTH) - req0 ( - clock, - ctl.reset - ), - req1 ( - clock, - ctl.reset - ); - stream_if #(`DDR_RESP_WIDTH) - resp0 ( - clock, - ctl.reset - ), - resp1 ( - clock, - ctl.reset - ); - stream_if #(`DDR_A_WIDTH) - aw ( - clock, - ctl.reset - ), - ar ( - clock, - ctl.reset - ); - stream_if #(`DDR_W_WIDTH) w ( - clock, - ctl.reset - ); - stream_if #(`DDR_B_WIDTH) b ( - clock, - ctl.reset - ); - stream_if #(`DDR_R_WIDTH) r ( - clock, - ctl.reset - ); - Bridge dut ( - .clock(clock), - .reset(ctl.reset), - `include `DDR_PORTS - ); - initial begin - int invalid_case; - uvm_config_db#(virtual ddr_control_if)::set(null, "uvm_test_top", "ctl", ctl); - uvm_config_db#(virtual stream_if #(`DDR_REQ_WIDTH))::set(null, "uvm_test_top", "req0", req0); - uvm_config_db#(virtual stream_if #(`DDR_REQ_WIDTH))::set(null, "uvm_test_top", "req1", req1); - uvm_config_db#(virtual stream_if #(`DDR_RESP_WIDTH))::set(null, "uvm_test_top", "resp0", resp0); - uvm_config_db#(virtual stream_if #(`DDR_RESP_WIDTH))::set(null, "uvm_test_top", "resp1", resp1); - uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::set(null, "uvm_test_top", "aw", aw); - uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::set(null, "uvm_test_top", "ar", ar); - uvm_config_db#(virtual stream_if #(`DDR_W_WIDTH))::set(null, "uvm_test_top", "w", w); - uvm_config_db#(virtual stream_if #(`DDR_B_WIDTH))::set(null, "uvm_test_top", "b", b); - uvm_config_db#(virtual stream_if #(`DDR_R_WIDTH))::set(null, "uvm_test_top", "r", r); - if ($value$plusargs("invalid_case=%d", invalid_case)) - uvm_config_db#(int)::set(null, "uvm_test_top", "invalid_case", invalid_case); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/package.sv deleted file mode 100644 index 146228cff..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/package.sv +++ /dev/null @@ -1,599 +0,0 @@ -`include "profile.svh" -`define DF(K, F) `DDR_``K``_``F``_OFFSET +: `DDR_``K``_``F``_WIDTH -package ddr_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function chandle ddr_ref_create(); - import "DPI-C" function void ddr_ref_destroy(input chandle model); - import "DPI-C" function void ddr_ref_program( - input chandle model, - input longint unsigned address, - input bit [511:0] data - ); - import "DPI-C" function int unsigned ddr_ref_read( - input chandle model, - input longint unsigned address, - output bit [511:0] data - ); - import "DPI-C" function int unsigned ddr_ref_write( - input chandle model, - input longint unsigned address, - input bit [511:0] data, - input longint unsigned mask - ); - typedef longint unsigned u64; - typedef bit [`DDR_REQ_WIDTH-1:0] req_t; - class result_item extends uvm_sequence_item; - bit [511:0] data; - bit error; - int tag, client; - `uvm_object_utils_begin(result_item) - - `uvm_field_int(data, UVM_DEFAULT) - `uvm_field_int(error, UVM_DEFAULT) - - `uvm_field_int(tag, UVM_DEFAULT) - `uvm_field_int(client, UVM_DEFAULT) - - `uvm_object_utils_end - function new(string name = "result_item"); - super.new(name); - endfunction - endclass - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual ddr_control_if ctl; - virtual stream_if #(`DDR_REQ_WIDTH) req[2]; - virtual stream_if #(`DDR_RESP_WIDTH) resp[2]; - virtual stream_if #(`DDR_A_WIDTH) aw, ar; - virtual stream_if #(`DDR_W_WIDTH) w; - virtual stream_if #(`DDR_B_WIDTH) b; - virtual stream_if #(`DDR_R_WIDTH) r; - keyed_scoreboard #(result_item) scoreboard; - chandle model; - typedef struct { - req_t packet; - int client; - bit done; - } command; - typedef struct { - int key, beat, serial; - bit [511:0] line; - } reading; - typedef struct {int key, due, serial;} writing; - typedef struct packed { - bit [511:0] data; - bit [63:0] mask; - } wline; - command commands[int]; - reading reading_ids[int]; - writing writing_ids[int]; - int aw_ids[$], aw_keys[$], aw_serials[$]; - wline w_lines[$]; - wline assembling; - int assembling_beat = 0; - int read_error[u64], write_error[u64]; - bit hold_aw = 0, hold_w = 0, hold_ar = 0, hold_b = 0, hold_r = 0, hold_resp[2] = '{0, 0}; - bit r_active = 0, b_active = 0; - int r_id = 0, b_id = 0, last_r_id = -1, last_r_serial = -1, r_cursor = 0, r_beats = 0; - logic [`DDR_R_WIDTH-1:0] r_packet; - logic [`DDR_B_WIDTH-1:0] b_packet; - int - cycle = 0, - accepted = 0, - returned[2] = '{0, 0}, - cancelled = 0, - peak = 0, - aw_count = 0, - w_bursts = 0, - ar_count = 0; - int - request_stalls = 0, - response_stalls = 0, - aw_stalls = 0, - w_stalls = 0, - interleaves = 0, - error_results = 0; - int invalid_case = -1; - int b_reorders = 0, r_reorders = 0, max_b_serial = -1, max_r_serial = -1; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 1ms; - endfunction - function void verify_contract(bit ok, string message); - if (!ok) `uvm_fatal("DDR_CONTRACT", message) - endfunction - function bit [511:0] pattern(int seed); - bit [511:0] data; - for (int i = 0; i < 8; i++) - data[i*64+:64]=64'h0123456789abcdef^(64'h0101010101010101*(seed+i))^(64'h1<<((seed+i*7)%64)); - return data; - endfunction - function void initialize(u64 address, int seed); - ddr_ref_program(model, address, pattern(seed)); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - model = ddr_ref_create(); - scoreboard = keyed_scoreboard#(result_item)::type_id::create("scoreboard", this); - verify_contract(uvm_config_db#(virtual ddr_control_if)::get(this, "", "ctl", ctl), - "Missing control"); - for (int c = 0; c < 2; c++) begin - verify_contract(uvm_config_db#(virtual stream_if #(`DDR_REQ_WIDTH))::get( - this, "", $sformatf("req%0d", c), req[c]), "Missing request"); - verify_contract(uvm_config_db#(virtual stream_if #(`DDR_RESP_WIDTH))::get( - this, "", $sformatf("resp%0d", c), resp[c]), "Missing response"); - end - verify_contract(uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::get(this, "", "aw", aw), - "Missing AW"); - verify_contract(uvm_config_db#(virtual stream_if #(`DDR_A_WIDTH))::get(this, "", "ar", ar), - "Missing AR"); - verify_contract(uvm_config_db#(virtual stream_if #(`DDR_W_WIDTH))::get(this, "", "w", w), - "Missing W"); - verify_contract(uvm_config_db#(virtual stream_if #(`DDR_B_WIDTH))::get(this, "", "b", b), - "Missing B"); - verify_contract(uvm_config_db#(virtual stream_if #(`DDR_R_WIDTH))::get(this, "", "r", r), - "Missing R"); - void'(uvm_config_db#(int)::get(this, "", "invalid_case", invalid_case)); - endfunction - function int address_owner(u64 address, bit write); - int found = -1; - foreach (commands[key]) - if (commands[key].packet[ - `DF(REQ, ADDR) - ] == address && commands[key].packet[ - `DF(REQ, WRITE) - ] == write) begin - verify_contract(found == -1, "Ambiguous fixture address"); - found = key; - end - verify_contract(found != -1, "AXI address without accepted line command"); - return found; - endfunction - function int observe_address(logic [`DDR_A_WIDTH-1:0] a, bit write); - int key, id; - verify_contract(!$isunknown(a), "Unknown AXI address payload"); - key = address_owner(a[`DF(A, ADDR)], write); - id = a[`DF(A, ID)]; - verify_contract(id >= `DDR_ID_BASE && id < `DDR_ID_BASE + `DDR_SLOTS, - "AXI ID outside configured namespace"); - verify_contract((id - `DDR_ID_BASE) / `DDR_SLOTS_PER_CLIENT == commands[key].client, - "AXI ID crossed client namespace"); - verify_contract(a[`DF(A, LEN)] == `DDR_BEATS - 1 && a[`DF(A, SIZE)] == $clog2( - (`DDR_BEAT_BYTES)) && a[`DF(A, BURST)] == 1, "Wrong AXI burst geometry"); - verify_contract(a[`DF(A, LOCK)] == 0 && a[`DF(A, CACHE)] == 0 && a[`DF(A, PROT)] == 0 && a[ - `DF(A, QOS)] == 0 && a[`DF(A, REGION)] == 0, "Unexpected AXI attributes"); - return key; - endfunction - task service(); - forever begin - @(ctl.sample); - if (!ctl.sample.reset) begin - cycle++; - for (int c = 0; c < 2; c++) begin - if (req[c].sample.valid && !req[c].sample.ready) request_stalls++; - if (resp[c].sample.valid && !resp[c].sample.ready) response_stalls++; - if (req[c].sample.valid && req[c].sample.ready) begin - result_item expected = new; - req_t q = req[c].sample.bits; - int key = (c << 12) | int'(q[`DF(REQ, ID)]); - bit [511:0] line; - verify_contract(!commands.exists(key), "Accepted duplicate live client tag"); - verify_contract(ddr_ref_read(model, q[`DF(REQ, ADDR)], line), - "Uninitialized fixture line"); - commands[key] = '{q, c, 0}; - expected.set_transaction_id(key); - expected.client = c; - expected.tag = q[`DF(REQ, ID)]; - expected.error = q[`DF(REQ, WRITE)] ? write_error.exists(q[`DF(REQ, ADDR)]) : - read_error.exists(q[`DF(REQ, ADDR)]); - expected.data = (q[`DF(REQ, WRITE)] || expected.error) ? '0 : line; - scoreboard.expected_export.write(expected); - accepted++; - if (commands.num() > peak) peak = commands.num(); - end - end - if (aw.sample.valid && !aw.sample.ready) aw_stalls++; - if (w.sample.valid && !w.sample.ready) w_stalls++; - if (aw.sample.valid && aw.sample.ready) begin - int key = observe_address(aw.sample.bits, 1); - int id = aw.sample.bits[`DF(A, ID)]; - aw_ids.push_back(id); - aw_keys.push_back(key); - aw_serials.push_back(aw_count); - aw_count++; - end - if (w.sample.valid && w.sample.ready) begin - verify_contract(!$isunknown(w.sample.bits), "Unknown W payload"); - verify_contract(w.sample.bits[`DF(W, LAST)] == (assembling_beat == `DDR_BEATS - 1), - "Wrong WLAST"); - assembling.data[assembling_beat*`DDR_DATA_BITS+:`DDR_DATA_BITS] = w.sample.bits[ - `DF(W, DATA) - ]; - assembling.mask[assembling_beat*(`DDR_BEAT_BYTES)+:(`DDR_BEAT_BYTES)] = w.sample.bits[ - `DF(W, STRB) - ]; - if (assembling_beat == `DDR_BEATS - 1) begin - w_lines.push_back(assembling); - assembling = '0; - assembling_beat = 0; - w_bursts++; - end else assembling_beat++; - end - if (aw_ids.size() && w_lines.size()) begin - int id = aw_ids.pop_front(); - int key = aw_keys.pop_front(); - int serial = aw_serials.pop_front(); - wline line = w_lines.pop_front(); - req_t q = commands[key].packet; - verify_contract(!writing_ids.exists(id) && !reading_ids.exists(id), - "AXI ID reused while active"); - verify_contract(line.data === q[`DF(REQ, DATA)] && line.mask === q[`DF(REQ, MASK)], - "AW/W order, payload or byte mask mismatch"); - verify_contract(ddr_ref_write(model, q[`DF(REQ, ADDR)], line.data, line.mask), - "Write outside initialized DDR"); - writing_ids[id] = '{key, cycle + 7, serial}; - end - if (ar.sample.valid && ar.sample.ready) begin - int key = observe_address(ar.sample.bits, 0); - int id = ar.sample.bits[`DF(A, ID)]; - bit [511:0] line; - verify_contract(!reading_ids.exists(id) && !writing_ids.exists(id), - "AXI ID reused while active"); - verify_contract(ddr_ref_read(model, ar.sample.bits[`DF(A, ADDR)], line), - "Read outside initialized DDR"); - reading_ids[id] = '{key, 0, ar_count, line}; - ar_count++; - end - if (r.sample.valid && r.sample.ready) begin - int key = reading_ids[r_id].key; - r_beats++; - if (last_r_id >= 0 && last_r_id != r_id && reading_ids.exists( - last_r_id - ) && reading_ids[last_r_id].serial == last_r_serial) - interleaves++; - last_r_id = r_id; - last_r_serial = reading_ids[r_id].serial; - if (reading_ids[r_id].beat == `DDR_BEATS - 1) begin - if (reading_ids[r_id].serial < max_r_serial) r_reorders++; - else max_r_serial = reading_ids[r_id].serial; - commands[key].done = 1; - reading_ids.delete(r_id); - end else reading_ids[r_id].beat++; - r_active = 0; - r_cursor = (r_id + 1) % 16; - end - if (b.sample.valid && b.sample.ready) begin - if (writing_ids[b_id].serial < max_b_serial) b_reorders++; - else max_b_serial = writing_ids[b_id].serial; - commands[writing_ids[b_id].key].done = 1; - writing_ids.delete(b_id); - b_active = 0; - end - for (int c = 0; c < 2; c++) - if (resp[c].sample.valid && resp[c].sample.ready) begin - result_item actual = new; - int key = (c << 12) | int'(resp[c].sample.bits[`DF(RESP, ID)]); - verify_contract(!$isunknown(resp[c].sample.bits) && commands.exists(key - ) && commands[key].done, - "Early/unknown line response before final R or B"); - actual.set_transaction_id(key); - actual.client = c; - actual.tag = resp[c].sample.bits[`DF(RESP, ID)]; - actual.data = resp[c].sample.bits[`DF(RESP, DATA)]; - actual.error = resp[c].sample.bits[`DF(RESP, ERROR)]; - scoreboard.actual_export.write(actual); - if (actual.error) error_results++; - returned[c]++; - commands.delete(key); - end - end - @(negedge ctl.clock); - aw.ready = !ctl.reset && !hold_aw && cycle % 5 != 0; - w.ready = !ctl.reset && !hold_w && cycle % 7 >= 2; - ar.ready = !ctl.reset && !hold_ar && cycle % 4 != 0; - for (int c = 0; c < 2; c++) resp[c].ready = !ctl.reset && !hold_resp[c] && cycle % 5 != c; - if (!r_active && !hold_r) - for (int off = 0; off < 16; off++) begin - int id = (r_cursor + off) % 16; - if (!r_active && reading_ids.exists(id)) begin - int key = reading_ids[id].key; - u64 address = commands[key].packet[`DF(REQ, ADDR)]; - int beat = reading_ids[id].beat; - r_active = 1; - r_id = id; - r_packet = '0; - r_packet[`DF(R, ID)] = id; - r_packet[`DF(R, DATA)] = reading_ids[id].line[beat*`DDR_DATA_BITS+:`DDR_DATA_BITS]; - r_packet[`DF(R, LAST)] = beat == `DDR_BEATS - 1; - if (read_error.exists(address) && beat == read_error[address]) - r_packet[`DF(R, RESP)] = beat % 2 ? 3 : 2; - end - end - if (!b_active && !hold_b) - for (int id = 15; id >= 0; id--) - if (!b_active && writing_ids.exists(id) && writing_ids[id].due <= cycle) begin - b_active = 1; - b_id = id; - b_packet = '0; - b_packet[`DF(B, ID)] = id; - if (write_error.exists(commands[writing_ids[id].key].packet[`DF(REQ, ADDR)])) - b_packet[ - `DF(B, RESP) - ] = write_error[commands[writing_ids[id].key].packet[ - `DF(REQ, ADDR) - ]]; - end - r.valid = !ctl.reset && r_active; - r.bits = r_packet; - b.valid = !ctl.reset && b_active; - b.bits = b_packet; - end - endtask - task submit(int client, int id, u64 address, bit write, bit [63:0] mask = '1); - req_t q = '0; - q[`DF(REQ, ID)] = id; - q[`DF(REQ, ADDR)] = address; - q[`DF(REQ, WRITE)] = write; - q[`DF(REQ, DATA)] = pattern(id + client * 17 + int'(address >> 6)); - q[`DF(REQ, MASK)] = mask; - @(negedge ctl.clock); - req[client].bits = q; - req[client].valid = 1; - do @(req[client].sample); while (!req[client].sample.ready); - @(negedge ctl.clock); - req[client].valid = 0; - endtask - task batch(u64 base, bit write); - for (int c = 0; c < 2; c++) - for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) - initialize(base + c * 4096 + i * 64, i + c * 7 + int'(base >> 6)); - fork - begin - for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) - submit(0, 12'h800 + i, base + i * 64, write, - i == 0 ? 0 : i == 1 ? '1 : i == 2 ? 64'h5555555555555555 : 64'haaaaaaaaaaaaaaaa); - end - begin - for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) - submit(1, 12'h800 + i, base + 4096 + i * 64, write); - end - join - endtask - task drain(); - wait (commands.num() == 0); - @(ctl.sample); - verify_contract( - !reading_ids.num()&&!writing_ids.num()&&!aw_ids.size()&&!w_lines.size()&&!assembling_beat&&!r_active&&!b_active, - "Backend did not drain"); - endtask - task joint_reset(); - @(negedge ctl.clock); - ctl.reset = 1; - req[0].valid = 0; - req[1].valid = 0; - repeat (3) @(ctl.sample); - cancelled += commands.num(); - commands.delete(); - reading_ids.delete(); - writing_ids.delete(); - aw_ids.delete(); - aw_keys.delete(); - aw_serials.delete(); - w_lines.delete(); - scoreboard.expected.delete(); - scoreboard.actual.delete(); - assembling = '0; - assembling_beat = 0; - r_active = 0; - b_active = 0; - @(negedge ctl.clock); - ctl.reset = 0; - endtask - function int finished_commands(); - int count = 0; - foreach (commands[key]) if (commands[key].done) count++; - return count; - endfunction - task invalid_scenario(); - int id, beats; - bit aw_seen; - if (invalid_case == 0) begin - submit(0, 0, 'h10001, 0); - end else if (invalid_case == 1) begin - @(negedge ctl.clock); - r.bits = '0; - r.bits[`DF(R, ID)] = 0; - r.valid = 1; - end else if (invalid_case == 2 || invalid_case == 6) begin - aw.ready = 1; - w.ready = invalid_case == 6; - submit(0, 0, 'h10000, 1); - beats = 0; - aw_seen = 0; - do begin - @(ctl.sample); - if (aw.sample.valid && aw.sample.ready) begin - id = aw.sample.bits[`DF(A, ID)]; - aw_seen = 1; - end - if (w.sample.valid && w.sample.ready) beats++; - end while (!aw_seen || (invalid_case == 6 && beats != `DDR_BEATS)); - @(negedge ctl.clock); - b.bits = '0; - b.bits[`DF(B, ID)] = id; - b.bits[`DF(B, RESP)] = invalid_case == 6 ? 1 : 0; - b.valid = 1; - end else begin - ar.ready = 1; - submit(0, 0, 'h10000, 0); - do @(ctl.sample); while (!ar.sample.valid || !ar.sample.ready); - id = ar.sample.bits[`DF(A, ID)]; - for (int beat = 0; beat < (invalid_case == 4 ? `DDR_BEATS : 1); beat++) begin - @(negedge ctl.clock); - r.bits = '0; - r.bits[`DF(R, ID)] = id; - r.bits[`DF(R, LAST)] = invalid_case == 3; - r.bits[`DF(R, RESP)] = invalid_case == 5 ? 1 : 0; - r.valid = 1; - do @(r.sample); while (!r.sample.ready); - end - end - repeat (8) @(ctl.sample); - `uvm_fatal("MISSING_ASSERTION", "Invalid protocol did not trigger expected DUT assertion") - endtask - task execute(); - int before_aw, before_w, before_ar, before_r; - int initial_returned; - ctl.reset = 1; - req[0].valid = 0; - req[1].valid = 0; - req[0].bits = '0; - req[1].bits = '0; - resp[0].ready = 0; - resp[1].ready = 0; - aw.ready = 0; - ar.ready = 0; - w.ready = 0; - b.valid = 0; - b.bits = '0; - r.valid = 0; - r.bits = '0; - repeat (5) @(negedge ctl.clock); - ctl.reset = 0; - if (invalid_case >= 0) begin - invalid_scenario(); - return; - end - fork - service(); - join_none - // W must progress even when AW has never been accepted; all eight slots stay live until B. - hold_aw = 1; - hold_b = 1; - hold_resp[0] = 1; - batch('h10000, 1); - wait (w_bursts == `DDR_SLOTS); - verify_contract(aw_count == 0 && returned[0] == 0 && returned[1] == 0, - "W-before-AW or B-delayed behavior failed"); - hold_aw = 0; - wait (aw_count == `DDR_SLOTS); - initialize('h18000, 99); - fork - submit(0, 12'h800, 'h18000, 1); - begin - hold_b = 0; - wait (returned[1] == `DDR_SLOTS_PER_CLIENT); - verify_contract(returned[0] == 0, "Client0 backpressure leaked"); - hold_resp[0] = 0; - end - join - drain(); - for (int c = 0; c < 2; c++) - for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) - submit(c, 12'h900 + i, 'h10000 + c * 4096 + i * 64, 0); - drain(); - // Conversely allow addresses to run ahead of every W beat. - before_aw = aw_count; - before_w = w_bursts; - hold_w = 1; - batch('h20000, 1); - wait (aw_count == before_aw + `DDR_SLOTS); - verify_contract(w_bursts == before_w, "AW-before-W stalled ordering failed"); - hold_w = 0; - drain(); - // Concurrent reads and writes: hold B until all independent reads return. - hold_b = 1; - initial_returned = returned[0] + returned[1]; - for (int c = 0; c < 2; c++) - for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) - initialize('h30000 + c * 4096 + i * 64, 30 + i); - fork - begin - for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) - submit(0, 12'ha00 + i, 'h30000 + i * 64, i[0]); - end - begin - for (int i = 0; i < `DDR_SLOTS_PER_CLIENT; i++) - submit(1, 12'ha00 + i, 'h31000 + i * 64, i[0]); - end - join - wait (returned[0] + returned[1] == initial_returned + 2 * ((`DDR_SLOTS_PER_CLIENT + 1) / 2)); - hold_b = 0; - drain(); - for (int i = 0; i < 4; i++) begin - initialize('h40000 + i * 64, 40 + i); - read_error['h40000+i*64] = i % `DDR_BEATS; - submit(i % 2, 12'hb00 + i, 'h40000 + i * 64, 0); - end - initialize('h41000, 44); - write_error['h41000] = 2; - submit(1, 12'hb10, 'h41000, 1); - initialize('h41040, 45); - write_error['h41040] = 3; - submit(0, 12'hb10, 'h41040, 1); - drain(); - before_ar = ar_count; - hold_r = 1; - batch('h48000, 0); - wait (ar_count == before_ar + `DDR_SLOTS); - r_cursor = `DDR_ID_BASE; - hold_r = 0; - drain(); - // Each byte strobe is independently checked, then read back through the reference. - for (int i = 0; i < 64; i++) begin - initialize('h50000 + i * 64, 50 + i); - submit(i % 2, 12'hc00 + i, 'h50000 + i * 64, 1, 64'h1 << i); - drain(); - submit(i % 2, 12'hd00 + i, 'h50000 + i * 64, 0); - drain(); - end - // Coordinated reset cancels queued AW-only, W-only and stalled line completions. - before_aw = aw_count; - hold_w = 1; - batch('h60000, 1); - wait (aw_count == before_aw + `DDR_SLOTS); - joint_reset(); - hold_w = 0; - before_w = w_bursts; - hold_aw = 1; - batch('h70000, 1); - wait (w_bursts == before_w + `DDR_SLOTS); - joint_reset(); - hold_aw = 0; - before_ar = ar_count; - before_r = r_beats; - hold_r = 1; - batch('h78000, 0); - wait (ar_count == before_ar + `DDR_SLOTS); - hold_r = 0; - wait (r_beats >= before_r + 1); - joint_reset(); - hold_resp[0] = 1; - hold_resp[1] = 1; - batch('h80000, 0); - do @(ctl.sample); while (finished_commands() != `DDR_SLOTS); - joint_reset(); - hold_resp[0] = 0; - hold_resp[1] = 0; - batch('h90000, 0); - drain(); - verify_contract( - peak==`DDR_SLOTS&&(`DDR_BEATS==1?interleaves==0:interleaves>0)&&b_reorders>0&&r_reorders>0&&request_stalls>0&&response_stalls>0&&aw_stalls>0&&w_stalls>0&&error_results==6&&cancelled==4*`DDR_SLOTS, - "Missing concurrency, stalls, errors or reset coverage"); - verify_contract(accepted == returned[0] + returned[1] + cancelled, "Lost line response"); - `uvm_info( - "DDR_PASS", - $sformatf( - "accepted=%0d returned=%0d/%0d cancelled=%0d peak=%0d errors=%0d readInterleave=%0d R/Breorder=%0d/%0d stalls req/resp/AW/W=%0d/%0d/%0d/%0d; full64B payload/mask/tag and all queues drained", - accepted, returned[0], returned[1], cancelled, peak, error_results, interleaves, - r_reorders, b_reorders, request_stalls, response_stalls, aw_stalls, w_stalls), - UVM_LOW) - endtask - function void final_phase(uvm_phase phase); - ddr_ref_destroy(model); - super.final_phase(phase); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/profile.svh b/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/profile.svh deleted file mode 100644 index 6e72f7a55..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/resources/profile.svh +++ /dev/null @@ -1,20 +0,0 @@ -`ifndef DDR_PROFILE_SVH -`define DDR_PROFILE_SVH -`ifdef DDR_PROFILE64 -`define DDR_CONFIG "ddr64_config.svh" -`define DDR_PORTS "ddr64_ports.svh" -`elsif DDR_PROFILE256 -`define DDR_CONFIG "ddr256_config.svh" -`define DDR_PORTS "ddr256_ports.svh" -`elsif DDR_PROFILE512 -`define DDR_CONFIG "ddr512_config.svh" -`define DDR_PORTS "ddr512_ports.svh" -`elsif DDR_PROFILESIX -`define DDR_CONFIG "ddr_six_config.svh" -`define DDR_PORTS "ddr_six_ports.svh" -`else -`define DDR_CONFIG "ddr_config.svh" -`define DDR_PORTS "ddr_ports.svh" -`endif -`include `DDR_CONFIG -`endif diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Emit.scala deleted file mode 100644 index a30caeda8..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Emit.scala +++ /dev/null @@ -1,68 +0,0 @@ -package memcore.memory.ddr - -import chisel3._ -import java.nio.file.{Files, Paths} -import memcore.bus.axi4 -import memcore.bus.chi.snf.{LineRequest, LineResponse} - -object Emit extends App { - - def emit(p: Params, name: String, header: String): Unit = { - Files.createDirectories(Paths.get("build")) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Bridge(p), - firtoolOpts = args, - args = Array("--target-dir", s"build/$name", "--split-verilog") - ) - val defs = scala.collection.mutable.ArrayBuffer[String]( - s"`define DDR_DATA_BITS ${p.dataBits}", - s"`define DDR_BEAT_BYTES ${p.axi.bytes}", - s"`define DDR_BEATS ${p.beats}", - s"`define DDR_SLOTS ${p.slots}", - s"`define DDR_SLOTS_PER_CLIENT ${p.slotsPerClient}", - s"`define DDR_ID_BASE ${p.idBase}" - ) - val ports = scala.collection.mutable.ArrayBuffer[String]() - def channel( - kind: String, - prefix: String, - signal: String, - bundle: Bundle, - define: Boolean = true - ): Unit = { - var offset = 0 - for ((name, field) <- bundle.elements.toSeq) { - val width = field.getWidth - if (define) { - defs += s"`define DDR_${kind}_${name.toUpperCase}_OFFSET $offset" - defs += s"`define DDR_${kind}_${name.toUpperCase}_WIDTH $width" - } - ports += s".${prefix}_bits_$name($signal.bits[$offset +: $width])" - offset += width - } - if (define) defs += s"`define DDR_${kind}_WIDTH $offset" - ports += s".${prefix}_valid($signal.valid)" - ports += s".${prefix}_ready($signal.ready)" - } - for (c <- 0 until p.clients) { - channel("REQ", s"io_request_$c", s"req$c", new LineRequest(p.line), c == 0) - channel("RESP", s"io_response_$c", s"resp$c", new LineResponse(p.line), c == 0) - } - channel("A", "io_axi_aw", "aw", new axi4.Address(p.axi)) - channel("A", "io_axi_ar", "ar", new axi4.Address(p.axi), false) - channel("W", "io_axi_w", "w", new axi4.WriteData(p.axi)) - channel("B", "io_axi_b", "b", new axi4.WriteResponse(p.axi)) - channel("R", "io_axi_r", "r", new axi4.ReadData(p.axi)) - Files.writeString( - Paths.get(s"build/${header}_config.svh"), - "`ifndef DDR_CONFIG_SVH\n`define DDR_CONFIG_SVH\n" + defs.mkString("\n") + "\n`endif\n" - ) - Files.writeString(Paths.get(s"build/${header}_ports.svh"), ports.mkString(",\n") + "\n") - } - - emit(Params(idBase = 4), "Bridge", "ddr") - emit(Params(dataBits = 64, idBase = 4), "Bridge64", "ddr64") - emit(Params(dataBits = 256, idBase = 4), "Bridge256", "ddr256") - emit(Params(dataBits = 512, idBase = 4), "Bridge512", "ddr512") - emit(Params(slotsPerClient = 3, idBase = 4), "BridgeSix", "ddr_six") -} diff --git a/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Params.scala b/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Params.scala deleted file mode 100644 index 2aa1eed33..000000000 --- a/arch/src/main/scala/framework/mem-core/ddr/src/main/scala/Params.scala +++ /dev/null @@ -1,20 +0,0 @@ -package memcore.memory.ddr - -import memcore.bus.chi -import memcore.bus.axi4 - -case class Params( - line: chi.Params = chi.Params(), - clients: Int = 2, - slotsPerClient: Int = 4, - dataBits: Int = 128, - idBits: Int = 4, - idBase: Int = 0) { - require(line.addressBits >= 6) - require(clients > 0 && slotsPerClient > 0) - require(Set(64, 128, 256, 512).contains(dataBits)) - val slots = clients * slotsPerClient - require(slots > 1 && idBase >= 0 && BigInt(idBase) + slots <= (BigInt(1) << idBits)) - val axi = axi4.Params(line.addressBits, dataBits, idBits) - val beats = 512 / dataBits -} diff --git a/arch/src/main/scala/framework/mem-core/fetch/build.sc b/arch/src/main/scala/framework/mem-core/fetch/build.sc deleted file mode 100644 index bccef01f1..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/build.sc +++ /dev/null @@ -1,10 +0,0 @@ -import mill._ -import mill.scalalib._ -object fetch extends SbtModule { - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") - override def mainClass = Some("memcore.memory.fetch.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch.el b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch.el deleted file mode 100644 index b72d899ed..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch.el +++ /dev/null @@ -1,50 +0,0 @@ -// Format Version: 2 -// 4-byte groups, 16-bit halfword/RVC alignment, one 64-bit word request. -CHECKSUM: "1717534490 3493169012" -INSTANCE: fetch_tb.dut -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Fetch privilege must be U, S or M" -Block 2 "3303546747" "if (1)" -Block 3 "784000845" "$error(\"Assertion failed: Fetch privilege must be U, S or M\n at Fetch.scala:73 assert(io.context.privilege =/= 2.U, \\"Fetch privilege must be U, S or M\\")\n\");" -Block 5 "3303546747" "if (1)" -Block 6 "2616558672" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Fetch reset vector must be instruction aligned" -Block 10 "1637879550" "if (1)" -Block 11 "1142822639" "$error(\"Assertion failed: Fetch reset vector must be instruction aligned\n at Fetch.scala:74 assert(pc(log2Ceil(p.instructionBytes) - 1, 0) === 0.U, \\"Fetch reset vector must be instruction aligned\\")\n\");" -Block 13 "1637879550" "if (1)" -Block 14 "2917930836" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Fetch maintenance response must acknowledge completion" -Block 18 "2033796801" "if (1)" -Block 19 "2717912749" "$error(\"Assertion failed: Fetch maintenance response must acknowledge completion\n at Fetch.scala:105 assert(io.maintained.bits, \\"Fetch maintenance response must acknowledge completion\\")\n\");" -Block 21 "2033796801" "if (1)" -Block 22 "2015233156" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Fetch redirect must be instruction aligned" -Block 26 "2247434341" "if (1)" -Block 27 "1751189408" "$error(\"Assertion failed: Fetch redirect must be instruction aligned\n at Fetch.scala:109 assert(io.redirect.bits(log2Ceil(p.instructionBytes) - 1, 0) === 0.U, \\"Fetch redirect must be instruction aligned\\")\n\");" -Block 29 "2247434341" "if (1)" -Block 30 "1248359523" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Fetch flush requires a restart redirect" -Block 34 "1762317919" "if (1)" -Block 35 "4112421248" "$error(\"Assertion failed: Fetch flush requires a restart redirect\n at Fetch.scala:115 assert(io.redirect.valid, \\"Fetch flush requires a restart redirect\\")\n\");" -Block 37 "1762317919" "if (1)" -Block 38 "564188382" "$fatal;" - -CHECKSUM: "1717534490 1587966699" -INSTANCE: fetch_tb.dut -ANNOTATION: "Only full enabled contract-failure vectors: privilege 2, odd start/redirect PC, incomplete maintenance, or flush without restart. Each is independently tested by an exact negative target; all idle/reset/pure comparator vectors remain measured." -Condition 2 "1830893280" "(_GEN_1 & (io_context_privilege == 2'h2)) 1 -1" (3 "11") -Condition 4 "2086071732" "(_GEN_1 & pc[0]) 1 -1" (3 "11") -Condition 5 "1305965348" "(_GEN_0 & ((~reset)) & ((~io_maintained_bits))) 1 -1" (4 "111") -Condition 6 "1743659560" "(io_redirect_valid & ((~reset)) & io_redirect_bits[0]) 1 -1" (4 "111") -Condition 7 "2120892050" "(io_flush & ((~reset)) & ((~io_redirect_valid))) 1 -1" (4 "111") - -CHECKSUM: "1717534490 202402385" -INSTANCE: fetch_tb.dut -ANNOTATION: "Word addresses are assigned pc & ~7; execute and maintenance request bits are literal true. Every deliverable packet comes from an accepted halfword-aligned fetch, so PC bit 0 is zero; its full 4-byte group mask is 11 or 10, so mask bit 1 is one. Reset-only PC/readPc/input values remain measured, and no instruction data is waived." -Toggle io_packet_bits_pc [0] "net io_packet_bits_pc[63:0]" -Toggle io_packet_bits_mask [1] "net io_packet_bits_mask[1:0]" -Toggle io_request_bits_addr [2:0] "net io_request_bits_addr[63:0]" -Toggle io_request_bits_execute "net io_request_bits_execute" -Toggle io_maintenance_bits "net io_maintenance_bits" -Toggle command_addr [2:0] "reg command_addr[63:0]" -Toggle answer_pc [0] "reg answer_pc[63:0]" -Toggle answer_mask [1] "reg answer_mask[1:0]" diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch.f b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch.f deleted file mode 100644 index 2102a5b94..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Fetch/filelist.f -@RESOURCES@/fetch_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_ack.f b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_ack.f deleted file mode 100644 index 49acb9a66..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_ack.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+FETCH_TOP=fetch_bad_ack_tb -+define+FETCH_BAD_ACK -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Fetch/filelist.f -@RESOURCES@/fetch_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_ack.toml b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_ack.toml deleted file mode 100644 index b4785e84f..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_ack.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Fetch maintenance response must acknowledge completion" diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_flush.f b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_flush.f deleted file mode 100644 index e80751594..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_flush.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+FETCH_TOP=fetch_bad_flush_tb -+define+FETCH_BAD_FLUSH -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Fetch/filelist.f -@RESOURCES@/fetch_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_flush.toml b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_flush.toml deleted file mode 100644 index d829076ff..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_flush.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Fetch flush requires a restart redirect" diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_privilege.f b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_privilege.f deleted file mode 100644 index 036ddd1f2..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_privilege.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+FETCH_TOP=fetch_bad_privilege_tb -+define+FETCH_BAD_PRIVILEGE -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Fetch/filelist.f -@RESOURCES@/fetch_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_privilege.toml b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_privilege.toml deleted file mode 100644 index b6e96cd3f..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_privilege.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Fetch privilege must be U, S or M" diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_redirect.f b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_redirect.f deleted file mode 100644 index 80ea2e2f3..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_redirect.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+FETCH_TOP=fetch_bad_redirect_tb -+define+FETCH_BAD_REDIRECT -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Fetch/filelist.f -@RESOURCES@/fetch_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_redirect.toml b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_redirect.toml deleted file mode 100644 index 1f743d176..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_redirect.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Fetch redirect must be instruction aligned" diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_reset.f b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_reset.f deleted file mode 100644 index 598dcc455..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_reset.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+FETCH_TOP=fetch_bad_reset_tb -+define+FETCH_BAD_RESET -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/Fetch/filelist.f -@RESOURCES@/fetch_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_reset.toml b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_reset.toml deleted file mode 100644 index b6a254790..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_bad_reset.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Fetch reset vector must be instruction aligned" diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_tb.sv b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_tb.sv deleted file mode 100644 index 6ac96319b..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/fetch_tb.sv +++ /dev/null @@ -1,87 +0,0 @@ -`include "fetch_config.svh" -`ifndef FETCH_TOP -`define FETCH_TOP fetch_tb -`endif -module `FETCH_TOP; - import uvm_pkg::*; - import fetch_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - fetch_control_if control (clock); - stream_if #(`FETCH_REQUEST_WIDTH) request ( - clock, - control.reset - ); - stream_if #(`FETCH_RESPONSE_WIDTH) response ( - clock, - control.reset - ); - // CPU redirection explicitly cancels a held frontend packet; it never cancels the word port. - stream_if #(`FETCH_PACKET_WIDTH) sink_if ( - clock, - control.reset || control.redirect_valid || control.flush - ); - stream_if #(1) maintenance ( - clock, - control.reset - ); - stream_if #(1) maintained ( - clock, - control.reset - ); - bit read_pending = 0, maintenance_pending = 0; - always @(posedge clock) begin - if (control.reset) begin - read_pending = 0; - maintenance_pending = 0; - end else begin - if (request.valid && request.ready) begin - if (read_pending || maintenance_pending) $fatal(1, "Fetch overlapped backend transactions"); - read_pending = 1; - end - if (response.valid && response.ready) begin - if (!read_pending) $fatal(1, "Fetch accepted an unsolicited word response"); - read_pending = 0; - end - if (maintenance.valid && maintenance.ready) begin - if (read_pending || maintenance_pending) - $fatal(1, "Maintenance began before draining the word"); - maintenance_pending = 1; - end - if (maintained.valid && maintained.ready) begin - if (!maintenance_pending) - $fatal(1, "Fetch accepted an unsolicited maintenance acknowledgement"); - maintenance_pending = 0; - end - if ((control.redirect_valid || control.flush) && sink_if.valid) - $fatal(1, "Cancelled packet was visible to the CPU"); - end - end - Fetch dut ( - .clock(clock), - .reset(control.reset), - .io_resetVector(control.reset_vector), - .io_context_privilege(control.privilege), - .io_context_satp(control.satp), - .io_context_sum(control.sum), - .io_context_mxr(control.mxr), - .io_redirect_valid(control.redirect_valid), - .io_redirect_bits(control.redirect_pc), - .io_flush(control.flush), - .io_invalidate(1'b0), - .io_npc(control.npc), - `include "fetch_ports.svh" - ); - initial begin - uvm_config_db#(virtual fetch_control_if)::set(null, "uvm_test_top*", "control", control); - uvm_config_db#(virtual stream_if #(`FETCH_REQUEST_WIDTH))::set(null, "uvm_test_top*", "request", - request); - uvm_config_db#(virtual stream_if #(`FETCH_RESPONSE_WIDTH))::set(null, "uvm_test_top*", - "response", response); - uvm_config_db#(virtual stream_if #(`FETCH_PACKET_WIDTH))::set(null, "uvm_test_top*", "packet", - sink_if); - uvm_config_db#(virtual stream_if #(1))::set(null, "uvm_test_top*", "maintenance", maintenance); - uvm_config_db#(virtual stream_if #(1))::set(null, "uvm_test_top*", "maintained", maintained); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/interface.sv b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/interface.sv deleted file mode 100644 index 459a4b770..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/interface.sv +++ /dev/null @@ -1,14 +0,0 @@ -interface fetch_control_if ( - input logic clock -); - logic reset, redirect_valid, flush; - logic [63:0] reset_vector, redirect_pc, satp, npc; - logic [1:0] privilege; - logic sum, mxr; - clocking cb @(posedge clock); - default input #1step output #0; - input npc; - output reset, redirect_valid, redirect_pc, flush, privilege, satp, mxr; - output context_sum = sum; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/package.sv deleted file mode 100644 index 52ad5b29b..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/package.sv +++ /dev/null @@ -1,20 +0,0 @@ -`include "fetch_config.svh" -package fetch_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `define FF(kind, field) `FETCH_``kind``_``field``_OFFSET +: `FETCH_``kind``_``field``_WIDTH - import "DPI-C" function longint unsigned fetch_ref_word( - input longint unsigned address, - input int unsigned version - ); - import "DPI-C" function int unsigned fetch_ref_group( - input longint unsigned address, - input int unsigned version - ); - import "DPI-C" function int unsigned fetch_ref_instruction( - input longint unsigned address, - input int unsigned version - ); - `include "test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/test.svh b/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/test.svh deleted file mode 100644 index 5bce60020..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/resources/test.svh +++ /dev/null @@ -1,410 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual fetch_control_if control; - virtual stream_if #(`FETCH_REQUEST_WIDTH) request; - virtual stream_if #(`FETCH_RESPONSE_WIDTH) response; - virtual stream_if #(`FETCH_PACKET_WIDTH) packet; - virtual stream_if #(1) maintenance, maintained; - int checked = 0, instructions = 0, reads = 0, drains = 0, barriers = 0, cancellations = 0; - longint unsigned cursor; - bit pending_half = 0; - bit [15:0] first_half; - longint unsigned partial_pc; - bit [63:0] saved_word; - bit [`FETCH_REQUEST_WIDTH-1:0] saved_request; - int version = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 200us; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual fetch_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`FETCH_REQUEST_WIDTH))::get( - this, "", "request", request - ) || !uvm_config_db#(virtual stream_if #(`FETCH_RESPONSE_WIDTH))::get( - this, "", "response", response - ) || !uvm_config_db#(virtual stream_if #(`FETCH_PACKET_WIDTH))::get( - this, "", "packet", packet - ) || !uvm_config_db#(virtual stream_if #(1))::get( - this, "", "maintenance", maintenance - ) || !uvm_config_db#(virtual stream_if #(1))::get( - this, "", "maintained", maintained - )) - `uvm_fatal("VIF", "Fetch interfaces missing") - endfunction - task redirect(longint unsigned target, bit flush = 0); - control.cb.redirect_pc <= target; - control.cb.redirect_valid <= 1; - control.cb.flush <= flush; - @(control.cb); - control.cb.redirect_valid <= 0; - control.cb.flush <= 0; - cursor = target; - pending_half = 0; - endtask - task offered(longint unsigned expected_pc); - do @(request.sample); while (!request.sample.valid); - saved_request = request.sample.bits; - if (saved_request[ - `FF(REQUEST, ADDR) - ] !== (expected_pc & ~64'h7) || saved_request[ - `FF(REQUEST, CONTEXT_PRIVILEGE) - ] !== control.privilege || saved_request[ - `FF(REQUEST, CONTEXT_SATP) - ] !== control.satp || saved_request[ - `FF(REQUEST, CONTEXT_SUM) - ] !== control.sum || saved_request[ - `FF(REQUEST, CONTEXT_MXR) - ] !== control.mxr || saved_request[ - `FF(REQUEST, EXECUTE) - ] !== 1) - `uvm_fatal("READ", "Word request changed PC/context/execute metadata") - saved_word = fetch_ref_word(expected_pc & ~64'h7, version); - endtask - task accept_word(); - request.ready <= 1; - @(request.sample); - request.ready <= 0; - reads++; - endtask - task reply(bit pf = 0, bit af = 0); - bit [511:0] line; - response.bits <= '0; - response.bits[`FF(RESPONSE, DATA)] <= saved_word; - for (int word = 0; word < 8; word++) - line[word*64+:64] = - fetch_ref_word((saved_request[`FF(REQUEST, ADDR)] & ~64'h3f) + word * 8, version); - response.bits[`FF(RESPONSE, LINE)] <= line; - response.bits[`FF(RESPONSE, LINEEXECUTABLE)] <= 0; - response.bits[`FF(RESPONSE, PAGEFAULT)] <= pf; - response.bits[`FF(RESPONSE, ACCESSFAULT)] <= af; - response.valid <= 1; - do @(response.sample); while (!response.sample.ready); - response.valid <= 0; - endtask - task check_packet(bit pf = 0, bit af = 0); - bit [31:0] data; - bit [ 1:0] mask; - bit [63:0] pc; - do @(packet.sample); while (!packet.sample.valid); - pc = packet.sample.bits[`FF(PACKET, PC)]; - data = packet.sample.bits[`FF(PACKET, DATA)]; - mask = packet.sample.bits[`FF(PACKET, MASK)]; - if (pc !== cursor || data !== (pf || af ? 0 : fetch_ref_group( - cursor, version - )) || mask !== (cursor[1] ? 2'b10 : 2'b11) || packet.sample.bits[ - `FF(PACKET, PAGEFAULT) - ] !== pf || packet.sample.bits[ - `FF(PACKET, ACCESSFAULT) - ] !== af) - `uvm_fatal("PACKET", $sformatf( - "Fetch packet mismatch cursor=%h got=%h data=%h", cursor, pc, data)) - // Independent instruction reconstruction exercises the same packet boundaries that IBuf consumes. - for (int lane = 0; lane < 2; lane++) - if (mask[lane]) begin - longint unsigned here = (cursor & ~64'h3) + lane * 2; - bit [15:0] halfword = data[lane*16+:16]; - if (pf || af) begin - if (pending_half && here !== partial_pc + 2) - `uvm_fatal("SECOND_HALF", "Fault did not apply to the second halfword") - pending_half = 0; - end else if (pending_half) begin - if ({halfword, first_half} !== fetch_ref_instruction( - partial_pc, version - ) || here !== partial_pc + 2) - `uvm_fatal("RVC", - "32-bit instruction split across groups was reconstructed incorrectly") - pending_half = 0; - instructions++; - end else if (halfword[1:0] != 2'b11) begin - if (halfword !== fetch_ref_instruction(here, version)) - `uvm_fatal("RVC", "Compressed instruction mismatch") - instructions++; - end else begin - first_half = halfword; - partial_pc = here; - pending_half = 1; - end - end - repeat (3) @(control.cb); - packet.ready <= 1; - @(packet.sample); - packet.ready <= 0; - checked++; - cursor = (cursor & ~64'h3) + 4; - endtask - task fetch_one(bit pf = 0, bit af = 0); - offered(cursor); - repeat (3) @(control.cb); - accept_word(); - repeat (2) @(control.cb); - reply(pf, af); - check_packet(pf, af); - endtask - task discarded_reply(); - reply(); - @(control.cb); - if (packet.valid) `uvm_fatal("CANCEL", "Old fetch response escaped redirection") - drains++; - endtask - task fence_ack(); - do @(maintenance.sample); while (!maintenance.sample.valid); - repeat (3) begin - @(control.cb); - if (request.valid || packet.valid) - `uvm_fatal("FENCE", "Fetch proceeded before maintenance request") - end - maintenance.ready <= 1; - @(maintenance.sample); - maintenance.ready <= 0; - repeat (3) begin - @(control.cb); - if (request.valid || packet.valid) - `uvm_fatal("FENCE", "Fetch proceeded before maintenance completed") - end -`ifdef FETCH_BAD_ACK - maintained.bits <= 0; -`else - maintained.bits <= 1; -`endif - maintained.valid <= 1; - do @(maintained.sample); while (!maintained.sample.ready); - maintained.valid <= 0; - maintained.bits <= 0; - barriers++; - endtask - task reset_fetch(int phase_index); - offered(cursor); - if (phase_index != 0) begin - accept_word(); - if (phase_index == 2) begin - reply(); - do @(packet.sample); while (!packet.sample.valid); - end - end - control.cb.reset <= 1; - response.valid <= 0; - maintained.valid <= 1; - maintained.bits <= 0; - control.cb.redirect_valid <= 0; - control.cb.flush <= 0; - control.cb.privilege <= 2; - control.cb.satp <= '1; - control.reset_vector <= '1; - repeat (2) @(control.cb); - control.cb.redirect_valid <= 1; - control.cb.redirect_pc <= '1; - control.cb.flush <= 1; - repeat (2) @(control.cb); - control.cb.redirect_valid <= 0; - control.cb.flush <= 1; - control.reset_vector <= 0; - repeat (2) @(control.cb); - control.cb.flush <= 0; - maintained.valid <= 0; - control.cb.privilege <= 3; - control.cb.satp <= 0; - repeat (2) @(control.cb); - control.cb.reset <= 0; - cursor = control.reset_vector; - pending_half = 0; - cancellations++; - fetch_one(); - endtask - task execute(); - control.reset = 1; -`ifdef FETCH_BAD_RESET - control.reset_vector = 1; -`else - control.reset_vector = 0; -`endif - control.redirect_valid = 0; - control.redirect_pc = 0; - control.flush = 0; -`ifdef FETCH_BAD_PRIVILEGE - control.privilege = 2; -`else - control.privilege = 3; -`endif - control.satp = 0; - control.sum = 0; - control.mxr = 0; - request.ready = 0; - response.valid = 0; - response.bits = '0; - packet.ready = 0; - maintenance.ready = 0; - maintained.valid = 0; - maintained.bits = 1; - cursor = 0; - repeat (4) @(control.cb); - control.cb.reset <= 0; -`ifdef FETCH_BAD_REDIRECT - redirect(1); - repeat (10) @(control.cb); - `uvm_fatal("NO_ASSERT", "Odd instruction PC was accepted") -`elsif FETCH_BAD_FLUSH - control.cb.flush <= 1; - repeat (10) @(control.cb); - `uvm_fatal("NO_ASSERT", "Flush without restart was accepted") -`else - for (int i = 0; i < 8; i++) fetch_one(); - // Zero-latency word owner: response VALID begins in the request handshake cycle. - offered(cursor); - fork - accept_word(); - reply(); - join - check_packet(); - // A stalled offer is immutable even after redirect and live context changes. - offered(cursor); - control.cb.privilege <= 2; - control.reset_vector <= 1; - @(control.cb); - control.cb.privilege <= 0; - control.reset_vector <= 0; - control.cb.satp <= 64'h8000123456789000; - control.cb.context_sum <= 1; - control.cb.mxr <= 1; - redirect(2); - repeat (3) @(control.cb); - if (request.bits !== saved_request || !request.valid) - `uvm_fatal("OFFER", "Redirect withdrew or changed a backpressured word offer") - accept_word(); - redirect(6); - discarded_reply(); - for (int i = 0; i < 4; i++) fetch_one(); - // Same-cycle response and redirect must drain the old word and suppress the packet. - offered(cursor); - accept_word(); - fork - redirect(62); - begin - discarded_reply(); - end - join - for (int i = 0; i < 4; i++) fetch_one(); - // A redirect in the same cycle as accepting the old word still drains that word. - offered(cursor); - fork - accept_word(); - redirect(66); - join - discarded_reply(); - for (int i = 0; i < 3; i++) fetch_one(); - // Cancel an already buffered packet under CPU backpressure. - offered(cursor); - accept_word(); - reply(); - do @(packet.sample); while (!packet.sample.valid); - packet.ready <= 1; - redirect(126); - packet.ready <= 0; - for (int i = 0; i < 3; i++) fetch_one(); - // At PC=4094 a 32-bit instruction starts at the last halfword of the page. - redirect(4094); - fetch_one(); - if (!pending_half) - `uvm_fatal("SECOND_HALF", "Page boundary did not leave a partial instruction") - fetch_one(1, 0); - redirect(8192); - fetch_one(0, 1); - // Flush must drain an old accepted read before requesting external maintenance. - offered(cursor); - accept_word(); - version = 1; - redirect(0, 1); - discarded_reply(); - fence_ack(); - for (int i = 0; i < 4; i++) fetch_one(); - // A fence arriving with a maintenance handshake needs another ordered acknowledgement. - offered(cursor); - accept_word(); - reply(); - do @(packet.sample); while (!packet.sample.valid); - redirect(18, 1); - do @(maintenance.sample); while (!maintenance.sample.valid); - fork - begin - maintenance.ready <= 1; - @(maintenance.sample); - maintenance.ready <= 0; - end - redirect(30, 1); - join - repeat (3) @(control.cb); - maintained.valid <= 1; - maintained.bits <= 1; - do @(maintained.sample); while (!maintained.sample.ready); - maintained.valid <= 0; - maintained.bits <= 0; - barriers++; - fence_ack(); - for (int i = 0; i < 3; i++) fetch_one(); - // Byte-memory patterns vary all transported instruction bits without duplicating the execution decoder. - for (int image = 2; image < 36; image++) begin - version = image; - control.cb.privilege <= image % 3 == 0 ? 3 : (image % 3 == 1 ? 1 : 0); - control.cb.satp <= image[0] ? 64'h8fffffffffffffff : 0; - control.cb.context_sum <= image[0]; - control.cb.mxr <= !image[0]; - redirect(image * 64); - for (int i = 0; i < 3; i++) fetch_one(); - end - control.cb.privilege <= 3; - control.cb.satp <= 0; - redirect(64'h2000); - fetch_one(); - for (int bit_index = 3; bit_index < 64; bit_index++) begin - redirect(64'h1 << bit_index); - fetch_one(0, bit_index >= 44); - end - for (int phase_index = 0; phase_index < 3; phase_index++) reset_fetch(phase_index); - // Joint reset cancels an accepted maintenance transaction and ignores its reset-cycle payload. - offered(cursor); - accept_word(); - reply(); - do @(packet.sample); while (!packet.sample.valid); - redirect(0, 1); - do @(maintenance.sample); while (!maintenance.sample.valid); - maintenance.ready <= 1; - @(maintenance.sample); - maintenance.ready <= 0; - control.cb.reset <= 1; - maintained.valid <= 1; - maintained.bits <= 0; - repeat (2) @(control.cb); - maintained.valid <= 0; - control.cb.reset <= 0; - cursor = control.reset_vector; - pending_half = 0; - cancellations++; - fetch_one(); - // READY can be high with VALID=0; unrelated payload cannot create a transaction. - request.ready <= 1; - maintenance.ready <= 1; - control.cb.redirect_pc <= '1; - @(control.cb); - control.cb.redirect_pc <= 0; - // Explicitly terminate the autonomous producer and cancel any trailing halfword by joint reset. - control.cb.reset <= 1; - response.valid <= 0; - maintained.valid <= 0; - pending_half = 0; - repeat (2) @(control.cb); - if (request.valid || packet.valid || maintenance.valid) - `uvm_fatal("RESET", "Reset retained a live frontend transaction") - `uvm_info("FETCH", $sformatf( - "Checked %0d packets, reconstructed %0d instructions, %0d accepted reads, %0d stale drains, %0d maintenance acknowledgements, %0d joint-reset cancellations", - checked, - instructions, - reads, - drains, - barriers, - cancellations - ), UVM_LOW) -`endif - endtask -endclass diff --git a/arch/src/main/scala/framework/mem-core/fetch/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/fetch/src/main/scala/Emit.scala deleted file mode 100644 index b4a8f6420..000000000 --- a/arch/src/main/scala/framework/mem-core/fetch/src/main/scala/Emit.scala +++ /dev/null @@ -1,72 +0,0 @@ -package memcore.memory.fetch - -import java.nio.file.{Files, Paths} - -object Emit extends App { - val p = Params() - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Fetch(p), - firtoolOpts = args, - args = Array("--target-dir", "build/Fetch", "--split-verilog") - ) - val definitions = scala.collection.mutable.ArrayBuffer[String]() - val ports = scala.collection.mutable.ArrayBuffer[String]() - - def fields( - kind: String, - prefix: String, - instance: String, - fs: Seq[(String, Int)] - ): Unit = { - var offset = 0 - for ((name, width) <- fs) { - definitions += s"`define FETCH_${kind}_${name.toUpperCase}_OFFSET $offset" - definitions += s"`define FETCH_${kind}_${name.toUpperCase}_WIDTH $width" - ports += s".$prefix$name($instance.bits[$offset +: $width])" - offset += width - } - definitions += s"`define FETCH_${kind}_WIDTH $offset" - } - - fields( - "REQUEST", - "io_request_bits_", - "request", - Seq( - "addr" -> 64, - "context_privilege" -> 2, - "context_satp" -> 64, - "context_sum" -> 1, - "context_mxr" -> 1, - "execute" -> 1 - ) - ) - fields( - "RESPONSE", - "io_response_bits_", - "response", - Seq("data" -> 64, "line" -> 512, "lineExecutable" -> 1, "pageFault" -> 1, "accessFault" -> 1) - ) - fields( - "PACKET", - "io_packet_bits_", - "sink_if", - Seq("pc" -> 64, "data" -> 32, "mask" -> p.lanes, "pageFault" -> 1, "accessFault" -> 1) - ) - for ( - (prefix, instance) <- Seq( - "request" -> "request", - "response" -> "response", - "packet" -> "sink_if", - "maintenance" -> "maintenance", - "maintained" -> "maintained" - ) - ) { - ports += s".io_${prefix}_valid($instance.valid)" - ports += s".io_${prefix}_ready($instance.ready)" - } - ports += ".io_maintenance_bits(maintenance.bits[0])" - ports += ".io_maintained_bits(maintained.bits[0])" - Files.write(Paths.get("build/fetch_config.svh"), (definitions.mkString("\n") + "\n").getBytes) - Files.write(Paths.get("build/fetch_ports.svh"), ports.mkString(",\n").getBytes) -} diff --git a/arch/src/main/scala/framework/mem-core/interlock/build.sc b/arch/src/main/scala/framework/mem-core/interlock/build.sc deleted file mode 100644 index 90648d244..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/build.sc +++ /dev/null @@ -1,10 +0,0 @@ -import mill._ -import mill.scalalib._ -object interlock extends SbtModule { - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") - override def mainClass = Some("memcore.memory.interlock.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/Cargo.lock b/arch/src/main/scala/framework/mem-core/interlock/src/csrc/Cargo.lock deleted file mode 100644 index a2560a2a9..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "interlock-reference" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/Cargo.toml b/arch/src/main/scala/framework/mem-core/interlock/src/csrc/Cargo.toml deleted file mode 100644 index ce45bc7a7..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "interlock-reference" -version = "0.1.0" -edition = "2021" -[lib] -crate-type = ["cdylib", "staticlib"] diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/src/lib.rs b/arch/src/main/scala/framework/mem-core/interlock/src/csrc/src/lib.rs deleted file mode 100644 index a61a9079a..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod ownership; -pub use ownership::*; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/src/ownership.rs b/arch/src/main/scala/framework/mem-core/interlock/src/csrc/src/ownership.rs deleted file mode 100644 index 3b3a906ee..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/csrc/src/ownership.rs +++ /dev/null @@ -1,99 +0,0 @@ -//! Transaction facts from real handshakes. No hardware cursor or phase FSM is mirrored. -use std::ffi::c_void; -#[derive(Clone, Copy)] -struct Access { base: u64, bytes: u64, write: bool } -struct Transaction { - id: u32, age: u64, sealed: bool, ranges: Vec, - pre: Vec, post: Vec, granted: bool, dma_ack: bool, -} -struct Model { - capacity: usize, address_bits: u32, line_bytes: u64, max_ranges: usize, - age: u64, live: Vec, maintenance: Option<(u32, u32, usize)>, -} -impl Model { - fn transaction(&self,id:u32)->&Transaction { self.live.iter().find(|t|t.id==id).expect("unknown transaction") } - fn transaction_mut(&mut self,id:u32)->&mut Transaction { self.live.iter_mut().find(|t|t.id==id).expect("unknown transaction") } - fn overlap(a:Access,b:Access)->bool { a.baseOption { - t.ranges.first().map(|first| { - let base=t.ranges.iter().map(|a|a.base).min().unwrap(); - let end=t.ranges.iter().map(|a|a.base+a.bytes).max().unwrap(); - Access{base,bytes:end-base,write:first.write} - }) - } - fn older_conflict(&self,t:&Transaction)->bool { - let a=Self::bounds(t).expect("memory grant without ranges"); - self.live.iter().filter(|old|old.age(p:*mut c_void)->&'a mut Model { &mut *p.cast::() } -#[no_mangle] -pub extern "C" fn interlock_ref_create(entries:u32,address_bits:u32,line_bytes:u32,max_ranges:u32)->*mut c_void { - assert!(entries>0&&address_bits<64&&line_bytes.is_power_of_two()&&max_ranges>0); - Box::into_raw(Box::new(Model{capacity:entries as usize,address_bits,line_bytes:line_bytes as u64,max_ranges:max_ranges as usize,age:0,live:Vec::new(),maintenance:None})).cast() -} -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_destroy(p:*mut c_void) { drop(Box::from_raw(p.cast::())); } -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_reset(p:*mut c_void) { let m=model(p);m.live.clear();m.maintenance=None;m.age=0; } -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_live(p:*mut c_void)->u32 { model(p).live.len() as u32 } -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_reserve(p:*mut c_void,id:u32) { - let m=model(p);assert!(m.live.len()0&&base.checked_add(bytes).is_some_and(|end|end<=(1<u8 { - let m=model(p);assert!(bytes.is_power_of_two()&&bytes<=8&&addr%bytes as u64==0); - assert!(addr/m.line_bytes==(addr+bytes as u64-1)/m.line_bytes); - if older!=0||reserve_now!=0{return 0;} - let cpu=Access{base:addr,bytes:bytes as u64,write:write!=0}; - u8::from(!m.live.iter().any(|t|!t.sealed||t.ranges.iter().any(|&a|Model::overlap(cpu,a)&&(cpu.write||a.write)))) -} -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_maint_offer(p:*mut c_void,id:u32,op:u32,first:u64,last:u64) { - let m=model(p);assert!(m.maintenance.is_none());let t=m.transaction(id);assert!(t.sealed&&!t.ranges.is_empty()); - let write=t.ranges[0].write; - if op==2 { assert!(write&&t.granted&&t.dma_ack); } - else { assert!(op==u32::from(write)&&!t.granted&&!m.older_conflict(t)); } - let done=if op==2{&t.post}else{&t.pre}; - let index=t.ranges.iter().enumerate().position(|(i,a)|!done[i]&&a.base==first&&a.base+a.bytes-m.line_bytes==last).expect("wrong or duplicate maintenance extent"); - m.maintenance=Some((id,op,index)); -} -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_maint_ack(p:*mut c_void,id:u32) { - let m=model(p);let (owner,op,index)=m.maintenance.take().expect("ACK without maintenance request");assert_eq!(owner,id); - let t=m.transaction_mut(id);if op==2{t.post[index]=true;}else{t.pre[index]=true;} -} -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_grant(p:*mut c_void,id:u32) { - let m=model(p);let t=m.transaction(id);assert!(t.sealed&&!t.ranges.is_empty()&&t.pre.iter().all(|b|*b)&&!t.granted&&!m.older_conflict(t)); - m.transaction_mut(id).granted=true; -} -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_dma_ack(p:*mut c_void,id:u32) { let t=model(p).transaction_mut(id);assert!(t.granted&&!t.dma_ack);t.dma_ack=true; } -#[no_mangle] -pub unsafe extern "C" fn interlock_ref_complete(p:*mut c_void,id:u32) { - let m=model(p);let t=m.transaction(id);assert!(t.sealed); - assert!(t.ranges.is_empty()||(t.granted&&t.dma_ack&&(!t.ranges[0].write||t.post.iter().all(|b|*b))),"premature completion"); - m.live.retain(|t|t.id!=id); -} diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/control_if.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/control_if.sv deleted file mode 100644 index 758c6018e..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/control_if.sv +++ /dev/null @@ -1,5 +0,0 @@ -interface interlock_control_if ( - input logic clock -); - bit start = 0, done = 0; -endinterface diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock.f deleted file mode 100644 index c6f620e16..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.f deleted file mode 100644 index c0dd50220..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_cancel_info_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.toml deleted file mode 100644 index 07bf6ed6b..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock cancel conflicts with the same command" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info_scenario.svh deleted file mode 100644 index 7cbcd7f94..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info_scenario.svh +++ /dev/null @@ -1,8 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -@(negedge clock); -cancel.valid = 1; -`ILF(cancel.bits, CANCEL, ID) = 7; -access_info.valid = 1; -`ILF(access_info.bits, ACCESS_INFO, ID) = 7; -`ILF(access_info.bits, ACCESS_INFO, LAST) = 1; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info_tb.sv deleted file mode 100644 index 2f719bd40..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_info_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_cancel_info_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_cancel_info_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.f deleted file mode 100644 index c3fdbb8f5..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_cancel_sealed_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.toml deleted file mode 100644 index dc66a7902..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock cancel requires an unsealed reservation" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed_scenario.svh deleted file mode 100644 index ee4ca4cec..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed_scenario.svh +++ /dev/null @@ -1,6 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 64, 0, 1); -@(negedge clock); -cancel.valid = 1; -`ILF(cancel.bits, CANCEL, ID) = 7; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed_tb.sv deleted file mode 100644 index 7550d92a2..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_sealed_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_cancel_sealed_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_cancel_sealed_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.f deleted file mode 100644 index ea5d2bb9d..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_cancel_unknown_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.toml deleted file mode 100644 index ebac80229..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock cancel ID has no reservation" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown_scenario.svh deleted file mode 100644 index a4a8587f2..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown_scenario.svh +++ /dev/null @@ -1,4 +0,0 @@ -// verilog_syntax: parse-as-statements -@(negedge clock); -cancel.valid = 1; -`ILF(cancel.bits, CANCEL, ID) = 7; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown_tb.sv deleted file mode 100644 index 298015b8f..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cancel_unknown_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_cancel_unknown_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_cancel_unknown_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.f deleted file mode 100644 index 7755f7b96..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_capacity_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.toml deleted file mode 100644 index 286cf7f26..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock range list must seal within capacity" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity_scenario.svh deleted file mode 100644 index f5f2761d3..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity_scenario.svh +++ /dev/null @@ -1,3 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -for (int i = 0; i < 8; i++) info(7, 1, 'h4000 + i * 128, 64, 0, 0); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity_tb.sv deleted file mode 100644 index 179a82e97..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_capacity_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_capacity_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_capacity_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.f deleted file mode 100644 index 6248b967c..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_cpu_alignment_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.toml deleted file mode 100644 index 34829bec4..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock CPU access must be naturally aligned" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment_scenario.svh deleted file mode 100644 index e41448a14..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment_scenario.svh +++ /dev/null @@ -1,5 +0,0 @@ -// verilog_syntax: parse-as-statements -@(negedge clock); -`ILF(cpu_query.bits, CPU_QUERY, VALID) = 1; -`ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = 3; -`ILF(cpu_query.bits, CPU_QUERY, PADDR) = 'h4001; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment_tb.sv deleted file mode 100644 index 59c2bc926..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_alignment_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_cpu_alignment_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_cpu_alignment_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.f deleted file mode 100644 index 5b860ef1a..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_cpu_size_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.toml deleted file mode 100644 index 75d374e71..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock CPU size must be 1, 2, 4 or 8 bytes" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size_scenario.svh deleted file mode 100644 index 42fc29d8b..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size_scenario.svh +++ /dev/null @@ -1,4 +0,0 @@ -// verilog_syntax: parse-as-statements -@(negedge clock); -`ILF(cpu_query.bits, CPU_QUERY, VALID) = 1; -`ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = 4; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size_tb.sv deleted file mode 100644 index ddccba87c..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_cpu_size_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_cpu_size_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_cpu_size_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.f deleted file mode 100644 index 02ddcb98c..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_direction_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.toml deleted file mode 100644 index 6f78b4bfd..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock ranges must have one access direction" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction_scenario.svh deleted file mode 100644 index 34d02d785..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction_scenario.svh +++ /dev/null @@ -1,4 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 64, 0, 0); -info(7, 1, 'h5000, 64, 1, 1); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction_tb.sv deleted file mode 100644 index 6e55aee8b..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_direction_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_direction_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_direction_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.f deleted file mode 100644 index 31dbfb80c..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_dma_failed_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.toml deleted file mode 100644 index 10a115658..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock DMA failed" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed_scenario.svh deleted file mode 100644 index 3b5b1b04d..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed_scenario.svh +++ /dev/null @@ -1,15 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 64, 1); -maintenance_accept(); -maintenance_ack(7, 1); -do @(posedge clock); while (!maintained.ready); -@(negedge clock); -maintained.valid = 0; -grant.ready = 1; -do @(posedge clock); while (!grant.valid); -@(negedge clock); -grant.ready = 0; -done.valid = 1; -`ILF(done.bits, DONE, TAG) = 7; -`ILF(done.bits, DONE, OK) = 0; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed_tb.sv deleted file mode 100644 index 8c8a0c957..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_failed_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_dma_failed_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_dma_failed_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.f deleted file mode 100644 index 40a4d9123..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_dma_ungranted_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.toml deleted file mode 100644 index c03f2dc8f..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock DMA done requires a granted access" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted_scenario.svh deleted file mode 100644 index 4b6151177..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted_scenario.svh +++ /dev/null @@ -1,7 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 64, 1); -@(negedge clock); -done.valid = 1; -`ILF(done.bits, DONE, TAG) = 7; -`ILF(done.bits, DONE, OK) = 1; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted_tb.sv deleted file mode 100644 index f803fa281..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_ungranted_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_dma_ungranted_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_dma_ungranted_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.f deleted file mode 100644 index a4ff5ef32..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_dma_unknown_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.toml deleted file mode 100644 index c8de0ff0b..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock DMA done tag has no reservation" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown_scenario.svh deleted file mode 100644 index 05f9f5e37..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown_scenario.svh +++ /dev/null @@ -1,5 +0,0 @@ -// verilog_syntax: parse-as-statements -@(negedge clock); -done.valid = 1; -`ILF(done.bits, DONE, TAG) = 7; -`ILF(done.bits, DONE, OK) = 1; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown_tb.sv deleted file mode 100644 index 2095fc13a..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_dma_unknown_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_dma_unknown_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_dma_unknown_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.f deleted file mode 100644 index 3120d9da8..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_duplicate_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.toml deleted file mode 100644 index 5b2a37f26..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock dispatch ID is already live" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate_scenario.svh deleted file mode 100644 index e99e77c88..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate_scenario.svh +++ /dev/null @@ -1,5 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -@(negedge clock); -dispatch.valid = 1; -`ILF(dispatch.bits, DISPATCH, ID) = 7; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate_tb.sv deleted file mode 100644 index c175edc15..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_duplicate_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_duplicate_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_duplicate_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.f deleted file mode 100644 index 0f4e5672c..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_empty_range_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.toml deleted file mode 100644 index b228dd89f..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock memory range is empty" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range_scenario.svh deleted file mode 100644 index a8fe960d9..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range_scenario.svh +++ /dev/null @@ -1,3 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 0, 0); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range_tb.sv deleted file mode 100644 index b5d405352..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_empty_range_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_empty_range_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_empty_range_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.f deleted file mode 100644 index a1500033b..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_info_duplicate_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.toml deleted file mode 100644 index 8d437efdb..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock access info is already declared" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate_scenario.svh deleted file mode 100644 index 2e1c233b5..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate_scenario.svh +++ /dev/null @@ -1,5 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 64, 0); -@(negedge clock); -access_info.valid = 1; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate_tb.sv deleted file mode 100644 index ea5583f8d..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_duplicate_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_info_duplicate_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_info_duplicate_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.f deleted file mode 100644 index c188f0811..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_info_unknown_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.toml deleted file mode 100644 index fd97b4bd0..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock access info ID has no reservation" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown_scenario.svh deleted file mode 100644 index 74f805ad4..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown_scenario.svh +++ /dev/null @@ -1,4 +0,0 @@ -// verilog_syntax: parse-as-statements -@(negedge clock); -access_info.valid = 1; -`ILF(access_info.bits, ACCESS_INFO, ID) = 7; diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown_tb.sv deleted file mode 100644 index 833111221..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_info_unknown_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_info_unknown_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_info_unknown_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.f deleted file mode 100644 index e477f0ec8..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_maint_failed_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.toml deleted file mode 100644 index ae0c93089..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock maintenance failed" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed_scenario.svh deleted file mode 100644 index 54db9679d..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed_scenario.svh +++ /dev/null @@ -1,5 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 64, 1); -maintenance_accept(); -maintenance_ack(7, 0); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed_tb.sv deleted file mode 100644 index e01336b5f..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_failed_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_maint_failed_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_maint_failed_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.f deleted file mode 100644 index 6e82b0296..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_maint_tag_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.toml deleted file mode 100644 index 03417e409..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock maintenance response tag mismatch" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag_scenario.svh deleted file mode 100644 index 4c06f89cb..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag_scenario.svh +++ /dev/null @@ -1,5 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, 'h4000, 64, 1); -maintenance_accept(); -maintenance_ack(8, 1); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag_tb.sv deleted file mode 100644 index 34f465476..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_tag_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_maint_tag_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_maint_tag_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.f deleted file mode 100644 index 4969360ce..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_maint_unknown_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.toml deleted file mode 100644 index 46923a720..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock maintenance response has no request" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown_scenario.svh deleted file mode 100644 index 2c5a2779e..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown_scenario.svh +++ /dev/null @@ -1,2 +0,0 @@ -// verilog_syntax: parse-as-statements -maintenance_ack(7, 1); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown_tb.sv deleted file mode 100644 index 8d557bd0e..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_maint_unknown_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_maint_unknown_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_maint_unknown_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.f deleted file mode 100644 index f101005ad..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_no_memory_seal_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.toml deleted file mode 100644 index 5fee3763a..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock no-memory declaration must seal an empty command" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal_scenario.svh deleted file mode 100644 index 401845ab8..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal_scenario.svh +++ /dev/null @@ -1,3 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 0, 0, 0, 0, 0); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal_tb.sv deleted file mode 100644 index a16997398..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_no_memory_seal_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_no_memory_seal_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_no_memory_seal_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.cm_hier b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.cm_hier deleted file mode 100644 index 241373a00..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.cm_hier +++ /dev/null @@ -1,9 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.maintenance -+tree @TOP@.maintained -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.cancel diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.f b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.f deleted file mode 100644 index af0de0558..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Interlock/filelist.f -@RESOURCES@/interlock_bad_overflow_range_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.toml b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.toml deleted file mode 100644 index 5b613055b..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Interlock memory range exceeds physical width" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range_scenario.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range_scenario.svh deleted file mode 100644 index aa5305eed..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range_scenario.svh +++ /dev/null @@ -1,3 +0,0 @@ -// verilog_syntax: parse-as-statements -reserve(7); -info(7, 1, (64'd1 << 44) - 64, 65, 0); diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range_tb.sv deleted file mode 100644 index 41a5eaef3..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_bad_overflow_range_tb.sv +++ /dev/null @@ -1,3 +0,0 @@ -`define INTERLOCK_NEGATIVE_TB interlock_bad_overflow_range_tb -`define INTERLOCK_NEGATIVE_SCENARIO "interlock_bad_overflow_range_scenario.svh" -`include "negative_tb.svh" diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_tb.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_tb.sv deleted file mode 100644 index 1bc7f983f..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/interlock_tb.sv +++ /dev/null @@ -1,743 +0,0 @@ -module interlock_tb; - import uvm_pkg::*; - import interlock_pkg::*; - `include "uvm_macros.svh" - `include "interlock_config.svh" - `define ILF(B, K, F) B[`INTERLOCK_``K``_``F``_OFFSET +: `INTERLOCK_``K``_``F``_WIDTH] - logic clock = 0, reset = 1, cpu_allow; - always #5 clock = ~clock; - interlock_control_if ctl (clock); - stream_if #( - .WIDTH(`INTERLOCK_DISPATCH_WIDTH) - ) dispatch ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_CANCEL_WIDTH) - ) cancel ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_ACCESS_INFO_WIDTH) - ) access_info ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_MAINTENANCE_WIDTH) - ) maintenance ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_MAINTAINED_WIDTH) - ) maintained ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_GRANT_WIDTH) - ) grant ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_DONE_WIDTH) - ) done ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_COMPLETE_WIDTH) - ) complete ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_CPU_QUERY_WIDTH) - ) cpu_query ( - clock, - reset - ); - Interlock dut ( - .clock(clock), - .reset(reset), - `include "interlock_ports.svh" - ); - chandle model; - int grants[256], completions[256], maintenance_ops[256][3]; - int checked_queries = 0, reserved = 0, acknowledged = 0, released = 0; - bit model_ready = 0; - int cache_pending_refills = 0; - function automatic void check(bit value, string message); - if (!value) `uvm_fatal("INTERLOCK_CONTRACT", message) - endfunction - always @(posedge clock) begin - if (model_ready) begin - if (reset) begin - interlock_ref_reset(model); - foreach (grants[i]) begin - grants[i] = 0; - completions[i] = 0; - for (int j = 0; j < 3; j++) maintenance_ops[i][j] = 0; - end - end else begin - if (`ILF(cpu_query.bits, CPU_QUERY, VALID)) begin - check(cpu_allow === interlock_ref_cpu_allow( - model, - `ILF(cpu_query.bits, CPU_QUERY, PADDR), - 1 << - `ILF(cpu_query.bits, CPU_QUERY, SIZELOG2), - `ILF(cpu_query.bits, CPU_QUERY, WRITE), - `ILF(cpu_query.bits, CPU_QUERY, OLDERDISPATCHPENDING), - dispatch.valid && dispatch.ready - ), "CPU permit disagrees with transaction oracle"); - checked_queries++; - end - if (dispatch.valid && dispatch.ready) begin - int id; - id = `ILF(dispatch.bits, DISPATCH, ID); - interlock_ref_reserve(model, id); - reserved++; - grants[id] = 0; - completions[id] = 0; - for (int j = 0; j < 3; j++) maintenance_ops[id][j] = 0; - end - if (cancel.valid && cancel.ready) - interlock_ref_cancel(model, `ILF(cancel.bits, CANCEL, ID)); - if (access_info.valid && access_info.ready) - interlock_ref_info(model, `ILF(access_info.bits, ACCESS_INFO, ID), - `ILF(access_info.bits, ACCESS_INFO, HASMEMORY), - `ILF(access_info.bits, ACCESS_INFO, BASE), - `ILF(access_info.bits, ACCESS_INFO, BYTES), - `ILF(access_info.bits, ACCESS_INFO, WRITE), - `ILF(access_info.bits, ACCESS_INFO, LAST)); - if (maintenance.valid && maintenance.ready) begin - interlock_ref_maint_offer(model, `ILF(maintenance.bits, MAINTENANCE, TAG), - `ILF(maintenance.bits, MAINTENANCE, OP), - `ILF(maintenance.bits, MAINTENANCE, FIRSTLINE), - `ILF(maintenance.bits, MAINTENANCE, LASTLINE)); - maintenance_ops[ - `ILF(maintenance.bits, MAINTENANCE, TAG) - ][ - `ILF(maintenance.bits, MAINTENANCE, OP) - ]++; - end - if (maintained.valid && maintained.ready) begin - check(`ILF(maintained.bits, MAINTAINED, OK), "normal cache BFM failed ACK"); - interlock_ref_maint_ack(model, `ILF(maintained.bits, MAINTAINED, TAG)); - end - if (grant.valid && grant.ready) begin - interlock_ref_grant(model, `ILF(grant.bits, GRANT, TAG)); - grants[`ILF(grant.bits, GRANT, TAG)]++; - end - if (done.valid && done.ready) begin - check(`ILF(done.bits, DONE, OK), "normal DMA BFM failed ACK"); - interlock_ref_dma_ack(model, `ILF(done.bits, DONE, TAG)); - acknowledged++; - end - if (complete.valid && complete.ready) begin - interlock_ref_complete(model, `ILF(complete.bits, COMPLETE, TAG)); - completions[`ILF(complete.bits, COMPLETE, TAG)]++; - released++; - end - end - end - end - task automatic query(longint unsigned addr, bit write, bit permit, int cycles = 3, bit older = 0, - int size_log2 = 3); - @(negedge clock); - cpu_query.bits = '0; - `ILF(cpu_query.bits, CPU_QUERY, VALID) = 1; - `ILF(cpu_query.bits, CPU_QUERY, PADDR) = addr; - `ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = size_log2; - `ILF(cpu_query.bits, CPU_QUERY, WRITE) = write; - `ILF(cpu_query.bits, CPU_QUERY, OLDERDISPATCHPENDING) = older; - #1; - check(cpu_allow === permit, "directed CPU query incorrect or exceeded combinational timing"); - repeat (cycles) begin - @(posedge clock); - #1; - check(cpu_allow === permit, "CPU permit changed during directed query"); - end - @(negedge clock); - cpu_query.bits = '0; - endtask - task automatic reserve(int id); - @(negedge clock); - dispatch.bits = '0; - `ILF(dispatch.bits, DISPATCH, ID) = id; - dispatch.valid = 1; - do @(posedge clock); while (!dispatch.ready); - @(negedge clock); - dispatch.valid = 0; - endtask - task automatic declare_access(int id, bit memory, longint unsigned base, bytes, bit write, - bit last = 1); - @(negedge clock); - access_info.bits = '0; - `ILF(access_info.bits, ACCESS_INFO, ID) = id; - `ILF(access_info.bits, ACCESS_INFO, HASMEMORY) = memory; - `ILF(access_info.bits, ACCESS_INFO, BASE) = base; - `ILF(access_info.bits, ACCESS_INFO, BYTES) = bytes; - `ILF(access_info.bits, ACCESS_INFO, WRITE) = write; - `ILF(access_info.bits, ACCESS_INFO, LAST) = last; - access_info.valid = 1; - do @(posedge clock); while (!access_info.ready); - @(negedge clock); - access_info.valid = 0; - endtask - // External cache BFM: maintenance ACK is delayed until old request/refill work drains. - task automatic maintain(int expected_tag = -1, int expected_op = -1, int old_refill_delay = 7); - int tag, op; - logic [`INTERLOCK_MAINTENANCE_WIDTH-1:0] packet; - wait (maintenance.valid); - packet = maintenance.bits; - tag = `ILF(packet, MAINTENANCE, TAG); - op = `ILF(packet, MAINTENANCE, OP); - if (expected_tag >= 0) check(tag == expected_tag, "unexpected maintenance owner"); - if (expected_op >= 0) check(op == expected_op, "unexpected maintenance operation"); - @(negedge clock); - cpu_query.bits = '0; - `ILF(cpu_query.bits, CPU_QUERY, VALID) = 1; - `ILF(cpu_query.bits, CPU_QUERY, PADDR) = `ILF(packet, MAINTENANCE, FIRSTLINE); - `ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = 3; - `ILF(cpu_query.bits, CPU_QUERY, WRITE) = (op == 0); - repeat (5) begin - @(posedge clock); - #1; - check(maintenance.valid && maintenance.bits === packet, - "maintenance offer changed under backpressure"); - end - @(negedge clock); - maintenance.ready = 1; - @(posedge clock); - check(maintenance.valid, "maintenance request disappeared"); - @(negedge clock); - maintenance.ready = 0; - cache_pending_refills = 1; - // Independent external refill work completes later; ACK waits for that event. - fork - begin - repeat (old_refill_delay) @(posedge clock); - #1; - cache_pending_refills = 0; - end - join_none - wait (cache_pending_refills == 0); - @(negedge clock); - check(!maintained.valid, "cache ACK preceded old refill drain"); - maintained.bits = '0; - `ILF(maintained.bits, MAINTAINED, TAG) = tag; - `ILF(maintained.bits, MAINTAINED, OK) = 1; - maintained.valid = 1; - do @(posedge clock); while (!maintained.ready); - @(negedge clock); - maintained.valid = 0; - maintained.bits = '0; - cpu_query.bits = '0; - endtask - task automatic finish_dma(int id); - @(negedge clock); - done.bits = '0; - `ILF(done.bits, DONE, TAG) = id; - `ILF(done.bits, DONE, OK) = 1; - done.valid = 1; - do @(posedge clock); while (!done.ready); - @(negedge clock); - done.valid = 0; - done.bits = '0; - endtask - task automatic await_grant(int id); - wait (grants[id] == 1); - @(negedge clock); - endtask - task automatic await_complete(int id); - wait (completions[id] == 1); - @(negedge clock); - endtask - task automatic joint_reset(); - @(negedge clock); - reset = 1; - dispatch.valid = 0; - access_info.valid = 0; - maintained.valid = 0; - done.valid = 0; - maintenance.ready = 0; - grant.ready = 0; - complete.ready = 0; - cpu_query.bits = '0; - repeat (4) @(posedge clock); - @(negedge clock); - reset = 0; - endtask - initial begin - uvm_config_db#(virtual interlock_control_if)::set(null, "*", "vif", ctl); - run_test("protocol_test"); - end - initial begin - dispatch.valid = 0; - dispatch.bits = '0; - cancel.valid = 0; - cancel.bits = '0; - access_info.valid = 0; - access_info.bits = '0; - maintenance.ready = 0; - maintained.valid = 0; - maintained.bits = '0; - grant.ready = 0; - done.valid = 0; - done.bits = '0; - complete.ready = 0; - cpu_query.valid = 0; - cpu_query.ready = 0; - cpu_query.bits = '0; - wait (ctl.start); - check( - `INTERLOCK_ENTRIES==4&&`INTERLOCK_ADDRESS_BITS==44&&`INTERLOCK_ID_BITS==8&&`INTERLOCK_LINE_BYTES==64, - "unexpected declared verification profile"); - model = interlock_ref_create(`INTERLOCK_ENTRIES, `INTERLOCK_ADDRESS_BITS, `INTERLOCK_LINE_BYTES, - `INTERLOCK_MAX_RANGES); - model_ready = 1; - joint_reset(); - query('h4000, 0, 0, 4, 1); - query('h4000, 0, 1); - query(0, 1, 1); - // Same-cycle dispatch must deny even a read/read CPU query until access info is known. - @(negedge clock); - dispatch.bits = '0; - `ILF(dispatch.bits, DISPATCH, ID) = 255; - dispatch.valid = 1; - cpu_query.bits = '0; - `ILF(cpu_query.bits, CPU_QUERY, VALID) = 1; - `ILF(cpu_query.bits, CPU_QUERY, PADDR) = 'h4000; - `ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = 3; - #1; - check(!cpu_allow, "same-cycle dispatch was invisible to CPU"); - @(posedge clock); - check(dispatch.ready, "first reservation blocked"); - @(negedge clock); - dispatch.valid = 0; - cpu_query.bits = '0; - query('h9000, 0, 0, 6); - declare_access(255, 0, 0, 0, 0); - wait (complete.valid); - repeat (8) begin - @(posedge clock); - #1; - check(complete.valid && `ILF(complete.bits, COMPLETE, TAG) == 255, - "non-memory completion unstable"); - end - @(negedge clock); - complete.ready = 1; - await_complete(255); - check(grants[255] == 0, "non-memory registration was sent to DMA"); - query('h9000, 1, 1); - // Fill all register entries, show R/R parallel grants and reverse-order DMA retirement. - @(negedge clock); - complete.ready = 0; - for (int id = 1; id <= 4; id++) begin - reserve(id); - declare_access(id, 1, 'h4000, 128, 0); - end - query('h4000, 0, 1); - query('h4000, 1, 0); - query('h4080, 1, 1); - @(negedge clock); - dispatch.valid = 1; - `ILF(dispatch.bits, DISPATCH, ID) = 5; - repeat (8) begin - @(posedge clock); - #1; - check(!dispatch.ready, "full table accepted a fifth reservation"); - end - // Keep the fifth offer valid until a real slot is released and it handshakes. - // Grant offers must remain stable while the external DMA consumer stalls. - maintain(); - wait (grant.valid); - repeat (8) begin - @(posedge clock); - #1; - check(grant.valid, "grant vanished under backpressure"); - end - @(negedge clock); - grant.ready = 1; - for (int i = 0; i < 3; i++) maintain(); - for (int id = 1; id <= 4; id++) await_grant(id); - @(negedge clock); - complete.ready = 1; - finish_dma(4); - await_complete(4); - do @(posedge clock); while (!dispatch.ready); - @(negedge clock); - dispatch.valid = 0; - declare_access(5, 0, 0, 0, 0); - await_complete(5); - for (int id = 3; id >= 1; id--) begin - finish_dma(id); - await_complete(id); - end - query('h4000, 1, 1); - check(interlock_ref_live(model) == 0, "R/R retirement leaked reservations"); - // RAW: CPU read and dependent NPU read must wait for write ACK AND post-invalidate. - reserve(20); - declare_access(20, 1, 'h6003, 61, 1); - reserve(21); - declare_access(21, 1, 'h6000, 64, 0); - query('h6000, 0, 0); - query('h6040, 1, 1); - maintain(20, 1, 15); - await_grant(20); - @(negedge clock); - done.bits = '0; - `ILF(done.bits, DONE, TAG) = 20; - `ILF(done.bits, DONE, OK) = 1; - query('h6000, 0, 0, 32); - check(grants[21] == 0, "dependent RAW grant issued before final DMA ACK"); - finish_dma(20); - query('h6000, 0, 0, 8); - check(completions[20] == 0 && grants[21] == 0, "write retired before post-invalidate"); - @(negedge clock); - complete.ready = 0; - maintain(20, 2, 13); - wait (complete.valid); - query('h6000, 0, 0, 8); - check(grants[21] == 0, "RAW grant overtook stalled predecessor completion"); - @(negedge clock); - complete.ready = 1; - await_complete(20); - maintain(21, 0); - await_grant(21); - query('h6000, 0, 1); - query('h6000, 1, 0); - finish_dma(21); - await_complete(21); - query('h6000, 1, 1); - // WAR then WAW enforce age while independent ranges retain forward progress. - reserve(30); - declare_access(30, 1, 'h7000, 64, 0); - reserve(31); - declare_access(31, 1, 'h7000, 64, 1); - reserve(32); - declare_access(32, 1, 'h7000, 64, 1); - reserve(33); - declare_access(33, 1, 'h8000, 64, 0); - maintain(); - maintain(); - await_grant(30); - await_grant(33); - check(grants[31] == 0 && grants[32] == 0, "WAR/WAW bypassed older operation"); - query('h9000, 1, 1); - finish_dma(33); - await_complete(33); - finish_dma(30); - await_complete(30); - maintain(31, 1); - await_grant(31); - check(grants[32] == 0, "WAW bypassed older writer"); - finish_dma(31); - maintain(31, 2); - await_complete(31); - maintain(32, 1); - await_grant(32); - finish_dma(32); - maintain(32, 2); - await_complete(32); - // Unknown older reservation blocks a younger known nonoverlapping memory access. - reserve(40); - reserve(41); - declare_access(41, 1, 'hA000, 64, 0); - query('hB000, 0, 0, 6); - check(!maintenance.valid && grants[41] == 0, "younger access overtook unknown reservation"); - declare_access(40, 0, 0, 0, 0); - await_complete(40); - maintain(41, 0); - await_grant(41); - finish_dma(41); - await_complete(41); - // Reuse a fully retired ID at the highest legal PA line. - reserve(255); - declare_access(255, 1, (64'd1 << 44) - 64, 64, 1); - query((64'd1 << 44) - 8, 0, 0); - query((64'd1 << 44) - 72, 1, 1); - maintain(255, 1); - await_grant(255); - finish_dma(255); - maintain(255, 2); - await_complete(255); - query((64'd1 << 44) - 8, 0, 1); - // Joint reset cancels an unknown reservation and a granted DMA operation externally. - reserve(50); - declare_access(50, 1, 'hB000, 64, 0); - maintain(50, 0); - await_grant(50); - reserve(51); - query('hC000, 0, 0); - joint_reset(); - query('hB000, 1, 1); - check(interlock_ref_live(model) == 0, "joint reset did not cancel outstanding operations"); - @(negedge clock); - grant.ready = 1; - complete.ready = 1; - reserve(50); - declare_access(50, 1, 'hB000, 64, 0); - maintain(50, 0); - await_grant(50); - finish_dma(50); - await_complete(50); - // Reachable report holes: same-cycle dispatch+info holds info until its later handshake. - @(negedge clock); - dispatch.bits = '0; - access_info.bits = '0; - `ILF(dispatch.bits, DISPATCH, ID) = 254; - dispatch.valid = 1; - `ILF(access_info.bits, ACCESS_INFO, ID) = 254; - `ILF(access_info.bits, ACCESS_INFO, LAST) = 1; - access_info.valid = 1; - cpu_query.bits = '0; - `ILF(cpu_query.bits, CPU_QUERY, VALID) = 1; - `ILF(cpu_query.bits, CPU_QUERY, PADDR) = 'h4000; - `ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = 3; - #1; - check(!cpu_allow, "same-cycle info bypassed reservation hazard"); - @(posedge clock); - check(dispatch.ready && !access_info.ready, "new info must await its reservation handshake"); - @(negedge clock); - dispatch.valid = 0; - do @(posedge clock); while (!access_info.ready); - @(negedge clock); - access_info.valid = 0; - cpu_query.bits = '0; - await_complete(254); - // Each physical register slot sees complementary full-width IDs and high PA tags. - for (int slot = 0; slot < 4; slot++) begin - reserve(8'hf0 ^ slot); - declare_access(8'hf0 ^ slot, 1, (64'd1 << 44) - 64, 64, 0); - end - for (int slot = 0; slot < 4; slot++) maintain(); - for (int slot = 0; slot < 4; slot++) await_grant(8'hf0 ^ slot); - for (int slot = 3; slot >= 0; slot--) begin - finish_dma(8'hf0 ^ slot); - await_complete(8'hf0 ^ slot); - end - for (int slot = 0; slot < 4; slot++) begin - reserve(8'h0f ^ slot); - declare_access(8'h0f ^ slot, 1, 0, 64'd1 << 44, 1); - end - for (int slot = 0; slot < 4; slot++) begin - maintain(8'h0f ^ slot, 1); - await_grant(8'h0f ^ slot); - finish_dma(8'h0f ^ slot); - maintain(8'h0f ^ slot, 2); - await_complete(8'h0f ^ slot); - end - // Extent input high bits are legal: no artificial small-range fixture restriction. - for (int bit_index = 0; bit_index < 44; bit_index++) begin - reserve(170); - declare_access(170, 1, 0, 64'd1 << bit_index, 0); - maintain(170, 0, 2); - await_grant(170); - finish_dma(170); - await_complete(170); - end - // Sub-line extents are rounded to the cache-maintenance line ownership unit. - reserve(171); - declare_access(171, 1, 'h3003f, 1, 0); - query('h30000, 1, 0); - query('h30040, 1, 1); - maintain(171, 0); - await_grant(171); - finish_dma(171); - await_complete(171); - // CPU byte/half/word accesses are legal and naturally aligned, up to one line. - for (int size_log2 = 0; size_log2 <= 3; size_log2++) - for (int offset = 0; offset < 8; offset += (1 << size_log2)) - query('h20000 + offset, 1, 1, 1, 0, size_log2); - // Recycle low slots while higher older slots remain live; dependencies must not alias a new occupant. - for (int slot = 0; slot < 4; slot++) begin - reserve(100 + slot); - declare_access(100 + slot, 1, 'hD000, 64, 0); - end - for (int slot = 0; slot < 4; slot++) maintain(); - for (int slot = 0; slot < 4; slot++) await_grant(100 + slot); - for (int slot = 0; slot < 4; slot++) begin - finish_dma(100 + slot); - await_complete(100 + slot); - reserve(200 + slot); - declare_access(200 + slot, 1, 'hD000, 64, 1); - end - for (int slot = 0; slot < 4; slot++) begin - maintain(200 + slot, 1); - await_grant(200 + slot); - finish_dma(200 + slot); - maintain(200 + slot, 2); - await_complete(200 + slot); - end - // Joint synchronous reset cancels legal in-flight producer offers at its first edge. - reserve(60); - declare_access(60, 1, 'hF000, 64, 0); - maintain(60, 0); - await_grant(60); - reserve(61); - declare_access(61, 1, 'h10000, 64, 1); - wait (maintenance.valid); - @(negedge clock); - maintenance.ready = 1; - @(posedge clock); - @(negedge clock); - maintenance.ready = 0; - reserve(62); - @(negedge clock); - reset = 1; - dispatch.bits = '0; - `ILF(dispatch.bits, DISPATCH, ID) = 63; - dispatch.valid = 1; - access_info.bits = '0; - `ILF(access_info.bits, ACCESS_INFO, ID) = 62; - `ILF(access_info.bits, ACCESS_INFO, LAST) = 1; - access_info.valid = 1; - maintained.bits = '0; - `ILF(maintained.bits, MAINTAINED, TAG) = 61; - `ILF(maintained.bits, MAINTAINED, OK) = 1; - maintained.valid = 1; - done.bits = '0; - `ILF(done.bits, DONE, TAG) = 60; - `ILF(done.bits, DONE, OK) = 1; - done.valid = 1; - cpu_query.bits = '0; - `ILF(cpu_query.bits, CPU_QUERY, VALID) = 1; - `ILF(cpu_query.bits, CPU_QUERY, PADDR) = 'hF000; - `ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = 3; - @(posedge clock); - @(negedge clock); - dispatch.valid = 0; - access_info.valid = 0; - maintained.valid = 0; - done.valid = 0; - cpu_query.bits = '0; - repeat (3) @(posedge clock); - @(negedge clock); - reset = 0; - query(0, 1, 1); - check(interlock_ref_live(model) == 0, "joint reset left producer-offer reservations live"); - // Fast external endpoints may offer ACK in the same cycle as request/grant acceptance. - @(negedge clock); - grant.ready = 0; - reserve(0); - declare_access(0, 1, 'hE000, 64, 0); - wait (maintenance.valid); - @(negedge clock); - maintenance.ready = 1; - maintained.bits = '0; - `ILF(maintained.bits, MAINTAINED, TAG) = 0; - `ILF(maintained.bits, MAINTAINED, OK) = 1; - maintained.valid = 1; - @(posedge clock); - check(maintenance.valid && !maintained.ready, - "fast cache response did not wait for accepted request"); - @(negedge clock); - maintenance.ready = 0; - do @(posedge clock); while (!maintained.ready); - @(negedge clock); - maintained.valid = 0; - maintained.bits = '0; - wait (grant.valid); - @(negedge clock); - grant.ready = 1; - done.bits = '0; - `ILF(done.bits, DONE, TAG) = 0; - `ILF(done.bits, DONE, OK) = 1; - done.valid = 1; - @(posedge clock); - check(grant.valid && !done.ready, "early DMA completion did not wait for accepted grant"); - @(negedge clock); - grant.ready = 0; - do @(posedge clock); while (!done.ready); - @(negedge clock); - done.valid = 0; - done.bits = '0; - grant.ready = 1; - await_complete(0); - // An unsealed eight-extent list blocks all CPU queries; sealing exposes exact holes. - reserve(80); - for (int extent = 0; extent < 8; extent++) begin - declare_access(80, 1, 'h200000 + extent * 128, 64, 1, extent == 7); - if (extent != 7) begin - query('h300000, 0, 0, 1); - check(!maintenance.valid && grants[80] == 0, - "partial unsealed list reached maintenance/grant"); - end - end - query('h200040, 1, 1); - query('h200000, 0, 0); - for (int extent = 0; extent < 8; extent++) begin - maintain(80, 1, 2); - check(grants[80] == 0 || extent == 7, "grant before every pre-maintenance ACK"); - end - await_grant(80); - finish_dma(80); - for (int extent = 0; extent < 8; extent++) begin - maintain(80, 2, 2); - if (extent != 7) check(completions[80] == 0, "complete before every post-invalidate ACK"); - end - await_complete(80); - query('h200000, 1, 1); - check(maintenance_ops[80][1] == 8 && maintenance_ops[80][2] == 8, - "eight extents were not all maintained"); - // Preflight failure cancels unknown/partial reservations without producing a fake completion. - reserve(81); - query('h300000, 0, 0, 1); - @(negedge clock); - cancel.bits = '0; - `ILF(cancel.bits, CANCEL, ID) = 81; - cancel.valid = 1; - do @(posedge clock); while (!cancel.ready); - @(negedge clock); - cancel.valid = 0; - query('h300000, 0, 1, 1); - check(completions[81] == 0 && grants[81] == 0, "empty cancel became grant/complete"); - reserve(82); - declare_access(82, 1, 'h400000, 64, 0, 0); - query('h500000, 0, 0, 1); - @(negedge clock); - cancel.bits = '0; - `ILF(cancel.bits, CANCEL, ID) = 82; - cancel.valid = 1; - do @(posedge clock); while (!cancel.ready); - @(negedge clock); - cancel.valid = 0; - query('h500000, 0, 1, 1); - check(completions[82] == 0 && grants[82] == 0, "partial cancel became grant/complete"); - reserve(82); - declare_access(82, 0, 0, 0, 0); - await_complete(82); - // Inactive payloads are unconstrained by valid: exercise arithmetic carry without an illegal handshake. - @(negedge clock); - access_info.valid = 0; - access_info.bits = '1; - cpu_query.bits = '0; - `ILF(cpu_query.bits, CPU_QUERY, SIZELOG2) = 4; - repeat (2) begin - @(posedge clock); - #1; - check(cpu_allow, "inactive CPU query affected permit"); - end - @(negedge clock); - access_info.bits = '0; - cpu_query.bits = '0; - repeat (2) @(posedge clock); - check(interlock_ref_live(model) == 0, "normal test left unmatched transactions"); - `uvm_info("CHECKED", $sformatf( - "queries=%0d reserved=%0d dmaACK=%0d completions=%0d", - checked_queries, - reserved, - acknowledged, - released - ), UVM_LOW) - interlock_ref_destroy(model); - model_ready = 0; - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/negative_tb.svh b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/negative_tb.svh deleted file mode 100644 index b5a74c3c4..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/negative_tb.svh +++ /dev/null @@ -1,136 +0,0 @@ -module `INTERLOCK_NEGATIVE_TB; - import uvm_pkg::*; - import interlock_pkg::*; - `include "uvm_macros.svh" - `include "interlock_config.svh" - `define ILF(B, K, F) B[`INTERLOCK_``K``_``F``_OFFSET +: `INTERLOCK_``K``_``F``_WIDTH] - logic clock = 0, reset = 1, cpu_allow; - always #5 clock = ~clock; - interlock_control_if ctl (clock); - stream_if #( - .WIDTH(`INTERLOCK_DISPATCH_WIDTH) - ) dispatch ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_CANCEL_WIDTH) - ) cancel ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_ACCESS_INFO_WIDTH) - ) access_info ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_MAINTENANCE_WIDTH) - ) maintenance ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_MAINTAINED_WIDTH) - ) maintained ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_GRANT_WIDTH) - ) grant ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_DONE_WIDTH) - ) done ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_COMPLETE_WIDTH) - ) complete ( - clock, - reset - ); - stream_if #( - .WIDTH(`INTERLOCK_CPU_QUERY_WIDTH) - ) cpu_query ( - clock, - reset - ); - Interlock dut ( - .clock(clock), - .reset(reset), - `include "interlock_ports.svh" - ); - task automatic reserve(int id); - @(negedge clock); - dispatch.bits = '0; - `ILF(dispatch.bits, DISPATCH, ID) = id; - dispatch.valid = 1; - do @(posedge clock); while (!dispatch.ready); - @(negedge clock); - dispatch.valid = 0; - endtask - task automatic info(int id, bit memory, longint unsigned base, bytes, bit write, bit last = 1); - @(negedge clock); - access_info.bits = '0; - `ILF(access_info.bits, ACCESS_INFO, ID) = id; - `ILF(access_info.bits, ACCESS_INFO, HASMEMORY) = memory; - `ILF(access_info.bits, ACCESS_INFO, BASE) = base; - `ILF(access_info.bits, ACCESS_INFO, BYTES) = bytes; - `ILF(access_info.bits, ACCESS_INFO, WRITE) = write; - `ILF(access_info.bits, ACCESS_INFO, LAST) = last; - access_info.valid = 1; - do @(posedge clock); while (!access_info.ready); - @(negedge clock); - access_info.valid = 0; - endtask - task automatic maintenance_accept(); - wait (maintenance.valid); - @(negedge clock); - maintenance.ready = 1; - @(posedge clock); - @(negedge clock); - maintenance.ready = 0; - endtask - task automatic maintenance_ack(int id, bit ok); - @(negedge clock); - maintained.bits = '0; - `ILF(maintained.bits, MAINTAINED, TAG) = id; - `ILF(maintained.bits, MAINTAINED, OK) = ok; - maintained.valid = 1; - endtask - initial begin - uvm_config_db#(virtual interlock_control_if)::set(null, "*", "vif", ctl); - run_test("protocol_test"); - end - initial begin - dispatch.valid = 0; - dispatch.bits = '0; - cancel.valid = 0; - cancel.bits = '0; - access_info.valid = 0; - access_info.bits = '0; - maintenance.ready = 0; - maintained.valid = 0; - maintained.bits = '0; - grant.ready = 0; - done.valid = 0; - done.bits = '0; - complete.ready = 0; - cpu_query.bits = '0; - cpu_query.valid = 0; - cpu_query.ready = 0; - wait (ctl.start); - repeat (4) @(posedge clock); - @(negedge clock); - reset = 0; - `include `INTERLOCK_NEGATIVE_SCENARIO - repeat (20) @(posedge clock); - `uvm_fatal("MISSING_ASSERTION", "illegal input did not produce the declared RTL assertion") - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/package.sv deleted file mode 100644 index 57425508c..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/resources/package.sv +++ /dev/null @@ -1,84 +0,0 @@ -package interlock_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function chandle interlock_ref_create( - input int unsigned entries, - address_bits, - line_bytes, - max_ranges - ); - import "DPI-C" function void interlock_ref_destroy(input chandle model); - import "DPI-C" function void interlock_ref_reset(input chandle model); - import "DPI-C" function int unsigned interlock_ref_live(input chandle model); - import "DPI-C" function void interlock_ref_reserve( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void interlock_ref_info( - input chandle model, - input int unsigned id, - input byte unsigned has_mem, - input longint unsigned base, - bytes, - input byte unsigned write, - last - ); - import "DPI-C" function byte unsigned interlock_ref_cpu_allow( - input chandle model, - input longint unsigned addr, - input int unsigned bytes, - input byte unsigned write, - older_pending, - reserve_now - ); - import "DPI-C" function void interlock_ref_maint_offer( - input chandle model, - input int unsigned id, - op, - input longint unsigned first_line, - last_line - ); - import "DPI-C" function void interlock_ref_maint_ack( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void interlock_ref_grant( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void interlock_ref_dma_ack( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void interlock_ref_complete( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void interlock_ref_cancel( - input chandle model, - input int unsigned id - ); - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual interlock_control_if vif; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 500us; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual interlock_control_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "missing control interface") - endfunction - task execute(); - @(negedge vif.clock); - vif.start = 1; - wait (vif.done); - `uvm_info( - "INTERLOCK_PASS", - "Checked live reservations, CPU permits, maintenance/grant/ACK/completion handshakes with independent transaction oracle", - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Emit.scala deleted file mode 100644 index e617b117a..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Emit.scala +++ /dev/null @@ -1,60 +0,0 @@ -package memcore.memory.interlock - -import chisel3._ -import java.nio.file.{Files, Paths} - -object Emit extends App { - val p = Params() - Files.createDirectories(Paths.get("build")) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Interlock(p), - firtoolOpts = args, - args = Array("--target-dir", "build/Interlock", "--split-verilog") - ) - - val definitions = scala.collection.mutable.ArrayBuffer[String]( - s"`define INTERLOCK_ENTRIES ${p.entries}", - s"`define INTERLOCK_ADDRESS_BITS ${p.addressBits}", - s"`define INTERLOCK_ID_BITS ${p.idBits}", - s"`define INTERLOCK_LINE_BYTES ${p.lineBytes}", - s"`define INTERLOCK_MAX_RANGES ${p.maxRanges}" - ) - - val ports = scala.collection.mutable.ArrayBuffer[String]() - - def channel( - kind: String, - prefix: String, - signal: String, - data: Bundle, - decoupled: Boolean = true - ): Unit = { - var offset = 0 - for ((name, field) <- data.elements.toSeq) { - val width = field.getWidth - definitions += s"`define INTERLOCK_${kind}_${name.toUpperCase}_OFFSET $offset" - definitions += s"`define INTERLOCK_${kind}_${name.toUpperCase}_WIDTH $width" - ports += s".$prefix$name($signal.bits[$offset +: $width])" - offset += width - } - definitions += s"`define INTERLOCK_${kind}_WIDTH $offset" - if (decoupled) { - val channelPrefix = prefix.stripSuffix("bits_") - ports += s".${channelPrefix}valid($signal.valid)" - ports += s".${channelPrefix}ready($signal.ready)" - } - } - - channel("DISPATCH", "io_dispatch_bits_", "dispatch", new Dispatch(p)) - channel("CANCEL", "io_cancel_bits_", "cancel", new Dispatch(p)) - channel("ACCESS_INFO", "io_accessInfo_bits_", "access_info", new AccessInfo(p)) - channel("MAINTENANCE", "io_maintenance_bits_", "maintenance", new Maintenance(p)) - channel("MAINTAINED", "io_maintained_bits_", "maintained", new Acknowledgement(p)) - channel("GRANT", "io_grant_bits_", "grant", new Tag(p)) - channel("DONE", "io_done_bits_", "done", new Acknowledgement(p)) - channel("COMPLETE", "io_complete_bits_", "complete", new Tag(p)) - channel("CPU_QUERY", "io_cpuQuery_", "cpu_query", new CpuQuery(p), decoupled = false) - ports += ".io_cpuAllow(cpu_allow)" - Files.write(Paths.get("build/interlock_config.svh"), (definitions.mkString("\n") + "\n").getBytes) - Files.write(Paths.get("build/interlock_ports.svh"), ports.mkString(",\n").getBytes) -} diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Interface.scala b/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Interface.scala deleted file mode 100644 index c4aaa94d7..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Interface.scala +++ /dev/null @@ -1,38 +0,0 @@ -package memcore.memory.interlock - -import chisel3._ - -class Dispatch(p: Params) extends Bundle { val id = UInt(p.idBits.W) } - -class AccessInfo(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val hasMemory = Bool() - val base = UInt(p.addressBits.W) - val bytes = UInt((p.addressBits + 1).W) - val write = Bool() - val last = Bool() -} - -class CpuQuery(p: Params) extends Bundle { - val valid = Bool() - val paddr = UInt(p.addressBits.W) - val sizeLog2 = UInt(3.W) - val write = Bool() - val olderDispatchPending = Bool() -} - -object MaintenanceOp { val Clean = 0; val CleanInvalidate = 1; val Invalidate = 2 } - -class Maintenance(p: Params) extends Bundle { - val tag = UInt(p.idBits.W) - val firstLine = UInt(p.addressBits.W) - val lastLine = UInt(p.addressBits.W) - val op = UInt(2.W) -} - -class Tag(p: Params) extends Bundle { val tag = UInt(p.idBits.W) } - -class Acknowledgement(p: Params) extends Bundle { - val tag = UInt(p.idBits.W) - val ok = Bool() -} diff --git a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Params.scala b/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Params.scala deleted file mode 100644 index 8c2c69735..000000000 --- a/arch/src/main/scala/framework/mem-core/interlock/src/main/scala/Params.scala +++ /dev/null @@ -1,17 +0,0 @@ -package memcore.memory.interlock - -case class Params( - entries: Int = 4, - addressBits: Int = 44, - idBits: Int = 8, - lineBytes: Int = 64, - maxRanges: Int = 8) { - require(entries >= 2 && (entries & (entries - 1)) == 0) - require(addressBits >= 4 && idBits >= 1) - require(lineBytes >= 8 && (lineBytes & (lineBytes - 1)) == 0) - require(maxRanges >= 1 && (maxRanges & (maxRanges - 1)) == 0) - val rangeIndexBits: Int = math.max(1, Integer.numberOfTrailingZeros(maxRanges)) - val rangeCountBits: Int = Integer.SIZE - Integer.numberOfLeadingZeros(maxRanges) - val lineBits: Int = Integer.numberOfTrailingZeros(lineBytes) - require(addressBits > lineBits) -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/build.sc b/arch/src/main/scala/framework/mem-core/mesh_shm/build.sc deleted file mode 100644 index 9b0de30f9..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/build.sc +++ /dev/null @@ -1,28 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -object axis extends ChiselModule { - override def moduleRoot = os.pwd / os.up / "axis" -} - -object bank extends ChiselModule { - override def moduleRoot = os.pwd / os.up / "bank" - override def moduleDeps = Seq(axis) -} - -object mesh_shm extends ChiselModule { - override def moduleRoot = os.pwd - override def moduleDeps = Seq(bank) - override def mainClass = Some("memcore.memory.mesh_shm.Emit") - -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router.f b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router.f deleted file mode 100644 index 935fe9d7d..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/protocol/axis/interface.sv -+incdir+@RESOURCES@ -@RESOURCES@/router_package.sv --F @RTL@/MeshRouter/filelist.f -@RESOURCES@/router_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_package.sv b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_package.sv deleted file mode 100644 index 4a91ab97c..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/router_package.sv +++ /dev/null @@ -1,13 +0,0 @@ -package router_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - typedef virtual axis_if #(128, 8, 4, 45) router_vif; - import "DPI-C" function int unsigned mesh_route( - input int unsigned destination, - input int unsigned row, - input int unsigned col - ); - `include "router_item.svh" - `include "router_test.svh" -endpackage diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared.f b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared.f deleted file mode 100644 index 52096c242..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/resources/shared.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/protocol/axis/interface.sv -+incdir+@RESOURCES@ -@RESOURCES@/transfer_if.sv -@RESOURCES@/shared_package.sv --F @RTL@/MeshSharedMem/filelist.f -@RESOURCES@/shared_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/Emit.scala deleted file mode 100644 index 8102772ae..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/Emit.scala +++ /dev/null @@ -1,15 +0,0 @@ -package memcore.memory.mesh_shm - -object Emit extends App { - val p = MeshSharedMemParams.prototype - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new MeshRouter(p, 1, 1), - firtoolOpts = args, - args = Array("--target-dir", "build/MeshRouter", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new MeshSharedMem(p), - firtoolOpts = args, - args = Array("--target-dir", "build/MeshSharedMem", "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshChannel.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshChannel.scala deleted file mode 100644 index 10b55ccbe..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshChannel.scala +++ /dev/null @@ -1,10 +0,0 @@ -package memcore.memory.mesh_shm - -import chisel3._ -import chisel3.util._ - -/** Core-facing shared-bank port. Non-conflicting requests may be outstanding together. */ -class MeshChannel(p: MeshSharedMemParams) extends Bundle { - val request = Flipped(Decoupled(new MeshClientRequest(p))) - val response = Decoupled(new MeshClientResponse(p)) -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshClientEndpoint.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshClientEndpoint.scala deleted file mode 100644 index 18279b39c..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshClientEndpoint.scala +++ /dev/null @@ -1,110 +0,0 @@ -package memcore.memory.mesh_shm - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -/** Converts one client operation class into packets injected at one Mesh node. */ -@instantiable -class MeshClientEndpoint( - p: MeshSharedMemParams, - row: Int, - col: Int, - channel: Int, - acceptedWrite: Option[Boolean]) - extends Module { - require(row >= 0 && row < p.rows && col >= 0 && col < p.cols) - require(channel >= 0 && channel < p.networkClientChannels) - - @public - val io = IO(new Bundle { - val client = new MeshChannel(p) - val inFlight = Output(UInt(log2Ceil(p.maxInFlight + 1).W)) - val tags = Output(UInt((1 << p.tagBits).W)) - - val mesh = new Bundle { - val request = Decoupled(new MeshPacket(p)) - val response = Flipped(Decoupled(new MeshPacket(p))) - } - - }) - - val inFlight = RegInit(0.U(log2Ceil(p.maxInFlight + 1).W)) - val tags = RegInit(0.U((1 << p.tagBits).W)) - val errorPending = RegInit(false.B) - val errorTag = Reg(UInt(p.tagBits.W)) - val errorWrite = Reg(Bool()) - val request = io.client.request - val response = io.client.response - val available = inFlight < p.maxInFlight.U && !tags(request.bits.tag) - val bankValid = request.bits.bank < p.visibleBankCount.U - val bankRows = VecInit((0 until p.bankCount).map(i => (i / p.cols).U(p.rowBits.W))) - val bankCols = VecInit((0 until p.bankCount).map(i => (i % p.cols).U(p.colBits.W))) - io.inFlight := inFlight - io.tags := tags - - acceptedWrite.foreach { write => - when(request.valid) { - assert(request.bits.write === write.B, "Mesh client operation used the wrong directional endpoint") - } - } - - io.mesh.request.valid := request.valid && available && !errorPending && bankValid - io.mesh.request.bits := 0.U.asTypeOf(new MeshPacket(p)) - io.mesh.request.bits.tdest := Cat(bankRows(request.bits.bank), bankCols(request.bits.bank)) - io.mesh.request.bits.tdata := request.bits.data - io.mesh.request.bits.tkeep := request.bits.mask - io.mesh.request.bits.tid := request.bits.tag - io.mesh.request.bits.tlast := true.B - io.mesh.request.bits.tuser := Cat( - row.U(p.rowBits.W), - col.U(p.colBits.W), - channel.U(p.channelBits.W), - request.bits.addr, - request.bits.write, - false.B - ) - request.ready := available && !errorPending && Mux(bankValid, io.mesh.request.ready, true.B) - when(request.fire && !bankValid) { - errorPending := true.B - errorTag := request.bits.tag - errorWrite := request.bits.write - } - - val responseValid = RegInit(false.B) - val responseBits = Reg(new MeshClientResponse(p)) - val responseNext = Wire(Decoupled(new MeshClientResponse(p))) - response.valid := responseValid - response.bits := responseBits - responseNext.ready := !responseValid || response.ready - when(responseNext.ready) { - responseValid := responseNext.valid - when(responseNext.valid)(responseBits := responseNext.bits) - } - responseNext.valid := errorPending || io.mesh.response.valid - responseNext.bits := 0.U.asTypeOf(responseNext.bits) - responseNext.bits.data := Mux(errorPending, 0.U, io.mesh.response.bits.data) - responseNext.bits.tag := Mux(errorPending, errorTag, io.mesh.response.bits.tag) - responseNext.bits.tkeep := Fill(p.maskBits, 1.U(1.W)) - responseNext.bits.tlast := true.B - responseNext.bits.tuser := Cat( - Mux(errorPending, errorWrite, io.mesh.response.bits.write), - errorPending || io.mesh.response.bits.error - ) - io.mesh.response.ready := responseNext.ready && !errorPending - - when(io.mesh.response.valid) { - assert(io.mesh.response.bits.channel === channel.U) - assert(inFlight =/= 0.U && tags(io.mesh.response.bits.tag)) - } - when(response.fire) { - assert(inFlight =/= 0.U && tags(response.bits.tag)) - } - when(responseNext.fire && errorPending)(errorPending := false.B) - val issued = Mux(request.fire, 1.U((1 << p.tagBits).W) << request.bits.tag, 0.U) - val completed = Mux(response.fire, 1.U((1 << p.tagBits).W) << response.bits.tag, 0.U) - tags := (tags | issued) & ~completed - when(request.fire =/= response.fire) { - inFlight := Mux(request.fire, inFlight + 1.U, inFlight - 1.U) - } -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshControlInterface.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshControlInterface.scala deleted file mode 100644 index 045f60001..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshControlInterface.scala +++ /dev/null @@ -1,53 +0,0 @@ -package memcore.memory.mesh_shm - -import chisel3._ -import chisel3.util._ - -/** Contiguous rows from one Core-local Bank to another Core-local Bank. */ -class MeshTransferCommand(p: MeshSharedMemParams) extends Bundle { - val sourceCore = UInt(8.W) - val sourceBank = UInt(p.localBankBits.W) - val sourceAddr = UInt(16.W) - val targetCore = UInt(8.W) - val targetBank = UInt(p.localBankBits.W) - val targetAddr = UInt(16.W) - val rows = UInt(17.W) - val tag = UInt(p.tagBits.W) -} - -class MeshTransferCompletion(p: MeshSharedMemParams) extends Bundle { - val tag = UInt(p.tagBits.W) - val error = Bool() -} - -class MeshLocalBankRequest( - addressBits: Int, - bankBits: Int, - dataBits: Int, - tagBits: Int) - extends Bundle { - val addr = UInt(addressBits.W) - val write = Bool() - val data = UInt(dataBits.W) - val mask = UInt((dataBits / 8).W) - val tag = UInt(tagBits.W) - val bank = UInt(bankBits.W) -} - -class MeshLocalBankResponse(dataBits: Int, tagBits: Int) extends Bundle { - val data = UInt(dataBits.W) - val tag = UInt(tagBits.W) - val error = Bool() -} - -/** SharedMem initiates both the source read and target write. */ -class MeshLocalBankPort( - addressBits: Int, - bankBits: Int, - dataBits: Int, - tagBits: Int) - extends Bundle { - - val request = Decoupled(new MeshLocalBankRequest(addressBits, bankBits, dataBits, tagBits)) - val response = Flipped(Decoupled(new MeshLocalBankResponse(dataBits, tagBits))) -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshCoreEndpoint.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshCoreEndpoint.scala deleted file mode 100644 index 523dff63f..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshCoreEndpoint.scala +++ /dev/null @@ -1,59 +0,0 @@ -package memcore.memory.mesh_shm - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -/** The private Bank port stays beside its Core; only packets cross the Mesh. */ -@instantiable -class MeshCoreEndpoint( - p: MeshSharedMemParams, - core: Int, - row: Int, - col: Int) - extends Module { - - @public - val io = IO(new Bundle { - val request = Flipped(Decoupled(new MeshPacket(p))) - val response = Decoupled(new MeshPacket(p)) - val bank = new MeshLocalBankPort(p.addressBits, p.localBankBits, p.dataBits, p.tagBits) - }) - - val pending = RegInit(false.B) - val request = Reg(new MeshPacket(p)) - io.bank.request.valid := io.request.valid && !pending - io.bank.request.bits.bank := io.request.bits.localBank - io.bank.request.bits.addr := io.request.bits.addr - io.bank.request.bits.write := io.request.bits.write - io.bank.request.bits.data := io.request.bits.data - io.bank.request.bits.mask := io.request.bits.mask - io.bank.request.bits.tag := io.request.bits.tag - io.request.ready := io.bank.request.ready && !pending - when(io.request.fire) { - assert(io.request.bits.privateRequest && io.request.bits.core === core.U) - assert(io.request.bits.destRow === row.U && io.request.bits.destCol === col.U) - request := io.request.bits - pending := true.B - } - io.response.valid := pending && io.bank.response.valid - io.response.bits := request - io.response.bits.tdest := Cat(request.sourceRow, request.sourceCol) - io.response.bits.tdata := Mux(request.write, 0.U, io.bank.response.bits.data) - io.response.bits.tuser := Cat( - request.core, - request.localBank, - true.B, - row.U(p.rowBits.W), - col.U(p.colBits.W), - request.channel, - request.addr, - request.write, - io.bank.response.bits.error - ) - io.bank.response.ready := pending && io.response.ready - when(io.bank.response.valid) { - assert(pending && io.bank.response.bits.tag === request.tag) - } - when(io.response.fire)(pending := false.B) -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMem.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMem.scala deleted file mode 100644 index b5148c73c..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMem.scala +++ /dev/null @@ -1,246 +0,0 @@ -package memcore.memory.mesh_shm - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} - -@instantiable -class MeshSharedMem(p: MeshSharedMemParams) extends Module { - - @public - val io = IO(new Bundle { - val channels = Vec(p.totalChannels, new MeshChannel(p)) - val external = Vec(p.externalChannels, new MeshChannel(p)) - val transferCommand = Flipped(Decoupled(new MeshTransferCommand(p))) - val transferCompletion = Decoupled(new MeshTransferCompletion(p)) - val localBanks = Vec(p.cores.size, new MeshLocalBankPort(p.addressBits, p.localBankBits, p.dataBits, p.tagBits)) - val bankWrites = Vec(p.bankCount, Decoupled(new MeshClientRequest(p))) - }) - - val transfer: Instance[MeshTransferController] = Instantiate(new MeshTransferController(p)) - transfer.io.command.valid := io.transferCommand.valid - transfer.io.command.bits := io.transferCommand.bits - io.transferCommand.ready := transfer.io.command.ready - io.transferCompletion.valid := transfer.io.completion.valid - io.transferCompletion.bits := transfer.io.completion.bits - transfer.io.completion.ready := io.transferCompletion.ready - - val ordering = Instantiate(new MeshChannelOrdering(p, p.totalChannels + p.externalChannels)) - for (channel <- 0 until p.totalChannels) { - ordering.io.client(channel).request.valid := io.channels(channel).request.valid - ordering.io.client(channel).request.bits := io.channels(channel).request.bits - io.channels(channel).request.ready := ordering.io.client(channel).request.ready - io.channels(channel).response <> ordering.io.client(channel).response - } - for (channel <- 0 until p.externalChannels) { - val index = p.totalChannels + channel - ordering.io.client(index).request.valid := io.external(channel).request.valid - ordering.io.client(index).request.bits := io.external(channel).request.bits - io.external(channel).request.ready := ordering.io.client(index).request.ready - io.external(channel).response <> ordering.io.client(index).response - } - - val directionalEndpoints = p.inputChannelLocations.zip(p.outputChannelLocations).zipWithIndex.flatMap { - case ((inputLocation, outputLocation), channel) => - val read = Instantiate(new MeshClientEndpoint( - p, - inputLocation._1, - inputLocation._2, - 2 * channel, - Some(false) - )) - val write = Instantiate(new MeshClientEndpoint( - p, - outputLocation._1, - outputLocation._2, - 2 * channel + 1, - Some(true) - )) - val clientRequest = ordering.io.ordered(channel).request - val available = (read.io.inFlight +& write.io.inFlight) < p.maxInFlight.U && - !read.io.tags(clientRequest.bits.tag) && !write.io.tags(clientRequest.bits.tag) - read.io.client.request.valid := clientRequest.valid && available && !clientRequest.bits.write - read.io.client.request.bits := clientRequest.bits - write.io.client.request.valid := clientRequest.valid && available && clientRequest.bits.write - write.io.client.request.bits := clientRequest.bits - clientRequest.ready := available && Mux( - clientRequest.bits.write, - write.io.client.request.ready, - read.io.client.request.ready - ) - - val responses = Module(new RRArbiter(new MeshClientResponse(p), 2)) - responses.io.in(0) <> read.io.client.response - responses.io.in(1) <> write.io.client.response - ordering.io.ordered(channel).response <> responses.io.out - Seq( - (inputLocation._1, inputLocation._2, 2 * channel, read), - (outputLocation._1, outputLocation._2, 2 * channel + 1, write) - ) - } - - val externalEndpoints = (0 until p.externalChannels).map { index => - val channel = 2 * p.totalChannels + index - val orderingChannel = p.totalChannels + index - val endpoint = Instantiate(new MeshClientEndpoint(p, 0, 0, channel, None)) - endpoint.io.client <> ordering.io.ordered(orderingChannel) - (0, 0, channel, endpoint) - } - - val clientEndpoints = directionalEndpoints ++ externalEndpoints - - for (core <- p.cores.indices if p.cores(core).localBankNode.isEmpty) { - io.localBanks(core).request.valid := false.B - io.localBanks(core).request.bits := 0.U.asTypeOf(io.localBanks(core).request.bits) - io.localBanks(core).response.ready := false.B - } - - val requests: Seq[Seq[Instance[MeshCreditRouter]]] = - Seq.tabulate(p.rows, p.cols)((row, col) => Instantiate(new MeshCreditRouter(p, row, col))) - val responses: Seq[Seq[Instance[MeshCreditRouter]]] = - Seq.tabulate(p.rows, p.cols)((row, col) => Instantiate(new MeshCreditRouter(p, row, col))) - val banks: Seq[Seq[Instance[MeshBankNode]]] = - Seq.tabulate(p.rows, p.cols)((row, col) => Instantiate(new MeshBankNode(p, row, col))) - for { - row <- 0 until p.rows - col <- 0 until p.cols - } { - val requestRouter = requests(row)(col) - val responseRouter = responses(row)(col) - val bank = banks(row)(col) - val bankIndex = row * p.cols + col - val requestEgress = Instantiate(new MeshCreditToDecoupled(p)) - requestEgress.io.in <> requestRouter.io.out(MeshDirection.endpoint) - val localRequest = requestEgress.io.out - io.bankWrites( - bankIndex - ).valid := localRequest.valid && !localRequest.bits.privateRequest && localRequest.bits.write && bank.io.request.ready - io.bankWrites(bankIndex).bits.bank := bankIndex.U - io.bankWrites(bankIndex).bits.tuser := Cat(bank.io.request.bits.addr, bank.io.request.bits.write) - io.bankWrites(bankIndex).bits.tlast := true.B - io.bankWrites(bankIndex).bits.data := bank.io.request.bits.data - io.bankWrites(bankIndex).bits.mask := bank.io.request.bits.mask - io.bankWrites(bankIndex).bits.tag := bank.io.request.bits.tag - bank.io.request.valid := localRequest.valid && !localRequest.bits.privateRequest && - (!localRequest.bits.write || io.bankWrites(bankIndex).ready) - bank.io.request.bits := localRequest.bits - val privateCores = - p.cores.indices.filter(core => p.localBankLocations(core).contains((row, col))) - val privateReady = Wire(Vec(privateCores.size, Bool())) - val replies = Module(new RRArbiter(new MeshPacket(p), privateCores.size + 1) { - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(privateCores.size + 1)).W)) - when(io.out.fire)(pointer := io.chosen) - pointer - } - }) - replies.io.in(0) <> bank.io.response - for ((core, index) <- privateCores.zipWithIndex) { - val endpoint: Instance[MeshCoreEndpoint] = Instantiate(new MeshCoreEndpoint(p, core, row, col)) - endpoint.io.request.valid := localRequest.valid && localRequest.bits.privateRequest && localRequest.bits.core === core.U - endpoint.io.request.bits := localRequest.bits - privateReady(index) := localRequest.bits.core === core.U && endpoint.io.request.ready - io.localBanks(core) <> endpoint.io.bank - replies.io.in(index + 1) <> endpoint.io.response - } - localRequest.ready := Mux( - localRequest.bits.privateRequest, - (if (privateCores.nonEmpty) privateReady.asUInt.orR else false.B), - bank.io.request.ready && (!localRequest.bits.write || io.bankWrites(bankIndex).ready) - ) - when(localRequest.valid && localRequest.bits.privateRequest) { - assert( - privateCores.map(core => localRequest.bits.core === core.U).reduceOption(_ || _).getOrElse(false.B), - "private request reached a node without its Core" - ) - } - val responseIngress = Instantiate(new MeshDecoupledToCredit(p)) - responseIngress.io.in <> replies.io.out - responseRouter.io.in(MeshDirection.endpoint) <> responseIngress.io.out - - val attached = clientEndpoints.filter { case (r, c, _, _) => r == row && c == col } - val transferAttached = row == 0 && col == 0 - val attachmentCount = attached.size + (if (transferAttached) 1 else 0) - val requestIngress = Instantiate(new MeshDecoupledToCredit(p)) - requestRouter.io.in(MeshDirection.endpoint) <> requestIngress.io.out - val responseEgress = Instantiate(new MeshCreditToDecoupled(p)) - responseEgress.io.in <> responseRouter.io.out(MeshDirection.endpoint) - val network = responseEgress.io.out - if (attachmentCount > 0) { - val arbiter = Module(new RRArbiter(new MeshPacket(p), attachmentCount) { - override lazy val lastGrant = { - val pointer = RegInit(0.U(math.max(1, log2Ceil(attachmentCount)).W)) - when(io.out.fire)(pointer := io.chosen) - pointer - } - }) - val localResponseReady = Wire(Vec(attachmentCount, Bool())) - for (((_, _, channel, endpoint), port) <- attached.zipWithIndex) { - arbiter.io.in(port) <> endpoint.io.mesh.request - endpoint.io.mesh.response.valid := network.valid && network.bits.channel === channel.U - endpoint.io.mesh.response.bits := network.bits - localResponseReady(port) := network.bits.channel === channel.U && endpoint.io.mesh.response.ready - } - if (transferAttached) { - val port = attached.size - arbiter.io.in(port) <> transfer.io.mesh.request - transfer.io.mesh.response.valid := network.valid && network.bits.channel === p.transferChannel.U - transfer.io.mesh.response.bits := network.bits - localResponseReady(port) := network.bits.channel === p.transferChannel.U && transfer.io.mesh.response.ready - } - requestIngress.io.in <> arbiter.io.out - network.ready := localResponseReady.asUInt.orR - when(network.valid) { - assert( - (attached.map { case (_, _, channel, _) => - network.bits.channel === channel.U - } ++ - (if (transferAttached) Seq(network.bits.channel === p.transferChannel.U) else Seq.empty)).reduce(_ || _) - ) - } - } else { - requestIngress.io.in.valid := false.B - requestIngress.io.in.bits := 0.U.asTypeOf(new MeshPacket(p)) - network.ready := false.B - when(network.valid) { - assert(false.B, "response reached a node without an attached channel") - } - } - - if (col + 1 < p.cols) { - requests(row)(col + 1).io.in(MeshDirection.west) <> requestRouter.io.out(MeshDirection.east) - requestRouter.io.in(MeshDirection.east) <> requests(row)(col + 1).io.out(MeshDirection.west) - responses(row)(col + 1).io.in(MeshDirection.west) <> responseRouter.io.out(MeshDirection.east) - responseRouter.io.in(MeshDirection.east) <> responses(row)(col + 1).io.out(MeshDirection.west) - } - if (row + 1 < p.rows) { - requests(row + 1)(col).io.in(MeshDirection.north) <> requestRouter.io.out(MeshDirection.south) - requestRouter.io.in(MeshDirection.south) <> requests(row + 1)(col).io.out(MeshDirection.north) - responses(row + 1)(col).io.in(MeshDirection.north) <> responseRouter.io.out(MeshDirection.south) - responseRouter.io.in(MeshDirection.south) <> responses(row + 1)(col).io.out(MeshDirection.north) - } - - for (router <- Seq(requestRouter, responseRouter)) { - if (col == 0) { - router.io.in(MeshDirection.west).valid := false.B - router.io.in(MeshDirection.west).bits := 0.U.asTypeOf(new MeshPacket(p)) - router.io.out(MeshDirection.west).credit := false.B - } - if (col == p.cols - 1) { - router.io.in(MeshDirection.east).valid := false.B - router.io.in(MeshDirection.east).bits := 0.U.asTypeOf(new MeshPacket(p)) - router.io.out(MeshDirection.east).credit := false.B - } - if (row == 0) { - router.io.in(MeshDirection.north).valid := false.B - router.io.in(MeshDirection.north).bits := 0.U.asTypeOf(new MeshPacket(p)) - router.io.out(MeshDirection.north).credit := false.B - } - if (row == p.rows - 1) { - router.io.in(MeshDirection.south).valid := false.B - router.io.in(MeshDirection.south).bits := 0.U.asTypeOf(new MeshPacket(p)) - router.io.out(MeshDirection.south).credit := false.B - } - } - } -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMemInterface.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMemInterface.scala deleted file mode 100644 index 1d591f544..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMemInterface.scala +++ /dev/null @@ -1,37 +0,0 @@ -package memcore.memory.mesh_shm - -import chisel3._ - -class MeshClientRequest(p: MeshSharedMemParams) - extends memcore.bus.axi.Beat(p.dataBits, p.tagBits, p.bankBits, p.addressBits + 1) { - def bank = tdest - def addr = tuser(p.addressBits, 1) - def write = tuser(0) - def mask = tkeep - def tag = tid -} - -class MeshClientResponse(p: MeshSharedMemParams) extends memcore.bus.axi.Beat(p.dataBits, p.tagBits, 0, 2) { - def tag = tid - def error = tuser(0) - def write = tuser(1) -} - -/** One row transaction: AXI-S payload plus routed event metadata in TUSER. */ -class MeshPacket(p: MeshSharedMemParams) - extends memcore.bus.axi.Beat(p.dataBits, p.tagBits, p.rowBits + p.colBits, p.packetUserBits) { - def privateRequest = tuser(p.eventBits) - def localBank = tuser(p.eventBits + p.localBankBits, p.eventBits + 1) - def core = tuser(p.packetUserBits - 1, p.eventBits + p.localBankBits + 1) - def destRow = tdest(p.rowBits + p.colBits - 1, p.colBits) - def destCol = tdest(p.colBits - 1, 0) - def error = tuser(0) - def write = tuser(1) - def addr = tuser(p.addressBits + 1, 2) - def channel = tuser(p.addressBits + p.channelBits + 1, p.addressBits + 2) - def sourceCol = tuser(p.addressBits + p.channelBits + p.colBits + 1, p.addressBits + p.channelBits + 2) - def sourceRow = - tuser(p.addressBits + p.channelBits + p.colBits + p.rowBits + 1, p.addressBits + p.channelBits + p.colBits + 2) - def mask = tkeep - def tag = tid -} diff --git a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMemParams.scala b/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMemParams.scala deleted file mode 100644 index b9fc26d98..000000000 --- a/arch/src/main/scala/framework/mem-core/mesh_shm/src/main/scala/MeshSharedMemParams.scala +++ /dev/null @@ -1,151 +0,0 @@ -package memcore.memory.mesh_shm - -import chisel3.util.log2Ceil -import memcore.memory.bank.BankSetParams - -case class MeshCoreAttachment( - inputBankIds: Seq[Int], - outputBankIds: Seq[Int], - localBankNode: Option[Int]) { - require(inputBankIds.size == outputBankIds.size, "mesh core input/output channel counts must match") -} - -case class MeshSharedMemParams( - rows: Int, - cols: Int, - entriesPerBank: Int, - dataBits: Int, - tagBits: Int, - cores: Seq[MeshCoreAttachment], - maxInFlight: Int = 4, - linkBufferDepth: Int = 2, - localBankBits: Int = 10, - visibleBanks: Int = 0, - externalChannels: Int = 0) { - require(rows > 0 && cols > 0) - require(entriesPerBank <= 65536) - require(entriesPerBank >= 2 && (entriesPerBank & (entriesPerBank - 1)) == 0) - require(dataBits > 0 && dataBits % 8 == 0) - require(tagBits > 0 && localBankBits > 0 && cores.nonEmpty) - require(cores.forall(c => - (c.inputBankIds ++ c.outputBankIds ++ c.localBankNode).forall(id => id >= 0 && id < rows * cols) - )) - - require(externalChannels >= 0) - require(linkBufferDepth > 0) - require(maxInFlight > 0 && maxInFlight <= (1 << tagBits)) - val bankParams = BankSetParams(dataBits, 1, entriesPerBank, tagBits) - val bankCount: Int = rows * cols - val visibleBankCount: Int = if (visibleBanks == 0) bankCount else visibleBanks - require(visibleBankCount > 0 && visibleBankCount <= bankCount) - require(cores.exists(_.inputBankIds.nonEmpty), "mesh requires a data endpoint") - require(cores.forall(core => core.inputBankIds.nonEmpty == core.localBankNode.nonEmpty)) - val totalChannels: Int = cores.map(_.inputBankIds.size).sum - val inputChannelLocations: Seq[(Int, Int)] = - cores.flatMap(c => c.inputBankIds.map(id => (id / cols, id % cols))) - val outputChannelLocations: Seq[(Int, Int)] = - cores.flatMap(c => c.outputBankIds.map(id => (id / cols, id % cols))) - val rowBits: Int = math.max(1, log2Ceil(rows)) - val colBits: Int = math.max(1, log2Ceil(cols)) - val bankBits: Int = math.max(1, log2Ceil(bankCount)) - val addressBits: Int = 16 - val networkClientChannels: Int = 2 * totalChannels + externalChannels - val transferChannel: Int = networkClientChannels - val channelBits: Int = math.max(1, log2Ceil(networkClientChannels + 1)) - val coreBits: Int = math.max(1, log2Ceil(cores.size)) - val linkCreditBits: Int = math.max(1, log2Ceil(linkBufferDepth + 1)) - - val localBankLocations: Seq[Option[(Int, Int)]] = - cores.map(_.localBankNode.map(id => (id / cols, id % cols))) - - val eventBits: Int = rowBits + colBits + channelBits + addressBits + 2 - val packetUserBits: Int = eventBits + localBankBits + 9 - val maskBits: Int = dataBits / 8 - - // Core order in this sequence defines the flattened io.channels layout. - def channelsForCore(coreIndex: Int): Range = { - require(coreIndex >= 0 && coreIndex < cores.size) - val start = cores.take(coreIndex).map(_.inputBankIds.size).sum - start until (start + cores(coreIndex).inputBankIds.size) - } - -} - -object MeshSharedMemParams { - - def dimensions(bankCount: Int): (Int, Int) = { - require(bankCount > 1) - val rows = (math.sqrt(bankCount.toDouble).toInt to 1 by -1).find(bankCount % _ == 0).get - (rows, bankCount / rows) - } - - /** - * Assigns each channel lane to a directed chain of adjacent Mesh nodes. - * - * Core order defines the stages within each lane; a Core may own fewer lanes - * than its neighbors. A two-dimensional Mesh with an even dimension uses a - * Hamiltonian cycle, so the last stage and reused lanes remain adjacent. - */ - def directionalAttachments( - rows: Int, - cols: Int, - channelsPerCore: Seq[Int] - ): Seq[MeshCoreAttachment] = { - require(rows > 0 && cols > 0) - require(channelsPerCore.nonEmpty && channelsPerCore.forall(_ > 0)) - val bankCount = rows * cols - val channelCount = channelsPerCore.sum - val canWrap = bankCount == 2 || rows > 1 && cols > 1 && (rows % 2 == 0 || cols % 2 == 0) - require(bankCount > 1) - require( - canWrap || channelCount < bankCount, - "a Mesh without a Hamiltonian cycle cannot wrap directional channels" - ) - - def cycle(r: Int, c: Int): Seq[(Int, Int)] = { - require(c % 2 == 0 && r > 1) - val firstColumn = (0 until r).map(row => (row, 0)) - val inner = (1 until c).flatMap { col => - val rowOrder = if (col % 2 == 1) (r - 1 to 1 by -1) else (1 until r) - rowOrder.map(row => (row, col)) - } - val firstRow = (c - 1 to 1 by -1).map(col => (0, col)) - firstColumn ++ inner ++ firstRow - } - - val coordinates = - if (rows % 2 == 0 && cols > 1) cycle(cols, rows).map { case (row, col) => (col, row) } - else if (cols % 2 == 0 && rows > 1) cycle(rows, cols) - else (0 until rows).flatMap { row => - val columns = if (row % 2 == 0) (0 until cols) else (cols - 1 to 0 by -1) - columns.map(col => (row, col)) - } - val bankOrder = coordinates.map { case (row, col) => row * cols + col } - val channelOrder = (0 until channelsPerCore.max).flatMap { channel => - channelsPerCore.indices.collect { case core if channel < channelsPerCore(core) => (core, channel) } - } - val locations = channelOrder.zipWithIndex.map { case (owner, index) => - owner -> ( - bankOrder(index % bankOrder.size), - bankOrder((index + 1) % bankOrder.size) - ) - }.toMap - - channelsPerCore.indices.map { core => - val inputs = (0 until channelsPerCore(core)).map(channel => locations(core -> channel)._1) - val outputs = (0 until channelsPerCore(core)).map(channel => locations(core -> channel)._2) - MeshCoreAttachment(inputs, outputs, Some(inputs.head)) - } - } - - // Experimental 3x4 mesh: type 0/1/2 use 1/2/4 channels, with 2/2/1 cores. - val prototype: MeshSharedMemParams = MeshSharedMemParams( - rows = 3, - cols = 4, - entriesPerBank = 1024, - dataBits = 128, - tagBits = 8, - cores = directionalAttachments(3, 4, Seq(1, 1, 2, 2, 4)) - ) - -} diff --git a/arch/src/main/scala/framework/mem-core/mmu/build.sc b/arch/src/main/scala/framework/mem-core/mmu/build.sc deleted file mode 100644 index 7266ec37d..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/build.sc +++ /dev/null @@ -1,30 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object chi extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = os.pwd / os.up / "chi" -} - -object mmu extends ChiselModule { - override def mainClass = Some("memcore.memory.mmu.Emit") - override def moduleRoot = os.pwd - override def moduleDeps = Seq(chi) -} diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/csrc/src/translation.rs b/arch/src/main/scala/framework/mem-core/mmu/src/csrc/src/translation.rs deleted file mode 100644 index f878e8ae7..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/csrc/src/translation.rs +++ /dev/null @@ -1,209 +0,0 @@ -use std::collections::{HashMap, VecDeque}; -use std::ffi::c_void; - -#[derive(Clone, Copy)] -struct Pte { - value: u64, - error: bool, -} - -struct Model { - address_bits: u32, - memory: HashMap, - expected_reads: VecDeque, -} - -#[derive(Default)] -struct Translation { - address: u64, - page_fault: bool, - access_fault: bool, - level: u32, -} - -impl Translation { - fn fault(page: bool, level: u32) -> Self { - Self { - page_fault: page, - access_fault: !page, - level, - ..Self::default() - } - } -} - -struct Request { - address: u64, - mode: u32, - root: u64, - privilege: u32, - write: bool, - execute: bool, - sum: bool, - mxr: bool, -} - -impl Model { - // Sequential architectural reference. Memory has only explicitly programmed - // PTEs; Err(address) identifies an incomplete fixture, never an invented PTE. - fn walk(&self, r: &Request, reads: &mut Vec) -> Result { - let max_pa = (1u64 << self.address_bits) - 1; - if r.privilege == 3 || r.mode == 0 { - return Ok(if r.address > max_pa { - Translation::fault(false, 0) - } else { - Translation { address: r.address, ..Translation::default() } - }); - } - if r.address > 0x0000_003f_ffff_ffff && r.address < 0xffff_ffc0_0000_0000 { - return Ok(Translation::fault(true, 2)); - } - let mut ppn = r.root; - for level in (0..=2u32).rev() { - let vpn = (r.address >> (12 + 9 * level)) & 511; - let address = ppn * 4096 + vpn * 8; - if address > max_pa { - return Ok(Translation::fault(false, level)); - } - reads.push(address); - let pte = *self.memory.get(&address).ok_or(address)?; - if pte.error { - return Ok(Translation::fault(false, level)); - } - let value = pte.value; - let valid = value & 1 != 0; - let readable = value & 2 != 0; - let writable = value & 4 != 0; - let executable = value & 8 != 0; - if !valid || (writable && !readable) || value >> 54 != 0 { - return Ok(Translation::fault(true, level)); - } - ppn = (value >> 10) & ((1u64 << 44) - 1); - if readable || executable { - let user_page = value & 16 != 0; - let privilege_ok = if r.privilege == 0 { - user_page - } else { - !user_page || (r.sum && !r.execute) - }; - let operation_ok = if r.execute { - executable - } else if r.write { - writable - } else { - readable || (r.mxr && executable) - }; - let accessed = value & 64 != 0; - let dirty = value & 128 != 0; - let aligned = ppn & ((1u64 << (9 * level)) - 1) == 0; - if !privilege_ok || !operation_ok || !accessed || (r.write && !dirty) || !aligned { - return Ok(Translation::fault(true, level)); - } - let page_bytes = 4096u64 << (9 * level); - let address = ppn * 4096 + (r.address & (page_bytes - 1)); - return Ok(if address > max_pa { - Translation::fault(false, level) - } else { - Translation { address, level, ..Translation::default() } - }); - } - if level == 0 || value & 0xd0 != 0 { - return Ok(Translation::fault(true, level)); - } - } - unreachable!("the bottom-level non-leaf faults before leaving the loop") - } -} - -#[no_mangle] -pub extern "C" fn mmu_ref_create(address_bits: u32) -> *mut c_void { - assert!((44..=52).contains(&address_bits)); - Box::into_raw(Box::new(Model { - address_bits, - memory: HashMap::new(), - expected_reads: VecDeque::new(), - })).cast() -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_destroy(model: *mut c_void) { - drop(Box::from_raw(model.cast::())); -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_clear(model: *mut c_void) { - let model = &mut *model.cast::(); - assert!(model.expected_reads.is_empty(), "fixture changed with unmatched PTE reads"); - model.memory.clear(); -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_program(model: *mut c_void, address: u64, value: u64, error: u32) { - assert_eq!(address % 8, 0); - (*model.cast::()).memory.insert(address, Pte { value, error: error != 0 }); -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_read(model: *mut c_void, address: u64, value: *mut u64, error: *mut u32) -> u32 { - if let Some(pte) = (*model.cast::()).memory.get(&address) { - *value = pte.value; - *error = pte.error as u32; - 1 - } else { - 0 - } -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_pending(model: *mut c_void) -> u32 { - (*model.cast::()).expected_reads.len() as u32 -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_peek(model: *mut c_void, address: *mut u64) -> u32 { - if let Some(next) = (*model.cast::()).expected_reads.front() { - *address = *next; - 1 - } else { - 0 - } -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_consume(model: *mut c_void, address: u64) -> u32 { - let expected = &mut (*model.cast::()).expected_reads; - if expected.front() != Some(&address) { - return 0; - } - expected.pop_front(); - 1 -} - -#[no_mangle] -pub unsafe extern "C" fn mmu_ref_translate( - model: *mut c_void, address: u64, mode: u32, root: u64, privilege: u32, - write: u32, execute: u32, sum: u32, mxr: u32, - paddr: *mut u64, page_fault: *mut u32, access_fault: *mut u32, level: *mut u32, -) -> u32 { - assert!(mode == 0 || mode == 8); - assert!(matches!(privilege, 0 | 1 | 3)); - assert!(write == 0 || execute == 0); - assert!(root < (1u64 << 44)); - let model = &mut *model.cast::(); - let request = Request { address, mode, root, privilege, write: write != 0, execute: execute != 0, sum: sum != 0, mxr: mxr != 0 }; - let mut reads = Vec::new(); - match model.walk(&request, &mut reads) { - Ok(result) => { - model.expected_reads.extend(reads); - *paddr = result.address; - *page_fault = result.page_fault as u32; - *access_fault = result.access_fault as u32; - *level = result.level; - 0 - } - Err(address) => { - *paddr = address; - 1 - } - } -} diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker.f b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker.f deleted file mode 100644 index 35b035a74..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/walker_interface.sv -@RESOURCES@/walker_package.sv --F @RTL@/Walker/filelist.f -@RESOURCES@/walker_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access.f b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access.f deleted file mode 100644 index 258b768a9..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/walker_interface.sv -@RESOURCES@/walker_negative_package.sv --F @RTL@/Walker/filelist.f -@RESOURCES@/walker_bad_access_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access.toml b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access.toml deleted file mode 100644 index 9a665fd6f..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Walker access cannot both write and execute" diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access_tb.sv b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access_tb.sv deleted file mode 100644 index 77977e2a4..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_access_tb.sv +++ /dev/null @@ -1,44 +0,0 @@ -`include "walker_config.svh" -module walker_bad_access_tb; - import uvm_pkg::*; - import walker_negative_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - walker_control_if control (clock); - stream_if #(`WALK_REQ_WIDTH) source_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESP_WIDTH) sink_if ( - clock, - control.reset - ); - stream_if #(`WALK_ACCESS_WIDTH) access_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESULT_WIDTH) result_if ( - clock, - control.reset - ); - Walker dut ( - .clock(clock), - .reset(control.reset), - .io_config_mode(control.mode), - .io_config_rootPpn(control.root_ppn), - `include "walker_ports.svh" - ); - initial begin - uvm_config_db#(virtual walker_control_if)::set(null, "uvm_test_top", "control", control); - uvm_config_db#(virtual stream_if #(`WALK_REQ_WIDTH))::set(null, "uvm_test_top", "req", - source_if); - uvm_config_db#(virtual stream_if #(`WALK_RESP_WIDTH))::set(null, "uvm_test_top", "resp", - sink_if); - uvm_config_db#(virtual stream_if #(`WALK_ACCESS_WIDTH))::set(null, "uvm_test_top", "access", - access_if); - uvm_config_db#(virtual stream_if #(`WALK_RESULT_WIDTH))::set(null, "uvm_test_top", "result", - result_if); - uvm_config_db#(int)::set(null, "uvm_test_top", "invalid_case", 2); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode.f b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode.f deleted file mode 100644 index 3e9e6d826..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/walker_interface.sv -@RESOURCES@/walker_negative_package.sv --F @RTL@/Walker/filelist.f -@RESOURCES@/walker_bad_mode_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode.toml b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode.toml deleted file mode 100644 index fbd6a3acf..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Walker requires Bare or Sv39 mode" diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode_tb.sv b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode_tb.sv deleted file mode 100644 index 21e941816..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_mode_tb.sv +++ /dev/null @@ -1,44 +0,0 @@ -`include "walker_config.svh" -module walker_bad_mode_tb; - import uvm_pkg::*; - import walker_negative_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - walker_control_if control (clock); - stream_if #(`WALK_REQ_WIDTH) source_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESP_WIDTH) sink_if ( - clock, - control.reset - ); - stream_if #(`WALK_ACCESS_WIDTH) access_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESULT_WIDTH) result_if ( - clock, - control.reset - ); - Walker dut ( - .clock(clock), - .reset(control.reset), - .io_config_mode(control.mode), - .io_config_rootPpn(control.root_ppn), - `include "walker_ports.svh" - ); - initial begin - uvm_config_db#(virtual walker_control_if)::set(null, "uvm_test_top", "control", control); - uvm_config_db#(virtual stream_if #(`WALK_REQ_WIDTH))::set(null, "uvm_test_top", "req", - source_if); - uvm_config_db#(virtual stream_if #(`WALK_RESP_WIDTH))::set(null, "uvm_test_top", "resp", - sink_if); - uvm_config_db#(virtual stream_if #(`WALK_ACCESS_WIDTH))::set(null, "uvm_test_top", "access", - access_if); - uvm_config_db#(virtual stream_if #(`WALK_RESULT_WIDTH))::set(null, "uvm_test_top", "result", - result_if); - uvm_config_db#(int)::set(null, "uvm_test_top", "invalid_case", 0); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege.f b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege.f deleted file mode 100644 index b19f01b46..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/walker_interface.sv -@RESOURCES@/walker_negative_package.sv --F @RTL@/Walker/filelist.f -@RESOURCES@/walker_bad_privilege_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege.toml b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege.toml deleted file mode 100644 index 9a567c024..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Walker effective privilege must be U, S or M" diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege_tb.sv b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege_tb.sv deleted file mode 100644 index cf19dbc56..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_bad_privilege_tb.sv +++ /dev/null @@ -1,44 +0,0 @@ -`include "walker_config.svh" -module walker_bad_privilege_tb; - import uvm_pkg::*; - import walker_negative_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - walker_control_if control (clock); - stream_if #(`WALK_REQ_WIDTH) source_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESP_WIDTH) sink_if ( - clock, - control.reset - ); - stream_if #(`WALK_ACCESS_WIDTH) access_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESULT_WIDTH) result_if ( - clock, - control.reset - ); - Walker dut ( - .clock(clock), - .reset(control.reset), - .io_config_mode(control.mode), - .io_config_rootPpn(control.root_ppn), - `include "walker_ports.svh" - ); - initial begin - uvm_config_db#(virtual walker_control_if)::set(null, "uvm_test_top", "control", control); - uvm_config_db#(virtual stream_if #(`WALK_REQ_WIDTH))::set(null, "uvm_test_top", "req", - source_if); - uvm_config_db#(virtual stream_if #(`WALK_RESP_WIDTH))::set(null, "uvm_test_top", "resp", - sink_if); - uvm_config_db#(virtual stream_if #(`WALK_ACCESS_WIDTH))::set(null, "uvm_test_top", "access", - access_if); - uvm_config_db#(virtual stream_if #(`WALK_RESULT_WIDTH))::set(null, "uvm_test_top", "result", - result_if); - uvm_config_db#(int)::set(null, "uvm_test_top", "invalid_case", 1); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_interface.sv b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_interface.sv deleted file mode 100644 index 895f6d148..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_interface.sv +++ /dev/null @@ -1,7 +0,0 @@ -interface walker_control_if ( - input logic clock -); - logic reset; - logic [3:0] mode; - logic [43:0] root_ppn; -endinterface diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_negative_package.sv b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_negative_package.sv deleted file mode 100644 index e4ff511f8..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_negative_package.sv +++ /dev/null @@ -1,64 +0,0 @@ -`include "walker_config.svh" -package walker_negative_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual walker_control_if control; - virtual stream_if #(`WALK_REQ_WIDTH) req; - virtual stream_if #(`WALK_RESP_WIDTH) resp; - virtual stream_if #(`WALK_ACCESS_WIDTH) access; - virtual stream_if #(`WALK_RESULT_WIDTH) result; - int invalid_case; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual walker_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`WALK_REQ_WIDTH))::get( - this, "", "req", req - ) || !uvm_config_db#(virtual stream_if #(`WALK_RESP_WIDTH))::get( - this, "", "resp", resp - ) || !uvm_config_db#(virtual stream_if #(`WALK_ACCESS_WIDTH))::get( - this, "", "access", access - ) || !uvm_config_db#(virtual stream_if #(`WALK_RESULT_WIDTH))::get( - this, "", "result", result - ) || !uvm_config_db#(int)::get( - this, "", "invalid_case", invalid_case - )) - `uvm_fatal("VIF", "walker negative interface missing") - endfunction - task execute(); - control.reset = 1; - control.mode = 8; - control.root_ppn = 16; - req.valid = 0; - req.bits = '0; - resp.ready = 1; - access.ready = 1; - result.valid = 0; - result.bits = '0; - repeat (4) @(negedge control.clock); - control.reset = 0; - req.bits[`WALK_REQ_PRIVILEGE_OFFSET+:`WALK_REQ_PRIVILEGE_WIDTH] = 1; - case (invalid_case) - 0: control.mode = 9; - 1: req.bits[`WALK_REQ_PRIVILEGE_OFFSET+:`WALK_REQ_PRIVILEGE_WIDTH] = 2; - 2: begin - req.bits[`WALK_REQ_WRITE_OFFSET] = 1; - req.bits[`WALK_REQ_EXECUTE_OFFSET] = 1; - end - default: `uvm_fatal("CASE", "Unknown invalid walker request") - endcase - req.valid = 1; - `uvm_info("INVALID_INPUT", $sformatf("Driving walker contract violation %0d", invalid_case), - UVM_LOW) - repeat (10) @(negedge control.clock); - `uvm_fatal("MISSING_ASSERTION", - "Illegal walker input did not trigger the specified contract assertion") - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_package.sv b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_package.sv deleted file mode 100644 index 35f7a04e7..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_package.sv +++ /dev/null @@ -1,458 +0,0 @@ -`include "walker_config.svh" -`define WF(K, F) `WALK_``K``_``F``_OFFSET +: `WALK_``K``_``F``_WIDTH -package walker_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - - import "DPI-C" function chandle mmu_ref_create(input int unsigned address_bits); - import "DPI-C" function void mmu_ref_destroy(input chandle model); - import "DPI-C" function void mmu_ref_clear(input chandle model); - import "DPI-C" function void mmu_ref_program( - input chandle model, - input longint unsigned address, - value, - input int unsigned error - ); - import "DPI-C" function int unsigned mmu_ref_read( - input chandle model, - input longint unsigned address, - output longint unsigned value, - output int unsigned error - ); - import "DPI-C" function int unsigned mmu_ref_pending(input chandle model); - import "DPI-C" function int unsigned mmu_ref_peek( - input chandle model, - output longint unsigned address - ); - import "DPI-C" function int unsigned mmu_ref_consume( - input chandle model, - input longint unsigned address - ); - import "DPI-C" function int unsigned mmu_ref_translate( - input chandle model, - input longint unsigned address, - input int unsigned mode, - input longint unsigned root, - input int unsigned privilege, - write, - execute, - sum, - mxr, - output longint unsigned paddr, - output int unsigned page_fault, - access_fault, - level - ); - typedef bit [63:0] u64; - typedef logic [`WALK_REQ_WIDTH-1:0] req_t; - class translation extends uvm_sequence_item; - bit [63:0] paddr; - bit page_fault, access_fault; - bit [1:0] level; - `uvm_object_utils_begin(translation) - `uvm_field_int(paddr, UVM_DEFAULT) - `uvm_field_int(page_fault, UVM_DEFAULT) - `uvm_field_int(access_fault, UVM_DEFAULT) - `uvm_field_int(level, UVM_DEFAULT) - `uvm_object_utils_end - function new(string name = "translation"); - super.new(name); - endfunction - endclass - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual walker_control_if control; - virtual stream_if #(`WALK_REQ_WIDTH) req; - virtual stream_if #(`WALK_RESP_WIDTH) resp; - virtual stream_if #(`WALK_ACCESS_WIDTH) access; - virtual stream_if #(`WALK_RESULT_WIDTH) result; - in_order_scoreboard #(translation) scoreboard; - // Only explicitly programmed PTEs exist. There is no implicit zero/pattern RAM. - chandle model; - int - expected = 0, - accepted = 0, - returned = 0, - reads = 0, - cycle = 0, - req_stalls = 0, - access_stalls = 0, - resp_stalls = 0; - int page_faults = 0, access_faults = 0, successes[3] = '{0, 0, 0}; - int due = 0; - bit pending_result = 0, result_active = 0, hold_access = 0, hold_response = 0; - logic [`WALK_RESULT_WIDTH-1:0] result_packet; - string current_case; - localparam u64 MAX_PA = (64'h1 << `WALK_RESP_PADDR_WIDTH) - 1; - localparam u64 HIGH_PPN = 64'h1 << (`WALK_RESP_PADDR_WIDTH - 12); - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 1ms; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - scoreboard = in_order_scoreboard#(translation)::type_id::create("scoreboard", this); - model = mmu_ref_create(`WALK_RESP_PADDR_WIDTH); - if (!uvm_config_db#(virtual walker_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`WALK_REQ_WIDTH))::get( - this, "", "req", req - ) || !uvm_config_db#(virtual stream_if #(`WALK_RESP_WIDTH))::get( - this, "", "resp", resp - ) || !uvm_config_db#(virtual stream_if #(`WALK_ACCESS_WIDTH))::get( - this, "", "access", access - ) || !uvm_config_db#(virtual stream_if #(`WALK_RESULT_WIDTH))::get( - this, "", "result", result - )) - `uvm_fatal("VIF", "Walker interfaces missing") - endfunction - function req_t request(u64 va, int operation = 0, int privilege = 1, bit sum = 0, bit mxr = 0); - req_t r = '0; - r[`WF(REQ, VADDR)] = va; - r[`WF(REQ, WRITE)] = operation == 1; - r[`WF(REQ, EXECUTE)] = operation == 2; - r[`WF(REQ, PRIVILEGE)] = privilege; - r[`WF(REQ, SUM)] = sum; - r[`WF(REQ, MXR)] = mxr; - return r; - endfunction - function void program_pte(u64 address, u64 value, bit error = 0); - mmu_ref_program(model, address, value, error); - endfunction - function u64 pte(u64 address); - longint unsigned value; - int unsigned error; - if (!mmu_ref_read(model, address, value, error)) - `uvm_fatal("FIXTURE", "Attempt to modify an unprogrammed PTE") - return value; - endfunction - function translation golden(req_t r, int mode, u64 root); - longint unsigned address; - int unsigned page_fault, access_fault, level; - translation t = new(current_case); - if (mmu_ref_translate( - model, - r[ - `WF(REQ, VADDR) - ], - mode, - root, - r[ - `WF(REQ, PRIVILEGE) - ], - r[ - `WF(REQ, WRITE) - ], - r[ - `WF(REQ, EXECUTE) - ], - r[ - `WF(REQ, SUM) - ], - r[ - `WF(REQ, MXR) - ], - address, - page_fault, - access_fault, - level - ) != 0) - `uvm_fatal("FIXTURE", $sformatf("%s uninitialized golden PTE %h", current_case, address)) - t.paddr = address; - t.page_fault = page_fault; - t.access_fault = access_fault; - t.level = level; - return t; - endfunction - function u64 pte_address(u64 va, int level); - return (64'h12 - level) * 4096 + ((va >> (12 + 9 * level)) & 511) * 8; - endfunction - function void prepare(u64 va, int leaf_level, u64 flags = 64'hcf, u64 mapped = 64'h80000000); - mmu_ref_clear(model); - for (int level = 2; level >= leaf_level; level--) begin - if (level == leaf_level) - program_pte(pte_address(va, level), ((mapped >> 12) << 10) | flags); - else program_pte(pte_address(va, level), ((64'h13 - level) << 10) | 1); - end - endfunction - task service(); - forever begin - @(req.sample); - if (!control.reset) begin - cycle++; - if (req.sample.valid && req.sample.ready) accepted++; - if (req.sample.valid && !req.sample.ready) req_stalls++; - if (access.sample.valid && !access.sample.ready) access_stalls++; - if (resp.sample.valid && !resp.sample.ready) resp_stalls++; - if (result_active && result.sample.valid && result.sample.ready) begin - result_active = 0; - pending_result = 0; - end - if (access.sample.valid) begin - u64 address; - longint unsigned next_address, value; - int unsigned error; - if ($isunknown(access.sample.bits)) `uvm_fatal("PTE_X", "Unknown PTE memory request") - address = access.sample.bits[`WF(ACCESS, ADDR)]; - if (!mmu_ref_peek(model, next_address) || address !== next_address) - `uvm_fatal("PTE_SEQUENCE", $sformatf( - "%s unexpected PTE address %h (remaining=%0d)", - current_case, - address, - mmu_ref_pending( - model - ) - )) - if (access.sample.bits[ - `WF(ACCESS, WRITE) - ] || access.sample.bits[ - `WF(ACCESS, DATA) - ] != 0 || access.sample.bits[ - `WF(ACCESS, MASK) - ] != 0 || access.sample.bits[ - `WF(ACCESS, ATOMIC) - ] != 0 || access.sample.bits[ - `WF(ACCESS, ATOMICWORD) - ] != 0) - `uvm_fatal("PTE_WRITE", "Svade profile permits only ordinary 64-bit PTE reads") - if (!mmu_ref_read(model, address, value, error)) - `uvm_fatal("UNINITIALIZED_PTE", "DUT read an unprogrammed memory location") - if (access.sample.ready) begin - if (pending_result) - `uvm_fatal("OUTSTANDING", "Walker issued more than one PTE access") - if (!mmu_ref_consume(model, address)) - `uvm_fatal("PTE_SEQUENCE", "Reference PTE queue changed before handshake") - reads++; - result_packet = '0; - result_packet[`WF(RESULT, DATA)] = value; - result_packet[`WF(RESULT, ERROR)] = error; - due = cycle + 2 + reads % 4; - pending_result = 1; - end - end - if (resp.sample.valid && resp.sample.ready) begin - translation actual = new(current_case); - if ($isunknown(resp.sample.bits)) - `uvm_fatal("RESPONSE_X", "Unknown translation response") - actual.paddr = resp.sample.bits[`WF(RESP, PADDR)]; - actual.page_fault = resp.sample.bits[`WF(RESP, PAGEFAULT)]; - actual.access_fault = resp.sample.bits[`WF(RESP, ACCESSFAULT)]; - actual.level = resp.sample.bits[`WF(RESP, LEVEL)]; - if (actual.page_fault && actual.access_fault) - `uvm_fatal("FAULT_KIND", "Fault kinds must be mutually exclusive") - if ((actual.page_fault || actual.access_fault) && actual.paddr != 0) - `uvm_fatal("FAULT_PA", "Fault response exposed a physical address") - scoreboard.actual_export.write(actual); - returned++; - if (actual.page_fault) page_faults++; - else if (actual.access_fault) access_faults++; - else successes[actual.level]++; - end - end - @(negedge control.clock); - access.ready = !control.reset && !hold_access && cycle % 5 >= 2; - resp.ready = !control.reset && !hold_response && cycle % 7 >= 3; - if (pending_result && !result_active && cycle >= due) result_active = 1; - result.valid = result_active; - result.bits = result_packet; - end - endtask - task send(req_t r, int mode = 8, u64 root = 16, bit change_inputs = 0); - @(negedge control.clock); - control.mode = mode; - control.root_ppn = root; - req.bits = r; - req.valid = 1; - do @(req.sample); while (!req.sample.ready); - @(negedge control.clock); - req.valid = 0; - if (change_inputs) begin - control.mode = 0; - control.root_ppn = 0; - req.bits = ~r; - end - endtask - task run_case(string name, req_t r, int mode = 8, u64 root = 16, bit change_inputs = 0); - translation prediction; - current_case = name; - if (mmu_ref_pending(model) || pending_result) - `uvm_fatal("PENDING", "Previous translation not drained") - prediction = golden(r, mode, root); - scoreboard.expected_export.write(prediction); - expected++; - send(r, mode, root, change_inputs); - scoreboard.wait_checked(expected); - @(negedge control.clock); - if (mmu_ref_pending(model) || pending_result) - `uvm_fatal("PENDING", "Translation skipped expected PTE reads") - endtask - task execute(); - u64 va = 64'h123456789, flags, mapped; - int permissions[5] = '{1, 3, 4, 5, 7}; - control.reset = 1; - control.mode = 0; - control.root_ppn = 0; - req.valid = 0; - req.bits = '0; - resp.ready = 0; - access.ready = 0; - result.valid = 0; - result.bits = '0; - repeat (5) @(negedge control.clock); - control.reset = 0; - fork - service(); - join_none - // Explicitly force a stalled PTE request, then hold the response while a - // second request is pending. Both expected paths use initialized table words. - prepare(va, 0, 64'h43, 64'h81234000); - current_case = "back_to_back_read_then_execute_fault"; - scoreboard.expected_export.write(golden(request(va), 8, 16)); - expected++; - scoreboard.expected_export.write(golden(request(va, 2), 8, 16)); - expected++; - hold_access = 1; - hold_response = 1; - send(request(va)); - fork - send(request(va, 2)); - begin - wait (access.sample.valid); - repeat (8) @(req.sample); - hold_access = 0; - wait (resp.sample.valid); - repeat (8) @(req.sample); - hold_response = 0; - end - join - scoreboard.wait_checked(expected); - @(negedge control.clock); - if (mmu_ref_pending(model) || pending_result) - `uvm_fatal("PENDING", "Back-to-back translations failed to drain") - // Three page sizes and both canonical sign regions, including page edges. - for (int level = 0; level < 3; level++) begin - for (int position = 0; position < 3; position++) begin - u64 size = 64'h1000 << (9 * level); - u64 address = (va & ~(size - 1)) + (position == 0 ? 0 : position == 1 ? 123 : size - 1); - prepare(address, level); - run_case($sformatf("page_size_%0d_edge_%0d", level, position), request(address)); - address |= 64'hffffffc000000000; - prepare(address, level); - run_case($sformatf("negative_canonical_%0d_%0d", level, position), request(address)); - end - end - // Permissions: all valid RWX forms, U/S, U-page, SUM, MXR, load/store/fetch. - foreach (permissions[i]) - for (int privilege = 0; privilege < 2; privilege++) - for (int user_page = 0; user_page < 2; user_page++) - for (int sum = 0; sum < 2; sum++) - for (int mxr = 0; mxr < 2; mxr++) - for (int operation = 0; operation < 3; operation++) begin - flags = 64'hc1 | (u64'(permissions[i]) << 1) | (u64'(user_page) << 4); - prepare(va, 0, flags, 64'h81234000); - run_case($sformatf( - "perm_%0d_prv%0d_u%0d_sum%0d_mxr%0d_op%0d", - permissions[i], - privilege, - user_page, - sum, - mxr, - operation - ), request(va, operation, privilege, sum, mxr)); - end - for (int level = 0; level < 3; level++) - for (int ad = 0; ad < 4; ad++) - for (int operation = 0; operation < 3; operation++) begin - flags = 64'hf | (u64'(ad) << 6); - prepare(va, level, flags); - run_case($sformatf("svade_level%0d_ad%0d_op%0d", level, ad, operation), request( - va, operation)); - end - for (int level = 0; level < 3; level++) begin - for (int reserved_bit = 54; reserved_bit < 64; reserved_bit++) begin - prepare(va, level); - program_pte(pte_address(va, level), pte(pte_address(va, level) - ) | (64'h1 << reserved_bit)); - run_case($sformatf("reserved_leaf_l%0d_bit%0d", level, reserved_bit), request(va)); - end - for (int encoding = 0; encoding < 3; encoding++) begin - prepare(va, level, encoding == 0 ? 64'hce : encoding == 1 ? 64'hc5 : 64'hcd); - run_case($sformatf("invalid_pte_l%0d_encoding%0d", level, encoding), request(va)); - end - prepare(va, level); - program_pte(pte_address(va, level), '1, 1); - run_case($sformatf("backend_error_l%0d", level), request(va)); - prepare(va, level, 64'hcf, HIGH_PPN * 4096); - run_case($sformatf("physical_leaf_overflow_l%0d", level), request(va)); - prepare(va, level, 64'hf, HIGH_PPN * 4096); - run_case($sformatf("pte_fault_precedes_final_pa_overflow_l%0d", level), request(va)); - end - for (int level = 1; level <= 2; level++) begin - for (int bit_index = 0; bit_index < 9 * level; bit_index++) begin - prepare(va, level); - program_pte(pte_address(va, level), pte(pte_address(va, level) - ) | (64'h1 << (10 + bit_index))); - run_case($sformatf("superpage_alignment_l%0d_bit%0d", level, bit_index), request(va)); - end - for (int bad = 0; bad < 3; bad++) begin - prepare(va, 0); - program_pte(pte_address(va, level), pte(pte_address(va, level) - ) | (64'h1 << (bad == 0 ? 4 : bad == 1 ? 6 : 7))); - run_case($sformatf("nonleaf_reserved_l%0d_%0d", level, bad), request(va)); - end - for (int bit_index = 54; bit_index < 64; bit_index++) begin - prepare(va, 0); - program_pte(pte_address(va, level), pte(pte_address(va, level)) | (64'h1 << bit_index)); - run_case($sformatf("nonleaf_upper_reserved_l%0d_%0d", level, bit_index), request(va)); - end - prepare(va, 0); - program_pte(pte_address(va, level), (HIGH_PPN << 10) | 1); - run_case($sformatf("next_table_address_overflow_l%0d", level), request(va)); - end - prepare(va, 0); - program_pte(pte_address(va, 0), (64'h13 << 10) | 1); - run_case("nonleaf_at_bottom", request(va)); - prepare(va, 0, 64'h3ef); // G and both RSW bits are permitted and ignored for translation. - for (int level = 1; level <= 2; level++) - program_pte(pte_address(va, level), pte(pte_address(va, level)) | 64'h320); - run_case("software_and_global_bits", request(va)); - prepare(va, 0, 64'h59); - run_case("latched_root_sum_mxr_privilege", request(va, 0, 1, 1, 1), 8, 16, 1); - mmu_ref_clear(model); - run_case("root_physical_overflow", request(va), 8, HIGH_PPN); - run_case("positive_noncanonical", request(64'h0000004000000000)); - run_case("upper_noncanonical", request(64'hffffffbfffffffff)); - for (int operation = 0; operation < 3; operation++) begin - for (int privilege = 0; privilege < 2; privilege++) begin - run_case("bare_noncanonical_physical", request(64'h8000000000, operation, privilege), 0, - 0); - run_case("bare_last_physical_byte", request(MAX_PA, operation, privilege), 0, 0); - run_case("bare_physical_overflow", request(MAX_PA + 1, operation, privilege), 0, 0); - end - run_case("machine_ignores_sv39", request(64'h8000000000, operation, 3), 8, HIGH_PPN); - run_case("machine_physical_overflow", request(MAX_PA + 1, operation, 3), 8, 16); - end - // Single-address translations on opposite sides of a base-page boundary. - prepare(64'hfff, 0, 64'hcf, MAX_PA & ~64'hfff); - run_case("last_byte_before_physical_limit", request(64'hfff)); - prepare(64'h1000, 0, 64'hcf, MAX_PA + 1); - run_case("next_page_physical_overflow", request(64'h1000)); - // The last PTE word of the highest implemented physical table page is legal. - mmu_ref_clear(model); - program_pte(MAX_PA - 7, 64'hcf); - run_case("last_physical_pte_word", request(64'hffffffffc0000000), 8, MAX_PA >> 12); - if(expected!=accepted||expected!=returned||!req_stalls||!access_stalls||!resp_stalls||!page_faults||!access_faults||!successes[0]||!successes[1]||!successes[2]) - `uvm_fatal("SCENARIOS", "Walker scenarios or handshake checks were not exercised") - `uvm_info( - "CHECKS", - $sformatf( - "Sv39/Svade walker checked=%0d PTEreads=%0d successLevels=%0d/%0d/%0d PF=%0d AF=%0d stalls(req/access/resp)=%0d/%0d/%0d; explicit PTE dictionary, no implicit writes", - expected, reads, successes[0], successes[1], successes[2], page_faults, - access_faults, req_stalls, access_stalls, resp_stalls), UVM_LOW) - endtask - function void final_phase(uvm_phase phase); - mmu_ref_destroy(model); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_tb.sv b/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_tb.sv deleted file mode 100644 index 646a52c6c..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/resources/walker_tb.sv +++ /dev/null @@ -1,43 +0,0 @@ -`include "walker_config.svh" -module walker_tb; - import uvm_pkg::*; - import walker_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - walker_control_if control (clock); - stream_if #(`WALK_REQ_WIDTH) source_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESP_WIDTH) sink_if ( - clock, - control.reset - ); - stream_if #(`WALK_ACCESS_WIDTH) access_if ( - clock, - control.reset - ); - stream_if #(`WALK_RESULT_WIDTH) result_if ( - clock, - control.reset - ); - Walker dut ( - .clock(clock), - .reset(control.reset), - .io_config_mode(control.mode), - .io_config_rootPpn(control.root_ppn), - `include "walker_ports.svh" - ); - initial begin - uvm_config_db#(virtual walker_control_if)::set(null, "uvm_test_top", "control", control); - uvm_config_db#(virtual stream_if #(`WALK_REQ_WIDTH))::set(null, "uvm_test_top", "req", - source_if); - uvm_config_db#(virtual stream_if #(`WALK_RESP_WIDTH))::set(null, "uvm_test_top", "resp", - sink_if); - uvm_config_db#(virtual stream_if #(`WALK_ACCESS_WIDTH))::set(null, "uvm_test_top", "access", - access_if); - uvm_config_db#(virtual stream_if #(`WALK_RESULT_WIDTH))::set(null, "uvm_test_top", "result", - result_if); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/mmu/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/mmu/src/main/scala/Emit.scala deleted file mode 100644 index 95c87ffd3..000000000 --- a/arch/src/main/scala/framework/mem-core/mmu/src/main/scala/Emit.scala +++ /dev/null @@ -1,58 +0,0 @@ -package memcore.memory.mmu - -import java.nio.file.{Files, Paths} -import memcore.bus.chi.Params - -object Emit extends App { - val p = Params() - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Walker(p), - firtoolOpts = args, - args = Array("--target-dir", "build/Walker", "--split-verilog") - ) - val walkerDefinitions = scala.collection.mutable.ArrayBuffer[String]() - val walkerPorts = scala.collection.mutable.ArrayBuffer[String]() - - def walkerChannel( - kind: String, - port: String, - signal: String, - fs: Seq[(String, Int)] - ): Unit = { - var offset = 0 - for ((field, width) <- fs) { - walkerDefinitions += s"`define WALK_${kind}_${field.toUpperCase}_OFFSET $offset" - walkerDefinitions += s"`define WALK_${kind}_${field.toUpperCase}_WIDTH $width" - walkerPorts += s".io_${port}_bits_$field($signal.bits[$offset +: $width])" - offset += width - } - walkerDefinitions += s"`define WALK_${kind}_WIDTH $offset" - walkerPorts += s".io_${port}_valid($signal.valid)" - walkerPorts += s".io_${port}_ready($signal.ready)" - } - - walkerChannel( - "REQ", - "req", - "source_if", - Seq("vaddr" -> 64, "write" -> 1, "execute" -> 1, "privilege" -> 2, "sum" -> 1, "mxr" -> 1) - ) - walkerChannel( - "RESP", - "resp", - "sink_if", - Seq("paddr" -> p.addressBits, "pageFault" -> 1, "accessFault" -> 1, "level" -> 2) - ) - walkerChannel( - "ACCESS", - "access", - "access_if", - Seq("addr" -> p.addressBits, "write" -> 1, "data" -> 64, "mask" -> 8, "atomic" -> 4, "atomicWord" -> 1) - ) - walkerChannel("RESULT", "result", "result_if", Seq("data" -> 64, "error" -> 1)) - Files.writeString( - Paths.get("build/walker_config.svh"), - "`ifndef WALKER_CONFIG_SVH\n`define WALKER_CONFIG_SVH\n" + walkerDefinitions.mkString("\n") + "\n`endif\n" - ) - Files.writeString(Paths.get("build/walker_ports.svh"), walkerPorts.mkString(",\n") + "\n") -} diff --git a/arch/src/main/scala/framework/mem-core/preflight/build.sc b/arch/src/main/scala/framework/mem-core/preflight/build.sc deleted file mode 100644 index b28c2c7e0..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/build.sc +++ /dev/null @@ -1,35 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object chi extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = os.pwd / os.up / "chi" -} - -object mmu extends ChiselModule { - override def moduleRoot = os.pwd / os.up / "mmu" - override def moduleDeps = Seq(chi) -} - -object preflight extends ChiselModule { - override def moduleRoot = os.pwd - override def moduleDeps = Seq(chi, mmu) - override def mainClass = Some("memcore.memory.preflight.Emit") -} diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/package.sv deleted file mode 100644 index 3ab987bbb..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/package.sv +++ /dev/null @@ -1,543 +0,0 @@ -`ifdef PREFLIGHT_LINE_READ -`include "preflight64_config.svh" -`else -`include "preflight_config.svh" -`endif -`define PF(K, F) `PF_``K``_``F``_OFFSET +: `PF_``K``_``F``_WIDTH -package preflight_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - typedef longint unsigned u64; - typedef bit [`PF_CMD_WIDTH-1:0] command_t; - import "DPI-C" function chandle preflight_ref_create( - input int unsigned address_bits, - beat - ); - import "DPI-C" function void preflight_ref_destroy(input chandle model); - import "DPI-C" function void preflight_ref_program( - input chandle model, - input longint unsigned address, - value, - input int unsigned error - ); - import "DPI-C" function void preflight_ref_policy( - input chandle model, - input int unsigned pte_valid, - input longint unsigned pte, - input int unsigned range_valid, - input longint unsigned first, - last - ); - import "DPI-C" function int unsigned preflight_ref_submit( - input chandle model, - input int unsigned id, - input longint unsigned base, - input int unsigned rows, - cols, - span, - col_stride, - row_stride, - write, - mode, - input longint unsigned root, - input int unsigned privilege, - sum, - mxr - ); - import "DPI-C" function int unsigned preflight_ref_count( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void preflight_ref_expected( - input chandle model, - input int unsigned id, - index, - output longint unsigned va, - pa, - output int unsigned bytes, - write, - last, - error - ); - import "DPI-C" function int unsigned preflight_ref_authorize( - input chandle model, - input int unsigned id, - input longint unsigned pa, - input int unsigned bytes, - write, - pte, - privilege, - output int unsigned allow - ); - import "DPI-C" function int unsigned preflight_ref_pte( - input chandle model, - input int unsigned id, - input longint unsigned address, - output longint unsigned value, - output int unsigned error - ); - import "DPI-C" function int unsigned preflight_ref_pending( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void preflight_ref_retire( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void preflight_ref_reset(input chandle model); - class result_item extends uvm_sequence_item; - bit [`PF_OUT_WIDTH-1:0] packet; - `uvm_object_utils_begin(result_item) - `uvm_field_int(packet, UVM_DEFAULT) - `uvm_object_utils_end - function new(string name = "result_item"); - super.new(name); - endfunction - endclass - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual preflight_control_if ctl; - virtual stream_if #(`PF_CMD_WIDTH) cmd; - virtual stream_if #(`PF_OUT_WIDTH) prepared; - virtual stream_if #(`PF_AUTH_WIDTH) authorization; - virtual stream_if #(`PF_PERMIT_WIDTH) permission; - virtual stream_if #(`PF_PTE_WIDTH) pte_req; - virtual stream_if #(`PF_PTERESP_WIDTH) pte_resp; - keyed_scoreboard #(result_item) scoreboard; - chandle model; - bit live[int]; - int output_index[int]; - int cycle = 0, submitted = 0, retired = 0, segments = 0, failures[7] = '{default: 0}, peak = 0; - int - auth_count = 0, - pte_count = 0, - command_stalls = 0, - output_stalls = 0, - auth_stalls = 0, - cancelled = 0; - bit hold_output = 0, hold_permission = 0, auth_pending = 0, pte_pending = 0, pte_grant = 0; - int auth_id, pte_id, grant_id, auth_due, pte_due; - bit auth_allow, auth_pte; - u64 grant_pa; - logic [`PF_PERMIT_WIDTH-1:0] permit_packet; - logic [`PF_PTERESP_WIDTH-1:0] pte_packet; - u64 next_table = 'h10000; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 1ms; - endfunction - function void verify_contract(bit ok, string message); - if (!ok) `uvm_fatal("PREFLIGHT", message) - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - model = preflight_ref_create(`PF_ADDRESS_BITS, `PF_BEAT_BYTES); - scoreboard = keyed_scoreboard#(result_item)::type_id::create("scoreboard", this); - verify_contract(uvm_config_db#(virtual preflight_control_if)::get(this, "", "ctl", ctl), - "Missing control"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_CMD_WIDTH))::get(this, "", "cmd", cmd), - "Missing command"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::get( - this, "", "prepared", prepared), "Missing output"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_AUTH_WIDTH))::get( - this, "", "authorization", authorization), "Missing authorization"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_PERMIT_WIDTH))::get( - this, "", "permission", permission), "Missing permission"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_PTE_WIDTH))::get( - this, "", "pte_req", pte_req), "Missing PTE request"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_PTERESP_WIDTH))::get( - this, "", "pte_resp", pte_resp), "Missing PTE response"); - endfunction - function command_t request(int id, u64 base, int rows, int span, int row_stride, int cols = 1, - int col_stride = 0, bit write = 0, int mode = 0, u64 root = 0, - int privilege = 1, bit sum = 0, bit mxr = 0); - command_t q = '0; - q[`PF(CMD, ID)] = id; - q[`PF(CMD, BASEVA)] = base; - q[`PF(CMD, ROWS)] = rows; - q[`PF(CMD, COLUMNS)] = cols; - q[`PF(CMD, SPANBYTES)] = span; - q[`PF(CMD, ROWSTRIDE)] = row_stride; - q[`PF(CMD, COLUMNSTRIDE)] = col_stride; - q[`PF(CMD, WRITE)] = write; - q[`PF(CMD, MODE)] = mode; - q[`PF(CMD, ROOTPPN)] = root; - q[`PF(CMD, PRIVILEGE)] = privilege; - q[`PF(CMD, SUM)] = sum; - q[`PF(CMD, MXR)] = mxr; - return q; - endfunction - function u64 mapping(u64 va, int pages, u64 flags = 'hc7, int backend_error_page = -1); - u64 root = next_table; - next_table += 'h3000; - preflight_ref_program(model, root + ((va >> 30) & 511) * 8, ((root + 4096) >> 12) << 10 | 1, - 0); - preflight_ref_program(model, root + 4096 + ((va >> 21) & 511) * 8, - ((root + 8192) >> 12) << 10 | 1, 0); - for (int i = 0; i < pages; i++) begin - u64 a = (va & ~64'hfff) + i * 4096; - // Deliberately noncontiguous 4KiB physical pages; no identity-map shortcut. - preflight_ref_program(model, root + 8192 + ((a >> 12) & 511) * 8, - ((64'h800000 + i * 8192) >> 12) << 10 | flags, - i == backend_error_page); - end - return root >> 12; - endfunction - function void predict(command_t q); - int id = q[`PF(CMD, ID)]; - int count; - verify_contract(!live.exists(id), "Duplicate admitted command ID"); - verify_contract(preflight_ref_submit( - model, - id, - q[ - `PF(CMD, BASEVA) - ], - q[ - `PF(CMD, ROWS) - ], - q[ - `PF(CMD, COLUMNS) - ], - q[ - `PF(CMD, SPANBYTES) - ], - q[ - `PF(CMD, COLUMNSTRIDE) - ], - q[ - `PF(CMD, ROWSTRIDE) - ], - q[ - `PF(CMD, WRITE) - ], - q[ - `PF(CMD, MODE) - ], - q[ - `PF(CMD, ROOTPPN) - ], - q[ - `PF(CMD, PRIVILEGE) - ], - q[ - `PF(CMD, SUM) - ], - q[ - `PF(CMD, MXR) - ] - ) == 0, "Uninitialized PTE in reference fixture"); - live[id] = 1; - output_index[id] = 0; - submitted++; - if (live.num() > peak) peak = live.num(); - count = preflight_ref_count(model, id); - for (int i = 0; i < count; i++) begin - result_item expected = new; - u64 va, pa; - int unsigned bytes, write, last, error; - preflight_ref_expected(model, id, i, va, pa, bytes, write, last, error); - expected.packet = '0; - expected.packet[`PF(OUT, ID)] = id; - expected.packet[`PF(OUT, VA)] = va; - expected.packet[`PF(OUT, PA)] = pa; - expected.packet[`PF(OUT, BYTES)] = bytes; - expected.packet[`PF(OUT, WRITE)] = write; - expected.packet[`PF(OUT, LAST)] = last; - expected.packet[`PF(OUT, ERROR)] = error; - expected.set_transaction_id((id << 4) | i); - scoreboard.expected_export.write(expected); - end - endfunction - task service(); - forever begin - @(ctl.sample); - if (!ctl.sample.reset) begin - cycle++; - if (cmd.sample.valid && !cmd.sample.ready) command_stalls++; - if (prepared.sample.valid && !prepared.sample.ready) output_stalls++; - if (authorization.sample.valid && !authorization.sample.ready) auth_stalls++; - if (cmd.sample.valid && cmd.sample.ready) predict(cmd.sample.bits); - if (permission.sample.valid && permission.sample.ready) begin - verify_contract(auth_pending, "Unsolicited permission consumed"); - if (auth_pte && auth_allow) begin - pte_grant = 1; - grant_id = auth_id; - end - auth_pending = 0; - end - if (authorization.sample.valid && authorization.sample.ready) begin - int unsigned allow; - int id = authorization.sample.bits[`PF(AUTH, ID)]; - verify_contract(!$isunknown(authorization.sample.bits) && !auth_pending, - "Overlapping/unknown authorization"); - verify_contract(preflight_ref_authorize( - model, - id, - authorization.sample.bits[ - `PF(AUTH, PA) - ], - authorization.sample.bits[ - `PF(AUTH, BYTES) - ], - authorization.sample.bits[ - `PF(AUTH, WRITE) - ], - authorization.sample.bits[ - `PF(AUTH, ISPTE) - ], - authorization.sample.bits[ - `PF(AUTH, PRIVILEGE) - ], - allow - ) == 1, "Authorization lost snapshot, range or PTE S privilege"); - auth_pending = 1; - auth_id = id; - auth_allow = allow; - auth_pte = authorization.sample.bits[`PF(AUTH, ISPTE)]; - grant_pa = authorization.sample.bits[`PF(AUTH, PA)]; - auth_due = cycle + 3; - permit_packet = '0; - permit_packet[`PF(PERMIT, ID)] = id; - permit_packet[`PF(PERMIT, ALLOW)] = allow; - auth_count++; - end - if (pte_resp.sample.valid && pte_resp.sample.ready) pte_pending = 0; - if (pte_req.sample.valid && pte_req.sample.ready) begin - u64 value; - int unsigned error; - verify_contract(pte_grant && !pte_pending && !$isunknown(pte_req.sample.bits), - "PTE accessed before authorization"); - verify_contract(pte_req.sample.bits[`PF(PTE, ADDR)] == grant_pa && !pte_req.sample.bits[ - `PF(PTE, WRITE)] && pte_req.sample.bits[`PF(PTE, ATOMIC) - ] == 0 && !pte_req.sample.bits[`PF(PTE, ATOMICWORD) - ] && pte_req.sample.bits[`PF(PTE, MASK)] == 0, - "PTE request changed address or attempted writeback"); - verify_contract(preflight_ref_pte(model, grant_id, grant_pa, value, error) == 1, - "PTE address chain differs from architectural walker"); - pte_packet = '0; - pte_packet[`PF(PTERESP, DATA)] = value; - pte_packet[`PF(PTERESP, ERROR)] = error; - pte_id = grant_id; - pte_pending = 1; - pte_due = cycle + 2 + (pte_count % 4); - pte_grant = 0; - pte_count++; - end - if (prepared.sample.valid) begin - int id = prepared.sample.bits[`PF(OUT, ID)]; - verify_contract(live.exists(id) && !$isunknown(prepared.sample.bits), - "Unsolicited/unknown prepared plan"); - verify_contract( - preflight_ref_pending(model, id - ) == 0 && !(auth_pending && auth_id == id) && !(pte_pending && pte_id == id), - "Partial plan exposed before all translations/permissions completed"); - if (prepared.sample.ready) begin - result_item actual = new; - actual.packet = prepared.sample.bits; - actual.set_transaction_id((id << 4) | output_index[id]); - scoreboard.actual_export.write(actual); - output_index[id]++; - segments++; - if (prepared.sample.bits[`PF(OUT, LAST)]) begin - failures[int'(prepared.sample.bits[`PF(OUT, ERROR)])]++; - retired++; - live.delete(id); - output_index.delete(id); - preflight_ref_retire(model, id); - end - end - end - end - @(negedge ctl.clock); - authorization.ready = !ctl.reset && !auth_pending && cycle % 5 >= 2; - permission.valid = !ctl.reset && auth_pending && !hold_permission && cycle >= auth_due; - permission.bits = permit_packet; - pte_req.ready = !ctl.reset && !pte_pending && cycle % 4 != 0; - pte_resp.valid = !ctl.reset && pte_pending && cycle >= pte_due; - pte_resp.bits = pte_packet; - prepared.ready = !ctl.reset && !hold_output && cycle % 7 >= 3; - end - endtask - task submit(command_t q); - @(negedge ctl.clock); - cmd.bits = q; - cmd.valid = 1; - do @(cmd.sample); while (!cmd.sample.ready); - @(negedge ctl.clock); - cmd.valid = 0; - cmd.bits = ~q; - endtask - task drain(); - wait (live.num() == 0); - @(ctl.sample); - verify_contract(!auth_pending && !pte_pending && !pte_grant, - "Preparation endpoints did not drain"); - endtask - task execute(); - u64 root, root2; - int prior, gap_stride; - command_t q; - ctl.reset = 1; - cmd.valid = 0; - cmd.bits = '0; - prepared.ready = 0; - authorization.ready = 0; - permission.valid = 0; - permission.bits = '0; - pte_req.ready = 0; - pte_resp.valid = 0; - pte_resp.bits = '0; - repeat (5) @(negedge ctl.clock); - ctl.reset = 0; - fork - service(); - join_none - // Current BERT maximum: 1024 dense 16B rows normalize to five pages. - root = mapping('h400000f0, 9); - root2 = mapping('h50000000, 9); - hold_output = 1; - submit(request(1, 'h400000f0, 1024, 16, 16, 1, 0, 0, 8, root)); - submit(request(2, 'h50000000, 2048, 16, 16, 1, 0, 1, 8, root2)); // exactly eight pages - submit(request(3, 'h60000ff8, 1, 16, 16)); // alignment-expanded beat crosses page - submit(request(4, 'h70000000, 1, 16, 16)); - wait (prepared.sample.valid); - repeat (8) @(ctl.sample); - fork - submit(request(5, 'h70002000, 1, 16, 16)); - begin - repeat (12) @(ctl.sample); - hold_output = 0; - end - join - drain(); - submit(request(6, 'h50000000, 2304, 16, 16, 1, 0, 0, 8, root2)); - drain(); // ninth page: one error, zero successful output - submit(request(7, 'h60000000, 2, 16, 8192, 2, 4096)); - drain(); // sparse 2D, four extents - submit(request(8, 'h60000000, 1, 16, 16)); - drain(); // progress immediately after capacity error - root = mapping('h40010000, 1, 'h43); - submit(request(9, 'h40010000, 1, 16, 16, 1, 0, 1, 8, root)); - drain(); - root = mapping('h40020000, 1, 'h7); - submit(request(10, 'h40020000, 1, 16, 16, 1, 0, 0, 8, root)); - drain(); - root = mapping('h40030000, 1, 'hc7, 0); - submit(request(11, 'h40030000, 1, 16, 16, 1, 0, 0, 8, root)); - drain(); - root = mapping('h40040000, 1); - preflight_ref_policy(model, 1, (root << 12) + ((64'h40040000 >> 30) & 511) * 8, 0, 0, 0); - submit(request(12, 'h40040000, 1, 16, 16, 1, 0, 0, 8, root)); - drain(); - preflight_ref_policy(model, 0, 0, 1, 'h800000, 'h80000f); - submit(request(13, 'h40040000, 1, 16, 16, 1, 0, 0, 8, root)); - drain(); - preflight_ref_policy(model, 0, 0, 1, 'h71000000, 'h71000003); - // Denied bytes precede the user span: the aligned transport range is still authorized. - submit(request(14, 'h71000004, 1, 4, 4)); - drain(); - preflight_ref_policy(model, 0, 0, 0, 0, 0); - submit(request(15, (64'h1 << `PF_ADDRESS_BITS) - 16, 1, 32, 32)); - drain(); - submit(request(16, 64'hfffffffffffffff8, 1, 16, 16)); - drain(); - submit(request(17, 'h80000000, 0, 16, 16)); - drain(); - q = request(18, 'h80000000, 1, 16, 16); - q[`PF(CMD, MODE)] = 9; - submit(q); - drain(); - root = mapping('h40050000, 1, 'h59); - submit(request(19, 'h40050000, 1, 16, 16, 1, 0, 0, 8, root, 1, 1, 1)); - drain(); - // External snapshot service captures policy at command acceptance, not from later live settings. - hold_permission = 1; - submit(request(20, 'h72000000, 1, 16, 16)); - preflight_ref_policy(model, 0, 0, 1, 'h72000000, 'h7200000f); - wait (auth_pending); - repeat (8) @(ctl.sample); - hold_permission = 0; - drain(); - preflight_ref_policy(model, 0, 0, 0, 0, 0); - submit(request(21, 'h73000000, 1, 16, 256, 16, 16)); - drain(); // columns retains all 32 input bits - q = request(22, 64'hfffffffffffff000, -1, -1, -1, -1, -1); - submit(q); - drain(); - // Write authorization covers actual bytes; read authorization covers transport padding. - prior = failures[4]; - preflight_ref_policy(model, 0, 0, 1, 'h74000000, 'h74000003); - submit(request(23, 'h74000004, 1, 4, 4, 1, 0, 1)); - drain(); - verify_contract(failures[4] == prior, "write was denied for untouched prefix padding"); - submit(request(24, 'h74000004, 1, 4, 4)); - drain(); - verify_contract(failures[4] == prior + 1, "read failed to authorize actual prefix overread"); - prior = failures[4]; - preflight_ref_policy(model, 0, 0, 1, 'h74000108, 'h7400013f); - submit(request(25, 'h74000104, 1, 4, 4, 1, 0, 1)); - drain(); - verify_contract(failures[4] == prior, "write was denied for untouched tail padding"); - submit(request(26, 'h74000104, 1, 4, 4)); - drain(); - verify_contract(failures[4] == prior + 1, "read failed to authorize actual tail overread"); - prior = failures[4]; - preflight_ref_policy(model, 0, 0, 1, 'h74002ff0, 'h74002ffb); - submit(request(27, 'h74002ffc, 1, 8, 8, 1, 0, 1)); - drain(); - verify_contract(failures[4] == prior, "cross-page write included untouched bytes"); - submit(request(28, 'h74002ffc, 1, 8, 8)); - drain(); - verify_contract(failures[4] == prior + 1, "cross-page read omitted padded transport bytes"); - preflight_ref_policy(model, 0, 0, 0, 0, 0); - // MobileNet's gather has nine aligned segments on two noncontiguous PA pages. - // Every same-page gap must be explicitly authorized before the plan retains a union. - root = mapping('h74003e00, 2); - prior = failures[5]; - submit(request(29, 'h74003e00, 1, 16, 4032, 8, 72, 0, 8, root)); - drain(); - verify_contract(failures[5] == prior, "Authorized MobileNet gather exceeded range capacity"); - // Splitting preserves the exact accessed bytes and keeps each list within eight records. - prior = failures[5]; - submit(request(30, 'h74003e00, 1, 16, 4032, 4, 72, 0, 8, root)); - drain(); - submit(request(31, 'h74003f20, 1, 16, 4032, 4, 72, 0, 8, root)); - drain(); - verify_contract(failures[5] == prior, "Split MobileNet gather exceeded range capacity"); - // A denied hole must keep both individually legal spans, rather than poison the command. - gap_stride = `PF_BEAT_BYTES == 16 ? 72 : 2 * `PF_BEAT_BYTES + 8; - preflight_ref_policy(model, 0, 0, 1, 'h800e00 + `PF_BEAT_BYTES, - 'h800e00 + (gap_stride / `PF_BEAT_BYTES) * `PF_BEAT_BYTES - 1); - prior = failures[4]; - submit(request(32, 'h74003e00, 1, 16, 4032, 2, gap_stride, 0, 8, root)); - drain(); - verify_contract(failures[4] == prior, "Denied union rejected individually authorized pixels"); - // Denial of an actual transport byte must still yield one terminal fault, no partial plan. - preflight_ref_policy(model, 0, 0, 1, 'h800e00 + gap_stride, 'h800e00 + gap_stride + 7); - submit(request(33, 'h74003e00, 1, 16, 4032, 2, gap_stride, 0, 8, root)); - drain(); - verify_contract(failures[4] == prior + 1, - "Actual MobileNet pixel bytes escaped authorization"); - preflight_ref_policy(model, 0, 0, 0, 0, 0); - verify_contract( - peak==4&&submitted==retired&&command_stalls>0&&output_stalls>0&&auth_stalls>0, - "Missing concurrency/backpressure/drain coverage"); - for (int error = 1; error < 7; error++) - verify_contract(failures[error] > 0, "Missing typed error scenario"); - verify_contract(scoreboard.checked == segments, "Not every output descriptor was compared"); - `uvm_info( - "PREFLIGHT_PASS", - $sformatf( - "commands=%0d descriptors=%0d peak=%0d PTE=%0d auth=%0d errors shape/overflow/PF/AF/capacity/context=%0d/%0d/%0d/%0d/%0d/%0d stalls cmd/out/auth=%0d/%0d/%0d; all plans/permissions drained", - retired, segments, peak, pte_count, auth_count, failures[1], failures[2], failures[3], - failures[4], failures[5], failures[6], command_stalls, output_stalls, auth_stalls), - UVM_LOW) - endtask - function void final_phase(uvm_phase phase); - preflight_ref_destroy(model); - super.final_phase(phase); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight.f b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight.f deleted file mode 100644 index 23dc0b937..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Preflight/filelist.f -@RESOURCES@/preflight_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight64.f b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight64.f deleted file mode 100644 index d6c9d5270..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight64.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+PREFLIGHT_LINE_READ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/control_if.sv -@RESOURCES@/package.sv --F @RTL@/Preflight64/filelist.f -@RESOURCES@/preflight_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight_tb.sv b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight_tb.sv deleted file mode 100644 index 0453c4e45..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/preflight_tb.sv +++ /dev/null @@ -1,59 +0,0 @@ -`ifdef PREFLIGHT_LINE_READ -`include "preflight64_config.svh" -`define PREFLIGHT_TOP preflight64_tb -`else -`include "preflight_config.svh" -`define PREFLIGHT_TOP preflight_tb -`endif -module `PREFLIGHT_TOP; - import uvm_pkg::*; - import preflight_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - preflight_control_if ctl (clock); - stream_if #(`PF_CMD_WIDTH) cmd ( - clock, - ctl.reset - ); - stream_if #(`PF_OUT_WIDTH) prepared ( - clock, - ctl.reset - ); - stream_if #(`PF_AUTH_WIDTH) authorization ( - clock, - ctl.reset - ); - stream_if #(`PF_PERMIT_WIDTH) permission ( - clock, - ctl.reset - ); - stream_if #(`PF_PTE_WIDTH) pte_req ( - clock, - ctl.reset - ); - stream_if #(`PF_PTERESP_WIDTH) pte_resp ( - clock, - ctl.reset - ); - Preflight dut ( - .clock(clock), - .reset(ctl.reset), - `include "preflight_ports.svh" - ); - initial begin - uvm_config_db#(virtual preflight_control_if)::set(null, "uvm_test_top", "ctl", ctl); - uvm_config_db#(virtual stream_if #(`PF_CMD_WIDTH))::set(null, "uvm_test_top", "cmd", cmd); - uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::set(null, "uvm_test_top", "prepared", - prepared); - uvm_config_db#(virtual stream_if #(`PF_AUTH_WIDTH))::set(null, "uvm_test_top", "authorization", - authorization); - uvm_config_db#(virtual stream_if #(`PF_PERMIT_WIDTH))::set(null, "uvm_test_top", "permission", - permission); - uvm_config_db#(virtual stream_if #(`PF_PTE_WIDTH))::set(null, "uvm_test_top", "pte_req", - pte_req); - uvm_config_db#(virtual stream_if #(`PF_PTERESP_WIDTH))::set(null, "uvm_test_top", "pte_resp", - pte_resp); - run_test("protocol_test"); - end -endmodule -`undef PREFLIGHT_TOP diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map.cm_hier b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map.cm_hier deleted file mode 100644 index f77ff296d..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map.cm_hier +++ /dev/null @@ -1,5 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.reserve -+tree @TOP@.prepared -+tree @TOP@.mapped -+tree @TOP@.retire diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map.f b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map.f deleted file mode 100644 index 2739ee5eb..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/prepared_map_control_if.sv -@RESOURCES@/prepared_map_package.sv --F @RTL@/PreparedMap/filelist.f -@RESOURCES@/prepared_map_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_control_if.sv b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_control_if.sv deleted file mode 100644 index eb7f6d0c7..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_control_if.sv +++ /dev/null @@ -1,12 +0,0 @@ -`include "preflight_config.svh" -interface prepared_map_control_if ( - input logic clock -); - logic reset, hold_ready; - logic [`PF_QUERY_WIDTH-1:0] query_0, query_1; - logic [`PF_RESULT_WIDTH-1:0] result_0, result_1; - clocking sample @(posedge clock); - default input #1step; - input reset, query_0, query_1, result_0, result_1; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_package.sv b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_package.sv deleted file mode 100644 index 85a9280d1..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_package.sv +++ /dev/null @@ -1,336 +0,0 @@ -`include "preflight_config.svh" -`define PF(K, F) `PF_``K``_``F``_OFFSET +: `PF_``K``_``F``_WIDTH -package prepared_map_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - typedef longint unsigned u64; - import "DPI-C" function chandle pmap_ref_create(input int unsigned address_bits); - import "DPI-C" function void pmap_ref_destroy(input chandle model); - import "DPI-C" function void pmap_ref_reset(input chandle model); - import "DPI-C" function void pmap_ref_reserve( - input chandle model, - input int unsigned id - ); - import "DPI-C" function void pmap_ref_prepared( - input chandle model, - input int unsigned id, - input u64 va, - pa, - input int unsigned bytes, - write, - last, - error - ); - import "DPI-C" function int unsigned pmap_ref_ready( - input chandle model, - input int unsigned id, - fire, - output int unsigned error, - output u64 va - ); - import "DPI-C" function void pmap_ref_query( - input chandle model, - input int unsigned valid, - id, - input u64 va, - input int unsigned bytes, - write, - retiring, - retiring_id, - output int unsigned hit, - output u64 pa, - output int unsigned error - ); - import "DPI-C" function void pmap_ref_release( - input chandle model, - input int unsigned id - ); - import "DPI-C" function int unsigned pmap_ref_pending(input chandle model); - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual prepared_map_control_if ctl; - virtual stream_if #(`PF_RESERVE_WIDTH) reserve; - virtual stream_if #(`PF_OUT_WIDTH) prepared; - virtual stream_if #(`PF_MAP_READY_WIDTH) mapped; - virtual stream_if #(`PF_RELEASE_WIDTH) retire; - chandle model; - bit notified[int]; - int status[int]; - int - queries = 0, - hits = 0, - misses = 0, - allocations = 0, - notifications = 0, - retired = 0, - stalls = 0, - dual = 0, - cancelled = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 1ms; - endfunction - function void verify_contract(bit ok, string message); - if (!ok) `uvm_fatal("PREPARED_MAP", message) - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - verify_contract(uvm_config_db#(virtual prepared_map_control_if)::get(this, "", "ctl", ctl), - "ctl missing"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_RESERVE_WIDTH))::get( - this, "", "reserve", reserve), "reserve missing"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::get( - this, "", "prepared", prepared), "prepared missing"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_MAP_READY_WIDTH))::get( - this, "", "mapped", mapped), "mapped missing"); - verify_contract(uvm_config_db#(virtual stream_if #(`PF_RELEASE_WIDTH))::get( - this, "", "retire", retire), "retire missing"); - model = pmap_ref_create(`PF_ADDRESS_BITS); - endfunction - function void query_check(logic [`PF_QUERY_WIDTH-1:0] q, logic [`PF_RESULT_WIDTH-1:0] actual); - int hit, error; - u64 pa; - pmap_ref_query(model, q[`PF(QUERY, VALID)], q[`PF(QUERY, ID)], q[`PF(QUERY, VA)], q[ - `PF(QUERY, BYTES)], q[`PF(QUERY, WRITE)], - retire.sample.valid && retire.sample.ready, retire.sample.bits[`PF(RELEASE, ID) - ], hit, pa, error); - verify_contract(actual[`PF(RESULT, HIT)] === 1'(hit) && actual[`PF(RESULT, PA) - ] === `PF_ADDRESS_BITS'(pa) && actual[`PF(RESULT, ERROR)] === 3'(error), - $sformatf( - "query id%0d VA%h bytes%0d expected hit%0d PA%h error%0d got %h", - q[ - `PF(QUERY, ID) - ], - q[ - `PF(QUERY, VA) - ], - q[ - `PF(QUERY, BYTES) - ], - hit, - pa, - error, - actual - )); - if (q[`PF(QUERY, VALID)]) begin - queries++; - if (hit) hits++; - else misses++; - end - endfunction - task monitor(); - forever begin - @(ctl.sample); - if (ctl.sample.reset) begin - pmap_ref_reset(model); - notified.delete(); - status.delete(); - end else begin - query_check(ctl.sample.query_0, ctl.sample.result_0); - query_check(ctl.sample.query_1, ctl.sample.result_1); - if (ctl.sample.query_0[`PF(QUERY, VALID)] && ctl.sample.query_1[`PF(QUERY, VALID)]) - dual++; - if (mapped.sample.valid) begin - int error, id; - u64 va; - id = mapped.sample.bits[`PF(MAP_READY, ID)]; - verify_contract(pmap_ref_ready(model, id, mapped.sample.ready, error, va) == 1, - "unexpected notification"); - verify_contract(mapped.sample.bits[`PF(MAP_READY, ERROR) - ] === 3'(error) && mapped.sample.bits[`PF(MAP_READY, VA)] === va, - "notification mismatch"); - if (mapped.sample.ready) begin - notified[id] = 1; - status[id] = error; - notifications++; - end else stalls++; - end - if (retire.sample.valid && retire.sample.ready) begin - int id; - id = retire.sample.bits[`PF(RELEASE, ID)]; - pmap_ref_release(model, id); - notified.delete(id); - retired++; - end - if (reserve.sample.valid && reserve.sample.ready) begin - pmap_ref_reserve(model, reserve.sample.bits[`PF(RESERVE, ID)]); - allocations++; - end - if (prepared.sample.valid && prepared.sample.ready) - pmap_ref_prepared(model, prepared.sample.bits[`PF(OUT, ID)], prepared.sample.bits[ - `PF(OUT, VA)], prepared.sample.bits[`PF(OUT, PA)], - prepared.sample.bits[`PF(OUT, BYTES)], prepared.sample.bits[ - `PF(OUT, WRITE)], prepared.sample.bits[`PF(OUT, LAST)], - prepared.sample.bits[`PF(OUT, ERROR)]); - end - end - endtask - task reserve_tag(int id); - @(negedge ctl.clock); - reserve.bits = '0; - reserve.bits[`PF(RESERVE, ID)] = id; - reserve.valid = 1; - do @(ctl.sample); while (!reserve.sample.ready); - @(negedge ctl.clock); - reserve.valid = 0; - endtask - task record(int id, u64 va, u64 pa, int bytes, bit write, bit last, int error = 0); - @(negedge ctl.clock); - prepared.bits = '0; - prepared.bits[`PF(OUT, ID)] = id; - prepared.bits[`PF(OUT, VA)] = va; - prepared.bits[`PF(OUT, PA)] = pa; - prepared.bits[`PF(OUT, BYTES)] = bytes; - prepared.bits[`PF(OUT, WRITE)] = write; - prepared.bits[`PF(OUT, LAST)] = last; - prepared.bits[`PF(OUT, ERROR)] = error; - prepared.valid = 1; - do @(ctl.sample); while (!prepared.sample.ready); - @(negedge ctl.clock); - prepared.valid = 0; - endtask - function logic [`PF_QUERY_WIDTH-1:0] query(int id, u64 va, int bytes, bit write); - logic [`PF_QUERY_WIDTH-1:0] q = '0; - q[`PF(QUERY, VALID)] = 1; - q[`PF(QUERY, ID)] = id; - q[`PF(QUERY, VA)] = va; - q[`PF(QUERY, BYTES)] = bytes; - q[`PF(QUERY, WRITE)] = write; - return q; - endfunction - task probes(logic [`PF_QUERY_WIDTH-1:0] a, logic [`PF_QUERY_WIDTH-1:0] b, int clocks = 3); - @(negedge ctl.clock); - ctl.query_0 = a; - ctl.query_1 = b; - repeat (clocks) @(ctl.sample); - @(negedge ctl.clock); - ctl.query_0 = '0; - ctl.query_1 = '0; - endtask - task wait_ready(int id, int error = 0); - while (!notified.exists(id)) @(negedge ctl.clock); - verify_contract(status[id] == error, "wrong terminal status"); - endtask - task retire_tag(int id, bit probe_release = 0); - @(negedge ctl.clock); - retire.bits = '0; - retire.bits[`PF(RELEASE, ID)] = id; - retire.valid = 1; - if (probe_release) begin - ctl.query_0 = query(id, 'h1010, 8, 0); - ctl.query_1 = query(2, 'h4004, 8, 1); - end - do @(ctl.sample); while (!retire.sample.ready); - @(negedge ctl.clock); - retire.valid = 0; - ctl.query_0 = '0; - ctl.query_1 = '0; - endtask - task execute(); - ctl.reset = 1; - ctl.hold_ready = 0; - ctl.query_0 = '0; - ctl.query_1 = '0; - reserve.valid = 0; - reserve.bits = '0; - prepared.valid = 0; - prepared.bits = '0; - retire.valid = 0; - retire.bits = '0; - fork - monitor(); - join_none - repeat (3) @(negedge ctl.clock); - ctl.reset = 0; - reserve_tag(1); - record(1, 'h1000, 'h8000, 4096, 0, 0); - probes(query(1, 'h1010, 16, 0), query(1, 'h1010, 16, 1)); - @(negedge ctl.clock); - ctl.hold_ready = 1; - record(1, 'h2000, 'hc000, 4096, 0, 1); - probes(query(1, 'h1010, 16, 0), query(1, 'h2010, 16, 0), 8); - @(negedge ctl.clock); - ctl.hold_ready = 0; - wait_ready(1); - probes(query(1, 'h1ffe, 2, 0), query(1, 'h2ffe, 2, 0)); - probes(query(1, 'h1fff, 2, 0), query(1, 'h3000, 4, 0)); - probes(query(1, 'h1000, 16, 1), query(1, 64'hffffffffffffffff, 2, 0)); - probes(query(1, 'h1000, 0, 0), query(99, 'h1000, 8, 0)); - reserve_tag(2); - record(2, 'h4000, 'h9000, 64, 1, 0); - record(2, 64'hffffffffffffffe0, (64'd1 << `PF_ADDRESS_BITS) - 32, 32, 1, 1); - wait_ready(2); - probes(query(1, 'h1f00, 16, 0), query(2, 'h4004, 16, 1)); - probes(query(2, 64'hfffffffffffffff8, 8, 1), query(2, 64'hfffffffffffffff8, 9, 1)); - retire_tag(1, 1); - probes(query(1, 'h1010, 16, 0), query(2, 'h4004, 16, 1)); - reserve_tag(1); - record(1, 'h6000, 'ha000, 128, 0, 1); - wait_ready(1); - probes(query(1, 'h1010, 16, 0), query(1, 'h6040, 64, 0)); - reserve_tag(3); - record(3, 'h7000, 'hb000, 64, 0, 0); - record(3, 'h8000, 0, 0, 0, 1, 3); - wait_ready(3, 3); - probes(query(3, 'h7004, 8, 0), query(3, 'h8000, 8, 0)); - reserve_tag(4); - fork - reserve_tag(5); - begin - repeat (6) begin - @(negedge ctl.clock); - verify_contract(!reserve.ready, "full table accepted fifth reserve"); - end - record(4, 'h9000, 0, 0, 0, 1, 4); - wait_ready(4, 4); - retire_tag(4); - end - join - for (int i = 0; i < 8; i++) begin - record(5, 'h10000 + i * 4096, 'h30000 + (7 - i) * 4096, 32, 0, i == 7); - end - wait_ready(5); - probes(query(5, 'h17018, 8, 0), query(5, 'h10008, 8, 0)); - retire_tag(1); - retire_tag(2); - retire_tag(3); - retire_tag(5); - // Retain an explicitly authorized same-page union and a discontiguous next PA page. - reserve_tag(10); - record(10, 'h74003e00, 'h800e00, 512, 0, 0); - record(10, 'h74004000, 'h802000, 16, 0, 1); - wait_ready(10); - probes(query(10, 'h74003e10, 16, 0), query(10, 'h74004000, 16, 0)); - probes(query(10, 'h74003df0, 16, 0), query(10, 'h74004010, 8, 0)); - probes(query(10, 'h74003ff8, 16, 0), query(10, 'h74003e48, 16, 1)); - retire_tag(10); - reserve_tag(9); - record(9, 'h9000, 'h19000, 64, 0, 1); - // Reset cancels a terminal notification still under backpressure. - @(negedge ctl.clock); - ctl.hold_ready = 1; - repeat (2) @(negedge ctl.clock); - cancelled += pmap_ref_pending(model); - ctl.reset = 1; - repeat (3) @(negedge ctl.clock); - ctl.reset = 0; - ctl.hold_ready = 0; - probes(query(9, 'h9000, 8, 0), query(1, 'h6000, 8, 0)); - repeat (3) @(negedge ctl.clock); - verify_contract(pmap_ref_pending(model) == 0, "mapping remained after retirement/reset"); - verify_contract(hits > 0 && misses > 0 && dual > 0 && stalls >= 4 && cancelled == 1, - "required checks did not execute"); - `uvm_info( - "PREPARED_MAP", - $sformatf( - "Checked %0d queries, %0d hits, %0d misses; reserve=%0d ready=%0d retire=%0d stalls=%0d dual=%0d reset_cancel=%0d", - queries, hits, misses, allocations, notifications, retired, stalls, dual, cancelled), - UVM_LOW) - endtask - function void final_phase(uvm_phase phase); - pmap_ref_destroy(model); - super.final_phase(phase); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_tb.sv b/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_tb.sv deleted file mode 100644 index ab5d4426c..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/resources/prepared_map_tb.sv +++ /dev/null @@ -1,42 +0,0 @@ -`include "preflight_config.svh" -module prepared_map_tb; - import uvm_pkg::*; - import prepared_map_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - prepared_map_control_if ctl (clock); - stream_if #(`PF_RESERVE_WIDTH) reserve ( - clock, - ctl.reset - ); - stream_if #(`PF_OUT_WIDTH) prepared ( - clock, - ctl.reset - ); - stream_if #(`PF_MAP_READY_WIDTH) mapped ( - clock, - ctl.reset - ); - stream_if #(`PF_RELEASE_WIDTH) retire ( - clock, - ctl.reset - ); - assign mapped.ready = !ctl.reset && !ctl.hold_ready; - PreparedMap dut ( - .clock(clock), - .reset(ctl.reset), - `include "prepared_map_ports.svh" - ); - initial begin - uvm_config_db#(virtual prepared_map_control_if)::set(null, "uvm_test_top", "ctl", ctl); - uvm_config_db#(virtual stream_if #(`PF_RESERVE_WIDTH))::set(null, "uvm_test_top", "reserve", - reserve); - uvm_config_db#(virtual stream_if #(`PF_OUT_WIDTH))::set(null, "uvm_test_top", "prepared", - prepared); - uvm_config_db#(virtual stream_if #(`PF_MAP_READY_WIDTH))::set(null, "uvm_test_top", "mapped", - mapped); - uvm_config_db#(virtual stream_if #(`PF_RELEASE_WIDTH))::set(null, "uvm_test_top", "retire", - retire); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Emit.scala deleted file mode 100644 index d09224e96..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Emit.scala +++ /dev/null @@ -1,94 +0,0 @@ -package memcore.memory.preflight - -import chisel3._ -import java.nio.file.{Files, Paths} -import memcore.bus.chi.rnf.{CacheAccess, CacheResult} - -object Emit extends App { - val p = Params() - Files.createDirectories(Paths.get("build")) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Preflight(p), - firtoolOpts = args, - args = Array("--target-dir", "build/Preflight", "--split-verilog") - ) - - val defs = scala.collection.mutable.ArrayBuffer[String]( - s"`define PF_ADDRESS_BITS ${p.bus.addressBits}", - s"`define PF_BEAT_BYTES ${p.beatBytes}" - ) - - val ports = scala.collection.mutable.ArrayBuffer[String]() - - def channel( - kind: String, - prefix: String, - signal: String, - bundle: Bundle - ): Unit = { - var offset = 0 - for ((name, field) <- bundle.elements.toSeq) { - val width = field.getWidth - defs += s"`define PF_${kind}_${name.toUpperCase}_OFFSET $offset" - defs += s"`define PF_${kind}_${name.toUpperCase}_WIDTH $width" - // firtool drops zero-width fields from the module ports. - if (width > 0) ports += s".${prefix}_bits_$name($signal.bits[$offset +: $width])" - offset += width - } - defs += s"`define PF_${kind}_WIDTH $offset" - ports += s".${prefix}_valid($signal.valid)"; ports += s".${prefix}_ready($signal.ready)" - } - - channel("CMD", "io_command", "cmd", new Command(p)) - channel("OUT", "io_prepared", "prepared", new PreparedSegment(p)) - channel("AUTH", "io_authorization", "authorization", new Authorization(p)) - channel("PERMIT", "io_permission", "permission", new Permission(p)) - channel("PTE", "io_pteRequest", "pte_req", new CacheAccess(p.bus)) - channel("PTERESP", "io_pteResponse", "pte_resp", new CacheResult) - Files.writeString(Paths.get("build/preflight_config.svh"), defs.mkString("\n") + "\n") - Files.writeString(Paths.get("build/preflight_ports.svh"), ports.mkString(",\n") + "\n") - val lineRead = p.copy(beatBytes = 64) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Preflight(lineRead), - firtoolOpts = args, - args = Array("--target-dir", "build/Preflight64", "--split-verilog") - ) - Files.writeString( - Paths.get("build/preflight64_config.svh"), - defs.map(d => if (d.startsWith("`define PF_BEAT_BYTES ")) s"`define PF_BEAT_BYTES ${lineRead.beatBytes}" else d) - .mkString("\n") + "\n" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new PreparedMap(p), - firtoolOpts = args, - args = Array("--target-dir", "build/PreparedMap", "--split-verilog") - ) - ports.clear() - channel("RESERVE", "io_reserve", "reserve", new MapTag(p)) - channel("MAP_READY", "io_ready", "mapped", new MapReady(p)) - channel("RELEASE", "io_release", "retire", new MapTag(p)) - // PreparedSegment uses the existing PF_OUT layout. - var preparedOffset = 0 - for ((name, field) <- (new PreparedSegment(p)).elements.toSeq) { - ports += s".io_prepared_bits_$name(prepared.bits[$preparedOffset +: ${field.getWidth}])" - preparedOffset += field.getWidth - } - ports += ".io_prepared_valid(prepared.valid)"; ports += ".io_prepared_ready(prepared.ready)" - for ( - (kind, prefix, signal, bundle) <- Seq( - ("QUERY", "io_queries", "query", new MapQuery(p)), - ("RESULT", "io_results", "result", new MapResult(p)) - ) - ) { - var offset = 0 - for ((name, field) <- bundle.elements.toSeq) { - defs += s"`define PF_${kind}_${name.toUpperCase}_OFFSET $offset" - defs += s"`define PF_${kind}_${name.toUpperCase}_WIDTH ${field.getWidth}" - for (port <- 0 until 2) ports += s".${prefix}_${port}_$name(ctl.${signal}_$port[$offset +: ${field.getWidth}])" - offset += field.getWidth - } - defs += s"`define PF_${kind}_WIDTH $offset" - } - Files.writeString(Paths.get("build/preflight_config.svh"), defs.mkString("\n") + "\n") - Files.writeString(Paths.get("build/prepared_map_ports.svh"), ports.mkString(",\n") + "\n") -} diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Interface.scala b/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Interface.scala deleted file mode 100644 index c2ec8605d..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Interface.scala +++ /dev/null @@ -1,50 +0,0 @@ -package memcore.memory.preflight - -import chisel3._ - -/** Shape describes transport bytes, including any read beyond the user's useful payload. */ -class Command(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val baseVA = UInt(64.W) - val rows = UInt(32.W) - val columns = UInt(32.W) - val spanBytes = UInt(32.W) - val columnStride = UInt(32.W) - val rowStride = UInt(32.W) - val write = Bool() - val mode = UInt(4.W) - val rootPpn = UInt(44.W) - val privilege = UInt(2.W) - val sum = Bool() - val mxr = Bool() -} - -object Error { - val Ok = 0; val Shape = 1; val Overflow = 2; val PageFault = 3 - val AccessFault = 4; val Capacity = 5; val Context = 6 -} - -/** Failure returns one terminal zero-PA record. The caller retains successful maps and IDs until DMA retirement. */ -class PreparedSegment(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val va = UInt(64.W) - val pa = UInt(p.bus.addressBits.W) - val bytes = UInt(13.W) - val write = Bool() - val last = Bool() - val error = UInt(3.W) -} - -class Authorization(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val pa = UInt(p.bus.addressBits.W) - val bytes = UInt(13.W) - val write = Bool() - val isPte = Bool() - val privilege = UInt(2.W) -} - -class Permission(p: Params) extends Bundle { - val id = UInt(p.idBits.W) - val allow = Bool() -} diff --git a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Params.scala b/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Params.scala deleted file mode 100644 index 8651b8464..000000000 --- a/arch/src/main/scala/framework/mem-core/preflight/src/main/scala/Params.scala +++ /dev/null @@ -1,15 +0,0 @@ -package memcore.memory.preflight - -import memcore.bus.chi - -case class Params( - bus: chi.Params = chi.Params(), - contexts: Int = 4, - maxRanges: Int = 8, - idBits: Int = 8, - beatBytes: Int = 16) { - require(contexts >= 2 && (contexts & (contexts - 1)) == 0) - require(maxRanges == 8) - require(idBits > 0 && bus.addressBits >= 12 && bus.addressBits < 64) - require(beatBytes >= 8 && beatBytes <= 64 && (beatBytes & (beatBytes - 1)) == 0) -} diff --git a/arch/src/main/scala/framework/mem-core/queue/build.sc b/arch/src/main/scala/framework/mem-core/queue/build.sc deleted file mode 100644 index 3fffcdc99..000000000 --- a/arch/src/main/scala/framework/mem-core/queue/build.sc +++ /dev/null @@ -1,10 +0,0 @@ -import mill._ -import mill.scalalib._ - -object queue extends SbtModule { - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} diff --git a/arch/src/main/scala/framework/mem-core/spm/src/main/resources/interface.sv b/arch/src/main/scala/framework/mem-core/spm/src/main/resources/interface.sv deleted file mode 100644 index b00b96c00..000000000 --- a/arch/src/main/scala/framework/mem-core/spm/src/main/resources/interface.sv +++ /dev/null @@ -1,81 +0,0 @@ -interface spm_if ( - input logic clock, - reset -); - import uvm_pkg::*; - import ip_pkg::*; - import spm_pkg::*; - `include "uvm_macros.svh" - logic valid = 0, ready, write = 0; - logic [ 63:0] address = 0; - logic [ 2:0] size = 0; - logic [127:0] data = 0; - logic [ 15:0] mask = 0; - logic response_valid, response_ready = 0, error; - logic [127:0] result; - bit cancel = 0, read_only = 0; - chandle model; - in_order_scoreboard #(response_item) scoreboard; - bit enabled = 0, held = 0; - logic [128:0] held_response; - - always @(posedge clock) - if (enabled) begin - if (reset || cancel) begin - scoreboard.cancel_pending(); - held = 0; - if (response_valid) `uvm_fatal("CANCEL", "response survived reset/cancel") - end else begin - if (held && (!response_valid || {error, result} !== held_response)) - `uvm_fatal("HOLD", "response changed under backpressure") - held = response_valid && !response_ready; - held_response = {error, result}; - if (valid && ready) begin - response_item item; - longint unsigned low, high; - int status; - item = new; - status = spm_access(model, address, size, write, mask, data[63:0], data[127:64], - read_only, low, high); - if (status == 2) `uvm_fatal("REFERENCE", "read before explicit SRAM initialization") - item.data = {high, low}; - item.error = status != 0; - scoreboard.write_expected(item); - end - if (response_valid && response_ready) begin - response_item item; - item = new; - item.data = result; - item.error = error; - scoreboard.write_actual(item); - end - end - end - task send(input logic [63:0] addr, input logic [2:0] width, input bit wr, - input logic [127:0] value = 0, input logic [15:0] keep = 0); - @(negedge clock); - address = addr; - size = width; - write = wr; - data = value; - mask = keep; - valid = 1; - do @(posedge clock); while (!ready); - @(negedge clock); - valid = 0; - endtask - task take(output logic [127:0] value); - @(negedge clock); - response_ready = 1; - do @(posedge clock); while (!response_valid); - value = result; - @(negedge clock); - response_ready = 0; - endtask - task access (input logic [63:0] addr, input logic [2:0] width, input bit wr, - input logic [127:0] value = 0, input logic [15:0] keep = 0); - logic [127:0] ignored; - send(addr, width, wr, value, keep); - take(ignored); - endtask -endinterface diff --git a/arch/src/main/scala/framework/mem-core/spm/src/main/resources/package.sv b/arch/src/main/scala/framework/mem-core/spm/src/main/resources/package.sv deleted file mode 100644 index 32e9788e8..000000000 --- a/arch/src/main/scala/framework/mem-core/spm/src/main/resources/package.sv +++ /dev/null @@ -1,33 +0,0 @@ -package spm_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function chandle spm_create( - input longint unsigned base, - input int unsigned bytes - ); - import "DPI-C" function void spm_destroy(input chandle model); - import "DPI-C" function int unsigned spm_access( - input chandle model, - input longint unsigned address, - input int unsigned size, - input byte unsigned write, - input int unsigned mask, - input longint unsigned low, - high, - input byte unsigned read_only, - output longint unsigned out_low, - out_high - ); - class response_item extends uvm_sequence_item; - logic [127:0] data; - logic error; - `uvm_object_utils_begin(response_item) - `uvm_field_int(data, UVM_ALL_ON) - `uvm_field_int(error, UVM_ALL_ON) - `uvm_object_utils_end - function new(string name = "response_item"); - super.new(name); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/mem-core/spm/src/main/scala/Interfaces.scala b/arch/src/main/scala/framework/mem-core/spm/src/main/scala/Interfaces.scala deleted file mode 100644 index f9c6c7531..000000000 --- a/arch/src/main/scala/framework/mem-core/spm/src/main/scala/Interfaces.scala +++ /dev/null @@ -1,35 +0,0 @@ -package memcore.memory.spm - -import chisel3._ -import chisel3.util._ -import memcore.memory.bank.BankSetParams - -/** One local byte-address window. This does not describe a DDR mapping. */ -case class Params(base: BigInt, bytes: Int, dataBits: Int = 128) { - require(dataBits >= 64 && dataBits % 8 == 0 && isPow2(dataBits / 8)) - val beatBytes = dataBits / 8 - require(base >= 0 && base % beatBytes == 0 && base + bytes <= (BigInt(1) << 64)) - require(bytes >= 2 * beatBytes && isPow2(bytes)) - val laneBits = log2Ceil(beatBytes) - val sizeBits = log2Ceil(laneBits + 2) - val bank = BankSetParams(dataBits, banks = 1, entriesPerBank = bytes / beatBytes) -} - -/** Naturally aligned 1..beatBytes access. Data and mask start at address, not the SRAM row. */ -class Request(p: Params) extends Bundle { - val address = UInt(64.W) - val size = UInt(p.sizeBits.W) - val write = Bool() - val data = UInt(p.dataBits.W) - val mask = UInt(p.beatBytes.W) -} - -class Response(p: Params) extends Bundle { - val data = UInt(p.dataBits.W) - val error = Bool() -} - -class Port(p: Params) extends Bundle { - val request = Decoupled(new Request(p)) - val response = Flipped(Decoupled(new Response(p))) -} diff --git a/arch/src/main/scala/framework/mem-core/spm/src/main/scala/Memory.scala b/arch/src/main/scala/framework/mem-core/spm/src/main/scala/Memory.scala deleted file mode 100644 index 3e90a98fa..000000000 --- a/arch/src/main/scala/framework/mem-core/spm/src/main/scala/Memory.scala +++ /dev/null @@ -1,59 +0,0 @@ -package memcore.memory.spm - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.memory.bank.Bank - -/** - * One outstanding transaction on one physical read/write port. Writes acknowledge after - * SRAM acceptance; reads return after the synchronous read. Responses hold until accepted. - * Reset cancels protocol state; SRAM contents, including already accepted writes, remain. - */ -@instantiable -class Memory(p: Params) extends Module { - - @public val io = IO(new Bundle { - val port = Flipped(new Port(p)) - val readOnly = Input(Bool()) - val busy = Output(Bool()) - }) - - val bank = Instantiate(new Bank(p.bank)) - val idle :: memory :: rejected :: Nil = Enum(3) - val state = RegInit(idle) - val offset = Reg(UInt(p.laneBits.W)) - val savedMask = Reg(UInt(p.beatBytes.W)) - val request = io.port.request.bits - val count = 1.U(64.W) << request.size - val end = request.address +& count - - val widthMask = MuxLookup(request.size, 0.U(p.beatBytes.W))( - (0 to p.laneBits).map(size => size.U -> ((BigInt(1) << (1 << size)) - 1).U(p.beatBytes.W)) - ) - - val legal = request.size <= p.laneBits.U && (request.address & (count - 1.U)) === 0.U && - request.address >= p.base.U && end <= (p.base + p.bytes).U && - !(request.write && (io.readOnly || (request.mask & ~widthMask) =/= 0.U)) - - val lane = request.address(p.laneBits - 1, 0) - io.port.request.ready := state === idle && !reset.asBool && Mux(legal, bank.io.request.ready, true.B) - io.busy := state =/= idle - bank.io.request.valid := state === idle && !reset.asBool && io.port.request.valid && legal - bank.io.request.bits.addr := (request.address - p.base.U) >> p.laneBits - bank.io.request.bits.write := request.write - bank.io.request.bits.data := request.data << (lane << 3) - bank.io.request.bits.mask := request.mask << lane - bank.io.request.bits.tag := 0.U - when(io.port.request.fire) { - offset := lane - savedMask := widthMask - state := Mux(legal, memory, rejected) - } - val dataMask = Cat((0 until p.beatBytes).reverse.map(i => Fill(8, savedMask(i)))) - io.port.response.valid := !reset.asBool && (state === rejected || (state === memory && bank.io.response.valid)) - io.port.response.bits.data := Mux(state === rejected, 0.U, (bank.io.response.bits.data >> (offset << 3)) & dataMask) - io.port.response.bits.error := state === rejected || bank.io.response.bits.error - bank.io.response.ready := state === memory && io.port.response.ready && !reset.asBool - when(io.port.response.fire)(state := idle) -} diff --git a/arch/src/main/scala/framework/mem-core/tls/src/csrc/Cargo.lock b/arch/src/main/scala/framework/mem-core/tls/src/csrc/Cargo.lock deleted file mode 100644 index 73011b8a8..000000000 --- a/arch/src/main/scala/framework/mem-core/tls/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "tls_ref" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/mem-core/tls/src/csrc/Cargo.toml b/arch/src/main/scala/framework/mem-core/tls/src/csrc/Cargo.toml deleted file mode 100644 index 4591b837f..000000000 --- a/arch/src/main/scala/framework/mem-core/tls/src/csrc/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "tls_ref" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib"] -path = "../../../spm/src/csrc/src/lib.rs" diff --git a/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls.f b/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls.f deleted file mode 100644 index 13249c547..000000000 --- a/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls.f +++ /dev/null @@ -1,10 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv -@RESOURCES@/../../../../spm/src/main/resources/package.sv -@RESOURCES@/../../../../spm/src/main/resources/interface.sv -+incdir+@RTL@ --F @RTL@/Store/filelist.f -@RESOURCES@/tls_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls_tb.sv b/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls_tb.sv deleted file mode 100644 index f1064cb1b..000000000 --- a/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls_tb.sv +++ /dev/null @@ -1,121 +0,0 @@ -`include "config.svh" -module tls_tb; - import uvm_pkg::*; - import ip_pkg::*; - import ip_control_test_pkg::*; - import spm_pkg::*; - `include "uvm_macros.svh" - logic clock = 0; - always #5 clock = ~clock; - logic [1:0] reset = '1, active = '0; - ip_control_if control (clock); - virtual spm_if ports[4]; - chandle models[2]; - for (genvar i = 0; i < 4; i++) begin : endpoints - spm_if bus ( - clock, - reset[i/2] - ); - initial ports[i] = bus; - end - for (genvar tile = 0; tile < 2; tile++) begin : tiles - Store dut ( - .clock(clock), - .reset(reset[tile]), - .io_running(active[tile]), - .io_cancel(endpoints[tile*2+1].bus.cancel), - .io_busy(), - .io_host_request_valid(endpoints[tile*2+0].bus.valid), - .io_host_request_ready(endpoints[tile*2+0].bus.ready), - .io_host_request_bits_address(endpoints[tile*2+0].bus.address), - .io_host_request_bits_size(endpoints[tile*2+0].bus.size), - .io_host_request_bits_write(endpoints[tile*2+0].bus.write), - .io_host_request_bits_data(endpoints[tile*2+0].bus.data), - .io_host_request_bits_mask(endpoints[tile*2+0].bus.mask), - .io_host_response_valid(endpoints[tile*2+0].bus.response_valid), - .io_host_response_ready(endpoints[tile*2+0].bus.response_ready), - .io_host_response_bits_data(endpoints[tile*2+0].bus.result), - .io_host_response_bits_error(endpoints[tile*2+0].bus.error), - .io_local_request_valid(endpoints[tile*2+1].bus.valid), - .io_local_request_ready(endpoints[tile*2+1].bus.ready), - .io_local_request_bits_address(endpoints[tile*2+1].bus.address), - .io_local_request_bits_size(endpoints[tile*2+1].bus.size), - .io_local_request_bits_write(endpoints[tile*2+1].bus.write), - .io_local_request_bits_data(endpoints[tile*2+1].bus.data), - .io_local_request_bits_mask(endpoints[tile*2+1].bus.mask), - .io_local_response_valid(endpoints[tile*2+1].bus.response_valid), - .io_local_response_ready(endpoints[tile*2+1].bus.response_ready), - .io_local_response_bits_data(endpoints[tile*2+1].bus.result), - .io_local_response_bits_error(endpoints[tile*2+1].bus.error) - ); - end - initial begin - foreach (models[tile]) models[tile] = spm_create(`LOCAL_BASE, `LOCAL_BYTES); - #0; - foreach (ports[i]) begin - ports[i].model = models[i/2]; - ports[i].scoreboard = new($sformatf("port_%0d", i), null); - ports[i].enabled = 1; - end - uvm_config_db#(virtual ip_control_if)::set(null, "uvm_test_top", "vif", control); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - final foreach (models[i]) spm_destroy(models[i]); - initial begin : exercise - logic [127:0] value; - wait (control.start); - repeat (3) @(negedge clock); - reset = 0; - for (int row = 0; row < `LOCAL_BYTES; row += 16) begin - ports[0].access(`LOCAL_BASE + row, 4, 1, 0, '1); - ports[2].access(`LOCAL_BASE + row, 4, 1, '1, '1); - end - @(negedge clock); - active = 3; - ports[1].access(`LOCAL_BASE, 3, 1, 64'h123456789abcdef0, 16'hff); - ports[3].access(`LOCAL_BASE, 4, 0); - ports[1].access(`LOCAL_BASE + 15, 0, 1, 8'ha5, 1); - ports[1].access(`LOCAL_BASE, 4, 0); - ports[1].send(`LOCAL_BASE, 4, 0); - wait (ports[1].response_valid); - repeat (5) @(negedge clock); - ports[3].access(`LOCAL_BASE + 16, 4, 0); - ports[1].take(value); - ports[1].access(`LOCAL_BASE + 3, 2, 0); - ports[1].access(`LOCAL_BASE + `LOCAL_BYTES, 3, 0); - ports[1].access(64'h80000000, 3, 0); - ports[1].access(`LOCAL_BASE, 5, 0); - ports[1].access(`LOCAL_BASE + 8, 3, 1, 64'h1122334455667788, 16'ha5); - ports[1].access(`LOCAL_BASE + 8, 3, 0); - ports[1].access((64'h1 << 32) | `LOCAL_BASE, 3, 0); - ports[1].access(64'hfffffffffffffff8, 3, 0); - ports[1].access(`LOCAL_BASE, 0, 1, '1, 3); - ports[1].send(`LOCAL_BASE, 3, 0); - ports[1].cancel = 1; - @(negedge clock); - ports[1].cancel = 0; - repeat (5) begin - @(negedge clock); - if (ports[1].response_valid) `uvm_fatal("CANCEL", "old TLS response survived cancellation") - end - ports[1].access(`LOCAL_BASE, 3, 0); - ports[1].send(`LOCAL_BASE + 16, 0, 1, 8'h5a, 1); - wait (ports[1].response_valid); - @(negedge clock); - reset[0] = 1; - active[0] = 0; - repeat (2) @(negedge clock); - reset[0] = 0; - ports[0].access(`LOCAL_BASE + 16, 0, 0); - ports[3].access(`LOCAL_BASE + 16, 0, 0); - repeat (2) @(negedge clock); - foreach (ports[i]) begin - if (ports[i].scoreboard.expected_queue.size() || ports[i].scoreboard.actual_queue.size()) - `uvm_fatal("DRAIN", "local storage still has unmatched accesses") - `uvm_info("CHECKED", $sformatf("port %0d: %0d comparisons", i, ports[i].scoreboard.checked), - UVM_LOW) - end - control.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/tls/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/tls/src/main/scala/Emit.scala deleted file mode 100644 index 721fc9b24..000000000 --- a/arch/src/main/scala/framework/mem-core/tls/src/main/scala/Emit.scala +++ /dev/null @@ -1,18 +0,0 @@ -package memcore.memory.tls - -import java.nio.file.{Files, Paths} - -object Emit extends App { - val memory = memcore.memory.spm.Params(base = BigInt("10000", 16), bytes = 64) - val output = Paths.get("src/main/scala/framework/mem-core/tls/build") - Files.createDirectories(output) - Files.writeString( - output.resolve("config.svh"), - s"`define LOCAL_BASE 64'h${memory.base.toString(16)}\n`define LOCAL_BYTES ${memory.bytes}\n" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Store(memory), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Store").toString, "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/mem-core/tls/src/main/scala/Store.scala b/arch/src/main/scala/framework/mem-core/tls/src/main/scala/Store.scala deleted file mode 100644 index 0e03879ba..000000000 --- a/arch/src/main/scala/framework/mem-core/tls/src/main/scala/Store.scala +++ /dev/null @@ -1,48 +0,0 @@ -package memcore.memory.tls - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.memory.spm.{Memory, Params, Port} - -/** - * A context-private local store. Loader and execution access it in separate ownership phases. - * Cancellation drains an accepted local access and discards its response before reuse. - */ -@instantiable -class Store(p: Params, localReadOnly: Boolean = false) extends Module { - - @public val io = IO(new Bundle { - val running = Input(Bool()) - val cancel = Input(Bool()) - val host = Flipped(new Port(p)) - val local = Flipped(new Port(p)) - val busy = Output(Bool()) - }) - - val memory = Instantiate(new Memory(p)) - val busy = RegInit(false.B) - val owner = Reg(Bool()) - val cancelled = RegInit(false.B) - val discard = owner && (cancelled || io.cancel) - memory.io.port.request.valid := !busy && !reset.asBool && - Mux(io.running, io.local.request.valid && !io.cancel, io.host.request.valid) - memory.io.port.request.bits := Mux(io.running, io.local.request.bits, io.host.request.bits) - memory.io.readOnly := io.running && localReadOnly.B - io.local.request.ready := !busy && io.running && !io.cancel && memory.io.port.request.ready - io.host.request.ready := !busy && !io.running && memory.io.port.request.ready - when(memory.io.port.request.fire) { - busy := true.B - owner := io.running - cancelled := false.B - } - when(busy && owner && io.cancel)(cancelled := true.B) - io.local.response.valid := busy && owner && !discard && memory.io.port.response.valid - io.host.response.valid := busy && !owner && memory.io.port.response.valid - io.local.response.bits := memory.io.port.response.bits - io.host.response.bits := memory.io.port.response.bits - memory.io.port.response.ready := busy && - (discard || Mux(owner, io.local.response.ready, io.host.response.ready)) - when(memory.io.port.response.fire) { busy := false.B; cancelled := false.B } - io.busy := busy -} diff --git a/arch/src/main/scala/framework/mem-core/tss/src/csrc/Cargo.lock b/arch/src/main/scala/framework/mem-core/tss/src/csrc/Cargo.lock deleted file mode 100644 index fdcc5b948..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "tss_ref" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/mem-core/tss/src/csrc/Cargo.toml b/arch/src/main/scala/framework/mem-core/tss/src/csrc/Cargo.toml deleted file mode 100644 index 146688e0a..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/csrc/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "tss_ref" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib"] -path = "../../../spm/src/csrc/src/lib.rs" diff --git a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.cm_hier b/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.cm_hier deleted file mode 100644 index 80afe3c28..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.cm_hier +++ /dev/null @@ -1,2 +0,0 @@ -+tree @TOP@.tiles[0].dut -+tree @TOP@.tiles[1].dut diff --git a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.f b/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.f deleted file mode 100644 index 7459e2657..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.f +++ /dev/null @@ -1,10 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv -@RESOURCES@/../../../../spm/src/main/resources/package.sv -@RESOURCES@/../../../../spm/src/main/resources/interface.sv -+incdir+@RTL@ --F @RTL@/Store/filelist.f -@RESOURCES@/tss_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.toml b/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.toml deleted file mode 100644 index c18e5753d..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss.toml +++ /dev/null @@ -1 +0,0 @@ -rtl_scopes = ["tiles[0].dut", "tiles[1].dut"] diff --git a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss_tb.sv b/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss_tb.sv deleted file mode 100644 index 4d895911e..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/main/resources/tss_tb.sv +++ /dev/null @@ -1,151 +0,0 @@ -`include "config.svh" -module tss_tb; - import uvm_pkg::*; - import ip_pkg::*; - import ip_control_test_pkg::*; - import spm_pkg::*; - `include "uvm_macros.svh" - logic clock = 0; - always #5 clock = ~clock; - logic [1:0] reset = '1, active = '0; - ip_control_if control (clock); - virtual spm_if ports[6]; - chandle models[2]; - for (genvar i = 0; i < 6; i++) begin : endpoints - spm_if bus ( - clock, - reset[i/3] - ); - initial ports[i] = bus; - end - for (genvar tile = 0; tile < 2; tile++) begin : tiles - Store dut ( - .clock(clock), - .reset(reset[tile]), - .io_inUse(active[tile]), - .io_busy_0(), - .io_busy_1(), - .io_hostBusy(), - .io_cancel_0(endpoints[tile*3].bus.cancel), - .io_cancel_1(endpoints[tile*3+1].bus.cancel), - .io_clients_0_request_valid(endpoints[tile*3+0].bus.valid), - .io_clients_0_request_ready(endpoints[tile*3+0].bus.ready), - .io_clients_0_request_bits_address(endpoints[tile*3+0].bus.address), - .io_clients_0_request_bits_size(endpoints[tile*3+0].bus.size), - .io_clients_0_request_bits_write(endpoints[tile*3+0].bus.write), - .io_clients_0_request_bits_data(endpoints[tile*3+0].bus.data), - .io_clients_0_request_bits_mask(endpoints[tile*3+0].bus.mask), - .io_clients_0_response_valid(endpoints[tile*3+0].bus.response_valid), - .io_clients_0_response_ready(endpoints[tile*3+0].bus.response_ready), - .io_clients_0_response_bits_data(endpoints[tile*3+0].bus.result), - .io_clients_0_response_bits_error(endpoints[tile*3+0].bus.error), - .io_clients_1_request_valid(endpoints[tile*3+1].bus.valid), - .io_clients_1_request_ready(endpoints[tile*3+1].bus.ready), - .io_clients_1_request_bits_address(endpoints[tile*3+1].bus.address), - .io_clients_1_request_bits_size(endpoints[tile*3+1].bus.size), - .io_clients_1_request_bits_write(endpoints[tile*3+1].bus.write), - .io_clients_1_request_bits_data(endpoints[tile*3+1].bus.data), - .io_clients_1_request_bits_mask(endpoints[tile*3+1].bus.mask), - .io_clients_1_response_valid(endpoints[tile*3+1].bus.response_valid), - .io_clients_1_response_ready(endpoints[tile*3+1].bus.response_ready), - .io_clients_1_response_bits_data(endpoints[tile*3+1].bus.result), - .io_clients_1_response_bits_error(endpoints[tile*3+1].bus.error), - .io_host_request_valid(endpoints[tile*3+2].bus.valid), - .io_host_request_ready(endpoints[tile*3+2].bus.ready), - .io_host_request_bits_address(endpoints[tile*3+2].bus.address), - .io_host_request_bits_size(endpoints[tile*3+2].bus.size), - .io_host_request_bits_write(endpoints[tile*3+2].bus.write), - .io_host_request_bits_data(endpoints[tile*3+2].bus.data), - .io_host_request_bits_mask(endpoints[tile*3+2].bus.mask), - .io_host_response_valid(endpoints[tile*3+2].bus.response_valid), - .io_host_response_ready(endpoints[tile*3+2].bus.response_ready), - .io_host_response_bits_data(endpoints[tile*3+2].bus.result), - .io_host_response_bits_error(endpoints[tile*3+2].bus.error) - ); - end - initial begin - foreach (models[tile]) models[tile] = spm_create(`LOCAL_BASE, `LOCAL_BYTES); - #0; - foreach (ports[i]) begin - ports[i].model = models[i/3]; - ports[i].scoreboard = new($sformatf("port_%0d", i), null); - ports[i].enabled = 1; - end - uvm_config_db#(virtual ip_control_if)::set(null, "uvm_test_top", "vif", control); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - final foreach (models[i]) spm_destroy(models[i]); - initial begin : exercise - logic [127:0] value; - wait (control.start); - repeat (3) @(negedge clock); - reset = 0; - for (int row = 0; row < `LOCAL_BYTES; row += 16) begin - ports[2].access(`LOCAL_BASE + row, 4, 1, 0, '1); - ports[5].access(`LOCAL_BASE + row, 4, 1, '1, '1); - end - @(negedge clock); - active = 3; - fork - begin - ports[0].access(`LOCAL_BASE, 3, 1, 64'h123456789abcdef0, 16'hff); - ports[0].access(`LOCAL_BASE + 8, 3, 1, 1, 16'hff); - end - begin - do begin - ports[1].send(`LOCAL_BASE + 8, 3, 0); - ports[1].take(value); - end while (value != 1); - ports[1].access(`LOCAL_BASE, 3, 0); - end - join - ports[3].access(`LOCAL_BASE, 4, 0); - ports[0].send(`LOCAL_BASE + 16, 3, 1, 64'h1122, 16'hff); - wait (ports[0].response_valid); - ports[1].access(`LOCAL_BASE + 16, 3, 0); - repeat (5) @(negedge clock); - ports[0].take(value); - fork - for (int n = 0; n < 8; n++) ports[0].access(`LOCAL_BASE + 32, 0, 1, n, 1); - for (int n = 0; n < 8; n++) ports[1].access(`LOCAL_BASE + 32, 0, 1, n + 16, 1); - join - ports[1].access(`LOCAL_BASE + 32, 0, 0); - ports[0].access(`LOCAL_BASE + 1, 3, 0); - ports[0].access(`LOCAL_BASE + `LOCAL_BYTES, 3, 0); - ports[0].access(64'h80000000, 3, 0); - ports[0].access(`LOCAL_BASE, 5, 0); - ports[0].access(`LOCAL_BASE + 8, 3, 1, 64'h1122334455667788, 16'ha5); - ports[1].access(`LOCAL_BASE + 8, 3, 0); - ports[0].access((64'h1 << 32) | `LOCAL_BASE, 3, 0); - ports[0].access(64'hfffffffffffffff8, 3, 0); - ports[0].access(`LOCAL_BASE, 0, 1, '1, 3); - ports[0].send(`LOCAL_BASE, 3, 0); - ports[0].cancel = 1; - @(negedge clock); - ports[0].cancel = 0; - ports[1].access(`LOCAL_BASE + 16, 3, 0); - repeat (5) begin - @(negedge clock); - if (ports[0].response_valid) `uvm_fatal("CANCEL", "old TSS response survived cancellation") - end - ports[0].access(`LOCAL_BASE, 3, 0); - ports[0].send(`LOCAL_BASE + 16, 0, 1, 8'ha5, 1); - wait (ports[0].response_valid); - @(negedge clock); - reset[0] = 1; - active[0] = 0; - repeat (2) @(negedge clock); - reset[0] = 0; - ports[2].access(`LOCAL_BASE + 16, 0, 0); - ports[3].access(`LOCAL_BASE + 16, 0, 0); - repeat (2) @(negedge clock); - foreach (ports[i]) begin - if (ports[i].scoreboard.expected_queue.size() || ports[i].scoreboard.actual_queue.size()) - `uvm_fatal("DRAIN", "local storage still has unmatched accesses") - `uvm_info("CHECKED", $sformatf("port %0d: %0d comparisons", i, ports[i].scoreboard.checked), - UVM_LOW) - end - control.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/mem-core/tss/src/main/scala/Emit.scala b/arch/src/main/scala/framework/mem-core/tss/src/main/scala/Emit.scala deleted file mode 100644 index c84d552a9..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/main/scala/Emit.scala +++ /dev/null @@ -1,18 +0,0 @@ -package memcore.memory.tss - -import java.nio.file.{Files, Paths} - -object Emit extends App { - val memory = memcore.memory.spm.Params(base = BigInt("20000", 16), bytes = 64) - val output = Paths.get("src/main/scala/framework/mem-core/tss/build") - Files.createDirectories(output) - Files.writeString( - output.resolve("config.svh"), - s"`define LOCAL_BASE 64'h${memory.base.toString(16)}\n`define LOCAL_BYTES ${memory.bytes}\n" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Store(Params(memory, ports = 2)), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Store").toString, "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/mem-core/tss/src/main/scala/Store.scala b/arch/src/main/scala/framework/mem-core/tss/src/main/scala/Store.scala deleted file mode 100644 index 93f543ee7..000000000 --- a/arch/src/main/scala/framework/mem-core/tss/src/main/scala/Store.scala +++ /dev/null @@ -1,86 +0,0 @@ -package memcore.memory.tss - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.memory.spm.{Memory, Params => MemoryParams, Port, Response} - -case class Params(memory: MemoryParams, ports: Int) { - require(ports > 0) -} - -/** - * One shared SRAM, fair arbitration, one outstanding request and one response slot per port. - * SRAM service is serialized; a held response does not block other clients. inUse covers the - * lifetime of the cooperating group, not just one Ant. Cancel drains only its owning port. - */ -@instantiable -class Store(p: Params) extends Module { - val count = p.ports + 1 - val host = p.ports - val indexBits = log2Ceil(count) - - @public val io = IO(new Bundle { - val inUse = Input(Bool()) - val cancel = Input(Vec(p.ports, Bool())) - val clients = Vec(p.ports, Flipped(new Port(p.memory))) - val host = Flipped(new Port(p.memory)) - val busy = Output(Vec(p.ports, Bool())) - val hostBusy = Output(Bool()) - }) - - val memory = Instantiate(new Memory(p.memory)) - val ports = io.clients.toSeq :+ io.host - val occupied = RegInit(VecInit(Seq.fill(count)(false.B))) - val available = RegInit(VecInit(Seq.fill(count)(false.B))) - val cancelled = RegInit(VecInit(Seq.fill(count)(false.B))) - val answers = Reg(Vec(count, new Response(p.memory))) - val active = RegInit(false.B) - val owner = Reg(UInt(indexBits.W)) - val next = RegInit(0.U(indexBits.W)) - val cancel = VecInit(io.cancel.toSeq :+ false.B) - - val eligible = VecInit(ports.zipWithIndex.map { case (port, i) => - val allowed = - if (i == host) !io.inUse && !occupied.take(p.ports).reduce(_ || _) - else io.inUse && !occupied(host) - port.request.valid && !occupied(i) && !cancel(i) && allowed && !reset.asBool - }) - - val after = VecInit((0 until count).map(i => eligible(i) && i.U >= next)).asUInt - val chosen = Mux(after.orR, PriorityEncoder(after), PriorityEncoder(eligible.asUInt)) - memory.io.readOnly := false.B - memory.io.port.request.valid := !active && eligible.asUInt.orR - memory.io.port.request.bits := VecInit(ports.map(_.request.bits))(chosen) - memory.io.port.response.ready := active - when(memory.io.port.request.fire) { - active := true.B - owner := chosen - occupied(chosen) := true.B - cancelled(chosen) := false.B - next := Mux(chosen === (count - 1).U, 0.U, chosen + 1.U) - } - for ((port, i) <- ports.zipWithIndex) { - port.request.ready := !active && eligible(i) && chosen === i.U && memory.io.port.request.ready - port.response.valid := available(i) && !cancelled(i) && !cancel(i) && !reset.asBool - port.response.bits := answers(i) - when(cancel(i) && occupied(i))(cancelled(i) := true.B) - when(port.response.fire || (cancel(i) && available(i))) { - available(i) := false.B - occupied(i) := false.B - cancelled(i) := false.B - } - } - when(memory.io.port.response.fire) { - active := false.B - when(cancelled(owner) || cancel(owner)) { - occupied(owner) := false.B - cancelled(owner) := false.B - }.otherwise { - answers(owner) := memory.io.port.response.bits - available(owner) := true.B - } - } - io.busy := VecInit(occupied.take(p.ports)) - io.hostBusy := occupied(host) -} diff --git a/arch/src/main/scala/framework/memdomain/MemDomain.scala b/arch/src/main/scala/framework/memdomain/MemDomain.scala deleted file mode 100644 index 740010090..000000000 --- a/arch/src/main/scala/framework/memdomain/MemDomain.scala +++ /dev/null @@ -1,332 +0,0 @@ -package framework.memdomain - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BankRead, BankWrite} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.frontend.globalrs.{GlobalSchedComplete, GlobalSchedIssue} -import framework.top.GlobalConfig -import framework.memdomain.backend.MemRequestIO -import framework.memdomain.backend.mmio.MmioPool -import framework.memdomain.backend.shared.SharedMemLayout -import framework.memdomain.frontend.MemFrontend -import framework.memdomain.frontend.mem.{Footprint, KernelMemoryBridge, MemConfigerIO} -import framework.memdomain.frontend.mem.dma.DmaPort -import framework.memdomain.midend.MemMidend -import framework.memdomain.backend.MemBackend -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.isa.{MvoverISA, MvoverPort} -import memcore.memory.mesh_shm.MeshLocalBankPort - -@instantiable -class MemDomain(val b: GlobalConfig) extends Module { - val totalMmioRead = b.ballDomain.ballIdMappings.map(_.mmioReadBW).sum - val totalMmioWrite = b.ballDomain.ballIdMappings.map(_.mmioWriteBW).sum - val totalBallRead = b.ballDomain.ballIdMappings.map(_.inBW).sum - val totalBallWrite = b.ballDomain.ballIdMappings.map(_.outBW).sum - val kernelPorts = if (b.rvv.enable) b.rvv.memoryPorts else 0 - val sharedHashCount = if (b.memDomain.sharedEnable) SharedMemLayout.totalBank(b) else 0 - - @public - val io = IO(new Bundle { -// Command Channel - val global_issue_i = Flipped(Decoupled(new GlobalSchedIssue(b))) - val global_complete_o = Decoupled(new GlobalSchedComplete(b)) - val busy = Output(Bool()) - val inst_ids = Input(Vec(b.frontend.rob_entries, UInt(64.W))) - val footprints = Output(Vec(3, new Footprint(b))) - -// Inside Channel - val ballChannelActive = Input(Vec(b.ballDomain.ballNum, Bool())) - val ballChannelReady = Output(Vec(b.ballDomain.ballNum, Bool())) - - val ballDomain = new Bundle { - val bankRead = Vec(totalBallRead, new BankRead(b)) - val bankWrite = Vec(totalBallWrite, new BankWrite(b)) - val mmioRead = Vec(totalMmioRead, new MmioRead(b)) - val mmioWrite = Vec(totalMmioWrite, new MmioWrite(b)) - } - - val kernel = - if (b.rvv.enable) Some(new KernelMemoryBridge(b)) - else None - -// Outside Channel - val dma = new DmaPort(b.memDomain.dma_buswidth) - val hartid = Input(UInt(b.tile.xLen.W)) - -// Shared memory path - val shared_mem_req = Vec(SharedMemLayout.channelPerHart(b), new MemRequestIO(b)) - val mvover = new MvoverPort - - val meshLocalBank = Flipped(new MeshLocalBankPort( - MvoverISA.AddressBits, - MvoverISA.BankBits, - b.memDomain.bankWidth, - math.max(1, log2Ceil(b.frontend.rob_entries)) - )) - - val shared_config = Decoupled(new MemConfigerIO(b)) - val shared_query_valid = Output(Bool()) - val shared_query_vbank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val shared_query_group_count = Input(UInt(b.memDomain.groupCountWidth.W)) - - val bank_hashes = - if (b.sim.diffTest) { - Some(Output(Vec(b.memDomain.bankNum + sharedHashCount, new PhysicalBankHash(b)))) - } else { - None - } - - val shared_bank_hashes = - if (b.sim.diffTest && b.memDomain.sharedEnable) { - Some(Input(Vec(sharedHashCount, new PhysicalBankHash(b)))) - } else { - None - } - - }) - - val frontend: Instance[MemFrontend] = Instantiate(new MemFrontend(b)) - val midend: Instance[MemMidend] = Instantiate(new MemMidend(b)) - val backend: Instance[MemBackend] = Instantiate(new MemBackend(b)) - - io.bank_hashes.foreach(_ := backend.io.bank_hashes.get) - backend.io.shared_bank_hashes.foreach(_ := io.shared_bank_hashes.get) - - // Connect query interface from frontend to backend - backend.io.query_vbank_id := frontend.io.query_vbank_id - backend.io.query_is_shared := frontend.io.query_is_shared - frontend.io.query_group_count := backend.io.query_group_count - frontend.io.clearBusy := backend.io.clearBusy - frontend.io.hartid := io.hartid - midend.io.inst_ids := io.inst_ids - - // Shared query: backend delegates shared query to external SharedMemBackend - backend.io.shared_query_group_count := io.shared_query_group_count - io.shared_query_valid := backend.io.shared_query_valid - io.shared_query_vbank_id := backend.io.shared_query_vbank_id - -//===----------------------------------------------------------------------===// -// Connection with outside (all in frontend) -//===----------------------------------------------------------------------===// - frontend.io.global_issue_i <> io.global_issue_i - frontend.io.global_complete_o <> io.global_complete_o - io.mvover <> frontend.io.mvover - io.busy := frontend.io.busy - io.footprints := frontend.io.footprints - - io.dma <> frontend.io.dma - - // Ball Domain interface connects to midend unified bankRead/bankWrite - // Indices [0, totalBallRead) are balldomain; last index is frontend (DMA) - for (i <- 0 until totalBallRead) { - midend.io.bankRead(i).bankRead <> io.ballDomain.bankRead(i) - midend.io.bankRead(i).is_shared := false.B - } - - midend.io.ballChannelActive := io.ballChannelActive - io.ballChannelReady := midend.io.ballChannelReady - - for (i <- 0 until totalBallWrite) { - midend.io.bankWrite(i).bankWrite <> io.ballDomain.bankWrite(i) - midend.io.bankWrite(i).is_shared := false.B - } - - if (b.rvv.enable) { - val kernel = io.kernel.get - val dma = frontend.io.kernel.get - dma.load <> kernel.load - kernel.image <> dma.image - kernel.result <> dma.result - dma.abort := kernel.abort - kernel.busy := dma.busy - kernel.ready := kernel.active - for (port <- 0 until kernelPorts) { - val read = backend.io.kernel_req(port) - read.read <> kernel.bankRead(port).io - read.write.req.valid := false.B - read.write.req.bits := 0.U.asTypeOf(read.write.req.bits) - read.write.resp.ready := true.B - read.bank_id := kernel.bankRead(port).bank_id - read.group_id := kernel.bankRead(port).group_id - read.rob_id := kernel.bankRead(port).rob_id - read.inst_id := io.inst_ids(kernel.bankRead(port).rob_id) - read.hart_id := io.hartid - read.is_shared := false.B - - val write = backend.io.kernel_req(kernelPorts + port) - write.write <> kernel.bankWrite(port).io - write.read.req.valid := false.B - write.read.req.bits := 0.U.asTypeOf(write.read.req.bits) - write.read.resp.ready := true.B - write.bank_id := kernel.bankWrite(port).bank_id - write.group_id := kernel.bankWrite(port).group_id - write.rob_id := kernel.bankWrite(port).rob_id - write.inst_id := io.inst_ids(kernel.bankWrite(port).rob_id) - write.hart_id := io.hartid - write.is_shared := false.B - } - } - - midend.io.bankRead(totalBallRead).bankRead <> frontend.io.interdma.bankRead - midend.io.bankRead(totalBallRead).is_shared := frontend.io.interdma.read_is_shared - midend.io.hartid := io.hartid - - for (i <- 1 until b.memDomain.bankChannel) { - midend.io.mem_req(i) <> backend.io.mem_req(i) - } - if (b.memDomain.sharedEnable && b.memDomain.bankWidth == 128) { - // Reserve channel 0 for Mesh-initiated private-Bank access only when its - // ordinary request/response pair has drained. This preserves the normal - // AccPipe route and prevents one-cycle SRAM responses from being lost. - val normal = midend.io.mem_req(0) - val routed = backend.io.mem_req(0) - val local = io.meshLocalBank - val normalReadPending = RegInit(false.B) - val normalWritePending = RegInit(false.B) - val localPending = RegInit(false.B) - val localIsWrite = RegInit(false.B) - val localTag = Reg(chiselTypeOf(local.request.bits.tag)) - val normalActive = normalReadPending || normalWritePending || - normal.read.req.valid || normal.write.req.valid - val localRequest = local.request.valid && !localPending && !normalActive - val localRead = localRequest && !local.request.bits.write - val localWrite = localRequest && local.request.bits.write - - routed.bank_id := Mux(localRequest, local.request.bits.bank, normal.bank_id) - routed.group_id := Mux(localRequest, 0.U, normal.group_id) - routed.is_shared := Mux(localRequest, false.B, normal.is_shared) - routed.hart_id := Mux(localRequest, io.hartid, normal.hart_id) - routed.rob_id := Mux(localRequest, 0.U, normal.rob_id) - routed.inst_id := Mux(localRequest, 0.U, normal.inst_id) - routed.read.req.valid := localRead || (normal.read.req.valid && !localPending) - routed.read.req.bits.addr := Mux(localRead, local.request.bits.addr, normal.read.req.bits.addr) - routed.write.req.valid := localWrite || (normal.write.req.valid && !localPending) - routed.write.req.bits.addr := Mux(localWrite, local.request.bits.addr, normal.write.req.bits.addr) - routed.write.req.bits.data := Mux(localWrite, local.request.bits.data, normal.write.req.bits.data) - routed.write.req.bits.mask := Mux(localWrite, VecInit(local.request.bits.mask.asBools), normal.write.req.bits.mask) - normal.read.req.ready := !localPending && routed.read.req.ready - normal.write.req.ready := !localPending && routed.write.req.ready - local.request.ready := !normalActive && !localPending && - Mux(local.request.bits.write, routed.write.req.ready, routed.read.req.ready) - - routed.read.resp.ready := Mux(localPending && !localIsWrite, local.response.ready, normal.read.resp.ready) - routed.write.resp.ready := Mux(localPending && localIsWrite, local.response.ready, normal.write.resp.ready) - normal.read.resp.valid := routed.read.resp.valid && !localPending - normal.read.resp.bits := routed.read.resp.bits - normal.write.resp.valid := routed.write.resp.valid && !localPending - normal.write.resp.bits := routed.write.resp.bits - local.response.valid := localPending && Mux(localIsWrite, routed.write.resp.valid, routed.read.resp.valid) - local.response.bits.data := routed.read.resp.bits.data - local.response.bits.tag := localTag - local.response.bits.error := false.B - - when(normal.read.req.fire)(normalReadPending := true.B) - when(normal.read.resp.fire)(normalReadPending := false.B) - when(normal.write.req.fire)(normalWritePending := true.B) - when(normal.write.resp.fire)(normalWritePending := false.B) - when(local.request.fire) { - assert(local.request.bits.bank <= b.frontend.vbank_id_upper_bound.U) - assert(local.request.bits.addr < b.memDomain.bankEntries.U) - assert(!local.request.bits.write || local.request.bits.mask.andR) - localPending := true.B - localIsWrite := local.request.bits.write - localTag := local.request.bits.tag - } - when(local.response.fire)(localPending := false.B) - } else { - midend.io.mem_req(0) <> backend.io.mem_req(0) - io.meshLocalBank.request.ready := false.B - io.meshLocalBank.response.valid := false.B - io.meshLocalBank.response.bits := 0.U.asTypeOf(io.meshLocalBank.response.bits) - } - backend.io.config <> frontend.io.config - -//===----------------------------------------------------------------------===// -// MMIO subsystem wiring -//===----------------------------------------------------------------------===// - val loaderBankWrite = frontend.io.interdma.bankWrite - val dmaBankWrite = midend.io.bankWrite(totalBallWrite).bankWrite - midend.io.bankWrite(totalBallWrite).is_shared := frontend.io.interdma.write_is_shared - - if (b.memDomain.mmioEnable) { - val mmioPool: Instance[MmioPool] = Instantiate(new MmioPool(b)) - - // Write path: route MemLoader's bankWrite to MmioPool when is_mvin_mmio_active - val destIsMmio = frontend.io.is_mvin_mmio_active - - // MMIO is one globally encoded byte space. Each DMA beat is 16 bytes and - // MmioPool stripes those bytes across the five physical byte banks. - val mmioWriteAddr = frontend.io.mmio_addr + loaderBankWrite.io.req.bits.addr * (b.memDomain.bankWidth / 8).U - val mmioByteMask = Wire(Vec(b.memDomain.bankMaskLen, Bool())) - for (k <- 0 until b.memDomain.bankMaskLen) { - mmioByteMask(k) := k.U < frontend.io.mmio_col - } - - dmaBankWrite.bank_id := loaderBankWrite.bank_id - dmaBankWrite.rob_id := loaderBankWrite.rob_id - dmaBankWrite.ball_id := loaderBankWrite.ball_id - dmaBankWrite.group_id := loaderBankWrite.group_id - - // Route write to MMIO or main bank based on is_mvin_mmio_active - mmioPool.io.write.req.valid := loaderBankWrite.io.req.valid && destIsMmio - mmioPool.io.write.req.bits.addr := loaderBankWrite.io.req.bits.addr - mmioPool.io.write.req.bits.data := loaderBankWrite.io.req.bits.data - mmioPool.io.write.req.bits.mask := mmioByteMask - mmioPool.io.writeAddr := mmioWriteAddr - - // Main bank write (when NOT mvin_mmio). - dmaBankWrite.io.req.valid := loaderBankWrite.io.req.valid && !destIsMmio - dmaBankWrite.io.req.bits := loaderBankWrite.io.req.bits - - // Request ready mux: select MMIO or main bank ready - loaderBankWrite.io.req.ready := Mux( - destIsMmio, - mmioPool.io.write.req.ready, - dmaBankWrite.io.req.ready - ) - - // Response mux: select MMIO or main bank response - loaderBankWrite.io.resp.valid := Mux( - destIsMmio, - mmioPool.io.write.resp.valid, - dmaBankWrite.io.resp.valid - ) - loaderBankWrite.io.resp.bits := Mux( - destIsMmio, - mmioPool.io.write.resp.bits, - dmaBankWrite.io.resp.bits - ) - - // Ready signals - mmioPool.io.write.resp.ready := loaderBankWrite.io.resp.ready && destIsMmio - dmaBankWrite.io.resp.ready := loaderBankWrite.io.resp.ready && !destIsMmio - - // Ball read path: connect every configured Blink MMIO line to MmioPool. - for (i <- 0 until totalMmioRead) { - mmioPool.io.ballReq(i) <> io.ballDomain.mmioRead(i).req - io.ballDomain.mmioRead(i).resp <> mmioPool.io.ballResp(i) - } - for (i <- 0 until totalMmioWrite) { - mmioPool.io.ballWriteReq(i) <> io.ballDomain.mmioWrite(i).req - } - } else { - dmaBankWrite <> loaderBankWrite - assert(!frontend.io.is_mvin_mmio_active, "MemDomain MMIO is disabled, but mvin_mmio was issued") - - for (i <- 0 until totalMmioRead) { - io.ballDomain.mmioRead(i).req.ready := false.B - io.ballDomain.mmioRead(i).resp.valid := false.B - io.ballDomain.mmioRead(i).resp.bits := 0.U.asTypeOf(io.ballDomain.mmioRead(i).resp.bits) - } - for (i <- 0 until totalMmioWrite) { - io.ballDomain.mmioWrite(i).req.ready := false.B - } - } - - // Shared path passthrough - io.shared_mem_req <> backend.io.shared_mem_req - io.shared_config <> backend.io.shared_config -} diff --git a/arch/src/main/scala/framework/memdomain/backend/MTrace.scala b/arch/src/main/scala/framework/memdomain/backend/MTrace.scala deleted file mode 100644 index f8c551536..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/MTrace.scala +++ /dev/null @@ -1,115 +0,0 @@ -package framework.memdomain.backend - -import chisel3._ -import chisel3.util._ -import framework.dpi.DpiGuard - -class MTraceDPI extends BlackBox with HasBlackBoxInline { - - val io = IO(new Bundle { - val clock = Input(Clock()) - val reset = Input(Bool()) - val is_write = Input(UInt(8.W)) - val is_shared = Input(UInt(8.W)) - val channel = Input(UInt(32.W)) - val hart_id = Input(UInt(64.W)) - val rob_id = Input(UInt(32.W)) - val inst_id = Input(UInt(64.W)) - val vbank_id = Input(UInt(32.W)) - val pbank_id = Input(UInt(32.W)) - val group_id = Input(UInt(32.W)) - val addr = Input(UInt(32.W)) - val write_mask = Input(UInt(32.W)) - val data_lo = Input(UInt(64.W)) - val data_hi = Input(UInt(64.W)) - val enable = Input(Bool()) - }) - - setInline( - "MTraceDPI.v", - """ - |module MTraceDPI( - | input clock, - | input reset, - | input [7:0] is_write, - | input [7:0] is_shared, - | input [31:0] channel, - | input [63:0] hart_id, - | input [31:0] rob_id, - | input [63:0] inst_id, - | input [31:0] vbank_id, - | input [31:0] pbank_id, - | input [31:0] group_id, - | input [31:0] addr, - | input [31:0] write_mask, - | input [63:0] data_lo, - | input [63:0] data_hi, - | input enable - |); - |""".stripMargin + DpiGuard.wrapMTrace(""" - | import "DPI-C" context function void dpi_mtrace( - | input int unsigned is_write, - | input int unsigned is_shared, - | input int unsigned channel, - | input int unsigned hart_id_lo, - | input int unsigned hart_id_hi, - | input int unsigned rob_id, - | input int unsigned inst_id_lo, - | input int unsigned inst_id_hi, - | input int unsigned vbank_id, - | input int unsigned pbank_id, - | input int unsigned group_id, - | input int unsigned addr, - | input int unsigned write_mask, - | input int unsigned data_lo_lo, - | input int unsigned data_lo_hi, - | input int unsigned data_hi_lo, - | input int unsigned data_hi_hi - | ); - | reg [7:0] is_write_reg; - | reg [7:0] is_shared_reg; - | reg [31:0] channel_reg; - | reg [63:0] hart_id_reg; - | reg [31:0] rob_id_reg; - | reg [63:0] inst_id_reg; - | reg [31:0] vbank_id_reg; - | reg [31:0] pbank_id_reg; - | reg [31:0] group_id_reg; - | reg [31:0] addr_reg; - | reg [31:0] write_mask_reg; - | reg [63:0] data_lo_reg; - | reg [63:0] data_hi_reg; - | reg valid_reg; - | - | always @(posedge clock) begin - | if (reset) begin - | valid_reg <= 1'b0; - | end else begin - | if (valid_reg) begin - | dpi_mtrace(is_write_reg, is_shared_reg, channel_reg, hart_id_reg[31:0], hart_id_reg[63:32], rob_id_reg, inst_id_reg[31:0], inst_id_reg[63:32], vbank_id_reg, pbank_id_reg, group_id_reg, addr_reg, write_mask_reg, data_lo_reg[31:0], data_lo_reg[63:32], data_hi_reg[31:0], data_hi_reg[63:32]); - | end - | - | valid_reg <= enable; - | if (enable) begin - | is_write_reg <= is_write; - | is_shared_reg <= is_shared; - | channel_reg <= channel; - | hart_id_reg <= hart_id; - | rob_id_reg <= rob_id; - | inst_id_reg <= inst_id; - | vbank_id_reg <= vbank_id; - | pbank_id_reg <= pbank_id; - | group_id_reg <= group_id; - | addr_reg <= addr; - | write_mask_reg <= write_mask; - | data_lo_reg <= data_lo; - | data_hi_reg <= data_hi; - | end - | end - | end - |""".stripMargin) + - """ - |endmodule - """.stripMargin - ) -} diff --git a/arch/src/main/scala/framework/memdomain/backend/MemBackend.scala b/arch/src/main/scala/framework/memdomain/backend/MemBackend.scala deleted file mode 100644 index 1faa814c0..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/MemBackend.scala +++ /dev/null @@ -1,292 +0,0 @@ -package framework.memdomain.backend - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import chisel3.util._ -import framework.memdomain.frontend.mem.MemConfigerIO -import framework.top.GlobalConfig -import framework.memdomain.backend.privatepath.PrivateMemBackend -import framework.memdomain.backend.shared.SharedMemLayout -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.backend.banks.{SramReadIO, SramWriteIO} - -class MemRequestIO(b: GlobalConfig) extends Bundle { - val write = Flipped(new SramWriteIO(b)) // midend sends write req into backend - val read = Flipped(new SramReadIO(b)) // midend sends read req into backend - val bank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val group_id = Output(UInt(b.memDomain.groupIdWidth.W)) - val is_shared = Output(Bool()) - val hart_id = Output(UInt(b.tile.xLen.W)) - val rob_id = Output(UInt(log2Up(b.frontend.rob_entries).W)) - val inst_id = Output(UInt(64.W)) -} - -@instantiable -class MemBackend(val b: GlobalConfig) extends Module { - val kernelRequests = if (b.rvv.enable) 2 * b.rvv.memoryPorts else 0 - - val sharedHashCount = if (b.memDomain.sharedEnable) SharedMemLayout.totalBank(b) else 0 - - @public - val io = IO(new Bundle { - val mem_req = Vec(b.memDomain.bankChannel, Flipped(new MemRequestIO(b))) - val kernel_req = Vec(kernelRequests, Flipped(new MemRequestIO(b))) - val config = Flipped(Decoupled(new MemConfigerIO(b))) - - // Shared path — exposed to tile level for multi-core sharing. - // Number of shared ports per core comes from TOML sharedInputChannels / computeCoreIds.size. - val shared_mem_req = Vec(SharedMemLayout.channelPerHart(b), new MemRequestIO(b)) - val shared_config = Decoupled(new MemConfigerIO(b)) - - // Query interface: shared query goes out, private query handled internally. - val shared_query_valid = Output(Bool()) - val shared_query_vbank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val shared_query_group_count = Input(UInt(b.memDomain.groupCountWidth.W)) - - // Original query interface from frontend - val query_vbank_id = Input(UInt(b.memDomain.vbankIdWidth.W)) - val query_is_shared = Input(Bool()) - val query_group_count = Output(UInt(b.memDomain.groupCountWidth.W)) - val clearBusy = Output(Bool()) - - val bank_hashes = - if (b.sim.diffTest) { - Some(Output(Vec(b.memDomain.bankNum + sharedHashCount, new PhysicalBankHash(b)))) - } else { - None - } - - val shared_bank_hashes = - if (b.sim.diffTest && b.memDomain.sharedEnable) { - Some(Input(Vec(SharedMemLayout.totalBank(b), new PhysicalBankHash(b)))) - } else { - None - } - - }) - - // Keep the private backend datapath unchanged and isolate it in a dedicated module. - val privateBackend: Instance[PrivateMemBackend] = Instantiate(new PrivateMemBackend(b)) - for (i <- 0 until kernelRequests) { - privateBackend.io.kernel_req(i) <> io.kernel_req(i) - when(io.kernel_req(i).read.req.valid || io.kernel_req(i).write.req.valid) { - assert(!io.kernel_req(i).is_shared) - assert(io.kernel_req(i).bank_id <= b.frontend.vbank_id_upper_bound.U) - } - } - val sharedChannelPerHart = SharedMemLayout.channelPerHart(b) - - io.bank_hashes.foreach { states => - for (i <- 0 until b.memDomain.bankNum) { - states(i) := privateBackend.io.bank_hashes.get(i) - } - for (i <- 0 until sharedHashCount) { - states(b.memDomain.bankNum + i) := io.shared_bank_hashes.get(i) - } - } - - // Route config to the selected backend only. - val cfgToShared = io.config.bits.is_shared - privateBackend.io.config.valid := io.config.valid && !cfgToShared - privateBackend.io.config.bits := io.config.bits - if (b.memDomain.sharedEnable) { - io.shared_config.valid := io.config.valid && cfgToShared - io.shared_config.bits := io.config.bits - io.config.ready := Mux(cfgToShared, io.shared_config.ready, privateBackend.io.config.ready) - } else { - io.shared_config.valid := false.B - io.shared_config.bits := DontCare - io.config.ready := Mux(cfgToShared, false.B, privateBackend.io.config.ready) - when(io.config.valid && cfgToShared) { - assert(false.B, "MemBackend shared config received while sharedMem is disabled\n") - } - } - - privateBackend.io.query_vbank_id := io.query_vbank_id - io.clearBusy := privateBackend.io.clearBusy - - if (b.memDomain.sharedEnable) { - // Shared query and request routing are only elaborated for shared chips. - io.shared_query_valid := io.query_is_shared - io.shared_query_vbank_id := io.query_vbank_id - io.query_group_count := Mux( - io.query_is_shared, - io.shared_query_group_count, - privateBackend.io.query_group_count - ) - - // Per-channel request routing: is_shared=0 -> private, is_shared=1 -> shared IO. - // Route selection is latched at request fire to keep response demux stable. - val readPending = RegInit(VecInit(Seq.fill(b.memDomain.bankChannel)(false.B))) - val writePending = RegInit(VecInit(Seq.fill(b.memDomain.bankChannel)(false.B))) - val readRouteShared = RegInit(VecInit(Seq.fill(b.memDomain.bankChannel)(false.B))) - val writeRouteShared = RegInit(VecInit(Seq.fill(b.memDomain.bankChannel)(false.B))) - - // Shared IO defaults - for (i <- 0 until sharedChannelPerHart) { - io.shared_mem_req(i).bank_id := 0.U - io.shared_mem_req(i).group_id := 0.U - io.shared_mem_req(i).is_shared := false.B - io.shared_mem_req(i).hart_id := 0.U - io.shared_mem_req(i).rob_id := 0.U - io.shared_mem_req(i).inst_id := 0.U - - io.shared_mem_req(i).read.req.valid := false.B - io.shared_mem_req(i).read.req.bits := DontCare - io.shared_mem_req(i).read.resp.ready := false.B - - io.shared_mem_req(i).write.req.valid := false.B - io.shared_mem_req(i).write.req.bits := DontCare - io.shared_mem_req(i).write.resp.ready := false.B - } - - for (i <- 0 until b.memDomain.bankChannel) { - val useSharedReqRaw = io.mem_req(i).bank_id >= b.frontend.shared_bank_id_base.U && - io.mem_req(i).bank_id < b.memDomain.virtualBankCount.U - val canUseSharedPort = i < sharedChannelPerHart - val hasReq = io.mem_req(i).read.req.valid || io.mem_req(i).write.req.valid - when(useSharedReqRaw && !canUseSharedPort.B && hasReq) { - assert( - false.B, - p"shared request on unsupported channel i=$i (sharedChannelPerHart=$sharedChannelPerHart, bankChannel=${b.memDomain.bankChannel})" - ) - } - val useSharedReq = useSharedReqRaw && canUseSharedPort.B - val useSharedReadResp = Mux(readPending(i), readRouteShared(i), useSharedReq) - val useSharedWriteResp = Mux(writePending(i), writeRouteShared(i), useSharedReq) - - when(io.mem_req(i).read.req.fire) { - readPending(i) := true.B - readRouteShared(i) := useSharedReq - } - when(io.mem_req(i).read.resp.fire) { - readPending(i) := false.B - } - - when(io.mem_req(i).write.req.fire) { - writePending(i) := true.B - writeRouteShared(i) := useSharedReq - } - when(io.mem_req(i).write.resp.fire) { - writePending(i) := false.B - } - - // Metadata is passed to both backends; only selected backend receives valid req/resp-ready. - privateBackend.io.mem_req(i).bank_id := io.mem_req(i).bank_id - privateBackend.io.mem_req(i).group_id := io.mem_req(i).group_id - privateBackend.io.mem_req(i).is_shared := useSharedReq - privateBackend.io.mem_req(i).hart_id := io.mem_req(i).hart_id - privateBackend.io.mem_req(i).rob_id := io.mem_req(i).rob_id - privateBackend.io.mem_req(i).inst_id := io.mem_req(i).inst_id - - // Read request route - privateBackend.io.mem_req(i).read.req.valid := io.mem_req(i).read.req.valid && !useSharedReq - privateBackend.io.mem_req(i).read.req.bits := io.mem_req(i).read.req.bits - - // Write request route - privateBackend.io.mem_req(i).write.req.valid := io.mem_req(i).write.req.valid && !useSharedReq - privateBackend.io.mem_req(i).write.req.bits := io.mem_req(i).write.req.bits - - // Response ready route (selected by latched request route when pending). - privateBackend.io.mem_req(i).read.resp.ready := io.mem_req(i).read.resp.ready && !useSharedReadResp - privateBackend.io.mem_req(i).write.resp.ready := io.mem_req(i).write.resp.ready && !useSharedWriteResp - - if (i < sharedChannelPerHart) { - io.shared_mem_req(i).bank_id := io.mem_req(i).bank_id - io.shared_mem_req(i).group_id := io.mem_req(i).group_id - io.shared_mem_req(i).is_shared := useSharedReq - io.shared_mem_req(i).hart_id := io.mem_req(i).hart_id - io.shared_mem_req(i).rob_id := io.mem_req(i).rob_id - io.shared_mem_req(i).inst_id := io.mem_req(i).inst_id - - io.shared_mem_req(i).read.req.valid := io.mem_req(i).read.req.valid && useSharedReq - io.shared_mem_req(i).read.req.bits := io.mem_req(i).read.req.bits - io.mem_req(i).read.req.ready := Mux( - useSharedReq, - io.shared_mem_req(i).read.req.ready, - privateBackend.io.mem_req(i).read.req.ready - ) - - io.shared_mem_req(i).write.req.valid := io.mem_req(i).write.req.valid && useSharedReq - io.shared_mem_req(i).write.req.bits := io.mem_req(i).write.req.bits - io.mem_req(i).write.req.ready := Mux( - useSharedReq, - io.shared_mem_req(i).write.req.ready, - privateBackend.io.mem_req(i).write.req.ready - ) - - io.shared_mem_req(i).read.resp.ready := io.mem_req(i).read.resp.ready && useSharedReadResp - io.shared_mem_req(i).write.resp.ready := io.mem_req(i).write.resp.ready && useSharedWriteResp - - io.mem_req(i).read.resp.valid := Mux( - useSharedReadResp, - io.shared_mem_req(i).read.resp.valid, - privateBackend.io.mem_req(i).read.resp.valid - ) - io.mem_req(i).read.resp.bits := Mux( - useSharedReadResp, - io.shared_mem_req(i).read.resp.bits, - privateBackend.io.mem_req(i).read.resp.bits - ) - - io.mem_req(i).write.resp.valid := Mux( - useSharedWriteResp, - io.shared_mem_req(i).write.resp.valid, - privateBackend.io.mem_req(i).write.resp.valid - ) - io.mem_req(i).write.resp.bits := Mux( - useSharedWriteResp, - io.shared_mem_req(i).write.resp.bits, - privateBackend.io.mem_req(i).write.resp.bits - ) - } else { - io.mem_req(i).read.req.ready := privateBackend.io.mem_req(i).read.req.ready - io.mem_req(i).write.req.ready := privateBackend.io.mem_req(i).write.req.ready - io.mem_req(i).read.resp.valid := privateBackend.io.mem_req(i).read.resp.valid - io.mem_req(i).read.resp.bits := privateBackend.io.mem_req(i).read.resp.bits - io.mem_req(i).write.resp.valid := privateBackend.io.mem_req(i).write.resp.valid - io.mem_req(i).write.resp.bits := privateBackend.io.mem_req(i).write.resp.bits - } - } - } else { - io.shared_query_valid := false.B - io.shared_query_vbank_id := 0.U - io.query_group_count := privateBackend.io.query_group_count - - for (i <- 0 until SharedMemLayout.channelPerHart(b)) { - io.shared_mem_req(i).bank_id := 0.U - io.shared_mem_req(i).group_id := 0.U - io.shared_mem_req(i).is_shared := false.B - io.shared_mem_req(i).hart_id := 0.U - io.shared_mem_req(i).rob_id := 0.U - io.shared_mem_req(i).inst_id := 0.U - io.shared_mem_req(i).read.req.valid := false.B - io.shared_mem_req(i).read.req.bits := DontCare - io.shared_mem_req(i).read.resp.ready := false.B - io.shared_mem_req(i).write.req.valid := false.B - io.shared_mem_req(i).write.req.bits := DontCare - io.shared_mem_req(i).write.resp.ready := false.B - } - - for (i <- 0 until b.memDomain.bankChannel) { - privateBackend.io.mem_req(i).bank_id := io.mem_req(i).bank_id - privateBackend.io.mem_req(i).group_id := io.mem_req(i).group_id - privateBackend.io.mem_req(i).is_shared := false.B - privateBackend.io.mem_req(i).hart_id := io.mem_req(i).hart_id - privateBackend.io.mem_req(i).rob_id := io.mem_req(i).rob_id - privateBackend.io.mem_req(i).inst_id := io.mem_req(i).inst_id - privateBackend.io.mem_req(i).read.req.valid := io.mem_req(i).read.req.valid - privateBackend.io.mem_req(i).read.req.bits := io.mem_req(i).read.req.bits - privateBackend.io.mem_req(i).read.resp.ready := io.mem_req(i).read.resp.ready - io.mem_req(i).read.req.ready := privateBackend.io.mem_req(i).read.req.ready - io.mem_req(i).read.resp.valid := privateBackend.io.mem_req(i).read.resp.valid - io.mem_req(i).read.resp.bits := privateBackend.io.mem_req(i).read.resp.bits - privateBackend.io.mem_req(i).write.req.valid := io.mem_req(i).write.req.valid - privateBackend.io.mem_req(i).write.req.bits := io.mem_req(i).write.req.bits - privateBackend.io.mem_req(i).write.resp.ready := io.mem_req(i).write.resp.ready - io.mem_req(i).write.req.ready := privateBackend.io.mem_req(i).write.req.ready - io.mem_req(i).write.resp.valid := privateBackend.io.mem_req(i).write.resp.valid - io.mem_req(i).write.resp.bits := privateBackend.io.mem_req(i).write.resp.bits - } - } -} diff --git a/arch/src/main/scala/framework/memdomain/backend/accpipe/AccPipe.scala b/arch/src/main/scala/framework/memdomain/backend/accpipe/AccPipe.scala deleted file mode 100644 index ecf7c1035..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/accpipe/AccPipe.scala +++ /dev/null @@ -1,95 +0,0 @@ -package framework.memdomain.backend.accpipe - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -import framework.top.GlobalConfig -import framework.memdomain.backend.banks.{SramReadIO, SramWriteIO} -import framework.memdomain.backend.MemRequestIO - -@instantiable -class AccPipe(val b: GlobalConfig) extends Module { - - @public - val io = IO(new Bundle { - val sramRead = Flipped(new SramReadIO(b)) - val sramWrite = Flipped(new SramWriteIO(b)) - - val mem_req = Flipped(new MemRequestIO(b)) - - val busy = Output(Bool()) - val group_id = Output(UInt(b.memDomain.groupIdWidth.W)) - val bank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val hart_id = Output(UInt(b.tile.xLen.W)) - }) - - // Each group has its own physical bank, so no address shifting is needed. - - //group_id output - val group_id_reg = RegInit(0.U(b.memDomain.groupIdWidth.W)) - io.group_id := group_id_reg - - //Bank_id output - val bank_id_reg = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - io.bank_id := bank_id_reg - val hart_id_reg = RegInit(0.U(b.tile.xLen.W)) - io.hart_id := hart_id_reg - - val rd_hold = RegInit(false.B) - val rd_data_reg = RegInit(0.U(b.memDomain.bankWidth.W)) - val wr_hold = RegInit(false.B) - val wr_ok_reg = RegInit(false.B) - val busy_reg = RegInit(false.B) - - val canStart = !busy_reg && !rd_hold && !wr_hold - val hasWriteReq = io.mem_req.write.req.valid - val wrReq = canStart && hasWriteReq - val rdReq = canStart && !hasWriteReq && io.mem_req.read.req.valid - - io.sramRead.req.valid := rdReq - io.sramRead.req.bits.addr := io.mem_req.read.req.bits.addr - io.sramRead.resp.ready := !rd_hold - - io.sramWrite.req.valid := wrReq - io.sramWrite.req.bits.addr := io.mem_req.write.req.bits.addr - io.sramWrite.req.bits.data := io.mem_req.write.req.bits.data - io.sramWrite.req.bits.mask := io.mem_req.write.req.bits.mask - io.sramWrite.resp.ready := !wr_hold - - io.mem_req.read.req.ready := canStart && !hasWriteReq && io.sramRead.req.ready - io.mem_req.read.resp.valid := rd_hold - io.mem_req.read.resp.bits.data := rd_data_reg - - io.mem_req.write.req.ready := canStart && io.sramWrite.req.ready - io.mem_req.write.resp.valid := wr_hold - io.mem_req.write.resp.bits.ok := wr_ok_reg - - when(rd_hold && io.mem_req.read.resp.ready) { - rd_hold := false.B - }.elsewhen(io.sramRead.resp.fire) { - rd_hold := true.B - rd_data_reg := io.sramRead.resp.bits.data - } - - when(wr_hold && io.mem_req.write.resp.ready) { - wr_hold := false.B - }.elsewhen(io.sramWrite.resp.fire) { - wr_hold := true.B - wr_ok_reg := io.sramWrite.resp.bits.ok - } - - when(io.mem_req.read.req.fire || io.mem_req.write.req.fire) { - group_id_reg := io.mem_req.group_id - bank_id_reg := io.mem_req.bank_id - hart_id_reg := io.mem_req.hart_id - } - when(io.mem_req.read.req.fire || io.mem_req.write.req.fire) { - busy_reg := true.B - } - when(io.mem_req.read.resp.fire || io.mem_req.write.resp.fire) { - busy_reg := false.B - } - - io.busy := busy_reg -} diff --git a/arch/src/main/scala/framework/memdomain/backend/banks/SramBank.scala b/arch/src/main/scala/framework/memdomain/backend/banks/SramBank.scala deleted file mode 100644 index 9a3f2ebf3..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/banks/SramBank.scala +++ /dev/null @@ -1,91 +0,0 @@ -package framework.memdomain.backend.banks - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig - -/** - * SramBank: Pure SRAM bank - * Simple read/write memory without any accumulation logic - * Each bank is a single-port SRAM. `clear` zeroes every row locally, one row per cycle; - * the ports refuse requests while `clearing`, so banks clear in parallel. - */ -@instantiable -class SramBank(val b: GlobalConfig) extends Module { - require(b.memDomain.bankEntries >= 2 && b.memDomain.bankEntries <= 65536) - val indexBits = log2Ceil(b.memDomain.bankEntries) - val mask_len = b.memDomain.bankMaskLen - val mask_elem = UInt((b.memDomain.bankWidth / mask_len).W) - - @public - val io = IO(new Bundle { - val sramRead = new SramReadIO(b) - val sramWrite = new SramWriteIO(b) - val clear = Input(Bool()) - val clearing = Output(Bool()) - }) - - val mem = SyncReadMem(b.memDomain.bankEntries, Vec(mask_len, mask_elem)) - - // ----------------------------------------------------------------------------- - // Local clear - // ----------------------------------------------------------------------------- - val clearing = RegInit(false.B) - val clearRow = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - io.clearing := clearing - when(io.clear) { - clearing := true.B - clearRow := 0.U - }.elsewhen(clearing) { - clearRow := clearRow + 1.U - when(clearRow === (b.memDomain.bankEntries - 1).U)(clearing := false.B) - } - - val readPending = RegNext(io.sramRead.req.fire, false.B) - val readHeld = RegInit(false.B) - val readData = Reg(UInt(b.memDomain.bankWidth.W)) - val writeValid = RegInit(false.B) - - io.sramRead.resp.valid := readPending || readHeld - io.sramWrite.resp.valid := writeValid - io.sramWrite.resp.bits.ok := true.B - io.sramWrite.req.ready := !io.clear && !clearing && (!writeValid || io.sramWrite.resp.ready) - io.sramRead.req.ready := !io.clear && !clearing && !io.sramWrite.req.fire && - (!io.sramRead.resp.valid || io.sramRead.resp.ready) - - val ren = io.sramRead.req.fire - val wen = clearing || io.sramWrite.req.fire - - when(io.sramRead.req.fire) { - assert(io.sramRead.req.bits.addr < b.memDomain.bankEntries.U, "SRAM read exceeds bank depth") - } - when(io.sramWrite.req.fire) { - assert(io.sramWrite.req.bits.addr < b.memDomain.bankEntries.U, "SRAM write exceeds bank depth") - } - - val rdata = mem.readWrite( - Mux( - clearing, - clearRow, - Mux(wen, io.sramWrite.req.bits.addr(indexBits - 1, 0), io.sramRead.req.bits.addr(indexBits - 1, 0)) - ), - Mux(clearing, 0.U.asTypeOf(Vec(mask_len, mask_elem)), io.sramWrite.req.bits.data.asTypeOf(Vec(mask_len, mask_elem))), - Mux(clearing, VecInit(Seq.fill(mask_len)(true.B)), io.sramWrite.req.bits.mask), - ren || wen, - wen - ) - - io.sramRead.resp.bits.data := Mux(readHeld, readData, rdata.asUInt) - when(readPending && !io.sramRead.resp.ready) { - readHeld := true.B - readData := rdata.asUInt - }.elsewhen(io.sramRead.resp.fire) { - readHeld := false.B - } - when(io.sramWrite.req.fire) { - writeValid := true.B - }.elsewhen(io.sramWrite.resp.fire) { - writeValid := false.B - } -} diff --git a/arch/src/main/scala/framework/memdomain/backend/banks/SramIO.scala b/arch/src/main/scala/framework/memdomain/backend/banks/SramIO.scala deleted file mode 100644 index 488db7ae5..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/banks/SramIO.scala +++ /dev/null @@ -1,36 +0,0 @@ -package framework.memdomain.backend.banks - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig - -/** - * Generic SRAM interface definitions - */ -class SramReadReq(val b: GlobalConfig) extends Bundle { - val addr = UInt(16.W) -} - -class SramReadResp(val b: GlobalConfig) extends Bundle { - val data = UInt(b.memDomain.bankWidth.W) -} - -class SramReadIO(val b: GlobalConfig) extends Bundle { - val req = Flipped(Decoupled(new SramReadReq(b))) - val resp = Decoupled(new SramReadResp(b)) -} - -class SramWriteReq(val b: GlobalConfig) extends Bundle { - val addr = UInt(16.W) - val mask = Vec(b.memDomain.bankMaskLen, Bool()) - val data = UInt(b.memDomain.bankWidth.W) -} - -class SramWriteIO(val b: GlobalConfig) extends Bundle { - val req = Flipped(Decoupled(new SramWriteReq(b))) - val resp = Decoupled(new SramWriteResp(b)) -} - -class SramWriteResp(val b: GlobalConfig) extends Bundle { - val ok = Bool() -} diff --git a/arch/src/main/scala/framework/memdomain/backend/banks/btrace/BTraceDPI.scala b/arch/src/main/scala/framework/memdomain/backend/banks/btrace/BTraceDPI.scala deleted file mode 100644 index 52db28039..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/banks/btrace/BTraceDPI.scala +++ /dev/null @@ -1,91 +0,0 @@ -package framework.memdomain.backend.banks.btrace - -import chisel3._ -import chisel3.util._ -import framework.dpi.DpiGuard -import framework.top.GlobalConfig - -class BTraceRecord extends Bundle { - val instId = UInt(64.W) - val hartId = UInt(64.W) - val w0Vbank = UInt(32.W) - val w0Hash = UInt(32.W) -} - -class PhysicalBankHash(val b: GlobalConfig) extends Bundle { - val valid = Bool() - val hartId = UInt(b.tile.xLen.W) - val vbankId = UInt(b.memDomain.vbankIdWidth.W) - val pbankId = UInt(32.W) - val groupId = UInt(32.W) - val statusHash = UInt(32.W) -} - -class BTraceDPI extends BlackBox with HasBlackBoxInline { - - val io = IO(new Bundle { - val clock = Input(Clock()) - val reset = Input(Bool()) - val instId = Input(UInt(64.W)) - val hartId = Input(UInt(64.W)) - val ownerHartId = Input(UInt(64.W)) - val w0Vbank = Input(UInt(32.W)) - val w0Hash = Input(UInt(32.W)) - val fire = Input(Bool()) - val produced = Input(UInt(64.W)) - val idle = Input(Bool()) - }) - - setInline( - "BTraceDPI.v", - """ - |module BTraceDPI( - | input clock, - | input reset, - | input [63:0] instId, - | input [63:0] hartId, - | input [63:0] ownerHartId, - | input [31:0] w0Vbank, - | input [31:0] w0Hash, - | input fire, - | input [63:0] produced, - | input idle - |); - |""".stripMargin + DpiGuard.wrapBTrace( - """ - | export "DPI-C" function btrace_snapshot; - | function void btrace_snapshot( - | output int unsigned hart_lo, - | output int unsigned hart_hi, - | output int unsigned produced_lo, - | output int unsigned produced_hi, - | output int unsigned is_idle - | ); - | hart_lo = hartId[31:0]; - | hart_hi = hartId[63:32]; - | produced_lo = produced[31:0]; - | produced_hi = produced[63:32]; - | is_idle = {31'b0, idle}; - | endfunction - | import "DPI-C" context function void dpi_btrace( - | input int unsigned inst_id_lo, - | input int unsigned inst_id_hi, - | input int unsigned hart_id_lo, - | input int unsigned hart_id_hi, - | input int unsigned owner_hart_id_lo, - | input int unsigned owner_hart_id_hi, - | input int unsigned w0_vbank, - | input int unsigned w0_hash - | ); - | always @(posedge clock) begin - | if (!reset && fire) begin - | dpi_btrace(instId[31:0], instId[63:32], hartId[31:0], hartId[63:32], ownerHartId[31:0], ownerHartId[63:32], w0Vbank, w0Hash); - | end - | end - |""".stripMargin - ) + - """ - |endmodule - |""".stripMargin - ) -} diff --git a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioBank.scala b/arch/src/main/scala/framework/memdomain/backend/mmio/MmioBank.scala deleted file mode 100644 index 4864c5257..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioBank.scala +++ /dev/null @@ -1,62 +0,0 @@ -package framework.memdomain.backend.mmio - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig - -/** - * MmioBank: single MMIO SRAM bank. - * - * Internal storage : mmioBankEntries x mmioReadWidth bits (default 1024 x 8 = 1 KB) - * Write port : one byte; MmioPool distributes DMA beats across banks. - * Read port : one byte. - * - * Read latency : 1 cycle. Write/read uses single-port semantics; write wins. - */ -@instantiable -class MmioBank(val b: GlobalConfig) extends Module { - - val numEntries = b.memDomain.mmioBankEntries - require( - b.memDomain.mmioBankWidth == b.memDomain.mmioReadWidth, - "MmioBank requires one physical MMIO element per read" - ) - require( - b.memDomain.mmioReadWidth == 8, - "MmioBank requires 8-bit MMIO elements" - ) - - @public - val io = IO(new Bundle { - val write = new MmioBankWriteIO(b) - val read = new MmioBankReadIO(b) - }) - - val mem = SyncReadMem(numEntries, UInt(b.memDomain.mmioReadWidth.W)) - - val readPending = RegNext(io.read.req.fire, false.B) - val readHeld = RegInit(false.B) - val readData = Reg(UInt(b.memDomain.mmioReadWidth.W)) - - io.read.resp.valid := readPending || readHeld - io.write.req.ready := true.B - io.read.req.ready := !io.write.req.valid && (!io.read.resp.valid || io.read.resp.ready) - val ren = io.read.req.fire - val wen = io.write.req.fire - - val rdata = mem.readWrite( - Mux(wen, io.write.req.bits.addr, io.read.req.bits.addr), - io.write.req.bits.data, - ren || wen, - wen - ) - - io.read.resp.bits.data := Mux(readHeld, readData, rdata) - when(readPending && !io.read.resp.ready) { - readHeld := true.B - readData := rdata - }.elsewhen(io.read.resp.fire) { - readHeld := false.B - } -} diff --git a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioIO.scala b/arch/src/main/scala/framework/memdomain/backend/mmio/MmioIO.scala deleted file mode 100644 index 345990326..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioIO.scala +++ /dev/null @@ -1,51 +0,0 @@ -package framework.memdomain.backend.mmio - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig - -/** - * Internal bundles for the MMIO subsystem. - * - * Naming convention: - * *Req / *Resp = Bundles that travel through Decoupled channels - * *Port = Bundles that group req+resp together - */ - -// ===== Per-MmioBank internal read port ===== -class MmioBankReadReq(b: GlobalConfig) extends Bundle { - val addr = UInt(log2Ceil(b.memDomain.mmioBankEntries).W) -} - -class MmioBankReadResp(b: GlobalConfig) extends Bundle { - val data = UInt(b.memDomain.mmioReadWidth.W) -} - -class MmioBankReadIO(b: GlobalConfig) extends Bundle { - val req = Flipped(Decoupled(new MmioBankReadReq(b))) - val resp = Decoupled(new MmioBankReadResp(b)) -} - -class MmioBankWriteReq(b: GlobalConfig) extends Bundle { - val addr = UInt(log2Ceil(b.memDomain.mmioBankEntries).W) - val data = UInt(b.memDomain.mmioReadWidth.W) -} - -class MmioBankWriteIO(b: GlobalConfig) extends Bundle { - val req = Flipped(Decoupled(new MmioBankWriteReq(b))) -} - -// ===== Ball-facing MMIO ports (carried over BlinkIO) ===== -class MmioReadReq(b: GlobalConfig) extends Bundle { - // MMIO is one globally encoded byte address space. - val addr = UInt(log2Ceil(b.memDomain.mmioTotalBytes).W) -} - -class MmioReadResp(b: GlobalConfig) extends Bundle { - val data = UInt(b.memDomain.mmioReadWidth.W) -} - -class MmioWriteReq(b: GlobalConfig) extends Bundle { - val addr = UInt(log2Ceil(b.memDomain.mmioTotalBytes).W) - val data = UInt(b.memDomain.mmioReadWidth.W) -} diff --git a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioPool.scala b/arch/src/main/scala/framework/memdomain/backend/mmio/MmioPool.scala deleted file mode 100644 index 48dd02d4e..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioPool.scala +++ /dev/null @@ -1,165 +0,0 @@ -package framework.memdomain.backend.mmio - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.memdomain.backend.banks.SramWriteIO -import framework.top.GlobalConfig - -/** - * MmioPool: top-level MMIO subsystem wrapper. - * - * Instantiates: - * - Vec of MmioBanks (one per mmioBankNum) - * - MmioRouter (translates Ball requests to bank requests) - * - * External interfaces: - * - write: from MemLoader (mvin_mmio DMA path) - * - ballReq/ballResp/ballWriteReq: from Balls (via MemDomain wiring) - */ -@instantiable -class MmioPool(val b: GlobalConfig) extends Module { - - val mmioReadPorts = b.ballDomain.ballIdMappings.map(_.mmioReadBW).sum - val mmioWritePorts = b.ballDomain.ballIdMappings.map(_.mmioWriteBW).sum - - @public - val io = IO(new Bundle { - // DMA write path (from MemLoader) - val write = new SramWriteIO(b) - val writeAddr = Input(UInt(log2Ceil(b.memDomain.mmioTotalBytes).W)) - - // Ball read path (from MemDomain, one port per configured Blink line) - val ballReq = - Vec(mmioReadPorts, Flipped(Decoupled(new MmioReadReq(b)))) - val ballResp = Vec(mmioReadPorts, Decoupled(new MmioReadResp(b))) - val ballWriteReq = Vec(mmioWritePorts, Flipped(Decoupled(new MmioWriteReq(b)))) - }) - - // Instantiate components - val banks: Seq[Instance[MmioBank]] = Seq.fill(b.memDomain.mmioBankNum)(Instantiate(new MmioBank(b))) - val router: Instance[MmioRouter] = Instantiate(new MmioRouter(b)) - - // MMIO read addresses are already globally encoded; no region translation. - router.io.ballReq <> io.ballReq - io.ballResp <> router.io.ballResp - - // Wire router to banks (read path) - for (i <- 0 until b.memDomain.mmioBankNum) { - banks(i).io.read.req <> router.io.bankReadReq(i) - banks(i).io.read.resp <> router.io.bankReadResp(i) - } - - // A DMA beat is 128 bits. The globally encoded byte address is striped over - // the homogeneous 8-bit MMIO banks, at most one byte per bank each cycle. - val dmaBytes = b.memDomain.bankWidth / 8 - val writeActive = RegInit(false.B) - val writeAddrReg = RegInit(0.U(log2Ceil(b.memDomain.mmioTotalBytes).W)) - val writeDataReg = RegInit(0.U(b.memDomain.bankWidth.W)) - val writeMaskReg = RegInit(VecInit(Seq.fill(b.memDomain.bankMaskLen)(false.B))) - val writeCursorReg = RegInit(0.U(log2Ceil(dmaBytes).W)) - - io.write.req.ready := !writeActive - when(io.write.req.fire) { - assert(io.writeAddr + dmaBytes.U <= b.memDomain.mmioTotalBytes.U, "MmioPool: DMA write exceeds MMIO address space") - writeActive := true.B - writeAddrReg := io.writeAddr - writeDataReg := io.write.req.bits.data - writeMaskReg := io.write.req.bits.mask - writeCursorReg := 0.U - } - - val finalWriteGroup = writeActive && - (writeCursorReg +& b.memDomain.mmioBankNum.U >= dmaBytes.U) - - if (mmioWritePorts == 0) { - for (bankIdx <- 0 until b.memDomain.mmioBankNum) { - val byteMatch = VecInit((0 until b.memDomain.mmioBankNum).map { offset => - val byteIdx = writeCursorReg +& offset.U - val byteAddr = writeAddrReg + byteIdx - byteIdx < dmaBytes.U && (byteAddr % b.memDomain.mmioBankNum.U === bankIdx.U) - }) - val selectedOffset = PriorityEncoder(byteMatch) - val selectedByte = writeCursorReg +& selectedOffset - val selectedAddr = writeAddrReg + selectedByte - val dmaWriteValid = writeActive && byteMatch.asUInt.orR - val dmaWriteAddr = (selectedAddr / b.memDomain.mmioBankNum.U)(log2Ceil(b.memDomain.mmioBankEntries) - 1, 0) - val selectedData = (writeDataReg >> (selectedByte << 3.U))(7, 0) - val dmaWriteData = Mux(writeMaskReg(selectedByte), selectedData, 0.U) - banks(bankIdx).io.write.req.valid := dmaWriteValid - banks(bankIdx).io.write.req.bits.addr := dmaWriteAddr - banks(bankIdx).io.write.req.bits.data := dmaWriteData - } - } else { - val bwV = RegInit(VecInit(Seq.fill(mmioWritePorts)(false.B))) - val bwA = Reg(Vec(mmioWritePorts, UInt(log2Ceil(b.memDomain.mmioTotalBytes).W))) - val bwD = Reg(Vec(mmioWritePorts, UInt(b.memDomain.mmioReadWidth.W))) - - for (bankIdx <- 0 until b.memDomain.mmioBankNum) { - val byteMatch = VecInit((0 until b.memDomain.mmioBankNum).map { offset => - val byteIdx = writeCursorReg +& offset.U - val byteAddr = writeAddrReg + byteIdx - byteIdx < dmaBytes.U && (byteAddr % b.memDomain.mmioBankNum.U === bankIdx.U) - }) - val selectedOffset = PriorityEncoder(byteMatch) - val selectedByte = writeCursorReg +& selectedOffset - val selectedAddr = writeAddrReg + selectedByte - val dmaWriteValid = writeActive && byteMatch.asUInt.orR - val dmaWriteAddr = (selectedAddr / b.memDomain.mmioBankNum.U)(log2Ceil(b.memDomain.mmioBankEntries) - 1, 0) - val selectedData = (writeDataReg >> (selectedByte << 3.U))(7, 0) - val dmaWriteData = Mux(writeMaskReg(selectedByte), selectedData, 0.U) - - val ballMatches = VecInit((0 until mmioWritePorts).map { i => - bwV(i) && (bwA(i) % b.memDomain.mmioBankNum.U === bankIdx.U) - }) - val ballIdx = PriorityEncoder(ballMatches) - val ballAddr = bwA(ballIdx) - assert( - PopCount(ballMatches) <= 1.U, - "MmioPool: multiple Ball writes target one MMIO bank in a cycle" - ) - assert( - !(writeActive && ballMatches.asUInt.orR), - "MmioPool: DMA and Ball write target one MMIO bank in a cycle" - ) - banks(bankIdx).io.write.req.valid := dmaWriteValid || ballMatches.asUInt.orR - banks(bankIdx).io.write.req.bits.addr := Mux( - dmaWriteValid, - dmaWriteAddr, - (ballAddr / b.memDomain.mmioBankNum.U)(log2Ceil(b.memDomain.mmioBankEntries) - 1, 0) - ) - banks(bankIdx).io.write.req.bits.data := Mux(dmaWriteValid, dmaWriteData, bwD(ballIdx)) - } - - for (i <- 0 until mmioWritePorts) { - val targetBank = bwA(i) % b.memDomain.mmioBankNum.U - val targetReady = Mux1H( - UIntToOH(targetBank, b.memDomain.mmioBankNum), - banks.map(_.io.write.req.ready) - ) - val taken = bwV(i) && !writeActive && targetReady - io.ballWriteReq(i).ready := !bwV(i) || taken - when(taken && !io.ballWriteReq(i).fire) { - bwV(i) := false.B - } - when(io.ballWriteReq(i).fire) { - bwV(i) := true.B - bwA(i) := io.ballWriteReq(i).bits.addr - bwD(i) := io.ballWriteReq(i).bits.data - } - assert( - !(io.ballWriteReq(i).valid && - (io.ballWriteReq(i).bits.addr >= b.memDomain.mmioTotalBytes.U)), - "MmioPool: Ball write exceeds the MMIO address space" - ) - } - } - - io.write.resp.valid := finalWriteGroup - io.write.resp.bits.ok := true.B - when(io.write.resp.fire) { - writeActive := false.B - }.elsewhen(writeActive && !finalWriteGroup) { - writeCursorReg := writeCursorReg + b.memDomain.mmioBankNum.U - } -} diff --git a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioRouter.scala b/arch/src/main/scala/framework/memdomain/backend/mmio/MmioRouter.scala deleted file mode 100644 index 0b3dc30e0..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/mmio/MmioRouter.scala +++ /dev/null @@ -1,137 +0,0 @@ -package framework.memdomain.backend.mmio - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig - -/** - * MmioRouter: translates Ball mmioRead requests into MmioBank requests. - * - * Per configured Blink MMIO line: - * 1. Receive a globally encoded MMIO byte address. - * 2. Decode the byte address: - * bank_id = absAddr % mmioBankNum - * rowInBank = absAddr / mmioBankNum - * 3. Route req to bankPorts(bank_id). - * - * Concurrency rule (enforced by software): - * At any time, each mmio_bank is accessed by at most one requester. - * Therefore no arbitration is required — bank ports are driven by exactly the matching Ball. - */ -@instantiable -class MmioRouter(val b: GlobalConfig) extends Module { - - val clientNum = b.ballDomain.ballIdMappings.map(_.mmioReadBW).sum - val mmioBankNum = b.memDomain.mmioBankNum - - @public - val io = IO(new Bundle { - // From Balls (via MemDomain wiring) - val ballReq = Vec(clientNum, Flipped(Decoupled(new MmioReadReq(b)))) - val ballResp = Vec(clientNum, Decoupled(new MmioReadResp(b))) - - // Bank ports (one per MmioBank) - val bankReadReq = Vec(mmioBankNum, Decoupled(new MmioBankReadReq(b))) - val bankReadResp = - Vec(mmioBankNum, Flipped(Decoupled(new MmioBankReadResp(b)))) - }) - - val ballAbsAddr = Wire( - Vec(clientNum, UInt(log2Ceil(b.memDomain.mmioTotalBytes).W)) - ) - - val ballBankId = Wire(Vec(clientNum, UInt(log2Ceil(mmioBankNum).W))) - - val ballRowInBank = Wire( - Vec(clientNum, UInt(log2Ceil(b.memDomain.mmioBankEntries).W)) - ) - - for (i <- 0 until clientNum) { - ballAbsAddr(i) := io.ballReq(i).bits.addr - ballBankId(i) := (ballAbsAddr(i) % mmioBankNum.U)( - log2Ceil(mmioBankNum) - 1, - 0 - ) - ballRowInBank(i) := (ballAbsAddr(i) / mmioBankNum.U)( - log2Ceil(b.memDomain.mmioBankEntries) - 1, - 0 - ) - - assert( - !(io.ballReq(i).valid && (ballAbsAddr(i) >= b.memDomain.mmioTotalBytes.U)), - "MmioRouter: read exceeds the MMIO address space\n" - ) - } - - // Per-bank request routing - for (bankIdx <- 0 until mmioBankNum) { - if (clientNum == 0) { - io.bankReadReq(bankIdx).valid := false.B - io.bankReadReq(bankIdx).bits.addr := 0.U - } else { - val ballMatches = VecInit((0 until clientNum).map { i => - io.ballReq(i).valid && (ballBankId(i) === bankIdx.U) - }) - val matchedBall = PriorityEncoder(ballMatches) - val anyMatch = ballMatches.asUInt.orR - - io.bankReadReq(bankIdx).valid := anyMatch - io.bankReadReq(bankIdx).bits.addr := ballRowInBank(matchedBall) - - assert( - PopCount(ballMatches) <= 1.U, - "MmioRouter: bank %d hit by multiple requesters in the same cycle (software invariant violated)\n", - bankIdx.U - ) - } - } - - // Per-Ball req.ready - for (i <- 0 until clientNum) { - io.ballReq(i).ready := io.bankReadReq(ballBankId(i)).ready - } - - // Response routing: each Ball has a small skid buffer holding in-flight bank_id - val inFlightBankPerBall = Seq.fill(clientNum)( - Module( - new Queue( - UInt(log2Ceil(mmioBankNum).W), - entries = 2, - pipe = true, - flow = false - ) - ) - ) - - for (i <- 0 until clientNum) { - inFlightBankPerBall(i).io.enq.valid := io.ballReq(i).fire - inFlightBankPerBall(i).io.enq.bits := ballBankId(i) - } - - for (i <- 0 until clientNum) { - val q = inFlightBankPerBall(i) - val srcBank = q.io.deq.bits - val srcRespVal = io.bankReadResp(srcBank).valid - - io.ballResp(i).valid := q.io.deq.valid && srcRespVal - io.ballResp(i).bits.data := io.bankReadResp(srcBank).bits.data - - q.io.deq.ready := io.ballResp(i).ready && srcRespVal - } - - for (bankIdx <- 0 until mmioBankNum) { - if (clientNum == 0) { - io.bankReadResp(bankIdx).ready := false.B - } else { - val anyExpect = VecInit((0 until clientNum).map { i => - inFlightBankPerBall(i).io.deq.valid && - (inFlightBankPerBall(i).io.deq.bits === bankIdx.U) && - io.ballResp(i).ready - }).asUInt.orR - io.bankReadResp(bankIdx).ready := anyExpect - } - } -} diff --git a/arch/src/main/scala/framework/memdomain/backend/privatepath/PrivateMemBackend.scala b/arch/src/main/scala/framework/memdomain/backend/privatepath/PrivateMemBackend.scala deleted file mode 100644 index d8874f672..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/privatepath/PrivateMemBackend.scala +++ /dev/null @@ -1,440 +0,0 @@ -package framework.memdomain.backend.privatepath - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.memdomain.frontend.mem.MemConfigerIO -import framework.memdomain.backend.{MTraceDPI, MemRequestIO} -import sims.hash.BankHashMonitor -import framework.memdomain.backend.accpipe.AccPipe -import framework.memdomain.backend.banks.SramBank -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.top.GlobalConfig - -@instantiable -class PrivateMemBackend(val b: GlobalConfig) extends Module { - val kernelRequests = if (b.rvv.enable) 2 * b.rvv.memoryPorts else 0 - val requestCount = b.memDomain.bankChannel + kernelRequests - - @public - val io = IO(new Bundle { - val mem_req = Vec(b.memDomain.bankChannel, Flipped(new MemRequestIO(b))) - val kernel_req = Vec(kernelRequests, Flipped(new MemRequestIO(b))) - val config = Flipped(Decoupled(new MemConfigerIO(b))) - - // Query interface for frontend to get group count - val query_vbank_id = Input(UInt(b.memDomain.vbankIdWidth.W)) - val query_group_count = Output(UInt(b.memDomain.groupCountWidth.W)) - val clearBusy = Output(Bool()) - val bank_hashes = if (b.sim.diffTest) Some(Output(Vec(b.memDomain.bankNum, new PhysicalBankHash(b)))) else None - }) - - val requests = io.mem_req.toSeq ++ io.kernel_req.toSeq - - val banks: Seq[Instance[SramBank]] = Seq.fill(b.memDomain.bankNum)(Instantiate(new SramBank(b))) - val accPipes: Seq[Instance[AccPipe]] = Seq.fill(requestCount)(Instantiate(new AccPipe(b))) - - val hashMonitors: Option[Seq[Instance[BankHashMonitor]]] = - if (b.sim.diffTest) { - Some(Seq.fill(b.memDomain.bankNum)(Instantiate(new BankHashMonitor(b)))) - } else { - None - } - - // Per-channel memory trace DPI-C modules to avoid losing simultaneous events - val mtraces = Seq.fill(requestCount)(Module(new MTraceDPI)) - for (mt <- mtraces) { - mt.io.clock := clock - mt.io.reset := reset.asBool - mt.io.is_write := 0.U - mt.io.is_shared := 0.U - mt.io.channel := 0.U - mt.io.hart_id := 0.U - mt.io.rob_id := 0.U - mt.io.inst_id := 0.U - mt.io.vbank_id := 0.U - mt.io.pbank_id := 0.U - mt.io.group_id := 0.U - mt.io.addr := 0.U - mt.io.write_mask := 0.U - mt.io.data_lo := 0.U - mt.io.data_hi := 0.U - mt.io.enable := false.B - } - - // ----------------------------------------------------------------------------- - // Mapping table - // ----------------------------------------------------------------------------- - class MappingTableEntry extends Bundle { - val valid = Bool() - val hart_id = UInt(b.tile.xLen.W) - val vbank_id = UInt(b.memDomain.vbankIdWidth.W) - val is_multi = Bool() - val group_id = UInt(b.memDomain.groupIdWidth.W) - } - - val mappingTable = RegInit(VecInit(Seq.fill(b.memDomain.bankNum)(0.U.asTypeOf(new MappingTableEntry)))) - // The frontend contract reserves [0, vbank_id_upper_bound] for private - // banks. Keep the direct route table at that architectural size. - val privateVbankCount = b.frontend.vbank_id_upper_bound + 1 - require( - privateVbankCount > 0 && privateVbankCount <= b.memDomain.virtualBankCount, - s"private vbank table size ($privateVbankCount) exceeds virtualBankCount(${b.memDomain.virtualBankCount})" - ) - val groupIndexWidth = log2Up(b.memDomain.bankNum) - val pbankIndexWidth = log2Up(b.memDomain.bankNum) - val privateVbankIdWidth = log2Up(privateVbankCount) - - // A non-multi vbank has exactly one route, so keep that common case as a - // small indexed table. Multi-bank mappings retain their per-group entries - // in mappingTable and use the fallback scan below. This avoids the much - // larger 32x24 valid+pbank register matrix while still removing the full - // mapping scan from the overwhelmingly common single-bank path. - val singleRouteValid = RegInit(VecInit(Seq.fill(privateVbankCount)(false.B))) - - val singleRoutePbank = RegInit( - VecInit(Seq.fill(privateVbankCount)(0.U(pbankIndexWidth.W))) - ) - - // The private namespace is bounded by the frontend contract, while - // multi-bank groups remain represented individually in mappingTable. - val groupCountByVbank = RegInit( - VecInit(Seq.fill(privateVbankCount)(0.U(b.memDomain.groupCountWidth.W))) - ) - - def addEntry( - hart_id: UInt, - vbank_id: UInt, - pbank_id: UInt, - is_multi: Bool, - group_id: UInt - ): Unit = { - val entry = mappingTable(pbank_id) - entry.valid := true.B - entry.hart_id := hart_id - entry.vbank_id := vbank_id - entry.is_multi := is_multi - entry.group_id := group_id - } - - def deleteEntry(vbank_id: UInt): Unit = { - for (i <- 0 until b.memDomain.bankNum) { - when(mappingTable(i).valid && mappingTable(i).vbank_id === vbank_id) { - mappingTable(i).valid := false.B - } - } - } - - // ----------------------------------------------------------------------------- - // Default Value - // ----------------------------------------------------------------------------- - - for (i <- 0 until requestCount) { - accPipes(i).io.mem_req.write <> requests(i).write - accPipes(i).io.mem_req.read <> requests(i).read - accPipes(i).io.mem_req.bank_id := requests(i).bank_id - accPipes(i).io.mem_req.group_id := requests(i).group_id - accPipes(i).io.mem_req.is_shared := requests(i).is_shared - accPipes(i).io.mem_req.hart_id := requests(i).hart_id - accPipes(i).io.mem_req.rob_id := requests(i).rob_id - accPipes(i).io.mem_req.inst_id := requests(i).inst_id - - // Bank-side defaults (only driven when a bank is actually connected) - accPipes(i).io.sramRead.req.ready := false.B - accPipes(i).io.sramRead.resp.valid := false.B - accPipes(i).io.sramRead.resp.bits := DontCare - - accPipes(i).io.sramWrite.req.ready := false.B - accPipes(i).io.sramWrite.resp.valid := false.B - accPipes(i).io.sramWrite.resp.bits := DontCare - } - - banks.zipWithIndex.foreach { - case (bank, _) => - bank.io.sramRead.req.valid := false.B - bank.io.sramRead.req.bits := DontCare - bank.io.sramRead.resp.ready := true.B - - bank.io.sramWrite.req.valid := false.B - bank.io.sramWrite.req.bits := DontCare - bank.io.sramWrite.resp.ready := true.B - bank.io.clear := false.B - } - io.clearBusy := VecInit(banks.map(_.io.clearing)).asUInt.orR - - val realloc = io.config.bits.group_id === 0.U - val freePbankMask = - VecInit(mappingTable.map(entry => !entry.valid || (realloc && entry.vbank_id === io.config.bits.vbank_id))) - val hasFreePbank = freePbankMask.asUInt.orR - io.config.ready := !io.config.bits.alloc || hasFreePbank - when(io.config.valid && io.config.bits.alloc && !hasFreePbank) { - assert(false.B, "PrivateMemBackend allocation failed: no free physical bank\n") - } - - // ----------------------------------------------------------------------------- - // Bank Alloc/Release - // ----------------------------------------------------------------------------- - - hashMonitors.foreach(_.foreach(_.io.bind := false.B)) - - when(io.config.fire) { - val vbank = io.config.bits.vbank_id - val vbankIdx = vbank(privateVbankIdWidth - 1, 0) - when(io.config.bits.transfer) { - val source = io.config.bits.source_bank_id - val sourceIdx = source(privateVbankIdWidth - 1, 0) - val sourceCount = groupCountByVbank(sourceIdx) - val targetCount = groupCountByVbank(vbankIdx) - val totalCount = targetCount +& sourceCount - assert(source =/= vbank && sourceCount =/= 0.U, "Private bank transfer requires a distinct allocated source") - assert(totalCount <= b.memDomain.bankNum.U, "Private bank transfer exceeds physical bank count") - for (entry <- mappingTable) { - when(entry.valid && entry.vbank_id === source) { - entry.vbank_id := vbank - entry.group_id := targetCount + entry.group_id - entry.is_multi := totalCount > 1.U - } - when(entry.valid && entry.vbank_id === vbank) { - entry.is_multi := totalCount > 1.U - } - } - singleRouteValid(sourceIdx) := false.B - singleRouteValid(vbankIdx) := totalCount === 1.U - when(totalCount === 1.U)(singleRoutePbank(vbankIdx) := singleRoutePbank(sourceIdx)) - groupCountByVbank(sourceIdx) := 0.U - groupCountByVbank(vbankIdx) := totalCount - }.elsewhen(io.config.bits.alloc) { - val freePbank = PriorityEncoder(freePbankMask) - hashMonitors.foreach { hashes => - for (i <- 0 until b.memDomain.bankNum) { - hashes(i).io.bind := freePbank === i.U - } - } - // The bank hash already treats a newly bound bank as zero, so the clear writes need no hash update. - for (i <- 0 until b.memDomain.bankNum) { - when(io.config.bits.clear && freePbank === i.U)(banks(i).io.clear := true.B) - } - // Match bemu mset: realloc of the same vbank frees prior physical banks first. - // MemConfiger emits one fire per group; only group 0 drops the old mapping. - when(io.config.bits.group_id === 0.U) { - deleteEntry(io.config.bits.vbank_id) - singleRouteValid(vbankIdx) := false.B - } - addEntry( - io.config.bits.hart_id, - io.config.bits.vbank_id, - freePbank, - io.config.bits.is_multi, - io.config.bits.group_id - ) - when(!io.config.bits.is_multi) { - singleRoutePbank(vbankIdx) := freePbank - singleRouteValid(vbankIdx) := true.B - } - groupCountByVbank(vbankIdx) := Mux(io.config.bits.is_multi, io.config.bits.group_id +& 1.U, 1.U) - }.otherwise { - deleteEntry(io.config.bits.vbank_id) - singleRouteValid(vbankIdx) := false.B - groupCountByVbank(vbankIdx) := 0.U - } - } - - // ----------------------------------------------------------------------------- - // Query interface: return group count for a given vbank_id - // ----------------------------------------------------------------------------- - val queryVbankId = RegNext(io.query_vbank_id, 0.U) - val queryVbank = queryVbankId(privateVbankIdWidth - 1, 0) - io.query_group_count := RegNext(groupCountByVbank(queryVbank), 0.U) - - // ----------------------------------------------------------------------------- - // Connect AccPipe and Banks - // ----------------------------------------------------------------------------- - private def emitTrace( - ch: Int, - isWrite: UInt, - pbankId: UInt, - addr: UInt, - writeMask: UInt, - dataLo: UInt, - dataHi: UInt, - en: Bool - ): Unit = { - mtraces(ch).io.is_write := isWrite - mtraces(ch).io.is_shared := requests(ch).is_shared.asUInt - mtraces(ch).io.channel := ch.U - mtraces(ch).io.hart_id := requests(ch).hart_id - mtraces(ch).io.rob_id := requests(ch).rob_id - mtraces(ch).io.inst_id := requests(ch).inst_id - mtraces(ch).io.vbank_id := requests(ch).bank_id - mtraces(ch).io.pbank_id := pbankId - mtraces(ch).io.group_id := requests(ch).group_id - mtraces(ch).io.addr := addr - mtraces(ch).io.write_mask := writeMask - mtraces(ch).io.data_lo := dataLo - mtraces(ch).io.data_hi := dataHi - mtraces(ch).io.enable := en - } - - // The route is held for the complete AccPipe transaction. Keep the route - // vectors in one place so the bank-side command mux and response demux share - // exactly the same ownership predicate. - val activeRouteOHs = Seq.fill(requestCount)(Wire(UInt(b.memDomain.bankNum.W))) - - for (i <- 0 until requestCount) { - val routePbankReg = RegInit(0.U(pbankIndexWidth.W)) - val routeValidReg = RegInit(false.B) - val routePending = RegInit(false.B) - val requestVbank = requests(i).bank_id - val requestVbankIdx = requestVbank(privateVbankIdWidth - 1, 0) - val requestGroup = requests(i).group_id(groupIndexWidth - 1, 0) - val singleRoute = singleRoutePbank(requestVbankIdx) - val singleValid = singleRouteValid(requestVbankIdx) - val multiRouteMatch = VecInit(mappingTable.map(entry => - entry.valid && entry.is_multi && - entry.vbank_id === requestVbank && entry.group_id === requestGroup - )) - val multiRoute = PriorityEncoder(multiRouteMatch) - val multiValid = multiRouteMatch.asUInt.orR - val requestRoute = Mux(singleValid, singleRoute, multiRoute) - val requestRouteValid = singleValid || multiValid - // Route lookup is deliberately one cycle ahead of the SRAM request. The - // route is captured while the AccPipe is idle, then held through the - // entire request/response lifetime. This removes the live mapping-table - // lookup from every wide bank-side mux select while preserving one request - // per channel per cycle once the pipe is running. - val requestSeen = requests(i).read.req.valid || requests(i).write.req.valid - val activeRoute = routePbankReg - val requestActive = routePending || accPipes(i).io.busy - val activeRouteValid = routeValidReg && requestActive - val activeRouteOH = UIntToOH(activeRoute, b.memDomain.bankNum) & - Fill(b.memDomain.bankNum, activeRouteValid) - - activeRouteOHs(i) := activeRouteOH - - when(!routePending && !accPipes(i).io.busy && requestSeen && requestRouteValid) { - routePbankReg := requestRoute - routeValidReg := true.B - routePending := true.B - } - when(requests(i).read.req.fire || requests(i).write.req.fire) { - routePending := false.B - } - - // Requests are attributed once the selected SRAM accepts them. - when(accPipes(i).io.sramRead.req.fire) { - emitTrace(i, 0.U, activeRoute, accPipes(i).io.sramRead.req.bits.addr, 0.U, 0.U, 0.U, true.B) - } - - // Arrival trace: the write has crossed arbitration and is accepted by the - // selected SPM SRAM port. Keep attribution metadata from the same AccPipe - // transaction and data from the bank-side request. - when(accPipes(i).io.sramWrite.req.fire) { - emitTrace( - i, - 1.U, - activeRoute, - accPipes(i).io.sramWrite.req.bits.addr, - accPipes(i).io.sramWrite.req.bits.mask.asUInt, - accPipes(i).io.sramWrite.req.bits.data(63, 0), - accPipes(i).io.sramWrite.req.bits.data(127, 64), - true.B - ) - } - - } - - // Fixed RVV request slots share the physical-bank arbitration with ordinary - // channels. Capture the accepted request's owner for the one-cycle SRAM - // response; a stalled requester must never receive another owner's response. - val responseRoutes = Seq.fill(requestCount)(WireDefault(0.U(b.memDomain.bankNum.W))) - val responseRouteBits = Seq.fill(requestCount)(Wire(Vec(b.memDomain.bankNum, Bool()))) - - for (j <- 0 until b.memDomain.bankNum) { - val arb = Module(new RRArbiter( - new Bundle { - val write = Bool() - val addr = UInt(16.W) - val data = UInt(b.memDomain.bankWidth.W) - val mask = Vec(b.memDomain.bankMaskLen, Bool()) - }, - requestCount - )) - - for (i <- 0 until requestCount) { - val pipe = accPipes(i) - arb.io.in(i).valid := activeRouteOHs(i)(j) && - (pipe.io.sramRead.req.valid || pipe.io.sramWrite.req.valid) - arb.io.in(i).bits.write := pipe.io.sramWrite.req.valid - arb.io.in(i).bits.addr := Mux( - pipe.io.sramWrite.req.valid, - pipe.io.sramWrite.req.bits.addr, - pipe.io.sramRead.req.bits.addr - ) - arb.io.in(i).bits.data := pipe.io.sramWrite.req.bits.data - arb.io.in(i).bits.mask := pipe.io.sramWrite.req.bits.mask - when(activeRouteOHs(i)(j)) { - pipe.io.sramRead.req.ready := arb.io.in(i).ready && !arb.io.in(i).bits.write - pipe.io.sramWrite.req.ready := arb.io.in(i).ready && arb.io.in(i).bits.write - } - responseRouteBits(i)(j) := RegNext(arb.io.in(i).fire, false.B) - } - - val bank = banks(j) - bank.io.sramRead.req.valid := arb.io.out.valid && !arb.io.out.bits.write - bank.io.sramRead.req.bits.addr := arb.io.out.bits.addr - bank.io.sramWrite.req.valid := arb.io.out.valid && arb.io.out.bits.write - bank.io.sramWrite.req.bits.addr := arb.io.out.bits.addr - bank.io.sramWrite.req.bits.data := arb.io.out.bits.data - bank.io.sramWrite.req.bits.mask := arb.io.out.bits.mask - arb.io.out.ready := Mux(arb.io.out.bits.write, bank.io.sramWrite.req.ready, bank.io.sramRead.req.ready) - - hashMonitors.foreach { hashes => - val monitor = hashes(j) - bank.io.sramWrite.req.valid := arb.io.out.valid && arb.io.out.bits.write && monitor.io.write.ready - monitor.io.write.valid := arb.io.out.valid && arb.io.out.bits.write && bank.io.sramWrite.req.ready - arb.io.out.ready := Mux( - arb.io.out.bits.write, - bank.io.sramWrite.req.ready && monitor.io.write.ready, - bank.io.sramRead.req.ready - ) - monitor.io.write.bits.addr := arb.io.out.bits.addr - monitor.io.write.bits.mask := arb.io.out.bits.mask - monitor.io.write.bits.data := arb.io.out.bits.data - } - bank.io.sramRead.resp.ready := VecInit((0 until requestCount).map { i => - responseRouteBits(i)(j) && accPipes(i).io.sramRead.resp.ready - }).asUInt.orR - bank.io.sramWrite.resp.ready := VecInit((0 until requestCount).map { i => - responseRouteBits(i)(j) && accPipes(i).io.sramWrite.resp.ready - }).asUInt.orR - when(bank.io.sramRead.resp.valid)(assert(bank.io.sramRead.resp.ready)) - when(bank.io.sramWrite.resp.valid)(assert(bank.io.sramWrite.resp.ready)) - } - - for (i <- 0 until requestCount) { - responseRoutes(i) := responseRouteBits(i).asUInt - val readRespSel = VecInit((0 until b.memDomain.bankNum).map { j => - responseRoutes(i)(j) && banks(j).io.sramRead.resp.valid - }).asUInt - val writeRespSel = VecInit((0 until b.memDomain.bankNum).map { j => - responseRoutes(i)(j) && banks(j).io.sramWrite.resp.valid - }).asUInt - accPipes(i).io.sramRead.resp.valid := readRespSel.orR - accPipes(i).io.sramRead.resp.bits.data := (0 until b.memDomain.bankNum).map { j => - Fill(b.memDomain.bankWidth, responseRoutes(i)(j)) & banks(j).io.sramRead.resp.bits.data - }.reduce(_ | _) - accPipes(i).io.sramWrite.resp.valid := writeRespSel.orR - accPipes(i).io.sramWrite.resp.bits.ok := writeRespSel.orR - } - - io.bank_hashes.foreach { states => - for (i <- 0 until b.memDomain.bankNum) { - states(i).valid := mappingTable(i).valid - states(i).hartId := mappingTable(i).hart_id - states(i).vbankId := mappingTable(i).vbank_id - states(i).pbankId := i.U - states(i).groupId := mappingTable(i).group_id - states(i).statusHash := hashMonitors.get(i).io.statusHash - } - } -} diff --git a/arch/src/main/scala/framework/memdomain/backend/shared/PhysicalPort.scala b/arch/src/main/scala/framework/memdomain/backend/shared/PhysicalPort.scala deleted file mode 100644 index 64ab7da5b..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/shared/PhysicalPort.scala +++ /dev/null @@ -1,23 +0,0 @@ -package framework.memdomain.backend.shared - -import chisel3._ -import chisel3.util._ - -/** Tile shared SRAM byte space; not a Core virtual-bank or private-bank address. */ -class SharedPhysicalRequest(dataBits: Int) extends Bundle { - val address = UInt(64.W) - val write = Bool() - val data = UInt(dataBits.W) - val mask = UInt((dataBits / 8).W) -} - -class SharedPhysicalResponse(dataBits: Int) extends Bundle { - val data = UInt(dataBits.W) - val error = Bool() -} - -/** One outstanding transaction per port; a write response acknowledges actual SRAM completion. */ -class SharedPhysicalPort(dataBits: Int) extends Bundle { - val request = Decoupled(new SharedPhysicalRequest(dataBits)) - val response = Flipped(Decoupled(new SharedPhysicalResponse(dataBits))) -} diff --git a/arch/src/main/scala/framework/memdomain/backend/shared/SharedLease.scala b/arch/src/main/scala/framework/memdomain/backend/shared/SharedLease.scala deleted file mode 100644 index 39959d48b..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/shared/SharedLease.scala +++ /dev/null @@ -1,17 +0,0 @@ -package framework.memdomain.backend.shared - -import chisel3._ -import chisel3.util._ - -class SharedLeaseRequest extends Bundle { - val endpoint = UInt(32.W) - val vbank = UInt(16.W) - val group = UInt(16.W) - val release = Bool() -} - -/** One outstanding mapping lease operation; export returns its physical byte base. */ -class SharedLeasePort extends Bundle { - val request = Decoupled(new SharedLeaseRequest) - val response = Flipped(Decoupled(UInt(64.W))) -} diff --git a/arch/src/main/scala/framework/memdomain/backend/shared/SharedMem.scala b/arch/src/main/scala/framework/memdomain/backend/shared/SharedMem.scala deleted file mode 100644 index c874b4fac..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/shared/SharedMem.scala +++ /dev/null @@ -1,39 +0,0 @@ -package framework.memdomain.backend.shared - -import framework.top.GlobalConfig - -object SharedMemLayout { - - def totalBank(b: GlobalConfig): Int = { - require(b.memDomain.sharedEnable, "shared memory is disabled") - require(b.memDomain.sharedBankNum > 0, "sharedBankNum must be > 0") - b.memDomain.sharedBankNum - } - - def channelPerHart(b: GlobalConfig): Int = { - if (!b.memDomain.sharedEnable) { - return 0 - } - val computeCores = b.memDomain.computeCoreIds.size - require(computeCores > 0, "shared memory requires compute cores") - require( - b.memDomain.sharedInputChannels > 0, - s"sharedInputChannels(${b.memDomain.sharedInputChannels}) must be > 0" - ) - require( - b.memDomain.sharedInputChannels % computeCores == 0, - s"sharedInputChannels(${b.memDomain.sharedInputChannels}) must be divisible by computeCores($computeCores)" - ) - if (b.memDomain.sharedInputChannels > 32) { - require( - b.memDomain.sharedInputChannels == computeCores, - s"sharedInputChannels(${b.memDomain.sharedInputChannels}) must equal computeCores($computeCores) when > 32" - ) - } - val ch = b.memDomain.sharedInputChannels / computeCores - require(ch > 0, s"channelPerHart($ch) must be > 0") - ch - } - - def totalChannel(b: GlobalConfig): Int = if (b.memDomain.sharedEnable) b.memDomain.sharedInputChannels else 0 -} diff --git a/arch/src/main/scala/framework/memdomain/backend/shared/SharedMemBackend.scala b/arch/src/main/scala/framework/memdomain/backend/shared/SharedMemBackend.scala deleted file mode 100644 index e9f8209af..000000000 --- a/arch/src/main/scala/framework/memdomain/backend/shared/SharedMemBackend.scala +++ /dev/null @@ -1,588 +0,0 @@ -package framework.memdomain.backend.shared - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.memdomain.backend.{MTraceDPI, MemRequestIO} -import sims.hash.BankHashMonitor -import framework.memdomain.backend.accpipe.AccPipe -import framework.memdomain.backend.banks.SramBank -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.frontend.mem.MemConfigerIO -import framework.top.GlobalConfig -import memcore.memory.mesh_shm.{MeshCoreAttachment, MeshLocalBankPort, MeshSharedMem, MeshSharedMemParams} -import framework.memdomain.isa.{MvoverISA, MvoverPort} - -@instantiable -class SharedMemBackend(val b: GlobalConfig, useMesh: Boolean = false, externalPhysicalPorts: Int = 0) extends Module { - require(externalPhysicalPorts >= 0 && (externalPhysicalPorts == 0 || useMesh)) - val nCores = b.memDomain.nCores - val totalBanks = SharedMemLayout.totalBank(b) - val totalChannel = SharedMemLayout.totalChannel(b) - val sharedConfig = b.copy(memDomain = b.memDomain.copy(bankEntries = b.memDomain.sharedBankEntries)) - val tagBits = math.max(1, log2Ceil(b.frontend.rob_entries)) - - @public - val io = IO(new Bundle { - val mem_req = Vec(totalChannel, Flipped(new MemRequestIO(b))) - val lease = Option.when(externalPhysicalPorts > 0)(Flipped(new SharedLeasePort)) - val leaseOwnerHartId = Option.when(externalPhysicalPorts > 0)(Input(UInt(b.tile.xLen.W))) - val physical = Vec(externalPhysicalPorts, Flipped(new SharedPhysicalPort(b.memDomain.bankWidth))) - val physicalOwnerHartId = if (externalPhysicalPorts > 0) Some(Input(UInt(b.tile.xLen.W))) else None - val mvover = Flipped(new MvoverPort) - val localBanks = - Vec(nCores, new MeshLocalBankPort(MvoverISA.AddressBits, MvoverISA.BankBits, b.memDomain.bankWidth, tagBits)) - val config = Flipped(Decoupled(new MemConfigerIO(b))) - - // Query interface for frontend to get group count - val query_valid = Input(Vec(nCores, Bool())) - val query_hart_id = Input(Vec(nCores, UInt(b.tile.xLen.W))) - val query_vbank_id = Input(Vec(nCores, UInt(b.memDomain.vbankIdWidth.W))) - val query_group_count = Output(Vec(nCores, UInt(b.memDomain.groupCountWidth.W))) - val bank_hashes = if (b.sim.diffTest) Some(Output(Vec(totalBanks, new PhysicalBankHash(b)))) else None - }) - - val meshParams = - if (useMesh) { - require(b.memDomain.bankWidth == 128 && b.memDomain.bankMaskLen == 16) - require(b.memDomain.sharedInputChannels % b.memDomain.computeCoreIds.size == 0) - val physicalBanks = totalBanks - val (rows, columns) = MeshSharedMemParams.dimensions(physicalBanks) - val perCore = SharedMemLayout.channelPerHart(b) - val computeCores = b.memDomain.computeCoreIds - val attachments = MeshSharedMemParams.directionalAttachments( - rows, - columns, - Seq.fill(computeCores.size)(perCore) - ) - Some(MeshSharedMemParams( - rows = rows, - cols = columns, - entriesPerBank = b.memDomain.sharedBankEntries, - dataBits = b.memDomain.bankWidth, - tagBits = tagBits, - localBankBits = MvoverISA.BankBits, - cores = (0 until nCores).map { core => - val computeIndex = computeCores.indexOf(core) - if (computeIndex < 0) MeshCoreAttachment(Seq.empty, Seq.empty, None) - else attachments(computeIndex) - }, - externalChannels = externalPhysicalPorts, - visibleBanks = totalBanks - )) - } else None - - val mesh: Option[Instance[MeshSharedMem]] = meshParams.map(p => Instantiate(new MeshSharedMem(p))) - mesh.foreach { network => - require(network.io.transferCommand.bits.sourceAddr.getWidth <= 16) - require(network.io.transferCommand.bits.sourceCore.getWidth <= 8) - network.io.transferCommand.valid := io.mvover.command.valid - network.io.transferCommand.bits.sourceCore := io.mvover.command.bits.sourceCore - network.io.transferCommand.bits.targetCore := io.mvover.command.bits.targetCore - network.io.transferCommand.bits.sourceBank := io.mvover.command.bits.sourceBank - network.io.transferCommand.bits.targetBank := io.mvover.command.bits.targetBank - network.io.transferCommand.bits.sourceAddr := io.mvover.command.bits.sourceAddr - network.io.transferCommand.bits.targetAddr := io.mvover.command.bits.targetAddr - network.io.transferCommand.bits.rows := io.mvover.command.bits.rows - network.io.transferCommand.bits.tag := 0.U - io.mvover.command.ready := network.io.transferCommand.ready - io.mvover.completion.valid := network.io.transferCompletion.valid - io.mvover.completion.bits := network.io.transferCompletion.bits.error - network.io.transferCompletion.ready := io.mvover.completion.ready - for ((local, index) <- network.io.localBanks.zipWithIndex) { - io.localBanks(index).request.valid := local.request.valid - io.localBanks(index).request.bits := local.request.bits - local.request.ready := io.localBanks(index).request.ready - local.response.valid := io.localBanks(index).response.valid - local.response.bits := io.localBanks(index).response.bits - io.localBanks(index).response.ready := local.response.ready - } - } - if (!useMesh) { - io.mvover.command.ready := false.B - io.mvover.completion.valid := false.B - io.mvover.completion.bits := false.B - for (local <- io.localBanks) { - local.request.valid := false.B - local.request.bits := 0.U.asTypeOf(local.request.bits) - local.response.ready := false.B - } - } - - val banks: Seq[Instance[SramBank]] = - if (useMesh) Seq.empty else Seq.fill(totalBanks)(Instantiate(new SramBank(sharedConfig))) - val accPipes: Seq[Instance[AccPipe]] = Seq.fill(totalChannel)(Instantiate(new AccPipe(b))) - - val hashMonitors: Option[Seq[Instance[BankHashMonitor]]] = - if (b.sim.diffTest) { - Some(Seq.fill(totalBanks)(Instantiate(new BankHashMonitor(sharedConfig)))) - } else { - None - } - - // Per-channel memory trace DPI-C modules to avoid losing simultaneous events - val mtraces = Seq.fill(totalChannel + externalPhysicalPorts)(Module(new MTraceDPI)) - for (mt <- mtraces) { - mt.io.clock := clock - mt.io.reset := reset.asBool - mt.io.is_write := 0.U - mt.io.is_shared := 0.U - mt.io.channel := 0.U - mt.io.hart_id := 0.U - mt.io.rob_id := 0.U - mt.io.inst_id := 0.U - mt.io.vbank_id := 0.U - mt.io.pbank_id := 0.U - mt.io.group_id := 0.U - mt.io.addr := 0.U - mt.io.write_mask := 0.U - mt.io.data_lo := 0.U - mt.io.data_hi := 0.U - mt.io.enable := false.B - } - - // ----------------------------------------------------------------------------- - // Mapping table - // ----------------------------------------------------------------------------- - class MappingTableEntry extends Bundle { - val valid = Bool() - val hart_id = UInt(b.tile.xLen.W) - val vbank_id = UInt(b.memDomain.vbankIdWidth.W) - val is_multi = Bool() - val group_id = UInt(b.memDomain.groupIdWidth.W) - } - - val mappingTable = RegInit(VecInit(Seq.fill(totalBanks)(0.U.asTypeOf(new MappingTableEntry)))) - val leases = Option.when(externalPhysicalPorts > 0)(RegInit(VecInit(Seq.fill(totalBanks)(0.U(32.W))))) - io.lease.foreach { port => - val pending = RegInit(false.B) - val result = Reg(UInt(64.W)) - val matches = VecInit(mappingTable.map(entry => - entry.valid && - entry.hart_id === io.leaseOwnerHartId.get && entry.vbank_id === port.request.bits.vbank && - entry.group_id === port.request.bits.group - )) - val index = PriorityEncoder(matches) - port.request.ready := !pending && !io.config.valid && !reset.asBool - port.response.valid := pending && !reset.asBool - port.response.bits := result - when(port.request.fire) { - assert(PopCount(matches) === 1.U, "Shared lease requires one allocated owner/bank/group") - val count = leases.get(index) - when(port.request.bits.release) { - assert(count =/= 0.U, "Shared lease release has no export") - count := count - 1.U - result := 0.U - }.otherwise { - assert(count =/= "hffffffff".U, "Shared lease count overflow") - count := count + 1.U - result := index * (b.memDomain.sharedBankEntries * (b.memDomain.bankWidth / 8)).U(64.W) - } - pending := true.B - } - when(port.response.fire)(pending := false.B) - } - - mesh.foreach { network => - val rowBits = log2Ceil(b.memDomain.sharedBankEntries) - val bytesPerRow = b.memDomain.bankWidth / 8 - val byteBits = log2Ceil(bytesPerRow) - for (i <- 0 until externalPhysicalPorts) { - val physical = io.physical(i) - val channel = network.io.external(i) - val bank = physical.request.bits.address >> (rowBits + byteBits) - val legal = bank < totalBanks.U && physical.request.bits.address(byteBits - 1, 0) === 0.U - val bankIndex = bank(math.max(1, log2Ceil(totalBanks)) - 1, 0) - val allocated = VecInit(mappingTable.map(_.valid))(bankIndex) - when(physical.request.valid && !reset.asBool) { - assert(legal && allocated, "T2T physical access requires an allocated shared bank and aligned address") - } - channel.request.valid := physical.request.valid && legal && allocated - channel.request.bits := 0.U.asTypeOf(channel.request.bits) - channel.request.bits.bank := bankIndex - channel.request.bits.tuser := Cat( - physical.request.bits.address(rowBits + byteBits - 1, byteBits).pad(16), - physical.request.bits.write - ) - channel.request.bits.data := physical.request.bits.data - channel.request.bits.mask := physical.request.bits.mask - channel.request.bits.tag := 0.U - channel.request.bits.tlast := true.B - physical.request.ready := channel.request.ready && legal && allocated - physical.response.valid := channel.response.valid - physical.response.bits.data := channel.response.bits.data - physical.response.bits.error := channel.response.bits.error - channel.response.ready := physical.response.ready - } - } - for (i <- 0 until externalPhysicalPorts) { - val physical = io.physical(i) - val held = Reg(new SharedPhysicalRequest(b.memDomain.bankWidth)) - val entry = Reg(new MappingTableEntry) - val pending = RegInit(false.B) - val bankBits = log2Ceil(b.memDomain.sharedBankEntries * (b.memDomain.bankWidth / 8)) - when(physical.request.fire) { - assert(!pending, "Physical shared port accepts one outstanding request") - held := physical.request.bits - entry := mappingTable((physical.request.bits.address >> bankBits)(math.max(1, log2Ceil(totalBanks)) - 1, 0)) - pending := true.B - } - when(physical.response.valid && !reset.asBool)(assert(pending)) - when(physical.response.fire)(pending := false.B) - val trace = mtraces(totalChannel + i) - val value = Mux(held.write, held.data, physical.response.bits.data) - trace.io.is_write := held.write.asUInt - trace.io.is_shared := 1.U - trace.io.channel := (totalChannel + i).U - trace.io.hart_id := io.physicalOwnerHartId.get - trace.io.vbank_id := entry.vbank_id - trace.io.pbank_id := held.address >> bankBits - trace.io.group_id := entry.group_id - trace.io.addr := (held.address >> 4) & (b.memDomain.sharedBankEntries - 1).U - trace.io.write_mask := Mux(held.write, held.mask, 0.U) - trace.io.data_lo := value(63, 0) - trace.io.data_hi := value(127, 64) - trace.io.enable := physical.response.fire && !physical.response.bits.error - } - - def addEntry( - hart_id: UInt, - vbank_id: UInt, - pbank_id: UInt, - is_multi: Bool, - group_id: UInt - ): Unit = { - val duplicate = mappingTable.map(entry => - entry.valid && - !(group_id === 0.U && entry.hart_id === hart_id && entry.vbank_id === vbank_id) && - (entry.hart_id === hart_id) && - (entry.vbank_id === vbank_id) && - (entry.group_id === group_id) - ).reduce(_ || _) - when(duplicate) { - assert(false.B, "SharedMemBackend duplicate allocation: hart=%d vbank=%d group=%d\n", hart_id, vbank_id, group_id) - } - - val entry = mappingTable(pbank_id) - leases.foreach(counts => assert(counts(pbank_id) === 0.U, "Shared bank allocation replaces live exports")) - entry.valid := true.B - entry.hart_id := hart_id - entry.vbank_id := vbank_id - entry.is_multi := is_multi - entry.group_id := group_id - } - - def deleteEntry(hart_id: UInt, vbank_id: UInt): Unit = { - val found = - mappingTable.map(entry => entry.valid && entry.vbank_id === vbank_id && entry.hart_id === hart_id).reduce(_ || _) - when(!found) { - assert(false.B, "SharedMemBackend release missing allocation: hart=%d vbank=%d\n", hart_id, vbank_id) - } - clearVbank(hart_id, vbank_id) - } - - def clearVbank(hart_id: UInt, vbank_id: UInt): Unit = { - for (i <- 0 until totalBanks) { - when(mappingTable(i).valid && mappingTable(i).vbank_id === vbank_id && mappingTable(i).hart_id === hart_id) { - leases.foreach(counts => assert(counts(i) === 0.U, "Shared bank mapping changed with live exports")) - mappingTable(i).valid := false.B - } - } - } - - // ----------------------------------------------------------------------------- - // Default Value - // ----------------------------------------------------------------------------- - - for (i <- 0 until totalChannel) { - accPipes(i).io.mem_req.write <> io.mem_req(i).write - accPipes(i).io.mem_req.read <> io.mem_req(i).read - accPipes(i).io.mem_req.bank_id := io.mem_req(i).bank_id - accPipes(i).io.mem_req.group_id := io.mem_req(i).group_id - accPipes(i).io.mem_req.is_shared := io.mem_req(i).is_shared - accPipes(i).io.mem_req.hart_id := io.mem_req(i).hart_id - accPipes(i).io.mem_req.rob_id := io.mem_req(i).rob_id - accPipes(i).io.mem_req.inst_id := io.mem_req(i).inst_id - - // Bank-side defaults (only driven when a bank is actually connected) - accPipes(i).io.sramRead.req.ready := false.B - accPipes(i).io.sramRead.resp.valid := false.B - accPipes(i).io.sramRead.resp.bits := DontCare - - accPipes(i).io.sramWrite.req.ready := false.B - accPipes(i).io.sramWrite.resp.valid := false.B - accPipes(i).io.sramWrite.resp.bits := DontCare - } - - banks.zipWithIndex.foreach { - case (bank, _) => - bank.io.sramRead.req.valid := false.B - bank.io.sramRead.req.bits := DontCare - bank.io.sramRead.resp.ready := true.B - - bank.io.sramWrite.req.valid := false.B - bank.io.sramWrite.req.bits := DontCare - bank.io.sramWrite.resp.ready := true.B - bank.io.clear := false.B - } - - val realloc = io.config.bits.group_id === 0.U - - val freePbankMask = VecInit(mappingTable.map(entry => - !entry.valid || - (realloc && entry.hart_id === io.config.bits.hart_id && entry.vbank_id === io.config.bits.vbank_id) - )) - - val hasFreePbank = freePbankMask.asUInt.orR - io.config.ready := !io.config.bits.alloc || hasFreePbank - when(io.config.valid && io.config.bits.alloc && !hasFreePbank) { - assert(false.B, "SharedMemBackend allocation failed: no free physical shared bank\n") - } - - // ----------------------------------------------------------------------------- - // Bank Alloc/Release - // ----------------------------------------------------------------------------- - - if (!useMesh) { - hashMonitors.foreach { hashes => - for (j <- 0 until totalBanks) { - hashes(j).io.write.valid := false.B - hashes(j).io.write.bits.addr := banks(j).io.sramWrite.req.bits.addr - hashes(j).io.write.bits.mask := banks(j).io.sramWrite.req.bits.mask - hashes(j).io.write.bits.data := banks(j).io.sramWrite.req.bits.data - } - } - } - - hashMonitors.foreach(_.foreach(_.io.bind := false.B)) - - when(io.config.fire) { - when(io.config.bits.transfer) { - val source = io.config.bits.source_bank_id - val target = io.config.bits.vbank_id - val owner = io.config.bits.hart_id - val sourceCount = PopCount(mappingTable.map(e => e.valid && e.hart_id === owner && e.vbank_id === source)) - val targetCount = PopCount(mappingTable.map(e => e.valid && e.hart_id === owner && e.vbank_id === target)) - val totalCount = sourceCount +& targetCount - assert(source =/= target && sourceCount =/= 0.U, "Shared bank transfer requires a distinct allocated source") - assert(totalCount <= totalBanks.U, "Shared bank transfer exceeds physical bank count") - for ((entry, physical) <- mappingTable.zipWithIndex) { - when(entry.valid && entry.hart_id === owner && entry.vbank_id === source) { - leases.foreach(counts => assert(counts(physical) === 0.U, "Shared bank transfer source has live exports")) - entry.vbank_id := target - entry.group_id := targetCount + entry.group_id - entry.is_multi := totalCount > 1.U - } - when(entry.valid && entry.hart_id === owner && entry.vbank_id === target) { - entry.is_multi := totalCount > 1.U - } - } - }.elsewhen(io.config.bits.alloc) { - when(io.config.bits.group_id === 0.U) { - clearVbank(io.config.bits.hart_id, io.config.bits.vbank_id) - } - val pbankId = PriorityEncoder(freePbankMask) - hashMonitors.foreach { hashes => - for (i <- 0 until totalBanks) { - hashes(i).io.bind := pbankId === i.U - } - } - printf( - p"[SharedMemBackend][ALLOC] hart=${io.config.bits.hart_id} vbank=0x${Hexadecimal(io.config.bits.vbank_id)} " + - p"group=${io.config.bits.group_id} pbank=$pbankId is_multi=${io.config.bits.is_multi}\n" - ) - addEntry( - io.config.bits.hart_id, - io.config.bits.vbank_id, - pbankId, - io.config.bits.is_multi, - io.config.bits.group_id - ) - }.otherwise { - printf( - p"[SharedMemBackend][RELEASE] hart=${io.config.bits.hart_id} vbank=0x${Hexadecimal(io.config.bits.vbank_id)}\n" - ) - deleteEntry(io.config.bits.hart_id, io.config.bits.vbank_id) - } - } - - // ----------------------------------------------------------------------------- - // Query interface: return group count for a given vbank_id - // ----------------------------------------------------------------------------- - for (q <- 0 until nCores) { - val queryValid = RegNext(io.query_valid(q), false.B) - val queryHart = RegNext(io.query_hart_id(q), 0.U) - val queryVbank = RegNext(io.query_vbank_id(q), 0.U) - val groupCounts = mappingTable.map { entry => - val matches = queryValid && - entry.valid && - (entry.hart_id === queryHart) && - (entry.vbank_id === queryVbank) - val count = Mux(entry.is_multi, entry.group_id +& 1.U, 1.U) - Mux(matches, count, 0.U) - } - - io.query_group_count(q) := RegNext(groupCounts.reduce((a, b) => Mux(a > b, a, b)), 0.U) - val queryHit = groupCounts.map(_ =/= 0.U).reduce(_ || _) - when(queryValid && !queryHit) { - printf( - p"[SharedMemBackend][QUERY_MISS] q=$q hart=$queryHart " + - p"vbank=0x${Hexadecimal(queryVbank)} returned group_count=0\n" - ) - } - } - - // ----------------------------------------------------------------------------- - // Connect AccPipe and Banks - // ----------------------------------------------------------------------------- - private def emitTrace( - ch: Int, - isWrite: UInt, - pbankId: UInt, - addr: UInt, - writeMask: UInt, - dataLo: UInt, - dataHi: UInt, - en: Bool - ): Unit = { - mtraces(ch).io.is_write := isWrite - mtraces(ch).io.is_shared := io.mem_req(ch).is_shared.asUInt - mtraces(ch).io.channel := ch.U - mtraces(ch).io.hart_id := io.mem_req(ch).hart_id - mtraces(ch).io.rob_id := io.mem_req(ch).rob_id - mtraces(ch).io.inst_id := io.mem_req(ch).inst_id - mtraces(ch).io.vbank_id := io.mem_req(ch).bank_id - mtraces(ch).io.pbank_id := pbankId - mtraces(ch).io.group_id := io.mem_req(ch).group_id - mtraces(ch).io.addr := addr - mtraces(ch).io.write_mask := writeMask - mtraces(ch).io.data_lo := dataLo - mtraces(ch).io.data_hi := dataHi - mtraces(ch).io.enable := en - } - - for (i <- 0 until totalChannel) { - val activeHart = Mux(accPipes(i).io.busy, accPipes(i).io.hart_id, io.mem_req(i).hart_id) - val activeBank = Mux(accPipes(i).io.busy, accPipes(i).io.bank_id, io.mem_req(i).bank_id) - val activeGroup = Mux(accPipes(i).io.busy, accPipes(i).io.group_id, io.mem_req(i).group_id) - val req_valid = io.mem_req(i).read.req.valid || io.mem_req(i).write.req.valid || accPipes(i).io.busy - - val tracePbankId = Wire(UInt(32.W)) - tracePbankId := 0.U - for (j <- 0 until totalBanks) { - val trace_hit_bank = mappingTable(j).valid && - (mappingTable(j).hart_id === activeHart) && - (mappingTable(j).vbank_id === activeBank) && - (!mappingTable(j).is_multi || - (mappingTable(j).is_multi && (mappingTable(j).group_id === activeGroup))) - when(trace_hit_bank) { - tracePbankId := j.U - } - } - - // Memory trace: read request - when(io.mem_req(i).read.req.fire) { - emitTrace(i, 0.U, tracePbankId, io.mem_req(i).read.req.bits.addr, 0.U, 0.U, 0.U, true.B) - } - - // Arrival trace: observe acceptance at the selected shared SPM SRAM port. - when(accPipes(i).io.sramWrite.req.fire) { - emitTrace( - i, - 1.U, - tracePbankId, - accPipes(i).io.sramWrite.req.bits.addr, - accPipes(i).io.sramWrite.req.bits.mask.asUInt, - accPipes(i).io.sramWrite.req.bits.data(63, 0), - accPipes(i).io.sramWrite.req.bits.data(127, 64), - true.B - ) - } - - if (useMesh) { - val channel = mesh.get.io.channels(i) - val mapped = mappingTable.map(entry => - entry.valid && - entry.hart_id === activeHart && - entry.vbank_id === activeBank && - (!entry.is_multi || entry.group_id === activeGroup) - ).reduce(_ || _) - val writeReq = accPipes(i).io.sramWrite.req.valid - val nextTag = RegInit(0.U(tagBits.W)) - - channel.request.valid := mapped && (writeReq || accPipes(i).io.sramRead.req.valid) - channel.request.bits.bank := tracePbankId - channel.request.bits.tuser := Cat( - Mux(writeReq, accPipes(i).io.sramWrite.req.bits.addr, accPipes(i).io.sramRead.req.bits.addr), - writeReq - ) - channel.request.bits.tlast := true.B - channel.request.bits.data := accPipes(i).io.sramWrite.req.bits.data - channel.request.bits.mask := accPipes(i).io.sramWrite.req.bits.mask.asUInt - channel.request.bits.tag := nextTag - accPipes(i).io.sramRead.req.ready := mapped && !writeReq && channel.request.ready - accPipes(i).io.sramWrite.req.ready := mapped && channel.request.ready - accPipes(i).io.sramRead.resp.valid := channel.response.valid && !channel.response.bits.write - accPipes(i).io.sramRead.resp.bits.data := channel.response.bits.data - accPipes(i).io.sramWrite.resp.valid := channel.response.valid && channel.response.bits.write - accPipes(i).io.sramWrite.resp.bits.ok := !channel.response.bits.error - channel.response.ready := Mux( - channel.response.bits.write, - accPipes(i).io.sramWrite.resp.ready, - accPipes(i).io.sramRead.resp.ready - ) - when(channel.request.fire) { - nextTag := nextTag + 1.U - } - } else { - for (j <- 0 until totalBanks) { - val hit_bank = mappingTable(j).valid && - (mappingTable(j).hart_id === activeHart) && - (mappingTable(j).vbank_id === activeBank) && - (!mappingTable(j).is_multi || - (mappingTable(j).is_multi && (mappingTable(j).group_id === activeGroup))) - - when(hit_bank && req_valid) { - banks(j).io.sramRead <> accPipes(i).io.sramRead - banks(j).io.sramWrite <> accPipes(i).io.sramWrite - hashMonitors.foreach { hashes => - val monitor = hashes(j) - val physical = banks(j).io.sramWrite.req - val offered = accPipes(i).io.sramWrite.req - physical.valid := offered.valid && monitor.io.write.ready - monitor.io.write.valid := offered.valid && physical.ready - offered.ready := physical.ready && monitor.io.write.ready - } - } - } - } - } - - mesh.foreach(_.io.bankWrites.foreach(_.ready := true.B)) - - hashMonitors.foreach { hashes => - for (j <- 0 until totalBanks) { - val monitor = hashes(j) - if (useMesh) { - val write = mesh.get.io.bankWrites(j) - write.ready := monitor.io.write.ready - monitor.io.write.valid := write.valid - monitor.io.write.bits.addr := write.bits.addr - monitor.io.write.bits.mask := write.bits.mask.asBools - monitor.io.write.bits.data := write.bits.data - - } - } - } - - io.bank_hashes.foreach { states => - for (i <- 0 until totalBanks) { - states(i).valid := mappingTable(i).valid - states(i).hartId := mappingTable(i).hart_id - states(i).vbankId := mappingTable(i).vbank_id - states(i).pbankId := i.U - states(i).groupId := mappingTable(i).group_id - states(i).statusHash := hashMonitors.get(i).io.statusHash - } - } -} diff --git a/arch/src/main/scala/framework/memdomain/boot/MemBoot.scala b/arch/src/main/scala/framework/memdomain/boot/MemBoot.scala deleted file mode 100644 index d583b491d..000000000 --- a/arch/src/main/scala/framework/memdomain/boot/MemBoot.scala +++ /dev/null @@ -1,16 +0,0 @@ -package framework.memdomain.boot - -import framework.memdomain.isa.MemISA -import framework.system.core.rocket.BuckyballCommand -import framework.top.GlobalConfig - -/** Internal MemDomain boot command program for one accelerator instance. */ -object MemBoot { - val BootBankId = 0 - - def initializationCommands(b: GlobalConfig): Seq[BuckyballCommand] = - Seq(MemISA.mset(MemISA.MsetArgs(BootBankId, columns = b.memDomain.bankNum, alloc = true, clear = true))) - - def releaseCommands(b: GlobalConfig): Seq[BuckyballCommand] = - Seq(MemISA.mset(MemISA.MsetArgs(BootBankId, columns = 0, alloc = false, clear = false))) -} diff --git a/arch/src/main/scala/framework/memdomain/configs/MemDomainParam.scala b/arch/src/main/scala/framework/memdomain/configs/MemDomainParam.scala deleted file mode 100644 index 4ab76565b..000000000 --- a/arch/src/main/scala/framework/memdomain/configs/MemDomainParam.scala +++ /dev/null @@ -1,89 +0,0 @@ -package framework.memdomain.configs - -import upickle.default._ - -/** - * MemDomain Parameter - * - * Real values are injected by `ChipLoader`. The no-arg `apply()` is a - * stub used only by `GlobalConfig()` before bundle overrides land — matching - * how `BallDomainParam()` returns an empty default. - */ -case class MemDomainParam( - bankNum: Int, - bankWidth: Int, - bankEntries: Int, - bankMaskLen: Int, - virtualBankCount: Int, - sharedEnable: Boolean, - sharedEntries: Int, - sharedBankEntries: Int, - sharedBankNum: Int, - sharedInputChannels: Int, - sharedDefaultGroupCount: Int, - nCores: Int, - computeCoreIds: Seq[Int], - tlb_size: Int, - dma_n_xacts: Int, - dma_burst_maxbytes: Int, - bankChannel: Int, - max_in_flight_mem_reqs: Int, - dma_buswidth: Int, - memAddrLen: Int, - tmaReadChannel: Int, - tmaWriteChannel: Int, - mmioEnable: Boolean, - mmioBankNum: Int, - mmioBankEntries: Int, - mmioBankWidth: Int, - mmioReadWidth: Int) { - - if (sharedEnable) { - require(sharedBankEntries >= 2 && sharedBankEntries <= 65536) - require(sharedEntries == sharedBankNum * sharedBankEntries, "Shared pool size must equal banks times rows per bank") - } - - // MMIO derived values - val mmioBankBytes: Int = mmioBankEntries * (mmioBankWidth / 8) - val mmioTotalBytes: Int = mmioBankNum * mmioBankBytes - val vbankIdWidth: Int = chisel3.util.log2Up(virtualBankCount) - val maxGroupCount: Int = math.max(bankNum, if (sharedEnable) sharedBankNum else bankNum) - val groupCountWidth: Int = chisel3.util.log2Up(maxGroupCount + 1) - val groupIdWidth: Int = chisel3.util.log2Up(maxGroupCount) -} - -object MemDomainParam { - implicit val rw: ReadWriter[MemDomainParam] = macroRW - - /** Stub default; real values come from ChipLoader. */ - def apply(): MemDomainParam = MemDomainParam( - bankNum = 0, - bankWidth = 0, - bankEntries = 0, - bankMaskLen = 0, - virtualBankCount = 0, - sharedEnable = false, - sharedEntries = 0, - sharedBankEntries = 0, - sharedBankNum = 0, - sharedInputChannels = 0, - sharedDefaultGroupCount = 0, - nCores = 0, - computeCoreIds = Seq.empty, - tlb_size = 0, - dma_n_xacts = 0, - dma_burst_maxbytes = 0, - bankChannel = 0, - max_in_flight_mem_reqs = 0, - dma_buswidth = 0, - memAddrLen = 0, - tmaReadChannel = 0, - tmaWriteChannel = 0, - mmioEnable = false, - mmioBankNum = 0, - mmioBankEntries = 0, - mmioBankWidth = 0, - mmioReadWidth = 0 - ) - -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/MemFrontend.scala b/arch/src/main/scala/framework/memdomain/frontend/MemFrontend.scala deleted file mode 100644 index 6961a069a..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/MemFrontend.scala +++ /dev/null @@ -1,263 +0,0 @@ -package framework.memdomain.frontend - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import framework.memdomain.frontend.mem.dma.{DmaError, DmaPort, DmaReadCommand} -import framework.memdomain.frontend.mem.{ - Footprint, - KernelDma, - KernelDmaPort, - MemConfiger, - MemConfigerIO, - MemLoader, - MemStorer -} -import framework.frontend.globalrs.{GlobalSchedComplete, GlobalSchedIssue} -import framework.frontend.decoder.GISA.MVIN_KERNEL_BITPAT -import framework.balldomain.blink.{BankRead, BankWrite} -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.memdomain.frontend.cmd.decoder.MemDomainDecoder -import framework.memdomain.frontend.cmd.rs.MemReservationStation -import framework.memdomain.utils.pmc.MemCyclePMC -import framework.memdomain.isa.{MvoverISA, MvoverPort} - -/** - * MemFrontend: - * Provides DMA interface and Ball Domain interface - */ -@instantiable -class MemFrontend(val b: GlobalConfig) extends Module { - - @public - val io = IO(new Bundle { - // Issue interface from global RS (single channel) - val global_issue_i = Flipped(Decoupled(new GlobalSchedIssue(b))) - // Report completion to global RS (single channel) - val global_complete_o = Decoupled(new GlobalSchedComplete(b)) - val mvover = new MvoverPort - - // Bank read/write interface - used by load/store - val interdma = new Bundle { - val bankRead = Flipped(new BankRead(b)) - val bankWrite = Flipped(new BankWrite(b)) - val read_is_shared = Output(Bool()) - val write_is_shared = Output(Bool()) - } - - val dma = new DmaPort(b.memDomain.dma_buswidth) - - val config = Decoupled(new MemConfigerIO(b)) - - // MMIO outputs (to MmioPool via MemDomain) - val is_mvin_mmio_active = Output(Bool()) - val mmio_addr = Output(UInt(17.W)) - val mmio_col = Output(UInt(8.W)) - - // Query interface to backend for group count - val query_vbank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val query_is_shared = Output(Bool()) - val query_group_count = Input(UInt(b.memDomain.groupCountWidth.W)) - val clearBusy = Input(Bool()) - - val hartid = Input(UInt(b.tile.xLen.W)) - val footprints = Output(Vec(3, new Footprint(b))) - - val kernel = if (b.rvv.enable) Some(new KernelDmaPort(b)) else None - - // Busy signal - val busy = Output(Bool()) - }) - - val memDecoder: Instance[MemDomainDecoder] = Instantiate(new MemDomainDecoder(b)) - val memRs: Instance[MemReservationStation] = Instantiate(new MemReservationStation(b)) - val memLoader: Instance[MemLoader] = Instantiate(new MemLoader(b)) - val memStorer: Instance[MemStorer] = Instantiate(new MemStorer(b)) - val pmc: Instance[MemCyclePMC] = Instantiate(new MemCyclePMC(b)) - - val configer: Instance[MemConfiger] = Instantiate(new MemConfiger(b)) - -// ----------------------------------------------------------------------------- -// Global RS -> MemDecoder -// ----------------------------------------------------------------------------- - val isMvover = io.global_issue_i.bits.cmd.cmd.funct === MvoverISA.Funct.U - val mvoverPending = RegInit(false.B) - val mvoverRobId = Reg(chiselTypeOf(io.global_issue_i.bits.rob_id)) - val mvoverIsSub = Reg(Bool()) - val mvoverSubRobId = Reg(chiselTypeOf(io.global_issue_i.bits.sub_rob_id)) - - io.mvover.command.valid := io.global_issue_i.valid && isMvover && !mvoverPending - io.mvover.command.bits.sourceCore := io.global_issue_i.bits.cmd.cmd.rs1Data(7, 0) - io.mvover.command.bits.targetCore := io.global_issue_i.bits.cmd.cmd.rs1Data(15, 8) - io.mvover.command.bits.sourceBank := io.global_issue_i.bits.cmd.cmd.rs1Data(25, 16) - io.mvover.command.bits.targetBank := io.global_issue_i.bits.cmd.cmd.rs1Data(35, 26) - io.mvover.command.bits.sourceAddr := io.global_issue_i.bits.cmd.cmd.rs2Data(15, 0) - io.mvover.command.bits.targetAddr := io.global_issue_i.bits.cmd.cmd.rs2Data(31, 16) - io.mvover.command.bits.rows := io.global_issue_i.bits.cmd.cmd.rs2Data(47, 32) +& 1.U - when(io.mvover.command.fire) { - mvoverPending := true.B - mvoverRobId := io.global_issue_i.bits.rob_id - mvoverIsSub := io.global_issue_i.bits.is_sub - mvoverSubRobId := io.global_issue_i.bits.sub_rob_id - } - memDecoder.io.cmd_i.valid := io.global_issue_i.valid && !isMvover && !mvoverPending - memDecoder.io.cmd_i.bits := io.global_issue_i.bits.cmd - io.global_issue_i.ready := !mvoverPending && - Mux(isMvover, io.mvover.command.ready, memDecoder.io.cmd_i.ready) - - // Config signal goes to backend - io.config <> configer.io.config - - val selectStorerQuery = memStorer.io.query_valid - io.query_vbank_id := Mux(selectStorerQuery, memStorer.io.query_vbank_id, memLoader.io.query_vbank_id) - io.query_is_shared := Mux(selectStorerQuery, memStorer.io.query_is_shared, memLoader.io.query_is_shared) - memLoader.io.query_group_count := io.query_group_count - memStorer.io.query_group_count := io.query_group_count - - when( - memLoader.io.query_valid && memStorer.io.query_valid && - memLoader.io.query_is_shared && memStorer.io.query_is_shared && - memLoader.io.query_vbank_id =/= memStorer.io.query_vbank_id - ) { - assert(false.B, "MemFrontend shared query conflict: loader and storer query different vbanks in the same cycle\n") - } - -// ----------------------------------------------------------------------------- -// MemDecoder -> MemReservationStation -// ----------------------------------------------------------------------------- - // Connect decoded instruction and global rob_id - memRs.io.mem_decode_cmd_i.valid := memDecoder.io.mem_decode_cmd_o.valid - memRs.io.mem_decode_cmd_i.bits.cmd := memDecoder.io.mem_decode_cmd_o.bits - memRs.io.mem_decode_cmd_i.bits.rob_id := io.global_issue_i.bits.rob_id - memRs.io.mem_decode_cmd_i.bits.is_sub := io.global_issue_i.bits.is_sub - memRs.io.mem_decode_cmd_i.bits.sub_rob_id := io.global_issue_i.bits.sub_rob_id - memDecoder.io.mem_decode_cmd_o.ready := memRs.io.mem_decode_cmd_i.ready - -// ----------------------------------------------------------------------------- -// MemReservationStation -> MemLoader/MemStorer -// ----------------------------------------------------------------------------- - memLoader.io.cmdReq <> memRs.io.issue_o.ld - memStorer.io.cmdReq <> memRs.io.issue_o.st - configer.io.cmdReq <> memRs.io.issue_o.cf - configer.io.hartid := io.hartid - configer.io.clearBusy := io.clearBusy - memRs.io.commit_i.ld <> memLoader.io.cmdResp - memRs.io.commit_i.st <> memStorer.io.cmdResp - memRs.io.commit_i.cf <> configer.io.cmdResp - -//===-------------------------------------------------------------------===//-- -// PMC - Performance Monitor Counter -// ----------------------------------------------------------------------------- - pmc.io.ldReq_i.valid := memRs.io.issue_o.ld.fire - pmc.io.ldReq_i.bits := memRs.io.issue_o.ld.bits - pmc.io.stReq_i.valid := memRs.io.issue_o.st.fire - pmc.io.stReq_i.bits := memRs.io.issue_o.st.bits - pmc.io.ldResp_o.valid := memLoader.io.cmdResp.fire - pmc.io.ldResp_o.bits := memLoader.io.cmdResp.bits - pmc.io.stResp_o.valid := memStorer.io.cmdResp.fire - pmc.io.stResp_o.bits := memStorer.io.cmdResp.bits - - // A logical request is accepted into this queue before its external transport fires. - // Keep both producer identity and transfer stable while the transport is backpressured. - val readQueue = Module(new Queue(new DmaReadCommand, 1)) - val readerOwned = RegInit(false.B) - val kernelOwner = RegInit(false.B) - io.dma.read.valid := readQueue.io.deq.valid && !readerOwned - io.dma.read.bits := readQueue.io.deq.bits - readQueue.io.deq.ready := io.dma.read.ready && !readerOwned - when(io.dma.read.fire) { - readerOwned := true.B - kernelOwner := io.dma.read.bits.producer === 2.U - } - when(io.dma.readResult.fire && io.dma.readResult.bits.last)(readerOwned := false.B) - when(io.dma.readResult.valid) { - assert(readerOwned || io.dma.read.fire, "MemFrontend DMA read result has no accepted transport owner") - } - io.footprints(0) := memLoader.io.footprint - io.footprints(1) := memStorer.io.footprint - io.footprints(2) := 0.U.asTypeOf(new Footprint(b)) - if (b.rvv.enable) { - val kernelDma: Instance[KernelDma] = Instantiate(new KernelDma(b)) - io.footprints(2) := kernelDma.io.footprint - kernelDma.io.kernel <> io.kernel.get - val selectKernel = kernelDma.io.request.valid - readQueue.io.enq.valid := kernelDma.io.request.valid || memLoader.io.dmaReq.valid - readQueue.io.enq.bits.transfer := Mux(selectKernel, kernelDma.io.request.bits, memLoader.io.dmaReq.bits) - readQueue.io.enq.bits.producer := Mux(selectKernel, 2.U, 0.U) - kernelDma.io.request.ready := selectKernel && readQueue.io.enq.ready - memLoader.io.dmaReq.ready := !selectKernel && readQueue.io.enq.ready - // KernelDma treats enqueue acceptance as an obligation to drain even after abort. - kernelDma.io.response.valid := readerOwned && kernelOwner && io.dma.readResult.valid - kernelDma.io.response.bits := io.dma.readResult.bits - memLoader.io.dmaResp.valid := readerOwned && !kernelOwner && io.dma.readResult.valid - memLoader.io.dmaResp.bits := io.dma.readResult.bits - io.dma.readResult.ready := readerOwned && Mux(kernelOwner, kernelDma.io.response.ready, memLoader.io.dmaResp.ready) - } else { - readQueue.io.enq.valid := memLoader.io.dmaReq.valid - readQueue.io.enq.bits.transfer := memLoader.io.dmaReq.bits - readQueue.io.enq.bits.producer := 0.U - memLoader.io.dmaReq.ready := readQueue.io.enq.ready - memLoader.io.dmaResp.valid := readerOwned && io.dma.readResult.valid - memLoader.io.dmaResp.bits := io.dma.readResult.bits - io.dma.readResult.ready := readerOwned && memLoader.io.dmaResp.ready - } - io.dma.write <> memStorer.io.dmaReq - io.dma.writeData <> memStorer.io.dmaData - memStorer.io.dmaResp <> io.dma.writeResult - - // MemLoader owns the frontend write channel until its write response returns, - // preserving BankWrite's request/response pairing. - val frontendWriteBusy = RegInit(false.B) - - io.interdma.bankWrite.io.req.valid := !frontendWriteBusy && memLoader.io.bankWrite.io.req.valid - io.interdma.bankWrite.io.req.bits := memLoader.io.bankWrite.io.req.bits - io.interdma.bankWrite.bank_id := memLoader.io.bankWrite.bank_id - io.interdma.bankWrite.rob_id := memLoader.io.bankWrite.rob_id - io.interdma.bankWrite.ball_id := memLoader.io.bankWrite.ball_id - io.interdma.bankWrite.group_id := memLoader.io.bankWrite.group_id - memLoader.io.bankWrite.io.req.ready := !frontendWriteBusy && io.interdma.bankWrite.io.req.ready - - io.interdma.bankWrite.io.resp.ready := frontendWriteBusy && memLoader.io.bankWrite.io.resp.ready - memLoader.io.bankWrite.io.resp.valid := frontendWriteBusy && io.interdma.bankWrite.io.resp.valid - memLoader.io.bankWrite.io.resp.bits := io.interdma.bankWrite.io.resp.bits - - when(io.interdma.bankWrite.io.req.fire)(frontendWriteBusy := true.B) - when(io.interdma.bankWrite.io.resp.fire)(frontendWriteBusy := false.B) - - memStorer.io.bankRead <> io.interdma.bankRead - io.interdma.read_is_shared := memStorer.io.is_shared - io.interdma.write_is_shared := memLoader.io.is_shared - - // MMIO signals from MemConfiger and MemLoader, exposed to MemDomain - io.is_mvin_mmio_active := memLoader.io.is_mvin_mmio_active - io.mmio_addr := memLoader.io.mmio_addr - io.mmio_col := memLoader.io.mmio_col - - // Completion signal connected to global RS - val mvoverComplete = mvoverPending && io.mvover.completion.valid - io.global_complete_o.valid := mvoverComplete || memRs.io.complete_o.valid - io.global_complete_o.bits.rob_id := Mux(mvoverComplete, mvoverRobId, memRs.io.complete_o.bits.rob_id) - io.global_complete_o.bits.is_sub := Mux(mvoverComplete, mvoverIsSub, memRs.io.complete_o.bits.is_sub) - io.global_complete_o.bits.sub_rob_id := Mux(mvoverComplete, mvoverSubRobId, memRs.io.complete_o.bits.sub_rob_id) - io.global_complete_o.bits.fault := Mux( - mvoverComplete, - 0.U.asTypeOf(io.global_complete_o.bits.fault), - memRs.io.complete_o.bits.fault - ) - when(mvoverComplete && io.mvover.completion.bits) { - io.global_complete_o.bits.fault.error := DmaError.Bank.U - } - - memRs.io.complete_o.ready := io.global_complete_o.ready && !mvoverComplete - io.mvover.completion.ready := io.global_complete_o.ready && mvoverPending - when(io.mvover.completion.fire) { - mvoverPending := false.B - } - - // Busy signal - // Simple busy signal - io.busy := mvoverPending || !memRs.io.complete_o.ready || io.kernel.map(_.busy).getOrElse(false.B) || - readQueue.io.deq.valid || readerOwned || io.dma.readBusy || io.dma.writeBusy -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/cmd/decoder/DISA.scala b/arch/src/main/scala/framework/memdomain/frontend/cmd/decoder/DISA.scala deleted file mode 100644 index ea91c3892..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/cmd/decoder/DISA.scala +++ /dev/null @@ -1,17 +0,0 @@ -package framework.memdomain.frontend.cmd.decoder - -import chisel3._ -import chisel3.util._ -import framework.memdomain.isa.MemISA - -object DISA { - val MVOVER_BITPAT = BitPat("b0001101") - val MSET_FUNCT = MemISA.MsetFunct - // enable=010 (1 write), opcode group - val MSET_BITPAT = BitPat("b0100000") // 32 (0x20) — enable=010, opcode=0 - val MVIN_BITPAT = BitPat("b0100001") // 33 (0x21) — enable=010, opcode=1 - val MVIN_2D_BITPAT = BitPat("b0100010") // 34 (0x22) — 2D DMA into bank rows - val MVIN_MMIO_BITPAT = BitPat("b0100011") // 35 (0x23) — enable=010, opcode=3 (DMA into MMIO) - // enable=001 (1 read), opcode group - val MVOUT_BITPAT = BitPat("b0010000") // 16 (0x10) — enable=001, opcode=0 -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/cmd/decoder/DomainDecoder.scala b/arch/src/main/scala/framework/memdomain/frontend/cmd/decoder/DomainDecoder.scala deleted file mode 100644 index cce719409..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/cmd/decoder/DomainDecoder.scala +++ /dev/null @@ -1,162 +0,0 @@ -package framework.memdomain.frontend.cmd.decoder - -import chisel3._ -import chisel3.util._ -import framework.frontend.decoder.{DomainId, PostGDCmd} -import framework.top.GlobalConfig -import framework.memdomain.frontend.cmd.decoder.DISA._ -import freechips.rocketchip.tile._ -import chisel3.experimental.hierarchy.{instantiable, public} - -// Detailed decode output for Mem domain -class MemDecodeCmd(b: GlobalConfig) extends Bundle { - // Shared memory access marker: raw BB_BANK0 id >= frontend.sharedBankIdBase. - val is_shared = Bool() - val is_load = Bool() - val is_store = Bool() - val is_config = Bool() - val clear = Bool() - val transfer = Bool() - val source_bank_id = UInt(b.memDomain.vbankIdWidth.W) - - val is_mvin_mmio = Bool() - val is_mvin_2d = Bool() - - val mem_addr = UInt(b.memDomain.memAddrLen.W) - val iter = UInt(b.frontend.iter_len.W) - val bank_id = UInt(b.memDomain.vbankIdWidth.W) - val special = UInt(64.W) -} - -// LS decode fields (iter removed from decode table — always from rs1[63:48]) -object LSDecodeFields extends Enumeration { - type Field = Value - val LD_EN, ST_EN, MEMADDR, BANK_ID, SPECIAL, VALID = Value -} - -// Default constants for Mem decoder -object MemDefaultConstants { - val Y = true.B - val N = false.B - val DADDR = 0.U(15.W) - val DSPECIAL = 0.U(64.W) -} - -@instantiable -class MemDomainDecoder(val b: GlobalConfig) extends Module { - import MemDefaultConstants._ - - val sharedBankIdBase = b.frontend.shared_bank_id_base - require(sharedBankIdBase > 0, s"sharedBankIdBase($sharedBankIdBase) must be > 0") - require( - sharedBankIdBase < (1 << b.frontend.bank_id_len), - s"sharedBankIdBase($sharedBankIdBase) must fit in bankIdLen(${b.frontend.bank_id_len})" - ) - require( - sharedBankIdBase > b.frontend.vbank_id_upper_bound, - s"sharedBankIdBase($sharedBankIdBase) must be > vbankIdUpperBound(${b.frontend.vbank_id_upper_bound})" - ) - - @public - val io = IO(new Bundle { - val cmd_i = Flipped(Decoupled(new PostGDCmd(b))) - val mem_decode_cmd_o = Decoupled(new MemDecodeCmd(b)) - }) - - val memAddrLen = b.memDomain.memAddrLen - val bankIdLen = b.frontend.bank_id_len - val iterLen = b.frontend.iter_len - - // Only process Mem instructions - io.cmd_i.ready := io.mem_decode_cmd_o.ready - - val func7 = io.cmd_i.bits.cmd.funct - val rs1 = io.cmd_i.bits.cmd.rs1Data - val rs2 = io.cmd_i.bits.cmd.rs2Data - - // Unified encoding: - // rs1[9:0] = read/config bank (BANK0) - // rs1[29:20] = write bank (BANK2) - // rs1[63:30] = iter (34-bit) - // funct7[6:4] = enable (bank access flags, decoded by GlobalDecoder) - // rs2[38:0] = mem_addr (for MVIN/MVOUT, 39-bit) - // - // mvin_mmio: rs1[63:30]=row (iter), rs2[38:0]=dram_addr, rs2[55:39]=mmio_addr, rs2[63:56]=col - import LSDecodeFields._ - // format: off - val ls_default_decode = List(N, N, DADDR, DADDR, DSPECIAL, N) - val ls_decode_list = ListLookup(func7, ls_default_decode, Array( - MSET_BITPAT -> List(N, N, 0.U(memAddrLen.W), rs1(bankIdLen - 1, 0), rs2, Y), - MVIN_BITPAT -> List(Y, N, rs2(memAddrLen - 1, 0), rs1(bankIdLen + 19, 20), rs2, Y), - MVIN_2D_BITPAT -> List(Y, N, Cat(rs2(35, 0), 0.U(3.W)), rs1(bankIdLen + 19, 20), rs2, Y), - MVOUT_BITPAT -> List(N, Y, rs2(memAddrLen - 1, 0), rs1(bankIdLen - 1, 0), rs2, Y), - MVIN_MMIO_BITPAT -> List(Y, N, rs2(memAddrLen - 1, 0), 0.U(bankIdLen.W), rs2, Y) - ) - ) - // format: on - - assert( - !(io.cmd_i.fire && !ls_decode_list(LSDecodeFields.VALID.id).asBool), - s"MemDomainDecoder: Invalid command opcode, func7 = 0x%x\n", - func7 - ) - -// ----------------------------------------------------------------------------- -// Output assignment -// ----------------------------------------------------------------------------- - io.mem_decode_cmd_o.valid := io.cmd_i.valid && (io.cmd_i.bits.domain_id === DomainId.MEM) - - val transfer = func7 === MSET_BITPAT && rs2(12) - val raw_bank_id = Mux(func7 === MVIN_BITPAT || func7 === MVIN_2D_BITPAT || transfer, rs1(29, 20), rs1(9, 0)) - val shared_bank_id = b.memDomain.sharedEnable.B && raw_bank_id >= sharedBankIdBase.U - // format: off - // BB_BANK0 is encoded in rs1[9:0]. Keep the configured architectural ID; - // private and shared banks occupy disjoint ranges in that namespace. - io.mem_decode_cmd_o.bits.is_shared := io.mem_decode_cmd_o.valid && shared_bank_id - io.mem_decode_cmd_o.bits.is_load := Mux(io.mem_decode_cmd_o.valid, ls_decode_list(LSDecodeFields.LD_EN.id).asBool, false.B) - io.mem_decode_cmd_o.bits.is_store := Mux(io.mem_decode_cmd_o.valid, ls_decode_list(LSDecodeFields.ST_EN.id).asBool, false.B) - io.mem_decode_cmd_o.bits.is_config := Mux(io.mem_decode_cmd_o.valid, func7 === MSET_BITPAT, false.B) - io.mem_decode_cmd_o.bits.transfer := io.mem_decode_cmd_o.valid && transfer - io.mem_decode_cmd_o.bits.source_bank_id := rs1(bankIdLen - 1, 0) - io.mem_decode_cmd_o.bits.clear := io.mem_decode_cmd_o.valid && (func7 === MSET_BITPAT) && rs2(11) - io.mem_decode_cmd_o.bits.is_mvin_mmio := Mux(io.mem_decode_cmd_o.valid, func7 === MVIN_MMIO_BITPAT, false.B) - io.mem_decode_cmd_o.bits.is_mvin_2d := Mux(io.mem_decode_cmd_o.valid, func7 === MVIN_2D_BITPAT, false.B) - io.mem_decode_cmd_o.bits.mem_addr := Mux(io.mem_decode_cmd_o.valid, ls_decode_list(LSDecodeFields.MEMADDR.id).asUInt, 0.U(b.memDomain.memAddrLen.W)) - io.mem_decode_cmd_o.bits.iter := Mux(io.mem_decode_cmd_o.valid, rs1(63, 30), 0.U(iterLen.W)) - - when(io.cmd_i.fire && func7 === MSET_BITPAT) { - when(transfer) { - assert(rs1(63, 30) === 0.U && rs1(19, 10) === 0.U, "MSET transfer reserves rs1 outside BANK0/BANK2") - assert(rs2 === (1.U << 12), "MSET transfer reserves other rs2 bits") - assert(rs1(9, 0) =/= rs1(29, 20), "MSET transfer requires distinct source and target") - assert((rs1(9, 0) >= sharedBankIdBase.U) === (rs1(29, 20) >= sharedBankIdBase.U), "MSET transfer requires one bank namespace") - }.otherwise { - assert(rs1(63, 10) === 0.U, "MSET reserves rs1[63:10]") - assert(rs2(63, 12) === 0.U, "MSET reserves rs2[63:12]") - } - assert(!(rs2(11) && !rs2(10)), "MSET clear requires alloc=1") - } - when(io.cmd_i.fire && (func7 === MVIN_BITPAT || func7 === MVOUT_BITPAT)) { - assert(rs2(63) || rs2(62, 58) === 0.U, "Whole-bank DMA reserves selected group bits") - } - when(io.cmd_i.fire && func7 === MVIN_2D_BITPAT) { - val height = rs1(63, 30) - val pixelBytes = rs2(42, 36) << 3 - val width = rs2(61, 59) +& 1.U - val validBytes = Mux(rs2(62), 8.U(5.W), 16.U(5.W)) - assert(height =/= 0.U, "MVIN_2D height must be non-zero") - assert(rs2(42, 36) =/= 0.U, "MVIN_2D pixel bytes must be non-zero") - assert(rs2(52, 43) =/= 0.U, "MVIN_2D source width must be non-zero") - assert(validBytes <= pixelBytes, "MVIN_2D valid bytes exceed pixel bytes") - assert((rs2(58, 53) +& (height * width)) <= Mux(shared_bank_id, b.memDomain.sharedBankEntries.U, b.memDomain.bankEntries.U), "MVIN_2D destination exceeds bank") - assert(!rs2(63), "MVIN_2D reserves rs2[63]") - } - when(io.cmd_i.fire && (func7 === MVIN_BITPAT || func7 === MVIN_2D_BITPAT)) { - assert(rs1(19, 0) === 0.U, "MVIN write-bank encoding reserves rs1[19:0]") - } - - val ls_bank_id = ls_decode_list(LSDecodeFields.BANK_ID.id).asUInt - io.mem_decode_cmd_o.bits.bank_id := Mux(io.mem_decode_cmd_o.valid, Mux(transfer, raw_bank_id, ls_bank_id), 0.U(b.memDomain.vbankIdWidth.W)) - io.mem_decode_cmd_o.bits.special := Mux(io.mem_decode_cmd_o.valid, ls_decode_list(LSDecodeFields.SPECIAL.id).asUInt, 0.U(64.W)) - // format: on -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/cmd/rs/README.md b/arch/src/main/scala/framework/memdomain/frontend/cmd/rs/README.md deleted file mode 100644 index d72da7ced..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/cmd/rs/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# Reservation Station (RS) Module - -## Overview - -Reservation station module for memory domain instruction scheduling, located at `framework/memdomain/frontend/cmd/rs`. Local Mem RS is a small FIFO scheduler; global reorder lives in Frontend `GlobalROB`. - -## File Structure - -``` -rs/ -└── reservationStation.scala - Memory reservation station -``` - -## MemReservationStation - -### Interface - -```scala -class MemReservationStation(implicit b: CustomBuckyballConfig, p: Parameters) extends Module { - val io = IO(new Bundle { - val mem_decode_cmd_i = Flipped(Decoupled(new MemDecodeCmd)) - val rs_rocc_o = new Bundle { - val resp = Decoupled(new RoCCResponseBB) - val busy = Output(Bool()) - } - val issue_o = new MemIssueInterface - val commit_i = new MemCommitInterface - }) -} -``` - -### Issue Interface - -```scala -class MemIssueInterface(implicit b: CustomBuckyballConfig, p: Parameters) extends Bundle { - val ld = Decoupled(new MemRsIssue) // Load instruction issue - val st = Decoupled(new MemRsIssue) // Store instruction issue -} -``` - -### Commit Interface - -```scala -class MemCommitInterface(implicit b: CustomBuckyballConfig, p: Parameters) extends Bundle { - val ld = Flipped(Decoupled(new MemRsComplete)) // Load completion - val st = Flipped(Decoupled(new MemRsComplete)) // Store completion -} -``` - -## Related Modules - -- [Memory Domain](../README.md) -- [Memory Controller](../mem/README.md) -- [DMA Engines](../dma/README.md) diff --git a/arch/src/main/scala/framework/memdomain/frontend/cmd/rs/reservationStation.scala b/arch/src/main/scala/framework/memdomain/frontend/cmd/rs/reservationStation.scala deleted file mode 100644 index a049aef47..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/cmd/rs/reservationStation.scala +++ /dev/null @@ -1,136 +0,0 @@ -package framework.memdomain.frontend.cmd.rs - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental._ -import framework.memdomain.frontend.cmd.decoder.MemDecodeCmd -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig - -// Mem domain issue interface - includes global rob_id -class MemRsIssue(val b: GlobalConfig) extends Bundle { - val cmd = new MemDecodeCmd(b) - // Global ROB ID - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) -} - -// Mem domain completion interface -class MemRsComplete(val b: GlobalConfig) extends Bundle { - val fault = new framework.memdomain.frontend.mem.dma.DmaStatus - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) -} - -// Mem domain issue interface combination (Load + Store) -class MemIssueInterface(val b: GlobalConfig) extends Bundle { - val ld = Decoupled(new MemRsIssue(b)) - val st = Decoupled(new MemRsIssue(b)) - val cf = Decoupled(new MemRsIssue(b)) -} - -// Mem domain completion interface combination (Load + Store) -class MemCommitInterface(val b: GlobalConfig) extends Bundle { - val ld = Flipped(Decoupled(new MemRsComplete(b))) - val st = Flipped(Decoupled(new MemRsComplete(b))) - val cf = Flipped(Decoupled(new MemRsComplete(b))) -} - -// Local Mem reservation station - simple FIFO scheduler -@instantiable -class MemReservationStation(val b: GlobalConfig) extends Module { - - @public - val io = IO(new Bundle { - - // Decoded instruction input (with global rob_id) - val mem_decode_cmd_i = Flipped(new DecoupledIO(new Bundle { - val cmd = new MemDecodeCmd(b) - // Global ROB ID - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) - })) - - // Rs -> MemLoader/MemStorer - val issue_o = new MemIssueInterface(b) - val commit_i = new MemCommitInterface(b) - - // Output completion signal (with global rob_id, single channel) - val complete_o = Decoupled(new MemRsComplete(b)) - }) - - // Simple FIFO queue, only for buffering - val fifo = Module(new Queue( - new Bundle { - val cmd = new MemDecodeCmd(b) - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) - }, - entries = 4 - )) // Small buffer is sufficient - -// ----------------------------------------------------------------------------- -// Inbound - FIFO enqueue -// ----------------------------------------------------------------------------- - fifo.io.enq <> io.mem_decode_cmd_i - -// ----------------------------------------------------------------------------- -// Outbound - instruction issue (dispatch based on is_load/is_store) -// ----------------------------------------------------------------------------- - val headEntry = fifo.io.deq.bits - - // Load issue - io.issue_o.ld.valid := fifo.io.deq.valid && headEntry.cmd.is_load - io.issue_o.ld.bits.cmd := headEntry.cmd - io.issue_o.ld.bits.rob_id := headEntry.rob_id - io.issue_o.ld.bits.is_sub := headEntry.is_sub - io.issue_o.ld.bits.sub_rob_id := headEntry.sub_rob_id - - // Store issue - io.issue_o.st.valid := fifo.io.deq.valid && headEntry.cmd.is_store - io.issue_o.st.bits.cmd := headEntry.cmd - io.issue_o.st.bits.rob_id := headEntry.rob_id - io.issue_o.st.bits.is_sub := headEntry.is_sub - io.issue_o.st.bits.sub_rob_id := headEntry.sub_rob_id - - // Config issue - io.issue_o.cf.valid := fifo.io.deq.valid && headEntry.cmd.is_config - io.issue_o.cf.bits.cmd := headEntry.cmd - io.issue_o.cf.bits.rob_id := headEntry.rob_id - io.issue_o.cf.bits.is_sub := headEntry.is_sub - io.issue_o.cf.bits.sub_rob_id := headEntry.sub_rob_id - - // FIFO deq.ready - can only dequeue when target unit is ready - fifo.io.deq.ready := - (headEntry.cmd.is_load && io.issue_o.ld.ready) || - (headEntry.cmd.is_store && io.issue_o.st.ready) || - (headEntry.cmd.is_config && io.issue_o.cf.ready) - -// ----------------------------------------------------------------------------- -// Completion signal processing - directly forward to global RS -// ----------------------------------------------------------------------------- - val completeArb = Module(new Arbiter(new MemRsComplete(b), 3)) - - completeArb.io.in(0).valid := io.commit_i.ld.valid - completeArb.io.in(0).bits := io.commit_i.ld.bits - io.commit_i.ld.ready := completeArb.io.in(0).ready - - completeArb.io.in(1).valid := io.commit_i.st.valid - completeArb.io.in(1).bits := io.commit_i.st.bits - io.commit_i.st.ready := completeArb.io.in(1).ready - - completeArb.io.in(2).valid := io.commit_i.cf.valid - completeArb.io.in(2).bits := io.commit_i.cf.bits - io.commit_i.cf.ready := completeArb.io.in(2).ready - - // Forward completion signal (with global rob_id) - io.complete_o.valid := completeArb.io.out.valid - io.complete_o.bits := completeArb.io.out.bits - completeArb.io.out.ready := io.complete_o.ready -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/Footprint.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/Footprint.scala deleted file mode 100644 index cb9f83bab..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/Footprint.scala +++ /dev/null @@ -1,21 +0,0 @@ -package framework.memdomain.frontend.mem - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.memdomain.frontend.mem.dma.DmaStatus - -class Footprint(b: GlobalConfig) extends Bundle { - val valid = Bool() - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val is_sub = Bool() - val sub_rob_id = UInt(log2Up(b.frontend.sub_rob_depth * 4).W) - val baseVA = UInt(64.W) - val rows = UInt(64.W) - val columns = UInt(64.W) - val spanBytes = UInt(64.W) - val columnStride = UInt(64.W) - val rowStride = UInt(64.W) - val write = Bool() - val fault = new DmaStatus -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/KernelDma.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/KernelDma.scala deleted file mode 100644 index de0033ab5..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/KernelDma.scala +++ /dev/null @@ -1,145 +0,0 @@ -package framework.memdomain.frontend.mem - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.memdomain.frontend.mem.dma.{BBReadRequest, BBReadResponse, DmaError, DmaStatus} -import framework.top.GlobalConfig -import framework.balldomain.blink.{BankRead, BankWrite} - -class KernelDmaRequest(b: GlobalConfig) extends Bundle { - val rob_id = UInt(log2Ceil(b.frontend.rob_entries).W) - val address = UInt(64.W) - val bytes = UInt(32.W) -} - -class KernelDmaPort(b: GlobalConfig) extends Bundle { - val load = Flipped(Decoupled(new KernelDmaRequest(b))) - val image = Decoupled(UInt(32.W)) - val result = Decoupled(new DmaStatus) - val abort = Input(Bool()) - val busy = Output(Bool()) -} - -class KernelMemoryBridge(b: GlobalConfig) extends KernelDmaPort(b) { - val bankRead = Vec(b.rvv.memoryPorts, new BankRead(b)) - val bankWrite = Vec(b.rvv.memoryPorts, new BankWrite(b)) - val active = Input(Bool()) - val ready = Output(Bool()) -} - -@instantiable -class KernelDma(val b: GlobalConfig) extends Module { - val beatWords = b.memDomain.dma_buswidth / 32 - val beatBytes = b.memDomain.dma_buswidth / 8 - require(beatWords > 0 && b.memDomain.dma_buswidth % 32 == 0) - - @public - val io = IO(new Bundle { - val footprint = Output(new Footprint(b)) - val kernel = new KernelDmaPort(b) - val request = Decoupled(new BBReadRequest) - val response = Flipped(Decoupled(new BBReadResponse(b.memDomain.dma_buswidth))) - }) - - val idle :: request :: receive :: output :: terminal :: Nil = Enum(5) - val state = RegInit(idle) - val descriptor = Reg(new KernelDmaRequest(b)) - val remaining = Reg(UInt(32.W)) - val data = Reg(UInt(b.memDomain.dma_buswidth.W)) - val words = Reg(UInt(math.max(1, log2Ceil(beatWords + 1)).W)) - val index = Reg(UInt(math.max(1, log2Ceil(beatWords)).W)) - val last = Reg(Bool()) - val discarded = RegInit(false.B) - val result = RegInit(0.U.asTypeOf(new DmaStatus)) - val owner = Reg(UInt(log2Ceil(b.frontend.rob_entries).W)) - - def readSpan(bytes: UInt): UInt = - (bytes.pad(64) + (beatBytes - 1).U) & (~(BigInt(beatBytes) - 1) & ((BigInt(1) << 64) - 1)).U(64.W) - - def invalidShape(address: UInt, bytes: UInt): Bool = { - val end = address +& (readSpan(bytes) - 1.U) - bytes === 0.U || bytes(1, 0) =/= 0.U || end(64) - } - - // The read DMA fills a complete output beat even for a short final payload. - // The preflight layer then aligns this span around the original VA. - io.footprint := 0.U.asTypeOf(new Footprint(b)) - io.footprint.valid := state =/= idle - io.footprint.rob_id := owner - io.footprint.baseVA := descriptor.address - io.footprint.rows := 1.U - io.footprint.columns := 1.U - io.footprint.spanBytes := readSpan(descriptor.bytes) - io.footprint.fault.error := Mux(invalidShape(descriptor.address, descriptor.bytes), DmaError.Shape.U, DmaError.None.U) - io.footprint.fault.address := Mux(io.footprint.fault.error =/= 0.U, descriptor.address, 0.U) - - io.kernel.load.ready := state === idle && !io.kernel.abort - io.kernel.busy := state =/= idle - io.request.valid := state === request && !io.kernel.abort - io.request.bits := 0.U.asTypeOf(io.request.bits) - io.request.bits.vaddr := descriptor.address - io.request.bits.len := descriptor.bytes - io.request.bits.groups := 1.U - io.request.bits.stride := 1.U - io.response.ready := state === receive - io.kernel.image.valid := state === output && !discarded && !io.kernel.abort - io.kernel.image.bits := (data >> (index * 32.U))(31, 0) - io.kernel.result.valid := state === terminal - io.kernel.result.bits := result - - // Abort cancels a load, never an execution. Once offered, the terminal result - // is immutable even if the image parser subsequently rejects the image. - when(io.kernel.abort && state =/= idle && state =/= terminal) { - discarded := true.B - when(result.error === DmaError.None.U) { - result.error := DmaError.Cancelled.U - result.address := descriptor.address - } - when(state === request)(state := terminal) - when(state === output)(state := Mux(last, terminal, receive)) - } - - when(io.kernel.load.fire) { - val invalid = invalidShape(io.kernel.load.bits.address, io.kernel.load.bits.bytes) - owner := io.kernel.load.bits.rob_id - descriptor := io.kernel.load.bits - remaining := io.kernel.load.bits.bytes >> 2 - discarded := false.B - result.error := Mux(invalid, DmaError.Shape.U, DmaError.None.U) - result.address := Mux(invalid, io.kernel.load.bits.address, 0.U) - state := Mux(invalid, terminal, request) - } - when(io.request.fire)(state := receive) - when(io.response.fire) { - data := io.response.bits.data - words := Mux(remaining < beatWords.U, remaining, beatWords.U) - index := 0.U - last := io.response.bits.last - remaining := remaining - Mux(remaining < beatWords.U, remaining, beatWords.U) - when(io.response.bits.fault.error =/= DmaError.None.U) { - when(result.error === DmaError.None.U || result.error === DmaError.Cancelled.U) { - result := io.response.bits.fault - } - discarded := true.B - when(io.response.bits.last)(state := terminal) - }.elsewhen(discarded || io.kernel.abort) { - when(io.response.bits.last)(state := terminal) - }.elsewhen(io.response.bits.last =/= (remaining <= beatWords.U) || remaining === 0.U) { - result.error := DmaError.Protocol.U - result.address := descriptor.address + (descriptor.bytes - (remaining << 2)) - discarded := true.B - when(io.response.bits.last)(state := terminal) - }.otherwise { - state := output - } - } - when(state === output && io.kernel.image.fire) { - when(index +& 1.U === words) { - state := Mux(last, terminal, receive) - }.otherwise { - index := index + 1.U - } - } - when(io.kernel.result.fire)(state := idle) -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/MemConfiger.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/MemConfiger.scala deleted file mode 100644 index 9bf49fb88..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/MemConfiger.scala +++ /dev/null @@ -1,141 +0,0 @@ -package framework.memdomain.frontend.mem - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.memdomain.backend.shared.SharedMemLayout -import framework.memdomain.frontend.cmd.rs.{MemRsComplete, MemRsIssue} -import chisel3.experimental.hierarchy.{instantiable, public} - -class MemConfigerIO(val b: GlobalConfig) extends Bundle { - val vbank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val is_shared = Output(Bool()) - val is_multi = Output(Bool()) - val alloc = Output(Bool()) - val transfer = Output(Bool()) - val source_bank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - // Zero the physical bank this allocation binds; the backend clears it locally. - val clear = Output(Bool()) - val group_id = Output(UInt(b.memDomain.groupIdWidth.W)) - val hart_id = Output(UInt(b.tile.xLen.W)) -} - -@instantiable -class MemConfiger(val b: GlobalConfig) extends Module { - val rob_id_width = log2Up(b.frontend.rob_entries) - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new MemRsIssue(b))) - val cmdResp = Decoupled(new MemRsComplete(b)) - val config = Decoupled(new MemConfigerIO(b)) - val hartid = Input(UInt(b.tile.xLen.W)) - // A backend bank is still clearing; MSET.clear completes only after it drains. - val clearBusy = Input(Bool()) - }) - - val idle :: config :: clearWait :: resp :: Nil = Enum(4) - - val state = RegInit(idle) - val transfer_reg = RegInit(false.B) - val source_bank_id_reg = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val alloc_reg = RegInit(false.B) - val is_shared_reg = RegInit(false.B) - val col_reg = RegInit(0.U(b.memDomain.groupCountWidth.W)) - val clear_reg = RegInit(false.B) - val vbank_id_reg = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val rob_id_reg = RegInit(0.U(rob_id_width.W)) - val is_sub_reg = RegInit(false.B) - val sub_rob_id_reg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - val counter = RegInit(0.U(b.memDomain.groupCountWidth.W)) - - io.config.bits.is_multi := false.B - io.config.bits.is_shared := false.B - io.config.bits.transfer := false.B - io.config.bits.source_bank_id := 0.U - io.config.bits.alloc := false.B - io.config.bits.clear := false.B - io.config.bits.vbank_id := 0.U(b.memDomain.vbankIdWidth.W) - io.config.bits.group_id := 0.U - io.config.bits.hart_id := io.hartid - io.config.valid := false.B - io.cmdResp.valid := false.B - io.cmdResp.bits := 0.U.asTypeOf(io.cmdResp.bits) - io.cmdResp.bits.rob_id := 0.U(rob_id_width.W) - io.cmdResp.bits.is_sub := false.B - io.cmdResp.bits.sub_rob_id := 0.U - - io.cmdReq.ready := state === idle - - when(state === idle) { - when(io.cmdReq.valid) { - when(io.cmdReq.fire) { - val rawCol = io.cmdReq.bits.cmd.special(9, 5) - val alloc = io.cmdReq.bits.cmd.special(10) - val fullCol = - if (b.memDomain.sharedEnable) { - Mux( - io.cmdReq.bits.cmd.is_shared, - SharedMemLayout.totalBank(b).U(col_reg.getWidth.W), - b.memDomain.bankNum.U(col_reg.getWidth.W) - ) - } else { - b.memDomain.bankNum.U(col_reg.getWidth.W) - } - - state := config - col_reg := Mux( - io.cmdReq.bits.cmd.transfer, - 1.U, - Mux(alloc && rawCol === 0.U, fullCol, Mux(rawCol > 1.U, rawCol, 1.U)) - ) - transfer_reg := io.cmdReq.bits.cmd.transfer - source_bank_id_reg := io.cmdReq.bits.cmd.source_bank_id - alloc_reg := alloc - clear_reg := io.cmdReq.bits.cmd.clear - is_shared_reg := io.cmdReq.bits.cmd.is_shared - vbank_id_reg := io.cmdReq.bits.cmd.bank_id - rob_id_reg := io.cmdReq.bits.rob_id - is_sub_reg := io.cmdReq.bits.is_sub - sub_rob_id_reg := io.cmdReq.bits.sub_rob_id - assert( - !(io.cmdReq.bits.cmd.clear && io.cmdReq.bits.cmd.is_shared), - "MSET clear is currently supported for private banks only" - ) - } - } - - }.elsewhen(state === config) { - io.config.bits.is_multi := col_reg > 1.U - io.config.bits.is_shared := is_shared_reg - io.config.bits.transfer := transfer_reg - io.config.bits.source_bank_id := source_bank_id_reg - io.config.bits.alloc := alloc_reg - io.config.bits.clear := clear_reg && alloc_reg - io.config.bits.vbank_id := vbank_id_reg - io.config.bits.group_id := counter(b.memDomain.groupIdWidth - 1, 0) - io.config.valid := true.B - - when(io.config.fire) { - when(counter === col_reg - 1.U) { - counter := 0.U - state := Mux(clear_reg && alloc_reg, clearWait, resp) - }.otherwise { - counter := counter + 1.U - } - } - }.elsewhen(state === clearWait) { - // Each bound bank started clearing when its allocation fired. - when(!io.clearBusy)(state := resp) - }.elsewhen(state === resp) { - io.cmdResp.valid := true.B - io.cmdResp.bits.rob_id := rob_id_reg - io.cmdResp.bits.is_sub := is_sub_reg - io.cmdResp.bits.sub_rob_id := sub_rob_id_reg - - when(io.cmdResp.fire) { - state := idle - counter := 0.U - } - } -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/MemLoader.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/MemLoader.scala deleted file mode 100644 index ab74d4abb..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/MemLoader.scala +++ /dev/null @@ -1,303 +0,0 @@ -package framework.memdomain.frontend.mem - -import chisel3._ -import chisel3.util._ -import framework.memdomain.frontend.cmd.rs.{MemRsComplete, MemRsIssue} -import framework.memdomain.frontend.mem.dma.{BBReadRequest, BBReadResponse, DmaError, DmaStatus} -import freechips.rocketchip.rocket.MStatus -import framework.balldomain.blink.BankWrite -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig - -@instantiable -class MemLoader(val b: GlobalConfig) extends Module { - require( - b.memDomain.bankWidth == 128 && b.memDomain.dma_buswidth == 128, - "MemLoader requires 128-bit bank and DMA beats" - ) - val rob_id_width = log2Up(b.frontend.rob_entries) - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new MemRsIssue(b))) - val cmdResp = Decoupled(new MemRsComplete(b)) - val footprint = Output(new Footprint(b)) - - val dmaReq = Decoupled(new BBReadRequest()) - val dmaResp = Flipped(Decoupled(new BBReadResponse(b.memDomain.bankWidth))) - - val bankWrite = Flipped(new BankWrite(b)) - - // MMIO routing hint: tells upper level (MemDomain) to route bankWrite to MmioPool - // When is_mvin_mmio_active is true, mmio_addr/col carry MMIO destination info - val is_mvin_mmio_active = Output(Bool()) - val mmio_addr = Output(UInt(17.W)) // rs2[55:39]: MMIO byte address - val mmio_col = Output(UInt(8.W)) // rs2[63:56]: valid bytes per row - - val query_valid = Output(Bool()) - val query_vbank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val query_is_shared = Output(Bool()) - val query_group_count = Input(UInt(b.memDomain.groupCountWidth.W)) - - // Propagate decoded shared/private access intent. - val is_shared = Output(Bool()) - }) - - val s_idle :: s_setup :: s_query :: s_query_wait :: s_mul :: s_dma_req :: s_dma_wait :: s_wait_write_resp :: s_done :: Nil = - Enum(9) - val state = RegInit(s_idle) - - val rob_id_reg = RegInit(0.U(rob_id_width.W)) - val is_sub_reg = RegInit(false.B) - val sub_rob_id_reg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - val mem_addr_reg = Reg(UInt(b.memDomain.memAddrLen.W)) - val iter_cmd = Reg(UInt(b.frontend.iter_len.W)) - val iter_reg = Reg(UInt(b.frontend.iter_len.W)) - val resp_count = RegInit(0.U(log2Up(16).W)) - val wr_bank_reg = Reg(UInt(b.memDomain.vbankIdWidth.W)) - val stride_reg = Reg(UInt(19.W)) - val is_shared_reg = RegInit(false.B) - val fault = RegInit(0.U.asTypeOf(new DmaStatus)) - val footprintValid = RegInit(false.B) - - // Group counter for multi-bank writes - val group_counter = RegInit(0.U(b.memDomain.groupCountWidth.W)) - val group_count_reg = RegInit(0.U(b.memDomain.groupCountWidth.W)) - val selected_group = RegInit(false.B) - val group_base = RegInit(0.U(5.W)) - val rowAddr = RegInit(0.U(16.W)) - - // MMIO routing info (latched at cmdReq.fire, exposed to upper level) - val is_mvin_mmio_reg = RegInit(false.B) - val is_mvin_2d_reg = RegInit(false.B) - val valid_bytes_reg = RegInit(16.U(5.W)) - val pixel_bytes_reg = RegInit(0.U(10.W)) - val source_width_reg = RegInit(0.U(10.W)) - val tile_width_reg = RegInit(0.U(4.W)) - val mmio_addr_reg = RegInit(0.U(17.W)) - val mmio_col_reg = RegInit(0.U(8.W)) - - io.is_mvin_mmio_active := is_mvin_mmio_reg - io.mmio_addr := mmio_addr_reg - io.mmio_col := mmio_col_reg - - val shapeRows = iter_cmd.pad(64) - val shapeColumns = Mux(is_mvin_2d_reg, tile_width_reg.pad(64), 1.U(64.W)) - val shapeSpan = - Mux(is_mvin_2d_reg, (b.memDomain.dma_buswidth / 8).U(64.W), (group_count_reg * (b.memDomain.bankWidth / 8).U).pad(64)) - val shapeColumnStride = Mux(is_mvin_2d_reg, pixel_bytes_reg.pad(64), 0.U(64.W)) - val linearRowStride = group_count_reg * stride_reg * (b.memDomain.bankWidth / 8).U - val shapeRowStride = Mux(is_mvin_2d_reg, (source_width_reg * pixel_bytes_reg).pad(64), linearRowStride.pad(64)) - val shapeBeats = iter_cmd * Mux(is_mvin_2d_reg, tile_width_reg.pad(group_count_reg.getWidth), group_count_reg) - val shapeEnd = mem_addr_reg.pad(64) +& ((shapeRows - 1.U) * shapeRowStride) +& - ((shapeColumns - 1.U) * shapeColumnStride) +& (shapeSpan - 1.U) - - val storageRows = Mux(is_shared_reg, b.memDomain.sharedBankEntries.U, b.memDomain.bankEntries.U) - val destinationRows = iter_cmd * Mux(is_mvin_2d_reg, tile_width_reg, 1.U) - val destinationStart = Reg(UInt(16.W)) - when(io.cmdReq.fire) { - destinationStart := Mux(io.cmdReq.bits.cmd.is_mvin_2d, io.cmdReq.bits.cmd.special(58, 53), 0.U) - } - - val shapeError = - (!is_mvin_mmio_reg && (destinationStart +& destinationRows) > storageRows) || iter_cmd === 0.U || group_count_reg === 0.U || stride_reg === 0.U || - (shapeBeats >> b.frontend.iter_len).orR || (shapeEnd >> b.memDomain.memAddrLen).orR || - (is_mvin_2d_reg && (pixel_bytes_reg === 0.U || source_width_reg < tile_width_reg)) - - io.footprint := 0.U.asTypeOf(new Footprint(b)) - io.footprint.valid := footprintValid - io.footprint.rob_id := rob_id_reg - io.footprint.is_sub := is_sub_reg - io.footprint.sub_rob_id := sub_rob_id_reg - io.footprint.baseVA := mem_addr_reg - io.footprint.rows := shapeRows - io.footprint.columns := shapeColumns - io.footprint.spanBytes := shapeSpan - io.footprint.columnStride := shapeColumnStride - io.footprint.rowStride := shapeRowStride - io.footprint.write := false.B - io.footprint.fault.error := Mux(shapeError, DmaError.Shape.U, DmaError.None.U) - io.footprint.fault.address := Mux(shapeError, mem_addr_reg.pad(64), 0.U) - - // ----------------------------- - // pending latch for 1-beat DMA -> bankWrite - // ----------------------------- - val pending = RegInit(false.B) - val latData = Reg(UInt(b.memDomain.bankWidth.W)) - val latLast = RegInit(false.B) - - // ----------------------------- - // defaults - // ----------------------------- - io.cmdReq.ready := (state === s_idle) - - io.dmaReq.valid := (state === s_dma_req) - io.dmaReq.bits.vaddr := mem_addr_reg - io.dmaReq.bits.len := iter_reg * (b.memDomain.bankWidth / 8).U - io.dmaReq.bits.status := 0.U.asTypeOf(new MStatus) - io.dmaReq.bits.stride := stride_reg - io.dmaReq.bits.groups := group_count_reg - io.dmaReq.bits.is_2d := is_mvin_2d_reg - io.dmaReq.bits.pixel_bytes := pixel_bytes_reg - io.dmaReq.bits.source_width := source_width_reg - io.dmaReq.bits.tile_width := tile_width_reg - - // only accept DMA beat when waiting AND no pending beat buffered - io.dmaResp.ready := (state === s_dma_wait) && !pending - - // bank write request driven from pending - io.bankWrite.io.req.valid := pending - io.bankWrite.io.req.bits.addr := rowAddr - - val validDataMask = VecInit( - (0 until b.memDomain.bankWidth / 8).map(i => Mux(i.U < valid_bytes_reg, "hff".U(8.W), 0.U(8.W))) - ).asUInt - - io.bankWrite.io.req.bits.data := Mux(is_mvin_2d_reg, latData & validDataMask, latData) - io.bankWrite.io.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(true.B)) - - // IMPORTANT: always ready for write response (avoid deadlock) - io.bankWrite.io.resp.ready := true.B - - io.bankWrite.rob_id := rob_id_reg - io.bankWrite.bank_id := wr_bank_reg - io.bankWrite.ball_id := 0.U - io.bankWrite.group_id := group_base + group_counter - io.is_shared := is_shared_reg - - // cmdResp (Decoupled): hold valid until accepted - io.cmdResp.valid := (state === s_done) - io.cmdResp.bits := 0.U.asTypeOf(new MemRsComplete(b)) - io.cmdResp.bits.rob_id := rob_id_reg - io.cmdResp.bits.is_sub := is_sub_reg - io.cmdResp.bits.sub_rob_id := sub_rob_id_reg - io.cmdResp.bits.fault := fault - - // ----------------------------- - // Receive load instruction (both mvin and mvin_mmio go through is_load path) - // ----------------------------- - when(io.cmdReq.fire && io.cmdReq.bits.cmd.is_load) { - rob_id_reg := io.cmdReq.bits.rob_id - is_sub_reg := io.cmdReq.bits.is_sub - sub_rob_id_reg := io.cmdReq.bits.sub_rob_id - mem_addr_reg := io.cmdReq.bits.cmd.mem_addr - wr_bank_reg := io.cmdReq.bits.cmd.bank_id - resp_count := 0.U - pending := false.B - fault := 0.U.asTypeOf(new DmaStatus) - footprintValid := io.cmdReq.bits.cmd.is_mvin_mmio || io.cmdReq.bits.cmd.is_mvin_2d - iter_cmd := io.cmdReq.bits.cmd.iter - latLast := false.B - group_counter := 0.U - rowAddr := Mux( - io.cmdReq.bits.cmd.is_mvin_2d, - io.cmdReq.bits.cmd.special(58, 53), - 0.U - ) - is_shared_reg := io.cmdReq.bits.cmd.is_shared - - val regular = !io.cmdReq.bits.cmd.is_mvin_mmio && !io.cmdReq.bits.cmd.is_mvin_2d - selected_group := regular && io.cmdReq.bits.cmd.special(63) - group_base := Mux(regular && io.cmdReq.bits.cmd.special(63), io.cmdReq.bits.cmd.special(62, 58), 0.U) - is_mvin_mmio_reg := io.cmdReq.bits.cmd.is_mvin_mmio - is_mvin_2d_reg := io.cmdReq.bits.cmd.is_mvin_2d - valid_bytes_reg := Mux(io.cmdReq.bits.cmd.special(62), 8.U(5.W), 16.U(5.W)) - pixel_bytes_reg := io.cmdReq.bits.cmd.special(42, 36) << 3 - source_width_reg := io.cmdReq.bits.cmd.special(52, 43) - tile_width_reg := io.cmdReq.bits.cmd.special(61, 59) +& 1.U - when(io.cmdReq.bits.cmd.is_mvin_mmio) { - mmio_addr_reg := io.cmdReq.bits.cmd.special(55, 39) - mmio_col_reg := io.cmdReq.bits.cmd.special(63, 56) - iter_reg := io.cmdReq.bits.cmd.iter - group_count_reg := 1.U - stride_reg := 1.U - state := s_dma_req - }.elsewhen(io.cmdReq.bits.cmd.is_mvin_2d) { - iter_reg := io.cmdReq.bits.cmd.iter * (io.cmdReq.bits.cmd.special(61, 59) +& 1.U) - group_count_reg := 1.U - stride_reg := 1.U - state := s_dma_req - }.otherwise { - stride_reg := io.cmdReq.bits.cmd.special(57, 39) - iter_cmd := io.cmdReq.bits.cmd.iter - state := s_setup - } - } - - io.query_valid := state === s_setup || state === s_query || state === s_query_wait - io.query_vbank_id := wr_bank_reg - io.query_is_shared := is_shared_reg && io.query_valid - - when(state === s_setup) { - state := s_query - } - - when(state === s_query) { - state := s_query_wait - } - - when(state === s_query_wait) { - assert(io.query_group_count >= 1.U, "MemLoader groups must be >= 1") - when(selected_group) { - assert(group_base < io.query_group_count, "MemLoader selected group is outside bank allocation") - } - group_count_reg := Mux(selected_group, 1.U, io.query_group_count) - state := s_mul - } - - when(state === s_mul) { - iter_reg := iter_cmd * group_count_reg - state := s_dma_req - footprintValid := true.B - } - - // DMA req accepted - when(io.dmaReq.fire) { - state := s_dma_wait - resp_count := 0.U - } - - // Latch DMA beat into pending buffer - when(io.dmaResp.fire) { - when(io.dmaResp.bits.fault.error =/= DmaError.None.U || fault.error =/= DmaError.None.U) { - when(fault.error === DmaError.None.U)(fault := io.dmaResp.bits.fault) - when(io.dmaResp.bits.last)(state := s_done) - }.otherwise { - pending := true.B - latData := io.dmaResp.bits.data - latLast := io.dmaResp.bits.last - } - } - - // When bankWrite request handshakes, consume pending beat - when(io.bankWrite.io.req.fire) { - pending := false.B - resp_count := resp_count + 1.U - - when(latLast) { - state := s_wait_write_resp - }.otherwise { - state := s_wait_write_resp - } - } - - // Wait for each write response before accepting the next DMA beat. - when(state === s_wait_write_resp && io.bankWrite.io.resp.fire) { - when(!io.bankWrite.io.resp.bits.ok && fault.error === DmaError.None.U) { - fault.error := DmaError.Bank.U - fault.address := rowAddr * (b.memDomain.bankWidth / 8).U - } - when(group_counter + 1.U < group_count_reg) { - group_counter := group_counter + 1.U - }.otherwise { - group_counter := 0.U - rowAddr := rowAddr + 1.U - } - state := Mux(latLast, s_done, s_dma_wait) - } - - when(state === s_done && io.cmdResp.fire) { - state := s_idle - footprintValid := false.B - } -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/MemStorer.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/MemStorer.scala deleted file mode 100644 index 18ad106b2..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/MemStorer.scala +++ /dev/null @@ -1,212 +0,0 @@ -package framework.memdomain.frontend.mem - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.memdomain.frontend.cmd.rs.{MemRsComplete, MemRsIssue} -import framework.memdomain.frontend.mem.dma.{BBWriteCommand, BBWriteData, BBWriteResponse, DmaError, DmaStatus} -import framework.balldomain.blink.BankRead -import chisel3.experimental.hierarchy.{instantiable, public} - -@instantiable -class MemStorer(val b: GlobalConfig) extends Module { - require( - b.memDomain.bankWidth == 128 && b.memDomain.dma_buswidth == 128, - "MemStorer requires 128-bit bank and DMA beats" - ) - val lineBytes = b.memDomain.bankWidth / 8 - val lgLine = log2Ceil(lineBytes) - val robBits = log2Up(b.frontend.rob_entries) - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new MemRsIssue(b))) - val cmdResp = Decoupled(new MemRsComplete(b)) - val footprint = Output(new Footprint(b)) - val dmaReq = Decoupled(new BBWriteCommand) - val dmaData = Decoupled(new BBWriteData(b.memDomain.bankWidth)) - val dmaResp = Flipped(Decoupled(new BBWriteResponse)) - val bankRead = Flipped(new BankRead(b)) - val query_valid = Output(Bool()) - val query_vbank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val query_is_shared = Output(Bool()) - val query_group_count = Input(UInt(b.memDomain.groupCountWidth.W)) - val is_shared = Output(Bool()) - }) - - val idle :: setup :: query :: queryWait :: prepare :: descriptor :: stream :: waitResponse :: drain :: done :: Nil = - Enum(10) - val state = RegInit(idle) - val rob = RegInit(0.U(robBits.W)) - val isSub = RegInit(false.B) - val subRob = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - val iterations = RegInit(0.U(b.frontend.iter_len.W)) - val bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val stride = RegInit(1.U(19.W)) - val groups = RegInit(1.U(b.memDomain.groupCountWidth.W)) - val shared = RegInit(false.B) - val baseVA = RegInit(0.U(64.W)) - val cursor = RegInit(0.U(64.W)) - val fault = RegInit(0.U.asTypeOf(new DmaStatus)) - val footprintValid = RegInit(false.B) - val readRow = RegInit(0.U(b.frontend.iter_len.W)) - val readGroup = RegInit(0.U(b.memDomain.groupCountWidth.W)) - val readsFinished = RegInit(false.B) - val finalDescriptor = RegInit(false.B) - - val selectedGroup = RegInit(false.B) - val groupBase = RegInit(0.U(5.W)) - - // Each accepted bank read reserves a data slot until its beat is sent. - // Ordered last tags include requests whose synchronous response has not yet arrived. - val lastTags = Module(new Queue(Bool(), 2, flow = true)) - val data = Module(new Queue(new BBWriteData(b.memDomain.bankWidth), 2, pipe = true)) - val reserved = lastTags.io.count +& data.io.count - val empty = lastTags.io.count === 0.U && data.io.count === 0.U - - val span = groups * lineBytes.U - val rowStride = groups * stride * lineBytes.U - val totalBytes = iterations * span - val descriptorBytes = Mux(stride === 1.U, totalBytes, span) - val shapeEnd = baseVA +& ((iterations - 1.U) * rowStride) +& (span - 1.U) - - val strideOverflow = - if (rowStride.getWidth > b.memDomain.memAddrLen) - (rowStride >> b.memDomain.memAddrLen).orR - else false.B - - val lengthOverflow = if (descriptorBytes.getWidth > 32) (descriptorBytes >> 32).orR else false.B - val storageRows = Mux(shared, b.memDomain.sharedBankEntries.U, b.memDomain.bankEntries.U) - - val shapeError = iterations === 0.U || iterations > storageRows || - groups === 0.U || groups > (BigInt(1) << b.memDomain.groupIdWidth).U || stride === 0.U || - (shapeEnd >> b.memDomain.memAddrLen).orR || strideOverflow || lengthOverflow - - io.cmdReq.ready := state === idle - when(io.cmdReq.fire) { - assert(io.cmdReq.bits.cmd.is_store, "MemStorer received a non-store command") - assert(empty, "MemStorer reused a command with pending bank reads or data") - rob := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRob := io.cmdReq.bits.sub_rob_id - iterations := io.cmdReq.bits.cmd.iter - bank := io.cmdReq.bits.cmd.bank_id - selectedGroup := io.cmdReq.bits.cmd.special(63) - groupBase := Mux(io.cmdReq.bits.cmd.special(63), io.cmdReq.bits.cmd.special(62, 58), 0.U) - stride := io.cmdReq.bits.cmd.special(57, 39) - shared := io.cmdReq.bits.cmd.is_shared - baseVA := io.cmdReq.bits.cmd.mem_addr - cursor := io.cmdReq.bits.cmd.mem_addr - readRow := 0.U - readGroup := 0.U - readsFinished := false.B - fault := 0.U.asTypeOf(new DmaStatus) - footprintValid := false.B - state := setup - } - io.query_valid := state === setup || state === query || state === queryWait - io.query_vbank_id := bank - io.query_is_shared := shared && io.query_valid - when(state === setup)(state := query) - when(state === query)(state := queryWait) - when(state === queryWait) { - when(selectedGroup) { - assert(groupBase < io.query_group_count, "MemStorer selected group is outside bank allocation") - } - groups := Mux(selectedGroup, 1.U, io.query_group_count) - state := prepare - } - when(state === prepare) { - footprintValid := true.B - when(shapeError) { - fault.error := DmaError.Shape.U - fault.address := baseVA - state := drain - }.otherwise(state := descriptor) - } - - io.footprint := 0.U.asTypeOf(new Footprint(b)) - io.footprint.valid := footprintValid - io.footprint.rob_id := rob - io.footprint.is_sub := isSub - io.footprint.sub_rob_id := subRob - io.footprint.baseVA := baseVA - io.footprint.rows := iterations - io.footprint.columns := 1.U - io.footprint.spanBytes := span - io.footprint.columnStride := 0.U - io.footprint.rowStride := rowStride - io.footprint.write := true.B - io.footprint.fault.error := Mux(shapeError, DmaError.Shape.U, DmaError.None.U) - io.footprint.fault.address := Mux(shapeError, baseVA, 0.U) - - io.dmaReq.valid := state === descriptor - io.dmaReq.bits.vaddr := cursor - io.dmaReq.bits.len := descriptorBytes - when(io.dmaReq.fire) { - assert(empty, "MemStorer started a descriptor with pending bank reads or data") - finalDescriptor := stride === 1.U || readRow === iterations - 1.U - readsFinished := false.B - state := stream - } - - val lastGroup = readGroup === groups - 1.U - val lastRow = readRow === iterations - 1.U - val lastRead = lastGroup && (stride =/= 1.U || lastRow) - io.bankRead.rob_id := rob - io.bankRead.bank_id := bank - io.bankRead.ball_id := 0.U - io.bankRead.group_id := groupBase + readGroup - io.bankRead.io.req.bits.addr := readRow - io.is_shared := shared - io.bankRead.io.req.valid := state === stream && !readsFinished && - (reserved < 2.U || io.dmaData.fire) - lastTags.io.enq.valid := io.bankRead.io.req.fire - lastTags.io.enq.bits := lastRead - when(io.bankRead.io.req.fire) { - assert(lastTags.io.enq.ready, "MemStorer bank read has no reserved last-tag slot") - when(lastGroup) { - readGroup := 0.U - when(!lastRow)(readRow := readRow + 1.U) - }.otherwise(readGroup := readGroup + 1.U) - when(lastRead)(readsFinished := true.B) - } - - data.io.enq.valid := io.bankRead.io.resp.valid && lastTags.io.deq.valid - data.io.enq.bits.data := io.bankRead.io.resp.bits.data - data.io.enq.bits.last := lastTags.io.deq.bits - io.bankRead.io.resp.ready := data.io.enq.ready && lastTags.io.deq.valid - lastTags.io.deq.ready := io.bankRead.io.resp.fire - when(io.bankRead.io.resp.valid) { - assert(lastTags.io.deq.valid, "MemStorer received an unreserved bank response") - assert(data.io.enq.ready, "MemStorer reserved bank response exceeded its data buffer") - } - assert(reserved <= 2.U, "MemStorer exceeded its two reserved data slots") - - io.dmaData.valid := state === stream && data.io.deq.valid - io.dmaData.bits := data.io.deq.bits - data.io.deq.ready := state === drain || (state === stream && io.dmaData.ready) - when(io.dmaData.fire && io.dmaData.bits.last)(state := waitResponse) - - // WriteDma consumes the complete descriptor even after an AXI/map error, and - // returns exactly one terminal response. Do not advance a strided row before it. - io.dmaResp.ready := state === waitResponse - when(io.dmaResp.fire) { - when(io.dmaResp.bits.fault.error =/= DmaError.None.U || !io.dmaResp.bits.done) { - when(io.dmaResp.bits.fault.error =/= DmaError.None.U)(fault := io.dmaResp.bits.fault) - .otherwise { fault.error := DmaError.Protocol.U; fault.address := cursor } - state := drain - }.elsewhen(finalDescriptor)(state := drain) - .otherwise { cursor := cursor + rowStride; state := descriptor } - } - when(state === drain && empty)(state := done) - io.cmdResp.valid := state === done - io.cmdResp.bits := 0.U.asTypeOf(new MemRsComplete(b)) - io.cmdResp.bits.rob_id := rob - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRob - io.cmdResp.bits.fault := fault - when(io.cmdResp.fire) { footprintValid := false.B; state := idle } -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/Bundles.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/dma/Bundles.scala deleted file mode 100644 index 261beaf26..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/Bundles.scala +++ /dev/null @@ -1,38 +0,0 @@ -package framework.memdomain.frontend.mem.dma - -import chisel3._ -import freechips.rocketchip.rocket.MStatus - -class BBReadRequest extends Bundle { - val vaddr = UInt(64.W) - val len = UInt(32.W) - val status = new MStatus - val stride = UInt(19.W) - val groups = UInt(6.W) - val is_2d = Bool() - val pixel_bytes = UInt(10.W) - val source_width = UInt(10.W) - val tile_width = UInt(4.W) -} - -class BBReadResponse(dataWidth: Int) extends Bundle { - val data = UInt(dataWidth.W) - val last = Bool() - val addrcounter = UInt(16.W) - val fault = new DmaStatus -} - -class BBWriteCommand extends Bundle { - val vaddr = UInt(64.W) - val len = UInt(32.W) -} - -class BBWriteData(dataWidth: Int) extends Bundle { - val data = UInt(dataWidth.W) - val last = Bool() -} - -class BBWriteResponse extends Bundle { - val done = Bool() - val fault = new DmaStatus -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/Dma.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/dma/Dma.scala deleted file mode 100644 index 4c78f78e4..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/Dma.scala +++ /dev/null @@ -1,69 +0,0 @@ -package framework.memdomain.frontend.mem.dma - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import memcore.memory.preflight.{MapQuery, MapResult, Params} -import memcore.bus.axi4 - -class DmaDecision(p: Params) extends Bundle { - val valid = Bool() - val parentId = UInt(p.idBits.W) - val fault = new DmaStatus -} - -@instantiable -class Dma(b: GlobalConfig, prepared: Params, axiParams: axi4.Params) extends Module { - - @public - val io = IO(new Bundle { - val dma = Flipped(new DmaPort(b.memDomain.dma_buswidth)) - // One decision for each physical producer: loader, storer, kernel image. - val decisions = Input(Vec(3, new DmaDecision(prepared))) - val queries = Output(Vec(2, new MapQuery(prepared))) - val mappings = Input(Vec(2, new MapResult(prepared))) - val axi = new axi4.Port(axiParams) - }) - - val reader: Instance[ReadDma] = Instantiate(new ReadDma(b, prepared, axiParams)) - val writer: Instance[WriteDma] = Instantiate(new WriteDma(b, prepared, axiParams)) - val kernel = io.dma.read.bits.producer === 2.U - val readDecision = Mux(kernel, io.decisions(2), io.decisions(0)) - when(io.dma.read.valid) { - assert(io.dma.read.bits.producer === 0.U || kernel, "DMA read has an unknown producer") - } - reader.io.req.valid := io.dma.read.valid - reader.io.req.bits := io.dma.read.bits.transfer - io.dma.read.ready := reader.io.req.ready - io.dma.readResult <> reader.io.resp - reader.io.parentId := readDecision.parentId - reader.io.decisionValid := readDecision.valid - reader.io.decisionFault := readDecision.fault - writer.io.req <> io.dma.write - writer.io.data <> io.dma.writeData - io.dma.writeResult <> writer.io.resp - writer.io.parentId := io.decisions(1).parentId - writer.io.decisionValid := io.decisions(1).valid - writer.io.decisionFault := io.decisions(1).fault - io.dma.readBusy := reader.io.busy - io.dma.writeBusy := writer.io.busy - io.queries(0) := reader.io.query - io.queries(1) := writer.io.query - reader.io.mapping := io.mappings(0) - writer.io.mapping := io.mappings(1) - - // AXI read and write channels retain independent command owners. - io.axi.ar <> reader.io.axi.ar - reader.io.axi.r <> io.axi.r - io.axi.aw <> writer.io.axi.aw - io.axi.w <> writer.io.axi.w - writer.io.axi.b <> io.axi.b - reader.io.axi.aw.ready := false.B - reader.io.axi.w.ready := false.B - reader.io.axi.b.valid := false.B - reader.io.axi.b.bits := 0.U.asTypeOf(new axi4.WriteResponse(axiParams)) - writer.io.axi.ar.ready := false.B - writer.io.axi.r.valid := false.B - writer.io.axi.r.bits := 0.U.asTypeOf(new axi4.ReadData(axiParams)) -} diff --git a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/DmaPort.scala b/arch/src/main/scala/framework/memdomain/frontend/mem/dma/DmaPort.scala deleted file mode 100644 index bafde2c10..000000000 --- a/arch/src/main/scala/framework/memdomain/frontend/mem/dma/DmaPort.scala +++ /dev/null @@ -1,21 +0,0 @@ -package framework.memdomain.frontend.mem.dma - -import chisel3._ -import chisel3.util._ - -class DmaReadCommand extends Bundle { - val transfer = new BBReadRequest - // Indices of MemFrontend's physical footprints: loader=0, storer=1, kernel image=2. - val producer = UInt(2.W) -} - -/** Accelerator-side logical transfers. Translation and external-memory transport belong to the system. */ -class DmaPort(dataBits: Int) extends Bundle { - val read = Decoupled(new DmaReadCommand) - val readResult = Flipped(Decoupled(new BBReadResponse(dataBits))) - val write = Decoupled(new BBWriteCommand) - val writeData = Decoupled(new BBWriteData(dataBits)) - val writeResult = Flipped(Decoupled(new BBWriteResponse)) - val readBusy = Input(Bool()) - val writeBusy = Input(Bool()) -} diff --git a/arch/src/main/scala/framework/memdomain/isa/MemISA.scala b/arch/src/main/scala/framework/memdomain/isa/MemISA.scala deleted file mode 100644 index d056b1bf9..000000000 --- a/arch/src/main/scala/framework/memdomain/isa/MemISA.scala +++ /dev/null @@ -1,31 +0,0 @@ -package framework.memdomain.isa - -import framework.system.core.rocket.BuckyballCommand - -/** Shared MemDomain ISA encoding helpers. */ -object MemISA { - val MsetFunct = 0x20 - - final case class MsetArgs( - bankId: Int, - row: Int = 0, - columns: Int, - alloc: Boolean, - clear: Boolean) - - def mset(args: MsetArgs): BuckyballCommand = { - require(args.bankId >= 0 && args.bankId < (1 << 10), s"MSET bank ID out of range: ${args.bankId}") - require(args.row >= 0 && args.row < (1 << 5), s"MSET row out of range: ${args.row}") - require(args.columns >= 0 && args.columns <= 32, s"MSET column count out of range: ${args.columns}") - require(!args.clear || args.alloc, "MSET clear requires allocation") - - val rs1 = BigInt(args.bankId) - val rs2 = - BigInt(args.row) | - (BigInt(args.columns) << 5) | - (if (args.alloc) BigInt(1) << 10 else 0) | - (if (args.clear) BigInt(1) << 11 else 0) - BuckyballCommand(MsetFunct, rs1, rs2) - } - -} diff --git a/arch/src/main/scala/framework/memdomain/isa/MvoverIO.scala b/arch/src/main/scala/framework/memdomain/isa/MvoverIO.scala deleted file mode 100644 index 14e231dcf..000000000 --- a/arch/src/main/scala/framework/memdomain/isa/MvoverIO.scala +++ /dev/null @@ -1,27 +0,0 @@ -package framework.memdomain.isa - -import chisel3._ -import chisel3.util._ - -object MvoverISA { - val Funct = 0x0d - val CoreBits = 8 - val BankBits = 10 - val AddressBits = 16 -} - -/** Contiguous rows between logical Core-local Banks in the same Tile. */ -class MvoverCommand extends Bundle { - val sourceCore = UInt(MvoverISA.CoreBits.W) - val sourceBank = UInt(MvoverISA.BankBits.W) - val sourceAddr = UInt(MvoverISA.AddressBits.W) - val targetCore = UInt(MvoverISA.CoreBits.W) - val targetBank = UInt(MvoverISA.BankBits.W) - val targetAddr = UInt(MvoverISA.AddressBits.W) - val rows = UInt((MvoverISA.AddressBits + 1).W) -} - -class MvoverPort extends Bundle { - val command = Decoupled(new MvoverCommand) - val completion = Flipped(Decoupled(Bool())) -} diff --git a/arch/src/main/scala/framework/memdomain/midend/MemMidend.scala b/arch/src/main/scala/framework/memdomain/midend/MemMidend.scala deleted file mode 100644 index 971dfc3ee..000000000 --- a/arch/src/main/scala/framework/memdomain/midend/MemMidend.scala +++ /dev/null @@ -1,277 +0,0 @@ -package framework.memdomain.midend - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.balldomain.blink.{BankRead, BankWrite} -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.memdomain.backend.MemRequestIO - -// BankRead/BankWrite with is_shared flag, used for unified midend interface -class BankReadWithShared(val b: GlobalConfig) extends Bundle { - val bankRead = new BankRead(b) - val is_shared = Input(Bool()) -} - -class BankWriteWithShared(val b: GlobalConfig) extends Bundle { - val bankWrite = new BankWrite(b) - val is_shared = Input(Bool()) -} - -/** - * MemMidend: Midend module for memory scheduling - * Connects MemFrontend to MemManager - * - * Unified interface: bankRead/bankWrite Vecs include both balldomain and frontend requests. - * The last entry is frontend DMA. - * All requests go through the same mapping table and channel allocation logic. - */ -@instantiable -class MemMidend(val b: GlobalConfig) extends Module { - val totalBallRead = b.ballDomain.ballIdMappings.map(_.inBW).sum - val totalBallWrite = b.ballDomain.ballIdMappings.map(_.outBW).sum - - // Total slots: balldomain entries + 1 frontend entry - val ownerCount = b.ballDomain.ballNum - val totalRead = totalBallRead + 1 - val totalWrite = totalBallWrite + 1 - - @public - val io = IO(new Bundle { - // Unified read/write interfaces: indices [0, totalBallRead) are balldomain, - // The final port belongs to frontend DMA. - val bankRead = Vec(totalRead, new BankReadWithShared(b)) - val bankWrite = Vec(totalWrite, new BankWriteWithShared(b)) - val ballChannelActive = Input(Vec(b.ballDomain.ballNum, Bool())) - val ballChannelReady = Output(Vec(b.ballDomain.ballNum, Bool())) - - val hartid = Input(UInt(b.tile.xLen.W)) - val inst_ids = Input(Vec(b.frontend.rob_entries, UInt(64.W))) - - // Output to backend (MemManager) - val mem_req = Vec(b.memDomain.bankChannel, new MemRequestIO(b)) - }) - - // ----------------------------------------------------------------------------- - // Mapping table for tracking all requests (balldomain + frontend) - // ----------------------------------------------------------------------------- - class MappingTableEntry extends Bundle { - val valid = Bool() - val isRead = Bool() - val id = UInt(log2Ceil(math.max(totalRead, totalWrite)).W) - val leased = Bool() - val owner = UInt(math.max(1, log2Ceil(ownerCount)).W) - } - - val mappingTable = RegInit(VecInit(Seq.fill(b.memDomain.bankChannel)(0.U.asTypeOf(new MappingTableEntry)))) - - // A route stays assigned until its Ball releases it (or a frontend channel - // drains), so maintain the reverse ownership at allocation time. This keeps - // readiness and pending detection from repeatedly scanning every backend - // channel for the same port. - val portSlots = math.max(totalRead, totalWrite) - val readPortAllocated = RegInit(VecInit(Seq.fill(portSlots)(false.B))) - val writePortAllocated = RegInit(VecInit(Seq.fill(portSlots)(false.B))) - - def isAllocated(isRead: Bool, id: UInt): Bool = - Mux(isRead, readPortAllocated(id), writePortAllocated(id)) - - val readPortOwner = b.ballDomain.ballIdMappings.zipWithIndex.flatMap { case (mapping, ball) => - Seq.fill(mapping.inBW)(ball) - } - - val writePortOwner = b.ballDomain.ballIdMappings.zipWithIndex.flatMap { case (mapping, ball) => - Seq.fill(mapping.outBW)(ball) - } - - val ownerActive = io.ballChannelActive - val ownerReady = Wire(Vec(ownerCount, Bool())) - for (ball <- 0 until b.ballDomain.ballNum) { - val readReady = readPortOwner.zipWithIndex - .filter(_._1 == ball) - .map { case (_, port) => isAllocated(true.B, port.U) } - .foldLeft(true.B)(_ && _) - val writeReady = writePortOwner.zipWithIndex - .filter(_._1 == ball) - .map { case (_, port) => isAllocated(false.B, port.U) } - .foldLeft(true.B)(_ && _) - ownerReady(ball) := ownerActive(ball) && readReady && writeReady - } - - io.ballChannelReady := VecInit(ownerReady.take(b.ballDomain.ballNum)) - - // Allocate exactly one channel per cycle. Ball channels are allocated before - // their data traffic can start; frontend DMA channels are demand-allocated. - for (i <- 0 until totalRead) { - io.bankRead(i).bankRead.io.req.ready := false.B - io.bankRead(i).bankRead.io.resp.valid := false.B - io.bankRead(i).bankRead.io.resp.bits := DontCare - - } - - for (i <- 0 until totalWrite) { - io.bankWrite(i).bankWrite.io.req.ready := false.B - io.bankWrite(i).bankWrite.io.resp.valid := false.B - io.bankWrite(i).bankWrite.io.resp.bits := DontCare - } - - val pendingReads = VecInit((0 until totalRead).map { i => - val active = - if (i < totalBallRead) { - ownerActive(readPortOwner(i)) - - } else { - io.bankRead(i).bankRead.io.req.valid - } - active && !isAllocated(true.B, i.U) - }) - - val pendingWrites = VecInit((0 until totalWrite).map { i => - val active = - if (i < totalBallWrite) { - ownerActive(writePortOwner(i)) - - } else { - io.bankWrite(i).bankWrite.io.req.valid - } - active && !isAllocated(false.B, i.U) - }) - - val pending = VecInit(pendingReads ++ pendingWrites) - val freeChannels = mappingTable.map(entry => !entry.valid) - - when(pending.asUInt.orR && freeChannels.reduce(_ || _)) { - val channel = PriorityEncoder(freeChannels) - val port = PriorityEncoder(pending) - mappingTable(channel).valid := true.B - mappingTable(channel).isRead := port < totalRead.U - mappingTable(channel).id := Mux(port < totalRead.U, port, port - totalRead.U) - mappingTable(channel).leased := Mux( - port < totalRead.U, - MuxLookup(port, false.B)((0 until totalBallRead).map(i => i.U -> true.B).toSeq), - MuxLookup(port - totalRead.U, false.B)((0 until totalBallWrite).map(i => i.U -> true.B).toSeq) - ) - mappingTable(channel).owner := Mux( - port < totalRead.U, - MuxLookup(port, 0.U)(readPortOwner.zipWithIndex.map { case (ball, i) => i.U -> ball.U }.toSeq), - MuxLookup(port - totalRead.U, 0.U)(writePortOwner.zipWithIndex.map { case (ball, i) => i.U -> ball.U }.toSeq) - ) - - when(port < totalRead.U) { - readPortAllocated(port) := true.B - }.otherwise { - writePortAllocated(port - totalRead.U) := true.B - } - } - - // Connect mapped entries to backend channels - for (i <- 0 until b.memDomain.bankChannel) { - io.mem_req(i).read.req.valid := false.B - io.mem_req(i).read.req.bits := DontCare - io.mem_req(i).read.resp.ready := false.B - io.mem_req(i).write.req.valid := false.B - io.mem_req(i).write.req.bits := DontCare - io.mem_req(i).write.resp.ready := false.B - io.mem_req(i).bank_id := 0.U - io.mem_req(i).group_id := 0.U - io.mem_req(i).is_shared := false.B - io.mem_req(i).hart_id := io.hartid - io.mem_req(i).rob_id := 0.U - io.mem_req(i).inst_id := 0.U - - val isRead = mappingTable(i).isRead - val rid = mappingTable(i).id - val wid = mappingTable(i).id - val ballRead = io.bankRead(rid).bankRead.io - val ballWrite = io.bankWrite(wid).bankWrite.io - val rbank_id = io.bankRead(rid).bankRead.bank_id - val wbank_id = io.bankWrite(wid).bankWrite.bank_id - val rgroup_id = io.bankRead(rid).bankRead.group_id - val wgroup_id = io.bankWrite(wid).bankWrite.group_id - val r_shared = io.bankRead(rid).is_shared - val w_shared = io.bankWrite(wid).is_shared - val rrob_id = io.bankRead(rid).bankRead.rob_id - val wrob_id = io.bankWrite(wid).bankWrite.rob_id - // A released Ball cannot issue new traffic, but its final bank response - // still needs the existing route until the physical channel drains. - val ballRouteOpen = !mappingTable(i).leased || - ownerReady(mappingTable(i).owner) || - !ownerActive(mappingTable(i).owner) - - when(mappingTable(i).valid) { - when(isRead) { - when(ballRouteOpen) { - io.mem_req(i).read <> ballRead - io.mem_req(i).bank_id := rbank_id - io.mem_req(i).group_id := rgroup_id - io.mem_req(i).is_shared := r_shared - io.mem_req(i).rob_id := rrob_id - io.mem_req(i).inst_id := io.inst_ids(rrob_id) - } - }.otherwise { - when(ballRouteOpen) { - io.mem_req(i).write <> ballWrite - io.mem_req(i).bank_id := wbank_id - io.mem_req(i).group_id := wgroup_id - io.mem_req(i).is_shared := w_shared - io.mem_req(i).rob_id := wrob_id - io.mem_req(i).inst_id := io.inst_ids(wrob_id) - } - } - } - } - - // Mapping table release - for (i <- 0 until b.memDomain.bankChannel) { - val releaseCounter = RegInit(0.U(5.W)) - val readPending = RegInit(0.U(32.W)) - val writePending = RegInit(0.U(32.W)) - when(io.mem_req(i).read.req.fire =/= io.mem_req(i).read.resp.fire) { - readPending := Mux(io.mem_req(i).read.req.fire, readPending + 1.U, readPending - 1.U) - } - when(io.mem_req(i).write.req.fire =/= io.mem_req(i).write.resp.fire) { - writePending := Mux(io.mem_req(i).write.req.fire, writePending + 1.U, writePending - 1.U) - } - - // Releasing a Ball's logical channel can precede the final bank response. - // Keep its physical route until that response has been consumed; otherwise - // the response loses its consumer and leaves the AccPipe busy. - val ballReleased = mappingTable(i).leased && - !ownerActive(mappingTable(i).owner) - val channelDrained = (readPending === 0.U) && (writePending === 0.U) && !(io.mem_req(i).read.resp.valid || - io.mem_req(i).write.resp.valid || io.mem_req(i).read.req.valid || - io.mem_req(i).write.req.valid) - - when(mappingTable(i).valid && ballReleased && channelDrained) { - mappingTable(i).valid := false.B - mappingTable(i).isRead := false.B - mappingTable(i).id := 0.U - mappingTable(i).leased := false.B - mappingTable(i).owner := 0.U - when(mappingTable(i).isRead) { - readPortAllocated(mappingTable(i).id) := false.B - }.otherwise { - writePortAllocated(mappingTable(i).id) := false.B - } - releaseCounter := 0.U - }.elsewhen(mappingTable(i).valid && !mappingTable(i).leased && channelDrained) { - releaseCounter := releaseCounter + 1.U - - when(releaseCounter === 16.U) { - releaseCounter := 0.U - mappingTable(i).valid := false.B - mappingTable(i).isRead := false.B - mappingTable(i).id := 0.U - mappingTable(i).leased := false.B - mappingTable(i).owner := 0.U - when(mappingTable(i).isRead) { - readPortAllocated(mappingTable(i).id) := false.B - }.otherwise { - writePortAllocated(mappingTable(i).id) := false.B - } - } - }.otherwise { - releaseCounter := 0.U - } - } -} diff --git a/arch/src/main/scala/framework/memdomain/utils/pmc/MemCyclePMC.scala b/arch/src/main/scala/framework/memdomain/utils/pmc/MemCyclePMC.scala deleted file mode 100644 index 5f9c02239..000000000 --- a/arch/src/main/scala/framework/memdomain/utils/pmc/MemCyclePMC.scala +++ /dev/null @@ -1,79 +0,0 @@ -package framework.memdomain.utils.pmc - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.memdomain.frontend.cmd.rs.{MemRsComplete, MemRsIssue} -import chisel3.experimental.hierarchy.{instantiable, public} - -@instantiable -class MemCyclePMC(val b: GlobalConfig) extends Module { - - @public - val io = IO(new Bundle { - val ldReq_i = Input(Valid(new MemRsIssue(b))) - val stReq_i = Input(Valid(new MemRsIssue(b))) - val ldResp_o = Input(Valid(new MemRsComplete(b))) - val stResp_o = Input(Valid(new MemRsComplete(b))) - val ldTotalCycles = Output(UInt(64.W)) - val stTotalCycles = Output(UInt(64.W)) - }) - - val cycleCounter = RegInit(0.U(64.W)) - cycleCounter := cycleCounter + 1.U - - val startTime = Reg(Vec(b.frontend.rob_entries, UInt(64.W))) - val ldTotalCycles = RegInit(0.U(64.W)) - val stTotalCycles = RegInit(0.U(64.W)) - - // DPI-C trace modules for load and store - val ldPmcTrace = Module(new MemPMCTraceDPI) - val stPmcTrace = Module(new MemPMCTraceDPI) - - ldPmcTrace.io.clock := clock - ldPmcTrace.io.reset := reset.asBool - ldPmcTrace.io.is_store := 0.U - ldPmcTrace.io.rob_id := 0.U - ldPmcTrace.io.elapsed := 0.U - ldPmcTrace.io.enable := false.B - - stPmcTrace.io.clock := clock - stPmcTrace.io.reset := reset.asBool - stPmcTrace.io.is_store := 1.U - stPmcTrace.io.rob_id := 0.U - stPmcTrace.io.elapsed := 0.U - stPmcTrace.io.enable := false.B - - when(io.ldReq_i.valid) { - startTime(io.ldReq_i.bits.rob_id) := cycleCounter - } - - when(io.stReq_i.valid) { - startTime(io.stReq_i.bits.rob_id) := cycleCounter - } - - when(io.ldResp_o.valid) { - val robId = io.ldResp_o.bits.rob_id - val elapsed = cycleCounter - startTime(robId) - ldTotalCycles := ldTotalCycles + elapsed - - // DPI-C trace output - ldPmcTrace.io.rob_id := robId - ldPmcTrace.io.elapsed := elapsed - ldPmcTrace.io.enable := true.B - } - - when(io.stResp_o.valid) { - val robId = io.stResp_o.bits.rob_id - val elapsed = cycleCounter - startTime(robId) - stTotalCycles := stTotalCycles + elapsed - - // DPI-C trace output - stPmcTrace.io.rob_id := robId - stPmcTrace.io.elapsed := elapsed - stPmcTrace.io.enable := true.B - } - - io.ldTotalCycles := ldTotalCycles - io.stTotalCycles := stTotalCycles -} diff --git a/arch/src/main/scala/framework/memdomain/utils/pmc/MemPMCTraceDPI.scala b/arch/src/main/scala/framework/memdomain/utils/pmc/MemPMCTraceDPI.scala deleted file mode 100644 index 69ba68b8c..000000000 --- a/arch/src/main/scala/framework/memdomain/utils/pmc/MemPMCTraceDPI.scala +++ /dev/null @@ -1,64 +0,0 @@ -package framework.memdomain.utils.pmc - -import chisel3._ -import chisel3.util._ -import framework.dpi.DpiGuard - -class MemPMCTraceDPI extends BlackBox with HasBlackBoxInline { - - val io = IO(new Bundle { - val clock = Input(Clock()) - val reset = Input(Bool()) - val is_store = Input(UInt(8.W)) - val rob_id = Input(UInt(32.W)) - val elapsed = Input(UInt(64.W)) - val enable = Input(Bool()) - }) - - setInline( - "MemPMCTraceDPI.v", - """ - |module MemPMCTraceDPI( - | input clock, - | input reset, - | input [7:0] is_store, - | input [31:0] rob_id, - | input [63:0] elapsed, - | input enable - |); - |""".stripMargin + DpiGuard.wrapPMCTrace( - """ - | import "DPI-C" context function void dpi_mem_pmctrace( - | input int unsigned is_store, - | input int unsigned rob_id, - | input int unsigned elapsed_lo, - | input int unsigned elapsed_hi - | ); - | reg [7:0] is_store_reg; - | reg [31:0] rob_id_reg; - | reg [63:0] elapsed_reg; - | reg valid_reg; - | - | always @(posedge clock) begin - | if (reset) begin - | valid_reg <= 1'b0; - | end else begin - | if (valid_reg) begin - | dpi_mem_pmctrace(is_store_reg, rob_id_reg, elapsed_reg[31:0], elapsed_reg[63:32]); - | end - | - | valid_reg <= enable; - | if (enable) begin - | is_store_reg <= is_store; - | rob_id_reg <= rob_id; - | elapsed_reg <= elapsed; - | end - | end - | end - |""".stripMargin - ) + - """ - |endmodule - """.stripMargin - ) -} diff --git a/arch/src/main/scala/framework/memdomain/verification/.gitignore b/arch/src/main/scala/framework/memdomain/verification/.gitignore deleted file mode 100644 index 5efea52c3..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/build/ -/src/csrc/target/ -/cm.log -/ucli.key diff --git a/arch/src/main/scala/framework/memdomain/verification/src/csrc/Cargo.lock b/arch/src/main/scala/framework/memdomain/verification/src/csrc/Cargo.lock deleted file mode 100644 index 2c71631e6..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/csrc/Cargo.lock +++ /dev/null @@ -1,268 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "bebop-bank-hash" -version = "0.1.0" -dependencies = [ - "serde", - "serde_json", - "snafu", -] - -[[package]] -name = "bebop-elf" -version = "0.1.0" -dependencies = [ - "bebop-memory", -] - -[[package]] -name = "bebop-memory" -version = "0.1.0" -dependencies = [ - "log", - "serde", - "serde_json", - "sha2", -] - -[[package]] -name = "bebop-rtl-trace" -version = "0.1.0" -dependencies = [ - "bebop-bank-hash", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "cfg-if" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "libc" -version = "0.2.190" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" - -[[package]] -name = "log" -version = "0.4.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "memdomain-ack-reference" -version = "0.1.0" -dependencies = [ - "bebop-elf", - "bebop-memory", - "bebop-rtl-trace", -] - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", -] - -[[package]] -name = "serde_json" -version = "1.0.151" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "snafu" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" -dependencies = [ - "snafu-derive", -] - -[[package]] -name = "snafu-derive" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "syn" -version = "2.0.119" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" - -[[package]] -name = "unicode-ident" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/arch/src/main/scala/framework/memdomain/verification/src/csrc/Cargo.toml b/arch/src/main/scala/framework/memdomain/verification/src/csrc/Cargo.toml deleted file mode 100644 index b335a27c0..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/csrc/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "memdomain-ack-reference" -version = "0.1.0" -edition = "2021" -[lib] -crate-type = ["cdylib", "staticlib"] - -[dependencies] -bebop-rtl-trace = { path = "../../../../../../../../../bebop/src/nodes/lib/rtl-trace" } -bebop-elf = { path = "../../../../../../../../../bebop/src/nodes/lib/elf" } -bebop-memory = { path = "../../../../../../../../../bebop/src/nodes/lib/memory" } diff --git a/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/lib.rs b/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/lib.rs deleted file mode 100644 index e256fc4f7..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/lib.rs +++ /dev/null @@ -1,12 +0,0 @@ -mod memory; -mod tile_peer; -pub use memory::*; -#[path = "../../../../../mem-core/ddr/src/csrc/src/lib.rs"] -mod ddr_reference; - -#[no_mangle] -pub extern "C" fn admission_trace_init() { - bebop_rtl_trace::init_trace(std::path::Path::new("build/admission_trace"), - bebop_rtl_trace::TraceConfig { itrace: true, mtrace: true, pmctrace: true, - ctrace: false, banktrace: false }).expect("initialize AdmissionSystem trace"); -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/memory.rs b/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/memory.rs deleted file mode 100644 index db01c630f..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/memory.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! Byte-addressed golden storage, independent of DUT state and completion timing. -use std::collections::BTreeMap; -#[derive(Default)] -pub struct Memory(BTreeMap); -#[no_mangle] -pub extern "C" fn ack_ref_create() -> *mut Memory { Box::into_raw(Box::new(Memory::default())) } -#[no_mangle] -pub unsafe extern "C" fn ack_ref_destroy(m: *mut Memory) { drop(Box::from_raw(m)); } -#[no_mangle] -pub unsafe extern "C" fn ack_ref_write(m: *mut Memory, addr: u64, lo: u64, hi: u64, mask: u32) { - let data = (lo as u128 | ((hi as u128) << 64)).to_le_bytes(); - for (i, byte) in data.into_iter().enumerate() { if mask & (1 << i) != 0 { (*m).0.insert(addr + i as u64, byte); } } -} -#[no_mangle] -pub unsafe extern "C" fn ack_ref_check(m: *mut Memory, addr: u64, lo: u64, hi: u64) -> u32 { - let data = (lo as u128 | ((hi as u128) << 64)).to_le_bytes(); - u32::from(data.iter().enumerate().all(|(i, byte)| (*m).0.get(&(addr + i as u64)) == Some(byte))) -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/tile_peer.rs b/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/tile_peer.rs deleted file mode 100644 index cc62540c5..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/csrc/src/tile_peer.rs +++ /dev/null @@ -1,147 +0,0 @@ -use bebop_memory::Memory; -use std::cell::RefCell; -use std::collections::BTreeSet; -use std::ffi::{c_char, c_void, CStr}; -use std::path::Path; - -const BASE: u64 = 0x8000_0000; -const BYTES: usize = 16usize << 30; - -// External DDR starts zeroed. Storage is allocated one line at a time, using -// the maintained DDR reference for every program/read/write operation. -struct TileMemory { - model: *mut c_void, - initialized: RefCell>, -} -impl TileMemory { - fn new() -> Self { - Self { model: crate::ddr_reference::ddr_ref_create(), initialized: RefCell::new(BTreeSet::new()) } - } - fn line(&self, address: u64) -> [u32; 16] { - let mut data = [0u32; 16]; - if self.initialized.borrow_mut().insert(address) { - unsafe { crate::ddr_reference::ddr_ref_program(self.model, address, data.as_ptr()); } - } - assert_eq!(unsafe { crate::ddr_reference::ddr_ref_read(self.model, address, data.as_mut_ptr()) }, 1, - "DDR reference line must be initialized"); - data - } - fn check(offset: usize, bytes: usize) { - assert!(offset.checked_add(bytes).is_some_and(|end| end <= BYTES), "Tile DDR access outside 16 GiB"); - } -} -impl Drop for TileMemory { - fn drop(&mut self) { unsafe { crate::ddr_reference::ddr_ref_destroy(self.model); } } -} -impl Memory for TileMemory { - fn len(&self) -> usize { BYTES } - fn read_buffer(&self, offset: usize, output: &mut [u8]) { - Self::check(offset, output.len()); - let mut cursor = 0; - while cursor < output.len() { - let address = BASE + (offset + cursor) as u64; - let line = self.line(address & !63); - let start = (address & 63) as usize; - let count = (64 - start).min(output.len() - cursor); - for i in 0..count { - output[cursor + i] = (line[(start + i) / 4] >> (((start + i) % 4) * 8)) as u8; - } - cursor += count; - } - } - fn write_buffer(&self, offset: usize, input: &[u8]) { - Self::check(offset, input.len()); - let mut cursor = 0; - while cursor < input.len() { - let address = BASE + (offset + cursor) as u64; - let mut line = self.line(address & !63); - let start = (address & 63) as usize; - let count = (64 - start).min(input.len() - cursor); - for i in 0..count { - let shift = ((start + i) % 4) * 8; - line[(start + i) / 4] = (line[(start + i) / 4] & !(255 << shift)) | ((input[cursor + i] as u32) << shift); - } - unsafe { crate::ddr_reference::ddr_ref_program(self.model, address & !63, line.as_ptr()); } - cursor += count; - } - } - fn fill(&self, offset: usize, bytes: usize, value: u8) { - Self::check(offset, bytes); - let block = [value; 64]; - for cursor in (0..bytes).step_by(64) { self.write_buffer(offset + cursor, &block[..64.min(bytes - cursor)]); } - } -} -thread_local! { static DDR: RefCell> = const { RefCell::new(None) }; } -thread_local! { static EXPECTED: RefCell> = const { RefCell::new(None) }; } - -#[no_mangle] -pub unsafe extern "C" fn tile_memory_init(image_path: *const c_char, expected_path: *const c_char) -> u64 { - let image = CStr::from_ptr(image_path).to_str().expect("Tile image path UTF-8"); - let expected = CStr::from_ptr(expected_path).to_str().expect("Tile expected bytes path UTF-8"); - assert!(!image.is_empty() && !expected.is_empty(), "Explicit Tile image and expected-byte paths are required"); - let memory = TileMemory::new(); - let entry = bebop_elf::load_elf(image, &memory, BASE, 0).expect("load Tile ELF").entry; - assert!((BASE..BASE + BYTES as u64).contains(&entry), "Tile entry outside DDR"); - DDR.with(|ddr| { assert!(ddr.borrow().is_none(), "Tile peer initialized twice"); *ddr.borrow_mut() = Some(memory); }); - EXPECTED.with(|path| { *path.borrow_mut() = Some(expected.to_owned()); }); - bebop_rtl_trace::init_trace(Path::new("build/tile_system_trace"), bebop_rtl_trace::TraceConfig { - itrace: true, mtrace: true, pmctrace: true, ctrace: true, banktrace: true, - }).expect("initialize actual Tile RTL traces"); - println!("TILE_PEER_IMAGE image={image} entry={entry:#x} DDR=16GiB zero-initialized"); - entry -} - -#[no_mangle] -pub extern "C" fn tile_peer_read64(address: u64) -> u64 { - DDR.with(|ddr| { - let memory = ddr.borrow(); - let mut data = [0; 8]; - memory.as_ref().expect("Tile peer not initialized").read_buffer(address.checked_sub(BASE).expect("AXI address below DDR") as usize, &mut data); - u64::from_le_bytes(data) - }) -} -#[no_mangle] -pub extern "C" fn tile_peer_write128(address: u64, lo: u64, hi: u64, mask: u32) { - assert_eq!(mask & !0xffff, 0, "AXI 128-bit write strobe width"); - assert_eq!(address & 15, 0, "AXI DDR write beat alignment"); - DDR.with(|ddr| { - let memory = ddr.borrow(); - let memory = memory.as_ref().expect("Tile peer not initialized"); - let offset = address.checked_sub(BASE).expect("AXI address below DDR") as usize; - TileMemory::check(offset, 16); - let mut data = memory.line(address & !63); - let incoming = (lo as u128 | ((hi as u128) << 64)).to_le_bytes(); - let start = (address & 63) as usize; - for i in 0..16 { - let shift = ((start + i) % 4) * 8; - data[(start + i) / 4] = (data[(start + i) / 4] & !(255 << shift)) | ((incoming[i] as u32) << shift); - } - assert_eq!(unsafe { crate::ddr_reference::ddr_ref_write(memory.model, address & !63, - data.as_ptr(), (mask as u64) << start) }, 1, "DDR AXI write to initialized reference line"); - }); -} -#[no_mangle] -pub extern "C" fn tile_peer_finish() { - let expected = EXPECTED.with(|path| path.borrow().as_ref().expect("Explicit Tile expected bytes path required").clone()); - let text = std::fs::read_to_string(&expected).expect("read expected Tile DDR output bytes"); - let mut compared = 0; - DDR.with(|ddr| { - let memory = ddr.borrow(); - let memory = memory.as_ref().expect("Tile peer not initialized"); - for line in text.lines().filter(|line| !line.trim().is_empty()) { - let fields: Vec<_> = line.split_whitespace().collect(); - assert_eq!(fields.len(), 2, "Expected DDR row must contain address and hex bytes"); - let address = u64::from_str_radix(fields[0].trim_start_matches("0x"), 16).expect("expected DDR address"); - assert!(!fields[1].is_empty() && fields[1].len() % 2 == 0, "expected DDR byte string"); - let bytes: Vec<_> = (0..fields[1].len()).step_by(2).map(|i| - u8::from_str_radix(&fields[1][i..i + 2], 16).expect("expected DDR byte")).collect(); - let mut actual = vec![0; bytes.len()]; - memory.read_buffer(address.checked_sub(BASE).expect("expected DDR address below RAM") as usize, &mut actual); - assert_eq!(actual, bytes, "Actual Tile NPU DDR bytes differ at {address:#x}"); - compared += actual.len(); - } - }); - assert!(compared > 0, "No independent Tile DDR output comparisons"); - println!("TILE_PEER_DDR_COMPARE bytes={compared} expected={expected}"); - bebop_rtl_trace::write_trace_summary(Path::new("build/tile_system_trace")).expect("write Tile RTL trace summary"); -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/ack_package.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/ack_package.sv deleted file mode 100644 index 47a9f7bb8..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/ack_package.sv +++ /dev/null @@ -1,17 +0,0 @@ -package ack_pkg; - import "DPI-C" function chandle ack_ref_create(); - import "DPI-C" function void ack_ref_destroy(input chandle model); - import "DPI-C" function void ack_ref_write( - input chandle model, - input longint unsigned addr, - lo, - hi, - input int unsigned mask - ); - import "DPI-C" function int unsigned ack_ref_check( - input chandle model, - input longint unsigned addr, - lo, - hi - ); -endpackage diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.cm_hier b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.cm_hier deleted file mode 100644 index cdc734792..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.cm_hier +++ /dev/null @@ -1,3 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.command_if -+tree @TOP@.complete_if diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.f deleted file mode 100644 index 3022d4f8c..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/AdmissionSystem/filelist.f -@RESOURCES@/admission_system_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.toml b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.toml deleted file mode 100644 index eb0e075b4..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system.toml +++ /dev/null @@ -1 +0,0 @@ -chips = ["goban"] diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system_tb.sv deleted file mode 100644 index ab66f8308..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/admission_system_tb.sv +++ /dev/null @@ -1,479 +0,0 @@ -`include "admission_system_config.svh" -module admission_system_tb; - import uvm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function chandle ddr_ref_create(); - import "DPI-C" function void admission_trace_init(); - import "DPI-C" function void dpi_bdb_set_clk(input longint unsigned cycle); - import "DPI-C" function void ddr_ref_destroy(input chandle model); - import "DPI-C" function void ddr_ref_program( - input chandle model, - input longint unsigned address, - input bit [511:0] data - ); - import "DPI-C" function int unsigned ddr_ref_read( - input chandle model, - input longint unsigned address, - output bit [511:0] data - ); - import "DPI-C" function int unsigned ddr_ref_write( - input chandle model, - input longint unsigned address, - input bit [511:0] data, - input longint unsigned mask - ); - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "admission_system_signals.svh" -AdmissionSystem dut ( - `include "admission_system_ports.svh" - ); - `include "admission_system_clocking.svh" -stream_if #(64 + 64 + 8) command_if ( - clock, - reset - ); - assign command_if.valid = io_core_command_valid; - assign command_if.ready = io_core_command_ready; - assign command_if.bits = { - io_core_command_bits_instruction_rs1Data, - io_core_command_bits_instruction_rs2Data, - io_core_command_bits_tag - }; - stream_if #(8) complete_if ( - clock, - reset - ); - assign complete_if.valid = io_core_complete_valid; - assign complete_if.ready = io_core_complete_ready; - assign complete_if.bits = io_core_complete_bits_tag; - chandle model; - bit returned[4] = '{default: 0}; - bit [511:0] line, wline; - bit [63:0] wmask; - int - cycle = 0, - completed = 0, - reads = 0, - writes = 0, - backs = 0, - aw_id = 0, - ar_id = 0, - rbeat = 0, - rbeats = 0, - wbeat = 0, - wbeats = 0, - r_due = 0, - b_due = 0, - maint_due = 0; - bit reading = 0, writing = 0, committed = 0, roff = 0, boff = 0, maint_pending = 0, allow_b = 1; - localparam int ROWS = `ADMISSION_ROWS; - localparam int BYTES = ROWS * 16; - localparam int EXPECTED_BURSTS = (ROWS + 255) / 256; - localparam longint unsigned INPUT_BASE = 'h80010000; - localparam longint unsigned OUTPUT_BASE = 'h80020000; - localparam longint unsigned UNALIGNED_BASE = 'h80024ff3; - int expected_total_b = EXPECTED_BURSTS, expected_store_tag = 2; - bit unaligned_phase = 0, expectedFaultPhase = 0; - int fault_start_aw = 0, fault_start_b = 0, fault_start_completed = 0; - longint unsigned aw_address = 0, ar_address = 0; - int maint_tag = 0; - bit finish_wait = 0; - function automatic bit [511:0] pattern(int line_index = 0); - bit [511:0] data; - for (int i = 0; i < 64; i++) data[i*8+:8] = line_index * 37 + i * 11 + 3; - return data; - endfunction - task automatic verify_contract(bit ok, string message); - if (!ok) `uvm_fatal("ADMISSION_SYSTEM", message) - endtask - task automatic memory_and_checks(); - forever begin - @(sample); - if (!sample.reset) begin - cycle++; - dpi_bdb_set_clk(cycle); - if (!expectedFaultPhase) - verify_contract( - !sample.io_halted, $sformatf( - "halted error%0d address%h", sample.io_fault_error, sample.io_fault_address)); - else begin - verify_contract(!sample.io_core_complete_valid, - "errored store published Core completion"); - if (writes > fault_start_aw || sample.io_halted) - verify_contract(!sample.io_core_cpuAllow, "errored write range was released to CPU"); - if (sample.io_halted) - verify_contract( - sample.io_fault_error == 2 && sample.io_fault_address == OUTPUT_BASE && - sample.io_faultTag == 0, - "late BERROR halted with incorrect typed fault or tag"); - end - if (sample.io_core_complete_valid && sample.io_core_complete_ready) begin - int id; - id = sample.io_core_complete_bits_tag; - verify_contract(!returned[id], "duplicate Core completion"); - returned[id] = 1; - completed++; - if (id == expected_store_tag) - verify_contract( - writes == expected_total_b && - backs + int'(sample.io_axi_b_valid && sample.io_axi_b_ready) == expected_total_b && - (!writing || (sample.io_axi_b_valid && sample.io_axi_b_ready)), - "MVOUT Core release preceded final B for the whole bank"); - end - if (sample.io_core_maintenance_valid && sample.io_core_maintenance_ready) begin - verify_contract(!maint_pending, "maintenance overlap"); - maint_pending = 1; - maint_tag = sample.io_core_maintenance_bits_tag; - maint_due = cycle + 8; - end - if (sample.io_core_maintained_valid && sample.io_core_maintained_ready) maint_pending = 0; - if (sample.io_axi_ar_valid && sample.io_axi_ar_ready) begin - verify_contract( - !reading && sample.io_axi_ar_bits_size == 4 && sample.io_axi_ar_bits_burst == 1 && - sample.io_axi_ar_bits_addr[3:0] == 0 && sample.io_axi_ar_bits_len + 1 <= 256 && - sample.io_axi_ar_bits_addr[11:0] + (sample.io_axi_ar_bits_len + 1) * 16 <= 4096, - "wrong AR"); - verify_contract( - sample.io_axi_ar_bits_addr == INPUT_BASE + reads * 4096 && - sample.io_axi_ar_bits_len + 1 == ((ROWS - reads * 256) < 256 ? ROWS - reads * 256 : 256), - "read did not fill a legal 256-beat/page segment"); - reading = 1; - ar_id = sample.io_axi_ar_bits_id; - rbeat = 0; - rbeats = sample.io_axi_ar_bits_len + 1; - ar_address = sample.io_axi_ar_bits_addr; - r_due = cycle + 4; - reads++; - end - if (sample.io_axi_r_valid && sample.io_axi_r_ready) begin - roff = 0; - rbeat++; - r_due = cycle + 3; - if (rbeat == rbeats) reading = 0; - end - if (sample.io_axi_aw_valid && sample.io_axi_aw_ready) begin - verify_contract( - !writing && sample.io_axi_aw_bits_size == 4 && sample.io_axi_aw_bits_burst == 1 && - sample.io_axi_aw_bits_addr[3:0] == 0 && sample.io_axi_aw_bits_len + 1 <= 256 && - sample.io_axi_aw_bits_addr[11:0] + (sample.io_axi_aw_bits_len + 1) * 16 <= 4096, - "wrong AW"); - if (expectedFaultPhase) - verify_contract( - writes == fault_start_aw && sample.io_axi_aw_bits_addr == OUTPUT_BASE && - sample.io_axi_aw_bits_len == 255, - "BERROR case emitted a later page or lost its first 256-beat burst"); - else if (!unaligned_phase) - verify_contract( - sample.io_axi_aw_bits_addr == OUTPUT_BASE + writes * 4096 && - sample.io_axi_aw_bits_len + 1 == ((ROWS - writes * 256) < 256 ? ROWS - writes * 256 : 256), - "write did not fill a legal 256-beat/page segment"); - else begin - int segment; - segment = writes - EXPECTED_BURSTS; - verify_contract( - (segment == 0 && sample.io_axi_aw_bits_addr == (UNALIGNED_BASE & ~64'd15) && sample.io_axi_aw_bits_len == 0) || - (segment == 1 && sample.io_axi_aw_bits_addr == ((UNALIGNED_BASE + 13) & ~64'd15) && sample.io_axi_aw_bits_len == 3), - "unaligned cross-page AW segments wrong"); - end - writing = 1; - aw_id = sample.io_axi_aw_bits_id; - aw_address = sample.io_axi_aw_bits_addr; - wbeats = sample.io_axi_aw_bits_len + 1; - wbeat = 0; - wline = '0; - wmask = '0; - committed = 0; - boff = 0; - writes++; - end - if (sample.io_axi_w_valid && sample.io_axi_w_ready) begin - verify_contract(writing && sample.io_axi_w_bits_last == (wbeat + 1 == wbeats), - "W lacks actual AW or WLAST incorrect"); - if (unaligned_phase) - verify_contract( - sample.io_axi_w_bits_strb == (writes == EXPECTED_BURSTS + 1 ? 'hfff8 : wbeat == wbeats - 1 ? 'h0007 : 'hffff), - "unaligned cross-page first/tail WSTRB incorrect"); - begin - longint unsigned beat_address; - int line_offset; - beat_address = aw_address + wbeat * 16; - line_offset = beat_address & 63; - wline = '0; - wmask = '0; - wline[line_offset*8+:128] = sample.io_axi_w_bits_data; - wmask[line_offset+:16] = sample.io_axi_w_bits_strb; - verify_contract(ddr_ref_write(model, beat_address & ~64'd63, wline, wmask) == 1, - "DDR beat commit failed across reference-line mapping"); - end - wbeat++; - if (wbeat == wbeats) begin - committed = 1; - b_due = cycle + 8; - end - end - if (sample.io_axi_b_valid && sample.io_axi_b_ready) begin - writing = 0; - boff = 0; - backs++; - end - if(sample.io_core_command_valid && sample.io_core_command_bits_instruction_opcode=='h2b && sample.io_core_command_bits_instruction_funct==2 && - (!returned[expected_store_tag] || backs < expected_total_b || writing)) - verify_contract(!sample.io_core_command_ready, - "Task finish published with pending DDR/ROB work"); - end - @(negedge clock); - io_axi_ar_ready = !reading && cycle % 5 != 0; - io_axi_aw_ready = !writing && cycle % 4 != 0; - io_axi_w_ready = writing && wbeat < wbeats && cycle % 3 != 0; - if (reading && !roff && cycle >= r_due) begin - io_axi_r_valid = 1; - io_axi_r_bits_id = ar_id; - begin - longint unsigned beat_address; - int line_offset; - beat_address = ar_address + rbeat * 16; - line_offset = beat_address & 63; - verify_contract(ddr_ref_read(model, beat_address & ~64'd63, line) == 1, - "unprogrammed DDR line during long read burst"); - io_axi_r_bits_data = line[line_offset*8+:128]; - end - io_axi_r_bits_resp = 0; - io_axi_r_bits_last = rbeat + 1 == rbeats; - roff = 1; - end - if (!roff) io_axi_r_valid = 0; - io_axi_b_valid = writing && committed && cycle >= b_due && allow_b; - io_axi_b_bits_id = aw_id; - io_axi_b_bits_resp = expectedFaultPhase ? 2 : 0; - io_core_maintenance_ready = !maint_pending; - io_core_maintained_valid = maint_pending && cycle >= maint_due; - io_core_maintained_bits_tag = maint_tag; - io_core_maintained_bits_ok = 1; - end - endtask - task automatic controller(int fn, longint unsigned a, longint unsigned b, - output longint unsigned result); - @(negedge clock); - io_controller_cmd_bits_opcode = 'h2b; - io_controller_cmd_bits_funct = fn; - io_controller_cmd_bits_rs1Data = a; - io_controller_cmd_bits_rs2Data = b; - io_controller_cmd_bits_rd = 10; - io_controller_cmd_valid = 1; - io_controller_resp_ready = 1; - do @(sample); while (!sample.io_controller_cmd_ready); - @(negedge clock); - io_controller_cmd_valid = 0; - do @(sample); while (!sample.io_controller_resp_valid); - result = sample.io_controller_resp_bits_data; - @(negedge clock); - io_controller_resp_ready = 0; - endtask - task automatic command(int tag, int opcode, int fn, longint unsigned a, longint unsigned b, - bit permission = 1); - @(negedge clock); - io_core_reserve_bits_id = tag; - io_core_reserve_valid = 1; - do @(sample); while (!sample.io_core_reserve_ready); - @(negedge clock); - io_core_reserve_valid = 0; - io_core_command_bits_tag = tag; - io_core_command_bits_satp = 0; - io_core_command_bits_effectivePrivilege = 1; - io_core_command_bits_instruction_opcode = opcode; - io_core_command_bits_instruction_funct = fn; - io_core_command_bits_instruction_funct3 = 3; - io_core_command_bits_instruction_xs1 = 1; - io_core_command_bits_instruction_xs2 = 1; - io_core_command_bits_instruction_rd = 10; - io_core_command_bits_instruction_xd = opcode == 'h2b; - io_core_command_bits_instruction_rs1Data = a; - io_core_command_bits_instruction_rs2Data = b; - io_core_command_bits_pmp_0_cfg_a = 1; - io_core_command_bits_pmp_0_cfg_r = permission; - io_core_command_bits_pmp_0_cfg_w = permission; - io_core_command_bits_pmp_0_addr = 'h20400000; - io_core_command_valid = 1; - do @(sample); while (!sample.io_core_command_ready); - @(negedge clock); - io_core_command_valid = 0; - // The live input is now denied. Permission must keep the accepted snapshot. - io_core_command_bits_pmp_0_cfg_r = 0; - io_core_command_bits_pmp_0_cfg_w = 0; - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 1ms); - run_test("protocol_test"); - end - initial begin - longint unsigned value; - bit [511:0] actual, first_expected, second_expected, source_data; - `include "admission_system_init.svh" -admission_trace_init(); - model = ddr_ref_create(); - verify_contract(ROWS > 256 && BYTES % 64 == 0, - "fixture must exercise a full bank across pages"); - for (int index = 0; index < BYTES / 64; index++) begin - ddr_ref_program(model, INPUT_BASE + index * 64, pattern(index)); - ddr_ref_program(model, OUTPUT_BASE + index * 64, '0); - end - wait (ctl.start); - repeat (5) @(negedge clock); - reset = 0; - io_core_complete_ready = 1; - io_core_response_ready = 1; - fork - memory_and_checks(); - join_none - wait (io_npuIdle); - command(3, 'h7b, 32, 0, 'h421, 0); - wait (returned[3]); - returned[3] = 0; - for (int i = 0; i < 7; i++) controller(7, i, i == 6 ? `ADMISSION_SIGNATURE : 0, value); - controller(0, 0, `ADMISSION_SIGNATURE, value); - verify_contract(value == 0, "actual signature task submit failed"); - command(3, 'h2b, 8, 0, 0); - wait (returned[3]); - returned[3] = 0; - io_blockedResponse = 1; - io_core_cpuQuery_valid = 1; - io_core_cpuQuery_paddr = INPUT_BASE; - io_core_cpuQuery_sizeLog2 = 3; - io_core_cpuQuery_write = 1; - command(0, 'h7b, 33, 64'(ROWS) << 30, INPUT_BASE | (64'd1 << 39)); - @(sample); - verify_contract(!sample.io_core_cpuAllow, "CPU write passed reserved NPU read range"); - repeat (12) @(sample); - verify_contract(!returned[0], "context released at request acceptance"); - @(negedge clock); - io_blockedResponse = 0; - wait (returned[0]); - @(sample); - verify_contract(sample.io_core_cpuAllow, "CPU remained blocked after read completion"); - @(negedge clock); - allow_b = 0; - io_core_cpuQuery_paddr = OUTPUT_BASE; - io_core_cpuQuery_write = 0; - command(2, 'h7b, 16, 64'(ROWS) << 30, OUTPUT_BASE | (64'd1 << 39)); - wait (committed); - @(sample); - verify_contract(!sample.io_core_cpuAllow, - "CPU read passed NPU write before real DDR B and post-maintenance"); - fork - command(3, 'h2b, 2, 0, 0, 0); - begin - finish_wait = 1; - repeat (12) @(sample); - verify_contract(!returned[2], "MVOUT released before real B"); - @(negedge clock); - finish_wait = 0; - allow_b = 1; - end - join - wait (returned[2] && returned[3] && io_workDrained); - @(sample); - verify_contract(sample.io_core_cpuAllow, - "CPU read did not resume after actual write completion"); - verify_contract( - reads == EXPECTED_BURSTS && writes == EXPECTED_BURSTS && - backs == writes && !writing && !reading, - "full-bank AXI segments not drained"); - for (int index = 0; index < BYTES / 64; index++) - verify_contract(ddr_ref_read(model, OUTPUT_BASE + index * 64, actual) == 1 && actual == pattern( - index), "full-bank MVIN/MVOUT DDR golden mismatch"); - controller(1, 0, 0, value); - verify_contract(value == 1, "Task finish failed to publish actual completion"); - // Reuse the released tag for a tiny real WriteDma carry/mask transaction. - ddr_ref_program(model, UNALIGNED_BASE & ~64'd63, {64{8'ha5}}); - ddr_ref_program(model, (UNALIGNED_BASE + 13) & ~64'd63, {64{8'ha5}}); - @(negedge clock); - returned[0] = 0; - expected_store_tag = 0; - expected_total_b = EXPECTED_BURSTS + 2; - unaligned_phase = 1; - allow_b = 0; - committed = 0; - io_core_cpuQuery_paddr = UNALIGNED_BASE & ~64'd7; - io_core_cpuQuery_write = 0; - command(0, 'h7b, 16, 64'd4 << 30, UNALIGNED_BASE | (64'd1 << 39)); - wait (committed); - repeat (12) begin - @(sample); - verify_contract(!returned[0] && !sample.io_core_cpuAllow, - "unaligned store released before its delayed first B"); - end - @(negedge clock); - allow_b = 1; - wait (returned[0] && io_workDrained); - @(sample); - verify_contract( - backs == expected_total_b && writes == expected_total_b && !writing && !reading && - sample.io_core_cpuAllow, - "unaligned Core release did not join both final B responses"); - source_data = pattern(0); - first_expected = {64{8'ha5}}; - second_expected = {64{8'ha5}}; - first_expected[51*8+:13*8] = source_data[0+:13*8]; - second_expected[0+:51*8] = source_data[13*8+:51*8]; - verify_contract(ddr_ref_read(model, UNALIGNED_BASE & ~64'd63, actual - ) == 1 && actual == first_expected, - "unaligned head carry corrupted requested data or three sentinel bytes"); - verify_contract(ddr_ref_read(model, (UNALIGNED_BASE + 13) & ~64'd63, actual - ) == 1 && actual == second_expected, - "cross-page carry/tail corrupted requested data or thirteen sentinel bytes"); - // First-page SLVERR is terminal: drain the remaining raw input without - // issuing later AWs, and retain the Admission reservation instead of release. - @(negedge clock); - fault_start_aw = writes; - fault_start_b = backs; - fault_start_completed = completed; - expectedFaultPhase = 1; - unaligned_phase = 0; - allow_b = 1; - committed = 0; - returned[0] = 0; - io_core_cpuQuery_paddr = OUTPUT_BASE; - io_core_cpuQuery_write = 0; - command(0, 'h7b, 16, 64'(ROWS) << 30, OUTPUT_BASE | (64'd1 << 39)); - do @(sample); while (!(sample.io_halted && sample.io_npuIdle)); - repeat (24) begin - @(sample); - verify_contract( - sample.io_halted && sample.io_npuIdle && sample.io_fault_error == 2 && - sample.io_fault_address == OUTPUT_BASE && sample.io_faultTag == 0, - $sformatf( - "BERROR terminal error=%0d addr=%h tag=%0d idle=%0d halted=%0d expected=%h", - sample.io_fault_error, - sample.io_fault_address, - sample.io_faultTag, - sample.io_npuIdle, - sample.io_halted, - OUTPUT_BASE - )); - verify_contract( - writes == fault_start_aw + 1 && backs == fault_start_b + 1 && - !writing && !reading && !sample.io_axi_aw_valid, - "BERROR emitted an extra AW/B or left an AXI transaction live"); - verify_contract( - !returned[0] && completed == fault_start_completed && - !sample.io_core_complete_valid && !sample.io_core_cpuAllow && !sample.io_workDrained, - "BERROR released its Core/CPU reservation instead of retaining the failed context"); - end - `uvm_info("ADMISSION_BERROR", - "First 256-beat B=SLVERR: remaining input drained, one AW/B only, fault tag/address retained and no Core/CPU release", - UVM_LOW) - `uvm_info("ADMISSION_SYSTEM", $sformatf( - "Completed%0d AR%0d AW%0d B%0d; actual Goban signature/PMP frozen, Task finish joined", - completed, - reads, - writes, - backs - ), UVM_LOW) - ddr_ref_destroy(model); - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.cm_hier b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.cm_hier deleted file mode 100644 index 122b5840e..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.cm_hier +++ /dev/null @@ -1,4 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.command_if -+tree @TOP@.response_if -+tree @TOP@.complete_if diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.f deleted file mode 100644 index 5e931ccdb..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/ControllerSystem/filelist.f -@RESOURCES@/controller_system_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.toml b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.toml deleted file mode 100644 index eb0e075b4..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system.toml +++ /dev/null @@ -1 +0,0 @@ -chips = ["goban"] diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system_tb.sv deleted file mode 100644 index ba533dd70..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/controller_system_tb.sv +++ /dev/null @@ -1,305 +0,0 @@ -`include "controller_system_config.svh" -module controller_system_tb; - import uvm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function void admission_trace_init(); - import "DPI-C" function void dpi_bdb_set_clk(input longint unsigned cycle); - import "DPI-C" function chandle ack_ref_create(); - import "DPI-C" function void ack_ref_destroy(input chandle model); - import "DPI-C" function void ack_ref_write( - input chandle model, - input longint unsigned address, - input longint unsigned lo, - input longint unsigned hi, - input int unsigned mask - ); - import "DPI-C" function int unsigned ack_ref_check( - input chandle model, - input longint unsigned address, - input longint unsigned lo, - input longint unsigned hi - ); - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "controller_system_signals.svh" -ControllerSystem dut ( - `include "controller_system_ports.svh" - ); - `include "controller_system_clocking.svh" -stream_if #(136) command_if ( - clock, - reset - ); - assign command_if.valid = io_core_command_valid; - assign command_if.ready = io_core_command_ready; - assign command_if.bits = { - io_core_command_bits_instruction_rs1Data, - io_core_command_bits_instruction_rs2Data, - io_core_command_bits_tag - }; - stream_if #(69) response_if ( - clock, - reset - ); - assign response_if.valid = io_core_response_valid; - assign response_if.ready = io_core_response_ready; - assign response_if.bits = {io_core_response_bits_rd, io_core_response_bits_data}; - stream_if #(8) complete_if ( - clock, - reset - ); - assign complete_if.valid = io_core_complete_valid; - assign complete_if.ready = io_core_complete_ready; - assign complete_if.bits = io_core_complete_bits_tag; - chandle model; - int cycle = 0, commands = 0, completions = 0, moves = 0, move_done = 0, target_writes = 0; - bit protect = 0; - function automatic bit [127:0] row_data(int row); - bit [127:0] data; - for (int j = 0; j < 16; j++) data[j*8+:8] = row * 31 + j * 7 + 3; - return data; - endfunction - task automatic verify_contract(bit ok, string message); - if (!ok) `uvm_fatal("CONTROLLER_SYSTEM", message) - endtask - task automatic monitor_contract(); - forever begin - @(sample); - if (!sample.reset) begin - cycle++; - dpi_bdb_set_clk(cycle); - verify_contract(!sample.io_moveFault_valid, "real bank move reported failure"); - if (sample.io_moveAccepted) moves++; - if (sample.io_moveCompleted) move_done++; - if (sample.io_bankWriteAccepted_1) target_writes++; - if (sample.io_core_command_valid && sample.io_core_command_ready) commands++; - if (sample.io_core_complete_valid && sample.io_core_complete_ready) begin - verify_contract(sample.io_core_complete_bits_tag == 0, "wrong completion owner"); - completions++; - end - if (protect) - verify_contract(!sample.io_core_cpuAllow, "pending move allowed CPU publication"); - end - end - endtask - task automatic issue(int opcode, int fn, longint unsigned a, longint unsigned b, - longint unsigned root = 'h8000000000012345); - @(negedge clock); - io_core_reserve_bits_id = 0; - io_core_reserve_valid = 1; - do @(sample); while (!sample.io_core_reserve_ready); - @(negedge clock); - io_core_reserve_valid = 0; - io_core_command_bits_tag = 0; - io_core_command_bits_satp = root; - io_core_command_bits_instruction_opcode = opcode; - io_core_command_bits_instruction_funct = fn; - io_core_command_bits_instruction_funct3 = 3; - io_core_command_bits_instruction_xs1 = 1; - io_core_command_bits_instruction_xs2 = 1; - io_core_command_bits_instruction_xd = opcode == 'h2b; - io_core_command_bits_instruction_rd = 11; - io_core_command_bits_instruction_rs1Data = a; - io_core_command_bits_instruction_rs2Data = b; - io_core_command_valid = 1; - if (opcode == 'h7b && (fn == 13 || fn == 0)) protect = 1; - do @(sample); while (!sample.io_core_command_ready); - @(negedge clock); - io_core_command_valid = 0; - endtask - task automatic receive_response(output longint unsigned value); - do @(sample); while (!sample.io_core_response_valid); - repeat (5) begin - @(sample); - verify_contract(sample.io_core_response_valid && sample.io_core_response_bits_rd == 11, - "response withdrawn/wrong rd"); - end - value = sample.io_core_response_bits_data; - @(negedge clock); - io_core_response_ready = 1; - do @(sample); while (!sample.io_core_response_valid); - @(negedge clock); - io_core_response_ready = 0; - endtask - task automatic complete_command(); - do @(sample); while (!sample.io_core_complete_valid); - repeat (5) begin - @(sample); - verify_contract(sample.io_core_complete_valid && sample.io_core_complete_bits_tag == 0, - "completion withdrawn/wrong tag"); - end - @(negedge clock); - io_core_complete_ready = 1; - do @(sample); while (!sample.io_core_complete_valid); - @(negedge clock); - io_core_complete_ready = 0; - protect = 0; - endtask - task automatic task_call(int fn, longint unsigned a, longint unsigned b, - output longint unsigned value); - issue('h2b, fn, a, b); - @(sample); - verify_contract(sample.io_core_cpuAllow, "task RPC unnecessarily blocked CPU memory"); - receive_response(value); - complete_command(); - endtask - task automatic worker_call(int fn, longint unsigned a, output longint unsigned value); - @(negedge clock); - io_workers_0_cmd_bits_opcode = 'h2b; - io_workers_0_cmd_bits_funct = fn; - io_workers_0_cmd_bits_rs1Data = a; - io_workers_0_cmd_bits_rs2Data = 0; - io_workers_0_cmd_bits_rd = 12; - io_workers_0_cmd_valid = 1; - io_workers_0_resp_ready = 0; - do @(sample); while (!sample.io_workers_0_cmd_ready); - @(negedge clock); - io_workers_0_cmd_valid = 0; - wait (io_workers_0_resp_valid); - repeat (3) @(sample); - value = sample.io_workers_0_resp_bits_data; - @(negedge clock); - io_workers_0_resp_ready = 1; - do @(sample); while (!sample.io_workers_0_resp_valid); - @(negedge clock); - io_workers_0_resp_ready = 0; - endtask - task automatic bank0(int row, bit write, bit [127:0] data, output bit [127:0] result); - @(negedge clock); - io_access_0_request_bits_bank = 0; - io_access_0_request_bits_addr = row; - io_access_0_request_bits_write = write; - io_access_0_request_bits_data = data; - io_access_0_request_bits_mask = '1; - io_access_0_request_bits_tag = 3; - io_access_0_request_valid = 1; - io_access_0_response_ready = 1; - do @(sample); while (!sample.io_access_0_request_ready); - @(negedge clock); - io_access_0_request_valid = 0; - do @(sample); while (!sample.io_access_0_response_valid); - verify_contract( - sample.io_access_0_response_bits_tag == 3 && !sample.io_access_0_response_bits_error, - "bank0 response owner/error"); - result = sample.io_access_0_response_bits_data; - @(negedge clock); - io_access_0_response_ready = 0; - endtask - task automatic bank1(int row, output bit [127:0] result); - @(negedge clock); - io_access_1_request_bits_bank = 0; - io_access_1_request_bits_addr = row; - io_access_1_request_bits_write = 0; - io_access_1_request_bits_mask = '1; - io_access_1_request_bits_tag = 4; - io_access_1_request_valid = 1; - io_access_1_response_ready = 1; - do @(sample); while (!sample.io_access_1_request_ready); - @(negedge clock); - io_access_1_request_valid = 0; - do @(sample); while (!sample.io_access_1_response_valid); - verify_contract( - sample.io_access_1_response_bits_tag == 4 && !sample.io_access_1_response_bits_error, - "bank1 response owner/error"); - result = sample.io_access_1_response_bits_data; - @(negedge clock); - io_access_1_response_ready = 0; - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 1ms); - run_test("protocol_test"); - end - initial begin - longint unsigned value; - bit [127:0] actual; - `include "controller_system_init.svh" -admission_trace_init(); - model = ack_ref_create(); - io_allowBankResponse_0 = 1; - io_allowBankResponse_1 = 1; - io_allowBankResponse_2 = 1; - io_allowBankResponse_3 = 1; - wait (ctl.start); - repeat (5) @(negedge clock); - reset = 0; - io_core_cpuQuery_valid = 1; - io_core_cpuQuery_paddr = 'h80001000; - io_core_cpuQuery_sizeLog2 = 3; - io_core_cpuQuery_write = 1; - fork - monitor_contract(); - join_none - task_call(3, 0, 0, value); - verify_contract(value == 4, "actual worker count"); - task_call(4, 0, 0, value); - verify_contract(value == `CONTROLLER_SIGNATURE, "actual signature mismatch"); - for (int i = 0; i < 7; i++) begin - task_call(7, i, i == 6 ? `CONTROLLER_SIGNATURE : 'h80000000 + i * 64, value); - verify_contract(value == 0, "task stage failed"); - end - task_call(0, 0, `CONTROLLER_SIGNATURE, value); - verify_contract(value == 0, "task submit failed"); - worker_call(8, 0, value); - verify_contract(value == 1, "worker did not receive real task"); - worker_call(9, 7, value); - verify_contract(value == 'h8000000000012345, "TaskController did not capture controller satp"); - worker_call(9, 0, value); - verify_contract(value == 'h80000000, "worker context corrupted"); - worker_call(2, 0, value); - verify_contract(value == 0, "worker finish failed"); - task_call(1, 0, 0, value); - verify_contract(value == 1, "controller did not observe success"); - for (int r = 3; r < 6; r++) begin - bit [127:0] expected; - expected = row_data(r); - bank0(r, 1, expected, actual); - ack_ref_write(model, r * 16, expected[63:0], expected[127:64], 'hffff); - end - fork - begin - wait (target_writes == 3); - @(negedge clock); - io_allowBankResponse_1 = 0; - repeat (12) begin - @(sample); - verify_contract( - !sample.io_core_response_valid && !sample.io_core_complete_valid && move_done == 0, - "controller completed before final actual bank ACK"); - end - @(negedge clock); - io_allowBankResponse_1 = 1; - end - begin - issue('h7b, 13, 64'd1 << 8, 64'd3 | (64'd7 << 16) | (64'd2 << 32)); - receive_response(value); - verify_contract(value == 0, "MVO response failed"); - complete_command(); - end - join - verify_contract(moves == 1 && move_done == 1 && target_writes == 3, - "real MVO request/completion count"); - for (int r = 7; r < 10; r++) begin - bank1(r, actual); - verify_contract(ack_ref_check(model, (r - 4) * 16, actual[63:0], actual[127:64]) == 1, - "MVO row data mismatch"); - end - @(sample); - verify_contract(sample.io_core_cpuAllow, - "CPU publication did not resume after move completion"); - verify_contract(commands == completions, - "accepted controller command not completed exactly once"); - `uvm_info("CONTROLLER_SYSTEM", $sformatf( - "Commands%0d completions%0d actual MVO%0d/%0d targetRows%0d; Task/satp/Bank/finalACK checked", - commands, - completions, - moves, - move_done, - target_writes - ), UVM_LOW) - ack_ref_destroy(model); - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma.cm_hier b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma.cm_hier deleted file mode 100644 index e8cdffc65..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma.cm_hier +++ /dev/null @@ -1,4 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.source_if -+tree @TOP@.sink_if -+tree @TOP@.result_if diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma.f deleted file mode 100644 index 05a60c3bb..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/KernelDma/filelist.f -@RESOURCES@/kernel_dma_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma_tb.sv deleted file mode 100644 index 801952487..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/kernel_dma_tb.sv +++ /dev/null @@ -1,348 +0,0 @@ -module kernel_dma_tb; - import uvm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "kernel_dma_signals.svh" -KernelDma dut ( - `include "kernel_dma_ports.svh" - ); - `include "kernel_dma_clocking.svh" -stream_if #(64 + 32) source_if ( - clock, - reset || io_kernel_abort - ); - assign source_if.valid = io_request_valid; - assign source_if.ready = io_request_ready; - assign source_if.bits = {io_request_bits_vaddr, io_request_bits_len}; - stream_if #(32) sink_if ( - clock, - reset || io_kernel_abort - ); - assign sink_if.valid = io_kernel_image_valid; - assign sink_if.ready = io_kernel_image_ready; - assign sink_if.bits = io_kernel_image_bits; - stream_if #(68) result_if ( - clock, - reset - ); - assign result_if.valid = io_kernel_result_valid; - assign result_if.ready = io_kernel_result_ready; - assign result_if.bits = {io_kernel_result_bits_error, io_kernel_result_bits_address}; - int images = 0, results = 0, requests = 0, cases = 0, checked = 0; - int expected_images = 0, expected_error = 0; - int expected_rob = 0; - longint unsigned base = 0, fault_address = 0; - int length = 0; - function automatic logic [31:0] word_value(int index); - return 32'hc1700081 ^ (index * 32'h1020305); - endfunction - function automatic logic [127:0] beat_value(int first); - logic [127:0] data; - for (int i = 0; i < 4; i++) data[32*i+:32] = word_value(first + i); - return data; - endfunction - task automatic verify_contract(bit condition, string message); - if (!condition) `uvm_fatal("KERNEL_DMA", message) - endtask - task automatic observe(); - forever begin - @(sample); - if (!sample.reset) begin - verify_contract(sample.io_footprint_valid == sample.io_kernel_busy, - "footprint lifetime differs from transaction ownership"); - if (sample.io_footprint_valid) begin - longint unsigned rounded_span; - rounded_span = (64'(length) + 15) & ~64'd15; - verify_contract( - sample.io_footprint_rob_id==expected_rob && - !sample.io_footprint_is_sub && sample.io_footprint_sub_rob_id==0 && - sample.io_footprint_baseVA==base && sample.io_footprint_rows==1 && - sample.io_footprint_columns==1 && sample.io_footprint_spanBytes==rounded_span && - sample.io_footprint_columnStride==0 && sample.io_footprint_rowStride==0 && - !sample.io_footprint_write, - "captured footprint changed or lost full VA/rounded transport span"); - verify_contract( - sample.io_footprint_fault_error==(expected_error==7 ? 7 : 0) && - sample.io_footprint_fault_address==(expected_error==7 ? base : 0), - "footprint shape fault changed while stalled"); - checked++; - end - if (sample.io_request_valid && sample.io_request_ready) begin - requests++; - verify_contract( - requests==1 && sample.io_request_bits_vaddr==base && - sample.io_request_bits_len==length && sample.io_request_bits_groups==1 && - sample.io_request_bits_stride==1 && !sample.io_request_bits_is_2d, - "wrong DMA request or duplicate dispatch"); - end - if (sample.io_kernel_image_valid && sample.io_kernel_image_ready) begin - verify_contract(sample.io_kernel_busy && images < expected_images, - "bad/later image word escaped"); - verify_contract(sample.io_kernel_image_bits === word_value(images), - "image word order/value differs from independent sequence"); - images++; - checked++; - end - if (sample.io_kernel_result_valid) begin - verify_contract(sample.io_kernel_busy && !sample.io_kernel_load_ready, - "terminal result did not retain ownership"); - verify_contract(images == expected_images, - "terminal result before all permitted image words consumed"); - verify_contract( - sample.io_kernel_result_bits_error==expected_error && - sample.io_kernel_result_bits_address==fault_address, - "wrong terminal full status/address"); - if (sample.io_kernel_result_ready) results++; - checked++; - end - end - end - endtask - task automatic configure(longint unsigned address, int bytes, int words, int error = 0, - longint unsigned error_address = 0); - @(negedge clock); - verify_contract(!io_kernel_busy && io_kernel_load_ready, - "previous transaction did not release"); - base = address; - length = bytes; - expected_images = words; - expected_error = error; - fault_address = error_address; - images = 0; - results = 0; - requests = 0; - cases++; - expected_rob = cases % (1 << $bits(io_rob_id)); - io_rob_id = expected_rob; - io_kernel_image_ready = 0; - io_kernel_result_ready = 0; - io_request_ready = 0; - io_response_valid = 0; - io_response_bits_fault_error = 0; - io_response_bits_fault_address = 0; - io_kernel_load_bits_address = address; - io_kernel_load_bits_bytes = bytes; - io_kernel_load_valid = 1; - do @(sample); while (!sample.io_kernel_load_ready); - @(negedge clock); - io_kernel_load_valid = 0; - io_rob_id = ~expected_rob; - endtask - task automatic accept_request(); - wait (io_request_valid); - repeat (4) begin - @(sample); - verify_contract( - sample.io_kernel_busy && !sample.io_kernel_load_ready && !sample.io_kernel_result_valid, - "request stalled ownership wrong"); - end - @(negedge clock); - io_request_ready = 1; - do @(sample); while (!sample.io_request_valid); - @(negedge clock); - io_request_ready = 0; - endtask - task automatic response(int first, bit last, int error = 0, longint unsigned address = 0); - @(negedge clock); - io_response_bits_data = beat_value(first); - io_response_bits_last = last; - io_response_bits_addrcounter = first / 4; - io_response_bits_fault_error = error; - io_response_bits_fault_address = address; - io_response_valid = 1; - do @(sample); while (!sample.io_response_ready); - @(negedge clock); - io_response_valid = 0; - endtask - task automatic consume_words(int count); - @(negedge clock); - io_kernel_image_ready = 1; - wait (images == count); - @(negedge clock); - io_kernel_image_ready = 0; - endtask - task automatic no_terminal(int cycles); - repeat (cycles) begin - @(sample); - verify_contract( - sample.io_kernel_busy && !sample.io_kernel_result_valid && - !sample.io_kernel_load_ready && results==0, - "early terminal before accepted read drained"); - end - endtask - task automatic retire_result(int required_requests); - wait (io_kernel_result_valid); - repeat (8) @(sample); - @(negedge clock); - io_kernel_abort = 1; // Held terminal must be immutable, including success. - repeat (4) @(sample); - @(negedge clock); - io_kernel_abort = 0; - io_kernel_result_ready = 1; - do @(sample); while (!sample.io_kernel_result_valid); - @(negedge clock); - io_kernel_result_ready = 0; - repeat (4) @(sample); - @(negedge clock); - verify_contract( - results==1 && requests==required_requests && !io_kernel_busy && io_kernel_load_ready && - !io_kernel_result_valid && !io_kernel_image_valid && !io_footprint_valid, - "duplicate result or ownership/footprint not released"); - endtask - task automatic recovery(); - configure('h9000 + cases * 64, 4, 1); - accept_request(); - response(0, 1); - consume_words(1); - retire_result(1); - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - initial begin - `include "kernel_dma_init.svh" - wait (ctl.start); - repeat (4) @(sample); - @(negedge clock); - reset = 0; - fork - observe(); - join_none - // A partial final beat emits exactly five words; success requires the final image handshake. - configure('h1000, 20, 5); - accept_request(); - response(0, 0); - no_terminal(8); - consume_words(4); - response(4, 1); - no_terminal(8); - consume_words(5); - retire_result(1); - // Original unaligned VA is preserved; preflight later aligns [1,17) to [0,32). - configure(1, 4, 1); - accept_request(); - response(0, 1); - no_terminal(8); - consume_words(1); - retire_result(1); - // First and middle faults preserve all 64 address bits and drop later data while draining. - configure('h2000, 48, 0, 3, 64'h123456789abcde00); - accept_request(); - response(0, 0, 3, 64'h123456789abcde00); - no_terminal(8); - response(4, 0); - no_terminal(8); - response(8, 1, 4, 'h2222); - retire_result(1); - recovery(); - configure('h3000, 48, 4, 4, 64'hfedcba9876543210); - accept_request(); - response(0, 0); - consume_words(4); - response(4, 0, 4, 64'hfedcba9876543210); - no_terminal(8); - response(8, 1); - retire_result(1); - recovery(); - // Abort an offered, unaccepted DMA request: cancellation, no request handshake. - configure('h4000, 16, 0, 10, 'h4000); - wait (io_request_valid); - @(negedge clock); - io_kernel_abort = 1; - repeat (4) @(sample); - @(negedge clock); - io_kernel_abort = 0; - retire_result(0); - recovery(); - // Abort an accepted read: keep ownership until its terminal response is actually consumed. - configure('h5000, 32, 0, 10, 'h5000); - accept_request(); - @(negedge clock); - io_kernel_abort = 1; - no_terminal(8); - @(negedge clock); - io_kernel_abort = 0; - response(0, 0); - no_terminal(8); - response(4, 1); - retire_result(1); - recovery(); - // Abort a buffered non-final output after one word: flush the remaining words and drain. - configure('h6000, 32, 1, 10, 'h6000); - accept_request(); - response(0, 0); - consume_words(1); - @(negedge clock); - io_kernel_abort = 1; - no_terminal(4); - @(negedge clock); - io_kernel_abort = 0; - response(4, 1); - retire_result(1); - recovery(); - // Abort a fully buffered final beat with zero output acceptance. - configure('h6100, 16, 0, 10, 'h6100); - accept_request(); - response(0, 1); - @(negedge clock); - io_kernel_abort = 1; - repeat (4) @(sample); - @(negedge clock); - io_kernel_abort = 0; - retire_result(1); - recovery(); - // An abort asserted before load acceptance is not a transaction and produces no result. - @(negedge clock); - io_kernel_abort = 1; - io_kernel_load_valid = 1; - repeat (8) begin - @(sample); - verify_contract( - !sample.io_kernel_load_ready && !sample.io_kernel_busy && - !sample.io_kernel_result_valid && !sample.io_request_valid, - "unaccepted cancelled load generated work"); - end - @(negedge clock); - io_kernel_load_valid = 0; - io_kernel_abort = 0; - recovery(); - // Shape errors never dispatch a DMA request. - configure('h7000, 0, 0, 7, 'h7000); - retire_result(0); - recovery(); - configure('h7100, 6, 0, 7, 'h7100); - retire_result(0); - recovery(); - configure(64'hfffffffffffffffc, 8, 0, 7, 64'hfffffffffffffffc); - retire_result(0); - recovery(); - // Useful [VA,VA+4) fits, but the mandatory sixteen-byte transport span wraps. - configure(64'hfffffffffffffffc, 4, 0, 7, 64'hfffffffffffffffc); - retire_result(0); - recovery(); - // Early last and missing last are protocol failures, never fabricated success. - configure('h8000, 32, 0, 6, 'h8000); - accept_request(); - response(0, 1); - retire_result(1); - recovery(); - configure('h8100, 16, 0, 6, 'h8100); - accept_request(); - response(0, 0); - no_terminal(8); - response(4, 1); - retire_result(1); - recovery(); - `uvm_info("KERNEL_DMA_GATE", $sformatf( - "%0d cases, %0d status/word checks; fault/cancel/drain/held terminal and recovery", - cases, - checked - ), UVM_LOW) - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/loader_ack.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/loader_ack.f deleted file mode 100644 index 7fadeefec..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/loader_ack.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv -@RESOURCES@/ack_package.sv --F @RTL@/MemLoader/filelist.f -@RESOURCES@/loader_ack_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/loader_ack_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/loader_ack_tb.sv deleted file mode 100644 index c6c99f81c..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/loader_ack_tb.sv +++ /dev/null @@ -1,412 +0,0 @@ -module loader_ack_tb; - import uvm_pkg::*; - import ack_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 0; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "loader_ack_signals.svh" -stream_if #( - .WIDTH($bits( - io_cmdResp_bits_rob_id - ) + $bits( - io_cmdResp_bits_is_sub - ) + $bits( - io_cmdResp_bits_sub_rob_id - ) + $bits( - io_cmdResp_bits_fault_error - ) + $bits( - io_cmdResp_bits_fault_address - )) - ) sink_if ( - clock, - reset - ); - assign sink_if.valid = io_cmdResp_valid; - assign sink_if.ready = io_cmdResp_ready; - assign sink_if.bits = { - io_cmdResp_bits_rob_id, - io_cmdResp_bits_is_sub, - io_cmdResp_bits_sub_rob_id, - io_cmdResp_bits_fault_error, - io_cmdResp_bits_fault_address - }; - MemLoader dut ( - `include "loader_ack_ports.svh" - ); - stream_if #( - .WIDTH($bits( - io_cmdReq_bits_cmd_special - ) + $bits( - io_cmdReq_bits_cmd_mem_addr - ) + $bits( - io_cmdReq_bits_cmd_iter - ) + $bits( - io_cmdReq_bits_cmd_bank_id - )) - ) source_if ( - clock, - reset - ); - assign source_if.valid = io_cmdReq_valid; - assign source_if.ready = io_cmdReq_ready; - assign source_if.bits = { - io_cmdReq_bits_cmd_special, - io_cmdReq_bits_cmd_mem_addr, - io_cmdReq_bits_cmd_iter, - io_cmdReq_bits_cmd_bank_id - }; - stream_if #( - .WIDTH($bits( - io_footprint_rob_id - ) + $bits( - io_footprint_is_sub - ) + $bits( - io_footprint_sub_rob_id - ) + $bits( - io_footprint_baseVA - ) + $bits( - io_footprint_rows - ) + $bits( - io_footprint_columns - ) + $bits( - io_footprint_spanBytes - ) + $bits( - io_footprint_columnStride - ) + $bits( - io_footprint_rowStride - ) + $bits( - io_footprint_write - ) + $bits( - io_footprint_fault_error - ) + $bits( - io_footprint_fault_address - )) - ) footprint_if ( - clock, - reset - ); - assign footprint_if.valid = io_footprint_valid; - assign footprint_if.ready = io_cmdResp_valid && io_cmdResp_ready; - assign footprint_if.bits = { - io_footprint_rob_id, - io_footprint_is_sub, - io_footprint_sub_rob_id, - io_footprint_baseVA, - io_footprint_rows, - io_footprint_columns, - io_footprint_spanBytes, - io_footprint_columnStride, - io_footprint_rowStride, - io_footprint_write, - io_footprint_fault_error, - io_footprint_fault_address - }; - chandle model; - int completions = 0; - logic [127:0] expected, observed; - longint unsigned observed_addr; - int unsigned observed_mask; - function automatic logic [127:0] row_data(int row); - logic [127:0] data; - for (int j = 0; j < 16; j++) data[j*8+:8] = (row * 37 + j * 11 + 3) & 255; - return data; - endfunction - task automatic check(bit condition, string message); - if (!condition) `uvm_fatal("ACK_CONTRACT", message) - endtask - task automatic no_completion(int cycles); - repeat (cycles) begin - @(posedge clock); - #1; - check(!io_cmdResp_valid, "cmdResp.valid asserted before final ACK"); - check(!io_cmdReq_ready, "accepted next command before completion"); - check(completions == 0, "unexpected completion count before ACK"); - end - endtask - task automatic complete_command(int error = 0, longint unsigned address = 0); - wait (io_cmdResp_valid); - repeat (8) begin - @(posedge clock); - #1; - check( - io_cmdResp_valid && io_cmdResp_bits_rob_id==3 && io_cmdResp_bits_is_sub==1 && io_cmdResp_bits_sub_rob_id==2, - "completion changed under backpressure"); - check(io_cmdResp_bits_fault_error == error && io_cmdResp_bits_fault_address == address, - "completion fault changed under backpressure"); - check(!io_cmdReq_ready, "command accepted while completion stalled"); - end - @(negedge clock); - io_cmdResp_ready = 1; - @(posedge clock); - check(io_cmdResp_valid, "completion lost before handshake"); - completions++; - @(negedge clock); - io_cmdResp_ready = 0; - repeat (8) begin - @(posedge clock); - #1; - check(!io_cmdResp_valid, "duplicate completion"); - end - check(completions == 1 && io_cmdReq_ready, "completion count or idle readiness wrong"); - endtask - task automatic begin_load(longint unsigned address, int rows, int groups = 1, int stride = 1); - completions = 0; - @(negedge clock); - io_cmdReq_bits_cmd_mem_addr = address; - io_cmdReq_bits_cmd_iter = rows; - io_query_group_count = groups; - io_cmdReq_bits_cmd_special = 64'(stride) << 39; - io_dmaResp_bits_fault_error = 0; - io_dmaResp_bits_fault_address = 0; - io_cmdReq_valid = 1; - do @(posedge clock); while (!io_cmdReq_ready); - @(negedge clock); - io_cmdReq_valid = 0; - wait (io_dmaReq_valid); - check( - io_footprint_valid && io_footprint_baseVA==address && io_footprint_rows==rows && io_footprint_columns==1 && io_footprint_spanBytes==groups*16 && io_footprint_columnStride==0 && io_footprint_rowStride==groups*stride*16 && io_footprint_fault_error==0 && io_footprint_write==0, - "production footprint differs from actual command"); - check(io_dmaReq_bits_vaddr == address && io_dmaReq_bits_len == rows * groups * 16, - "followup load DMA shape wrong"); - @(negedge clock); - io_dmaReq_ready = 1; - @(posedge clock); - check(io_dmaReq_valid, "followup DMA request missing"); - @(negedge clock); - io_dmaReq_ready = 0; - endtask - task automatic read_beat(int row, bit last, int error = 0, longint unsigned address = 0); - @(negedge clock); - io_dmaResp_bits_data = row_data(row); - io_dmaResp_bits_last = last; - io_dmaResp_bits_addrcounter = row; - io_dmaResp_bits_fault_error = error; - io_dmaResp_bits_fault_address = address; - io_dmaResp_valid = 1; - do @(posedge clock); while (!io_dmaResp_ready); - @(negedge clock); - io_dmaResp_valid = 0; - endtask - task automatic bank_ack(int row, bit ok); - wait (io_bankWrite_io_req_valid); - check(io_bankWrite_io_req_bits_addr == row && io_bankWrite_io_req_bits_data === row_data(row), - "followup bank data/address wrong"); - @(negedge clock); - io_bankWrite_io_req_ready = 1; - @(posedge clock); - check(io_bankWrite_io_req_valid, "followup bank request missing"); - @(negedge clock); - io_bankWrite_io_req_ready = 0; - no_completion(16); - @(negedge clock); - io_bankWrite_io_resp_bits_ok = ok; - io_bankWrite_io_resp_valid = 1; - do @(posedge clock); while (!io_bankWrite_io_resp_ready); - @(negedge clock); - io_bankWrite_io_resp_valid = 0; - endtask - task automatic no_bank_write(int cycles); - repeat (cycles) begin - @(posedge clock); - #1; - check(!io_bankWrite_io_req_valid, "bank writes continued after first fault"); - end - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - initial begin - `include "loader_ack_init.svh" - wait (ctl.start); - model = ack_ref_create(); - @(negedge clock); - reset = 1; - repeat (4) @(posedge clock); - @(negedge clock); - reset = 0; - io_query_group_count = 1; - io_cmdReq_bits_cmd_mem_addr = 64'h1000; - io_cmdReq_bits_cmd_iter = 4; - io_cmdReq_bits_cmd_bank_id = 1; - io_cmdReq_bits_cmd_special = 64'd1 << 39; - io_cmdReq_bits_rob_id = 3; - io_cmdReq_bits_is_sub = 1; - io_cmdReq_bits_sub_rob_id = 2; - io_cmdReq_bits_cmd_is_load = 1; - io_cmdReq_valid = 1; - do @(posedge clock); while (!io_cmdReq_ready); - @(negedge clock); - io_cmdReq_valid = 0; - wait (io_dmaReq_valid); - check( - io_footprint_valid && io_footprint_baseVA=='h1000 && io_footprint_rows==4 && io_footprint_columns==1 && io_footprint_spanBytes==16 && io_footprint_rowStride==16 && !io_footprint_write && io_footprint_fault_error==0, - "initial loader footprint wrong"); - check( - io_dmaReq_bits_vaddr==64'h1000 && io_dmaReq_bits_len==64 && io_dmaReq_bits_stride==1 && io_dmaReq_bits_groups==1, - "load DMA request wrong"); - repeat (3) begin - @(posedge clock); - #1; - check(io_dmaReq_valid && io_dmaReq_bits_vaddr == 64'h1000, - "DMA request changed while stalled"); - end - @(negedge clock); - io_dmaReq_ready = 1; - @(posedge clock); - check(io_dmaReq_valid, "missing DMA handshake"); - @(negedge clock); - io_dmaReq_ready = 0; - for (int row = 0; row < 4; row++) begin - expected = row_data(row); - io_dmaResp_bits_data = expected; - io_dmaResp_bits_last = (row == 3); - io_dmaResp_bits_addrcounter = row; - io_dmaResp_valid = 1; - do @(posedge clock); while (!io_dmaResp_ready); - @(negedge clock); - io_dmaResp_valid = 0; - wait (io_bankWrite_io_req_valid); - check( - io_bankWrite_io_req_bits_addr==row && io_bankWrite_io_req_bits_data===expected && (&{io_bankWrite_io_req_bits_mask_0, io_bankWrite_io_req_bits_mask_1, io_bankWrite_io_req_bits_mask_2, io_bankWrite_io_req_bits_mask_3, io_bankWrite_io_req_bits_mask_4, io_bankWrite_io_req_bits_mask_5, io_bankWrite_io_req_bits_mask_6, io_bankWrite_io_req_bits_mask_7, io_bankWrite_io_req_bits_mask_8, io_bankWrite_io_req_bits_mask_9, io_bankWrite_io_req_bits_mask_10, io_bankWrite_io_req_bits_mask_11, io_bankWrite_io_req_bits_mask_12, io_bankWrite_io_req_bits_mask_13, io_bankWrite_io_req_bits_mask_14, io_bankWrite_io_req_bits_mask_15}) && io_bankWrite_bank_id==1 && io_bankWrite_rob_id==3 && io_bankWrite_group_id==0, - "bank write address/data/mask/tag wrong"); - repeat (2) begin - @(posedge clock); - #1; - check(io_bankWrite_io_req_valid && io_bankWrite_io_req_bits_data === expected, - "bank write changed under backpressure"); - end - @(negedge clock); - io_bankWrite_io_req_ready = 1; - @(posedge clock); - check(io_bankWrite_io_req_valid, "missing bank write handshake"); - ack_ref_write(model, io_bankWrite_io_req_bits_addr * 16, io_bankWrite_io_req_bits_data[63:0], - io_bankWrite_io_req_bits_data[127:64], 65535); - @(negedge clock); - io_bankWrite_io_req_ready = 0; - no_completion(row == 3 ? 32 : 2); - if (row == 3) - `uvm_info("FINAL_ACK_DELAY", - "bankWrite request accepted; cmdResp.valid remained zero for 32 clocks without bankWrite ACK", - UVM_LOW) - @(negedge clock); - io_bankWrite_io_resp_bits_ok = 1; - io_bankWrite_io_resp_valid = 1; - do @(posedge clock); while (!io_bankWrite_io_resp_ready); - @(negedge clock); - io_bankWrite_io_resp_valid = 0; - end - complete_command(); - for (int row = 0; row < 4; row++) begin - expected = row_data(row); - check(ack_ref_check(model, row * 16, expected[63:0], expected[127:64]) == 1, - "bank golden read after completion failed"); - end - // A failed bank ACK preserves the already-written prefix and drains all DMA beats. - begin_load('h2000, 4); - read_beat(0, 0); - bank_ack(0, 1); - read_beat(1, 0); - bank_ack(1, 0); - read_beat(2, 0, 4, 'hdead); - no_bank_write(8); - no_completion(16); - read_beat(3, 1); - no_bank_write(4); - complete_command(5, 16); - begin_load('h2100, 1); - read_beat(0, 1); - bank_ack(0, 1); - complete_command(); - // Failed DMA data must never reach a bank; subsequent errors cannot replace the first. - begin_load('h3000, 3); - read_beat(0, 0, 3, 'h3030); - no_bank_write(8); - read_beat(1, 0); - no_bank_write(8); - no_completion(16); - read_beat(2, 1, 2, 'h3040); - no_bank_write(4); - complete_command(3, 'h3030); - begin_load('h3100, 1); - read_beat(0, 1); - bank_ack(0, 1); - complete_command(); - begin_load('h8000, 2, 2, 3); - read_beat(0, 1, 3, 'h8000); - complete_command(3, 'h8000); - // 2D footprint retains original rows instead of flattened DMA beat count. - completions = 0; - @(negedge clock); - io_cmdReq_bits_cmd_mem_addr = 'ha000; - io_cmdReq_bits_cmd_iter = 2; - io_cmdReq_bits_cmd_is_mvin_2d = 1; - io_cmdReq_bits_cmd_special = (64'd1 << 59) | (64'd1 << 36) | (64'd4 << 43); - io_cmdReq_valid = 1; - do @(posedge clock); while (!io_cmdReq_ready); - @(negedge clock); - io_cmdReq_valid = 0; - wait (io_dmaReq_valid); - check( - io_footprint_valid && io_footprint_rows==2 && io_footprint_columns==2 && - io_footprint_spanBytes==16 && io_footprint_columnStride==8 && io_footprint_rowStride==32 && - io_footprint_fault_error==0 && io_dmaReq_bits_len==64, - "2D padded footprint wrong"); - @(negedge clock); - io_dmaReq_ready = 1; - @(posedge clock); - @(negedge clock); - io_dmaReq_ready = 0; - for (int row = 0; row < 4; row++) begin - read_beat(row, row == 3); - bank_ack(row, 1); - end - complete_command(); - @(negedge clock); - io_cmdReq_bits_cmd_is_mvin_2d = 0; - begin_load('ha100, 1); - read_beat(0, 1); - bank_ack(0, 1); - complete_command(); - // MMIO explicitly uses one group despite an unrelated query value of two. - completions = 0; - @(negedge clock); - io_query_group_count = 2; - io_cmdReq_bits_cmd_mem_addr = 'hb000; - io_cmdReq_bits_cmd_iter = 2; - io_cmdReq_bits_cmd_is_mvin_mmio = 1; - io_cmdReq_bits_cmd_special = (64'd16 << 56) | (64'h100 << 39); - io_cmdReq_valid = 1; - do @(posedge clock); while (!io_cmdReq_ready); - @(negedge clock); - io_cmdReq_valid = 0; - wait (io_dmaReq_valid); - check( - io_footprint_valid && io_footprint_rows==2 && io_footprint_columns==1 && - io_footprint_spanBytes==16 && io_footprint_columnStride==0 && io_footprint_rowStride==16 && - io_footprint_fault_error==0 && io_dmaReq_bits_groups==1, - "MMIO footprint must use one group"); - @(negedge clock); - io_dmaReq_ready = 1; - @(posedge clock); - @(negedge clock); - io_dmaReq_ready = 0; - read_beat(0, 1, 3, 'hb000); - complete_command(3, 'hb000); - @(negedge clock); - io_cmdReq_bits_cmd_is_mvin_mmio = 0; - begin_load('hb100, 1); - read_beat(0, 1); - bank_ack(0, 1); - complete_command(); - ack_ref_destroy(model); - `uvm_info( - "ACK_GATE", - "Late ACK, typed DMA/bank fault, drain, stable completion and successful followup checked", - UVM_LOW) - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.f deleted file mode 100644 index 7d45d7cc2..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/Memory/filelist.f -@RESOURCES@/memory_system_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.toml b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.toml deleted file mode 100644 index eb0e075b4..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system.toml +++ /dev/null @@ -1 +0,0 @@ -chips = ["goban"] diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system_tb.sv deleted file mode 100644 index 004333104..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/memory_system_tb.sv +++ /dev/null @@ -1,355 +0,0 @@ -module memory_system_tb; - import uvm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "memory_system_signals.svh" -Memory dut ( - `include "memory_system_ports.svh" - ); - bit line_valid[2], line_write[2], line_ready_out[2]; - bit [ 11:0] line_id [2]; - bit [ 43:0] line_addr[2]; - bit [ 63:0] line_mask[2]; - bit [511:0] line_data[2]; - logic line_ready[2], reply_valid[2], reply_error[2]; - logic [ 11:0] reply_id [2]; - logic [511:0] reply_data[2]; - assign io_lineRequest_0_valid = line_valid[0]; - assign io_lineRequest_0_bits_write = line_write[0]; - assign io_lineRequest_0_bits_id = line_id[0]; - assign io_lineRequest_0_bits_addr = line_addr[0]; - assign io_lineRequest_0_bits_data = line_data[0]; - assign io_lineRequest_0_bits_mask = line_mask[0]; - assign line_ready[0] = io_lineRequest_0_ready; - assign io_lineResponse_0_ready = line_ready_out[0]; - assign reply_valid[0] = io_lineResponse_0_valid; - assign reply_id[0] = io_lineResponse_0_bits_id; - assign reply_data[0] = io_lineResponse_0_bits_data; - assign reply_error[0] = io_lineResponse_0_bits_error; - assign io_lineRequest_1_valid = line_valid[1]; - assign io_lineRequest_1_bits_write = line_write[1]; - assign io_lineRequest_1_bits_id = line_id[1]; - assign io_lineRequest_1_bits_addr = line_addr[1]; - assign io_lineRequest_1_bits_data = line_data[1]; - assign io_lineRequest_1_bits_mask = line_mask[1]; - assign line_ready[1] = io_lineRequest_1_ready; - assign io_lineResponse_1_ready = line_ready_out[1]; - assign reply_valid[1] = io_lineResponse_1_valid; - assign reply_id[1] = io_lineResponse_1_bits_id; - assign reply_data[1] = io_lineResponse_1_bits_data; - assign reply_error[1] = io_lineResponse_1_bits_error; - `include "memory_system_clocking.svh" - typedef struct { - bit [63:0] addr; - int beat; - } read_t; - typedef struct { - bit [63:0] addr; - bit error; - } write_t; - read_t reads[int]; - write_t writes[int]; - bit [511:0] memory[bit [63:0]]; - int aw_ids[$]; - bit [63:0] aw_addr[$]; - bit [511:0] w_data[$]; - bit [63:0] w_mask[$]; - bit [511:0] assembly; - bit [63:0] assembly_mask; - int beat = 0, cycles = 0, read_bursts = 0, write_bursts = 0, checks = 0; - int dma_checks = 0; - bit hold_b = 0, read_error = 0, write_error = 0; - int r_id = 0, b_id = 0; - bit r_active = 0, b_active = 0; - function automatic void ck(bit ok, string message); - if (!ok) `uvm_fatal("MEMORY_SYSTEM", message) - endfunction - task automatic axi_service(); - forever begin - @(sample); - if (!sample.reset) begin - cycles++; - if (sample.io_axi_aw_valid && sample.io_axi_aw_ready) begin - ck( - sample.io_axi_aw_bits_len==3&&sample.io_axi_aw_bits_size==4&&sample.io_axi_aw_bits_burst==1&&!sample.io_axi_aw_bits_lock, - "AXI AW contract"); - aw_ids.push_back(sample.io_axi_aw_bits_id); - aw_addr.push_back(sample.io_axi_aw_bits_addr); - end - if (sample.io_axi_w_valid && sample.io_axi_w_ready) begin - ck(sample.io_axi_w_bits_last == (beat == 3), "AXI WLAST"); - assembly[128*beat+:128] = sample.io_axi_w_bits_data; - assembly_mask[16*beat+:16] = sample.io_axi_w_bits_strb; - if (beat == 3) begin - w_data.push_back(assembly); - w_mask.push_back(assembly_mask); - beat = 0; - end else beat++; - end - if (aw_ids.size() && w_data.size()) begin - int id = aw_ids.pop_front(); - bit [63:0] addr = aw_addr.pop_front(); - bit [511:0] data = w_data.pop_front(); - bit [63:0] mask = w_mask.pop_front(); - ck(!writes.exists(id), "AXI active write ID reused"); - if (!write_error) - for (int j = 0; j < 64; j++) if (mask[j]) memory[addr][j*8+:8] = data[j*8+:8]; - writes[id] = '{addr, write_error}; - write_bursts++; - end - if (sample.io_axi_ar_valid && sample.io_axi_ar_ready) begin - int id = sample.io_axi_ar_bits_id; - bit [63:0] addr = sample.io_axi_ar_bits_addr; - ck( - sample.io_axi_ar_bits_len==3&&sample.io_axi_ar_bits_size==4&&sample.io_axi_ar_bits_burst==1&&!sample.io_axi_ar_bits_lock, - "AXI AR contract"); - ck(!reads.exists(id), "AXI active read ID reused"); - ck(memory.exists(addr), "Read of uninitialized external memory"); - reads[id] = '{addr, 0}; - read_bursts++; - end - if (sample.io_axi_r_valid && sample.io_axi_r_ready) begin - if (reads[r_id].beat == 3) reads.delete(r_id); - else reads[r_id].beat++; - r_active = 0; - end - if (sample.io_axi_b_valid && sample.io_axi_b_ready) begin - writes.delete(b_id); - b_active = 0; - end - end - @(negedge clock); - io_axi_aw_ready = !reset && cycles % 5 != 0; - io_axi_w_ready = !reset && cycles % 7 >= 2; - io_axi_ar_ready = !reset && cycles % 4 != 0; - if (!r_active) - foreach (reads[id]) - if (!r_active) begin - r_active = 1; - r_id = id; - io_axi_r_bits_id = id; - io_axi_r_bits_data = memory[reads[id].addr][128*reads[id].beat+:128]; - io_axi_r_bits_last = reads[id].beat == 3; - io_axi_r_bits_resp = read_error && reads[id].beat == 1 ? 2 : 0; - end - if (!b_active && !hold_b) - foreach (writes[id]) - if (!b_active) begin - b_active = 1; - b_id = id; - io_axi_b_bits_id = id; - io_axi_b_bits_resp = writes[id].error ? 2 : 0; - end - io_axi_r_valid = !reset && r_active; - io_axi_b_valid = !reset && b_active; - end - endtask - task automatic issue_line(int agent, bit write, bit [63:0] addr, bit [63:0] data, int id); - @(negedge clock); - line_valid[agent] = 1; - line_write[agent] = write; - line_addr[agent] = addr; - line_data[agent] = '0; - line_data[agent][63:0] = data; - line_mask[agent] = 'hff; - line_id[agent] = id; - do @(sample); while (!sample.line_ready[agent]); - @(negedge clock); - line_valid[agent] = 0; - endtask - task automatic receive_line(int agent, int id, bit [511:0] data, bit write = 0); - do @(sample); while (!sample.reply_valid[agent]); - repeat (4) begin - ck(sample.reply_valid[agent] && sample.reply_id[agent] == id && !sample.reply_error[agent], - "Line owner/error or backpressure stability"); - if (!write) ck(sample.reply_data[agent] == data, "DDR line readback data"); - @(sample); - end - @(negedge clock); - line_ready_out[agent] = 1; - do @(sample); while (!sample.reply_valid[agent]); - checks++; - @(negedge clock); - line_ready_out[agent] = 0; - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 1ms); - run_test("protocol_test"); - end - initial begin - io_dma_0_aw_valid = 0; - io_dma_0_aw_bits_id = 6; - io_dma_0_aw_bits_addr = 'h80000080; - io_dma_0_aw_bits_len = 3; - io_dma_0_aw_bits_size = 4; - io_dma_0_aw_bits_burst = 1; - io_dma_0_aw_bits_lock = 0; - io_dma_0_aw_bits_cache = 0; - io_dma_0_aw_bits_prot = 0; - io_dma_0_aw_bits_qos = 0; - io_dma_0_aw_bits_region = 0; - io_dma_0_w_valid = 0; - io_dma_0_w_bits_data = 0; - io_dma_0_w_bits_strb = '1; - io_dma_0_w_bits_last = 0; - io_dma_0_b_ready = 0; - io_dma_0_ar_valid = 0; - io_dma_0_ar_bits_id = 6; - io_dma_0_ar_bits_addr = 'h80000080; - io_dma_0_ar_bits_len = 3; - io_dma_0_ar_bits_size = 4; - io_dma_0_ar_bits_burst = 1; - io_dma_0_ar_bits_lock = 0; - io_dma_0_ar_bits_cache = 0; - io_dma_0_ar_bits_prot = 0; - io_dma_0_ar_bits_qos = 0; - io_dma_0_ar_bits_region = 0; - io_dma_0_r_ready = 0; - for (int i = 0; i < 2; i++) begin - line_valid[i] = 0; - line_write[i] = 0; - line_addr[i] = 0; - line_data[i] = 0; - line_mask[i] = 0; - line_id[i] = 0; - line_ready_out[i] = 0; - end - io_axi_aw_ready = 0; - io_axi_ar_ready = 0; - io_axi_w_ready = 0; - io_axi_r_valid = 0; - io_axi_b_valid = 0; - io_axi_r_bits_id = 0; - io_axi_r_bits_data = 0; - io_axi_r_bits_last = 0; - io_axi_r_bits_resp = 0; - io_axi_b_bits_id = 0; - io_axi_b_bits_resp = 0; - memory['h80000000] = '0; - memory['h80000040] = '0; - wait (ctl.start); - repeat (5) @(negedge clock); - reset = 0; - fork - axi_service(); - join_none - // A retained NPU write response must not block independent coherence backing reads. - hold_b = 1; - @(negedge clock); - io_dma_0_aw_valid = 1; - do @(sample); while (!sample.io_dma_0_aw_ready); - @(negedge clock); - io_dma_0_aw_valid = 0; - for (int i = 0; i < 4; i++) begin - io_dma_0_w_valid = 1; - io_dma_0_w_bits_data = 128'('h100 + i); - io_dma_0_w_bits_last = i == 3; - do @(sample); while (!sample.io_dma_0_w_ready); - @(negedge clock); - end - io_dma_0_w_valid = 0; - wait (writes.num() != 0); - issue_line(0, 0, 'h80000000, 0, 19); - receive_line(0, 19, 0); - ck(io_outstanding == 1, "NPU write completed before its final AXI B"); - @(negedge clock); - hold_b = 0; - io_dma_0_b_ready = 1; - do @(sample); while (!sample.io_dma_0_b_valid); - ck(sample.io_dma_0_b_bits_id == 6 && sample.io_dma_0_b_bits_resp == 0, - "NPU write response owner or error"); - @(negedge clock); - io_dma_0_b_ready = 0; - io_dma_0_ar_valid = 1; - do @(sample); while (!sample.io_dma_0_ar_ready); - @(negedge clock); - io_dma_0_ar_valid = 0; - io_dma_0_r_ready = 1; - for (int i = 0; i < 4; i++) begin - do @(sample); while (!sample.io_dma_0_r_valid); - ck( - sample.io_dma_0_r_bits_id == 6 && sample.io_dma_0_r_bits_resp == 0 && - sample.io_dma_0_r_bits_data == 128'('h100 + i) && sample.io_dma_0_r_bits_last == (i == 3), - "NPU readback data, owner, error or RLAST"); - dma_checks++; - end - @(negedge clock); - io_dma_0_r_ready = 0; - fork - begin - issue_line(0, 1, 'h80000000, 23, 20); - receive_line(0, 20, 0, 1); - end - begin - issue_line(1, 1, 'h80000040, 31, 20); - receive_line(1, 20, 0, 1); - end - join - fork - begin - issue_line(0, 0, 'h80000000, 0, 21); - receive_line(0, 21, 23); - end - begin - issue_line(1, 0, 'h80000040, 0, 21); - receive_line(1, 21, 31); - end - join - // Neither client observes write completion before the final AXI B. - hold_b = 1; - issue_line(0, 1, 'h80000000, 41, 22); - wait (writes.num() != 0); - repeat (12) begin - @(sample); - ck(!sample.reply_valid[0] && sample.io_outstanding != 0, - "Backing write completed before final AXI B"); - end - @(negedge clock); - hold_b = 0; - receive_line(0, 22, 0, 1); - // Transport errors return to the originating coherence client. - read_error = 1; - issue_line(1, 0, 'h80000040, 0, 23); - do @(sample); while (!sample.reply_valid[1]); - ck(sample.reply_id[1] == 23 && sample.reply_error[1], "DDR R error owner"); - @(negedge clock); - line_ready_out[1] = 1; - do @(sample); while (!sample.reply_valid[1]); - checks++; - @(negedge clock); - line_ready_out[1] = 0; - read_error = 0; - write_error = 1; - issue_line(0, 1, 'h80000000, 99, 24); - do @(sample); while (!sample.reply_valid[0]); - ck(sample.reply_id[0] == 24 && sample.reply_error[0], "DDR B error owner"); - @(negedge clock); - line_ready_out[0] = 1; - do @(sample); while (!sample.reply_valid[0]); - checks++; - @(negedge clock); - line_ready_out[0] = 0; - write_error = 0; - issue_line(0, 0, 'h80000000, 0, 25); - receive_line(0, 25, 41); - repeat (8) @(sample); - ck( - !reads.num() && !writes.num() && !aw_ids.size() && !aw_addr.size() && - !w_data.size() && !w_mask.size() && !r_active && !b_active && sample.io_outstanding == 0, - "DDR transport did not drain"); - for (int i = 0; i < 2; i++) ck(!sample.reply_valid[i], "Unmatched backing response"); - ck(checks == 9 && dma_checks == 4, "Missing checked backing/DMA transactions"); - `uvm_info("MEMORY_SYSTEM_PASS", $sformatf( - "backing checked=%0d DMA beats=%0d AXI reads/writes=%0d/%0d; owners, backpressure, final B, R/B errors and drain checked", - checks, - dma_checks, - read_bursts, - write_bursts - ), UVM_LOW) - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma.cm_hier b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma.cm_hier deleted file mode 100644 index 772e1cfc5..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma.cm_hier +++ /dev/null @@ -1 +0,0 @@ -+tree @TOP@.dut diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma.f deleted file mode 100644 index 179db17e9..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/ReadDma/filelist.f -@RESOURCES@/read_dma_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma_tb.sv deleted file mode 100644 index 7ffc5b3fa..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/read_dma_tb.sv +++ /dev/null @@ -1,185 +0,0 @@ -module read_dma_tb; - import uvm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "read_dma_signals.svh" -ReadDma dut ( - `include "read_dma_ports.svh" - ); - `include "read_dma_clocking.svh" - int cases = 0, beats = 0, bursts = 0, checks = 0; - function automatic bit [127:0] data_at(longint unsigned address); - bit [127:0] value; - for (int i = 0; i < 16; ++i) value[i*8+:8] = 8'((address + i) ^ ((address + i) >> 8)); - return value; - endfunction - task automatic check(bit good, string reason); - checks++; - if (!good) `uvm_fatal("READ_DMA", reason) - endtask - task automatic run_case(longint unsigned base, int length, int groups, int stride, bit mode2d = 0, - int pixel = 16, int width = 1, int source = 1, int expected_error = 0, - bit bus_error = 0, int cancel_after = 0); - int count, seen, pending, issued, tick, fault_seen; - longint unsigned read_address, expected_address, query_address; - int query_bytes; - bit [127:0] held_data; - bit held, held_last; - bit [ 3:0] held_error; - bit [63:0] held_address; - count = (length + 15) / 16; - seen = 0; - pending = 0; - issued = 0; - fault_seen = 0; - held = 0; - cases++; - @(negedge clock); - io_req_bits_vaddr = base; - io_req_bits_len = length; - io_req_bits_groups = groups; - io_req_bits_stride = stride; - io_req_bits_is_2d = mode2d; - io_req_bits_pixel_bytes = pixel; - io_req_bits_tile_width = width; - io_req_bits_source_width = source; - io_req_valid = 1; - do @(sample); while (!sample.io_req_ready); - @(negedge clock); - io_req_valid = 0; - for (tick = 0; tick < 20000; ++tick) begin - io_mapping_hit = 1; - io_mapping_error = 0; - io_mapping_pa = io_query_va; - io_axi_ar_ready = pending == 0 && tick % 3 != 0; - io_axi_r_valid = pending != 0; - io_axi_r_bits_data = data_at(read_address); - io_axi_r_bits_last = pending == 1; - io_axi_r_bits_resp = bus_error && issued == 1 ? 2 : 0; - io_resp_ready = tick % 5 == 0; - @(sample); - if (held) - check( - sample.io_resp_valid && sample.io_resp_bits_data == held_data && - sample.io_resp_bits_last == held_last && sample.io_resp_bits_fault_error == held_error && - sample.io_resp_bits_fault_address == held_address, - "response changed under backpressure"); - held = sample.io_resp_valid && !sample.io_resp_ready; - held_data = sample.io_resp_bits_data; - held_last = sample.io_resp_bits_last; - held_error = sample.io_resp_bits_fault_error; - held_address = sample.io_resp_bits_fault_address; - if (sample.io_query_valid) begin - check( - !sample.io_query_write && sample.io_query_va[3:0] == 0 && - sample.io_query_bytes != 0 && sample.io_query_bytes % 16 == 0, - "invalid prepared footprint"); - query_address = sample.io_mapping_pa; - query_bytes = sample.io_query_bytes; - end - if (sample.io_axi_ar_valid && sample.io_axi_ar_ready) begin - check( - sample.io_axi_ar_bits_addr == query_address && - (int'(sample.io_axi_ar_bits_len)+1)*16 == query_bytes, - "burst differs from prepared footprint"); - check(sample.io_axi_ar_bits_size == 4 && sample.io_axi_ar_bits_burst == 1, - "AXI request must be 128-bit INCR"); - check( - sample.io_axi_ar_bits_addr[3:0] == 0 && - int'(sample.io_axi_ar_bits_addr[11:0]) + (int'(sample.io_axi_ar_bits_len)+1)*16 <= 4096, - "burst alignment/page boundary"); - pending = int'(sample.io_axi_ar_bits_len) + 1; - read_address = sample.io_axi_ar_bits_addr; - bursts++; - end - if (sample.io_axi_r_valid && sample.io_axi_r_ready) begin - pending--; - issued++; - read_address += 16; - end - if (sample.io_resp_valid && sample.io_resp_ready) begin - if (sample.io_resp_bits_fault_error != 0) begin - check(sample.io_resp_bits_fault_error == expected_error && sample.io_resp_bits_last, - "wrong terminal fault"); - check(pending == 0, "fault completed before accepted burst drained"); - fault_seen++; - end else begin - check(expected_error == 0 || bus_error, "shape failure dispatched data"); - expected_address = mode2d ? base + (seen/width)*source*pixel + (seen%width)*pixel : - base + (seen/groups)*groups*stride*16 + (seen%groups)*16; - check(sample.io_resp_bits_data == data_at(expected_address), - "data/stride/unaligned assembly mismatch"); - check(sample.io_resp_bits_addrcounter == seen, "beat index mismatch"); - if (!bus_error) check(sample.io_resp_bits_last == (seen + 1 == count), "last mismatch"); - seen++; - beats++; - end - if (sample.io_resp_bits_last) begin - check(fault_seen == (expected_error != 0), "missing/unexpected terminal error"); - if (!expected_error) check(seen == count, "incomplete response"); - @(negedge clock); - io_resp_ready = 0; - io_axi_r_valid = 0; - io_axi_ar_ready = 0; - return; - end - end - @(negedge clock); - if (cancel_after != 0 && tick == cancel_after) begin - check(issued == 0, "reset fixture must precede AXI issue"); - reset = 1; - io_axi_r_valid = 0; - io_axi_ar_ready = 0; - repeat (3) @(sample); - @(negedge clock); - reset = 0; - repeat (4) begin - @(sample); - check(!sample.io_resp_valid && !sample.io_axi_ar_valid && !sample.io_busy, - "reset leaked old work"); - end - return; - end - end - `uvm_fatal("READ_DMA", "request timed out") - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 2ms); - run_test("protocol_test"); - end - initial begin - `include "read_dma_init.svh" - wait (ctl.start); - repeat (4) @(sample); - @(negedge clock); - reset = 0; - io_decisionValid = 1; - run_case('h1000, 16, 1, 1); - run_case('h1ff0, 16 * 257, 1, 1); - run_case('h3003, 53, 3, 5); - run_case('h4ff9, 17, 1, 1); - run_case('h6000, 16 * 19, 3, 7); - run_case('h8000, 16 * 64, 63, 524287); - run_case('h8000, 16 * 63, 63, 524287); - run_case('h9000, 16 * 20, 1, 1, 1, 7, 3, 7); - run_case('haff0, 16 * 20, 1, 1, 1, 16, 3, 7); - run_case('hb000, 16 * 17, 1, 1, 1, 1023, 15, 1023); - run_case(64'hfffffffffffffff0, 16, 1, 1, 0, 16, 1, 1, 0); - run_case(64'hfffffffffffffff1, 16, 1, 1, 0, 16, 1, 1, 7); - run_case(64'hfffffff000000000, 16 * 65536, 63, 524287, 0, 16, 1, 1, 7); - run_case(64'hfffffffffff00000, 16 * 65536, 1, 1, 1, 1023, 15, 1023, 7); - run_case('hc000, 0, 1, 1, 0, 16, 1, 1, 7); - run_case('hc000, 16, 0, 1, 0, 16, 1, 1, 7); - run_case('hc000, 16, 1, 1, 1, 0, 3, 3, 7); - run_case('hd000, 64, 1, 1, 0, 16, 1, 1, 2, 1); - run_case('he000, 256, 3, 1, 0, 16, 1, 1, 0, 0, 5); - run_case('hf000, 32, 1, 1); - `uvm_info("READ_DMA", $sformatf( - "%0d cases %0d checks %0d beats %0d bursts", cases, checks, beats, bursts), UVM_LOW) - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease.cm_hier b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease.cm_hier deleted file mode 100644 index 772e1cfc5..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease.cm_hier +++ /dev/null @@ -1 +0,0 @@ -+tree @TOP@.dut diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease.f deleted file mode 100644 index 8e2244b16..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/SharedMemBackend/filelist.f -@RESOURCES@/shared_lease_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease_tb.sv deleted file mode 100644 index 3052eaa84..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/shared_lease_tb.sv +++ /dev/null @@ -1,163 +0,0 @@ -module shared_lease_tb; - import uvm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "shared_lease_signals.svh" -SharedMemBackend dut ( - `include "shared_lease_ports.svh" - ); - `include "shared_lease_clocking.svh" - int checks = 0, invalid_case = 0; - task automatic check(bit good, string reason); - checks++; - if (!good) `uvm_fatal("SHARED_LEASE", reason) - endtask - task automatic configure(bit alloc, int bank, int group_id = 0, bit transfer = 0, int source = 0); - @(negedge clock); - io_config_bits_alloc = alloc; - io_config_bits_vbank_id = bank; - io_config_bits_group_id = group_id; - io_config_bits_transfer = transfer; - io_config_bits_source_bank_id = source; - io_config_valid = 1; - do @(sample); while (!sample.io_config_ready); - @(negedge clock); - io_config_valid = 0; - endtask - task automatic request(bit release_lease, int bank = 8, int group_id = 0); - @(negedge clock); - io_lease_request_bits_release = release_lease; - io_lease_request_bits_vbank = bank; - io_lease_request_bits_group = group_id; - io_lease_request_valid = 1; - do @(sample); while (!sample.io_lease_request_ready); - @(negedge clock); - io_lease_request_valid = 0; - endtask - task automatic reply(longint unsigned expected); - repeat (5) begin - @(sample); - check(sample.io_lease_response_valid && sample.io_lease_response_bits == expected, - "lease response must remain stable under backpressure"); - check(!sample.io_lease_request_ready, "accepted second lease before response"); - end - @(negedge clock); - io_lease_response_ready = 1; - @(sample); - check(sample.io_lease_response_valid, "response lost"); - @(negedge clock); - io_lease_response_ready = 0; - endtask - task automatic data_access(bit write, bit [127:0] value); - @(negedge clock); - io_physical_0_request_valid = 1; - io_physical_0_request_bits_address = 128; - io_physical_0_request_bits_write = write; - io_physical_0_request_bits_data = value; - io_physical_0_request_bits_mask = '1; - do @(sample); while (!sample.io_physical_0_request_ready); - @(negedge clock); - io_physical_0_request_valid = 0; - do @(sample); while (!sample.io_physical_0_response_valid); - check(!sample.io_physical_0_response_bits_error, "leased bank data access failed"); - if (!write) - check(sample.io_physical_0_response_bits_data == value, "leased bank data mismatch"); - @(negedge clock); - io_physical_0_response_ready = 1; - @(sample); - @(negedge clock); - io_physical_0_response_ready = 0; - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - initial begin - `include "shared_lease_init.svh" - void'($value$plusargs("invalid_case=%d", invalid_case)); - wait (ctl.start); - repeat (4) @(sample); - @(negedge clock); - reset = 0; - io_config_bits_hart_id = 64'h100000002; - io_config_bits_is_shared = 1; - io_leaseOwnerHartId = 64'h100000002; - configure(1, 8); - configure(1, 8, 1); - request(0, 8, 1); - reply(128); - if (invalid_case == 1) configure(0, 8); - if (invalid_case == 2) configure(1, 8); - if (invalid_case == 3) configure(0, 9, 0, 1, 8); - if (invalid_case == 4) begin - request(1, 8, 0); - end - if (invalid_case == 5) begin - request(0, 10, 0); - end - if (invalid_case) begin - repeat (8) @(sample); - `uvm_fatal("SHARED_LEASE", "expected RTL assertion absent") - end - request(0, 8, 1); - reply(128); - request(1, 8, 1); - reply(0); - data_access(1, 128'hffeeddccbbaa99887766554433221100); - data_access(0, 128'hffeeddccbbaa99887766554433221100); - request(1, 8, 1); - reply(0); - // Config wins when both are offered; lease then observes the new mapping. - @(negedge clock); - io_config_valid = 1; - io_config_bits_alloc = 1; - io_config_bits_vbank_id = 9; - io_config_bits_group_id = 0; - io_lease_request_valid = 1; - io_lease_request_bits_release = 0; - io_lease_request_bits_vbank = 9; - io_lease_request_bits_group = 0; - @(sample); - check(sample.io_config_ready && !sample.io_lease_request_ready, "config priority deadlock"); - @(negedge clock); - io_config_valid = 0; - @(sample); - check(sample.io_lease_request_ready, "lease not accepted after config"); - @(negedge clock); - io_lease_request_valid = 0; - reply(256); - request(1, 9); - reply(0); - configure(0, 9, 0, 1, 8); - request(0, 9, 2); - reply(128); - request(1, 9, 2); - reply(0); - configure(0, 9); - configure(1, 8); - request(0); // Reset discards an unconsumed response and its lease. - @(negedge clock); - reset = 1; - repeat (3) @(sample); - @(negedge clock); - reset = 0; - repeat (3) begin - @(sample); - check(!sample.io_lease_response_valid, "old response survived reset"); - end - configure(1, 8); - request(0); - reply(0); - request(1); - reply(0); - configure(0, 8); - `uvm_info("SHARED_LEASE", $sformatf( - "%0d checks: leases, data sharing, config priority, backpressure, reset", checks), - UVM_LOW) - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/storer_ack.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/storer_ack.f deleted file mode 100644 index 4f677251e..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/storer_ack.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv -@RESOURCES@/ack_package.sv --F @RTL@/MemStorer/filelist.f -@RESOURCES@/storer_ack_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/storer_ack_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/storer_ack_tb.sv deleted file mode 100644 index cebf09d58..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/storer_ack_tb.sv +++ /dev/null @@ -1,366 +0,0 @@ -module storer_ack_tb; - import uvm_pkg::*; - import ack_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 0; - always #5 clock = ~clock; - ip_control_if ctl (clock); - `include "storer_ack_signals.svh" -stream_if #( - .WIDTH($bits( - io_cmdResp_bits_rob_id - ) + $bits( - io_cmdResp_bits_is_sub - ) + $bits( - io_cmdResp_bits_sub_rob_id - ) + $bits( - io_cmdResp_bits_fault_error - ) + $bits( - io_cmdResp_bits_fault_address - )) - ) sink_if ( - clock, - reset - ); - assign sink_if.valid = io_cmdResp_valid; - assign sink_if.ready = io_cmdResp_ready; - assign sink_if.bits = { - io_cmdResp_bits_rob_id, - io_cmdResp_bits_is_sub, - io_cmdResp_bits_sub_rob_id, - io_cmdResp_bits_fault_error, - io_cmdResp_bits_fault_address - }; - MemStorer dut ( - `include "storer_ack_ports.svh" - ); - stream_if #( - .WIDTH($bits( - io_cmdReq_bits_cmd_special - ) + $bits( - io_cmdReq_bits_cmd_mem_addr - ) + $bits( - io_cmdReq_bits_cmd_iter - ) + $bits( - io_cmdReq_bits_cmd_bank_id - )) - ) source_if ( - clock, - reset - ); - assign source_if.valid = io_cmdReq_valid; - assign source_if.ready = io_cmdReq_ready; - assign source_if.bits = { - io_cmdReq_bits_cmd_special, - io_cmdReq_bits_cmd_mem_addr, - io_cmdReq_bits_cmd_iter, - io_cmdReq_bits_cmd_bank_id - }; - stream_if #( - .WIDTH($bits( - io_footprint_rob_id - ) + $bits( - io_footprint_is_sub - ) + $bits( - io_footprint_sub_rob_id - ) + $bits( - io_footprint_baseVA - ) + $bits( - io_footprint_rows - ) + $bits( - io_footprint_columns - ) + $bits( - io_footprint_spanBytes - ) + $bits( - io_footprint_columnStride - ) + $bits( - io_footprint_rowStride - ) + $bits( - io_footprint_write - ) + $bits( - io_footprint_fault_error - ) + $bits( - io_footprint_fault_address - )) - ) footprint_if ( - clock, - reset - ); - assign footprint_if.valid = io_footprint_valid; - assign footprint_if.ready = io_cmdResp_valid && io_cmdResp_ready; - assign footprint_if.bits = { - io_footprint_rob_id, - io_footprint_is_sub, - io_footprint_sub_rob_id, - io_footprint_baseVA, - io_footprint_rows, - io_footprint_columns, - io_footprint_spanBytes, - io_footprint_columnStride, - io_footprint_rowStride, - io_footprint_write, - io_footprint_fault_error, - io_footprint_fault_address - }; - chandle model; - int completions = 0, store_cases = 0, descriptor_count = 0, data_count = 0; - logic [127:0] expected; - longint unsigned descriptor_address; - int descriptor_bytes; - logic [127:0] descriptor_data[$]; - function automatic logic [127:0] row_data(int row); - logic [127:0] data; - for (int j = 0; j < 16; j++) data[j*8+:8] = (row * 37 + j * 11 + 3) & 255; - return data; - endfunction - task automatic check(bit condition, string message); - if (!condition) `uvm_fatal("ACK_CONTRACT", message) - endtask - task automatic no_completion(int cycles); - repeat (cycles) begin - @(posedge clock); - #1; - check(!io_cmdResp_valid, "cmdResp.valid asserted before final descriptor ACK"); - check(!io_cmdReq_ready, "accepted next command before completion"); - check(completions == 0, "unexpected completion count before descriptor ACK"); - end - endtask - task automatic complete_command(int error = 0, longint unsigned address = 0); - wait (io_cmdResp_valid); - repeat (8) begin - @(posedge clock); - #1; - check( - io_cmdResp_valid && io_cmdResp_bits_rob_id == 3 && io_cmdResp_bits_is_sub == 1 && io_cmdResp_bits_sub_rob_id == 2, - "completion changed under backpressure"); - check(io_cmdResp_bits_fault_error == error && io_cmdResp_bits_fault_address == address, - "completion fault changed under backpressure"); - check(!io_cmdReq_ready, "command accepted while completion stalled"); - end - @(negedge clock); - io_cmdResp_ready = 1; - @(posedge clock); - check(io_cmdResp_valid, "completion lost before handshake"); - completions++; - @(negedge clock); - io_cmdResp_ready = 0; - repeat (8) begin - @(posedge clock); - #1; - check(!io_cmdResp_valid, "duplicate completion"); - end - check(completions == 1 && io_cmdReq_ready, "completion count or idle readiness wrong"); - endtask - task automatic begin_store(longint unsigned address, int rows, int groups = 1, int stride = 1); - completions = 0; - store_cases++; - descriptor_count = 0; - data_count = 0; - @(negedge clock); - io_cmdReq_bits_cmd_mem_addr = address; - io_cmdReq_bits_cmd_iter = rows; - io_query_group_count = groups; - io_cmdReq_bits_cmd_special = 64'(stride) << 39; - io_dmaResp_bits_fault_error = 0; - io_dmaResp_bits_fault_address = 0; - io_cmdReq_valid = 1; - do @(posedge clock); while (!io_cmdReq_ready); - @(negedge clock); - io_cmdReq_valid = 0; - endtask - task automatic accept_descriptor(longint unsigned address, int bytes); - wait (io_dmaReq_valid); - repeat (4) begin - @(posedge clock); - #1; - check(io_dmaReq_valid && io_dmaReq_bits_vaddr == address && io_dmaReq_bits_len == bytes, - "whole-descriptor address/length changed under backpressure"); - check(!io_cmdResp_valid, "descriptor offer released Core completion"); - end - @(negedge clock); - io_dmaReq_ready = 1; - @(posedge clock); - check(io_dmaReq_valid, "missing descriptor handshake"); - descriptor_address = io_dmaReq_bits_vaddr; - descriptor_bytes = io_dmaReq_bits_len; - descriptor_data.delete(); - descriptor_count++; - @(negedge clock); - io_dmaReq_ready = 0; - endtask - task automatic bank_and_data(int row, int group, logic [127:0] data, bit last); - wait (io_bankRead_io_req_valid); - check( - io_bankRead_io_req_bits_addr == row && io_bankRead_group_id == group && - io_bankRead_bank_id == 1 && io_bankRead_rob_id == 3, - "bank row/group/tag wrong"); - repeat (2) begin - @(posedge clock); - #1; - check( - io_bankRead_io_req_valid && io_bankRead_io_req_bits_addr == row && - io_bankRead_group_id == group, - "bank read changed under backpressure"); - end - @(negedge clock); - io_bankRead_io_req_ready = 1; - @(posedge clock); - check(io_bankRead_io_req_valid, "missing bank read handshake"); - @(negedge clock); - io_bankRead_io_req_ready = 0; - no_completion(2); - @(negedge clock); - io_bankRead_io_resp_bits_data = data; - io_bankRead_io_resp_valid = 1; - do @(posedge clock); while (!io_bankRead_io_resp_ready); - @(negedge clock); - io_bankRead_io_resp_valid = 0; - wait (io_dmaData_valid); - repeat (3) begin - @(posedge clock); - #1; - check(io_dmaData_valid && io_dmaData_bits_data === data && io_dmaData_bits_last == last, - "raw 128-bit bank stream changed under backpressure or was shifted for alignment"); - check(!io_dmaReq_valid, "duplicate descriptor emitted during its stream"); - check(!io_cmdResp_valid, "Core completion preceded final descriptor ACK"); - end - @(negedge clock); - io_dmaData_ready = 1; - @(posedge clock); - check(io_dmaData_valid, "missing raw DMA data handshake"); - descriptor_data.push_back(io_dmaData_bits_data); - data_count++; - @(negedge clock); - io_dmaData_ready = 0; - endtask - task automatic feed_descriptor(longint unsigned base, int rows, int groups = 1, int stride = 1, - int first_row = 0, int descriptor_rows = 0); - int count; - count = descriptor_rows == 0 ? rows : descriptor_rows; - accept_descriptor(base + first_row * groups * stride * 16, count * groups * 16); - check( - io_footprint_valid && io_footprint_baseVA == base && io_footprint_rows == rows && - io_footprint_columns == 1 && io_footprint_spanBytes == groups * 16 && - io_footprint_columnStride == 0 && io_footprint_rowStride == groups * stride * 16 && - io_footprint_fault_error == 0 && io_footprint_write, - "footprint differs from the complete store command"); - for (int row = first_row; row < first_row + count; row++) - for (int group = 0; group < groups; group++) - bank_and_data(row, group, row_data(row * groups + group), - row == first_row + count - 1 && group == groups - 1); - check(descriptor_data.size() * 16 == descriptor_bytes, "stream length differs from descriptor"); - endtask - task automatic dma_ack(bit done, int error = 0, longint unsigned address = 0); - no_completion(32); - if (done && error == 0) - foreach (descriptor_data[index]) - ack_ref_write(model, descriptor_address + index * 16, descriptor_data[index][63:0], - descriptor_data[index][127:64], 'hffff); - @(negedge clock); - io_dmaResp_bits_done = done; - io_dmaResp_bits_fault_error = error; - io_dmaResp_bits_fault_address = address; - io_dmaResp_valid = 1; - do @(posedge clock); while (!io_dmaResp_ready); - @(negedge clock); - io_dmaResp_valid = 0; - endtask - task automatic no_future_requests(); - repeat (8) begin - @(posedge clock); - #1; - check(!io_dmaReq_valid && !io_dmaData_valid && !io_bankRead_io_req_valid, - "failed final ACK launched residual DMA data/descriptor or a future bank read"); - end - endtask - task automatic recovery(longint unsigned address); - begin_store(address, 1); - feed_descriptor(address, 1); - dma_ack(1); - complete_command(); - check(descriptor_count == 1 && data_count == 1, - "recovery reused previous row/descriptor state"); - endtask - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - initial begin - logic [127:0] partial; - `include "storer_ack_init.svh" - wait (ctl.start); - model = ack_ref_create(); - @(negedge clock); - reset = 1; - repeat (4) @(posedge clock); - @(negedge clock); - reset = 0; - io_cmdReq_bits_cmd_bank_id = 1; - io_cmdReq_bits_rob_id = 3; - io_cmdReq_bits_is_sub = 1; - io_cmdReq_bits_sub_rob_id = 2; - io_cmdReq_bits_cmd_is_store = 1; - begin_store('h1000, 4); - feed_descriptor('h1000, 4); - check(descriptor_count == 1 && data_count == 4, "stride-one rows were not one long descriptor"); - dma_ack(1); - complete_command(); - for (int row = 0; row < 4; row++) begin - expected = row_data(row); - check(ack_ref_check(model, 'h1000 + row * 16, expected[63:0], expected[127:64]) == 1, - "DDR golden after final completion failed"); - end - // The packing/partial-burst failure belongs to WriteDma, not MemStorer. - begin_store('h4003, 4); - feed_descriptor('h4003, 4); - dma_ack(0, 3, 'h4000); - no_future_requests(); - complete_command(3, 'h4000); - recovery('h4100); - // Preserve already committed bytes when a later DMA segment faults. - begin_store('h5003, 4); - feed_descriptor('h5003, 4); - partial = ~row_data(0); - ack_ref_write(model, 'h5003, partial[63:0], partial[127:64], 'hffff); - expected = row_data(0); - ack_ref_write(model, 'h5003, expected[63:0], expected[127:64], 'h1fff); - partial[103:0] = expected[103:0]; - dma_ack(0, 4, 'h5010); - no_future_requests(); - complete_command(4, 'h5010); - check(ack_ref_check(model, 'h5003, partial[63:0], partial[127:64]) == 1, - "later-segment fault rolled back prior committed bytes"); - recovery('h5100); - begin_store('h6000, 1); - feed_descriptor('h6000, 1); - dma_ack(0); - no_future_requests(); - complete_command(6, 'h6000); - recovery('h6100); - // The descriptor keeps the unaligned VA, and data is sent unshifted once. - begin_store('h7003, 1); - feed_descriptor('h7003, 1); - dma_ack(1); - complete_command(); - expected = row_data(0); - check(descriptor_count == 1 && data_count == 1 && ack_ref_check( - model, 'h7003, expected[63:0], expected[127:64]) == 1, - "unaligned store changed raw bytes or emitted a second descriptor"); - // Each strided row has one groups*16-byte descriptor; fault suppresses row1. - begin_store('h8000, 2, 2, 3); - feed_descriptor('h8000, 2, 2, 3, 0, 1); - dma_ack(0, 3, 'h8000); - no_future_requests(); - complete_command(3, 'h8000); - check(descriptor_count == 1 && data_count == 2, "strided row did not group its raw bank beats"); - recovery('h8100); - check(store_cases == 10, "original store/recovery case count changed"); - ack_ref_destroy(model); - `uvm_info("ACK_GATE", - "10 existing cases: descriptor-final ACK delay, raw128 stable stream, typed fault/protocol fault, strided drain, stable completion and recovery", - UVM_LOW) - ctl.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_axi_peer.svh b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_axi_peer.svh deleted file mode 100644 index 062488e99..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_axi_peer.svh +++ /dev/null @@ -1,103 +0,0 @@ -typedef struct { - bit [63:0] addr; - int beat; -} axi_read_t; -typedef struct { - bit [63:0] addr; - int due; -} axi_write_t; -axi_read_t reads[int]; -axi_write_t writes[int]; -int aw_ids[$]; -bit [63:0] aw_addresses[$]; -bit [511:0] w_lines[$]; -bit [63:0] w_masks[$]; -bit [511:0] assembly; -bit [63:0] assembly_mask; -int w_beat = 0, r_id = 0, b_id = 0, r_cursor = 0; -bit r_active = 0, b_active = 0; -int axi_reads = 0, axi_writes = 0, final_b_waits = 0; -task automatic axi_service(); - forever begin - @(sample); - if (!sample.reset) begin - if (sample.io_axi_aw_valid && sample.io_axi_aw_ready) begin - ck( - sample.io_axi_aw_bits_len==3&&sample.io_axi_aw_bits_size==4&&sample.io_axi_aw_bits_burst==1&&!sample.io_axi_aw_bits_lock, - "Actual DDR AW line burst"); - aw_ids.push_back(sample.io_axi_aw_bits_id); - aw_addresses.push_back(sample.io_axi_aw_bits_addr); - end - if (sample.io_axi_w_valid && sample.io_axi_w_ready) begin - ck(sample.io_axi_w_bits_last == (w_beat == 3), "Actual DDR WLAST"); - assembly[w_beat*128+:128] = sample.io_axi_w_bits_data; - assembly_mask[w_beat*16+:16] = sample.io_axi_w_bits_strb; - if (w_beat == 3) begin - w_lines.push_back(assembly); - w_masks.push_back(assembly_mask); - w_beat = 0; - end else w_beat++; - end - if (aw_ids.size() && w_lines.size()) begin - int id = aw_ids.pop_front(); - bit [63:0] addr = aw_addresses.pop_front(); - bit [511:0] data = w_lines.pop_front(); - bit [63:0] mask = w_masks.pop_front(); - ck(!reads.exists(id) && !writes.exists(id), "Actual DDR active AXI ID reused for write"); - for (int beat = 0; beat < 4; beat++) - tile_peer_write128(addr + 16 * beat, data[128*beat+:64], data[128*beat+64+:64], - int'(mask[16*beat+:16])); - writes[id] = '{addr, cycles + 12}; - axi_writes++; - end - if (sample.io_axi_ar_valid && sample.io_axi_ar_ready) begin - int id = sample.io_axi_ar_bits_id; - bit [63:0] addr = sample.io_axi_ar_bits_addr; - ck( - sample.io_axi_ar_bits_len==3&&sample.io_axi_ar_bits_size==4&&sample.io_axi_ar_bits_burst==1&&!sample.io_axi_ar_bits_lock, - "Actual DDR AR line burst"); - ck(!reads.exists(id) && !writes.exists(id), "Actual DDR active AXI ID reused for read"); - reads[id] = '{addr, 0}; - axi_reads++; - end - if (sample.io_axi_r_valid && sample.io_axi_r_ready) begin - if (reads[r_id].beat == 3) reads.delete(r_id); - else reads[r_id].beat++; - r_active = 0; - r_cursor = (r_id + 1) % 16; - end - if (sample.io_axi_b_valid && sample.io_axi_b_ready) begin - writes.delete(b_id); - b_active = 0; - end - foreach (writes[id]) if (writes[id].due > cycles) final_b_waits++; - end - @(negedge clock); - io_axi_aw_ready = !reset && cycles % 5 != 0; - io_axi_w_ready = !reset && cycles % 7 >= 2; - io_axi_ar_ready = !reset && cycles % 4 != 0; - if (!r_active) - for (int off = 0; off < 16; off++) begin - int id = (r_cursor + off) % 16; - if (!r_active && reads.exists(id)) begin - bit [63:0] addr = reads[id].addr + 16 * reads[id].beat; - r_active = 1; - r_id = id; - io_axi_r_bits_id = id; - io_axi_r_bits_data = {tile_peer_read64(addr + 8), tile_peer_read64(addr)}; - io_axi_r_bits_last = reads[id].beat == 3; - io_axi_r_bits_resp = 0; - end - end - if (!b_active) - for (int id = 15; id >= 0; id--) - if (!b_active && writes.exists(id) && writes[id].due <= cycles) begin - b_active = 1; - b_id = id; - io_axi_b_bits_id = id; - io_axi_b_bits_resp = 0; - end - io_axi_r_valid = !reset && r_active; - io_axi_b_valid = !reset && b_active; - end -endtask diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.cm_hier b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.cm_hier deleted file mode 100644 index 772e1cfc5..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.cm_hier +++ /dev/null @@ -1 +0,0 @@ -+tree @TOP@.dut diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.f deleted file mode 100644 index 6cb2ed1e1..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/TileSystem/filelist.f -@RESOURCES@/tile_system_four_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.toml b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.toml deleted file mode 100644 index eb0e075b4..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four.toml +++ /dev/null @@ -1 +0,0 @@ -chips = ["goban"] diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four_config.svh b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four_config.svh deleted file mode 100644 index c10702927..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four_config.svh +++ /dev/null @@ -1,2 +0,0 @@ -`define TILE_IMAGE_PATH "../../../../../../../bb-tests/output/goban/workloads/src/CTest/chip/goban/goban-goban-ctest-chip-task_dma_four-multicore-baremetal" -`define TILE_EXPECT_PATH "../../../../../../../bb-tests/output/goban/workloads/src/CTest/chip/goban/goban-goban-ctest-chip-task_dma_four-multicore-baremetal.expect" diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four_tb.sv deleted file mode 100644 index acfb1599b..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_four_tb.sv +++ /dev/null @@ -1,4 +0,0 @@ -`include "tile_system_four_config.svh" -module tile_system_four_tb; - `include "tile_system_tb.svh" -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.cm_hier b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.cm_hier deleted file mode 100644 index 772e1cfc5..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.cm_hier +++ /dev/null @@ -1 +0,0 @@ -+tree @TOP@.dut diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.f b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.f deleted file mode 100644 index 6c26f8d5d..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv --F @RTL@/TileSystem/filelist.f -@RESOURCES@/tile_system_single_tb.sv diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.toml b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.toml deleted file mode 100644 index eb0e075b4..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single.toml +++ /dev/null @@ -1 +0,0 @@ -chips = ["goban"] diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single_config.svh b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single_config.svh deleted file mode 100644 index 6553cf780..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single_config.svh +++ /dev/null @@ -1,2 +0,0 @@ -`define TILE_IMAGE_PATH "../../../../../../../bb-tests/output/goban/workloads/src/CTest/chip/goban/goban-goban-ctest-chip-task_dma_single-multicore-baremetal" -`define TILE_EXPECT_PATH "../../../../../../../bb-tests/output/goban/workloads/src/CTest/chip/goban/goban-goban-ctest-chip-task_dma_single-multicore-baremetal.expect" diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single_tb.sv b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single_tb.sv deleted file mode 100644 index 49883f777..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_single_tb.sv +++ /dev/null @@ -1,4 +0,0 @@ -`include "tile_system_single_config.svh" -module tile_system_single_tb; - `include "tile_system_tb.svh" -endmodule diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_tb.svh b/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_tb.svh deleted file mode 100644 index ec2e42a74..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/resources/tile_system_tb.svh +++ /dev/null @@ -1,318 +0,0 @@ -`include "tile_system_config.svh" -import uvm_pkg::*; -import ip_control_test_pkg::*; -`include "uvm_macros.svh" -import "DPI-C" function longint unsigned tile_memory_init( - input string imagePath, - input string expectedPath -); -import "DPI-C" function longint unsigned tile_peer_read64(input longint unsigned address); -import "DPI-C" function void tile_peer_write128( - input longint unsigned address, - input longint unsigned lo, - input longint unsigned hi, - input int unsigned mask -); -import "DPI-C" function void tile_peer_finish(); -import "DPI-C" function void dpi_bdb_set_clk(input longint unsigned cycle); -logic clock = 0, reset = 1; -always #5 clock = ~clock; -ip_control_if ctl (clock); -`include "tile_system_signals.svh" -TileSystem dut ( - `include "tile_system_ports.svh" -); -logic device_valid[5], device_ready[5], device_write[5], device_normal[5]; -logic [63:0] device_addr[5], device_data[5]; -logic [5:0] device_tag[5]; -logic [2:0] device_size[5]; -logic [3:0] device_atomic[5]; -logic dev_response_valid[5], dev_response_ready[5]; -logic [ 5:0] dev_response_tag [5]; -logic [63:0] dev_response_data[5]; -logic retired[5], trapped[5], failure[5]; -logic [63:0] retired_pc[5], trap_pc[5], trap_cause[5], trap_value[5]; -assign device_valid[0] = io_deviceRequest_0_valid; -assign device_addr[0] = io_deviceRequest_0_bits_addr; -assign device_data[0] = io_deviceRequest_0_bits_data; -assign device_tag[0] = io_deviceRequest_0_bits_tag; -assign device_size[0] = io_deviceRequest_0_bits_size; -assign device_atomic[0] = io_deviceRequest_0_bits_atomic; -assign device_write[0] = io_deviceRequest_0_bits_write; -assign device_normal[0] = io_deviceRequest_0_bits_normal; -assign io_deviceRequest_0_ready = device_ready[0]; -assign dev_response_ready[0] = io_deviceResponse_0_ready; -assign io_deviceResponse_0_valid = dev_response_valid[0]; -assign io_deviceResponse_0_bits_tag = dev_response_tag[0]; -assign io_deviceResponse_0_bits_data = dev_response_data[0]; -assign io_deviceResponse_0_bits_error = 0; -assign retired[0] = io_retired_0; -assign retired_pc[0] = io_retiredPc_0; -assign trapped[0] = io_trapped_0; -assign trap_pc[0] = io_trapPc_0; -assign trap_cause[0] = io_trapCause_0; -assign trap_value[0] = io_trapValue_0; -assign failure[0] = io_failure_0_valid; -assign device_valid[1] = io_deviceRequest_1_valid; -assign device_addr[1] = io_deviceRequest_1_bits_addr; -assign device_data[1] = io_deviceRequest_1_bits_data; -assign device_tag[1] = io_deviceRequest_1_bits_tag; -assign device_size[1] = io_deviceRequest_1_bits_size; -assign device_atomic[1] = io_deviceRequest_1_bits_atomic; -assign device_write[1] = io_deviceRequest_1_bits_write; -assign device_normal[1] = io_deviceRequest_1_bits_normal; -assign io_deviceRequest_1_ready = device_ready[1]; -assign dev_response_ready[1] = io_deviceResponse_1_ready; -assign io_deviceResponse_1_valid = dev_response_valid[1]; -assign io_deviceResponse_1_bits_tag = dev_response_tag[1]; -assign io_deviceResponse_1_bits_data = dev_response_data[1]; -assign io_deviceResponse_1_bits_error = 0; -assign retired[1] = io_retired_1; -assign retired_pc[1] = io_retiredPc_1; -assign trapped[1] = io_trapped_1; -assign trap_pc[1] = io_trapPc_1; -assign trap_cause[1] = io_trapCause_1; -assign trap_value[1] = io_trapValue_1; -assign failure[1] = io_failure_1_valid; -assign device_valid[2] = io_deviceRequest_2_valid; -assign device_addr[2] = io_deviceRequest_2_bits_addr; -assign device_data[2] = io_deviceRequest_2_bits_data; -assign device_tag[2] = io_deviceRequest_2_bits_tag; -assign device_size[2] = io_deviceRequest_2_bits_size; -assign device_atomic[2] = io_deviceRequest_2_bits_atomic; -assign device_write[2] = io_deviceRequest_2_bits_write; -assign device_normal[2] = io_deviceRequest_2_bits_normal; -assign io_deviceRequest_2_ready = device_ready[2]; -assign dev_response_ready[2] = io_deviceResponse_2_ready; -assign io_deviceResponse_2_valid = dev_response_valid[2]; -assign io_deviceResponse_2_bits_tag = dev_response_tag[2]; -assign io_deviceResponse_2_bits_data = dev_response_data[2]; -assign io_deviceResponse_2_bits_error = 0; -assign retired[2] = io_retired_2; -assign retired_pc[2] = io_retiredPc_2; -assign trapped[2] = io_trapped_2; -assign trap_pc[2] = io_trapPc_2; -assign trap_cause[2] = io_trapCause_2; -assign trap_value[2] = io_trapValue_2; -assign failure[2] = io_failure_2_valid; -assign device_valid[3] = io_deviceRequest_3_valid; -assign device_addr[3] = io_deviceRequest_3_bits_addr; -assign device_data[3] = io_deviceRequest_3_bits_data; -assign device_tag[3] = io_deviceRequest_3_bits_tag; -assign device_size[3] = io_deviceRequest_3_bits_size; -assign device_atomic[3] = io_deviceRequest_3_bits_atomic; -assign device_write[3] = io_deviceRequest_3_bits_write; -assign device_normal[3] = io_deviceRequest_3_bits_normal; -assign io_deviceRequest_3_ready = device_ready[3]; -assign dev_response_ready[3] = io_deviceResponse_3_ready; -assign io_deviceResponse_3_valid = dev_response_valid[3]; -assign io_deviceResponse_3_bits_tag = dev_response_tag[3]; -assign io_deviceResponse_3_bits_data = dev_response_data[3]; -assign io_deviceResponse_3_bits_error = 0; -assign retired[3] = io_retired_3; -assign retired_pc[3] = io_retiredPc_3; -assign trapped[3] = io_trapped_3; -assign trap_pc[3] = io_trapPc_3; -assign trap_cause[3] = io_trapCause_3; -assign trap_value[3] = io_trapValue_3; -assign failure[3] = io_failure_3_valid; -assign device_valid[4] = io_deviceRequest_4_valid; -assign device_addr[4] = io_deviceRequest_4_bits_addr; -assign device_data[4] = io_deviceRequest_4_bits_data; -assign device_tag[4] = io_deviceRequest_4_bits_tag; -assign device_size[4] = io_deviceRequest_4_bits_size; -assign device_atomic[4] = io_deviceRequest_4_bits_atomic; -assign device_write[4] = io_deviceRequest_4_bits_write; -assign device_normal[4] = io_deviceRequest_4_bits_normal; -assign io_deviceRequest_4_ready = device_ready[4]; -assign dev_response_ready[4] = io_deviceResponse_4_ready; -assign io_deviceResponse_4_valid = dev_response_valid[4]; -assign io_deviceResponse_4_bits_tag = dev_response_tag[4]; -assign io_deviceResponse_4_bits_data = dev_response_data[4]; -assign io_deviceResponse_4_bits_error = 0; -assign retired[4] = io_retired_4; -assign retired_pc[4] = io_retiredPc_4; -assign trapped[4] = io_trapped_4; -assign trap_pc[4] = io_trapPc_4; -assign trap_cause[4] = io_trapCause_4; -assign trap_value[4] = io_trapValue_4; -assign failure[4] = io_failure_4_valid; -clocking sample @(posedge clock); - default input #1step; - input reset; - input device_valid,device_ready,device_write,device_normal,device_addr,device_data,device_tag,device_size,device_atomic; - input dev_response_valid,dev_response_ready,retired,retired_pc,trapped,trap_pc,trap_cause,trap_value,failure; - input io_memoryOutstanding; - input io_axi_aw_valid,io_axi_aw_ready,io_axi_aw_bits_id,io_axi_aw_bits_addr,io_axi_aw_bits_len,io_axi_aw_bits_size,io_axi_aw_bits_burst,io_axi_aw_bits_lock; - input io_axi_w_valid, io_axi_w_ready, io_axi_w_bits_data, io_axi_w_bits_strb, io_axi_w_bits_last; - input io_axi_ar_valid,io_axi_ar_ready,io_axi_ar_bits_id,io_axi_ar_bits_addr,io_axi_ar_bits_len,io_axi_ar_bits_size,io_axi_ar_bits_burst,io_axi_ar_bits_lock; - input io_axi_r_valid, io_axi_r_ready, io_axi_b_valid, io_axi_b_ready; -endclocking -int cycles = 0, retire_count[5], device_requests = 0, device_responses = 0; -bit exit_seen = 0, exit_ack = 0; -int exit_status = 0; -bit device_pending[5], device_exit[5]; -int device_due[5]; -bit [5:0] pending_tag[5]; -bit [63:0] pending_data[5]; -function automatic void ck(bit ok, string message); - if (!ok) `uvm_fatal("TILE_SYSTEM", message) -endfunction -`include "tile_axi_peer.svh" -task automatic device_service(); - forever begin - @(sample); - if (!sample.reset) - for (int i = 0; i < 5; i++) begin - if (sample.device_valid[i] && sample.device_ready[i]) begin - ck(!device_pending[i] && !sample.device_normal[i] && sample.device_atomic[i] == 0, - "Illegal actual Tile Device request/owner"); - device_pending[i] = 1; - pending_tag[i] = sample.device_tag[i]; - pending_data[i] = 0; - device_due[i] = cycles + 4; - device_exit[i] = 0; - device_requests++; - case (sample.device_addr[i]) - 'h60000000: begin - ck(i == 0 && sample.device_write[i] && sample.device_size[i] == 2 && !exit_seen, - "Completion must be one controller 32-bit Device write"); - exit_seen = 1; - exit_status = int'(sample.device_data[i][31:0]); - device_exit[i] = 1; - ck(exit_status == 0, $sformatf( - "CPU firmware exited status%0d pc%h", exit_status, sample.retired_pc[i])); - end - 'h10000000: begin - if (sample.device_write[i]) $write("%c", sample.device_data[i][7:0]); - end - 'h10000005: begin - ck(!sample.device_write[i], "UART status is read only"); - pending_data[i] = 'h60; - end - default: - ck(0, $sformatf( - "Unsupported actual Device address hart%0d addr%h", i, sample.device_addr[i])); - endcase - end - if (sample.dev_response_valid[i] && sample.dev_response_ready[i]) begin - ck(device_pending[i], "Device response without retained CPU owner"); - if (device_exit[i]) exit_ack = 1; - device_pending[i] = 0; - device_responses++; - end - end - @(negedge clock); - for (int i = 0; i < 5; i++) begin - device_ready[i] = !reset && !device_pending[i] && cycles % 5 != 0; - dev_response_valid[i] = !reset && device_pending[i] && device_due[i] <= cycles; - dev_response_tag[i] = pending_tag[i]; - dev_response_data[i] = pending_data[i]; - end - end -endtask -task automatic monitor_cpu(); - forever begin - @(sample); - if (!sample.reset) begin - cycles++; - dpi_bdb_set_clk(cycles); - for (int i = 0; i < 5; i++) begin - ck(!sample.failure[i], $sformatf("Actual NPU DMA failure hart%0d", i)); - if (sample.retired[i]) retire_count[i]++; - if (sample.trapped[i]) begin - ck(i != 0 && sample.trap_cause[i] == 8, $sformatf( - "Unexpected CPU trap hart%0d pc%h cause%h value%h", - i, - sample.trap_pc[i], - sample.trap_cause[i], - sample.trap_value[i] - )); - end - end - end - end -endtask -initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "*", "vif", ctl); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 1ms); - run_test("protocol_test"); -end -initial begin - longint unsigned entry; - ck(`TILE_CPU_COUNT == 5, "Tile gate requires actual Goban five-CPU profile"); - for (int i = 0; i < 5; i++) begin - device_ready[i] = 0; - dev_response_valid[i] = 0; - dev_response_tag[i] = 0; - dev_response_data[i] = 0; - device_pending[i] = 0; - device_exit[i] = 0; - device_due[i] = 0; - pending_tag[i] = 0; - pending_data[i] = 0; - retire_count[i] = 0; - end - io_axi_aw_ready = 0; - io_axi_ar_ready = 0; - io_axi_w_ready = 0; - io_axi_r_valid = 0; - io_axi_b_valid = 0; - io_axi_r_bits_id = 0; - io_axi_r_bits_data = 0; - io_axi_r_bits_last = 0; - io_axi_r_bits_resp = 0; - io_axi_b_bits_id = 0; - io_axi_b_bits_resp = 0; - io_interrupts_0_external = 0; - io_interrupts_0_software = 0; - io_interrupts_0_timer = 0; - io_interrupts_1_external = 0; - io_interrupts_1_software = 0; - io_interrupts_1_timer = 0; - io_interrupts_2_external = 0; - io_interrupts_2_software = 0; - io_interrupts_2_timer = 0; - io_interrupts_3_external = 0; - io_interrupts_3_software = 0; - io_interrupts_3_timer = 0; - io_interrupts_4_external = 0; - io_interrupts_4_software = 0; - io_interrupts_4_timer = 0; - entry = tile_memory_init(`TILE_IMAGE_PATH, `TILE_EXPECT_PATH); - io_resetVector_0 = entry; - io_resetVector_1 = entry; - io_resetVector_2 = entry; - io_resetVector_3 = entry; - io_resetVector_4 = entry; - wait (ctl.start); - repeat (5) @(negedge clock); - reset = 0; - fork - axi_service(); - device_service(); - monitor_cpu(); - join_none - wait (exit_ack); - do - @(sample); - while (reads.num() || writes.num() || aw_ids.size() || aw_addresses.size() || w_lines.size() || - w_masks.size() || w_beat || r_active || b_active || sample.io_memoryOutstanding != 0); - ck(device_requests == device_responses, "Device completions not drained"); - for (int i = 0; i < 5; i++) - ck(retire_count[i] > 0, $sformatf("Configured CPU%0d never executed", i)); - ck(axi_reads > 0 && axi_writes > 0 && final_b_waits > 0, - "Firmware did not exercise actual DDR read/write/finalB"); - tile_peer_finish(); - `uvm_info("TILE_SYSTEM_PASS", $sformatf( - "actual five CPU firmware cycles%0d DDR read/write%0d/%0d finalBwait%0d Device%0d/%0d status%0d; no RoCC BFM", - cycles, - axi_reads, - axi_writes, - final_b_waits, - device_requests, - device_responses, - exit_status - ), UVM_LOW) - ctl.done = 1; -end diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/AdmissionSystem.scala b/arch/src/main/scala/framework/memdomain/verification/src/main/scala/AdmissionSystem.scala deleted file mode 100644 index cdbaa8e1c..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/AdmissionSystem.scala +++ /dev/null @@ -1,117 +0,0 @@ -package framework.memdomain.verification - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import framework.top.GlobalConfig -import framework.system.core.accelerator.{Admission, BuckyballAccelerator} -import framework.system.core.rocket.RoCCIO -import framework.memdomain.backend.shared.{SharedMemBackend, SharedMemLayout} -import hier.core.rocket.AdmissionPorts -import hier.tile.TaskController -import memcore.memory.interlock.{Params => TrackingParams} -import memcore.memory.preflight.Params -import memcore.memory.cpu.PhysicalRegion -import memcore.bus.axi4 - -/** Actual NPU/ROB/task/translation/AXI join. Core and CMO peer use the declared BFM contract. */ -@instantiable -class AdmissionSystem( - b: GlobalConfig, - prepared: Params, - coreIndex: Int, - workerSignatures: Seq[BigInt], - regions: Seq[PhysicalRegion] -)( - implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - val tracking = TrackingParams(addressBits = prepared.bus.addressBits) - val axiParams = axi4.Params(prepared.bus.addressBits, 128, 4) - - @public - val io = IO(new Bundle { - val core = Flipped(new AdmissionPorts(tracking, nPMPs, prepared.bus)) - val controller = new RoCCIO(64) - val controllerSatp = Input(UInt(64.W)) - val axi = new memcore.bus.axi4.Port(axiParams) - val npuIdle = Output(Bool()) - val halted = Output(Bool()) - val fault = Output(new framework.memdomain.frontend.mem.dma.DmaStatus) - val faultTag = Output(UInt(tracking.idBits.W)) - val workDrained = Output(Bool()) - val blockedResponse = Input(Bool()) - }) - - val admission: Instance[Admission] = Instantiate(new Admission(b, tracking, prepared, regions, axiParams)) - val accelerator: Instance[BuckyballAccelerator] = Instantiate(new BuckyballAccelerator(b)) - val tasks: Instance[TaskController] = - Instantiate(new TaskController((1 to workerSignatures.size), workerSignatures, b.memDomain.nCores)) - admission.io.core <> io.core - accelerator.io.cmd <> admission.io.npuCommand - admission.io.npuResponse <> accelerator.io.resp - admission.io.allocation.valid := accelerator.io.allocation.valid - admission.io.allocation.bits := accelerator.io.allocation.bits.rob_id - admission.io.retired := accelerator.io.retired - admission.io.npuFault := accelerator.io.fault - admission.io.npuBusy := !accelerator.io.idle - admission.io.npuInterrupt := accelerator.io.interrupt - admission.io.footprints := accelerator.io.footprints - admission.io.dma <> accelerator.io.dma - accelerator.io.hartid := coreIndex.U - accelerator.io.sharedBankOwnerHartId := 0.U - accelerator.io.barrier_release := accelerator.io.barrier_arrive - admission.io.task <> tasks.io.ports(coreIndex) - tasks.io.ports(0) <> io.controller - tasks.io.satp := io.controllerSatp - for (i <- 1 until b.memDomain.nCores if i != coreIndex) { - tasks.io.ports(i).cmd.valid := false.B - tasks.io.ports(i).cmd.bits := 0.U.asTypeOf(tasks.io.ports(i).cmd.bits) - tasks.io.ports(i).resp.ready := true.B - tasks.io.ports(i).exception := false.B - } - io.npuIdle := accelerator.io.idle - io.halted := admission.io.halted; io.fault := admission.io.fault - io.faultTag := admission.io.faultTag; io.workDrained := admission.io.workDrained - - val shared: Instance[SharedMemBackend] = Instantiate(new SharedMemBackend(b, useMesh = true)) - shared.io.config <> accelerator.io.shared_config - shared.io.mvover <> accelerator.io.mvover - val channels = SharedMemLayout.channelPerHart(b) - val endpoint = b.memDomain.computeCoreIds.indexOf(coreIndex) - require(endpoint >= 0) - for (i <- shared.io.mem_req.indices) { - if (i >= endpoint * channels && i < (endpoint + 1) * channels) { - shared.io.mem_req(i) <> accelerator.io.shared_mem_req(i - endpoint * channels) - } else { - val q = shared.io.mem_req(i) - q.read.req.valid := false.B; q.read.req.bits := 0.U.asTypeOf(q.read.req.bits); q.read.resp.ready := true.B - q.write.req.valid := false.B; q.write.req.bits := 0.U.asTypeOf(q.write.req.bits); q.write.resp.ready := true.B - q.bank_id := 0.U; q.group_id := 0.U; q.is_shared := false.B; q.hart_id := 0.U; q.rob_id := 0.U; q.inst_id := 0.U - } - } - for (i <- 0 until b.memDomain.nCores) { - shared.io.query_valid(i) := (if (i == coreIndex) accelerator.io.shared_query_valid else false.B) - shared.io.query_hart_id(i) := i.U - shared.io.query_vbank_id(i) := (if (i == coreIndex) accelerator.io.shared_query_vbank_id else 0.U) - if (i == coreIndex) shared.io.localBanks(i) <> accelerator.io.meshLocalBank - else { - shared.io.localBanks(i).request.ready := false.B - shared.io.localBanks(i).response.valid := false.B - shared.io.localBanks(i).response.bits := 0.U.asTypeOf(shared.io.localBanks(i).response.bits) - } - } - accelerator.io.shared_query_group_count := shared.io.query_group_count(coreIndex) - accelerator.io.shared_bank_hashes.foreach(_ := shared.io.bank_hashes.get) - - io.axi.aw <> admission.io.axi.aw - io.axi.w <> admission.io.axi.w - io.axi.ar <> admission.io.axi.ar - admission.io.axi.r.valid := io.axi.r.valid && !io.blockedResponse - admission.io.axi.r.bits := io.axi.r.bits - io.axi.r.ready := admission.io.axi.r.ready && !io.blockedResponse - admission.io.axi.b.valid := io.axi.b.valid && !io.blockedResponse - admission.io.axi.b.bits := io.axi.b.bits - io.axi.b.ready := admission.io.axi.b.ready && !io.blockedResponse -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/ControllerSystem.scala b/arch/src/main/scala/framework/memdomain/verification/src/main/scala/ControllerSystem.scala deleted file mode 100644 index 210ff7769..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/ControllerSystem.scala +++ /dev/null @@ -1,107 +0,0 @@ -package framework.memdomain.verification - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import framework.top.GlobalConfig -import framework.system.core.ControllerAdmission -import framework.system.core.rocket.RoCCIO -import framework.system.tile.BankNetwork -import hier.core.rocket.AdmissionPorts -import hier.tile.TaskController -import memcore.memory.interlock.{Params => TrackingParams} -import memcore.bus.chi.{Params => ChiParams} -import memcore.memory.bank.{Bank, BankRequest, BankSetParams} - -/** Real controller/task/bank-move composition; CPU command and bank initialization are test peers. */ -@instantiable -class ControllerSystem(b: GlobalConfig, signatures: Seq[BigInt])(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - val tracking = TrackingParams() - val bankParams = - BankSetParams(b.memDomain.bankWidth, 1, b.memDomain.bankEntries, math.max(1, log2Ceil(b.frontend.rob_entries))) - val cores = b.memDomain.computeCoreIds - - @public - val io = IO(new Bundle { - val core = Flipped(new AdmissionPorts(tracking, nPMPs, ChiParams())) - val workers = Vec(signatures.size, new RoCCIO(64)) - - val access = Vec( - cores.size, - Flipped(new memcore.memory.mesh_shm.MeshLocalBankPort( - 16, - 10, - bankParams.dataBits, - bankParams.tagBits - )) - ) - - val allowBankResponse = Input(Vec(cores.size, Bool())) - val moveFault = Valid(new framework.memdomain.frontend.mem.dma.DmaStatus) - val moveAccepted = Output(Bool()) - val moveCompleted = Output(Bool()) - val bankWriteAccepted = Output(Vec(cores.size, Bool())) - }) - - val admission: Instance[ControllerAdmission] = - Instantiate(new ControllerAdmission(tracking, ChiParams(), moves = true)) - val tasks: Instance[TaskController] = - Instantiate(new TaskController(1 to signatures.size, signatures, b.memDomain.nCores)) - val network: Instance[BankNetwork] = Instantiate(new BankNetwork(b, cores, useMesh = true, controllerMove = true)) - admission.io.core <> io.core - admission.io.task <> tasks.io.ports(0) - tasks.io.satp := admission.io.taskSatp - for (i <- signatures.indices) { tasks.io.ports(i + 1) <> io.workers(i) } - admission.io.move <> network.io.controllerMvover.get - io.moveFault := admission.io.moveFault - io.moveAccepted := admission.io.move.command.fire - io.moveCompleted := admission.io.move.completion.fire - network.io.hartIds.zipWithIndex.foreach { case (hart, i) => hart := i.U } - for (i <- cores.indices) { - val port = network.io.compute(i) - port.config.valid := false.B; port.config.bits := 0.U.asTypeOf(port.config.bits) - port.queryValid := false.B; port.queryVbank := 0.U; port.barrierArrive := false.B - port.move.command.valid := false.B; port.move.command.bits := 0.U.asTypeOf(port.move.command.bits) - port.move.completion.ready := true.B - for (q <- port.requests) { - q.read.req.valid := false.B; q.read.req.bits := 0.U.asTypeOf(q.read.req.bits); q.read.resp.ready := true.B - q.write.req.valid := false.B; q.write.req.bits := 0.U.asTypeOf(q.write.req.bits); q.write.resp.ready := true.B - q.bank_id := 0.U; q.group_id := 0.U; q.is_shared := false.B - q.hart_id := 0.U; q.rob_id := 0.U; q.inst_id := 0.U - } - // Bank zero is the declared private endpoint used by this gate. Storage and ACK are the existing Bank IP. - val bank: Instance[Bank] = Instantiate(new Bank(bankParams)) - val choose = Module(new Arbiter(chiselTypeOf(port.local.request.bits), 2)) - choose.io.in(0) <> port.local.request - choose.io.in(1) <> io.access(i).request - bank.io.request.valid := choose.io.out.valid - choose.io.out.ready := bank.io.request.ready - bank.io.request.bits := 0.U.asTypeOf(new BankRequest(bankParams)) - bank.io.request.bits.addr := choose.io.out.bits.addr - bank.io.request.bits.write := choose.io.out.bits.write - bank.io.request.bits.data := choose.io.out.bits.data - bank.io.request.bits.mask := choose.io.out.bits.mask - bank.io.request.bits.tag := choose.io.out.bits.tag - val owner = Reg(UInt(1.W)) - io.bankWriteAccepted(i) := choose.io.in(0).fire && choose.io.in(0).bits.write - when(choose.io.out.fire) { - assert( - choose.io.out.bits.bank === 0.U && choose.io.out.bits.addr < bankParams.entriesPerBank.U, - "ControllerSystem endpoint is private bank zero within the actual bank capacity" - ) - owner := choose.io.chosen - } - port.local.response.bits := bank.io.response.bits - io.access(i).response.bits := bank.io.response.bits - port.local.response.valid := bank.io.response.valid && owner === 0.U && io.allowBankResponse(i) - io.access(i).response.valid := bank.io.response.valid && owner === 1.U - bank.io.response.ready := Mux( - owner === 0.U, - port.local.response.ready && io.allowBankResponse(i), - io.access(i).response.ready - ) - } -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/Emit.scala b/arch/src/main/scala/framework/memdomain/verification/src/main/scala/Emit.scala deleted file mode 100644 index c37e15172..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/Emit.scala +++ /dev/null @@ -1,321 +0,0 @@ -package framework.memdomain.verification - -import chisel3._ -import chisel3.reflect.DataMirror -import chisel3.ActualDirection -import framework.system.core.rocket.CpuParams -import java.nio.file.{Files, Paths} -import framework.system.configloader.{ChipLoader, RocketTileCore} -import framework.memdomain.frontend.mem.{KernelDma, MemConfiger, MemLoader, MemStorer} -import framework.memdomain.backend.privatepath.PrivateMemBackend -import framework.memdomain.backend.shared.SharedMemBackend - -object Emit extends App { - require( - args.length >= 4 && args(0) == "--chip" && args(2) == "--targets", - "Usage: memdomain_ack.run --chip --targets [firtool options]" - ) - val chip = args(1) - val targets = args(3).split(",").toSet - - val known = Set( - "mset_private", - "mset_shared", - "shared_lease", - "mset_configer", - "program_rob", - "program_decoder", - "loader_ack", - "read_dma", - "storer_ack", - "kernel_dma", - "admission_system", - "controller_system", - "memory_system", - "tile_system_single", - "tile_system_four" - ) - - require(targets.nonEmpty && targets.subsetOf(known), s"Unknown verification targets: ${targets -- known}") - val firtoolOptions = args.drop(4) - require(chip == "toy" || chip == "goban", s"MemDomain gates declare Toy or Goban profiles, got $chip") - val repo = Paths.get("..").toAbsolutePath.normalize - lazy val topology = ChipLoader.load(repo.resolve(s"examples/chips/$chip/configs/generated/chip.pb").toString) - - lazy val b = { - val configs = topology.tiles.flatMap(_.cores).collect { case RocketTileCore(_, Some(config)) => config } - require(configs.nonEmpty, "MemDomain gate requires a configured accelerator") - if (chip == "toy") require(configs.size == 1, "Toy ACK gate requires exactly one compute configuration") - val config = configs.head - require( - config.memDomain.bankWidth == 128 && config.memDomain.bankMaskLen == 16, - "ACK gate profile requires 16-byte bank rows and byte masks" - ) - config - } - - val output = repo.resolve("arch/src/main/scala/framework/memdomain/verification/build") - Files.createDirectories(output) - - def leaves(d: Data, name: String): Seq[(String, Data)] = d match { - case r: Record => r.elements.toSeq.flatMap { case (n, v) => leaves(v, s"${name}_$n") } - case v: Vec[_] => v.zipWithIndex.flatMap { case (e, i) => leaves(e, s"${name}_$i") }.toSeq - case e => if (e.getWidth > 0) Seq(name -> e) else Seq.empty - } - - lazy val msetConfig = { - val defaults = framework.top.GlobalConfig() - defaults.copy( - memDomain = defaults.memDomain.copy( - bankNum = 4, - bankWidth = 128, - bankEntries = 8, - bankMaskLen = 16, - virtualBankCount = 16, - sharedEnable = true, - sharedEntries = 32, - sharedBankEntries = 8, - sharedBankNum = 4, - sharedInputChannels = 2, - sharedDefaultGroupCount = 1, - nCores = 2, - computeCoreIds = Seq(0, 1), - bankChannel = 2, - dma_buswidth = 128, - memAddrLen = 39 - ), - frontend = defaults.frontend.copy( - rob_entries = 8, - bank_id_len = 10, - vbank_id_upper_bound = 7, - shared_bank_id_base = 8, - iter_len = 34, - sub_rob_depth = 4 - ), - tile = defaults.tile.copy(xLen = 64), - sim = defaults.sim.copy(diffTest = true) - ) - } - - lazy val programConfig = msetConfig.copy( - rvv = msetConfig.rvv.copy(enable = true), - sim = msetConfig.sim.copy(diffTest = false) - ) - - for ( - target <- Seq("mset_private", "mset_shared", "shared_lease", "mset_configer", "program_rob", "program_decoder") - if targets(target) - ) { - var interface: Data = null - val moduleName = target match { - case "mset_private" => "PrivateMemBackend" - case "mset_shared" => "SharedMemBackend" - case "shared_lease" => "SharedMemBackend" - case "mset_configer" => "MemConfiger" - case "program_rob" => "GlobalROB" - case "program_decoder" => "GlobalDecoder" - } - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - target match { - case "mset_private" => val m = new PrivateMemBackend(msetConfig); interface = m.io; m - case "mset_shared" => val m = new SharedMemBackend(msetConfig); interface = m.io; m - case "shared_lease" => - val m = new SharedMemBackend(msetConfig, useMesh = true, externalPhysicalPorts = 1); interface = m.io; m - case "mset_configer" => val m = new MemConfiger(msetConfig); interface = m.io; m - case "program_rob" => val m = new framework.frontend.globalrs.GlobalROB(programConfig); interface = m.io; m - case "program_decoder" => - val m = new framework.frontend.decoder.GlobalDecoder(programConfig); interface = m.io; m - }, - args = Array("--target-dir", output.resolve(moduleName).toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - val fields = leaves(interface, "io") - if (target == "shared_lease") Files.writeString( - output.resolve("shared_lease_clocking.svh"), - "clocking sample @(posedge clock);\ndefault input #1step;\ninput reset;\n" + - fields.map { case (n, _) => s"input $n;" }.mkString("\n") + "\nendclocking\n" - ) - Files.writeString( - output.resolve(s"${target}_signals.svh"), - fields.map { case (name, data) => s"logic [${data.getWidth - 1}:0] $name;" }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve(s"${target}_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ fields.map { case (name, _) => s".$name($name)" }).mkString( - ",\n" - ) + "\n" - ) - Files.writeString( - output.resolve(s"${target}_init.svh"), - fields.collect { - case (name, data) if DataMirror.directionOf(data) == ActualDirection.Input => s"$name = '0;" - }.mkString("\n") + "\n" - ) - } - - for (load <- Seq(true, false) if targets(if (load) "loader_ack" else "storer_ack")) { - var io: Data = null - val stem = if (load) "loader_ack" else "storer_ack" - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - if (load) { val m = new MemLoader(msetConfig); io = m.io; m } - else { val m = new MemStorer(msetConfig); io = m.io; m }, - args = Array("--target-dir", output.resolve(if (load) "MemLoader" else "MemStorer").toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - val fs = leaves(io, "io") - Files.writeString( - output.resolve(s"${stem}_signals.svh"), - fs.map { case (n, d) => s"logic [${d.getWidth - 1}:0] $n;" }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve(s"${stem}_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ fs.map { case (n, _) => s".$n($n)" }).mkString(",\n") + "\n" - ) - Files.writeString( - output.resolve(s"${stem}_init.svh"), - fs.collect { case (n, d) if DataMirror.directionOf(d) == ActualDirection.Input => s"$n = '0;" }.mkString( - "\n" - ) + "\n" - ) - } - - if (targets("read_dma")) { - var readIo: Data = null - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - { - val m = new framework.memdomain.frontend.mem.dma.ReadDma( - msetConfig, - memcore.memory.preflight.Params(beatBytes = 16), - memcore.bus.axi4.Params() - ); readIo = m.io; m - }, - args = Array("--target-dir", output.resolve("ReadDma").toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - val fields = leaves(readIo, "io") - Files.writeString( - output.resolve("read_dma_signals.svh"), - fields.map { case (n, d) => s"logic [${d.getWidth - 1}:0] $n;" }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("read_dma_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ fields.map { case (n, _) => s".$n($n)" }).mkString(",\n") + "\n" - ) - Files.writeString( - output.resolve("read_dma_init.svh"), - fields.collect { case (n, d) if DataMirror.directionOf(d) == ActualDirection.Input => s"$n = '0;" }.mkString( - "\n" - ) + "\n" - ) - Files.writeString( - output.resolve("read_dma_clocking.svh"), - "clocking sample @(posedge clock);\ndefault input #1step;\ninput reset;\n" + - fields.map { case (n, _) => s"input $n;" }.mkString("\n") + "\nendclocking\n" - ) - } - - if (targets("kernel_dma")) { - var kernelIo: Data = null - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - { val m = new KernelDma(b); kernelIo = m.io; m }, - args = Array("--target-dir", output.resolve("KernelDma").toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - val kernelFields = leaves(kernelIo, "io") - Files.writeString( - output.resolve("kernel_dma_signals.svh"), - kernelFields.map { case (n, d) => s"logic [${d.getWidth - 1}:0] $n;" }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("kernel_dma_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ kernelFields.map { case (n, _) => s".$n($n)" }).mkString(",\n") + "\n" - ) - Files.writeString( - output.resolve("kernel_dma_init.svh"), - kernelFields.collect { - case (n, d) if DataMirror.directionOf(d) == ActualDirection.Input => s"$n = '0;" - }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("kernel_dma_clocking.svh"), - "clocking sample @(posedge clock);\ndefault input #1step;\ninput reset;\n" + - kernelFields.map { case (n, _) => s"input $n;" }.mkString("\n") + "\nendclocking\n" - ) - } - if (targets("admission_system") || targets("controller_system")) { - // Task admission and the controller belong to compute tiles; the main tile has neither. - val tile = topology.mountedTiles.find(_.controller.isDefined).getOrElse( - throw new IllegalArgumentException(s"$chip: admission/controller gates need a compute tile") - ) - val selected = tile.cores.zipWithIndex.collectFirst { case (RocketTileCore(cpu, Some(config)), i) => - (cpu, config, i) - }.get - val cpu = selected._1; val config = selected._2; val coreIndex = selected._3 - val signatures = tile.signatures.tail - require( - signatures.size == tile.cores.size - 1 && config.memDomain.sharedEnable, - "AdmissionSystem requires the actual shared-bank/task-worker configuration" - ) - implicit val coreParameters: CpuParams = CpuParams( - core = framework.system.core.rocket.configs.RocketCpuParam.toRocketCoreParams( - cpu, - tile.param.xLen, - tile.param.pgLevels - ), - dcache = Some(freechips.rocketchip.rocket.DCacheParams()), - icache = Some(freechips.rocketchip.rocket.ICacheParams()), - btb = None, - physicalAddressBits = tile.param.paddrBits, - hartIdBits = 6, - beatBytes = 16, - blockBytes = 64 - ) - val prepared = memcore.memory.preflight.Params(beatBytes = 16) - val regions = - Seq(memcore.memory.cpu.PhysicalRegion(BigInt("80000000", 16), BigInt(16) << 20, true, true, true, true, true, true)) - if (targets("admission_system")) { - var interface: Data = null - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - { val m = new AdmissionSystem(config, prepared, coreIndex, signatures, regions); interface = m.io; m }, - args = Array("--target-dir", output.resolve("AdmissionSystem").toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - val fields = leaves(interface, "io") - Files.writeString( - output.resolve("admission_system_signals.svh"), - fields.map { case (n, d) => s"logic [${d.getWidth - 1}:0] $n;" }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("admission_system_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ fields.map { case (n, _) => s".$n($n)" }).mkString(",\n") + "\n" - ) - Files.writeString( - output.resolve("admission_system_init.svh"), - fields.collect { case (n, d) if DataMirror.directionOf(d) == ActualDirection.Input => s"$n = '0;" }.mkString( - "\n" - ) + "\n" - ) - Files.writeString( - output.resolve("admission_system_clocking.svh"), - "clocking sample @(posedge clock);\ndefault input #1step;\ninput reset;\n" + - fields.map { case (n, _) => s"input $n;" }.mkString("\n") + "\nendclocking\n" - ) - Files.writeString( - output.resolve("admission_system_config.svh"), - s"`define ADMISSION_ROWS ${config.memDomain.bankEntries}\n`define ADMISSION_SIGNATURE 64'h${config.coreSignature.toString(16)}\n`define ADMISSION_CORE $coreIndex\n" - ) - } - if (targets("controller_system")) EmitController(output, config, signatures, firtoolOptions) - } - - if (targets.exists(_.startsWith("tile_system_"))) { - require(chip == "goban", "Tile verification profile requires Goban") - val description = buckyball.config.Chip.parseFrom( - Files.readAllBytes(repo.resolve(s"examples/chips/$chip/configs/generated/chip.pb")) - ) - val platform = Class.forName(description.getMill.getVerilatorConfig).getDeclaredConstructor() - .newInstance().asInstanceOf[sims.soc.SystemTarget] - EmitTile(topology, output, firtoolOptions, platform.instantiate) - } - if (targets("memory_system")) EmitMemory(output, firtoolOptions) -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitController.scala b/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitController.scala deleted file mode 100644 index 0f746f709..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitController.scala +++ /dev/null @@ -1,62 +0,0 @@ -package framework.memdomain.verification - -import chisel3._ -import chisel3.reflect.DataMirror -import java.nio.file.{Files, Path} -import framework.system.core.rocket.CpuParams -import framework.top.GlobalConfig - -object EmitController { - - def apply( - output: Path, - b: GlobalConfig, - signatures: Seq[BigInt], - firtoolOptions: Array[String] - )( - implicit cpuParams: CpuParams - ): Unit = { - var interface: Data = null - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - { val m = new ControllerSystem(b, signatures); interface = m.io; m }, - args = Array("--target-dir", output.resolve("ControllerSystem").toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - def leaves(data: Data, name: String): Seq[(String, Data)] = data match { - case bundle: Record => bundle.elements.toSeq.flatMap { case (field, value) => leaves(value, s"${name}_$field") } - case vec: Vec[_] => vec.zipWithIndex.flatMap { case (value, i) => leaves(value, s"${name}_$i") }.toSeq - case value => Seq(name -> value) - } - val fields = leaves(interface, "io") - Files.writeString( - output.resolve("controller_system_signals.svh"), - fields.map { - case (name, data) => s"logic [${data.getWidth - 1}:0] $name;" - }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("controller_system_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ fields.map { case (name, _) => s".$name($name)" }).mkString( - ",\n" - ) + "\n" - ) - Files.writeString( - output.resolve("controller_system_init.svh"), - fields.collect { - case (name, data) if DataMirror.directionOf(data) == ActualDirection.Input => s"$name = '0;" - }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("controller_system_clocking.svh"), - "clocking sample @(posedge clock);\ndefault input #1step;\ninput reset;\n" + - fields.map { - case (name, _) => s"input $name;" - }.mkString("\n") + "\nendclocking\n" - ) - Files.writeString( - output.resolve("controller_system_config.svh"), - s"`define CONTROLLER_SIGNATURE 64'h${signatures.head.toString(16)}\n" - ) - } - -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitMemory.scala b/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitMemory.scala deleted file mode 100644 index 430c0470e..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitMemory.scala +++ /dev/null @@ -1,54 +0,0 @@ -package framework.memdomain.verification - -import chisel3._ -import chisel3.reflect.DataMirror -import java.nio.file.{Files, Path} -import framework.system.memory.Memory -import memcore.memory.ddr.{Params => DdrParams} - -object EmitMemory { - - def apply(output: Path, firtoolOptions: Array[String]): Unit = { - val ddr = DdrParams() - var interface: Data = null - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - { val m = new Memory(ddr, dmaMasters = 1); interface = m.io; m }, - args = Array("--target-dir", output.resolve("Memory").toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - def leaves(data: Data, name: String): Seq[(String, Data)] = data match { - case bundle: Record => bundle.elements.toSeq.flatMap { case (field, value) => leaves(value, s"${name}_$field") } - case vec: Vec[_] => vec.zipWithIndex.flatMap { case (value, i) => leaves(value, s"${name}_$i") }.toSeq - case value => Seq(name -> value) - } - val fields = leaves(interface, "io") - Files.writeString( - output.resolve("memory_system_signals.svh"), - fields.map { - case (name, data) => s"logic [${data.getWidth - 1}:0] $name;" - }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("memory_system_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ fields.map { case (name, _) => s".$name($name)" }).mkString( - ",\n" - ) + "\n" - ) - Files.writeString( - output.resolve("memory_system_init.svh"), - fields.collect { - case (name, data) if DataMirror.directionOf(data) == ActualDirection.Input => s"$name = '0;" - }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("memory_system_clocking.svh"), - "clocking sample @(posedge clock);\ndefault input #1step;\ninput reset;\n" + - "" + - "input line_ready,reply_valid,reply_id,reply_data,reply_error;\n" + - fields.map { - case (name, _) => s"input $name;" - }.mkString("\n") + "\nendclocking\n" - ) - } - -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitTile.scala b/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitTile.scala deleted file mode 100644 index 0798016e8..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/EmitTile.scala +++ /dev/null @@ -1,88 +0,0 @@ -package framework.memdomain.verification - -import java.nio.file.{Files, Path} -import chisel3._ -import chisel3.experimental.hierarchy.Instance -import framework.system.{System, SystemParams} -import framework.system.configloader.ExampleTopology -import memcore.bus.chi.{Params => ChiParams} -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.cache.configs.CacheParams -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.interlock.{Params => TrackingParams} - -/** Elaboration of the actual single-tile composition, with explicit verification cache geometry. */ -object EmitTile { - - def apply( - topology: ExampleTopology, - output: Path, - firtoolOptions: Array[String], - build: SystemParams => Instance[System] - ): Unit = { - require(topology.tiles.size == 1, "Tile gate requires one configured tile") - val t = topology.tiles.head - val chi = ChiParams() - val memory = CoherenceParams( - chi, - CacheParams(chi.addressBits, 64, 4, 2, 8, 4, 2), - agents = 2 * t.cores.size, - mshrEntries = 8, - homeId = 64 - ) - val regions = Seq( - PhysicalRegion(BigInt("80000000", 16), BigInt(256) << 20, true, true, true, true, true, true), - PhysicalRegion(BigInt("90000000", 16), BigInt(16) << 20, false, true, true, true, true, true), - PhysicalRegion(BigInt("10000000", 16), BigInt(4096), false, false, true, true, false, false), - PhysicalRegion(BigInt("60000000", 16), BigInt(4096), false, false, true, true, false, false) - ) - val slotsPerClient = 16 - val ddr = memcore.memory.ddr.Params( - chi, - clients = 1, - slotsPerClient = slotsPerClient, - dataBits = 128, - idBits = 4 - ) - var interface: Data = null - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - { - val system = new TileSystem( - topology, - 36, - memory, - RnfParams(chi, cacheLines = 8, banks = 2), - regions, - TrackingParams(addressBits = chi.addressBits), - ddr, - build - ) - interface = system.io - system - }, - args = Array("--target-dir", output.resolve("TileSystem").toString, "--split-verilog"), - firtoolOpts = firtoolOptions - ) - def leaves(data: Data, name: String): Seq[(String, Data)] = data match { - case record: Record => record.elements.toSeq.flatMap { case (n, v) => leaves(v, s"${name}_$n") } - case vector: Vec[_] => vector.zipWithIndex.flatMap { case (value, i) => leaves(value, s"${name}_$i") }.toSeq - case leaf => if (leaf.getWidth > 0) Seq(name -> leaf) else Seq.empty - } - val fields = leaves(interface, "io") - Files.writeString( - output.resolve("tile_system_signals.svh"), - fields.map { - case (name, data) => s"logic [${data.getWidth - 1}:0] $name;" - }.mkString("\n") + "\n" - ) - Files.writeString( - output.resolve("tile_system_ports.svh"), - (Seq(".clock(clock)", ".reset(reset)") ++ fields.map { case (name, _) => s".$name($name)" }).mkString( - ",\n" - ) + "\n" - ) - Files.writeString(output.resolve("tile_system_config.svh"), s"`define TILE_CPU_COUNT ${t.cores.size}\n") - } - -} diff --git a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/TileSystem.scala b/arch/src/main/scala/framework/memdomain/verification/src/main/scala/TileSystem.scala deleted file mode 100644 index c15909e4d..000000000 --- a/arch/src/main/scala/framework/memdomain/verification/src/main/scala/TileSystem.scala +++ /dev/null @@ -1,34 +0,0 @@ -package framework.memdomain.verification - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.configloader.ExampleTopology -import framework.system.{System, SystemParams} -import framework.system.memory.Memory -import framework.memdomain.frontend.mem.dma.DmaStatus -import hier.tile.memory.CoreInterrupts -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion, UncachedRequest, UncachedResponse} -import memcore.memory.interlock.{Params => TrackingParams} -import memcore.memory.ddr.{Params => DdrParams} - -/** Verification names the same production Tile/DDR composition. */ -@instantiable -class TileSystem( - topology: ExampleTopology, - cpuPhysicalBits: Int, - memory: CoherenceParams, - l1: RnfParams, - regions: Seq[PhysicalRegion], - tracking: TrackingParams, - ddr: DdrParams, - build: SystemParams => Instance[System]) - extends Module { - val system: Instance[framework.system.System] = - build(SystemParams(topology, cpuPhysicalBits, memory, l1, regions, tracking, ddr)) - @public - val io = IO(chiselTypeOf(system.dlink)) - io <> system.dlink -} diff --git a/arch/src/main/scala/framework/root/.gitignore b/arch/src/main/scala/framework/root/.gitignore deleted file mode 100644 index 36093f7bf..000000000 --- a/arch/src/main/scala/framework/root/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -out/ -build/ -test_run_dir/ -generated/ diff --git a/arch/src/main/scala/framework/root/README.md b/arch/src/main/scala/framework/root/README.md deleted file mode 100644 index 852c39e73..000000000 --- a/arch/src/main/scala/framework/root/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# root -ROOT: Rigid Organisation, Open Topology diff --git a/arch/src/main/scala/framework/root/root-chip/build.sc b/arch/src/main/scala/framework/root/root-chip/build.sc deleted file mode 100644 index 48c609b05..000000000 --- a/arch/src/main/scala/framework/root/root-chip/build.sc +++ /dev/null @@ -1,43 +0,0 @@ -import mill._ -import mill.scalalib._ - -trait ChiselModule extends SbtModule { - def moduleRoot: os.Path - - override def millSourcePath = moduleRoot - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") -} - -val frameworkRoot = os.pwd / os.up / os.up -val memCoreRoot = frameworkRoot / "mem-core" - -object queue extends SbtModule { - override def millSourcePath = os.pwd / os.up / os.up / "mem-core" / "queue" - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} - -object axis extends ChiselModule { - override def moduleRoot = memCoreRoot / "axis" -} - -object chi extends ChiselModule { - override def moduleDeps = Seq(queue) - override def moduleRoot = memCoreRoot / "chi" -} - -object bank extends ChiselModule { - override def moduleRoot = memCoreRoot / "bank" - override def moduleDeps = Seq(axis) -} - -object root_chip extends ChiselModule { - override def moduleRoot = os.pwd - override def mainClass = Some("hier.chip.mesh.Emit") - override def moduleDeps = Seq(axis, chi, bank) -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/csrc/Cargo.lock b/arch/src/main/scala/framework/root/root-chip/src/csrc/Cargo.lock deleted file mode 100644 index a561fabf0..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "root-chip-reference-model" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/root/root-chip/src/csrc/Cargo.toml b/arch/src/main/scala/framework/root/root-chip/src/csrc/Cargo.toml deleted file mode 100644 index 631fdb590..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/csrc/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name="root-chip-reference-model" -version="0.1.0" -edition="2021" -[lib] -crate-type=["cdylib"] diff --git a/arch/src/main/scala/framework/root/root-chip/src/csrc/src/lib.rs b/arch/src/main/scala/framework/root/root-chip/src/csrc/src/lib.rs deleted file mode 100644 index 56353a299..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/csrc/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod routing; -pub use routing::*; diff --git a/arch/src/main/scala/framework/root/root-chip/src/csrc/src/routing.rs b/arch/src/main/scala/framework/root/root-chip/src/csrc/src/routing.rs deleted file mode 100644 index ef20027e9..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/csrc/src/routing.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[no_mangle] -pub extern "C" fn mesh_route(x: i32, y: i32, dst_x: i32, dst_y: i32) -> i32 { - if dst_x < x { 4 } else if dst_x > x { 3 } else if dst_y < y { 1 } else if dst_y > y { 2 } else { 0 } -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node.f b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node.f deleted file mode 100644 index d33202880..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/codec_interface.sv -@RESOURCES@/bad_node_package.sv --F @RTL@/ChiMeshCodecLoopback/filelist.f -@RESOURCES@/bad_node_tb.sv diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node.toml b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node.toml deleted file mode 100644 index 4a4cc5ad7..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CHI Mesh target NodeID is not placed" diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node_package.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node_package.sv deleted file mode 100644 index eda5ed51f..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node_package.sv +++ /dev/null @@ -1,30 +0,0 @@ -package chi_bad_node_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual chi_codec_if vif; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual chi_codec_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "codec missing") - endfunction - task execute(); - vif.reset = 1; - vif.pause = 0; - vif.valid = 0; - vif.target = 0; - vif.data = 0; - vif.out_ready = 1; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - vif.valid = 1; - repeat (8) @(negedge vif.clock); - `uvm_fatal("MISSING_ASSERTION", "unplaced NodeID was not rejected") - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node_tb.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node_tb.sv deleted file mode 100644 index 54a913b26..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_node_tb.sv +++ /dev/null @@ -1,28 +0,0 @@ -module bad_node_tb; - import uvm_pkg::*; - import chi_bad_node_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_codec_if vif (clock); - ChiMeshCodecLoopback dut ( - .clock(clock), - .reset(vif.reset), - .io_pause(vif.pause), - .io_in_valid(vif.valid), - .io_in_ready(vif.ready), - .io_in_bits_targetNode(vif.target), - .io_in_bits_flit(vif.data), - .io_out_valid(vif.out_valid), - .io_out_ready(vif.out_ready), - .io_out_bits(vif.out_data), - .io_observedFire(vif.chunk_fire), - .io_observed_head(vif.chunk_head), - .io_observed_tail(vif.chunk_tail), - .io_observed_dstX(vif.chunk_x), - .io_observed_dstY(vif.chunk_y) - ); - initial begin - uvm_config_db#(virtual chi_codec_if)::set(null, "uvm_test_top", "vif", vif); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req.f b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req.f deleted file mode 100644 index 7df8ad137..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/fabric_interface.sv -@RESOURCES@/bad_requester_req_package.sv --F @RTL@/ChiMeshRequesterEndpoint/filelist.f -@RESOURCES@/bad_requester_req_tb.sv diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req.toml b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req.toml deleted file mode 100644 index 0cc7e830b..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "CHI requester endpoint received a REQ" diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req_package.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req_package.sv deleted file mode 100644 index 369fc4d1d..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req_package.sv +++ /dev/null @@ -1,57 +0,0 @@ -package chi_bad_requester_req_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual mesh_fabric_if vif; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - timeout = 10us; - if (!uvm_config_db#(virtual mesh_fabric_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "requester mesh interface missing") - endfunction - task execute(); - // Issue-H base-profile REQ is 137 bits, carried as five 32-bit mesh chunks. - bit [159:0] request; - request = '0; - request[4+:7] = 1; // TgtID: populated RN-F, deliberately the wrong role. - request[11+:7] = 2; // SrcID: another populated requester. - request[18+:12] = 7; // TxnID is opaque to transport. - request[50+:7] = 7'h01; // ReadShared. - request[58+:6] = 6; - request[64+:44] = 44'h100; - request[112] = 1; - request[119+:4] = 4'b1100; - request[123] = 1; - request[133] = 1; - vif.reset = 1; - vif.active = 0; - for (int vc = 0; vc < 4; vc++) begin - vif.valid[0][vc] = 0; - vif.data[0][vc] = 0; - vif.head[0][vc] = 0; - vif.tail[0][vc] = 0; - end - repeat (4) @(negedge vif.clock); - vif.reset = 0; - for (int chunk = 0; chunk < 5; chunk++) begin - vif.valid[0][0] = 1; - vif.data[0][0] = request[chunk*32+:32]; - vif.head[0][0] = chunk == 0; - vif.tail[0][0] = chunk == 4; - do @(posedge vif.clock); while (!vif.ready[0][0]); - @(negedge vif.clock); - end - vif.valid[0][0] = 0; - `uvm_info("ROLE_INPUT", - "Delivered all five REQ chunks: SrcID2, TgtID1, REQ VC0, correct HEAD/TAIL", - UVM_LOW) - repeat (8) @(negedge vif.clock); - `uvm_fatal("MISSING_ASSERTION", "requester accepted or discarded a wrong-role REQ") - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req_tb.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req_tb.sv deleted file mode 100644 index 7be33bbea..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/bad_requester_req_tb.sv +++ /dev/null @@ -1,126 +0,0 @@ -module bad_requester_req_tb; - import uvm_pkg::*; - import chi_bad_requester_req_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - mesh_fabric_if vif (clock); - ChiMeshRequesterEndpoint dut ( - .clock(clock), - .reset(vif.reset), - .io_chi_req_valid(1'b0), - .io_chi_req_bits_qos('0), - .io_chi_req_bits_tgtId('0), - .io_chi_req_bits_srcId('0), - .io_chi_req_bits_txnId('0), - .io_chi_req_bits_returnNid('0), - .io_chi_req_bits_stashNidValidEndian('0), - .io_chi_req_bits_returnTxnId('0), - .io_chi_req_bits_opcode('0), - .io_chi_req_bits_multiReq('0), - .io_chi_req_bits_size('0), - .io_chi_req_bits_addr('0), - .io_chi_req_bits_pas('0), - .io_chi_req_bits_likelyShared('0), - .io_chi_req_bits_allowRetry('0), - .io_chi_req_bits_order('0), - .io_chi_req_bits_pCrdType('0), - .io_chi_req_bits_memAttr('0), - .io_chi_req_bits_snpAttr('0), - .io_chi_req_bits_lpid('0), - .io_chi_req_bits_exclSnoopMe('0), - .io_chi_req_bits_expCompAck('0), - .io_chi_req_bits_tagOp('0), - .io_chi_req_bits_traceTag('0), - .io_chi_txRsp_valid(1'b0), - .io_chi_txRsp_bits_qos('0), - .io_chi_txRsp_bits_tgtId('0), - .io_chi_txRsp_bits_srcId('0), - .io_chi_txRsp_bits_txnId('0), - .io_chi_txRsp_bits_opcode('0), - .io_chi_txRsp_bits_respErr('0), - .io_chi_txRsp_bits_resp('0), - .io_chi_txRsp_bits_fwdState('0), - .io_chi_txRsp_bits_cBusy('0), - .io_chi_txRsp_bits_dbid('0), - .io_chi_txRsp_bits_pCrdType('0), - .io_chi_txRsp_bits_tagOp('0), - .io_chi_txRsp_bits_traceTag('0), - .io_chi_txRsp_bits_cacheLineId('0), - .io_chi_txDat_valid(1'b0), - .io_chi_txDat_bits_qos('0), - .io_chi_txDat_bits_tgtId('0), - .io_chi_txDat_bits_srcId('0), - .io_chi_txDat_bits_txnId('0), - .io_chi_txDat_bits_homeNid('0), - .io_chi_txDat_bits_opcode('0), - .io_chi_txDat_bits_respErr('0), - .io_chi_txDat_bits_resp('0), - .io_chi_txDat_bits_dataSource('0), - .io_chi_txDat_bits_dataPull('0), - .io_chi_txDat_bits_cBusy('0), - .io_chi_txDat_bits_dbid('0), - .io_chi_txDat_bits_ccid('0), - .io_chi_txDat_bits_dataId('0), - .io_chi_txDat_bits_cacheLineId('0), - .io_chi_txDat_bits_tagOp('0), - .io_chi_txDat_bits_tag('0), - .io_chi_txDat_bits_tagUpdate('0), - .io_chi_txDat_bits_traceTag('0), - .io_chi_txDat_bits_copyAtHome('0), - .io_chi_txDat_bits_numDat('0), - .io_chi_txDat_bits_replicate('0), - .io_chi_txDat_bits_be('0), - .io_chi_txDat_bits_data('0), - .io_chi_rxRsp_ready(1'b1), - .io_chi_rxDat_ready(1'b1), - .io_chi_snp_ready(1'b1), - .io_meshOut_0_ready(1'b1), - .io_meshIn_0_valid(vif.valid[0][0]), - .io_meshIn_0_ready(vif.ready[0][0]), - .io_meshIn_0_bits_srcX(1'b0), - .io_meshIn_0_bits_srcY(1'b1), - .io_meshIn_0_bits_dstX(1'b1), - .io_meshIn_0_bits_dstY(1'b0), - .io_meshIn_0_bits_vc(2'd0), - .io_meshIn_0_bits_head(vif.head[0][0]), - .io_meshIn_0_bits_tail(vif.tail[0][0]), - .io_meshIn_0_bits_payload(vif.data[0][0]), - .io_meshOut_1_ready(1'b1), - .io_meshIn_1_valid(vif.valid[0][1]), - .io_meshIn_1_ready(vif.ready[0][1]), - .io_meshIn_1_bits_srcX(1'b0), - .io_meshIn_1_bits_srcY(1'b1), - .io_meshIn_1_bits_dstX(1'b1), - .io_meshIn_1_bits_dstY(1'b0), - .io_meshIn_1_bits_vc(2'd1), - .io_meshIn_1_bits_head(vif.head[0][1]), - .io_meshIn_1_bits_tail(vif.tail[0][1]), - .io_meshIn_1_bits_payload(vif.data[0][1]), - .io_meshOut_2_ready(1'b1), - .io_meshIn_2_valid(vif.valid[0][2]), - .io_meshIn_2_ready(vif.ready[0][2]), - .io_meshIn_2_bits_srcX(1'b0), - .io_meshIn_2_bits_srcY(1'b1), - .io_meshIn_2_bits_dstX(1'b1), - .io_meshIn_2_bits_dstY(1'b0), - .io_meshIn_2_bits_vc(2'd2), - .io_meshIn_2_bits_head(vif.head[0][2]), - .io_meshIn_2_bits_tail(vif.tail[0][2]), - .io_meshIn_2_bits_payload(vif.data[0][2]), - .io_meshOut_3_ready(1'b1), - .io_meshIn_3_valid(vif.valid[0][3]), - .io_meshIn_3_ready(vif.ready[0][3]), - .io_meshIn_3_bits_srcX(1'b0), - .io_meshIn_3_bits_srcY(1'b1), - .io_meshIn_3_bits_dstX(1'b1), - .io_meshIn_3_bits_dstY(1'b0), - .io_meshIn_3_bits_vc(2'd3), - .io_meshIn_3_bits_head(vif.head[0][3]), - .io_meshIn_3_bits_tail(vif.tail[0][3]), - .io_meshIn_3_bits_payload(vif.data[0][3]) - ); - initial begin - uvm_config_db#(virtual mesh_fabric_if)::set(null, "uvm_test_top", "vif", vif); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec.el b/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec.el deleted file mode 100644 index 8cc9bd3d6..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec.el +++ /dev/null @@ -1,180 +0,0 @@ -// Format Version: 2 -// Reviewed for 389-bit DAT / 32-bit chunks / populated NodeIDs {1,2,3,64}, coordinated reset. - -CHECKSUM: "3171296859 1964414076" -INSTANCE: codec_tb.dut.tx -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI Mesh target NodeID is not placed" -Block 2 "3889754848" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI Mesh target NodeID is not placed" -Block 3 "2159607100" "$error(\"Assertion failed: CHI Mesh target NodeID is not placed\n at ChiMesh.scala:81 when(io.in.valid) { assert(placed, \\"CHI Mesh target NodeID is not placed\\") }\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI Mesh target NodeID is not placed" -Block 5 "3889754848" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI Mesh target NodeID is not placed" -Block 6 "4027875381" "$fatal;" - -CHECKSUM: "1704682488 3830468848" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet started without head" -Block 2 "4264478414" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet started without head" -Block 3 "1178357294" "$error(\"Assertion failed: CHI mesh packet started without head\n at ChiMesh.scala:133 assert(io.in.bits.head, \\"CHI mesh packet started without head\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet started without head" -Block 5 "4264478414" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet started without head" -Block 6 "520366355" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has a second head" -Block 10 "991107315" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has a second head" -Block 11 "1926190078" "$error(\"Assertion failed: CHI mesh packet has a second head\n at ChiMesh.scala:135 assert(!io.in.bits.head, \\"CHI mesh packet has a second head\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has a second head" -Block 13 "991107315" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has a second head" -Block 14 "1882298097" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has wrong length" -Block 18 "1840269416" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has wrong length" -Block 19 "74803256" "$error(\"Assertion failed: CHI mesh packet has wrong length\n at ChiMesh.scala:137 assert(io.in.bits.tail === (beat === (chunks - 1).U), \\"CHI mesh packet has wrong length\\")\n\");" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has wrong length" -Block 21 "1840269416" "if (1)" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: CHI mesh packet has wrong length" -Block 22 "1745333983" "$fatal;" - -CHECKSUM: "3171296859 4271561183" -INSTANCE: codec_tb.dut.tx -ANNOTATION: "NodeID rejection failure vector; a normally accepted request must target populated NodeIDs {1,2,3,64}; bad_node checks rejection separately" -Condition 1 "690909136" "(io_in_valid & ((~reset)) & ((~_GEN_1[io_in_bits_targetNode]))) 1 -1" (4 "111") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "First accepted fragment has HEAD; shared reset keeps both codec counters aligned, including the reset-disabled failure vector" -Condition 1 "2778753700" "(_GEN & ((~collecting)) & ((~reset)) & ((~io_in_bits_head))) 1 -1" (3 "1101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "First accepted fragment has HEAD; shared reset keeps both codec counters aligned, including the reset-disabled failure vector" -Condition 1 "2778753700" "(_GEN & ((~collecting)) & ((~reset)) & ((~io_in_bits_head))) 1 -1" (5 "1111") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "During collection packetizer HEAD remains false even if paused; shared reset cancels both endpoints together" -Condition 2 "142507406" "(_GEN & collecting & ((~reset)) & io_in_bits_head) 1 -1" (1 "0111") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "During collection packetizer HEAD remains false even if paused; shared reset cancels both endpoints together" -Condition 2 "142507406" "(_GEN & collecting & ((~reset)) & io_in_bits_head) 1 -1" (3 "1101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "During collection packetizer HEAD remains false even if paused; shared reset cancels both endpoints together" -Condition 2 "142507406" "(_GEN & collecting & ((~reset)) & io_in_bits_head) 1 -1" (5 "1111") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Packetizer and receiver counters advance on exactly the same handshake; accepted TAIL always matches beat 12, including coordinated reset" -Condition 3 "1479293268" "(_GEN & ((~reset)) & (io_in_bits_tail != (beat == 4'hc))) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Packetizer and receiver counters advance on exactly the same handshake; accepted TAIL always matches beat 12, including coordinated reset" -Condition 3 "1479293268" "(_GEN & ((~reset)) & (io_in_bits_tail != (beat == 4'hc))) 1 -1" (4 "111") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 10 "291367532" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'b0)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 12 "684678524" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'b1)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 14 "3689188981" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h2)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 16 "3799398757" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h3)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 18 "2405101270" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h4)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 20 "3066583494" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h5)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 22 "3217531303" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h6)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 24 "2253723319" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h7)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 26 "759940555" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h8)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 28 "349795035" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'h9)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 30 "811415760" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'ha)) 1 -1" (2 "101") - -CHECKSUM: "1704682488 2938369825" -INSTANCE: codec_tb.dut.rx -ANNOTATION: "Stored words are beats 0 through 11; legal TAIL can only accompany beat 12, so tail-at-stored-word vector is unreachable" -Condition 32 "961724593" "(_GEN & ((~io_in_bits_tail)) & (beat == 4'hb)) 1 -1" (2 "101") - -CHECKSUM: "2536693556 3895022188" -INSTANCE: codec_tb.dut -ANNOTATION: "Codec local injection coordinate is fixed (0,0) in this declared top" -Toggle io_observed_srcX "net io_observed_srcX" - -CHECKSUM: "2536693556 3895022188" -INSTANCE: codec_tb.dut -ANNOTATION: "Codec local injection coordinate is fixed (0,0) in this declared top" -Toggle io_observed_srcY "net io_observed_srcY" - -CHECKSUM: "2536693556 3895022188" -INSTANCE: codec_tb.dut -ANNOTATION: "This codec verification top transports only CHI DAT, VC2" -Toggle io_observed_vc "net io_observed_vc[1:0]" - -CHECKSUM: "3171296859 1555586989" -INSTANCE: codec_tb.dut.tx -ANNOTATION: "Elaboration-produced coordinate/presence lookup table consists entirely of literal constants; no datapath object selected by generated-name prefix" -Toggle _GEN "net _GEN[127:0]" - -CHECKSUM: "3171296859 1555586989" -INSTANCE: codec_tb.dut.tx -ANNOTATION: "Elaboration-produced coordinate/presence lookup table consists entirely of literal constants; no datapath object selected by generated-name prefix" -Toggle _GEN_0 "net _GEN_0[127:0]" - -CHECKSUM: "3171296859 1555586989" -INSTANCE: codec_tb.dut.tx -ANNOTATION: "Elaboration-produced coordinate/presence lookup table consists entirely of literal constants; no datapath object selected by generated-name prefix" -Toggle _GEN_1 "net _GEN_1[127:0]" - -CHECKSUM: "3171296859 1555586989" -INSTANCE: codec_tb.dut.tx -ANNOTATION: "Only NodeIDs 1,2,3,64 are admitted: accepted target bits 5 through 2 are always zero; bits 6,1,0 remain measured" -Toggle target [5:2] "reg target[6:0]" - -CHECKSUM: "3171296859 1555586989" -INSTANCE: codec_tb.dut.tx -ANNOTATION: "Only the upper 27 bits of the 416-bit zero-extended 389-bit flit are excluded; all useful bits remain checked" -Toggle _io_out_bits_payload_T_1 [415:389] "net _io_out_bits_payload_T_1[415:0]" diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec.f b/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec.f deleted file mode 100644 index d64509fc6..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/codec_interface.sv -@RESOURCES@/codec_package.sv --F @RTL@/ChiMeshCodecLoopback/filelist.f -@RESOURCES@/codec_tb.sv diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_interface.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_interface.sv deleted file mode 100644 index 7848dde2b..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_interface.sv +++ /dev/null @@ -1,8 +0,0 @@ -interface chi_codec_if ( - input logic clock -); - logic reset, pause, valid, ready, out_valid, out_ready; - logic [6:0] target; - logic [388:0] data, out_data; - logic chunk_fire, chunk_head, chunk_tail, chunk_x, chunk_y; -endinterface diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_package.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_package.sv deleted file mode 100644 index 39df7c68a..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_package.sv +++ /dev/null @@ -1,124 +0,0 @@ -package chi_codec_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual chi_codec_if vif; - bit [388:0] expected[$]; - bit [6:0] targets[$]; - int checked = 0; - int cycles = 0; - bit flow = 0; - bit stalled; - bit [388:0] held; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual chi_codec_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "codec missing") - endfunction - task monitor(); - forever begin - @(posedge vif.clock); - if (vif.reset) begin - expected.delete(); - targets.delete(); - stalled = 0; - end else begin - if (vif.valid && vif.ready) begin - expected.push_back(vif.data); - targets.push_back(vif.target); - end - if (vif.chunk_fire) begin - bit [6:0] target; - if (!targets.size()) `uvm_fatal("NODEMAP_EXTRA", "chunk without accepted target") - target = targets[0]; - if (vif.chunk_x !== (target & 1) || vif.chunk_y !== ((target >> 1) & 1)) - `uvm_fatal("NODEMAP", "CHI NodeID coordinate lookup mismatch") - if (vif.chunk_tail) target = targets.pop_front(); - end - if (stalled && (!vif.out_valid || vif.out_data !== held)) - `uvm_fatal("STABLE", "reassembled flit changed under stall") - if (vif.out_valid && vif.out_ready) begin - bit [388:0] golden; - if (!expected.size()) `uvm_fatal("EXTRA", "unexpected reassembled CHI DAT") - golden = expected.pop_front(); - if (vif.out_data !== golden) - `uvm_fatal("DATA", "CHI DAT bit packing/reassembly mismatch"); - checked++; - end - stalled = vif.out_valid && !vif.out_ready; - held = vif.out_data; - end - end - endtask - task flow_control(); - forever begin - @(negedge vif.clock); - cycles++; - if (flow) begin - vif.pause = cycles % 7 < 2; - vif.out_ready = cycles % 11 >= 4; - end else begin - vif.pause = 0; - vif.out_ready = 1; - end - end - endtask - task execute(); - vif.reset = 1; - vif.pause = 0; - vif.valid = 1; - vif.target = 0; - vif.data = 0; - vif.out_ready = 0; - fork - monitor(); - flow_control(); - join_none - repeat (4) @(negedge vif.clock); - vif.reset = 0; - vif.valid = 0; - // Invalid target metadata is legal when no transaction is offered. - for (int bitnum = 0; bitnum < 7; bitnum++) begin - vif.target = 1 << bitnum; - @(negedge vif.clock); - vif.target = 0; - @(negedge vif.clock); - end - vif.reset = 1; - vif.valid = 1; - vif.target = 0; - repeat (2) @(negedge vif.clock); - vif.reset = 0; - vif.valid = 0; - vif.target = 1; - flow = 1; - for (int packet = 0; packet < 32; packet++) begin - vif.target = packet % 4 == 0 ? 64 : packet % 4; - vif.valid = 1; - for (int bitnum = 0; bitnum < 389; bitnum++) vif.data[bitnum] = $urandom_range(0, 1); - do begin - @(posedge vif.clock); - if (vif.ready) begin - @(negedge vif.clock); - break; - end - @(negedge vif.clock); - end while (1); - - end - vif.valid = 0; - flow = 0; - while (checked != 32) @(negedge vif.clock); - repeat (8) @(negedge vif.clock); - `uvm_info( - "CHECKS", - "32 real 389-bit CHI DAT flits serialized to 13 mesh chunks and reassembled under backpressure", - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_tb.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_tb.sv deleted file mode 100644 index cfe70d6d7..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/codec_tb.sv +++ /dev/null @@ -1,28 +0,0 @@ -module codec_tb; - import uvm_pkg::*; - import chi_codec_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - chi_codec_if vif (clock); - ChiMeshCodecLoopback dut ( - .clock(clock), - .reset(vif.reset), - .io_pause(vif.pause), - .io_in_valid(vif.valid), - .io_in_ready(vif.ready), - .io_in_bits_targetNode(vif.target), - .io_in_bits_flit(vif.data), - .io_out_valid(vif.out_valid), - .io_out_ready(vif.out_ready), - .io_out_bits(vif.out_data), - .io_observedFire(vif.chunk_fire), - .io_observed_head(vif.chunk_head), - .io_observed_tail(vif.chunk_tail), - .io_observed_dstX(vif.chunk_x), - .io_observed_dstY(vif.chunk_y) - ); - initial begin - uvm_config_db#(virtual chi_codec_if)::set(null, "uvm_test_top", "vif", vif); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/fabric_interface.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/fabric_interface.sv deleted file mode 100644 index c855cd1ab..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/fabric_interface.sv +++ /dev/null @@ -1,11 +0,0 @@ -interface mesh_fabric_if ( - input logic clock -); - logic reset, active; - logic valid[5][4], ready[5][4], out_valid[5][4], out_ready[5][4]; - logic [1:0] x[5][4], y[5][4], src_x[5][4], src_y[5][4], vc[5][4]; - logic head[5][4], tail[5][4]; - logic [31:0] data[5][4], out_data[5][4]; - logic [1:0] out_src_x[5][4], out_src_y[5][4]; - logic out_head[5][4], out_tail[5][4]; -endinterface diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/network.f b/arch/src/main/scala/framework/root/root-chip/src/main/resources/network.f deleted file mode 100644 index fc8e0c60e..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/network.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/fabric_interface.sv -@RESOURCES@/network_package.sv --F @RTL@/MeshCreditNetwork/filelist.f -@RESOURCES@/network_tb.sv diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/network_package.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/network_package.sv deleted file mode 100644 index 58dc740c9..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/network_package.sv +++ /dev/null @@ -1,386 +0,0 @@ -package mesh_network_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual mesh_fabric_if vif; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - timeout = 1ms; - if (!uvm_config_db#(virtual mesh_fabric_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "fabric interface missing") - endfunction - typedef bit [33:0] golden_beat; - typedef golden_beat beat_queue[$]; - task route_matrix(); - beat_queue expected[4][4][4]; - int submitted[4][4]; - int checked = 0; - bit done; - bit [31:0] payload; - for (int i = 0; i < 4; i++) for (int v = 0; v < 4; v++) submitted[i][v] = 0; - for (int tick = 0; tick < 60000; tick++) begin - for (int source = 0; source < 4; source++) - for (int v = 0; v < 4; v++) begin - int index, destination, pattern; - index = submitted[source][v]; - destination = index / (64 * 3); - pattern = (index / 3) % 64; - payload = 32'h1 << (pattern % 32); - if (pattern >= 32) payload = ~payload; - payload=payload^(source*32'h9e3779b9)^(v*32'h7f4a7c15)^(destination*32'h85ebca6b); - vif.valid[source][v] = index < 768; - vif.x[source][v] = destination % 2; - vif.y[source][v] = destination / 2; - vif.data[source][v] = payload; - vif.head[source][v] = index % 3 == 0; - vif.tail[source][v] = index % 3 == 2; - vif.out_ready[source][v] = tick % 17 >= ((source + v) % 5); - end - @(posedge vif.clock); - for (int source = 0; source < 4; source++) - for (int v = 0; v < 4; v++) - if (vif.valid[source][v] && vif.ready[source][v]) begin - int destination; - destination = vif.y[source][v] * 2 + vif.x[source][v]; - expected[source][destination][v].push_back( - {vif.tail[source][v], vif.head[source][v], vif.data[source][v]}); - submitted[source][v]++; - end - for (int destination = 0; destination < 4; destination++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[destination][v] && vif.out_ready[destination][v]) begin - int source; - golden_beat golden; - source = vif.out_src_y[destination][v] * 2 + vif.out_src_x[destination][v]; - if (source >= 4 || !expected[source][destination][v].size()) - `uvm_fatal("MATRIX_EXTRA", "unmatched node/VC output") - golden = expected[source][destination][v].pop_front(); - if({vif.out_tail[destination][v],vif.out_head[destination][v],vif.out_data[destination][v]}!==golden) - `uvm_fatal("MATRIX_ORDER", "all-node source/destination/VC FIFO mismatch") - checked++; - end - @(negedge vif.clock); - done = checked == 12288; - if (done) begin - for (int source = 0; source < 4; source++) - for (int destination = 0; destination < 4; destination++) - for (int v = 0; v < 4; v++) - if (expected[source][destination][v].size()) - `uvm_fatal("MATRIX_PENDING", "unmatched accepted all-node request"); - break; - end - if (tick == 59999) - `uvm_fatal("MATRIX_TIMEOUT", "all-node walking/covering payload failed to drain") - end - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) begin - vif.valid[i][v] = 0; - vif.out_ready[i][v] = 1; - end - repeat (8) @(negedge vif.clock); - `uvm_info( - "CHECKS", - "12288 all-node/all-VC walking+complement payload flits compared by per(source,destination,VC) FIFO including packet boundaries", - UVM_LOW) - endtask - task isolated_paths(); - isolated_reset(); - // Reset cancels packet ownership; local input metadata is unconstrained while reset is asserted. - vif.reset = 1; - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) begin - vif.valid[i][v] = 1; - vif.head[i][v] = 0; - vif.tail[i][v] = 1; - vif.x[i][v] = i % 2; - vif.y[i][v] = i / 2; - vif.out_ready[i][v] = 1; - end - repeat (3) @(negedge vif.clock); - for (int i = 0; i < 4; i++) for (int v = 0; v < 4; v++) vif.valid[i][v] = 0; - vif.active = 0; - repeat (3) @(negedge vif.clock); - vif.reset = 0; - vif.active = 1; - repeat (20) begin - @(posedge vif.clock); - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[i][v]) - `uvm_fatal("RESET_FINAL", "reset-time metadata survived as a packet"); - @(negedge vif.clock); - end - // Exercise each physical route independently: fill its buffers, pause packet - // bodies while an owner is locked, then resume and compare every accepted flit. - for (int source = 0; source < 4; source++) - for (int path = 0; path < 6; path++) - for (int vc = 0; vc < 4; vc++) begin - int destination; - int submitted = 0, checked = 0, next_cycle = 0; - golden_beat expected[$]; - bit done = 0; - destination = path < 4 ? path : path == 4 ? (source ^ 3) : (source ^ 1); - for (int node = 0; node < 4; node++) - for (int v = 0; v < 4; v++) begin - vif.valid[node][v] = 0; - vif.out_ready[node][v] = 1; - end - for (int tick = 0; tick < 300; tick++) begin - vif.valid[source][vc] = submitted < 13 && tick >= next_cycle; - vif.x[source][vc] = vif.valid[source][vc] ? destination % 2 : (destination ^ 3) % 2; - vif.y[source][vc] = vif.valid[source][vc] ? destination / 2 : (destination ^ 3) / 2; - vif.head[source][vc] = submitted % 4 == 0; - vif.tail[source][vc] = submitted == 12 || submitted % 4 == 3; - vif.data[source][vc]=32'h61900000|(source<<16)|(destination<<12)|(vc<<8)|submitted; - vif.out_ready[destination][vc]=path>=4 || submitted>=12 || (tick>=45 && tick%13>=3); - @(posedge vif.clock); - if (vif.valid[source][vc] && vif.ready[source][vc]) begin - expected.push_back({vif.tail[source][vc], vif.head[source][vc], vif.data[source][vc] - }); - submitted++; - next_cycle = tick + 1 + (submitted == 12 ? 30 : submitted % 4 == 1 ? 7 : 0); - end - for (int node = 0; node < 4; node++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[node][v] && vif.out_ready[node][v]) begin - golden_beat golden; - if (node != destination || v != vc || !expected.size()) - `uvm_fatal("PATH_EXTRA", "isolated path delivered unexpected flit") - golden = expected.pop_front(); - if({vif.out_tail[node][v],vif.out_head[node][v],vif.out_data[node][v]}!==golden || vif.out_src_x[node][v]!=source%2 || vif.out_src_y[node][v]!=source/2) - `uvm_fatal("PATH_DATA", "isolated packet path mismatch") - checked++; - end - @(negedge vif.clock); - if (checked == 13) begin - done = 1; - break; - end - end - if (!done || expected.size()) - `uvm_fatal("PATH_TIMEOUT", "isolated route failed to drain") - vif.valid[source][vc] = 0; - vif.out_ready[destination][vc] = 1; - repeat (8) @(negedge vif.clock); - end - `uvm_info( - "CHECKS", - "1248 isolated-path packet flits checked with owner gaps and per-route FIFO saturation", - UVM_LOW) - endtask - task isolated_reset(); - // Reset each physical horizontal link/VC after a packet head, before its tail. - for (int source = 0; source < 4; source++) - for (int vc = 0; vc < 4; vc++) begin - repeat (12) @(negedge vif.clock); - vif.valid[source][vc] = 1; - vif.head[source][vc] = 1; - vif.tail[source][vc] = 0; - vif.x[source][vc] = (source ^ 1) % 2; - vif.y[source][vc] = source / 2; - vif.data[source][vc] = 32'hca110000 | (source << 8) | vc; - @(posedge vif.clock); - if (!vif.ready[source][vc]) - `uvm_fatal("RESET_CREDIT", "isolated reset launch did not have its initialized credit") - @(negedge vif.clock); - vif.reset = 1; - vif.active = 0; - vif.valid[source][vc] = 0; - repeat (3) @(negedge vif.clock); - vif.reset = 0; - vif.active = 1; - repeat (12) begin - @(posedge vif.clock); - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[i][v]) - `uvm_fatal("RESET_ISOLATED", "cancelled per-VC flit survived reset"); - @(negedge vif.clock); - end - end - endtask - task reset_sweep(); - for (int step = 2; step < 13; step++) begin - for (int source = 0; source < 4; source++) - for (int v = 0; v < 4; v++) begin - vif.valid[source][v] = 1; - vif.x[source][v] = (source ^ 3) % 2; - vif.y[source][v] = (source ^ 3) / 2; - vif.head[source][v] = 1; - vif.tail[source][v] = 1; - vif.data[source][v] = 32'hcc000000 | (source << 8) | v; - vif.out_ready[source][v] = 0; - end - repeat (step) @(negedge vif.clock); - vif.reset = 1; - vif.active = 0; - for (int source = 0; source < 4; source++) - for (int v = 0; v < 4; v++) vif.valid[source][v] = 0; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - vif.active = 1; - for (int source = 0; source < 4; source++) - for (int v = 0; v < 4; v++) vif.out_ready[source][v] = 1; - repeat (20) begin - @(posedge vif.clock); - for (int source = 0; source < 4; source++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[source][v]) - `uvm_fatal("RESET_STAGE", "cancelled pipeline-stage flit survived reset"); - @(negedge vif.clock); - end - end - endtask - task execute(); - int sent[4][4], received[4][4]; - bit complete; - bit stalled[4][4]; - bit [31:0] held[4][4]; - vif.reset = 1; - vif.active = 0; - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) begin - sent[i][v] = 0; - received[i][v] = 0; - stalled[i][v] = 0; - vif.valid[i][v] = 0; - vif.x[i][v] = 1; - vif.y[i][v] = 1; - vif.out_ready[i][v] = 1; - end - repeat (4) @(negedge vif.clock); - vif.reset = 0; - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) begin - vif.valid[i][v] = 1; - vif.head[i][v] = 1; - vif.tail[i][v] = 1; - vif.data[i][v] = 32'habcd; - end - repeat (4) begin - @(posedge vif.clock); - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) - if (vif.ready[i][v] || vif.out_valid[i][v]) - `uvm_fatal("INACTIVE", "inactive local path accepted traffic"); - @(negedge vif.clock); - end - for (int i = 0; i < 4; i++) for (int v = 0; v < 4; v++) vif.valid[i][v] = 0; - vif.active = 1; - vif.out_ready[3][0] = 0; - for (int tick = 0; tick < 1000; tick++) begin - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) begin - vif.valid[i][v] = sent[i][v] < 24; - vif.head[i][v] = sent[i][v] % 3 == 0; - vif.tail[i][v] = sent[i][v] % 3 == 2; - vif.data[i][v] = (i << 24) | (v << 16) | sent[i][v]; - end - @(posedge vif.clock); - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) if (vif.valid[i][v] && vif.ready[i][v]) sent[i][v]++; - for (int destination = 0; destination < 4; destination++) - for (int v = 0; v < 4; v++) begin - if(stalled[destination][v]&&(!vif.out_valid[destination][v]||vif.out_data[destination][v]!==held[destination][v])) - `uvm_fatal("STABLE", "network VC changed under stall") - stalled[destination][v] = vif.out_valid[destination][v] && !vif.out_ready[destination][v]; - held[destination][v] = vif.out_data[destination][v]; - end - for (int destination = 0; destination < 4; destination++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[destination][v] && vif.out_ready[destination][v]) begin - int source, seq; - source = vif.out_data[destination][v] >> 24; - seq = vif.out_data[destination][v] & 65535; - if(destination!=3||source>=4||seq!=received[source][v]||((vif.out_data[destination][v]>>16)&255)!=v) - `uvm_fatal("ORDER", "network lost, duplicated or reordered source/VC flit") - received[source][v]++; - end - @(negedge vif.clock); - if (tick == 350) begin - for (int i = 0; i < 4; i++) - for (int v = 1; v < 4; v++) - if (received[i][v] != 24) `uvm_fatal("ISOLATION", "REQ blocked unrelated CHI protocol VC") - vif.out_ready[3][0] = 1; - end - complete = 1; - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) if (received[i][v] != 24) complete = 0; - if (complete) break; - if (tick == 999) `uvm_fatal("TIMEOUT", "multi-hop network did not drain") - end - for (int i = 0; i < 4; i++) for (int v = 0; v < 4; v++) vif.valid[i][v] = 0; - repeat (10) @(negedge vif.clock); - // Reset cancels buffered traffic and starts a fresh per-VC credit epoch. - vif.out_ready[3][0] = 0; - vif.valid[0][0] = 1; - vif.head[0][0] = 1; - vif.tail[0][0] = 1; - vif.data[0][0] = 32'hdeadbeef; - do @(posedge vif.clock); while (!vif.ready[0][0]); - @(negedge vif.clock); - vif.valid[0][0] = 0; - repeat (12) @(negedge vif.clock); - vif.reset = 1; - vif.active = 0; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - vif.active = 1; - vif.out_ready[3][0] = 1; - repeat (20) begin - @(posedge vif.clock); - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[i][v]) `uvm_fatal("RESET", "pre-reset flit survived coordinated reset"); - @(negedge vif.clock); - end - vif.valid[0][0] = 1; - vif.data[0][0] = 32'h5eedca11; - do @(posedge vif.clock); while (!vif.ready[0][0]); - @(negedge vif.clock); - vif.valid[0][0] = 0; - do @(posedge vif.clock); while (!vif.out_valid[3][0]); - if (vif.out_data[3][0] !== 32'h5eedca11) `uvm_fatal("RESET", "fresh epoch payload mismatch"); - @(negedge vif.clock); - repeat (8) @(negedge vif.clock); - reset_sweep(); - route_matrix(); - isolated_paths(); - isolated_reset(); - // Reset cancels packet ownership; local input metadata is unconstrained while reset is asserted. - vif.reset = 1; - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) begin - vif.valid[i][v] = 1; - vif.head[i][v] = 0; - vif.tail[i][v] = 1; - vif.x[i][v] = i % 2; - vif.y[i][v] = i / 2; - vif.out_ready[i][v] = 1; - end - repeat (3) @(negedge vif.clock); - for (int i = 0; i < 4; i++) for (int v = 0; v < 4; v++) vif.valid[i][v] = 0; - vif.active = 0; - repeat (3) @(negedge vif.clock); - vif.reset = 0; - vif.active = 1; - repeat (20) begin - @(posedge vif.clock); - for (int i = 0; i < 4; i++) - for (int v = 0; v < 4; v++) - if (vif.out_valid[i][v]) - `uvm_fatal("RESET_FINAL", "reset-time metadata survived as a packet"); - @(negedge vif.clock); - end - `uvm_info( - "CHECKS", - "384 multi-hop packet flits checked by source/VC; RSP/DAT/SNP finish while REQ blocked", - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/network_tb.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/network_tb.sv deleted file mode 100644 index e5c897afc..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/network_tb.sv +++ /dev/null @@ -1,288 +0,0 @@ -module network_tb; - import uvm_pkg::*; - import mesh_network_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - mesh_fabric_if vif (clock); - MeshCreditNetwork dut ( - .clock(clock), - .reset(vif.reset), - .io_active(vif.active), - .io_localIn_0_0_valid(vif.valid[0][0]), - .io_localIn_0_0_ready(vif.ready[0][0]), - .io_localIn_0_0_bits_dstX(vif.x[0][0]), - .io_localIn_0_0_bits_dstY(vif.y[0][0]), - .io_localIn_0_0_bits_head(vif.head[0][0]), - .io_localIn_0_0_bits_tail(vif.tail[0][0]), - .io_localIn_0_0_bits_payload(vif.data[0][0]), - .io_localIn_0_0_bits_srcX(1'd0), - .io_localIn_0_0_bits_srcY(1'd0), - .io_localIn_0_0_bits_vc(2'd0), - .io_localOut_0_0_valid(vif.out_valid[0][0]), - .io_localOut_0_0_ready(vif.out_ready[0][0]), - .io_localOut_0_0_bits_payload(vif.out_data[0][0]), - .io_localOut_0_0_bits_srcX(vif.out_src_x[0][0]), - .io_localOut_0_0_bits_srcY(vif.out_src_y[0][0]), - .io_localOut_0_0_bits_head(vif.out_head[0][0]), - .io_localOut_0_0_bits_tail(vif.out_tail[0][0]), - .io_localIn_0_1_valid(vif.valid[0][1]), - .io_localIn_0_1_ready(vif.ready[0][1]), - .io_localIn_0_1_bits_dstX(vif.x[0][1]), - .io_localIn_0_1_bits_dstY(vif.y[0][1]), - .io_localIn_0_1_bits_head(vif.head[0][1]), - .io_localIn_0_1_bits_tail(vif.tail[0][1]), - .io_localIn_0_1_bits_payload(vif.data[0][1]), - .io_localIn_0_1_bits_srcX(1'd0), - .io_localIn_0_1_bits_srcY(1'd0), - .io_localIn_0_1_bits_vc(2'd1), - .io_localOut_0_1_valid(vif.out_valid[0][1]), - .io_localOut_0_1_ready(vif.out_ready[0][1]), - .io_localOut_0_1_bits_payload(vif.out_data[0][1]), - .io_localOut_0_1_bits_srcX(vif.out_src_x[0][1]), - .io_localOut_0_1_bits_srcY(vif.out_src_y[0][1]), - .io_localOut_0_1_bits_head(vif.out_head[0][1]), - .io_localOut_0_1_bits_tail(vif.out_tail[0][1]), - .io_localIn_0_2_valid(vif.valid[0][2]), - .io_localIn_0_2_ready(vif.ready[0][2]), - .io_localIn_0_2_bits_dstX(vif.x[0][2]), - .io_localIn_0_2_bits_dstY(vif.y[0][2]), - .io_localIn_0_2_bits_head(vif.head[0][2]), - .io_localIn_0_2_bits_tail(vif.tail[0][2]), - .io_localIn_0_2_bits_payload(vif.data[0][2]), - .io_localIn_0_2_bits_srcX(1'd0), - .io_localIn_0_2_bits_srcY(1'd0), - .io_localIn_0_2_bits_vc(2'd2), - .io_localOut_0_2_valid(vif.out_valid[0][2]), - .io_localOut_0_2_ready(vif.out_ready[0][2]), - .io_localOut_0_2_bits_payload(vif.out_data[0][2]), - .io_localOut_0_2_bits_srcX(vif.out_src_x[0][2]), - .io_localOut_0_2_bits_srcY(vif.out_src_y[0][2]), - .io_localOut_0_2_bits_head(vif.out_head[0][2]), - .io_localOut_0_2_bits_tail(vif.out_tail[0][2]), - .io_localIn_0_3_valid(vif.valid[0][3]), - .io_localIn_0_3_ready(vif.ready[0][3]), - .io_localIn_0_3_bits_dstX(vif.x[0][3]), - .io_localIn_0_3_bits_dstY(vif.y[0][3]), - .io_localIn_0_3_bits_head(vif.head[0][3]), - .io_localIn_0_3_bits_tail(vif.tail[0][3]), - .io_localIn_0_3_bits_payload(vif.data[0][3]), - .io_localIn_0_3_bits_srcX(1'd0), - .io_localIn_0_3_bits_srcY(1'd0), - .io_localIn_0_3_bits_vc(2'd3), - .io_localOut_0_3_valid(vif.out_valid[0][3]), - .io_localOut_0_3_ready(vif.out_ready[0][3]), - .io_localOut_0_3_bits_payload(vif.out_data[0][3]), - .io_localOut_0_3_bits_srcX(vif.out_src_x[0][3]), - .io_localOut_0_3_bits_srcY(vif.out_src_y[0][3]), - .io_localOut_0_3_bits_head(vif.out_head[0][3]), - .io_localOut_0_3_bits_tail(vif.out_tail[0][3]), - .io_localIn_1_0_valid(vif.valid[1][0]), - .io_localIn_1_0_ready(vif.ready[1][0]), - .io_localIn_1_0_bits_dstX(vif.x[1][0]), - .io_localIn_1_0_bits_dstY(vif.y[1][0]), - .io_localIn_1_0_bits_head(vif.head[1][0]), - .io_localIn_1_0_bits_tail(vif.tail[1][0]), - .io_localIn_1_0_bits_payload(vif.data[1][0]), - .io_localIn_1_0_bits_srcX(1'd1), - .io_localIn_1_0_bits_srcY(1'd0), - .io_localIn_1_0_bits_vc(2'd0), - .io_localOut_1_0_valid(vif.out_valid[1][0]), - .io_localOut_1_0_ready(vif.out_ready[1][0]), - .io_localOut_1_0_bits_payload(vif.out_data[1][0]), - .io_localOut_1_0_bits_srcX(vif.out_src_x[1][0]), - .io_localOut_1_0_bits_srcY(vif.out_src_y[1][0]), - .io_localOut_1_0_bits_head(vif.out_head[1][0]), - .io_localOut_1_0_bits_tail(vif.out_tail[1][0]), - .io_localIn_1_1_valid(vif.valid[1][1]), - .io_localIn_1_1_ready(vif.ready[1][1]), - .io_localIn_1_1_bits_dstX(vif.x[1][1]), - .io_localIn_1_1_bits_dstY(vif.y[1][1]), - .io_localIn_1_1_bits_head(vif.head[1][1]), - .io_localIn_1_1_bits_tail(vif.tail[1][1]), - .io_localIn_1_1_bits_payload(vif.data[1][1]), - .io_localIn_1_1_bits_srcX(1'd1), - .io_localIn_1_1_bits_srcY(1'd0), - .io_localIn_1_1_bits_vc(2'd1), - .io_localOut_1_1_valid(vif.out_valid[1][1]), - .io_localOut_1_1_ready(vif.out_ready[1][1]), - .io_localOut_1_1_bits_payload(vif.out_data[1][1]), - .io_localOut_1_1_bits_srcX(vif.out_src_x[1][1]), - .io_localOut_1_1_bits_srcY(vif.out_src_y[1][1]), - .io_localOut_1_1_bits_head(vif.out_head[1][1]), - .io_localOut_1_1_bits_tail(vif.out_tail[1][1]), - .io_localIn_1_2_valid(vif.valid[1][2]), - .io_localIn_1_2_ready(vif.ready[1][2]), - .io_localIn_1_2_bits_dstX(vif.x[1][2]), - .io_localIn_1_2_bits_dstY(vif.y[1][2]), - .io_localIn_1_2_bits_head(vif.head[1][2]), - .io_localIn_1_2_bits_tail(vif.tail[1][2]), - .io_localIn_1_2_bits_payload(vif.data[1][2]), - .io_localIn_1_2_bits_srcX(1'd1), - .io_localIn_1_2_bits_srcY(1'd0), - .io_localIn_1_2_bits_vc(2'd2), - .io_localOut_1_2_valid(vif.out_valid[1][2]), - .io_localOut_1_2_ready(vif.out_ready[1][2]), - .io_localOut_1_2_bits_payload(vif.out_data[1][2]), - .io_localOut_1_2_bits_srcX(vif.out_src_x[1][2]), - .io_localOut_1_2_bits_srcY(vif.out_src_y[1][2]), - .io_localOut_1_2_bits_head(vif.out_head[1][2]), - .io_localOut_1_2_bits_tail(vif.out_tail[1][2]), - .io_localIn_1_3_valid(vif.valid[1][3]), - .io_localIn_1_3_ready(vif.ready[1][3]), - .io_localIn_1_3_bits_dstX(vif.x[1][3]), - .io_localIn_1_3_bits_dstY(vif.y[1][3]), - .io_localIn_1_3_bits_head(vif.head[1][3]), - .io_localIn_1_3_bits_tail(vif.tail[1][3]), - .io_localIn_1_3_bits_payload(vif.data[1][3]), - .io_localIn_1_3_bits_srcX(1'd1), - .io_localIn_1_3_bits_srcY(1'd0), - .io_localIn_1_3_bits_vc(2'd3), - .io_localOut_1_3_valid(vif.out_valid[1][3]), - .io_localOut_1_3_ready(vif.out_ready[1][3]), - .io_localOut_1_3_bits_payload(vif.out_data[1][3]), - .io_localOut_1_3_bits_srcX(vif.out_src_x[1][3]), - .io_localOut_1_3_bits_srcY(vif.out_src_y[1][3]), - .io_localOut_1_3_bits_head(vif.out_head[1][3]), - .io_localOut_1_3_bits_tail(vif.out_tail[1][3]), - .io_localIn_2_0_valid(vif.valid[2][0]), - .io_localIn_2_0_ready(vif.ready[2][0]), - .io_localIn_2_0_bits_dstX(vif.x[2][0]), - .io_localIn_2_0_bits_dstY(vif.y[2][0]), - .io_localIn_2_0_bits_head(vif.head[2][0]), - .io_localIn_2_0_bits_tail(vif.tail[2][0]), - .io_localIn_2_0_bits_payload(vif.data[2][0]), - .io_localIn_2_0_bits_srcX(1'd0), - .io_localIn_2_0_bits_srcY(1'd1), - .io_localIn_2_0_bits_vc(2'd0), - .io_localOut_2_0_valid(vif.out_valid[2][0]), - .io_localOut_2_0_ready(vif.out_ready[2][0]), - .io_localOut_2_0_bits_payload(vif.out_data[2][0]), - .io_localOut_2_0_bits_srcX(vif.out_src_x[2][0]), - .io_localOut_2_0_bits_srcY(vif.out_src_y[2][0]), - .io_localOut_2_0_bits_head(vif.out_head[2][0]), - .io_localOut_2_0_bits_tail(vif.out_tail[2][0]), - .io_localIn_2_1_valid(vif.valid[2][1]), - .io_localIn_2_1_ready(vif.ready[2][1]), - .io_localIn_2_1_bits_dstX(vif.x[2][1]), - .io_localIn_2_1_bits_dstY(vif.y[2][1]), - .io_localIn_2_1_bits_head(vif.head[2][1]), - .io_localIn_2_1_bits_tail(vif.tail[2][1]), - .io_localIn_2_1_bits_payload(vif.data[2][1]), - .io_localIn_2_1_bits_srcX(1'd0), - .io_localIn_2_1_bits_srcY(1'd1), - .io_localIn_2_1_bits_vc(2'd1), - .io_localOut_2_1_valid(vif.out_valid[2][1]), - .io_localOut_2_1_ready(vif.out_ready[2][1]), - .io_localOut_2_1_bits_payload(vif.out_data[2][1]), - .io_localOut_2_1_bits_srcX(vif.out_src_x[2][1]), - .io_localOut_2_1_bits_srcY(vif.out_src_y[2][1]), - .io_localOut_2_1_bits_head(vif.out_head[2][1]), - .io_localOut_2_1_bits_tail(vif.out_tail[2][1]), - .io_localIn_2_2_valid(vif.valid[2][2]), - .io_localIn_2_2_ready(vif.ready[2][2]), - .io_localIn_2_2_bits_dstX(vif.x[2][2]), - .io_localIn_2_2_bits_dstY(vif.y[2][2]), - .io_localIn_2_2_bits_head(vif.head[2][2]), - .io_localIn_2_2_bits_tail(vif.tail[2][2]), - .io_localIn_2_2_bits_payload(vif.data[2][2]), - .io_localIn_2_2_bits_srcX(1'd0), - .io_localIn_2_2_bits_srcY(1'd1), - .io_localIn_2_2_bits_vc(2'd2), - .io_localOut_2_2_valid(vif.out_valid[2][2]), - .io_localOut_2_2_ready(vif.out_ready[2][2]), - .io_localOut_2_2_bits_payload(vif.out_data[2][2]), - .io_localOut_2_2_bits_srcX(vif.out_src_x[2][2]), - .io_localOut_2_2_bits_srcY(vif.out_src_y[2][2]), - .io_localOut_2_2_bits_head(vif.out_head[2][2]), - .io_localOut_2_2_bits_tail(vif.out_tail[2][2]), - .io_localIn_2_3_valid(vif.valid[2][3]), - .io_localIn_2_3_ready(vif.ready[2][3]), - .io_localIn_2_3_bits_dstX(vif.x[2][3]), - .io_localIn_2_3_bits_dstY(vif.y[2][3]), - .io_localIn_2_3_bits_head(vif.head[2][3]), - .io_localIn_2_3_bits_tail(vif.tail[2][3]), - .io_localIn_2_3_bits_payload(vif.data[2][3]), - .io_localIn_2_3_bits_srcX(1'd0), - .io_localIn_2_3_bits_srcY(1'd1), - .io_localIn_2_3_bits_vc(2'd3), - .io_localOut_2_3_valid(vif.out_valid[2][3]), - .io_localOut_2_3_ready(vif.out_ready[2][3]), - .io_localOut_2_3_bits_payload(vif.out_data[2][3]), - .io_localOut_2_3_bits_srcX(vif.out_src_x[2][3]), - .io_localOut_2_3_bits_srcY(vif.out_src_y[2][3]), - .io_localOut_2_3_bits_head(vif.out_head[2][3]), - .io_localOut_2_3_bits_tail(vif.out_tail[2][3]), - .io_localIn_3_0_valid(vif.valid[3][0]), - .io_localIn_3_0_ready(vif.ready[3][0]), - .io_localIn_3_0_bits_dstX(vif.x[3][0]), - .io_localIn_3_0_bits_dstY(vif.y[3][0]), - .io_localIn_3_0_bits_head(vif.head[3][0]), - .io_localIn_3_0_bits_tail(vif.tail[3][0]), - .io_localIn_3_0_bits_payload(vif.data[3][0]), - .io_localIn_3_0_bits_srcX(1'd1), - .io_localIn_3_0_bits_srcY(1'd1), - .io_localIn_3_0_bits_vc(2'd0), - .io_localOut_3_0_valid(vif.out_valid[3][0]), - .io_localOut_3_0_ready(vif.out_ready[3][0]), - .io_localOut_3_0_bits_payload(vif.out_data[3][0]), - .io_localOut_3_0_bits_srcX(vif.out_src_x[3][0]), - .io_localOut_3_0_bits_srcY(vif.out_src_y[3][0]), - .io_localOut_3_0_bits_head(vif.out_head[3][0]), - .io_localOut_3_0_bits_tail(vif.out_tail[3][0]), - .io_localIn_3_1_valid(vif.valid[3][1]), - .io_localIn_3_1_ready(vif.ready[3][1]), - .io_localIn_3_1_bits_dstX(vif.x[3][1]), - .io_localIn_3_1_bits_dstY(vif.y[3][1]), - .io_localIn_3_1_bits_head(vif.head[3][1]), - .io_localIn_3_1_bits_tail(vif.tail[3][1]), - .io_localIn_3_1_bits_payload(vif.data[3][1]), - .io_localIn_3_1_bits_srcX(1'd1), - .io_localIn_3_1_bits_srcY(1'd1), - .io_localIn_3_1_bits_vc(2'd1), - .io_localOut_3_1_valid(vif.out_valid[3][1]), - .io_localOut_3_1_ready(vif.out_ready[3][1]), - .io_localOut_3_1_bits_payload(vif.out_data[3][1]), - .io_localOut_3_1_bits_srcX(vif.out_src_x[3][1]), - .io_localOut_3_1_bits_srcY(vif.out_src_y[3][1]), - .io_localOut_3_1_bits_head(vif.out_head[3][1]), - .io_localOut_3_1_bits_tail(vif.out_tail[3][1]), - .io_localIn_3_2_valid(vif.valid[3][2]), - .io_localIn_3_2_ready(vif.ready[3][2]), - .io_localIn_3_2_bits_dstX(vif.x[3][2]), - .io_localIn_3_2_bits_dstY(vif.y[3][2]), - .io_localIn_3_2_bits_head(vif.head[3][2]), - .io_localIn_3_2_bits_tail(vif.tail[3][2]), - .io_localIn_3_2_bits_payload(vif.data[3][2]), - .io_localIn_3_2_bits_srcX(1'd1), - .io_localIn_3_2_bits_srcY(1'd1), - .io_localIn_3_2_bits_vc(2'd2), - .io_localOut_3_2_valid(vif.out_valid[3][2]), - .io_localOut_3_2_ready(vif.out_ready[3][2]), - .io_localOut_3_2_bits_payload(vif.out_data[3][2]), - .io_localOut_3_2_bits_srcX(vif.out_src_x[3][2]), - .io_localOut_3_2_bits_srcY(vif.out_src_y[3][2]), - .io_localOut_3_2_bits_head(vif.out_head[3][2]), - .io_localOut_3_2_bits_tail(vif.out_tail[3][2]), - .io_localIn_3_3_valid(vif.valid[3][3]), - .io_localIn_3_3_ready(vif.ready[3][3]), - .io_localIn_3_3_bits_dstX(vif.x[3][3]), - .io_localIn_3_3_bits_dstY(vif.y[3][3]), - .io_localIn_3_3_bits_head(vif.head[3][3]), - .io_localIn_3_3_bits_tail(vif.tail[3][3]), - .io_localIn_3_3_bits_payload(vif.data[3][3]), - .io_localIn_3_3_bits_srcX(1'd1), - .io_localIn_3_3_bits_srcY(1'd1), - .io_localIn_3_3_bits_vc(2'd3), - .io_localOut_3_3_valid(vif.out_valid[3][3]), - .io_localOut_3_3_ready(vif.out_ready[3][3]), - .io_localOut_3_3_bits_payload(vif.out_data[3][3]), - .io_localOut_3_3_bits_srcX(vif.out_src_x[3][3]), - .io_localOut_3_3_bits_srcY(vif.out_src_y[3][3]), - .io_localOut_3_3_bits_head(vif.out_head[3][3]), - .io_localOut_3_3_bits_tail(vif.out_tail[3][3]) - ); - initial begin - uvm_config_db#(virtual mesh_fabric_if)::set(null, "uvm_test_top", "vif", vif); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router.el b/arch/src/main/scala/framework/root/root-chip/src/main/resources/router.el deleted file mode 100644 index 8962495b3..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router.el +++ /dev/null @@ -1,502 +0,0 @@ -// Format Version: 2 -// Reviewed exact objects; native URG checksums/signatures and bit/transition scope preserved. - -CHECKSUM: "1802735833 964344264" -INSTANCE: router_tb.dut -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Mesh flit arrived on wrong VC port" -Block 2 "2419507893" "if (1)" -Block 3 "3668136291" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 5 "2419507893" "if (1)" -Block 6 "2132654405" "$fatal;" -Block 26 "1224552289" "if (1)" -Block 27 "409298446" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 29 "1224552289" "if (1)" -Block 30 "3185282600" "$fatal;" -Block 50 "1934849908" "if (1)" -Block 51 "1166025150" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 53 "1934849908" "if (1)" -Block 54 "3762211224" "$fatal;" -Block 74 "65211867" "if (1)" -Block 75 "4087705017" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 77 "65211867" "if (1)" -Block 78 "1444644255" "$fatal;" -Block 98 "1600740343" "if (1)" -Block 99 "2104806949" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 101 "1600740343" "if (1)" -Block 102 "3637126659" "$fatal;" -Block 122 "2168220930" "if (1)" -Block 123 "2564218234" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 125 "2168220930" "if (1)" -Block 126 "1030239580" "$fatal;" -Block 146 "3814076770" "if (1)" -Block 147 "2257455454" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 149 "3814076770" "if (1)" -Block 150 "590406008" "$fatal;" -Block 170 "619444456" "if (1)" -Block 171 "938657885" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 173 "619444456" "if (1)" -Block 174 "2454462587" "$fatal;" -Block 194 "2288343919" "if (1)" -Block 195 "2897478908" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 197 "2288343919" "if (1)" -Block 198 "151710938" "$fatal;" -Block 218 "2768801095" "if (1)" -Block 219 "3659715001" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 221 "2768801095" "if (1)" -Block 222 "2140944799" "$fatal;" -Block 242 "147697006" "if (1)" -Block 243 "253650898" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 245 "147697006" "if (1)" -Block 246 "2862649332" "$fatal;" -Block 266 "1789391118" "if (1)" -Block 267 "289846262" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 269 "1789391118" "if (1)" -Block 270 "3036175312" "$fatal;" -Block 290 "3616373105" "if (1)" -Block 291 "2174420298" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 293 "3616373105" "if (1)" -Block 294 "606465388" "$fatal;" -Block 314 "2063850230" "if (1)" -Block 315 "450742763" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 317 "2063850230" "if (1)" -Block 318 "3210944973" "$fatal;" -Block 338 "1450052830" "if (1)" -Block 339 "1816861870" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 341 "1450052830" "if (1)" -Block 342 "3388339336" "$fatal;" -Block 362 "171706947" "if (1)" -Block 363 "2243434851" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 365 "171706947" "if (1)" -Block 366 "537325893" "$fatal;" -Block 386 "1750488611" "if (1)" -Block 387 "2615201095" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 389 "1750488611" "if (1)" -Block 390 "1046357345" "$fatal;" -Block 410 "2951454633" "if (1)" -Block 411 "714869828" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 413 "2951454633" "if (1)" -Block 414 "2401428578" "$fatal;" -Block 434 "56966190" "if (1)" -Block 435 "2983903461" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 437 "56966190" "if (1)" -Block 438 "342108355" "$fatal;" -Block 458 "772336134" "if (1)" -Block 459 "3343652256" "$error(\"Assertion failed: Mesh flit arrived on wrong VC port\n at Mesh.scala:101 assert(input.bits.vc === vc.U, \\"Mesh flit arrived on wrong VC port\\")\n\");" -Block 461 "772336134" "if (1)" -Block 462 "1660091782" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Mesh destination outside topology" -Block 10 "3524853549" "if (1)" -Block 11 "3668459727" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 13 "3524853549" "if (1)" -Block 14 "127688172" "$fatal;" -Block 34 "624517000" "if (1)" -Block 35 "572034051" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 37 "624517000" "if (1)" -Block 38 "4281110816" "$fatal;" -Block 58 "2799459455" "if (1)" -Block 59 "65291866" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 61 "2799459455" "if (1)" -Block 62 "3738196857" "$fatal;" -Block 82 "2466509191" "if (1)" -Block 83 "537394510" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 85 "2466509191" "if (1)" -Block 86 "4248035437" "$fatal;" -Block 106 "243241476" "if (1)" -Block 107 "429169505" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 109 "243241476" "if (1)" -Block 110 "3298854466" "$fatal;" -Block 130 "2931492635" "if (1)" -Block 131 "1613403034" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 133 "2931492635" "if (1)" -Block 134 "3172896441" "$fatal;" -Block 154 "1503307710" "if (1)" -Block 155 "2560253782" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 157 "1503307710" "if (1)" -Block 158 "1169048181" "$fatal;" -Block 178 "3473843463" "if (1)" -Block 179 "3677482553" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 181 "3473843463" "if (1)" -Block 182 "101053210" "$fatal;" -Block 202 "4208003327" "if (1)" -Block 203 "4174533933" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 205 "4208003327" "if (1)" -Block 206 "635848718" "$fatal;" -Block 226 "1739418492" "if (1)" -Block 227 "3242275586" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 229 "1739418492" "if (1)" -Block 230 "477441569" "$fatal;" -Block 250 "3594899581" "if (1)" -Block 251 "4186328242" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 253 "3594899581" "if (1)" -Block 254 "615666065" "$fatal;" -Block 274 "560230616" "if (1)" -Block 275 "20349054" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 277 "560230616" "if (1)" -Block 278 "3691158877" "$fatal;" -Block 298 "1461373702" "if (1)" -Block 299 "4071637836" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 301 "1461373702" "if (1)" -Block 302 "797204079" "$fatal;" -Block 322 "1657104126" "if (1)" -Block 323 "3511941208" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 325 "1657104126" "if (1)" -Block 326 "208135547" "$fatal;" -Block 346 "4290315645" "if (1)" -Block 347 "3904937591" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 349 "4290315645" "if (1)" -Block 350 "905216852" "$fatal;" -Block 370 "1602064482" "if (1)" -Block 371 "2440992396" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 373 "1602064482" "if (1)" -Block 374 "1279953839" "$fatal;" -Block 394 "2824614087" "if (1)" -Block 395 "1775131200" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 397 "2824614087" "if (1)" -Block 398 "3036301155" "$fatal;" -Block 418 "1053307518" "if (1)" -Block 419 "711322415" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 421 "1053307518" "if (1)" -Block 422 "4149343756" "$fatal;" -Block 442 "186097542" "if (1)" -Block 443 "159797307" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 445 "186097542" "if (1)" -Block 446 "3568454936" "$fatal;" -Block 466 "2523659269" "if (1)" -Block 467 "806696468" "$error(\"Assertion failed: Mesh destination outside topology\n at Mesh.scala:102 assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, \\"Mesh destination outside topology\\")\n\");" -Block 469 "2523659269" "if (1)" -Block 470 "3978373943" "$fatal;" -ANNOTATION: "Assertion-failure diagnostic only; assertion remains enabled: Mesh packet body without ownership" -Block 18 "1479776755" "if (1)" -Block 19 "3696211781" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 21 "1479776755" "if (1)" -Block 22 "2048665000" "$fatal;" -Block 42 "1003602289" "if (1)" -Block 43 "1541976043" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 45 "1003602289" "if (1)" -Block 46 "4256906502" "$fatal;" -Block 66 "1944574667" "if (1)" -Block 67 "1172873431" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 69 "1944574667" "if (1)" -Block 70 "3820696122" "$fatal;" -Block 90 "4087040474" "if (1)" -Block 91 "2177760972" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 93 "4087040474" "if (1)" -Block 94 "664693793" "$fatal;" -Block 114 "2364201432" "if (1)" -Block 115 "965280461" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 117 "2364201432" "if (1)" -Block 118 "2681901088" "$fatal;" -Block 138 "575965992" "if (1)" -Block 139 "662191603" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 141 "575965992" "if (1)" -Block 142 "2167123742" "$fatal;" -Block 162 "2780275615" "if (1)" -Block 163 "1919327153" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 165 "2780275615" "if (1)" -Block 166 "3560245596" "$fatal;" -Block 186 "3620965993" "if (1)" -Block 187 "829912204" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 189 "3620965993" "if (1)" -Block 190 "2535777889" "$fatal;" -Block 210 "1583181966" "if (1)" -Block 211 "1087336062" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 213 "1583181966" "if (1)" -Block 214 "3869047955" "$fatal;" -Block 234 "1962668773" "if (1)" -Block 235 "588553913" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 237 "1962668773" "if (1)" -Block 238 "2236025940" "$fatal;" -Block 258 "1046265680" "if (1)" -Block 259 "3929352611" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 261 "1046265680" "if (1)" -Block 262 "1281801038" "$fatal;" -Block 282 "1608998260" "if (1)" -Block 283 "764998666" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 285 "1608998260" "if (1)" -Block 286 "2345363175" "$fatal;" -Block 306 "574513879" "if (1)" -Block 307 "1302867378" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 309 "574513879" "if (1)" -Block 310 "3959144287" "$fatal;" -Block 330 "1689751111" "if (1)" -Block 331 "1315827385" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 333 "1689751111" "if (1)" -Block 334 "3896409172" "$fatal;" -Block 354 "2345279621" "if (1)" -Block 355 "1865992791" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 357 "2345279621" "if (1)" -Block 358 "3379255482" "$fatal;" -Block 378 "3602701937" "if (1)" -Block 379 "664587898" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 381 "3602701937" "if (1)" -Block 382 "2177850519" "$fatal;" -Block 402 "2372097411" "if (1)" -Block 403 "739169264" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 405 "2372097411" "if (1)" -Block 406 "2321367325" "$fatal;" -Block 426 "2353113249" "if (1)" -Block 427 "2459213730" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 429 "2353113249" "if (1)" -Block 430 "885460303" "$fatal;" -Block 450 "1647330288" "if (1)" -Block 451 "289923572" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 453 "1647330288" "if (1)" -Block 454 "3071570713" "$fatal;" -Block 474 "3712945988" "if (1)" -Block 475 "2331052331" "$error(\"Assertion failed: Mesh packet body without ownership\n at Mesh.scala:107 assert(owned, \\"Mesh packet body without ownership\\")\n\");" -Block 477 "3712945988" "if (1)" -Block 478 "748854214" "$fatal;" - -CHECKSUM: "1802735833 3870451462" -INSTANCE: router_tb.dut -ANNOTATION: "Input READY implies selected input VALID: READY requires output VALID and this input selected, and output VALID is selected input VALID" -Condition 1 "3092155993" "(io_in_0_0_ready_0 & io_in_0_0_valid) 1 -1" (2 "10") -Condition 3 "3850033337" "(io_in_1_0_ready_0 & io_in_1_0_valid) 1 -1" (2 "10") -Condition 5 "1752407670" "(io_in_2_0_ready_0 & io_in_2_0_valid) 1 -1" (2 "10") -Condition 7 "1756263612" "(io_in_3_0_ready_0 & io_in_3_0_valid) 1 -1" (2 "10") -Condition 9 "3996989854" "(io_in_4_0_ready_0 & io_in_4_0_valid) 1 -1" (2 "10") -Condition 11 "2231740155" "(io_in_0_1_ready_0 & io_in_0_1_valid) 1 -1" (2 "10") -Condition 13 "3627151387" "(io_in_1_1_ready_0 & io_in_1_1_valid) 1 -1" (2 "10") -Condition 15 "1429780180" "(io_in_2_1_ready_0 & io_in_2_1_valid) 1 -1" (2 "10") -Condition 17 "4175394999" "(io_in_3_1_ready_0 & io_in_3_1_valid) 1 -1" (2 "10") -Condition 19 "2118923669" "(io_in_4_1_ready_0 & io_in_4_1_valid) 1 -1" (2 "10") -Condition 21 "3932843002" "(io_in_0_2_ready_0 & io_in_0_2_valid) 1 -1" (2 "10") -Condition 23 "3076406554" "(io_in_1_2_ready_0 & io_in_1_2_valid) 1 -1" (2 "10") -Condition 25 "978811861" "(io_in_2_2_ready_0 & io_in_2_2_valid) 1 -1" (2 "10") -Condition 27 "1711358307" "(io_in_3_2_ready_0 & io_in_3_2_valid) 1 -1" (2 "10") -Condition 29 "3767667777" "(io_in_4_2_ready_0 & io_in_4_2_valid) 1 -1" (2 "10") -Condition 31 "2343211812" "(io_in_0_3_ready_0 & io_in_0_3_valid) 1 -1" (2 "10") -Condition 33 "3600729540" "(io_in_1_3_ready_0 & io_in_1_3_valid) 1 -1" (2 "10") -Condition 35 "1536656139" "(io_in_2_3_ready_0 & io_in_2_3_valid) 1 -1" (2 "10") -Condition 37 "4134554984" "(io_in_3_3_ready_0 & io_in_3_3_valid) 1 -1" (2 "10") -Condition 39 "1893924938" "(io_in_4_3_ready_0 & io_in_4_3_valid) 1 -1" (2 "10") -ANNOTATION: "Accepted flit must match the explicit VC port; this vector is the VC-mismatch assertion failure" -Condition 41 "4070260215" "(_GEN_76 & ((|io_in_0_0_bits_vc))) 1 -1" (3 "11") -Condition 60 "3671467409" "(_GEN_78 & ((|io_in_1_0_bits_vc))) 1 -1" (3 "11") -Condition 79 "3127070248" "(_GEN_80 & ((|io_in_2_0_bits_vc))) 1 -1" (3 "11") -Condition 98 "4085502683" "(_GEN_82 & ((|io_in_3_0_bits_vc))) 1 -1" (3 "11") -Condition 117 "2704255070" "(_GEN_84 & ((|io_in_4_0_bits_vc))) 1 -1" (3 "11") -Condition 136 "4209678335" "(_GEN_86 & (io_in_0_1_bits_vc != 2'b1)) 1 -1" (3 "11") -Condition 156 "3794511059" "(_GEN_88 & (io_in_1_1_bits_vc != 2'b1)) 1 -1" (3 "11") -Condition 176 "2865133531" "(_GEN_90 & (io_in_2_1_bits_vc != 2'b1)) 1 -1" (3 "11") -Condition 196 "725272976" "(_GEN_92 & (io_in_3_1_bits_vc != 2'b1)) 1 -1" (3 "11") -Condition 216 "2694844148" "(_GEN_94 & (io_in_4_1_bits_vc != 2'b1)) 1 -1" (3 "11") -Condition 236 "4160240023" "(_GEN_96 & (io_in_0_2_bits_vc != 2'h2)) 1 -1" (3 "11") -Condition 256 "4013492923" "(_GEN_98 & (io_in_1_2_bits_vc != 2'h2)) 1 -1" (3 "11") -Condition 276 "1238580357" "(_GEN_100 & (io_in_2_2_bits_vc != 2'h2)) 1 -1" (3 "11") -Condition 296 "3358556878" "(_GEN_102 & (io_in_3_2_bits_vc != 2'h2)) 1 -1" (3 "11") -Condition 316 "1135953322" "(_GEN_104 & (io_in_4_2_bits_vc != 2'h2)) 1 -1" (3 "11") -Condition 336 "2633490133" "(_GEN_106 & (io_in_0_3_bits_vc != 2'h3)) 1 -1" (3 "11") -Condition 356 "2218192377" "(_GEN_108 & (io_in_1_3_bits_vc != 2'h3)) 1 -1" (3 "11") -Condition 376 "3436920561" "(_GEN_110 & (io_in_2_3_bits_vc != 2'h3)) 1 -1" (3 "11") -Condition 396 "1294434490" "(_GEN_112 & (io_in_3_3_bits_vc != 2'h3)) 1 -1" (3 "11") -Condition 416 "3334293470" "(_GEN_114 & (io_in_4_3_bits_vc != 2'h3)) 1 -1" (3 "11") -ANNOTATION: "Accepted destination must be inside the declared 3x3 topology; this vector is the range-assertion failure" -Condition 42 "1924237034" "(_GEN_76 & ( ~ ((io_in_0_0_bits_dstX != 2'h3) & (io_in_0_0_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 61 "1403403801" "(_GEN_78 & ( ~ ((io_in_1_0_bits_dstX != 2'h3) & (io_in_1_0_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 80 "254368599" "(_GEN_80 & ( ~ ((io_in_2_0_bits_dstX != 2'h3) & (io_in_2_0_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 99 "1476507923" "(_GEN_82 & ( ~ ((io_in_3_0_bits_dstX != 2'h3) & (io_in_3_0_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 118 "2495079852" "(_GEN_84 & ( ~ ((io_in_4_0_bits_dstX != 2'h3) & (io_in_4_0_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 138 "26791419" "(_GEN_86 & ( ~ ((io_in_0_1_bits_dstX != 2'h3) & (io_in_0_1_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 158 "546272520" "(_GEN_88 & ( ~ ((io_in_1_1_bits_dstX != 2'h3) & (io_in_1_1_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 178 "2228200946" "(_GEN_90 & ( ~ ((io_in_2_1_bits_dstX != 2'h3) & (io_in_2_1_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 198 "3555149750" "(_GEN_92 & ( ~ ((io_in_3_1_bits_dstX != 2'h3) & (io_in_3_1_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 218 "525409033" "(_GEN_94 & ( ~ ((io_in_4_1_bits_dstX != 2'h3) & (io_in_4_1_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 238 "3572317890" "(_GEN_96 & ( ~ ((io_in_0_2_bits_dstX != 2'h3) & (io_in_0_2_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 258 "4126836273" "(_GEN_98 & ( ~ ((io_in_1_2_bits_dstX != 2'h3) & (io_in_1_2_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 278 "3777107480" "(_GEN_100 & ( ~ ((io_in_2_2_bits_dstX != 2'h3) & (io_in_2_2_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 298 "3054171228" "(_GEN_102 & ( ~ ((io_in_3_2_bits_dstX != 2'h3) & (io_in_3_2_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 318 "2059184355" "(_GEN_104 & ( ~ ((io_in_4_2_bits_dstX != 2'h3) & (io_in_4_2_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 338 "4019421364" "(_GEN_106 & ( ~ ((io_in_0_3_bits_dstX != 2'h3) & (io_in_0_3_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 358 "3464771655" "(_GEN_108 & ( ~ ((io_in_1_3_bits_dstX != 2'h3) & (io_in_1_3_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 378 "1791289533" "(_GEN_110 & ( ~ ((io_in_2_3_bits_dstX != 2'h3) & (io_in_2_3_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 398 "1038879481" "(_GEN_112 & ( ~ ((io_in_3_3_bits_dstX != 2'h3) & (io_in_3_3_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -Condition 418 "4049229382" "(_GEN_114 & ( ~ ((io_in_4_3_bits_dstX != 2'h3) & (io_in_4_3_bits_dstY != 2'h3)) )) 1 -1" (3 "11") -ANNOTATION: "Accepted packet body must retain output/VC ownership; this vector is the body-without-ownership assertion failure" -Condition 47 "3947639187" "(_GEN_75 & ((~io_in_0_0_bits_head)) & ((~reset)) & ( ~ ((_owned_T & owners_0_3 & (owners_0_4 == 3'b0)) | (_owned_T_4 & owners_1_3 & (owners_1_4 == 3'b0)) | (_owned_T_8 & owners_2_3 & (owners_2_4 == 3'b0)) | (_owned_T_12 & owners_3_3 & (owners_3_4 == 3'b0)) | (_owned_T_16 & owners_4_3 & (owners_4_4 == 3'b0))) )) 1 -1" (5 "1111") -Condition 66 "1972422343" "(_GEN_77 & ((~io_in_1_0_bits_head)) & ((~reset)) & ( ~ ((_owned_T_23 & owners_0_3 & (owners_0_4 == 3'b1)) | (_owned_T_27 & owners_1_3 & (owners_1_4 == 3'b1)) | (_owned_T_31 & owners_2_3 & (owners_2_4 == 3'b1)) | (_owned_T_35 & owners_3_3 & (owners_3_4 == 3'b1)) | (_owned_T_39 & owners_4_3 & (owners_4_4 == 3'b1))) )) 1 -1" (5 "1111") -Condition 85 "3114650597" "(_GEN_79 & ((~io_in_2_0_bits_head)) & ((~reset)) & ( ~ ((_owned_T_46 & owners_0_3 & (owners_0_4 == 3'h2)) | (_owned_T_50 & owners_1_3 & (owners_1_4 == 3'h2)) | (_owned_T_54 & owners_2_3 & (owners_2_4 == 3'h2)) | (_owned_T_58 & owners_3_3 & (owners_3_4 == 3'h2)) | (_owned_T_62 & owners_4_3 & (owners_4_4 == 3'h2))) )) 1 -1" (5 "1111") -Condition 104 "3746486545" "(_GEN_81 & ((~io_in_3_0_bits_head)) & ((~reset)) & ( ~ ((_owned_T_69 & owners_0_3 & (owners_0_4 == 3'h3)) | (_owned_T_73 & owners_1_3 & (owners_1_4 == 3'h3)) | (_owned_T_77 & owners_2_3 & (owners_2_4 == 3'h3)) | (_owned_T_81 & owners_3_3 & (owners_3_4 == 3'h3)) | (_owned_T_85 & owners_4_3 & (owners_4_4 == 3'h3))) )) 1 -1" (5 "1111") -Condition 123 "1611394691" "(_GEN_83 & ((~io_in_4_0_bits_head)) & ((~reset)) & ( ~ ((_owned_T_92 & owners_0_3 & (owners_0_4 == 3'h4)) | (_owned_T_96 & owners_1_3 & (owners_1_4 == 3'h4)) | (_owned_T_100 & owners_2_3 & (owners_2_4 == 3'h4)) | (_owned_T_104 & owners_3_3 & (owners_3_4 == 3'h4)) | (_owned_T_108 & owners_4_3 & (owners_4_4 == 3'h4))) )) 1 -1" (5 "1111") -Condition 143 "1457370676" "(_GEN_85 & ((~io_in_0_1_bits_head)) & ((~reset)) & ( ~ ((_owned_T_115 & owners_0_3_1 & (owners_0_4_1 == 3'b0)) | (_owned_T_119 & owners_1_3_1 & (owners_1_4_1 == 3'b0)) | (_owned_T_123 & owners_2_3_1 & (owners_2_4_1 == 3'b0)) | (_owned_T_127 & owners_3_3_1 & (owners_3_4_1 == 3'b0)) | (_owned_T_131 & owners_4_3_1 & (owners_4_4_1 == 3'b0))) )) 1 -1" (5 "1111") -Condition 163 "4245520960" "(_GEN_87 & ((~io_in_1_1_bits_head)) & ((~reset)) & ( ~ ((_owned_T_138 & owners_0_3_1 & (owners_0_4_1 == 3'b1)) | (_owned_T_142 & owners_1_3_1 & (owners_1_4_1 == 3'b1)) | (_owned_T_146 & owners_2_3_1 & (owners_2_4_1 == 3'b1)) | (_owned_T_150 & owners_3_3_1 & (owners_3_4_1 == 3'b1)) | (_owned_T_154 & owners_4_3_1 & (owners_4_4_1 == 3'b1))) )) 1 -1" (5 "1111") -Condition 183 "486659993" "(_GEN_89 & ((~io_in_2_1_bits_head)) & ((~reset)) & ( ~ ((_owned_T_161 & owners_0_3_1 & (owners_0_4_1 == 3'h2)) | (_owned_T_165 & owners_1_3_1 & (owners_1_4_1 == 3'h2)) | (_owned_T_169 & owners_2_3_1 & (owners_2_4_1 == 3'h2)) | (_owned_T_173 & owners_3_3_1 & (owners_3_4_1 == 3'h2)) | (_owned_T_177 & owners_4_3_1 & (owners_4_4_1 == 3'h2))) )) 1 -1" (5 "1111") -Condition 203 "1083250543" "(_GEN_91 & ((~io_in_3_1_bits_head)) & ((~reset)) & ( ~ ((_owned_T_184 & owners_0_3_1 & (owners_0_4_1 == 3'h3)) | (_owned_T_188 & owners_1_3_1 & (owners_1_4_1 == 3'h3)) | (_owned_T_192 & owners_2_3_1 & (owners_2_4_1 == 3'h3)) | (_owned_T_196 & owners_3_3_1 & (owners_3_4_1 == 3'h3)) | (_owned_T_200 & owners_4_3_1 & (owners_4_4_1 == 3'h3))) )) 1 -1" (5 "1111") -Condition 223 "355838805" "(_GEN_93 & ((~io_in_4_1_bits_head)) & ((~reset)) & ( ~ ((_owned_T_207 & owners_0_3_1 & (owners_0_4_1 == 3'h4)) | (_owned_T_211 & owners_1_3_1 & (owners_1_4_1 == 3'h4)) | (_owned_T_215 & owners_2_3_1 & (owners_2_4_1 == 3'h4)) | (_owned_T_219 & owners_3_3_1 & (owners_3_4_1 == 3'h4)) | (_owned_T_223 & owners_4_3_1 & (owners_4_4_1 == 3'h4))) )) 1 -1" (5 "1111") -Condition 243 "3816971184" "(_GEN_95 & ((~io_in_0_2_bits_head)) & ((~reset)) & ( ~ ((_owned_T_230 & owners_0_3_2 & (owners_0_4_2 == 3'b0)) | (_owned_T_234 & owners_1_3_2 & (owners_1_4_2 == 3'b0)) | (_owned_T_238 & owners_2_3_2 & (owners_2_4_2 == 3'b0)) | (_owned_T_242 & owners_3_3_2 & (owners_3_4_2 == 3'b0)) | (_owned_T_246 & owners_4_3_2 & (owners_4_4_2 == 3'b0))) )) 1 -1" (5 "1111") -Condition 263 "226870005" "(_GEN_97 & ((~io_in_1_2_bits_head)) & ((~reset)) & ( ~ ((_owned_T_253 & owners_0_3_2 & (owners_0_4_2 == 3'b1)) | (_owned_T_257 & owners_1_3_2 & (owners_1_4_2 == 3'b1)) | (_owned_T_261 & owners_2_3_2 & (owners_2_4_2 == 3'b1)) | (_owned_T_265 & owners_3_3_2 & (owners_3_4_2 == 3'b1)) | (_owned_T_269 & owners_4_3_2 & (owners_4_4_2 == 3'b1))) )) 1 -1" (5 "1111") -Condition 283 "2142325087" "(_GEN_99 & ((~io_in_2_2_bits_head)) & ((~reset)) & ( ~ ((_owned_T_276 & owners_0_3_2 & (owners_0_4_2 == 3'h2)) | (_owned_T_280 & owners_1_3_2 & (owners_1_4_2 == 3'h2)) | (_owned_T_284 & owners_2_3_2 & (owners_2_4_2 == 3'h2)) | (_owned_T_288 & owners_3_3_2 & (owners_3_4_2 == 3'h2)) | (_owned_T_292 & owners_4_3_2 & (owners_4_4_2 == 3'h2))) )) 1 -1" (5 "1111") -Condition 303 "168759350" "(_GEN_101 & ((~io_in_3_2_bits_head)) & ((~reset)) & ( ~ ((_owned_T_299 & owners_0_3_2 & (owners_0_4_2 == 3'h3)) | (_owned_T_303 & owners_1_3_2 & (owners_1_4_2 == 3'h3)) | (_owned_T_307 & owners_2_3_2 & (owners_2_4_2 == 3'h3)) | (_owned_T_311 & owners_3_3_2 & (owners_3_4_2 == 3'h3)) | (_owned_T_315 & owners_4_3_2 & (owners_4_4_2 == 3'h3))) )) 1 -1" (5 "1111") -Condition 323 "1234006216" "(_GEN_103 & ((~io_in_4_2_bits_head)) & ((~reset)) & ( ~ ((_owned_T_322 & owners_0_3_2 & (owners_0_4_2 == 3'h4)) | (_owned_T_326 & owners_1_3_2 & (owners_1_4_2 == 3'h4)) | (_owned_T_330 & owners_2_3_2 & (owners_2_4_2 == 3'h4)) | (_owned_T_334 & owners_3_3_2 & (owners_3_4_2 == 3'h4)) | (_owned_T_338 & owners_4_3_2 & (owners_4_4_2 == 3'h4))) )) 1 -1" (5 "1111") -Condition 343 "3686430807" "(_GEN_105 & ((~io_in_0_3_bits_head)) & ((~reset)) & ( ~ ((_owned_T_345 & owners_0_3_3 & (owners_0_4_3 == 3'b0)) | (_owned_T_349 & owners_1_3_3 & (owners_1_4_3 == 3'b0)) | (_owned_T_353 & owners_2_3_3 & (owners_2_4_3 == 3'b0)) | (_owned_T_357 & owners_3_3_3 & (owners_3_4_3 == 3'b0)) | (_owned_T_361 & owners_4_3_3 & (owners_4_4_3 == 3'b0))) )) 1 -1" (5 "1111") -Condition 363 "3447495620" "(_GEN_107 & ((~io_in_1_3_bits_head)) & ((~reset)) & ( ~ ((_owned_T_368 & owners_0_3_3 & (owners_0_4_3 == 3'b1)) | (_owned_T_372 & owners_1_3_3 & (owners_1_4_3 == 3'b1)) | (_owned_T_376 & owners_2_3_3 & (owners_2_4_3 == 3'b1)) | (_owned_T_380 & owners_3_3_3 & (owners_3_4_3 == 3'b1)) | (_owned_T_384 & owners_4_3_3 & (owners_4_4_3 == 3'b1))) )) 1 -1" (5 "1111") -Condition 383 "2754832380" "(_GEN_109 & ((~io_in_2_3_bits_head)) & ((~reset)) & ( ~ ((_owned_T_391 & owners_0_3_3 & (owners_0_4_3 == 3'h2)) | (_owned_T_395 & owners_1_3_3 & (owners_1_4_3 == 3'h2)) | (_owned_T_399 & owners_2_3_3 & (owners_2_4_3 == 3'h2)) | (_owned_T_403 & owners_3_3_3 & (owners_3_4_3 == 3'h2)) | (_owned_T_407 & owners_4_3_3 & (owners_4_4_3 == 3'h2))) )) 1 -1" (5 "1111") -Condition 403 "2964337691" "(_GEN_111 & ((~io_in_3_3_bits_head)) & ((~reset)) & ( ~ ((_owned_T_414 & owners_0_3_3 & (owners_0_4_3 == 3'h3)) | (_owned_T_418 & owners_1_3_3 & (owners_1_4_3 == 3'h3)) | (_owned_T_422 & owners_2_3_3 & (owners_2_4_3 == 3'h3)) | (_owned_T_426 & owners_3_3_3 & (owners_3_4_3 == 3'h3)) | (_owned_T_430 & owners_4_3_3 & (owners_4_4_3 == 3'h3))) )) 1 -1" (5 "1111") -Condition 423 "531256894" "(_GEN_113 & ((~io_in_4_3_bits_head)) & ((~reset)) & ( ~ ((_owned_T_437 & owners_0_3_3 & (owners_0_4_3 == 3'h4)) | (_owned_T_441 & owners_1_3_3 & (owners_1_4_3 == 3'h4)) | (_owned_T_445 & owners_2_3_3 & (owners_2_4_3 == 3'h4)) | (_owned_T_449 & owners_3_3_3 & (owners_3_4_3 == 3'h4)) | (_owned_T_453 & owners_4_3_3 & (owners_4_4_3 == 3'h4))) )) 1 -1" (5 "1111") -ANNOTATION: "Tail/capture subexpression is evaluated only in output-FIRE branch; FIRE needs READY while capture needs !READY, so capture cannot be asserted in this branch" -Condition 518 "2956550500" "(_GEN_8[owners_0_2] & ((~_owners_T_3))) 1 -1" (2 "10") -Condition 522 "708526675" "(_GEN_8[owners_1_2] & ((~_owners_T_8))) 1 -1" (2 "10") -Condition 526 "1710407644" "(_GEN_8[owners_2_2] & ((~_owners_T_13))) 1 -1" (2 "10") -Condition 530 "855132190" "(_GEN_8[owners_3_2] & ((~_owners_T_18))) 1 -1" (2 "10") -Condition 534 "1570867619" "(_GEN_8[owners_4_2] & ((~_owners_T_23))) 1 -1" (2 "10") -Condition 538 "3016253282" "(_GEN_27[owners_0_2_1] & ((~_owners_T_28))) 1 -1" (2 "10") -Condition 542 "2555122219" "(_GEN_27[owners_1_2_1] & ((~_owners_T_33))) 1 -1" (2 "10") -Condition 546 "2458396873" "(_GEN_27[owners_2_2_1] & ((~_owners_T_38))) 1 -1" (2 "10") -Condition 550 "3558935641" "(_GEN_27[owners_3_2_1] & ((~_owners_T_43))) 1 -1" (2 "10") -Condition 554 "64083681" "(_GEN_27[owners_4_2_1] & ((~_owners_T_48))) 1 -1" (2 "10") -Condition 558 "1343425993" "(_GEN_46[owners_0_2_2] & ((~_owners_T_53))) 1 -1" (2 "10") -Condition 562 "721917084" "(_GEN_46[owners_1_2_2] & ((~_owners_T_58))) 1 -1" (2 "10") -Condition 566 "1790583670" "(_GEN_46[owners_2_2_2] & ((~_owners_T_63))) 1 -1" (2 "10") -Condition 570 "4116221512" "(_GEN_46[owners_3_2_2] & ((~_owners_T_68))) 1 -1" (2 "10") -Condition 574 "3731495116" "(_GEN_46[owners_4_2_2] & ((~_owners_T_73))) 1 -1" (2 "10") -Condition 578 "834226852" "(_GEN_65[owners_0_2_3] & ((~_owners_T_78))) 1 -1" (2 "10") -Condition 582 "3439348919" "(_GEN_65[owners_1_2_3] & ((~_owners_T_83))) 1 -1" (2 "10") -Condition 586 "3351541333" "(_GEN_65[owners_2_2_3] & ((~_owners_T_88))) 1 -1" (2 "10") -Condition 590 "2733104725" "(_GEN_65[owners_3_2_3] & ((~_owners_T_93))) 1 -1" (2 "10") -Condition 594 "632686821" "(_GEN_65[owners_4_2_3] & ((~_owners_T_98))) 1 -1" (2 "10") -ANNOTATION: "Capture is VALID & !READY & !lockValid; capture and an existing lock cannot both be true, so !capture=0 / lockValid=1 is unreachable" -Condition 519 "4125160278" "(((~_owners_T_3)) | owners_0_3) 1 -1" (2 "01") -Condition 523 "3698058731" "(((~_owners_T_8)) | owners_1_3) 1 -1" (2 "01") -Condition 527 "2771202367" "(((~_owners_T_13)) | owners_2_3) 1 -1" (2 "01") -Condition 531 "229547046" "(((~_owners_T_18)) | owners_3_3) 1 -1" (2 "01") -Condition 535 "2665684153" "(((~_owners_T_23)) | owners_4_3) 1 -1" (2 "01") -Condition 539 "2806063833" "(((~_owners_T_28)) | owners_0_3_1) 1 -1" (2 "01") -Condition 543 "3170010768" "(((~_owners_T_33)) | owners_1_3_1) 1 -1" (2 "01") -Condition 547 "3394554053" "(((~_owners_T_38)) | owners_2_3_1) 1 -1" (2 "01") -Condition 551 "950194497" "(((~_owners_T_43)) | owners_3_3_1) 1 -1" (2 "01") -Condition 555 "2952743903" "(((~_owners_T_48)) | owners_4_3_1) 1 -1" (2 "01") -Condition 559 "1705755745" "(((~_owners_T_53)) | owners_0_3_2) 1 -1" (2 "01") -Condition 563 "3367075034" "(((~_owners_T_58)) | owners_1_3_2) 1 -1" (2 "01") -Condition 567 "2469408776" "(((~_owners_T_63)) | owners_2_3_2) 1 -1" (2 "01") -Condition 571 "3671899352" "(((~_owners_T_68)) | owners_3_3_2) 1 -1" (2 "01") -Condition 575 "1467654804" "(((~_owners_T_73)) | owners_4_3_2) 1 -1" (2 "01") -Condition 579 "1545107880" "(((~_owners_T_78)) | owners_0_3_3) 1 -1" (2 "01") -Condition 583 "2118368442" "(((~_owners_T_83)) | owners_1_3_3) 1 -1" (2 "01") -Condition 587 "149287663" "(((~_owners_T_88)) | owners_2_3_3) 1 -1" (2 "01") -Condition 591 "1576121327" "(((~_owners_T_93)) | owners_3_3_3) 1 -1" (2 "01") -Condition 595 "2584552825" "(((~_owners_T_98)) | owners_4_3_3) 1 -1" (2 "01") - -CHECKSUM: "1802735833 384767911" -INSTANCE: router_tb.dut -ANNOTATION: "Literal zero prepended to the 3-bit round-robin cursor; only padded bit 3 is excluded" -Toggle _GEN [3] "net _GEN[3:0]" -Toggle _GEN_10 [3] "net _GEN_10[3:0]" -Toggle _GEN_12 [3] "net _GEN_12[3:0]" -Toggle _GEN_14 [3] "net _GEN_14[3:0]" -Toggle _GEN_16 [3] "net _GEN_16[3:0]" -Toggle _GEN_18 [3] "net _GEN_18[3:0]" -Toggle _GEN_29 [3] "net _GEN_29[3:0]" -Toggle _GEN_31 [3] "net _GEN_31[3:0]" -Toggle _GEN_33 [3] "net _GEN_33[3:0]" -Toggle _GEN_35 [3] "net _GEN_35[3:0]" -Toggle _GEN_37 [3] "net _GEN_37[3:0]" -Toggle _GEN_48 [3] "net _GEN_48[3:0]" -Toggle _GEN_50 [3] "net _GEN_50[3:0]" -Toggle _GEN_52 [3] "net _GEN_52[3:0]" -Toggle _GEN_54 [3] "net _GEN_54[3:0]" -Toggle _GEN_56 [3] "net _GEN_56[3:0]" -Toggle _GEN_67 [3] "net _GEN_67[3:0]" -Toggle _GEN_69 [3] "net _GEN_69[3:0]" -Toggle _GEN_71 [3] "net _GEN_71[3:0]" -Toggle _GEN_73 [3] "net _GEN_73[3:0]" -ANNOTATION: "Result is explicitly modulo the 5 input directions; only values 0..4 are possible, so only bit 3 is constant zero" -Toggle _owners_ordered_T_1 [3] "net _owners_ordered_T_1[3:0]" -Toggle _owners_ordered_T_3 [3] "net _owners_ordered_T_3[3:0]" -Toggle _owners_ordered_T_5 [3] "net _owners_ordered_T_5[3:0]" -Toggle _owners_ordered_T_7 [3] "net _owners_ordered_T_7[3:0]" -Toggle owners_fairChoice [3] "net owners_fairChoice[3:0]" -Toggle _owners_ordered_T_11 [3] "net _owners_ordered_T_11[3:0]" -Toggle _owners_ordered_T_13 [3] "net _owners_ordered_T_13[3:0]" -Toggle _owners_ordered_T_15 [3] "net _owners_ordered_T_15[3:0]" -Toggle _owners_ordered_T_17 [3] "net _owners_ordered_T_17[3:0]" -Toggle owners_fairChoice_1 [3] "net owners_fairChoice_1[3:0]" -Toggle _owners_ordered_T_21 [3] "net _owners_ordered_T_21[3:0]" -Toggle _owners_ordered_T_23 [3] "net _owners_ordered_T_23[3:0]" -Toggle _owners_ordered_T_25 [3] "net _owners_ordered_T_25[3:0]" -Toggle _owners_ordered_T_27 [3] "net _owners_ordered_T_27[3:0]" -Toggle owners_fairChoice_2 [3] "net owners_fairChoice_2[3:0]" -Toggle _owners_ordered_T_31 [3] "net _owners_ordered_T_31[3:0]" -Toggle _owners_ordered_T_33 [3] "net _owners_ordered_T_33[3:0]" -Toggle _owners_ordered_T_35 [3] "net _owners_ordered_T_35[3:0]" -Toggle _owners_ordered_T_37 [3] "net _owners_ordered_T_37[3:0]" -Toggle owners_fairChoice_3 [3] "net owners_fairChoice_3[3:0]" -Toggle _owners_ordered_T_41 [3] "net _owners_ordered_T_41[3:0]" -Toggle _owners_ordered_T_43 [3] "net _owners_ordered_T_43[3:0]" -Toggle _owners_ordered_T_45 [3] "net _owners_ordered_T_45[3:0]" -Toggle _owners_ordered_T_47 [3] "net _owners_ordered_T_47[3:0]" -Toggle owners_fairChoice_4 [3] "net owners_fairChoice_4[3:0]" -Toggle _owners_ordered_T_51 [3] "net _owners_ordered_T_51[3:0]" -Toggle _owners_ordered_T_53 [3] "net _owners_ordered_T_53[3:0]" -Toggle _owners_ordered_T_55 [3] "net _owners_ordered_T_55[3:0]" -Toggle _owners_ordered_T_57 [3] "net _owners_ordered_T_57[3:0]" -Toggle owners_fairChoice_5 [3] "net owners_fairChoice_5[3:0]" -Toggle _owners_ordered_T_61 [3] "net _owners_ordered_T_61[3:0]" -Toggle _owners_ordered_T_63 [3] "net _owners_ordered_T_63[3:0]" -Toggle _owners_ordered_T_65 [3] "net _owners_ordered_T_65[3:0]" -Toggle _owners_ordered_T_67 [3] "net _owners_ordered_T_67[3:0]" -Toggle owners_fairChoice_6 [3] "net owners_fairChoice_6[3:0]" -Toggle _owners_ordered_T_71 [3] "net _owners_ordered_T_71[3:0]" -Toggle _owners_ordered_T_73 [3] "net _owners_ordered_T_73[3:0]" -Toggle _owners_ordered_T_75 [3] "net _owners_ordered_T_75[3:0]" -Toggle _owners_ordered_T_77 [3] "net _owners_ordered_T_77[3:0]" -Toggle owners_fairChoice_7 [3] "net owners_fairChoice_7[3:0]" -Toggle _owners_ordered_T_81 [3] "net _owners_ordered_T_81[3:0]" -Toggle _owners_ordered_T_83 [3] "net _owners_ordered_T_83[3:0]" -Toggle _owners_ordered_T_85 [3] "net _owners_ordered_T_85[3:0]" -Toggle _owners_ordered_T_87 [3] "net _owners_ordered_T_87[3:0]" -Toggle owners_fairChoice_8 [3] "net owners_fairChoice_8[3:0]" -Toggle _owners_ordered_T_91 [3] "net _owners_ordered_T_91[3:0]" -Toggle _owners_ordered_T_93 [3] "net _owners_ordered_T_93[3:0]" -Toggle _owners_ordered_T_95 [3] "net _owners_ordered_T_95[3:0]" -Toggle _owners_ordered_T_97 [3] "net _owners_ordered_T_97[3:0]" -Toggle owners_fairChoice_9 [3] "net owners_fairChoice_9[3:0]" -Toggle _owners_ordered_T_101 [3] "net _owners_ordered_T_101[3:0]" -Toggle _owners_ordered_T_103 [3] "net _owners_ordered_T_103[3:0]" -Toggle _owners_ordered_T_105 [3] "net _owners_ordered_T_105[3:0]" -Toggle _owners_ordered_T_107 [3] "net _owners_ordered_T_107[3:0]" -Toggle owners_fairChoice_10 [3] "net owners_fairChoice_10[3:0]" -Toggle _owners_ordered_T_111 [3] "net _owners_ordered_T_111[3:0]" -Toggle _owners_ordered_T_113 [3] "net _owners_ordered_T_113[3:0]" -Toggle _owners_ordered_T_115 [3] "net _owners_ordered_T_115[3:0]" -Toggle _owners_ordered_T_117 [3] "net _owners_ordered_T_117[3:0]" -Toggle owners_fairChoice_11 [3] "net owners_fairChoice_11[3:0]" -Toggle _owners_ordered_T_121 [3] "net _owners_ordered_T_121[3:0]" -Toggle _owners_ordered_T_123 [3] "net _owners_ordered_T_123[3:0]" -Toggle _owners_ordered_T_125 [3] "net _owners_ordered_T_125[3:0]" -Toggle _owners_ordered_T_127 [3] "net _owners_ordered_T_127[3:0]" -Toggle owners_fairChoice_12 [3] "net owners_fairChoice_12[3:0]" -Toggle _owners_ordered_T_131 [3] "net _owners_ordered_T_131[3:0]" -Toggle _owners_ordered_T_133 [3] "net _owners_ordered_T_133[3:0]" -Toggle _owners_ordered_T_135 [3] "net _owners_ordered_T_135[3:0]" -Toggle _owners_ordered_T_137 [3] "net _owners_ordered_T_137[3:0]" -Toggle owners_fairChoice_13 [3] "net owners_fairChoice_13[3:0]" -Toggle _owners_ordered_T_141 [3] "net _owners_ordered_T_141[3:0]" -Toggle _owners_ordered_T_143 [3] "net _owners_ordered_T_143[3:0]" -Toggle _owners_ordered_T_145 [3] "net _owners_ordered_T_145[3:0]" -Toggle _owners_ordered_T_147 [3] "net _owners_ordered_T_147[3:0]" -Toggle owners_fairChoice_14 [3] "net owners_fairChoice_14[3:0]" -Toggle _owners_ordered_T_151 [3] "net _owners_ordered_T_151[3:0]" -Toggle _owners_ordered_T_153 [3] "net _owners_ordered_T_153[3:0]" -Toggle _owners_ordered_T_155 [3] "net _owners_ordered_T_155[3:0]" -Toggle _owners_ordered_T_157 [3] "net _owners_ordered_T_157[3:0]" -Toggle owners_fairChoice_15 [3] "net owners_fairChoice_15[3:0]" -Toggle _owners_ordered_T_161 [3] "net _owners_ordered_T_161[3:0]" -Toggle _owners_ordered_T_163 [3] "net _owners_ordered_T_163[3:0]" -Toggle _owners_ordered_T_165 [3] "net _owners_ordered_T_165[3:0]" -Toggle _owners_ordered_T_167 [3] "net _owners_ordered_T_167[3:0]" -Toggle owners_fairChoice_16 [3] "net owners_fairChoice_16[3:0]" -Toggle _owners_ordered_T_171 [3] "net _owners_ordered_T_171[3:0]" -Toggle _owners_ordered_T_173 [3] "net _owners_ordered_T_173[3:0]" -Toggle _owners_ordered_T_175 [3] "net _owners_ordered_T_175[3:0]" -Toggle _owners_ordered_T_177 [3] "net _owners_ordered_T_177[3:0]" -Toggle owners_fairChoice_17 [3] "net owners_fairChoice_17[3:0]" -Toggle _owners_ordered_T_181 [3] "net _owners_ordered_T_181[3:0]" -Toggle _owners_ordered_T_183 [3] "net _owners_ordered_T_183[3:0]" -Toggle _owners_ordered_T_185 [3] "net _owners_ordered_T_185[3:0]" -Toggle _owners_ordered_T_187 [3] "net _owners_ordered_T_187[3:0]" -Toggle owners_fairChoice_18 [3] "net owners_fairChoice_18[3:0]" -Toggle _owners_ordered_T_191 [3] "net _owners_ordered_T_191[3:0]" -Toggle _owners_ordered_T_193 [3] "net _owners_ordered_T_193[3:0]" -Toggle _owners_ordered_T_195 [3] "net _owners_ordered_T_195[3:0]" -Toggle _owners_ordered_T_197 [3] "net _owners_ordered_T_197[3:0]" -Toggle owners_fairChoice_19 [3] "net owners_fairChoice_19[3:0]" diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router.f b/arch/src/main/scala/framework/root/root-chip/src/main/resources/router.f deleted file mode 100644 index 5f48e24aa..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router.f +++ /dev/null @@ -1,7 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/fabric_interface.sv -@RESOURCES@/router_package.sv --F @RTL@/MeshRouter/filelist.f -@RESOURCES@/router_tb.sv diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router_package.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/router_package.sv deleted file mode 100644 index 9a459f26a..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router_package.sv +++ /dev/null @@ -1,211 +0,0 @@ -package mesh_router_pkg; - import "DPI-C" function int mesh_route( - int x, - int y, - int dst_x, - int dst_y - ); - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual mesh_fabric_if vif; - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - timeout = 300us; - if (!uvm_config_db#(virtual mesh_fabric_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "fabric interface missing") - endfunction - task packet(int source, int direction, int channel, bit stall_head = 0, int length = 3); - int dx, dy; - dx = direction == 4 ? 0 : direction == 3 ? 2 : 1; - dy = direction == 1 ? 0 : direction == 2 ? 2 : 1; - vif.src_x[source][channel] = (source + direction) % 3; - vif.src_y[source][channel] = (source + channel) % 3; - for (int beat = 0; beat < length; beat++) begin - if (beat == 1) begin - vif.valid[source][channel] = 0; - vif.out_ready[direction][channel] = 1; - // VALID-low packet gap may carry unrelated metadata; ownership must survive. - vif.x[source][channel] = dx == 0 ? 2 : 0; - vif.y[source][channel] = dy == 0 ? 2 : 0; - repeat (2) @(negedge vif.clock); - end - vif.valid[source][channel] = 1; - vif.x[source][channel] = dx; - vif.y[source][channel] = dy; - vif.head[source][channel] = beat == 0; - vif.tail[source][channel] = beat == length - 1; - vif.data[source][channel] = $urandom; - vif.out_ready[direction][channel] = beat == 0 && !stall_head; - if (beat != 0 || stall_head) - repeat (2) begin - @(posedge vif.clock); - if(vif.ready[source][channel]||vif.out_data[direction][channel]!==vif.data[source][channel]) - `uvm_fatal("STABLE", "packet beat unstable while blocked"); - @(negedge vif.clock); - end - vif.out_ready[direction][channel] = 1; - @(posedge vif.clock); - if (!vif.ready[source][channel] || !vif.out_valid[mesh_route( - 1, 1, dx, dy - )][channel] || vif.out_data[direction][channel] !== vif.data[source][channel] || - vif.out_src_x[direction][channel] !== vif.src_x[source][channel] || - vif.out_src_y[direction][channel] !== vif.src_y[source][channel]) - `uvm_fatal("PACKET", "route or metadata mismatch") - @(negedge vif.clock); - end - vif.valid[source][channel] = 0; - endtask - task execute(); - int counts [5][4]; - int fired; - bit accepted[5][4]; - vif.reset = 1; - vif.active = 0; - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) begin - vif.valid[i][v] = 0; - vif.src_x[i][v] = (i + v) % 3; - vif.src_y[i][v] = (i + v) / 3 % 3; - vif.vc[i][v] = v; - vif.x[i][v] = 1; - vif.y[i][v] = 1; - vif.head[i][v] = 1; - vif.tail[i][v] = 1; - vif.data[i][v] = 32'h100 + i; - vif.out_ready[i][v] = 1; - end - repeat (4) @(negedge vif.clock); - // Reset cancels transactions; metadata is unconstrained during reset. - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) begin - vif.valid[i][v] = 1; - vif.x[i][v] = 3; - vif.y[i][v] = 3; - vif.vc[i][v] = (v + 1) % 4; - vif.head[i][v] = 0; - end - for (int source = 0; source < 5; source++) begin - for (int i = 0; i < 5; i++) for (int v = 0; v < 4; v++) vif.valid[i][v] = i == source; - repeat (2) @(negedge vif.clock); - end - for (int i = 0; i < 5; i++) for (int v = 0; v < 4; v++) vif.valid[i][v] = 0; - vif.reset = 0; - repeat (4) @(negedge vif.clock); - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) begin - vif.x[i][v] = 3; - vif.y[i][v] = 1; - end - repeat (2) @(negedge vif.clock); - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) begin - vif.x[i][v] = 1; - vif.y[i][v] = 3; - end - repeat (2) @(negedge vif.clock); - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) begin - vif.x[i][v] = 1; - vif.y[i][v] = 1; - vif.vc[i][v] = v; - vif.head[i][v] = 1; - end - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) begin - counts[i][v] = 0; - vif.valid[i][v] = 1; - end - for (int tick = 0; tick < 100; tick++) begin - @(posedge vif.clock); - fired = 0; - for (int v = 0; v < 4; v++) begin - fired = 0; - for (int i = 0; i < 5; i++) begin - accepted[i][v] = vif.ready[i][v]; - if (vif.ready[i][v]) begin - fired++; - counts[i][v]++; - if (vif.out_data[0][v] !== vif.data[i][v]) - `uvm_fatal("DATA", "selected source mismatch"); - end - end - if (fired != 1) - `uvm_fatal("CONTENTION", $sformatf("%0d inputs retired for one output", fired)); - end - @(negedge vif.clock); - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) - if (accepted[i][v]) begin - if (counts[i][v] == 20) vif.valid[i][v] = 0; - else begin - vif.data[i][v] = $urandom; - vif.src_x[i][v] = counts[i][v] % 3; - vif.src_y[i][v] = (counts[i][v] / 3) % 3; - end - end - end - for (int i = 0; i < 5; i++) - for (int v = 0; v < 4; v++) begin - if (counts[i][v] != 20) `uvm_fatal("FAIR", "persistent contender starved"); - vif.valid[i][v] = 0; - end - // A blocked REQ must not stop independent RSP, DAT and SNP ports. - vif.valid[4][0] = 1; - vif.data[4][0] = 32'h104; - vif.out_ready[0][0] = 0; - for (int v = 1; v < 4; v++) vif.valid[0][v] = 1; - repeat (20) begin - @(posedge vif.clock); - if (vif.ready[4][0]) `uvm_fatal("STALL", "blocked REQ retired") - for (int v = 1; v < 4; v++) - if (!vif.ready[0][v] || !vif.out_valid[0][v]) - `uvm_fatal("ISOLATION", "other CHI VC blocked by REQ") - @(negedge vif.clock); - end - for (int v = 1; v < 4; v++) vif.valid[0][v] = 0; - vif.out_ready[0][0] = 1; - @(posedge vif.clock); - if (!vif.ready[4][0] || vif.out_data[0][0] !== 32'h104) - `uvm_fatal("RESUME", "stalled request was not preserved") - @(negedge vif.clock); - vif.valid[4][0] = 0; - // Unlocked and empty output may be backpressured independently of a source. - for (int direction = 0; direction < 5; direction++) - for (int channel = 0; channel < 4; channel++) vif.out_ready[direction][channel] = 0; - repeat (2) @(negedge vif.clock); - for (int direction = 0; direction < 5; direction++) - for (int channel = 0; channel < 4; channel++) vif.out_ready[direction][channel] = 1; - // Each source seed and each offset from the round-robin cursor is exercised. - for (int channel = 0; channel < 4; channel++) - for (int direction = 0; direction < 5; direction++) - for (int seed = 0; seed < 5; seed++) - for (int offset = 0; offset < 5; offset++) begin - packet(seed, direction, channel, offset % 2); - packet((seed + 1 + offset) % 5, direction, channel); - end - for (int channel = 0; channel < 4; channel++) - for (int direction = 0; direction < 5; direction++) - for (int source = 0; source < 5; source++) packet(source, direction, channel, 0, 1); - // Input metadata is unconstrained with VALID low; no packet may retire. - for (int value = 0; value < 4; value++) begin - for (int i = 0; i < 5; i++) for (int v = 0; v < 4; v++) vif.vc[i][v] = value; - repeat (2) @(negedge vif.clock); - end - vif.reset = 1; - repeat (4) @(negedge vif.clock); - vif.reset = 0; - for (int i = 0; i < 5; i++) for (int v = 0; v < 4; v++) vif.vc[i][v] = v; - repeat (4) @(negedge vif.clock); - `uvm_info( - "CHECKS", - "400 fair contended flits, 60 VC-isolation flits, 3100 routed packet flits; all cursor offsets and source coordinates with per-beat backpressure", - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router_tb.sv b/arch/src/main/scala/framework/root/root-chip/src/main/resources/router_tb.sv deleted file mode 100644 index f5627697d..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/resources/router_tb.sv +++ /dev/null @@ -1,355 +0,0 @@ -module router_tb; - import uvm_pkg::*; - import mesh_router_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - mesh_fabric_if vif (clock); - MeshRouter dut ( - .clock(clock), - .reset(vif.reset), - .io_in_0_0_valid(vif.valid[0][0]), - .io_in_0_0_ready(vif.ready[0][0]), - .io_in_0_0_bits_dstX(vif.x[0][0]), - .io_in_0_0_bits_dstY(vif.y[0][0]), - .io_in_0_0_bits_head(vif.head[0][0]), - .io_in_0_0_bits_tail(vif.tail[0][0]), - .io_in_0_0_bits_payload(vif.data[0][0]), - .io_in_0_0_bits_srcX(vif.src_x[0][0]), - .io_in_0_0_bits_srcY(vif.src_y[0][0]), - .io_in_0_0_bits_vc(vif.vc[0][0]), - .io_out_0_0_valid(vif.out_valid[0][0]), - .io_out_0_0_ready(vif.out_ready[0][0]), - .io_out_0_0_bits_payload(vif.out_data[0][0]), - .io_out_0_0_bits_srcX(vif.out_src_x[0][0]), - .io_out_0_0_bits_srcY(vif.out_src_y[0][0]), - .io_out_0_0_bits_head(vif.out_head[0][0]), - .io_out_0_0_bits_tail(vif.out_tail[0][0]), - .io_in_0_1_valid(vif.valid[0][1]), - .io_in_0_1_ready(vif.ready[0][1]), - .io_in_0_1_bits_dstX(vif.x[0][1]), - .io_in_0_1_bits_dstY(vif.y[0][1]), - .io_in_0_1_bits_head(vif.head[0][1]), - .io_in_0_1_bits_tail(vif.tail[0][1]), - .io_in_0_1_bits_payload(vif.data[0][1]), - .io_in_0_1_bits_srcX(vif.src_x[0][1]), - .io_in_0_1_bits_srcY(vif.src_y[0][1]), - .io_in_0_1_bits_vc(vif.vc[0][1]), - .io_out_0_1_valid(vif.out_valid[0][1]), - .io_out_0_1_ready(vif.out_ready[0][1]), - .io_out_0_1_bits_payload(vif.out_data[0][1]), - .io_out_0_1_bits_srcX(vif.out_src_x[0][1]), - .io_out_0_1_bits_srcY(vif.out_src_y[0][1]), - .io_out_0_1_bits_head(vif.out_head[0][1]), - .io_out_0_1_bits_tail(vif.out_tail[0][1]), - .io_in_0_2_valid(vif.valid[0][2]), - .io_in_0_2_ready(vif.ready[0][2]), - .io_in_0_2_bits_dstX(vif.x[0][2]), - .io_in_0_2_bits_dstY(vif.y[0][2]), - .io_in_0_2_bits_head(vif.head[0][2]), - .io_in_0_2_bits_tail(vif.tail[0][2]), - .io_in_0_2_bits_payload(vif.data[0][2]), - .io_in_0_2_bits_srcX(vif.src_x[0][2]), - .io_in_0_2_bits_srcY(vif.src_y[0][2]), - .io_in_0_2_bits_vc(vif.vc[0][2]), - .io_out_0_2_valid(vif.out_valid[0][2]), - .io_out_0_2_ready(vif.out_ready[0][2]), - .io_out_0_2_bits_payload(vif.out_data[0][2]), - .io_out_0_2_bits_srcX(vif.out_src_x[0][2]), - .io_out_0_2_bits_srcY(vif.out_src_y[0][2]), - .io_out_0_2_bits_head(vif.out_head[0][2]), - .io_out_0_2_bits_tail(vif.out_tail[0][2]), - .io_in_0_3_valid(vif.valid[0][3]), - .io_in_0_3_ready(vif.ready[0][3]), - .io_in_0_3_bits_dstX(vif.x[0][3]), - .io_in_0_3_bits_dstY(vif.y[0][3]), - .io_in_0_3_bits_head(vif.head[0][3]), - .io_in_0_3_bits_tail(vif.tail[0][3]), - .io_in_0_3_bits_payload(vif.data[0][3]), - .io_in_0_3_bits_srcX(vif.src_x[0][3]), - .io_in_0_3_bits_srcY(vif.src_y[0][3]), - .io_in_0_3_bits_vc(vif.vc[0][3]), - .io_out_0_3_valid(vif.out_valid[0][3]), - .io_out_0_3_ready(vif.out_ready[0][3]), - .io_out_0_3_bits_payload(vif.out_data[0][3]), - .io_out_0_3_bits_srcX(vif.out_src_x[0][3]), - .io_out_0_3_bits_srcY(vif.out_src_y[0][3]), - .io_out_0_3_bits_head(vif.out_head[0][3]), - .io_out_0_3_bits_tail(vif.out_tail[0][3]), - .io_in_1_0_valid(vif.valid[1][0]), - .io_in_1_0_ready(vif.ready[1][0]), - .io_in_1_0_bits_dstX(vif.x[1][0]), - .io_in_1_0_bits_dstY(vif.y[1][0]), - .io_in_1_0_bits_head(vif.head[1][0]), - .io_in_1_0_bits_tail(vif.tail[1][0]), - .io_in_1_0_bits_payload(vif.data[1][0]), - .io_in_1_0_bits_srcX(vif.src_x[1][0]), - .io_in_1_0_bits_srcY(vif.src_y[1][0]), - .io_in_1_0_bits_vc(vif.vc[1][0]), - .io_out_1_0_valid(vif.out_valid[1][0]), - .io_out_1_0_ready(vif.out_ready[1][0]), - .io_out_1_0_bits_payload(vif.out_data[1][0]), - .io_out_1_0_bits_srcX(vif.out_src_x[1][0]), - .io_out_1_0_bits_srcY(vif.out_src_y[1][0]), - .io_out_1_0_bits_head(vif.out_head[1][0]), - .io_out_1_0_bits_tail(vif.out_tail[1][0]), - .io_in_1_1_valid(vif.valid[1][1]), - .io_in_1_1_ready(vif.ready[1][1]), - .io_in_1_1_bits_dstX(vif.x[1][1]), - .io_in_1_1_bits_dstY(vif.y[1][1]), - .io_in_1_1_bits_head(vif.head[1][1]), - .io_in_1_1_bits_tail(vif.tail[1][1]), - .io_in_1_1_bits_payload(vif.data[1][1]), - .io_in_1_1_bits_srcX(vif.src_x[1][1]), - .io_in_1_1_bits_srcY(vif.src_y[1][1]), - .io_in_1_1_bits_vc(vif.vc[1][1]), - .io_out_1_1_valid(vif.out_valid[1][1]), - .io_out_1_1_ready(vif.out_ready[1][1]), - .io_out_1_1_bits_payload(vif.out_data[1][1]), - .io_out_1_1_bits_srcX(vif.out_src_x[1][1]), - .io_out_1_1_bits_srcY(vif.out_src_y[1][1]), - .io_out_1_1_bits_head(vif.out_head[1][1]), - .io_out_1_1_bits_tail(vif.out_tail[1][1]), - .io_in_1_2_valid(vif.valid[1][2]), - .io_in_1_2_ready(vif.ready[1][2]), - .io_in_1_2_bits_dstX(vif.x[1][2]), - .io_in_1_2_bits_dstY(vif.y[1][2]), - .io_in_1_2_bits_head(vif.head[1][2]), - .io_in_1_2_bits_tail(vif.tail[1][2]), - .io_in_1_2_bits_payload(vif.data[1][2]), - .io_in_1_2_bits_srcX(vif.src_x[1][2]), - .io_in_1_2_bits_srcY(vif.src_y[1][2]), - .io_in_1_2_bits_vc(vif.vc[1][2]), - .io_out_1_2_valid(vif.out_valid[1][2]), - .io_out_1_2_ready(vif.out_ready[1][2]), - .io_out_1_2_bits_payload(vif.out_data[1][2]), - .io_out_1_2_bits_srcX(vif.out_src_x[1][2]), - .io_out_1_2_bits_srcY(vif.out_src_y[1][2]), - .io_out_1_2_bits_head(vif.out_head[1][2]), - .io_out_1_2_bits_tail(vif.out_tail[1][2]), - .io_in_1_3_valid(vif.valid[1][3]), - .io_in_1_3_ready(vif.ready[1][3]), - .io_in_1_3_bits_dstX(vif.x[1][3]), - .io_in_1_3_bits_dstY(vif.y[1][3]), - .io_in_1_3_bits_head(vif.head[1][3]), - .io_in_1_3_bits_tail(vif.tail[1][3]), - .io_in_1_3_bits_payload(vif.data[1][3]), - .io_in_1_3_bits_srcX(vif.src_x[1][3]), - .io_in_1_3_bits_srcY(vif.src_y[1][3]), - .io_in_1_3_bits_vc(vif.vc[1][3]), - .io_out_1_3_valid(vif.out_valid[1][3]), - .io_out_1_3_ready(vif.out_ready[1][3]), - .io_out_1_3_bits_payload(vif.out_data[1][3]), - .io_out_1_3_bits_srcX(vif.out_src_x[1][3]), - .io_out_1_3_bits_srcY(vif.out_src_y[1][3]), - .io_out_1_3_bits_head(vif.out_head[1][3]), - .io_out_1_3_bits_tail(vif.out_tail[1][3]), - .io_in_2_0_valid(vif.valid[2][0]), - .io_in_2_0_ready(vif.ready[2][0]), - .io_in_2_0_bits_dstX(vif.x[2][0]), - .io_in_2_0_bits_dstY(vif.y[2][0]), - .io_in_2_0_bits_head(vif.head[2][0]), - .io_in_2_0_bits_tail(vif.tail[2][0]), - .io_in_2_0_bits_payload(vif.data[2][0]), - .io_in_2_0_bits_srcX(vif.src_x[2][0]), - .io_in_2_0_bits_srcY(vif.src_y[2][0]), - .io_in_2_0_bits_vc(vif.vc[2][0]), - .io_out_2_0_valid(vif.out_valid[2][0]), - .io_out_2_0_ready(vif.out_ready[2][0]), - .io_out_2_0_bits_payload(vif.out_data[2][0]), - .io_out_2_0_bits_srcX(vif.out_src_x[2][0]), - .io_out_2_0_bits_srcY(vif.out_src_y[2][0]), - .io_out_2_0_bits_head(vif.out_head[2][0]), - .io_out_2_0_bits_tail(vif.out_tail[2][0]), - .io_in_2_1_valid(vif.valid[2][1]), - .io_in_2_1_ready(vif.ready[2][1]), - .io_in_2_1_bits_dstX(vif.x[2][1]), - .io_in_2_1_bits_dstY(vif.y[2][1]), - .io_in_2_1_bits_head(vif.head[2][1]), - .io_in_2_1_bits_tail(vif.tail[2][1]), - .io_in_2_1_bits_payload(vif.data[2][1]), - .io_in_2_1_bits_srcX(vif.src_x[2][1]), - .io_in_2_1_bits_srcY(vif.src_y[2][1]), - .io_in_2_1_bits_vc(vif.vc[2][1]), - .io_out_2_1_valid(vif.out_valid[2][1]), - .io_out_2_1_ready(vif.out_ready[2][1]), - .io_out_2_1_bits_payload(vif.out_data[2][1]), - .io_out_2_1_bits_srcX(vif.out_src_x[2][1]), - .io_out_2_1_bits_srcY(vif.out_src_y[2][1]), - .io_out_2_1_bits_head(vif.out_head[2][1]), - .io_out_2_1_bits_tail(vif.out_tail[2][1]), - .io_in_2_2_valid(vif.valid[2][2]), - .io_in_2_2_ready(vif.ready[2][2]), - .io_in_2_2_bits_dstX(vif.x[2][2]), - .io_in_2_2_bits_dstY(vif.y[2][2]), - .io_in_2_2_bits_head(vif.head[2][2]), - .io_in_2_2_bits_tail(vif.tail[2][2]), - .io_in_2_2_bits_payload(vif.data[2][2]), - .io_in_2_2_bits_srcX(vif.src_x[2][2]), - .io_in_2_2_bits_srcY(vif.src_y[2][2]), - .io_in_2_2_bits_vc(vif.vc[2][2]), - .io_out_2_2_valid(vif.out_valid[2][2]), - .io_out_2_2_ready(vif.out_ready[2][2]), - .io_out_2_2_bits_payload(vif.out_data[2][2]), - .io_out_2_2_bits_srcX(vif.out_src_x[2][2]), - .io_out_2_2_bits_srcY(vif.out_src_y[2][2]), - .io_out_2_2_bits_head(vif.out_head[2][2]), - .io_out_2_2_bits_tail(vif.out_tail[2][2]), - .io_in_2_3_valid(vif.valid[2][3]), - .io_in_2_3_ready(vif.ready[2][3]), - .io_in_2_3_bits_dstX(vif.x[2][3]), - .io_in_2_3_bits_dstY(vif.y[2][3]), - .io_in_2_3_bits_head(vif.head[2][3]), - .io_in_2_3_bits_tail(vif.tail[2][3]), - .io_in_2_3_bits_payload(vif.data[2][3]), - .io_in_2_3_bits_srcX(vif.src_x[2][3]), - .io_in_2_3_bits_srcY(vif.src_y[2][3]), - .io_in_2_3_bits_vc(vif.vc[2][3]), - .io_out_2_3_valid(vif.out_valid[2][3]), - .io_out_2_3_ready(vif.out_ready[2][3]), - .io_out_2_3_bits_payload(vif.out_data[2][3]), - .io_out_2_3_bits_srcX(vif.out_src_x[2][3]), - .io_out_2_3_bits_srcY(vif.out_src_y[2][3]), - .io_out_2_3_bits_head(vif.out_head[2][3]), - .io_out_2_3_bits_tail(vif.out_tail[2][3]), - .io_in_3_0_valid(vif.valid[3][0]), - .io_in_3_0_ready(vif.ready[3][0]), - .io_in_3_0_bits_dstX(vif.x[3][0]), - .io_in_3_0_bits_dstY(vif.y[3][0]), - .io_in_3_0_bits_head(vif.head[3][0]), - .io_in_3_0_bits_tail(vif.tail[3][0]), - .io_in_3_0_bits_payload(vif.data[3][0]), - .io_in_3_0_bits_srcX(vif.src_x[3][0]), - .io_in_3_0_bits_srcY(vif.src_y[3][0]), - .io_in_3_0_bits_vc(vif.vc[3][0]), - .io_out_3_0_valid(vif.out_valid[3][0]), - .io_out_3_0_ready(vif.out_ready[3][0]), - .io_out_3_0_bits_payload(vif.out_data[3][0]), - .io_out_3_0_bits_srcX(vif.out_src_x[3][0]), - .io_out_3_0_bits_srcY(vif.out_src_y[3][0]), - .io_out_3_0_bits_head(vif.out_head[3][0]), - .io_out_3_0_bits_tail(vif.out_tail[3][0]), - .io_in_3_1_valid(vif.valid[3][1]), - .io_in_3_1_ready(vif.ready[3][1]), - .io_in_3_1_bits_dstX(vif.x[3][1]), - .io_in_3_1_bits_dstY(vif.y[3][1]), - .io_in_3_1_bits_head(vif.head[3][1]), - .io_in_3_1_bits_tail(vif.tail[3][1]), - .io_in_3_1_bits_payload(vif.data[3][1]), - .io_in_3_1_bits_srcX(vif.src_x[3][1]), - .io_in_3_1_bits_srcY(vif.src_y[3][1]), - .io_in_3_1_bits_vc(vif.vc[3][1]), - .io_out_3_1_valid(vif.out_valid[3][1]), - .io_out_3_1_ready(vif.out_ready[3][1]), - .io_out_3_1_bits_payload(vif.out_data[3][1]), - .io_out_3_1_bits_srcX(vif.out_src_x[3][1]), - .io_out_3_1_bits_srcY(vif.out_src_y[3][1]), - .io_out_3_1_bits_head(vif.out_head[3][1]), - .io_out_3_1_bits_tail(vif.out_tail[3][1]), - .io_in_3_2_valid(vif.valid[3][2]), - .io_in_3_2_ready(vif.ready[3][2]), - .io_in_3_2_bits_dstX(vif.x[3][2]), - .io_in_3_2_bits_dstY(vif.y[3][2]), - .io_in_3_2_bits_head(vif.head[3][2]), - .io_in_3_2_bits_tail(vif.tail[3][2]), - .io_in_3_2_bits_payload(vif.data[3][2]), - .io_in_3_2_bits_srcX(vif.src_x[3][2]), - .io_in_3_2_bits_srcY(vif.src_y[3][2]), - .io_in_3_2_bits_vc(vif.vc[3][2]), - .io_out_3_2_valid(vif.out_valid[3][2]), - .io_out_3_2_ready(vif.out_ready[3][2]), - .io_out_3_2_bits_payload(vif.out_data[3][2]), - .io_out_3_2_bits_srcX(vif.out_src_x[3][2]), - .io_out_3_2_bits_srcY(vif.out_src_y[3][2]), - .io_out_3_2_bits_head(vif.out_head[3][2]), - .io_out_3_2_bits_tail(vif.out_tail[3][2]), - .io_in_3_3_valid(vif.valid[3][3]), - .io_in_3_3_ready(vif.ready[3][3]), - .io_in_3_3_bits_dstX(vif.x[3][3]), - .io_in_3_3_bits_dstY(vif.y[3][3]), - .io_in_3_3_bits_head(vif.head[3][3]), - .io_in_3_3_bits_tail(vif.tail[3][3]), - .io_in_3_3_bits_payload(vif.data[3][3]), - .io_in_3_3_bits_srcX(vif.src_x[3][3]), - .io_in_3_3_bits_srcY(vif.src_y[3][3]), - .io_in_3_3_bits_vc(vif.vc[3][3]), - .io_out_3_3_valid(vif.out_valid[3][3]), - .io_out_3_3_ready(vif.out_ready[3][3]), - .io_out_3_3_bits_payload(vif.out_data[3][3]), - .io_out_3_3_bits_srcX(vif.out_src_x[3][3]), - .io_out_3_3_bits_srcY(vif.out_src_y[3][3]), - .io_out_3_3_bits_head(vif.out_head[3][3]), - .io_out_3_3_bits_tail(vif.out_tail[3][3]), - .io_in_4_0_valid(vif.valid[4][0]), - .io_in_4_0_ready(vif.ready[4][0]), - .io_in_4_0_bits_dstX(vif.x[4][0]), - .io_in_4_0_bits_dstY(vif.y[4][0]), - .io_in_4_0_bits_head(vif.head[4][0]), - .io_in_4_0_bits_tail(vif.tail[4][0]), - .io_in_4_0_bits_payload(vif.data[4][0]), - .io_in_4_0_bits_srcX(vif.src_x[4][0]), - .io_in_4_0_bits_srcY(vif.src_y[4][0]), - .io_in_4_0_bits_vc(vif.vc[4][0]), - .io_out_4_0_valid(vif.out_valid[4][0]), - .io_out_4_0_ready(vif.out_ready[4][0]), - .io_out_4_0_bits_payload(vif.out_data[4][0]), - .io_out_4_0_bits_srcX(vif.out_src_x[4][0]), - .io_out_4_0_bits_srcY(vif.out_src_y[4][0]), - .io_out_4_0_bits_head(vif.out_head[4][0]), - .io_out_4_0_bits_tail(vif.out_tail[4][0]), - .io_in_4_1_valid(vif.valid[4][1]), - .io_in_4_1_ready(vif.ready[4][1]), - .io_in_4_1_bits_dstX(vif.x[4][1]), - .io_in_4_1_bits_dstY(vif.y[4][1]), - .io_in_4_1_bits_head(vif.head[4][1]), - .io_in_4_1_bits_tail(vif.tail[4][1]), - .io_in_4_1_bits_payload(vif.data[4][1]), - .io_in_4_1_bits_srcX(vif.src_x[4][1]), - .io_in_4_1_bits_srcY(vif.src_y[4][1]), - .io_in_4_1_bits_vc(vif.vc[4][1]), - .io_out_4_1_valid(vif.out_valid[4][1]), - .io_out_4_1_ready(vif.out_ready[4][1]), - .io_out_4_1_bits_payload(vif.out_data[4][1]), - .io_out_4_1_bits_srcX(vif.out_src_x[4][1]), - .io_out_4_1_bits_srcY(vif.out_src_y[4][1]), - .io_out_4_1_bits_head(vif.out_head[4][1]), - .io_out_4_1_bits_tail(vif.out_tail[4][1]), - .io_in_4_2_valid(vif.valid[4][2]), - .io_in_4_2_ready(vif.ready[4][2]), - .io_in_4_2_bits_dstX(vif.x[4][2]), - .io_in_4_2_bits_dstY(vif.y[4][2]), - .io_in_4_2_bits_head(vif.head[4][2]), - .io_in_4_2_bits_tail(vif.tail[4][2]), - .io_in_4_2_bits_payload(vif.data[4][2]), - .io_in_4_2_bits_srcX(vif.src_x[4][2]), - .io_in_4_2_bits_srcY(vif.src_y[4][2]), - .io_in_4_2_bits_vc(vif.vc[4][2]), - .io_out_4_2_valid(vif.out_valid[4][2]), - .io_out_4_2_ready(vif.out_ready[4][2]), - .io_out_4_2_bits_payload(vif.out_data[4][2]), - .io_out_4_2_bits_srcX(vif.out_src_x[4][2]), - .io_out_4_2_bits_srcY(vif.out_src_y[4][2]), - .io_out_4_2_bits_head(vif.out_head[4][2]), - .io_out_4_2_bits_tail(vif.out_tail[4][2]), - .io_in_4_3_valid(vif.valid[4][3]), - .io_in_4_3_ready(vif.ready[4][3]), - .io_in_4_3_bits_dstX(vif.x[4][3]), - .io_in_4_3_bits_dstY(vif.y[4][3]), - .io_in_4_3_bits_head(vif.head[4][3]), - .io_in_4_3_bits_tail(vif.tail[4][3]), - .io_in_4_3_bits_payload(vif.data[4][3]), - .io_in_4_3_bits_srcX(vif.src_x[4][3]), - .io_in_4_3_bits_srcY(vif.src_y[4][3]), - .io_in_4_3_bits_vc(vif.vc[4][3]), - .io_out_4_3_valid(vif.out_valid[4][3]), - .io_out_4_3_ready(vif.out_ready[4][3]), - .io_out_4_3_bits_payload(vif.out_data[4][3]), - .io_out_4_3_bits_srcX(vif.out_src_x[4][3]), - .io_out_4_3_bits_srcY(vif.out_src_y[4][3]), - .io_out_4_3_bits_head(vif.out_head[4][3]), - .io_out_4_3_bits_tail(vif.out_tail[4][3]) - ); - initial begin - uvm_config_db#(virtual mesh_fabric_if)::set(null, "uvm_test_top", "vif", vif); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/ChiMesh.scala b/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/ChiMesh.scala deleted file mode 100644 index ec2e28ddf..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/ChiMesh.scala +++ /dev/null @@ -1,475 +0,0 @@ -package hier.chip.mesh - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi._ - -/** - * CHI uses four non-overlapping Mesh VCs. A production tile mesh reserves at - * least four more VCs for bulk DMA and future system traffic. - */ -object ChiMeshVirtualChannel { - val Request = 0 - val Response = 1 - val Data = 2 - val Snoop = 3 - val Required = 4 -} - -/** - * Static placement of CHI NodeIDs with an explicit set of populated endpoints. - */ -case class ChiMeshNodeMap( - xByNode: Seq[Int], - yByNode: Seq[Int], - mesh: MeshParams, - presentNodes: Set[Int]) { - require(xByNode.nonEmpty && xByNode.size == yByNode.size) - require(presentNodes.nonEmpty && presentNodes.forall(node => node >= 0 && node < xByNode.size)) - require(xByNode.forall(x => x >= 0 && x < mesh.xNodes)) - require(yByNode.forall(y => y >= 0 && y < mesh.yNodes)) - def coordinateBits: Int = math.max(1, log2Ceil(xByNode.size)) -} - -/** A packed CHI flit with the routing destination kept outside protocol bits. */ -class ChiMeshMessage(flitBits: Int, nodeIdBits: Int) extends Bundle { - val targetNode = UInt(nodeIdBits.W) - val flit = UInt(flitBits.W) -} - -/** - * Turns one atomic CHI flit into a fixed-length wormhole packet. - * - * CHI channel identity is carried by the mesh VC, so a packetizer instance is - * dedicated to one channel. `targetNode` is routing metadata, never serialized - * into the CHI flit. This makes request, response, data and snoop packetizers - * identical even though their target fields differ. - */ -@instantiable -class ChiMeshPacketizer( - mesh: MeshParams, - flitBits: Int, - virtualChannel: Int, - localX: Int, - localY: Int, - nodeMap: ChiMeshNodeMap) - extends Module { - require(flitBits > 0) - require(virtualChannel >= 0 && virtualChannel < mesh.virtualChannels) - require(localX >= 0 && localX < mesh.xNodes && localY >= 0 && localY < mesh.yNodes) - require(nodeMap.mesh == mesh) - - val chunks = (flitBits + mesh.payloadBits - 1) / mesh.payloadBits - val beatBits = math.max(1, log2Ceil(chunks)) - val paddedBits = chunks * mesh.payloadBits - val message = new ChiMeshMessage(flitBits, nodeMap.coordinateBits) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(message)) - val out = Decoupled(new MeshFlit(mesh)) - }) - - val busy = RegInit(false.B) - val beat = RegInit(0.U(beatBits.W)) - val flit = Reg(UInt(flitBits.W)) - val target = Reg(UInt(nodeMap.coordinateBits.W)) - val xLookup = VecInit(nodeMap.xByNode.map(_.U(mesh.xBits.W))) - val yLookup = VecInit(nodeMap.yByNode.map(_.U(mesh.yBits.W))) - val present = VecInit(nodeMap.xByNode.indices.map(node => nodeMap.presentNodes.contains(node).B)) - val padded = Wire(UInt(paddedBits.W)) - padded := flit - - val placed = io.in.bits.targetNode < nodeMap.xByNode.size.U && present(io.in.bits.targetNode) - io.in.ready := !busy && placed - when(io.in.valid)(assert(placed, "CHI Mesh target NodeID is not placed")) - when(io.in.fire) { - flit := io.in.bits.flit - target := io.in.bits.targetNode - beat := 0.U - busy := true.B - } - - io.out.valid := busy - io.out.bits.srcX := localX.U - io.out.bits.srcY := localY.U - io.out.bits.dstX := xLookup(target) - io.out.bits.dstY := yLookup(target) - io.out.bits.vc := virtualChannel.U - io.out.bits.head := beat === 0.U - io.out.bits.tail := beat === (chunks - 1).U - io.out.bits.payload := (padded >> (beat * mesh.payloadBits.U))(mesh.payloadBits - 1, 0) - when(io.out.fire) { - when(io.out.bits.tail) { - busy := false.B - }.otherwise { - beat := beat + 1.U - } - } -} - -/** Reassembles one fixed-length CHI packet after the mesh VC has been demultiplexed. */ -@instantiable -class ChiMeshDepacketizer(mesh: MeshParams, flitBits: Int, virtualChannel: Int) extends Module { - require(flitBits > 0) - require(virtualChannel >= 0 && virtualChannel < mesh.virtualChannels) - val chunks = (flitBits + mesh.payloadBits - 1) / mesh.payloadBits - val beatBits = math.max(1, log2Ceil(chunks)) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshFlit(mesh))) - val out = Decoupled(UInt(flitBits.W)) - }) - - val collecting = RegInit(false.B) - val beat = RegInit(0.U(beatBits.W)) - val words = Reg(Vec(math.max(1, chunks - 1), UInt(mesh.payloadBits.W))) - val result = Reg(UInt(flitBits.W)) - val resultValid = RegInit(false.B) - - io.in.ready := !resultValid - io.out.valid := resultValid - io.out.bits := result - when(io.in.fire) { - assert(io.in.bits.vc === virtualChannel.U, "CHI packet arrived on wrong mesh VC") - when(!collecting) { - assert(io.in.bits.head, "CHI mesh packet started without head") - }.otherwise { - assert(!io.in.bits.head, "CHI mesh packet has a second head") - } - assert(io.in.bits.tail === (beat === (chunks - 1).U), "CHI mesh packet has wrong length") - if (chunks > 1) { when(!io.in.bits.tail)(words(beat) := io.in.bits.payload) } - when(io.in.bits.tail) { - result := (if (chunks == 1) io.in.bits.payload else Cat(Seq(io.in.bits.payload) ++ words.reverse))(flitBits - 1, 0) - resultValid := true.B - collecting := false.B - beat := 0.U - }.otherwise { - collecting := true.B - beat := beat + 1.U - } - } - when(io.out.fire) { - resultValid := false.B - } -} - -/** - * One tile's CHI-facing Mesh endpoint. - * - * It owns packet boundaries and CHI channel-to-VC assignment. Its Mesh ports - * carry opaque packets, so a router can be changed without changing cache or - * Home protocol state. Snoop destinations are explicit sideband because CHI - * Snp flits identify their source but do not contain a target NodeID. - */ -@instantiable -class ChiMeshEndpoint( - p: Params, - mesh: MeshParams, - localX: Int, - localY: Int, - nodeMap: ChiMeshNodeMap) - extends Module { - require(mesh.virtualChannels >= ChiMeshVirtualChannel.Required) - require(nodeMap.mesh == mesh && nodeMap.coordinateBits == p.nodeIdBits) - val reqBits = (new RequestFlit(p)).flitWidth - val rspBits = (new ResponseFlit(p)).flitWidth - val datBits = (new DataFlit(p)).flitWidth - val snpBits = (new SnoopFlit(p)).flitWidth - val message = (bits: Int) => new ChiMeshMessage(bits, p.nodeIdBits) - - @public - val io = IO(new Bundle { - val txReq = Flipped(Decoupled(message(reqBits))) - val txRsp = Flipped(Decoupled(message(rspBits))) - val txDat = Flipped(Decoupled(message(datBits))) - val txSnp = Flipped(Decoupled(message(snpBits))) - val rxReq = Decoupled(UInt(reqBits.W)) - val rxRsp = Decoupled(UInt(rspBits.W)) - val rxDat = Decoupled(UInt(datBits.W)) - val rxSnp = Decoupled(UInt(snpBits.W)) - val meshOut = Vec(mesh.virtualChannels, Decoupled(new MeshFlit(mesh))) - val meshIn = Vec(mesh.virtualChannels, Flipped(Decoupled(new MeshFlit(mesh)))) - }) - - val txReq: Instance[ChiMeshPacketizer] = - Instantiate(new ChiMeshPacketizer(mesh, reqBits, ChiMeshVirtualChannel.Request, localX, localY, nodeMap)) - val txRsp: Instance[ChiMeshPacketizer] = - Instantiate(new ChiMeshPacketizer(mesh, rspBits, ChiMeshVirtualChannel.Response, localX, localY, nodeMap)) - val txDat: Instance[ChiMeshPacketizer] = - Instantiate(new ChiMeshPacketizer(mesh, datBits, ChiMeshVirtualChannel.Data, localX, localY, nodeMap)) - val txSnp: Instance[ChiMeshPacketizer] = - Instantiate(new ChiMeshPacketizer(mesh, snpBits, ChiMeshVirtualChannel.Snoop, localX, localY, nodeMap)) - txReq.io.in <> io.txReq - txRsp.io.in <> io.txRsp - txDat.io.in <> io.txDat - txSnp.io.in <> io.txSnp - io.meshOut(ChiMeshVirtualChannel.Request) <> txReq.io.out - io.meshOut(ChiMeshVirtualChannel.Response) <> txRsp.io.out - io.meshOut(ChiMeshVirtualChannel.Data) <> txDat.io.out - io.meshOut(ChiMeshVirtualChannel.Snoop) <> txSnp.io.out - for (vc <- ChiMeshVirtualChannel.Required until mesh.virtualChannels) { - io.meshOut(vc).valid := false.B - io.meshOut(vc).bits := 0.U.asTypeOf(new MeshFlit(mesh)) - io.meshIn(vc).ready := false.B - } - val rxReq: Instance[ChiMeshDepacketizer] = - Instantiate(new ChiMeshDepacketizer(mesh, reqBits, ChiMeshVirtualChannel.Request)) - val rxRsp: Instance[ChiMeshDepacketizer] = - Instantiate(new ChiMeshDepacketizer(mesh, rspBits, ChiMeshVirtualChannel.Response)) - val rxDat: Instance[ChiMeshDepacketizer] = - Instantiate(new ChiMeshDepacketizer(mesh, datBits, ChiMeshVirtualChannel.Data)) - val rxSnp: Instance[ChiMeshDepacketizer] = - Instantiate(new ChiMeshDepacketizer(mesh, snpBits, ChiMeshVirtualChannel.Snoop)) - val receivers = Seq(rxReq, rxRsp, rxDat, rxSnp) - for ((receiver, vc) <- receivers.zipWithIndex) { receiver.io.in <> io.meshIn(vc) } - io.rxReq <> rxReq.io.out - io.rxRsp <> rxRsp.io.out - io.rxDat <> rxDat.io.out - io.rxSnp <> rxSnp.io.out -} - -/** Typed adapter for a cache or NPU CHI requester port. */ -@instantiable -class ChiMeshRequesterEndpoint( - p: Params, - mesh: MeshParams, - localX: Int, - localY: Int, - nodeMap: ChiMeshNodeMap) - extends Module { - - @public - val io = IO(new Bundle { - val chi = Flipped(new RequesterPort(p)) - val meshOut = Vec(mesh.virtualChannels, Decoupled(new MeshFlit(mesh))) - val meshIn = Vec(mesh.virtualChannels, Flipped(Decoupled(new MeshFlit(mesh)))) - }) - - val endpoint: Instance[ChiMeshEndpoint] = Instantiate(new ChiMeshEndpoint(p, mesh, localX, localY, nodeMap)) - endpoint.io.txReq.valid := io.chi.req.valid - endpoint.io.txReq.bits.targetNode := io.chi.req.bits.tgtId - endpoint.io.txReq.bits.flit := io.chi.req.bits.packed - io.chi.req.ready := endpoint.io.txReq.ready - endpoint.io.txRsp.valid := io.chi.txRsp.valid - endpoint.io.txRsp.bits.targetNode := io.chi.txRsp.bits.tgtId - endpoint.io.txRsp.bits.flit := io.chi.txRsp.bits.packed - io.chi.txRsp.ready := endpoint.io.txRsp.ready - endpoint.io.txDat.valid := io.chi.txDat.valid - endpoint.io.txDat.bits.targetNode := io.chi.txDat.bits.tgtId - endpoint.io.txDat.bits.flit := io.chi.txDat.bits.packed - io.chi.txDat.ready := endpoint.io.txDat.ready - endpoint.io.txSnp.valid := false.B - endpoint.io.txSnp.bits := 0.U.asTypeOf(endpoint.io.txSnp.bits) - endpoint.io.rxRsp.ready := io.chi.rxRsp.ready - io.chi.rxRsp.valid := endpoint.io.rxRsp.valid - io.chi.rxRsp.bits.unpack(endpoint.io.rxRsp.bits) - endpoint.io.rxDat.ready := io.chi.rxDat.ready - io.chi.rxDat.valid := endpoint.io.rxDat.valid - io.chi.rxDat.bits.unpack(endpoint.io.rxDat.bits) - endpoint.io.rxSnp.ready := io.chi.snp.ready - io.chi.snp.valid := endpoint.io.rxSnp.valid - io.chi.snp.bits.unpack(endpoint.io.rxSnp.bits) - endpoint.io.rxReq.ready := false.B - when(endpoint.io.rxReq.valid)(assert(false.B, "CHI requester endpoint received a REQ")) - io.meshOut <> endpoint.io.meshOut - endpoint.io.meshIn <> io.meshIn -} - -/** Typed adapter for one directory Home and all of its snoop targets. */ -@instantiable -class ChiMeshHomeEndpoint( - p: Params, - mesh: MeshParams, - localX: Int, - localY: Int, - nodeMap: ChiMeshNodeMap, - agents: Int) - extends Module { - require(agents >= 1 && agents < (1 << p.nodeIdBits)) - val snpBits = (new SnoopFlit(p)).flitWidth - - @public - val io = IO(new Bundle { - val req = Decoupled(new RequestFlit(p)) - val rxRsp = Decoupled(new ResponseFlit(p)) - val rxDat = Decoupled(new DataFlit(p)) - val rsp = Flipped(Decoupled(new ResponseFlit(p))) - val dat = Flipped(Decoupled(new DataFlit(p))) - val snp = Vec(agents, Flipped(Decoupled(new SnoopFlit(p)))) - val meshOut = Vec(mesh.virtualChannels, Decoupled(new MeshFlit(mesh))) - val meshIn = Vec(mesh.virtualChannels, Flipped(Decoupled(new MeshFlit(mesh)))) - }) - - val endpoint: Instance[ChiMeshEndpoint] = Instantiate(new ChiMeshEndpoint(p, mesh, localX, localY, nodeMap)) - endpoint.io.txReq.valid := false.B - endpoint.io.txReq.bits := 0.U.asTypeOf(endpoint.io.txReq.bits) - endpoint.io.txRsp.valid := io.rsp.valid - endpoint.io.txRsp.bits.targetNode := io.rsp.bits.tgtId - endpoint.io.txRsp.bits.flit := io.rsp.bits.packed - io.rsp.ready := endpoint.io.txRsp.ready - endpoint.io.txDat.valid := io.dat.valid - endpoint.io.txDat.bits.targetNode := io.dat.bits.tgtId - endpoint.io.txDat.bits.flit := io.dat.bits.packed - io.dat.ready := endpoint.io.txDat.ready - val snpArbiter = Module(new RRArbiter(new ChiMeshMessage(snpBits, p.nodeIdBits), agents)) - for (agent <- 0 until agents) { - snpArbiter.io.in(agent).valid := io.snp(agent).valid - snpArbiter.io.in(agent).bits.targetNode := (agent + 1).U - snpArbiter.io.in(agent).bits.flit := io.snp(agent).bits.packed - io.snp(agent).ready := snpArbiter.io.in(agent).ready - } - endpoint.io.txSnp <> snpArbiter.io.out - endpoint.io.rxReq.ready := io.req.ready - io.req.valid := endpoint.io.rxReq.valid - io.req.bits.unpack(endpoint.io.rxReq.bits) - endpoint.io.rxRsp.ready := io.rxRsp.ready - io.rxRsp.valid := endpoint.io.rxRsp.valid - io.rxRsp.bits.unpack(endpoint.io.rxRsp.bits) - endpoint.io.rxDat.ready := io.rxDat.ready - io.rxDat.valid := endpoint.io.rxDat.valid - io.rxDat.bits.unpack(endpoint.io.rxDat.bits) - endpoint.io.rxSnp.ready := true.B - when(endpoint.io.rxSnp.valid)(assert(false.B, "Home received a CHI snoop")) - io.meshOut <> endpoint.io.meshOut - endpoint.io.meshIn <> io.meshIn -} - -/** Native verification target for Home snoop target-sideband routing. */ -class ChiMeshHomeSnoopLoopback extends Module { - val chi = Params() - val mesh = MeshParams(xNodes = 2, yNodes = 1, payloadBits = 32, virtualChannels = 8) - - val map = ChiMeshNodeMap( - Seq.tabulate(1 << chi.nodeIdBits)(node => if (node == 1) 1 else 0), - Seq.fill(1 << chi.nodeIdBits)(0), - mesh, - presentNodes = (0 until (1 << chi.nodeIdBits)).toSet - ) - - val snpBits = (new SnoopFlit(chi)).flitWidth - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(UInt(snpBits.W))) - val out = Decoupled(UInt(snpBits.W)) - }) - - val home: Instance[ChiMeshHomeEndpoint] = - Instantiate(new ChiMeshHomeEndpoint(chi, mesh, localX = 0, localY = 0, nodeMap = map, agents = 2)) - val receiver: Instance[ChiMeshEndpoint] = - Instantiate(new ChiMeshEndpoint(chi, mesh, localX = 1, localY = 0, nodeMap = map)) - home.io.snp(0).valid := io.in.valid - home.io.snp(0).bits.unpack(io.in.bits) - io.in.ready := home.io.snp(0).ready - home.io.snp(1).valid := false.B - home.io.snp(1).bits := 0.U.asTypeOf(home.io.snp(1).bits) - home.io.rsp.valid := false.B - home.io.rsp.bits := 0.U.asTypeOf(home.io.rsp.bits) - home.io.dat.valid := false.B - home.io.dat.bits := 0.U.asTypeOf(home.io.dat.bits) - home.io.req.ready := true.B - home.io.rxRsp.ready := true.B - home.io.rxDat.ready := true.B - receiver.io.txReq.valid := false.B - receiver.io.txReq.bits := 0.U.asTypeOf(receiver.io.txReq.bits) - receiver.io.txRsp.valid := false.B - receiver.io.txRsp.bits := 0.U.asTypeOf(receiver.io.txRsp.bits) - receiver.io.txDat.valid := false.B - receiver.io.txDat.bits := 0.U.asTypeOf(receiver.io.txDat.bits) - receiver.io.txSnp.valid := false.B - receiver.io.txSnp.bits := 0.U.asTypeOf(receiver.io.txSnp.bits) - receiver.io.rxReq.ready := true.B - receiver.io.rxRsp.ready := true.B - receiver.io.rxDat.ready := true.B - io.out <> receiver.io.rxSnp - receiver.io.meshIn <> home.io.meshOut - for (vc <- 0 until mesh.virtualChannels) { - home.io.meshIn(vc).valid := false.B - home.io.meshIn(vc).bits := 0.U.asTypeOf(new MeshFlit(mesh)) - receiver.io.meshOut(vc).ready := true.B - } -} - -/** Native endpoint verification target: a real CHI Dat channel traverses its Mesh endpoint. */ -class ChiMeshEndpointLoopback extends Module { - val chi = Params() - val mesh = MeshParams(xNodes = 2, yNodes = 2, payloadBits = 32, virtualChannels = 8) - - val map = ChiMeshNodeMap( - Seq.tabulate(1 << chi.nodeIdBits)(_ & 1), - Seq.tabulate(1 << chi.nodeIdBits)(node => (node >> 1) & 1), - mesh, - presentNodes = (0 until (1 << chi.nodeIdBits)).toSet - ) - - val datBits = (new DataFlit(chi)).flitWidth - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new ChiMeshMessage(datBits, chi.nodeIdBits))) - val out = Decoupled(UInt(datBits.W)) - }) - - val endpoint: Instance[ChiMeshEndpoint] = - Instantiate(new ChiMeshEndpoint(chi, mesh, localX = 0, localY = 0, nodeMap = map)) - endpoint.io.txDat <> io.in - endpoint.io.txReq.valid := false.B - endpoint.io.txReq.bits := 0.U.asTypeOf(endpoint.io.txReq.bits) - endpoint.io.txRsp.valid := false.B - endpoint.io.txRsp.bits := 0.U.asTypeOf(endpoint.io.txRsp.bits) - endpoint.io.txSnp.valid := false.B - endpoint.io.txSnp.bits := 0.U.asTypeOf(endpoint.io.txSnp.bits) - endpoint.io.rxReq.ready := true.B - endpoint.io.rxRsp.ready := true.B - endpoint.io.rxSnp.ready := true.B - io.out <> endpoint.io.rxDat - endpoint.io.meshIn <> endpoint.io.meshOut -} - -/** Native verification target for packetization and reassembly without a mesh topology. */ -class ChiMeshCodecLoopback extends Module { - val mesh = MeshParams(xNodes = 2, yNodes = 2, payloadBits = 32, virtualChannels = 4) - val chi = Params() - - val map = ChiMeshNodeMap( - Seq.tabulate(1 << chi.nodeIdBits)(_ & 1), - Seq.tabulate(1 << chi.nodeIdBits)(node => (node >> 1) & 1), - mesh, - presentNodes = Set(1, 2, 3, 64) - ) - - val flitBits = (new DataFlit(chi)).flitWidth - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new ChiMeshMessage(flitBits, map.coordinateBits))) - val out = Decoupled(UInt(flitBits.W)) - val pause = Input(Bool()) - val observedFire = Output(Bool()) - val observedValid = Output(Bool()) - val observed = Output(new MeshFlit(mesh)) - }) - - val tx: Instance[ChiMeshPacketizer] = Instantiate(new ChiMeshPacketizer( - mesh, - flitBits, - virtualChannel = ChiMeshVirtualChannel.Data, - localX = 0, - localY = 0, - nodeMap = map - )) - - val rx: Instance[ChiMeshDepacketizer] = - Instantiate(new ChiMeshDepacketizer(mesh, flitBits, virtualChannel = ChiMeshVirtualChannel.Data)) - io.in <> tx.io.in - rx.io.in.valid := tx.io.out.valid && !io.pause - rx.io.in.bits := tx.io.out.bits - tx.io.out.ready := rx.io.in.ready && !io.pause - io.out <> rx.io.out - io.observedFire := tx.io.out.fire - io.observedValid := tx.io.out.valid - io.observed := tx.io.out.bits -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/Emit.scala b/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/Emit.scala deleted file mode 100644 index f80e06dee..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/Emit.scala +++ /dev/null @@ -1,35 +0,0 @@ -package hier.chip.mesh - -object Emit extends App { - val p = MeshParams(xNodes = 3, yNodes = 3, payloadBits = 32, virtualChannels = 4) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new MeshRouter(p, 1, 1), - firtoolOpts = args, - args = Array("--target-dir", "build/MeshRouter", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new MeshCreditNetwork(MeshParams(2, 2, 32, 4), linkDepth = 2), - firtoolOpts = args, - args = Array("--target-dir", "build/MeshCreditNetwork", "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new ChiMeshCodecLoopback, - firtoolOpts = args, - args = Array("--target-dir", "build/ChiMeshCodecLoopback", "--split-verilog") - ) - val chi = memcore.bus.chi.Params() - val endpointMesh = MeshParams(2, 2, 32, 4) - - val endpointNodes = ChiMeshNodeMap( - Seq.tabulate(1 << chi.nodeIdBits)(_ & 1), - Seq.tabulate(1 << chi.nodeIdBits)(node => (node >> 1) & 1), - endpointMesh, - presentNodes = Set(1, 2, 64) - ) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new ChiMeshRequesterEndpoint(chi, endpointMesh, 1, 0, endpointNodes), - firtoolOpts = args, - args = Array("--target-dir", "build/ChiMeshRequesterEndpoint", "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/Mesh.scala b/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/Mesh.scala deleted file mode 100644 index 02fd950a9..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/Mesh.scala +++ /dev/null @@ -1,126 +0,0 @@ -package hier.chip.mesh - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -object MeshDirection { - val Local = 0 - val North = 1 - val South = 2 - val East = 3 - val West = 4 - val Count = 5 -} - -case class MeshParams( - xNodes: Int, - yNodes: Int, - payloadBits: Int, - virtualChannels: Int = 2) { - require(xNodes >= 1 && yNodes >= 1) - require(payloadBits > 0) - require(virtualChannels >= 1 && isPow2(virtualChannels)) - val xBits = math.max(1, log2Ceil(xNodes)) - val yBits = math.max(1, log2Ceil(yNodes)) - val vcBits = math.max(1, log2Ceil(virtualChannels)) -} - -/** Generic wormhole flit. Packet content is opaque to the mesh. */ -class MeshFlit(p: MeshParams) extends Bundle { - val srcX = UInt(p.xBits.W) - val srcY = UInt(p.yBits.W) - val dstX = UInt(p.xBits.W) - val dstY = UInt(p.yBits.W) - val vc = UInt(p.vcBits.W) - val head = Bool() - val tail = Bool() - val payload = UInt(p.payloadBits.W) -} - -/** Deterministic XY routing; only physically connected directions own state. */ -@instantiable -class MeshRouter(p: MeshParams, x: Int, y: Int) extends Module { - require(x >= 0 && x < p.xNodes && y >= 0 && y < p.yNodes) - val flit = new MeshFlit(p) - - val connected = Seq(MeshDirection.Local) ++ - Seq(MeshDirection.North).filter(_ => y > 0) ++ - Seq(MeshDirection.South).filter(_ => y < p.yNodes - 1) ++ - Seq(MeshDirection.East).filter(_ => x < p.xNodes - 1) ++ - Seq(MeshDirection.West).filter(_ => x > 0) - - val count = connected.size - val inputBits = math.max(1, log2Ceil(count)) - - @public - val io = IO(new Bundle { - val in = Vec(MeshDirection.Count, Vec(p.virtualChannels, Flipped(Decoupled(flit)))) - val out = Vec(MeshDirection.Count, Vec(p.virtualChannels, Decoupled(flit))) - }) - - def route(value: MeshFlit): UInt = Mux( - value.dstX < x.U, - MeshDirection.West.U, - Mux( - value.dstX > x.U, - MeshDirection.East.U, - Mux(value.dstY < y.U, MeshDirection.North.U, Mux(value.dstY > y.U, MeshDirection.South.U, MeshDirection.Local.U)) - ) - ) - - for { - direction <- (0 until MeshDirection.Count).filterNot(connected.contains) - vc <- 0 until p.virtualChannels - } { - io.in(direction)(vc).ready := false.B - io.out(direction)(vc).valid := false.B - io.out(direction)(vc).bits := 0.U.asTypeOf(flit) - } - for (vc <- 0 until p.virtualChannels) { - val inputs = connected.map(direction => io.in(direction)(vc)) - val payloads = VecInit(inputs.map(_.bits)) - val routes = inputs.map(input => route(input.bits)) - val owners = connected.map { out => - val lockValid = RegInit(false.B) - val lockInput = Reg(UInt(inputBits.W)) - val cursor = RegInit(0.U(inputBits.W)) - val request = VecInit(inputs.indices.map(in => inputs(in).valid && routes(in) === out.U)) - val ordered = VecInit(inputs.indices.map(offset => request((cursor +& offset.U) % count.U))) - val fairChoice = (cursor +& PriorityEncoder(ordered)) % count.U - val selected = Mux(lockValid, lockInput, fairChoice) - val valid = Mux(lockValid, VecInit(inputs.map(_.valid))(lockInput), request.asUInt.orR) - io.out(out)(vc).valid := valid - io.out(out)(vc).bits := payloads(selected) - when(valid && !io.out(out)(vc).ready && !lockValid) { - lockValid := true.B - lockInput := selected - } - when(io.out(out)(vc).fire) { - when(io.out(out)(vc).bits.tail) { - lockValid := false.B - cursor := Mux(selected === (count - 1).U, 0.U, selected + 1.U) - }.otherwise { - lockValid := true.B - lockInput := selected - } - } - (out, selected, lockValid, lockInput) - } - for ((input, index) <- inputs.zipWithIndex) { - input.ready := owners.map { case (out, selected, _, _) => - io.out(out)(vc).ready && io.out(out)(vc).valid && selected === index.U - }.reduce(_ || _) - when(input.fire) { - assert(input.bits.vc === vc.U, "Mesh flit arrived on wrong VC port") - assert(input.bits.dstX < p.xNodes.U && input.bits.dstY < p.yNodes.U, "Mesh destination outside topology") - when(!input.bits.head) { - val owned = owners.map { case (out, _, locked, owner) => - routes(index) === out.U && locked && owner === index.U - }.reduce(_ || _) - assert(owned, "Mesh packet body without ownership") - } - } - } - } -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/MeshBulk.scala b/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/MeshBulk.scala deleted file mode 100644 index a8c5199d9..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/MeshBulk.scala +++ /dev/null @@ -1,192 +0,0 @@ -package hier.chip.mesh - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.axi.Beat - -/** One AXI-stream beat plus the Mesh destination selected at packet start. */ -class MeshBulkBeat(dataBits: Int, nodeIdBits: Int) extends Bundle { - require(dataBits > 0 && dataBits % 8 == 0) - val targetNode = UInt(nodeIdBits.W) - val data = UInt(dataBits.W) - val keep = UInt((dataBits / 8).W) - val last = Bool() -} - -/** VC5 control descriptor for one remote scratchpad DMA transaction. */ -class BulkDescriptor(nodeIdBits: Int, addressBits: Int = 32) extends Bundle { - val sourceNode = UInt(nodeIdBits.W) - val targetNode = UInt(nodeIdBits.W) - val write = Bool() - val scratchpadAddr = UInt(addressBits.W) - val bytes = UInt(32.W) - val txnId = UInt(8.W) -} - -/** VC6 acknowledgement or terminal completion for a bulk descriptor. */ -class BulkCompletion(nodeIdBits: Int) extends Bundle { - val targetNode = UInt(nodeIdBits.W) - val txnId = UInt(8.W) - val ready = Bool() - val error = Bool() -} - -/** One-flit control packet for VC5 descriptors and VC6 acknowledgements. */ -class MeshBulkControlTx( - mesh: MeshParams, - payloadBits: Int, - virtualChannel: Int, - localX: Int, - localY: Int, - nodeMap: ChiMeshNodeMap) - extends Module { - require(payloadBits > 0 && payloadBits <= mesh.payloadBits) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshBulkBeat(payloadBits, nodeMap.coordinateBits))) - val out = Decoupled(new MeshFlit(mesh)) - }) - - val xLookup = VecInit(nodeMap.xByNode.map(_.U(mesh.xBits.W))) - val yLookup = VecInit(nodeMap.yByNode.map(_.U(mesh.yBits.W))) - io.out.valid := io.in.valid - io.out.bits.srcX := localX.U - io.out.bits.srcY := localY.U - io.out.bits.dstX := xLookup(io.in.bits.targetNode) - io.out.bits.dstY := yLookup(io.in.bits.targetNode) - io.out.bits.vc := virtualChannel.U - io.out.bits.head := true.B - io.out.bits.tail := true.B - io.out.bits.payload := io.in.bits.data - io.in.ready := io.out.ready -} - -class MeshBulkControlRx(mesh: MeshParams, payloadBits: Int, virtualChannel: Int) extends Module { - require(payloadBits > 0 && payloadBits <= mesh.payloadBits) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshFlit(mesh))) - val out = Decoupled(UInt(payloadBits.W)) - }) - - io.out.valid := io.in.valid - io.out.bits := io.in.bits.payload(payloadBits - 1, 0) - io.in.ready := io.out.ready - when(io.in.fire) { - assert( - io.in.bits.vc === virtualChannel.U && io.in.bits.head && io.in.bits.tail, - "Bulk control packet must be one flit on its assigned VC" - ) - } -} - -/** - * Maps one AXI-stream transaction to one non-interleaved Mesh packet. - * - * Mesh `head` is the first AXI beat and `tail` is the beat carrying AXI - * `last`. The Mesh router therefore holds the selected output across the full - * long stream, rather than arbitrating again at every beat. - */ -@instantiable -class MeshBulkPacketizer( - mesh: MeshParams, - dataBits: Int, - virtualChannel: Int, - localX: Int, - localY: Int, - nodeMap: ChiMeshNodeMap) - extends Module { - require(mesh.payloadBits >= dataBits + dataBits / 8) - require(virtualChannel >= 0 && virtualChannel < mesh.virtualChannels) - require(nodeMap.mesh == mesh) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshBulkBeat(dataBits, nodeMap.coordinateBits))) - val out = Decoupled(new MeshFlit(mesh)) - }) - - val first = RegInit(true.B) - val target = Reg(UInt(nodeMap.coordinateBits.W)) - val xLookup = VecInit(nodeMap.xByNode.map(_.U(mesh.xBits.W))) - val yLookup = VecInit(nodeMap.yByNode.map(_.U(mesh.yBits.W))) - val selectedTarget = Mux(first, io.in.bits.targetNode, target) - val payload = Wire(UInt(mesh.payloadBits.W)) - payload := Cat(io.in.bits.keep, io.in.bits.data) - io.out.valid := io.in.valid - io.out.bits.srcX := localX.U - io.out.bits.srcY := localY.U - io.out.bits.dstX := xLookup(selectedTarget) - io.out.bits.dstY := yLookup(selectedTarget) - io.out.bits.vc := virtualChannel.U - io.out.bits.head := first - io.out.bits.tail := io.in.bits.last - io.out.bits.payload := payload - io.in.ready := io.out.ready - when(io.in.fire) { - assert(io.in.bits.targetNode < nodeMap.xByNode.size.U, "Bulk Mesh target NodeID is not placed") - when(!first)(assert(io.in.bits.targetNode === target, "Bulk Mesh target changed inside AXI packet")) - target := io.in.bits.targetNode - first := io.in.bits.last - } -} - -/** Recreates AXI-stream beat boundaries from a single bulk Mesh VC. */ -@instantiable -class MeshBulkDepacketizer(mesh: MeshParams, dataBits: Int, virtualChannel: Int) extends Module { - require(mesh.payloadBits >= dataBits + dataBits / 8) - require(virtualChannel >= 0 && virtualChannel < mesh.virtualChannels) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshFlit(mesh))) - val out = Decoupled(new Beat(dataBits)) - }) - - val first = RegInit(true.B) - io.out.valid := io.in.valid - io.out.bits.data := io.in.bits.payload(dataBits - 1, 0) - io.out.bits.keep := io.in.bits.payload(dataBits + dataBits / 8 - 1, dataBits) - io.out.bits.last := io.in.bits.tail - io.out.bits.id := 0.U - io.out.bits.dest := 0.U - io.out.bits.user := 0.U - io.in.ready := io.out.ready - when(io.in.fire) { - assert(io.in.bits.vc === virtualChannel.U, "Bulk packet arrived on wrong Mesh VC") - assert(io.in.bits.head === first, "Bulk Mesh packet head mismatch") - first := io.in.bits.tail - } -} - -/** Native verification target for long AXI-stream packet ownership over Mesh. */ -class MeshBulkLoopback extends Module { - val mesh = MeshParams(xNodes = 2, yNodes = 2, payloadBits = 320, virtualChannels = 8) - - val map = ChiMeshNodeMap( - Seq.tabulate(128)(_ & 1), - Seq.tabulate(128)(node => (node >> 1) & 1), - mesh, - presentNodes = (0 until 128).toSet - ) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshBulkBeat(256, 7))) - val out = Decoupled(new Beat(256)) - val observed = Output(new MeshFlit(mesh)) - val observedValid = Output(Bool()) - }) - - val tx: Instance[MeshBulkPacketizer] = - Instantiate(new MeshBulkPacketizer(mesh, 256, virtualChannel = 4, localX = 0, localY = 0, map)) - val rx: Instance[MeshBulkDepacketizer] = Instantiate(new MeshBulkDepacketizer(mesh, 256, virtualChannel = 4)) - tx.io.in <> io.in - rx.io.in <> tx.io.out - io.out <> rx.io.out - io.observed := tx.io.out.bits - io.observedValid := tx.io.out.valid -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/MeshCredit.scala b/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/MeshCredit.scala deleted file mode 100644 index 2f57f3d42..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/MeshCredit.scala +++ /dev/null @@ -1,165 +0,0 @@ -package hier.chip.mesh - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} - -/** One directed physical Mesh link. Each VC returns its own buffer credits. */ -class MeshCreditLink(p: MeshParams) extends Bundle { - val valid = Output(Bool()) - val flit = Output(new MeshFlit(p)) - val credit = Input(Vec(p.virtualChannels, Bool())) -} - -/** - * Sender half of a credit-based Mesh link. - * - * Credits start at zero and are supplied by the receiver after link activation. - * A coordinated reset is the only supported way to stop an active link. - */ -@instantiable -class MeshCreditTx(p: MeshParams, maxCredits: Int) extends Module { - require(maxCredits >= 1 && maxCredits <= 255) - val creditBits = math.max(1, log2Ceil(maxCredits + 1)) - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - val in = Vec(p.virtualChannels, Flipped(Decoupled(new MeshFlit(p)))) - val link = new MeshCreditLink(p) - val credits = Output(UInt(creditBits.W)) - }) - - val credits = RegInit(VecInit(Seq.fill(p.virtualChannels)(0.U(creditBits.W)))) - val returned = RegNext(io.link.credit, VecInit(Seq.fill(p.virtualChannels)(false.B))) - val cursor = RegInit(0.U(p.vcBits.W)) - val eligible = VecInit((0 until p.virtualChannels).map(vc => io.active && io.in(vc).valid && credits(vc) =/= 0.U)) - val ordered = VecInit((0 until p.virtualChannels).map(offset => eligible((cursor +& offset.U) % p.virtualChannels.U))) - val selected = (cursor +& PriorityEncoder(ordered)) % p.virtualChannels.U - val send = eligible.asUInt.orR - for (vc <- 0 until p.virtualChannels) { - io.in(vc).ready := send && selected === vc.U - val consumed = send && selected === vc.U - when(returned(vc) =/= consumed)(credits(vc) := Mux(returned(vc), credits(vc) + 1.U, credits(vc) - 1.U)) - when(returned(vc) && !consumed)(assert(credits(vc) < maxCredits.U, "Mesh TX credit overflow")) - when(consumed)(assert(io.in(vc).bits.vc === vc.U, "Mesh TX wrong VC port")) - } - when(send)(cursor := Mux(selected === (p.virtualChannels - 1).U, 0.U, selected + 1.U)) - io.link.valid := RegNext(send, false.B) - io.link.flit := RegEnable(io.in(selected).bits, 0.U.asTypeOf(new MeshFlit(p)), send) - io.credits := credits(selected) - val wasActive = RegNext(io.active, false.B) - when(wasActive)(assert(io.active, "Mesh TX requires coordinated reset to stop")) -} - -/** Receiver half of a credit-based Mesh link with independent FIFO and credit pool per VC. */ -@instantiable -class MeshCreditRx(p: MeshParams, depth: Int) extends Module { - require(depth >= 1 && depth <= 255) - val creditBits = math.max(1, log2Ceil(depth + 1)) - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - val link = Flipped(new MeshCreditLink(p)) - val out = Vec(p.virtualChannels, Decoupled(new MeshFlit(p))) - val occupancy = Output(Vec(p.virtualChannels, UInt(creditBits.W))) - }) - - val queues = Seq.fill(p.virtualChannels)(Module(new Queue(new MeshFlit(p), depth, pipe = true))) - val advertised = RegInit(VecInit(Seq.fill(p.virtualChannels)(0.U(creditBits.W)))) - val enqueueReady = Wire(Vec(p.virtualChannels, Bool())) - for (vc <- 0 until p.virtualChannels) { - val queue = queues(vc) - val accepted = io.link.valid && io.link.flit.vc === vc.U - val grant = io.active && (advertised(vc) +& queue.io.count) < depth.U - io.link.credit(vc) := grant - queue.io.enq.valid := accepted - queue.io.enq.bits := io.link.flit - enqueueReady(vc) := queue.io.enq.ready - io.out(vc) <> queue.io.deq - when(grant =/= accepted) { - advertised(vc) := Mux(grant, advertised(vc) + 1.U, advertised(vc) - 1.U) - } - io.occupancy(vc) := queue.io.count - } - when(io.link.valid) { - assert(io.active, "Mesh RX flit on inactive link") - assert(advertised(io.link.flit.vc) =/= 0.U, "Mesh RX flit without returned credit") - assert(enqueueReady(io.link.flit.vc), "Mesh RX buffer overflow") - } - for (vc <- 0 until p.virtualChannels) { - assert((advertised(vc) +& queues(vc).io.count) <= depth.U, "Mesh RX credit conservation") - } - val wasActive = RegNext(io.active, false.B) - when(wasActive)(assert(io.active, "Mesh RX requires coordinated reset to stop")) -} - -/** 2D Mesh with explicit independent local VC ports and per-VC physical credits. */ -@instantiable -class MeshCreditNetwork(p: MeshParams, linkDepth: Int = 2) extends Module { - require(linkDepth >= 1 && linkDepth <= 255) - val flit = new MeshFlit(p) - val routers: Seq[Seq[Instance[MeshRouter]]] = - Seq.tabulate(p.yNodes, p.xNodes)((y, x) => Instantiate(new MeshRouter(p, x, y))) - private def index(x: Int, y: Int): Int = y * p.xNodes + x - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - val localIn = Vec(p.xNodes * p.yNodes, Vec(p.virtualChannels, Flipped(Decoupled(flit)))) - val localOut = Vec(p.xNodes * p.yNodes, Vec(p.virtualChannels, Decoupled(flit))) - }) - - def wireLink(source: Vec[DecoupledIO[MeshFlit]], sink: Vec[DecoupledIO[MeshFlit]]): Unit = { - val tx: Instance[MeshCreditTx] = Instantiate(new MeshCreditTx(p, linkDepth)); - val rx: Instance[MeshCreditRx] = Instantiate(new MeshCreditRx(p, linkDepth)) - tx.io.active := io.active; rx.io.active := io.active - tx.io.in <> source; rx.io.link <> tx.io.link; sink <> rx.io.out - } - - for { - y <- 0 until p.yNodes - x <- 0 until p.xNodes - } { - val router = routers(y)(x) - for (vc <- 0 until p.virtualChannels) { - val input = io.localIn(index(x, y))(vc) - val output = io.localOut(index(x, y))(vc) - router.io.in(MeshDirection.Local)(vc).valid := io.active && input.valid - router.io.in(MeshDirection.Local)(vc).bits := input.bits - input.ready := io.active && router.io.in(MeshDirection.Local)(vc).ready - output.valid := io.active && router.io.out(MeshDirection.Local)(vc).valid - output.bits := router.io.out(MeshDirection.Local)(vc).bits - router.io.out(MeshDirection.Local)(vc).ready := io.active && output.ready - } - for (vc <- 0 until p.virtualChannels) { - for ( - direction <- Seq(MeshDirection.West).filter(_ => x == 0) ++ Seq(MeshDirection.East).filter(_ => - x == p.xNodes - 1 - ) ++ Seq(MeshDirection.North).filter(_ => y == 0) ++ Seq(MeshDirection.South).filter(_ => - y == p.yNodes - 1 - ) - ) { - router.io.in(direction)(vc).valid := false.B; router.io.in(direction)(vc).bits := 0.U.asTypeOf(flit) - router.io.out(direction)(vc).ready := false.B - } - } - } - for { - y <- 0 until p.yNodes - x <- 0 until p.xNodes - 1 - } { - wireLink(routers(y)(x).io.out(MeshDirection.East), routers(y)(x + 1).io.in(MeshDirection.West)) - wireLink(routers(y)(x + 1).io.out(MeshDirection.West), routers(y)(x).io.in(MeshDirection.East)) - } - for { - y <- 0 until p.yNodes - 1 - x <- 0 until p.xNodes - } { - wireLink(routers(y)(x).io.out(MeshDirection.South), routers(y + 1)(x).io.in(MeshDirection.North)) - wireLink(routers(y + 1)(x).io.out(MeshDirection.North), routers(y)(x).io.in(MeshDirection.South)) - } -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/ScratchpadBulkWriteSink.scala b/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/ScratchpadBulkWriteSink.scala deleted file mode 100644 index 3c8c9984e..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/ScratchpadBulkWriteSink.scala +++ /dev/null @@ -1,50 +0,0 @@ -package hier.chip.mesh - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import memcore.bus.axi.Beat -import memcore.memory.bank.{BankSet, BankSetParams} - -/** NoC extension that binds VC5/VC4/VC6 DMA traffic to a local scratchpad. */ -class ScratchpadBulkWriteSink(p: BankSetParams, nodeIdBits: Int) extends Module { - - val io = IO(new Bundle { - val descriptor = Flipped(Decoupled(new BulkDescriptor(nodeIdBits))) - val bulkIn = Flipped(Decoupled(new Beat(p.dataBits))) - val completion = Decoupled(new BulkCompletion(nodeIdBits)) - }) - - val scratchpad: Instance[BankSet] = Instantiate(new BankSet(p)) - val idle :: issue :: streaming :: complete :: Nil = Enum(4) - val state = RegInit(idle) - val descriptor = Reg(new BulkDescriptor(nodeIdBits)) - val beats = descriptor.bytes / p.bytes.U - io.descriptor.ready := state === idle - when(io.descriptor.fire) { - assert( - io.descriptor.bits.write && io.descriptor.bits.bytes =/= 0.U && - io.descriptor.bits.bytes % p.bytes.U === 0.U, - "Scratchpad bulk sink accepts aligned nonempty writes only" - ) - descriptor := io.descriptor.bits - state := issue - } - scratchpad.io.command.valid := state === issue - scratchpad.io.command.bits.write := true.B - scratchpad.io.command.bits.addr := descriptor.scratchpadAddr(p.addressBits - 1, 0) - scratchpad.io.command.bits.beats := beats - when(scratchpad.io.command.fire)(state := streaming) - scratchpad.io.write.valid := state === streaming && io.bulkIn.valid - scratchpad.io.write.bits := io.bulkIn.bits - io.bulkIn.ready := state === streaming && scratchpad.io.write.ready - scratchpad.io.read.ready := true.B - scratchpad.io.done.ready := state === streaming - when(scratchpad.io.done.fire)(state := complete) - io.completion.valid := state === complete - io.completion.bits.targetNode := descriptor.sourceNode - io.completion.bits.txnId := descriptor.txnId - io.completion.bits.ready := false.B - io.completion.bits.error := scratchpad.io.done.bits - when(io.completion.fire)(state := idle) -} diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/TileMeshEndpoint.scala b/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/TileMeshEndpoint.scala deleted file mode 100644 index 6c50290a3..000000000 --- a/arch/src/main/scala/framework/root/root-chip/src/main/scala/chip/mesh/TileMeshEndpoint.scala +++ /dev/null @@ -1,95 +0,0 @@ -package hier.chip.mesh - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi._ -import memcore.bus.axi.Beat - -/** - * One requester tile's combined coherent and bulk Mesh endpoint. - * - * CHI uses VC0--VC3 through `ChiMeshRequesterEndpoint`; bulk AXI streams use - * VC4. Each VC has independent flow control; physical links arbitrate flits - * using the receiver credit pool for that VC. - */ -@instantiable -class TileMeshEndpoint( - p: Params, - mesh: MeshParams, - localX: Int, - localY: Int, - nodeMap: ChiMeshNodeMap, - bulkDataBits: Int = 256) - extends Module { - require(mesh.virtualChannels >= 5) - require(mesh.payloadBits >= bulkDataBits + bulkDataBits / 8) - require(nodeMap.coordinateBits == p.nodeIdBits) - - @public - val io = IO(new Bundle { - val chi = Flipped(new RequesterPort(p)) - val bulkIn = Flipped(Decoupled(new MeshBulkBeat(bulkDataBits, p.nodeIdBits))) - val bulkOut = Decoupled(new Beat(bulkDataBits)) - val meshOut = Vec(mesh.virtualChannels, Decoupled(new MeshFlit(mesh))) - val meshIn = Vec(mesh.virtualChannels, Flipped(Decoupled(new MeshFlit(mesh)))) - }) - - val chiEndpoint: Instance[ChiMeshRequesterEndpoint] = - Instantiate(new ChiMeshRequesterEndpoint(p, mesh, localX, localY, nodeMap)) - val bulkTx: Instance[MeshBulkPacketizer] = - Instantiate(new MeshBulkPacketizer(mesh, bulkDataBits, virtualChannel = 4, localX, localY, nodeMap)) - val bulkRx: Instance[MeshBulkDepacketizer] = - Instantiate(new MeshBulkDepacketizer(mesh, bulkDataBits, virtualChannel = 4)) - chiEndpoint.io.chi <> io.chi - bulkTx.io.in <> io.bulkIn - for (vc <- 0 until mesh.virtualChannels) { - if (vc < 4) { - io.meshOut(vc) <> chiEndpoint.io.meshOut(vc) - chiEndpoint.io.meshIn(vc) <> io.meshIn(vc) - } else { - chiEndpoint.io.meshOut(vc).ready := false.B - chiEndpoint.io.meshIn(vc).valid := false.B - chiEndpoint.io.meshIn(vc).bits := 0.U.asTypeOf(new MeshFlit(mesh)) - if (vc == 4) { io.meshOut(vc) <> bulkTx.io.out; bulkRx.io.in <> io.meshIn(vc) } - else { - io.meshOut(vc).valid := false.B; io.meshOut(vc).bits := 0.U.asTypeOf(new MeshFlit(mesh)); - io.meshIn(vc).ready := false.B - } - } - } - io.bulkOut <> bulkRx.io.out -} - -/** Native bulk-path verification target for the combined tile endpoint. */ -class TileMeshBulkLoopback extends Module { - val chi = Params() - val mesh = MeshParams(xNodes = 2, yNodes = 2, payloadBits = 320, virtualChannels = 8) - - val map = ChiMeshNodeMap( - Seq.tabulate(1 << chi.nodeIdBits)(_ & 1), - Seq.tabulate(1 << chi.nodeIdBits)(node => (node >> 1) & 1), - mesh, - presentNodes = (0 until (1 << chi.nodeIdBits)).toSet - ) - - @public - val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshBulkBeat(256, chi.nodeIdBits))) - val out = Decoupled(new Beat(256)) - }) - - val endpoint: Instance[TileMeshEndpoint] = Instantiate(new TileMeshEndpoint(chi, mesh, 0, 0, map)) - endpoint.io.bulkIn <> io.in - io.out <> endpoint.io.bulkOut - endpoint.io.chi.req.valid := false.B - endpoint.io.chi.req.bits := 0.U.asTypeOf(endpoint.io.chi.req.bits) - endpoint.io.chi.txRsp.valid := false.B - endpoint.io.chi.txRsp.bits := 0.U.asTypeOf(endpoint.io.chi.txRsp.bits) - endpoint.io.chi.txDat.valid := false.B - endpoint.io.chi.txDat.bits := 0.U.asTypeOf(endpoint.io.chi.txDat.bits) - endpoint.io.chi.snp.ready := true.B - endpoint.io.chi.rxRsp.ready := true.B - endpoint.io.chi.rxDat.ready := true.B - endpoint.io.meshIn <> endpoint.io.meshOut -} diff --git a/arch/src/main/scala/framework/root/root-core/src/csrc/Cargo.lock b/arch/src/main/scala/framework/root/root-core/src/csrc/Cargo.lock deleted file mode 100644 index 4a1308c3e..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "root-core-reference-model" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/root/root-core/src/csrc/Cargo.toml b/arch/src/main/scala/framework/root/root-core/src/csrc/Cargo.toml deleted file mode 100644 index f9c892cdb..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/csrc/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "root-core-reference-model" -version = "0.1.0" -edition = "2021" -[lib] -crate-type = ["cdylib"] diff --git a/arch/src/main/scala/framework/root/root-core/src/csrc/src/lib.rs b/arch/src/main/scala/framework/root/root-core/src/csrc/src/lib.rs deleted file mode 100644 index 879e3a09c..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/csrc/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod memory; -pub use memory::*; diff --git a/arch/src/main/scala/framework/root/root-core/src/csrc/src/memory.rs b/arch/src/main/scala/framework/root/root-core/src/csrc/src/memory.rs deleted file mode 100644 index 61d5cf2de..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/csrc/src/memory.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::ffi::{c_char, c_void, CStr}; -const BASE: u64 = 0x8000_0000; -const BYTES: usize = 16 * 1024 * 1024; - -#[no_mangle] -pub unsafe extern "C" fn core_ref_create(path: *const c_char) -> *mut c_void { - let path = CStr::from_ptr(path).to_str().unwrap(); - let text = std::fs::read_to_string(path).unwrap(); - let mut memory = vec![0u8; BYTES]; - let mut address = BASE; - for token in text.split_whitespace() { - if let Some(value) = token.strip_prefix('@') { address = u64::from_str_radix(value, 16).unwrap(); } - else { - memory[usize::try_from(address - BASE).unwrap()] = u8::from_str_radix(token, 16).unwrap(); - address += 1; - } - } - Box::into_raw(Box::new(memory)).cast() -} -#[no_mangle] -pub unsafe extern "C" fn core_ref_destroy(model: *mut c_void) { - drop(Box::from_raw(model.cast::>())); -} -#[no_mangle] -pub unsafe extern "C" fn core_ref_read(model: *mut c_void, address: u64, output: *mut u32) { - assert!(address % 64 == 0); - let bytes = &*model.cast::>(); - let start = usize::try_from(address - BASE).unwrap(); - for i in 0..16 { *output.add(i) = u32::from_le_bytes(bytes[start + i*4..start + i*4 + 4].try_into().unwrap()); } -} -#[no_mangle] -pub unsafe extern "C" fn core_ref_write(model: *mut c_void, address: u64, data: *const u32, mask: *const u32) { - assert!(address % 64 == 0); - let bytes = &mut *model.cast::>(); - let start = usize::try_from(address - BASE).unwrap(); - let mask = u64::from(*mask) | (u64::from(*mask.add(1)) << 32); - for i in 0..16 { - let word = (*data.add(i)).to_le_bytes(); - for (b, value) in word.into_iter().enumerate() { - if mask & (1u64 << (i*4+b)) != 0 { bytes[start + i*4 + b] = value; } - } - } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.S b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.S deleted file mode 100644 index 08870aa0a..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.S +++ /dev/null @@ -1,84 +0,0 @@ -.option norelax -.section .text.start,"ax",@progbits -.globl _start -_start: - la t0,trap_handler - csrw mtvec,t0 - li s0,0x10000000 - li t0,0x20400000 - csrw pmpaddr0,t0 - li t0,0x0f - csrw pmpcfg0,t0 - li t0,0x8000000000080006 - csrw satp,t0 - li t0,3<<11 - csrc mstatus,t0 - li t0,(1<<11)|(1<<17)|(1<<18)|(1<<19) - csrs mstatus,t0 - li s1,0x400040c0 - li t0,0x1111222233334444 - mv t1,s1 -.globl admit_old_load -admit_old_load: - ld t3,0(s1) -.globl admit_cmd0 -admit_cmd0: - .insn r 0x7b,3,0x11,zero,t0,t1 -.globl admit_young_load -admit_young_load: - ld t4,8(s1) - li t5,0x0123456789abcdef - bne t3,t5,fail - li t5,0x8877665544332211 - bne t4,t5,fail - li t0,(1<<17)|(1<<18)|(1<<19) - csrc mstatus,t0 - li t0,0x20800000 - csrw pmpaddr0,t0 - li t0,0x0b - csrw pmpcfg0,t0 - li t0,0x8000300000080009 - csrw satp,t0 - li t0,0x5555666677778888 - li t1,0x400040c8 -.globl admit_cmd1 -admit_cmd1: - .insn r 0x7b,3,0x12,zero,t0,t1 -.globl admit_fault -admit_fault: - .insn r 0x7b,1,0x13,zero,t0,t1 -.globl admit_after_fault -admit_after_fault: - sd zero,0(s0) -1: j 1b -fail: - li t0,1 - sd t0,0(s0) -2: j 2b -.section .text.trap,"ax",@progbits -.option push -.option norvc -trap_handler: - csrr t0,mcause - li t1,2 - bne t0,t1,fail - csrr t0,mepc - la t1,admit_fault - bne t0,t1,fail - addi t0,t0,4 - csrw mepc,t0 - mret -.option pop -.section .data,"aw",@progbits -.quad 0x0123456789abcdef,0x8877665544332211 -.section .pt.root,"aw",@progbits -.zero 8 -.quad 0x20001c01 -.zero 4080 -.section .pt.mid,"aw",@progbits -.quad 0x20002001 -.zero 4088 -.section .pt.leaf,"aw",@progbits -.zero 32 -.quad 0x200010c7 -.zero 4056 diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.cm_hier b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.cm_hier deleted file mode 100644 index b8cd95204..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.cm_hier +++ /dev/null @@ -1,12 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.tracker -+tree @TOP@.reserve -+tree @TOP@.command -+tree @TOP@.complete -+tree @TOP@.cancelled -+tree @TOP@.info -+tree @TOP@.cancel_request -+tree @TOP@.mem_req -+tree @TOP@.mem_resp -+tree @TOP@.uncached_req -+tree @TOP@.uncached_resp diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.f deleted file mode 100644 index 4bb12da19..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.f +++ /dev/null @@ -1,10 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/admission_control.sv -@RESOURCES@/admission_package.sv --F @RTL@/AdmissionCore/filelist.f --F @RTL@/AdmissionInterlock/filelist.f -@RESOURCES@/admission_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.ld b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.ld deleted file mode 100644 index ea50912c3..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission.ld +++ /dev/null @@ -1,18 +0,0 @@ -OUTPUT_ARCH(riscv) -ENTRY(_start) -SECTIONS -{ - . = 0x80000000; - .text.start : { *(.text.start) } - . = 0x80000200; - .text.trap : { *(.text.trap) } - . = 0x800040c0; - .data : { *(.data) } - . = 0x80006000; - .pt.root : { *(.pt.root) } - . = 0x80007000; - .pt.mid : { *(.pt.mid) } - . = 0x80008000; - .pt.leaf : { *(.pt.leaf) } - /DISCARD/ : { *(.riscv.attributes) *(.comment) } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge.f deleted file mode 100644 index cbf89d5fb..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/admission_bridge_control.sv -@RESOURCES@/admission_bridge_package.sv --F @RTL@/AdmissionBridge/filelist.f -@RESOURCES@/admission_bridge_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_bad_late_fault.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_bad_late_fault.f deleted file mode 100644 index 301acda12..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_bad_late_fault.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+AB_TB=admission_bridge_bad_late_fault_tb -+define+AB_LATE_FAULT -+incdir+@RESOURCES@ -+incdir+@RTL@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/admission_bridge_control.sv -@RESOURCES@/admission_bridge_package.sv --F @RTL@/AdmissionBridge/filelist.f -@RESOURCES@/admission_bridge_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_bad_late_fault.toml b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_bad_late_fault.toml deleted file mode 100644 index ccb662d7d..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_bad_late_fault.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "AdmissionBridge fault arrived after retirement offer" diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_control.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_control.sv deleted file mode 100644 index 151b47d51..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_control.sv +++ /dev/null @@ -1,17 +0,0 @@ -`include "admission_bridge_config.svh" -interface admission_bridge_control_if ( - input logic clock -); - logic reset, boot_allocation, allocation_valid; - logic fault_valid, unbound_fault_valid; - logic [`AB_FAULT_WIDTH-1:0] fault_bits, unbound_fault_bits; - logic [`AB_ROB_BITS-1:0] allocation_id, lookup_id[3]; - logic [`AB_ROB_ENTRIES-1:0] retired; - logic lookup_valid[3]; - logic [`AB_SNAP_WIDTH-1:0] lookup_bits[3]; - clocking sample @(posedge clock); - default input #1step; - input fault_valid, unbound_fault_valid, fault_bits, unbound_fault_bits; - input reset, allocation_valid, allocation_id, retired, lookup_id, lookup_valid, lookup_bits; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_package.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_package.sv deleted file mode 100644 index abe45361e..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_package.sv +++ /dev/null @@ -1,354 +0,0 @@ -`include "admission_bridge_config.svh" -package admission_bridge_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `define RF(F) `AB_RET_``F``_OFFSET +: `AB_RET_``F``_WIDTH - `define FF(F) `AB_FAULT_``F``_OFFSET +: `AB_FAULT_``F``_WIDTH - `define SF(F) `AB_SNAP_``F``_OFFSET +: `AB_SNAP_``F``_WIDTH - typedef bit [`AB_SNAP_WIDTH-1:0] snapshot_t; - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual admission_bridge_control_if control; - virtual stream_if #(`AB_SNAP_WIDTH) command; - virtual stream_if #(`AB_RET_WIDTH) retirement; - virtual stream_if #(`AB_NPU_WIDTH) npu; - typedef struct { - snapshot_t snapshot; - int rob; - bit pending; - bit [3:0] error; - bit [63:0] address; - } context_t; - context_t contexts[int]; - int accepted = 0, notices = 0, cancelled = 0, peak = 0, lookup_hits = 0, lookup_misses = 0; - bit late_fault = 0; - int bound_faults = 0, repeated_faults = 0, unbound_faults = 0, error_notices = 0; - int multi_retire = 0, boot_ignored = 0, nonrob = 0, stalls = 0, cycle = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 100us; - endfunction - function void verify_contract(bit ok, string message); - if (!ok) `uvm_fatal("ADMISSION_BRIDGE", message) - endfunction - function snapshot_t packet(int tag, int funct, int seed); - snapshot_t value; - // All CSR/PMP/instruction bits participate in independent exact comparisons. - for (int i = 0; i < `AB_SNAP_WIDTH; i++) - value[i] = ((i * 17 + seed * 7) % (seed % 19 + 3)) % 2; - value[`SF(TAG)] = tag; - value[`SF(INSTRUCTION_FUNCT)] = funct; - return value; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - void'(uvm_config_db#(bit)::get(this, "", "late_fault", late_fault)); - verify_contract(uvm_config_db#(virtual admission_bridge_control_if)::get( - this, "", "control", control), "Missing control"); - verify_contract(uvm_config_db#(virtual stream_if #(`AB_SNAP_WIDTH))::get( - this, "", "command", command), "Missing command"); - verify_contract(uvm_config_db#(virtual stream_if #(`AB_RET_WIDTH))::get( - this, "", "retirement", retirement), "Missing retirement"); - verify_contract(uvm_config_db#(virtual stream_if #(`AB_NPU_WIDTH))::get(this, "", "npu", npu), - "Missing NPU"); - endfunction - task monitor(); - forever begin - @(control.sample); - if (control.sample.reset) begin - cancelled += contexts.num(); - contexts.delete(); - end else begin - int retiring = 0; - cycle++; - foreach (contexts[tag]) - if (contexts[tag].rob >= 0 && control.sample.retired[contexts[tag].rob]) begin - contexts[tag].pending = 1; - retiring++; - end - if (retiring > 1) multi_retire++; - begin - int bound = -1; - foreach (contexts[tag]) - if (contexts[tag].rob == control.sample.fault_bits[`FF(ROB_ID)]) bound = tag; - verify_contract( - control.sample.unbound_fault_valid == (control.sample.fault_valid && bound < 0), - "Unknown/boot fault was lost or misrouted"); - if (control.sample.unbound_fault_valid) begin - verify_contract(control.sample.unbound_fault_bits === control.sample.fault_bits, - "Unbound fault payload changed"); - unbound_faults++; - end - if (control.sample.fault_valid && bound >= 0 && control.sample.fault_bits[ - `FF(ERROR) - ] != 0 && !late_fault) begin - if (contexts[bound].error == 0) begin - contexts[bound].error = control.sample.fault_bits[`FF(ERROR)]; - contexts[bound].address = control.sample.fault_bits[`FF(ADDRESS)]; - bound_faults++; - end else repeated_faults++; - end - end - for (int port = 0; port < 3; port++) begin - int found = -1; - foreach (contexts[tag]) - if (!contexts[tag].pending && contexts[tag].rob == control.sample.lookup_id[port]) begin - verify_contract(found == -1, "Reference duplicate ROB"); - found = tag; - end - verify_contract(control.sample.lookup_valid[port] == (found >= 0), - "Lookup live/retired visibility mismatch"); - if (found >= 0) begin - verify_contract(control.sample.lookup_bits[port] === contexts[found].snapshot, - "Lookup lost snapshot or changed CSR/PMP context"); - lookup_hits++; - end else lookup_misses++; - end - verify_contract( - (command.sample.valid&&command.sample.ready)==(npu.sample.valid&&npu.sample.ready), - "CPU/NPU acceptance not atomic"); - if (command.sample.valid && !command.sample.ready) stalls++; - if (npu.sample.valid) - verify_contract( - npu.sample.bits === command.sample.bits[`AB_INSTRUCTION_OFFSET+:`AB_NPU_WIDTH], - "NPU instruction differs from CPU snapshot"); - if (control.sample.allocation_valid && !(npu.sample.valid && npu.sample.ready)) begin - verify_contract(contexts.num() == 0, "Boot overlapped external contexts"); - boot_ignored++; - end - // Check retirement while stalled as well as when accepted. - if (retirement.sample.valid) begin - int tag = retirement.sample.bits[`RF(SNAPSHOT_TAG)]; - verify_contract(contexts.exists(tag) && contexts[tag].pending, - "Unexpected retirement notice"); - verify_contract( - retirement.sample.bits[`AB_RET_SNAPSHOT_OFFSET+:`AB_SNAP_WIDTH]===contexts[tag].snapshot, - "Retirement changed full snapshot"); - verify_contract(retirement.sample.bits[`RF(ERROR) - ] === contexts[tag].error && retirement.sample.bits[`RF(ADDRESS) - ] === contexts[tag].address, - "Retirement lost first fault/error/full64 address"); - end - if (command.sample.valid && command.sample.ready) begin - int tag = command.sample.bits[`SF(TAG)]; - int funct = command.sample.bits[`SF(INSTRUCTION_FUNCT)]; - int rob = funct <= 1 ? -1 : int'(control.sample.allocation_id); - verify_contract(!contexts.exists(tag) && contexts.num() < `AB_ENTRIES, - "Reused tag/context in retirement acceptance cycle"); - if (rob >= 0) - foreach (contexts[old]) - verify_contract(contexts[old].rob != rob, - "ROB reused before old retirement acknowledgement"); - verify_contract(control.sample.allocation_valid == (rob >= 0), - "Wrong allocation ownership"); - contexts[tag] = '{command.sample.bits, rob, rob < 0, 0, 0}; - accepted++; - if (rob < 0) nonrob++; - if (contexts.num() > peak) peak = contexts.num(); - end - if (retirement.sample.valid && retirement.sample.ready) begin - int tag = retirement.sample.bits[`RF(SNAPSHOT_TAG)]; - if (contexts[tag].error != 0) error_notices++; - contexts.delete(tag); - notices++; - end - end - @(negedge control.clock); - for (int port = 0; port < 3; port++) - control.lookup_id[port] = (cycle + port * 5) % `AB_ROB_ENTRIES; - end - endtask - task submit(int tag, int rob, int funct = 2, int seed = 3); - @(negedge control.clock); - control.allocation_id = rob; - command.bits = packet(tag, funct, seed); - command.valid = 1; - do @(command.sample); while (!command.sample.ready); - @(negedge control.clock); - command.valid = 0; - command.bits = ~command.bits; - endtask - task retire(bit [`AB_ROB_ENTRIES-1:0] mask); - @(negedge control.clock); - control.retired = mask; - @(control.sample); - @(negedge control.clock); - control.retired = 0; - endtask - task fault(int rob, int error, bit [63:0] address, bit [`AB_ROB_ENTRIES-1:0] mask = 0); - @(negedge control.clock); - control.fault_bits = '0; - control.fault_bits[`FF(ROB_ID)] = rob; - control.fault_bits[`FF(ERROR)] = error; - control.fault_bits[`FF(ADDRESS)] = address; - control.fault_valid = 1; - control.retired = mask; - @(control.sample); - @(negedge control.clock); - control.fault_valid = 0; - control.retired = 0; - control.fault_bits = ~control.fault_bits; - endtask - task blocked(int cycles); - repeat (cycles) begin - @(control.sample); - verify_contract(!command.sample.ready && !npu.sample.valid, "Blocked binding reached NPU"); - end - endtask - task drain(); - wait (contexts.num() == 0); - repeat (3) @(control.sample); - verify_contract(!retirement.sample.valid, "Notice remained after all acknowledgements"); - endtask - task execute(); - control.fault_valid = 0; - control.fault_bits = 0; - control.reset = 1; - control.boot_allocation = 0; - control.retired = 0; - control.allocation_id = 0; - for (int i = 0; i < 3; i++) control.lookup_id[i] = 0; - command.valid = 0; - command.bits = 0; - npu.ready = 0; - retirement.ready = 0; - fork - monitor(); - join_none - repeat (4) @(negedge control.clock); - control.reset = 0; - if (late_fault) begin - npu.ready = 1; - submit(86, 8, 2, 9); - retire(16'h100); - wait (retirement.valid); - fault(8, 7, 64'hf123456789abcdef); - repeat (4) @(control.sample); - `uvm_fatal("MISSING_ASSERTION", "Late fault did not trigger DUT ordering assertion") - return; - end - for (int i = 0; i < 3; i++) begin - @(negedge control.clock); - control.boot_allocation = 1; - control.allocation_id = i; - @(control.sample); - end - @(negedge control.clock); - control.boot_allocation = 0; - fault(14, 7, 64'hfedcba9876543210); - // A stable external offer must survive NPU backpressure without reservation. - fork - submit(0, 0, 2, 7); - begin - repeat (5) @(control.sample); - verify_contract(contexts.num() == 0, "NPU stall reserved a context"); - @(negedge control.clock); - npu.ready = 1; - end - join - submit(127, 3, 3, 11); - submit(255, 7, 4, 17); - submit(37, 15, 5, 23); - fault(0, 3, 64'h8123456789abcdef); - fault(0, 4, 64'hfedcba9876543210); - fault(7, 0, 64'hffffffffffffffff); - fault(14, 5, 64'habcdef0123456789); - fork - submit(2, 9, 6, 29); - begin - @(negedge control.clock); - repeat (2) @(control.sample); - blocked(4); - fault(3, 9, 64'hfedcba9876543210, 16'h8089); - wait (retirement.valid); - blocked(4); - @(negedge control.clock); - retirement.ready = 1; - end - join - wait (contexts.num() == 1); - repeat (18) @(control.sample); - retire(16'h200); - drain(); - // A duplicated caller tag remains blocked after ROB retire until notice acceptance. - @(negedge control.clock); - retirement.ready = 0; - submit(42, 5, 8, 31); - fork - submit(42, 6, 9, 37); - begin - repeat (2) @(control.sample); - blocked(4); - retire(16'h20); - wait (retirement.valid); - blocked(4); - @(negedge control.clock); - retirement.ready = 1; - end - join - repeat (18) @(control.sample); - retire(16'h40); - drain(); - // Old pending notices keep their ROB binding even with a different caller tag. - @(negedge control.clock); - retirement.ready = 0; - submit(55, 11, 10, 41); - retire(16'h800); - wait (retirement.valid); - fork - submit(56, 11, 11, 43); - begin - repeat (2) @(control.sample); - blocked(5); - @(negedge control.clock); - retirement.ready = 1; - end - join - repeat (18) @(control.sample); - retire(16'h800); - drain(); - @(negedge control.clock); - retirement.ready = 0; - submit(73, 0, 15, 45); - submit(70, 0, 0, 47); - submit(71, 0, 1, 53); - repeat (5) @(control.sample); - @(negedge control.clock); - retirement.ready = 1; - retire(16'h1); - drain(); - // Coordinated reset cancels both a held notice and another unretired binding. - @(negedge control.clock); - retirement.ready = 0; - submit(80, 2, 12, 59); - submit(81, 4, 13, 61); - retire(16'h4); - wait (retirement.valid); - @(negedge control.clock); - control.reset = 1; - repeat (3) @(negedge control.clock); - control.reset = 0; - retirement.ready = 1; - repeat (18) @(control.sample); - submit(80, 2, 14, 67); - repeat (18) @(control.sample); - retire(16'h4); - drain(); - verify_contract( - accepted==notices+cancelled&&cancelled==2&&peak==4&&boot_ignored==3&&nonrob==2&&multi_retire>0&&lookup_hits>0&&lookup_misses>0&&stalls>0&&bound_faults==2&&repeated_faults==1&&unbound_faults==2&&error_notices==2, - "Missing lifecycle coverage or lost notice"); - `uvm_info( - "ADMISSION_FAULT_PASS", - $sformatf( - "firstFaults=%0d repeatedIgnored=%0d unboundForwarded=%0d errorNotices=%0d; same-cycle fault+retire and full64 addresses checked", - bound_faults, repeated_faults, unbound_faults, error_notices), UVM_LOW) - `uvm_info( - "ADMISSION_BRIDGE_PASS", - $sformatf( - "accepted=%0d notices=%0d resetCancelled=%0d peak=%0d bootIgnored=%0d nonROB=%0d multiRetire=%0d lookup hit/miss=%0d/%0d stalls=%0d; complete snapshot/tag/PMP retained and all contexts drained", - accepted, notices, cancelled, peak, boot_ignored, nonrob, multi_retire, lookup_hits, - lookup_misses, stalls), UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_tb.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_tb.sv deleted file mode 100644 index 93cdff5fb..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_bridge_tb.sv +++ /dev/null @@ -1,43 +0,0 @@ -`include "admission_bridge_config.svh" -`ifndef AB_TB -`define AB_TB admission_bridge_tb -`endif -module `AB_TB; - import uvm_pkg::*; - import admission_bridge_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - admission_bridge_control_if control (clock); - stream_if #(`AB_SNAP_WIDTH) command ( - clock, - control.reset - ); - stream_if #(`AB_RET_WIDTH) retirement ( - clock, - control.reset - ); - stream_if #(`AB_NPU_WIDTH) npu ( - clock, - control.reset - ); - assign control.allocation_valid=control.boot_allocation || - (npu.valid&&npu.ready&&npu.bits[`AB_NPU_FUNCT_OFFSET+:`AB_NPU_FUNCT_WIDTH]>1); - AdmissionBridge dut ( - .clock(clock), - .reset(control.reset), - `include "admission_bridge_ports.svh" - ); - initial begin - uvm_config_db#(virtual admission_bridge_control_if)::set(null, "uvm_test_top", "control", - control); - uvm_config_db#(virtual stream_if #(`AB_SNAP_WIDTH))::set(null, "uvm_test_top", "command", - command); - uvm_config_db#(virtual stream_if #(`AB_RET_WIDTH))::set(null, "uvm_test_top", "retirement", - retirement); - uvm_config_db#(virtual stream_if #(`AB_NPU_WIDTH))::set(null, "uvm_test_top", "npu", npu); -`ifdef AB_LATE_FAULT - uvm_config_db#(bit)::set(null, "uvm_test_top", "late_fault", 1); -`endif - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_control.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_control.sv deleted file mode 100644 index ee6908e7e..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_control.sv +++ /dev/null @@ -1,13 +0,0 @@ -`include "admission_config.svh" -interface admission_control_if ( - input logic clock -); - logic reset, retired, trapped, cancelled; - logic [63:0] retired_pc, trap_pc, trap_cause, trap_value; - logic [`ADMIT_MEMORY_OUTSTANDING_BITS-1:0] outstanding; - logic [`ADMIT_OUTSTANDING_BITS-1:0] admission_outstanding; - logic [`ADMIT_QUERY_WIDTH-1:0] query; - logic cpu_allow, accelerator_irq; - logic block_maintenance_response, maintenance_response_pending; - bit start = 0, finished = 0, hold_complete = 1; -endinterface diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_package.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_package.sv deleted file mode 100644 index 9fab23374..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_package.sv +++ /dev/null @@ -1,40 +0,0 @@ -package admission_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function chandle core_ref_create(input string path); - import "DPI-C" function void core_ref_destroy(input chandle model); - import "DPI-C" function void core_ref_read( - input chandle model, - input longint unsigned address, - output bit [511:0] data - ); - import "DPI-C" function void core_ref_write( - input chandle model, - input longint unsigned address, - input bit [511:0] data, - input bit [63:0] mask - ); - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual admission_control_if control; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 1ms; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual admission_control_if)::get(this, "", "control", control)) - `uvm_fatal("VIF", "missing admission interface") - endfunction - task execute(); - @(negedge control.clock); - control.start = 1; - wait (control.finished); - `uvm_info( - "ADMISSION_PASS", - "Actual Rocket/Core reservations, context snapshots, younger request ordering and completion/cancel verified", - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_tb.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_tb.sv deleted file mode 100644 index 23b9ca853..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/admission_tb.sv +++ /dev/null @@ -1,605 +0,0 @@ -module admission_tb; - import uvm_pkg::*; - import admission_pkg::*; - `include "uvm_macros.svh" - `include "admission_config.svh" - `include "admission_fixture.svh" - `define AF(B, K, F) B[`ADMIT_``K``_``F``_OFFSET +: `ADMIT_``K``_``F``_WIDTH] - logic clock = 0; - always #5 clock = ~clock; - admission_control_if control (clock); - stream_if #(`ADMIT_PTE_WIDTH) pte_req ( - clock, - control.reset - ); - stream_if #(`ADMIT_PTERESP_WIDTH) pte_resp ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_RESERVE_WIDTH) - ) reserve ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_COMMAND_WIDTH) - ) command ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_COMPLETE_WIDTH) - ) complete ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_CANCELLED_WIDTH) - ) cancelled ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_RESPONSE_WIDTH) - ) response ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_MAINTENANCE_WIDTH) - ) maintenance ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_MAINTAINED_WIDTH) - ) maintained ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_MEMREQ_WIDTH) - ) mem_req ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_MEMRESP_WIDTH) - ) mem_resp ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_UNCACHED_WIDTH) - ) uncached_req ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_URESP_WIDTH) - ) uncached_resp ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_RESERVE_WIDTH) - ) info ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`ADMIT_RESERVE_WIDTH) - ) cancel_request ( - clock, - control.reset - ); - logic tracker_complete_valid, tracker_complete_ready, tracker_cancel_ready; - logic [`ADMIT_COMPLETE_WIDTH-1:0] tracker_complete_tag; - logic maintenance_valid, grant_valid; - AdmissionCore dut ( - .clock(clock), - .reset(control.reset), - .io_resetVector(64'h80000000), - .io_time(64'b0), - .io_timerInterrupt(1'b0), - .io_softwareInterrupt(1'b0), - .io_externalInterrupt(1'b0), - .io_retired(control.retired), - .io_retiredPc(control.retired_pc), - .io_trapped(control.trapped), - .io_trapCause(control.trap_cause), - .io_trapValue(control.trap_value), - .io_trapPc(control.trap_pc), - .io_cancelledData(control.cancelled), - .io_memoryOutstanding(control.outstanding), - .io_blockMaintenanceResponse(control.block_maintenance_response), - .io_maintenanceResponsePending(control.maintenance_response_pending), - `include "admission_ports.svh" - ); - // The first opaque command owns one write range; real maintenance uses this Core and Home. - logic tracker_allow, tracker_maint_ready, tracker_done_ready, tracker_maintained_ready; - logic [7:0] tracker_grant_tag; - logic [7:0] tracker_maint_tag; - logic [43:0] tracker_maint_first, tracker_maint_last; - logic [1:0] tracker_maint_op; - logic dma_done_valid = 0; - logic [7:0] dma_done_tag = 0; - bit use_tracker_maintenance = 0; - assign control.cpu_allow = tracker_allow; - Interlock tracker ( - .clock(clock), - .reset(control.reset), - .io_dispatch_valid(reserve.valid), - .io_dispatch_ready(reserve.ready), - .io_dispatch_bits_id(reserve.bits), - .io_cancel_valid(cancel_request.valid && cancelled.ready), - .io_cancel_ready(tracker_cancel_ready), - .io_cancel_bits_id(cancel_request.bits), - .io_accessInfo_valid(info.valid), - .io_accessInfo_ready(info.ready), - .io_accessInfo_bits_id(info.bits), - .io_accessInfo_bits_hasMemory(1'b1), - .io_accessInfo_bits_base(44'h800040c0), - .io_accessInfo_bits_bytes(45'd64), - .io_accessInfo_bits_write(1'b1), - .io_accessInfo_bits_last(1'b1), - .io_cpuQuery_valid(`AF(control.query, QUERY, VALID)), - .io_cpuQuery_paddr(`AF(control.query, QUERY, PADDR)), - .io_cpuQuery_sizeLog2(`AF(control.query, QUERY, SIZELOG2)), - .io_cpuQuery_write(`AF(control.query, QUERY, WRITE)), - .io_cpuQuery_olderDispatchPending(`AF(control.query, QUERY, OLDERDISPATCHPENDING)), - .io_cpuAllow(tracker_allow), - .io_maintenance_valid(maintenance_valid), - .io_maintenance_bits_tag(tracker_maint_tag), - .io_maintenance_bits_firstLine(tracker_maint_first), - .io_maintenance_bits_lastLine(tracker_maint_last), - .io_maintenance_bits_op(tracker_maint_op), - .io_maintenance_ready(tracker_maint_ready), - .io_maintained_valid(use_tracker_maintenance && maintained.valid), - .io_maintained_ready(tracker_maintained_ready), - .io_maintained_bits_tag(`AF(maintained.bits, MAINTAINED, TAG)), - .io_maintained_bits_ok(`AF(maintained.bits, MAINTAINED, OK)), - .io_grant_valid(grant_valid), - .io_grant_bits_tag(tracker_grant_tag), - .io_grant_ready(1'b1), - .io_done_valid(dma_done_valid), - .io_done_ready(tracker_done_ready), - .io_done_bits_tag(dma_done_tag), - .io_done_bits_ok(1'b1), - .io_complete_valid(tracker_complete_valid), - .io_complete_ready(tracker_complete_ready), - .io_complete_bits_tag(tracker_complete_tag) - ); - assign complete.valid = tracker_complete_valid && !control.hold_complete; - assign complete.bits = tracker_complete_tag; - assign tracker_complete_ready = complete.ready && !control.hold_complete; - assign cancelled.valid = cancel_request.valid && tracker_cancel_ready; - assign cancelled.bits = cancel_request.bits; - assign cancel_request.ready = tracker_cancel_ready && cancelled.ready; - typedef struct { - int id, due; - longint unsigned addr; - bit write; - bit [511:0] data; - bit [63:0] mask; - } memory_entry; - memory_entry pending_memory[$], active_memory; - bit - memory_active = 0, - uncached_active = 0, - model_ready = 0, - old_retired = 0, - young_retired = 0, - exit_seen = 0, - exit_retired = 0; - bit cmo_done = 0; - int cmo_requests = 0, cmo_acks = 0, cmo_drain_waits = 0, cmo_late_rsp_waits = 0; - int - cycle = 0, - reservations = 0, - commands_seen = 0, - traps = 0, - completed = 0, - cancelled_count = 0, - old_pte_waits = 0; - logic [`ADMIT_URESP_WIDTH-1:0] uncached_result; - int uncached_due = 0; - int external_ptes = 0, pte_stalls = 0, grants = 0, dma_done_count = 0, blocked_cmo = 0; - bit live_tags[`ADMIT_ENTRIES]; - logic [`ADMIT_COMMAND_WIDTH-1:0] snapshots[2]; - chandle memory; - function automatic void check(bit condition, string message); - if (!condition) `uvm_fatal("ADMISSION", message) - endfunction - always @(posedge clock) begin - if (model_ready && !control.reset) begin - cycle++; - if (pte_req.valid && pte_req.ready) external_ptes++; - if (pte_resp.valid && !pte_resp.ready) pte_stalls++; - if (grant_valid) begin - check(tracker_grant_tag == `AF(snapshots[0], COMMAND, TAG), "wrong DMA grant tag"); - grants++; - end - if (dma_done_valid && tracker_done_ready) dma_done_count++; - if (maintenance.valid && !maintenance.ready) cmo_drain_waits++; - if (maintenance.valid && maintenance.ready) begin - check( - dut.core.cache.io_olderRequestsDrained && !dut.core.cacheOffer && !dut.core.cacheOwner && - !dut.core.uncachedOffer && !dut.core.uncachedOwner && !dut.core.pteDenied, - "CMO accepted before cache/PTE owners drained"); - if (!use_tracker_maintenance) check(dut.core.lsu.io_idle, "old CPU owner bypassed drain"); - else if (dut.core.state != 0 || !dut.core.lsu.io_idle) begin - check(!control.cpu_allow && `AF(control.query, QUERY, VALID), - "CMO bypassed a nonblocked busy broker"); - blocked_cmo++; - end - check(old_pte_waits > 0, "CMO fixture did not overlap actual page-table read"); - cmo_requests++; - end - if (control.maintenance_response_pending && control.block_maintenance_response) begin - cmo_late_rsp_waits++; - check(!maintained.valid, "CMO acknowledged before delayed real CHI completion"); - end - if (maintained.valid) begin - check(`AF(maintained.bits, MAINTAINED, TAG) == (use_tracker_maintenance ? - `AF(snapshots[0], COMMAND, TAG) : 8'he1) && `AF(maintained.bits, MAINTAINED, OK), - "Actual Core CMO failed or changed tag/status"); - end - if (maintained.valid && maintained.ready) cmo_acks++; - if (!use_tracker_maintenance) - check(!maintenance_valid && !grant_valid, "early tracker maintenance/grant"); - if (control.retired) begin - if (control.retired_pc == `ADMIT_OLD_LOAD_PC) old_retired = 1; - if (control.retired_pc == `ADMIT_YOUNG_LOAD_PC) begin - check(commands_seen >= 1, "young load retired before descriptor delivery"); - young_retired = 1; - end - if (control.retired_pc == `CORE_EXIT_PC) exit_retired = 1; - end - if (control.trapped) begin - check(traps == 0 && control.trap_cause == 2 && control.trap_pc == `ADMIT_FAULT_PC, - "unexpected precise trap"); - traps++; - end - if(dut.core.lsu.io_cpu_req_valid&&dut.core.lsu.io_cpu_req_ready&&dut.core.lsu.io_pc==`ADMIT_YOUNG_LOAD_PC) - check(reservations >= 1, "young EX memory accepted ahead of older RoCC reservation"); - if (reserve.valid && reserve.ready) begin - int tag; - tag = reserve.bits; - check(old_retired && tag < `ADMIT_ENTRIES && !live_tags[tag] && reservations < 2, - "early/duplicate/faulting reservation"); - live_tags[tag] = 1; - reservations++; - `uvm_info("RESERVE", $sformatf("tag%0d pc=%h", tag, dut.core.cpu.io_rocc_cmd_bits_pc), - UVM_LOW) - end - if (command.valid && command.ready) begin - int tag; - tag = `AF(command.bits, COMMAND, TAG); - check(commands_seen < 2 && live_tags[tag], "snapshot has no live reservation"); - snapshots[commands_seen] = command.bits; - commands_seen++; - `uvm_info("COMMAND", $sformatf("tag%0d pc=%h satp=%h priv%0d", tag, - `AF(command.bits, COMMAND, INSTRUCTION_PC), - `AF(command.bits, COMMAND, SATP), `AF(command.bits, COMMAND, - EFFECTIVEPRIVILEGE)), - UVM_LOW) - end - if (complete.valid && complete.ready) begin - check(live_tags[complete.bits], "duplicate completion"); - live_tags[complete.bits] = 0; - completed++; - end - if (cancelled.valid && cancelled.ready) begin - check(live_tags[cancelled.bits], "duplicate cancellation"); - live_tags[cancelled.bits] = 0; - cancelled_count++; - end - if (mem_resp.valid && mem_resp.ready) begin - check(memory_active && `AF(mem_resp.bits, MEMRESP, ID) == active_memory.id, - "DDR response identity changed"); - if (active_memory.write) - core_ref_write(memory, active_memory.addr, active_memory.data, active_memory.mask); - memory_active = 0; - end - if (mem_req.valid && mem_req.ready) begin - memory_entry item; - item.id = `AF(mem_req.bits, MEMREQ, ID); - item.addr = `AF(mem_req.bits, MEMREQ, ADDR); - item.write = `AF(mem_req.bits, MEMREQ, WRITE); - item.data = `AF(mem_req.bits, MEMREQ, DATA); - item.mask = `AF(mem_req.bits, MEMREQ, MASK); - if (!item.write) core_ref_read(memory, item.addr, item.data); - item.due = cycle + (item.addr == 64'h80006000 ? 40 : 9); - pending_memory.push_back(item); - if (item.addr == 64'h80006000) old_pte_waits++; - end - if (uncached_resp.valid && uncached_resp.ready) uncached_active = 0; - if (uncached_req.valid && uncached_req.ready) begin - check(!uncached_active, "uncached owner overlapped"); - uncached_result = '0; - `AF(uncached_result, URESP, TAG) = `AF(uncached_req.bits, UNCACHED, TAG); - uncached_due = cycle + 11; - check(`AF(uncached_req.bits, UNCACHED, WRITE) && `AF(uncached_req.bits, UNCACHED, ADDR) - == 64'h10000000, "unexpected device/PTE side effect"); - check(`AF(uncached_req.bits, UNCACHED, DATA) == 0, "firmware reported admission failure"); - exit_seen = 1; - uncached_active = 1; - end - end - end - always @(negedge clock) begin - if (model_ready && !control.reset) begin - mem_req.ready = cycle % 5 != 0; - if (!memory_active) begin - mem_resp.valid = 0; - for (int i = pending_memory.size() - 1; i >= 0; i--) - if (!memory_active && pending_memory[i].due <= cycle) begin - active_memory = pending_memory[i]; - pending_memory.delete(i); - memory_active = 1; - mem_resp.bits = '0; - `AF(mem_resp.bits, MEMRESP, ID) = active_memory.id; - `AF(mem_resp.bits, MEMRESP, DATA) = active_memory.write ? '0 : active_memory.data; - mem_resp.valid = 1; - end - end - uncached_req.ready = !uncached_active && cycle % 5 != 0; - uncached_resp.valid = uncached_active && cycle >= uncached_due; - uncached_resp.bits = uncached_result; - end - end - task automatic external_pte(input longint unsigned addr, expected, input bit error); - @(negedge clock); - pte_req.bits = '0; - `AF(pte_req.bits, PTE, ADDR) = addr; - pte_req.valid = 1; - do @(posedge clock); while (!pte_req.ready); - @(negedge clock); - pte_req.valid = 0; - wait (pte_resp.valid); - repeat (4) begin - @(posedge clock); - check(pte_resp.valid && `AF(pte_resp.bits, PTERESP, ERROR) == error && - `AF(pte_resp.bits, PTERESP, DATA) == expected, - "external PTE response owner/data/error unstable"); - end - @(negedge clock); - pte_resp.ready = 1; - do @(posedge clock); while (!pte_resp.valid); - @(negedge clock); - pte_resp.ready = 0; - endtask - task automatic check_context(logic [`ADMIT_COMMAND_WIDTH-1:0] packet, bit second); - check(`AF(packet, COMMAND, INSTRUCTION_OPCODE) == 7'h7b && - `AF(packet, COMMAND, INSTRUCTION_FUNCT3) == 3 && `AF(packet, COMMAND, INSTRUCTION_XS1) && - `AF(packet, COMMAND, INSTRUCTION_XS2) && !`AF(packet, COMMAND, INSTRUCTION_XD), - "stock RoCC classification/operands altered"); - check(`AF(packet, COMMAND, INSTRUCTION_PC) == (second ? `ADMIT_CMD1_PC : `ADMIT_CMD0_PC), - "command PC changed"); - check(`AF(packet, COMMAND, INSTRUCTION_RS1DATA) - == (second ? 64'h5555666677778888 : 64'h1111222233334444), "rs1 snapshot changed"); - check(`AF(packet, COMMAND, INSTRUCTION_RS2DATA) == (second ? 64'h400040c8 : 64'h400040c0), - "rs2 snapshot changed"); - check(`AF(packet, COMMAND, SATP) == (second ? 64'h8000300000080009 : 64'h8000000000080006), - "satp snapshot changed"); - check(`AF(packet, COMMAND, EFFECTIVEPRIVILEGE) == 1 && `AF(packet, COMMAND, SUM) - == !second && `AF(packet, COMMAND, MXR) == !second, "privilege/SUM/MXR snapshot changed"); - check(`AF(packet, COMMAND, PMP_0_ADDR) == (second ? 64'h20800000 : 64'h20400000) && - `AF(packet, COMMAND, PMP_0_CFG_A) == 1 && `AF(packet, COMMAND, PMP_0_CFG_R) && - `AF(packet, COMMAND, PMP_0_CFG_W) && `AF(packet, COMMAND, PMP_0_CFG_X) == !second && ! - `AF(packet, COMMAND, PMP_0_CFG_L), "PMP snapshot changed"); - endtask - always @(negedge clock) begin - if (use_tracker_maintenance) begin - maintenance.valid = maintenance_valid; - maintenance.bits = '0; - `AF(maintenance.bits, MAINTENANCE, TAG) = tracker_maint_tag; - `AF(maintenance.bits, MAINTENANCE, FIRSTLINE) = tracker_maint_first; - `AF(maintenance.bits, MAINTENANCE, LASTLINE) = tracker_maint_last; - `AF(maintenance.bits, MAINTENANCE, OP) = tracker_maint_op; - maintained.ready = tracker_maintained_ready; - end - end - always_comb tracker_maint_ready = use_tracker_maintenance && maintenance.ready; - initial begin - uvm_config_db#(virtual admission_control_if)::set(null, "*", "control", control); - run_test("protocol_test"); - end - initial begin - pte_req.valid = 0; - pte_req.bits = '0; - pte_resp.ready = 0; - maintenance.valid = 0; - maintenance.bits = '0; - maintained.ready = 0; - control.block_maintenance_response = 1; - control.reset = 1; - control.accelerator_irq = 0; - command.ready = 0; - response.valid = 0; - response.bits = '0; - info.valid = 0; - info.bits = '0; - cancel_request.valid = 0; - cancel_request.bits = '0; - mem_req.ready = 0; - mem_resp.valid = 0; - mem_resp.bits = '0; - uncached_req.ready = 0; - uncached_resp.valid = 0; - uncached_resp.bits = '0; - foreach (live_tags[i]) live_tags[i] = 0; - wait (control.start); - memory = core_ref_create(`ADMIT_IMAGE); - model_ready = 1; - repeat (4) @(posedge clock); - @(negedge clock); - control.reset = 0; - fork - begin - wait (old_pte_waits > 0); - @(negedge clock); - maintenance.bits = '0; - `AF(maintenance.bits, MAINTENANCE, TAG) = 8'he1; - `AF(maintenance.bits, MAINTENANCE, FIRSTLINE) = 44'h800040c0; - `AF(maintenance.bits, MAINTENANCE, LASTLINE) = 44'h800040c0; - `AF(maintenance.bits, MAINTENANCE, OP) = 0; - maintenance.valid = 1; - do @(posedge clock); while (!maintenance.ready); - @(negedge clock); - maintenance.valid = 0; - wait (control.maintenance_response_pending); - repeat (8) begin - @(posedge clock); - #1; - check(!maintained.valid && reservations == 0, - "Late CMO completion released new admission"); - end - @(negedge clock); - control.block_maintenance_response = 0; - wait (maintained.valid); - begin - logic [`ADMIT_MAINTAINED_WIDTH-1:0] held; - held = maintained.bits; - repeat (8) begin - @(posedge clock); - #1; - check(maintained.valid && maintained.bits === held && reservations == 0, - "Held CMO ACK changed or released new admission"); - end - end - @(negedge clock); - maintained.ready = 1; - do @(posedge clock); while (!maintained.valid); - @(negedge clock); - maintained.ready = 0; - cmo_done = 1; - end - join_none - wait (command.valid); - begin - logic [`ADMIT_COMMAND_WIDTH-1:0] held; - held = command.bits; - check_context(held, 0); - repeat (8) begin - @(posedge clock); - #1; - check(command.valid && command.bits === held && reservations == 1 && !young_retired, - "snapshot/young request escaped admission stall"); - end - end - @(negedge clock); - command.ready = 1; - wait (commands_seen == 1); - @(negedge clock); - command.ready = 0; - // A younger authorized physical query waits behind the unknown reservation. - wait (!control.cpu_allow && `AF(control.query, QUERY, VALID)); - external_pte(64'h80006008, 64'h20001c01, 0); - external_pte(64'ha0000008, 0, 1); - external_pte(64'h10000000, 0, 1); - @(negedge clock); - pte_req.bits = '0; - `AF(pte_req.bits, PTE, ADDR) = 44'h80006008; - pte_req.valid = 1; - do @(posedge clock); while (!pte_req.ready); - @(negedge clock); - pte_req.valid = 0; - wait (pte_resp.valid); - check(!`AF(pte_resp.bits, PTERESP, ERROR) && `AF(pte_resp.bits, PTERESP, DATA) - == 64'h20001c01, "external PTE did not return the coherent cached value"); - use_tracker_maintenance = 1; - info.bits = `AF(snapshots[0], COMMAND, TAG); - info.valid = 1; - do @(posedge clock); while (!info.ready); - @(negedge clock); - info.valid = 0; - wait (maintenance_valid); - repeat (8) begin - @(posedge clock); - #1; - check(pte_resp.valid && !tracker_maint_ready && !young_retired, - "held external PTE did not block maintenance or younger access"); - end - @(negedge clock); - pte_resp.ready = 1; - do @(posedge clock); while (!pte_resp.valid); - @(negedge clock); - pte_resp.ready = 0; - wait (grants == 1); - wait (!control.cpu_allow && - `AF(control.query, QUERY, VALID) - && - `AF(control.query, QUERY, PADDR) - == 44'h800040c8 && dut.core.state != 0); - repeat (8) begin - @(posedge clock); - #1; - check(!young_retired, "young CPU escaped live write range"); - end - @(negedge clock); - control.hold_complete = 0; - dma_done_tag = `AF(snapshots[0], COMMAND, TAG); - dma_done_valid = 1; - do @(posedge clock); while (!tracker_done_ready); - @(negedge clock); - dma_done_valid = 0; - wait (command.valid); - check_context(command.bits, 1); - check(completed == 1, "second command proceeded before real first completion"); - @(negedge clock); - command.ready = 1; - wait (commands_seen == 2); - @(negedge clock); - command.ready = 0; - cancel_request.bits = `AF(snapshots[1], COMMAND, TAG); - cancel_request.valid = 1; - do @(posedge clock); while (!cancel_request.ready); - @(negedge clock); - cancel_request.valid = 0; - control.hold_complete = 0; - wait(cmo_done&&exit_seen&&exit_retired&&completed==1&&cancelled_count==1&&control.admission_outstanding==0&&control.outstanding==0); - repeat (8) @(posedge clock); - check( - reservations == 2 && commands_seen == 2 && traps == 1 && old_pte_waits > 0 && young_retired, - "actual Core admission fixture did not complete required observations"); - check(!memory_active && pending_memory.size() == 0 && !uncached_active, - "memory responses not drained"); - check( - cmo_requests == 3 && cmo_acks == 3 && cmo_drain_waits > 0 && cmo_late_rsp_waits >= 8 && - external_ptes==4 && pte_stalls>=8 && grants==1 && dma_done_count==1 && blocked_cmo>0, - $sformatf( - "CMO requirements req%0d ack%0d oldwait%0d latewait%0d pte%0d stalls%0d grant%0d done%0d blocked%0d", - cmo_requests, - cmo_acks, - cmo_drain_waits, - cmo_late_rsp_waits, - external_ptes, - pte_stalls, - grants, - dma_done_count, - blocked_cmo - )); - `uvm_info("CORE_MAINTENANCE_PASS", $sformatf( - "request=%0d ack=%0d olderOwnerWait=%0d lateCHIWait=%0d; actual Core CMO and held ACK checked", - cmo_requests, - cmo_acks, - cmo_drain_waits, - cmo_late_rsp_waits - ), UVM_LOW) - `uvm_info("CHECKED", $sformatf( - "reserve%0d cmd%0d complete%0d cancel%0d preciseTrap%0d oldPTE%0d", - reservations, - commands_seen, - completed, - cancelled_count, - traps, - old_pte_waits - ), UVM_LOW) - core_ref_destroy(memory); - model_ready = 0; - control.finished = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/build_fixture.py b/arch/src/main/scala/framework/root/root-core/src/main/resources/build_fixture.py deleted file mode 100644 index 6c909b30a..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/build_fixture.py +++ /dev/null @@ -1,108 +0,0 @@ -from pathlib import Path -import argparse -import subprocess - -parser = argparse.ArgumentParser() -parser.add_argument("destination") -parser.add_argument( - "fixture", - choices=("core", "supervisor", "multicore", "irq", "admission", "fpu"), - nargs="?", - default="core", -) -options = parser.parse_args() -output = Path(options.destination) -fixture = options.fixture -output.mkdir(parents=True, exist_ok=True) -source = Path(__file__).resolve().parent -subprocess.run( - [ - "riscv64-none-elf-gcc", - ( - "-march=rv64imafdc_zicsr_zifencei" - if fixture == "fpu" - else "-march=rv64imac_zicsr_zifencei" - ), - "-mabi=lp64", - "-nostdlib", - "-nostartfiles", - "-Wl,--build-id=none", - "-Wl,--no-relax", - "-T", - str(source / f"{fixture}.ld"), - str(source / f"{fixture}.S"), - "-o", - str(output / f"{fixture}.elf"), - ], - check=True, -) -subprocess.run( - [ - "riscv64-none-elf-objcopy", - "-O", - "verilog", - str(output / f"{fixture}.elf"), - str(output / f"{fixture}.hex"), - ], - check=True, -) -image = output / f"{fixture}.hex" -image.write_bytes(image.read_bytes().replace(b"\r\n", b"\n")) -disassembly = subprocess.run( - [ - "riscv64-none-elf-objdump", - "-d", - str(output / f"{fixture}.elf"), - ], - check=True, - capture_output=True, - text=True, -).stdout -(output / f"{fixture}.dis").write_text(disassembly) -import re - -exits = re.findall(r"^\s*([0-9a-f]+):[^\n]*\bsd\s+zero,0\(s0\)", disassembly, re.M) -if len(exits) != 1: - raise RuntimeError(f"Expected one successful exit SD in firmware, found {exits}") -definitions = [f"`define CORE_EXIT_PC 64'h{exits[0]}"] -if fixture in ("supervisor", "irq", "admission", "fpu"): - symbols = subprocess.run( - [ - "riscv64-none-elf-nm", - "--defined-only", - str(output / f"{fixture}.elf"), - ], - check=True, - capture_output=True, - text=True, - ).stdout - addresses = { - name: int(address, 16) - for address, _, name in (line.split() for line in symbols.splitlines()) - } -if fixture == "irq": - for name in ("irq_load", "irq_amo", "irq_store"): - definitions.append(f"`define {name.upper()}_PC 64'h{addresses[name]:x}") -if fixture == "fpu": - definitions.append(f"`define FPU_ILLEGAL_PC 64'h{addresses['fpu_illegal']:x}") -if fixture == "admission": - for name, address in sorted(addresses.items()): - if name.startswith("admit_"): - definitions.append(f"`define {name.upper()}_PC 64'h{address:x}") -if fixture == "supervisor": - traps = [ - ("trap_boundary", 1, 0x4000A008), - ("trap_store", 15, 0x40008000), - ("trap_sum", 13, 0x40006000), - ("trap_user_load", 13, 0x40008000), - ("trap_user_ecall", 8, 0), - ("trap_supervisor_ecall", 9, 0), - ] - definitions.append(f"`define SUPERVISOR_TRAP_COUNT {len(traps)}") - for index, (name, cause, value) in enumerate(traps): - definitions += [ - f"`define SUPERVISOR_TRAP_{index}_PC 64'h{addresses[name] - 0x40000000:x}", - f"`define SUPERVISOR_TRAP_{index}_CAUSE 64'd{cause}", - f"`define SUPERVISOR_TRAP_{index}_VALUE 64'h{value:x}", - ] -(output / f"{fixture}_fixture.svh").write_text("\n".join(definitions) + "\n") diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.S b/arch/src/main/scala/framework/root/root-core/src/main/resources/core.S deleted file mode 100644 index 5cde0ed34..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.S +++ /dev/null @@ -1,203 +0,0 @@ -.option norelax -.section .text.start,"ax",@progbits -.globl _start -_start: - la t0, trap_handler - csrw mtvec, t0 - li s0, 0x10000000 - la s1, data - li s9, -1 - j tests - -.section .text.trap,"ax",@progbits -.option push -.option norvc -trap_handler: - csrr t0, mcause - bne t0, s9, fail - csrr t0, mtval - bne t0, s10, fail - csrr t0, mepc - addi t0, t0, 4 - csrw mepc, t0 - li s9, -1 - mret -.option pop - -.section .text.tests,"ax",@progbits -tests: - li s2, 1 - sd s2, 8(s0) - li t0, 0x8877665544332211 - sd t0, 0(s1) - ld t1, 0(s1) - bne t0, t1, fail - lb t0, 7(s1) - li t1, -120 - bne t0, t1, fail - lbu t0, 7(s1) - li t1, 0x88 - bne t0, t1, fail - lh t0, 6(s1) - li t1, -30601 - bne t0, t1, fail - lhu t0, 6(s1) - li t1, 0x8877 - bne t0, t1, fail - lw t0, 4(s1) - li t1, 0xffffffff88776655 - bne t0, t1, fail - lwu t0, 4(s1) - li t1, 0x88776655 - bne t0, t1, fail - li t0, 0xaa - sb t0, 1(s1) - li t0, 0xbeef - sh t0, 2(s1) - li t0, 0xcafebabe - sw t0, 4(s1) - ld t0, 0(s1) - li t1, 0xcafebabebeefaa11 - bne t0, t1, fail - - li s2, 2 - sd s2, 8(s0) - li t0, 0x7fffffff - sw t0, 0(s1) - li t0, 1 - amoadd.w t1, t0, (s1) - li t2, 0x7fffffff - bne t1, t2, fail - ld t1, 0(s1) - li t2, 0xcafebabe80000000 - bne t1, t2, fail - addi t3, s1, 4 - amoadd.w t1, t0, (t3) - li t2, 0xffffffffcafebabe - bne t1, t2, fail - ld t1, 0(s1) - li t2, 0xcafebabf80000000 - bne t1, t2, fail - li t0, 0x1122334455667788 - amoswap.d t1, t0, (s1) - bne t1, t2, fail - ld t1, 0(s1) - bne t1, t0, fail - - li s2, 3 - sd s2, 8(s0) - li t4, 32 - j lr_word - .balign 512 -lr_word: - lr.w t0, (s1) - addi t0, t0, 1 - sc.w t1, t0, (s1) - beqz t1, word_done - addi t4, t4, -1 - bnez t4, lr_word - j fail -word_done: - ld t0, 0(s1) - li t1, 0x1122334455667789 - bne t0, t1, fail - li t4, 32 - j lr_double - .balign 512 -lr_double: - lr.d t0, (s1) - addi t0, t0, 1 - sc.d t1, t0, (s1) - beqz t1, double_done - addi t4, t4, -1 - bnez t4, lr_double - j fail -double_done: - ld t0, 0(s1) - li t1, 0x112233445566778a - bne t0, t1, fail - - li s2, 4 - sd s2, 8(s0) - li t0, 1 - beqz t0, fail - li t0, 0 - bnez t0, fail - li t0, 0xdeadbeef - beq zero, zero, branch_done - sd t0, 0(s1) -branch_done: - ld t0, 0(s1) - li t1, 0x112233445566778a - bne t0, t1, fail - jal ra, patch_target - li t0, 7 - bne a0, t0, fail - la t0, patch_target - li t1, 0x00900513 - sw t1, 0(t0) - fence.i - jal ra, patch_target - li t0, 9 - bne a0, t0, fail - - li s2, 5 - sd s2, 8(s0) -.option push -.option norvc - li s9, 11 - li s10, 0 - ecall - li t1, -1 - bne s9, t1, fail - li t3, 0x20000000 - li s9, 5 - mv s10, t3 - ld t0, 0(t3) - li t1, -1 - bne s9, t1, fail - li s9, 7 - sd t0, 0(t3) - li t1, -1 - bne s9, t1, fail -.option pop - - li s2, 6 - sd s2, 8(s0) - srli t0, s1, 2 - csrw pmpaddr0, t0 - addi t0, t0, 8 - csrw pmpaddr1, t0 - li t0, 0x8800 - csrw pmpcfg0, t0 -.option push -.option norvc - li s9, 5 - mv s10, s1 - ld t0, 0(s1) - li t1, -1 - bne s9, t1, fail - li s9, 7 - sd t0, 0(s1) - li t1, -1 - bne s9, t1, fail -.option pop - li s2, 7 - sd s2, 8(s0) - sd zero, 0(s0) -1: j 1b - -fail: - ori t0, s2, 0x100 - sd t0, 0(s0) -2: j 2b - -.balign 8 -patch_target: - .word 0x00700513 - ret - -.section .data,"aw",@progbits -.balign 8 -data: - .quad 0 diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.cm_hier b/arch/src/main/scala/framework/root/root-core/src/main/resources/core.cm_hier deleted file mode 100644 index 0f4845487..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.cm_hier +++ /dev/null @@ -1,5 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.mem_req -+tree @TOP@.mem_resp -+tree @TOP@.uncached_req -+tree @TOP@.uncached_resp diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/core.f deleted file mode 100644 index 2fb40b16b..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/core_interface.sv -@RESOURCES@/core_package.sv --F @RTL@/Core/filelist.f -@RESOURCES@/core_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.ld b/arch/src/main/scala/framework/root/root-core/src/main/resources/core.ld deleted file mode 100644 index 7b1347647..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core.ld +++ /dev/null @@ -1,14 +0,0 @@ -OUTPUT_ARCH(riscv) -ENTRY(_start) -SECTIONS -{ - . = 0x80000000; - .text.start : { *(.text.start) } - . = 0x80000200; - .text.trap : { *(.text.trap) } - . = 0x80000400; - .text.tests : { *(.text.tests) } - . = 0x800040c0; - .data : { *(.data) } - /DISCARD/ : { *(.riscv.attributes) *(.comment) } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_interface.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/core_interface.sv deleted file mode 100644 index 0fa5a6b79..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_interface.sv +++ /dev/null @@ -1,17 +0,0 @@ -interface core_control_if ( - input logic clock -); - logic reset; - logic timer_irq, software_irq, external_irq; - logic load_waiting, amo_completed, store_completed; - logic [2:0] target_access; - logic cancelled, retired, trapped; - logic [63:0] retired_pc, trap_cause, trap_value, trap_pc; - logic [2:0] outstanding; - clocking cb @(posedge clock); - default input #1step output #0; - output reset, timer_irq, software_irq, external_irq; - input load_waiting, amo_completed, store_completed, target_access; - input cancelled, outstanding, retired, retired_pc, trapped, trap_cause, trap_value, trap_pc; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_package.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/core_package.sv deleted file mode 100644 index ae084188b..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_package.sv +++ /dev/null @@ -1,34 +0,0 @@ -`include "core_config.svh" -`ifdef IRQ_PROFILE -`define TEST_IMAGE `IRQ_IMAGE -`include "irq_fixture.svh" -`elsif SUPERVISOR_PROFILE -`define TEST_IMAGE `SUPERVISOR_IMAGE -`include "supervisor_fixture.svh" -`elsif FPU_PROFILE -`define TEST_IMAGE `FPU_IMAGE -`include "fpu_fixture.svh" -`else -`define TEST_IMAGE `CORE_IMAGE -`include "core_fixture.svh" -`endif -package core_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `define CF(kind, field) `CORE_``kind``_``field``_OFFSET +: `CORE_``kind``_``field``_WIDTH - import "DPI-C" function chandle core_ref_create(input string path); - import "DPI-C" function void core_ref_destroy(input chandle model); - import "DPI-C" function void core_ref_read( - input chandle model, - input longint unsigned address, - output bit [511:0] data - ); - import "DPI-C" function void core_ref_write( - input chandle model, - input longint unsigned address, - input bit [511:0] data, - input bit [63:0] mask - ); - `include "core_test.svh" -endpackage diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_tb.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/core_tb.sv deleted file mode 100644 index c95275f47..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_tb.sv +++ /dev/null @@ -1,85 +0,0 @@ -`include "core_config.svh" -`ifdef IRQ_PROFILE -`define CORE_TB irq_tb -`define CORE_DUT Irq -`elsif SUPERVISOR_PROFILE -`define CORE_TB supervisor_tb -`define CORE_DUT Supervisor -`elsif FPU_PROFILE -`define CORE_TB fpu_tb -`define CORE_DUT Fpu -`else -`define CORE_TB core_tb -`define CORE_DUT Core -`endif -module `CORE_TB; - import uvm_pkg::*; - import core_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - logic [63:0] time_value = 0; - always @(posedge clock) time_value <= control.reset ? 0 : time_value + 1; - core_control_if control (clock); - stream_if #(`CORE_MEMREQ_WIDTH) mem_req ( - clock, - control.reset - ); - stream_if #(`CORE_MEMRESP_WIDTH) mem_resp ( - clock, - control.reset - ); - stream_if #(`CORE_UNCACHED_WIDTH) uncached_req ( - clock, - control.reset - ); - stream_if #(`CORE_URESP_WIDTH) uncached_resp ( - clock, - control.reset - ); - `CORE_DUT dut ( - .clock(clock), - .reset(control.reset), - .io_resetVector(64'h80000000), - .io_time(time_value), -`ifdef IRQ_PROFILE - .io_timerInterrupt(control.timer_irq), - .io_softwareInterrupt(control.software_irq), - .io_externalInterrupt(control.external_irq), -`else - .io_timerInterrupt(1'b0), - .io_softwareInterrupt(1'b0), - .io_externalInterrupt(1'b0), -`endif - .io_retired(control.retired), - .io_retiredPc(control.retired_pc), - .io_trapped(control.trapped), - .io_trapCause(control.trap_cause), - .io_trapValue(control.trap_value), - .io_trapPc(control.trap_pc), - .io_cancelledData(control.cancelled), - .io_memoryOutstanding(control.outstanding), - `include "core_ports.svh" - ); -`ifdef IRQ_PROFILE - `include "irq_fixture.svh" - wire demand_fire = dut.core.cache.io_access_valid && dut.core.cache.io_access_ready; - assign control.target_access[0] = demand_fire && dut.core.cache.io_access_bits_addr == 44'h800040c0 && !dut.core.cache.io_access_bits_write && dut.core.cache.io_access_bits_atomic == 0; - assign control.target_access[1] = demand_fire && dut.core.cache.io_access_bits_addr == 44'h800040c8 && dut.core.cache.io_access_bits_atomic == 2; - assign control.target_access[2] = demand_fire && dut.core.cache.io_access_bits_addr == 44'h800040d0 && dut.core.cache.io_access_bits_write && dut.core.cache.io_access_bits_atomic == 0; - assign control.load_waiting = dut.core.lsu.io_response_ready && dut.core.lsu.instructionPc == `IRQ_LOAD_PC; - assign control.amo_completed = dut.core.lsu.io_cpu_req_ready && !dut.core.lsu.io_idle && dut.core.lsu.instructionPc == `IRQ_AMO_PC; - assign control.store_completed = dut.core.lsu.io_cpu_req_ready && !dut.core.lsu.io_idle && dut.core.lsu.instructionPc == `IRQ_STORE_PC; -`endif - initial begin - uvm_config_db#(virtual core_control_if)::set(null, "uvm_test_top*", "control", control); - uvm_config_db#(virtual stream_if #(`CORE_MEMREQ_WIDTH))::set(null, "uvm_test_top*", "mem_req", - mem_req); - uvm_config_db#(virtual stream_if #(`CORE_MEMRESP_WIDTH))::set(null, "uvm_test_top*", "mem_resp", - mem_resp); - uvm_config_db#(virtual stream_if #(`CORE_UNCACHED_WIDTH))::set(null, "uvm_test_top*", - "uncached_req", uncached_req); - uvm_config_db#(virtual stream_if #(`CORE_URESP_WIDTH))::set(null, "uvm_test_top*", - "uncached_resp", uncached_resp); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_test.svh b/arch/src/main/scala/framework/root/root-core/src/main/resources/core_test.svh deleted file mode 100644 index abf0426c5..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/core_test.svh +++ /dev/null @@ -1,321 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual core_control_if control; - virtual stream_if #(`CORE_MEMREQ_WIDTH) mem_req; - virtual stream_if #(`CORE_MEMRESP_WIDTH) mem_resp; - virtual stream_if #(`CORE_UNCACHED_WIDTH) uncached_req; - virtual stream_if #(`CORE_URESP_WIDTH) uncached_resp; - chandle model; - bit [`CORE_MEMRESP_WIDTH-1:0] memory_queue[$]; - bit [`CORE_URESP_WIDTH-1:0] mmio_queue[$]; - bit memory_active = 0, mmio_active = 0, exited = 0; - int stage = 0, cycles = 0, reads = 0, writes = 0, cancellations = 0; - longint unsigned exit_code; -`ifdef IRQ_PROFILE - localparam int STAGES = 3; - localparam int TRAPS = 3; - bit [2:0] raised = 0, target_retired = 0; - int acknowledgements = 0, memory_release = 0; - int target_retire_count[3] = '{default: 0}; - int target_access_count[3] = '{default: 0}; -`elsif SUPERVISOR_PROFILE - localparam int STAGES = 3; - localparam int TRAPS = `SUPERVISOR_TRAP_COUNT; -`elsif FPU_PROFILE - localparam int STAGES = 4; - localparam int TRAPS = 1; -`else - localparam int STAGES = 7; - localparam int TRAPS = 5; -`endif - int traps = 0, retired = 0; - bit exit_retired = 0; - longint unsigned exit_pc; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 5ms; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual core_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`CORE_MEMREQ_WIDTH))::get( - this, "", "mem_req", mem_req - ) || !uvm_config_db#(virtual stream_if #(`CORE_MEMRESP_WIDTH))::get( - this, "", "mem_resp", mem_resp - ) || !uvm_config_db#(virtual stream_if #(`CORE_UNCACHED_WIDTH))::get( - this, "", "uncached_req", uncached_req - ) || !uvm_config_db#(virtual stream_if #(`CORE_URESP_WIDTH))::get( - this, "", "uncached_resp", uncached_resp - )) - `uvm_fatal("VIF", "Core verification interfaces missing") - model = core_ref_create(`TEST_IMAGE); - endfunction - task drive_memory(); - forever begin - @(posedge control.clock); - if (control.reset) begin - mem_req.ready <= 0; - uncached_req.ready <= 0; - mem_resp.valid <= 0; - mem_resp.bits <= '0; - uncached_resp.valid <= 0; - uncached_resp.bits <= '0; - memory_queue.delete(); - mmio_queue.delete(); - memory_active = 0; - mmio_active = 0; - end else begin - cycles++; -`ifdef IRQ_PROFILE - foreach (target_access_count[i]) if (control.target_access[i]) target_access_count[i]++; -`endif - if (control.cancelled) cancellations++; - if (control.retired) begin - retired++; -`ifdef IRQ_PROFILE - if (control.retired_pc == `IRQ_LOAD_PC) begin - target_retired[0] = 1; - target_retire_count[0]++; - end - if (control.retired_pc == `IRQ_AMO_PC) begin - target_retired[1] = 1; - target_retire_count[1]++; - end - if (control.retired_pc == `IRQ_STORE_PC) begin - target_retired[2] = 1; - target_retire_count[2]++; - end -`endif - if (exited && control.retired_pc == exit_pc) exit_retired = 1; - end - if (control.trapped) begin - int expected_cause = traps == 0 ? 11 : (traps % 2 == 1 ? 5 : 7); - longint unsigned expected_value = traps == 0 ? 0 : (traps <= 2 ? 64'h20000000 : 64'h800040c0); -`ifdef IRQ_PROFILE - expected_cause = traps == 0 ? 7 : (traps == 1 ? 3 : 11); - expected_value = 0; - if (traps >= 3 || !raised[traps] || !target_retired[traps] || - control.trap_cause != (64'h8000000000000000 | expected_cause) || control.trap_value != 0) - `uvm_fatal("IRQ", - "Interrupt preceded original retirement or had an unexpected cause/value") -`elsif SUPERVISOR_PROFILE - longint unsigned expected_pc; - case (traps) - 0: begin - expected_cause = `SUPERVISOR_TRAP_0_CAUSE; - expected_value = `SUPERVISOR_TRAP_0_VALUE; - expected_pc = `SUPERVISOR_TRAP_0_PC; - end - 1: begin - expected_cause = `SUPERVISOR_TRAP_1_CAUSE; - expected_value = `SUPERVISOR_TRAP_1_VALUE; - expected_pc = `SUPERVISOR_TRAP_1_PC; - end - 2: begin - expected_cause = `SUPERVISOR_TRAP_2_CAUSE; - expected_value = `SUPERVISOR_TRAP_2_VALUE; - expected_pc = `SUPERVISOR_TRAP_2_PC; - end - 3: begin - expected_cause = `SUPERVISOR_TRAP_3_CAUSE; - expected_value = `SUPERVISOR_TRAP_3_VALUE; - expected_pc = `SUPERVISOR_TRAP_3_PC; - end - 4: begin - expected_cause = `SUPERVISOR_TRAP_4_CAUSE; - expected_value = `SUPERVISOR_TRAP_4_VALUE; - expected_pc = `SUPERVISOR_TRAP_4_PC; - end - 5: begin - expected_cause = `SUPERVISOR_TRAP_5_CAUSE; - expected_value = `SUPERVISOR_TRAP_5_VALUE; - expected_pc = `SUPERVISOR_TRAP_5_PC; - end - endcase - if (control.trap_pc != expected_pc) - `uvm_fatal("TRAP_PC", "Supervisor trap PC differs from fixture symbol") -`elsif FPU_PROFILE - expected_cause = 2; - expected_value = 64'h53; - if (control.trap_pc != `FPU_ILLEGAL_PC) - `uvm_fatal("TRAP_PC", "FPU FS-off trap PC differs from fixture symbol") -`endif -`ifndef IRQ_PROFILE - if (traps >= TRAPS || control.trap_cause != expected_cause || control.trap_value != expected_value) - `uvm_fatal("TRAP", $sformatf( - "Unexpected precise trap count=%0d cause=%h value=%h pc=%h", - traps, - control.trap_cause, - control.trap_value, - control.trap_pc - )) -`endif - traps++; - end -`ifdef IRQ_PROFILE - if (!raised[1] && acknowledgements == 1 && control.amo_completed) begin - control.software_irq <= 1; - raised[1] = 1; - `uvm_info("IRQ_RAISE", "Software level asserted while AMO result was buffered", UVM_LOW) - end - if (!raised[2] && acknowledgements == 2 && control.store_completed) begin - control.external_irq <= 1; - raised[2] = 1; - `uvm_info("IRQ_RAISE", "External level asserted while store completion was buffered", - UVM_LOW) - end -`endif - mem_req.ready <= cycles % 7 > 1; - uncached_req.ready <= cycles % 7 > 1; - if (mem_req.valid && mem_req.ready) begin - bit [`CORE_MEMRESP_WIDTH-1:0] result = '0; - bit [511:0] data; - longint unsigned address = mem_req.bits[`CF(MEMREQ, ADDR)]; -`ifdef IRQ_PROFILE - if (!raised[0] && address == 64'h800040c0 && !mem_req.bits[`CF(MEMREQ, WRITE)]) begin - if (!control.load_waiting) - `uvm_fatal("IRQ_TIMING", "Cold data DDR request did not belong to receiving load") - control.timer_irq <= 1; - raised[0] = 1; - memory_release = cycles + 20; - `uvm_info("IRQ_RAISE", - "Timer level asserted with cold-load DDR response held for 20 cycles", - UVM_LOW) - end -`endif - result[`CF(MEMRESP, ID)] = mem_req.bits[`CF(MEMREQ, ID)]; - if (mem_req.bits[`CF(MEMREQ, WRITE)]) begin - core_ref_write(model, address, mem_req.bits[`CF(MEMREQ, DATA)], mem_req.bits[ - `CF(MEMREQ, MASK)]); - writes++; - end else begin - core_ref_read(model, address, data); - result[`CF(MEMRESP, DATA)] = data; - reads++; - end - memory_queue.push_back(result); - end - if (uncached_req.valid && uncached_req.ready) begin - bit [`CORE_URESP_WIDTH-1:0] result = '0; - longint unsigned address = uncached_req.bits[`CF(UNCACHED, ADDR)]; - longint unsigned value = uncached_req.bits[`CF(UNCACHED, DATA)]; - bit bad_mmio, mmio_handled = 0; - result[`CF(URESP, TAG)] = uncached_req.bits[`CF(UNCACHED, TAG)]; - bad_mmio = !uncached_req.bits[ - `CF(UNCACHED, WRITE) - ] || uncached_req.bits[ - `CF(UNCACHED, SIZE) - ] != 3 || (address != 64'h10000000 && address != 64'h10000008); -`ifdef IRQ_PROFILE - if (uncached_req.bits[ - `CF(UNCACHED, WRITE) - ] && uncached_req.bits[ - `CF(UNCACHED, SIZE) - ] == 3 && address == 64'h10000010) - bad_mmio = 0; -`endif - if (bad_mmio) `uvm_fatal("MMIO", "Unexpected or expanded Core MMIO operation") -`ifdef IRQ_PROFILE - if (address == 64'h10000010) begin - longint unsigned expected = acknowledgements == 0 ? 64'h80 : (acknowledgements == 1 ? 64'h8 : 64'h800); - if (acknowledgements >= 3 || traps != acknowledgements + 1 || !raised[acknowledgements] || value != expected) - `uvm_fatal("IRQ_ACK", "IRQ ACK did not match its pending level/cause") - case (acknowledgements) - 0: control.timer_irq <= 0; - 1: control.software_irq <= 0; - 2: control.external_irq <= 0; - endcase - acknowledgements++; - mmio_handled = 1; - end -`endif - if (!mmio_handled && address == 64'h10000008) begin - if (value != stage + 1 || value > STAGES) - `uvm_fatal("STAGE", "Firmware stage signature out of order") - stage = value; - `uvm_info("STAGE", $sformatf("Completed firmware stage %0d", stage), UVM_LOW) - end else if (!mmio_handled) begin - exit_code = value; - // The final exit SD has rd/tag zero; observe its later precise architectural retirement. - exit_pc = `CORE_EXIT_PC; - if (value != 0 || stage != STAGES) - `uvm_fatal("FIRMWARE", $sformatf("Firmware failed: stage=%0d exit=%h", stage, value)) - exited = 1; - end - mmio_queue.push_back(result); - end - if (memory_active && mem_resp.ready) memory_active = 0; - if (mmio_active && uncached_resp.ready) mmio_active = 0; - begin - bit release_mem; - release_mem = !memory_active && memory_queue.size() != 0 && cycles % 5 == 0; -`ifdef IRQ_PROFILE - release_mem = release_mem && cycles >= memory_release; -`endif - if (release_mem) begin - mem_resp.bits <= memory_queue.pop_front(); - memory_active = 1; - end - end - if (!mmio_active && mmio_queue.size() != 0 && cycles % 5 == 0) begin - uncached_resp.bits <= mmio_queue.pop_front(); - mmio_active = 1; - end - mem_resp.valid <= memory_active; - uncached_resp.valid <= mmio_active; - end - end - endtask - task execute(); - control.reset = 1; - control.timer_irq = 0; - control.software_irq = 0; - control.external_irq = 0; - mem_req.ready = 0; - uncached_req.ready = 0; - mem_resp.valid = 0; - mem_resp.bits = '0; - uncached_resp.valid = 0; - uncached_resp.bits = '0; - repeat (5) @(control.cb); - control.cb.reset <= 0; - fork - drive_memory(); - join_none - wait (exited && exit_retired); - if (traps != TRAPS || retired < 100) - `uvm_fatal("COMPLETION", "Firmware ended without expected architectural execution/traps") -`ifdef IRQ_PROFILE - if (raised != 3'b111 || target_retired != 3'b111 || acknowledgements != 3 || - control.timer_irq || control.software_irq || control.external_irq) - `uvm_fatal("IRQ_COMPLETION", - "IRQ gate finished without all held levels, retirements and acknowledgements") - foreach (target_access_count[i]) - if (target_access_count[i] != 1 || target_retire_count[i] != 1) - `uvm_fatal("IRQ_ONCE", $sformatf( - "Target %0d executed %0d cache demands and retired %0d times", - i, - target_access_count[i], - target_retire_count[i] - )) - `uvm_info("IRQ_CHECKED", - "All three deferred targets executed and retired exactly once; three held-level ACKs drained", - UVM_LOW) -`endif - do - @(control.cb); - while (memory_queue.size() != 0 || mmio_queue.size() != 0 || memory_active || mmio_active); - `uvm_info( - "CORE", - $sformatf( - "Firmware checked %0d stages, exit=0; cycles=%0d memory reads=%0d writes=%0d S1/S2 cancellations=%0d precise traps=%0d retired=%0d", - STAGES, cycles, reads, writes, cancellations, traps, retired), UVM_LOW) - control.cb.reset <= 1; - repeat (3) @(control.cb); - endtask - function void final_phase(uvm_phase phase); - core_ref_destroy(model); - super.final_phase(phase); - endfunction -endclass diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.S b/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.S deleted file mode 100644 index f2a7e20a8..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.S +++ /dev/null @@ -1,148 +0,0 @@ -.option norelax -.section .text.start,"ax",@progbits -.globl _start -_start: - la t0,trap_handler - csrw mtvec,t0 - li s0,0x10000000 - la s1,data - li s9,-1 - li t0,0x6000 - csrs mstatus,t0 - csrw fcsr,zero - j tests -.section .text.trap,"ax",@progbits -.option push -.option norvc -trap_handler: - csrr t0,mcause - bne t0,s9,fail - csrr t0,mtval - li t1,0x53 - bne t0,t1,fail - csrr t0,mepc - addi t0,t0,4 - csrw mepc,t0 - li s9,-1 - mret -.option pop -.section .text.tests,"ax",@progbits -tests: - li t0,1 - sd t0,8(s0) - flw ft0,0(s1) - fmv.x.w t0,ft0 - li t1,0x3fc00000 - bne t0,t1,fail - fld ft4,8(s1) - fmv.x.d t0,ft4 - li t1,0x3ff8000000000000 - bne t0,t1,fail - flw ft1,4(s1) - fld ft5,16(s1) - fsw ft0,32(s1) - fsd ft4,40(s1) - lw t0,32(s1) - li t1,0x3fc00000 - bne t0,t1,fail - ld t0,40(s1) - li t1,0x3ff8000000000000 - bne t0,t1,fail - li t0,2 - sd t0,8(s0) - fadd.s ft2,ft0,ft1 - fmul.s ft2,ft2,ft0 - fsw ft2,36(s1) - lw t0,36(s1) - li t1,0x40b40000 - bne t0,t1,fail - li s2,16 -fp_dependencies: - fmul.d ft6,ft4,ft5 - fadd.d ft6,ft6,ft4 - fsub.d ft6,ft6,ft5 - fdiv.d ft6,ft6,ft4 - fsd ft6,48(s1) - fld ft7,48(s1) - fmv.x.d t0,ft7 - li t1,0x3ffc000000000000 - bne t0,t1,fail - addi s2,s2,-1 - bnez s2,fp_dependencies - li t0,3 - sd t0,8(s0) - fsflags zero - li t0,0x3f800000 - fmv.w.x ft0,t0 - li t0,0x40400000 - fmv.w.x ft1,t0 - fdiv.s ft2,ft0,ft1 - fmv.x.w t0,ft2 - li t1,0x3eaaaaab - bne t0,t1,fail - frflags t0 - li t1,1 - bne t0,t1,fail - fsflags zero - li t0,0x3ff0000000000000 - fmv.d.x ft4,t0 - fmv.d.x ft5,zero - fdiv.d ft6,ft4,ft5 - fmv.x.d t0,ft6 - li t1,0x7ff0000000000000 - bne t0,t1,fail - frflags t0 - li t1,8 - bne t0,t1,fail - fsflags zero - fmv.w.x ft0,zero - fdiv.s ft1,ft0,ft0 - fmv.x.w t0,ft1 - li t1,0x7fc00000 - bne t0,t1,fail - frflags t0 - li t1,16 - bne t0,t1,fail - fsflags zero - li t0,1 - fsrm t0 - li t0,0x3fc00000 - fmv.w.x ft0,t0 - fcvt.w.s t0,ft0,dyn - li t1,1 - bne t0,t1,fail - frflags t0 - li t1,1 - bne t0,t1,fail - fsrm zero - li t0,4 - sd t0,8(s0) - li s9,2 - li t0,0x6000 - csrc mstatus,t0 -.globl fpu_illegal -.option push -.option norvc -fpu_illegal: - .word 0x00000053 -.option pop - li t0,0x6000 - csrs mstatus,t0 - li t0,0x3f800000 - fmv.w.x ft0,t0 - fadd.s ft1,ft0,ft0 - fmv.x.w t0,ft1 - li t1,0x40000000 - bne t0,t1,fail - sd zero,0(s0) -1:j 1b -fail: - li t0,1 - sd t0,0(s0) -1:j 1b -.section .data,"aw",@progbits -.balign 64 -data: - .word 0x3fc00000,0x40100000 - .dword 0x3ff8000000000000,0x4002000000000000 - .zero 40 diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.f deleted file mode 100644 index beb8e9085..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+FPU_PROFILE -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/core_interface.sv -@RESOURCES@/core_package.sv --F @RTL@/Fpu/filelist.f -@RESOURCES@/core_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.ld b/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.ld deleted file mode 100644 index 7b1347647..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/fpu.ld +++ /dev/null @@ -1,14 +0,0 @@ -OUTPUT_ARCH(riscv) -ENTRY(_start) -SECTIONS -{ - . = 0x80000000; - .text.start : { *(.text.start) } - . = 0x80000200; - .text.trap : { *(.text.trap) } - . = 0x80000400; - .text.tests : { *(.text.tests) } - . = 0x800040c0; - .data : { *(.data) } - /DISCARD/ : { *(.riscv.attributes) *(.comment) } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.S b/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.S deleted file mode 100644 index 6a79ac32a..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.S +++ /dev/null @@ -1,97 +0,0 @@ -.option norelax -.option norvc -.section .text.start,"ax",@progbits -.globl _start -_start: - li s0, 0x10000000 - la s1, data - li s2, 0 - li s3, 0 - la t0, trap_handler - csrw mtvec, t0 - li s7, 7 - li t0, 0x80 - csrw mie, t0 - li t0, 8 - csrs mstatus, t0 - j tests - -.section .text.trap,"ax",@progbits -trap_handler: - // t4/t5/t6 are reserved to this handler, so interrupted operands survive. - csrr t4, mcause - li t5, 0x8000000000000000 - or t5, t5, s7 - bne t4, t5, fail - csrr t4, mtval - bnez t4, fail - li t5, 1 - sll t5, t5, s7 - csrr t6, mip - and t6, t6, t5 - beqz t6, fail - ld t4, 64(s1) - addi t4, t4, 1 - sd t4, 64(s1) - sd t5, 16(s0) - addi s3, s3, 1 - mret - -.section .text.tests,"ax",@progbits -tests: -irq_load: - ld s8, 0(s1) - li t0, 0x8877665544332211 - bne s8, t0, fail -timer_wait: - li t0, 1 - bne s3, t0, timer_wait - li s2, 1 - sd s2, 8(s0) - li s7, 3 - li t0, 8 - csrw mie, t0 - addi t2, s1, 8 - li t0, 1 -irq_amo: - amoadd.d.aqrl s8, t0, (t2) - bnez s8, fail - ld t1, 8(s1) - bne t1, t0, fail -software_wait: - li t0, 2 - bne s3, t0, software_wait - li s2, 2 - sd s2, 8(s0) - li s7, 11 - li t0, 0x800 - csrw mie, t0 - li t0, 0x55 -irq_store: - sd t0, 16(s1) -external_wait: - li t0, 3 - bne s3, t0, external_wait - ld t0, 16(s1) - li t1, 0x55 - bne t0, t1, fail - ld t0, 64(s1) - li t1, 3 - bne t0, t1, fail - ld t0, 8(s1) - li t1, 1 - bne t0, t1, fail - li s2, 3 - sd s2, 8(s0) - sd zero, 0(s0) -1: j 1b -fail: - ori t0, s2, 0x100 - sd t0, 0(s0) -2: j 2b - -.section .data,"aw",@progbits -data: - .dword 0x8877665544332211 - .dword 0, 0, 0, 0, 0, 0, 0 - .dword 0 diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.cm_hier b/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.cm_hier deleted file mode 100644 index 0f4845487..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.cm_hier +++ /dev/null @@ -1,5 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.mem_req -+tree @TOP@.mem_resp -+tree @TOP@.uncached_req -+tree @TOP@.uncached_resp diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.f deleted file mode 100644 index b9b3967a8..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+IRQ_PROFILE -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/core_interface.sv -@RESOURCES@/core_package.sv --F @RTL@/Irq/filelist.f -@RESOURCES@/core_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.ld b/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.ld deleted file mode 100644 index 7b1347647..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/irq.ld +++ /dev/null @@ -1,14 +0,0 @@ -OUTPUT_ARCH(riscv) -ENTRY(_start) -SECTIONS -{ - . = 0x80000000; - .text.start : { *(.text.start) } - . = 0x80000200; - .text.trap : { *(.text.trap) } - . = 0x80000400; - .text.tests : { *(.text.tests) } - . = 0x800040c0; - .data : { *(.data) } - /DISCARD/ : { *(.riscv.attributes) *(.comment) } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu.f deleted file mode 100644 index ccd73eebf..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/lsu_interface.sv -@RESOURCES@/lsu_package.sv --F @RTL@/Lsu/filelist.f -@RESOURCES@/lsu_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_interface.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_interface.sv deleted file mode 100644 index 11210ed35..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_interface.sv +++ /dev/null @@ -1,14 +0,0 @@ -interface lsu_control_if ( - input logic clock -); - logic reset, s1_kill, s2_kill, cancel_offer; - logic [45:0] pc; - logic [63:0] s1_data; - logic nack, ordered, idle, cancelled; - logic ma_ld, ma_st, pf_ld, pf_st, ae_ld, ae_st; - clocking cb @(posedge clock); - default input #1step output #0; - output reset, s1_kill, s2_kill, pc, s1_data, cancel_offer; - input nack, ordered, idle, cancelled, ma_ld, ma_st, pf_ld, pf_st, ae_ld, ae_st; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_package.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_package.sv deleted file mode 100644 index 460760b35..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_package.sv +++ /dev/null @@ -1,8 +0,0 @@ -`include "lsu_config.svh" -package lsu_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `define LF(kind, field) `CORE_``kind``_``field``_OFFSET +: `CORE_``kind``_``field``_WIDTH - `include "lsu_test.svh" -endpackage diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_tb.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_tb.sv deleted file mode 100644 index c7afb3d4c..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_tb.sv +++ /dev/null @@ -1,59 +0,0 @@ -`include "lsu_config.svh" -module lsu_tb; - import uvm_pkg::*; - import lsu_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - lsu_control_if control (clock); - // EX structural replay may withdraw an unaccepted HellaCache offer; backend streams remain irrevocable. - stream_if #(`CORE_CPU_WIDTH) source_if ( - clock, - control.reset || control.cancel_offer - ); - stream_if #(`CORE_RETURN_WIDTH) sink_if ( - clock, - control.reset - ); - stream_if #(`CORE_VIRTUAL_WIDTH) virtual_req ( - clock, - control.reset - ); - stream_if #(`CORE_RESULT_WIDTH) virtual_resp ( - clock, - control.reset - ); - assign sink_if.ready = 1; - Lsu dut ( - .clock(clock), - .reset(control.reset), - .io_pc(control.pc), - .io_cpu_s1_kill(control.s1_kill), - .io_cpu_s2_kill(control.s2_kill), - .io_cpu_s1_data_data(control.s1_data), - .io_cpu_s1_data_mask(8'hff), - .io_cpu_keep_clock_enabled(1'b1), - .io_cpu_s2_nack(control.nack), - .io_cpu_ordered(control.ordered), - .io_idle(control.idle), - .io_cancelled(control.cancelled), - .io_cpu_s2_xcpt_ma_ld(control.ma_ld), - .io_cpu_s2_xcpt_ma_st(control.ma_st), - .io_cpu_s2_xcpt_pf_ld(control.pf_ld), - .io_cpu_s2_xcpt_pf_st(control.pf_st), - .io_cpu_s2_xcpt_ae_ld(control.ae_ld), - .io_cpu_s2_xcpt_ae_st(control.ae_st), - `include "lsu_ports.svh" - ); - initial begin - uvm_config_db#(virtual lsu_control_if)::set(null, "uvm_test_top*", "control", control); - uvm_config_db#(virtual stream_if #(`CORE_CPU_WIDTH))::set(null, "uvm_test_top*", "cpu", - source_if); - uvm_config_db#(virtual stream_if #(`CORE_RETURN_WIDTH))::set(null, "uvm_test_top*", "result", - sink_if); - uvm_config_db#(virtual stream_if #(`CORE_VIRTUAL_WIDTH))::set(null, "uvm_test_top*", - "virtual_req", virtual_req); - uvm_config_db#(virtual stream_if #(`CORE_RESULT_WIDTH))::set(null, "uvm_test_top*", - "virtual_resp", virtual_resp); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_test.svh b/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_test.svh deleted file mode 100644 index ff30cab48..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/lsu_test.svh +++ /dev/null @@ -1,193 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual lsu_control_if control; - virtual stream_if #(`CORE_CPU_WIDTH) cpu; - virtual stream_if #(`CORE_RETURN_WIDTH) result; - virtual stream_if #(`CORE_VIRTUAL_WIDTH) virtual_req; - virtual stream_if #(`CORE_RESULT_WIDTH) virtual_resp; - int issued = 0, checked = 0, killed = 0, rejected = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 50us; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual lsu_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`CORE_CPU_WIDTH))::get( - this, "", "cpu", cpu - ) || !uvm_config_db#(virtual stream_if #(`CORE_RETURN_WIDTH))::get( - this, "", "result", result - ) || !uvm_config_db#(virtual stream_if #(`CORE_VIRTUAL_WIDTH))::get( - this, "", "virtual_req", virtual_req - ) || !uvm_config_db#(virtual stream_if #(`CORE_RESULT_WIDTH))::get( - this, "", "virtual_resp", virtual_resp - )) - `uvm_fatal("VIF", "LSU interfaces missing") - endfunction - task start(int cmd = 1, longint unsigned address = 64'h800040c0); - bit [`CORE_CPU_WIDTH-1:0] packet = '0; - packet[`LF(CPU, ADDR)] = address; - packet[`LF(CPU, TAG)] = 2; - packet[`LF(CPU, CMD)] = cmd; - packet[`LF(CPU, SIZE)] = 3; - packet[`LF(CPU, DPRV)] = 3; - packet[`LF(CPU, SIGNED)] = 1; - packet[`LF(CPU, NO_RESP)] = cmd == 1; - packet[`LF(CPU, DATA)] = 64'hdeadbeef; // EX req.data is not the later S1 operand. - control.cb.pc <= 46'h80000100; - control.cb.s1_kill <= 0; - control.cb.s2_kill <= 0; - control.cb.s1_data <= 64'h0123456789abcdef; - cpu.bits <= packet; - cpu.valid <= 1; - do @(cpu.sample); while (!cpu.sample.ready); - cpu.valid <= 0; - endtask - task no_backend(); - repeat (3) begin - @(control.cb); - if (virtual_req.valid || result.valid) - `uvm_fatal("KILL", "Killed instruction had a memory/CPU side effect") - end - killed++; - endtask - task execute_operation(int cmd = 1, bit fault = 0); - start(cmd); - @(control.cb); - @(control.cb); - if (!control.cb.nack) `uvm_fatal("S2", "Original operation did not nack at its precise S2") - do @(virtual_req.sample); while (!virtual_req.sample.valid); - if (virtual_req.sample.bits[ - `LF(VIRTUAL, TAG) - ] != 2 || virtual_req.sample.bits[ - `LF(VIRTUAL, VADDR) - ] != 64'h800040c0 || virtual_req.sample.bits[ - `LF(VIRTUAL, DATA) - ] != 64'h0123456789abcdef || virtual_req.sample.bits[ - `LF(VIRTUAL, WRITE) - ] != (cmd == 1) || virtual_req.sample.bits[ - `LF(VIRTUAL, EXECUTE) - ] != 0) - `uvm_fatal("OPERAND", "S1 operand/tag/address incorrectly captured") - repeat (3) @(control.cb); - virtual_req.ready <= 1; - @(virtual_req.sample); - virtual_req.ready <= 0; - issued++; - repeat (4) @(control.cb); - virtual_resp.bits <= '0; - virtual_resp.bits[`LF(RESULT, TAG)] <= 2; - virtual_resp.bits[`LF(RESULT, DATA)] <= cmd == 0 ? 64'h89abcdef01234567 : 0; - virtual_resp.bits[`LF(RESULT, ACCESSFAULT)] <= fault; - virtual_resp.valid <= 1; - do @(virtual_resp.sample); while (!virtual_resp.sample.ready); - virtual_resp.valid <= 0; - @(control.cb); - if (!control.cb.ordered || result.valid) - `uvm_fatal("ORDER", - "Buffered completed operation still blocked ordered decode or returned early") - endtask - task reject_identity(bit wrong_pc); - bit [`CORE_CPU_WIDTH-1:0] packet = '0; - packet[`LF(CPU, ADDR)] = 64'h800040c0; - packet[`LF(CPU, TAG)] = wrong_pc ? 2 : 4; - packet[`LF(CPU, CMD)] = 1; - packet[`LF(CPU, SIZE)] = 3; - packet[`LF(CPU, DPRV)] = 3; - control.cb.pc <= wrong_pc ? 46'h80000104 : 46'h80000100; - cpu.bits <= packet; - cpu.valid <= 1; - repeat (3) begin - @(cpu.sample); - if (cpu.sample.ready || result.valid || virtual_req.valid) - `uvm_fatal("IDENTITY", "A different tag/PC consumed the buffered completion") - end - control.cb.cancel_offer <= 1; - cpu.valid <= 0; - @(control.cb); - control.cb.cancel_offer <= 0; - rejected++; - endtask - task replay(int cmd = 1, bit fault = 0, bit misaligned = 0, - longint unsigned address = 64'h800040c0); - start(cmd, address); - @(control.cb); - do @(result.sample); while (!result.sample.valid); - if (result.sample.bits[ - `LF(RETURN, TAG) - ] != 2 || result.sample.bits[ - `LF(RETURN, CMD) - ] != cmd || result.sample.bits[ - `LF(RETURN, ADDR) - ] != address || result.sample.bits[ - `LF(RETURN, HAS_DATA) - ] != (cmd == 0 && !fault && !misaligned) || control.cb.ae_ld != (cmd == 0 && fault) || - control.cb.ae_st != (cmd == 1 && fault) || control.cb.ma_ld != (cmd == 0 && misaligned) - || control.cb.ma_st != (cmd == 1 && misaligned) || control.cb.nack || virtual_req.valid) - `uvm_fatal("REPLAY", "Replay return/exception/tag was not aligned with its own S2") - if (cmd == 0 && !fault && !misaligned && result.sample.bits[ - `LF(RETURN, DATA) - ] != 64'h89abcdef01234567) - `uvm_fatal("DATA", "Replay data changed") - checked++; - @(control.cb); - endtask - task execute(); - control.reset = 1; - control.cancel_offer = 0; - control.s1_kill = 0; - control.s2_kill = 0; - control.pc = 0; - control.s1_data = 0; - cpu.valid = 0; - cpu.bits = '0; - virtual_req.ready = 0; - virtual_resp.valid = 0; - virtual_resp.bits = '0; - repeat (4) @(control.cb); - control.cb.reset <= 0; - @(control.cb); - start(); - control.cb.s1_kill <= 1; - no_backend(); - control.cb.s1_kill <= 0; - start(); - @(control.cb); - control.cb.s2_kill <= 1; - no_backend(); - control.cb.s2_kill <= 0; - execute_operation(); - reject_identity(0); - reject_identity(1); - start(); - control.cb.s1_kill <= 1; - no_backend(); - control.cb.s1_kill <= 0; - start(); - @(control.cb); - control.cb.s2_kill <= 1; - no_backend(); - control.cb.s2_kill <= 0; - replay(); - execute_operation(0); - replay(0); - execute_operation(0, 1); - replay(0, 1); - execute_operation(1, 1); - replay(1, 1); - start(0, 64'h800040c1); - @(control.cb); - @(control.cb); - repeat (3) @(control.cb); - if (virtual_req.valid) `uvm_fatal("MISALIGN", "Misaligned request reached backend") - replay(0, 0, 1, 64'h800040c1); - if (issued != 4 || checked != 5 || killed != 4 || rejected != 2) - `uvm_fatal("COUNTS", "LSU contract cases incomplete") - `uvm_info( - "LSU", - $sformatf( - "Checked %0d precise replay completions, %0d real backend executions, %0d original/replay S1/S2 kills, %0d rejected tag/PC offers", - checked, issued, killed, rejected), UVM_LOW) - endtask -endclass diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance.cm_hier b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance.cm_hier deleted file mode 100644 index 6bd7e07ab..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance.cm_hier +++ /dev/null @@ -1,5 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.source_if -+tree @TOP@.sink_if -+tree @TOP@.chi_req -+tree @TOP@.chi_rsp diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance.f deleted file mode 100644 index 716a51f6a..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/maintenance_interface.sv -@RESOURCES@/maintenance_package.sv --F @RTL@/Maintenance/filelist.f -@RESOURCES@/maintenance_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.cm_hier b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.cm_hier deleted file mode 100644 index 6bd7e07ab..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.cm_hier +++ /dev/null @@ -1,5 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.source_if -+tree @TOP@.sink_if -+tree @TOP@.chi_req -+tree @TOP@.chi_rsp diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.f deleted file mode 100644 index 49e36bc22..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.f +++ /dev/null @@ -1,11 +0,0 @@ -+define+MAINTENANCE_TB=maintenance_bad_response_tb -+define+MAINTENANCE_BAD_RESPONSE -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/maintenance_interface.sv -@RESOURCES@/maintenance_package.sv --F @RTL@/Maintenance/filelist.f -@RESOURCES@/maintenance_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.toml b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.toml deleted file mode 100644 index 33bb516f3..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_bad_response.toml +++ /dev/null @@ -1 +0,0 @@ -expected_assertion = "Maintenance response has no issued request" diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_interface.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_interface.sv deleted file mode 100644 index 7a062bbb3..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_interface.sv +++ /dev/null @@ -1,10 +0,0 @@ -interface maintenance_control_if ( - input logic clock -); - logic reset, drained, idle; - clocking cb @(posedge clock); - default input #1step output #0; - output reset, drained; - input idle; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_package.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_package.sv deleted file mode 100644 index 0374a573e..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_package.sv +++ /dev/null @@ -1,166 +0,0 @@ -`include "maintenance_config.svh" -package maintenance_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - `define MF(kind, field) `CORE_``kind``_``field``_OFFSET +: `CORE_``kind``_``field``_WIDTH - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual maintenance_control_if control; - virtual stream_if #(`CORE_RANGE_WIDTH) request; - virtual stream_if #(`CORE_ACK_WIDTH) response; - virtual stream_if #(`CORE_REQ_WIDTH) req; - virtual stream_if #(`CORE_RSP_WIDTH) rsp; - int checked_lines = 0, checked_ranges = 0; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 50us; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual maintenance_control_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`CORE_RANGE_WIDTH))::get( - this, "", "request", request - ) || !uvm_config_db#(virtual stream_if #(`CORE_ACK_WIDTH))::get( - this, "", "response", response - ) || !uvm_config_db#(virtual stream_if #(`CORE_REQ_WIDTH))::get( - this, "", "req", req - ) || !uvm_config_db#(virtual stream_if #(`CORE_RSP_WIDTH))::get( - this, "", "rsp", rsp - )) - `uvm_fatal("VIF", "Maintenance interfaces missing") - endfunction - task reset_all(); - control.cb.reset <= 1; - control.cb.drained <= 0; - request.valid <= 0; - request.bits <= '0; - response.ready <= 0; - req.ready <= 0; - rsp.valid <= 0; - rsp.bits <= '0; - repeat (3) @(control.cb); - control.cb.reset <= 0; - repeat (2) @(control.cb); - if (!control.cb.idle || req.valid || response.valid) - `uvm_fatal("RESET", "Joint reset retained work") - endtask - task start_range(int tag, int op, longint unsigned first, int lines); - bit [`CORE_RANGE_WIDTH-1:0] bits = '0; - bits[`MF(RANGE, TAG)] = tag; - bits[`MF(RANGE, OP)] = op; - bits[`MF(RANGE, FIRSTLINE)] = first; - bits[`MF(RANGE, LASTLINE)] = first + (lines - 1) * 64; - request.bits <= bits; - request.valid <= 1; - do @(request.sample); while (!request.sample.ready); - request.valid <= 0; - endtask - task line_completion(longint unsigned address, int opcode, bit error = 0); - bit [`CORE_REQ_WIDTH-1:0] expected = '0; - bit [`CORE_RSP_WIDTH-1:0] reply = '0; - int home = `MAINTENANCE_HOME_BASE + ((address >> 6) % `MAINTENANCE_HOME_COUNT); - expected[`MF(REQ, SRCID)] = `MAINTENANCE_NODE; - expected[`MF(REQ, TGTID)] = home; - expected[`MF(REQ, TXNID)] = `MAINTENANCE_TXN; - expected[`MF(REQ, ADDR)] = address; - expected[`MF(REQ, SIZE)] = 6; - expected[`MF(REQ, OPCODE)] = opcode; - expected[`MF(REQ, SNPATTR)] = 1; - expected[`MF(REQ, MEMATTR)] = 12; - expected[`MF(REQ, ALLOWRETRY)] = 1; - do @(req.sample); while (!req.sample.valid); - if (req.sample.bits !== expected) - `uvm_fatal("REQ", $sformatf( - "CMO mismatch at %h: %h != %h", address, req.sample.bits, expected)) - repeat (3) begin - @(req.sample); - if (!req.sample.valid || req.sample.bits !== expected || response.valid) - `uvm_fatal("STALL", "CMO changed or completed while request was stalled") - end - req.ready <= 1; - @(req.sample); - req.ready <= 0; - repeat (5) begin - @(rsp.sample); - if (req.valid || response.valid) - `uvm_fatal("EARLY", "Range advanced before actual Home completion") - end - reply[`MF(RSP, SRCID)] = home; - reply[`MF(RSP, TGTID)] = `MAINTENANCE_NODE; - reply[`MF(RSP, TXNID)] = `MAINTENANCE_TXN; - reply[`MF(RSP, OPCODE)] = 4; - reply[`MF(RSP, RESPERR)] = error ? 3 : 0; - rsp.bits <= reply; - rsp.valid <= 1; - do @(rsp.sample); while (!rsp.sample.ready); - rsp.valid <= 0; - checked_lines++; - endtask - task finish_range(int tag, bit ok); - bit [`CORE_ACK_WIDTH-1:0] expected = '0; - expected[`MF(ACK, TAG)] = tag; - expected[`MF(ACK, OK)] = ok; - do @(response.sample); while (!response.sample.valid); - repeat (4) begin - if (response.sample.bits !== expected || req.valid || request.ready) - `uvm_fatal("ACK", "Bad completion or accepted new range before retirement") - @(response.sample); - end - response.ready <= 1; - @(response.sample); - response.ready <= 0; - @(control.cb); - if (!control.cb.idle) `uvm_fatal("IDLE", "Completed range did not retire") - checked_ranges++; - endtask - task execute(); - reset_all(); -`ifdef MAINTENANCE_BAD_RESPONSE - rsp.valid <= 1; - repeat (3) @(control.cb); - `uvm_fatal("MISSING_ASSERT", "Unissued Home response was not rejected") -`else - for (int op = 0; op < 3; op++) begin - int lines = 3 - op; - control.cb.drained <= 0; - start_range(16 + op, op, 64'h80000000, lines); - repeat (4) begin - @(control.cb); - if (req.valid || response.valid) `uvm_fatal("DRAIN", "CMO overtook older CPU accesses") - end - control.cb.drained <= 1; - for (int i = 0; i < lines; i++) line_completion(64'h80000000 + i * 64, 8 + op); - finish_range(16 + op, 1); - end - start_range(255, 1, 64'h80000100, 4); - line_completion(64'h80000100, 9); - line_completion(64'h80000140, 9, 1); - finish_range(255, 0); - start_range(0, 0, 64'hfffffffffc0, 1); - line_completion(64'hfffffffffc0, 8); - finish_range(0, 1); - control.cb.drained <= 0; - start_range(42, 0, 64'h80000000, 1); - reset_all(); - control.cb.drained <= 1; - start_range(43, 2, 64'h80000000, 1); - do @(req.sample); while (!req.sample.valid); - req.ready <= 1; - @(req.sample); - req.ready <= 0; - reset_all(); - control.cb.drained <= 1; - start_range(44, 2, 64'h80000040, 1); - line_completion(64'h80000040, 10); - finish_range(44, 1); - if (checked_lines != 10 || checked_ranges != 6) - `uvm_fatal("COUNTS", "Maintenance scenarios incomplete") - `uvm_info("MAINTENANCE", - "Checked 10 CMO lines, 6 ranges, delayed Home completion, errors and joint reset", - UVM_LOW) -`endif - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_tb.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_tb.sv deleted file mode 100644 index 073ec3e26..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/maintenance_tb.sv +++ /dev/null @@ -1,46 +0,0 @@ -`include "maintenance_config.svh" -`ifndef MAINTENANCE_TB -`define MAINTENANCE_TB maintenance_tb -`endif -module `MAINTENANCE_TB; - import uvm_pkg::*; - import maintenance_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - maintenance_control_if control (clock); - stream_if #(`CORE_RANGE_WIDTH) source_if ( - clock, - control.reset - ); - stream_if #(`CORE_ACK_WIDTH) sink_if ( - clock, - control.reset - ); - stream_if #(`CORE_REQ_WIDTH) chi_req ( - clock, - control.reset - ); - stream_if #(`CORE_RSP_WIDTH) chi_rsp ( - clock, - control.reset - ); - Maintenance dut ( - .clock(clock), - .reset(control.reset), - .io_drained(control.drained), - .io_idle(control.idle), - `include "maintenance_ports.svh" - ); - initial begin - uvm_config_db#(virtual maintenance_control_if)::set(null, "uvm_test_top*", "control", control); - uvm_config_db#(virtual stream_if #(`CORE_RANGE_WIDTH))::set(null, "uvm_test_top*", "request", - source_if); - uvm_config_db#(virtual stream_if #(`CORE_ACK_WIDTH))::set(null, "uvm_test_top*", "response", - sink_if); - uvm_config_db#(virtual stream_if #(`CORE_REQ_WIDTH))::set(null, "uvm_test_top*", "req", - chi_req); - uvm_config_db#(virtual stream_if #(`CORE_RSP_WIDTH))::set(null, "uvm_test_top*", "rsp", - chi_rsp); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/multicore.S b/arch/src/main/scala/framework/root/root-core/src/main/resources/multicore.S deleted file mode 100644 index 17aa3bb5a..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/multicore.S +++ /dev/null @@ -1,108 +0,0 @@ -.option norelax -.option norvc -.macro check_payload - ld t0, 0(s1) - li t1, 0x8877665544332211 - bne t0, t1, fail - ld t0, 32(s1) - li t1, 0x0102030405060708 - bne t0, t1, fail - ld t0, 40(s1) - li t1, 0x1122334455667788 - bne t0, t1, fail - ld t0, 48(s1) - li t1, 0x8000000000000001 - bne t0, t1, fail - ld t0, 56(s1) - li t1, 0xffeeddccbbaa9988 - bne t0, t1, fail -.endm -.section .text.start,"ax",@progbits -.globl _start -_start: - li s0, 0x10000000 - la s1, shared - li s2, 0 - la t0, trap_handler - csrw mtvec, t0 - csrw mie, zero - csrr s5, mhartid - beqz s5, producer - li t0, 1 - beq s5, t0, consumer - j fail - -.section .text.trap,"ax",@progbits -trap_handler: - j fail - -.section .text.tests,"ax",@progbits -producer: - li t0, 0x8877665544332211 - sd t0, 0(s1) - sd zero, 24(s1) - li t0, 0x0102030405060708 - sd t0, 32(s1) - li t0, 0x1122334455667788 - sd t0, 40(s1) - li t0, 0x8000000000000001 - sd t0, 48(s1) - li t0, 0xffeeddccbbaa9988 - sd t0, 56(s1) - fence rw, rw - li t0, 1 - sd t0, 8(s1) - li s2, 1 - sd s2, 8(s0) - li s3, 4096 -producer_wait: - ld t0, 16(s1) - bnez t0, producer_check - addi s3, s3, -1 - bnez s3, producer_wait - j fail -producer_check: - li t1, 1 - bne t0, t1, fail - fence r, rw - ld t0, 24(s1) - bne t0, t1, fail - check_payload - j finish -consumer: - li s3, 4096 -consumer_wait: - ld t0, 8(s1) - bnez t0, consumer_check - addi s3, s3, -1 - bnez s3, consumer_wait - j fail -consumer_check: - li t1, 1 - bne t0, t1, fail - fence r, rw - check_payload - li s2, 1 - sd s2, 8(s0) - addi t2, s1, 24 - li t0, 1 - amoadd.d.aqrl t1, t0, (t2) - bnez t1, fail - ld t1, 24(s1) - bne t1, t0, fail - fence rw, w - sd t0, 16(s1) -finish: - li s2, 2 - sd s2, 8(s0) - sd zero, 0(s0) -1: j 1b -fail: - ori t0, s2, 0x100 - sd t0, 0(s0) -2: j 2b - -.section .data,"aw",@progbits -.balign 64 -shared: - .dword 0, 0, 0, 0, 0, 0, 0, 0 diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/multicore.ld b/arch/src/main/scala/framework/root/root-core/src/main/resources/multicore.ld deleted file mode 100644 index 7b1347647..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/multicore.ld +++ /dev/null @@ -1,14 +0,0 @@ -OUTPUT_ARCH(riscv) -ENTRY(_start) -SECTIONS -{ - . = 0x80000000; - .text.start : { *(.text.start) } - . = 0x80000200; - .text.trap : { *(.text.trap) } - . = 0x80000400; - .text.tests : { *(.text.tests) } - . = 0x800040c0; - .data : { *(.data) } - /DISCARD/ : { *(.riscv.attributes) *(.comment) } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/permission.cm_hier b/arch/src/main/scala/framework/root/root-core/src/main/resources/permission.cm_hier deleted file mode 100644 index 772e1cfc5..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/permission.cm_hier +++ /dev/null @@ -1 +0,0 @@ -+tree @TOP@.dut diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/permission.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/permission.f deleted file mode 100644 index 8081d7f26..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/permission.f +++ /dev/null @@ -1,5 +0,0 @@ -+incdir+@RESOURCES@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv --F @RTL@/PermissionVerification/filelist.f -@RESOURCES@/permission_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/permission_tb.sv b/arch/src/main/scala/framework/root/root-core/src/main/resources/permission_tb.sv deleted file mode 100644 index f77e4a83c..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/permission_tb.sv +++ /dev/null @@ -1,216 +0,0 @@ -interface permission_control_if ( - input logic clock -); - logic reset, context_valid, request_valid, request_ready, response_valid, response_ready; - logic write, is_pte, allow; - logic [1:0] privilege; - logic [7:0] id, context_id, response_id; - logic [12:0] bytes; - logic [43:0] address; - logic [31:0] pmp_config; - logic [63:0] pmp_addr[4]; - clocking cb @(posedge clock); - default input #1step output #0; - output reset, context_valid, request_valid, response_ready, write, is_pte; - output privilege, id, context_id, bytes, address, pmp_config, pmp_addr; - input request_ready, response_valid, response_id, allow; - endclocking -endinterface - -package permission_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual permission_control_if control; - int checked = 0; - localparam longint unsigned BASE = 'h80000000; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 100us; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual permission_control_if)::get(this, "", "control", control)) - `uvm_fatal("VIF", "Permission interface missing") - endfunction - task initialize(); - control.cb.reset <= 1; - control.cb.context_valid <= 0; - control.cb.request_valid <= 0; - control.cb.response_ready <= 0; - control.cb.pmp_config <= 0; - control.cb.context_id <= 0; - control.cb.id <= 0; - control.cb.address <= 0; - control.cb.bytes <= 0; - control.cb.write <= 0; - control.cb.is_pte <= 0; - control.cb.privilege <= 0; - foreach (control.pmp_addr[i]) control.cb.pmp_addr[i] <= 0; - repeat (3) @(control.cb); - control.cb.reset <= 0; - repeat (2) @(control.cb); - endtask - task check_range(longint unsigned address, int bytes, bit write, int privilege, bit expected, - bit is_pte = 0, bit context_valid = 1, bit wrong_tag = 0, bit mutate = 0, - int hold = 0); - int tag = checked + 1; - control.cb.id <= tag; - control.cb.context_id <= wrong_tag ? tag + 1 : tag; - control.cb.context_valid <= context_valid; - control.cb.address <= address; - control.cb.bytes <= bytes; - control.cb.write <= write; - control.cb.is_pte <= is_pte; - control.cb.privilege <= privilege; - control.cb.request_valid <= 1; - do @(control.cb); while (!control.cb.request_ready); - control.cb.request_valid <= 0; - if (mutate) begin - control.cb.pmp_config <= 0; - foreach (control.pmp_addr[i]) control.cb.pmp_addr[i] <= 0; - control.cb.context_valid <= 0; - control.cb.context_id <= tag + 1; - end - do @(control.cb); while (!control.cb.response_valid); - for (int cycle = 0; cycle <= hold; ++cycle) begin - if (!control.cb.response_valid || control.cb.response_id !== tag || - control.cb.allow !== expected || control.cb.request_ready) - `uvm_fatal("PERMISSION", $sformatf( - "case%0d PA=%h bytes=%0d write=%0d prv=%0d expected=%b got=%b", - tag, - address, - bytes, - write, - privilege, - expected, - control.cb.allow - )) - if (cycle < hold) @(control.cb); - end - control.cb.response_ready <= 1; - @(control.cb); - control.cb.response_ready <= 0; - @(control.cb); - ++checked; - endtask - task execute(); - initialize(); - check_range(BASE, 64, 0, 3, 1); - check_range(BASE, 64, 0, 1, 0); - check_range('h90000000, 64, 0, 3, 1); - check_range('h90000000, 64, 1, 3, 0); - check_range(BASE + 'h2000000, 64, 0, 3, 0); - check_range(BASE, 0, 0, 3, 0); - check_range(BASE, 4097, 0, 3, 0); - check_range(BASE, 64, 0, 2, 0); - check_range(BASE, 64, 0, 3, 0, 0, 0); - check_range(BASE, 64, 0, 3, 0, 0, 1, 1); - check_range('h1000000000, 64, 0, 3, 0); - - control.cb.pmp_config <= 'h1b; // NAPOT R/W, 8 KiB at BASE. - control.cb.pmp_addr[0] <= BASE + 'hffc; - check_range(BASE, 4096, 0, 1, 1); - check_range(BASE + 1, 127, 0, 1, 1); - check_range(BASE + 8192 - 64, 64, 1, 1, 1); - check_range(BASE + 8192, 64, 0, 1, 0); - check_range(BASE, 8, 0, 1, 1, 1); - check_range(BASE, 8, 1, 1, 0, 1); - check_range(BASE, 8, 0, 0, 0, 1); - check_range(BASE, 64, 0, 1, 0, 1); - - control.cb.pmp_config <= 'h99; // Locked NAPOT read-only, applies in M-mode too. - check_range(BASE, 64, 0, 3, 1); - check_range(BASE, 64, 1, 3, 0); - check_range(BASE, 4096, 0, 1, 1, 0, 1, 0, 1, 9); // Captured PMP survives changes. - - // Higher-priority 64-byte read-only entry in the middle of a writable region. - control.cb.pmp_config <= 'h1b19; - control.cb.pmp_addr[0] <= BASE + 64 + 28; - control.cb.pmp_addr[1] <= BASE + 'hffc; - check_range(BASE, 192, 0, 1, 1); - check_range(BASE, 192, 1, 1, 0); - check_range(BASE, 64, 1, 1, 1); - check_range(BASE + 64, 64, 1, 1, 0); - check_range(BASE + 128, 64, 1, 1, 1); - - // Two adjacent TOR entries: check each actual line rather than rejecting a page envelope. - control.cb.pmp_config <= 'h0b0900; - control.cb.pmp_addr[0] <= BASE; - control.cb.pmp_addr[1] <= BASE + 64; - control.cb.pmp_addr[2] <= BASE + 128; - check_range(BASE, 128, 0, 1, 1); - check_range(BASE, 64, 1, 1, 0); - check_range(BASE + 64, 64, 1, 1, 1); - check_range(BASE, 128, 1, 1, 0); - - // A denied hole blocks a proposed union while both original pixels remain permitted. - control.cb.pmp_config <= 'h1b18; - control.cb.pmp_addr[0] <= BASE + 20; // Higher-priority 16-byte denied NAPOT hole at BASE+16. - control.cb.pmp_addr[1] <= BASE + 'hffc; - check_range(BASE, 96, 0, 1, 0); - check_range(BASE, 16, 0, 1, 1); - check_range(BASE + 64, 32, 0, 1, 1); - - // Cancel an accepted long check by joint reset. No stale response may escape. - control.cb.pmp_config <= 'h1b; - control.cb.pmp_addr[0] <= BASE + 'hffc; - control.cb.context_valid <= 1; - control.cb.context_id <= 99; - control.cb.id <= 99; - control.cb.address <= BASE; - control.cb.bytes <= 4096; - control.cb.write <= 0; - control.cb.is_pte <= 0; - control.cb.privilege <= 1; - control.cb.request_valid <= 1; - do @(control.cb); while (!control.cb.request_ready); - control.cb.request_valid <= 0; - repeat (4) @(control.cb); - initialize(); - repeat (8) begin - @(control.cb); - if (control.cb.response_valid) `uvm_fatal("RESET", "Old permission response survived reset") - end - check_range(BASE, 64, 0, 3, 1); - `uvm_info("PASS", $sformatf("%0d frozen-context PMP/PMA checks passed", checked), UVM_LOW) - endtask - endclass -endpackage - -module permission_tb; - import uvm_pkg::*; - import permission_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - permission_control_if control (clock); - PermissionVerification dut ( - .clock(clock), - .reset(control.reset), - .io_request_valid(control.request_valid), - .io_request_ready(control.request_ready), - .io_request_bits_id(control.id), - .io_request_bits_pa(control.address), - .io_request_bits_bytes(control.bytes), - .io_request_bits_write(control.write), - .io_request_bits_isPte(control.is_pte), - .io_request_bits_privilege(control.privilege), - .io_response_valid(control.response_valid), - .io_response_ready(control.response_ready), - .io_response_bits_id(control.response_id), - .io_response_bits_allow(control.allow), - .io_contextValid(control.context_valid), - .io_contextId(control.context_id), - .io_pmpConfig(control.pmp_config), - .io_pmpAddresses_0(control.pmp_addr[0]), - .io_pmpAddresses_1(control.pmp_addr[1]), - .io_pmpAddresses_2(control.pmp_addr[2]), - .io_pmpAddresses_3(control.pmp_addr[3]) - ); - initial begin - uvm_config_db#(virtual permission_control_if)::set(null, "uvm_test_top*", "control", control); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.S b/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.S deleted file mode 100644 index caf82779d..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.S +++ /dev/null @@ -1,221 +0,0 @@ -.option norelax -.option norvc - -.macro expect_trap cause, value - li s9, \cause - li s10, \value -.endm - -.macro check_trap failure - li t0, -1 - bne s9, t0, \failure -.endm - -.section .text.start,"ax",@progbits -.globl _start -_start: - li s0, 0x10000000 - li s2, 1 - li s9, -1 - li s11, 0 - la t0, trap_handler - csrw mtvec, t0 - csrw medeleg, zero - csrw mideleg, zero - csrw mie, zero - // The standalone Core has no RoCC accelerator; TIME must still be readable. - csrr t0, time - beqz t0, fail - li t0, 2 - csrw mcounteren, t0 - csrw scounteren, t0 - li t0, 0x20002800 - csrw pmpaddr0, t0 - li t0, 0x20002803 - csrw pmpaddr1, t0 - li t0, -1 - csrw pmpaddr2, t0 - li t0, 0x1f191d - csrw pmpcfg0, t0 - li t0, 0x8000000000080010 - csrw satp, t0 - sfence.vma - - // MPRV=S translates data while machine instruction fetch remains physical. - li t0, 0x21800 - csrc mstatus, t0 - li t0, 0x20800 - csrs mstatus, t0 - li t1, 0x40008000 - ld t2, 0(t1) - li t0, 0x20000 - csrc mstatus, t0 - li t0, 0x1122334455667788 - bne t2, t0, fail - sd s2, 8(s0) - li t0, 0x40004000 - csrw mepc, t0 - mret - -.section .text.trap,"ax",@progbits -trap_handler: - csrr t3, mcause - bne t3, s9, fail - csrr t0, mtval - bne t0, s10, fail - addi s11, s11, 1 - li s9, -1 - li t1, 1 - beq t3, t1, return_boundary - li t1, 8 - beq t3, t1, return_supervisor - li t1, 9 - beq t3, t1, finish - csrr t0, mepc - addi t0, t0, 4 - csrw mepc, t0 - mret -return_boundary: - la t0, boundary_return - li t1, 0x40000000 - sub t0, t0, t1 - csrw mepc, t0 - mret -return_supervisor: - sd s2, 8(s0) - // U ECALL returns through M to an explicit S continuation. - li t0, 0x1800 - csrc mstatus, t0 - li t0, 0x800 - csrs mstatus, t0 - la t0, supervisor_return - li t1, 0x40000000 - sub t0, t0, t1 - csrw mepc, t0 - mret -finish: - li t0, 6 - bne s11, t0, fail - li s2, 3 - sd s2, 8(s0) - sd zero, 0(s0) -1: j 1b -fail: - ori t0, s2, 0x100 - sd t0, 0(s0) -2: j 2b - -.section .text.supervisor,"ax",@progbits -supervisor: - csrr t0, time - beqz t0, supervisor_fail - li s2, 2 - li s1, 0x40008000 - li t0, 0x8877665544332211 - sd t0, 0(s1) - ld t1, 0(s1) - bne t0, t1, supervisor_fail - // Exercise both fetch and data through a canonical upper-half Sv39 alias. - li s3, 0xffffffffc0008000 - la t0, high_alias - li t1, 0xffffffff80000000 - add t0, t0, t1 - jalr ra, 0(t0) - // The allowed RX range is exactly one 8-byte fetch word. - li s4, 0 - li t0, 0x4000a000 - jalr ra, 0(t0) - li t0, 1 - bne s4, t0, supervisor_fail - expect_trap 1, 0x4000a008 - li t0, 0x4000a008 - jr t0 -boundary_return: - check_trap supervisor_fail - // Change the actual leaf via a writable page-table alias, then fence. - li t0, 0x40012040 - li t1, 0x200020c3 - sd t1, 0(t0) - sfence.vma s1, zero - expect_trap 15, 0x40008000 -trap_store: - sd zero, 0(s1) - check_trap supervisor_fail - ld t0, 0(s1) - li t1, 0x8877665544332211 - bne t0, t1, supervisor_fail - // Supervisor cannot read a user page with SUM clear. - expect_trap 13, 0x40006000 - li t0, 0x40006000 -trap_sum: - ld t1, 0(t0) - check_trap supervisor_fail - li t0, 0x40006000 - csrw sepc, t0 - li t0, 0x100 - csrc sstatus, t0 - sret -supervisor_return: - check_trap supervisor_fail - expect_trap 9, 0 -trap_supervisor_ecall: - ecall -3: j 3b -supervisor_fail: - ebreak - j supervisor_fail -high_alias: - ld t1, 0(s3) - li t2, 0x8877665544332211 - bne t1, t2, supervisor_fail - ret - -.section .text.user,"ax",@progbits -user: - csrr t0, time - beqz t0, user_fail - // User access to a supervisor-only leaf must fault. - expect_trap 13, 0x40008000 - li t0, 0x40008000 -trap_user_load: - ld t1, 0(t0) - check_trap user_fail - expect_trap 8, 0 -trap_user_ecall: - ecall -4: j 4b -user_fail: - ebreak - j user_fail - -.section .data,"aw",@progbits - .dword 0x1122334455667788 - -.section .text.boundary,"ax",@progbits - addi s4, s4, 1 - ret -trap_boundary: - nop - -.section .pagetables,"aw",@progbits -.balign 4096 -root: - .dword 0 - .dword 0x20004401 - .zero 4072 - .dword 0x20004401 -middle: - .dword 0x20004801 - .zero 4088 -leaf: - .zero 32 - .dword 0x200010cb - .dword 0 - .dword 0x200018db - .dword 0 - .dword 0x200020c7 - .dword 0 - .dword 0x200028cb - .zero 56 - .dword 0x200048c7 - .zero 3944 diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.cm_hier b/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.cm_hier deleted file mode 100644 index 0f4845487..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.cm_hier +++ /dev/null @@ -1,5 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.mem_req -+tree @TOP@.mem_resp -+tree @TOP@.uncached_req -+tree @TOP@.uncached_resp diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.f b/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.f deleted file mode 100644 index 1ae752420..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.f +++ /dev/null @@ -1,13 +0,0 @@ -+define+SUPERVISOR_PROFILE -// SATP PPN/ASID have no architectural reset value; select concrete bits for this 4-state run. -+define+RANDOMIZE_REG_INIT -+define+RANDOM=32'b0 -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/stream_if.sv -@VERIFY@/uvm/src/ip/package.sv -@RESOURCES@/core_interface.sv -@RESOURCES@/core_package.sv --F @RTL@/Supervisor/filelist.f -@RESOURCES@/core_tb.sv diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.ld b/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.ld deleted file mode 100644 index 8db53fc3c..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/resources/supervisor.ld +++ /dev/null @@ -1,20 +0,0 @@ -OUTPUT_ARCH(riscv) -ENTRY(_start) -SECTIONS -{ - . = 0x80000000; - .text.start : { *(.text.start) } - . = 0x80002000; - .text.trap : { *(.text.trap) } - . = 0x80004000; - .text.supervisor : { *(.text.supervisor) } - . = 0x80006000; - .text.user : { *(.text.user) } - . = 0x80008000; - .data : { *(.data) } - . = 0x8000a000; - .text.boundary : { *(.text.boundary) } - . = 0x80010000; - .pagetables : { *(.pagetables) } - /DISCARD/ : { *(.riscv.attributes) *(.comment) } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/Emit.scala b/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/Emit.scala deleted file mode 100644 index e8628b215..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/Emit.scala +++ /dev/null @@ -1,481 +0,0 @@ -package hier.core.rocket - -import java.nio.file.{Files, Paths} -import scala.sys.process._ -import framework.system.core.rocket.{CpuParams, RocketParameters} -import freechips.rocketchip.tile.FPUParams -import freechips.rocketchip.rocket.{ - ASIdBits, - DCacheParams, - HellaCacheReq, - HellaCacheResp, - ICacheParams, - RocketCoreParams -} -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion, VirtualMemoryRequest, VirtualMemoryResponse} -import chisel3._ -import memcore.memory.cache.configs.CacheParams -import memcore.memory.coherence.configs.CoherenceParams -import memcore.bus.chi.{Params => ChiParams} - -object Emit extends App { - val root = Paths.get("src/main/scala/framework/root/root-core").toAbsolutePath - val output = root.resolve("build") - Files.createDirectories(output) - require( - Process(Seq("python", root.resolve("src/main/resources/build_fixture.py").toString, output.toString)).! == 0, - "Core fixture generation failed" - ) - - implicit val cpuParams: CpuParams = CpuParams( - core = new RocketCoreParams( - useVM = false, - useUser = false, - useSupervisor = false, - useHypervisor = false, - useDebug = false, - fpu = None, - nPMPs = 4, - clockGate = false, - haveCFlush = false, - haveCease = false, - haveSimTimeout = false - ) { override val pmpGranularity = 8 }, - dcache = Some(DCacheParams(nSets = 64, nWays = 2)), - icache = Some(ICacheParams()), - btb = None, - physicalAddressBits = 44, - hartIdBits = 1, - beatBytes = 8, - blockBytes = 64 - ) - - implicit val upstream = RocketParameters(cpuParams) - - val memory = CoherenceParams(ChiParams(), CacheParams(44, 64, 4, 2, 8, 4, 2), 2, 4, 64) - - val floatingParameters = cpuParams.copy(core = cpuParams.core.copy(fpu = Some(FPUParams(minFLen = 16, fLen = 64)))) - - val regions = Seq( - PhysicalRegion(BigInt("80000000", 16), BigInt(16) << 20, true, true, true, true, true, true), - PhysicalRegion(BigInt("10000000", 16), BigInt(4096), false, false, true, true, false, false) - ) - - require( - Process(Seq("python", root.resolve("src/main/resources/build_fixture.py").toString, output.toString, "fpu")).! == 0, - "FPU fixture generation failed" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Verification(memory, regions, "Fpu")(floatingParameters), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Fpu").toString, "--split-verilog") - ) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Verification(memory, regions, "Core"), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Core").toString, "--split-verilog") - ) - - val supervisorParameters = cpuParams.copy( - core = new RocketCoreParams( - useVM = true, - useUser = true, - useSupervisor = true, - useHypervisor = false, - useDebug = false, - fpu = None, - nPMPs = 4, - clockGate = false, - haveCFlush = false, - haveCease = false, - haveSimTimeout = false - ) { override val pmpGranularity = 8 }, - dcache = Some(DCacheParams(nSets = 64, nWays = 2)), - icache = Some(ICacheParams()), - btb = None, - physicalAddressBits = 36, - asIdBits = 16 - ) - - require( - Process( - Seq("python", root.resolve("src/main/resources/build_fixture.py").toString, output.toString, "supervisor") - ).! == 0, - "Supervisor fixture generation failed" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Verification(memory, regions, "Supervisor")(supervisorParameters), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Supervisor").toString, "--split-verilog") - ) - - require( - Process(Seq("python", root.resolve("src/main/resources/build_fixture.py").toString, output.toString, "irq")).! == 0, - "IRQ fixture generation failed" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Verification(memory, regions, "Irq"), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Irq").toString, "--split-verilog") - ) - - val definitions = scala.collection.mutable.ArrayBuffer[String]( - s"""`define IRQ_IMAGE "${output.resolve("irq.hex")}"""", - s"""`define FPU_IMAGE "${output.resolve("fpu.hex")}"""", - s"""`define SUPERVISOR_IMAGE "${output.resolve("supervisor.hex")}"""", - s"""`define CORE_IMAGE "${output.resolve("core.hex")}"""" - ) - - val ports = scala.collection.mutable.ArrayBuffer[String]() - - def channel( - kind: String, - prefix: String, - instance: String, - fields: Seq[(String, Int)] - ): Unit = { - var offset = 0 - for ((name, width) <- fields) { - definitions += s"`define CORE_${kind}_${name.toUpperCase}_OFFSET $offset" - definitions += s"`define CORE_${kind}_${name.toUpperCase}_WIDTH $width" - ports += s".$prefix$name($instance.bits[$offset +: $width])" - offset += width - } - definitions += s"`define CORE_${kind}_WIDTH $offset" - } - - channel( - "MEMREQ", - "io_memoryRequest_bits_", - "mem_req", - Seq("id" -> 12, "addr" -> 44, "write" -> 1, "data" -> 512, "mask" -> 64) - ) - channel("MEMRESP", "io_memoryResponse_bits_", "mem_resp", Seq("id" -> 12, "data" -> 512, "error" -> 1)) - channel( - "UNCACHED", - "io_uncachedRequest_bits_", - "uncached_req", - Seq("addr" -> 64, "tag" -> 6, "size" -> 3, "write" -> 1, "data" -> 64) - ) - channel("URESP", "io_uncachedResponse_bits_", "uncached_resp", Seq("tag" -> 6, "data" -> 64, "error" -> 1)) - for ( - (prefix, instance) <- Seq( - "memoryRequest" -> "mem_req", - "memoryResponse" -> "mem_resp", - "uncachedRequest" -> "uncached_req", - "uncachedResponse" -> "uncached_resp" - ) - ) { - ports += s".io_${prefix}_valid($instance.valid)" - ports += s".io_${prefix}_ready($instance.ready)" - } - Files.write(output.resolve("core_config.svh"), (definitions.mkString("\n") + "\n").getBytes) - Files.write(output.resolve("core_ports.svh"), ports.mkString(",\n").getBytes) - val cp = CpuMemParams(memory.chi, tagBits = 6) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Lsu(cp), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Lsu").toString, "--split-verilog") - ) - definitions.clear() - ports.clear() - - def elements(bundle: Bundle): Seq[(String, Int)] = bundle.elements.toSeq.collect { - case (name, field) if field.getWidth > 0 => name -> field.getWidth - } - - channel("CPU", "io_cpu_req_bits_", "source_if", elements(new HellaCacheReq)) - channel("RETURN", "io_cpu_resp_bits_", "sink_if", elements(new HellaCacheResp)) - channel("VIRTUAL", "io_request_bits_", "virtual_req", elements(new VirtualMemoryRequest(cp))) - channel("RESULT", "io_response_bits_", "virtual_resp", elements(new VirtualMemoryResponse(cp))) - for ((prefix, instance) <- Seq("cpu_req" -> "source_if", "request" -> "virtual_req", "response" -> "virtual_resp")) { - ports += s".io_${prefix}_valid($instance.valid)" - ports += s".io_${prefix}_ready($instance.ready)" - } - ports += ".io_cpu_resp_valid(sink_if.valid)" - for (i <- 0 until 4) { - for (name <- Seq("l", "res", "a", "x", "w", "r")) ports += s".io_pmp_${i}_cfg_$name('0)" - ports += s".io_pmp_${i}_addr('0)" - ports += s".io_pmp_${i}_mask('0)" - } - ports += ".io_context_privilege(2'd3)" - ports += ".io_context_satp(64'd0)" - ports += ".io_context_sum(1'b0)" - ports += ".io_context_mxr(1'b0)" - ports += ".io_maintenance_ready(1'b0)" - ports += ".io_maintained_valid(1'b0)" - ports += ".io_maintained_bits(1'b0)" - Files.write(output.resolve("lsu_config.svh"), (definitions.mkString("\n") + "\n").getBytes) - Files.write(output.resolve("lsu_ports.svh"), ports.mkString(",\n").getBytes) - - val tracking = memcore.memory.interlock.Params() - val rnf = memcore.bus.chi.rnf.RnfParams(memory.chi, homeCount = 2) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Maintenance(rnf, tracking), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Maintenance").toString, "--split-verilog") - ) - definitions.clear() - ports.clear() - definitions += s"`define MAINTENANCE_HOME_BASE ${rnf.homeId}" - definitions += s"`define MAINTENANCE_HOME_COUNT ${rnf.homeCount}" - definitions += s"`define MAINTENANCE_NODE ${rnf.nodeId}" - definitions += s"`define MAINTENANCE_TXN ${rnf.banks}" - channel("RANGE", "io_request_bits_", "source_if", elements(new memcore.memory.interlock.Maintenance(tracking))) - channel("ACK", "io_response_bits_", "sink_if", elements(new memcore.memory.interlock.Acknowledgement(tracking))) - channel("REQ", "io_req_bits_", "chi_req", elements(new memcore.bus.chi.RequestFlit(memory.chi))) - channel("RSP", "io_rsp_bits_", "chi_rsp", elements(new memcore.bus.chi.ResponseFlit(memory.chi))) - for ( - (prefix, instance) <- Seq("request" -> "source_if", "response" -> "sink_if", "req" -> "chi_req", "rsp" -> "chi_rsp") - ) { - ports += s".io_${prefix}_valid($instance.valid)" - ports += s".io_${prefix}_ready($instance.ready)" - } - Files.write(output.resolve("maintenance_config.svh"), (definitions.mkString("\n") + "\n").getBytes) - Files.write(output.resolve("maintenance_ports.svh"), ports.mkString(",\n").getBytes) - - require( - Process( - Seq("python", root.resolve("src/main/resources/build_fixture.py").toString, output.toString, "admission") - ).! == 0, - "Admission fixture generation failed" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Verification( - memory, - regions :+ PhysicalRegion(BigInt("a0000000", 16), BigInt(4096), false, false, true, true, true, true), - "AdmissionCore", - Some(Commands(compute = true, scheduler = true, tracking = tracking)) - )(supervisorParameters), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("AdmissionCore").toString, "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new memcore.memory.interlock.Interlock(tracking), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("AdmissionInterlock").toString, "--split-verilog") - ) - - val admissionDefinitions = scala.collection.mutable.ArrayBuffer[String]( - s"""`define ADMIT_IMAGE "${output.resolve("admission.hex")}"""", - s"`define ADMIT_ENTRIES ${tracking.entries}", - s"`define ADMIT_PMP_COUNT ${supervisorParameters.core.nPMPs}", - s"`define ADMIT_OUTSTANDING_BITS ${chisel3.util.log2Ceil(tracking.entries + 1)}", - s"`define ADMIT_MEMORY_OUTSTANDING_BITS ${chisel3.util.log2Ceil(memory.mshrEntries + 1)}" - ) - - val admissionPorts = scala.collection.mutable.ArrayBuffer[String]() - - def flatten(data: Data, prefix: String = ""): Seq[(String, Int)] = data match { - case record: Record => record.elements.toSeq.flatMap { case (name, value) => - flatten(value, if (prefix.isEmpty) name else s"${prefix}_$name") - } - case vector: Vec[_] => vector.zipWithIndex.flatMap { case (value, i) => flatten(value, s"${prefix}_$i") } - case field if field.getWidth > 0 => Seq(prefix -> field.getWidth) - case _ => Seq.empty - } - - def admissionLayout( - kind: String, - prefix: String, - signal: String, - bundle: Data - ): Unit = { - var offset = 0 - for ((name, width) <- flatten(bundle)) { - admissionDefinitions += s"`define ADMIT_${kind}_${name.toUpperCase}_OFFSET $offset" - admissionDefinitions += s"`define ADMIT_${kind}_${name.toUpperCase}_WIDTH $width" - admissionPorts += s".$prefix$name($signal[$offset +: $width])" - offset += width - } - admissionDefinitions += s"`define ADMIT_${kind}_WIDTH $offset" - } - - for ( - (kind, port, signal, bundle) <- Seq( - ( - "RESERVE", - "admission_reserve", - "reserve", - new memcore.memory.interlock.Dispatch(tracking) - ), - ( - "COMMAND", - "admission_command", - "command", - new CommandSnapshot(tracking, supervisorParameters.core.nPMPs)( - supervisorParameters - ) - ), - ( - "COMPLETE", - "admission_complete", - "complete", - new memcore.memory.interlock.Tag(tracking) - ), - ( - "CANCELLED", - "admission_cancelled", - "cancelled", - new memcore.memory.interlock.Tag(tracking) - ), - ( - "RESPONSE", - "admission_response", - "response", - new framework.system.core.rocket.RoCCResponseBB - ), - ( - "MAINTENANCE", - "admission_maintenance", - "maintenance", - new memcore.memory.interlock.Maintenance(tracking) - ), - ( - "MAINTAINED", - "admission_maintained", - "maintained", - new memcore.memory.interlock.Acknowledgement(tracking) - ), - ( - "PTE", - "admission_pteRequest", - "pte_req", - new memcore.bus.chi.rnf.CacheAccess(memory.chi) - ), - ( - "PTERESP", - "admission_pteResponse", - "pte_resp", - new memcore.bus.chi.rnf.CacheResult - ), - ( - "MEMREQ", - "memoryRequest", - "mem_req", - new memcore.bus.chi.snf.LineRequest(memory.chi) - ), - ( - "MEMRESP", - "memoryResponse", - "mem_resp", - new memcore.bus.chi.snf.LineResponse(memory.chi) - ), - ( - "UNCACHED", - "uncachedRequest", - "uncached_req", - new memcore.memory.cpu.UncachedRequest(cp) - ), - ( - "URESP", - "uncachedResponse", - "uncached_resp", - new memcore.memory.cpu.UncachedResponse(cp) - ) - ) - ) { - admissionLayout(kind, s"io_${port}_bits_", s"$signal.bits", bundle) - admissionPorts += s".io_${port}_valid($signal.valid)" - admissionPorts += s".io_${port}_ready($signal.ready)" - } - admissionLayout("QUERY", "io_admission_cpuQuery_", "control.query", new memcore.memory.interlock.CpuQuery(tracking)) - admissionPorts ++= Seq( - ".io_admission_cpuAllow(control.cpu_allow)", - ".io_admission_cpuProbeAllow(control.cpu_allow)", - ".io_admission_interrupt(control.accelerator_irq)", - ".io_admission_outstanding(control.admission_outstanding)" - ) - Files.write(output.resolve("admission_config.svh"), (admissionDefinitions.mkString("\n") + "\n").getBytes) - Files.write(output.resolve("admission_ports.svh"), admissionPorts.mkString(",\n").getBytes) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new AdmissionBridge(16, tracking, supervisorParameters.core.nPMPs, Seq(1))(supervisorParameters), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("AdmissionBridge").toString, "--split-verilog") - ) - - val bridgeDefinitions = scala.collection.mutable.ArrayBuffer[String]( - "`define AB_ROB_ENTRIES 16", - "`define AB_ROB_BITS 4", - s"`define AB_ENTRIES ${tracking.entries}" - ) - - val bridgePorts = scala.collection.mutable.ArrayBuffer[String]() - val bridgeSnapshot = new CommandSnapshot(tracking, supervisorParameters.core.nPMPs)(supervisorParameters) - - def bridgeLayout( - kind: String, - prefix: String, - signal: String, - data: Data, - define: Boolean - ): Unit = { - var offset = 0 - for ((name, width) <- flatten(data)) { - if (define) { - bridgeDefinitions += s"`define AB_${kind}_${name.toUpperCase}_OFFSET $offset" - bridgeDefinitions += s"`define AB_${kind}_${name.toUpperCase}_WIDTH $width" - } - bridgePorts += s".$prefix$name($signal[$offset +: $width])" - offset += width - } - if (define) bridgeDefinitions += s"`define AB_${kind}_WIDTH $offset" - } - - bridgeLayout("SNAP", "io_command_bits_", "command.bits", bridgeSnapshot, true) - val bridgeRetirement = - new AdmissionRetirement(tracking, supervisorParameters.core.nPMPs)(supervisorParameters) - bridgeLayout("RET", "io_retirement_bits_", "retirement.bits", bridgeRetirement, true) - bridgeLayout("FAULT", "io_fault_bits_", "control.fault_bits", new RobFault(16), true) - bridgeLayout("FAULT", "io_unboundFault_bits_", "control.unbound_fault_bits", new RobFault(16), false) - bridgePorts ++= Seq(".io_fault_valid(control.fault_valid)", ".io_unboundFault_valid(control.unbound_fault_valid)") - val snapshotStart = - flatten(bridgeRetirement).takeWhile { case (name, _) => !name.startsWith("snapshot_") }.map(_._2).sum - bridgeDefinitions += s"`define AB_RET_SNAPSHOT_OFFSET $snapshotStart" - bridgeLayout("NPU", "io_npuCommand_bits_", "npu.bits", new framework.system.core.rocket.RoCCCommandBB, true) - for ((port, signal) <- Seq("command" -> "command", "retirement" -> "retirement", "npuCommand" -> "npu")) { - bridgePorts += s".io_${port}_valid($signal.valid)" - bridgePorts += s".io_${port}_ready($signal.ready)" - } - for (i <- 0 until 3) { - bridgeLayout("SNAP", s"io_lookup_${i}_snapshot_bits_", s"control.lookup_bits[$i]", bridgeSnapshot, false) - bridgePorts += s".io_lookup_${i}_snapshot_valid(control.lookup_valid[$i])" - bridgePorts += s".io_lookup_${i}_robId(control.lookup_id[$i])" - } - bridgePorts += ".io_lookupTag_tag(8'd0)" - bridgePorts ++= Seq( - ".io_allocation_valid(control.allocation_valid)", - ".io_allocation_bits(control.allocation_id)", - ".io_retired(control.retired)" - ) - val instructionStart = - flatten(bridgeSnapshot).takeWhile { case (name, _) => !name.startsWith("instruction_") }.map(_._2).sum - bridgeDefinitions += s"`define AB_INSTRUCTION_OFFSET $instructionStart" - Files.writeString( - output.resolve("admission_bridge_config.svh"), - "`ifndef ADMISSION_BRIDGE_CONFIG\n`define ADMISSION_BRIDGE_CONFIG\n" + bridgeDefinitions.mkString( - "\n" - ) + "\n`endif\n" - ) - Files.writeString(output.resolve("admission_bridge_ports.svh"), bridgePorts.mkString(",\n") + "\n") - - val preparationConfig = memcore.memory.preflight.Params(bus = memory.chi, beatBytes = 64) - - val preparationRegions = Seq( - PhysicalRegion(BigInt("80000000", 16), BigInt(16) << 20, true, true, true, true, true, true), - PhysicalRegion(BigInt("90000000", 16), BigInt(4096), false, false, true, false, false, true) - ) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new PermissionVerification(preparationConfig, preparationRegions)(supervisorParameters), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("PermissionVerification").toString, "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Preparation(preparationConfig, preparationRegions)(supervisorParameters), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Preparation").toString, "--split-verilog") - ) - -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/PermissionVerification.scala b/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/PermissionVerification.scala deleted file mode 100644 index dc7afaa6d..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/PermissionVerification.scala +++ /dev/null @@ -1,39 +0,0 @@ -package hier.core.rocket - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import freechips.rocketchip.rocket.PMPConfig -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.preflight.{Authorization, Permission, Params => PreparationParams} -import chisel3.util.Decoupled - -@instantiable -class PermissionVerification(config: PreparationParams, regions: Seq[PhysicalRegion])(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - require(nPMPs == 4) - - @public - val io = IO(new Bundle { - val request = Flipped(Decoupled(new Authorization(config))) - val response = Decoupled(new Permission(config)) - val contextValid = Input(Bool()) - val contextId = Input(UInt(config.idBits.W)) - val pmpConfig = Input(UInt(32.W)) - // PMP address CSRs expressed as CSR << 2, including their NAPOT encoding. - val pmpAddresses = Input(Vec(4, UInt(64.W))) - }) - - val permission: Instance[PermissionCheck] = Instantiate(new PermissionCheck(config, regions)) - permission.io.request <> io.request - io.response <> permission.io.response - permission.io.contextValid := io.contextValid - permission.io.contextId := io.contextId - for (i <- 0 until 4) { - val pmp = permission.io.pmp(i) - pmp.cfg := io.pmpConfig(8 * i + 7, 8 * i).asTypeOf(new PMPConfig) - pmp.addr := io.pmpAddresses(i)(paddrBits - 1, 2) - pmp.mask := pmp.computeMask - } -} diff --git a/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/Verification.scala b/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/Verification.scala deleted file mode 100644 index cbfd4aa17..000000000 --- a/arch/src/main/scala/framework/root/root-core/src/main/scala/rocket/Verification.scala +++ /dev/null @@ -1,98 +0,0 @@ -package hier.core.rocket - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import freechips.rocketchip.tile.TraceBundle -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion, UncachedRequest, UncachedResponse} -import memcore.memory.coherence.Coherence -import memcore.memory.coherence.configs.CoherenceParams -import memcore.bus.chi.Fabric -import memcore.bus.chi.rnf.RnfParams -import memcore.bus.chi.snf.{LineRequest, LineResponse} - -/** Verification system: the production Core's data and instruction L1s share a canonical Home. */ -@instantiable -class Verification( - memory: CoherenceParams, - regions: Seq[PhysicalRegion], - topName: String, - commands: Option[Commands] = None -)( - implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - override def desiredName: String = topName - require(memory.agents == 2) - val cp = CpuMemParams(memory.chi, tagBits = 6) - - @public - val io = IO(new Bundle { - val timerInterrupt = Input(Bool()) - val softwareInterrupt = Input(Bool()) - val externalInterrupt = Input(Bool()) - val resetVector = Input(UInt(64.W)) - val time = Input(UInt(64.W)) - val memoryRequest = Decoupled(new LineRequest(memory.chi)) - val memoryResponse = Flipped(Decoupled(new LineResponse(memory.chi))) - val uncachedRequest = Decoupled(new UncachedRequest(cp)) - val uncachedResponse = Flipped(Decoupled(new UncachedResponse(cp))) - val trace = Output(new TraceBundle) - val retired = Output(Bool()) - val retiredPc = Output(UInt(64.W)) - val trapped = Output(Bool()) - val trapCause = Output(UInt(64.W)) - val trapValue = Output(UInt(64.W)) - val trapPc = Output(UInt(64.W)) - val cancelledData = Output(Bool()) - val memoryOutstanding = Output(UInt(log2Ceil(memory.mshrEntries + 1).W)) - val blockMaintenanceResponse = commands.map(_ => Input(Bool())) - val maintenanceResponsePending = commands.map(_ => Output(Bool())) - val admission = commands.map(mode => new AdmissionPorts(mode.tracking, cpuParams.core.nPMPs, memory.chi)) - }) - - val config = RnfParams(memory.chi, nodeId = 1, cacheLines = 8, homeId = memory.homeId, homeCount = 1, banks = 2) - val core: Instance[Core] = Instantiate(new Core(config, config.copy(nodeId = 2), regions, commands)) - val home: Instance[Coherence] = Instantiate(new Coherence(memory)) - val fabric: Instance[Fabric] = Instantiate(new Fabric(memory.chi, memory.agents, memory.homeId)) - commands.foreach(_ => io.admission.get <> core.clink.admission.get) - core.clink.resetVector := io.resetVector - core.clink.time := io.time - core.clink.timerInterrupt := io.timerInterrupt - core.clink.softwareInterrupt := io.softwareInterrupt - core.clink.externalInterrupt := io.externalInterrupt - core.clink.supervisorExternalInterrupt := false.B - core.clink.hartId := 0.U - io.uncachedRequest <> core.clink.uncachedRequest - core.clink.uncachedResponse <> io.uncachedResponse - io.trace := core.clink.trace - io.retired := core.clink.retired - io.retiredPc := core.clink.retiredPc - io.trapped := core.clink.trapped - io.trapCause := core.clink.trapCause - io.trapValue := core.clink.trapValue - io.trapPc := core.clink.trapPc - io.cancelledData := core.clink.cancelledData - fabric.io.active := true.B - fabric.io.blockRequesterRsp := false.B - fabric.io.requesters(0) <> core.clink.chi - commands.foreach { _ => - val response = fabric.io.requesters(0).rxRsp - val isMaintenance = response.bits.txnId === config.banks.U - val hold = io.blockMaintenanceResponse.get && isMaintenance - core.clink.chi.rxRsp.valid := response.valid && !hold - response.ready := core.clink.chi.rxRsp.ready && !hold - io.maintenanceResponsePending.get := response.valid && isMaintenance - } - fabric.io.requesters(1) <> core.clink.instructionChi - home.io.req <> fabric.io.req - home.io.rxRsp <> fabric.io.rxRsp - home.io.rxDat <> fabric.io.rxDat - fabric.io.rsp <> home.io.rsp - fabric.io.dat <> home.io.dat - fabric.io.snp <> home.io.snp - io.memoryRequest <> home.io.memoryReq - home.io.memoryResp <> io.memoryResponse - io.memoryOutstanding := home.io.outstanding -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/csrc/Cargo.lock b/arch/src/main/scala/framework/root/root-tile/src/csrc/Cargo.lock deleted file mode 100644 index e2fc5f03e..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/csrc/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "tile-memory-reference-model" -version = "0.1.0" diff --git a/arch/src/main/scala/framework/root/root-tile/src/csrc/Cargo.toml b/arch/src/main/scala/framework/root/root-tile/src/csrc/Cargo.toml deleted file mode 100644 index 445a6b832..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/csrc/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "tile-memory-reference-model" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib"] -path = "src/lib.rs" diff --git a/arch/src/main/scala/framework/root/root-tile/src/csrc/src/consistency.rs b/arch/src/main/scala/framework/root/root-tile/src/csrc/src/consistency.rs deleted file mode 100644 index 04b5b4f44..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/csrc/src/consistency.rs +++ /dev/null @@ -1,80 +0,0 @@ -//! Physical DDR and CPU-visible architectural memory are distinct golden stores. -//! CPU dirty writes do not commit DDR. Only observed Home write ACKs or the mock DMA do. -use std::{collections::BTreeMap, ffi::c_void}; -struct Memory { - ddr: BTreeMap, - visible: BTreeMap, -} -fn initial(address: u64) -> [u8; 64] { - let mut line = [0; 64]; - for (i, byte) in line.iter_mut().enumerate() { - *byte = ((address >> 6) as u8).wrapping_mul(19).wrapping_add((i as u8).wrapping_mul(13)) ^ 0x5a; - } - line -} -fn line(map: &BTreeMap, address: u64) -> [u8; 64] { - *map.get(&address).unwrap_or(&initial(address)) -} -fn write_word(map: &mut BTreeMap, address: u64, data: u64, mask: u32) { - assert!(address < 1 << 44 && address & 7 == 0); - let base = address & !63; - let target = map.entry(base).or_insert_with(|| initial(base)); - for (i, byte) in data.to_le_bytes().iter().enumerate() { - if mask & (1 << i) != 0 { target[(address & 63) as usize + i] = *byte; } - } -} -unsafe fn memory<'a>(ptr: *mut c_void) -> &'a mut Memory { &mut *ptr.cast::() } -#[no_mangle] -pub extern "C" fn cons_ref_create() -> *mut c_void { - Box::into_raw(Box::new(Memory { ddr: BTreeMap::new(), visible: BTreeMap::new() })).cast() -} -#[no_mangle] -pub unsafe extern "C" fn cons_ref_destroy(ptr: *mut c_void) { drop(Box::from_raw(ptr.cast::())); } -#[no_mangle] -pub unsafe extern "C" fn cons_ref_expected64(ptr: *mut c_void, address: u64) -> u64 { - assert!(address & 7 == 0); - let bytes = line(&memory(ptr).visible, address & !63); - u64::from_le_bytes(bytes[(address & 63) as usize..(address & 63) as usize + 8].try_into().unwrap()) -} -#[no_mangle] -pub unsafe extern "C" fn cons_ref_cpu_commit(ptr: *mut c_void, address: u64, data: u64, mask: u32) { - write_word(&mut memory(ptr).visible, address, data, mask); -} -#[no_mangle] -pub unsafe extern "C" fn cons_ref_ddr_read(ptr: *mut c_void, address: u64, output: *mut u32) { - assert!(address & 63 == 0 && address < 1 << 44); - let bytes = line(&memory(ptr).ddr, address); - for (i, word) in std::slice::from_raw_parts_mut(output, 16).iter_mut().enumerate() { - *word = u32::from_le_bytes(bytes[i * 4..i * 4 + 4].try_into().unwrap()); - } -} -#[no_mangle] -pub unsafe extern "C" fn cons_ref_write_matches(ptr: *mut c_void, address: u64, data: *const u32, mask: *const u32) -> u32 { - assert!(address & 63 == 0 && address < 1 << 44); - let wanted = line(&memory(ptr).visible, address); - let data = std::slice::from_raw_parts(data, 16); - let mask = std::slice::from_raw_parts(mask, 2); - u32::from((0..64).all(|i| mask[i / 32] & (1 << (i % 32)) == 0 || wanted[i] == (data[i / 4] >> (8 * (i % 4))) as u8)) -} -#[no_mangle] -pub unsafe extern "C" fn cons_ref_ddr_commit(ptr: *mut c_void, address: u64, data: *const u32, mask: *const u32) { - assert!(address & 63 == 0 && address < 1 << 44); - let data = std::slice::from_raw_parts(data, 16); - let mask = std::slice::from_raw_parts(mask, 2); - let target = memory(ptr).ddr.entry(address).or_insert_with(|| initial(address)); - for (i, byte) in target.iter_mut().enumerate() { - if mask[i / 32] & (1 << (i % 32)) != 0 { *byte = (data[i / 4] >> (8 * (i % 4))) as u8; } - } -} -#[no_mangle] -pub unsafe extern "C" fn cons_ref_ddr_visible(ptr: *mut c_void, address: u64) -> u32 { - assert!(address & 63 == 0); - let m = memory(ptr); - u32::from(line(&m.ddr, address) == line(&m.visible, address)) -} -#[no_mangle] -pub unsafe extern "C" fn cons_ref_dma_write(ptr: *mut c_void, address: u64, data: u64, mask: u32) { - let m = memory(ptr); - write_word(&mut m.ddr, address, data, mask); - write_word(&mut m.visible, address, data, mask); -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/csrc/src/lib.rs b/arch/src/main/scala/framework/root/root-tile/src/csrc/src/lib.rs deleted file mode 100644 index e1552417d..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/csrc/src/lib.rs +++ /dev/null @@ -1,15 +0,0 @@ -// Reuse the actual leaf reference implementations, including their exported DPI API. -#[path = "../../../../../mem-core/coherence/src/csrc/src/memory.rs"] -mod line_memory; -#[path = "../../../../../mem-core/mmu/src/csrc/src/translation.rs"] -mod mmu; -#[path = "../../../../../mem-core/cpu_mem/src/csrc/src/access.rs"] -mod cpu_mem; - -#[path = "../../../../root-core/src/csrc/src/memory.rs"] -mod core_memory; - -mod consistency; - -#[path = "../../../../../mem-core/ddr/src/csrc/src/memory.rs"] -mod ddr_memory; diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system.f deleted file mode 100644 index f75e2acf5..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/cache_system_interface.sv -@RESOURCES@/cache_system_package.sv --F @RTL@/CacheSystem/filelist.f -@RESOURCES@/cache_system_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_interface.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_interface.sv deleted file mode 100644 index 3c33cdf15..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_interface.sv +++ /dev/null @@ -1,35 +0,0 @@ -`include "cache_system_config.svh" -interface cache_system_if ( - input logic clock -); - logic reset, active, block_requester_rsp; - logic access_valid[2], access_ready[2], access_write[2], access_atomicWord[2]; - logic [`COH_REQ_ADDR_WIDTH-1:0] access_addr[2]; - logic [63:0] access_data[2], result_data[2]; - logic [7:0] access_mask [2]; - logic [3:0] access_atomic[2]; - logic result_valid[2], result_ready[2], result_error[2]; - logic [`COH_OUTSTANDING_BITS-1:0] outstanding; - logic req_valid, rsp_valid, dat_valid, snp_valid, rx_rsp_valid, rx_dat_valid; - logic [`COH_REQ_WIDTH-1:0] req_bits; - logic [`COH_RSP_WIDTH-1:0] rsp_bits, rx_rsp_bits; - logic [`COH_DAT_WIDTH-1:0] dat_bits, rx_dat_bits; - logic [`COH_SNP_CHANNEL_WIDTH-1:0] snp_bits; - clocking sample @(posedge clock); - default input #1step; - input reset,active,block_requester_rsp,access_atomicWord,access_valid,access_ready,access_write,access_addr,access_data,access_mask,access_atomic; - input result_valid, result_ready, result_error, result_data, outstanding; - input req_valid, rsp_valid, dat_valid, snp_valid, rx_rsp_valid, rx_dat_valid; - input req_bits, rsp_bits, dat_bits, snp_bits, rx_rsp_bits, rx_dat_bits; - endclocking - for (genvar core = 0; core < 2; core++) begin - assert property (@(posedge clock) disable iff(reset) access_valid[core]&&!access_ready[core] |=> access_valid[core]&&$stable( - {access_addr[core],access_write[core],access_data[core],access_mask[core],access_atomic[core],access_atomicWord[core]} - )) - else $fatal(1, "CacheSystem client changed stalled command"); - assert property (@(posedge clock) disable iff(reset) result_valid[core]&&!result_ready[core] |=> result_valid[core]&&$stable( - {result_data[core], result_error[core]} - )) - else $fatal(1, "CacheSystem changed stalled result"); - end -endinterface diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_package.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_package.sv deleted file mode 100644 index 1cfdac6a0..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_package.sv +++ /dev/null @@ -1,606 +0,0 @@ -`include "cache_system_config.svh" -`define CSF(K, F) `COH_``K``_``F``_OFFSET +: `COH_``K``_``F``_WIDTH -package cache_system_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - typedef logic [`COH_REQ_WIDTH-1:0] req_t; - typedef logic [`COH_RSP_WIDTH-1:0] rsp_t; - typedef logic [`COH_DAT_WIDTH-1:0] dat_t; - typedef bit [`COH_MEMRESP_WIDTH-1:0] mem_t; - import "DPI-C" function chandle coherence_ref_create(); - import "DPI-C" function void coherence_ref_destroy(input chandle model); - import "DPI-C" function void coherence_ref_read( - input chandle model, - input longint unsigned addr, - output bit [511:0] data - ); - import "DPI-C" function void coherence_ref_write( - input chandle model, - input longint unsigned addr, - input bit [511:0] data - ); - class result_item extends uvm_sequence_item; - bit [63:0] data; - bit error; - `uvm_object_utils_begin(result_item) - `uvm_field_int(data, UVM_DEFAULT) - `uvm_field_int(error, UVM_DEFAULT) - `uvm_object_utils_end - function new(string name = "result_item"); - super.new(name); - endfunction - endclass - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual cache_system_if vif; - virtual stream_if #(`COH_MEMREQ_WIDTH) mem_req; - virtual stream_if #(`COH_MEMRESP_WIDTH) mem_resp; - in_order_scoreboard #(result_item) scoreboard[2]; - chandle architectural, backing; - typedef struct packed { - bit write; - bit [63:0] addr, data; - bit [7:0] mask; - } command_t; - typedef command_t command_queue[$]; - command_queue commands[2]; - typedef struct { - mem_t bits; - int due, serial; - } memory_entry; - memory_entry pending[$]; - req_t requests[int]; - bit [511:0] read_expected[int], snoop_expected[int]; - longint unsigned wb_address[int]; - int - fill_seen[int], - fill_dbid[int], - snoop_seen[int], - snoop_target[int], - wb_seen[int], - wb_owner[int], - slot_owner[int]; - bit ack_pending[int]; - int submitted[2] = '{0, 0}; - int result_stalls[2] = '{0, 0}; - int home_requests = 0, rsp_block_cycles = 0, release_cycle = -1, ack_resume_cycles = -1; - int - cycle = 0, - peak = 0, - memory_reads = 0, - memory_writes = 0, - memory_serial = 0, - last_memory_serial = -1; - int full_lines = 0, snoop_lines = 0, copybacks = 0, dirty_snoops = 0, acks = 0, errors = 0; - bit - hold_memory = 0, - hold_results = 0, - mem_active = 0, - saw_reorder = 0, - saw_dbid_mismatch = 0, - saw_large_snoop = 0; - bit [`COH_MEMRESP_WIDTH-1:0] mem_packet; - localparam bit [63:0] ERROR_ADDRESS = 64'hf000; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 1ms; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if(`COH_AGENTS!=2||`COH_MSHRS!=4||`COH_SETS!=4||`COH_WAYS!=2||`COH_DATA_BITS!=256) - `uvm_fatal( - "CONFIG", - "CacheSystem regression requires 2 requesters, 4 MSHRs/sets, 2 ways and 256-bit DAT") - if (!uvm_config_db#(virtual cache_system_if)::get( - this, "", "control", vif - ) || !uvm_config_db#(virtual stream_if #(`COH_MEMREQ_WIDTH))::get( - this, "", "mem_req", mem_req - ) || !uvm_config_db#(virtual stream_if #(`COH_MEMRESP_WIDTH))::get( - this, "", "mem_resp", mem_resp - )) - `uvm_fatal("VIF", "CacheSystem interfaces missing") - for (int i = 0; i < 2; i++) - scoreboard[i] = - in_order_scoreboard#(result_item)::type_id::create($sformatf("scoreboard_%0d", i), this); - architectural = coherence_ref_create(); - backing = coherence_ref_create(); - endfunction - function void observe_cpu(); - for (int core = 0; core < 2; core++) begin - if (vif.sample.access_valid[core] && vif.sample.access_ready[core]) begin - command_t cmd; - bit [511:0] line; - result_item expected; - cmd = '{ - vif.sample.access_write[core], - 64'(vif.sample.access_addr[core]), - vif.sample.access_data[core], - vif.sample.access_mask[core] - }; - if (vif.sample.access_atomicWord[core] !== 1'b0) - `uvm_fatal("PROFILE", "This regression drives atomicWord=False") - commands[core].push_back(cmd); - coherence_ref_read(architectural, cmd.addr & ~64'h3f, line); - expected = new; - expected.error = cmd.addr == ERROR_ADDRESS; - expected.data = expected.error ? 0 : line[int'(cmd.addr&63)*8+:64]; - scoreboard[core].expected_export.write(expected); - submitted[core]++; - end - if (vif.sample.result_valid[core] && vif.sample.result_ready[core]) begin - command_t cmd; - bit [511:0] line; - result_item actual; - if ($isunknown({vif.sample.result_data[core], vif.sample.result_error[core]})) - `uvm_fatal("RESULT_X", "Unknown CPU result") - if (!commands[core].size()) - `uvm_fatal("EXTRA_RESULT", "CPU result without an accepted command") - cmd = commands[core].pop_front(); - actual = new; - actual.data = vif.sample.result_data[core]; - actual.error = vif.sample.result_error[core]; - scoreboard[core].actual_export.write(actual); - if (actual.error) errors++; - if (cmd.write && !actual.error) begin - coherence_ref_read(architectural, cmd.addr & ~64'h3f, line); - for (int b = 0; b < 8; b++) - if (cmd.mask[b]) line[(int'(cmd.addr&63)+b)*8+:8] = cmd.data[b*8+:8]; - coherence_ref_write(architectural, cmd.addr & ~64'h3f, line); - end - end - end - endfunction - function void observe_protocol(); - if (vif.sample.req_valid && $isunknown(vif.sample.req_bits)) - `uvm_fatal("REQ_X", "Unknown accepted RequestFlit") - if (vif.sample.rsp_valid && $isunknown(vif.sample.rsp_bits)) - `uvm_fatal("RSP_X", "Unknown accepted ResponseFlit") - if (vif.sample.dat_valid && $isunknown(vif.sample.dat_bits)) - `uvm_fatal("DAT_X", "Unknown accepted DataFlit") - if (vif.sample.snp_valid && $isunknown(vif.sample.snp_bits)) - `uvm_fatal("SNP_X", "Unknown accepted DirectedSnoop") - if (vif.sample.rx_rsp_valid && $isunknown(vif.sample.rx_rsp_bits)) - `uvm_fatal("RX_RSP_X", "Unknown requester response") - if (vif.sample.rx_dat_valid && $isunknown(vif.sample.rx_dat_bits)) - `uvm_fatal("RX_DAT_X", "Unknown requester data") - if (vif.sample.req_valid) begin - req_t r; - int key; - home_requests++; - r = vif.sample.req_bits; - key = (int'(r[`CSF(REQ, SRCID)]) << 1) | int'(r[`CSF(REQ, TXNID)]); - if (r[ - `CSF(REQ, TGTID) - ] != 64 || r[ - `CSF(REQ, SRCID) - ] < 1 || r[ - `CSF(REQ, SRCID) - ] > 2 || r[ - `CSF(REQ, TXNID) - ] >= 2) - `uvm_fatal("REQ_ID", $sformatf( - "Requester identity or bank TxnID corrupted src=%0d tgt=%0d txn=%0d packet=%h", - r[ - `CSF(REQ, SRCID) - ], - r[ - `CSF(REQ, TGTID) - ], - r[ - `CSF(REQ, TXNID) - ], - r - )) - requests[key] = r; - if (r[`CSF(REQ, OPCODE)] == `COH_READ_UNIQUE || r[`CSF(REQ, OPCODE)] == `COH_READ_NSD) begin - coherence_ref_read(architectural, r[`CSF(REQ, ADDR)], read_expected[key]); - fill_seen[key] = 0; - end - end - if (vif.sample.rsp_valid) begin - rsp_t r; - int key, id; - r = vif.sample.rsp_bits; - key = (int'(r[`CSF(RSP, TGTID)]) << 1) | int'(r[`CSF(RSP, TXNID)]); - id = r[`CSF(RSP, DBID)]; - if (!requests.exists(key) || r[`CSF(RSP, SRCID)] != 64 || r[`CSF(RSP, RESPERR)] != 0) - `uvm_fatal("RSP_ID", "Unexpected completion identity or error") - if (r[`CSF(RSP, OPCODE)] == `COH_COMP_DBID) begin - if (id >= 4 || requests[key][`CSF(REQ, OPCODE)] != `COH_WRITEBACK) - `uvm_fatal("WB_ID", "Writeback grant does not match request") - wb_address[id] = requests[key][`CSF(REQ, ADDR)]; - wb_owner[id] = r[`CSF(RSP, TGTID)]; - wb_seen[id] = 0; - end else if (r[`CSF(RSP, OPCODE)] != `COH_COMP) - `uvm_fatal("RETRY", "Canonical Home profile must backpressure REQ, not generate Retry") - end - if (vif.sample.dat_valid) begin - dat_t d; - int key, b, id; - bit failed; - bit [255:0] expected; - d = vif.sample.dat_bits; - key = (int'(d[`CSF(DAT, TGTID)]) << 1) | int'(d[`CSF(DAT, TXNID)]); - id = d[`CSF(DAT, DBID)]; - b = int'(d[`CSF(DAT, DATAID)]) >> 1; - failed = d[`CSF(DAT, RESPERR)] != 0; - if (!requests.exists( - key - ) || !read_expected.exists( - key - ) || id >= 4 || d[ - `CSF(DAT, SRCID) - ] != 64 || d[ - `CSF(DAT, HOMENID) - ] != 64 || d[ - `CSF(DAT, OPCODE) - ] != `COH_COMP_DATA) - `uvm_fatal("DAT_ID", "CompData bank TxnID, HomeNID or DBID mismatch") - if (d[`CSF(DAT, DATAID)] != 0 && d[`CSF(DAT, DATAID)] != 2) - `uvm_fatal("DATAID", "Invalid 256-bit DataID") - if (fill_seen[key] & (1 << b)) `uvm_fatal("DUPLICATE", "Duplicate CompData beat") - if (fill_seen[key] && fill_dbid[key] != id) - `uvm_fatal("DBID", "CompData DBID changed within line") - if (failed != (requests[key][`CSF(REQ, ADDR)] == ERROR_ADDRESS)) - `uvm_fatal("ERROR", "Unexpected backend error status") - expected = failed ? '0 : read_expected[key][b*256+:256]; - if (d[`CSF(DAT, DATA)] !== expected || d[`CSF(DAT, BE)] !== (failed ? 32'h0 : 32'hffffffff)) - `uvm_fatal("LINE_DATA", "Full 64-byte CompData payload/BE mismatch") - if (failed && d[`CSF(DAT, RESPERR)] != 3) `uvm_fatal("NDERR", "Read fault was not NDERR") - if (!fill_seen[key]) begin - fill_dbid[key] = id; - slot_owner[id] = key; - end - fill_seen[key] |= 1 << b; - if (id != d[`CSF(DAT, TXNID)]) saw_dbid_mismatch = 1; - if (fill_seen[key] == 3) begin - full_lines++; - ack_pending[id] = 1; - end - end - if (vif.sample.snp_valid) begin - int id, target; - bit [63:0] address; - id = vif.sample.snp_bits[`CSF(SNP, TXNID)]; - target = vif.sample.snp_bits[`CSF(SNP, TARGET)]; - address = 64'(vif.sample.snp_bits[`CSF(SNP, ADDR)]) << 3; - if (id >= 4 || target < 1 || target > 2 || vif.sample.snp_bits[`CSF(SNP, SRCID)] != 64) - `uvm_fatal("SNP_ID", "Directed snoop lost target/slot/address semantics") - coherence_ref_read(architectural, address, snoop_expected[id]); - snoop_seen[id] = 0; - snoop_target[id] = target; - if (id >= 2) saw_large_snoop = 1; - end - if (vif.sample.rx_rsp_valid) begin - rsp_t r; - int id; - r = vif.sample.rx_rsp_bits; - id = r[`CSF(RSP, TXNID)]; - if (r[`CSF(RSP, OPCODE)] == `COH_COMP_ACK) begin - if (!ack_pending.exists( - id - ) || !ack_pending[id] || slot_owner[id] >> 1 != r[ - `CSF(RSP, SRCID) - ] || r[ - `CSF(RSP, TGTID) - ] != 64) - `uvm_fatal("ACK_ID", "CompAck must return the Home DBID, not original bank TxnID") - ack_pending[id] = 0; - acks++; - end else begin - if (r[ - `CSF(RSP, OPCODE) - ] != `COH_SNP_RESP || !snoop_target.exists( - id - ) || r[ - `CSF(RSP, SRCID) - ] != snoop_target[id] || r[ - `CSF(RSP, TGTID) - ] != 64) - `uvm_fatal("RX_RSP", "Snoop response identity mismatch") - end - end - if (vif.sample.rx_dat_valid) begin - dat_t d; - int id, b; - bit [511:0] expected; - d = vif.sample.rx_dat_bits; - id = d[`CSF(DAT, TXNID)]; - b = int'(d[`CSF(DAT, DATAID)]) >> 1; - if (id >= 4 || (d[`CSF(DAT, DATAID)] != 0 && d[`CSF(DAT, DATAID)] != 2)) - `uvm_fatal("RETURN_ID", "Snoop/writeback must return full Home slot ID") - if (d[`CSF(DAT, OPCODE)] == `COH_SNP_DATA) begin - if (!snoop_expected.exists( - id - ) || d[ - `CSF(DAT, SRCID) - ] != snoop_target[id] || d[ - `CSF(DAT, TGTID) - ] != 64 || (snoop_seen[id] & (1 << b))) - `uvm_fatal("SNP_DATA", "Unmatched or duplicate snoop data") - expected = snoop_expected[id]; - snoop_seen[id] |= 1 << b; - if (snoop_seen[id] == 3) begin - snoop_lines++; - if (d[`CSF(DAT, RESP)] & 4) dirty_snoops++; - end - end else if (d[`CSF(DAT, OPCODE)] == `COH_COPYBACK_DATA) begin - if (!wb_address.exists( - id - ) || d[ - `CSF(DAT, SRCID) - ] != wb_owner[id] || d[ - `CSF(DAT, TGTID) - ] != 64 || (wb_seen[id] & (1 << b))) - `uvm_fatal("WB_DATA", "Unmatched or duplicate copyback") - coherence_ref_read(architectural, wb_address[id], expected); - wb_seen[id] |= 1 << b; - if (wb_seen[id] == 3) copybacks++; - if (d[`CSF(DAT, RESP)] == 0) expected = '0; - end else `uvm_fatal("RETURN_OPCODE", "Unexpected requester data opcode") - if (d[`CSF(DAT, DATA)] !== expected[b*256+:256]) - `uvm_fatal("RETURN_PAYLOAD", "Full-line snoop/copyback data mismatch") - end - endfunction - task service(); - forever begin - @(vif.sample); - if (!vif.sample.reset) begin - cycle++; - if (vif.sample.outstanding > peak) peak = vif.sample.outstanding; - observe_cpu(); - observe_protocol(); - for (int core = 0; core < 2; core++) - if (vif.sample.result_valid[core] && !vif.sample.result_ready[core]) - result_stalls[core]++; - if (vif.sample.block_requester_rsp) begin - rsp_block_cycles++; - if (!vif.sample.active || vif.sample.rx_rsp_valid) - `uvm_fatal("RSP_BLOCK", "RSP stall changed active or allowed a blocked response") - end - if (release_cycle >= 0 && ack_resume_cycles < 0 && acks >= 4) - ack_resume_cycles = cycle - release_cycle; - if (mem_active && mem_resp.sample.valid && mem_resp.sample.ready) mem_active = 0; - if (mem_req.sample.valid && mem_req.sample.ready) begin - memory_entry entry; - bit [511:0] line, expected; - bit write; - longint unsigned addr; - if ($isunknown( - { - mem_req.sample.bits[`CSF(MEMREQ, ID)], - mem_req.sample.bits[`CSF(MEMREQ, ADDR)], - mem_req.sample.bits[`CSF(MEMREQ, WRITE)] - } - )) - `uvm_fatal("MEM_X", "Unknown backing command") - entry.bits = '0; - entry.bits[`CSF(MEMRESP, ID)] = mem_req.sample.bits[`CSF(MEMREQ, ID)]; - addr = mem_req.sample.bits[`CSF(MEMREQ, ADDR)]; - write = mem_req.sample.bits[`CSF(MEMREQ, WRITE)]; - if (addr[5:0] != 0) `uvm_fatal("MEM_ALIGN", "Backing request not line aligned") - foreach (pending[i]) - if (pending[i].bits[`CSF(MEMRESP, ID)] == entry.bits[`CSF(MEMRESP, ID)]) - `uvm_fatal("MEM_ID", "Reused live backing ID") - if (write) begin - if ($isunknown( - { - mem_req.sample.bits[`CSF(MEMREQ, DATA)], - mem_req.sample.bits[`CSF(MEMREQ, MASK)] - } - )) - `uvm_fatal("MEM_WRITE_X", "Unknown backing write payload") - line = mem_req.sample.bits[`CSF(MEMREQ, DATA)]; - coherence_ref_read(architectural, addr, expected); - if (mem_req.sample.bits[`CSF(MEMREQ, MASK)] !== '1 || line !== expected) - `uvm_fatal("MEM_WRITE", "Backing write differs from architectural full line") - coherence_ref_write(backing, addr, line); - memory_writes++; - end else begin - coherence_ref_read(backing, addr, line); - memory_reads++; - end - entry.bits[`CSF(MEMRESP, DATA)] = line; - entry.bits[`CSF(MEMRESP, ERROR)] = !write && addr == ERROR_ADDRESS; - entry.due = cycle + 9 + (3 - int'(entry.bits[`CSF(MEMRESP, ID)])) * 3; - entry.serial = memory_serial++; - pending.push_back(entry); - end - end - @(negedge vif.clock); - mem_req.ready = !vif.reset && cycle % 7 != 0; - for (int core = 0; core < 2; core++) - vif.result_ready[core] = !vif.reset && !hold_results && (cycle + core) % 11 >= 3; - if (!mem_active && !hold_memory) begin - for (int i = pending.size() - 1; i >= 0; i--) - if (pending[i].due <= cycle) begin - memory_entry entry; - entry = pending[i]; - pending.delete(i); - mem_packet = entry.bits; - mem_active = 1; - if (entry.serial < last_memory_serial) saw_reorder = 1; - last_memory_serial = entry.serial; - break; - end - end - mem_resp.valid = mem_active; - mem_resp.bits = mem_packet; - end - endtask - task send(int core, longint unsigned address, bit write = 0, bit [63:0] data = 0, - bit [7:0] mask = 8'hff); - @(negedge vif.clock); - vif.access_addr[core] = address; - vif.access_write[core] = write; - vif.access_data[core] = data; - vif.access_mask[core] = mask; - vif.access_atomic[core] = 0; - vif.access_atomicWord[core] = 0; - vif.access_valid[core] = 1; - do @(vif.sample); while (!vif.sample.access_ready[core]); - @(negedge vif.clock); - vif.access_valid[core] = 0; - endtask - task settle(); - for (int core = 0; core < 2; core++) scoreboard[core].wait_checked(submitted[core]); - do @(vif.sample); while (vif.sample.outstanding != 0 || pending.size() != 0 || mem_active); - repeat (8) @(vif.sample); - endtask - task execute(); - vif.reset = 1; - vif.active = 0; - vif.block_requester_rsp = 0; - mem_req.ready = 0; - mem_resp.valid = 0; - mem_resp.bits = '0; - for (int i = 0; i < 2; i++) begin - vif.access_valid[i] = 0; - vif.access_addr[i] = 0; - vif.access_write[i] = 0; - vif.access_data[i] = 0; - vif.access_mask[i] = '1; - vif.access_atomic[i] = 0; - vif.access_atomicWord[i] = 0; - vif.result_ready[i] = 0; - end - repeat (5) @(negedge vif.clock); - vif.reset = 0; - vif.active = 1; - fork - service(); - join_none - repeat (12) @(vif.sample); - // Completion ACKs remain in the physical Rx FIFOs while DAT and CPU result - // channels make progress. Hold CPU results first, then issue more REQs while - // all four Home slots still wait for their ACKs. - hold_memory = 1; - hold_results = 1; - @(negedge vif.clock); - vif.block_requester_rsp = 1; - fork - begin - send(0, 'h0); - send(0, 'h40); - end - begin - send(1, 'h80); - send(1, 'hc0); - end - join - wait (pending.size() == 4); - if (peak != 4) - `uvm_fatal("CONCURRENCY", "Four independent misses did not occupy all Home MSHRs") - hold_memory = 0; - wait (vif.sample.result_valid[0] && vif.sample.result_valid[1]); - repeat (24) begin - @(vif.sample); - if(vif.sample.outstanding!=4||acks!=0||scoreboard[0].checked!=0||scoreboard[1].checked!=0) - `uvm_fatal("ACK_LIFETIME", "Blocked ACK or CPU result prematurely retired") - end - hold_results = 0; - scoreboard[0].wait_checked(2); - scoreboard[1].wait_checked(2); - repeat (24) begin - @(vif.sample); - if (vif.sample.outstanding != 4 || acks != 0 || full_lines != 4) - `uvm_fatal("ACK_LIFETIME", "Home released a slot before receiving CompAck") - end - fork - send(0, 'h800); - send(1, 'h880); - join - repeat (16) begin - @(vif.sample); - if(vif.sample.outstanding!=4||home_requests!=4||scoreboard[0].checked!=2||scoreboard[1].checked!=2) - `uvm_fatal("REQ_PRESSURE", "New request advanced while all slots awaited blocked ACKs") - end - @(negedge vif.clock); - release_cycle = cycle; - vif.block_requester_rsp = 0; - begin - bit drained = 0; - for (int elapsed = 0; elapsed < 256; elapsed++) begin - @(vif.sample); - if(scoreboard[0].checked==submitted[0]&&scoreboard[1].checked==submitted[1]&& - vif.sample.outstanding==0&&pending.size()==0&&!mem_active)begin - drained = 1; - break; - end - end - if(!drained||ack_resume_cycles<0||ack_resume_cycles>32||result_stalls[0]<20||result_stalls[1]<20) - `uvm_fatal("ACK_PROGRESS", - "CompAck/blocked REQ/result channels did not drain within the progress bound") - end - settle(); - `uvm_info( - "BACKPRESSURE", - $sformatf( - "4 MSHRs held for CompAck with live credit links; CPU stall cycles=%0d/%0d RSP blocked=%0d first4ACK resume=%0d cycles; pending REQs and all results drained", - result_stalls[0], result_stalls[1], rsp_block_cycles, ack_resume_cycles), UVM_LOW) - // Prime two RN1 lines, then keep slots0/1 waiting for unrelated backing reads. - send(0, 'h80); - settle(); - send(0, 'hc0); - settle(); - hold_memory = 1; - send(0, 'h400); - send(0, 'h440); - wait (pending.size() == 2); - send(1, 'h80, 1, 64'h0123456789abcdef); - send(1, 'hc0, 1, 64'h8877665544332211); - wait (scoreboard[1].checked == submitted[1]); - if (!saw_large_snoop) - `uvm_fatal("SNP_SLOT", "No snoop used a Home slot beyond RN bank-ID range") - hold_memory = 0; - settle(); - send(0, 'h80); - settle(); // Dirty owner at RN2 supplies the newest data and downgrades. - send(0, 'h80, 1, 64'hfedcba9876543210); - settle(); - send(0, 'h88, 1, 64'hdecafbad76543210, 8'h3c); - settle(); - send(1, 'h80); - settle(); - send(0, 'h80, 1, 64'h445566778899aabb); - settle(); - send(0, 'h280); - settle(); // Same private index: CopyBackWriteData into shared L2. - send(1, 'h480); - settle(); - send(1, 'h680); - settle(); // L2 same-set replacement flushes dirty data. - send(1, 'h80); - settle(); - send(0, ERROR_ADDRESS); - settle(); - if(!saw_dbid_mismatch||!saw_reorder||!dirty_snoops||!copybacks||!memory_writes||errors!=1||acks!=full_lines) - `uvm_fatal("SCENARIOS", $sformatf( - "dbid=%0b reorder=%0b dirty=%0d copybacks=%0d writes=%0d errors=%0d ack/lines=%0d/%0d", - saw_dbid_mismatch, - saw_reorder, - dirty_snoops, - copybacks, - memory_writes, - errors, - acks, - full_lines - )) - foreach (ack_pending[id]) - if (ack_pending[id]) `uvm_fatal("PENDING_ACK", "Unmatched completion acknowledgement") - for (int core = 0; core < 2; core++) - if (commands[core].size()) `uvm_fatal("PENDING_CPU", "Unmatched accepted CPU command") - `uvm_info( - "CHECKS", - $sformatf( - "canonical 2RN-F x2banks + shared L2: CPU checked=%0d/%0d full64B=%0d snoopLines=%0d dirtySnoops=%0d copybacks=%0d memoryWrites=%0d peakMSHR=%0d NDERR=%0d", - scoreboard[0].checked, scoreboard[1].checked, full_lines, snoop_lines, dirty_snoops, - copybacks, memory_writes, peak, errors), UVM_LOW) - endtask - function void final_phase(uvm_phase phase); - coherence_ref_destroy(architectural); - coherence_ref_destroy(backing); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_tb.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_tb.sv deleted file mode 100644 index 7e16d4ad2..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/cache_system_tb.sv +++ /dev/null @@ -1,32 +0,0 @@ -`include "cache_system_config.svh" -module cache_system_tb; - import uvm_pkg::*; - import cache_system_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - cache_system_if control (clock); - stream_if #(`COH_MEMREQ_WIDTH) mem_req ( - clock, - control.reset - ); - stream_if #(`COH_MEMRESP_WIDTH) mem_resp ( - clock, - control.reset - ); - CacheSystem dut ( - .clock(clock), - .reset(control.reset), - .io_active(control.active), - .io_blockRequesterRsp(control.block_requester_rsp), - .io_outstanding(control.outstanding), - `include "cache_system_ports.svh" - ); - initial begin - uvm_config_db#(virtual cache_system_if)::set(null, "uvm_test_top", "control", control); - uvm_config_db#(virtual stream_if #(`COH_MEMREQ_WIDTH))::set(null, "uvm_test_top", "mem_req", - mem_req); - uvm_config_db#(virtual stream_if #(`COH_MEMRESP_WIDTH))::set(null, "uvm_test_top", "mem_resp", - mem_resp); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency.cm_hier b/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency.cm_hier deleted file mode 100644 index 140ec3277..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency.cm_hier +++ /dev/null @@ -1,12 +0,0 @@ -+tree @TOP@.dut -+tree @TOP@.access_0 -+tree @TOP@.access_1 -+tree @TOP@.result_0 -+tree @TOP@.result_1 -+tree @TOP@.dispatch -+tree @TOP@.access_info -+tree @TOP@.grant -+tree @TOP@.done -+tree @TOP@.complete -+tree @TOP@.mem_req -+tree @TOP@.mem_resp diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency.f deleted file mode 100644 index ea556227a..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/consistency_control.sv -@RESOURCES@/consistency_package.sv --F @RTL@/Consistency/filelist.f -@RESOURCES@/consistency_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_control.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_control.sv deleted file mode 100644 index a82d901ab..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_control.sv +++ /dev/null @@ -1,13 +0,0 @@ -`include "consistency_config.svh" -interface consistency_control_if ( - input logic clock -); - logic reset; - logic cpu_allow, older_dispatch_pending, older_requests_drained; - logic block_requester_rsp, block_requester_data; - logic [`CONS_OUTSTANDING_BITS-1:0] outstanding; - logic req_valid, snp_valid; - logic [`CONS_REQ_WIDTH-1:0] req_bits; - logic [`CONS_SNP_CHANNEL_WIDTH-1:0] snp_bits; - bit start = 0, finished = 0; -endinterface diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_package.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_package.sv deleted file mode 100644 index b486df7f6..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_package.sv +++ /dev/null @@ -1,66 +0,0 @@ -package consistency_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function chandle cons_ref_create(); - import "DPI-C" function void cons_ref_destroy(input chandle model); - import "DPI-C" function longint unsigned cons_ref_expected64( - input chandle model, - input longint unsigned address - ); - import "DPI-C" function void cons_ref_cpu_commit( - input chandle model, - input longint unsigned address, - data, - input int unsigned mask - ); - import "DPI-C" function void cons_ref_ddr_read( - input chandle model, - input longint unsigned address, - output bit [511:0] data - ); - import "DPI-C" function int unsigned cons_ref_write_matches( - input chandle model, - input longint unsigned address, - input bit [511:0] data, - input bit [63:0] mask - ); - import "DPI-C" function void cons_ref_ddr_commit( - input chandle model, - input longint unsigned address, - input bit [511:0] data, - input bit [63:0] mask - ); - import "DPI-C" function int unsigned cons_ref_ddr_visible( - input chandle model, - input longint unsigned address - ); - import "DPI-C" function void cons_ref_dma_write( - input chandle model, - input longint unsigned address, - data, - input int unsigned mask - ); - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual consistency_control_if control; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 1ms; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual consistency_control_if)::get(this, "", "control", control)) - `uvm_fatal("VIF", "missing consistency control") - endfunction - task execute(); - @(negedge control.clock); - control.start = 1; - wait (control.finished); - `uvm_info( - "CONSISTENCY_PASS", - "Real L1/CMO/Home DDR data, partial DMA bytes, delayed ACK/old fill and CPU permit checks completed", - UVM_LOW) - endtask - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_tb.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_tb.sv deleted file mode 100644 index 2bfad093a..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/consistency_tb.sv +++ /dev/null @@ -1,564 +0,0 @@ -module consistency_tb; - import uvm_pkg::*; - import consistency_pkg::*; - `include "uvm_macros.svh" - `include "consistency_config.svh" - `define CF(B, K, F) B[`CONS_``K``_``F``_OFFSET +: `CONS_``K``_``F``_WIDTH] - logic clock = 0; - always #5 clock = ~clock; - consistency_control_if control (clock); - stream_if #(.WIDTH(`CONS_CPU_WIDTH)) - access_0 ( - clock, - control.reset - ), - access_1 ( - clock, - control.reset - ); - stream_if #(.WIDTH(`CONS_RESULT_WIDTH)) - result_0 ( - clock, - control.reset - ), - result_1 ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`CONS_DISPATCH_WIDTH) - ) dispatch ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`CONS_INFO_WIDTH) - ) access_info ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`CONS_GRANT_WIDTH) - ) grant ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`CONS_DONE_WIDTH) - ) done ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`CONS_COMPLETE_WIDTH) - ) complete ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`CONS_MEMREQ_WIDTH) - ) mem_req ( - clock, - control.reset - ); - stream_if #( - .WIDTH(`CONS_MEMRESP_WIDTH) - ) mem_resp ( - clock, - control.reset - ); - Consistency dut ( - .clock(clock), - .reset(control.reset), - `include "consistency_ports.svh" - ); - localparam longint unsigned A='h4000,B='h4040,P='h10000,X='hC000,Y='h8040,E='h20000,R='h30000+3*`CONS_LINE_BYTES; - localparam longint unsigned EVICT = E + `CONS_L1_LINES * `CONS_LINE_BYTES; - typedef struct { - longint unsigned addr, data, expected; - int unsigned mask; - bit write; - } cpu_entry; - typedef struct { - int unsigned id; - longint unsigned addr; - bit write; - bit [511:0] data; - bit [63:0] mask; - int due; - } memory_entry; - cpu_entry cpu_pending[2][$]; - memory_entry memory_pending[$], active_memory; - bit memory_active = 0, memory_live[`CONS_MSHRS]; - chandle model; - bit model_ready = 0, hold_write = 0, hold_read = 0, old_fill_seen = 0; - longint unsigned hold_write_addr, hold_read_addr; - int cycle = 0, checked[2], submitted[2], grants[256], completions[256]; - int memory_reads = 0, memory_writes = 0, memory_acks = 0, peak = 0, cmo_count[3], peer_snoops = 0; - function automatic void check(bit condition, string message); - if (!condition) `uvm_fatal("CONSISTENCY", message) - endfunction - function automatic void observe_cpu(int core, bit valid, ready, - logic [`CONS_CPU_WIDTH-1:0] request, bit response_valid, - response_ready, logic [`CONS_RESULT_WIDTH-1:0] response); - if (valid && ready) begin - cpu_entry item; - check(!$isunknown(request), "unknown CPU request"); - check(`CF(request, CPU, ATOMIC) == 0 && !`CF(request, CPU, ATOMICWORD), - "physical consistency profile uses ordinary 64-bit CPU accesses"); - item.addr = `CF(request, CPU, ADDR); - item.data = `CF(request, CPU, DATA); - item.mask = `CF(request, CPU, MASK); - item.write = `CF(request, CPU, WRITE); - item.expected = cons_ref_expected64(model, item.addr); - cpu_pending[core].push_back(item); - submitted[core]++; - `uvm_info("CPU_ACCEPT", $sformatf("core%0d addr=%h write%0d", core, item.addr, item.write), - UVM_LOW) - end - if (response_valid && response_ready) begin - cpu_entry item; - check(cpu_pending[core].size() != 0, "CPU response without accepted access"); - item = cpu_pending[core].pop_front(); - check(!$isunknown(response) && !`CF(response, RESULT, ERROR), "CPU memory access failed"); - check(`CF(response, RESULT, DATA) === item.expected, $sformatf( - "core%0d addr=%h result=%h expected=%h", - core, - item.addr, - `CF(response, RESULT, DATA), - item.expected - )); - if (item.write) cons_ref_cpu_commit(model, item.addr, item.data, item.mask); - checked[core]++; - `uvm_info("CPU_RESULT", $sformatf("core%0d addr=%h data=%h", core, item.addr, - `CF(response, RESULT, DATA)), UVM_LOW) - end - endfunction - always @(posedge clock) begin - if (model_ready && !control.reset) begin - cycle++; - if (control.outstanding > peak) peak = control.outstanding; - observe_cpu(0, access_0.valid, access_0.ready, access_0.bits, result_0.valid, result_0.ready, - result_0.bits); - observe_cpu(1, access_1.valid, access_1.ready, access_1.bits, result_1.valid, result_1.ready, - result_1.bits); - if (dispatch.valid && dispatch.ready) begin - int id; - id = `CF(dispatch.bits, DISPATCH, ID); - grants[id] = 0; - completions[id] = 0; - end - if (grant.valid && grant.ready) begin - grants[`CF(grant.bits, GRANT, TAG)]++; - `uvm_info("NPU_GRANT", $sformatf("tag%0d", `CF(grant.bits, GRANT, TAG)), UVM_LOW) - end - if (complete.valid && complete.ready) begin - completions[`CF(complete.bits, COMPLETE, TAG)]++; - `uvm_info("NPU_COMPLETE", $sformatf("tag%0d", `CF(complete.bits, COMPLETE, TAG)), UVM_LOW) - end - if (control.req_valid) - `uvm_info("HOME_REQUEST", $sformatf( - "node%0d txn%0d op%h addr=%h", - `CF(control.req_bits, REQ, SRCID), - `CF(control.req_bits, REQ, TXNID), - `CF(control.req_bits, REQ, OPCODE), - `CF(control.req_bits, REQ, ADDR) - ), UVM_LOW) - if (control.snp_valid) - `uvm_info("HOME_SNOOP", $sformatf( - "node%0d op%h addr=%h", - control.snp_bits[`CONS_SNP_TARGET_OFFSET+:`CONS_SNP_TARGET_WIDTH], - `CF(control.snp_bits, SNP, OPCODE), - 64'( - `CF(control.snp_bits, SNP, ADDR) - ) << 3 - ), UVM_LOW) - if (control.req_valid && - `CF(control.req_bits, REQ, SRCID) - == 1 && - `CF(control.req_bits, REQ, TXNID) - == 2) begin - int opcode; - opcode = `CF(control.req_bits, REQ, OPCODE); - check(opcode inside {'h08, 'h09, 'h0a}, "maintenance requester emitted a non-CMO opcode"); - check(`CF(control.req_bits, REQ, TGTID) == `CONS_HOME, "CMO routed to wrong Home"); - cmo_count[opcode-'h08]++; - `uvm_info("CMO_REQUEST", $sformatf("op%h addr=%h", opcode, `CF(control.req_bits, REQ, - ADDR)), UVM_LOW) - end - if(control.snp_valid&&control.snp_bits[`CONS_SNP_TARGET_OFFSET+:`CONS_SNP_TARGET_WIDTH]==2) - peer_snoops++; - if (mem_resp.valid && mem_resp.ready) begin - check(memory_active && `CF(mem_resp.bits, MEMRESP, ID) == active_memory.id, - "memory response identity mismatch"); - if (active_memory.write) - cons_ref_ddr_commit(model, active_memory.addr, active_memory.data, active_memory.mask); - memory_live[active_memory.id] = 0; - memory_active = 0; - memory_acks++; - `uvm_info("DDR_ACK", $sformatf("id%0d addr=%h write%0d", active_memory.id, - active_memory.addr, active_memory.write), UVM_LOW) - end - if (mem_req.valid && mem_req.ready) begin - memory_entry item; - check(!$isunknown(mem_req.bits), "unknown real Home memory request"); - item.id = `CF(mem_req.bits, MEMREQ, ID); - item.addr = `CF(mem_req.bits, MEMREQ, ADDR); - item.write = `CF(mem_req.bits, MEMREQ, WRITE); - item.data = `CF(mem_req.bits, MEMREQ, DATA); - item.mask = `CF(mem_req.bits, MEMREQ, MASK); - check(item.id < `CONS_MSHRS && !memory_live[item.id], "Home reused a live DDR ID"); - check((item.addr & 63) == 0, "Home DDR request not a 64-byte line"); - memory_live[item.id] = 1; - if (item.write) begin - check(item.mask === '1 && cons_ref_write_matches(model, item.addr, item.data, item.mask - ) == 1, "Home writeback lost CPU-visible dirty bytes"); - memory_writes++; - end else begin - cons_ref_ddr_read(model, item.addr, item.data); - memory_reads++; - if (item.addr == X) old_fill_seen = 1; - end - item.due = cycle + 7 + (3 - item.id) * 3; - memory_pending.push_back(item); - `uvm_info("DDR_REQUEST", $sformatf("id%0d addr=%h write%0d", item.id, item.addr, - item.write), UVM_LOW) - end - end - end - // DDR BFM: reads and write ACKs can reorder across distinct real Home MSHR IDs. - always @(negedge clock) begin - if (model_ready && !control.reset) begin - mem_req.ready = (cycle % 7) != 0; - if (!memory_active) begin - mem_resp.valid = 0; - for (int i = memory_pending.size() - 1; i >= 0; i--) begin - if(!memory_active&&memory_pending[i].due<=cycle&& - !(hold_write&&memory_pending[i].write&&memory_pending[i].addr==hold_write_addr)&& - !(hold_read&&!memory_pending[i].write&&memory_pending[i].addr==hold_read_addr)) begin - active_memory = memory_pending[i]; - memory_pending.delete(i); - memory_active = 1; - mem_resp.bits = '0; - `CF(mem_resp.bits, MEMRESP, ID) = active_memory.id; - `CF(mem_resp.bits, MEMRESP, DATA) = active_memory.write ? '0 : active_memory.data; - mem_resp.valid = 1; - end - end - end - end - end - task automatic cpu_request(int core, longint unsigned address, bit write = 0, - longint unsigned data = 0, int unsigned mask = 255); - logic [`CONS_CPU_WIDTH-1:0] packet; - packet = '0; - `CF(packet, CPU, ADDR) = address; - `CF(packet, CPU, WRITE) = write; - `CF(packet, CPU, DATA) = data; - `CF(packet, CPU, MASK) = write ? mask : 0; - @(negedge clock); - if (core == 0) begin - access_0.bits = packet; - access_0.valid = 1; - do @(posedge clock); while (!access_0.ready); - @(negedge clock); - access_0.valid = 0; - end else begin - access_1.bits = packet; - access_1.valid = 1; - do @(posedge clock); while (!access_1.ready); - @(negedge clock); - access_1.valid = 0; - end - endtask - task automatic cpu_access(int core, longint unsigned address, bit write = 0, - longint unsigned data = 0, int unsigned mask = 255); - int goal; - goal = checked[core] + 1; - cpu_request(core, address, write, data, mask); - wait (checked[core] == goal); - @(negedge clock); - endtask - task automatic reserve_access(int tag, longint unsigned base, bytes, bit write); - @(negedge clock); - dispatch.bits = '0; - `CF(dispatch.bits, DISPATCH, ID) = tag; - dispatch.valid = 1; - do @(posedge clock); while (!dispatch.ready); - @(negedge clock); - dispatch.valid = 0; - access_info.bits = '0; - `CF(access_info.bits, INFO, ID) = tag; - `CF(access_info.bits, INFO, HASMEMORY) = 1; - `CF(access_info.bits, INFO, BASE) = base; - `CF(access_info.bits, INFO, BYTES) = bytes; - `CF(access_info.bits, INFO, WRITE) = write; - `CF(access_info.bits, INFO, LAST) = 1; - access_info.valid = 1; - do @(posedge clock); while (!access_info.ready); - @(negedge clock); - access_info.valid = 0; - endtask - task automatic dma_done(int tag); - @(negedge clock); - done.bits = '0; - `CF(done.bits, DONE, TAG) = tag; - `CF(done.bits, DONE, OK) = 1; - done.valid = 1; - do @(posedge clock); while (!done.ready); - @(negedge clock); - done.valid = 0; - done.bits = '0; - endtask - task automatic await_complete(int tag); - wait (completions[tag] == 1); - @(negedge clock); - endtask - task automatic block_cpu_read(longint unsigned address, int clocks = 32); - @(negedge clock); - access_0.bits = '0; - `CF(access_0.bits, CPU, ADDR) = address; - access_0.valid = 1; - repeat (clocks) begin - @(posedge clock); - #1; - check(!control.cpu_allow && !access_0.ready, - "CPU target read passed before DMA ACK/post-invalidate"); - end - endtask - task automatic release_cpu_read(); - int goal; - goal = checked[0] + 1; - do @(posedge clock); while (!access_0.ready); - @(negedge clock); - access_0.valid = 0; - wait (checked[0] == goal); - @(negedge clock); - endtask - initial begin - uvm_config_db#(virtual consistency_control_if)::set(null, "*", "control", control); - run_test("protocol_test"); - end - initial begin - control.reset = 1; - control.older_dispatch_pending = 0; - control.older_requests_drained = 1; - control.block_requester_rsp = 0; - control.block_requester_data = 0; - access_0.valid = 0; - access_0.bits = '0; - access_1.valid = 0; - access_1.bits = '0; - result_0.ready = 1; - result_1.ready = 1; - dispatch.valid = 0; - dispatch.bits = '0; - access_info.valid = 0; - access_info.bits = '0; - grant.ready = 1; - done.valid = 0; - done.bits = '0; - complete.ready = 1; - mem_req.ready = 0; - mem_resp.valid = 0; - mem_resp.bits = '0; - foreach (memory_live[i]) memory_live[i] = 0; - foreach (checked[i]) begin - checked[i] = 0; - submitted[i] = 0; - end - foreach (grants[i]) begin - grants[i] = 0; - completions[i] = 0; - end - foreach (cmo_count[i]) cmo_count[i] = 0; - wait (control.start); - check( - `CONS_AGENTS==2&&`CONS_SLOTS==4&&`CONS_PA_BITS==44&&`CONS_LINE_BYTES==64&&`CONS_L1_LINES==8&&`CONS_L2_SETS==16, - "unexpected frozen physical profile"); - model = cons_ref_create(); - model_ready = 1; - repeat (4) @(posedge clock); - @(negedge clock); - control.reset = 0; - // CPU publication hint stalls only until the older unregistered dispatch is recorded. - @(negedge clock); - control.older_dispatch_pending = 1; - access_0.bits = '0; - `CF(access_0.bits, CPU, ADDR) = B; - access_0.valid = 1; - repeat (8) begin - @(posedge clock); - #1; - check(!control.cpu_allow && !access_0.ready, "older dispatch hint did not block CPU"); - end - @(negedge clock); - control.older_dispatch_pending = 0; - release_cpu_read(); - // Real dirty L1 data must reach DDR before a mock NPU read is granted. - cpu_access(0, A, 1, 64'h1122334455667788); - check(cons_ref_ddr_visible(model, A) == 0, "CPU dirty store incorrectly updated DDR golden"); - hold_write = 1; - hold_write_addr = A; - reserve_access(10, A, 64, 0); - wait (memory_writes > 0 || grants[10] > 0); - check(memory_writes > 0, "dirty pre-Clean granted without a real DDR writeback"); - repeat (32) begin - @(posedge clock); - #1; - check(grants[10] == 0 && completions[10] == 0, "NPU read granted before real DDR write ACK"); - end - cpu_access(1, B); // Independent requester/range progresses while dirty write ACK is held. - @(negedge clock); - hold_write = 0; - wait (grants[10] == 1); - check(cons_ref_ddr_visible(model, A) == 1, "pre-Clean did not publish dirty CPU bytes to DDR"); - cpu_access(0, A); // R/R overlap completes while NPU read has not sent done. - check(completions[10] == 0, "NPU read unexpectedly completed without done"); - dma_done(10); - await_complete(10); - // Preserve dirty neighbour bytes around a partial NPU write and invalidate all real holders. - cpu_access(0, P, 1, 64'h0102030405060708); - cpu_access(0, P + 16, 1, 64'h9192939495969798); - cpu_access(1, P); - cpu_access(1, P + 16); - control.older_requests_drained = 0; - reserve_access(20, P, 64, 1); - begin - int prior_cmo_count; - prior_cmo_count = cmo_count[1]; - repeat (12) begin - @(posedge clock); - #1; - check(grants[20] == 0 && cmo_count[1] == prior_cmo_count, - "maintenance admitted before external old requests drained"); - end - end - @(negedge clock); - control.older_requests_drained = 1; - wait (grants[20] == 1); - check(cons_ref_ddr_visible(model, P) == 1, "pre-CleanInvalidate lost dirty neighbour bytes"); - cons_ref_dma_write(model, P + 8, 64'hfeedface89abcdef, 15); - block_cpu_read(P + 8, 32); - check(completions[20] == 0, "write completed before DMA done ACK"); - @(negedge clock); - complete.ready = 0; - dma_done(20); - wait (complete.valid); - repeat (8) begin - @(posedge clock); - #1; - check(!control.cpu_allow && !access_0.ready && complete.valid && - `CF(complete.bits, COMPLETE, TAG) == 20, - "CPU read bypassed stalled final range completion"); - end - @(negedge clock); - complete.ready = 1; - await_complete(20); - release_cpu_read(); - for (int word = 0; word < 8; word++) begin - cpu_access(0, P + word * 8); - cpu_access(1, P + word * 8); - end - // An old dirty CopyBack can leave L1 yet remain in the explicit transport FIFO. - cpu_access(0, E, 1, 64'h12345678badc0ffe); - @(negedge clock); - control.block_requester_data = 1; - cpu_access(0, EVICT); // Same L1 slot, different real Home set. - check(cons_ref_ddr_visible(model, E) == 0, - "old dirty CopyBack reached DDR despite transport stall"); - reserve_access(40, E, 64, 0); - repeat (32) begin - @(posedge clock); - #1; - check(grants[40] == 0 && completions[40] == 0, "CMO granted ahead of old delayed CopyBack"); - end - @(negedge clock); - control.block_requester_data = 0; - wait (grants[40] == 1); - check(cons_ref_ddr_visible(model, E) == 1, "old CopyBack did not publish DDR before grant"); - dma_done(40); - await_complete(40); - // A CompAck accepted by L1 can still be queued at the real Home RX credit boundary. - @(negedge clock); - control.block_requester_rsp = 1; - cpu_access(0, R); - reserve_access(50, R, 64, 0); - repeat (24) begin - @(posedge clock); - #1; - check(grants[50] == 0 && completions[50] == 0, "CMO overtook old undelivered CompAck"); - end - @(negedge clock); - control.block_requester_rsp = 0; - wait (grants[50] == 1); - check(cons_ref_ddr_visible(model, R) == 1, "CompAck drain changed read-only DDR data"); - dma_done(50); - await_complete(50); - // Delay an actual older CPU miss/fill and its result, then admit maintenance only after both drain. - hold_read = 1; - hold_read_addr = X; - result_0.ready = 0; - control.older_requests_drained = 0; - cpu_request(0, X); - wait (old_fill_seen); - reserve_access(30, X, 64, 1); - cpu_access(1, Y); - repeat (16) begin - @(posedge clock); - #1; - check(grants[30] == 0, "maintenance overtook an older delayed fill"); - end - @(negedge clock); - hold_read = 0; - wait (result_0.valid); - repeat (8) begin - @(posedge clock); - #1; - check(grants[30] == 0, "maintenance overtook unaccepted older CPU result"); - end - @(negedge clock); - result_0.ready = 1; - wait (checked[0] == submitted[0]); - repeat (8) begin - @(posedge clock); - #1; - check(grants[30] == 0, "maintenance ignored olderRequestsDrained=0 after fill"); - end - @(negedge clock); - control.older_requests_drained = 1; - wait (grants[30] == 1); - check(cons_ref_ddr_visible(model, X) == 1, "old-fill pre-maintenance altered DDR bytes"); - cons_ref_dma_write(model, X, 64'hd0d1d2d3d4d5d6d7, 255); - block_cpu_read(X, 32); - dma_done(30); - await_complete(30); - release_cpu_read(); - cpu_access(1, X); - wait (control.outstanding == 0 && !memory_active && memory_pending.size() == 0); - repeat (8) @(posedge clock); - check( - checked[0]==submitted[0]&&checked[1]==submitted[1]&&cpu_pending[0].size()==0&&cpu_pending[1].size()==0, - "unmatched CPU accesses remain"); - check( - memory_writes>0&&cmo_count[0]>0&&cmo_count[1]>=2&&cmo_count[2]>=2&&peer_snoops>0, - "required real CMO/writeback/holder snoops were absent"); - check(peak >= 2, "independent old-fill/peer traffic did not exercise multiple Home MSHRs"); - `uvm_info("CHECKED", $sformatf( - "CPU%0d/%0d DDRread%0d/write%0d/ACK%0d CMO%0d/%0d/%0d peerSnp%0d peak%0d", - checked[0], - checked[1], - memory_reads, - memory_writes, - memory_acks, - cmo_count[0], - cmo_count[1], - cmo_count[2], - peer_snoops, - peak - ), UVM_LOW) - cons_ref_destroy(model); - model_ready = 0; - control.finished = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile.f deleted file mode 100644 index 3c3a49006..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/tile_interface.sv -@RESOURCES@/tile_package.sv --F @RTL@/Tile/filelist.f -@RESOURCES@/tile_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_interface.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_interface.sv deleted file mode 100644 index b22250e83..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_interface.sv +++ /dev/null @@ -1,20 +0,0 @@ -`include "cache_system_config.svh" -interface tile_if ( - input logic clock -); - logic reset; - logic retired[2], trapped[2]; - logic [63:0] retiredPc[2], trapCause[2], trapValue[2], trapPc[2]; - logic [2:0] outstanding; - logic req_valid, rsp_valid, dat_valid, snp_valid, rx_rsp_valid, rx_dat_valid; - logic [`COH_REQ_WIDTH-1:0] req_bits; - logic [`COH_RSP_WIDTH-1:0] rsp_bits, rx_rsp_bits; - logic [`COH_DAT_WIDTH-1:0] dat_bits, rx_dat_bits; - logic [`COH_SNP_CHANNEL_WIDTH-1:0] snp_bits; - clocking sample @(posedge clock); - default input #1step; - input reset, retired, retiredPc, trapped, trapCause, trapValue, trapPc, outstanding; - input req_valid, rsp_valid, dat_valid, snp_valid, rx_rsp_valid, rx_dat_valid; - input req_bits, rsp_bits, dat_bits, snp_bits, rx_rsp_bits, rx_dat_bits; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_package.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_package.sv deleted file mode 100644 index bb4f142b8..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_package.sv +++ /dev/null @@ -1,415 +0,0 @@ -`include "tile_config.svh" -`include "cache_system_config.svh" -`include "multicore_fixture.svh" -`define TF(K, F) `TILE_``K``_``F``_OFFSET +: `TILE_``K``_``F``_WIDTH -`define TH(K, F) `COH_``K``_``F``_OFFSET +: `COH_``K``_``F``_WIDTH -package tile_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function chandle core_ref_create(input string path); - import "DPI-C" function void core_ref_destroy(input chandle model); - import "DPI-C" function void core_ref_read( - input chandle model, - input longint unsigned address, - output bit [511:0] data - ); - import "DPI-C" function void core_ref_write( - input chandle model, - input longint unsigned address, - input bit [511:0] data, - input bit [63:0] mask - ); - typedef logic [`COH_REQ_WIDTH-1:0] req_t; - typedef logic [`COH_RSP_WIDTH-1:0] rsp_t; - typedef logic [`COH_DAT_WIDTH-1:0] dat_t; - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual tile_if control; - virtual stream_if #(`TILE_MEMREQ_WIDTH) mem_req; - virtual stream_if #(`TILE_MEMRESP_WIDTH) mem_resp; - virtual stream_if #(`TILE_UNCACHED_WIDTH) uncached_req[2]; - virtual stream_if #(`TILE_URESP_WIDTH) uncached_resp[2]; - chandle model; - typedef struct { - bit [`TILE_MEMRESP_WIDTH-1:0] packet; - int due; - } memory_entry; - memory_entry memory_queue[$]; - bit memory_active = 0, mmio_pending[2] = '{0, 0}; - bit [`TILE_MEMRESP_WIDTH-1:0] memory_packet; - bit [`TILE_URESP_WIDTH-1:0] mmio_packet[2]; - int mmio_due[2], mmio_wait[2] = '{0, 0}, memory_wait = 0; - int stages[2] = '{0, 0}, retired[2] = '{0, 0}, mmio_stalls[2] = '{0, 0}; - bit exited[2] = '{0, 0}, exit_retired[2] = '{0, 0}; - int cycles = 0, reads = 0, writes = 0, memory_stalls = 0, node_requests[2] = '{0, 0}; - req_t requests[int]; - int fill_seen[int], fill_dbid[int], ack_owner[int]; - bit [511:0] fill_line[int], return_line[int]; - longint unsigned return_address[int]; - int return_owner[int], return_seen[int]; - bit return_is_snoop[int]; - int grants = 0, snoops = 0, writebacks = 0, acks = 0, dirty_snoops = 0; - bit final_line_seen = 0; - localparam longint unsigned SHARED = 64'h800040c0; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 5ms; - endfunction - function bit [511:0] final_line(); - return { - 64'hffeeddccbbaa9988, - 64'h8000000000000001, - 64'h1122334455667788, - 64'h0102030405060708, - 64'd1, - 64'd1, - 64'd1, - 64'h8877665544332211 - }; - endfunction - function void build_phase(uvm_phase phase); - bit [511:0] initial_line; - super.build_phase(phase); - if (!uvm_config_db#(virtual tile_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`TILE_MEMREQ_WIDTH))::get( - this, "", "mem_req", mem_req - ) || !uvm_config_db#(virtual stream_if #(`TILE_MEMRESP_WIDTH))::get( - this, "", "mem_resp", mem_resp - )) - `uvm_fatal("VIF", "Tile interfaces missing") - for (int h = 0; h < 2; h++) begin - if (!uvm_config_db#(virtual stream_if #(`TILE_UNCACHED_WIDTH))::get( - this, "", $sformatf("uncached_req_%0d", h), uncached_req[h] - ) || !uvm_config_db#(virtual stream_if #(`TILE_URESP_WIDTH))::get( - this, "", $sformatf("uncached_resp_%0d", h), uncached_resp[h] - )) - `uvm_fatal("VIF", "Per-hart MMIO interface missing") - end - model = core_ref_create(`TILE_IMAGE); - core_ref_read(model, SHARED, initial_line); - if (initial_line !== 512'b0) - `uvm_fatal("FIXTURE", "Shared 64-byte fixture must be explicitly initialized to zero") - endfunction - function void observe_protocol(); - if (control.sample.req_valid) begin - req_t r = control.sample.req_bits; - int node, key; - if ($isunknown(r)) `uvm_fatal("CHI_X", "Unknown accepted request") - node = r[`TH(REQ, SRCID)]; - key = (node << 1) | int'(r[`TH(REQ, TXNID)]); - if (node < 1 || node > 2 || r[ - `TH(REQ, TGTID) - ] != 64 || r[ - `TH(REQ, TXNID) - ] >= 2 || requests.exists( - key - )) - `uvm_fatal("REQ_ID", "Requester node/bank identity or duplicate transaction") - requests[key] = r; - node_requests[node-1]++; - end - if (control.sample.rsp_valid) begin - rsp_t r = control.sample.rsp_bits; - int key, id; - key = (int'(r[`TH(RSP, TGTID)]) << 1) | int'(r[`TH(RSP, TXNID)]); - id = r[`TH(RSP, DBID)]; - if ($isunknown( - r - ) || !requests.exists( - key - ) || r[ - `TH(RSP, SRCID) - ] != 64 || r[ - `TH(RSP, RESPERR) - ] != 0) - `uvm_fatal("RSP_ID", "Unmatched Home response or error") - if (r[`TH(RSP, OPCODE)] == `COH_COMP_DBID) begin - if (id >= 4 || return_owner.exists( - id - ) || requests[key][ - `TH(REQ, OPCODE) - ] != `COH_WRITEBACK) - `uvm_fatal("WB_ID", "Invalid writeback DBID/descriptor") - return_owner[id] = r[`TH(RSP, TGTID)]; - return_address[id] = requests[key][`TH(REQ, ADDR)]; - return_seen[id] = 0; - return_line[id] = '0; - return_is_snoop[id] = 0; - end else if (r[`TH(RSP, OPCODE)] != `COH_COMP) - `uvm_fatal("RSP_OPCODE", "Unexpected retry/response") - requests.delete(key); - end - if (control.sample.dat_valid) begin - dat_t d = control.sample.dat_bits; - int key, id, beat; - key = (int'(d[`TH(DAT, TGTID)]) << 1) | int'(d[`TH(DAT, TXNID)]); - id = d[`TH(DAT, DBID)]; - beat = int'(d[`TH(DAT, DATAID)]) >> 1; - if ($isunknown( - d - ) || !requests.exists( - key - ) || id >= 4 || d[ - `TH(DAT, SRCID) - ] != 64 || d[ - `TH(DAT, HOMENID) - ] != 64 || d[ - `TH(DAT, OPCODE) - ] != `COH_COMP_DATA || d[ - `TH(DAT, RESPERR) - ] != 0 || d[ - `TH(DAT, BE) - ] != 32'hffffffff || (d[ - `TH(DAT, DATAID) - ] != 0 && d[ - `TH(DAT, DATAID) - ] != 2)) - `uvm_fatal("COMP_DATA", "Invalid coherent grant") - if (!fill_seen.exists(key)) begin - fill_seen[key] = 0; - fill_line[key] = '0; - fill_dbid[key] = id; - end - if ((fill_seen[key] & (1 << beat)) || fill_dbid[key] != id) - `uvm_fatal("COMP_BEAT", "Duplicate beat or changed DBID") - fill_line[key][beat*256+:256] = d[`TH(DAT, DATA)]; - fill_seen[key] |= 1 << beat; - if (fill_seen[key] == 3) begin - if (requests[key][`TH(REQ, ADDR)] == SHARED && fill_line[key] === final_line()) - final_line_seen = 1; - if (ack_owner.exists(id)) `uvm_fatal("ACK_SLOT", "Home DBID reused before CompAck") - ack_owner[id] = d[`TH(DAT, TGTID)]; - grants++; - requests.delete(key); - fill_seen.delete(key); - fill_dbid.delete(key); - fill_line.delete(key); - end - end - if (control.sample.snp_valid) begin - int id = control.sample.snp_bits[`TH(SNP, TXNID)]; - int target = control.sample.snp_bits[`TH(SNP, TARGET)]; - if ($isunknown( - control.sample.snp_bits - ) || id >= 4 || target < 1 || target > 2 || return_owner.exists( - id - ) || control.sample.snp_bits[ - `TH(SNP, SRCID) - ] != 64) - `uvm_fatal("SNOOP_ID", "Invalid directed snoop descriptor") - return_owner[id] = target; - return_address[id] = 64'(control.sample.snp_bits[`TH(SNP, ADDR)]) << 3; - return_seen[id] = 0; - return_line[id] = '0; - return_is_snoop[id] = 1; - end - if (control.sample.rx_rsp_valid) begin - rsp_t r = control.sample.rx_rsp_bits; - int id = r[`TH(RSP, TXNID)]; - if ($isunknown(r) || r[`TH(RSP, TGTID)] != 64) - `uvm_fatal("RX_RSP", "Invalid requester response") - if (r[`TH(RSP, OPCODE)] == `COH_COMP_ACK) begin - if (!ack_owner.exists(id) || ack_owner[id] != r[`TH(RSP, SRCID)]) - `uvm_fatal("ACK", "CompAck did not return Home DBID/owner") - ack_owner.delete(id); - acks++; - end else begin - if (r[ - `TH(RSP, OPCODE) - ] != `COH_SNP_RESP || !return_owner.exists( - id - ) || !return_is_snoop[id] || return_owner[id] != r[ - `TH(RSP, SRCID) - ] || return_seen[id] != 0) - `uvm_fatal("SNOOP_RSP", "Unmatched snoop response") - return_owner.delete(id); - return_seen.delete(id); - return_address.delete(id); - return_line.delete(id); - return_is_snoop.delete(id); - end - end - if (control.sample.rx_dat_valid) begin - dat_t d = control.sample.rx_dat_bits; - int id = d[`TH(DAT, TXNID)]; - int beat = int'(d[`TH(DAT, DATAID)]) >> 1; - if ($isunknown( - d - ) || !return_owner.exists( - id - ) || return_owner[id] != d[ - `TH(DAT, SRCID) - ] || d[ - `TH(DAT, TGTID) - ] != 64 || d[ - `TH(DAT, RESPERR) - ] != 0 || (d[ - `TH(DAT, DATAID) - ] != 0 && d[ - `TH(DAT, DATAID) - ] != 2) || (return_seen[id] & (1 << beat)) || d[ - `TH(DAT, BE) - ] != 32'hffffffff) - `uvm_fatal("RETURN_DATA", "Unmatched/duplicate/partial return data") - if (d[`TH(DAT, OPCODE)] != (return_is_snoop[id] ? `COH_SNP_DATA : `COH_COPYBACK_DATA)) - `uvm_fatal("RETURN_OPCODE", "Wrong return role") - return_line[id][beat*256+:256] = d[`TH(DAT, DATA)]; - return_seen[id] |= 1 << beat; - if (return_seen[id] == 3) begin - if(return_is_snoop[id]&&return_address[id]==SHARED&&return_line[id]===final_line()) - final_line_seen = 1; - if (return_is_snoop[id]) begin - snoops++; - if (d[`TH(DAT, RESP)] & 4) dirty_snoops++; - end else writebacks++; - return_owner.delete(id); - return_seen.delete(id); - return_address.delete(id); - return_line.delete(id); - return_is_snoop.delete(id); - end - end - endfunction - task service(); - forever begin - @(control.sample); - if (!control.sample.reset) begin - cycles++; - observe_protocol(); - if (mem_resp.sample.valid && mem_resp.sample.ready) memory_active = 0; - for (int h = 0; h < 2; h++) begin - if (uncached_resp[h].sample.valid && uncached_resp[h].sample.ready) mmio_pending[h] = 0; - if (control.sample.trapped[h]) - `uvm_fatal("TRAP", $sformatf( - "hart%0d cause=%h value=%h pc=%h", - h, - control.sample.trapCause[h], - control.sample.trapValue[h], - control.sample.trapPc[h] - )) - if (control.sample.retired[h]) begin - retired[h]++; - if (control.sample.retiredPc[h] == `CORE_EXIT_PC) begin - if (!exited[h] || exit_retired[h]) - `uvm_fatal("EXIT_RETIRE", "Exit retired before MMIO success or twice") - exit_retired[h] = 1; - end - end - if (uncached_req[h].sample.valid && !uncached_req[h].sample.ready) begin - mmio_stalls[h]++; - mmio_wait[h]++; - end - if (uncached_req[h].sample.valid && uncached_req[h].sample.ready) begin - longint unsigned address = uncached_req[h].sample.bits[`TF(UNCACHED, ADDR)]; - longint unsigned value = uncached_req[h].sample.bits[`TF(UNCACHED, DATA)]; - if (mmio_pending[h] || exited[h] || $isunknown( - uncached_req[h].sample.bits - ) || !uncached_req[h].sample.bits[ - `TF(UNCACHED, WRITE) - ] || uncached_req[h].sample.bits[ - `TF(UNCACHED, SIZE) - ] != 3 || (address != 64'h10000000 && address != 64'h10000008)) - `uvm_fatal("MMIO", "Unexpected per-hart MMIO operation") - if (address == 64'h10000008) begin - if (value != stages[h] + 1 || value > 2) - `uvm_fatal("STAGE", "Per-hart stage out of order") - stages[h] = value; - end else begin - if (value != 0 || stages[h] != 2) - `uvm_fatal("FIRMWARE", $sformatf( - "hart%0d failed stage%0d code%h", h, stages[h], value)) - exited[h] = 1; - end - mmio_packet[h] = '0; - mmio_packet[h][`TF(URESP, TAG)] = uncached_req[h].sample.bits[`TF(UNCACHED, TAG)]; - mmio_pending[h] = 1; - mmio_due[h] = cycles + 5 + h; - mmio_wait[h] = 0; - end - end - if (mem_req.sample.valid && !mem_req.sample.ready) begin - memory_stalls++; - memory_wait++; - end - if (mem_req.sample.valid && mem_req.sample.ready) begin - memory_entry e; - bit [511:0] line; - longint unsigned address = mem_req.sample.bits[`TF(MEMREQ, ADDR)]; - if ($isunknown(mem_req.sample.bits) || address[5:0] != 0) - `uvm_fatal("MEMORY", "Unknown/unaligned backend request") - e.packet = '0; - e.packet[`TF(MEMRESP, ID)] = mem_req.sample.bits[`TF(MEMREQ, ID)]; - if (mem_req.sample.bits[`TF(MEMREQ, WRITE)]) begin - core_ref_write(model, address, mem_req.sample.bits[`TF(MEMREQ, DATA)], - mem_req.sample.bits[`TF(MEMREQ, MASK)]); - writes++; - end else begin - core_ref_read(model, address, line); - e.packet[`TF(MEMRESP, DATA)] = line; - reads++; - end - e.due = cycles + 5 + (reads + writes) % 7; - memory_queue.push_back(e); - memory_wait = 0; - end - end - @(negedge control.clock); - mem_req.ready = !control.reset && memory_wait >= 2; - for (int h = 0; h < 2; h++) begin - uncached_req[h].ready = !control.reset && !mmio_pending[h] && mmio_wait[h] >= 2; - uncached_resp[h].valid = !control.reset && mmio_pending[h] && cycles >= mmio_due[h]; - uncached_resp[h].bits = mmio_packet[h]; - end - if (!memory_active) - for (int q = memory_queue.size() - 1; q >= 0; q--) - if (memory_queue[q].due <= cycles) begin - memory_packet = memory_queue[q].packet; - memory_queue.delete(q); - memory_active = 1; - break; - end - mem_resp.valid = !control.reset && memory_active; - mem_resp.bits = memory_packet; - end - endtask - task execute(); - control.reset = 1; - mem_req.ready = 0; - mem_resp.valid = 0; - mem_resp.bits = '0; - for (int h = 0; h < 2; h++) begin - uncached_req[h].ready = 0; - uncached_resp[h].valid = 0; - uncached_resp[h].bits = '0; - end - repeat (5) @(negedge control.clock); - control.reset = 0; - fork - service(); - join_none - wait (exit_retired[0] && exit_retired[1]); - do - @(control.sample); - while (control.sample.outstanding || memory_active || memory_queue.size() || - mmio_pending[0] || mmio_pending[1] || requests.num() || fill_seen.num() || - ack_owner.num() || return_owner.num()); - if(!final_line_seen||!dirty_snoops||!node_requests[0]||!node_requests[1]||!memory_stalls||!mmio_stalls[0]||!mmio_stalls[1]) - `uvm_fatal( - "SCENARIOS", - "Missing final full64B coherent line, dirty snoop, distinct nodes or backpressure") - `uvm_info( - "TILE", - $sformatf( - "2 real harts stages=2/2 exits=0/0 retired=%0d/%0d; CHI req=%0d/%0d grants=%0d snoopLines=%0d dirty=%0d copybacks=%0d acks=%0d; DDR R/W=%0d/%0d stalls=%0d MMIO stalls=%0d/%0d; final64B checked and all descriptors drained", - retired[0], retired[1], node_requests[0], node_requests[1], grants, snoops, - dirty_snoops, writebacks, acks, reads, writes, memory_stalls, mmio_stalls[0], - mmio_stalls[1]), UVM_LOW) - endtask - function void final_phase(uvm_phase phase); - core_ref_destroy(model); - super.final_phase(phase); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_tb.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_tb.sv deleted file mode 100644 index be26d3420..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/tile_tb.sv +++ /dev/null @@ -1,58 +0,0 @@ -`include "tile_config.svh" -module tile_tb; - import uvm_pkg::*; - import tile_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - tile_if control (clock); - stream_if #(`TILE_MEMREQ_WIDTH) mem_req ( - clock, - control.reset - ); - stream_if #(`TILE_MEMRESP_WIDTH) mem_resp ( - clock, - control.reset - ); - stream_if #(`TILE_UNCACHED_WIDTH) - uncached_req_0 ( - clock, - control.reset - ), - uncached_req_1 ( - clock, - control.reset - ); - stream_if #(`TILE_URESP_WIDTH) - uncached_resp_0 ( - clock, - control.reset - ), - uncached_resp_1 ( - clock, - control.reset - ); - Tile dut ( - .clock(clock), - .reset(control.reset), - .io_resetVector(64'h80000000), - .io_time(64'b0), - .io_outstanding(control.outstanding), - `include "tile_ports.svh" - ); - initial begin - uvm_config_db#(virtual tile_if)::set(null, "uvm_test_top", "control", control); - uvm_config_db#(virtual stream_if #(`TILE_MEMREQ_WIDTH))::set(null, "uvm_test_top", "mem_req", - mem_req); - uvm_config_db#(virtual stream_if #(`TILE_MEMRESP_WIDTH))::set(null, "uvm_test_top", "mem_resp", - mem_resp); - uvm_config_db#(virtual stream_if #(`TILE_UNCACHED_WIDTH))::set( - null, "uvm_test_top", "uncached_req_0", uncached_req_0); - uvm_config_db#(virtual stream_if #(`TILE_UNCACHED_WIDTH))::set( - null, "uvm_test_top", "uncached_req_1", uncached_req_1); - uvm_config_db#(virtual stream_if #(`TILE_URESP_WIDTH))::set(null, "uvm_test_top", - "uncached_resp_0", uncached_resp_0); - uvm_config_db#(virtual stream_if #(`TILE_URESP_WIDTH))::set(null, "uvm_test_top", - "uncached_resp_1", uncached_resp_1); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache.f deleted file mode 100644 index 33340ae79..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic.f deleted file mode 100644 index ee4b21ef4..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_atomic_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic.toml deleted file mode 100644 index 366502210..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual LSU does not accept Fence or unknown atomics" -invalid_case = 1 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic_write.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic_write.f deleted file mode 100644 index 793f41de9..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic_write.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_atomic_write_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic_write.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic_write.toml deleted file mode 100644 index 5c3b59521..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_atomic_write.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual LSU atomic requires a 32 or 64 bit operand" -invalid_case = 2 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_atomic.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_atomic.f deleted file mode 100644 index 17d40a9c5..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_atomic.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_execute_atomic_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_atomic.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_atomic.toml deleted file mode 100644 index 24accfd7f..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_atomic.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual instruction word requires unsigned 64 bit read without atomic" -invalid_case = 7 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_signed.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_signed.f deleted file mode 100644 index 5f1106e80..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_signed.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_execute_signed_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_signed.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_signed.toml deleted file mode 100644 index 75aaca461..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_signed.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual instruction word requires unsigned 64 bit read without atomic" -invalid_case = 9 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_size.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_size.f deleted file mode 100644 index 8e4146c1d..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_size.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_execute_size_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_size.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_size.toml deleted file mode 100644 index 73034135d..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_size.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual instruction word requires unsigned 64 bit read without atomic" -invalid_case = 8 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_write.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_write.f deleted file mode 100644 index 83023cc9a..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_write.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_execute_write_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_write.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_write.toml deleted file mode 100644 index acc1d20e0..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_execute_write.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual instruction word requires unsigned 64 bit read without atomic" -invalid_case = 6 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_mode.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_mode.f deleted file mode 100644 index 496ef0a7c..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_mode.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_mode_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_mode.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_mode.toml deleted file mode 100644 index 4eba68ed5..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_mode.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual LSU requires Bare or Sv39 mode" -invalid_case = 4 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_operand.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_operand.f deleted file mode 100644 index f2c5dba67..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_operand.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_operand_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_operand.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_operand.toml deleted file mode 100644 index 401130c64..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_operand.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual LSU atomic requires a 32 or 64 bit operand" -invalid_case = 3 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_privilege.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_privilege.f deleted file mode 100644 index c0895f988..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_privilege.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_privilege_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_privilege.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_privilege.toml deleted file mode 100644 index 3666d1758..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_privilege.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual LSU effective privilege must be U, S or M" -invalid_case = 5 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_size.f b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_size.f deleted file mode 100644 index 897a4abc0..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_size.f +++ /dev/null @@ -1,10 +0,0 @@ -+define+VIRTUAL_CACHE_TB=virtual_cache_bad_size_tb -+incdir+@VERIFY@/uvm/src/ip -+incdir+@RTL@ -+incdir+@RESOURCES@ -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/stream_if.sv -@RESOURCES@/virtual_cache_interface.sv -@RESOURCES@/virtual_cache_package.sv --F @RTL@/VirtualCacheSystem/filelist.f -@RESOURCES@/virtual_cache_tb.sv diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_size.toml b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_size.toml deleted file mode 100644 index b51b400fc..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_bad_size.toml +++ /dev/null @@ -1,2 +0,0 @@ -expected_assertion = "Virtual LSU size must be 1, 2, 4 or 8 bytes" -invalid_case = 0 diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_interface.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_interface.sv deleted file mode 100644 index 3d8d7d9ee..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_interface.sv +++ /dev/null @@ -1,18 +0,0 @@ -`include "virtual_cache_config.svh" -interface virtual_cache_if ( - input logic clock -); - logic reset, active, conflict_trace; - int conflict_wb, conflict_dbid, conflict_beats, conflict_ack; - logic [`VM_OUTSTANDING_BITS-1:0] outstanding; - logic translation_valid, physical_valid, cache_valid, cache_pte, eviction_valid; - logic [`VM_TRANSLATION_WIDTH-1:0] translation_bits; - logic [`VM_PHYSICAL_WIDTH-1:0] physical_bits; - logic [`VM_CACHE_WIDTH-1:0] cache_bits; - logic [`VM_CACHE_ADDR_WIDTH-1:0] eviction_addr; - clocking sample @(posedge clock); - default input #1step; - input reset,active,outstanding,translation_valid,physical_valid,cache_valid,cache_pte,eviction_valid; - input translation_bits, physical_bits, cache_bits, eviction_addr; - endclocking -endinterface diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_package.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_package.sv deleted file mode 100644 index bfcceade4..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_package.sv +++ /dev/null @@ -1,1127 +0,0 @@ -`include "virtual_cache_config.svh" -`define VF(K, F) `VM_``K``_``F``_OFFSET +: `VM_``K``_``F``_WIDTH -package virtual_cache_pkg; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - typedef bit [63:0] u64; - typedef logic [`VM_REQ_WIDTH-1:0] req_t; - typedef logic [`VM_MEMRESP_WIDTH-1:0] mem_t; - import "DPI-C" function chandle mmu_ref_create(input int unsigned address_bits); - import "DPI-C" function void mmu_ref_destroy(input chandle model); - import "DPI-C" function void mmu_ref_program( - input chandle model, - input longint unsigned address, - value, - input int unsigned error - ); - import "DPI-C" function int unsigned mmu_ref_pending(input chandle model); - import "DPI-C" function int unsigned mmu_ref_peek( - input chandle model, - output longint unsigned address - ); - import "DPI-C" function int unsigned mmu_ref_consume( - input chandle model, - input longint unsigned address - ); - import "DPI-C" function int unsigned mmu_ref_translate( - input chandle model, - input longint unsigned address, - input int unsigned mode, - input longint unsigned root, - input int unsigned privilege, - write, - execute, - sum, - mxr, - output longint unsigned paddr, - output int unsigned page_fault, - access_fault, - level - ); - import "DPI-C" function void cpu_mem_ref_prepare( - input longint unsigned addr, - input int unsigned size, - write, - input longint unsigned data, - input int unsigned atomic, - cacheable, - normal, - address_bits, - output int unsigned target, - misaligned, - access_fault, - output longint unsigned bus_addr, - bus_data, - output int unsigned bus_mask, - atomic_word - ); - import "DPI-C" function longint unsigned cpu_mem_ref_result( - input longint unsigned addr, - input int unsigned size, - write, - signed_load, - atomic, - cacheable, - input longint unsigned raw, - input int unsigned error - ); - class result_item extends uvm_sequence_item; - bit [5:0] tag; - u64 data; - bit misaligned, page_fault, access_fault; - `uvm_object_utils_begin(result_item) - `uvm_field_int(tag, UVM_DEFAULT) - `uvm_field_int(data, UVM_DEFAULT) - `uvm_field_int(misaligned, UVM_DEFAULT) - `uvm_field_int(page_fault, UVM_DEFAULT) - `uvm_field_int(access_fault, UVM_DEFAULT) - `uvm_object_utils_end - function new(string name = "result_item"); - super.new(name); - endfunction - endclass - class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual virtual_cache_if control; - virtual stream_if #(`VM_AUTH_WIDTH) auth_req; - virtual stream_if #(1) auth_resp; - virtual stream_if #(`VM_REQ_WIDTH) req; - virtual stream_if #(`VM_RESP_WIDTH) resp; - virtual stream_if #(`VM_MEMREQ_WIDTH) mem_req; - virtual stream_if #(`VM_MEMRESP_WIDTH) mem_resp; - virtual stream_if #(`VM_UNCACHED_WIDTH) uncached_req; - virtual stream_if #(`VM_URESP_WIDTH) uncached_resp; - in_order_scoreboard #(result_item) scoreboard; - chandle mmu; - bit [511:0] backing[u64], architectural[u64]; - bit [7:0] mmio_backing[u64], mmio_arch[u64]; - bit line_error[u64], mmio_error[u64]; - int evictions[u64]; - typedef struct { - mem_t bits; - int due; - } memory_entry; - memory_entry pending[$]; - bit mem_active = 0, uncached_active = 0, uncached_pending = 0; - mem_t memory_packet; - logic [`VM_URESP_WIDTH-1:0] uncached_packet; - int - uncached_due, - cycle = 0, - expected = 0, - accepted = 0, - returned = 0, - pte_reads = 0, - data_accesses = 0, - uncached_accesses = 0; - int - memory_reads = 0, - memory_writes = 0, - req_stalls = 0, - resp_stalls = 0, - mem_stalls = 0, - uncached_stalls = 0; - int page_faults = 0, access_faults = 0, misalignments = 0, sc_successes = 0, sc_evictions = 0; - int invalid_case = -1; - bit deny_final = 0, deny_pte = 0; - u64 denied_pte = 0; - bit - final_queried = 0, - auth_pending = 0, - auth_allow = 0, - auth_is_pte = 0, - pte_permit = 0, - final_permit = 0; - u64 auth_address = 0, pte_permitted_address = 0; - int auth_due = 0, auth_queries = 0, auth_denials = 0, auth_stalls = 0; - - bit - hold_response = 0, - case_active = 0, - expect_translation = 0, - expect_physical = 0, - expect_cache = 0, - expect_uncached = 0; - bit saw_translation = 0, saw_physical = 0, saw_cache = 0, saw_uncached = 0, commit_write = 0; - int uncached_pte_reads = 0; - int expected_region = 0, commit_bytes = 0; - u64 expected_pa = 0, commit_data = 0; - logic [`VM_TRANSLATION_WIDTH-1:0] translation_expected; - logic [`VM_PHYSICAL_WIDTH-1:0] physical_expected; - logic [`VM_CACHE_WIDTH-1:0] cache_expected; - logic [`VM_UNCACHED_WIDTH-1:0] uncached_expected; - req_t current; - string current_case; - u64 next_table = `VM_DDR_BASE + 64'h10000; - localparam u64 DATA = `VM_DDR_BASE + 64'h800180; - localparam u64 EVICT_DATA = `VM_DDR_BASE + 64'h810000; - localparam u64 ERROR_DATA = `VM_DDR_BASE + 64'h900180; - localparam u64 MAX_PA = (64'h1 << `VM_CACHE_ADDR_WIDTH) - 1; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 2ms; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - scoreboard = in_order_scoreboard#(result_item)::type_id::create("scoreboard", this); - mmu = mmu_ref_create(`VM_CACHE_ADDR_WIDTH); - if (!uvm_config_db#(virtual stream_if #(`VM_AUTH_WIDTH))::get( - this, "", "auth_req", auth_req - ) || !uvm_config_db#(virtual stream_if #(1))::get( - this, "", "auth_resp", auth_resp - ) || !uvm_config_db#(virtual virtual_cache_if)::get( - this, "", "control", control - ) || !uvm_config_db#(virtual stream_if #(`VM_REQ_WIDTH))::get( - this, "", "req", req - ) || !uvm_config_db#(virtual stream_if #(`VM_RESP_WIDTH))::get( - this, "", "resp", resp - ) || !uvm_config_db#(virtual stream_if #(`VM_MEMREQ_WIDTH))::get( - this, "", "mem_req", mem_req - ) || !uvm_config_db#(virtual stream_if #(`VM_MEMRESP_WIDTH))::get( - this, "", "mem_resp", mem_resp - ) || !uvm_config_db#(virtual stream_if #(`VM_UNCACHED_WIDTH))::get( - this, "", "uncached_req", uncached_req - ) || !uvm_config_db#(virtual stream_if #(`VM_URESP_WIDTH))::get( - this, "", "uncached_resp", uncached_resp - )) - `uvm_fatal("VIF", "Virtual cache interfaces missing") - void'(uvm_config_db#(int)::get(this, "", "invalid_case", invalid_case)); - endfunction - function int region(u64 address, int bytes); - if (address > MAX_PA || bytes - 1 > MAX_PA - address) return 0; - if(address>=`VM_READ_ONLY_BASE&&address+bytes<=`VM_WRITE_ONLY_BASE+`VM_WRITE_ONLY_BYTES) - return 1; - if(address>=`VM_DDR_BASE&&address-`VM_DDR_BASE<`VM_DDR_BYTES&&bytes<=`VM_DDR_BYTES-(address-`VM_DDR_BASE)) - return 1; - if(address>=`VM_MMIO_BASE&&address-`VM_MMIO_BASE<`VM_MMIO_BYTES&&bytes<=`VM_MMIO_BYTES-(address-`VM_MMIO_BASE)) - return 2; - if(address>=`VM_SHORT_MMIO_BASE&&address-`VM_SHORT_MMIO_BASE<`VM_SHORT_MMIO_BYTES&&bytes<=`VM_SHORT_MMIO_BYTES-(address-`VM_SHORT_MMIO_BASE)) - return 2; - if(address>=`VM_NORMAL_RAM_BASE&&address+bytes<=`VM_NORMAL_RAM_BASE+`VM_NORMAL_RAM_BYTES) - return 3; - return 0; - endfunction - function void initialize_line(u64 address, bit [511:0] value); - u64 base = address & ~64'h3f; - if (backing.exists(base)) `uvm_fatal("FIXTURE", "Attempt to reinitialize a live DDR line") - backing[base] = value; - architectural[base] = value; - endfunction - function void program_pte(u64 address, u64 value, bit error = 0); - u64 base = address & ~64'h3f; - if (region(address, 8) == 1 || region(address, 8) == 3) begin - if (!backing.exists(base)) initialize_line(base, '0); - backing[base][int'(address&63)*8+:64] = value; - architectural[base] = backing[base]; - if (error) line_error[base] = 1; - end else if (!error) `uvm_fatal("FIXTURE", "Non-DDR PTE must be an explicit access error") - mmu_ref_program(mmu, address, value, error); - endfunction - function u64 mapping(u64 va, u64 pa, u64 flags = 64'hcf, int error_level = -1); - u64 root = next_table; - next_table += 64'h3000; - for (int level = 2; level >= 0; level--) begin - u64 table = root + (2 - level) * 4096; - u64 address = table + ((va >> (12 + 9 * level)) & 511) * 8; - program_pte(address, - level == 0 ? ((pa >> 12) << 10) | flags : (((table + 4096) >> 12) << 10) | 1, - level == error_level); - end - return root >> 12; - endfunction - function req_t command(u64 address, int size = 3, bit write = 0, bit signed_load = 0, - u64 data = 0, int atomic = 0, int mode = 0, u64 root = 0, - int privilege = 1, bit sum = 0, bit mxr = 0, bit execute = 0); - req_t r = '0; - r[`VF(REQ, EXECUTE)] = execute; - r[`VF(REQ, VADDR)] = address; - r[`VF(REQ, SIZE)] = size; - r[`VF(REQ, WRITE)] = write; - r[`VF(REQ, SIGNED)] = signed_load; - r[`VF(REQ, DATA)] = data; - r[`VF(REQ, ATOMIC)] = atomic; - r[`VF(REQ, SATPMODE)] = mode; - r[`VF(REQ, ROOTPPN)] = root; - r[`VF(REQ, PRIVILEGE)] = privilege; - r[`VF(REQ, SUM)] = sum; - r[`VF(REQ, MXR)] = mxr; - return r; - endfunction - function u64 read_arch(u64 address, int bytes, int kind); - u64 value = 0; - for (int i = 0; i < bytes; i++) begin - u64 a = address + i; - if (kind == 1 || kind == 3) begin - if (!architectural.exists(a & ~64'h3f)) - `uvm_fatal("FIXTURE", $sformatf("Uninitialized architecture DDR %h", a)) - value |= u64'(architectural[a&~64'h3f][int'(a&63)*8+:8]) << (i * 8); - end else begin - if (!mmio_arch.exists(a)) - `uvm_fatal("FIXTURE", $sformatf("Uninitialized architecture MMIO %h", a)) - value |= u64'(mmio_arch[a]) << (i * 8); - end - end - return value; - endfunction - function void predict(req_t r, int sc_result); - int unsigned target, misaligned, af, mask, word, pf, level; - longint unsigned pa, bus_addr, bus_data; - u64 raw, old, operand, next_value, saved_pte; - int size, atom, bytes; - bit write, failed; - result_item answer = new(current_case); - if (mmu_ref_pending(mmu)) - `uvm_fatal("PTE_PENDING", "Previous reference PTE chain not drained") - current = r; - case_active = 1; - commit_write = 0; - pte_permit = 0; - final_permit = 0; - final_queried = 0; - expect_translation = 0; - expect_physical = 0; - expect_cache = 0; - expect_uncached = 0; - saw_translation = 0; - saw_physical = 0; - saw_cache = 0; - saw_uncached = 0; - translation_expected = '0; - physical_expected = '0; - cache_expected = '0; - uncached_expected = '0; - size = r[`VF(REQ, SIZE)]; - bytes = 1 << size; - atom = r[`VF(REQ, ATOMIC)]; - write = r[`VF(REQ, WRITE)]; - answer.tag = r[`VF(REQ, TAG)]; - // Before translation only alignment is meaningful; use physical reference - // again with the translated PA for its actual range/classification result. - cpu_mem_ref_prepare(r[`VF(REQ, VADDR)], size, write, r[`VF(REQ, DATA)], atom, 1, 1, - `VM_CACHE_ADDR_WIDTH, target, misaligned, af, bus_addr, bus_data, mask, - word); - answer.misaligned = misaligned; - if (!misaligned) begin - expect_translation = 1; - if (deny_pte) begin - saved_pte = read_arch(denied_pte, 8, 1); - mmu_ref_program(mmu, denied_pte, saved_pte, 1); - end - if (mmu_ref_translate( - mmu, - r[ - `VF(REQ, VADDR) - ], - r[ - `VF(REQ, SATPMODE) - ], - r[ - `VF(REQ, ROOTPPN) - ], - r[ - `VF(REQ, PRIVILEGE) - ], - write || (atom >= 1 && atom <= 9) || atom == 11, - r[ - `VF(REQ, EXECUTE) - ], - r[ - `VF(REQ, SUM) - ], - r[ - `VF(REQ, MXR) - ], - pa, - pf, - af, - level - ) != 0) - `uvm_fatal("FIXTURE", $sformatf("Reference requested an unprogrammed PTE %h", pa)) - if (deny_pte) mmu_ref_program(mmu, denied_pte, saved_pte, 0); - translation_expected[`VF(TRANSLATION, PADDR)] = pa; - translation_expected[`VF(TRANSLATION, PAGEFAULT)] = pf; - translation_expected[`VF(TRANSLATION, ACCESSFAULT)] = af; - translation_expected[`VF(TRANSLATION, LEVEL)] = level; - answer.page_fault = pf; - answer.access_fault = af; - if (!pf && !af) begin - expected_pa = pa; - expected_region = region(pa, bytes); - if(deny_final||(pa>=`VM_READ_ONLY_BASE&&pa<`VM_READ_ONLY_BASE+64&&(write||atom!=0||r[ - `VF(REQ, EXECUTE) - ])) || (pa >= `VM_WRITE_ONLY_BASE && pa < `VM_WRITE_ONLY_BASE + 64 && - (!write || atom != 0 || r[ - `VF(REQ, EXECUTE) - ])) || expected_region == 0 || - (atom != 0 && expected_region != 1 && expected_region != 3) || (r[ - `VF(REQ, EXECUTE) - ] && expected_region != 1 && expected_region != 3)) - answer.access_fault = 1; - else begin - expect_physical = 1; - physical_expected[`VF(PHYSICAL, ADDR)] = pa; - physical_expected[`VF(PHYSICAL, TAG)] = r[`VF(REQ, TAG)]; - physical_expected[`VF(PHYSICAL, SIZE)] = size; - physical_expected[`VF(PHYSICAL, WRITE)] = write; - physical_expected[`VF(PHYSICAL, SIGNED)] = r[`VF(REQ, SIGNED)]; - physical_expected[`VF(PHYSICAL, DATA)] = r[`VF(REQ, DATA)]; - physical_expected[`VF(PHYSICAL, ATOMIC)] = atom; - physical_expected[`VF(PHYSICAL, CACHEABLE)] = expected_region == 1; - physical_expected[`VF(PHYSICAL, NORMAL)] = expected_region == 1 || expected_region == 3; - cpu_mem_ref_prepare(pa, size, write, r[`VF(REQ, DATA)], atom, expected_region == 1, - expected_region == 1 || expected_region == 3, `VM_CACHE_ADDR_WIDTH, - target, misaligned, af, bus_addr, bus_data, mask, word); - answer.access_fault = af; - if (target != 0) begin - expect_cache = target == 1; - expect_uncached = target == 2; - cache_expected[`VF(CACHE, ADDR)] = bus_addr; - cache_expected[`VF(CACHE, WRITE)] = write; - cache_expected[`VF(CACHE, DATA)] = bus_data; - cache_expected[`VF(CACHE, MASK)] = mask; - cache_expected[`VF(CACHE, ATOMIC)] = atom; - cache_expected[`VF(CACHE, ATOMICWORD)] = word; - uncached_expected[`VF(UNCACHED, ADDR)] = pa; - uncached_expected[`VF(UNCACHED, TAG)] = r[`VF(REQ, TAG)]; - uncached_expected[`VF(UNCACHED, SIZE)] = size; - uncached_expected[`VF(UNCACHED, WRITE)] = write; - uncached_expected[`VF(UNCACHED, DATA)] = r[`VF(REQ, DATA)]; - uncached_expected[`VF(UNCACHED, ATOMIC)] = atom; - uncached_expected[ - `VF(UNCACHED, NORMAL) - ] = expected_region == 1 || expected_region == 3; - failed = expected_region != 2 ? - (line_error.exists(pa & ~64'h3f) && line_error[pa&~64'h3f]) : - (mmio_error.exists(pa) && mmio_error[pa]); - answer.access_fault = failed; - if (!failed) begin - old = read_arch(pa, bytes, expected_region); - operand = r[`VF(REQ, DATA)]; - next_value = operand; - if (atom != 0) begin - raw = (bytes == 4 && old[31]) ? old | 64'hffffffff00000000 : old; - case (atom) - 1: next_value = operand; - 2: next_value = old + operand; - 3: next_value = old ^ operand; - 4: next_value = old & operand; - 5: next_value = old | operand; - 6: - next_value = bytes == 4 ? - ($signed(old[31:0]) < $signed(operand[31:0]) ? old : operand) : - ($signed(old) < $signed(operand) ? old : operand); - 7: - next_value = bytes == 4 ? - ($signed(old[31:0]) > $signed(operand[31:0]) ? old : operand) : - ($signed(old) > $signed(operand) ? old : operand); - 8: - next_value=bytes==4?(old[31:0]operand[31:0]?old:operand):(old>operand?old:operand); - 10: next_value = old; - 11: begin - if (sc_result < 0 || sc_result > 1) - `uvm_fatal("SC_ORACLE", - "SC requires an explicit architectural scenario outcome") - raw = sc_result; - end - default: `uvm_fatal("ATOMIC", "Unsupported test operation") - endcase - end else raw = target == 1 ? read_arch(pa & ~64'h7, 8, 1) : old; - answer.data = cpu_mem_ref_result(pa, size, write, r[`VF(REQ, SIGNED)], atom, - target == 1, raw, 0); - commit_write = write || (atom >= 1 && atom <= 9) || (atom == 11 && sc_result == 0); - commit_bytes = bytes; - commit_data = next_value; - end - end - end - end - end - scoreboard.expected_export.write(answer); - expected++; - endfunction - function void observations(); - if (control.sample.eviction_valid) evictions[u64'(control.sample.eviction_addr)]++; - if (control.sample.cache_valid) begin - u64 address; - longint unsigned next_pte; - logic [`VM_CACHE_WIDTH-1:0] wanted; - if (!case_active || $isunknown({control.sample.cache_pte, control.sample.cache_bits})) - `uvm_fatal("CACHE_EVENT", "Unknown or unsolicited cache operation") - address = control.sample.cache_bits[`VF(CACHE, ADDR)]; - if (control.sample.cache_pte) begin - if (!pte_permit || pte_permitted_address != address || region( - address, 8 - ) != 1 || !mmu_ref_peek( - mmu, next_pte - ) || address != next_pte) - `uvm_fatal("PTE_CHAIN", $sformatf( - "%s wrong real-cache PTE address %h", current_case, address)) - wanted = '0; - wanted[`VF(CACHE, ADDR)] = next_pte; - if (control.sample.cache_bits !== wanted) - `uvm_fatal("PTE_WRITE", - "PTE access must be an ordinary cache read, never A/D writeback") - if (!mmu_ref_consume(mmu, address)) - `uvm_fatal("PTE_CHAIN", "PTE reference queue mismatch") - pte_permit = 0; - pte_reads++; - end else begin - if(!expect_cache||saw_cache||!saw_translation||!saw_physical||control.sample.cache_bits!==cache_expected) - `uvm_fatal("CACHE_DATA", $sformatf( - "%s wrong PA/lane/mask/operand/atomic cache request", current_case)) - saw_cache = 1; - data_accesses++; - end - end - if (control.sample.translation_valid) begin - longint unsigned denied; - if (!expect_translation || saw_translation || $isunknown( - control.sample.translation_bits - ) || control.sample.translation_bits !== translation_expected) - `uvm_fatal("TRANSLATION", $sformatf( - "%s wrong translation result got=%h expected=%h", - current_case, - control.sample.translation_bits, - translation_expected - )) - if (mmu_ref_pending(mmu)) begin - if (!mmu_ref_peek( - mmu, denied - ) || (region( - denied, 8 - ) != 2 && region( - denied, 8 - ) != 0 && (!deny_pte || denied != denied_pte)) || !translation_expected[ - `VF(TRANSLATION, ACCESSFAULT) - ]) - `uvm_fatal("PTE_CHAIN", "Translation skipped a required cache PTE read") - if (!mmu_ref_consume(mmu, denied) || mmu_ref_pending(mmu)) - `uvm_fatal("PMA", "Unexpected denied-PTE reference chain") - end - saw_translation = 1; - end - if (control.sample.physical_valid) begin - if (!final_permit || !expect_physical || saw_physical || !saw_translation || $isunknown( - control.sample.physical_bits - ) || control.sample.physical_bits !== physical_expected) - `uvm_fatal("PHYSICAL", $sformatf( - "%s PA classification or captured command changed", current_case)) - saw_physical = 1; - end - endfunction - task service(); - forever begin - @(control.sample); - if (!control.sample.reset) begin - cycle++; - if (req.sample.valid && req.sample.ready) accepted++; - if (req.sample.valid && !req.sample.ready) req_stalls++; - if (resp.sample.valid && !resp.sample.ready) resp_stalls++; - if (mem_req.sample.valid && !mem_req.sample.ready) mem_stalls++; - if (uncached_req.sample.valid && !uncached_req.sample.ready) uncached_stalls++; - if (auth_req.sample.valid && !auth_req.sample.ready) auth_stalls++; - if (auth_resp.sample.valid && auth_resp.sample.ready) begin - if (!auth_pending) `uvm_fatal("AUTH_RESPONSE", "Authorization response without query") - if (auth_allow) begin - if (auth_is_pte) begin - pte_permit = 1; - pte_permitted_address = auth_address; - end else final_permit = 1; - end else auth_denials++; - auth_pending = 0; - end - if (auth_req.sample.valid && auth_req.sample.ready) begin - logic [`VM_AUTH_WIDTH-1:0] wanted = '0; - longint unsigned pte; - if (auth_pending || $isunknown(auth_req.sample.bits)) - `uvm_fatal("AUTH_QUERY", "Overlapping or unknown authorization query") - auth_is_pte = auth_req.sample.bits[`VF(AUTH, ISPTE)]; - if (auth_is_pte) begin - if (!mmu_ref_peek(mmu, pte)) `uvm_fatal("AUTH_PTE", "Unsolicited PTE authorization") - wanted[`VF(AUTH, PADDR)] = pte; - wanted[`VF(AUTH, SIZE)] = 3; - wanted[`VF(AUTH, READ)] = 1; - wanted[`VF(AUTH, PRIVILEGE)] = 1; - wanted[`VF(AUTH, ISPTE)] = 1; - end else begin - if (final_queried) `uvm_fatal("AUTH_FINAL", "Repeated final query") - final_queried = 1; - if (!saw_translation || translation_expected[ - `VF(TRANSLATION, PAGEFAULT) - ] || translation_expected[ - `VF(TRANSLATION, ACCESSFAULT) - ]) - `uvm_fatal("AUTH_FINAL", "Final authorization before valid translation") - wanted[`VF(AUTH, PADDR)] = translation_expected[`VF(TRANSLATION, PADDR)]; - wanted[`VF(AUTH, SIZE)] = current[`VF(REQ, SIZE)]; - wanted[ - `VF(AUTH, READ) - ] = !current[ - `VF(REQ, EXECUTE) - ] && !current[ - `VF(REQ, WRITE) - ] && current[ - `VF(REQ, ATOMIC) - ] != 11; - wanted[ - `VF(AUTH, WRITE) - ] = current[ - `VF(REQ, WRITE) - ] || (current[ - `VF(REQ, ATOMIC) - ] != 0 && current[ - `VF(REQ, ATOMIC) - ] != 10); - wanted[`VF(AUTH, EXECUTE)] = current[`VF(REQ, EXECUTE)]; - wanted[`VF(AUTH, PRIVILEGE)] = current[`VF(REQ, PRIVILEGE)]; - end - if (auth_req.sample.bits !== wanted) - `uvm_fatal("AUTH_FIELDS", $sformatf( - "%s got=%h expected=%h", current_case, auth_req.sample.bits, wanted)) - auth_address = auth_req.sample.bits[`VF(AUTH, PADDR)]; - auth_allow = !(auth_is_pte ? (deny_pte && auth_address == denied_pte) : deny_final); - auth_pending = 1; - auth_due = cycle + 3 + (auth_queries % 5); - auth_queries++; - end - observations(); - if (mem_active && mem_resp.sample.valid && mem_resp.sample.ready) mem_active = 0; - if (uncached_active && uncached_resp.sample.valid && uncached_resp.sample.ready) begin - uncached_active = 0; - uncached_pending = 0; - end - if (mem_req.sample.valid && mem_req.sample.ready) begin - memory_entry e; - u64 address; - bit write, failed; - bit [511:0] line; - if ($isunknown( - { - mem_req.sample.bits[`VF(MEMREQ, ID)], - mem_req.sample.bits[`VF(MEMREQ, ADDR)], - mem_req.sample.bits[`VF(MEMREQ, WRITE)] - } - )) - `uvm_fatal("DDR_X", "Unknown DDR request") - address = mem_req.sample.bits[`VF(MEMREQ, ADDR)]; - write = mem_req.sample.bits[`VF(MEMREQ, WRITE)]; - if (address[5:0] != 0 || region(address, 64) != 1 || !backing.exists(address)) - `uvm_fatal("DDR_DICTIONARY", $sformatf("Unexpected/uninitialized DDR line %h", address - )) - failed = !write && line_error.exists(address) && line_error[address]; - if (write) begin - if ($isunknown( - { - mem_req.sample.bits[`VF(MEMREQ, DATA)], - mem_req.sample.bits[`VF(MEMREQ, MASK)] - } - )) - `uvm_fatal("DDR_WRITE_X", "Unknown DDR write") - line = mem_req.sample.bits[`VF(MEMREQ, DATA)]; - if (mem_req.sample.bits[`VF(MEMREQ, MASK)] !== '1 || line !== architectural[address]) - `uvm_fatal("DDR_WRITE", "Cache writeback corrupted full initialized line") - backing[address] = line; - memory_writes++; - end else memory_reads++; - e.bits = '0; - e.bits[`VF(MEMRESP, ID)] = mem_req.sample.bits[`VF(MEMREQ, ID)]; - e.bits[`VF(MEMRESP, DATA)] = backing[address]; - e.bits[`VF(MEMRESP, ERROR)] = failed; - e.due = cycle + 5 + (int'(e.bits[`VF(MEMRESP, ID)]) % 4); - pending.push_back(e); - end - if (uncached_req.sample.valid && uncached_req.sample.ready) begin - u64 address, value, next_pte; - int bytes, kind; - bit failed, write, is_pte; - address = uncached_req.sample.bits[`VF(UNCACHED, ADDR)]; - bytes = 1 << uncached_req.sample.bits[`VF(UNCACHED, SIZE)]; - write = uncached_req.sample.bits[`VF(UNCACHED, WRITE)]; - kind = region(address, bytes); - is_pte = !saw_physical && mmu_ref_pending(mmu) != 0; - if (uncached_pending || $isunknown(uncached_req.sample.bits)) - `uvm_fatal("UNCACHED_OWNER", "Overlapping or unknown uncached request") - if (is_pte) begin - logic [`VM_UNCACHED_WIDTH-1:0] wanted = '0; - if (!pte_permit || pte_permitted_address != address || kind != 3 || !mmu_ref_peek( - mmu, next_pte - ) || next_pte != address) - `uvm_fatal("PTE_CHAIN", "Wrong authorized uncached PTE address") - wanted[`VF(UNCACHED, ADDR)] = address; - wanted[`VF(UNCACHED, TAG)] = current[`VF(REQ, TAG)]; - wanted[`VF(UNCACHED, SIZE)] = 3; - wanted[`VF(UNCACHED, NORMAL)] = 1; - if (uncached_req.sample.bits !== wanted || !mmu_ref_consume(mmu, address)) - `uvm_fatal("PTE_REQUEST", "Uncached PTE contract mismatch") - pte_permit = 0; - pte_reads++; - uncached_pte_reads++; - end else begin - if(!expect_uncached||saw_uncached||!saw_physical||uncached_req.sample.bits!==uncached_expected) - `uvm_fatal("UNCACHED_REQUEST", $sformatf( - "%s unexpected uncached request", current_case)) - saw_uncached = 1; - end - if (kind != 2 && kind != 3) `uvm_fatal("UNCACHED_REGION", "Invalid uncached region") - failed = kind == 3 ? (line_error.exists(address & ~64'h3f) && line_error[address&~64'h3f - ]) : (mmio_error.exists(address) && mmio_error[address]); - value = 0; - for (int i = 0; i < bytes; i++) begin - if (kind == 3) begin - if (!backing.exists(address & ~64'h3f)) - `uvm_fatal("RAM_DICTIONARY", "Uninitialized normal RAM") - value |= u64'(backing[address&~64'h3f][int'((address+i)&63)*8+:8]) << (i * 8); - if (write && !failed) - backing[address&~64'h3f][int'((address+i)&63)*8+:8]=uncached_req.sample.bits[`VM_UNCACHED_DATA_OFFSET+i*8+:8]; - end else begin - if (!mmio_backing.exists(address + i)) - `uvm_fatal("MMIO_DICTIONARY", "Uninitialized device") - value |= u64'(mmio_backing[address+i]) << (i * 8); - if (write && !failed) - mmio_backing[address+i]=uncached_req.sample.bits[`VM_UNCACHED_DATA_OFFSET+i*8+:8]; - end - end - uncached_packet = '0; - uncached_packet[`VF(URESP, TAG)] = uncached_req.sample.bits[`VF(UNCACHED, TAG)]; - uncached_packet[`VF(URESP, DATA)] = write ? 0 : value; - uncached_packet[`VF(URESP, ERROR)] = failed; - uncached_due = cycle + 7; - uncached_pending = 1; - uncached_accesses++; - end - if (resp.sample.valid && resp.sample.ready) begin - result_item actual = new(current_case); - if (!case_active || $isunknown(resp.sample.bits)) - `uvm_fatal("RESULT", "Unknown or unsolicited virtual result") - actual.tag = resp.sample.bits[`VF(RESP, TAG)]; - actual.data = resp.sample.bits[`VF(RESP, DATA)]; - actual.misaligned = resp.sample.bits[`VF(RESP, MISALIGNED)]; - actual.page_fault = resp.sample.bits[`VF(RESP, PAGEFAULT)]; - actual.access_fault = resp.sample.bits[`VF(RESP, ACCESSFAULT)]; - if(saw_translation!=expect_translation||saw_physical!=expect_physical||saw_cache!=expect_cache||saw_uncached!=expect_uncached||mmu_ref_pending( - mmu - )) - `uvm_fatal("SIDE_EFFECT", - "Fault suppressed or successful operation skipped a required stage") - if (auth_pending || final_queried != (expect_translation && !translation_expected[ - `VF(TRANSLATION, PAGEFAULT) - ] && !translation_expected[ - `VF(TRANSLATION, ACCESSFAULT) - ])) - `uvm_fatal("AUTH_DRAIN", "Missing final authorization or response before decision") - scoreboard.actual_export.write(actual); - returned++; - case_active = 0; - if (actual.page_fault) page_faults++; - if (actual.access_fault) access_faults++; - if (actual.misaligned) misalignments++; - if (commit_write) - for (int i = 0; i < commit_bytes; i++) begin - u64 a = expected_pa + i; - if (expected_region == 1 || expected_region == 3) - architectural[a&~64'h3f][int'(a&63)*8+:8] = commit_data[i*8+:8]; - else mmio_arch[a] = commit_data[i*8+:8]; - end - end - end - @(negedge control.clock); - auth_req.ready = !control.reset && !auth_pending && cycle % 7 >= 4; - auth_resp.valid = !control.reset && auth_pending && cycle >= auth_due; - auth_resp.bits = auth_allow; - mem_req.ready = !control.reset && cycle % 7 >= 2; - uncached_req.ready = !control.reset && cycle % 9 >= 4; - resp.ready = !control.reset && !hold_response && cycle % 11 >= 4; - if (!mem_active) - for (int i = pending.size() - 1; i >= 0; i--) - if (pending[i].due <= cycle) begin - memory_packet = pending[i].bits; - pending.delete(i); - mem_active = 1; - break; - end - mem_resp.valid = mem_active; - mem_resp.bits = memory_packet; - if (uncached_pending && cycle >= uncached_due) uncached_active = 1; - uncached_resp.valid = uncached_active; - uncached_resp.bits = uncached_packet; - end - endtask - task run_case(string name, req_t packet, int sc_result = -1, bit perturb = 0, bit drain = 1); - current_case = name; - packet[`VF(REQ, TAG)] = expected & 63; - predict(packet, sc_result); - @(negedge control.clock); - req.bits = packet; - req.valid = 1; - do @(req.sample); while (!req.sample.ready); - @(negedge control.clock); - req.valid = 0; - if (perturb) req.bits = ~packet; - scoreboard.wait_checked(expected); - if (drain) begin - do - @(control.sample); - while (control.sample.outstanding != 0 || pending.size() || mem_active || uncached_pending); - repeat (8) @(control.sample); - end - endtask - task execute(); - bit [511:0] pattern; - u64 root, va, prior; - int sizes[4] = '{0, 1, 2, 3}; - auth_req.ready = 0; - auth_resp.valid = 0; - auth_resp.bits = 0; - control.conflict_trace = 0; - control.reset = 1; - control.active = 0; - req.valid = 0; - req.bits = '0; - resp.ready = 0; - mem_req.ready = 0; - mem_resp.valid = 0; - mem_resp.bits = '0; - uncached_req.ready = 0; - uncached_resp.valid = 0; - uncached_resp.bits = '0; - repeat (5) @(negedge control.clock); - control.reset = 0; - control.active = 1; - if (invalid_case >= 0) begin - req_t bad = command(1); - bad[`VF(REQ, TAG)] = 23; - case (invalid_case) - 0: bad[`VF(REQ, SIZE)] = 4; - 1: bad[`VF(REQ, ATOMIC)] = 12; - 2: begin - bad[`VF(REQ, ATOMIC)] = 2; - bad[`VF(REQ, WRITE)] = 1; - end - 3: begin - bad[`VF(REQ, ATOMIC)] = 2; - bad[`VF(REQ, SIZE)] = 1; - end - 4: bad[`VF(REQ, SATPMODE)] = 9; - 5: bad[`VF(REQ, PRIVILEGE)] = 2; - 6: begin - bad[`VF(REQ, EXECUTE)] = 1; - bad[`VF(REQ, WRITE)] = 1; - end - 7: begin - bad[`VF(REQ, EXECUTE)] = 1; - bad[`VF(REQ, ATOMIC)] = 10; - end - 8: begin - bad[`VF(REQ, EXECUTE)] = 1; - bad[`VF(REQ, SIZE)] = 2; - end - 9: begin - bad[`VF(REQ, EXECUTE)] = 1; - bad[`VF(REQ, SIGNED)] = 1; - end - default: `uvm_fatal("CASE", "Unknown virtual negative case") - endcase - req.bits = bad; - req.valid = 1; - `uvm_info("INVALID_INPUT", $sformatf( - "Driving illegal virtual command %0d with misaligned VA", - invalid_case), UVM_LOW) - repeat (10) @(negedge control.clock); - `uvm_fatal("MISSING_ASSERTION", "Illegal command hidden by early fault response") - end - for (int w = 0; w < 8; w++) - pattern[w*64+:64] = 64'h88776655fedcba80 ^ (64'h0101010101010101 * w); - initialize_line(`VM_DDR_BASE + 64'h4c0, pattern); - begin - bit [511:0] conflict_data = pattern; - conflict_data[63:0] = 64'hcafebabebeefaa11; - initialize_line(`VM_DDR_BASE + 64'h40c0, conflict_data); - end - initialize_line(`VM_READ_ONLY_BASE, pattern); - initialize_line(`VM_WRITE_ONLY_BASE, pattern); - initialize_line(DATA, pattern); - initialize_line(DATA + 4096, pattern); - initialize_line(DATA + 8192, pattern); - initialize_line(EVICT_DATA, pattern); - initialize_line(ERROR_DATA, pattern); - initialize_line(`VM_DDR_BASE + `VM_DDR_BYTES - 64, pattern); - line_error[ERROR_DATA&~64'h3f] = 1; - for (int i = 0; i < 32; i++) begin - mmio_arch[`VM_MMIO_BASE+i] = 8'h80 + i; - mmio_backing[`VM_MMIO_BASE+i] = 8'h80 + i; - end - for (int i = 0; i < 5; i++) begin - mmio_arch[`VM_SHORT_MMIO_BASE+i] = 8'hf0 + i; - mmio_backing[`VM_SHORT_MMIO_BASE+i] = 8'hf0 + i; - end - mmio_error[`VM_MMIO_BASE+24] = 1; - fork - service(); - join_none - repeat (12) @(control.sample); - // Match the actual Core conflict sequence without idle cycles or HN drain - // between returned requests: code and data have the same bank/index. - control.conflict_trace = 1; - run_case("conflict_code_first", command( - `VM_DDR_BASE + 64'h4c0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1), -1, 0, 0); - run_case("conflict_store_W", command(`VM_DDR_BASE + 64'h40c0, 2, 1, 0, 64'h7fffffff), -1, 0, - 0); - run_case("conflict_code_after_store", command( - `VM_DDR_BASE + 64'h4d8, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1), -1, 0, 0); - run_case("conflict_code_AMO_word", command( - `VM_DDR_BASE + 64'h4e0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1), -1, 0, 0); - run_case("conflict_AMO_W", command(`VM_DDR_BASE + 64'h40c0, 2, 0, 0, 1, 2), -1, 0, 0); - run_case("conflict_code_after_AMO", command( - `VM_DDR_BASE + 64'h4e0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1), -1, 0, 0); - run_case("conflict_data_readback", command(`VM_DDR_BASE + 64'h40c0), -1, 0, 0); - do - @(control.sample); - while (control.sample.outstanding || pending.size() || mem_active || uncached_pending); - if(control.conflict_wb!=2||control.conflict_dbid!=2||control.conflict_beats!=4||control.conflict_ack!=6) - `uvm_fatal("CONFLICT_CHI", $sformatf( - "Missing WB/DBID/data/ACK: %0d/%0d/%0d/%0d", - control.conflict_wb, - control.conflict_dbid, - control.conflict_beats, - control.conflict_ack - )) - control.conflict_trace = 0; - `uvm_info( - "CONFLICT", - "Code/store/code/AMO.W/code collision completed without idle-cycle insertion; all queues drained", - UVM_LOW) - // Hold the second request unchanged until the first response retires, then - // accept and check it normally; never withdraw a stalled VALID. - current_case = "single_outstanding_snapshot"; - begin - req_t first = command(DATA, 0, 0, 1); - req_t second = command(`VM_MMIO_BASE, 3, 1, 0, 64'hdeadbeef); - first[`VF(REQ, TAG)] = 31; - second[`VF(REQ, TAG)] = 17; - predict(first, -1); - hold_response = 1; - @(negedge control.clock); - req.bits = first; - req.valid = 1; - do @(req.sample); while (!req.sample.ready); - @(negedge control.clock); - req.bits = second; - wait (resp.sample.valid); - repeat (12) begin - @(req.sample); - if (req.sample.ready) - `uvm_fatal("OUTSTANDING", "Accepted a second command before retirement"); - end - @(negedge control.clock); - hold_response = 0; - scoreboard.wait_checked(expected); - @(negedge control.clock); - current_case = "pending_request_after_retirement"; - predict(second, -1); - do @(req.sample); while (!req.sample.ready); - @(negedge control.clock); - req.valid = 0; - scoreboard.wait_checked(expected); - do - @(control.sample); - while (control.sample.outstanding || pending.size() || mem_active || uncached_pending); - repeat (8) @(control.sample); - end - foreach (sizes[s]) - for (int offset = 0; offset < 8; offset += (1 << sizes[s])) - for (int sign_load = 0; sign_load < 2; sign_load++) - run_case("bare_lane_sign", command(DATA + offset, sizes[s], 0, sign_load)); - foreach (sizes[s]) begin - run_case("store_mask", command( - DATA + (sizes[s] == 0 ? 7 : sizes[s] == 1 ? 2 : sizes[s] == 2 ? 4 : 0), - sizes[s], - 1, - 0, - 64'hfe12345689abcdef - )); - run_case("masked_store_readback", command(DATA)); - end - // The numerical VA deliberately belongs to the opposite physical class. - va = 64'h10000180; - root = mapping(va, DATA); - run_case("VA_MMIO_maps_DDR", command(va, 3, 0, 0, 0, 0, 8, root), -1, 1); - run_case("LR_D", command(va, 3, 0, 0, 0, 10, 8, root)); - run_case("SC_D_success", command(va, 3, 0, 0, 64'h8899aabbccddeeff, 11, 8, root), 0); - sc_successes++; - run_case("LR_W_upper", command(va + 4, 2, 0, 0, 0, 10, 8, root)); - run_case("SC_W_success", command(va + 4, 2, 0, 0, 64'h7fffffff, 11, 8, root), 0); - sc_successes++; - for (int op = 1; op <= 9; op++) begin - run_case("AMO_W_upper", command(va + 4, 2, 0, 0, 64'h80000001 + op, op, 8, root)); - run_case("AMO_neighbor_preserved", command(va, 2, 0, 0, 0, 0, 8, root)); - end - run_case("dirty_private_eviction", command(DATA + 4096)); - run_case("dirty_shared_eviction_to_DDR", command(DATA + 8192)); - if (memory_writes == 0) - `uvm_fatal("DDR_WRITEBACK", "Dirty data never reached the explicit DDR dictionary") - run_case("DDR_writeback_readback", command(DATA)); - va = 64'h80000010; - root = mapping(va, `VM_MMIO_BASE + 16); - run_case("VA_DDR_maps_MMIO", command(va, 1, 0, 1, 0, 0, 8, root), -1, 1); - run_case("MMIO_store", command(va, 1, 1, 0, 64'hf123, 0, 8, root)); - run_case("MMIO_readback", command(va, 1, 0, 1, 0, 0, 8, root)); - run_case("MMIO_backend_error", command(`VM_MMIO_BASE + 24, 3)); - run_case("DDR_backend_error", command(ERROR_DATA)); - run_case("MMIO_atomic_denied", command(`VM_MMIO_BASE, 2, 0, 0, 1, 2)); - run_case("unknown_PA", command(64'h40000000)); - run_case("short_MMIO_last_byte", command(`VM_SHORT_MMIO_BASE + 4, 0, 0, 1)); - run_case("cross_region_word", command(`VM_SHORT_MMIO_BASE + 4, 2)); - run_case("cross_region_double", command(`VM_SHORT_MMIO_BASE, 3)); - run_case("DDR_last_byte", command(`VM_DDR_BASE + `VM_DDR_BYTES - 1, 0)); - run_case("DDR_after_end", command(`VM_DDR_BASE + `VM_DDR_BYTES, 0)); - run_case("misalign_before_missing_PTE", command(1, 3, 0, 0, 0, 0, 8, 0)); - run_case("noncanonical", command(64'h4000000000, 0, 0, 0, 0, 0, 8, 0)); - va = 64'h4180; - root = mapping(va, DATA, 64'h43); - run_case("LR_read_only", command(va, 3, 0, 0, 0, 10, 8, root)); - for (int op = 1; op <= 9; op++) - run_case("AMO_needs_store_permission", command(va, 3, 0, 0, 1, op, 8, root)); - run_case("SC_needs_store_permission", command(va, 3, 0, 0, 1, 11, 8, root)); - root = mapping(va, DATA, 64'h4f); - run_case("Svade_dirty_fault", command(va, 3, 1, 0, 1, 0, 8, root)); - root = mapping(va, DATA, 64'hf); - run_case("Svade_A_bit_page_fault", command(va, 3, 0, 0, 0, 0, 8, root)); - root = mapping(va, DATA, 64'h59); - run_case("SUM_denied", command(va, 3, 0, 0, 0, 0, 8, root, 1, 0, 1)); - run_case("SUM_MXR_allowed", command(va, 3, 0, 0, 0, 0, 8, root, 1, 1, 1), -1, 1); - run_case("U_load_ignores_SUM", command(va, 3, 0, 0, 0, 0, 8, root, 0, 0, 1)); - root = mapping(va, DATA, 64'hcf); - run_case("U_cannot_read_supervisor_page", command(va, 3, 0, 0, 0, 0, 8, root, 0)); - run_case("M_bypasses_satp", command(DATA, 3, 0, 0, 0, 0, 8, 0, 3)); - root = mapping(64'hffffffc000000180, DATA); - run_case("negative_canonical_kernel_VA", command(64'hffffffc000000180, 3, 0, 0, 0, 0, 8, root - )); - root = mapping(va, DATA, 64'hcf, 1); - run_case("PTE_backend_error", command(va, 3, 0, 0, 0, 0, 8, root)); - mmu_ref_program(mmu, `VM_MMIO_BASE, 0, 1); - run_case("PTE_cannot_access_MMIO", command(va, 3, 0, 0, 0, 0, 8, `VM_MMIO_BASE >> 12)); - mmu_ref_program(mmu, 64'h20000000, 0, 1); - run_case("unknown_PTE_region", command(va, 3, 0, 0, 0, 0, 8, 64'h20000)); - root = mapping(va, 64'h40000180); - run_case("translated_unknown_PA", command(va, 3, 0, 0, 0, 0, 8, root)); - root = mapping(va, 64'h1 << `VM_CACHE_ADDR_WIDTH); - run_case("translated_PA_overflow", command(va, 3, 0, 0, 0, 0, 8, root)); - // Instruction words share the real PTE/data cache path; Fetch owns RVC assembly. - va = 64'h4180; - root = mapping(va, DATA, 64'h49); - run_case("X_only_data_denied", command(va, 3, 0, 0, 0, 0, 8, root)); - run_case("X_only_instruction_word", command(va, 3, 0, 0, 0, 0, 8, root, 1, 0, 0, 1), -1, 1); - run_case("U_execute_supervisor_denied", command(va, 3, 0, 0, 0, 0, 8, root, 0, 1, 1, 1)); - root = mapping(va, DATA, 64'h59); - run_case("U_execute_user_word", command(va, 3, 0, 0, 0, 0, 8, root, 0, 0, 0, 1)); - run_case("S_execute_user_SUM_denied", command(va, 3, 0, 0, 0, 0, 8, root, 1, 1, 1, 1)); - root = mapping(va, DATA, 64'h43); - run_case("readable_nonX_instruction_denied", command(va, 3, 0, 0, 0, 0, 8, root, 1, 0, 1, 1)); - run_case("instruction_MMIO_no_side_effect", command( - `VM_MMIO_BASE, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1)); - root = mapping(va, `VM_MMIO_BASE + 384, 64'h49); - run_case("translated_instruction_MMIO_denied", command(va, 3, 0, 0, 0, 0, 8, root, 1, 0, 0, 1 - )); - // Two explicit aligned requests straddle a page; this owner never splits accesses. - root = mapping(64'h4ff8, DATA, 64'h49); - initialize_line((DATA & ~64'hfff) + 64'hfc0, pattern); - run_case("page_last_instruction_word", command(64'h4ff8, 3, 0, 0, 0, 0, 8, root, 1, 0, 0, 1)); - // The next PTE is explicitly initialized invalid in the same fixture line. - mmu_ref_program(mmu, (root << 12) + 8192 + 5 * 8, 0, 0); - run_case("page_second_instruction_PF", command(64'h5000, 3, 0, 0, 0, 0, 8, root, 1, 0, 0, 1)); - root = mapping(64'h5000, 64'h40000000, 64'h49); - run_case("page_second_instruction_AF", command(64'h5000, 3, 0, 0, 0, 0, 8, root, 1, 0, 0, 1)); - run_case("PMA_read_only_load", command(`VM_READ_ONLY_BASE)); - run_case("PMA_read_only_store_denied", command(`VM_READ_ONLY_BASE, 3, 1, 0, 64'hbadbad)); - run_case("PMA_read_only_AMO_denied", command(`VM_READ_ONLY_BASE, 2, 0, 0, 1, 2)); - run_case("PMA_cached_nonexecutable", command( - `VM_READ_ONLY_BASE, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1)); - run_case("PMA_write_only_read_denied", command(`VM_WRITE_ONLY_BASE)); - // Authorization uses a policy captured for this accepted command. Denials - // reuse the MMU backend-error result contract without modifying DDR data. - deny_final = 1; - run_case("authorization_deny_load", command(DATA), -1, 1); - run_case("authorization_deny_store", command(DATA, 3, 1, 0, 64'hdeadbeef), -1, 1); - run_case("authorization_deny_AMO", command(DATA, 2, 0, 0, 1, 2), -1, 1); - run_case("authorization_deny_execute", command(DATA, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1), -1, 1); - run_case("authorization_deny_MMIO", command(`VM_MMIO_BASE, 3, 1, 0, 64'hbadbad), -1, 1); - deny_final = 0; - run_case("denied_writes_preserve_DDR", command(DATA)); - run_case("denied_write_preserves_MMIO", command(`VM_MMIO_BASE)); - for (int denied_level = 0; denied_level < 3; denied_level++) begin - va = 64'h4180; - root = mapping(va, DATA); - denied_pte=(root<<12)+(2-denied_level)*4096+((va>>(12+9*denied_level))&511)*8; - deny_pte = 1; - run_case("authorization_deny_PTE", command(va, 3, 0, 0, 0, 0, 8, root), -1, 1); - deny_pte = 0; - run_case("PTE_authorization_reenabled", command(va, 3, 0, 0, 0, 0, 8, root)); - end - if (auth_denials != 8 || !auth_stalls || auth_pending) - `uvm_fatal("AUTH_SCENARIOS", "Authorization denials/stalls/drain missing") - `uvm_info( - "AUTHORIZATION", - $sformatf( - "queries=%0d denies=%0d queryStalls=%0d; PTE/final fields and no-before-grant backend checked", - auth_queries, auth_denials, auth_stalls), UVM_LOW) - // No reservation timing model: require a real clean Evict of the LR line - // during the subsequent walk and independently check SC's architectural 1. - va = 64'h4000; - root = mapping(va, EVICT_DATA); - run_case("LR_before_PTE_eviction", command(EVICT_DATA, 3, 0, 0, 0, 10)); - prior = evictions.exists(EVICT_DATA) ? evictions[EVICT_DATA] : 0; - run_case("SC_after_real_PTE_eviction", command(va, 3, 0, 0, 64'h1234, 11, 8, root), 1); - if (!evictions.exists(EVICT_DATA) || evictions[EVICT_DATA] <= prior) - `uvm_fatal("SC_EVICTION", "No real cache eviction witnessed between LR and failed SC") - sc_evictions++; - run_case("failed_SC_preserves_data", command(EVICT_DATA)); - initialize_line(`VM_NORMAL_RAM_BASE, 512'hfedcba98765432100123456789abcdef); - run_case("normal_RAM_load", command(`VM_NORMAL_RAM_BASE)); - run_case("normal_RAM_execute", command(`VM_NORMAL_RAM_BASE, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1)); - run_case("normal_RAM_store", command(`VM_NORMAL_RAM_BASE + 8, 3, 1, 0, 64'h8123456789abcdef)); - run_case("normal_RAM_store_readback", command(`VM_NORMAL_RAM_BASE + 8)); - next_table = `VM_NORMAL_RAM_BASE + 64'h10000; - va = 64'h12345000; - root = mapping(va, `VM_NORMAL_RAM_BASE); - run_case("uncached_PTE_uncached_data", command(va, 3, 0, 0, 0, 0, 8, root), -1, 1); - run_case("uncached_PTE_execute", command(va, 3, 0, 0, 0, 0, 8, root, 1, 0, 0, 1)); - va = 64'h12345180; - root = mapping(va, DATA); - run_case("uncached_PTE_cached_data", command(va, 3, 0, 0, 0, 0, 8, root)); - root = mapping(va, DATA, 8'h0f); - run_case("uncached_PTE_missing_AD_fault", command(va, 3, 0, 0, 0, 0, 8, root)); - root = mapping(va, DATA); - begin - u64 address = (root << 12) + ((va >> 30) & 511) * 8; - program_pte(address, read_arch(address, 8, 3), 1); - end - run_case("uncached_PTE_backend_error", command(va, 3, 0, 0, 0, 0, 8, root)); - root = mapping(va, DATA); - denied_pte = (root << 12) + ((va >> 30) & 511) * 8; - deny_pte = 1; - run_case("uncached_PTE_authorization_denied", command(va, 3, 0, 0, 0, 0, 8, root), -1, 1); - deny_pte = 0; - line_error[`VM_NORMAL_RAM_BASE] = 1; - run_case("normal_RAM_execute_error", command( - `VM_NORMAL_RAM_BASE, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1)); - run_case("normal_RAM_load_error", command(`VM_NORMAL_RAM_BASE)); - if (uncached_pte_reads < 10) - `uvm_fatal("UNCACHED_PTE", "Uncached page walk cases were not exercised") - if(expected!=accepted||expected!=returned||!pte_reads||!data_accesses||!uncached_accesses||!req_stalls||!resp_stalls||!mem_stalls||!uncached_stalls||!page_faults||!access_faults||!misalignments||!sc_successes||!sc_evictions) - `uvm_fatal("SCENARIOS", "Virtual LSU scenarios not all exercised") - `uvm_info( - "CHECKS", - $sformatf( - "virtual LSU checked=%0d realCachePTE=%0d dataCache=%0d MMIO=%0d DDRreads/writes=%0d/%0d PF/AF/misaligned=%0d/%0d/%0d SCsuccess/evicted=%0d/%0d stalls(req/resp/DDR/MMIO)=%0d/%0d/%0d/%0d; all queues drained", - expected, pte_reads, data_accesses, uncached_accesses, memory_reads, memory_writes, - page_faults, access_faults, misalignments, sc_successes, sc_evictions, req_stalls, - resp_stalls, mem_stalls, uncached_stalls), UVM_LOW) - endtask - function void final_phase(uvm_phase phase); - mmu_ref_destroy(mmu); - endfunction - endclass -endpackage diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_tb.sv b/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_tb.sv deleted file mode 100644 index d5b4387ba..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/resources/virtual_cache_tb.sv +++ /dev/null @@ -1,102 +0,0 @@ -`include "virtual_cache_config.svh" -`ifndef VIRTUAL_CACHE_TB -`define VIRTUAL_CACHE_TB virtual_cache_tb -`endif -module `VIRTUAL_CACHE_TB; - import uvm_pkg::*; - import virtual_cache_pkg::*; - logic clock = 0; - always #5 clock = ~clock; - virtual_cache_if control (clock); - stream_if #(`VM_AUTH_WIDTH) auth_req ( - clock, - control.reset - ); - stream_if #(1) auth_resp ( - clock, - control.reset - ); - stream_if #(`VM_REQ_WIDTH) source_if ( - clock, - control.reset - ); - stream_if #(`VM_RESP_WIDTH) sink_if ( - clock, - control.reset - ); - stream_if #(`VM_MEMREQ_WIDTH) mem_req ( - clock, - control.reset - ); - stream_if #(`VM_MEMRESP_WIDTH) mem_resp ( - clock, - control.reset - ); - stream_if #(`VM_UNCACHED_WIDTH) uncached_req ( - clock, - control.reset - ); - stream_if #(`VM_URESP_WIDTH) uncached_resp ( - clock, - control.reset - ); - VirtualCacheSystem dut ( - .clock(clock), - .reset(control.reset), - .io_active(control.active), - .io_outstanding(control.outstanding), - `include "virtual_cache_ports.svh" - ); - // Observe existing HN boundary; no DUT logic is added. rxDat/rxRsp ready - // are structurally true and optimized out of emitted RTL. - always @(posedge clock) begin - if (control.reset) begin - control.conflict_wb = 0; - control.conflict_dbid = 0; - control.conflict_beats = 0; - control.conflict_ack = 0; - end else if (control.conflict_trace) begin - if (dut.caches.home.home.io_req_valid && dut.caches.home.home.io_req_ready) begin - if (dut.caches.home.home.io_req_bits_opcode == 7'h1b) control.conflict_wb++; - $display("[CONFLICT_CHI] REQ op=%h txn=%h addr=%h", dut.caches.home.home.io_req_bits_opcode, - dut.caches.home.home.io_req_bits_txnId, dut.caches.home.home.io_req_bits_addr); - end - if (dut.caches.home.home.io_rsp_valid && dut.caches.home.home.io_rsp_ready) begin - if (dut.caches.home.home.io_rsp_bits_opcode == 5'h05) control.conflict_dbid++; - $display("[CONFLICT_CHI] RSP op=%h txn=%h dbid=%h", dut.caches.home.home.io_rsp_bits_opcode, - dut.caches.home.home.io_rsp_bits_txnId, dut.caches.home.home.io_rsp_bits_dbid); - end - if (dut.caches.home.home.io_rxDat_valid && dut.caches.home.home.io_rxDat_bits_opcode==4'h2) begin - control.conflict_beats++; - $display( - "[CONFLICT_CHI] RXDAT op=%h txn=%h beat=%h", dut.caches.home.home.io_rxDat_bits_opcode, - dut.caches.home.home.io_rxDat_bits_txnId, dut.caches.home.home.io_rxDat_bits_dataId); - end - if (dut.caches.home.home.io_rxRsp_valid && dut.caches.home.home.io_rxRsp_bits_opcode==5'h02) begin - control.conflict_ack++; - $display("[CONFLICT_CHI] RXRSP op=%h txn=%h", dut.caches.home.home.io_rxRsp_bits_opcode, - dut.caches.home.home.io_rxRsp_bits_txnId); - end - end - end - initial begin - int invalid_case; - uvm_config_db#(virtual stream_if #(`VM_AUTH_WIDTH))::set(null, "uvm_test_top", "auth_req", - auth_req); - uvm_config_db#(virtual stream_if #(1))::set(null, "uvm_test_top", "auth_resp", auth_resp); - uvm_config_db#(virtual virtual_cache_if)::set(null, "uvm_test_top", "control", control); - uvm_config_db#(virtual stream_if #(`VM_REQ_WIDTH))::set(null, "uvm_test_top", "req", source_if); - uvm_config_db#(virtual stream_if #(`VM_RESP_WIDTH))::set(null, "uvm_test_top", "resp", sink_if); - uvm_config_db#(virtual stream_if #(`VM_MEMREQ_WIDTH))::set(null, "uvm_test_top", "mem_req", - mem_req); - uvm_config_db#(virtual stream_if #(`VM_MEMRESP_WIDTH))::set(null, "uvm_test_top", "mem_resp", - mem_resp); - uvm_config_db#(virtual stream_if #(`VM_UNCACHED_WIDTH))::set(null, "uvm_test_top", - "uncached_req", uncached_req); - uvm_config_db#(virtual stream_if #(`VM_URESP_WIDTH))::set(null, "uvm_test_top", "uncached_resp", - uncached_resp); - if ($value$plusargs("invalid_case=%d", invalid_case)) - uvm_config_db#(int)::set(null, "uvm_test_top", "invalid_case", invalid_case); - run_test("protocol_test"); - end -endmodule diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/TaskAdmission.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/TaskAdmission.scala deleted file mode 100644 index 64c67e690..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/TaskAdmission.scala +++ /dev/null @@ -1,55 +0,0 @@ -package hier.tile - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import hier.core.rocket.CommandSnapshot -import framework.system.core.rocket.{RoCCIO, RoCCResponseBB} -import memcore.memory.interlock.{Params => TrackingParams, Tag} - -/** Routes an accepted task command without allocating an accelerator ROB entry. */ -@instantiable -class TaskAdmission(tracking: TrackingParams, pmps: Int)(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - - @public - val io = IO(new Bundle { - val command = Flipped(Decoupled(new CommandSnapshot(tracking, pmps))) - val task = Flipped(new RoCCIO(64)) - val response = Decoupled(new RoCCResponseBB(64)) - val release = Decoupled(new Tag(tracking)) - val satp = Output(UInt(64.W)) - // Worker completion cannot publish a result while its accepted NPU/DMA/cache work remains live. - val workDrained = Input(Bool()) - }) - - val idle :: responding :: releasing :: Nil = Enum(3) - val state = RegInit(idle) - val tag = Reg(UInt(tracking.idBits.W)) - val satp = Reg(UInt(64.W)) - val operation = io.command.bits.instruction.funct - val finishTask = operation === 2.U || operation === 14.U || - (operation >= 16.U && operation <= 19.U) - val eligible = state === idle && (!finishTask || io.workDrained) && !reset.asBool - io.task.cmd.valid := io.command.valid && eligible - io.task.cmd.bits := io.command.bits.instruction - io.command.ready := io.task.cmd.ready && eligible - io.task.exception := false.B - when(io.command.fire) { - assert(io.command.bits.instruction.opcode === "h2b".U, "TaskAdmission received a compute instruction") - tag := io.command.bits.tag - satp := io.command.bits.satp - state := responding - } - // TaskController samples the root on the command handshake, including the first staged field. - io.satp := Mux(state === idle && io.command.valid, io.command.bits.satp, satp) - io.response.valid := state === responding && io.task.resp.valid && !reset.asBool - io.response.bits := io.task.resp.bits - io.task.resp.ready := state === responding && io.response.ready && !reset.asBool - when(io.response.fire)(state := releasing) - io.release.valid := state === releasing && !reset.asBool - io.release.bits.tag := tag - when(io.release.fire)(state := idle) -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/TaskController.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/TaskController.scala deleted file mode 100644 index f62d0a3dc..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/TaskController.scala +++ /dev/null @@ -1,108 +0,0 @@ -package hier.tile - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.core.rocket.RoCCIO - -/** Linux submits compiled kernels; workers consume contexts without joining Linux SMP. */ -@instantiable -class TaskController(coreIds: Seq[Int], signatures: Seq[BigInt], cpuCount: Int) extends Module { - require(coreIds == (1 to signatures.size) && signatures.nonEmpty) - require(cpuCount == coreIds.size + 1, "task workers must include every non-controller CPU") - - @public - val io = IO(new Bundle { - val ports = Vec(cpuCount, new RoCCIO(64)) - val satp = Input(UInt(64.W)) - }) - - val count = coreIds.size - val kinds = VecInit(signatures.map(_.U(64.W))) - val contexts = Reg(Vec(count, Vec(7, UInt(64.W)))) - val roots = Reg(Vec(count, UInt(64.W))) - val active = RegInit(VecInit(Seq.fill(count)(false.B))) - val available = RegInit(VecInit(Seq.fill(count)(false.B))) - val results = RegInit(VecInit(Seq.fill(count)(1.U(2.W)))) - val staged = Reg(Vec(7, UInt(64.W))) - val stagedRoot = Reg(UInt(64.W)) - val stagedFields = RegInit(0.U(3.W)) - val workspace = RegInit(0.U(64.W)) - - for (cpu <- 0 until cpuCount) { - val port = io.ports(cpu) - val worker = coreIds.indexOf(cpu) - val pending = RegInit(false.B) - val rd = Reg(UInt(5.W)) - val response = Reg(UInt(64.W)) - val command = port.cmd.bits - val op = command.funct - val index = command.rs1Data - val value = command.rs2Data - val slot = index(math.max(1, log2Ceil(count)) - 1, 0) - val field = index(2, 0) - val inRange = index < count.U - val free = VecInit((0 until count).map(i => !active(i) && kinds(i) === value)).asUInt.orR - val waitReady = - if (cpu == 0) - Mux(op === 1.U, Mux(index === count.U, free, !inRange || !active(slot)), true.B) - else if (worker >= 0) Mux(op === 8.U, available(worker), true.B) - else true.B - port.cmd.ready := !pending && waitReady - port.resp.valid := pending - port.resp.bits.rd := rd - port.resp.bits.data := response - port.busy := pending - port.interrupt := false.B - when(port.resp.fire)(pending := false.B) - when(port.cmd.fire) { - pending := true.B - rd := command.rd - response := 2.U - if (cpu == 0) { - switch(op) { - is(0.U) { - when(inRange && !active(slot) && stagedFields === 7.U && - stagedRoot === io.satp && kinds(slot) === value && staged(6) === value) { - contexts(slot) := staged - roots(slot) := stagedRoot - active(slot) := true.B - available(slot) := true.B - results(slot) := 0.U - stagedFields := 0.U - response := 0.U - } - } - is(1.U)(response := Mux(index === count.U, 0.U, Mux(inRange, results(slot), 2.U))) - is(3.U)(response := count.U) - is(4.U)(response := Mux(inRange, kinds(slot), 0.U)) - is(5.U)(response := workspace) - is(6.U) { workspace := index; response := 0.U } - is(7.U) { - when(index === 0.U || (index < 7.U && index === stagedFields && stagedRoot === io.satp)) { - staged(field) := value - stagedFields := field + 1.U - when(index === 0.U)(stagedRoot := io.satp) - response := 0.U - } - } - is(10.U)(response := Mux(inRange, results(slot), 2.U)) - } - } else if (worker >= 0) { - switch(op) { - is(2.U) { - when(active(worker) && !available(worker)) { - active(worker) := false.B - results(worker) := Mux(index === 0.U, 1.U, 2.U) - response := 0.U - } - } - is(5.U)(response := contexts(worker)(4)) - is(11.U)(when(active(worker) && !available(worker))(response := 0.U)) - is(8.U) { available(worker) := false.B; response := 1.U } - is(9.U)(response := Mux(index === 7.U, roots(worker), Mux(index < 7.U, contexts(worker)(field), 0.U))) - } - } - } - } -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/CacheSystem.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/CacheSystem.scala deleted file mode 100644 index 16bc528b3..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/CacheSystem.scala +++ /dev/null @@ -1,59 +0,0 @@ -package hier.tile.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi._ -import memcore.bus.chi.rnf.{BankedChiCache, CacheAccess, CacheResult, RnfParams} -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.coherence.configs.CoherenceParams - -/** Independent canonical cache composition with physical active-link CHI channels. */ -@instantiable -class CacheSystem(p: CoherenceParams) extends Module { - require(p.agents == 2 && p.homeId == 64 && p.mshrEntries == 4) - val c = p.chi - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - val blockRequesterRsp = Input(Bool()) - val access = Vec(p.agents, Flipped(Decoupled(new CacheAccess(c)))) - val result = Vec(p.agents, Decoupled(new CacheResult)) - val memoryReq = Decoupled(new LineRequest(c)) - val memoryResp = Flipped(Decoupled(new LineResponse(c))) - val outstanding = Output(UInt(log2Ceil(p.mshrEntries + 1).W)) - val observedReq = Output(Valid(new RequestFlit(c))) - val observedRsp = Output(Valid(new ResponseFlit(c))) - val observedDat = Output(Valid(new DataFlit(c))) - val observedSnp = Output(Valid(new DirectedSnoop(c))) - val observedRxRsp = Output(Valid(new ResponseFlit(c))) - val observedRxDat = Output(Valid(new DataFlit(c))) - }) - - val home: Instance[Home] = Instantiate(new Home(p)) - - val cores: Seq[Instance[BankedChiCache]] = Seq.tabulate(p.agents)(i => - Instantiate(new BankedChiCache( - RnfParams(c, nodeId = i + 1, cacheLines = 8, homeId = p.homeId, homeCount = 1, banks = 2) - )) - ) - - home.io.active := io.active - home.io.blockRequesterRsp := io.blockRequesterRsp - io.memoryReq <> home.io.memoryReq - home.io.memoryResp <> io.memoryResp - io.outstanding := home.io.outstanding - for (i <- 0 until p.agents) { - cores(i).io.access <> io.access(i) - io.result(i) <> cores(i).io.result - home.io.requesters(i) <> cores(i).io.chi - } - io.observedReq := home.io.observedReq - io.observedRsp := home.io.observedRsp - io.observedDat := home.io.observedDat - io.observedSnp := home.io.observedSnp - io.observedRxRsp := home.io.observedRxRsp - io.observedRxDat := home.io.observedRxDat - -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Composition.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Composition.scala deleted file mode 100644 index 73d8bc78e..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Composition.scala +++ /dev/null @@ -1,175 +0,0 @@ -package hier.tile.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.CpuParams -import hier.core.rocket.{AdmissionPorts, Commands, CpuCLink} -import hier.tile.TaskController -import framework.system.core.rocket.RoCCIO -import memcore.bus.chi.RequesterPort -import memcore.bus.chi.rnf.RnfParams -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion, UncachedRequest, UncachedResponse} -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.interlock.{Params => TrackingParams} - -/** Roles describe command capabilities; a scheduler may also have compute capability. */ -case class CoreRole(compute: Boolean, scheduler: Boolean) { - require(compute || scheduler) -} - -object CoreRole { - val Controller = CoreRole(compute = false, scheduler = true) - val Compute = CoreRole(compute = true, scheduler = false) -} - -case class CorePlacement( - l1: RnfParams, - cpu: CpuParams, - role: CoreRole) - -class CoreInterrupts extends Bundle { - val timer = Bool() - val software = Bool() - val external = Bool() - val supervisorExternal = Bool() -} - -/** Explicit CPU and task assembly; the Chip owner supplies each Core admission. */ -@instantiable -class Composition( - memory: CoherenceParams, - placements: Seq[CorePlacement], - regions: Seq[PhysicalRegion], - tracking: TrackingParams, - workerCoreIds: Seq[Int], - signatures: Seq[BigInt], - controlCoreIds: Seq[Int] = Nil) - extends Module { - val localCoreIds = placements.indices.filterNot(controlCoreIds.contains) - require(controlCoreIds.distinct.size == controlCoreIds.size) - require(controlCoreIds.forall(placements.indices.contains)) - // Control requesters leave the tile; the remaining cores use the local Home. - require(placements.nonEmpty && 2 * placements.size == memory.agents) - require(workerCoreIds.size == signatures.size && signatures.forall(s => s >= 0 && s < (BigInt(1) << 64))) - require( - workerCoreIds.forall(i => i > 0 && i < placements.size), - "Task worker IDs must explicitly name non-controller placements" - ) - require(placements.head.role.scheduler, "TaskController port zero must be a scheduler-capable placement") - val c = memory.chi - val cp = CpuMemParams(c, tagBits = 6) - require(tracking.addressBits == c.addressBits) - placements.zipWithIndex.foreach { case (entry, index) => - require(entry.l1.chi == c && entry.l1.homeId == memory.homeId && entry.l1.homeCount == 1) - // Memory/Fabric routes node i+1. This is a CHI node contract, never a hart-ID derivation. - require(entry.l1.nodeId == index + 1, "Core CHI node must match its Memory requester port") - } - - @public - val io = IO(new Bundle { - - // Caller routes opcode 0x2b commands and supplies their captured controller SATP. - val cores = MixedVec(placements.zipWithIndex.map { case (entry, index) => - val controlIndex = controlCoreIds.indexOf(index) - val localIndex = localCoreIds.indexOf(index) - val node = if (controlIndex >= 0) controlIndex + 1 else localIndex + 1 - new CpuCLink(entry.l1.copy(nodeId = node), Some(Commands(entry.role.compute, entry.role.scheduler, tracking)))( - entry.cpu - ) - }.map(Flipped(_))) - - val taskControl = Vec(placements.size, new RoCCIO(64)) - val controllerSatp = Input(UInt(64.W)) - val hartIds = Input(Vec(placements.size, UInt(64.W))) - val resetVector = Input(Vec(placements.size, UInt(64.W))) - val time = Input(UInt(64.W)) - val interrupts = Input(Vec(placements.size, new CoreInterrupts)) - val admission = - MixedVec(placements.map(entry => new AdmissionPorts(tracking, entry.cpu.core.nPMPs, c)(entry.cpu))) - val uncachedRequest = Vec(placements.size, Decoupled(new UncachedRequest(cp))) - val uncachedResponse = Vec(placements.size, Flipped(Decoupled(new UncachedResponse(cp)))) - val control = Vec(2 * controlCoreIds.size, new RequesterPort(c)) - val backingReq = Vec(if (localCoreIds.nonEmpty) 1 else 0, Decoupled(new LineRequest(c))) - val backingResp = Vec(if (localCoreIds.nonEmpty) 1 else 0, Flipped(Decoupled(new LineResponse(c)))) - val homeOutstanding = Output(UInt(log2Ceil(memory.mshrEntries + 1).W)) - val retired = Output(Vec(placements.size, Bool())) - val retiredPc = Output(Vec(placements.size, UInt(64.W))) - val trapped = Output(Vec(placements.size, Bool())) - val trapCause = Output(Vec(placements.size, UInt(64.W))) - val trapValue = Output(Vec(placements.size, UInt(64.W))) - val trapPc = Output(Vec(placements.size, UInt(64.W))) - }) - - if (workerCoreIds.nonEmpty) { - val taskController: Instance[TaskController] = - Instantiate(new TaskController(workerCoreIds, signatures, placements.size)) - taskController.io.ports <> io.taskControl - taskController.io.satp := io.controllerSatp - } else { - // A Tile without task workers reports zero workers (op 3) and an invalid index (2) otherwise. - for (port <- io.taskControl) { - val pending = RegInit(false.B) - val rd = Reg(UInt(5.W)) - val response = Reg(UInt(64.W)) - port.cmd.ready := !pending - port.resp.valid := pending - port.resp.bits.rd := rd - port.resp.bits.data := response - port.busy := pending - port.interrupt := false.B - when(port.cmd.fire) { - pending := true.B - rd := port.cmd.bits.rd - response := Mux(port.cmd.bits.funct === 3.U, 0.U, 2.U) - } - when(port.resp.fire)(pending := false.B) - } - } - for (port <- io.taskControl) { - when(port.cmd.valid) { - assert(port.cmd.bits.opcode === "h2b".U, "Composition task port requires an opcode 0x2b command") - } - } - - val memorySystem = Option.when(localCoreIds.nonEmpty)( - Instantiate(new Memory(memory.copy(agents = 2 * localCoreIds.size))) - ) - - io.homeOutstanding := 0.U - memorySystem.foreach { m => - io.backingReq <> m.io.backingReq - m.io.backingResp <> io.backingResp - io.homeOutstanding := m.io.homeOutstanding - } - - for (index <- placements.indices) { - val controlIndex = controlCoreIds.indexOf(index) - val localIndex = localCoreIds.indexOf(index) - val core = io.cores(index) - core.hartId := io.hartIds(index) - core.resetVector := io.resetVector(index) - core.time := io.time - core.timerInterrupt := io.interrupts(index).timer - core.softwareInterrupt := io.interrupts(index).software - core.externalInterrupt := io.interrupts(index).external - core.supervisorExternalInterrupt := io.interrupts(index).supervisorExternal - io.admission(index) <> core.admission.get - io.uncachedRequest(index) <> core.uncachedRequest - core.uncachedResponse <> io.uncachedResponse(index) - if (controlIndex >= 0) { - io.control(controlIndex) <> core.chi - io.control(controlCoreIds.size + controlIndex) <> core.instructionChi - } else { - memorySystem.get.io.coherent(localIndex) <> core.chi - memorySystem.get.io.coherent(localCoreIds.size + localIndex) <> core.instructionChi - } - io.retired(index) := core.retired - io.retiredPc(index) := core.retiredPc - io.trapped(index) := core.trapped - io.trapCause(index) := core.trapCause - io.trapValue(index) := core.trapValue - io.trapPc(index) := core.trapPc - } -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Consistency.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Consistency.scala deleted file mode 100644 index 08b91ef24..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Consistency.scala +++ /dev/null @@ -1,91 +0,0 @@ -package hier.tile.memory - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import hier.core.rocket.{Cache => CoreCache} -import memcore.bus.chi.{DataFlit, DirectedSnoop, RequestFlit} -import memcore.bus.chi.rnf.{BankedChiCache, CacheAccess, CacheAtomic, CacheResult, RnfParams} -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.interlock.{AccessInfo, Acknowledgement, Dispatch, Interlock, Params, Tag} - -/** Physical-access verification composition: actual L1s, one shared L2 and one core-local interlock. */ -@instantiable -class Consistency(memory: CoherenceParams, l1: RnfParams, tracking: Params) extends Module { - require(memory.agents >= 2 && l1.homeCount == 1 && l1.homeId == memory.homeId) - require(l1.chi == memory.chi && tracking.addressBits == memory.chi.addressBits) - val c = memory.chi - - @public - val io = IO(new Bundle { - val access = Flipped(Vec(memory.agents, Decoupled(new CacheAccess(c)))) - val result = Vec(memory.agents, Decoupled(new CacheResult)) - val dispatch = Flipped(Decoupled(new Dispatch(tracking))) - val accessInfo = Flipped(Decoupled(new AccessInfo(tracking))) - val grant = Decoupled(new Tag(tracking)) - val done = Flipped(Decoupled(new Acknowledgement(tracking))) - val complete = Decoupled(new Tag(tracking)) - val olderDispatchPending = Input(Bool()) - val olderRequestsDrained = Input(Bool()) - val blockRequesterRsp = Input(Bool()) - val blockRequesterData = Input(Bool()) - val cpuAllow = Output(Bool()) - val memoryReq = Decoupled(new LineRequest(c)) - val memoryResp = Flipped(Decoupled(new LineResponse(c))) - val observedReq = Output(Valid(new RequestFlit(c))) - val observedSnp = Output(Valid(new DirectedSnoop(c))) - val outstanding = Output(UInt(log2Ceil(memory.mshrEntries + 1).W)) - }) - - val home: Instance[Home] = Instantiate(new Home(memory)) - home.io.active := true.B - home.io.blockRequesterRsp := io.blockRequesterRsp - io.memoryReq <> home.io.memoryReq - home.io.memoryResp <> io.memoryResp - io.observedReq := home.io.observedReq - io.observedSnp := home.io.observedSnp - io.outstanding := home.io.outstanding - - val interlock: Instance[Interlock] = Instantiate(new Interlock(tracking)) - interlock.io.cancel.valid := false.B - interlock.io.cancel.bits := 0.U.asTypeOf(new Dispatch(tracking)) - interlock.io.dispatch <> io.dispatch - interlock.io.accessInfo <> io.accessInfo - io.grant <> interlock.io.grant - interlock.io.done <> io.done - io.complete <> interlock.io.complete - val access = io.access(0).bits - interlock.io.cpuQuery.valid := io.access(0).valid && access.atomic =/= CacheAtomic.Fence.U - interlock.io.cpuQuery.paddr := access.addr - interlock.io.cpuQuery.sizeLog2 := Mux(access.atomicWord, 2.U, 3.U) - interlock.io.cpuQuery.write := access.write || access.atomic === CacheAtomic.SC.U || - (access.atomic >= CacheAtomic.Swap.U && access.atomic <= CacheAtomic.MaxU.U) - interlock.io.cpuQuery.olderDispatchPending := io.olderDispatchPending - io.cpuAllow := interlock.io.cpuAllow - - val first: Instance[CoreCache] = Instantiate(new CoreCache(l1.copy(nodeId = 1), tracking)) - first.io.access.valid := io.access(0).valid && interlock.io.cpuAllow - first.io.access.bits := access - io.access(0).ready := first.io.access.ready && interlock.io.cpuAllow - io.result(0) <> first.io.result - first.io.olderRequestsDrained := io.olderRequestsDrained - first.io.maintenance <> interlock.io.maintenance - interlock.io.maintained <> first.io.maintained - home.io.requesters(0) <> first.io.chi - // Verification-only delayed transport, including a complete old writeback - // accepted by L1 but not yet delivered to Home. - val delayedData = Module(new Queue(new DataFlit(c), c.beatsPerLine)) - delayedData.io.enq <> first.io.chi.txDat - home.io.requesters(0).txDat.valid := delayedData.io.deq.valid && !io.blockRequesterData - home.io.requesters(0).txDat.bits := delayedData.io.deq.bits - delayedData.io.deq.ready := home.io.requesters(0).txDat.ready && !io.blockRequesterData - for (i <- 1 until memory.agents) { - val cache: Instance[BankedChiCache] = Instantiate(new BankedChiCache(l1.copy(nodeId = i + 1))) - cache.io.access <> io.access(i) - io.result(i) <> cache.io.result - home.io.requesters(i) <> cache.io.chi - } -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Emit.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Emit.scala deleted file mode 100644 index d9135dd3a..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Emit.scala +++ /dev/null @@ -1,468 +0,0 @@ -package hier.tile.memory - -import java.nio.file.{Files, Paths} -import scala.sys.process._ -import framework.system.core.rocket.CpuParams -import freechips.rocketchip.rocket.{DCacheParams, ICacheParams, RocketCoreParams} -import memcore.bus.chi.rnf.RnfParams -import memcore.bus.chi._ -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.cache.configs.CacheParams -import memcore.memory.coherence.configs.CoherenceParams - -object Emit extends App { - val root = Paths.get("src/main/scala/framework/root/root-tile").toAbsolutePath - val output = root.resolve("build") - Files.createDirectories(output) - // Frozen verification profile: two identical 2-bank/8-line RN-Fs, one Home64. - val chi = Params() - val p = CoherenceParams(chi, CacheParams(chi.addressBits, 64, 4, 2, 8, 4, 2), 2, 4, 64) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new CacheSystem(p), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("CacheSystem").toString, "--split-verilog") - ) - val definitions = scala.collection.mutable.ArrayBuffer[String]() - val ports = scala.collection.mutable.ArrayBuffer[String]() - def fields(flit: Flit): Seq[(String, Int)] = - flit.fieldsLSB.map(field => flit.elements.find(_._2 eq field).get._1 -> field.getWidth) - - def layout( - namespace: String, - definitions: scala.collection.mutable.Buffer[String], - kind: String, - fs: Seq[(String, Int)] - ): Unit = { - var offset = 0 - for ((name, width) <- fs if width > 0) { - definitions += s"`define ${namespace}_${kind}_${name.toUpperCase}_OFFSET $offset" - definitions += s"`define ${namespace}_${kind}_${name.toUpperCase}_WIDTH $width" - offset += width - } - definitions += s"`define ${namespace}_${kind}_WIDTH $offset" - } - - def bind( - ports: scala.collection.mutable.Buffer[String], - prefix: String, - signal: String, - fs: Seq[(String, Int)] - ): Unit = { - var offset = 0 - for ((name, width) <- fs if width > 0) { - ports += s".$prefix$name($signal[$offset +: $width])" - offset += width - } - } - - val req = fields(new RequestFlit(chi)) - val rsp = fields(new ResponseFlit(chi)) - val dat = fields(new DataFlit(chi)) - val snp = fields(new SnoopFlit(chi)) - val lineReq = new LineRequest(chi) - val lineResp = new LineResponse(chi) - val memReq = Seq("id", "addr", "write", "data", "mask").map(name => name -> lineReq.elements(name).getWidth) - val memResp = Seq("id", "data", "error").map(name => name -> lineResp.elements(name).getWidth) - for ( - (kind, fs) <- Seq("REQ" -> req, "RSP" -> rsp, "DAT" -> dat, "SNP" -> snp, "MEMREQ" -> memReq, "MEMRESP" -> memResp) - ) { layout("COH", definitions, kind, fs) } - val snpWidth = snp.map(_._2).sum - definitions += s"`define COH_SNP_TARGET_OFFSET $snpWidth" - definitions += s"`define COH_SNP_TARGET_WIDTH ${chi.nodeIdBits}" - definitions += s"`define COH_SNP_CHANNEL_WIDTH ${snpWidth + chi.nodeIdBits}" - for ( - (name, value) <- Seq( - "AGENTS" -> p.agents, - "MSHRS" -> p.mshrEntries, - "HOME" -> p.homeId, - "SETS" -> p.cache.sets, - "WAYS" -> p.cache.ways, - "DATA_BITS" -> chi.dataBits, - "OUTSTANDING_BITS" -> chisel3.util.log2Ceil(p.mshrEntries + 1), - "READ_NSD" -> Opcode.ReadNotSharedDirty, - "READ_UNIQUE" -> Opcode.ReadUnique, - "WRITEBACK" -> Opcode.WriteBackFull, - "COMP" -> Opcode.Comp, - "COMP_DBID" -> Opcode.CompDBIDResp, - "COMP_ACK" -> Opcode.CompAck, - "COMP_DATA" -> Opcode.CompData, - "SNP_RESP" -> Opcode.SnpResp, - "SNP_DATA" -> Opcode.SnpRespData, - "COPYBACK_DATA" -> Opcode.CopyBackWriteData - ) - ) { - definitions += s"`define COH_$name $value" - } - - val observedChannels = Seq( - ("observedReq", "req", req), - ("observedRsp", "rsp", rsp), - ("observedDat", "dat", dat), - ("observedRxRsp", "rx_rsp", rsp), - ("observedRxDat", "rx_dat", dat) - ) - - for (core <- 0 until p.agents) { - for (field <- Seq("addr", "write", "data", "mask", "atomic", "atomicWord")) { - ports += s".io_access_${core}_bits_$field(control.access_$field[$core])" - } - ports += s".io_access_${core}_valid(control.access_valid[$core])" - ports += s".io_access_${core}_ready(control.access_ready[$core])" - ports += s".io_result_${core}_valid(control.result_valid[$core])" - ports += s".io_result_${core}_ready(control.result_ready[$core])" - ports += s".io_result_${core}_bits_data(control.result_data[$core])" - ports += s".io_result_${core}_bits_error(control.result_error[$core])" - } - for ((port, signal, fs) <- observedChannels) { - bind(ports, s"io_${port}_bits_", s"control.${signal}_bits", fs) - ports += s".io_${port}_valid(control.${signal}_valid)" - } - bind(ports, "io_observedSnp_bits_flit_", "control.snp_bits", snp) - ports += s".io_observedSnp_bits_targetNode(control.snp_bits[$snpWidth +: ${chi.nodeIdBits}])" - ports += ".io_observedSnp_valid(control.snp_valid)" - bind(ports, "io_memoryReq_bits_", "mem_req.bits", memReq) - bind(ports, "io_memoryResp_bits_", "mem_resp.bits", memResp) - ports ++= Seq( - ".io_memoryReq_valid(mem_req.valid)", - ".io_memoryReq_ready(mem_req.ready)", - ".io_memoryResp_valid(mem_resp.valid)", - ".io_memoryResp_ready(mem_resp.ready)" - ) - Files.writeString( - output.resolve("cache_system_config.svh"), - "`ifndef CACHE_SYSTEM_CONFIG_SVH\n`define CACHE_SYSTEM_CONFIG_SVH\n" + definitions.mkString("\n") + "\n`endif\n" - ) - Files.writeString(output.resolve("cache_system_ports.svh"), ports.mkString(",\n") + "\n") - - // The short uncached window makes aligned cross-region accesses testable. - val regions = Seq( - PhysicalRegion(BigInt("80000000", 16), 0x1000000, true, true, true, true, true, true), - PhysicalRegion(BigInt("10000000", 16), 0x1000, false, false, true, true, false, false), - PhysicalRegion(BigInt("10002000", 16), 5, false, false, true, true, false, false), - PhysicalRegion(BigInt("90000000", 16), 64, true, false, true, false, false, true), - PhysicalRegion(BigInt("90000040", 16), 64, true, false, false, true, false, true), - PhysicalRegion(BigInt("a0000000", 16), 0x100000, false, true, true, true, true, true) - ) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new VirtualCacheSystem(p, regions), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("VirtualCacheSystem").toString, "--split-verilog") - ) - val vd = scala.collection.mutable.ArrayBuffer[String]() - val vp = scala.collection.mutable.ArrayBuffer[String]() - - def virtualFields( - kind: String, - prefix: String, - signal: String, - fs: Seq[(String, Int)] - ): Unit = { - layout("VM", vd, kind, fs) - bind(vp, prefix, signal, fs) - } - - virtualFields( - "REQ", - "io_request_bits_", - "source_if.bits", - Seq( - "vaddr" -> 64, - "tag" -> 6, - "size" -> 3, - "write" -> 1, - "execute" -> 1, - "signed" -> 1, - "data" -> 64, - "atomic" -> 4, - "privilege" -> 2, - "sum" -> 1, - "mxr" -> 1, - "satpMode" -> 4, - "rootPpn" -> 44 - ) - ) - virtualFields( - "AUTH", - "io_authorizationRequest_bits_", - "auth_req.bits", - Seq("paddr" -> 64, "size" -> 3, "read" -> 1, "write" -> 1, "execute" -> 1, "isPte" -> 1, "privilege" -> 2) - ) - vp += ".io_authorizationResponse_bits(auth_resp.bits)" - virtualFields( - "RESP", - "io_response_bits_", - "sink_if.bits", - Seq("tag" -> 6, "data" -> 64, "misaligned" -> 1, "pageFault" -> 1, "accessFault" -> 1) - ) - virtualFields("MEMREQ", "io_memoryReq_bits_", "mem_req.bits", memReq) - virtualFields("MEMRESP", "io_memoryResp_bits_", "mem_resp.bits", memResp) - virtualFields( - "UNCACHED", - "io_uncachedRequest_bits_", - "uncached_req.bits", - Seq("addr" -> 64, "tag" -> 6, "size" -> 3, "write" -> 1, "data" -> 64, "atomic" -> 4, "normal" -> 1) - ) - virtualFields("URESP", "io_uncachedResponse_bits_", "uncached_resp.bits", Seq("tag" -> 6, "data" -> 64, "error" -> 1)) - virtualFields( - "TRANSLATION", - "io_observedTranslation_bits_", - "control.translation_bits", - Seq( - "paddr" -> chi.addressBits, - "pageFault" -> 1, - "accessFault" -> 1, - "level" -> 2 - ) - ) - virtualFields( - "PHYSICAL", - "io_observedPhysical_bits_", - "control.physical_bits", - Seq( - "addr" -> 64, - "tag" -> 6, - "size" -> 3, - "write" -> 1, - "signed" -> 1, - "data" -> 64, - "atomic" -> 4, - "cacheable" -> 1, - "normal" -> 1 - ) - ) - virtualFields( - "CACHE", - "io_observedCache_bits_access_", - "control.cache_bits", - Seq("addr" -> chi.addressBits, "write" -> 1, "data" -> 64, "mask" -> 8, "atomic" -> 4, "atomicWord" -> 1) - ) - vp ++= Seq( - ".io_observedTranslation_valid(control.translation_valid)", - ".io_observedPhysical_valid(control.physical_valid)", - ".io_observedCache_valid(control.cache_valid)", - ".io_observedCache_bits_pte(control.cache_pte)", - ".io_observedEviction_valid(control.eviction_valid)", - ".io_observedEviction_bits(control.eviction_addr)" - ) - for ( - (port, signal) <- Seq( - "request" -> "source_if", - "response" -> "sink_if", - "memoryReq" -> "mem_req", - "memoryResp" -> "mem_resp", - "authorizationRequest" -> "auth_req", - "authorizationResponse" -> "auth_resp", - "uncachedRequest" -> "uncached_req", - "uncachedResponse" -> "uncached_resp" - ) - ) { - vp += s".io_${port}_valid($signal.valid)" - vp += s".io_${port}_ready($signal.ready)" - } - for ( - (name, region) <- Seq( - "DDR" -> regions(0), - "MMIO" -> regions(1), - "SHORT_MMIO" -> regions(2), - "READ_ONLY" -> regions(3), - "WRITE_ONLY" -> regions(4), - "NORMAL_RAM" -> regions(5) - ) - ) { - vd += s"`define VM_${name}_BASE 64'h${region.base.toString(16)}" - vd += s"`define VM_${name}_BYTES 64'h${region.bytes.toString(16)}" - } - vd += s"`define VM_OUTSTANDING_BITS ${chisel3.util.log2Ceil(p.mshrEntries + 1)}" - Files.writeString( - output.resolve("virtual_cache_config.svh"), - "`ifndef VIRTUAL_CACHE_CONFIG_SVH\n`define VIRTUAL_CACHE_CONFIG_SVH\n" + vd.mkString("\n") + "\n`endif\n" - ) - Files.writeString(output.resolve("virtual_cache_ports.svh"), vp.mkString(",\n") + "\n") - - implicit val cpu: CpuParams = CpuParams( - core = new RocketCoreParams( - useVM = false, - useUser = false, - useSupervisor = false, - useHypervisor = false, - useDebug = false, - fpu = None, - nPMPs = 4, - clockGate = false, - haveCFlush = false, - haveCease = false, - haveSimTimeout = false - ) { override val pmpGranularity = 8 }, - dcache = Some(DCacheParams(nSets = 64, nWays = 2)), - icache = Some(ICacheParams()), - btb = None, - physicalAddressBits = 44, - hartIdBits = 1, - beatBytes = 8, - blockBytes = 64 - ) - - val builder = Paths.get("src/main/scala/framework/root/root-core/src/main/resources/build_fixture.py").toAbsolutePath - require( - Process(Seq("python", builder.toString, output.toString, "multicore")).! == 0, - "Two-Core fixture generation failed" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Tile(p.copy(agents = 2 * p.agents), RnfParams(chi, cacheLines = 8, banks = 2), regions), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Tile").toString, "--split-verilog") - ) - - val td = scala.collection.mutable.ArrayBuffer[String]( - s"""`define TILE_IMAGE "${output.resolve("multicore.hex")}"""" - ) - - val tp = scala.collection.mutable.ArrayBuffer[String]() - - bind(tp, "io_memoryReq_bits_", "mem_req.bits", memReq) - bind(tp, "io_memoryResp_bits_", "mem_resp.bits", memResp) - tp ++= Seq( - ".io_memoryReq_valid(mem_req.valid)", - ".io_memoryReq_ready(mem_req.ready)", - ".io_memoryResp_valid(mem_resp.valid)", - ".io_memoryResp_ready(mem_resp.ready)" - ) - val uncached = Seq("addr" -> 64, "tag" -> 6, "size" -> 3, "write" -> 1, "data" -> 64) - val uncachedResp = Seq("tag" -> 6, "data" -> 64, "error" -> 1) - for ((kind, fs) <- Seq("MEMREQ" -> memReq, "MEMRESP" -> memResp, "UNCACHED" -> uncached, "URESP" -> uncachedResp)) { - layout("TILE", td, kind, fs) - } - for (i <- 0 until p.agents) { - bind(tp, s"io_uncachedRequest_${i}_bits_", s"uncached_req_$i.bits", uncached) - bind(tp, s"io_uncachedResponse_${i}_bits_", s"uncached_resp_$i.bits", uncachedResp) - tp ++= Seq( - s".io_uncachedRequest_${i}_valid(uncached_req_$i.valid)", - s".io_uncachedRequest_${i}_ready(uncached_req_$i.ready)", - s".io_uncachedResponse_${i}_valid(uncached_resp_$i.valid)", - s".io_uncachedResponse_${i}_ready(uncached_resp_$i.ready)" - ) - for (field <- Seq("retired", "retiredPc", "trapped", "trapCause", "trapValue", "trapPc")) - tp += s".io_${field}_$i(control.$field[$i])" - } - for ((port, signal, fs) <- observedChannels) { - bind(tp, s"io_${port}_bits_", s"control.${signal}_bits", fs) - tp += s".io_${port}_valid(control.${signal}_valid)" - } - bind(tp, "io_observedSnp_bits_flit_", "control.snp_bits", snp) - tp += s".io_observedSnp_bits_targetNode(control.snp_bits[$snpWidth +: ${chi.nodeIdBits}])" - tp += ".io_observedSnp_valid(control.snp_valid)" - Files.writeString(output.resolve("tile_config.svh"), td.mkString("\n") + "\n") - Files.writeString(output.resolve("tile_ports.svh"), tp.mkString(",\n") + "\n") - - val tracking = memcore.memory.interlock.Params() - val consistencyMemory = p.copy(cache = p.cache.copy(sets = 16)) - val consistencyL1 = RnfParams(chi, cacheLines = 8, banks = 2) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Consistency(consistencyMemory, consistencyL1, tracking), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Consistency").toString, "--split-verilog") - ) - - val cd = scala.collection.mutable.ArrayBuffer[String]( - s"`define CONS_AGENTS ${p.agents}", - s"`define CONS_HOME ${p.homeId}", - s"`define CONS_MSHRS ${p.mshrEntries}", - s"`define CONS_OUTSTANDING_BITS ${chisel3.util.log2Ceil(p.mshrEntries + 1)}", - s"`define CONS_PA_BITS ${tracking.addressBits}", - s"`define CONS_LINE_BYTES ${tracking.lineBytes}", - s"`define CONS_L1_LINES ${consistencyL1.cacheLines}", - s"`define CONS_L2_SETS ${consistencyMemory.cache.sets}", - s"`define CONS_SLOTS ${tracking.entries}" - ) - - val cp = scala.collection.mutable.ArrayBuffer[String]() - def members(bundle: chisel3.Bundle): Seq[(String, Int)] = - bundle.elements.toSeq.collect { case (name, field) if field.getWidth > 0 => name -> field.getWidth } - val accessFields = members(new memcore.bus.chi.rnf.CacheAccess(chi)) - val resultFields = members(new memcore.bus.chi.rnf.CacheResult) - layout("CONS", cd, "CPU", accessFields) - layout("CONS", cd, "RESULT", resultFields) - - def connect(port: String, signal: String, fs: Seq[(String, Int)]): Unit = { - bind(cp, s"io_${port}_bits_", s"$signal.bits", fs) - cp += s".io_${port}_valid($signal.valid)" - cp += s".io_${port}_ready($signal.ready)" - } - - for (i <- 0 until p.agents) { - connect(s"access_$i", s"access_$i", accessFields) - connect(s"result_$i", s"result_$i", resultFields) - } - for ( - (kind, port, signal, fs) <- Seq( - ( - "DISPATCH", - "dispatch", - "dispatch", - members(new memcore.memory.interlock.Dispatch(tracking)) - ), - ( - "INFO", - "accessInfo", - "access_info", - members(new memcore.memory.interlock.AccessInfo(tracking)) - ), - ("GRANT", "grant", "grant", members(new memcore.memory.interlock.Tag(tracking))), - ("DONE", "done", "done", members(new memcore.memory.interlock.Acknowledgement(tracking))), - ("COMPLETE", "complete", "complete", members(new memcore.memory.interlock.Tag(tracking))), - ("MEMREQ", "memoryReq", "mem_req", memReq), - ("MEMRESP", "memoryResp", "mem_resp", memResp) - ) - ) { - layout("CONS", cd, kind, fs) - connect(port, signal, fs) - } - layout("CONS", cd, "REQ", req) - layout("CONS", cd, "SNP", snp) - cd += s"`define CONS_SNP_TARGET_OFFSET $snpWidth" - cd += s"`define CONS_SNP_TARGET_WIDTH ${chi.nodeIdBits}" - cd += s"`define CONS_SNP_CHANNEL_WIDTH ${snpWidth + chi.nodeIdBits}" - bind(cp, "io_observedReq_bits_", "control.req_bits", req) - bind(cp, "io_observedSnp_bits_flit_", "control.snp_bits", snp) - cp ++= Seq( - ".io_observedReq_valid(control.req_valid)", - ".io_observedSnp_valid(control.snp_valid)", - s".io_observedSnp_bits_targetNode(control.snp_bits[$snpWidth +: ${chi.nodeIdBits}])", - ".io_olderDispatchPending(control.older_dispatch_pending)", - ".io_olderRequestsDrained(control.older_requests_drained)", - ".io_blockRequesterRsp(control.block_requester_rsp)", - ".io_blockRequesterData(control.block_requester_data)", - ".io_cpuAllow(control.cpu_allow)", - ".io_outstanding(control.outstanding)" - ) - Files.writeString(output.resolve("consistency_config.svh"), cd.mkString("\n") + "\n") - Files.writeString(output.resolve("consistency_ports.svh"), cp.mkString(",\n") + "\n") - // Five-core Goban shape elaboration only: explicit placement and opaque test signatures. - // This does not load a PB profile; the production Tile is assembled in framework.system.tile. - val compositionCpu = cpu.copy(hartIdBits = 3) - val compositionMemory = p.copy(agents = 10) - - val compositionPlacements = Seq( - CorePlacement(RnfParams(chi, nodeId = 1, cacheLines = 8, banks = 2), compositionCpu, CoreRole.Controller), - CorePlacement(RnfParams(chi, nodeId = 2, cacheLines = 8, banks = 2), compositionCpu, CoreRole.Compute), - CorePlacement(RnfParams(chi, nodeId = 3, cacheLines = 8, banks = 2), compositionCpu, CoreRole.Compute), - CorePlacement(RnfParams(chi, nodeId = 4, cacheLines = 8, banks = 2), compositionCpu, CoreRole.Compute), - CorePlacement(RnfParams(chi, nodeId = 5, cacheLines = 8, banks = 2), compositionCpu, CoreRole.Compute) - ) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Composition( - compositionMemory, - compositionPlacements, - regions, - memcore.memory.interlock.Params(addressBits = chi.addressBits), - workerCoreIds = Seq(1, 2, 3, 4), - signatures = Seq(BigInt(1), BigInt(2), BigInt(3), BigInt(4)) - ), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Composition").toString, "--split-verilog") - ) - -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Home.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Home.scala deleted file mode 100644 index 9b245bc1c..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Home.scala +++ /dev/null @@ -1,52 +0,0 @@ -package hier.tile.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi._ -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.coherence.Coherence -import memcore.memory.coherence.configs.CoherenceParams - -/** One Tile-owned shared L2/Home and its explicit requester fabric. */ -@instantiable -class Home(p: CoherenceParams) extends Module { - val c = p.chi - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - val blockRequesterRsp = Input(Bool()) - val requesters = Vec(p.agents, Flipped(new RequesterPort(c))) - val memoryReq = Decoupled(new LineRequest(c)) - val memoryResp = Flipped(Decoupled(new LineResponse(c))) - val outstanding = Output(UInt(log2Ceil(p.mshrEntries + 1).W)) - val observedReq = Output(Valid(new RequestFlit(c))) - val observedRsp = Output(Valid(new ResponseFlit(c))) - val observedDat = Output(Valid(new DataFlit(c))) - val observedSnp = Output(Valid(new DirectedSnoop(c))) - val observedRxRsp = Output(Valid(new ResponseFlit(c))) - val observedRxDat = Output(Valid(new DataFlit(c))) - }) - - val home: Instance[Coherence] = Instantiate(new Coherence(p)) - val fabric: Instance[Fabric] = Instantiate(new Fabric(c, p.agents, p.homeId)) - fabric.io.active := io.active - fabric.io.blockRequesterRsp := io.blockRequesterRsp - fabric.io.requesters <> io.requesters - home.io.req <> fabric.io.req - home.io.rxRsp <> fabric.io.rxRsp - home.io.rxDat <> fabric.io.rxDat - fabric.io.rsp <> home.io.rsp - fabric.io.dat <> home.io.dat - fabric.io.snp <> home.io.snp - io.memoryReq <> home.io.memoryReq - home.io.memoryResp <> io.memoryResp - io.outstanding := home.io.outstanding - io.observedReq.valid := home.io.req.fire; io.observedReq.bits := home.io.req.bits - io.observedRsp.valid := home.io.rsp.fire; io.observedRsp.bits := home.io.rsp.bits - io.observedDat.valid := home.io.dat.fire; io.observedDat.bits := home.io.dat.bits - io.observedSnp.valid := home.io.snp.fire; io.observedSnp.bits := home.io.snp.bits - io.observedRxRsp.valid := home.io.rxRsp.fire; io.observedRxRsp.bits := home.io.rxRsp.bits - io.observedRxDat.valid := home.io.rxDat.fire; io.observedRxDat.bits := home.io.rxDat.bits -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Memory.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Memory.scala deleted file mode 100644 index 6119e39bc..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Memory.scala +++ /dev/null @@ -1,30 +0,0 @@ -package hier.tile.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi.RequesterPort -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.coherence.configs.CoherenceParams - -/** CPU coherent Home and shared L2 with one backing-memory client. */ -@instantiable -class Memory(p: CoherenceParams) extends Module { - val c = p.chi - - @public - val io = IO(new Bundle { - val coherent = Vec(p.agents, Flipped(new RequesterPort(c))) - val backingReq = Vec(1, Decoupled(new LineRequest(c))) - val backingResp = Vec(1, Flipped(Decoupled(new LineResponse(c)))) - val homeOutstanding = Output(UInt(log2Ceil(p.mshrEntries + 1).W)) - }) - - val home: Instance[Home] = Instantiate(new Home(p)) - home.io.active := true.B - home.io.blockRequesterRsp := false.B - home.io.requesters <> io.coherent - io.backingReq(0) <> home.io.memoryReq - home.io.memoryResp <> io.backingResp(0) - io.homeOutstanding := home.io.outstanding -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Tile.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Tile.scala deleted file mode 100644 index 2aafa747a..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/Tile.scala +++ /dev/null @@ -1,82 +0,0 @@ -package hier.tile.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import hier.core.rocket.Core -import memcore.bus.chi._ -import memcore.bus.chi.rnf.RnfParams -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion, UncachedRequest, UncachedResponse} -import memcore.memory.coherence.configs.CoherenceParams - -@instantiable -class Tile(memory: CoherenceParams, l1: RnfParams, regions: Seq[PhysicalRegion])(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - require(l1.chi == memory.chi && l1.homeId == memory.homeId && l1.homeCount == 1) - val c = memory.chi - val cp = CpuMemParams(c, tagBits = 6) - - val cores = memory.agents / 2 - require(memory.agents == 2 * cores) - - @public - val io = IO(new Bundle { - val resetVector = Input(UInt(64.W)) - val time = Input(UInt(64.W)) - val memoryReq = Decoupled(new LineRequest(c)) - val memoryResp = Flipped(Decoupled(new LineResponse(c))) - val uncachedRequest = Vec(cores, Decoupled(new UncachedRequest(cp))) - val uncachedResponse = Vec(cores, Flipped(Decoupled(new UncachedResponse(cp)))) - val retired = Output(Vec(cores, Bool())) - val retiredPc = Output(Vec(cores, UInt(64.W))) - val trapped = Output(Vec(cores, Bool())) - val trapCause = Output(Vec(cores, UInt(64.W))) - val trapValue = Output(Vec(cores, UInt(64.W))) - val trapPc = Output(Vec(cores, UInt(64.W))) - val outstanding = Output(UInt(log2Ceil(memory.mshrEntries + 1).W)) - val observedReq = Output(Valid(new RequestFlit(c))) - val observedRsp = Output(Valid(new ResponseFlit(c))) - val observedDat = Output(Valid(new DataFlit(c))) - val observedSnp = Output(Valid(new DirectedSnoop(c))) - val observedRxRsp = Output(Valid(new ResponseFlit(c))) - val observedRxDat = Output(Valid(new DataFlit(c))) - }) - - val home: Instance[Home] = Instantiate(new Home(memory)) - home.io.active := true.B - home.io.blockRequesterRsp := false.B - io.memoryReq <> home.io.memoryReq - home.io.memoryResp <> io.memoryResp - io.outstanding := home.io.outstanding - io.observedReq := home.io.observedReq - io.observedRsp := home.io.observedRsp - io.observedDat := home.io.observedDat - io.observedSnp := home.io.observedSnp - io.observedRxRsp := home.io.observedRxRsp - io.observedRxDat := home.io.observedRxDat - // Requester i is core i's data L1; requester cores + i is its instruction L1. - val core: Seq[Instance[Core]] = - Seq.tabulate(cores)(i => Instantiate(new Core(l1.copy(nodeId = i + 1), l1.copy(nodeId = cores + i + 1), regions))) - for (i <- 0 until cores) { - core(i).clink.timerInterrupt := false.B - core(i).clink.softwareInterrupt := false.B - core(i).clink.externalInterrupt := false.B - core(i).clink.supervisorExternalInterrupt := false.B - core(i).clink.hartId := i.U - core(i).clink.resetVector := io.resetVector - core(i).clink.time := io.time - home.io.requesters(i) <> core(i).clink.chi - home.io.requesters(cores + i) <> core(i).clink.instructionChi - io.uncachedRequest(i) <> core(i).clink.uncachedRequest - core(i).clink.uncachedResponse <> io.uncachedResponse(i) - io.retired(i) := core(i).clink.retired - io.retiredPc(i) := core(i).clink.retiredPc - io.trapped(i) := core(i).clink.trapped - io.trapCause(i) := core(i).clink.trapCause - io.trapValue(i) := core(i).clink.trapValue - io.trapPc(i) := core(i).clink.trapPc - } -} diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/VirtualCacheSystem.scala b/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/VirtualCacheSystem.scala deleted file mode 100644 index 2a07408a1..000000000 --- a/arch/src/main/scala/framework/root/root-tile/src/main/scala/memory/VirtualCacheSystem.scala +++ /dev/null @@ -1,78 +0,0 @@ -package hier.tile.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi.Opcode -import memcore.bus.chi.rnf.CacheAccess -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.cpu.{ - CpuMemParams, - CpuMemoryRequest, - PhysicalAuthorization, - PhysicalRegion, - UncachedRequest, - UncachedResponse, - VirtualMemory, - VirtualMemoryRequest, - VirtualMemoryResponse -} -import memcore.memory.coherence.configs.CoherenceParams - -/** Independent verification composition: one virtual client and the two-RN-F shared-L2 cache system. */ -@instantiable -class VirtualCacheSystem(p: CoherenceParams, regions: Seq[PhysicalRegion]) extends Module { - val cp = CpuMemParams(p.chi, tagBits = 6) - - @public - val io = IO(new Bundle { - val active = Input(Bool()) - val request = Flipped(Decoupled(new VirtualMemoryRequest(cp))) - val response = Decoupled(new VirtualMemoryResponse(cp)) - val authorizationRequest = Decoupled(new PhysicalAuthorization) - val authorizationResponse = Flipped(Decoupled(Bool())) - val memoryReq = Decoupled(new LineRequest(p.chi)) - val memoryResp = Flipped(Decoupled(new LineResponse(p.chi))) - val uncachedRequest = Decoupled(new UncachedRequest(cp)) - val uncachedResponse = Flipped(Decoupled(new UncachedResponse(cp))) - val observedTranslation = Output(Valid(new memcore.memory.mmu.Response(p.chi))) - val observedPhysical = Output(Valid(new CpuMemoryRequest(cp))) - - val observedCache = Output(Valid(new Bundle { - val pte = Bool() - val access = new CacheAccess(p.chi) - })) - - val observedEviction = Output(Valid(UInt(p.chi.addressBits.W))) - val outstanding = Output(UInt(log2Ceil(p.mshrEntries + 1).W)) - }) - - val virtualMemory: Instance[VirtualMemory] = Instantiate(new VirtualMemory(cp, regions)) - val caches: Instance[CacheSystem] = Instantiate(new CacheSystem(p)) - virtualMemory.io.active := io.active - virtualMemory.io.flushTranslation := false.B - virtualMemory.io.lookup.config := 0.U.asTypeOf(virtualMemory.io.lookup.config) - virtualMemory.io.lookup.req := 0.U.asTypeOf(virtualMemory.io.lookup.req) - virtualMemory.io.request <> io.request - io.response <> virtualMemory.io.response - io.authorizationRequest <> virtualMemory.io.authorizationRequest - virtualMemory.io.authorizationResponse <> io.authorizationResponse - io.uncachedRequest <> virtualMemory.io.uncachedRequest - virtualMemory.io.uncachedResponse <> io.uncachedResponse - caches.io.active := io.active - caches.io.blockRequesterRsp := false.B - caches.io.access(0) <> virtualMemory.io.cacheRequest - virtualMemory.io.cacheResponse <> caches.io.result(0) - caches.io.access(1).valid := false.B - caches.io.access(1).bits := 0.U.asTypeOf(caches.io.access(1).bits) - caches.io.result(1).ready := false.B - when(caches.io.result(1).valid)(assert(false.B, "Virtual LSU inactive requester returned data")) - io.memoryReq <> caches.io.memoryReq - caches.io.memoryResp <> io.memoryResp - io.outstanding := caches.io.outstanding - io.observedTranslation := virtualMemory.io.observedTranslation - io.observedPhysical := virtualMemory.io.observedPhysical - io.observedCache := virtualMemory.io.observedCache - io.observedEviction.valid := caches.io.observedReq.valid && caches.io.observedReq.bits.opcode === Opcode.Evict.U - io.observedEviction.bits := caches.io.observedReq.bits.addr -} diff --git a/arch/src/main/scala/framework/rvv/LICENSE b/arch/src/main/scala/framework/rvv/LICENSE deleted file mode 100644 index 261eeb9e9..000000000 --- a/arch/src/main/scala/framework/rvv/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/arch/src/main/scala/framework/rvv/README.md b/arch/src/main/scala/framework/rvv/README.md deleted file mode 100644 index 3ea42ae69..000000000 --- a/arch/src/main/scala/framework/rvv/README.md +++ /dev/null @@ -1 +0,0 @@ -# rvv diff --git a/arch/src/main/scala/framework/rvv/build.sc b/arch/src/main/scala/framework/rvv/build.sc deleted file mode 100644 index 9645d6c4e..000000000 --- a/arch/src/main/scala/framework/rvv/build.sc +++ /dev/null @@ -1,64 +0,0 @@ -import mill._ -import mill.scalalib._ - -val archRoot = os.pwd / os.up / os.up / os.up / os.up / os.up - -object cde extends SbtModule { - override def millSourcePath = archRoot / "thirdparty" / "rocket-chip" / "dependencies" / "cde" - override def scalaVersion = "2.13.16" - - override def sources = T.sources { - super.sources() ++ Seq(PathRef(millSourcePath / "cde" / "src" / "chipsalliance")) - } - -} - -object hardfloat extends SbtModule { - override def millSourcePath = archRoot / "thirdparty" / "berkeley-hardfloat" - override def scalaVersion = "2.13.16" - override def moduleDeps = Seq(cde) - - override def sources = T.sources { - super.sources() ++ Seq(PathRef(millSourcePath / "hardfloat" / "src" / "main" / "scala")) - } - - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") -} - -object rvv extends SbtModule { - override def millSourcePath = os.pwd - override def scalaVersion = "2.13.16" - override def moduleDeps = Seq(hardfloat) - - override def sources = T.sources { - super.sources() ++ Seq( - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "arith" / "src" / "main" / "scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "top" / "GlobalConfig.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "top" / "configs" / "SimParam.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "memdomain" / "configs" / "MemDomainParam.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "frontend" / "configs" / "FrontendParam.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "configs" / "BallDomainParam.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "system" / "tile" / "configs" / "TileParam.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "blink" / "blink.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "blink" / "bank.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "blink" / "status.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "blink" / "baseball.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "blink" / "SubRobRow.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "blink" / "mmio" / "MmioRead.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "blink" / "mmio" / "MmioWrite.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "memdomain" / "backend" / "banks" / "SramIO.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "memdomain" / "backend" / "mmio" / "MmioIO.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "rs" / "interfaces.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "balldomain" / "decoder" / "BallDecodeCmd.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "frontend" / "decoder" / "PostGDCmd.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "frontend" / "scoreboard" / "BankAccessInfo.scala"), - PathRef(archRoot / "src" / "main" / "scala" / "framework" / "system" / "core" / "rocket" / "RoCCCommandBB.scala") - ) - } - - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0", ivy"com.lihaoyi::upickle:3.3.1") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") - override def mainClass = Some("framework.rvv.Emit") -} diff --git a/arch/src/main/scala/framework/rvv/src/csrc/Cargo.lock b/arch/src/main/scala/framework/rvv/src/csrc/Cargo.lock deleted file mode 100644 index c99fadbc9..000000000 --- a/arch/src/main/scala/framework/rvv/src/csrc/Cargo.lock +++ /dev/null @@ -1,135 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "bitflags" -version = "2.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rustc_apfloat" -version = "0.2.3+llvm-462a31f5a5ab" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" -dependencies = [ - "bitflags", - "smallvec", -] - -[[package]] -name = "rvv" -version = "0.1.0" -dependencies = [ - "rustc_apfloat", -] - -[[package]] -name = "rvv-reference-model" -version = "0.1.0" -dependencies = [ - "rvv", - "serde_json", -] - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.151" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "smallvec" -version = "1.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" - -[[package]] -name = "syn" -version = "3.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "unicode-ident" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/arch/src/main/scala/framework/rvv/src/csrc/Cargo.toml b/arch/src/main/scala/framework/rvv/src/csrc/Cargo.toml deleted file mode 100644 index 7162fbd21..000000000 --- a/arch/src/main/scala/framework/rvv/src/csrc/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "rvv-reference-model" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib", "staticlib"] - -[dependencies] -rvv = { path = "../../../../../../../../bebop/src/nodes/bemu/rvv" } - -[build-dependencies] -serde_json = "1" - -[workspace] diff --git a/arch/src/main/scala/framework/rvv/src/csrc/build.rs b/arch/src/main/scala/framework/rvv/src/csrc/build.rs deleted file mode 100644 index 80727754a..000000000 --- a/arch/src/main/scala/framework/rvv/src/csrc/build.rs +++ /dev/null @@ -1,62 +0,0 @@ -use std::{env, fs, path::PathBuf, process::Command}; -fn main() { - let root = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()) - .ancestors() - .nth(8) - .unwrap() - .to_path_buf(); - let ip = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()).parent().unwrap().parent().unwrap().to_path_buf(); - let parameters = ip.join("src/main/resources/rvv.json"); - println!("cargo:rerun-if-changed={}", parameters.display()); - let config: serde_json::Value = serde_json::from_str(&fs::read_to_string(parameters).unwrap()).unwrap(); - let value = |key: &str| config[key].as_u64().unwrap() as usize; - let constants = format!("const VLEN: usize = {};\nconst ELEN: usize = {};\nconst IBUF_BYTES: usize = {};\n", - value("vLen"), value("eLen"), value("iBufWords") * 4); - fs::write(PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("params.rs"), constants).unwrap(); - let rtl = ip.join("build/KernelEngine"); - fs::create_dir_all(&rtl).unwrap(); - fs::write(rtl.join("rvv_config.svh"), format!("`ifndef RVV_MEMORY_PORTS\n`define RVV_MEMORY_PORTS {}\n`endif\n", value("memoryPorts"))).unwrap(); - let mut connections = String::new(); - for port in 0..value("memoryPorts") { - for (direction, fields) in [ - ("Read", vec![("bank_id", "read_bank"), ("group_id", "read_group"), ("io_req_valid", "read_valid"), ("io_req_ready", "read_ready"), ("io_req_bits_addr", "read_row"), ("io_resp_valid", "read_response_valid"), ("io_resp_ready", "read_response_ready"), ("io_resp_bits_data", "read_data")]), - ("Write", vec![("bank_id", "write_bank"), ("group_id", "write_group"), ("io_req_valid", "write_valid"), ("io_req_ready", "write_ready"), ("io_req_bits_addr", "write_row"), ("io_resp_valid", "write_response_valid"), ("io_resp_ready", "write_response_ready"), ("io_req_bits_data", "write_data"), ("io_resp_bits_ok", "write_ok")]), - ] { - for (field, signal) in fields { - connections += &format!(".io_bank{direction}_{port}_{field}(vif.{signal}[{port}]),\n"); - } - } - for bit in 0..16 { - connections += &format!(".io_bankWrite_{port}_io_req_bits_mask_{bit}(vif.write_mask[{port}][{bit}]),\n"); - } - } - connections.truncate(connections.trim_end().len() - 1); - fs::write(rtl.join("rvv_ports.svh"), connections).unwrap(); - let source = root.join("bb-tests/workloads/lib/bbsw/kernels/rvv"); - for name in [ - "build_images.py", - "kernel.ld", - "kernels.h", - "silu.cpp", - "snake.cpp", - "quant.cpp", - "math/exp.cpp", - "math/sin.cpp", - "math/cos.cpp", - "math/erf.cpp", - "rope.cpp", - "trig.cpp", - "math/math.h", - "math/constants.h", - ] { - println!("cargo:rerun-if-changed={}", source.join(name).display()); - } - let output = PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("kernels"); - assert!(Command::new("python3") - .arg(source.join("build_images.py")) - .arg("--output") - .arg(output) - .status() - .unwrap() - .success()); -} diff --git a/arch/src/main/scala/framework/rvv/src/csrc/src/lib.rs b/arch/src/main/scala/framework/rvv/src/csrc/src/lib.rs deleted file mode 100644 index 9cce67869..000000000 --- a/arch/src/main/scala/framework/rvv/src/csrc/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod ffi; -mod model; diff --git a/arch/src/main/scala/framework/rvv/src/csrc/src/model.rs b/arch/src/main/scala/framework/rvv/src/csrc/src/model.rs deleted file mode 100644 index fcc242e65..000000000 --- a/arch/src/main/scala/framework/rvv/src/csrc/src/model.rs +++ /dev/null @@ -1,903 +0,0 @@ -include!(concat!(env!("OUT_DIR"), "/params.rs")); - -use rvv::{Engine, Fault, Memory, MemoryError}; -use std::path::Path; - -pub const BANK_BYTES: usize = 4096; -pub const BANKS: usize = 6; -#[derive(Clone)] -pub struct Banks(pub Vec); -impl Banks { - fn range(&self, address: u64, bytes: usize) -> Result, MemoryError> { - let bank = (address >> 32) as usize; - let offset = (address & 0xffff_ffff) as usize; - if bank >= BANKS || offset + bytes > BANK_BYTES { - return Err(MemoryError); - } - Ok(bank * BANK_BYTES + offset..bank * BANK_BYTES + offset + bytes) - } - fn put(&mut self, bank: usize, offset: usize, data: &[u8]) { - self.0[bank * BANK_BYTES + offset..bank * BANK_BYTES + offset + data.len()] - .copy_from_slice(data); - } -} -impl Memory for Banks { - fn ball_command(&mut self, _: u32, _: u64, _: u64) -> Result { - Err(MemoryError) - } - fn read(&mut self, address: u64, bytes: usize) -> Result { - let mut value = [0u8; 8]; - value[..bytes].copy_from_slice(&self.0[self.range(address, bytes)?]); - Ok(u64::from_le_bytes(value)) - } - fn write(&mut self, address: u64, bytes: usize, value: u64) -> Result<(), MemoryError> { - let range = self.range(address, bytes)?; - self.0[range].copy_from_slice(&value.to_le_bytes()[..bytes]); - Ok(()) - } -} -pub struct Case { - pub name: String, - pub program: Vec, - pub constants: Vec, - pub entry: u32, - pub args: [u64; 8], - pub initial: Banks, - pub expected_cause: u32, - pub fixed_expected: Option, -} -pub struct Model { - pub cases: Vec, - pub current: usize, - pub actual: Banks, - pub expected: Banks, - pub engine: Engine, - pub fault: Option, -} -fn i(op: u32, rd: u32, f: u32, rs: u32, imm: i32) -> u32 { - ((imm as u32 & 4095) << 20) | rs << 15 | f << 12 | rd << 7 | op -} -fn s(op: u32, f: u32, rs1: u32, rs2: u32, imm: u32) -> u32 { - (imm >> 5) << 25 | rs2 << 20 | rs1 << 15 | f << 12 | (imm & 31) << 7 | op -} -fn r(op: u32, rd: u32, f: u32, rs1: u32, rs2: u32, f7: u32) -> u32 { - f7 << 25 | rs2 << 20 | rs1 << 15 | f << 12 | rd << 7 | op -} -fn initial() -> Banks { - let mut b = Banks(vec![0; BANKS * BANK_BYTES]); - for offset in 0..BANK_BYTES { - b.0[BANK_BYTES + offset] = (offset as u8).wrapping_mul(73).wrapping_add(19); - } - for n in 0..256 { - let x = (n as f32 - 123.0) / 31.0; - b.put(2, n * 4, &x.to_le_bytes()); - b.put(3, n * 4, &(0.25 + n as f32 / 128.0).to_le_bytes()); - } - b -} -fn micro(name: &str, words: &[u32], args: [u64; 8]) -> Case { - Case { - name: name.into(), - program: words.iter().flat_map(|w| w.to_le_bytes()).collect(), - constants: vec![0; 512], - entry: 0, - args, - initial: initial(), - expected_cause: 0, - fixed_expected: None, - } -} -fn elf(path: &Path, name: &str, args: [u64; 8]) -> Case { - let bytes = std::fs::read(path).unwrap(); - assert_eq!(&bytes[..7], b"\x7fELF\x02\x01\x01"); - let u64_at = |p| u64::from_le_bytes(bytes[p..p + 8].try_into().unwrap()); - let u32_at = |p| u32::from_le_bytes(bytes[p..p + 4].try_into().unwrap()); - let u16_at = |p| u16::from_le_bytes(bytes[p..p + 2].try_into().unwrap()); - assert_eq!(u16_at(18), 243); - let mut c = micro(name, &[], args); - c.entry = u32::try_from(u64_at(24)).expect("kernel entry exceeds local address width"); - assert_eq!(u16_at(58), 64, "invalid ELF64 section header size"); - let sections = usize::try_from(u64_at(40)).unwrap(); - for n in 0..u16_at(60) as usize { - let p = sections.checked_add(n.checked_mul(64).unwrap()).unwrap(); - let flags = u64_at(p + 8); - let address = u32::try_from(u64_at(p + 16)).expect("kernel section exceeds local address width"); - let size = usize::try_from(u64_at(p + 32)).unwrap(); - if flags & 2 == 0 || size == 0 { - continue; - } - let offset = usize::try_from(u64_at(p + 24)).unwrap(); - if flags & 4 != 0 { - assert_eq!(address, 0); - c.program = bytes[offset..offset.checked_add(size).unwrap()].to_vec(); - } else { - assert_ne!( - u32_at(p + 4), - 8, - "allocated NOBITS requires explicit initialization" - ); - assert_eq!(address, 0x40000000); - assert_eq!(flags & 1, 0, "kernel globals must be read-only"); - c.constants.resize(size.max(512), 0); - c.constants[..size].copy_from_slice(&bytes[offset..offset.checked_add(size).unwrap()]); - } - } - assert!(!c.program.is_empty() && c.program.len() <= 4096 && c.program.len() % 4 == 0); - c -} -// Independent integer arithmetic gold: the BEMU Engine does not implement these fixed-point opcodes. -fn fixed_gold(case: &Case, bits: usize, op: u32, mode: u32) -> Banks { - let mut gold = case.initial.clone(); - let bytes = bits / 8; - let mask = (1i128 << bits) - 1; - let minimum = -(1i128 << (bits - 1)); - let maximum = (1i128 << (bits - 1)) - 1; - let read = |offset: usize, size: usize| { - let mut value = [0; 8]; - value[..size].copy_from_slice(&case.initial.0[2 * BANK_BYTES + offset..2 * BANK_BYTES + offset + size]); - u64::from_le_bytes(value) as i128 - }; - let signed = |value: i128, width: usize| if value & (1i128 << (width - 1)) != 0 { value - (1i128 << width) } else { value }; - let mut saturated = false; - for lane in 0..5 { - let a = read(lane * bytes, bytes); - let b = read(128 + lane * bytes, bytes); - let mut result = a; - if mode != 3 || lane >= 2 { - let (value, shift) = match op { - 8 => (a + b, 1), 9 => (signed(a, bits) + signed(b, bits), 1), - 10 => (a - b, 1), 11 => (signed(a, bits) - signed(b, bits), 1), - 39 => (signed(a, bits) * signed(b, bits), bits - 1), - 42 => (a, b as usize & (bits - 1)), - 43 => (signed(a, bits), b as usize & (bits - 1)), - 46 => (read(lane * bytes * 2, bytes * 2), b as usize & (bits * 2 - 1)), - 47 => (signed(read(lane * bytes * 2, bytes * 2), bits * 2), b as usize & (bits * 2 - 1)), - _ => unreachable!(), - }; - let truncated = value >> shift; - let half = shift != 0 && value & (1i128 << (shift - 1)) != 0; - let low = shift != 0 && value & ((1i128 << (shift - 1)) - 1) != 0; - let increment = match mode { - 0 => half, 1 => half && (low || truncated & 1 != 0), 2 => false, - 3 => truncated & 1 == 0 && value & ((1i128 << shift) - 1) != 0, - _ => unreachable!(), - }; - result = truncated + i128::from(increment); - if op == 39 || op == 47 { - saturated |= result < minimum || result > maximum; - result = result.clamp(minimum, maximum); - } else if op == 46 { - saturated |= result > mask; - result = result.min(mask); - } - } - gold.put(1, lane * bytes, &((result & mask) as u64).to_le_bytes()[..bytes]); - } - gold.put(1, 128, &u32::from(saturated).to_le_bytes()); - gold -} - -// Additional coverage for balanced integer reductions and bounded rounding carry. -fn integer_depth_cases(args: [u64; 8]) -> Vec { - let mut cases = vec![]; - for sew in 0..3 { - let bits = 8usize << sew; - let mask = u64::MAX >> (64 - bits); - let values = [1u64 << (bits - 1), mask >> 1, 0, 1, mask]; - let width = [0, 5, 6, 7][sew as usize]; - for reduction in [true, false] { - let operations: Vec<(u32, u32)> = if reduction { - (0..8).map(|op| (op, 2)).collect() - } else { - let mut ops = vec![(8, 2), (9, 2), (10, 2), (11, 2), (39, 0), (42, 0), (43, 0)]; - if sew < 2 { ops.extend([(46, 0), (47, 0)]); } - ops - }; - for (op, kind) in operations { - for mode in 0..(if reduction { 5 } else { 4 }) { - let mut a = args; - a[2] = 5; - let lmul = if 5 * bits > VLEN { 1 } else { 0 }; - let mut words = vec![ - i(0x57, 5, 7, 12, ((sew << 3) | lmul) as i32), - (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, - i(0x13, 6, 0, 11, 128), - (1 << 25) | (6 << 15) | (width << 12) | (16 << 7) | 7, - (1 << 25) | (11 << 15) | (width << 12) | (24 << 7) | 7, - ]; - let mut vm = 1; - if reduction { - if mode == 1 { - words.push((0x19 << 26) | (1 << 25) | (8 << 20) | (3 << 12) | 0x57); - vm = 0; - } else if mode == 2 { - words.push((0x1b << 26) | (1 << 25) | (2 << 12) | 0x57); - vm = 0; - } else if mode == 3 { - words.push(i(0x57, 5, 7, 0, 0xc00 | (sew << 3) as i32)); - } else if mode == 4 { - words.push(i(0x73, 0, 5, 1, 8)); - } - } else { - words.push(i(0x73, 0, 5, mode, 0xa)); - words.push(i(0x73, 0, 5, 0, 9)); - if op == 46 || op == 47 { - words.push((1 << 25) | (11 << 15) | ([5, 6, 7][sew as usize] << 12) | (8 << 7) | 7); - } - } - if !reduction && mode == 3 { words.push(i(0x73, 0, 5, 2, 8)); } - words.push((op << 26) | (vm << 25) | (8 << 20) | (16 << 15) | (kind << 12) | (24 << 7) | 0x57); - words.push(i(0x57, 5, 7, 12, ((sew << 3) | lmul) as i32)); - words.push((1 << 25) | (10 << 15) | (width << 12) | (24 << 7) | 0x27); - if !reduction { - words.push(i(0x73, 6, 2, 0, 9)); - words.push(s(0x23, 2, 10, 6, 128)); - } - words.push(0x8067); - let mut case = micro(&format!("depth_{}_e{bits}_op{op}_mode{mode}", if reduction {"reduce"} else {"round"}), &words, a); - for (index, value) in values.into_iter().enumerate() { - let bytes = bits / 8; - let rhs = if reduction || op == 39 || kind == 2 { value } else { [0, 1, bits as u64 - 1, bits as u64 / 2, 3][index] }; - case.initial.put(2, index * bytes, &value.to_le_bytes()[..bytes]); - case.initial.put(2, 128 + index * bytes, &rhs.to_le_bytes()[..bytes]); - } - if !reduction { case.fixed_expected = Some(fixed_gold(&case, bits, op, mode)); } - if reduction && mode == 4 { case.expected_cause = 2; } - cases.push(case); - } - } - } - } - cases -} - -impl Model { - pub fn new(kernel_dir: &Path) -> Self { - let mut cases = vec![]; - let mut args = [0; 8]; - args[0] = 1 << 32; - args[1] = 2 << 32; - for kernel in ["silu", "swiglu", "snake", "quant"] { - let lengths: &[u32] = if kernel == "quant" { - &[32, 64] - } else { - &[1, 31, 33, 65] - }; - for &n in lengths { - let mut a = args; - match kernel { - "swiglu" => { - a[2] = 3 << 32; - a[3] = u64::from(n); - } - "snake" => { - a[2] = 2; - a[3] = u64::from(n); - a[4] = 48; - a[5] = 56; - } - "quant" => { - a[1] = args[0] + 1024; - a[2] = args[1]; - a[3] = u64::from(n); - a[4] = args[0] + 1536; - } - _ => a[2] = u64::from(n), - } - if kernel == "silu" { a[3] = args[0] + 1024; } - if kernel == "swiglu" { a[4] = args[0] + 1024; } - let mut c = elf( - &kernel_dir.join(format!("{kernel}.elf")), - &format!("{kernel}_{n}"), - a, - ); - if kernel == "snake" { - c.initial.put( - 0, - 48, - &[ - 0.0f32.to_le_bytes(), - 0.5f32.to_le_bytes(), - 0.25f32.to_le_bytes(), - (-0.25f32).to_le_bytes(), - ] - .concat(), - ); - } - cases.push(c); - } - } - // Unaligned loads/stores and every RV32M operation, including divide by zero and overflow. - for &(a, b) in &[ - (17u32, 3u32), - (0x80000000, 0xffffffff), - (0xffffffef, 0), - (0xffffffff, 7), - ] { - let mut aargs = args; - aargs[2] = u64::from(a); - aargs[3] = u64::from(b); - let mut words = vec![]; - for f in 0..8 { - words.push(r(0x33, 5, f, 12, 13, 1)); - words.push(s(0x23, 2, 10, 5, f * 4 + 1)); - } - for f in [0, 1, 2, 4, 5] { - words.push(i(3, 6, f, 10, 1)); - words.push(s(0x23, 2, 10, 6, 48 + f * 4)); - } - words.push(0x8067); - cases.push(micro(&format!("scalar_m_{a:x}_{b:x}"), &words, aargs)); - } - for funct in [0, 1, 4, 5, 6, 7] { - for &(left, right) in &[(0xffffffff, 1), (7, 7)] { - let mut a = args; - a[2] = left; - a[3] = right; - let words = [ - i(0x13, 5, 0, 0, 0), - (13 << 20) | (12 << 15) | (funct << 12) | (4 << 8) | 0x63, - i(0x13, 5, 0, 0, 1), - s(0x23, 2, 10, 5, 0), - 0x8067, - ]; - cases.push(micro( - &format!("branch_{funct}_{left:x}_{right:x}"), - &words, - a, - )); - } - } - for sew in [2, 3] { - for (vl, start) in [(0, 0), (4, 0), (4, 1), (4, 4)] { - let width = if sew == 2 { 6 } else { 7 }; - let words = [ - i(0x57, 5, 7, 4, 0xc00 | (sew << 3)), - (1 << 25) | (11 << 15) | (width << 12) | (9 << 7) | 7, - i(0x57, 5, 7, 12, (sew << 3) | 3), - i(0x73, 0, 5, start, 8), - r(0x57, 9, 6, 13, 0, 33), - i(0x57, 5, 7, 4, 0xc00 | (sew << 3)), - (1 << 25) | (10 << 15) | (width << 12) | (9 << 7) | 0x27, - i(0x73, 7, 2, 0, 1), - s(0x23, 2, 10, 7, 64), - 0x8067, - ]; - let mut a = args; - a[2] = vl; - a[3] = 0x80000001; - let mut c = micro(&format!("scalar_insert_e{}_vl{vl}_start{start}", 8 << sew), &words, a); - if sew == 3 && ELEN == 32 { c.expected_cause = 2; } - cases.push(c); - } - } - for scalar in [0x3f800000u64, 0x7f800123] { - for (vl, start) in [(0, 0), (4, 0), (4, 1), (4, 4)] { - let words = [ - i(0x57, 5, 7, 4, 0xc00 | (2 << 3)), - (1 << 25) | (11 << 15) | (6 << 12) | (9 << 7) | 7, - i(7, 3, 2, 13, 0), - i(0x57, 5, 7, 12, (2 << 3) | 3), - i(0x73, 0, 5, start, 8), - r(0x57, 9, 5, 3, 0, 33), - i(0x57, 5, 7, 4, 0xc00 | (2 << 3)), - (1 << 25) | (10 << 15) | (6 << 12) | (9 << 7) | 0x27, - i(0x73, 7, 2, 0, 1), - s(0x23, 2, 10, 7, 64), - 0x8067, - ]; - let mut a = args; - a[2] = vl; - a[3] = 3 << 32; - let mut c = micro(&format!("floating_insert_{scalar:x}_vl{vl}_start{start}"), &words, a); - c.initial.put(3, 0, &scalar.to_le_bytes()); - cases.push(c); - } - } - let mut a = args; - a[2] = 17; - cases.push(micro( - "vset_avl_register_rules", - &[ - i(0x57, 5, 7, 12, 16), - s(0x23, 2, 10, 5, 0), - i(0x57, 5, 7, 0, 16), - s(0x23, 2, 10, 5, 4), - i(0x57, 0, 7, 0, 16), - i(0x73, 5, 2, 0, 0xc20), - s(0x23, 2, 10, 5, 8), - i(0x57, 5, 7, 3, 0xc10u32 as i32), - s(0x23, 2, 10, 5, 12), - 0x8067, - ], - a, - )); - // SEW and integer/fractional LMUL, VL=0, mask/tail preservation, CSR visibility. - for sew in 0..4 { - for lmul in [0, 1, 2, 3, 5, 6, 7] { - for avl in [0, 1, 17, 63] { - let vt = (sew << 3) | lmul; - let mut a = args; - a[2] = avl; - let width = [0, 5, 6, 7][sew as usize]; - let words = [ - i(0x57, 5, 7, 12, vt), - (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, - r(0x57, 16, 0, 8, 8, 0) | (1 << 25), - (1 << 25) | (10 << 15) | (width << 12) | (16 << 7) | 0x27, - i(0x73, 6, 2, 0, 0xc20), - s(0x23, 2, 10, 6, 1536), - i(0x73, 6, 2, 0, 0xc21), - s(0x23, 2, 10, 6, 1540), - 0x8067, - ]; - let mut c = micro( - &format!("vector_sew{}_lmul{lmul}_avl{avl}", 8 << sew), - &words, - a, - ); - if (8 << sew) > ELEN as i32 || (lmul >= 5 && (8 << sew) > (ELEN as i32 >> (8 - lmul))) { - c.expected_cause = 2; - } - cases.push(c); - } - } - } - for sew in 0..4 { - let mut a = args; - a[2] = 17; - let width = [0, 5, 6, 7][sew as usize]; - let words = [ - i(0x57, 5, 7, 12, sew << 3), - (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, - (0x18 << 26) | (1 << 25) | (8 << 20) | (3 << 12) | 0x57, - (8 << 20) | (8 << 15) | (16 << 7) | 0x57, - (1 << 25) | (10 << 15) | (width << 12) | (16 << 7) | 0x27, - 0x8067, - ]; - let mut c = micro(&format!("masked_sew{}", 8 << sew), &words, a); - if sew == 3 && ELEN == 32 { c.expected_cause = 2; } - cases.push(c); - } - for sew in [2, 3] { - let mut a = args; - a[2] = 17; - let width = if sew == 2 { 6 } else { 7 }; - let mut c = micro( - &format!("vector_floating_{}", 8 << sew), - &[ - i(0x57, 5, 7, 12, sew << 3), - (1 << 25) | (11 << 15) | (width << 12) | (8 << 7) | 7, - (1 << 25) | (8 << 20) | (8 << 15) | (1 << 12) | (16 << 7) | 0x57, - (0x2c << 26) | (1 << 25) | (8 << 20) | (8 << 15) | (1 << 12) | (16 << 7) | 0x57, - (1 << 25) | (10 << 15) | (width << 12) | (16 << 7) | 0x27, - (0x12 << 26) - | (1 << 25) - | (16 << 20) - | (1 << 15) - | (1 << 12) - | (24 << 7) - | 0x57, - i(0x13, 7, 0, 10, 1024), - (1 << 25) | (7 << 15) | (width << 12) | (24 << 7) | 0x27, - i(0x73, 6, 2, 0, 1), - s(0x23, 2, 10, 6, 1536), - 0x8067, - ], - a, - ); - if sew == 3 { - if ELEN == 32 { c.expected_cause = 2; } - for n in 0..17 { - c.initial - .put(2, n * 8, &(n as f64 / 4.0 - 2.0).to_le_bytes()); - } - } - cases.push(c); - } - // Scalar IEEE arithmetic/FMA/conversion flags are stored through CSR reads. - for fmt in 0..2 { - let load_f = if fmt == 0 { 2 } else { 3 }; - let step = if fmt == 0 { 4 } else { 8 }; - let mut c = micro( - &format!("floating_{}", 32 << fmt), - &[ - i(7, 1, load_f, 11, 0), - i(7, 2, load_f, 11, step), - r(0x53, 3, 0, 1, 2, fmt), - (2 << 27) | (fmt << 25) | (2 << 20) | (1 << 15) | (4 << 7) | 0x43, - s(0x27, load_f, 10, 3, 0), - s(0x27, load_f, 10, 4, 8), - r(0x53, 5, 0, 1, 0, 0x60 | fmt), - s(0x23, 2, 10, 5, 16), - i(0x73, 6, 2, 0, 1), - s(0x23, 2, 10, 6, 20), - 0x8067, - ], - args, - ); - if fmt == 0 { - c.initial.put( - 2, - 0, - &[1.5f32.to_le_bytes(), (-2.25f32).to_le_bytes()].concat(), - ); - } else { - if ELEN == 32 { c.expected_cause = 2; } - c.initial.put( - 2, - 0, - &[1.5f64.to_le_bytes(), (-2.25f64).to_le_bytes()].concat(), - ); - } - cases.push(c); - } - // Raw IEEE encodings preserve signaling NaNs and signed zero in the inputs. - for fmt in 0..2 { - let fixtures: &[(&str, [u64; 3])] = if fmt == 0 { - &[ - ("zeros", [0, 0x80000000, 0x3f800000]), - ("division_by_zero", [0x3f800000, 0, 0xbf800000]), - ("negative_zero", [0x80000000, 0x40000000, 0]), - ("qnan", [0x7fc12345, 0x3f800000, 0xbf800000]), - ("snan", [0x7f812345, 0x3f800000, 0]), - ("infinities", [0x7f800000, 0, 0xff800000]), - ("subnormal", [1, 0x3f000000, 0x00800000]), - ("overflow", [0x7f7fffff, 0x40000000, 0xff7fffff]), - ("conversion_ties", [0x3fc00000, 0xc0200000, 0x3f800000]), - ("division_inexact", [0x3f800000, 0x40400000, 0x80000000]), - ("fused_cancellation", [0x3f800001, 0x3f800001, 0xbf800000]), - ] - } else { - &[ - ("zeros", [0, 0x8000000000000000, 0x3ff0000000000000]), - ( - "division_by_zero", - [0x3ff0000000000000, 0, 0xbff0000000000000], - ), - ("negative_zero", [0x8000000000000000, 0x4000000000000000, 0]), - ( - "qnan", - [0x7ff8123456789abc, 0x3ff0000000000000, 0xbff0000000000000], - ), - ("snan", [0x7ff0123456789abc, 0x3ff0000000000000, 0]), - ("infinities", [0x7ff0000000000000, 0, 0xfff0000000000000]), - ("subnormal", [1, 0x3fe0000000000000, 0x0010000000000000]), - ( - "overflow", - [0x7fefffffffffffff, 0x4000000000000000, 0xffefffffffffffff], - ), - ( - "conversion_ties", - [0x3ff8000000000000, 0xc004000000000000, 0x3ff0000000000000], - ), - ( - "division_inexact", - [0x3ff0000000000000, 0x4008000000000000, 0x8000000000000000], - ), - ( - "fused_cancellation", - [0x3ff0000000000001, 0x3ff0000000000001, 0xbff0000000000000], - ), - ("narrowing_tie", [0x3ff0000010000000, 0x3ff0000000000000, 0]), - ] - }; - let width = if fmt == 0 { 2 } else { 3 }; - let bytes = 1usize << width; - for frm in 0..5 { - for &(name, values) in fixtures { - if frm != 0 - && !matches!( - name, - "conversion_ties" | "fused_cancellation" | "narrowing_tie" - ) - { - continue; - } - let mut a = args; - a[2] = 0x01000001; - a[3] = 0xffffffff; - let mut words = vec![i(0x73, 0, 5, frm, 2)]; - for f in 1..=3 { - words.push(i(7, f, width, 11, (f as usize - 1) as i32 * bytes as i32)); - } - // Clear and sample fflags for every operation; result and flags share a 32-byte record. - let mut operations = vec![]; - for op in [0, 4, 8, 12] { - operations.push((r(0x53, 4, 7, 1, 2, op | fmt), s(0x27, width, 10, 4, 0))); - } - for op in [0x43, 0x47, 0x4b, 0x4f] { - operations.push(( - (3 << 27) - | (fmt << 25) - | (2 << 20) - | (1 << 15) - | (7 << 12) - | (4 << 7) - | op, - s(0x27, width, 10, 4, 0), - )); - } - for source in [1, 2] { - for unsigned in 0..2 { - operations.push(( - r(0x53, 5, 7, source, unsigned, 0x60 | fmt), - s(0x23, 2, 10, 5, 0), - )); - } - } - for unsigned in 0..2 { - operations.push(( - r(0x53, 4, 7, 12 + unsigned, unsigned, 0x68 | fmt), - s(0x27, width, 10, 4, 0), - )); - } - let target = 1 - fmt; - if fmt == 1 { operations.push(( - r(0x53, 4, 7, 1, fmt, 0x20 | target), - s(0x27, if target == 0 { 2 } else { 3 }, 10, 4, 0), - )); } - for (n, (operation, store)) in operations.into_iter().enumerate() { - let offset = n as u32 * 32; - words.extend([ - i(0x73, 0, 5, 0, 1), - operation, - store | ((offset >> 5) << 25), - i(0x73, 6, 2, 0, 1), - s(0x23, 2, 10, 6, offset + 16), - ]); - } - words.extend([i(0x73, 6, 2, 0, 2), s(0x23, 2, 10, 6, 1536), 0x8067]); - let mut c = micro( - &format!("scalar_ieee_{}_frm{frm}_{name}", 32 << fmt), - &words, - a, - ); - if fmt == 1 && ELEN == 32 { c.expected_cause = 2; } - for (n, bits) in values.into_iter().enumerate() { - c.initial.put(2, n * bytes, &bits.to_le_bytes()[..bytes]); - } - cases.push(c); - } - // The vector case runs the same exceptional inputs lane by lane. - let mut a = args; - a[2] = fixtures.len() as u64; - a[3] = 3 << 32; - a[4] = 5 << 32; - let vwidth: u32 = if fmt == 0 { 6 } else { 7 }; - let load = - |vd: u32, rs: u32| (1 << 25) | (rs << 15) | (vwidth << 12) | (vd << 7) | 7; - let store = |vd: u32| (1 << 25) | (7 << 15) | (vwidth << 12) | (vd << 7) | 0x27; - let mut words = vec![ - i(0x73, 0, 5, frm, 2), - i(0x57, 5, 7, 12, (fmt as i32 + 2) << 3), - load(8, 11), - load(12, 13), - ]; - for (n, operation) in [0, 2, 0x24, 0x20, 0x2c].into_iter().enumerate() { - let offset = n as u32 * 128; - words.extend([ - i(0x73, 0, 5, 0, 1), - load(20, 14), - (operation << 26) - | (1 << 25) - | (12 << 20) - | (8 << 15) - | (1 << 12) - | (20 << 7) - | 0x57, - i(0x13, 7, 0, 10, offset as i32), - store(20), - i(0x73, 6, 2, 0, 1), - s(0x23, 2, 10, 6, 1024 + n as u32 * 4), - ]); - } - for unsigned in 0..2 { - words.extend([ - i(0x73, 0, 5, 0, 1), - (0x12 << 26) - | (1 << 25) - | (8 << 20) - | ((1 - unsigned) << 15) - | (1 << 12) - | (24 << 7) - | 0x57, - i(0x13, 7, 0, 10, 640 + unsigned as i32 * 128), - store(24), - i(0x73, 6, 2, 0, 1), - s(0x23, 2, 10, 6, 1044 + unsigned * 4), - ]); - } - words.push(0x8067); - let mut c = micro(&format!("vector_ieee_{}_frm{frm}", 32 << fmt), &words, a); - if fmt == 1 && ELEN == 32 { c.expected_cause = 2; } - for (n, (_, values)) in fixtures.iter().enumerate() { - for (bank, bits) in [(2, values[0]), (3, values[1]), (5, values[2])] { - c.initial.put(bank, n * bytes, &bits.to_le_bytes()[..bytes]); - } - } - cases.push(c); - } - } - cases.push(micro( - "bank_line_crossing_word", - &[i(3, 5, 2, 11, 15), s(0x23, 2, 10, 5, 15), 0x8067], - args, - )); - let mut double_load = micro( - "bank_line_crossing_double", - &[i(7, 5, 3, 11, 11), s(0x27, 3, 10, 5, 11), 0x8067], - args, - ); - if ELEN == 32 { double_load.expected_cause = 2; } - cases.push(double_load); - let mut a = args; - a[2] = 17; - let mut c = micro( - "illegal_group_alignment", - &[ - i(0x57, 5, 7, 12, 17), - (1 << 25) | (11 << 15) | (6 << 12) | (9 << 7) | 7, - ], - a, - ); - c.expected_cause = 2; - cases.push(c); - cases.push(micro("illegal_opcode", &[0xffffffff], args)); - cases.push(micro("load_fault", &[i(3, 5, 2, 11, 0)], { - let mut a = args; - a[1] = 6 << 32; - a - })); - cases.push(micro("store_fault", &[s(0x23, 2, 10, 11, 0)], { - let mut a = args; - a[0] = (1 << 32) + 4094; - a - })); - for (n, cause) in [ - (cases.len() - 3, 2), - (cases.len() - 2, 5), - (cases.len() - 1, 7), - ] { - cases[n].expected_cause = cause; - } - cases.extend(integer_depth_cases(args)); - let mut local = micro( - "local_stack_and_constants", - &[ - i(0x13, 2, 0, 2, -16), - s(0x23, 2, 2, 11, 0), - s(0x23, 2, 2, 12, 4), - i(3, 5, 2, 2, 0), - i(3, 6, 2, 2, 4), - s(0x23, 2, 10, 5, 0), - s(0x23, 2, 10, 6, 4), - 0x400003b7, - i(3, 5, 2, 7, 0), - s(0x23, 2, 10, 5, 8), - i(0x13, 2, 0, 2, 16), - 0x8067, - ], - args, - ); - local.constants[..4].copy_from_slice(&0x72cafe31u32.to_le_bytes()); - cases.push(local); - let mut readonly = micro( - "store_readonly_constants", - &[0x400002b7, s(0x23, 2, 5, 11, 0), 0x8067], - args, - ); - readonly.expected_cause = 7; - cases.push(readonly); - for kernel in ["sin", "cos"] { - let values: Vec = (0..65).map(|n| (n as f32 - 32.0) * 0.75).chain([ - 120.0, -120.0, 10000.0, -10000.0, 1e10, -1e10, 1e30, -1e30, - f32::MAX, -f32::MAX, f32::from_bits(1), -f32::from_bits(1), - 0.0, -0.0, f32::INFINITY, f32::NEG_INFINITY, f32::NAN, - ]).collect(); - let mut a = args; - a[2] = values.len() as u64; - a[3] = a[0] + 1024; - let mut c = elf(&kernel_dir.join(format!("{kernel}.elf")), &format!("{kernel}_fp32_range"), a); - for (n, value) in values.iter().enumerate() { c.initial.put(2, n * 4, &value.to_le_bytes()); } - cases.push(c); - } - for kernel in ["gelu", "tanh"] { - let mut a = args; - a[2] = 37; - a[3] = a[0] + 1024; - cases.push(elf(&kernel_dir.join(format!("{kernel}.elf")), &format!("{kernel}_fp32"), a)); - } - let b = initial(); - Self { - cases, - current: 0, - actual: b.clone(), - expected: b, - engine: Engine::new(VLEN, ELEN, IBUF_BYTES), - fault: None, - } - } - pub fn select(&mut self, n: usize) { - self.current = n; - let c = &self.cases[n]; - self.actual = c.initial.clone(); - let descriptor = [ - u64::from(c.entry), - c.program.len() as u64, - 0x40002000, - u64::from(c.args[0]), - u64::from(c.args[1]), - u64::from(c.args[2]), - u64::from(c.args[3]), - u64::from(c.args[4]), - u64::from(c.args[5]), - u64::from(c.args[6]), - u64::from(c.args[7]), - 0, - ]; - for (index, value) in descriptor.iter().enumerate() { - self.actual.put(1, 2048 + index * 8, &value.to_le_bytes()); - } - self.expected = match &c.fixed_expected { - Some(gold) => { - let mut expected = gold.clone(); - let descriptor = BANK_BYTES + 2048..BANK_BYTES + 2144; - expected.0[descriptor.clone()].copy_from_slice(&self.actual.0[descriptor]); - expected - } - None => self.actual.clone(), - }; - self.engine.load_constants(n % 2, &c.constants); - self.engine.load_program(n % 2, &c.program); - self.fault = if c.fixed_expected.is_some() { None } else { self - .engine - .run( - n % 2, - c.entry, - c.program.len() as u32, - c.args.map(u64::from), - 0x40002000, - &mut self.expected, - ) - .err() }; - assert_eq!( - self.fault.map(|f| f.cause).unwrap_or(0), - c.expected_cause, - "unexpected reference outcome for {}: {:?}", - c.name, self.fault - ); - let read_float = |banks: &Banks, address: u64| { - f32::from_le_bytes(banks.0[banks.range(address, 4).unwrap()].try_into().unwrap()) - }; - let operation = c.name.split('_').next().unwrap(); - if matches!(operation, "silu" | "swiglu" | "snake" | "sin" | "cos") { - let length = c.args[if matches!(operation, "swiglu" | "snake") { 3 } else { 2 }] as usize; - let channels = if operation == "snake" { c.args[2] as usize } else { 1 }; - for channel in 0..channels { - for index in 0..length { - let offset = ((channel * length + index) * 4) as u64; - let x = read_float(&c.initial, c.args[1] + offset); - let expected = match operation { - "silu" => x / (1.0 + (-x).exp()), - "swiglu" => x / (1.0 + (-x).exp()) * read_float(&c.initial, c.args[2] + offset), - "snake" => { - let alpha = read_float(&c.initial, c.args[4] + channel as u64 * 4).exp(); - let beta = read_float(&c.initial, c.args[5] + channel as u64 * 4).exp() + 1e-9; - let sine = (x * alpha).sin(); - x + sine * sine / beta - } - "sin" => x.sin(), - "cos" => x.cos(), - _ => unreachable!(), - }; - let actual = read_float(&self.expected, c.args[0] + offset); - assert!(if expected.is_nan() { actual.is_nan() } else { - actual == expected || (actual - expected).abs() <= 2e-6 * (1.0 + expected.abs()) - }, "{} element {index}: {actual} != libm {expected}", c.name); - } - } - } - eprintln!("RVV case {n}: {}", c.name); - } -} diff --git a/arch/src/main/scala/framework/rvv/src/main/resources/package.sv b/arch/src/main/scala/framework/rvv/src/main/resources/package.sv deleted file mode 100644 index 9395630e6..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/resources/package.sv +++ /dev/null @@ -1,97 +0,0 @@ -package rvv_pkg; - timeunit 1ns; timeprecision 1ps; - import uvm_pkg::*; - import ip_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function void rvv_model_init(); - import "DPI-C" function int unsigned rvv_case_count(); - import "DPI-C" function void rvv_case_select(int unsigned n); - import "DPI-C" function longint unsigned rvv_case_meta(int unsigned field); - import "DPI-C" function int unsigned rvv_image_bytes(); - import "DPI-C" function int unsigned rvv_program_word(int unsigned n); - import "DPI-C" function int unsigned rvv_image_word(int unsigned n); - import "DPI-C" function void rvv_memory_masked_write( - longint unsigned address, - longint unsigned data, - int unsigned mask - ); - import "DPI-C" function longint unsigned rvv_expected_status(int unsigned field); - import "DPI-C" function longint unsigned rvv_memory_read( - longint unsigned address, - int unsigned size - ); - import "DPI-C" function int unsigned rvv_memory_error( - longint unsigned address, - int unsigned size - ); - import "DPI-C" function void rvv_memory_write( - longint unsigned address, - int unsigned size, - longint unsigned data, - int unsigned mask - ); - import "DPI-C" function int unsigned rvv_compare_memory(); - import "DPI-C" function int unsigned rvv_compare_image(); - - class launch_item extends uvm_sequence_item; - bit execute; - bit protocol_fault; - bit [31:0] protocol_cause, protocol_pc, protocol_instruction; - bit [63:0] protocol_tval; - bit [3:0] rob; - `uvm_object_utils(launch_item) - function new(string name = "launch_item"); - super.new(name); - endfunction - endclass - class completion_item extends uvm_sequence_item; - bit fault; - bit [15:0] write_bank; - bit [3:0] rob; - bit [31:0] pc, instruction, cause; - bit [63:0] tval; - `uvm_object_utils_begin(completion_item) - `uvm_field_int(fault, UVM_DEFAULT) - `uvm_field_int(rob, UVM_DEFAULT) - `uvm_field_int(write_bank, UVM_DEFAULT) - `uvm_field_int(pc, UVM_DEFAULT) - `uvm_field_int(instruction, UVM_DEFAULT | UVM_NOCOMPARE) - `uvm_field_int(cause, UVM_DEFAULT) - `uvm_field_int(tval, UVM_DEFAULT) - `uvm_object_utils_end - function new(string name = "completion_item"); - super.new(name); - endfunction - function bit do_compare(uvm_object rhs, uvm_comparer comparer); - completion_item other; - if (!$cast(other, rhs)) return 0; - return super.do_compare(rhs, comparer) && (!fault || instruction == other.instruction); - endfunction - endclass - class kernel_model extends reference_model #(launch_item, completion_item); - `uvm_component_utils(kernel_model) - function new(string name, uvm_component parent); - super.new(name, parent); - endfunction - function void write(launch_item t); - completion_item expected = completion_item::type_id::create("expected"); - expected.rob = t.rob; - expected.fault = t.execute ? rvv_expected_status(0) : 0; - expected.pc = t.execute ? rvv_expected_status(1) : 0; - expected.instruction = t.execute ? rvv_expected_status(2) : 0; - expected.cause = t.execute ? rvv_expected_status(3) : 0; - expected.tval = t.execute ? rvv_expected_status(4) : 0; - if (t.protocol_fault) begin - expected.fault = 1; - expected.pc = t.protocol_pc; - expected.instruction = t.protocol_instruction; - expected.cause = t.protocol_cause; - expected.tval = t.protocol_tval; - end - expected.write_bank = t.execute && !expected.fault ? 1 : 0; - expected_ap.write(expected); - endfunction - endclass - typedef checked_env#(launch_item, completion_item, kernel_model) kernel_env; - `include "test.svh" -endpackage diff --git a/arch/src/main/scala/framework/rvv/src/main/resources/test.svh b/arch/src/main/scala/framework/rvv/src/main/resources/test.svh deleted file mode 100644 index 2d7e7bf8c..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/resources/test.svh +++ /dev/null @@ -1,265 +0,0 @@ -class protocol_test extends ip_test; - `uvm_component_utils(protocol_test) - virtual rvv_if vif; - kernel_env test_env; - int unsigned completed = 0; - bit protocol_fault = 0; - bit reject_write = 0; - bit [31:0] protocol_cause, protocol_pc, protocol_instruction; - bit [63:0] protocol_tval; - int unsigned requests[`RVV_MEMORY_PORTS]; - function new(string name, uvm_component parent); - super.new(name, parent); - timeout = 40ms; - endfunction - function void build_phase(uvm_phase phase); - super.build_phase(phase); - if (!uvm_config_db#(virtual rvv_if)::get(this, "", "vif", vif)) - `uvm_fatal("VIF", "RVV interface missing") - test_env = kernel_env::type_id::create("env", this); - endfunction - task memory_port(int p); - bit read_pending = 0, write_pending = 0; - int read_delay, write_delay; - bit [31:0] seed = 32'h629731a5 ^ (p * 32'h1234567); - longint unsigned address; - forever begin - @(negedge vif.clock); - seed = {seed[30:0], seed[31] ^ seed[21] ^ seed[1] ^ seed[0]}; - if (vif.reset) begin - read_pending = 0; - write_pending = 0; - vif.read_ready[p] = 0; - vif.write_ready[p] = 0; - vif.read_response_valid[p] = 0; - vif.write_response_valid[p] = 0; - end else begin - vif.read_ready[p] = !read_pending && seed[0]; - vif.write_ready[p] = !write_pending && seed[1]; - if (read_pending && read_delay != 0) read_delay--; - if (write_pending && write_delay != 0) write_delay--; - vif.read_response_valid[p] = read_pending && read_delay == 0; - vif.write_response_valid[p] = write_pending && write_delay == 0; - end - @(posedge vif.clock); - if (!vif.reset) begin - if (vif.read_valid[p] && vif.read_ready[p]) begin - requests[p]++; - read_pending = 1; - read_delay = 1 + seed[4:2]; - address = (longint'(vif.read_bank[p] + vif.read_group[p]) << 32) | (int'(vif.read_row[p]) << 4); - vif.read_data[p] = {rvv_memory_read(address + 8, 3), rvv_memory_read(address, 3)}; - end - if (vif.write_valid[p] && vif.write_ready[p]) begin - requests[p]++; - write_pending = 1; - write_delay = 1 + seed[7:5]; - address = (longint'(vif.write_bank[p] + vif.write_group[p]) << 32) | (int'(vif.write_row[p]) << 4); - if (!reject_write) begin - rvv_memory_masked_write(address, vif.write_data[p][63:0], vif.write_mask[p][7:0]); - rvv_memory_masked_write(address + 8, vif.write_data[p][127:64], - vif.write_mask[p][15:8]); - end - vif.write_ok[p] = !reject_write; - end - if (vif.read_response_valid[p] && vif.read_response_ready[p]) read_pending = 0; - if (vif.write_response_valid[p] && vif.write_response_ready[p]) write_pending = 0; - end - end - endtask - task monitor(); - forever begin - @(posedge vif.clock); - if (!vif.reset) begin - if (vif.command_valid && vif.command_ready) begin - launch_item item = launch_item::type_id::create("launch"); - item.execute = vif.command_funct7 == 79; - item.protocol_fault = protocol_fault; - item.protocol_cause = protocol_cause; - item.protocol_tval = protocol_tval; - item.protocol_pc = protocol_pc; - item.protocol_instruction = protocol_instruction; - item.rob = vif.command_rob; - test_env.input_export.write(item); - end - if (vif.done_valid && vif.done_ready) begin - completion_item item = completion_item::type_id::create("completion"); - item.fault = vif.done_fault; - item.pc = vif.done_pc; - item.instruction = vif.done_instruction; - item.cause = vif.done_cause; - item.tval = vif.done_tval; - item.rob = vif.done_rob; - item.write_bank = vif.done_write_bank; - test_env.output_export.write(item); - end - end - end - endtask - task command(int funct7, longint unsigned rs1, longint unsigned rs2); - @(negedge vif.clock); - vif.command_funct7 = funct7; - vif.command_rs1 = rs1; - vif.command_rs2 = rs2; - vif.command_rob = completed % 16; - vif.command_valid = 1; - do @(posedge vif.clock); while (!vif.command_ready); - @(negedge vif.clock); - vif.command_valid = 0; - endtask - task retire(); - bit [250:0] held; - while (!vif.done_valid) @(negedge vif.clock); - held = { - vif.done_rob, - vif.done_write_bank, - vif.done_fault, - vif.done_pc, - vif.done_instruction, - vif.done_cycles, - vif.done_cause, - vif.done_tval, - vif.done_fflags, - vif.done_vxsat - }; - repeat (5) begin - @(negedge vif.clock); - if(!vif.done_valid || held !== {vif.done_rob,vif.done_write_bank,vif.done_fault,vif.done_pc, - vif.done_instruction,vif.done_cycles,vif.done_cause,vif.done_tval,vif.done_fflags,vif.done_vxsat}) - `uvm_fatal("COMPLETION", "completion changed under backpressure") - if (vif.command_ready || !vif.busy) - `uvm_fatal("COMPLETION", "completion allowed command before retirement") - end - vif.done_ready = 1; - @(negedge vif.clock); - vif.done_ready = 0; - completed++; - test_env.scoreboard.wait_checked(completed); - endtask - task upload(); - command(44, (longint'(rvv_image_bytes()) << 30) | ((6 + rvv_case_meta(0)) << 20), 64'h10000000); - for (int word_index = 0; word_index < rvv_image_bytes() / 4; word_index++) begin - @(negedge vif.clock); - vif.image_valid = 1; - vif.image_data = rvv_image_word(word_index); - do @(posedge vif.clock); while (!vif.image_ready); - end - @(negedge vif.clock); - vif.image_valid = 0; - retire(); - if (!rvv_compare_image()) `uvm_fatal("IMAGE", "image load modified compute banks") - endtask - task execute(); - vif.reset = 1; - vif.image_valid = 0; - vif.command_valid = 0; - vif.done_ready = 0; - foreach (vif.read_ready[p]) begin - vif.read_ready[p] = 0; - vif.write_ready[p] = 0; - vif.read_response_valid[p] = 0; - vif.write_response_valid[p] = 0; - vif.read_data[p] = 0; - vif.write_ok[p] = 0; - end - rvv_model_init(); - fork - begin - for (int p = 0; p < `RVV_MEMORY_PORTS; p++) begin - automatic int port = p; - fork - memory_port(port); - join_none - end - wait fork; - end - monitor(); - join_none - repeat (4) @(negedge vif.clock); - vif.reset = 0; - for (int case_index = 0; case_index < rvv_case_count(); case_index++) begin - rvv_case_select(case_index); - upload(); - command(79, (64'd1 << 20) | ((6 + rvv_case_meta(0)) << 10), 2048); - while (!vif.done_valid) @(negedge vif.clock); - if (!rvv_compare_memory()) - `uvm_fatal("REFERENCE", $sformatf("case %0d bank contents differ", case_index)) - retire(); - `uvm_info("RVV", $sformatf("case %0d completed and compared", case_index), UVM_LOW) - end - foreach (requests[p]) - if (requests[p] == 0) `uvm_fatal("PORTS", $sformatf("bank port %0d was never exercised", p)) - // A malformed header reports a command fault before any code/data is written. - protocol_fault = 1; - protocol_cause = 2; - protocol_tval = 32'hdeadbeef; - command(44, (longint'(rvv_image_bytes()) << 30) | (64'd6 << 20), 0); - for (int index = 0; index < 6; index++) begin - @(negedge vif.clock); - vif.image_valid = 1; - vif.image_data = index == 0 ? 32'hdeadbeef : rvv_image_word(index); - do @(posedge vif.clock); while (!vif.image_ready); - end - @(negedge vif.clock); - vif.image_valid = 0; - retire(); - protocol_tval = 25; - command(44, (64'd25 << 30) | (64'd6 << 20), 0); - retire(); - protocol_cause = 1; - protocol_tval = (64'd1 << 20) | (64'd6 << 10); - command(79, protocol_tval, 2048); - retire(); - protocol_cause = 2; - protocol_tval = 127; - command(127, 0, 0); - retire(); - // Writable globals and BSS are forbidden in a kernel image. - for (int invalid_field = 3; invalid_field <= 5; invalid_field += 2) begin - protocol_cause = 2; - protocol_tval = invalid_field == 3 ? 32'h50000 : 4; - command(44, (longint'(rvv_image_bytes()) << 30) | (64'd6 << 20), 0); - for (int index = 0; index < 6; index++) begin - @(negedge vif.clock); - vif.image_valid = 1; - vif.image_data = index == invalid_field ? - (invalid_field == 3 ? 32'h50000 : 4) : rvv_image_word(index); - do @(posedge vif.clock); while (!vif.image_ready); - end - @(negedge vif.clock); - vif.image_valid = 0; - retire(); - end - // A failed compute-bank write acknowledgement is an execution fault. - protocol_fault = 0; - rvv_case_select(14); - upload(); - protocol_fault = 1; - protocol_cause = 7; - protocol_tval = (64'd1 << 32) + 1; - protocol_pc = 4; - protocol_instruction = rvv_program_word(1); - reject_write = 1; - command(79, (64'd1 << 20) | ((6 + rvv_case_meta(0)) << 10), 2048); - retire(); - reject_write = 0; - protocol_pc = 0; - protocol_instruction = 0; - protocol_fault = 0; - // Reset cancels a command while it waits for a descriptor bank response. - rvv_case_select(rvv_case_count() - 2); - upload(); - command(79, (64'd1 << 20) | ((6 + rvv_case_meta(0)) << 10), 2048); - do @(posedge vif.clock); while (!(vif.read_valid[0] && vif.read_ready[0])); - @(negedge vif.clock); - vif.reset = 1; - test_env.scoreboard.cancel_pending(); - repeat (3) @(negedge vif.clock); - vif.reset = 0; - repeat (5) begin - @(negedge vif.clock); - if (vif.done_valid || vif.busy) `uvm_fatal("RESET", "reset did not cancel pending kernel") - end - `uvm_info("RVV", $sformatf("checked %0d Blink commands", completed), UVM_LOW) - endtask -endclass diff --git a/arch/src/main/scala/framework/rvv/src/main/resources/topkernel.f b/arch/src/main/scala/framework/rvv/src/main/resources/topkernel.f deleted file mode 100644 index 975cf783a..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/resources/topkernel.f +++ /dev/null @@ -1,8 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -@VERIFY@/uvm/src/ip/package.sv -+incdir+@RESOURCES@ -+incdir+@RTL@/KernelEngine -@RESOURCES@/interface.sv -@RESOURCES@/package.sv --F @RTL@/KernelEngine/filelist.f -@RESOURCES@/topkernel_tb.sv diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/Emit.scala b/arch/src/main/scala/framework/rvv/src/main/scala/Emit.scala deleted file mode 100644 index c153ea413..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/Emit.scala +++ /dev/null @@ -1,35 +0,0 @@ -package framework.rvv - -import framework.top.GlobalConfig -import framework.rvv.configs.RvvParam -import upickle.default.read - -object Emit extends App { - val defaults = GlobalConfig() - - val b = defaults.copy( - rvv = read[RvvParam](os.read(os.pwd / "src" / "main" / "resources" / "rvv.json")), - memDomain = defaults.memDomain.copy( - bankNum = 6, - bankWidth = 128, - bankEntries = 256, - bankMaskLen = 16, - virtualBankCount = 6, - nCores = 1, - computeCoreIds = Seq(0) - ), - frontend = defaults.frontend.copy( - rob_entries = 16, - bank_id_len = 3, - vbank_id_upper_bound = 5, - iter_len = 16 - ), - tile = defaults.tile.copy(xLen = 64) - ) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new KernelEngine(b), - firtoolOpts = args, - args = Array("--target-dir", "build/KernelEngine", "--split-verilog") - ) -} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/configs/RvvParam.scala b/arch/src/main/scala/framework/rvv/src/main/scala/configs/RvvParam.scala deleted file mode 100644 index 345b157fc..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/configs/RvvParam.scala +++ /dev/null @@ -1,45 +0,0 @@ -package framework.rvv.configs - -import chisel3.util.{isPow2, log2Ceil} -import upickle.default._ - -case class RvvParam( - enable: Boolean, - laneNumber: Int, - vLen: Int, - eLen: Int, - iBufWords: Int, - memoryPorts: Int) { - require(laneNumber > 0) - require(vLen >= 128 && vLen >= eLen && isPow2(vLen)) - require(eLen == 32 || eLen == 64) - require(iBufWords > 0 && isPow2(iBufWords)) - require(memoryPorts > 0) - require(laneNumber <= memoryPorts) - - val wordBits: Int = eLen - val wordOffsetBits: Int = log2Ceil(wordBits) - val maxSew: Int = log2Ceil(eLen / 8) - val wordsPerRegister: Int = vLen / wordBits - val registerBanks: Int = math.min(laneNumber, wordsPerRegister) - require(isPow2(registerBanks), "RVV register bank count must be a power of two") - - val elementsPerRegister: Int = vLen / eLen - val constBytes: Int = 4096 - val stackBytes: Int = 4096 -} - -object RvvParam { - - def apply(): RvvParam = RvvParam( - enable = false, - laneNumber = 4, - vLen = 1024, - eLen = 32, - iBufWords = 1024, - memoryPorts = 4 - ) - - implicit val rw: ReadWriter[RvvParam] = macroRW[RvvParam] - -} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/core/VectorCore.scala b/arch/src/main/scala/framework/rvv/src/main/scala/core/VectorCore.scala deleted file mode 100644 index 39df8c3c3..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/core/VectorCore.scala +++ /dev/null @@ -1,921 +0,0 @@ -package framework.rvv - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig - -@instantiable -class VectorCore(val b: GlobalConfig) extends Module { - val p = b.rvv - - @public - val io = IO(new Bundle { - val initialize = Input(Bool()) - val roundingMode = Input(UInt(3.W)) - val vxrm = Input(UInt(2.W)) - val vstartWrite = Flipped(Valid(UInt(64.W))) - val vl = Output(UInt(32.W)) - val vtype = Output(UInt(64.W)) - val vstart = Output(UInt(64.W)) - val issue = Flipped(Decoupled(new VectorIssue)) - val result = Decoupled(new VectorResult) - val memoryRequest = Vec(p.memoryPorts, Decoupled(new VectorMemoryRequest)) - val memoryResponse = - Vec(p.memoryPorts, Flipped(Decoupled(new VectorMemoryResponse))) - val busy = Output(Bool()) - }) - - val idle :: snapshot :: capture :: execute :: commit :: Nil = Enum(5) - val state = RegInit(idle) - val resultValid = RegInit(false.B) - val result = RegInit(0.U.asTypeOf(new VectorResult)) - val instruction = Reg(UInt(32.W)) - val scalar1 = Reg(UInt(64.W)) - val scalar2 = Reg(UInt(64.W)) - val floatScalar = Reg(UInt(64.W)) - val vl = RegInit(0.U(32.W)) - val vtype = RegInit("h8000000000000000".U(64.W)) - val vstart = RegInit(0.U(64.W)) - val base = Reg(UInt(32.W)) - val readPortCount = math.max(p.laneNumber, p.memoryPorts) - val wordsPerRegister = p.wordsPerRegister - val maskWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) - val operandWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) - val sourceWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) - val destinationWords = Reg(Vec(readPortCount, UInt(p.wordBits.W))) - val readPhase = RegInit(0.U(3.W)) - val readSent = RegInit(false.B) - val snapshotSent = RegInit(false.B) - val firstBatch = RegInit(true.B) - val writeSent = RegInit(false.B) - val resumeState = Reg(UInt(3.W)) - val finishRequested = WireDefault(false.B) - val accumulator = Reg(UInt(p.wordBits.W)) - val count = Reg(UInt(32.W)) - val offered = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) - val issued = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) - val received = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) - val memoryData = Reg(Vec(p.memoryPorts, UInt(p.wordBits.W))) - val memoryFailed = RegInit(VecInit(Seq.fill(p.memoryPorts)(false.B))) - val divBatchValid = RegInit(false.B) - val divActive = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) - val divStarted = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) - val divDone = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) - val divData = Reg(Vec(p.laneNumber, UInt(p.wordBits.W))) - val fpStarted = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) - val fpDone = RegInit(VecInit(Seq.fill(p.laneNumber)(false.B))) - val fpData = Reg(Vec(p.laneNumber, UInt(p.wordBits.W))) - val fpFlags = Reg(Vec(p.laneNumber, UInt(5.W))) - val fpLegal = Reg(Vec(p.laneNumber, Bool())) - - val opcode = instruction(6, 0) - val rd = instruction(11, 7) - val kind = instruction(14, 12) - val rs1 = instruction(19, 15) - val rs2 = instruction(24, 20) - val vm = instruction(25) - val op = instruction(31, 26) - val sew = vtype(4, 3) - val lmul = Cat(vtype(2), vtype(2, 0)).asSInt - val configure = opcode === "h57".U && kind === 7.U - val load = opcode === "h07".U - val store = opcode === "h27".U - val memory = load || store - val fp = opcode === "h57".U && (kind === 1.U || kind === 5.U) - val integer = - opcode === "h57".U && (kind === 0.U || kind === 2.U || kind === 3.U || kind === 4.U || kind === 6.U) - val divide = integer && (kind === 2.U || kind === 6.U) && op >= 32.U && op <= 35.U - val multiply = kind === 2.U || kind === 6.U - val vectorOperand = kind === 0.U || kind === 1.U || kind === 2.U - val maskLogical = integer && kind === 2.U && op >= 24.U && op <= 31.U - - val compare = - (integer && !multiply && (op >= 24.U && op <= 31.U || op === 17.U || op === 19.U)) || - (fp && Seq(24, 25, 27, 28, 29, 31).map(n => op === n.U).reduce(_ || _)) - - val carryOperation = integer && !multiply && op >= 16.U && op <= 19.U - val carryMask = carryOperation && op(0) - val merge = op === 23.U && (fp || integer && !multiply) - val scalarMoveOut = - op === 16.U && rs1 === 0.U && vm && (kind === 2.U || kind === 1.U) - val scalarMoveIn = - op === 16.U && rs2 === 0.U && vm && (kind === 6.U || kind === 5.U) - val population = integer && kind === 2.U && op === 16.U && rs1 === 16.U - val first = integer && kind === 2.U && op === 16.U && rs1 === 17.U - val identity = - integer && kind === 2.U && op === 20.U && rs1 === 17.U && rs2 === 0.U - val intReduction = integer && kind === 2.U && op <= 7.U - val fpReduction = - fp && kind === 1.U && (op === 1.U || op === 3.U || op === 5.U || op === 7.U) - val reduction = intReduction || fpReduction - val wholeMove = integer && kind === 3.U && op === 39.U - val memMode = instruction(27, 26) - - val memWidth = MuxLookup(kind, 0.U(2.W))( - Seq(0.U -> 0.U, 5.U -> 1.U, 6.U -> 2.U, 7.U -> 3.U) - ) - - val wholeMemory = memory && memMode === 0.U && rs2 === 8.U - val indexed = memMode(0) - val memSew = Mux(indexed && !wholeMemory, sew, memWidth) - val intWide = integer && multiply && op >= 48.U - val intNarrow = integer && !multiply && op >= 44.U && op <= 47.U - val floatWide = fp && op === 18.U && rs1 >= 8.U && rs1 <= 15.U - val floatNarrow = fp && op === 18.U && rs1 >= 16.U && rs1 <= 23.U - val extension = integer && multiply && op === 18.U - - val sourceSew = Mux( - memory, - Mux(indexed, memWidth, memSew), - Mux( - intNarrow || floatNarrow || intWide && op >= 52.U && op <= 55.U, - sew +& 1.U, - Mux( - extension, - sew - Mux(rs1(2, 1) === 1.U, 3.U, Mux(rs1(2, 1) === 2.U, 2.U, 1.U)), - sew - ) - ) - ) - - val destinationSew = - Mux(memory, memSew, Mux(intWide || floatWide, sew +& 1.U, sew)) - - val limit = Mux( - wholeMemory, - ((instruction(31, 29) +& 1.U) * p.vLen.U) >> (memWidth +& 3.U), - Mux(wholeMove, ((rs1 +& 1.U) * p.vLen.U) >> (sew +& 3.U), vl) - ) - - def element(word: UInt, index: UInt, width: UInt): UInt = { - val shift = (index << (width.pad(3) + 3.U))(p.wordOffsetBits - 1, 0) - val mask = MuxLookup(width, Fill(p.wordBits, 1.U(1.W)))( - Seq(0.U -> 255.U, 1.U -> 65535.U, 2.U -> "hffffffff".U) - ) - ((word >> shift) & mask)(p.eLen - 1, 0) - } - - def maskElement(word: UInt, index: UInt): Bool = (word >> index(p.wordOffsetBits - 1, 0))(0) - - def wordAddress( - register: UInt, - index: UInt, - width: UInt, - maskBit: Bool = false.B - ): UInt = { - val shift = Mux(maskBit, index, index << (width.pad(3) + 3.U)) - val address = Mux(maskBit, register, register + (shift >> log2Ceil(p.vLen)))(4, 0) - val within = shift.pad(log2Ceil(p.vLen))(log2Ceil(p.vLen) - 1, 0) - address * wordsPerRegister.U + (within >> p.wordOffsetBits) - } - - def groupLegal(register: UInt, width: UInt): Bool = { - val exponent = lmul.pad(6) + width.zext - sew.zext - val registers = 1.U(6.W) << Mux(exponent > 0.S, exponent.asUInt, 0.U) - exponent >= -3.S && exponent <= 3.S && (register & (registers - 1.U)) === 0.U && register +& registers <= 32.U - } - - def overlapLegal( - destination: UInt, - destinationWidth: UInt, - operand: UInt, - operandWidth: UInt - ): Bool = { - val destinationExponent = lmul.pad(6) + destinationWidth.zext - sew.zext - val operandExponent = lmul.pad(6) + operandWidth.zext - sew.zext - val destinationCount = 1 - .U(6.W) << Mux(destinationExponent > 0.S, destinationExponent.asUInt, 0.U) - val operandCount = - 1.U(6.W) << Mux(operandExponent > 0.S, operandExponent.asUInt, 0.U) - val overlap = - destination < operand +& operandCount && operand < destination +& destinationCount - !overlap || destinationWidth === operandWidth || - (destinationWidth < operandWidth && destination === operand) || - (destinationWidth > operandWidth && operandExponent >= 0.S && destination +& destinationCount === operand +& operandCount) - } - - def maximum(width: UInt, multiplier: UInt): UInt = { - val elements = p.vLen.U(32.W) >> (width.pad(3) + 3.U) - Mux(multiplier(2), elements >> (8.U - multiplier), elements << multiplier) - } - - val vlmax = maximum(sew, vtype(2, 0)) - val registers: Instance[Operands] = Instantiate(new Operands(b)) - registers.io.initialize := io.initialize - val writes = Wire(chiselTypeOf(registers.io.write.bits)) - val pendingWrites = Reg(chiselTypeOf(registers.io.write.bits)) - for (lane <- 0 until p.laneNumber) { - writes(lane) := 0.U.asTypeOf(writes(lane)) - } - registers.io.write.valid := state === commit && !writeSent && !io.initialize - registers.io.write.bits := pendingWrites - registers.io.writeDone.ready := state === commit && writeSent && !io.initialize - when(registers.io.write.fire)(writeSent := true.B) - when(registers.io.writeDone.fire) { - writeSent := false.B - state := resumeState - } - - def writeElement( - lane: Int, - index: UInt, - width: UInt, - data: UInt, - maskBit: Bool = false.B - ): Unit = { - val bitOffset = Mux(maskBit, index, index << (width.pad(3) + 3.U)) - val within = bitOffset.pad(log2Ceil(p.vLen))(log2Ceil(p.vLen) - 1, 0) - val mask = Mux( - maskBit, - 1.U(p.wordBits.W), - MuxLookup(width, Fill(p.wordBits, 1.U(1.W)))( - Seq(0.U -> 255.U, 1.U -> 65535.U, 2.U -> "hffffffff".U) - ) - ) - writes(lane).valid := true.B - writes(lane).address := wordAddress(rd, index, width, maskBit) - writes(lane).mask := (mask << within(p.wordOffsetBits - 1, 0))(p.wordBits - 1, 0) - writes(lane).data := (data << within(p.wordOffsetBits - 1, 0))(p.wordBits - 1, 0) - } - - def finish( - fault: Bool = false.B, - cause: UInt = 0.U, - value: UInt = 0.U - ): Unit = { - finishRequested := true.B - state := idle - resultValid := true.B - result.fault := fault - result.cause := cause - result.tval := value - when(!fault)(vstart := 0.U) - } - - def nextBatch(step: UInt): Unit = { - when(base + step >= limit)(finish()).otherwise { - base := base + step - state := capture - readPhase := 0.U - } - offered.foreach(_ := false.B) - issued.foreach(_ := false.B) - received.foreach(_ := false.B) - memoryFailed.foreach(_ := false.B) - divBatchValid := false.B - divStarted.foreach(_ := false.B) - divDone.foreach(_ := false.B) - fpStarted.foreach(_ := false.B) - fpDone.foreach(_ := false.B) - } - - io.vl := vl - io.vtype := vtype - io.vstart := vstart - io.busy := state =/= idle || resultValid || registers.io.busy - io.issue.ready := state === idle && !resultValid && !registers.io.busy && !io.initialize - io.result.valid := resultValid && state === idle && !registers.io.busy - io.result.bits := result - when(io.result.fire)(resultValid := false.B) - when(io.vstartWrite.valid)(vstart := io.vstartWrite.bits) - when(io.issue.fire) { - instruction := io.issue.bits.instruction - scalar1 := io.issue.bits.scalar1 - scalar2 := io.issue.bits.scalar2 - floatScalar := io.issue.bits.floating - state := snapshot - readPhase := 0.U - readSent := false.B - snapshotSent := false.B - firstBatch := true.B - writeSent := false.B - result := 0.U.asTypeOf(new VectorResult) - base := vstart - count := 0.U - offered.foreach(_ := false.B) - issued.foreach(_ := false.B) - received.foreach(_ := false.B) - memoryFailed.foreach(_ := false.B) - divBatchValid := false.B - divStarted.foreach(_ := false.B) - divDone.foreach(_ := false.B) - fpStarted.foreach(_ := false.B) - fpDone.foreach(_ := false.B) - } - - val instructionLegal = Wire(Bool()) - val alu: Seq[Instance[IALU]] = Seq.fill(p.laneNumber)(Instantiate(new IALU(p.eLen))) - val falu: Seq[Instance[FALU]] = Seq.fill(p.laneNumber)(Instantiate(new FALU(p.eLen))) - val laneA = Wire(Vec(p.laneNumber, UInt(p.wordBits.W))) - val laneB = Wire(Vec(p.laneNumber, UInt(p.wordBits.W))) - val laneSelected = Wire(Vec(p.laneNumber, Bool())) - val laneValue = Wire(Vec(p.laneNumber, UInt(p.wordBits.W))) - val laneFlags = Wire(Vec(p.laneNumber, UInt(5.W))) - val laneLegal = Wire(Vec(p.laneNumber, Bool())) - val fpComplete = Wire(Vec(p.laneNumber, Bool())) - val gather = - integer && !multiply && (op === 12.U || op === 14.U && kind === 0.U) - val slide = - integer && !multiply && (op === 14.U || op === 15.U) && kind =/= 0.U - val slideOne = integer && multiply && (op === 14.U || op === 15.U) - - val operandReads = (0 until readPortCount).map { port => - val index = base + port.U - val scalar = Mux(kind === 3.U, Cat(Fill(59, rs1(4)), rs1), scalar1) - val floatingScalar = Mux(sew === 2.U && !floatScalar(63, 32).andR, "h7fc00000".U, floatScalar) - val b = - Mux( - vectorOperand, - element(operandWords(port), index, Mux(gather && op === 14.U, 1.U, sew)), - Mux(fp, floatingScalar, scalar) - ) - val slideOffset = Mux(slideOne, 1.U, Mux(kind === 3.U, rs1, scalar1)) - val sourceIndex = Mux(gather && !slide, b, Mux(op === 14.U, index - slideOffset, index + slideOffset)) - val permuteRead = gather || slide || slideOne - // element already keeps only the low group bit offset; upper index bits cannot affect its read. - val a = element( - sourceWords(port), - Mux(scalarMoveOut, 0.U, Mux(permuteRead, sourceIndex(31, 0), index)), - Mux(permuteRead, sew, sourceSew) - ) - val c = element(destinationWords(port), index, Mux(fp, sew, destinationSew)) - (a, b, c, sourceIndex) - } - - val permuteRead = gather || slide || slideOne - registers.io.snapshot.valid := state === snapshot && permuteRead && !snapshotSent && !io.initialize - registers.io.snapshot.bits := rs2 - registers.io.snapshotDone.ready := state === snapshot && snapshotSent && !io.initialize - when(registers.io.snapshot.fire)(snapshotSent := true.B) - when(state === snapshot && !permuteRead || registers.io.snapshotDone.fire) { - state := Mux(configure, execute, capture) - readPhase := 0.U - readSent := false.B - } - - registers.io.read.valid := state === capture && !readSent && !io.initialize - registers.io.readResult.ready := state === capture && readSent && !io.initialize - for (port <- 0 until readPortCount) { - val index = base + port.U - val maskSource = maskLogical || population || first - val sourceIndex = Mux(scalarMoveOut, 0.U, Mux(permuteRead, operandReads(port)._4(31, 0), index)) - val sourceWidth = Mux(scalarMoveOut || permuteRead, sew, sourceSew) - val shift = sourceIndex << (sourceWidth.pad(3) + 3.U) - registers.io.read.bits(port).address := MuxLookup(readPhase, wordAddress(rs1, 0.U, sew))( - Seq( - 0.U -> wordAddress(0.U, index, sew, true.B), - 1.U -> wordAddress(rs1, index, Mux(gather && op === 14.U, 1.U, sew), maskLogical), - 2.U -> Mux(permuteRead, shift >> p.wordOffsetBits, wordAddress(rs2, sourceIndex, sourceWidth, maskSource)), - 3.U -> wordAddress(rd, index, Mux(fp, sew, destinationSew)) - ) - ) - registers.io.read.bits(port).snapshot := readPhase === 2.U && permuteRead - } - when(registers.io.read.fire)(readSent := true.B) - when(registers.io.readResult.fire) { - readSent := false.B - switch(readPhase) { - is(0.U)(maskWords := registers.io.readResult.bits) - is(1.U)(operandWords := registers.io.readResult.bits) - is(2.U)(sourceWords := registers.io.readResult.bits) - is(3.U)(destinationWords := registers.io.readResult.bits) - is(4.U) { - accumulator := registers.io.readResult.bits(0) & MuxLookup(sew, Fill(p.eLen, 1.U(1.W)))( - Seq(0.U -> 255.U, 1.U -> 65535.U, 2.U -> "hffffffff".U) - ) - firstBatch := false.B - } - } - when(readPhase === 4.U || readPhase === 3.U && !firstBatch) { - state := execute - }.otherwise(readPhase := readPhase + 1.U) - } - - for (lane <- 0 until p.laneNumber) { - val index = base + lane.U - val selected = maskElement(maskWords(lane), index) - val scalar = Mux( - kind === 3.U, - Cat(Fill(59, rs1(4)), rs1), - scalar1 - ) - val floatingScalar = - Mux(sew === 2.U && !floatScalar(63, 32).andR, "h7fc00000".U, floatScalar) - val b = operandReads(lane)._2 - val slideOffset = Mux(slideOne, 1.U, Mux(kind === 3.U, rs1, scalar1)) - val up = op === 14.U - val sourceIndex = operandReads(lane)._4 - val a = operandReads(lane)._1 - laneA(lane) := a - laneB(lane) := b - laneSelected( - lane - ) := index < limit && (wholeMove || vm || selected || merge || carryOperation) && - (!slide || !up || index >= slideOffset) - alu(lane).io.divRequest.valid := state === execute && instructionLegal && divide && - divBatchValid && divActive(lane) && !divStarted(lane) && !io.initialize - alu(lane).io.divRequest.bits.a := a - alu(lane).io.divRequest.bits.b := b - alu(lane).io.divRequest.bits.sew := sew - alu(lane).io.divRequest.bits.signed := op(0) - alu(lane).io.divRequest.bits.remainder := op(1) - alu(lane).io.divResponse.ready := state === execute && divide && divBatchValid && - divStarted(lane) && !divDone(lane) && !io.initialize - alu(lane).io.clear := io.initialize - when(alu(lane).io.divRequest.fire)(divStarted(lane) := true.B) - when(alu(lane).io.divResponse.fire) { - divData(lane) := alu(lane).io.divResponse.bits - divDone(lane) := true.B - } - alu(lane).io.a := a - alu(lane).io.b := b - alu(lane).io.c := operandReads(lane)._3 - alu(lane).io.carry := (selected && (!carryMask || !vm)) || merge && vm - alu(lane).io.vxrm := io.vxrm - alu(lane).io.selector := rs1 - alu(lane).io.sew := sew - alu(lane).io.funct6 := op - alu(lane).io.multiplyClass := multiply - falu(lane).io.a := a - falu(lane).io.b := Mux(fpReduction, accumulator, b) - falu(lane).io.c := operandReads(lane)._3 - falu(lane).io.sew := sew - falu(lane).io.op := Mux( - fpReduction, - Mux(op === 5.U, 4.U, Mux(op === 7.U, 6.U, 0.U)), - op - ) - falu(lane).io.subop := rs1 - falu(lane).io.roundingMode := io.roundingMode - val fpRequired = - laneSelected(lane) && !merge && (!fpReduction || (lane == 0).B) - falu( - lane - ).io.start := state === execute && instructionLegal && fp && fpRequired && !fpStarted( - lane - ) && falu( - lane - ).io.ready - fpComplete(lane) := !fpRequired || fpDone(lane) || falu(lane).io.valid - laneFlags(lane) := Mux( - merge, - 0.U, - Mux(fpDone(lane), fpFlags(lane), falu(lane).io.flags) - ) - laneLegal(lane) := merge || Mux( - fpDone(lane), - fpLegal(lane), - falu(lane).io.legal - ) - val floatingValue = Mux(fpDone(lane), fpData(lane), falu(lane).io.result) - val gathered = Mux(sourceIndex < vlmax, a, 0.U) - val slid = Mux( - slideOne && Mux(up, index === 0.U, index === vl - 1.U), - scalar, - Mux(!up && sourceIndex >= vlmax, 0.U, a) - ) - val maskA = maskElement(sourceWords(lane), index) - val maskB = maskElement(operandWords(lane), index) - val maskResult = MuxLookup(op, false.B)( - Seq( - 24.U -> (maskA && !maskB), - 25.U -> (maskA && maskB), - 26.U -> (maskA || maskB), - 27.U -> (maskA ^ maskB), - 28.U -> (maskA || !maskB), - 29.U -> !(maskA && maskB), - 30.U -> !(maskA || maskB), - 31.U -> !(maskA ^ maskB) - ) - ) - laneValue(lane) := Mux( - fp, - Mux(merge, Mux(vm || selected, b, a), floatingValue), - Mux( - wholeMove, - a, - Mux( - maskLogical, - maskResult, - Mux( - identity, - index, - Mux( - slide || slideOne, - slid, - Mux(gather, gathered, alu(lane).io.result) - ) - ) - ) - ) - ) - when(falu(lane).io.start)(fpStarted(lane) := true.B) - when(state === execute && falu(lane).io.valid) { - fpDone(lane) := true.B - fpData(lane) := falu(lane).io.result - fpFlags(lane) := falu(lane).io.flags - fpLegal(lane) := falu(lane).io.legal - } - } - - val destinationGroupLegal = - compare || maskLogical || population || first || reduction || scalarMoveOut || scalarMoveIn || - groupLegal(rd, destinationSew) - - val sourceGroupLegal = - maskLogical || population || first || scalarMoveIn || scalarMoveOut || groupLegal( - rs2, - sourceSew - ) - - val operandGroupLegal = - !vectorOperand || op === 18.U || reduction || scalarMoveOut || maskLogical || population || first || - groupLegal(rs1, sew) - - val aluLegal = alu - .map(_.io.legal) - .reduce(_ && _) || maskLogical || identity || gather || slide || slideOne || - scalarMoveOut || scalarMoveIn || population || first || intReduction || wholeMove - - val fpOperationLegal = Seq(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16, 18, 19, 23, 24, 25, 27, 28, 29, 31, 32, 33, 36, 39, - 40, 41, 42, 43, 44, 45, 46, 47) - .map(n => op === n.U) - .reduce(_ || _) - - val ordinaryLegal = - !vtype( - 63 - ) && sourceSew <= p.maxSew.U && destinationSew <= p.maxSew.U && destinationGroupLegal && sourceGroupLegal && operandGroupLegal && - (!(intWide || floatWide || intNarrow || floatNarrow || extension) || - overlapLegal(rd, destinationSew, rs2, sourceSew) && - (!vectorOperand || op === 18.U || overlapLegal( - rd, - destinationSew, - rs1, - sew - ))) && - (vm || rd =/= 0.U || compare || maskLogical || population || first || reduction) && - (!merge || !vm || rs2 === 0.U) && (!maskLogical || vm) && - (!carryOperation || carryMask || !vm) && - (!(reduction || population || first) || vstart === 0.U) && - (!fp || sew >= 2.U && fpOperationLegal && io.roundingMode <= 4.U && - (scalarMoveOut || scalarMoveIn || merge || falu(0).io.legal) && - (op =/= 16.U || scalarMoveOut || scalarMoveIn) && - (op =/= 18.U && op =/= 19.U || kind === 1.U) && (!merge || kind === 5.U)) && - (!integer || aluLegal) - - val wholeMoveLegal = vm && vstart === 0.U && Seq(0, 1, 3, 7) - .map(n => rs1 === n.U) - .reduce(_ || _) && - (rd & rs1) === 0.U && (rs2 & rs1) === 0.U && rd +& rs1 < 32.U && rs2 +& rs1 < 32.U - - val memoryLegal = memWidth <= p.maxSew.U && memSew <= p.maxSew.U && Seq( - 0, - 5, - 6, - 7 - ).map(n => kind === n.U).reduce(_ || _) && !instruction(28) && - Mux( - wholeMemory, - vm && Seq(0, 1, 3, 7) - .map(n => instruction(31, 29) === n.U) - .reduce(_ || _) && - (rd & instruction(31, 29)) === 0.U && rd +& instruction( - 31, - 29 - ) < 32.U && (!store || memWidth === 0.U), - !vtype(63) && instruction( - 31, - 29 - ) === 0.U && (memMode =/= 0.U || rs2 === 0.U) && - groupLegal(rd, memSew) && (!indexed || groupLegal( - rs2, - memWidth - )) && (vm || store || rd =/= 0.U) - ) - - val legal = Mux( - memory, - memoryLegal, - Mux(wholeMove, wholeMoveLegal, (integer || fp) && ordinaryLegal) - ) - - instructionLegal := legal - - val required = Wire(Vec(p.memoryPorts, Bool())) - val addresses = Wire(Vec(p.memoryPorts, UInt(64.W))) - val canceled = Wire(Vec(p.memoryPorts, Bool())) - for (port <- 0 until p.memoryPorts) { - val index = base + port.U - required( - port - ) := (port < p.laneNumber).B && index < limit && (wholeMemory || vm || maskElement( - maskWords(port), - index - )) - addresses(port) := scalar1 + Mux( - indexed && !wholeMemory, - operandReads(port)._1, - Mux(memMode === 2.U, index * scalar2, index << memSew) - ) - val earlierFailure = (0 until port) - .map { prior => - required(prior) && (memoryFailed(prior) || io - .memoryResponse(prior) - .fire && io.memoryResponse(prior).bits.error) - } - .foldLeft(false.B)(_ || _) - val earlierPending = (0 until port) - .map { prior => - val distance = addresses(port) - addresses(prior) - val reverseDistance = addresses(prior) - addresses(port) - val collision = - store && (distance < (1.U << memSew) || reverseDistance < (1.U << memSew)) - required(prior) && (memMode === 3.U || collision) && !received(prior) - } - .foldLeft(false.B)(_ || _) - canceled(port) := !issued(port) && !offered(port) && earlierFailure - io.memoryRequest(port) - .valid := state === execute && memory && legal && required( - port - ) && !issued( - port - ) && !earlierPending && !canceled(port) - io.memoryRequest(port).bits.address := addresses(port) - io.memoryRequest(port).bits.write := store - io.memoryRequest(port).bits.data := operandReads(port)._3 - io.memoryRequest(port).bits.mask := MuxLookup(memSew, 255.U(8.W))( - Seq(0.U -> 1.U, 1.U -> 3.U, 2.U -> 15.U) - ) - io.memoryRequest(port).bits.size := memSew - io.memoryResponse(port).ready := state === execute && memory && issued( - port - ) && !received(port) - when(io.memoryRequest(port).valid)(offered(port) := true.B) - when(io.memoryRequest(port).fire)(issued(port) := true.B) - when(io.memoryResponse(port).fire) { - received(port) := true.B - memoryData(port) := io.memoryResponse(port).bits.data - memoryFailed(port) := io.memoryResponse(port).bits.error - } - } - - val memoryComplete = (0 until p.memoryPorts) - .map(i => !required(i) || canceled(i) || received(i) || io.memoryResponse(i).fire) - .reduce(_ && _) - - val failures = VecInit( - (0 until p.memoryPorts).map(i => - required(i) && (memoryFailed(i) || io - .memoryResponse(i) - .fire && io.memoryResponse(i).bits.error) - ) - ) - - val failureIndex = PriorityEncoder(failures.asUInt) - - when(state === execute) { - when(configure) { - val immediate = instruction(31, 30) === 3.U - val registerType = instruction(31, 25) === 64.U - val requested = Mux( - registerType, - scalar2, - Mux(immediate, instruction(29, 20), instruction(30, 20)) - ) - val requestedSew = requested(5, 3) - val requestedLmul = requested(2, 0) - val typeLegal = - requested(63, 8) === 0.U && requestedSew <= p.maxSew.U && requestedLmul =/= 4.U && - (!requestedLmul( - 2 - ) || (8.U << requestedSew) <= (p.eLen.U >> (8.U - requestedLmul))) - val newMaximum = maximum(requestedSew, requestedLmul) - val keepLength = !immediate && rs1 === 0.U && rd === 0.U - when( - !(immediate || registerType || !instruction( - 31 - )) || typeLegal && keepLength && newMaximum =/= vlmax - ) { - finish(true.B, 2.U, instruction) - }.otherwise { - val avl = Mux( - immediate, - rs1, - Mux(rs1 =/= 0.U, scalar1, Mux(rd =/= 0.U, newMaximum, vl)) - ) - val length = Mux(typeLegal, Mux(avl < newMaximum, avl, newMaximum), 0.U) - vl := length - vtype := Mux(typeLegal, requested, "h8000000000000000".U) - result.scalarWrite := rd =/= 0.U - result.scalarData := length - finish() - } - }.elsewhen(!legal) { - finish(true.B, 2.U, instruction) - }.elsewhen(scalarMoveOut) { - val value = element(sourceWords(0), 0.U, sew) - when(fp) { - result.floatWrite := true.B - result.floatData := Mux( - sew === 2.U, - Cat("hffffffff".U(32.W), value(31, 0)), - value - ) - }.otherwise { - result.scalarWrite := rd =/= 0.U - result.scalarData := MuxLookup(sew, value)( - Seq( - 0.U -> Cat(Fill(56, value(7)), value(7, 0)), - 1.U -> Cat(Fill(48, value(15)), value(15, 0)), - 2.U -> Cat(Fill(32, value(31)), value(31, 0)) - ) - ) - } - finish() - }.elsewhen(scalarMoveIn) { - when(vl =/= 0.U && vstart < vl) { - val scalar = Mux( - sew === 2.U && !floatScalar(63, 32).andR, - "h7fc00000".U, - floatScalar - ) - writeElement( - 0, - 0.U, - sew, - Mux(fp, scalar, scalar1) - ) - } - finish() - }.elsewhen(base >= limit) { - when(population || first) { - result.scalarWrite := rd =/= 0.U - result.scalarData := Mux(first, "hffffffffffffffff".U, count) - } - finish() - }.elsewhen(memory) { - when(memoryComplete) { - for (lane <- 0 until p.laneNumber) { - when( - load && required( - lane - ) && (!failures.asUInt.orR || lane.U < failureIndex) - ) { - writeElement( - lane, - base + lane.U, - memSew, - Mux( - io.memoryResponse(lane).fire, - io.memoryResponse(lane).bits.data, - memoryData(lane) - ) - ) - } - } - when(failures.asUInt.orR) { - vstart := base + failureIndex - finish(true.B, Mux(load, 5.U, 7.U), addresses(failureIndex)) - }.otherwise(nextBatch(p.laneNumber.U)) - } - }.elsewhen(population || first) { - val selected = VecInit((0 until p.laneNumber).map { lane => - val index = base + lane.U - index < vl && maskElement(sourceWords(lane), index) && - (vm || maskElement(maskWords(lane), index)) - }) - val total = count + PopCount(selected) - count := total - when(first && selected.asUInt.orR || base + p.laneNumber.U >= vl) { - result.scalarWrite := rd =/= 0.U - result.scalarData := Mux( - first, - Mux( - selected.asUInt.orR, - base + PriorityEncoder(selected.asUInt), - "hffffffff".U - ), - total - ) - finish() - }.otherwise { - base := base + p.laneNumber.U - readPhase := 0.U - state := capture - } - }.elsewhen(intReduction) { - // All integer reductions are associative at the destination width. Keep - // the original leaf order and ignore masked lanes instead of serializing them. - def combine(left: (Bool, UInt), right: (Bool, UInt)): (Bool, UInt) = { - val (leftValid, a) = left - val (rightValid, b) = right - val width = 8.U(7.W) << sew - val sa = (a << (p.eLen.U - width))(p.eLen - 1, 0).asSInt - val sb = (b << (p.eLen.U - width))(p.eLen - 1, 0).asSInt - val value = MuxLookup(op, a + b)(Seq( - 1.U -> (a & b), - 2.U -> (a | b), - 3.U -> (a ^ b), - 4.U -> Mux(b < a, b, a), - 5.U -> Mux(sb < sa, b, a), - 6.U -> Mux(b > a, b, a), - 7.U -> Mux(sb > sa, b, a) - )) - (leftValid || rightValid, Mux(!leftValid, b, Mux(!rightValid, a, value))) - } - def tree(values: Seq[(Bool, UInt)]): (Bool, UInt) = { - if (values.size == 1) values.head - else { - val (left, right) = values.splitAt(values.size / 2) - combine(tree(left), tree(right)) - } - } - val reduced = tree(Seq(true.B -> accumulator) ++ - (0 until p.laneNumber).map(lane => laneSelected(lane) -> laneA(lane)))._2 - accumulator := reduced - when(base + p.laneNumber.U >= vl)(writeElement(0, 0.U, sew, reduced)) - nextBatch(p.laneNumber.U) - }.elsewhen(fpReduction) { - when(fpComplete(0)) { - when(laneSelected(0) && !laneLegal(0))(finish(true.B, 2.U, instruction)) - .otherwise { - val value = Mux(laneSelected(0), laneValue(0), accumulator) - accumulator := value - when(laneSelected(0))(result.flags := result.flags | laneFlags(0)) - when(base + 1.U >= vl)(writeElement(0, 0.U, sew, value)) - nextBatch(1.U) - } - } - }.elsewhen(divide) { - when(!io.initialize) { - when(!divBatchValid) { - divActive := laneSelected - divBatchValid := true.B - }.elsewhen((0 until p.laneNumber).map(i => !divActive(i) || divDone(i)).reduce(_ && _)) { - for (lane <- 0 until p.laneNumber) { - when(divActive(lane)) { - writeElement(lane, base + lane.U, destinationSew, divData(lane)) - } - } - nextBatch(p.laneNumber.U) - } - } - }.elsewhen(!fp || fpComplete.reduce(_ && _)) { - val floatingLegal = (0 until p.laneNumber) - .map(i => !laneSelected(i) || laneLegal(i)) - .reduce(_ && _) - when(fp && !floatingLegal)(finish(true.B, 2.U, instruction)).otherwise { - for (lane <- 0 until p.laneNumber) { - when(laneSelected(lane)) { - writeElement( - lane, - base + lane.U, - destinationSew, - laneValue(lane), - compare || maskLogical - ) - } - } - when(fp) { - result.flags := result.flags | (0 until p.laneNumber) - .map(i => Mux(laneSelected(i), laneFlags(i), 0.U)) - .reduce(_ | _) - } - when(integer && !wholeMove) { - result.saturated := result.saturated || (0 until p.laneNumber) - .map(i => laneSelected(i) && alu(i).io.saturated) - .reduce(_ || _) - } - nextBatch(p.laneNumber.U) - } - } - } - when(state === execute && writes.map(_.valid).reduce(_ || _) && !io.initialize) { - pendingWrites := writes - resumeState := Mux(finishRequested, idle, capture) - state := commit - writeSent := false.B - } - when(io.initialize) { - state := idle - readSent := false.B - snapshotSent := false.B - writeSent := false.B - firstBatch := true.B - readPhase := 0.U - resultValid := false.B - vl := 0.U - vtype := "h8000000000000000".U - vstart := 0.U - divBatchValid := false.B - divStarted.foreach(_ := false.B) - divDone.foreach(_ := false.B) - fpStarted.foreach(_ := false.B) - fpDone.foreach(_ := false.B) - offered.foreach(_ := false.B) - issued.foreach(_ := false.B) - received.foreach(_ := false.B) - } -} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/interfaces/KernelBlinkIO.scala b/arch/src/main/scala/framework/rvv/src/main/scala/interfaces/KernelBlinkIO.scala deleted file mode 100644 index b7d38a14c..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/interfaces/KernelBlinkIO.scala +++ /dev/null @@ -1,41 +0,0 @@ -package framework.rvv - -import chisel3._ -import chisel3.util._ -import framework.balldomain.blink.BlinkPortIO -import framework.top.GlobalConfig - -class KernelBlinkIO(b: GlobalConfig) - extends BlinkPortIO(b, b.rvv.memoryPorts, b.rvv.memoryPorts, new KernelRequest(b), new KernelResponse(b)) { - // Images contain instructions and read-only constants; mutable globals are - // rejected. Temporary state stays inside the kernel; data outputs use banks. - val ballRequest = Decoupled(new RvvBallCommand) - val ballResponse = Flipped(Decoupled(UInt(64.W))) - val image = Flipped(Decoupled(UInt(32.W))) - val imageTerminal = Flipped(Decoupled(new ImageTerminal)) - // Sample diagnostics with cmdResp; no separate completion handshake. - val result = Output(new KernelCompletion) -} - -class KernelCommand extends Bundle { - val funct7 = UInt(7.W) - val rs1 = UInt(64.W) - val rs2 = UInt(64.W) -} - -class KernelRequest(b: GlobalConfig) extends Bundle { - val cmd = new KernelCommand - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val read_groups = UInt(b.memDomain.groupCountWidth.W) - val write_groups = UInt(b.memDomain.groupCountWidth.W) -} - -class KernelResponse(b: GlobalConfig) extends Bundle { - val rob_id = UInt(log2Up(b.frontend.rob_entries).W) - val write_bank = UInt(16.W) -} - -class ImageTerminal extends Bundle { - val error = UInt(4.W) - val address = UInt(64.W) -} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/interfaces/RvvInterface.scala b/arch/src/main/scala/framework/rvv/src/main/scala/interfaces/RvvInterface.scala deleted file mode 100644 index 6411b07ef..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/interfaces/RvvInterface.scala +++ /dev/null @@ -1,88 +0,0 @@ -package framework.rvv - -import chisel3._ - -class ProgramWrite extends Bundle { - val buffer = Bool() - val first = Bool() - val address = UInt(32.W) - val data = UInt(32.W) -} - -class KernelLaunch extends Bundle { - val iBuffer = Bool() - val entry = UInt(32.W) - val end = UInt(32.W) - val stack = UInt(64.W) - val args = Vec(8, UInt(64.W)) -} - -class KernelCompletion extends Bundle { - val fault = Bool() - val pc = UInt(32.W) - val instruction = UInt(32.W) - val cycles = UInt(64.W) - val cause = UInt(32.W) - val tval = UInt(64.W) - val fflags = UInt(5.W) - val vxsat = Bool() -} - -class VectorMemoryRequest extends Bundle { - val address = UInt(64.W) - val write = Bool() - val data = UInt(64.W) - val mask = UInt(8.W) - val size = UInt(2.W) -} - -class VectorMemoryResponse extends Bundle { - val data = UInt(64.W) - val error = Bool() -} - -class VectorIssue extends Bundle { - val instruction = UInt(32.W) - val scalar1 = UInt(64.W) - val scalar2 = UInt(64.W) - val floating = UInt(64.W) -} - -class VectorResult extends Bundle { - val fault = Bool() - val cause = UInt(32.W) - val tval = UInt(64.W) - val scalarWrite = Bool() - val scalarData = UInt(64.W) - val floatWrite = Bool() - val floatData = UInt(64.W) - val flags = UInt(5.W) - val saturated = Bool() -} - -class ImageLoad extends Bundle { - val buffer = Bool() - val bytes = UInt(32.W) -} - -class ImageResult extends Bundle { - val fault = Bool() - val cause = UInt(32.W) - val tval = UInt(32.W) - val entry = UInt(32.W) - val textBytes = UInt(32.W) - val constBytes = UInt(32.W) -} - -class RvvBallCommand extends Bundle { - val funct7 = UInt(7.W) - val rs1 = UInt(64.W) - val rs2 = UInt(64.W) -} - -class BankLayout(b: framework.top.GlobalConfig) extends Bundle { - val readBank = UInt(b.memDomain.vbankIdWidth.W) - val writeBank = UInt(b.memDomain.vbankIdWidth.W) - val readGroups = UInt(b.memDomain.groupCountWidth.W) - val writeGroups = UInt(b.memDomain.groupCountWidth.W) -} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/memory/BankPort.scala b/arch/src/main/scala/framework/rvv/src/main/scala/memory/BankPort.scala deleted file mode 100644 index dd9f4d84e..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/memory/BankPort.scala +++ /dev/null @@ -1,109 +0,0 @@ -package framework.rvv - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.balldomain.blink.{BankRead, BankWrite} -import framework.top.GlobalConfig - -@instantiable -class BankPort(val b: GlobalConfig) extends Module { - val lineBytes = b.memDomain.bankWidth / 8 - val offsetBits = log2Ceil(lineBytes) - require(b.memDomain.bankWidth >= 64 && isPow2(b.memDomain.bankWidth)) - require(lineBytes <= 65536) - require(b.memDomain.bankMaskLen == lineBytes) - - @public - val io = IO(new Bundle { - val request = Flipped(Decoupled(new VectorMemoryRequest)) - val response = Decoupled(new VectorMemoryResponse) - val read = Flipped(new BankRead(b)) - val write = Flipped(new BankWrite(b)) - val readBank = Input(UInt(b.memDomain.vbankIdWidth.W)) - val writeBank = Input(UInt(b.memDomain.vbankIdWidth.W)) - val readGroups = Input(UInt(b.memDomain.groupCountWidth.W)) - val writeGroups = Input(UInt(b.memDomain.groupCountWidth.W)) - val robId = Input(UInt(log2Up(b.frontend.rob_entries).W)) - }) - - val idle :: readRequest :: readResponse :: writeRequest :: writeResponse :: complete :: Nil = Enum(6) - val state = RegInit(idle) - val request = Reg(new VectorMemoryRequest) - val robId = Reg(UInt(log2Up(b.frontend.rob_entries).W)) - val consumed = RegInit(0.U(4.W)) - val result = RegInit(0.U(64.W)) - val error = RegInit(false.B) - - val byteCount = (1.U(4.W) << request.size)(3, 0) - val byteAddress = request.address(31, 0) +& consumed - val lineOffset = byteAddress(offsetBits - 1, 0) - val remaining = byteCount - consumed - val lineRemaining = lineBytes.U - lineOffset - val chunk = Mux(remaining <= lineRemaining, remaining, lineRemaining)(3, 0) - val nextConsumed = consumed + chunk - val lineAddress = byteAddress >> offsetBits - val readShifted = io.read.io.resp.bits.data >> (lineOffset << 3) - val chunkMask = ((1.U(65.W) << (chunk << 3)) - 1.U)(63, 0) - val readChunk = readShifted.pad(64)(63, 0) & chunkMask - - io.request.ready := state === idle - io.response.valid := state === complete - io.response.bits.data := result - io.response.bits.error := error - val localBank = request.address(63, 32) - val fromRead = localBank < io.readGroups - val parentBank = Mux(fromRead, io.readBank, io.writeBank) - val parentGroup = Mux(fromRead, localBank, localBank - io.readGroups) - io.read.bank_id := parentBank - io.read.rob_id := robId - io.read.ball_id := 0.U - io.read.group_id := parentGroup - io.read.io.req.valid := state === readRequest - io.read.io.req.bits.addr := lineAddress - io.read.io.resp.ready := state === readResponse - io.write.bank_id := parentBank - io.write.rob_id := robId - io.write.ball_id := 0.U - io.write.group_id := parentGroup - io.write.io.req.valid := state === writeRequest - io.write.io.req.bits.addr := lineAddress - io.write.io.req.bits.data := (request.data >> (consumed << 3)).pad(b.memDomain.bankWidth) << (lineOffset << 3) - io.write.io.resp.ready := state === writeResponse - for (i <- 0 until lineBytes) { - val sourceByte = consumed + i.U - lineOffset - io.write.io.req.bits.mask(i) := i.U >= lineOffset && i.U < lineOffset +& chunk && request.mask(sourceByte(2, 0)) - } - - when(io.request.fire) { - request := io.request.bits - robId := io.robId - consumed := 0.U - result := 0.U - val end = io.request.bits.address(31, 0) +& (1.U(33.W) << io.request.bits.size) - val invalid = io.request.bits.address(63, 32) >= (io.readGroups +& io.writeGroups) || - (io.request.bits.write && io.request.bits.address(63, 32) < io.readGroups) || - end > (BigInt(b.memDomain.bankEntries) * lineBytes).U - error := invalid - state := Mux(invalid, complete, Mux(io.request.bits.write, writeRequest, readRequest)) - } - when(io.read.io.req.fire) { - state := readResponse - } - when(io.read.io.resp.fire) { - result := result | (readChunk << (consumed << 3)) - consumed := nextConsumed - state := Mux(nextConsumed === byteCount, complete, readRequest) - } - when(io.write.io.req.fire) { - state := writeResponse - } - when(io.write.io.resp.fire) { - consumed := nextConsumed - error := !io.write.io.resp.bits.ok - state := Mux(!io.write.io.resp.bits.ok || nextConsumed === byteCount, complete, writeRequest) - } - when(io.response.fire) { - state := idle - } -} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/top/Execution.scala b/arch/src/main/scala/framework/rvv/src/main/scala/top/Execution.scala deleted file mode 100644 index 0a35e7296..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/top/Execution.scala +++ /dev/null @@ -1,354 +0,0 @@ -package framework.rvv - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig - -@instantiable -class Execution(val b: GlobalConfig) extends Module { - val p = b.rvv - - @public - val io = IO(new Bundle { - val program = Flipped(Decoupled(new ProgramWrite)) - val launch = Flipped(Decoupled(new KernelLaunch)) - val done = Decoupled(new KernelCompletion) - val busy = Output(Bool()) - val ballRequest = Decoupled(new RvvBallCommand) - val ballResponse = Flipped(Decoupled(UInt(64.W))) - val memoryRequest = Vec(p.memoryPorts, Decoupled(new VectorMemoryRequest)) - val memoryResponse = Vec(p.memoryPorts, Flipped(Decoupled(new VectorMemoryResponse))) - }) - - val iBuf: Seq[Instance[IBuf]] = Seq.fill(2)(Instantiate(new IBuf(b))) - val registers: Instance[ScalarRF] = Instantiate(new ScalarRF) - val scalar: Instance[ScalarExecution] = Instantiate(new ScalarExecution) - val floating: Instance[ScalarFloating] = Instantiate(new ScalarFloating(p.eLen)) - val vector: Instance[VectorCore] = Instantiate(new VectorCore(b)) - - val idle :: fetch :: fetchWait :: execute :: vectorWait :: scalarWait :: floatWait :: memorySend :: memoryWait :: ballSend :: ballWait :: completed :: Nil = - Enum(12) - - val state = RegInit(idle) - val activeBuffer = RegInit(false.B) - val end = Reg(UInt(32.W)) - val pc = Reg(UInt(32.W)) - val instruction = RegInit(0.U(32.W)) - val cycles = RegInit(0.U(64.W)) - val fault = RegInit(false.B) - val cause = RegInit(0.U(32.W)) - val tval = RegInit(0.U(64.W)) - val fflags = RegInit(0.U(5.W)) - val frm = RegInit(0.U(3.W)) - val vxrm = RegInit(0.U(2.W)) - val vxsat = RegInit(false.B) - val scalarTarget = Reg(UInt(5.W)) - val scalarNextPc = Reg(UInt(32.W)) - - val scalarDivide = (instruction(6, 0) === "h33".U || instruction(6, 0) === "h3b".U) && instruction( - 31, - 25 - ) === 1.U && instruction(14, 12) >= 4.U - - val memory = Reg(new VectorMemoryRequest) - val memoryFloating = Reg(Bool()) - val memoryUnsigned = Reg(Bool()) - - def finish(isFault: Bool, code: UInt, value: UInt): Unit = { - state := completed - fault := isFault - cause := code - tval := value - } - - io.ballRequest.valid := state === ballSend - io.ballRequest.bits.funct7 := instruction(31, 25) - io.ballRequest.bits.rs1 := registers.io.xReadData1 - io.ballRequest.bits.rs2 := registers.io.xReadData2 - io.ballResponse.ready := state === ballWait - when(io.ballRequest.fire)(state := ballWait) - - val running = state =/= idle && state =/= completed - io.busy := state =/= idle - io.launch.ready := state === idle - io.done.valid := state === completed - io.done.bits.fault := fault - io.done.bits.pc := pc - io.done.bits.instruction := instruction - io.done.bits.cycles := cycles - io.done.bits.cause := cause - io.done.bits.tval := tval - io.done.bits.fflags := fflags - io.done.bits.vxsat := vxsat - - for (i <- 0 until 2) { - iBuf(i).io.upload.valid := io.program.valid && io.program.bits.buffer === i.U - iBuf(i).io.upload.bits := io.program.bits - iBuf(i).io.uploadEnabled := !io.launch.valid && (!running || activeBuffer =/= i.U) - iBuf(i).io.fetchEnabled := state === fetch && activeBuffer === i.U && pc < end - iBuf(i).io.fetchAddress := pc - } - io.program.ready := Mux(io.program.bits.buffer, iBuf(1).io.upload.ready, iBuf(0).io.upload.ready) - val imageWords = Mux(io.launch.bits.iBuffer, iBuf(1).io.wordCount, iBuf(0).io.wordCount) - val fetchLoaded = Mux(activeBuffer, iBuf(1).io.loaded, iBuf(0).io.loaded) - val fetchedInstruction = Mux(activeBuffer, iBuf(1).io.instruction, iBuf(0).io.instruction) - - registers.io.initialize.valid := io.launch.fire - registers.io.initialize.bits := io.launch.bits - registers.io.xReadAddress1 := instruction(19, 15) - registers.io.xReadAddress2 := instruction(24, 20) - registers.io.fReadAddress1 := instruction(19, 15) - registers.io.fReadAddress2 := instruction(24, 20) - registers.io.fReadAddress3 := instruction(31, 27) - registers.io.xWrite.valid := false.B - registers.io.xWrite.bits.address := instruction(11, 7) - registers.io.xWrite.bits.data := 0.U - registers.io.fWrite.valid := false.B - registers.io.fWrite.bits.address := instruction(11, 7) - registers.io.fWrite.bits.data := 0.U - scalar.io.divRequest.valid := false.B - scalar.io.divRequest.bits.a := registers.io.xReadData1 - scalar.io.divRequest.bits.b := registers.io.xReadData2 - scalar.io.divRequest.bits.sew := Mux(instruction(6, 0) === "h3b".U, 2.U, 3.U) - scalar.io.divRequest.bits.signed := !instruction(12) - scalar.io.divRequest.bits.remainder := instruction(13) - scalar.io.divResponse.ready := state === scalarWait && !io.launch.fire - scalar.io.clear := io.launch.fire - scalar.io.instruction := instruction - scalar.io.pc := pc - scalar.io.source1 := registers.io.xReadData1 - scalar.io.source2 := registers.io.xReadData2 - floating.io.instruction := instruction - floating.io.source1 := registers.io.fReadData1 - floating.io.source2 := registers.io.fReadData2 - floating.io.source3 := registers.io.fReadData3 - floating.io.xSource := registers.io.xReadData1 - floating.io.frm := frm - floating.io.start := false.B - vector.io.initialize := false.B - vector.io.roundingMode := frm - vector.io.vxrm := vxrm - vector.io.vstartWrite.valid := false.B - vector.io.vstartWrite.bits := 0.U - vector.io.issue.valid := false.B - vector.io.issue.bits.instruction := instruction - vector.io.issue.bits.scalar1 := registers.io.xReadData1 - vector.io.issue.bits.scalar2 := registers.io.xReadData2 - vector.io.issue.bits.floating := registers.io.fReadData1 - vector.io.result.ready := state === vectorWait - - for (i <- 0 until p.memoryPorts) { - io.memoryRequest(i) <> vector.io.memoryRequest(i) - vector.io.memoryResponse(i) <> io.memoryResponse(i) - } - when(io.ballResponse.fire) { - registers.io.xWrite.valid := instruction(11, 7) =/= 0.U - registers.io.xWrite.bits.data := io.ballResponse.bits - pc := pc + 4.U - state := fetch - } - when(state === scalarWait && scalar.io.divResponse.fire) { - registers.io.xWrite.valid := true.B - registers.io.xWrite.bits.address := scalarTarget - registers.io.xWrite.bits.data := Mux( - instruction(6, 0) === "h3b".U, - Cat(Fill(32, scalar.io.divResponse.bits(31)), scalar.io.divResponse.bits(31, 0)), - scalar.io.divResponse.bits - ) - pc := scalarNextPc - state := fetch - } - when(state === memorySend || state === memoryWait) { - io.memoryRequest(0).valid := state === memorySend - io.memoryRequest(0).bits := memory - vector.io.memoryRequest(0).ready := false.B - vector.io.memoryResponse(0).valid := false.B - io.memoryResponse(0).ready := state === memoryWait - } - - val opcode = instruction(6, 0) - val funct3 = instruction(14, 12) - val vectorInstruction = opcode === "h57".U || - (opcode === "h07".U || opcode === "h27".U) && (funct3 === 0.U || funct3 >= 5.U) - val floatingInstruction = opcode === "h53".U || opcode === "h43".U || opcode === "h47".U || - opcode === "h4b".U || opcode === "h4f".U - val load = opcode === "h03".U || opcode === "h07".U - val store = opcode === "h23".U || opcode === "h27".U - val csrAddress = instruction(31, 20) - val csrKnown = WireDefault(true.B) - val csrValue = WireDefault(0.U(64.W)) - switch(csrAddress) { - is("h001".U)(csrValue := fflags) - is("h002".U)(csrValue := frm) - is("h003".U)(csrValue := Cat(frm, fflags)) - is("h008".U)(csrValue := vector.io.vstart) - is("h009".U)(csrValue := vxsat) - is("h00a".U)(csrValue := vxrm) - is("h00f".U)(csrValue := Cat(vxrm, vxsat)) - is("hc20".U)(csrValue := vector.io.vl) - is("hc21".U)(csrValue := vector.io.vtype) - is("hc22".U)(csrValue := (p.vLen / 8).U) - } - csrKnown := Seq(0x001, 0x002, 0x003, 0x008, 0x009, 0x00a, 0x00f, 0xc20, 0xc21, 0xc22) - .map(a => csrAddress === a.U).reduce(_ || _) - val csrSource = Mux(funct3(2), instruction(19, 15), registers.io.xReadData1) - val csrWrite = funct3(1, 0) === 1.U || instruction(19, 15) =/= 0.U - - val csrNew = MuxLookup(funct3(1, 0), csrValue)(Seq( - 1.U -> csrSource, - 2.U -> (csrValue | csrSource), - 3.U -> (csrValue & ~csrSource) - )) - - when(running)(cycles := cycles + 1.U) - when(io.done.fire)(state := idle) - when(io.launch.fire) { - activeBuffer := io.launch.bits.iBuffer - pc := io.launch.bits.entry - end := io.launch.bits.end - instruction := 0.U - cycles := 0.U - fflags := 0.U - frm := 0.U - fault := false.B - cause := 0.U - tval := 0.U - state := fetch - when(io.launch.bits.entry(1, 0).orR || io.launch.bits.end(1, 0).orR) { - finish(true.B, 0.U, io.launch.bits.entry) - }.elsewhen(io.launch.bits.entry >= io.launch.bits.end || io.launch.bits.end > imageWords * 4.U) { - finish(true.B, 1.U, io.launch.bits.entry) - } - } - when(state === fetch) { - when(pc === end)(finish(false.B, 0.U, 0.U)) - .elsewhen(pc(1, 0).orR)(finish(true.B, 0.U, pc)) - .elsewhen(pc >= end || !fetchLoaded)(finish(true.B, 1.U, pc)) - .otherwise(state := fetchWait) - } - when(state === fetchWait) { instruction := fetchedInstruction; state := execute } - when(state === execute) { - when(opcode === "h7b".U && funct3 === 3.U) { - state := ballSend - }.elsewhen(vectorInstruction) { - vector.io.issue.valid := true.B - when(vector.io.issue.fire)(state := vectorWait) - }.elsewhen(floatingInstruction) { - when(!floating.io.legal)(finish(true.B, 2.U, instruction)) - .otherwise { - floating.io.start := floating.io.ready - when(floating.io.ready)(state := floatWait) - } - }.elsewhen(load || store) { - val fp = opcode === "h07".U || opcode === "h27".U - val size = Mux(fp, Mux(funct3 === 3.U, 3.U, 2.U), funct3(1, 0)) - val legal = Mux( - fp, - funct3 === 2.U || (p.eLen == 64).B && funct3 === 3.U, - Mux(store, funct3 <= 3.U, funct3 <= 6.U) - ) - val immediate = Mux( - store, - Cat(Fill(52, instruction(31)), instruction(31, 25), instruction(11, 7)), - Cat(Fill(52, instruction(31)), instruction(31, 20)) - ) - val address = registers.io.xReadData1 + immediate - memory.address := address - memory.write := store - memory.data := Mux(fp, registers.io.fReadData2, registers.io.xReadData2) - memory.size := size - memory.mask := MuxLookup(size, 1.U(8.W))(Seq(0.U -> 1.U, 1.U -> 3.U, 2.U -> 15.U, 3.U -> 255.U)) - memoryFloating := fp - memoryUnsigned := funct3(2) - when(!legal)(finish(true.B, 2.U, instruction)) - .otherwise(state := memorySend) - }.elsewhen(opcode === "h73".U) { - when(funct3 === 0.U) { - when(instruction === "h00100073".U)(finish(true.B, 3.U, pc)) - .elsewhen(instruction === "h00000073".U)(finish(true.B, 11.U, 0.U)) - .otherwise(finish(true.B, 2.U, instruction)) - }.elsewhen(funct3 === 4.U || !csrKnown || csrWrite && csrAddress(11, 10) === 3.U) { - finish(true.B, 2.U, instruction) - }.otherwise { - registers.io.xWrite.valid := true.B - registers.io.xWrite.bits.data := csrValue - when(csrWrite) { - switch(csrAddress) { - is("h001".U)(fflags := csrNew(4, 0)) - is("h002".U)(frm := csrNew(2, 0)) - is("h003".U) { frm := csrNew(7, 5); fflags := csrNew(4, 0) } - is("h008".U) { vector.io.vstartWrite.valid := true.B; vector.io.vstartWrite.bits := csrNew } - is("h009".U)(vxsat := csrNew(0)) - is("h00a".U)(vxrm := csrNew(1, 0)) - is("h00f".U) { vxrm := csrNew(2, 1); vxsat := csrNew(0) } - } - } - pc := pc + 4.U - state := fetch - } - }.elsewhen(!scalar.io.legal)(finish(true.B, 2.U, instruction)) - .elsewhen(scalar.io.nextPc(63, 32).orR)(finish(true.B, 1.U, scalar.io.nextPc)) - .elsewhen(scalar.io.nextPc(1, 0).orR)(finish(true.B, 0.U, scalar.io.nextPc)) - .elsewhen(scalarDivide) { - scalar.io.divRequest.valid := true.B - when(scalar.io.divRequest.fire) { - scalarTarget := scalar.io.destination - scalarNextPc := scalar.io.nextPc - state := scalarWait - } - }.otherwise { - registers.io.xWrite.valid := scalar.io.write - registers.io.xWrite.bits.data := scalar.io.result - pc := scalar.io.nextPc - state := fetch - } - } - when(state === memorySend && io.memoryRequest(0).fire)(state := memoryWait) - when(state === memoryWait && io.memoryResponse(0).fire) { - when(io.memoryResponse(0).bits.error)(finish(true.B, Mux(memory.write, 7.U, 5.U), memory.address)) - .otherwise { - when(!memory.write) { - val data = io.memoryResponse(0).bits.data - when(memoryFloating) { - registers.io.fWrite.valid := true.B - registers.io.fWrite.bits.data := Mux(memory.size === 2.U, Cat("hffffffff".U(32.W), data(31, 0)), data) - }.otherwise { - registers.io.xWrite.valid := true.B - registers.io.xWrite.bits.data := MuxLookup(memory.size, data)(Seq( - 0.U -> Cat(Fill(56, !memoryUnsigned && data(7)), data(7, 0)), - 1.U -> Cat(Fill(48, !memoryUnsigned && data(15)), data(15, 0)), - 2.U -> Cat(Fill(32, !memoryUnsigned && data(31)), data(31, 0)) - )) - } - } - pc := pc + 4.U - state := fetch - } - } - when((state === floatWait || state === execute && floatingInstruction) && floating.io.valid) { - when(!floating.io.legal)(finish(true.B, 2.U, instruction)) - .otherwise { - registers.io.fWrite.valid := floating.io.floatWrite - registers.io.fWrite.bits.data := floating.io.result - registers.io.xWrite.valid := floating.io.integerWrite - registers.io.xWrite.bits.data := floating.io.result - fflags := fflags | floating.io.flags - pc := pc + 4.U - state := fetch - } - } - when(vector.io.result.fire) { - when(vector.io.result.bits.fault)(finish(true.B, vector.io.result.bits.cause, vector.io.result.bits.tval)) - .otherwise { - registers.io.xWrite.valid := vector.io.result.bits.scalarWrite - registers.io.xWrite.bits.data := vector.io.result.bits.scalarData - registers.io.fWrite.valid := vector.io.result.bits.floatWrite - registers.io.fWrite.bits.data := vector.io.result.bits.floatData - fflags := fflags | vector.io.result.bits.flags - vxsat := vxsat || vector.io.result.bits.saturated - pc := pc + 4.U - state := fetch - } - } -} diff --git a/arch/src/main/scala/framework/rvv/src/main/scala/top/KernelEngine.scala b/arch/src/main/scala/framework/rvv/src/main/scala/top/KernelEngine.scala deleted file mode 100644 index 05f361cdf..000000000 --- a/arch/src/main/scala/framework/rvv/src/main/scala/top/KernelEngine.scala +++ /dev/null @@ -1,241 +0,0 @@ -package framework.rvv - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BallStatus, HasBallStatus} -import framework.top.GlobalConfig - -@instantiable -class KernelEngine(val b: GlobalConfig) extends Module with HasBallStatus { - val p = b.rvv - require(p.enable, "RVV must be explicitly enabled") - - @public - val io = IO(new KernelBlinkIO(b)) - def status: BallStatus = io.status - - val execution: Instance[Execution] = Instantiate(new Execution(b)) - val loader: Instance[ImageLoader] = Instantiate(new ImageLoader(b)) - val local: Instance[KernelMemory] = Instantiate(new KernelMemory(b)) - val ports: Seq[Instance[BankPort]] = Seq.fill(p.memoryPorts)(Instantiate(new BankPort(b))) - val idle :: loading :: descriptorSend :: descriptorWait :: launch :: executing :: completed :: Nil = Enum(7) - - val state = RegInit(idle) - val command = Reg(new KernelRequest(b)) - val channels = RegInit(false.B) - val validImage = RegInit(VecInit(Seq.fill(2)(false.B))) - val imageEntry = Reg(Vec(2, UInt(32.W))) - val imageBytes = Reg(Vec(2, UInt(32.W))) - val imageConstants = Reg(Vec(2, UInt(32.W))) - val buffer = Reg(Bool()) - val descriptor = Reg(Vec(12, UInt(64.W))) - val word = Reg(UInt(4.W)) - val terminalSeen = RegInit(false.B) - val terminalFailed = RegInit(false.B) - val terminalCancelled = RegInit(false.B) - val result = RegInit(0.U.asTypeOf(new KernelCompletion)) - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === completed && (command.cmd.funct7 =/= 44.U || result.fault || terminalSeen) - io.cmdResp.bits.rob_id := command.rob_id - io.cmdResp.bits.write_bank := Mux(command.cmd.funct7 === 79.U && !result.fault, command.cmd.rs1(29, 20), 0.U) - io.result := result - io.status.idle := state === idle - io.status.running := state =/= idle - when(state =/= idle && io.channelReady)(channels := true.B) - - val loadBank = io.cmdReq.bits.cmd.rs1(29, 20) - val loadBankValid = loadBank >= b.memDomain.virtualBankCount.U && loadBank < (b.memDomain.virtualBankCount + 2).U - val loadBuffer = loadBank - b.memDomain.virtualBankCount.U - loader.io.start.valid := io.cmdReq.fire && io.cmdReq.bits.cmd.funct7 === 44.U && loadBankValid - loader.io.start.bits.buffer := loadBuffer(0) - loader.io.start.bits.bytes := io.cmdReq.bits.cmd.rs1(63, 30) - loader.io.image <> io.image - io.imageTerminal.ready := state =/= idle && command.cmd.funct7 === 44.U && !terminalSeen - val terminalError = io.imageTerminal.fire && io.imageTerminal.bits.error =/= 0.U - loader.io.abort := terminalError - when(io.imageTerminal.fire) { - terminalSeen := true.B - terminalFailed := io.imageTerminal.bits.error =/= 0.U - terminalCancelled := io.imageTerminal.bits.error === 10.U - when(terminalError) { - validImage(buffer) := false.B - when(!result.fault) { - result.fault := true.B - result.cause := 5.U - result.tval := io.imageTerminal.bits.address - } - } - } - loader.io.done.ready := state === loading - io.ballRequest <> execution.io.ballRequest - execution.io.ballResponse <> io.ballResponse - execution.io.program <> loader.io.program - execution.io.launch.valid := state === launch - execution.io.launch.bits.iBuffer := buffer - execution.io.launch.bits.entry := descriptor(0)(31, 0) - execution.io.launch.bits.end := descriptor(1)(31, 0) - execution.io.launch.bits.stack := descriptor(2) - for (i <- 0 until 8) (execution.io.launch.bits.args(i) := descriptor(i + 3)) - execution.io.done.ready := state === executing - - local.io.buffer := buffer - local.io.loading := state === loading - local.io.constBytes := imageConstants(buffer) - for (i <- 0 until p.memoryPorts) { - io.bankRead(i) <> ports(i).io.read - io.bankWrite(i) <> ports(i).io.write - ports(i).io.robId := command.rob_id - ports(i).io.readBank := command.cmd.rs1(9, 0) - ports(i).io.writeBank := command.cmd.rs1(29, 20) - ports(i).io.readGroups := command.read_groups - ports(i).io.writeGroups := command.write_groups - val address = execution.io.memoryRequest(i).bits.address - val internal = address >= "h40000000".U && address < "h40002000".U - ports(i).io.request.valid := state === executing && channels && !internal && execution.io.memoryRequest(i).valid - ports(i).io.request.bits := execution.io.memoryRequest(i).bits - local.io.request(i).valid := state === executing && internal && execution.io.memoryRequest(i).valid - local.io.request(i).bits := execution.io.memoryRequest(i).bits - execution.io.memoryRequest(i).ready := state === executing && Mux( - internal, - local.io.request(i).ready, - channels && ports(i).io.request.ready - ) - execution.io.memoryResponse(i).valid := state === executing && (local.io.response(i).valid || ports( - i - ).io.response.valid) - execution.io.memoryResponse(i).bits := Mux( - local.io.response(i).valid, - local.io.response(i).bits, - ports(i).io.response.bits - ) - ports(i).io.response.ready := state === executing && execution.io.memoryResponse(i).ready - local.io.response(i).ready := state === executing && execution.io.memoryResponse(i).ready - } - loader.io.memoryRequest.ready := state === loading && local.io.request(0).ready - loader.io.memoryResponse.valid := state === loading && local.io.response(0).valid - loader.io.memoryResponse.bits := local.io.response(0).bits - when(state === loading) { - local.io.request(0).valid := loader.io.memoryRequest.valid - local.io.request(0).bits := loader.io.memoryRequest.bits - local.io.response(0).ready := loader.io.memoryResponse.ready - } - when(state === descriptorSend) { - ports(0).io.request.valid := channels - ports(0).io.request.bits.address := (command.read_groups.pad(64) << 32) + command.cmd.rs2 + word * 8.U - ports(0).io.request.bits.write := false.B - ports(0).io.request.bits.data := 0.U - ports(0).io.request.bits.mask := 255.U - ports(0).io.request.bits.size := 3.U - } - when(state === descriptorWait)(ports(0).io.response.ready := true.B) - - when(io.cmdReq.fire) { - command := io.cmdReq.bits - channels := io.channelReady - result := 0.U.asTypeOf(new KernelCompletion) - terminalSeen := false.B - terminalFailed := false.B - terminalCancelled := false.B - switch(io.cmdReq.bits.cmd.funct7) { - is(44.U) { - buffer := loadBuffer(0) - validImage(loadBuffer(0)) := false.B - state := loading - when(!loadBankValid || io.cmdReq.bits.cmd.rs1(19, 0).orR) { - result.fault := true.B - result.cause := 2.U - result.tval := io.cmdReq.bits.cmd.rs1 - state := completed - } - } - is(79.U) { - val program = io.cmdReq.bits.cmd.rs1(19, 10) - val programValid = program >= b.memDomain.virtualBankCount.U && program < (b.memDomain.virtualBankCount + 2).U - val slot = program - b.memDomain.virtualBankCount.U - buffer := slot(0) - word := 0.U - state := descriptorSend - when(!programValid || !validImage( - slot(0) - ) || io.cmdReq.bits.read_groups === 0.U || io.cmdReq.bits.write_groups === 0.U || - io.cmdReq.bits.cmd.rs1(63, 30).orR || io.cmdReq.bits.cmd.rs1(9, 0) === io.cmdReq.bits.cmd.rs1(29, 20) || - io.cmdReq.bits.cmd.rs2(63, 32).orR || (io.cmdReq.bits.cmd.rs2( - 31, - 0 - ) +& 96.U) > (b.memDomain.bankEntries * (b.memDomain.bankWidth / 8)).U) { - result.fault := true.B - result.cause := 1.U - result.tval := io.cmdReq.bits.cmd.rs1 - state := completed - } - } - is(32.U) { - val program = io.cmdReq.bits.cmd.rs1(9, 0) - val slot = program - b.memDomain.virtualBankCount.U - when(program < b.memDomain.virtualBankCount.U || program >= (b.memDomain.virtualBankCount + 2).U || - io.cmdReq.bits.cmd.rs1(63, 10).orR || io.cmdReq.bits.cmd.rs2.orR) { - result.fault := true.B - result.cause := 2.U - result.tval := io.cmdReq.bits.cmd.rs1 - }.otherwise(validImage(slot(0)) := false.B) - state := completed - } - - } - when(io.cmdReq.bits.cmd.funct7 =/= 44.U && io.cmdReq.bits.cmd.funct7 =/= 79.U && io.cmdReq.bits.cmd.funct7 =/= 32.U) { - result.fault := true.B - result.cause := 2.U - result.tval := io.cmdReq.bits.cmd.funct7 - state := completed - } - } - when(loader.io.done.fire) { - val nativeFault = loader.io.done.bits.fault && loader.io.done.bits.cause =/= 5.U - when((!terminalFailed && !terminalError) || - (nativeFault && (terminalCancelled || (io.imageTerminal.fire && io.imageTerminal.bits.error === 10.U)))) { - result.fault := loader.io.done.bits.fault - result.cause := loader.io.done.bits.cause - result.tval := loader.io.done.bits.tval - } - imageEntry(buffer) := loader.io.done.bits.entry - imageBytes(buffer) := loader.io.done.bits.textBytes - imageConstants(buffer) := loader.io.done.bits.constBytes - state := completed - } - when(state === descriptorSend && ports(0).io.request.fire)(state := descriptorWait) - when(state === descriptorWait && ports(0).io.response.fire) { - when(ports(0).io.response.bits.error) { - result.fault := true.B - result.cause := 5.U - result.tval := (command.read_groups.pad(64) << 32) + command.cmd.rs2 + word * 8.U - state := completed - }.otherwise { - descriptor(word) := ports(0).io.response.bits.data - word := word + 1.U - state := Mux(word === 11.U, launch, descriptorSend) - } - } - when(state === launch) { - when(descriptor(0) =/= imageEntry(buffer) || descriptor(1) =/= imageBytes(buffer) || descriptor( - 11 - ) =/= 0.U || descriptor(2) =/= "h40002000".U) { - execution.io.launch.valid := false.B - result.fault := true.B - result.cause := 1.U - result.tval := descriptor(0) - state := completed - }.elsewhen(execution.io.launch.fire)(state := executing) - } - when(execution.io.done.fire) { - result := execution.io.done.bits - state := completed - } - when(io.cmdResp.fire) { - when(command.cmd.funct7 === 44.U && !result.fault && terminalSeen && !terminalFailed) { - validImage(buffer) := true.B - } - state := idle - } -} diff --git a/arch/src/main/scala/framework/system/Device.scala b/arch/src/main/scala/framework/system/Device.scala deleted file mode 100644 index e78cb95e0..000000000 --- a/arch/src/main/scala/framework/system/Device.scala +++ /dev/null @@ -1,13 +0,0 @@ -package framework.system - -import chisel3.experimental.hierarchy.{instantiable, Instantiate} -import framework.top.GlobalConfig -import framework.system.GlobalConfigOps._ -import framework.system.tile.{MainTile, TileFactory} - -@instantiable -class Device(b: GlobalConfig) extends System(b.systemParams) { - val main = Instantiate(new MainTile(b.forTile(0))) - val tiles = b.systemDesign.get.tiles.indices.drop(1).map(i => TileFactory.instantiate(b.forTile(i))) - DeviceConnection.connect(b.systemParams, dlink, Seq(main.tlink) ++ tiles.map(_.tlink)) -} diff --git a/arch/src/main/scala/framework/system/DeviceConnection.scala b/arch/src/main/scala/framework/system/DeviceConnection.scala deleted file mode 100644 index a642bae32..000000000 --- a/arch/src/main/scala/framework/system/DeviceConnection.scala +++ /dev/null @@ -1,131 +0,0 @@ -package framework.system - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.dlink.DLinkIO -import framework.system.tile.tlink.{Fabric, TLinkIO} -import framework.system.memory.Memory -import framework.system.device.{BootRomLines, Devices} -import memcore.memory.cpu.CpuMemParams - -object DeviceConnection { - - def connect(p: SystemParams, dlink: DLinkIO, tilePorts: Seq[TLinkIO]): Unit = { - val topology = p.topology - val memory = p.memory - val ddr = p.ddr - val devices = p.devices - val tiles = topology.tiles - val hartIds = tiles.flatMap(_.hartIds) - require(ddr.clients == 1 && ddr.line == memory.chi) - val n = hartIds.size - - val dmaMasters = tiles.size - - val cp = CpuMemParams(memory.chi, tagBits = 6) - - val controlCounts = tiles.map(_.hartIds.size) - val controlBase = controlCounts.scanLeft(0)(_ + _) - val controlCount = controlCounts.sum - - require(tilePorts.size == tiles.size, "Designed tile count must match software topology") - val linkedTiles = tilePorts.indices.filter(i => tilePorts(i).t2t.isDefined) - if (linkedTiles.nonEmpty) { - val geometries = linkedTiles.map(i => tiles(i).sharedStorage.get.bytes) - require(geometries.distinct.size == 1, "TLink tiles must share the same public shared-storage geometry") - val fabric = Instantiate(new Fabric(ddr.axi, linkedTiles, geometries.head)) - for ((tile, port) <- linkedTiles.zipWithIndex) { - fabric.io.source(port) <> tilePorts(tile).t2t.get.tx - tilePorts(tile).t2t.get.rx <> fabric.io.target(port) - } - } - - val backing: Instance[Memory] = Instantiate(new Memory(ddr, dmaMasters + 1)) - val chipDevices = Instantiate(new Devices(devices, cp, hartIds)) - chipDevices.io.sources := 0.U - val external = Instantiate(new framework.system.memory.UncachedAxi(cp, ddr.axi, n)) - val scuParams = devices.scu - val scu = Instantiate(new sims.scu.SystemControl(n, cp, scuParams)) - scu.io.hartIds := VecInit(hartIds.map(_.U(32.W))) - for (core <- hartIds.indices) { - val request = chipDevices.io.externalRequest(core) - val response = chipDevices.io.externalResponse(core) - val inScu = request.bits.addr >= scuParams.baseAddress.U && - request.bits.addr < (scuParams.baseAddress + scuParams.totalSizeBytes).U - val selectedScu = RegInit(false.B) - when(request.fire)(selectedScu := inScu) - scu.io.request(core).valid := request.valid && inScu - scu.io.request(core).bits := request.bits - external.io.request(core).valid := request.valid && !inScu - external.io.request(core).bits := request.bits - request.ready := Mux(inScu, scu.io.request(core).ready, external.io.request(core).ready) - response.valid := Mux(selectedScu, scu.io.response(core).valid, external.io.response(core).valid) - response.bits := Mux(selectedScu, scu.io.response(core).bits, external.io.response(core).bits) - scu.io.response(core).ready := selectedScu && response.ready - external.io.response(core).ready := !selectedScu && response.ready - } - scu.io.failure := VecInit(tilePorts.flatMap(_.failure)) - scu.io.workDrained := VecInit(tilePorts.flatMap(_.workDrained)) - backing.io.dma(dmaMasters) <> external.io.axi - dlink.axi <> backing.io.axi - - val coreBase = tiles.scanLeft(0)(_ + _.hartIds.size) - for ((tile, index) <- tilePorts.zipWithIndex) { - tile.tileId := index.U - for ((port, id) <- tile.hartIds.zip(tiles(index).hartIds)) port := id.U(64.W) - for ((port, id) <- tile.executionIds.zip(tiles(index).executionIds)) port := id.U(64.W) - tile.time := chipDevices.io.time - for (local <- tiles(index).hartIds.indices) { - val core = coreBase(index) + local - tile.resetVector(local) := devices.bootrom.get.base.U - tile.interrupts(local) := chipDevices.io.interrupts(core) - chipDevices.io.request(core) <> tile.uncachedRequest(local) - tile.uncachedResponse(local) <> chipDevices.io.response(core) - } - // Tile-local requester IDs keep homogeneous tiles identical. Only endpoint identity - // changes at the shared control fabric; clocks, transactions and backpressure do not. - for (lane <- 0 until (if (index == 0) 0 else 2 * controlCounts(index))) { - val localNode = lane + 1 - val globalIndex = controlBase(index) + lane % controlCounts(index) + - (if (lane >= controlCounts(index)) controlCount else 0) - val globalNode = globalIndex + 1 - val source = tile.control(lane) - val remoteCount = controlCount - controlCounts.head - val remoteIndex = controlBase(index) - controlCounts.head + lane % controlCounts(index) + - (if (lane >= controlCounts(index)) remoteCount else 0) - val target = tilePorts.head.remoteControl(remoteIndex) - target <> source - target.req.bits.srcId := globalNode.U - target.txRsp.bits.srcId := globalNode.U - target.txDat.bits.srcId := globalNode.U - source.rxRsp.bits.tgtId := localNode.U - source.rxDat.bits.tgtId := localNode.U - when(source.req.valid) { - assert(source.req.bits.srcId === localNode.U && source.req.bits.returnNid === 0.U) - } - when(source.txRsp.valid)(assert(source.txRsp.bits.srcId === localNode.U)) - when(source.txDat.valid)(assert(source.txDat.bits.srcId === localNode.U)) - when(target.rxRsp.valid)(assert(target.rxRsp.bits.tgtId === globalNode.U)) - when(target.rxDat.valid)(assert(target.rxDat.bits.tgtId === globalNode.U)) - } - if (index == 0) { - val request = tile.backingRequest(0) - val response = tile.backingResponse(0) - devices.bootrom match { - case Some(rom) => - val lines = Instantiate(new BootRomLines(rom, memory.chi)) - lines.io.request <> request - response <> lines.io.response - backing.io.lineRequest(0) <> lines.io.memoryRequest - lines.io.memoryResponse <> backing.io.lineResponse(0) - case None => - backing.io.lineRequest(0) <> request - response <> backing.io.lineResponse(0) - } - } - backing.io.dma(index) <> tile.mem - } - } - -} diff --git a/arch/src/main/scala/framework/system/GlobalConfigOps.scala b/arch/src/main/scala/framework/system/GlobalConfigOps.scala deleted file mode 100644 index 4db33ce60..000000000 --- a/arch/src/main/scala/framework/system/GlobalConfigOps.scala +++ /dev/null @@ -1,186 +0,0 @@ -package framework.system - -import framework.top.GlobalConfig -import framework.top.configs._ -import framework.system.configloader._ -import framework.system.core.{AntCoreParams, RocketCoreParams} -import framework.system.tile.{TileParameters, TileParams} -import framework.system.device._ - -/** Pure configuration projection and IP-parameter derivation; no hardware is instantiated here. */ -object GlobalConfigOps { - def accelerator(b: GlobalConfig): AcceleratorConfig = - AcceleratorConfig(b.memDomain, b.frontend, b.rvv, b.ballDomain, b.tile, b.sim, b.coreSignature) - def accelerator(a: AcceleratorConfig): GlobalConfig = - GlobalConfig(a.memDomain, a.frontend, a.rvv, a.ballDomain, a.tile, a.sim, a.coreSignature) - - def core(name: String, c: TileCore): CoreDesign = c match { - case RocketTileCore(cpu, b) => CoreDesign(name, Some(cpu), None, b.map(accelerator)) - case AntTileCore(local, b) => CoreDesign(name, None, Some(local), Some(accelerator(b))) - case _ => throw new IllegalArgumentException("Unsupported core design") - } - - def core(c: CoreDesign): TileCore = c.rocket match { - case Some(cpu) => RocketTileCore(cpu, c.accelerator.map(accelerator)) - case None => AntTileCore(c.local.get, accelerator(c.accelerator.get)) - } - - def shared(s: SharedConfig): SharedStorageParams = SharedStorageParams(s.bankBits, s.bankEntries, s.banks) - - def design(t: TileTopology): TileDesign = { - require(t.coreFactories.size == t.cores.size, "Every core requires a configured design") - TileDesign( - t.factory, - t.param, - t.main, - t.coreFactories.zip(t.cores).map { case (n, c) => core(n, c) }, - t.signatures, - t.controller, - t.sharedStorage.map(s => SharedConfig(s.bankBits, s.bankEntries, s.banks)) - ) - } - - def fromSystem(p: SystemParams): GlobalConfig = { - val d = p.devices - val devices = DevicesConfig( - d.clint.base, - d.clint.bytes, - d.clint.tickCycles, - d.plic.base, - d.plic.bytes, - d.plic.sources, - d.plic.priorityBits, - d.bootrom.map(r => RomConfig(r.base, r.bytes, r.image)), - d.scu.baseAddress, - d.scu.strideBytes, - d.scu.totalSizeBytes, - d.scu.maxHarts - ) - val platform = PlatformConfig(p.cpuPhysicalBits, p.memory, p.l1, p.regions, p.tracking, p.ddr, devices) - GlobalConfig().copy(systemDesign = - Some(SystemDesign( - p.topology.tiles.map(t => framework.top.configs.TilePlacement(design(t), t.hartIds, t.executionIds)), - platform - )) - ) - } - - implicit class DesignParameters(val b: GlobalConfig) extends AnyVal { - - def systemParams: SystemParams = { - val s = b.systemDesign.get - val p = s.platform - val d = p.devices - val topology = ExampleTopology(s.tiles.map { t => - val v = t.design - TileTopology( - v.param, - v.main, - v.cores.map(core), - t.hartIds, - t.executionIds, - v.signatures, - v.controller, - v.sharedStorage.map(shared), - v.cores.map(_.factory), - v.factory - ) - }) - val devices = DeviceParams( - ClintParams(d.clintBase, d.clintBytes, d.tickCycles), - PlicParams(d.plicBase, d.plicBytes, d.sources, d.priorityBits), - d.bootrom.map(r => BootRomParams(r.base, r.bytes, r.image)), - sims.scu.SCUParams(d.scuBase, d.scuStride, d.scuBytes, d.scuMaxHarts) - ) - SystemParams(topology, p.cpuPhysicalBits, p.memory, p.l1, p.regions, p.tracking, p.ddr, devices) - } - - def forTile(index: Int): GlobalConfig = { - val s = b.systemDesign.get - val ids = s.tiles.flatMap(_.hartIds) - GlobalConfig().copy(tileDesign = Some(TileScope(s.tiles(index).design, s.platform, ids.max, ids.size, s.tiles.size))) - } - - def tileLinkParams: framework.system.tile.TileLinkParams = { - val t = b.tileDesign.get - framework.system.tile.TileLinkParams( - t.design.main, - t.design.cores.count(_.rocket.isDefined), - t.design.cores.size, - if (t.design.main) t.controls else 0, - t.platform.memory.chi, - t.platform.ddr.axi, - t.design.sharedStorage.isDefined - ) - } - - def tileParams: TileParams = { - val t = b.tileDesign.get - val p = t.platform - val cpus = t.design.cores.flatMap(_.rocket).map(cpu => - TileParameters.cpuParameters(cpu, t.design.param, t.maxHartId, p.cpuPhysicalBits) - ) - TileParams( - t.design.main, - t.design.cores.map(core), - t.design.signatures, - t.design.controller, - cpus, - p.memory.copy(agents = 2 * cpus.size), - p.l1, - p.regions, - p.tracking, - t.controls, - p.ddr.axi, - t.design.sharedStorage.map(shared), - t.tiles - ) - } - - def forCore(index: Int): GlobalConfig = { - val t = b.tileDesign.get - val c = t.design.cores(index) - val cpus = t.design.cores.flatMap(_.rocket) - val selected = c.accelerator.map(accelerator).getOrElse(GlobalConfig()) - val signature = if (c.local.isDefined) selected.coreSignature else BigInt(0) - selected.copy(coreDesign = Some(CoreScope( - c.factory, - c.rocket, - c.local, - c.accelerator.isDefined, - t.design.controller.map(i => t.design.cores(i).rocket.get), - t.design.param, - t.platform, - t.maxHartId, - if (c.rocket.isDefined) t.design.cores.take(index).count(_.rocket.isDefined) else 0, - cpus.size, - signature - ))) - } - - def rocketParams: RocketCoreParams = { - val c = b.coreDesign.get - require(c.rocket.isDefined) - val p = c.platform - val cpu = TileParameters.cpuParameters(c.rocket.get, c.tile, c.maxHartId, p.cpuPhysicalBits) - RocketCoreParams( - cpu, - p.l1.copy(nodeId = c.cpuIndex + 1), - p.l1.copy(nodeId = c.cpuCount + c.cpuIndex + 1), - p.regions, - hier.core.rocket.Commands(c.hasAccelerator, true, p.tracking), - Option.when(c.hasAccelerator)(b) - ) - } - - def antParams: AntCoreParams = { - val c = b.coreDesign.get - require(c.local.isDefined && c.hasAccelerator) - val p = c.platform - val cpu = TileParameters.cpuParameters(c.controller.get, c.tile, c.maxHartId, p.cpuPhysicalBits) - AntCoreParams(b, c.local.get, p.tracking, p.memory.chi, c.signature, cpu) - } - - } - -} diff --git a/arch/src/main/scala/framework/system/System.scala b/arch/src/main/scala/framework/system/System.scala deleted file mode 100644 index 8c1b798f6..000000000 --- a/arch/src/main/scala/framework/system/System.scala +++ /dev/null @@ -1,31 +0,0 @@ -package framework.system - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.configloader.ExampleTopology -import framework.system.device.DeviceParams -import framework.system.dlink.{DLinkIO, HasDLink} -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.interlock.{Params => TrackingParams} -import memcore.memory.ddr.{Params => DdrParams} - -case class SystemParams( - topology: ExampleTopology, - cpuPhysicalBits: Int, - memory: CoherenceParams, - l1: RnfParams, - regions: Seq[PhysicalRegion], - tracking: TrackingParams, - ddr: DdrParams, - devices: DeviceParams = DeviceParams()) { - - def toGlobalConfig: framework.top.GlobalConfig = GlobalConfigOps.fromSystem(this) - -} - -@instantiable -abstract class System(p: SystemParams) extends Module with HasDLink { - @public val dlink = IO(new DLinkIO(p)) -} diff --git a/arch/src/main/scala/framework/system/configloader/ChipLoader.scala b/arch/src/main/scala/framework/system/configloader/ChipLoader.scala deleted file mode 100644 index 93d9a507a..000000000 --- a/arch/src/main/scala/framework/system/configloader/ChipLoader.scala +++ /dev/null @@ -1,499 +0,0 @@ -package framework.system.configloader - -import buckyball.config.{ - BoomCpuConfig, - Chip, - CoreInstance, - CpuConfig, - FrontendConfig, - MemDomainConfig, - RocketCpuConfig, - RvvConfig, - SharedMemConfig, - SpmConfig, - TileKind, - TileParamConfig, - TilePlacement -} -import java.nio.file.{Files, Path, Paths} -import framework.balldomain.configs.{BallDomainParam, BallISAEntry, BallIdMapping} -import framework.frontend.configs.FrontendParam -import framework.rvv.configs.RvvParam -import framework.memdomain.configs.MemDomainParam -import framework.system.core.boom.configs.{BoomCpuParam, BoomDCacheParam, BoomICacheParam} -import framework.system.core.rocket.configs._ -import framework.system.tile.configs.TileParam -import framework.top.GlobalConfig -import scala.jdk.CollectionConverters._ - -/** Load ExampleTopology from chip.pb. */ -object ChipLoader { - - def load(pbPath: String): ExampleTopology = { - val path = Paths.get(pbPath) - if (!Files.isRegularFile(path)) { - throw new RuntimeException(s"chip.pb does not exist: $pbPath") - } - val repo = repoRoot(path) - val chip = Chip.parseFrom(Files.readAllBytes(path)) - val cores = chip.getCoresList.asScala.toSeq - if (cores.isEmpty) { - throw new RuntimeException(s"chip.pb has no cores: $pbPath") - } - val tiles = chip.getTilesList.asScala.map(parseTile(_, cores, repo)).toSeq - require( - tiles.size == chip.getNTiles, - s"chip.pb declares n_tiles=${chip.getNTiles} but defines ${tiles.size} tile(s) in $pbPath" - ) - val hartIds = tiles.flatMap(_.hartIds) - require(hartIds.forall(_ >= 0), "hart_id must fit a nonnegative Scala Int") - require(hartIds.distinct.size == hartIds.size, "chip hart_id values must be globally unique") - ExampleTopology(tiles) - } - - def repoRoot(pb: Path): Path = { - val abs = pb.toAbsolutePath.normalize - val s = abs.toString - val marker = "/examples/chips/" - val i = s.lastIndexOf(marker) - if (i < 0) { - throw new RuntimeException(s"chip.pb is not under examples/chips: $pb") - } - Paths.get(s.substring(0, i)) - } - - def repoFile(repo: Path, rel: String, what: String): String = { - if (rel.isEmpty) { - throw new RuntimeException(s"$what is empty") - } - val p = Paths.get(rel) - if (p.isAbsolute) { - throw new RuntimeException(s"$what must be repo-relative: $rel") - } - val resolved = repo.resolve(p).normalize() - if (!Files.isRegularFile(resolved)) { - throw new RuntimeException(s"$what missing: $resolved") - } - resolved.toString - } - - def parseTile(tile: TilePlacement, cores: Seq[CoreInstance], repo: Path): TileTopology = { - val indices = tile.getCoreIndicesList.asScala.map(_.toInt).toSeq - val nCores = indices.size - val computeCoreIds = - indices.zipWithIndex.collect { case (index, slot) if cores(index).getBalldomain.getBallNum > 0 => slot } - val hasBuckyball = computeCoreIds.nonEmpty - val tileParam = parseTileParam(tile.getParam) - - val shared = tile.getSharedMem - val virtualBankCount = tile.getVirtualBankCount - if (hasBuckyball) { - require(virtualBankCount > 0, s"tile ${tile.getPath}: virtual_bank_count must be > 0") - } - val sharedBankNum = - if (shared.getEnable) { - require(shared.getBankWidth == 128, s"tile ${tile.getPath}: shared bank width must be 128 bits") - indices.iterator.map(idx => cores(idx)).filter(_.getBalldomain.getBallNum > 0).foreach { core => - require( - core.getMem.getBank.getWidth == shared.getBankWidth, - s"tile ${tile.getPath}: all Buckyball cores must use shared bank width ${shared.getBankWidth}" - ) - } - require(shared.getEntries > 0, s"tile ${tile.getPath}: shared entries must be > 0") - require( - shared.getBankEntries > 0 && shared.getEntries % shared.getBankEntries == 0, - s"tile ${tile.getPath}: shared entries ${shared.getEntries} must be divisible by shared bank entries ${shared.getBankEntries}" - ) - shared.getEntries / shared.getBankEntries - } else 0 - val tss = Option.when(tile.hasTss)(parseSpm(tile.getTss)) - val tileCores = indices.map { idx => - parseCore(cores(idx), tileParam, shared, sharedBankNum, virtualBankCount, nCores, computeCoreIds, tss, repo) - } - val ants = indices.filter(i => cores(i).getCpu.getKind == "ant") - ants.zipWithIndex.foreach { case (idx, context) => - require( - !cores(idx).hasHartId && cores(idx).hasAntContextId && cores(idx).getAntContextId == context, - s"core $idx: Ant requires explicit local context, without a CPU hart" - ) - } - val hartIds = indices.filterNot(ants.contains).map { idx => - require(cores(idx).hasHartId && !cores(idx).hasAntContextId, s"core $idx: CPU requires hart_id only") - cores(idx).getHartId - } - val main = tile.getKind match { - case TileKind.TILE_KIND_MAIN => true - case TileKind.TILE_KIND_TILE => false - case kind => throw new IllegalArgumentException(s"Unknown tile kind: $kind") - } - require(!main || !tile.hasControllerCoreIndex, "The main tile has no task controller") - val controller = Option.when(tile.hasControllerCoreIndex) { - val local = indices.indexOf(tile.getControllerCoreIndex) - require(local >= 0, s"tile ${tile.getPath}: controller must belong to the tile") - local - } - require(tile.getFactoryClass.isEmpty == main, "Only mounted tiles must specify a factory") - TileTopology( - tileParam, - main, - tileCores, - hartIds, - indices, - if (controller.isDefined) indices.map(idx => coreSignature(cores(idx), tss, shared)) else Nil, - controller, - Option.when(shared.getEnable)(SharedStorageParams(shared.getBankWidth, shared.getBankEntries, sharedBankNum)), - indices.map(idx => cores(idx).getFactoryClass), - tile.getFactoryClass - ) - } - - def parseCore( - core: CoreInstance, - tile: TileParam, - shared: SharedMemConfig, - sharedBankNum: Int, - virtualBankCount: Int, - nCores: Int, - computeCoreIds: Seq[Int], - tss: Option[memcore.memory.spm.Params], - repo: Path - ): TileCore = { - require(core.hasCpu, s"core ${core.getPkg}: missing cpu config") - val cpu = core.getCpu - val kind = cpu.getKind - kind match { - case "rocket" => - require(cpu.hasRocket, s"core ${core.getPkg}: missing cpu.rocket") - RocketTileCore( - parseRocketCpu(cpu.getRocket), - parseAccelerator(core, tile, shared, sharedBankNum, virtualBankCount, nCores, computeCoreIds, tss, repo) - ) - case "ant" => - require(cpu.hasAnt && tss.isDefined, s"core ${core.getPkg}: Ant requires local geometry and tile TSS") - val a = cpu.getAnt - require(a.hasTls, s"core ${core.getPkg}: missing Ant TLS") - val local = framework.ant.Params(a.getCodeBytes, parseSpm(a.getTls), tss.get, a.getTaskBits) - val accelerator = - parseAccelerator(core, tile, shared, sharedBankNum, virtualBankCount, nCores, computeCoreIds, tss, repo) - require(accelerator.isDefined, "A compute Ant must have an NPU endpoint") - AntTileCore(local, accelerator.get) - case "boom" => - if (core.getBalldomain.getBallNum > 0) { - throw new RuntimeException(s"core ${core.getPkg}: kind=boom forbids balldomain") - } - if (!cpu.hasBoom) { - throw new RuntimeException(s"core ${core.getPkg}: kind=boom missing cpu.boom") - } - BoomTileCore(parseBoomCpu(cpu.getBoom)) - case other => - throw new RuntimeException(s"core ${core.getPkg}: unsupported kind '$other'") - } - } - - def parseAccelerator( - core: CoreInstance, - tile: TileParam, - shared: SharedMemConfig, - sharedBankNum: Int, - virtualBankCount: Int, - nCores: Int, - computeCoreIds: Seq[Int], - tss: Option[memcore.memory.spm.Params], - repo: Path - ): Option[GlobalConfig] = { - val domain = core.getBalldomain - if (domain.getBallNum == 0) { - return None - } - require(core.hasFrontend, s"core ${core.getPkg} missing frontend config") - require(core.hasRvv, s"core ${core.getPkg} missing rvv config") - require(core.getRvv.hasEnable, s"core ${core.getPkg} must explicitly set rvv.enable") - require(tile.coreDataBytes > 0, s"core ${core.getPkg} missing tile config") - val frontend = core.getFrontend - require( - virtualBankCount <= (1 << frontend.getBankIdLen), - s"core ${core.getPkg}: virtual_bank_count $virtualBankCount does not fit bank_id_len ${frontend.getBankIdLen}" - ) - require( - frontend.getVbankIdUpperBound < virtualBankCount, - s"core ${core.getPkg}: private vbank upper bound ${frontend.getVbankIdUpperBound} must be below virtual_bank_count $virtualBankCount" - ) - if (shared.getEnable) { - require( - frontend.getSharedBankIdBase > frontend.getVbankIdUpperBound, - s"core ${core.getPkg}: shared bank base must be above the private vbank range" - ) - require( - frontend.getSharedBankIdBase <= virtualBankCount, - s"core ${core.getPkg}: shared bank base ${frontend.getSharedBankIdBase} exceeds virtual_bank_count $virtualBankCount" - ) - } - - val buckyball = GlobalConfig().copy( - coreSignature = coreSignature(core, tss, shared), - ballDomain = parseBallDomain(core, repo), - frontend = parseFrontend(core.getFrontend), - rvv = parseRvv(core.getRvv), - tile = tile, - memDomain = parseMemDomain(core.getMem, shared, sharedBankNum, virtualBankCount, nCores, computeCoreIds) - ) - Some(buckyball) - } - - def parseSpm(value: SpmConfig): memcore.memory.spm.Params = - memcore.memory.spm.Params(BigInt(java.lang.Long.toUnsignedString(value.getBase)), value.getBytes, value.getDataBits) - - def coreSignature( - core: CoreInstance, - tss: Option[memcore.memory.spm.Params], - sharedMemory: SharedMemConfig - ): BigInt = { - require(core.hasMem && core.getMem.hasBank, s"core ${core.getPkg}: task signature requires explicit mem.bank") - val bytes = new java.io.ByteArrayOutputStream() - def text(value: String): Unit = { - bytes.write(value.getBytes(java.nio.charset.StandardCharsets.UTF_8)); bytes.write(0) - } - def integer(value: Long, width: Int = 8): Unit = - (0 until width).foreach(i => bytes.write(((value >>> (8 * i)) & 255).toInt)) - text(core.getPkg) - val bank = core.getMem.getBank - Seq(bank.getNum, bank.getWidth, bank.getEntries).foreach(value => integer(value.toLong)) - core.getBalldomain.getIsaList.asScala.sortBy(_.getFunct7).foreach { entry => - text(entry.getMnemonic); integer(entry.getFunct7.toLong, 4) - } - core.getBalldomain.getMappingsList.asScala.sortBy(_.getBallClass.split('.').last).foreach { mapping => - text(mapping.getBallClass.split('.').last) - integer(mapping.getInBw.toLong); integer(mapping.getOutBw.toLong) - mapping.getBallParamsMap.asScala.toSeq.sortBy(_._1).foreach { case (name, value) => text(name); text(value) } - } - if (core.getCpu.getKind == "ant") { - val a = core.getCpu.getAnt - val shared = tss.get - text("ant") - Seq( - a.getCodeBytes.toLong, - a.getTls.getBase, - a.getTls.getBytes.toLong, - a.getTls.getDataBits.toLong, - a.getTaskBits.toLong, - shared.base.toLong, - shared.bytes.toLong, - shared.dataBits.toLong, - sharedMemory.getBankEntries.toLong, - sharedMemory.getEntries.toLong - ).foreach(v => integer(v)) - } - val mask = (BigInt(1) << 64) - 1 - bytes.toByteArray.foldLeft(BigInt("cbf29ce484222325", 16)) { (hash, value) => - ((hash ^ BigInt(value & 255)) * BigInt("100000001b3", 16)) & mask - } - } - - def parseBallDomain(core: CoreInstance, repo: Path): BallDomainParam = { - val domain = core.getBalldomain - val mappings = domain.getMappingsList.asScala.map { m => - val params = m.getBallParamsMap.asScala.toMap - require(m.getInBw > 0 && m.getOutBw > 0, s"Ball ${m.getBallName}: BBus widths must be positive") - val config = Some(repoFile(repo, m.getConfigPath, s"Ball ${m.getBallName} config")) - - BallIdMapping( - ballId = m.getBallId, - ballName = m.getBallName, - ballClass = m.getBallClass, - config = config, - inBW = m.getInBw, - outBW = m.getOutBw, - mmioReadBW = m.getMmioReadBw, - mmioWriteBW = m.getMmioWriteBw, - ballParams = params - ) - }.toSeq - val isa = domain.getIsaList.asScala.map { e => - BallISAEntry(mnemonic = e.getMnemonic, funct7 = e.getFunct7, bid = e.getBid) - }.toSeq - BallDomainParam(ballNum = domain.getBallNum, ballIdMappings = mappings, ballISA = isa) - } - - def parseMemDomain( - mem: MemDomainConfig, - shared: SharedMemConfig, - sharedBankNum: Int, - virtualBankCount: Int, - nCores: Int, - computeCoreIds: Seq[Int] - ): MemDomainParam = { - val bank = mem.getBank - val dma = mem.getDma - val tlb = mem.getTlb - val tma = mem.getTma - val mmio = mem.getMmio - MemDomainParam( - bankNum = bank.getNum, - bankWidth = bank.getWidth, - bankEntries = bank.getEntries, - bankMaskLen = bank.getMaskLen, - virtualBankCount = virtualBankCount, - sharedEnable = shared.getEnable, - sharedEntries = shared.getEntries, - sharedBankEntries = shared.getBankEntries, - sharedBankNum = sharedBankNum, - sharedInputChannels = shared.getInputChannels, - sharedDefaultGroupCount = shared.getDefaultGroupCount, - nCores = nCores, - computeCoreIds = computeCoreIds, - tlb_size = tlb.getSize, - dma_n_xacts = dma.getNXacts, - dma_burst_maxbytes = dma.getBurstMaxBytes, - bankChannel = bank.getChannel, - max_in_flight_mem_reqs = dma.getMaxInFlightMemReqs, - dma_buswidth = dma.getBusWidth, - memAddrLen = mem.getMem.getAddrLen, - tmaReadChannel = tma.getReadChannel, - tmaWriteChannel = tma.getWriteChannel, - mmioEnable = mmio.getEnable, - mmioBankNum = mmio.getBankNum, - mmioBankEntries = mmio.getBankEntries, - mmioBankWidth = mmio.getBankWidth, - mmioReadWidth = mmio.getReadWidth - ) - } - - def parseFrontend(frontend: FrontendConfig): FrontendParam = - FrontendParam( - rob_entries = frontend.getRobEntries, - rs_out_of_order_response = frontend.getRsOutOfOrderResponse, - bank_id_len = frontend.getBankIdLen, - vbank_id_upper_bound = frontend.getVbankIdUpperBound, - shared_bank_id_base = frontend.getSharedBankIdBase, - iter_len = frontend.getIterLen, - sub_rob_enable = frontend.getSubRobEnable, - sub_rob_depth = frontend.getSubRobDepth - ) - - def parseRvv(rvv: RvvConfig): RvvParam = - RvvParam( - enable = rvv.getEnable, - laneNumber = rvv.getLaneNumber, - vLen = rvv.getVLen, - eLen = rvv.getELen, - iBufWords = rvv.getIBufWords, - memoryPorts = rvv.getMemoryPorts - ) - - def parseTileParam(param: TileParamConfig): TileParam = - TileParam( - coreDataBytes = param.getCoreDataBytes, - xLen = param.getXLen, - vaddrBits = param.getVaddrBits, - paddrBits = param.getPaddrBits, - pgIdxBits = param.getPgIdxBits, - pgLevels = param.getPgLevels, - nPMPs = param.getNPmps - ) - - def parseRocketCpu(rocket: RocketCpuConfig): RocketCpuParam = { - val mulDiv = rocket.getMulDiv - val fpu = rocket.getFpu - val dcache = rocket.getDcache - val icache = rocket.getIcache - val btb = rocket.getBtb - RocketCpuParam( - mtvecInitEnable = rocket.getMtvecInitEnable, - useUser = rocket.getUseUser, - useSupervisor = rocket.getUseSupervisor, - useHypervisor = rocket.getUseHypervisor, - useDebug = rocket.getUseDebug, - useAtomics = rocket.getUseAtomics, - useAtomicsOnlyForIO = rocket.getUseAtomicsOnlyForIO, - useCompressed = rocket.getUseCompressed, - useRVE = rocket.getUseRVE, - useConditionalZero = rocket.getUseConditionalZero, - nLocalInterrupts = rocket.getNLocalInterrupts, - useNMI = rocket.getUseNMI, - nBreakpoints = rocket.getNBreakpoints, - useBPWatch = rocket.getUseBPWatch, - mcontextWidth = rocket.getMcontextWidth, - scontextWidth = rocket.getScontextWidth, - nPerfCounters = rocket.getNPerfCounters, - haveBasicCounters = rocket.getHaveBasicCounters, - misaWritable = rocket.getMisaWritable, - mtvecInit = BigInt(java.lang.Long.toUnsignedString(rocket.getMtvecInit)), - mtvecWritable = rocket.getMtvecWritable, - fastLoadWord = rocket.getFastLoadWord, - fastLoadByte = rocket.getFastLoadByte, - branchPredictionModeCSR = rocket.getBranchPredictionModeCSR, - clockGate = rocket.getClockGate, - mvendorid = rocket.getMvendorid, - mimpid = rocket.getMimpid, - haveCease = rocket.getHaveCease, - haveSimTimeout = rocket.getHaveSimTimeout, - useVM = rocket.getUseVm, - useZba = rocket.getUseZba, - useZbb = rocket.getUseZbb, - useZbs = rocket.getUseZbs, - haveCFlush = rocket.getHaveCFlush, - mulDiv = MulDivParam( - divUnroll = mulDiv.getDivUnroll, - divEarlyOutGranularity = mulDiv.getDivEarlyOutGranularity, - enable = mulDiv.getEnable, - mulUnroll = mulDiv.getMulUnroll, - mulEarlyOut = mulDiv.getMulEarlyOut, - divEarlyOut = mulDiv.getDivEarlyOut - ), - fpu = FPUParam( - divSqrt = fpu.getDivSqrt, - sfmaLatency = fpu.getSfmaLatency, - dfmaLatency = fpu.getDfmaLatency, - fpmuLatency = fpu.getFpmuLatency, - ifpuLatency = fpu.getIfpuLatency, - enable = fpu.getEnable, - minFLen = fpu.getMinFLen, - fLen = fpu.getFLen - ), - dcache = DCacheParam( - clockGate = dcache.getClockGate, - nSets = dcache.getNSets, - nWays = dcache.getNWays, - nMSHRs = dcache.getNMshrs - ), - icache = ICacheParam( - prefetch = icache.getPrefetch, - nSets = icache.getNSets, - nWays = icache.getNWays - ), - btb = BTBParam( - nMatchBits = btb.getNMatchBits, - nPages = btb.getNPages, - updatesOutOfOrder = btb.getUpdatesOutOfOrder, - bhtEnable = btb.getBhtEnable, - bhtEntries = btb.getBhtEntries, - bhtCounterLength = btb.getBhtCounterLength, - bhtHistoryLength = btb.getBhtHistoryLength, - bhtHistoryBits = btb.getBhtHistoryBits, - enable = btb.getEnable, - nEntries = btb.getNEntries, - nRAS = btb.getNRas - ) - ) - } - - def parseBoomCpu(boom: BoomCpuConfig): BoomCpuParam = { - val dcache = boom.getDcache - val icache = boom.getIcache - BoomCpuParam( - fetchWidth = boom.getFetchWidth, - decodeWidth = boom.getDecodeWidth, - numRobEntries = boom.getNumRobEntries, - dcache = BoomDCacheParam( - nSets = dcache.getNSets, - nWays = dcache.getNWays, - nMSHRs = dcache.getNMshrs - ), - icache = BoomICacheParam( - nSets = icache.getNSets, - nWays = icache.getNWays - ) - ) - } - -} diff --git a/arch/src/main/scala/framework/system/configloader/Topology.scala b/arch/src/main/scala/framework/system/configloader/Topology.scala deleted file mode 100644 index 035f8b5e0..000000000 --- a/arch/src/main/scala/framework/system/configloader/Topology.scala +++ /dev/null @@ -1,50 +0,0 @@ -package framework.system.configloader - -import framework.system.core.boom.configs.BoomCpuParam -import framework.system.core.rocket.configs.RocketCpuParam -import framework.system.tile.configs.TileParam -import framework.top.GlobalConfig - -/** Chip topology loaded from chip.pb: the main tile first, then configured TLink tiles. */ -case class ExampleTopology(tiles: Seq[TileTopology]) { - require(tiles.nonEmpty && tiles.head.main && tiles.tail.forall(!_.main), "Tile 0 alone is the main tile") - def mainTile: TileTopology = tiles.head - def mountedTiles: Seq[TileTopology] = tiles.tail -} - -sealed trait TileCore - -case class RocketTileCore( - cpu: RocketCpuParam, - buckyball: Option[GlobalConfig]) - extends TileCore - -case class BoomTileCore(cpu: BoomCpuParam) extends TileCore - -case class AntTileCore(local: framework.ant.Params, buckyball: GlobalConfig) extends TileCore - -case class SharedStorageParams(bankBits: Int, bankEntries: Int, banks: Int) { - require(bankBits == 128 && bankEntries > 0 && banks > 0) - val bankBytes: Int = bankEntries * (bankBits / 8) - val bytes: BigInt = BigInt(banks) * bankBytes -} - -/** - * Per-tile topology: ordered cores. CPU kind is per-core; shared memory is tile-level. - * CPU cores have system hart IDs; Ant contexts remain tile-local resources. - */ -case class TileTopology( - param: TileParam, - main: Boolean, - cores: Seq[TileCore], - hartIds: Seq[Int], - executionIds: Seq[Int], - signatures: Seq[BigInt], - controller: Option[Int], - sharedStorage: Option[SharedStorageParams], - coreFactories: Seq[String], - factory: String) { - val cpuCoreIds: Seq[Int] = cores.indices.filter(i => !cores(i).isInstanceOf[AntTileCore]) - require(hartIds.size == cpuCoreIds.size && executionIds.size == cores.size) - require(!main || controller.isEmpty, "The main tile has no task controller") -} diff --git a/arch/src/main/scala/framework/system/core/AntEndpoint.scala b/arch/src/main/scala/framework/system/core/AntEndpoint.scala deleted file mode 100644 index ccd8d19e6..000000000 --- a/arch/src/main/scala/framework/system/core/AntEndpoint.scala +++ /dev/null @@ -1,19 +0,0 @@ -package framework.system.core - -import framework.system.GlobalConfigOps._ -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.core.clink.{CLinkIO, HasCLink} -import framework.system.core.rocket.CpuParams - -/** Typed Ant/NPU boundary; implementations own Ant and TLS, while the tile owns TSS. */ -@instantiable -abstract class AntEndpoint(val b: framework.top.GlobalConfig) extends Module with HasCLink { - val p = b.antParams - val local = p.local - val tracking = p.tracking - val bus = p.bus - val signature = p.signature - implicit val cpuParams: CpuParams = p.cpu - @public val clink = IO(new CLinkIO(b, local, tracking, bus)) -} diff --git a/arch/src/main/scala/framework/system/core/ControllerAdmission.scala b/arch/src/main/scala/framework/system/core/ControllerAdmission.scala deleted file mode 100644 index f5282bb43..000000000 --- a/arch/src/main/scala/framework/system/core/ControllerAdmission.scala +++ /dev/null @@ -1,104 +0,0 @@ -package framework.system.core - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import framework.system.core.rocket.RoCCIO -import framework.memdomain.isa.{MvoverISA, MvoverPort} -import framework.memdomain.frontend.mem.dma.{DmaError, DmaStatus} -import hier.core.rocket.{AdmissionPorts, CommandSnapshot} -import hier.tile.TaskAdmission -import memcore.bus.chi.{Params => ChiParams} -import memcore.memory.interlock.{Params => TrackingParams} - -/** CPU-only controller command path: task RPC and bank moves have explicit completion owners. */ -@instantiable -class ControllerAdmission(tracking: TrackingParams, bus: ChiParams, moves: Boolean)(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - require(xLen == 64) - - @public - val io = IO(new Bundle { - val core = Flipped(new AdmissionPorts(tracking, nPMPs, bus)) - val task = Flipped(new RoCCIO(64)) - val taskSatp = Output(UInt(64.W)) - val taskContext = Output(new CommandSnapshot(tracking, nPMPs)) - val move = new MvoverPort - val moveFault = Valid(new DmaStatus) - }) - - val idle :: task :: moving :: moveResponse :: releasing :: Nil = Enum(5) - val state = RegInit(idle) - val tag = Reg(UInt(tracking.idBits.W)) - val rd = Reg(UInt(5.W)) - val failedMove = Reg(Bool()) - val instruction = io.core.command.bits.instruction - val isTask = instruction.opcode === "h2b".U - val isMove = moves.B && instruction.opcode === "h7b".U && instruction.funct === MvoverISA.Funct.U - val controller: Instance[TaskAdmission] = Instantiate(new TaskAdmission(tracking, nPMPs)) - controller.io.task <> io.task - io.taskSatp := controller.io.satp - // The Ant launch handshake is the task command handshake; freeze this complete snapshot there. - io.taskContext := io.core.command.bits - controller.io.workDrained := true.B // This endpoint has no NPU or DMA client. - controller.io.command.valid := state === idle && io.core.command.valid && isTask - controller.io.command.bits := io.core.command.bits - - io.core.reserve.ready := !reset.asBool - io.core.command.ready := state === idle && !reset.asBool && - Mux(isTask, controller.io.command.ready, isMove && io.move.command.ready) - when(io.core.command.valid && state === idle && !reset.asBool) { - assert(isTask || isMove, "CPU-only controller received an unsupported compute instruction") - } - io.move.command.valid := state === idle && io.core.command.valid && isMove && !reset.asBool - io.move.command.bits.sourceCore := instruction.rs1Data(7, 0) - io.move.command.bits.targetCore := instruction.rs1Data(15, 8) - io.move.command.bits.sourceBank := instruction.rs1Data(25, 16) - io.move.command.bits.targetBank := instruction.rs1Data(35, 26) - io.move.command.bits.sourceAddr := instruction.rs2Data(15, 0) - io.move.command.bits.targetAddr := instruction.rs2Data(31, 16) - io.move.command.bits.rows := instruction.rs2Data(47, 32) +& 1.U - when(io.core.command.fire) { - tag := io.core.command.bits.tag - rd := instruction.rd - state := Mux(isTask, task, moving) - } - io.move.completion.ready := state === moving && !reset.asBool - when(io.move.completion.fire) { - failedMove := io.move.completion.bits - state := moveResponse - } - io.moveFault.valid := io.move.completion.fire && io.move.completion.bits - io.moveFault.bits.error := DmaError.Bank.U - io.moveFault.bits.address := 0.U // Existing move completion supplies no byte address. - - io.core.response.valid := Mux(state === task, controller.io.response.valid, state === moveResponse) && !reset.asBool - io.core.response.bits := controller.io.response.bits - when(state === moveResponse) { - io.core.response.bits.rd := rd - io.core.response.bits.data := failedMove.asUInt - } - controller.io.response.ready := state === task && io.core.response.ready - when(state === moveResponse && io.core.response.fire)(state := releasing) - io.core.complete.valid := (state === releasing || (state === task && controller.io.release.valid)) && !reset.asBool - io.core.complete.bits.tag := Mux(state === task, controller.io.release.bits.tag, tag) - controller.io.release.ready := state === task && io.core.complete.ready - when(io.core.complete.fire)(state := idle) - io.core.cancelled.valid := false.B - io.core.cancelled.bits := 0.U.asTypeOf(io.core.cancelled.bits) - io.core.interrupt := false.B - - // A move may have retired at CPU dispatch while its work is still pending here. - // Do not let a following CPU store publish a completion flag before the bank move ends. - val pendingMove = io.core.command.valid && isMove - io.core.cpuAllow := !(pendingMove || state === moving || state === moveResponse || state === releasing) - io.core.cpuProbeAllow := io.core.cpuAllow - io.core.maintenance.valid := false.B - io.core.maintenance.bits := 0.U.asTypeOf(io.core.maintenance.bits) - io.core.maintained.ready := true.B - io.core.pteRequest.valid := false.B - io.core.pteRequest.bits := 0.U.asTypeOf(io.core.pteRequest.bits) - io.core.pteResponse.ready := true.B -} diff --git a/arch/src/main/scala/framework/system/core/CoreConnection.scala b/arch/src/main/scala/framework/system/core/CoreConnection.scala deleted file mode 100644 index cf6d7bd92..000000000 --- a/arch/src/main/scala/framework/system/core/CoreConnection.scala +++ /dev/null @@ -1,80 +0,0 @@ -package framework.system.core - -import chisel3._ -import chisel3.experimental.hierarchy.Instance -import framework.ant.LocalExecution -import framework.system.core.accelerator.{AntAdmission, BuckyballAccelerator} -import framework.system.core.clink.CLinkIO - -object CoreConnection { - - def accelerator(link: AcceleratorCLink, accelerator: Instance[BuckyballAccelerator]): Unit = { - accelerator.io.cmd <> link.npu.command - link.npu.response <> accelerator.io.resp - link.npu.allocation := accelerator.io.allocation - link.npu.retired := accelerator.io.retired - link.npu.fault := accelerator.io.fault - link.npu.busy := !accelerator.io.idle - link.npu.interrupt := accelerator.io.interrupt - link.npu.footprints := accelerator.io.footprints - link.mem <> accelerator.io.dma - accelerator.io.hartid := link.hartId - accelerator.io.sharedBankOwnerHartId := link.shmOwner - link.shm.requests <> accelerator.io.shared_mem_req - link.shm.move <> accelerator.io.mvover - accelerator.io.meshLocalBank <> link.shm.local - link.shm.config <> accelerator.io.shared_config - link.shm.queryValid := accelerator.io.shared_query_valid - link.shm.queryVbank := accelerator.io.shared_query_vbank_id - accelerator.io.shared_query_group_count := link.shm.queryGroups - link.shm.barrierArrive := accelerator.io.barrier_arrive - accelerator.io.barrier_release := link.shm.barrierRelease - accelerator.io.shared_bank_hashes.foreach(_ := link.sharedHashes.get) - } - - def ant( - clink: CLinkIO, - execution: Instance[LocalExecution], - accelerator: Instance[BuckyballAccelerator], - issuer: Instance[AntAdmission], - signature: BigInt - ): Unit = { - execution.io.local <> clink.local - clink.ctrl.fingerprint := signature.U(64.W) - clink.ctrl.online := true.B - issuer.io.bind.valid := clink.ctrl.bind.valid - issuer.io.bind.bits.task := clink.ctrl.bind.bits.task - issuer.io.bind.bits.snapshot := clink.ctrl.bind.bits.snapshot - issuer.io.command <> execution.io.command - execution.io.response <> issuer.io.response - issuer.io.cancel := clink.local.cancel - issuer.io.workDrained := clink.ctrl.workDrained - issuer.io.halted := clink.ctrl.halted - clink.ctrl.drained := issuer.io.drained - execution.io.npuDrained := issuer.io.drained - clink.admission <> issuer.io.admission - clink.memory <> issuer.io.memory - accelerator.io.cmd <> clink.npu.command - clink.npu.response <> accelerator.io.resp - clink.npu.allocation := accelerator.io.allocation - clink.npu.retired := accelerator.io.retired - clink.npu.fault := accelerator.io.fault - clink.npu.busy := !accelerator.io.idle - clink.npu.interrupt := accelerator.io.interrupt - clink.npu.footprints := accelerator.io.footprints - clink.mem <> accelerator.io.dma - accelerator.io.hartid := clink.ctrl.hartId - accelerator.io.sharedBankOwnerHartId := clink.ctrl.shmOwner - clink.shm.requests <> accelerator.io.shared_mem_req - clink.shm.move <> accelerator.io.mvover - accelerator.io.meshLocalBank <> clink.shm.local - clink.shm.config <> accelerator.io.shared_config - clink.shm.queryValid := accelerator.io.shared_query_valid - clink.shm.queryVbank := accelerator.io.shared_query_vbank_id - accelerator.io.shared_query_group_count := clink.shm.queryGroups - clink.shm.barrierArrive := accelerator.io.barrier_arrive - accelerator.io.barrier_release := clink.shm.barrierRelease - accelerator.io.shared_bank_hashes.foreach(_ := clink.sharedHashes.get) - } - -} diff --git a/arch/src/main/scala/framework/system/core/CoreFactory.scala b/arch/src/main/scala/framework/system/core/CoreFactory.scala deleted file mode 100644 index 7b65ded35..000000000 --- a/arch/src/main/scala/framework/system/core/CoreFactory.scala +++ /dev/null @@ -1,24 +0,0 @@ -package framework.system.core - -import chisel3.experimental.hierarchy.Instance -import framework.top.GlobalConfig - -trait RocketCoreFactory { - def instantiate(b: GlobalConfig): Instance[RocketEndpoint] -} - -trait AntCoreFactory { - def instantiate(b: GlobalConfig): Instance[AntEndpoint] -} - -object CoreFactory { - - def rocket(b: GlobalConfig): Instance[RocketEndpoint] = - Class.forName(b.coreDesign.get.factory + "$") - .getField("MODULE$").get(null).asInstanceOf[RocketCoreFactory].instantiate(b) - - def ant(b: GlobalConfig): Instance[AntEndpoint] = - Class.forName(b.coreDesign.get.factory + "$") - .getField("MODULE$").get(null).asInstanceOf[AntCoreFactory].instantiate(b) - -} diff --git a/arch/src/main/scala/framework/system/core/CoreParams.scala b/arch/src/main/scala/framework/system/core/CoreParams.scala deleted file mode 100644 index 9fce8d980..000000000 --- a/arch/src/main/scala/framework/system/core/CoreParams.scala +++ /dev/null @@ -1,15 +0,0 @@ -package framework.system.core - -import framework.ant.{Params => AntParams} -import framework.system.core.rocket.CpuParams -import framework.top.GlobalConfig -import memcore.bus.chi -import memcore.memory.interlock.{Params => TrackingParams} - -case class AntCoreParams( - buckyball: GlobalConfig, - local: AntParams, - tracking: TrackingParams, - bus: chi.Params, - signature: BigInt, - cpu: CpuParams) diff --git a/arch/src/main/scala/framework/system/core/RocketEndpoint.scala b/arch/src/main/scala/framework/system/core/RocketEndpoint.scala deleted file mode 100644 index aa8aa0ac9..000000000 --- a/arch/src/main/scala/framework/system/core/RocketEndpoint.scala +++ /dev/null @@ -1,49 +0,0 @@ -package framework.system.core - -import framework.system.GlobalConfigOps._ -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.core.clink.{CLink, HasCLink, NpuEvents, ShmPort} -import framework.system.core.rocket.CpuParams -import framework.top.GlobalConfig -import framework.memdomain.frontend.mem.dma.DmaPort -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.backend.shared.SharedMemLayout -import hier.core.rocket.{Commands, CpuCLink} -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.cpu.PhysicalRegion - -case class RocketCoreParams( - cpu: CpuParams, - data: RnfParams, - instruction: RnfParams, - regions: Seq[PhysicalRegion], - commands: Commands, - buckyball: Option[GlobalConfig]) - -class AcceleratorCLink(b: GlobalConfig) extends Bundle { - val hartId = Input(UInt(64.W)) - val shmOwner = Input(UInt(64.W)) - val npu = new NpuEvents(b) - val shm = new ShmPort(b) - val mem = new DmaPort(b.memDomain.dma_buswidth) - - val sharedHashes = - if (b.sim.diffTest && b.memDomain.sharedEnable) - Some(Input(Vec(SharedMemLayout.totalBank(b), new PhysicalBankHash(b)))) - else None - -} - -class RocketCLink(p: RocketCoreParams) extends CLink { - val cpu = new CpuCLink(p.data, Some(p.commands))(p.cpu) - val accelerator = p.buckyball.map(new AcceleratorCLink(_)) -} - -/** Reusable implementation; the concrete design and class selection live in examples/cores. */ -@instantiable -abstract class RocketEndpoint(b: framework.top.GlobalConfig) extends Module with HasCLink { - val p = b.rocketParams - @public val clink = IO(new RocketCLink(p)) -} diff --git a/arch/src/main/scala/framework/system/core/accelerator/Admission.scala b/arch/src/main/scala/framework/system/core/accelerator/Admission.scala deleted file mode 100644 index cb5940491..000000000 --- a/arch/src/main/scala/framework/system/core/accelerator/Admission.scala +++ /dev/null @@ -1,420 +0,0 @@ -package framework.system.core.accelerator - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import framework.top.GlobalConfig -import framework.system.core.rocket.{RoCCCommandBB, RoCCIO, RoCCResponseBB} -import framework.frontend.decoder.GISA -import framework.memdomain.frontend.cmd.decoder.DISA -import framework.memdomain.frontend.mem.Footprint -import framework.memdomain.frontend.mem.dma.{Dma, DmaDecision, DmaError, DmaPort, DmaStatus} -import hier.core.rocket.{AdmissionBridge, AdmissionPorts, Preparation, RobFault} -import hier.tile.TaskAdmission -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.interlock.{AccessInfo, Interlock, Params => TrackingParams} -import memcore.memory.preflight.{Command, Error, Params => PreparationParams} -import memcore.bus.axi4 - -/** Per-compute-core admission, physical preparation and completion join. */ -@instantiable -class Admission( - b: GlobalConfig, - tracking: TrackingParams, - prepared: PreparationParams, - regions: Seq[PhysicalRegion], - axiParams: axi4.Params -)( - implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - require(tracking.entries == 4 && tracking.idBits == prepared.idBits) - require(tracking.addressBits == prepared.bus.addressBits && tracking.maxRanges == prepared.maxRanges) - require(prepared.beatBytes == axiParams.bytes && b.memDomain.dma_buswidth == axiParams.dataBits) - val parents = tracking.entries - val n = 8 - val internalTracking = tracking.copy(entries = n) - val internalPrepared = prepared.copy(contexts = n) - val indexBits = log2Ceil(n) - require(tracking.idBits >= indexBits) - - @public - val io = IO(new Bundle { - val core = Flipped(new AdmissionPorts(tracking, nPMPs, prepared.bus)) - val npuCommand = Decoupled(new RoCCCommandBB) - val npuResponse = Flipped(Decoupled(new RoCCResponseBB)) - val allocation = Flipped(Valid(UInt(log2Ceil(b.frontend.rob_entries).W))) - val retired = Input(UInt(b.frontend.rob_entries.W)) - val npuFault = Flipped(Valid(new RobFault(b.frontend.rob_entries))) - // Actual scheduler/boot work, not merely RoCC ROB backpressure. - val npuBusy = Input(Bool()) - val npuInterrupt = Input(Bool()) - val footprints = Input(Vec(3, new Footprint(b))) - val dma = Flipped(new DmaPort(b.memDomain.dma_buswidth)) - val task = Flipped(new RoCCIO(64)) - val taskSatp = Output(UInt(64.W)) - val axi = new axi4.Port(axiParams) - val halted = Output(Bool()) - val fault = Output(new DmaStatus) - val faultTag = Output(UInt(tracking.idBits.W)) - val faultHasTag = Output(Bool()) - val workDrained = Output(Bool()) - }) - - val bridge: Instance[AdmissionBridge] = Instantiate(new AdmissionBridge( - b.frontend.rob_entries, - tracking, - nPMPs, - Seq(GISA.BARRIER_BITPAT.value.toInt) - )) - - val interlock: Instance[Interlock] = Instantiate(new Interlock(internalTracking)) - val preparation: Instance[Preparation] = Instantiate(new Preparation(internalPrepared, regions)) - val dma: Instance[Dma] = Instantiate(new Dma(b, internalPrepared, axiParams)) - val task: Instance[TaskAdmission] = Instantiate(new TaskAdmission(tracking, nPMPs)) - - val live = RegInit(VecInit(Seq.fill(n)(false.B))) - val memory = RegInit(VecInit(Seq.fill(n)(false.B))) - val barrier = RegInit(VecInit(Seq.fill(n)(false.B))) - val retirementSeen = RegInit(VecInit(Seq.fill(n)(false.B))) - val interlockDone = RegInit(VecInit(Seq.fill(n)(false.B))) - val noMemoryPending = RegInit(VecInit(Seq.fill(n)(false.B))) - val shapeClaimed = RegInit(VecInit(Seq.fill(n)(false.B))) - val preparedStarted = RegInit(VecInit(Seq.fill(n)(false.B))) - val mapReady = RegInit(VecInit(Seq.fill(n)(false.B))) - val grantSeen = RegInit(VecInit(Seq.fill(n)(false.B))) - val doneIssued = RegInit(VecInit(Seq.fill(n)(false.B))) - val errors = RegInit(VecInit(Seq.fill(n)(0.U.asTypeOf(new DmaStatus)))) - val halted = RegInit(false.B) - val firstFault = RegInit(0.U.asTypeOf(new DmaStatus)) - val firstTag = RegInit(0.U(tracking.idBits.W)) - val firstHasTag = RegInit(false.B) - val parentTags = Reg(Vec(n, UInt(tracking.idBits.W))) - val childDispatchPending = RegInit(VecInit(Seq.fill(3)(false.B))) - def index(tag: UInt): UInt = tag(indexBits - 1, 0) - def known(tag: UInt): Bool = tag < n.U && live(index(tag)) - - def stop( - error: UInt, - address: UInt, - tag: UInt, - hasTag: Bool - ): Unit = { - when(!halted) { - halted := true.B; firstFault.error := error; firstFault.address := address - firstTag := Mux(tag >= parents.U && known(tag), parentTags(index(tag)), tag) - firstHasTag := hasTag - } - } - - def dmaError(error: UInt): UInt = MuxLookup(error, DmaError.Context.U)(Seq( - Error.Ok.U -> DmaError.None.U, - Error.Shape.U -> DmaError.Shape.U, - Error.Overflow.U -> DmaError.Shape.U, - Error.PageFault.U -> DmaError.PageFault.U, - Error.AccessFault.U -> DmaError.AccessFault.U, - Error.Capacity.U -> DmaError.Capacity.U, - Error.Context.U -> DmaError.Context.U - )) - - val command = io.core.command - val isTask = command.bits.instruction.opcode === "h2b".U - val isBarrier = !isTask && command.bits.instruction.funct === GISA.BARRIER_BITPAT - - val isMemory = !isTask && (command.bits.instruction.funct === DISA.MVIN_BITPAT || - command.bits.instruction.funct === DISA.MVIN_2D_BITPAT || - command.bits.instruction.funct === DISA.MVIN_MMIO_BITPAT || - command.bits.instruction.funct === DISA.MVOUT_BITPAT || - command.bits.instruction.funct === GISA.MVIN_KERNEL_BITPAT) - - val barrierProtect = barrier.asUInt.orR || (command.valid && isBarrier) - - val unknownSubMemory = - if (b.frontend.sub_rob_enable) - VecInit((0 until parents).map(i => live(i) && !memory(i) && !retirementSeen(i) && !barrier(i))).asUInt.orR - else false.B - - val childDispatch = childDispatchPending.asUInt.orR - val dispatchProducer = PriorityEncoder(childDispatchPending) - interlock.io.dispatch.valid := !halted && (childDispatch || - (io.core.reserve.valid && !barrierProtect && !unknownSubMemory)) - interlock.io.dispatch.bits := io.core.reserve.bits - when(childDispatch)(interlock.io.dispatch.bits.id := parents.U + dispatchProducer) - io.core.reserve.ready := interlock.io.dispatch.ready && !halted && - !barrierProtect && !unknownSubMemory && !childDispatch - when(interlock.io.dispatch.fire && childDispatch)(childDispatchPending(dispatchProducer) := false.B) - when(io.core.reserve.fire) { - val tag = io.core.reserve.bits.id - assert(tag < parents.U && !live(index(tag)), "Admission reused a live Core tag") - parentTags(index(tag)) := tag - live(index(tag)) := true.B; memory(index(tag)) := false.B; barrier(index(tag)) := false.B - retirementSeen(index(tag)) := false.B; interlockDone(index(tag)) := false.B - noMemoryPending(index(tag)) := false.B; shapeClaimed(index(tag)) := false.B; preparedStarted(index(tag)) := false.B - mapReady(index(tag)) := false.B; grantSeen(index(tag)) := false.B; doneIssued(index(tag)) := false.B - errors(index(tag)) := 0.U.asTypeOf(new DmaStatus) - } - bridge.io.command.valid := command.valid && !isTask && !halted - bridge.io.command.bits := command.bits - task.io.command.valid := command.valid && isTask && !halted - task.io.command.bits := command.bits - command.ready := !halted && Mux(isTask, task.io.command.ready, bridge.io.command.ready) - when(command.fire) { - assert(command.bits.tag < parents.U && known(command.bits.tag), "Admission command has no reserved Core tag") - val slot = index(command.bits.tag) - memory(slot) := isMemory; barrier(slot) := isBarrier - noMemoryPending(slot) := !isMemory - } - io.npuCommand <> bridge.io.npuCommand - bridge.io.allocation := io.allocation; bridge.io.retired := io.retired; bridge.io.fault := io.npuFault - io.task <> task.io.task; io.taskSatp := task.io.satp - val responses = Module(new Arbiter(new RoCCResponseBB, 2)) - responses.io.in(0) <> task.io.response; responses.io.in(1) <> io.npuResponse - io.core.response <> responses.io.out - io.core.interrupt := io.npuInterrupt || io.task.interrupt - interlock.io.cpuQuery := io.core.cpuQuery - io.core.cpuAllow := interlock.io.cpuAllow && !halted && !barrierProtect && !unknownSubMemory - io.core.cpuProbeAllow := interlock.io.cpuProbeAllow && !halted && !barrierProtect && !unknownSubMemory - io.core.maintenance <> interlock.io.maintenance - interlock.io.maintained <> io.core.maintained - io.core.pteRequest <> preparation.io.pteRequest - preparation.io.pteResponse <> io.core.pteResponse - interlock.io.cancel.valid := false.B; interlock.io.cancel.bits := 0.U.asTypeOf(interlock.io.cancel.bits) - io.core.cancelled.valid := false.B; io.core.cancelled.bits := 0.U.asTypeOf(io.core.cancelled.bits) - - // Notices never wait for other tags: otherwise a high-priority barrier notice - // could prevent older ROB retirements from reaching the completion join. - bridge.io.retirement.ready := true.B - when(bridge.io.retirement.fire) { - val notice = bridge.io.retirement.bits - assert(known(notice.snapshot.tag), "Admission retirement has no live Core tag") - retirementSeen(index(notice.snapshot.tag)) := true.B - when(notice.error =/= DmaError.None.U) { - errors(index(notice.snapshot.tag)).error := notice.error - errors(index(notice.snapshot.tag)).address := notice.address - stop(notice.error, notice.address, notice.snapshot.tag, true.B) - } - } - task.io.release.ready := true.B - when(task.io.release.fire) { - assert(known(task.io.release.bits.tag), "Task release has no live Core tag") - retirementSeen(index(task.io.release.bits.tag)) := true.B - } - interlock.io.complete.ready := true.B - when(interlock.io.complete.fire) { - assert(known(interlock.io.complete.bits.tag), "Interlock complete has no live Core tag") - interlockDone(index(interlock.io.complete.bits.tag)) := true.B - } - when(bridge.io.unboundFault.valid) { - stop(bridge.io.unboundFault.bits.error, bridge.io.unboundFault.bits.address, 0.U, false.B) - } - - val producerBound = RegInit(VecInit(Seq.fill(3)(false.B))) - val producerTag = Reg(Vec(3, UInt(tracking.idBits.W))) - val producerSub = RegInit(VecInit(Seq.fill(3)(false.B))) - val producerRob = Reg(Vec(3, UInt(log2Ceil(b.frontend.rob_entries).W))) - val producerSubRob = Reg(Vec(3, UInt(log2Ceil(b.frontend.sub_rob_depth * 4).W))) - val producerMatches = Wire(Vec(3, Bool())) - val shapes = Reg(Vec(3, new Command(prepared))) - val shapePending = RegInit(VecInit(Seq.fill(3)(false.B))) - for (i <- 0 until 3) { - bridge.io.lookup(i).robId := io.footprints(i).rob_id - val footprint = io.footprints(i) - val snapshot = bridge.io.lookup(i).snapshot - val childTag = (parents + i).U(tracking.idBits.W) - producerMatches(i) := footprint.valid && footprint.rob_id === producerRob(i) && - footprint.is_sub === producerSub(i) && (!producerSub(i) || footprint.sub_rob_id === producerSubRob(i)) - when(producerBound(i) && !producerMatches(i)) { - when(producerSub(i))(retirementSeen(parents + i) := true.B) - .otherwise(producerBound(i) := false.B) - } - when(footprint.valid && !producerBound(i) && (!footprint.is_sub || !live(parents + i)) && - !halted && !io.npuFault.valid) { - when(!snapshot.valid || !known(snapshot.bits.tag) || - (!footprint.is_sub && !memory(index(snapshot.bits.tag))) || - (footprint.is_sub && !b.frontend.sub_rob_enable.B)) { - stop(DmaError.Context.U, footprint.baseVA, snapshot.bits.tag, snapshot.valid) - }.otherwise { - val tag = Mux(footprint.is_sub, childTag, snapshot.bits.tag); val slot = index(tag) - when(footprint.is_sub) { - live(slot) := true.B; memory(slot) := true.B; barrier(slot) := false.B - parentTags(slot) := snapshot.bits.tag - retirementSeen(slot) := false.B; interlockDone(slot) := false.B - noMemoryPending(slot) := false.B; preparedStarted(slot) := false.B - mapReady(slot) := false.B; grantSeen(slot) := false.B; doneIssued(slot) := false.B - errors(slot) := 0.U.asTypeOf(new DmaStatus) - childDispatchPending(i) := true.B - } - producerSub(i) := footprint.is_sub - producerRob(i) := footprint.rob_id; producerSubRob(i) := footprint.sub_rob_id - val narrow = footprint.rows(63, 32).orR || footprint.columns(63, 32).orR || - footprint.spanBytes(63, 32).orR || footprint.columnStride(63, 32).orR || footprint.rowStride(63, 32).orR - producerBound(i) := true.B; producerTag(i) := tag - shapes(i).id := tag; shapes(i).baseVA := footprint.baseVA - shapes(i).rows := footprint.rows(31, 0); shapes(i).columns := footprint.columns(31, 0) - shapes(i).spanBytes := footprint.spanBytes(31, 0); shapes(i).columnStride := footprint.columnStride(31, 0) - shapes(i).rowStride := footprint.rowStride(31, 0); shapes(i).write := footprint.write - shapes(i).mode := snapshot.bits.satp(63, 60); shapes(i).rootPpn := snapshot.bits.satp(43, 0) - shapes(i).privilege := snapshot.bits.effectivePrivilege - shapes(i).sum := snapshot.bits.sum; shapes(i).mxr := snapshot.bits.mxr - when(narrow || footprint.fault.error =/= DmaError.None.U) { - val error = Mux(narrow, DmaError.Shape.U, footprint.fault.error) - errors(slot).error := error - errors(slot).address := Mux(narrow, footprint.baseVA, footprint.fault.address) - stop(error, Mux(narrow, footprint.baseVA, footprint.fault.address), snapshot.bits.tag, true.B) - }.otherwise { - when(!footprint.is_sub && shapeClaimed(slot)) { - errors(slot).error := DmaError.Context.U; errors(slot).address := footprint.baseVA - stop(DmaError.Context.U, footprint.baseVA, tag, true.B) - }.otherwise { shapeClaimed(slot) := true.B; shapePending(i) := true.B } - } - } - } - } - val prepareOffer = RegInit(false.B) - val prepareProducer = Reg(UInt(2.W)) - when(!prepareOffer && shapePending.asUInt.orR && !halted) { - prepareOffer := true.B; prepareProducer := PriorityEncoder(shapePending) - } - preparation.io.command.valid := prepareOffer && !halted - preparation.io.command.bits := shapes(prepareProducer) - when(preparation.io.command.fire) { - prepareOffer := false.B; shapePending(prepareProducer) := false.B - preparedStarted(index(shapes(prepareProducer).id)) := true.B - } - val preparationTag = preparation.io.contextTag - bridge.io.lookupTag.tag := Mux(preparationTag >= parents.U, parentTags(index(preparationTag)), preparationTag) - preparation.io.contextValid := bridge.io.lookupTag.snapshot.valid - preparation.io.contextId := preparationTag - preparation.io.contextPmp := bridge.io.lookupTag.snapshot.bits.pmp - - val infos = Module(new Queue(new AccessInfo(internalTracking), 2)) - interlock.io.accessInfo <> infos.io.deq - val range = preparation.io.ranges - val infoOffer = RegInit(false.B) - val infoRange = RegInit(false.B) - val infoPacket = Reg(new AccessInfo(internalTracking)) - val pure = PriorityEncoder(noMemoryPending) - when(!infoOffer && !halted && - ((range.valid && range.bits.error === Error.Ok.U) || (!range.valid && noMemoryPending.asUInt.orR))) { - infoOffer := true.B; infoRange := range.valid && range.bits.error === Error.Ok.U - infoPacket := 0.U.asTypeOf(infoPacket) - infoPacket.id := Mux(range.valid, range.bits.id, pure) - infoPacket.hasMemory := range.valid - infoPacket.base := Mux(range.valid, range.bits.pa, 0.U) - infoPacket.bytes := Mux(range.valid, range.bits.bytes, 0.U) - infoPacket.write := range.valid && range.bits.write - infoPacket.last := Mux(range.valid, range.bits.last, true.B) - } - infos.io.enq.valid := infoOffer && !halted - infos.io.enq.bits := infoPacket - range.ready := Mux(range.bits.error =/= Error.Ok.U, true.B, infoOffer && infoRange && infos.io.enq.ready && !halted) - when(infos.io.enq.fire) { - infoOffer := false.B - when(!infoRange)(noMemoryPending(index(infoPacket.id)) := false.B) - } - when(range.fire && range.bits.error =/= Error.Ok.U) { - val error = dmaError(range.bits.error) - errors(index(range.bits.id)).error := error; errors(index(range.bits.id)).address := range.bits.va - stop(error, range.bits.va, range.bits.id, true.B) - } - preparation.io.ready.ready := true.B - when(preparation.io.ready.fire) { - val value = preparation.io.ready.bits - mapReady(index(value.id)) := true.B - when(value.error =/= Error.Ok.U) { - errors(index(value.id)).error := dmaError(value.error); errors(index(value.id)).address := value.va - stop(dmaError(value.error), value.va, value.id, true.B) - } - } - interlock.io.grant.ready := true.B - when(interlock.io.grant.fire)(grantSeen(index(interlock.io.grant.bits.tag)) := true.B) - - preparation.io.queries := dma.io.queries - for (i <- 0 until 2) { - dma.io.mappings(i) := preparation.io.results(i) - when(halted) { - dma.io.mappings(i).hit := false.B; dma.io.mappings(i).pa := 0.U - dma.io.mappings(i).error := Error.Context.U - } - } - when(io.npuFault.valid && io.npuFault.bits.error =/= DmaError.None.U) { - stop(io.npuFault.bits.error, io.npuFault.bits.address, bridge.io.faultTag.bits, bridge.io.faultTag.valid) - } - val readOwnerTag = Reg(UInt(tracking.idBits.W)) - val writeOwnerTag = Reg(UInt(tracking.idBits.W)) - when(io.dma.read.fire)(readOwnerTag := Mux(io.dma.read.bits.producer === 2.U, producerTag(2), producerTag(0))) - when(io.dma.write.fire)(writeOwnerTag := producerTag(1)) - dma.io.dma <> io.dma - io.axi <> dma.io.axi - for (i <- 0 until 3) { - val tag = producerTag(i); val slot = index(tag) - dma.io.decisions(i) := 0.U.asTypeOf(new DmaDecision(prepared)) - dma.io.decisions(i).parentId := tag - dma.io.decisions(i).valid := producerBound(i) && producerMatches(i) && - (!producerSub(i) || !retirementSeen(slot)) && - ((mapReady(slot) && grantSeen(slot)) || errors(slot).error =/= DmaError.None.U || halted) - dma.io.decisions(i).fault := Mux(halted, firstFault, errors(slot)) - } - when(io.dma.readResult.fire && io.dma.readResult.bits.fault.error =/= DmaError.None.U) { - stop(io.dma.readResult.bits.fault.error, io.dma.readResult.bits.fault.address, readOwnerTag, true.B) - } - when(io.dma.writeResult.fire && io.dma.writeResult.bits.fault.error =/= DmaError.None.U) { - stop(io.dma.writeResult.bits.fault.error, io.dma.writeResult.bits.fault.address, writeOwnerTag, true.B) - } - - val doneCandidates = VecInit((0 until n).map(i => - live(i) && memory(i) && retirementSeen(i) && - mapReady(i) && grantSeen(i) && !doneIssued(i) && errors(i).error === DmaError.None.U - )) - - val doneOffer = RegInit(false.B) - val doneTag = Reg(UInt(indexBits.W)) - when(!doneOffer && doneCandidates.asUInt.orR && !halted) { - doneOffer := true.B; doneTag := PriorityEncoder(doneCandidates) - } - interlock.io.done.valid := doneOffer && !halted - interlock.io.done.bits.tag := doneTag; interlock.io.done.bits.ok := true.B - when(interlock.io.done.fire) { doneIssued(doneTag) := true.B; doneOffer := false.B } - - val releasable = VecInit((0 until n).map(i => - live(i) && retirementSeen(i) && interlockDone(i) && - errors(i).error === DmaError.None.U && (!barrier(i) || - (PopCount(live) === 1.U && !io.npuBusy && !io.dma.readBusy && !io.dma.writeBusy)) && - (if (i < parents) !VecInit((parents until parents + 3).map(j => live(j) && parentTags(j) === i.U)).asUInt.orR - else true.B) - )) - - val releaseOffer = RegInit(false.B) - val releaseTag = Reg(UInt(indexBits.W)) - when(!releaseOffer && releasable.asUInt.orR && !halted) { - releaseOffer := true.B; releaseTag := PriorityEncoder(releasable) - } - val releasingMap = memory(releaseTag) && preparedStarted(releaseTag) - val releasingChild = releaseTag >= parents.U - val releaseReady = releasingChild || io.core.complete.ready - preparation.io.release.valid := releaseOffer && releasingMap && releaseReady && !halted - preparation.io.release.bits.id := releaseTag - io.core.complete.valid := releaseOffer && !releasingChild && !halted && (!releasingMap || preparation.io.release.ready) - io.core.complete.bits.tag := releaseTag - val releaseFire = releaseOffer && releaseReady && !halted && (!releasingMap || preparation.io.release.ready) - when(releaseFire) { - assert(!releasingMap || preparation.io.release.fire, "Admission map/Core release is not atomic") - live(releaseTag) := false.B; barrier(releaseTag) := false.B; releaseOffer := false.B - for (i <- 0 until 3) { - when(releaseTag === (parents + i).U)(producerBound(i) := false.B) - } - } - val otherLive = (live.asUInt & ~(1.U(n.W) << index(command.bits.tag))).orR - - val drained = !otherLive && !io.npuBusy && !io.dma.readBusy && !io.dma.writeBusy && - !prepareOffer && !shapePending.asUInt.orR && - !interlock.io.maintenance.valid && !halted - - task.io.workDrained := drained - io.workDrained := !live.asUInt.orR && !io.npuBusy && !io.dma.readBusy && !io.dma.writeBusy && - !halted - io.halted := halted; io.fault := firstFault; io.faultTag := firstTag; io.faultHasTag := firstHasTag -} diff --git a/arch/src/main/scala/framework/system/core/accelerator/AntAdmission.scala b/arch/src/main/scala/framework/system/core/accelerator/AntAdmission.scala deleted file mode 100644 index 46188b2c4..000000000 --- a/arch/src/main/scala/framework/system/core/accelerator/AntAdmission.scala +++ /dev/null @@ -1,147 +0,0 @@ -package framework.system.core.accelerator - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.ant -import framework.system.core.rocket.CpuParams -import hier.core.rocket.{AdmissionPorts, CommandSnapshot} -import memcore.bus.chi -import memcore.bus.chi.rnf.{CacheAccess, CacheResult} -import memcore.memory.interlock.{Acknowledgement, CpuQuery, Maintenance, Params => TrackingParams} - -/** Controller-side cache services; requests retain their original admission tag. */ -class AntMemoryPort(tracking: TrackingParams, bus: chi.Params) extends Bundle { - val cpuQuery = Input(new CpuQuery(tracking)) - val cpuAllow = Output(Bool()) - val cpuProbeAllow = Output(Bool()) - val maintenance = Decoupled(new Maintenance(tracking)) - val maintained = Flipped(Decoupled(new Acknowledgement(tracking))) - val pteRequest = Decoupled(new CacheAccess(bus)) - val pteResponse = Flipped(Decoupled(new CacheResult)) -} - -/** Binds local Ant execution to the controller's immutable task address-space snapshot. */ -@instantiable -class AntAdmission( - p: ant.Params, - tracking: TrackingParams, - pmps: Int, - bus: chi.Params -)( - implicit val cpuParams: CpuParams) - extends Module { - require(tracking.entries == 4) - - @public val io = IO(new Bundle { - - val bind = Flipped(Valid(new Bundle { - val task = UInt(p.taskBits.W) - val snapshot = new CommandSnapshot(tracking, pmps) - })) - - val command = Flipped(Decoupled(new ant.Command(p))) - val response = Decoupled(new ant.Response(p)) - val cancel = Input(Bool()) - val workDrained = Input(Bool()) - val halted = Input(Bool()) - val drained = Output(Bool()) - val admission = new AdmissionPorts(tracking, pmps, bus) - val memory = new AntMemoryPort(tracking, bus) - }) - - val bound = RegInit(false.B) - val task = Reg(UInt(p.taskBits.W)) - val context = Reg(new CommandSnapshot(tracking, pmps)) - val live = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) - val sent = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) - val pending = RegInit(false.B) - val packet = Reg(new CommandSnapshot(tracking, pmps)) - val responsePending = RegInit(false.B) - val responseRd = Reg(UInt(5.W)) - val responseDelivered = RegInit(false.B) - val free = VecInit(live.map(!_)).asUInt - val selected = PriorityEncoder(free) - val port = io.admission - io.drained := !live.asUInt.orR && !pending && !responsePending && io.workDrained - when(io.bind.valid) { - assert(io.drained && !io.command.valid, "Ant context rebound before previous task drained") - bound := true.B - task := io.bind.bits.task - context := io.bind.bits.snapshot - } - val capacity = bound && free.orR && !pending && !responsePending && !io.cancel && !io.bind.valid && !reset.asBool - port.reserve.valid := io.command.valid && capacity - port.reserve.bits.id := selected - io.command.ready := capacity && port.reserve.ready - port.command.valid := pending - port.command.bits := packet - when(io.command.fire) { - val instruction = io.command.bits.instruction - assert(io.command.bits.task === task, "Ant command uses another task context") - assert(instruction(6, 0) === "h7b".U, "Ant admission only accepts NPU commands") - packet := context - packet.tag := selected - packet.instruction.raw_inst := instruction - packet.instruction.pc := io.command.bits.pc - packet.instruction.funct := instruction(31, 25) - packet.instruction.funct3 := instruction(14, 12) - packet.instruction.rs2 := instruction(24, 20) - packet.instruction.rs1 := instruction(19, 15) - packet.instruction.xd := instruction(14) - packet.instruction.xs1 := instruction(13) - packet.instruction.xs2 := instruction(12) - packet.instruction.rd := instruction(11, 7) - packet.instruction.opcode := instruction(6, 0) - packet.instruction.rs1Data := io.command.bits.rs1 - packet.instruction.rs2Data := io.command.bits.rs2 - live(selected) := true.B - sent(selected) := false.B - pending := true.B - when(instruction(14)) { - responsePending := true.B - responseDelivered := false.B - responseRd := instruction(11, 7) - } - } - when(port.command.fire) { - pending := false.B - sent(packet.tag(1, 0)) := true.B - when(packet.instruction.xd)(responseDelivered := true.B) - } - port.complete.ready := true.B - when(port.complete.fire) { - val tag = port.complete.bits.tag - assert( - tag < tracking.entries.U && live(tag(1, 0)) && sent(tag(1, 0)), - "Ant completion has no delivered live command" - ) - live(tag(1, 0)) := false.B - sent(tag(1, 0)) := false.B - } - port.cancelled.ready := true.B - assert(!port.cancelled.valid, "Ant cancellation must drain accepted NPU commands") - assert(!io.halted && !port.interrupt, "Ant NPU backend fault") - port.outstanding := PopCount(live) - io.response.valid := port.response.valid && responsePending - io.response.bits.task := task - io.response.bits.rd := port.response.bits.rd - io.response.bits.data := port.response.bits.data - io.response.bits.error := false.B - port.response.ready := io.response.ready && responsePending - when(port.response.valid) { - assert( - responsePending && (responseDelivered || (port.command.fire && packet.instruction.xd)), - "Ant received an unsolicited NPU response" - ) - assert(port.response.bits.rd === responseRd, "Ant NPU response destination changed") - } - when(port.response.fire)(responsePending := false.B) - port.cpuQuery := io.memory.cpuQuery - io.memory.cpuAllow := port.cpuAllow - io.memory.cpuProbeAllow := port.cpuProbeAllow - io.memory.maintenance <> port.maintenance - port.maintained <> io.memory.maintained - io.memory.pteRequest <> port.pteRequest - port.pteResponse <> io.memory.pteResponse -} diff --git a/arch/src/main/scala/framework/system/core/accelerator/AntMemory.scala b/arch/src/main/scala/framework/system/core/accelerator/AntMemory.scala deleted file mode 100644 index a374fd1ba..000000000 --- a/arch/src/main/scala/framework/system/core/accelerator/AntMemory.scala +++ /dev/null @@ -1,73 +0,0 @@ -package framework.system.core.accelerator - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.chi -import memcore.memory.interlock.{Params => TrackingParams} - -/** Independent fair, single-owner cache-maintenance and page-table service arbiters. */ -@instantiable -class AntMemory(ports: Int, tracking: TrackingParams, bus: chi.Params) extends Module { - require(ports > 0) - - @public val io = IO(new Bundle { - val clients = Vec(ports, Flipped(new AntMemoryPort(tracking, bus))) - val controller = new AntMemoryPort(tracking, bus) - }) - - io.controller.cpuAllow := io.clients.map(_.cpuAllow).reduce(_ && _) - io.controller.cpuProbeAllow := io.clients.map(_.cpuProbeAllow).reduce(_ && _) - io.clients.foreach(_.cpuQuery := io.controller.cpuQuery) - - def arbitrate[T <: Data, R <: Data]( - requests: Seq[DecoupledIO[T]], - responses: Seq[DecoupledIO[R]], - request: DecoupledIO[T], - response: DecoupledIO[R] - ): Unit = { - val ownerBits = math.max(1, log2Ceil(ports)) - val next = RegInit(0.U(ownerBits.W)) - val owner = Reg(UInt(ownerBits.W)) - val occupied = RegInit(false.B) - val sent = RegInit(false.B) - val packet = Reg(chiselTypeOf(request.bits)) - val candidates = VecInit(requests.map(_.valid)).asUInt - val later = VecInit(requests.zipWithIndex.map { case (r, i) => r.valid && i.U >= next }).asUInt - val selected = PriorityEncoder(Mux(later.orR, later, candidates)) - for (i <- 0 until ports) { - requests(i).ready := !occupied && selected === i.U && !reset.asBool - responses(i).valid := occupied && sent && owner === i.U && response.valid - responses(i).bits := response.bits - when(requests(i).fire) { - packet := requests(i).bits - owner := i.U - occupied := true.B - sent := false.B - next := (if (i + 1 == ports) 0 else i + 1).U - } - } - request.valid := occupied && !sent && !reset.asBool - request.bits := packet - when(request.fire)(sent := true.B) - response.ready := occupied && sent && VecInit(responses.map(_.ready))(owner) && !reset.asBool - when(response.valid)(assert(occupied && sent, "Ant cache service returned a response without a sent owner")) - when(response.fire) { - occupied := false.B - sent := false.B - } - } - - arbitrate( - io.clients.map(_.maintenance), - io.clients.map(_.maintained), - io.controller.maintenance, - io.controller.maintained - ) - arbitrate( - io.clients.map(_.pteRequest), - io.clients.map(_.pteResponse), - io.controller.pteRequest, - io.controller.pteResponse - ) -} diff --git a/arch/src/main/scala/framework/system/core/accelerator/BuckyballAccelerator.scala b/arch/src/main/scala/framework/system/core/accelerator/BuckyballAccelerator.scala deleted file mode 100644 index 186a95d42..000000000 --- a/arch/src/main/scala/framework/system/core/accelerator/BuckyballAccelerator.scala +++ /dev/null @@ -1,219 +0,0 @@ -package framework.system.core.accelerator - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} - -import framework.top.GlobalConfig -import framework.frontend.Frontend -import framework.frontend.globalrs.RobAllocation -import hier.core.rocket.RobFault -import framework.system.core.rocket.{RoCCCommandBB, RoCCResponseBB} -import framework.memdomain.MemDomain -import framework.memdomain.backend.MemRequestIO -import framework.memdomain.backend.shared.SharedMemLayout -import framework.memdomain.frontend.mem.{Footprint, MemConfigerIO} -import framework.memdomain.frontend.mem.dma.DmaPort -import framework.balldomain.BallDomain -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.isa.{MvoverISA, MvoverPort} -import memcore.memory.mesh_shm.MeshLocalBankPort - -/** Accelerator domains with an explicit DMA transport boundary owned by the enclosing system. */ -@instantiable -class BuckyballAccelerator(val b: GlobalConfig) extends Module { - val totalBallRead = b.ballDomain.ballIdMappings.map(_.inBW).sum - val totalBallWrite = b.ballDomain.ballIdMappings.map(_.outBW).sum - val sharedHashCount = if (b.memDomain.sharedEnable) SharedMemLayout.totalBank(b) else 0 - - @public - val io = IO(new Bundle { - // RoCC command/response (connected to Rocket core inside tile) - val cmd = Flipped(Decoupled(new RoCCCommandBB(b.tile.xLen))) - val resp = Decoupled(new RoCCResponseBB(b.tile.xLen)) - val busy = Output(Bool()) - val idle = Output(Bool()) - val interrupt = Output(Bool()) - val hartid = Input(UInt(b.tile.xLen.W)) - val sharedBankOwnerHartId = Input(UInt(b.tile.xLen.W)) - // Includes boot allocations; external context binds only on io.cmd.fire. - val allocation = Valid(new RobAllocation(b)) - val retired = Output(UInt(b.frontend.rob_entries.W)) - val fault = Valid(new RobFault(b.frontend.rob_entries)) - val firstFault = Output(Valid(new RobFault(b.frontend.rob_entries))) - val footprints = Output(Vec(3, new Footprint(b))) - - val dma = new DmaPort(b.memDomain.dma_buswidth) - - // Shared memory path — exposed to tile level for multi-core SharedMemBackend - val shared_mem_req = Vec(SharedMemLayout.channelPerHart(b), new MemRequestIO(b)) - val mvover = new MvoverPort - - val meshLocalBank = Flipped(new MeshLocalBankPort( - MvoverISA.AddressBits, - MvoverISA.BankBits, - b.memDomain.bankWidth, - math.max(1, log2Ceil(b.frontend.rob_entries)) - )) - - val shared_config = Decoupled(new MemConfigerIO(b)) - val shared_query_valid = Output(Bool()) - val shared_query_vbank_id = Output(UInt(b.memDomain.vbankIdWidth.W)) - val shared_query_group_count = Input(UInt(b.memDomain.groupCountWidth.W)) - - val shared_bank_hashes = - if (b.sim.diffTest && b.memDomain.sharedEnable) { - Some(Input(Vec(sharedHashCount, new PhysicalBankHash(b)))) - } else { - None - } - - // Barrier interface — connected to tile-level BarrierUnit - val barrier_arrive = Output(Bool()) - val barrier_release = Input(Bool()) - }) - - // --- Instantiate domains --- - val frontend: Instance[Frontend] = Instantiate(new Frontend(b)) - val ballDomain: Instance[BallDomain] = Instantiate(new BallDomain(b)) - val memDomain: Instance[MemDomain] = Instantiate(new MemDomain(b)) - val rvv = if (b.rvv.enable) Some(Instantiate(new Rvv(b))) else None - frontend.io.hartid := io.hartid - frontend.io.sharedBankOwnerHartId := io.sharedBankOwnerHartId - frontend.io.bank_hashes.foreach(_ := memDomain.io.bank_hashes.get) - memDomain.io.shared_bank_hashes.foreach(_ := io.shared_bank_hashes.get) - io.allocation := frontend.io.allocation - io.retired := frontend.io.retired - io.footprints := memDomain.io.footprints - - // --- Frontend <- cmd --- - frontend.io.cmd.valid := io.cmd.valid - frontend.io.cmd.bits.cmd := io.cmd.bits - io.cmd.ready := frontend.io.cmd.ready - - // --- Frontend -> BallDomain --- - ballDomain.global_issue_i <> frontend.io.ball_issue_o - frontend.io.ball_complete_i <> ballDomain.global_complete_o - - // --- BallDomain -> Frontend (SubROB requests) --- - for (i <- 0 until b.ballDomain.ballNum) { - frontend.io.ball_subrob_req_i(i) <> ballDomain.subRobReq(i) - } - - memDomain.io.ballChannelActive := ballDomain.ballChannelActive - ballDomain.ballChannelReady := memDomain.io.ballChannelReady - - // --- Frontend -> MemDomain --- - memDomain.io.global_issue_i <> frontend.io.mem_issue_o - frontend.io.mem_complete_i <> memDomain.io.global_complete_o - // Observe the accepted completion before the scheduler's ID-only queue. - // Its arbiter accepts at most one of these domain completions per cycle. - val memoryFault = memDomain.io.global_complete_o.fire && memDomain.io.global_complete_o.bits.fault.error =/= 0.U - val ballFault = ballDomain.global_complete_o.fire && ballDomain.global_complete_o.bits.fault.error =/= 0.U - val rvvFault = if (b.rvv.enable) rvv.get.complete.fire && rvv.get.complete.bits.fault.error =/= 0.U else false.B - - val faultCompletion = Mux( - memoryFault, - memDomain.io.global_complete_o.bits, - if (b.rvv.enable) Mux(rvvFault, rvv.get.complete.bits, ballDomain.global_complete_o.bits) - else ballDomain.global_complete_o.bits - ) - - io.fault.valid := memoryFault || ballFault || rvvFault - io.fault.bits.rob_id := faultCompletion.rob_id - io.fault.bits.error := faultCompletion.fault.error - io.fault.bits.address := faultCompletion.fault.address - val firstFault = RegInit(0.U.asTypeOf(Valid(new RobFault(b.frontend.rob_entries)))) - when(io.fault.valid && !firstFault.valid)(firstFault := io.fault) - io.firstFault := firstFault - memDomain.io.hartid := io.hartid - memDomain.io.inst_ids := frontend.io.inst_ids - - if (b.rvv.enable) { - val controller = rvv.get - controller.command <> frontend.io.kernel_issue_o.get - frontend.io.kernel_complete_i.get <> controller.complete - controller.memory <> memDomain.io.kernel.get - ballDomain.internalCommand.get <> controller.ballRequest - controller.ballResponse <> ballDomain.internalComplete.get - ballDomain.ownerRobId.get := controller.ownerRobId - ballDomain.kernelBanks.get := controller.banks - } - - // --- BallDomain <-> MemDomain (bankRead with pipeline register to break comb loops) --- - for (i <- 0 until totalBallRead) { - val bankReadReqWithIds = Wire(Decoupled(new Bundle { - val bank_id = chiselTypeOf(ballDomain.bankRead(i).bank_id) - val rob_id = chiselTypeOf(ballDomain.bankRead(i).rob_id) - val ball_id = chiselTypeOf(ballDomain.bankRead(i).ball_id) - val group_id = chiselTypeOf(ballDomain.bankRead(i).group_id) - val req = chiselTypeOf(ballDomain.bankRead(i).io.req.bits) - })) - - bankReadReqWithIds.valid := ballDomain.bankRead(i).io.req.valid - bankReadReqWithIds.bits.bank_id := ballDomain.bankRead(i).bank_id - bankReadReqWithIds.bits.rob_id := ballDomain.bankRead(i).rob_id - bankReadReqWithIds.bits.ball_id := ballDomain.bankRead(i).ball_id - bankReadReqWithIds.bits.group_id := ballDomain.bankRead(i).group_id - bankReadReqWithIds.bits.req := ballDomain.bankRead(i).io.req.bits - ballDomain.bankRead(i).io.req.ready := bankReadReqWithIds.ready - - val bankReadReqQueue = Module(new Queue(chiselTypeOf(bankReadReqWithIds.bits), 8)) - bankReadReqQueue.io.enq <> bankReadReqWithIds - val bankReadReqQ = bankReadReqQueue.io.deq - - memDomain.io.ballDomain.bankRead(i).io.req.valid := bankReadReqQ.valid - memDomain.io.ballDomain.bankRead(i).io.req.bits := bankReadReqQ.bits.req - memDomain.io.ballDomain.bankRead(i).bank_id := bankReadReqQ.bits.bank_id - memDomain.io.ballDomain.bankRead(i).rob_id := bankReadReqQ.bits.rob_id - memDomain.io.ballDomain.bankRead(i).ball_id := bankReadReqQ.bits.ball_id - memDomain.io.ballDomain.bankRead(i).group_id := bankReadReqQ.bits.group_id - bankReadReqQ.ready := memDomain.io.ballDomain.bankRead(i).io.req.ready - - ballDomain.bankRead(i).io.resp <> memDomain.io.ballDomain.bankRead(i).io.resp - } - - ballDomain.bankWrite <> memDomain.io.ballDomain.bankWrite - - // --- BallDomain <-> MemDomain (MMIO read path) --- - ballDomain.mmioRead <> memDomain.io.ballDomain.mmioRead - ballDomain.mmioWrite <> memDomain.io.ballDomain.mmioWrite - - io.dma <> memDomain.io.dma - - // --- Shared memory passthrough --- - io.shared_mem_req <> memDomain.io.shared_mem_req - io.mvover <> memDomain.io.mvover - io.meshLocalBank <> memDomain.io.meshLocalBank - io.shared_config <> memDomain.io.shared_config - io.shared_query_valid := memDomain.io.shared_query_valid - io.shared_query_vbank_id := memDomain.io.shared_query_vbank_id - memDomain.io.shared_query_group_count := io.shared_query_group_count - - // --- Barrier passthrough --- - io.barrier_arrive := frontend.io.barrier_arrive - frontend.io.barrier_release := io.barrier_release - - // --- Response & status --- - io.resp <> frontend.io.resp - io.busy := frontend.io.busy - io.idle := frontend.io.idle && !io.dma.readBusy && !io.dma.writeBusy - // Typed completion failures are delivered through the admission fault path. - // firstFault is retained diagnostic state, not an additional sticky IRQ. - io.interrupt := (if (b.rvv.enable) rvv.get.fault else false.B) - - // --- Busy watchdog --- - // BootRom clears and initializes the local memories before the external - // command interface becomes ready. That initialization can legitimately - // exceed the runtime watchdog limit, so only start monitoring after the - // first external command has handshaken. - val runtime_started = RegInit(false.B) - when(io.cmd.fire) { - runtime_started := true.B - } - val busy_counter = RegInit(0.U(32.W)) - busy_counter := Mux(runtime_started && frontend.io.busy, busy_counter + 1.U, 0.U) - assert(busy_counter < 10000000.U, "BuckyballAccelerator: busy for too long!") -} diff --git a/arch/src/main/scala/framework/system/core/accelerator/Rvv.scala b/arch/src/main/scala/framework/system/core/accelerator/Rvv.scala deleted file mode 100644 index a7e685f78..000000000 --- a/arch/src/main/scala/framework/system/core/accelerator/Rvv.scala +++ /dev/null @@ -1,120 +0,0 @@ -package framework.system.core.accelerator - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.rvv.{KernelEngine, RvvBallCommand} - -import framework.frontend.globalrs.{GlobalSchedComplete, GlobalSchedIssue} -import framework.memdomain.frontend.mem.{KernelDmaRequest, KernelMemoryBridge} - -import framework.memdomain.frontend.mem.dma.{DmaError, DmaStatus} - -@instantiable -class Rvv(val b: GlobalConfig) extends Module { - require(b.rvv.enable) - @public val command = IO(Flipped(Decoupled(new GlobalSchedIssue(b)))) - @public val complete = IO(Decoupled(new GlobalSchedComplete(b))) - @public val memory = IO(Flipped(new KernelMemoryBridge(b))) - @public val fault = IO(Output(Bool())) - @public val ballRequest = IO(Decoupled(new RvvBallCommand)) - @public val ballResponse = IO(Flipped(Decoupled(UInt(64.W)))) - @public val ownerRobId = IO(Output(UInt(log2Up(b.frontend.rob_entries).W))) - @public val banks = IO(Output(new framework.rvv.BankLayout(b))) - @public val busy = IO(Output(Bool())) - - val engine: Instance[KernelEngine] = Instantiate(new KernelEngine(b)) - val loadOwner = RegInit(false.B) - val loadPending = RegInit(false.B) - val load = Reg(new KernelDmaRequest(b)) - val loadAccepted = RegInit(false.B) - val terminalSeen = RegInit(false.B) - val terminalStatus = Reg(new DmaStatus) - val failed = RegInit(false.B) - val runOwner = RegInit(false.B) - val isLoad = command.bits.cmd.cmd.funct === 44.U - engine.io.ballRequest <> ballRequest - engine.io.ballResponse <> ballResponse - engine.io.cmdReq.valid := command.valid - engine.io.cmdReq.bits.cmd.funct7 := command.bits.cmd.cmd.funct - engine.io.cmdReq.bits.cmd.rs1 := command.bits.cmd.cmd.rs1Data - engine.io.cmdReq.bits.cmd.rs2 := command.bits.cmd.cmd.rs2Data - engine.io.cmdReq.bits.rob_id := command.bits.rob_id - engine.io.cmdReq.bits.read_groups := command.bits.cmd.op1_col - engine.io.cmdReq.bits.write_groups := command.bits.cmd.wr_col - command.ready := engine.io.cmdReq.ready - busy := engine.io.status.running - val owner = Reg(UInt(log2Up(b.frontend.rob_entries).W)) - ownerRobId := owner - val layout = Reg(new framework.rvv.BankLayout(b)) - banks := layout - when(command.fire && command.bits.cmd.cmd.funct === 79.U) { - layout.readBank := command.bits.cmd.cmd.rs1Data(9, 0) - layout.writeBank := command.bits.cmd.cmd.rs1Data(29, 20) - layout.readGroups := command.bits.cmd.op1_col - layout.writeGroups := command.bits.cmd.wr_col - } - when(command.fire) { - assert(!command.bits.is_sub, "RVV command must belong to the main ROB") - owner := command.bits.rob_id - } - when(engine.io.cmdReq.fire) { - loadOwner := isLoad - runOwner := command.bits.cmd.cmd.funct === 79.U - when(isLoad) { - load.address := command.bits.cmd.cmd.rs2Data - load.bytes := command.bits.cmd.cmd.rs1Data(63, 30) - load.rob_id := command.bits.rob_id - loadPending := true.B - loadAccepted := false.B - terminalSeen := false.B - } - } - - val cancel = loadOwner && engine.io.cmdResp.valid && engine.io.result.fault && !terminalSeen - memory.load.valid := loadPending && !cancel - memory.load.bits := load - when(memory.load.fire || cancel)(loadPending := false.B) - memory.abort := cancel - when(memory.load.fire)(loadAccepted := true.B) - engine.io.imageTerminal.valid := loadOwner && loadAccepted && !terminalSeen && memory.result.valid - engine.io.imageTerminal.bits.error := memory.result.bits.error - engine.io.imageTerminal.bits.address := memory.result.bits.address - memory.result.ready := loadOwner && loadAccepted && !terminalSeen && engine.io.imageTerminal.ready - when(memory.result.fire) { - terminalSeen := true.B - terminalStatus := memory.result.bits - } - engine.io.image <> memory.image - memory.active := engine.io.status.running && !loadOwner - engine.io.channelReady := memory.ready - for (port <- 0 until b.rvv.memoryPorts) { - memory.bankRead(port) <> engine.io.bankRead(port) - memory.bankWrite(port) <> engine.io.bankWrite(port) - } - - val drained = !loadOwner || (!memory.busy && !loadPending && (!loadAccepted || terminalSeen)) - complete.valid := engine.io.cmdResp.valid && drained - complete.bits.rob_id := engine.io.cmdResp.bits.rob_id - complete.bits.is_sub := false.B - complete.bits.sub_rob_id := 0.U - complete.bits.fault := 0.U.asTypeOf(new DmaStatus) - when( - loadOwner && terminalSeen && terminalStatus.error =/= DmaError.None.U && terminalStatus.error =/= DmaError.Cancelled.U - ) { - complete.bits.fault := terminalStatus - }.elsewhen(engine.io.result.fault && (engine.io.result.cause === 7.U || !loadOwner && engine.io.result.cause === 5.U)) { - complete.bits.fault.error := DmaError.Bank.U - complete.bits.fault.address := engine.io.result.tval - }.elsewhen(engine.io.result.fault && engine.io.result.cause =/= 5.U) { - complete.bits.fault.error := DmaError.Shape.U - complete.bits.fault.address := Mux(loadOwner, load.address, engine.io.result.tval) - }.elsewhen(loadOwner && terminalSeen && terminalStatus.error =/= DmaError.None.U) { - complete.bits.fault := terminalStatus - } - - engine.io.cmdResp.ready := complete.ready && drained - when(engine.io.cmdResp.fire && !loadOwner && engine.io.result.fault)(failed := true.B) - fault := failed -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/csrc/src/lib.rs b/arch/src/main/scala/framework/system/core/ant/src/csrc/src/lib.rs deleted file mode 100644 index 679bed2f2..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/csrc/src/lib.rs +++ /dev/null @@ -1,20 +0,0 @@ -#[path = "../../../../../../mem-core/spm/src/csrc/src/lib.rs"] -mod storage; - -#[no_mangle] -pub extern "C" fn ant_arithmetic(a: u64, b: u64, operation: u32, word: bool) -> u64 { - let (a, b) = if word { (a as u32 as u64, b as u32 as u64) } else { (a, b) }; - let signed = |x: u64| if word { x as u32 as i32 as i64 } else { x as i64 }; - let value = match operation { - 0 => a.wrapping_mul(b), - 1 => (((a as i64 as i128) * (b as i64 as i128)) >> 64) as u64, - 2 => (((a as i64 as i128) * (b as i128)) >> 64) as u64, - 3 => (((a as u128) * (b as u128)) >> 64) as u64, - 4 => if b == 0 { u64::MAX } else { signed(a).wrapping_div(signed(b)) as u64 }, - 5 => if b == 0 { u64::MAX } else { a / b }, - 6 => if b == 0 { a } else { signed(a).wrapping_rem(signed(b)) as u64 }, - 7 => if b == 0 { a } else { a % b }, - _ => unreachable!(), - }; - if word { value as u32 as i32 as i64 as u64 } else { value } -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/ant.f b/arch/src/main/scala/framework/system/core/ant/src/main/resources/ant.f deleted file mode 100644 index 028d3899d..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/resources/ant.f +++ /dev/null @@ -1,10 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv -@RESOURCES@/../../../../../../mem-core/spm/src/main/resources/package.sv -@RESOURCES@/../../../../../../mem-core/spm/src/main/resources/interface.sv -+incdir+@RTL@ --F @RTL@/Execution/filelist.f -@RESOURCES@/ant_tb.sv diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/ant_tb.sv b/arch/src/main/scala/framework/system/core/ant/src/main/resources/ant_tb.sv deleted file mode 100644 index cdbe4e93d..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/resources/ant_tb.sv +++ /dev/null @@ -1,314 +0,0 @@ -`include "config.svh" -module ant_tb; - import uvm_pkg::*; - import ip_pkg::*; - import spm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - import "DPI-C" function longint unsigned ant_arithmetic( - longint unsigned a, - b, - int unsigned operation, - bit word_mode - ); - logic clock = 0, reset = 1, in_use = 0; - always #5 clock = ~clock; - ip_control_if control (clock); - virtual spm_if ports[5]; - for (genvar i = 0; i < 5; i++) begin : endpoints - spm_if bus ( - clock, - reset - ); - initial ports[i] = bus; - end - logic [1:0] start_valid = 0, start_ready, result_valid, result_ready = 0; - logic [1:0] cancel = 0, drained = '1, was_cancelled, running; - logic [31:0] task_id[2] = '{1, 2}, result_task[2]; - logic [63:0] value[2]; - logic [63:0] entry[2] = '{0, 0}, code_end[2] = '{16, 16}; - logic [1:0] command_valid, command_ready = '1, response_valid = 0, response_ready; - logic [31:0] command_instruction[2], command_task[2], response_task[2] = '{1, 2}; - logic [63:0] command_a[2], command_b[2], response_data[2] = '{64'd55, 64'd55}; - logic [4:0] response_rd[2] = '{5'd10, 5'd10}; - logic [1:0] response_error = 0, retired_valid; - logic [63:0] retired_pc[2]; - int checks = 0; - Execution dut ( - .clock(clock), - .reset(reset), - .io_inUse(in_use), - .io_start_0_valid(start_valid[0]), - .io_start_0_ready(start_ready[0]), - .io_start_0_bits_task(task_id[0]), - .io_start_0_bits_entry(entry[0]), - .io_start_0_bits_codeEnd(code_end[0]), - .io_start_0_bits_argument(`TLS_BASE), - .io_start_0_bits_stack(`TLS_END), - .io_result_0_valid(result_valid[0]), - .io_result_0_ready(result_ready[0]), - .io_result_0_bits_task(result_task[0]), - .io_result_0_bits_cancelled(was_cancelled[0]), - .io_result_0_bits_value(value[0]), - .io_cancel_0(cancel[0]), - .io_npuDrained_0(drained[0]), - .io_running_0(running[0]), - .io_command_0_ready(command_ready[0]), - .io_command_0_valid(command_valid[0]), - .io_command_0_bits_instruction(command_instruction[0]), - .io_command_0_bits_task(command_task[0]), - .io_command_0_bits_rs1(command_a[0]), - .io_command_0_bits_rs2(command_b[0]), - .io_retired_0_valid(retired_valid[0]), - .io_retired_0_bits_pc(retired_pc[0]), - .io_response_0_valid(response_valid[0]), - .io_response_0_ready(response_ready[0]), - .io_response_0_bits_task(response_task[0]), - .io_response_0_bits_rd(response_rd[0]), - .io_response_0_bits_data(response_data[0]), - .io_response_0_bits_error(response_error[0]), - .io_start_1_valid(start_valid[1]), - .io_start_1_ready(start_ready[1]), - .io_start_1_bits_task(task_id[1]), - .io_start_1_bits_entry(entry[1]), - .io_start_1_bits_codeEnd(code_end[1]), - .io_start_1_bits_argument(`TLS_BASE), - .io_start_1_bits_stack(`TLS_END), - .io_result_1_valid(result_valid[1]), - .io_result_1_ready(result_ready[1]), - .io_result_1_bits_task(result_task[1]), - .io_result_1_bits_cancelled(was_cancelled[1]), - .io_result_1_bits_value(value[1]), - .io_cancel_1(cancel[1]), - .io_npuDrained_1(drained[1]), - .io_running_1(running[1]), - .io_command_1_ready(command_ready[1]), - .io_command_1_valid(command_valid[1]), - .io_command_1_bits_instruction(command_instruction[1]), - .io_command_1_bits_task(command_task[1]), - .io_command_1_bits_rs1(command_a[1]), - .io_command_1_bits_rs2(command_b[1]), - .io_retired_1_valid(retired_valid[1]), - .io_retired_1_bits_pc(retired_pc[1]), - .io_response_1_valid(response_valid[1]), - .io_response_1_ready(response_ready[1]), - .io_response_1_bits_task(response_task[1]), - .io_response_1_bits_rd(response_rd[1]), - .io_response_1_bits_data(response_data[1]), - .io_response_1_bits_error(response_error[1]), - .io_code_0_request_valid(endpoints[0].bus.valid), - .io_code_0_request_ready(endpoints[0].bus.ready), - .io_code_0_request_bits_address(endpoints[0].bus.address), - .io_code_0_request_bits_size(endpoints[0].bus.size), - .io_code_0_request_bits_write(endpoints[0].bus.write), - .io_code_0_request_bits_data(endpoints[0].bus.data), - .io_code_0_request_bits_mask(endpoints[0].bus.mask), - .io_code_0_response_valid(endpoints[0].bus.response_valid), - .io_code_0_response_ready(endpoints[0].bus.response_ready), - .io_code_0_response_bits_data(endpoints[0].bus.result), - .io_code_0_response_bits_error(endpoints[0].bus.error), - .io_data_0_request_valid(endpoints[1].bus.valid), - .io_data_0_request_ready(endpoints[1].bus.ready), - .io_data_0_request_bits_address(endpoints[1].bus.address), - .io_data_0_request_bits_size(endpoints[1].bus.size), - .io_data_0_request_bits_write(endpoints[1].bus.write), - .io_data_0_request_bits_data(endpoints[1].bus.data), - .io_data_0_request_bits_mask(endpoints[1].bus.mask), - .io_data_0_response_valid(endpoints[1].bus.response_valid), - .io_data_0_response_ready(endpoints[1].bus.response_ready), - .io_data_0_response_bits_data(endpoints[1].bus.result), - .io_data_0_response_bits_error(endpoints[1].bus.error), - .io_code_1_request_valid(endpoints[2].bus.valid), - .io_code_1_request_ready(endpoints[2].bus.ready), - .io_code_1_request_bits_address(endpoints[2].bus.address), - .io_code_1_request_bits_size(endpoints[2].bus.size), - .io_code_1_request_bits_write(endpoints[2].bus.write), - .io_code_1_request_bits_data(endpoints[2].bus.data), - .io_code_1_request_bits_mask(endpoints[2].bus.mask), - .io_code_1_response_valid(endpoints[2].bus.response_valid), - .io_code_1_response_ready(endpoints[2].bus.response_ready), - .io_code_1_response_bits_data(endpoints[2].bus.result), - .io_code_1_response_bits_error(endpoints[2].bus.error), - .io_data_1_request_valid(endpoints[3].bus.valid), - .io_data_1_request_ready(endpoints[3].bus.ready), - .io_data_1_request_bits_address(endpoints[3].bus.address), - .io_data_1_request_bits_size(endpoints[3].bus.size), - .io_data_1_request_bits_write(endpoints[3].bus.write), - .io_data_1_request_bits_data(endpoints[3].bus.data), - .io_data_1_request_bits_mask(endpoints[3].bus.mask), - .io_data_1_response_valid(endpoints[3].bus.response_valid), - .io_data_1_response_ready(endpoints[3].bus.response_ready), - .io_data_1_response_bits_data(endpoints[3].bus.result), - .io_data_1_response_bits_error(endpoints[3].bus.error), - .io_shared_request_valid(endpoints[4].bus.valid), - .io_shared_request_ready(endpoints[4].bus.ready), - .io_shared_request_bits_address(endpoints[4].bus.address), - .io_shared_request_bits_size(endpoints[4].bus.size), - .io_shared_request_bits_write(endpoints[4].bus.write), - .io_shared_request_bits_data(endpoints[4].bus.data), - .io_shared_request_bits_mask(endpoints[4].bus.mask), - .io_shared_response_valid(endpoints[4].bus.response_valid), - .io_shared_response_ready(endpoints[4].bus.response_ready), - .io_shared_response_bits_data(endpoints[4].bus.result), - .io_shared_response_bits_error(endpoints[4].bus.error) - ); - initial begin - #0; - uvm_config_db#(virtual ip_control_if)::set(null, "uvm_test_top", "vif", control); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 2ms); - run_test("protocol_test"); - end - task automatic put(int port, longint unsigned address, logic [127:0] data, bit wide = 0); - logic [127:0] reply; - ports[port].send(address, wide ? 4 : 2, 1, data, wide ? 16'hffff : 16'h000f); - ports[port].take(reply); - if (ports[port].error) `uvm_fatal("LOAD", "management write failed") - endtask - task automatic launch(int id); - @(negedge clock); - start_valid[id] = 1; - do @(posedge clock); while (!start_ready[id]); - @(negedge clock); - start_valid[id] = 0; - endtask - task automatic finish(int id, longint unsigned expected, bit cancelled = 0); - wait (result_valid[id]); - if (was_cancelled[id] !== cancelled || value[id] !== expected || result_task[id] !== task_id[id]) - `uvm_fatal("RESULT", $sformatf( - "context %0d task %0d: cancelled %b value %h expected %h", - id, - result_task[id], - was_cancelled[id], - value[id], - expected - )) - repeat (3) begin - @(negedge clock); - if (!result_valid[id] || value[id] !== expected) `uvm_fatal("HOLD", "completion changed") - end - result_ready[id] = 1; - @(negedge clock); - result_ready[id] = 0; - checks++; - endtask - initial begin : exercise - longint unsigned operands[8] = '{ - 0, - 1, - 64'hffffffffffffffff, - 64'h8000000000000000, - 64'h7fffffffffffffff, - 64'h80000000, - 64'hffffffff, - 37 - }; - logic [31:0] operation; - wait (control.start); - repeat (3) @(negedge clock); - reset = 0; - // Real RV64IM programs: LD operands, arithmetic, ECALL returning a0. - // Golden arithmetic uses independent Rust i128/u128 and defined division corners. - for (int w = 0; w < 2; w++) - for (int op = 0; op < 8; op++) - if (!w || op == 0 || op >= 4) - for (int sample = 0; sample < 64; sample ++) begin - operation = 32'h02b50533 | (op << 12) | (w << 3); - put(0, 0, {32'h00000073, operation, 32'h00053503, 32'h00853583}, 1); - put(1, `TLS_BASE, {operands[sample%8], operands[sample/8]}, 1); - in_use = 1; - launch(0); - finish(0, ant_arithmetic(operands[sample/8], operands[sample%8], op, w)); - in_use = 0; - end - // One producer and one consumer share data then publish/observe a flag. - put(4, `TSS_BASE, 0, 1); - put(0, 0, {32'h0062b023, 32'h02a00313, 32'h000202b7, 32'h00000013}, 1); - put(0, 16, {32'h02a00513, 32'h0062b423, 32'h00100313, 32'h00000013}, 1); - put(0, 32, 32'h00000073); - // Consumer loops while flag is zero, then returns shared data. - put(2, 0, {32'h0002b503, 32'hfe030ee3, 32'h0082b303, 32'h000202b7}, 1); - put(2, 16, 32'h00000073); - code_end[0] = 36; - code_end[1] = 20; - in_use = 1; - fork - launch(0); - launch(1); - join - fork - finish(0, 42); - finish(1, 42); - join - in_use = 0; - // Identical TLS addresses hold different values in the two contexts. - put(0, 0, {96'b0, 32'h00053503}, 1); - put(0, 4, 32'h00000073); - put(2, 0, {96'b0, 32'h00053503}, 1); - put(2, 4, 32'h00000073); - put(1, `TLS_BASE, 128'd17, 1); - put(3, `TLS_BASE, 128'd29, 1); - code_end[0] = 8; - code_end[1] = 8; - in_use = 1; - fork - launch(0); - launch(1); - join - fork - finish(0, 17); - finish(1, 29); - join - in_use = 0; - code_end[0] = 16; - // Raw custom instruction and opaque DDR pointer survive submission backpressure. - put(0, 0, {32'h00000073, 32'h12b5757b, 32'h00053503, 32'h00853583}, 1); - put(1, `TLS_BASE, {64'hfedcba9876543210, 64'h80001000}, 1); - command_ready[0] = 0; - in_use = 1; - launch(0); - wait (command_valid[0]); - // A busy Ant must not lock the other Ant's private loader. - put(3, `TLS_BASE, 128'd91, 1); - repeat (7) begin - @(negedge clock); - if (!command_valid[0] || command_instruction[0] !== 32'h12b5757b || - command_task[0] !== task_id[0] || command_a[0] !== 64'h80001000 || - command_b[0] !== 64'hfedcba9876543210) - `uvm_fatal("COMMAND", "raw NPU command changed under backpressure") - end - drained[0] = 0; - command_ready[0] = 1; - @(negedge clock); - response_valid[0] = 1; - do @(posedge clock); while (!response_ready[0]); - @(negedge clock); - response_valid[0] = 0; - repeat (20) begin - @(negedge clock); - if (result_valid[0]) `uvm_fatal("DRAIN", "task completed before NPU drained") - end - drained[0] = 1; - finish(0, 55); - in_use = 0; - // Cancel an actual in-flight DIV; reusing the context must not consume its old result. - put(0, 0, {32'h00000073, 32'h02b54533, 32'h00053503, 32'h00853583}, 1); - put(1, `TLS_BASE, {64'd37, 64'h7fffffffffffffff}, 1); - in_use = 1; - launch(0); - do @(posedge clock); while (!(retired_valid[0] && retired_pc[0] == 4)); - repeat (8) @(negedge clock); - cancel[0] = 1; - @(negedge clock); - cancel[0] = 0; - finish(0, 0, 1); - in_use = 0; - put(0, 0, {64'b0, 32'h00000073, 32'h00b00513}, 1); - code_end[0] = 8; - in_use = 1; - launch(0); - finish(0, 11); - in_use = 0; - `uvm_info("ANT", $sformatf("%0d task completions checked", checks), UVM_LOW) - control.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/control.f b/arch/src/main/scala/framework/system/core/ant/src/main/resources/control.f deleted file mode 100644 index 7c06037d8..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/resources/control.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv -+incdir+@RTL@ --F @RTL@/Control/filelist.f -@RESOURCES@/control_if.sv -@RESOURCES@/control_tb.sv diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/control_tb.sv b/arch/src/main/scala/framework/system/core/ant/src/main/resources/control_tb.sv deleted file mode 100644 index 966d9c0b0..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/resources/control_tb.sv +++ /dev/null @@ -1,141 +0,0 @@ -`include "config.svh" -module control_tb; - import uvm_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if control (clock); - ant_control_if management ( - clock, - reset - ); - logic [1:0] online = '1, start_valid, start_ready = '1, result_valid = 0, result_ready, cancel; - logic storage_busy = 0, in_use; - logic [31:0] task_id[2], result_task[2] = '{7, 8}; - logic [63:0] entry[2], code_end[2], argument[2], stack[2]; - int checked = 0, launches = 0; - Control dut ( - .clock(clock), - .reset(reset), - .io_request_valid(management.valid), - .io_request_ready(management.ready), - .io_request_bits_operation(management.operation), - .io_request_bits_context(management.context_id), - .io_request_bits_field(management.field), - .io_request_bits_data(management.data), - .io_response_valid(management.reply_valid), - .io_response_ready(management.reply_ready), - .io_response_bits(management.reply), - .io_storageBusy(storage_busy), - .io_inUse(in_use), - .io_signatures_0(64'h1234), - .io_online_0(online[0]), - .io_cancel_0(cancel[0]), - .io_start_0_valid(start_valid[0]), - .io_start_0_ready(start_ready[0]), - .io_start_0_bits_task(task_id[0]), - .io_start_0_bits_entry(entry[0]), - .io_start_0_bits_codeEnd(code_end[0]), - .io_start_0_bits_argument(argument[0]), - .io_start_0_bits_stack(stack[0]), - .io_result_0_valid(result_valid[0]), - .io_result_0_ready(result_ready[0]), - .io_result_0_bits_task(result_task[0]), - .io_result_0_bits_cancelled(1'b0), - .io_result_0_bits_value(64'h42), - .io_signatures_1(64'h1235), - .io_online_1(online[1]), - .io_cancel_1(cancel[1]), - .io_start_1_valid(start_valid[1]), - .io_start_1_ready(start_ready[1]), - .io_start_1_bits_task(task_id[1]), - .io_start_1_bits_entry(entry[1]), - .io_start_1_bits_codeEnd(code_end[1]), - .io_start_1_bits_argument(argument[1]), - .io_start_1_bits_stack(stack[1]), - .io_result_1_valid(result_valid[1]), - .io_result_1_ready(result_ready[1]), - .io_result_1_bits_task(result_task[1]), - .io_result_1_bits_cancelled(1'b0), - .io_result_1_bits_value(64'h42) - ); - always @(posedge clock) - if (!reset) begin - for (int i = 0; i < 2; i++) - if (start_valid[i] && start_ready[i]) begin - if(task_id[i] !== 7+i || entry[i] !== 0 || code_end[i] !== 16 || - argument[i] !== `TLS_BASE || stack[i] !== `TLS_END) - `uvm_fatal("START", "local descriptor changed") - launches++; - end - end - initial begin - uvm_config_db#(virtual ip_control_if)::set(null, "uvm_test_top", "vif", control); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - task automatic transact(int op, int ctx, int f, longint unsigned value, - longint unsigned expected = 0); - longint unsigned actual; - management.transfer(op, ctx, f, value, actual); - if (actual !== expected) - `uvm_fatal("CONTROL", $sformatf("data %h expected %h", actual, expected)) - checked++; - endtask - task automatic descriptor(int ctx); - transact(1, ctx, 0, 7 + ctx); - transact(1, ctx, 1, 0); - transact(1, ctx, 2, 16); - transact(1, ctx, 3, `TLS_BASE); - transact(1, ctx, 4, `TLS_END); - transact(1, ctx, 5, 64'h1234 + ctx); - endtask - initial begin - wait (control.start); - repeat (3) @(negedge clock); - reset = 0; - transact(0, 0, 7, 0, 2); - transact(0, 1, 0, 0, 64'h1235); - transact(0, 0, 1, 0, 1); - transact(0, 0, 2, 0, 256); - transact(0, 0, 3, 0, `TLS_BASE); - transact(0, 0, 4, 0, 256); - transact(0, 0, 5, 0, `TSS_BASE); - transact(0, 0, 6, 0, 256); - transact(5, 0, 0, 0); - descriptor(0); - transact(1, 0, 5, 64'h1234); - start_ready[0] = 0; - fork - transact(2, 0, 0, 0); - begin - wait (start_valid[0]); - repeat (5) begin - @(negedge clock); - if (!start_valid[0] || management.ready) `uvm_fatal("STALL", "launch was not held") - end - start_ready[0] = 1; - end - join - transact(0, 0, 1, 0, 3); - descriptor(1); - transact(2, 1, 0, 0); - @(negedge clock); - result_valid = 3; - do @(posedge clock); while (result_ready != 3); - @(negedge clock); - result_valid = 0; - transact(0, 0, 1, 0, 5); - transact(0, 0, 8, 0, 7); - transact(0, 0, 9, 0, 64'h42); - transact(0, 1, 8, 0, 8); - transact(3, 0, 0, 0); - transact(3, 1, 0, 0); - transact(6, 0, 0, 0); - if (in_use || launches != 2) `uvm_fatal("GROUP", "group lifetime or launch count wrong") - `uvm_info("CONTROL", $sformatf("%0d transactions checked; %0d launches", checked, launches), - UVM_LOW) - control.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/service.f b/arch/src/main/scala/framework/system/core/ant/src/main/resources/service.f deleted file mode 100644 index 04e91c71c..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/resources/service.f +++ /dev/null @@ -1,9 +0,0 @@ -+incdir+@VERIFY@/uvm/src/ip -+incdir+@VERIFY@/uvm/src/ip/test -@VERIFY@/uvm/src/ip/package.sv -@VERIFY@/uvm/src/ip/test/control_if.sv -@VERIFY@/uvm/src/ip/test/control_test_package.sv -+incdir+@RTL@ --F @RTL@/Service/filelist.f -@RESOURCES@/control_if.sv -@RESOURCES@/service_tb.sv diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/service_tb.sv b/arch/src/main/scala/framework/system/core/ant/src/main/resources/service_tb.sv deleted file mode 100644 index a5a2d0ae1..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/resources/service_tb.sv +++ /dev/null @@ -1,160 +0,0 @@ -`include "config.svh" -module service_tb; - import uvm_pkg::*; - import ip_pkg::*; - import ip_control_test_pkg::*; - `include "uvm_macros.svh" - logic clock = 0, reset = 1; - always #5 clock = ~clock; - ip_control_if control (clock); - ant_control_if management ( - clock, - reset - ); - logic [1:0] drained = '1, command_valid, command_ready = '1, response_valid = 0, response_ready; - logic [31:0] command_instruction[2], command_task[2], response_task[2] = '{0, 0}; - logic [63:0] command_a[2], command_b[2], response_data[2] = '{0, 0}, retired_pc[2]; - logic [4:0] response_rd[2] = '{0, 0}; - logic [1:0] response_error = 0, retired_valid, cancel_npu; - int completed = 0; - ServiceVerification dut ( - .clock(clock), - .reset(reset), - .io_npuDrained_0(drained[0]), - .io_command_0_ready(command_ready[0]), - .io_command_0_valid(command_valid[0]), - .io_command_0_bits_instruction(command_instruction[0]), - .io_command_0_bits_task(command_task[0]), - .io_command_0_bits_rs1(command_a[0]), - .io_command_0_bits_rs2(command_b[0]), - .io_retired_0_valid(retired_valid[0]), - .io_retired_0_bits_pc(retired_pc[0]), - .io_response_0_valid(response_valid[0]), - .io_response_0_ready(response_ready[0]), - .io_response_0_bits_task(response_task[0]), - .io_response_0_bits_rd(response_rd[0]), - .io_response_0_bits_data(response_data[0]), - .io_response_0_bits_error(response_error[0]), - .io_npuDrained_1(drained[1]), - .io_command_1_ready(command_ready[1]), - .io_command_1_valid(command_valid[1]), - .io_command_1_bits_instruction(command_instruction[1]), - .io_command_1_bits_task(command_task[1]), - .io_command_1_bits_rs1(command_a[1]), - .io_command_1_bits_rs2(command_b[1]), - .io_retired_1_valid(retired_valid[1]), - .io_retired_1_bits_pc(retired_pc[1]), - .io_response_1_valid(response_valid[1]), - .io_response_1_ready(response_ready[1]), - .io_response_1_bits_task(response_task[1]), - .io_response_1_bits_rd(response_rd[1]), - .io_response_1_bits_data(response_data[1]), - .io_response_1_bits_error(response_error[1]), - .io_request_valid(management.valid), - .io_request_ready(management.ready), - .io_request_bits_operation(management.operation), - .io_request_bits_context(management.context_id), - .io_request_bits_field(management.field), - .io_request_bits_data(management.data), - .io_reply_valid(management.reply_valid), - .io_reply_ready(management.reply_ready), - .io_reply_bits(management.reply), - .io_signatures_0(64'h1234), - .io_online_0(1'b1), - .io_cancelNpu_0(cancel_npu[0]), - .io_signatures_1(64'h1235), - .io_online_1(1'b1), - .io_cancelNpu_1(cancel_npu[1]) - ); - initial begin - #0; - uvm_config_db#(virtual ip_control_if)::set(null, "uvm_test_top", "vif", control); - uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); - run_test("protocol_test"); - end - task automatic command(int op, int ctx, int field, longint unsigned value = 0); - longint unsigned reply; - management.transfer(op, ctx, field, value, reply); - endtask - task automatic put(int port, longint unsigned address, logic [127:0] data); - longint unsigned result, offset; - int space, ctx; - space = port == 4 ? 2 : port % 2; - ctx = port == 4 ? 0 : port / 2; - offset = address - (space == 0 ? 0 : space == 1 ? `TLS_BASE : `TSS_BASE); - for (int half = 0; half < 2; half++) begin - command(8 + 2 * space, ctx, offset + 8 * half, data[64*half+:64]); - management.transfer(7 + 2 * space, ctx, offset + 8 * half, 0, result); - if (result !== data[64*half+:64]) `uvm_fatal("LOAD", "local storage readback mismatch") - end - endtask - task automatic launch(int ctx, int task_id); - command(1, ctx, 0, task_id); - command(1, ctx, 1, 0); - command(1, ctx, 2, 8); - command(1, ctx, 3, `TLS_BASE); - command(1, ctx, 4, `TLS_END); - command(1, ctx, 5, 64'h1234 + ctx); - command(2, ctx, 0); - endtask - task automatic finish(int ctx, int task_id, longint unsigned value, bit cancelled = 0); - longint unsigned reply, status; - do management.transfer(0, ctx, 1, 0, status); while (!(status & 4)); - if (status !== (cancelled ? 13 : 5)) `uvm_fatal("RESULT", "wrong terminal state") - management.transfer(0, ctx, 8, 0, reply); - if (reply !== task_id) `uvm_fatal("RESULT", "wrong task ID") - management.transfer(0, ctx, 9, 0, reply); - if (reply !== value) `uvm_fatal("RESULT", "wrong terminal value") - command(3, ctx, 0); - completed++; - endtask - initial begin - longint unsigned reply; - wait (control.start); - repeat (3) @(negedge clock); - reset = 0; - put(1, `TLS_BASE, 128'h1111); - put(3, `TLS_BASE, 128'h2222); - put(4, `TSS_BASE, 128'h3333); - put(0, 0, {64'b0, 32'h00000073, 32'h02500513}); - put(2, 0, {64'b0, 32'h00000073, 32'h01d00513}); - command(5, 0, 0); - launch(0, 1); - launch(1, 2); - finish(0, 1, 37); - finish(1, 2, 29); - command(6, 0, 0); - put(0, 0, {64'b0, 32'h00000073, 32'h0000007b}); - command_ready[0] = 0; - command(5, 0, 0); - launch(0, 3); - drained[0] = 0; - wait (command_valid[0]); - command(4, 0, 0); - management.transfer(0, 0, 1, 0, reply); - if (reply !== 3) `uvm_fatal("DRAIN", "task completed before external work drained") - drained[0] = 1; - finish(0, 3, 0, 1); - command(6, 0, 0); - // A new task must not inherit the old task's cancellation. - command_ready[0] = 1; - put(0, 0, {64'b0, 32'h00000073, 32'h00900513}); - command(5, 0, 0); - launch(0, 4); - finish(0, 4, 9); - command(6, 0, 0); - // Cancellation remains effective after ECALL while backend work is still pending. - command(5, 0, 0); - launch(0, 5); - drained[0] = 0; - repeat (30) @(negedge clock); - command(4, 0, 0); - drained[0] = 1; - finish(0, 5, 0, 1); - command(6, 0, 0); - `uvm_info("SERVICE", $sformatf( - "%0d real Ant tasks; %0d control transactions checked", completed, management.checked - ), UVM_LOW) - control.done = 1; - end -endmodule diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Ant.scala b/arch/src/main/scala/framework/system/core/ant/src/main/scala/Ant.scala deleted file mode 100644 index 944f6cc73..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Ant.scala +++ /dev/null @@ -1,282 +0,0 @@ -package framework.ant - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import framework.arith.{DivRem, Multiply} -import memcore.memory.spm - -/** - * One RV64IM execution context. Ordinary memory is strictly local. NPU work may remain - * in flight after a command is accepted; the owner supplies drained for final completion. - * Cancellation also reaches the local stores, whose accepted operations must drain before reuse. - */ -@instantiable -class Ant(p: Params) extends Module { - val code = spm.Params(0, p.codeBytes, p.data.dataBits) - - @public val io = IO(new Bundle { - val start = Flipped(Decoupled(new Start(p))) - val result = Decoupled(new Completion(p)) - val cancel = Input(Bool()) - val drained = Input(Bool()) - val running = Output(Bool()) - val localDone = Output(Bool()) - val imem = new spm.Port(code) - val tls = new spm.Port(p.data) - val tss = new spm.Port(p.shared) - val command = Decoupled(new Command(p)) - val response = Flipped(Decoupled(new Response(p))) - val retired = Output(Valid(new Retire)) - }) - - val (idle :: fetch :: fetchWait :: execute :: memory :: memoryWait :: - multiply :: multiplyWait :: divide :: divideWait :: issue :: response :: complete :: Nil) = Enum(13) - val state = RegInit(idle) - val registers = Reg(Vec(32, UInt(64.W))) - val pc = Reg(UInt(64.W)) - val instruction = Reg(UInt(32.W)) - val task = Reg(UInt(p.taskBits.W)) - val codeEnd = Reg(UInt(64.W)) - val completion = Reg(new Completion(p)) - val cancelled = RegInit(false.B) - val multiplyPending = RegInit(false.B) - val dividePending = RegInit(false.B) - val live = !io.cancel && !reset.asBool - val memoryAddress = Reg(UInt(64.W)) - val memoryShared = Reg(Bool()) - val decoder = Instantiate(new Decode) - val multiplier = Instantiate(new Multiply) - val divider = Instantiate(new DivRem(64)) - decoder.io.instruction := instruction - val d = decoder.io.decoded - val a = Mux(d.rs1 === 0.U, 0.U, registers(d.rs1)) - val b = Mux(d.rs2 === 0.U, 0.U, registers(d.rs2)) - val nextPc = pc + 4.U - - io.start.ready := state === idle && io.drained && !io.cancel && !reset.asBool - io.result.valid := state === complete && io.drained && !multiplyPending && !dividePending && !reset.asBool - io.result.bits := completion - io.running := state =/= idle - io.localDone := state === complete - io.retired.valid := false.B - io.retired.bits := 0.U.asTypeOf(new Retire) - - def retire(rd: UInt, value: UInt, next: UInt): Unit = { - when(live) { - when(rd =/= 0.U)(registers(rd) := value) - io.retired.valid := true.B - io.retired.bits.pc := pc - io.retired.bits.instruction := instruction - io.retired.bits.rd := rd - io.retired.bits.data := Mux(rd === 0.U, 0.U, value) - pc := next - state := fetch - } - } - - def finish(value: UInt, wasCancelled: Bool = false.B): Unit = { - completion.task := task - completion.value := value - completion.cancelled := wasCancelled - state := complete - } - - def word(value: UInt): UInt = Mux(d.word, value(31, 0).asSInt.pad(64).asUInt, value) - - when(io.start.fire) { - for (r <- registers) r := 0.U - registers(2) := io.start.bits.stack - registers(4) := p.data.base.U - registers(10) := io.start.bits.argument - pc := io.start.bits.entry - codeEnd := io.start.bits.codeEnd - task := io.start.bits.task - cancelled := false.B - val valid = io.start.bits.entry(1, 0) === 0.U && io.start.bits.codeEnd(1, 0) === 0.U && - io.start.bits.codeEnd <= p.codeBytes.U && (io.start.bits.entry +& 4.U) <= io.start.bits.codeEnd && - io.start.bits.argument >= p.data.base.U && io.start.bits.argument < (p.data.base + p.data.bytes).U && - io.start.bits.stack > p.data.base.U && io.start.bits.stack <= (p.data.base + p.data.bytes).U && - io.start.bits.stack(3, 0) === 0.U - assert(valid, "Invalid Ant local execution descriptor") - when(valid)(state := fetch) - } - - val fetchAllowed = pc(1, 0) === 0.U && (pc +& 4.U) <= codeEnd - io.imem.request.valid := state === fetch && fetchAllowed && live - io.imem.request.bits := 0.U.asTypeOf(io.imem.request.bits) - io.imem.request.bits.address := pc - io.imem.request.bits.size := 2.U - io.imem.response.ready := state === fetchWait || (state === complete && cancelled) - when(state === fetch && live) { - assert(fetchAllowed, "Ant instruction address invalid: pc=0x%x", pc) - when(io.imem.request.fire)(state := fetchWait) - } - when(state === fetchWait && io.imem.response.fire && live) { - assert(!io.imem.response.bits.error, "Ant instruction storage access failed: pc=0x%x", pc) - when(!io.imem.response.bits.error) { - instruction := io.imem.response.bits.data(31, 0) - state := execute - } - } - - val operand = Mux(d.immediateAlu, d.immediate, b) - val shift = Mux(d.word, Cat(0.U(1.W), operand(4, 0)), operand(5, 0)) - val logical = Mux(d.word, Cat(0.U(32.W), a(31, 0)), a) - val signed = Mux(d.word, a(31, 0).asSInt.pad(64).asUInt, a) - - val alu = MuxLookup(d.funct3, 0.U(64.W))(Seq( - 0.U -> Mux(d.subtract, a - operand, a + operand), - 1.U -> (a << shift)(63, 0), - 2.U -> (a.asSInt < operand.asSInt).asUInt, - 3.U -> (a < operand).asUInt, - 4.U -> (a ^ operand), - 5.U -> Mux(d.arithmeticShift, (signed.asSInt >> shift).asUInt, logical >> shift), - 6.U -> (a | operand), - 7.U -> (a & operand) - )) - - val branchTaken = MuxLookup(d.funct3, false.B)(Seq( - 0.U -> (a === b), - 1.U -> (a =/= b), - 4.U -> (a.asSInt < b.asSInt), - 5.U -> (a.asSInt >= b.asSInt), - 6.U -> (a < b), - 7.U -> (a >= b) - )) - - val address = a + d.immediate - val size = d.funct3(1, 0) - val bytes = 1.U(64.W) << size - val end = address +& bytes - val inTls = address >= p.data.base.U && end <= (p.data.base + p.data.bytes).U - val inTss = address >= p.shared.base.U && end <= (p.shared.base + p.shared.bytes).U - when(state === execute && live) { - switch(d.kind) { - is(Kind.illegal)(assert(false.B, "Ant illegal instruction: pc=0x%x instruction=0x%x", pc, instruction)) - is(Kind.lui)(retire(d.rd, d.immediate, nextPc)) - is(Kind.auipc)(retire(d.rd, pc + d.immediate, nextPc)) - is(Kind.alu)(retire(d.rd, word(alu), nextPc)) - is(Kind.jal, Kind.jalr, Kind.branch) { - val target = Mux( - d.kind === Kind.jalr, - (a + d.immediate) & ~1.U(64.W), - Mux(d.kind === Kind.branch && !branchTaken, nextPc, pc + d.immediate) - ) - assert(target(1, 0) === 0.U, "Ant branch target misaligned: pc=0x%x target=0x%x", pc, target) - retire(Mux(d.kind === Kind.branch, 0.U, d.rd), nextPc, target) - } - is(Kind.load, Kind.store) { - assert((address & (bytes - 1.U)) === 0.U, "Ant load/store misaligned: pc=0x%x address=0x%x", pc, address) - assert(inTls || inTss, "Ant load/store outside TLS/TSS: pc=0x%x address=0x%x", pc, address) - when((address & (bytes - 1.U)) === 0.U && (inTls || inTss)) { - memoryAddress := address - memoryShared := inTss - state := memory - } - } - is(Kind.multiply)(state := multiply) - is(Kind.divide)(state := divide) - is(Kind.custom)(state := issue) - is(Kind.fence)(retire(0.U, 0.U, nextPc)) // Ordinary LSU operations already complete in order. - is(Kind.exit) { - assert(registers(17) === 0.U, "Ant unsupported ECALL: pc=0x%x a7=0x%x", pc, registers(17)) - when(registers(17) === 0.U)(finish(registers(10))) - } - is(Kind.breakpoint)(assert(false.B, "Ant breakpoint: pc=0x%x", pc)) - } - } - - val mask = MuxLookup(size, 0.U(p.data.beatBytes.W))(Seq( - 0.U -> 1.U, - 1.U -> 3.U, - 2.U -> 15.U, - 3.U -> 255.U - )) - - for ((port, shared) <- Seq((io.tls, false), (io.tss, true))) { - port.request.valid := state === memory && memoryShared === shared.B && live - port.request.bits := 0.U.asTypeOf(port.request.bits) - port.request.bits.address := memoryAddress - port.request.bits.size := size - port.request.bits.write := d.kind === Kind.store - port.request.bits.data := b - port.request.bits.mask := Mux(d.kind === Kind.store, mask, 0.U) - port.response.ready := (state === memoryWait && memoryShared === shared.B) || (state === complete && cancelled) - } - when(io.tls.request.fire || io.tss.request.fire)(state := memoryWait) - val memoryResponse = Mux(memoryShared, io.tss.response.bits, io.tls.response.bits) - val memoryValid = Mux(memoryShared, io.tss.response.valid, io.tls.response.valid) - - val loaded = MuxLookup(d.funct3, memoryResponse.data(63, 0))(Seq( - 0.U -> memoryResponse.data(7, 0).asSInt.pad(64).asUInt, - 1.U -> memoryResponse.data(15, 0).asSInt.pad(64).asUInt, - 2.U -> memoryResponse.data(31, 0).asSInt.pad(64).asUInt, - 4.U -> memoryResponse.data(7, 0).pad(64), - 5.U -> memoryResponse.data(15, 0).pad(64), - 6.U -> memoryResponse.data(31, 0).pad(64) - )) - - when(state === memoryWait && memoryValid && live) { - assert(!memoryResponse.error, "Ant local memory access failed: pc=0x%x address=0x%x", pc, memoryAddress) - when(!memoryResponse.error)(retire(Mux(d.kind === Kind.store, 0.U, d.rd), loaded, nextPc)) - } - - multiplier.io.request.valid := state === multiply && live - multiplier.io.request.bits.a := a - multiplier.io.request.bits.b := b - multiplier.io.request.bits.high := d.funct3 =/= 0.U - multiplier.io.request.bits.signedA := d.funct3 === 1.U || d.funct3 === 2.U - multiplier.io.request.bits.signedB := d.funct3 === 1.U - multiplier.io.response.ready := state === multiplyWait || (state === complete && cancelled) - when(multiplier.io.request.fire) { state := multiplyWait; multiplyPending := true.B } - when(multiplier.io.response.fire)(multiplyPending := false.B) - when(state === multiplyWait && multiplier.io.response.fire && live) { - retire(d.rd, word(multiplier.io.response.bits), nextPc) - } - divider.io.clear := false.B - divider.io.request.valid := state === divide && live - divider.io.request.bits.a := a - divider.io.request.bits.b := b - divider.io.request.bits.sew := Mux(d.word, 2.U, 3.U) - divider.io.request.bits.signed := !d.funct3(0) - divider.io.request.bits.remainder := d.funct3(1) - divider.io.response.ready := state === divideWait || (state === complete && cancelled) - when(divider.io.request.fire) { state := divideWait; dividePending := true.B } - when(divider.io.response.fire)(dividePending := false.B) - when(state === divideWait && divider.io.response.fire && live) { - retire(d.rd, word(divider.io.response.bits), nextPc) - } - - io.command.valid := state === issue && live - io.command.bits.task := task - io.command.bits.pc := pc - io.command.bits.instruction := instruction - io.command.bits.rs1 := Mux(instruction(13), a, 0.U) - io.command.bits.rs2 := Mux(instruction(12), b, 0.U) - io.response.ready := state === response || (state === complete && cancelled) - when(io.command.fire) { - when(instruction(14))(state := response) - .otherwise(retire(0.U, 0.U, nextPc)) - } - when(state === response && io.response.fire && live) { - assert(io.response.bits.task === task && io.response.bits.rd === d.rd, "Ant NPU response does not match its task") - assert(!io.response.bits.error, "Ant NPU response failed: pc=0x%x data=0x%x", pc, io.response.bits.data) - when(io.response.bits.task === task && io.response.bits.rd === d.rd && !io.response.bits.error) { - retire(d.rd, io.response.bits.data, nextPc) - } - } - when(io.result.fire)(state := idle) - when(io.cancel && state =/= idle && state =/= complete) { - finish(0.U, true.B) - cancelled := true.B - io.retired.valid := false.B - } - // Local execution may have ended while the task still waits for NPU/DMA drain. - // Once a completion is published, preserve it under response backpressure. - when(io.cancel && state === complete && !io.result.valid) { - completion.cancelled := true.B - completion.value := 0.U - cancelled := true.B - } -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Control.scala b/arch/src/main/scala/framework/system/core/ant/src/main/scala/Control.scala deleted file mode 100644 index 782e51664..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Control.scala +++ /dev/null @@ -1,143 +0,0 @@ -package framework.ant - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -object ControlOp { - val Query = 0 - val Stage = 1 - val Start = 2 - val Acknowledge = 3 - val Cancel = 4 - val Acquire = 5 - val Release = 6 -} - -class ControlRequest extends Bundle { - val operation = UInt(7.W) - val context = UInt(32.W) - val field = UInt(32.W) - val data = UInt(64.W) -} - -/** - * Task registers for one software owner. Invalid descriptors and resource reuse are - * programming errors. Execution contract violations assert at their source. - */ -@instantiable -class Control(p: Params, contexts: Int) extends Module { - require(contexts > 0 && p.taskBits <= 64) - - @public val io = IO(new Bundle { - val request = Flipped(Decoupled(new ControlRequest)) - val response = Decoupled(UInt(64.W)) - val signatures = Input(Vec(contexts, UInt(64.W))) - val online = Input(Vec(contexts, Bool())) - val storageBusy = Input(Bool()) - val start = Vec(contexts, Decoupled(new Start(p))) - val result = Vec(contexts, Flipped(Decoupled(new Completion(p)))) - val cancel = Output(Vec(contexts, Bool())) - val inUse = Output(Bool()) - val available = Output(Vec(contexts, Bool())) - }) - - val group = RegInit(false.B) - val owned = RegInit(VecInit(Seq.fill(contexts)(false.B))) - val done = RegInit(VecInit(Seq.fill(contexts)(false.B))) - val staged = Reg(Vec(contexts, Vec(6, UInt(64.W)))) - val fields = RegInit(VecInit(Seq.fill(contexts)(0.U(6.W)))) - val results = Reg(Vec(contexts, new Completion(p))) - val pending = RegInit(false.B) - val answer = Reg(UInt(64.W)) - val req = io.request.bits - val index = req.context(math.max(1, log2Ceil(contexts)) - 1, 0) - val inRange = req.context < contexts.U - val free = !owned(index) && !done(index) - val canStart = inRange && io.online(index) && free && group && - fields(index) === 63.U && staged(index)(5) === io.signatures(index) - val launching = req.operation === ControlOp.Start.U && canStart - io.request.ready := !pending && !reset.asBool && (!launching || io.start(index).ready) - io.response.valid := pending && !reset.asBool - io.response.bits := answer - io.inUse := group - when(io.response.fire)(pending := false.B) - for (i <- 0 until contexts) { - io.available(i) := io.online(i) && !owned(i) && !done(i) - io.start(i).valid := io.request.valid && !pending && !reset.asBool && launching && req.context === i.U - io.start(i).bits.task := staged(i)(0) - io.start(i).bits.entry := staged(i)(1) - io.start(i).bits.codeEnd := staged(i)(2) - io.start(i).bits.argument := staged(i)(3) - io.start(i).bits.stack := staged(i)(4) - // Cancel accepts immediately, independently of the start-ready path. - io.cancel(i) := io.request.valid && !pending && !reset.asBool && - req.operation === ControlOp.Cancel.U && req.context === i.U && owned(i) - io.result(i).ready := owned(i) && !done(i) && !reset.asBool - when(io.result(i).valid && !reset.asBool) { - assert(owned(i) && !done(i), "Ant result arrived without a live task") - assert(io.result(i).bits.task === staged(i)(0), "Ant completion task does not match its submission") - } - when(io.result(i).fire) { - results(i) := io.result(i).bits - owned(i) := false.B - done(i) := true.B - } - } - when(io.request.fire) { - pending := true.B - answer := 0.U - assert(req.operation <= ControlOp.Release.U, "Invalid Ant control operation") - when(req.operation <= ControlOp.Cancel.U && !(req.operation === ControlOp.Query.U && req.field === 7.U)) { - assert(inRange, "Invalid Ant context index") - } - switch(req.operation) { - is(ControlOp.Query.U) { - assert(req.field < 10.U, "Invalid Ant query field") - when(req.field >= 8.U)(assert(done(index), "Ant result read before completion")) - answer := MuxLookup(req.field, 0.U)(Seq( - 0.U -> io.signatures(index), - 1.U -> Cat(done(index) && results(index).cancelled, done(index), owned(index), io.online(index)), - 2.U -> p.codeBytes.U, - 3.U -> p.data.base.U, - 4.U -> p.data.bytes.U, - 5.U -> p.shared.base.U, - 6.U -> p.shared.bytes.U, - 7.U -> contexts.U, - 8.U -> results(index).task, - 9.U -> results(index).value - )) - } - is(ControlOp.Stage.U) { - assert(free, "Ant descriptor overwritten before resource release") - assert(req.field < 6.U, "Invalid Ant descriptor field") - when(req.field === 0.U)(assert((req.data >> p.taskBits) === 0.U, "Ant task ID overflow")) - staged(index)(req.field(2, 0)) := req.data - fields(index) := fields(index) | UIntToOH(req.field, 6) - } - is(ControlOp.Start.U) { - assert(io.online(index), "Offline Ant context submitted") - assert(free, "Ant context reused before completion acknowledgement") - assert(group, "Ant submitted without TSS group ownership") - assert(fields(index) === 63.U, "Incomplete Ant descriptor") - assert(staged(index)(5) === io.signatures(index), "Ant program fingerprint mismatch") - when(canStart) { owned(index) := true.B; fields(index) := 0.U } - } - is(ControlOp.Acknowledge.U) { - assert(done(index), "Ant completion acknowledged before result") - done(index) := false.B - } - is(ControlOp.Acquire.U) { - assert(!group && !io.storageBusy, "Ant group acquired before loader completion or previous release") - group := true.B - } - is(ControlOp.Release.U) { - assert( - group && !owned.asUInt.orR && !done.asUInt.orR && !io.storageBusy, - "Ant group released before tasks and storage drained" - ) - group := false.B - } - } - } -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Emit.scala b/arch/src/main/scala/framework/system/core/ant/src/main/scala/Emit.scala deleted file mode 100644 index 4ae83658c..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Emit.scala +++ /dev/null @@ -1,69 +0,0 @@ -package framework.ant - -import java.nio.file.{Files, Paths} -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.memory.spm - -object Emit extends App { - val p = Params(256, spm.Params(0x10000, 256), spm.Params(0x20000, 256)) - val output = Paths.get("src/main/scala/framework/system/core/ant/build") - Files.createDirectories(output) - Files.writeString( - output.resolve("config.svh"), - s"`define TLS_BASE 64'h${p.data.base.toString(16)}\n" + - s"`define TSS_BASE 64'h${p.shared.base.toString(16)}\n" + - s"`define TLS_END 64'h${(p.data.base + p.data.bytes).toString(16)}\n" - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Execution(p, contexts = 2), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Execution").toString, "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new Control(p, contexts = 2), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Control").toString, "--split-verilog") - ) - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new ServiceVerification(p, contexts = 2), - firtoolOpts = args, - args = Array("--target-dir", output.resolve("Service").toString, "--split-verilog") - ) -} - -@instantiable -class ServiceVerification(p: Params, contexts: Int) extends Module { - - @public val io = IO(new Bundle { - val request = Flipped(Decoupled(new ControlRequest)) - val reply = Decoupled(UInt(64.W)) - val signatures = Input(Vec(contexts, UInt(64.W))) - val online = Input(Vec(contexts, Bool())) - val command = Vec(contexts, Decoupled(new Command(p))) - val response = Vec(contexts, Flipped(Decoupled(new Response(p)))) - val npuDrained = Input(Vec(contexts, Bool())) - val cancelNpu = Output(Vec(contexts, Bool())) - val launched = Output(Vec(contexts, Valid(new Start(p)))) - val retired = Output(Vec(contexts, Valid(new Retire))) - val idle = Output(Bool()) - }) - - val service = Instantiate(new Service(p, contexts)) - service.io.request <> io.request - io.reply <> service.io.reply - service.io.signatures := io.signatures - service.io.online := io.online - io.launched := service.io.launched - io.retired := service.io.retired - io.idle := service.io.idle - for (i <- 0 until contexts) { - val execution = Instantiate(new LocalExecution(p)) - execution.io.local <> service.io.locals(i) - execution.io.npuDrained := io.npuDrained(i) - io.command(i) <> execution.io.command - execution.io.response <> io.response(i) - io.cancelNpu(i) := service.io.locals(i).cancel - } -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Execution.scala b/arch/src/main/scala/framework/system/core/ant/src/main/scala/Execution.scala deleted file mode 100644 index 91d5497d1..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Execution.scala +++ /dev/null @@ -1,55 +0,0 @@ -package framework.ant - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.memory.{spm, tss} - -/** A tile-local group. The controller holds inUse until all cooperating tasks have finished. */ -@instantiable -class Execution(p: Params, contexts: Int) extends Module { - require(contexts > 0) - val codeParams = spm.Params(0, p.codeBytes, p.data.dataBits) - - @public val io = IO(new Bundle { - val inUse = Input(Bool()) - val start = Vec(contexts, Flipped(Decoupled(new Start(p)))) - val result = Vec(contexts, Decoupled(new Completion(p))) - val cancel = Input(Vec(contexts, Bool())) - val npuDrained = Input(Vec(contexts, Bool())) - val running = Output(Vec(contexts, Bool())) - val storageBusy = Output(Bool()) - val command = Vec(contexts, Decoupled(new Command(p))) - val response = Vec(contexts, Flipped(Decoupled(new Response(p)))) - val retired = Output(Vec(contexts, Valid(new Retire))) - val code = Vec(contexts, Flipped(new spm.Port(codeParams))) - val data = Vec(contexts, Flipped(new spm.Port(p.data))) - val shared = Flipped(new spm.Port(p.shared)) - }) - - val shared = Instantiate(new tss.Store(tss.Params(p.shared, contexts))) - // A premature group release cannot grant management access over a running task. - shared.io.inUse := io.inUse || io.running.asUInt.orR - shared.io.host <> io.shared - shared.io.cancel := io.cancel - val privateBusy = Wire(Vec(contexts, Bool())) - io.storageBusy := privateBusy.asUInt.orR || shared.io.busy.asUInt.orR || shared.io.hostBusy - for (i <- 0 until contexts) { - val execution = Instantiate(new LocalExecution(p)) - privateBusy(i) := execution.io.local.storageBusy - execution.io.local.inUse := io.inUse - execution.io.local.start <> io.start(i) - io.result(i) <> execution.io.local.result - io.running(i) := execution.io.local.running - io.retired(i) := execution.io.local.retired - execution.io.local.cancel := io.cancel(i) - execution.io.local.sharedBusy := shared.io.busy(i) - execution.io.local.sharedHostBusy := shared.io.hostBusy - execution.io.npuDrained := io.npuDrained(i) - io.command(i) <> execution.io.command - execution.io.response <> io.response(i) - execution.io.local.code <> io.code(i) - execution.io.local.data <> io.data(i) - shared.io.clients(i) <> execution.io.local.shared - } -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Interfaces.scala b/arch/src/main/scala/framework/system/core/ant/src/main/scala/Interfaces.scala deleted file mode 100644 index 794982dba..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Interfaces.scala +++ /dev/null @@ -1,70 +0,0 @@ -package framework.ant - -import chisel3._ -import chisel3.util._ -import memcore.memory.spm - -/** RV64IM local execution; no compressed, floating-point, atomic or privileged instructions. */ -case class Params( - codeBytes: Int, - data: spm.Params, - shared: spm.Params, - taskBits: Int = 32) { - require(codeBytes >= 32 && isPow2(codeBytes)) - require(data.base >= codeBytes && shared.base >= codeBytes) - require(data.base + data.bytes <= shared.base || shared.base + shared.bytes <= data.base) - require(data.dataBits == shared.dataBits) - require(taskBits > 0) -} - -class Start(p: Params) extends Bundle { - val task = UInt(p.taskBits.W) - val entry = UInt(64.W) - val codeEnd = UInt(64.W) - val argument = UInt(64.W) - val stack = UInt(64.W) -} - -class Completion(p: Params) extends Bundle { - val task = UInt(p.taskBits.W) - val value = UInt(64.W) - val cancelled = Bool() -} - -class Command(p: Params) extends Bundle { - val task = UInt(p.taskBits.W) - val pc = UInt(64.W) - val instruction = UInt(32.W) - val rs1 = UInt(64.W) - val rs2 = UInt(64.W) -} - -class Response(p: Params) extends Bundle { - val task = UInt(p.taskBits.W) - val rd = UInt(5.W) - val data = UInt(64.W) - val error = Bool() -} - -class Retire extends Bundle { - val pc = UInt(64.W) - val instruction = UInt(32.W) - val rd = UInt(5.W) - val data = UInt(64.W) -} - -/** Core-side local execution and uncached management boundary. */ -class LocalPort(p: Params) extends Bundle { - val inUse = Input(Bool()) - val start = Flipped(Decoupled(new Start(p))) - val result = Decoupled(new Completion(p)) - val cancel = Input(Bool()) - val sharedBusy = Input(Bool()) - val sharedHostBusy = Input(Bool()) - val running = Output(Bool()) - val storageBusy = Output(Bool()) - val retired = Output(Valid(new Retire)) - val code = Flipped(new spm.Port(spm.Params(0, p.codeBytes, p.data.dataBits))) - val data = Flipped(new spm.Port(p.data)) - val shared = new spm.Port(p.shared) -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/LocalExecution.scala b/arch/src/main/scala/framework/system/core/ant/src/main/scala/LocalExecution.scala deleted file mode 100644 index 781064686..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/LocalExecution.scala +++ /dev/null @@ -1,42 +0,0 @@ -package framework.ant - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.memory.{spm, tls} - -@instantiable -class LocalExecution(p: Params) extends Module { - - @public val io = IO(new Bundle { - val local = new LocalPort(p) - val npuDrained = Input(Bool()) - val command = Decoupled(new Command(p)) - val response = Flipped(Decoupled(new Response(p))) - }) - - val ant = Instantiate(new Ant(p)) - val image = Instantiate(new tls.Store(spm.Params(0, p.codeBytes, p.data.dataBits), localReadOnly = true)) - val data = Instantiate(new tls.Store(p.data)) - io.local.storageBusy := image.io.busy || data.io.busy - ant.io.start.valid := io.local.start.valid && io.local.inUse && !io.local.sharedHostBusy - ant.io.start.bits := io.local.start.bits - io.local.start.ready := ant.io.start.ready && io.local.inUse && !io.local.sharedHostBusy - io.local.result <> ant.io.result - io.local.running := ant.io.running - io.local.retired := ant.io.retired - ant.io.cancel := io.local.cancel - ant.io.drained := io.npuDrained && !image.io.busy && !data.io.busy && !io.local.sharedBusy - io.command <> ant.io.command - ant.io.response <> io.response - val claimed = ant.io.running || (io.local.inUse && io.local.start.valid) - image.io.running := claimed - data.io.running := claimed - image.io.cancel := io.local.cancel - data.io.cancel := io.local.cancel - image.io.host <> io.local.code - data.io.host <> io.local.data - image.io.local <> ant.io.imem - data.io.local <> ant.io.tls - io.local.shared <> ant.io.tss -} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Service.scala b/arch/src/main/scala/framework/system/core/ant/src/main/scala/Service.scala deleted file mode 100644 index 876efbf41..000000000 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Service.scala +++ /dev/null @@ -1,102 +0,0 @@ -package framework.ant - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.memory.{spm, tss} - -/** - * Tile execution service. The caller supplies the uncached management transport and - * task-bound NPU/DMA owner; neither CPU cache nor address translation lives here. - */ -@instantiable -class Service(p: Params, contexts: Int) extends Module { - - @public val io = IO(new Bundle { - val request = Flipped(Decoupled(new ControlRequest)) - val reply = Decoupled(UInt(64.W)) - val signatures = Input(Vec(contexts, UInt(64.W))) - val online = Input(Vec(contexts, Bool())) - val locals = Vec(contexts, Flipped(new LocalPort(p))) - val launched = Output(Vec(contexts, Valid(new Start(p)))) - val retired = Output(Vec(contexts, Valid(new Retire))) - val idle = Output(Bool()) - }) - - val control = Instantiate(new Control(p, contexts)) - val sharedStore = Instantiate(new tss.Store(tss.Params(p.shared, contexts))) - val running = VecInit(io.locals.map(_.running)) - val storageBusy = io.locals.map(_.storageBusy).reduce(_ || _) || - sharedStore.io.busy.asUInt.orR || sharedStore.io.hostBusy - control.io.signatures := io.signatures - control.io.online := io.online - control.io.storageBusy := storageBusy - sharedStore.io.inUse := control.io.inUse || running.asUInt.orR - sharedStore.io.cancel := control.io.cancel - for (i <- 0 until contexts) { - io.locals(i).inUse := control.io.inUse - io.locals(i).start <> control.io.start(i) - control.io.result(i) <> io.locals(i).result - io.locals(i).cancel := control.io.cancel(i) - io.locals(i).sharedBusy := sharedStore.io.busy(i) - io.locals(i).sharedHostBusy := sharedStore.io.hostBusy - sharedStore.io.clients(i) <> io.locals(i).shared - io.retired(i) := io.locals(i).retired - } - val idle :: controlReply :: loading :: loadReply :: Nil = Enum(4) - val state = RegInit(idle) - val request = io.request.bits - val isLoad = request.operation >= 7.U - val space = (request.operation - 7.U) >> 1 - val write = !request.operation(0) - val shared = space === 2.U - val localIndex = request.context(math.max(1, log2Ceil(contexts)) - 1, 0) - val offset = request.field - val bytes = MuxLookup(space, 0.U)(Seq(0.U -> p.codeBytes.U, 1.U -> p.data.bytes.U, 2.U -> p.shared.bytes.U)) - val localBase = MuxLookup(space, 0.U(64.W))(Seq(0.U -> 0.U, 1.U -> p.data.base.U, 2.U -> p.shared.base.U)) - val valid = request.operation <= 12.U && Mux(shared, request.context === 0.U, request.context < contexts.U) && - offset(2, 0) === 0.U && (offset +& 8.U) <= bytes - val permitted = - Mux(shared, !control.io.inUse, !running(localIndex) && (!write || control.io.available(localIndex))) - val ports = io.locals.map(_.code).toSeq ++ io.locals.map(_.data).toSeq :+ sharedStore.io.host - val selected = Mux(shared, (2 * contexts).U, Mux(space === 0.U, request.context, request.context +& contexts.U)) - val owner = Reg(UInt(log2Ceil(ports.size).W)) - val value = Reg(UInt(64.W)) - for ((port, i) <- ports.zipWithIndex) { - port.request.valid := state === idle && io.request.valid && isLoad && valid && permitted && selected === i.U && !reset.asBool - port.request.bits := 0.U.asTypeOf(port.request.bits) - port.request.bits.address := localBase + offset - port.request.bits.size := 3.U - port.request.bits.write := write - port.request.bits.data := request.data - port.request.bits.mask := Mux(write, 255.U, 0.U) - port.response.ready := state === loading && owner === i.U && !reset.asBool - } - when(state === idle && io.request.valid && isLoad && !reset.asBool) { - assert(valid, "Invalid Ant local loader address or operation") - assert(permitted, "Ant local storage loader conflicts with execution ownership") - } - control.io.request.valid := state === idle && io.request.valid && !isLoad && !reset.asBool - control.io.request.bits := request - io.request.ready := state === idle && !reset.asBool && - Mux(isLoad, valid && permitted && VecInit(ports.map(_.request.ready))(selected), control.io.request.ready) - when(io.request.fire) { - state := Mux(isLoad, loading, controlReply) - owner := selected - } - val response = VecInit(ports.map(_.response.bits))(owner) - when(state === loading && VecInit(ports.map(_.response.valid))(owner)) { - assert(!response.error, "Ant local loader storage access failed") - value := response.data(63, 0) - state := loadReply - } - io.reply.valid := (state === loadReply || (state === controlReply && control.io.response.valid)) && !reset.asBool - io.reply.bits := Mux(state === loadReply, value, control.io.response.bits) - control.io.response.ready := state === controlReply && io.reply.ready && !reset.asBool - when(io.reply.fire)(state := idle) - io.idle := state === idle && !running.asUInt.orR && !storageBusy - for (i <- 0 until contexts) { - io.launched(i).valid := control.io.start(i).fire - io.launched(i).bits := control.io.start(i).bits - } -} diff --git a/arch/src/main/scala/framework/system/core/clink/Shm.scala b/arch/src/main/scala/framework/system/core/clink/Shm.scala deleted file mode 100644 index 929c53e4e..000000000 --- a/arch/src/main/scala/framework/system/core/clink/Shm.scala +++ /dev/null @@ -1,30 +0,0 @@ -package framework.system.core.clink - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig -import framework.memdomain.backend.MemRequestIO -import framework.memdomain.backend.shared.SharedMemLayout -import framework.memdomain.frontend.mem.MemConfigerIO -import framework.memdomain.isa.{MvoverISA, MvoverPort} -import memcore.memory.mesh_shm.MeshLocalBankPort - -/** Core-facing access to the tile's shared bank network. */ -class ShmPort(b: GlobalConfig) extends Bundle { - val requests = Vec(SharedMemLayout.channelPerHart(b), new MemRequestIO(b)) - val move = new MvoverPort - - val local = Flipped(new MeshLocalBankPort( - MvoverISA.AddressBits, - MvoverISA.BankBits, - b.memDomain.bankWidth, - math.max(1, log2Ceil(b.frontend.rob_entries)) - )) - - val config = Decoupled(new MemConfigerIO(b)) - val queryValid = Output(Bool()) - val queryVbank = Output(UInt(b.memDomain.vbankIdWidth.W)) - val queryGroups = Input(UInt(b.memDomain.groupCountWidth.W)) - val barrierArrive = Output(Bool()) - val barrierRelease = Input(Bool()) -} diff --git a/arch/src/main/scala/framework/system/core/clink/base.scala b/arch/src/main/scala/framework/system/core/clink/base.scala deleted file mode 100644 index 009619c6e..000000000 --- a/arch/src/main/scala/framework/system/core/clink/base.scala +++ /dev/null @@ -1,7 +0,0 @@ -package framework.system.core.clink - -import chisel3.Bundle - -class CLink extends Bundle - -trait HasCLink { def clink: CLink } diff --git a/arch/src/main/scala/framework/system/core/clink/clink.scala b/arch/src/main/scala/framework/system/core/clink/clink.scala deleted file mode 100644 index 4939e46a0..000000000 --- a/arch/src/main/scala/framework/system/core/clink/clink.scala +++ /dev/null @@ -1,69 +0,0 @@ -package framework.system.core.clink - -import chisel3._ -import chisel3.util._ -import framework.ant.{Params => AntParams, LocalPort} -import framework.top.GlobalConfig -import framework.frontend.globalrs.RobAllocation -import framework.memdomain.frontend.mem.Footprint -import framework.memdomain.frontend.mem.dma.DmaPort -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.backend.shared.SharedMemLayout -import framework.system.core.accelerator.AntMemoryPort -import framework.system.core.rocket.{CpuParams, RoCCCommandBB, RoCCResponseBB} -import hier.core.rocket.{AdmissionPorts, CommandSnapshot, RobFault} -import memcore.bus.chi -import memcore.memory.interlock.{Params => TrackingParams} - -/** Task address-space binding and identity; accepted work drains before context reuse. */ -class Control(p: AntParams, tracking: TrackingParams, pmps: Int)(implicit val cpuParams: CpuParams) extends Bundle { - val hartId = Input(UInt(64.W)) - val shmOwner = Input(UInt(64.W)) - val fingerprint = Output(UInt(64.W)) - val online = Output(Bool()) - - val bind = Flipped(Valid(new Bundle { - val task = UInt(p.taskBits.W) - val snapshot = new CommandSnapshot(tracking, pmps) - })) - - val drained = Output(Bool()) - val workDrained = Input(Bool()) - val halted = Input(Bool()) -} - -/** The tile admits commands and joins their actual ROB, memory and fault lifetimes. */ -class NpuEvents(b: GlobalConfig) extends Bundle { - val command = Flipped(Decoupled(new RoCCCommandBB(b.tile.xLen))) - val response = Decoupled(new RoCCResponseBB(b.tile.xLen)) - val allocation = Valid(new RobAllocation(b)) - val retired = Output(UInt(b.frontend.rob_entries.W)) - val fault = Valid(new RobFault(b.frontend.rob_entries)) - val busy = Output(Bool()) - val interrupt = Output(Bool()) - val footprints = Output(Vec(3, new Footprint(b))) -} - -/** Real NPU/issuer boundary; translation, cache maintenance and task-group storage stay in Tile. */ -class CLinkIO( - b: GlobalConfig, - ant: AntParams, - tracking: TrackingParams, - bus: chi.Params -)( - implicit val cpuParams: CpuParams) - extends CLink { - val ctrl = new Control(ant, tracking, cpuParams.core.nPMPs) - val local = new LocalPort(ant) - val admission = new AdmissionPorts(tracking, cpuParams.core.nPMPs, bus) - val memory = new AntMemoryPort(tracking, bus) - val npu = new NpuEvents(b) - val shm = new ShmPort(b) - val mem = new DmaPort(b.memDomain.dma_buswidth) - - val sharedHashes = - if (b.sim.diffTest && b.memDomain.sharedEnable) - Some(Input(Vec(SharedMemLayout.totalBank(b), new PhysicalBankHash(b)))) - else None - -} diff --git a/arch/src/main/scala/framework/system/core/rocket/BuckyballCommand.scala b/arch/src/main/scala/framework/system/core/rocket/BuckyballCommand.scala deleted file mode 100644 index eba52e452..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/BuckyballCommand.scala +++ /dev/null @@ -1,13 +0,0 @@ -package framework.system.core.rocket - -/** - * Elaboration-time representation of one Buckyball custom instruction. - * Domain ISA builders own the funct7 and operand encodings; boot only streams - * these already-formed commands. - */ -case class BuckyballCommand(funct: Int, rs1: BigInt = 0, rs2: BigInt = 0) - -object BuckyballCommand { - val Custom3Opcode = 0x7b - val Custom3Funct3 = 3 -} diff --git a/arch/src/main/scala/framework/system/core/rocket/CoreParameters.scala b/arch/src/main/scala/framework/system/core/rocket/CoreParameters.scala deleted file mode 100644 index c963e31cb..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/CoreParameters.scala +++ /dev/null @@ -1,175 +0,0 @@ -// See LICENSE.SiFive for license details. - -package freechips.rocketchip.tile - -import chisel3._ -import chisel3.util.isPow2 -import org.chipsalliance.cde.config._ -import freechips.rocketchip.rocket._ -import freechips.rocketchip.util._ - -case object MaxHartIdBits extends Field[Int] -case object CorePAddrBits extends Field[Int] - -// These parameters can be varied per-core -trait CoreParams { - val bootFreqHz: BigInt - val useVM: Boolean - val useHypervisor: Boolean - val useUser: Boolean - val useSupervisor: Boolean - val useDebug: Boolean - val useAtomics: Boolean - val useAtomicsOnlyForIO: Boolean - val useCompressed: Boolean - val useVector: Boolean = false - val vectorUseDCache: Boolean = false - val useRVE: Boolean - val useConditionalZero: Boolean - val useZba: Boolean - val useZbb: Boolean - val useZbs: Boolean - val mulDiv: Option[MulDivParams] - val fpu: Option[FPUParams] - val fetchWidth: Int - val decodeWidth: Int - val retireWidth: Int - val instBits: Int - val nLocalInterrupts: Int - val useNMI: Boolean - val nPMPs: Int - val pmpGranularity: Int - val nBreakpoints: Int - val useBPWatch: Boolean - val mcontextWidth: Int - val scontextWidth: Int - val nPerfCounters: Int - val haveBasicCounters: Boolean - val haveFSDirty: Boolean - val misaWritable: Boolean - val haveCFlush: Boolean - val nL2TLBEntries: Int - val nL2TLBWays: Int - val nPTECacheEntries: Int - val mtvecInit: Option[BigInt] - val mtvecWritable: Boolean - val traceHasWdata: Boolean - val xLen: Int - val pgLevels: Int - def traceCustom: Option[Data] = None - def customIsaExt: Option[String] = None - def customCSRs(implicit p: Parameters): CustomCSRs = new CustomCSRs - - def hasSupervisorMode: Boolean = useSupervisor || useVM - def instBytes: Int = instBits / 8 - def fetchBytes: Int = fetchWidth * instBytes - def lrscCycles: Int - - def dcacheReqTagBits: Int = 6 - - def minFLen: Int = 32 - - def vLen: Int = 0 - def eLen: Int = 0 - def vfLen: Int = 0 - def vfh: Boolean = false - def vExts: Seq[String] = Nil - def hasV: Boolean = vLen >= 128 && eLen >= 64 && vfLen >= 64 - def vMemDataBits: Int = 0 - - def useBitmanip = useZba && useZbb && useZbs -} - -trait HasCoreParameters extends HasTileParameters { - - override lazy val paddrBits: Int = { - val bits = p(CorePAddrBits) - require(bits <= maxPAddrBits) - bits - } - - val coreParams: CoreParams = tileParams.core - - val minFLen = coreParams.fpu.map(_ => coreParams.minFLen).getOrElse(0) - val fLen = coreParams.fpu.map(_.fLen).getOrElse(0) - - val usingMulDiv = coreParams.mulDiv.nonEmpty - val usingFPU = coreParams.fpu.nonEmpty - val usingAtomics = coreParams.useAtomics - val usingAtomicsOnlyForIO = coreParams.useAtomicsOnlyForIO - val usingAtomicsInCache = usingAtomics && !usingAtomicsOnlyForIO - val usingCompressed = coreParams.useCompressed - val usingVector = coreParams.useVector - val usingNMI = coreParams.useNMI - val usingConditionalZero = coreParams.useConditionalZero - - val retireWidth = coreParams.retireWidth - val fetchWidth = coreParams.fetchWidth - val decodeWidth = coreParams.decodeWidth - - val fetchBytes = coreParams.fetchBytes - val coreInstBits = coreParams.instBits - val coreInstBytes = coreInstBits / 8 - val coreDataBits = xLen max fLen max vMemDataBits - val coreDataBytes = coreDataBits / 8 - def coreMaxAddrBits = paddrBits max vaddrBitsExtended - - val nBreakpoints = coreParams.nBreakpoints - val nPMPs = coreParams.nPMPs - val pmpGranularity = coreParams.pmpGranularity - val nPerfCounters = coreParams.nPerfCounters - val mtvecInit = coreParams.mtvecInit - val mtvecWritable = coreParams.mtvecWritable - val customIsaExt = coreParams.customIsaExt - val traceHasWdata = coreParams.traceHasWdata - - def vLen = coreParams.vLen - def eLen = coreParams.eLen - def vfLen = coreParams.vfLen - def vMemDataBits = if (usingVector) coreParams.vMemDataBits else 0 - def maxVLMax = vLen - - if (usingVector) { - require(isPow2(vLen), s"vLen ($vLen) must be a power of 2") - require(eLen >= 32 && vLen % eLen == 0, s"eLen must divide vLen ($vLen) and be no less than 32") - require(eLen == 32 || eLen == 64) - require(vfLen <= eLen) - require(!coreParams.vfh || (vfLen >= 32 && coreParams.minFLen <= 16)) - } - - if (coreParams.useVM) { - if (coreParams.xLen == 32) { - require(coreParams.pgLevels == 2) - } else { - require(coreParams.pgLevels >= 3) - } - } - - lazy val hartIdLen: Int = p(MaxHartIdBits) - - lazy val resetVectorLen: Int = { - val externalLen = paddrBits - require(externalLen <= xLen, s"External reset vector length ($externalLen) must be <= XLEN ($xLen)") - require( - externalLen <= vaddrBitsExtended, - s"External reset vector length ($externalLen) must be <= virtual address bit width ($vaddrBitsExtended)" - ) - externalLen - } - - // Print out log of committed instructions and their writeback values. - // Requires post-processing due to out-of-order writebacks. - val enableCommitLog = false - -} - -abstract class CoreModule(implicit val p: Parameters) extends Module with HasCoreParameters - -abstract class CoreBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) with HasCoreParameters - -// This is a raw commit trace from the core, not the TraceCoreInterface -class TraceBundle(implicit val p: Parameters) extends Bundle with HasCoreParameters { - val insns = Vec(coreParams.retireWidth, new TracedInstruction) - val time = UInt(64.W) - val custom = coreParams.traceCustom -} diff --git a/arch/src/main/scala/framework/system/core/rocket/CpuInterfaces.scala b/arch/src/main/scala/framework/system/core/rocket/CpuInterfaces.scala deleted file mode 100644 index 7d63d73eb..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/CpuInterfaces.scala +++ /dev/null @@ -1,170 +0,0 @@ -// See LICENSE.Berkeley and LICENSE.SiFive for the Rocket interface definitions. -package freechips.rocketchip.rocket - -import chisel3._ -import chisel3.util._ -import org.chipsalliance.cde.config.Parameters -import freechips.rocketchip.tile.{CoreBundle, HasCoreParameters, TileInterrupts} - -/** CPU-facing interfaces, independent of the upstream cache and TileLink implementations. */ -class FrontendReq(implicit p: Parameters) extends CoreBundle()(p) { - val pc = UInt(vaddrBitsExtended.W) - val speculative = Bool() -} - -class FrontendExceptions extends Bundle { - val pf = new Bundle { val inst = Bool() } - val gf = new Bundle { val inst = Bool() } - val ae = new Bundle { val inst = Bool() } -} - -class FrontendResp(implicit p: Parameters) extends CoreBundle()(p) { - val btb = new BTBResp - val pc = UInt(vaddrBitsExtended.W) - val data = UInt((fetchWidth * coreInstBits).W) - val mask = Bits(fetchWidth.W) - val xcpt = new FrontendExceptions - val replay = Bool() -} - -class FrontendPerfEvents extends Bundle { - val acquire = Bool() - val tlbMiss = Bool() -} - -class SFenceReq(implicit p: Parameters) extends CoreBundle()(p) { - val rs1 = Bool() - val rs2 = Bool() - val addr = UInt(vaddrBits.W) - val asid = UInt((asIdBits max 1).W) - val hv = Bool() - val hg = Bool() -} - -class FrontendIO(implicit p: Parameters) extends CoreBundle()(p) { - val might_request = Output(Bool()) - val clock_enabled = Input(Bool()) - val req = Valid(new FrontendReq) - val sfence = Valid(new SFenceReq) - val resp = Flipped(Decoupled(new FrontendResp)) - val gpa = Flipped(Valid(UInt(vaddrBitsExtended.W))) - val gpa_is_pte = Input(Bool()) - val btb_update = Valid(new BTBUpdate) - val bht_update = Valid(new BHTUpdate) - val ras_update = Valid(new RASUpdate) - val flush_icache = Output(Bool()) - val npc = Input(UInt(vaddrBitsExtended.W)) - val perf = Input(new FrontendPerfEvents) - val progress = Output(Bool()) -} - -trait HasCoreMemOp extends HasCoreParameters { - val addr = UInt(coreMaxAddrBits.W) - - val idx = Option.when(usingVM && lgCacheBlockBytes + log2Up(tileParams.dcache.get.nSets) > pgIdxBits)( - UInt(coreMaxAddrBits.W) - ) - - val tag = UInt((coreParams.dcacheReqTagBits + log2Ceil(dcacheArbPorts)).W) - val cmd = UInt(M_SZ.W) - val size = UInt(log2Ceil(log2Ceil(coreDataBytes) + 1).W) - val signed = Bool() - val dprv = UInt(PRV.SZ.W) - val dv = Bool() -} - -trait HasCoreData extends HasCoreParameters { - val data = UInt(coreDataBits.W) - val mask = UInt(coreDataBytes.W) -} - -class HellaCacheReqInternal(implicit p: Parameters) extends CoreBundle()(p) with HasCoreMemOp { - val phys = Bool() - val no_resp = Bool() - val no_alloc = Bool() - val no_xcpt = Bool() -} - -class HellaCacheReq(implicit p: Parameters) extends HellaCacheReqInternal()(p) with HasCoreData - -class HellaCacheResp(implicit p: Parameters) extends CoreBundle()(p) with HasCoreMemOp with HasCoreData { - val replay = Bool() - val has_data = Bool() - val data_word_bypass = UInt(coreDataBits.W) - val data_raw = UInt(coreDataBits.W) - val store_data = UInt(coreDataBits.W) -} - -class AlignmentExceptions extends Bundle { - val ld = Bool() - val st = Bool() -} - -class HellaCacheExceptions extends Bundle { - val ma = new AlignmentExceptions - val pf = new AlignmentExceptions - val gf = new AlignmentExceptions - val ae = new AlignmentExceptions -} - -class HellaCacheWriteData(implicit p: Parameters) extends CoreBundle()(p) with HasCoreData - -class HellaCachePerfEvents extends Bundle { - val acquire = Bool() - val release = Bool() - val grant = Bool() - val tlbMiss = Bool() - val blocked = Bool() - val canAcceptStoreThenLoad = Bool() - val canAcceptStoreThenRMW = Bool() - val canAcceptLoadThenLoad = Bool() - val storeBufferEmptyAfterLoad = Bool() - val storeBufferEmptyAfterStore = Bool() -} - -class HellaCacheIO(implicit p: Parameters) extends CoreBundle()(p) { - val req = Decoupled(new HellaCacheReq) - val s1_kill = Output(Bool()) - val s1_data = Output(new HellaCacheWriteData) - val s2_nack = Input(Bool()) - val s2_nack_cause_raw = Input(Bool()) - val s2_kill = Output(Bool()) - val s2_uncached = Input(Bool()) - val s2_paddr = Input(UInt(paddrBits.W)) - val resp = Flipped(Valid(new HellaCacheResp)) - val replay_next = Input(Bool()) - val s2_xcpt = Input(new HellaCacheExceptions) - val s2_gpa = Input(UInt(vaddrBitsExtended.W)) - val s2_gpa_is_pte = Input(Bool()) - val uncached_resp = Option.when(tileParams.dcache.get.separateUncachedResp)(Flipped(Decoupled(new HellaCacheResp))) - val ordered = Input(Bool()) - val store_pending = Input(Bool()) - val perf = Input(new HellaCachePerfEvents) - val keep_clock_enabled = Output(Bool()) - val clock_enabled = Input(Bool()) -} - -class PTWPerfEvents extends Bundle { - val l2miss = Bool() - val l2hit = Bool() - val pte_miss = Bool() - val pte_hit = Bool() -} - -class DatapathPTWIO(implicit p: Parameters) extends CoreBundle()(p) { - val ptbr = Input(new PTBR) - val hgatp = Input(new PTBR) - val vsatp = Input(new PTBR) - val sfence = Flipped(Valid(new SFenceReq)) - val status = Input(new MStatus) - val hstatus = Input(new HStatus) - val gstatus = Input(new MStatus) - val pmp = Input(Vec(nPMPs, new PMP)) - val perf = Output(new PTWPerfEvents) - val customCSRs = Flipped(coreParams.customCSRs) - val clock_enabled = Output(Bool()) -} - -class CoreInterrupts(val hasBeu: Boolean)(implicit p: Parameters) extends TileInterrupts()(p) { - val buserror = Option.when(hasBeu)(Bool()) -} diff --git a/arch/src/main/scala/framework/system/core/rocket/CpuParams.scala b/arch/src/main/scala/framework/system/core/rocket/CpuParams.scala deleted file mode 100644 index 4c2ef32ae..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/CpuParams.scala +++ /dev/null @@ -1,15 +0,0 @@ -package framework.system.core.rocket - -import freechips.rocketchip.rocket.{BTBParams, DCacheParams, ICacheParams, RocketCoreParams} - -/** Explicit CPU metadata; the upstream parameter conversion stays in the Rocket adapter. */ -case class CpuParams( - core: RocketCoreParams, - dcache: Option[DCacheParams], - icache: Option[ICacheParams], - btb: Option[BTBParams], - physicalAddressBits: Int, - hartIdBits: Int, - beatBytes: Int = 8, - blockBytes: Int = 64, - asIdBits: Int = 0) diff --git a/arch/src/main/scala/framework/system/core/rocket/Parameters.scala b/arch/src/main/scala/framework/system/core/rocket/Parameters.scala deleted file mode 100644 index 732867221..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/Parameters.scala +++ /dev/null @@ -1,24 +0,0 @@ -package framework.system.core.rocket - -import org.chipsalliance.cde.config.Parameters -import freechips.rocketchip.tile.{CoreBeatBytes, CorePAddrBits, CpuTileParams, HasCoreParameters, MaxHartIdBits, TileKey} -import freechips.rocketchip.rocket.ASIdBits -import freechips.rocketchip.subsystem.CacheBlockBytes - -object RocketParameters { - - def apply(cpu: CpuParams): Parameters = Parameters.empty.alterPartial { - case TileKey => CpuTileParams(core = cpu.core, dcache = cpu.dcache, icache = cpu.icache, btb = cpu.btb) - case CorePAddrBits => cpu.physicalAddressBits - case MaxHartIdBits => cpu.hartIdBits - case CoreBeatBytes => cpu.beatBytes - case CacheBlockBytes => cpu.blockBytes - case ASIdBits => cpu.asIdBits - } - -} - -trait HasCpuParameters extends HasCoreParameters { - implicit val cpuParams: CpuParams - implicit lazy val p: Parameters = RocketParameters(cpuParams) -} diff --git a/arch/src/main/scala/framework/system/core/rocket/RoCCCommandBB.scala b/arch/src/main/scala/framework/system/core/rocket/RoCCCommandBB.scala deleted file mode 100644 index da6f1194e..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/RoCCCommandBB.scala +++ /dev/null @@ -1,36 +0,0 @@ -package framework.system.core.rocket - -import chisel3._ -import chisel3.util._ - -/** RoCC response bundle */ -class RoCCResponseBB(xLen: Int = 64) extends Bundle { - val rd = Bits(5.W) - val data = Bits(xLen.W) -} - -/** RoCC interface between a core and an accelerator. */ -class RoCCIO(xLen: Int = 64) extends Bundle { - val cmd = Flipped(Decoupled(new RoCCCommandBB(xLen))) - val resp = Decoupled(new RoCCResponseBB(xLen)) - val busy = Output(Bool()) - val interrupt = Output(Bool()) - val exception = Input(Bool()) -} - -/** RoCC command bundle */ -class RoCCCommandBB(xLen: Int = 64) extends Bundle { - val raw_inst = UInt(32.W) - val pc = UInt(xLen.W) - val funct = UInt(7.W) - val funct3 = UInt(3.W) - val rs2 = Bits(5.W) - val rs1 = Bits(5.W) - val xd = Bool() - val xs1 = Bool() - val xs2 = Bool() - val rd = Bits(5.W) - val opcode = UInt(7.W) - val rs1Data = UInt(xLen.W) - val rs2Data = UInt(xLen.W) -} diff --git a/arch/src/main/scala/framework/system/core/rocket/RocketCoreBB.scala b/arch/src/main/scala/framework/system/core/rocket/RocketCoreBB.scala deleted file mode 100644 index 441caf85c..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/RocketCoreBB.scala +++ /dev/null @@ -1,1486 +0,0 @@ -// See LICENSE.Berkeley for license details. -// See LICENSE.SiFive for license details. - -package framework.system.core.rocket - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import chisel3.withClock -import freechips.rocketchip.tile._ -import freechips.rocketchip.util._ -import freechips.rocketchip.util.property -import scala.collection.mutable.ArrayBuffer -import freechips.rocketchip.rocket._ - -/** RoCC core IO — used inside Rocket core. */ -class RoCCCoreIOBB(val nRoCCCSRs: Int = 0)(implicit val cpuParams: CpuParams) extends Bundle with HasCpuParameters { - val cmd = Flipped(Decoupled(new RoCCCommandBB)) - val resp = Decoupled(new RoCCResponseBB) - val mem = new HellaCacheIO - val busy = Output(Bool()) - val interrupt = Output(Bool()) - val exception = Input(Bool()) - val csrs = Flipped(Vec(nRoCCCSRs, new CustomCSRIO)) -} - -@instantiable -class RocketBB( - roccCSRs: Seq[CustomCSR], - coreHasBuckyball: Boolean, - hasTaskControl: Boolean, - val hasBusError: Boolean, - dcacheFlushOnFenceI: Boolean, - dcacheCanSupportCFlushLine: Boolean -)( - implicit val cpuParams: CpuParams) - extends Module - with HasRocketCoreParameters - with HasCpuParameters { - def nTotalRoCCCSRs = roccCSRs.size - - override val usingRoCC = coreHasBuckyball || hasTaskControl - - @public val io = IO(new CoreBundle()(p) { - val hartid = Input(UInt(hartIdLen.W)) - val reset_vector = Input(UInt(resetVectorLen.W)) - val interrupts = Input(new CoreInterrupts(hasBusError)) - val interruptReady = Input(Bool()) - val imem = new FrontendIO - val dmem = new HellaCacheIO - val dmemPc = Output(UInt(vaddrBitsExtended.W)) - val roccOlderPending = Output(Bool()) - val roccCommitEligible = Output(Bool()) - val ptw = Flipped(new DatapathPTWIO()) - val fpu = Flipped(new FPUCoreIO()) - val rocc = Flipped(new RoCCCoreIOBB(nTotalRoCCCSRs)) - val trace = Output(new TraceBundle) - val bpwatch = Output(Vec(coreParams.nBreakpoints, new BPWatch(coreParams.retireWidth))) - val cease = Output(Bool()) - val wfi = Output(Bool()) - val traceStall = Input(Bool()) - val vector = if (usingVector) Some(Flipped(new VectorCoreIO)) else None - }) - - import ALU._ - - val clock_en_reg = RegInit(true.B) - val long_latency_stall = Reg(Bool()) - val id_reg_pause = Reg(Bool()) - val imem_might_request_reg = Reg(Bool()) - val clock_en = WireDefault(true.B) - - val gated_clock = - if (!rocketParams.clockGate) clock - else ClockGate(clock, clock_en, "rocket_clock_gate") - - class RocketImpl { // entering gated-clock domain - - // performance counters - def pipelineIDToWB[T <: Data](x: T): T = - RegEnable(RegEnable(RegEnable(x, !ctrl_killd), ex_pc_valid), mem_pc_valid) - - val perfEvents = new EventSets(Seq( - new EventSet( - (mask, hits) => Mux(wb_xcpt, mask(0), wb_valid && pipelineIDToWB((mask & hits).orR)), - Seq( - ("exception", () => false.B), - ("load", () => id_ctrl.mem && id_ctrl.mem_cmd === M_XRD && !id_ctrl.fp), - ("store", () => id_ctrl.mem && id_ctrl.mem_cmd === M_XWR && !id_ctrl.fp), - ("amo", () => usingAtomics.B && id_ctrl.mem && (isAMO(id_ctrl.mem_cmd) || id_ctrl.mem_cmd.isOneOf(M_XLR, M_XSC))), - ("system", () => id_ctrl.csr =/= CSR.N), - ( - "arith", - () => - id_ctrl.wxd && !(id_ctrl.jal || id_ctrl.jalr || id_ctrl.mem || id_ctrl.fp || id_ctrl.mul || id_ctrl.div || id_ctrl.csr =/= CSR.N) - ), - ("branch", () => id_ctrl.branch), - ("jal", () => id_ctrl.jal), - ("jalr", () => id_ctrl.jalr) - ) - ++ (if (!usingMulDiv) Seq() - else - Seq( - ("mul", () => if (pipelinedMul) id_ctrl.mul else id_ctrl.div && (id_ctrl.alu_fn & FN_DIV) =/= FN_DIV), - ("div", () => if (pipelinedMul) id_ctrl.div else id_ctrl.div && (id_ctrl.alu_fn & FN_DIV) === FN_DIV) - )) - ++ (if (!usingFPU) Seq() - else - Seq( - ("fp load", () => id_ctrl.fp && io.fpu.dec.ldst && io.fpu.dec.wen), - ("fp store", () => id_ctrl.fp && io.fpu.dec.ldst && !io.fpu.dec.wen), - ("fp add", () => id_ctrl.fp && io.fpu.dec.fma && io.fpu.dec.swap23), - ("fp mul", () => id_ctrl.fp && io.fpu.dec.fma && !io.fpu.dec.swap23 && !io.fpu.dec.ren3), - ("fp mul-add", () => id_ctrl.fp && io.fpu.dec.fma && io.fpu.dec.ren3), - ("fp div/sqrt", () => id_ctrl.fp && (io.fpu.dec.div || io.fpu.dec.sqrt)), - ( - "fp other", - () => id_ctrl.fp && !(io.fpu.dec.ldst || io.fpu.dec.fma || io.fpu.dec.div || io.fpu.dec.sqrt) - ) - )) - ), - new EventSet( - (mask, hits) => (mask & hits).orR, - Seq( - ( - "load-use interlock", - () => id_ex_hazard && ex_ctrl.mem || id_mem_hazard && mem_ctrl.mem || id_wb_hazard && wb_ctrl.mem - ), - ("long-latency interlock", () => id_sboard_hazard), - ( - "csr interlock", - () => - id_ex_hazard && ex_ctrl.csr =/= CSR.N || id_mem_hazard && mem_ctrl.csr =/= CSR.N || id_wb_hazard && wb_ctrl.csr =/= CSR.N - ), - ("I$ blocked", () => icache_blocked), - ("D$ blocked", () => id_ctrl.mem && dcache_blocked), - ("branch misprediction", () => take_pc_mem && mem_direction_misprediction), - ( - "control-flow target misprediction", - () => take_pc_mem && mem_misprediction && mem_cfi && !mem_direction_misprediction && !icache_blocked - ), - ("flush", () => wb_reg_flush_pipe), - ("replay", () => replay_wb) - ) - ++ (if (!usingMulDiv) Seq() - else - Seq( - ( - "mul/div interlock", - () => - id_ex_hazard && (ex_ctrl.mul || ex_ctrl.div) || id_mem_hazard && (mem_ctrl.mul || mem_ctrl.div) || id_wb_hazard && wb_ctrl.div - ) - )) - ++ (if (!usingFPU) Seq() - else - Seq( - ( - "fp interlock", - () => - id_ex_hazard && ex_ctrl.fp || id_mem_hazard && mem_ctrl.fp || id_wb_hazard && wb_ctrl.fp || id_ctrl.fp && id_stall_fpu - ) - )) - ), - new EventSet( - (mask, hits) => (mask & hits).orR, - Seq( - ("I$ miss", () => io.imem.perf.acquire), - ("D$ miss", () => io.dmem.perf.acquire), - ("D$ release", () => io.dmem.perf.release), - ("ITLB miss", () => io.imem.perf.tlbMiss), - ("DTLB miss", () => io.dmem.perf.tlbMiss), - ("L2 TLB miss", () => io.ptw.perf.l2miss) - ) - ) - )) - - val pipelinedMul = usingMulDiv && mulDivParams.mulUnroll == xLen - - val decode_table = { - (if (usingMulDiv) new MDecode(pipelinedMul) +: (xLen > 32).option(new M64Decode(pipelinedMul)).toSeq else Nil) ++: - (if (usingAtomics) new ADecode +: (xLen > 32).option(new A64Decode).toSeq else Nil) ++: - (if (fLen >= 32) new FDecode +: (xLen > 32).option(new F64Decode).toSeq else Nil) ++: - (if (fLen >= 64) new DDecode +: (xLen > 32).option(new D64Decode).toSeq else Nil) ++: - (if (minFLen == 16) - new HDecode +: (xLen > 32).option(new H64Decode).toSeq ++: (fLen >= 64).option(new HDDecode).toSeq - else Nil) ++: - (usingRoCC.option(new RoCCDecode)) ++: - (if (xLen == 32) new I32Decode else new I64Decode) +: - (usingVM.option(new SVMDecode)) ++: - (usingSupervisor.option(new SDecode)) ++: - (usingHypervisor.option(new HypervisorDecode)) ++: - ((usingHypervisor && (xLen == 64)).option(new Hypervisor64Decode)) ++: - (usingDebug.option(new DebugDecode)) ++: - (usingNMI.option(new NMIDecode)) ++: - (usingConditionalZero.option(new ConditionalZeroDecode)) ++: - Seq(new FenceIDecode(dcacheFlushOnFenceI || coreParams.haveCFlush)) ++: - coreParams.haveCFlush.option(new CFlushDecode(dcacheCanSupportCFlushLine)) ++: - rocketParams.haveCease.option(new CeaseDecode) ++: - usingVector.option(new VCFGDecode) ++: - (if (coreParams.useZba) new ZbaDecode +: (xLen > 32).option(new Zba64Decode).toSeq else Nil) ++: - (if (coreParams.useZbb) Seq(new ZbbDecode, if (xLen == 32) new Zbb32Decode else new Zbb64Decode) else Nil) ++: - coreParams.useZbs.option(new ZbsDecode) ++: - Seq(new IDecode) - } flatMap (_.table) - - val ex_ctrl = Reg(new IntCtrlSigs) - val mem_ctrl = Reg(new IntCtrlSigs) - val wb_ctrl = Reg(new IntCtrlSigs) - - val ex_reg_xcpt_interrupt = Reg(Bool()) - val ex_reg_valid = Reg(Bool()) - val ex_reg_rvc = Reg(Bool()) - val ex_reg_btb_resp = Reg(new BTBResp) - val ex_reg_xcpt = Reg(Bool()) - val ex_reg_flush_pipe = Reg(Bool()) - val ex_reg_load_use = Reg(Bool()) - val ex_reg_cause = Reg(UInt()) - val ex_reg_replay = Reg(Bool()) - val ex_reg_pc = Reg(UInt()) - val ex_reg_mem_size = Reg(UInt()) - val ex_reg_hls = Reg(Bool()) - val ex_reg_inst = Reg(Bits()) - val ex_reg_raw_inst = Reg(UInt()) - val ex_reg_wphit = Reg(Vec(nBreakpoints, Bool())) - val ex_reg_set_vconfig = Reg(Bool()) - - val mem_reg_xcpt_interrupt = Reg(Bool()) - val mem_reg_valid = Reg(Bool()) - val mem_reg_rvc = Reg(Bool()) - val mem_reg_btb_resp = Reg(new BTBResp) - val mem_reg_xcpt = Reg(Bool()) - val mem_reg_replay = Reg(Bool()) - val mem_reg_flush_pipe = Reg(Bool()) - val mem_reg_cause = Reg(UInt()) - val mem_reg_slow_bypass = Reg(Bool()) - val mem_reg_load = Reg(Bool()) - val mem_reg_store = Reg(Bool()) - val mem_reg_set_vconfig = Reg(Bool()) - val mem_reg_sfence = Reg(Bool()) - val mem_reg_pc = Reg(UInt()) - val mem_reg_inst = Reg(Bits()) - val mem_reg_mem_size = Reg(UInt()) - val mem_reg_hls_or_dv = Reg(Bool()) - val mem_reg_raw_inst = Reg(UInt()) - val mem_reg_wdata = Reg(Bits()) - val mem_reg_rs2 = Reg(Bits()) - val mem_br_taken = Reg(Bool()) - val take_pc_mem = Wire(Bool()) - val mem_reg_wphit = Reg(Vec(nBreakpoints, Bool())) - - val wb_reg_valid = Reg(Bool()) - val wb_reg_xcpt = Reg(Bool()) - val wb_reg_replay = Reg(Bool()) - val wb_reg_flush_pipe = Reg(Bool()) - val wb_reg_cause = Reg(UInt()) - val wb_reg_set_vconfig = Reg(Bool()) - val wb_reg_sfence = Reg(Bool()) - val wb_reg_pc = Reg(UInt()) - val wb_reg_mem_size = Reg(UInt()) - val wb_reg_hls_or_dv = Reg(Bool()) - val wb_reg_hfence_v = Reg(Bool()) - val wb_reg_hfence_g = Reg(Bool()) - val wb_reg_inst = Reg(Bits()) - val wb_reg_raw_inst = Reg(UInt()) - val wb_reg_wdata = Reg(Bits()) - val wb_reg_rs2 = Reg(Bits()) - val take_pc_wb = Wire(Bool()) - val wb_reg_wphit = Reg(Vec(nBreakpoints, Bool())) - - val take_pc_mem_wb = take_pc_wb || take_pc_mem - val take_pc = take_pc_mem_wb - - // decode stage - val ibuf = Module(new IBuf) - val id_expanded_inst = ibuf.io.inst.map(_.bits.inst) - val id_raw_inst = ibuf.io.inst.map(_.bits.raw) - val id_inst = id_expanded_inst.map(_.bits) - ibuf.io.imem <> io.imem.resp - ibuf.io.kill := take_pc - - require(decodeWidth == 1 /* TODO */ && retireWidth == decodeWidth) - require(!(coreParams.useRVE && coreParams.fpu.nonEmpty), "Can't select both RVE and floating-point") - require(!(coreParams.useRVE && coreParams.useHypervisor), "Can't select both RVE and Hypervisor") - val id_ctrl = Wire(new IntCtrlSigs).decode(id_inst(0), decode_table) - - val lgNXRegs = if (coreParams.useRVE) 4 else 5 - val regAddrMask = (1 << lgNXRegs) - 1 - - def decodeReg(x: UInt) = (x.extract(x.getWidth - 1, lgNXRegs).asBool, x(lgNXRegs - 1, 0)) - val (id_raddr3_illegal, id_raddr3) = decodeReg(id_expanded_inst(0).rs3) - val (id_raddr2_illegal, id_raddr2) = decodeReg(id_expanded_inst(0).rs2) - val (id_raddr1_illegal, id_raddr1) = decodeReg(id_expanded_inst(0).rs1) - val (id_waddr_illegal, id_waddr) = decodeReg(id_expanded_inst(0).rd) - - val id_load_use = Wire(Bool()) - val id_reg_fence = RegInit(false.B) - val id_ren = IndexedSeq(id_ctrl.rxs1, id_ctrl.rxs2) - val id_raddr = IndexedSeq(id_raddr1, id_raddr2) - val rf = new RegFile(regAddrMask, xLen) - val id_rs = id_raddr.map(rf.read _) - val ctrl_killd = Wire(Bool()) - - // A scheduler needs the custom ISA even without compute Balls. - val csrP = - if (usingRoCC) p - else p.alterPartial { case CoreRoCCCount => 0 } - - val csr = Module(new CSRFile( - perfEvents, - coreParams.customCSRs.decls, - roccCSRs, - hasBusError - )(csrP) { - override def usingRoCC: Boolean = RocketBB.this.usingRoCC - }) - - val id_csr_en = id_ctrl.csr.isOneOf(CSR.S, CSR.C, CSR.W) - val id_system_insn = id_ctrl.csr === CSR.I - val id_csr_ren = id_ctrl.csr.isOneOf(CSR.S, CSR.C) && id_expanded_inst(0).rs1 === 0.U - val id_csr = Mux(id_system_insn && id_ctrl.mem, CSR.N, Mux(id_csr_ren, CSR.R, id_ctrl.csr)) - val id_csr_flush = id_system_insn || (id_csr_en && !id_csr_ren && csr.io.decode(0).write_flush) - val id_set_vconfig = - Seq(Instructions.VSETVLI, Instructions.VSETIVLI, Instructions.VSETVL).map(_ === id_inst(0)).orR && usingVector.B - - id_ctrl.vec := false.B - if (usingVector) { - val v_decode = rocketParams.vector.get.decoder(p) - v_decode.io.inst := id_inst(0) - v_decode.io.vconfig := csr.io.vector.get.vconfig - when(v_decode.io.legal) { - id_ctrl.legal := !csr.io.vector.get.vconfig.vtype.vill - id_ctrl.fp := v_decode.io.fp - id_ctrl.rocc := false.B - id_ctrl.branch := false.B - id_ctrl.jal := false.B - id_ctrl.jalr := false.B - id_ctrl.rxs2 := v_decode.io.read_rs2 - id_ctrl.rxs1 := v_decode.io.read_rs1 - id_ctrl.mem := false.B - id_ctrl.rfs1 := v_decode.io.read_frs1 - id_ctrl.rfs2 := false.B - id_ctrl.rfs3 := false.B - id_ctrl.wfd := v_decode.io.write_frd - id_ctrl.mul := false.B - id_ctrl.div := false.B - id_ctrl.wxd := v_decode.io.write_rd - id_ctrl.csr := CSR.N - id_ctrl.fence_i := false.B - id_ctrl.fence := false.B - id_ctrl.amo := false.B - id_ctrl.dp := false.B - id_ctrl.vec := true.B - } - } - - val id_illegal_insn = !id_ctrl.legal || - (id_ctrl.mul || id_ctrl.div) && !csr.io.status.isa('m' - 'a') || - id_ctrl.amo && !csr.io.status.isa('a' - 'a') || - id_ctrl.fp && (csr.io.decode(0).fp_illegal || (io.fpu.illegal_rm && !id_ctrl.vec)) || - (id_ctrl.vec) && (csr.io.decode(0).vector_illegal || csr.io.vector.map(_.vconfig.vtype.vill).getOrElse( - false.B - )) || - id_ctrl.dp && !csr.io.status.isa('d' - 'a') || - ibuf.io.inst(0).bits.rvc && !csr.io.status.isa('c' - 'a') || - id_raddr2_illegal && id_ctrl.rxs2 || - id_raddr1_illegal && id_ctrl.rxs1 || - id_waddr_illegal && id_ctrl.wxd || - id_ctrl.rocc && csr.io.decode(0).rocc_illegal || - id_csr_en && (csr.io.decode(0).read_illegal || !id_csr_ren && csr.io.decode(0).write_illegal) || - !ibuf.io.inst(0).bits.rvc && (id_system_insn && csr.io.decode(0).system_illegal) - - val id_virtual_insn = id_ctrl.legal && - ((id_csr_en && !(!id_csr_ren && csr.io.decode(0).write_illegal) && csr.io.decode(0).virtual_access_illegal) || - (!ibuf.io.inst(0).bits.rvc && id_system_insn && csr.io.decode(0).virtual_system_illegal)) - - // stall decode for fences (now, for AMO.rl; later, for AMO.aq and FENCE) - val id_amo_aq = id_inst(0)(26) - val id_amo_rl = id_inst(0)(25) - val id_fence_succ = id_inst(0)(23, 20) - val id_fence_next = id_ctrl.fence || id_ctrl.amo && id_amo_aq - val id_mem_busy = !io.dmem.ordered || io.dmem.req.valid - when(!id_mem_busy)(id_reg_fence := false.B) - - val id_rocc_busy = usingRoCC.B && - (io.rocc.busy || ex_reg_valid && ex_ctrl.rocc || - mem_reg_valid && mem_ctrl.rocc || wb_reg_valid && wb_ctrl.rocc) - - val id_csr_rocc_write = roccCSRs.map(_.id.U === id_inst(0)(31, 20)).orR && id_csr_en && !id_csr_ren - val id_vec_busy = io.vector.map(v => v.backend_busy || v.trap_check_busy).getOrElse(false.B) - - val id_do_fence = WireDefault(id_rocc_busy && (id_ctrl.fence || id_csr_rocc_write) || - id_vec_busy && id_ctrl.fence || - id_mem_busy && (id_ctrl.amo && id_amo_rl || id_ctrl.fence_i || id_reg_fence && (id_ctrl.mem || id_ctrl.rocc))) - - val bpu = Module(new BreakpointUnit(nBreakpoints)) - bpu.io.status := csr.io.status - bpu.io.bp := csr.io.bp - bpu.io.pc := ibuf.io.pc - bpu.io.ea := mem_reg_wdata - bpu.io.mcontext := csr.io.mcontext - bpu.io.scontext := csr.io.scontext - - val id_xcpt0 = ibuf.io.inst(0).bits.xcpt0 - val id_xcpt1 = ibuf.io.inst(0).bits.xcpt1 - - val deliveredInterrupt = csr.io.interrupt && io.interruptReady - - val (id_xcpt, id_cause) = checkExceptions(List( - (deliveredInterrupt, csr.io.interrupt_cause), - (bpu.io.debug_if, CSR.debugTriggerCause.U), - (bpu.io.xcpt_if, Causes.breakpoint.U), - (id_xcpt0.pf.inst, Causes.fetch_page_fault.U), - (id_xcpt0.gf.inst, Causes.fetch_guest_page_fault.U), - (id_xcpt0.ae.inst, Causes.fetch_access.U), - (id_xcpt1.pf.inst, Causes.fetch_page_fault.U), - (id_xcpt1.gf.inst, Causes.fetch_guest_page_fault.U), - (id_xcpt1.ae.inst, Causes.fetch_access.U), - (id_virtual_insn, Causes.virtual_instruction.U), - (id_illegal_insn, Causes.illegal_instruction.U) - )) - - val idCoverCauses = List( - (CSR.debugTriggerCause, "DEBUG_TRIGGER"), - (Causes.breakpoint, "BREAKPOINT"), - (Causes.fetch_access, "FETCH_ACCESS"), - (Causes.illegal_instruction, "ILLEGAL_INSTRUCTION") - ) ++ (if (usingVM) - List( - (Causes.fetch_page_fault, "FETCH_PAGE_FAULT") - ) - else Nil) - - coverExceptions(id_xcpt, id_cause, "DECODE", idCoverCauses) - - val dcache_bypass_data = - if (fastLoadByte) io.dmem.resp.bits.data(xLen - 1, 0) - else if (fastLoadWord) io.dmem.resp.bits.data_word_bypass(xLen - 1, 0) - else wb_reg_wdata - - // detect bypass opportunities - val ex_waddr = ex_reg_inst(11, 7) & regAddrMask.U - val mem_waddr = mem_reg_inst(11, 7) & regAddrMask.U - val wb_waddr = wb_reg_inst(11, 7) & regAddrMask.U - - val bypass_sources = IndexedSeq( - (true.B, 0.U, 0.U), // treat reading x0 as a bypass - (ex_reg_valid && ex_ctrl.wxd, ex_waddr, mem_reg_wdata), - (mem_reg_valid && mem_ctrl.wxd && !mem_ctrl.mem, mem_waddr, wb_reg_wdata), - (mem_reg_valid && mem_ctrl.wxd, mem_waddr, dcache_bypass_data) - ) - - val id_bypass_src = id_raddr.map(raddr => bypass_sources.map(s => s._1 && s._2 === raddr)) - - // execute stage - val bypass_mux = bypass_sources.map(_._3) - val ex_reg_rs_bypass = Reg(Vec(id_raddr.size, Bool())) - val ex_reg_rs_lsb = Reg(Vec(id_raddr.size, UInt(log2Ceil(bypass_sources.size).W))) - val ex_reg_rs_msb = Reg(Vec(id_raddr.size, UInt())) - - val ex_rs = - for (i <- 0 until id_raddr.size) - yield Mux(ex_reg_rs_bypass(i), bypass_mux(ex_reg_rs_lsb(i)), Cat(ex_reg_rs_msb(i), ex_reg_rs_lsb(i))) - - val ex_imm = ImmGen(ex_ctrl.sel_imm, ex_reg_inst) - val ex_rs1shl = Mux(ex_reg_inst(3), ex_rs(0)(31, 0), ex_rs(0)) << ex_reg_inst(14, 13) - - val ex_op1 = MuxLookup(ex_ctrl.sel_alu1, 0.S)(Seq( - A1_RS1 -> ex_rs(0).asSInt, - A1_PC -> ex_reg_pc.asSInt, - A1_RS1SHL -> (if (rocketParams.useZba) ex_rs1shl.asSInt else 0.S) - )) - - val ex_op2_oh = - UIntToOH(Mux(ex_ctrl.sel_alu2(0), (ex_reg_inst >> 20).asUInt, ex_rs(1))(log2Ceil(xLen) - 1, 0)).asSInt - - val ex_op2 = MuxLookup(ex_ctrl.sel_alu2, 0.S)(Seq( - A2_RS2 -> ex_rs(1).asSInt, - A2_IMM -> ex_imm, - A2_SIZE -> Mux(ex_reg_rvc, 2.S, 4.S) - ) ++ (if (coreParams.useZbs) - Seq( - A2_RS2OH -> ex_op2_oh, - A2_IMMOH -> ex_op2_oh - ) - else Nil)) - - val (ex_new_vl, ex_new_vconfig) = - if (usingVector) { - val ex_new_vtype = VType.fromUInt(MuxCase( - ex_rs(1), - Seq( - ex_reg_inst(31, 30).andR -> ex_reg_inst(29, 20), - !ex_reg_inst(31) -> ex_reg_inst(30, 20) - ) - )) - val ex_avl = Mux( - ex_ctrl.rxs1, - Mux( - ex_reg_inst(19, 15) === 0.U, - Mux(ex_reg_inst(11, 7) === 0.U, csr.io.vector.get.vconfig.vl, ex_new_vtype.vlMax), - ex_rs(0) - ), - ex_reg_inst(19, 15) - ) - val ex_new_vl = ex_new_vtype.vl(ex_avl, csr.io.vector.get.vconfig.vl, false.B, false.B, false.B) - val ex_new_vconfig = Wire(new VConfig) - ex_new_vconfig.vtype := ex_new_vtype - ex_new_vconfig.vl := ex_new_vl - (Some(ex_new_vl), Some(ex_new_vconfig)) - } else { (None, None) } - - val alu = Module(new ALU) - alu.io.dw := ex_ctrl.alu_dw - alu.io.fn := ex_ctrl.alu_fn - alu.io.in2 := ex_op2.asUInt - alu.io.in1 := ex_op1.asUInt - - // multiplier and divider - val div = Module(new MulDiv(if (pipelinedMul) mulDivParams.copy(mulUnroll = 0) else mulDivParams, width = xLen)) - div.io.req.valid := ex_reg_valid && ex_ctrl.div - div.io.req.bits.dw := ex_ctrl.alu_dw - div.io.req.bits.fn := ex_ctrl.alu_fn - div.io.req.bits.in1 := ex_rs(0) - div.io.req.bits.in2 := ex_rs(1) - div.io.req.bits.tag := ex_waddr - - val mul = pipelinedMul.option { - val m = Module(new PipelinedMultiplier(xLen, 2)) - m.io.req.valid := ex_reg_valid && ex_ctrl.mul - m.io.req.bits := div.io.req.bits - m - } - - ex_reg_valid := !ctrl_killd - ex_reg_replay := !take_pc && ibuf.io.inst(0).valid && ibuf.io.inst(0).bits.replay - ex_reg_xcpt := !ctrl_killd && id_xcpt - ex_reg_xcpt_interrupt := !take_pc && ibuf.io.inst(0).valid && deliveredInterrupt - - when(!ctrl_killd) { - ex_ctrl := id_ctrl - ex_reg_rvc := ibuf.io.inst(0).bits.rvc - ex_ctrl.csr := id_csr - when(id_ctrl.fence && id_fence_succ === 0.U)(id_reg_pause := true.B) - when(id_fence_next)(id_reg_fence := true.B) - when(id_xcpt) { // pass PC down ALU writeback pipeline for badaddr - ex_ctrl.alu_fn := FN_ADD - ex_ctrl.alu_dw := DW_XPR - ex_ctrl.sel_alu1 := A1_RS1 // badaddr := instruction - ex_ctrl.sel_alu2 := A2_ZERO - when(id_xcpt1.asUInt.orR) { // badaddr := PC+2 - ex_ctrl.sel_alu1 := A1_PC - ex_ctrl.sel_alu2 := A2_SIZE - ex_reg_rvc := true.B - } - when(bpu.io.xcpt_if || id_xcpt0.asUInt.orR) { // badaddr := PC - ex_ctrl.sel_alu1 := A1_PC - ex_ctrl.sel_alu2 := A2_ZERO - } - } - ex_reg_flush_pipe := id_ctrl.fence_i || id_csr_flush - ex_reg_load_use := id_load_use - ex_reg_hls := usingHypervisor.B && id_system_insn && id_ctrl.mem_cmd.isOneOf(M_XRD, M_XWR, M_HLVX) - ex_reg_mem_size := Mux(usingHypervisor.B && id_system_insn, id_inst(0)(27, 26), id_inst(0)(13, 12)) - when(id_ctrl.mem_cmd.isOneOf(M_SFENCE, M_HFENCEV, M_HFENCEG, M_FLUSH_ALL)) { - ex_reg_mem_size := Cat(id_raddr2 =/= 0.U, id_raddr1 =/= 0.U) - } - when(id_ctrl.mem_cmd === M_SFENCE && csr.io.status.v) { - ex_ctrl.mem_cmd := M_HFENCEV - } - if (dcacheFlushOnFenceI || coreParams.haveCFlush) { - when(id_ctrl.fence_i) { - ex_reg_mem_size := 0.U - } - } - - for (i <- 0 until id_raddr.size) { - val do_bypass = id_bypass_src(i).reduce(_ || _) - val bypass_src = PriorityEncoder(id_bypass_src(i)) - ex_reg_rs_bypass(i) := do_bypass - ex_reg_rs_lsb(i) := bypass_src - when(id_ren(i) && !do_bypass) { - ex_reg_rs_lsb(i) := id_rs(i)(log2Ceil(bypass_sources.size) - 1, 0) - ex_reg_rs_msb(i) := id_rs(i) >> log2Ceil(bypass_sources.size) - } - } - when(id_illegal_insn || id_virtual_insn) { - val inst = Mux(ibuf.io.inst(0).bits.rvc, id_raw_inst(0)(15, 0), id_raw_inst(0)) - ex_reg_rs_bypass(0) := false.B - ex_reg_rs_lsb(0) := inst(log2Ceil(bypass_sources.size) - 1, 0) - ex_reg_rs_msb(0) := inst >> log2Ceil(bypass_sources.size) - } - } - when(!ctrl_killd || deliveredInterrupt || ibuf.io.inst(0).bits.replay) { - ex_reg_cause := id_cause - ex_reg_inst := id_inst(0) - ex_reg_raw_inst := id_raw_inst(0) - ex_reg_pc := ibuf.io.pc - ex_reg_btb_resp := ibuf.io.btb_resp - ex_reg_wphit := bpu.io.bpwatch.map(bpw => bpw.ivalid(0)) - ex_reg_set_vconfig := id_set_vconfig && !id_xcpt - } - - // replay inst in ex stage? - val ex_pc_valid = ex_reg_valid || ex_reg_replay || ex_reg_xcpt_interrupt - val wb_dcache_miss = wb_ctrl.mem && !io.dmem.resp.valid - - val replay_ex_structural = ex_ctrl.mem && !io.dmem.req.ready || - ex_ctrl.div && !div.io.req.ready || - ex_ctrl.vec && !io.vector.map(_.ex.ready).getOrElse(true.B) - - val replay_ex_load_use = wb_dcache_miss && ex_reg_load_use - val replay_ex = ex_reg_replay || (ex_reg_valid && (replay_ex_structural || replay_ex_load_use)) - val ctrl_killx = take_pc_mem_wb || replay_ex || !ex_reg_valid - // detect 2-cycle load-use delay for LB/LH/SC - val ex_slow_bypass = ex_ctrl.mem_cmd === M_XSC || ex_reg_mem_size < 2.U - val ex_sfence = - usingVM.B && ex_ctrl.mem && (ex_ctrl.mem_cmd === M_SFENCE || ex_ctrl.mem_cmd === M_HFENCEV || ex_ctrl.mem_cmd === M_HFENCEG) - - val (ex_xcpt, ex_cause) = checkExceptions(List( - (ex_reg_xcpt_interrupt || ex_reg_xcpt, ex_reg_cause) - )) - - val exCoverCauses = idCoverCauses - coverExceptions(ex_xcpt, ex_cause, "EXECUTE", exCoverCauses) - - // memory stage - val mem_pc_valid = mem_reg_valid || mem_reg_replay || mem_reg_xcpt_interrupt - - val mem_br_target = mem_reg_pc.asSInt + - Mux( - mem_ctrl.branch && mem_br_taken, - ImmGen(IMM_SB, mem_reg_inst), - Mux(mem_ctrl.jal, ImmGen(IMM_UJ, mem_reg_inst), Mux(mem_reg_rvc, 2.S, 4.S)) - ) - - val mem_npc = (Mux( - mem_ctrl.jalr || mem_reg_sfence, - encodeVirtualAddress(mem_reg_wdata, mem_reg_wdata).asSInt, - mem_br_target - ) & -2.S).asUInt - - val mem_wrong_npc = - Mux( - ex_pc_valid, - mem_npc =/= ex_reg_pc, - Mux(ibuf.io.inst(0).valid || ibuf.io.imem.valid, mem_npc =/= ibuf.io.pc, true.B) - ) - - val mem_npc_misaligned = !csr.io.status.isa('c' - 'a') && mem_npc(1) && !mem_reg_sfence - val mem_int_wdata = - Mux(!mem_reg_xcpt && (mem_ctrl.jalr ^ mem_npc_misaligned), mem_br_target, mem_reg_wdata.asSInt).asUInt - val mem_cfi = mem_ctrl.branch || mem_ctrl.jalr || mem_ctrl.jal - val mem_cfi_taken = (mem_ctrl.branch && mem_br_taken) || mem_ctrl.jalr || mem_ctrl.jal - val mem_direction_misprediction = mem_ctrl.branch && mem_br_taken =/= (usingBTB.B && mem_reg_btb_resp.taken) - val mem_misprediction = if (usingBTB) mem_wrong_npc else mem_cfi_taken - take_pc_mem := mem_reg_valid && !mem_reg_xcpt && (mem_misprediction || mem_reg_sfence) - - mem_reg_valid := !ctrl_killx - mem_reg_replay := !take_pc_mem_wb && replay_ex - mem_reg_xcpt := !ctrl_killx && ex_xcpt - mem_reg_xcpt_interrupt := !take_pc_mem_wb && ex_reg_xcpt_interrupt - - // on pipeline flushes, cause mem_npc to hold the sequential npc, which - // will drive the W-stage npc mux - when(mem_reg_valid && mem_reg_flush_pipe) { - mem_reg_sfence := false.B - }.elsewhen(ex_pc_valid) { - mem_ctrl := ex_ctrl - mem_reg_rvc := ex_reg_rvc - mem_reg_load := ex_ctrl.mem && isRead(ex_ctrl.mem_cmd) - mem_reg_store := ex_ctrl.mem && isWrite(ex_ctrl.mem_cmd) - mem_reg_sfence := ex_sfence - mem_reg_btb_resp := ex_reg_btb_resp - mem_reg_flush_pipe := ex_reg_flush_pipe - mem_reg_slow_bypass := ex_slow_bypass - mem_reg_wphit := ex_reg_wphit - mem_reg_set_vconfig := ex_reg_set_vconfig - - mem_reg_cause := ex_cause - mem_reg_inst := ex_reg_inst - mem_reg_raw_inst := ex_reg_raw_inst - mem_reg_mem_size := ex_reg_mem_size - mem_reg_hls_or_dv := io.dmem.req.bits.dv - mem_reg_pc := ex_reg_pc - // IDecode ensured they are 1H - mem_reg_wdata := Mux(ex_reg_set_vconfig, ex_new_vl.getOrElse(alu.io.out), alu.io.out) - mem_br_taken := alu.io.cmp_out - - when(ex_ctrl.rxs2 && (ex_ctrl.mem || ex_ctrl.rocc || ex_sfence)) { - val size = Mux(ex_ctrl.rocc, log2Ceil(xLen / 8).U, ex_reg_mem_size) - mem_reg_rs2 := new StoreGen(size, 0.U, ex_rs(1), coreDataBytes).data - } - if (usingVector) { - when(ex_reg_set_vconfig) { - mem_reg_rs2 := ex_new_vconfig.get.asUInt - } - } - when(ex_ctrl.jalr && csr.io.status.debug) { - // flush I$ on D-mode JALR to effect uncached fetch without D$ flush - mem_ctrl.fence_i := true.B - mem_reg_flush_pipe := true.B - } - } - - val mem_breakpoint = (mem_reg_load && bpu.io.xcpt_ld) || (mem_reg_store && bpu.io.xcpt_st) - val mem_debug_breakpoint = (mem_reg_load && bpu.io.debug_ld) || (mem_reg_store && bpu.io.debug_st) - - val (mem_ldst_xcpt, mem_ldst_cause) = checkExceptions(List( - (mem_debug_breakpoint, CSR.debugTriggerCause.U), - (mem_breakpoint, Causes.breakpoint.U) - )) - - val (mem_xcpt, mem_cause) = checkExceptions(List( - (mem_reg_xcpt_interrupt || mem_reg_xcpt, mem_reg_cause), - (mem_reg_valid && mem_npc_misaligned, Causes.misaligned_fetch.U), - (mem_reg_valid && mem_ldst_xcpt, mem_ldst_cause) - )) - - val memCoverCauses = (exCoverCauses ++ List( - (CSR.debugTriggerCause, "DEBUG_TRIGGER"), - (Causes.breakpoint, "BREAKPOINT"), - (Causes.misaligned_fetch, "MISALIGNED_FETCH") - )).distinct - - coverExceptions(mem_xcpt, mem_cause, "MEMORY", memCoverCauses) - - val dcache_kill_mem = mem_reg_valid && mem_ctrl.wxd && io.dmem.replay_next // structural hazard on writeback port - val fpu_kill_mem = mem_reg_valid && mem_ctrl.fp && io.fpu.nack_mem - val vec_kill_mem = mem_reg_valid && mem_ctrl.mem && io.vector.map(_.mem.block_mem).getOrElse(false.B) - val vec_kill_all = mem_reg_valid && io.vector.map(_.mem.block_all).getOrElse(false.B) - val replay_mem = dcache_kill_mem || mem_reg_replay || fpu_kill_mem || vec_kill_mem || vec_kill_all - val killm_common = dcache_kill_mem || take_pc_wb || mem_reg_xcpt || !mem_reg_valid - div.io.kill := killm_common && RegNext(div.io.req.fire) - val ctrl_killm = killm_common || mem_xcpt || fpu_kill_mem || vec_kill_mem - - // writeback stage - wb_reg_valid := !ctrl_killm - wb_reg_replay := replay_mem && !take_pc_wb - wb_reg_xcpt := mem_xcpt && !take_pc_wb && !io.vector.map(_.mem.block_all).getOrElse(false.B) - wb_reg_flush_pipe := !ctrl_killm && mem_reg_flush_pipe - when(mem_pc_valid) { - wb_ctrl := mem_ctrl - wb_reg_sfence := mem_reg_sfence - wb_reg_wdata := Mux(!mem_reg_xcpt && mem_ctrl.fp && mem_ctrl.wxd, io.fpu.toint_data, mem_int_wdata) - when(mem_ctrl.rocc || mem_reg_sfence || mem_reg_set_vconfig) { - wb_reg_rs2 := mem_reg_rs2 - } - wb_reg_cause := mem_cause - wb_reg_inst := mem_reg_inst - wb_reg_raw_inst := mem_reg_raw_inst - wb_reg_mem_size := mem_reg_mem_size - wb_reg_hls_or_dv := mem_reg_hls_or_dv - wb_reg_hfence_v := mem_ctrl.mem_cmd === M_HFENCEV - wb_reg_hfence_g := mem_ctrl.mem_cmd === M_HFENCEG - wb_reg_pc := mem_reg_pc - wb_reg_wphit := mem_reg_wphit | bpu.io.bpwatch.map { bpw => - (bpw.rvalid(0) && mem_reg_load) || (bpw.wvalid(0) && mem_reg_store) - } - wb_reg_set_vconfig := mem_reg_set_vconfig - } - - val (wb_xcpt, wb_cause) = checkExceptions(List( - (wb_reg_xcpt, wb_reg_cause), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.pf.st, Causes.store_page_fault.U), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.pf.ld, Causes.load_page_fault.U), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.gf.st, Causes.store_guest_page_fault.U), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.gf.ld, Causes.load_guest_page_fault.U), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ae.st, Causes.store_access.U), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ae.ld, Causes.load_access.U), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ma.st, Causes.misaligned_store.U), - (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ma.ld, Causes.misaligned_load.U) - )) - - val wbCoverCauses = List( - (Causes.misaligned_store, "MISALIGNED_STORE"), - (Causes.misaligned_load, "MISALIGNED_LOAD"), - (Causes.store_access, "STORE_ACCESS"), - (Causes.load_access, "LOAD_ACCESS") - ) ++ (if (usingVM) - List( - (Causes.store_page_fault, "STORE_PAGE_FAULT"), - (Causes.load_page_fault, "LOAD_PAGE_FAULT") - ) - else Nil) ++ (if (usingHypervisor) - List( - (Causes.store_guest_page_fault, "STORE_GUEST_PAGE_FAULT"), - (Causes.load_guest_page_fault, "LOAD_GUEST_PAGE_FAULT") - ) - else Nil) - - coverExceptions(wb_xcpt, wb_cause, "WRITEBACK", wbCoverCauses) - - val wb_pc_valid = wb_reg_valid || wb_reg_replay || wb_reg_xcpt - val wb_wxd = wb_reg_valid && wb_ctrl.wxd - val wb_set_sboard = wb_ctrl.div || wb_dcache_miss || wb_ctrl.rocc || wb_ctrl.vec - val replay_wb_common = io.dmem.s2_nack || wb_reg_replay - val replay_wb_rocc = wb_reg_valid && wb_ctrl.rocc && !io.rocc.cmd.ready - val replay_wb_csr: Bool = wb_reg_valid && csr.io.rw_stall - val replay_wb_vec = wb_reg_valid && io.vector.map(_.wb.replay).getOrElse(false.B) - val replay_wb = replay_wb_common || replay_wb_rocc || replay_wb_csr || replay_wb_vec - take_pc_wb := replay_wb || wb_xcpt || csr.io.eret || wb_reg_flush_pipe - - // writeback arbitration - val dmem_resp_xpu = !io.dmem.resp.bits.tag(0).asBool - val dmem_resp_fpu = io.dmem.resp.bits.tag(0).asBool - val dmem_resp_waddr = io.dmem.resp.bits.tag(5, 1) - val dmem_resp_valid = io.dmem.resp.valid && io.dmem.resp.bits.has_data - val dmem_resp_replay = dmem_resp_valid && io.dmem.resp.bits.replay - - class LLWB extends Bundle { - val data = UInt(xLen.W) - val tag = UInt(5.W) - } - - val ll_arb = Module(new Arbiter(new LLWB, 3)) // div, rocc, vec - ll_arb.io.in.foreach(_.valid := false.B) - ll_arb.io.in.foreach(_.bits := DontCare) - val ll_wdata = WireInit(ll_arb.io.out.bits.data) - val ll_waddr = WireInit(ll_arb.io.out.bits.tag) - val ll_wen = WireInit(ll_arb.io.out.fire) - ll_arb.io.out.ready := !wb_wxd - - div.io.resp.ready := ll_arb.io.in(0).ready - ll_arb.io.in(0).valid := div.io.resp.valid - ll_arb.io.in(0).bits.data := div.io.resp.bits.data - ll_arb.io.in(0).bits.tag := div.io.resp.bits.tag - - if (usingRoCC) { - io.rocc.resp.ready := ll_arb.io.in(1).ready - ll_arb.io.in(1).valid := io.rocc.resp.valid - ll_arb.io.in(1).bits.data := io.rocc.resp.bits.data - ll_arb.io.in(1).bits.tag := io.rocc.resp.bits.rd - } else { - // tie off RoCC - io.rocc.resp.ready := false.B - io.rocc.mem.req.ready := false.B - } - - io.vector.map { v => - v.resp.ready := Mux(v.resp.bits.fp, !(dmem_resp_valid && dmem_resp_fpu), ll_arb.io.in(2).ready) - ll_arb.io.in(2).valid := v.resp.valid && !v.resp.bits.fp - ll_arb.io.in(2).bits.data := v.resp.bits.data - ll_arb.io.in(2).bits.tag := v.resp.bits.rd - } - // Dont care mem since not all RoCC need accessing memory - io.rocc.mem := DontCare - - when(dmem_resp_replay && dmem_resp_xpu) { - ll_arb.io.out.ready := false.B - ll_waddr := dmem_resp_waddr - ll_wen := true.B - } - - val wb_valid = wb_reg_valid && !replay_wb && !wb_xcpt - val wb_wen = wb_valid && wb_ctrl.wxd - val rf_wen = wb_wen || ll_wen - val rf_waddr = Mux(ll_wen, ll_waddr, wb_waddr) - - val rf_wdata = Mux( - dmem_resp_valid && dmem_resp_xpu, - io.dmem.resp.bits.data(xLen - 1, 0), - Mux( - ll_wen, - ll_wdata, - Mux( - wb_ctrl.csr =/= CSR.N, - csr.io.rw.rdata, - Mux(wb_ctrl.mul, mul.map(_.io.resp.bits.data).getOrElse(wb_reg_wdata), wb_reg_wdata) - ) - ) - ) - - when(rf_wen)(rf.write(rf_waddr, rf_wdata)) - - // hook up control/status regfile - csr.io.ungated_clock := clock - csr.io.decode(0).inst := id_inst(0) - csr.io.exception := wb_xcpt - csr.io.cause := wb_cause - csr.io.retire := wb_valid - csr.io.inst(0) := (if (usingCompressed) - Cat(Mux(wb_reg_raw_inst(1, 0).andR, wb_reg_inst >> 16, 0.U), wb_reg_raw_inst(15, 0)) - else wb_reg_inst) - csr.io.interrupts := io.interrupts - csr.io.hartid := io.hartid - io.fpu.fcsr_rm := csr.io.fcsr_rm - val vector_fcsr_flags = io.vector.map(_.set_fflags.bits).getOrElse(0.U(5.W)) - val vector_fcsr_flags_valid = io.vector.map(_.set_fflags.valid).getOrElse(false.B) - csr.io.fcsr_flags.valid := io.fpu.fcsr_flags.valid | vector_fcsr_flags_valid - csr.io.fcsr_flags.bits := (io.fpu.fcsr_flags.bits & Fill(5, io.fpu.fcsr_flags.valid)) | (vector_fcsr_flags & Fill( - 5, - vector_fcsr_flags_valid - )) - io.fpu.time := csr.io.time(31, 0) - io.fpu.hartid := io.hartid - csr.io.rocc_interrupt := io.rocc.interrupt - csr.io.pc := wb_reg_pc - - val tval_dmem_addr = !wb_reg_xcpt - - val tval_any_addr = tval_dmem_addr || - wb_reg_cause.isOneOf( - Causes.breakpoint.U, - Causes.fetch_access.U, - Causes.fetch_page_fault.U, - Causes.fetch_guest_page_fault.U - ) - - val tval_inst = wb_reg_cause === Causes.illegal_instruction.U - val tval_valid = wb_xcpt && (tval_any_addr || tval_inst) - csr.io.gva := wb_xcpt && (tval_any_addr && csr.io.status.v || tval_dmem_addr && wb_reg_hls_or_dv) - csr.io.tval := Mux(tval_valid, encodeVirtualAddress(wb_reg_wdata, wb_reg_wdata), 0.U) - - val (htval, mhtinst_read_pseudo) = { - val htval_valid_imem = wb_reg_xcpt && wb_reg_cause === Causes.fetch_guest_page_fault.U - val htval_imem = Mux(htval_valid_imem, io.imem.gpa.bits, 0.U) - assert(!htval_valid_imem || io.imem.gpa.valid) - - val htval_valid_dmem = - wb_xcpt && tval_dmem_addr && io.dmem.s2_xcpt.gf.asUInt.orR && !io.dmem.s2_xcpt.pf.asUInt.orR - val htval_dmem = Mux(htval_valid_dmem, io.dmem.s2_gpa, 0.U) - - val htval = (htval_dmem | htval_imem) >> hypervisorExtraAddrBits - // read pseudoinstruction if a guest-page fault is caused by an implicit memory access for VS-stage address - // translation - val mhtinst_read_pseudo = (io.imem.gpa_is_pte && htval_valid_imem) || (io.dmem.s2_gpa_is_pte && htval_valid_dmem) - (htval, mhtinst_read_pseudo) - } - - csr.io.vector.foreach { v => - v.set_vconfig.valid := wb_reg_set_vconfig && wb_reg_valid - v.set_vconfig.bits := wb_reg_rs2.asTypeOf(new VConfig) - v.set_vs_dirty := wb_valid && wb_ctrl.vec - v.set_vstart.valid := wb_valid && wb_reg_set_vconfig - v.set_vstart.bits := 0.U - } - - io.vector.foreach { v => - when(v.wb.retire || v.wb.xcpt || wb_ctrl.vec) { - csr.io.pc := v.wb.pc - csr.io.retire := v.wb.retire - csr.io.inst(0) := v.wb.inst - when(v.wb.xcpt && !wb_reg_xcpt) { - wb_xcpt := true.B - wb_cause := v.wb.cause - csr.io.tval := v.wb.tval - } - } - v.wb.store_pending := io.dmem.store_pending - v.wb.vxrm := csr.io.vector.get.vxrm - v.wb.frm := csr.io.fcsr_rm - csr.io.vector.get.set_vxsat := v.set_vxsat - when(v.set_vconfig.valid) { - csr.io.vector.get.set_vconfig.valid := true.B - csr.io.vector.get.set_vconfig.bits := v.set_vconfig.bits - } - when(v.set_vstart.valid) { - csr.io.vector.get.set_vstart.valid := true.B - csr.io.vector.get.set_vstart.bits := v.set_vstart.bits - } - } - - csr.io.htval := htval - csr.io.mhtinst_read_pseudo := mhtinst_read_pseudo - io.ptw.ptbr := csr.io.ptbr - io.ptw.hgatp := csr.io.hgatp - io.ptw.vsatp := csr.io.vsatp - (io.ptw.customCSRs.csrs zip csr.io.customCSRs).map { case (lhs, rhs) => lhs <> rhs } - io.ptw.status := csr.io.status - io.ptw.hstatus := csr.io.hstatus - io.ptw.gstatus := csr.io.gstatus - io.ptw.pmp := csr.io.pmp - csr.io.rw.addr := wb_reg_inst(31, 20) - csr.io.rw.cmd := CSR.maskCmd(wb_reg_valid, wb_ctrl.csr) - csr.io.rw.wdata := wb_reg_wdata - - io.rocc.csrs <> csr.io.roccCSRs - io.trace.time := csr.io.time - io.trace.insns := csr.io.trace - if (rocketParams.debugROB.isDefined) { - val sz = rocketParams.debugROB.get.size - if (sz < 1) { // use unsynthesizable ROB - val csr_trace_with_wdata = WireInit(csr.io.trace(0)) - csr_trace_with_wdata.wdata.get := rf_wdata - val should_wb = WireInit((wb_ctrl.wfd || (wb_ctrl.wxd && wb_waddr =/= 0.U)) && !csr.io.trace(0).exception) - val has_wb = WireInit(wb_ctrl.wxd && wb_wen && !wb_set_sboard) - val wb_addr = WireInit(wb_waddr + Mux(wb_ctrl.wfd, 32.U, 0.U)) - - io.vector.foreach { v => - when(v.wb.retire) { - should_wb := v.wb.rob_should_wb - has_wb := false.B - wb_addr := Cat(v.wb.rob_should_wb_fp, csr_trace_with_wdata.insn(11, 7)) - } - } - - DebugROB.pushTrace(clock, reset, io.hartid, csr_trace_with_wdata, should_wb, has_wb, wb_addr) - - io.trace.insns(0) := DebugROB.popTrace(clock, reset, io.hartid) - - DebugROB.pushWb(clock, reset, io.hartid, ll_wen, rf_waddr, rf_wdata) - } else { // synthesizable ROB (no FPRs) - require(!usingVector, "Synthesizable ROB does not support vector implementations") - val csr_trace_with_wdata = WireInit(csr.io.trace(0)) - csr_trace_with_wdata.wdata.get := rf_wdata - - val debug_rob = Module(new HardDebugROB(sz, 32)) - debug_rob.io.i_insn := csr_trace_with_wdata - debug_rob.io.should_wb := (wb_ctrl.wfd || (wb_ctrl.wxd && wb_waddr =/= 0.U)) && - !csr.io.trace(0).exception - debug_rob.io.has_wb := wb_ctrl.wxd && wb_wen && !wb_set_sboard - debug_rob.io.tag := wb_waddr + Mux(wb_ctrl.wfd, 32.U, 0.U) - - debug_rob.io.wb_val := ll_wen - debug_rob.io.wb_tag := rf_waddr - debug_rob.io.wb_data := rf_wdata - - io.trace.insns(0) := debug_rob.io.o_insn - } - } else { - io.trace.insns := csr.io.trace - } - for (((iobpw, wphit), bp) <- io.bpwatch zip wb_reg_wphit zip csr.io.bp) { - iobpw.valid(0) := wphit - iobpw.action := bp.control.action - // tie off bpwatch valids - iobpw.rvalid.foreach(_ := false.B) - iobpw.wvalid.foreach(_ := false.B) - iobpw.ivalid.foreach(_ := false.B) - } - - val hazard_targets = Seq( - (id_ctrl.rxs1 && id_raddr1 =/= 0.U, id_raddr1), - (id_ctrl.rxs2 && id_raddr2 =/= 0.U, id_raddr2), - (id_ctrl.wxd && id_waddr =/= 0.U, id_waddr) - ) - - val fp_hazard_targets = Seq( - (io.fpu.dec.ren1, id_raddr1), - (io.fpu.dec.ren2, id_raddr2), - (io.fpu.dec.ren3, id_raddr3), - (io.fpu.dec.wen, id_waddr) - ) - - val sboard = new Scoreboard(32, true) - sboard.clear(ll_wen, ll_waddr) - - def id_sboard_clear_bypass(r: UInt) = - // ll_waddr arrives late when D$ has ECC, so reshuffle the hazard check - if (!tileParams.dcache.get.dataECC.isDefined) ll_wen && ll_waddr === r - else div.io.resp.fire && div.io.resp.bits.tag === r || dmem_resp_replay && dmem_resp_xpu && dmem_resp_waddr === r - - val id_sboard_hazard = checkHazards(hazard_targets, rd => sboard.read(rd) && !id_sboard_clear_bypass(rd)) - sboard.set(wb_set_sboard && wb_wen, wb_waddr) - - // stall for RAW/WAW hazards on CSRs, loads, AMOs, and mul/div in execute stage. - val ex_cannot_bypass = - ex_ctrl.csr =/= CSR.N || ex_ctrl.jalr || ex_ctrl.mem || ex_ctrl.mul || ex_ctrl.div || ex_ctrl.fp || ex_ctrl.rocc || ex_ctrl.vec - val data_hazard_ex = ex_ctrl.wxd && checkHazards(hazard_targets, _ === ex_waddr) - val fp_data_hazard_ex = id_ctrl.fp && ex_ctrl.wfd && checkHazards(fp_hazard_targets, _ === ex_waddr) - val id_ex_hazard = ex_reg_valid && (data_hazard_ex && ex_cannot_bypass || fp_data_hazard_ex) - - // stall for RAW/WAW hazards on CSRs, LB/LH, and mul/div in memory stage. - val mem_mem_cmd_bh = - if (fastLoadWord) (!fastLoadByte).B && mem_reg_slow_bypass - else true.B - - val mem_cannot_bypass = - mem_ctrl.csr =/= CSR.N || mem_ctrl.mem && mem_mem_cmd_bh || mem_ctrl.mul || mem_ctrl.div || mem_ctrl.fp || mem_ctrl.rocc || mem_ctrl.vec - val data_hazard_mem = mem_ctrl.wxd && checkHazards(hazard_targets, _ === mem_waddr) - val fp_data_hazard_mem = id_ctrl.fp && mem_ctrl.wfd && checkHazards(fp_hazard_targets, _ === mem_waddr) - val id_mem_hazard = mem_reg_valid && (data_hazard_mem && mem_cannot_bypass || fp_data_hazard_mem) - id_load_use := mem_reg_valid && data_hazard_mem && mem_ctrl.mem - - val id_vconfig_hazard = id_ctrl.vec && ( - (ex_reg_valid && ex_reg_set_vconfig) || - (mem_reg_valid && mem_reg_set_vconfig) || - (wb_reg_valid && wb_reg_set_vconfig) - ) - - // stall for RAW/WAW hazards on load/AMO misses and mul/div in writeback. - val data_hazard_wb = wb_ctrl.wxd && checkHazards(hazard_targets, _ === wb_waddr) - val fp_data_hazard_wb = id_ctrl.fp && wb_ctrl.wfd && checkHazards(fp_hazard_targets, _ === wb_waddr) - val id_wb_hazard = wb_reg_valid && (data_hazard_wb && wb_set_sboard || fp_data_hazard_wb) - - val id_stall_fpu = - if (usingFPU) { - val fp_sboard = new Scoreboard(32) - fp_sboard.set(((wb_dcache_miss || wb_ctrl.vec) && wb_ctrl.wfd || io.fpu.sboard_set) && wb_valid, wb_waddr) - val v_ll = io.vector.map(v => v.resp.fire && v.resp.bits.fp).getOrElse(false.B) - fp_sboard.clear((dmem_resp_replay && dmem_resp_fpu) || v_ll, io.fpu.ll_resp_tag) - fp_sboard.clear(io.fpu.sboard_clr, io.fpu.sboard_clra) - - checkHazards(fp_hazard_targets, fp_sboard.read _) - } else false.B - - val dcache_blocked = { - // speculate that a blocked D$ will unblock the cycle after a Grant - val blocked = Reg(Bool()) - blocked := !io.dmem.req.ready && io.dmem.clock_enabled && !io.dmem.perf.grant && (blocked || io.dmem.req.valid || io.dmem.s2_nack) - blocked && !io.dmem.perf.grant - } - - val rocc_blocked = RegInit(false.B) - rocc_blocked := !wb_xcpt && !io.rocc.cmd.ready && (io.rocc.cmd.valid || rocc_blocked) - - val ctrl_stalld = - id_ex_hazard || id_mem_hazard || id_wb_hazard || id_sboard_hazard || - id_vconfig_hazard || - csr.io.singleStep && (ex_reg_valid || mem_reg_valid || wb_reg_valid) || - id_csr_en && csr.io.decode(0).fp_csr && !io.fpu.fcsr_rdy || - id_csr_en && csr.io.decode(0).vector_csr && id_vec_busy || - id_ctrl.fp && id_stall_fpu || - id_ctrl.mem && dcache_blocked || // reduce activity during D$ misses - id_ctrl.rocc && rocc_blocked || // reduce activity while RoCC is busy - id_ctrl.div && (!(div.io.req.ready || (div.io.resp.valid && !wb_wxd)) || div.io.req.valid) || // reduce odds of replay - !clock_en || - id_do_fence || io.rocc.busy || - csr.io.csr_stall || - id_reg_pause || - io.traceStall - - ctrl_killd := !ibuf.io.inst(0).valid || ibuf.io.inst( - 0 - ).bits.replay || take_pc_mem_wb || ctrl_stalld || deliveredInterrupt - - io.imem.req.valid := take_pc - io.imem.req.bits.speculative := !take_pc_wb - io.imem.req.bits.pc := - Mux( - wb_xcpt || csr.io.eret, - csr.io.evec, // exception or [m|s]ret - Mux( - replay_wb, - wb_reg_pc, // replay - mem_npc - ) - ) // flush or branch misprediction - io.imem.flush_icache := wb_reg_valid && wb_ctrl.fence_i && !io.dmem.s2_nack - io.imem.might_request := { - imem_might_request_reg := ex_pc_valid || mem_pc_valid || io.ptw.customCSRs.disableICacheClockGate || io.vector.map( - _.trap_check_busy - ).getOrElse(false.B) - imem_might_request_reg - } - io.imem.progress := RegNext(wb_reg_valid && !replay_wb_common) - io.imem.sfence.valid := wb_reg_valid && wb_reg_sfence - io.imem.sfence.bits.rs1 := wb_reg_mem_size(0) - io.imem.sfence.bits.rs2 := wb_reg_mem_size(1) - io.imem.sfence.bits.addr := wb_reg_wdata - io.imem.sfence.bits.asid := wb_reg_rs2 - io.imem.sfence.bits.hv := wb_reg_hfence_v - io.imem.sfence.bits.hg := wb_reg_hfence_g - io.ptw.sfence := io.imem.sfence - - ibuf.io.inst(0).ready := !ctrl_stalld - - io.imem.btb_update.valid := mem_reg_valid && !take_pc_wb && mem_wrong_npc && (!mem_cfi || mem_cfi_taken) - io.imem.btb_update.bits.isValid := mem_cfi - io.imem.btb_update.bits.cfiType := - Mux( - (mem_ctrl.jal || mem_ctrl.jalr) && mem_waddr(0), - CFIType.call, - Mux( - mem_ctrl.jalr && (mem_reg_inst(19, 15) & regAddrMask.U) === BitPat("b00?01"), - CFIType.ret, - Mux(mem_ctrl.jal || mem_ctrl.jalr, CFIType.jump, CFIType.branch) - ) - ) - io.imem.btb_update.bits.target := io.imem.req.bits.pc - io.imem.btb_update.bits.br_pc := (if (usingCompressed) mem_reg_pc + Mux(mem_reg_rvc, 0.U, 2.U) else mem_reg_pc) - io.imem.btb_update.bits.pc := ~(~io.imem.btb_update.bits.br_pc | (coreInstBytes * fetchWidth - 1).U) - io.imem.btb_update.bits.prediction := mem_reg_btb_resp - io.imem.btb_update.bits.taken := DontCare - - io.imem.bht_update.valid := mem_reg_valid && !take_pc_wb - io.imem.bht_update.bits.pc := io.imem.btb_update.bits.pc - io.imem.bht_update.bits.taken := mem_br_taken - io.imem.bht_update.bits.mispredict := mem_wrong_npc - io.imem.bht_update.bits.branch := mem_ctrl.branch - io.imem.bht_update.bits.prediction := mem_reg_btb_resp.bht - - // Connect RAS in Frontend - io.imem.ras_update := DontCare - - io.fpu.valid := !ctrl_killd && id_ctrl.fp - io.fpu.killx := ctrl_killx - io.fpu.killm := killm_common - io.fpu.inst := id_inst(0) - io.fpu.fromint_data := ex_rs(0) - io.fpu.ll_resp_val := dmem_resp_valid && dmem_resp_fpu - io.fpu.ll_resp_data := (if (minFLen == 32) io.dmem.resp.bits.data_word_bypass else io.dmem.resp.bits.data) - io.fpu.ll_resp_type := io.dmem.resp.bits.size - io.fpu.ll_resp_tag := dmem_resp_waddr - io.fpu.keep_clock_enabled := io.ptw.customCSRs.disableCoreClockGate - - io.fpu.v_sew := csr.io.vector.map(_.vconfig.vtype.vsew).getOrElse(0.U) - - io.vector.map { v => - when(!(dmem_resp_valid && dmem_resp_fpu)) { - io.fpu.ll_resp_val := v.resp.valid && v.resp.bits.fp - io.fpu.ll_resp_data := v.resp.bits.data - io.fpu.ll_resp_type := v.resp.bits.size - io.fpu.ll_resp_tag := v.resp.bits.rd - } - } - - io.vector.foreach { v => - v.ex.valid := ex_reg_valid && (ex_ctrl.vec || rocketParams.vector.get.issueVConfig.B && ex_reg_set_vconfig) && !ctrl_killx - v.ex.inst := ex_reg_inst - v.ex.vconfig := csr.io.vector.get.vconfig - v.ex.vstart := Mux(mem_reg_valid && mem_ctrl.vec || wb_reg_valid && wb_ctrl.vec, 0.U, csr.io.vector.get.vstart) - v.ex.rs1 := ex_rs(0) - v.ex.rs2 := ex_rs(1) - v.ex.pc := ex_reg_pc - v.mem.frs1 := io.fpu.store_data - v.killm := killm_common - v.status := csr.io.status - } - - io.dmemPc := ex_reg_pc - io.dmem.req.valid := ex_reg_valid && ex_ctrl.mem - val ex_dcache_tag = Cat(ex_waddr, ex_ctrl.fp) - require(coreParams.dcacheReqTagBits >= ex_dcache_tag.getWidth) - io.dmem.req.bits.tag := ex_dcache_tag - io.dmem.req.bits.cmd := ex_ctrl.mem_cmd - io.dmem.req.bits.size := ex_reg_mem_size - io.dmem.req.bits.signed := !Mux(ex_reg_hls, ex_reg_inst(20), ex_reg_inst(14)) - io.dmem.req.bits.phys := false.B - io.dmem.req.bits.addr := encodeVirtualAddress(ex_rs(0), alu.io.adder_out) - io.dmem.req.bits.idx.foreach(_ := io.dmem.req.bits.addr) - io.dmem.req.bits.dprv := Mux(ex_reg_hls, csr.io.hstatus.spvp, csr.io.status.dprv) - io.dmem.req.bits.dv := ex_reg_hls || csr.io.status.dv - io.dmem.req.bits.no_resp := !isRead(ex_ctrl.mem_cmd) || (!ex_ctrl.fp && ex_waddr === 0.U) - io.dmem.req.bits.no_alloc := DontCare - io.dmem.req.bits.no_xcpt := DontCare - io.dmem.req.bits.data := DontCare - io.dmem.req.bits.mask := DontCare - - io.dmem.s1_data.data := (if (fLen == 0) mem_reg_rs2 - else Mux(mem_ctrl.fp, Fill(coreDataBits / fLen, io.fpu.store_data), mem_reg_rs2)) - io.dmem.s1_data.mask := DontCare - - io.dmem.s1_kill := killm_common || mem_ldst_xcpt || fpu_kill_mem || vec_kill_mem - io.dmem.s2_kill := false.B - // don't let D$ go to sleep if we're probably going to use it soon - io.dmem.keep_clock_enabled := ibuf.io.inst(0).valid && id_ctrl.mem && !csr.io.csr_stall - - // Pipeline classification, not a second accelerator decoder. Readiness is deliberately absent. - io.roccOlderPending := usingRoCC.B && - ((mem_reg_valid && mem_ctrl.rocc) || (wb_reg_valid && wb_ctrl.rocc)) - io.roccCommitEligible := usingRoCC.B && wb_reg_valid && wb_ctrl.rocc && !replay_wb_common && !wb_xcpt - io.rocc.cmd.valid := wb_reg_valid && wb_ctrl.rocc && !replay_wb_common - io.rocc.exception := wb_xcpt && csr.io.status.xs.orR - io.rocc.cmd.bits.raw_inst := wb_reg_inst - io.rocc.cmd.bits.funct := wb_reg_inst(31, 25) - io.rocc.cmd.bits.funct3 := wb_reg_inst(14, 12) - io.rocc.cmd.bits.rs2 := wb_reg_inst(24, 20) - io.rocc.cmd.bits.rs1 := wb_reg_inst(19, 15) - io.rocc.cmd.bits.pc := wb_reg_pc - io.rocc.cmd.bits.xd := wb_reg_inst(14) - io.rocc.cmd.bits.xs1 := wb_reg_inst(13) - io.rocc.cmd.bits.xs2 := wb_reg_inst(12) - io.rocc.cmd.bits.rd := wb_reg_inst(11, 7) - io.rocc.cmd.bits.opcode := wb_reg_inst(6, 0) - io.rocc.cmd.bits.rs1Data := wb_reg_wdata - io.rocc.cmd.bits.rs2Data := wb_reg_rs2 - - // gate the clock - val unpause = - csr.io.time(rocketParams.lgPauseCycles - 1, 0) === 0.U || csr.io.inhibit_cycle || io.dmem.perf.release || take_pc - when(unpause)(id_reg_pause := false.B) - io.cease := csr.io.status.cease && !clock_en_reg - io.wfi := csr.io.status.wfi - if (rocketParams.clockGate) { - long_latency_stall := csr.io.csr_stall || io.dmem.perf.blocked || id_reg_pause && !unpause - clock_en := clock_en_reg || ex_pc_valid || (!long_latency_stall && io.imem.resp.valid) - clock_en_reg := - ex_pc_valid || mem_pc_valid || wb_pc_valid || // instruction in flight - io.ptw.customCSRs.disableCoreClockGate || // chicken bit - !div.io.req.ready || // mul/div in flight - usingFPU.B && !io.fpu.fcsr_rdy || // long-latency FPU in flight - io.dmem.replay_next || // long-latency load replaying - (!long_latency_stall && (ibuf.io.inst(0).valid || io.imem.resp.valid)) // instruction pending - - assert(!(ex_pc_valid || mem_pc_valid || wb_pc_valid) || clock_en) - } - - // evaluate performance counters - val icache_blocked = !(io.imem.resp.valid || RegNext(io.imem.resp.valid)) - csr.io.counters foreach { c => c.inc := RegNext(perfEvents.evaluate(c.eventSel)) } - - val coreMonitorBundle = Wire(new CoreMonitorBundle(xLen, fLen)) - - coreMonitorBundle.clock := clock - coreMonitorBundle.reset := reset - coreMonitorBundle.hartid := io.hartid - coreMonitorBundle.timer := csr.io.time(31, 0) - coreMonitorBundle.valid := csr.io.trace(0).valid && !csr.io.trace(0).exception - coreMonitorBundle.pc := csr.io.trace(0).iaddr(vaddrBitsExtended - 1, 0).sextTo(xLen) - coreMonitorBundle.wrenx := wb_wen && !wb_set_sboard - coreMonitorBundle.wrenf := false.B - coreMonitorBundle.wrdst := wb_waddr - coreMonitorBundle.wrdata := rf_wdata - coreMonitorBundle.rd0src := wb_reg_inst(19, 15) - coreMonitorBundle.rd0val := RegNext(RegNext(ex_rs(0))) - coreMonitorBundle.rd1src := wb_reg_inst(24, 20) - coreMonitorBundle.rd1val := RegNext(RegNext(ex_rs(1))) - coreMonitorBundle.inst := csr.io.trace(0).insn - coreMonitorBundle.excpt := csr.io.trace(0).exception - coreMonitorBundle.priv_mode := csr.io.trace(0).priv - - if (enableCommitLog) { - val t = csr.io.trace(0) - val rd = wb_waddr - val wfd = wb_ctrl.wfd - val wxd = wb_ctrl.wxd - val has_data = wb_wen && !wb_set_sboard - - when(t.valid && !t.exception) { - when(wfd) { - printf("%d 0x%x (0x%x) f%d p%d 0xXXXXXXXXXXXXXXXX\n", t.priv, t.iaddr, t.insn, rd, rd + 32.U) - } - .elsewhen(wxd && rd =/= 0.U && has_data) { - printf("%d 0x%x (0x%x) x%d 0x%x\n", t.priv, t.iaddr, t.insn, rd, rf_wdata) - } - .elsewhen(wxd && rd =/= 0.U && !has_data) { - printf("%d 0x%x (0x%x) x%d p%d 0xXXXXXXXXXXXXXXXX\n", t.priv, t.iaddr, t.insn, rd, rd) - } - .otherwise { - printf("%d 0x%x (0x%x)\n", t.priv, t.iaddr, t.insn) - } - } - - when(ll_wen && rf_waddr =/= 0.U) { - printf("x%d p%d 0x%x\n", rf_waddr, rf_waddr, rf_wdata) - } - } else { - when(csr.io.trace(0).valid) { - printf( - "C%d: %d [%d] pc=[%x] W[r%d=%x][%d] R[r%d=%x] R[r%d=%x] inst=[%x] DASM(%x) wb_xcpt:%d\n", - io.hartid, - coreMonitorBundle.timer, - coreMonitorBundle.valid, - coreMonitorBundle.pc, - Mux(wb_ctrl.wxd || wb_ctrl.wfd, coreMonitorBundle.wrdst, 0.U), - Mux(coreMonitorBundle.wrenx, coreMonitorBundle.wrdata, 0.U), - coreMonitorBundle.wrenx, - Mux(wb_ctrl.rxs1 || wb_ctrl.rfs1, coreMonitorBundle.rd0src, 0.U), - Mux(wb_ctrl.rxs1 || wb_ctrl.rfs1, coreMonitorBundle.rd0val, 0.U), - Mux(wb_ctrl.rxs2 || wb_ctrl.rfs2, coreMonitorBundle.rd1src, 0.U), - Mux(wb_ctrl.rxs2 || wb_ctrl.rfs2, coreMonitorBundle.rd1val, 0.U), - coreMonitorBundle.inst, - coreMonitorBundle.inst, - wb_xcpt - ) - } - } - - // CoreMonitorBundle for late latency writes - val xrfWriteBundle = Wire(new CoreMonitorBundle(xLen, fLen)) - - xrfWriteBundle.clock := clock - xrfWriteBundle.reset := reset - xrfWriteBundle.hartid := io.hartid - xrfWriteBundle.timer := csr.io.time(31, 0) - xrfWriteBundle.valid := false.B - xrfWriteBundle.pc := 0.U - xrfWriteBundle.wrdst := rf_waddr - xrfWriteBundle.wrenx := rf_wen && !(csr.io.trace(0).valid && wb_wen && (wb_waddr === rf_waddr)) - xrfWriteBundle.wrenf := false.B - xrfWriteBundle.wrdata := rf_wdata - xrfWriteBundle.rd0src := 0.U - xrfWriteBundle.rd0val := 0.U - xrfWriteBundle.rd1src := 0.U - xrfWriteBundle.rd1val := 0.U - xrfWriteBundle.inst := 0.U - xrfWriteBundle.excpt := false.B - xrfWriteBundle.priv_mode := csr.io.trace(0).priv - - if (rocketParams.haveSimTimeout) - PlusArg.timeout( - name = "max_core_cycles", - docstring = "Kill the emulation after INT rdtime cycles. Off if 0." - )(csr.io.time) - - } // leaving gated-clock domain - val rocketImpl = withClock(gated_clock)(new RocketImpl) - - def checkExceptions(x: Seq[(Bool, UInt)]) = - (WireInit(x.map(_._1).reduce(_ || _)), WireInit(PriorityMux(x))) - - def coverExceptions( - exceptionValid: Bool, - cause: UInt, - labelPrefix: String, - coverCausesLabels: Seq[(Int, String)] - ): Unit = { - for ((coverCause, label) <- coverCausesLabels) { - property.cover(exceptionValid && (cause === coverCause.U), s"${labelPrefix}_$label") - } - } - - def checkHazards(targets: Seq[(Bool, UInt)], cond: UInt => Bool) = - targets.map(h => h._1 && cond(h._2)).reduce(_ || _) - - def encodeVirtualAddress(a0: UInt, ea: UInt) = - if (vaddrBitsExtended == vaddrBits) ea - else { - // efficient means to compress 64-bit VA into vaddrBits+1 bits - // (VA is bad if VA(vaddrBits) != VA(vaddrBits-1)) - val b = vaddrBitsExtended - 1 - val a = (a0 >> b).asSInt - val msb = Mux(a === 0.S || a === -1.S, ea(b), !ea(b - 1)) - Cat(msb, ea(b - 1, 0)) - } - - class Scoreboard(n: Int, zero: Boolean = false) { - def set(en: Bool, addr: UInt): Unit = update(en, _next | mask(en, addr)) - def clear(en: Bool, addr: UInt): Unit = update(en, _next & ~mask(en, addr)) - def read(addr: UInt): Bool = r(addr) - def readBypassed(addr: UInt): Bool = _next(addr) - - val _r = RegInit(0.U(n.W)) - val r = if (zero) (_r >> 1 << 1) else _r - private var _next = r - private var ens = false.B - def mask(en: Bool, addr: UInt) = Mux(en, 1.U << addr, 0.U) - - def update(en: Bool, update: UInt) = { - _next = update - ens = ens || en - when(ens)(_r := _next) - } - - } - -} - -class RegFile(n: Int, w: Int, zero: Boolean = false) { - val rf = Reg(Vec(n, UInt(w.W))) - def access(addr: UInt) = rf(~addr(log2Up(n) - 1, 0)) - val reads = ArrayBuffer[(UInt, UInt)]() - private var canRead = true - - def read(addr: UInt) = { - require(canRead) - reads += addr -> Wire(UInt()) - reads.last._2 := Mux(zero.B && addr === 0.U, 0.U, access(addr)) - reads.last._2 - } - - def write(addr: UInt, data: UInt) = { - canRead = false - when(addr =/= 0.U) { - access(addr) := data - for ((raddr, rdata) <- reads) - when(addr === raddr)(rdata := data) - } - } - -} - -object ImmGen { - - def apply(sel: UInt, inst: UInt) = { - val sign = Mux(sel === IMM_Z, 0.S, inst(31).asSInt) - val b30_20 = Mux(sel === IMM_U, inst(30, 20).asSInt, sign) - val b19_12 = Mux(sel =/= IMM_U && sel =/= IMM_UJ, sign, inst(19, 12).asSInt) - val b11 = Mux( - sel === IMM_U || sel === IMM_Z, - 0.S, - Mux(sel === IMM_UJ, inst(20).asSInt, Mux(sel === IMM_SB, inst(7).asSInt, sign)) - ) - val b10_5 = Mux(sel === IMM_U || sel === IMM_Z, 0.U, inst(30, 25)) - val b4_1 = Mux( - sel === IMM_U, - 0.U, - Mux(sel === IMM_S || sel === IMM_SB, inst(11, 8), Mux(sel === IMM_Z, inst(19, 16), inst(24, 21))) - ) - val b0 = Mux(sel === IMM_S, inst(7), Mux(sel === IMM_I, inst(20), Mux(sel === IMM_Z, inst(15), 0.U))) - - Cat(sign, b30_20, b19_12, b11, b10_5, b4_1, b0).asSInt - } - -} diff --git a/arch/src/main/scala/framework/system/core/rocket/RocketCoreParameters.scala b/arch/src/main/scala/framework/system/core/rocket/RocketCoreParameters.scala deleted file mode 100644 index 607096033..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/RocketCoreParameters.scala +++ /dev/null @@ -1,34 +0,0 @@ -// See LICENSE.Berkeley and LICENSE.SiFive for Rocket CSR semantics. -package freechips.rocketchip.rocket - -import chisel3._ -import org.chipsalliance.cde.config.Parameters -import freechips.rocketchip.tile._ -import freechips.rocketchip.util._ - -trait HasRocketCoreParameters extends HasCoreParameters { - lazy val rocketParams = tileParams.core.asInstanceOf[RocketCoreParams] - val fastLoadWord = rocketParams.fastLoadWord - val fastLoadByte = rocketParams.fastLoadByte - val mulDivParams = rocketParams.mulDiv.getOrElse(MulDivParams()) - require(!fastLoadByte || fastLoadWord) - require(!rocketParams.haveFSDirty) -} - -class RocketCustomCSRs(implicit p: Parameters) extends CustomCSRs with HasRocketCoreParameters { - override def bpmCSR = rocketParams.branchPredictionModeCSR.option(CustomCSR(bpmCSRId, 1, Some(BigInt(0)))) - - override def chickenCSR = { - val dc = tileParams.dcache.get - val mask = BigInt(dc.clockGate.toInt | (rocketParams.clockGate.toInt << 1) | - (rocketParams.clockGate.toInt << 2) | (1 << 3) | (dc.scratch.isEmpty.toInt << 9) | - (tileParams.icache.get.prefetch.toInt << 17)) - Some(CustomCSR(chickenCSRId, mask, Some(mask))) - } - - def disableICachePrefetch = getOrElse(chickenCSR, _.value(17), true.B) - def marchid = CustomCSR.constant(CSRs.marchid, BigInt(1)) - def mvendorid = CustomCSR.constant(CSRs.mvendorid, BigInt(rocketParams.mvendorid)) - def mimpid = CustomCSR.constant(CSRs.mimpid, BigInt(rocketParams.mimpid)) - override def decls = super.decls :+ marchid :+ mvendorid :+ mimpid -} diff --git a/arch/src/main/scala/framework/system/core/rocket/TileInterrupts.scala b/arch/src/main/scala/framework/system/core/rocket/TileInterrupts.scala deleted file mode 100644 index 4e3ea2a1b..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/TileInterrupts.scala +++ /dev/null @@ -1,21 +0,0 @@ -// See LICENSE.SiFive for the Rocket interface definitions. -package freechips.rocketchip.tile - -import chisel3._ -import org.chipsalliance.cde.config.Parameters - -class NMI(val w: Int) extends Bundle { - val rnmi = Bool() - val rnmi_interrupt_vector = UInt(w.W) - val rnmi_exception_vector = UInt(w.W) -} - -class TileInterrupts(implicit p: Parameters) extends CoreBundle()(p) { - val debug = Bool() - val mtip = Bool() - val msip = Bool() - val meip = Bool() - val seip = Option.when(usingSupervisor)(Bool()) - val lip = Vec(coreParams.nLocalInterrupts, Bool()) - val nmi = Option.when(usingNMI)(new NMI(resetVectorLen)) -} diff --git a/arch/src/main/scala/framework/system/core/rocket/TileParameters.scala b/arch/src/main/scala/framework/system/core/rocket/TileParameters.scala deleted file mode 100644 index 5e93631ad..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/TileParameters.scala +++ /dev/null @@ -1,78 +0,0 @@ -// See LICENSE.Berkeley and LICENSE.SiFive for Rocket interface definitions. -package freechips.rocketchip.tile { - - import org.chipsalliance.cde.config.{Field, Parameters} - import chisel3.util.{log2Ceil, log2Up} - import freechips.rocketchip.rocket._ - - case object TileKey extends Field[CpuTileParams] - case object CoreBeatBytes extends Field[Int] - case object CoreRoCCCount extends Field[Int](0) - - trait HasTileParameters { - implicit val p: Parameters - def tileParams: CpuTileParams = p(TileKey) - def usingVM = tileParams.core.useVM - def usingSupervisor = tileParams.core.hasSupervisorMode - def usingUser = tileParams.core.useUser || usingSupervisor - def usingHypervisor = usingVM && tileParams.core.useHypervisor - def usingDebug = tileParams.core.useDebug - def usingRoCC = p(CoreRoCCCount) > 0 - def usingBTB = tileParams.btb.exists(_.nEntries > 0) - def usingPTW = usingVM - def usingDataScratchpad = tileParams.dcache.flatMap(_.scratch).isDefined - def xLen = tileParams.core.xLen - def xBytes = xLen / 8 - def iLen = 32 - def pgIdxBits = 12 - def pgLevelBits = 10 - log2Ceil(xLen / 32) - def pgLevels = tileParams.core.pgLevels - def minPgLevels = { val levels = if (xLen == 32) 2 else 3; require(pgLevels >= levels); levels } - def maxSVAddrBits = pgIdxBits + pgLevels * pgLevelBits - def maxHypervisorExtraAddrBits = 2 - def hypervisorExtraAddrBits = if (usingHypervisor) maxHypervisorExtraAddrBits else 0 - def maxHVAddrBits = maxSVAddrBits + hypervisorExtraAddrBits - def asIdBits = p(ASIdBits) - def vmIdBits = p(VMIdBits) - - lazy val maxPAddrBits: Int = { - require(xLen == 32 || xLen == 64); if (!usingVM) xLen else if (xLen == 32) 34 else 56 - } - - lazy val paddrBits: Int = p(CorePAddrBits) - - def vaddrBits: Int = - if (usingVM) { - val bits = maxHVAddrBits - require(bits == xLen || xLen > bits && bits > paddrBits) - bits - } else (paddrBits + 1).min(xLen) - - def vpnBits = vaddrBits - pgIdxBits - def ppnBits = paddrBits - pgIdxBits - def vpnBitsExtended = vpnBits + (if (vaddrBits < xLen) 1 + (if (usingHypervisor) 1 else 0) else 0) - def vaddrBitsExtended = vpnBitsExtended + pgIdxBits - def cacheBlockBytes = p(freechips.rocketchip.subsystem.CacheBlockBytes) - def lgCacheBlockBytes = log2Up(cacheBlockBytes) - def masterPortBeatBytes = p(CoreBeatBytes) - def dcacheArbPorts = 1 + (if (usingVM) 1 else 0) + (if (usingDataScratchpad) 1 else 0) + p(CoreRoCCCount) + - (if (tileParams.core.useVector && tileParams.core.vectorUseDCache) 1 else 0) - } - -} - -package freechips.rocketchip.rocket { - import org.chipsalliance.cde.config.Field - case object ASIdBits extends Field[Int](0) - case object VMIdBits extends Field[Int](0) -} - -package freechips.rocketchip.subsystem { - import org.chipsalliance.cde.config.Field - case object CacheBlockBytes extends Field[Int] -} - -package freechips.rocketchip.devices.debug { - import org.chipsalliance.cde.config.Field - case object DebugModuleKey extends Field[Option[DebugModuleParams]](Some(DebugModuleParams())) -} diff --git a/arch/src/main/scala/framework/system/core/rocket/VectorInterfaces.scala b/arch/src/main/scala/framework/system/core/rocket/VectorInterfaces.scala deleted file mode 100644 index e2377278e..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/VectorInterfaces.scala +++ /dev/null @@ -1,53 +0,0 @@ -// See LICENSE.Berkeley for Rocket's CPU/vector interface contract. -package freechips.rocketchip.rocket - -import chisel3._ -import chisel3.util._ -import org.chipsalliance.cde.config.Parameters -import freechips.rocketchip.tile._ - -class VectorCoreIO(implicit p: Parameters) extends CoreBundle { - val status = Input(new MStatus) - - val ex = new Bundle { - val valid = Input(Bool()); val ready = Output(Bool()); val inst = Input(UInt(32.W)) - val pc = Input(UInt(vaddrBitsExtended.W)); val vconfig = Input(new VConfig) - val vstart = Input(UInt(log2Ceil(maxVLMax).W)); val rs1 = Input(UInt(xLen.W)); val rs2 = Input(UInt(xLen.W)) - } - - val killm = Input(Bool()) - - val mem = new Bundle { - val frs1 = Input(UInt(fLen.W)); val block_mem = Output(Bool()); val block_all = Output(Bool()) - } - - val wb = new Bundle { - val store_pending = Input(Bool()); val replay = Output(Bool()); val retire = Output(Bool()) - val inst = Output(UInt(32.W)); val rob_should_wb = Output(Bool()); val rob_should_wb_fp = Output(Bool()) - val pc = Output(UInt(vaddrBitsExtended.W)); val xcpt = Output(Bool()) - val cause = Output(UInt(log2Ceil(Causes.all.max).W)); val tval = Output(UInt(coreMaxAddrBits.W)) - val vxrm = Input(UInt(2.W)); val frm = Input(UInt(3.W)) - } - - val resp = Decoupled(new Bundle { - val fp = Bool(); val size = UInt(2.W); val rd = UInt(5.W); val data = UInt((xLen max fLen).W) - }) - - val set_vstart = Valid(UInt(log2Ceil(maxVLMax).W)) - val set_vxsat = Output(Bool()) - val set_vconfig = Valid(new VConfig) - val set_fflags = Valid(UInt(5.W)) - val trap_check_busy = Output(Bool()) - val backend_busy = Output(Bool()) -} - -abstract class RocketVectorDecoder(implicit p: Parameters) extends CoreModule { - - val io = IO(new Bundle { - val inst = Input(UInt(32.W)); val vconfig = Input(new VConfig) - val vector = Output(Bool()); val legal = Output(Bool()); val fp = Output(Bool()) - val read_rs1 = Output(Bool()); val read_rs2 = Output(Bool()); val read_frs1 = Output(Bool()) - val write_rd = Output(Bool()); val write_frd = Output(Bool()) - }) - -} diff --git a/arch/src/main/scala/framework/system/core/rocket/configs/RocketCpuParam.scala b/arch/src/main/scala/framework/system/core/rocket/configs/RocketCpuParam.scala deleted file mode 100644 index b36f8d2f9..000000000 --- a/arch/src/main/scala/framework/system/core/rocket/configs/RocketCpuParam.scala +++ /dev/null @@ -1,378 +0,0 @@ -package framework.system.core.rocket.configs { - - import upickle.default._ - import freechips.rocketchip.rocket.{BHTParams, BTBParams, DCacheParams, ICacheParams, MulDivParams, RocketCoreParams} - import freechips.rocketchip.tile.FPUParams - - /** - * JSON-serializable Rocket CPU configuration parameters. - * - * Optional features (mulDiv, fpu, btb) use an `enable` flag instead of Option[T], - * so JSON stays as a plain dictionary (upickle serializes Option as array). - */ - - case class MulDivParam( - divUnroll: Int, - divEarlyOutGranularity: Int, - enable: Boolean, - mulUnroll: Int, - mulEarlyOut: Boolean, - divEarlyOut: Boolean) - - object MulDivParam { - implicit val rw: ReadWriter[MulDivParam] = macroRW - - def toMulDivParams(p: MulDivParam): Option[MulDivParams] = - if (p.enable) Some(MulDivParams( - divUnroll = p.divUnroll, - divEarlyOutGranularity = p.divEarlyOutGranularity, - mulUnroll = p.mulUnroll, - mulEarlyOut = p.mulEarlyOut, - divEarlyOut = p.divEarlyOut - )) - else None - - } - - case class FPUParam( - divSqrt: Boolean, - sfmaLatency: Int, - dfmaLatency: Int, - fpmuLatency: Int, - ifpuLatency: Int, - enable: Boolean, - minFLen: Int, - fLen: Int) - - object FPUParam { - implicit val rw: ReadWriter[FPUParam] = macroRW - - def toFPUParams(p: FPUParam): Option[FPUParams] = - if (p.enable) Some(FPUParams( - divSqrt = p.divSqrt, - sfmaLatency = p.sfmaLatency, - dfmaLatency = p.dfmaLatency, - fpmuLatency = p.fpmuLatency, - ifpuLatency = p.ifpuLatency, - minFLen = p.minFLen, - fLen = p.fLen - )) - else None - - } - - case class DCacheParam( - clockGate: Boolean, - nSets: Int, - nWays: Int, - nMSHRs: Int) - - object DCacheParam { - implicit val rw: ReadWriter[DCacheParam] = macroRW - - def toDCacheParams(p: DCacheParam, rowBits: Int, blockBytes: Int): DCacheParams = DCacheParams( - clockGate = p.clockGate, - nSets = p.nSets, - nWays = p.nWays, - rowBits = rowBits, - nMSHRs = p.nMSHRs, - blockBytes = blockBytes - ) - - } - - case class ICacheParam( - prefetch: Boolean, - nSets: Int, - nWays: Int) - - object ICacheParam { - implicit val rw: ReadWriter[ICacheParam] = macroRW - - def toICacheParams(p: ICacheParam, rowBits: Int, blockBytes: Int): ICacheParams = ICacheParams( - prefetch = p.prefetch, - nSets = p.nSets, - nWays = p.nWays, - rowBits = rowBits, - blockBytes = blockBytes - ) - - } - - case class BTBParam( - nMatchBits: Int, - nPages: Int, - updatesOutOfOrder: Boolean, - bhtEnable: Boolean, - bhtEntries: Int, - bhtCounterLength: Int, - bhtHistoryLength: Int, - bhtHistoryBits: Int, - enable: Boolean, - nEntries: Int, - nRAS: Int) - - object BTBParam { - implicit val rw: ReadWriter[BTBParam] = macroRW - - def toBTBParams(p: BTBParam): Option[BTBParams] = - if (p.enable) Some(BTBParams( - nMatchBits = p.nMatchBits, - nPages = p.nPages, - updatesOutOfOrder = p.updatesOutOfOrder, - bhtParams = - if (p.bhtEnable) Some(BHTParams(p.bhtEntries, p.bhtCounterLength, p.bhtHistoryLength, p.bhtHistoryBits)) - else None, - nEntries = p.nEntries, - nRAS = p.nRAS - )) - else None - - } - - case class RocketCpuParam( - mtvecInitEnable: Boolean, - useUser: Boolean, - useSupervisor: Boolean, - useHypervisor: Boolean, - useDebug: Boolean, - useAtomics: Boolean, - useAtomicsOnlyForIO: Boolean, - useCompressed: Boolean, - useRVE: Boolean, - useConditionalZero: Boolean, - nLocalInterrupts: Int, - useNMI: Boolean, - nBreakpoints: Int, - useBPWatch: Boolean, - mcontextWidth: Int, - scontextWidth: Int, - nPerfCounters: Int, - haveBasicCounters: Boolean, - misaWritable: Boolean, - mtvecInit: BigInt, - mtvecWritable: Boolean, - fastLoadWord: Boolean, - fastLoadByte: Boolean, - branchPredictionModeCSR: Boolean, - clockGate: Boolean, - mvendorid: Int, - mimpid: Int, - haveCease: Boolean, - haveSimTimeout: Boolean, - useVM: Boolean, - useZba: Boolean, - useZbb: Boolean, - useZbs: Boolean, - haveCFlush: Boolean, - mulDiv: MulDivParam, - fpu: FPUParam, - dcache: DCacheParam, - icache: ICacheParam, - btb: BTBParam) - - object RocketCpuParam { - implicit val rw: ReadWriter[RocketCpuParam] = macroRW - - /** - * Convert to rocket-chip RocketCoreParams. - * Cache geometry, physical address width and bus widths are supplied explicitly by the core. - */ - def toRocketCoreParams(p: RocketCpuParam, xLen: Int, pgLevels: Int): RocketCoreParams = RocketCoreParams( - xLen = xLen, - pgLevels = pgLevels, - useUser = p.useUser, - useSupervisor = p.useSupervisor, - useHypervisor = p.useHypervisor, - useDebug = p.useDebug, - useAtomics = p.useAtomics, - useAtomicsOnlyForIO = p.useAtomicsOnlyForIO, - useCompressed = p.useCompressed, - useRVE = p.useRVE, - useConditionalZero = p.useConditionalZero, - nLocalInterrupts = p.nLocalInterrupts, - useNMI = p.useNMI, - nBreakpoints = p.nBreakpoints, - useBPWatch = p.useBPWatch, - mcontextWidth = p.mcontextWidth, - scontextWidth = p.scontextWidth, - nPerfCounters = p.nPerfCounters, - haveBasicCounters = p.haveBasicCounters, - misaWritable = p.misaWritable, - mtvecInit = if (p.mtvecInitEnable) Some(p.mtvecInit) else None, - mtvecWritable = p.mtvecWritable, - fastLoadWord = p.fastLoadWord, - fastLoadByte = p.fastLoadByte, - branchPredictionModeCSR = p.branchPredictionModeCSR, - clockGate = p.clockGate, - mvendorid = p.mvendorid, - mimpid = p.mimpid, - haveCease = p.haveCease, - haveSimTimeout = p.haveSimTimeout, - useVM = p.useVM, - useZba = p.useZba, - useZbb = p.useZbb, - useZbs = p.useZbs, - haveCFlush = p.haveCFlush, - mulDiv = MulDivParam.toMulDivParams(p.mulDiv), - fpu = FPUParam.toFPUParams(p.fpu) - ) - - def toDCacheParams(p: RocketCpuParam, rowBits: Int, blockBytes: Int): DCacheParams = - DCacheParam.toDCacheParams(p.dcache, rowBits, blockBytes) - - def toICacheParams(p: RocketCpuParam, rowBits: Int, blockBytes: Int): ICacheParams = - ICacheParam.toICacheParams(p.icache, rowBits, blockBytes) - - def toBTBParams(p: RocketCpuParam): Option[BTBParams] = - BTBParam.toBTBParams(p.btb) - } - -} - -package freechips.rocketchip.rocket { - import chisel3._ - import chisel3.util._ - import org.chipsalliance.cde.config.Parameters - import freechips.rocketchip.tile._ - - case class DCacheParams( - nSets: Int = 64, - nWays: Int = 4, - rowBits: Int = 64, - subWordBits: Option[Int] = None, - replacementPolicy: String = "random", - nTLBSets: Int = 1, - nTLBWays: Int = 32, - nTLBBasePageSectors: Int = 4, - nTLBSuperpages: Int = 4, - tagECC: Option[String] = None, - dataECC: Option[String] = None, - dataECCBytes: Int = 1, - nMSHRs: Int = 1, - nSDQ: Int = 17, - nRPQ: Int = 16, - nMMIOs: Int = 1, - blockBytes: Int = 64, - separateUncachedResp: Boolean = false, - acquireBeforeRelease: Boolean = false, - pipelineWayMux: Boolean = false, - clockGate: Boolean = false, - scratch: Option[BigInt] = None) { - require(scratch.isEmpty || nWays == 1) - require(scratch.isEmpty || nMSHRs == 0) - if (scratch.isEmpty) require(isPow2(nSets)) - } - - case class ICacheParams( - nSets: Int = 64, - nWays: Int = 4, - rowBits: Int = 128, - nTLBSets: Int = 1, - nTLBWays: Int = 32, - nTLBBasePageSectors: Int = 4, - nTLBSuperpages: Int = 4, - cacheIdBits: Int = 0, - tagECC: Option[String] = None, - dataECC: Option[String] = None, - itimAddr: Option[BigInt] = None, - prefetch: Boolean = false, - blockBytes: Int = 64, - latency: Int = 2, - fetchBytes: Int = 4) - - case class RocketCoreParams( - xLen: Int = 64, - pgLevels: Int = 3, - bootFreqHz: BigInt = 0, - useVM: Boolean = true, - useUser: Boolean = false, - useSupervisor: Boolean = false, - useHypervisor: Boolean = false, - useDebug: Boolean = true, - useAtomics: Boolean = true, - useAtomicsOnlyForIO: Boolean = false, - useCompressed: Boolean = true, - useRVE: Boolean = false, - useConditionalZero: Boolean = false, - useZba: Boolean = false, - useZbb: Boolean = false, - useZbs: Boolean = false, - nLocalInterrupts: Int = 0, - useNMI: Boolean = false, - nBreakpoints: Int = 1, - useBPWatch: Boolean = false, - mcontextWidth: Int = 0, - scontextWidth: Int = 0, - nPMPs: Int = 8, - nPerfCounters: Int = 0, - haveBasicCounters: Boolean = true, - haveCFlush: Boolean = false, - misaWritable: Boolean = true, - nL2TLBEntries: Int = 0, - nL2TLBWays: Int = 1, - nPTECacheEntries: Int = 8, - mtvecInit: Option[BigInt] = Some(BigInt(0)), - mtvecWritable: Boolean = true, - fastLoadWord: Boolean = true, - fastLoadByte: Boolean = false, - branchPredictionModeCSR: Boolean = false, - clockGate: Boolean = false, - mvendorid: Int = 0, - mimpid: Int = 0x20181004, - mulDiv: Option[MulDivParams] = Some(MulDivParams()), - fpu: Option[FPUParams] = Some(FPUParams()), - debugROB: Option[DebugROBParams] = None, - haveCease: Boolean = true, - haveSimTimeout: Boolean = true, - vector: Option[RocketCoreVectorParams] = None, - enableTraceCoreIngress: Boolean = false) - extends CoreParams { - val lgPauseCycles = 5 - val haveFSDirty = false - val pmpGranularity = if (useHypervisor) 4096 else 4 - val fetchWidth = if (useCompressed) 2 else 1 - val decodeWidth = fetchWidth / (if (useCompressed) 2 else 1) - val retireWidth = 1 - val instBits = if (useCompressed) 16 else 32 - val lrscCycles = 80 - val traceHasWdata = debugROB.isDefined - override def minFLen = fpu.map(_.minFLen).getOrElse(32) - override def customCSRs(implicit p: Parameters) = new RocketCustomCSRs - override val useVector = vector.isDefined - override val vectorUseDCache = vector.exists(_.useDCache) - override def vLen = vector.map(_.vLen).getOrElse(0) - override def eLen = vector.map(_.eLen).getOrElse(0) - override def vfLen = vector.map(_.vfLen).getOrElse(0) - override def vfh = vector.exists(_.vfh) - override def vMemDataBits = vector.map(_.vMemDataBits).getOrElse(0) - override def vExts = vector.map(_.vExts).getOrElse(Nil) - } - - case class RocketCoreVectorParams( - vLen: Int, - eLen: Int, - vfLen: Int, - vfh: Boolean, - vMemDataBits: Int, - decoder: Parameters => RocketVectorDecoder, - useDCache: Boolean, - issueVConfig: Boolean, - vExts: Seq[String]) - -} - -package freechips.rocketchip.tile { - import freechips.rocketchip.rocket._ - - case class CpuTileParams( - core: CoreParams, - dcache: Option[DCacheParams], - icache: Option[ICacheParams], - btb: Option[BTBParams]) - -} - -package freechips.rocketchip.devices.debug { - case class DebugModuleParams(nDscratch: Int = 1, debugEntry: BigInt = 0x800, debugException: BigInt = 0x808) -} diff --git a/arch/src/main/scala/framework/system/core/seed b/arch/src/main/scala/framework/system/core/seed deleted file mode 160000 index f8ba87b5c..000000000 --- a/arch/src/main/scala/framework/system/core/seed +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f8ba87b5c73c2c78aa9e08e3b92324cbed8e72de diff --git a/arch/src/main/scala/framework/system/device/BootRom.scala b/arch/src/main/scala/framework/system/device/BootRom.scala deleted file mode 100644 index 7cdc799f1..000000000 --- a/arch/src/main/scala/framework/system/device/BootRom.scala +++ /dev/null @@ -1,86 +0,0 @@ -package framework.system.device - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import chisel3.util._ -import memcore.bus.chi.{Params => ChiParams} -import memcore.bus.chi.snf.{LineRequest, LineResponse} - -/** - * The BootROM image (`bootrom/bootrom.rv64.img`, built from `bootrom.S`) with its two config dwords - * filled: the hart count at offset 0x8 and the DRAM entry at offset 0x10. - */ -case class BootRomParams(base: BigInt, bytes: BigInt, image: Seq[Byte]) { - require(base % 64 == 0 && bytes % 64 == 0 && image.size <= bytes, "BootROM must be whole cache lines") -} - -object BootRom { - - def apply( - base: BigInt, - bytes: BigInt, - hartCount: Int, - entry: BigInt - ): BootRomParams = { - val stream = getClass.getResourceAsStream("/bootrom/bootrom.rv64.img") - require(stream != null, "missing BootROM resource bootrom/bootrom.rv64.img") - val image = - try stream.readAllBytes().toSeq - finally stream.close() - require(image.size >= 24 && image.slice(8, 24).forall(_ == 0), "BootROM image lacks its zeroed config dwords") - def dword(value: BigInt): Seq[Byte] = (0 until 8).map(i => ((value >> (8 * i)) & 0xff).toByte) - BootRomParams(base, bytes, image.take(8) ++ dword(hartCount) ++ dword(entry) ++ image.drop(24)) - } - -} - -/** - * Serves BootROM lines on a tile's L2 backing path and forwards every other line to memory. The - * ROM is a cacheable, executable, read-only region, so fetches and loads reach it as line reads; - * a write to a ROM line can only come from a faulty requester and answers with an error. - */ -@instantiable -class BootRomLines(p: BootRomParams, chi: ChiParams) extends Module { - - @public val io = IO(new Bundle { - val request = Flipped(Decoupled(new LineRequest(chi))) - val response = Decoupled(new LineResponse(chi)) - val memoryRequest = Decoupled(new LineRequest(chi)) - val memoryResponse = Flipped(Decoupled(new LineResponse(chi))) - }) - - val lines = p.image.padTo(((p.image.size + 63) / 64) * 64, 0.toByte).grouped(64).map { line => - line.zipWithIndex.map { case (byte, i) => BigInt(byte & 0xff) << (8 * i) }.sum - }.toSeq - - val rom = VecInit(lines.map(_.U(512.W))) - - val offset = io.request.bits.addr - p.base.U - val inRom = io.request.bits.addr >= p.base.U && io.request.bits.addr < (p.base + p.bytes).U - val line = offset >> 6 - val pending = RegInit(false.B) - val id = Reg(UInt(chi.txnIdBits.W)) - val data = Reg(UInt(512.W)) - val error = Reg(Bool()) - - io.memoryRequest.valid := io.request.valid && !inRom - io.memoryRequest.bits := io.request.bits - io.request.ready := Mux(inRom, !pending, io.memoryRequest.ready) - when(io.request.fire && inRom) { - pending := true.B - id := io.request.bits.id - data := Mux(line < lines.size.U, rom(line(log2Ceil(lines.size.max(2)) - 1, 0)), 0.U) - error := io.request.bits.write - } - - // Memory responses keep priority; the ROM answer waits at most until memory is idle. - io.response.valid := io.memoryResponse.valid || pending - io.response.bits := Mux(io.memoryResponse.valid, io.memoryResponse.bits, 0.U.asTypeOf(io.response.bits)) - when(!io.memoryResponse.valid) { - io.response.bits.id := id - io.response.bits.data := data - io.response.bits.error := error - } - io.memoryResponse.ready := io.response.ready - when(io.response.fire && !io.memoryResponse.valid)(pending := false.B) -} diff --git a/arch/src/main/scala/framework/system/device/Clint.scala b/arch/src/main/scala/framework/system/device/Clint.scala deleted file mode 100644 index 613233947..000000000 --- a/arch/src/main/scala/framework/system/device/Clint.scala +++ /dev/null @@ -1,86 +0,0 @@ -package framework.system.device - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import chisel3.util._ - -/** One register access presented for a single cycle; data is right-aligned at `addr`. */ -class DeviceAccess extends Bundle { - val addr = UInt(64.W) - val write = Bool() - val size = UInt(3.W) - val data = UInt(64.W) -} - -/** Register-block interface: `read` and `error` answer the access presented in the same cycle. */ -class DevicePort extends Bundle { - val access = Flipped(Valid(new DeviceAccess)) - val read = Output(UInt(64.W)) - val error = Output(Bool()) -} - -/** `mtime` advances once every `tickCycles` clock cycles; the device tree timebase must match. */ -case class ClintParams(base: BigInt = BigInt("2000000", 16), bytes: BigInt = BigInt("10000", 16), tickCycles: Int = 1000) { - require(tickCycles >= 1) -} - -/** Standard CLINT layout: msip at 4*hart, mtimecmp at 0x4000 + 8*hart, mtime at 0xbff8. */ -@instantiable -class Clint(p: ClintParams, hartIds: Seq[Int]) extends Module { - require(hartIds.distinct.size == hartIds.size && hartIds.max < 4095) - val n = hartIds.size - - @public val io = IO(new Bundle { - val port = new DevicePort - val msip = Output(Vec(n, Bool())) - val mtip = Output(Vec(n, Bool())) - val time = Output(UInt(64.W)) - }) - - val msip = RegInit(VecInit(Seq.fill(n)(false.B))) - val mtimecmp = RegInit(VecInit(Seq.fill(n)(~0.U(64.W)))) - val mtime = RegInit(0.U(64.W)) - val ticks = RegInit(0.U(log2Ceil(p.tickCycles + 1).W)) - io.time := mtime - - val access = io.port.access.bits - val offset = access.addr - p.base.U - val write = io.port.access.valid && access.write - val word = access.size === 2.U - val double = access.size === 3.U - - // Each register answers a 32-bit access to either half, and a 64-bit access to its base. - val hits = Wire(Vec(2 * n + 1, Bool())) - val reads = Wire(Vec(2 * n + 1, UInt(64.W))) - - def register( - slot: Int, - at: BigInt, - state: UInt, - update: UInt => Unit, - wide: Boolean - ): Unit = { - val value = WireDefault(UInt(64.W), state) - val low = word && offset === at.U - val high = wide.B && word && offset === (at + 4).U - val full = wide.B && double && offset === at.U - hits(slot) := low || high || full - reads(slot) := Mux(high, value(63, 32), Mux(low, value(31, 0), value)) - when(write && low)(update(if (wide) Cat(value(63, 32), access.data(31, 0)) else access.data(31, 0))) - when(write && high)(update(Cat(access.data(31, 0), value(31, 0)))) - when(write && full)(update(access.data)) - } - - ticks := Mux(ticks === (p.tickCycles - 1).U, 0.U, ticks + 1.U) - when(ticks === (p.tickCycles - 1).U)(mtime := mtime + 1.U) - for ((hart, i) <- hartIds.zipWithIndex) { - register(i, 4 * hart, msip(i).asUInt, v => msip(i) := v(0), wide = false) - register(n + i, 0x4000 + 8 * BigInt(hart), mtimecmp(i), v => mtimecmp(i) := v, wide = true) - io.msip(i) := msip(i) - io.mtip(i) := mtime >= mtimecmp(i) - } - register(2 * n, 0xbff8, mtime, v => mtime := v, wide = true) - - io.port.read := Mux1H(hits, reads) - io.port.error := !hits.asUInt.orR -} diff --git a/arch/src/main/scala/framework/system/device/Devices.scala b/arch/src/main/scala/framework/system/device/Devices.scala deleted file mode 100644 index 598b4c3ea..000000000 --- a/arch/src/main/scala/framework/system/device/Devices.scala +++ /dev/null @@ -1,101 +0,0 @@ -package framework.system.device - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import chisel3.util._ -import hier.tile.memory.CoreInterrupts -import memcore.memory.cpu.{CpuMemParams, UncachedRequest, UncachedResponse} - -/** Chip devices; the BootROM, when present, is served on the tiles' line paths rather than here. */ -case class DeviceParams( - clint: ClintParams = ClintParams(), - plic: PlicParams = PlicParams(), - bootrom: Option[BootRomParams] = None, - scu: sims.scu.SCUParams = sims.scu.SCUParams()) - -/** - * Chip device decode for the per-core uncached device ports. CLINT and PLIC are served inside; - * every other device address leaves on the same core's external port. Each core port carries one - * outstanding request, and the internal register blocks take one access per cycle in round robin. - */ -@instantiable -class Devices(p: DeviceParams, cp: CpuMemParams, hartIds: Seq[Int]) extends Module { - val n = hartIds.size - - @public val io = IO(new Bundle { - val request = Vec(n, Flipped(Decoupled(new UncachedRequest(cp)))) - val response = Vec(n, Decoupled(new UncachedResponse(cp))) - val externalRequest = Vec(n, Decoupled(new UncachedRequest(cp))) - val externalResponse = Vec(n, Flipped(Decoupled(new UncachedResponse(cp)))) - val sources = Input(UInt(p.plic.sources.W)) - val interrupts = Output(Vec(n, new CoreInterrupts)) - val time = Output(UInt(64.W)) - }) - - val clint = Instantiate(new Clint(p.clint, hartIds)) - val plic = Instantiate(new Plic(p.plic, hartIds)) - io.time := clint.io.time - plic.io.sources := io.sources - for (i <- 0 until n) { - io.interrupts(i).timer := clint.io.mtip(i) - io.interrupts(i).software := clint.io.msip(i) - io.interrupts(i).external := plic.io.meip(i) - io.interrupts(i).supervisorExternal := plic.io.seip(i) - } - - def within(addr: UInt, base: BigInt, bytes: BigInt): Bool = addr >= base.U && addr < (base + bytes).U - val inClint = io.request.map(r => within(r.bits.addr, p.clint.base, p.clint.bytes)) - val inPlic = io.request.map(r => within(r.bits.addr, p.plic.base, p.plic.bytes)) - val internal = inClint.zip(inPlic).map { case (c, l) => c || l } - - val pending = RegInit(VecInit(Seq.fill(n)(false.B))) - val outside = RegInit(VecInit(Seq.fill(n)(false.B))) - val tag = Reg(Vec(n, UInt(cp.tagBits.W))) - val data = Reg(Vec(n, UInt(64.W))) - val error = Reg(Vec(n, Bool())) - val idle = (0 until n).map(i => !pending(i) && !outside(i)) - - // Round-robin grant among the cores presenting an internal access this cycle. - val wants = VecInit((0 until n).map(i => io.request(i).valid && internal(i) && idle(i))).asUInt - val last = RegInit(0.U(log2Ceil(n + 1).W)) - val rotated = (0 until n).map(k => (k.U +& last +& 1.U) % n.U) - val grant = PriorityMux(rotated.map(i => wants(i)), rotated) - val granted = wants.orR - when(granted)(last := grant) - - val access = io.request(grant).bits - for (port <- Seq(clint.io.port, plic.io.port)) { - port.access.bits.addr := access.addr - port.access.bits.write := access.write - port.access.bits.size := access.size - port.access.bits.data := access.data - } - clint.io.port.access.valid := granted && VecInit(inClint)(grant) - plic.io.port.access.valid := granted && VecInit(inPlic)(grant) - val internalRead = Mux(VecInit(inClint)(grant), clint.io.port.read, plic.io.port.read) - val internalError = Mux(VecInit(inClint)(grant), clint.io.port.error, plic.io.port.error) - - for (i <- 0 until n) { - val request = io.request(i) - val response = io.response(i) - val external = io.externalRequest(i) - val mine = granted && grant === i.U - external.valid := request.valid && !internal(i) && idle(i) - external.bits := request.bits - request.ready := idle(i) && Mux(internal(i), mine, external.ready) - when(request.fire) { - when(internal(i)) { - pending(i) := true.B - tag(i) := request.bits.tag - data(i) := Mux(request.bits.write, 0.U, internalRead) - error(i) := internalError - }.otherwise(outside(i) := true.B) - } - response.valid := pending(i) || (outside(i) && io.externalResponse(i).valid) - response.bits.tag := Mux(pending(i), tag(i), io.externalResponse(i).bits.tag) - response.bits.data := Mux(pending(i), data(i), io.externalResponse(i).bits.data) - response.bits.error := Mux(pending(i), error(i), io.externalResponse(i).bits.error) - io.externalResponse(i).ready := outside(i) && response.ready - when(response.fire) { pending(i) := false.B; outside(i) := false.B } - } -} diff --git a/arch/src/main/scala/framework/system/dlink/dlink.scala b/arch/src/main/scala/framework/system/dlink/dlink.scala deleted file mode 100644 index e75e5772c..000000000 --- a/arch/src/main/scala/framework/system/dlink/dlink.scala +++ /dev/null @@ -1,11 +0,0 @@ -package framework.system.dlink - -import chisel3._ -import framework.system.SystemParams - -/** Unified memory access; address decoding belongs to the platform. */ -class DLinkIO(p: SystemParams) extends Bundle { - val axi = new memcore.bus.axi4.Port(p.ddr.axi) -} - -trait HasDLink { def dlink: DLinkIO } diff --git a/arch/src/main/scala/framework/system/memory/Memory.scala b/arch/src/main/scala/framework/system/memory/Memory.scala deleted file mode 100644 index 3f38b277d..000000000 --- a/arch/src/main/scala/framework/system/memory/Memory.scala +++ /dev/null @@ -1,33 +0,0 @@ -package framework.system.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.ddr.{Params => DdrParams, Bridge} - -/** CPU coherence backing traffic and NPU DMA meet at the DDR AXI arbiter. */ -@instantiable -class Memory(ddr: DdrParams, dmaMasters: Int = 0) extends Module { - - @public - val io = IO(new Bundle { - val lineRequest = Vec(ddr.clients, Flipped(Decoupled(new LineRequest(ddr.line)))) - val lineResponse = Vec(ddr.clients, Decoupled(new LineResponse(ddr.line))) - val dma = Vec(dmaMasters, Flipped(new memcore.bus.axi4.Port(ddr.axi))) - val axi = new memcore.bus.axi4.Port(ddr.axi) - val outstanding = Output(UInt(log2Ceil(ddr.slots + 2 * (1 << ddr.axi.idBits) + 1).W)) - }) - - val bridge: Instance[Bridge] = Instantiate(new Bridge(ddr)) - bridge.io.request <> io.lineRequest - io.lineResponse <> bridge.io.response - val fabric: Instance[memcore.bus.axi4.Interconnect] = - Instantiate(new memcore.bus.axi4.Interconnect(ddr.axi, dmaMasters + 1)) - fabric.io.in(0) <> bridge.io.axi - io.axi <> fabric.io.out - for (i <- 0 until dmaMasters) { - fabric.io.in(i + 1) <> io.dma(i) - } - io.outstanding := bridge.io.outstanding +& fabric.io.outstanding -} diff --git a/arch/src/main/scala/framework/system/memory/UncachedAxi.scala b/arch/src/main/scala/framework/system/memory/UncachedAxi.scala deleted file mode 100644 index 7380988f8..000000000 --- a/arch/src/main/scala/framework/system/memory/UncachedAxi.scala +++ /dev/null @@ -1,73 +0,0 @@ -package framework.system.memory - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import memcore.bus.axi4 -import memcore.memory.cpu.{CpuMemParams, UncachedRequest, UncachedResponse} - -/** Uncached CPU accesses share one AXI master; responses retain the CPU port and tag. */ -@instantiable -class UncachedAxi(cp: CpuMemParams, ap: axi4.Params, ports: Int) extends Module { - - @public - val io = IO(new Bundle { - val request = Vec(ports, Flipped(Decoupled(new UncachedRequest(cp)))) - val response = Vec(ports, Decoupled(new UncachedResponse(cp))) - val axi = new axi4.Port(ap) - }) - - val indexBits = math.max(1, log2Ceil(ports)) - val idle :: issue :: waitResponse :: respond :: Nil = Enum(4) - val state = RegInit(idle) - val cursor = RegInit(0.U(indexBits.W)) - val owner = Reg(UInt(indexBits.W)) - val command = Reg(new UncachedRequest(cp)) - val answer = Reg(new UncachedResponse(cp)) - val addressSent = RegInit(false.B) - val dataSent = RegInit(false.B) - val ordered = (0 until ports).map(i => (cursor +& i.U) % ports.U) - val selected = if (ports == 1) 0.U else PriorityMux(ordered.map(i => io.request(i).valid), ordered) - val present = io.request.map(_.valid).reduce(_ || _) - for (i <- 0 until ports) { - io.request(i).ready := state === idle && present && selected === i.U - io.response(i).valid := state === respond && owner === i.U - io.response(i).bits := answer - when(io.response(i).fire)(state := idle) - } - when(state === idle && present) { - owner := selected - command := (if (ports == 1) io.request.head.bits else io.request(selected).bits) - addressSent := false.B - dataSent := false.B - cursor := Mux(selected === (ports - 1).U, 0.U, selected + 1.U) - state := issue - } - for (address <- Seq(io.axi.aw, io.axi.ar)) { - address.bits := 0.U.asTypeOf(address.bits) - address.bits.addr := command.addr - address.bits.size := command.size - address.bits.burst := 1.U - } - io.axi.aw.valid := state === issue && command.write && !addressSent - io.axi.ar.valid := state === issue && !command.write - val shift = command.addr(log2Ceil(ap.bytes) - 1, 0) - io.axi.w.valid := state === issue && command.write && !dataSent - io.axi.w.bits.data := command.data << (shift << 3) - io.axi.w.bits.strb := (((1.U((ap.bytes + 1).W) << (1.U << command.size)) - 1.U) << shift) - io.axi.w.bits.last := true.B - when(io.axi.aw.fire)(addressSent := true.B) - when(io.axi.w.fire)(dataSent := true.B) - when(io.axi.ar.fire || (command.write && (addressSent || io.axi.aw.fire) && (dataSent || io.axi.w.fire))) { - when(state === issue)(state := waitResponse) - } - io.axi.r.ready := state === waitResponse && !command.write - io.axi.b.ready := state === waitResponse && command.write - when(io.axi.r.fire || io.axi.b.fire) { - when(io.axi.r.fire)(assert(io.axi.r.bits.last)) - answer.tag := command.tag - answer.data := Mux(command.write, 0.U, io.axi.r.bits.data >> (shift << 3)) - answer.error := Mux(command.write, io.axi.b.bits.resp =/= 0.U, io.axi.r.bits.resp =/= 0.U) - state := respond - } -} diff --git a/arch/src/main/scala/framework/system/tile/BankNetwork.scala b/arch/src/main/scala/framework/system/tile/BankNetwork.scala deleted file mode 100644 index 6946efdd6..000000000 --- a/arch/src/main/scala/framework/system/tile/BankNetwork.scala +++ /dev/null @@ -1,113 +0,0 @@ -package framework.system.tile - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.system.core.clink.ShmPort -import framework.memdomain.backend.shared.{SharedLeasePort, SharedMemBackend, SharedMemLayout, SharedPhysicalPort} -import framework.memdomain.backend.banks.btrace.PhysicalBankHash -import framework.memdomain.frontend.mem.MemConfigerIO -import framework.memdomain.isa.{MvoverCommand, MvoverPort} - -/** Shared banks and inter-core bank movement; transport and CPU command decoding stay outside. */ -@instantiable -class BankNetwork( - b: GlobalConfig, - enabledCoreIds: Seq[Int], - useMesh: Boolean, - controllerMove: Boolean, - physicalPorts: Int = 0) - extends Module { - require(b.memDomain.sharedEnable) - require(enabledCoreIds.nonEmpty && enabledCoreIds == b.memDomain.computeCoreIds) - require(enabledCoreIds.distinct.size == enabledCoreIds.size && - enabledCoreIds.forall(i => i >= 0 && i < b.memDomain.nCores)) - val nCores = b.memDomain.nCores - val channels = SharedMemLayout.channelPerHart(b) - val controllerPorts = if (controllerMove) 1 else 0 - val movePorts = enabledCoreIds.size + controllerPorts - - @public - val io = IO(new Bundle { - val lease = Option.when(physicalPorts > 0)(Flipped(new SharedLeasePort)) - val physical = Vec(physicalPorts, Flipped(new SharedPhysicalPort(b.memDomain.bankWidth))) - val compute = Vec(enabledCoreIds.size, Flipped(new ShmPort(b))) - val hartIds = Input(Vec(nCores, UInt(b.tile.xLen.W))) - val controllerMvover = if (controllerMove) Some(Flipped(new MvoverPort)) else None - val bankHashes = - if (b.sim.diffTest) Some(Output(Vec(SharedMemLayout.totalBank(b), new PhysicalBankHash(b)))) else None - }) - - val sharedBackend: Instance[SharedMemBackend] = Instantiate(new SharedMemBackend(b, useMesh, physicalPorts)) - io.lease.foreach { lease => - sharedBackend.io.lease.get <> lease - val endpoint = lease.request.bits.endpoint - val owner = MuxLookup(endpoint, 0.U(b.tile.xLen.W))( - enabledCoreIds.zipWithIndex.map { case (physical, logical) => logical.U -> io.hartIds(physical) } - ) - sharedBackend.io.leaseOwnerHartId.get := owner - when(lease.request.valid && !reset.asBool) { - assert(endpoint < enabledCoreIds.size.U, "Shared lease endpoint is not a configured NPU") - } - } - sharedBackend.io.physical <> io.physical - sharedBackend.io.physicalOwnerHartId.foreach(_ := io.hartIds(0)) - io.bankHashes.foreach(_ := sharedBackend.io.bank_hashes.get) - for ((physical, compute) <- enabledCoreIds.zipWithIndex) { - val port = io.compute(compute) - for (channel <- 0 until channels) { - sharedBackend.io.mem_req(compute * channels + channel) <> port.requests(channel) - } - sharedBackend.io.localBanks(physical) <> port.local - sharedBackend.io.query_valid(physical) := port.queryValid - sharedBackend.io.query_hart_id(physical) := io.hartIds(physical) - sharedBackend.io.query_vbank_id(physical) := port.queryVbank - port.queryGroups := sharedBackend.io.query_group_count(physical) - } - for (physical <- 0 until nCores if !enabledCoreIds.contains(physical)) { - val local = sharedBackend.io.localBanks(physical) - local.request.ready := false.B - local.response.valid := false.B - local.response.bits := 0.U.asTypeOf(local.response.bits) - sharedBackend.io.query_valid(physical) := false.B - sharedBackend.io.query_hart_id(physical) := 0.U - sharedBackend.io.query_vbank_id(physical) := 0.U - } - - val moveArb = Module(new Arbiter(new MvoverCommand, movePorts)) - val moveOwner = RegInit(0.U(math.max(1, log2Ceil(movePorts)).W)) - val completionReady = Wire(Vec(movePorts, Bool())) - io.controllerMvover.foreach { port => - moveArb.io.in(0) <> port.command - port.completion.valid := sharedBackend.io.mvover.completion.valid && moveOwner === 0.U - port.completion.bits := sharedBackend.io.mvover.completion.bits - completionReady(0) := port.completion.ready - } - for (compute <- enabledCoreIds.indices) { - val port = compute + controllerPorts - moveArb.io.in(port) <> io.compute(compute).move.command - io.compute(compute).move.completion.valid := sharedBackend.io.mvover.completion.valid && moveOwner === port.U - io.compute(compute).move.completion.bits := sharedBackend.io.mvover.completion.bits - completionReady(port) := io.compute(compute).move.completion.ready - } - sharedBackend.io.mvover.command <> moveArb.io.out - - def physicalComputeCore(logical: UInt): UInt = MuxLookup(logical, nCores.U(8.W))( - enabledCoreIds.zipWithIndex.map { case (physical, logicalIndex) => logicalIndex.U -> physical.U(8.W) } - ) - - sharedBackend.io.mvover.command.bits.sourceCore := physicalComputeCore(moveArb.io.out.bits.sourceCore) - sharedBackend.io.mvover.command.bits.targetCore := physicalComputeCore(moveArb.io.out.bits.targetCore) - sharedBackend.io.mvover.completion.ready := completionReady(moveOwner) - when(moveArb.io.out.fire)(moveOwner := moveArb.io.chosen) - - val configArb = Module(new Arbiter(new MemConfigerIO(b), enabledCoreIds.size)) - for (compute <- enabledCoreIds.indices) { configArb.io.in(compute) <> io.compute(compute).config } - sharedBackend.io.config <> configArb.io.out - val barrier: Instance[BarrierUnit] = Instantiate(new BarrierUnit(enabledCoreIds.size)) - for (compute <- enabledCoreIds.indices) { - barrier.io.arrive(compute) := io.compute(compute).barrierArrive - io.compute(compute).barrierRelease := barrier.io.release(compute) - } -} diff --git a/arch/src/main/scala/framework/system/tile/BarrierUnit.scala b/arch/src/main/scala/framework/system/tile/BarrierUnit.scala deleted file mode 100644 index 5f54584a8..000000000 --- a/arch/src/main/scala/framework/system/tile/BarrierUnit.scala +++ /dev/null @@ -1,35 +0,0 @@ -package framework.system.tile - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} - -/** - * BarrierUnit — tile-level hardware barrier for multi-core synchronization. - * - * Each core's BuckyballAccelerator raises arrive(i) after its ROB drains. - * When all N cores have arrived, release(i) is asserted for one cycle, - * allowing all cores to proceed simultaneously. - * - * @param nCores number of cores in this tile - */ -@instantiable -class BarrierUnit(val nCores: Int) extends Module { - - @public - val io = IO(new Bundle { - val arrive = Input(Vec(nCores, Bool())) - val release = Output(Vec(nCores, Bool())) - }) - - val arrived = RegInit(VecInit(Seq.fill(nCores)(false.B))) - val allArrived = arrived.asUInt.andR - - for (i <- 0 until nCores) { - when(io.arrive(i))(arrived(i) := true.B) - io.release(i) := allArrived - } - - when(allArrived) { - for (i <- 0 until nCores) { arrived(i) := false.B } - } -} diff --git a/arch/src/main/scala/framework/system/tile/ComputeTilePlatform.scala b/arch/src/main/scala/framework/system/tile/ComputeTilePlatform.scala deleted file mode 100644 index 42bb2e90e..000000000 --- a/arch/src/main/scala/framework/system/tile/ComputeTilePlatform.scala +++ /dev/null @@ -1,228 +0,0 @@ -package framework.system.tile - -import chisel3._ -import framework.system.tile.{BankNetwork, TileEndpoint, TileParams} -import framework.system.tile.tlink.{ - HasTLink, - TLinkIO, - T2TParams, - T2TTransfer, - SharedAxiEndpoint, - Control => TransferControl -} -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.ant.Service -import framework.system.configloader.{AntTileCore, RocketTileCore} -import framework.system.core.{ControllerAdmission, RocketCLink} -import hier.core.rocket.Commands -import framework.system.core.rocket.CpuParams -import framework.system.core.accelerator.{Admission, AntMemory} -import framework.memdomain.frontend.mem.dma.DmaStatus -import framework.memdomain.backend.shared.SharedMemLayout -import hier.tile.memory.{Composition, CorePlacement, CoreRole} -import memcore.bus.axi4 -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.interlock.{Params => TrackingParams} -import memcore.memory.preflight.{Params => PreparationParams} - -/** One coherent Linux controller, local Ant execution contexts, and their NPU endpoints. */ -@instantiable -class ComputeTilePlatform(p: TileParams) extends TileEndpoint(p.linkParams, false) { - require(!p.main && p.controller.contains(0) && p.cpus.size == 1) - - val controller = p.cores.head match { - case core: RocketTileCore => core - case _ => throw new IllegalArgumentException("Compute controller must be Rocket") - } - - val workers = p.cores.tail.map { - case core: AntTileCore => core - case _ => throw new IllegalArgumentException("Compute workers must be Ant") - } - - val signatures = p.signatures.tail - val cpu = p.cpus.head - val memory = p.memory - val l1 = p.l1 - val regions = p.regions - val tracking = p.tracking - val axiParams = p.axi - val tiles = p.tiles - require(controller.buckyball.isEmpty && workers.nonEmpty && workers.size == signatures.size) - val local = workers.head.local - require(workers.forall(_.local == local), "Ant local storage geometry must be homogeneous within a tile") - val base = workers.head.buckyball - val enabled = 1 to workers.size - require(workers.forall(w => - w.buckyball.memDomain.sharedEnable && - w.buckyball.memDomain.computeCoreIds == enabled && w.buckyball.memDomain.nCores == workers.size + 1 - )) - val c = memory.chi - val prepared = - PreparationParams(c, tracking.entries, tracking.maxRanges, tracking.idBits, beatBytes = axiParams.bytes) - - val tileMemory = Instantiate(new axi4.Interconnect(axiParams, workers.size)) - tlink.mem <> tileMemory.io.out - - val composition = Instantiate(new Composition( - memory.copy(agents = 2), - Seq(CorePlacement(l1.copy(nodeId = 1), cpu, CoreRole.Controller)), - regions, - tracking, - Nil, - Nil, - Seq(0) - )) - - @public val controllerLink = IO(Flipped(new RocketCLink(p.rocket(0)))) - - @public val workerLinks = IO(MixedVec((1 until p.cores.size).map { i => - val params = p.ant(i) - Flipped(new framework.system.core.clink.CLinkIO( - params.buckyball, - params.local, - params.tracking, - params.bus - )(params.cpu)) - })) - - composition.io.cores(0) <> controllerLink.cpu - - val controllerAdmission = Instantiate(new ControllerAdmission(tracking, c, moves = true)(cpu)) - val service = Instantiate(new Service(local, workers.size)) - val broker = Instantiate(new AntMemory(workers.size, tracking, c)) - val banks = Instantiate(new BankNetwork(base, enabled, useMesh = true, controllerMove = true, physicalPorts = 3)) - - val transferParams = T2TParams( - sharedBytes = BigInt(SharedMemLayout.totalBank(base)) * base.memDomain.sharedBankEntries * axiParams.bytes, - bankBytes = base.memDomain.sharedBankEntries * axiParams.bytes, - tiles = tiles, - axi = axiParams - ) - - val transferControl = Instantiate(new TransferControl(transferParams)) - val transfer = Instantiate(new T2TTransfer(transferParams)) - val sharedEndpoint = Instantiate(new SharedAxiEndpoint(transferParams)) - transferControl.io.tileId := tlink.tileId - transfer.io.command <> transferControl.io.command - transferControl.io.completion <> transfer.io.completion - banks.io.physical(0) <> transfer.io.source - banks.io.physical(1) <> sharedEndpoint.io.read - banks.io.physical(2) <> sharedEndpoint.io.write - banks.io.lease.get <> transferControl.io.lease.get - tlink.t2t.get.tx <> transfer.io.mem - sharedEndpoint.io.mem <> tlink.t2t.get.rx - composition.io.hartIds := tlink.hartIds - composition.io.resetVector := tlink.resetVector - composition.io.time := tlink.time - composition.io.interrupts := tlink.interrupts - tlink.control <> composition.io.control - tlink.uncachedRequest <> composition.io.uncachedRequest - composition.io.uncachedResponse <> tlink.uncachedResponse - tlink.retired := composition.io.retired - tlink.retiredPc := composition.io.retiredPc - tlink.trapped := composition.io.trapped - tlink.trapCause := composition.io.trapCause - tlink.trapValue := composition.io.trapValue - tlink.trapPc := composition.io.trapPc - - // No hidden CPU polls the legacy task port. Only the controller drives this service. - composition.io.controllerSatp := 0.U - for (port <- composition.io.taskControl) { - port.cmd.valid := false.B - port.cmd.bits := 0.U.asTypeOf(port.cmd.bits) - port.resp.ready := true.B - port.exception := false.B - } - val core = composition.io.admission(0) - controllerAdmission.io.core <> core - val task = controllerAdmission.io.task - val rd = Reg(UInt(5.W)) - val isTransfer = task.cmd.bits.funct >= 13.U - transferControl.io.request.bits.operation := task.cmd.bits.funct - transferControl.io.request.bits.context := task.cmd.bits.rs1Data(63, 32) - transferControl.io.request.bits.field := task.cmd.bits.rs1Data(31, 0) - transferControl.io.request.bits.data := task.cmd.bits.rs2Data - transferControl.io.request.valid := task.cmd.valid && isTransfer - service.io.request.valid := task.cmd.valid && !isTransfer - service.io.request.bits.operation := task.cmd.bits.funct - service.io.request.bits.context := task.cmd.bits.rs1Data(63, 32) - service.io.request.bits.field := task.cmd.bits.rs1Data(31, 0) - service.io.request.bits.data := task.cmd.bits.rs2Data - task.cmd.ready := Mux(isTransfer, transferControl.io.request.ready, service.io.request.ready) - when(task.cmd.fire) { - assert(task.cmd.bits.opcode === "h2b".U && task.cmd.bits.funct3 === 7.U, "Invalid Ant management instruction") - rd := task.cmd.bits.rd - } - task.resp.valid := service.io.reply.valid || transferControl.io.reply.valid - task.resp.bits.rd := rd - task.resp.bits.data := Mux(transferControl.io.reply.valid, transferControl.io.reply.bits, service.io.reply.bits) - transferControl.io.reply.ready := task.resp.ready - assert(!(service.io.reply.valid && transferControl.io.reply.valid), "Concurrent tile management responses") - service.io.reply.ready := task.resp.ready - task.busy := !service.io.idle || !transferControl.io.idle - task.interrupt := false.B - banks.io.hartIds := tlink.executionIds - banks.io.controllerMvover.get <> controllerAdmission.io.move - - broker.io.controller.cpuQuery := core.cpuQuery - core.cpuAllow := controllerAdmission.io.core.cpuAllow && broker.io.controller.cpuAllow - core.cpuProbeAllow := controllerAdmission.io.core.cpuProbeAllow && broker.io.controller.cpuProbeAllow - core.maintenance <> broker.io.controller.maintenance - broker.io.controller.maintained <> core.maintained - core.pteRequest <> broker.io.controller.pteRequest - broker.io.controller.pteResponse <> core.pteResponse - - val failures = Wire(Vec(workers.size, Valid(new DmaStatus))) - val drained = Wire(Vec(workers.size, Bool())) - for ((worker, i) <- workers.zipWithIndex) { - val b = worker.buckyball - val link = workerLinks(i) - val admission = Instantiate(new Admission(b, tracking, prepared, regions, axiParams)(cpu)) - link.ctrl.bind.valid := service.io.launched(i).valid - link.ctrl.bind.bits.task := service.io.launched(i).bits.task - link.ctrl.bind.bits.snapshot := controllerAdmission.io.taskContext - service.io.locals(i) <> link.local - link.ctrl.workDrained := admission.io.workDrained - link.ctrl.halted := admission.io.halted - drained(i) := link.ctrl.drained - service.io.signatures(i) := link.ctrl.fingerprint - service.io.online(i) := link.ctrl.online - link.admission <> admission.io.core - link.memory <> broker.io.clients(i) - admission.io.task.cmd.ready := false.B - admission.io.task.resp.valid := false.B - admission.io.task.resp.bits := 0.U.asTypeOf(admission.io.task.resp.bits) - admission.io.task.busy := false.B - admission.io.task.interrupt := false.B - link.npu.command <> admission.io.npuCommand - admission.io.npuResponse <> link.npu.response - admission.io.allocation.valid := link.npu.allocation.valid - admission.io.allocation.bits := link.npu.allocation.bits.rob_id - admission.io.retired := link.npu.retired - admission.io.npuFault := link.npu.fault - admission.io.npuBusy := link.npu.busy - admission.io.npuInterrupt := link.npu.interrupt - admission.io.footprints := link.npu.footprints - admission.io.dma <> link.mem - tileMemory.io.in(i) <> admission.io.axi - failures(i).valid := admission.io.halted - failures(i).bits := admission.io.fault - link.ctrl.hartId := tlink.executionIds(i + 1) - link.ctrl.shmOwner := tlink.executionIds(0) - banks.io.compute(i) <> link.shm - link.sharedHashes.foreach(_ := banks.io.bankHashes.get) - } - tlink.failure(0).valid := failures.map(_.valid).reduce(_ || _) || controllerAdmission.io.moveFault.valid - tlink.failure(0).bits := Mux( - controllerAdmission.io.moveFault.valid, - controllerAdmission.io.moveFault.bits, - PriorityMux(failures.map(f => f.valid -> f.bits)) - ) - tlink.workDrained( - 0 - ) := service.io.idle && transferControl.io.idle && !sharedEndpoint.io.busy && drained.asUInt.andR && core.outstanding === 0.U -} diff --git a/arch/src/main/scala/framework/system/tile/MainTile.scala b/arch/src/main/scala/framework/system/tile/MainTile.scala deleted file mode 100644 index df8744801..000000000 --- a/arch/src/main/scala/framework/system/tile/MainTile.scala +++ /dev/null @@ -1,15 +0,0 @@ -package framework.system.tile - -import chisel3.experimental.hierarchy.{instantiable, Instantiate} -import framework.top.GlobalConfig -import framework.system.GlobalConfigOps._ -import framework.system.core.CoreFactory - -@instantiable -class MainTile(b: GlobalConfig) extends TileEndpoint(b.tileLinkParams, true) { - require(b.tileDesign.get.design.factory.isEmpty, "The main tile is built into the device") - val p = b.tileParams - val platform = Instantiate(new MainTilePlatform(p)) - val cores = p.cores.indices.map(i => CoreFactory.rocket(b.forCore(i))) - TileConnection.main(platform, tlink, cores.map(_.clink)) -} diff --git a/arch/src/main/scala/framework/system/tile/MainTilePlatform.scala b/arch/src/main/scala/framework/system/tile/MainTilePlatform.scala deleted file mode 100644 index d9c428a99..000000000 --- a/arch/src/main/scala/framework/system/tile/MainTilePlatform.scala +++ /dev/null @@ -1,406 +0,0 @@ -package framework.system.tile - -import chisel3._ -import framework.system.tile.{BankNetwork, TileEndpoint, TileParams} -import framework.system.tile.tlink.{ - HasTLink, - TLinkIO, - T2TParams, - T2TTransfer, - SharedAxiEndpoint, - Control => TransferControl -} -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, RoCCIO} -import framework.system.configloader.{AntTileCore, RocketTileCore, SharedStorageParams, TileCore, TileTopology} -import framework.system.core.rocket.configs.RocketCpuParam -import framework.system.core.{ControllerAdmission, RocketCLink} -import hier.core.rocket.Commands -import framework.system.core.accelerator.Admission -import framework.memdomain.frontend.mem.dma.DmaStatus -import framework.memdomain.backend.shared.{SharedMemLayout, SharedPhysicalPort, SharedPhysicalRequest} -import framework.memdomain.backend.MTraceDPI -import memcore.memory.mesh_shm.{MeshSharedMem, MeshSharedMemParams} -import hier.tile.memory.{Composition, CoreInterrupts, CorePlacement, CoreRole} -import memcore.bus.chi.RequesterPort -import memcore.bus.chi.rnf.RnfParams -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion, UncachedRequest, UncachedResponse} -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.interlock.{Params => TrackingParams} -import memcore.memory.preflight.{Params => PreparationParams} - -/** A PB-defined Tile assembled explicitly from its CPU, cache, accelerator and bank IPs. */ -@instantiable -class MainTilePlatform(p: TileParams) extends TileEndpoint(p.linkParams, true) { - val coreTypes = p.cores - val cpuParameters = p.cpus - val memory = p.memory - val l1 = p.l1 - val regions = p.regions - val tracking = p.tracking - val controlCores = p.controls - val axiParams = p.axi - val sharedStorage = p.sharedStorage - val tiles = p.tiles - require(p.controller.isEmpty, "Main tile has no hidden task controller") - require(coreTypes.size == cpuParameters.size) - require(memory.agents == 2 * coreTypes.size) - - val cores = coreTypes.map { - case core: RocketTileCore => core - case _ => throw new IllegalArgumentException("This Tile requires Rocket CPUs") - } - - // A CPU-only Tile (for example a main tile without Buckyball cores) has no banks or NPU DMA. - val enabled = cores.indices.filter(i => cores(i).buckyball.isDefined) - val base = enabled.headOption.map(i => cores(i).buckyball.get) - // Shared banks form the Tile bank mesh; otherwise every accelerator keeps its private banks. - val shared = base.exists(_.memDomain.sharedEnable) - for (b <- base) { - require(b.memDomain.nCores == cores.size) - require(b.memDomain.computeCoreIds == enabled) - } - for { - i <- enabled - base <- base - } { - val b = cores(i).buckyball.get - require(b.tile.xLen == 64 && b.memDomain.nCores == cores.size && b.memDomain.sharedEnable == shared) - require(b.memDomain.bankWidth == base.memDomain.bankWidth && - b.memDomain.dma_buswidth == base.memDomain.dma_buswidth && - b.memDomain.bankChannel == base.memDomain.bankChannel && - b.frontend.rob_entries == base.frontend.rob_entries) - } - val c = memory.chi - val cp = CpuMemParams(c, tagBits = 6) - val prepared = - PreparationParams(c, tracking.entries, tracking.maxRanges, tracking.idBits, beatBytes = axiParams.bytes) - - val placements = cores.indices.map { i => - CorePlacement( - l1.copy(nodeId = i + 1), - cpuParameters(i), - CoreRole(compute = cores(i).buckyball.isDefined, scheduler = true) - ) - } - - val controlCoreIds = cores.indices - - // TLink and NPU requests share the same bank network when main has an accelerator. - val controllerMove = shared && cores.head.buckyball.isEmpty - - val banks: Option[Instance[BankNetwork]] = Option.when(shared)( - Instantiate(new BankNetwork( - base.get, - enabled, - useMesh = true, - controllerMove = controllerMove, - physicalPorts = if (sharedStorage.isDefined) 4 else 0 - )) - ) - - val mainTransfer = sharedStorage.map { geometry => - require(enabled.isEmpty || shared, "Main NPU and TLink require one shared bank network") - val p = T2TParams(geometry.bytes, geometry.bankBytes, tiles, axiParams) - val control = Instantiate(new TransferControl(p, localAccess = true, leaseAccess = banks.isDefined)) - val transfer = Instantiate(new T2TTransfer(p)) - val endpoint = Instantiate(new SharedAxiEndpoint(p)) - val physical = Wire(Vec(4, Flipped(new SharedPhysicalPort(128)))) - val pending = RegInit(VecInit(Seq.fill(4)(false.B))) - banks match { - case Some(network) => - val b = base.get - require( - b.memDomain.bankWidth == geometry.bankBits && - b.memDomain.sharedBankEntries == geometry.bankEntries && - SharedMemLayout.totalBank(b) == geometry.banks, - "Main NPU and TLink storage geometry must agree" - ) - network.io.physical <> physical - network.io.lease.get <> control.io.lease.get - for ((port, index) <- physical.zipWithIndex) { - when(port.request.fire) { - assert(!pending(index), "Main SHM port accepts one outstanding request") - pending(index) := true.B - } - when(port.response.fire) { - assert(pending(index) && !port.response.bits.error, "Main SHM response has no owner or failed") - pending(index) := false.B - } - } - case None => - val (rows, columns) = MeshSharedMemParams.dimensions(geometry.banks) - val core = MeshSharedMemParams.directionalAttachments(rows, columns, Seq(1)).head - val shm = Instantiate(new MeshSharedMem(MeshSharedMemParams( - rows = rows, - cols = columns, - entriesPerBank = geometry.bankEntries, - dataBits = geometry.bankBits, - tagBits = 8, - cores = Seq(core), - externalChannels = 4, - visibleBanks = geometry.banks - ))) - // Main has no private NPU endpoint. Its CPU management and T2T use physical SHM ports. - for (channel <- shm.io.channels) { - channel.request.valid := false.B - channel.request.bits := 0.U.asTypeOf(channel.request.bits) - channel.response.ready := true.B - } - shm.io.transferCommand.valid := false.B - shm.io.transferCommand.bits := 0.U.asTypeOf(shm.io.transferCommand.bits) - shm.io.transferCompletion.ready := true.B - for (local <- shm.io.localBanks) { - local.request.ready := false.B - local.response.valid := false.B - local.response.bits := 0.U.asTypeOf(local.response.bits) - } - shm.io.bankWrites.foreach(_.ready := true.B) - val bankBits = log2Ceil(geometry.bankBytes) - for ((port, index) <- physical.zipWithIndex) { - val channel = shm.io.external(index) - val held = Reg(new SharedPhysicalRequest(128)) - when(port.request.valid && !reset.asBool) { - assert( - port.request.bits.address(3, 0) === 0.U && port.request.bits.address < geometry.bytes.U, - "Main SHM physical access is unaligned or exceeds capacity" - ) - } - channel.request.valid := port.request.valid - channel.request.bits := 0.U.asTypeOf(channel.request.bits) - channel.request.bits.bank := port.request.bits.address >> bankBits - channel.request.bits.tuser := Cat(port.request.bits.address(bankBits - 1, 4).pad(16), port.request.bits.write) - channel.request.bits.data := port.request.bits.data - channel.request.bits.mask := port.request.bits.mask - channel.request.bits.tag := 0.U - channel.request.bits.tlast := true.B - port.request.ready := channel.request.ready - port.response.valid := channel.response.valid - port.response.bits.data := channel.response.bits.data - port.response.bits.error := channel.response.bits.error - channel.response.ready := port.response.ready - when(port.request.fire) { - assert(!pending(index), "Main SHM port accepts one outstanding request") - pending(index) := true.B - held := port.request.bits - } - when(port.response.fire) { - assert(pending(index) && !port.response.bits.error, "Main SHM response has no owner or failed") - pending(index) := false.B - } - val trace = Module(new MTraceDPI) - val value = Mux(held.write, held.data, port.response.bits.data) - trace.io.clock := clock - trace.io.reset := reset.asBool - trace.io.hart_id := tlink.executionIds(0) - trace.io.channel := (shm.io.channels.size + index).U - trace.io.is_shared := 1.U - trace.io.is_write := held.write.asUInt - trace.io.rob_id := 0.U - trace.io.inst_id := 0.U - trace.io.vbank_id := 0.U - trace.io.group_id := 0.U - trace.io.pbank_id := held.address >> bankBits - trace.io.addr := held.address(bankBits - 1, 4) - trace.io.write_mask := Mux(held.write, held.mask, 0.U) - trace.io.data_lo := value(63, 0) - trace.io.data_hi := value(127, 64) - trace.io.enable := port.response.fire - } - } - control.io.tileId := tlink.tileId - transfer.io.command <> control.io.command - control.io.completion <> transfer.io.completion - physical(0) <> transfer.io.source - physical(1) <> endpoint.io.read - physical(2) <> endpoint.io.write - physical(3) <> control.io.local.get - tlink.t2t.get.tx <> transfer.io.mem - endpoint.io.mem <> tlink.t2t.get.rx - (control, endpoint, pending.asUInt.orR) - } - - val tileMemory = Option.when(enabled.nonEmpty)( - Instantiate(new memcore.bus.axi4.Interconnect(axiParams, enabled.size)) - ) - - tileMemory match { - case Some(fabric) => tlink.mem <> fabric.io.out - case None => - tlink.mem.aw.valid := false.B - tlink.mem.aw.bits := 0.U.asTypeOf(tlink.mem.aw.bits) - tlink.mem.w.valid := false.B - tlink.mem.w.bits := 0.U.asTypeOf(tlink.mem.w.bits) - tlink.mem.ar.valid := false.B - tlink.mem.ar.bits := 0.U.asTypeOf(tlink.mem.ar.bits) - tlink.mem.b.ready := true.B - tlink.mem.r.ready := true.B - } - - val composition: Instance[Composition] = Instantiate(new Composition( - memory, - placements, - regions, - tracking, - Nil, - Nil, - controlCoreIds - )) - - composition.io.hartIds := tlink.hartIds - composition.io.resetVector := tlink.resetVector - composition.io.time := tlink.time - composition.io.interrupts := tlink.interrupts - tlink.uncachedRequest <> composition.io.uncachedRequest - composition.io.uncachedResponse <> tlink.uncachedResponse - locally { - val memorySystem = Instantiate(new hier.tile.memory.Memory(memory.copy(agents = 2 * controlCores))) - for (lane <- 0 until 2 * cores.size) { - val localNode = lane + 1 - val globalIndex = lane % cores.size + (if (lane >= cores.size) controlCores else 0) - val source = composition.io.control(lane) - val target = memorySystem.io.coherent(globalIndex) - target <> source - target.req.bits.srcId := (globalIndex + 1).U - target.txRsp.bits.srcId := (globalIndex + 1).U - target.txDat.bits.srcId := (globalIndex + 1).U - source.rxRsp.bits.tgtId := localNode.U - source.rxDat.bits.tgtId := localNode.U - } - val remoteCount = controlCores - cores.size - for (lane <- 0 until 2 * remoteCount) { - val globalIndex = cores.size + lane % remoteCount + (if (lane >= remoteCount) controlCores else 0) - memorySystem.io.coherent(globalIndex) <> tlink.remoteControl(lane) - } - tlink.backingRequest <> memorySystem.io.backingReq - memorySystem.io.backingResp <> tlink.backingResponse - } - tlink.retired := composition.io.retired - tlink.retiredPc := composition.io.retiredPc - tlink.trapped := composition.io.trapped - tlink.trapCause := composition.io.trapCause - tlink.trapValue := composition.io.trapValue - tlink.trapPc := composition.io.trapPc - banks.foreach(_.io.hartIds := tlink.executionIds) - tlink.failure := 0.U.asTypeOf(tlink.failure) - - def connectTask(i: Int, task: RoCCIO, npuIdle: Bool = true.B): Unit = { - val target = composition.io.taskControl(i) - if (i == 0 && mainTransfer.isDefined) { - val (control, _, _) = mainTransfer.get - val transfer = task.cmd.bits.funct >= 13.U - val accessesStorage = task.cmd.bits.funct === 14.U || - (task.cmd.bits.funct >= 16.U && task.cmd.bits.funct <= 19.U) - val transferReady = !accessesStorage || npuIdle - val rd = Reg(UInt(5.W)) - control.io.request.valid := task.cmd.valid && transfer && transferReady - control.io.request.bits.operation := task.cmd.bits.funct - control.io.request.bits.context := task.cmd.bits.rs1Data(63, 32) - control.io.request.bits.field := task.cmd.bits.rs1Data(31, 0) - control.io.request.bits.data := task.cmd.bits.rs2Data - target.cmd.valid := task.cmd.valid && !transfer - target.cmd.bits := task.cmd.bits - target.exception := task.exception - task.cmd.ready := Mux(transfer, control.io.request.ready && transferReady, target.cmd.ready) - when(task.cmd.fire) { - assert(task.cmd.bits.opcode === "h2b".U && task.cmd.bits.funct3 === 7.U, "Invalid main management instruction") - rd := task.cmd.bits.rd - } - assert(!(control.io.reply.valid && target.resp.valid), "Concurrent main management responses") - task.resp.valid := control.io.reply.valid || target.resp.valid - task.resp.bits.rd := Mux(control.io.reply.valid, rd, target.resp.bits.rd) - task.resp.bits.data := Mux(control.io.reply.valid, control.io.reply.bits, target.resp.bits.data) - control.io.reply.ready := task.resp.ready - target.resp.ready := task.resp.ready - task.busy := !control.io.idle || target.busy - task.interrupt := target.interrupt - } else task <> target - } - - @public val coreLinks = IO(MixedVec(cores.indices.map(i => Flipped(new RocketCLink(p.rocket(i)))))) - for (i <- cores.indices) { - composition.io.cores(i) <> coreLinks(i).cpu - cores(i).buckyball match { - case Some(b) => - val accelerator = coreLinks(i).accelerator.get - val admission: Instance[Admission] = - Instantiate(new Admission(b, tracking, prepared, regions, axiParams)(cpuParameters(i))) - admission.io.core <> composition.io.admission(i) - connectTask(i, admission.io.task, !accelerator.npu.busy) - if (i == 0) composition.io.controllerSatp := admission.io.taskSatp - if (i == 0 && mainTransfer.isDefined) { - // A younger NPU command cannot pass an active local storage-management command. - val storageIdle = mainTransfer.get._1.io.idle - accelerator.npu.command.valid := admission.io.npuCommand.valid && storageIdle - accelerator.npu.command.bits := admission.io.npuCommand.bits - admission.io.npuCommand.ready := accelerator.npu.command.ready && storageIdle - } else accelerator.npu.command <> admission.io.npuCommand - admission.io.npuResponse <> accelerator.npu.response - admission.io.allocation.valid := accelerator.npu.allocation.valid - admission.io.allocation.bits := accelerator.npu.allocation.bits.rob_id - admission.io.retired := accelerator.npu.retired - admission.io.npuFault := accelerator.npu.fault - admission.io.npuBusy := accelerator.npu.busy - admission.io.npuInterrupt := accelerator.npu.interrupt - admission.io.footprints := accelerator.npu.footprints - admission.io.dma <> accelerator.mem - val bankEndpoint = enabled.indexOf(i) - tileMemory.get.io.in(bankEndpoint) <> admission.io.axi - accelerator.hartId := tlink.executionIds(i) - accelerator.shmOwner := tlink.executionIds(0) - banks match { - case Some(network) => - val port = network.io.compute(bankEndpoint) - port.requests <> accelerator.shm.requests - port.move <> accelerator.shm.move - accelerator.shm.local <> port.local - port.config <> accelerator.shm.config - port.queryValid := accelerator.shm.queryValid - port.queryVbank := accelerator.shm.queryVbank - accelerator.shm.queryGroups := port.queryGroups - port.barrierArrive := accelerator.shm.barrierArrive - accelerator.shm.barrierRelease := port.barrierRelease - accelerator.sharedHashes.foreach(_ := network.io.bankHashes.get) - case None => - // Private banks: no shared channels exist, and mesh moves and barriers stay idle. - accelerator.shm.move.command.ready := false.B - accelerator.shm.move.completion.valid := false.B - accelerator.shm.move.completion.bits := false.B - accelerator.shm.local.request.valid := false.B - accelerator.shm.local.request.bits := 0.U.asTypeOf(accelerator.shm.local.request.bits) - accelerator.shm.local.response.ready := true.B - accelerator.shm.config.ready := false.B - accelerator.shm.queryGroups := 0.U - accelerator.shm.barrierRelease := false.B - } - tlink.failure(i).valid := admission.io.halted - tlink.failure(i).bits := admission.io.fault - tlink.workDrained(i) := admission.io.workDrained && - (if (i == 0) mainTransfer.map { case (control, endpoint, storageBusy) => - control.io.idle && !endpoint.io.busy && !storageBusy - }.getOrElse(true.B) - else true.B) - case None => - val admission: Instance[ControllerAdmission] = - Instantiate(new ControllerAdmission(tracking, c, moves = i == 0 && controllerMove)(cpuParameters(i))) - admission.io.core <> composition.io.admission(i) - connectTask(i, admission.io.task) - if (i == 0) composition.io.controllerSatp := admission.io.taskSatp - if (i == 0 && controllerMove) { - banks.get.io.controllerMvover.get <> admission.io.move - } else { - admission.io.move.command.ready := false.B - admission.io.move.completion.valid := false.B - admission.io.move.completion.bits := false.B - } - tlink.failure(i) := admission.io.moveFault - tlink.workDrained(i) := composition.io.admission(i).outstanding === 0.U && - (if (i == 0) mainTransfer.map { case (control, endpoint, storageBusy) => - control.io.idle && !endpoint.io.busy && !storageBusy - }.getOrElse(true.B) - else true.B) - } - } -} diff --git a/arch/src/main/scala/framework/system/tile/TileConnection.scala b/arch/src/main/scala/framework/system/tile/TileConnection.scala deleted file mode 100644 index 2c27661f5..000000000 --- a/arch/src/main/scala/framework/system/tile/TileConnection.scala +++ /dev/null @@ -1,30 +0,0 @@ -package framework.system.tile - -import chisel3._ -import chisel3.experimental.hierarchy.Instance -import framework.system.core.RocketCLink -import framework.system.core.clink.CLinkIO -import framework.system.tile.tlink.TLinkIO - -/** Cross-level binding is shared by all tile designs. */ -object TileConnection { - - def main(platform: Instance[MainTilePlatform], link: TLinkIO, cores: Seq[RocketCLink]): Unit = { - require(cores.size == platform.coreLinks.size) - cores.zipWithIndex.foreach { case (core, i) => platform.coreLinks(i) <> core } - link <> platform.tlink - } - - def compute( - platform: Instance[ComputeTilePlatform], - link: TLinkIO, - controller: RocketCLink, - workers: Seq[CLinkIO] - ): Unit = { - require(workers.size == platform.workerLinks.size) - platform.controllerLink <> controller - workers.zipWithIndex.foreach { case (core, i) => platform.workerLinks(i) <> core } - link <> platform.tlink - } - -} diff --git a/arch/src/main/scala/framework/system/tile/TileEndpoint.scala b/arch/src/main/scala/framework/system/tile/TileEndpoint.scala deleted file mode 100644 index b87580867..000000000 --- a/arch/src/main/scala/framework/system/tile/TileEndpoint.scala +++ /dev/null @@ -1,74 +0,0 @@ -package framework.system.tile - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance} -import framework.system.configloader.{AntTileCore, RocketTileCore, SharedStorageParams, TileCore} -import framework.system.core.rocket.CpuParams -import framework.system.core.{AntCoreParams, RocketCoreParams} -import hier.core.rocket.Commands -import framework.system.tile.tlink.{HasTLink, TLinkIO} -import memcore.bus.axi4 -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.interlock.{Params => TrackingParams} - -case class TileLinkParams( - main: Boolean, - cpus: Int, - executions: Int, - controls: Int, - bus: memcore.bus.chi.Params, - axi: axi4.Params, - sharedStorage: Boolean) - -/** Structural parameters only; tile IDs and hart IDs arrive through TLink. */ -case class TileParams( - main: Boolean, - cores: Seq[TileCore], - signatures: Seq[BigInt], - controller: Option[Int], - cpus: Seq[CpuParams], - memory: CoherenceParams, - l1: RnfParams, - regions: Seq[PhysicalRegion], - tracking: TrackingParams, - controls: Int, - axi: axi4.Params, - sharedStorage: Option[SharedStorageParams], - tiles: Int) { - - def linkParams: TileLinkParams = - TileLinkParams(main, cpus.size, cores.size, if (main) controls else 0, memory.chi, axi, sharedStorage.isDefined) - - def rocket(index: Int): RocketCoreParams = { - val core = cores(index) match { - case value: RocketTileCore => value - case _ => throw new IllegalArgumentException("Rocket parameters require a Rocket slot") - } - val cpuIndex = cores.take(index).count(_.isInstanceOf[RocketTileCore]) - RocketCoreParams( - cpus(cpuIndex), - l1.copy(nodeId = cpuIndex + 1), - l1.copy(nodeId = cpus.size + cpuIndex + 1), - regions, - Commands(core.buckyball.isDefined, true, tracking), - core.buckyball - ) - } - - def ant(index: Int): AntCoreParams = { - val core = cores(index) match { - case value: AntTileCore => value - case _ => throw new IllegalArgumentException("Ant parameters require an Ant slot") - } - AntCoreParams(core.buckyball, core.local, tracking, memory.chi, signatures(index), cpus.head) - } - -} - -@instantiable -abstract class TileEndpoint(p: TileLinkParams, main: Boolean) extends Module with HasTLink { - require(p.main == main, "Software tile role must match the designed TLink role") - @public val tlink = IO(new TLinkIO(p.main, p.cpus, p.executions, p.controls, p.bus, p.axi, p.sharedStorage)) -} diff --git a/arch/src/main/scala/framework/system/tile/TileFactory.scala b/arch/src/main/scala/framework/system/tile/TileFactory.scala deleted file mode 100644 index 5abafe0ad..000000000 --- a/arch/src/main/scala/framework/system/tile/TileFactory.scala +++ /dev/null @@ -1,18 +0,0 @@ -package framework.system.tile - -import chisel3.experimental.hierarchy.Instance -import framework.top.GlobalConfig - -trait TileFactory { - def instantiate(b: GlobalConfig): Instance[TileEndpoint] -} - -object TileFactory { - - def instantiate(b: GlobalConfig): Instance[TileEndpoint] = { - require(!b.tileDesign.get.design.main, "The main tile is built into the device") - Class.forName(b.tileDesign.get.design.factory + "$") - .getField("MODULE$").get(null).asInstanceOf[TileFactory].instantiate(b) - } - -} diff --git a/arch/src/main/scala/framework/system/tile/TileParameters.scala b/arch/src/main/scala/framework/system/tile/TileParameters.scala deleted file mode 100644 index b1b4a592b..000000000 --- a/arch/src/main/scala/framework/system/tile/TileParameters.scala +++ /dev/null @@ -1,34 +0,0 @@ -package framework.system.tile - -import chisel3.util.log2Ceil -import framework.system.core.rocket.CpuParams -import framework.system.core.rocket.configs.RocketCpuParam - -object TileParameters { - - def cpuParameters( - cpu: RocketCpuParam, - t: framework.system.tile.configs.TileParam, - maxHartId: Int, - physicalAddressBits: Int - ): CpuParams = { - require(physicalAddressBits > t.pgIdxBits && physicalAddressBits <= t.paddrBits) - require( - !cpu.useVM || physicalAddressBits < t.vaddrBits, - "Rocket requires platform physical addresses narrower than its virtual address format" - ) - val core = RocketCpuParam.toRocketCoreParams(cpu, t.xLen, t.pgLevels).copy( - nPMPs = t.nPMPs - ) - CpuParams( - core = core, - dcache = Some(RocketCpuParam.toDCacheParams(cpu, t.coreDataBytes * 8, 64)), - icache = Some(RocketCpuParam.toICacheParams(cpu, t.coreDataBytes * 8, 64)), - btb = RocketCpuParam.toBTBParams(cpu), - physicalAddressBits = physicalAddressBits, - hartIdBits = math.max(1, log2Ceil(maxHartId + 1)), - beatBytes = t.coreDataBytes - ) - } - -} diff --git a/arch/src/main/scala/framework/system/tile/configs/TileParam.scala b/arch/src/main/scala/framework/system/tile/configs/TileParam.scala deleted file mode 100644 index bb59efd4d..000000000 --- a/arch/src/main/scala/framework/system/tile/configs/TileParam.scala +++ /dev/null @@ -1,27 +0,0 @@ -package framework.system.tile.configs - -import upickle.default._ - -case class TileParam( - coreDataBytes: Int, - xLen: Int, - vaddrBits: Int, - paddrBits: Int, - pgIdxBits: Int, - pgLevels: Int, - nPMPs: Int) // Physical Memory Protection entries, typically 8 or 16 - -object TileParam { - implicit val rw: ReadWriter[TileParam] = macroRW - - def apply(): TileParam = TileParam( - coreDataBytes = 0, - xLen = 0, - vaddrBits = 0, - paddrBits = 0, - pgIdxBits = 0, - pgLevels = 0, - nPMPs = 0 - ) - -} diff --git a/arch/src/main/scala/framework/system/tile/tlink/Control.scala b/arch/src/main/scala/framework/system/tile/tlink/Control.scala deleted file mode 100644 index d52264a50..000000000 --- a/arch/src/main/scala/framework/system/tile/tlink/Control.scala +++ /dev/null @@ -1,129 +0,0 @@ -package framework.system.tile.tlink - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.ant.ControlRequest -import framework.memdomain.backend.shared.{SharedLeasePort, SharedPhysicalPort} - -/** The controller stages one transfer; the start instruction returns after destination commit. */ -@instantiable -class Control(p: T2TParams, localAccess: Boolean = false, leaseAccess: Boolean = true) extends Module { - - @public val io = IO(new Bundle { - val request = Flipped(Decoupled(new ControlRequest)) - val reply = Decoupled(UInt(64.W)) - val tileId = Input(UInt(p.tileBits.W)) - val command = Decoupled(new T2TCommand(p)) - val completion = Flipped(Decoupled(Bool())) - val idle = Output(Bool()) - val local = Option.when(localAccess)(new SharedPhysicalPort(128)) - val lease = Option.when(leaseAccess)(new SharedLeasePort) - }) - - val idle :: running :: reading :: leasing :: responding :: Nil = Enum(5) - val state = RegInit(idle) - val fields = RegInit(0.U(4.W)) - val staged = Reg(Vec(4, UInt(64.W))) - val answer = Reg(UInt(64.W)) - val request = io.request.bits - val start = request.operation === 14.U - val access = request.operation === 16.U || request.operation === 17.U - val lease = request.operation === 18.U || request.operation === 19.U - val upperHalf = Reg(Bool()) - val localWrite = Reg(Bool()) - val localReady = io.local.map(_.request.ready).getOrElse(false.B) - val leaseReady = io.lease.map(_.request.ready).getOrElse(false.B) - io.lease.foreach { port => - port.request.valid := state === idle && io.request.valid && lease && !reset.asBool - port.request.bits.endpoint := request.context - port.request.bits.vbank := request.field(31, 16) - port.request.bits.group := request.field(15, 0) - port.request.bits.release := request.operation === 19.U - port.response.ready := state === leasing && !reset.asBool - when(port.response.fire) { - answer := port.response.bits - state := responding - } - } - io.local.foreach { port => - val address = Cat(request.context, request.field) - port.request.valid := state === idle && io.request.valid && access && !reset.asBool - port.request.bits.address := address & ~15.U(64.W) - port.request.bits.write := request.operation === 17.U - port.request.bits.data := Cat(request.data, request.data) - port.request.bits.mask := Mux(address(3), "hff00".U, "h00ff".U) - port.response.ready := state === reading && !reset.asBool - when(port.response.fire) { - assert(!port.response.bits.error, "Main shared-storage access failed") - answer := Mux(localWrite, 0.U, Mux(upperHalf, port.response.bits.data(127, 64), port.response.bits.data(63, 0))) - state := responding - } - } - io.command.valid := state === idle && io.request.valid && start && !reset.asBool - io.command.bits.sourceByteAddress := staged(0) - io.command.bits.targetTile := staged(1) - io.command.bits.targetByteAddress := staged(2) - io.command.bits.bytes := staged(3) - io.request.ready := state === idle && !reset.asBool && - Mux(lease, leaseReady, Mux(access, localReady, !start || io.command.ready)) - io.reply.valid := state === responding && !reset.asBool - io.reply.bits := answer - io.completion.ready := state === running && !reset.asBool - io.idle := state === idle - when(io.request.valid && state === idle && !reset.asBool) { - assert( - (request.operation >= 13.U && request.operation <= 15.U) || - (localAccess.B && access) || (leaseAccess.B && lease), - "Invalid TLink management operation" - ) - } - when(io.request.fire) { - when(!access && !lease)(assert(request.context === 0.U, "TLink management has one tile-owned transfer context")) - answer := 0.U - state := responding - when(lease) { - assert(request.data === 0.U, "TLink shared lease reserves rs2") - state := leasing - } - when(access) { - val address = Cat(request.context, request.field) - assert(address(2, 0) === 0.U && (address +& 8.U) <= p.sharedBytes.U, "Invalid main shared-storage access") - upperHalf := address(3) - localWrite := request.operation === 17.U - state := reading - } - switch(request.operation) { - is(13.U) { - assert(request.field < 4.U, "Invalid TLink transfer descriptor field") - staged(request.field(1, 0)) := request.data - fields := fields | UIntToOH(request.field, 4) - } - is(14.U) { - assert(request.field === 0.U, "TLink start reserves the field selector") - assert(fields.andR, "Incomplete TLink transfer descriptor") - assert( - staged(1) < p.tiles.U && staged(1) =/= io.tileId, - "TLink transfer must target another tile" - ) - assert(staged(3) > 0.U && (staged(3) >> 32) === 0.U, "Invalid TLink transfer size") - fields := 0.U - state := running - } - is(15.U) { - assert(request.field < 4.U, "Invalid TLink geometry query") - answer := MuxLookup(request.field, 0.U)(Seq( - 0.U -> p.sharedBytes.U(64.W), - 1.U -> p.bankBytes.U(64.W), - 2.U -> io.tileId.pad(64), - 3.U -> p.tiles.U(64.W) - )) - } - } - } - when(io.completion.fire) { - assert(!io.completion.bits, "TLink transfer failed") - state := responding - } - when(io.reply.fire)(state := idle) -} diff --git a/arch/src/main/scala/framework/system/tile/tlink/Fabric.scala b/arch/src/main/scala/framework/system/tile/tlink/Fabric.scala deleted file mode 100644 index 5ad7d0088..000000000 --- a/arch/src/main/scala/framework/system/tile/tlink/Fabric.scala +++ /dev/null @@ -1,80 +0,0 @@ -package framework.system.tile.tlink - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import memcore.bus.axi4 - -/** Tile-addressed shared-storage fabric, separate from the DDR and CPU coherence paths. */ -@instantiable -class Fabric(p: axi4.Params, tileIds: Seq[Int], sharedBytes: BigInt) extends Module { - require(tileIds.nonEmpty && tileIds.distinct.size == tileIds.size && tileIds.forall(_ >= 0)) - require(p.dataBits == 128 && sharedBytes >= p.bytes) - val localBits = log2Ceil(sharedBytes) - val indexBits = math.max(1, log2Ceil(tileIds.size)) - require(p.addressBits >= localBits + log2Ceil(tileIds.max + 1)) - - @public val io = IO(new Bundle { - val source = Vec(tileIds.size, Flipped(new axi4.Port(p))) - val target = Vec(tileIds.size, new axi4.Port(p)) - }) - - val targets = tileIds.indices.map(_ => Instantiate(new axi4.Interconnect(p, tileIds.size))) - for (i <- tileIds.indices) { io.target(i) <> targets(i).io.out } - - for (source <- tileIds.indices) { - val port = io.source(source) - val reading = RegInit(false.B) - val writing = RegInit(false.B) - val readTarget = Reg(UInt(indexBits.W)) - val writeTarget = Reg(UInt(indexBits.W)) - val readMatches = VecInit(tileIds.map(id => (port.ar.bits.addr >> localBits) === id.U)) - val writeMatches = VecInit(tileIds.map(id => (port.aw.bits.addr >> localBits) === id.U)) - val readSelection = PriorityEncoder(readMatches) - val writeSelection = PriorityEncoder(writeMatches) - - def check(address: axi4.Address, matched: Bool): Unit = { - val offset = address.addr(localBits - 1, 0) - val bytes = (address.len +& 1.U) << log2Ceil(p.bytes) - assert(matched, "TLink address does not select a shared-storage tile") - assert( - address.size === log2Ceil(p.bytes).U && address.burst === 1.U && !address.lock, - "TLink requires full-width INCR bursts" - ) - assert( - offset(log2Ceil(p.bytes) - 1, 0) === 0.U && (offset +& bytes) <= sharedBytes.U, - "TLink burst exceeds shared storage or is unaligned" - ) - assert((offset.pad(12)(11, 0) +& bytes) <= 4096.U, "TLink burst crosses a 4KiB boundary") - } - when(port.ar.valid && !reading)(check(port.ar.bits, readMatches.asUInt.orR)) - when(port.aw.valid && !writing)(check(port.aw.bits, writeMatches.asUInt.orR)) - when(port.ar.fire) { reading := true.B; readTarget := readSelection } - when(port.r.fire && port.r.bits.last)(reading := false.B) - when(port.aw.fire) { writing := true.B; writeTarget := writeSelection } - when(port.b.fire)(writing := false.B) - - port.ar.ready := !reading && readMatches.asUInt.orR && VecInit(targets.map(_.io.in(source).ar.ready))(readSelection) - port.aw.ready := !writing && writeMatches.asUInt.orR && VecInit(targets.map(_.io.in(source).aw.ready))( - writeSelection - ) - port.w.ready := writing && VecInit(targets.map(_.io.in(source).w.ready))(writeTarget) - port.r.valid := reading && VecInit(targets.map(_.io.in(source).r.valid))(readTarget) - port.r.bits := VecInit(targets.map(_.io.in(source).r.bits))(readTarget) - port.b.valid := writing && VecInit(targets.map(_.io.in(source).b.valid))(writeTarget) - port.b.bits := VecInit(targets.map(_.io.in(source).b.bits))(writeTarget) - for (target <- tileIds.indices) { - val route = targets(target).io.in(source) - route.ar.valid := port.ar.valid && !reading && readMatches(target) - route.ar.bits := port.ar.bits - route.ar.bits.addr := port.ar.bits.addr(localBits - 1, 0) - route.aw.valid := port.aw.valid && !writing && writeMatches(target) - route.aw.bits := port.aw.bits - route.aw.bits.addr := port.aw.bits.addr(localBits - 1, 0) - route.w.valid := port.w.valid && writing && writeTarget === target.U - route.w.bits := port.w.bits - route.r.ready := port.r.ready && reading && readTarget === target.U - route.b.ready := port.b.ready && writing && writeTarget === target.U - } - } -} diff --git a/arch/src/main/scala/framework/system/tile/tlink/T2T.scala b/arch/src/main/scala/framework/system/tile/tlink/T2T.scala deleted file mode 100644 index 05b5b0177..000000000 --- a/arch/src/main/scala/framework/system/tile/tlink/T2T.scala +++ /dev/null @@ -1,127 +0,0 @@ -package framework.system.tile.tlink - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import framework.memdomain.backend.shared.SharedPhysicalPort -import memcore.bus.axi4 -import memcore.memory.queue.Queue - -case class T2TParams( - sharedBytes: BigInt, - bankBytes: Int, - tiles: Int, - axi: axi4.Params) { - require(sharedBytes > 0 && bankBytes >= 16 && isPow2(bankBytes) && sharedBytes % bankBytes == 0) - require(tiles > 0 && axi.dataBits == 128) - val sharedAddressBits: Int = math.max(1, log2Ceil(sharedBytes)) - val tileBits: Int = math.max(1, log2Ceil(tiles)) - require(sharedAddressBits + tileBits <= axi.addressBits && sharedAddressBits <= 32) -} - -class T2TCommand(p: T2TParams) extends Bundle { - val sourceByteAddress = UInt(64.W) - val targetTile = UInt(p.tileBits.W) - val targetByteAddress = UInt(64.W) - val bytes = UInt(32.W) -} - -/** - * Push shared SRAM bytes to another Tile. Starts are 16-byte aligned; the final beat may be partial. - * One command is active. Bank mappings remain owned by the caller through completion. - * Reset is coordinated with the fabric and SRAM ports, not an independent transaction cancellation. - */ -@instantiable -class T2TTransfer(p: T2TParams) extends Module { - - @public val io = IO(new Bundle { - val command = Flipped(Decoupled(new T2TCommand(p))) - val completion = Decoupled(Bool()) // false = no error, after the final destination write response - val source = new SharedPhysicalPort(128) - val mem = new axi4.Port(p.axi) - }) - - val idle :: plan :: stream :: reply :: complete :: Nil = Enum(5) - val state = RegInit(idle) - val source = Reg(UInt(p.sharedAddressBits.W)) - val target = Reg(UInt(p.sharedAddressBits.W)) - val tile = Reg(UInt(p.tileBits.W)) - val remaining = Reg(UInt((p.sharedAddressBits + 1).W)) - val burstBytes = Reg(UInt(13.W)) - val beats = Reg(UInt(9.W)) - val issued = RegInit(0.U(9.W)) - val sent = RegInit(0.U(9.W)) - val pending = RegInit(false.B) - val awSent = RegInit(false.B) - val data = Instantiate(new Queue(UInt(128.W), 2)) - val live = !reset.asBool - io.command.ready := state === idle && live - io.completion.valid := state === complete && live - io.completion.bits := false.B - when(io.completion.fire)(state := idle) - when(io.command.fire) { - val cmd = io.command.bits - assert(cmd.bytes =/= 0.U && cmd.sourceByteAddress(3, 0) === 0.U && cmd.targetByteAddress(3, 0) === 0.U) - assert(cmd.targetTile < p.tiles.U, "T2T target must be a Tile system ID") - assert( - cmd.sourceByteAddress +& cmd.bytes <= p.sharedBytes.U && cmd.targetByteAddress +& cmd.bytes <= p.sharedBytes.U, - "T2T command exceeds Tile shared SRAM" - ) - source := cmd.sourceByteAddress; target := cmd.targetByteAddress - tile := cmd.targetTile; remaining := cmd.bytes; state := plan - } - def minimum(a: UInt, b: UInt): UInt = Mux(a < b, a, b) - val sourcePage = 4096.U(13.W) - source.pad(12)(11, 0) - val targetPage = 4096.U(13.W) - target.pad(12)(11, 0) - val sourceBank = p.bankBytes.U - (source & (p.bankBytes - 1).U) - val targetBank = p.bankBytes.U - (target & (p.bankBytes - 1).U) - val nextBytes = Seq(remaining, sourcePage, targetPage, sourceBank, targetBank, 4096.U).reduce(minimum) - when(state === plan) { - burstBytes := nextBytes; beats := (nextBytes +& 15.U) >> 4 - issued := 0.U; sent := 0.U; pending := false.B; awSent := false.B; state := stream - assert(data.io.count === 0.U && nextBytes =/= 0.U) - } - io.mem.aw.valid := state === stream && !awSent && live - io.mem.aw.bits := 0.U.asTypeOf(new axi4.Address(p.axi)) - io.mem.aw.bits.addr := Cat(tile, target(p.sharedAddressBits - 1, 0)) - io.mem.aw.bits.len := beats - 1.U - io.mem.aw.bits.size := 4.U - io.mem.aw.bits.burst := 1.U - when(io.mem.aw.fire)(awSent := true.B) - io.source.request.valid := state === stream && issued < beats && !pending && data.io.count < 2.U && live - io.source.request.bits.address := source + (issued << 4) - io.source.request.bits.write := false.B - io.source.request.bits.data := 0.U - io.source.request.bits.mask := 0.U - when(io.source.request.fire) { pending := true.B; issued := issued + 1.U } - data.io.enq.valid := state === stream && pending && io.source.response.valid && !io.source.response.bits.error && live - data.io.enq.bits := io.source.response.bits.data - io.source.response.ready := state === stream && pending && data.io.enq.ready && live - when(io.source.response.valid && live) { - assert(pending && !io.source.response.bits.error, "T2T shared source read failed") - } - when(io.source.response.fire)(pending := false.B) - val last = sent +& 1.U === beats - val tail = burstBytes(3, 0) - io.mem.w.valid := state === stream && awSent && data.io.deq.valid && live - io.mem.w.bits.data := data.io.deq.bits - io.mem.w.bits.strb := Mux(last && tail =/= 0.U, (1.U(17.W) << tail) - 1.U, "hffff".U) - io.mem.w.bits.last := last - data.io.deq.ready := state === stream && awSent && io.mem.w.ready && live - when(io.mem.w.fire) { sent := sent + 1.U; when(last)(state := reply) } - io.mem.b.ready := state === reply && live - when(io.mem.b.fire) { - assert(io.mem.b.bits.id === 0.U && io.mem.b.bits.resp === 0.U, "T2T destination write failed") - assert(!pending && data.io.count === 0.U && issued === beats && sent === beats) - when(io.mem.b.bits.id === 0.U && io.mem.b.bits.resp === 0.U) { - when(remaining === burstBytes)(state := complete).otherwise { - source := source + burstBytes; target := target + burstBytes - remaining := remaining - burstBytes; state := plan - } - } - } - io.mem.ar.valid := false.B - io.mem.ar.bits := 0.U.asTypeOf(new axi4.Address(p.axi)) - io.mem.r.ready := false.B - when(live)(assert(!io.mem.r.valid, "T2T push engine received an unsolicited AXI read response")) -} diff --git a/arch/src/main/scala/framework/system/tile/tlink/tlink.scala b/arch/src/main/scala/framework/system/tile/tlink/tlink.scala deleted file mode 100644 index 203f52eae..000000000 --- a/arch/src/main/scala/framework/system/tile/tlink/tlink.scala +++ /dev/null @@ -1,54 +0,0 @@ -package framework.system.tile.tlink - -import chisel3._ -import chisel3.util._ -import hier.tile.memory.CoreInterrupts -import framework.memdomain.frontend.mem.dma.DmaStatus -import memcore.bus.{axi4, chi} -import memcore.bus.chi.RequesterPort -import memcore.bus.chi.snf.{LineRequest, LineResponse} -import memcore.memory.cpu.{CpuMemParams, UncachedRequest, UncachedResponse} - -class SharedPort(p: axi4.Params) extends Bundle { - val tx = new axi4.Port(p) - val rx = Flipped(new axi4.Port(p)) -} - -/** Tile boundary: coherent CPU control traffic and one independent DDR data port. */ -class TLinkIO( - main: Boolean, - cpus: Int, - executions: Int, - controls: Int, - bus: chi.Params, - axi: axi4.Params, - sharedStorage: Boolean) - extends Bundle { - val cp = CpuMemParams(bus, tagBits = 6) - val tileId = Input(UInt(32.W)) - val t2t = Option.when(sharedStorage)(new SharedPort(axi)) - val hartIds = Input(Vec(cpus, UInt(64.W))) - val executionIds = Input(Vec(executions, UInt(64.W))) - val resetVector = Input(Vec(cpus, UInt(64.W))) - val time = Input(UInt(64.W)) - val interrupts = Input(Vec(cpus, new CoreInterrupts)) - val uncachedRequest = Vec(cpus, Decoupled(new UncachedRequest(cp))) - val uncachedResponse = Vec(cpus, Flipped(Decoupled(new UncachedResponse(cp)))) - val control = Vec(if (main) 0 else 2 * cpus, new RequesterPort(bus)) - val remoteControl = Vec(if (main) 2 * (controls - cpus) else 0, Flipped(new RequesterPort(bus))) - val backingRequest = Vec(if (main) 1 else 0, Decoupled(new LineRequest(bus))) - val backingResponse = Vec(if (main) 1 else 0, Flipped(Decoupled(new LineResponse(bus)))) - val mem = new axi4.Port(axi) - val failure = Output(Vec(cpus, Valid(new DmaStatus))) - val workDrained = Output(Vec(cpus, Bool())) - val retired = Output(Vec(cpus, Bool())) - val retiredPc = Output(Vec(cpus, UInt(64.W))) - val trapped = Output(Vec(cpus, Bool())) - val trapCause = Output(Vec(cpus, UInt(64.W))) - val trapValue = Output(Vec(cpus, UInt(64.W))) - val trapPc = Output(Vec(cpus, UInt(64.W))) -} - -trait HasTLink { - def tlink: TLinkIO -} diff --git a/arch/src/main/scala/framework/thread/ant/build.sc b/arch/src/main/scala/framework/thread/ant/build.sc new file mode 100644 index 000000000..d06fa8611 --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/build.sc @@ -0,0 +1,25 @@ +import mill._ +import $file.^.^.^.^.^.^.settings + + +val ipRoot: os.Path = millSourcePath +trait Emitter extends settings.IpModule { + override def moduleRoot = ipRoot + override def mainClass = Some("framework.thread.ant.Emit") +} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("configs/AntParam.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Ant.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Decode.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/Executor.scala")), + PathRef(millSourcePath / os.RelPath("src/main/scala/LocalStore.scala")) + ) +) + +def emitSources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("src/main/scala/Emit.scala")) + ) +) diff --git a/arch/src/main/scala/framework/thread/ant/configs/AntParam.scala b/arch/src/main/scala/framework/thread/ant/configs/AntParam.scala new file mode 100644 index 000000000..8b8f53043 --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/configs/AntParam.scala @@ -0,0 +1,35 @@ +package framework.thread.ant.configs + +import buckyball.config.AntConfig +import chisel3.util.isPow2 +import framework.ip.memory.spm +import upickle.default.{macroRW, ReadWriter} + +case class AntParam( + codeBytes: Int, + data: spm.Params, + shared: spm.Params, + taskBits: Int) { + require(codeBytes >= 32 && isPow2(codeBytes)) + require(data.base >= codeBytes && shared.base >= codeBytes) + require(data.base + data.bytes <= shared.base || shared.base + shared.bytes <= data.base) + require(data.dataBits == shared.dataBits) + require(taskBits > 0) +} + +object AntParam { + + def fromProto(p: AntConfig, shared: spm.Params): AntParam = { + require(p.hasTls, "Ant TLS configuration is required") + val tls = p.getTls + AntParam( + p.getCodeBytes, + spm.Params(BigInt(java.lang.Long.toUnsignedString(tls.getBase)), tls.getBytes, tls.getDataBits), + shared, + p.getTaskBits + ) + } + + implicit val spmRW: ReadWriter[spm.Params] = macroRW + implicit val rw: ReadWriter[AntParam] = macroRW +} diff --git a/arch/src/main/scala/framework/thread/ant/configs/default.toml b/arch/src/main/scala/framework/thread/ant/configs/default.toml new file mode 100644 index 000000000..dd065ba9d --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/configs/default.toml @@ -0,0 +1,7 @@ +codeBytes = 0x4000 +taskBits = 32 + +[tls] +base = 0x10000 +bytes = 0x4000 +dataBits = 128 diff --git a/arch/src/main/scala/framework/system/core/ant/src/csrc/Cargo.lock b/arch/src/main/scala/framework/thread/ant/src/csrc/Cargo.lock similarity index 100% rename from arch/src/main/scala/framework/system/core/ant/src/csrc/Cargo.lock rename to arch/src/main/scala/framework/thread/ant/src/csrc/Cargo.lock diff --git a/arch/src/main/scala/framework/system/core/ant/src/csrc/Cargo.toml b/arch/src/main/scala/framework/thread/ant/src/csrc/Cargo.toml similarity index 100% rename from arch/src/main/scala/framework/system/core/ant/src/csrc/Cargo.toml rename to arch/src/main/scala/framework/thread/ant/src/csrc/Cargo.toml diff --git a/arch/src/main/scala/framework/thread/ant/src/csrc/src/lib.rs b/arch/src/main/scala/framework/thread/ant/src/csrc/src/lib.rs new file mode 100644 index 000000000..e85ae86dc --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/csrc/src/lib.rs @@ -0,0 +1,20 @@ +#[path = "../../../../../ip/spm/src/csrc/src/lib.rs"] +mod storage; + +#[no_mangle] +pub extern "C" fn ant_arithmetic(a: u64, b: u64, operation: u32, word: bool) -> u64 { + let (a, b) = if word { (a as u32 as u64, b as u32 as u64) } else { (a, b) }; + let signed = |x: u64| if word { x as u32 as i32 as i64 } else { x as i64 }; + let value = match operation { + 0 => a.wrapping_mul(b), + 1 => (((a as i64 as i128) * (b as i64 as i128)) >> 64) as u64, + 2 => (((a as i64 as i128) * (b as i128)) >> 64) as u64, + 3 => (((a as u128) * (b as u128)) >> 64) as u64, + 4 => if b == 0 { u64::MAX } else { signed(a).wrapping_div(signed(b)) as u64 }, + 5 => if b == 0 { u64::MAX } else { a / b }, + 6 => if b == 0 { a } else { signed(a).wrapping_rem(signed(b)) as u64 }, + 7 => if b == 0 { a } else { a % b }, + _ => unreachable!(), + }; + if word { value as u32 as i32 as i64 as u64 } else { value } +} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/ant.cm_hier b/arch/src/main/scala/framework/thread/ant/src/main/resources/ant.cm_hier similarity index 100% rename from arch/src/main/scala/framework/system/core/ant/src/main/resources/ant.cm_hier rename to arch/src/main/scala/framework/thread/ant/src/main/resources/ant.cm_hier diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/ant.f b/arch/src/main/scala/framework/thread/ant/src/main/resources/ant.f new file mode 100644 index 000000000..b27fde31e --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/ant.f @@ -0,0 +1,10 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/scenario_if.sv +@VERIFY@/uvm/src/core/scenario_package.sv +@RESOURCES@/../../../../../ip/spm/src/main/resources/package.sv +@RESOURCES@/../../../../../ip/spm/src/main/resources/interface.sv ++incdir+@RTL@ +-F @RTL@/Ant/filelist.f +@RESOURCES@/ant_tb.sv diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/ant_tb.sv b/arch/src/main/scala/framework/thread/ant/src/main/resources/ant_tb.sv new file mode 100644 index 000000000..e62ab19ae --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/ant_tb.sv @@ -0,0 +1,315 @@ +`include "config.svh" +module ant_tb; + import uvm_pkg::*; + import verification_pkg::*; + import spm_pkg::*; + import scenario_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function longint unsigned ant_arithmetic( + longint unsigned a, + b, + int unsigned operation, + bit word_mode + ); + logic clock = 0, reset = 1, in_use = 0; + always #5 clock = ~clock; + scenario_if control (clock); + spm_if host ( + clock, + reset + ); + logic [1:0] start_valid = 0, start_ready, result_valid, result_ready = 0; + logic [1:0] cancel = 0, drained = '1, was_cancelled, running; + logic [31:0] task_id[2] = '{1, 2}, result_task[2]; + logic [63:0] value[2]; + logic [63:0] entry[2] = '{0, 0}, code_end[2] = '{16, 16}; + logic [1:0] command_valid, command_ready = '1, retired_valid; + logic [31:0] command_instruction[2]; + logic [63:0] command_a[2], command_b[2], retired_pc[2]; + logic [1:0] transfer_valid = 0, transfer_ready, transferred_valid, transferred_ready = 0; + logic [1:0] transfer_code = 0, transfer_to_local = 0, transferred_result; + logic [31:0] transfer_local[2], transfer_shared[2], transfer_bytes[2]; + int checks = 0; + AntVerification dut ( + .clock(clock), + .reset(reset), + .io_inUse(in_use), + .io_rocc_0_cmd_valid(command_valid[0]), + .io_rocc_0_cmd_ready(command_ready[0]), + .io_rocc_0_cmd_bits_instruction_raw_inst(command_instruction[0]), + .io_rocc_0_cmd_bits_instruction_rs1Data(command_a[0]), + .io_rocc_0_cmd_bits_instruction_rs2Data(command_b[0]), + .io_rocc_0_busy(!drained[0]), + .io_rocc_0_interrupt(1'b0), + .io_rocc_0_resp_valid(1'b0), + .io_rocc_0_resp_bits_rd(5'd0), + .io_rocc_0_resp_bits_data(64'd0), + .io_retired_0_valid(retired_valid[0]), + .io_retired_0_bits_pc(retired_pc[0]), + .io_rocc_1_cmd_valid(command_valid[1]), + .io_rocc_1_cmd_ready(command_ready[1]), + .io_rocc_1_cmd_bits_instruction_raw_inst(command_instruction[1]), + .io_rocc_1_cmd_bits_instruction_rs1Data(command_a[1]), + .io_rocc_1_cmd_bits_instruction_rs2Data(command_b[1]), + .io_rocc_1_busy(!drained[1]), + .io_rocc_1_interrupt(1'b0), + .io_rocc_1_resp_valid(1'b0), + .io_rocc_1_resp_bits_rd(5'd0), + .io_rocc_1_resp_bits_data(64'd0), + .io_retired_1_valid(retired_valid[1]), + .io_retired_1_bits_pc(retired_pc[1]), + .io_start_0_valid(start_valid[0]), + .io_start_0_ready(start_ready[0]), + .io_start_0_bits_task(task_id[0]), + .io_start_0_bits_entry(entry[0]), + .io_start_0_bits_codeEnd(code_end[0]), + .io_start_0_bits_argument(`TLS_BASE), + .io_start_0_bits_stack(`TLS_END), + .io_result_0_valid(result_valid[0]), + .io_result_0_ready(result_ready[0]), + .io_result_0_bits_task(result_task[0]), + .io_result_0_bits_cancelled(was_cancelled[0]), + .io_result_0_bits_value(value[0]), + .io_cancel_0(cancel[0]), + .io_running_0(running[0]), + .io_transfer_0_valid(transfer_valid[0]), + .io_transfer_0_ready(transfer_ready[0]), + .io_transfer_0_bits_code(transfer_code[0]), + .io_transfer_0_bits_toLocal(transfer_to_local[0]), + .io_transfer_0_bits_localOffset(transfer_local[0]), + .io_transfer_0_bits_sharedOffset(transfer_shared[0]), + .io_transfer_0_bits_bytes(transfer_bytes[0]), + .io_transferred_0_valid(transferred_valid[0]), + .io_transferred_0_ready(transferred_ready[0]), + .io_transferred_0_bits(transferred_result[0]), + .io_start_1_valid(start_valid[1]), + .io_start_1_ready(start_ready[1]), + .io_start_1_bits_task(task_id[1]), + .io_start_1_bits_entry(entry[1]), + .io_start_1_bits_codeEnd(code_end[1]), + .io_start_1_bits_argument(`TLS_BASE), + .io_start_1_bits_stack(`TLS_END), + .io_result_1_valid(result_valid[1]), + .io_result_1_ready(result_ready[1]), + .io_result_1_bits_task(result_task[1]), + .io_result_1_bits_cancelled(was_cancelled[1]), + .io_result_1_bits_value(value[1]), + .io_cancel_1(cancel[1]), + .io_running_1(running[1]), + .io_transfer_1_valid(transfer_valid[1]), + .io_transfer_1_ready(transfer_ready[1]), + .io_transfer_1_bits_code(transfer_code[1]), + .io_transfer_1_bits_toLocal(transfer_to_local[1]), + .io_transfer_1_bits_localOffset(transfer_local[1]), + .io_transfer_1_bits_sharedOffset(transfer_shared[1]), + .io_transfer_1_bits_bytes(transfer_bytes[1]), + .io_transferred_1_valid(transferred_valid[1]), + .io_transferred_1_ready(transferred_ready[1]), + .io_transferred_1_bits(transferred_result[1]), + .io_host_request_valid(host.valid), + .io_host_request_ready(host.ready), + .io_host_request_bits_address(host.address), + .io_host_request_bits_size(host.size), + .io_host_request_bits_write(host.write), + .io_host_request_bits_data(host.data), + .io_host_request_bits_mask(host.mask), + .io_host_response_valid(host.response_valid), + .io_host_response_ready(host.response_ready), + .io_host_response_bits_data(host.result), + .io_host_response_bits_error(host.error) + ); + initial begin + #0; + uvm_config_db#(virtual scenario_if)::set(null, "uvm_test_top", "vif", control); + uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 10ms); + run_test("protocol_test"); + end + task automatic copy_local(int ctx, bit code, bit to_local, int offset, int bytes); + @(negedge clock); + transfer_code[ctx] = code; + transfer_to_local[ctx] = to_local; + transfer_local[ctx] = offset; + transfer_shared[ctx] = `TSS_SCRATCH - `TSS_BASE; + transfer_bytes[ctx] = bytes; + transfer_valid[ctx] = 1; + do @(posedge clock); while (!transfer_ready[ctx]); + @(negedge clock); + transfer_valid[ctx] = 0; + wait (transferred_valid[ctx]); + repeat (3) begin + @(negedge clock); + if (!transferred_valid[ctx] || !transferred_result[ctx]) + `uvm_fatal("TRANSFER", "transfer completion changed under backpressure") + end + transferred_ready[ctx] = 1; + @(negedge clock); + transferred_ready[ctx] = 0; + endtask + task automatic put(int port, longint unsigned address, logic [127:0] data, bit wide = 0); + logic [127:0] word; + int ctx, offset, bytes; + if (port == 4) begin + host.access(address, wide ? 4 : 2, 1, data, wide ? 16'hffff : 16'h000f); + end else begin + ctx = port / 2; + offset = address - (port % 2 ? `TLS_BASE : 0); + bytes = wide ? 16 : 8; + word = data; + if (!wide) begin + copy_local(ctx, !(port % 2), 0, offset & ~7, 8); + host.send(`TSS_SCRATCH, 3, 0); + host.take(word); + word[(offset%8)*8+:32] = data[31:0]; + end + host.access(`TSS_SCRATCH, wide ? 4 : 3, 1, word, wide ? 16'hffff : 16'h00ff); + copy_local(ctx, !(port % 2), 1, offset & ~7, bytes); + end + if (host.error) `uvm_fatal("LOAD", "TSS access failed") + endtask + task automatic launch(int id); + @(negedge clock); + start_valid[id] = 1; + do @(posedge clock); while (!start_ready[id]); + @(negedge clock); + start_valid[id] = 0; + endtask + task automatic finish(int id, longint unsigned expected, bit cancelled = 0); + wait (result_valid[id]); + if (was_cancelled[id] !== cancelled || value[id] !== expected || result_task[id] !== task_id[id]) + `uvm_fatal("RESULT", $sformatf( + "context %0d task %0d: cancelled %b value %h expected %h", + id, + result_task[id], + was_cancelled[id], + value[id], + expected + )) + repeat (3) begin + @(negedge clock); + if (!result_valid[id] || value[id] !== expected) `uvm_fatal("HOLD", "completion changed") + end + result_ready[id] = 1; + @(negedge clock); + result_ready[id] = 0; + checks++; + endtask + initial begin : exercise + longint unsigned operands[8] = '{ + 0, + 1, + 64'hffffffffffffffff, + 64'h8000000000000000, + 64'h7fffffffffffffff, + 64'h80000000, + 64'hffffffff, + 37 + }; + logic [31:0] operation; + wait (control.start); + repeat (3) @(negedge clock); + reset = 0; + // Real RV64IM programs: LD operands, arithmetic, ECALL returning a0. + // Golden arithmetic uses independent Rust i128/u128 and defined division corners. + for (int w = 0; w < 2; w++) + for (int op = 0; op < 8; op++) + if (!w || op == 0 || op >= 4) + for (int sample = 0; sample < 64; sample ++) begin + operation = 32'h02b50533 | (op << 12) | (w << 3); + put(0, 0, {32'h00000073, operation, 32'h00053503, 32'h00853583}, 1); + put(1, `TLS_BASE, {operands[sample%8], operands[sample/8]}, 1); + in_use = 1; + launch(0); + finish(0, ant_arithmetic(operands[sample/8], operands[sample%8], op, w)); + in_use = 0; + end + // One producer and one consumer share data then publish/observe a flag. + put(4, `TSS_BASE, 0, 1); + put(0, 0, {32'h0062b023, 32'h02a00313, 32'h000202b7, 32'h00000013}, 1); + put(0, 16, {32'h02a00513, 32'h0062b423, 32'h00100313, 32'h00000013}, 1); + put(0, 32, 32'h00000073); + // Consumer loops while flag is zero, then returns shared data. + put(2, 0, {32'h0002b503, 32'hfe030ee3, 32'h0082b303, 32'h000202b7}, 1); + put(2, 16, 32'h00000073); + code_end[0] = 36; + code_end[1] = 20; + in_use = 1; + fork + launch(0); + launch(1); + join + fork + finish(0, 42); + finish(1, 42); + join + in_use = 0; + // Identical TLS addresses hold different values in the two contexts. + put(0, 0, {96'b0, 32'h00053503}, 1); + put(0, 4, 32'h00000073); + put(2, 0, {96'b0, 32'h00053503}, 1); + put(2, 4, 32'h00000073); + put(1, `TLS_BASE, 128'd17, 1); + put(3, `TLS_BASE, 128'd29, 1); + code_end[0] = 8; + code_end[1] = 8; + in_use = 1; + fork + launch(0); + launch(1); + join + fork + finish(0, 17); + finish(1, 29); + join + in_use = 0; + code_end[0] = 16; + // Raw custom instruction and opaque DDR pointer survive submission backpressure. + put(0, 0, {32'h03700513, 32'h12b5307b, 32'h00053503, 32'h00853583}, 1); + put(0, 16, 32'h00000073); + code_end[0] = 20; + put(1, `TLS_BASE, {64'hfedcba9876543210, 64'h80001000}, 1); + command_ready[0] = 0; + in_use = 1; + launch(0); + wait (command_valid[0]); + // A busy Ant must not lock the other Ant's private loader. + put(3, `TLS_BASE, 128'd91, 1); + repeat (7) begin + @(negedge clock); + if (!command_valid[0] || command_instruction[0] !== 32'h12b5307b || + command_a[0] !== 64'h80001000 || + command_b[0] !== 64'hfedcba9876543210) + `uvm_fatal("COMMAND", "raw NPU command changed under backpressure") + end + drained[0] = 0; + command_ready[0] = 1; + @(negedge clock); + repeat (20) begin + @(negedge clock); + if (result_valid[0]) `uvm_fatal("DRAIN", "task completed before NPU drained") + end + drained[0] = 1; + finish(0, 55); + in_use = 0; + code_end[0] = 16; + // Cancel an actual in-flight DIV; reusing the context must not consume its old result. + put(0, 0, {32'h00000073, 32'h02b54533, 32'h00053503, 32'h00853583}, 1); + put(1, `TLS_BASE, {64'd37, 64'h7fffffffffffffff}, 1); + in_use = 1; + launch(0); + do @(posedge clock); while (!(retired_valid[0] && retired_pc[0] == 4)); + repeat (8) @(negedge clock); + cancel[0] = 1; + @(negedge clock); + cancel[0] = 0; + finish(0, 0, 1); + in_use = 0; + put(0, 0, {64'b0, 32'h00000073, 32'h00b00513}, 1); + code_end[0] = 8; + in_use = 1; + launch(0); + finish(0, 11); + in_use = 0; + `uvm_info("ANT", $sformatf("%0d task completions checked", checks), UVM_LOW) + control.done = 1; + end +endmodule diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/control.cm_hier b/arch/src/main/scala/framework/thread/ant/src/main/resources/control.cm_hier similarity index 100% rename from arch/src/main/scala/framework/system/core/ant/src/main/resources/control.cm_hier rename to arch/src/main/scala/framework/thread/ant/src/main/resources/control.cm_hier diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/control.f b/arch/src/main/scala/framework/thread/ant/src/main/resources/control.f new file mode 100644 index 000000000..8e955d5fa --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/control.f @@ -0,0 +1,9 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/scenario_if.sv +@VERIFY@/uvm/src/core/scenario_package.sv ++incdir+@RTL@ +-F @RTL@/Control/filelist.f +@RESOURCES@/scenario_if.sv +@RESOURCES@/control_tb.sv diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/control_if.sv b/arch/src/main/scala/framework/thread/ant/src/main/resources/control_if.sv similarity index 100% rename from arch/src/main/scala/framework/system/core/ant/src/main/resources/control_if.sv rename to arch/src/main/scala/framework/thread/ant/src/main/resources/control_if.sv diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/control_tb.sv b/arch/src/main/scala/framework/thread/ant/src/main/resources/control_tb.sv new file mode 100644 index 000000000..f2e395c99 --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/control_tb.sv @@ -0,0 +1,176 @@ +`include "config.svh" +module control_tb; + import uvm_pkg::*; + import scenario_pkg::*; + `include "uvm_macros.svh" + logic clock = 0, reset = 1; + always #5 clock = ~clock; + scenario_if control (clock); + ant_control_if management ( + clock, + reset + ); + logic [1:0] start_valid, start_ready = '1, result_valid = 0, result_ready, cancel; + logic storage_busy = 0, in_use; + logic [31:0] task_id[2], result_task[2] = '{7, 8}; + logic [63:0] entry[2], code_end[2], argument[2], stack[2]; + logic [63:0] signature[2]; + logic [63:0] request_satp = 64'h8000000000000bad, launch_satp[2]; + logic [1:0] request_privilege = 0, launch_privilege[2]; + logic request_sum = 0, request_mxr = 0; + logic [1:0] launch_sum, launch_mxr; + logic [63:0] expected_satp[2] = '{64'h8000000000000123, 64'h8000000000000456}; + logic [1:0] expected_privilege[2] = '{2'd1, 2'd0}; + logic [1:0] expected_sum = 2'b01, expected_mxr = 2'b10; + int checked = 0, launches = 0; + Control dut ( + .clock(clock), + .reset(reset), + .io_request_valid(management.valid), + .io_request_ready(management.ready), + .io_request_bits_control_operation(management.operation), + .io_request_bits_control_context(management.context_id), + .io_request_bits_control_field(management.field), + .io_request_bits_control_data(management.data), + .io_request_bits_context_satp(request_satp), + .io_request_bits_context_effectivePrivilege(request_privilege), + .io_request_bits_context_sum(request_sum), + .io_request_bits_context_mxr(request_mxr), + .io_response_valid(management.reply_valid), + .io_response_ready(management.reply_ready), + .io_response_bits(management.reply), + .io_storageBusy(storage_busy), + .io_inUse(in_use), + .io_cancel_0(cancel[0]), + .io_start_0_valid(start_valid[0]), + .io_start_0_ready(start_ready[0]), + .io_start_0_bits_descriptor_task(task_id[0]), + .io_start_0_bits_descriptor_entry(entry[0]), + .io_start_0_bits_descriptor_codeEnd(code_end[0]), + .io_start_0_bits_descriptor_argument(argument[0]), + .io_start_0_bits_descriptor_stack(stack[0]), + .io_start_0_bits_context_satp(launch_satp[0]), + .io_start_0_bits_context_effectivePrivilege(launch_privilege[0]), + .io_start_0_bits_context_sum(launch_sum[0]), + .io_start_0_bits_context_mxr(launch_mxr[0]), + .io_result_0_valid(result_valid[0]), + .io_result_0_ready(result_ready[0]), + .io_result_0_bits_task(result_task[0]), + .io_result_0_bits_cancelled(1'b0), + .io_result_0_bits_value(64'h42), + .io_cancel_1(cancel[1]), + .io_start_1_valid(start_valid[1]), + .io_start_1_ready(start_ready[1]), + .io_start_1_bits_descriptor_task(task_id[1]), + .io_start_1_bits_descriptor_entry(entry[1]), + .io_start_1_bits_descriptor_codeEnd(code_end[1]), + .io_start_1_bits_descriptor_argument(argument[1]), + .io_start_1_bits_descriptor_stack(stack[1]), + .io_start_1_bits_context_satp(launch_satp[1]), + .io_start_1_bits_context_effectivePrivilege(launch_privilege[1]), + .io_start_1_bits_context_sum(launch_sum[1]), + .io_start_1_bits_context_mxr(launch_mxr[1]), + .io_result_1_valid(result_valid[1]), + .io_result_1_ready(result_ready[1]), + .io_result_1_bits_task(result_task[1]), + .io_result_1_bits_cancelled(1'b0), + .io_result_1_bits_value(64'h42) + `include "control_pmp.svh" + ); + always @(posedge clock) + if (!reset) begin + for (int i = 0; i < 2; i++) + if (start_valid[i] && start_ready[i]) begin + if(task_id[i] !== 7+i || entry[i] !== 0 || code_end[i] !== 16 || + argument[i] !== `TLS_BASE || stack[i] !== `TLS_END) + `uvm_fatal("START", "local descriptor changed") + if (launch_satp[i] !== expected_satp[i] || launch_privilege[i] !== expected_privilege[i] || + launch_sum[i] !== expected_sum[i] || launch_mxr[i] !== expected_mxr[i]) + `uvm_fatal("CONTEXT", "launch context does not belong to the accepted start command") + launches++; + end + end + initial begin + uvm_config_db#(virtual scenario_if)::set(null, "uvm_test_top", "vif", control); + uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); + run_test("protocol_test"); + end + task automatic transact(int op, int ctx, int f, longint unsigned value, + longint unsigned expected = 0); + longint unsigned actual; + management.transfer(op, ctx, f, value, actual); + if (actual !== expected) + `uvm_fatal("CONTROL", $sformatf("data %h expected %h", actual, expected)) + checked++; + endtask + task automatic descriptor(int ctx); + transact(1, ctx, 0, 7 + ctx); + transact(1, ctx, 1, 0); + transact(1, ctx, 2, 16); + transact(1, ctx, 3, `TLS_BASE); + transact(1, ctx, 4, `TLS_END); + transact(1, ctx, 5, signature[ctx]); + endtask + initial begin + wait (control.start); + repeat (3) @(negedge clock); + reset = 0; + transact(0, 0, 7, 0, 2); + management.transfer(0, 0, 0, 0, signature[0]); + management.transfer(0, 1, 0, 0, signature[1]); + transact(0, 0, 1, 0, 1); + transact(0, 0, 2, 0, `CODE_BYTES); + transact(0, 0, 3, 0, `TLS_BASE); + transact(0, 0, 4, 0, `TLS_BYTES); + transact(0, 0, 5, 0, `TSS_BASE); + transact(0, 0, 6, 0, `TSS_BYTES); + transact(5, 0, 0, 0); + descriptor(0); + transact(1, 0, 5, signature[0]); + // Staging was performed with another address context. Only Start selects the task's context. + request_satp = expected_satp[0]; + request_privilege = expected_privilege[0]; + request_sum = expected_sum[0]; + request_mxr = expected_mxr[0]; + start_ready[0] = 0; + fork + transact(2, 0, 0, 0); + begin + wait (start_valid[0]); + repeat (5) begin + @(negedge clock); + if (!start_valid[0] || management.ready || launches != 0) + `uvm_fatal("STALL", "launch was accepted before the thread became ready") + if (task_id[0] !== 7 || launch_satp[0] !== expected_satp[0] || + launch_privilege[0] !== expected_privilege[0] || + launch_sum[0] !== expected_sum[0] || launch_mxr[0] !== expected_mxr[0]) + `uvm_fatal("STALL", "launch descriptor and address context changed under backpressure") + end + start_ready[0] = 1; + end + join + transact(0, 0, 1, 0, 3); + descriptor(1); + request_satp = expected_satp[1]; + request_privilege = expected_privilege[1]; + request_sum = expected_sum[1]; + request_mxr = expected_mxr[1]; + transact(2, 1, 0, 0); + @(negedge clock); + result_valid = 3; + do @(posedge clock); while (result_ready != 3); + @(negedge clock); + result_valid = 0; + transact(0, 0, 1, 0, 5); + transact(0, 0, 8, 0, 7); + transact(0, 0, 9, 0, 64'h42); + transact(0, 1, 8, 0, 8); + transact(3, 0, 0, 0); + transact(3, 1, 0, 0); + transact(6, 0, 0, 0); + if (in_use || launches != 2) `uvm_fatal("GROUP", "group lifetime or launch count wrong") + `uvm_info("CONTROL", $sformatf("%0d transactions checked; %0d launches", checked, launches), + UVM_LOW) + control.done = 1; + end +endmodule diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/resources/service.cm_hier b/arch/src/main/scala/framework/thread/ant/src/main/resources/service.cm_hier similarity index 100% rename from arch/src/main/scala/framework/system/core/ant/src/main/resources/service.cm_hier rename to arch/src/main/scala/framework/thread/ant/src/main/resources/service.cm_hier diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/service.f b/arch/src/main/scala/framework/thread/ant/src/main/resources/service.f new file mode 100644 index 000000000..4a391fa4e --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/service.f @@ -0,0 +1,9 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/scenario_if.sv +@VERIFY@/uvm/src/core/scenario_package.sv ++incdir+@RTL@ +-F @RTL@/Service/filelist.f +@RESOURCES@/scenario_if.sv +@RESOURCES@/service_tb.sv diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/service_tb.sv b/arch/src/main/scala/framework/thread/ant/src/main/resources/service_tb.sv new file mode 100644 index 000000000..286e24c1d --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/service_tb.sv @@ -0,0 +1,153 @@ +`include "config.svh" +module service_tb; + import uvm_pkg::*; + import verification_pkg::*; + import scenario_pkg::*; + `include "uvm_macros.svh" + logic clock = 0, reset = 1; + always #5 clock = ~clock; + scenario_if control (clock); + ant_control_if management ( + clock, + reset + ); + logic [1:0] drained = '1, command_valid, command_ready = '1, retired_valid; + logic [31:0] command_instruction[2]; + logic [63:0] command_a[2], command_b[2], retired_pc[2]; + int completed = 0; + ServiceVerification dut ( + .clock(clock), + .reset(reset), + .io_rocc_0_cmd_valid(command_valid[0]), + .io_rocc_0_cmd_ready(command_ready[0]), + .io_rocc_0_cmd_bits_instruction_raw_inst(command_instruction[0]), + .io_rocc_0_cmd_bits_instruction_rs1Data(command_a[0]), + .io_rocc_0_cmd_bits_instruction_rs2Data(command_b[0]), + .io_rocc_0_busy(!drained[0]), + .io_rocc_0_interrupt(1'b0), + .io_rocc_0_resp_valid(1'b0), + .io_rocc_0_resp_bits_rd(5'd0), + .io_rocc_0_resp_bits_data(64'd0), + .io_retired_0_valid(retired_valid[0]), + .io_retired_0_bits_pc(retired_pc[0]), + .io_rocc_1_cmd_valid(command_valid[1]), + .io_rocc_1_cmd_ready(command_ready[1]), + .io_rocc_1_cmd_bits_instruction_raw_inst(command_instruction[1]), + .io_rocc_1_cmd_bits_instruction_rs1Data(command_a[1]), + .io_rocc_1_cmd_bits_instruction_rs2Data(command_b[1]), + .io_rocc_1_busy(!drained[1]), + .io_rocc_1_interrupt(1'b0), + .io_rocc_1_resp_valid(1'b0), + .io_rocc_1_resp_bits_rd(5'd0), + .io_rocc_1_resp_bits_data(64'd0), + .io_retired_1_valid(retired_valid[1]), + .io_retired_1_bits_pc(retired_pc[1]), + .io_request_valid(management.valid), + .io_request_ready(management.ready), + .io_request_bits_operation(management.operation), + .io_request_bits_context(management.context_id), + .io_request_bits_field(management.field), + .io_request_bits_data(management.data), + .io_reply_valid(management.reply_valid), + .io_reply_ready(management.reply_ready), + .io_reply_bits(management.reply) + ); + initial begin + #0; + uvm_config_db#(virtual scenario_if)::set(null, "uvm_test_top", "vif", control); + uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); + run_test("protocol_test"); + end + task automatic command(int op, int ctx, int field, longint unsigned value = 0); + longint unsigned reply; + management.transfer(op, ctx, field, value, reply); + endtask + task automatic put(int port, longint unsigned address, logic [127:0] data); + longint unsigned result, offset, shared_offset, descriptor; + int space, ctx; + space = port == 4 ? 2 : port % 2; + ctx = port == 4 ? 0 : port / 2; + offset = address - (space == 0 ? 0 : space == 1 ? `TLS_BASE : `TSS_BASE); + shared_offset = space == 2 ? offset : `TSS_SCRATCH - `TSS_BASE; + for (int half = 0; half < 2; half++) + command(12, 0, shared_offset + 8 * half, data[64*half+:64]); + if (space != 2) begin + descriptor = (64'd16 << 32) | shared_offset; + command(8 + 2 * space, ctx, offset, descriptor); + command(7 + 2 * space, ctx, offset, descriptor); + end + for (int half = 0; half < 2; half++) begin + management.transfer(11, 0, shared_offset + 8 * half, 0, result); + if (result !== data[64*half+:64]) `uvm_fatal("LOAD", "TSS transfer readback mismatch") + end + endtask + task automatic launch(int ctx, int task_id); + command(1, ctx, 0, task_id); + command(1, ctx, 1, 0); + command(1, ctx, 2, 8); + command(1, ctx, 3, `TLS_BASE); + command(1, ctx, 4, `TLS_END); + command(1, ctx, 5, ctx ? `SIGNATURE_1 : `SIGNATURE_0); + command(2, ctx, 0); + endtask + task automatic finish(int ctx, int task_id, longint unsigned value, bit cancelled = 0); + longint unsigned reply, status; + do management.transfer(0, ctx, 1, 0, status); while (!(status & 4)); + if (status !== (cancelled ? 13 : 5)) `uvm_fatal("RESULT", "wrong terminal state") + management.transfer(0, ctx, 8, 0, reply); + if (reply !== task_id) `uvm_fatal("RESULT", "wrong task ID") + management.transfer(0, ctx, 9, 0, reply); + if (reply !== value) `uvm_fatal("RESULT", "wrong terminal value") + command(3, ctx, 0); + completed++; + endtask + initial begin + longint unsigned reply; + wait (control.start); + repeat (3) @(negedge clock); + reset = 0; + put(1, `TLS_BASE, 128'h1111); + put(3, `TLS_BASE, 128'h2222); + put(4, `TSS_BASE, 128'h3333); + put(0, 0, {64'b0, 32'h00000073, 32'h02500513}); + put(2, 0, {64'b0, 32'h00000073, 32'h01d00513}); + command(5, 0, 0); + launch(0, 1); + launch(1, 2); + finish(0, 1, 37); + finish(1, 2, 29); + command(6, 0, 0); + put(0, 0, {64'b0, 32'h00000073, 32'h0000307b}); + command_ready[0] = 0; + command(5, 0, 0); + launch(0, 3); + drained[0] = 0; + wait (command_valid[0]); + command(4, 0, 0); + management.transfer(0, 0, 1, 0, reply); + if (reply !== 3) `uvm_fatal("DRAIN", "task completed before external work drained") + drained[0] = 1; + finish(0, 3, 0, 1); + command(6, 0, 0); + // A new task must not inherit the old task's cancellation. + command_ready[0] = 1; + put(0, 0, {64'b0, 32'h00000073, 32'h00900513}); + command(5, 0, 0); + launch(0, 4); + finish(0, 4, 9); + command(6, 0, 0); + // Cancellation remains effective after ECALL while backend work is still pending. + command(5, 0, 0); + launch(0, 5); + drained[0] = 0; + repeat (30) @(negedge clock); + command(4, 0, 0); + drained[0] = 1; + finish(0, 5, 0, 1); + command(6, 0, 0); + `uvm_info("SERVICE", $sformatf( + "%0d real Ant tasks; %0d control transactions checked", completed, management.checked + ), UVM_LOW) + control.done = 1; + end +endmodule diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/suite.toml b/arch/src/main/scala/framework/thread/ant/src/main/resources/suite.toml new file mode 100644 index 000000000..e5b06b45d --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/suite.toml @@ -0,0 +1,71 @@ +[model] +manifest = "src/csrc/Cargo.toml" +library = "libant_ref" + +[[tests]] +functional_groups = [] +name = "ant" +top = "ant_tb" +test = "protocol_test" +filelist = "ant.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "control" +top = "control_tb" +test = "protocol_test" +filelist = "control.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "service" +top = "service_tb" +test = "protocol_test" +filelist = "service.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" + +[[tests]] +functional_groups = [] +name = "tls" +top = "tls_tb" +test = "protocol_test" +filelist = "tls.f" +timeout_ns = 100000 +wall_timeout_s = 120 +kind = "functional" +rtl_scopes = ["tiles[0].dut", "tiles[1].dut"] +metrics = ["line", "cond", "tgl"] +sink_ready = 1 +sink_stall = 3 +designs = [] +plusargs = [] +assertion = "" diff --git a/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls.cm_hier b/arch/src/main/scala/framework/thread/ant/src/main/resources/tls.cm_hier similarity index 100% rename from arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls.cm_hier rename to arch/src/main/scala/framework/thread/ant/src/main/resources/tls.cm_hier diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/tls.f b/arch/src/main/scala/framework/thread/ant/src/main/resources/tls.f new file mode 100644 index 000000000..fcce13b16 --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/tls.f @@ -0,0 +1,10 @@ ++incdir+@VERIFY@/uvm/src/core ++incdir+@VERIFY@/uvm/src/core +@VERIFY@/uvm/src/core/package.sv +@VERIFY@/uvm/src/core/scenario_if.sv +@VERIFY@/uvm/src/core/scenario_package.sv +@RESOURCES@/../../../../../ip/spm/src/main/resources/package.sv +@RESOURCES@/../../../../../ip/spm/src/main/resources/interface.sv ++incdir+@RTL@ +-F @RTL@/Store/filelist.f +@RESOURCES@/tls_tb.sv diff --git a/arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls.toml b/arch/src/main/scala/framework/thread/ant/src/main/resources/tls.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/tls/src/main/resources/tls.toml rename to arch/src/main/scala/framework/thread/ant/src/main/resources/tls.toml diff --git a/arch/src/main/scala/framework/thread/ant/src/main/resources/tls_tb.sv b/arch/src/main/scala/framework/thread/ant/src/main/resources/tls_tb.sv new file mode 100644 index 000000000..7fe8d8de1 --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/resources/tls_tb.sv @@ -0,0 +1,121 @@ +`include "config.svh" +module tls_tb; + import uvm_pkg::*; + import verification_pkg::*; + import scenario_pkg::*; + import spm_pkg::*; + `include "uvm_macros.svh" + logic clock = 0; + always #5 clock = ~clock; + logic [1:0] reset = '1, active = '0; + scenario_if control (clock); + virtual spm_if ports[4]; + chandle models[2]; + for (genvar i = 0; i < 4; i++) begin : endpoints + spm_if bus ( + clock, + reset[i/2] + ); + initial ports[i] = bus; + end + for (genvar core = 0; core < 2; core++) begin : cores + LocalStore dut ( + .clock(clock), + .reset(reset[core]), + .io_running(active[core]), + .io_cancel(endpoints[core*2+1].bus.cancel), + .io_busy(), + .io_host_request_valid(endpoints[core*2+0].bus.valid), + .io_host_request_ready(endpoints[core*2+0].bus.ready), + .io_host_request_bits_address(endpoints[core*2+0].bus.address), + .io_host_request_bits_size(endpoints[core*2+0].bus.size), + .io_host_request_bits_write(endpoints[core*2+0].bus.write), + .io_host_request_bits_data(endpoints[core*2+0].bus.data), + .io_host_request_bits_mask(endpoints[core*2+0].bus.mask), + .io_host_response_valid(endpoints[core*2+0].bus.response_valid), + .io_host_response_ready(endpoints[core*2+0].bus.response_ready), + .io_host_response_bits_data(endpoints[core*2+0].bus.result), + .io_host_response_bits_error(endpoints[core*2+0].bus.error), + .io_local_request_valid(endpoints[core*2+1].bus.valid), + .io_local_request_ready(endpoints[core*2+1].bus.ready), + .io_local_request_bits_address(endpoints[core*2+1].bus.address), + .io_local_request_bits_size(endpoints[core*2+1].bus.size), + .io_local_request_bits_write(endpoints[core*2+1].bus.write), + .io_local_request_bits_data(endpoints[core*2+1].bus.data), + .io_local_request_bits_mask(endpoints[core*2+1].bus.mask), + .io_local_response_valid(endpoints[core*2+1].bus.response_valid), + .io_local_response_ready(endpoints[core*2+1].bus.response_ready), + .io_local_response_bits_data(endpoints[core*2+1].bus.result), + .io_local_response_bits_error(endpoints[core*2+1].bus.error) + ); + end + initial begin + foreach (models[core]) models[core] = spm_create(`LOCAL_BASE, `LOCAL_BYTES); + #0; + foreach (ports[i]) begin + ports[i].model = models[i/2]; + ports[i].scoreboard = new($sformatf("port_%0d", i), null); + ports[i].enabled = 1; + end + uvm_config_db#(virtual scenario_if)::set(null, "uvm_test_top", "vif", control); + uvm_config_db#(time)::set(null, "uvm_test_top", "timeout", 100us); + run_test("protocol_test"); + end + final foreach (models[i]) spm_destroy(models[i]); + initial begin : exercise + logic [127:0] value; + wait (control.start); + repeat (3) @(negedge clock); + reset = 0; + for (int row = 0; row < `LOCAL_BYTES; row += 16) begin + ports[0].access(`LOCAL_BASE + row, 4, 1, 0, '1); + ports[2].access(`LOCAL_BASE + row, 4, 1, '1, '1); + end + @(negedge clock); + active = 3; + ports[1].access(`LOCAL_BASE, 3, 1, 64'h123456789abcdef0, 16'hff); + ports[3].access(`LOCAL_BASE, 4, 0); + ports[1].access(`LOCAL_BASE + 15, 0, 1, 8'ha5, 1); + ports[1].access(`LOCAL_BASE, 4, 0); + ports[1].send(`LOCAL_BASE, 4, 0); + wait (ports[1].response_valid); + repeat (5) @(negedge clock); + ports[3].access(`LOCAL_BASE + 16, 4, 0); + ports[1].take(value); + ports[1].access(`LOCAL_BASE + 3, 2, 0); + ports[1].access(`LOCAL_BASE + `LOCAL_BYTES, 3, 0); + ports[1].access(64'h80000000, 3, 0); + ports[1].access(`LOCAL_BASE, 5, 0); + ports[1].access(`LOCAL_BASE + 8, 3, 1, 64'h1122334455667788, 16'ha5); + ports[1].access(`LOCAL_BASE + 8, 3, 0); + ports[1].access((64'h1 << 32) | `LOCAL_BASE, 3, 0); + ports[1].access(64'hfffffffffffffff8, 3, 0); + ports[1].access(`LOCAL_BASE, 0, 1, '1, 3); + ports[1].send(`LOCAL_BASE, 3, 0); + ports[1].cancel = 1; + @(negedge clock); + ports[1].cancel = 0; + repeat (5) begin + @(negedge clock); + if (ports[1].response_valid) `uvm_fatal("CANCEL", "old TLS response survived cancellation") + end + ports[1].access(`LOCAL_BASE, 3, 0); + ports[1].send(`LOCAL_BASE + 16, 0, 1, 8'h5a, 1); + wait (ports[1].response_valid); + @(negedge clock); + reset[0] = 1; + active[0] = 0; + repeat (2) @(negedge clock); + reset[0] = 0; + ports[0].access(`LOCAL_BASE + 16, 0, 0); + ports[3].access(`LOCAL_BASE + 16, 0, 0); + repeat (2) @(negedge clock); + foreach (ports[i]) begin + if (ports[i].scoreboard.pending() || ports[i].scoreboard.pending()) + `uvm_fatal("DRAIN", "local storage still has unmatched accesses") + `uvm_info("CHECKED", $sformatf("port %0d: %0d comparisons", i, ports[i].scoreboard.checked), + UVM_LOW) + end + control.done = 1; + end +endmodule diff --git a/arch/src/main/scala/framework/thread/ant/src/main/scala/Ant.scala b/arch/src/main/scala/framework/thread/ant/src/main/scala/Ant.scala new file mode 100644 index 000000000..9f7f73d9d --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/scala/Ant.scala @@ -0,0 +1,163 @@ +package framework.thread.ant + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.thread.ant.configs.AntParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.thread.ant.{Completion, Executor, Retire, Start} +import framework.core.memory.access.preflight.PmpEntry +import framework.acc.frontend.RoCCIO +import framework.ip.memory.spm + +/** Address space and permissions captured from the task's launch command. */ +class TaskContext(permission: PmpParams) extends Bundle { + val satp = UInt(64.W) + val effectivePrivilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + val pmp = + Vec(permission.entries, new PmpEntry(permission)) +} + +/** One accepted launch binds the task context and starts its local program together. */ +class Launch(b: AntParam, permission: PmpParams) extends Bundle { + val descriptor = new Start(b) + val context = new TaskContext(permission) +} + +/** Byte offsets into the TSS and the selected private store; transfers use 8-byte beats. */ +class Transfer extends Bundle { + val code = Bool() + val toLocal = Bool() + val localOffset = UInt(32.W) + val sharedOffset = UInt(32.W) + val bytes = UInt(32.W) +} + +/** One thread task: launch, cancellation and completion. Storage accesses use independent interfaces. */ +class Task(b: AntParam, permission: PmpParams) extends Bundle { + val transfer = Flipped(Decoupled(new Transfer)) + val transferred = Decoupled(Bool()) + val inUse = Input(Bool()) + val start = Flipped(Decoupled(new Launch(b, permission))) + val result = Decoupled(new Completion(b)) + val cancel = Input(Bool()) + val running = Output(Bool()) + val storageBusy = Output(Bool()) + +} + +@instantiable +class Ant(val b: AntParam, permission: PmpParams) extends Module { + + @public + val io = IO(new Bundle { + val trace = Output(Valid(new Retire)) + val task = new Task(b, permission) + + val shared = new spm.Port(b.shared) + val rocc = Flipped(new RoCCIO(64, permission)) + + }) + + val executor: Instance[Executor] = Instantiate(new Executor(b, permission)) + val image = Instantiate(new LocalStore(b, 0)) + val data = Instantiate(new LocalStore(b, 1)) + val loader = Instantiate(new Loader(b)) + + io.trace := executor.io.retired + io.task.storageBusy := executor.io.storageBusy || loader.io.busy + executor.io.start.valid := io.task.start.valid && !loader.io.busy + executor.io.start.bits := io.task.start.bits + io.task.start.ready := executor.io.start.ready && !loader.io.busy + loader.io.transfer.valid := io.task.transfer.valid && !executor.io.storageClaimed && !io.task.start.valid + loader.io.transfer.bits := io.task.transfer.bits + io.task.transfer.ready := loader.io.transfer.ready && !executor.io.storageClaimed && !io.task.start.valid + io.task.transferred <> loader.io.transferred + io.task.result <> executor.io.result + io.task.running := executor.io.running + executor.io.cancel := io.task.cancel + executor.io.inUse := io.task.inUse + executor.io.localBusy(0) := image.io.busy + executor.io.localBusy(1) := data.io.busy + image.io.running := executor.io.storageClaimed + data.io.running := executor.io.storageClaimed + image.io.cancel := io.task.cancel + data.io.cancel := io.task.cancel + image.io.host <> loader.io.code + data.io.host <> loader.io.data + image.io.local <> executor.io.imem + data.io.local <> executor.io.tls + io.shared.request.valid := Mux(loader.io.busy, loader.io.shared.request.valid, executor.io.tss.request.valid) + io.shared.request.bits := Mux(loader.io.busy, loader.io.shared.request.bits, executor.io.tss.request.bits) + loader.io.shared.request.ready := loader.io.busy && io.shared.request.ready + executor.io.tss.request.ready := !loader.io.busy && io.shared.request.ready + loader.io.shared.response.valid := loader.io.busy && io.shared.response.valid + loader.io.shared.response.bits := io.shared.response.bits + executor.io.tss.response.valid := !loader.io.busy && io.shared.response.valid + executor.io.tss.response.bits := io.shared.response.bits + io.shared.response.ready := Mux(loader.io.busy, loader.io.shared.response.ready, executor.io.tss.response.ready) + io.rocc <> executor.io.rocc +} + +/** Moves data between TSS and private storage without exposing private SRAM ports. */ +@instantiable +class Loader(b: AntParam) extends Module { + + @public + val io = IO(new Bundle { + val transfer = Flipped(Decoupled(new Transfer)) + val transferred = Decoupled(Bool()) + val busy = Output(Bool()) + val code = new spm.Port(spm.Params(0, b.codeBytes, b.data.dataBits)) + val data = new spm.Port(b.data) + val shared = new spm.Port(b.shared) + }) + + val idle :: reading :: readResponse :: writing :: writeResponse :: completed :: Nil = Enum(6) + val state = RegInit(idle) + val transfer = Reg(new Transfer) + val offset = Reg(UInt(32.W)) + val value = Reg(UInt(64.W)) + io.transfer.ready := state === idle + io.transferred.valid := state === completed + io.transferred.bits := true.B + io.busy := state =/= idle + when(io.transfer.fire) { + val t = io.transfer.bits + assert(t.bytes =/= 0.U && !(t.localOffset | t.sharedOffset | t.bytes)(2, 0).orR) + assert((t.localOffset +& t.bytes) <= Mux(t.code, b.codeBytes.U, b.data.bytes.U)) + assert((t.sharedOffset +& t.bytes) <= b.shared.bytes.U) + transfer := t + offset := 0.U + state := reading + } + val local = Mux(transfer.code, 0.U, 1.U) + val source = Mux(transfer.toLocal, 2.U, local) + val destination = Mux(transfer.toLocal, local, 2.U) + val ports = Seq(io.code, io.data, io.shared) + for ((port, i) <- ports.zipWithIndex) { + port.request.valid := (state === reading && source === i.U) || (state === writing && destination === i.U) + port.request.bits := 0.U.asTypeOf(port.request.bits) + val base = Seq(BigInt(0), b.data.base, b.shared.base)(i) + port.request.bits.address := base.U + (if (i == 2) transfer.sharedOffset else transfer.localOffset) + offset + port.request.bits.size := 3.U + port.request.bits.write := state === writing + port.request.bits.data := value + port.request.bits.mask := Mux(state === writing, 255.U, 0.U) + port.response.ready := (state === readResponse && source === i.U) || (state === writeResponse && destination === i.U) + when(port.request.fire)(state := Mux(state === reading, readResponse, writeResponse)) + when(port.response.fire) { + assert(!port.response.bits.error, "Ant TSS transfer failed") + when(state === readResponse) { + value := port.response.bits.data(63, 0) + state := writing + }.otherwise { + when(offset + 8.U === transfer.bytes)(state := completed) + .otherwise { offset := offset + 8.U; state := reading } + } + } + } + when(io.transferred.fire)(state := idle) +} diff --git a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Decode.scala b/arch/src/main/scala/framework/thread/ant/src/main/scala/Decode.scala similarity index 97% rename from arch/src/main/scala/framework/system/core/ant/src/main/scala/Decode.scala rename to arch/src/main/scala/framework/thread/ant/src/main/scala/Decode.scala index d2dc96c51..d35f5ebc3 100644 --- a/arch/src/main/scala/framework/system/core/ant/src/main/scala/Decode.scala +++ b/arch/src/main/scala/framework/thread/ant/src/main/scala/Decode.scala @@ -1,4 +1,4 @@ -package framework.ant +package framework.thread.ant import chisel3._ import chisel3.util._ @@ -25,7 +25,8 @@ class Decoded extends Bundle { @instantiable class Decode extends Module { - @public val io = IO(new Bundle { + @public + val io = IO(new Bundle { val instruction = Input(UInt(32.W)) val decoded = Output(new Decoded) }) diff --git a/arch/src/main/scala/framework/thread/ant/src/main/scala/Emit.scala b/arch/src/main/scala/framework/thread/ant/src/main/scala/Emit.scala new file mode 100644 index 000000000..b64d05625 --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/scala/Emit.scala @@ -0,0 +1,148 @@ +package framework.thread.ant + +import buckyball.config.Design +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.Instantiate +import framework.core.controller.configs.TasksParam +import framework.core.controller.task.{Control, Service} +import framework.thread.ant.{Completion, Retire, Start} +import framework.acc.frontend.RoCCIO +import framework.core.tss.{Store => TssStore} +import framework.core.tss.configs.{Params => TssParam} +import framework.core.memory.access.preflight.configs.PmpParams +import framework.device.cbus.ControlRequest +import framework.ip.memory.spm +import java.nio.file.{Files, Paths} +import scala.jdk.CollectionConverters._ + +/** Direct task/transfer verification around real Ants and their shared storage. */ +class AntVerification(p: TasksParam) extends Module { + require(p.locals.size == 2 && p.locals.distinct.size == 1) + val local = p.locals.head + + val io = IO(new Bundle { + val inUse = Input(Bool()) + val start = Vec(2, Flipped(Decoupled(new Start(local)))) + val result = Vec(2, Decoupled(new Completion(local))) + val transfer = Vec(2, Flipped(Decoupled(new Transfer))) + val transferred = Vec(2, Decoupled(Bool())) + val cancel = Input(Vec(2, Bool())) + val running = Output(Vec(2, Bool())) + val rocc = Vec(2, Flipped(new RoCCIO(64, p.permission))) + val retired = Output(Vec(2, Valid(new Retire))) + val host = Flipped(new spm.Port(p.tss.memory)) + }) + + val ants = Seq.fill(2)(Instantiate(new Ant(local, p.permission))) + val shared = Instantiate(new TssStore(p.tss)) + shared.io.host <> io.host + shared.io.inUse := io.inUse || ants.map(_.io.task.storageBusy).reduce(_ || _) + shared.io.cancel := io.cancel + for ((ant, i) <- ants.zipWithIndex) { + ant.io.task.inUse := io.inUse + ant.io.task.start.valid := io.start(i).valid && !shared.io.busy(i) && !shared.io.hostBusy + ant.io.task.start.bits.descriptor := io.start(i).bits + ant.io.task.start.bits.context := 0.U.asTypeOf(ant.io.task.start.bits.context) + io.start(i).ready := ant.io.task.start.ready && !shared.io.busy(i) && !shared.io.hostBusy + io.result(i).valid := ant.io.task.result.valid && !shared.io.busy(i) + io.result(i).bits := ant.io.task.result.bits + ant.io.task.result.ready := io.result(i).ready && !shared.io.busy(i) + ant.io.task.transfer <> io.transfer(i) + io.transferred(i) <> ant.io.task.transferred + ant.io.task.cancel := io.cancel(i) + io.running(i) := ant.io.task.running + io.rocc(i) <> ant.io.rocc + io.retired(i) := ant.io.trace + shared.io.clients(i) <> ant.io.shared + } +} + +/** Production management service connected to real Ant/TSS instances. */ +class ServiceVerification(p: TasksParam) extends Module { + + val io = IO(new Bundle { + val request = Flipped(Decoupled(new ControlRequest)) + val reply = Decoupled(UInt(64.W)) + val rocc = Vec(p.count, Flipped(new RoCCIO(64, p.permission))) + val retired = Output(Vec(p.count, Valid(new Retire))) + }) + + val service = Instantiate(new Service(p)) + val shared = Instantiate(new TssStore(p.tss)) + service.io.request.valid := io.request.valid + service.io.request.bits.control := io.request.bits + service.io.request.bits.context := 0.U.asTypeOf(service.io.request.bits.context) + io.request.ready := service.io.request.ready + io.reply <> service.io.reply + shared.io.host <> service.io.tss.host + shared.io.inUse := service.io.tss.inUse + shared.io.cancel := service.io.tss.cancel + service.io.tss.busy := shared.io.busy + service.io.tss.hostBusy := shared.io.hostBusy + for (i <- 0 until p.count) { + val ant = Instantiate(new Ant(p.locals(i), p.permission)) + ant.io.task <> service.io.tasks(i) + shared.io.clients(i) <> ant.io.shared + io.rocc(i) <> ant.io.rocc + io.retired(i) := ant.io.trace + } +} + +object Emit extends App { + require(args.length == 4 && args(0) == "--config" && args(2) == "--core") + val design = Design.parseFrom(Files.readAllBytes(Paths.get(args(1)))) + val core = design.getDevice.getCoresList.asScala.find(_.getHartId == args(3).toInt).get + val shared = core.getTss + val memory = spm.Params(BigInt(java.lang.Long.toUnsignedString(shared.getBase)), shared.getBytes, shared.getDataBits) + require(core.getThreadIndicesCount >= 2, "Ant verification requires two configured hardware threads") + val threads = core.getThreadIndicesList.asScala.take(2).map(i => design.getThreads(i.intValue)).toSeq + val locals = threads.map(t => configs.AntParam.fromProto(t.getCpu.getAnt, memory)) + + val p = TasksParam( + locals, + threads.map(t => BigInt(java.lang.Long.toUnsignedString(t.getSignature))), + TssParam(memory, 2), + PmpParams(core.getParam.getPaddrBits, core.getParam.getNPmps) + ) + + require(locals.distinct.size == 1 && locals.head.taskBits == 32 && memory.dataBits == 128) + require(locals.head.data.base == 0x10000 && memory.base == 0x20000, "Shared-program fixture address encoding mismatch") + val directory = Paths.get("build") + Files.createDirectories(directory) + Files.writeString( + directory.resolve("config.svh"), + s"""`define LOCAL_BASE 64'h${locals.head.data.base.toString(16)} +`define LOCAL_BYTES ${locals.head.data.bytes} +`define TLS_BASE 64'h${locals.head.data.base.toString(16)} +`define TLS_END 64'h${(locals.head.data.base + locals.head.data.bytes).toString(16)} +`define TSS_BASE 64'h${memory.base.toString(16)} +`define TSS_SCRATCH 64'h${(memory.base + memory.bytes - 32).toString(16)} +`define CODE_BYTES ${locals.head.codeBytes} +`define TLS_BYTES ${locals.head.data.bytes} +`define TSS_BYTES ${memory.bytes} +`define SIGNATURE_0 64'h${p.signatures(0).toString(16)} +`define SIGNATURE_1 64'h${p.signatures(1).toString(16)} +""" + ) + val fields = Seq("cfg_l", "cfg_res", "cfg_a", "cfg_x", "cfg_w", "cfg_r", "addr", "mask") + Files.writeString( + directory.resolve("control_pmp.svh"), + (for { + i <- 0 until p.permission.entries + field <- fields + } yield s", .io_request_bits_context_pmp_${i}_$field('0)").mkString("\n") + ) + Seq[(String, () => RawModule)]( + "Store" -> (() => new LocalStore(locals.head, 1)), + "Ant" -> (() => new AntVerification(p)), + "Control" -> (() => new Control(p)), + "Service" -> (() => new ServiceVerification(p)) + ).foreach { case (name, top) => + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + top(), + Array("--target-dir", s"build/$name", "--split-verilog"), + firtoolOpts = Array("--disable-all-randomization") + ) + } +} diff --git a/arch/src/main/scala/framework/thread/ant/src/main/scala/Executor.scala b/arch/src/main/scala/framework/thread/ant/src/main/scala/Executor.scala new file mode 100644 index 000000000..780e3d1fa --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/scala/Executor.scala @@ -0,0 +1,329 @@ +package framework.thread.ant + +import framework.thread.ant.configs.AntParam +import framework.core.memory.access.preflight.configs.PmpParams + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.lib.arith.{DivRem, Multiply} +import framework.ip.memory.spm +import framework.thread.ant.{Launch, TaskContext} +import framework.acc.frontend.RoCCIO + +class Start(b: AntParam) extends Bundle { + val p = b + + val task = UInt(p.taskBits.W) + val entry = UInt(64.W) + val codeEnd = UInt(64.W) + val argument = UInt(64.W) + val stack = UInt(64.W) +} + +class Completion(b: AntParam) extends Bundle { + val p = b + + val task = UInt(p.taskBits.W) + val value = UInt(64.W) + val cancelled = Bool() +} + +class Retire extends Bundle { + val pc = UInt(64.W) + val instruction = UInt(32.W) + val rd = UInt(5.W) + val data = UInt(64.W) +} + +/** + * One RV64IM execution context. Ordinary memory is strictly local. NPU work may remain + * in flight after a command is accepted; completion waits for accelerator and storage drain. + * Cancellation also reaches the local stores, whose accepted operations must drain before reuse. + */ +@instantiable +class Executor(val b: AntParam, permission: PmpParams) extends Module { + val p = b + val code = spm.Params(0, p.codeBytes, p.data.dataBits) + + @public + val io = IO(new Bundle { + + val start = Flipped(Decoupled(new Launch(b, permission))) + + val result = Decoupled(new Completion(b)) + + val cancel = Input(Bool()) + val inUse = Input(Bool()) + val localBusy = Input(Vec(2, Bool())) + val storageBusy = Output(Bool()) + val storageClaimed = Output(Bool()) + val running = Output(Bool()) + + val imem = new spm.Port(code) + + val tls = new spm.Port(p.data) + + val tss = new spm.Port(p.shared) + + val rocc = Flipped(new RoCCIO(64, permission)) + + val retired = Output(Valid(new Retire)) + }) + + val (idle :: fetch :: fetchWait :: execute :: memory :: memoryWait :: + multiply :: multiplyWait :: divide :: divideWait :: issue :: complete :: Nil) = Enum(12) + val state = RegInit(idle) + val registers = Reg(Vec(32, UInt(64.W))) + val pc = Reg(UInt(64.W)) + val instruction = Reg(UInt(32.W)) + val task = Reg(UInt(p.taskBits.W)) + val codeEnd = Reg(UInt(64.W)) + + val completion = Reg(new Completion(b)) + val context = Reg(new TaskContext(permission)) + io.storageBusy := io.localBusy.asUInt.orR + io.storageClaimed := state =/= idle || (io.inUse && io.start.valid) + val drained = !io.storageBusy && !io.rocc.busy + + val cancelled = RegInit(false.B) + val multiplyPending = RegInit(false.B) + val dividePending = RegInit(false.B) + val live = !io.cancel && !reset.asBool + val memoryAddress = Reg(UInt(64.W)) + val memoryShared = Reg(Bool()) + val decoder = Instantiate(new Decode) + val multiplier = Instantiate(new Multiply) + val divider = + Instantiate(new DivRem(64)) + decoder.io.instruction := instruction + val d = decoder.io.decoded + val a = Mux(d.rs1 === 0.U, 0.U, registers(d.rs1)) + val rhs = Mux(d.rs2 === 0.U, 0.U, registers(d.rs2)) + val nextPc = pc + 4.U + + io.start.ready := state === idle && io.inUse && drained && !io.cancel && !reset.asBool + io.result.valid := state === complete && drained && !multiplyPending && !dividePending && !reset.asBool + io.result.bits := completion + io.running := state =/= idle + io.retired.valid := false.B + io.retired.bits := 0.U.asTypeOf(new Retire) + + def retire(rd: UInt, value: UInt, next: UInt): Unit = { + when(live) { + when(rd =/= 0.U)(registers(rd) := value) + io.retired.valid := true.B + io.retired.bits.pc := pc + io.retired.bits.instruction := instruction + io.retired.bits.rd := rd + io.retired.bits.data := Mux(rd === 0.U, 0.U, value) + pc := next + state := fetch + } + } + + def finish(value: UInt, wasCancelled: Bool = false.B): Unit = { + completion.task := task + completion.value := value + completion.cancelled := wasCancelled + state := complete + } + + def word(value: UInt): UInt = Mux(d.word, value(31, 0).asSInt.pad(64).asUInt, value) + + when(io.start.fire) { + context := io.start.bits.context + for (r <- registers) r := 0.U + registers(2) := io.start.bits.descriptor.stack + registers(4) := p.data.base.U + registers(10) := io.start.bits.descriptor.argument + pc := io.start.bits.descriptor.entry + codeEnd := io.start.bits.descriptor.codeEnd + task := io.start.bits.descriptor.task + cancelled := false.B + val valid = io.start.bits.descriptor.entry(1, 0) === 0.U && io.start.bits.descriptor.codeEnd(1, 0) === 0.U && + io.start.bits.descriptor.codeEnd <= p.codeBytes.U && (io.start.bits.descriptor.entry +& 4.U) <= io.start.bits.descriptor.codeEnd && + io.start.bits.descriptor.argument >= p.data.base.U && io.start.bits.descriptor.argument < (p.data.base + p.data.bytes).U && + io.start.bits.descriptor.stack > p.data.base.U && io.start.bits.descriptor.stack <= (p.data.base + p.data.bytes).U && + io.start.bits.descriptor.stack(3, 0) === 0.U + assert(valid, "Invalid Ant local execution descriptor") + when(valid)(state := fetch) + } + + val fetchAllowed = pc(1, 0) === 0.U && (pc +& 4.U) <= codeEnd + io.imem.request.valid := state === fetch && fetchAllowed && live + io.imem.request.bits := 0.U.asTypeOf(io.imem.request.bits) + io.imem.request.bits.address := pc + io.imem.request.bits.size := 2.U + io.imem.response.ready := state === fetchWait || (state === complete && cancelled) + when(state === fetch && live) { + assert(fetchAllowed, "Ant instruction address invalid: pc=0x%x", pc) + when(io.imem.request.fire)(state := fetchWait) + } + when(state === fetchWait && io.imem.response.fire && live) { + assert(!io.imem.response.bits.error, "Ant instruction storage access failed: pc=0x%x", pc) + when(!io.imem.response.bits.error) { + instruction := io.imem.response.bits.data(31, 0) + state := execute + } + } + + val operand = Mux(d.immediateAlu, d.immediate, rhs) + val shift = Mux(d.word, Cat(0.U(1.W), operand(4, 0)), operand(5, 0)) + val logical = Mux(d.word, Cat(0.U(32.W), a(31, 0)), a) + val signed = Mux(d.word, a(31, 0).asSInt.pad(64).asUInt, a) + + val alu = MuxLookup(d.funct3, 0.U(64.W))(Seq( + 0.U -> Mux(d.subtract, a - operand, a + operand), + 1.U -> (a << shift)(63, 0), + 2.U -> (a.asSInt < operand.asSInt).asUInt, + 3.U -> (a < operand).asUInt, + 4.U -> (a ^ operand), + 5.U -> Mux(d.arithmeticShift, (signed.asSInt >> shift).asUInt, logical >> shift), + 6.U -> (a | operand), + 7.U -> (a & operand) + )) + + val branchTaken = MuxLookup(d.funct3, false.B)(Seq( + 0.U -> (a === rhs), + 1.U -> (a =/= rhs), + 4.U -> (a.asSInt < rhs.asSInt), + 5.U -> (a.asSInt >= rhs.asSInt), + 6.U -> (a < rhs), + 7.U -> (a >= rhs) + )) + + val address = a + d.immediate + val size = d.funct3(1, 0) + val bytes = 1.U(64.W) << size + val end = address +& bytes + val inTls = address >= p.data.base.U && end <= (p.data.base + p.data.bytes).U + val inTss = address >= p.shared.base.U && end <= (p.shared.base + p.shared.bytes).U + when(state === execute && live) { + switch(d.kind) { + is(Kind.illegal)(assert(false.B, "Ant illegal instruction: pc=0x%x instruction=0x%x", pc, instruction)) + is(Kind.lui)(retire(d.rd, d.immediate, nextPc)) + is(Kind.auipc)(retire(d.rd, pc + d.immediate, nextPc)) + is(Kind.alu)(retire(d.rd, word(alu), nextPc)) + is(Kind.jal, Kind.jalr, Kind.branch) { + val target = Mux( + d.kind === Kind.jalr, + (a + d.immediate) & ~1.U(64.W), + Mux(d.kind === Kind.branch && !branchTaken, nextPc, pc + d.immediate) + ) + assert(target(1, 0) === 0.U, "Ant branch target misaligned: pc=0x%x target=0x%x", pc, target) + retire(Mux(d.kind === Kind.branch, 0.U, d.rd), nextPc, target) + } + is(Kind.load, Kind.store) { + assert((address & (bytes - 1.U)) === 0.U, "Ant load/store misaligned: pc=0x%x address=0x%x", pc, address) + assert(inTls || inTss, "Ant load/store outside TLS/TSS: pc=0x%x address=0x%x", pc, address) + when((address & (bytes - 1.U)) === 0.U && (inTls || inTss)) { + memoryAddress := address + memoryShared := inTss + state := memory + } + } + is(Kind.multiply)(state := multiply) + is(Kind.divide)(state := divide) + is(Kind.custom)(state := issue) + is(Kind.fence)(retire(0.U, 0.U, nextPc)) // Ordinary LSU operations already complete in order. + is(Kind.exit) { + assert(registers(17) === 0.U, "Ant unsupported ECALL: pc=0x%x a7=0x%x", pc, registers(17)) + when(registers(17) === 0.U)(finish(registers(10))) + } + is(Kind.breakpoint)(assert(false.B, "Ant breakpoint: pc=0x%x", pc)) + } + } + + val mask = MuxLookup(size, 0.U(p.data.beatBytes.W))(Seq( + 0.U -> 1.U, + 1.U -> 3.U, + 2.U -> 15.U, + 3.U -> 255.U + )) + + for ((port, shared) <- Seq((io.tls, false), (io.tss, true))) { + port.request.valid := state === memory && memoryShared === shared.B && live + port.request.bits := 0.U.asTypeOf(port.request.bits) + port.request.bits.address := memoryAddress + port.request.bits.size := size + port.request.bits.write := d.kind === Kind.store + port.request.bits.data := rhs + port.request.bits.mask := Mux(d.kind === Kind.store, mask, 0.U) + port.response.ready := (state === memoryWait && memoryShared === shared.B) || (state === complete && cancelled) + } + when(io.tls.request.fire || io.tss.request.fire)(state := memoryWait) + val memoryResponse = Mux(memoryShared, io.tss.response.bits, io.tls.response.bits) + val memoryValid = Mux(memoryShared, io.tss.response.valid, io.tls.response.valid) + + val loaded = MuxLookup(d.funct3, memoryResponse.data(63, 0))(Seq( + 0.U -> memoryResponse.data(7, 0).asSInt.pad(64).asUInt, + 1.U -> memoryResponse.data(15, 0).asSInt.pad(64).asUInt, + 2.U -> memoryResponse.data(31, 0).asSInt.pad(64).asUInt, + 4.U -> memoryResponse.data(7, 0).pad(64), + 5.U -> memoryResponse.data(15, 0).pad(64), + 6.U -> memoryResponse.data(31, 0).pad(64) + )) + + when(state === memoryWait && memoryValid && live) { + assert(!memoryResponse.error, "Ant local memory access failed: pc=0x%x address=0x%x", pc, memoryAddress) + when(!memoryResponse.error)(retire(Mux(d.kind === Kind.store, 0.U, d.rd), loaded, nextPc)) + } + + multiplier.io.request.valid := state === multiply && live + multiplier.io.request.bits.a := a + multiplier.io.request.bits.b := rhs + multiplier.io.request.bits.high := d.funct3 =/= 0.U + multiplier.io.request.bits.signedA := d.funct3 === 1.U || d.funct3 === 2.U + multiplier.io.request.bits.signedB := d.funct3 === 1.U + multiplier.io.response.ready := state === multiplyWait || (state === complete && cancelled) + when(multiplier.io.request.fire) { state := multiplyWait; multiplyPending := true.B } + when(multiplier.io.response.fire)(multiplyPending := false.B) + when(state === multiplyWait && multiplier.io.response.fire && live) { + retire(d.rd, word(multiplier.io.response.bits), nextPc) + } + divider.io.clear := false.B + divider.io.request.valid := state === divide && live + divider.io.request.bits.a := a + divider.io.request.bits.b := rhs + divider.io.request.bits.sew := Mux(d.word, 2.U, 3.U) + divider.io.request.bits.signed := !d.funct3(0) + divider.io.request.bits.remainder := d.funct3(1) + divider.io.response.ready := state === divideWait || (state === complete && cancelled) + when(divider.io.request.fire) { state := divideWait; dividePending := true.B } + when(divider.io.response.fire)(dividePending := false.B) + when(state === divideWait && divider.io.response.fire && live) { + retire(d.rd, word(divider.io.response.bits), nextPc) + } + + io.rocc.cmd.valid := state === issue && live + io.rocc.cmd.bits.instruction.raw_inst := instruction + io.rocc.cmd.bits.instruction.pc := pc + io.rocc.cmd.bits.instruction.rs1Data := Mux(instruction(13), a, 0.U) + io.rocc.cmd.bits.instruction.rs2Data := Mux(instruction(12), rhs, 0.U) + io.rocc.cmd.bits.satp := context.satp + io.rocc.cmd.bits.effectivePrivilege := context.effectivePrivilege + io.rocc.cmd.bits.sum := context.sum + io.rocc.cmd.bits.mxr := context.mxr + io.rocc.cmd.bits.pmp := context.pmp + io.rocc.resp.ready := false.B + when(io.rocc.cmd.valid) { + assert(!instruction(14), "Ant accelerator commands do not write integer registers") + } + when(io.rocc.cmd.fire)(retire(0.U, 0.U, nextPc)) + assert(!io.rocc.resp.valid, "Ant received an unsolicited RoCC response") + assert(!io.rocc.interrupt, "Ant accelerator fault") + when(io.result.fire)(state := idle) + when(io.cancel && state =/= idle && state =/= complete) { + finish(0.U, true.B) + cancelled := true.B + io.retired.valid := false.B + } + // Local execution may have ended while the task still waits for NPU/DMA drain. + // Once a completion is published, preserve it under response backpressure. + when(io.cancel && state === complete && !io.result.valid) { + completion.cancelled := true.B + completion.value := 0.U + cancelled := true.B + } +} diff --git a/arch/src/main/scala/framework/thread/ant/src/main/scala/LocalStore.scala b/arch/src/main/scala/framework/thread/ant/src/main/scala/LocalStore.scala new file mode 100644 index 000000000..382011997 --- /dev/null +++ b/arch/src/main/scala/framework/thread/ant/src/main/scala/LocalStore.scala @@ -0,0 +1,61 @@ +package framework.thread.ant + +import framework.thread.ant.configs.AntParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.ip.memory.spm.{Memory, Params, Port} + +/** + * A context-private local store. Loader and execution access it in separate ownership phases. + * Cancellation drains an accepted local access and discards its response before reuse. + */ +@instantiable +class LocalStore(val b: AntParam, index: Int) extends Module { + val local = b + + val (p, localReadOnly) = index match { + case 0 => (Params(0, local.codeBytes, local.data.dataBits), true) + case 1 => (local.data, false) + } + + @public + val io = IO(new Bundle { + val running = Input(Bool()) + val cancel = Input(Bool()) + + val host = Flipped(new Port(p)) + + val local = Flipped(new Port(p)) + + val busy = Output(Bool()) + }) + + val memory = Instantiate(new Memory(p)) + + val busy = RegInit(false.B) + val owner = Reg(Bool()) + val cancelled = RegInit(false.B) + val discard = owner && (cancelled || io.cancel) + memory.io.port.request.valid := !busy && !reset.asBool && + Mux(io.running, io.local.request.valid && !io.cancel, io.host.request.valid) + memory.io.port.request.bits := Mux(io.running, io.local.request.bits, io.host.request.bits) + memory.io.readOnly := io.running && localReadOnly.B + io.local.request.ready := !busy && io.running && !io.cancel && memory.io.port.request.ready + io.host.request.ready := !busy && !io.running && memory.io.port.request.ready + when(memory.io.port.request.fire) { + busy := true.B + owner := io.running + cancelled := false.B + } + when(busy && owner && io.cancel)(cancelled := true.B) + io.local.response.valid := busy && owner && !discard && memory.io.port.response.valid + io.host.response.valid := busy && !owner && memory.io.port.response.valid + io.local.response.bits := memory.io.port.response.bits + io.host.response.bits := memory.io.port.response.bits + memory.io.port.response.ready := busy && + (discard || Mux(owner, io.local.response.ready, io.host.response.ready)) + when(memory.io.port.response.fire) { busy := false.B; cancelled := false.B } + io.busy := busy +} diff --git a/arch/src/main/scala/framework/thread/build.sc b/arch/src/main/scala/framework/thread/build.sc new file mode 100644 index 000000000..a1ccd93a1 --- /dev/null +++ b/arch/src/main/scala/framework/thread/build.sc @@ -0,0 +1,16 @@ +import mill._ +import $file.ant.{build => ant_build} +import $file.tlink.{build => tlink_build} + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("configs/ThreadParam.scala")) + ) ++ + ant_build.sources() ++ + tlink_build.sources() +) + +def emitSources = T.sources( + ant_build.emitSources() ++ + tlink_build.emitSources() +) diff --git a/arch/src/main/scala/framework/thread/configs/ThreadParam.scala b/arch/src/main/scala/framework/thread/configs/ThreadParam.scala new file mode 100644 index 000000000..ac361a1bc --- /dev/null +++ b/arch/src/main/scala/framework/thread/configs/ThreadParam.scala @@ -0,0 +1,39 @@ +package framework.thread.configs + +import buckyball.config.{CorePlacement, ThreadInstance} +import framework.ip.memory.spm +import framework.acc.configs.AccParam +import framework.thread.ant.configs.AntParam + +case class ThreadParam( + id: Int, + name: String, + component: String, + rtlClass: String, + signature: BigInt, + commandBits: Int, + acc: Option[AccParam], + ant: Option[AntParam]) { + require(ant.isEmpty || commandBits == 64, "Ant implements a 64-bit RoCC command interface") +} + +object ThreadParam { + + def fromProto(thread: ThreadInstance, core: CorePlacement): ThreadParam = { + require(thread.hasLocalId && thread.hasSignature) + require(core.hasTss && thread.hasCpu && thread.getCpu.getKind == "ant" && thread.getCpu.hasAnt) + val tss = core.getTss + val shared = spm.Params(BigInt(java.lang.Long.toUnsignedString(tss.getBase)), tss.getBytes, tss.getDataBits) + ThreadParam( + id = thread.getLocalId, + name = thread.getRole, + component = thread.getPkg, + rtlClass = thread.getRtlClass, + signature = BigInt(java.lang.Long.toUnsignedString(thread.getSignature)), + commandBits = core.getParam.getXLen, + acc = Some(AccParam.fromProto(thread, core)), + ant = Some(AntParam.fromProto(thread.getCpu.getAnt, shared)) + ) + } + +} diff --git a/arch/src/main/scala/framework/thread/tlink/basethread.scala b/arch/src/main/scala/framework/thread/tlink/basethread.scala new file mode 100644 index 000000000..e60207d8a --- /dev/null +++ b/arch/src/main/scala/framework/thread/tlink/basethread.scala @@ -0,0 +1,9 @@ +package framework.thread.tlink + +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +trait HasTLink { + @public + val tlink: TLinkIO +} diff --git a/arch/src/main/scala/framework/thread/tlink/build.sc b/arch/src/main/scala/framework/thread/tlink/build.sc new file mode 100644 index 000000000..295276ed8 --- /dev/null +++ b/arch/src/main/scala/framework/thread/tlink/build.sc @@ -0,0 +1,12 @@ +import mill._ + +def sources = T.sources( + Seq( + PathRef(millSourcePath / os.RelPath("basethread.scala")), + PathRef(millSourcePath / os.RelPath("tlink.scala")) + ) +) + +def emitSources = T.sources( + Seq.empty[PathRef] +) diff --git a/arch/src/main/scala/framework/thread/tlink/tlink.scala b/arch/src/main/scala/framework/thread/tlink/tlink.scala new file mode 100644 index 000000000..e998e34a5 --- /dev/null +++ b/arch/src/main/scala/framework/thread/tlink/tlink.scala @@ -0,0 +1,82 @@ +package framework.thread.tlink + +import sims.configs.SimParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.core.tbus.configs.{Params => TBusParam} + +import chisel3._ +import chisel3.util._ +import framework.core.tbus.router.{LeasePort, PhysicalPort} +import framework.acc.memdomain.{BankPort, LayoutConfig, MvoverISA, MvoverPort} +import framework.acc.memdomain.bank_router.BankMapping +import framework.ip.memory.mesh_shm.MeshLocalBankPort +import framework.thread.configs.ThreadParam +import framework.thread.ant.Task +import framework.core.memory.MemoryPort +import framework.core.memory.access.preflight.configs.PmpParams +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.acc.memdomain.PhysicalBankHash +import framework.ip.memory.spm +import framework.ip.bus.axi4 + +/** Thread-facing shared-bank access, lifetime and synchronization. */ +class BankAccess(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val sharedPorts = memParam.sharedPorts + require(sharedPorts >= 0 && (memParam.sharedEnable || sharedPorts == 0)) + val requests = Vec(sharedPorts, new BankPort(frontendParam, memParam)) + + val local = Flipped(new MeshLocalBankPort( + MvoverISA.AddressBits, + MvoverISA.BankBits, + memParam.bankWidth, + math.max(1, log2Ceil(frontendParam.rob_entries)) + )) + + val config = Decoupled(new LayoutConfig(frontendParam, memParam)) + val mapping = Decoupled(new BankMapping(frontendParam)) +} + +class SharedLink(frontendParam: FrontendParam, memParam: MemDomainParam) extends Bundle { + val sharedPorts = memParam.sharedPorts + val bank = new BankAccess(frontendParam, memParam) + val move = new MvoverPort + val barrierArrive = Output(Bool()) + val barrierRelease = Input(Bool()) +} + +/** Complete thread boundary; each service retains its independent handshake. */ +class TLinkIO( + thread: ThreadParam, + permission: PmpParams, + tracking: TrackingParam, + walker: WalkerParam, + simp: SimParam) + extends Bundle { + val sharedPorts = thread.acc.get.memDomain.sharedPorts + val diffTest = simp.diffTest + val task = new Task(thread.ant.get, permission) + val tss = new spm.Port(thread.ant.get.shared) + val memory = new MemoryPort(tracking, walker) + val ddr = new axi4.Port(thread.acc.get.transport.axi) + val shm = new SharedLink(thread.acc.get.frontend, thread.acc.get.memDomain) + + val sharedHashes = Option.when(diffTest && thread.acc.get.memDomain.sharedEnable)( + Input(Vec(thread.acc.get.memDomain.sharedBankNum, new PhysicalBankHash(thread.acc.get.frontend))) + ) + +} + +/** External shared-memory access and optional bank-movement commands. */ +class ExternalLink(memParam: MemDomainParam, tbusParam: TBusParam) extends Bundle { + val p = tbusParam + val lease = Option.when(p.router.physicalPorts > 0)(Flipped(new LeasePort)) + + val memory = Vec( + p.router.physicalPorts, + Flipped(new PhysicalPort(memParam.bankWidth)) + ) + + val move = Option.when(p.externalMove)(Flipped(new MvoverPort)) +} diff --git a/arch/src/main/scala/framework/top/GlobalConfig.scala b/arch/src/main/scala/framework/top/GlobalConfig.scala deleted file mode 100644 index 887b3ad3c..000000000 --- a/arch/src/main/scala/framework/top/GlobalConfig.scala +++ /dev/null @@ -1,40 +0,0 @@ -package framework.top - -import upickle.default.{macroRW, ReadWriter} -import chisel3.experimental.SerializableModuleParameter -import framework.memdomain.configs.MemDomainParam -import framework.frontend.configs.FrontendParam -import framework.rvv.configs.RvvParam -import framework.balldomain.configs.BallDomainParam -import framework.system.tile.configs.TileParam -import framework.top.configs.SimParam - -case class GlobalConfig( - memDomain: MemDomainParam, - frontend: FrontendParam, - rvv: RvvParam, - ballDomain: BallDomainParam, - tile: TileParam, - sim: SimParam, - coreSignature: BigInt = 0, - systemDesign: Option[framework.top.configs.SystemDesign] = None, - tileDesign: Option[framework.top.configs.TileScope] = None, - coreDesign: Option[framework.top.configs.CoreScope] = None) - extends SerializableModuleParameter - -object GlobalConfig { - - def apply(): GlobalConfig = { - GlobalConfig( - memDomain = MemDomainParam(), - frontend = FrontendParam(), - rvv = RvvParam(), - ballDomain = BallDomainParam(), - tile = TileParam(), - sim = SimParam() - ) - } - - implicit val rw: ReadWriter[GlobalConfig] = macroRW[GlobalConfig] - -} diff --git a/arch/src/main/scala/framework/top/configs/DesignConfig.scala b/arch/src/main/scala/framework/top/configs/DesignConfig.scala deleted file mode 100644 index 7a1fc5ef6..000000000 --- a/arch/src/main/scala/framework/top/configs/DesignConfig.scala +++ /dev/null @@ -1,130 +0,0 @@ -package framework.top.configs - -import upickle.default.{macroRW, ReadWriter} -import framework.frontend.configs.FrontendParam -import framework.memdomain.configs.MemDomainParam -import framework.rvv.configs.RvvParam -import framework.balldomain.configs.BallDomainParam -import framework.system.core.rocket.configs.RocketCpuParam -import framework.system.tile.configs.TileParam -import memcore.bus.chi -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.cache.configs.CacheParams -import memcore.memory.{interlock, spm} -import memcore.memory.ddr.{Params => DdrParams} - -case class AcceleratorConfig( - memDomain: MemDomainParam, - frontend: FrontendParam, - rvv: RvvParam, - ballDomain: BallDomainParam, - tile: TileParam, - sim: SimParam, - coreSignature: BigInt) - -object AcceleratorConfig { implicit val rw: ReadWriter[AcceleratorConfig] = macroRW } - -case class CoreDesign( - factory: String, - rocket: Option[RocketCpuParam], - local: Option[framework.ant.Params], - accelerator: Option[AcceleratorConfig]) { - require(rocket.isDefined != local.isDefined) -} - -object CoreDesign { - import DesignCodecs._ - implicit val rw: ReadWriter[CoreDesign] = macroRW -} - -case class SharedConfig(bankBits: Int, bankEntries: Int, banks: Int) -object SharedConfig { implicit val rw: ReadWriter[SharedConfig] = macroRW } - -case class TileDesign( - factory: String, - param: TileParam, - main: Boolean, - cores: Seq[CoreDesign], - signatures: Seq[BigInt], - controller: Option[Int], - sharedStorage: Option[SharedConfig]) - -object TileDesign { implicit val rw: ReadWriter[TileDesign] = macroRW } - -case class DevicesConfig( - clintBase: BigInt, - clintBytes: BigInt, - tickCycles: Int, - plicBase: BigInt, - plicBytes: BigInt, - sources: Int, - priorityBits: Int, - bootrom: Option[RomConfig], - scuBase: BigInt, - scuStride: BigInt, - scuBytes: BigInt, - scuMaxHarts: Int) - -object DevicesConfig { implicit val rw: ReadWriter[DevicesConfig] = macroRW } -case class RomConfig(base: BigInt, bytes: BigInt, image: Seq[Byte]) -object RomConfig { implicit val rw: ReadWriter[RomConfig] = macroRW } - -case class PlatformConfig( - cpuPhysicalBits: Int, - memory: CoherenceParams, - l1: RnfParams, - regions: Seq[PhysicalRegion], - tracking: interlock.Params, - ddr: DdrParams, - devices: DevicesConfig) - -object PlatformConfig { - import DesignCodecs._ - implicit val rw: ReadWriter[PlatformConfig] = macroRW -} - -case class TilePlacement(design: TileDesign, hartIds: Seq[Int], executionIds: Seq[Int]) -object TilePlacement { implicit val rw: ReadWriter[TilePlacement] = macroRW } -case class SystemDesign(tiles: Seq[TilePlacement], platform: PlatformConfig) -object SystemDesign { implicit val rw: ReadWriter[SystemDesign] = macroRW } - -case class TileScope( - design: TileDesign, - platform: PlatformConfig, - maxHartId: Int, - controls: Int, - tiles: Int) - -object TileScope { implicit val rw: ReadWriter[TileScope] = macroRW } - -case class CoreScope( - factory: String, - rocket: Option[RocketCpuParam], - local: Option[framework.ant.Params], - hasAccelerator: Boolean, - controller: Option[RocketCpuParam], - tile: TileParam, - platform: PlatformConfig, - maxHartId: Int, - cpuIndex: Int, - cpuCount: Int, - signature: BigInt) - -object CoreScope { - import DesignCodecs._ - implicit val rw: ReadWriter[CoreScope] = macroRW -} - -object DesignCodecs { - implicit val chiRW: ReadWriter[chi.Params] = macroRW - implicit val cacheRW: ReadWriter[CacheParams] = macroRW - implicit val coherenceRW: ReadWriter[CoherenceParams] = macroRW - implicit val rnfRW: ReadWriter[RnfParams] = macroRW - implicit val regionRW: ReadWriter[PhysicalRegion] = macroRW - implicit val trackingRW: ReadWriter[interlock.Params] = macroRW - implicit val ddrRW: ReadWriter[DdrParams] = macroRW - implicit val spmRW: ReadWriter[spm.Params] = macroRW - implicit val antRW: ReadWriter[framework.ant.Params] = macroRW -} diff --git a/arch/src/main/scala/framework/top/configs/SimParam.scala b/arch/src/main/scala/framework/top/configs/SimParam.scala deleted file mode 100644 index 2dc810c81..000000000 --- a/arch/src/main/scala/framework/top/configs/SimParam.scala +++ /dev/null @@ -1,9 +0,0 @@ -package framework.top.configs - -import upickle.default._ - -case class SimParam(diffTest: Boolean = false) - -object SimParam { - implicit val rw: ReadWriter[SimParam] = macroRW -} diff --git a/arch/src/main/scala/sims/build.sc b/arch/src/main/scala/sims/build.sc new file mode 100644 index 000000000..9432aef2b --- /dev/null +++ b/arch/src/main/scala/sims/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.framework.{build => framework_build} + +val simRoot: os.Path = millSourcePath + +object library extends SbtModule { + override def millSourcePath = simRoot + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(framework_build.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0", ivy"com.lihaoyi::upickle:3.3.1") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-deprecation", "-feature", "-Ymacro-annotations") + override def sources = T.sources { + os.walk(simRoot).filter(path => path.ext == "scala" && !path.toString.contains("/out/")).map(PathRef(_)) + } +} diff --git a/arch/src/main/scala/sims/configs/SimParam.scala b/arch/src/main/scala/sims/configs/SimParam.scala new file mode 100644 index 000000000..92353a3c1 --- /dev/null +++ b/arch/src/main/scala/sims/configs/SimParam.scala @@ -0,0 +1,31 @@ +package sims.configs + +import buckyball.config.SimHardware + +import upickle.default._ + +case class MemoryParam( + lineBytes: Int, + clockHz: BigInt) + +object MemoryParam { implicit val rw: ReadWriter[MemoryParam] = macroRW } + +/** Differential debugging may delay completion; passive trace does not use this flag. */ +case class SimParam(diffTest: Boolean, memory: MemoryParam) + +object SimParam { + + def fromProto(p: SimHardware): SimParam = { + require(p.isInitialized, "Simulation configuration is incomplete") + val memory = p.getMemory + SimParam( + p.getDiffTest, + MemoryParam( + memory.getLineBytes, + BigInt(java.lang.Long.toUnsignedString(memory.getClockHz)) + ) + ) + } + + implicit val rw: ReadWriter[SimParam] = macroRW +} diff --git a/arch/src/main/scala/sims/core/CoreTarget.scala b/arch/src/main/scala/sims/core/CoreTarget.scala new file mode 100644 index 000000000..e31c0d89e --- /dev/null +++ b/arch/src/main/scala/sims/core/CoreTarget.scala @@ -0,0 +1,71 @@ +package sims.core + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} +import _root_.configs.GlobalConfig +import framework.core.clink.CLinkIO +import framework.device.dlink.BaseDevice +import framework.device.ddr.Bridge +import framework.device.peripherals.Controller +import framework.device.peripherals.bootrom.BootRomLines +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.LineResponse +import framework.lib.arbiter.RRArbiter +import framework.lib.queue.{Queue => ResponseQueue} +import sims.soc.SystemTarget + +/** A registered core supplies its CLink; the simulator owns its external environment. */ +abstract class CoreTarget(b: GlobalConfig) extends SystemTarget(b) { + def instantiateCore(): CLinkIO + def instantiate(b: GlobalConfig): Instance[BaseDevice] = + Instantiate(new CoreEnvironment(b, () => instantiateCore())) +} + +@instantiable +class CoreEnvironment(b: GlobalConfig, instantiateCore: () => CLinkIO) extends BaseDevice(b.device.ddr.axi) { + val Seq(corep) = b.device.cores + val core = instantiateCore() + require(core.p.harts == 1 && core.p.backing && core.p.requesters == 0) + require(core.c2c.isEmpty && core.banks.isEmpty) + require(b.device.ddr.clients == 1) + + val peripherals = Instantiate(new Controller(b.device, b.device.ddr.axi, corep.memory.memoryAccess.memory)) + + val decoder = Instantiate(new axi4.Decoder( + Seq( + (b.device.clint.base, b.device.clint.bytes), + (b.device.plic.base, b.device.plic.bytes), + (b.device.scu.baseAddress, b.device.scu.totalSizeBytes) + ), + b.device.ddr.axi + )) + + decoder.io.in <> core.memory.uncached + peripherals.io.axi <> decoder.io.matched + peripherals.io.hardwareInterrupt.sources := 0.U + core.interrupts := peripherals.io.hardwareInterrupt.harts + + val bridge = Instantiate(new Bridge(b.device.ddr)) + val memory = Instantiate(new axi4.Interconnect(axi4.InterconnectParam(b.device.ddr.axi, 3, None))) + memory.io.in(0) <> core.memory.ddr + memory.io.in(1) <> decoder.io.other + memory.io.in(2) <> bridge.io.axi + dlink.axi <> memory.io.out + + val rom = Instantiate(new BootRomLines(b.device.bootrom, corep.memory.home.memory.chi)) + val backing = core.memory.backing.get + val inRom = backing.request.bits.addr >= b.device.bootrom.base.U && + backing.request.bits.addr < (b.device.bootrom.base + b.device.bootrom.bytes).U + rom.io.request.valid := backing.request.valid && inRom + rom.io.request.bits := backing.request.bits + bridge.io.request(0).valid := backing.request.valid && !inRom + bridge.io.request(0).bits := backing.request.bits + backing.request.ready := Mux(inRom, rom.io.request.ready, bridge.io.request(0).ready) + val answers = Instantiate(new RRArbiter(new LineResponse(corep.memory.home.memory.chi), 2)) + answers.io.in(0) <> rom.io.response + answers.io.in(1) <> bridge.io.response(0) + val response = Instantiate(new ResponseQueue(new LineResponse(corep.memory.home.memory.chi), 1, false, false)) + response.io.enq <> answers.io.out + backing.response <> response.io.deq +} diff --git a/arch/src/main/scala/sims/hash/BankHash.scala b/arch/src/main/scala/sims/hash/BankHash.scala deleted file mode 100644 index 32d4482a5..000000000 --- a/arch/src/main/scala/sims/hash/BankHash.scala +++ /dev/null @@ -1,75 +0,0 @@ -package sims.hash - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import framework.top.GlobalConfig - -class BankHashWrite(val b: GlobalConfig) extends Bundle { - val addr = UInt(log2Ceil(b.memDomain.bankEntries).W) - val mask = Vec(b.memDomain.bankMaskLen, Bool()) - val data = UInt(b.memDomain.bankWidth.W) -} - -@instantiable -class BankHashMonitor(val b: GlobalConfig) extends Module { - require(b.memDomain.bankWidth == 128, "bank hash requires 128-bit bank rows") - require(b.memDomain.bankMaskLen == 16, "bank hash requires byte write masks") - - @public - val io = IO(new Bundle { - val bind = Input(Bool()) - val write = Flipped(Decoupled(new BankHashWrite(b))) - val statusHash = Output(UInt(32.W)) - }) - - val shadow = SyncReadMem(b.memDomain.bankEntries, UInt(128.W)) - val validRows = RegInit(VecInit(Seq.fill(b.memDomain.bankEntries)(false.B))) - val generation = RegInit(0.U(8.W)) - val statusHash = RegInit(0.U(32.W)) - - def rotateLeft(value: UInt, amount: Int): UInt = - Cat(value(31 - amount, 0), value(31, 32 - amount)) - - def rowHash(addr: UInt, row: UInt): UInt = { - val address = addr.pad(32) - val mixed = row(31, 0) ^ rotateLeft(row(63, 32), 7) ^ rotateLeft(row(95, 64), 13) ^ - rotateLeft(row(127, 96), 21) ^ rotateLeft(address, 11) - Mux(row.orR, mixed, 0.U) - } - - val writeValid = RegInit(false.B) - val writeBits = Reg(new BankHashWrite(b)) - val writeGeneration = Reg(UInt(generation.getWidth.W)) - val commit = writeValid && writeGeneration === generation && !io.bind - io.write.ready := !writeValid && !io.bind - val oldRow = Wire(UInt(128.W)) - val oldBytes = oldRow.asTypeOf(Vec(16, UInt(8.W))) - val writeBytes = writeBits.data.asTypeOf(Vec(16, UInt(8.W))) - val newBytes = VecInit((0 until 16).map(i => Mux(writeBits.mask(i), writeBytes(i), oldBytes(i)))) - val newRow = newBytes.asUInt - val nextHash = statusHash - rowHash(writeBits.addr, oldRow) + rowHash(writeBits.addr, newRow) - - val readData = - shadow.readWrite(Mux(commit, writeBits.addr, io.write.bits.addr), newRow, io.write.fire || commit, commit) - oldRow := Mux(validRows(writeBits.addr), readData, 0.U) - - when(io.write.fire) { - writeValid := true.B - writeBits := io.write.bits - writeGeneration := generation - } - - io.statusHash := statusHash - - when(io.bind) { - generation := generation + 1.U - statusHash := 0.U - writeValid := false.B - validRows.foreach(_ := false.B) - }.elsewhen(commit) { - validRows(writeBits.addr) := true.B - statusHash := nextHash - writeValid := false.B - } -} diff --git a/arch/src/main/scala/sims/p2e/P2ETop.scala b/arch/src/main/scala/sims/p2e/P2ETop.scala index d83f7e650..421c2838e 100644 --- a/arch/src/main/scala/sims/p2e/P2ETop.scala +++ b/arch/src/main/scala/sims/p2e/P2ETop.scala @@ -1,16 +1,22 @@ package sims.p2e +import chisel3.experimental.hierarchy.public + +import chisel3.experimental.hierarchy.instantiable + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import _root_.configs.GlobalConfig + import chisel3._ import chisel3.experimental.Analog import chisel3.util._ -import memcore.bus.axi4 -import memcore.memory.queue.Queue +import framework.ip.bus.axi4 +import framework.lib.queue.Queue import sims.soc.{SimSoc, SystemTarget} -/** A chip on the P2E board: the System over the 16-GiB DDR4 macro at 0x80000000. */ -abstract class P2ETarget(pb: String) extends SystemTarget(pb) { - override val dramBytes: BigInt = BigInt(16) << 30 -} +/** A design on the P2E board: the System over the 16-GiB DDR4 macro at 0x80000000. */ +abstract class P2ETarget(b: GlobalConfig) extends SystemTarget(b) /** Board pins of the P2E top; bebop's Tcl drives `sys_rstn`/`soc_hold` and reads the status pins. */ class P2ETopIO extends Bundle { @@ -46,7 +52,9 @@ class P2ETopIO extends Bundle { } /** AXI slave port of the DDR4 macro, as named on the macro (`_ddr4_s_axi_*`). */ -class Ddr4AxiSlave(idBits: Int) extends Bundle { +class Ddr4AxiSlave(port: Int) extends Bundle { + val idBits = XepicDdr4Dc1.portIdBits(port) + val awid = Input(UInt(idBits.W)) val awaddr = Input(UInt(64.W)) val awlen = Input(UInt(8.W)) @@ -86,10 +94,14 @@ class Ddr4AxiSlave(idBits: Int) extends Bundle { val rvalid = Output(Bool()) } +object XepicDdr4Dc1 { val portIdBits = Seq(11, 4) } + /** The board DDR4 controller; vcom replaces this stub with the `xepic_ddr4_dc1` netlist macro. */ +@instantiable class XepicDdr4Dc1 extends BlackBox { override def desiredName = "xepic_ddr4_dc1" + @public val io = IO(new Bundle { val sys_rstn = Input(Bool()) val c0_sys_clk_p = Input(Bool()) @@ -121,8 +133,10 @@ class XepicDdr4Dc1 extends BlackBox { val axi_clk = Input(Clock()) val c0_ddr4_ui_clk = Output(Clock()) // Flattened as the macro's `s0_ddr4_s_axi_*` and `s1_ddr4_s_axi_*` ports. - val s0_ddr4_s_axi = new Ddr4AxiSlave(11) - val s1_ddr4_s_axi = new Ddr4AxiSlave(4) + val s0_ddr4_s_axi = + new Ddr4AxiSlave(0) + val s1_ddr4_s_axi = + new Ddr4AxiSlave(1) }) } @@ -132,19 +146,26 @@ class XepicDdr4Dc1 extends BlackBox { * narrow bursts, so only lanes move: beat k of a burst starting at A uses lane (A + 16k)[4]. * CPU 0x80000000 is DDR offset 0, and the System's IDs widen to the macro's 11 bits. */ -class P2EDdrAdapter(system: axi4.Params, base: BigInt) extends Module { +@instantiable +class P2EDdrAdapter(b: GlobalConfig) extends Module { + val system = b.device.ddr.axi + val base = b.device.ddr.base + require(system.dataBits == 128, "P2E DDR adapter maps 16-byte beats onto the 32-byte macro bus") - require(system.idBits <= 11) + require(system.idBits <= XepicDdr4Dc1.portIdBits.head) + @public val io = IO(new Bundle { - val in = Flipped(new axi4.Port(system)) - val out = Flipped(new Ddr4AxiSlave(11)) + val in = Flipped(new axi4.Port(b.device.ddr.axi)) + val out = + Flipped(new Ddr4AxiSlave(0)) }) val lanes = 1 << system.idBits // Write lanes: W beats follow AW order, so queue each accepted burst's first lane. - val writeLane = Module(new Queue(Bool(), 8)) + val writeLane = Instantiate(new Queue(Bool(), 8, false, false)) + val writeOdd = RegInit(false.B) val writeUpper = writeLane.io.deq.bits ^ writeOdd io.out.awid := io.in.aw.bits.id @@ -222,10 +243,14 @@ class P2EDdrAdapter(system: axi4.Params, base: BigInt) extends Module { * DDR adapter and the System, all on `user_clk`. The SoC leaves reset only after calibration and * after the host releases `soc_hold`. */ -class P2EHarness(target: SystemTarget, diffTest: Boolean) extends RawModule { +@instantiable +class P2EHarness(b: GlobalConfig, target: SystemTarget) extends RawModule { + val diffTest = b.sim.diffTest + + @public val io = FlatIO(new P2ETopIO) - val ddr = Module(new XepicDdr4Dc1) + val ddr = Instantiate(new XepicDdr4Dc1) ddr.io.sys_rstn := io.sys_rstn ddr.io.c0_sys_clk_p := io.c0_sys_clk_p ddr.io.c0_sys_clk_n := io.c0_sys_clk_n @@ -257,8 +282,11 @@ class P2EHarness(target: SystemTarget, diffTest: Boolean) extends RawModule { val socReset = !io.sys_rstn || !ddr.io.c0_init_calib_complete || io.soc_hold withClockAndReset(io.user_clk, socReset) { - val soc = Module(new SimSoc(target, diffTest)) - val adapter = Module(new P2EDdrAdapter(soc.axiParams, target.dramBase)) + val soc = Instantiate(new SimSoc( + b, + target + )) + val adapter = Instantiate(new P2EDdrAdapter(b)) adapter.io.in <> soc.io.axi ddr.io.s0_ddr4_s_axi <> adapter.io.out } @@ -273,9 +301,16 @@ class P2EHarness(target: SystemTarget, diffTest: Boolean) extends RawModule { } /** The P2E top module bebop's vvac/vcom flow builds; `P2ETop.top` is the harness. */ -class P2ETop(target: SystemTarget, diffTest: Boolean) extends RawModule { - val io = IO(new P2ETopIO) - val top = Module(new P2EHarness(target, diffTest)) +class P2ETop(b: GlobalConfig, target: SystemTarget) extends RawModule { + val diffTest = b.sim.diffTest + + val io = IO(new P2ETopIO) + + val top = Instantiate(new P2EHarness( + b, + target + )) + // The DDR4 data pins stay on the macro inside `top`: vcom binds the macro's own pins, and a top // inout driving the macro is rejected. for ((name, port) <- io.elements) port match { diff --git a/arch/src/main/scala/sims/p2e/TargetConfig.scala b/arch/src/main/scala/sims/p2e/TargetConfig.scala index eb0812a48..6d6b2a51d 100644 --- a/arch/src/main/scala/sims/p2e/TargetConfig.scala +++ b/arch/src/main/scala/sims/p2e/TargetConfig.scala @@ -1,46 +1,28 @@ package sims.p2e import _root_.circt.stage.ChiselStage +import sims.soc.SystemTarget -//===----------------------------------------------------------------------===// -object Elaborate extends App { - if (args.isEmpty) { - println("Usage: Elaborate [--difftest] [firtool-opts...] -o=") - println("Example: Elaborate sims.p2e.PebbleP2ETarget") - sys.exit(1) - } - val targetClassName = args(0) - println(s"Elaborating P2ETop with target: $targetClassName") - - val target: P2ETarget = - try { - Class.forName(targetClassName).getDeclaredConstructor().newInstance() match { - case p2e: P2ETarget => p2e - case _ => - println(s"Error: $targetClassName is not a P2ETarget") - sys.exit(1) - } - } catch { - case e: ClassNotFoundException => - println(s"Error: target class not found: $targetClassName") - sys.exit(1) - } - - val rawFirtoolOpts = args.drop(1) +/** The design owns configuration decoding and concrete component types. */ +object Elaborate { - val outDir = rawFirtoolOpts.collectFirst { - case opt if opt.startsWith("-o=") => opt.stripPrefix("-o=") - }.getOrElse { - throw new Exception("missing -o= in firtool opts") + def main(args: Array[String]): Unit = { + require(args.length >= 6 && args(0) == "--config" && args(2) == "--target-class" && args(4) == "--target-dir") + val original = buckyball.config.Design.parseFrom(java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(args(1)))) + val options = args.drop(6) + val design = original.toBuilder.setSims(original.getSims.toBuilder + .setDiffTest(original.getSims.getDiffTest || options.contains("--difftest"))).build() + val target = Class.forName(args(3)).getConstructor(classOf[buckyball.config.Design]) + .newInstance(design).asInstanceOf[SystemTarget] + emit(target, args(5), options.filterNot(_ == "--difftest")) } - val firtoolOpts = rawFirtoolOpts.filterNot { opt => - opt == "--split-verilog" || opt == "--difftest" || opt.startsWith("-o=") + def emit(target: SystemTarget, directory: String, firtoolOpts: Array[String]): Unit = { + ChiselStage.emitSystemVerilogFile( + new P2ETop(target.b, target), + args = Array("--target-dir", directory, "--split-verilog"), + firtoolOpts = firtoolOpts + ) } - ChiselStage.emitSystemVerilogFile( - new P2ETop(target, diffTest = rawFirtoolOpts.contains("--difftest")), - firtoolOpts = firtoolOpts, - args = Array("--target-dir", outDir, "--split-verilog") - ) } diff --git a/arch/src/main/scala/sims/scu/SCU.scala b/arch/src/main/scala/sims/scu/SCU.scala deleted file mode 100644 index bf4f8587d..000000000 --- a/arch/src/main/scala/sims/scu/SCU.scala +++ /dev/null @@ -1,162 +0,0 @@ -package sims.scu - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util.HasBlackBoxInline - -/** - * SCU is a global multi-hart System Control Unit (see SystemControl). It - * provides a per-hart sub-region for UART output and simulation exit. Each hart accesses its own sub-region via - * `baseAddress + hartId * strideBytes`. - * - * Address layout (default values): - * baseAddress = 0x60000000 - * strideBytes = 0x40000 (256 KiB per hart) - * totalSizeBytes = 0x10000000 (256 MiB total, supports up to 1024 harts) - * maxHarts = 64 - * - * Within each hart's sub-region: - * +0x00000: simExit register (write triggers $finish with exit code) - * +0x20000: uartTx register (write outputs a byte via DPI-C) - * +0x20004: uartRx register (read pops a byte from host input) - * +0x20005: status register (read-only, bit0 rx-valid, bit5/6 tx-ready) - * +0x20006: ready register (read/write software-visible byte) - * - * @param baseAddress physical base address (must be aligned to totalSizeBytes) - * @param strideBytes bytes per hart (must be power of two) - * @param totalSizeBytes total address window (must be power of two, - * >= maxHarts * strideBytes) - * @param maxHarts number of hart sub-regions. Accesses targeting - * hartId >= maxHarts return an access error. - */ -case class SCUParams( - baseAddress: BigInt = BigInt("60000000", 16), - strideBytes: BigInt = BigInt("40000", 16), - totalSizeBytes: BigInt = BigInt("10000000", 16), - maxHarts: Int = 64) { - require( - strideBytes > 0 && (strideBytes & (strideBytes - 1)) == 0, - s"SCU strideBytes ($strideBytes) must be a positive power of two" - ) - require( - totalSizeBytes > 0 && (totalSizeBytes & (totalSizeBytes - 1)) == 0, - s"SCU totalSizeBytes ($totalSizeBytes) must be a positive power of two" - ) - require(maxHarts > 0, s"SCU maxHarts ($maxHarts) must be positive") - require( - BigInt(maxHarts) * strideBytes <= totalSizeBytes, - s"SCU maxHarts ($maxHarts) * strideBytes ($strideBytes) = " + - s"${BigInt(maxHarts) * strideBytes} exceeds totalSizeBytes ($totalSizeBytes)" - ) -} - -/** - * Single DPI-C bridge module for all harts. The hart_id is supplied as an - * input signal rather than being baked into the module name, so only one - * BlackBox is generated and DPI-C imports are not duplicated. - * - * Has separate hart_id inputs for uart and exit operations because different - * harts may write uart and exit simultaneously. - */ -@instantiable -class SCUWriteDPI extends BlackBox with HasBlackBoxInline { - override def desiredName = "SCUWriteDPI" - - @public - val io = IO(new Bundle { - val clock = Input(Clock()) - val reset = Input(Bool()) - val uart_hart_id = Input(UInt(32.W)) - val uart_valid = Input(Bool()) - val uart_data = Input(UInt(8.W)) - val exit_hart_id = Input(UInt(32.W)) - val exit_valid = Input(Bool()) - val exit_code = Input(UInt(32.W)) - }) - - setInline( - "SCUWriteDPI.v", - s""" - |import "DPI-C" context function void scu_uart_write(input int unsigned hart_id, input int unsigned ch); - |import "DPI-C" context function void scu_sim_exit(input int unsigned hart_id, input int unsigned code); - | - |module SCUWriteDPI( - | input clock, - | input reset, - | input [31:0] uart_hart_id, - | input uart_valid, - | input [7:0] uart_data, - | input [31:0] exit_hart_id, - | input exit_valid, - | input [31:0] exit_code - |); - | always @(posedge clock) begin - | if (!reset) begin - | if (uart_valid) begin - | scu_uart_write(uart_hart_id, {24'h0, uart_data}); - | end - | - | if (exit_valid) begin - | scu_sim_exit(exit_hart_id, exit_code); - | end - | end - | end - |endmodule - """.stripMargin - ) -} - -@instantiable -class SCUReadDPI extends BlackBox with HasBlackBoxInline { - override def desiredName = "SCUReadDPI" - - @public - val io = IO(new Bundle { - val clock = Input(Clock()) - val reset = Input(Bool()) - val hart_id = Input(UInt(32.W)) - val enable = Input(Bool()) - val pop = Input(Bool()) - val rx_valid = Output(Bool()) - val rx_data = Output(UInt(8.W)) - }) - - setInline( - "SCUReadDPI.v", - s""" - |import "DPI-C" context function void scu_uart_rx_sample( - | input int unsigned hart_id, - | input int unsigned pop, - | output int unsigned valid, - | output int unsigned data - |); - | - |module SCUReadDPI( - | input clock, - | input reset, - | input [31:0] hart_id, - | input enable, - | input pop, - | output reg rx_valid, - | output reg [7:0] rx_data - |); - | integer valid; - | integer data; - | always @(posedge clock) begin - | if (reset) begin - | rx_valid <= 1'b0; - | rx_data <= 8'h0; - | end else if (enable) begin - | scu_uart_rx_sample(hart_id, {31'h0, pop}, valid, data); - | if (pop) begin - | rx_valid <= 1'b0; - | end else begin - | rx_valid <= (valid != 0); - | rx_data <= data[7:0]; - | end - | end - | end - |endmodule - """.stripMargin - ) -} diff --git a/arch/src/main/scala/sims/scu/SystemControl.scala b/arch/src/main/scala/sims/scu/SystemControl.scala deleted file mode 100644 index b6316f324..000000000 --- a/arch/src/main/scala/sims/scu/SystemControl.scala +++ /dev/null @@ -1,114 +0,0 @@ -package sims.scu - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import chisel3.util._ -import memcore.memory.cpu.{CpuMemParams, UncachedRequest, UncachedResponse} - -/** - * SCU on the System device ports, with the register layout documented on SCUParams. The hart sub-region is - * selected by address, so software keeps using `base + hartId * stride` regardless of port. - * Data is right-aligned at `addr`, as on every uncached CPU port. - */ -@instantiable -class SystemControl(ports: Int, p: CpuMemParams, params: SCUParams = SCUParams()) extends Module { - val strideBits = log2Ceil(params.strideBytes) - - @public - val io = IO(new Bundle { - val request = Vec(ports, Flipped(Decoupled(new UncachedRequest(p)))) - val response = Vec(ports, Decoupled(new UncachedResponse(p))) - val hartIds = Input(Vec(ports, UInt(32.W))) - val failure = Input(Vec(ports, Valid(new framework.memdomain.frontend.mem.dma.DmaStatus))) - val workDrained = Input(Vec(ports, Bool())) - }) - - val faulted = SystemControl.observe(io.failure.toSeq, io.hartIds.toSeq) - - for (i <- 0 until ports) { - val request = io.request(i) - val response = io.response(i) - val write: Instance[SCUWriteDPI] = Instantiate(new SCUWriteDPI) - val read: Instance[SCUReadDPI] = Instantiate(new SCUReadDPI) - val ready = RegInit(0.U(8.W)) - - val pending = RegInit(false.B) - val tag = Reg(UInt(p.tagBits.W)) - val error = Reg(Bool()) - val data = Reg(UInt(64.W)) - val status = Reg(Bool()) - - val offset = request.bits.addr - params.baseAddress.U - val hart = offset(31, strideBits) - val register = offset(strideBits - 1, 0) - val inside = request.bits.addr >= params.baseAddress.U && - request.bits.addr < (params.baseAddress + params.totalSizeBytes).U && hart < params.maxHarts.U - val fire = request.fire - val wr = request.bits.write - val exit = inside && register === 0x00000.U && wr - val uartTx = inside && register === 0x20000.U && wr - val uartRx = inside && register === 0x20004.U && !wr - val uartSt = inside && register === 0x20005.U && !wr - val readyReg = inside && register === 0x20006.U - - write.io.clock := clock - write.io.reset := reset.asBool - write.io.exit_hart_id := hart - write.io.exit_valid := fire && exit && !faulted - write.io.exit_code := request.bits.data(31, 0) - write.io.uart_hart_id := hart - write.io.uart_valid := fire && uartTx - write.io.uart_data := request.bits.data(7, 0) - - // The DPI samples on this edge; status reads answer with the fresh sample next cycle. - read.io.clock := clock - read.io.reset := reset.asBool - read.io.hart_id := hart - read.io.enable := fire && (uartRx || uartSt) - read.io.pop := fire && uartRx - - request.ready := !pending && (!request.valid || !exit || io.workDrained(i)) - when(fire) { - pending := true.B - tag := request.bits.tag - error := !(exit || uartTx || uartRx || uartSt || readyReg) - status := uartSt - data := Mux(uartRx, read.io.rx_data, Mux(readyReg && !wr, ready, 0.U)) - when(readyReg && wr)(ready := request.bits.data(7, 0)) - } - - response.valid := pending - response.bits.tag := tag - response.bits.error := error - response.bits.data := Mux(status, "h60".U | read.io.rx_valid.asUInt, data) - when(response.fire)(pending := false.B) - } -} - -object SystemControl { - - def observe(failures: Seq[ValidIO[framework.memdomain.frontend.mem.dma.DmaStatus]], hartIds: Seq[UInt]): Bool = { - val failed = RegInit(false.B) - val pending = VecInit(failures.map(_.valid)) - val first = PriorityEncoder(pending) - val exit = Instantiate(new SCUWriteDPI) - exit.io.clock := Module.clock - exit.io.reset := Module.reset.asBool - exit.io.uart_hart_id := 0.U - exit.io.uart_valid := false.B - exit.io.uart_data := 0.U - exit.io.exit_hart_id := VecInit(hartIds)(first) - exit.io.exit_valid := pending.asUInt.orR && !failed - exit.io.exit_code := 0x200.U | VecInit(failures.map(_.bits.error))(first) - when(pending.asUInt.orR)(failed := true.B) - for ((failure, index) <- failures.zipWithIndex) { - when(failure.valid && !failed && first === index.U) { - printf( - cf"[SCU] hart ${hartIds(index)} NPU failure: error=${failure.bits.error} address=0x${failure.bits.address}%x\n" - ) - } - } - pending.asUInt.orR || failed - } - -} diff --git a/arch/src/main/scala/sims/soc/SimSoc.scala b/arch/src/main/scala/sims/soc/SimSoc.scala index e4f37085f..e9bafb0b4 100644 --- a/arch/src/main/scala/sims/soc/SimSoc.scala +++ b/arch/src/main/scala/sims/soc/SimSoc.scala @@ -1,99 +1,38 @@ package sims.soc -import chisel3._ -import chisel3.experimental.hierarchy.Instance -import framework.system.{System, SystemParams} -import framework.system.configloader.{AntTileCore, ChipLoader, RocketTileCore} -import framework.system.device.{BootRom, DeviceParams} -import memcore.bus.axi4 -import memcore.bus.chi.{Params => ChiParams} -import memcore.bus.chi.rnf.RnfParams -import memcore.memory.cache.configs.CacheParams -import memcore.memory.coherence.configs.CoherenceParams -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion} -import memcore.memory.ddr.{Params => DdrParams} -import memcore.memory.interlock.{Params => TrackingParams} -import sims.scu.{SCUParams} - -/** A chip on the explicit System: one PB plus the platform's DRAM window. */ -abstract class SystemTarget(val pb: String) { - def instantiate(p: SystemParams): Instance[System] - val dramBase: BigInt = BigInt("80000000", 16) - val dramBytes: BigInt = BigInt(512) << 20 -} - -/** - * The explicit System with its SCU and NPU-fault abort. Every platform (Verilator, P2E) supplies - * the DRAM behind `io.axi` and nothing else. - */ -class SimSoc(target: SystemTarget, diffTest: Boolean, mainOnly: Boolean = false) extends Module { - - val loaded = ChipLoader.load(target.pb) - - val topology = loaded.copy(tiles = loaded.tiles.filter(tile => !mainOnly || tile.main).map(tile => - tile.copy(cores = tile.cores.map { - case RocketTileCore(cpu, Some(config)) => - RocketTileCore(cpu, Some(config.copy(sim = config.sim.copy(diffTest = diffTest)))) - case AntTileCore(local, config) => - AntTileCore(local, config.copy(sim = config.sim.copy(diffTest = diffTest))) - case core => core - }) - )) +import chisel3.experimental.hierarchy.public - val scu = SCUParams() +import chisel3.experimental.hierarchy.instantiable - val chi = ChiParams() +import _root_.configs.GlobalConfig - // Per-tile template (System sets each tile's agent count). Tags and directories are still - // registers in these IPs; sizes stay modest until they move to SRAM. - val memory = CoherenceParams( - chi, - CacheParams(chi.addressBits, 64, 64, 4, 8, 4, 2), - agents = 2, - mshrEntries = 8, - homeId = math.max(64, 2 * topology.tiles.map(_.hartIds.size).sum + 1) - ) - - // Every hart starts in the BootROM, which validates mhartid and jumps to the DRAM entry. - val hartIdList = topology.tiles.flatMap(_.hartIds) - require(hartIdList.sorted == hartIdList.indices, "BootROM validates harts as 0 until the hart count") - val bootrom = BootRom(BigInt("10000", 16), BigInt("10000", 16), hartIdList.size, target.dramBase) - val devices = DeviceParams(bootrom = Some(bootrom), scu = scu) +import chisel3._ +import chisel3.experimental.hierarchy.Instance +import framework.device.dlink.BaseDevice +import framework.device.configs.DeviceParam +import framework.ip.bus.axi4 +import sims.configs.SimParam + +/** A design supplies its parameters and explicitly instantiates its DLink implementation. */ +abstract class SystemTarget(val b: GlobalConfig) { + def instantiate(b: GlobalConfig): Instance[BaseDevice] + val dramBase: BigInt = b.device.ddr.base + val dramBytes: BigInt = b.device.ddr.bytes +} - val regions = Seq( - PhysicalRegion(target.dramBase, target.dramBytes, true, true, true, true, true, true), - PhysicalRegion(bootrom.base, bootrom.bytes, true, true, true, false, false, true), - PhysicalRegion(devices.clint.base, devices.clint.bytes, false, false, true, true, false, false), - PhysicalRegion(devices.plic.base, devices.plic.bytes, false, false, true, true, false, false), - PhysicalRegion(scu.baseAddress, scu.totalSizeBytes, false, false, true, true, false, false) - ) +@instantiable +class SimSoc(b: GlobalConfig, target: SystemTarget) extends Module { + val diffTest = b.sim.diffTest - val slotsPerClient = 16 + val p = b.device - val ddr = DdrParams( - chi, - clients = 1, - slotsPerClient = slotsPerClient, - dataBits = 128, - idBits = 4 - ) + @public + val io = IO(new Bundle { - val axiParams: axi4.Params = ddr.axi + val axi = new axi4.Port(p.ddr.axi) - val io = IO(new Bundle { - val axi = new axi4.Port(axiParams) }) - val system: Instance[System] = target.instantiate(SystemParams( - topology, - 36, - memory, - RnfParams(chi, cacheLines = 64, banks = 2, homeId = memory.homeId), - regions, - TrackingParams(addressBits = chi.addressBits), - ddr, - devices - )) - + val system: Instance[BaseDevice] = target.instantiate(b) io.axi <> system.dlink.axi } diff --git a/arch/src/main/scala/sims/verilator/BBsimDRAM.scala b/arch/src/main/scala/sims/verilator/BBsimDRAM.scala index 5d8c1f9b1..6b0f9277a 100644 --- a/arch/src/main/scala/sims/verilator/BBsimDRAM.scala +++ b/arch/src/main/scala/sims/verilator/BBsimDRAM.scala @@ -1,48 +1,49 @@ package sims.verilator +import _root_.configs.GlobalConfig + import chisel3._ import chisel3.experimental.hierarchy.{instantiable, public} import chisel3.experimental.IntParam import chisel3.util.HasBlackBoxInline -import memcore.bus.axi4.{Params => AxiParams, Port => AxiPort} +import framework.ip.bus.axi4.{Params => AxiParams, Port => AxiPort} @instantiable -class BBSimDRAM( - memSize: BigInt, - lineSize: Int, - clockFreqHz: BigInt, - memBase: BigInt, - params: AxiParams, - chipId: Int) +class BBSimDRAM(b: GlobalConfig, deviceId: Int) extends BlackBox( Map( - "MEM_SIZE" -> IntParam(memSize), - "LINE_SIZE" -> IntParam(lineSize), - "ADDR_BITS" -> IntParam(params.addressBits), - "DATA_BITS" -> IntParam(params.dataBits), - "ID_BITS" -> IntParam(params.idBits), - "CLOCK_HZ" -> IntParam(clockFreqHz), - "MEM_BASE" -> IntParam(memBase), - "CHIP_ID" -> IntParam(chipId) + "MEM_SIZE" -> IntParam(b.device.ddr.bytes), + "LINE_SIZE" -> IntParam(b.sim.memory.lineBytes), + "ADDR_BITS" -> IntParam(b.device.ddr.axi.addressBits), + "DATA_BITS" -> IntParam(b.device.ddr.axi.dataBits), + "ID_BITS" -> IntParam(b.device.ddr.axi.idBits), + "CLOCK_HZ" -> IntParam(b.sim.memory.clockHz), + "MEM_BASE" -> IntParam(b.device.ddr.base), + "DEVICE_ID" -> IntParam(deviceId) ) ) with HasBlackBoxInline { + val memSize = b.device.ddr.bytes + val lineSize = b.sim.memory.lineBytes + val clockFreqHz = b.sim.memory.clockHz + val memBase = b.device.ddr.base + val axiParams = b.device.ddr.axi @public val io = IO(new Bundle { val clock = Input(Clock()) val reset = Input(Reset()) - val axi = Flipped(new AxiPort(params)) + val axi = Flipped(new AxiPort(b.device.ddr.axi)) }) - require(Set(64, 128).contains(params.dataBits), "BBSimDRAM supports 64 or 128 bit AXI beats") + require(Set(64, 128).contains(axiParams.dataBits), "BBSimDRAM supports 64 or 128 bit AXI beats") setInline( "BBSimDRAM.v", """ |import "DPI-C" function chandle bbsim_memory_init |( - | input int chip_id, + | input int device_id, | input longint mem_size, | input longint word_size, | input longint line_size, @@ -98,7 +99,7 @@ class BBSimDRAM( | CLOCK_HZ = 100000, | STRB_BITS = DATA_BITS/8, | MEM_BASE = 0, - | CHIP_ID = 0)( + | DEVICE_ID = 0)( | input clock, | input reset, | output axi_aw_ready, @@ -191,7 +192,7 @@ class BBSimDRAM( | | always @(posedge clock) begin | if (!initialized) begin - | channel = bbsim_memory_init(CHIP_ID, MEM_SIZE, WORD_SIZE, LINE_SIZE, ID_BITS, CLOCK_HZ, MEM_BASE); + | channel = bbsim_memory_init(DEVICE_ID, MEM_SIZE, WORD_SIZE, LINE_SIZE, ID_BITS, CLOCK_HZ, MEM_BASE); | initialized = 1'b1; | end | diff --git a/arch/src/main/scala/sims/verilator/BdbClkDPI.scala b/arch/src/main/scala/sims/verilator/BdbClkDPI.scala index 65cf198dc..685d76c96 100644 --- a/arch/src/main/scala/sims/verilator/BdbClkDPI.scala +++ b/arch/src/main/scala/sims/verilator/BdbClkDPI.scala @@ -25,6 +25,7 @@ class BdbClkDPI extends BlackBox with HasBlackBoxInline { | input wire clock, | input wire reset |); + |`ifndef BEBOP_PARALLEL | reg [63:0] cnt; | always @(posedge clock) begin | if (reset) begin @@ -35,6 +36,7 @@ class BdbClkDPI extends BlackBox with HasBlackBoxInline { | cnt <= cnt + 64'd1; | end | end + |`endif |endmodule """.stripMargin ) diff --git a/arch/src/main/scala/sims/verilator/SystemHarness.scala b/arch/src/main/scala/sims/verilator/SystemHarness.scala index 7e341971f..c024ce447 100644 --- a/arch/src/main/scala/sims/verilator/SystemHarness.scala +++ b/arch/src/main/scala/sims/verilator/SystemHarness.scala @@ -1,22 +1,33 @@ package sims.verilator +import _root_.configs.GlobalConfig + import chisel3._ import chisel3.experimental.hierarchy.Instantiate import chisel3.experimental.hierarchy.Instance import sims.soc.{SimSoc, SystemTarget} /** Verilator top for the explicit System. Keeps the BBSimHarness name bebop builds against. */ -class SystemHarness(target: SystemTarget, diffTest: Boolean, mainOnly: Boolean = false) extends Module { +class SystemHarness(b: GlobalConfig, target: SystemTarget) extends Module { + val diffTest = b.sim.diffTest + override def desiredName = "BBSimHarness" val bdbClkDpi: Instance[BdbClkDPI] = Instantiate(new BdbClkDPI) bdbClkDpi.io.clock := clock bdbClkDpi.io.reset := reset.asBool - val soc = Module(new SimSoc(target, diffTest, mainOnly)) + val soc = Instantiate(new SimSoc( + b, + target + )) val dram: Instance[BBSimDRAM] = - Instantiate(new BBSimDRAM(target.dramBytes, 64, 1000, target.dramBase, soc.axiParams, chipId = 0)) + Instantiate(new BBSimDRAM( + b, + 0 + )) + dram.io.clock := clock dram.io.reset := reset dram.io.axi <> soc.io.axi diff --git a/arch/src/main/scala/sims/verilator/TargetConfig.scala b/arch/src/main/scala/sims/verilator/TargetConfig.scala index 06d2affb4..62f6a0d3a 100644 --- a/arch/src/main/scala/sims/verilator/TargetConfig.scala +++ b/arch/src/main/scala/sims/verilator/TargetConfig.scala @@ -2,55 +2,27 @@ package sims.verilator import _root_.circt.stage.ChiselStage import sims.soc.SystemTarget -import framework.system.configloader.ChipLoader -import java.nio.file.{Files, Paths} -//===----------------------------------------------------------------------===// -object Elaborate extends App { - if (args.isEmpty) { - println("Usage: Elaborate [firtool-opts...]") - println("Example: Elaborate sims.verilator.GobanVerilatorTarget") - sys.exit(1) - } - val targetClassName = args(0) - println(s"Elaborating BBSimHarness with target: $targetClassName") - - val target: SystemTarget = - try { - Class.forName(targetClassName).getDeclaredConstructor().newInstance() match { - case system: SystemTarget => system - case _ => - println(s"Error: $targetClassName is not a SystemTarget") - sys.exit(1) - } - } catch { - case e: ClassNotFoundException => - println(s"Error: target class not found: $targetClassName") - sys.exit(1) - } +/** The design owns configuration decoding and concrete component types. */ +object Elaborate { - val rawFirtoolOpts = args.drop(1) - val mainOnly = rawFirtoolOpts.contains("--main-only") - - val outDir = rawFirtoolOpts.collectFirst { - case opt if opt.startsWith("-o=") => opt.stripPrefix("-o=") - }.getOrElse { - throw new Exception("missing -o= in firtool opts") + def main(args: Array[String]): Unit = { + require(args.length >= 6 && args(0) == "--config" && args(2) == "--target-class" && args(4) == "--target-dir") + val original = buckyball.config.Design.parseFrom(java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(args(1)))) + val options = args.drop(6) + val design = original.toBuilder.setSims(original.getSims.toBuilder + .setDiffTest(original.getSims.getDiffTest || options.contains("--difftest"))).build() + val target = Class.forName(args(3)).getConstructor(classOf[buckyball.config.Design]) + .newInstance(design).asInstanceOf[SystemTarget] + emit(target, args(5), options.filterNot(_ == "--difftest")) } - val firtoolOpts = rawFirtoolOpts.filterNot { opt => - opt == "--split-verilog" || opt == "--difftest" || opt == "--main-only" || opt.startsWith("-o=") + def emit(target: SystemTarget, directory: String, firtoolOpts: Array[String]): Unit = { + ChiselStage.emitSystemVerilogFile( + new SystemHarness(target.b, target), + args = Array("--target-dir", directory, "--split-verilog"), + firtoolOpts = firtoolOpts + ) } - ChiselStage.emitSystemVerilogFile( - new SystemHarness(target, diffTest = rawFirtoolOpts.contains("--difftest"), mainOnly = mainOnly), - firtoolOpts = firtoolOpts, - args = Array("--target-dir", outDir, "--split-verilog") - ) - val hierarchy = Paths.get(outDir).resolve("hierarchy.vlt") - if (!mainOnly && ChipLoader.load(target.pb).tiles.exists(!_.main)) { - Files.writeString(hierarchy, "`verilator_config\nhier_block -module \"ComputeTile\"\n") - } else { - Files.deleteIfExists(hierarchy) - } } diff --git a/arch/thirdparty/build.sc b/arch/thirdparty/build.sc new file mode 100644 index 000000000..d409c958a --- /dev/null +++ b/arch/thirdparty/build.sc @@ -0,0 +1,101 @@ +import mill._ +import mill.scalalib._ +val archRoot: os.Path = millSourcePath / os.up + +object protoJava extends JavaModule { + + def protoDir = T { + archRoot / os.up / "bbdev" / "api" / "steps" / "config" / "scripts" / "proto" + } + + def protoSources = T.sources { + Seq(PathRef(protoDir() / "design.proto"), PathRef(protoDir() / "hardware.proto")) ++ + Seq("balls", "cores").flatMap(kind => + os.walk(archRoot / os.up / "examples" / "components" / kind).filter(p => p.ext == "proto" && !p.toString.contains("/thirdparty/") && !p.toString.contains("/out/")).map(PathRef(_))) + } + + def generatedSources = T { + val dir = protoDir() + val protos = protoSources().map(_.path) + val command = Seq("protoc", s"--java_out=${T.dest}") ++ + protos.map(path => s"-I${path / os.up}").distinct ++ protos.map(_.toString) + os.proc(command).call() + Seq(PathRef(T.dest)) + } + + override def ivyDeps = Agg(ivy"com.google.protobuf:protobuf-java:4.35.1") +} + +object hardfloat extends SbtModule { + override def millSourcePath = + archRoot / "thirdparty" / "berkeley-hardfloat" + override def scalaVersion = "2.13.16" + override def scalacOptions = Seq("-Ymacro-annotations") + + // Override sources to match build.sbt behavior + override def sources = T.sources { + super.sources() ++ Seq( + PathRef(millSourcePath / "hardfloat" / "src" / "main" / "scala") + ) + } + + override def ivyDeps = Agg( + ivy"org.chipsalliance::chisel:6.7.0" + ) + + override def scalacPluginIvyDeps = Agg( + ivy"org.chipsalliance:::chisel-plugin:6.7.0" + ) + +} + +object midas_target_utils extends SbtModule { + override def millSourcePath = + archRoot / os.up / "thirdparty" / "firesim" / "sim" / "midas" / "targetutils" + override def scalaVersion = "2.13.16" + + override def ivyDeps = Agg( + ivy"org.chipsalliance::chisel:6.7.0" + ) + + override def scalacPluginIvyDeps = Agg( + ivy"org.chipsalliance:::chisel-plugin:6.7.0" + ) + +} + +object gemmini extends SbtModule { + override def millSourcePath = + archRoot / "thirdparty" / "gemmini" + override def scalaVersion = "2.13.16" + override def scalacOptions = Seq("-Ymacro-annotations") + + override def sources = T.sources { + Seq( + "Arithmetic", + "Dataflow", + "MeshWithDelays", + "Mesh", + "Tile", + "PE", + "Util", + "Transposer", + "TagQueue", + "Pipeline", + "Shifter", + "SyncMem" + ) + .map(name => PathRef(millSourcePath / "src" / "main" / "scala" / "gemmini" / s"$name.scala")) + } + + override def moduleDeps = Seq(hardfloat) + + override def ivyDeps = Agg( + ivy"org.chipsalliance::chisel:6.7.0" + ) + + override def scalacPluginIvyDeps = Agg( + ivy"org.chipsalliance:::chisel-plugin:6.7.0" + ) + +} diff --git a/arch/thirdparty/rocket-chip b/arch/thirdparty/rocket-chip deleted file mode 160000 index a4b4643bb..000000000 --- a/arch/thirdparty/rocket-chip +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a4b4643bbcbe6d6deb6cfa8b95954cbf518a7073 diff --git a/arch/uvm.toml b/arch/uvm.toml index caeae8009..25a059dcd 100644 --- a/arch/uvm.toml +++ b/arch/uvm.toml @@ -1,49 +1,259 @@ -ips = ["ant", "tls", "tss", "preflight", "ddr", "interlock", "memdomain", "root_core", "root_tile", "fetch", "mmu", "cpu_mem", "axis", "bank", "cache", "coherence", "chi", "root_chip", "mesh_shm", "rvv"] - [paths] -ant = "arch/src/main/scala/framework/system/core/ant" -tls = "arch/src/main/scala/framework/mem-core/tls" -tss = "arch/src/main/scala/framework/mem-core/tss" -preflight = "arch/src/main/scala/framework/mem-core/preflight" -ddr = "arch/src/main/scala/framework/mem-core/ddr" -interlock = "arch/src/main/scala/framework/mem-core/interlock" -memdomain = "arch/src/main/scala/framework/memdomain/verification" -axis = "arch/src/main/scala/framework/mem-core/axis" -bank = "arch/src/main/scala/framework/mem-core/bank" -cache = "arch/src/main/scala/framework/mem-core/cache" -coherence = "arch/src/main/scala/framework/mem-core/coherence" -mesh_shm = "arch/src/main/scala/framework/mem-core/mesh_shm" -rvv = "arch/src/main/scala/framework/rvv" -chi = "arch/src/main/scala/framework/mem-core/chi" -root_chip = "arch/src/main/scala/framework/root/root-chip" -cpu_mem = "arch/src/main/scala/framework/mem-core/cpu_mem" -mmu = "arch/src/main/scala/framework/mem-core/mmu" -fetch = "arch/src/main/scala/framework/mem-core/fetch" -root_tile = "arch/src/main/scala/framework/root/root-tile" - -root_core = "arch/src/main/scala/framework/root/root-core" - -[build.root_core] -directory = "arch" -module = "root_core" - -[build.root_tile] -directory = "arch" -module = "root_tile" - -[build.memdomain] -directory = "arch" -module = "memdomain_ack" -arguments = ["--chip", "{chip}", "--targets", "{targets}"] - -[build.tls] -directory = "arch" -module = "tls" +tbus_router="arch/src/main/scala/framework/core/tbus/src/main/scala/router" +core_cache="arch/src/main/scala/framework/core/memory/access/src/main/scala/cache" +home="arch/src/main/scala/framework/core/memory/home" +admission="arch/src/main/scala/framework/core/controller/admission" +tol1 = "arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/tol1" +toddr = "arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr" +tomem = "arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem" +prm = "arch/src/main/scala/framework/acc/memdomain/src/main/scala/prm" +bank_router = "arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router" +global_rob = "arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global" +boot = "arch/src/main/scala/framework/acc/frontend/src/main/scala/boot" +bat = "arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/bat" +scoreboard = "arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/scoreboard" +tss = "arch/src/main/scala/framework/core/tss" +preflight = "arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight" +ddr = "arch/src/main/scala/framework/device/ddr" +interlock = "arch/src/main/scala/framework/ip/interlock" +axis = "arch/src/main/scala/framework/ip/axis" +axi4 = "arch/src/main/scala/framework/ip/axi4" +bank = "arch/src/main/scala/framework/ip/bank" +spm = "arch/src/main/scala/framework/ip/spm" +cache = "arch/src/main/scala/framework/ip/cache" +coherence = "arch/src/main/scala/framework/ip/coherence" +chi = "arch/src/main/scala/framework/ip/chi" +mem_access = "arch/src/main/scala/framework/core/memory/access" +mmu = "arch/src/main/scala/framework/core/memory/access/src/main/scala/mmu" +fetch = "arch/src/main/scala/framework/core/execution/fetch" + + +instruction = "arch/src/main/scala/framework/core/execution/instruction" +fpu = "arch/src/main/scala/framework/core/execution/fpu" +muldiv = "arch/src/main/scala/framework/core/execution/muldiv" +privileged = "arch/src/main/scala/framework/core/execution/privileged" +integer = "arch/src/main/scala/framework/core/execution/integer" +register_file = "arch/src/main/scala/framework/core/execution/register_file" [build.tss] -directory = "arch" -module = "tss" +directory = "arch/src/main/scala/framework" +module = "core.tss" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + +[build.global_rob] +directory = "arch/src/main/scala/framework" +module = "acc.frontend.scheduler.rob.global" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + + +[build.boot] +directory = "arch/src/main/scala/framework" +module = "acc.frontend.boot" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + + +[build.bank_router] +directory = "arch/src/main/scala/framework" +module = "acc.memdomain.bank_router" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + +[build.axis] +directory = "arch/src/main/scala/framework" +module = "ip.axis" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + +[build.axi4] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "ip.axi4" + +[build.chi] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "ip.chi" + +[build.bank] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "ip.bank" + +[build.cache] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "ip.cache" + +[build.coherence] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "ip.coherence" + +[build.interlock] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "ip.interlock" + +[build.spm] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "ip.spm" + +[build.ddr] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "device.ddr" + +[build.mmu] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "core.memory.access.mmu" + +[build.fetch] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "core.execution.fetch" + +[build.mem_access] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "core.memory.access" + +[build.preflight] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "core.memory.access.preflight" + + +[build.bat] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "acc.frontend.scheduler.bat" + +[build.scoreboard] +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] +directory = "arch/src/main/scala/framework" +module = "acc.frontend.scheduler.scoreboard" + + +[build.instruction] +directory = "arch/src/main/scala/framework" +module = "core.execution.instruction" +arguments = ["--config", "{config}"] + +[build.fpu] +directory = "arch/src/main/scala/framework" +module = "core.execution.fpu" +arguments = ["--config", "{config}"] + +[build.muldiv] +directory = "arch/src/main/scala/framework" +module = "core.execution.muldiv" +arguments = ["--config", "{config}"] + +[build.privileged] +directory = "arch/src/main/scala/framework" +module = "core.execution.privileged" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + +[build.integer] +directory = "arch/src/main/scala/framework" +module = "core.execution.integer" +arguments = ["--config", "{config}"] + +[build.register_file] +directory = "arch/src/main/scala/framework" +module = "core.execution.register_file" +arguments = ["--config", "{config}"] + +[modules.frontend] +level = "domain" +path = "arch/src/main/scala/framework/acc/frontend" + +[modules.frontend.build] +directory = "arch/src/main/scala/framework" +module = "acc.frontend" +arguments = ["--config", "{repo}/examples/designs/{design}/output/generated/design.pb"] + +[modules.memdomain] +level = "domain" +path = "arch/src/main/scala/framework/acc/memdomain" + +[modules.memdomain.build] +directory = "arch/src/main/scala/framework" +module = "acc.memdomain" +arguments = ["--design", "{design}"] + +[modules.gpdomain] +level = "domain" +path = "arch/src/main/scala/framework/acc/gpdomain" + +[modules.gpdomain.build] +directory = "arch/src/main/scala/framework" +module = "acc.gpdomain" +arguments = ["--config", "{repo}/examples/designs/{design}/output/generated/design.pb"] + +[modules.balldomain] +level = "domain" +path = "arch/src/main/scala/framework/acc/balldomain" + +[modules.balldomain.build] +runtime = [{ directory = "examples", module = "library" }] +directory = "arch/src/main/scala/framework" +module = "acc.balldomain" +arguments = ["--config", "{repo}/examples/designs/{design}/output/generated/design.pb"] + +[modules.ant] +level = "thread" +path = "arch/src/main/scala/framework/thread/ant" + +[modules.ant.build] +directory = "arch/src/main/scala/framework" +module = "thread.ant" +arguments = ["--config", "{repo}/examples/designs/{design}/output/generated/design.pb", "--core", "1"] + +[modules.mesh_shm] +level = "subsystem" +path = "arch/src/main/scala/framework/ip/mesh_shm" + +[modules.mesh_shm.build] +directory = "arch/src/main/scala/framework" +module = "ip.mesh_shm" + +[build.prm] +directory = "arch/src/main/scala/framework" +module = "acc.memdomain.prm" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + +[build.tol1] +directory = "arch/src/main/scala/framework" +module = "acc.memdomain.tomem.tol1" +arguments = ["--config", "{config}", "--output-dir", "{rtl}"] + +[build.toddr] +directory="arch/src/main/scala/framework" +module="acc.memdomain.tomem.toddr" +arguments=["--config","{config}","--output-dir","{rtl}"] + +[build.tomem] +directory="arch/src/main/scala/framework" +module="acc.memdomain.tomem" +arguments=["--config","{config}","--output-dir","{rtl}"] + +[build.admission] +directory="arch/src/main/scala/framework" +module="core.controller.admission" +arguments=["--config","{config}","--output-dir","{rtl}"] + +[build.home] +directory="arch/src/main/scala/framework" +module="core.memory.home" +arguments=["--config","{config}","--output-dir","{rtl}"] + +[build.core_cache] +directory="arch/src/main/scala/framework" +module="core.memory.access.cache" +arguments=["--config","{config}","--output-dir","{rtl}"] -[build.ant] -directory = "arch" -module = "ant" +[build.tbus_router] +directory="arch/src/main/scala/framework" +module="core.tbus.router" +arguments=["--config","{config}","--output-dir","{rtl}"] diff --git a/bb-tests/CMakeLists.txt b/bb-tests/CMakeLists.txt index df6afe5a9..7dc5f899b 100644 --- a/bb-tests/CMakeLists.txt +++ b/bb-tests/CMakeLists.txt @@ -1,10 +1,4 @@ -project(bb-tests) cmake_minimum_required(VERSION 3.26) -cmake_policy(SET CMP0003 NEW) - -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - -set(BB_TESTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(WORKLOADS_DIR ${BB_TESTS_DIR}/workloads) +project(BuckyballTests C CXX) add_subdirectory(workloads) diff --git a/bb-tests/thirdparty/busybox b/bb-tests/thirdparty/busybox deleted file mode 160000 index 1a64f6a20..000000000 --- a/bb-tests/thirdparty/busybox +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1a64f6a20aaf6ea4dbba68bbfa8cc1ab7e5c57c4 diff --git a/bb-tests/thirdparty/linux b/bb-tests/thirdparty/linux deleted file mode 160000 index ffc253263..000000000 --- a/bb-tests/thirdparty/linux +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ffc253263a1375a65fa6c9f62a893e9767fbebfa diff --git a/bb-tests/thirdparty/opensbi b/bb-tests/thirdparty/opensbi deleted file mode 160000 index c0f87f10d..000000000 --- a/bb-tests/thirdparty/opensbi +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c0f87f10d1bfb9e72a84ddfafb5604ee1bfe9d04 diff --git a/bb-tests/workloads/CMakeLists.txt b/bb-tests/workloads/CMakeLists.txt index 588799bc1..277b7c022 100644 --- a/bb-tests/workloads/CMakeLists.txt +++ b/bb-tests/workloads/CMakeLists.txt @@ -1,10 +1,102 @@ -cmake_minimum_required(VERSION 3.12) +cmake_minimum_required(VERSION 3.26) project(BuckyballWorkloads C CXX) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -include(common/CMakeLists.txt) +set(WORKLOAD_DIR ${CMAKE_CURRENT_LIST_DIR}) +set(WORKLOAD_SRC_DIR ${WORKLOAD_DIR}/src) + +get_filename_component(BUCKYBALL_REPO_ROOT ${WORKLOAD_DIR}/../.. ABSOLUTE) +set(WORKLOAD_LIB_DIR ${WORKLOAD_DIR}/lib) +set(BAREMETAL_DIR ${BUCKYBALL_REPO_ROOT}/stack/os/baremetal) + +find_package(Python3 REQUIRED COMPONENTS Interpreter) +find_program(ELF_CC riscv64-unknown-elf-gcc REQUIRED) +find_program(ELF_CXX riscv64-unknown-elf-g++ REQUIRED) +find_program(ELF_OBJCOPY riscv64-unknown-elf-objcopy REQUIRED) +set(LINUX_CC ${CMAKE_C_COMPILER}) +set(LINUX_CXX ${CMAKE_CXX_COMPILER}) + +set(BUCKYBALL_WORKLOAD_DESIGN "" CACHE STRING "Buckyball design for workloads") +if(BUCKYBALL_WORKLOAD_DESIGN STREQUAL "") + message( + FATAL_ERROR + "BUCKYBALL_WORKLOAD_DESIGN is required. Pass -DBUCKYBALL_WORKLOAD_DESIGN=." + ) +endif() +if(NOT IS_DIRECTORY "${BUCKYBALL_REPO_ROOT}/examples/designs/${BUCKYBALL_WORKLOAD_DESIGN}") + message(FATAL_ERROR "Buckyball design does not exist: ${BUCKYBALL_WORKLOAD_DESIGN}") +endif() + +if(NOT DEFINED BUCKYBALL_DESIGN_PB OR BUCKYBALL_DESIGN_PB STREQUAL "") + message(FATAL_ERROR "BUCKYBALL_DESIGN_PB is required") +endif() + +if(NOT EXISTS "${BUCKYBALL_DESIGN_PB}") + message(FATAL_ERROR "missing Design.pb: ${BUCKYBALL_DESIGN_PB}") +endif() +# Everything this build needs from design.pb: ISA headers and the DESIGN_* lists. +execute_process( + COMMAND + "${Python3_EXECUTABLE}" "${BUCKYBALL_REPO_ROOT}/stack/scripts/design.py" + --design-pb "${BUCKYBALL_DESIGN_PB}" --target "${BUCKYBALL_SIM_TARGET}" + --out "${CMAKE_BINARY_DIR}/generated" + COMMAND_ERROR_IS_FATAL ANY +) +set_property( + DIRECTORY + APPEND + PROPERTY + CMAKE_CONFIGURE_DEPENDS + "${BUCKYBALL_DESIGN_PB}" + "${BUCKYBALL_REPO_ROOT}/stack/scripts/design.py" +) +include("${CMAKE_BINARY_DIR}/generated/design.cmake") +# runtime, runtime-baremetal, runtime-linux and engine of every program here: +# built by `bbdev stack --build --device`. +set(BUCKYBALL_ISA_DIR "${CMAKE_BINARY_DIR}/generated/isa") +set(BUCKYBALL_CONTROL_TARGET "core${DESIGN_CONTROL_HART}") + +set(BBSIM_LD ${BAREMETAL_DIR}/bbsim.ld) +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=rv64gc") +set(CMAKE_EXE_LINKER_FLAGS + "${CMAKE_EXE_LINKER_FLAGS} -static -Wl,--no-dynamic-linker" +) + +set(OUTPUT_BIN_DIR "${BUCKYBALL_WORKLOAD_OUTPUT}") + +set(CTEST_MAX_LINES 100) +set(BBHW_MEM_C ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/dma.c) +set(BUCKYBALL_CTEST_C_FLAGS + -g + -fno-common + -O2 + -static + -march=rv64gc + -mcmodel=medany + -fno-builtin-printf + -specs=nano.specs + -specs=nosys.specs + -nostartfiles + -Wl,-T,${BBSIM_LD} +) +file( + GLOB BUCKYBALL_BALL_WORKLOAD_DIRS + CONFIGURE_DEPENDS + "${BUCKYBALL_REPO_ROOT}/examples/components/balls/*/workloads" +) +file( + GLOB BUCKYBALL_BALL_ISA_DEPS + CONFIGURE_DEPENDS + "${BUCKYBALL_REPO_ROOT}/examples/components/balls/*/workloads/isa/*" +) + +set(BUDDY_MLIR_DIR ${BUCKYBALL_REPO_ROOT}/stack/thirdparty/buddy-mlir) + +set(BUCKYBALL_RISCV_MATTR "+xbuckyball,+m,+D") -add_subdirectory(${WORKLOAD_LIB_DIR} ${CMAKE_CURRENT_BINARY_DIR}/lib) add_subdirectory(src) -add_custom_target(sync-bin DEPENDS sync-native-bin sync-mlirtest-bin sync-soctest-bin) +add_custom_target( + sync-bin + DEPENDS sync-native-bin sync-mlirtest-bin sync-soctest-bin +) add_custom_target(build-all ALL DEPENDS sync-bin) diff --git a/bb-tests/workloads/common/CMakeLists.txt b/bb-tests/workloads/common/CMakeLists.txt deleted file mode 100644 index c9f866308..000000000 --- a/bb-tests/workloads/common/CMakeLists.txt +++ /dev/null @@ -1,233 +0,0 @@ -get_filename_component(WORKLOAD_DIR ${CMAKE_CURRENT_LIST_DIR}/.. ABSOLUTE) -set(WORKLOAD_SRC_DIR ${WORKLOAD_DIR}/src) - -get_filename_component(BUCKYBALL_REPO_ROOT ${WORKLOAD_DIR}/../.. ABSOLUTE) -set(WORKLOAD_LIB_DIR ${WORKLOAD_DIR}/lib) -include_directories(${BUCKYBALL_REPO_ROOT}/stack/runtime/include) -set(BBSW_BAREMETAL_DIR ${WORKLOAD_LIB_DIR}/bbsw/baremetal) - -function(buckyball_ctest_name OUT CORE TEST) - if("${CORE}" STREQUAL "" OR "${TEST}" STREQUAL "") - message(FATAL_ERROR "buckyball_ctest_name requires core and test") - endif() - set(${OUT} "${BUCKYBALL_WORKLOAD_CHIP}-${CORE}-ctest-${TEST}" PARENT_SCOPE) -endfunction() - -function(buckyball_mlirtest_name OUT CORE TEST) - if("${CORE}" STREQUAL "" OR "${TEST}" STREQUAL "") - message(FATAL_ERROR "buckyball_mlirtest_name requires core and test") - endif() - set(${OUT} "${BUCKYBALL_WORKLOAD_CHIP}-${CORE}-mlirtest-${TEST}" PARENT_SCOPE) -endfunction() - -function(buckyball_soctest_name OUT CORE TEST) - if("${CORE}" STREQUAL "" OR "${TEST}" STREQUAL "") - message(FATAL_ERROR "buckyball_soctest_name requires core and test") - endif() - set(${OUT} "${BUCKYBALL_WORKLOAD_CHIP}-${CORE}-soctest-${TEST}" PARENT_SCOPE) -endfunction() - -set(RISCV_GNU_TOOLCHAIN $ENV{RISCV}) - -set(BUCKYBALL_WORKLOAD_CHIP "" CACHE STRING "Buckyball chip for workloads") -if(BUCKYBALL_WORKLOAD_CHIP STREQUAL "") - message(FATAL_ERROR - "BUCKYBALL_WORKLOAD_CHIP is required. Pass -DBUCKYBALL_WORKLOAD_CHIP=.") -endif() -set(BUCKYBALL_WORKLOAD_SOURCE_CHIP "${BUCKYBALL_WORKLOAD_CHIP}" CACHE STRING - "Buckyball chip source tree for workloads") -if(NOT IS_DIRECTORY - "${BUCKYBALL_REPO_ROOT}/examples/chips/${BUCKYBALL_WORKLOAD_SOURCE_CHIP}") - message(FATAL_ERROR - "Buckyball workload source chip does not exist: ${BUCKYBALL_WORKLOAD_SOURCE_CHIP}") -endif() -message(STATUS - "Buckyball workload instance: ${BUCKYBALL_WORKLOAD_CHIP} (source: ${BUCKYBALL_WORKLOAD_SOURCE_CHIP})") - -if(NOT IS_ABSOLUTE "${Python3_EXECUTABLE}") - find_program(BUCKYBALL_PYTHON3_EXECUTABLE NAMES python3) - if(NOT BUCKYBALL_PYTHON3_EXECUTABLE) - message(FATAL_ERROR "python3 is required to build workloads") - endif() - set(Python3_EXECUTABLE "${BUCKYBALL_PYTHON3_EXECUTABLE}") -endif() - -foreach(_BUCKYBALL_REQ IN ITEMS - BUCKYBALL_CHIP_PB) - if(NOT DEFINED ${_BUCKYBALL_REQ} OR "${${_BUCKYBALL_REQ}}" STREQUAL "") - message(FATAL_ERROR - "${_BUCKYBALL_REQ} must be passed by bbdev config --install (cmake.defs)") - endif() -endforeach() - -if(NOT EXISTS "${BUCKYBALL_CHIP_PB}") - message(FATAL_ERROR "missing Chip.pb: ${BUCKYBALL_CHIP_PB}") -endif() -set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS - "${BUCKYBALL_CHIP_PB}" - "${BUCKYBALL_REPO_ROOT}/stack/compiler/scripts/pb_to_target_registry.py") -set(BUCKYBALL_ISA_DIR "${CMAKE_BINARY_DIR}/generated/isa") -execute_process( - COMMAND "${Python3_EXECUTABLE}" - "${BUCKYBALL_REPO_ROOT}/stack/compiler/scripts/pb_to_target_registry.py" - --repo "${BUCKYBALL_REPO_ROOT}" - --chip-pb "${BUCKYBALL_CHIP_PB}" - --isa-dir "${BUCKYBALL_ISA_DIR}" - --print-targets - --print-target-balls - RESULT_VARIABLE BUCKYBALL_TARGET_REGISTRY_RESULT - OUTPUT_VARIABLE BUCKYBALL_TARGETS_RAW - ERROR_VARIABLE BUCKYBALL_TARGET_REGISTRY_ERROR) -if(NOT BUCKYBALL_TARGET_REGISTRY_RESULT EQUAL 0) - message(FATAL_ERROR - "failed to generate target ISA headers: ${BUCKYBALL_TARGET_REGISTRY_ERROR}") -endif() -string(REPLACE "\n" ";" BUCKYBALL_TARGETS "${BUCKYBALL_TARGETS_RAW}") -execute_process( - COMMAND "${Python3_EXECUTABLE}" - "${BUCKYBALL_REPO_ROOT}/stack/compiler/scripts/pb_to_target_registry.py" - --repo "${BUCKYBALL_REPO_ROOT}" - --chip-pb "${BUCKYBALL_CHIP_PB}" - --print-bank-targets - RESULT_VARIABLE BUCKYBALL_BANK_TARGETS_RESULT - OUTPUT_VARIABLE BUCKYBALL_BANK_TARGETS_RAW - ERROR_VARIABLE BUCKYBALL_BANK_TARGETS_ERROR) -if(NOT BUCKYBALL_BANK_TARGETS_RESULT EQUAL 0) - message(FATAL_ERROR - "failed to get compiler targets with banks: ${BUCKYBALL_BANK_TARGETS_ERROR}") -endif() -string(REPLACE "\n" ";" BUCKYBALL_BANK_TARGETS - "${BUCKYBALL_BANK_TARGETS_RAW}") -list(REMOVE_ITEM BUCKYBALL_BANK_TARGETS "") -execute_process( - COMMAND "${Python3_EXECUTABLE}" - "${BUCKYBALL_REPO_ROOT}/stack/compiler/scripts/pb_to_target_registry.py" - --repo "${BUCKYBALL_REPO_ROOT}" - --chip-pb "${BUCKYBALL_CHIP_PB}" - --print-core-workload-targets - RESULT_VARIABLE BUCKYBALL_CORE_TARGETS_RESULT - OUTPUT_VARIABLE BUCKYBALL_CORE_TARGETS_RAW - ERROR_VARIABLE BUCKYBALL_CORE_TARGETS_ERROR) -if(NOT BUCKYBALL_CORE_TARGETS_RESULT EQUAL 0) - message(FATAL_ERROR - "failed to get Core workload targets: ${BUCKYBALL_CORE_TARGETS_ERROR}") -endif() -string(REPLACE "\n" ";" BUCKYBALL_CORE_TARGETS - "${BUCKYBALL_CORE_TARGETS_RAW}") -list(REMOVE_ITEM BUCKYBALL_CORE_TARGETS "") -set(BUCKYBALL_TARGET_BALL_LINES) -foreach(BUCKYBALL_TARGET_LINE ${BUCKYBALL_TARGETS}) - if(BUCKYBALL_TARGET_LINE MATCHES "^([^:]+):(.*)$") - set(BUCKYBALL_TARGET_BALLS_${CMAKE_MATCH_1} "${CMAKE_MATCH_2}") - list(APPEND BUCKYBALL_TARGET_BALL_LINES ${BUCKYBALL_TARGET_LINE}) - endif() -endforeach() -list(REMOVE_ITEM BUCKYBALL_TARGETS ${BUCKYBALL_TARGET_BALL_LINES} "") -if(NOT BUCKYBALL_TARGETS) - message(FATAL_ERROR "Chip.pb has no compiler targets") -endif() -foreach(_BUCKYBALL_TARGET IN LISTS BUCKYBALL_TARGETS) - if(NOT EXISTS "${BUCKYBALL_ISA_DIR}/${_BUCKYBALL_TARGET}/params.h") - message(FATAL_ERROR - "missing generated params.h for target ${_BUCKYBALL_TARGET}") - endif() -endforeach() -message(STATUS "Buckyball compiler targets: ${BUCKYBALL_TARGETS}") -message(STATUS "Buckyball bank targets: ${BUCKYBALL_BANK_TARGETS}") -message(STATUS "Buckyball Core workload targets: ${BUCKYBALL_CORE_TARGETS}") -execute_process( - COMMAND "${Python3_EXECUTABLE}" - "${BUCKYBALL_REPO_ROOT}/stack/compiler/scripts/pb_to_target_registry.py" - --repo "${BUCKYBALL_REPO_ROOT}" --chip-pb "${BUCKYBALL_CHIP_PB}" - --print-cpu-targets - RESULT_VARIABLE _cpu_result OUTPUT_VARIABLE _cpu_targets ERROR_VARIABLE _cpu_error) -if(NOT _cpu_result EQUAL 0) - message(FATAL_ERROR "failed to get CPU workload targets: ${_cpu_error}") -endif() -string(REPLACE "\n" ";" BUCKYBALL_CPU_TARGETS "${_cpu_targets}") -list(REMOVE_ITEM BUCKYBALL_CPU_TARGETS "") - -function(buckyball_assign_physical_banks TARGET OUT) - list(FIND BUCKYBALL_BANK_TARGETS "${TARGET}" _bank_target_index) - if(_bank_target_index EQUAL -1) - message(FATAL_ERROR "${TARGET}: no physical banks") - endif() - set(${OUT} "-assign-physical-banks" PARENT_SCOPE) -endfunction() - -set(BBSIM_LD ${BBSW_BAREMETAL_DIR}/bbsim.ld) -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=rv64gc") -set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -Wl,--no-dynamic-linker") - -function(buckyball_chip_ball_dirs OUT_VAR TARGET) - set(_raw "${BUCKYBALL_TARGET_BALLS_${TARGET}}") - if(_raw STREQUAL "") - set(${OUT_VAR} "" PARENT_SCOPE) - return() - endif() - string(REPLACE "," ";" _dirs "${_raw}") - foreach(_dir IN LISTS _dirs) - if(_dir STREQUAL "") - message(FATAL_ERROR "empty ball_dir on target ${TARGET}") - endif() - endforeach() - set(${OUT_VAR} "${_dirs}" PARENT_SCOPE) -endfunction() - -function(buckyball_add_ball_workload_subdirs KIND) - if(NOT KIND STREQUAL "ctests" AND NOT KIND STREQUAL "mlir_tests") - message(FATAL_ERROR "unknown ball workload kind: ${KIND}") - endif() - foreach(BUCKYBALL_TARGET IN LISTS BUCKYBALL_TARGETS) - if(NOT BUCKYBALL_TARGET IN_LIST BUCKYBALL_CPU_TARGETS) - continue() - endif() - buckyball_chip_ball_dirs(_ball_dirs ${BUCKYBALL_TARGET}) - foreach(BUCKYBALL_BALL_DIR IN LISTS _ball_dirs) - set(_src - ${BUCKYBALL_REPO_ROOT}/examples/balls/${BUCKYBALL_BALL_DIR}/workloads/${KIND}) - if(NOT IS_DIRECTORY ${_src}) - if(KIND STREQUAL "mlir_tests") - continue() - endif() - message(FATAL_ERROR "missing Ball ${KIND} directory: ${_src}") - endif() - if(KIND STREQUAL "ctests") - set(BUCKYBALL_CTEST_TARGET ${BUCKYBALL_TARGET}) - add_subdirectory( - ${_src} - ${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}/ctests) - else() - set(BUCKYBALL_MLIR_ACTIVE_TARGET ${BUCKYBALL_TARGET}) - add_subdirectory( - ${_src} - ${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}) - endif() - endforeach() - endforeach() -endfunction() - -function(buckyball_add_core_workload_subdirs KIND) - foreach(_entry IN LISTS BUCKYBALL_CORE_TARGETS) - if(NOT _entry MATCHES "^([^:]+):([^:]+):([0-9]+)$") - message(FATAL_ERROR "invalid Core workload target: ${_entry}") - endif() - set(BUCKYBALL_CORE_WORKLOAD_TARGET "${CMAKE_MATCH_1}") - if(KIND STREQUAL "ctests" AND NOT BUCKYBALL_CORE_WORKLOAD_TARGET IN_LIST BUCKYBALL_CPU_TARGETS) - continue() - endif() - set(_pkg "${CMAKE_MATCH_2}") - set(BUCKYBALL_CORE_WORKLOAD_PROFILE "${CMAKE_MATCH_3}") - set(_src - "${BUCKYBALL_REPO_ROOT}/examples/cores/${_pkg}/workloads/${KIND}") - if(IS_DIRECTORY "${_src}") - add_subdirectory("${_src}" - "${CMAKE_CURRENT_BINARY_DIR}/cores/${BUCKYBALL_CORE_WORKLOAD_TARGET}/${KIND}") - endif() - endforeach() -endfunction() - -set(ELF_CC ${RISCV_GNU_TOOLCHAIN}/bin/riscv64-unknown-elf-gcc) -set(LINUX_CC ${RISCV_GNU_TOOLCHAIN}/bin/riscv64-unknown-linux-gnu-gcc) -set(BUILD_BIN_DIR ${CMAKE_BINARY_DIR}) -set(OUTPUT_BIN_DIR ${WORKLOAD_DIR}/../output/${BUCKYBALL_WORKLOAD_CHIP}/workloads) diff --git a/bb-tests/workloads/lib/CMakeLists.txt b/bb-tests/workloads/lib/CMakeLists.txt deleted file mode 100644 index 04e30376e..000000000 --- a/bb-tests/workloads/lib/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -add_subdirectory(bbhw) -add_subdirectory(${BUCKYBALL_REPO_ROOT}/stack/runtime runtime EXCLUDE_FROM_ALL) - -if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/kernel/CMakeLists.txt") - add_subdirectory(kernel) -endif() diff --git a/bb-tests/workloads/lib/bbhw/CMakeLists.txt b/bb-tests/workloads/lib/bbhw/CMakeLists.txt deleted file mode 100644 index 195ef1c6e..000000000 --- a/bb-tests/workloads/lib/bbhw/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -# bbhw: ISA macros (headers) + dma.c (dma_touch). -# Include path is WORKLOAD_LIB_DIR (workloads/lib), so #include works. -# dma.c is compiled into each ctest via bb-tests/workloads/CMakeLists.txt. diff --git a/bb-tests/workloads/lib/bbhw/isa/01_barrier.c b/bb-tests/workloads/lib/bbhw/isa/01_barrier.c index 02aef6904..e8bc9b8a4 100644 --- a/bb-tests/workloads/lib/bbhw/isa/01_barrier.c +++ b/bb-tests/workloads/lib/bbhw/isa/01_barrier.c @@ -6,14 +6,14 @@ #define BB_BARRIER_FUNC7 1 /** - * bb_barrier() - hardware multi-core barrier synchronization. + * bb_barrier() - hardware multi-thread barrier synchronization. * * Semantics: - * 1. Waits for this core's own instruction ROB to drain. - * 2. Signals arrive to the tile-level BarrierUnit. - * 3. Stalls until all nCores cores have arrived (hardware all-reduce). + * 1. Waits for this thread's own instruction ROB to drain. + * 2. Signals arrive to the core-level BarrierUnit. + * 3. Stalls until all nThreads threads have arrived (hardware all-reduce). * - * All cores in the same BBTile must call bb_barrier() at the same point. + * All threads in the same BBCore must call bb_barrier() at the same point. */ #define bb_barrier() BUCKYBALL_INSTRUCTION_R_R(0, 0, BB_BARRIER_FUNC7) diff --git a/bb-tests/workloads/lib/bbhw/isa/13_mvover.c b/bb-tests/workloads/lib/bbhw/isa/13_mvover.c index 82579abe0..c77023ddb 100644 --- a/bb-tests/workloads/lib/bbhw/isa/13_mvover.c +++ b/bb-tests/workloads/lib/bbhw/isa/13_mvover.c @@ -5,20 +5,20 @@ enum { BB_MVOVER_FUNC7 = 0x0d }; -/* Move contiguous rows from a private Bank of one Core to another in the same - * Tile. Bank IDs are Core-local virtual IDs; row addresses are not byte +/* Move contiguous rows from a private Bank of one Thread to another in the same + * Core. Bank IDs are Thread-local virtual IDs; row addresses are not byte * addresses. */ -static inline void bb_mvover(uint32_t source_core, uint32_t source_bank, - uint32_t source_row, uint32_t target_core, +static inline void bb_mvover(uint32_t source_thread, uint32_t source_bank, + uint32_t source_row, uint32_t target_thread, uint32_t target_bank, uint32_t target_row, uint32_t rows) { - if (source_core > 255 || target_core > 255 || source_bank > 1023 || + if (source_thread > 255 || target_thread > 255 || source_bank > 1023 || target_bank > 1023 || source_row > 65535 || target_row > 65535 || !rows || rows > 65536) __builtin_trap(); - uint64_t rs1 = FIELD(source_core, 0, 7) | FIELD(target_core, 8, 15) | + uint64_t rs1 = FIELD(source_thread, 0, 7) | FIELD(target_thread, 8, 15) | FIELD(source_bank, 16, 25) | FIELD(target_bank, 26, 35); uint64_t rs2 = FIELD(source_row, 0, 15) | FIELD(target_row, 16, 31) | FIELD(rows - 1, 32, 47); diff --git a/bb-tests/workloads/lib/bbhw/isa/14_task.c b/bb-tests/workloads/lib/bbhw/isa/14_task.c deleted file mode 100644 index 382ffb65b..000000000 --- a/bb-tests/workloads/lib/bbhw/isa/14_task.c +++ /dev/null @@ -1,52 +0,0 @@ -#ifndef BB_TASK_ISA_H -#define BB_TASK_ISA_H - -#include "isa.h" - -static inline uintptr_t bb_task_submit(uintptr_t worker, uintptr_t signature) { - uintptr_t result; - asm volatile(".insn r 0x2b, 7, 0, %0, %1, %2" - : "=r"(result) - : "r"(worker), "r"(signature) - : "memory"); - return result; -} - -static inline uintptr_t bb_task_status(uintptr_t worker) { - uintptr_t result; - asm volatile(".insn r 0x2b, 7, 1, %0, %1, %2" - : "=r"(result) - : "r"(worker), "r"((uintptr_t)0) - : "memory"); - return result; -} - -static inline uintptr_t bb_task_worker_count(void) { - uintptr_t result; - asm volatile(".insn r 0x2b, 7, 3, %0, %1, %2" - : "=r"(result) - : "r"((uintptr_t)0), "r"((uintptr_t)0) - : "memory"); - return result; -} - -static inline uintptr_t bb_task_core_signature(uintptr_t core) { - uintptr_t result; - asm volatile(".insn r 0x2b, 7, 4, %0, %1, %2" - : "=r"(result) - : "r"(core), "r"((uintptr_t)0) - : "memory"); - return result; -} - -static inline uintptr_t bb_task_write_descriptor(uintptr_t field, - uintptr_t value) { - uintptr_t result; - asm volatile(".insn r 0x2b, 7, 7, %0, %1, %2" - : "=r"(result) - : "r"(field), "r"(value) - : "memory"); - return result; -} - -#endif diff --git a/bb-tests/workloads/lib/bbhw/isa/16_mvout.c b/bb-tests/workloads/lib/bbhw/isa/16_mvout.c index ca92b8987..4a2d9efb0 100644 --- a/bb-tests/workloads/lib/bbhw/isa/16_mvout.c +++ b/bb-tests/workloads/lib/bbhw/isa/16_mvout.c @@ -2,7 +2,9 @@ #define _BB_MVOUT_H_ #include "isa.h" +#if defined(__linux__) #include +#endif #define BB_MVOUT_FUNC7 16 diff --git a/bb-tests/workloads/lib/bbhw/isa/32_mset.c b/bb-tests/workloads/lib/bbhw/isa/32_mset.c index d7df73012..34fd5c675 100644 --- a/bb-tests/workloads/lib/bbhw/isa/32_mset.c +++ b/bb-tests/workloads/lib/bbhw/isa/32_mset.c @@ -2,7 +2,9 @@ #define _BB_MSET_H_ #include "isa.h" +#if defined(__linux__) #include +#endif #define BB_MSET_FUNC7 32 diff --git a/bb-tests/workloads/lib/bbhw/isa/34_mvin_2d.c b/bb-tests/workloads/lib/bbhw/isa/34_mvin_2d.c index 57971cb41..d993f2567 100644 --- a/bb-tests/workloads/lib/bbhw/isa/34_mvin_2d.c +++ b/bb-tests/workloads/lib/bbhw/isa/34_mvin_2d.c @@ -5,12 +5,12 @@ #define BB_MVIN_2D_FUNC7 34 -// rs1: write bank in BANK2 and tile height; [19:0] reserved. +// rs1: write bank in BANK2 and core height; [19:0] reserved. // rs2: [35:0] address / 8, [42:36] physical pixel bytes / 8, // [52:43] source width, [58:53] destination row base, -// [61:59] tile width - 1, [62] valid bytes (0 = 16, 1 = 8). +// [61:59] core width - 1, [62] valid bytes (0 = 16, 1 = 8). #define bb_mvin_2d(mem_addr, bank_id, height, pixel_bytes, source_width, \ - dst_base, tile_width, valid_bytes) \ + dst_base, core_width, valid_bytes) \ do { \ const uintptr_t bb_mvin_2d_addr = (uintptr_t)(mem_addr); \ const uint64_t bb_mvin_2d_valid = (valid_bytes); \ @@ -21,7 +21,7 @@ (BB_BANK2(bank_id) | BB_ITER(height)), \ (FIELD(bb_mvin_2d_addr >> 3, 0, 35) | \ FIELD((pixel_bytes) / 8, 36, 42) | FIELD(source_width, 43, 52) | \ - FIELD(dst_base, 53, 58) | FIELD((tile_width) - 1, 59, 61) | \ + FIELD(dst_base, 53, 58) | FIELD((core_width) - 1, 59, 61) | \ FIELD(bb_mvin_2d_valid == 8, 62, 62)), \ BB_MVIN_2D_FUNC7); \ } while (0) diff --git a/bb-tests/workloads/lib/bbhw/isa/bb_func7.h b/bb-tests/workloads/lib/bbhw/isa/bb_func7.h index 5b712fd4b..7db8ad0bd 100644 --- a/bb-tests/workloads/lib/bbhw/isa/bb_func7.h +++ b/bb-tests/workloads/lib/bbhw/isa/bb_func7.h @@ -4,5 +4,6 @@ #include "ballISA.h" #define BB_FUNC7(mne) BB_FUNC7_##mne +#define BB_FUNC7_INSTANCE(id, mne) BB_FUNC7_BALL_##id##_##mne #endif diff --git a/bb-tests/workloads/lib/bbhw/isa/isa.c b/bb-tests/workloads/lib/bbhw/isa/isa.c deleted file mode 100644 index 283156402..000000000 --- a/bb-tests/workloads/lib/bbhw/isa/isa.c +++ /dev/null @@ -1 +0,0 @@ -#include "isa.h" diff --git a/bb-tests/workloads/lib/bbhw/isa/isa.h b/bb-tests/workloads/lib/bbhw/isa/isa.h index c733d66f9..2d2923360 100644 --- a/bb-tests/workloads/lib/bbhw/isa/isa.h +++ b/bb-tests/workloads/lib/bbhw/isa/isa.h @@ -21,23 +21,26 @@ typedef int32_t result_t; #define FIELD(val, start_bit, end_bit) \ (((val) & ((2UL << ((end_bit) - (start_bit))) - 1)) << (start_bit)) -// rs1 bank field helpers (10-bit each) +// rs1 bank fields: 10-bit sources and 9-bit architectural destination. #define BB_BANK0(id) FIELD(id, 0, 9) #define BB_BANK1(id) FIELD(id, 10, 19) -#define BB_BANK2(id) FIELD(id, 20, 29) +#define BB_BANK2(id) FIELD(id, 20, 28) +// The architectural destination also supplies an old-value read operand. +#define BB_READ_WR_OLD (1ULL << 29) +#define BB_UPDATE_WR (1ULL << 63) -// rs1 iter field (34-bit, bits 30-63) -#define BB_ITER(n) FIELD(n, 30, 63) +// rs1 iter field (33-bit, bits 30-62); bit 63 declares an in-place update. +#define BB_ITER(n) FIELD(n, 30, 62) // funct7 encoding: [6:4]=enable, [3:0]=opcode // enable: 000=none, 001=1rd, 010=1wr, 011=1rd+1wr, 100=2rd+1wr // 101/110/111 = none (extended opcode space) // Generic RISC-V custom instruction macro (funct3 always 0x3 = CUSTOM3_RS1_RS2) -#ifdef BUCKYBALL_ANT_HOST -#include +#ifdef BUCKYBALL_STREAM +#include #define BUCKYBALL_INSTRUCTION_R_R(rs1_val, rs2_val, func7) \ - ant_emit((func7), (uint64_t)(rs1_val), (uint64_t)(rs2_val)) + stream_emit((func7), (uint64_t)(rs1_val), (uint64_t)(rs2_val)) #else #define BUCKYBALL_INSTRUCTION_R_R(rs1_val, rs2_val, func7) \ asm volatile(".insn r " STR(CUSTOM_3) ", 3, %c2, x0, %0, %1" \ @@ -51,7 +54,6 @@ typedef int32_t result_t; #include "01_barrier.c" #include "12_kernel.c" #include "13_mvover.c" -#include "14_task.c" #include "16_mvout.c" #include "32_mset.c" #include "33_mvin.c" diff --git a/bb-tests/workloads/lib/bbsw/CRunnerUtils/CRunnerUtils.cpp b/bb-tests/workloads/lib/bbsw/CRunnerUtils/CRunnerUtils.cpp deleted file mode 100644 index 33821e886..000000000 --- a/bb-tests/workloads/lib/bbsw/CRunnerUtils/CRunnerUtils.cpp +++ /dev/null @@ -1,75 +0,0 @@ -// The code here is from the upstream llvm, you can check -// llvm/mlir/include/mlir/ExecutionEngine/CRunnerUtils.cpp - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -extern "C" void memrefCopy(int64_t elemSize, UnrankedMemRefType *srcArg, - UnrankedMemRefType *dstArg) { - DynamicMemRefType src(*srcArg); - DynamicMemRefType dst(*dstArg); - - int64_t rank = src.rank; - MLIR_MSAN_MEMORY_IS_INITIALIZED(src.sizes, rank * sizeof(int64_t)); - - // Handle empty shapes -> nothing to copy. - for (int rankp = 0; rankp < rank; ++rankp) - if (src.sizes[rankp] == 0) - return; - - char *srcPtr = src.data + src.offset * elemSize; - char *dstPtr = dst.data + dst.offset * elemSize; - - if (rank == 0) { - memcpy(dstPtr, srcPtr, elemSize); - return; - } - - int64_t *indices = static_cast(alloca(sizeof(int64_t) * rank)); - int64_t *srcStrides = static_cast(alloca(sizeof(int64_t) * rank)); - int64_t *dstStrides = static_cast(alloca(sizeof(int64_t) * rank)); - - // Initialize index and scale strides. - for (int rankp = 0; rankp < rank; ++rankp) { - indices[rankp] = 0; - srcStrides[rankp] = src.strides[rankp] * elemSize; - dstStrides[rankp] = dst.strides[rankp] * elemSize; - } - - int64_t readIndex = 0, writeIndex = 0; - for (;;) { - // Copy over the element, byte by byte. - memcpy(dstPtr + writeIndex, srcPtr + readIndex, elemSize); - // Advance index and read position. - for (int64_t axis = rank - 1; axis >= 0; --axis) { - // Advance at current axis. - auto newIndex = ++indices[axis]; - readIndex += srcStrides[axis]; - writeIndex += dstStrides[axis]; - // If this is a valid index, we have our next index, so continue copying. - if (src.sizes[axis] != newIndex) - break; - // We reached the end of this axis. If this is axis 0, we are done. - if (axis == 0) - return; - // Else, reset to 0 and undo the advancement of the linear index that - // this axis had. Then continue with the axis one outer. - indices[axis] = 0; - readIndex -= src.sizes[axis] * srcStrides[axis]; - writeIndex -= dst.sizes[axis] * dstStrides[axis]; - } - } -} - -extern "C" void printF64(double d) { fprintf(stdout, "%lg", d); } - -extern "C" void printNewline() { fputc('\n', stdout); } diff --git a/bb-tests/workloads/lib/bbsw/baremetal/bbsim.ld b/bb-tests/workloads/lib/bbsw/baremetal/bbsim.ld deleted file mode 100644 index a33fda064..000000000 --- a/bb-tests/workloads/lib/bbsw/baremetal/bbsim.ld +++ /dev/null @@ -1,63 +0,0 @@ -/* bbsim.ld — baremetal linker script for BBSimHarness (DRAM at 0x8000_0000) */ -/* Bootrom jumps directly to 0x80000000 (ELF entry), _start must be first. */ -OUTPUT_ARCH("riscv") -ENTRY(_start) - -SECTIONS { - . = 0x80000000; - - .text : { - *(.text.init) /* crt0.S _start must be first */ - *(.text.startup .text.startup.*) - *(.text .text.*) - *(.gnu.linkonce.t.*) - } - - .rodata : { - *(.rodata .rodata.*) - *(.srodata .srodata.*) - *(.gnu.linkonce.r.*) - } - - .data : { - *(.data .data.*) - *(.sdata .sdata.*) - *(.gnu.linkonce.d.*) - *(.gnu.linkonce.s.*) - PROVIDE(_edata = .); - } - - .tdata ALIGN(16) : { - PROVIDE(__tls_start = .); - *(.tdata .tdata.* .gnu.linkonce.td.*) - PROVIDE(__tdata_end = .); - } - - .tbss ALIGN(8) (NOLOAD) : { - PROVIDE(__bss_start = .); - *(.tbss .tbss.* .gnu.linkonce.tb.*) - . = ALIGN(8); - PROVIDE(__tls_end = .); - } - . = ADDR(.tbss) + SIZEOF(.tbss); - - /* All zero-init sections: crt0.S clears __bss_start .. __bss_end, - so heap (at _end) starts after ALL bss including sbss. */ - .bss (NOLOAD) : { - *(.bss .bss.*) - *(.sbss .sbss.*) - *(.gnu.linkonce.b.*) - *(.gnu.linkonce.sb.*) - *(COMMON) - . = ALIGN(8); - PROVIDE(__bss_end = .); - } - - .noinit (NOLOAD) : { - *(.noinit .noinit.*) - } - - . = ALIGN(16); - PROVIDE(end = .); - PROVIDE(_end = .); -} diff --git a/bb-tests/workloads/lib/bbsw/baremetal/crt0.S b/bb-tests/workloads/lib/bbsw/baremetal/crt0.S deleted file mode 100644 index 81ec0f2aa..000000000 --- a/bb-tests/workloads/lib/bbsw/baremetal/crt0.S +++ /dev/null @@ -1,71 +0,0 @@ -/* crt0.S — minimal startup for BBSimHarness baremetal - * Bootrom jumps to 0x80000000 (this code). - * Sets up stack and TLS, clears BSS, then calls main. */ -#include -.section .text.init,"ax",@progbits -.global _start -_start: - /* The generated target profile selects its test hart before boot. */ - csrr t0, mhartid - li t1, BB_TEST_HART - bne t0, t1, park_hart - - /* Enable FPU: set mstatus.FS = 01 (Initial) */ - li t0, 0x2000 # mstatus.FS = 01 (bit 13) - csrs mstatus, t0 - - /* Set up trap vector (mtvec) to trap_entry */ - la t0, trap_entry - csrw mtvec, t0 - - /* stack pointer: top of the 256 MiB BBSim DRAM window */ - li sp, 0x90000000 - - .option push - .option norelax - la tp, __tls_start - .option pop - - /* clear BSS */ - la a0, __bss_start - la a1, __bss_end -bss_loop: - bgeu a0, a1, bss_done - sd zero, 0(a0) - addi a0, a0, 8 - j bss_loop -bss_done: - - li a0, 0 - li a1, 0 - call main - - /* if main returns, write its return code to sim_exit */ - mv t0, a0 - li a0, 0x60000000 - sw t0, 0(a0) -1: j 1b - -/* Trap handler: print mcause and mepc, then exit */ -.align 4 -trap_entry: - /* Save context */ - addi sp, sp, -16 - sd ra, 8(sp) - sd a0, 0(sp) - - /* Read mcause and mepc for debugging */ - csrr a0, mcause - csrr a1, mepc - - /* Exit simulation with error code */ - li a0, 0x60000000 - li t0, 1 - sw t0, 0(a0) - - /* Infinite loop */ -1: j 1b - -park_hart: - wfi - j park_hart diff --git a/bb-tests/workloads/lib/bbsw/baremetal/multicore/multicore.h b/bb-tests/workloads/lib/bbsw/baremetal/multicore/multicore.h deleted file mode 100644 index 134a49c8c..000000000 --- a/bb-tests/workloads/lib/bbsw/baremetal/multicore/multicore.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef BBSW_BAREMETAL_MULTICORE_H -#define BBSW_BAREMETAL_MULTICORE_H - -#include -#include - -static inline uint32_t bb_get_hart_id(void) { - uint32_t hart; - asm volatile("csrr %0, mhartid" : "=r"(hart)); - return hart; -} - -static inline core_id_t bb_get_core_id(void) { - return bb_topology_core_id(bb_get_hart_id()); -} - -#endif diff --git a/bb-tests/workloads/lib/bbsw/baremetal/start.S b/bb-tests/workloads/lib/bbsw/baremetal/start.S deleted file mode 100644 index 3b943056c..000000000 --- a/bb-tests/workloads/lib/bbsw/baremetal/start.S +++ /dev/null @@ -1,100 +0,0 @@ -.section .text.init -.global _start - -#include - -_start: - # Get hart ID - csrr t0, mhartid - - # Enable the FPU on every hart: mstatus.FS = Initial - li t1, 0x2000 - csrs mstatus, t1 - - # Set up stack pointer for each hart - # Each hart gets 4KB stack space (0x1000 bytes) - li sp, 0x90000000 - li t1, 0x1000 - mul t1, t0, t1 - sub sp, sp, t1 - # Each hart owns a TLS copy at the top of its 4KB stack slot. - la t2, __tls_start - la t3, __tdata_end - la t4, __tls_end - sub t5, t4, t2 - addi t5, t5, 15 - andi t5, t5, -16 - sub sp, sp, t5 - mv tp, sp - mv t6, tp -.Lcopy_tls: - bgeu t2, t3, .Lzero_tls - lbu a0, 0(t2) - sb a0, 0(t6) - addi t2, t2, 1 - addi t6, t6, 1 - j .Lcopy_tls -.Lzero_tls: - bgeu t2, t4, .Lrun_main - sb zero, 0(t6) - addi t2, t2, 1 - addi t6, t6, 1 - j .Lzero_tls -.Lrun_main: - # Loaders write only the file image, so .bss can hold a previous run's values on P2E DDR. - # The reporting hart clears it, then releases the others through the last flag write. - csrr t0, mhartid - la t1, .Lbss_ready - li t2, BB_REPORT_HART - bne t0, t2, .Lwait_bss - la t2, __bss_start - la t3, __bss_end - addi t4, t3, -63 -.Lclear_bss_blocks: - bgeu t2, t4, .Lclear_bss - sd zero, 0(t2) - sd zero, 8(t2) - sd zero, 16(t2) - sd zero, 24(t2) - sd zero, 32(t2) - sd zero, 40(t2) - sd zero, 48(t2) - sd zero, 56(t2) - addi t2, t2, 64 - j .Lclear_bss_blocks -.Lclear_bss: - bgeu t2, t3, .Lbss_cleared - sd zero, 0(t2) - addi t2, t2, 8 - j .Lclear_bss -.Lbss_cleared: - li t2, 1 - fence rw, w - sw t2, 0(t1) - j .Lcall_main -.Lwait_bss: - lw t2, 0(t1) - beqz t2, .Lwait_bss - fence r, rw -.Lcall_main: - # Jump to main for all harts; the workload dispatches by core id. - li a0, 0 - li a1, 0 - call main - - # Core 0 of the work tile reports the workload result. Other harts remain parked. - csrr t0, mhartid - li t1, BB_REPORT_HART - bne t0, t1, 1f - li t1, 0x60000000 - sw a0, 0(t1) -1: - wfi - j 1b - -# Outside .bss so the clear cannot reset it; a stale 1 from a previous run is -# overwritten by the image load because .data is part of the file image. -.section .data -.balign 4 -.Lbss_ready: - .word 0 diff --git a/bb-tests/workloads/lib/bbsw/baremetal/syscalls.c b/bb-tests/workloads/lib/bbsw/baremetal/syscalls.c deleted file mode 100644 index a37464db6..000000000 --- a/bb-tests/workloads/lib/bbsw/baremetal/syscalls.c +++ /dev/null @@ -1,18 +0,0 @@ -#include - -#define MMIO_SIM_EXIT ((volatile uint32_t *)0x60000000UL) -#define MMIO_UART_TX ((volatile uint32_t *)0x60020000UL) - -int _write(int fd, const char *buf, int len) { - (void)fd; - for (int i = 0; i < len; ++i) { - *MMIO_UART_TX = (uint32_t)(unsigned char)buf[i]; - } - return len; -} - -void __attribute__((noreturn)) _exit(int code) { - *MMIO_SIM_EXIT = (uint32_t)code; - while (1) { - } -} diff --git a/bb-tests/workloads/lib/bbsw/baremetal/test.c b/bb-tests/workloads/lib/bbsw/baremetal/test.c deleted file mode 100644 index fa2b77579..000000000 --- a/bb-tests/workloads/lib/bbsw/baremetal/test.c +++ /dev/null @@ -1,12 +0,0 @@ -#include -#include - -#define MMIO_SIM_EXIT ((volatile uint32_t *)0x60000000UL) - -void bb_test_pass(void) { *MMIO_SIM_EXIT = 0; } - -void bb_test_fail(void) { - *MMIO_SIM_EXIT = 1; - while (1) { - } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/CMakeLists.txt b/bb-tests/workloads/lib/bbsw/kernels/CMakeLists.txt deleted file mode 100644 index aa9a73446..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../../../.." ABSOLUTE) -add_library(scalar_kernels STATIC mxfp8/pack.cpp mxfp8/quant.cpp) -target_compile_options(scalar_kernels PRIVATE -O2) -target_compile_features(scalar_kernels PRIVATE cxx_std_20) -target_include_directories(scalar_kernels PRIVATE - "${REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") diff --git a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/CMakeLists.txt b/bb-tests/workloads/lib/bbsw/kernels/mxfp8/CMakeLists.txt deleted file mode 100644 index 09552e568..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../.." ABSOLUTE) -add_library(mxfp8_pack STATIC pack.cpp) -target_compile_options(mxfp8_pack PRIVATE -O2) -target_compile_features(mxfp8_pack PRIVATE cxx_std_20) -target_include_directories(mxfp8_pack PRIVATE - "${REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") - -add_library(mxfp8_pack_ant STATIC EXCLUDE_FROM_ALL pack.cpp) -target_link_libraries(mxfp8_pack_ant PRIVATE runtime_ant_tasks) -target_compile_options(mxfp8_pack_ant PRIVATE -O2) -target_compile_features(mxfp8_pack_ant PRIVATE cxx_std_20) -target_include_directories(mxfp8_pack_ant PRIVATE - "${REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") diff --git a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/pack.cpp b/bb-tests/workloads/lib/bbsw/kernels/mxfp8/pack.cpp deleted file mode 100644 index 4fbc78c13..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/pack.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "quant.h" - -extern "C" void _mlir_ciface_mxfp8_quant(StridedMemRefType *input, - StridedMemRefType *output, - int64_t tileRows, int64_t tileK, - int64_t panelStride) { - int64_t m = input->sizes[0], k = input->sizes[1]; - int64_t mt = (m + tileRows - 1) / tileRows, kt = (k + tileK - 1) / tileK; - int64_t bytes = panelStride; - if (k % 32 || tileK % 32 || output->sizes[0] != mt * kt * bytes) { - fprintf(stderr, "mxfp8: invalid activation packing shape\n"); - abort(); - } - auto *source = input->data + input->offset; - auto *dest = output->data + output->offset; - const int64_t rowStride = input->strides[0], columnStride = input->strides[1]; - const int64_t outputStride = output->strides[0]; - for (int64_t mr = 0; mr < mt; ++mr) - for (int64_t kr = 0; kr < kt; ++kr) - for (int64_t row = 0; row < tileRows; ++row) { - int64_t width = std::min(tileK, k - kr * tileK); - int64_t base = (mr * kt + kr) * bytes; - if (mr * tileRows + row >= m && outputStride == 1) { - std::memset(dest + base + row * width, 0, width); - std::memset(dest + base + tileRows * width + row * (width / 32), 127, - width / 32); - continue; - } - alignas(16) uint32_t staging[1024]; - alignas(16) uint8_t packed[1056]; - for (int64_t begin = 0; begin < width; begin += 1024) { - int64_t count = std::min(1024, width - begin); - int64_t r = mr * tileRows + row, c = kr * tileK + begin; - int64_t valid = - r < m ? std::max(0, std::min(count, k - c)) : 0; - const uint32_t *values; - if (valid == count && columnStride == 1) { - values = - reinterpret_cast(source + r * rowStride + c); - } else { - for (int64_t i = 0; i < valid; ++i) - staging[i] = __builtin_bit_cast( - uint32_t, source[r * rowStride + (c + i) * columnStride]); - std::fill(staging + valid, staging + count, 0); - values = staging; - } - mxfp8_quant(values, packed, count); - int64_t codes = base + row * width + begin; - int64_t scales = - base + tileRows * width + row * (width / 32) + begin / 32; - if (outputStride == 1) { - std::memcpy(dest + codes, packed, count); - std::memcpy(dest + scales, packed + count, count / 32); - } else { - for (int64_t i = 0; i < count; ++i) - dest[(codes + i) * outputStride] = packed[i]; - for (int64_t i = 0; i < count / 32; ++i) - dest[(scales + i) * outputStride] = packed[count + i]; - } - } - } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/quant.cpp b/bb-tests/workloads/lib/bbsw/kernels/mxfp8/quant.cpp deleted file mode 100644 index 258d63c20..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/quant.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "quant.h" -#include -#include -#include - -static uint32_t roundedShift(uint32_t value, unsigned shift) { - if (shift > 31) - return 0; - uint32_t half = uint32_t(1) << (shift - 1); - return (value + half - 1 + ((value >> shift) & 1)) >> shift; -} - -static uint8_t encode(uint32_t bits, int blockExponent) { - uint8_t sign = (bits >> 24) & 128; - if (!(bits & 0x7fffffff)) - return sign; - int exponent = int((bits >> 23) & 255); - uint32_t fraction = bits & 0x7fffff; - if (exponent == 0) { - int shift = __builtin_clz(fraction) - 8; - fraction = (fraction << shift) & 0x7fffff; - exponent = 1 - shift; - } - exponent -= blockExponent + 120; - uint32_t code = - exponent <= 0 ? roundedShift(fraction | 0x800000, unsigned(21 - exponent)) - : unsigned(exponent * 8) + roundedShift(fraction, 20); - return sign | std::min(code, 126u); -} - -extern "C" void mxfp8_quant(const uint32_t *input, uint8_t *output, - uint32_t count) { - for (uint32_t begin = 0; begin < count; begin += 32) { - uint32_t maximum = 0; - for (uint32_t i = 0; i < 32; ++i) - maximum = std::max(maximum, input[begin + i] & 0x7fffffff); - if (maximum >= 0x7f800000) { - fputs("mxfp8: non-finite activation\n", stderr); - abort(); - } - int exponent = maximum == 0 ? 0 : std::max(-127, int(maximum >> 23) - 135); - output[count + begin / 32] = uint8_t(exponent + 127); - for (uint32_t i = 0; i < 32; ++i) - output[begin + i] = encode(input[begin + i], exponent); - } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/quant.h b/bb-tests/workloads/lib/bbsw/kernels/mxfp8/quant.h deleted file mode 100644 index 4d82efd3e..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/mxfp8/quant.h +++ /dev/null @@ -1,5 +0,0 @@ -#pragma once -#include - -extern "C" void mxfp8_quant(const uint32_t *input, uint8_t *output, - uint32_t count); diff --git a/bb-tests/workloads/lib/bbsw/kernels/mxquant/CMakeLists.txt b/bb-tests/workloads/lib/bbsw/kernels/mxquant/CMakeLists.txt deleted file mode 100644 index fcb5dc2fe..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/mxquant/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../.." ABSOLUTE) -add_library(mxquant_backend STATIC dispatch.cpp) -target_link_libraries(mxquant_backend PRIVATE runtime) -target_compile_options(mxquant_backend PRIVATE -O2) -target_compile_features(mxquant_backend PRIVATE cxx_std_20) -target_include_directories(mxquant_backend PRIVATE - ${RUNTIME_PARAMS} "${REPO_ROOT}/bb-tests/workloads/lib" - "${REPO_ROOT}/examples/balls/mxquant/workloads") - -add_library(mxquant_backend_ant STATIC EXCLUDE_FROM_ALL dispatch.cpp) -target_link_libraries(mxquant_backend_ant PRIVATE runtime_ant_tasks) -target_compile_options(mxquant_backend_ant PRIVATE -O2) -target_compile_features(mxquant_backend_ant PRIVATE cxx_std_20) -target_include_directories(mxquant_backend_ant PRIVATE - ${RUNTIME_PARAMS} "${REPO_ROOT}/bb-tests/workloads/lib" - "${REPO_ROOT}/examples/balls/mxquant/workloads") diff --git a/bb-tests/workloads/lib/bbsw/kernels/mxquant/dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/mxquant/dispatch.cpp deleted file mode 100644 index a99ca82b3..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/mxquant/dispatch.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include "../mxfp8/quant.h" -#include -#include - -extern "C" void mxfp8_quant(const uint32_t *input, uint8_t *output, - uint32_t count) { - constexpr unsigned inputBank = 0, outputBank = 1; - bb_mem_alloc(inputBank, 1, 1); - bb_mem_alloc(outputBank, 1, 1); - bb_mvin((uintptr_t)input, inputBank, count / 4, 1); - bb_mxquant(inputBank, outputBank, count); - bb_mvout((uintptr_t)output, outputBank, (count + count / 32 + 15) / 16, 1); - bb_mem_release(inputBank); - bb_mem_release(outputBank); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/CMakeLists.txt b/bb-tests/workloads/lib/bbsw/kernels/rvv/CMakeLists.txt deleted file mode 100644 index 746874b0c..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/CMakeLists.txt +++ /dev/null @@ -1,57 +0,0 @@ -get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../.." ABSOLUTE) -add_library(rvv_kernels STATIC dispatch.cpp matmul_dispatch.cpp cache_dispatch.cpp normalization_dispatch.cpp pointwise_dispatch.cpp activation_dispatch.cpp flash_attention_dispatch.cpp flash_attention_mxfp8_dispatch.cpp) -target_link_libraries(rvv_kernels PRIVATE runtime rvv_images) -target_compile_options(rvv_kernels PRIVATE -O2) -target_compile_features(rvv_kernels PRIVATE cxx_std_20) -target_include_directories(rvv_kernels PRIVATE - ${RUNTIME_PARAMS} - "${REPO_ROOT}/bb-tests/workloads/lib" - "${REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") - -set(IMAGE_SOURCE "${CMAKE_CURRENT_BINARY_DIR}/images.cpp") -set(IMAGE_HEADER "${CMAKE_CURRENT_BINARY_DIR}/images.h") -add_custom_command( - OUTPUT "${IMAGE_SOURCE}" "${IMAGE_HEADER}" - BYPRODUCTS silu.o snake.o quant.o dequant.o matmul.o norm.o softmax.o rope.o exp.o sin.o cos.o - layernorm.o gelu.o tanh.o erf.o pointwise.o layernorm.elf gelu.elf tanh.elf pointwise.elf - trig.o sin.elf cos.elf flash_attention.o flash_attention.elf - flash_attention_mxfp8.o flash_attention_mxfp8.elf - silu.elf swiglu.elf snake.elf quant.elf cache_quant.elf dequant.elf matmul.elf norm.elf norm_no_weight.elf softmax.elf attention_softmax.elf logsumexp_softmax.elf rope.elf - COMMAND "${PYTHON}" "${CMAKE_CURRENT_SOURCE_DIR}/build_images.py" - --output "${CMAKE_CURRENT_BINARY_DIR}" - DEPENDS build_images.py kernel.ld kernels.h silu.cpp snake.cpp quant.cpp dequant.cpp matmul.cpp - norm.cpp softmax.cpp rope.cpp layernorm.cpp gelu.cpp tanh.cpp trig.cpp pointwise.cpp math/erf.cpp - flash_attention.cpp flash_attention.h flash_attention_mxfp8.cpp math/mxfp8.h - math/exp.cpp math/sin.cpp math/cos.cpp math/math.h math/constants.h - VERBATIM) -add_custom_target(rvv-images DEPENDS "${IMAGE_SOURCE}" "${IMAGE_HEADER}") -add_library(rvv_images STATIC "${IMAGE_SOURCE}") -target_compile_features(rvv_images PRIVATE cxx_std_20) -add_dependencies(rvv_images rvv-images) -target_include_directories(rvv_images PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") -target_include_directories(rvv_kernels PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") - -add_library(rvv_quant_backend STATIC quant_dispatch.cpp) -target_link_libraries(rvv_quant_backend PRIVATE runtime rvv_images) -target_compile_options(rvv_quant_backend PRIVATE -O2) -target_compile_features(rvv_quant_backend PRIVATE cxx_std_20) -target_include_directories(rvv_quant_backend PRIVATE - ${RUNTIME_PARAMS} "${REPO_ROOT}/bb-tests/workloads/lib" - "${CMAKE_CURRENT_BINARY_DIR}") -add_dependencies(rvv_quant_backend rvv-images) - -add_library(rvv_kernels_ant STATIC EXCLUDE_FROM_ALL dispatch.cpp matmul_dispatch.cpp cache_dispatch.cpp normalization_dispatch.cpp pointwise_dispatch.cpp activation_dispatch.cpp flash_attention_dispatch.cpp flash_attention_mxfp8_dispatch.cpp) -target_link_libraries(rvv_kernels_ant PRIVATE runtime_ant_tasks rvv_images) -target_compile_options(rvv_kernels_ant PRIVATE -O2) -target_compile_features(rvv_kernels_ant PRIVATE cxx_std_20) -target_include_directories(rvv_kernels_ant PRIVATE ${RUNTIME_PARAMS} - "${REPO_ROOT}/bb-tests/workloads/lib" "${CMAKE_CURRENT_BINARY_DIR}" - "${REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") -add_dependencies(rvv_kernels_ant rvv-images) -add_library(rvv_quant_backend_ant STATIC EXCLUDE_FROM_ALL quant_dispatch.cpp) -target_link_libraries(rvv_quant_backend_ant PRIVATE runtime_ant_tasks rvv_images) -target_compile_options(rvv_quant_backend_ant PRIVATE -O2) -target_compile_features(rvv_quant_backend_ant PRIVATE cxx_std_20) -target_include_directories(rvv_quant_backend_ant PRIVATE ${RUNTIME_PARAMS} - "${REPO_ROOT}/bb-tests/workloads/lib" "${CMAKE_CURRENT_BINARY_DIR}") -add_dependencies(rvv_quant_backend_ant rvv-images) diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/activation_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/activation_dispatch.cpp deleted file mode 100644 index d6f74ee35..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/activation_dispatch.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include "images.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 3, writeBank = 4; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr size_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); - -int64_t offset(const DynamicMemRefType &value, int64_t index) { - int64_t result = value.offset; - for (int64_t axis = value.rank - 1; axis >= 0; --axis) { - result += (index % value.sizes[axis]) * value.strides[axis]; - index /= value.sizes[axis]; - } - return result; -} -} // namespace - -static void unary(UnrankedMemRefType *output, - UnrankedMemRefType *input, - const images::KernelImage &image) { - DynamicMemRefType out(*output), in(*input); - if (out.rank != in.rank) - abort(); - int64_t count = 1; - for (int64_t axis = in.rank - 1; axis >= 0; --axis) { - if (in.sizes[axis] <= 0 || out.sizes[axis] != in.sizes[axis]) - abort(); - count *= in.sizes[axis]; - } - alignas(16) float source[bankBytes / sizeof(float)]; - alignas(16) float result[bankBytes / sizeof(float)]; - for (unsigned bank = 0; bank < 3; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mem_transfer(0, readBank); - bb_mem_transfer(1, writeBank); - bb_mem_transfer(2, writeBank); - mvin_kernel(image.bytes, image.size, programBank); - uint32_t rounding; - asm volatile("csrr %0, frm" : "=r"(rounding)::"memory"); - for (int64_t begin = 0; begin < count; begin += bankBytes / sizeof(float)) { - uint32_t length = - std::min(bankBytes / sizeof(float), count - begin); - for (uint32_t i = 0; i < length; ++i) - std::memcpy(source + i, in.data + offset(in, begin + i), sizeof(float)); - std::fill(source + length, source + (length + 3) / 4 * 4, 0.0f); - bb_mvin_group((uintptr_t)source, readBank, 0, (length + 3) / 4, 1); - alignas(16) kernel_launch packet{ - image.entry, - image.text_bytes, - 0x40002000, - {uint64_t(2) << 32, 0, length, - (uint64_t(1) << 32) + offsetof(kernel_launch, reserved), - rounding << 5}, - 0}; - bb_mvin_group((uintptr_t)&packet, writeBank, 0, sizeof(packet) / 16, 1); - run_kernel(readBank, programBank, writeBank, 0); - bb_mvout_group((uintptr_t)result, writeBank, 1, (length + 3) / 4, 1); - bb_mvout_group((uintptr_t)&packet, writeBank, 0, sizeof(packet) / 16, 1); - asm volatile("csrs fflags, %0" ::"r"(packet.reserved) : "memory"); - for (uint32_t i = 0; i < length; ++i) - std::memcpy(out.data + offset(out, begin + i), result + i, sizeof(float)); - } - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} - -extern "C" void _mlir_ciface_rvv_tanh(UnrankedMemRefType *output, - UnrankedMemRefType *input) { - unary(output, input, images::tanh); -} -extern "C" void _mlir_ciface_rvv_sin(UnrankedMemRefType *output, - UnrankedMemRefType *input) { - unary(output, input, images::sin); -} -extern "C" void _mlir_ciface_rvv_cos(UnrankedMemRefType *output, - UnrankedMemRefType *input) { - unary(output, input, images::cos); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/build_images.py b/bb-tests/workloads/lib/bbsw/kernels/rvv/build_images.py deleted file mode 100644 index 5f4537eb6..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/build_images.py +++ /dev/null @@ -1,198 +0,0 @@ -import argparse -from pathlib import Path -import struct -import subprocess - -KERNELS = { - "silu": "rvv_silu_launch", - "swiglu": "rvv_swiglu_launch", - "snake": "rvv_snake", - "quant": "rvv_quant", - "cache_quant": "rvv_cache_quant", - "dequant": "rvv_dequant", - "matmul": "rvv_matmul", - "norm": "rvv_norm_launch", - "norm_no_weight": "rvv_norm_no_weight_launch", - "softmax": "rvv_softmax_launch", - "attention_softmax": "rvv_attention_softmax_launch", - "logsumexp_softmax": "rvv_logsumexp_softmax_launch", - "rope": "rvv_rope_launch", - "layernorm": "rvv_layernorm", - "gelu": "rvv_gelu", - "tanh": "rvv_tanh_launch", - "sin": "rvv_sin_launch", - "cos": "rvv_cos_launch", - "flash_attention": "rvv_flash_attention_launch", - "flash_attention_mxfp8": "rvv_flash_attention_mxfp8_launch", - "pointwise": "rvv_pointwise", -} - - -def read_segments(path): - elf = path.read_bytes() - header = struct.unpack_from("<16sHHIQQQIHHHHHH", elf) - ident, kind, machine = header[:3] - entry, shoff, shsize, shnum, names_index = ( - header[4], - header[6], - header[11], - header[12], - header[13], - ) - if ident[:6] != b"\x7fELF\x02\x01" or kind != 2 or machine != 243: - raise ValueError(f"{path}: expected a linked little-endian RISC-V ELF64") - sections = [ - struct.unpack_from(" 4096 - or text[5] % 4 - or entry >= text[5] - or entry % 4 - ): - raise ValueError(f"{path}: invalid RVV instruction memory layout") - data_address = 0x40000000 - initialized = b"" - if ".rodata" in sections and sections[".rodata"][5]: - section = sections[".rodata"] - if section[1] != 1 or section[3] != data_address or section[5] > 4096: - raise ValueError(f"{path}: invalid RVV constant memory layout") - initialized = elf[section[4] : section[4] + section[5]] - initialized += bytes((-len(initialized)) % 4) - return elf[text[4] : text[4] + text[5]], initialized, entry - - -def array(name, content): - rows = [ - " " + ", ".join(f"0x{value:02x}" for value in content[offset : offset + 16]) - for offset in range(0, len(content), 16) - ] - return f"alignas(4) const uint8_t {name}[] = {{\n" + ",\n".join(rows) + "\n};\n" - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--output", required=True, type=Path) - args = parser.parse_args() - source = Path(__file__).resolve().parent - tools = source.parents[5] / "result/bin" - args.output.mkdir(parents=True, exist_ok=True) - objects = [] - for filename in ( - "silu.cpp", - "snake.cpp", - "quant.cpp", - "dequant.cpp", - "matmul.cpp", - "norm.cpp", - "softmax.cpp", - "rope.cpp", - "layernorm.cpp", - "gelu.cpp", - "tanh.cpp", - "trig.cpp", - "flash_attention.cpp", - "flash_attention_mxfp8.cpp", - "pointwise.cpp", - "math/erf.cpp", - "math/exp.cpp", - "math/sin.cpp", - "math/cos.cpp", - ): - output = args.output / (Path(filename).stem + ".o") - subprocess.run( - [ - str(tools / "clang++"), - "--target=riscv64", - "-march=rv64imf_zve32f_zvl128b", - "-mabi=lp64f", - "-mcmodel=medany", - "-ffreestanding", - "-O2", - "-ffp-contract=off", - "-ffunction-sections", - "-fdata-sections", - *( - ["-fno-vectorize", "-fno-slp-vectorize"] - if filename == "math/erf.cpp" - else [] - ), - "-c", - str(source / filename), - "-o", - str(output), - ], - check=True, - ) - objects.append(output) - arrays = [] - images = [] - for name, symbol in KERNELS.items(): - path = args.output / (name + ".elf") - subprocess.run( - [ - str(tools / "riscv64-unknown-elf-ld"), - "-m", - "elf64lriscv", - "--gc-sections", - "-e", - symbol, - "-T", - str(source / "kernel.ld"), - *map(str, objects), - "-o", - str(path), - ], - check=True, - ) - text, data, entry = read_segments(path) - image = ( - struct.pack("<6I", 0x31564B52, len(text), entry, 0x40000000, len(data), 0) - + text - + data - ) - arrays.append(array(name + "_image", image)) - images.append( - f"const KernelImage {name}{{{name}_image, sizeof({name}_image), " - f"{entry}, {len(text)}}};" - ) - print( - f"{name}: image={len(image)} text={len(text)} data={len(data)} " - f"bss=0 entry={entry}" - ) - (args.output / "images.h").write_text( - "#pragma once\n#include \n\nnamespace images {\n" - "struct KernelImage {\n" - " const uint8_t *bytes;\n uint32_t size;\n" - " uint32_t entry;\n uint32_t text_bytes;\n};\n" - + "".join(f"extern const KernelImage {name};\n" for name in KERNELS) - + "}\n" - ) - (args.output / "images.cpp").write_text( - '#include "images.h"\n\nnamespace images {\nnamespace {\n' - + "\n".join(arrays) - + "}\n" - + "\n".join(images) - + "\n}\n" - ) - - -if __name__ == "__main__": - main() diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/cache_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/cache_dispatch.cpp deleted file mode 100644 index a78d85c93..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/cache_dispatch.cpp +++ /dev/null @@ -1,179 +0,0 @@ -#include "images.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 4, writeBank = 5; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr uint32_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); -constexpr int64_t chunk = bankBytes / 128 * 32; -static_assert(chunk > 0 && bankBytes <= 65536); - -template int64_t elements(const DynamicMemRefType &value) { - if (!value.rank || value.sizes[value.rank - 1] <= 0) - abort(); - int64_t count = 1; - for (int64_t axis = 0; axis < value.rank; ++axis) { - if (value.sizes[axis] <= 0) - abort(); - count *= value.sizes[axis]; - } - return count; -} - -template -void copyRows(DynamicMemRefType &value, T *packed, int64_t first, - int64_t count, bool write) { - const int64_t width = value.sizes[value.rank - 1]; - const int64_t stride = value.strides[value.rank - 1]; - for (int64_t done = 0; done < count;) { - int64_t index = first + done; - const int64_t length = std::min(count - done, width - index % width); - int64_t offset = value.offset; - for (int64_t axis = value.rank - 1; axis >= 0; --axis) { - offset += index % value.sizes[axis] * value.strides[axis]; - index /= value.sizes[axis]; - } - T *row = value.data + offset; - if (stride == 1) { - if (write) - std::memcpy(row, packed + done, length * sizeof(T)); - else - std::memcpy(packed + done, row, length * sizeof(T)); - } else { - for (int64_t column = 0; column < length; ++column) { - if (write) - row[column * stride] = packed[done + column]; - else - packed[done + column] = row[column * stride]; - } - } - done += length; - } -} - -void check(DynamicMemRefType &codes, - DynamicMemRefType &scales) { - if (codes.rank != scales.rank || codes.sizes[codes.rank - 1] % 32) - abort(); - for (int64_t axis = 0; axis < codes.rank; ++axis) - if (scales.sizes[axis] != - codes.sizes[axis] / (axis == codes.rank - 1 ? 32 : 1)) - abort(); -} - -void load(const images::KernelImage &image, bool encoding) { - for (unsigned bank = 0; bank < 4; ++bank) - bb_mem_alloc(bank, 1, 1); - if (encoding) { - bb_mem_transfer(3, readBank); - bb_mem_transfer(0, writeBank); - bb_mem_transfer(1, writeBank); - bb_mem_transfer(2, writeBank); - } else { - bb_mem_transfer(2, readBank); - bb_mem_transfer(3, readBank); - bb_mem_transfer(0, writeBank); - bb_mem_transfer(1, writeBank); - } - mvin_kernel(image.bytes, image.size, programBank); -} - -void launch(const images::KernelImage &image, uint32_t count, bool encoding) { - const uint64_t descriptor = uint64_t(encoding ? 1 : 2) << 32; - alignas(16) kernel_launch packet{}; - packet.entry = image.entry; - packet.end = image.text_bytes; - packet.stack = 0x40002000; - packet.args[0] = uint64_t(encoding ? 2 : 3) << 32; - packet.args[1] = encoding ? uint64_t(3) << 32 : 0; - packet.args[2] = encoding ? 0 : uint64_t(1) << 32; - packet.args[3] = count; - packet.args[4] = descriptor + offsetof(kernel_launch, reserved); - bb_mvin_group((uintptr_t)&packet, writeBank, 0, sizeof(packet) / 16, 1); - run_kernel(readBank, programBank, writeBank, 0); - alignas(16) kernel_launch completed; - bb_mvout_group((uintptr_t)&completed, writeBank, 0, sizeof(completed) / 16, - 1); - if (std::memcmp(&packet, &completed, offsetof(kernel_launch, reserved)) || - completed.reserved) { - fputs("MXFP8 cache kernel rejected non-finite values, NaN encoding, or " - "FP32 overflow\n", - stderr); - abort(); - } -} - -void release() { - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} -} // namespace - -extern "C" void -_mlir_ciface_rvv_mxfp8_encode(UnrankedMemRefType *codes, - UnrankedMemRefType *scales, - UnrankedMemRefType *input) { - DynamicMemRefType c(*codes), s(*scales); - DynamicMemRefType in(*input); - check(c, s); - int64_t count = elements(in); - if (elements(c) != count || elements(s) != count / 32 || in.rank != c.rank) - abort(); - for (int64_t axis = 0; axis < in.rank; ++axis) - if (in.sizes[axis] != c.sizes[axis]) - abort(); - alignas(16) float staging[chunk]; - alignas(16) int8_t codeData[chunk]; - alignas(16) int8_t scaleData[(chunk / 32 + 15) / 16 * 16]; - load(images::cache_quant, true); - for (int64_t begin = 0; begin < count; begin += chunk) { - uint32_t length = std::min(chunk, count - begin); - copyRows(in, staging, begin, length, false); - bb_mvin_group((uintptr_t)staging, readBank, 0, length / 4, 1); - launch(images::cache_quant, length, true); - bb_mvout_group((uintptr_t)codeData, writeBank, 1, length / 16, 1); - bb_mvout_group((uintptr_t)scaleData, writeBank, 2, (length / 32 + 15) / 16, - 1); - copyRows(c, codeData, begin, length, true); - copyRows(s, scaleData, begin / 32, length / 32, true); - } - release(); -} - -extern "C" void -_mlir_ciface_rvv_mxfp8_decode(UnrankedMemRefType *output, - UnrankedMemRefType *codes, - UnrankedMemRefType *scales) { - DynamicMemRefType out(*output); - DynamicMemRefType c(*codes), s(*scales); - check(c, s); - int64_t count = elements(c); - if (elements(out) != count || elements(s) != count / 32 || out.rank != c.rank) - abort(); - for (int64_t axis = 0; axis < out.rank; ++axis) - if (out.sizes[axis] != c.sizes[axis]) - abort(); - alignas(16) float staging[chunk]; - alignas(16) int8_t codeData[chunk]; - alignas(16) int8_t scaleData[(chunk / 32 + 15) / 16 * 16]{}; - load(images::dequant, false); - for (int64_t begin = 0; begin < count; begin += chunk) { - uint32_t length = std::min(chunk, count - begin); - copyRows(c, codeData, begin, length, false); - copyRows(s, scaleData, begin / 32, length / 32, false); - bb_mvin_group((uintptr_t)codeData, readBank, 0, length / 16, 1); - bb_mvin_group((uintptr_t)scaleData, readBank, 1, (length / 32 + 15) / 16, - 1); - launch(images::dequant, length, false); - bb_mvout_group((uintptr_t)staging, writeBank, 1, length / 4, 1); - copyRows(out, staging, begin, length, true); - } - release(); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/dequant.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/dequant.cpp deleted file mode 100644 index c83e0ea40..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/dequant.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include "kernels.h" -#include - -extern "C" void rvv_dequant(uint32_t *output, const uint8_t *input, - const uint8_t *scales, size_t count, - uint32_t *status) { - *status = 0; - for (size_t block = 0; block < count / 32; ++block) { - unsigned scale = scales[block]; - if (scale == 255) { - *status = 1; - return; - } - for (size_t begin = 0; begin < 32;) { - size_t vl = __riscv_vsetvl_e32m1(32 - begin); - auto bytes = __riscv_vle8_v_u8mf4(input + block * 32 + begin, vl); - auto code = - __riscv_vzext_vf2_u32m1(__riscv_vzext_vf2_u16mf2(bytes, vl), vl); - auto sign = - __riscv_vsll_vx_u32m1(__riscv_vand_vx_u32m1(code, 128, vl), 24, vl); - auto magnitude = __riscv_vand_vx_u32m1(code, 127, vl); - if (__riscv_vcpop_m_b32(__riscv_vmseq_vx_u32m1_b32(magnitude, 127, vl), - vl)) { - *status = 1; - return; - } - auto exponent = __riscv_vsrl_vx_u32m1(magnitude, 3, vl); - auto fraction = __riscv_vand_vx_u32m1(code, 7, vl); - auto subnormal = __riscv_vmseq_vx_u32m1_b32(exponent, 0, vl); - auto highest = __riscv_vmv_v_x_u32m1(0, vl); - highest = __riscv_vmerge_vxm_u32m1( - highest, 1, __riscv_vmsgeu_vx_u32m1_b32(fraction, 2, vl), vl); - highest = __riscv_vmerge_vxm_u32m1( - highest, 2, __riscv_vmsgeu_vx_u32m1_b32(fraction, 4, vl), vl); - auto normalExponent = __riscv_vadd_vx_i32m1( - __riscv_vreinterpret_v_u32m1_i32m1(exponent), int(scale) - 7, vl); - auto subExponent = __riscv_vadd_vx_i32m1( - __riscv_vreinterpret_v_u32m1_i32m1(highest), int(scale) - 9, vl); - auto resultExponent = - __riscv_vmerge_vvm_i32m1(normalExponent, subExponent, subnormal, vl); - auto overflow = __riscv_vmand_mm_b32( - __riscv_vmsge_vx_i32m1_b32(resultExponent, 255, vl), - __riscv_vmsne_vx_u32m1_b32(magnitude, 0, vl), vl); - if (__riscv_vcpop_m_b32(overflow, vl)) { - *status = 2; - return; - } - auto normalSignificand = - __riscv_vsll_vx_u32m1(__riscv_vor_vx_u32m1(fraction, 8, vl), 20, vl); - auto subSignificand = __riscv_vsll_vv_u32m1( - fraction, __riscv_vrsub_vx_u32m1(highest, 23, vl), vl); - auto significand = __riscv_vmerge_vvm_u32m1( - normalSignificand, subSignificand, subnormal, vl); - auto normalBits = __riscv_vor_vv_u32m1( - __riscv_vsll_vx_u32m1( - __riscv_vreinterpret_v_i32m1_u32m1(resultExponent), 23, vl), - __riscv_vand_vx_u32m1(significand, 0x7fffff, vl), vl); - auto tinyBits = __riscv_vsrl_vv_u32m1( - significand, - __riscv_vreinterpret_v_i32m1_u32m1( - __riscv_vrsub_vx_i32m1(resultExponent, 1, vl)), - vl); - auto bits = __riscv_vmerge_vvm_u32m1( - normalBits, tinyBits, - __riscv_vmsle_vx_i32m1_b32(resultExponent, 0, vl), vl); - bits = __riscv_vmerge_vxm_u32m1( - bits, 0, __riscv_vmseq_vx_u32m1_b32(magnitude, 0, vl), vl); - __riscv_vse32_v_u32m1(output + block * 32 + begin, - __riscv_vor_vv_u32m1(bits, sign, vl), vl); - begin += vl; - } - } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/dispatch.cpp deleted file mode 100644 index 28ab8222d..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/dispatch.cpp +++ /dev/null @@ -1,442 +0,0 @@ -#include "images.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 4, writeBank = 5; -constexpr unsigned outputBank = 3, inputBank = 0, upBank = 1; -constexpr uint64_t descriptorAddress = uint64_t(2) << 32; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr int64_t chunk = 1024; -constexpr uint32_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); -static_assert(bankBytes >= 4096 && bankBytes <= 65536 && bankBytes % 16 == 0); - -int64_t elements(const DynamicMemRefType &value) { - int64_t count = 1; - for (int64_t axis = value.rank - 1; axis >= 0; --axis) { - if (value.sizes[axis] <= 0 || - (value.sizes[axis] != 1 && value.strides[axis] != count)) { - fputs("RVV kernel requires a positive contiguous FP32 tensor\n", stderr); - abort(); - } - count *= value.sizes[axis]; - } - return count; -} - -void load(const images::KernelImage &image, unsigned buffer) { - for (unsigned bank = 0; bank < 4; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mem_transfer(2, readBank); - bb_mem_transfer(3, readBank); - bb_mem_transfer(0, writeBank); - bb_mem_transfer(1, writeBank); - mvin_kernel(image.bytes, image.size, programBank + buffer); -} - -uint32_t roundingMode() { - uint32_t rounding; - asm volatile("csrr %0, frm" : "=r"(rounding)::"memory"); - return rounding << 5; -} - -void launch(const void *descriptor, uint32_t bytes, unsigned buffer) { - bb_mvin_group((uintptr_t)descriptor, writeBank, 0, bytes / 16, 1); - run_kernel(readBank, programBank + buffer, writeBank, 0); -} - -void send(const float *input, unsigned bank, uint32_t count, float *staging) { - if (count % 4) { - std::memcpy(staging, input, count * sizeof(float)); - std::fill_n(staging + count, 4 - count % 4, 0.0f); - input = staging; - } - bb_mvin_group((uintptr_t)input, readBank, bank, (count + 3) / 4, 1); -} - -void receive(float *output, uint32_t count, float *staging) { - bb_mvout_group((uintptr_t)(count % 4 ? staging : output), writeBank, 1, - (count + 3) / 4, 1); - if (count % 4) - std::memcpy(output, staging, count * sizeof(float)); -} - -// Tensor launch entries return private exception flags in the launch descriptor -// reserved field. -void receiveTensor(float *output, uint32_t count, float *staging) { - receive(output, count, staging); - alignas(16) kernel_launch descriptor; - bb_mvout_group((uintptr_t)&descriptor, writeBank, 0, sizeof(descriptor) / 16, - 1); - asm volatile("csrs fflags, %0" ::"r"(descriptor.reserved) : "memory"); -} - -void release(unsigned buffer) { - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank + buffer); -} -} // namespace - -extern "C" void _mlir_ciface_rvv_silu(UnrankedMemRefType *output, - UnrankedMemRefType *input) { - DynamicMemRefType out(*output), in(*input); - int64_t count = elements(in); - if (elements(out) != count) - abort(); - alignas(16) float staging[chunk]; - const uint32_t rounding = roundingMode(); - load(images::silu, 0); - for (int64_t begin = 0; begin < count; begin += chunk) { - uint32_t length = std::min(chunk, count - begin); - send(in.data + in.offset + begin, inputBank, length, staging); - kernel_launch call{ - images::silu.entry, - images::silu.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, length, - descriptorAddress + offsetof(kernel_launch, reserved), rounding}, - 0}; - launch(&call, sizeof(call), 0); - receiveTensor(out.data + out.offset + begin, length, staging); - } - release(0); -} - -extern "C" void _mlir_ciface_rvv_swiglu(UnrankedMemRefType *output, - UnrankedMemRefType *gate, - UnrankedMemRefType *up) { - DynamicMemRefType out(*output), a(*gate), b(*up); - int64_t count = elements(a); - if (elements(out) != count || elements(b) != count) - abort(); - alignas(16) float staging[chunk]; - const uint32_t rounding = roundingMode(); - load(images::swiglu, 0); - for (int64_t begin = 0; begin < count; begin += chunk) { - uint32_t length = std::min(chunk, count - begin); - send(a.data + a.offset + begin, inputBank, length, staging); - send(b.data + b.offset + begin, upBank, length, staging); - kernel_launch call{images::swiglu.entry, - images::swiglu.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, - uint64_t(upBank) << 32, length, - descriptorAddress + offsetof(kernel_launch, reserved), - rounding}, - 0}; - launch(&call, sizeof(call), 0); - receiveTensor(out.data + out.offset + begin, length, staging); - } - release(0); -} - -extern "C" void _mlir_ciface_rvv_snake(UnrankedMemRefType *output, - UnrankedMemRefType *input, - UnrankedMemRefType *logAlpha, - UnrankedMemRefType *logBeta) { - DynamicMemRefType out(*output), in(*input), alpha(*logAlpha), - beta(*logBeta); - int64_t channels = elements(alpha), count = elements(in); - if (elements(beta) != channels || elements(out) != count || count % channels) - abort(); - int64_t width = count / channels; - alignas(16) float staging[chunk]; - load(images::snake, 1); - // Short channels fit together in one input bank and one coefficient packet. - const int64_t batch = - width <= chunk - ? std::min(chunk / width, - (bankBytes - sizeof(kernel_launch)) / 8) - : 1; - alignas(16) uint8_t descriptor[bankBytes]; - for (int64_t channel = 0; channel < channels; channel += batch) { - uint32_t grouped = std::min(batch, channels - channel); - for (int64_t begin = 0; begin < width; begin += chunk) { - uint32_t length = std::min(chunk, width - begin); - int64_t offset = channel * width + begin; - uint32_t elements = grouped * length; - send(in.data + in.offset + offset, inputBank, elements, staging); - kernel_launch call{ - images::snake.entry, - images::snake.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, grouped, - length, descriptorAddress + sizeof(kernel_launch), - descriptorAddress + sizeof(kernel_launch) + grouped * 4}, - 0}; - uint32_t bytes = (sizeof(call) + grouped * 2 * sizeof(float) + 15) & ~15U; - std::memset(descriptor, 0, bytes); - std::memcpy(descriptor, &call, sizeof(call)); - std::memcpy(descriptor + sizeof(call), - alpha.data + alpha.offset + channel, grouped * sizeof(float)); - std::memcpy(descriptor + sizeof(call) + grouped * sizeof(float), - beta.data + beta.offset + channel, grouped * sizeof(float)); - launch(descriptor, bytes, 1); - receive(out.data + out.offset + offset, elements, staging); - } - } - release(1); -} - -extern "C" void _mlir_ciface_rvv_norm(UnrankedMemRefType *output, - UnrankedMemRefType *input, - UnrankedMemRefType *weight, - uint32_t meanMultiplierBits, - uint32_t epsilonBits) { - DynamicMemRefType out(*output), in(*input), w(*weight); - if (in.rank < 1 || in.strides[in.rank - 1] != 1) - abort(); - int64_t count = 1; - for (int64_t axis = 0; axis < in.rank; ++axis) { - if (in.sizes[axis] <= 0) - abort(); - count *= in.sizes[axis]; - } - if (elements(out) != count) - abort(); - int64_t width = in.sizes[in.rank - 1]; - if (elements(w) != width || width > bankBytes / sizeof(float)) { - fputs("RVV norm requires one complete row and its weights in banks\n", - stderr); - abort(); - } - alignas(16) float staging[bankBytes / sizeof(float)]; - const uint32_t rounding = roundingMode(); - load(images::norm, 0); - send(w.data + w.offset, upBank, width, staging); - for (int64_t begin = 0; begin < count; begin += width) { - int64_t row = begin / width, offset = in.offset; - for (int64_t axis = in.rank - 2; axis >= 0; --axis) { - offset += (row % in.sizes[axis]) * in.strides[axis]; - row /= in.sizes[axis]; - } - send(in.data + offset, inputBank, width, staging); - kernel_launch call{images::norm.entry, - images::norm.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, - uint64_t(upBank) << 32, uint32_t(width), - meanMultiplierBits, epsilonBits, - descriptorAddress + offsetof(kernel_launch, reserved), - rounding}, - 0}; - launch(&call, sizeof(call), 0); - receiveTensor(out.data + out.offset + begin, width, staging); - } - release(0); -} - -extern "C" void _mlir_ciface_rvv_norm_no_weight( - UnrankedMemRefType *output, UnrankedMemRefType *input, - uint32_t meanMultiplierBits, uint32_t epsilonBits) { - DynamicMemRefType out(*output), in(*input); - if (in.rank < 1 || in.strides[in.rank - 1] != 1) - abort(); - int64_t count = 1; - for (int64_t axis = 0; axis < in.rank; ++axis) { - if (in.sizes[axis] <= 0) - abort(); - count *= in.sizes[axis]; - } - if (elements(out) != count) - abort(); - int64_t width = in.sizes[in.rank - 1]; - if (width > bankBytes / sizeof(float)) { - fputs("RVV norm requires one complete row in a bank\n", stderr); - abort(); - } - alignas(16) float staging[bankBytes / sizeof(float)]; - const uint32_t rounding = roundingMode(); - load(images::norm_no_weight, 0); - for (int64_t begin = 0; begin < count; begin += width) { - int64_t row = begin / width, offset = in.offset; - for (int64_t axis = in.rank - 2; axis >= 0; --axis) { - offset += (row % in.sizes[axis]) * in.strides[axis]; - row /= in.sizes[axis]; - } - send(in.data + offset, inputBank, width, staging); - kernel_launch call{images::norm_no_weight.entry, - images::norm_no_weight.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, - uint32_t(width), meanMultiplierBits, epsilonBits, - descriptorAddress + offsetof(kernel_launch, reserved), - rounding}, - 0}; - launch(&call, sizeof(call), 0); - receiveTensor(out.data + out.offset + begin, width, staging); - } - release(0); -} - -static void tensorSoftmax(UnrankedMemRefType *output, - UnrankedMemRefType *input, - const images::KernelImage &image) { - DynamicMemRefType out(*output), in(*input); - int64_t count = elements(in); - if (in.rank < 1 || elements(out) != count) - abort(); - int64_t width = in.sizes[in.rank - 1]; - if (width > bankBytes / sizeof(float)) { - fputs("RVV softmax requires one complete row in a bank\n", stderr); - abort(); - } - int64_t rows = count / width, batch = bankBytes / sizeof(float) / width; - alignas(16) float staging[bankBytes / sizeof(float)]; - const uint32_t rounding = roundingMode(); - load(image, 0); - for (int64_t row = 0; row < rows; row += batch) { - uint32_t grouped = std::min(batch, rows - row), length = grouped * width; - send(in.data + in.offset + row * width, inputBank, length, staging); - kernel_launch call{image.entry, - image.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, - grouped, uint32_t(width), - descriptorAddress + offsetof(kernel_launch, reserved), - rounding}, - 0}; - launch(&call, sizeof(call), 0); - receiveTensor(out.data + out.offset + row * width, length, staging); - } - release(0); -} - -extern "C" void _mlir_ciface_rvv_softmax(UnrankedMemRefType *output, - UnrankedMemRefType *input) { - tensorSoftmax(output, input, images::softmax); -} - -extern "C" void -_mlir_ciface_rvv_logsumexp_softmax(UnrankedMemRefType *output, - UnrankedMemRefType *input) { - tensorSoftmax(output, input, images::logsumexp_softmax); -} - -extern "C" void -_mlir_ciface_rvv_attention_softmax(UnrankedMemRefType *output, - UnrankedMemRefType *input, - UnrankedMemRefType *mask, - uint32_t scaleBits, uint32_t maskedBits) { - DynamicMemRefType out(*output), in(*input); - DynamicMemRefType condition(*mask); - int64_t count = elements(in); - if (in.rank != 4 || in.sizes[0] != 1 || elements(out) != count || - condition.rank != 4 || condition.sizes[0] != 1 || - condition.sizes[1] != 1 || - (condition.sizes[2] != 1 && condition.sizes[2] != in.sizes[2]) || - condition.sizes[3] != in.sizes[3] || condition.strides[3] != 1 || - (condition.sizes[2] != 1 && condition.strides[2] != in.sizes[3])) - abort(); - int64_t width = in.sizes[3]; - int64_t maskQueries = condition.sizes[2]; - if (width > bankBytes / sizeof(float) || maskQueries * width > bankBytes) { - fputs("RVV attention softmax requires one complete row in a bank\n", - stderr); - abort(); - } - int64_t rows = count / width, batch = bankBytes / sizeof(float) / width; - alignas(16) float staging[bankBytes / sizeof(float)]; - alignas(16) uint8_t maskStaging[bankBytes]; - const uint32_t rounding = roundingMode(); - load(images::attention_softmax, 0); - uint32_t maskBytes = maskQueries * width; - std::memcpy(maskStaging, condition.data + condition.offset, maskBytes); - std::memset(maskStaging + maskBytes, 0, (16 - maskBytes % 16) % 16); - bb_mvin_group((uintptr_t)maskStaging, readBank, upBank, (maskBytes + 15) / 16, - 1); - for (int64_t row = 0; row < rows;) { - uint32_t grouped = std::min(batch, rows - row); - uint32_t length = grouped * width; - send(in.data + in.offset + row * width, inputBank, length, staging); - alignas(16) - uint8_t descriptor[sizeof(kernel_launch) + 4 * sizeof(uint32_t)]{}; - kernel_launch call{images::attention_softmax.entry, - images::attention_softmax.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, - uint64_t(upBank) << 32, grouped, uint32_t(width), - descriptorAddress + sizeof(kernel_launch), - descriptorAddress + offsetof(kernel_launch, reserved), - rounding}, - 0}; - std::memcpy(descriptor, &call, sizeof(call)); - uint32_t parameters[]{scaleBits, maskedBits, uint32_t(maskQueries), - uint32_t(row % maskQueries)}; - std::memcpy(descriptor + sizeof(call), parameters, sizeof(parameters)); - launch(descriptor, sizeof(descriptor), 0); - receiveTensor(out.data + out.offset + row * width, length, staging); - row += grouped; - } - release(0); -} - -extern "C" void _mlir_ciface_rvv_rope(UnrankedMemRefType *output, - UnrankedMemRefType *input, - UnrankedMemRefType *frequencies, - UnrankedMemRefType *positions, - uint32_t scaleBits) { - DynamicMemRefType out(*output), in(*input), freq(*frequencies); - DynamicMemRefType pos(*positions); - int64_t count = elements(in); - if (in.rank != 4 || in.sizes[0] != 1 || elements(out) != count) - abort(); - int64_t heads = in.sizes[1], rows = in.sizes[2], width = in.sizes[3]; - if (width % 2 || elements(freq) != width / 2 || pos.rank != 1 || - pos.sizes[0] != rows || pos.strides[0] != 1 || - width > bankBytes / sizeof(float)) { - fputs( - "RVV RoPE requires complete even-width heads and matching positions\n", - stderr); - abort(); - } - int64_t batch = - std::min(bankBytes / sizeof(float) / width, - (bankBytes - sizeof(kernel_launch)) / sizeof(int32_t)); - alignas(16) float staging[bankBytes / sizeof(float)]; - alignas(16) uint8_t descriptor[bankBytes]; - const uint32_t rounding = roundingMode(); - load(images::rope, 0); - send(freq.data + freq.offset, upBank, width / 2, staging); - const int64_t rowChunk = std::min(batch, rows); - const int64_t headChunk = bankBytes / sizeof(float) / (rowChunk * width); - for (int64_t head = 0; head < heads; head += headChunk) - for (int64_t row = 0; row < rows; row += rowChunk) { - uint32_t groupedRows = std::min(rowChunk, rows - row); - uint32_t groupedHeads = std::min(headChunk, heads - head); - uint32_t length = groupedHeads * groupedRows * width; - int64_t begin = (head * rows + row) * width; - send(in.data + in.offset + begin, inputBank, length, staging); - kernel_launch call{ - images::rope.entry, - images::rope.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, - uint64_t(upBank) << 32, descriptorAddress + sizeof(kernel_launch), - groupedRows, - (uint64_t(scaleBits) << 32) | (groupedHeads << 16) | uint32_t(width), - descriptorAddress + offsetof(kernel_launch, reserved), rounding}, - 0}; - uint32_t bytes = - (sizeof(call) + groupedRows * sizeof(int32_t) + 15) & ~15U; - std::memset(descriptor, 0, bytes); - std::memcpy(descriptor, &call, sizeof(call)); - auto packedPositions = - reinterpret_cast(descriptor + sizeof(call)); - for (size_t index = 0; index < groupedRows; ++index) { - const int64_t position = pos.data[pos.offset + row + index]; - if (position < INT32_MIN || position > INT32_MAX) - abort(); - packedPositions[index] = static_cast(position); - } - launch(descriptor, bytes, 0); - receiveTensor(out.data + out.offset + begin, length, staging); - } - release(0); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention.cpp deleted file mode 100644 index 55ecfa411..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention.cpp +++ /dev/null @@ -1,131 +0,0 @@ -#include "flash_attention.h" -#include "math/math.h" - -static float exponential(float value) { - return __riscv_vfmv_f_s_f32m1_f32( - rvv_exp(__riscv_vfmv_v_f_f32m1(value, 1), 1)); -} - -extern "C" void -rvv_flash_attention_launch(const float *query, const float *panel, - unsigned char *descriptor, float *output, - float *blockOutput, uint64_t dimensions, - uint64_t parameters, uint32_t *flags) { - const unsigned phase = dimensions & 255, rows = (dimensions >> 8) & 255; - const unsigned width = (dimensions >> 16) & 65535; - const unsigned begin = (dimensions >> 32) & 255; - const unsigned count = (dimensions >> 40) & 255; - const unsigned keyCount = (dimensions >> 48) & 255; - const uint32_t rounding = parameters >> 32; - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - union { - uint32_t bits; - float value; - } scale{uint32_t(parameters)}; - auto *state = - reinterpret_cast(descriptor + rvv_flash::stateOffset); - auto *mask = reinterpret_cast(descriptor + rvv_flash::maskOffset); - if (phase == 0) { - for (unsigned row = 0; row < rows; ++row) { - state->sum[row] = 0; - state->maximum[row] = -1.0e30f; - for (unsigned d = 0; d < width; d += rvv_flash::lanes) - __riscv_vse32_v_f32m2(output + row * width + d, - __riscv_vfmv_v_f_f32m2(0, rvv_flash::lanes), - rvv_flash::lanes); - } - } else if (phase == 1) { - for (unsigned row = 0; row < rows; ++row) - for (unsigned k = 0; k < count; ++k) { - auto partial = __riscv_vfmv_v_f_f32m2(0, rvv_flash::lanes); - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto q = - __riscv_vle32_v_f32m2(query + row * width + d, rvv_flash::lanes); - auto key = - __riscv_vle32_v_f32m2(panel + k * width + d, rvv_flash::lanes); - partial = __riscv_vfmacc_vv_f32m2(partial, q, key, rvv_flash::lanes); - } - auto sum = __riscv_vfredosum_vs_f32m2_f32m1( - partial, __riscv_vfmv_v_f_f32m1(0, 1), rvv_flash::lanes); - float score = __riscv_vfmv_f_s_f32m1_f32(sum) * scale.value; - state->scores[row * rvv_flash::keys + begin + k] = - score + mask[row * rvv_flash::lanes + k]; - } - } else if (phase == 2) { - for (unsigned row = 0; row < rows; ++row) { - float maximum = -1.0e30f; - for (unsigned k = 0; k < keyCount; ++k) { - float score = state->scores[row * rvv_flash::keys + k]; - if (score > maximum) - maximum = score; - } - float sum = 0; - for (unsigned k = 0; k < keyCount; ++k) { - float p = - exponential(state->scores[row * rvv_flash::keys + k] - maximum); - state->scores[row * rvv_flash::keys + k] = p; - sum = sum + p; - } - state->blockMaximum[row] = maximum; - state->blockSum[row] = sum; - float merged = - maximum > state->maximum[row] ? maximum : state->maximum[row]; - state->mergedMaximum[row] = merged; - state->alpha[row] = exponential(state->maximum[row] - merged); - state->beta[row] = exponential(maximum - merged); - } - } else if (phase == 3) { - for (unsigned row = 0; row < rows; ++row) - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto partial = - begin == 0 ? __riscv_vfmv_v_f_f32m2(0, rvv_flash::lanes) - : __riscv_vle32_v_f32m2(blockOutput + row * width + d, - rvv_flash::lanes); - for (unsigned k = 0; k < count; ++k) { - auto value = - __riscv_vle32_v_f32m2(panel + k * width + d, rvv_flash::lanes); - float p = state->scores[row * rvv_flash::keys + begin + k]; - partial = - __riscv_vfmacc_vf_f32m2(partial, p, value, rvv_flash::lanes); - } - __riscv_vse32_v_f32m2(blockOutput + row * width + d, partial, - rvv_flash::lanes); - } - } else if (phase == 4) { - for (unsigned row = 0; row < rows; ++row) { - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto old = - __riscv_vle32_v_f32m2(output + row * width + d, rvv_flash::lanes); - auto block = __riscv_vle32_v_f32m2(blockOutput + row * width + d, - rvv_flash::lanes); - old = __riscv_vfmul_vf_f32m2(old, state->alpha[row], rvv_flash::lanes); - block = - __riscv_vfmul_vf_f32m2(block, state->beta[row], rvv_flash::lanes); - auto merged = __riscv_vfadd_vv_f32m2(old, block, rvv_flash::lanes); - __riscv_vse32_v_f32m2(output + row * width + d, merged, - rvv_flash::lanes); - } - float old = state->sum[row] * state->alpha[row]; - float block = state->blockSum[row] * state->beta[row]; - state->sum[row] = old + block; - state->maximum[row] = state->mergedMaximum[row]; - } - } else if (phase == 5) { - for (unsigned row = 0; row < rows; ++row) { - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto value = - __riscv_vle32_v_f32m2(output + row * width + d, rvv_flash::lanes); - value = - __riscv_vfdiv_vf_f32m2(value, state->sum[row], rvv_flash::lanes); - __riscv_vse32_v_f32m2(output + row * width + d, value, - rvv_flash::lanes); - } - mask[row] = state->sum[row]; - } - } else { - asm volatile("ebreak" ::: "memory"); - } - uint32_t status; - asm volatile("csrr %0, fflags" : "=r"(status)::"memory"); - *flags = status; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention.h b/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention.h deleted file mode 100644 index 9d1a8c7f3..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once -#include -#include - -namespace rvv_flash { -constexpr unsigned queries = 16, keys = 64, lanes = 16; -constexpr unsigned maskOffset = 128; -constexpr unsigned stateOffset = maskOffset + queries * lanes * sizeof(float); -struct State { - float sum[queries], maximum[queries]; - float scores[queries * keys]; - float blockMaximum[queries], blockSum[queries]; - float alpha[queries], beta[queries], mergedMaximum[queries]; -}; -} // namespace rvv_flash diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_dispatch.cpp deleted file mode 100644 index 9c13df397..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_dispatch.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include "flash_attention.h" -#include "images.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 5, writeBank = 6; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr size_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); -struct alignas(16) Packet { - kernel_launch launch; - uint64_t padding[4]; - float mask[rvv_flash::queries * rvv_flash::lanes]; -}; -static_assert(offsetof(Packet, mask) == rvv_flash::maskOffset); -static_assert(sizeof(Packet) == rvv_flash::stateOffset); -static_assert(rvv_flash::stateOffset + sizeof(rvv_flash::State) <= bankBytes); - -void panel(const DynamicMemRefType &source, int64_t batch, int64_t head, - int64_t begin, unsigned rows, unsigned width, float *staging) { - for (unsigned row = 0; row < rows; ++row) - std::memcpy(staging + row * width, - source.data + source.offset + batch * source.strides[0] + - head * source.strides[1] + - (begin + row) * source.strides[2], - width * sizeof(float)); -} -} // namespace - -extern "C" void _mlir_ciface_rvv_flash_attention( - UnrankedMemRefType *output, UnrankedMemRefType *scores, - UnrankedMemRefType *query, UnrankedMemRefType *key, - UnrankedMemRefType *value, UnrankedMemRefType *mask, - uint32_t scaleBits) { - DynamicMemRefType out(*output), score(*scores), q(*query), k(*key), - v(*value), condition(*mask); - if (q.rank != 4 || k.rank != 4 || v.rank != 4 || out.rank != 4 || - score.rank != 3 || condition.rank != 4) - abort(); - const int64_t batches = q.sizes[0], heads = q.sizes[1], queries = q.sizes[2]; - const int64_t keys = k.sizes[2], width = q.sizes[3]; - if (batches <= 0 || heads <= 0 || queries <= 0 || keys <= 0 || width <= 0 || - width % rvv_flash::lanes || width * sizeof(float) > bankBytes || - k.sizes[0] != batches || v.sizes[0] != batches || k.sizes[1] != heads || - v.sizes[1] != heads || v.sizes[2] != keys || k.sizes[3] != width || - v.sizes[3] != width || condition.sizes[0] != batches || - condition.sizes[1] != 1 || condition.sizes[2] != queries || - condition.sizes[3] != keys || q.strides[3] != 1 || k.strides[3] != 1 || - v.strides[3] != 1 || condition.strides[3] != 1 || out.strides[3] != 1 || - score.strides[2] != 1) - abort(); - for (unsigned axis = 0; axis < 4; ++axis) - if (out.sizes[axis] != q.sizes[axis] || - (axis < 3 && score.sizes[axis] != q.sizes[axis])) - abort(); - const unsigned tileRows = - std::min(rvv_flash::queries, bankBytes / (width * sizeof(float))); - const unsigned tileKeys = - std::min(rvv_flash::lanes, bankBytes / (width * sizeof(float))); - alignas(16) float staging[bankBytes / sizeof(float)]; - Packet packet{}; - packet.launch.entry = images::flash_attention.entry; - packet.launch.end = images::flash_attention.text_bytes; - packet.launch.stack = 0x40002000; - packet.launch.args[0] = 0; - packet.launch.args[1] = uint64_t(1) << 32; - packet.launch.args[2] = uint64_t(2) << 32; - packet.launch.args[3] = uint64_t(3) << 32; - packet.launch.args[4] = uint64_t(4) << 32; - packet.launch.args[7] = - (uint64_t(2) << 32) + offsetof(kernel_launch, reserved); - uint32_t rounding; - asm volatile("csrr %0, frm" : "=r"(rounding)::"memory"); - packet.launch.args[6] = uint64_t(scaleBits) | (uint64_t(rounding << 5) << 32); - for (unsigned bank = 0; bank < 5; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mem_transfer(0, readBank); - bb_mem_transfer(1, readBank); - for (unsigned bank = 2; bank < 5; ++bank) - bb_mem_transfer(bank, writeBank); - mvin_kernel(images::flash_attention.bytes, images::flash_attention.size, - programBank); - auto launch = [&](unsigned phase, unsigned rows, unsigned keyCount, - unsigned begin = 0, unsigned count = 0) { - packet.launch.args[5] = uint64_t(phase) | (uint64_t(rows) << 8) | - (uint64_t(width) << 16) | (uint64_t(begin) << 32) | - (uint64_t(count) << 40) | - (uint64_t(keyCount) << 48); - packet.launch.reserved = 0; - bb_mvin_group((uintptr_t)&packet, writeBank, 0, sizeof(packet) / 16, 1); - run_kernel(readBank, programBank, writeBank, 0); - bb_mvout_group( - (uintptr_t)&packet, writeBank, 0, - phase == 5 ? sizeof(packet) / 16 : sizeof(kernel_launch) / 16, 1); - asm volatile("csrs fflags, %0" ::"r"(packet.launch.reserved) : "memory"); - }; - for (int64_t batch = 0; batch < batches; ++batch) - for (int64_t head = 0; head < heads; ++head) - for (int64_t queryBegin = 0; queryBegin < queries; - queryBegin += tileRows) { - unsigned rows = std::min(tileRows, queries - queryBegin); - panel(q, batch, head, queryBegin, rows, width, staging); - bb_mvin_group((uintptr_t)staging, readBank, 0, rows * width / 4, 1); - launch(0, rows, 0); - for (int64_t keyBegin = 0; keyBegin < keys; - keyBegin += rvv_flash::keys) { - unsigned keyCount = - std::min(rvv_flash::keys, keys - keyBegin); - for (unsigned begin = 0; begin < keyCount; begin += tileKeys) { - unsigned count = std::min(tileKeys, keyCount - begin); - panel(k, batch, head, keyBegin + begin, count, width, staging); - bb_mvin_group((uintptr_t)staging, readBank, 1, count * width / 4, - 1); - for (unsigned row = 0; row < rows; ++row) - std::memcpy(packet.mask + row * rvv_flash::lanes, - condition.data + condition.offset + - batch * condition.strides[0] + - (queryBegin + row) * condition.strides[2] + - keyBegin + begin, - count * sizeof(float)); - launch(1, rows, keyCount, begin, count); - } - launch(2, rows, keyCount); - for (unsigned begin = 0; begin < keyCount; begin += tileKeys) { - unsigned count = std::min(tileKeys, keyCount - begin); - panel(v, batch, head, keyBegin + begin, count, width, staging); - bb_mvin_group((uintptr_t)staging, readBank, 1, count * width / 4, - 1); - launch(3, rows, keyCount, begin, count); - } - launch(4, rows, keyCount); - } - launch(5, rows, 0); - bb_mvout_group((uintptr_t)staging, writeBank, 1, rows * width / 4, 1); - for (unsigned row = 0; row < rows; ++row) { - std::memcpy(out.data + out.offset + batch * out.strides[0] + - head * out.strides[1] + - (queryBegin + row) * out.strides[2], - staging + row * width, width * sizeof(float)); - std::memcpy(score.data + score.offset + batch * score.strides[0] + - head * score.strides[1] + queryBegin + row, - packet.mask + row, sizeof(float)); - } - } - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_mxfp8.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_mxfp8.cpp deleted file mode 100644 index 619a7dd7a..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_mxfp8.cpp +++ /dev/null @@ -1,144 +0,0 @@ -#include "flash_attention.h" -#include "math/math.h" -#include "math/mxfp8.h" - -static float exponential(float value) { - return __riscv_vfmv_f_s_f32m1_f32( - rvv_exp(__riscv_vfmv_v_f_f32m1(value, 1), 1)); -} - -extern "C" void -rvv_flash_attention_mxfp8_launch(const float *query, const uint8_t *panel, - unsigned char *descriptor, float *output, - float *blockOutput, uint64_t dimensions, - uint64_t parameters, uint32_t *flags) { - const unsigned phase = dimensions & 255, rows = (dimensions >> 8) & 255; - const unsigned width = (dimensions >> 16) & 65535; - const unsigned begin = (dimensions >> 32) & 255; - const unsigned count = (dimensions >> 40) & 255; - const unsigned keyCount = (dimensions >> 48) & 255; - const uint32_t rounding = parameters >> 32; - uint32_t decodeStatus = 0; - const auto *scales = panel + count * width; - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - union { - uint32_t bits; - float value; - } scale{uint32_t(parameters)}; - auto *state = - reinterpret_cast(descriptor + rvv_flash::stateOffset); - auto *mask = reinterpret_cast(descriptor + rvv_flash::maskOffset); - if (phase == 0) { - for (unsigned row = 0; row < rows; ++row) { - state->sum[row] = 0; - state->maximum[row] = -1.0e30f; - for (unsigned d = 0; d < width; d += rvv_flash::lanes) - __riscv_vse32_v_f32m2(output + row * width + d, - __riscv_vfmv_v_f_f32m2(0, rvv_flash::lanes), - rvv_flash::lanes); - } - } else if (phase == 1) { - for (unsigned row = 0; row < rows; ++row) - for (unsigned k = 0; k < count; ++k) { - auto partial = __riscv_vfmv_v_f_f32m2(0, rvv_flash::lanes); - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto q = - __riscv_vle32_v_f32m2(query + row * width + d, rvv_flash::lanes); - auto key = - rvv_mxfp8_load16(panel + k * width + d, - scales[k * (width / 32) + d / 32], decodeStatus); - if (decodeStatus) { - *flags = decodeStatus; - return; - } - partial = __riscv_vfmacc_vv_f32m2(partial, q, key, rvv_flash::lanes); - } - auto sum = __riscv_vfredosum_vs_f32m2_f32m1( - partial, __riscv_vfmv_v_f_f32m1(0, 1), rvv_flash::lanes); - float score = __riscv_vfmv_f_s_f32m1_f32(sum) * scale.value; - state->scores[row * rvv_flash::keys + begin + k] = - score + mask[row * rvv_flash::lanes + k]; - } - } else if (phase == 2) { - for (unsigned row = 0; row < rows; ++row) { - float maximum = -1.0e30f; - for (unsigned k = 0; k < keyCount; ++k) { - float score = state->scores[row * rvv_flash::keys + k]; - if (score > maximum) - maximum = score; - } - float sum = 0; - for (unsigned k = 0; k < keyCount; ++k) { - float p = - exponential(state->scores[row * rvv_flash::keys + k] - maximum); - state->scores[row * rvv_flash::keys + k] = p; - sum = sum + p; - } - state->blockMaximum[row] = maximum; - state->blockSum[row] = sum; - float merged = - maximum > state->maximum[row] ? maximum : state->maximum[row]; - state->mergedMaximum[row] = merged; - state->alpha[row] = exponential(state->maximum[row] - merged); - state->beta[row] = exponential(maximum - merged); - } - } else if (phase == 3) { - for (unsigned row = 0; row < rows; ++row) - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto partial = - begin == 0 ? __riscv_vfmv_v_f_f32m2(0, rvv_flash::lanes) - : __riscv_vle32_v_f32m2(blockOutput + row * width + d, - rvv_flash::lanes); - for (unsigned k = 0; k < count; ++k) { - auto value = - rvv_mxfp8_load16(panel + k * width + d, - scales[k * (width / 32) + d / 32], decodeStatus); - if (decodeStatus) { - *flags = decodeStatus; - return; - } - float p = state->scores[row * rvv_flash::keys + begin + k]; - partial = - __riscv_vfmacc_vf_f32m2(partial, p, value, rvv_flash::lanes); - } - __riscv_vse32_v_f32m2(blockOutput + row * width + d, partial, - rvv_flash::lanes); - } - } else if (phase == 4) { - for (unsigned row = 0; row < rows; ++row) { - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto old = - __riscv_vle32_v_f32m2(output + row * width + d, rvv_flash::lanes); - auto block = __riscv_vle32_v_f32m2(blockOutput + row * width + d, - rvv_flash::lanes); - old = __riscv_vfmul_vf_f32m2(old, state->alpha[row], rvv_flash::lanes); - block = - __riscv_vfmul_vf_f32m2(block, state->beta[row], rvv_flash::lanes); - auto merged = __riscv_vfadd_vv_f32m2(old, block, rvv_flash::lanes); - __riscv_vse32_v_f32m2(output + row * width + d, merged, - rvv_flash::lanes); - } - float old = state->sum[row] * state->alpha[row]; - float block = state->blockSum[row] * state->beta[row]; - state->sum[row] = old + block; - state->maximum[row] = state->mergedMaximum[row]; - } - } else if (phase == 5) { - for (unsigned row = 0; row < rows; ++row) { - for (unsigned d = 0; d < width; d += rvv_flash::lanes) { - auto value = - __riscv_vle32_v_f32m2(output + row * width + d, rvv_flash::lanes); - value = - __riscv_vfdiv_vf_f32m2(value, state->sum[row], rvv_flash::lanes); - __riscv_vse32_v_f32m2(output + row * width + d, value, - rvv_flash::lanes); - } - mask[row] = state->sum[row]; - } - } else { - asm volatile("ebreak" ::: "memory"); - } - uint32_t status; - asm volatile("csrr %0, fflags" : "=r"(status)::"memory"); - *flags = status; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_mxfp8_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_mxfp8_dispatch.cpp deleted file mode 100644 index 384295e5a..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/flash_attention_mxfp8_dispatch.cpp +++ /dev/null @@ -1,187 +0,0 @@ -#include "flash_attention.h" -#include "images.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 5, writeBank = 6; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr size_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); -struct alignas(16) Packet { - kernel_launch launch; - uint64_t padding[4]; - float mask[rvv_flash::queries * rvv_flash::lanes]; -}; -static_assert(offsetof(Packet, mask) == rvv_flash::maskOffset); -static_assert(sizeof(Packet) == rvv_flash::stateOffset); -static_assert(rvv_flash::stateOffset + sizeof(rvv_flash::State) <= bankBytes); - -void panel(const DynamicMemRefType &source, int64_t batch, int64_t head, - int64_t begin, unsigned rows, unsigned width, float *staging) { - for (unsigned row = 0; row < rows; ++row) - std::memcpy(staging + row * width, - source.data + source.offset + batch * source.strides[0] + - head * source.strides[1] + - (begin + row) * source.strides[2], - width * sizeof(float)); -} - -void packedPanel(const DynamicMemRefType &codes, - const DynamicMemRefType &scales, int64_t batch, - int64_t head, int64_t begin, unsigned rows, unsigned width, - int8_t *staging) { - for (unsigned row = 0; row < rows; ++row) { - std::memcpy(staging + row * width, - codes.data + codes.offset + batch * codes.strides[0] + - head * codes.strides[1] + (begin + row) * codes.strides[2], - width); - std::memcpy(staging + rows * width + row * (width / 32), - scales.data + scales.offset + batch * scales.strides[0] + - head * scales.strides[1] + - (begin + row) * scales.strides[2], - width / 32); - } -} -} // namespace - -extern "C" void _mlir_ciface_rvv_flash_attention_mxfp8( - UnrankedMemRefType *output, UnrankedMemRefType *scores, - UnrankedMemRefType *query, UnrankedMemRefType *keyCodes, - UnrankedMemRefType *keyScales, - UnrankedMemRefType *valueCodes, - UnrankedMemRefType *valueScales, UnrankedMemRefType *mask, - uint32_t scaleBits) { - DynamicMemRefType out(*output), score(*scores), q(*query), - condition(*mask); - DynamicMemRefType k(*keyCodes), ks(*keyScales), v(*valueCodes), - vs(*valueScales); - if (q.rank != 4 || k.rank != 4 || v.rank != 4 || out.rank != 4 || - score.rank != 3 || condition.rank != 4 || ks.rank != 4 || vs.rank != 4) - abort(); - const int64_t batches = q.sizes[0], heads = q.sizes[1], queries = q.sizes[2]; - const int64_t keys = k.sizes[2], width = q.sizes[3]; - if (batches <= 0 || heads <= 0 || queries <= 0 || keys <= 0 || width <= 0 || - width % 32 || width * sizeof(float) > bankBytes || - k.sizes[0] != batches || v.sizes[0] != batches || k.sizes[1] <= 0 || - heads % k.sizes[1] || v.sizes[1] != k.sizes[1] || v.sizes[2] != keys || - k.sizes[3] != width || v.sizes[3] != width || - condition.sizes[0] != batches || condition.sizes[1] != 1 || - condition.sizes[2] != queries || condition.sizes[3] != keys || - q.strides[3] != 1 || k.strides[3] != 1 || v.strides[3] != 1 || - condition.strides[3] != 1 || out.strides[3] != 1 || score.strides[2] != 1) - abort(); - for (unsigned axis = 0; axis < 4; ++axis) - if (ks.sizes[axis] != (axis == 3 ? width / 32 : k.sizes[axis]) || - vs.sizes[axis] != ks.sizes[axis]) - abort(); - if (ks.strides[3] != 1 || vs.strides[3] != 1) - abort(); - for (unsigned axis = 0; axis < 4; ++axis) - if (out.sizes[axis] != q.sizes[axis] || - (axis < 3 && score.sizes[axis] != q.sizes[axis])) - abort(); - const unsigned tileRows = - std::min(rvv_flash::queries, bankBytes / (width * sizeof(float))); - const unsigned tileKeys = - std::min(rvv_flash::lanes, bankBytes / (width * sizeof(float))); - alignas(16) float staging[bankBytes / sizeof(float)]; - alignas(16) int8_t packed[bankBytes]; - Packet packet{}; - packet.launch.entry = images::flash_attention_mxfp8.entry; - packet.launch.end = images::flash_attention_mxfp8.text_bytes; - packet.launch.stack = 0x40002000; - packet.launch.args[0] = 0; - packet.launch.args[1] = uint64_t(1) << 32; - packet.launch.args[2] = uint64_t(2) << 32; - packet.launch.args[3] = uint64_t(3) << 32; - packet.launch.args[4] = uint64_t(4) << 32; - packet.launch.args[7] = - (uint64_t(2) << 32) + offsetof(kernel_launch, reserved); - uint32_t rounding; - asm volatile("csrr %0, frm" : "=r"(rounding)::"memory"); - packet.launch.args[6] = uint64_t(scaleBits) | (uint64_t(rounding << 5) << 32); - for (unsigned bank = 0; bank < 5; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mem_transfer(0, readBank); - bb_mem_transfer(1, readBank); - for (unsigned bank = 2; bank < 5; ++bank) - bb_mem_transfer(bank, writeBank); - mvin_kernel(images::flash_attention_mxfp8.bytes, - images::flash_attention_mxfp8.size, programBank); - auto launch = [&](unsigned phase, unsigned rows, unsigned keyCount, - unsigned begin = 0, unsigned count = 0) { - packet.launch.args[5] = uint64_t(phase) | (uint64_t(rows) << 8) | - (uint64_t(width) << 16) | (uint64_t(begin) << 32) | - (uint64_t(count) << 40) | - (uint64_t(keyCount) << 48); - packet.launch.reserved = 0; - bb_mvin_group((uintptr_t)&packet, writeBank, 0, sizeof(packet) / 16, 1); - run_kernel(readBank, programBank, writeBank, 0); - bb_mvout_group( - (uintptr_t)&packet, writeBank, 0, - phase == 5 ? sizeof(packet) / 16 : sizeof(kernel_launch) / 16, 1); - if (packet.launch.reserved & ~uint64_t(31)) { - fputs("flash_attention_mxfp8: invalid packed KV data\n", stderr); - abort(); - } - asm volatile("csrs fflags, %0" ::"r"(packet.launch.reserved) : "memory"); - }; - for (int64_t batch = 0; batch < batches; ++batch) - for (int64_t head = 0; head < heads; ++head) - for (int64_t queryBegin = 0; queryBegin < queries; - queryBegin += tileRows) { - unsigned rows = std::min(tileRows, queries - queryBegin); - panel(q, batch, head, queryBegin, rows, width, staging); - bb_mvin_group((uintptr_t)staging, readBank, 0, rows * width / 4, 1); - launch(0, rows, 0); - for (int64_t keyBegin = 0; keyBegin < keys; - keyBegin += rvv_flash::keys) { - unsigned keyCount = - std::min(rvv_flash::keys, keys - keyBegin); - for (unsigned begin = 0; begin < keyCount; begin += tileKeys) { - unsigned count = std::min(tileKeys, keyCount - begin); - packedPanel(k, ks, batch, head / (heads / k.sizes[1]), - keyBegin + begin, count, width, packed); - bb_mvin_group((uintptr_t)packed, readBank, 1, - (count * width + count * (width / 32) + 15) / 16, 1); - for (unsigned row = 0; row < rows; ++row) - std::memcpy(packet.mask + row * rvv_flash::lanes, - condition.data + condition.offset + - batch * condition.strides[0] + - (queryBegin + row) * condition.strides[2] + - keyBegin + begin, - count * sizeof(float)); - launch(1, rows, keyCount, begin, count); - } - launch(2, rows, keyCount); - for (unsigned begin = 0; begin < keyCount; begin += tileKeys) { - unsigned count = std::min(tileKeys, keyCount - begin); - packedPanel(v, vs, batch, head / (heads / k.sizes[1]), - keyBegin + begin, count, width, packed); - bb_mvin_group((uintptr_t)packed, readBank, 1, - (count * width + count * (width / 32) + 15) / 16, 1); - launch(3, rows, keyCount, begin, count); - } - launch(4, rows, keyCount); - } - launch(5, rows, 0); - bb_mvout_group((uintptr_t)staging, writeBank, 1, rows * width / 4, 1); - for (unsigned row = 0; row < rows; ++row) { - std::memcpy(out.data + out.offset + batch * out.strides[0] + - head * out.strides[1] + - (queryBegin + row) * out.strides[2], - staging + row * width, width * sizeof(float)); - std::memcpy(score.data + score.offset + batch * score.strides[0] + - head * score.strides[1] + queryBegin + row, - packet.mask + row, sizeof(float)); - } - } - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/gelu.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/gelu.cpp deleted file mode 100644 index cc54994e9..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/gelu.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "kernels.h" - -extern "C" float rvv_erff(float value); - -extern "C" void rvv_gelu(float *output, const float *input, size_t count, - uint32_t *flags, uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - for (size_t index = 0; index < count; ++index) { - const float value = input[index]; - const float scaled = value * 0x1.6a09e6p-1f; - const float error = rvv_erff(scaled); - const float shifted = error + 1.0f; - const float halved = value * 0.5f; - output[index] = halved * shifted; - } - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/kernel.ld b/bb-tests/workloads/lib/bbsw/kernels/rvv/kernel.ld deleted file mode 100644 index b43527119..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/kernel.ld +++ /dev/null @@ -1,13 +0,0 @@ -SECTIONS { - . = 0; - .text : { *(.text*) } - ASSERT(SIZEOF(.text) <= 4096, "RVV instruction memory overflow") - . = 0x40000000; - .rodata : { *(.rodata*) *(.srodata*) } - ASSERT(SIZEOF(.rodata) <= 4096, "RVV constant memory overflow") - .data : { *(.data*) *(.sdata*) *(.tdata*) } - .bss (NOLOAD) : { *(.bss*) *(.sbss*) *(.tbss*) *(COMMON) } - ASSERT(SIZEOF(.data) == 0, "RVV kernels cannot define writable global data") - ASSERT(SIZEOF(.bss) == 0, "RVV kernels cannot define writable global data") - /DISCARD/ : { *(.eh_frame*) *(.comment*) *(.riscv.attributes*) } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/kernels.h b/bb-tests/workloads/lib/bbsw/kernels/rvv/kernels.h deleted file mode 100644 index 93a1134f4..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/kernels.h +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once -#include -#include - -extern "C" { -void rvv_flash_attention_mxfp8_launch(const float *query, const uint8_t *panel, - unsigned char *descriptor, float *output, - float *blockOutput, uint64_t dimensions, - uint64_t parameters, uint32_t *flags); -void rvv_layernorm(float *output, const float *input, const float *weight, - const float *bias, uint64_t widthAndBias, - uint64_t parameters, uint32_t *flags, uint32_t rounding); -void rvv_gelu(float *output, const float *input, size_t count, uint32_t *flags, - uint32_t rounding); -void rvv_pointwise(float *output, const float *lhs, const float *rhs, - size_t count, uint32_t opcode, uint32_t *flags, - uint32_t rounding); -void rvv_matmul(float *output, const float *lhs, const float *rhs, size_t rows, - size_t cols, size_t inner); -void rvv_quant(uint8_t *output, uint8_t *scales, const uint32_t *input, - size_t count, uint32_t *status); -void rvv_cache_quant(uint8_t *output, uint8_t *scales, const uint32_t *input, - size_t count, uint32_t *status); -void rvv_dequant(uint32_t *output, const uint8_t *input, const uint8_t *scales, - size_t count, uint32_t *status); -void rvv_silu(float *output, const float *input, size_t count); -void rvv_swiglu(float *output, const float *gate, const float *up, - size_t count); -void rvv_snake(float *output, const float *input, size_t channels, - size_t length, const float *log_alpha, const float *log_beta); -void rvv_norm(float *output, const float *input, const float *weight, - size_t width, uint32_t meanMultiplierBits, uint32_t epsilonBits); -void rvv_norm_no_weight(float *output, const float *input, size_t width, - uint32_t meanMultiplierBits, uint32_t epsilonBits); -void rvv_softmax(float *output, const float *input, size_t rows, size_t width); -void rvv_logsumexp_softmax(float *output, const float *input, size_t rows, - size_t width); -void rvv_rope(float *output, const float *input, const float *frequencies, - const int32_t *positions, size_t rows, size_t headDimAndHeads); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/layernorm.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/layernorm.cpp deleted file mode 100644 index 15f3e00fc..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/layernorm.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "kernels.h" -#include - -extern "C" void rvv_layernorm(float *output, const float *input, - const float *weight, const float *bias, - uint64_t widthAndBias, uint64_t parameters, - uint32_t *flags, uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - const bool hasBias = widthAndBias >> 32; - const size_t width = uint32_t(widthAndBias); - const float multiplier = __builtin_bit_cast(float, uint32_t(parameters)); - const float epsilon = __builtin_bit_cast(float, uint32_t(parameters >> 32)); - auto sum = __riscv_vfmv_v_f_f32m1(0, 1); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(input + offset, vl); - sum = __riscv_vfredosum_vs_f32m1_f32m1(values, sum, vl); - offset += vl; - } - const float mean = __riscv_vfmv_f_s_f32m1_f32(sum) * multiplier; - sum = __riscv_vfmv_v_f_f32m1(0, 1); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(input + offset, vl); - auto centered = __riscv_vfsub_vf_f32m1(values, mean, vl); - auto squares = __riscv_vfmul_vv_f32m1(centered, centered, vl); - sum = __riscv_vfredosum_vs_f32m1_f32m1(squares, sum, vl); - offset += vl; - } - const float variance = __riscv_vfmv_f_s_f32m1_f32(sum) * multiplier; - const float biased = variance + epsilon; - float root; - asm volatile("fsqrt.s %0, %1" : "=f"(root) : "f"(biased)); - const float scale = 1.0f / root; - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(input + offset, vl); - auto centered = __riscv_vfsub_vf_f32m1(values, mean, vl); - auto normalized = __riscv_vfmul_vf_f32m1(centered, scale, vl); - auto weights = __riscv_vle32_v_f32m1(weight + offset, vl); - auto weighted = __riscv_vfmul_vv_f32m1(normalized, weights, vl); - if (hasBias) { - auto biases = __riscv_vle32_v_f32m1(bias + offset, vl); - weighted = __riscv_vfadd_vv_f32m1(weighted, biases, vl); - } - __riscv_vse32_v_f32m1(output + offset, weighted, vl); - offset += vl; - } - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/constants.h b/bb-tests/workloads/lib/bbsw/kernels/rvv/math/constants.h deleted file mode 100644 index b82cc5c26..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/constants.h +++ /dev/null @@ -1,39 +0,0 @@ -/* Single-precision e^x function. - Copyright (C) 2017-2025 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#pragma once -#include -static const float exp_table[32] = { - 0x1.0000000000000p+0f, 0x1.059b0e0000000p+0f, 0x1.0b55860000000p+0f, - 0x1.11301e0000000p+0f, 0x1.172b840000000p+0f, 0x1.1d48740000000p+0f, - 0x1.2387a60000000p+0f, 0x1.29e9e00000000p+0f, 0x1.306fe00000000p+0f, - 0x1.371a740000000p+0f, 0x1.3dea640000000p+0f, 0x1.44e0860000000p+0f, - 0x1.4bfdae0000000p+0f, 0x1.5342b60000000p+0f, 0x1.5ab07e0000000p+0f, - 0x1.6247ec0000000p+0f, 0x1.6a09e60000000p+0f, 0x1.71f75e0000000p+0f, - 0x1.7a11480000000p+0f, 0x1.82589a0000000p+0f, 0x1.8ace540000000p+0f, - 0x1.93737c0000000p+0f, 0x1.9c49180000000p+0f, 0x1.a5503c0000000p+0f, - 0x1.ae89fa0000000p+0f, 0x1.b7f7700000000p+0f, 0x1.c199be0000000p+0f, - 0x1.cb720e0000000p+0f, 0x1.d5818e0000000p+0f, 0x1.dfc9740000000p+0f, - 0x1.ea4afa0000000p+0f, 0x1.f507660000000p+0f, -}; -static const uint32_t inv_pio4[24] = { - - 0xa2, 0xa2f9, 0xa2f983, 0xa2f9836e, 0xf9836e4e, 0x836e4e44, - 0x6e4e4415, 0x4e441529, 0x441529fc, 0x1529fc27, 0x29fc2757, 0xfc2757d1, - 0x2757d1f5, 0x57d1f534, 0xd1f534dd, 0xf534ddc0, 0x34ddc0db, 0xddc0db62, - 0xc0db6295, 0xdb629599, 0x6295993c, 0x95993c43, 0x993c4390, 0x3c439041}; diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/cos.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/math/cos.cpp deleted file mode 100644 index 6b9785b66..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/cos.cpp +++ /dev/null @@ -1,99 +0,0 @@ -// Vector adaptation of glibc 2.42 s_cosf.c and s_sincosf.h/sincosf_poly.h. -// Copyright (C) 2018-2025 Free Software Foundation, Inc. -// LGPL-2.1-or-later; see ../../../thirdparty/glibc/COPYING.LIB. -#include "constants.h" -#include "math.h" - -extern "C" vfloat32m1_t rvv_cos(vfloat32m1_t value, size_t vl) { - auto bits = __riscv_vreinterpret_u32m1(value); - auto top = - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 20, vl), 0x7ff, vl); - auto tiny = __riscv_vmsltu_vx_u32m1_b32(top, 0x398, vl); - auto fast = __riscv_vmsltu_vx_u32m1_b32(top, 0x42f, vl); - fast = __riscv_vmandn_mm_b32(fast, tiny, vl); - auto finite = __riscv_vmsltu_vx_u32m1_b32(top, 0x7f8, vl); - auto input = - __riscv_vmerge_vvm_f32m1(__riscv_vfmv_v_f_f32m1(0, vl), value, fast, vl); - auto x = input; - auto r = __riscv_vfmul_vf_f32m1(x, 0x1.45f306dc9c883p23, vl); - auto n = __riscv_vfcvt_rtz_x_f_v_i32m1(r, vl); - n = __riscv_vsra_vx_i32m1(__riscv_vadd_vx_i32m1(n, 0x800000, vl), 24, vl); - auto small = __riscv_vmsltu_vx_u32m1_b32(top, 0x3f4, vl); - n = __riscv_vmerge_vxm_i32m1(n, 0, small, vl); - auto nd = __riscv_vfcvt_f_x_v_f32m1(n, vl); - x = __riscv_vfnmsac_vf_f32m1(x, 0x1.921fb4p0f, nd, vl); - x = __riscv_vfnmsac_vf_f32m1(x, 0x1.4442d2p-24f, nd, vl); - auto quadrant = n; - auto large = __riscv_vmandn_mm_b32(finite, fast, vl); - large = __riscv_vmandn_mm_b32(large, tiny, vl); - if (__riscv_vcpop_m_b32(large, vl)) { - auto index = - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 26, vl), 15, vl); - index = __riscv_vsll_vx_u32m1(index, 2, vl); - auto a0 = __riscv_vluxei32_v_u32m1(inv_pio4, index, vl); - auto a1 = __riscv_vluxei32_v_u32m1(inv_pio4 + 4, index, vl); - auto a2 = __riscv_vluxei32_v_u32m1(inv_pio4 + 8, index, vl); - auto shift = - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 23, vl), 7, vl); - auto xi = __riscv_vor_vx_u32m1(__riscv_vand_vx_u32m1(bits, 0xffffff, vl), - 0x800000, vl); - xi = __riscv_vsll_vv_u32m1(xi, shift, vl); - auto low0 = __riscv_vmulhu_vv_u32m1(xi, a2, vl); - auto low1 = __riscv_vmul_vv_u32m1(xi, a1, vl); - auto low = __riscv_vadd_vv_u32m1(low0, low1, vl); - auto carry_mask = __riscv_vmsltu_vv_u32m1_b32(low, low1, vl); - auto high = __riscv_vadd_vv_u32m1(__riscv_vmul_vv_u32m1(xi, a0, vl), - __riscv_vmulhu_vv_u32m1(xi, a1, vl), vl); - auto carry = __riscv_vmerge_vxm_u32m1(__riscv_vmv_v_x_u32m1(0, vl), 1, - carry_mask, vl); - high = __riscv_vadd_vv_u32m1(high, carry, vl); - auto q = __riscv_vsrl_vx_u32m1(__riscv_vadd_vx_u32m1(high, 0x20000000, vl), - 30, vl); - high = __riscv_vsub_vv_u32m1(high, __riscv_vsll_vx_u32m1(q, 30, vl), vl); - auto reduced = - __riscv_vfcvt_f_x_v_f32m1(__riscv_vreinterpret_i32m1(high), vl); - reduced = __riscv_vfmul_vf_f32m1(reduced, 0x1.921fb4p-30f, vl); - auto fraction = __riscv_vfcvt_f_xu_v_f32m1(low, vl); - reduced = __riscv_vfmacc_vf_f32m1(reduced, 0x1.921fb4p-62f, fraction, vl); - auto q32 = __riscv_vreinterpret_i32m1(q); - n = __riscv_vmerge_vvm_i32m1(n, q32, large, vl); - auto sign = __riscv_vreinterpret_i32m1(__riscv_vsrl_vx_u32m1(bits, 31, vl)); - quadrant = __riscv_vmerge_vvm_i32m1( - quadrant, __riscv_vadd_vv_i32m1(q32, sign, vl), large, vl); - x = __riscv_vmerge_vvm_f32m1(x, reduced, large, vl); - } - auto x2 = __riscv_vfmul_vv_f32m1(x, x, vl); - auto negative = - __riscv_vmsne_vx_i32m1_b32(__riscv_vand_vx_i32m1(quadrant, 2, vl), 0, vl); - // sign[] is {1,-1,-1,1}; cosine polynomial sign depends on bit 1. - auto signbit = __riscv_vxor_vv_i32m1( - quadrant, __riscv_vsra_vx_i32m1(quadrant, 1, vl), vl); - auto sine_negative = - __riscv_vmsne_vx_i32m1_b32(__riscv_vand_vx_i32m1(signbit, 1, vl), 0, vl); - auto sx = __riscv_vmerge_vvm_f32m1(x, __riscv_vfneg_v_f32m1(x, vl), - sine_negative, vl); - auto x3 = __riscv_vfmul_vv_f32m1(sx, x2, vl); - auto s1 = - __riscv_vfmacc_vf_f32m1(__riscv_vfmv_v_f_f32m1(0x1.1107605230bc4p-7, vl), - -0x1.994eb3774cf24p-13, x2, vl); - auto s = __riscv_vfmacc_vf_f32m1(sx, -0x1.555545995a603p-3, x3, vl); - auto x7 = __riscv_vfmul_vv_f32m1(x3, x2, vl); - s = __riscv_vfmacc_vv_f32m1(s, s1, x7, vl); - auto x4 = __riscv_vfmul_vv_f32m1(x2, x2, vl); - auto c2 = __riscv_vfmacc_vf_f32m1( - __riscv_vfmv_v_f_f32m1(-0x1.6c087e89a359dp-10, vl), 0x1.99343027bf8c3p-16, - x2, vl); - auto c1 = __riscv_vfmacc_vf_f32m1(__riscv_vfmv_v_f_f32m1(1, vl), - -0x1.ffffffd0c621cp-2, x2, vl); - auto c = __riscv_vfmacc_vf_f32m1(c1, 0x1.55553e1068f19p-5, x4, vl); - auto x6 = __riscv_vfmul_vv_f32m1(x4, x2, vl); - c = __riscv_vfmacc_vv_f32m1(c, c2, x6, vl); - c = __riscv_vmerge_vvm_f32m1(c, __riscv_vfneg_v_f32m1(c, vl), negative, vl); - auto odd = __riscv_vmsne_vx_i32m1_b32(__riscv_vand_vx_i32m1(n, 1, vl), 0, vl); - auto result = __riscv_vmerge_vvm_f32m1(c, s, odd, vl); - result = __riscv_vfmerge_vfm_f32m1(result, 1, tiny, vl); - auto invalid = __riscv_vmnot_m_b32(finite, vl); - auto difference = __riscv_vfsub_vv_f32m1_m(invalid, value, value, vl); - auto nan = __riscv_vfdiv_vv_f32m1_m(invalid, difference, difference, vl); - return __riscv_vmerge_vvm_f32m1(result, nan, invalid, vl); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/erf.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/math/erf.cpp deleted file mode 100644 index 1ae8a2201..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/erf.cpp +++ /dev/null @@ -1,248 +0,0 @@ -/* Error function evaluated in binary32. - -Copyright (c) 2022-2024 Alexei Sibidanov. - -This file is part of the CORE-MATH project -project (file src/binary32/erf/erff.c revision bc385c2). - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -#include -#include -// GNU glibc 2.42 sysdeps/ieee754/flt-32/s_erff.c. -static uint32_t asuint(float x) { return __builtin_bit_cast(uint32_t, x); } - -extern "C" float rvv_erff(float x) { - /* for 7 <= i < 63, C[i-7] is a degree-7 polynomial approximation of - erf(i/16+1/32+x) for -1/32 <= x <= 1/32 */ - static const float C[56][8] = { - {0x1.f86faa9428f9cp-2f, 0x1.cfc41e36c7dfap-1f, -0x1.b2c7dc53508b9p-2f, - -0x1.5a9de93fa556ep-3f, 0x1.731793dbb01b5p-3f, 0x1.133e06426cf18p-6f, - -0x1.a12a6289cafd8p-5f, 0x1.717d6f1d6f557p-9f}, - {0x1.1855a5fd3dd50p-1f, 0x1.b3aafcc27502fp-1f, -0x1.cee5ac8e92bb2p-2f, - -0x1.fa02983ca2d79p-4f, 0x1.77cd746cb1922p-3f, -0x1.fa6f277886487p-10f, - -0x1.8de75458db416p-5f, 0x1.00899c98551c9p-7f}, - {0x1.32a54cb8db67ap-1f, 0x1.96164fafd8de5p-1f, -0x1.e23a7ea0c9ad3p-2f, - -0x1.3f5ee15671cf4p-4f, 0x1.70e468a3d72d9p-3f, -0x1.3da68037cfc99p-6f, - -0x1.69ed9ba1f9839p-5f, 0x1.8cab9244a4ff4p-7f}, - {0x1.4b13713ad3513p-1f, 0x1.7791b886e7405p-1f, -0x1.ecef423109bf5p-2f, - -0x1.15c3c5cec6847p-5f, 0x1.5f688fc931ba6p-3f, -0x1.1da63ed190037p-5f, - -0x1.38427ca63cca4p-5f, 0x1.fa00e52525e17p-7f}, - {0x1.61955607dd15dp-1f, 0x1.58a445da7c74ep-1f, -0x1.ef6c246a0f66cp-2f, - 0x1.e83e0d9d61330p-8f, 0x1.44cc65535bc9fp-3f, -0x1.87d3c4860435dp-5f, - -0x1.f90b10501169bp-6f, 0x1.22295856d427ap-6f}, - {0x1.762870f720c6fp-1f, 0x1.39ccc1b136d5cp-1f, -0x1.ea4feea4e4744p-2f, - 0x1.715e5952ebfbap-5f, 0x1.22cdbd83c75c4p-3f, -0x1.da50aa1d925b6p-5f, - -0x1.754dc0a29b4ddp-6f, 0x1.350b6bef9392cp-6f}, - {0x1.88d1cd474a2e0p-1f, 0x1.1b7e98fe26219p-1f, -0x1.de65a22ce1419p-2f, - 0x1.40686a3f16400p-4f, 0x1.f6b0cbb216b2bp-4f, -0x1.09c7c903edd57p-4f, - -0x1.da7529fde641p-7f, 0x1.362a7a0588eabp-6f}, - {0x1.999d4192a5717p-1f, 0x1.fc3ee5d1524b3p-2f, -0x1.cc990045b55c8p-2f, - 0x1.b37338e68b37dp-4f, 0x1.a0d120c872ea7p-4f, -0x1.19bb2b07ecff6p-4f, - -0x1.a110f5f593aafp-8f, 0x1.272c15a57720ep-6f}, - {0x1.a89c850b7d54dp-1f, 0x1.c40b0729ed54ap-2f, -0x1.b5eaaef0a2346p-2f, - 0x1.0847c7dacbae1p-3f, 0x1.47de0ba6d18fbp-4f, -0x1.1d9de77a4b648p-4f, - 0x1.30ffbe56f0726p-10f, 0x1.0a9cb99feea01p-6f}, - {0x1.b5e62fce16096p-1f, 0x1.8eed36b886d95p-2f, -0x1.9b64a06e50705p-2f, - 0x1.2bb6e2c744df5p-3f, 0x1.dee3261ca61bcp-5f, -0x1.16996004f7da5p-4f, - 0x1.fdff37bae983ep-8f, 0x1.c750083e65f9ap-7f}, - {0x1.c194b1d49a184p-1f, 0x1.5d4fd33729015p-2f, -0x1.7e0f4f045addbp-2f, - 0x1.444bc66c31a1bp-3f, 0x1.356dbf16ec8f1p-5f, -0x1.0643de0906cd8p-4f, - 0x1.b281af7bd3a2cp-7f, 0x1.6b97eaa2c6abdp-7f}, - {0x1.cbc54b476248ep-1f, 0x1.2f7cc3fe6f423p-2f, -0x1.5ee8429e36de8p-2f, - 0x1.52a8395f96177p-3f, 0x1.313761ba257dcp-6f, -0x1.dcf844d5fed8fp-5f, - 0x1.1e1420f475fa9p-6f, 0x1.091c7dc1e18b2p-7f}, - {0x1.d4970f9ce00d9p-1f, 0x1.059f59af7a905p-2f, -0x1.3eda354de36c3p-2f, - 0x1.57b85ad439779p-3f, 0x1.8e913b9778136p-10f, -0x1.a2893bd3435f4p-5f, - 0x1.4d3a90e37164ap-6f, 0x1.4ce7f6e19a902p-8f}, - {0x1.dc29fb60715b0p-1f, 0x1.bf8e1b1ca2277p-3f, -0x1.1eb7095e5d6d2p-2f, - 0x1.549ea6f7a64f4p-3f, -0x1.b10f12f3877a3p-7f, -0x1.61420c8f7156ap-5f, - 0x1.674f1f92a8812p-6f, 0x1.25543ffd74d52p-9f}, - {0x1.e29e22a89d767p-1f, 0x1.7bd5c7df3fe99p-3f, -0x1.fe674494077bfp-3f, - 0x1.4a9feacf86578p-3f, -0x1.a008269076644p-6f, -0x1.1cf0e8fb4f1cbp-5f, - 0x1.6e0d2ef105fb3p-6f, -0x1.367205fbd7876p-12f}, - {0x1.e812fc64db36ap-1f, 0x1.3fda6bc016991p-3f, -0x1.c1cb278627920p-3f, - 0x1.3b10512314f1ep-3f, -0x1.1e6457bb1b9a9p-5f, -0x1.b1f6474e2388cp-6f, - 0x1.640a5345f7ec7p-6f, -0x1.3dae5a997fdbp-9f}, - {0x1.eca6ccd709544p-1f, 0x1.0b3f52ce8c380p-3f, -0x1.8885019f63c6dp-3f, - 0x1.274275fc91a05p-3f, -0x1.57f73699a8372p-5f, -0x1.3076a305fc7cep-6f, - 0x1.4c6ae04843a41p-6f, -0x1.0be5fcf5ecc91p-8f}, - {0x1.f0762fde45ee7p-1f, 0x1.bb1c972f23e4ap-4f, -0x1.5341e3c01b58dp-3f, - 0x1.107929f6f0b60p-3f, -0x1.7e1b34f976c02p-5f, -0x1.73b62589c234ap-7f, - 0x1.2a97ee1876486p-6f, -0x1.595f40a3150fep-8f}, - {0x1.f39bc242e43e6p-1f, 0x1.6c7e64e7281c5p-4f, -0x1.2274b86835fd3p-3f, - 0x1.efb890e5c770dp-4f, -0x1.92c7db16847e0p-5f, -0x1.45477db5e2dd4p-8f, - 0x1.01fc6165fc866p-6f, -0x1.8845509030c2cp-8f}, - {0x1.f62fe80272419p-1f, 0x1.297db960e4f5dp-4f, -0x1.ecb83b087c04fp-4f, - 0x1.bce18363ca3d1p-4f, -0x1.985aaf776482cp-5f, 0x1.cd953efdae886p-12f, - 0x1.ab9a0b89b54ffp-7f, -0x1.9b5e576ccc31cp-8f}, - {0x1.f848acb544e95p-1f, 0x1.e1d4cf1e24501p-5f, -0x1.9e12e1fde5552p-4f, - 0x1.8a27806df3d1bp-4f, -0x1.91674e5eb3319p-5f, 0x1.3bc75595b2db8p-8f, - 0x1.51bc537ac61afp-7f, -0x1.96b23b19ea04dp-8f}, - {0x1.f9f9ba8d3c733p-1f, 0x1.83298d7172108p-5f, -0x1.58d101f905a75p-4f, - 0x1.58f1456f8639bp-4f, -0x1.808d1850b8231p-5f, 0x1.0c1bd99c348a7p-7f, - 0x1.f61e9d7bc48cap-8f, -0x1.7f07c13441774p-8f}, - {0x1.fb54641aebbc9p-1f, 0x1.34ac36ad8dafap-5f, -0x1.1c8ec267f9405p-4f, - 0x1.2a52c5d841848p-4f, -0x1.68541c02b3b6bp-5f, 0x1.5afe400196379p-7f, - 0x1.565b2d6eda3d6p-8f, -0x1.596aaff29e739p-8f}, - {0x1.fc67bcf2d7b8fp-1f, 0x1.e85c449e377efp-6f, -0x1.d177f166c07c6p-5f, - 0x1.fe23b7584b504p-5f, -0x1.4b12109613313p-5f, 0x1.8d9905c0acf7dp-7f, - 0x1.9265032a669dap-9f, -0x1.2ac4a6dbcbf3ep-8f}, - {0x1.fd40bd6d7a785p-1f, 0x1.7f5188610ddc7p-6f, -0x1.7954423f7c998p-5f, - 0x1.af5baae33887fp-5f, -0x1.2ad77c7cbc474p-5f, 0x1.a7b8c47ec2a51p-7f, - 0x1.46646ee094bccp-10f, -0x1.ef19d8db8673p-9f}, - {0x1.fdea6e062d0c9p-1f, 0x1.2a875b5ffab58p-6f, -0x1.2f3178cd6dcd5p-5f, - 0x1.68d1c45b94182p-5f, -0x1.09648ed3aeaefp-5f, 0x1.ad8b150d38164p-7f, - -0x1.e9a6023d9429fp-13f, -0x1.8722d19ee2e8ep-9f}, - {0x1.fe6e1742f7cf5p-1f, 0x1.cd5ec93c1243ap-7f, -0x1.e2ff3aaacb386p-6f, - 0x1.2aa4e5823cc89p-5f, -0x1.d049842dbe399p-6f, 0x1.a34edb21ab302p-7f, - -0x1.676e5996c7f9bp-10f, -0x1.23b01a35140bfp-9f}, - {0x1.fed37386190fbp-1f, 0x1.61beae53b72c2p-7f, -0x1.7d6193f22c3c1p-6f, - 0x1.e947279e3bb7dp-6f, -0x1.906031b97ca97p-6f, 0x1.8d14d62561755p-7f, - -0x1.1f245e7178882p-9f, -0x1.9257d4eb47685p-10f}, - {0x1.ff20e0a7ba8c2p-1f, 0x1.0d1d69569b839p-7f, -0x1.2a8ca0dc02752p-6f, - 0x1.8cc071b709751p-6f, -0x1.54a149f1b070cp-6f, 0x1.6e9137b13412cp-7f, - -0x1.6577ed3d8e83bp-9f, -0x1.e9c1a5178a289p-11f}, - {0x1.ff5b8fb26f5f6p-1f, 0x1.9646f35a7663cp-8f, -0x1.cf68ed9311b0bp-7f, - 0x1.3e8735b5a694fp-6f, -0x1.1e1612d026fdfp-6f, 0x1.4afd8e6ca636dp-7f, - -0x1.8c375170ccb22p-9f, -0x1.c799443c4fd3bp-12f}, - {0x1.ff87b1913e853p-1f, 0x1.30499b5039596p-8f, -0x1.64964201ec8bap-7f, - 0x1.fa73d7eafba98p-7f, -0x1.daa3022141fbbp-7f, 0x1.2509444c063b7p-7f, - -0x1.99482a2f8a0a1p-9f, -0x1.403d1f76c9454p-15f}, - {0x1.ffa89fe5b3625p-1f, 0x1.c4412bf4b8f35p-9f, -0x1.100f347126cf0p-7f, - 0x1.8ebda07671d40p-7f, -0x1.850c6a31c98c1p-7f, 0x1.fdac860c67d21p-8f, - -0x1.927d03d2ba12cp-9f, 0x1.0ff620b4190fep-12f}, - {0x1.ffc10194fcb64p-1f, 0x1.4d78bba8ca621p-9f, -0x1.9ba107a443e02p-8f, - 0x1.36f273fbc04ccp-7f, -0x1.3b38716ac7e6fp-7f, 0x1.b3fe0181914acp-8f, - -0x1.7d3fe7de98c5cp-9f, 0x1.ea31f8e5317f7p-12f}, - {0x1.ffd2eae369a07p-1f, 0x1.e7f232d9e266cp-10f, -0x1.34c7442dd48d9p-8f, - 0x1.e066bed070a0bp-8f, -0x1.f914f3c42fc0dp-8f, 0x1.6f4664ed2260fp-8f, - -0x1.5e59910761d24p-9f, 0x1.39cbb6e84c126p-11f}, - {0x1.ffdff92db56e5p-1f, 0x1.6235fbd7a4373p-10f, -0x1.cb5e029b9e56ap-9f, - 0x1.6fa4c7ef274dap-8f, -0x1.903a089a835f3p-8f, 0x1.30f12e0ca1901p-8f, - -0x1.39d21b6957f99p-9f, 0x1.5d3f8495a703cp-11f}, - {0x1.ffe96a78a04a9p-1f, 0x1.fe41cd9bb4f2cp-11f, -0x1.52d7b28966c0cp-9f, - 0x1.16c192d86a1a7p-8f, -0x1.39bfce951100cp-8f, 0x1.f376a7869f9e3p-9f, - -0x1.12e6cef999c4fp-9f, 0x1.66acd4d667b5p-11f}, - {0x1.fff0312b010b5p-1f, 0x1.6caa0d3583018p-11f, -0x1.efb729f4cf75bp-10f, - 0x1.a2da7cebe12acp-9f, -0x1.e6c27a24bc759p-9f, 0x1.93b1f4d8ea65p-9f, - -0x1.d82050aa94a08p-10f, 0x1.5cd7dc75d6cbap-11f}, - {0x1.fff50456dab8cp-1f, 0x1.0295ef6591865p-11f, -0x1.679880e95a4dap-10f, - 0x1.37d38e3a5c8ebp-9f, -0x1.75b3708aebb8fp-9f, 0x1.4231c4b4b0296p-9f, - -0x1.8e26476489318p-10f, 0x1.45c3b570dd924p-11f}, - {0x1.fff86cfd3e657p-1f, 0x1.6be02102b353dp-12f, -0x1.02b157780d6aep-10f, - 0x1.cc1d886861133p-10f, -0x1.1bff6f12ec9abp-9f, 0x1.fc0f77bd9c736p-10f, - -0x1.4a3320bd0959dp-10f, 0x1.267f8b4f95d2p-11f}, - {0x1.fffad0b901755p-1f, 0x1.fc0d55470cf5ep-13f, -0x1.7121aff5e820ep-11f, - 0x1.506d6992f7de5p-10f, -0x1.ab595d3ecd0d6p-10f, 0x1.8bdd79daaf754p-10f, - -0x1.0d9b090f997c1p-10f, 0x1.031ab9fd1c7dap-11f}, - {0x1.fffc7a37857d2p-1f, 0x1.5feada379d8a5p-13f, -0x1.05304df58f3aap-11f, - 0x1.e79c081b8600fp-11f, -0x1.3e5dbe33232e0p-10f, 0x1.30eb208200729p-10f, - -0x1.b1d493b147945p-11f, 0x1.bd587bbc071bep-12f}, - {0x1.fffd9fdeabccep-1f, 0x1.e3bcf436a1a49p-14f, -0x1.6e953111ef0a1p-12f, - 0x1.5e3edf6768654p-11f, -0x1.d5be67c0547a4p-11f, 0x1.d07d9ffa1d435p-11f, - -0x1.58328f5f358cap-11f, 0x1.76d42d95c42c4p-12f}, - {0x1.fffe68f4fa777p-1f, 0x1.49e17724f4cddp-14f, -0x1.fe48c44e229c1p-13f, - 0x1.f2bd95d76f188p-12f, -0x1.57388cb12d011p-11f, 0x1.5decc25c5c079p-11f, - -0x1.0d7499d1b0d2dp-11f, 0x1.359332c94ecdcp-12f}, - {0x1.fffef1960d85dp-1f, 0x1.be6abbb10a4cdp-15f, -0x1.6040381a8c313p-13f, - 0x1.5fff1dde9ee9dp-12f, -0x1.f0c933efa9971p-12f, 0x1.04cbf4a5cd760p-11f, - -0x1.a07f150af6dadp-12f, 0x1.f68dd183426bap-13f}, - {0x1.ffff4db27f146p-1f, 0x1.2bb5cc22e5cd8p-15f, -0x1.e25894899f526p-14f, - 0x1.ec8a8e5a72757p-13f, -0x1.64256ae0a3cf9p-12f, 0x1.80a836c18c46cp-12f, - -0x1.3dea401af6775p-12f, 0x1.915ddff3fe0d1p-13f}, - {0x1.ffff8b500e77cp-1f, 0x1.8f4ccca7fc769p-16f, -0x1.478cffe305946p-14f, - 0x1.559f04adde504p-13f, -0x1.f9e1577d6961dp-13f, 0x1.18bda53c14716p-12f, - -0x1.df8634c35541cp-13f, 0x1.3bb5c6b616337p-13f}, - {0x1.ffffb43555b5fp-1f, 0x1.07ebd2a2d26c8p-16f, -0x1.b93e442a37f2bp-15f, - 0x1.d5cf15159ce28p-14f, -0x1.63f5e1469c006p-13f, 0x1.95a03acebac18p-13f, - -0x1.656e5e2a1f8e2p-13f, 0x1.e98c437189bdep-14f}, - {0x1.ffffcf23ff5fcp-1f, 0x1.5a2adfa0b492cp-17f, -0x1.26c88270759f0p-15f, - 0x1.40473572b99a8p-14f, -0x1.f057cbde578a5p-14f, 0x1.22178d1c3c948p-13f, - -0x1.0765b61a0d859p-13f, 0x1.765b3ea03ddbep-14f}, - {0x1.ffffe0bd3e852p-1f, 0x1.c282cd3957a72p-18f, -0x1.86ad6dfa44faap-16f, - 0x1.b0f313f03a029p-15f, -0x1.56e44abecd255p-14f, 0x1.9ad1ecfe34a89p-14f, - -0x1.7fe4033478618p-14f, 0x1.1a8184e049fbfp-14f}, - {0x1.ffffec2641a9ep-1f, 0x1.22df29821407ep-18f, -0x1.00c902a6cfd98p-16f, - 0x1.22234eb88671fp-15f, -0x1.d57a181c9e6e1p-15f, 0x1.200c283b54a90p-14f, - -0x1.14b4c3295a7d0p-14f, 0x1.a4f966f713bdep-15f}, - {0x1.fffff37d63a36p-1f, 0x1.74adc8f405eecp-19f, -0x1.4ed4228e44858p-17f, - 0x1.81918baea92bap-16f, -0x1.3e81b17a0009cp-15f, 0x1.9004a36116436p-15f, - -0x1.8aa1ba400e076p-15f, 0x1.35cd4e2340a9ep-15f}, - {0x1.fffff82cdcf1bp-1f, 0x1.d9c73698fa87dp-20f, -0x1.b11017ec67115p-18f, - 0x1.fc0dfadf653f8p-17f, -0x1.ac4e03cd2dfc2p-16f, 0x1.131806b5abbc5p-15f, - -0x1.1672ef66fcaafp-15f, 0x1.c2882c7debed7p-16f}, - {0x1.fffffb248c39dp-1f, 0x1.2acee2f5ec66ap-20f, -0x1.15cc570408a36p-18f, - 0x1.4be757bbb75a3p-17f, -0x1.1d6aa5f8d2940p-16f, 0x1.76c5937d5105ep-16f, - -0x1.84dffc3ca9302p-16f, 0x1.43c8315f2c30ap-16f}, - {0x1.fffffd01f36afp-1f, 0x1.75fa8dbc840bap-21f, -0x1.6186da0133f5ap-19f, - 0x1.ae023231e1af5p-18f, -0x1.790812f7ca394p-17f, 0x1.f9c25656d0ef2p-17f, - -0x1.0cc66682e304cp-16f, 0x1.cc170a75d6f9cp-17f}, - {0x1.fffffe2ba0ea5p-1f, 0x1.d06ad6ecde88ep-22f, -0x1.be46aa8edc9a1p-20f, - 0x1.143860c7840b8p-18f, -0x1.edaba78fb1260p-18f, 0x1.52138a96ecee2p-17f, - -0x1.6fca538c4e2eep-17f, 0x1.434040640bcefp-17f}, - {0x1.fffffee3cc32cp-1f, 0x1.1e1e857adb8ddp-22f, -0x1.1769ce5f2a6e8p-20f, - 0x1.5fe5d479b0543p-19f, -0x1.405d865c94c2ap-18f, 0x1.bfc94feb96afcp-18f, - -0x1.f245d5f3e8358p-18f, 0x1.c142456acf443p-18f}, - }; - float ax = __builtin_fabsf(x); - uint32_t ux = asuint(ax); - float s = x; - float z = ax; - /* 0x407ad444 corresponds to x = 0x1.f5a888p+1 = 3.91921..., which is the - largest float such that erf(x) does not round to 1 (to nearest). */ - if ((ux > 0x407ad444u)) { - float os = __builtin_copysignf(1.0f, x); - if (ux > (0xffu << 23)) - return x + x; /* nan */ - if (ux == (0xffu << 23)) - return os; /* +-inf */ - return os - 0x1p-25f * os; - } - uint32_t i = 16.0f * ax; - float v = static_cast(i); - /* 0x3ee00000 corresponds to x = 0.4375, for smaller x we have i < 7. */ - if ((ux < 0x3ee00000u)) { - static const float c[] = {0x1.20dd750429b6dp+0f, -0x1.812746b0375fbp-2f, - 0x1.ce2f219fd6f45p-4f, -0x1.b82ce2cbf0838p-6f, - 0x1.565bb655adb85p-8f, -0x1.c025bfc879c94p-11f, - 0x1.f81718f61309cp-14f, -0x1.cc67bd88f5867p-17f}; - float z2 = s * s, z4 = z2 * z2, z8 = z4 * z4; - float c0 = c[0] + z2 * c[1]; - float c2 = c[2] + z2 * c[3]; - float c4 = c[4] + z2 * c[5]; - float c6 = c[6] + z2 * c[7]; - c0 += z4 * c2; - c4 += z4 * c6; - c0 += z8 * c4; - return s * c0; - } - z = (z - 0.03125f) - 0.0625f * v; - const float *c = C[i - 7]; - float z2 = z * z, z4 = z2 * z2; - float c0 = c[0] + z * c[1]; - float c2 = c[2] + z * c[3]; - float c4 = c[4] + z * c[5]; - float c6 = c[6] + z * c[7]; - c0 += z2 * c2; - c4 += z2 * c6; - c0 += z4 * c4; - return __builtin_copysignf(c0, s); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/exp.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/math/exp.cpp deleted file mode 100644 index 8acc0b093..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/exp.cpp +++ /dev/null @@ -1,66 +0,0 @@ -// Vector adaptation of glibc 2.42 sysdeps/ieee754/flt-32/e_expf.c. -// Copyright (C) 2017-2025 Free Software Foundation, Inc. -// LGPL-2.1-or-later; see ../../../thirdparty/glibc/COPYING.LIB. -#include "constants.h" -#include "math.h" - -extern "C" vfloat32m1_t rvv_exp(vfloat32m1_t value, size_t vl) { - auto bits = __riscv_vreinterpret_u32m1(value); - auto magnitude = __riscv_vand_vx_u32m1(bits, 0x7fffffff, vl); - auto finite = __riscv_vmsltu_vx_u32m1_b32(magnitude, 0x7f800000, vl); - auto ordered = __riscv_vmerge_vvm_f32m1(__riscv_vfmv_v_f_f32m1(0, vl), value, - finite, vl); - auto normal = __riscv_vmfge_vf_f32m1_b32(ordered, -0x1.9d1d9ep6f, vl); - normal = __riscv_vmand_mm_b32( - normal, __riscv_vmfle_vf_f32m1_b32(ordered, 0x1.62e42ep6f, vl), vl); - normal = __riscv_vmand_mm_b32(normal, finite, vl); - auto input = __riscv_vmerge_vvm_f32m1(__riscv_vfmv_v_f_f32m1(0, vl), value, - normal, vl); - auto scaled = __riscv_vfmul_vf_f32m1(input, 0x1.715476p5f, vl); - auto kd = __riscv_vfadd_vf_f32m1(scaled, 0x1.8p23f, vl); - kd = __riscv_vfsub_vf_f32m1(kd, 0x1.8p23f, vl); - auto ki = __riscv_vfcvt_rtz_x_f_v_i32m1(kd, vl); - auto r = __riscv_vfnmsac_vf_f32m1(input, 0x1.62e4p-6f, kd, vl); - r = __riscv_vfnmsac_vf_f32m1(r, 0x1.7f7d1cp-25f, kd, vl); - auto index = __riscv_vand_vx_u32m1(__riscv_vreinterpret_u32m1(ki), 31, vl); - index = __riscv_vsll_vx_u32m1(index, 2, vl); - auto table = __riscv_vluxei32_v_f32m1(exp_table, index, vl); - auto exponent = __riscv_vsra_vx_i32m1(ki, 5, vl); - auto bounded = __riscv_vmax_vx_i32m1(exponent, -126, vl); - bounded = __riscv_vmin_vx_i32m1(bounded, 127, vl); - auto tail = __riscv_vsub_vv_i32m1(exponent, bounded, vl); - auto scale_bits = - __riscv_vsll_vx_i32m1(__riscv_vadd_vx_i32m1(bounded, 127, vl), 23, vl); - auto tail_bits = - __riscv_vsll_vx_i32m1(__riscv_vadd_vx_i32m1(tail, 127, vl), 23, vl); - auto polynomial = __riscv_vfmacc_vf_f32m1(__riscv_vfmv_v_f_f32m1(0.5f, vl), - 1.0f / 6.0f, r, vl); - auto y = __riscv_vfadd_vf_f32m1(r, 1.0f, vl); - y = __riscv_vfmacc_vv_f32m1(y, polynomial, __riscv_vfmul_vv_f32m1(r, r, vl), - vl); - y = __riscv_vfmul_vv_f32m1(y, table, vl); - y = __riscv_vfmul_vv_f32m1( - y, __riscv_vreinterpret_f32m1(__riscv_vreinterpret_u32m1(tail_bits)), vl); - auto result = __riscv_vfmul_vv_f32m1( - y, __riscv_vreinterpret_f32m1(__riscv_vreinterpret_u32m1(scale_bits)), - vl); - auto under = __riscv_vmflt_vf_f32m1_b32(ordered, -0x1.9d1d9ep6f, vl); - under = __riscv_vmand_mm_b32(under, finite, vl); - auto helper = __riscv_vfmv_v_f_f32m1(0x1.4p-75f, vl); - helper = __riscv_vfmul_vv_f32m1_m(under, helper, helper, vl); - result = __riscv_vmerge_vvm_f32m1(result, helper, under, vl); - auto zero = __riscv_vmflt_vf_f32m1_b32(ordered, -0x1.9fe368p6f, vl); - result = __riscv_vfmerge_vfm_f32m1(result, 0, zero, vl); - auto over = __riscv_vmfgt_vf_f32m1_b32(ordered, 0x1.62e42ep6f, vl); - auto overflow = __riscv_vmand_mm_b32(over, finite, vl); - helper = __riscv_vfmv_v_f_f32m1(0x1p97f, vl); - helper = __riscv_vfmul_vv_f32m1_m(overflow, helper, helper, vl); - result = __riscv_vmerge_vvm_f32m1(result, helper, overflow, vl); - auto infinity = __riscv_vmseq_vx_u32m1_b32(bits, 0x7f800000, vl); - result = __riscv_vfmerge_vfm_f32m1(result, __builtin_inff(), infinity, vl); - auto negative_infinity = __riscv_vmseq_vx_u32m1_b32(bits, 0xff800000, vl); - result = __riscv_vfmerge_vfm_f32m1(result, 0, negative_infinity, vl); - auto nan = __riscv_vmsgtu_vx_u32m1_b32(magnitude, 0x7f800000, vl); - return __riscv_vmerge_vvm_f32m1( - result, __riscv_vfadd_vv_f32m1_m(nan, value, value, vl), nan, vl); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/math.h b/bb-tests/workloads/lib/bbsw/kernels/rvv/math/math.h deleted file mode 100644 index ef5af1af0..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/math.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once -#include - -extern "C" vfloat32m1_t rvv_exp(vfloat32m1_t value, size_t vl); -extern "C" vfloat32m1_t rvv_sin(vfloat32m1_t value, size_t vl); -extern "C" vfloat32m1_t rvv_cos(vfloat32m1_t value, size_t vl); diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/mxfp8.h b/bb-tests/workloads/lib/bbsw/kernels/rvv/math/mxfp8.h deleted file mode 100644 index 89299bc7e..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/mxfp8.h +++ /dev/null @@ -1,64 +0,0 @@ -#pragma once -#include -#include - -static inline vfloat32m2_t rvv_mxfp8_load16(const uint8_t *input, - unsigned scale, uint32_t &status) { - constexpr size_t vl = 16; - if (scale == 255) { - status = 1U << 8; - return __riscv_vfmv_v_f_f32m2(0, vl); - } - auto bytes = __riscv_vle8_v_u8mf2(input, vl); - auto code = __riscv_vzext_vf2_u32m2(__riscv_vzext_vf2_u16m1(bytes, vl), vl); - auto sign = - __riscv_vsll_vx_u32m2(__riscv_vand_vx_u32m2(code, 128, vl), 24, vl); - auto magnitude = __riscv_vand_vx_u32m2(code, 127, vl); - if (__riscv_vcpop_m_b16(__riscv_vmseq_vx_u32m2_b16(magnitude, 127, vl), vl)) { - status = 1U << 8; - return __riscv_vfmv_v_f_f32m2(0, vl); - } - auto exponent = __riscv_vsrl_vx_u32m2(magnitude, 3, vl); - auto fraction = __riscv_vand_vx_u32m2(code, 7, vl); - auto subnormal = __riscv_vmseq_vx_u32m2_b16(exponent, 0, vl); - auto highest = __riscv_vmv_v_x_u32m2(0, vl); - highest = __riscv_vmerge_vxm_u32m2( - highest, 1, __riscv_vmsgeu_vx_u32m2_b16(fraction, 2, vl), vl); - highest = __riscv_vmerge_vxm_u32m2( - highest, 2, __riscv_vmsgeu_vx_u32m2_b16(fraction, 4, vl), vl); - auto normalExponent = __riscv_vadd_vx_i32m2( - __riscv_vreinterpret_v_u32m2_i32m2(exponent), int(scale) - 7, vl); - auto subExponent = __riscv_vadd_vx_i32m2( - __riscv_vreinterpret_v_u32m2_i32m2(highest), int(scale) - 9, vl); - auto resultExponent = - __riscv_vmerge_vvm_i32m2(normalExponent, subExponent, subnormal, vl); - auto overflow = - __riscv_vmand_mm_b16(__riscv_vmsge_vx_i32m2_b16(resultExponent, 255, vl), - __riscv_vmsne_vx_u32m2_b16(magnitude, 0, vl), vl); - if (__riscv_vcpop_m_b16(overflow, vl)) { - status = 2U << 8; - return __riscv_vfmv_v_f_f32m2(0, vl); - } - auto normalSignificand = - __riscv_vsll_vx_u32m2(__riscv_vor_vx_u32m2(fraction, 8, vl), 20, vl); - auto subSignificand = __riscv_vsll_vv_u32m2( - fraction, __riscv_vrsub_vx_u32m2(highest, 23, vl), vl); - auto significand = __riscv_vmerge_vvm_u32m2(normalSignificand, subSignificand, - subnormal, vl); - auto normalBits = __riscv_vor_vv_u32m2( - __riscv_vsll_vx_u32m2(__riscv_vreinterpret_v_i32m2_u32m2(resultExponent), - 23, vl), - __riscv_vand_vx_u32m2(significand, 0x7fffff, vl), vl); - auto tinyBits = - __riscv_vsrl_vv_u32m2(significand, - __riscv_vreinterpret_v_i32m2_u32m2( - __riscv_vrsub_vx_i32m2(resultExponent, 1, vl)), - vl); - auto bits = __riscv_vmerge_vvm_u32m2( - normalBits, tinyBits, __riscv_vmsle_vx_i32m2_b16(resultExponent, 0, vl), - vl); - bits = __riscv_vmerge_vxm_u32m2( - bits, 0, __riscv_vmseq_vx_u32m2_b16(magnitude, 0, vl), vl); - return __riscv_vreinterpret_v_u32m2_f32m2( - __riscv_vor_vv_u32m2(bits, sign, vl)); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/sin.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/math/sin.cpp deleted file mode 100644 index 63212a3fd..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/math/sin.cpp +++ /dev/null @@ -1,107 +0,0 @@ -// Vector adaptation of glibc 2.42 s_sinf.c and s_sincosf.h/sincosf_poly.h. -// Copyright (C) 2018-2025 Free Software Foundation, Inc. -// LGPL-2.1-or-later; see ../../../thirdparty/glibc/COPYING.LIB. -#include "constants.h" -#include "math.h" - -extern "C" vfloat32m1_t rvv_sin(vfloat32m1_t value, size_t vl) { - auto bits = __riscv_vreinterpret_u32m1(value); - auto top = - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 20, vl), 0x7ff, vl); - auto tiny = __riscv_vmsltu_vx_u32m1_b32(top, 0x398, vl); - auto fast = __riscv_vmsltu_vx_u32m1_b32(top, 0x42f, vl); - fast = __riscv_vmandn_mm_b32(fast, tiny, vl); - auto finite = __riscv_vmsltu_vx_u32m1_b32(top, 0x7f8, vl); - auto input = - __riscv_vmerge_vvm_f32m1(__riscv_vfmv_v_f_f32m1(0, vl), value, fast, vl); - auto x = input; - auto r = __riscv_vfmul_vf_f32m1(x, 0x1.45f306dc9c883p23, vl); - auto n = __riscv_vfcvt_rtz_x_f_v_i32m1(r, vl); - n = __riscv_vsra_vx_i32m1(__riscv_vadd_vx_i32m1(n, 0x800000, vl), 24, vl); - auto small = __riscv_vmsltu_vx_u32m1_b32(top, 0x3f4, vl); - n = __riscv_vmerge_vxm_i32m1(n, 0, small, vl); - auto nd = __riscv_vfcvt_f_x_v_f32m1(n, vl); - x = __riscv_vfnmsac_vf_f32m1(x, 0x1.921fb4p0f, nd, vl); - x = __riscv_vfnmsac_vf_f32m1(x, 0x1.4442d2p-24f, nd, vl); - auto quadrant = n; - auto large = __riscv_vmandn_mm_b32(finite, fast, vl); - large = __riscv_vmandn_mm_b32(large, tiny, vl); - if (__riscv_vcpop_m_b32(large, vl)) { - auto index = - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 26, vl), 15, vl); - index = __riscv_vsll_vx_u32m1(index, 2, vl); - auto a0 = __riscv_vluxei32_v_u32m1(inv_pio4, index, vl); - auto a1 = __riscv_vluxei32_v_u32m1(inv_pio4 + 4, index, vl); - auto a2 = __riscv_vluxei32_v_u32m1(inv_pio4 + 8, index, vl); - auto shift = - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 23, vl), 7, vl); - auto xi = __riscv_vor_vx_u32m1(__riscv_vand_vx_u32m1(bits, 0xffffff, vl), - 0x800000, vl); - xi = __riscv_vsll_vv_u32m1(xi, shift, vl); - auto low0 = __riscv_vmulhu_vv_u32m1(xi, a2, vl); - auto low1 = __riscv_vmul_vv_u32m1(xi, a1, vl); - auto low = __riscv_vadd_vv_u32m1(low0, low1, vl); - auto carry_mask = __riscv_vmsltu_vv_u32m1_b32(low, low1, vl); - auto high = __riscv_vadd_vv_u32m1(__riscv_vmul_vv_u32m1(xi, a0, vl), - __riscv_vmulhu_vv_u32m1(xi, a1, vl), vl); - auto carry = __riscv_vmerge_vxm_u32m1(__riscv_vmv_v_x_u32m1(0, vl), 1, - carry_mask, vl); - high = __riscv_vadd_vv_u32m1(high, carry, vl); - auto q = __riscv_vsrl_vx_u32m1(__riscv_vadd_vx_u32m1(high, 0x20000000, vl), - 30, vl); - high = __riscv_vsub_vv_u32m1(high, __riscv_vsll_vx_u32m1(q, 30, vl), vl); - auto reduced = - __riscv_vfcvt_f_x_v_f32m1(__riscv_vreinterpret_i32m1(high), vl); - reduced = __riscv_vfmul_vf_f32m1(reduced, 0x1.921fb4p-30f, vl); - auto fraction = __riscv_vfcvt_f_xu_v_f32m1(low, vl); - reduced = __riscv_vfmacc_vf_f32m1(reduced, 0x1.921fb4p-62f, fraction, vl); - auto q32 = __riscv_vreinterpret_i32m1(q); - n = __riscv_vmerge_vvm_i32m1(n, q32, large, vl); - auto sign = __riscv_vreinterpret_i32m1(__riscv_vsrl_vx_u32m1(bits, 31, vl)); - quadrant = __riscv_vmerge_vvm_i32m1( - quadrant, __riscv_vadd_vv_i32m1(q32, sign, vl), large, vl); - x = __riscv_vmerge_vvm_f32m1(x, reduced, large, vl); - } - auto x2 = __riscv_vfmul_vv_f32m1(x, x, vl); - auto negative = - __riscv_vmsne_vx_i32m1_b32(__riscv_vand_vx_i32m1(quadrant, 2, vl), 0, vl); - // sign[] is {1,-1,-1,1}; cosine polynomial sign depends on bit 1. - auto signbit = __riscv_vxor_vv_i32m1( - quadrant, __riscv_vsra_vx_i32m1(quadrant, 1, vl), vl); - auto sine_negative = - __riscv_vmsne_vx_i32m1_b32(__riscv_vand_vx_i32m1(signbit, 1, vl), 0, vl); - auto sx = __riscv_vmerge_vvm_f32m1(x, __riscv_vfneg_v_f32m1(x, vl), - sine_negative, vl); - auto x3 = __riscv_vfmul_vv_f32m1(sx, x2, vl); - auto s1 = - __riscv_vfmacc_vf_f32m1(__riscv_vfmv_v_f_f32m1(0x1.1107605230bc4p-7, vl), - -0x1.994eb3774cf24p-13, x2, vl); - auto s = __riscv_vfmacc_vf_f32m1(sx, -0x1.555545995a603p-3, x3, vl); - auto x7 = __riscv_vfmul_vv_f32m1(x3, x2, vl); - s = __riscv_vfmacc_vv_f32m1(s, s1, x7, vl); - auto x4 = __riscv_vfmul_vv_f32m1(x2, x2, vl); - auto c2 = __riscv_vfmacc_vf_f32m1( - __riscv_vfmv_v_f_f32m1(-0x1.6c087e89a359dp-10, vl), 0x1.99343027bf8c3p-16, - x2, vl); - auto c1 = __riscv_vfmacc_vf_f32m1(__riscv_vfmv_v_f_f32m1(1, vl), - -0x1.ffffffd0c621cp-2, x2, vl); - auto c = __riscv_vfmacc_vf_f32m1(c1, 0x1.55553e1068f19p-5, x4, vl); - auto x6 = __riscv_vfmul_vv_f32m1(x4, x2, vl); - c = __riscv_vfmacc_vv_f32m1(c, c2, x6, vl); - c = __riscv_vmerge_vvm_f32m1(c, __riscv_vfneg_v_f32m1(c, vl), negative, vl); - auto odd = __riscv_vmsne_vx_i32m1_b32(__riscv_vand_vx_i32m1(n, 1, vl), 0, vl); - auto result = __riscv_vmerge_vvm_f32m1(s, c, odd, vl); - auto subnormal = __riscv_vmsltu_vx_u32m1_b32(top, 0x008, vl); - if (__riscv_vcpop_m_b32(subnormal, vl)) { - auto extended = value; - auto square = __riscv_vfmul_vv_f32m1_m(subnormal, extended, extended, vl); - auto rounded = square; - // glibc forces this conversion for its underflow/inexact side effects. - asm volatile("" : : "vr"(rounded)); - } - result = __riscv_vmerge_vvm_f32m1(result, value, tiny, vl); - auto invalid = __riscv_vmnot_m_b32(finite, vl); - auto difference = __riscv_vfsub_vv_f32m1_m(invalid, value, value, vl); - auto nan = __riscv_vfdiv_vv_f32m1_m(invalid, difference, difference, vl); - return __riscv_vmerge_vvm_f32m1(result, nan, invalid, vl); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/matmul.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/matmul.cpp deleted file mode 100644 index 5932082b3..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/matmul.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "kernels.h" -#include - -extern "C" void rvv_matmul(float *output, const float *lhs, const float *rhs, - size_t rows, size_t cols, size_t inner) { - for (size_t row = 0; row < rows; ++row) { - for (size_t col = 0; col < cols;) { - const size_t vl = __riscv_vsetvl_e32m1(cols - col); - auto sum = __riscv_vle32_v_f32m1(output + row * cols + col, vl); - for (size_t k = 0; k < inner; ++k) { - auto weight = __riscv_vle32_v_f32m1(rhs + k * cols + col, vl); - auto product = __riscv_vfmul_vf_f32m1(weight, lhs[row * inner + k], vl); - sum = __riscv_vfadd_vv_f32m1(sum, product, vl); - } - __riscv_vse32_v_f32m1(output + row * cols + col, sum, vl); - col += vl; - } - } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/matmul_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/matmul_dispatch.cpp deleted file mode 100644 index 5a9476713..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/matmul_dispatch.cpp +++ /dev/null @@ -1,212 +0,0 @@ -#include "images.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 4, writeBank = 5; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr uint32_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); -static_assert(bankBytes >= 4096 && bankBytes <= 65536 && bankBytes % 16 == 0); -} // namespace - -extern "C" void _mlir_ciface_rvv_matmul(UnrankedMemRefType *output, - UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs) { - DynamicMemRefType out(*output), a(*lhs), b(*rhs); - if ((out.rank != 2 && out.rank != 3) || a.rank != out.rank || - b.rank != out.rank) { - fputs("RVV matmul requires matching rank-two or rank-three tensors\n", - stderr); - abort(); - } - const int64_t axis = out.rank - 2; - const int64_t rows = a.sizes[axis], inner = a.sizes[axis + 1]; - const int64_t cols = b.sizes[axis + 1]; - const int64_t batches = axis ? out.sizes[0] : 1; - if (rows <= 0 || inner <= 0 || cols <= 0 || batches <= 0 || - b.sizes[axis] != inner || out.sizes[axis] != rows || - out.sizes[axis + 1] != cols || - (axis && (a.sizes[0] != batches || b.sizes[0] != batches))) { - fprintf(stderr, - "RVV matmul shape mismatch: output=[%lld,%lld,%lld], " - "lhs=[%lld,%lld,%lld], rhs=[%lld,%lld,%lld]\n", - (long long)(axis ? out.sizes[0] : 1), (long long)out.sizes[axis], - (long long)out.sizes[axis + 1], (long long)(axis ? a.sizes[0] : 1), - (long long)a.sizes[axis], (long long)a.sizes[axis + 1], - (long long)(axis ? b.sizes[0] : 1), (long long)b.sizes[axis], - (long long)b.sizes[axis + 1]); - abort(); - } - alignas(16) float aTile[bankBytes / sizeof(float)]; - alignas(16) float bTile[bankBytes / sizeof(float)]; - alignas(16) float cTile[16 * 16]; - for (unsigned bank = 0; bank < 4; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mem_transfer(2, readBank); - bb_mem_transfer(3, readBank); - bb_mem_transfer(0, writeBank); - bb_mem_transfer(1, writeBank); - mvin_kernel(images::matmul.bytes, images::matmul.size, programBank); - for (int64_t batch = 0; batch < batches; ++batch) { - const float *aSource = - a.data + a.offset + (axis ? batch * a.strides[0] : 0); - const float *bSource = - b.data + b.offset + (axis ? batch * b.strides[0] : 0); - float *cSource = - out.data + out.offset + (axis ? batch * out.strides[0] : 0); - for (int64_t row = 0; row < rows; row += 16) { - const uint32_t m = std::min(16, rows - row); - for (int64_t col = 0; col < cols; col += 16) { - const uint32_t n = std::min(16, cols - col); - const uint32_t count = m * n; - float *cDirect = - cSource + row * out.strides[axis] + col * out.strides[axis + 1]; - const bool directC = - out.strides[axis + 1] == 1 && (m == 1 || out.strides[axis] == n) && - count % 4 == 0 && (reinterpret_cast(cDirect) & 15) == 0; - if (directC) { - bb_mvin_group((uintptr_t)cDirect, writeBank, 1, count / 4, 1); - } else { - for (uint32_t r = 0; r < m; ++r) - for (uint32_t c = 0; c < n; ++c) - cTile[r * n + c] = cSource[(row + r) * out.strides[axis] + - (col + c) * out.strides[axis + 1]]; - std::fill(cTile + count, cTile + (count + 3) / 4 * 4, 0.0f); - bb_mvin_group((uintptr_t)cTile, writeBank, 1, (count + 3) / 4, 1); - } - const int64_t tileK = bankBytes / sizeof(float) / std::max(m, n); - for (int64_t begin = 0; begin < inner; begin += tileK) { - const uint32_t k = std::min(tileK, inner - begin); - const float *aDirect = - aSource + row * a.strides[axis] + begin * a.strides[axis + 1]; - const float *bDirect = - bSource + begin * b.strides[axis] + col * b.strides[axis + 1]; - const bool directA = a.strides[axis + 1] == 1 && - (m == 1 || a.strides[axis] == k) && - m * k % 4 == 0 && - (reinterpret_cast(aDirect) & 15) == 0; - const bool directB = b.strides[axis + 1] == 1 && - (k == 1 || b.strides[axis] == n) && - k * n % 4 == 0 && - (reinterpret_cast(bDirect) & 15) == 0; - if (directA) { - bb_mvin_group((uintptr_t)aDirect, readBank, 0, m * k / 4, 1); - } else { - for (uint32_t r = 0; r < m; ++r) { - const float *source = aSource + (row + r) * a.strides[axis] + - begin * a.strides[axis + 1]; - if (a.strides[axis + 1] == 1) - std::memcpy(aTile + r * k, source, k * sizeof(float)); - else - for (uint32_t i = 0; i < k; ++i) - aTile[r * k + i] = source[i * a.strides[axis + 1]]; - } - std::fill(aTile + m * k, aTile + (m * k + 3) / 4 * 4, 0.0f); - bb_mvin_group((uintptr_t)aTile, readBank, 0, (m * k + 3) / 4, 1); - } - if (directB) { - bb_mvin_group((uintptr_t)bDirect, readBank, 1, k * n / 4, 1); - } else { - for (uint32_t i = 0; i < k; ++i) { - const float *source = bSource + (begin + i) * b.strides[axis] + - col * b.strides[axis + 1]; - if (b.strides[axis + 1] == 1) - std::memcpy(bTile + i * n, source, n * sizeof(float)); - else - for (uint32_t c = 0; c < n; ++c) - bTile[i * n + c] = source[c * b.strides[axis + 1]]; - } - std::fill(bTile + k * n, bTile + (k * n + 3) / 4 * 4, 0.0f); - bb_mvin_group((uintptr_t)bTile, readBank, 1, (k * n + 3) / 4, 1); - } - kernel_launch call{images::matmul.entry, - images::matmul.text_bytes, - 0x40002000, - {uint64_t(3) << 32, 0, uint64_t(1) << 32, m, n, k}, - 0}; - bb_mvin_group((uintptr_t)&call, writeBank, 0, sizeof(call) / 16, 1); - run_kernel(readBank, programBank, writeBank, 0); - } - if (directC) { - bb_mvout_group((uintptr_t)cDirect, writeBank, 1, count / 4, 1); - } else { - bb_mvout_group((uintptr_t)cTile, writeBank, 1, (count + 3) / 4, 1); - for (uint32_t r = 0; r < m; ++r) - for (uint32_t c = 0; c < n; ++c) - cSource[(row + r) * out.strides[axis] + - (col + c) * out.strides[axis + 1]] = cTile[r * n + c]; - } - } - } - } - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} - -extern "C" void -_mlir_ciface_rvv_matmul_transpose_rhs(UnrankedMemRefType *output, - UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs) { - DynamicMemRefType b(*rhs); - if (b.rank == 2) { - StridedMemRefType view{b.basePtr, - b.data, - b.offset, - {b.sizes[1], b.sizes[0]}, - {b.strides[1], b.strides[0]}}; - UnrankedMemRefType transposed{2, &view}; - _mlir_ciface_rvv_matmul(output, lhs, &transposed); - } else if (b.rank == 3) { - StridedMemRefType view{ - b.basePtr, - b.data, - b.offset, - {b.sizes[0], b.sizes[2], b.sizes[1]}, - {b.strides[0], b.strides[2], b.strides[1]}}; - UnrankedMemRefType transposed{3, &view}; - _mlir_ciface_rvv_matmul(output, lhs, &transposed); - } else { - fputs("RVV transposed matmul requires rank-two or rank-three RHS\n", - stderr); - abort(); - } -} - -namespace { -void broadcastMatmul(UnrankedMemRefType *output, - UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs, bool transpose) { - DynamicMemRefType out(*output), b(*rhs); - if (out.rank != 3 || b.rank != 4 || b.sizes[0] != 1 || b.sizes[1] != 1) { - fputs("RVV broadcast matmul requires one rank-four RHS head\n", stderr); - abort(); - } - StridedMemRefType view{ - b.basePtr, - b.data, - b.offset, - {out.sizes[0], b.sizes[transpose ? 3 : 2], b.sizes[transpose ? 2 : 3]}, - {0, b.strides[transpose ? 3 : 2], b.strides[transpose ? 2 : 3]}}; - UnrankedMemRefType repeated{3, &view}; - _mlir_ciface_rvv_matmul(output, lhs, &repeated); -} -} // namespace - -extern "C" void -_mlir_ciface_rvv_matmul_broadcast_rhs(UnrankedMemRefType *output, - UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs) { - broadcastMatmul(output, lhs, rhs, false); -} - -extern "C" void _mlir_ciface_rvv_matmul_broadcast_transpose_rhs( - UnrankedMemRefType *output, UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs) { - broadcastMatmul(output, lhs, rhs, true); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/norm.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/norm.cpp deleted file mode 100644 index 43259fd60..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/norm.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include "kernels.h" -#include - -template -static void norm(float *output, const float *input, const float *weight, - size_t width, uint32_t meanMultiplierBits, - uint32_t epsilonBits) { - auto sum = __riscv_vfmv_v_f_f32m1(0, 1); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(input + offset, vl); - auto squares = __riscv_vfmul_vv_f32m1(values, values, vl); - sum = __riscv_vfredosum_vs_f32m1_f32m1(squares, sum, vl); - offset += vl; - } - auto meanMultiplier = - __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(meanMultiplierBits, 1)); - auto epsilon = - __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(epsilonBits, 1)); - auto mean = __riscv_vfmul_vv_f32m1(sum, meanMultiplier, 1); - auto biased = __riscv_vfadd_vv_f32m1(mean, epsilon, 1); - auto root = __riscv_vfsqrt_v_f32m1(biased, 1); - auto reciprocal = __riscv_vfrdiv_vf_f32m1(root, 1, 1); - const float scale = __riscv_vfmv_f_s_f32m1_f32(reciprocal); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(input + offset, vl); - auto scaled = __riscv_vfmul_vf_f32m1(values, scale, vl); - if constexpr (weighted) { - auto weights = __riscv_vle32_v_f32m1(weight + offset, vl); - scaled = __riscv_vfmul_vv_f32m1(weights, scaled, vl); - } - __riscv_vse32_v_f32m1(output + offset, scaled, vl); - offset += vl; - } -} - -extern "C" void rvv_norm(float *output, const float *input, const float *weight, - size_t width, uint32_t mean, uint32_t epsilon) { - norm(output, input, weight, width, mean, epsilon); -} - -extern "C" void rvv_norm_no_weight(float *output, const float *input, - size_t width, uint32_t mean, - uint32_t epsilon) { - norm(output, input, nullptr, width, mean, epsilon); -} - -extern "C" void rvv_norm_launch(float *output, const float *input, - const float *weight, size_t width, - uint32_t mean, uint32_t epsilon, - uint32_t *flags, uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - rvv_norm(output, input, weight, width, mean, epsilon); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} - -extern "C" void rvv_norm_no_weight_launch(float *output, const float *input, - size_t width, uint32_t mean, - uint32_t epsilon, uint32_t *flags, - uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - rvv_norm_no_weight(output, input, width, mean, epsilon); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/normalization_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/normalization_dispatch.cpp deleted file mode 100644 index fa69cf8da..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/normalization_dispatch.cpp +++ /dev/null @@ -1,178 +0,0 @@ -#include "images.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 5, writeBank = 6; -constexpr unsigned inputBank = 0, weightBank = 1, biasBank = 2; -constexpr unsigned descriptorBank = 3, outputBank = 4; -constexpr uint64_t descriptorAddress = uint64_t(descriptorBank) << 32; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr uint32_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); - -int64_t elements(const DynamicMemRefType &value) { - int64_t count = 1; - for (int64_t axis = value.rank - 1; axis >= 0; --axis) { - if (value.sizes[axis] <= 0 || - (value.sizes[axis] != 1 && value.strides[axis] != count)) { - fputs("RVV normalization requires contiguous FP32 tensors\n", stderr); - abort(); - } - count *= value.sizes[axis]; - } - return count; -} -void load(const images::KernelImage &image) { - for (unsigned bank = 0; bank < 5; ++bank) - bb_mem_alloc(bank, 1, 1); - for (unsigned bank = 0; bank < 3; ++bank) - bb_mem_transfer(bank, readBank); - bb_mem_transfer(descriptorBank, writeBank); - bb_mem_transfer(outputBank, writeBank); - mvin_kernel(image.bytes, image.size, programBank); -} -void send(const float *input, unsigned bank, uint32_t count, float *staging) { - if (count % 4) { - std::memcpy(staging, input, count * sizeof(float)); - std::fill_n(staging + count, 4 - count % 4, 0.0f); - input = staging; - } - bb_mvin_group((uintptr_t)input, readBank, bank, (count + 3) / 4, 1); -} -void launch(kernel_launch &descriptor) { - bb_mvin_group((uintptr_t)&descriptor, writeBank, 0, sizeof(descriptor) / 16, - 1); - run_kernel(readBank, programBank, writeBank, 0); -} -void receive(float *output, uint32_t count, float *staging) { - bb_mvout_group((uintptr_t)(count % 4 ? staging : output), writeBank, 1, - (count + 3) / 4, 1); - alignas(16) kernel_launch descriptor; - bb_mvout_group((uintptr_t)&descriptor, writeBank, 0, sizeof(descriptor) / 16, - 1); - if (count % 4) - std::memcpy(output, staging, count * sizeof(float)); - asm volatile("csrs fflags, %0" ::"r"(descriptor.reserved) : "memory"); -} -void release() { - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} -uint32_t roundingMode() { - uint32_t rounding; - asm volatile("csrr %0, frm" : "=r"(rounding)::"memory"); - return rounding << 5; -} -} // namespace - -static void layernorm(UnrankedMemRefType *output, - UnrankedMemRefType *input, - UnrankedMemRefType *weight, - UnrankedMemRefType *bias, uint32_t multiplier, - uint32_t epsilon) { - DynamicMemRefType out(*output), in(*input), w(*weight); - const int64_t count = elements(in); - const int64_t width = in.sizes[in.rank - 1]; - if (elements(out) != count || elements(w) != width || - width > bankBytes / sizeof(float)) { - fputs("RVV LayerNorm requires one complete row and weights in banks\n", - stderr); - abort(); - } - alignas(16) float staging[bankBytes / sizeof(float)]; - const uint32_t rounding = roundingMode(); - load(images::layernorm); - send(w.data + w.offset, weightBank, width, staging); - if (bias) { - DynamicMemRefType b(*bias); - if (elements(b) != width) - abort(); - send(b.data + b.offset, biasBank, width, staging); - } - for (int64_t begin = 0; begin < count; begin += width) { - send(in.data + in.offset + begin, inputBank, width, staging); - alignas(16) kernel_launch call{ - images::layernorm.entry, - images::layernorm.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, - uint64_t(weightBank) << 32, uint64_t(biasBank) << 32, - uint64_t(width) | (uint64_t(bias != nullptr) << 32), - (uint64_t(epsilon) << 32) | multiplier, - descriptorAddress + offsetof(kernel_launch, reserved), rounding}, - 0}; - launch(call); - receive(out.data + out.offset + begin, width, staging); - } - release(); -} - -extern "C" void _mlir_ciface_rvv_layernorm(UnrankedMemRefType *output, - UnrankedMemRefType *input, - UnrankedMemRefType *weight, - UnrankedMemRefType *bias, - uint32_t multiplier, - uint32_t epsilon) { - layernorm(output, input, weight, bias, multiplier, epsilon); -} - -extern "C" void _mlir_ciface_rvv_layernorm_no_bias( - UnrankedMemRefType *output, UnrankedMemRefType *input, - UnrankedMemRefType *weight, uint32_t multiplier, uint32_t epsilon) { - layernorm(output, input, weight, nullptr, multiplier, epsilon); -} - -extern "C" void _mlir_ciface_rvv_gelu(UnrankedMemRefType *output, - UnrankedMemRefType *input) { - DynamicMemRefType out(*output), in(*input); - const int64_t count = elements(out); - int64_t inputCount = 1; - bool contiguous = true; - for (int64_t axis = in.rank - 1; axis >= 0; --axis) { - if (in.sizes[axis] != 1 && in.strides[axis] != inputCount) - contiguous = false; - inputCount *= in.sizes[axis]; - } - if (inputCount != count) - abort(); - alignas(16) float staging[bankBytes / sizeof(float)]; - const uint32_t rounding = roundingMode(); - load(images::gelu); - for (int64_t begin = 0; begin < count;) { - const uint32_t length = - std::min(bankBytes / sizeof(float), count - begin); - if (contiguous) { - send(in.data + in.offset + begin, inputBank, length, staging); - } else { - for (uint32_t index = 0; index < length; ++index) { - int64_t remaining = begin + index, offset = in.offset; - for (int64_t axis = in.rank - 1; axis >= 0; --axis) { - offset += (remaining % in.sizes[axis]) * in.strides[axis]; - remaining /= in.sizes[axis]; - } - staging[index] = in.data[offset]; - } - std::fill_n(staging + length, (4 - length % 4) % 4, 0.0f); - bb_mvin_group((uintptr_t)staging, readBank, inputBank, (length + 3) / 4, - 1); - } - alignas(16) kernel_launch call{ - images::gelu.entry, - images::gelu.text_bytes, - 0x40002000, - {uint64_t(outputBank) << 32, uint64_t(inputBank) << 32, length, - descriptorAddress + offsetof(kernel_launch, reserved), rounding, 0, 0, - 0}, - 0}; - launch(call); - receive(out.data + out.offset + begin, length, staging); - begin += length; - } - release(); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/pack.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/pack.cpp deleted file mode 100644 index 1d4161321..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/pack.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include -#include -#include - -extern "C" void rvv_pack(uint32_t *output, const uint32_t *input, size_t fullK, - size_t fullN, size_t startK, size_t startN, - size_t countK, size_t columns) { - for (size_t column = 0; column < columns; ++column) - for (size_t offset = 0; offset < countK;) { - const size_t vl = __riscv_vsetvl_e32m1(countK - offset); - auto value = __riscv_vmv_v_x_u32m1(0, vl); - if (startN + column < fullN) - value = __riscv_vlse32_v_u32m1(input + (startK + offset) * fullN + - startN + column, - fullN * sizeof(uint32_t), vl); - __riscv_vse32_v_u32m1(output + column * countK + offset, value, vl); - offset += vl; - } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/pointwise.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/pointwise.cpp deleted file mode 100644 index e8d1b99b2..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/pointwise.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "kernels.h" -#include - -extern "C" void rvv_pointwise(float *output, const float *lhs, const float *rhs, - size_t count, uint32_t operation, uint32_t *flags, - uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - for (size_t first = 0; first < count;) { - size_t vl = __riscv_vsetvl_e32m1(count - first); - auto a = __riscv_vle32_v_f32m1(lhs + first, vl); - vfloat32m1_t result; - if (operation == 4) { - result = __riscv_vfneg_v_f32m1(a, vl); - } else if (operation == 5) { - result = __riscv_vfabs_v_f32m1(a, vl); - } else { - auto b = __riscv_vle32_v_f32m1(rhs + first, vl); - switch (operation) { - case 0: - result = __riscv_vfadd_vv_f32m1(a, b, vl); - break; - case 1: - result = __riscv_vfsub_vv_f32m1(a, b, vl); - break; - case 2: - result = __riscv_vfmul_vv_f32m1(a, b, vl); - break; - case 3: - result = __riscv_vfdiv_vv_f32m1(a, b, vl); - break; - case 6: - case 7: { - result = operation == 6 ? __riscv_vfmax_vv_f32m1(a, b, vl) - : __riscv_vfmin_vv_f32m1(a, b, vl); - auto nan = - __riscv_vmor_mm_b32(__riscv_vmfne_vv_f32m1_b32(a, a, vl), - __riscv_vmfne_vv_f32m1_b32(b, b, vl), vl); - auto quiet = __riscv_vfadd_vv_f32m1_m(nan, a, b, vl); - result = __riscv_vmerge_vvm_f32m1(result, quiet, nan, vl); - break; - } - default: - __builtin_trap(); - } - } - __riscv_vse32_v_f32m1(output + first, result, vl); - first += vl; - } - uint32_t status; - asm volatile("csrr %0, fflags" : "=r"(status)::"memory"); - *flags = status; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/pointwise_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/pointwise_dispatch.cpp deleted file mode 100644 index ee31ac579..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/pointwise_dispatch.cpp +++ /dev/null @@ -1,145 +0,0 @@ -#include "images.h" -#include -#include -#include -#include -#include -#include - -namespace { -constexpr unsigned readBank = 4, writeBank = 5; -constexpr unsigned programBank = VIRTUAL_BANK_NUM; -constexpr size_t bankBytes = BANK_LINES * (BANK_WIDTH / 8); -constexpr size_t chunk = bankBytes / sizeof(float); - -bool linear(const DynamicMemRefType &value, - const DynamicMemRefType &shape) { - if (value.rank != shape.rank) - return false; - int64_t stride = 1; - for (int64_t axis = value.rank - 1; axis >= 0; --axis) { - if (value.sizes[axis] != shape.sizes[axis] || - (value.sizes[axis] != 1 && value.strides[axis] != stride)) - return false; - stride *= value.sizes[axis]; - } - return true; -} - -int64_t offset(const DynamicMemRefType &value, - const DynamicMemRefType &shape, int64_t index) { - int64_t result = value.offset; - for (int64_t axis = shape.rank - 1; axis >= 0; --axis) { - int64_t coordinate = index % shape.sizes[axis]; - index /= shape.sizes[axis]; - int64_t inputAxis = axis - (shape.rank - value.rank); - if (inputAxis >= 0 && value.sizes[inputAxis] != 1) - result += coordinate * value.strides[inputAxis]; - } - return result; -} - -void run(UnrankedMemRefType *output, UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs, uint32_t operation) { - DynamicMemRefType out(*output), a(*lhs); - int64_t count = 1; - for (int64_t axis = 0; axis < out.rank; ++axis) { - if (out.sizes[axis] <= 0) - abort(); - count *= out.sizes[axis]; - } - for (auto *input : {lhs, rhs}) { - if (!input) - continue; - DynamicMemRefType value(*input); - if (value.rank > out.rank) - abort(); - for (int64_t axis = 0; axis < value.rank; ++axis) - if (value.sizes[axis] != 1 && - value.sizes[axis] != out.sizes[axis + out.rank - value.rank]) - abort(); - } - alignas(16) float aTile[chunk], bTile[chunk], result[chunk]; - const bool outputLinear = linear(out, out); - uint32_t rounding; - asm volatile("csrr %0, frm" : "=r"(rounding)::"memory"); - for (unsigned bank = 0; bank < 4; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mem_transfer(2, readBank); - bb_mem_transfer(3, readBank); - bb_mem_transfer(0, writeBank); - bb_mem_transfer(1, writeBank); - mvin_kernel(images::pointwise.bytes, images::pointwise.size, programBank); - for (int64_t first = 0; first < count; first += chunk) { - uint32_t length = std::min(chunk, count - first); - auto pack = [&](const DynamicMemRefType &value, float *staging) { - if (linear(value, out)) { - const float *source = value.data + value.offset + first; - if (length % 4 == 0) - return source; - std::memcpy(staging, source, length * sizeof(float)); - } else { - bool scalar = true; - for (int64_t axis = 0; axis < value.rank; ++axis) - scalar &= value.sizes[axis] == 1; - if (scalar) - std::fill_n(staging, length, value.data[value.offset]); - else - for (uint32_t index = 0; index < length; ++index) - staging[index] = value.data[offset(value, out, first + index)]; - } - std::fill(staging + length, staging + (length + 3) / 4 * 4, 0.0f); - return static_cast(staging); - }; - bb_mvin_group((uintptr_t)pack(a, aTile), readBank, 0, (length + 3) / 4, 1); - if (rhs) { - DynamicMemRefType b(*rhs); - bb_mvin_group((uintptr_t)pack(b, bTile), readBank, 1, (length + 3) / 4, - 1); - } - alignas(16) kernel_launch call{ - images::pointwise.entry, - images::pointwise.text_bytes, - 0x40002000, - {uint64_t(3) << 32, 0, uint64_t(1) << 32, length, operation, - (uint64_t(2) << 32) + offsetof(kernel_launch, reserved), - rounding << 5}, - 0}; - bb_mvin_group((uintptr_t)&call, writeBank, 0, sizeof(call) / 16, 1); - run_kernel(readBank, programBank, writeBank, 0); - float *destination = outputLinear && length % 4 == 0 - ? out.data + out.offset + first - : result; - bb_mvout_group((uintptr_t)destination, writeBank, 1, (length + 3) / 4, 1); - bb_mvout_group((uintptr_t)&call, writeBank, 0, sizeof(call) / 16, 1); - asm volatile("csrs fflags, %0" ::"r"(call.reserved) : "memory"); - if (destination == result) { - if (outputLinear) - std::memcpy(out.data + out.offset + first, result, - length * sizeof(float)); - else - for (uint32_t index = 0; index < length; ++index) - out.data[offset(out, out, first + index)] = result[index]; - } - } - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} -} // namespace - -extern "C" void _mlir_ciface_rvv_binary(UnrankedMemRefType *output, - UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs, - uint32_t operation) { - if (operation > 7 || operation == 4 || operation == 5) - abort(); - run(output, lhs, rhs, operation); -} -extern "C" void _mlir_ciface_rvv_unary(UnrankedMemRefType *output, - UnrankedMemRefType *input, - uint32_t operation) { - if (operation < 4 || operation > 5) - abort(); - run(output, input, nullptr, operation); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/quant.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/quant.cpp deleted file mode 100644 index 180c2540a..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/quant.cpp +++ /dev/null @@ -1,131 +0,0 @@ -#include "kernels.h" -#include -#include - -template -static void quant(uint8_t *output, uint8_t *scales, const uint32_t *input, - size_t count, uint32_t *status) { - *status = 0; - for (size_t block = 0; block < count / 32; ++block) { - uint32_t maximum = 0; - for (size_t begin = 0; begin < 32;) { - size_t vl = __riscv_vsetvl_e32m1(32 - begin); - auto bits = __riscv_vle32_v_u32m1(input + block * 32 + begin, vl); - auto magnitude = __riscv_vand_vx_u32m1(bits, 0x7fffffff, vl); - auto seed = __riscv_vmv_v_x_u32m1(maximum, vl); - maximum = __riscv_vmv_x_s_u32m1_u32( - __riscv_vredmaxu_vs_u32m1_u32m1(magnitude, seed, vl)); - begin += vl; - } - if (maximum >= 0x7f800000) { - *status = 1; - return; - } - int blockExponent = maximum == 0 ? 0 : int(maximum >> 23) - 135; - if constexpr (bounded) { - if ((maximum & 0x7fffff) > 0x600000) - ++blockExponent; - if (blockExponent > 119) - blockExponent = 119; - } - if (blockExponent < -127) - blockExponent = -127; - scales[block] = uint8_t(blockExponent + 127); - for (size_t begin = 0; begin < 32;) { - size_t vl = __riscv_vsetvl_e32m1(32 - begin); - auto bits = __riscv_vle32_v_u32m1(input + block * 32 + begin, vl); - auto sign = - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 24, vl), 128, vl); - auto fraction = __riscv_vand_vx_u32m1(bits, 0x7fffff, vl); - auto exponent = __riscv_vreinterpret_v_u32m1_i32m1( - __riscv_vand_vx_u32m1(__riscv_vsrl_vx_u32m1(bits, 23, vl), 255, vl)); - auto subnormal = - __riscv_vmand_mm_b32(__riscv_vmseq_vx_i32m1_b32(exponent, 0, vl), - __riscv_vmsne_vx_u32m1_b32(fraction, 0, vl), vl); - if (__riscv_vcpop_m_b32(subnormal, vl)) { - auto normalized = fraction; - auto shift = __riscv_vmv_v_x_u32m1(0, vl); - for (unsigned step = 16; step; step >>= 1) { - auto selected = - __riscv_vmsltu_vx_u32m1_b32(normalized, 1U << (24 - step), vl); - auto moved = __riscv_vsll_vx_u32m1(normalized, step, vl); - normalized = - __riscv_vmerge_vvm_u32m1(normalized, moved, selected, vl); - auto increment = __riscv_vadd_vx_u32m1(shift, step, vl); - shift = __riscv_vmerge_vvm_u32m1(shift, increment, selected, vl); - } - fraction = __riscv_vmerge_vvm_u32m1( - fraction, __riscv_vand_vx_u32m1(normalized, 0x7fffff, vl), - subnormal, vl); - auto adjusted = __riscv_vrsub_vx_i32m1( - __riscv_vreinterpret_v_u32m1_i32m1(shift), 1, vl); - exponent = __riscv_vmerge_vvm_i32m1(exponent, adjusted, subnormal, vl); - } - exponent = __riscv_vsub_vx_i32m1(exponent, blockExponent + 120, vl); - auto underflow = __riscv_vmsle_vx_i32m1_b32(exponent, 0, vl); - vuint32m1_t code; - if (__riscv_vcpop_m_b32(underflow, vl) == 0) { - auto parity = __riscv_vand_vx_u32m1( - __riscv_vsrl_vx_u32m1(fraction, 20, vl), 1, vl); - auto rounded = __riscv_vsrl_vx_u32m1( - __riscv_vadd_vv_u32m1(__riscv_vadd_vx_u32m1(fraction, 0x7ffff, vl), - parity, vl), - 20, vl); - code = - __riscv_vadd_vv_u32m1(rounded, - __riscv_vreinterpret_v_i32m1_u32m1( - __riscv_vsll_vx_i32m1(exponent, 3, vl)), - vl); - } else { - auto shift = __riscv_vmv_v_x_u32m1(20, vl); - auto underShift = __riscv_vreinterpret_v_i32m1_u32m1( - __riscv_vrsub_vx_i32m1(exponent, 21, vl)); - shift = __riscv_vmerge_vvm_u32m1(shift, underShift, underflow, vl); - auto tooSmall = __riscv_vmsgtu_vx_u32m1_b32(shift, 31, vl); - shift = __riscv_vminu_vx_u32m1(shift, 31, vl); - auto value = __riscv_vmerge_vvm_u32m1( - fraction, __riscv_vor_vx_u32m1(fraction, 0x800000, vl), underflow, - vl); - auto half = - __riscv_vsll_vv_u32m1(__riscv_vmv_v_x_u32m1(1, vl), - __riscv_vsub_vx_u32m1(shift, 1, vl), vl); - auto parity = __riscv_vand_vx_u32m1( - __riscv_vsrl_vv_u32m1(value, shift, vl), 1, vl); - auto rounded = __riscv_vsrl_vv_u32m1( - __riscv_vadd_vv_u32m1( - __riscv_vsub_vx_u32m1(__riscv_vadd_vv_u32m1(value, half, vl), 1, - vl), - parity, vl), - shift, vl); - rounded = __riscv_vmerge_vxm_u32m1(rounded, 0, tooSmall, vl); - auto normal = - __riscv_vadd_vv_u32m1(rounded, - __riscv_vreinterpret_v_i32m1_u32m1( - __riscv_vsll_vx_i32m1(exponent, 3, vl)), - vl); - code = __riscv_vmerge_vvm_u32m1(normal, rounded, underflow, vl); - } - code = __riscv_vminu_vx_u32m1(code, 126, vl); - auto zero = __riscv_vmseq_vx_u32m1_b32( - __riscv_vand_vx_u32m1(bits, 0x7fffffff, vl), 0, vl); - code = __riscv_vmerge_vxm_u32m1(code, 0, zero, vl); - code = __riscv_vor_vv_u32m1(code, sign, vl); - auto narrow = __riscv_vnsrl_wx_u16mf2(code, 0, vl); - __riscv_vse8_v_u8mf4(output + block * 32 + begin, - __riscv_vnsrl_wx_u8mf4(narrow, 0, vl), vl); - begin += vl; - } - } -} - -extern "C" void rvv_quant(uint8_t *output, uint8_t *scales, - const uint32_t *input, size_t count, - uint32_t *status) { - quant(output, scales, input, count, status); -} - -extern "C" void rvv_cache_quant(uint8_t *output, uint8_t *scales, - const uint32_t *input, size_t count, - uint32_t *status) { - quant(output, scales, input, count, status); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/quant_dispatch.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/quant_dispatch.cpp deleted file mode 100644 index 682fe9238..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/quant_dispatch.cpp +++ /dev/null @@ -1,41 +0,0 @@ -#include "../mxfp8/quant.h" -#include "images.h" -#include -#include -#include -#include - -extern "C" void mxfp8_quant(const uint32_t *input, uint8_t *output, - uint32_t count) { - constexpr unsigned readBank = 4, writeBank = 5; - constexpr unsigned programBank = VIRTUAL_BANK_NUM; - for (unsigned bank = 0; bank < 3; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mem_transfer(2, readBank); - bb_mem_transfer(0, writeBank); - bb_mem_transfer(1, writeBank); - mvin_kernel(images::quant.bytes, images::quant.size, programBank); - bb_mvin_group((uintptr_t)input, readBank, 0, count / 4, 1); - alignas(16) uint8_t descriptor[sizeof(kernel_launch)]{}; - kernel_launch call{images::quant.entry, - images::quant.text_bytes, - 0x40002000, - {uint64_t(2) << 32, (uint64_t(2) << 32) + count, 0, count, - (uint64_t(1) << 32) + offsetof(kernel_launch, reserved)}, - 0}; - std::memcpy(descriptor, &call, sizeof(call)); - bb_mvin_group((uintptr_t)descriptor, writeBank, 0, sizeof(descriptor) / 16, - 1); - run_kernel(readBank, programBank, writeBank, 0); - bb_mvout_group((uintptr_t)descriptor, writeBank, 0, sizeof(descriptor) / 16, - 1); - bb_mvout_group((uintptr_t)output, writeBank, 1, - (count + count / 32 + 15) / 16, 1); - if (descriptor[offsetof(kernel_launch, reserved)]) { - fputs("mxfp8: non-finite activation\n", stderr); - abort(); - } - bb_mem_release(readBank); - bb_mem_release(writeBank); - release_kernel(programBank); -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/rope.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/rope.cpp deleted file mode 100644 index d3e58ee3a..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/rope.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "kernels.h" -#include "math/math.h" - -extern "C" void rvv_rope(float *output, const float *input, - const float *frequencies, const int32_t *positions, - size_t rows, size_t headDimAndHeads) { - const size_t headDim = headDimAndHeads & 65535; - const size_t heads = (headDimAndHeads >> 16) & 65535; - const uint32_t scaleBits = headDimAndHeads >> 32; - const float scale = __riscv_vfmv_f_s_f32m1_f32( - __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(scaleBits, 1))); - const size_t half = headDim / 2; - for (size_t row = 0; row < rows; ++row) { - auto integer = __riscv_vle32_v_i32m1(positions + row, 1); - auto converted = __riscv_vfcvt_f_x_v_f32m1(integer, 1); - const float position = __riscv_vfmv_f_s_f32m1_f32(converted); - for (size_t offset = 0; offset < half;) { - const size_t vl = __riscv_vsetvl_e32m1(half - offset); - auto frequency = __riscv_vle32_v_f32m1(frequencies + offset, vl); - auto angle = __riscv_vfmul_vf_f32m1(frequency, position, vl); - angle = __riscv_vfadd_vf_f32m1(angle, 0, vl); - auto cosine = rvv_cos(angle, vl); - cosine = __riscv_vfmul_vf_f32m1(cosine, scale, vl); - for (size_t head = 0; head < heads; ++head) { - const float *source = input + (head * rows + row) * headDim; - float *destination = output + (head * rows + row) * headDim; - auto lower = __riscv_vle32_v_f32m1(source + offset, vl); - auto upper = __riscv_vle32_v_f32m1(source + half + offset, vl); - __riscv_vse32_v_f32m1(destination + offset, - __riscv_vfmul_vv_f32m1(lower, cosine, vl), vl); - __riscv_vse32_v_f32m1(destination + half + offset, - __riscv_vfmul_vv_f32m1(upper, cosine, vl), vl); - } - frequency = __riscv_vle32_v_f32m1(frequencies + offset, vl); - angle = __riscv_vfmul_vf_f32m1(frequency, position, vl); - angle = __riscv_vfadd_vf_f32m1(angle, 0, vl); - auto sine = rvv_sin(angle, vl); - sine = __riscv_vfmul_vf_f32m1(sine, scale, vl); - for (size_t head = 0; head < heads; ++head) { - const float *source = input + (head * rows + row) * headDim; - float *destination = output + (head * rows + row) * headDim; - auto lower = __riscv_vle32_v_f32m1(source + offset, vl); - auto upper = __riscv_vle32_v_f32m1(source + half + offset, vl); - auto rotatedLower = - __riscv_vfmul_vv_f32m1(__riscv_vfneg_v_f32m1(upper, vl), sine, vl); - auto rotatedUpper = __riscv_vfmul_vv_f32m1(lower, sine, vl); - auto lowerProduct = __riscv_vle32_v_f32m1(destination + offset, vl); - auto upperProduct = - __riscv_vle32_v_f32m1(destination + half + offset, vl); - __riscv_vse32_v_f32m1( - destination + offset, - __riscv_vfadd_vv_f32m1(lowerProduct, rotatedLower, vl), vl); - __riscv_vse32_v_f32m1( - destination + half + offset, - __riscv_vfadd_vv_f32m1(upperProduct, rotatedUpper, vl), vl); - } - offset += vl; - } - } -} - -extern "C" void rvv_rope_launch(float *output, const float *input, - const float *frequencies, - const int32_t *positions, size_t rows, - size_t headDimAndHeads, uint32_t *flags, - uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - rvv_rope(output, input, frequencies, positions, rows, headDimAndHeads); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/silu.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/silu.cpp deleted file mode 100644 index 6c5fe1795..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/silu.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "kernels.h" -#include "math/math.h" - -extern "C" void rvv_silu(float *output, const float *input, size_t count) { - for (size_t offset = 0; offset < count;) { - const size_t vl = __riscv_vsetvl_e32m1(count - offset); - auto value = __riscv_vle32_v_f32m1(input + offset, vl); - auto denominator = __riscv_vfadd_vf_f32m1( - rvv_exp(__riscv_vfneg_v_f32m1(value, vl), vl), 1, vl); - auto sigmoid = __riscv_vfrdiv_vf_f32m1(denominator, 1, vl); - auto result = __riscv_vfmul_vv_f32m1(value, sigmoid, vl); - __riscv_vse32_v_f32m1(output + offset, result, vl); - offset += vl; - } -} - -extern "C" void rvv_swiglu(float *output, const float *gate, const float *up, - size_t count) { - for (size_t offset = 0; offset < count;) { - const size_t vl = __riscv_vsetvl_e32m1(count - offset); - auto value = __riscv_vle32_v_f32m1(gate + offset, vl); - auto other = __riscv_vle32_v_f32m1(up + offset, vl); - auto denominator = __riscv_vfadd_vf_f32m1( - rvv_exp(__riscv_vfneg_v_f32m1(value, vl), vl), 1, vl); - auto sigmoid = __riscv_vfrdiv_vf_f32m1(denominator, 1, vl); - auto result = __riscv_vfmul_vv_f32m1( - __riscv_vfmul_vv_f32m1(value, sigmoid, vl), other, vl); - __riscv_vse32_v_f32m1(output + offset, result, vl); - offset += vl; - } -} - -extern "C" void rvv_silu_launch(float *output, const float *input, size_t count, - uint32_t *flags, uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - rvv_silu(output, input, count); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} - -extern "C" void rvv_swiglu_launch(float *output, const float *gate, - const float *up, size_t count, - uint32_t *flags, uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - rvv_swiglu(output, gate, up, count); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/snake.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/snake.cpp deleted file mode 100644 index 49d3b193a..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/snake.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "kernels.h" -#include "math/math.h" - -extern "C" void rvv_snake(float *output, const float *input, size_t channels, - size_t length, const float *log_alpha, - const float *log_beta) { - for (size_t channel = 0; channel < channels; ++channel) { - auto alpha = rvv_exp(__riscv_vfmv_v_f_f32m1(log_alpha[channel], 1), 1); - auto beta = __riscv_vfadd_vf_f32m1( - rvv_exp(__riscv_vfmv_v_f_f32m1(log_beta[channel], 1), 1), 1e-9f, 1); - auto reciprocal = __riscv_vfrdiv_vf_f32m1(beta, 1, 1); - const float scale = __riscv_vfmv_f_s_f32m1_f32(alpha); - const float inverse = __riscv_vfmv_f_s_f32m1_f32(reciprocal); - for (size_t offset = 0; offset < length;) { - const size_t vl = __riscv_vsetvl_e32m1(length - offset); - const size_t index = channel * length + offset; - auto value = __riscv_vle32_v_f32m1(input + index, vl); - auto angle = __riscv_vfmul_vf_f32m1(value, scale, vl); - auto sine = rvv_sin(angle, vl); - auto square = __riscv_vfmul_vv_f32m1(sine, sine, vl); - auto scaled = __riscv_vfmul_vf_f32m1(square, inverse, vl); - auto result = __riscv_vfadd_vv_f32m1(value, scaled, vl); - __riscv_vse32_v_f32m1(output + index, result, vl); - offset += vl; - } - } -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/softmax.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/softmax.cpp deleted file mode 100644 index f859cf04e..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/softmax.cpp +++ /dev/null @@ -1,183 +0,0 @@ -#include "kernels.h" -#include "math/math.h" - -extern "C" void rvv_softmax(float *output, const float *input, size_t rows, - size_t width) { - for (size_t row = 0; row < rows; ++row) { - const float *source = input + row * width; - float *destination = output + row * width; - auto maximum = __riscv_vfmv_v_f_f32m1(-0x1.fffffep127f, 1); - bool hasNaN = false; - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(source + offset, vl); - auto magnitude = __riscv_vand_vx_u32m1(__riscv_vreinterpret_u32m1(values), - 0x7fffffff, vl); - hasNaN |= - __riscv_vcpop_m_b32( - __riscv_vmsgtu_vx_u32m1_b32(magnitude, 0x7f800000, vl), vl) != 0; - maximum = __riscv_vfredmax_vs_f32m1_f32m1(values, maximum, vl); - offset += vl; - } - if (hasNaN) - maximum = - __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(0x7fc00000, 1)); - const float peak = __riscv_vfmv_f_s_f32m1_f32(maximum); - auto sum = __riscv_vfmv_v_f_f32m1(0, 1); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(source + offset, vl); - auto centered = __riscv_vfsub_vf_f32m1(values, peak, vl); - auto exponential = rvv_exp(centered, vl); - __riscv_vse32_v_f32m1(destination + offset, exponential, vl); - sum = __riscv_vfredosum_vs_f32m1_f32m1(exponential, sum, vl); - offset += vl; - } - auto reciprocal = __riscv_vfrdiv_vf_f32m1(sum, 1, 1); - const float scale = __riscv_vfmv_f_s_f32m1_f32(reciprocal); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(destination + offset, vl); - auto result = __riscv_vfmul_vf_f32m1(values, scale, vl); - __riscv_vse32_v_f32m1(destination + offset, result, vl); - offset += vl; - } - } -} - -extern "C" void rvv_softmax_launch(float *output, const float *input, - size_t rows, size_t width, uint32_t *flags, - uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - rvv_softmax(output, input, rows, width); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} - -extern "C" void rvv_attention_softmax_launch(float *output, const float *input, - const uint8_t *mask, size_t rows, - size_t width, - const uint32_t *parameters, - uint32_t *flags, - uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - union { - uint32_t bits; - float value; - } scale{parameters[0]}, masked{parameters[1]}; - for (size_t row = 0; row < rows; ++row) - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(input + row * width + offset, vl); - auto scaled = __riscv_vfmul_vf_f32m1(values, scale.value, vl); - auto bytes = __riscv_vle8_v_u8mf4( - mask + ((row + parameters[3]) % parameters[2]) * width + offset, vl); - auto selected = __riscv_vmsne_vx_u8mf4_b32(bytes, 0, vl); - auto result = - __riscv_vfmerge_vfm_f32m1(scaled, masked.value, selected, vl); - __riscv_vse32_v_f32m1(output + row * width + offset, result, vl); - offset += vl; - } - rvv_softmax(output, output, rows, width); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} - -// Softmax sums are positive and finite for finite scores. Preserve IEEE special -// values when the input row contains infinities or NaNs. -static vfloat32m1_t logSum(vfloat32m1_t value) { - constexpr size_t vl = 1; - auto bits = __riscv_vreinterpret_u32m1(value); - auto fraction = __riscv_vand_vx_u32m1(bits, 0x007fffff, vl); - auto mantissa = __riscv_vreinterpret_f32m1( - __riscv_vor_vx_u32m1(fraction, 0x3f800000, vl)); - auto exponent = __riscv_vsub_vx_i32m1( - __riscv_vreinterpret_i32m1(__riscv_vsrl_vx_u32m1(bits, 23, vl)), 127, vl); - auto upper = __riscv_vmfge_vf_f32m1_b32(mantissa, 0x1.6a09e6p0f, vl); - mantissa = __riscv_vmerge_vvm_f32m1( - mantissa, __riscv_vfmul_vf_f32m1(mantissa, 0.5f, vl), upper, vl); - exponent = __riscv_vmerge_vvm_i32m1( - exponent, __riscv_vadd_vx_i32m1(exponent, 1, vl), upper, vl); - auto numerator = __riscv_vfsub_vf_f32m1(mantissa, 1, vl); - auto denominator = __riscv_vfadd_vf_f32m1(mantissa, 1, vl); - auto t = __riscv_vfdiv_vv_f32m1(numerator, denominator, vl); - auto square = __riscv_vfmul_vv_f32m1(t, t, vl); - auto term = t, series = t; - for (unsigned odd = 3; odd <= 15; odd += 2) { - term = __riscv_vfmul_vv_f32m1(term, square, vl); - series = __riscv_vfadd_vv_f32m1( - series, __riscv_vfdiv_vf_f32m1(term, float(odd), vl), vl); - } - auto e = __riscv_vfcvt_f_x_v_f32m1(exponent, vl); - auto result = - __riscv_vfadd_vv_f32m1(__riscv_vfmul_vf_f32m1(e, 0x1.62e4p-1f, vl), - __riscv_vfmul_vf_f32m1(series, 2, vl), vl); - result = __riscv_vfadd_vv_f32m1( - result, __riscv_vfmul_vf_f32m1(e, 0x1.7f7d1cp-20f, vl), vl); - auto zero = __riscv_vmseq_vx_u32m1_b32(bits, 0, vl); - auto infinity = __riscv_vmseq_vx_u32m1_b32(bits, 0x7f800000, vl); - auto nan = __riscv_vmsgtu_vx_u32m1_b32(bits, 0x7f800000, vl); - auto negativeInfinity = - __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(0xff800000, vl)); - auto quietNaN = - __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(0x7fc00000, vl)); - result = __riscv_vmerge_vvm_f32m1(result, negativeInfinity, zero, vl); - result = __riscv_vmerge_vvm_f32m1(result, value, infinity, vl); - return __riscv_vmerge_vvm_f32m1(result, quietNaN, nan, vl); -} - -extern "C" void rvv_logsumexp_softmax(float *output, const float *input, - size_t rows, size_t width) { - for (size_t row = 0; row < rows; ++row) { - const float *source = input + row * width; - float *destination = output + row * width; - auto maximum = __riscv_vfmv_v_f_f32m1(-0x1.fffffep127f, 1); - bool hasNaN = false; - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(source + offset, vl); - auto magnitude = __riscv_vand_vx_u32m1(__riscv_vreinterpret_u32m1(values), - 0x7fffffff, vl); - hasNaN |= - __riscv_vcpop_m_b32( - __riscv_vmsgtu_vx_u32m1_b32(magnitude, 0x7f800000, vl), vl) != 0; - maximum = __riscv_vfredmax_vs_f32m1_f32m1(values, maximum, vl); - offset += vl; - } - if (hasNaN) - maximum = - __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(0x7fc00000, 1)); - const float peak = __riscv_vfmv_f_s_f32m1_f32(maximum); - auto sum = __riscv_vfmv_v_f_f32m1(0, 1); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(source + offset, vl); - auto exponential = rvv_exp(__riscv_vfsub_vf_f32m1(values, peak, vl), vl); - sum = __riscv_vfredosum_vs_f32m1_f32m1(exponential, sum, vl); - offset += vl; - } - auto logsum = __riscv_vfadd_vv_f32m1(maximum, logSum(sum), 1); - const float normalization = __riscv_vfmv_f_s_f32m1_f32(logsum); - for (size_t offset = 0; offset < width;) { - const size_t vl = __riscv_vsetvl_e32m1(width - offset); - auto values = __riscv_vle32_v_f32m1(source + offset, vl); - auto result = - rvv_exp(__riscv_vfsub_vf_f32m1(values, normalization, vl), vl); - __riscv_vse32_v_f32m1(destination + offset, result, vl); - offset += vl; - } - } -} - -extern "C" void rvv_logsumexp_softmax_launch(float *output, const float *input, - size_t rows, size_t width, - uint32_t *flags, - uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - rvv_logsumexp_softmax(output, input, rows, width); - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/tanh.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/tanh.cpp deleted file mode 100644 index e46e5d62d..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/tanh.cpp +++ /dev/null @@ -1,54 +0,0 @@ -#include "math/math.h" -#include - -extern "C" void rvv_tanh_launch(float *output, const float *input, size_t count, - uint32_t *flags, uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - for (size_t offset = 0; offset < count;) { - const size_t vl = __riscv_vsetvl_e32m1(count - offset); - auto values = __riscv_vle32_v_f32m1(input + offset, vl); - auto bits = __riscv_vreinterpret_u32m1(values); - auto magnitude = __riscv_vand_vx_u32m1(bits, 0x7fffffff, vl); - auto finite = __riscv_vmsleu_vx_u32m1_b32(magnitude, 0x7f7fffff, vl); - auto absolute = - __riscv_vmerge_vvm_f32m1(__riscv_vfmv_v_f_f32m1(0, vl), - __riscv_vfabs_v_f32m1(values, vl), finite, vl); - auto saturated = __riscv_vfmin_vf_f32m1(absolute, 10, vl); - auto e = rvv_exp(__riscv_vfmul_vf_f32m1(saturated, -2, vl), vl); - auto numerator = __riscv_vfrsub_vf_f32m1(e, 1, vl); - auto denominator = __riscv_vfadd_vf_f32m1(e, 1, vl); - auto result = __riscv_vfdiv_vv_f32m1(numerator, denominator, vl); - // The odd series avoids exponential cancellation near the origin. - auto bounded = __riscv_vfmin_vf_f32m1(absolute, 0.125f, vl); - auto square = __riscv_vfmul_vv_f32m1(bounded, bounded, vl); - auto polynomial = __riscv_vfadd_vf_f32m1( - __riscv_vfmul_vf_f32m1(square, -17.0f / 315.0f, vl), 2.0f / 15.0f, vl); - polynomial = __riscv_vfadd_vf_f32m1( - __riscv_vfmul_vv_f32m1(polynomial, square, vl), -1.0f / 3.0f, vl); - polynomial = __riscv_vfmul_vv_f32m1(polynomial, square, vl); - auto series = __riscv_vfadd_vv_f32m1( - bounded, __riscv_vfmul_vv_f32m1(polynomial, bounded, vl), vl); - auto small = __riscv_vmfle_vf_f32m1_b32(absolute, 0.125f, vl); - result = __riscv_vmerge_vvm_f32m1(result, series, small, vl); - auto infinity = __riscv_vmseq_vx_u32m1_b32(magnitude, 0x7f800000, vl); - auto nan = __riscv_vmsgtu_vx_u32m1_b32(magnitude, 0x7f800000, vl); - auto quiet = - __riscv_vreinterpret_f32m1(__riscv_vor_vx_u32m1(bits, 0x00400000, vl)); - result = __riscv_vmerge_vvm_f32m1(result, __riscv_vfmv_v_f_f32m1(1, vl), - infinity, vl); - result = __riscv_vmerge_vvm_f32m1(result, quiet, nan, vl); - auto signaling = __riscv_vmand_mm_b32( - nan, - __riscv_vmseq_vx_u32m1_b32(__riscv_vand_vx_u32m1(bits, 0x00400000, vl), - 0, vl), - vl); - if (__riscv_vcpop_m_b32(signaling, vl)) - asm volatile("csrs fflags, %0" ::"r"(uint32_t(16)) : "memory"); - result = __riscv_vfsgnj_vv_f32m1(result, values, vl); - __riscv_vse32_v_f32m1(output + offset, result, vl); - offset += vl; - } - uint32_t result; - asm volatile("csrr %0, fflags" : "=r"(result)::"memory"); - *flags = result; -} diff --git a/bb-tests/workloads/lib/bbsw/kernels/rvv/trig.cpp b/bb-tests/workloads/lib/bbsw/kernels/rvv/trig.cpp deleted file mode 100644 index d4dde4e03..000000000 --- a/bb-tests/workloads/lib/bbsw/kernels/rvv/trig.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "math/math.h" -#include - -template -static void trig(float *output, const float *input, size_t count, - uint32_t *flags, uint32_t rounding) { - asm volatile("csrw fcsr, %0" ::"r"(rounding) : "memory"); - for (size_t offset = 0; offset < count;) { - size_t vl = __riscv_vsetvl_e32m1(count - offset); - auto values = __riscv_vle32_v_f32m1(input + offset, vl); - auto result = sine ? rvv_sin(values, vl) : rvv_cos(values, vl); - __riscv_vse32_v_f32m1(output + offset, result, vl); - offset += vl; - } - uint32_t status; - asm volatile("csrr %0, fflags" : "=r"(status)::"memory"); - *flags = status; -} -extern "C" void rvv_sin_launch(float *output, const float *input, size_t count, - uint32_t *flags, uint32_t rounding) { - trig(output, input, count, flags, rounding); -} -extern "C" void rvv_cos_launch(float *output, const float *input, size_t count, - uint32_t *flags, uint32_t rounding) { - trig(output, input, count, flags, rounding); -} diff --git a/bb-tests/workloads/lib/bbsw/linux/multicore/multicore.h b/bb-tests/workloads/lib/bbsw/linux/multicore/multicore.h deleted file mode 100644 index 1ece5e551..000000000 --- a/bb-tests/workloads/lib/bbsw/linux/multicore/multicore.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef BBSW_LINUX_MULTICORE_H -#define BBSW_LINUX_MULTICORE_H - -#include -#include -#include -#include - -static inline uint32_t bb_get_hart_id(void) { - unsigned cpu; - if (syscall(SYS_getcpu, &cpu, 0, 0) != 0) - __builtin_trap(); - return cpu; -} - -static inline core_id_t bb_get_core_id(void) { - return bb_topology_core_id(bb_get_hart_id()); -} - -#endif diff --git a/bb-tests/workloads/lib/bbsw/linux/test.c b/bb-tests/workloads/lib/bbsw/linux/test.c deleted file mode 100644 index 3272fa403..000000000 --- a/bb-tests/workloads/lib/bbsw/linux/test.c +++ /dev/null @@ -1,6 +0,0 @@ -#include -#include - -void bb_test_pass(void) {} - -void bb_test_fail(void) { exit(1); } diff --git a/bb-tests/workloads/lib/bbsw/test.h b/bb-tests/workloads/lib/bbsw/test.h deleted file mode 100644 index de53b0fa7..000000000 --- a/bb-tests/workloads/lib/bbsw/test.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef BBSW_TEST_H -#define BBSW_TEST_H - -#ifdef __cplusplus -extern "C" { -#endif - -void bb_test_pass(void); -void bb_test_fail(void) __attribute__((noreturn)); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/bb-tests/workloads/lib/buckyball.h b/bb-tests/workloads/lib/buckyball.h index e3801c438..de78240e8 100644 --- a/bb-tests/workloads/lib/buckyball.h +++ b/bb-tests/workloads/lib/buckyball.h @@ -3,25 +3,11 @@ #include -// String macros (from xcustom.h) -// #define STR1(x) #x -// #ifndef STR -// #define STR(x) STR1(x) -// #endif - -#define CAT_(A, B) A##B -#define CAT(A, B) CAT_(A, B) - // Data type for matrix elements typedef int8_t elem_t; typedef int32_t result_t; -#define MULTICORE_INIT(hart_id) \ - __attribute__((constructor)) static void _multicore_init() { \ - multicore(hart_id); \ - } - -static inline void multicore(int target_hart_id) { +static inline void multithread(int target_hart_id) { int hart_id; asm volatile("csrr %0, mhartid" : "=r"(hart_id)); diff --git a/bb-tests/workloads/lib/kernel b/bb-tests/workloads/lib/kernel deleted file mode 160000 index 9e16330e6..000000000 --- a/bb-tests/workloads/lib/kernel +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9e16330e6df5154e20b17d7b86891400407fcedb diff --git a/bb-tests/workloads/scripts/build.py b/bb-tests/workloads/scripts/build.py index fff1a58ec..a37499e03 100644 --- a/bb-tests/workloads/scripts/build.py +++ b/bb-tests/workloads/scripts/build.py @@ -1,31 +1,22 @@ from __future__ import annotations -import argparse -import os import re import shlex import shutil import subprocess +import sys from pathlib import Path -def _repo(raw: str | Path) -> Path: - root = Path(raw).resolve() - if not root.is_dir(): - raise ValueError(f"repo does not exist: {root}") - return root - - def _run( cmd: list[str], *, cwd: Path, prefix: str, - env: dict | None = None, logger: object | None = None, task_scope: str | None = None, ) -> None: if logger is None: - result = subprocess.run(cmd, cwd=cwd, env=env) + result = subprocess.run(cmd, cwd=cwd) else: from utils.stream_run import stream_run_logger @@ -36,156 +27,70 @@ def _run( stdout_prefix=prefix, stderr_prefix=prefix, task_scope=task_scope, - env=env, ) if result.returncode != 0: raise RuntimeError(f"command failed ({result.returncode}): {' '.join(cmd)}") -def _cmake_defs(repo: Path, chip: str) -> dict[str, str]: - path = ( - repo - / "examples" - / "chips" - / chip - / "configs" - / "generated" - / "workload" - / "cmake.defs" - ) - if not path.is_file(): - raise RuntimeError(f"missing {path}; run bbdev config --install") - values: dict[str, str] = {} - for line in path.read_text(encoding="utf-8").splitlines(): - key, sep, value = line.partition("=") - if not sep: - raise RuntimeError(f"bad cmake.defs line in {path}: {line!r}") - if key in values: - raise RuntimeError(f"duplicate cmake.defs key {key} in {path}") - values[key] = value - required = ("BUCKYBALL_WORKLOAD_CHIP", "BUCKYBALL_CHIP_PB") - missing = [k for k in required if k not in values] - if missing: - raise RuntimeError(f"{path} missing {missing}") - return values - - -def _require_riscv() -> Path: - raw = os.environ.get("RISCV", "") - if not raw: - raise RuntimeError("RISCV is unset; enter nix develop") - root = Path(raw) - if not root.is_dir(): - raise RuntimeError(f"RISCV is not a directory: {root}") - return root - - -def _workload_src(repo: Path) -> Path: - src = repo / "bb-tests" / "workloads" - cmake = src / "CMakeLists.txt" - if not cmake.is_file(): - raise RuntimeError(f"missing {cmake}") - return src - - -def _workload_build_dir(repo: Path, instance: str) -> Path: - return repo / "bb-tests" / "workloads" / "build" / instance - - def build_workload( repo: str | Path, - chip: str, + design: str, *, - instance: str | None = None, - chip_pb: str | Path | None = None, + selection: str, ctest: bool = False, mlirtest: bool = False, soctest: bool = False, - stable: bool = False, logger: object | None = None, task_scope: str | None = None, ) -> None: - if not re.fullmatch("[A-Za-z0-9_-]+", chip): - raise ValueError(f"invalid chip: {chip}") - build_instance = instance or chip - if not re.fullmatch("[A-Za-z0-9_-]+", build_instance): - raise ValueError(f"invalid workload build instance: {build_instance}") - if (instance is None) != (chip_pb is None): - raise ValueError("workload variant requires both instance and chip_pb") + if not re.fullmatch("[A-Za-z0-9_-]+", design): + raise ValueError(f"invalid design: {design}") if sum((ctest, mlirtest, soctest)) > 1: raise ValueError("--ctest, --mlirtest and --soctest are mutually exclusive") - root = _repo(repo) - defs = _cmake_defs(root, chip) - if chip_pb is not None: - selected_pb = Path(chip_pb).resolve() - if not selected_pb.is_file(): - raise RuntimeError(f"missing {selected_pb}") - defs["BUCKYBALL_WORKLOAD_CHIP"] = build_instance - defs["BUCKYBALL_WORKLOAD_SOURCE_CHIP"] = chip - defs["BUCKYBALL_CHIP_PB"] = str(selected_pb) - compiler_build = ( - root - / "stack" - / "compiler" - / "thirdparty" - / "buddy-mlir" - / "build" - / build_instance + root = Path(repo).resolve() + design_pb = root / "examples/designs" / design / "output/generated/design.pb" + sys.path.insert(0, str(root)) + from stack.scripts.design import load + + target = load(design_pb).simulations[selection] + if not target.directory: + raise ValueError(f"{design}: unregistered simulation entry {selection}") + defs = { + "BUCKYBALL_WORKLOAD_DESIGN": design, + "BUCKYBALL_DESIGN_PB": str(design_pb), + } + defs["BUCKYBALL_WORKLOAD_SCOPE"] = ( + "ctest" + if ctest + else "mlirtest" if mlirtest else "soctest" if soctest else "all" ) - riscv = _require_riscv() - project_python = riscv / "bin" / "python3" - python = ( - str(project_python) if project_python.is_file() else shutil.which("python3") + defs["BUCKYBALL_SIM_TARGET"] = selection + defs["BUCKYBALL_DESIGN_WORKLOAD_DIR"] = str(root / target.workloads) + defs["BUCKYBALL_WORKLOAD_OUTPUT"] = str( + root / "examples/designs" / design / "output/sims" / selection / "workloads" + ) + tools = {} + for name in ( + "cmake", + "riscv64-unknown-elf-gcc", + "riscv64-unknown-elf-g++", + "riscv64-unknown-elf-objcopy", + "riscv64-unknown-linux-gnu-gcc", + "riscv64-unknown-linux-gnu-g++", + ): + executable = shutil.which(name) + if executable is None: + raise FileNotFoundError(f"{name} is required; enter nix develop") + tools[name] = str(Path(executable).absolute()) + source = root / "bb-tests" / "workloads" + build = ( + root + / "examples/designs" + / design + / "output/sims" + / selection + / "workloads-build" ) - if not python: - raise RuntimeError("python3 not in PATH; enter nix develop") - linux_cc = riscv / "bin" / "riscv64-unknown-linux-gnu-gcc" - linux_cxx = riscv / "bin" / "riscv64-unknown-linux-gnu-g++" - if not linux_cc.is_file() or not linux_cxx.is_file(): - raise RuntimeError(f"missing RISC-V linux toolchain under {riscv / 'bin'}") - source_root = _workload_src(root) - build_root = _workload_build_dir(root, build_instance) - env = os.environ.copy() - env["PATH"] = f"{riscv / 'bin'}:{env.get('PATH', '')}" - env["RISCV"] = str(riscv) - env["CC"] = str(linux_cc) - env["CXX"] = str(linux_cxx) - if not (ctest or soctest): - compiler_config = {} - for line in (compiler_build / "CMakeCache.txt").read_text().splitlines(): - key, sep, value = line.partition("=") - name = key.split(":")[0] - if sep and name in ("BUDDY_EXTERNAL_DIALECTS_DIR", "BUCKYBALL_CHIP_PB"): - if name in compiler_config: - raise RuntimeError(f"duplicate {name} in compiler CMakeCache.txt") - compiler_config[name] = value - if ( - Path(compiler_config["BUDDY_EXTERNAL_DIALECTS_DIR"]).resolve() - != root / "stack" / "compiler" - ): - raise RuntimeError("compiler does not use this repository's stack/compiler") - if ( - Path(compiler_config["BUCKYBALL_CHIP_PB"]).resolve() - != Path(defs["BUCKYBALL_CHIP_PB"]).resolve() - ): - raise RuntimeError("compiler Chip.pb does not match the workload Chip.pb") - _run( - [ - "cmake", - "--build", - str(compiler_build), - "--target", - "buddy-opt", - "buddy-translate", - ], - cwd=root, - env=env, - prefix="workload compiler", - logger=logger, - task_scope=task_scope, - ) - src = source_root - build = build_root target = ( "sync-ctest-bin" if ctest @@ -197,67 +102,39 @@ def build_workload( ) build.mkdir(parents=True, exist_ok=True) cmake_args = [ - "cmake", + tools["cmake"], "-G", "Ninja", "-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_C_FLAGS_RELEASE=-O2 -UNDEBUG", "-DCMAKE_CXX_FLAGS_RELEASE=-O2 -UNDEBUG", "-S", - str(src), + str(source), "-B", str(build), - f"-DPython3_EXECUTABLE={python}", - f"-DCMAKE_C_COMPILER={linux_cc}", - f"-DCMAKE_CXX_COMPILER={linux_cxx}", + f"-DPython3_EXECUTABLE={sys.executable}", + f"-DCMAKE_C_COMPILER={tools['riscv64-unknown-linux-gnu-gcc']}", + f"-DCMAKE_CXX_COMPILER={tools['riscv64-unknown-linux-gnu-g++']}", ] - if not (ctest or soctest): - cmake_args.append(f"-DBUCKYBALL_STABLE={'ON' if stable else 'OFF'}") + for variable, executable in ( + ("ELF_CC", "riscv64-unknown-elf-gcc"), + ("ELF_CXX", "riscv64-unknown-elf-g++"), + ("ELF_OBJCOPY", "riscv64-unknown-elf-objcopy"), + ): + cmake_args.append(f"-D{variable}={tools[executable]}") for key, value in defs.items(): cmake_args.append(f"-D{key}={value}") _run( cmake_args, cwd=root, - env=env, prefix="workload configure", logger=logger, task_scope=task_scope, ) - ninja = ["ninja", "-C", str(build), f"-j{1}"] - ninja.append(target) _run( - ninja, + [tools["cmake"], "--build", str(build), "--parallel", "1", "--target", target], cwd=root, - env=env, prefix="workload build", logger=logger, task_scope=task_scope, ) - - -def main() -> None: - parser = argparse.ArgumentParser(description="Build chip workloads") - parser.add_argument("--repo", type=Path, required=True) - parser.add_argument("--chip", required=True) - parser.add_argument("--instance") - parser.add_argument("--chip-pb", type=Path) - scope = parser.add_mutually_exclusive_group() - scope.add_argument("--ctest", action="store_true") - scope.add_argument("--mlirtest", action="store_true") - scope.add_argument("--soctest", action="store_true") - parser.add_argument("--stable", action="store_true") - args = parser.parse_args() - build_workload( - args.repo, - args.chip, - instance=args.instance, - chip_pb=args.chip_pb, - ctest=args.ctest, - mlirtest=args.mlirtest, - soctest=args.soctest, - stable=args.stable, - ) - - -if __name__ == "__main__": - main() diff --git a/bb-tests/workloads/src/CMakeLists.txt b/bb-tests/workloads/src/CMakeLists.txt index d3009b022..3dfae01a8 100644 --- a/bb-tests/workloads/src/CMakeLists.txt +++ b/bb-tests/workloads/src/CMakeLists.txt @@ -1,7 +1,32 @@ -set(CTEST_WORKLOAD_DIR ${WORKLOAD_SRC_DIR}/CTest) -set(MLIR_WORKLOAD_DIR ${WORKLOAD_SRC_DIR}/MLIRTest) -set(SOC_WORKLOAD_DIR ${WORKLOAD_SRC_DIR}/SocTest) - add_subdirectory(CTest) add_subdirectory(MLIRTest) add_subdirectory(SocTest) + +# =============================================================== +# Design-specific workloads +# =============================================================== +if(NOT DEFINED BUCKYBALL_DESIGN_WORKLOAD_DIR) + message( + FATAL_ERROR + "BUCKYBALL_DESIGN_WORKLOAD_DIR must select a registered simulation workload entry" + ) +endif() + +if(NOT EXISTS ${BUCKYBALL_DESIGN_WORKLOAD_DIR}/CMakeLists.txt) + message( + FATAL_ERROR + "Buckyball design workloads do not exist: ${BUCKYBALL_DESIGN_WORKLOAD_DIR}" + ) +endif() + +add_subdirectory( + ${BUCKYBALL_DESIGN_WORKLOAD_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/CTest/designs/${BUCKYBALL_WORKLOAD_DESIGN} +) + +if(NOT TARGET design-workloads-build) + message( + FATAL_ERROR + "Buckyball design workloads must define target design-workloads-build: ${BUCKYBALL_WORKLOAD_DESIGN}" + ) +endif() diff --git a/bb-tests/workloads/src/CTest/CMakeLists.txt b/bb-tests/workloads/src/CTest/CMakeLists.txt index 94a1040c8..201540ae2 100644 --- a/bb-tests/workloads/src/CTest/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/CMakeLists.txt @@ -1,278 +1,267 @@ # =============================================================== # Generic functions to help add CTest workloads # =============================================================== -set(CTEST_MAX_LINES 100) -set(BBHW_MEM_C ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/dma.c) -set(BUCKYBALL_CTEST_C_FLAGS - -g -fno-common -O2 -static -march=rv64gc -mcmodel=medany - -fno-builtin-printf -specs=nano.specs -specs=nosys.specs -nostartfiles - -Wl,-T,${BBSIM_LD} -) -file(GLOB BUCKYBALL_BALL_WORKLOAD_DIRS CONFIGURE_DEPENDS - "${BUCKYBALL_REPO_ROOT}/examples/balls/*/workloads") -file(GLOB BUCKYBALL_BALL_ISA_DEPS CONFIGURE_DEPENDS - "${BUCKYBALL_REPO_ROOT}/examples/balls/*/workloads/isa/*") - function(buckyball_enforce_ctest_line_limit SOURCE_FILE) - if(IS_ABSOLUTE "${SOURCE_FILE}") - set(_src "${SOURCE_FILE}") - else() - set(_src "${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_FILE}") - endif() - if(NOT EXISTS "${_src}") - message(FATAL_ERROR "ctest source does not exist: ${_src}") - endif() + if(IS_ABSOLUTE "${SOURCE_FILE}") + set(_src "${SOURCE_FILE}") + else() + set(_src "${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_FILE}") + endif() + if(NOT EXISTS "${_src}") + message(FATAL_ERROR "ctest source does not exist: ${_src}") + endif() - file(READ "${_src}" _content) - if(_content STREQUAL "") - set(_n 0) - else() - string(REGEX MATCHALL "\n" _newlines "${_content}") - list(LENGTH _newlines _n) - string(REGEX MATCH "[^\n]$" _no_trailing_nl "${_content}") - if(_no_trailing_nl) - math(EXPR _n "${_n} + 1") + file(READ "${_src}" _content) + if(_content STREQUAL "") + set(_n 0) + else() + string(REGEX MATCHALL "\n" _newlines "${_content}") + list(LENGTH _newlines _n) + string(REGEX MATCH "[^\n]$" _no_trailing_nl "${_content}") + if(_no_trailing_nl) + math(EXPR _n "${_n} + 1") + endif() endif() - endif() - if(_n GREATER CTEST_MAX_LINES) - message(FATAL_ERROR - "ctest exceeds ${CTEST_MAX_LINES} lines: ${_src} (${_n} lines). " - "Split it into smaller focused ctests. " - "Do not evade this check by moving functional code into headers.") - endif() + if(_n GREATER CTEST_MAX_LINES) + message( + FATAL_ERROR + "ctest exceeds ${CTEST_MAX_LINES} lines: ${_src} (${_n} lines). " + "Split it into smaller focused ctests. " + "Do not evade this check by moving functional code into headers." + ) + endif() endfunction() function(buckyball_ctest_deps OUT_DEPS SOURCE_DIR SOURCE_FILE) - if(NOT DEFINED BUCKYBALL_CTEST_TARGET OR BUCKYBALL_CTEST_TARGET STREQUAL "") - message(FATAL_ERROR "BUCKYBALL_CTEST_TARGET is required for C tests") - endif() - list(FIND BUCKYBALL_TARGETS "${BUCKYBALL_CTEST_TARGET}" _buckyball_target_index) - if(_buckyball_target_index EQUAL -1) - message(FATAL_ERROR "unknown C test target: ${BUCKYBALL_CTEST_TARGET}") - endif() - file(GLOB BUCKYBALL_ISA_DEPS CONFIGURE_DEPENDS - ${WORKLOAD_LIB_DIR}/bbhw/isa/*) - set(DEPS - ${SOURCE_DIR}/${SOURCE_FILE} - ${WORKLOAD_LIB_DIR}/buckyball.c - ${WORKLOAD_LIB_DIR}/buckyball.h - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/ballISA.h - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h - ${BUCKYBALL_ISA_DIR}/chip/topology.h - ${BBHW_MEM_C} - ${BUCKYBALL_REPO_ROOT}/stack/runtime/include/dma.h - ${BUCKYBALL_CHIP_COMMON_SOURCES} - ${BUCKYBALL_CHIP_COMMON_HEADERS} - ${BUCKYBALL_ISA_DEPS} - ${BUCKYBALL_BALL_ISA_DEPS}) - file(GLOB _LOCAL_HDRS CONFIGURE_DEPENDS ${SOURCE_DIR}/*.h) - list(APPEND DEPS ${_LOCAL_HDRS}) + if(NOT DEFINED BUCKYBALL_CTEST_TARGET OR BUCKYBALL_CTEST_TARGET STREQUAL "") + message(FATAL_ERROR "BUCKYBALL_CTEST_TARGET is required for C tests") + endif() + list( + FIND DESIGN_TARGETS + "${BUCKYBALL_CTEST_TARGET}" + _buckyball_target_index + ) + if(_buckyball_target_index EQUAL -1) + message(FATAL_ERROR "unknown C test target: ${BUCKYBALL_CTEST_TARGET}") + endif() + file( + GLOB BUCKYBALL_ISA_DEPS + CONFIGURE_DEPENDS + ${WORKLOAD_LIB_DIR}/bbhw/isa/* + ) + set(DEPS + ${SOURCE_DIR}/${SOURCE_FILE} + ${WORKLOAD_LIB_DIR}/buckyball.c + ${WORKLOAD_LIB_DIR}/buckyball.h + ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/ballISA.h + ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h + ${BUCKYBALL_ISA_DIR}/design/instances.h + ${BBHW_MEM_C} + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/dma.h + ${BUCKYBALL_DESIGN_COMMON_SOURCES} + ${BUCKYBALL_DESIGN_COMMON_HEADERS} + ${BUCKYBALL_ISA_DEPS} + ${BUCKYBALL_BALL_ISA_DEPS} + ) + file(GLOB _LOCAL_HDRS CONFIGURE_DEPENDS ${SOURCE_DIR}/*.h) + list(APPEND DEPS ${_LOCAL_HDRS}) - set(${OUT_DEPS} ${DEPS} PARENT_SCOPE) + set(${OUT_DEPS} ${DEPS} PARENT_SCOPE) endfunction() function(add_buckyball_linux_ctest TEST_NAME SOURCE_DIR SOURCE_FILE) - set(EXECUTABLE "${TEST_NAME}-linux") + set(EXECUTABLE "${TEST_NAME}-linux") - add_executable(${EXECUTABLE} - ${SOURCE_DIR}/${SOURCE_FILE} - ${WORKLOAD_LIB_DIR}/buckyball.c - ${BBHW_MEM_C} - ${BUCKYBALL_CHIP_COMMON_SOURCES}) - target_include_directories(${EXECUTABLE} PRIVATE - ${WORKLOAD_LIB_DIR} - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} - ${BUCKYBALL_CHIP_COMMON_INCLUDE_DIRS} - ${BUCKYBALL_BALL_WORKLOAD_DIRS} - ${SOURCE_DIR}) - set_target_properties(${EXECUTABLE} PROPERTIES LINKER_LANGUAGE C) + add_executable( + ${EXECUTABLE} + ${SOURCE_DIR}/${SOURCE_FILE} + ${WORKLOAD_LIB_DIR}/buckyball.c + ${BBHW_MEM_C} + ${BUCKYBALL_DESIGN_COMMON_SOURCES} + ) + target_include_directories( + ${EXECUTABLE} + PRIVATE + ${WORKLOAD_LIB_DIR} + ${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services + ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} + ${BUCKYBALL_DESIGN_COMMON_INCLUDE_DIRS} + ${BUCKYBALL_BALL_WORKLOAD_DIRS} + ${SOURCE_DIR} + ) + set_target_properties(${EXECUTABLE} PROPERTIES LINKER_LANGUAGE C) - add_custom_target(${TEST_NAME}-linux-build - DEPENDS ${EXECUTABLE}) + add_custom_target(${TEST_NAME}-linux-build DEPENDS ${EXECUTABLE}) endfunction() function(add_buckyball_baremetal_ctest TEST_NAME SOURCE_DIR SOURCE_FILE) - set(EXECUTABLE "${TEST_NAME}-baremetal") - buckyball_ctest_deps(TEST_DEPS ${SOURCE_DIR} ${SOURCE_FILE}) + set(EXECUTABLE "${TEST_NAME}-baremetal") + buckyball_ctest_deps(TEST_DEPS ${SOURCE_DIR} ${SOURCE_FILE}) - set(BUCKYBALL_CHIP_COMMON_INCLUDE_FLAGS) - foreach(CHIP_COMMON_INCLUDE_DIR ${BUCKYBALL_CHIP_COMMON_INCLUDE_DIRS}) - list(APPEND BUCKYBALL_CHIP_COMMON_INCLUDE_FLAGS -I${CHIP_COMMON_INCLUDE_DIR}) - endforeach() - set(BALL_ISA_INCLUDE_FLAGS) - foreach(BALL_WORKLOAD_DIR ${BUCKYBALL_BALL_WORKLOAD_DIRS}) - list(APPEND BALL_ISA_INCLUDE_FLAGS -I${BALL_WORKLOAD_DIR}) - endforeach() + set(BUCKYBALL_DESIGN_COMMON_INCLUDE_FLAGS) + foreach(DESIGN_COMMON_INCLUDE_DIR ${BUCKYBALL_DESIGN_COMMON_INCLUDE_DIRS}) + list( + APPEND BUCKYBALL_DESIGN_COMMON_INCLUDE_FLAGS + -I${DESIGN_COMMON_INCLUDE_DIR} + ) + endforeach() + set(BALL_ISA_INCLUDE_FLAGS) + foreach(BALL_WORKLOAD_DIR ${BUCKYBALL_BALL_WORKLOAD_DIRS}) + list(APPEND BALL_ISA_INCLUDE_FLAGS -I${BALL_WORKLOAD_DIR}) + endforeach() - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMAND ${ELF_CC} ${BUCKYBALL_CTEST_C_FLAGS} - -o ${EXECUTABLE} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${WORKLOAD_LIB_DIR}/buckyball.c - ${BBHW_MEM_C} - ${BUCKYBALL_CHIP_COMMON_SOURCES} - ${SOURCE_DIR}/${SOURCE_FILE} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -I${BUCKYBALL_ISA_DIR}/chip - -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} - ${BUCKYBALL_CHIP_COMMON_INCLUDE_FLAGS} - ${BALL_ISA_INCLUDE_FLAGS} - -I${SOURCE_DIR} - DEPENDS - ${TEST_DEPS} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSIM_LD} - COMMENT "Building baremetal executable: ${EXECUTABLE}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + COMMAND + ${ELF_CC} ${BUCKYBALL_CTEST_C_FLAGS} -o ${EXECUTABLE} + ${BAREMETAL_DIR}/crt0.S ${WORKLOAD_LIB_DIR}/buckyball.c + ${BBHW_MEM_C} ${BUCKYBALL_DESIGN_COMMON_SOURCES} + ${SOURCE_DIR}/${SOURCE_FILE} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} -I${BUCKYBALL_ISA_DIR}/design + -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} + ${BUCKYBALL_DESIGN_COMMON_INCLUDE_FLAGS} ${BALL_ISA_INCLUDE_FLAGS} + -I${SOURCE_DIR} + DEPENDS ${TEST_DEPS} ${BAREMETAL_DIR}/crt0.S ${BBSIM_LD} + COMMENT "Building baremetal executable: ${EXECUTABLE}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) - add_custom_target(${TEST_NAME}-baremetal-build - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE}) + add_custom_target( + ${TEST_NAME}-baremetal-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + ) endfunction() function(add_buckyball_ctest SOURCE_FILE) - if(BUCKYBALL_WORKLOAD_CHIP STREQUAL "") - message(FATAL_ERROR - "BUCKYBALL_WORKLOAD_CHIP must be set before add_buckyball_ctest()") - endif() - get_filename_component(_stem ${SOURCE_FILE} NAME_WE) - if(NOT DEFINED BUCKYBALL_CTEST_TARGET OR BUCKYBALL_CTEST_TARGET STREQUAL "") - message(FATAL_ERROR "BUCKYBALL_CTEST_TARGET is required for ${SOURCE_FILE}") - endif() - buckyball_ctest_name(TEST_NAME ${BUCKYBALL_CTEST_TARGET} ${_stem}) - set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - - buckyball_enforce_ctest_line_limit(${SOURCE_FILE}) - buckyball_ctest_deps(TEST_DEPS ${SOURCE_DIR} ${SOURCE_FILE}) + if(BUCKYBALL_WORKLOAD_DESIGN STREQUAL "") + message( + FATAL_ERROR + "BUCKYBALL_WORKLOAD_DESIGN must be set before add_buckyball_ctest()" + ) + endif() + get_filename_component(_stem ${SOURCE_FILE} NAME_WE) + if(NOT DEFINED BUCKYBALL_CTEST_TARGET OR BUCKYBALL_CTEST_TARGET STREQUAL "") + message( + FATAL_ERROR + "BUCKYBALL_CTEST_TARGET is required for ${SOURCE_FILE}" + ) + endif() + set(TEST_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-${_stem}" + ) + set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - add_buckyball_linux_ctest(${TEST_NAME} ${SOURCE_DIR} ${SOURCE_FILE}) - add_buckyball_baremetal_ctest(${TEST_NAME} ${SOURCE_DIR} ${SOURCE_FILE}) + buckyball_enforce_ctest_line_limit(${SOURCE_FILE}) - add_custom_target(${TEST_NAME}-ctest-build - DEPENDS - ${TEST_NAME}-linux-build - ${TEST_NAME}-baremetal-build - COMMENT "Building CTest workload ${TEST_NAME}" - ) + add_buckyball_linux_ctest(${TEST_NAME} ${SOURCE_DIR} ${SOURCE_FILE}) + add_buckyball_baremetal_ctest(${TEST_NAME} ${SOURCE_DIR} ${SOURCE_FILE}) - if(DEFINED BUCKYBALL_CTEST_GROUP_TARGET) - add_dependencies(${BUCKYBALL_CTEST_GROUP_TARGET} ${TEST_NAME}-ctest-build) - endif() -endfunction() + add_custom_target( + ${TEST_NAME}-ctest-build + DEPENDS ${TEST_NAME}-linux-build ${TEST_NAME}-baremetal-build + COMMENT "Building CTest workload ${TEST_NAME}" + ) -function(add_buckyball_ctests) - foreach(SOURCE_FILE ${ARGV}) - add_buckyball_ctest(${SOURCE_FILE}) - endforeach() + if(DEFINED BUCKYBALL_CTEST_GROUP_TARGET) + add_dependencies( + ${BUCKYBALL_CTEST_GROUP_TARGET} + ${TEST_NAME}-ctest-build + ) + endif() endfunction() -add_custom_target(core-ctests-build) +add_custom_target(component-ctests-build) -function(add_buckyball_core_workload NAME) - cmake_parse_arguments(ARG "" "" "SOURCES" ${ARGN}) - if(NOT DEFINED BUCKYBALL_CORE_WORKLOAD_TARGET OR - BUCKYBALL_CORE_WORKLOAD_TARGET STREQUAL "") - message(FATAL_ERROR "Core workload target is required for ${NAME}") - endif() - if(NOT DEFINED BUCKYBALL_CORE_WORKLOAD_PROFILE OR - BUCKYBALL_CORE_WORKLOAD_PROFILE STREQUAL "" OR NOT ARG_SOURCES) - message(FATAL_ERROR "Core workload profile and sources are required for ${NAME}") - endif() - set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_CORE_WORKLOAD_TARGET}") - buckyball_ctest_name(_test ${BUCKYBALL_CTEST_TARGET} core-${NAME}) - set(_elf "${_test}-multicore-baremetal") - set(_sources) - foreach(_source IN LISTS ARG_SOURCES) - buckyball_enforce_ctest_line_limit("${_source}") - list(APPEND _sources "${CMAKE_CURRENT_SOURCE_DIR}/${_source}") - endforeach() - set(_ball_includes) - foreach(_dir IN LISTS BUCKYBALL_BALL_WORKLOAD_DIRS) - list(APPEND _ball_includes -I${_dir}) - endforeach() - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${_elf}" - COMMAND ${RISCV_GNU_TOOLCHAIN}/bin/riscv64-unknown-elf-g++ - ${BUCKYBALL_CTEST_C_FLAGS} - -DBB_CORE_WORKLOAD_PROFILE=${BUCKYBALL_CORE_WORKLOAD_PROFILE} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -I${WORKLOAD_LIB_DIR}/bbsw/baremetal/multicore - -I${BUCKYBALL_ISA_DIR}/chip - -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} - ${_ball_includes} - -o ${_elf} - ${WORKLOAD_LIB_DIR}/bbsw/baremetal/start.S - ${_sources} - ${WORKLOAD_LIB_DIR}/buckyball.c - ${BBHW_MEM_C} - DEPENDS ${_sources} - ${WORKLOAD_LIB_DIR}/bbsw/baremetal/start.S - ${BUCKYBALL_ISA_DIR}/chip/topology.h - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/ballISA.h - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - add_custom_target(${_test}-core-build - DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${_elf}") - add_dependencies(core-ctests-build ${_test}-core-build) +function(add_buckyball_component_workload NAME) + cmake_parse_arguments(ARG "" "" "SOURCES" ${ARGN}) + if( + NOT DEFINED BUCKYBALL_CTEST_TARGET + OR BUCKYBALL_CTEST_TARGET STREQUAL "" + OR NOT ARG_SOURCES + ) + message( + FATAL_ERROR + "Workload target and sources are required for ${NAME}" + ) + endif() + set(_profile "${DESIGN_TARGET_ID_${BUCKYBALL_CTEST_TARGET}}") + set(_test + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-pipeline-${NAME}" + ) + set(_elf "${_test}-multicore-baremetal") + set(_sources) + foreach(_source IN LISTS ARG_SOURCES) + buckyball_enforce_ctest_line_limit("${_source}") + list(APPEND _sources "${CMAKE_CURRENT_SOURCE_DIR}/${_source}") + endforeach() + file(GLOB_RECURSE _headers CONFIGURE_DEPENDS "${WORKLOAD_LIB_DIR}/*.h") + set(_ball_includes) + foreach(_dir IN LISTS BUCKYBALL_BALL_WORKLOAD_DIRS) + list(APPEND _ball_includes -I${_dir}) + file(GLOB_RECURSE _ball_headers CONFIGURE_DEPENDS "${_dir}/*.h") + list(APPEND _headers ${_ball_headers}) + endforeach() + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${_elf}" + COMMAND + ${ELF_CXX} ${BUCKYBALL_CTEST_C_FLAGS} + -DBB_COMPONENT_WORKLOAD_PROFILE=${_profile} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} -I${BAREMETAL_DIR}/multithread + -I${BUCKYBALL_ISA_DIR}/design + -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} ${_ball_includes} + -o ${_elf} ${BAREMETAL_DIR}/start.S ${_sources} + ${WORKLOAD_LIB_DIR}/buckyball.c ${BBHW_MEM_C} + DEPENDS + ${_sources} + ${_headers} + ${BAREMETAL_DIR}/start.S + ${BUCKYBALL_ISA_DIR}/design/instances.h + ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/ballISA.h + ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + add_custom_target( + ${_test}-thread-build + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${_elf}" + ) + add_dependencies(${BUCKYBALL_CTEST_GROUP_TARGET} ${_test}-thread-build) endfunction() # =============================================================== -# Common workloads for all chips -# =============================================================== -add_subdirectory(base) -add_subdirectory(tlb) -add_subdirectory(tutorial) -add_subdirectory(embench) -add_subdirectory(coremark) -buckyball_add_core_workload_subdirs(ctests) - -# =============================================================== -# Chip-specific workloads +# Common workloads for all designs # =============================================================== -set(BUCKYBALL_CHIP_WORKLOAD_DIR - ${BUCKYBALL_REPO_ROOT}/examples/chips/${BUCKYBALL_WORKLOAD_SOURCE_CHIP}/workloads) - -if(NOT EXISTS ${BUCKYBALL_CHIP_WORKLOAD_DIR}/CMakeLists.txt) - message(FATAL_ERROR - "Buckyball chip workloads do not exist: ${BUCKYBALL_CHIP_WORKLOAD_DIR}") -endif() - -add_subdirectory(${BUCKYBALL_CHIP_WORKLOAD_DIR} - ${CMAKE_CURRENT_BINARY_DIR}/chips/${BUCKYBALL_WORKLOAD_SOURCE_CHIP}) - -if(NOT TARGET chip-workloads-build) - message(FATAL_ERROR - "Buckyball chip workloads must define target chip-workloads-build: ${BUCKYBALL_WORKLOAD_CHIP}") -endif() - -add_custom_target(ctest-workloads-build - DEPENDS - base-ctests-build - tlb-ctests-build - tutorial-ctests-build - core-ctests-build - chip-ctests-build - COMMENT "Building CTest workloads" +add_custom_target( + ctest-workloads-build + DEPENDS design-ctests-build + COMMENT "Building registered CTest workloads" ) -add_custom_target(sync-ctest-bin - COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src/CTest - COMMAND rsync -a --delete --exclude=CMakeFiles - "--exclude=*/mlir_tests/**" "--include=*/" - "--include=*-baremetal" "--include=*-linux" - "--include=*-baremetal.expect" "--exclude=*" - ${CMAKE_CURRENT_BINARY_DIR}/ ${OUTPUT_BIN_DIR}/src/CTest/ - DEPENDS ctest-workloads-build - COMMENT "Syncing CTest binaries to output directory" +add_custom_target( + sync-ctest-bin + COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src/CTest + COMMAND + rsync -a --delete --exclude=CMakeFiles "--exclude=*/mlir_tests/**" + "--include=*/" "--include=*-baremetal" "--include=*-linux" + "--include=*-baremetal.expect" "--exclude=*" + ${CMAKE_CURRENT_BINARY_DIR}/ ${OUTPUT_BIN_DIR}/src/CTest/ + DEPENDS ctest-workloads-build + COMMENT "Syncing CTest binaries to output directory" ) -add_custom_target(sync-native-bin - COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src/CTest - COMMAND rsync -a --delete --exclude=CMakeFiles "--exclude=mlir_tests" "--include=*/" - "--include=*-baremetal" "--include=*-baremetal.expect" - "--include=*-linux" "--include=*-run" "--exclude=*" - ${CMAKE_CURRENT_BINARY_DIR}/ ${OUTPUT_BIN_DIR}/src/CTest/ - DEPENDS ctest-workloads-build embench-all coremark-all - COMMENT "Syncing CTest and benchmark binaries") +add_custom_target( + sync-native-bin + COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src/CTest + COMMAND + rsync -a --delete --exclude=CMakeFiles "--exclude=mlir_tests" + "--include=*/" "--include=*-baremetal" "--include=*-baremetal.expect" + "--include=*-linux" "--include=*-run" "--exclude=*" + ${CMAKE_CURRENT_BINARY_DIR}/ ${OUTPUT_BIN_DIR}/src/CTest/ + DEPENDS ctest-workloads-build + COMMENT "Syncing CTest and benchmark binaries" +) diff --git a/bb-tests/workloads/src/CTest/base/CMakeLists.txt b/bb-tests/workloads/src/CTest/base/CMakeLists.txt index f4fdbdd2b..6d7d55b5f 100644 --- a/bb-tests/workloads/src/CTest/base/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/base/CMakeLists.txt @@ -1,32 +1,38 @@ add_custom_target(base-ctests-build) set(BUCKYBALL_CTEST_GROUP_TARGET base-ctests-build) -foreach(BUCKYBALL_CTEST_TARGET IN LISTS BUCKYBALL_CPU_TARGETS) - set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h") - foreach(_name BANK_NUM BANK_WIDTH BANK_LINES BB_PRIVATE_VBANK_MAX) - file(STRINGS "${_params}" _line REGEX "^#define ${_name} ") - string(REGEX REPLACE ".* " "" _${_name} "${_line}") - endforeach() - if(_BB_PRIVATE_VBANK_MAX EQUAL 0) - continue() - endif() - math(EXPR _word_remainder "${_BANK_WIDTH} % 32") - if(_BANK_NUM LESS 4 OR _BB_PRIVATE_VBANK_MAX LESS 2 OR _BANK_LINES LESS 24 OR - _BANK_WIDTH LESS 64 OR _BANK_WIDTH GREATER 2048 OR NOT _word_remainder EQUAL 0) - message(FATAL_ERROR - "base DMA tests require 4 banks for ACC, vbank2, >=24 rows for 2D, and 64..2048-bit rows divisible by 32; ${BUCKYBALL_CTEST_TARGET}: banks=${_BANK_NUM}, privateMax=${_BB_PRIVATE_VBANK_MAX}, lines=${_BANK_LINES}, width=${_BANK_WIDTH}") - endif() - if(_BB_PRIVATE_VBANK_MAX GREATER_EQUAL 4) - add_buckyball_ctest(mset_transfer_test.c) - endif() - add_buckyball_ctests( - mvin_mvout_group_test.c - mvin_2d_test.c - mvin_2d_pixel_stride_test.c - mvin_mvout_acc_test.c - mvin_mvout_stride_test.c - mvin_mvout_test.c - mvin_mvout_unaligned_test.c - mvin_unaligned_test.c - ) +foreach(BUCKYBALL_CTEST_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h") + foreach(_name BANK_NUM BANK_WIDTH BANK_LINES BB_PRIVATE_VBANK_MAX) + file(STRINGS "${_params}" _line REGEX "^#define ${_name} ") + string(REGEX REPLACE ".* " "" _${_name} "${_line}") + endforeach() + if(_BB_PRIVATE_VBANK_MAX EQUAL 0) + continue() + endif() + math(EXPR _word_remainder "${_BANK_WIDTH} % 32") + if( + _BANK_NUM LESS 4 + OR _BB_PRIVATE_VBANK_MAX LESS 2 + OR _BANK_LINES LESS 24 + OR _BANK_WIDTH LESS 64 + OR _BANK_WIDTH GREATER 2048 + OR NOT _word_remainder EQUAL 0 + ) + message( + FATAL_ERROR + "base DMA tests require 4 banks for ACC, vbank2, >=24 rows for 2D, and 64..2048-bit rows divisible by 32; ${BUCKYBALL_CTEST_TARGET}: banks=${_BANK_NUM}, privateMax=${_BB_PRIVATE_VBANK_MAX}, lines=${_BANK_LINES}, width=${_BANK_WIDTH}" + ) + endif() + if(_BB_PRIVATE_VBANK_MAX GREATER_EQUAL 4) + add_buckyball_ctest(mset_transfer_test.c) + endif() + add_buckyball_ctest(mvin_mvout_group_test.c) + add_buckyball_ctest(mvin_2d_test.c) + add_buckyball_ctest(mvin_2d_pixel_stride_test.c) + add_buckyball_ctest(mvin_mvout_acc_test.c) + add_buckyball_ctest(mvin_mvout_stride_test.c) + add_buckyball_ctest(mvin_mvout_test.c) + add_buckyball_ctest(mvin_mvout_unaligned_test.c) + add_buckyball_ctest(mvin_unaligned_test.c) endforeach() diff --git a/bb-tests/workloads/src/CTest/base/mvin_2d_test.c b/bb-tests/workloads/src/CTest/base/mvin_2d_test.c index 5949fc055..8b1ec2f9f 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_2d_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_2d_test.c @@ -9,21 +9,22 @@ enum { SOURCE_WIDTH = 7, PANEL_BYTES = (BANK_WIDTH / 8), CHANNELS = (PANEL_BYTES + PANEL_BYTES / 2), - TILE_HEIGHT = 4, - TILE_WIDTH = 3, + CORE_HEIGHT = 4, + CORE_WIDTH = 3, START_Y = 1, START_X = 2, - TILE_ROWS = (TILE_HEIGHT * TILE_WIDTH) + CORE_ROWS = (CORE_HEIGHT * CORE_WIDTH) }; static elem_t input[SOURCE_HEIGHT * SOURCE_WIDTH * CHANNELS] __attribute__((aligned(128))); -static elem_t output[2 * TILE_ROWS * PANEL_BYTES] __attribute__((aligned(128))); +static elem_t output[2 * CORE_ROWS * PANEL_BYTES] __attribute__((aligned(128))); +static elem_t dirty[2 * CORE_ROWS * PANEL_BYTES] __attribute__((aligned(128))); static int check_output(void) { for (int panel = 0; panel < 2; ++panel) { - for (int y = 0; y < TILE_HEIGHT; ++y) { - for (int x = 0; x < TILE_WIDTH; ++x) { + for (int y = 0; y < CORE_HEIGHT; ++y) { + for (int x = 0; x < CORE_WIDTH; ++x) { for (int lane = 0; lane < PANEL_BYTES; ++lane) { int channel = panel * PANEL_BYTES + lane; elem_t expected = @@ -32,7 +33,7 @@ static int check_output(void) { CHANNELS + channel] : 0; - int row = panel * TILE_ROWS + y * TILE_WIDTH + x; + int row = panel * CORE_ROWS + y * CORE_WIDTH + x; elem_t actual = output[row * PANEL_BYTES + lane]; if (actual != expected) { printf("mvin_2d mismatch panel=%d y=%d x=%d lane=%d expected=%d " @@ -50,18 +51,22 @@ static int check_output(void) { int main(void) { for (int i = 0; i < SOURCE_HEIGHT * SOURCE_WIDTH * CHANNELS; ++i) input[i] = (elem_t)((i * 13 + 7) & 0x7f); - for (int i = 0; i < 2 * TILE_ROWS * PANEL_BYTES; ++i) + for (int i = 0; i < 2 * CORE_ROWS * PANEL_BYTES; ++i) { output[i] = (elem_t)-1; + dirty[i] = (elem_t)0x5a; + } uint32_t bank = 0; bb_mem_alloc(bank, 1, 1); + // Tail lanes must be zeroed even when the destination already contains data. + bb_mvin((uintptr_t)dirty, bank, 2 * CORE_ROWS, 1); uintptr_t source = (uintptr_t)&input[(START_Y * SOURCE_WIDTH + START_X) * CHANNELS]; - bb_mvin_2d(source, bank, TILE_HEIGHT, CHANNELS, SOURCE_WIDTH, 0, TILE_WIDTH, + bb_mvin_2d(source, bank, CORE_HEIGHT, CHANNELS, SOURCE_WIDTH, 0, CORE_WIDTH, PANEL_BYTES); - bb_mvin_2d(source + PANEL_BYTES, bank, TILE_HEIGHT, CHANNELS, SOURCE_WIDTH, - TILE_ROWS, TILE_WIDTH, CHANNELS - PANEL_BYTES); - bb_mvout((uintptr_t)output, bank, 2 * TILE_ROWS, 1); + bb_mvin_2d(source + PANEL_BYTES, bank, CORE_HEIGHT, CHANNELS, SOURCE_WIDTH, + CORE_ROWS, CORE_WIDTH, CHANNELS - PANEL_BYTES); + bb_mvout((uintptr_t)output, bank, 2 * CORE_ROWS, 1); if (!check_output()) return 1; diff --git a/bb-tests/workloads/src/CTest/coremark/CMakeLists.txt b/bb-tests/workloads/src/CTest/coremark/CMakeLists.txt index 6820d958d..c57961177 100644 --- a/bb-tests/workloads/src/CTest/coremark/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/coremark/CMakeLists.txt @@ -1,65 +1,71 @@ -set(ELF_CC "riscv64-unknown-elf-gcc") - -set(CTEST_COREMARK_WORKLOAD_DIR ${CTEST_WORKLOAD_DIR}/coremark) +set(CTEST_COREMARK_WORKLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}) set(COREMARK_PORT_DIR ${CTEST_COREMARK_WORKLOAD_DIR}/buckyball) set(COREMARK_SRC_DIR ${CTEST_COREMARK_WORKLOAD_DIR}/src) -set(BBSIM_LD ${BBSW_BAREMETAL_DIR}/bbsim.ld) -list(GET BUCKYBALL_CPU_TARGETS 0 COREMARK_CORE) +set(COREMARK_THREAD ${BUCKYBALL_CONTROL_TARGET}) set(COREMARK_SOURCES - ${COREMARK_SRC_DIR}/core_list_join.c - ${COREMARK_SRC_DIR}/core_main.c - ${COREMARK_SRC_DIR}/core_matrix.c - ${COREMARK_SRC_DIR}/core_state.c - ${COREMARK_SRC_DIR}/core_util.c - ${COREMARK_PORT_DIR}/core_portme.c - ${COREMARK_PORT_DIR}/main.c + ${COREMARK_SRC_DIR}/core_list_join.c + ${COREMARK_SRC_DIR}/core_main.c + ${COREMARK_SRC_DIR}/core_matrix.c + ${COREMARK_SRC_DIR}/core_state.c + ${COREMARK_SRC_DIR}/core_util.c + ${COREMARK_PORT_DIR}/core_portme.c + ${COREMARK_PORT_DIR}/main.c ) set(COREMARK_HEADERS - ${COREMARK_SRC_DIR}/coremark.h - ${COREMARK_PORT_DIR}/core_portme.h + ${COREMARK_SRC_DIR}/coremark.h + ${COREMARK_PORT_DIR}/core_portme.h ) set(COREMARK_C_FLAGS - -g -fno-common -O2 -static -std=gnu99 -march=rv64gc -mcmodel=medany - -fno-builtin-printf -specs=nano.specs -specs=nosys.specs -nostartfiles - -Wl,-T,${BBSIM_LD} - -I${COREMARK_SRC_DIR} - -I${COREMARK_PORT_DIR} - -I${BUCKYBALL_ISA_DIR}/${COREMARK_CORE} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -ffunction-sections -fdata-sections - -Wl,--gc-sections - -DITERATIONS=10 - -DBUCKYBALL_WORKLOAD=1 + -g + -fno-common + -O2 + -static + -std=gnu99 + -march=rv64gc + -mcmodel=medany + -fno-builtin-printf + -specs=nano.specs + -specs=nosys.specs + -nostartfiles + -Wl,-T,${BBSIM_LD} + -I${COREMARK_SRC_DIR} + -I${COREMARK_PORT_DIR} + -I${BUCKYBALL_ISA_DIR}/control + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} + -ffunction-sections + -fdata-sections + -Wl,--gc-sections + -DITERATIONS=10 + -DBUCKYBALL_WORKLOAD=1 ) -function(add_coremark_target) - buckyball_ctest_name(COREMARK_NAME ${COREMARK_CORE} benchmark-coremark) - set(EXECUTABLE "${COREMARK_NAME}-baremetal") +set(COREMARK_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${COREMARK_THREAD}-ctest-benchmark-coremark" +) +set(EXECUTABLE "${COREMARK_NAME}-baremetal") - add_custom_command( +add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMAND ${ELF_CC} ${COREMARK_C_FLAGS} -o ${EXECUTABLE} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - ${BBSW_BAREMETAL_DIR}/test.c - ${COREMARK_SOURCES} - -lm - DEPENDS ${COREMARK_SOURCES} - ${COREMARK_HEADERS} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - ${BBSW_BAREMETAL_DIR}/test.c - ${BUCKYBALL_ISA_DIR}/${COREMARK_CORE}/params.h + COMMAND + ${ELF_CC} ${COREMARK_C_FLAGS} -o ${EXECUTABLE} ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c ${COREMARK_SOURCES} -lm + DEPENDS + ${COREMARK_SOURCES} + ${COREMARK_HEADERS} + ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c + ${BUCKYBALL_ISA_DIR}/control/params.h COMMENT "Building coremark: ${EXECUTABLE}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) +) - add_custom_target(coremark-all +add_custom_target( + coremark-all DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMENT "Building coremark for Buckyball") -endfunction() - -add_coremark_target() + COMMENT "Building coremark for Buckyball" +) diff --git a/bb-tests/workloads/src/CTest/coremark/buckyball/main.c b/bb-tests/workloads/src/CTest/coremark/buckyball/main.c index 92755e170..91ab33f0b 100644 --- a/bb-tests/workloads/src/CTest/coremark/buckyball/main.c +++ b/bb-tests/workloads/src/CTest/coremark/buckyball/main.c @@ -1,16 +1,3 @@ -#include -#include - int coremark_main(void); -int main(void) { - int ret = coremark_main(); - if (ret == 0) { - bb_test_pass(); - while (1) { - } - } - - printf("CoreMark failed with code %d\n", ret); - bb_test_fail(); -} +int main(void) { return coremark_main(); } diff --git a/bb-tests/workloads/src/CTest/coremark/src/core_state.c b/bb-tests/workloads/src/CTest/coremark/src/core_state.c index 7a7a91a8b..e956cb45e 100644 --- a/bb-tests/workloads/src/CTest/coremark/src/core_state.c +++ b/bb-tests/workloads/src/CTest/coremark/src/core_state.c @@ -28,8 +28,8 @@ Topic: Description implementation is used instead, trading speed of direct coding for ease of maintenance. - Since the main goal of using a state machine in CoreMark is to excercise -the switch/if behaviour, we are using a small moore machine. + Since the main goal of using a state machine in CoreMark is to +excercise the switch/if behaviour, we are using a small moore machine. In particular, this machine tests type of string input, trying to determine whether the input is a number or something else. diff --git a/bb-tests/workloads/src/CTest/coremark/src/coremark.h b/bb-tests/workloads/src/CTest/coremark/src/coremark.h index 8f80d25bc..f014a0b46 100644 --- a/bb-tests/workloads/src/CTest/coremark/src/coremark.h +++ b/bb-tests/workloads/src/CTest/coremark/src/coremark.h @@ -151,7 +151,7 @@ typedef struct RESULTS_S { core_portable port; } core_results; -/* Multicore execution handling */ +/* Multithread execution handling */ #if (MULTITHREAD > 1) ee_u8 core_start_parallel(core_results *res); ee_u8 core_stop_parallel(core_results *res); diff --git a/bb-tests/workloads/src/CTest/embench/CMakeLists.txt b/bb-tests/workloads/src/CTest/embench/CMakeLists.txt index 93a110695..f4afb9029 100644 --- a/bb-tests/workloads/src/CTest/embench/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/embench/CMakeLists.txt @@ -1,174 +1,205 @@ -set(ELF_CC "riscv64-unknown-elf-gcc") -set(ELF_OBJCOPY "riscv64-unknown-elf-objcopy") - # Embench: baremetal benchmarks sharing toy's linker script and startup -# code in lib/bbsw/baremetal/. -set(CTEST_EMBENCH_WORKLOAD_DIR ${CTEST_WORKLOAD_DIR}/embench) +# code in stack/os/baremetal/. +set(CTEST_EMBENCH_WORKLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}) set(EMBENCH_SUPPORT_DIR ${CTEST_EMBENCH_WORKLOAD_DIR}/support) set(EMBENCH_SRC_DIR ${CTEST_EMBENCH_WORKLOAD_DIR}/src) -set(BBSIM_LD ${BBSW_BAREMETAL_DIR}/bbsim.ld) -list(GET BUCKYBALL_CPU_TARGETS 0 EMBENCH_CORE) +set(EMBENCH_THREAD ${BUCKYBALL_CONTROL_TARGET}) set(EMBENCH_SUPPORT_SOURCES - ${EMBENCH_SUPPORT_DIR}/main.c - ${EMBENCH_SUPPORT_DIR}/beebsc.c - ${EMBENCH_SUPPORT_DIR}/boardsupport.c - ${EMBENCH_SUPPORT_DIR}/chipsupport.c + ${EMBENCH_SUPPORT_DIR}/main.c + ${EMBENCH_SUPPORT_DIR}/beebsc.c + ${EMBENCH_SUPPORT_DIR}/boardsupport.c + ${EMBENCH_SUPPORT_DIR}/chipsupport.c ) set(EMBENCH_TOP_SUPPORT_SOURCES - ${EMBENCH_SUPPORT_DIR}/top.c - ${EMBENCH_SUPPORT_DIR}/beebsc.c - ${EMBENCH_SUPPORT_DIR}/boardsupport.c - ${EMBENCH_SUPPORT_DIR}/chipsupport.c + ${EMBENCH_SUPPORT_DIR}/top.c + ${EMBENCH_SUPPORT_DIR}/beebsc.c + ${EMBENCH_SUPPORT_DIR}/boardsupport.c + ${EMBENCH_SUPPORT_DIR}/chipsupport.c ) set(EMBENCH_C_FLAGS - -g -fno-common -O2 -static -march=rv64gc -mcmodel=medany - -fno-builtin-printf -specs=nano.specs -specs=nosys.specs -nostartfiles - -Wl,-T,${BBSIM_LD} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -I${BUCKYBALL_ISA_DIR}/${EMBENCH_CORE} - -ffunction-sections -fdata-sections - -Wl,--gc-sections - -I${EMBENCH_SUPPORT_DIR} + -g + -fno-common + -O2 + -static + -march=rv64gc + -mcmodel=medany + -fno-builtin-printf + -specs=nano.specs + -specs=nosys.specs + -nostartfiles + -Wl,-T,${BBSIM_LD} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_ISA_DIR}/control + -ffunction-sections + -fdata-sections + -Wl,--gc-sections + -I${EMBENCH_SUPPORT_DIR} ) set(EMBENCH_TOP_OBJ_FLAGS - -g -fno-common -O2 -march=rv64gc -mcmodel=medany - -fno-builtin-printf - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -I${BUCKYBALL_ISA_DIR}/${EMBENCH_CORE} - -ffunction-sections -fdata-sections - -I${EMBENCH_SUPPORT_DIR} + -g + -fno-common + -O2 + -march=rv64gc + -mcmodel=medany + -fno-builtin-printf + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_ISA_DIR}/control + -ffunction-sections + -fdata-sections + -I${EMBENCH_SUPPORT_DIR} ) function(add_embench_target BMARK_NAME) - file(GLOB BMARK_SOURCES "${EMBENCH_SRC_DIR}/${BMARK_NAME}/*.c") - buckyball_ctest_name(EMBENCH_NAME ${EMBENCH_CORE} - benchmark-embench_${BMARK_NAME}) - set(EXECUTABLE "${EMBENCH_NAME}-baremetal") - - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMAND ${ELF_CC} ${EMBENCH_C_FLAGS} -o ${EXECUTABLE} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - ${EMBENCH_SUPPORT_SOURCES} - ${BMARK_SOURCES} - -lm - DEPENDS ${BMARK_SOURCES} ${EMBENCH_SUPPORT_SOURCES} - ${BBSIM_LD} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - COMMENT "Building embench: ${EXECUTABLE}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - - add_custom_target(embench_${BMARK_NAME} - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMENT "Building embench ${BMARK_NAME}") - - if(BMARK_NAME STREQUAL "matmult-int" OR BMARK_NAME STREQUAL "sglib-combined") - buckyball_ctest_name(CHECK_NAME ${EMBENCH_CORE} check-embench_${BMARK_NAME}) - set(CHECK_EXECUTABLE ${CHECK_NAME}-baremetal) - set(CHECK_SUPPORT_SOURCES ${EMBENCH_SUPPORT_SOURCES}) - list(REMOVE_ITEM CHECK_SUPPORT_SOURCES ${EMBENCH_SUPPORT_DIR}/main.c) + file(GLOB BMARK_SOURCES "${EMBENCH_SRC_DIR}/${BMARK_NAME}/*.c") + set(EMBENCH_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${EMBENCH_THREAD}-ctest-benchmark-embench_${BMARK_NAME}" + ) + set(EXECUTABLE "${EMBENCH_NAME}-baremetal") + add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${CHECK_EXECUTABLE} - COMMAND ${ELF_CC} ${EMBENCH_C_FLAGS} -o ${CHECK_EXECUTABLE} - ${BBSW_BAREMETAL_DIR}/crt0.S ${BBSW_BAREMETAL_DIR}/syscalls.c - ${EMBENCH_SUPPORT_DIR}/check.c ${CHECK_SUPPORT_SOURCES} ${BMARK_SOURCES} -lm - DEPENDS ${EMBENCH_SUPPORT_DIR}/check.c ${CHECK_SUPPORT_SOURCES} ${BMARK_SOURCES} - ${BBSW_BAREMETAL_DIR}/crt0.S ${BBSW_BAREMETAL_DIR}/syscalls.c ${BBSIM_LD} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - add_custom_target(embench_check_${BMARK_NAME} - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${CHECK_EXECUTABLE}) - add_dependencies(embench_${BMARK_NAME} embench_check_${BMARK_NAME}) - endif() + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + COMMAND + ${ELF_CC} ${EMBENCH_C_FLAGS} -o ${EXECUTABLE} + ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c + ${EMBENCH_SUPPORT_SOURCES} ${BMARK_SOURCES} -lm + DEPENDS + ${BMARK_SOURCES} + ${EMBENCH_SUPPORT_SOURCES} + ${BBSIM_LD} + ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c + COMMENT "Building embench: ${EXECUTABLE}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + add_custom_target( + embench_${BMARK_NAME} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + COMMENT "Building embench ${BMARK_NAME}" + ) + + if( + BMARK_NAME STREQUAL "matmult-int" + OR BMARK_NAME STREQUAL "sglib-combined" + ) + set(CHECK_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${EMBENCH_THREAD}-ctest-check-embench_${BMARK_NAME}" + ) + set(CHECK_EXECUTABLE ${CHECK_NAME}-baremetal) + set(CHECK_SUPPORT_SOURCES ${EMBENCH_SUPPORT_SOURCES}) + list(REMOVE_ITEM CHECK_SUPPORT_SOURCES ${EMBENCH_SUPPORT_DIR}/main.c) + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${CHECK_EXECUTABLE} + COMMAND + ${ELF_CC} ${EMBENCH_C_FLAGS} -o ${CHECK_EXECUTABLE} + ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c + ${EMBENCH_SUPPORT_DIR}/check.c ${CHECK_SUPPORT_SOURCES} + ${BMARK_SOURCES} -lm + DEPENDS + ${EMBENCH_SUPPORT_DIR}/check.c + ${CHECK_SUPPORT_SOURCES} + ${BMARK_SOURCES} + ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c + ${BBSIM_LD} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + add_custom_target( + embench_check_${BMARK_NAME} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${CHECK_EXECUTABLE} + ) + add_dependencies(embench_${BMARK_NAME} embench_check_${BMARK_NAME}) + endif() endfunction() macro(add_embench_top_object BMARK_NAME PREFIX SOURCE_FILES) - set(OBJ "${CMAKE_CURRENT_BINARY_DIR}/${PREFIX}.top.o") - - add_custom_command( - OUTPUT ${OBJ} - COMMAND ${ELF_CC} -r ${EMBENCH_TOP_OBJ_FLAGS} - -Dbenchmark=${PREFIX}_benchmark - -Dinitialise_benchmark=${PREFIX}_initialise_benchmark - -Dwarm_caches=${PREFIX}_warm_caches - -Dverify_benchmark=${PREFIX}_verify_benchmark - -o ${OBJ} - ${SOURCE_FILES} - COMMAND ${ELF_OBJCOPY} - --keep-global-symbol=${PREFIX}_benchmark - --keep-global-symbol=${PREFIX}_initialise_benchmark - --keep-global-symbol=${PREFIX}_warm_caches - --keep-global-symbol=${PREFIX}_verify_benchmark - ${OBJ} - DEPENDS ${SOURCE_FILES} - COMMENT "Building embench top object: ${BMARK_NAME}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - - list(APPEND EMBENCH_TOP_OBJECTS ${OBJ}) + set(OBJ "${CMAKE_CURRENT_BINARY_DIR}/${PREFIX}.top.o") + + add_custom_command( + OUTPUT ${OBJ} + COMMAND + ${ELF_CC} -r ${EMBENCH_TOP_OBJ_FLAGS} + -Dbenchmark=${PREFIX}_benchmark + -Dinitialise_benchmark=${PREFIX}_initialise_benchmark + -Dwarm_caches=${PREFIX}_warm_caches + -Dverify_benchmark=${PREFIX}_verify_benchmark -o ${OBJ} + ${SOURCE_FILES} + COMMAND + ${ELF_OBJCOPY} --keep-global-symbol=${PREFIX}_benchmark + --keep-global-symbol=${PREFIX}_initialise_benchmark + --keep-global-symbol=${PREFIX}_warm_caches + --keep-global-symbol=${PREFIX}_verify_benchmark ${OBJ} + DEPENDS ${SOURCE_FILES} + COMMENT "Building embench top object: ${BMARK_NAME}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + list(APPEND EMBENCH_TOP_OBJECTS ${OBJ}) endmacro() macro(add_embench_top_benchmark BMARK_NAME PREFIX) - file(GLOB BMARK_SOURCES "${EMBENCH_SRC_DIR}/${BMARK_NAME}/*.c") - add_embench_top_object(${BMARK_NAME} ${PREFIX} "${BMARK_SOURCES}") + file(GLOB BMARK_SOURCES "${EMBENCH_SRC_DIR}/${BMARK_NAME}/*.c") + add_embench_top_object(${BMARK_NAME} ${PREFIX} "${BMARK_SOURCES}") endmacro() function(add_embench_top_target) - set(EMBENCH_TOP_OBJECTS) - - add_embench_top_benchmark(aha-mont64 aha_mont64) - add_embench_top_benchmark(crc32 crc32) - add_embench_top_benchmark(edn edn) - add_embench_top_benchmark(huffbench huffbench) - add_embench_top_benchmark(matmult-int matmult_int) - add_embench_top_benchmark(md5sum md5sum) - add_embench_top_benchmark(minver minver) - add_embench_top_benchmark(nbody nbody) - add_embench_top_benchmark(nettle-aes nettle_aes) - add_embench_top_benchmark(nettle-sha256 nettle_sha256) - add_embench_top_benchmark(nsichneu nsichneu) - add_embench_top_benchmark(picojpeg picojpeg) - add_embench_top_benchmark(primecount primecount) - add_embench_top_benchmark(qrduino qrduino) - add_embench_top_benchmark(sglib-combined sglib_combined) - add_embench_top_benchmark(slre slre) - add_embench_top_benchmark(st st) - add_embench_top_benchmark(statemate statemate) - add_embench_top_benchmark(tarfind tarfind) - add_embench_top_benchmark(ud ud) - add_embench_top_benchmark(wikisort wikisort) - - buckyball_ctest_name(EMBENCH_TOP_NAME ${EMBENCH_CORE} benchmark-embench_top) - set(EXECUTABLE "${EMBENCH_TOP_NAME}-baremetal") - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMAND ${ELF_CC} ${EMBENCH_C_FLAGS} -o ${EXECUTABLE} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - ${EMBENCH_TOP_SUPPORT_SOURCES} - ${EMBENCH_TOP_OBJECTS} - ${BBSW_BAREMETAL_DIR}/test.c - -lm - DEPENDS ${EMBENCH_TOP_SUPPORT_SOURCES} - ${BBSIM_LD} - ${EMBENCH_TOP_OBJECTS} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - ${BBSW_BAREMETAL_DIR}/test.c - COMMENT "Building embench top: ${EXECUTABLE}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - - add_custom_target(embench_top DEPENDS - ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMENT "Building embench top workload for Buckyball" - ) + set(EMBENCH_TOP_OBJECTS) + + add_embench_top_benchmark(aha-mont64 aha_mont64) + add_embench_top_benchmark(crc32 crc32) + add_embench_top_benchmark(edn edn) + add_embench_top_benchmark(huffbench huffbench) + add_embench_top_benchmark(matmult-int matmult_int) + add_embench_top_benchmark(md5sum md5sum) + add_embench_top_benchmark(minver minver) + add_embench_top_benchmark(nbody nbody) + add_embench_top_benchmark(nettle-aes nettle_aes) + add_embench_top_benchmark(nettle-sha256 nettle_sha256) + add_embench_top_benchmark(nsichneu nsichneu) + add_embench_top_benchmark(picojpeg picojpeg) + add_embench_top_benchmark(primecount primecount) + add_embench_top_benchmark(qrduino qrduino) + add_embench_top_benchmark(sglib-combined sglib_combined) + add_embench_top_benchmark(slre slre) + add_embench_top_benchmark(st st) + add_embench_top_benchmark(statemate statemate) + add_embench_top_benchmark(tarfind tarfind) + add_embench_top_benchmark(ud ud) + add_embench_top_benchmark(wikisort wikisort) + + set(EMBENCH_TOP_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${EMBENCH_THREAD}-ctest-benchmark-embench_top" + ) + set(EXECUTABLE "${EMBENCH_TOP_NAME}-baremetal") + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + COMMAND + ${ELF_CC} ${EMBENCH_C_FLAGS} -o ${EXECUTABLE} + ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c + ${EMBENCH_TOP_SUPPORT_SOURCES} ${EMBENCH_TOP_OBJECTS} -lm + DEPENDS + ${EMBENCH_TOP_SUPPORT_SOURCES} + ${BBSIM_LD} + ${EMBENCH_TOP_OBJECTS} + ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c + COMMENT "Building embench top: ${EXECUTABLE}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + add_custom_target( + embench_top + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + COMMENT "Building embench top workload for Buckyball" + ) endfunction() #------------------------------------------------------------------------------- @@ -201,14 +232,31 @@ add_embench_target(wikisort) add_embench_top_target() # Master embench build target (excludes cubic due to long double link issue) -add_custom_target(embench-all DEPENDS - embench_aha-mont64 embench_crc32 embench_edn - embench_huffbench embench_matmult-int embench_md5sum embench_minver - embench_nbody - embench_nettle-aes embench_nettle-sha256 embench_nsichneu - embench_picojpeg embench_primecount embench_qrduino embench_sglib-combined - embench_slre embench_st embench_statemate embench_tarfind embench_ud - embench_wikisort - embench_top - COMMENT "Building all embench benchmarks for Buckyball (21/22, cubic excluded)" +add_custom_target( + embench-all + DEPENDS + embench_aha-mont64 + embench_crc32 + embench_edn + embench_huffbench + embench_matmult-int + embench_md5sum + embench_minver + embench_nbody + embench_nettle-aes + embench_nettle-sha256 + embench_nsichneu + embench_picojpeg + embench_primecount + embench_qrduino + embench_sglib-combined + embench_slre + embench_st + embench_statemate + embench_tarfind + embench_ud + embench_wikisort + embench_top + COMMENT + "Building all embench benchmarks for Buckyball (21/22, cubic excluded)" ) diff --git a/bb-tests/workloads/src/CTest/embench/support/chipsupport.c b/bb-tests/workloads/src/CTest/embench/support/chipsupport.c index 5c229a8d0..e0f3de0fe 100644 --- a/bb-tests/workloads/src/CTest/embench/support/chipsupport.c +++ b/bb-tests/workloads/src/CTest/embench/support/chipsupport.c @@ -1,4 +1,4 @@ -/* Buckyball chip support implementation for Embench +/* Buckyball design support implementation for Embench Adapted for Buckyball baremetal environment. @@ -6,11 +6,11 @@ #include "chipsupport.h" -/* Initialize chip +/* Initialize design - For Buckyball, no chip-specific initialization is needed. + For Buckyball, no design-specific initialization is needed. All necessary setup is done by crt0.S/start.S. */ -void initialise_chip(void) { - /* Empty - no chip-specific initialization needed */ +void initialise_design(void) { + /* Empty - no design-specific initialization needed */ } diff --git a/bb-tests/workloads/src/CTest/embench/support/chipsupport.h b/bb-tests/workloads/src/CTest/embench/support/chipsupport.h index 8c26ec4b4..d62d69ed3 100644 --- a/bb-tests/workloads/src/CTest/embench/support/chipsupport.h +++ b/bb-tests/workloads/src/CTest/embench/support/chipsupport.h @@ -1,4 +1,4 @@ -/* Buckyball chip support header for Embench +/* Buckyball design support header for Embench Adapted for Buckyball baremetal environment. @@ -7,7 +7,7 @@ #ifndef CHIPSUPPORT_H #define CHIPSUPPORT_H -/* Chip initialization function */ -void initialise_chip(void); +/* Design initialization function */ +void initialise_design(void); #endif /* CHIPSUPPORT_H */ diff --git a/bb-tests/workloads/src/CTest/rvv/rvv_kernel.c b/bb-tests/workloads/src/CTest/rvv/rvv_kernel.c index a54b6632a..05fc9bab2 100644 --- a/bb-tests/workloads/src/CTest/rvv/rvv_kernel.c +++ b/bb-tests/workloads/src/CTest/rvv/rvv_kernel.c @@ -1,4 +1,5 @@ #include +#include #include #include diff --git a/bb-tests/workloads/src/CTest/tlb/CMakeLists.txt b/bb-tests/workloads/src/CTest/tlb/CMakeLists.txt index b6088f0aa..ead8a138b 100644 --- a/bb-tests/workloads/src/CTest/tlb/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/tlb/CMakeLists.txt @@ -1,14 +1,12 @@ add_custom_target(tlb-ctests-build) set(BUCKYBALL_CTEST_GROUP_TARGET tlb-ctests-build) -foreach(BUCKYBALL_CTEST_TARGET IN LISTS BUCKYBALL_CPU_TARGETS) - add_buckyball_ctests( - tlb_bare_mode_test.c - tlb_repeated_sfence_test.c - tlb_sfence_flush_test.c - tlb_sfence_remap_test.c - tlb_sv39_4k_alias_test.c - tlb_sv39_identity_test.c - tlb_sv39_nonidentity_test.c - ) +foreach(BUCKYBALL_CTEST_TARGET IN LISTS DESIGN_CPU_TARGETS) + add_buckyball_ctest(tlb_bare_mode_test.c) + add_buckyball_ctest(tlb_repeated_sfence_test.c) + add_buckyball_ctest(tlb_sfence_flush_test.c) + add_buckyball_ctest(tlb_sfence_remap_test.c) + add_buckyball_ctest(tlb_sv39_4k_alias_test.c) + add_buckyball_ctest(tlb_sv39_identity_test.c) + add_buckyball_ctest(tlb_sv39_nonidentity_test.c) endforeach() diff --git a/bb-tests/workloads/src/CTest/tlb/tlb_common.h b/bb-tests/workloads/src/CTest/tlb/tlb_common.h index a11b6c2c1..0ec971df8 100644 --- a/bb-tests/workloads/src/CTest/tlb/tlb_common.h +++ b/bb-tests/workloads/src/CTest/tlb/tlb_common.h @@ -28,6 +28,9 @@ #define TLB_SATP_MODE_BARE 0UL #define TLB_PMPCFG_NAPOT 0x18UL #define TLB_PMPCFG_RWX 0x07UL +#define TLB_MSTATUS_MPRV (1UL << 17) +#define TLB_MSTATUS_MPP (3UL << 11) +#define TLB_MSTATUS_MPP_S (1UL << 11) #define TLB_DIM 16 @@ -116,10 +119,15 @@ static void tlb_enable_sv39(void) { tlb_fence_rw(); asm volatile("csrw satp, %0" ::"r"(satp_val) : "memory"); asm volatile("sfence.vma" ::: "memory"); + uintptr_t status; + asm volatile("csrr %0, mstatus" : "=r"(status)); + status = (status & ~TLB_MSTATUS_MPP) | TLB_MSTATUS_MPRV | TLB_MSTATUS_MPP_S; + asm volatile("csrw mstatus, %0" ::"r"(status) : "memory"); tlb_fence_rw(); } static void tlb_disable_vm(void) { + asm volatile("csrc mstatus, %0" ::"r"(TLB_MSTATUS_MPRV) : "memory"); uint64_t satp_val = TLB_SATP_MODE_BARE << 60; tlb_fence_rw(); asm volatile("csrw satp, %0" ::"r"(satp_val) : "memory"); @@ -160,8 +168,8 @@ static int tlb_finish(const char *name, int passed) { } static void tlb_select_hart(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif } diff --git a/bb-tests/workloads/src/CTest/tutorial/CMakeLists.txt b/bb-tests/workloads/src/CTest/tutorial/CMakeLists.txt index 9f2607c76..3b59bb9c1 100644 --- a/bb-tests/workloads/src/CTest/tutorial/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/tutorial/CMakeLists.txt @@ -1,14 +1,18 @@ add_custom_target(tutorial-ctests-build) set(BUCKYBALL_CTEST_GROUP_TARGET tutorial-ctests-build) -# Baremetal single-core only (matches Bazel //bb-tests/.../CTest/tutorial). -foreach(BUCKYBALL_CTEST_TARGET IN LISTS BUCKYBALL_CPU_TARGETS) - buckyball_ctest_name(BUCKYBALL_TUTORIAL_TEST - ${BUCKYBALL_CTEST_TARGET} tutorial) - add_buckyball_baremetal_ctest( - ${BUCKYBALL_TUTORIAL_TEST} - ${CMAKE_CURRENT_SOURCE_DIR} - tutorial.c) - add_dependencies(tutorial-ctests-build - ${BUCKYBALL_TUTORIAL_TEST}-baremetal-build) +# Runs on the selected bare-metal hart. +foreach(BUCKYBALL_CTEST_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_TUTORIAL_TEST + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-tutorial" + ) + add_buckyball_baremetal_ctest( + ${BUCKYBALL_TUTORIAL_TEST} + ${CMAKE_CURRENT_SOURCE_DIR} + tutorial.c + ) + add_dependencies( + tutorial-ctests-build + ${BUCKYBALL_TUTORIAL_TEST}-baremetal-build + ) endforeach() diff --git a/bb-tests/workloads/src/MLIRTest/CMakeLists.txt b/bb-tests/workloads/src/MLIRTest/CMakeLists.txt index 2f4424e0f..e7e7edeab 100644 --- a/bb-tests/workloads/src/MLIRTest/CMakeLists.txt +++ b/bb-tests/workloads/src/MLIRTest/CMakeLists.txt @@ -1,315 +1,268 @@ -include(${CMAKE_CURRENT_LIST_DIR}/compiler/CMakeLists.txt) -set(MLIRTEST_WORKLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - -# The workload root chooses the compiler instance for this chip's core. -if(NOT DEFINED BUDDY_BINARY_DIR) - message(FATAL_ERROR - "BUDDY_BINARY_DIR must be set by the workload root.") -endif() - -if(NOT DEFINED LLVM_BUILD_DIR) - message(FATAL_ERROR "LLVM_BUILD_DIR is not set") -endif() - -if(NOT DEFINED BUDDY_OPT) - message(FATAL_ERROR "BUDDY_OPT is not set") -endif() - # Bank-level / ball-op MLIR -> baremetal + linux ELF (same pipeline as toy MLIRTest). function(buckyball_enforce_mlir_test_line_limit SOURCE_FILE) - if(NOT EXISTS "${SOURCE_FILE}") - message(FATAL_ERROR "MLIR test source does not exist: ${SOURCE_FILE}") - endif() - file(READ "${SOURCE_FILE}" _content) - string(REGEX MATCHALL "\n" _newlines "${_content}") - list(LENGTH _newlines _lines) - string(REGEX MATCH "[^\n]$" _trailing_content "${_content}") - if(_trailing_content) - math(EXPR _lines "${_lines} + 1") - endif() - if(_lines GREATER 100) - message(FATAL_ERROR - "MLIR test exceeds 100 lines: ${SOURCE_FILE} (${_lines} lines). " - "Split it into one focused test per file.") - endif() -endfunction() - -function(buckyball_require_smatmul_bank_shape TARGET) - set(_params "${BUCKYBALL_ISA_DIR}/${TARGET}/params.h") - file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") - file(STRINGS "${_params}" _bank_width REGEX "^#define BANK_WIDTH ") - string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") - string(REGEX REPLACE ".* " "" _width "${_bank_width}") - math(EXPR _line_remainder "${_lines} % 64") - if(NOT (_width EQUAL 128) OR _lines LESS 64 OR - NOT (_line_remainder EQUAL 0)) - message(FATAL_ERROR - "SMatMul BankOp test requires BANK_WIDTH=128 and BANK_LINES a multiple " - "of 64 (got width=${_width}, lines=${_lines})") - endif() -endfunction() - -function(buckyball_require_quant_i32_bank_shape TARGET) - set(_params "${BUCKYBALL_ISA_DIR}/${TARGET}/params.h") - file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") - string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") - math(EXPR _remainder "${_lines} % 4") - if(_lines LESS 4 OR NOT (_remainder EQUAL 0)) - message(FATAL_ERROR - "ToInt8 INT32 BankOp test requires BANK_LINES divisible by four " - "(got lines=${_lines})") - endif() -endfunction() - -function(buckyball_require_mxfp2int_bank_shape TARGET) - set(_params "${BUCKYBALL_ISA_DIR}/${TARGET}/params.h") - file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") - string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") - math(EXPR _remainder "${_lines} % 32") - if(_lines LESS 32 OR NOT (_remainder EQUAL 0)) - message(FATAL_ERROR - "Mxfp2Int BankOp test requires BANK_LINES divisible by 32 " - "(got lines=${_lines})") - endif() -endfunction() - -function(buckyball_require_min_bank_lines TARGET MINIMUM BALL) - set(_params "${BUCKYBALL_ISA_DIR}/${TARGET}/params.h") - file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") - string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") - if(_lines LESS MINIMUM) - message(FATAL_ERROR - "${BALL} BankOp test requires BANK_LINES >= ${MINIMUM} (got ${_lines})") - endif() + if(NOT EXISTS "${SOURCE_FILE}") + message(FATAL_ERROR "MLIR test source does not exist: ${SOURCE_FILE}") + endif() + file(READ "${SOURCE_FILE}" _content) + string(REGEX MATCHALL "\n" _newlines "${_content}") + list(LENGTH _newlines _lines) + string(REGEX MATCH "[^\n]$" _trailing_content "${_content}") + if(_trailing_content) + math(EXPR _lines "${_lines} + 1") + endif() + if(_lines GREATER 100) + message( + FATAL_ERROR + "MLIR test exceeds 100 lines: ${SOURCE_FILE} (${_lines} lines). " + "Split it into one focused test per file." + ) + endif() endfunction() function(add_buckyball_mlir_test NAME) - cmake_parse_arguments(ARG "" "TARGET;CRUNNER_SOURCE" "PASSES;LIBRARIES;BAREMETAL_LIBRARIES" ${ARGN}) + cmake_parse_arguments( + ARG + "" + "TARGET" + "PASSES;LIBRARIES;BAREMETAL_LIBRARIES" + ${ARGN} + ) - if(NOT DEFINED BUCKYBALL_MLIR_TEST_PREFIX) - message(FATAL_ERROR "BUCKYBALL_MLIR_TEST_PREFIX is not set") - endif() - if(BUCKYBALL_WORKLOAD_CHIP STREQUAL "") - message(FATAL_ERROR "BUCKYBALL_WORKLOAD_CHIP must be set before mlir tests") - endif() - if(NOT ARG_TARGET) - message(FATAL_ERROR "${NAME}: TARGET is required") - endif() - list(FIND BUCKYBALL_TARGETS "${ARG_TARGET}" TARGET_INDEX) - if(TARGET_INDEX EQUAL -1) - message(FATAL_ERROR "${NAME}: unknown Buckyball target ${ARG_TARGET}") - endif() - buckyball_assign_physical_banks(${ARG_TARGET} ASSIGN_PHYSICAL_BANKS) - set(BB_BALL_ISA_INCLUDE_DIR "${BUCKYBALL_ISA_DIR}/${ARG_TARGET}") - set(BB_BALL_ISA_H "${BB_BALL_ISA_INCLUDE_DIR}/ballISA.h") - set(BB_PARAMS_H "${BB_BALL_ISA_INCLUDE_DIR}/params.h") - if(NOT EXISTS "${BB_BALL_ISA_H}") - message(FATAL_ERROR "${NAME}: missing target ISA header ${BB_BALL_ISA_H}") - endif() - if(NOT EXISTS "${BB_PARAMS_H}") - message(FATAL_ERROR "${NAME}: missing target params.h ${BB_PARAMS_H}") - endif() - set(MLIR_SRC ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}.mlir) - set(MLIR_TEMPLATE ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}.mlir.in) - if(EXISTS ${MLIR_TEMPLATE}) - file(STRINGS ${BB_PARAMS_H} _bank_lines REGEX "^#define BANK_LINES ") - file(STRINGS ${BB_PARAMS_H} _bank_width REGEX "^#define BANK_WIDTH ") - file(STRINGS ${BB_PARAMS_H} _bank_isqrt REGEX "^#define BANK_ISQRT ") - string(REGEX REPLACE ".* " "" BB_BANK_LINES "${_bank_lines}") - string(REGEX REPLACE ".* " "" BB_BANK_WIDTH_BITS "${_bank_width}") - string(REGEX REPLACE ".* " "" BB_BANK_ISQRT "${_bank_isqrt}") - math(EXPR BB_BANK_I8_LANES "${BB_BANK_WIDTH_BITS} / 8") - math(EXPR BB_BANK_I32_LANES "${BB_BANK_WIDTH_BITS} / 32") - math(EXPR BB_BANK_SQUARE "${BB_BANK_ISQRT} * ${BB_BANK_ISQRT}") - set(BB_IM2COL_SIDE ${BB_BANK_ISQRT}) - if(BB_IM2COL_SIDE GREATER 16) - set(BB_IM2COL_SIDE 16) + if(NOT DEFINED BUCKYBALL_MLIR_TEST_PREFIX) + message(FATAL_ERROR "BUCKYBALL_MLIR_TEST_PREFIX is not set") endif() - math(EXPR BB_IM2COL_INPUT_ROWS "${BB_IM2COL_SIDE} * ${BB_IM2COL_SIDE}") - math(EXPR BB_SMM_M "${BB_BANK_LINES} / 4") - math(EXPR BB_SMM_CONFIG "${BB_SMM_M} | (16 << 12) | (16 << 24)") - # QUANT_I32_TO_I8 has a seven-bit outputHeight field, so one instruction - # can consume at most 508 packed INT32 bank rows. - math(EXPR BB_QUANT_I32_INPUT_ROWS "${BB_BANK_LINES} - (${BB_BANK_LINES} % 4)") - if(BB_QUANT_I32_INPUT_ROWS GREATER 508) - set(BB_QUANT_I32_INPUT_ROWS 508) + if(BUCKYBALL_WORKLOAD_DESIGN STREQUAL "") + message( + FATAL_ERROR + "BUCKYBALL_WORKLOAD_DESIGN must be set before mlir tests" + ) endif() - math(EXPR BB_QUANT_I32_OUTPUT_ROWS "${BB_QUANT_I32_INPUT_ROWS} / 4") - math(EXPR BB_MXFP_BLOCKS "${BB_BANK_LINES} / 2") - math(EXPR BB_MXFP_MMIO_ROWS "${BB_MXFP_BLOCKS} / 16") - if(BB_BANK_LINES LESS 1 OR BB_BANK_ISQRT LESS 1 OR BB_BANK_I8_LANES LESS 1 OR - BB_BANK_I32_LANES LESS 1) - message(FATAL_ERROR "${NAME}: invalid generated bank parameters") + if(NOT ARG_TARGET) + message(FATAL_ERROR "${NAME}: TARGET is required") endif() - set(MLIR_SRC ${CMAKE_CURRENT_BINARY_DIR}/${ARG_TARGET}-${NAME}.mlir) - configure_file(${MLIR_TEMPLATE} ${MLIR_SRC} @ONLY) - endif() - set(MAIN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}_main.cpp) - buckyball_enforce_mlir_test_line_limit(${MLIR_SRC}) - buckyball_enforce_mlir_test_line_limit(${MAIN_SRC}) - set(OBJ ${CMAKE_CURRENT_BINARY_DIR}/${ARG_TARGET}-${NAME}.o) - set(PREFIX ${BUCKYBALL_MLIR_TEST_PREFIX}) - set(TEST_CASE ${NAME}) - if(NAME MATCHES "^${PREFIX}_(.+)$") - set(TEST_CASE ${CMAKE_MATCH_1}) - endif() - buckyball_mlirtest_name(TEST_NAME ${ARG_TARGET} ${PREFIX}_${TEST_CASE}) - set(BAREMETAL_BIN ${TEST_NAME}-baremetal) - set(LINUX_BIN ${TEST_NAME}-linux) - set(BAREMETAL_TARGET ${BAREMETAL_BIN}-build) - set(LINUX_TARGET ${LINUX_BIN}-build) - set(GROUP_TARGET ${TEST_NAME}) - - set(BBSIM_LD ${BBSW_BAREMETAL_DIR}/bbsim.ld) - set(C_FLAGS -g -fno-common -O2 -static -march=rv64gc -mcmodel=medany - -fno-builtin-printf -specs=nano.specs -specs=nosys.specs -nostartfiles - -DBAREMETAL -Wl,-T,${BBSIM_LD}) - set(LINUX_CXX ${RISCV_GNU_TOOLCHAIN}/bin/riscv64-unknown-linux-gnu-g++) - set(LINUX_FLAGS -static -Wl,--no-dynamic-linker -march=rv64gc) - set(CRUNNER_UTILS_SRC ${WORKLOAD_LIB_DIR}/bbsw/CRunnerUtils/CRunnerUtils.cpp) - if(ARG_CRUNNER_SOURCE) - set(CRUNNER_UTILS_SRC "${ARG_CRUNNER_SOURCE}") - endif() - set(BBHW_MEM_C ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/dma.c) - set(LLVM_MLIR_EXECUTION_ENGINE_DIR - ${BUDDY_MLIR_DIR}/llvm/mlir/include/mlir/ExecutionEngine) - - if(ARG_PASSES) - set(MLIR_PASSES) - foreach(PASS IN LISTS ARG_PASSES) - if(PASS STREQUAL "-assign-physical-banks") - list(APPEND MLIR_PASSES ${ASSIGN_PHYSICAL_BANKS}) - else() - list(APPEND MLIR_PASSES ${PASS}) - endif() - endforeach() - else() - set(MLIR_PASSES - ${ASSIGN_PHYSICAL_BANKS} - ${BUCKYBALL_LOWER_BANK_SSA_TO_INTRINSICS} - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - endif() - string(JOIN " " MLIR_PASS_COMMAND ${MLIR_PASSES}) + list(FIND DESIGN_TARGETS "${ARG_TARGET}" TARGET_INDEX) + if(TARGET_INDEX EQUAL -1) + message(FATAL_ERROR "${NAME}: unknown Buckyball target ${ARG_TARGET}") + endif() + if(NOT ARG_TARGET IN_LIST DESIGN_BANK_TARGETS) + message(FATAL_ERROR "${ARG_TARGET}: no physical banks") + endif() + set(ASSIGN_PHYSICAL_BANKS -assign-physical-banks) + set(BIN "${DESIGN_COMPILER}/bin") + # buddy-opt with this target's compiler plugins. + string(JOIN " " OPT ${DESIGN_TARGET_OPT_${ARG_TARGET}}) + set(BB_BALL_ISA_INCLUDE_DIR "${BUCKYBALL_ISA_DIR}/${ARG_TARGET}") + set(BB_BALL_ISA_H "${BB_BALL_ISA_INCLUDE_DIR}/ballISA.h") + set(BB_PARAMS_H "${BB_BALL_ISA_INCLUDE_DIR}/params.h") + if(NOT EXISTS "${BB_BALL_ISA_H}") + message( + FATAL_ERROR + "${NAME}: missing target ISA header ${BB_BALL_ISA_H}" + ) + endif() + if(NOT EXISTS "${BB_PARAMS_H}") + message(FATAL_ERROR "${NAME}: missing target params.h ${BB_PARAMS_H}") + endif() + set(MLIR_SRC ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}.mlir) + set(MLIR_TEMPLATE ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}.mlir.in) + if(EXISTS ${MLIR_TEMPLATE}) + file(STRINGS ${BB_PARAMS_H} _bank_lines REGEX "^#define BANK_LINES ") + file(STRINGS ${BB_PARAMS_H} _bank_width REGEX "^#define BANK_WIDTH ") + file(STRINGS ${BB_PARAMS_H} _bank_isqrt REGEX "^#define BANK_ISQRT ") + string(REGEX REPLACE ".* " "" BB_BANK_LINES "${_bank_lines}") + string(REGEX REPLACE ".* " "" BB_BANK_WIDTH_BITS "${_bank_width}") + string(REGEX REPLACE ".* " "" BB_BANK_ISQRT "${_bank_isqrt}") + math(EXPR BB_BANK_I8_LANES "${BB_BANK_WIDTH_BITS} / 8") + math(EXPR BB_BANK_I32_LANES "${BB_BANK_WIDTH_BITS} / 32") + if( + BB_BANK_LINES LESS 1 + OR BB_BANK_ISQRT LESS 1 + OR BB_BANK_I8_LANES LESS 1 + OR BB_BANK_I32_LANES LESS 1 + ) + message(FATAL_ERROR "${NAME}: invalid generated bank parameters") + endif() + set(MLIR_SRC ${CMAKE_CURRENT_BINARY_DIR}/${ARG_TARGET}-${NAME}.mlir) + configure_file(${MLIR_TEMPLATE} ${MLIR_SRC} @ONLY) + endif() + set(MAIN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}_main.cpp) + buckyball_enforce_mlir_test_line_limit(${MLIR_SRC}) + buckyball_enforce_mlir_test_line_limit(${MAIN_SRC}) + set(OBJ ${CMAKE_CURRENT_BINARY_DIR}/${ARG_TARGET}-${NAME}.o) + set(PREFIX ${BUCKYBALL_MLIR_TEST_PREFIX}) + set(TEST_CASE ${NAME}) + if(NAME MATCHES "^${PREFIX}_(.+)$") + set(TEST_CASE ${CMAKE_MATCH_1}) + endif() + set(TEST_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${ARG_TARGET}-mlirtest-${PREFIX}_${TEST_CASE}" + ) + set(BAREMETAL_BIN ${TEST_NAME}-baremetal) + set(LINUX_BIN ${TEST_NAME}-linux) + set(BAREMETAL_TARGET ${BAREMETAL_BIN}-build) + set(LINUX_TARGET ${LINUX_BIN}-build) + set(GROUP_TARGET ${TEST_NAME}) - add_custom_command( - OUTPUT ${OBJ} - COMMAND bash -o pipefail -c - "${BUDDY_OPT} ${MLIR_SRC} --target=${ARG_TARGET} ${MLIR_PASS_COMMAND} | ${BUDDY_TRANSLATE} --buddy-to-llvmir | ${BUDDY_LLC} -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=${BUCKYBALL_RISCV_MATTR} -float-abi=hard -o ${OBJ}" - DEPENDS ${MLIR_SRC} ${BUDDY_OPT} - COMMENT "Building ${NAME}.o from ${PREFIX} MLIR" - VERBATIM) + set(C_FLAGS + -g + -fno-common + -O2 + -static + -march=rv64gc + -mcmodel=medany + -fno-builtin-printf + -specs=nano.specs + -specs=nosys.specs + -nostartfiles + -DBAREMETAL + -Wl,-T,${BBSIM_LD} + ) + set(LINUX_FLAGS -static -Wl,--no-dynamic-linker -march=rv64gc) + set(CRUNNER_UTILS_SRC + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/mlir.cpp + ) + set(BBHW_MEM_C ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/dma.c) + set(LLVM_MLIR_EXECUTION_ENGINE_DIR + ${BUDDY_MLIR_DIR}/llvm/mlir/include/mlir/ExecutionEngine + ) - set(MLIR_LINK_LIBRARIES) - foreach(LIBRARY IN LISTS ARG_LIBRARIES) - if(NOT TARGET ${LIBRARY}) - message(FATAL_ERROR "${NAME}: unknown runtime library ${LIBRARY}") + if(ARG_PASSES) + set(MLIR_PASSES ${ARG_PASSES}) + else() + set(MLIR_PASSES + ${ASSIGN_PHYSICAL_BANKS} + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-arith-to-llvm + -convert-math-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts + ) endif() - list(APPEND MLIR_LINK_LIBRARIES "$") - endforeach() + string(JOIN " " MLIR_PASS_COMMAND ${MLIR_PASSES}) - set(MLIR_BAREMETAL_LIBRARIES ${MLIR_LINK_LIBRARIES}) - if(ARG_BAREMETAL_LIBRARIES) - set(MLIR_BAREMETAL_LIBRARIES) - foreach(LIBRARY IN LISTS ARG_BAREMETAL_LIBRARIES) - if(NOT TARGET ${LIBRARY}) - message(FATAL_ERROR "${NAME}: unknown bare-metal library ${LIBRARY}") - endif() - list(APPEND MLIR_BAREMETAL_LIBRARIES "$") + add_custom_command( + OUTPUT ${OBJ} + COMMAND + bash -o pipefail -c + "${OPT} ${MLIR_SRC} ${MLIR_PASS_COMMAND} | ${BIN}/buddy-translate --buddy-to-llvmir | ${BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=${BUCKYBALL_RISCV_MATTR} -float-abi=hard -o ${OBJ}" + DEPENDS ${MLIR_SRC} ${DESIGN_TARGET_PLUGIN_${ARG_TARGET}} + COMMENT "Building ${NAME}.o from ${PREFIX} MLIR" + VERBATIM + ) + + set(MLIR_LINK_LIBRARIES) + foreach(LIBRARY IN LISTS ARG_LIBRARIES) + if(NOT TARGET ${LIBRARY}) + message(FATAL_ERROR "${NAME}: unknown runtime library ${LIBRARY}") + endif() + list(APPEND MLIR_LINK_LIBRARIES "$") endforeach() - endif() - add_custom_command( - OUTPUT ${BAREMETAL_BIN} - COMMAND ${ELF_CC} ${C_FLAGS} - -I${LLVM_MLIR_EXECUTION_ENGINE_DIR} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -I${BB_BALL_ISA_INCLUDE_DIR} - -o ${CMAKE_CURRENT_BINARY_DIR}/${BAREMETAL_BIN} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - ${CRUNNER_UTILS_SRC} - ${BBHW_MEM_C} - ${MAIN_SRC} - ${OBJ} ${MLIR_BAREMETAL_LIBRARIES} - DEPENDS ${OBJ} ${ARG_BAREMETAL_LIBRARIES} ${ARG_LIBRARIES} ${MAIN_SRC} ${CRUNNER_UTILS_SRC} ${BBHW_MEM_C} - ${BB_BALL_ISA_H} ${BB_PARAMS_H} - ${BBSW_BAREMETAL_DIR}/crt0.S ${BBSW_BAREMETAL_DIR}/syscalls.c - ${BBSIM_LD} - COMMENT "Linking ${BAREMETAL_BIN}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - VERBATIM) + set(MLIR_BAREMETAL_LIBRARIES ${MLIR_LINK_LIBRARIES}) + if(ARG_BAREMETAL_LIBRARIES) + set(MLIR_BAREMETAL_LIBRARIES) + foreach(LIBRARY IN LISTS ARG_BAREMETAL_LIBRARIES) + if(NOT TARGET ${LIBRARY}) + message( + FATAL_ERROR + "${NAME}: unknown bare-metal library ${LIBRARY}" + ) + endif() + list(APPEND MLIR_BAREMETAL_LIBRARIES "$") + endforeach() + endif() - add_custom_command( - OUTPUT ${LINUX_BIN} - COMMAND ${LINUX_CXX} ${LINUX_FLAGS} - -I${LLVM_MLIR_EXECUTION_ENGINE_DIR} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -I${BB_BALL_ISA_INCLUDE_DIR} - -o ${CMAKE_CURRENT_BINARY_DIR}/${LINUX_BIN} - ${CRUNNER_UTILS_SRC} - -x c ${BBHW_MEM_C} -x none - ${MAIN_SRC} - ${OBJ} ${MLIR_LINK_LIBRARIES} - DEPENDS ${OBJ} ${ARG_LIBRARIES} ${MAIN_SRC} ${CRUNNER_UTILS_SRC} ${BBHW_MEM_C} ${BB_BALL_ISA_H} + add_custom_command( + OUTPUT ${BAREMETAL_BIN} + COMMAND + ${ELF_CC} ${C_FLAGS} -I${LLVM_MLIR_EXECUTION_ENGINE_DIR} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} -I${BB_BALL_ISA_INCLUDE_DIR} -o + ${CMAKE_CURRENT_BINARY_DIR}/${BAREMETAL_BIN} ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c ${CRUNNER_UTILS_SRC} ${BBHW_MEM_C} + ${MAIN_SRC} ${OBJ} ${MLIR_BAREMETAL_LIBRARIES} + DEPENDS + ${OBJ} + ${ARG_BAREMETAL_LIBRARIES} + ${ARG_LIBRARIES} + ${MAIN_SRC} + ${CRUNNER_UTILS_SRC} + ${BBHW_MEM_C} + ${BB_BALL_ISA_H} ${BB_PARAMS_H} - COMMENT "Linking ${LINUX_BIN}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - VERBATIM) + ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c + ${BBSIM_LD} + COMMENT "Linking ${BAREMETAL_BIN}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM + ) - add_custom_target(${BAREMETAL_TARGET} - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${BAREMETAL_BIN}) - add_custom_target(${LINUX_TARGET} - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LINUX_BIN}) - add_custom_target(${GROUP_TARGET} - DEPENDS ${BAREMETAL_TARGET} ${LINUX_TARGET}) + add_custom_command( + OUTPUT ${LINUX_BIN} + COMMAND + ${LINUX_CXX} ${LINUX_FLAGS} -I${LLVM_MLIR_EXECUTION_ENGINE_DIR} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} -I${BB_BALL_ISA_INCLUDE_DIR} -o + ${CMAKE_CURRENT_BINARY_DIR}/${LINUX_BIN} ${CRUNNER_UTILS_SRC} -x c + ${BBHW_MEM_C} -x none ${MAIN_SRC} ${OBJ} ${MLIR_LINK_LIBRARIES} + DEPENDS + ${OBJ} + ${ARG_LIBRARIES} + ${MAIN_SRC} + ${CRUNNER_UTILS_SRC} + ${BBHW_MEM_C} + ${BB_BALL_ISA_H} + ${BB_PARAMS_H} + COMMENT "Linking ${LINUX_BIN}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM + ) - if(DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - add_dependencies(${BUCKYBALL_MLIR_GROUP_TARGET} ${GROUP_TARGET}) - endif() -endfunction() + add_custom_target( + ${BAREMETAL_TARGET} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${BAREMETAL_BIN} + ) + add_custom_target( + ${LINUX_TARGET} + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${LINUX_BIN} + ) + add_custom_target( + ${GROUP_TARGET} + DEPENDS ${BAREMETAL_TARGET} ${LINUX_TARGET} + ) -add_subdirectory(${MLIRTEST_WORKLOAD_DIR}/base - ${CMAKE_CURRENT_BINARY_DIR}/mlir_tests/base) + if(DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + add_dependencies(${BUCKYBALL_MLIR_GROUP_TARGET} ${GROUP_TARGET}) + endif() +endfunction() add_custom_target(balls-mlir-tests-build) -buckyball_add_ball_workload_subdirs(mlir_tests) - -add_custom_target(MLIRTest-all ALL DEPENDS - base-mlirtest-build - balls-mlir-tests-build -) +add_custom_target(MLIRTest-all ALL) -set(CHIP_MLIR_TEST_DIR - ${BUCKYBALL_REPO_ROOT}/examples/chips/${BUCKYBALL_WORKLOAD_SOURCE_CHIP}/workloads/mlir_tests) -add_custom_target(chip-mlirtests-build) -if(EXISTS ${CHIP_MLIR_TEST_DIR}/CMakeLists.txt) - add_subdirectory(${CHIP_MLIR_TEST_DIR} - ${CMAKE_CURRENT_BINARY_DIR}/CTest/chips/${BUCKYBALL_WORKLOAD_SOURCE_CHIP}/mlir_tests) -endif() +add_custom_target(design-mlirtests-build) -add_custom_target(mlirtest-workloads-build - DEPENDS - MLIRTest-all - chip-mlirtests-build - COMMENT "Building MLIRTest workloads" +add_custom_target( + mlirtest-workloads-build + DEPENDS MLIRTest-all design-mlirtests-build + COMMENT "Building MLIRTest workloads" ) -add_custom_target(sync-mlirtest-bin - COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src - COMMAND rsync -a --delete --exclude=CMakeFiles "--include=*/" - "--include=/mlir_tests/***" "--include=/balls/***" - "--include=/CTest/chips/*/mlir_tests/***" "--exclude=*" - ${CMAKE_CURRENT_BINARY_DIR}/ ${OUTPUT_BIN_DIR}/src/ - DEPENDS mlirtest-workloads-build - COMMENT "Syncing MLIRTest binaries to output directory" +add_custom_target( + sync-mlirtest-bin + COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src + COMMAND + rsync -a --delete --exclude=CMakeFiles "--include=*/" + "--include=*-mlirtest-*-baremetal" "--include=*-mlirtest-*-linux" + "--exclude=*" ${CMAKE_BINARY_DIR}/src/ ${OUTPUT_BIN_DIR}/src/ + DEPENDS mlirtest-workloads-build + COMMENT "Syncing registered MLIRTest binaries" ) diff --git a/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt b/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt index 7fb131aa1..4a96f2676 100644 --- a/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt +++ b/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt @@ -3,69 +3,11 @@ add_custom_target(base-mlirtest-build) set(BUCKYBALL_MLIR_GROUP_TARGET base-mlirtest-build) set(BUCKYBALL_MLIR_TEST_PREFIX base) -foreach(BUCKYBALL_TARGET IN LISTS BUCKYBALL_CPU_TARGETS) - if(NOT BUCKYBALL_TARGET IN_LIST BUCKYBALL_BANK_TARGETS) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + if(NOT BUCKYBALL_TARGET IN_LIST DESIGN_BANK_TARGETS) continue() endif() - add_buckyball_mlir_test(base_memref_copy_dynamic - TARGET ${BUCKYBALL_TARGET} - CRUNNER_SOURCE "${BUCKYBALL_REPO_ROOT}/stack/runtime/src/mlir.cpp" - PASSES - ${BUCKYBALL_LOWER_BUCKYBALL} - -expand-strided-metadata - -convert-scf-to-cf - -convert-cf-to-llvm - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - - add_buckyball_mlir_test(base_mvin_mvout_16x16_i8 - TARGET ${BUCKYBALL_TARGET} - PASSES - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - - add_buckyball_mlir_test(base_mset_transfer - TARGET ${BUCKYBALL_TARGET} - PASSES - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - - add_buckyball_mlir_test(base_dma_group - TARGET ${BUCKYBALL_TARGET} - PASSES - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - - add_buckyball_mlir_test(base_bank_mvin_mvout_16x16_i8 - TARGET ${BUCKYBALL_TARGET}) + add_buckyball_mlir_test(base_memref_copy_dynamic TARGET ${BUCKYBALL_TARGET}) + add_buckyball_mlir_test(base_mvin_mvout_16x16_i8 TARGET ${BUCKYBALL_TARGET}) + add_buckyball_mlir_test(base_bank_mvin_mvout_16x16_i8 TARGET ${BUCKYBALL_TARGET}) endforeach() diff --git a/bb-tests/workloads/src/MLIRTest/base/base_bank_mvin_mvout_16x16_i8.mlir b/bb-tests/workloads/src/MLIRTest/base/base_bank_mvin_mvout_16x16_i8.mlir index 32a9315cc..e1ff7fb5a 100644 --- a/bb-tests/workloads/src/MLIRTest/base/base_bank_mvin_mvout_16x16_i8.mlir +++ b/bb-tests/workloads/src/MLIRTest/base/base_bank_mvin_mvout_16x16_i8.mlir @@ -6,8 +6,6 @@ func.func @main() -> i8 { %c1 = arith.constant 1 : index %n = arith.constant 16 : index %c16 = arith.constant 16 : i32 - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 %input = memref.alloc() alignment = 64 : memref<16x16xi8> %output = memref.alloc() alignment = 64 : memref<16x16xi8> @@ -22,12 +20,10 @@ func.func @main() -> i8 { } } - %bank = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %bank %depth %stride - : memref<16x16xi8> i64 i64 i64 - %stored = buckyball.bank_mvout %output %loaded %depth %stride - : memref<16x16xi8> i64 i64 i64 - buckyball.bank_release %stored : i64 + %bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded = buckyball.bank_mvin %input, %bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + %stored = buckyball.bank_mvout %output, %loaded <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.bank_release %stored func.call @check_result(%output) : (memref<16x16xi8>) -> () memref.dealloc %input : memref<16x16xi8> diff --git a/bb-tests/workloads/src/MLIRTest/base/base_memref_copy_dynamic.mlir b/bb-tests/workloads/src/MLIRTest/base/base_memref_copy_dynamic.mlir new file mode 100644 index 000000000..a3703f84d --- /dev/null +++ b/bb-tests/workloads/src/MLIRTest/base/base_memref_copy_dynamic.mlir @@ -0,0 +1,15 @@ +// The base workload pipeline explicitly runs lower-buckyball before MemRefToLLVM. +// Static strides remain visible to Buckyball's own export patterns. +func.func @copy_unit(%input: memref>, + %output: memref>) + attributes {llvm.emit_c_interface} { + memref.copy %input, %output : memref> to memref> + return +} + +func.func @copy_gap(%input: memref>, + %output: memref>) + attributes {llvm.emit_c_interface} { + memref.copy %input, %output : memref> to memref> + return +} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_memref_copy_dynamic.mlir.in b/bb-tests/workloads/src/MLIRTest/base/base_memref_copy_dynamic.mlir.in deleted file mode 100644 index 06cfcfb7f..000000000 --- a/bb-tests/workloads/src/MLIRTest/base/base_memref_copy_dynamic.mlir.in +++ /dev/null @@ -1,15 +0,0 @@ -// The base workload pipeline explicitly runs lower-buckyball before MemRefToLLVM. -// Target and static strides remain visible to Buckyball's own export patterns. -func.func @copy_unit(%input: memref>, - %output: memref>) - attributes {llvm.emit_c_interface, buckyball.target = "@ARG_TARGET@"} { - memref.copy %input, %output : memref> to memref> - return -} - -func.func @copy_gap(%input: memref>, - %output: memref>) - attributes {llvm.emit_c_interface, buckyball.target = "@ARG_TARGET@"} { - memref.copy %input, %output : memref> to memref> - return -} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_mvin_mvout_16x16_i8.mlir b/bb-tests/workloads/src/MLIRTest/base/base_mvin_mvout_16x16_i8.mlir index 7f3115c68..17ef2a197 100644 --- a/bb-tests/workloads/src/MLIRTest/base/base_mvin_mvout_16x16_i8.mlir +++ b/bb-tests/workloads/src/MLIRTest/base/base_mvin_mvout_16x16_i8.mlir @@ -6,9 +6,6 @@ func.func @main() -> i8 { %c1 = arith.constant 1 : index %n = arith.constant 16 : index %c16 = arith.constant 16 : i32 - %bank = arith.constant 0 : i64 - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 %input = memref.alloc() alignment = 64 : memref<16x16xi8> %output = memref.alloc() alignment = 64 : memref<16x16xi8> @@ -23,10 +20,10 @@ func.func @main() -> i8 { } } - buckyball.mset %bank : i64 - buckyball.mvin %input %bank %depth %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mvout %output %bank %depth %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mset %bank : i64 + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvout %output <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> func.call @check_result(%output) : (memref<16x16xi8>) -> () memref.dealloc %input : memref<16x16xi8> diff --git a/bb-tests/workloads/src/MLIRTest/compiler/CMakeLists.txt b/bb-tests/workloads/src/MLIRTest/compiler/CMakeLists.txt deleted file mode 100644 index a969a34bf..000000000 --- a/bb-tests/workloads/src/MLIRTest/compiler/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -set(BUDDY_MLIR_DIR ${BUCKYBALL_REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir) -set(BUDDY_BINARY_DIR ${BUDDY_MLIR_DIR}/build/${BUCKYBALL_WORKLOAD_CHIP}/bin) -set(BUDDY_OPT ${BUDDY_BINARY_DIR}/buddy-opt) -set(BUDDY_TRANSLATE ${BUDDY_BINARY_DIR}/buddy-translate) -set(BUDDY_LLC ${BUDDY_BINARY_DIR}/buddy-llc) -set(LLVM_BUILD_DIR ${BUDDY_MLIR_DIR}/llvm/build) - -option(BUCKYBALL_STABLE "Use stable LLVM Buckyball extensions" OFF) -if(BUCKYBALL_STABLE) - set(BUCKYBALL_RISCV_MATTR "+xbbpebble,+xbuckyball,+m,+D") - set(BUCKYBALL_LOWER_BUCKYBALL "--lower-buckyball=stable=true") - - set(BUCKYBALL_LOWER_BANK_SSA_TO_INTRINSICS - "--lower-bank-ssa-to-intrinsics=stable=true") - -else() - set(BUCKYBALL_RISCV_MATTR "+xbuckyball,+m,+D") - set(BUCKYBALL_LOWER_BUCKYBALL -lower-buckyball) - - set(BUCKYBALL_LOWER_BANK_SSA_TO_INTRINSICS - -lower-bank-ssa-to-intrinsics) - -endif() -message(STATUS "Buckyball RISC-V mattr: ${BUCKYBALL_RISCV_MATTR}") diff --git a/bb-tests/workloads/src/SocTest/CMakeLists.txt b/bb-tests/workloads/src/SocTest/CMakeLists.txt index cf7be0508..95b1b97a1 100644 --- a/bb-tests/workloads/src/SocTest/CMakeLists.txt +++ b/bb-tests/workloads/src/SocTest/CMakeLists.txt @@ -1,62 +1,65 @@ # =============================================================== -# SoC tests: CPU-visible chip devices (CLINT, PLIC, SCU) exercised on -# the chip's boot hart, independent of any Ball ISA. +# SoC tests: CPU-visible design devices (CLINT, PLIC, SCU) exercised on +# the design's boot hart, independent of any Ball ISA. # =============================================================== set(SOCTEST_MAX_LINES 100) set(BUCKYBALL_SOCTEST_C_FLAGS - -g -fno-common -O2 -static -march=rv64gc -mcmodel=medany - -fno-builtin-printf -specs=nano.specs -specs=nosys.specs -nostartfiles - -Wl,-T,${BBSIM_LD} + -g + -fno-common + -O2 + -static + -march=rv64gc + -mcmodel=medany + -fno-builtin-printf + -specs=nano.specs + -specs=nosys.specs + -nostartfiles + -Wl,-T,${BBSIM_LD} ) -# The target whose generated profile boots hart 0 owns the SoC tests. -unset(BUCKYBALL_SOCTEST_TARGET) -foreach(_target IN LISTS BUCKYBALL_TARGETS) - file(STRINGS "${BUCKYBALL_ISA_DIR}/${_target}/params.h" _boot - REGEX "^#define BB_TEST_HART 0$") - if(_boot) - set(BUCKYBALL_SOCTEST_TARGET ${_target}) - break() - endif() -endforeach() -if(NOT BUCKYBALL_SOCTEST_TARGET) - message(FATAL_ERROR - "SoC tests need a target with BB_TEST_HART 0; targets: ${BUCKYBALL_TARGETS}") -endif() +set(BUCKYBALL_SOCTEST_TARGET ${BUCKYBALL_CONTROL_TARGET}) add_custom_target(soctests-build) function(add_buckyball_soctest SOURCE_FILE) - get_filename_component(_stem ${SOURCE_FILE} NAME_WE) - buckyball_soctest_name(TEST_NAME ${BUCKYBALL_SOCTEST_TARGET} ${_stem}) - set(SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_FILE}) - file(STRINGS ${SOURCE} _lines) - list(LENGTH _lines _n) - if(_n GREATER SOCTEST_MAX_LINES) - message(FATAL_ERROR - "soctest exceeds ${SOCTEST_MAX_LINES} lines: ${SOURCE} (${_n} lines)") - endif() + get_filename_component(_stem ${SOURCE_FILE} NAME_WE) + set(TEST_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_SOCTEST_TARGET}-soctest-${_stem}" + ) + set(SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_FILE}) + file(STRINGS ${SOURCE} _lines) + list(LENGTH _lines _n) + if(_n GREATER SOCTEST_MAX_LINES) + message( + FATAL_ERROR + "soctest exceeds ${SOCTEST_MAX_LINES} lines: ${SOURCE} (${_n} lines)" + ) + endif() - set(EXECUTABLE "${TEST_NAME}-baremetal") - set(PARAMS ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_SOCTEST_TARGET}/params.h) - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} - COMMAND ${ELF_CC} ${BUCKYBALL_SOCTEST_C_FLAGS} - -o ${EXECUTABLE} - ${BBSW_BAREMETAL_DIR}/crt0.S - ${BBSW_BAREMETAL_DIR}/syscalls.c - ${SOURCE} - -I${BUCKYBALL_ISA_DIR}/chip - -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_SOCTEST_TARGET} - -I${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS ${SOURCE} ${PARAMS} ${BBSW_BAREMETAL_DIR}/crt0.S ${BBSIM_LD} - ${BBSW_BAREMETAL_DIR}/syscalls.c ${CMAKE_CURRENT_SOURCE_DIR}/soc.h - COMMENT "Building SoC test: ${EXECUTABLE}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - add_custom_target(${TEST_NAME}-baremetal-build - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE}) - add_dependencies(soctests-build ${TEST_NAME}-baremetal-build) + set(EXECUTABLE "${TEST_NAME}-baremetal") + set(PARAMS ${BUCKYBALL_ISA_DIR}/control/params.h) + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + COMMAND + ${ELF_CC} ${BUCKYBALL_SOCTEST_C_FLAGS} -o ${EXECUTABLE} + ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c ${SOURCE} + -I${BUCKYBALL_ISA_DIR}/control -I${BUCKYBALL_ISA_DIR}/design + -I${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS + ${SOURCE} + ${PARAMS} + ${BAREMETAL_DIR}/crt0.S + ${BBSIM_LD} + ${BAREMETAL_DIR}/syscalls.c + ${CMAKE_CURRENT_SOURCE_DIR}/soc.h + COMMENT "Building SoC test: ${EXECUTABLE}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + add_custom_target( + ${TEST_NAME}-baremetal-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} + ) + add_dependencies(soctests-build ${TEST_NAME}-baremetal-build) endfunction() add_buckyball_soctest(clint_timer_interrupt.c) @@ -68,10 +71,13 @@ add_buckyball_soctest(lrsc_word.c) add_buckyball_soctest(lrsc_translated.c) add_buckyball_soctest(plic_register_access.c) -add_custom_target(sync-soctest-bin - COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src/SocTest - COMMAND rsync -a --delete --exclude=CMakeFiles "--include=*/" - "--include=*-baremetal" "--include=*-baremetal.expect" "--exclude=*" - ${CMAKE_CURRENT_BINARY_DIR}/ ${OUTPUT_BIN_DIR}/src/SocTest/ - DEPENDS soctests-build - COMMENT "Syncing SoC test binaries to output directory") +add_custom_target( + sync-soctest-bin + COMMAND ${CMAKE_COMMAND} -E make_directory ${OUTPUT_BIN_DIR}/src/SocTest + COMMAND + rsync -a --delete --exclude=CMakeFiles "--include=*/" + "--include=*-baremetal" "--include=*-baremetal.expect" "--exclude=*" + ${CMAKE_CURRENT_BINARY_DIR}/ ${OUTPUT_BIN_DIR}/src/SocTest/ + DEPENDS soctests-build + COMMENT "Syncing SoC test binaries to output directory" +) diff --git a/bb-tests/workloads/src/SocTest/soc.h b/bb-tests/workloads/src/SocTest/soc.h index f6bbe9e73..fe111b313 100644 --- a/bb-tests/workloads/src/SocTest/soc.h +++ b/bb-tests/workloads/src/SocTest/soc.h @@ -1,7 +1,7 @@ #pragma once #include -// Chip device map, matching framework.system.device.DeviceParams. +// Design device map, matching framework.system.device.DeviceParams. #define CLINT_BASE 0x02000000UL #define PLIC_BASE 0x0C000000UL diff --git a/bbdev/api/services/workflow_tools.py b/bbdev/api/services/workflow_tools.py index 67a2a83f9..0cd2b90fd 100644 --- a/bbdev/api/services/workflow_tools.py +++ b/bbdev/api/services/workflow_tools.py @@ -12,7 +12,14 @@ def get_name(self) -> str: return "call_workflow_api" def get_description(self) -> str: - return "Call internal workflow API endpoints.\n Available endpoints:\n - /verilator/verilog: Generate Verilog\n - /verilator/build: Build verilator (params: jobs)\n - /verilator/sim: Run simulation (params: binary, batch)\n - /workload/clean: Clean workload output directory\n - /workload/build: Build workload (params: chip, stable, ctest, mlirtest)\n - /model/build: Build model recipe (params: chip, model, ctrace, dtrace)" + return ( + "Call internal workflow API endpoints.\n Available endpoints:\n -" + " /verilator/verilog: Generate Verilog\n - /verilator/build: Build" + " verilator (params: jobs)\n - /verilator/sim: Run simulation (params:" + " binary, batch)\n - /workload/clean: Clean workload output directory\n " + " - /workload/build: Build workload (params: design, stable, ctest," + " mlirtest)\n - /model/build: Build model recipe (params: design, model)" + ) def get_parameters(self) -> Dict[str, Any]: return { diff --git a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py index cdff96dae..02bbe2c93 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_api.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_api.step.py @@ -1,3 +1,4 @@ +from utils.event_common import require_target import os import sys @@ -10,9 +11,7 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.event_common import require_chip -from utils.path import get_buckyball_path -from steps.bebop.bemu.scripts.model_sim import model_run_commands +from utils.event_common import require_design config = { "name": "bebop-bemu-sim-api", @@ -25,65 +24,18 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - if "pk" in body or "host-io" in body: - return ApiResponse( - status=400, body={"error": "Unknown parameter: pk or host-io"} - ) - if "system" in body: - return ApiResponse(status=400, body={"error": "Unknown parameter: system"}) - if body.get("core_index") is not None and any( - key in body for key in ("dtb", "initrd", "memory-mib", "load-manifest") - ): - return ApiResponse( - status=400, body={"error": "chip boot options cannot select a single core"} - ) - if body.get("load-manifest") and any( - key in body for key in ("dtb", "initrd", "memory-mib") - ): - return ApiResponse( - status=400, - body={"error": "load-manifest conflicts with dtb, initrd and memory-mib"}, - ) - chip = body.get("chip", "") try: - require_chip({"chip": chip}) - except ValueError as e: - return ApiResponse( - status=400, - body={ - "success": False, - "failure": True, - "returncode": 400, - "message": str(e), - }, - ) - - binary = body.get("binary", "") - if body.get("model"): - try: - model_run_commands(get_buckyball_path(), body) - except (ValueError, KeyError, OSError) as error: - return ApiResponse( - status=400, - body={ - "success": False, - "failure": True, - "returncode": 400, - "message": str(error), - }, - ) - elif not binary: - return ApiResponse( - status=400, - body={ - "success": False, - "failure": True, - "returncode": 400, - "message": "model or binary parameter is required", - }, - ) + selection = require_target(body, ("device", "core")) + require_design(body) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + if not body.get("binary"): + return ApiResponse(status=400, body={"error": "binary parameter is required"}) await ctx.enqueue( - {"topic": "bebop.bemu.sim", "data": {**body, "_trace_id": ctx.trace_id}} + { + "topic": "bebop.bemu.sim", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 894f517c8..8233dc2ef 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -19,17 +19,17 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) from utils.path import ( - bebop_cargo_env, + bebop_target_dir, + bebop_bin_path, get_buckyball_path, log_dir, - workloads_output_root, + design_output_root, ) from utils.stream_run import stream_run_logger_async from utils.search_workload import search_workload -from steps.bebop.bemu.scripts.model_sim import model_run_commands from utils.event_common import check_result, get_origin_trace_id from utils.process_registry import cancellation_requested -from bemu_common import bemu_chip_binary, bemu_manifest +from bemu_common import bemu_binary, bemu_manifest from steps.bebop.performance_report import performance_report from utils.reports import source_context @@ -42,124 +42,22 @@ } -def clean_model_trace(binary_dir: str) -> None: - trace_dir = Path(binary_dir) / "trace" - if trace_dir.exists(): - shutil.rmtree(trace_dir) - - -def resolve_bemu_binary(bbdir: str, chip: str, binary_name: str) -> str | None: - if Path(binary_name).name != binary_name: - return None - workload_root = Path(workloads_output_root(bbdir)) / chip / "workloads" - workload = search_workload(str(workload_root), binary_name) - if workload is not None: - return workload - kernel = Path(bbdir) / "bb-tests" / "output" / "kernel" / chip / binary_name - return str(kernel) if kernel.is_file() else None +def resolve_bemu_binary( + bbdir: str, design: str, binary_name: str, selection: str +) -> str | None: + workload_root = ( + Path(design_output_root(bbdir, design, selection=selection)) / "workloads" + ) + return search_workload(str(workload_root), binary_name) async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - chip = input_data.get("chip") - if not chip: - ctx.logger.error("Missing required parameter: chip must be specified") - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": "missing_chip"}, - trace_id=origin_tid, - ) - return - try: - bemu_cargo_manifest = bemu_manifest(chip, bbdir) - except ValueError as e: - ctx.logger.error(str(e)) - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": "invalid_chip", "chip": chip}, - trace_id=origin_tid, - ) - return - if input_data.get("model"): - report_context = source_context(bbdir, chip) - params = {key: value for key, value in input_data.items() if key != "_trace_id"} - commands, run_log = model_run_commands(bbdir, params) - run_log.mkdir(parents=True, exist_ok=True) - ctx.logger.info(f"Model simulation logs: {run_log}") - filenames = ( - ("run.log",) - if input_data.get("reuse-simulator", False) - else ("build.log", "run.log") - ) - for command, filename in zip(commands, filenames, strict=True): - command_line = shlex.join(["nix", "develop", "-c", *command]) - command_line = f"set -o pipefail; {command_line} 2>&1 | tee {shlex.quote(str(run_log / filename))}" - result = await stream_run_logger_async( - cmd=command_line, - executable="/bin/bash", - logger=ctx.logger, - cwd=bbdir, - task_scope=origin_tid, - stdout_prefix="model simulation", - stderr_prefix="model simulation", - ) - if cancellation_requested(origin_tid): - return - if result.returncode: - break - try: - report_path = performance_report( - bbdir, - report_context, - origin_tid, - "bemu", - run_log, - (run_log / "run.log").read_text(), - result.returncode, - ) - except (OSError, ValueError, KeyError) as error: - await check_result( - ctx, - 1, - extra_fields={ - "task": "report", - "error": str(error), - "log_dir": str(run_log), - }, - trace_id=origin_tid, - ) - return - await check_result( - ctx, - result.returncode, - continue_run=False, - extra_fields={ - "chip": chip, - "model": params["model"], - "report_path": report_path, - "log_dir": str(run_log), - }, - trace_id=origin_tid, - ) - return - binary_name = input_data.get("binary", "") - try: - binary_path = resolve_bemu_binary(bbdir, chip, binary_name) - except ValueError as error: - ctx.logger.error(str(error)) - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": "binary_resolution_error", "binary": binary_name}, - trace_id=origin_tid, - ) - return + design = input_data["design"] + bemu_cargo_manifest = bemu_manifest(design, bbdir, selection=input_data["target"]) + binary_name = input_data["binary"] + binary_path = resolve_bemu_binary(bbdir, design, binary_name, input_data["target"]) if binary_path is None: ctx.logger.error(f"binary not found: {binary_name}") await check_result( @@ -172,38 +70,35 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return ctx.logger.info(f"binary_path: {binary_path}") binary_dir = os.path.dirname(binary_path) - perfetto_target = None - if input_data.get("tool-profile"): - if not binary_name.endswith("-run"): - raise ValueError(f"tool-profile binary must end with '-run': {binary_name}") - perfetto_target = f"{binary_name.removesuffix('-run')}-perfetto" - if perfetto_target: - clean_model_trace(binary_dir) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - run_log = log_dir(bbdir, chip, "verilog", timestamp, "bemu", binary_name) + run_log = log_dir( + bbdir, + design, + "verilog", + timestamp, + "bemu", + binary_name, + selection=input_data["target"], + ) os.makedirs(run_log, exist_ok=True) - core_index = input_data.get("core_index") - if "system" in input_data: - raise ValueError("Unknown parameter: system") - if core_index is not None and any( - key in input_data for key in ("dtb", "initrd", "memory-mib", "load-manifest") - ): - raise ValueError("chip boot options cannot select a single core") - entry = [bemu_chip_binary(chip) if core_index is None else "bebop-bemu", "--"] - cargo_args = [ + target = Path( + bebop_target_dir(bbdir, design, "bemu", selection=input_data["target"]) + ) + installed = Path( + bebop_bin_path(bbdir, design, "bemu", selection=input_data["target"]) + ) + build_args = [ "cargo", - "run", + "build", "--release", "--manifest-path", str(bemu_cargo_manifest), + "--target-dir", + str(target), "--bin", - *entry, - "--elf", - binary_path, - "--log-dir", - run_log, - ] - cargo_args[1:1] = [ + bemu_binary(), + "--features", + "bemu", "--config", 'build.rustflags=["-C", "target-cpu=native"]', "--config", @@ -211,34 +106,51 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--config", "profile.release.codegen-units=1", ] - for option in ("dtb", "initrd", "load-manifest"): + build_result = await stream_run_logger_async( + cmd=shlex.join(["nix", "develop", "-c", *build_args]), + executable="/bin/bash", + logger=ctx.logger, + cwd=bbdir, + task_scope=origin_tid, + stdout_prefix="BEMU build", + stderr_prefix="BEMU build", + ) + if cancellation_requested(origin_tid): + return + if build_result.returncode: + await check_result( + ctx, + build_result.returncode, + continue_run=False, + extra_fields={"task": "build", "design": design}, + trace_id=origin_tid, + ) + return + installed.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(target / "release/bebop-bemu", installed) + run_args = [str(installed), "--elf", binary_path, "--log-dir", run_log] + # `bbdev kernel --build` lays the kernel and the model into -ddr.bin + # next to the firmware ELF. + ddr_image = Path(binary_path).with_suffix("").as_posix() + "-ddr.bin" + if os.path.isfile(ddr_image): + run_args.extend(["--ddr-image", ddr_image]) + for option in ("dtb", "initrd"): if input_data.get(option): - cargo_args.extend([f"--{option}", input_data[option]]) + run_args.extend([f"--{option}", input_data[option]]) if input_data.get("memory-mib") is not None: - cargo_args.extend(["--memory-mib", str(input_data["memory-mib"])]) - if core_index is not None: - cargo_args.extend(["--core-index", str(core_index)]) + run_args.extend(["--memory-mib", str(input_data["memory-mib"])]) if input_data.get("disasm"): - cargo_args.append("--disasm") + run_args.append("--disasm") if input_data.get("tool-profile"): - cargo_args.append("--tool-profile") + run_args.append("--tool-profile") for trace_name in ("itrace", "mtrace"): if input_data.get(trace_name, False): - cargo_args.append(f"--{trace_name}") - arguments = input_data.get("arguments", []) - if arguments and core_index is None: - raise ValueError("chip boot does not accept process arguments") - if arguments: - if not isinstance(arguments, list) or not all( - isinstance(arg, str) for arg in arguments - ): - raise ValueError("arguments must be a list of strings") - cargo_args.extend(["--", *arguments]) - inner_cmd = f"cd {shlex.quote(binary_dir)} && {shlex.join(cargo_args)}" + run_args.append(f"--{trace_name}") + inner_cmd = f"cd {shlex.quote(binary_dir)} && {shlex.join(run_args)}" run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" run_cmd = f"set -o pipefail; {run_cmd} 2>&1 | tee {shlex.quote(str(Path(run_log) / 'run.log'))}" ctx.logger.info(f"Running bebop bemu: {run_cmd}") - report_context = source_context(bbdir, chip) + report_context = source_context(bbdir, design) run_result = await stream_run_logger_async( cmd=run_cmd, logger=ctx.logger, @@ -247,7 +159,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: stdout_prefix="bebop bemu", stderr_prefix="bebop bemu", task_scope=origin_tid, - env={**os.environ.copy(), **bebop_cargo_env(bbdir, chip)}, + env=os.environ.copy(), ) if cancellation_requested(origin_tid): return @@ -278,41 +190,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "task": "bemu", "report_path": report_path, "binary": binary_path, - "chip": chip, + "design": design, "log_dir": run_log, "timestamp": timestamp, }, trace_id=origin_tid, ) return - perfetto_path = None - if perfetto_target: - perfetto_cmd = f"cmake --build {shlex.quote(f'{bbdir}/bb-tests/build')} --target {shlex.quote(perfetto_target)}" - ctx.logger.info(f"Generating Perfetto trace: {perfetto_cmd}") - perfetto_result = await stream_run_logger_async( - cmd=perfetto_cmd, - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="perfetto", - stderr_prefix="perfetto", - ) - perfetto_path = f"{binary_dir}/trace/perfetto.json" - if perfetto_result.returncode != 0: - await check_result( - ctx, - perfetto_result.returncode, - continue_run=False, - extra_fields={ - "task": "perfetto", - "binary": binary_path, - "log_dir": run_log, - "timestamp": timestamp, - "perfetto_target": perfetto_target, - "perfetto": perfetto_path, - }, - trace_id=origin_tid, - ) - return await check_result( ctx, 0, @@ -321,11 +205,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "task": "bemu", "report_path": report_path, "binary": binary_path, - "chip": chip, + "design": design, "log_dir": run_log, "timestamp": timestamp, - "perfetto_target": perfetto_target, - "perfetto": perfetto_path, }, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py index dc987026e..fe889b77a 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -1,3 +1,4 @@ +from utils.event_common import require_target import os import sys from motia import ApiRequest, ApiResponse, FlowContext, api @@ -8,7 +9,7 @@ scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.event_common import require_chip +from utils.event_common import require_design config = { "name": "bebop-bemu-batch-api", @@ -21,11 +22,20 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - chip = body.get("chip") try: - require_chip({"chip": chip}) + selection = require_target(body, ("device",)) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + design = body.get("design") + try: + require_design({"design": design}) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) + junit = body.get("junit") + if junit is not None and (not isinstance(junit, str) or not os.path.isabs(junit)): + return ApiResponse( + status=400, body={"error": "junit must be an absolute file path"} + ) test_type = body.get("test") if not test_type: return ApiResponse( @@ -45,13 +55,15 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: { "topic": "bebop.bemu.batch", "data": { - "chip": chip, + "design": design, "test": test_type, + "junit": junit, "jobs": body.get("jobs", 1), "clean-before": body.get( "clean-before", body.get("clean_before", False) ), "_trace_id": ctx.trace_id, + "target": selection, }, } ) diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index 36bfed88a..0e6854ac1 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -2,10 +2,14 @@ bebop bemu batch event handler Runs bebop bemu batch regression: - 1. Build the selected chip's BEMU wrapper + 1. Build the selected design's BEMU wrapper 2. Run the BEMU regression harness """ +from pathlib import Path +from utils.path import design_output_root + + import os import shlex import sys @@ -20,7 +24,12 @@ scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.path import bebop_cargo_env, get_buckyball_path, workloads_output_root +from utils.path import ( + bebop_target_dir, + bebop_bin_path, + get_buckyball_path, + workloads_output_root, +) from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from bemu_common import bemu_manifest @@ -38,35 +47,41 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - chip = input_data.get("chip") - if not chip: - ctx.logger.error("Missing required parameter: chip must be specified") + design = input_data.get("design") + if not design: + ctx.logger.error("Missing required parameter: design must be specified") await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + extra_fields={"error": "missing_design"}, trace_id=origin_tid, ) return try: - bemu_cargo_manifest = bemu_manifest(chip, bbdir) + bemu_cargo_manifest = bemu_manifest( + design, bbdir, selection=input_data["target"] + ) except ValueError as e: ctx.logger.error(str(e)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "invalid_chip", "chip": chip}, + extra_fields={"error": "invalid_design", "design": design}, trace_id=origin_tid, ) return - elf_root = workloads_output_root(bbdir) + elf_root = str( + Path(design_output_root(bbdir, design, selection=input_data["target"])) + / "workloads" + ) env = os.environ.copy() - env.update(bebop_cargo_env(bbdir, chip)) test_type = input_data.get("test", "bare-tests") try: - workload_toml = resolve_workload_toml(chip, "bemu", test_type, bbdir) + workload_toml = resolve_workload_toml( + design, "bemu", test_type, bbdir, selection=input_data["target"] + ) except ValueError as e: ctx.logger.error(str(e)) await check_result( @@ -76,13 +91,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "error": "invalid_regression", "test": test_type, - "chip": chip, + "design": design, }, trace_id=origin_tid, ) return ctx.logger.info(f"Running {test_type} with workload config: {workload_toml}") - build_cmd = f"nix develop -c cargo build --manifest-path {shlex.quote(str(bemu_cargo_manifest))} --tests" + target = bebop_target_dir(bbdir, design, "bemu", selection=input_data["target"]) + build_cmd = ( + "nix develop -c cargo build --manifest-path" + f" {shlex.quote(str(bemu_cargo_manifest))} --target-dir {shlex.quote(target)}" + " --features bemu --tests --bin bebop-bemu" + ) ctx.logger.info("Building bebop bemu (tests)...") build_result = await stream_run_logger_async( cmd=build_cmd, @@ -112,8 +132,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ] if input_data.get("clean-before", input_data.get("clean_before", False)): harness_args.append("--clean-before") + if input_data.get("junit"): + harness_args.extend(["--junit", input_data["junit"]]) harness = shlex.join(harness_args) - test_cmd = f"nix develop -c cargo test --manifest-path {shlex.quote(str(bemu_cargo_manifest))} --test test_bemu {harness}" + test_cmd = ( + "nix develop -c cargo test --manifest-path" + f" {shlex.quote(str(bemu_cargo_manifest))} --target-dir {shlex.quote(target)}" + f" --features bemu --test test_bemu {harness}" + ) ctx.logger.info(f"Running bebop bemu regression: {test_cmd}") run_result = await stream_run_logger_async( cmd=test_cmd, @@ -130,7 +156,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "task": "batch", "backend": "bemu", - "chip": chip, + "design": design, "test_type": test_type, "workload_toml": workload_toml, }, diff --git a/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py b/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py index eaef1930d..2b59f727c 100644 --- a/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py +++ b/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py @@ -1,3 +1,4 @@ +from utils.event_common import require_target import os import sys @@ -10,8 +11,8 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.path import get_buckyball_path -from bemu_analysis import abs_log_dir, chip_maps +from utils.path import get_buckyball_path, run_log_dir +from bemu_analysis import design_maps config = { "name": "bebop-bemu-analysis-api", @@ -21,6 +22,7 @@ "enqueues": ["bebop.bemu.analysis"], } + def _fail(message: str, status: int = 400) -> ApiResponse: return ApiResponse( status=status, @@ -35,16 +37,20 @@ def _fail(message: str, status: int = 400) -> ApiResponse: async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - allowed = {"chip", "log-dir", "log_dir", "itrace", "mtrace"} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + allowed = {"device", "core", "design", "run", "itrace", "mtrace"} unknown = sorted(k for k in body if k not in allowed) if unknown: return _fail(f"unknown parameter(s): {', '.join(unknown)}") - chip = body.get("chip", "") - if not chip: - return _fail("chip parameter is required") + design = body.get("design", "") + if not design: + return _fail("design parameter is required") try: - chip_maps(get_buckyball_path(), chip) + design_maps(get_buckyball_path(), design) except ValueError as e: return _fail(str(e)) @@ -53,9 +59,8 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: if not itrace and not mtrace: return _fail("need --itrace and/or --mtrace") - log_dir = body.get("log-dir", body.get("log_dir", "")) try: - log_dir = str(abs_log_dir(log_dir)) + run_log_dir(get_buckyball_path(), body.get("run")) except ValueError as e: return _fail(str(e)) @@ -63,11 +68,12 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: { "topic": "bebop.bemu.analysis", "data": { - "chip": chip, - "log-dir": log_dir, + "design": design, + "run": body["run"], "itrace": itrace, "mtrace": mtrace, "_trace_id": ctx.trace_id, + "target": selection, }, } ) diff --git a/bbdev/api/steps/bebop/bemu/04_analysis_event.step.py b/bbdev/api/steps/bebop/bemu/04_analysis_event.step.py index e2e25be77..70aa8377c 100644 --- a/bbdev/api/steps/bebop/bemu/04_analysis_event.step.py +++ b/bbdev/api/steps/bebop/bemu/04_analysis_event.step.py @@ -10,9 +10,9 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.path import get_buckyball_path +from utils.path import get_buckyball_path, run_log_dir from utils.event_common import check_result, get_origin_trace_id -from bemu_analysis import abs_log_dir, analysis_dir, chip_maps, write_report +from bemu_analysis import analysis_dir, design_maps, write_report config = { "name": "bebop-bemu-analysis", @@ -25,23 +25,27 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) - chip = input_data.get("chip") - if not chip: - ctx.logger.error("Missing required parameter: chip") + design = input_data.get("design") + if not design: + ctx.logger.error("Missing required parameter: design") await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, trace_id=origin_tid, ) return try: - log_path = abs_log_dir(input_data.get("log-dir") or input_data.get("log_dir")) + log_path = run_log_dir(get_buckyball_path(), input_data.get("run")) except ValueError as e: ctx.logger.error(str(e)) await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "log_dir_not_absolute", "message": str(e)}, + ctx, + 1, + continue_run=False, + extra_fields={"error": "invalid_run", "message": str(e)}, trace_id=origin_tid, ) return @@ -49,7 +53,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: itrace = bool(input_data.get("itrace", False)) mtrace = bool(input_data.get("mtrace", False)) try: - names, matrix, depth = chip_maps(get_buckyball_path(), chip) + names, matrix, depth = design_maps(get_buckyball_path(), design) text = analysis_dir( log_path, names=names, @@ -62,8 +66,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: except ValueError as e: ctx.logger.error(str(e)) await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "analysis_failed", "message": str(e), "log_dir": str(log_path)}, + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "analysis_failed", + "message": str(e), + "log_dir": str(log_path), + }, trace_id=origin_tid, ) return @@ -76,7 +86,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "bemu-analysis", - "chip": chip, + "design": design, "log_dir": str(log_path), "analysis": str(out), }, diff --git a/bbdev/api/steps/bebop/bemu/scripts/bank_replay.py b/bbdev/api/steps/bebop/bemu/scripts/bank_replay.py new file mode 100644 index 000000000..c1d479c3f --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/scripts/bank_replay.py @@ -0,0 +1,402 @@ +import argparse +from array import array +from collections import deque +import csv +import heapq +import json +import math +from pathlib import Path +import sys +import time + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + +repo = Path(__file__).resolve().parents[6] +sys.path.insert(0, str(repo)) +from stack.scripts.design import load + +sys.path.insert(0, str(Path(__file__).resolve().parents[4])) +from utils.path import get_buckyball_path, run_log_dir + + +class Replay: + def __init__(self, thread, virtual_banks, calibration): + self.capacity = thread.mem.bank.num + self.depth = thread.frontend.rob_entries + self.alias_base = virtual_banks + (2 if thread.gpdomain.enable else 0) + self.bids = { + i.funct7: ball.ball_id + for ball in thread.balldomain.balls + for i in ball.instructions + } + self.calibration = calibration + self.now = 0 + self.frontier = deque() + self.pending = [] + self.tail = 0 + self.next_alloc = 0 + self.next_issue = 0 + self.mapping = {} + self.current_alias = list(range(self.alias_base)) + self.alias_busy = set() + self.alias_live = set() + self.alias_readers = [0] * (self.alias_base + self.depth) + self.busy_mask = 0 + self.busy_count = 0 + self.count = 0 + self.events = [ + array("Q", [0]), + array("B", [0]), + array("B", [0]), + array("B", [0]), + ] + self.commands = 0 + self.frontend_stalls = 0 + + def record(self): + values = (self.count, len(self.alias_live), self.busy_count) + assert ( + values[0] <= self.capacity + and values[1] <= self.depth + and values[2] <= values[0] + ) + if values != tuple(a[-1] for a in self.events[1:]): + self.events[0].append(self.now) + for column, value in zip(self.events[1:], values): + column.append(value) + + def enqueue(self, event): + f, a, b = event["f"], event["a"], event["b"] + mode = f >> 4 + reads = [a & 1023] if mode in (1, 3, 4) or (f == 32 and b & 4096) else [] + if mode == 4: + reads.append((a >> 10) & 1023) + write = ( + ((a & 1023) if f == 32 and not b & 4096 else (a >> 20) & 1023) + if mode in (2, 3, 4) and f != 35 + else None + ) + assert all(v < self.alias_base for v in reads) and ( + write is None or write < self.alias_base + ) + config = f == 32 + bid = self.bids[f] if f in self.bids else None + uses = set(reads) + if write is not None: + uses.add(write) + read_mask = sum(1 << v for v in set(reads)) + write_mask = 0 if write is None else 1 << write + mask = read_mask | write_mask + deps = [] + for older in self.frontier: + conflict = bool( + read_mask & older["write_mask"] or write_mask & older["mask"] + ) + config_conflict = ( + config + and older["config"] + or (config or older["config"]) + and bool(mask & older["mask"]) + ) + if config_conflict: + deps.append((older, "committed")) + elif conflict or bid is not None and bid == older["bid"]: + deps.append((older, "done")) + alias = self.alias_base + self.tail if write is not None else None + renamed_reads = [self.current_alias[v] for v in reads] + entry = { + "f": f, + "a": a, + "b": b, + "base": event["base"], + "reads": reads, + "write": write, + "writes": set() if write is None else {write}, + "uses": uses, + "config": config, + "bid": bid, + "alias": alias, + "renamed_reads": renamed_reads, + "deps": deps, + "issued": False, + "done": False, + "committed": False, + "allocated": self.now, + "physical": [], + "number": self.commands, + "mask": mask, + "write_mask": write_mask, + } + if alias is not None: + assert alias not in self.alias_live + self.current_alias[write] = alias + self.alias_live.add(alias) + self.alias_busy.add(alias) + for alias_read in renamed_reads: + self.alias_readers[alias_read] += 1 + self.frontier.append(entry) + self.tail = (self.tail + 1) % self.depth + self.next_alloc = self.now + 1 + self.commands += 1 + + def advance(self, incoming, arrival): + while True: + while self.pending and self.pending[0][0] <= self.now: + _, _, entry = heapq.heappop(self.pending) + entry["done"] = True + entry["completed"] = self.now + self.busy_mask &= ~entry["mask"] + self.busy_count -= len(entry["physical"]) + for alias_read in entry["renamed_reads"]: + self.alias_readers[alias_read] -= 1 + if entry["alias"] is not None: + self.alias_busy.remove(entry["alias"]) + committed = False + while ( + self.frontier + and self.frontier[0]["done"] + and self.frontier[0]["completed"] < self.now + ): + entry = self.frontier.popleft() + entry["committed"] = True + committed = True + if entry["alias"] is not None: + self.alias_live.remove(entry["alias"]) + if self.current_alias[entry["write"]] == entry["alias"]: + self.current_alias[entry["write"]] = entry["write"] + if entry["config"]: + v, b = entry["a"] & 1023, entry["b"] + if b & 4096: + target = (entry["a"] >> 20) & 1023 + self.mapping[target] = self.mapping.get( + target, [] + ) + self.mapping.pop(v) + else: + self.mapping.pop(v, None) + if b & 1024: + groups = (b >> 5) & 31 + groups = groups if groups else self.capacity + occupied = {p for ps in self.mapping.values() for p in ps} + available = [ + p for p in range(self.capacity) if p not in occupied + ] + assert len(available) >= groups + self.mapping[v] = available[:groups] + self.count = sum(map(len, self.mapping.values())) + if self.now >= self.next_issue: + for entry in self.frontier: + if ( + entry["issued"] + or entry["allocated"] >= self.now + or entry["mask"] & self.busy_mask + ): + continue + if any(not dep[field] for dep, field in entry["deps"]) or any( + a in self.alias_busy for a in entry["renamed_reads"] + ): + continue + entry["issued"] = True + entry["deps"] = [] + entry["physical"] = [ + p + for v in entry["uses"] + if not entry["config"] + for p in self.mapping[v] + ] + self.busy_mask |= entry["mask"] + self.busy_count += len(entry["physical"]) + f, a, b = entry["f"], entry["a"], entry["b"] + factor = self.calibration[str(f)] + base = entry["base"] + if f in (16, 33, 49, 69, 70, 72) and not ( + f in (16, 33) and b >> 63 + ): + v = ( + (a & 1023) + if f in (16, 49, 69, 70, 72) + else (a >> 20) & 1023 + ) + base *= len(self.mapping[v]) + if ( + factor.get("model") + == "effective_rows*K + bias_rows + writeback_rows + startup" + ): + rows, k = b & 4095, a >> 30 + features = ( + (7.125 if rows == 1 else rows) * k, + ((b >> 24) & 1) * rows, + ((b >> 25) & 1) * rows, + 1, + ) + duration = sum( + x * y for x, y in zip(features, factor["coefficients"]) + ) + else: + duration = base * factor["scale"] + assert duration > 0 + entry["issue_cycle"] = self.now + completion = self.now + math.ceil(duration) + heapq.heappush(self.pending, (completion, entry["number"], entry)) + self.next_issue = self.now + 1 + break + # The RTL forbids allocating on a commit cycle and reusing an alias + # still referenced by any incomplete entry. + alias = self.alias_base + self.tail + alias_referenced = ( + alias in self.alias_busy or self.alias_readers[alias] != 0 + ) + if ( + incoming is not None + and arrival <= self.now + and not committed + and len(self.frontier) < self.depth + and self.now >= self.next_alloc + and not alias_referenced + ): + self.enqueue(incoming) + self.record() + return + self.record() + if incoming is None and not self.frontier: + return + candidates = [] + if incoming is not None and arrival > self.now: + candidates.append(arrival) + if self.pending: + candidates.append(self.pending[0][0]) + if self.frontier and self.frontier[0]["done"]: + candidates.append(max(self.now + 1, self.frontier[0]["completed"] + 1)) + if any( + not e["issued"] + and (not any(not dep[field] for dep, field in e["deps"])) + and not any(a in self.alias_busy for a in e["renamed_reads"]) + and not e["mask"] & self.busy_mask + for e in self.frontier + ): + candidates.append(max(self.now + 1, self.next_issue)) + if committed: + candidates.append(self.now + 1) + if ( + incoming is not None + and len(self.frontier) < self.depth + and not alias_referenced + and self.next_alloc > self.now + ): + candidates.append(self.next_alloc) + assert candidates, "Replay deadlock" + self.now = min(t for t in candidates if t > self.now) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--design-pb", type=Path, required=True) + parser.add_argument("--thread-index", type=int, required=True) + parser.add_argument("--run", required=True) + parser.add_argument("--calibration", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + run_log = run_log_dir(get_buckyball_path(), args.run) + assert all( + p.is_absolute() for p in [args.design_pb, args.calibration, args.output_dir] + ) + design = load(args.design_pb) + thread = design.threads[args.thread_index] + core = next(t for t in design.device.cores if args.thread_index in t.thread_indices) + model = Replay( + thread, core.virtual_bank_count, json.loads(args.calibration.read_text()) + ) + started = time.monotonic() + first_cycle = None + with (run_log / "bdb.ndjson").open() as f: + for line in f: + e = json.loads(line) + if e["type"] != "itrace": + continue + if first_cycle is None: + first_cycle = e["issue_cycle"] + funct = int(e["funct"], 16) + if funct == 1: + model.advance(None, 0) + continue + base = e["modeled_cycles"] + if base is None: + raise ValueError("This kernel has no validated timing model") + model.advance( + { + "f": funct, + "a": int(e["rs1"], 16), + "b": int(e["rs2"], 16), + "base": base, + }, + e["issue_cycle"] - first_cycle, + ) + model.advance(None, 0) + model.events[0].append(model.now) + for a in model.events[1:]: + a.append(a[-1]) + args.output_dir.mkdir(exist_ok=True, parents=True) + values = np.array(model.events, dtype=np.uint64) + np.savez_compressed( + args.output_dir / "bank-replay-events.npz", + cycle=values[0], + allocated_physical_banks=values[1], + live_bat_aliases=values[2], + busy_physical_banks=values[3], + ) + result = { + "method": ( + "Event-driven replay of architectural-bank dependencies before the DMA issue-order " + "fix, BAT alias lifetime and MSET physical mappings. Completion latencies are RTL- " + "calibrated estimates; BEMU CPU issue times are arrival constraints. Not cycle- " + "accurate RTL or FPGA measurement." + ), + "limitations": [ + ( + "The four-slot thread admission interface is not modeled; ROB occupancy may exceed " + "what the current issue path can supply." + ), + "The per-producer DMA issue-order dependency added after the MVOUT deadlock diagnosis is not modeled.", + "Issue transport/endpoint backpressure is not modeled cycle by cycle.", + "Latency calibration extrapolates outside measured shapes.", + "Alias count is not a physical-bank count.", + ], + "log_dir": str(run_log), + "design_pb": str(args.design_pb), + "thread_index": args.thread_index, + "rob_entries": model.depth, + "physical_banks": model.capacity, + "bank_bytes": thread.mem.bank.width // 8 * thread.mem.bank.entries, + "instructions": model.commands, + "estimated_cycles": model.now, + "offline_seconds": time.monotonic() - started, + "physical_allocation_peak": int(values[1].max()), + "bat_alias_peak": int(values[2].max()), + "busy_physical_peak": int(values[3].max()), + "final_physical_banks": model.count, + "final_bat_aliases": len(model.alias_live), + } + (args.output_dir / "results.json").write_text(json.dumps(result, indent=2) + "\n") + fig, axes = plt.subplots(3, 1, figsize=(11, 7), constrained_layout=True) + for ax, y, label, capacity, color in zip( + axes, + values[1:], + [ + "Mapped physical data banks", + "Live BAT aliases", + "In-flight reserved physical banks", + ], + [model.capacity, model.depth, model.capacity], + ["#2563eb", "#d97706", "#0d9488"], + ): + ax.step(values[0] / 1e6, y, where="post", linewidth=0.65, color=color) + ax.set_ylabel(label, fontsize=9) + ax.set_ylim(0, capacity + 1) + ax.grid(alpha=0.2) + axes[-1].set_xlabel("Estimated replay cycles (million; not FPGA time)") + fig.savefig(args.output_dir / "bank-replay.png", dpi=180) + fig.savefig(args.output_dir / "bank-replay.pdf") + print(json.dumps(result, indent=2), flush=True) diff --git a/bbdev/api/steps/bebop/bemu/scripts/bank_replay_native.py b/bbdev/api/steps/bebop/bemu/scripts/bank_replay_native.py new file mode 100644 index 000000000..a30c79baa --- /dev/null +++ b/bbdev/api/steps/bebop/bemu/scripts/bank_replay_native.py @@ -0,0 +1,459 @@ +import argparse +import json +import math +from pathlib import Path +import sys +import time + +import numpy as np +from numba import njit + +sys.path.insert(0, str(Path(__file__).resolve().parents[6])) +from stack.scripts.design import load + +sys.path.insert(0, str(Path(__file__).resolve().parents[4])) +from utils.path import get_buckyball_path, run_log_dir + + +@njit(cache=True) +def simulate( + commands, capacity, depth, alias_base, bids, scales, shape_model, coefficients +): + valid = np.zeros(depth, np.bool_) + issued = np.zeros(depth, np.bool_) + done = np.zeros(depth, np.bool_) + config = np.zeros(depth, np.bool_) + function = np.zeros(depth, np.int64) + operand1 = np.zeros(depth, np.uint64) + operand2 = np.zeros(depth, np.uint64) + base_cycles = np.zeros(depth, np.float64) + write_bank = np.full(depth, -1, np.int64) + read0 = np.full(depth, -1, np.int64) + read1 = np.full(depth, -1, np.int64) + uses = np.zeros(depth, np.uint64) + writes = np.zeros(depth, np.uint64) + dependencies = np.zeros(depth, np.uint64) + config_dependencies = np.zeros(depth, np.uint64) + allocated = np.zeros(depth, np.int64) + completion = np.zeros(depth, np.int64) + physical_count = np.zeros(depth, np.int64) + current_alias = np.arange(alias_base, dtype=np.int64) + alias_busy = np.zeros(depth, np.bool_) + alias_live = np.zeros(depth, np.bool_) + alias_readers = np.zeros(alias_base + depth, np.int64) + owners = np.full(capacity, -1, np.int64) + groups = np.zeros(alias_base, np.int64) + n = len(commands) + times = np.zeros(n * 4 + 2, np.uint64) + mapped = np.zeros(n * 4 + 2, np.uint8) + aliases = np.zeros(n * 4 + 2, np.uint8) + reserved = np.zeros(n * 4 + 2, np.uint8) + points = 1 + now = 0 + head = 0 + tail = 0 + live_entries = 0 + mapped_count = 0 + alias_count = 0 + busy_count = 0 + busy_mask = np.uint64(0) + next_alloc = 0 + next_issue = 0 + index = 0 + while True: + completed_mask = np.uint64(0) + for slot in range(depth): + if ( + valid[slot] + and issued[slot] + and not done[slot] + and completion[slot] <= now + ): + done[slot] = True + completed_mask |= np.uint64(1) << np.uint64(slot) + busy_mask &= ~uses[slot] + busy_count -= physical_count[slot] + if write_bank[slot] >= 0: + alias_busy[slot] = False + if read0[slot] >= 0: + alias_readers[read0[slot]] -= 1 + if read1[slot] >= 0: + alias_readers[read1[slot]] -= 1 + committed = False + committed_mask = np.uint64(0) + while live_entries and done[head] and completion[head] < now: + slot = head + valid[slot] = False + live_entries -= 1 + committed = True + committed_mask |= np.uint64(1) << np.uint64(slot) + if write_bank[slot] >= 0: + alias_live[slot] = False + alias_count -= 1 + if current_alias[write_bank[slot]] == alias_base + slot: + current_alias[write_bank[slot]] = write_bank[slot] + if config[slot]: + v = np.int64(operand1[slot] & np.uint64(1023)) + b = operand2[slot] + if b & np.uint64(4096): + target = np.int64( + (operand1[slot] >> np.uint64(20)) & np.uint64(1023) + ) + for p in range(capacity): + if owners[p] == v: + owners[p] = target + groups[target] += groups[v] + groups[v] = 0 + else: + mapped_count -= groups[v] + groups[v] = 0 + for p in range(capacity): + if owners[p] == v: + owners[p] = -1 + if b & np.uint64(1024): + count = np.int64((b >> np.uint64(5)) & np.uint64(31)) + if count == 0: + count = capacity + for p in range(capacity): + if owners[p] < 0 and count: + owners[p] = v + count -= 1 + groups[v] += 1 + mapped_count += 1 + assert count == 0 + head = (head + 1) % depth + for slot in range(depth): + dependencies[slot] &= ~(completed_mask | committed_mask) + config_dependencies[slot] &= ~committed_mask + if now >= next_issue: + for offset in range(depth): + slot = (head + offset) % depth + if ( + not valid[slot] + or issued[slot] + or allocated[slot] >= now + or uses[slot] & busy_mask + ): + continue + if dependencies[slot] or config_dependencies[slot]: + continue + if read0[slot] >= alias_base and alias_busy[read0[slot] - alias_base]: + continue + if read1[slot] >= alias_base and alias_busy[read1[slot] - alias_base]: + continue + issued[slot] = True + busy_mask |= uses[slot] + count = 0 + if not config[slot]: + for v in range(alias_base): + if uses[slot] & (np.uint64(1) << np.uint64(v)): + assert groups[v] > 0 + count += groups[v] + physical_count[slot] = count + busy_count += count + f = function[slot] + a = operand1[slot] + b = operand2[slot] + base = base_cycles[slot] + if f == 16 or f == 33 or f == 49 or f == 69 or f == 70 or f == 72: + if not ((f == 16 or f == 33) and b >> np.uint64(63)): + v = ( + np.int64(a & np.uint64(1023)) + if f == 16 or f == 49 or f == 69 or f == 70 or f == 72 + else np.int64((a >> np.uint64(20)) & np.uint64(1023)) + ) + base *= groups[v] + if shape_model[f]: + rows = float(b & np.uint64(4095)) + k = float(a >> np.uint64(30)) + effective = 7.125 if rows == 1 else rows + duration = coefficients[f, 0] * (effective * k) + duration += ( + coefficients[f, 1] + * float((b >> np.uint64(24)) & np.uint64(1)) + * rows + ) + duration += ( + coefficients[f, 2] + * float((b >> np.uint64(25)) & np.uint64(1)) + * rows + ) + duration += coefficients[f, 3] + else: + duration = base * scales[f] + assert duration > 0 + completion[slot] = now + np.int64(math.ceil(duration)) + next_issue = now + 1 + break + can_allocate = ( + index < n + and not committed + and live_entries < depth + and now >= next_alloc + and not alias_busy[tail] + and alias_readers[alias_base + tail] == 0 + ) + if can_allocate and np.int64(commands[index]["arrival"]) <= now: + slot = tail + f = np.int64(commands[index]["f"]) + a = commands[index]["a"] + b = commands[index]["b"] + mode = f >> 4 + r0 = ( + np.int64(a & np.uint64(1023)) + if mode == 1 + or mode == 3 + or mode == 4 + or f == 32 + and b & np.uint64(4096) + else -1 + ) + r1 = np.int64((a >> np.uint64(10)) & np.uint64(1023)) if mode == 4 else -1 + w = -1 + if (mode == 2 or mode == 3 or mode == 4) and f != 35: + w = ( + np.int64(a & np.uint64(1023)) + if f == 32 and not b & np.uint64(4096) + else np.int64((a >> np.uint64(20)) & np.uint64(1023)) + ) + assert r0 < alias_base and r1 < alias_base and w < alias_base + read_mask = np.uint64(0) + if r0 >= 0: + read_mask |= np.uint64(1) << np.uint64(r0) + if r1 >= 0: + read_mask |= np.uint64(1) << np.uint64(r1) + write_mask = np.uint64(0) if w < 0 else np.uint64(1) << np.uint64(w) + mask = read_mask | write_mask + deps = np.uint64(0) + conf_deps = np.uint64(0) + for older in range(depth): + if not valid[older]: + continue + config_conflict = ( + f == 32 + and config[older] + or (f == 32 or config[older]) + and bool(mask & uses[older]) + ) + conflict = bool(read_mask & writes[older] or write_mask & uses[older]) + same_ball = bids[f] >= 0 and bids[f] == bids[function[older]] + if config_conflict: + conf_deps |= np.uint64(1) << np.uint64(older) + elif not done[older] and (conflict or same_ball): + deps |= np.uint64(1) << np.uint64(older) + valid[slot] = True + issued[slot] = False + done[slot] = False + config[slot] = f == 32 + function[slot] = f + operand1[slot] = a + operand2[slot] = b + base_cycles[slot] = commands[index]["base"] + allocated[slot] = now + physical_count[slot] = 0 + write_bank[slot] = w + uses[slot] = mask + writes[slot] = write_mask + dependencies[slot] = deps + config_dependencies[slot] = conf_deps + read0[slot] = -1 if r0 < 0 else current_alias[r0] + read1[slot] = -1 if r1 < 0 else current_alias[r1] + if read0[slot] >= 0: + alias_readers[read0[slot]] += 1 + if read1[slot] >= 0: + alias_readers[read1[slot]] += 1 + if w >= 0: + assert not alias_live[slot] + current_alias[w] = alias_base + slot + alias_live[slot] = True + alias_busy[slot] = True + alias_count += 1 + tail = (tail + 1) % depth + live_entries += 1 + index += 1 + next_alloc = now + 1 + admitted = True + else: + admitted = False + assert ( + busy_count >= 0 + and busy_count <= mapped_count + and mapped_count <= capacity + and alias_count <= depth + ) + if ( + mapped_count != mapped[points - 1] + or alias_count != aliases[points - 1] + or busy_count != reserved[points - 1] + ): + times[points] = now + mapped[points] = mapped_count + aliases[points] = alias_count + reserved[points] = busy_count + points += 1 + if admitted: + continue + if index == n and live_entries == 0: + break + next_time = np.int64(9223372036854775807) + if index < n and np.int64(commands[index]["arrival"]) > now: + next_time = min(next_time, np.int64(commands[index]["arrival"])) + for slot in range(depth): + if not valid[slot]: + continue + if issued[slot] and not done[slot]: + next_time = min(next_time, completion[slot]) + if ( + not issued[slot] + and not dependencies[slot] + and not config_dependencies[slot] + and not uses[slot] & busy_mask + ): + hazard = ( + read0[slot] >= alias_base + and alias_busy[read0[slot] - alias_base] + or read1[slot] >= alias_base + and alias_busy[read1[slot] - alias_base] + ) + if not hazard: + next_time = min(next_time, max(now + 1, next_issue)) + if live_entries and done[head]: + next_time = min(next_time, max(now + 1, completion[head] + 1)) + if committed: + next_time = min(next_time, now + 1) + if ( + index < n + and live_entries < depth + and not alias_busy[tail] + and alias_readers[alias_base + tail] == 0 + and next_alloc > now + ): + next_time = min(next_time, next_alloc) + assert next_time > now and next_time < 9223372036854775807 + now = next_time + times[points] = now + mapped[points] = mapped_count + aliases[points] = alias_count + reserved[points] = busy_count + return ( + times[: points + 1], + mapped[: points + 1], + aliases[: points + 1], + reserved[: points + 1], + now, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--design-pb", type=Path, required=True) + parser.add_argument("--thread-index", type=int, required=True) + parser.add_argument("--run", required=True) + parser.add_argument("--calibration", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + run_log = run_log_dir(get_buckyball_path(), args.run) + assert all( + p.is_absolute() for p in [args.design_pb, args.calibration, args.output_dir] + ) + design = load(args.design_pb) + thread = design.threads[args.thread_index] + core = next(t for t in design.device.cores if args.thread_index in t.thread_indices) + capacity = thread.mem.bank.num + depth = thread.frontend.rob_entries + alias_base = core.virtual_bank_count + (2 if thread.gpdomain.enable else 0) + assert capacity < 256 and depth < 64 and alias_base < 64 + calibration = json.loads(args.calibration.read_text()) + bids = np.full(128, -1, np.int64) + scales = np.zeros(128) + shape_model = np.zeros(128, np.bool_) + coefficients = np.zeros((128, 4)) + for ball in thread.balldomain.balls: + for isa in ball.instructions: + bids[isa.funct7] = ball.ball_id + for f, row in calibration.items(): + if "model" in row: + assert ( + row["model"] + == "effective_rows*K + bias_rows + writeback_rows + startup" + ) + shape_model[int(f)] = True + coefficients[int(f)] = row["coefficients"] + else: + scales[int(f)] = row["scale"] + dtype = np.dtype( + [("f", "u1"), ("a", "u8"), ("b", "u8"), ("base", "u8"), ("arrival", "u8")] + ) + started = time.monotonic() + + def instructions(): + first = None + with (run_log / "bdb.ndjson").open() as stream: + for line in stream: + e = json.loads(line) + if e["type"] != "itrace": + continue + f = int(e["funct"], 16) + assert f != 1, "Barrier replay requires a separate drain boundary" + if first is None: + first = e["issue_cycle"] + assert e["modeled_cycles"] is not None + assert str(f) in calibration + yield ( + f, + int(e["rs1"], 16), + int(e["rs2"], 16), + e["modeled_cycles"], + e["issue_cycle"] - first, + ) + + commands = np.fromiter(instructions(), dtype=dtype) + parsed = time.monotonic() + t, m, a, b, end = simulate( + commands, capacity, depth, alias_base, bids, scales, shape_model, coefficients + ) + finished = time.monotonic() + args.output_dir.mkdir(parents=True, exist_ok=True) + np.savez_compressed( + args.output_dir / "bank-replay-events.npz", + cycle=t, + allocated_physical_banks=m, + live_bat_aliases=a, + busy_physical_banks=b, + ) + result = { + "method": ( + "Native event-driven replay of architectural-bank dependencies before the DMA issue- " + "order fix, BAT alias lifetime and MSET physical mappings. Calibrated completion " + "estimates; not cycle-accurate RTL or FPGA measurement." + ), + "limitations": [ + ( + "The four-slot thread admission interface is not modeled; ROB occupancy may exceed " + "what the current issue path can supply." + ), + "The per-producer DMA issue-order dependency added after the MVOUT deadlock diagnosis is not modeled.", + "Issue transport/endpoint backpressure is not modeled cycle by cycle.", + "Physical mapping changes are approximated at MSET commit, rather than actual backend config handshake.", + "Latency calibration extrapolates outside measured shapes.", + "Alias count is not a physical-bank count.", + ], + "log_dir": str(run_log), + "design_pb": str(args.design_pb), + "thread_index": args.thread_index, + "rob_entries": depth, + "physical_banks": capacity, + "bank_bytes": thread.mem.bank.width // 8 * thread.mem.bank.entries, + "instructions": len(commands), + "estimated_cycles": end, + "offline_seconds": finished - started, + "parse_seconds": parsed - started, + "replay_and_jit_seconds": finished - parsed, + "physical_allocation_peak": int(m.max()), + "bat_alias_peak": int(a.max()), + "busy_physical_peak": int(b.max()), + "final_physical_banks": int(m[-1]), + "final_bat_aliases": int(a[-1]), + } + (args.output_dir / "results.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result, indent=2), flush=True) diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py index 204addaf3..96244d7ce 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py @@ -13,22 +13,13 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[4])) -def abs_log_dir(value: object) -> Path: - if not isinstance(value, str) or not value: - raise ValueError("log-dir must be an absolute path") - path = Path(value) - if not path.is_absolute(): - raise ValueError(f"log-dir must be an absolute path: {value}") - return path - - def _hex_u(value: object, field: str, line_no: int) -> int: if not isinstance(value, str) or not value.startswith("0x"): raise ValueError(f"bdb.ndjson:{line_no}: {field} must be a 0x hex string") return int(value, 16) -def chip_maps(bbdir: str, chip: str) -> tuple[dict[int, str], set[int], int]: +def design_maps(bbdir: str, design: str) -> tuple[dict[int, str], set[int], int]: system_isa = { 0: "fence", 1: "barrier", @@ -38,68 +29,34 @@ def chip_maps(bbdir: str, chip: str) -> tuple[dict[int, str], set[int], int]: 34: "mmio_set", 35: "mvin_mmio", } - path = ( - Path(bbdir) - / "examples" - / "chips" - / chip - / "configs" - / "generated" - / "config" - / "config.json" - ) - data = json.loads(path.read_text(encoding="utf-8")) - cores = [] - for tile in data["designs"]["tiles"]: - cores.extend(tile["cores"]) - if len(cores) != 1: - # isomorphic tiles share the same core pkg; take first unique balldomain+_file - seen = [] - uniq = [] - for core in cores: - key = core["balldomain"]["_file"] - if key not in seen: - seen.append(key) - uniq.append(core) - if len(uniq) != 1: - raise ValueError(f"chip {chip} has no unique topology core") - core = uniq[0] - else: - core = cores[0] - - ball = core["balldomain"] - isa = ball["ballISA"] - ball_path = ball["_file"] - + from utils.path import load_design_config + + data = load_design_config(bbdir, design) + configurations = { + ( + p.config.balldomain.SerializeToString(deterministic=True), + p.config.mem.bank.entries, + ): p.config + for p in data.profiles + } + if len(configurations) != 1: + raise ValueError( + f"design {design} requires a unique instruction map and bank depth for this analysis" + ) + config = next(iter(configurations.values())) names: dict[int, str] = dict(system_isa) matrix: set[int] = set() - seen: set[int] = set() - for entry in isa: - if not isinstance(entry, dict): - raise ValueError(f"ballISA entry must be a table: {ball_path}") - funct = entry.get("funct7") - mnemonic = entry.get("mnemonic") - if not isinstance(funct, int) or funct < 0: - raise ValueError(f"ballISA funct7 must be a non-negative int: {ball_path}") - if not isinstance(mnemonic, str) or not mnemonic: - raise ValueError( - f"ballISA mnemonic must be a non-empty string: {ball_path}" - ) - if funct in seen: - raise ValueError(f"duplicate ballISA funct7 {funct}: {ball_path}") - if funct in system_isa: - raise ValueError( - f"ballISA funct7 {funct} ({mnemonic}) collides with ISA {system_isa[funct]}: {ball_path}" - ) - seen.add(funct) - names[funct] = mnemonic.lower() - if mnemonic in {"MATRIX", "MATRIX_F32", "SMATMUL_OS"}: - matrix.add(funct) - - mem = core["memdomain"] - bank = mem["bank"] - entries = bank["entries"] - return names, matrix, entries + for instance in config.balldomain.balls: + for instruction in instance.instructions: + funct, mnemonic = instruction.funct7, instruction.mnemonic + if funct in names: + raise ValueError( + f"design {design}: duplicate or reserved funct7 {funct}" + ) + names[funct] = mnemonic.lower() + if mnemonic in {"MATRIX", "MATRIX_F32", "SMATMUL_OS"}: + matrix.add(funct) + return names, matrix, config.mem.bank.entries def _mnk(rs1: int, rs2: int) -> tuple[int, int, int]: @@ -118,7 +75,7 @@ def analysis_dir( if bank_depth <= 0: raise ValueError(f"bank_depth must be a positive int, got {bank_depth}") if not log_dir.is_dir(): - raise ValueError(f"log-dir is not a directory: {log_dir}") + raise ValueError(f"run is not a directory: {log_dir}") bdb = log_dir / "bdb.ndjson" if not bdb.is_file(): raise ValueError(f"missing bdb.ndjson: {bdb}") @@ -218,22 +175,22 @@ def write_report(log_dir: Path, text: str) -> Path: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--chip", required=True) - parser.add_argument("--log-dir", required=True, type=Path) + parser.add_argument("--design", required=True) + parser.add_argument("--run", required=True) parser.add_argument("--bbdir", type=Path) parser.add_argument("--itrace", action="store_true") parser.add_argument("--mtrace", action="store_true") args = parser.parse_args(argv) if not args.itrace and not args.mtrace: raise SystemExit("need --itrace and/or --mtrace") - log_dir = abs_log_dir(str(args.log_dir)) api_root = Path(__file__).resolve().parents[4] if str(api_root) not in sys.path: sys.path.insert(0, str(api_root)) - from utils.path import get_buckyball_path + from utils.path import get_buckyball_path, run_log_dir bbdir = args.bbdir if args.bbdir is not None else Path(get_buckyball_path()) - names, matrix, depth = chip_maps(str(bbdir), args.chip) + log_dir = run_log_dir(bbdir, args.run) + names, matrix, depth = design_maps(str(bbdir), args.design) text = analysis_dir( log_dir, names=names, diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py index b0aafeece..9128a1543 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_common.py @@ -1,6 +1,7 @@ from pathlib import Path -from utils.event_common import require_chip +from utils.event_common import require_design +from utils.path import runner_manifest def _repo(bbdir: str | None) -> Path: @@ -9,24 +10,14 @@ def _repo(bbdir: str | None) -> Path: return Path(bbdir or get_buckyball_path()) -def bemu_manifest(chip: str, bbdir: str | None = None) -> Path: - chip = require_chip({"chip": chip}) +def bemu_manifest(design: str, bbdir: str | None = None, *, selection: str) -> Path: + design = require_design({"design": design}) root = _repo(bbdir) - path = ( - root - / "examples" - / "chips" - / chip - / "configs" - / "generated" - / "bemu" - / "Cargo.toml" - ) + path = runner_manifest(root, design, selection) if not path.is_file(): - raise FileNotFoundError(f"missing {path}; run bbdev config --install") + raise FileNotFoundError(f"missing design manifest: {path}") return path -def bemu_chip_binary(chip: str) -> str: - """The chip's BEMU boot entry in its generated crate.""" - return f"bebop-chip-{require_chip({'chip': chip})}" +def bemu_binary() -> str: + return "bebop-bemu" diff --git a/bbdev/api/steps/bebop/bemu/scripts/model_sim.py b/bbdev/api/steps/bebop/bemu/scripts/model_sim.py deleted file mode 100644 index 5482bd2e3..000000000 --- a/bbdev/api/steps/bebop/bemu/scripts/model_sim.py +++ /dev/null @@ -1,144 +0,0 @@ -import json -import importlib.util -import re -import tomllib -from datetime import datetime -from pathlib import Path - -from utils.path import log_dir - - -def model_run_commands(repo: str, params: dict): - unknown = params.keys() - { - "chip", - "model", - "reuse-simulator", - "itrace", - "mtrace", - "firmware", - "guest-memory-mib", - "load-manifest", - } - if unknown: - raise ValueError(f"Unknown model simulation parameters: {sorted(unknown)}") - for key in ("chip", "model"): - if not isinstance(params.get(key), str) or not re.fullmatch( - r"[A-Za-z0-9_-]+", params[key] - ): - raise ValueError(f"Invalid or missing {key}") - root = Path(repo) - if not isinstance(params.get("reuse-simulator", False), bool): - raise ValueError("reuse-simulator must be a boolean") - for flag in ("itrace", "mtrace"): - if not isinstance(params.get(flag, False), bool): - raise ValueError(f"{flag} must be a boolean") - chip, model = params["chip"], params["model"] - config = root / "examples/models" / chip / model / "configs/running-param.toml" - build = root / "stack/models/build" / chip / model - packages = list((build / "artifact").glob("*.rax")) - if len(packages) != 1: - raise ValueError( - f"Expected one built RAX under {build / 'artifact'}; build the model first" - ) - layout = json.loads((build / "artifact/layout.json").read_text()) - kind = layout["execution"]["kind"] - settings = tomllib.loads(config.read_text()) - manifest = root / "examples/chips" / chip / "configs/generated/bemu/Cargo.toml" - if kind == "native" and "tile_indices" not in settings: - binary = "bebop-bemu" - python = root / "result/bin/python3" - elif kind == "python" or (kind == "native" and "tile_indices" in settings): - binary = f"bebop-chip-{chip}" - python = root / "result/bin/python3" - else: - raise ValueError(f"Unknown model execution kind: {kind}") - firmware = None - endpoint = layout["execution"].get("p2e", {}) - if endpoint.get("task_runtime") == "ant" or ( - kind == "native" and layout["execution"]["input"] == "resource" - ): - if kind != "native" or endpoint.get("kind") != "native": - raise ValueError("prepared model execution must run inside the Linux guest") - binary = f"bebop-chip-{chip}" - memory = params.get("guest-memory-mib", settings["memory_mib"]) - if memory is not None: - if isinstance(memory, bool) or not str(memory).isdigit() or int(memory) < 1: - raise ValueError("guest-memory-mib must be a positive integer") - memory = int(memory) - spec = importlib.util.spec_from_file_location( - "buckyball_model_guest", root / "stack/serving/guest.py" - ) - guest = importlib.util.module_from_spec(spec) - spec.loader.exec_module(guest) - firmware, guest_memory, _ = guest.firmware_profile( - root, chip, model, params.get("firmware"), memory - ) - elif any( - key in params for key in ("firmware", "guest-memory-mib", "load-manifest") - ): - raise ValueError("firmware options require a packaged Linux guest model") - target = root / "bebop/target" / chip - stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - log = Path(log_dir(repo, chip, "verilog", stamp, "bemu", model)) - commands = [ - [ - "cargo", - "build", - "--release", - "--config", - 'build.rustflags=["-C", "target-cpu=native"]', - "--config", - 'profile.release.lto="thin"', - "--config", - "profile.release.codegen-units=1", - "--manifest-path", - str(manifest), - "--target-dir", - str(target), - "--bin", - binary, - ], - [ - str(python), - "-u", - "-m", - "stack.serving.cli", - "--model", - str(packages[0]), - "--loader", - str(build / "loader/load-model"), - "--simulator", - str(target / "release" / binary), - "--log-dir", - str(log), - "--run-config", - str(config), - ], - ] - if firmware is not None: - commands[-1].extend( - ["--firmware", str(firmware), "--guest-memory-mib", str(guest_memory)] - ) - if firmware.name.endswith("-ddr.elf"): - manifest = firmware.with_suffix(".load.json") - if ( - "load-manifest" in params - and Path(params["load-manifest"]).resolve() != manifest.resolve() - ): - raise ValueError( - "load-manifest does not belong to the requested firmware" - ) - if not manifest.is_file(): - raise ValueError(f"MODEL_DDR load manifest missing: {manifest}") - commands[-1].extend(["--load-manifest", str(manifest)]) - elif "load-manifest" in params: - raise ValueError("load-manifest requires a MODEL_DDR firmware") - if params.get("reuse-simulator", False): - simulator = target / "release" / binary - if not simulator.is_file(): - raise ValueError(f"Reusable model simulator missing: {simulator}") - commands = commands[1:] - commands[-1].extend( - f"--{flag}" for flag in ("itrace", "mtrace") if params.get(flag, False) - ) - return commands, log diff --git a/bbdev/api/steps/bebop/p2e/00_clean_api.step.py b/bbdev/api/steps/bebop/p2e/00_clean_api.step.py index e7001eb9a..b2bc18016 100644 --- a/bbdev/api/steps/bebop/p2e/00_clean_api.step.py +++ b/bbdev/api/steps/bebop/p2e/00_clean_api.step.py @@ -1,6 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import get_buckyball_path config = { @@ -15,11 +16,23 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - await ctx.enqueue({ - "topic": "bebop.p2e.clean", - "data": {**body, "chip": chip, "task": "clean", "_trace_id": ctx.trace_id}, - }) + await ctx.enqueue( + { + "topic": "bebop.p2e.clean", + "data": { + **body, + "design": design, + "task": "clean", + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/00_clean_event.step.py b/bbdev/api/steps/bebop/p2e/00_clean_event.step.py index b479cb159..ed4bb2a0f 100644 --- a/bbdev/api/steps/bebop/p2e/00_clean_event.step.py +++ b/bbdev/api/steps/bebop/p2e/00_clean_event.step.py @@ -7,7 +7,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -25,16 +25,24 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() try: - chip = require_chip(input_data) + design = require_design(input_data) except ValueError as error: ctx.logger.error(str(error)) await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, trace_id=origin_tid, ) return - build_dir = rtl_dir(bbdir, chip, "p2e", input_data.get("output_dir")) + build_dir = rtl_dir( + bbdir, + design, + "p2e", + input_data.get("output_dir"), + selection=input_data["target"], + ) command = f"rm -rf {build_dir}" result = await stream_run_logger_async( diff --git a/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py index d49eeaa5b..fb0520eed 100644 --- a/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py +++ b/bbdev/api/steps/bebop/p2e/01_verilog_api.step.py @@ -1,6 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import get_buckyball_path config = { @@ -15,11 +16,20 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - data = {"chip": chip, "diff": bool(body.get("diff", False))} + data = {"design": design, "diff": bool(body.get("diff", False))} if body.get("output-dir"): data["output_dir"] = body["output-dir"] - await ctx.enqueue({"topic": "bebop.p2e.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "bebop.p2e.verilog", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py index a1bc91789..5690ce0b7 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py @@ -1,7 +1,8 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api from pathlib import Path -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import get_buckyball_path, rtl_dir config = { @@ -17,21 +18,34 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: bbdir = get_buckyball_path() body = request.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) vsrc_dir = rtl_dir( - bbdir, chip, "p2e", body.get("vsrc_dir") or body.get("vsrc-dir") + bbdir, + design, + "p2e", + body.get("vsrc_dir") or body.get("vsrc-dir"), + selection=selection, ) stop_after = body.get("stop_after", body.get("stop-after")) if stop_after is not None and stop_after not in ("vsyn", "vcom"): - return ApiResponse(status=400, body={"error": "stop-after must be vsyn or vcom"}) + return ApiResponse( + status=400, body={"error": "stop-after must be vsyn or vcom"} + ) output_dir = body.get("output_dir") or body.get("output-dir") if stop_after and output_dir and Path(output_dir).exists(): - return ApiResponse(status=400, body={"error": "Resource assessment requires a fresh output_dir"}) + return ApiResponse( + status=400, + body={"error": "Resource assessment requires a fresh output_dir"}, + ) data = { - "chip": chip, + "design": design, "diff": bool(body.get("diff", False)), "itrace": bool(body.get("itrace", False)), "mtrace": bool(body.get("mtrace", False)), @@ -39,8 +53,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "output_dir": output_dir, "stop_after": stop_after, } - await ctx.enqueue({ - "topic": "bebop.p2e.buildbitstream", - "data": {**data, "_trace_id": ctx.trace_id}, - }) + await ctx.enqueue( + { + "topic": "bebop.p2e.buildbitstream", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index e6087bacc..f3ce81aba 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -6,6 +6,9 @@ 2. Run bebop build p2e with rtl_dir and out_dir 3. Validate generated bitstream and runtime artifacts """ + +from utils.path import runner_manifest + import os import shlex import shutil @@ -19,8 +22,8 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.event_common import require_chip -from utils.path import bebop_cargo_env, get_buckyball_path, rtl_dir +from utils.event_common import require_design +from utils.path import bebop_target_dir, get_buckyball_path, rtl_dir from steps.bebop.p2e.scripts.timescale import normalize_p2e_timescale from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -40,67 +43,100 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bebop_dir = f"{bbdir}/bebop" try: - chip = require_chip(input_data) + design = require_design(input_data) except ValueError as error: ctx.logger.error(str(error)) await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, trace_id=origin_tid, ) return - vsrc_dir = rtl_dir(bbdir, chip, "p2e", input_data.get("vsrc_dir")) + vsrc_dir = rtl_dir( + bbdir, design, "p2e", input_data.get("vsrc_dir"), selection=input_data["target"] + ) normalize_p2e_timescale(vsrc_dir, ctx.logger) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") diff = bool(input_data.get("diff", False)) stop_after = input_data.get("stop_after") - build_dir = ( - input_data.get("output_dir") - or f"{bebop_dir}/build/{chip}{'-diff' if diff else ''}-{timestamp}" + build_dir = input_data.get("output_dir") or str( + Path(bbdir) + / "examples/designs" + / design + / "output/p2e" + / f"{timestamp}{'-diff' if diff else ''}" ) build_path = Path(build_dir).resolve() - build_root = (Path(bebop_dir) / "build").resolve() + build_root = (Path(bbdir) / "examples/designs" / design / "output/p2e").resolve() if build_root not in build_path.parents: raise ValueError(f"P2E output_dir must be under {build_root}: {build_path}") if stop_after and build_path.exists(): - raise ValueError(f"Resource assessment requires a fresh output_dir: {build_path}") + raise ValueError( + f"Resource assessment requires a fresh output_dir: {build_path}" + ) if build_path.exists(): shutil.rmtree(build_path) build_path.mkdir(parents=True) - manifest = Path(bbdir) / "examples" / "chips" / chip / "generated" / "bebop" / "Cargo.toml" + manifest = runner_manifest(bbdir, design, input_data["target"]) features = ["p2e"] for trace in ("itrace", "mtrace"): if input_data.get(trace, False): features.append(f"bebop-p2e/{trace}") build_env = { **os.environ, - **bebop_cargo_env(bbdir, chip), "VSRC_PATH": vsrc_dir, "OUT_PATH": str(build_path), - "CARGO_TARGET_DIR": os.path.join(bebop_dir, "target", f"{chip}-p2e{'-diff' if diff else ''}"), } if diff: features.append("bemu") - build_cmd = shlex.join([ - "nix", "develop", "--ignore-env", - "--keep-env-var", "HOME", - "--keep-env-var", "ALL_PROXY", - "--keep-env-var", "CARGO_TARGET_DIR", - "--keep-env-var", "VSRC_PATH", - "--keep-env-var", "OUT_PATH", - "-c", "env", "-u", "LD_LIBRARY_PATH", "cargo", "run", "--release", - "--manifest-path", str(manifest), - "--bin", "bebop", - "--features", ",".join(features), - "--", "build", "p2e", - "--rtl-dir", vsrc_dir, - "--out-dir", build_dir, - *(["--diff"] if diff else []), - *(["--itrace"] if input_data.get("itrace", False) else []), - *(["--mtrace"] if input_data.get("mtrace", False) else []), - *(["--stop-after", stop_after] if stop_after else []), - ]) + build_cmd = shlex.join( + [ + "nix", + "develop", + "--ignore-env", + "--keep-env-var", + "HOME", + "--keep-env-var", + "ALL_PROXY", + "--keep-env-var", + "VSRC_PATH", + "--keep-env-var", + "OUT_PATH", + "-c", + "env", + "-u", + "LD_LIBRARY_PATH", + "cargo", + "run", + "--release", + "--manifest-path", + str(manifest), + "--target-dir", + bebop_target_dir(bbdir, design, "p2e", selection=input_data["target"]), + "--bin", + "bebop", + "--features", + ",".join(features), + "--", + "build", + "p2e", + "--design-dir", + str(Path(bbdir) / "examples/designs" / design), + "--entry-dir", + str(Path(manifest).parent), + "--rtl-dir", + vsrc_dir, + "--out-dir", + build_dir, + *(["--diff"] if diff else []), + *(["--itrace"] if input_data.get("itrace", False) else []), + *(["--mtrace"] if input_data.get("mtrace", False) else []), + *(["--stop-after", stop_after] if stop_after else []), + ] + ) build_cmd = f"cd {shlex.quote(bbdir)} && {build_cmd}" ctx.logger.info("Building bebop p2e runtime case ...") build_result = await stream_run_logger_async( @@ -113,7 +149,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") - libvctb_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so") + libvctb_path = os.path.join( + build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm", "libvCtb.so" + ) bitstream_path = os.path.join(build_dir, "fpgaCompDir", "bitstream.bit") runtime_path = os.path.join(build_dir, "bebop-p2e") extra_fields = { @@ -129,9 +167,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: } if stop_after: extra_fields = { - "task": "resource-assessment", "stop_after": stop_after, - "runtime_ready": False, "vsrc_dir": vsrc_dir, "build_dir": build_dir, - "diff": diff, "timestamp": timestamp, + "task": "resource-assessment", + "stop_after": stop_after, + "runtime_ready": False, + "vsrc_dir": vsrc_dir, + "build_dir": build_dir, + "diff": diff, + "timestamp": timestamp, } else: extra_fields["runtime_ready"] = build_result.returncode == 0 diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py index b6d403566..1e3bef4c4 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_api.step.py @@ -1,4 +1,4 @@ -from pathlib import Path +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api from steps.bebop.p2e.scripts.runtime_case import validate_runtime_reuse @@ -14,35 +14,29 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - if not isinstance(body.get("reuse-runtime", False), bool): - return ApiResponse(status=400, body={"error": "reuse-runtime must be a boolean"}) - image = body.get("image", "") - manifest = body.get("load-manifest", "") - bitstream = body.get("bitstream", "") - if not all(isinstance(value, str) for value in (image, manifest, bitstream)): - return ApiResponse(status=400, body={"error": "image, load-manifest and bitstream must be strings"}) - if bool(image) == bool(manifest) or not bitstream: + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + if not body.get("image") or not body.get("bitstream"): return ApiResponse( - status=400, - body={ - "success": False, - "failure": True, - "returncode": 400, - "message": "exactly one of image and load-manifest, and bitstream, are required", - }, + status=400, body={"error": "image and bitstream are required"} ) - if manifest and body.get("diff", False): - return ApiResponse(status=400, body={"error": "load-manifest cannot use single-ELF diff"}) - if not Path(bitstream).is_file(): - return ApiResponse(status=400, body={"error": f"P2E bitstream is not a file: {bitstream}"}) + bitstream = body["bitstream"] if body.get("reuse-runtime", False): try: - validate_runtime_reuse(bitstream, bool(body.get("diff", False)), - bool(body.get("itrace", False)), bool(body.get("mtrace", False))) + validate_runtime_reuse( + bitstream, + bool(body.get("diff", False)), + bool(body.get("itrace", False)), + bool(body.get("mtrace", False)), + ) except (ValueError, OSError) as error: return ApiResponse(status=400, body={"error": str(error)}) - await ctx.enqueue({ - "topic": "bebop.p2e.runworkload", - "data": {**body, "_trace_id": ctx.trace_id}, - }) + await ctx.enqueue( + { + "topic": "bebop.p2e.runworkload", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index bd946d0d2..276e191f3 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -4,10 +4,14 @@ Loads a kernel image into FPGA and runs the workload via bebop CLI: 1. Resolve image name to .hex file under bb-tests/output/ 2. Validate bitstream .bit file path - 3. Run bebop run p2e --image --bitstream [--multi-fpga] [--wave] [--wave-start ] + 3. Run bebop --image --bitstream [--multi-fpga] [--wave] [--wave-start ] """ +from utils.path import runner_manifest + + import glob +import hashlib import os import re import shlex @@ -22,8 +26,8 @@ scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.event_common import require_chip -from utils.path import bebop_cargo_env, get_buckyball_path, log_dir +from utils.event_common import require_design +from utils.path import bebop_target_dir, bebop_bin_path, get_buckyball_path, log_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from resolve_image import resolve_image @@ -58,21 +62,20 @@ def case_uses_multi_fpga(build_dir: str) -> bool: async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) try: - chip = require_chip(input_data) + design = require_design(input_data) except ValueError as error: ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + extra_fields={"error": "missing_design"}, trace_id=origin_tid, ) return bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" image_name = input_data.get("image", "") - manifest_path = input_data.get("load-manifest", "") bitstream = input_data.get("bitstream", "") multi_fpga = bool(input_data.get("multi-fpga", False)) fpga_location = ( @@ -135,28 +138,20 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return wave = True try: - if bool(image_name) == bool(manifest_path): - raise ValueError("exactly one of image and load-manifest is required") - if manifest_path and diff: - raise ValueError("load-manifest cannot use single-ELF diff") - if manifest_path: - manifest_path = str(Path(manifest_path).resolve(strict=True)) - if not Path(manifest_path).is_file(): - raise ValueError("load-manifest must be a regular JSON file") - image_path = None - elf_path = None - image_name = Path(manifest_path).stem - else: - image_path = resolve_image(bbdir, image_name, chip) - image_base = os.path.splitext(image_path)[0] - elf_path = f"{image_base}.elf" if os.path.isfile(f"{image_base}.elf") else image_base - if diff and not os.path.isfile(elf_path): - raise ValueError(f"workload ELF for P2E DiffTest not found: {elf_path}") + image_path = resolve_image(bbdir, image_name, design) + image_base = os.path.splitext(image_path)[0] + elf_path = ( + f"{image_base}.elf" if os.path.isfile(f"{image_base}.elf") else image_base + ) except (ValueError, OSError, TypeError) as error: ctx.logger.error(str(error)) - await check_result(ctx, 1, continue_run=False, - extra_fields={"error": "invalid_cold_load_inputs", "message": str(error)}, - trace_id=origin_tid) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "invalid_cold_load_inputs", "message": str(error)}, + trace_id=origin_tid, + ) return if not bitstream or not os.path.isfile(bitstream): ctx.logger.error(f"bitstream .bit file not found: {bitstream}") @@ -185,7 +180,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"Detected multi-FPGA P2E case: {build_dir}") timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") run_log = log_dir( - bbdir, chip, "p2e", timestamp, "p2e", image_name, input_data.get("vsrc_dir") + bbdir, + design, + "p2e", + timestamp, + "p2e", + image_name, + input_data.get("vsrc_dir"), + selection=input_data["target"], ) os.makedirs(run_log, exist_ok=True) runtime_lib_dir = os.path.join(build_dir, "vvacDir", "runtimeDir", "lib", "lib_arm") @@ -209,18 +211,19 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - manifest = os.path.join( - bbdir, "examples", "chips", chip, "generated", "bebop", "Cargo.toml" - ) - build_env = {**os.environ, **bebop_cargo_env(bbdir, chip), "OUT_PATH": build_dir} - if diff: - build_env["CARGO_TARGET_DIR"] = os.path.join( - bebop_dir, "target", f"{chip}-p2e-diff" - ) + manifest = str(runner_manifest(bbdir, design, input_data["target"])) + build_env = {**os.environ, "OUT_PATH": build_dir} reuse_runtime = input_data.get("reuse-runtime", False) if reuse_runtime: - validate_runtime_reuse(bitstream, diff, bool(input_data.get("itrace", False)), bool(input_data.get("mtrace", False))) - ctx.logger.info(f"Reusing selected P2E runtime without rebuilding: {bebop_p2e_path}") + validate_runtime_reuse( + bitstream, + diff, + bool(input_data.get("itrace", False)), + bool(input_data.get("mtrace", False)), + ) + ctx.logger.info( + f"Reusing selected P2E runtime without rebuilding: {bebop_p2e_path}" + ) else: modes = trace_modes(bitstream) features = ["p2e"] @@ -239,8 +242,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--keep-env-var", "ALL_PROXY", "--keep-env-var", - "CARGO_TARGET_DIR", - "--keep-env-var", "OUT_PATH", "-c", "cargo", @@ -248,8 +249,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--release", "--manifest-path", manifest, + "--target-dir", + bebop_target_dir(bbdir, design, "p2e", selection=input_data["target"]), "--bin", - "bebop", + "bebop-p2e", "--features", ",".join(features), ] @@ -272,16 +275,50 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - target_release = os.path.join(build_env["CARGO_TARGET_DIR"], "release") + target_release = os.path.join( + bebop_target_dir(bbdir, design, "p2e", selection=input_data["target"]), + "release", + ) staged_runtime = f"{bebop_p2e_path}.new" - shutil.copy2(os.path.join(target_release, "bebop"), staged_runtime) + shutil.copy2(os.path.join(target_release, "bebop-p2e"), staged_runtime) os.replace(staged_runtime, bebop_p2e_path) - run_args = ["nix", "develop", "-c", "env", "-u", "LD_LIBRARY_PATH", bebop_p2e_path, - "run", "p2e", "--bitstream", bitstream, "--log-dir", run_log, - "--fpga-location", fpga_location] - run_args += ["--load-manifest", manifest_path] if manifest_path else ["--image", image_path] + rust_library = Path(target_release) / "deps/libbebop_p2e.so" + shutil.copy2(rust_library, Path(runtime_lib_dir) / "libbebop_p2e.so") + design_libraries = Path(bbdir) / "examples/designs" / design / "output/lib" + design_libraries.mkdir(parents=True, exist_ok=True) + shutil.copy2(rust_library, design_libraries / "libbebop_p2e.so") + installed_runtime = Path( + bebop_bin_path(bbdir, design, "p2e", selection=input_data["target"]) + ) + installed_runtime.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(bebop_p2e_path, installed_runtime) + run_args = [ + "nix", + "develop", + "-c", + "env", + "-u", + "LD_LIBRARY_PATH", + bebop_p2e_path, + "--bitstream", + bitstream, + "--log-dir", + run_log, + "--fpga-location", + fpga_location, + ] + run_args += ["--image", image_path] run_cmd = f"cd {shlex.quote(bbdir)} && {shlex.join(run_args)}" - report_context = source_context(bbdir, chip) + report_context = source_context(bbdir, design) + with open(bitstream, "rb") as source: + report_context["provenance"]["bitstream_sha256"] = hashlib.file_digest( + source, "sha256" + ).hexdigest() + with open(image_path, "rb") as source: + report_context["provenance"]["executable_sha256"] = hashlib.file_digest( + source, "sha256" + ).hexdigest() + if multi_fpga: run_cmd += " --multi-fpga" if wave: @@ -305,9 +342,22 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) uart_logs = sorted(Path(run_log).glob("uart_hart_*.log")) try: - report_path = performance_report(bbdir, report_context, origin_tid, "p2e", run_log, "\n".join(path.read_text() for path in uart_logs), run_result.returncode) + report_path = performance_report( + bbdir, + report_context, + origin_tid, + "p2e", + run_log, + "\n".join(path.read_text() for path in uart_logs), + run_result.returncode, + ) except (OSError, ValueError, KeyError) as error: - await check_result(ctx, 1, extra_fields={"task": "report", "log_dir": run_log, "error": str(error)}, trace_id=origin_tid) + await check_result( + ctx, + 1, + extra_fields={"task": "report", "log_dir": run_log, "error": str(error)}, + trace_id=origin_tid, + ) return await check_result( ctx, @@ -317,7 +367,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "task": "runworkload", "report_path": report_path, "image": image_path, - "load_manifest": manifest_path or None, "bitstream": bitstream, "build_dir": build_dir, "log_dir": run_log, diff --git a/bbdev/api/steps/bebop/p2e/04_batch_api.step.py b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py index 940d813d6..6e84c7e0e 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_api.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_api.step.py @@ -1,3 +1,4 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api config = { @@ -11,6 +12,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + try: + selection = require_target(body, ("device",)) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) bitstream = body.get("bitstream", "") if not bitstream: @@ -24,37 +29,48 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: }, ) - chip = body.get("chip") - if not chip: + design = body.get("design") + if not design: return ApiResponse( status=400, - body={"error": "Missing required parameter: --chip must be specified"} + body={"error": "Missing required parameter: --design must be specified"}, ) test_type = body.get("test") if not test_type: return ApiResponse( status=400, - body={"error": "Missing required parameter: --test must be specified (bare-tests or linux-tests)"} + body={ + "error": "Missing required parameter: --test must be specified (bare-tests or linux-tests)" + }, ) if test_type not in ["bare-tests", "linux-tests"]: return ApiResponse( status=400, - body={"error": f"Invalid test type: {test_type}. Must be 'bare-tests' or 'linux-tests'"} + body={ + "error": f"Invalid test type: {test_type}. Must be 'bare-tests' or 'linux-tests'" + }, ) fpga_location = body.get("fpga-location", "0.A") if not isinstance(fpga_location, str) or not fpga_location: - return ApiResponse(status=400, body={"error": "--fpga-location must be a location such as 1.A"}) + return ApiResponse( + status=400, body={"error": "--fpga-location must be a location such as 1.A"} + ) diff = bool(body.get("diff", False)) data = { - "chip": chip, + "design": design, "bitstream": bitstream, "test": test_type, "diff": diff, "fpga-location": fpga_location, } - await ctx.enqueue({"topic": "bebop.p2e.batch", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "bebop.p2e.batch", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py index a4053ad34..d10027b7a 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py @@ -7,11 +7,15 @@ 3. Run the P2E regression harness serially on one FPGA """ +from utils.path import runner_manifest +from pathlib import Path +from utils.path import design_output_root + + import os import re import shlex import sys -from pathlib import Path from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) @@ -20,7 +24,7 @@ bebop_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) if bebop_path not in sys.path: sys.path.insert(0, bebop_path) -from utils.path import bebop_cargo_env, get_buckyball_path, workloads_output_root +from utils.path import bebop_target_dir, get_buckyball_path, workloads_output_root from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from utils.workload_manifest import resolve_workload_toml @@ -69,23 +73,27 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - chip = input_data.get("chip") - if not chip: - ctx.logger.error("Missing required parameter: chip must be specified") + design = input_data.get("design") + if not design: + ctx.logger.error("Missing required parameter: design must be specified") await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + extra_fields={"error": "missing_design"}, trace_id=origin_tid, ) return - elf_root = workloads_output_root(bbdir) - cargo_env = bebop_cargo_env(bbdir, chip) + elf_root = str( + Path(design_output_root(bbdir, design, selection=input_data["target"])) + / "workloads" + ) test_type = input_data.get("test", "bare-tests") diff = bool(input_data.get("diff", False)) try: - workload_toml = resolve_workload_toml(chip, "p2e", test_type, bbdir, diff=diff) + workload_toml = resolve_workload_toml( + design, "p2e", test_type, bbdir, diff=diff, selection=input_data["target"] + ) except ValueError as e: ctx.logger.error(str(e)) await check_result( @@ -95,7 +103,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "error": "invalid_regression", "test": test_type, - "chip": chip, + "design": design, }, trace_id=origin_tid, ) @@ -104,23 +112,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"Running {test_type} with workload config: {workload_toml} diff={diff}" ) ctx.logger.info(f"P2E case dir (from bitstream): {build_dir}") - manifest = Path(bebop_dir) / "Cargo.toml" + manifest = runner_manifest(bbdir, design, input_data["target"]) features = ["p2e"] - runtime_env = {**os.environ.copy(), **cargo_env} + runtime_env = os.environ.copy() if diff: - manifest = ( - Path(bbdir) - / "examples" - / "chips" - / chip - / "generated" - / "bebop" - / "Cargo.toml" - ) features.append("bemu") - runtime_env["CARGO_TARGET_DIR"] = os.path.join( - bebop_dir, "target", f"{chip}-p2e-diff" - ) feature_arg = ",".join(features) rtcfg_path = os.path.join(build_dir, "vvacDir", "runtimeDir", "rtcfg") libvctb_path = os.path.join( @@ -144,7 +140,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return cargo_out = f'''--config="env.OUT_PATH='{build_dir}'"''' - build_cmd = f"nix develop -c cargo {cargo_out} build --release --manifest-path {shlex.quote(str(manifest))} --features {shlex.quote(feature_arg)} --tests" + build_cmd = ( + f"nix develop -c cargo {cargo_out} build --release --target-dir" + f" {shlex.quote(bebop_target_dir(bbdir, design, 'p2e', selection=input_data['target']))}" + f" --manifest-path {shlex.quote(str(manifest))} --features" + f" {shlex.quote(feature_arg)} --tests" + ) ctx.logger.info("Building bebop p2e (tests)...") build_result = await stream_run_logger_async( cmd=build_cmd, @@ -179,7 +180,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if diff: harness_args.append("--diff") harness = shlex.join(harness_args) - test_cmd = f"nix develop -c cargo {cargo_out} test --release --manifest-path {shlex.quote(str(manifest))} --features {shlex.quote(feature_arg)} --test test_p2e {harness}" + test_cmd = ( + f"nix develop -c cargo {cargo_out} test --release --target-dir" + f" {shlex.quote(bebop_target_dir(bbdir, design, 'p2e', selection=input_data['target']))}" + f" --manifest-path {shlex.quote(str(manifest))} --features" + f" {shlex.quote(feature_arg)} --test test_p2e {harness}" + ) ctx.logger.info(f"Running bebop p2e regression: {test_cmd}") run_result = await stream_run_logger_async( cmd=test_cmd, @@ -192,7 +198,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields = { "task": "batch", "backend": "p2e", - "chip": chip, + "design": design, "bitstream": bitstream, "build_dir": build_dir, "test_type": test_type, diff --git a/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py b/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py index db0c18264..ce47d7287 100644 --- a/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py +++ b/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py @@ -2,7 +2,7 @@ import os -def resolve_image(bbdir: str, image_name: str, chip: str) -> str: +def resolve_image(bbdir: str, image_name: str, design: str) -> str: image_name = image_name.replace(r"\_", "_") if os.path.isabs(image_name): raise ValueError( @@ -13,7 +13,7 @@ def resolve_image(bbdir: str, image_name: str, chip: str) -> str: bbdir, "bb-tests", "output", - chip, + design, "workloads", "**", f"{image_name}.hex", @@ -21,14 +21,14 @@ def resolve_image(bbdir: str, image_name: str, chip: str) -> str: recursive=True, ) kernel_image = os.path.join( - bbdir, "bb-tests", "output", "kernel", chip, f"{image_name}.hex" + bbdir, "stack", "output", "kernel", design, f"{image_name}.hex" ) if os.path.isfile(kernel_image): matches.append(kernel_image) if len(matches) != 1: raise ValueError( f"expected one image for {image_name!r} under " - f"bb-tests/output/{chip}/workloads/ or bb-tests/output/kernel/{chip}/, " + f"bb-tests/output/{design}/workloads/ or stack/output/kernel/{design}/, " f"found {len(matches)}" ) return matches[0] diff --git a/bbdev/api/steps/bebop/performance_report.py b/bbdev/api/steps/bebop/performance_report.py index 591462089..f4a270343 100644 --- a/bbdev/api/steps/bebop/performance_report.py +++ b/bbdev/api/steps/bebop/performance_report.py @@ -1,286 +1,13 @@ -import hashlib -import importlib.util -import json -import re -import tomllib from pathlib import Path -from utils.reports import new_report, attachment, finish_report -from utils.report_schema import number, nonempty, CATEGORIES - - -def parse_trace(data, categories): - groups = {} - for event in data["traceEvents"]: - if event.get("ph") != "X" or event.get("cat") != "buddy.trace": - continue - args = event["args"] - counter = args["counter"] - if counter not in ("riscv-cycle", "x86-tsc"): - raise ValueError(f"invalid trace counter: {counter}") - unit = "cycle" if counter == "riscv-cycle" else "tick" - path = args["id_path"] - if not isinstance(path, list) or not path: - raise ValueError("trace id_path must be a non-empty list") - for part in path: - number(part, "trace id", integer=True) - controller = int(number(args["controller"], "controller", integer=True)) - core = int(number(args["core"], "core", integer=True)) - platform = args["platform"] - if platform not in ("linux", "baremetal"): - raise ValueError(f"invalid trace platform: {platform}") - number(event["pid"], "pid", integer=True, positive=platform == "linux") - number(event["tid"], "tid", integer=True, positive=platform == "linux") - if platform == "baremetal" and event["pid"] != 0: - raise ValueError("baremetal trace requires pid=0") - key = (controller, core, tuple(path)) - identity = f"controller-{controller}/core-{core}/" + "-".join(map(str, path)) - start, end = args[f"start_{unit}"], args[f"end_{unit}"] - if any( - isinstance(value, bool) - or not isinstance(value, int) - or value < 0 - or value >= 2**64 - for value in (start, end) - ): - raise ValueError("counter boundaries must be unsigned 64-bit integers") - elapsed = number(args[f"elapsed_{unit}"], "elapsed_cycle", integer=True) - if end < start or end - start != elapsed or event["dur"] != elapsed: - raise ValueError(f"inconsistent cycle interval for {identity}") - name = nonempty(event["name"], "operator name") - category = categories.get(name, "Other") - if category not in CATEGORIES: - raise ValueError(f"unknown operator category: {category}") - groups.setdefault(key, []).append( - { - "id": identity, - "name": name, - "category": category, - "counter": counter, - "duration": int(elapsed), - "start": int(start), - "end": int(end), - "level": len(path) - 1, - } - ) - if not groups: - raise ValueError("no buddy.trace cycle intervals found") - if ( - len({record["counter"] for records in groups.values() for record in records}) - != 1 - ): - raise ValueError("performance trace cannot mix counter sources") - for key, records in groups.items(): - records.sort(key=lambda op: op["start"]) - for previous, current in zip(records, records[1:]): - if previous["name"] != current["name"]: - raise ValueError(f"trace scope has inconsistent names: {key}") - if previous["start"] + previous["duration"] > current["start"]: - raise ValueError(f"overlapping calls of trace scope {key}") - for (controller, core, path), records in groups.items(): - if len(path) == 1: - continue - for op in records: - parents = [ - parent - for (owner, _, parent_path), calls in groups.items() - if owner == controller and parent_path == path[:-1] - for parent in calls - if parent["start"] <= op["start"] - and op["start"] + op["duration"] <= parent["start"] + parent["duration"] - ] - if len(parents) != 1: - raise ValueError( - f"trace scope {controller}/{core}/{path} has {len(parents)} parents" - ) - start = min(record["start"] for records in groups.values() for record in records) - end = max( - record["start"] + record["duration"] - for records in groups.values() - for record in records - ) - if start == end: - raise ValueError("trace span must be positive") - operators = [] - for records in groups.values(): - first = records[0] - unit = "cycle" if first["counter"] == "riscv-cycle" else "tick" - plural = "cycles" if unit == "cycle" else "ticks" - operators.append( - { - "id": first["id"], - "name": first["name"], - "category": first["category"], - "counter": first["counter"], - "level": first["level"], - "calls": len(records), - plural: sum(record["duration"] for record in records), - f"start_{unit}": first["start"] - start, - f"end_{unit}": records[-1]["end"] - start, - } - ) - return int(end - start), operators +from utils.reports import new_report, finish_report def performance_report(repo, context, trace_id, backend, log, stdout, returncode): - repo, log = Path(repo), Path(log) - records = [ - json.loads(line.split("@BBPERF ", 1)[1]) - for line in stdout.splitlines() - if "@BBPERF " in line - ] - if not records: - report = new_report(context, f"{backend}.execution", trace_id, backend) - report["execution"] = { - "status": "pass" if returncode == 0 else "fail", - "returncode": returncode, - "performance": "not_collected", - } - return finish_report(report, log / "report") - models = {record["model"] for record in records} - if len(models) != 1: - raise ValueError("one model is required per performance run") - if len({record["unit"] for record in records}) != 1: - raise ValueError("benchmark throughput units differ") - model = models.pop() - if not re.fullmatch(r"[A-Za-z0-9_-]+", model): - raise ValueError("invalid benchmark model name") - recipe = repo / f"examples/models/{context['chip']}/{model}" - settings = tomllib.loads((recipe / "configs/report.toml").read_text()) - report = new_report(context, f"{backend}.performance", trace_id, backend) - output = log / "report" - report["clock_hz"] = settings.get(backend, {}).get("clock_hz") - expected = settings.get("expected_classes") - counters = {record["counter"] for record in records} - if len(counters) != 1 or not counters <= {"riscv-cycle", "x86-tsc"}: - raise ValueError("benchmark records require one explicit counter source") - benchmark_counter = counters.pop() - benchmark_count_field = "cycles" if benchmark_counter == "riscv-cycle" else "ticks" - correct = samples = elapsed = measured = 0 - for index, record in enumerate(records): - number(record["samples"], "samples", positive=True, integer=True) - number(record["elapsed_ns"], "elapsed_ns", positive=True, integer=True) - number( - record[benchmark_count_field], - benchmark_count_field, - positive=True, - integer=True, - ) - samples += record["samples"] - elapsed += record["elapsed_ns"] - other = "ticks" if benchmark_count_field == "cycles" else "cycles" - if other in record: - raise ValueError("benchmark record mixes cycle and tick fields") - measured += record[benchmark_count_field] - if "correct" in record: - number(record["correct"], "correct samples", integer=True) - if record["correct"] > record["samples"]: - raise ValueError("correct samples exceed sample count") - correct += record["correct"] - elif expected is not None: - if len(expected) != len(records): - raise ValueError("reference labels do not match the benchmark inputs") - if sum(record["predictions"].values()) != record["samples"]: - raise ValueError("prediction counts do not match sample count") - correct += record["predictions"].get(str(expected[index]), 0) - accuracy = ( - correct / samples - if expected is not None or all("correct" in record for record in records) - else None - ) - build = repo / f"stack/models/build/{context['chip']}/{model}" - workload = hashlib.sha256() - for file in sorted((build / "artifact").rglob("*")): - if file.is_file(): - workload.update(file.relative_to(build / "artifact").as_posix().encode()) - workload.update(file.read_bytes()) - run_config = recipe / "configs/running-param.toml" - inputs = tomllib.loads(run_config.read_text())["inputs"] - dataset = hashlib.sha256() - for value in inputs: - if isinstance(value, str) and (run_config.parent / value).is_file(): - dataset.update((run_config.parent / value).read_bytes()) - else: - dataset.update(json.dumps(value, sort_keys=True).encode()) - operators = {} - all_events = [] - trace_span = 0 - trace_counter = None - trace_roots = sorted( - { - file.parents[3] - for file in log.rglob("trace-*.txt") - if file.parent.name == "cycle" - } - ) - if trace_roots: - spec = importlib.util.spec_from_file_location( - "bbdev_perfetto", repo / "stack/serving/trace/perfetto.py" - ) - perfetto = importlib.util.module_from_spec(spec) - spec.loader.exec_module(perfetto) - metadata = [build / f"models/{model}/subgraph0.trace.mlir"] - for root in trace_roots: - data = perfetto.build_perfetto(root, recipe / "trace/trace.toml", metadata) - span, ops = parse_trace(data, settings.get("operator_categories", {})) - counter = ops[0]["counter"] - if trace_counter is not None and trace_counter != counter: - raise ValueError("performance roots cannot mix counter sources") - if counter != benchmark_counter: - raise ValueError("benchmark and trace counter sources differ") - trace_counter = counter - count_field = "cycles" if counter == "riscv-cycle" else "ticks" - offset = trace_span - trace_span += span - unit = "cycle" if counter == "riscv-cycle" else "tick" - for op in ops: - op[f"start_{unit}"] += offset - op[f"end_{unit}"] += offset - if op["id"] in operators: - operators[op["id"]][count_field] += op[count_field] - operators[op["id"]]["calls"] += op["calls"] - operators[op["id"]][f"end_{unit}"] = op[f"end_{unit}"] - else: - operators[op["id"]] = op - all_events.extend(data["traceEvents"]) - counter = trace_counter if trace_roots else benchmark_counter - count_field = "cycles" if counter == "riscv-cycle" else "ticks" - result = { - "counter": counter, - "name": model, + report = new_report(context, f"{backend}.execution", trace_id, backend) + report["execution"] = { "status": "pass" if returncode == 0 else "fail", - "workload_hash": workload.hexdigest(), - "dataset_hash": dataset.hexdigest(), - count_field: trace_span if trace_roots else measured, - "accuracy": accuracy, - "accuracy_metric": records[0].get( - "accuracy_metric", settings["accuracy_metric"] - ), - "latency_ms": elapsed / samples / 1e6, - "throughput": {"value": samples * 1e9 / elapsed, "unit": records[0]["unit"]}, - "error": None, - "trace_url": None, - "operators": list(operators.values()), - "benchmark": { - "samples": samples, - "elapsed_ns": elapsed, - "scope": "model_execution", - }, + "returncode": returncode, + "performance": "not_collected", } - if returncode != 0: - result.update( - **{count_field: None}, - accuracy=None, - latency_ms=None, - throughput=None, - operators=[], - error=f"model exited with {returncode}", - ) - elif all_events: - path = log / "perfetto.json" - path.write_text(json.dumps({"traceEvents": all_events}) + "\n") - result["trace_url"] = attachment( - report, output, path, f"model-{model}-perfetto.json" - ) - report["models"].append(result) - return finish_report(report, output) + return finish_report(report, Path(log) / "report") diff --git a/bbdev/api/steps/bebop/vcs/01_verilog_api.step.py b/bbdev/api/steps/bebop/vcs/01_verilog_api.step.py index 27b6bd565..77f7f9b23 100644 --- a/bbdev/api/steps/bebop/vcs/01_verilog_api.step.py +++ b/bbdev/api/steps/bebop/vcs/01_verilog_api.step.py @@ -1,10 +1,24 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design, require_target -config = {"name": "bebop-vcs-verilog-api", "flows": ["bebop"], "triggers": [api("POST", "/bebop/vcs/verilog")], "enqueues": ["vcs.verilog"]} +config = { + "name": "bebop-vcs-verilog-api", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/vcs/verilog")], + "enqueues": ["vcs.verilog"], +} async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - await ctx.enqueue({"topic": "vcs.verilog", "data": {"chip": require_chip(req.body or {}), "bebop_vcs": True, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "vcs.verilog", + "data": { + "design": require_design(req.body or {}), + "target": require_target(req.body or {}), + "_trace_id": ctx.trace_id, + }, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/vcs/02_build_api.step.py b/bbdev/api/steps/bebop/vcs/02_build_api.step.py index aee98a5c7..598e57a65 100644 --- a/bbdev/api/steps/bebop/vcs/02_build_api.step.py +++ b/bbdev/api/steps/bebop/vcs/02_build_api.step.py @@ -1,11 +1,31 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design -config = {"name": "bebop-vcs-build-api", "flows": ["bebop"], "triggers": [api("POST", "/bebop/vcs/build")], "enqueues": ["vcs.build"]} +config = { + "name": "bebop-vcs-build-api", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/vcs/build")], + "enqueues": ["vcs.build"], +} async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - await ctx.enqueue({"topic": "vcs.build", "data": {"chip": require_chip(body), "jobs": body.get("jobs", 16), "bebop_vcs": True, "_trace_id": ctx.trace_id}}) + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + await ctx.enqueue( + { + "topic": "vcs.build", + "data": { + "design": require_design(body), + "jobs": body.get("jobs", 16), + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/vcs/03_batch_api.step.py b/bbdev/api/steps/bebop/vcs/03_batch_api.step.py index 28e7aeb81..aac5c160e 100644 --- a/bbdev/api/steps/bebop/vcs/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/vcs/03_batch_api.step.py @@ -1,11 +1,40 @@ +from utils.event_common import require_target +import os + from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design -config = {"name": "bebop-vcs-batch-api", "flows": ["bebop"], "triggers": [api("POST", "/bebop/vcs/batch")], "enqueues": ["bebop.vcs.batch"]} +config = { + "name": "bebop-vcs-batch-api", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/vcs/batch")], + "enqueues": ["bebop.vcs.batch"], +} async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - await ctx.enqueue({"topic": "bebop.vcs.batch", "data": {"chip": require_chip(body), "test": body.get("test", "bare-tests"), "_trace_id": ctx.trace_id}}) + try: + selection = require_target(body, ("device",)) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + junit = body.get("junit") + if junit is not None and (not isinstance(junit, str) or not os.path.isabs(junit)): + return ApiResponse( + status=400, body={"error": "junit must be an absolute file path"} + ) + await ctx.enqueue( + { + "topic": "bebop.vcs.batch", + "data": { + "design": require_design(body), + "test": body.get("test", "bare-tests"), + "junit": junit, + "jobs": body.get("jobs", 1), + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/vcs/03_batch_event.step.py b/bbdev/api/steps/bebop/vcs/03_batch_event.step.py index 0015e6920..786663635 100644 --- a/bbdev/api/steps/bebop/vcs/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/vcs/03_batch_event.step.py @@ -1,62 +1,85 @@ -import os +from pathlib import Path import shlex -import subprocess -import sys -import tomllib from datetime import datetime -from pathlib import Path from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) -bebop_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -if bebop_path not in sys.path: - sys.path.insert(0, bebop_path) - +from utils.event_common import check_result, get_origin_trace_id, require_design +from utils.path import ( + get_buckyball_path, + design_output_root, + bebop_target_dir, + runner_manifest, + log_dir, +) from utils.workload_manifest import resolve_workload_toml -from utils.event_common import check_result, get_origin_trace_id, require_chip -from utils.path import get_buckyball_path, log_dir, rtl_dir, workloads_output_root -from utils.search_workload import search_workload from utils.stream_run import stream_run_logger_async -config = {"name": "bebop-vcs-batch", "flows": ["bebop"], "triggers": [queue("bebop.vcs.batch")], "enqueues": []} +config = { + "name": "bebop-vcs-batch", + "flows": ["bebop"], + "triggers": [queue("bebop.vcs.batch")], + "enqueues": [], +} async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) - try: - chip = require_chip(input_data) - bbdir = get_buckyball_path() - test_type = input_data.get("test", "bare-tests") - manifest = resolve_workload_toml(chip, "verilator", test_type, bbdir) - except ValueError as error: - await check_result(ctx, 1, extra_fields={"error": str(error)}, trace_id=origin_tid) - return - simv = Path(rtl_dir(bbdir, chip, "verilog")) / "vcs" / "simv" - if not simv.is_file(): - await check_result(ctx, 1, extra_fields={"error": "missing VCS simv: run bebop-vcs --verilog and --build first"}, trace_id=origin_tid) - return - tests = tomllib.loads(Path(manifest).read_text())["workloads"]["tests"] + design = require_design(input_data) + repo = get_buckyball_path() + selection = input_data["target"] + tests = input_data.get("test", "bare-tests") + manifest = resolve_workload_toml(design, "vcs", tests, repo, selection=selection) + output = Path(design_output_root(repo, design, selection=selection)) stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - report = Path(log_dir(bbdir, chip, "verilog", stamp, "bebop-vcs", test_type)) - report.mkdir(parents=True) - libc = subprocess.check_output(["g++", "-print-file-name=libc.so.6"], text=True).strip() - ball_names = [path.name for path in (Path(bbdir) / "examples" / "balls").iterdir() if path.is_dir()] - counts = {ball: [0, 0] for ball in ball_names} - passed = 0 - for name in tests: - ball = next((item for item in ball_names if f"-ctest-{item}" in name), "framework") - counts.setdefault(ball, [0, 0])[1] += 1 - binary = search_workload(workloads_output_root(bbdir), name) - if binary is None: - continue - command = " ".join(("env", f"LD_LIBRARY_PATH={shlex.quote(str(Path(libc).parent))}", shlex.quote(str(simv)), f"+elf={shlex.quote(binary)}", "+batch", "+trace=none", "+no-wave", f"+dramsim_ini_dir={shlex.quote(str(Path(bbdir) / 'result/share/dramsim3/configs'))}", f"+dramsim_outdir={shlex.quote(str(report / 'dramsim'))}")) - result = await stream_run_logger_async(cmd=command, logger=ctx.logger, cwd=str(simv.parent), stdout_prefix="bebop vcs batch", stderr_prefix="bebop vcs batch") - if result.returncode == 0: - counts[ball][0] += 1 - passed += 1 - index = report / "test-index.txt" - index.write_text("ball_dir vcs\n" + "\n".join(f"{ball} {ok}/{total}" for ball, (ok, total) in sorted(counts.items()) if total) + "\n") - await check_result(ctx, 0 if passed == len(tests) else 1, extra_fields={"task": "batch", "passed": passed, "total": len(tests), "test_index": str(index)}, trace_id=origin_tid) + report = ( + Path( + log_dir( + repo, design, "tapeout", stamp, "bebop-vcs", tests, selection=selection + ) + ) + / origin_tid + ) + command = [ + "cargo", + "test", + "--release", + "--manifest-path", + str(runner_manifest(repo, design, selection)), + "--target-dir", + bebop_target_dir(repo, design, "vcs", selection=selection), + "--test", + "test_vcs", + "--", + "--workload-toml", + manifest, + "--bb-tests-root", + str(output / "workloads"), + "--simv", + str(output / "vcs/simv"), + "--dram-configs", + str(Path(repo) / "result/share/dramsim3/configs"), + "--log-dir", + str(report), + "--jobs", + str(input_data.get("jobs", 1)), + ] + if input_data.get("junit"): + command.extend(["--junit", input_data["junit"]]) + result = await stream_run_logger_async( + cmd=shlex.join(command), + logger=ctx.logger, + cwd=repo, + stdout_prefix="bebop vcs batch", + stderr_prefix="bebop vcs batch", + ) + await check_result( + ctx, + result.returncode, + extra_fields={ + "task": "batch", + "backend": "vcs", + "log_dir": str(report), + "junit": input_data.get("junit"), + }, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/bebop/vcs/04_sim_api.step.py b/bbdev/api/steps/bebop/vcs/04_sim_api.step.py new file mode 100644 index 000000000..1ef7c2b48 --- /dev/null +++ b/bbdev/api/steps/bebop/vcs/04_sim_api.step.py @@ -0,0 +1,26 @@ +from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.event_common import require_design, require_target + +config = { + "name": "bebop-vcs-sim-api", + "flows": ["bebop"], + "triggers": [api("POST", "/bebop/vcs/sim")], + "enqueues": ["vcs.sim"], +} + + +async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = req.body or {} + await ctx.enqueue( + { + "topic": "vcs.sim", + "data": { + "design": require_design(body), + "target": require_target(body, ("device", "core")), + "binary": body["binary"], + "wave": body.get("wave", False), + "_trace_id": ctx.trace_id, + }, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py index 0126a91be..8c142eb57 100644 --- a/bbdev/api/steps/bebop/verilator/01_clean_api.step.py +++ b/bbdev/api/steps/bebop/verilator/01_clean_api.step.py @@ -1,6 +1,6 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip -from utils.path import get_buckyball_path, rtl_dir +from utils.event_common import require_design config = { "name": "bebop-verilator-clean-api", @@ -12,15 +12,20 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = req.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - build_dir = rtl_dir(bbdir, chip, "verilog", body.get("output_dir")) - data = {"chip": chip, "output_dir": build_dir} + data = {"design": design} await ctx.enqueue( - {"topic": "bebop.verilator.clean", "data": {**data, "_trace_id": ctx.trace_id}} + { + "topic": "bebop.verilator.clean", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py index ff645b680..62e35bfcc 100644 --- a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py +++ b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py @@ -5,7 +5,7 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -23,18 +23,23 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() try: - chip = require_chip(input_data) + design = require_design(input_data) except ValueError as error: ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + extra_fields={"error": "missing_design"}, trace_id=origin_tid, ) return - build_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("output_dir")) + build_dir = rtl_dir( + bbdir, + design, + "verilog", + selection=input_data["target"], + ) command = f"rm -rf {build_dir}" result = await stream_run_logger_async( cmd=command, @@ -47,15 +52,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, result.returncode, continue_run=input_data.get("from_run_workflow", False), - extra_fields={"task": "clean", "output_dir": build_dir}, + extra_fields={"task": "clean", "rtl_dir": build_dir}, trace_id=origin_tid, ) if input_data.get("from_run_workflow"): - data = { - k: v for k, v in input_data.items() if k not in ("output_dir", "vsrc_dir") - } - if input_data.get("_explicit_output_dir"): - data["output_dir"] = build_dir + data = dict(input_data) await ctx.enqueue( - {"topic": "bebop.verilator.run.verilog", "data": {**data, "task": "run"}} + { + "topic": "bebop.verilator.run.verilog", + "data": {**data, "task": "run", "target": input_data["target"]}, + } ) diff --git a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py index d405b618e..79f32514c 100644 --- a/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py +++ b/bbdev/api/steps/bebop/verilator/02_verilog_api.step.py @@ -1,5 +1,7 @@ +from utils.event_common import require_target + from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design config = { "name": "bebop-verilator-verilog-api", @@ -12,30 +14,31 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) if body.get("balltype"): return ApiResponse( status=400, body={"error": "ball-only verilog belongs to uvm; use bbdev uvm --verilog"}, ) try: - chip = require_chip(body) + design = require_design(body) except ValueError as e: return ApiResponse( status=400, body={ "status": "error", "message": str(e), - "example": 'bbdev bebop verilator --verilog "--chip toy"', + "example": 'bbdev bebop verilator --verilog "--design toy"', }, ) - data = {"chip": chip, "diff": bool(body.get("diff", False))} - data["main_only"] = bool(body.get("main-only", False)) - if body.get("output-dir"): - data["output_dir"] = body["output-dir"] + data = {"design": design, "diff": bool(body.get("diff", False))} await ctx.enqueue( { "topic": "bebop.verilator.verilog", - "data": {**data, "_trace_id": ctx.trace_id}, + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, } ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_api.step.py b/bbdev/api/steps/bebop/verilator/03_build_api.step.py index 4c20084ed..8e8fd04f0 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_api.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_api.step.py @@ -1,6 +1,6 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip -from utils.path import get_buckyball_path, rtl_dir +from utils.event_common import require_design config = { "name": "bebop-verilator-build-api", @@ -12,29 +12,24 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = request.body or {} - unsupported = [name for name in ("vsrc_dir", "output_dir") if name in body] - if unsupported: - return ApiResponse( - status=400, - body={ - "error": f"Unsupported parameter(s): {', '.join(unsupported)}. Use --vsrc-dir/--output-dir." - }, - ) try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) data = { - "chip": chip, + "design": design, "jobs": body.get("jobs", 16), "diff": bool(body.get("diff", False)), - "vsrc_dir": rtl_dir( - bbdir, chip, "verilog", body.get("vsrc-dir") or body.get("output-dir") - ), } await ctx.enqueue( - {"topic": "bebop.verilator.build", "data": {**data, "_trace_id": ctx.trace_id}} + { + "topic": "bebop.verilator.build", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index ec52e4119..5739b5a98 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -1,11 +1,15 @@ """ bebop verilator build event handler -Builds bebop with verilator feature and VSRC_PATH +Builds the entry's Verilator runner; bebop-verilator finds the entry's RTL by its output layout """ +from utils.path import runner_manifest + + import os import shlex +import shutil import sys from pathlib import Path from motia import FlowContext, queue @@ -16,8 +20,8 @@ scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.event_common import require_chip -from utils.path import bebop_cargo_env, bebop_target_dir, get_buckyball_path, rtl_dir +from utils.event_common import require_design +from utils.path import bebop_target_dir, bebop_bin_path, get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id from build_marker import write_build_marker @@ -34,40 +38,20 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - bebop_dir = f"{bbdir}/bebop" - chip = require_chip(input_data) - vsrc_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("vsrc_dir")) + design = require_design(input_data) + vsrc_dir = rtl_dir( + bbdir, + design, + "verilog", + selection=input_data["target"], + ) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") diff = bool(input_data.get("diff", False)) - build_dir = bebop_dir - manifest = Path(bbdir) / "bebop" / "Cargo.toml" + manifest = runner_manifest(bbdir, design, input_data["target"]) + build_dir = str(manifest.parent) features = ["verilator"] if diff: - manifest = ( - Path(bbdir) - / "examples" - / "chips" - / chip - / "generated" - / "bebop" - / "Cargo.toml" - ) - build_dir = str(manifest.parent) features.append("bemu") - dramsim_header = os.path.join(bbdir, "result", "include", "dramsim3.h") - if not os.path.isfile(dramsim_header): - ctx.logger.info( - f"Nix environment is missing {dramsim_header}; rebuilding result" - ) - env_result = await stream_run_logger_async( - cmd="nix build", - logger=ctx.logger, - cwd=bbdir, - stdout_prefix="bebop verilator difftest environment", - stderr_prefix="bebop verilator difftest environment", - ) - if env_result.returncode != 0: - raise RuntimeError(f"nix build failed: {env_result.returncode}") jobs = input_data.get("jobs", 16) cargo_build_cmd = shlex.join( [ @@ -77,7 +61,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--manifest-path", str(manifest), "--bin", - "bebop", + "bebop-verilator", + "--target-dir", + bebop_target_dir( + bbdir, design, "verilator", selection=input_data["target"] + ), "--features", ",".join(features), "--jobs", @@ -86,7 +74,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) build_inner = f"cd {shlex.quote(str(build_dir))} && exec {cargo_build_cmd}" build_cmd = f"nix develop -c sh -c {shlex.quote(build_inner)}" - env = {**os.environ, "VSRC_PATH": vsrc_dir, **bebop_cargo_env(bbdir, chip)} ctx.logger.info(f"Building bebop verilator (diff={diff}) ...") build_result = await stream_run_logger_async( cmd=build_cmd, @@ -94,12 +81,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cwd=bbdir, stdout_prefix="bebop verilator build", stderr_prefix="bebop verilator build", - env=env, ) - bebop_bin = os.path.join(bebop_target_dir(bbdir, chip), "release", "bebop") - target_dir = bebop_target_dir(bbdir, chip) + bebop_bin = bebop_bin_path( + bbdir, design, "verilator", selection=input_data["target"] + ) + target_dir = bebop_target_dir( + bbdir, design, "verilator", selection=input_data["target"] + ) if build_result.returncode == 0: - write_build_marker(target_dir, chip, vsrc_dir, bebop_bin, diff=diff) + Path(bebop_bin).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(Path(target_dir) / "release/bebop-verilator", bebop_bin) + write_build_marker(target_dir, design, vsrc_dir, bebop_bin, diff=diff) await check_result( ctx, build_result.returncode, @@ -109,7 +101,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "vsrc_dir": vsrc_dir, "binary": bebop_bin, "diff": diff, - "chip": chip, + "design": design, }, trace_id=origin_tid, ) @@ -121,9 +113,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "topic": "bebop.verilator.run.sim", "data": { **input_data, - "vsrc_dir": vsrc_dir, - "chip": chip, + "design": design, "task": "run", + "target": input_data["target"], }, } ) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_api.step.py b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py index ccff49aa8..495aa2314 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_api.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_api.step.py @@ -1,7 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip -from utils.path import get_buckyball_path +from utils.event_common import require_design config = { "name": "bebop-verilator-sim-api", @@ -14,8 +14,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - if "batch" in body: - return ApiResponse(status=400, body={"error": "Unsupported parameter: batch"}) + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) binary = body.get("binary", "") if not binary: return ApiResponse( @@ -28,15 +30,15 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: }, ) try: - chip = require_chip(body) + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - data = {**body, "chip": chip, "_trace_id": ctx.trace_id} - if "vsrc-dir" in data: - data["vsrc_dir"] = data.pop("vsrc-dir") - await ctx.enqueue({ - "topic": "bebop.verilator.sim", - "data": data, - }) + data = {**body, "design": design, "_trace_id": ctx.trace_id} + await ctx.enqueue( + { + "topic": "bebop.verilator.sim", + "data": {**(data), "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index 327446725..57d13c69b 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -4,6 +4,10 @@ Run a guest ELF through Verilator. """ +from pathlib import Path +from utils.path import design_output_root + + import json import os import shlex @@ -18,13 +22,13 @@ scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import ( bebop_target_dir, + bebop_bin_path, get_buckyball_path, log_dir, rtl_dir, - workloads_output_root, ) from utils.stream_run import stream_run_logger_async from utils.search_workload import search_workload @@ -41,42 +45,47 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: - if "batch" in input_data: - raise ValueError("Unsupported parameter: batch") origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() bebop_dir = f"{bbdir}/bebop" try: - chip = require_chip(input_data) + design = require_design(input_data) except ValueError as error: ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + extra_fields={"error": "missing_design"}, trace_id=origin_tid, ) return diff = bool(input_data.get("diff", False)) - vsrc_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("vsrc_dir")) + vsrc_dir = rtl_dir( + bbdir, + design, + "verilog", + selection=input_data["target"], + ) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") build_dir = bebop_dir - chip = input_data.get("chip") + design = input_data.get("design") if diff: - if not isinstance(chip, str) or not chip: + if not isinstance(design, str) or not design: ctx.logger.error( - "Missing required parameter: --chip (required with --diff)" + "Missing required parameter: --design (required with --diff)" ) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + extra_fields={"error": "missing_design"}, trace_id=origin_tid, ) return - bebop_bin = os.path.join(bebop_target_dir(bbdir, chip), "release", "bebop") + bebop_bin = bebop_bin_path( + bbdir, design, "verilator", selection=input_data["target"] + ) if not os.path.isfile(bebop_bin): ctx.logger.error(f"bebop binary does not exist: {bebop_bin}") await check_result( @@ -87,7 +96,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - target_dir = bebop_target_dir(bbdir, chip) + target_dir = bebop_target_dir( + bbdir, design, "verilator", selection=input_data["target"] + ) marker_path = build_marker_path(target_dir) try: marker = read_build_marker(target_dir) @@ -118,7 +129,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: expect_vsrc = os.path.abspath(vsrc_dir) expect_bin = os.path.abspath(bebop_bin) if ( - marker.get("config") != chip + marker.get("config") != design or marker.get("vsrc_dir") != expect_vsrc or marker.get("binary") != expect_bin or (bool(marker.get("diff", False)) != diff) @@ -142,11 +153,23 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return binary_name = input_data.get("binary", "") try: - binary_path = search_workload(workloads_output_root(bbdir), binary_name) + binary_path = search_workload( + str( + Path(design_output_root(bbdir, design, selection=input_data["target"])) + / "workloads" + ), + binary_name, + ) except ValueError as error: await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "binary_resolution_error", "binary": binary_name, "message": str(error)}, + ctx, + 1, + continue_run=False, + extra_fields={ + "error": "binary_resolution_error", + "binary": binary_name, + "message": str(error), + }, trace_id=origin_tid, ) return @@ -164,12 +187,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") run_log = log_dir( bbdir, - chip, + design, "verilog", timestamp, "verilator", binary_name, - input_data.get("vsrc_dir"), + selection=input_data["target"], ) os.makedirs(run_log, exist_ok=True) wave_arg = ( @@ -181,18 +204,28 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: for trace_name in ("itrace", "mtrace", "pmctrace", "ctrace", "banktrace"): if input_data.get(trace_name, False): trace_args += f" --{trace_name}" - run_cmd = f"{shlex.quote(bebop_bin)} run verilator --elf={shlex.quote(binary_path)} --log-dir={shlex.quote(run_log)}{(' --diff' if diff else '')}{wave_arg}{trace_args}" + run_cmd = ( + f"{shlex.quote(bebop_bin)} --elf={shlex.quote(binary_path)}" + f" --log-dir={shlex.quote(run_log)}{(' --diff' if diff else '')}{wave_arg}{trace_args}" + ) if diff: dram_library = f"{bbdir}/result/lib/libdramsim3.so" - dram_rpath = subprocess.check_output( - ["patchelf", "--print-rpath", dram_library], text=True - ).strip().split(os.pathsep) + dram_rpath = ( + subprocess.check_output( + ["patchelf", "--print-rpath", dram_library], text=True + ) + .strip() + .split(os.pathsep) + ) cpp_libraries = [ - os.path.join(path, "libstdc++.so.6") for path in dram_rpath + os.path.join(path, "libstdc++.so.6") + for path in dram_rpath if os.path.isfile(os.path.join(path, "libstdc++.so.6")) ] if len(cpp_libraries) != 1: - raise ValueError(f"DRAMSim requires one C++ runtime in its RPATH: {dram_rpath}") + raise ValueError( + f"DRAMSim requires one C++ runtime in its RPATH: {dram_rpath}" + ) preload = os.pathsep.join( ( path @@ -212,7 +245,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"export LD_PRELOAD={shlex.quote(preload)} && " f"exec {run_cmd}" ) - run_cmd = f"cd {shlex.quote(bbdir)} && nix develop -c sh -c {shlex.quote(run_inner)}" + run_cmd = ( + f"cd {shlex.quote(bbdir)} && nix develop -c sh -c {shlex.quote(run_inner)}" + ) ctx.logger.info(f"Running bebop verilator (diff={diff}): {run_cmd}") stdout_prefix = "bebop verilator difftest" if diff else "bebop verilator" stderr_prefix = stdout_prefix @@ -231,7 +266,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "task": "verilator", "backend": "guest-elf", "diff": diff, - "chip": chip, + "design": design, "binary": binary_path, "log_dir": run_log, "bank_diff": os.path.join(run_log, "diff.ndjson") if diff else None, diff --git a/bbdev/api/steps/bebop/verilator/05_run_api.step.py b/bbdev/api/steps/bebop/verilator/05_run_api.step.py index 25b138a8f..957d7d0d4 100644 --- a/bbdev/api/steps/bebop/verilator/05_run_api.step.py +++ b/bbdev/api/steps/bebop/verilator/05_run_api.step.py @@ -1,5 +1,6 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design config = { "name": "bebop-verilator-run-api", @@ -12,15 +13,12 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - if "batch" in body: - return ApiResponse(status=400, body={"error": "Unsupported parameter: batch"}) - if "output_dir" in body: - return ApiResponse( - status=400, - body={"error": "Unsupported parameter: output_dir. Use --output-dir."}, - ) try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) binary = body.get("binary", "") @@ -36,7 +34,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: ) diff = body.get("diff", False) data = { - "chip": chip, + "design": design, "binary": binary, "itrace": body.get("itrace", False), "mtrace": body.get("mtrace", False), @@ -48,14 +46,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "jobs": body.get("jobs", 16), "from_run_workflow": True, } - output_dir = body.get("output-dir") - if output_dir is not None: - data["output_dir"] = output_dir - data["_explicit_output_dir"] = True await ctx.enqueue( { "topic": "bebop.verilator.run.clean", - "data": {**data, "_trace_id": ctx.trace_id}, + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, } ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py index abfe23394..2344e2fb2 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -1,6 +1,7 @@ +from utils.event_common import require_target +import os from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip -from utils.path import get_buckyball_path, rtl_dir +from utils.event_common import require_design config = { "name": "bebop-verilator-batch-api", @@ -12,13 +13,21 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = request.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device",)) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) diff = bool(body.get("diff", False)) + junit = body.get("junit") + if junit is not None and (not isinstance(junit, str) or not os.path.isabs(junit)): + return ApiResponse( + status=400, body={"error": "junit must be an absolute file path"} + ) test_type = body.get("test") if not test_type: return ApiResponse( @@ -30,18 +39,22 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: if test_type not in ("bare-tests", "linux-tests"): return ApiResponse( status=400, - body={"error": f"Invalid test type: {test_type}. Must be 'bare-tests' or 'linux-tests'"}, + body={ + "error": f"Invalid test type: {test_type}. Must be 'bare-tests' or 'linux-tests'" + }, ) - vsrc_dir = rtl_dir(bbdir, chip, "verilog", body.get("vsrc_dir")) data = { - "chip": chip, - "vsrc_dir": vsrc_dir, + "design": design, "test": test_type, + "junit": junit, "jobs": body.get("jobs", 1), "clean-before": body.get("clean-before", body.get("clean_before", False)), "diff": diff, } await ctx.enqueue( - {"topic": "bebop.verilator.batch", "data": {**data, "_trace_id": ctx.trace_id}} + { + "topic": "bebop.verilator.batch", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index b72afeaf4..3ed3f88a4 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -4,6 +4,11 @@ Runs bebop verilator batch regression (requires prior --build). """ +from pathlib import Path +from utils.path import design_output_root +from utils.path import runner_manifest + + import os import shlex import sys @@ -15,12 +20,11 @@ bebop_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) if bebop_path not in sys.path: sys.path.insert(0, bebop_path) -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import ( - bebop_cargo_env, + bebop_target_dir, get_buckyball_path, rtl_dir, - workloads_output_root, ) from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -38,25 +42,32 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - bebop_dir = f"{bbdir}/bebop" try: - chip = require_chip(input_data) + design = require_design(input_data) except ValueError as error: ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + extra_fields={"error": "missing_design"}, trace_id=origin_tid, ) return - elf_root = workloads_output_root(bbdir) + elf_root = str( + Path(design_output_root(bbdir, design, selection=input_data["target"])) + / "workloads" + ) test_type = input_data.get("test", "bare-tests") diff = bool(input_data.get("diff", False)) try: workload_toml = resolve_workload_toml( - chip, "verilator", test_type, bbdir, diff=diff + design, + "verilator", + test_type, + bbdir, + diff=diff, + selection=input_data["target"], ) except ValueError as e: ctx.logger.error(str(e)) @@ -67,7 +78,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "error": "invalid_regression", "test": test_type, - "chip": chip, + "design": design, }, trace_id=origin_tid, ) @@ -75,16 +86,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info( f"Running {test_type} with workload config: {workload_toml} diff={diff}" ) - vsrc_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("vsrc_dir")) - vsrc_config = shlex.quote(f"env.VSRC_PATH='{vsrc_dir}'") - env = os.environ.copy() - env.update(bebop_cargo_env(bbdir, chip)) - env.update({"VSRC_PATH": vsrc_dir}) - manifest = f"{bebop_dir}/Cargo.toml" - features = "verilator" - if diff: - manifest = f"{bbdir}/examples/chips/{chip}/generated/bebop/Cargo.toml" - features = "verilator,bemu" + vsrc_dir = rtl_dir( + bbdir, + design, + "verilog", + selection=input_data["target"], + ) + manifest = str(runner_manifest(bbdir, design, input_data["target"])) + features = "verilator,bemu" if diff else "verilator" harness_args = [ "--", "--workload-toml", @@ -92,7 +101,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--bb-tests-root", elf_root, "--arch-config", - chip, + design, "--jobs", str(input_data.get("jobs", 1)), ] @@ -100,8 +109,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: harness_args.append("--clean-before") if diff: harness_args.append("--diff") + if input_data.get("junit"): + harness_args.extend(["--junit", input_data["junit"]]) harness = shlex.join(harness_args) - test_cmd = f"nix develop -c cargo --config={vsrc_config} test --release --manifest-path {shlex.quote(manifest)} --features {shlex.quote(features)} --test test_verilator {harness}" + test_cmd = ( + "nix develop -c cargo test --release --target-dir" + f" {shlex.quote(bebop_target_dir(bbdir, design, 'verilator', selection=input_data['target']))}" + f" --manifest-path {shlex.quote(manifest)} --features {shlex.quote(features)}" + f" --test test_verilator {harness}" + ) ctx.logger.info(f"Running bebop verilator regression: {test_cmd}") run_result = await stream_run_logger_async( cmd=test_cmd, @@ -109,7 +125,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: cwd=bbdir, stdout_prefix="bebop verilator batch", stderr_prefix="bebop verilator batch", - env=env, ) await check_result( ctx, @@ -118,7 +133,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "task": "batch", "backend": "difftest" if diff else "verilator", - "chip": chip, + "design": design, "vsrc_dir": vsrc_dir, "test_type": test_type, "diff": diff, diff --git a/bbdev/api/steps/compiler/01_build_api.step.py b/bbdev/api/steps/compiler/01_build_api.step.py deleted file mode 100644 index 44a6fecff..000000000 --- a/bbdev/api/steps/compiler/01_build_api.step.py +++ /dev/null @@ -1,30 +0,0 @@ -import os -import re -import sys - -from motia import ApiRequest, ApiResponse, FlowContext, api - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path - -config = { - "name": "build-compiler-api", - "description": "build compiler", - "flows": ["compiler"], - "triggers": [api("POST", "/compiler/build")], - "enqueues": ["compiler.build"], -} - - -async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - body = request.body or {} - if "core" in body: - raise ValueError("compiler build takes --chip, not --core") - chip = body.get("chip") - if not isinstance(chip, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", chip): - raise ValueError(f"Invalid compiler chip: {chip}") - await ctx.enqueue({"topic": "compiler.build", "data": {**body, "_trace_id": ctx.trace_id}}) - return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/compiler/01_build_event.step.py b/bbdev/api/steps/compiler/01_build_event.step.py deleted file mode 100644 index 432ccc861..000000000 --- a/bbdev/api/steps/compiler/01_build_event.step.py +++ /dev/null @@ -1,60 +0,0 @@ -import asyncio -import importlib.util -import os -import re -import sys - -from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.path import get_buckyball_path - -_compiler_build_path = os.path.join(os.path.dirname(__file__), "scripts", "build.py") -_spec = importlib.util.spec_from_file_location("compiler_build_module", _compiler_build_path) -if _spec is None or _spec.loader is None: - raise RuntimeError(f"cannot load {_compiler_build_path}") -compiler_build = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(compiler_build) - -from utils.event_common import check_result, get_origin_trace_id - -config = { - "name": "build-compiler", - "description": "build compiler", - "flows": ["compiler"], - "triggers": [queue("compiler.build")], - "enqueues": [], -} - - -async def handler(input_data: dict, ctx: FlowContext) -> None: - origin_tid = get_origin_trace_id(input_data, ctx) - bbdir = get_buckyball_path() - - chip = input_data.get("chip") - if not isinstance(chip, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", chip): - raise ValueError(f"Invalid compiler chip: {chip}") - - try: - await asyncio.to_thread( - compiler_build.build_compiler, - bbdir, - chip=chip, - logger=ctx.logger, - task_scope=origin_tid, - ) - except Exception as exc: - ctx.logger.error(f"Compiler build failed: {exc}") - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"task": "build", "chip": chip, "error": str(exc)}, - trace_id=origin_tid, - ) - return - - await check_result(ctx, 0, continue_run=False, trace_id=origin_tid) diff --git a/bbdev/api/steps/compiler/scripts/build.py b/bbdev/api/steps/compiler/scripts/build.py deleted file mode 100644 index a4530078a..000000000 --- a/bbdev/api/steps/compiler/scripts/build.py +++ /dev/null @@ -1,231 +0,0 @@ -from __future__ import annotations - -import argparse -import fcntl -import os -import re -import shlex -import shutil -import subprocess -from pathlib import Path - - -def _run( - cmd: list[str], - *, - repo: Path, - logger: object | None = None, - task_scope: str | None = None, - output_prefix: str, -) -> None: - cmd = ["nix", "develop", str(repo), "-c", *cmd] - if logger is None: - result = subprocess.run(cmd, cwd=repo) - else: - from utils.stream_run import stream_run_logger - - result = stream_run_logger( - cmd=shlex.join(cmd), - logger=logger, - cwd=str(repo), - stdout_prefix=output_prefix, - stderr_prefix=output_prefix, - task_scope=task_scope, - ) - if result.returncode != 0: - detail = result.stderr.strip() if logger is not None else "" - if not detail and logger is not None: - lines = result.stdout.splitlines() - failed = [i for i, line in enumerate(lines) if "FAILED:" in line] - if failed: - start = failed[0] - detail = "\n".join(lines[start:]) - else: - detail = "\n".join(lines[-20:]) - if detail: - raise RuntimeError( - f"command failed ({result.returncode}): {' '.join(cmd)}\n{detail}" - ) - raise RuntimeError(f"command failed ({result.returncode}): {' '.join(cmd)}") - - -def compiler_python(repo: str | Path) -> str: - root = Path(repo).resolve() - candidates = [root / "result" / "bin" / "python3"] - path_python = shutil.which("python3") - if path_python: - candidates.append(Path(path_python)) - for candidate in candidates: - if not candidate.is_file(): - continue - probe = subprocess.run( - [str(candidate), "-c", "import nanobind"], capture_output=True - ) - if probe.returncode == 0: - return str(candidate) - checked = ", ".join(str(candidate) for candidate in candidates) - raise RuntimeError(f"python3 with nanobind not found (checked: {checked})") - - -def build_llvm( - repo: str | Path, *, logger: object | None = None, task_scope: str | None = None -) -> Path: - root = Path(repo).resolve() - buddy = root / "stack" / "compiler" / "thirdparty" / "buddy-mlir" - llvm_src = buddy / "llvm" / "llvm" - llvm_build = buddy / "llvm" / "build" - if not llvm_src.is_dir(): - raise RuntimeError(f"missing LLVM source: {llvm_src}") - - python = compiler_python(root) - cmake = [ - "cmake", - "-G", - "Ninja", - "-S", - str(llvm_src), - "-B", - str(llvm_build), - "-DLLVM_ENABLE_PROJECTS=mlir;clang", - "-DLLVM_ENABLE_RUNTIMES=openmp", - "-DLLVM_TARGETS_TO_BUILD=host;RISCV", - "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DOPENMP_ENABLE_LIBOMPTARGET=OFF", - "-DLIBOMP_LIBFLAGS=-lrt", - "-DCMAKE_BUILD_TYPE=RELEASE", - "-DMLIR_ENABLE_BINDINGS_PYTHON=ON", - f"-DPython3_EXECUTABLE={python}", - f"-DPython_EXECUTABLE={python}", - ] - llvm_build.mkdir(parents=True, exist_ok=True) - with open(llvm_build / ".bbdev.lock", "a+b") as lockf: - fcntl.flock(lockf.fileno(), fcntl.LOCK_EX) - if not (llvm_build / "build.ninja").is_file(): - _run( - cmake, - repo=root, - logger=logger, - task_scope=task_scope, - output_prefix="compiler llvm configure", - ) - _run( - ["ninja", "-C", str(llvm_build), "-j", str(os.cpu_count() or 1)], - repo=root, - logger=logger, - task_scope=task_scope, - output_prefix="compiler llvm build", - ) - mlir_cmake = llvm_build / "lib" / "cmake" / "mlir" - if not mlir_cmake.is_dir(): - raise RuntimeError(f"LLVM/MLIR build failed: missing {mlir_cmake}") - return llvm_build - - -def compiler_build_dir(repo: str | Path, instance: str) -> Path: - return ( - Path(repo).resolve() - / "stack" - / "compiler" - / "thirdparty" - / "buddy-mlir" - / "build" - / instance - ) - - -def build_compiler( - repo: str | Path, - *, - chip: str, - instance: str | None = None, - chip_pb: str | Path | None = None, - logger: object | None = None, - task_scope: str | None = None, -) -> Path: - root = Path(repo).resolve() - if not chip: - raise ValueError("chip is required") - build_instance = instance or chip - if not re.fullmatch(r"[A-Za-z0-9_-]+", build_instance): - raise ValueError(f"invalid compiler build instance: {build_instance}") - if (instance is None) != (chip_pb is None): - raise ValueError("compiler variant requires both instance and chip_pb") - selected_pb = ( - Path(chip_pb).resolve() - if chip_pb is not None - else root / "examples" / "chips" / chip / "configs" / "generated" / "chip.pb" - ) - if not selected_pb.is_file(): - raise RuntimeError(f"missing {selected_pb}; generate the requested chip.pb first") - buddy = root / "stack" / "compiler" / "thirdparty" / "buddy-mlir" - llvm = build_llvm(root, logger=logger, task_scope=task_scope) - python = compiler_python(root) - build = compiler_build_dir(root, build_instance) - build.mkdir(parents=True, exist_ok=True) - cmake = [ - "cmake", - "-G", - "Ninja", - "-S", - str(buddy), - "-B", - str(build), - f"-DBUDDY_EXTERNAL_DIALECTS_DIR={root / 'stack' / 'compiler'}", - f"-DBUCKYBALL_CHIP_PB={selected_pb}", - f"-DMLIR_DIR={llvm / 'lib' / 'cmake' / 'mlir'}", - f"-DLLVM_DIR={llvm / 'lib' / 'cmake' / 'llvm'}", - "-DLLVM_ENABLE_ASSERTIONS=ON", - "-DCMAKE_BUILD_TYPE=RELEASE", - "-DBUDDY_MLIR_ENABLE_PYTHON_PACKAGES=ON", - f"-DPython3_EXECUTABLE={python}", - f"-DPython_EXECUTABLE={python}", - ] - _run( - cmake, - repo=root, - logger=logger, - task_scope=task_scope, - output_prefix="compiler configure", - ) - _run( - [ - "ninja", - "-C", - str(build), - "-j", - str(os.cpu_count() or 1), - "buddy-opt", - "buddy-translate", - "buddy-llc", - "rax-pack", - "python-package-buddy", - "BuddyMLIRPythonModules", - ], - repo=root, - logger=logger, - task_scope=task_scope, - output_prefix="compiler build", - ) - for tool in ("buddy-opt", "buddy-translate", "buddy-llc", "rax-pack"): - if not (build / "bin" / tool).is_file(): - raise RuntimeError(f"compiler build failed: missing {build / 'bin' / tool}") - return build - - -def main() -> None: - parser = argparse.ArgumentParser(description="Build a chip compiler instance") - parser.add_argument("--repo", type=Path, required=True) - parser.add_argument("--chip", required=True) - parser.add_argument("--instance") - parser.add_argument("--chip-pb", type=Path) - args = parser.parse_args() - build_compiler( - args.repo, - chip=args.chip, - instance=args.instance, - chip_pb=args.chip_pb, - ) - - -if __name__ == "__main__": - main() diff --git a/bbdev/api/steps/config/.gitignore b/bbdev/api/steps/config/.gitignore index e3efcb307..08616723a 100644 --- a/bbdev/api/steps/config/.gitignore +++ b/bbdev/api/steps/config/.gitignore @@ -1 +1 @@ -scripts/chip_pb2.py +scripts/design_pb2.py diff --git a/bbdev/api/steps/config/01_install_api.step.py b/bbdev/api/steps/config/01_install_api.step.py index 244b95a58..6f5f8fb03 100644 --- a/bbdev/api/steps/config/01_install_api.step.py +++ b/bbdev/api/steps/config/01_install_api.step.py @@ -1,8 +1,9 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api config = { "name": "config-install-api", - "description": "install all chip configs under examples/chips", + "description": "install all design configs under examples/designs", "flows": ["config"], "triggers": [api("POST", "/config/install")], "enqueues": ["config.install"], @@ -11,10 +12,24 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - unknown = set(body) - {"chip"} + if "design" in body: + try: + require_target(body) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + unknown = set(body) - {"design", "device", "core"} if unknown: - return ApiResponse(status=400, body={"error": f"Unexpected parameters: {sorted(unknown)}"}) - if "chip" in body and (not isinstance(body["chip"], str) or not body["chip"].replace("_", "").replace("-", "").isalnum()): - return ApiResponse(status=400, body={"error": "--chip requires a chip name"}) - await ctx.enqueue({"topic": "config.install", "data": {**body, "_trace_id": ctx.trace_id}}) + return ApiResponse( + status=400, body={"error": f"Unexpected parameters: {sorted(unknown)}"} + ) + if "design" in body and ( + not isinstance(body["design"], str) + or not body["design"].replace("_", "").replace("-", "").isalnum() + ): + return ApiResponse( + status=400, body={"error": "--design requires a design name"} + ) + await ctx.enqueue( + {"topic": "config.install", "data": {**body, "_trace_id": ctx.trace_id}} + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/config/01_install_event.step.py b/bbdev/api/steps/config/01_install_event.step.py index bd356595c..0fbe3905d 100644 --- a/bbdev/api/steps/config/01_install_event.step.py +++ b/bbdev/api/steps/config/01_install_event.step.py @@ -3,6 +3,7 @@ import os import shlex import sys +import tempfile from pathlib import Path from motia import FlowContext, queue @@ -19,7 +20,7 @@ config = { "name": "config-install", - "description": "install all chip configs under examples/chips", + "description": "install all design configs under examples/designs", "flows": ["config"], "triggers": [queue("config.install")], "enqueues": [], @@ -39,113 +40,91 @@ def _load_script(name: str): async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = Path(get_buckyball_path()) - chips = bbdir / "examples" / "chips" + designs = bbdir / "examples" / "designs" - selected = input_data.get("chip") - tops = [chips / selected / "configs" / "chip.toml"] if selected else sorted(chips.glob("*/configs/chip.toml")) + selected = input_data.get("design") + tops = ( + [designs / selected / "configs" / "design.toml"] + if selected + else sorted(designs.glob("*/configs/design.toml")) + ) if any(not top.is_file() for top in tops): - raise ValueError(f"unknown chip: {selected}") + raise ValueError(f"design {selected!r} has no configs/design.toml assembly") if not tops: - raise ValueError(f"no configs/chip.toml under {chips}") + raise ValueError(f"no configs/design.toml under {designs}") toml2json = _load_script("1_toml2json.py") derive = _load_script("2_parameter_derivation.py") - # step 1: generate chip config JSONs from toml files + # step 1: generate design config JSONs from toml files json_outs: list[Path] = [] for top in tops: - data = toml2json.toml2json(top) - out = top.parent / "generated" / "config" / "config.json" + data = toml2json.design2json(top) + out = top.parent.parent / "output" / "generated" / "config" / "config.json" out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + out.write_text( + json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) json_outs.append(out) - ctx.logger.info(f"Wrote chip config JSON: {out}") + ctx.logger.info(f"Wrote design config JSON: {out}") # step 2: derive computed parameters from config JSONs derived_outs: list[Path] = [] for top, json_path in zip(tops, json_outs): - chip = top.parent.parent.name + design = top.parent.parent.name data = json.loads(json_path.read_text(encoding="utf-8")) derived_path = json_path.parent / "derived.json" - derive.write_derived(data, bbdir, chip, derived_path) + derive.write_derived(data, bbdir, design, derived_path) derived_outs.append(derived_path) ctx.logger.info(f"Wrote derived config: {derived_path}") - # step 3.1: generate chip_pb2.py from chip.proto + # step 3.1: generate design_pb2.py from design.proto scripts = Path(scripts_dir) - cmd = ( - f"protoc -I{shlex.quote(str(scripts / 'proto'))} " - f"--python_out={shlex.quote(str(scripts))} " - f"{shlex.quote(str(scripts / 'proto' / 'chip.proto'))}" - ) - ctx.logger.info(f"Running protoc: {cmd}") - result = await stream_run_logger_async( - cmd=cmd, - logger=ctx.logger, - stdout_prefix="protoc", - stderr_prefix="protoc", - task_scope=origin_tid, - ) - if result.returncode != 0: - raise RuntimeError(f"protoc failed with returncode={result.returncode}") - ctx.logger.info(f"Generated chip python bindings: {scripts / 'chip_pb2.py'}") - - # step 3.2: generate chip.pb for all components build + with tempfile.TemporaryDirectory(prefix="bbdev-proto-") as generated: + schemas = [ + scripts / "proto" / "design.proto", + scripts / "proto" / "hardware.proto", + *sorted( + Path(bbdir).glob("examples/components/balls/*/configs/*_params.proto") + ), + *sorted( + Path(bbdir).glob("examples/components/cores/*/configs/*_params.proto") + ), + ] + includes = sorted({schema.parent for schema in schemas}) + cmd = " ".join( + [ + "protoc", + *(f"-I{shlex.quote(str(directory))}" for directory in includes), + f"--python_out={shlex.quote(generated)}", + *(shlex.quote(str(schema)) for schema in schemas), + ] + ) + ctx.logger.info(f"Running protoc: {cmd}") + result = await stream_run_logger_async( + cmd=cmd, + logger=ctx.logger, + stdout_prefix="protoc", + stderr_prefix="protoc", + task_scope=origin_tid, + ) + if result.returncode != 0: + raise RuntimeError(f"protoc failed with returncode={result.returncode}") + for source in Path(generated).glob("*_pb2.py"): + content = source.read_bytes() + destination = scripts / source.name + if not destination.is_file() or destination.read_bytes() != content: + destination.write_bytes(content) + ctx.logger.info(f"Generated design python bindings: {scripts / 'design_pb2.py'}") + + # step 3.2: design.pb, the one input of every build system json2proto = _load_script("3_json2proto.py") pb_outs: list[Path] = [] for json_path, derived_path in zip(json_outs, derived_outs): - out = json_path.parent / "chip.pb" - json2proto.write_chip_pb(json_path, derived_path, out, bbdir) + out = json_path.parent.parent / "design.pb" + json2proto.write_design_pb(json_path, derived_path, out, bbdir) pb_outs.append(out) - ctx.logger.info(f"Wrote chip protobuf: {out}") - - # step 4: translate PB to different build systems - step4 = _load_script("4_install.py") - installed: list[dict[str, str]] = [] - for top, src_pb, json_path in zip(tops, pb_outs, json_outs): - chip_name = top.parent.parent.name - chip = step4.load_chip(src_pb) - gen = top.parent / "generated" - - # A. TO Arch BUILD SYSTEM - arch = step4.install_arch(src_pb, gen) - ctx.logger.info(f"Installed arch pb: {arch}") - - # B. TO BEMU BUILD SYSTEM - bemu = step4.install_bemu(chip, bbdir, gen) - ctx.logger.info(f"Installed bemu crate: {bemu}") - - bebop = step4.install_bebop(gen) - ctx.logger.info(f"Installed bebop crate: {bebop}") - - # C. TO Workload BUILD SYSTEM - workload = step4.install_workload(chip, bbdir, chip_name, gen) - ctx.logger.info(f"Installed workload defs: {workload}") - - default_design = Path(json.loads(json_path.read_text(encoding="utf-8"))["designs"]["_file"]).resolve() - designs_dir = top.parent / "designs" - if not designs_dir.is_dir(): - raise ValueError(f"missing {designs_dir}") - for design in sorted(designs_dir.glob("*.toml")): - if design.resolve() == default_design: - continue - vdata = toml2json.toml2json(top, design=design) - vjson = gen / "config" / f"{design.stem}.json" - vjson.write_text(json.dumps(vdata, indent=2, sort_keys=True) + "\n", encoding="utf-8") - vderived = gen / "config" / f"{design.stem}.derived.json" - derive.write_derived(vdata, bbdir, chip_name, vderived) - vpb = json2proto.write_chip_pb(vjson, vderived, gen / f"{design.stem}.pb", bbdir) - ctx.logger.info(f"Installed arch variant pb: {vpb}") - - installed.append( - { - "chip": chip_name, - "arch": str(arch), - "bemu": str(bemu), - "bebop": str(bebop), - "workload": str(workload), - } - ) + ctx.logger.info(f"Wrote design protobuf: {out}") await check_result( ctx, @@ -155,8 +134,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "task": "install", "json": [str(p) for p in json_outs], "derived": [str(p) for p in derived_outs], - "pb2": str(scripts / "chip_pb2.py"), - "install": installed, + "pb2": str(scripts / "design_pb2.py"), + "design_pb": [str(p) for p in pb_outs], }, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/config/scripts/1_toml2json.py b/bbdev/api/steps/config/scripts/1_toml2json.py index 671b822ea..5ee24f0dc 100644 --- a/bbdev/api/steps/config/scripts/1_toml2json.py +++ b/bbdev/api/steps/config/scripts/1_toml2json.py @@ -25,6 +25,57 @@ def _load(path: Path) -> dict[str, Any]: return data +IP_DEFAULTS = { + "uncached": "arch/src/main/scala/framework/core/memory/uncached/configs/default.toml", + "fetch": "arch/src/main/scala/framework/core/execution/fetch/src/main/scala/configs/default.toml", + "fpu": "arch/src/main/scala/framework/core/execution/fpu/src/main/scala/configs/default.toml", + "controller": "arch/src/main/scala/framework/core/controller/configs/default.toml", + "axi": "arch/src/main/scala/framework/ip/axi4/configs/default.toml", + "requesters": "arch/src/main/scala/framework/ip/coherence/configs/requesters.toml", + "ddr": "arch/src/main/scala/framework/device/ddr/configs/default.toml", + "bootrom": "arch/src/main/scala/framework/device/peripherals/bootrom/configs/default.toml", + "clint": "arch/src/main/scala/framework/device/peripherals/clint/configs/default.toml", + "plic": "arch/src/main/scala/framework/device/peripherals/plic/configs/default.toml", + "scu": "arch/src/main/scala/framework/device/peripherals/scu/configs/default.toml", + "frontend": "arch/src/main/scala/framework/acc/frontend/configs/default.toml", + "memdomain": "arch/src/main/scala/framework/acc/memdomain/configs/default.toml", + "gpdomain": "arch/src/main/scala/framework/acc/gpdomain/configs/default.toml", + "transport": ( + "arch/src/main/scala/framework/acc/" + "memdomain/src/main/scala/tomem/configs/default.toml" + ), +} +CONFIG_REFERENCES = set(IP_DEFAULTS) | { + "device", + "sims", + "accelerator", + "balldomain", + "cpu", +} + + +def _reference(base: Path, value: str, slot: str | None) -> Path: + if value == "default": + if slot not in IP_DEFAULTS: + _die(f"{base}: {slot!r} requires an explicit configuration path") + return Path(__file__).resolve().parents[5] / IP_DEFAULTS[slot] + return (base / value).resolve() + + +def _merge(left: dict, right: dict, path: Path) -> dict: + result = dict(left) + for key, value in right.items(): + if key == "_file": + continue + if key not in result: + result[key] = value + elif isinstance(result[key], dict) and isinstance(value, dict): + result[key] = _merge(result[key], value, path / key) + else: + _die(f"{path}: include key collision: {key}") + return result + + def _walk_file( path: Path, includes: list[str], @@ -53,26 +104,44 @@ def _walk( base: Path, includes: list[str], stack: list[str], + slot: str | None = None, ) -> Any: if isinstance(obj, dict): inc = obj.get("include") if inc is not None: if not isinstance(inc, str) or not inc: _die(f"{base}: include must be a non-empty string") - loaded = _walk_file((base / inc).resolve(), includes, stack) + loaded = _walk_file(_reference(base, inc, slot), includes, stack) rest = {k: v for k, v in obj.items() if k != "include"} walked = _walk(rest, base, includes, stack) if not isinstance(walked, dict): _die(f"{base}: include siblings must be a table") - overlap = (set(loaded) & set(walked)) - {"_file"} - if overlap: - _die(f"{base / inc}: include key collision: {sorted(overlap)}") - merged = dict(loaded) - merged.update(walked) - return merged - return {k: _walk(v, base, includes, stack) for k, v in obj.items()} + return _merge(loaded, walked, base / inc) + result = {} + for key, value in obj.items(): + if key == "config" and value == "default" and "ip" in obj: + repo = Path(__file__).resolve().parents[5] + catalog = _load(repo / "arch/uvm.toml") + path = repo / catalog["paths"][obj["ip"]] / "configs/uvm.toml" + result[key] = _walk_file(path, includes, stack) + elif key == "param" and value == "default": + component = obj["component"] + if not isinstance(component, str) or not component.isidentifier(): + _die(f"{base}: invalid Ball component {component!r}") + path = ( + Path(__file__).resolve().parents[5] + / "examples/components/balls" + / component + / "configs/default.toml" + ) + result[key] = _walk_file(path, includes, stack) + else: + result[key] = _walk(value, base, includes, stack, key) + return result if isinstance(obj, list): - return [_walk(x, base, includes, stack) for x in obj] + return [_walk(x, base, includes, stack, slot) for x in obj] + if isinstance(obj, str) and obj == "default" and slot in CONFIG_REFERENCES: + return _walk_file(_reference(base, obj, slot), includes, stack) if isinstance(obj, str) and obj.endswith(".toml"): path = (base / obj).resolve() if not path.is_file(): @@ -95,9 +164,9 @@ def toml2json(top: Path, design: Path | None = None) -> dict[str, Any]: raise FileNotFoundError(f"missing TOML: {design}") design = design.resolve() raw = _load(path) - if not isinstance(raw.get("designs"), dict): - _die(f"{path}: missing [designs]") - raw["designs"] = {"include": design.relative_to(path.parent).as_posix()} + if not isinstance(raw.get("device"), dict): + _die(f"{path}: missing [device]") + raw["device"] = {"include": design.relative_to(path.parent).as_posix()} includes.append(str(path)) data = _walk(raw, path.parent, includes, [str(path)]) if not isinstance(data, dict): @@ -111,10 +180,20 @@ def toml2json(top: Path, design: Path | None = None) -> dict[str, Any]: return data +def design2json(top: Path) -> dict[str, Any]: + data = toml2json(top) + data["workloads"] = _walk_file( + Path(top).resolve().parent.parent / "workloads/configs/tests.toml", + data["includes"], + [], + ) + return data + + def main() -> None: if len(sys.argv) != 2: - _die(f"usage: {sys.argv[0]} TOP.toml") - out = toml2json(Path(sys.argv[1])) + _die(f"usage: {sys.argv[0]} DESIGN.toml") + out = design2json(Path(sys.argv[1])) sys.stdout.write(json.dumps(out, indent=2, sort_keys=True) + "\n") diff --git a/bbdev/api/steps/config/scripts/2_parameter_derivation.py b/bbdev/api/steps/config/scripts/2_parameter_derivation.py index f63ff6bed..b39c6c176 100644 --- a/bbdev/api/steps/config/scripts/2_parameter_derivation.py +++ b/bbdev/api/steps/config/scripts/2_parameter_derivation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Derive computed chip parameters from step1 config.json.""" +"""Derive computed design parameters from step1 config.json.""" from __future__ import annotations @@ -18,15 +18,6 @@ def _die(msg: str) -> None: raise ValueError(msg) -def _factory(config: dict[str, Any], context: str) -> str: - value = config["factory"] - if not isinstance(value, str) or len(value.split(".")) < 2 or not all( - part.isidentifier() for part in value.split(".") - ): - _die(f"{context}: factory must be a Scala companion FQCN") - return value - - def _repo_rel(repo: Path, path: str | Path) -> str: p = Path(path) if not p.is_absolute(): @@ -34,154 +25,45 @@ def _repo_rel(repo: Path, path: str | Path) -> str: return p.resolve().relative_to(repo.resolve()).as_posix() -def _walk(obj: Any) -> Iterator[Any]: - if isinstance(obj, dict): - yield obj - for key, value in obj.items(): - if key != "includes": - yield from _walk(value) - elif isinstance(obj, list): - for item in obj: - yield from _walk(item) - - -def core_pkg(rel: str) -> str | None: - parts = Path(rel).parts - if "cores" not in parts: - return None - idx = parts.index("cores") - if idx + 1 >= len(parts): - return None - return parts[idx + 1] - - def _require_id(obj: dict[str, Any], key: str, ctx: str) -> int: val = obj.get(key) - if not isinstance(val, int) or val < 0: + if type(val) is not int or val < 0: _die(f"{ctx}: missing {key} (non-negative int required)") return val -def _require_id_list(obj: dict[str, Any], key: str, count: int, ctx: str) -> list[int]: - val = obj.get(key) - if not isinstance(val, list) or len(val) != count: - _die(f"{ctx}: {key} must be a list of length {count}") - out: list[int] = [] - for item in val: - if not isinstance(item, int) or item < 0: - _die(f"{ctx}: {key} entries must be non-negative ints") - out.append(item) - if len(set(out)) != len(out): - _die(f"{ctx}: {key} has duplicates: {out}") - return out - - -def _tile_cores(tile: dict[str, Any]) -> list[dict[str, Any]]: - cores = tile.get("cores") - if isinstance(cores, list): - out: list[dict[str, Any]] = [] - seen: set[int] = set() - for i, core in enumerate(cores): - if not isinstance(core, dict): - _die("cores entry must be a table") - core_id = _require_id(core, "core_id", f"cores[{i}]") - if core_id in seen: - _die(f"duplicate core_id {core_id} in tile") - seen.add(core_id) - out.append(core) - expected = set(range(len(out))) - if seen != expected: - _die(f"tile core_id must be exactly 0..{len(out)-1}, got {sorted(seen)}") - out.sort(key=lambda c: c["core_id"]) - return out - - template = tile.get("coreTemplate") - if isinstance(template, dict): - count = template.get("count") - if not isinstance(count, int) or count < 1: - _die("[coreTemplate].count must be a positive int") - core_ids = _require_id_list(template, "core_ids", count, "coreTemplate") - expected = set(range(count)) - if set(core_ids) != expected: - _die(f"coreTemplate.core_ids must be exactly 0..{count-1}, got {core_ids}") - base = { - k: v - for k, v in template.items() - if k not in ("count", "core_ids") - } - out = [] - for core_id in sorted(core_ids): - core = dict(base) - core["core_id"] = core_id - out.append(core) - return out - _die("tile must define [[cores]] or [coreTemplate]") - - -def iter_topology_tiles(topo: dict[str, Any]) -> list[dict[str, Any]]: - """The built-in main tile is tile 0; configured mounts follow in list order.""" - for key in ("compute_tiles", "tileTemplate", "main_core"): - if key in topo: - _die(f"invalid topology field: {key}") - main = topo["main_tile"] - if not isinstance(main, dict): - _die("main_tile must be a table") - out = [dict(main, tile_id=0, kind="main")] - groups = topo.get("tiles", []) - if not isinstance(groups, list): - _die("tiles must be an array of tables") - for group in groups: - count = group["count"] - if type(count) is not int or count < 1: - _die("tile count must be a positive int") - base = {k: v for k, v in group.items() if k != "count"} - for _ in range(count): - out.append(dict(base, tile_id=len(out), kind="tile")) - return out - - -def iter_cores(topo: dict[str, Any]) -> Iterator[dict[str, Any]]: - for tile in iter_topology_tiles(topo): - yield from _tile_cores(tile) - - -def unique_cores(topo: dict[str, Any]) -> list[str]: - seen: set[str] = set() - out: list[str] = [] - for obj in _walk(topo): - if not isinstance(obj, dict): - continue - rel = obj.get("_file") - if not isinstance(rel, str): - continue - pkg = core_pkg(rel) - if pkg and pkg not in seen: - seen.add(pkg) - out.append(pkg) - if not out: - _die("topology has no cores//configs/*.toml include") - return out +def _core_threads(core: dict[str, Any]) -> list[dict[str, Any]]: + threads = core["threads"] + if not isinstance(threads, list): + _die("threads must be an explicit instance list") + ids = [_require_id(thread, "id", "thread") for thread in threads] + if len(set(ids)) != len(ids): + _die("duplicate thread ID within core") + return threads + + +def configured_cores(design: dict[str, Any]) -> list[dict[str, Any]]: + cores = design["cores"] + if not isinstance(cores, list): + _die("design cores must be an explicit instance list") + harts = [_require_id(core, "hartId", "core") for core in cores] + if len(set(harts)) != len(harts): + _die("duplicate hart ID") + for core in cores: + if "count" in core or "factory" in core: + _die( + "core instances must be declared directly; hardware is assembled by the design" + ) + return cores -def tile_files(topo: dict[str, Any]) -> list[str]: - seen: set[str] = set() - tiles: list[str] = [] - for obj in _walk(topo): - if not isinstance(obj, dict): - continue - rel = obj.get("_file") - if not isinstance(rel, str) or "/tiles/" not in rel or rel in seen: - continue - seen.add(rel) - tiles.append(rel) - tiles.sort() - if not tiles: - _die("topology has no tile file") - return tiles +def iter_threads(topo: dict[str, Any]) -> Iterator[dict[str, Any]]: + for core in configured_cores(topo): + yield from _core_threads(core) -def bank_params(core: dict, pkg: str) -> tuple[int, int, int]: - mem = core.get("memdomain") +def bank_params(thread: dict, pkg: str) -> tuple[int, int, int]: + mem = thread.get("memdomain") if not isinstance(mem, dict): _die(f"{pkg}: missing memdomain") bank = mem.get("bank") @@ -197,107 +79,64 @@ def bank_params(core: dict, pkg: str) -> tuple[int, int, int]: return num, width, entries -def _ball_dir(ball_class: str) -> str: - if not ball_class.startswith("examples.balls."): - _die(f"malformed ballClass: {ball_class!r}") - directory = ball_class[len("examples.balls.") :].split(".", 1)[0] - if not directory: - _die(f"malformed ballClass: {ball_class!r}") - return directory - - -def _balldomain_base_dir(repo: Path, core: dict, pkg: str) -> str: - bd = core.get("balldomain") +def _balldomain_base_dir(repo: Path, thread: dict, pkg: str) -> str: + bd = thread.get("balldomain") if isinstance(bd, dict): rel = bd.get("_file") if isinstance(rel, str): return _repo_rel(repo, Path(rel).parent) - return f"examples/cores/{pkg}/configs/balldomains" - + raise ValueError(f"{pkg}: missing BallDomain configuration source") -def _config_path(repo: Path, config: object) -> str: - if isinstance(config, str): - return _repo_rel(repo, config) - if isinstance(config, dict): - raw = config.get("_file") - if isinstance(raw, str): - return _repo_rel(repo, raw) - _die(f"ball mapping config must be a path or include _file: {config!r}") - -def _ball_num(core: dict) -> int: - bd = normalize_ball_domain(core) +def _ball_num(thread: dict) -> int: + bd = normalize_ball_domain(thread) if not isinstance(bd, dict): return 0 - ball_num = bd.get("ballNum") - if ball_num is None: - return 0 - if not isinstance(ball_num, int) or ball_num < 0: - _die("balldomain.ballNum must be a non-negative int") - return ball_num - - -def _n_tiles(topo: dict[str, Any]) -> int: - if "top" in topo: - _die("invalid topology field: top") - return len(iter_topology_tiles(topo)) + return len(bd["balls"]) -def _derive_cores(repo: Path, topo: dict[str, Any]) -> list[dict[str, Any]]: +def _derive_threads(repo: Path, topo: dict[str, Any]) -> list[dict[str, Any]]: out: list[dict[str, Any]] = [] - for index, core in enumerate(iter_cores(topo)): - rel = core.get("_file") + for index, thread in enumerate(iter_threads(topo)): + rel = thread.get("_file") if not isinstance(rel, str): - _die("core config missing _file") - pkg = core_pkg(rel) - if not pkg: - _die(f"unsupported core config path: {rel}") - role = core.get("name") + _die("thread config missing _file") + pkg = thread["component"] + if not isinstance(pkg, str) or not pkg.isidentifier(): + _die(f"invalid component name: {pkg!r}") + role = thread.get("name") if role is not None and not isinstance(role, str): - _die("core name must be a string") - core_id = _require_id(core, "core_id", f"core index {index}") - ball_num = _ball_num(core) - if isinstance(core.get("memdomain"), dict): - num, width, entries = bank_params(core, pkg) + _die("thread name must be a string") + thread_id = _require_id(thread, "id", f"thread index {index}") + ball_num = _ball_num(thread) + if isinstance(thread.get("memdomain"), dict): + num, width, entries = bank_params(thread, pkg) elif ball_num > 0: - _die(f"{pkg}: buckyball core missing memdomain") + _die(f"{pkg}: buckyball thread missing memdomain") else: num, width, entries = 0, 0, 0 entry: dict[str, Any] = { "index": index, - "core_id": core_id, + "thread_id": thread_id, "pkg": pkg, "role": role or "", - "cpu_kind": core["cpu"]["kind"], - "factory_class": _factory(core, pkg), + "profile": thread["compiler"]["name"], + "cpu_kind": thread["cpu"]["kind"] if "cpu" in thread else "", "config_path": _repo_rel(repo, rel), - "balldomain_base_dir": _balldomain_base_dir(repo, core, pkg), + "balldomain_base_dir": _balldomain_base_dir(repo, thread, pkg), "bank_num": num, "bank_width": width, "bank_entries": entries, "ball_num": ball_num, } if ball_num > 0: - bd = normalize_ball_domain(core) + bd = normalize_ball_domain(thread) if not isinstance(bd, dict): _die(f"{pkg}: missing balldomain") - mappings = bd.get("ballIdMappings") - if not isinstance(mappings, list): - _die(f"{pkg}: missing ballIdMappings") - entry["mappings"] = [] - for mapping in mappings: - if not isinstance(mapping, dict): - _die("ballIdMappings entry must be a table") - ball_class = mapping.get("ballClass") - if not isinstance(ball_class, str) or not ball_class: - _die(f"ballClass must be a non-empty string: {mapping!r}") - entry["mappings"].append( - { - "ball_id": mapping.get("ballId"), - "ball_dir": _ball_dir(ball_class), - "config_path": _config_path(repo, mapping.get("config")), - } - ) + entry["balls"] = [ + {"ball_id": ball["id"], "ball_dir": ball["component"]} + for ball in bd["balls"] + ] out.append(entry) return out @@ -309,35 +148,41 @@ def _validate_spm(config: dict[str, Any]) -> None: if bits < 64 or bits & (bits - 1): _die("SPM dataBits must be a power of two >= 64") beat = bits // 8 - if size < 2 * beat or size & (size - 1) or base < 0 or base % beat or base + size > 1 << 64: - _die("SPM requires an aligned base and power-of-two capacity within the 64-bit address space") + if ( + size < 2 * beat + or size & (size - 1) + or base < 0 + or base % beat + or base + size > 1 << 64 + ): + _die( + "SPM requires an aligned base and power-of-two capacity within the 64-bit address space" + ) -def _derive_tiles( - repo: Path, topo: dict[str, Any], cores: list[dict[str, Any]] +def _derive_cores( + repo: Path, topo: dict[str, Any], threads: list[dict[str, Any]] ) -> list[dict[str, Any]]: placements: list[dict[str, Any]] = [] offset = 0 - for tile in iter_topology_tiles(topo): - tile_id = _require_id(tile, "tile_id", "tile") - tile_path = tile.get("_file") - if not isinstance(tile_path, str): - _die("tile missing _file") - tile_cores = _tile_cores(tile) - n = len(tile_cores) - controller = tile.get("controllerCore") - if controller is not None and (type(controller) is not int or controller not in range(n)): - _die(f"tile_id={tile_id}: controllerCore must name a local core") - if tile["kind"] == "main" and controller is not None: - _die("main tile has no task controller") - ants = [core for core in tile_cores if core["cpu"]["kind"] == "ant"] + for core in configured_cores(topo): + core_path = core.get("_file") + if not isinstance(core_path, str): + _die("core missing _file") + core_threads = _core_threads(core) + n = len(core_threads) + ants = [ + thread + for thread in core_threads + if "cpu" in thread and thread["cpu"]["kind"] == "ant" + ] if ants: - shared_spm = tile.get("tss") + shared_spm = core.get("tss") if not isinstance(shared_spm, dict): - _die("Ant compute tile requires explicit tss geometry") + _die("Ant compute core requires explicit tss geometry") _validate_spm(shared_spm) - for core in ants: - params = core["cpu"]["config"] + for thread in ants: + params = thread["cpu"]["config"] tls = params["tls"] _validate_spm(tls) code = params["codeBytes"] @@ -347,102 +192,69 @@ def _derive_tiles( _die("Ant taskBits must be in [1,64]") if min(tls["base"], shared_spm["base"]) < code: _die("Ant code overlaps TLS/TSS") - if not (tls["base"] + tls["bytes"] <= shared_spm["base"] or - shared_spm["base"] + shared_spm["bytes"] <= tls["base"]): + if not ( + tls["base"] + tls["bytes"] <= shared_spm["base"] + or shared_spm["base"] + shared_spm["bytes"] <= tls["base"] + ): _die("Ant TLS/TSS windows overlap") if tls["dataBits"] != shared_spm["dataBits"]: _die("Ant TLS/TSS access widths must match") - shared = tile.get("sharedMem") + shared = core.get("sharedMem") if isinstance(shared, dict): if shared["bankWidth"] != 128: _die("sharedMem.bankWidth must be 128 bits") rows = shared["bankEntries"] if type(rows) is not int or rows < 2 or rows > 65536 or rows & (rows - 1): _die("sharedMem.bankEntries must be a power of two in [2,65536]") - if shared["enable"] and (shared["entries"] <= 0 or shared["entries"] % rows): + if shared["enable"] and ( + shared["entries"] <= 0 or shared["entries"] % rows + ): _die("sharedMem.entries must be a positive multiple of bankEntries") - vbc = 0 - if isinstance(shared, dict): - raw = shared.get("virtualBankCount") - if isinstance(raw, int) and raw > 0: - vbc = raw - if vbc == 0 and n > 0: - bank_nums = [cores[i]["bank_num"] for i in range(offset, offset + n)] - if any(bank_nums): - vbc = max(bank_nums) + vbc = _require_id(shared, "virtualBankCount", "shared memory") indices = list(range(offset, offset + n)) - has_buckyball = any(cores[i]["ball_num"] > 0 for i in indices) - mem_ball_channel_num = 0 - if has_buckyball: - raw = tile.get("memBallChannelNum") - if not isinstance(raw, int): - _die("tile with Buckyball cores must define memBallChannelNum") - mem_ball_channel_num = raw - if tile["kind"] == "main": - if "factory" in tile: - _die("main tile must not specify a factory") - factory = "" - else: - factory = _factory(tile, str(tile_path)) placements.append( { - "tile_id": tile_id, - "kind": tile["kind"], - "factory_class": factory, - "path": _repo_rel(repo, tile_path), - "core_indices": indices, - "cores_per_tile": n, + "hart_id": core["hartId"], + "path": _repo_rel(repo, core_path), + "thread_indices": indices, + "threads_per_core": n, "virtual_bank_count": vbc, - "mem_ball_channel_num": mem_ball_channel_num, } ) - if controller is not None: - placements[-1]["controller_core_index"] = indices[controller] offset += n return placements -def _bemu_balls(repo: Path, topo: dict[str, Any]) -> list[dict[str, str]]: +def _bemu_balls(repo: Path, profiles: list[dict[str, Any]]) -> list[dict[str, str]]: seen: set[str] = set() balls: list[dict[str, str]] = [] - for core in iter_cores(topo): - bd = normalize_ball_domain(core) + for profile in profiles: + bd = normalize_ball_domain(profile["config"]) if not isinstance(bd, dict): continue - mappings = bd.get("ballIdMappings") - isa = bd.get("ballISA") - if not isinstance(mappings, list) or not isinstance(isa, list): - continue - bid_to_class = { - m["ballId"]: m["ballClass"] - for m in mappings - if isinstance(m, dict) - and isinstance(m.get("ballId"), int) - and isinstance(m.get("ballClass"), str) - } - pkg = core_pkg(core.get("_file", "")) or "core" - for entry in isa: - if not isinstance(entry, dict): - _die("ballISA entry must be a table") - funct7 = entry.get("funct7") - bid = entry.get("bid") - if not isinstance(funct7, int) or not isinstance(bid, int): - _die(f"ballISA entry must have funct7 and bid: {entry!r}") - if funct7 == 0: - _die(f"core {pkg}: funct7 zero is not a Buckyball instruction") - if funct7 in {1, 16, 32, 33, 34, 35}: + for ball in bd["balls"]: + instructions = ball["instructions"] + if any(entry["funct7"] == 0 for entry in instructions): + _die("funct7 zero is not a Buckyball instruction") + if not any( + entry["funct7"] not in {1, 16, 32, 33, 34, 35} for entry in instructions + ): continue - ball_class = bid_to_class.get(bid) - if not ball_class: - _die( - f"core {pkg}: ballISA funct7 {funct7} " - f"references missing bid {bid}" - ) + ball_dir = ball["component"] + ball_class = f"examples.balls.{ball_dir}.{ball['name']}" if ball_class in seen: continue seen.add(ball_class) - ball_dir = _ball_dir(ball_class) - emu_lib = repo / "examples" / "balls" / ball_dir / "emu" / "src" / "lib.rs" + emu_lib = ( + repo + / "examples" + / "components" + / "balls" + / ball_dir + / "emu" + / "src" + / "lib.rs" + ) if not emu_lib.is_file(): _die(f"missing BEMU ball source for {ball_class}: {emu_lib}") balls.append( @@ -455,175 +267,147 @@ def _bemu_balls(repo: Path, topo: dict[str, Any]) -> list[dict[str, str]]: return balls -def _bemu_paths(repo: Path, chip: str, topo: dict[str, Any]) -> tuple[str, int]: - """The chip's own bebop-chip entry (empty for the default tile runner) and the tile it runs.""" - tiles = iter_topology_tiles(topo) - mounted = [t["tile_id"] for t in tiles if t["kind"] == "tile"] - main = repo / "examples" / "chips" / chip / "emu" / "src" / "main.rs" - entry = f"examples/chips/{chip}/emu/src/main.rs" if main.is_file() else "" - return entry, mounted[0] if mounted else 0 - - -def _ball_ctest_dirs(repo: Path, core: dict[str, Any]) -> list[str]: - bd = normalize_ball_domain(core) +def _ball_ctest_dirs(repo: Path, thread: dict[str, Any]) -> list[str]: + bd = normalize_ball_domain(thread) if not isinstance(bd, dict): - _die("core missing balldomain for ctest dirs") - mappings = bd.get("ballIdMappings") - if not isinstance(mappings, list): - _die("core missing ballIdMappings for ctest dirs") + _die("thread missing balldomain for ctest dirs") dirs: list[str] = [] - for mapping in mappings: - if not isinstance(mapping, dict): - _die("ballIdMappings entry must be a table") - ball_class = mapping.get("ballClass") - if not isinstance(ball_class, str): - _die("ballIdMappings entry missing ballClass") - ball_dir = _ball_dir(ball_class) - path = repo / "examples" / "balls" / ball_dir / "workloads" / "ctests" + for ball in bd["balls"]: + path = ( + repo + / "examples" + / "components" + / "balls" + / ball["component"] + / "workloads" + / "ctests" + ) if not path.is_dir(): _die(f"missing {path}") dirs.append(_repo_rel(repo, path)) return dirs -def _target_name(role: str, pkg: str) -> str: - if role: - return role - return pkg +def compiler_profiles(device: dict[str, Any]) -> list[dict[str, Any]]: + profiles = [] + for core in configured_cores(device): + for thread in core["threads"]: + declaration = thread["compiler"] + profiles.append( + { + "name": declaration["name"], + "component": declaration["component"], + "config": thread, + } + ) + for core in configured_cores(device): + if "accelerator" in core: + declaration = core["compiler"] + profiles.append( + { + "name": declaration["name"], + "component": declaration["component"], + "config": core["accelerator"], + } + ) + return profiles def _derive_targets( - repo: Path, topo: dict[str, Any], cores: list[dict[str, Any]] + repo: Path, profiles: list[dict[str, Any]] ) -> dict[str, dict[str, Any]]: - targets: dict[str, dict[str, Any]] = {} - topo_by_pkg: dict[str, dict[str, Any]] = {} - for core in iter_cores(topo): - rel = core.get("_file") - if not isinstance(rel, str): - continue - pkg = core_pkg(rel) - if pkg and pkg not in topo_by_pkg: - topo_by_pkg[pkg] = core - - for inst in cores: - name = _target_name(inst["role"], inst["pkg"]) - if name in targets: - continue - pkg = inst["pkg"] - topo_core = topo_by_pkg.get(pkg) - if topo_core is None: - _die(f"topology missing core package {pkg}") - target: dict[str, Any] = { + targets = {} + for profile in profiles: + name, pkg, config = profile["name"], profile["component"], profile["config"] + num, width, entries = bank_params(config, pkg) + targets[name] = { "pkg": pkg, "compiler": "compiler", - "bank": { - "num": inst["bank_num"], - "width": inst["bank_width"], - "entries": inst["bank_entries"], - }, - "ball_ctest_dirs": ( - _ball_ctest_dirs(repo, topo_core) if inst["ball_num"] > 0 else [] - ), + "bank": {"num": num, "width": width, "entries": entries}, + "ball_ctest_dirs": _ball_ctest_dirs(repo, config), } - targets[name] = target return targets -def _derive_harts(tiles: list[dict[str, Any]], cores: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Only system CPUs receive hart IDs; Ant contexts are tile-local resources.""" - harts: list[dict[str, Any]] = [] - for tile in tiles: - context = 0 - for local, core_index in enumerate(tile["core_indices"]): - inst = cores[core_index] - if inst["core_id"] != local: - _die(f"tile {tile['tile_id']}: core IDs must match ordered slots") - if inst["cpu_kind"] == "ant": - inst["ant_context_id"] = context - context += 1 - continue - harts.append({ - "hart_id": len(harts), "tile_id": tile["tile_id"], - "core_id": local, "core_index": core_index, "visible": True, - "target": _target_name(inst["role"], inst["pkg"]), - "pkg": inst["pkg"], "role": inst["role"], - }) - return harts +def _derive_harts( + cores: list[dict[str, Any]], threads: list[dict[str, Any]] +) -> list[dict[str, Any]]: + return [{"hart_id": core["hart_id"]} for core in cores] -def _workload(chip: str, targets: dict[str, dict[str, Any]], repo: Path) -> dict[str, Any]: +def _workload( + design: str, targets: dict[str, dict[str, Any]], repo: Path +) -> dict[str, Any]: defs: dict[str, str] = { - "BUCKYBALL_WORKLOAD_CHIP": chip, - "BUCKYBALL_CARGO_TARGET_DIR": str(repo / "bebop" / "target" / chip), + "BUCKYBALL_WORKLOAD_DESIGN": design, } return {"cmake_param": defs} -def derive(data: dict[str, Any], repo: Path, chip: str) -> dict[str, Any]: +def derive(data: dict[str, Any], repo: Path, design: str) -> dict[str, Any]: repo = repo.resolve() - topo = data.get("designs") + topo = data.get("device") if not isinstance(topo, dict): - _die(f"{chip}: missing designs") + _die(f"{design}: missing device") root_file = data.get("_file") if not isinstance(root_file, str): - _die(f"{chip}: missing root _file") + _die(f"{design}: missing root _file") design_file = topo.get("_file") if not isinstance(design_file, str): - _die(f"{chip}: designs missing _file") + _die(f"{design}: device missing _file") includes_raw = data.get("includes") if not isinstance(includes_raw, list): - _die(f"{chip}: includes must be a list") + _die(f"{design}: includes must be a list") - cores = _derive_cores(repo, topo) - tiles = _derive_tiles(repo, topo, cores) - targets = _derive_targets(repo, topo, cores) - harts = _derive_harts(tiles, cores) - n_tiles = _n_tiles(topo) - - chip_main, tile_index = _bemu_paths(repo, chip, topo) + threads = _derive_threads(repo, topo) + cores = _derive_cores(repo, topo, threads) + profiles = compiler_profiles(topo) + targets = _derive_targets(repo, profiles) + harts = _derive_harts(cores, threads) + if sorted(hart["hart_id"] for hart in harts) != list(range(len(harts))): + _die("complete design hart IDs must be contiguous from zero") + n_cores = len(configured_cores(topo)) return { - "chip": chip, - "name": Path(_repo_rel(repo, design_file)).stem, - "chip_path": _repo_rel(repo, root_file), - "tile_config_path": _repo_rel(repo, design_file), + "design": design, + "name": Path(root_file).parent.parent.name, + "design_path": _repo_rel(repo, root_file), + "core_config_path": _repo_rel(repo, design_file), "includes": [_repo_rel(repo, p) for p in includes_raw], - "n_tiles": n_tiles, + "n_cores": n_cores, "targets": targets, "harts": harts, + "threads": threads, "cores": cores, - "tiles": tiles, "bemu": { - "chip_main": chip_main, - "tile_index": tile_index, - "balls": _bemu_balls(repo, topo), + "balls": _bemu_balls(repo, profiles), }, - "workload": _workload(chip, targets, repo), + "workload": _workload(design, targets, repo), } -def write_derived(data: dict[str, Any], repo: Path, chip: str, out: Path) -> dict[str, Any]: - derived = derive(data, repo, chip) +def write_derived( + data: dict[str, Any], repo: Path, design: str, out: Path +) -> dict[str, Any]: + derived = derive(data, repo, design) out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(derived, indent=2, sort_keys=True) + "\n", encoding="utf-8") + out.write_text( + json.dumps(derived, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) return derived def main() -> None: if len(sys.argv) not in (4, 5): - _die(f"usage: {sys.argv[0]} REPO CHIP CONFIG.json [OUT.json]") + _die(f"usage: {sys.argv[0]} REPO DESIGN CONFIG.json [OUT.json]") repo = Path(sys.argv[1]) - chip = sys.argv[2] + design = sys.argv[2] config = Path(sys.argv[3]) - out = ( - Path(sys.argv[4]) - if len(sys.argv) == 5 - else config.parent / "derived.json" - ) + out = Path(sys.argv[4]) if len(sys.argv) == 5 else config.parent / "derived.json" data = json.loads(config.read_text(encoding="utf-8")) - derived = write_derived(data, repo, chip, out) + derived = write_derived(data, repo, design, out) sys.stdout.write(json.dumps(derived, indent=2, sort_keys=True) + "\n") diff --git a/bbdev/api/steps/config/scripts/3_json2proto.py b/bbdev/api/steps/config/scripts/3_json2proto.py index 532100ff2..e9f8372a0 100644 --- a/bbdev/api/steps/config/scripts/3_json2proto.py +++ b/bbdev/api/steps/config/scripts/3_json2proto.py @@ -1,10 +1,14 @@ #!/usr/bin/env python3 -"""Build Chip protobuf from config.json + derived.json.""" +"""Build Design protobuf from config.json + derived.json.""" from __future__ import annotations import importlib.util import json +import hashlib +import re +import tomllib +from google.protobuf.json_format import ParseDict import sys from pathlib import Path from typing import Any @@ -12,8 +16,9 @@ if str(Path(__file__).resolve().parent) not in sys.path: sys.path.insert(0, str(Path(__file__).resolve().parent)) -import chip_pb2 as pb # noqa: E402 +import design_pb2 as pb # noqa: E402 from ball_normalize import normalize_ball_domain +from ball_parameters import pack_parameters def _load_derive(): @@ -36,15 +41,8 @@ def _rel(bbdir: Path, path: str | Path) -> str: return p.as_posix() -def _ball_dir(ball_class: str) -> str: - parts = ball_class.split(".") - if len(parts) < 3 or parts[0] != "examples" or parts[1] != "balls": - raise ValueError(f"bad ballClass: {ball_class}") - return parts[2] - - -def _raw_core_list(designs: dict[str, Any]) -> list[dict[str, Any]]: - return list(_derive.iter_cores(designs)) +def _raw_thread_list(designs: dict[str, Any]) -> list[dict[str, Any]]: + return list(_derive.iter_threads(designs)) def _fill_bank(msg: pb.BankConfig, d: dict[str, Any]) -> None: @@ -52,20 +50,15 @@ def _fill_bank(msg: pb.BankConfig, d: dict[str, Any]) -> None: msg.width = d["width"] msg.entries = d["entries"] msg.mask_len = d["maskLen"] - msg.channel = d["channel"] + msg.request_ports = d["requestPorts"] + msg.layout_enable = d["layoutEnable"] def _fill_mem(msg: pb.MemDomainConfig, d: dict[str, Any], bbdir: Path) -> None: msg.source_path = _rel(bbdir, d["_file"]) _fill_bank(msg.bank, d["bank"]) dma = d["dma"] - msg.dma.n_xacts = dma["nXacts"] - msg.dma.burst_max_bytes = dma["burstMaxBytes"] msg.dma.bus_width = dma["busWidth"] - msg.dma.max_in_flight_mem_reqs = dma["maxInFlightMemReqs"] - msg.tlb.size = d["tlb"]["size"] - msg.tma.read_channel = d["tma"]["readChannel"] - msg.tma.write_channel = d["tma"]["writeChannel"] mmio = d["mmio"] msg.mmio.enable = mmio["enable"] msg.mmio.bank_num = mmio["bankNum"] @@ -76,127 +69,45 @@ def _fill_mem(msg: pb.MemDomainConfig, d: dict[str, Any], bbdir: Path) -> None: def _fill_ball(msg: pb.BallDomain, d: dict[str, Any], bbdir: Path) -> None: - msg.source_path = _rel(bbdir, d["_file"]) - msg.ball_num = d["ballNum"] - for m in d["ballIdMappings"]: - e = msg.mappings.add() - e.ball_id = m["ballId"] - e.ball_name = m["ballName"] - e.ball_class = m["ballClass"] - e.ball_dir = _ball_dir(m["ballClass"]) - e.config_path = _rel(bbdir, m["config"]["_file"]) - params = m["config"]["ball"] - for key, value in params.items(): - e.ball_params[key] = str(value) - e.in_bw = m["inBW"] - e.out_bw = m["outBW"] - if "mmioReadBW" in m: - e.mmio_read_bw = m["mmioReadBW"] - if "mmioWriteBW" in m: - e.mmio_write_bw = m["mmioWriteBW"] - for item in d["ballISA"]: - e = msg.isa.add() - e.mnemonic = item["mnemonic"] - e.funct7 = item["funct7"] - e.bid = item["bid"] - - -def _fill_rocket(msg: pb.RocketCpuConfig, d: dict[str, Any]) -> None: - msg.mtvec_init_enable = d["mtvecInitEnable"] - msg.use_user = d["useUser"] - msg.use_supervisor = d["useSupervisor"] - msg.use_hypervisor = d["useHypervisor"] - msg.use_debug = d["useDebug"] - msg.use_atomics = d["useAtomics"] - msg.use_atomics_only_for_i_o = d["useAtomicsOnlyForIO"] - msg.use_compressed = d["useCompressed"] - msg.use_r_v_e = d["useRVE"] - msg.use_conditional_zero = d["useConditionalZero"] - msg.n_local_interrupts = d["nLocalInterrupts"] - msg.use_n_m_i = d["useNMI"] - msg.n_breakpoints = d["nBreakpoints"] - msg.use_b_p_watch = d["useBPWatch"] - msg.mcontext_width = d["mcontextWidth"] - msg.scontext_width = d["scontextWidth"] - msg.n_perf_counters = d["nPerfCounters"] - msg.have_basic_counters = d["haveBasicCounters"] - msg.misa_writable = d["misaWritable"] - msg.mtvec_init = d["mtvecInit"] - msg.mtvec_writable = d["mtvecWritable"] - msg.fast_load_word = d["fastLoadWord"] - msg.fast_load_byte = d["fastLoadByte"] - msg.branch_prediction_mode_c_s_r = d["branchPredictionModeCSR"] - msg.clock_gate = d["clockGate"] - msg.mvendorid = d["mvendorid"] - msg.mimpid = d["mimpid"] - msg.have_cease = d["haveCease"] - msg.have_sim_timeout = d["haveSimTimeout"] - for field in ("useHypervisor", "useDebug", "useNMI"): - if d[field]: - raise ValueError(f"Rocket {field} has no platform interface") - if d["nLocalInterrupts"] != 0: - raise ValueError("Rocket local interrupts have no platform interface") - msg.mul_div.div_unroll = d["mulDiv"]["divUnroll"] - msg.mul_div.div_early_out_granularity = d["mulDiv"]["divEarlyOutGranularity"] - msg.fpu.div_sqrt = d["fpu"]["divSqrt"] - msg.fpu.sfma_latency = d["fpu"]["sfmaLatency"] - msg.fpu.dfma_latency = d["fpu"]["dfmaLatency"] - msg.fpu.fpmu_latency = d["fpu"]["fpmuLatency"] - msg.fpu.ifpu_latency = d["fpu"]["ifpuLatency"] - msg.dcache.clock_gate = d["dcache"]["clockGate"] - msg.icache.prefetch = d["icache"]["prefetch"] - msg.btb.n_match_bits = d["btb"]["nMatchBits"] - msg.btb.n_pages = d["btb"]["nPages"] - msg.btb.updates_out_of_order = d["btb"]["updatesOutOfOrder"] - msg.btb.bht_enable = d["btb"]["bhtEnable"] - msg.btb.bht_entries = d["btb"]["bhtEntries"] - msg.btb.bht_counter_length = d["btb"]["bhtCounterLength"] - msg.btb.bht_history_length = d["btb"]["bhtHistoryLength"] - msg.btb.bht_history_bits = d["btb"]["bhtHistoryBits"] - msg.use_vm = d["useVM"] - msg.use_zba = d["useZba"] - msg.use_zbb = d["useZbb"] - msg.use_zbs = d["useZbs"] - msg.have_c_flush = d["haveCFlush"] - md = d["mulDiv"] - msg.mul_div.enable = md["enable"] - msg.mul_div.mul_unroll = md["mulUnroll"] - msg.mul_div.mul_early_out = md["mulEarlyOut"] - msg.mul_div.div_early_out = md["divEarlyOut"] - fpu = d["fpu"] - msg.fpu.enable = fpu["enable"] - msg.fpu.min_f_len = fpu["minFLen"] - msg.fpu.f_len = fpu["fLen"] - dc = d["dcache"] - msg.dcache.n_sets = dc["nSets"] - msg.dcache.n_ways = dc["nWays"] - msg.dcache.n_mshrs = dc["nMSHRs"] - ic = d["icache"] - msg.icache.n_sets = ic["nSets"] - msg.icache.n_ways = ic["nWays"] - btb = d["btb"] - msg.btb.enable = btb["enable"] - msg.btb.n_entries = btb["nEntries"] - msg.btb.n_ras = btb["nRAS"] - - -def _fill_boom(msg: pb.BoomCpuConfig, d: dict[str, Any]) -> None: - msg.fetch_width = d["fetchWidth"] - msg.decode_width = d["decodeWidth"] - msg.num_rob_entries = d["numRobEntries"] - dc = d["dcache"] - msg.dcache.n_sets = dc["nSets"] - msg.dcache.n_ways = dc["nWays"] - msg.dcache.n_mshrs = dc["nMSHRs"] - ic = d["icache"] - msg.icache.n_sets = ic["nSets"] - msg.icache.n_ways = ic["nWays"] + for ball in d["balls"]: + instance = msg.balls.add() + instance.ball_id = ball["id"] + instance.ball_name = ball["name"] + instance.ball_dir = ball["component"] + classes = [] + root = bbdir / "examples/components/balls" / ball["component"] / "arch" + for source in sorted(root.rglob("*Ball.scala")): + text = source.read_text() + names = re.findall( + r"class (\w+)\([^{}]*?\)\s+extends Module\s+with HasBlink\b", text, re.S + ) + if names: + package = re.search(r"(?m)^package ([\w.]+)", text).group(1) + classes.extend(f"{package}.{name}" for name in names) + if len(classes) != 1: + raise ValueError( + f"{ball['component']}: expected one Ball implementation, found {classes}" + ) + instance.rtl_class = classes[0] + port = ball["port"] + instance.in_bw = port["inBW"] + instance.out_bw = port["outBW"] + instance.mmio_read_bw = port["mmioReadBW"] + instance.mmio_write_bw = port["mmioWriteBW"] + pack_parameters( + instance, + {key: value for key, value in ball["param"].items() if key != "_file"}, + ) + for instruction in ball["instructions"]: + entry = instance.instructions.add() + entry.mnemonic = instruction["mnemonic"] + entry.funct7 = instruction["funct7"] def _check_cpu_kind(cpu: dict[str, Any], pkg: str) -> str: kind = cpu.get("kind") - if kind not in ("rocket", "boom", "ant"): - raise ValueError(f"{pkg}: kind must be 'rocket', 'boom' or 'ant', got {kind!r}") + if kind != "ant": + raise ValueError(f"{pkg}: kind must be 'ant', got {kind!r}") if not isinstance(cpu.get("config"), dict): raise ValueError(f"{pkg}: cpu.config must reference a TOML file") return kind @@ -212,15 +123,16 @@ def _fill_frontend(msg: pb.FrontendConfig, d: dict[str, Any], bbdir: Path) -> No msg.iter_len = d["iterLen"] msg.sub_rob_enable = d["subRobEnable"] msg.sub_rob_depth = d["subRobDepth"] + msg.bank_alias_enable = d["bankAliasEnable"] -def _fill_rvv(msg: pb.RvvConfig, d: dict[str, Any], bbdir: Path) -> None: +def _fill_gpdomain(msg: pb.GpDomainConfig, d: dict[str, Any], bbdir: Path) -> None: if type(d["enable"]) is not bool: - raise ValueError("rvv.enable must be an explicit boolean") + raise ValueError("gpdomain.enable must be an explicit boolean") msg.enable = d["enable"] for key in ("laneNumber", "vLen", "eLen", "iBufWords", "memoryPorts"): if type(d[key]) is not int or d[key] <= 0: - raise ValueError(f"rvv.{key} must be a positive integer") + raise ValueError(f"gpdomain.{key} must be a positive integer") msg.source_path = _rel(bbdir, d["_file"]) msg.lane_number = d["laneNumber"] msg.v_len = d["vLen"] @@ -229,8 +141,8 @@ def _fill_rvv(msg: pb.RvvConfig, d: dict[str, Any], bbdir: Path) -> None: msg.memory_ports = d["memoryPorts"] -def _fill_tile_params(msg: pb.TileParamConfig, d: dict[str, Any]) -> None: - msg.core_data_bytes = d["coreDataBytes"] +def _fill_core_params(msg: pb.CoreParamConfig, d: dict[str, Any]) -> None: + msg.thread_data_bytes = d["threadDataBytes"] msg.x_len = d["xLen"] msg.vaddr_bits = d["vaddrBits"] msg.paddr_bits = d["paddrBits"] @@ -249,144 +161,403 @@ def _fill_cpu(msg: pb.CpuConfig, cpu: dict[str, Any], context: str, bbdir: Path) kind = _check_cpu_kind(cpu, context) msg.kind = kind msg.source_path = _rel(bbdir, cpu["_file"]) - if kind == "rocket": - _fill_rocket(msg.rocket, cpu["config"]) - elif kind == "boom": - _fill_boom(msg.boom, cpu["config"]) - else: - config = cpu["config"] - msg.ant.code_bytes = config["codeBytes"] - msg.ant.task_bits = config["taskBits"] - _fill_spm(msg.ant.tls, config["tls"]) + config = cpu["config"] + msg.ant.code_bytes = config["codeBytes"] + msg.ant.task_bits = config["taskBits"] + _fill_spm(msg.ant.tls, config["tls"]) return kind -def _fill_core(ci: pb.CoreInstance, raw: dict[str, Any], meta: dict[str, Any], bbdir: Path) -> None: +def _hardware_fields(value: Any) -> Any: + if isinstance(value, dict): + return { + key: _hardware_fields(item) for key, item in value.items() if key != "_file" + } + if isinstance(value, list): + return [_hardware_fields(item) for item in value] + return value + + +def _thread_signature(thread: pb.ThreadInstance) -> int: + hardware = pb.ThreadInstance() + hardware.CopyFrom(thread) + for ball in hardware.balldomain.balls: + ball.ClearField("rtl_class") + for field in ( + "index", + "role", + "config_path", + "source_path", + "rtl_class", + "balldomain_base_dir", + "local_id", + "signature", + "transport", + "profile", + ): + hardware.ClearField(field) + for message in (hardware.cpu, hardware.mem, hardware.frontend, hardware.gpdomain): + message.ClearField("source_path") + digest = hashlib.sha256(hardware.SerializeToString(deterministic=True)).digest() + return int.from_bytes(digest[:8], "little") + + +def _fill_thread( + ci: pb.ThreadInstance, + raw: dict[str, Any], + meta: dict[str, Any], + bbdir: Path, + *, + hardware: bool, +) -> None: ci.index = meta["index"] + ci.local_id = meta["thread_id"] ci.role = meta["role"] ci.pkg = meta["pkg"] + ci.profile = raw["compiler"]["name"] if "compiler" in raw else meta["role"] ci.config_path = meta["config_path"] - if raw["factory"] != meta["factory_class"]: - raise ValueError("core factory differs between config and derived metadata") - ci.factory_class = raw["factory"] + source = (Path(raw["_file"]).parent / raw["source"]).resolve() + if not source.is_dir(): + raise FileNotFoundError(f"missing component source directory: {source}") + ci.source_path = _rel(bbdir, source) + if hardware: + classes = [] + for implementation in sorted((source / "arch").rglob("*.scala")): + text = implementation.read_text() + names = re.findall( + r"class (\w+)\([^{}]*?\)\s+extends Module\s+with HasTLink\b", text, re.S + ) + if names: + package = re.search(r"(?m)^package ([\w.]+)", text).group(1) + classes.extend(f"{package}.{name}" for name in names) + if len(classes) != 1: + raise ValueError( + f"{source}: expected one Thread implementation, found {classes}" + ) + ci.rtl_class = classes[0] ci.balldomain_base_dir = meta["balldomain_base_dir"] - kind = _fill_cpu(ci.cpu, raw["cpu"], meta["pkg"], bbdir) + if "cpu" in raw: + _fill_cpu(ci.cpu, raw["cpu"], meta["pkg"], bbdir) domain = normalize_ball_domain(raw) if domain is not None: _fill_ball(ci.balldomain, domain, bbdir) if "memdomain" in raw: _fill_mem(ci.mem, raw["memdomain"], bbdir) - if kind == "boom": - bd = domain - if isinstance(bd, dict) and bd.get("ballNum", 0): - raise ValueError(f"{meta['pkg']}: kind=boom forbids balldomain.ballNum > 0") if "frontend" in raw: _fill_frontend(ci.frontend, raw["frontend"], bbdir) - if "rvv" in raw: - _fill_rvv(ci.rvv, raw["rvv"], bbdir) - - -def _fill_tile(tp: pb.TilePlacement, meta: dict[str, Any], proto: dict[str, Any]) -> None: - tp.path = meta["path"] - if meta["kind"] == "main": - if "factory" in proto or meta["factory_class"]: - raise ValueError("main tile must not specify a factory") - else: - if proto["factory"] != meta["factory_class"]: - raise ValueError("tile factory differs between config and derived metadata") - tp.factory_class = proto["factory"] - tp.kind = pb.TILE_KIND_MAIN if meta["kind"] == "main" else pb.TILE_KIND_TILE - tp.virtual_bank_count = meta["virtual_bank_count"] - tp.core_indices.extend(meta["core_indices"]) - if "tss" in proto: - _fill_spm(tp.tss, proto["tss"]) - if "controller_core_index" in meta: - tp.controller_core_index = meta["controller_core_index"] - tp.mem_ball_channel_num = meta["mem_ball_channel_num"] - _fill_tile_params(tp.param, proto) - sm = proto["sharedMem"] - tp.shared_mem.enable = sm["enable"] - tp.shared_mem.entries = sm["entries"] - tp.shared_mem.bank_entries = sm["bankEntries"] - tp.shared_mem.bank_width = sm["bankWidth"] - tp.shared_mem.input_channels = sm["inputChannels"] - tp.shared_mem.default_group_count = sm["defaultGroupCount"] - if "virtualBankCount" in sm: - tp.shared_mem.virtual_bank_count = sm["virtualBankCount"] - if "memCycles" in sm: - tp.shared_mem.mem_cycles = sm["memCycles"] - - - -def fill_chip(config: dict[str, Any], derived: dict[str, Any], bbdir: Path) -> pb.Chip: - """ - Hardware numbers live in config.json; identities/paths live in derived.json. - chip.pb is a curated schema, so we pick from both instead of dumping either file. - - Core/tile lists vs templates are already expanded in step2 — import that walker, - do not write a second one. - - Pair each tile's hardware parameters with its own derived placement. - """ - b = pb.Chip() + if "gpdomain" in raw: + _fill_gpdomain(ci.gpdomain, raw["gpdomain"], bbdir) + if "transport" in raw: + ParseDict(_hardware_fields(raw["transport"]), ci.transport) + ci.signature = _thread_signature(ci) + + +def fill_design( + config: dict[str, Any], derived: dict[str, Any], bbdir: Path +) -> pb.Design: + """Serialize explicitly declared instances and their typed parameters into design.pb.""" + b = pb.Design() b.name = derived["name"] - b.chip_path = derived["chip_path"] - b.topology_path = derived["tile_config_path"] - b.n_tiles = derived["n_tiles"] + b.design_path = derived["design_path"] + b.n_cores = derived["n_cores"] b.includes.extend(derived["includes"]) - sims = config["sims"] - if "verilator" in sims: - b.mill.verilator_config = sims["verilator"] - if "p2e" in sims: - b.mill.p2e_config = sims["p2e"] - - raw_cores = _raw_core_list(config["designs"]) - meta_cores = derived["cores"] - if len(raw_cores) != len(meta_cores): + for key in ("ddr", "clint", "plic", "scu"): + ParseDict(_hardware_fields(config["device"][key]), getattr(b.device, key)) + bootrom = config["device"]["bootrom"] + image_path = bbdir / bootrom["image"] + image = bytearray(image_path.read_bytes()) + if len(image) < 24 or len(image) > bootrom["bytes"]: + raise ValueError( + "BootROM image must contain its configuration header and fit the ROM" + ) + if any(image[8:24]): raise ValueError( - f"core count mismatch: config={len(raw_cores)} derived={len(meta_cores)}" + "BootROM image configuration header must contain zero placeholders" ) - hart_ids = {hart["core_index"]: hart["hart_id"] for hart in derived["harts"]} - for index, (raw, meta) in enumerate(zip(raw_cores, meta_cores)): - core = b.cores.add() - _fill_core(core, raw, meta, bbdir) - if core.cpu.kind == "ant": - core.ant_context_id = meta["ant_context_id"] - else: - core.hart_id = hart_ids[index] - - raw_tiles = _derive.iter_topology_tiles(config["designs"]) - for raw, meta in zip(raw_tiles, derived["tiles"], strict=True): - _fill_tile(b.tiles.add(), meta, raw) + image[8:16] = b.n_cores.to_bytes(8, "little") + image[16:24] = bootrom["entry"].to_bytes(8, "little") + b.device.bootrom.base = bootrom["base"] + b.device.bootrom.bytes = bootrom["bytes"] + b.device.bootrom.image.extend(byte if byte < 128 else byte - 256 for byte in image) + b.includes.append(_rel(bbdir, image_path)) + + ParseDict( + {key: _hardware_fields(config["sims"][key]) for key in ("diffTest", "memory")}, + b.sims, + ) + if not b.device.IsInitialized() or not b.sims.IsInitialized(): + raise ValueError("Device and simulation parameters must be complete") + + root = Path(derived["design_path"]).parent.parent + for name, registration in config["sims"]["targets"].items(): + if name == "_file": + continue + if name != "device" and not re.fullmatch(r"core/[A-Za-z0-9_-]+", name): + raise ValueError(f"invalid simulation target: {name}") + if name.startswith("core/") and "regression" in registration: + raise ValueError(f"{name}: regression belongs to the device entry") + target = b.simulations[name] + target.directory = str(root / "sims" / name) + for field in ("emu", "stack", "workloads"): + setattr(target, field, str(root / "sims" / name / field)) + target.arch.directory = str(root / "sims" / name / "arch") + target.arch.module = "simulation" + target.arch.config = str(root / "output/generated/design.pb") + target.entry_class = registration["config"] + target.top = "BBSimHarness" + target.stack_name = registration["stack_name"] + manifest = bbdir / target.emu / "Cargo.toml" + target.emu_package = tomllib.loads(manifest.read_text())["package"]["name"] + for resource in ("regression", "kernel", "tapeout"): + if resource in registration: + target.resources[resource] = str(root / registration[resource]) + for entry in ( + Path(target.arch.directory) / "build.sc", + Path(target.stack) / "CMakeLists.txt", + Path(target.workloads) / "CMakeLists.txt", + ): + if not (bbdir / entry).is_file(): + raise FileNotFoundError(bbdir / entry) + from verification import ip_registrations + + profiles = ip_registrations( + bbdir, config["sims"]["verification"]["registry"]["ips"] + ) + b.verification.ips.extend(dict.fromkeys(profile.ip for profile in profiles)) + b.verification.profiles.extend(profiles) + + for raw in _derive.configured_cores(config["device"]): + _append_core(b, raw, bbdir) + + if "sharedBankFabric" in config["device"]: + fabric = config["device"]["sharedBankFabric"] + harts = {core.hart_id for core in b.device.cores} + owner, participants = fabric["ownerHart"], fabric["participantHarts"] + if ( + type(owner) is not int + or owner not in harts + or not participants + or owner in participants + or len(set(participants)) != len(participants) + or set(participants) | {owner} != harts + or len(fabric["bankNodes"]) != len(harts) + ): + raise ValueError( + "sharedBankFabric must explicitly connect each declared hart once" + ) + ParseDict(_hardware_fields(fabric), b.device.shared_bank_fabric) + + domains = [ + core.accelerator.balldomain + for core in b.device.cores + if core.HasField("accelerator") + ] + domains.extend(thread.balldomain for thread in b.threads) + b.verification.balls.extend( + dict.fromkeys(ball.ball_dir for domain in domains for ball in domain.balls) + ) targets = derived["targets"] - for name, target in targets.items(): + compiled = {} + for profile in _derive.compiler_profiles(config["device"]): + name = profile["name"] + raw = profile["config"] + meta = { + "index": 0, + "thread_id": 0, + "role": name, + "pkg": profile["component"], + "config_path": _rel(bbdir, raw["_file"]), + "balldomain_base_dir": _derive._balldomain_base_dir( + bbdir, raw, profile["component"] + ), + } + instance = pb.ThreadInstance() + _fill_thread(instance, raw, meta, bbdir, hardware=False) + if name in compiled: + if compiled[name].config.signature != instance.signature: + raise ValueError( + f"compiler {name}: instances have different parameters; declare distinct compiler names" + ) + continue p = b.profiles.add() + compiled[name] = p p.name = name - p.pkg = target["pkg"] - p.compiler = target["compiler"] - bank = target["bank"] - p.bank_num = bank["num"] - p.bank_width = bank["width"] - p.bank_entries = bank["entries"] - p.ball_ctest_dirs.extend(target["ball_ctest_dirs"]) + p.pkg = profile["component"] + p.compiler = "compiler" + bank = targets[name]["bank"] + p.bank_num, p.bank_width, p.bank_entries = ( + bank["num"], + bank["width"], + bank["entries"], + ) + p.ball_ctest_dirs.extend(targets[name]["ball_ctest_dirs"]) + p.config.CopyFrom(instance) + + for name, selected in config["workloads"].get("profiles", {}).items(): + if name == "_file": + continue + if ( + type(selected) is not int + or selected < 0 + or not any( + c.hart_id == selected and c.profile == name for c in b.device.cores + ) + ): + raise ValueError( + f"compiler {name}: test hart is not a direct accelerator binding" + ) + compiled[name].test_core = selected bemu = derived["bemu"] - b.bemu.chip_main = bemu["chip_main"] - b.bemu.tile_index = bemu["tile_index"] for ball in bemu["balls"]: e = b.bemu.balls.add() e.ball_class = ball["ball_class"] e.ball_dir = ball["ball_dir"] e.emu_lib = ball["emu_lib"] + harts = {core.hart_id for core in b.device.cores} + tests = config["workloads"] + for key, field in (("controlHart", "control_hart"), ("reportHart", "report_hart")): + hart = tests[key] + if type(hart) is not int or hart not in harts: + raise ValueError(f"workload {key}: unknown hart {hart!r}") + setattr(b.workload, field, hart) + for key, val in derived["workload"]["cmake_param"].items(): b.workload.cmake_param[key] = val return b -def write_chip_pb( +def write_thread_pb(raw: dict[str, Any], out_path: Path, bbdir: Path, pkg: str) -> Path: + """One thread config, independent of any design: the compiler input.""" + rel = _rel(bbdir, raw["_file"]) + meta = { + "index": 0, + "thread_id": 0, + "role": "", + "pkg": pkg, + "config_path": rel, + "balldomain_base_dir": _derive._balldomain_base_dir(bbdir, raw, pkg), + } + thread = pb.ThreadInstance() + _fill_thread(thread, raw, meta, bbdir, hardware=True) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(thread.SerializeToString()) + return out_path + + +def _append_core(design: pb.Design, raw: dict[str, Any], bbdir: Path) -> None: + core = design.device.cores.add() + core.path = _rel(bbdir, raw["_file"]) + component = raw.get("component") + if not isinstance(component, str) or not component: + raise ValueError("Core component must be explicitly named") + core.component = component + core.profile = raw["compiler"]["name"] if "compiler" in raw else "" + core.hart_id = raw["hartId"] + core.reset_vector = raw["resetVector"] + core.mtime_address = raw["mtimeAddress"] + for key in ("memory", "fetch", "fpu", "interconnect", "c2c", "controller"): + ParseDict(_hardware_fields(raw[key]), getattr(core, key)) + if "tbus" in raw: + ParseDict(_hardware_fields(raw["tbus"]), core.tbus) + if "dmaCache" in raw: + ParseDict(_hardware_fields(raw["dmaCache"]), core.dma_cache) + if not core.IsInitialized(): + raise ValueError("Core IP configuration is incomplete") + if "implementation" in raw: + from importlib import import_module + + implementation = import_module( + f"{raw['implementation']['component']}_params_pb2" + ).Params() + ParseDict( + _hardware_fields( + {k: v for k, v in raw["implementation"].items() if k != "component"} + ), + implementation, + ) + core.implementation.Pack(implementation, deterministic=True) + if "param" in raw or "implementation" not in raw: + _fill_core_params(core.param, raw["param"]) + if "tss" in raw: + _fill_spm(core.tss, raw["tss"]) + shared = raw["sharedMem"] + core.virtual_bank_count = shared["virtualBankCount"] + core.shared_mem.enable = shared["enable"] + core.shared_mem.entries = shared["entries"] + core.shared_mem.bank_entries = shared["bankEntries"] + core.shared_mem.bank_width = shared["bankWidth"] + core.shared_mem.input_channels = shared["inputChannels"] + core.shared_mem.default_group_count = shared["defaultGroupCount"] + core.shared_mem.virtual_bank_count = shared["virtualBankCount"] + core.shared_mem.mem_cycles = shared["memCycles"] + if "accelerator" in raw: + acc = raw["accelerator"] + _fill_ball(core.accelerator.balldomain, acc["balldomain"], bbdir) + _fill_mem(core.accelerator.mem, acc["memdomain"], bbdir) + _fill_frontend(core.accelerator.frontend, acc["frontend"], bbdir) + _fill_gpdomain(core.accelerator.gpdomain, acc["gpdomain"], bbdir) + ParseDict(_hardware_fields(acc["transport"]), core.accelerator.transport) + threads = raw["threads"] + ids = [thread["id"] for thread in threads] + if any(type(value) is not int or value < 0 for value in ids) or len( + set(ids) + ) != len(ids): + raise ValueError( + "Core-local thread IDs must be explicit, non-negative and unique" + ) + offset = len(design.threads) + for local, thread in enumerate(threads): + index = offset + local + package = thread["component"] + if not isinstance(package, str) or not package.isidentifier(): + raise ValueError("invalid thread component") + meta = { + "index": index, + "thread_id": thread["id"], + "role": thread["name"], + "pkg": package, + "config_path": _rel(bbdir, thread["_file"]), + "balldomain_base_dir": _derive._balldomain_base_dir(bbdir, thread, package), + } + item = design.threads.add() + _fill_thread(item, thread, meta, bbdir, hardware=True) + core.thread_indices.append(index) + # The signature names the kind of core software runs on; where an instance + # sits (hart, config file, NoC nodes) is not part of it. + hardware = pb.CorePlacement() + hardware.CopyFrom(core) + for field in ("hart_id", "path", "thread_indices"): + hardware.ClearField(field) + + def clear_nodes(message): + for field, value in message.ListFields(): + if field.name == "node_id": + message.node_id = 0 + elif field.message_type is not None: + for item in [value] if hasattr(value, "ListFields") else value: + clear_nodes(item) + + clear_nodes(hardware) + payload = hardware.SerializeToString(deterministic=True) + b"".join( + design.threads[index].signature.to_bytes(8, "little") + for index in core.thread_indices + ) + core.signature = int.from_bytes(hashlib.sha256(payload).digest()[:8], "little") + + +def write_core_pb(raw: dict[str, Any], out_path: Path, bbdir: Path) -> Path: + design = pb.Design(name=f"hart-{raw['hartId']}", n_cores=1) + _append_core(design, raw, bbdir) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(design.SerializeToString(deterministic=True)) + return out_path + + +def write_design_pb( config_path: Path, derived_path: Path, out_path: Path, @@ -394,7 +565,39 @@ def write_chip_pb( ) -> Path: config = json.loads(config_path.read_text(encoding="utf-8")) derived = json.loads(derived_path.read_text(encoding="utf-8")) - chip = fill_chip(config, derived, bbdir) + design = fill_design(config, derived, bbdir) out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_bytes(chip.SerializeToString()) + out_path.write_bytes(design.SerializeToString()) return out_path + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Build a configured core protobuf using the standard TOML expansion" + ) + source_kind = parser.add_mutually_exclusive_group(required=True) + source_kind.add_argument("--core", type=Path) + source_kind.add_argument("--design", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--repo", type=Path, required=True) + options = parser.parse_args() + source = Path(__file__).with_name("1_toml2json.py") + spec = importlib.util.spec_from_file_location("toml_expansion", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + if options.core is not None: + output = write_core_pb( + module.toml2json(options.core), options.output, options.repo + ) + else: + data = module.design2json(options.design) + derived = _derive.derive( + data, options.repo, options.design.resolve().parent.parent.name + ) + design = fill_design(data, derived, options.repo) + options.output.parent.mkdir(parents=True, exist_ok=True) + options.output.write_bytes(design.SerializeToString(deterministic=True)) + output = options.output + print(output) diff --git a/bbdev/api/steps/config/scripts/4_install.py b/bbdev/api/steps/config/scripts/4_install.py deleted file mode 100644 index 2b0556d81..000000000 --- a/bbdev/api/steps/config/scripts/4_install.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Install Chip.pb into arch / bemu / compiler / workload build trees.""" - -from __future__ import annotations -import os -import shutil -import sys -from pathlib import Path - -if str(Path(__file__).resolve().parent) not in sys.path: - sys.path.insert(0, str(Path(__file__).resolve().parent)) -import chip_pb2 as pb - - -def load_chip(pb_path: Path) -> pb.Chip: - chip = pb.Chip() - chip.ParseFromString(pb_path.read_bytes()) - if not chip.name: - raise ValueError(f"empty Chip.name in {pb_path}") - if not chip.cores: - raise ValueError(f"no cores in {pb_path}") - return chip - - -def _emit_dispatch(chip: pb.Chip, bbdir: Path, out: Path) -> None: - balls = list(chip.bemu.balls) - if not balls: - out.write_text( - 'use crate::inst::instruction::ExecContext;\n\npub fn execute_known(\n _ball_class: &str,\n _funct: u32,\n _xs1: u64,\n _xs2: u64,\n _ctx: &mut ExecContext,\n) -> u64 {\n panic!("no BEMU ball implementation")\n}\n\npub fn cycles_after_issue(_ball_class: &str, _funct: u32, _xs1: u64, _xs2: u64) -> u64 { panic!("no BEMU latency contract") }\n', - encoding="utf-8", - ) - return - lines = ["use crate::inst::instruction::ExecContext;", ""] - for ball in balls: - if not ball.ball_dir.isidentifier(): - raise ValueError(f"ball_dir is not a Rust identifier: {ball.ball_dir!r}") - lib = (bbdir / ball.emu_lib).resolve() - if not lib.is_file(): - raise FileNotFoundError(f"missing ball emu: {lib}") - path = str(lib) - if '"' in path or "\\" in path: - raise ValueError(f"ball emu path not usable in rust #[path]: {path}") - lines.append(f'#[path = "{path}"]') - lines.append(f"mod {ball.ball_dir};") - lines.append("") - - def chain(fn: str, ctx: bool, panic_msg: str) -> list[str]: - extra = ", ctx" if ctx else "" - body = [f" {balls[0].ball_dir}::{fn}(ball_class, funct, xs1, xs2{extra})"] - for ball in balls[1:]: - body.append( - f" .or_else(|| {ball.ball_dir}::{fn}(ball_class, funct, xs1, xs2{extra}))" - ) - body.append(f' .unwrap_or_else(|| panic!("{panic_msg}"))') - return body - - lines += [ - "pub fn execute_known(", - " ball_class: &str,", - " funct: u32,", - " xs1: u64,", - " xs2: u64,", - " ctx: &mut ExecContext,", - ") -> u64 {", - ] - lines += chain( - "execute_known", - True, - "no BEMU ball implementation for ballClass={ball_class} funct7={funct}", - ) - lines += ["}", ""] - lines += [ - "pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> u64 {", - ] - lines += chain("cycles_after_issue", False, - "no BEMU latency contract for ballClass={ball_class} funct7={funct}") - lines += ["}", ""] - out.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def install_arch(src_pb: Path, gen: Path) -> Path: - dest = gen / "chip.pb" - dest.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src_pb, dest) - return dest - - -def install_bemu(chip: pb.Chip, bbdir: Path, gen: Path) -> Path: - src = bbdir / "bebop" / "src" / "nodes" / "bemu" / "chip" - cargo = src / "Cargo.toml" - build_rs = src / "build.rs" - if not cargo.is_file(): - raise FileNotFoundError(f"missing {cargo}") - if not build_rs.is_file(): - raise FileNotFoundError(f"missing {build_rs}") - bemu = gen / "bemu" - bemu.mkdir(parents=True, exist_ok=True) - _emit_dispatch(chip, bbdir, bemu / "dispatch.rs") - target = bemu / build_rs.name - target.unlink(missing_ok=True) - target.symlink_to(os.path.relpath(build_rs, bemu)) - # Each chip owns its crate: the single-core bebop-bemu plus its bebop-chip- entry, which - # is the chip's own emu/src/main.rs when present and the default tile runner otherwise. - entry = bbdir / chip.bemu.chip_main if chip.bemu.chip_main else src / "main.rs" - manifest = bemu / "Cargo.toml" - manifest.unlink(missing_ok=True) - manifest.write_text( - cargo.read_text(encoding="utf-8") - + f'\n[[bin]]\nname = "bebop-chip-{chip.name}"\npath = "{os.path.relpath(entry, bemu)}"\n', - encoding="utf-8", - ) - return manifest - - -def install_bebop(gen: Path) -> Path: - bebop = gen.parent.parent / "generated" / "bebop" - bebop.mkdir(parents=True, exist_ok=True) - manifest = bebop / "Cargo.toml" - manifest.write_text( - '[workspace]\nresolver = "2"\n\n[package]\nname = "bebop"\nversion = "0.1.0"\nedition = "2021"\nbuild = "../../../../../bebop/build.rs"\n\n[[bin]]\nname = "bebop"\npath = "../../../../../bebop/src/main.rs"\n\n[[test]]\nname = "test_verilator"\npath = "../../../../../bebop/tests/test_verilator.rs"\nharness = false\nrequired-features = ["verilator"]\n\n[[test]]\nname = "test_p2e"\npath = "../../../../../bebop/tests/test_p2e.rs"\nharness = false\nrequired-features = ["p2e"]\n\n[features]\ndefault = []\nverilator = ["dep:bebop-verilator"]\np2e = ["dep:bebop-p2e", "bebop-bemu?/p2e"]\nbemu = ["dep:bebop-bemu", "bebop-p2e?/diff"]\n\n[dependencies]\nbebop-verilator = { path = "../../../../../bebop/src/nodes/verilator", optional = true }\nbebop-p2e = { path = "../../../../../bebop/src/nodes/p2e", optional = true }\nbebop-bemu = { path = "../../configs/generated/bemu", optional = true }\nbebop-dasm = { path = "../../../../../bebop/src/nodes/lib/dasm" }\nbebop-bank-hash = { path = "../../../../../bebop/src/nodes/lib/bank-hash" }\nbebop-bemu-profile = { path = "../../../../../bebop/src/nodes/lib/bemu-profile" }\nbebop-fd-redirect = { path = "../../../../../bebop/src/nodes/lib/fd-redirect" }\nbebop-rtl-trace = { path = "../../../../../bebop/src/nodes/lib/rtl-trace" }\nbebop-uart = { path = "../../../../../bebop/src/nodes/lib/uart" }\nclap = { version = "4", features = ["derive"] }\nlibc = "0.2"\nlog = "0.4"\nenv_logger = "0.11"\nnix = { version = "0.29", features = ["fs", "mman", "signal", "process"] }\ntoml = "0.8"\ncamino = "1.1"\nsnafu = "0.8"\nserde = { version = "1", features = ["derive"] }\nserde_json = "1"\nduct = "0.13"\n\n[dev-dependencies]\nlibtest-mimic = "0.8"\nassert_cmd = "2"\nwalkdir = "2"\nchrono = { version = "0.4", default-features = false, features = ["clock"] }\n', - encoding="utf-8", - ) - return manifest - - -def install_workload(chip: pb.Chip, bbdir: Path, name: str, gen: Path) -> Path: - if not chip.profiles: - raise ValueError(f"chip {name}: no compiler profiles") - defs = { - "BUCKYBALL_WORKLOAD_CHIP": name, - "BUCKYBALL_CHIP_PB": str(gen / "chip.pb"), - } - out = gen / "workload" / "cmake.defs" - out.parent.mkdir(parents=True, exist_ok=True) - body = "".join((f"{k}={v}\n" for k, v in sorted(defs.items()))) - out.write_text(body, encoding="utf-8") - return out diff --git a/bbdev/api/steps/config/scripts/ball_normalize.py b/bbdev/api/steps/config/scripts/ball_normalize.py index 1cdd80d3c..872cdd4c0 100644 --- a/bbdev/api/steps/config/scripts/ball_normalize.py +++ b/bbdev/api/steps/config/scripts/ball_normalize.py @@ -1,37 +1,45 @@ -"""Validate and copy external Ball mappings.""" +"""Validate Ball instances and their directly owned instructions.""" from copy import deepcopy from typing import Any -def normalize_ball_domain(core: dict[str, Any]) -> dict[str, Any] | None: - raw = core.get("balldomain") +def normalize_ball_domain(thread: dict[str, Any]) -> dict[str, Any] | None: + raw = thread.get("balldomain") if raw is None: return None - if not isinstance(raw, dict): - raise ValueError("Ball domain must be a table") domain = deepcopy(raw) - mappings = domain.get("ballIdMappings") - isa = domain.get("ballISA") - if not isinstance(mappings, list) or not isinstance(isa, list): - raise ValueError("Ball domain requires ballIdMappings and ballISA lists") - if any(not isinstance(entry, dict) for entry in isa): - raise ValueError("Ball ISA entries must be tables") - if type(domain.get("ballNum")) is not int or domain["ballNum"] != len(mappings): - raise ValueError("ballNum must equal the number of mappings") - for index, mapping in enumerate(mappings): - if not isinstance(mapping, dict) or type(mapping.get("ballId")) is not int or mapping["ballId"] != index: - raise ValueError("Ball IDs must enumerate mappings without gaps or reordering") - if "builtin" in mapping: - raise ValueError("Ball mappings must select external Balls") - klass = mapping.get("ballClass") - config = mapping.get("config") - if not isinstance(klass, str) or not klass.startswith("examples.balls."): - raise ValueError("external Ball class must be under examples.balls") - if not isinstance(config, dict) or not isinstance(config.get("_file"), str) or not isinstance(config.get("ball"), dict): - raise ValueError("external Ball requires a resolved config with [ball] parameters") - if any(type(mapping.get(key)) is not int or mapping[key] <= 0 for key in ("inBW", "outBW")): - raise ValueError("external Ball inBW/outBW must be positive integers") - if any(entry.get("funct7") == 15 or entry.get("mnemonic") == "RUN_KERNEL" for entry in isa): - raise ValueError("RUN_KERNEL is a global control instruction, not a Ball instruction") + balls = domain["balls"] + ids, names, codes = set(), set(), set() + for ball in balls: + identity, name = ball["id"], ball["name"] + if type(identity) is not int or not 0 <= identity < 32 or identity in ids: + raise ValueError(f"Invalid or duplicate Ball ID: {identity}") + if name in names: + raise ValueError(f"Duplicate Ball instance name: {name}") + ids.add(identity) + names.add(name) + if not ball["component"].isidentifier(): + raise ValueError(f"Invalid Ball component: {ball['component']}") + port = ball["port"] + if any( + type(port[key]) is not int or port[key] <= 0 for key in ("inBW", "outBW") + ): + raise ValueError( + "Ball input and output bandwidths must be positive integers" + ) + if any( + type(port[key]) is not int or port[key] < 0 + for key in ("mmioReadBW", "mmioWriteBW") + ): + raise ValueError("Ball MMIO bandwidths must be non-negative integers") + mnemonics = set() + for instruction in ball["instructions"]: + code, mnemonic = instruction["funct7"], instruction["mnemonic"] + if type(code) is not int or not 0 <= code < 128 or code in codes: + raise ValueError(f"Invalid or duplicate instruction encoding: {code}") + if mnemonic in mnemonics or code == 15 or mnemonic == "RUN_KERNEL": + raise ValueError(f"Invalid or duplicate Ball instruction: {mnemonic}") + codes.add(code) + mnemonics.add(mnemonic) return domain diff --git a/bbdev/api/steps/config/scripts/ball_parameters.py b/bbdev/api/steps/config/scripts/ball_parameters.py new file mode 100644 index 000000000..1bf89dd9a --- /dev/null +++ b/bbdev/api/steps/config/scripts/ball_parameters.py @@ -0,0 +1,31 @@ +"""Typed parameter messages owned by individual Ball components.""" + +from importlib import import_module +from google.protobuf.json_format import ParseDict + + +def pack_parameters(instance, values): + message = import_module(f"{instance.ball_dir}_params_pb2").Params() + ParseDict(values, message) + missing = [ + field.name + for field in message.DESCRIPTOR.fields + if not message.HasField(field.name) + ] + if missing: + raise ValueError(f"{instance.ball_name}: missing parameters {missing}") + instance.parameters.Pack(message, deterministic=True) + + +def unpack_parameters(instance): + message = import_module(f"{instance.ball_dir}_params_pb2").Params() + if not instance.parameters.Unpack(message): + raise ValueError(f"{instance.ball_name}: wrong parameter message type") + missing = [ + field.name + for field in message.DESCRIPTOR.fields + if not message.HasField(field.name) + ] + if missing: + raise ValueError(f"{instance.ball_name}: missing parameters {missing}") + return message diff --git a/bbdev/api/steps/config/scripts/boom_params_pb2.py b/bbdev/api/steps/config/scripts/boom_params_pb2.py new file mode 100644 index 000000000..02f769d2a --- /dev/null +++ b/bbdev/api/steps/config/scripts/boom_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: boom_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "boom_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x11\x62oom_params.proto\x12\x19\x62uckyball.components.boom"\x19\n\x06Params\x12\x0f\n\x07profile\x18\x01 \x02(\tB\x1d\n\x19\x62uckyball.components.boomP\x01' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "boom_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\031buckyball.components.boomP\001" + _globals["_PARAMS"]._serialized_start = 48 + _globals["_PARAMS"]._serialized_end = 73 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/f32add_params_pb2.py b/bbdev/api/steps/config/scripts/f32add_params_pb2.py new file mode 100644 index 000000000..8d13feb6b --- /dev/null +++ b/bbdev/api/steps/config/scripts/f32add_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: f32add_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "f32add_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x13\x66\x33\x32\x61\x64\x64_params.proto\x12\x16\x62uckyball.balls.f32add"\x08\n\x06ParamsB\x1a\n\x16\x62uckyball.balls.f32addP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "f32add_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\026buckyball.balls.f32addP\001" + _globals["_PARAMS"]._serialized_start = 47 + _globals["_PARAMS"]._serialized_end = 55 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/gemmini_params_pb2.py b/bbdev/api/steps/config/scripts/gemmini_params_pb2.py new file mode 100644 index 000000000..81c248f11 --- /dev/null +++ b/bbdev/api/steps/config/scripts/gemmini_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: gemmini_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "gemmini_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x14gemmini_params.proto\x12\x17\x62uckyball.balls.gemmini"\xfc\x02\n\x06Params\x12\x15\n\x08meshRows\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x18\n\x0bmeshColumns\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x15\n\x08tileRows\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x18\n\x0btileColumns\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12\x17\n\ninputWidth\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x15\n\x08\x61\x63\x63Width\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x1f\n\x12spatialOutputWidth\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12\x18\n\x0btileLatency\x18\x08 \x01(\x05H\x07\x88\x01\x01\x12\x18\n\x0boutputDelay\x18\t \x01(\x05H\x08\x88\x01\x01\x42\x0b\n\t_meshRowsB\x0e\n\x0c_meshColumnsB\x0b\n\t_tileRowsB\x0e\n\x0c_tileColumnsB\r\n\x0b_inputWidthB\x0b\n\t_accWidthB\x15\n\x13_spatialOutputWidthB\x0e\n\x0c_tileLatencyB\x0e\n\x0c_outputDelayB\x1b\n\x17\x62uckyball.balls.gemminiP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "gemmini_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\027buckyball.balls.gemminiP\001" + _globals["_PARAMS"]._serialized_start = 50 + _globals["_PARAMS"]._serialized_end = 430 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/hardware_pb2.py b/bbdev/api/steps/config/scripts/hardware_pb2.py new file mode 100644 index 000000000..389cbeec6 --- /dev/null +++ b/bbdev/api/steps/config/scripts/hardware_pb2.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: hardware.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "hardware.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x0ehardware.proto\x12\x10\x62uckyball.config"L\n\x0b\x43hiHardware\x12\x14\n\x0cnode_id_bits\x18\x01 \x02(\r\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x02 \x02(\r\x12\x11\n\tdata_bits\x18\x03 \x02(\r"G\n\x0b\x41xiHardware\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x01 \x02(\r\x12\x11\n\tdata_bits\x18\x02 \x02(\r\x12\x0f\n\x07id_bits\x18\x03 \x02(\r"r\n\x10TrackingHardware\x12\x0f\n\x07\x65ntries\x18\x01 \x02(\r\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x02 \x02(\r\x12\x0f\n\x07id_bits\x18\x03 \x02(\r\x12\x12\n\nline_bytes\x18\x04 \x02(\r\x12\x12\n\nmax_ranges\x18\x05 \x02(\r"\x98\x01\n\x0eRegionHardware\x12\x0c\n\x04\x62\x61se\x18\x01 \x02(\x04\x12\r\n\x05\x62ytes\x18\x02 \x02(\x04\x12\x11\n\tcacheable\x18\x03 \x02(\x08\x12\x12\n\nexecutable\x18\x04 \x02(\x08\x12\x10\n\x08readable\x18\x05 \x02(\x08\x12\x10\n\x08writable\x18\x06 \x02(\x08\x12\x0e\n\x06\x61tomic\x18\x07 \x02(\x08\x12\x0e\n\x06normal\x18\x08 \x02(\x08"\x95\x01\n\rCacheHardware\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x01 \x02(\r\x12\x12\n\nline_bytes\x18\x02 \x02(\r\x12\x0c\n\x04sets\x18\x03 \x02(\r\x12\x0c\n\x04ways\x18\x04 \x02(\r\x12\x0f\n\x07id_bits\x18\x05 \x02(\r\x12\x15\n\rmetadata_bits\x18\x06 \x02(\r\x12\x16\n\x0eresponse_depth\x18\x07 \x02(\r"\x91\x01\n\x0bRnfHardware\x12\x0f\n\x07node_id\x18\x01 \x02(\r\x12\x13\n\x0b\x63\x61\x63he_lines\x18\x02 \x02(\r\x12\x0f\n\x07home_id\x18\x03 \x02(\r\x12\x12\n\nhome_count\x18\x04 \x02(\r\x12\r\n\x05\x62\x61nks\x18\x05 \x02(\r\x12\x13\n\x0bline_result\x18\x06 \x02(\x08\x12\r\n\x05probe\x18\x07 \x02(\x08J\x04\x08\x08\x10\t"\xa8\x01\n\x0cHomeHardware\x12.\n\x05\x63\x61\x63he\x18\x01 \x02(\x0b\x32\x1f.buckyball.config.CacheHardware\x12\x0e\n\x06\x61gents\x18\x02 \x02(\r\x12\x14\n\x0cmshr_entries\x18\x03 \x02(\r\x12\x0f\n\x07home_id\x18\x04 \x02(\r\x12\x31\n\nrequesters\x18\x05 \x02(\x0b\x32\x1d.buckyball.config.ChiHardware"\x8c\x01\n\x13PreparationHardware\x12*\n\x03\x62us\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.ChiHardware\x12\x10\n\x08\x63ontexts\x18\x02 \x02(\r\x12\x12\n\nmax_ranges\x18\x03 \x02(\r\x12\x0f\n\x07id_bits\x18\x04 \x02(\r\x12\x12\n\nbeat_bytes\x18\x05 \x02(\r"\xe1\x01\n\x11TransportHardware\x12\x34\n\x08tracking\x18\x01 \x02(\x0b\x32".buckyball.config.TrackingHardware\x12\x37\n\x08prepared\x18\x02 \x02(\x0b\x32%.buckyball.config.PreparationHardware\x12*\n\x03\x61xi\x18\x03 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware\x12\x31\n\x07regions\x18\x04 \x03(\x0b\x32 .buckyball.config.RegionHardware"\xc6\x01\n\x0b\x44\x64rHardware\x12+\n\x04line\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.ChiHardware\x12\x0f\n\x07\x63lients\x18\x02 \x02(\r\x12\x18\n\x10slots_per_client\x18\x03 \x02(\r\x12\x11\n\tdata_bits\x18\x04 \x02(\r\x12\x0f\n\x07id_bits\x18\x05 \x02(\r\x12\x0f\n\x07id_base\x18\x06 \x02(\r\x12\x0c\n\x04\x62\x61se\x18\x07 \x02(\x04\x12\r\n\x05\x62ytes\x18\x08 \x02(\x04\x12\r\n\x05model\x18\t \x02(\x04"=\n\x0f\x42ootRomHardware\x12\x0c\n\x04\x62\x61se\x18\x01 \x02(\x04\x12\r\n\x05\x62ytes\x18\x02 \x02(\x04\x12\r\n\x05image\x18\x03 \x03(\x11"S\n\rClintHardware\x12\x10\n\x08hart_ids\x18\x01 \x03(\r\x12\x0c\n\x04\x62\x61se\x18\x02 \x02(\x04\x12\r\n\x05\x62ytes\x18\x03 \x02(\x04\x12\x13\n\x0btick_cycles\x18\x04 \x02(\r"e\n\x0cPlicHardware\x12\x10\n\x08hart_ids\x18\x01 \x03(\r\x12\x0c\n\x04\x62\x61se\x18\x02 \x02(\x04\x12\r\n\x05\x62ytes\x18\x03 \x02(\x04\x12\x0f\n\x07sources\x18\x04 \x02(\r\x12\x15\n\rpriority_bits\x18\x05 \x02(\r"f\n\x0bScuHardware\x12\x14\n\x0c\x62\x61se_address\x18\x01 \x02(\x04\x12\x14\n\x0cstride_bytes\x18\x02 \x02(\x04\x12\x18\n\x10total_size_bytes\x18\x03 \x02(\x04\x12\x11\n\tmax_harts\x18\x04 \x02(\r"E\n\x11SimMemoryHardware\x12\x12\n\nline_bytes\x18\x03 \x02(\r\x12\x10\n\x08\x63lock_hz\x18\x04 \x02(\x04J\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03"[\n\x0bSimHardware\x12\x11\n\tdiff_test\x18\x01 \x02(\x08\x12\x33\n\x06memory\x18\x02 \x02(\x0b\x32#.buckyball.config.SimMemoryHardwareJ\x04\x08\x03\x10\x04"\xa3\x01\n\x12MemoryAccessConfig\x12\x17\n\x0f\x61\x63\x63\x65ss_tag_bits\x18\x02 \x02(\r\x12\x1b\n\x13translation_entries\x18\x03 \x02(\r\x12\x18\n\x10result_line_bits\x18\x04 \x02(\r\x12\x31\n\x07regions\x18\x06 \x03(\x0b\x32 .buckyball.config.RegionHardwareJ\x04\x08\x01\x10\x02J\x04\x08\x05\x10\x06"r\n\x0c\x44\x43\x61\x63heConfig\x12,\n\x05\x63\x61\x63he\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.RnfHardware\x12\x34\n\x08tracking\x18\x02 \x02(\x0b\x32".buckyball.config.TrackingHardware"K\n\x0eUncachedConfig\x12\r\n\x05ports\x18\x01 \x02(\r\x12*\n\x03\x61xi\x18\x02 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware"\x8b\x02\n\x0cMemoryConfig\x12\x32\n\x08uncached\x18\x06 \x02(\x0b\x32 .buckyball.config.UncachedConfig\x12\x34\n\x06\x61\x63\x63\x65ss\x18\x02 \x02(\x0b\x32$.buckyball.config.MemoryAccessConfig\x12-\n\x06icache\x18\x03 \x02(\x0b\x32\x1d.buckyball.config.RnfHardware\x12.\n\x06\x64\x63\x61\x63he\x18\x04 \x02(\x0b\x32\x1e.buckyball.config.DCacheConfig\x12,\n\x04home\x18\x05 \x02(\x0b\x32\x1e.buckyball.config.HomeHardwareJ\x04\x08\x01\x10\x02"(\n\x0b\x46\x65tchConfig\x12\x19\n\x11instruction_bytes\x18\x01 \x02(\r"\x1a\n\tFpuConfig\x12\r\n\x05\x66_len\x18\x01 \x02(\r"O\n\x12InterconnectConfig\x12*\n\x03\x61xi\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware\x12\r\n\x05slots\x18\x02 \x02(\r"\\\n\tC2CConfig\x12\x0f\n\x07\x65nabled\x18\x01 \x02(\x08\x12\x12\n\npeer_cores\x18\x02 \x02(\r\x12*\n\x03\x61xi\x18\x03 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware" \n\nTBusConfig\x12\x12\n\nbank_nodes\x18\x01 \x03(\r"!\n\x10\x43ontrollerConfig\x12\r\n\x05moves\x18\x01 \x02(\x08\x42\x14\n\x10\x62uckyball.configP\x01' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "hardware_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\020buckyball.configP\001" + _globals["_CHIHARDWARE"]._serialized_start = 36 + _globals["_CHIHARDWARE"]._serialized_end = 112 + _globals["_AXIHARDWARE"]._serialized_start = 114 + _globals["_AXIHARDWARE"]._serialized_end = 185 + _globals["_TRACKINGHARDWARE"]._serialized_start = 187 + _globals["_TRACKINGHARDWARE"]._serialized_end = 301 + _globals["_REGIONHARDWARE"]._serialized_start = 304 + _globals["_REGIONHARDWARE"]._serialized_end = 456 + _globals["_CACHEHARDWARE"]._serialized_start = 459 + _globals["_CACHEHARDWARE"]._serialized_end = 608 + _globals["_RNFHARDWARE"]._serialized_start = 611 + _globals["_RNFHARDWARE"]._serialized_end = 756 + _globals["_HOMEHARDWARE"]._serialized_start = 759 + _globals["_HOMEHARDWARE"]._serialized_end = 927 + _globals["_PREPARATIONHARDWARE"]._serialized_start = 930 + _globals["_PREPARATIONHARDWARE"]._serialized_end = 1070 + _globals["_TRANSPORTHARDWARE"]._serialized_start = 1073 + _globals["_TRANSPORTHARDWARE"]._serialized_end = 1298 + _globals["_DDRHARDWARE"]._serialized_start = 1301 + _globals["_DDRHARDWARE"]._serialized_end = 1499 + _globals["_BOOTROMHARDWARE"]._serialized_start = 1501 + _globals["_BOOTROMHARDWARE"]._serialized_end = 1562 + _globals["_CLINTHARDWARE"]._serialized_start = 1564 + _globals["_CLINTHARDWARE"]._serialized_end = 1647 + _globals["_PLICHARDWARE"]._serialized_start = 1649 + _globals["_PLICHARDWARE"]._serialized_end = 1750 + _globals["_SCUHARDWARE"]._serialized_start = 1752 + _globals["_SCUHARDWARE"]._serialized_end = 1854 + _globals["_SIMMEMORYHARDWARE"]._serialized_start = 1856 + _globals["_SIMMEMORYHARDWARE"]._serialized_end = 1925 + _globals["_SIMHARDWARE"]._serialized_start = 1927 + _globals["_SIMHARDWARE"]._serialized_end = 2018 + _globals["_MEMORYACCESSCONFIG"]._serialized_start = 2021 + _globals["_MEMORYACCESSCONFIG"]._serialized_end = 2184 + _globals["_DCACHECONFIG"]._serialized_start = 2186 + _globals["_DCACHECONFIG"]._serialized_end = 2300 + _globals["_UNCACHEDCONFIG"]._serialized_start = 2302 + _globals["_UNCACHEDCONFIG"]._serialized_end = 2377 + _globals["_MEMORYCONFIG"]._serialized_start = 2380 + _globals["_MEMORYCONFIG"]._serialized_end = 2647 + _globals["_FETCHCONFIG"]._serialized_start = 2649 + _globals["_FETCHCONFIG"]._serialized_end = 2689 + _globals["_FPUCONFIG"]._serialized_start = 2691 + _globals["_FPUCONFIG"]._serialized_end = 2717 + _globals["_INTERCONNECTCONFIG"]._serialized_start = 2719 + _globals["_INTERCONNECTCONFIG"]._serialized_end = 2798 + _globals["_C2CCONFIG"]._serialized_start = 2800 + _globals["_C2CCONFIG"]._serialized_end = 2892 + _globals["_TBUSCONFIG"]._serialized_start = 2894 + _globals["_TBUSCONFIG"]._serialized_end = 2926 + _globals["_CONTROLLERCONFIG"]._serialized_start = 2928 + _globals["_CONTROLLERCONFIG"]._serialized_end = 2961 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/im2col_params_pb2.py b/bbdev/api/steps/config/scripts/im2col_params_pb2.py new file mode 100644 index 000000000..ac744c174 --- /dev/null +++ b/bbdev/api/steps/config/scripts/im2col_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: im2col_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "im2col_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x13im2col_params.proto\x12\x16\x62uckyball.balls.im2col"\x9e\x01\n\x06Params\x12\x17\n\ninputWidth\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x14\n\x07maxIter\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x15\n\x08maxKSize\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x17\n\nmaxPadding\x18\x04 \x01(\x05H\x03\x88\x01\x01\x42\r\n\x0b_inputWidthB\n\n\x08_maxIterB\x0b\n\t_maxKSizeB\r\n\x0b_maxPaddingB\x1a\n\x16\x62uckyball.balls.im2colP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "im2col_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\026buckyball.balls.im2colP\001" + _globals["_PARAMS"]._serialized_start = 48 + _globals["_PARAMS"]._serialized_end = 206 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/int2fp_params_pb2.py b/bbdev/api/steps/config/scripts/int2fp_params_pb2.py new file mode 100644 index 000000000..61c5f0b33 --- /dev/null +++ b/bbdev/api/steps/config/scripts/int2fp_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: int2fp_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "int2fp_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x13int2fp_params.proto\x12\x16\x62uckyball.balls.int2fp"V\n\x06Params\x12\x16\n\tinputType\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\noutputType\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\x0c\n\n_inputTypeB\r\n\x0b_outputTypeB\x1a\n\x16\x62uckyball.balls.int2fpP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "int2fp_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\026buckyball.balls.int2fpP\001" + _globals["_PARAMS"]._serialized_start = 47 + _globals["_PARAMS"]._serialized_end = 133 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/int8add_params_pb2.py b/bbdev/api/steps/config/scripts/int8add_params_pb2.py new file mode 100644 index 000000000..1ff7736a2 --- /dev/null +++ b/bbdev/api/steps/config/scripts/int8add_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: int8add_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "int8add_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x14int8add_params.proto\x12\x17\x62uckyball.balls.int8add"\x08\n\x06ParamsB\x1b\n\x17\x62uckyball.balls.int8addP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "int8add_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\027buckyball.balls.int8addP\001" + _globals["_PARAMS"]._serialized_start = 49 + _globals["_PARAMS"]._serialized_end = 57 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/int8mul_params_pb2.py b/bbdev/api/steps/config/scripts/int8mul_params_pb2.py new file mode 100644 index 000000000..7271656e3 --- /dev/null +++ b/bbdev/api/steps/config/scripts/int8mul_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: int8mul_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "int8mul_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x14int8mul_params.proto\x12\x17\x62uckyball.balls.int8mul"\x08\n\x06ParamsB\x1b\n\x17\x62uckyball.balls.int8mulP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "int8mul_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\027buckyball.balls.int8mulP\001" + _globals["_PARAMS"]._serialized_start = 49 + _globals["_PARAMS"]._serialized_end = 57 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/lut_params_pb2.py b/bbdev/api/steps/config/scripts/lut_params_pb2.py new file mode 100644 index 000000000..9d42d2466 --- /dev/null +++ b/bbdev/api/steps/config/scripts/lut_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: lut_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "lut_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x10lut_params.proto\x12\x13\x62uckyball.balls.lut"\x08\n\x06ParamsB\x17\n\x13\x62uckyball.balls.lutP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "lut_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\023buckyball.balls.lutP\001" + _globals["_PARAMS"]._serialized_start = 41 + _globals["_PARAMS"]._serialized_end = 49 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/matadd_params_pb2.py b/bbdev/api/steps/config/scripts/matadd_params_pb2.py new file mode 100644 index 000000000..fe65713c8 --- /dev/null +++ b/bbdev/api/steps/config/scripts/matadd_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: matadd_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "matadd_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x13matadd_params.proto\x12\x16\x62uckyball.balls.matadd"\x08\n\x06ParamsB\x1a\n\x16\x62uckyball.balls.mataddP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "matadd_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\026buckyball.balls.mataddP\001" + _globals["_PARAMS"]._serialized_start = 47 + _globals["_PARAMS"]._serialized_end = 55 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/maxpool_params_pb2.py b/bbdev/api/steps/config/scripts/maxpool_params_pb2.py new file mode 100644 index 000000000..68512d9ff --- /dev/null +++ b/bbdev/api/steps/config/scripts/maxpool_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: maxpool_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "maxpool_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x14maxpool_params.proto\x12\x17\x62uckyball.balls.maxpool"\x08\n\x06ParamsB\x1b\n\x17\x62uckyball.balls.maxpoolP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "maxpool_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\027buckyball.balls.maxpoolP\001" + _globals["_PARAMS"]._serialized_start = 49 + _globals["_PARAMS"]._serialized_end = 57 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/mxfp2int_params_pb2.py b/bbdev/api/steps/config/scripts/mxfp2int_params_pb2.py new file mode 100644 index 000000000..447b0de63 --- /dev/null +++ b/bbdev/api/steps/config/scripts/mxfp2int_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: mxfp2int_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "mxfp2int_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x15mxfp2int_params.proto\x12\x18\x62uckyball.balls.mxfp2int"0\n\x06Params\x12\x17\n\nmxfpFormat\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\r\n\x0b_mxfpFormatB\x1c\n\x18\x62uckyball.balls.mxfp2intP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "mxfp2int_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\030buckyball.balls.mxfp2intP\001" + _globals["_PARAMS"]._serialized_start = 51 + _globals["_PARAMS"]._serialized_end = 99 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/mxmm_params_pb2.py b/bbdev/api/steps/config/scripts/mxmm_params_pb2.py new file mode 100644 index 000000000..1883aa8c3 --- /dev/null +++ b/bbdev/api/steps/config/scripts/mxmm_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: mxmm_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "mxmm_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x11mxmm_params.proto\x12\x14\x62uckyball.balls.mxmm"P\n\x06Params\x12\x15\n\x08tileRows\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x15\n\x08tileCols\x18\x02 \x01(\x05H\x01\x88\x01\x01\x42\x0b\n\t_tileRowsB\x0b\n\t_tileColsB\x18\n\x14\x62uckyball.balls.mxmmP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "mxmm_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\024buckyball.balls.mxmmP\001" + _globals["_PARAMS"]._serialized_start = 43 + _globals["_PARAMS"]._serialized_end = 123 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/mxquant_params_pb2.py b/bbdev/api/steps/config/scripts/mxquant_params_pb2.py new file mode 100644 index 000000000..38024b677 --- /dev/null +++ b/bbdev/api/steps/config/scripts/mxquant_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: mxquant_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "mxquant_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x14mxquant_params.proto\x12\x17\x62uckyball.balls.mxquant"\x08\n\x06ParamsB\x1b\n\x17\x62uckyball.balls.mxquantP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "mxquant_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\027buckyball.balls.mxquantP\001" + _globals["_PARAMS"]._serialized_start = 49 + _globals["_PARAMS"]._serialized_end = 57 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/proto/chip.proto b/bbdev/api/steps/config/scripts/proto/chip.proto deleted file mode 100644 index 3042253aa..000000000 --- a/bbdev/api/steps/config/scripts/proto/chip.proto +++ /dev/null @@ -1,346 +0,0 @@ -// chip.proto is the Chip schema. -// Java: mill protoJava; Python: chip_pb2.py (generated by config install step3). -syntax = "proto3"; - -package buckyball.config; - -option java_package = "buckyball.config"; -option java_multiple_files = true; - -message MillConfig { - string verilator_config = 1; - string p2e_config = 2; -} - -message BankConfig { - uint32 num = 1; - uint32 width = 2; - uint32 entries = 3; - uint32 mask_len = 4; - uint32 channel = 5; -} - -message MmioConfig { - bool enable = 1; - uint32 bank_num = 2; - uint32 bank_entries = 3; - uint32 bank_width = 4; - uint32 read_width = 5; -} - -message DmaConfig { - uint32 n_xacts = 1; - uint32 burst_max_bytes = 2; - uint32 bus_width = 3; - uint32 max_in_flight_mem_reqs = 4; -} - -message TlbConfig { - uint32 size = 1; -} - -message TmaConfig { - uint32 read_channel = 1; - uint32 write_channel = 2; -} - -message MemAddrConfig { - uint32 addr_len = 1; -} - -message MemDomainConfig { - BankConfig bank = 1; - DmaConfig dma = 2; - TlbConfig tlb = 3; - TmaConfig tma = 4; - MmioConfig mmio = 5; - MemAddrConfig mem = 6; - string source_path = 7; -} - -message BallIdMapping { - uint32 ball_id = 1; - string ball_name = 2; - string ball_class = 3; - string ball_dir = 4; - string config_path = 5; - uint32 in_bw = 6; - uint32 out_bw = 7; - uint32 mmio_read_bw = 8; - uint32 mmio_write_bw = 9; - map ball_params = 10; -} - -message BallIsaEntry { - string mnemonic = 1; - uint32 funct7 = 2; - uint32 bid = 3; -} - -message BallDomain { - uint32 ball_num = 1; - repeated BallIdMapping mappings = 2; - repeated BallIsaEntry isa = 3; - string source_path = 4; -} - -message RocketMulDivConfig { - uint32 div_unroll = 5; - uint32 div_early_out_granularity = 6; - - bool enable = 1; - uint32 mul_unroll = 2; - bool mul_early_out = 3; - bool div_early_out = 4; -} - -message RocketFpuConfig { - bool div_sqrt = 4; - uint32 sfma_latency = 5; - uint32 dfma_latency = 6; - uint32 fpmu_latency = 7; - uint32 ifpu_latency = 8; - - bool enable = 1; - uint32 min_f_len = 2; - uint32 f_len = 3; -} - -message RocketDCacheConfig { - bool clock_gate = 4; - - uint32 n_sets = 1; - uint32 n_ways = 2; - uint32 n_mshrs = 3; -} - -message RocketICacheConfig { - bool prefetch = 3; - - uint32 n_sets = 1; - uint32 n_ways = 2; -} - -message RocketBtbConfig { - uint32 n_match_bits = 4; - uint32 n_pages = 5; - bool updates_out_of_order = 6; - bool bht_enable = 7; - uint32 bht_entries = 8; - uint32 bht_counter_length = 9; - uint32 bht_history_length = 10; - uint32 bht_history_bits = 11; - - bool enable = 1; - uint32 n_entries = 2; - uint32 n_ras = 3; -} - -message RocketCpuConfig { - bool mtvec_init_enable = 41; - bool use_user = 13; - bool use_supervisor = 14; - bool use_hypervisor = 15; - bool use_debug = 16; - bool use_atomics = 17; - bool use_atomics_only_for_i_o = 18; - bool use_compressed = 19; - bool use_r_v_e = 20; - bool use_conditional_zero = 21; - uint32 n_local_interrupts = 22; - bool use_n_m_i = 23; - uint32 n_breakpoints = 24; - bool use_b_p_watch = 25; - uint32 mcontext_width = 26; - uint32 scontext_width = 27; - uint32 n_perf_counters = 28; - bool have_basic_counters = 29; - bool misa_writable = 30; - uint64 mtvec_init = 31; - bool mtvec_writable = 32; - bool fast_load_word = 33; - bool fast_load_byte = 34; - bool branch_prediction_mode_c_s_r = 35; - bool clock_gate = 36; - uint32 mvendorid = 37; - uint32 mimpid = 38; - bool have_cease = 39; - bool have_sim_timeout = 40; - - bool use_vm = 3; - bool use_zba = 4; - bool use_zbb = 5; - bool use_zbs = 6; - bool have_c_flush = 7; - RocketMulDivConfig mul_div = 8; - RocketFpuConfig fpu = 9; - RocketDCacheConfig dcache = 10; - RocketICacheConfig icache = 11; - RocketBtbConfig btb = 12; -} - -message BoomDCacheConfig { - uint32 n_sets = 1; - uint32 n_ways = 2; - uint32 n_mshrs = 3; -} - -message BoomICacheConfig { - uint32 n_sets = 1; - uint32 n_ways = 2; -} - -message BoomCpuConfig { - uint32 fetch_width = 1; - uint32 decode_width = 2; - uint32 num_rob_entries = 3; - BoomDCacheConfig dcache = 4; - BoomICacheConfig icache = 5; -} - -message FrontendConfig { - uint32 rob_entries = 1; - bool rs_out_of_order_response = 2; - uint32 bank_id_len = 3; - uint32 vbank_id_upper_bound = 4; - uint32 shared_bank_id_base = 5; - uint32 iter_len = 6; - bool sub_rob_enable = 7; - uint32 sub_rob_depth = 8; - string source_path = 9; -} - -message RvvConfig { - uint32 lane_number = 1; - uint32 v_len = 2; - uint32 e_len = 3; - uint32 i_buf_words = 4; - uint32 memory_ports = 5; - string source_path = 6; - optional bool enable = 7; -} - -message TileParamConfig { - uint32 core_data_bytes = 1; - uint32 x_len = 2; - uint32 vaddr_bits = 3; - uint32 paddr_bits = 4; - uint32 pg_idx_bits = 5; - uint32 pg_levels = 6; - uint32 n_pmps = 7; -} - -message SpmConfig { - uint64 base = 1; - uint32 bytes = 2; - uint32 data_bits = 3; -} - -message AntConfig { - uint32 code_bytes = 1; - SpmConfig tls = 2; - uint32 task_bits = 3; -} - -message CpuConfig { - string kind = 1; - oneof cpu { - RocketCpuConfig rocket = 2; - BoomCpuConfig boom = 3; - AntConfig ant = 5; - } - string source_path = 4; -} - -message CoreInstance { - uint32 index = 1; - string role = 2; - string pkg = 3; - string config_path = 4; - BallDomain balldomain = 5; - MemDomainConfig mem = 6; - string balldomain_base_dir = 7; - CpuConfig cpu = 8; - FrontendConfig frontend = 9; - optional uint32 hart_id = 10; - RvvConfig rvv = 11; - optional uint32 ant_context_id = 12; - string factory_class = 13; -} - -message BemuBall { - string ball_class = 1; - string ball_dir = 2; - string emu_lib = 3; -} - -message BemuConfig { - string chip_main = 1; - uint32 tile_index = 2; - repeated BemuBall balls = 3; -} - -message SharedMemConfig { - bool enable = 1; - uint32 entries = 2; - uint32 input_channels = 3; - uint32 default_group_count = 4; - uint32 virtual_bank_count = 5; - uint32 mem_cycles = 6; - uint32 bank_entries = 7; - uint32 bank_width = 8; -} - -// Tile 0 is the main tile: every core is Linux visible. Compute tiles follow and are -// homogeneous: core 0 controls the tile and the other cores are hidden task workers. -enum TileKind { - TILE_KIND_MAIN = 0; - TILE_KIND_TILE = 1; -} - -message TilePlacement { - string path = 1; - uint32 virtual_bank_count = 2; - repeated uint32 core_indices = 3; - uint32 mem_ball_channel_num = 4; - reserved 5; - SharedMemConfig shared_mem = 6; - TileParamConfig param = 7; - // Optional local task controller, expressed as a global core index. - optional uint32 controller_core_index = 8; - TileKind kind = 9; - SpmConfig tss = 10; - string factory_class = 11; -} - -message WorkloadConfig { - reserved 1; - map cmake_param = 2; -} - -message CompilerProfile { - string name = 1; - string pkg = 2; - string compiler = 3; - uint32 bank_num = 4; - uint32 bank_width = 5; - uint32 bank_entries = 6; - repeated string ball_ctest_dirs = 7; -} - -message Chip { - string name = 1; - MillConfig mill = 2; - string topology_path = 3; - repeated string includes = 4; - reserved 5; - uint32 n_tiles = 6; - repeated TilePlacement tiles = 7; - repeated CoreInstance cores = 8; - reserved 9; - BemuConfig bemu = 10; - WorkloadConfig workload = 11; - string chip_path = 12; - repeated CompilerProfile profiles = 13; - reserved 14; -} diff --git a/bbdev/api/steps/config/scripts/proto/design.proto b/bbdev/api/steps/config/scripts/proto/design.proto new file mode 100644 index 000000000..019983822 --- /dev/null +++ b/bbdev/api/steps/config/scripts/proto/design.proto @@ -0,0 +1,335 @@ +// design.proto is the Design schema. +// Java: mill protoJava; Python: design_pb2.py (generated by config install step3). +syntax = "proto3"; + +package buckyball.config; + +import "google/protobuf/any.proto"; +import "hardware.proto"; + +option java_package = "buckyball.config"; +option java_multiple_files = true; + +message MillConfig { + reserved 1, 2; + map targets = 3; + map tops = 4; + map resources = 5; +} + +message BankConfig { + uint32 num = 1; + uint32 width = 2; + uint32 entries = 3; + uint32 mask_len = 4; + uint32 request_ports = 5; + optional bool layout_enable = 6; +} + +message MmioConfig { + bool enable = 1; + uint32 bank_num = 2; + uint32 bank_entries = 3; + uint32 bank_width = 4; + uint32 read_width = 5; +} + +message DmaConfig { + uint32 bus_width = 3; +} + +message MemAddrConfig { + uint32 addr_len = 1; +} + +message MemDomainConfig { + BankConfig bank = 1; + DmaConfig dma = 2; + MmioConfig mmio = 5; + MemAddrConfig mem = 6; + string source_path = 7; +} + +message BallInstance { + uint32 ball_id = 1; + string ball_name = 2; + string ball_dir = 3; + uint32 in_bw = 4; + uint32 out_bw = 5; + uint32 mmio_read_bw = 6; + uint32 mmio_write_bw = 7; + repeated BallInstruction instructions = 8; + google.protobuf.Any parameters = 9; + string rtl_class = 10; + reserved 11; +} + +message BallInstruction { + string mnemonic = 1; + uint32 funct7 = 2; +} + +message BallDomain { + repeated BallInstance balls = 1; +} + +message FrontendConfig { + uint32 rob_entries = 1; + bool rs_out_of_order_response = 2; + uint32 bank_id_len = 3; + uint32 vbank_id_upper_bound = 4; + uint32 shared_bank_id_base = 5; + uint32 iter_len = 6; + bool sub_rob_enable = 7; + uint32 sub_rob_depth = 8; + string source_path = 9; + optional bool bank_alias_enable = 10; +} + +message GpDomainConfig { + uint32 lane_number = 1; + uint32 v_len = 2; + uint32 e_len = 3; + uint32 i_buf_words = 4; + uint32 memory_ports = 5; + string source_path = 6; + optional bool enable = 7; +} + +message CoreParamConfig { + uint32 thread_data_bytes = 1; + uint32 x_len = 2; + uint32 vaddr_bits = 3; + uint32 paddr_bits = 4; + uint32 pg_idx_bits = 5; + uint32 pg_levels = 6; + uint32 n_pmps = 7; +} + +message SpmConfig { + uint64 base = 1; + uint32 bytes = 2; + uint32 data_bits = 3; +} + +message AntConfig { + uint32 code_bytes = 1; + SpmConfig tls = 2; + uint32 task_bits = 3; +} + +message CpuConfig { + string kind = 1; + oneof cpu { + AntConfig ant = 5; + } + string source_path = 4; +} + +message ThreadInstance { + uint32 index = 1; + string role = 2; + string pkg = 3; + string config_path = 4; + BallDomain balldomain = 5; + MemDomainConfig mem = 6; + string balldomain_base_dir = 7; + CpuConfig cpu = 8; + FrontendConfig frontend = 9; + reserved 10; + GpDomainConfig gpdomain = 11; + reserved 12; + TransportHardware transport = 13; + optional uint32 local_id = 14; + optional uint64 signature = 15; + string profile = 16; + string source_path = 17; + string rtl_class = 18; +} + +message BemuBall { + string ball_class = 1; + string ball_dir = 2; + string emu_lib = 3; +} + +message BemuConfig { + reserved 1, 2; + repeated BemuBall balls = 3; +} + +message SharedMemConfig { + bool enable = 1; + uint32 entries = 2; + uint32 input_channels = 3; + uint32 default_group_count = 4; + uint32 virtual_bank_count = 5; + uint32 mem_cycles = 6; + uint32 bank_entries = 7; + uint32 bank_width = 8; +} + +message LocalAccelerator { + BallDomain balldomain = 1; + MemDomainConfig mem = 2; + FrontendConfig frontend = 3; + GpDomainConfig gpdomain = 4; + TransportHardware transport = 5; +} + +message CorePlacement { + string path = 1; + uint32 virtual_bank_count = 2; + repeated uint32 thread_indices = 3; + reserved 5; + SharedMemConfig shared_mem = 6; + CoreParamConfig param = 7; + reserved 8; + reserved 9; + SpmConfig tss = 10; + reserved 11; + reserved 12; + google.protobuf.Any implementation = 13; + LocalAccelerator accelerator = 14; + string profile = 16; + reserved 17; + uint32 hart_id = 18; + uint64 reset_vector = 19; + uint64 mtime_address = 20; + MemoryConfig memory = 21; + FetchConfig fetch = 22; + FpuConfig fpu = 23; + InterconnectConfig interconnect = 24; + C2CConfig c2c = 25; + TBusConfig tbus = 26; + ControllerConfig controller = 27; + RnfHardware dma_cache = 28; + string component = 29; + // Identity of the core hardware and its threads; runtime jobs select cores by it. + uint64 signature = 30; + +} + +message WorkloadConfig { + reserved 1; + map cmake_param = 2; + optional uint32 control_hart = 3; + optional uint32 report_hart = 4; +} + +message CompilerProfile { + string name = 1; + string pkg = 2; + string compiler = 3; + uint32 bank_num = 4; + uint32 bank_width = 5; + uint32 bank_entries = 6; + repeated string ball_ctest_dirs = 7; + ThreadInstance config = 8; + optional uint32 test_core = 9; +} + +message Design { + string name = 1; + MillConfig mill = 2; + repeated string includes = 4; + reserved 5; + uint32 n_cores = 6; + reserved 7; + repeated ThreadInstance threads = 8; + reserved 9; + BemuConfig bemu = 10; + WorkloadConfig workload = 11; + string design_path = 12; + repeated CompilerProfile profiles = 13; + reserved 14; + DeviceConfig device = 15; + SimHardware sims = 16; + VerificationConfig verification = 17; + map simulations = 18; +} + +message SharedBankFabricConfig { + optional uint32 owner_hart = 1; + repeated uint32 participant_harts = 2; + repeated uint32 bank_nodes = 3; +} + +message DeviceConfig { + repeated CorePlacement cores = 1; + DdrHardware ddr = 2; + BootRomHardware bootrom = 3; + ClintHardware clint = 4; + PlicHardware plic = 5; + ScuHardware scu = 6; + SharedBankFabricConfig shared_bank_fabric = 7; +} + +message MillTarget { + string directory = 1; + string module = 2; + string config = 3; +} +message VerificationConfig { + repeated string balls = 1; + repeated string ips = 2; + reserved 3; + repeated VerificationProfile profiles = 4; +} + +message VerificationBuild { + string kind = 1; + string directory = 2; + string module = 3; + repeated string arguments = 4; +} + +message VerificationReference { + string manifest = 1; + string library = 2; +} + +message VerificationTest { + string name = 1; + string top = 2; + string test = 3; + string filelist = 4; + uint64 timeout_ns = 5; + uint32 wall_timeout_s = 6; + string kind = 7; + repeated string rtl_scopes = 8; + repeated string metrics = 9; + uint32 sink_ready = 10; + uint32 sink_stall = 11; + repeated string designs = 12; + repeated string plusargs = 13; + string assertion = 14; + repeated string functional_groups = 15; +} + +message VerificationProfile { + string ip = 1; + string profile = 2; + string config_path = 3; + string root = 4; + string parameters_json = 5; + map defines = 6; + VerificationBuild build = 7; + VerificationReference model = 8; + repeated VerificationTest tests = 9; + string resources = 10; + string scope = 11; +} + +message SimulationTarget { + string directory = 1; + MillTarget arch = 2; + string entry_class = 3; + string top = 4; + string emu = 5; + string emu_package = 6; + string stack = 7; + string stack_name = 8; + string workloads = 9; + map resources = 10; +} diff --git a/bbdev/api/steps/config/scripts/proto/hardware.proto b/bbdev/api/steps/config/scripts/proto/hardware.proto new file mode 100644 index 000000000..967aba249 --- /dev/null +++ b/bbdev/api/steps/config/scripts/proto/hardware.proto @@ -0,0 +1,158 @@ +syntax = "proto2"; +package buckyball.config; +option java_package = "buckyball.config"; +option java_multiple_files = true; + +message ChiHardware { + required uint32 node_id_bits = 1; + required uint32 address_bits = 2; + required uint32 data_bits = 3; +} +message AxiHardware { + required uint32 address_bits = 1; + required uint32 data_bits = 2; + required uint32 id_bits = 3; +} +message TrackingHardware { + required uint32 entries = 1; + required uint32 address_bits = 2; + required uint32 id_bits = 3; + required uint32 line_bytes = 4; + required uint32 max_ranges = 5; +} +message RegionHardware { + required uint64 base = 1; + required uint64 bytes = 2; + required bool cacheable = 3; + required bool executable = 4; + required bool readable = 5; + required bool writable = 6; + required bool atomic = 7; + required bool normal = 8; +} +message CacheHardware { + required uint32 address_bits = 1; + required uint32 line_bytes = 2; + required uint32 sets = 3; + required uint32 ways = 4; + required uint32 id_bits = 5; + required uint32 metadata_bits = 6; + required uint32 response_depth = 7; +} +message RnfHardware { + required uint32 node_id = 1; + required uint32 cache_lines = 2; + required uint32 home_id = 3; + required uint32 home_count = 4; + required uint32 banks = 5; + required bool line_result = 6; + required bool probe = 7; + reserved 8; +} +message HomeHardware { + required CacheHardware cache = 1; + required uint32 agents = 2; + required uint32 mshr_entries = 3; + required uint32 home_id = 4; + required ChiHardware requesters = 5; +} +message PreparationHardware { + required ChiHardware bus = 1; + required uint32 contexts = 2; + required uint32 max_ranges = 3; + required uint32 id_bits = 4; + required uint32 beat_bytes = 5; +} +message TransportHardware { + required TrackingHardware tracking = 1; + required PreparationHardware prepared = 2; + required AxiHardware axi = 3; + repeated RegionHardware regions = 4; +} + + +message DdrHardware { + required ChiHardware line = 1; + required uint32 clients = 2; + required uint32 slots_per_client = 3; + required uint32 data_bits = 4; + required uint32 id_bits = 5; + required uint32 id_base = 6; + // DDR address map; the top `model` bytes hold the model image, the rest is + // what the CPUs run in. + required uint64 base = 7; + required uint64 bytes = 8; + required uint64 model = 9; +} +message BootRomHardware { + required uint64 base = 1; + required uint64 bytes = 2; + repeated sint32 image = 3; +} +message ClintHardware { + repeated uint32 hart_ids = 1; + required uint64 base = 2; + required uint64 bytes = 3; + required uint32 tick_cycles = 4; +} +message PlicHardware { + repeated uint32 hart_ids = 1; + required uint64 base = 2; + required uint64 bytes = 3; + required uint32 sources = 4; + required uint32 priority_bits = 5; +} +message ScuHardware { + required uint64 base_address = 1; + required uint64 stride_bytes = 2; + required uint64 total_size_bytes = 3; + required uint32 max_harts = 4; +} +message SimMemoryHardware { + reserved 1, 2; + required uint32 line_bytes = 3; + required uint64 clock_hz = 4; +} +message SimHardware { + required bool diff_test = 1; + required SimMemoryHardware memory = 2; + reserved 3; +} + +message MemoryAccessConfig { + reserved 1; + required uint32 access_tag_bits = 2; + required uint32 translation_entries = 3; + required uint32 result_line_bits = 4; + reserved 5; + repeated RegionHardware regions = 6; +} +message DCacheConfig { + required RnfHardware cache = 1; + required TrackingHardware tracking = 2; +} +message UncachedConfig { + required uint32 ports = 1; + required AxiHardware axi = 2; +} +message MemoryConfig { + reserved 1; + required UncachedConfig uncached = 6; + required MemoryAccessConfig access = 2; + required RnfHardware icache = 3; + required DCacheConfig dcache = 4; + required HomeHardware home = 5; +} +message FetchConfig { required uint32 instruction_bytes = 1; } +message FpuConfig { required uint32 f_len = 1; } +message InterconnectConfig { + required AxiHardware axi = 1; + required uint32 slots = 2; +} +message C2CConfig { + required bool enabled = 1; + required uint32 peer_cores = 2; + required AxiHardware axi = 3; +} +message TBusConfig { repeated uint32 bank_nodes = 1; } +message ControllerConfig { required bool moves = 1; } diff --git a/bbdev/api/steps/config/scripts/redmule_params_pb2.py b/bbdev/api/steps/config/scripts/redmule_params_pb2.py new file mode 100644 index 000000000..9aa96f62f --- /dev/null +++ b/bbdev/api/steps/config/scripts/redmule_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: redmule_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "redmule_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x14redmule_params.proto\x12\x17\x62uckyball.balls.redmule"&\n\x06Params\x12\x12\n\x05lanes\x18\x01 \x01(\x05H\x00\x88\x01\x01\x42\x08\n\x06_lanesB\x1b\n\x17\x62uckyball.balls.redmuleP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "redmule_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\027buckyball.balls.redmuleP\001" + _globals["_PARAMS"]._serialized_start = 49 + _globals["_PARAMS"]._serialized_end = 87 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/relu_params_pb2.py b/bbdev/api/steps/config/scripts/relu_params_pb2.py new file mode 100644 index 000000000..cb3e7123f --- /dev/null +++ b/bbdev/api/steps/config/scripts/relu_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: relu_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "relu_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x11relu_params.proto\x12\x14\x62uckyball.balls.relu"\x08\n\x06ParamsB\x18\n\x14\x62uckyball.balls.reluP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "relu_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\024buckyball.balls.reluP\001" + _globals["_PARAMS"]._serialized_start = 43 + _globals["_PARAMS"]._serialized_end = 51 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/rocket_params_pb2.py b/bbdev/api/steps/config/scripts/rocket_params_pb2.py new file mode 100644 index 000000000..8c6cbead1 --- /dev/null +++ b/bbdev/api/steps/config/scripts/rocket_params_pb2.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: rocket_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "rocket_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x13rocket_params.proto\x12\x1b\x62uckyball.components.rocket"\xfa\x01\n\x06Params\x12\x1d\n\x15physical_address_bits\x18\x01 \x02(\r\x12\x14\n\x0chart_id_bits\x18\x02 \x02(\r\x12\x13\n\x0bpage_levels\x18\x03 \x02(\r\x12\x13\n\x0bpmp_entries\x18\x04 \x02(\r\x12\x16\n\x0evirtual_memory\x18\x05 \x02(\x08\x12\x12\n\nsupervisor\x18\x06 \x02(\x08\x12\x0c\n\x04user\x18\x07 \x02(\x08\x12\x16\n\x0e\x66loating_point\x18\x08 \x02(\x08\x12\x0f\n\x07\x61tomics\x18\t \x02(\x08\x12\x17\n\x0fmultiply_divide\x18\n \x02(\x08\x12\x15\n\rexternal_home\x18\x0b \x02(\x08\x42\x1f\n\x1b\x62uckyball.components.rocketP\x01' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "rocket_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = ( + b"\n\033buckyball.components.rocketP\001" + ) + _globals["_PARAMS"]._serialized_start = 53 + _globals["_PARAMS"]._serialized_end = 303 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/smatmul_params_pb2.py b/bbdev/api/steps/config/scripts/smatmul_params_pb2.py new file mode 100644 index 000000000..fa13c0b8f --- /dev/null +++ b/bbdev/api/steps/config/scripts/smatmul_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: smatmul_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "smatmul_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x14smatmul_params.proto\x12\x17\x62uckyball.balls.smatmul"P\n\x06Params\x12\x15\n\x08tileRows\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x15\n\x08tileCols\x18\x02 \x01(\x05H\x01\x88\x01\x01\x42\x0b\n\t_tileRowsB\x0b\n\t_tileColsB\x1b\n\x17\x62uckyball.balls.smatmulP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "smatmul_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\027buckyball.balls.smatmulP\001" + _globals["_PARAMS"]._serialized_start = 49 + _globals["_PARAMS"]._serialized_end = 129 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/toint8_params_pb2.py b/bbdev/api/steps/config/scripts/toint8_params_pb2.py new file mode 100644 index 000000000..ce767b8ea --- /dev/null +++ b/bbdev/api/steps/config/scripts/toint8_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: toint8_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "toint8_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x13toint8_params.proto\x12\x16\x62uckyball.balls.toint8"\x08\n\x06ParamsB\x1a\n\x16\x62uckyball.balls.toint8P\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "toint8_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\026buckyball.balls.toint8P\001" + _globals["_PARAMS"]._serialized_start = 47 + _globals["_PARAMS"]._serialized_end = 55 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/trace_params_pb2.py b/bbdev/api/steps/config/scripts/trace_params_pb2.py new file mode 100644 index 000000000..6c325cae2 --- /dev/null +++ b/bbdev/api/steps/config/scripts/trace_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: trace_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "trace_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x12trace_params.proto\x12\x15\x62uckyball.balls.trace"\x08\n\x06ParamsB\x19\n\x15\x62uckyball.balls.traceP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "trace_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\025buckyball.balls.traceP\001" + _globals["_PARAMS"]._serialized_start = 45 + _globals["_PARAMS"]._serialized_end = 53 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/transpose_params_pb2.py b/bbdev/api/steps/config/scripts/transpose_params_pb2.py new file mode 100644 index 000000000..7171096d3 --- /dev/null +++ b/bbdev/api/steps/config/scripts/transpose_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: transpose_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "transpose_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x16transpose_params.proto\x12\x19\x62uckyball.balls.transpose"T\n\x06Params\x12\x15\n\x08InputNum\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x17\n\ninputWidth\x18\x02 \x01(\x05H\x01\x88\x01\x01\x42\x0b\n\t_InputNumB\r\n\x0b_inputWidthB\x1d\n\x19\x62uckyball.balls.transposeP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "transpose_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\031buckyball.balls.transposeP\001" + _globals["_PARAMS"]._serialized_start = 53 + _globals["_PARAMS"]._serialized_end = 137 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/vector_params_pb2.py b/bbdev/api/steps/config/scripts/vector_params_pb2.py new file mode 100644 index 000000000..e1e8bf1ea --- /dev/null +++ b/bbdev/api/steps/config/scripts/vector_params_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: vector_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "vector_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x13vector_params.proto\x12\x16\x62uckyball.balls.vector"\xf6\x01\n\x06Params\x12\x15\n\x08InputNum\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x17\n\ninputWidth\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x11\n\x04lane\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x18\n\x0boutputWidth\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12\x1a\n\rnumMulThreads\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x1a\n\rnumCasThreads\x18\x06 \x01(\x05H\x05\x88\x01\x01\x42\x0b\n\t_InputNumB\r\n\x0b_inputWidthB\x07\n\x05_laneB\x0e\n\x0c_outputWidthB\x10\n\x0e_numMulThreadsB\x10\n\x0e_numCasThreadsB\x1a\n\x16\x62uckyball.balls.vectorP\x01\x62\x06proto3' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "vector_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = b"\n\026buckyball.balls.vectorP\001" + _globals["_PARAMS"]._serialized_start = 48 + _globals["_PARAMS"]._serialized_end = 294 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/verification.py b/bbdev/api/steps/config/scripts/verification.py new file mode 100644 index 000000000..915676256 --- /dev/null +++ b/bbdev/api/steps/config/scripts/verification.py @@ -0,0 +1,99 @@ +import importlib.util +import json +import tomllib +from pathlib import Path +from google.protobuf.json_format import ParseDict +import design_pb2 + + +def ip_registrations(repo, entries): + repo = Path(repo) + catalog = tomllib.loads((repo / "arch/uvm.toml").read_text()) + profiles = [] + identities = set() + for entry in entries: + if set(entry) != {"ip", "config"}: + raise ValueError("UVM registration must contain exactly ip and config") + ip = entry["ip"] + source = Path(entry["config"]["_file"]).relative_to(repo) + identity = (ip, source.stem) + if identity in identities: + raise ValueError(f"Duplicate UVM registration: {ip}/{source.stem}") + identities.add(identity) + root = repo / catalog["paths"][ip] + suite = tomllib.loads((root / "src/main/resources/suite.toml").read_text()) + tests = suite["tests"] + names = [test["name"] for test in tests] + if not names or len(names) != len(set(names)): + raise ValueError(f"Suite must declare unique test names: {root}") + for test in tests: + if test["kind"] not in {"functional", "assertion"}: + raise ValueError(f"Invalid test kind: {test['kind']}") + if test["kind"] == "assertion" and not test["assertion"]: + raise ValueError( + f"Assertion test requires an exact assertion message: {test['name']}" + ) + if test["timeout_ns"] <= 0 or test["wall_timeout_s"] <= 0: + raise ValueError( + f"Positive simulation and wall timeouts are required: {test['name']}" + ) + if not test["rtl_scopes"] or not set(test["metrics"]) <= { + "line", + "cond", + "tgl", + }: + raise ValueError( + f"Invalid DUT coverage scope or metrics: {test['name']}" + ) + if not (root / "src/main/resources" / test["filelist"]).is_file(): + raise FileNotFoundError(test["filelist"]) + config = entry["config"] + parameters = { + key: value for key, value in config["parameters"].items() if key != "_file" + } + defines = { + key: parameters[value] + for key, value in config["defines"].items() + if key != "_file" + } + profile = design_pb2.VerificationProfile( + ip=ip, + profile=source.stem, + config_path=str(source), + root=str(root.relative_to(repo)), + parameters_json=json.dumps(parameters), + defines={key: str(value) for key, value in defines.items()}, + resources="src/main/resources", + scope="ip", + ) + build = catalog["build"][ip] + profile.build.kind = "mill" + profile.build.directory = build["directory"] + profile.build.module = build["module"] + profile.build.arguments.extend(build["arguments"]) + ParseDict(suite["model"], profile.model) + for test in tests: + ParseDict(test, profile.tests.add()) + profiles.append(profile) + return profiles + + +def default_ip_config(repo, ip): + repo = Path(repo) + catalog = tomllib.loads((repo / "arch/uvm.toml").read_text()) + root = repo / catalog["paths"][ip] + spec = importlib.util.spec_from_file_location( + "verification_toml", Path(__file__).with_name("1_toml2json.py") + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + expanded = module.toml2json(root / "configs/uvm.toml") + profiles = ip_registrations(repo, [{"ip": ip, "config": expanded}]) + + config = design_pb2.Design(name=ip) + config.verification.ips.append(ip) + config.verification.profiles.extend(profiles) + output = root / "build/verification/default/design.pb" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(config.SerializeToString(deterministic=True)) + return output diff --git a/bbdev/api/steps/config/scripts/xiangshan_params_pb2.py b/bbdev/api/steps/config/scripts/xiangshan_params_pb2.py new file mode 100644 index 000000000..f83266f3f --- /dev/null +++ b/bbdev/api/steps/config/scripts/xiangshan_params_pb2.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: xiangshan_params.proto +# Protobuf Python Version: 7.35.1 +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "xiangshan_params.proto" +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +import hardware_pb2 as hardware__pb2 + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x16xiangshan_params.proto\x12\x1e\x62uckyball.components.xiangshan\x1a\x0ehardware.proto"\\\n\x06Params\x12\x0f\n\x07node_id\x18\x01 \x02(\r\x12\x0f\n\x07home_id\x18\x02 \x02(\r\x12\x30\n\tdma_cache\x18\x03 \x02(\x0b\x32\x1d.buckyball.config.RnfHardwareB"\n\x1e\x62uckyball.components.xiangshanP\x01' +) + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "xiangshan_params_pb2", _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals["DESCRIPTOR"]._loaded_options = None + _globals["DESCRIPTOR"]._serialized_options = ( + b"\n\036buckyball.components.xiangshanP\001" + ) + _globals["_PARAMS"]._serialized_start = 74 + _globals["_PARAMS"]._serialized_end = 166 +# @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/dc/01_area_api.step.py b/bbdev/api/steps/dc/01_area_api.step.py index 1a03447b6..6fa48d211 100644 --- a/bbdev/api/steps/dc/01_area_api.step.py +++ b/bbdev/api/steps/dc/01_area_api.step.py @@ -1,6 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design config = { "name": "dc-area-api", @@ -13,15 +14,20 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - chip = require_chip(body) + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + design = require_design(body) await ctx.enqueue( { "topic": "dc.verilog", "data": { - "chip": chip, + "design": design, "top": body.get("top") or "DigitalTop", "from_area_workflow": True, "_trace_id": ctx.trace_id, + "target": selection, }, } ) diff --git a/bbdev/api/steps/dc/01_area_event.step.py b/bbdev/api/steps/dc/01_area_event.step.py index cbd165031..3d7c7dcaf 100644 --- a/bbdev/api/steps/dc/01_area_event.step.py +++ b/bbdev/api/steps/dc/01_area_event.step.py @@ -52,7 +52,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, 1, continue_run=False, - extra_fields={"task": "dc", "error": "missing fixed DC analysis output directory"}, + extra_fields={ + "task": "dc", + "error": "missing fixed DC analysis output directory", + }, trace_id=origin_tid, ) return @@ -70,7 +73,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return try: - contract = get_tapeout_contract(get_buckyball_path(), input_data.get("chip"), top_module) + contract = get_tapeout_contract( + get_buckyball_path(), + input_data.get("design"), + top_module, + selection=input_data["target"], + ) except (OSError, ValueError) as exc: await check_result( ctx, @@ -96,8 +104,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: raise ValueError("missing generate_manifest with link_dbs") with open(gman, encoding="utf-8") as handle: link_dbs = json.load(handle).get("link_dbs") - if not isinstance(link_dbs, list) or not link_dbs: - raise ValueError(f"generate_manifest.link_dbs empty: {gman}") + if not isinstance(link_dbs, list): + raise ValueError(f"generate_manifest.link_dbs must be a list: {gman}") for path in link_dbs: if not isinstance(path, str) or not os.path.isfile(path): raise ValueError(f"sram db missing: {path}") @@ -105,7 +113,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "target_library": contract.target_library, "synthetic_library": contract.synthetic_library, "link_library": list(contract.link_library) + list(link_dbs), - "max_cores": contract.max_cores, + "max_threads": contract.max_threads, } run_config = write_run_tcl( os.path.join(analysis_dir, "run.tcl"), @@ -119,12 +127,20 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: }, ) except (OSError, ValueError, json.JSONDecodeError) as exc: - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "dc", "error": str(exc)}, trace_id=origin_tid) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "dc", "error": str(exc)}, + trace_id=origin_tid, + ) return script_path = contract.dc_script - report_context = source_context(get_buckyball_path(), contract.chip) + report_context = source_context(get_buckyball_path(), contract.design) dc_log = os.path.join(analysis_dir, "dc_shell.log") - ctx.logger.info(f"Running chip-owned DC synthesis for {contract.chip}, top {top_module}: {script_path}") + ctx.logger.info( + f"Running design-owned DC synthesis for {contract.design}, top {top_module}: {script_path}" + ) result = await stream_run_logger_async( cmd=( f"set -o pipefail; dc_shell -f {shlex.quote(str(script_path))} " @@ -144,15 +160,22 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "dc_report_dir": report_dir, "dc_log": dc_log, "run_config": str(run_config), - "chip": contract.chip, + "design": contract.design, "tapeout_dir": str(contract.root), "replace_manifest": input_data.get("replace_manifest"), } if result.returncode == 0: try: - extra_fields["report_path"] = area_report(report_context, origin_tid, report_dir, contract) + extra_fields["report_path"] = area_report( + report_context, origin_tid, report_dir, contract + ) except (OSError, ValueError) as error: - await check_result(ctx, 1, extra_fields={**extra_fields, "error": str(error)}, trace_id=origin_tid) + await check_result( + ctx, + 1, + extra_fields={**extra_fields, "error": str(error)}, + trace_id=origin_tid, + ) return await check_result( @@ -163,4 +186,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) if result.returncode == 0 and input_data.get("from_power_workflow"): - await ctx.enqueue({"topic": "dc.sim", "data": {**input_data, "area_dir": output_dir, "chip": contract.chip, "_trace_id": origin_tid}}) + await ctx.enqueue( + { + "topic": "dc.sim", + "data": { + **input_data, + "area_dir": output_dir, + "design": contract.design, + "_trace_id": origin_tid, + "target": input_data["target"], + }, + } + ) diff --git a/bbdev/api/steps/dc/02_verilog_api.step.py b/bbdev/api/steps/dc/02_verilog_api.step.py index 72dd1f2de..0556ece91 100644 --- a/bbdev/api/steps/dc/02_verilog_api.step.py +++ b/bbdev/api/steps/dc/02_verilog_api.step.py @@ -1,6 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design config = { "name": "dc-verilog-api", @@ -13,14 +14,19 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - chip = require_chip(body) + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + design = require_design(body) await ctx.enqueue( { "topic": "dc.verilog", "data": { - "chip": chip, + "design": design, "top": body.get("top") or "DigitalTop", "_trace_id": ctx.trace_id, + "target": selection, }, } ) diff --git a/bbdev/api/steps/dc/03_sim_event.step.py b/bbdev/api/steps/dc/03_sim_event.step.py index 833638430..7d55dd80d 100644 --- a/bbdev/api/steps/dc/03_sim_event.step.py +++ b/bbdev/api/steps/dc/03_sim_event.step.py @@ -1,3 +1,5 @@ +from pathlib import Path +from utils.path import design_output_root import os import shlex import shutil @@ -20,7 +22,7 @@ config = { "name": "dc-sim", - "description": "rerun chip-owned simulation and produce activity for PTPX", + "description": "rerun design-owned simulation and produce activity for PTPX", "flows": ["dc"], "triggers": [queue("dc.sim")], "enqueues": ["pt.run"], @@ -37,22 +39,48 @@ def _first(data: dict, *names: str): async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) - chip = input_data.get("chip") + design = input_data.get("design") top_module = input_data.get("top") or "DigitalTop" analysis_dir = input_data.get("analysis_dir") if not isinstance(analysis_dir, str) or not analysis_dir: - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": "missing analysis directory"}, trace_id=origin_tid) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "sim", "error": "missing analysis directory"}, + trace_id=origin_tid, + ) return try: - contract = get_tapeout_contract(get_buckyball_path(), input_data.get("chip"), top_module) + contract = get_tapeout_contract( + get_buckyball_path(), + input_data.get("design"), + top_module, + selection=input_data["target"], + ) except (OSError, ValueError) as exc: - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": str(exc)}, trace_id=origin_tid) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "sim", "error": str(exc)}, + trace_id=origin_tid, + ) return activity_format = str(_first(input_data, "format") or contract.power_format).lower() if activity_format not in {"saif", "vcd", "fsdb"}: - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": f"unsupported activity format: {activity_format}"}, trace_id=origin_tid) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "sim", + "error": f"unsupported activity format: {activity_format}", + }, + trace_id=origin_tid, + ) return # An explicit activity file is retained as a debugging escape hatch. Normal @@ -60,38 +88,118 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: explicit_activity = _first(input_data, "activity") if explicit_activity: if not os.path.isfile(explicit_activity): - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": f"activity file does not exist: {explicit_activity}"}, trace_id=origin_tid) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "sim", + "error": f"activity file does not exist: {explicit_activity}", + }, + trace_id=origin_tid, + ) return - await ctx.enqueue({"topic": "pt.run", "data": {**input_data, "activity": explicit_activity, "format": activity_format, "_trace_id": origin_tid}}) + await ctx.enqueue( + { + "topic": "pt.run", + "data": { + **input_data, + "activity": explicit_activity, + "format": activity_format, + "_trace_id": origin_tid, + "target": input_data["target"], + }, + } + ) + return + + if contract.power_sim_script is None: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "sim", + "error": f"design {design} has no gate-level power simulation entry; provide an activity file", + }, + trace_id=origin_tid, + ) return if shutil.which("bash") is None: - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": "bash is required for chip power_sim.sh"}, trace_id=origin_tid) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "sim", + "error": "bash is required for design power_sim.sh", + }, + trace_id=origin_tid, + ) return activity_dir = os.path.join(analysis_dir, "activity") os.makedirs(activity_dir, exist_ok=True) workload_name = _first(input_data, "workload") or contract.power_workload if not workload_name: - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": "missing workload name; pass --workload "}, trace_id=origin_tid) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "sim", + "error": "missing workload name; pass --workload ", + }, + trace_id=origin_tid, + ) return if os.path.isabs(str(workload_name)): - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": "workload must be a built ELF name, not an absolute path"}, trace_id=origin_tid) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "sim", + "error": "workload must be a built ELF name, not an absolute path", + }, + trace_id=origin_tid, + ) return - if not chip: - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": "missing chip"}, trace_id=origin_tid) + if not design: + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "sim", "error": "missing design"}, + trace_id=origin_tid, + ) return bbdir = get_buckyball_path() workload = None for root in ( - workloads_output_root(bbdir), - os.path.join(workload_build_dir(bbdir, chip), "src"), + str( + Path(design_output_root(bbdir, design, selection=input_data["target"])) + / "workloads" + ), + os.path.join( + workload_build_dir(bbdir, design, selection=input_data["target"]), "src" + ), ): workload = search_workload(root, str(workload_name)) if workload: break if not workload: - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": f"workload not found for chip {chip}: {workload_name}"}, trace_id=origin_tid) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={ + "task": "sim", + "error": f"workload not found for design {design}: {workload_name}", + }, + trace_id=origin_tid, + ) return try: start_ns, end_ns = resolve_power_window( @@ -100,7 +208,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: _first(input_data, "end_ns", "end-ns"), ) except ValueError as exc: - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "sim", "error": str(exc)}, trace_id=origin_tid) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "sim", "error": str(exc)}, + trace_id=origin_tid, + ) return netlist = os.path.join(input_data.get("area_dir") or "", f"{top_module}.v") sdc = os.path.join(input_data.get("area_dir") or "", f"{top_module}.sdc") @@ -112,7 +226,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "ACTIVITY_FILE": activity_path, "ACTIVITY_FORMAT": activity_format, "TOP": top_module, - "CONFIG": chip, + "CONFIG": design, "NETLIST": netlist, "SDC": sdc, "WORKLOAD": workload, @@ -123,8 +237,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) ctx.logger.info( - f"Running chip-owned power simulation for {contract.chip}; " - f"format={activity_format}, workload={workload or ''}, " + f"Running design-owned power simulation for {contract.design}; " + f"format={activity_format}, workload={workload or ''}, " f"window={start_ns or ''}..{end_ns or ''} ns" ) result = await stream_run_logger_async( @@ -136,7 +250,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) if result.returncode == 0 and not os.path.isfile(activity_path): result.returncode = 1 - error = f"chip power simulation completed without producing {activity_path}" + error = f"design power simulation completed without producing {activity_path}" else: error = None await check_result( @@ -145,7 +259,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "sim", - "chip": contract.chip, + "design": contract.design, "power_sim_script": str(contract.power_sim_script), "activity": activity_path, "format": activity_format, @@ -157,4 +271,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) if result.returncode == 0: - await ctx.enqueue({"topic": "pt.run", "data": {**input_data, "activity": activity_path, "format": activity_format, "strip_path": contract.power_strip_path, "_trace_id": origin_tid}}) + await ctx.enqueue( + { + "topic": "pt.run", + "data": { + **input_data, + "activity": activity_path, + "format": activity_format, + "strip_path": contract.power_strip_path, + "_trace_id": origin_tid, + "target": input_data["target"], + }, + } + ) diff --git a/bbdev/api/steps/dc/04_power_api.step.py b/bbdev/api/steps/dc/04_power_api.step.py index c9fd1e8e6..f730dfd12 100644 --- a/bbdev/api/steps/dc/04_power_api.step.py +++ b/bbdev/api/steps/dc/04_power_api.step.py @@ -1,6 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design config = { "name": "dc-power-api", @@ -13,16 +14,21 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - chip = require_chip(body) + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + design = require_design(body) await ctx.enqueue( { "topic": "dc.verilog", "data": { **body, - "chip": chip, + "design": design, "top": body.get("top") or "DigitalTop", "from_power_workflow": True, "_trace_id": ctx.trace_id, + "target": selection, }, } ) diff --git a/bbdev/api/steps/dc/scripts/emit_seq_mem_models.py b/bbdev/api/steps/dc/scripts/emit_seq_mem_models.py index 2a18a4e77..9e89c31b3 100644 --- a/bbdev/api/steps/dc/scripts/emit_seq_mem_models.py +++ b/bbdev/api/steps/dc/scripts/emit_seq_mem_models.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Emit behavioral models for FIRRTL externalized sequential memories. -These models are used only by gate-level activity simulation when a chip has +These models are used only by gate-level activity simulation when a design has not yet supplied PDK SRAM macro simulation collateral. They retain the external-memory interface from ``mems.conf`` so the DC netlist is simulated with functional memories rather than X-producing black boxes. @@ -14,7 +14,9 @@ from pathlib import Path -def write_masked(lines: list[str], *, addr: str, data: str, mask: str, width: int, gran: int) -> None: +def write_masked( + lines: list[str], *, addr: str, data: str, mask: str, width: int, gran: int +) -> None: lines.extend( [ f" for (int i = 0; i < {width // gran}; i++) begin", @@ -39,8 +41,20 @@ def emit(name: str, depth: int, width: int, ports: str, mask_gran: int | None) - " if (RW0_en) begin", " if (RW0_wmode) begin", ] - write_masked(lines, addr="RW0_addr", data="RW0_wdata", mask="RW0_wmask", width=width, gran=mask_gran or width) - lines += [" end", " RW0_rdata <= memory[RW0_addr];", " end", " end"] + write_masked( + lines, + addr="RW0_addr", + data="RW0_wdata", + mask="RW0_wmask", + width=width, + gran=mask_gran or width, + ) + lines += [ + " end", + " RW0_rdata <= memory[RW0_addr];", + " end", + " end", + ] elif ports == "rw": lines += [ " input logic [$clog2(" + str(depth) + ")-1:0] RW0_addr,", @@ -68,9 +82,21 @@ def emit(name: str, depth: int, width: int, ports: str, mask_gran: int | None) - ] if has_mask: lines.append(f" input logic [{width // mask_gran - 1}:0] W0_mask") - lines += [");", f" logic [{width - 1}:0] memory [0:{depth - 1}];", " always @(posedge R0_clk) if (R0_en) R0_data <= memory[R0_addr];", " always @(posedge W0_clk) if (W0_en) begin"] + lines += [ + ");", + f" logic [{width - 1}:0] memory [0:{depth - 1}];", + " always @(posedge R0_clk) if (R0_en) R0_data <= memory[R0_addr];", + " always @(posedge W0_clk) if (W0_en) begin", + ] if has_mask: - write_masked(lines, addr="W0_addr", data="W0_data", mask="W0_mask", width=width, gran=mask_gran or width) + write_masked( + lines, + addr="W0_addr", + data="W0_data", + mask="W0_mask", + width=width, + gran=mask_gran or width, + ) else: lines.append(" memory[W0_addr] <= W0_data;") lines += [" end"] @@ -109,7 +135,9 @@ def main() -> int: ) if not models: raise ValueError(f"no memories found in {args.mem_conf}") - args.output.write_text("`timescale 1ns/1ps\n\n" + "\n".join(models), encoding="utf-8") + args.output.write_text( + "`timescale 1ns/1ps\n\n" + "\n".join(models), encoding="utf-8" + ) return 0 diff --git a/bbdev/api/steps/dc/scripts/gate_dpi.cc b/bbdev/api/steps/dc/scripts/gate_dpi.cc index 77d9f053e..0b28a9fbe 100644 --- a/bbdev/api/steps/dc/scripts/gate_dpi.cc +++ b/bbdev/api/steps/dc/scripts/gate_dpi.cc @@ -16,27 +16,28 @@ extern "C" void scu_sim_exit(std::uint32_t hart, std::uint32_t code) { } extern "C" void scu_uart_rx_sample(std::uint32_t, std::uint32_t, - std::uint32_t *valid, - std::uint32_t *data) { + std::uint32_t *valid, std::uint32_t *data) { *valid = 0; *data = 0; } +extern "C" void dpi_ant_trace(std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t) {} extern "C" void dpi_itrace(std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t) {} + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t) {} extern "C" void dpi_mtrace(std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t) {} + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t) {} extern "C" void dpi_pmctrace(std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t) {} -extern "C" void dpi_mem_pmctrace(std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t) {} -extern "C" void dpi_mtrace_issue(std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t) {} + std::uint32_t) {} +extern "C" void dpi_mem_pmctrace(std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t) {} +extern "C" void dpi_mtrace_issue(std::uint32_t, std::uint32_t, std::uint32_t, + std::uint32_t, std::uint32_t, std::uint32_t) {} diff --git a/bbdev/api/steps/dc/scripts/ptpx.tcl b/bbdev/api/steps/dc/scripts/ptpx.tcl index 7139c27d2..4c27e50de 100755 --- a/bbdev/api/steps/dc/scripts/ptpx.tcl +++ b/bbdev/api/steps/dc/scripts/ptpx.tcl @@ -1,4 +1,4 @@ -# Shared PrimeTime PX implementation. Chip tapeout/power.tcl files are thin +# Shared PrimeTime PX implementation. Design tapeout/power.tcl files are thin # wrappers; this reports only standard-cell dynamic power. if {![info exists RUN_CONFIG] || $RUN_CONFIG eq ""} { error "bbdev must pass -x {set RUN_CONFIG }" } source [file normalize $RUN_CONFIG] diff --git a/bbdev/api/steps/dc/scripts/report.py b/bbdev/api/steps/dc/scripts/report.py index 76d5bc8c5..a32ff901e 100644 --- a/bbdev/api/steps/dc/scripts/report.py +++ b/bbdev/api/steps/dc/scripts/report.py @@ -21,7 +21,11 @@ def area_report(context, trace_id, report_dir, contract): "status": "pass", "cell_um2": float(values[0]), "scope": contract.top_module, - "conditions": f"DC · {Path(contract.target_library).name} · SDC {hashlib.sha256(contract.constraints_sdc.read_bytes()).hexdigest()}", + "conditions": ( + f"DC · {Path(contract.target_library).name} · library SHA256 " + f"{hashlib.sha256(Path(contract.target_library).read_bytes()).hexdigest()} · SDC " + f"{hashlib.sha256(contract.constraints_sdc.read_bytes()).hexdigest()}" + ), "constraints_met": "(VIOLATED)" not in constraints, "report_url": attachment(report, output, report_dir / "area.rpt", "area.rpt"), "constraints_report_url": attachment( diff --git a/bbdev/api/steps/dc/scripts/tapeout.py b/bbdev/api/steps/dc/scripts/tapeout.py index 6dbed01a2..a7774f95b 100644 --- a/bbdev/api/steps/dc/scripts/tapeout.py +++ b/bbdev/api/steps/dc/scripts/tapeout.py @@ -1,4 +1,4 @@ -"""Resolve chip-owned tapeout flow contracts.""" +"""Resolve design-owned tapeout flow contracts.""" from __future__ import annotations @@ -8,6 +8,7 @@ import tomllib from dataclasses import dataclass from pathlib import Path +from utils.path import design_resource_dir def _tcl_word(value: str) -> str: @@ -29,18 +30,18 @@ class SramGeom: @dataclass(frozen=True) class TapeoutContract: - chip: str + design: str root: Path dc_script: Path constraints_sdc: Path power_script: Path - power_sim_script: Path + power_sim_script: Path | None sram_mdf: Path top_module: str target_library: str synthetic_library: list[str] link_library: list[str] - max_cores: int + max_threads: int power_format: str power_start_ns: str | None power_end_ns: str | None @@ -60,8 +61,16 @@ def _p(root: Path, value: str) -> Path: def resolve_power_window( contract: TapeoutContract, start_ns: object | None, end_ns: object | None ) -> tuple[str | None, str | None]: - start = str(start_ns) if start_ns is not None and str(start_ns) != "" else contract.power_start_ns - end = str(end_ns) if end_ns is not None and str(end_ns) != "" else contract.power_end_ns + start = ( + str(start_ns) + if start_ns is not None and str(start_ns) != "" + else contract.power_start_ns + ) + end = ( + str(end_ns) + if end_ns is not None and str(end_ns) != "" + else contract.power_end_ns + ) if (start is None) != (end is None): raise ValueError("power start_ns and end_ns must be supplied together") if start is not None: @@ -84,8 +93,14 @@ def clock_period_ns(contract: TapeoutContract) -> float: return period -def get_tapeout_contract(bbdir: str | os.PathLike[str], chip: str, top: str | None = None) -> TapeoutContract: - root = Path(bbdir) / "examples" / "chips" / chip / "tapeout" +def get_tapeout_contract( + bbdir: str | os.PathLike[str], + design: str, + top: str | None = None, + *, + selection: str, +) -> TapeoutContract: + root = design_resource_dir(bbdir, design, "tapeout", selection=selection) with (root / "config.toml").open("rb") as f: s = tomllib.load(f)["tapeout"] @@ -96,23 +111,29 @@ def get_tapeout_contract(bbdir: str | os.PathLike[str], chip: str, top: str | No raise ValueError(f"duplicate tapeout.sram name {name}") bitwrite = row["bitwrite"] if "bitwrite" in row else False if not isinstance(bitwrite, bool): - raise ValueError(f"tapeout.sram {name}: bitwrite must be bool, got {bitwrite!r}") - sram_table[name] = SramGeom(name, row["words"], row["mux"], row["bits"], bitwrite) + raise ValueError( + f"tapeout.sram {name}: bitwrite must be bool, got {bitwrite!r}" + ) + sram_table[name] = SramGeom( + name, row["words"], row["mux"], row["bits"], bitwrite + ) pw = s.get("power_workload") return TapeoutContract( - chip=chip, + design=design, root=root, dc_script=_p(root, s["dc_script"]), constraints_sdc=_p(root, s["constraints_sdc"]), power_script=_p(root, s["power_script"]), - power_sim_script=_p(root, s["power_sim_script"]), + power_sim_script=( + _p(root, s["power_sim_script"]) if "power_sim_script" in s else None + ), sram_mdf=_p(root, s["sram_mdf"]), top_module=top or s["top"], target_library=str(_p(root, s["target_library"])), synthetic_library=[str(_p(root, p)) for p in s["synthetic_library"]], link_library=[str(_p(root, p)) for p in s.get("link_library", [])], - max_cores=s["max_cores"], + max_threads=s["max_threads"], power_format=s["power_format"], power_start_ns=s.get("power_start_ns"), power_end_ns=s.get("power_end_ns"), @@ -146,6 +167,10 @@ def write_run_tcl(path: str | os.PathLike[str], values: dict[str, object]) -> Pa def write_run_env(path: str | os.PathLike[str], values: dict[str, object]) -> Path: output = Path(path) output.parent.mkdir(parents=True, exist_ok=True) - lines = [f"{key}={shlex.quote(str(value))}" for key, value in values.items() if value is not None] + lines = [ + f"{key}={shlex.quote(str(value))}" + for key, value in values.items() + if value is not None + ] output.write_text("\n".join(lines) + "\n", encoding="utf-8") return output diff --git a/bbdev/api/steps/firesim/00_enumeratefpgas_api.step.py b/bbdev/api/steps/firesim/00_enumeratefpgas_api.step.py index da11565ad..855525837 100644 --- a/bbdev/api/steps/firesim/00_enumeratefpgas_api.step.py +++ b/bbdev/api/steps/firesim/00_enumeratefpgas_api.step.py @@ -1,6 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design config = { "name": "firesim-enumeratefpgas-api", @@ -14,13 +15,17 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) await ctx.enqueue( { "topic": "firesim.enumeratefpgas", - "data": {"chip": chip, "_trace_id": ctx.trace_id}, + "data": {"design": design, "_trace_id": ctx.trace_id, "target": selection}, } ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py b/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py index d05a9fce1..734147116 100644 --- a/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py +++ b/bbdev/api/steps/firesim/00_enumeratefpgas_event.step.py @@ -9,14 +9,14 @@ from utils.path import get_buckyball_path, sim_name from utils.stream_run import stream_run_logger_async -from utils.event_common import check_result, get_origin_trace_id, require_chip +from utils.event_common import check_result, get_origin_trace_id, require_design scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) from firesim_env import setup_firesim_env -from chip_yaml import firesim_cmd, prepare_yamls +from design_yaml import firesim_cmd, prepare_yamls config = { "name": "firesim-enumeratefpgas", @@ -32,9 +32,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() try: - chip = require_chip(input_data) - target_config = sim_name(bbdir, chip, "firesim") - yaml_dir = prepare_yamls(bbdir, chip, target_config) + design = require_design(input_data) + target_config = sim_name( + bbdir, design, "firesim", selection=input_data["target"] + ) + yaml_dir = prepare_yamls(bbdir, design, target_config) except ValueError as error: ctx.logger.error(str(error)) await check_result( @@ -47,7 +49,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return env = setup_firesim_env() - ctx.logger.info(f"firesim enumeratefpgas chip={chip}") + ctx.logger.info(f"firesim enumeratefpgas design={design}") result = await stream_run_logger_async( cmd=firesim_cmd("enumeratefpgas", yaml_dir), logger=ctx.logger, @@ -60,6 +62,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, result.returncode, continue_run=False, - extra_fields={"chip": chip, "yaml_dir": yaml_dir}, + extra_fields={"design": design, "yaml_dir": yaml_dir}, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/firesim/01_buildbitstream_api.step.py b/bbdev/api/steps/firesim/01_buildbitstream_api.step.py index 106f7e7e3..608be6c2d 100644 --- a/bbdev/api/steps/firesim/01_buildbitstream_api.step.py +++ b/bbdev/api/steps/firesim/01_buildbitstream_api.step.py @@ -1,6 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design config = { "name": "firesim-buildbitstream-api", @@ -14,13 +15,17 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) await ctx.enqueue( { "topic": "firesim.buildbitstream", - "data": {"chip": chip, "_trace_id": ctx.trace_id}, + "data": {"design": design, "_trace_id": ctx.trace_id, "target": selection}, } ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/01_buildbitstream_event.step.py b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py index eecd2e726..feed9d36d 100644 --- a/bbdev/api/steps/firesim/01_buildbitstream_event.step.py +++ b/bbdev/api/steps/firesim/01_buildbitstream_event.step.py @@ -9,14 +9,14 @@ from utils.path import get_buckyball_path, sim_name from utils.stream_run import stream_run_logger_async -from utils.event_common import check_result, get_origin_trace_id, require_chip +from utils.event_common import check_result, get_origin_trace_id, require_design scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) from firesim_env import setup_firesim_env -from chip_yaml import firesim_cmd, prepare_yamls +from design_yaml import firesim_cmd, prepare_yamls config = { "name": "firesim-buildbitstream", @@ -32,8 +32,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() try: - chip = require_chip(input_data) - target_config = sim_name(bbdir, chip, "firesim") + design = require_design(input_data) + target_config = sim_name( + bbdir, design, "firesim", selection=input_data["target"] + ) except ValueError as error: ctx.logger.error(str(error)) await check_result( @@ -45,7 +47,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - yaml_dir = prepare_yamls(bbdir, chip, target_config) + yaml_dir = prepare_yamls(bbdir, design, target_config) env = setup_firesim_env() au280_tcl = f"{bbdir}/thirdparty/firesim/platforms/xilinx_alveo_u280/cl_firesim/scripts/au280.tcl" @@ -61,7 +63,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f.write(fixed_content) ctx.logger.info("Fixed AU280 board_part version: 1.2 -> 1.0") - ctx.logger.info(f"firesim buildbitstream chip={chip} TARGET_CONFIG={target_config}") + ctx.logger.info( + f"firesim buildbitstream design={design} TARGET_CONFIG={target_config}" + ) result = await stream_run_logger_async( cmd=firesim_cmd("buildbitstream", yaml_dir), logger=ctx.logger, @@ -74,6 +78,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, result.returncode, continue_run=False, - extra_fields={"chip": chip, "target_config": target_config, "yaml_dir": yaml_dir}, + extra_fields={ + "design": design, + "target_config": target_config, + "yaml_dir": yaml_dir, + }, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/firesim/02_infrasetup_api.step.py b/bbdev/api/steps/firesim/02_infrasetup_api.step.py index b1f87b6ba..5e8dbb957 100644 --- a/bbdev/api/steps/firesim/02_infrasetup_api.step.py +++ b/bbdev/api/steps/firesim/02_infrasetup_api.step.py @@ -1,6 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design config = { "name": "firesim-infrasetup-api", @@ -14,13 +15,17 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) await ctx.enqueue( { "topic": "firesim.infrasetup", - "data": {"chip": chip, "_trace_id": ctx.trace_id}, + "data": {"design": design, "_trace_id": ctx.trace_id, "target": selection}, } ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/02_infrasetup_event.step.py b/bbdev/api/steps/firesim/02_infrasetup_event.step.py index 55ba00c5a..4a4a9e600 100644 --- a/bbdev/api/steps/firesim/02_infrasetup_event.step.py +++ b/bbdev/api/steps/firesim/02_infrasetup_event.step.py @@ -9,14 +9,14 @@ from utils.path import get_buckyball_path, sim_name from utils.stream_run import stream_run_logger_async -from utils.event_common import check_result, get_origin_trace_id, require_chip +from utils.event_common import check_result, get_origin_trace_id, require_design scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) from firesim_env import setup_firesim_env -from chip_yaml import firesim_cmd, prepare_yamls, recipe_name +from design_yaml import firesim_cmd, prepare_yamls, recipe_name config = { "name": "firesim-infrasetup", @@ -32,15 +32,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() try: - chip = require_chip(input_data) - target_config = sim_name(bbdir, chip, "firesim") - yaml_dir = prepare_yamls(bbdir, chip, target_config) + design = require_design(input_data) + target_config = sim_name( + bbdir, design, "firesim", selection=input_data["target"] + ) + yaml_dir = prepare_yamls(bbdir, design, target_config) hwdb = os.path.join(yaml_dir, "config_hwdb.yaml") text = open(hwdb, encoding="utf-8").read() if "bitstream_tar: null" in text or "bitstream_tar: file://" not in text: raise ValueError( - f"no bitstream for recipe {recipe_name(chip)}; " - f"run bbdev firesim --buildbitstream '--chip {chip}' first" + f"no bitstream for recipe {recipe_name(design)}; " + f"run bbdev firesim --buildbitstream '--design {design}' first" ) except ValueError as error: ctx.logger.error(str(error)) @@ -54,7 +56,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return env = setup_firesim_env() - ctx.logger.info(f"firesim infrasetup chip={chip} TARGET_CONFIG={target_config}") + ctx.logger.info(f"firesim infrasetup design={design} TARGET_CONFIG={target_config}") result = await stream_run_logger_async( cmd=firesim_cmd("infrasetup", yaml_dir), logger=ctx.logger, @@ -67,6 +69,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, result.returncode, continue_run=False, - extra_fields={"chip": chip, "target_config": target_config, "yaml_dir": yaml_dir}, + extra_fields={ + "design": design, + "target_config": target_config, + "yaml_dir": yaml_dir, + }, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/firesim/03_runworkload_api.step.py b/bbdev/api/steps/firesim/03_runworkload_api.step.py index 36f9ff897..4dff37470 100644 --- a/bbdev/api/steps/firesim/03_runworkload_api.step.py +++ b/bbdev/api/steps/firesim/03_runworkload_api.step.py @@ -1,6 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design config = { "name": "firesim-runworkload-api", @@ -14,11 +15,17 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - data = {"chip": chip, "_trace_id": ctx.trace_id} + data = {"design": design, "_trace_id": ctx.trace_id} if body.get("jobs") is not None: data["jobs"] = body["jobs"] - await ctx.enqueue({"topic": "firesim.runworkload", "data": data}) + await ctx.enqueue( + {"topic": "firesim.runworkload", "data": {**(data), "target": selection}} + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/firesim/03_runworkload_event.step.py b/bbdev/api/steps/firesim/03_runworkload_event.step.py index 3edec98db..41b36169f 100644 --- a/bbdev/api/steps/firesim/03_runworkload_event.step.py +++ b/bbdev/api/steps/firesim/03_runworkload_event.step.py @@ -9,14 +9,14 @@ from utils.path import get_buckyball_path, sim_name from utils.stream_run import stream_run_logger_async -from utils.event_common import check_result, get_origin_trace_id, require_chip +from utils.event_common import check_result, get_origin_trace_id, require_design scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) from firesim_env import setup_firesim_env -from chip_yaml import firesim_cmd, prepare_yamls, recipe_name +from design_yaml import firesim_cmd, prepare_yamls, recipe_name config = { "name": "firesim-runworkload", @@ -32,15 +32,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() try: - chip = require_chip(input_data) - target_config = sim_name(bbdir, chip, "firesim") - yaml_dir = prepare_yamls(bbdir, chip, target_config) + design = require_design(input_data) + target_config = sim_name( + bbdir, design, "firesim", selection=input_data["target"] + ) + yaml_dir = prepare_yamls(bbdir, design, target_config) hwdb = os.path.join(yaml_dir, "config_hwdb.yaml") text = open(hwdb, encoding="utf-8").read() if "bitstream_tar: null" in text or "bitstream_tar: file://" not in text: raise ValueError( - f"no bitstream for recipe {recipe_name(chip)}; " - f"run bbdev firesim --buildbitstream '--chip {chip}' first" + f"no bitstream for recipe {recipe_name(design)}; " + f"run bbdev firesim --buildbitstream '--design {design}' first" ) except ValueError as error: ctx.logger.error(str(error)) @@ -54,7 +56,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return env = setup_firesim_env() - ctx.logger.info(f"firesim runworkload chip={chip} TARGET_CONFIG={target_config}") + ctx.logger.info( + f"firesim runworkload design={design} TARGET_CONFIG={target_config}" + ) result = await stream_run_logger_async( cmd=firesim_cmd("runworkload", yaml_dir), logger=ctx.logger, @@ -67,6 +71,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, result.returncode, continue_run=False, - extra_fields={"chip": chip, "target_config": target_config, "yaml_dir": yaml_dir}, + extra_fields={ + "design": design, + "target_config": target_config, + "yaml_dir": yaml_dir, + }, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/firesim/README.md b/bbdev/api/steps/firesim/README.md index 8a9a69ea2..29258fecb 100644 --- a/bbdev/api/steps/firesim/README.md +++ b/bbdev/api/steps/firesim/README.md @@ -1,18 +1,18 @@ # FireSim Workflow -FireSim FPGA simulation. All commands take `--chip`; mill class comes from -`examples/chips//configs/chip.toml` `[sims].firesim`. +FireSim FPGA simulation. All commands take `--design`; mill class comes from +`examples/designs//configs/design.toml` `[sims].firesim`. -Manager YAMLs are generated per chip under `scripts/yaml//`. +Manager YAMLs are generated per design under `scripts/yaml//`. ## Commands ```bash -bbdev firesim --enumeratefpgas '--chip toy' -bbdev firesim --buildbitstream '--chip toy' -bbdev firesim --infrasetup '--chip toy' -bbdev firesim --runworkload '--chip toy' +bbdev firesim --enumeratefpgas '--design toy' +bbdev firesim --buildbitstream '--design toy' +bbdev firesim --infrasetup '--design toy' +bbdev firesim --runworkload '--design toy' ``` `buildbitstream` takes hours. `infrasetup` / `runworkload` require a prior -bitstream under `thirdparty/firesim/deploy/results-build` for that chip recipe. +bitstream under `thirdparty/firesim/deploy/results-build` for that design recipe. diff --git a/bbdev/api/steps/firesim/scripts/chip_yaml.py b/bbdev/api/steps/firesim/scripts/chip_yaml.py deleted file mode 100644 index 52b30c179..000000000 --- a/bbdev/api/steps/firesim/scripts/chip_yaml.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Prepare per-chip FireSim manager YAMLs from chip sims.firesim.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Optional - - -def recipe_name(chip: str) -> str: - return f"alveo_u280_firesim_{chip}_no_nic" - - -def _latest_bitstream(results: Path, recipe: str) -> Optional[Path]: - if not results.is_dir(): - return None - dirs = sorted( - (p for p in results.iterdir() if p.is_dir() and recipe in p.name), - key=lambda p: p.stat().st_mtime, - ) - for d in reversed(dirs): - matches = list(d.rglob("firesim.tar.gz")) - if matches: - return matches[0] - return None - - -def prepare_yamls(bbdir: str, chip: str, target_config: str) -> str: - """Write chip-scoped firesim yamls under scripts/yaml/. Return that dir.""" - if not target_config or "." not in target_config: - raise ValueError(f"invalid firesim TARGET_CONFIG: {target_config!r}") - recipe = recipe_name(chip) - script_dir = Path(bbdir) / "bbdev" / "api" / "steps" / "firesim" / "scripts" - out = script_dir / "yaml" / chip - out.mkdir(parents=True, exist_ok=True) - - deploy = Path(bbdir) / "thirdparty" / "firesim" / "deploy" - makefrag = script_dir / "makefrag" / "firesim" - if not makefrag.is_dir(): - raise FileNotFoundError(f"missing firesim makefrag: {makefrag}") - - build_dir = deploy / "FIRESIM_BUILD_DIR" - runs_dir = deploy / "FIRESIM_RUNS_DIR" - build_dir.mkdir(parents=True, exist_ok=True) - runs_dir.mkdir(parents=True, exist_ok=True) - - bitstream = _latest_bitstream(deploy / "results-build", recipe) - if bitstream is None: - bitstream_line = ( - " # bitstream_tar filled after buildbitstream; required for infrasetup/runworkload\n" - " bitstream_tar: null" - ) - else: - bitstream_line = f" bitstream_tar: file://{bitstream.resolve()}" - - (out / "config_build_recipes.yaml").write_text( - f"""# Auto-generated for chip={chip}. Do not edit by hand. -{recipe}: - PLATFORM: xilinx_alveo_u280 - TARGET_PROJECT: firesim - TARGET_PROJECT_MAKEFRAG: {makefrag} - DESIGN: FireSim - TARGET_CONFIG: {target_config} - PLATFORM_CONFIG: BaseXilinxAlveoU280Config - deploy_quintuplet: null - platform_config_args: - fpga_frequency: 30 - build_strategy: TIMING - post_build_hook: null - metasim_customruntimeconfig: null - bit_builder_recipe: bit-builder-recipes/xilinx_alveo_u280.yaml -""", - encoding="utf-8", - ) - - (out / "config_build.yaml").write_text( - f"""# Auto-generated for chip={chip}. Do not edit by hand. -build_farm: - base_recipe: build-farm-recipes/externally_provisioned.yaml - recipe_arg_overrides: - default_build_dir: {build_dir} - build_farm_hosts: - - localhost - -builds_to_run: - - {recipe} - -agfis_to_share: [] - -share_with_accounts: {{}} -""", - encoding="utf-8", - ) - - (out / "config_runtime.yaml").write_text( - f"""# Auto-generated for chip={chip}. Do not edit by hand. -run_farm: - base_recipe: run-farm-recipes/externally_provisioned.yaml - recipe_arg_overrides: - default_platform: XilinxAlveoU280InstanceDeployManager - default_simulation_dir: {runs_dir} - default_fpga_db: /opt/firesim-db.json - run_farm_hosts_to_use: - - localhost: one_fpgas_spec - -metasimulation: - metasimulation_enabled: false - metasimulation_host_simulator: verilator - metasimulation_only_plusargs: "+fesvr-step-size=128 +max-cycles=100000000" - metasimulation_only_vcs_plusargs: "+vcs+initreg+0 +vcs+initmem+0" - -target_config: - topology: no_net_config - no_net_num_nodes: 1 - link_latency: 6405 - switching_latency: 10 - net_bandwidth: 200 - profile_interval: -1 - default_hw_config: {recipe} - plusarg_passthrough: "" - -tracing: - enable: no - output_format: 0 - selector: 1 - start: 0 - end: -1 - -autocounter: - read_rate: 0 - -workload: - workload_name: interactive.json - terminate_on_completion: no - suffix_tag: null - -host_debug: - zero_out_dram: no - disable_synth_asserts: no - -synth_print: - start: 0 - end: -1 - cycle_prefix: yes -""", - encoding="utf-8", - ) - - (out / "config_hwdb.yaml").write_text( - f"""# Auto-generated for chip={chip}. bitstream_tar updated by buildbitstream / results-build scan. -{recipe}: -{bitstream_line} - deploy_quintuplet_override: null - custom_runtime_config: null -""", - encoding="utf-8", - ) - - return str(out) - - -def firesim_cmd(action: str, yaml_dir: str) -> str: - return ( - f"firesim {action}" - f" -a {yaml_dir}/config_hwdb.yaml" - f" -b {yaml_dir}/config_build.yaml" - f" -r {yaml_dir}/config_build_recipes.yaml" - f" -c {yaml_dir}/config_runtime.yaml" - ) diff --git a/bbdev/api/steps/firesim/scripts/design_yaml.py b/bbdev/api/steps/firesim/scripts/design_yaml.py new file mode 100644 index 000000000..eb4afd314 --- /dev/null +++ b/bbdev/api/steps/firesim/scripts/design_yaml.py @@ -0,0 +1,168 @@ +"""Prepare per-design FireSim manager YAMLs from design sims.firesim.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Optional + + +def recipe_name(design: str) -> str: + return f"alveo_u280_firesim_{design}_no_nic" + + +def _latest_bitstream(results: Path, recipe: str) -> Optional[Path]: + if not results.is_dir(): + return None + dirs = sorted( + (p for p in results.iterdir() if p.is_dir() and recipe in p.name), + key=lambda p: p.stat().st_mtime, + ) + for d in reversed(dirs): + matches = list(d.rglob("firesim.tar.gz")) + if matches: + return matches[0] + return None + + +def prepare_yamls(bbdir: str, design: str, target_config: str) -> str: + """Write design-scoped firesim yamls under scripts/yaml/. Return that dir.""" + if not target_config or "." not in target_config: + raise ValueError(f"invalid firesim TARGET_CONFIG: {target_config!r}") + recipe = recipe_name(design) + script_dir = Path(bbdir) / "bbdev" / "api" / "steps" / "firesim" / "scripts" + out = script_dir / "yaml" / design + out.mkdir(parents=True, exist_ok=True) + + deploy = Path(bbdir) / "thirdparty" / "firesim" / "deploy" + makefrag = script_dir / "makefrag" / "firesim" + if not makefrag.is_dir(): + raise FileNotFoundError(f"missing firesim makefrag: {makefrag}") + + build_dir = deploy / "FIRESIM_BUILD_DIR" + runs_dir = deploy / "FIRESIM_RUNS_DIR" + build_dir.mkdir(parents=True, exist_ok=True) + runs_dir.mkdir(parents=True, exist_ok=True) + + bitstream = _latest_bitstream(deploy / "results-build", recipe) + if bitstream is None: + bitstream_line = ( + " # bitstream_tar filled after buildbitstream; required for infrasetup/runworkload\n" + " bitstream_tar: null" + ) + else: + bitstream_line = f" bitstream_tar: file://{bitstream.resolve()}" + + (out / "config_build_recipes.yaml").write_text( + f"""# Auto-generated for design={design}. Do not edit by hand. +{recipe}: + PLATFORM: xilinx_alveo_u280 + TARGET_PROJECT: firesim + TARGET_PROJECT_MAKEFRAG: {makefrag} + DESIGN: FireSim + TARGET_CONFIG: {target_config} + PLATFORM_CONFIG: BaseXilinxAlveoU280Config + deploy_quintuplet: null + platform_config_args: + fpga_frequency: 30 + build_strategy: TIMING + post_build_hook: null + metasim_customruntimeconfig: null + bit_builder_recipe: bit-builder-recipes/xilinx_alveo_u280.yaml +""", + encoding="utf-8", + ) + + (out / "config_build.yaml").write_text( + f"""# Auto-generated for design={design}. Do not edit by hand. +build_farm: + base_recipe: build-farm-recipes/externally_provisioned.yaml + recipe_arg_overrides: + default_build_dir: {build_dir} + build_farm_hosts: + - localhost + +builds_to_run: + - {recipe} + +agfis_to_share: [] + +share_with_accounts: {{}} +""", + encoding="utf-8", + ) + + (out / "config_runtime.yaml").write_text( + f"""# Auto-generated for design={design}. Do not edit by hand. +run_farm: + base_recipe: run-farm-recipes/externally_provisioned.yaml + recipe_arg_overrides: + default_platform: XilinxAlveoU280InstanceDeployManager + default_simulation_dir: {runs_dir} + default_fpga_db: /opt/firesim-db.json + run_farm_hosts_to_use: + - localhost: one_fpgas_spec + +metasimulation: + metasimulation_enabled: false + metasimulation_host_simulator: verilator + metasimulation_only_plusargs: "+fesvr-step-size=128 +max-cycles=100000000" + metasimulation_only_vcs_plusargs: "+vcs+initreg+0 +vcs+initmem+0" + +target_config: + topology: no_net_config + no_net_num_nodes: 1 + link_latency: 6405 + switching_latency: 10 + net_bandwidth: 200 + profile_interval: -1 + default_hw_config: {recipe} + plusarg_passthrough: "" + +tracing: + enable: no + output_format: 0 + selector: 1 + start: 0 + end: -1 + +autocounter: + read_rate: 0 + +workload: + workload_name: interactive.json + terminate_on_completion: no + suffix_tag: null + +host_debug: + zero_out_dram: no + disable_synth_asserts: no + +synth_print: + start: 0 + end: -1 + cycle_prefix: yes +""", + encoding="utf-8", + ) + + (out / "config_hwdb.yaml").write_text( + f"""# Auto-generated for design={design}. bitstream_tar updated by buildbitstream / results-build scan. +{recipe}: +{bitstream_line} + deploy_quintuplet_override: null + custom_runtime_config: null +""", + encoding="utf-8", + ) + + return str(out) + + +def firesim_cmd(action: str, yaml_dir: str) -> str: + return ( + f"firesim {action}" + f" -a {yaml_dir}/config_hwdb.yaml" + f" -b {yaml_dir}/config_build.yaml" + f" -r {yaml_dir}/config_build_recipes.yaml" + f" -c {yaml_dir}/config_runtime.yaml" + ) diff --git a/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk b/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk index a7d2d7e26..01ec696ff 100644 --- a/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk +++ b/bbdev/api/steps/firesim/scripts/makefrag/firesim/config.mk @@ -9,7 +9,7 @@ makefile_dir := $(patsubst %/,%,$(dir $(makefile_path))) chipyard_dir := $(abspath $(makefile_dir)/../../../../../../../arch/thirdparty/chipyard) # These point at the main class of the target's Chisel generator -DESIGN_PACKAGE ?= firechip.chip +DESIGN_PACKAGE ?= firechip.design DESIGN ?= FireSim # Override to use our custom config package diff --git a/bbdev/api/steps/firesim/scripts/makefrag/firesim/driver.mk b/bbdev/api/steps/firesim/scripts/makefrag/firesim/driver.mk index 4042d8382..a9a0d9c60 100644 --- a/bbdev/api/steps/firesim/scripts/makefrag/firesim/driver.mk +++ b/bbdev/api/steps/firesim/scripts/makefrag/firesim/driver.mk @@ -12,7 +12,7 @@ else LRISCV=-lriscv endif -firechip_lib_dir = $(chipyard_dir)/generators/firechip/chip/src/main/cc +firechip_lib_dir = $(chipyard_dir)/generators/firechip/design/src/main/cc firechip_bridgestubs_lib_dir = $(chipyard_dir)/generators/firechip/bridgestubs/src/main/cc testchipip_csrc_dir = $(chipyard_dir)/generators/testchipip/src/main/resources/testchipip/csrc diff --git a/bbdev/api/steps/ip/01_generate_api.step.py b/bbdev/api/steps/ip/01_generate_api.step.py index d12372730..561971245 100644 --- a/bbdev/api/steps/ip/01_generate_api.step.py +++ b/bbdev/api/steps/ip/01_generate_api.step.py @@ -1,6 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design def req_arg(body: dict, name: str): @@ -19,12 +20,16 @@ def req_arg(body: dict, name: str): async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) data = { - "chip": chip, + "design": design, "consumer": req_arg(body, "consumer") or "dc", "top": req_arg(body, "top") or "DigitalTop", } @@ -33,12 +38,21 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: output_dir = req_arg(body, "output_dir") if output_dir is not None: if not isinstance(output_dir, str) or not output_dir: - return ApiResponse(status=400, body={"error": "output_dir must be a non-empty string"}) + return ApiResponse( + status=400, body={"error": "output_dir must be a non-empty string"} + ) data["output_dir"] = output_dir next_topic = req_arg(body, "next_topic") if next_topic is not None: if not isinstance(next_topic, str) or not next_topic: - return ApiResponse(status=400, body={"error": "next_topic must be a non-empty string"}) + return ApiResponse( + status=400, body={"error": "next_topic must be a non-empty string"} + ) data["next_topic"] = next_topic - await ctx.enqueue({"topic": "ip.generate", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "ip.generate", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/ip/01_generate_event.step.py b/bbdev/api/steps/ip/01_generate_event.step.py index 994d82d48..c662f65c5 100644 --- a/bbdev/api/steps/ip/01_generate_event.step.py +++ b/bbdev/api/steps/ip/01_generate_event.step.py @@ -12,11 +12,10 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.event_common import check_result, get_origin_trace_id, require_chip +from utils.event_common import check_result, get_origin_trace_id, require_design from utils.path import get_buckyball_path, rtl_dir from generate import generate_sram - config = { "name": "ip-generate", "description": "generate SRAM macros from elaborator mems.conf via MacroCompiler", @@ -29,7 +28,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) try: - chip = require_chip(input_data) + design = require_design(input_data) consumer = input_data.get("consumer") or "dc" if consumer not in ("dc", "yosys"): raise ValueError("consumer must be dc or yosys") @@ -37,8 +36,23 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if not isinstance(top, str) or not top: raise ValueError("top must be a non-empty string") bbdir = Path(get_buckyball_path()).resolve() - build_dir = Path(rtl_dir(bbdir, chip, "tapeout" if consumer == "dc" else "verilog", input_data.get("output_dir"))).resolve() - man = generate_sram(bbdir=bbdir, chip=chip, build_dir=build_dir) + build_dir = Path( + rtl_dir( + bbdir, + design, + "tapeout" if consumer == "dc" else "verilog", + input_data.get("output_dir"), + selection=input_data["target"], + ) + ).resolve() + man = generate_sram( + bbdir=bbdir, + design=design, + selection=input_data["target"], + build_dir=build_dir, + top_module=top, + source_list=build_dir / f"{consumer}_sources.list", + ) except Exception as exc: _, failure_result = await check_result( ctx, @@ -51,22 +65,25 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra = { "task": "ip.generate", - "chip": man["chip"], + "design": man["design"], "generate_manifest": man["generate_manifest"], "sram_macros_v": man["sram_macros_v"], "mem_count": man["mem_count"], "build_dir": str(build_dir), } - await check_result(ctx, 0, continue_run=True, extra_fields=extra, trace_id=origin_tid) + await check_result( + ctx, 0, continue_run=True, extra_fields=extra, trace_id=origin_tid + ) await ctx.enqueue( { "topic": "ip.replace", "data": { **input_data, - "chip": chip, + "design": design, "consumer": consumer, "top": top, "_trace_id": origin_tid, + "target": input_data["target"], }, } ) diff --git a/bbdev/api/steps/ip/02_replace_api.step.py b/bbdev/api/steps/ip/02_replace_api.step.py index 8e3118687..ae59b95f0 100644 --- a/bbdev/api/steps/ip/02_replace_api.step.py +++ b/bbdev/api/steps/ip/02_replace_api.step.py @@ -1,6 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design def req_arg(body: dict, name: str): @@ -19,12 +20,16 @@ def req_arg(body: dict, name: str): async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) data = { - "chip": chip, + "design": design, "consumer": req_arg(body, "consumer") or "dc", "top": req_arg(body, "top") or "DigitalTop", } @@ -33,12 +38,21 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: output_dir = req_arg(body, "output_dir") if output_dir is not None: if not isinstance(output_dir, str) or not output_dir: - return ApiResponse(status=400, body={"error": "output_dir must be a non-empty string"}) + return ApiResponse( + status=400, body={"error": "output_dir must be a non-empty string"} + ) data["output_dir"] = output_dir next_topic = req_arg(body, "next_topic") if next_topic is not None: if not isinstance(next_topic, str) or not next_topic: - return ApiResponse(status=400, body={"error": "next_topic must be a non-empty string"}) + return ApiResponse( + status=400, body={"error": "next_topic must be a non-empty string"} + ) data["next_topic"] = next_topic - await ctx.enqueue({"topic": "ip.replace", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "ip.replace", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/ip/02_replace_event.step.py b/bbdev/api/steps/ip/02_replace_event.step.py index 6312ec87f..e5335a667 100644 --- a/bbdev/api/steps/ip/02_replace_event.step.py +++ b/bbdev/api/steps/ip/02_replace_event.step.py @@ -12,7 +12,7 @@ if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -from utils.event_common import check_result, get_origin_trace_id, require_chip +from utils.event_common import check_result, get_origin_trace_id, require_design from utils.path import get_buckyball_path, rtl_dir from replace import replace_sources @@ -28,7 +28,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) try: - chip = require_chip(input_data) + design = require_design(input_data) consumer = input_data.get("consumer") or "dc" if consumer not in {"dc", "yosys"}: raise ValueError("consumer must be dc or yosys") @@ -37,11 +37,23 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: raise ValueError("top must be a non-empty string") bbdir = Path(get_buckyball_path()).resolve() - build_dir = Path(rtl_dir(bbdir, chip, "tapeout" if consumer == "dc" else "verilog", input_data.get("output_dir"))).resolve() + build_dir = Path( + rtl_dir( + bbdir, + design, + "tapeout" if consumer == "dc" else "verilog", + input_data.get("output_dir"), + selection=input_data["target"], + ) + ).resolve() source_list_path = build_dir / f"{consumer}_sources.list" if not source_list_path.is_file(): raise FileNotFoundError(f"missing source list: {source_list_path}") - sources = [line.strip() for line in source_list_path.read_text().splitlines() if line.strip()] + sources = [ + line.strip() + for line in source_list_path.read_text().splitlines() + if line.strip() + ] if not sources: raise ValueError(f"empty source list: {source_list_path}") @@ -88,26 +100,34 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, 1, continue_run=False, - extra_fields={"task": "ip.replace", "error": "next_topic must be a non-empty string"}, + extra_fields={ + "task": "ip.replace", + "error": "next_topic must be a non-empty string", + }, trace_id=origin_tid, ) return failure_result - await check_result(ctx, 0, continue_run=True, extra_fields=extra, trace_id=origin_tid) + await check_result( + ctx, 0, continue_run=True, extra_fields=extra, trace_id=origin_tid + ) await ctx.enqueue( { "topic": next_topic, "data": { **input_data, - "chip": chip, + "design": design, "consumer": consumer, "top": top_module, "source_list": result["source_list"], "replace_manifest": result["replace_manifest"], "generate_manifest": str(generate_manifest), "_trace_id": origin_tid, + "target": input_data["target"], }, } ) return - await check_result(ctx, 0, continue_run=False, extra_fields=extra, trace_id=origin_tid) + await check_result( + ctx, 0, continue_run=False, extra_fields=extra, trace_id=origin_tid + ) diff --git a/bbdev/api/steps/ip/scripts/generate.py b/bbdev/api/steps/ip/scripts/generate.py index 20fd4a41c..1b5d0a5db 100644 --- a/bbdev/api/steps/ip/scripts/generate.py +++ b/bbdev/api/steps/ip/scripts/generate.py @@ -11,8 +11,10 @@ from tapeout import SramGeom, get_tapeout_contract from macro_compiler import run_macro_compiler +from replace import source_scope from sram_compiler import generate_sram_dbs, leaf_names_from_macros + def pad_mems_conf(text: str) -> str: lines = [] for raw in text.splitlines(): @@ -55,8 +57,15 @@ def parse_mems(text: str) -> list[tuple[str, int, int]]: if not line: continue parts = line.split() - if len(parts) < 6 or parts[0] != "name" or parts[2] != "depth" or parts[4] != "width": - raise ValueError(f"mems.conf:{i}: expected 'name depth width ...', got {line!r}") + if ( + len(parts) < 6 + or parts[0] != "name" + or parts[2] != "depth" + or parts[4] != "width" + ): + raise ValueError( + f"mems.conf:{i}: expected 'name depth width ...', got {line!r}" + ) try: depth = int(parts[3]) width = int(parts[5]) @@ -92,7 +101,9 @@ def module_insts(text: str) -> dict[str, Counter[str]]: return out -def check_mapping(*, mems_text: str, macros_v: Path, mdf: dict[str, tuple[int, int]]) -> None: +def check_mapping( + *, mems_text: str, macros_v: Path, mdf: dict[str, tuple[int, int]] +) -> None: if not macros_v.is_file(): raise FileNotFoundError(f"missing macros verilog: {macros_v}") insts = module_insts(macros_v.read_text()) @@ -159,7 +170,9 @@ def geoms_from_macros( return geoms -def check_bitwrite_contract(geoms: list[SramGeom], mdf: Path, macros_v: Path, sram_table: dict[str, SramGeom]) -> None: +def check_bitwrite_contract( + geoms: list[SramGeom], mdf: Path, macros_v: Path, sram_table: dict[str, SramGeom] +) -> None: raw = json.loads(mdf.read_text()) if not isinstance(raw, list): raise ValueError(f"mdf must be a list: {mdf}") @@ -185,7 +198,9 @@ def check_bitwrite_contract(geoms: list[SramGeom], mdf: Path, macros_v: Path, sr mask_name = p.get("mask port name") if g.bitwrite: if mask_name != "BWEN": - raise ValueError(f"{g.name}: bitwrite requires mask port BWEN, got {mask_name!r}") + raise ValueError( + f"{g.name}: bitwrite requires mask port BWEN, got {mask_name!r}" + ) if p.get("mask port polarity") != "active low": raise ValueError(f"{g.name}: BWEN polarity must be active low") if p.get("mask granularity") != 1: @@ -195,7 +210,9 @@ def check_bitwrite_contract(geoms: list[SramGeom], mdf: Path, macros_v: Path, sr used = {g.name for g in geoms} for geom in sram_table.values(): if geom.bitwrite and geom.name not in used: - raise RuntimeError(f"bitwrite sram {geom.name} not instantiated in {macros_v}") + raise RuntimeError( + f"bitwrite sram {geom.name} not instantiated in {macros_v}" + ) def _leaf_paths(cache_dir: Path, name: str, corner: str) -> dict[str, str]: @@ -210,8 +227,11 @@ def _leaf_paths(cache_dir: Path, name: str, corner: str) -> dict[str, str]: def generate_sram( *, bbdir: str | Path, - chip: str, + design: str, + selection: str, build_dir: str | Path, + top_module: str, + source_list: str | Path, out_dir: str | Path | None = None, ) -> dict: bbdir = Path(bbdir).resolve() @@ -219,8 +239,16 @@ def generate_sram( src = build_dir / "mems.conf" if not src.is_file(): raise FileNotFoundError(f"missing elaborator mems.conf: {src}") - padded = pad_mems_conf(src.read_text()) - contract = get_tapeout_contract(bbdir, chip) + mems = src.read_text() + parse_mems(mems) + reachable, _ = source_scope(Path(source_list).read_text().splitlines(), top_module) + selected = "\n".join( + line + for line in mems.splitlines() + if line.strip() and line.split()[1] in reachable + ) + padded = pad_mems_conf(selected) if selected else "" + contract = get_tapeout_contract(bbdir, design, selection=selection) if out_dir is None: dest = build_dir / "ip-generate" else: @@ -233,35 +261,43 @@ def generate_sram( firrtl = dest / "sram_macros.fir" mems_out.write_text(padded) arch_dir = bbdir / "arch" - run_macro_compiler( - mems_conf=mems_out, - mdf=contract.sram_mdf, - verilog=verilog, - firrtl=firrtl, - arch_dir=arch_dir, - ) - check_mapping( - mems_text=padded, - macros_v=verilog, - mdf=load_mdf(contract.sram_mdf), - ) - geoms = geoms_from_macros( - macros_v=verilog, - sram_table=contract.sram_table, - mdf=contract.sram_mdf, - ) - check_bitwrite_contract(geoms, contract.sram_mdf, verilog, contract.sram_table) + if padded: + run_macro_compiler( + mems_conf=mems_out, + mdf=contract.sram_mdf, + verilog=verilog, + firrtl=firrtl, + arch_dir=arch_dir, + ) + check_mapping( + mems_text=padded, + macros_v=verilog, + mdf=load_mdf(contract.sram_mdf), + ) + geoms = geoms_from_macros( + macros_v=verilog, + sram_table=contract.sram_table, + mdf=contract.sram_mdf, + ) + check_bitwrite_contract(geoms, contract.sram_mdf, verilog, contract.sram_table) + else: + verilog.write_text("") + firrtl.write_text("") + geoms = [] ip_db = build_dir.parent / f"{build_dir.name}-ip-db" - db_paths, corner_tag = generate_sram_dbs( - geoms=geoms, - process=contract.sram_process, - corner=contract.sram_corner, - cache_dir=ip_db, - lc_shell=contract.lc_shell, - ) + if geoms: + db_paths, corner_tag = generate_sram_dbs( + geoms=geoms, + process=contract.sram_process, + corner=contract.sram_corner, + cache_dir=ip_db, + lc_shell=contract.lc_shell, + ) + else: + db_paths, corner_tag = [], contract.sram_corner names = [g.name for g in geoms] man = { - "chip": chip, + "design": design, "sram_mdf": str(contract.sram_mdf), "mems_conf": str(mems_out), "sram_macros_v": str(verilog), @@ -269,10 +305,7 @@ def generate_sram( "sram_corner": corner_tag, "ip_db": str(ip_db), "leaves": names, - "leaf_paths": { - g.name: _leaf_paths(ip_db, g.name, corner_tag) - for g in geoms - }, + "leaf_paths": {g.name: _leaf_paths(ip_db, g.name, corner_tag) for g in geoms}, "link_dbs": [str(p) for p in db_paths], "leaf_count": len(names), "mem_count": sum(1 for line in padded.splitlines() if line.strip()), diff --git a/bbdev/api/steps/ip/scripts/macro_compiler.py b/bbdev/api/steps/ip/scripts/macro_compiler.py index d6c4b6319..decd9a4ac 100644 --- a/bbdev/api/steps/ip/scripts/macro_compiler.py +++ b/bbdev/api/steps/ip/scripts/macro_compiler.py @@ -1,7 +1,6 @@ from __future__ import annotations import os -import shlex import sys from pathlib import Path @@ -10,17 +9,19 @@ sys.path.insert(0, utils_path) from utils.stream_run import stream_run +from utils.path import get_buckyball_path +from steps.mill.scripts.mill import mill_cmd def run_macro_compiler( *, mems_conf: Path, mdf: Path, verilog: Path, firrtl: Path, arch_dir: Path ) -> None: verilog.parent.mkdir(parents=True, exist_ok=True) - cmd = shlex.join( + cmd = mill_cmd( + get_buckyball_path(), + arch_dir, + "tapeout.runMain", [ - "mill", - "-i", - "tapeout.runMain", "tapeout.macros.MacroCompiler", "-n", str(mems_conf), @@ -32,7 +33,7 @@ def run_macro_compiler( str(mdf), "--mode", "strict", - ] + ], ) r = stream_run(cmd, cwd=str(arch_dir)) if r.returncode != 0: diff --git a/bbdev/api/steps/ip/scripts/replace.py b/bbdev/api/steps/ip/scripts/replace.py index 264a6a2c2..e0008dc82 100644 --- a/bbdev/api/steps/ip/scripts/replace.py +++ b/bbdev/api/steps/ip/scripts/replace.py @@ -4,49 +4,41 @@ import re from pathlib import Path -def replace_sources( - *, - source_paths: list[str], - top_module: str, - sram_macros_v: str | Path, - output_dir: str | Path, - consumer: str, - generate_manifest: str | Path | None = None, -) -> dict: - paths = [Path(path).resolve() for path in source_paths] - macros = Path(sram_macros_v).resolve() +def source_scope( + source_paths: list[str | Path], top_module: str +) -> tuple[set[str], set[Path]]: + paths = [Path(path).resolve() for path in source_paths] modules: dict[str, Path] = {} texts: dict[str, str] = {} for path in paths: - text = path.read_text() - match = re.search( - r"^\s*(?:\(\*.*?\*\)\s*)*module\s+([A-Za-z_][A-Za-z0-9_$]*)\b", - text, - re.M, + text = re.sub(r"/\*.*?\*/|//[^\n]*", "", path.read_text(), flags=re.S) + matches = list( + re.finditer( + r"^\s*(?:\(\*.*?\*\)\s*)*module\s+([A-Za-z_][A-Za-z0-9_$]*)\b", + text, + re.M, + ) ) - if match is None: - stripped = re.sub(r"/\*.*?\*/|//.*?$", "", text, flags=re.M | re.S) - if re.search(r"\bmodule\b", stripped): - raise RuntimeError(f"no module declaration: {path}") - continue - name = match.group(1) - if name in modules: - raise RuntimeError(f"duplicate module {name}: {modules[name]} vs {path}") - modules[name] = path - texts[name] = text + for match in matches: + name = match.group(1) + if name in modules: + raise RuntimeError( + f"duplicate module {name}: {modules[name]} vs {path}" + ) + modules[name] = path + end = re.search(r"\bendmodule\b", text[match.end() :]).end() + match.end() + texts[name] = text[match.end() : end] if top_module not in modules: raise ValueError(f"synthesis top is not defined in source list: {top_module}") - if not macros.is_file(): - raise FileNotFoundError(f"missing sram_macros.v: {macros}") reachable = {top_module} pending = [top_module] while pending: cur = pending.pop() for child, _inst in re.findall( - r"^\s*([A-Za-z_][A-Za-z0-9_$]*)\s*(?:#\s*\(.*?\)\s*)?([A-Za-z_][A-Za-z0-9_$]*)\s*\(", + r"\b([A-Za-z_][A-Za-z0-9_$]*)\s*(?:#\s*\(.*?\)\s*)?([A-Za-z_][A-Za-z0-9_$]*)\s*\(", texts[cur], re.M | re.S, ): @@ -54,7 +46,24 @@ def replace_sources( reachable.add(child) pending.append(child) - selected_paths = {modules[name] for name in reachable} + return reachable, {modules[name] for name in reachable} + + +def replace_sources( + *, + source_paths: list[str], + top_module: str, + sram_macros_v: str | Path, + output_dir: str | Path, + consumer: str, + generate_manifest: str | Path | None = None, +) -> dict: + paths = [Path(path).resolve() for path in source_paths] + macros = Path(sram_macros_v).resolve() + + if not macros.is_file(): + raise FileNotFoundError(f"missing sram_macros.v: {macros}") + reachable, selected_paths = source_scope(paths, top_module) selected = [path for path in paths if path in selected_paths] if macros not in selected: selected.append(macros) diff --git a/bbdev/api/steps/kernel/01_build_api.step.py b/bbdev/api/steps/kernel/01_build_api.step.py index 21c08ce97..d5e7b016b 100644 --- a/bbdev/api/steps/kernel/01_build_api.step.py +++ b/bbdev/api/steps/kernel/01_build_api.step.py @@ -1,5 +1,8 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api +from utils.event_common import require_design + config = { "name": "kernel-build-api", "description": "build RISC-V kernel + rootfs for Pegasus", @@ -11,13 +14,18 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - memory = body.get("guest-memory-mib", 512) - if isinstance(memory, bool) or not str(memory).isdigit() or not 1 <= int(memory) <= 16384: - return ApiResponse(status=400, body={"error": "guest-memory-mib must be an integer in 1..16384 (P2E DDR capacity)"}) - storage = body.get("model-storage", "initramfs") - if storage not in ("initramfs", "ddr"): - return ApiResponse(status=400, body={"error": "model-storage must be initramfs or ddr"}) - if storage == "ddr" and (not body.get("model") or int(memory) >= 16384): - return ApiResponse(status=400, body={"error": "ddr requires --model and guest-memory-mib < 16384"}) - await ctx.enqueue({"topic": "kernel.build", "data": {**body, "_trace_id": ctx.trace_id}}) + try: + selection = require_target(body, ("device",)) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + require_design(body) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + await ctx.enqueue( + { + "topic": "kernel.build", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index 1a9c439f0..5a0545533 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -1,9 +1,8 @@ +from utils.path import design_resource_dir import os -import json -import re +import shlex import shutil import sys -import tomllib from motia import FlowContext, queue @@ -11,150 +10,35 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path +from utils.path import get_buckyball_path, load_design_config from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id -# Import bin_to_hex converter -scripts_path = os.path.join(os.path.dirname(__file__), "scripts") -if scripts_path not in sys.path: - sys.path.insert(0, scripts_path) -from bin_to_hex import bin_to_hex - config = { "name": "kernel-build", - "description": "build RISC-V kernel + rootfs for image via bb-tests/workloads/lib/kernel", + "description": "build RISC-V kernel + rootfs for image via stack/os/linux", "flows": ["kernel"], "triggers": [queue("kernel.build")], "enqueues": [], } -def hart_count_params(input_data: dict) -> dict: - allowed = { - "hart-count", - "model", - "chip", - "interactive", - "guest-memory-mib", - "model-storage", - "_trace_id", - } - unknown = sorted(k for k in input_data if k not in allowed) - if unknown: - raise ValueError(f"unknown kernel build parameter(s): {', '.join(unknown)}") - - if "hart-count" in input_data: - value = input_data["hart-count"] - if isinstance(value, bool) or not str(value).isdigit() or int(value) < 1: - raise ValueError("hart-count must be a positive integer") - - chip = input_data.get("chip") - if chip: - derived = os.path.join( - get_buckyball_path(), - "examples", - "chips", - chip, - "configs", - "generated", - "config", - "derived.json", - ) - with open(derived) as stream: - harts = json.load(stream)["harts"] - count = sum(hart["visible"] for hart in harts) - if "hart-count" in input_data and int(input_data["hart-count"]) != count: - raise ValueError(f"hart-count must match chip topology: {count}") - else: - count = int(input_data.get("hart-count", 64)) - - if count < 1: - raise ValueError("hart-count must be at least 1") - - return {"count": count} - - -def kernel_interactive(input_data: dict) -> bool: - interactive = input_data.get("interactive", False) - if not isinstance(interactive, bool): - raise ValueError("interactive must be a boolean flag") - return interactive - - -def kernel_model(input_data: dict) -> str: - model = input_data.get("model", "") - if model in ("", None): - return "" - if not isinstance(model, str): - raise ValueError("model must be a string") - - if not re.fullmatch(r"[a-z0-9][a-z0-9-]*", model): - raise ValueError(f"invalid model name: {model}") - return model - - -def kernel_chip(input_data: dict, bbdir: str) -> str: - chip = input_data.get("chip", "") - if chip in ("", None): - return "" - if not isinstance(chip, str): - raise ValueError("chip must be a string") - if os.path.sep in chip or chip in {".", ".."}: - raise ValueError(f"invalid chip: {chip}") - - chip_dir = os.path.join(bbdir, "examples", "chips", chip) - if not os.path.isdir(chip_dir): - raise ValueError(f"unknown chip: {chip}") - - return chip - - def kernel_build_dir( - bbdir: str, - hart_params: dict, - model: str = "", - chip: str = "", - interactive: bool = False, - guest_memory_mib: int = 512, - model_storage: str = "initramfs", + bbdir: str, hart_count: int, design: str, interactive: bool, guest_memory_mib: int ) -> str: - count = hart_params["count"] suffix = "" if guest_memory_mib == 512 else f"-mem{guest_memory_mib}M" - if chip: - suffix += f"-chip-{chip}" - if model: - suffix += f"-model-{model}" + suffix += f"-design-{design}" if interactive: suffix += "-interactive" - if model_storage == "ddr": - suffix += "-ddr" - if count == 64: - return os.path.join(bbdir, "bb-tests", "build", f"kernel{suffix}") - return os.path.join( - bbdir, "bb-tests", "build", f"kernel-h{count}{suffix}" - ) + name = "kernel" if hart_count == 64 else f"kernel-h{hart_count}" + return os.path.join(bbdir, "bb-tests", "build", f"{name}{suffix}") -def fw_payload_name( - hart_params: dict, - model: str = "", - chip: str = "", - guest_memory_mib: int = 512, - model_storage: str = "initramfs", -) -> str: - count = hart_params["count"] - name = "fw_payload" - if count != 64: - name = f"{name}-h{count}" - if model: - name = f"{name}-{model}" - elif chip: - name = f"{name}-{chip}-linux" +def fw_payload_name(hart_count: int, design: str, guest_memory_mib: int) -> str: + name = "fw_payload" if hart_count == 64 else f"fw_payload-h{hart_count}" + name += f"-{design}-linux" if guest_memory_mib != 512: name += f"-mem{guest_memory_mib}M" - if model_storage == "ddr": - name += "-ddr" return name @@ -162,77 +46,35 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - kernel_src = os.path.join(bbdir, "bb-tests", "workloads", "lib", "kernel") - try: - model = kernel_model(input_data) - if model and not input_data.get("chip"): - raise ValueError("--model requires --chip") - chip = kernel_chip(input_data, bbdir) - memory_value = input_data.get("guest-memory-mib", 512) - if model and "guest-memory-mib" not in input_data: - config_path = os.path.join( - bbdir, "examples", "models", chip, model, "configs", "running-param.toml" - ) - with open(config_path, "rb") as stream: - memory_value = tomllib.load(stream).get("memory_mib", 512) - if ( - isinstance(memory_value, bool) - or not str(memory_value).isdigit() - or not 1 <= int(memory_value) <= 16384 - ): - raise ValueError( - "guest-memory-mib must be an integer in 1..16384 (P2E DDR capacity)" - ) - guest_memory_mib = int(memory_value) - model_storage = input_data.get("model-storage", "initramfs") - if model_storage not in ("initramfs", "ddr"): - raise ValueError("model-storage must be initramfs or ddr") - if model_storage == "ddr" and ( - not input_data.get("model") or guest_memory_mib >= 16384 - ): - raise ValueError("ddr requires --model and guest-memory-mib < 16384") - hart_params = hart_count_params(input_data) - interactive = kernel_interactive(input_data) - except ValueError as e: - ctx.logger.error(str(e)) - await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) - return - output_dir = os.path.join(bbdir, "bb-tests", "output", "kernel") - if chip: - output_dir = os.path.join(output_dir, chip) + kernel_src = os.path.join(bbdir, "stack", "os", "linux") + design = input_data["design"] + design_config = load_design_config(bbdir, design) + hart_count = len(design_config.device.cores) + ddr = design_config.device.ddr + guest_memory_mib = (ddr.bytes - ddr.model) // (1024 * 1024) + interactive = input_data.get("interactive", False) + output_dir = os.path.join( + bbdir, "examples/designs", design, "output/sims/device/workloads/kernel" + ) os.makedirs(output_dir, exist_ok=True) kernel_build = kernel_build_dir( - bbdir, - hart_params, - model, - chip, - interactive=interactive, - guest_memory_mib=guest_memory_mib, - model_storage=model_storage, + bbdir, hart_count, design, interactive, guest_memory_mib ) interactive_arg = "ON" if interactive else "OFF" - cmake_model_name = model + programs = design_resource_dir( + bbdir, design, "kernel", selection=input_data["target"] + ) # cmake configure configure_cmd = ( f"cmake -B {kernel_build} -S {kernel_src} " - f"-DBUCKYBALL_HART_COUNT={hart_params['count']} " + f"-DBUCKYBALL_HART_COUNT={hart_count} " f"-DBUCKYBALL_GUEST_MEMORY_MIB={guest_memory_mib} " - f"-DBUCKYBALL_MODEL_STORAGE={model_storage} " - f"-DBUCKYBALL_KERNEL_MODEL={cmake_model_name} " - f"-DBUCKYBALL_KERNEL_CHIP={chip} " + f"-DBUCKYBALL_KERNEL_DESIGN={design} " f"-DBUCKYBALL_KERNEL_INTERACTIVE={interactive_arg} " - f"-DBUCKYBALL_MODEL_DATASET=" + f"-DBUCKYBALL_MODEL_BASE={ddr.base + ddr.bytes - ddr.model:#x} " + f"-DBUCKYBALL_KERNEL_WORKLOADS={programs}" ) - if chip and not model: - workload_toml = os.path.join( - bbdir, "examples", "chips", chip, "kernel", "workloads.toml" - ) - if not os.path.isfile(workload_toml): - ctx.logger.error(f"chip workload toml not found: {workload_toml}") - await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) - return - configure_cmd += f" -DBUCKYBALL_WORKLOAD_TOML={workload_toml}" result = await stream_run_logger_async( cmd=configure_cmd, @@ -261,16 +103,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - # Convert fw_payload.bin to hex for P2E memory backdoor - payload_name = fw_payload_name( - hart_params, - cmake_model_name if model else "", - chip, - guest_memory_mib, - model_storage, - ) + payload_name = fw_payload_name(hart_count, design, guest_memory_mib) fw_payload_bin = os.path.join(output_dir, f"{payload_name}.bin") - fw_payload_hex = os.path.join(output_dir, f"{payload_name}.hex") fw_payload_elf = os.path.join(output_dir, f"{payload_name}.elf") built_payload_elf = os.path.join( kernel_build, @@ -281,30 +115,25 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "fw_payload.elf", ) - if not os.path.exists(fw_payload_bin): - ctx.logger.error(f"{payload_name}.bin not found") - await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) - return - if not os.path.isfile(built_payload_elf): - ctx.logger.error(f"fw_payload.elf not found: {built_payload_elf}") - await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) - return - shutil.copy2(built_payload_elf, fw_payload_elf) - ctx.logger.info(f"Converting {fw_payload_bin} to Verilog hex format for P2E...") - success = bin_to_hex(fw_payload_bin, fw_payload_hex, base_address=0x80000000) - if not success: - ctx.logger.error("Failed to convert fw_payload to hex") - await check_result(ctx, 1, continue_run=False, trace_id=origin_tid) - return - - storage_fields = {"model_storage": model_storage} - if model_storage == "ddr": - manifest = os.path.join(output_dir, f"{payload_name}.load.json") - if not os.path.isfile(manifest): - raise ValueError(f"Missing ddr load manifest: {manifest}") - storage_fields["load_manifest"] = manifest - await check_result( - ctx, 0, continue_run=False, extra_fields=storage_fields, trace_id=origin_tid + # The DDR image: the kernel and, when given, the model, each at its place. + # BEMU takes the bin, P2E the hex. + image = os.path.join(output_dir, f"{payload_name}-ddr") + pack = os.path.join(kernel_build, "image-pack") + model = input_data.get("model") + pack_cmd = [pack, f"{image}.bin", f"{image}.hex", str(ddr.bytes), fw_payload_bin] + if model: + pack_cmd += [ + os.path.join( + bbdir, "stack/models/archs/buckyball", design, model, "output/model.img" + ), + str(ddr.bytes - ddr.model), + ] + result = await stream_run_logger_async( + cmd=shlex.join(pack_cmd), + logger=ctx.logger, + stdout_prefix="image pack", + stderr_prefix="image pack", ) + await check_result(ctx, result.returncode, continue_run=False, trace_id=origin_tid) diff --git a/bbdev/api/steps/kernel/scripts/bin_to_hex.py b/bbdev/api/steps/kernel/scripts/bin_to_hex.py deleted file mode 100644 index 97fe16841..000000000 --- a/bbdev/api/steps/kernel/scripts/bin_to_hex.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os - - -def bin_to_hex(bin_file, hex_file, base_address=0): - """ - Convert a raw binary file to Verilog readmemh format. - - The output uses one byte per line with `@0xADDR` markers for the starting - offset (relative to base_address). The input is treated as a contiguous - byte stream starting at base_address; the resulting `@` offset is always 0 - because raw binaries have no internal address information. - - base_address is kept in the signature for API compatibility and is only - used to print the absolute address range. - """ - print(f"Converting {bin_file} -> {hex_file}") - if base_address != 0: - print(f"Base address offset: 0x{base_address:08X}") - - try: - with open(bin_file, "rb") as f: - data = f.read() - - if not data: - print("Error: Input file is empty") - return False - - with open(hex_file, "w") as f: - f.write("@0\n") - for byte in data: - f.write(f"{byte:02X}\n") - - print(f"Successfully generated {hex_file}") - print( - f"Address range: 0x{base_address:08X} - 0x{base_address + len(data) - 1:08X}" - ) - print(f"Total bytes: {len(data)}") - return True - - except Exception as e: - print(f"Error: Conversion failed - {e}") - return False - - -def main(): - if len(sys.argv) < 3 or len(sys.argv) > 4: - print("Usage: bin_to_hex.py [base_address]") - print(" base_address: optional hex address for reporting (e.g., 0x80000000)") - sys.exit(1) - - bin_file = sys.argv[1] - hex_file = sys.argv[2] - base_address = 0 - - if len(sys.argv) == 4: - base_str = sys.argv[3] - if base_str.startswith("0x") or base_str.startswith("0X"): - base_address = int(base_str, 16) - else: - base_address = int(base_str, 10) - - if not os.path.exists(bin_file): - print(f"Error: File not found: {bin_file}") - sys.exit(1) - - success = bin_to_hex(bin_file, hex_file, base_address=base_address) - sys.exit(0 if success else 1) - - -if __name__ == "__main__": - main() diff --git a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py index d661ce5fb..f99516b60 100644 --- a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py @@ -6,23 +6,17 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -mill_dir = os.path.dirname(__file__) -mill_scripts = os.path.join(mill_dir, "scripts") -if mill_dir not in sys.path: - sys.path.insert(0, mill_dir) -if mill_scripts not in sys.path: - sys.path.insert(0, mill_scripts) -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import get_buckyball_path from utils.path import rtl_out -import mill as mill_run +from steps.mill.scripts.mill import design_cmd from utils.event_common import check_result, get_origin_trace_id from utils.stream_run import stream_run_logger_async config = { "name": "verilator-verilog", - "description": "generate chip verilog code", + "description": "generate design verilog code", "flows": ["verilator"], "triggers": [queue("verilator.verilog")], "enqueues": ["verilator.build"], @@ -35,20 +29,32 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: arch = os.path.join(bbdir, "arch") try: - chip = require_chip(input_data) + design = require_design(input_data) mill_config, build_dir = rtl_out( - bbdir, chip, "verilog", input_data.get("output_dir"), + bbdir, + design, + "verilog", + input_data.get("output_dir"), + selection=input_data["target"], ) os.makedirs(build_dir, exist_ok=True) ctx.logger.info(f"Using mill config: {mill_config}") ctx.logger.info(f"Using build directory: {build_dir}") + command, directory = design_cmd( + bbdir, + design, + "verilator", + build_dir, + diff=bool(input_data.get("diff", False)), + selection=input_data["target"], + ) returncode = ( await stream_run_logger_async( - cmd=mill_run.elaborate_cmd("sims.verilator.Elaborate", mill_config, build_dir), - logger=ctx.logger, - cwd=arch, - stdout_prefix="verilator verilog", - stderr_prefix="verilator verilog", + cmd=command, + logger=ctx.logger, + cwd=directory, + stdout_prefix="verilator verilog", + stderr_prefix="verilator verilog", ) ).returncode except (ValueError, RuntimeError) as error: @@ -60,12 +66,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "task": "validation", "error": str(error), - "example": 'bbdev verilator --verilog "--chip toy"', + "example": 'bbdev verilator --verilog "--design toy"', }, trace_id=origin_tid, ) return failure_result - ctx.logger.info(f"Using chip: {chip}") + ctx.logger.info(f"Using design: {design}") await check_result( ctx, @@ -77,5 +83,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if input_data.get("from_run_workflow") and returncode == 0: await ctx.enqueue( - {"topic": "verilator.build", "data": {**input_data, "output_dir": build_dir, "task": "run"}} + { + "topic": "verilator.build", + "data": { + **input_data, + "output_dir": build_dir, + "task": "run", + "target": input_data["target"], + }, + } ) diff --git a/bbdev/api/steps/mill/02_vcs_verilog_event.step.py b/bbdev/api/steps/mill/02_vcs_verilog_event.step.py index acc8c98b8..2afc11508 100644 --- a/bbdev/api/steps/mill/02_vcs_verilog_event.step.py +++ b/bbdev/api/steps/mill/02_vcs_verilog_event.step.py @@ -6,17 +6,11 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -mill_dir = os.path.dirname(__file__) -mill_scripts = os.path.join(mill_dir, "scripts") -if mill_dir not in sys.path: - sys.path.insert(0, mill_dir) -if mill_scripts not in sys.path: - sys.path.insert(0, mill_scripts) -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import get_buckyball_path from utils.path import rtl_out -import mill as mill_run +from steps.mill.scripts.mill import elaborate_cmd from utils.event_common import check_result, get_origin_trace_id from utils.stream_run import stream_run_logger_async @@ -32,31 +26,41 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) try: - chip = require_chip(input_data) + design = require_design(input_data) except ValueError as exc: - await check_result(ctx, 1, extra_fields={"task": "verilog", "error": str(exc)}, trace_id=origin_tid) + await check_result( + ctx, + 1, + extra_fields={"task": "verilog", "error": str(exc)}, + trace_id=origin_tid, + ) return bbdir = get_buckyball_path() arch = os.path.join(bbdir, "arch") try: mill_config, build_dir = rtl_out( bbdir, - chip, - "verilog" if input_data.get("bebop_vcs") else "tapeout", - input_data.get("output_dir"), + design, + "tapeout", + selection=input_data["target"], ) os.makedirs(build_dir, exist_ok=True) returncode = ( await stream_run_logger_async( - cmd=mill_run.elaborate_cmd("sims.verilator.Elaborate", mill_config, build_dir), - logger=ctx.logger, - cwd=arch, - stdout_prefix="vcs verilog", - stderr_prefix="vcs verilog", + cmd=elaborate_cmd(design, input_data["target"], "verilog", build_dir), + logger=ctx.logger, + cwd=arch, + stdout_prefix="vcs verilog", + stderr_prefix="vcs verilog", ) ).returncode except (ValueError, RuntimeError) as exc: - await check_result(ctx, 1, extra_fields={"task": "verilog", "error": str(exc)}, trace_id=origin_tid) + await check_result( + ctx, + 1, + extra_fields={"task": "verilog", "error": str(exc)}, + trace_id=origin_tid, + ) return await check_result( ctx, @@ -66,4 +70,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) if input_data.get("from_run_workflow") and returncode == 0: - await ctx.enqueue({"topic": "vcs.build", "data": {**input_data, "output_dir": build_dir, "_trace_id": origin_tid}}) + await ctx.enqueue( + { + "topic": "vcs.build", + "data": { + **input_data, + "output_dir": build_dir, + "_trace_id": origin_tid, + "target": input_data["target"], + }, + } + ) diff --git a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py index eff2a57f7..c7c94415d 100644 --- a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py +++ b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py @@ -13,18 +13,12 @@ step_path = os.path.dirname(__file__) if step_path not in sys.path: sys.path.insert(0, step_path) -mill_dir = os.path.dirname(__file__) -mill_scripts = os.path.join(mill_dir, "scripts") -if mill_dir not in sys.path: - sys.path.insert(0, mill_dir) -if mill_scripts not in sys.path: - sys.path.insert(0, mill_scripts) -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import get_buckyball_path, log_dir from utils.stream_run import stream_run_logger_async from utils.path import rtl_out -import mill as mill_run +from steps.mill.scripts.mill import elaborate_cmd from utils.event_common import check_result, get_origin_trace_id config = { @@ -47,7 +41,9 @@ def load_yosys_config(): def prepare_yosys_verilog(build_dir: str, yosys_log_dir: str, logger): def _build_stub_from_header(src_path: str, content: str, force_blackbox: bool): - m = re.search(r"module\s+([A-Za-z_][A-Za-z0-9_]*)\s*\((.*?)\)\s*;", content, re.S) + m = re.search( + r"module\s+([A-Za-z_][A-Za-z0-9_]*)\s*\((.*?)\)\s*;", content, re.S + ) if m is None: raise RuntimeError(f"invalid module header format: {src_path}") mod_name = m.group(1) @@ -60,7 +56,9 @@ def _collect_defined_modules(sv_files): for path in sv_files: with open(path, "r") as f: content = f.read() - for m in re.finditer(r"^\s*module\s+([A-Za-z_][A-Za-z0-9_]*)\b", content, re.M): + for m in re.finditer( + r"^\s*module\s+([A-Za-z_][A-Za-z0-9_]*)\b", content, re.M + ): defined.add(m.group(1)) return defined @@ -70,14 +68,18 @@ def _extract_external_modules(extern_path): names = [] with open(extern_path, "r") as f: for line in f: - m = re.match(r"\s*//\s*external module\s+([A-Za-z_][A-Za-z0-9_]*)\s*$", line) + m = re.match( + r"\s*//\s*external module\s+([A-Za-z_][A-Za-z0-9_]*)\s*$", line + ) if m: names.append(m.group(1)) return names def _collect_ports(mod_name, sv_files): ports = set() - inst_re = re.compile(rf"\b{mod_name}\b\s+[A-Za-z_][A-Za-z0-9_]*\s*\((.*?)\)\s*;", re.S) + inst_re = re.compile( + rf"\b{mod_name}\b\s+[A-Za-z_][A-Za-z0-9_]*\s*\((.*?)\)\s*;", re.S + ) port_re = re.compile(r"\.([A-Za-z_][A-Za-z0-9_]*)\s*\(") for path in sv_files: with open(path, "r") as f: @@ -153,19 +155,23 @@ def _build_blackbox(mod_name, ports): if dpi_stubbed: logger.info( - f"Stubbed {len(dpi_stubbed)} DPI modules: {', '.join(dpi_stubbed[:10])}{'...' if len(dpi_stubbed) > 10 else ''}" + f"Stubbed {len(dpi_stubbed)} DPI modules:" + f" {', '.join(dpi_stubbed[:10])}{'...' if len(dpi_stubbed) > 10 else ''}" ) if pattern_stubbed: logger.info( - f"Stubbed {len(pattern_stubbed)} pattern modules: {', '.join(pattern_stubbed[:10])}{'...' if len(pattern_stubbed) > 10 else ''}" + f"Stubbed {len(pattern_stubbed)} pattern modules:" + f" {', '.join(pattern_stubbed[:10])}{'...' if len(pattern_stubbed) > 10 else ''}" ) if ext_stubbed: logger.info( - f"Stubbed {len(ext_stubbed)} external modules: {', '.join(ext_stubbed[:10])}{'...' if len(ext_stubbed) > 10 else ''}" + f"Stubbed {len(ext_stubbed)} external modules:" + f" {', '.join(ext_stubbed[:10])}{'...' if len(ext_stubbed) > 10 else ''}" ) if skipped: logger.info( - f"Skipped {len(skipped)} unsupported files: {', '.join(skipped[:10])}{'...' if len(skipped) > 10 else ''}" + f"Skipped {len(skipped)} unsupported files:" + f" {', '.join(skipped[:10])}{'...' if len(skipped) > 10 else ''}" ) logger.info(f"Prepared yosys source list: {source_list_path}") return source_list_path @@ -176,22 +182,30 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() arch = os.path.join(bbdir, "arch") try: - chip = require_chip(input_data) + design = require_design(input_data) mill_config, build_dir = rtl_out( - bbdir, chip, "verilog", input_data.get("output_dir"), + bbdir, + design, + "verilog", + input_data.get("output_dir"), + selection=input_data["target"], ) os.makedirs(build_dir, exist_ok=True) ctx.logger.info(f"Using mill config: {mill_config}") ctx.logger.info(f"Using build directory: {build_dir}") returncode = ( await stream_run_logger_async( - cmd=mill_run.elaborate_cmd( - "sims.verilator.Elaborate", mill_config, build_dir, seq_mem=True, - ), - logger=ctx.logger, - cwd=arch, - stdout_prefix="yosys verilog", - stderr_prefix="yosys verilog", + cmd=elaborate_cmd( + design, + input_data["target"], + "verilog", + build_dir, + seq_mem=True, + ), + logger=ctx.logger, + cwd=arch, + stdout_prefix="yosys verilog", + stderr_prefix="yosys verilog", ) ).returncode except (ValueError, RuntimeError) as error: @@ -214,9 +228,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return failure_result yosys_cfg = load_yosys_config() top_module = input_data.get("top") or yosys_cfg.get("top") or "DigitalTop" - yosys_log_dir = input_data.get("log_dir") or log_dir( - bbdir, chip, "synth", datetime.now().strftime("%Y-%m-%d-%H-%M"), - "yosys", top_module, input_data.get("output_dir"), + yosys_log_dir = log_dir( + bbdir, + design, + "synth", + datetime.now().strftime("%Y-%m-%d-%H-%M"), + "yosys", + top_module, + input_data.get("output_dir"), + selection=input_data["target"], ) ctx.logger.info(f"Yosys log dir: {yosys_log_dir}") mem_conf = os.path.join(build_dir, "mems.conf") @@ -245,7 +265,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, 0, continue_run=need_ip, - extra_fields={"task": "verilog", "source_list": source_list_path, "mem_conf": mem_conf}, + extra_fields={ + "task": "verilog", + "source_list": source_list_path, + "mem_conf": mem_conf, + }, trace_id=origin_tid, ) if not need_ip: @@ -256,13 +280,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "topic": "ip.generate", "data": { **input_data, - "chip": chip, + "design": design, "consumer": "yosys", "top": top_module, - "log_dir": yosys_log_dir, + "analysis_dir": yosys_log_dir, "next_topic": "yosys.synth", "task": "run", "_trace_id": origin_tid, + "target": input_data["target"], }, } ) diff --git a/bbdev/api/steps/mill/04_dc_verilog_event.step.py b/bbdev/api/steps/mill/04_dc_verilog_event.step.py index 6b8ed804c..5f331a8a4 100644 --- a/bbdev/api/steps/mill/04_dc_verilog_event.step.py +++ b/bbdev/api/steps/mill/04_dc_verilog_event.step.py @@ -14,9 +14,10 @@ sys.path.insert(0, step_path) from utils.event_common import check_result, get_origin_trace_id -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import get_buckyball_path, log_dir from utils.stream_run import stream_run_logger_async + mill_dir = os.path.join(os.path.dirname(__file__)) mill_scripts = os.path.join(mill_dir, "scripts") if mill_dir not in sys.path: @@ -24,7 +25,7 @@ if mill_scripts not in sys.path: sys.path.insert(0, mill_scripts) from utils.path import rtl_out -import mill as mill_run +from steps.mill.scripts.mill import elaborate_cmd config = { "name": "dc-verilog", @@ -63,7 +64,10 @@ def prepare_dc_verilog(build_dir: str): ) stub_dir = os.path.join(build_dir, "dc_stubs") os.makedirs(stub_dir, exist_ok=True) - skip_dirs = {os.path.join(build_dir, name) for name in ("ip-generate", "ip-replace", "dc_stubs")} + skip_dirs = { + os.path.join(build_dir, name) + for name in ("ip-generate", "ip-replace", "dc_stubs") + } kept = [] stubbed_dpi = [] for path in vsrcs: @@ -91,22 +95,30 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() arch = os.path.join(bbdir, "arch") try: - chip = require_chip(input_data) + design = require_design(input_data) mill_config, build_dir = rtl_out( - bbdir, chip, "tapeout", input_data.get("output_dir"), + bbdir, + design, + "tapeout", + input_data.get("output_dir"), + selection=input_data["target"], ) os.makedirs(build_dir, exist_ok=True) ctx.logger.info(f"Using mill config: {mill_config}") ctx.logger.info(f"Using build directory: {build_dir}") returncode = ( await stream_run_logger_async( - cmd=mill_run.elaborate_cmd( - "sims.verilator.Elaborate", mill_config, build_dir, seq_mem=True, - ), - logger=ctx.logger, - cwd=arch, - stdout_prefix="dc verilog", - stderr_prefix="dc verilog", + cmd=elaborate_cmd( + design, + input_data["target"], + "verilog", + build_dir, + seq_mem=True, + ), + logger=ctx.logger, + cwd=arch, + stdout_prefix="dc verilog", + stderr_prefix="dc verilog", ) ).returncode except (ValueError, RuntimeError) as error: @@ -128,7 +140,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return failure_result top_module = input_data.get("top") or "DigitalTop" - ctx.logger.info(f"Using DC RTL output directory: {build_dir}, synthesis top: {top_module}") + ctx.logger.info( + f"Using DC RTL output directory: {build_dir}, synthesis top: {top_module}" + ) mem_conf = os.path.join(build_dir, "mems.conf") try: @@ -169,7 +183,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: payload = { **input_data, - "chip": chip, + "design": design, "consumer": "dc", "top": top_module, "next_topic": "dc.area", @@ -177,12 +191,26 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: } if input_data.get("from_power_workflow"): payload["analysis_dir"] = log_dir( - bbdir, chip, "tapeout", datetime.now().strftime("%Y-%m-%d-%H-%M"), "dc", "power" + bbdir, + design, + "tapeout", + datetime.now().strftime("%Y-%m-%d-%H-%M"), + "dc", + "power", + selection=input_data["target"], ) else: payload["analysis_dir"] = log_dir( - bbdir, chip, "tapeout", datetime.now().strftime("%Y-%m-%d-%H-%M"), "dc", "area" + bbdir, + design, + "tapeout", + datetime.now().strftime("%Y-%m-%d-%H-%M"), + "dc", + "area", + selection=input_data["target"], ) - await ctx.enqueue({"topic": "ip.generate", "data": payload}) + await ctx.enqueue( + {"topic": "ip.generate", "data": {**(payload), "target": input_data["target"]}} + ) return diff --git a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py index a04f30288..fb0f2ff3d 100644 --- a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py @@ -5,24 +5,18 @@ """ import os -import shutil +import time import sys from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -mill_dir = os.path.dirname(__file__) -mill_scripts = os.path.join(mill_dir, "scripts") -if mill_dir not in sys.path: - sys.path.insert(0, mill_dir) -if mill_scripts not in sys.path: - sys.path.insert(0, mill_scripts) -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import get_buckyball_path from utils.stream_run import stream_run_logger_async from utils.path import rtl_out -import mill as mill_run +from steps.mill.scripts.mill import design_cmd from utils.event_common import check_result, get_origin_trace_id config = { @@ -52,28 +46,36 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() arch = os.path.join(bbdir, "arch") - chip = None + design = None try: - chip = require_chip(input_data) + design = require_design(input_data) mill_config, build_dir = rtl_out( - bbdir, chip, "verilog", input_data.get("output_dir") + bbdir, + design, + "verilog", + input_data.get("output_dir"), + selection=input_data["target"], ) - shutil.rmtree(build_dir, ignore_errors=True) + if not os.path.isabs(build_dir): + raise ValueError("RTL output directory must be absolute") + command, directory = design_cmd( + bbdir, + design, + "verilator", + build_dir, + diff=bool(input_data.get("diff", False)), + selection=input_data["target"], + ) + if os.path.exists(build_dir): + os.rename(build_dir, f"{build_dir}.previous-{time.time_ns()}") os.makedirs(build_dir) - ctx.logger.info(f"Using mill config: {mill_config}") + ctx.logger.info(f"Using mill target: {input_data["target"]}") ctx.logger.info(f"Using build directory: {build_dir}") - command = mill_run.elaborate_cmd( - "sims.verilator.Elaborate", mill_config, build_dir - ) - if input_data.get("diff"): - command += " --difftest" - if input_data.get("main_only"): - command += " --main-only" returncode = ( await stream_run_logger_async( cmd=command, logger=ctx.logger, - cwd=arch, + cwd=directory, stdout_prefix="bebop verilator verilog", stderr_prefix="bebop verilator verilog", ) @@ -87,12 +89,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "task": "validation", "error": str(error), - "example": 'bbdev bebop verilator --verilog "--chip toy"', + "example": 'bbdev bebop verilator --verilog "--design toy"', }, trace_id=origin_tid, ) return - ctx.logger.info(f"Using chip: {chip}") + ctx.logger.info(f"Using design: {design}") output_status = check_verilog_output(build_dir) if returncode != 0: await check_result( @@ -101,7 +103,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "verilog", - "chip": chip, + "design": design, "output_dir": build_dir, "output_status": output_status, }, @@ -110,7 +112,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return if not output_status["is_dir"] or output_status["sv_count"] == 0: ctx.logger.error( - f"Verilog output is invalid: {build_dir} (exists={output_status['exists']}, is_dir={output_status['is_dir']}, sv_count={output_status['sv_count']})" + f"Verilog output is invalid: {build_dir} (exists={output_status['exists']}," + f" is_dir={output_status['is_dir']}, sv_count={output_status['sv_count']})" ) await check_result( ctx, @@ -119,7 +122,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "task": "verilog", "error": "invalid_verilog_output", - "chip": chip, + "design": design, "output_dir": build_dir, "output_status": output_status, }, @@ -130,7 +133,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, returncode, continue_run=input_data.get("from_run_workflow", False) and returncode == 0, - extra_fields={"task": "verilog", "chip": chip, "output_dir": build_dir}, + extra_fields={"task": "verilog", "design": design, "output_dir": build_dir}, trace_id=origin_tid, ) if input_data.get("from_run_workflow") and returncode == 0: @@ -142,6 +145,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "output_dir": build_dir, "vsrc_dir": build_dir, "task": "run", + "target": input_data["target"], }, } ) diff --git a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py index 74478fba0..b1df85ad1 100644 --- a/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py +++ b/bbdev/api/steps/mill/07_bebop_p2e_verilog_event.step.py @@ -7,19 +7,13 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -mill_dir = os.path.dirname(__file__) -mill_scripts = os.path.join(mill_dir, "scripts") -if mill_dir not in sys.path: - sys.path.insert(0, mill_dir) -if mill_scripts not in sys.path: - sys.path.insert(0, mill_scripts) -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import get_buckyball_path from utils.path import rtl_out from steps.bebop.p2e.scripts.timescale import normalize_p2e_timescale from utils.stream_run import stream_run_logger_async -import mill as mill_run +from steps.mill.scripts.mill import elaborate_cmd from utils.event_common import check_result, get_origin_trace_id config = { @@ -36,25 +30,29 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() arch = os.path.join(bbdir, "arch") try: - chip = require_chip(input_data) + design = require_design(input_data) mill_config, build_dir = rtl_out( - bbdir, chip, "p2e", input_data.get("output_dir"), + bbdir, + design, + "p2e", + input_data.get("output_dir"), + selection=input_data["target"], ) shutil.rmtree(build_dir, ignore_errors=True) os.makedirs(build_dir) ctx.logger.info(f"Using mill config: {mill_config}") ctx.logger.info(f"Using build directory: {build_dir}") prefix = "bebop p2e verilog" - command = mill_run.elaborate_cmd("sims.p2e.Elaborate", mill_config, build_dir) + command = elaborate_cmd(design, input_data["target"], "p2e", build_dir) if input_data.get("diff"): command += " --difftest" returncode = ( await stream_run_logger_async( - cmd=command, - logger=ctx.logger, - cwd=arch, - stdout_prefix=prefix, - stderr_prefix=prefix, + cmd=command, + logger=ctx.logger, + cwd=arch, + stdout_prefix=prefix, + stderr_prefix=prefix, ) ).returncode if returncode == 0: @@ -62,12 +60,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: except (ValueError, RuntimeError) as error: ctx.logger.error(str(error)) await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip", "detail": str(error)}, + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design", "detail": str(error)}, trace_id=origin_tid, ) return - ctx.logger.info(f"Using P2E chip: {chip}") + ctx.logger.info(f"Using P2E design: {design}") ctx.logger.info(f"Using P2E output directory: {build_dir}") await check_result( @@ -76,7 +76,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "verilog", - "chip": chip, + "design": design, "output_dir": build_dir, "top_module": "P2ETop", }, diff --git a/bbdev/api/steps/mill/scripts/mill.py b/bbdev/api/steps/mill/scripts/mill.py index bd6b3dbea..45b8cf36f 100644 --- a/bbdev/api/steps/mill/scripts/mill.py +++ b/bbdev/api/steps/mill/scripts/mill.py @@ -2,17 +2,102 @@ import os import shlex +from utils.path import get_buckyball_path, load_design_config, simulation_target +from pathlib import Path -def elaborate_cmd(main: str, config: str, out: str, *, seq_mem: bool = False) -> str: - extra = "" - if seq_mem: - extra += ( - " -lowering-options=disallowLocalVariables" - f" --repl-seq-mem --repl-seq-mem-file={shlex.quote(os.path.join(out, 'mems.conf'))}" +def mill_cmd( + repo: str, directory: str | Path, target: str, arguments=(), *, show: bool = False +) -> str: + repo = Path(repo).resolve() + directory = (repo / directory).resolve() + designs = repo / "examples/designs" + if directory.is_relative_to(designs): + relative = directory.relative_to(designs) + owner = designs / relative.parts[0] + if directory.is_relative_to(owner / "sims"): + work = ( + owner + / "output/sims" + / directory.relative_to(owner / "sims").parent + / "mill" + ) + else: + work = owner / "output/mill" + else: + owner = ( + directory.parent + if directory.name == "arch" and directory.is_relative_to(repo / "examples") + else directory ) - return ( - f"flock /tmp/buckyball-mill.lock mill -i __.test.runMain {main} {config} " - "--disable-annotation-unknown --strip-debug-info -O=debug " - f"--split-verilog -o={shlex.quote(out)}{extra}" + work = owner / "output/mill" + work.mkdir(parents=True, exist_ok=True) + relative = Path(os.path.relpath(directory, work)) + imported = ".".join("^" if part == ".." else f"`{part}`" for part in relative.parts) + source = f"import $file.{imported}.{{build => source}}\n\ndef project = source\n" + launcher = work / "build.sc" + if not launcher.is_file() or launcher.read_text() != source: + launcher.write_text(source) + command = shlex.join( + [ + "flock", + "/tmp/buckyball-mill.lock", + "mill", + "--no-server", + *(["show"] if show else []), + f"project.{target}", + *arguments, + ] ) + return f"cd {shlex.quote(str(work))} && {command}" + + +def design_cmd( + repo: str, + design: str, + product: str, + out: str, + *, + selection: str, + diff: bool = False, +) -> tuple[str, str]: + registered = simulation_target(repo, design, selection) + build = registered.arch + directory = Path(repo) / build.directory + config = Path(repo) / build.config + command = mill_cmd( + repo, + directory, + build.module + ".runMain", + [ + "sims.p2e.Elaborate" if product == "p2e" else "sims.verilator.Elaborate", + "--config", + str(config), + "--target-class", + registered.entry_class, + "--target-dir", + out, + "--disable-annotation-unknown", + "--strip-debug-info", + "-O=debug", + *(["--difftest"] if diff else []), + ], + ) + return command, str(directory) + + +def elaborate_cmd( + design: str, selection: str, product: str, out: str, *, seq_mem: bool = False +) -> str: + command, _ = design_cmd( + get_buckyball_path(), design, product, out, selection=selection + ) + if seq_mem: + command += " " + shlex.join( + [ + "-lowering-options=disallowLocalVariables", + "--repl-seq-mem", + f"--repl-seq-mem-file={os.path.join(out, 'mems.conf')}", + ] + ) + return command diff --git a/bbdev/api/steps/model/01_build_api.step.py b/bbdev/api/steps/model/01_build_api.step.py index 2be9d208c..b9433bee1 100644 --- a/bbdev/api/steps/model/01_build_api.step.py +++ b/bbdev/api/steps/model/01_build_api.step.py @@ -1,15 +1,10 @@ -import os -import sys - from motia import ApiRequest, ApiResponse, FlowContext, api -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) -from steps.model.scripts.build import validate_model_build -from utils.path import get_buckyball_path +from utils.event_common import require_design config = { "name": "model-build-api", - "description": "Build a chip model recipe", + "description": "Build the model recipes of a design", "flows": ["model"], "triggers": [api("POST", "/model/build")], "enqueues": ["model.build"], @@ -19,8 +14,17 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} try: - validate_model_build(get_buckyball_path(), body) + design = require_design(body) except ValueError as error: return ApiResponse(status=400, body={"error": str(error)}) - await ctx.enqueue({"topic": "model.build", "data": {**body, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "model.build", + "data": { + "design": design, + "model": body.get("model"), + "_trace_id": ctx.trace_id, + }, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/model/01_build_event.step.py b/bbdev/api/steps/model/01_build_event.step.py index 704daae83..237f75779 100644 --- a/bbdev/api/steps/model/01_build_event.step.py +++ b/bbdev/api/steps/model/01_build_event.step.py @@ -1,3 +1,4 @@ +import asyncio import os import sys @@ -5,12 +6,12 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) from utils.event_common import check_result, get_origin_trace_id -from steps.model.scripts.build import build_model, validate_model_build +from steps.model.scripts.build import build_model from utils.path import get_buckyball_path config = { "name": "model-build", - "description": "Build a chip model recipe", + "description": "Build a design model recipe", "flows": ["model"], "triggers": [queue("model.build")], "enqueues": [], @@ -21,16 +22,24 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id = get_origin_trace_id(input_data, ctx) params = {key: value for key, value in input_data.items() if key != "_trace_id"} try: - repo = get_buckyball_path() - validate_model_build(repo, params) - build_model(repo, params["chip"], params.get("model"), - ctrace=params.get("ctrace", False), - dtrace=params.get("dtrace", False), - logger=ctx.logger, task_scope=trace_id) + await asyncio.to_thread( + build_model, + get_buckyball_path(), + params["design"], + params["model"], + logger=ctx.logger, + task_scope=trace_id, + ) except Exception as error: ctx.logger.error(str(error)) - await check_result(ctx, 1, continue_run=False, - extra_fields={"error": "model_build_failed", "detail": str(error)}, - trace_id=trace_id) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"error": "model_build_failed", "detail": str(error)}, + trace_id=trace_id, + ) return - await check_result(ctx, 0, continue_run=False, extra_fields=params, trace_id=trace_id) + await check_result( + ctx, 0, continue_run=False, extra_fields=params, trace_id=trace_id + ) diff --git a/bbdev/api/steps/model/scripts/build.py b/bbdev/api/steps/model/scripts/build.py index 1d20936e1..5d2071593 100644 --- a/bbdev/api/steps/model/scripts/build.py +++ b/bbdev/api/steps/model/scripts/build.py @@ -1,67 +1,73 @@ -import re import shlex from pathlib import Path from utils.stream_run import stream_run_logger -from steps.model.scripts.config import compile_config, cmake_arguments -def validate_model_build(repo: str, params: dict) -> list[str]: - unknown = params.keys() - {"chip", "model", "ctrace", "dtrace"} - if unknown: - raise ValueError(f"Unknown model build parameters: {', '.join(sorted(unknown))}") - for name in ("chip", "model"): - value = params.get(name) - if name == "model" and value is None: - continue - if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", value): - raise ValueError(f"Invalid or missing {name}: {value}") - recipes = Path(repo) / "examples/models" / params["chip"] - model = params.get("model") - if model is not None: - if not (recipes / model / "CMakeLists.txt").is_file(): - raise ValueError(f"Model recipe does not exist: {recipes / model}") - models = [model] - else: - models = sorted(path.parent.name for path in recipes.glob("*/CMakeLists.txt")) - if not models: - raise ValueError(f"No model recipes for chip: {params['chip']}") - for name in ("ctrace", "dtrace"): - if not isinstance(params.get(name, False), bool): - raise ValueError(f"{name} must be a boolean flag") - return models - - -def build_model(repo: str, chip: str, model: str | None = None, *, ctrace: bool = False, dtrace: bool = False, - logger, task_scope: str) -> None: - models = validate_model_build(repo, {"chip": chip, "model": model, "ctrace": ctrace, "dtrace": dtrace}) +def build_model( + repo: str, design: str, model: str | None = None, *, logger, task_scope: str +) -> None: + """Build one model recipe of `design`, or every recipe of it when `model` is None. + Models run on the whole device, so they build against its device entry.""" + recipes = Path(repo) / "examples/models" / design + models = ( + [model] + if model + else sorted(path.parent.name for path in recipes.glob("*/CMakeLists.txt")) + ) source = Path(repo) / "stack/models" - compiler = Path(repo) / "stack/compiler/thirdparty/buddy-mlir/build" / chip - command = ["cmake", "--build", str(compiler), "--target", "buddy-opt", - "buddy-translate", "buddy-llc", "rax-pack", "python-package-buddy", "--parallel", "4"] - result = stream_run_logger(cmd=shlex.join(command), cwd=repo, logger=logger, - stdout_prefix="model compiler", stderr_prefix="model compiler", task_scope=task_scope) - if result.returncode: - raise RuntimeError(f"Model compiler build failed ({result.returncode}): {shlex.join(command)}") + from steps.stack.scripts.build import build_design + + # Models compile with the design's compiler plugins and link its device runtime. + build_design( + repo, design=design, selection="device", logger=logger, task_scope=task_scope + ) for model in models: - build = source / "build" / chip / model - config = source / "models" / model / "configs/model.toml" - model_args = [] - if config.is_file(): - pb = build / "config/model.pb" - compile_config(config, pb, model) - model_args = cmake_arguments(pb, model) + build = source / "build" / design / model + build_command = ["cmake", "--build", str(build), "--parallel", "1"] + build_command.extend(["--target", f"{model}-model-build"]) + # A recipe reads the prepared model while it configures, so the model + # is prepared first, by its own project: its weights and checkpoints + # are rebuilt only when their inputs change. + prepare = source / "build" / "models" / model commands = [ - ["cmake", "-S", str(source), "-B", str(build), "-G", "Ninja", - f"-DCHIP={chip}", f"-DMODEL={model}", f"-DCTRACE={'ON' if ctrace else 'OFF'}", - f"-DDTRACE={'ON' if dtrace else 'OFF'}", *model_args], - ["cmake", "--build", str(build), "--parallel", "1"], + # Prepared weights can predate this build tree; use file timestamps. + [ + "cmake", + "-S", + str(source), + "-B", + str(prepare), + "-G", + "Unix Makefiles", + f"-DMODEL={model}", + ], + ["cmake", "--build", str(prepare), "--target", f"{model}-prepare"], + [ + "cmake", + "-S", + str(source), + "-B", + str(build), + "-G", + "Ninja", + "-DARCH=buckyball", + f"-DDESIGN={design}", + "-DBUCKYBALL_SIM_TARGET=device", + f"-DMODEL={model}", + ], + build_command, ] for command in commands: result = stream_run_logger( - cmd=shlex.join(command), cwd=repo, logger=logger, - stdout_prefix=f"model {model}", stderr_prefix=f"model {model}", + cmd=shlex.join(command), + cwd=repo, + logger=logger, + stdout_prefix=f"model {model}", + stderr_prefix=f"model {model}", task_scope=task_scope, ) if result.returncode: - raise RuntimeError(f"Model build failed ({result.returncode}): {shlex.join(command)}") + raise RuntimeError( + f"Model build failed ({result.returncode}): {shlex.join(command)}" + ) diff --git a/bbdev/api/steps/pt/01_ptpx_event.step.py b/bbdev/api/steps/pt/01_ptpx_event.step.py index d952f6231..ad3c9e786 100644 --- a/bbdev/api/steps/pt/01_ptpx_event.step.py +++ b/bbdev/api/steps/pt/01_ptpx_event.step.py @@ -11,7 +11,9 @@ scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: sys.path.insert(0, scripts_path) -dc_scripts_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "dc", "scripts")) +dc_scripts_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "dc", "scripts") +) if dc_scripts_path not in sys.path: sys.path.insert(0, dc_scripts_path) @@ -40,20 +42,35 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: error = "missing fixed DC analysis output directory" elif not isinstance(activity_path, str) or not activity_path: error = "missing required --activity" - elif not isinstance(activity_format, str) or activity_format not in {"saif", "vcd", "fsdb"}: + elif not isinstance(activity_format, str) or activity_format not in { + "saif", + "vcd", + "fsdb", + }: error = "missing or invalid --format; expected saif, vcd, or fsdb" elif shutil.which("pt_shell") is None: error = "pt_shell is not on PATH" else: error = None if error: - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "pt", "error": error}, trace_id=origin_tid) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "pt", "error": error}, + trace_id=origin_tid, + ) return try: netlist = os.path.join(analysis_dir, "outputs", f"{top_module}.v") sdc = os.path.join(analysis_dir, "outputs", f"{top_module}.sdc") - contract = get_tapeout_contract(get_buckyball_path(), input_data.get("chip"), top_module) + contract = get_tapeout_contract( + get_buckyball_path(), + input_data.get("design"), + top_module, + selection=input_data["target"], + ) start_ns, end_ns = resolve_power_window( contract, input_data.get("start_ns", input_data.get("start-ns")), @@ -63,7 +80,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "target_library": contract.target_library, "synthetic_library": contract.synthetic_library, "link_library": contract.link_library, - "max_cores": contract.max_cores, + "max_threads": contract.max_threads, } run_config = write_run_tcl( os.path.join(analysis_dir, "power-run.tcl"), @@ -82,7 +99,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) script = contract.power_script except (OSError, ValueError) as exc: - await check_result(ctx, 1, continue_run=False, extra_fields={"task": "pt", "error": str(exc)}, trace_id=origin_tid) + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "pt", "error": str(exc)}, + trace_id=origin_tid, + ) return result = await stream_run_logger_async( diff --git a/bbdev/api/steps/stack/01_build_api.step.py b/bbdev/api/steps/stack/01_build_api.step.py new file mode 100644 index 000000000..94e0458b4 --- /dev/null +++ b/bbdev/api/steps/stack/01_build_api.step.py @@ -0,0 +1,37 @@ +from utils.event_common import require_target +import os +import re +import sys + +from motia import ApiRequest, ApiResponse, FlowContext, api + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +config = { + "name": "build-stack-api", + "description": "build one level of a design's stack", + "flows": ["stack"], + "triggers": [api("POST", "/stack/build")], + "enqueues": ["stack.build"], +} + + +async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: + body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + if set(body) - {"design", "device", "core"}: + raise ValueError("stack build takes --design and --device or --core") + if not isinstance(body.get("design"), str): + raise ValueError("--design requires a design name") + await ctx.enqueue( + { + "topic": "stack.build", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } + ) + return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/stack/01_build_event.step.py b/bbdev/api/steps/stack/01_build_event.step.py new file mode 100644 index 000000000..f717052c6 --- /dev/null +++ b/bbdev/api/steps/stack/01_build_event.step.py @@ -0,0 +1,54 @@ +import asyncio +import importlib.util +import os +import sys + +from motia import FlowContext, queue + +utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if utils_path not in sys.path: + sys.path.insert(0, utils_path) + +from utils.path import get_buckyball_path + +_stack_build_path = os.path.join(os.path.dirname(__file__), "scripts", "build.py") +_spec = importlib.util.spec_from_file_location("stack_build_module", _stack_build_path) +if _spec is None or _spec.loader is None: + raise RuntimeError(f"cannot load {_stack_build_path}") +stack_build = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(stack_build) + +from utils.event_common import check_result, get_origin_trace_id + +config = { + "name": "build-stack", + "description": "Build one level of a design's stack", + "flows": ["stack"], + "triggers": [queue("stack.build")], + "enqueues": [], +} + + +async def handler(input_data: dict, ctx: FlowContext) -> None: + origin_tid = get_origin_trace_id(input_data, ctx) + bbdir = get_buckyball_path() + log = {"logger": ctx.logger, "task_scope": origin_tid} + try: + await asyncio.to_thread( + stack_build.build_design, + bbdir, + design=input_data["design"], + selection=input_data["target"], + **log, + ) + except Exception as exc: + ctx.logger.error(f"Stack build failed: {exc}") + await check_result( + ctx, + 1, + continue_run=False, + extra_fields={"task": "build", "error": str(exc)}, + trace_id=origin_tid, + ) + return + await check_result(ctx, 0, continue_run=False, trace_id=origin_tid) diff --git a/bbdev/api/steps/stack/scripts/build.py b/bbdev/api/steps/stack/scripts/build.py new file mode 100644 index 000000000..223633eeb --- /dev/null +++ b/bbdev/api/steps/stack/scripts/build.py @@ -0,0 +1,244 @@ +from __future__ import annotations + +import argparse +import fcntl +import os +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + + +def _run( + cmd: list[str], + *, + repo: Path, + logger: object | None = None, + task_scope: str | None = None, + output_prefix: str, +) -> None: + cmd = ["nix", "develop", str(repo), "-c", *cmd] + if logger is None: + result = subprocess.run(cmd, cwd=repo) + else: + from utils.stream_run import stream_run_logger + + result = stream_run_logger( + cmd=shlex.join(cmd), + logger=logger, + cwd=str(repo), + stdout_prefix=output_prefix, + stderr_prefix=output_prefix, + task_scope=task_scope, + ) + if result.returncode != 0: + detail = result.stderr.strip() if logger is not None else "" + if not detail and logger is not None: + lines = result.stdout.splitlines() + failed = [i for i, line in enumerate(lines) if "FAILED:" in line] + if failed: + start = failed[0] + detail = "\n".join(lines[start:]) + else: + detail = "\n".join(lines[-20:]) + if detail: + raise RuntimeError( + f"command failed ({result.returncode}): {' '.join(cmd)}\n{detail}" + ) + raise RuntimeError(f"command failed ({result.returncode}): {' '.join(cmd)}") + + +def compiler_python(repo: str | Path) -> str: + root = Path(repo).resolve() + candidates = [root / "result" / "bin" / "python3"] + path_python = shutil.which("python3") + if path_python: + candidates.append(Path(path_python)) + for candidate in candidates: + if not candidate.is_file(): + continue + probe = subprocess.run( + [str(candidate), "-c", "import nanobind"], capture_output=True + ) + if probe.returncode == 0: + return str(candidate) + checked = ", ".join(str(candidate) for candidate in candidates) + raise RuntimeError(f"python3 with nanobind not found (checked: {checked})") + + +def build_llvm( + repo: str | Path, *, logger: object | None = None, task_scope: str | None = None +) -> Path: + root = Path(repo).resolve() + buddy = root / "stack" / "thirdparty" / "buddy-mlir" + llvm_src = buddy / "llvm" / "llvm" + llvm_build = buddy / "llvm" / "build" + if not llvm_src.is_dir(): + raise RuntimeError(f"missing LLVM source: {llvm_src}") + + python = compiler_python(root) + cmake = [ + "cmake", + "-G", + "Ninja", + "-S", + str(llvm_src), + "-B", + str(llvm_build), + "-DLLVM_ENABLE_PROJECTS=mlir;clang", + "-DLLVM_ENABLE_RUNTIMES=openmp", + "-DLLVM_TARGETS_TO_BUILD=host;RISCV", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DOPENMP_ENABLE_LIBOMPTARGET=OFF", + "-DLIBOMP_LIBFLAGS=-lrt", + "-DCMAKE_BUILD_TYPE=RELEASE", + "-DMLIR_ENABLE_BINDINGS_PYTHON=ON", + f"-DPython3_EXECUTABLE={python}", + f"-DPython_EXECUTABLE={python}", + ] + llvm_build.mkdir(parents=True, exist_ok=True) + with open(llvm_build / ".bbdev.lock", "a+b") as lockf: + fcntl.flock(lockf.fileno(), fcntl.LOCK_EX) + if not (llvm_build / "build.ninja").is_file(): + _run( + cmake, + repo=root, + logger=logger, + task_scope=task_scope, + output_prefix="compiler llvm configure", + ) + _run( + ["ninja", "-C", str(llvm_build), "-j", str(os.cpu_count() or 1)], + repo=root, + logger=logger, + task_scope=task_scope, + output_prefix="compiler llvm build", + ) + mlir_cmake = llvm_build / "lib" / "cmake" / "mlir" + if not mlir_cmake.is_dir(): + raise RuntimeError(f"LLVM/MLIR build failed: missing {mlir_cmake}") + return llvm_build + + +def build_buddy( + repo: str | Path, *, logger: object | None = None, task_scope: str | None = None +) -> Path: + """The one buddy-mlir build every stack's compiler plugin is loaded into.""" + root = Path(repo).resolve() + buddy = root / "stack/thirdparty/buddy-mlir" + llvm = build_llvm(root, logger=logger, task_scope=task_scope) + python = compiler_python(root) + build = buddy / "build" + cmake = [ + "cmake", + "-G", + "Ninja", + "-S", + str(buddy), + "-B", + str(build), + f"-DMLIR_DIR={llvm / 'lib' / 'cmake' / 'mlir'}", + f"-DLLVM_DIR={llvm / 'lib' / 'cmake' / 'llvm'}", + "-DLLVM_ENABLE_ASSERTIONS=ON", + "-DCMAKE_BUILD_TYPE=RELEASE", + "-DBUDDY_MLIR_ENABLE_PYTHON_PACKAGES=ON", + f"-DPython3_EXECUTABLE={python}", + f"-DPython_EXECUTABLE={python}", + ] + with open(buddy / ".bbdev.lock", "a+b") as lockf: + fcntl.flock(lockf.fileno(), fcntl.LOCK_EX) + _run( + cmake, + repo=root, + logger=logger, + task_scope=task_scope, + output_prefix="buddy-mlir configure", + ) + _run( + [ + "ninja", + "-C", + str(build), + "-j", + str(os.cpu_count() or 1), + "buddy-opt", + "buddy-translate", + "buddy-llc", + "rax-pack", + "python-package-buddy", + "BuddyMLIRPythonModules", + ], + repo=root, + logger=logger, + task_scope=task_scope, + output_prefix="buddy-mlir build", + ) + return build + + +def _design(repo: Path, design: str): + sys.path.insert(0, str(repo)) + from stack.scripts import design as design_config + + return design_config, design_config.load( + repo / "examples/designs" / design / "output/generated/design.pb" + ) + + +def build_design( + repo: str | Path, + *, + design: str, + selection: str, + logger: object | None = None, + task_scope: str | None = None, +) -> None: + """Build the software stack of one simulation entry of `design`. + + sims//stack/CMakeLists.txt is the top of that build: it lists the + thread, core and device stacks the entry needs and builds each of them. + """ + root = Path(repo).resolve() + design_config, config = _design(root, design) + if selection not in config.simulations: + raise ValueError(f"{design}: unregistered simulation entry {selection}") + build = design_config.stack_build(config, selection) + design_config.generate(config, build / "generated", selection) + build_buddy(root, logger=logger, task_scope=task_scope) + _run( + [ + "cmake", + "-G", + "Ninja", + "-S", + config.simulations[selection].stack, + "-B", + str(build), + f"-DDESIGN_CMAKE={build / 'generated/design.cmake'}", + ], + repo=root, + logger=logger, + task_scope=task_scope, + output_prefix=f"stack {selection} configure", + ) + _run( + ["ninja", "-C", str(build), "-j", str(os.cpu_count() or 1)], + repo=root, + logger=logger, + task_scope=task_scope, + output_prefix=f"stack {selection} build", + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build the stacks of a design") + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--design", required=True) + parser.add_argument("--target", required=True) + args = parser.parse_args() + build_design(args.repo, design=args.design, selection=args.target) + + +if __name__ == "__main__": + main() diff --git a/bbdev/api/steps/uvm/00_waive_event.step.py b/bbdev/api/steps/uvm/00_waive_event.step.py index 90538afc1..acd1cc751 100644 --- a/bbdev/api/steps/uvm/00_waive_event.step.py +++ b/bbdev/api/steps/uvm/00_waive_event.step.py @@ -13,7 +13,7 @@ sys.path.insert(0, step_dir) from scripts.waive import apply_waivers -from utils.event_common import check_result, get_origin_trace_id, require_chip +from utils.event_common import check_result, get_origin_trace_id, require_design from utils.path import get_buckyball_path, rtl_dir config = { @@ -28,16 +28,38 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) try: - chip = require_chip(input_data) + design = require_design(input_data) task = input_data["uvm_task"] if task not in ("build", "run"): raise ValueError(f"invalid UVM task: {task}") bbdir = get_buckyball_path() - count = await asyncio.to_thread(apply_waivers, Path(rtl_dir(bbdir, chip, "tapeout"))) + count = await asyncio.to_thread( + apply_waivers, + Path(rtl_dir(bbdir, design, "tapeout", selection=input_data["target"])), + ) except (KeyError, ValueError) as error: - await check_result(ctx, 1, extra_fields={"task": "waive", "error": str(error)}, trace_id=origin_tid) + await check_result( + ctx, + 1, + extra_fields={"task": "waive", "error": str(error)}, + trace_id=origin_tid, + ) return - await check_result(ctx, 0, continue_run=True, extra_fields={"task": "waive", "count": count}, trace_id=origin_tid) + await check_result( + ctx, + 0, + continue_run=True, + extra_fields={"task": "waive", "count": count}, + trace_id=origin_tid, + ) await ctx.enqueue( - {"topic": f"uvm.{task}", "data": {**input_data, "force_rebuild": True, "_trace_id": origin_tid}} + { + "topic": f"uvm.{task}", + "data": { + **input_data, + "force_rebuild": True, + "_trace_id": origin_tid, + "target": input_data["target"], + }, + } ) diff --git a/bbdev/api/steps/uvm/01_build_api.step.py b/bbdev/api/steps/uvm/01_build_api.step.py index 123e9f793..bf6331295 100644 --- a/bbdev/api/steps/uvm/01_build_api.step.py +++ b/bbdev/api/steps/uvm/01_build_api.step.py @@ -1,3 +1,4 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api config = { @@ -10,31 +11,37 @@ def check_args(body: dict) -> str | None: - allowed = {"chip", "ball", "ip", "target"} + allowed = {"design", "device", "ball", "ip"} for key in body: if key not in allowed: return f"Unexpected parameter: --{key}" - if not body.get("chip") or body.get("chip") is True: - return "Missing required parameter: --chip" + if body.get("design") is True or ("design" in body and not body["design"]): + return "Parameter --design requires a value" + if not body.get("design") and not isinstance(body.get("ip"), str): + return "Standalone UVM requires --ip" if body.get("ball") is True: return "Parameter --ball requires a value" if body.get("ip") is True: return "Parameter --ip requires a value" if body.get("ball") and body.get("ip"): return "Parameters --ball and --ip are mutually exclusive" - if "target" in body: - if not isinstance(body["target"], str) or not body["target"].strip(): - return "Parameter --target requires a value" - if not body.get("ip"): - return "Parameter --target requires --ip" return None async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + try: + selection = require_target(body, ("device", "ip", "ball")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) error = check_args(body) if error: return ApiResponse(status=400, body={"error": error}) - await ctx.enqueue({"topic": "uvm.build", "data": {**body, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "uvm.build", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/uvm/01_build_event.step.py b/bbdev/api/steps/uvm/01_build_event.step.py index ce319c645..f3926c399 100644 --- a/bbdev/api/steps/uvm/01_build_event.step.py +++ b/bbdev/api/steps/uvm/01_build_event.step.py @@ -13,11 +13,12 @@ from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path -from scripts.uvm_common import run_uvm +from scripts.run import run_verification +from scripts.ball import run_design config = { "name": "uvm-build", - "description": "Build a Ball UVM simulation", + "description": "Build a UVM simulation", "flows": ["uvm"], "triggers": [queue("uvm.build")], "enqueues": [], @@ -29,16 +30,19 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bbdir = get_buckyball_path() try: - info = await asyncio.to_thread( - run_uvm, - bbdir, - input_data["chip"], - input_data.get("ball"), - input_data.get("ip"), - ctx, - False, - input_data.get("target"), - ) + if input_data.get("ball") is not None: + info = await asyncio.to_thread( + run_design, bbdir, input_data["design"], input_data["ball"], ctx, False + ) + else: + info = await asyncio.to_thread( + run_verification, + bbdir, + input_data.get("design"), + input_data.get("ip"), + ctx, + False, + ) except Exception as e: ctx.logger.error(str(e)) await check_result( @@ -52,7 +56,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result( ctx, - 0, + 1 if info.get("failures") else 0, continue_run=False, extra_fields={"task": "build", **info}, trace_id=origin_tid, diff --git a/bbdev/api/steps/uvm/02_run_api.step.py b/bbdev/api/steps/uvm/02_run_api.step.py index 3532fde4f..922f65939 100644 --- a/bbdev/api/steps/uvm/02_run_api.step.py +++ b/bbdev/api/steps/uvm/02_run_api.step.py @@ -1,3 +1,4 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api config = { @@ -10,31 +11,37 @@ def check_args(body: dict) -> str | None: - allowed = {"chip", "ball", "ip", "target"} + allowed = {"design", "device", "ball", "ip"} for key in body: if key not in allowed: return f"Unexpected parameter: --{key}" - if not body.get("chip") or body.get("chip") is True: - return "Missing required parameter: --chip" + if body.get("design") is True or ("design" in body and not body["design"]): + return "Parameter --design requires a value" + if not body.get("design") and not isinstance(body.get("ip"), str): + return "Standalone UVM requires --ip" if body.get("ball") is True: return "Parameter --ball requires a value" if body.get("ip") is True: return "Parameter --ip requires a value" if body.get("ball") and body.get("ip"): return "Parameters --ball and --ip are mutually exclusive" - if "target" in body: - if not isinstance(body["target"], str) or not body["target"].strip(): - return "Parameter --target requires a value" - if not body.get("ip"): - return "Parameter --target requires --ip" return None async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + try: + selection = require_target(body, ("device", "ip", "ball")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) error = check_args(body) if error: return ApiResponse(status=400, body={"error": error}) - await ctx.enqueue({"topic": "uvm.run", "data": {**body, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "uvm.run", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/uvm/02_run_event.step.py b/bbdev/api/steps/uvm/02_run_event.step.py index a1e24a397..beb9a08ec 100644 --- a/bbdev/api/steps/uvm/02_run_event.step.py +++ b/bbdev/api/steps/uvm/02_run_event.step.py @@ -13,15 +13,12 @@ from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path -from scripts.uvm_common import run_uvm -from steps.uvm.scripts.report import coverage_report, verification_targets -from utils.reports import source_context -from utils.path import log_dir -from datetime import datetime +from scripts.run import run_verification +from scripts.ball import run_design config = { "name": "uvm-run", - "description": "Build and run a Ball UVM simulation", + "description": "Build and run a UVM simulation", "flows": ["uvm"], "triggers": [queue("uvm.run")], "enqueues": [], @@ -31,28 +28,21 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - report_context = source_context(bbdir, input_data["chip"]) try: - info = await asyncio.to_thread( - run_uvm, - bbdir, - input_data["chip"], - input_data.get("ball"), - input_data.get("ip"), - ctx, - True, - input_data.get("target"), - ) - if "results" not in info: - info = {"chip": info["chip"], "results": [info], "failures": []} - stamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") - directory = log_dir(bbdir, input_data["chip"], "verilog", stamp, "uvm", f"report-{origin_tid}") - expected = verification_targets( - bbdir, input_data["chip"], input_data.get("ball"), - input_data.get("ip"), input_data.get("target"), - ) - info["report_path"] = coverage_report(report_context, origin_tid, info, directory, expected) + if input_data.get("ball") is not None: + info = await asyncio.to_thread( + run_design, bbdir, input_data["design"], input_data["ball"], ctx, True + ) + else: + info = await asyncio.to_thread( + run_verification, + bbdir, + input_data.get("design"), + input_data.get("ip"), + ctx, + True, + ) except Exception as e: ctx.logger.error(str(e)) await check_result( @@ -66,7 +56,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result( ctx, - 1 if info["failures"] else 0, + 1 if info.get("failures") else 0, continue_run=False, extra_fields={"task": "run", **info}, trace_id=origin_tid, diff --git a/bbdev/api/steps/uvm/scripts/ball.py b/bbdev/api/steps/uvm/scripts/ball.py new file mode 100644 index 000000000..3ee9fe85b --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/ball.py @@ -0,0 +1,377 @@ +import os +import re +import shlex +import sys +import tomllib +from datetime import datetime +from pathlib import Path +from utils.path import log_dir +from steps.mill.scripts.mill import mill_cmd +from utils.stream_run import stream_run_logger + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "config/scripts")) + + +def load_design(bbdir: str, design: str): + import design_pb2 + + path = ( + Path(bbdir) + / "examples" + / "designs" + / design + / "output" + / "generated" + / "design.pb" + ) + if not path.is_file(): + raise FileNotFoundError(f"missing {path}; run bbdev config --install") + msg = design_pb2.Design() + msg.ParseFromString(path.read_bytes()) + if not msg.name or not msg.device.cores: + raise ValueError(f"empty design.pb: {path}") + return msg + + +def ball_domain(design, ball: str | None = None): + domains = [ + core.accelerator.balldomain + for core in design.device.cores + if core.HasField("accelerator") + ] + domains.extend(thread.balldomain for thread in design.threads) + domains = [domain for domain in domains if domain.balls] + if not domains: + raise ValueError("design.pb has no BallDomain instances") + if ball is None: + domain = domains[0] + if any(other.balls != domain.balls for other in domains[1:]): + raise ValueError( + "design.pb has different BallDomains; select a Ball explicitly" + ) + return domain + matches = [ + (domain, mapping) + for domain in domains + for mapping in domain.balls + if mapping.ball_dir == ball + ] + if not matches: + raise ValueError(f"ball {ball!r} not in design.pb") + domain, mapping = matches[0] + if any(other != mapping for _, other in matches[1:]): + raise ValueError(f"ball {ball!r} has different instance definitions") + return domain + + +def selected_mappings(domain, ball: str | None): + if ball is None: + return list(domain.balls) + for m in domain.balls: + if m.ball_dir == ball: + return [m] + raise ValueError(f"ball {ball!r} not in design.pb") + + +def ball_owner(design, ball): + owners = [] + for core in design.device.cores: + accelerators = [core.accelerator] if core.HasField("accelerator") else [] + accelerators.extend(design.threads[index] for index in core.thread_indices) + owners.extend( + (acc, core) + for acc in accelerators + if any(item.ball_dir == ball for item in acc.balldomain.balls) + ) + if not owners: + raise ValueError(f"ball {ball!r} not in design.pb") + accelerator, core = owners[0] + for other, owner in owners[1:]: + if ( + other.mem.bank != accelerator.mem.bank + or other.frontend.rob_entries != accelerator.frontend.rob_entries + or other.frontend.sub_rob_depth != accelerator.frontend.sub_rob_depth + or owner.shared_mem != core.shared_mem + or owner.virtual_bank_count != core.virtual_bank_count + ): + raise ValueError( + f"ball {ball!r} has different verification interfaces across instances" + ) + return accelerator, core + + +def vcs_defines(domain, mapping, thread, core, parameters): + shared_bank_num = ( + core.shared_mem.entries // thread.mem.bank.entries + if core.shared_mem.enable + else 0 + ) + max_group_count = max(thread.mem.bank.num, shared_bank_num) + defs = [ + f"+define+BB_IN_BW={mapping.in_bw}", + f"+define+BB_OUT_BW={mapping.out_bw}", + f"+define+BB_MMIO_READ_BW={mapping.mmio_read_bw}", + f"+define+BB_MMIO_WRITE_BW={mapping.mmio_write_bw}", + f"+define+BB_BANK_ID_W={(core.virtual_bank_count - 1).bit_length()}", + f"+define+BB_GROUP_COUNT_W={max_group_count.bit_length()}", + f"+define+BB_GROUP_ID_W={(max_group_count - 1).bit_length()}", + f"+define+BB_BANK_ADDR_W={(thread.mem.bank.entries - 1).bit_length()}", + f"+define+BB_ROB_ID_W={(thread.frontend.rob_entries - 1).bit_length()}", + f"+define+BB_SUB_ROB_ID_W={(thread.frontend.sub_rob_depth * 4 - 1).bit_length()}", + ] + for e in mapping.instructions: + defs.append(f"+define+{e.mnemonic}_FUNCT7={e.funct7}") + from ball_parameters import unpack_parameters + + typed = unpack_parameters(mapping) + for define, parameter in parameters.items(): + defs.append(f"+define+{define}={getattr(typed, parameter)}") + return defs + + +def _filelist( + verify_dir: Path, + ball_dir: str, + uvm_rel: str, + rtl_dir: Path, + sim_dir: Path, +) -> str: + src = verify_dir / "filelists" / f"{ball_dir}_ball.f" + dst = sim_dir / f"{ball_dir}_ball.f" + sim_dir.mkdir(parents=True, exist_ok=True) + text = src.read_text() + dst.write_text( + text.replace("@UVM@", uvm_rel).replace("@RTL@", str(rtl_dir.resolve())) + ) + return str(dst) + + +def check_uvm_result( + result, target: str, expected_assertion: str | None = None +) -> None: + output = result.stdout + result.stderr + if expected_assertion is not None: + assertions = re.findall(r"Assertion failed: ([^\r\n]+)", output) + failure_counts = re.findall(r"UVM_(?:ERROR|FATAL)\s*:\s*(\d+)", output) + if ( + not assertions + or set(assertions) != {expected_assertion} + or "Fatal:" not in output + or any(int(count) for count in failure_counts) + or result.returncode < 0 + ): + raise RuntimeError( + f"Expected RTL assertion did not terminate {target}: {expected_assertion}" + ) + return + if result.returncode != 0: + raise RuntimeError(f"UVM failed for {target}") + counts = dict(re.findall(r"UVM_(ERROR|FATAL)\s*:\s*(\d+)", output)) + if set(counts) != {"ERROR", "FATAL"}: + raise RuntimeError(f"UVM report summary missing for {target}") + if counts != {"ERROR": "0", "FATAL": "0"}: + raise RuntimeError( + f"UVM failed for {target}: errors={counts['ERROR']} fatals={counts['FATAL']}" + ) + + +def build_ball( + bbdir: str, design_name: str, mill_cfg: str, domain, mapping, ctx +) -> None: + ball = mapping.ball_dir + verify_dir = Path(bbdir) / "examples" / "components" / "balls" / ball / "verify" + casegen = verify_dir / "casegen" / "Cargo.toml" + rtl_dir = ( + Path(bbdir) + / "examples/designs" + / design_name + / "output/sims/device/arch" + / mill_cfg + ) + design = load_design(bbdir, design_name) + thread, core = ball_owner(design, ball) + sim_dir = verify_dir.parent / "output" / "uvm" / design_name + uvm_rel = os.path.relpath(Path(bbdir) / "verify" / "uvm", verify_dir) + flist = _filelist(verify_dir, ball, uvm_rel, rtl_dir, sim_dir) + parameters_path = verify_dir / "parameters.toml" + parameters = ( + tomllib.loads(parameters_path.read_text())["defines"] + if parameters_path.is_file() + else {} + ) + cargo = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"cargo build --manifest-path {shlex.quote(str(casegen))} " + f"--target-dir {shlex.quote(str(verify_dir.parent / 'output' / 'casegen'))}" + ) + r = stream_run_logger( + cmd=cargo, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm dpi", + stderr_prefix="uvm dpi", + ) + if r.returncode != 0: + raise RuntimeError(f"cargo build failed for {ball}") + crate = tomllib.loads(casegen.read_text()).get("package", {}).get("name") + if not crate: + raise ValueError(f"package.name missing in {casegen}") + simv = sim_dir / "simv" + csrc = sim_dir / "csrc" + hier = sim_dir / "cm_hier.cfg" + sim_dir.mkdir(parents=True, exist_ok=True) + hier.write_text("+tree tb_top.dut\n") + script = ( + f"cd {shlex.quote(str(verify_dir))} && " + f"rm -rf {shlex.quote(str(csrc))} {shlex.quote(str(simv))} {shlex.quote(str(simv))}.daidir && " + f"mkdir -p {shlex.quote(str(sim_dir))} {shlex.quote(str(csrc))} && " + "vcs -full64 -sverilog -timescale=1ns/1ps -debug_access+all -top tb_top " + "${=VCS_UVM_ARGS} " + + " ".join( + shlex.quote(d) + for d in vcs_defines(domain, mapping, thread, core, parameters) + ) + + f" -cm line+cond+tgl+assert -cm_hier {shlex.quote(str(hier))} " + f"-Mdir={shlex.quote(str(csrc))} -o {shlex.quote(str(simv))} " + f"-f {shlex.quote(flist)}" + ) + vcs = f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command zsh -c {shlex.quote(script)}" + r = stream_run_logger( + cmd=vcs, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm vcs", + stderr_prefix="uvm vcs", + ) + if r.returncode != 0: + raise RuntimeError(f"vcs failed for {ball}") + + +def run_ball( + bbdir: str, design_name: str, mill_cfg: str, domain, mapping, ctx, cov_dir: str +) -> None: + ball = mapping.ball_dir + verify_dir = Path(bbdir) / "examples" / "components" / "balls" / ball / "verify" + simv = verify_dir.parent / "output" / "uvm" / design_name / "simv" + build_ball(bbdir, design_name, mill_cfg, domain, mapping, ctx) + crate = tomllib.loads((verify_dir / "casegen" / "Cargo.toml").read_text())[ + "package" + ]["name"] + dpi = ( + verify_dir.parent + / "output" + / "casegen" + / "debug" + / f"lib{crate.replace('-', '_')}" + ) + test = f"{ball}_ball_test" + verify_config = ( + verify_dir.parent / "output" / "uvm" / design_name / "verify_config.env" + ) + design = load_design(bbdir, design_name) + thread, _ = ball_owner(design, ball) + verify_config.write_text( + f"bank_entries={thread.mem.bank.entries}\nball_id={mapping.ball_id}\n" + ) + simv_q = shlex.quote(str(simv)) + script = ( + f"cd {shlex.quote(str(verify_dir))} && " + f'loader="$(patchelf --print-interpreter {simv_q})"; ' + f'library_path="$(patchelf --print-rpath {simv_q})"; ' + f"BB_VERIFY_CONFIG={shlex.quote(str(verify_config))} " + f'"$loader" --library-path "$library_path" {simv_q} -no_save ' + f"-sv_lib {shlex.quote(str(dpi))} " + f"+UVM_TESTNAME={shlex.quote(test)} +BID={mapping.ball_id} " + f"-cm line+cond+tgl+assert -cm_name {shlex.quote(test)}" + ) + cmd = f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command zsh -ic {shlex.quote(script)}" + r = stream_run_logger( + cmd=cmd, + logger=ctx.logger, + cwd=bbdir, + stdout_prefix="uvm run", + stderr_prefix="uvm run", + ) + check_uvm_result(r, f"{ball} test={test}") + Path(cov_dir).mkdir(parents=True, exist_ok=True) + urg = ( + f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " + f"urg -dir {shlex.quote(str(simv))}.vdb -format text -report {shlex.quote(cov_dir)}" + ) + r = stream_run_logger( + cmd=urg, + logger=ctx.logger, + cwd=str(verify_dir), + stdout_prefix="uvm urg", + stderr_prefix="uvm urg", + ) + if r.returncode != 0: + raise RuntimeError(f"urg failed for {ball}") + + +def dashboard_summary(cov_dir: str) -> dict[str, str]: + path = Path(cov_dir) / "dashboard.txt" + lines = path.read_text().splitlines() + i = 0 + while i < len(lines) and "Total Coverage Summary" not in lines[i]: + i += 1 + if i >= len(lines): + raise RuntimeError(f"Total Coverage Summary missing in {path}") + i += 1 + while i < len(lines) and not lines[i].strip().startswith("SCORE"): + i += 1 + if i + 1 >= len(lines): + raise RuntimeError(f"SCORE row missing in {path}") + keys = lines[i].split() + vals = lines[i + 1].split() + if len(vals) != len(keys): + raise RuntimeError(f"SCORE/value mismatch in {path}: {keys!r} vs {vals!r}") + return dict(zip(keys, vals)) + + +def run_design(bbdir: str, design: str, ball: str | None, ctx, do_run: bool) -> dict: + msg = load_design(bbdir, design) + domain = ball_domain(msg, ball) + mill_cfg = msg.simulations["device"].top + if not mill_cfg: + raise ValueError(f"design {design!r} has no sims.verilator for UVM") + maps = selected_mappings(domain, ball) + ran = [] + covs = [] + run_root = None + if do_run: + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") + run_name = "all" if ball is None else maps[0].ball_dir + run_root = log_dir(bbdir, design, "verilog", stamp, "uvm", run_name) + for m in maps: + if do_run: + cov = ( + os.path.join(run_root, m.ball_dir, "coverage") + if ball is None + else os.path.join(run_root, "coverage") + ) + run_ball(bbdir, design, mill_cfg, domain, m, ctx, cov) + if not (Path(cov) / "dashboard.txt").is_file(): + raise RuntimeError(f"urg wrote no dashboard.txt under {cov}") + covs.append((m.ball_dir, cov)) + else: + build_ball(bbdir, design, mill_cfg, domain, m, ctx) + ran.append(m.ball_dir) + info = {"design": design, "mill": mill_cfg, "balls": ran} + if do_run and ball is None: + index_dir = Path(run_root) / "coverage" + index_dir.mkdir(parents=True, exist_ok=True) + body = ["ball_dir score line cond toggle assert group result"] + for b, p in covs: + s = dashboard_summary(p) + body.append( + f"{b} {s['SCORE']} {s['LINE']} {s['COND']} {s['TOGGLE']} " + f"{s.get('ASSERT', '--')} {s.get('GROUP', '--')} pass" + ) + index = index_dir / "index.txt" + index.write_text("\n".join(body) + "\n") + info["index"] = str(index) + info["log"] = run_root + elif do_run: + info["log"] = run_root + return info diff --git a/bbdev/api/steps/uvm/scripts/build.py b/bbdev/api/steps/uvm/scripts/build.py new file mode 100644 index 000000000..b6d90b93c --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/build.py @@ -0,0 +1,186 @@ +import fcntl +import json +import shlex +import shutil +from pathlib import Path + +from steps.mill.scripts.mill import mill_cmd +from utils.stream_run import stream_run_logger + + +class VerificationFailure(RuntimeError): + def __init__(self, stage, test, logfile): + self.stage, self.test, self.logfile = stage, test, str(logfile) + super().__init__(f"{stage} failed: {test}; see {logfile}") + + +def execute(repo, arguments, cwd, logfile, timeout, ctx): + logfile = Path(logfile) + logfile.parent.mkdir(parents=True, exist_ok=True) + command = [ + "nix", + "develop", + str(Path(repo) / "verify"), + "--command", + "timeout", + "--kill-after=5s", + f"{timeout}s", + *map(str, arguments), + ] + result = stream_run_logger( + command, + ctx.logger, + cwd=str(cwd), + shell=False, + timeout=timeout + 30, + stdout_prefix="verification", + stderr_prefix="verification", + ) + logfile.write_text(result.stdout + "\n" + result.stderr) + return result + + +def build(repo, design, profile, logdir, ctx): + repo = Path(repo) + root = repo / profile["root"] + scope = design if design is not None else "default" + output = root / "build/verification" / scope / profile["profile"] + output.mkdir(parents=True, exist_ok=True) + lock = (root / "build/.verification.lock").open("w") + with lock: + fcntl.flock(lock, fcntl.LOCK_EX) + model = profile["model"] + if model["manifest"]: + result = execute( + repo, + [ + "cargo", + "build", + "--manifest-path", + root / model["manifest"], + "--target-dir", + root / "build/verification/reference", + ], + root, + Path(logdir) / "reference.log", + 600, + ctx, + ) + if result.returncode: + raise VerificationFailure( + "reference", "", Path(logdir) / "reference.log" + ) + parameters = output / "parameters.json" + parameters.write_text(json.dumps(profile["parameters"])) + rtl = output / "rtl" + registration = profile["build"] + if registration["kind"] != "mill": + raise ValueError(f"Unknown emitter kind: {registration['kind']}") + replacements = { + "{repo}": str(repo), + "{design}": design, + "{config}": str(parameters), + "{rtl}": str(rtl), + "{targets}": ",".join(test["name"] for test in profile["tests"]), + } + arguments = [] + for argument in registration["arguments"]: + for token, value in replacements.items(): + if token in argument: + if value is None: + raise ValueError( + f"{profile['ip']} requires a design for {token}" + ) + argument = argument.replace(token, value) + arguments.append(argument) + if profile["parameters"] and not any( + "{config}" in value for value in registration["arguments"] + ): + raise ValueError(f"Emitter does not consume parameters: {profile['ip']}") + command = mill_cmd( + repo, + registration["directory"], + registration["module"] + ".run", + arguments, + ) + result = execute( + repo, + ["zsh", "-c", command], + repo / registration["directory"], + Path(logdir) / "emit.log", + 900, + ctx, + ) + if result.returncode: + raise VerificationFailure("emit", "", Path(logdir) / "emit.log") + if not any("{rtl}" in value for value in registration["arguments"]): + shutil.copytree( + root / "build", + rtl, + dirs_exist_ok=True, + ignore=shutil.ignore_patterns( + "verification", "uvm", ".verification.lock", ".build.lock" + ), + ) + resources = root / profile["resources"] + compiled = {} + binaries = {} + for test in profile["tests"]: + directory = output / test["name"] + directory.mkdir(parents=True, exist_ok=True) + source = resources / test["filelist"] + filelist = directory / "filelist.f" + filelist.write_text( + source.read_text() + .replace("@VERIFY@", str(repo / "verify")) + .replace("@RESOURCES@", str(source.parent)) + .replace("@RTL@", str(rtl)) + ) + signature = ( + filelist.read_text(), + test["top"], + tuple(test["metrics"]), + tuple(test["rtl_scopes"]), + ) + if signature in compiled: + binaries[test["name"]] = compiled[signature] + continue + hierarchy = directory / "coverage.cfg" + hierarchy.write_text( + "".join( + f"+tree {test['top']}.{scope}\n" for scope in test["rtl_scopes"] + ) + ) + metrics = "+".join([*test["metrics"], "assert"]) + command = [ + "uvm-compile", + "-top", + test["top"], + "-debug_access+all", + "-f", + filelist, + "-o", + directory / "simv", + "-Mdir=" + str(directory / "csrc"), + *[ + f"+define+{name}={value}" + for name, value in profile["defines"].items() + ], + ] + if test["metrics"]: + command.extend(["-cm", metrics, "-cm_hier", hierarchy]) + result = execute( + repo, + command, + directory, + Path(logdir) / test["name"] / "compile.log", + 600, + ctx, + ) + if result.returncode: + raise VerificationFailure( + "compile", test["name"], Path(logdir) / test["name"] / "compile.log" + ) + compiled[signature] = directory / "simv" + binaries[test["name"]] = directory / "simv" + return {"output": output, "binaries": binaries} diff --git a/bbdev/api/steps/uvm/scripts/config.py b/bbdev/api/steps/uvm/scripts/config.py new file mode 100644 index 000000000..248b574cb --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/config.py @@ -0,0 +1,50 @@ +import json +import sys +from pathlib import Path +from google.protobuf.json_format import MessageToDict + + +def profiles(repo, design, ip): + repo = Path(repo) + sys.path.insert(0, str(repo / "bbdev/api/steps/config/scripts")) + import design_pb2 + + if design is None: + from verification import default_ip_config + + path = default_ip_config(repo, ip) + else: + path = repo / "examples/designs" / design / "output/generated/design.pb" + config = design_pb2.Design.FromString(path.read_bytes()) + entries = decode_profiles(config) + selected = [entry for entry in entries if ip is None or entry["ip"] == ip] + if not selected: + raise ValueError( + f"No IP verification registrations for design={design!r} ip={ip!r}" + ) + result = [] + for entry in selected: + tests = [ + test + for test in entry["tests"] + if not test["designs"] or design in test["designs"] + ] + if not tests: + raise ValueError(f"No applicable tests: {entry['ip']}/{entry['profile']}") + result.append({**entry, "tests": tests}) + return result + + +def decode_profiles(config): + entries = [] + for descriptor in config.verification.profiles: + entry = MessageToDict( + descriptor, + preserving_proto_field_name=True, + always_print_fields_with_no_presence=True, + ) + entry["parameters"] = json.loads(entry.pop("parameters_json")) + for test in entry["tests"]: + test["timeout_ns"] = int(test["timeout_ns"]) + entries.append(entry) + return entries diff --git a/bbdev/api/steps/uvm/scripts/coverage.py b/bbdev/api/steps/uvm/scripts/coverage.py new file mode 100644 index 000000000..f249e9cb5 --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/coverage.py @@ -0,0 +1,75 @@ +import re +from pathlib import Path + +from .build import VerificationFailure, execute + + +def summary(directory): + lines = (Path(directory) / "dashboard.txt").read_text().splitlines() + start = lines.index("Total Coverage Summary ") + for index in range(start + 1, len(lines)): + if lines[index].strip().startswith("SCORE"): + return { + name.lower(): None if value == "--" else float(value) + for name, value in zip( + lines[index].split(), lines[index + 1].split(), strict=True + ) + } + raise ValueError(f"Missing coverage summary: {directory}") + + +def collect(repo, test, database, output, ctx): + output = Path(output) + hierarchy = output.parent / "dut.cfg" + hierarchy.write_text( + "".join(f"+tree {test['top']}.{scope}\n" for scope in test["rtl_scopes"]) + ) + commands = [ + ( + ["urg", "-dir", database, "-format", "text", "-report", output / "all"], + "all", + ), + ( + [ + "urg", + "-dir", + database, + "-format", + "text", + "-hier", + hierarchy, + "-metric", + "+".join(test["metrics"]), + "-report", + output / "dut", + ], + "dut", + ), + ] + for command, name in commands: + logfile = output.parent / f"coverage-{name}.log" + result = execute(repo, command, output.parent, logfile, 120, ctx) + if result.returncode: + raise VerificationFailure("coverage", test["name"], logfile) + values = summary(output / "dut") + for metric in test["metrics"]: + name = {"tgl": "toggle", "cond": "cond", "line": "line"}[metric] + if values[name] is None: + raise VerificationFailure( + "coverage_metric", test["name"], output / "dut/dashboard.txt" + ) + groups = {} + if test["functional_groups"]: + text = (output / "all/grpinfo.txt").read_text() + for name in test["functional_groups"]: + match = re.search( + r"Group : " + re.escape(name) + r"\s.*?\nSCORE[^\n]*\n\s*([\d.]+)", + text, + re.DOTALL, + ) + if match is None: + raise VerificationFailure( + "functional_coverage", test["name"], output / "all/groups.txt" + ) + groups[name] = float(match[1]) + return {"dut": values, "functional": groups, "report": str(output)} diff --git a/bbdev/api/steps/uvm/scripts/report.py b/bbdev/api/steps/uvm/scripts/report.py index 764a21c7d..cf09b7d50 100644 --- a/bbdev/api/steps/uvm/scripts/report.py +++ b/bbdev/api/steps/uvm/scripts/report.py @@ -1,25 +1,26 @@ -import tomllib from pathlib import Path from utils.reports import new_report, attachment, finish_report +from utils.path import load_design_config from .uvm_common import dashboard_summary, load_ip -def verification_targets(repo, chip, ball=None, ip=None, target=None): +def verification_targets(repo, design, ball=None, ip=None): if ip is not None: return [ f"ip/{ip}/{item['name']}" - for item in load_ip(str(repo), ip, target, chip)["targets"] + for profile in load_ip(str(repo), ip, design) + for item in profile["targets"] ] if ball is not None: return [f"ball/{ball}"] - config = tomllib.loads( - (Path(repo) / f"examples/chips/{chip}/configs/chip.toml").read_text() - )["uvm"] - targets = [f"ball/{ball}" for ball in config["balls"]] - for ip in config["ips"]: + config = load_design_config(repo, design).verification + targets = [f"ball/{ball}" for ball in config.balls] + for ip in config.ips: targets.extend( - f"ip/{ip}/{target['name']}" for target in load_ip(str(repo), ip, chip=chip)["targets"] + f"ip/{ip}/{target['name']}" + for profile in load_ip(str(repo), ip, design=design) + for target in profile["targets"] ) return targets @@ -48,7 +49,8 @@ def coverage_report(context, trace_id, info, output, expected_targets): report["verification_targets"].append(target) if ( result.get("expected_assertions") - and target.split("/")[-1] in result["expected_assertions"] + and target.removeprefix(f"ip/{result['ip']}/") + in result["expected_assertions"] ): report["verification"].append( { diff --git a/bbdev/api/steps/uvm/scripts/run.py b/bbdev/api/steps/uvm/scripts/run.py new file mode 100644 index 000000000..6fb3dcfc2 --- /dev/null +++ b/bbdev/api/steps/uvm/scripts/run.py @@ -0,0 +1,210 @@ +import json +import re +import shutil +from datetime import datetime +from pathlib import Path + +from utils.path import log_dir +from .build import build, execute, VerificationFailure +from .config import profiles +from .coverage import collect + + +def judge(test, process, result_file): + text = process.stdout + "\n" + process.stderr + logfile = result_file.parent / "simulation.log" + if process.returncode in (124, 137): + raise VerificationFailure("wall_timeout", test["name"], logfile) + if "[PH_TIMEOUT]" in text: + raise VerificationFailure("simulation_timeout", test["name"], logfile) + severities = dict(re.findall(r"UVM_(ERROR|FATAL)\s*:\s*(\d+)", text)) + if test["kind"] == "assertion": + assertions = re.findall(r"Assertion failed: ([^\r\n]+)", text) + passed = ( + bool(assertions) + and set(assertions) == {test["assertion"]} + and "Fatal:" in text + ) + passed = passed and not any(int(value) for value in severities.values()) + if not passed: + raise VerificationFailure("expected_assertion", test["name"], logfile) + return {"status": "pass", "checks": 1} + if process.returncode: + raise VerificationFailure("simulation", test["name"], logfile) + if any(int(value) for value in severities.values()): + raise VerificationFailure("checking", test["name"], logfile) + if not result_file.is_file(): + raise VerificationFailure("result_missing", test["name"], logfile) + data = json.loads(result_file.read_text()) + required = { + "status", + "comparisons", + "checks", + "expected", + "received", + "cancelled", + "pending", + } + if set(data) != required: + raise ValueError(f"Invalid verification evidence: {result_file}") + passed = data["status"] == "pass" and data["comparisons"] + data["checks"] > 0 + passed = ( + passed + and data["pending"] == 0 + and data["expected"] == data["comparisons"] + data["cancelled"] + ) + passed = ( + passed + and data["received"] == data["comparisons"] + and severities == {"ERROR": "0", "FATAL": "0"} + ) + passed = passed and not re.search( + r"^(?:Error|Fatal):|^UVM_(?:ERROR|FATAL)\s+(?!:)", text, re.MULTILINE + ) + if not passed: + raise VerificationFailure("checking", test["name"], logfile) + return data + + +def run_profile(repo, design, profile, directory, do_run, ctx): + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + (directory / "configuration.json").write_text(json.dumps(profile, indent=2) + "\n") + artifacts = build(repo, design, profile, directory, ctx) + output = artifacts["output"] + records = [] + if not do_run: + return { + "ip": profile["ip"], + "profile": profile["profile"], + "status": "built", + "output": str(output), + } + for test in profile["tests"]: + record_dir = directory / test["name"] + record_dir.mkdir(parents=True, exist_ok=True) + evidence = record_dir / "evidence.json" + if evidence.exists(): + raise FileExistsError(evidence) + binary = artifacts["binaries"][test["name"]] + if test["metrics"]: + shutil.copytree(Path(str(binary) + ".vdb"), record_dir / "simulation.vdb") + arguments = [ + "uvm-simulate", + binary, + "-no_save", + f"+UVM_TESTNAME={test['test']}", + f"+RESULT={evidence}", + f"+TIMEOUT_NS={test['timeout_ns']}", + f"+SINK_READY={test['sink_ready']}", + f"+SINK_STALL={test['sink_stall']}", + *test["plusargs"], + ] + if test["metrics"]: + arguments.extend( + [ + "-cm", + "+".join([*test["metrics"], "assert"]), + "-cm_dir", + record_dir / "simulation.vdb", + ] + ) + if profile["model"]["manifest"]: + arguments.extend( + [ + "-sv_lib", + Path(repo) + / profile["root"] + / "build/verification/reference/debug" + / profile["model"]["library"], + ] + ) + process = execute( + repo, + arguments, + binary.parent, + record_dir / "simulation.log", + test["wall_timeout_s"], + ctx, + ) + verdict = judge(test, process, evidence) + record = { + "ip": profile["ip"], + "profile": profile["profile"], + "test": test["name"], + "id": f"{profile['scope']}/{profile['ip']}/{profile['profile']}/{test['name']}", + "kind": test["kind"], + "configuration": profile["config_path"], + "log": str(record_dir / "simulation.log"), + **verdict, + "coverage": None, + } + if ( + verdict["status"] == "pass" + and test["kind"] == "functional" + and test["metrics"] + ): + # Runtime databases belong to this run, never the reusable build directory. + record["coverage"] = collect( + repo, test, record_dir / "simulation.vdb", record_dir / "coverage", ctx + ) + (record_dir / "result.json").write_text(json.dumps(record, indent=2) + "\n") + records.append(record) + return { + "ip": profile["ip"], + "profile": profile["profile"], + "configuration": profile["config_path"], + "status": "pass", + "tests": records, + } + + +def run_verification(repo, design, ip, ctx, do_run): + selected = profiles(repo, design, ip) + return run_suite(repo, design, selected, ip, ctx, do_run) + + +def run_suite(repo, design, selected, name, ctx, do_run): + scope = design if design is not None else selected[0]["scope"] + stamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S-%f") + directory = Path( + log_dir( + repo, scope, "verilog", stamp, "uvm", name if name is not None else "suite" + ) + ) + directory.mkdir(parents=True) + records = [] + for profile in selected: + records.append( + run_profile( + repo, + design, + profile, + directory / profile["ip"] / profile["profile"], + do_run, + ctx, + ) + ) + planned = [ + f"{profile['scope']}/{profile['ip']}/{profile['profile']}/{test['name']}" + for profile in selected + for test in profile["tests"] + ] + report = { + "design": design, + "operation": "run" if do_run else "build", + "planned_tests": planned, + "status": "pass", + "profiles": records, + "log": str(directory), + } + path = directory / "result.json" + path.write_text(json.dumps(report, indent=2) + "\n") + return { + "design": design, + "success": True, + "results": records, + "failures": [], + "report_path": str(path), + "log": str(directory), + } diff --git a/bbdev/api/steps/uvm/scripts/test_report.py b/bbdev/api/steps/uvm/scripts/test_report.py index 17daa1657..bbc7b8864 100644 --- a/bbdev/api/steps/uvm/scripts/test_report.py +++ b/bbdev/api/steps/uvm/scripts/test_report.py @@ -1,13 +1,28 @@ from pathlib import Path -def discover_tests(bbdir: str, chip: str, ball: str) -> list[Path]: - root = Path(bbdir) / "bb-tests" / "output" / chip / "workloads" / "src" / "CTest" / "chips" / chip / "balls" / chip / ball / "ctests" +def discover_tests(bbdir: str, design: str, ball: str) -> list[Path]: + root = ( + Path(bbdir) + / "bb-tests" + / "output" + / design + / "workloads" + / "src" + / "CTest" + / "designs" + / design + / "balls" + / design + / ball + / "ctests" + ) return sorted(root.glob("*-baremetal")) def write_test_index(path: Path, rows: list[tuple[str, str, str]]) -> None: path.write_text( - "ball_dir bemu verilator\n" + "\n".join(f"{ball} {bemu} {verilator}" for ball, bemu, verilator in rows) + "\n" + "ball_dir bemu verilator\n" + + "\n".join(f"{ball} {bemu} {verilator}" for ball, bemu, verilator in rows) + + "\n" ) - diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index aa73346b1..fd504c971 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -1,3 +1,5 @@ +import json +import shutil import fcntl import os import re @@ -12,6 +14,7 @@ sys.path.insert(0, utils_path) from utils.path import get_buckyball_path, log_dir +from steps.mill.scripts.mill import mill_cmd from utils.stream_run import stream_run_logger config_scripts = os.path.abspath( @@ -20,90 +23,130 @@ sys.path.insert(0, config_scripts) -def load_chip(bbdir: str, chip: str): - import chip_pb2 +def load_design(bbdir: str, design: str): + import design_pb2 path = ( - Path(bbdir) / "examples" / "chips" / chip / "configs" / "generated" / "chip.pb" + Path(bbdir) + / "examples" + / "designs" + / design + / "output" + / "generated" + / "design.pb" ) if not path.is_file(): raise FileNotFoundError(f"missing {path}; run bbdev config --install") - msg = chip_pb2.Chip() + msg = design_pb2.Design() msg.ParseFromString(path.read_bytes()) - if not msg.name or not msg.cores: - raise ValueError(f"empty chip.pb: {path}") + if not msg.name or not msg.device.cores: + raise ValueError(f"empty design.pb: {path}") return msg -def load_chip_uvm(bbdir: str, chip: str) -> dict[str, list[str]]: - path = Path(bbdir) / "examples" / "chips" / chip / "configs" / "chip.toml" - config = tomllib.loads(path.read_text())["uvm"] - if set(config) != {"balls", "ips"}: - raise ValueError(f"{path}: [uvm] must define exactly balls and ips") +def load_design_uvm(bbdir: str, design: str) -> dict[str, list[str]]: + from utils.path import load_design_config + + msg = load_design_config(bbdir, design) + config = {"balls": list(msg.verification.balls), "ips": list(msg.verification.ips)} if not config["balls"] and not config["ips"]: - raise ValueError(f"{path}: [uvm] has no targets") + raise ValueError(f"design {design}: protobuf has no UVM targets") return config -def ball_domain(chip, ball: str | None = None): +def ball_domain(design, ball: str | None = None): + domains = [ + core.accelerator.balldomain + for core in design.device.cores + if core.HasField("accelerator") + ] + domains.extend(thread.balldomain for thread in design.threads) + domains = [domain for domain in domains if domain.balls] + if not domains: + raise ValueError("design.pb has no BallDomain instances") if ball is None: - domains = [core.balldomain for core in chip.cores] domain = domains[0] - for other in domains[1:]: - if other.mappings != domain.mappings or other.isa != domain.isa: - raise ValueError("chip.pb cores have different balldomains") + if any(other.balls != domain.balls for other in domains[1:]): + raise ValueError( + "design.pb has different BallDomains; select a Ball explicitly" + ) return domain - - matches = [] - for core in chip.cores: - mappings = [m for m in core.balldomain.mappings if m.ball_dir == ball] - if mappings: - mapping = mappings[0] - isa = [e for e in core.balldomain.isa if e.bid == mapping.ball_id] - matches.append((core.balldomain, mapping, isa)) + matches = [ + (domain, mapping) + for domain in domains + for mapping in domain.balls + if mapping.ball_dir == ball + ] if not matches: - raise ValueError(f"ball {ball!r} not in chip.pb") - - domain, mapping, isa = matches[0] - for _, other_mapping, other_isa in matches[1:]: - if other_mapping != mapping or other_isa != isa: - raise ValueError(f"ball {ball!r} has different definitions across cores") + raise ValueError(f"ball {ball!r} not in design.pb") + domain, mapping = matches[0] + if any(other != mapping for _, other in matches[1:]): + raise ValueError(f"ball {ball!r} has different instance definitions") return domain def selected_mappings(domain, ball: str | None): if ball is None: - return list(domain.mappings) - for m in domain.mappings: + return list(domain.balls) + for m in domain.balls: if m.ball_dir == ball: return [m] - raise ValueError(f"ball {ball!r} not in chip.pb") + raise ValueError(f"ball {ball!r} not in design.pb") + + +def ball_owner(design, ball): + owners = [] + for core in design.device.cores: + accelerators = [core.accelerator] if core.HasField("accelerator") else [] + accelerators.extend(design.threads[index] for index in core.thread_indices) + owners.extend( + (acc, core) + for acc in accelerators + if any(item.ball_dir == ball for item in acc.balldomain.balls) + ) + if not owners: + raise ValueError(f"ball {ball!r} not in design.pb") + accelerator, core = owners[0] + for other, owner in owners[1:]: + if ( + other.mem.bank != accelerator.mem.bank + or other.frontend.rob_entries != accelerator.frontend.rob_entries + or other.frontend.sub_rob_depth != accelerator.frontend.sub_rob_depth + or owner.shared_mem != core.shared_mem + or owner.virtual_bank_count != core.virtual_bank_count + ): + raise ValueError( + f"ball {ball!r} has different verification interfaces across instances" + ) + return accelerator, core -def vcs_defines(domain, mapping, core, tile, parameters): +def vcs_defines(domain, mapping, thread, core, parameters): shared_bank_num = ( - tile.shared_mem.entries // core.mem.bank.entries - if tile.shared_mem.enable + core.shared_mem.entries // thread.mem.bank.entries + if core.shared_mem.enable else 0 ) - max_group_count = max(core.mem.bank.num, shared_bank_num) + max_group_count = max(thread.mem.bank.num, shared_bank_num) defs = [ f"+define+BB_IN_BW={mapping.in_bw}", f"+define+BB_OUT_BW={mapping.out_bw}", f"+define+BB_MMIO_READ_BW={mapping.mmio_read_bw}", f"+define+BB_MMIO_WRITE_BW={mapping.mmio_write_bw}", - f"+define+BB_BANK_ID_W={(tile.virtual_bank_count - 1).bit_length()}", + f"+define+BB_BANK_ID_W={(core.virtual_bank_count - 1).bit_length()}", f"+define+BB_GROUP_COUNT_W={max_group_count.bit_length()}", f"+define+BB_GROUP_ID_W={(max_group_count - 1).bit_length()}", - f"+define+BB_BANK_ADDR_W={(core.mem.bank.entries - 1).bit_length()}", - f"+define+BB_ROB_ID_W={(core.frontend.rob_entries - 1).bit_length()}", - f"+define+BB_SUB_ROB_ID_W={(core.frontend.sub_rob_depth * 4 - 1).bit_length()}", + f"+define+BB_BANK_ADDR_W={(thread.mem.bank.entries - 1).bit_length()}", + f"+define+BB_ROB_ID_W={(thread.frontend.rob_entries - 1).bit_length()}", + f"+define+BB_SUB_ROB_ID_W={(thread.frontend.sub_rob_depth * 4 - 1).bit_length()}", ] - for e in domain.isa: - if e.bid == mapping.ball_id: - defs.append(f"+define+{e.mnemonic}_FUNCT7={e.funct7}") + for e in mapping.instructions: + defs.append(f"+define+{e.mnemonic}_FUNCT7={e.funct7}") + from ball_parameters import unpack_parameters + + typed = unpack_parameters(mapping) for define, parameter in parameters.items(): - defs.append(f"+define+{define}={mapping.ball_params[parameter]}") + defs.append(f"+define+{define}={getattr(typed, parameter)}") return defs @@ -118,85 +161,163 @@ def _filelist( dst = sim_dir / f"{ball_dir}_ball.f" sim_dir.mkdir(parents=True, exist_ok=True) text = src.read_text() - dst.write_text(text.replace("@UVM@", uvm_rel).replace("@RTL@", str(rtl_dir.resolve()))) - return str(dst.relative_to(verify_dir)) - - -def load_ip(bbdir: str, name: str, target_name: str | None = None, chip: str | None = None) -> dict: - registry = Path(bbdir) / "arch" / "uvm.toml" - registered = tomllib.loads(registry.read_text()) - if name not in registered["ips"]: - raise ValueError(f"IP {name!r} is not registered in {registry}") - root = Path(bbdir) / registered["paths"][name] - build = registered.get("build", {}).get(name, {}) - if set(build) - {"directory", "module", "arguments"}: + dst.write_text( + text.replace("@UVM@", uvm_rel).replace("@RTL@", str(rtl_dir.resolve())) + ) + return str(dst) + + +def load_ip(bbdir: str, name: str, design: str | None = None) -> list[dict]: + from verification import default_ip_config + + if design is not None: + msg = load_design(bbdir, design) + else: + import design_pb2 + + msg = design_pb2.Design.FromString(default_ip_config(bbdir, name).read_bytes()) + profiles = [ + profile + for profile in json.loads(msg.verification.ip_registry_json) + if profile["ip"] == name + ] + if not profiles: + raise ValueError(f"IP {name!r} has no UVM registrations in design {design!r}") + return [load_ip_profile(bbdir, name, design, profile) for profile in profiles] + + +def load_ip_profile(bbdir: str, name: str, design: str | None, profile: dict) -> dict: + root = Path(bbdir) / profile["definition"]["root"] + build = profile["definition"]["build"] + if set(build) - {"directory", "module", "arguments", "runtime"}: raise ValueError(f"Invalid IP build settings for {name!r}: {build}") directory = build.get("directory", str(root.relative_to(bbdir))) module = build.get("module", name) arguments = build.get("arguments", []) - if not isinstance(arguments, list) or not all(isinstance(value, str) for value in arguments): + if not isinstance(arguments, list) or not all( + isinstance(value, str) for value in arguments + ): raise ValueError(f"IP build arguments must be a list of strings: {name!r}") - if not isinstance(directory, str) or not directory or not isinstance(module, str) or not module: - raise ValueError(f"IP build directory and module must be nonempty strings: {name!r}") + if ( + not isinstance(directory, str) + or not directory + or not isinstance(module, str) + or not module + ): + raise ValueError( + f"IP build directory and module must be nonempty strings: {name!r}" + ) resources = root / "src" / "main" / "resources" - targets = [ - { - "name": filelist.stem, - "filelist": filelist.name, - "top": f"{filelist.stem}_tb", - "test": "protocol_test", - } - for filelist in sorted(resources.glob("*.f")) - ] + targets = [] + for filelist in sorted([*resources.glob("*.f"), *resources.glob("*/filelist.f")]): + grouped = filelist.parent != resources + organized = filelist.name == "filelist.f" + target = ( + filelist.parent.name if grouped else name if organized else filelist.stem + ) + targets.append( + { + "name": target, + "filelist": str(filelist.relative_to(resources)), + "top": f"{target}_tb", + "test": "protocol_test", + "exclusions": ( + str((filelist.parent / "coverage.el").relative_to(resources)) + if organized + else f"{target}.el" + ), + "hierarchy": ( + str((filelist.parent / "coverage.cm_hier").relative_to(resources)) + if organized + else f"{target}.cm_hier" + ), + } + ) for target in targets: - metadata = resources / f"{target['name']}.toml" + metadata = (resources / target["filelist"]).with_suffix(".toml") if metadata.is_file(): settings = tomllib.loads(metadata.read_text()) - if not settings or set(settings) - {"expected_assertion", "chips", "invalid_case", "rtl_scopes"}: + if not settings or set(settings) - { + "expected_assertion", + "designs", + "invalid_case", + "rtl_scopes", + }: raise ValueError(f"Invalid IP test settings: {metadata}") if "rtl_scopes" in settings: scopes = settings["rtl_scopes"] - if not isinstance(scopes, list) or not scopes or any( - not isinstance(scope, str) or not re.fullmatch( - r"[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?(?:\.[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?)*", scope - ) for scope in scopes + if ( + not isinstance(scopes, list) + or not scopes + or any( + not isinstance(scope, str) + or not re.fullmatch( + r"[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?(?:\.[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?)*", + scope, + ) + for scope in scopes + ) ): raise ValueError(f"Invalid RTL instance scopes: {metadata}") target["rtl_scopes"] = scopes if "invalid_case" in settings: case = settings["invalid_case"] - if type(case) is not int or case < 0 or "expected_assertion" not in settings: - raise ValueError(f"invalid_case requires a nonnegative integer and expected_assertion: {metadata}") + if ( + type(case) is not int + or case < 0 + or "expected_assertion" not in settings + ): + raise ValueError( + f"invalid_case requires a nonnegative integer and expected_assertion: {metadata}" + ) target["invalid_case"] = case if "expected_assertion" in settings: assertion = settings["expected_assertion"] if not isinstance(assertion, str) or not assertion or "\n" in assertion: raise ValueError(f"Expected one assertion message in {metadata}") target["expected_assertion"] = assertion - if "chips" in settings: - profiles = settings["chips"] - if (not isinstance(profiles, list) or not profiles or - not all(isinstance(value, str) and value.strip() and value == value.strip() for value in profiles)): - raise ValueError(f"chips must be a nonempty list of nonempty chip names: {metadata}") - if chip is None: - raise ValueError(f"Chip is required to select declared IP profiles: {metadata}") - target["chips"] = profiles + if "designs" in settings: + profiles = settings["designs"] + if ( + not isinstance(profiles, list) + or not profiles + or not all( + isinstance(value, str) + and value.strip() + and value == value.strip() + for value in profiles + ) + ): + raise ValueError( + f"designs must be a nonempty list of nonempty design names: {metadata}" + ) + if design is None: + raise ValueError( + f"Design is required to select declared IP profiles: {metadata}" + ) + target["designs"] = profiles if not targets: raise ValueError(f"registered IP {name!r} has no filelists under {resources}") - if target_name is not None: - targets = [target for target in targets if target["name"] == target_name] - if not targets: - raise ValueError(f"Unknown target {target_name!r} for IP {name!r}") - if "chips" in targets[0] and chip not in targets[0]["chips"]: - raise ValueError(f"Target {target_name!r} for IP {name!r} does not support chip {chip!r}; declared chips: {targets[0]['chips']}") - targets = [target for target in targets if "chips" not in target or chip in target["chips"]] + targets = [ + target + for target in targets + if "designs" not in target or design in target["designs"] + ] if not targets: - raise ValueError(f"IP {name!r} has no applicable targets for chip {chip!r}") + raise ValueError(f"IP {name!r} has no applicable targets for design {design!r}") + for target in targets: + target["target"] = target["name"] + target["name"] = f"{profile['profile']}/{target['name']}" return { + "profile": profile["profile"], + "config_path": profile["config_path"], + "parameters": profile["parameters"], + "defines": profile["defines"], "root": str(root.relative_to(bbdir)), "mill_module": module, "mill_directory": directory, "mill_arguments": arguments, + "runtime": build.get("runtime", []), "model": "src/csrc/Cargo.toml", "resources": "src/main/resources", "targets": targets, @@ -206,18 +327,21 @@ def load_ip(bbdir: str, name: str, target_name: str | None = None, chip: str | N def _ip_filelist( resources: Path, source: str, sim_dir: Path, bbdir: str, rtl: Path ) -> Path: - dst = sim_dir / source + source_path = resources / source + dst = sim_dir / source_path.name sim_dir.mkdir(parents=True, exist_ok=True) - text = (resources / source).read_text() + text = source_path.read_text() dst.write_text( text.replace("@VERIFY@", str(Path(bbdir) / "verify")) - .replace("@RESOURCES@", str(resources)) + .replace("@RESOURCES@", str(source_path.parent)) .replace("@RTL@", str(rtl)) ) return dst -def check_uvm_result(result, target: str, expected_assertion: str | None = None) -> None: +def check_uvm_result( + result, target: str, expected_assertion: str | None = None +) -> None: output = result.stdout + result.stderr if expected_assertion is not None: assertions = re.findall(r"Assertion failed: ([^\r\n]+)", output) @@ -229,13 +353,13 @@ def check_uvm_result(result, target: str, expected_assertion: str | None = None) or any(int(count) for count in failure_counts) or result.returncode < 0 ): - raise RuntimeError(f"Expected RTL assertion did not terminate {target}: {expected_assertion}") + raise RuntimeError( + f"Expected RTL assertion did not terminate {target}: {expected_assertion}" + ) return if result.returncode != 0: raise RuntimeError(f"UVM failed for {target}") - counts = dict( - re.findall(r"UVM_(ERROR|FATAL)\s*:\s*(\d+)", output) - ) + counts = dict(re.findall(r"UVM_(ERROR|FATAL)\s*:\s*(\d+)", output)) if set(counts) != {"ERROR", "FATAL"}: raise RuntimeError(f"UVM report summary missing for {target}") if counts != {"ERROR": "0", "FATAL": "0"}: @@ -244,13 +368,22 @@ def check_uvm_result(result, target: str, expected_assertion: str | None = None) ) -def build_ip(bbdir: str, chip: str, name: str, ctx, target_name: str | None = None) -> dict: - config = load_ip(bbdir, name, target_name, chip) +def build_ip(bbdir: str, design: str | None, name: str, ctx) -> dict: + targets = [] + for config in load_ip(bbdir, name, design): + build_ip_profile(bbdir, design, name, ctx, config) + targets.extend(config["targets"]) + return {"targets": targets} + + +def build_ip_profile( + bbdir: str, design: str | None, name: str, ctx, config: dict +) -> dict: root = Path(bbdir) / config["root"] resources = root / config["resources"] manifest = root / config["model"] rtl = root / "build" - sim_root = rtl / "uvm" / chip + sim_root = rtl / "uvm" / (design if design is not None else "default") rtl.mkdir(parents=True, exist_ok=True) build_lock = (rtl / ".build.lock").open("w") fcntl.flock(build_lock, fcntl.LOCK_EX) @@ -269,11 +402,81 @@ def build_ip(bbdir: str, chip: str, name: str, ctx, target_name: str | None = No if result.returncode != 0: raise RuntimeError(f"cargo build failed for IP {name}") - emit = shlex.join([ - str(Path(bbdir) / "result" / "bin" / "mill"), - config["mill_module"] + ".run", - *(argument.replace("{chip}", chip).replace("{targets}", ",".join(target["name"] for target in config["targets"])) for argument in config["mill_arguments"]), - ]) + profile_root = sim_root / config["profile"] + profile_root.mkdir(parents=True, exist_ok=True) + parameter_file = profile_root / "config.json" + parameter_file.write_text(json.dumps(config["parameters"])) + profile_rtl = profile_root / "rtl" + if design is None and any( + "{design}" in argument for argument in config["mill_arguments"] + ): + raise ValueError(f"IP {name} requires a design configuration") + if config["parameters"] and not any( + "{config}" in argument for argument in config["mill_arguments"] + ): + raise ValueError( + f"IP {name} emitter does not consume its UVM parameter configuration" + ) + arguments = [ + argument.replace("{repo}", str(Path(bbdir).resolve())) + .replace("{design}", design if design is not None else "") + .replace("{config}", str(parameter_file)) + .replace("{rtl}", str(profile_rtl)) + .replace( + "{targets}", ",".join(target["target"] for target in config["targets"]) + ) + for argument in config["mill_arguments"] + ] + if config["runtime"]: + classpath = [] + for entry in [ + {"directory": config["mill_directory"], "module": config["mill_module"]}, + *config["runtime"], + ]: + result = stream_run_logger( + cmd=mill_cmd( + bbdir, + entry["directory"], + entry["module"] + ".runClasspath", + show=True, + ), + logger=ctx.logger, + cwd=str(Path(bbdir) / entry["directory"]), + stdout_prefix="uvm classpath", + stderr_prefix="uvm classpath", + ) + if result.returncode != 0: + raise RuntimeError(f"Classpath build failed for IP {name}") + classpath.extend( + value.split(":", 3)[3] for value in json.loads(result.stdout) + ) + result = stream_run_logger( + cmd=mill_cmd( + bbdir, + config["mill_directory"], + config["mill_module"] + ".mainClass", + show=True, + ), + logger=ctx.logger, + cwd=str(Path(bbdir) / config["mill_directory"]), + stdout_prefix="uvm main", + stderr_prefix="uvm main", + ) + if result.returncode != 0: + raise RuntimeError(f"Main class lookup failed for IP {name}") + emit = shlex.join( + [ + str(Path(bbdir) / "result" / "bin" / "java"), + "-cp", + ":".join(dict.fromkeys(classpath)), + json.loads(result.stdout), + *arguments, + ] + ) + else: + emit = mill_cmd( + bbdir, config["mill_directory"], config["mill_module"] + ".run", arguments + ) result = stream_run_logger( cmd=emit, logger=ctx.logger, @@ -284,21 +487,40 @@ def build_ip(bbdir: str, chip: str, name: str, ctx, target_name: str | None = No if result.returncode != 0: raise RuntimeError(f"RTL generation failed for IP {name}") + if not any("{rtl}" in argument for argument in config["mill_arguments"]): + shutil.copytree( + rtl, + profile_rtl, + dirs_exist_ok=True, + ignore=shutil.ignore_patterns("uvm", ".build.lock"), + ) for target in config["targets"]: sim_dir = sim_root / target["name"] - filelist = _ip_filelist(resources, target["filelist"], sim_dir, bbdir, rtl) + filelist = _ip_filelist( + resources, target["filelist"], sim_dir, bbdir, profile_rtl + ) simv = sim_dir / "simv" csrc = sim_dir / "csrc" hier = sim_dir / "cm_hier.cfg" - declared_hier = resources / f"{target['name']}.cm_hier" + declared_hier = resources / target["hierarchy"] if declared_hier.is_file(): - scopes = [line.strip() for line in declared_hier.read_text().splitlines() if line.strip()] + scopes = [ + line.strip() + for line in declared_hier.read_text().splitlines() + if line.strip() + ] if not scopes or any( - not re.fullmatch(r"\+tree @TOP@\.[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?(?:\.[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?)*", line) + not re.fullmatch( + r"\+tree @TOP@\.[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?(?:\.[A-Za-z_][A-Za-z0-9_$]*(?:\[\d+\])?)*", + line, + ) for line in scopes ): raise ValueError(f"Invalid IP coverage hierarchy: {declared_hier}") - hier.write_text("\n".join(line.replace("@TOP@", target["top"]) for line in scopes) + "\n") + hier.write_text( + "\n".join(line.replace("@TOP@", target["top"]) for line in scopes) + + "\n" + ) else: hier.write_text( f"+tree {target['top']}.dut\n" @@ -311,6 +533,10 @@ def build_ip(bbdir: str, chip: str, name: str, ctx, target_name: str | None = No f"mkdir -p {shlex.quote(str(csrc))} && " f"vcs -full64 -sverilog -timescale=1ns/1ps -debug_access+all -top {shlex.quote(target['top'])} " "${=VCS_UVM_ARGS} " + + " ".join( + f"+define+{key}={value}" for key, value in config["defines"].items() + ) + + " " f"-cm line+cond+tgl+assert -cm_hier {shlex.quote(str(hier))} " f"-cm_assert_hier {shlex.quote(str(hier))} " f"-Mdir={shlex.quote(str(csrc))} -o {shlex.quote(str(simv))} " @@ -359,16 +585,24 @@ def report_ip_coverage( bbdir: str, root: Path, target: dict, simv: Path, cov_dir: Path, ctx ) -> None: cov_dir.mkdir(parents=True, exist_ok=True) - exclusions = root / "src" / "main" / "resources" / f"{target['name']}.el" + exclusions = root / "src/main/resources" / target["exclusions"] hierarchy = simv.parent / "rtl_hier.cfg" - hierarchy.write_text("".join( - f"+tree {target['top']}.{instance}\n" for instance in target.get("rtl_scopes", ["dut"]) - )) + hierarchy.write_text( + "".join( + f"+tree {target['top']}.{instance}\n" + for instance in target.get("rtl_scopes", ["dut"]) + ) + ) scope = f" -hier {shlex.quote(str(hierarchy))} -metric line+cond+tgl" reports = [(cov_dir, ""), (cov_dir / "rtl_raw", scope)] if exclusions.is_file(): reports[0] = (cov_dir, " -dump full_exclusions") - reports.append((cov_dir / "rtl", scope + f" -elfile {shlex.quote(str(exclusions))} -excl_strict")) + reports.append( + ( + cov_dir / "rtl", + scope + f" -elfile {shlex.quote(str(exclusions))} -excl_strict", + ) + ) for report, options in reports: if report.name == "rtl": validate_ip_exclusions(exclusions, simv.parent) @@ -394,13 +628,23 @@ def report_ip_coverage( ) -def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str, target_name: str | None = None) -> dict: - config = build_ip(bbdir, chip, name, ctx, target_name) +def run_ip(bbdir: str, design: str | None, name: str, ctx, cov_root: str) -> dict: + results = [ + run_ip_profile(bbdir, design, name, ctx, cov_root, config) + for config in load_ip(bbdir, name, design) + ] + return {"design": design, "results": results, "failures": []} + + +def run_ip_profile( + bbdir: str, design: str | None, name: str, ctx, cov_root: str, config: dict +) -> dict: + build_ip_profile(bbdir, design, name, ctx, config) root = Path(bbdir) / config["root"] manifest = root / config["model"] crate = tomllib.loads(manifest.read_text())["package"]["name"].replace("-", "_") model = manifest.parent / "target" / "debug" / f"lib{crate}" - sim_root = root / "build" / "uvm" / chip + sim_root = root / "build" / "uvm" / (design if design is not None else "default") coverage = [] expected_assertions = [] @@ -408,7 +652,11 @@ def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str, target_name: st simv = sim_root / target["name"] / "simv" simv_q = shlex.quote(str(simv)) cov_dir = Path(cov_root) / target["name"] / "coverage" - scenario = f"+invalid_case={target['invalid_case']} " if "invalid_case" in target else "" + scenario = ( + f"+invalid_case={target['invalid_case']} " + if "invalid_case" in target + else "" + ) script = ( f'loader="$(patchelf --print-interpreter {simv_q})"; ' f'library_path="$(patchelf --print-rpath {simv_q})"; ' @@ -428,7 +676,9 @@ def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str, target_name: st stderr_prefix="uvm run", ) expected_assertion = target.get("expected_assertion") - check_uvm_result(result, f"IP {name} target={target['name']}", expected_assertion) + check_uvm_result( + result, f"IP {name} target={target['name']}", expected_assertion + ) if expected_assertion is not None: expected_assertions.append(target["name"]) continue @@ -436,7 +686,7 @@ def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str, target_name: st report_ip_coverage(bbdir, root, target, simv, cov_dir, ctx) coverage.append((target["name"], cov_dir)) - index_dir = Path(cov_root) / "coverage" + index_dir = Path(cov_root) / config["profile"] / "coverage" index_dir.mkdir(parents=True, exist_ok=True) rows = ["target score line cond toggle assert group result"] for target, path in coverage: @@ -445,17 +695,24 @@ def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str, target_name: st f"{target} {summary['SCORE']} {summary['LINE']} {summary['COND']} " f"{summary['TOGGLE']} {summary.get('ASSERT', '--')} {summary.get('GROUP', '--')} pass" ) - rows.extend(f"{target} -- -- -- -- -- -- expected-assertion" for target in expected_assertions) + rows.extend( + f"{target} -- -- -- -- -- -- expected-assertion" + for target in expected_assertions + ) index = index_dir / "index.txt" index.write_text("\n".join(rows) + "\n") return { - "chip": chip, + "design": design, + "config_path": config["config_path"], + "parameters": config["parameters"], "ip": name, "targets": [t["name"] for t in config["targets"]], "expected_assertions": expected_assertions, "index": str(index), "rtl_raw_coverage": { - target["name"]: str(Path(cov_root) / target["name"] / "coverage" / "rtl_raw") + target["name"]: str( + Path(cov_root) / target["name"] / "coverage" / "rtl_raw" + ) for target in config["targets"] if target["name"] not in expected_assertions }, @@ -463,28 +720,40 @@ def run_ip(bbdir: str, chip: str, name: str, ctx, cov_root: str, target_name: st target["name"]: str(Path(cov_root) / target["name"] / "coverage" / "rtl") for target in config["targets"] if target["name"] not in expected_assertions - and (root / "src/main/resources" / f"{target['name']}.el").is_file() + and (root / "src/main/resources" / target["exclusions"]).is_file() }, "log": cov_root, } -def build_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx) -> None: +def build_ball( + bbdir: str, design_name: str, mill_cfg: str, domain, mapping, ctx +) -> None: ball = mapping.ball_dir - verify_dir = Path(bbdir) / "examples" / "balls" / ball / "verify" + verify_dir = Path(bbdir) / "examples" / "components" / "balls" / ball / "verify" casegen = verify_dir / "casegen" / "Cargo.toml" - rtl_dir = Path(bbdir) / "arch" / "build" / chip_name / mill_cfg - chip = load_chip(bbdir, chip_name) - core = next(core for core in chip.cores if any(item.ball_dir == ball for item in core.balldomain.mappings)) - tile = next(tile for tile in chip.tiles if core.index in tile.core_indices) - sim_dir = verify_dir / "build" / chip_name + rtl_dir = ( + Path(bbdir) + / "examples/designs" + / design_name + / "output/sims/device/arch" + / mill_cfg + ) + design = load_design(bbdir, design_name) + thread, core = ball_owner(design, ball) + sim_dir = verify_dir.parent / "output" / "uvm" / design_name uvm_rel = os.path.relpath(Path(bbdir) / "verify" / "uvm", verify_dir) flist = _filelist(verify_dir, ball, uvm_rel, rtl_dir, sim_dir) parameters_path = verify_dir / "parameters.toml" - parameters = tomllib.loads(parameters_path.read_text())["defines"] if parameters_path.is_file() else {} + parameters = ( + tomllib.loads(parameters_path.read_text())["defines"] + if parameters_path.is_file() + else {} + ) cargo = ( f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command " - f"cargo build --manifest-path {shlex.quote(str(casegen))}" + f"cargo build --manifest-path {shlex.quote(str(casegen))} " + f"--target-dir {shlex.quote(str(verify_dir.parent / 'output' / 'casegen'))}" ) r = stream_run_logger( cmd=cargo, @@ -509,7 +778,10 @@ def build_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx) f"mkdir -p {shlex.quote(str(sim_dir))} {shlex.quote(str(csrc))} && " "vcs -full64 -sverilog -timescale=1ns/1ps -debug_access+all -top tb_top " "${=VCS_UVM_ARGS} " - + " ".join(shlex.quote(d) for d in vcs_defines(domain, mapping, core, tile, parameters)) + + " ".join( + shlex.quote(d) + for d in vcs_defines(domain, mapping, thread, core, parameters) + ) + f" -cm line+cond+tgl+assert -cm_hier {shlex.quote(str(hier))} " f"-Mdir={shlex.quote(str(csrc))} -o {shlex.quote(str(simv))} " f"-f {shlex.quote(flist)}" @@ -527,29 +799,37 @@ def build_ball(bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx) def run_ball( - bbdir: str, chip_name: str, mill_cfg: str, domain, mapping, ctx, cov_dir: str + bbdir: str, design_name: str, mill_cfg: str, domain, mapping, ctx, cov_dir: str ) -> None: ball = mapping.ball_dir - verify_dir = Path(bbdir) / "examples" / "balls" / ball / "verify" - simv = verify_dir / "build" / chip_name / "simv" - build_ball(bbdir, chip_name, mill_cfg, domain, mapping, ctx) + verify_dir = Path(bbdir) / "examples" / "components" / "balls" / ball / "verify" + simv = verify_dir.parent / "output" / "uvm" / design_name / "simv" + build_ball(bbdir, design_name, mill_cfg, domain, mapping, ctx) crate = tomllib.loads((verify_dir / "casegen" / "Cargo.toml").read_text())[ "package" ]["name"] - dpi = verify_dir / "casegen" / "target" / "debug" / f"lib{crate.replace('-', '_')}" + dpi = ( + verify_dir.parent + / "output" + / "casegen" + / "debug" + / f"lib{crate.replace('-', '_')}" + ) test = f"{ball}_ball_test" - verify_config = verify_dir / "build" / chip_name / "verify_config.env" - chip = load_chip(bbdir, chip_name) - core = next(core for core in chip.cores if any(item.ball_dir == ball for item in core.balldomain.mappings)) + verify_config = ( + verify_dir.parent / "output" / "uvm" / design_name / "verify_config.env" + ) + design = load_design(bbdir, design_name) + thread, _ = ball_owner(design, ball) verify_config.write_text( - f"bank_entries={core.mem.bank.entries}\nball_id={mapping.ball_id}\n" + f"bank_entries={thread.mem.bank.entries}\nball_id={mapping.ball_id}\n" ) simv_q = shlex.quote(str(simv)) script = ( f"cd {shlex.quote(str(verify_dir))} && " f'loader="$(patchelf --print-interpreter {simv_q})"; ' f'library_path="$(patchelf --print-rpath {simv_q})"; ' - f'BB_VERIFY_CONFIG={shlex.quote(str(verify_config))} ' + f"BB_VERIFY_CONFIG={shlex.quote(str(verify_config))} " f'"$loader" --library-path "$library_path" {simv_q} -no_save ' f"-sv_lib {shlex.quote(str(dpi))} " f"+UVM_TESTNAME={shlex.quote(test)} +BID={mapping.ball_id} " @@ -600,12 +880,12 @@ def dashboard_summary(cov_dir: str) -> dict[str, str]: return dict(zip(keys, vals)) -def run_chip(bbdir: str, chip: str, ball: str | None, ctx, do_run: bool) -> dict: - msg = load_chip(bbdir, chip) +def run_design(bbdir: str, design: str, ball: str | None, ctx, do_run: bool) -> dict: + msg = load_design(bbdir, design) domain = ball_domain(msg, ball) - mill_cfg = msg.mill.verilator_config + mill_cfg = msg.simulations["device"].top if not mill_cfg: - raise ValueError(f"chip {chip!r} has no sims.verilator for UVM") + raise ValueError(f"design {design!r} has no sims.verilator for UVM") maps = selected_mappings(domain, ball) ran = [] covs = [] @@ -613,7 +893,7 @@ def run_chip(bbdir: str, chip: str, ball: str | None, ctx, do_run: bool) -> dict if do_run: stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") run_name = "all" if ball is None else maps[0].ball_dir - run_root = log_dir(bbdir, chip, "verilog", stamp, "uvm", run_name) + run_root = log_dir(bbdir, design, "verilog", stamp, "uvm", run_name) for m in maps: if do_run: cov = ( @@ -621,14 +901,14 @@ def run_chip(bbdir: str, chip: str, ball: str | None, ctx, do_run: bool) -> dict if ball is None else os.path.join(run_root, "coverage") ) - run_ball(bbdir, chip, mill_cfg, domain, m, ctx, cov) + run_ball(bbdir, design, mill_cfg, domain, m, ctx, cov) if not (Path(cov) / "dashboard.txt").is_file(): raise RuntimeError(f"urg wrote no dashboard.txt under {cov}") covs.append((m.ball_dir, cov)) else: - build_ball(bbdir, chip, mill_cfg, domain, m, ctx) + build_ball(bbdir, design, mill_cfg, domain, m, ctx) ran.append(m.ball_dir) - info = {"chip": chip, "mill": mill_cfg, "balls": ran} + info = {"design": design, "mill": mill_cfg, "balls": ran} if do_run and ball is None: index_dir = Path(run_root) / "coverage" index_dir.mkdir(parents=True, exist_ok=True) @@ -649,45 +929,69 @@ def run_chip(bbdir: str, chip: str, ball: str | None, ctx, do_run: bool) -> dict def run_uvm( - bbdir: str, chip: str, ball: str | None, ip: str | None, ctx, do_run: bool, target_name: str | None = None + bbdir: str, + design: str, + ball: str | None, + ip: str | None, + ctx, + do_run: bool, ) -> dict: - if target_name is not None and ip is None: - raise ValueError("Parameter --target requires --ip") if ball is not None: - return run_chip(bbdir, chip, ball, ctx, do_run) + return run_design(bbdir, design, ball, ctx, do_run) if ip is not None: if do_run: stamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") - run_name = f"{ip}-{target_name}" if target_name is not None else ip - run_root = log_dir(bbdir, chip, "verilog", stamp, "uvm", run_name) - return run_ip(bbdir, chip, ip, ctx, run_root, target_name) - config = build_ip(bbdir, chip, ip, ctx, target_name) + run_root = log_dir( + bbdir, + design if design is not None else "ip", + "verilog", + stamp, + "uvm", + ip, + ) + return run_ip(bbdir, design, ip, ctx, run_root) + config = build_ip(bbdir, design, ip, ctx) return { - "chip": chip, + "design": design, "ip": ip, "targets": [target["name"] for target in config["targets"]], } - targets = load_chip_uvm(bbdir, chip) + targets = load_design_uvm(bbdir, design) results, failures = [], [] for ball in targets["balls"]: try: - results.append(run_chip(bbdir, chip, ball, ctx, do_run)) + results.append(run_design(bbdir, design, ball, ctx, do_run)) except Exception as error: - if not do_run: raise + if not do_run: + raise ctx.logger.error(f"UVM ball/{ball} failed: {error}") failures.append({"target": f"ball/{ball}", "error": str(error)}) for ip in targets["ips"]: try: if do_run: stamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") - run_root = log_dir(bbdir, chip, "verilog", stamp, "uvm", ip) - results.append(run_ip(bbdir, chip, ip, ctx, run_root)) + run_root = log_dir( + bbdir, + design if design is not None else "ip", + "verilog", + stamp, + "uvm", + ip, + ) + results.extend(run_ip(bbdir, design, ip, ctx, run_root)["results"]) else: - config = build_ip(bbdir, chip, ip, ctx) - results.append({"chip": chip, "ip": ip, "targets": [item["name"] for item in config["targets"]]}) + config = build_ip(bbdir, design, ip, ctx) + results.append( + { + "design": design, + "ip": ip, + "targets": [item["name"] for item in config["targets"]], + } + ) except Exception as error: - if not do_run: raise + if not do_run: + raise ctx.logger.error(f"UVM ip/{ip} failed: {error}") failures.append({"target": f"ip/{ip}", "error": str(error)}) - return {"chip": chip, "results": results, "failures": failures} + return {"design": design, "results": results, "failures": failures} diff --git a/bbdev/api/steps/vcs/01_clean_api.step.py b/bbdev/api/steps/vcs/01_clean_api.step.py index 899228ed2..30142467f 100644 --- a/bbdev/api/steps/vcs/01_clean_api.step.py +++ b/bbdev/api/steps/vcs/01_clean_api.step.py @@ -1,8 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip -from utils.path import get_buckyball_path, rtl_dir - +from utils.event_common import require_design config = { "name": "vcs-clean-api", @@ -15,19 +14,18 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} - bbdir = get_buckyball_path() try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) await ctx.enqueue( { "topic": "vcs.clean", - "data": { - "chip": chip, - "output_dir": rtl_dir(bbdir, chip, "tapeout", body.get("output_dir")), - "_trace_id": ctx.trace_id, - }, + "data": {"design": design, "_trace_id": ctx.trace_id, "target": selection}, } ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/01_clean_event.step.py b/bbdev/api/steps/vcs/01_clean_event.step.py index c7a9c7ac7..0ff2c2676 100644 --- a/bbdev/api/steps/vcs/01_clean_event.step.py +++ b/bbdev/api/steps/vcs/01_clean_event.step.py @@ -1,3 +1,4 @@ +from utils.path import design_output_root import os import shutil import sys @@ -9,8 +10,7 @@ sys.path.insert(0, utils_path) from utils.event_common import check_result, get_origin_trace_id -from utils.path import get_buckyball_path, rtl_dir - +from utils.path import get_buckyball_path config = { "name": "vcs-clean", @@ -24,9 +24,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() - chip = input_data.get("chip") - build_dir = rtl_dir(bbdir, chip, "tapeout", input_data.get("output_dir")) - artifact_dir = os.path.join(build_dir, "vcs") + design = input_data.get("design") + artifact_dir = os.path.join( + design_output_root(bbdir, design, selection=input_data["target"]), "vcs" + ) if os.path.isdir(artifact_dir): shutil.rmtree(artifact_dir) - await check_result(ctx, 0, extra_fields={"task": "clean", "artifact_dir": artifact_dir}, trace_id=origin_tid) + await check_result( + ctx, + 0, + extra_fields={"task": "clean", "artifact_dir": artifact_dir}, + trace_id=origin_tid, + ) diff --git a/bbdev/api/steps/vcs/02_verilog_api.step.py b/bbdev/api/steps/vcs/02_verilog_api.step.py index 029129fab..07d8ddc90 100644 --- a/bbdev/api/steps/vcs/02_verilog_api.step.py +++ b/bbdev/api/steps/vcs/02_verilog_api.step.py @@ -1,7 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip - +from utils.event_common import require_design, require_target config = { "name": "vcs-verilog-api", @@ -13,10 +12,8 @@ def _request_data(body: dict) -> dict: - chip = require_chip(body) - data = {"chip": chip} - if body.get("output_dir"): - data["output_dir"] = body["output_dir"] + design = require_design(body) + data = {"design": design, "target": require_target(body)} return data @@ -25,5 +22,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: data = _request_data(req.body or {}) except ValueError as exc: return ApiResponse(status=400, body={"error": str(exc)}) - await ctx.enqueue({"topic": "vcs.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + {"topic": "vcs.verilog", "data": {**data, "_trace_id": ctx.trace_id}} + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/03_build_api.step.py b/bbdev/api/steps/vcs/03_build_api.step.py index 901fba742..c4f95f601 100644 --- a/bbdev/api/steps/vcs/03_build_api.step.py +++ b/bbdev/api/steps/vcs/03_build_api.step.py @@ -1,8 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip -from utils.path import get_buckyball_path, rtl_dir - +from utils.event_common import require_design config = { "name": "vcs-build-api", @@ -16,21 +15,32 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) jobs = body.get("jobs", 16) try: jobs = int(jobs) except (TypeError, ValueError): - return ApiResponse(status=400, body={"error": "jobs must be a positive integer"}) + return ApiResponse( + status=400, body={"error": "jobs must be a positive integer"} + ) if jobs <= 0: - return ApiResponse(status=400, body={"error": "jobs must be a positive integer"}) - bbdir = get_buckyball_path() + return ApiResponse( + status=400, body={"error": "jobs must be a positive integer"} + ) data = { - "chip": chip, + "design": design, "jobs": jobs, - "output_dir": rtl_dir(bbdir, chip, "tapeout", body.get("output_dir")), } - await ctx.enqueue({"topic": "vcs.build", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "vcs.build", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/03_build_event.step.py b/bbdev/api/steps/vcs/03_build_event.step.py index f1265cf80..1e9cc5373 100644 --- a/bbdev/api/steps/vcs/03_build_event.step.py +++ b/bbdev/api/steps/vcs/03_build_event.step.py @@ -1,225 +1,93 @@ -import glob -import os +from pathlib import Path import shlex import shutil -import subprocess -import sys -from pathlib import Path from motia import FlowContext, queue - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.event_common import check_result, get_origin_trace_id -from utils.event_common import require_chip -from utils.path import get_buckyball_path, rtl_dir +from utils.event_common import check_result, get_origin_trace_id, require_design +from utils.path import ( + get_buckyball_path, + rtl_dir, + design_output_root, + runner_manifest, + bebop_target_dir, + bebop_bin_path, +) from utils.stream_run import stream_run_logger_async - config = { "name": "vcs-build", - "description": "compile BBSimHarness with Synopsys VCS", + "description": "Build the independent Bebop VCS node and SoC simulator", "flows": ["vcs"], "triggers": [queue("vcs.build")], "enqueues": ["vcs.sim"], } -def testbench_source() -> str: - return r'''`timescale 1ns/1ps -module BBSimVcsHarness; - reg clock = 1'b0; - reg reset = 1'b1; - integer timeout_ns = 100000000; - - BBSimHarness dut (.clock(clock), .reset(reset)); - - always #0.5 clock = ~clock; - - initial begin - if ($value$plusargs("timeout-ns=%d", timeout_ns)) begin end - repeat (10) @(posedge clock); - @(negedge clock); - reset = 1'b0; - #(timeout_ns); - $fatal(1, "VCS simulation timed out after %0d ns", timeout_ns); - end - - initial begin - if ($test$plusargs("vpd")) $vcdpluson(0, dut); - if ($test$plusargs("vcd")) begin : enable_vcd - string vcd_file; - if (!$value$plusargs("vcd=%s", vcd_file)) vcd_file = "activity.vcd"; - $dumpfile(vcd_file); - $dumpvars(0, dut); - end - end -endmodule -''' - - -# The normal BBSim monitor has a Verilator-specific C++ main. VCS runs from -# a SystemVerilog testbench, so it needs only the memory model plus these DPI -# callbacks. Trace callbacks are intentionally no-ops; functional execution, -# UART, and SCU exit remain available to every chip using BBSimHarness. -def dpi_source() -> str: - return r'''#include -#include -#include - -extern "C" void dpi_bdb_set_clk(unsigned long long) {} -extern "C" void bbsim_memory_print_stats(); - -extern "C" void scu_uart_write(std::uint32_t, std::uint32_t ch) { - std::fputc(static_cast(ch & 0xff), stdout); - std::fflush(stdout); -} - -extern "C" void scu_sim_exit(std::uint32_t hart, std::uint32_t code) { - std::fprintf(stderr, "[SCU] hart %u: simulation exit code %u\\n", hart, code); - std::fflush(stderr); - bbsim_memory_print_stats(); - vpi_control(vpiFinish, static_cast(code)); -} - -extern "C" void scu_uart_rx_sample(std::uint32_t, std::uint32_t, - std::uint32_t *valid, std::uint32_t *data) { - *valid = 0; - *data = 0; -} - -extern "C" void dpi_itrace(std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t) {} -extern "C" void dpi_mtrace(std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t) {} -extern "C" void dpi_pmctrace(std::uint32_t, std::uint32_t, std::uint32_t, - std::uint32_t) {} -extern "C" void dpi_mem_pmctrace(std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t) {} -extern "C" void dpi_mtrace_issue(std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t, - std::uint32_t, std::uint32_t) {} -''' - - -def _pkg_config(flag: str, package: str) -> str: - result = subprocess.run(["pkg-config", flag, package], capture_output=True, text=True) - return result.stdout.strip() if result.returncode == 0 else "" - - async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) - try: - chip = require_chip(input_data) - except ValueError as exc: - await check_result(ctx, 1, extra_fields={"task": "build", "error": str(exc)}, trace_id=origin_tid) - return - if shutil.which("vcs") is None: - await check_result( - ctx, - 1, - extra_fields={"task": "build", "error": "vcs is not on PATH; source the EDA host environment before running bbdev vcs"}, - trace_id=origin_tid, - ) - return - - bbdir = get_buckyball_path() - build_dir = rtl_dir( - bbdir, - chip, - "verilog" if input_data.get("bebop_vcs") else "tapeout", - input_data.get("output_dir"), - ) - vsrcs = sorted( - path for path in ( - glob.glob(f"{build_dir}/**/*.v", recursive=True) - + glob.glob(f"{build_dir}/**/*.sv", recursive=True) - ) - if Path(path).name != "BBSimVcsHarness.sv" - ) - if not vsrcs: - await check_result(ctx, 1, extra_fields={"task": "build", "error": f"no RTL found under {build_dir}; run bbdev vcs --verilog first"}, trace_id=origin_tid) - return - - artifact_dir = Path(build_dir) / "vcs" - if artifact_dir.exists(): - shutil.rmtree(artifact_dir) - artifact_dir.mkdir(parents=True) - testbench = artifact_dir / "BBSimVcsHarness.sv" - dpi_shim = artifact_dir / "vcs_dpi.cc" - testbench.write_text(testbench_source(), encoding="utf-8") - dpi_shim.write_text(dpi_source(), encoding="utf-8") - - arch_dir = Path(bbdir) / "arch" - csrcs = [ - arch_dir / "src/csrc/src/monitor/ioe/BBSimDRAM.cc", - arch_dir / "src/csrc/src/monitor/ioe/mm.cc", - arch_dir / "src/csrc/src/monitor/ioe/mm_dramsim3.cc", - dpi_shim, - ] - missing = [str(path) for path in csrcs if not path.is_file()] - if missing: - await check_result(ctx, 1, extra_fields={"task": "build", "error": f"missing VCS DPI source(s): {', '.join(missing)}"}, trace_id=origin_tid) - return - - result_dir = Path(bbdir) / "result" - cflags = " ".join( - flag for flag in ( - "-std=c++17", - f"-I{result_dir / 'include'}", - f"-I{build_dir}", - f"-I{arch_dir / 'src/csrc/include'}", - "", - ) if flag - ) - ldflags = " ".join( - flag for flag in ( - "-ldramsim3 -lz -lstdc++", - f"-L{result_dir / 'lib'} -Wl,-rpath,{result_dir / 'lib'}", - ) if flag - ) - jobs = int(input_data.get("jobs", 16)) - simv = artifact_dir / "simv" - compile_log = artifact_dir / "vcs_compile.log" - sources = [*vsrcs, str(testbench), *(str(path) for path in csrcs)] - command = " ".join( + design = require_design(input_data) + repo = get_buckyball_path() + selection = input_data["target"] + target = Path(bebop_target_dir(repo, design, "vcs", selection=selection)) + output = Path(design_output_root(repo, design, selection=selection)) / "vcs" + rtl = rtl_dir(repo, design, "tapeout", selection=selection) + commands = [ [ - "env -u NIX_LDFLAGS -u NIX_CFLAGS_COMPILE -u NIX_LDFLAGS_FOR_TARGET -u NIX_CFLAGS_COMPILE_FOR_TARGET -u CPATH -u LIBRARY_PATH -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH -u CFLAGS -u CXXFLAGS -u LDFLAGS", - "vcs -full64 -sverilog -timescale=1ns/1ps -cpp g++ -cc g++ -ld g++", - "+define+PRINTF_COND=0" if input_data.get("bebop_vcs") else "", - "-top BBSimVcsHarness -debug_access+all -hsopt=off", - f"-j {jobs}", - f"-Mdir={shlex.quote(str(artifact_dir / 'csrc'))}", - f"-o {shlex.quote(str(simv))}", - f"-l {shlex.quote(str(compile_log))}", - f"+incdir+{shlex.quote(build_dir)}", - f"-CFLAGS {shlex.quote(cflags)}", - f"-LDFLAGS {shlex.quote(ldflags)}", - *(shlex.quote(source) for source in sources), - ] - ) - result = await stream_run_logger_async( - cmd=command, - logger=ctx.logger, - cwd=artifact_dir, - stdout_prefix="vcs build", - stderr_prefix="vcs build", - ) + "cargo", + "build", + "--release", + "--manifest-path", + str(runner_manifest(repo, design, selection)), + "--target-dir", + str(target), + "--bin", + "bebop", + "--bin", + "bebop-vcs", + ], + [ + str(target / "release/bebop"), + "build", + "vcs", + "--rtl-dir", + rtl, + "--artifact-dir", + str(output), + "--jobs", + str(input_data.get("jobs", 16)), + ], + ] + for command in commands: + result = await stream_run_logger_async( + cmd=shlex.join(command), + logger=ctx.logger, + cwd=repo, + stdout_prefix="vcs build", + stderr_prefix="vcs build", + ) + if result.returncode: + await check_result( + ctx, + result.returncode, + extra_fields={"task": "build"}, + trace_id=origin_tid, + ) + return + installed = Path(bebop_bin_path(repo, design, "vcs", selection=selection)) + installed.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(target / "release/bebop-vcs", installed) await check_result( ctx, - result.returncode, + 0, continue_run=input_data.get("from_run_workflow", False), - extra_fields={"task": "build", "simv": str(simv), "build_dir": build_dir, "compile_log": str(compile_log)}, + extra_fields={ + "task": "build", + "simv": str(output / "simv"), + "compile_log": str(output / "vcs_compile.log"), + }, trace_id=origin_tid, ) - if result.returncode == 0 and input_data.get("from_run_workflow"): - await ctx.enqueue({"topic": "vcs.sim", "data": {**input_data, "output_dir": build_dir, "_trace_id": origin_tid}}) + if input_data.get("from_run_workflow"): + await ctx.enqueue( + {"topic": "vcs.sim", "data": {**input_data, "_trace_id": origin_tid}} + ) diff --git a/bbdev/api/steps/vcs/04_sim_api.step.py b/bbdev/api/steps/vcs/04_sim_api.step.py index 4f17357e2..771d9a1d5 100644 --- a/bbdev/api/steps/vcs/04_sim_api.step.py +++ b/bbdev/api/steps/vcs/04_sim_api.step.py @@ -1,8 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip -from utils.path import get_buckyball_path - +from utils.event_common import require_design config = { "name": "vcs-sim-api", @@ -15,15 +14,29 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) binary = body.get("binary") try: - chip = require_chip(body) + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) if not isinstance(binary, str) or not binary: - return ApiResponse(status=400, body={"error": "Missing required parameter: --binary must be specified"}) - await ctx.enqueue({ - "topic": "vcs.sim", - "data": {**body, "chip": chip, "_trace_id": ctx.trace_id}, - }) + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --binary must be specified"}, + ) + await ctx.enqueue( + { + "topic": "vcs.sim", + "data": { + **body, + "design": design, + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/04_sim_event.step.py b/bbdev/api/steps/vcs/04_sim_event.step.py index 45371496b..3ada2dde7 100644 --- a/bbdev/api/steps/vcs/04_sim_event.step.py +++ b/bbdev/api/steps/vcs/04_sim_event.step.py @@ -1,3 +1,4 @@ +from utils.path import design_output_root import os import shlex import sys @@ -11,12 +12,15 @@ sys.path.insert(0, utils_path) from utils.event_common import check_result, get_origin_trace_id -from utils.event_common import require_chip -from utils.path import get_buckyball_path, log_dir, rtl_dir, workload_build_dir, workloads_output_root +from utils.event_common import require_design +from utils.path import ( + get_buckyball_path, + log_dir, + bebop_bin_path, +) from utils.search_workload import search_workload from utils.stream_run import stream_run_logger_async - config = { "name": "vcs-sim", "description": "run a VCS BBSimHarness simulation", @@ -26,65 +30,76 @@ } -def _resolve_binary(bbdir: str, chip: str, binary: str) -> str | None: +def _resolve_binary(bbdir: str, design: str, binary: str, selection: str) -> str | None: if os.path.isfile(binary): return os.path.abspath(binary) - for root in ( - workloads_output_root(bbdir), - os.path.join(workload_build_dir(bbdir, chip), "src"), - ): - found = search_workload(root, binary) - if found: - return found - return None + return search_workload( + str(Path(design_output_root(bbdir, design, selection=selection)) / "workloads"), + binary, + ) async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) try: - chip = require_chip(input_data) + design = require_design(input_data) except ValueError as exc: - await check_result(ctx, 1, extra_fields={"task": "sim", "error": str(exc)}, trace_id=origin_tid) + await check_result( + ctx, 1, extra_fields={"task": "sim", "error": str(exc)}, trace_id=origin_tid + ) return binary_name = input_data.get("binary") if not isinstance(binary_name, str) or not binary_name: - await check_result(ctx, 1, extra_fields={"task": "sim", "error": "missing binary"}, trace_id=origin_tid) + await check_result( + ctx, + 1, + extra_fields={"task": "sim", "error": "missing binary"}, + trace_id=origin_tid, + ) return bbdir = get_buckyball_path() - binary = _resolve_binary(bbdir, chip, binary_name) + binary = _resolve_binary(bbdir, design, binary_name, input_data["target"]) if binary is None: - await check_result(ctx, 1, extra_fields={"task": "sim", "error": f"binary not found: {binary_name}"}, trace_id=origin_tid) + await check_result( + ctx, + 1, + extra_fields={"task": "sim", "error": f"binary not found: {binary_name}"}, + trace_id=origin_tid, + ) return - build_dir = rtl_dir(bbdir, chip, "tapeout", input_data.get("output_dir")) - artifact_dir = Path(build_dir) / "vcs" + artifact_dir = ( + Path(design_output_root(bbdir, design, selection=input_data["target"])) / "vcs" + ) simv = artifact_dir / "simv" timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - run_log = Path(log_dir( - bbdir, chip, "tapeout", timestamp, "vcs", binary_name, - input_data.get("output_dir"), - )) - run_log.mkdir(parents=True) - timeout_ns = input_data.get("timeout_ns", input_data.get("timeout-ns", "100000000")) - if float(timeout_ns) <= 0: - raise ValueError("timeout-ns must be a positive number") - wave = input_data.get("waveform", input_data.get("wave")) - wave_arg = "+vcd" if str(wave).lower() == "vcd" else ("+vpd" if wave else "") - command = " ".join( - [ - "env -u LD_LIBRARY_PATH", - shlex.quote(str(simv)), - "+permissive", - f"+elf={shlex.quote(binary)}", - "+batch" if input_data.get("batch", False) else "", - wave_arg, - f"+dramsim_ini_dir={shlex.quote(str(Path(bbdir) / 'result/share/dramsim3/configs'))}", - f"+dramsim_outdir={shlex.quote(str(run_log / 'dramsim'))}", - f"+timeout-ns={shlex.quote(str(timeout_ns))}", - "+permissive-off", - ] + run_log = Path( + log_dir( + bbdir, + design, + "tapeout", + timestamp, + "vcs", + binary_name, + selection=input_data["target"], + ) ) + run_log.mkdir(parents=True) + command = [ + bebop_bin_path(bbdir, design, "vcs", selection=input_data["target"]), + "--simv", + str(simv), + "--elf", + binary, + "--dram-configs", + str(Path(bbdir) / "result/share/dramsim3/configs"), + "--log-dir", + str(run_log), + ] + wave = input_data.get("wave", False) + if wave: + command.extend(["--wave", "vpd" if wave is True else str(wave).lower()]) result = await stream_run_logger_async( - cmd=command, + cmd=shlex.join(command), logger=ctx.logger, cwd=artifact_dir, stdout_prefix="vcs sim", @@ -93,6 +108,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: await check_result( ctx, result.returncode, - extra_fields={"task": "sim", "simv": str(simv), "binary": binary, "log_dir": str(run_log)}, + extra_fields={ + "task": "sim", + "simv": str(simv), + "binary": binary, + "log_dir": str(run_log), + }, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/vcs/05_run_api.step.py b/bbdev/api/steps/vcs/05_run_api.step.py index f90f21290..40df9a4d2 100644 --- a/bbdev/api/steps/vcs/05_run_api.step.py +++ b/bbdev/api/steps/vcs/05_run_api.step.py @@ -1,8 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip -from utils.path import get_buckyball_path, rtl_dir - +from utils.event_common import require_design config = { "name": "vcs-run-api", @@ -15,28 +14,42 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) binary = body.get("binary") try: - chip = require_chip(body) + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) if not isinstance(binary, str) or not binary: - return ApiResponse(status=400, body={"error": "Missing required parameter: --binary must be specified"}) + return ApiResponse( + status=400, + body={"error": "Missing required parameter: --binary must be specified"}, + ) jobs = body.get("jobs", 16) try: jobs = int(jobs) except (TypeError, ValueError): - return ApiResponse(status=400, body={"error": "jobs must be a positive integer"}) + return ApiResponse( + status=400, body={"error": "jobs must be a positive integer"} + ) if jobs <= 0: - return ApiResponse(status=400, body={"error": "jobs must be a positive integer"}) - bbdir = get_buckyball_path() + return ApiResponse( + status=400, body={"error": "jobs must be a positive integer"} + ) data = { - "chip": chip, + "design": design, "binary": binary, "batch": bool(body.get("batch", False)), "jobs": jobs, - "output_dir": rtl_dir(bbdir, chip, "tapeout", body.get("output_dir")), "from_run_workflow": True, } - await ctx.enqueue({"topic": "vcs.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "vcs.verilog", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/vcs/README.md b/bbdev/api/steps/vcs/README.md index f4de374c9..fb7ee5820 100644 --- a/bbdev/api/steps/vcs/README.md +++ b/bbdev/api/steps/vcs/README.md @@ -1,7 +1,7 @@ # VCS workflow VCS follows the same user-facing stages as Verilator while keeping its own -simulator products below `arch/build//vcs/`: +simulator products below `examples/designs//output/sims//vcs/`: `vcs.verilog` is implemented under `steps/mill/02_vcs_verilog_event.step.py` (same mill elaboration as Verilator). Flow: @@ -11,11 +11,11 @@ vcs.verilog → vcs.build → vcs.sim ``` `bbdev vcs --run` routes through all three stages. The generated RTL is the -standard `BBSimHarness` elaboration; `vcs.build` adds a small SystemVerilog -testbench and a VCS-only DPI shim. The shim supplies BBSimDRAM, SCU UART, and +standard `BBSimHarness` elaboration; `vcs.build` invokes the independent node under `bebop/src/nodes/vcs`, +which owns the SystemVerilog testbench, DPI shim, build and execution. The shim supplies BBSimDRAM, SCU UART, and SCU exit callbacks without linking the Verilator-specific BBSim C++ main. The VCS flow is intended for RTL functional simulation. Gate-level power -simulation remains chip-owned under `examples/chips//tapeout/`, because -that flow must define the chip's workload, reset/boot skip interval, and SDF +simulation remains design-owned under `examples/designs//sims//tapeout/`, because +that flow must define the design's workload, reset/boot skip interval, and SDF policy. diff --git a/bbdev/api/steps/verilator/01_clean_api.step.py b/bbdev/api/steps/verilator/01_clean_api.step.py index 2376aa29a..4173ff4f4 100644 --- a/bbdev/api/steps/verilator/01_clean_api.step.py +++ b/bbdev/api/steps/verilator/01_clean_api.step.py @@ -1,7 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip -from utils.path import get_buckyball_path +from utils.event_common import require_design config = { "name": "verilator-clean-api", @@ -15,11 +15,23 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - await ctx.enqueue({ - "topic": "verilator.clean", - "data": {**body, "chip": chip, "task": "clean", "_trace_id": ctx.trace_id}, - }) + await ctx.enqueue( + { + "topic": "verilator.clean", + "data": { + **body, + "design": design, + "task": "clean", + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/01_clean_event.step.py b/bbdev/api/steps/verilator/01_clean_event.step.py index 5f4550cb3..d6470b264 100644 --- a/bbdev/api/steps/verilator/01_clean_event.step.py +++ b/bbdev/api/steps/verilator/01_clean_event.step.py @@ -8,7 +8,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -28,20 +28,23 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) try: - chip = require_chip(input_data) + design = require_design(input_data) except ValueError as error: ctx.logger.error(str(error)) await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, trace_id=origin_tid, ) return bbdir = get_buckyball_path() build_dir = rtl_dir( - bbdir, chip, + bbdir, + design, "verilog", - input_data.get("output_dir"), + selection=input_data["target"], ) # ================================================================================== # Execute operation @@ -62,7 +65,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, result.returncode, continue_run=input_data.get("from_run_workflow", False), - extra_fields={"task": "clean"}, trace_id=origin_tid, + extra_fields={"task": "clean"}, + trace_id=origin_tid, ) # ================================================================================== @@ -70,7 +74,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # ================================================================================== if input_data.get("from_run_workflow"): await ctx.enqueue( - {"topic": "verilator.verilog", "data": {**input_data, "output_dir": build_dir, "task": "run"}} + { + "topic": "verilator.verilog", + "data": {**input_data, "task": "run", "target": input_data["target"]}, + } ) return diff --git a/bbdev/api/steps/verilator/02_verilog_api.step.py b/bbdev/api/steps/verilator/02_verilog_api.step.py index 395fbfe45..1b1e90c25 100644 --- a/bbdev/api/steps/verilator/02_verilog_api.step.py +++ b/bbdev/api/steps/verilator/02_verilog_api.step.py @@ -1,10 +1,11 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design config = { "name": "verilator-verilog-api", - "description": "generate chip verilog code", + "description": "generate design verilog code", "flows": ["verilator"], "triggers": [api("POST", "/verilator/verilog")], "enqueues": ["verilator.verilog"], @@ -13,29 +14,28 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - - if body.get("balltype"): - return ApiResponse( - status=400, - body={ - "error": "ball-only verilog belongs to uvm; use bbdev uvm --verilog", - }, - ) + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) try: - chip = require_chip(body) + design = require_design(body) except ValueError as e: return ApiResponse( status=400, body={ "status": "error", "message": str(e), - "example": 'bbdev verilator --verilog "--chip toy"', + "example": 'bbdev verilator --verilog "--design toy"', }, ) - data = {"chip": chip} - if body.get("output_dir"): - data["output_dir"] = body["output_dir"] - await ctx.enqueue({"topic": "verilator.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + data = {"design": design} + await ctx.enqueue( + { + "topic": "verilator.verilog", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/03_build_api.step.py b/bbdev/api/steps/verilator/03_build_api.step.py index 023bbbc16..da7574b8c 100644 --- a/bbdev/api/steps/verilator/03_build_api.step.py +++ b/bbdev/api/steps/verilator/03_build_api.step.py @@ -1,7 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip -from utils.path import get_buckyball_path, rtl_dir +from utils.event_common import require_design config = { "name": "verilator-build-api", @@ -13,16 +13,23 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: - bbdir = get_buckyball_path() body = request.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) data = { - "chip": chip, + "design": design, "jobs": body.get("jobs", 16), - "output_dir": rtl_dir(bbdir, chip, "verilog", body.get("output_dir")), } - await ctx.enqueue({"topic": "verilator.build", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "verilator.build", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/03_build_event.step.py b/bbdev/api/steps/verilator/03_build_event.step.py index 70c574473..8c3ec5f11 100644 --- a/bbdev/api/steps/verilator/03_build_event.step.py +++ b/bbdev/api/steps/verilator/03_build_event.step.py @@ -11,7 +11,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import get_buckyball_path, rtl_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -27,14 +27,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) - chip = require_chip(input_data) + design = require_design(input_data) bbdir = get_buckyball_path() arch_dir = f"{bbdir}/arch" build_dir = rtl_dir( bbdir, - chip, + design, "verilog", - input_data.get("output_dir"), + selection=input_data["target"], ) coverage = input_data.get("coverage", False) ctx.logger.info(f"Using build directory: {build_dir}") @@ -131,7 +131,7 @@ def _gcc_lib_dir(soname: str) -> str: # Run verilator # ================================================================================== verilator_cmd = ( - f"verilator -MMD -cc --vpi --trace -O3 --x-assign fast --x-initial fast --noassert -Wno-fatal " + f"verilator -MMD -cc --vpi --trace -O3 --x-assign fast --x-initial fast --assert -Wno-fatal " f"--trace-fst --trace-threads 1 --output-split 10000 --output-split-cfuncs 100 " f"--unroll-count 256 " f"{'--coverage-line ' if coverage else ''}" @@ -193,7 +193,7 @@ def _gcc_lib_dir(soname: str) -> str: await ctx.enqueue( { "topic": "verilator.sim", - "data": {**input_data, "output_dir": build_dir, "task": "run"}, + "data": {**input_data, "task": "run", "target": input_data["target"]}, } ) diff --git a/bbdev/api/steps/verilator/04_sim_api.step.py b/bbdev/api/steps/verilator/04_sim_api.step.py index e9cf26916..d4cf19090 100644 --- a/bbdev/api/steps/verilator/04_sim_api.step.py +++ b/bbdev/api/steps/verilator/04_sim_api.step.py @@ -1,7 +1,7 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip -from utils.path import get_buckyball_path +from utils.event_common import require_design config = { "name": "verilator-sim-api", @@ -14,6 +14,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) binary = body.get("binary", "") if not binary: return ApiResponse( @@ -26,12 +30,20 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: }, ) try: - chip = require_chip(body) + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - await ctx.enqueue({ - "topic": "verilator.sim", - "data": {**body, "chip": chip, "task": "sim", "_trace_id": ctx.trace_id}, - }) + await ctx.enqueue( + { + "topic": "verilator.sim", + "data": { + **body, + "design": design, + "task": "sim", + "_trace_id": ctx.trace_id, + "target": selection, + }, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/04_sim_event.step.py b/bbdev/api/steps/verilator/04_sim_event.step.py index 51d893459..401dfc3e9 100644 --- a/bbdev/api/steps/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/verilator/04_sim_event.step.py @@ -1,3 +1,5 @@ +from pathlib import Path +from utils.path import design_output_root import os import shlex import struct @@ -9,8 +11,8 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.event_common import require_chip -from utils.path import get_buckyball_path, log_dir, rtl_dir, workloads_output_root +from utils.event_common import require_design +from utils.path import get_buckyball_path, log_dir, rtl_dir from utils.stream_run import stream_run_logger_async from utils.search_workload import search_workload from utils.event_common import check_result, get_origin_trace_id @@ -62,7 +64,10 @@ def validate_bbsim_elf(path: str) -> tuple[bool, str]: if p_paddr < dram_base or p_paddr + p_memsz > dram_base + dram_size: return ( False, - f"load segment {index} at 0x{p_paddr:x} is outside BBSimDRAM [0x{dram_base:x}, 0x{dram_base + dram_size:x})", + ( + f"load segment {index} at 0x{p_paddr:x} is outside" + f" BBSimDRAM [0x{dram_base:x}, 0x{dram_base + dram_size:x})" + ), ) except (OSError, struct.error): return (False, "cannot read a valid ELF64 program-header table") @@ -74,19 +79,24 @@ def validate_bbsim_elf(path: str) -> tuple[bool, str]: async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) try: - chip = require_chip(input_data) + design = require_design(input_data) except ValueError as error: ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + extra_fields={"error": "missing_design"}, trace_id=origin_tid, ) return bbdir = get_buckyball_path() - build_dir = rtl_dir(bbdir, chip, "verilog", input_data.get("output_dir")) + build_dir = rtl_dir( + bbdir, + design, + "verilog", + selection=input_data["target"], + ) ctx.logger.info(f"Using build directory: {build_dir}") timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") binary_name = input_data.get("binary", "") @@ -100,7 +110,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return coverage = input_data.get("coverage", False) - workload_root = workloads_output_root(bbdir) + workload_root = str( + Path(design_output_root(bbdir, design, selection=input_data["target"])) + / "workloads" + ) binary_path = search_workload(workload_root, binary_name) ctx.logger.info(f"binary_path: {binary_path}") if binary_path is None: @@ -125,7 +138,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: extra_fields={ "error": "unsupported_elf", "binary": binary_path, - "detail": f"direct Verilator requires a baremetal RISC-V ELF in BBSimDRAM; {elf_error}. Use bebop-verilator for Linux ELFs", + "detail": ( + "direct Verilator requires a baremetal RISC-V ELF in BBSimDRAM;" + f" {elf_error}. Use bebop-verilator for Linux ELFs" + ), }, trace_id=origin_tid, ) @@ -133,12 +149,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: topname = "BBSimHarness" run_log = log_dir( bbdir, - chip, + design, "verilog", timestamp, "verilator", binary_name, - input_data.get("output_dir"), + selection=input_data["target"], ) waveform_dir = os.path.join(run_log, "waveform") os.makedirs(run_log, exist_ok=True) @@ -218,7 +234,11 @@ def _gcc_lib_dir(soname: str) -> str: ) return dasm_cmd = shlex.join([dasm_bin, "dasm"]) - sim_cmd = f"export LD_LIBRARY_PATH={shlex.quote(ld_lib_path)}:$LD_LIBRARY_PATH; export BDB_SIM_META={shlex.quote(meta_path)}; {shlex.join(sim_args)} 2> >({dasm_cmd} > {shlex.quote(os.path.join(run_log, 'disasm.log'))})" + sim_cmd = ( + f"export LD_LIBRARY_PATH={shlex.quote(ld_lib_path)}:$LD_LIBRARY_PATH; export" + f" BDB_SIM_META={shlex.quote(meta_path)}; {shlex.join(sim_args)} 2>" + f" >({dasm_cmd} > {shlex.quote(os.path.join(run_log, 'disasm.log'))})" + ) script_dir = os.path.dirname(__file__) result = await stream_run_logger_async( cmd=sim_cmd, diff --git a/bbdev/api/steps/verilator/05_run_api.step.py b/bbdev/api/steps/verilator/05_run_api.step.py index 3810a06bf..3aeb4fc70 100644 --- a/bbdev/api/steps/verilator/05_run_api.step.py +++ b/bbdev/api/steps/verilator/05_run_api.step.py @@ -1,3 +1,6 @@ +from pathlib import Path +from utils.path import design_output_root +from utils.event_common import require_target import os import sys @@ -7,8 +10,8 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.event_common import require_chip -from utils.path import get_buckyball_path, workloads_output_root +from utils.event_common import require_design +from utils.path import get_buckyball_path from utils.search_workload import search_workload config = { @@ -23,7 +26,11 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) binary = body.get("binary", "") @@ -38,7 +45,14 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: }, ) - search_dir = workloads_output_root(get_buckyball_path()) + search_dir = str( + Path( + design_output_root( + get_buckyball_path(), body["design"], selection=selection + ) + ) + / "workloads" + ) if search_workload(search_dir, binary) is None: return ApiResponse( status=400, @@ -53,7 +67,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: ) data = { - "chip": chip, + "design": design, "binary": binary, "jobs": body.get("jobs", "16"), "batch": body.get("batch", False), @@ -67,5 +81,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "from_run_workflow": True, } - await ctx.enqueue({"topic": "verilator.run", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "verilator.run", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/verilator/README.md b/bbdev/api/steps/verilator/README.md index 8369f21c6..a2608df16 100644 --- a/bbdev/api/steps/verilator/README.md +++ b/bbdev/api/steps/verilator/README.md @@ -26,10 +26,10 @@ must be run with `bebop-verilator` instead. **Example**: ```bash # bbdev wrapper -bbdev verilator --run "--chip toy --jobs 16 --binary toy-toy-ctest-mvin_mvout_test-baremetal --batch" +bbdev verilator --run "--design toy --jobs 16 --binary toy-toy-ctest-mvin_mvout_test-baremetal --batch" # Raw command -curl -X POST http://localhost:5000/verilator/run -H "Content-Type: application/json" -d '{"chip":"toy", "jobs":8, "binary":"toy-toy-ctest-mvin_mvout_test-baremetal", "batch":true}' +curl -X POST http://localhost:5000/verilator/run -H "Content-Type: application/json" -d '{"design":"toy", "jobs":8, "binary":"toy-toy-ctest-mvin_mvout_test-baremetal", "batch":true}' ``` diff --git a/bbdev/api/steps/workload/00_clean_api.step.py b/bbdev/api/steps/workload/00_clean_api.step.py index b0c9503f5..47617cb97 100644 --- a/bbdev/api/steps/workload/00_clean_api.step.py +++ b/bbdev/api/steps/workload/00_clean_api.step.py @@ -1,10 +1,11 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design config = { "name": "workload-clean-api", - "description": "clean workload output directory for one chip", + "description": "clean workload output directory for one design", "flows": ["workload"], "triggers": [api("POST", "/workload/clean")], "enqueues": ["workload.clean"], @@ -14,16 +15,27 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as error: return ApiResponse(status=400, body={"error": str(error)}) - unknown = sorted(k for k in body if k not in {"chip"}) + unknown = sorted(k for k in body if k not in {"design"}) if unknown: return ApiResponse( status=400, - body={"error": f"Unknown workload clean parameter(s): {', '.join(unknown)}"}, + body={ + "error": f"Unknown workload clean parameter(s): {', '.join(unknown)}" + }, ) - await ctx.enqueue({"topic": "workload.clean", "data": {**body, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "workload.clean", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/00_clean_event.step.py b/bbdev/api/steps/workload/00_clean_event.step.py index a710be5dd..0c26105f1 100644 --- a/bbdev/api/steps/workload/00_clean_event.step.py +++ b/bbdev/api/steps/workload/00_clean_event.step.py @@ -8,13 +8,13 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.event_common import require_chip +from utils.event_common import require_design from utils.event_common import check_result, get_origin_trace_id -from utils.path import chip_output_root, get_buckyball_path, workload_build_dir +from utils.path import design_output_root, get_buckyball_path, workload_build_dir config = { "name": "workload-clean", - "description": "clean workload output directory for one chip", + "description": "clean workload output directory for one design", "flows": ["workload"], "triggers": [queue("workload.clean")], "enqueues": [], @@ -23,7 +23,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) - allowed = {"chip", "_trace_id"} + allowed = {"device", "core", "target", "design", "_trace_id"} unknown = sorted(k for k in input_data if k not in allowed) if unknown: ctx.logger.error(f"Unknown workload clean parameter(s): {', '.join(unknown)}") @@ -37,21 +37,21 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return try: - chip = require_chip(input_data) + design = require_design(input_data) except ValueError as error: ctx.logger.error(str(error)) await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + extra_fields={"error": "missing_design"}, trace_id=origin_tid, ) return bbdir = get_buckyball_path() - output_path = chip_output_root(bbdir, chip) - build_path = workload_build_dir(bbdir, chip) + output_path = design_output_root(bbdir, design, selection=input_data["target"]) + build_path = workload_build_dir(bbdir, design, selection=input_data["target"]) for path in (output_path, build_path): if os.path.exists(path): @@ -66,7 +66,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "task": "clean", - "chip": chip, + "design": design, "output_path": output_path, "build_path": build_path, }, diff --git a/bbdev/api/steps/workload/01_buidl_api.step.py b/bbdev/api/steps/workload/01_buidl_api.step.py index 12c31422b..9943a918c 100644 --- a/bbdev/api/steps/workload/01_buidl_api.step.py +++ b/bbdev/api/steps/workload/01_buidl_api.step.py @@ -1,3 +1,4 @@ +from utils.event_common import require_target import os import re import sys @@ -20,7 +21,11 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - allowed = {"chip", "stable", "ctest", "mlirtest", "soctest"} + try: + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + allowed = {"device", "core", "design", "ctest", "mlirtest", "soctest"} unknown = sorted((k for k in body if k not in allowed)) if unknown: return ApiResponse( @@ -29,24 +34,18 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "error": f"Unknown workload build parameter(s): {', '.join(unknown)}" }, ) - chip = body.get("chip") - if not chip: + design = body.get("design") + if not design: return ApiResponse( status=400, - body={"error": "Missing required parameter: --chip must be specified"}, + body={"error": "Missing required parameter: --design must be specified"}, ) - if not isinstance(chip, str) or not re.fullmatch("[A-Za-z0-9_-]+", chip): - return ApiResponse(status=400, body={"error": f"Invalid chip: {chip}"}) - chip_dir = Path(get_buckyball_path()) / "examples" / "chips" / chip - if not chip_dir.is_dir(): + if not isinstance(design, str) or not re.fullmatch("[A-Za-z0-9_-]+", design): + return ApiResponse(status=400, body={"error": f"Invalid design: {design}"}) + design_dir = Path(get_buckyball_path()) / "examples" / "designs" / design + if not design_dir.is_dir(): return ApiResponse( - status=400, body={"error": f"Workload chip does not exist: {chip}"} - ) - stable = body.get("stable", False) - if not isinstance(stable, bool): - return ApiResponse( - status=400, - body={"error": "Invalid parameter: stable must be a boolean flag"}, + status=400, body={"error": f"Workload design does not exist: {design}"} ) ctest = body.get("ctest", False) mlirtest = body.get("mlirtest", False) @@ -62,13 +61,15 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body={"error": "--ctest, --mlirtest and --soctest are mutually exclusive"}, ) data = { - "chip": chip, - "stable": stable, + "design": design, "ctest": ctest, "mlirtest": mlirtest, "soctest": soctest, } await ctx.enqueue( - {"topic": "workload.build", "data": {**data, "_trace_id": ctx.trace_id}} + { + "topic": "workload.build", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index 5c8929af2..aefb3ef27 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -21,6 +21,13 @@ raise RuntimeError(f"cannot load {_workload_build_path}") workload_build = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(workload_build) + +_stack_build_path = os.path.join( + os.path.dirname(__file__), "..", "stack", "scripts", "build.py" +) +_spec = importlib.util.spec_from_file_location("stack_build_module", _stack_build_path) +stack_build = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(stack_build) config = { "name": "workload-build", "description": "build workload", @@ -34,8 +41,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() allowed = { - "chip", - "stable", + "target", + "device", + "core", + "design", "ctest", "mlirtest", "soctest", @@ -52,46 +61,35 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return - chip = input_data.get("chip") - if not chip: - ctx.logger.error("Missing required parameter: chip must be specified") + design = input_data.get("design") + if not design: + ctx.logger.error("Missing required parameter: design must be specified") await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + extra_fields={"error": "missing_design"}, trace_id=origin_tid, ) return - if not isinstance(chip, str) or not re.fullmatch("[A-Za-z0-9_-]+", chip): - ctx.logger.error(f"Invalid chip: {chip}") + if not isinstance(design, str) or not re.fullmatch("[A-Za-z0-9_-]+", design): + ctx.logger.error(f"Invalid design: {design}") await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "invalid_chip", "chip": chip}, + extra_fields={"error": "invalid_design", "design": design}, trace_id=origin_tid, ) return - chip_dir = Path(bbdir) / "examples" / "chips" / chip - if not chip_dir.is_dir(): - ctx.logger.error(f"Workload chip does not exist: {chip}") + design_dir = Path(bbdir) / "examples" / "designs" / design + if not design_dir.is_dir(): + ctx.logger.error(f"Workload design does not exist: {design}") await check_result( ctx, 1, continue_run=False, - extra_fields={"error": "unknown_chip", "chip": chip}, - trace_id=origin_tid, - ) - return - stable = input_data.get("stable", False) - if not isinstance(stable, bool): - ctx.logger.error("Invalid parameter: stable must be a boolean flag") - await check_result( - ctx, - 1, - continue_run=False, - extra_fields={"error": "invalid_stable", "stable": stable}, + extra_fields={"error": "unknown_design", "design": design}, trace_id=origin_tid, ) return @@ -119,13 +117,21 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return try: + # Workloads link the device runtime; MLIR tests compile with its plugins. + stack_build.build_design( + bbdir, + design=design, + selection=input_data["target"], + logger=ctx.logger, + task_scope=origin_tid, + ) workload_build.build_workload( bbdir, - chip, + design, + selection=input_data["target"], ctest=ctest, mlirtest=mlirtest, soctest=soctest, - stable=stable, logger=ctx.logger, task_scope=origin_tid, ) @@ -137,7 +143,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: continue_run=False, extra_fields={ "error": "workload_build_failed", - "chip": chip, + "design": design, "detail": str(error), }, trace_id=origin_tid, @@ -148,7 +154,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: 0, continue_run=False, extra_fields={ - "chip": chip, + "design": design, "ctest": ctest, "mlirtest": mlirtest, "soctest": soctest, diff --git a/bbdev/api/steps/workload/02_tohex_api.step.py b/bbdev/api/steps/workload/02_tohex_api.step.py index b87b73f76..18e85fc38 100644 --- a/bbdev/api/steps/workload/02_tohex_api.step.py +++ b/bbdev/api/steps/workload/02_tohex_api.step.py @@ -1,10 +1,11 @@ +from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design config = { "name": "workload-tohex-api", - "description": "convert chip workload ELF to hex", + "description": "convert design workload ELF to hex", "flows": ["workload"], "triggers": [api("POST", "/workload/tohex")], "enqueues": ["workload.tohex"], @@ -14,16 +15,18 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + try: + design = require_design(body) except ValueError as error: return ApiResponse(status=400, body={"error": str(error)}) - unknown = sorted(k for k in body if k not in {"chip"}) - if unknown: - return ApiResponse( - status=400, - body={"error": f"Unknown workload tohex parameter(s): {', '.join(unknown)}"}, - ) - - await ctx.enqueue({"topic": "workload.tohex", "data": {**body, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "workload.tohex", + "data": {**body, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/workload/02_tohex_event.step.py b/bbdev/api/steps/workload/02_tohex_event.step.py index c7ef314cd..273cbace1 100644 --- a/bbdev/api/steps/workload/02_tohex_event.step.py +++ b/bbdev/api/steps/workload/02_tohex_event.step.py @@ -9,14 +9,14 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.event_common import require_chip +from utils.event_common import require_design from utils.event_common import check_result, get_origin_trace_id from utils.path import get_buckyball_path, workload_tests_root from utils.stream_run import stream_run_logger_async config = { "name": "workload-tohex", - "description": "convert all -baremetal ELF files under chip workload output to hex", + "description": "convert all -baremetal ELF files under design workload output to hex", "flows": ["workload"], "triggers": [queue("workload.tohex")], "enqueues": [], @@ -27,18 +27,22 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() try: - chip = require_chip(input_data) + design = require_design(input_data) except ValueError as error: ctx.logger.error(str(error)) await check_result( - ctx, 1, continue_run=False, - extra_fields={"error": "missing_chip"}, + ctx, + 1, + continue_run=False, + extra_fields={"error": "missing_design"}, trace_id=origin_tid, ) return script = f"{bbdir}/bbdev/api/steps/workload/scripts/elf2hex.py" - search_root = Path(workload_tests_root(bbdir, chip)) + search_root = Path( + workload_tests_root(bbdir, design, selection=input_data["target"]) + ) ctx.logger.info("Search root", {"path": str(search_root)}) diff --git a/bbdev/api/steps/workload/README.md b/bbdev/api/steps/workload/README.md index 8b640d616..56b92f0a1 100644 --- a/bbdev/api/steps/workload/README.md +++ b/bbdev/api/steps/workload/README.md @@ -7,14 +7,14 @@ Workload build workflow in Buckyball framework, used to build test workloads and ### `clean` **Endpoint**: `POST /workload/clean` -**Function**: Clean workload output directory for one chip. +**Function**: Clean workload output directory for one design. **Parameters**: -- **`chip`** - Required chip name. +- **`design`** - Required design name. **Examples**: ```bash -bbdev workload --clean "--chip toy" +bbdev workload --clean "--design toy" ``` ### `build` @@ -23,27 +23,27 @@ bbdev workload --clean "--chip toy" **Function**: Build workload **Parameters**: -- **`chip`** - Required chip name. Selects chip-specific workloads. +- **`design`** - Required design name. Selects design-specific workloads. - **`stable`** - Optional boolean flag. If set, build with stable LLVM Buckyball extensions. - **`ctest`** - Build CTest workloads only. - **`mlirtest`** - Build MLIRTest workloads only. - **`soctest`** - Build SoC test workloads only. -For chip workloads under paths like `*/chips/`, only the directory selected by `chip` is synced to `bb-tests/output//workloads`. +For design workloads under paths like `*/designs/`, only the directory selected by `design` is synced to `bb-tests/output//workloads`. **Examples**: ```bash # Build all workloads -bbdev workload --build "--chip toy" +bbdev workload --build "--design toy" # Build only CTest workloads -bbdev workload --build "--chip pebble --ctest" +bbdev workload --build "--design pebble --ctest" # Build only MLIRTest workloads -bbdev workload --build "--chip pebble --mlirtest" +bbdev workload --build "--design pebble --mlirtest" # Build only SoC test workloads -bbdev workload --build "--chip goban --soctest" +bbdev workload --build "--design goban --soctest" ``` **Response**: @@ -60,6 +60,6 @@ bbdev workload --build "--chip goban --soctest" ## Notes -- Workload build directory is `bb-tests/workloads/build/` +- Workload build directory is `bb-tests/workloads/build/` - Workload source code is distributed under `bb-tests/workloads/src` and `examples/*/*/workloads` -- Workload binaries are emitted under `bb-tests/output//workloads/src` +- Workload binaries are emitted under `bb-tests/output//workloads/src` diff --git a/bbdev/api/steps/yosys/01_run_api.step.py b/bbdev/api/steps/yosys/01_run_api.step.py index 667f52b8f..2ebd1d401 100644 --- a/bbdev/api/steps/yosys/01_run_api.step.py +++ b/bbdev/api/steps/yosys/01_run_api.step.py @@ -1,9 +1,10 @@ +from utils.event_common import require_target import os import sys from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: @@ -23,12 +24,30 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + unknown = body.keys() - { + "device", + "core", + "design", + "top", + "output-dir", + "output_dir", + "vcd", + } + if unknown: + return ApiResponse( + status=400, + body={"error": f"Unknown parameter(s): {', '.join(sorted(unknown))}"}, + ) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) data = { - "chip": chip, + "design": design, "top": req_arg(body, "top") or "DigitalTop", "vcd": req_arg(body, "vcd"), "from_run_workflow": True, @@ -36,8 +55,10 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: output_dir = req_arg(body, "output_dir") if output_dir: data["output_dir"] = output_dir - log_dir = req_arg(body, "log_dir") - if log_dir: - data["log_dir"] = log_dir - await ctx.enqueue({"topic": "yosys.run", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "yosys.run", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/yosys/02_verilog_api.step.py b/bbdev/api/steps/yosys/02_verilog_api.step.py index ed0a5a0d4..e27c8c229 100644 --- a/bbdev/api/steps/yosys/02_verilog_api.step.py +++ b/bbdev/api/steps/yosys/02_verilog_api.step.py @@ -1,9 +1,10 @@ +from utils.event_common import require_target import os import sys from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: @@ -23,16 +24,35 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + unknown = body.keys() - { + "device", + "core", + "design", + "top", + "output-dir", + "output_dir", + } + if unknown: + return ApiResponse( + status=400, + body={"error": f"Unknown parameter(s): {', '.join(sorted(unknown))}"}, + ) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - data = {"chip": chip, "top": req_arg(body, "top") or "DigitalTop"} + data = {"design": design, "top": req_arg(body, "top") or "DigitalTop"} output_dir = req_arg(body, "output_dir") if output_dir: data["output_dir"] = output_dir - log_dir = req_arg(body, "log_dir") - if log_dir: - data["log_dir"] = log_dir - await ctx.enqueue({"topic": "yosys.verilog", "data": {**data, "_trace_id": ctx.trace_id}}) + await ctx.enqueue( + { + "topic": "yosys.verilog", + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, + } + ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/yosys/03_synth_api.step.py b/bbdev/api/steps/yosys/03_synth_api.step.py index 2994e98dc..481239790 100644 --- a/bbdev/api/steps/yosys/03_synth_api.step.py +++ b/bbdev/api/steps/yosys/03_synth_api.step.py @@ -1,9 +1,10 @@ +from utils.event_common import require_target import os import sys from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_chip +from utils.event_common import require_design scripts_path = os.path.join(os.path.dirname(__file__), "scripts") if scripts_path not in sys.path: @@ -23,12 +24,30 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: body = req.body or {} try: - chip = require_chip(body) + selection = require_target(body, ("device", "core")) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) + unknown = body.keys() - { + "device", + "core", + "design", + "top", + "output-dir", + "output_dir", + "vcd", + } + if unknown: + return ApiResponse( + status=400, + body={"error": f"Unknown parameter(s): {', '.join(sorted(unknown))}"}, + ) + try: + design = require_design(body) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) data = { - "chip": chip, + "design": design, "consumer": "yosys", "top": req_arg(body, "top") or "DigitalTop", "next_topic": "yosys.synth", @@ -39,13 +58,10 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: vcd = req_arg(body, "vcd") if vcd: data["vcd"] = vcd - log_dir = req_arg(body, "log_dir") - if log_dir: - data["log_dir"] = log_dir await ctx.enqueue( { "topic": "ip.generate", - "data": {**data, "_trace_id": ctx.trace_id}, + "data": {**data, "_trace_id": ctx.trace_id, "target": selection}, } ) return ApiResponse(status=202, body={"trace_id": ctx.trace_id}) diff --git a/bbdev/api/steps/yosys/03_synth_event.step.py b/bbdev/api/steps/yosys/03_synth_event.step.py index 0a0a493ad..29875acda 100644 --- a/bbdev/api/steps/yosys/03_synth_event.step.py +++ b/bbdev/api/steps/yosys/03_synth_event.step.py @@ -10,7 +10,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.event_common import require_chip +from utils.event_common import require_design from utils.path import rtl_dir, get_buckyball_path, log_dir from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -24,7 +24,9 @@ } -def dynamic_power_report(power_report: str, activity_source: str) -> tuple[str, float] | None: +def dynamic_power_report( + power_report: str, activity_source: str +) -> tuple[str, float] | None: """Return OpenSTA's internal-plus-switching power, excluding leakage.""" match = re.search( r"^Total\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)", @@ -36,38 +38,53 @@ def dynamic_power_report(power_report: str, activity_source: str) -> tuple[str, internal, switching, _leakage, _total = (float(value) for value in match.groups()) dynamic = internal + switching return ( - "OpenSTA dynamic power\n" - f" Activity source: {activity_source}\n" - f" Internal: {internal:.6e} W\n" - f" Switching: {switching:.6e} W\n" - f" Dynamic (internal + switching): {dynamic:.6e} W\n", + ( + "OpenSTA dynamic power\n" + f" Activity source: {activity_source}\n" + f" Internal: {internal:.6e} W\n" + f" Switching: {switching:.6e} W\n" + f" Dynamic (internal + switching): {dynamic:.6e} W\n" + ), dynamic, ) def load_yosys_config(): - config_path = os.path.join(os.path.dirname(__file__), "scripts", "yosys-config.yaml") + config_path = os.path.join( + os.path.dirname(__file__), "scripts", "yosys-config.yaml" + ) if os.path.exists(config_path): with open(config_path, "r") as f: config = yaml.safe_load(f) or {} # Keep paths portable between the Nix shell and explicit user # configurations while retaining normal YAML scalar values. if isinstance(config.get("liberty"), str): - config["liberty"] = os.path.expandvars(os.path.expanduser(config["liberty"])) + config["liberty"] = os.path.expandvars( + os.path.expanduser(config["liberty"]) + ) if isinstance(config.get("vcd"), str) and config["vcd"]: config["vcd"] = os.path.expandvars(os.path.expanduser(config["vcd"])) return config return {} + async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() try: - chip = require_chip(input_data) - build_dir = rtl_dir(bbdir, chip, "synth", input_data.get("output_dir")) + design = require_design(input_data) + build_dir = rtl_dir( + bbdir, + design, + "synth", + input_data.get("output_dir"), + selection=input_data["target"], + ) except ValueError as exc: await check_result( - ctx, 1, continue_run=False, + ctx, + 1, + continue_run=False, extra_fields={"task": "synth", "error": str(exc)}, trace_id=origin_tid, ) @@ -85,7 +102,10 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx, 1, continue_run=False, - extra_fields={"task": "synth", "error": "missing prepared yosys source list"}, + extra_fields={ + "task": "synth", + "error": "missing prepared yosys source list", + }, trace_id=origin_tid, ) return failure_result @@ -104,8 +124,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return failure_result stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - yosys_output_dir = input_data.get("log_dir") or log_dir( - bbdir, chip, "synth", stamp, "yosys", top_module, input_data.get("output_dir"), + yosys_output_dir = input_data.get("analysis_dir") or log_dir( + bbdir, + design, + "synth", + stamp, + "yosys", + top_module, + input_data.get("output_dir"), + selection=input_data["target"], ) os.makedirs(yosys_output_dir, exist_ok=True) ctx.logger.info(f"Yosys log dir: {yosys_output_dir}") @@ -122,11 +149,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: if liberty and os.path.exists(liberty): f.write(f"dfflibmap -liberty {liberty}\n") f.write(f"abc -liberty {liberty}\n") - f.write(f"tee -o {yosys_output_dir}/hierarchy_report.txt stat -liberty {liberty}\n") + f.write( + f"tee -o {yosys_output_dir}/hierarchy_report.txt stat -liberty" + f" {liberty}\n" + ) f.write("flatten\n") f.write("opt\n") f.write(f"stat -liberty {liberty}\n") - f.write(f"tee -o {yosys_output_dir}/area_report.txt stat -liberty {liberty}\n") + f.write( + f"tee -o {yosys_output_dir}/area_report.txt stat -liberty {liberty}\n" + ) f.write(f"write_verilog {yosys_output_dir}/synth_netlist.v\n") else: f.write(f"tee -o {yosys_output_dir}/hierarchy_report.txt stat\n") @@ -154,7 +186,12 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: dynamic_power_report_file = f"{yosys_output_dir}/dynamic_power_report.txt" sta_returncode = 0 - if liberty and os.path.exists(liberty) and os.path.exists(netlist_file) and result.returncode == 0: + if ( + liberty + and os.path.exists(liberty) + and os.path.exists(netlist_file) + and result.returncode == 0 + ): clock_period = yosys_cfg.get("clock_period", 10.0) clock_name = yosys_cfg.get("clock_name", "clock") vcd = input_data.get("vcd") or yosys_cfg.get("vcd") @@ -182,10 +219,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: # naming convention. Some small modules are combinational and # therefore have no clock at all. f.write(f"set bb_clock_port [get_ports -quiet {clock_name}]\n") - f.write("if {[llength $bb_clock_port] == 0} { set bb_clock_port [get_ports -quiet *clk*] }\n") - f.write(f"if {{[llength $bb_clock_port] > 0}} {{ create_clock $bb_clock_port -name clk -period {clock_period} }}\n") - f.write(f"report_checks -path_delay max -format full > {timing_report_file}\n") - f.write(f"report_checks -path_delay max -format full\n") + f.write( + "if {[llength $bb_clock_port] == 0} { set bb_clock_port [get_ports" + " -quiet *clk*] }\n" + ) + f.write( + "if {[llength $bb_clock_port] > 0} { create_clock $bb_clock_port" + f" -name clk -period {clock_period} }}\n" + ) + f.write( + f"report_checks -path_delay max -format full > {timing_report_file}\n" + ) + f.write("report_checks -path_delay max -format full\n") if vcd and os.path.exists(vcd): f.write(f"read_vcd {vcd} -scope {top_module}\n") extra["power_activity_source"] = vcd @@ -194,11 +239,17 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: input_duty = yosys_cfg.get("input_duty", 0.5) clock_activity = yosys_cfg.get("clock_activity", 1.0) clock_duty = yosys_cfg.get("clock_duty", 0.5) - f.write(f"set_power_activity -input -activity {input_activity} -duty {input_duty}\n") + f.write( + f"set_power_activity -input -activity {input_activity} -duty" + f" {input_duty}\n" + ) # OpenSTA expects the clock collection after ``-clock``. # Applying the default rate to all clocks keeps this usable # for generated designs whose clock port is renamed. - f.write(f"if {{[llength [all_clocks]] > 0}} {{ set_power_activity -clock [all_clocks] -activity {clock_activity} -duty {clock_duty} }}\n") + f.write( + "if {[llength [all_clocks]] > 0} { set_power_activity -clock" + f" [all_clocks] -activity {clock_activity} -duty {clock_duty} }}\n" + ) extra["power_activity_source"] = "default" f.write(f"report_power -digits 4 > {power_report_file}\n") f.write("exit\n") diff --git a/bbdev/api/steps/yosys/scripts/yosys_log.py b/bbdev/api/steps/yosys/scripts/yosys_log.py index a79c3deef..c0bcd8565 100644 --- a/bbdev/api/steps/yosys/scripts/yosys_log.py +++ b/bbdev/api/steps/yosys/scripts/yosys_log.py @@ -1,12 +1,2 @@ -import os -from datetime import datetime - - def req_arg(body: dict, name: str): return body.get(name) or body.get(name.replace("_", "-")) - - -def make_yosys_log_dir(bbdir: str, trace_id: str) -> str: - stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - suffix = trace_id[:8] if trace_id else "no-trace" - return os.path.join(bbdir, "bbdev", "api", "steps", "yosys", "log", f"{stamp}-{suffix}") diff --git a/bbdev/api/tests/test_all_api_serial.py b/bbdev/api/tests/test_all_api_serial.py index d2b8510d9..1b59f4279 100644 --- a/bbdev/api/tests/test_all_api_serial.py +++ b/bbdev/api/tests/test_all_api_serial.py @@ -5,51 +5,51 @@ def main(): - test_dir = Path(__file__).resolve().parent - tests = [ - "test_api_compiler_build.py", - "test_api_workload_clean.py", - "test_api_workload_build.py", - "test_api_verilator_clean.py", - "test_api_verilator_verilog.py", - "test_api_verilator_build.py", - "test_api_verilator_sim.py", - "test_api_verilator_run.py", - "test_api_yosys_run.py", - "test_api_yosys_verilog.py", - "test_api_yosys_synth.py", - "test_api_dc_verilog.py", - "test_api_uvm_build.py", - "test_api_uvm_run.py", - "test_api_kernel_build.py", - "test_api_firesim_enumeratefpgas.py", - "test_api_firesim_buildbitstream.py", - "test_api_firesim_infrasetup.py", - "test_api_firesim_runworkload.py", - "test_api_bebop_bemu_sim.py", - "test_api_bebop_bemu_analysis.py", - "test_api_bebop_bemu_batch.py", - "test_api_bebop_verilator_clean.py", - "test_api_bebop_verilator_verilog.py", - "test_api_bebop_verilator_build.py", - "test_api_bebop_verilator_sim.py", - "test_api_bebop_verilator_run.py", - "test_api_bebop_verilator_batch.py", - "test_api_bebop_p2e_clean.py", - "test_api_bebop_p2e_verilog.py", - "test_api_bebop_p2e_buildbitstream.py", - "test_api_bebop_p2e_runworkload.py", - ] - for name in tests: - path = test_dir / name - if not path.exists(): - raise RuntimeError(f"missing test: {name}") - print(f"RUN {name}") - ret = subprocess.run(["python", str(path)], cwd=str(test_dir)) - if ret.returncode != 0: - raise RuntimeError(f"failed: {name}") - print(f"PASS: {len(tests)} api tests done.") + test_dir = Path(__file__).resolve().parent + tests = [ + "test_api_stack_build.py", + "test_api_workload_clean.py", + "test_api_workload_build.py", + "test_api_verilator_clean.py", + "test_api_verilator_verilog.py", + "test_api_verilator_build.py", + "test_api_verilator_sim.py", + "test_api_verilator_run.py", + "test_api_yosys_run.py", + "test_api_yosys_verilog.py", + "test_api_yosys_synth.py", + "test_api_dc_verilog.py", + "test_api_uvm_build.py", + "test_api_uvm_run.py", + "test_api_kernel_build.py", + "test_api_firesim_enumeratefpgas.py", + "test_api_firesim_buildbitstream.py", + "test_api_firesim_infrasetup.py", + "test_api_firesim_runworkload.py", + "test_api_bebop_bemu_sim.py", + "test_api_bebop_bemu_analysis.py", + "test_api_bebop_bemu_batch.py", + "test_api_bebop_verilator_clean.py", + "test_api_bebop_verilator_verilog.py", + "test_api_bebop_verilator_build.py", + "test_api_bebop_verilator_sim.py", + "test_api_bebop_verilator_run.py", + "test_api_bebop_verilator_batch.py", + "test_api_bebop_p2e_clean.py", + "test_api_bebop_p2e_verilog.py", + "test_api_bebop_p2e_buildbitstream.py", + "test_api_bebop_p2e_runworkload.py", + ] + for name in tests: + path = test_dir / name + if not path.exists(): + raise RuntimeError(f"missing test: {name}") + print(f"RUN {name}") + ret = subprocess.run(["python", str(path)], cwd=str(test_dir)) + if ret.returncode != 0: + raise RuntimeError(f"failed: {name}") + print(f"PASS: {len(tests)} api tests done.") if __name__ == "__main__": - main() + main() diff --git a/bbdev/api/tests/test_api_bebop_bemu_analysis.py b/bbdev/api/tests/test_api_bebop_bemu_analysis.py index 542aba417..d4ed24226 100644 --- a/bbdev/api/tests/test_api_bebop_bemu_analysis.py +++ b/bbdev/api/tests/test_api_bebop_bemu_analysis.py @@ -1,3 +1,32 @@ +from datetime import datetime +from pathlib import Path +import shlex +import shutil +import sys +from uuid import uuid4 + from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-bemu --analysis '--chip pebble --log-dir fixtures/bemu-analysis --itrace --mtrace'") +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from utils.path import get_buckyball_path, log_dir + +run_log = Path( + log_dir( + get_buckyball_path(), + "pebble", + "verilog", + datetime.now().strftime("%Y-%m-%d-%H-%M"), + "bemu", + f"analysis-test-{uuid4().hex}", + ) +) +run_log.mkdir(parents=True) +try: + shutil.copyfile( + Path(__file__).parent / "fixtures/bemu-analysis/bdb.ndjson", + run_log / "bdb.ndjson", + ) + arguments = f"--design pebble --run {run_log.name} --itrace --mtrace" + run_bbdev_case(f"bbdev bebop-bemu --analysis {shlex.quote(arguments)}") +finally: + shutil.rmtree(run_log) diff --git a/bbdev/api/tests/test_api_bebop_bemu_batch.py b/bbdev/api/tests/test_api_bebop_bemu_batch.py index 595ba40b6..94d11e0a4 100644 --- a/bbdev/api/tests/test_api_bebop_bemu_batch.py +++ b/bbdev/api/tests/test_api_bebop_bemu_batch.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-bemu --batch '--chip toy --test bare-tests'") +run_bbdev_case("bbdev bebop-bemu --batch '--design toy --test bare-tests'") diff --git a/bbdev/api/tests/test_api_bebop_bemu_sim.py b/bbdev/api/tests/test_api_bebop_bemu_sim.py index f72013459..9f36f0ce1 100644 --- a/bbdev/api/tests/test_api_bebop_bemu_sim.py +++ b/bbdev/api/tests/test_api_bebop_bemu_sim.py @@ -1,4 +1,6 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_relu_test-singlecore-baremetal'") -run_bbdev_case("bbdev bebop-bemu --sim '--chip toy --binary toy_relu_test-linux'") +run_bbdev_case( + "bbdev bebop-bemu --sim '--design toy --binary toy_relu_test-singlecore-baremetal'" +) +run_bbdev_case("bbdev bebop-bemu --sim '--design toy --binary toy_relu_test-linux'") diff --git a/bbdev/api/tests/test_api_bebop_p2e_buildbitstream.py b/bbdev/api/tests/test_api_bebop_p2e_buildbitstream.py index ddac6f048..e05880561 100644 --- a/bbdev/api/tests/test_api_bebop_p2e_buildbitstream.py +++ b/bbdev/api/tests/test_api_bebop_p2e_buildbitstream.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-p2e --buildbitstream '--chip toy'") +run_bbdev_case("bbdev bebop-p2e --buildbitstream '--design toy'") diff --git a/bbdev/api/tests/test_api_bebop_p2e_clean.py b/bbdev/api/tests/test_api_bebop_p2e_clean.py index c67a885ad..e1b431233 100644 --- a/bbdev/api/tests/test_api_bebop_p2e_clean.py +++ b/bbdev/api/tests/test_api_bebop_p2e_clean.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-p2e --clean '--chip toy'") +run_bbdev_case("bbdev bebop-p2e --clean '--design toy'") diff --git a/bbdev/api/tests/test_api_bebop_p2e_runworkload.py b/bbdev/api/tests/test_api_bebop_p2e_runworkload.py index 1d1b26d85..06a828bbb 100644 --- a/bbdev/api/tests/test_api_bebop_p2e_runworkload.py +++ b/bbdev/api/tests/test_api_bebop_p2e_runworkload.py @@ -1,3 +1,7 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-p2e --runworkload '--chip toy --image vecunit_matmul_ones --bitstream /home/wanghui/Code/buckyball/bebop/build/p2e/sims.p2e.P2EToyConfig/bitstream.bit'") +run_bbdev_case( + "bbdev bebop-p2e --runworkload '--design toy --image vecunit_matmul_ones" + " --bitstream" + " /home/wanghui/Code/buckyball/bebop/build/p2e/sims.p2e.P2EToyConfig/bitstream.bit'" +) diff --git a/bbdev/api/tests/test_api_bebop_p2e_verilog.py b/bbdev/api/tests/test_api_bebop_p2e_verilog.py index b3363f1ad..a50fa4f19 100644 --- a/bbdev/api/tests/test_api_bebop_p2e_verilog.py +++ b/bbdev/api/tests/test_api_bebop_p2e_verilog.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-p2e --verilog '--chip toy'") +run_bbdev_case("bbdev bebop-p2e --verilog '--design toy'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_batch.py b/bbdev/api/tests/test_api_bebop_verilator_batch.py index dd4077030..f81ba8d21 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_batch.py +++ b/bbdev/api/tests/test_api_bebop_verilator_batch.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --batch '--chip toy --test bare-tests'") +run_bbdev_case("bbdev bebop-verilator --batch '--design toy --test bare-tests'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_build.py b/bbdev/api/tests/test_api_bebop_verilator_build.py index 9dee15463..73a39f88d 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_build.py +++ b/bbdev/api/tests/test_api_bebop_verilator_build.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --build '--jobs 16 --chip toy'") +run_bbdev_case("bbdev bebop-verilator --build '--jobs 16 --design toy'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_clean.py b/bbdev/api/tests/test_api_bebop_verilator_clean.py index 780de8680..f9ce532a3 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_clean.py +++ b/bbdev/api/tests/test_api_bebop_verilator_clean.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --clean '--chip toy'") +run_bbdev_case("bbdev bebop-verilator --clean '--design toy'") diff --git a/bbdev/api/tests/test_api_bebop_verilator_run.py b/bbdev/api/tests/test_api_bebop_verilator_run.py index 712e0479f..3a9bede5e 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_run.py +++ b/bbdev/api/tests/test_api_bebop_verilator_run.py @@ -1,3 +1,5 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --run '--diff --no-wave --jobs 16 --binary toy_relu_test-singlecore-baremetal --chip toy'") +run_bbdev_case( + "bbdev bebop-verilator --run '--diff --no-wave --jobs 16 --binary toy_relu_test-singlecore-baremetal --design toy'" +) diff --git a/bbdev/api/tests/test_api_bebop_verilator_sim.py b/bbdev/api/tests/test_api_bebop_verilator_sim.py index 4b4cccdc8..2472e5e2e 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_sim.py +++ b/bbdev/api/tests/test_api_bebop_verilator_sim.py @@ -1,3 +1,6 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --sim '--binary toy_relu_test-singlecore-baremetal --chip toy --itrace --mtrace --pmctrace --ctrace --banktrace'") +run_bbdev_case( + "bbdev bebop-verilator --sim '--binary toy_relu_test-singlecore-baremetal --design" + " toy --itrace --mtrace --pmctrace --ctrace --banktrace'" +) diff --git a/bbdev/api/tests/test_api_bebop_verilator_verilog.py b/bbdev/api/tests/test_api_bebop_verilator_verilog.py index 1702bc0ae..8a6be92cc 100644 --- a/bbdev/api/tests/test_api_bebop_verilator_verilog.py +++ b/bbdev/api/tests/test_api_bebop_verilator_verilog.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev bebop-verilator --verilog '--chip toy'") +run_bbdev_case("bbdev bebop-verilator --verilog '--design toy'") diff --git a/bbdev/api/tests/test_api_compiler_build.py b/bbdev/api/tests/test_api_compiler_build.py deleted file mode 100644 index ef786df81..000000000 --- a/bbdev/api/tests/test_api_compiler_build.py +++ /dev/null @@ -1,3 +0,0 @@ -from _api_test_helper import run_bbdev_case - -run_bbdev_case("bbdev compiler --build '--chip toy'") diff --git a/bbdev/api/tests/test_api_dc_verilog.py b/bbdev/api/tests/test_api_dc_verilog.py index a6247c22f..b8c5533d0 100644 --- a/bbdev/api/tests/test_api_dc_verilog.py +++ b/bbdev/api/tests/test_api_dc_verilog.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev dc --verilog '--chip toy'") +run_bbdev_case("bbdev dc --verilog '--design toy'") diff --git a/bbdev/api/tests/test_api_firesim_buildbitstream.py b/bbdev/api/tests/test_api_firesim_buildbitstream.py index 3edefe465..320c515bf 100644 --- a/bbdev/api/tests/test_api_firesim_buildbitstream.py +++ b/bbdev/api/tests/test_api_firesim_buildbitstream.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev firesim --buildbitstream '--chip toy'") +run_bbdev_case("bbdev firesim --buildbitstream '--design toy'") diff --git a/bbdev/api/tests/test_api_firesim_enumeratefpgas.py b/bbdev/api/tests/test_api_firesim_enumeratefpgas.py index 337070c26..20654bbe4 100644 --- a/bbdev/api/tests/test_api_firesim_enumeratefpgas.py +++ b/bbdev/api/tests/test_api_firesim_enumeratefpgas.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev firesim --enumeratefpgas '--chip toy'") +run_bbdev_case("bbdev firesim --enumeratefpgas '--design toy'") diff --git a/bbdev/api/tests/test_api_firesim_infrasetup.py b/bbdev/api/tests/test_api_firesim_infrasetup.py index 7a619caa5..ac0a02ede 100644 --- a/bbdev/api/tests/test_api_firesim_infrasetup.py +++ b/bbdev/api/tests/test_api_firesim_infrasetup.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev firesim --infrasetup '--chip toy'") +run_bbdev_case("bbdev firesim --infrasetup '--design toy'") diff --git a/bbdev/api/tests/test_api_firesim_runworkload.py b/bbdev/api/tests/test_api_firesim_runworkload.py index e9c941513..b46c3a665 100644 --- a/bbdev/api/tests/test_api_firesim_runworkload.py +++ b/bbdev/api/tests/test_api_firesim_runworkload.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev firesim --runworkload '--chip toy'") +run_bbdev_case("bbdev firesim --runworkload '--design toy'") diff --git a/bbdev/api/tests/test_api_kernel_build.py b/bbdev/api/tests/test_api_kernel_build.py index ffb41469b..732a5e8db 100644 --- a/bbdev/api/tests/test_api_kernel_build.py +++ b/bbdev/api/tests/test_api_kernel_build.py @@ -1,4 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev kernel --build") -run_bbdev_case("bbdev kernel --build '--hart-count 64'") +run_bbdev_case("bbdev kernel --build '--design toy'") diff --git a/bbdev/api/tests/test_api_stack_build.py b/bbdev/api/tests/test_api_stack_build.py new file mode 100644 index 000000000..5fabf4627 --- /dev/null +++ b/bbdev/api/tests/test_api_stack_build.py @@ -0,0 +1,3 @@ +from _api_test_helper import run_bbdev_case + +run_bbdev_case("bbdev stack --build '--design toy'") diff --git a/bbdev/api/tests/test_api_uvm_build.py b/bbdev/api/tests/test_api_uvm_build.py index eba674164..491866b53 100644 --- a/bbdev/api/tests/test_api_uvm_build.py +++ b/bbdev/api/tests/test_api_uvm_build.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev uvm --build '--chip toy --ip axis'") +run_bbdev_case("bbdev uvm --build '--design toy --ip axis'") diff --git a/bbdev/api/tests/test_api_uvm_run.py b/bbdev/api/tests/test_api_uvm_run.py index eaca26f4a..dad9a320e 100644 --- a/bbdev/api/tests/test_api_uvm_run.py +++ b/bbdev/api/tests/test_api_uvm_run.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev uvm --run '--chip toy --ip axis'") +run_bbdev_case("bbdev uvm --run '--design toy --ip axis'") diff --git a/bbdev/api/tests/test_api_verilator_build.py b/bbdev/api/tests/test_api_verilator_build.py index 300da1ff2..3440ef00c 100644 --- a/bbdev/api/tests/test_api_verilator_build.py +++ b/bbdev/api/tests/test_api_verilator_build.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --build '--jobs 16 --chip toy'") +run_bbdev_case("bbdev verilator --build '--jobs 16 --design toy'") diff --git a/bbdev/api/tests/test_api_verilator_clean.py b/bbdev/api/tests/test_api_verilator_clean.py index e94bbb161..ee2f8a631 100644 --- a/bbdev/api/tests/test_api_verilator_clean.py +++ b/bbdev/api/tests/test_api_verilator_clean.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --clean '--chip toy'") +run_bbdev_case("bbdev verilator --clean '--design toy'") diff --git a/bbdev/api/tests/test_api_verilator_run.py b/bbdev/api/tests/test_api_verilator_run.py index c12691182..821209439 100644 --- a/bbdev/api/tests/test_api_verilator_run.py +++ b/bbdev/api/tests/test_api_verilator_run.py @@ -1,3 +1,5 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --run '--jobs 16 --binary toy-toy-ctest-mvin_mvout_test-baremetal --chip toy --batch'") +run_bbdev_case( + "bbdev verilator --run '--jobs 16 --binary toy-toy-ctest-mvin_mvout_test-baremetal --design toy --batch'" +) diff --git a/bbdev/api/tests/test_api_verilator_sim.py b/bbdev/api/tests/test_api_verilator_sim.py index 6c9ad70b2..6b0aea79b 100644 --- a/bbdev/api/tests/test_api_verilator_sim.py +++ b/bbdev/api/tests/test_api_verilator_sim.py @@ -1,3 +1,5 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --sim '--binary toy-toy-ctest-mvin_mvout_test-baremetal --batch --chip toy'") +run_bbdev_case( + "bbdev verilator --sim '--binary toy-toy-ctest-mvin_mvout_test-baremetal --batch --design toy'" +) diff --git a/bbdev/api/tests/test_api_verilator_verilog.py b/bbdev/api/tests/test_api_verilator_verilog.py index ddc5e8882..1c4b29475 100644 --- a/bbdev/api/tests/test_api_verilator_verilog.py +++ b/bbdev/api/tests/test_api_verilator_verilog.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev verilator --verilog '--chip toy'") +run_bbdev_case("bbdev verilator --verilog '--design toy'") diff --git a/bbdev/api/tests/test_api_workload_build.py b/bbdev/api/tests/test_api_workload_build.py index f4e5b5bef..54c3dcc97 100644 --- a/bbdev/api/tests/test_api_workload_build.py +++ b/bbdev/api/tests/test_api_workload_build.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev workload --build '--chip toy'") +run_bbdev_case("bbdev workload --build '--design toy'") diff --git a/bbdev/api/tests/test_api_workload_clean.py b/bbdev/api/tests/test_api_workload_clean.py index da87ffbb8..fc83995a8 100644 --- a/bbdev/api/tests/test_api_workload_clean.py +++ b/bbdev/api/tests/test_api_workload_clean.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev workload --clean '--chip toy'") +run_bbdev_case("bbdev workload --clean '--design toy'") diff --git a/bbdev/api/tests/test_api_workload_tohex.py b/bbdev/api/tests/test_api_workload_tohex.py index 5f3d36964..6faf6996a 100644 --- a/bbdev/api/tests/test_api_workload_tohex.py +++ b/bbdev/api/tests/test_api_workload_tohex.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev workload --tohex '--chip toy'") +run_bbdev_case("bbdev workload --tohex '--design toy'") diff --git a/bbdev/api/tests/test_api_yosys_run.py b/bbdev/api/tests/test_api_yosys_run.py index c8d133217..bcaa238e3 100644 --- a/bbdev/api/tests/test_api_yosys_run.py +++ b/bbdev/api/tests/test_api_yosys_run.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev yosys --run '--chip toy'") +run_bbdev_case("bbdev yosys --run '--design toy'") diff --git a/bbdev/api/tests/test_api_yosys_synth.py b/bbdev/api/tests/test_api_yosys_synth.py index 3f88bc892..032b9314e 100644 --- a/bbdev/api/tests/test_api_yosys_synth.py +++ b/bbdev/api/tests/test_api_yosys_synth.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev yosys --synth '--chip toy'") +run_bbdev_case("bbdev yosys --synth '--design toy'") diff --git a/bbdev/api/tests/test_api_yosys_verilog.py b/bbdev/api/tests/test_api_yosys_verilog.py index 057ad8cf8..a899a059c 100644 --- a/bbdev/api/tests/test_api_yosys_verilog.py +++ b/bbdev/api/tests/test_api_yosys_verilog.py @@ -1,3 +1,3 @@ from _api_test_helper import run_bbdev_case -run_bbdev_case("bbdev yosys --verilog '--chip toy'") +run_bbdev_case("bbdev yosys --verilog '--design toy'") diff --git a/bbdev/api/tests/test_bemu_binary.py b/bbdev/api/tests/test_bemu_binary.py deleted file mode 100644 index b52734427..000000000 --- a/bbdev/api/tests/test_bemu_binary.py +++ /dev/null @@ -1,50 +0,0 @@ -import importlib.util -from pathlib import Path -import tempfile -import unittest - - -API = Path(__file__).resolve().parents[1] -spec = importlib.util.spec_from_file_location( - "bemu_sim_event", API / "steps/bebop/bemu/02_sim_event.step.py" -) -event = importlib.util.module_from_spec(spec) -spec.loader.exec_module(event) - - -class BemuBinaryResolutionTest(unittest.TestCase): - def setUp(self): - self.temporary = tempfile.TemporaryDirectory() - self.addCleanup(self.temporary.cleanup) - self.root = Path(self.temporary.name) - - def binary(self, relative): - path = self.root / "bb-tests/output" / relative - path.parent.mkdir(parents=True, exist_ok=True) - path.touch() - return str(path) - - def test_same_model_image_on_two_chips(self): - for chip in ("toy", "pebble"): - self.binary(f"kernel/{chip}/fw_payload-alexnet.elf") - for chip in ("toy", "pebble"): - expected = self.root / f"bb-tests/output/kernel/{chip}/fw_payload-alexnet.elf" - self.assertEqual( - event.resolve_bemu_binary(str(self.root), chip, expected.name), - str(expected), - ) - - def test_workload_must_belong_to_requested_chip(self): - path = self.binary("toy/workloads/src/shared-test") - self.assertEqual(event.resolve_bemu_binary(str(self.root), "toy", "shared-test"), path) - self.assertIsNone(event.resolve_bemu_binary(str(self.root), "pebble", "shared-test")) - - def test_ambiguous_workload_within_chip_is_an_error(self): - self.binary("toy/workloads/src/one/shared-test") - self.binary("toy/workloads/src/two/shared-test") - with self.assertRaises(ValueError): - event.resolve_bemu_binary(str(self.root), "toy", "shared-test") - - -if __name__ == "__main__": - unittest.main() diff --git a/bbdev/api/tests/test_p2e_runtime_reuse.py b/bbdev/api/tests/test_p2e_runtime_reuse.py deleted file mode 100644 index 2657e0a6b..000000000 --- a/bbdev/api/tests/test_p2e_runtime_reuse.py +++ /dev/null @@ -1,54 +0,0 @@ -from pathlib import Path -import tempfile -import sys -import unittest - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from steps.bebop.p2e.scripts.runtime_case import validate_runtime_reuse - - -class RuntimeReuseTest(unittest.TestCase): - def setUp(self): - self.directory = tempfile.TemporaryDirectory() - self.addCleanup(self.directory.cleanup) - self.case = Path(self.directory.name) - self.bit = self.case / "fpgaCompDir/bitstream.bit" - for path in (self.bit, self.case / "bebop-p2e", - self.case / "vvacDir/runtimeDir/rtcfg", - self.case / "vvacDir/runtimeDir/lib/lib_arm/libvCtb.so"): - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(b"existing artifact") - self.mode = self.case / "p2e_trace_mode" - self.mode.write_text("btrace") - (self.case / "bebop-p2e").chmod(0o755) - - def test_valid_case_remains_unchanged(self): - before = {p: p.read_bytes() for p in self.case.rglob("*") if p.is_file()} - self.assertEqual(validate_runtime_reuse(str(self.bit), True), - self.case / "bebop-p2e") - self.assertEqual(before, {p: p.read_bytes() for p in self.case.rglob("*") if p.is_file()}) - - def test_missing_library_is_rejected(self): - (self.case / "vvacDir/runtimeDir/lib/lib_arm/libvCtb.so").unlink() - with self.assertRaisesRegex(ValueError, "incomplete"): - validate_runtime_reuse(str(self.bit), True) - - def test_diff_requires_trace_capability(self): - self.mode.write_text("none") - with self.assertRaisesRegex(ValueError, "does not support diff"): - validate_runtime_reuse(str(self.bit), True) - validate_runtime_reuse(str(self.bit), False) - - def test_unknown_protocol_is_rejected(self): - self.mode.write_text("unknown") - with self.assertRaisesRegex(ValueError, "Unknown"): - validate_runtime_reuse(str(self.bit), False) - - def test_nonexecutable_runtime_is_rejected(self): - (self.case / "bebop-p2e").chmod(0o644) - with self.assertRaisesRegex(ValueError, "not executable"): - validate_runtime_reuse(str(self.bit), True) - - -if __name__ == "__main__": - unittest.main() diff --git a/bbdev/api/tests/test_uvm_ip_profiles.py b/bbdev/api/tests/test_uvm_ip_profiles.py deleted file mode 100644 index dbfdf4f62..000000000 --- a/bbdev/api/tests/test_uvm_ip_profiles.py +++ /dev/null @@ -1,51 +0,0 @@ -import sys -import tempfile -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from steps.uvm.scripts.uvm_common import load_ip -from steps.uvm.scripts.report import verification_targets - - -class IpProfiles(unittest.TestCase): - def setUp(self): - self.temp = tempfile.TemporaryDirectory() - self.repo = Path(self.temp.name) - registry = self.repo / "arch/uvm.toml" - registry.parent.mkdir(parents=True) - registry.write_text('ips=["unit"]\n[paths]\nunit="ip"\n') - self.resources = self.repo / "ip/src/main/resources" - self.resources.mkdir(parents=True) - (self.resources / "common.f").write_text("") - (self.resources / "goban.f").write_text("") - (self.resources / "goban.toml").write_text('chips=["goban"]\n') - - def tearDown(self): - self.temp.cleanup() - - def test_build_and_report_choose_same_profile(self): - for chip, names in [("toy", ["common"]), ("goban", ["common", "goban"])]: - self.assertEqual([x["name"] for x in load_ip(str(self.repo), "unit", chip=chip)["targets"]], names) - self.assertEqual(verification_targets(self.repo, chip, ip="unit"), [f"ip/unit/{n}" for n in names]) - with self.assertRaisesRegex(ValueError, "does not support chip"): - load_ip(str(self.repo), "unit", "goban", "toy") - - def test_profile_and_negative_assertion_can_combine(self): - (self.resources / "goban.toml").write_text('chips=["goban"]\nexpected_assertion="precise failure"\n') - target = load_ip(str(self.repo), "unit", "goban", "goban")["targets"][0] - self.assertEqual(target["expected_assertion"], "precise failure") - - def test_invalid_metadata_and_empty_selection_fail(self): - for value in ['[]', '[""]', '[" goban"]', '[7]', '"goban"']: - (self.resources / "goban.toml").write_text(f"chips={value}\n") - with self.assertRaisesRegex(ValueError, "nonempty list"): - load_ip(str(self.repo), "unit", chip="goban") - (self.resources / "goban.toml").write_text('chips=["goban"]\n') - (self.resources / "common.toml").write_text('chips=["goban"]\n') - with self.assertRaisesRegex(ValueError, "no applicable targets"): - load_ip(str(self.repo), "unit", chip="toy") - - -if __name__ == "__main__": - unittest.main() diff --git a/bbdev/api/unit_tests/test_process_cleanup.py b/bbdev/api/unit_tests/test_process_cleanup.py deleted file mode 100644 index 7fe5f8246..000000000 --- a/bbdev/api/unit_tests/test_process_cleanup.py +++ /dev/null @@ -1,72 +0,0 @@ -import os -import shlex -import signal -import subprocess -import sys -import tempfile -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from utils.process import _exists, terminate_group -from utils.process_registry import cancel_process, register_process, unregister_process -from utils.stream_run import stream_run - - -class ProcessCleanupTest(unittest.TestCase): - def test_nested_session_exits_on_shutdown_and_cancel(self): - for cancel in (False, True): - with self.subTest(cancel=cancel): - parent = subprocess.Popen( - [ - sys.executable, - "-c", - "import subprocess, sys, time; " - "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(300)'], start_new_session=True); " - "print(child.pid, flush=True); time.sleep(300)", - ], - stdout=subprocess.PIPE, - text=True, - start_new_session=True, - ) - child_pid = int(parent.stdout.readline()) - try: - if cancel: - register_process("nested-session-test", parent.pid) - self.assertTrue(cancel_process("nested-session-test")) - parent.wait(timeout=2) - else: - terminate_group(parent, term_timeout=1) - self.assertFalse(_exists(child_pid)) - finally: - unregister_process("nested-session-test") - if _exists(child_pid): - os.kill(child_pid, signal.SIGKILL) - if parent.poll() is None: - parent.kill() - parent.wait() - parent.stdout.close() - - def test_timeout_exits_nested_session(self): - with tempfile.TemporaryDirectory() as directory: - child_pid_file = Path(directory) / "child.pid" - command = shlex.join( - [ - sys.executable, - "-c", - "import subprocess, sys, time; " - "child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(300)'], start_new_session=True); " - "open(sys.argv[1], 'w').write(str(child.pid)); time.sleep(300)", - str(child_pid_file), - ] - ) - try: - result = stream_run(command, timeout=1) - self.assertNotEqual(result.returncode, 0) - self.assertFalse(_exists(int(child_pid_file.read_text()))) - finally: - if child_pid_file.exists(): - child_pid = int(child_pid_file.read_text()) - if _exists(child_pid): - os.kill(child_pid, signal.SIGKILL) diff --git a/bbdev/api/utils/event_common.py b/bbdev/api/utils/event_common.py index b7574390b..d307a1e3d 100644 --- a/bbdev/api/utils/event_common.py +++ b/bbdev/api/utils/event_common.py @@ -6,18 +6,38 @@ from utils.process_registry import set_current_task_scope -def require_chip(data: dict) -> str: - chip = data.get("chip") - if not isinstance(chip, str) or not chip: - raise ValueError("Missing required parameter: --chip") + +def require_design(data: dict) -> str: + design = data.get("design") + if not isinstance(design, str) or not design: + raise ValueError("Missing required parameter: --design") extra = data.get("config") if isinstance(extra, str) and extra and extra != "None": raise ValueError( - "do not pass --config; mill class comes from chip config sims" + "do not pass --config; mill class comes from design config sims" + ) + if not re.fullmatch(r"[A-Za-z0-9_-]+", design): + raise ValueError(f"invalid design: {design}") + return design + + +def require_target(data: dict, kinds=("device", "core")) -> str: + selected = [kind for kind in ("device", "core", "ip", "ball") if kind in data] + if len(selected) != 1: + raise ValueError( + "Specify exactly one of --device, --core , --ip , --ball after --design" ) - if not re.fullmatch(r"[A-Za-z0-9_-]+", chip): - raise ValueError(f"invalid chip: {chip}") - return chip + kind = selected[0] + if kind not in kinds: + raise ValueError(f"--{kind} is not supported by this operation") + value = data[kind] + if kind == "device": + if value is not True: + raise ValueError("--device is a flag and takes no value") + return "device" + if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", value): + raise ValueError(f"--{kind} requires an entry name") + return f"{kind}/{value}" def get_origin_trace_id(input_data, ctx): @@ -26,12 +46,18 @@ def get_origin_trace_id(input_data, ctx): iii 0.7+ assigns a new trace_id per handler invocation, so the API step's trace_id must be forwarded explicitly via input_data["_trace_id"]. """ - trace_id = input_data["_trace_id"] if isinstance(input_data, dict) and "_trace_id" in input_data else ctx.trace_id + trace_id = ( + input_data["_trace_id"] + if isinstance(input_data, dict) and "_trace_id" in input_data + else ctx.trace_id + ) set_current_task_scope(trace_id) return trace_id -async def check_result(ctx, returncode, continue_run=False, extra_fields=None, trace_id=None): +async def check_result( + ctx, returncode, continue_run=False, extra_fields=None, trace_id=None +): """ Check returncode, create appropriate result objects and set state. diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index 4f2214c42..58e223844 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -1,6 +1,7 @@ import json import os import re +import sys from pathlib import Path @@ -15,85 +16,131 @@ def get_buckyball_path(): return root -def _chip_name(chip): - if not isinstance(chip, str) or not chip or chip == "None": - raise ValueError("missing required parameter: chip") - if not re.fullmatch("[A-Za-z0-9_.-]+", chip): - raise ValueError(f"invalid chip name: {chip}") - return chip +def _design_name(design): + if not isinstance(design, str) or not design or design == "None": + raise ValueError("missing required parameter: design") + if not re.fullmatch("[A-Za-z0-9_.-]+", design): + raise ValueError(f"invalid design name: {design}") + return design -def chip_output_root(bbdir, chip): - chip = _chip_name(chip) - return os.path.join(bbdir, "bb-tests", "output", chip) +def design_output_root(bbdir, design, *, selection): + design = _design_name(design) + return os.path.join(bbdir, "examples/designs", design, "output/sims", selection) def workloads_output_root(bbdir): return os.path.join(bbdir, "bb-tests", "output") -def workload_build_dir(bbdir, chip): - chip = _chip_name(chip) - return os.path.join(bbdir, "bb-tests", "workloads", "build", chip) +def workload_build_dir(bbdir, design, *, selection): + design = _design_name(design) + return os.path.join( + design_output_root(bbdir, design, selection=selection), "workloads-build" + ) -def workload_tests_root(bbdir, chip): - return os.path.join(chip_output_root(bbdir, chip), "workloads", "src") +def workload_tests_root(bbdir, design, *, selection): + return os.path.join( + design_output_root(bbdir, design, selection=selection), "workloads", "src" + ) -def bebop_target_dir(bbdir, chip): - chip = _chip_name(chip) - return os.path.join(bbdir, "bebop", "target", chip) +def bebop_target_dir(bbdir, design, backend, *, selection): + design = _design_name(design) + if backend not in {"bemu", "verilator", "p2e", "vcs"}: + raise ValueError(f"unknown Bebop backend: {backend}") + return os.path.join( + bbdir, + "examples", + "designs", + design, + "output", + "sims", + selection, + "emu", + backend, + "target", + ) -def bebop_cargo_env(bbdir, chip): - return {"CARGO_TARGET_DIR": bebop_target_dir(bbdir, chip)} +def bebop_bin_path(bbdir, design, backend, *, selection): + design = _design_name(design) + if backend not in {"bemu", "verilator", "p2e", "vcs"}: + raise ValueError(f"unknown Bebop backend: {backend}") + return os.path.join( + bbdir, + "examples", + "designs", + design, + "output", + "sims", + selection, + "bin", + f"bebop-{backend}", + ) -def chip_arch_root(bbdir, chip): - return os.path.join(bbdir, "arch", "build", _chip_name(chip)) +def design_arch_root(bbdir, design, *, selection): + return os.path.join( + bbdir, + "examples/designs", + _design_name(design), + "output/sims", + selection, + "arch", + ) -def sim_name(bbdir, chip, product): - if product == "verilog" or product == "synth": - sim_key = "verilator" - elif product == "tapeout": - sim_key = "tapeout" - elif product == "p2e": - sim_key = "p2e" - elif product == "firesim": - sim_key = "firesim" - else: - raise ValueError(f"invalid rtl product: {product}") +def load_design_config(bbdir, design): + sys.path.insert(0, str(Path(bbdir) / "bbdev/api/steps/config/scripts")) + import design_pb2 + path = ( Path(bbdir) - / "examples" - / "chips" - / _chip_name(chip) - / "configs" - / "generated" - / "config" - / "config.json" + / "examples/designs" + / _design_name(design) + / "output/generated/design.pb" ) - if not path.is_file(): - raise ValueError(f"missing {path}; run bbdev config --install") - sims = json.loads(path.read_text(encoding="utf-8"))["sims"] - if sim_key not in sims: - raise ValueError(f"chip {chip!r} has no sims.{sim_key} in {path}") - name = sims[sim_key] - return name + return design_pb2.Design.FromString(path.read_bytes()) + + +def simulation_target(bbdir, design, selection): + config = load_design_config(bbdir, design) + if selection not in config.simulations: + raise ValueError( + f"design {design!r} has no registered simulation entry {selection!r}" + ) + return config.simulations[selection] + +def runner_manifest(bbdir, design, selection): + """The simulation entry's emu crate: its model and the runners built from it.""" + return Path(bbdir) / simulation_target(bbdir, design, selection).emu / "Cargo.toml" -def rtl_dir(bbdir, chip, product, output_dir=None): - if output_dir: - return output_dir - return os.path.join(chip_arch_root(bbdir, chip), sim_name(bbdir, chip, product)) +def design_resource_dir(bbdir, design, name, *, selection): + resources = simulation_target(bbdir, design, selection).resources + if name not in resources: + raise ValueError(f"{design}/{selection} has no registered {name} directory") + return Path(bbdir) / resources[name] -def rtl_out(bbdir, chip, product, output_dir=None): - name = sim_name(bbdir, chip, product) - out = output_dir or os.path.join(chip_arch_root(bbdir, chip), name) - return (name, out) + +def sim_name(bbdir, design, product, *, selection): + target = simulation_target(bbdir, design, selection) + return "P2ETop" if product == "p2e" else target.top + + +def rtl_dir(bbdir, design, product, output_dir=None, *, selection): + name = sim_name(bbdir, design, product, selection=selection) + return output_dir or os.path.join( + design_arch_root(bbdir, design, selection=selection), name + ) + + +def rtl_out(bbdir, design, product, output_dir=None, *, selection): + name = sim_name(bbdir, design, product, selection=selection) + return name, rtl_dir(bbdir, design, product, output_dir, selection=selection) def _log_part(value, what): @@ -106,9 +153,21 @@ def _log_part(value, what): return value -def log_dir(bbdir, chip, product, stamp, tool, name, output_dir=None): - chip = _chip_name(chip) +def log_dir( + bbdir, design, product, stamp, tool, name, output_dir=None, *, selection=None +): + design = _design_name(design) stamp = _log_part(stamp, "timestamp") tool = _log_part(tool, "tool") name = _log_part(os.path.basename(name), "name") - return os.path.join(bbdir, "log", f"{stamp}-{chip}-{tool}-{name}") + if selection is not None: + name = _log_part(selection.replace("/", "-"), "target") + "-" + name + return os.path.join(bbdir, "log", f"{stamp}-{design}-{tool}-{name}") + + +def run_log_dir(bbdir, run): + root = (Path(bbdir) / "log").resolve() + path = (root / _log_part(run, "run")).resolve() + if path.parent != root: + raise ValueError(f"run must name a directory under {root}: {run!r}") + return path diff --git a/bbdev/api/utils/report_schema.py b/bbdev/api/utils/report_schema.py index dec86aae9..2427115d4 100644 --- a/bbdev/api/utils/report_schema.py +++ b/bbdev/api/utils/report_schema.py @@ -16,13 +16,13 @@ def report_directory(run): - if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", run["chip"]): - raise ValueError("chip must be a safe directory name") + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", run["design"]): + raise ValueError("design must be a safe directory name") timestamp = datetime.fromisoformat(run["timestamp"].replace("Z", "+00:00")) if timestamp.utcoffset() is None: raise ValueError("timestamp must include a timezone") - folder = timestamp.astimezone(timezone.utc).strftime("%Y-%m-%d-%H-%M") - return Path(run["chip"]) / folder + folder = run["id"] + return Path(run["design"]) / folder def nonempty(value, label): @@ -48,7 +48,7 @@ def number(value, label, *, positive=False, integer=False): def validate_run(run): - if run["schema_version"] != 2: + if run["schema_version"] != 3: raise ValueError("invalid schema_version") if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", run["id"]): raise ValueError("run id must be a safe file name") @@ -57,9 +57,9 @@ def validate_run(run): raise ValueError("timestamp must include a timezone") if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", run["repository"]): raise ValueError("repository must be owner/name") - if not re.fullmatch(r"[0-9a-f]{40}", run["commit"]): + if run["commit"] is not None and not re.fullmatch(r"[0-9a-f]{40}", run["commit"]): raise ValueError("commit must be a full 40-character SHA") - for key in ("branch", "chip", "backend", "config_hash", "producer"): + for key in ("branch", "design", "backend", "config_hash", "producer"): nonempty(run[key], key) if run["clock_hz"] is not None: number(run["clock_hz"], "clock_hz", positive=True) @@ -95,7 +95,9 @@ def validate_run(run): if targets - set(run["verification_targets"]): raise ValueError("verification results contain undeclared targets") names = set() - for model in run["models"]: + for model in run["workloads"]: + if model["kind"] not in {"model", "core", "ball"}: + raise ValueError("invalid workload kind") if model["latency_ms"] is not None: number(model["latency_ms"], "latency_ms", positive=True) if model["throughput"] is not None: @@ -115,10 +117,10 @@ def validate_run(run): raise ValueError("accuracy must be in [0,1]") nonempty(model["accuracy_metric"], "accuracy_metric") counter = model["counter"] - if counter not in ("riscv-cycle", "x86-tsc"): + if counter not in ("riscv-cycle", "rtl-cycle", "x86-tsc"): raise ValueError(f"unknown model counter: {counter}") - count_field = "cycles" if counter == "riscv-cycle" else "ticks" - other = "ticks" if counter == "riscv-cycle" else "cycles" + count_field = "cycles" if counter != "x86-tsc" else "ticks" + other = "ticks" if counter != "x86-tsc" else "cycles" if other in model: raise ValueError("model must not mix cycle and tick fields") if model["status"] == "pass": @@ -129,7 +131,12 @@ def validate_run(run): raise ValueError("passing model cannot contain an error") else: nonempty(model["error"], "model error") - if model[count_field] is not None or model["operators"]: + if ( + model[count_field] is not None + or model["operators"] + or model["latency_ms"] is not None + or model["throughput"] is not None + ): raise ValueError( "failed models cannot contain performance measurements" ) @@ -144,7 +151,7 @@ def validate_run(run): raise ValueError("unknown operator category") if op["counter"] != counter or other in op: raise ValueError("operator counter differs from its model") - unit = "cycle" if counter == "riscv-cycle" else "tick" + unit = "cycle" if counter != "x86-tsc" else "tick" opposite = "tick" if unit == "cycle" else "cycle" if f"start_{opposite}" in op or f"end_{opposite}" in op: raise ValueError("operator mixes cycle and tick intervals") @@ -162,37 +169,57 @@ def validate_run(run): def submission_errors(run): - missing = [] - if run["area"] is None: - missing.append("Area：缺少面积报告") - elif not run["area"]["constraints_met"]: - missing.append("Area：综合约束未满足") - if not run["models"]: - missing.append("Performance：没有模型结果") - for model in run["models"]: - if model["status"] != "pass": - missing.append(f"Performance：{model['name']} 未通过") - for key, label in ( - ("accuracy", "精度"), - ("latency_ms", "延迟"), - ("throughput", "吞吐"), - ): - if model[key] is None: - missing.append(f"Performance：{model['name']} 缺少{label}") - expected = set(run["verification_targets"]) - actual = {result["target"] for result in run["verification"]} - if not expected or expected != actual: - missing.append("Coverage：验证项不完整") - for result in run["verification"]: - if result["status"] != "pass" or result["coverage"] is None: - missing.append(f"Coverage：{result['target']} 未通过或缺少覆盖率报告") - if run.get("provenance", {}).get("source_dirty", False): - missing.append("源码包含未提交修改") - return missing + errors = [] + if not run["workloads"] and not run["verification"] and run["area"] is None: + errors.append("没有测量或验证结果") + if run["commit"] is None and not run.get("provenance", {}).get("executable_sha256"): + errors.append("缺少源码版本或执行产物哈希") + return errors def validate_submission(run): validate_run(run) + if run["verification"] or run["verification_targets"]: + raise ValueError( + "verification results belong in test reports, not the leaderboard" + ) + if run["backend"] == "dc": + if run["area"] is None or run["workloads"]: + raise ValueError( + "DC submissions require an area measurement without model results" + ) + if run["commit"] is None: + raise ValueError("DC area report requires a source revision") + else: + if run["area"] is not None: + raise ValueError( + "area measurements must be submitted as a separate DC report" + ) + if run["backend"] != "p2e" or not run["workloads"]: + raise ValueError("leaderboard requires FPGA end-to-end model results") + provenance = run["provenance"] + for field in ("bitstream_sha256", "executable_sha256"): + if not re.fullmatch(r"[0-9a-f]{64}", provenance.get(field, "")): + raise ValueError(f"FPGA model report requires {field}") + for result in run["workloads"]: + if ( + result["kind"] != "model" + or result["benchmark"]["scope"] != "model_execution" + ): + raise ValueError( + "core, Ball and operator tests are not leaderboard model runs" + ) + if result["counter"] != "riscv-cycle": + raise ValueError( + "FPGA model performance requires the device cycle counter" + ) + number( + result["benchmark"]["samples"], + "benchmark samples", + positive=True, + integer=True, + ) + nonempty(result["report_url"], "raw end-to-end benchmark log") missing = submission_errors(run) if missing: raise ValueError(f"submission blocked for {run['id']}: {'; '.join(missing)}") diff --git a/bbdev/api/utils/reports.py b/bbdev/api/utils/reports.py index 36e8539f8..5d03336c9 100644 --- a/bbdev/api/utils/reports.py +++ b/bbdev/api/utils/reports.py @@ -4,6 +4,7 @@ import subprocess from datetime import datetime, timezone from pathlib import Path +from utils.path import _design_name from utils.report_schema import ( report_directory, @@ -12,21 +13,12 @@ ) -def source_context(repo, chip): +def source_context(repo, design): repo = Path(repo) - - def plain(value): - if isinstance(value, dict): - return {key: plain(item) for key, item in value.items() if key != "_file"} - if isinstance(value, list): - return [plain(item) for item in value] - return value - - config = json.loads( - ( - repo / f"examples/chips/{chip}/configs/generated/config/config.json" - ).read_text() - ) + design = _design_name(design) + config = ( + repo / "examples/designs" / design / "output/generated/design.pb" + ).read_bytes() def git(*args): return subprocess.check_output(["git", *args], cwd=repo, text=True).strip() @@ -35,10 +27,8 @@ def git(*args): "repository": "DangoSys/buckyball", "branch": git("branch", "--show-current") or git("rev-parse", "HEAD"), "commit": git("rev-parse", "HEAD"), - "chip": chip, - "config_hash": hashlib.sha256( - json.dumps(plain(config), sort_keys=True).encode() - ).hexdigest(), + "design": design, + "config_hash": hashlib.sha256(config).hexdigest(), "provenance": { "source_dirty": bool( git( @@ -49,6 +39,7 @@ def git(*args): ) ), "workflow_url": None, + "comparable": True, }, } @@ -56,8 +47,8 @@ def git(*args): def new_report(context, producer, trace_id, backend): now = datetime.now(timezone.utc) return { - "schema_version": 2, - "id": f"{context['chip']}-{producer.replace('.', '-')}-{trace_id}", + "schema_version": 3, + "id": f"{context['design']}-{producer.replace('.', '-')}-{trace_id}", "timestamp": now.isoformat().replace("+00:00", "Z"), **context, "producer": producer, @@ -66,7 +57,7 @@ def new_report(context, producer, trace_id, backend): "area": None, "verification_targets": [], "verification": [], - "models": [], + "workloads": [], "artifacts": {}, } @@ -108,9 +99,10 @@ def verify_artifacts(report, directory): if row["report_url"] is not None ) references.extend( - model["trace_url"] - for model in report["models"] - if model["trace_url"] is not None + model[field] + for model in report["workloads"] + for field in ("trace_url", "report_url") + if model[field] is not None ) for url in references: if not url.startswith(prefix): diff --git a/bbdev/api/utils/workload_manifest.py b/bbdev/api/utils/workload_manifest.py index aa8e11516..d1902f1e1 100644 --- a/bbdev/api/utils/workload_manifest.py +++ b/bbdev/api/utils/workload_manifest.py @@ -1,30 +1,40 @@ import os from pathlib import Path -from .path import get_buckyball_path +from .path import get_buckyball_path, design_resource_dir -def validate_chip(chip: str | None) -> str: - if not chip: - raise ValueError("missing required parameter: chip") - if not isinstance(chip, str): - raise ValueError("chip must be a string") - if os.path.sep in chip or chip in {".", ".."}: - raise ValueError(f"invalid chip: {chip}") - return chip +def validate_design(design: str | None) -> str: + if not design: + raise ValueError("missing required parameter: design") + if not isinstance(design, str): + raise ValueError("design must be a string") + if os.path.sep in design or design in {".", ".."}: + raise ValueError(f"invalid design: {design}") + return design -def chip_regression_dir(chip: str, backend: str, bbdir: str | None = None) -> Path: - chip = validate_chip(chip) +def design_regression_dir( + design: str, backend: str, bbdir: str | None = None, *, selection: str +) -> Path: + if selection != "device": + raise ValueError( + "batch regression requires --device; use a single workload for --core" + ) + design = validate_design(design) root = Path(bbdir or get_buckyball_path()) - return root / "examples" / "chips" / chip / "regression" / "batch" / backend + return ( + design_resource_dir(root, design, "regression", selection=selection) / backend + ) def resolve_workload_toml( - chip: str, + design: str, backend: str, test_type: str, bbdir: str | None = None, diff: bool = False, + *, + selection: str, ) -> str: if test_type == "bare-tests": suffix = "bare" @@ -32,7 +42,7 @@ def resolve_workload_toml( suffix = "linux" else: raise ValueError(f"invalid test type: {test_type}") - regression_dir = chip_regression_dir(chip, backend, bbdir) + regression_dir = design_regression_dir(design, backend, bbdir, selection=selection) if diff: diff_toml = regression_dir / f"workloads-{suffix}-diff.toml" if diff_toml.is_file(): diff --git a/bbdev/bbdev b/bbdev/bbdev index 2051e5e5d..736b25ead 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -16,7 +16,11 @@ import shutil sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "api")) from utils.port import reserve_port from utils.process import clean_python_caches, stop_workspace_servers, terminate_group -from utils.workers import assert_sole_workspace_workers, motia_worker_count, write_server_ports +from utils.workers import ( + assert_sole_workspace_workers, + motia_worker_count, + write_server_ports, +) workflow_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "api") workspace_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -25,89 +29,257 @@ workspace_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Operations with string values accept sub-arguments; True means boolean-only (no sub-args). WORKFLOW_COMMANDS = { "verilator": { - "clean": 'Clean build directory. Args: "--chip [--output-dir ]"', - "verilog": 'Generate verilog files. Args: "--chip [--output-dir ]"', - "build": 'Build simulation executable. Args: "--chip [--jobs ] [--output-dir ]"', - "sim": 'Run simulation. Args: "--chip --binary [--batch] [--output-dir ]"', - "run": 'Integrated build+sim+run. Args: "--chip --binary [--batch] [--jobs ] [--output-dir ]"', - "cosim": 'Run cosimulation. Args: "--binary [--batch]"', + "clean": ( + 'Clean build directory. Args: "--design (--device | --core' + ' )"' + ), + "verilog": ( + 'Generate verilog files. Args: "--design (--device | --core' + ' )"' + ), + "build": ( + 'Build simulation executable. Args: "--design (--device | --core' + ' ) [--jobs ]"' + ), + "sim": ( + "Run a built simulation on a baremetal workload of the entry. Args:" + ' "--design (--device | --core ) --binary ' + " [--batch] [--no-wave] [--coverage] [--itrace] [--mtrace] [--pmctrace]" + ' [--ctrace] [--banktrace]"' + ), + "run": ( + 'Generate RTL, build and run. Args: "--design (--device | --core' + " ) --binary [--jobs ] [--batch] [--no-wave]" + ' [--coverage] [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--banktrace]"' + ), }, "vcs": { - "clean": 'Clean VCS artifacts. Args: "--chip [--output-dir ]"', - "verilog": 'Generate RTL for VCS. Args: "--chip [--output-dir ]"', - "build": 'Build VCS simulation executable. Args: "--chip [--jobs ] [--output-dir ]"', - "sim": 'Run a built VCS simulation. Args: "--chip --binary [--batch] [--output-dir ]"', - "run": 'Generate RTL, build, and run VCS. Args: "--chip --binary [--batch] [--jobs ] [--output-dir ]"', + "clean": ( + 'Clean the entry VCS build. Args: "--design (--device | --core' + ' )"' + ), + "verilog": ( + 'Generate the entry tapeout RTL for VCS. Args: "--design (--device' + ' | --core )"' + ), + "build": ( + 'Build the entry VCS simv. Args: "--design (--device | --core' + ' ) [--jobs ]"' + ), + "sim": ( + 'Run a built simv on a baremetal workload of the entry. Args: "--design' + " (--device | --core ) --binary [--batch]" + ' [--wave]"' + ), + "run": ( + 'Generate RTL, build and run VCS. Args: "--design (--device |' + ' --core ) --binary [--jobs ] [--batch]"' + ), }, "workload": { - "clean": 'Clean workload output directory. Args: "--chip "', - "build": 'Build test workloads. Args: "--chip [--stable] [--ctest|--mlirtest|--soctest]"', - "tohex": 'Convert chip workload ELF to hex. Args: "--chip "', + "clean": ( + 'Clean workload output directory. Args: "--design (--device |' + ' --core )"' + ), + "build": ( + 'Build test workloads. Args: "--design (--device | --core )' + ' [--ctest|--mlirtest|--soctest]"' + ), + "tohex": ( + 'Convert design workload ELF to hex. Args: "--design (--device |' + ' --core )"' + ), }, "model": { - "build": 'Build model recipes (all chip models when --model is omitted). Args: "--chip [--model ] [--ctrace] [--dtrace]"', + "build": ( + "Build model recipes on the device of a design (all its models when --model" + ' is omitted). Args: "--design [--model ]"' + ), }, "kernel": { - "build": 'Build RISC-V kernel + rootfs. Args: "[--chip ] [--model ] [--interactive] [--hart-count ] [--guest-memory-mib ] [--model-storage initramfs|ddr]"; --model requires --chip', + "build": ( + "Build RISC-V kernel + rootfs carrying the Linux programs of a design." + ' Args: "--design --device [--interactive]"' + ), }, "firesim": { - "enumeratefpgas": 'Enumerate FPGAs. Args: "--chip "', - "buildbitstream": 'Build bitstream. Args: "--chip "', - "infrasetup": 'Infrasetup. Args: "--chip "', - "runworkload": 'Run workload. Args: "--chip "', + "enumeratefpgas": ( + 'Enumerate FPGAs. Args: "--design (--device | --core )"' + ), + "buildbitstream": ( + 'Build bitstream. Args: "--design (--device | --core )"' + ), + "infrasetup": ( + 'Infrasetup. Args: "--design (--device | --core )"' + ), + "runworkload": ( + 'Run workload. Args: "--design (--device | --core )"' + ), }, - "compiler": { - "build": 'Build compiler. Args: "--chip "', + "stack": { + "build": ( + 'Build a registered simulation stack. Args: "--design (--device |' + ' --core )"' + ), }, "bebop-bemu": { - "sim": 'Run Bebop emulator simulation. Model args: "--chip --model [--reuse-simulator] [--itrace] [--mtrace]". ELF args: "--chip --binary [--disasm] [--tool-profile] [--itrace] [--mtrace]"; resolves files in the selected chip workload/kernel outputs', - "analysis": 'Analyze itrace/mtrace in an existing log dir. Args: "--chip --log-dir --itrace|--mtrace"', - "batch": 'Run Bebop BEMU batch regression. Args: "--chip --test [--jobs ] [--clean-before]"', + "sim": ( + 'Run Bebop emulator simulation. Args: "--design (--device | --core' + " ) --binary [--disasm] [--tool-profile] [--itrace]" + ' [--mtrace]"; resolves files in the selected design workload/kernel' + " outputs" + ), + "analysis": ( + 'Analyze itrace/mtrace in an existing log dir. Args: "--design ' + ' (--device | --core ) --run --itrace|--mtrace"' + ), + "batch": ( + 'Run Bebop BEMU batch regression. Args: "--design --device --test' + ' [--jobs ] [--clean-before]"' + ), }, "bebop-verilator": { - "clean": 'Clean Bebop Verilator build directory. Args: "--chip [--output-dir ]"', - "verilog": 'Generate Bebop Verilog files. Args: "--chip [--diff] [--main-only] [--output-dir ]"', - "build": 'Build Bebop Verilator simulation executable. Args: "--chip [--diff] [--jobs ] [--output-dir ]"', - "sim": 'Run Bebop Verilator simulation. Args: "--chip --binary [--log-dir ] [--fst-dir ] [--no-wave]"', - "run": 'Integrated Bebop Verilator build+sim+run. Args: "--binary [--diff] [--jobs ] [--chip ] [--log-dir ] [--fst-dir ] [--no-wave]"', - "batch": 'Run Bebop Verilator batch regression. Args: "--chip --test [--diff] [--clean-before]"', + "clean": ( + 'Clean the entry RTL. Args: "--design (--device | --core )"' + ), + "verilog": ( + 'Generate the entry RTL. Args: "--design (--device | --core' + ' ) [--diff]"' + ), + "build": ( + 'Build the entry Verilator runner. Args: "--design (--device |' + ' --core ) [--diff] [--jobs ]"' + ), + "sim": ( + 'Run a built runner on a workload of the entry. Args: "--design ' + " (--device | --core ) --binary [--diff] [--no-wave]" + ' [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--banktrace]"' + ), + "run": ( + 'Generate RTL, build and run. Args: "--design (--device | --core' + " ) --binary [--diff] [--jobs ] [--no-wave]" + ' [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--banktrace]"' + ), + "batch": ( + 'Run the entry regression. Args: "--design --device --test' + " [--diff] [--jobs ] [--clean-before]" + ' [--junit ]"' + ), }, "bebop-vcs": { - "verilog": 'Generate Bebop VCS RTL once. Args: "--chip "', - "build": 'Build the reusable Bebop VCS simv. Args: "--chip [--jobs ]"', - "batch": 'Run Bebop VCS workload batch. Args: "--chip --test "', + "sim": ( + 'Run one SoC workload through the Bebop VCS node. Args: "--design ' + ' (--device | --core ) --binary [--wave]"' + ), + "verilog": ( + 'Generate Bebop VCS RTL once. Args: "--design (--device | --core' + ' )"' + ), + "build": ( + 'Build the reusable Bebop VCS simv. Args: "--design (--device |' + ' --core ) [--jobs ]"' + ), + "batch": ( + 'Run design-owned VCS regression. Args: "--design --device --test' + ' [--jobs ] [--junit ]"' + ), }, "bebop-p2e": { - "clean": 'Clean Bebop P2E build directory. Args: "--chip [--output-dir ]"', - "verilog": 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--chip [--diff] [--output-dir ]"', - "buildbitstream": 'Build Bebop P2E runtime case. Args: "--chip [--diff] [--itrace] [--mtrace] [--vsrc-dir ] [--output-dir ] [--stop-after vsyn|vcom]"; stop-after produces resource assessment only, not a runnable case', - "runworkload": 'Cold-load Bebop P2E workload. Args: "--chip (--image | --load-manifest ) --bitstream [--fpga-location ] [--multi-fpga] [--wave] [--wave-start ] [--diff (image only)] [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--reuse-runtime]"', - "batch": 'Run Bebop P2E batch regression. Args: "--chip --bitstream --test [--diff] [--fpga-location ]"', + "clean": ( + 'Clean Bebop P2E build directory. Args: "--design (--device |' + ' --core ) [--output-dir ]"' + ), + "verilog": ( + 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--design' + ' (--device | --core ) [--diff] [--output-dir ]"' + ), + "buildbitstream": ( + 'Build Bebop P2E runtime case. Args: "--design (--device | --core' + " ) [--diff] [--itrace] [--mtrace] [--vsrc-dir ] [--output-dir" + ' ] [--stop-after vsyn|vcom]"; stop-after produces resource assessment' + " only, not a runnable case" + ), + "runworkload": ( + 'Cold-load Bebop P2E workload. Args: "--design (--device | --core' + " ) --image --bitstream [--fpga-location" + " ] [--multi-fpga] [--wave] [--wave-start ] [--diff] [--itrace]" + ' [--mtrace] [--pmctrace] [--ctrace] [--reuse-runtime]"' + ), + "batch": ( + 'Run Bebop P2E batch regression. Args: "--design --device' + " --bitstream --test [--diff]" + ' [--fpga-location ]"' + ), }, "dc": { - "verilog": 'Generate top-scoped DC RTL under arch/build//. Args: "--chip [--top ]"', - "area": 'Generate RTL, run DC synthesis, and report area under log////area. Args: "--chip [--top ]".', - "power": 'Generate RTL, run chip-owned power simulation, then run PrimeTime PX under log////power. Args: "--chip [--top ] [--workload ] [--start-ns ] [--end-ns ] [--activity --format ]".', + "verilog": ( + "Generate top-scoped DC RTL under arch/build//. Args:" + ' "--design (--device | --core ) [--top ]"' + ), + "area": ( + "Generate RTL, run DC synthesis, and report area under" + ' log////area. Args: "--design ' + ' (--device | --core ) [--top ]".' + ), + "power": ( + "Generate RTL, run design-owned power simulation, then run PrimeTime PX" + ' under log////power. Args: "--design ' + " (--device | --core ) [--top ] [--workload ] [--start-ns ] [--end-ns ] [--activity --format" + ' ]".' + ), }, "ip": { - "generate": 'Generate SRAM macros from elaborator mems.conf under arch/build//. Args: "--chip [--consumer ] [--top ] [--output-dir ] [--next-topic ]"', - "replace": 'Assemble top-scoped synthesis RTL plus SRAM macros under arch/build//. Args: "--chip [--consumer ] [--top ] [--output-dir ] [--next-topic ]"', + "generate": ( + "Generate SRAM macros from elaborator mems.conf under" + ' arch/build//. Args: "--design (--device | --core' + " ) [--consumer ] [--top ] [--output-dir ]" + ' [--next-topic ]"' + ), + "replace": ( + "Assemble top-scoped synthesis RTL plus SRAM macros under" + ' arch/build//. Args: "--design (--device | --core' + " ) [--consumer ] [--top ] [--output-dir ]" + ' [--next-topic ]"' + ), }, "yosys": { - "run": 'Run yosys flow. Args: "[--top ] [--chip ] [--output-dir ] [--log-dir ] [--vcd ]"', - "verilog": 'Generate verilog for yosys flow. Args: "[--chip ] [--output-dir ] [--log-dir ]"', - "synth": 'Run yosys synthesis. Args: "[--top ] [--output-dir ] [--log-dir ] [--vcd ]"', + "run": ( + 'Run yosys flow. Args: "[--top ] [--design (--device |' + ' --core )] [--output-dir ] [--vcd ]"' + ), + "verilog": ( + 'Generate verilog for yosys flow. Args: "[--design (--device |' + ' --core )] [--output-dir ]"' + ), + "synth": ( + 'Run yosys synthesis. Args: "[--top ] [--output-dir ]' + ' [--vcd ]"' + ), }, "uvm": { - "verilog": 'Generate isolated Ball RTL for UVM. Args: "--balltype --output-dir "', - "build": 'Build a Ball or IP UVM simulation. Args: "--chip [--ball | --ip [--target ]]"', - "run": 'Build and run a Ball or IP UVM simulation. Args: "--chip [--ball | --ip [--target ]]"', + "verilog": ( + 'Generate isolated Ball RTL for UVM. Args: "--balltype --output-dir' + ' "' + ), + "build": ( + 'Build a Ball or IP UVM simulation. Args: "[--design ] (--device |' + ' --ball | --ip )"' + ), + "run": ( + 'Build and run a Ball or IP UVM simulation. Args: "[--design ]' + ' (--device | --ball | --ip )"' + ), }, "config": { - "install": 'Install chip configurations. Args: "[--chip ]"', + "install": ( + 'Install design configurations. Args: "[--design (--device |' + ' --core )]"' + ), }, } + def make_iii_config( port: int, worker_port: int, @@ -135,11 +307,16 @@ def make_iii_config( "exporter: memory\n level: warn\n logs_console_output: false", ) if not exec_module: - marker = "\n # ── Exec Module (Python) ───────────────────────────────────────────────" + marker = ( + "\n # ── Exec Module (Python)" + " ───────────────────────────────────────────────" + ) if marker not in content: raise RuntimeError("failed to remove ExecModule from iii config") content = content.split(marker, 1)[0].rstrip() + "\n" else: + # Configuration generation must not restart active batch workers. + content = content.replace(" watch:\n - steps/**/*.py\n", "") motia_exec = "PYTHONUNBUFFERED=1 PYTHONPATH=. .venv/bin/motia dev --dir steps" single = f" exec:\n - {motia_exec}\n" if single not in content: @@ -208,7 +385,9 @@ def parse_args(argv: list[str]) -> argparse.Namespace: # start / stop subparsers.add_parser("start", parents=[common], help="Start dev server") - stop_parser = subparsers.add_parser("stop", parents=[common], help="Stop dev server") + stop_parser = subparsers.add_parser( + "stop", parents=[common], help="Stop dev server" + ) stop_parser.add_argument("--all", action="store_true", help="Stop all servers") # Build workflow subcommands from WORKFLOW_COMMANDS table @@ -257,7 +436,8 @@ def _normalize_workflow_args(argv: list[str]) -> list[str]: op_name, _value = token[2:].split("=", 1) if op_name in operations and operations[op_name] is not True: raise ValueError( - f"--{op_name} sub-arguments must be passed as a separate quoted string" + f"--{op_name} sub-arguments must be passed as a separate quoted" + " string" ) if token.startswith("--") and "=" not in token: op_name = token[2:] @@ -266,7 +446,11 @@ def _normalize_workflow_args(argv: list[str]) -> list[str]: j = i + 1 while j < len(argv): next_token = argv[j] - next_key = next_token[2:].split("=", 1)[0] if next_token.startswith("--") else "" + next_key = ( + next_token[2:].split("=", 1)[0] + if next_token.startswith("--") + else "" + ) if next_key in operations: break values.append(next_token) @@ -352,40 +536,52 @@ def _parse_sub_args(arg_string: str) -> dict: i += 1 else: raise ValueError(f"Unexpected argument: {token}") + unknown = args_dict.keys() & {"log-dir", "log_dir", "fst-dir", "fst_dir"} + if unknown: + raise ValueError(f"Unknown parameter(s): {', '.join(sorted(unknown))}") if "arguments" in args_dict: values = json.loads(args_dict["arguments"]) - if not isinstance(values, list) or not all(isinstance(value, str) for value in values): + if not isinstance(values, list) or not all( + isinstance(value, str) for value in values + ): raise ValueError("arguments must be a JSON list of strings") args_dict["arguments"] = values return args_dict def wait_for_http(url: str, check_fn, timeout: int, error_msg: str): - """Poll an HTTP endpoint until check_fn(response) returns True, or timeout.""" - for _ in range(timeout): - try: - resp = requests.get( - url, timeout=1, proxies={"http": None, "https": None, "all": None} - ) - if check_fn(resp): - return True - except requests.exceptions.ConnectionError: - pass - except requests.exceptions.RequestException: - return True # non-connection error means server is reachable - time.sleep(1) + """Wait for an actual HTTP response satisfying the readiness predicate.""" + deadline = time.monotonic() + timeout + with requests.Session() as session: + session.trust_env = False + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + response = session.get(url, timeout=min(1, remaining)) + if check_fn(response): + return True + except requests.exceptions.RequestException: + pass + remaining = deadline - time.monotonic() + if remaining > 0: + time.sleep(min(1, remaining)) print(error_msg) return False -# If you find that bbdev has failed without any logs, + +# If you find that bbdev has failed without any logs, # you can disable these log filters. These contain the bbdev runtime logs. def forward_iii_output(pipe, active_event: threading.Event, quiet: bool = False): """Forward task output to stdout. In quiet mode, suppress engine infrastructure logs.""" for line in iter(pipe.readline, ""): if not active_event.is_set(): continue - if quiet: - # Suppress engine infrastructure INFO/WARN; pass everything else + if quiet and not any( + marker in line for marker in ("ERROR", "error:", "Error:") + ): + # Preserve failures even when infrastructure progress is suppressed. if any( module in line for module in ( @@ -472,7 +668,9 @@ def run_server_mode(args, cmd_info): if proc.poll() is not None: sys.exit(proc.returncode or 1) try: - n = assert_sole_workspace_workers(worker_port, repo_root, min_local=need) + n = assert_sole_workspace_workers( + worker_port, repo_root, min_local=need + ) break except RuntimeError as e: msg = str(e) @@ -483,13 +681,12 @@ def run_server_mode(args, cmd_info): continue else: print( - f"Error: need {need} Motia worker(s) from this workspace within 60s", + f"Error: need {need} Motia worker(s) from this workspace" + " within 60s", file=sys.stderr, ) sys.exit(1) - print( - f"Workspace Motia isolation ok: {n} worker(s) on port {worker_port}" - ) + print(f"Workspace Motia isolation ok: {n} worker(s) on port {worker_port}") rc = proc.wait() if rc != 0: sys.exit(rc) @@ -569,7 +766,10 @@ def _submit_and_poll(port: int, api_path: str, cmd_info: dict, state_dir: str): elif state_data.get("processing") is True: legacy_processing_count += 1 if legacy_processing_count >= 30: - print("Error: Task is stuck in legacy processing state; worker did not advance the queue") + print( + "Error: Task is stuck in legacy processing state; worker did" + " not advance the queue" + ) return 1 else: legacy_processing_count = 0 @@ -685,7 +885,9 @@ def run_script_mode(args, cmd_info): ) output_thread = threading.Thread( - target=forward_iii_output, args=(proc.stdout, iii_output_active, True), daemon=True + target=forward_iii_output, + args=(proc.stdout, iii_output_active, True), + daemon=True, ) output_thread.start() @@ -696,8 +898,8 @@ def run_script_mode(args, cmd_info): if not wait_for_http( f"{base_url}/", check_fn=lambda r: True, - timeout=30, - error_msg="Server failed to start within 30 seconds", + timeout=120, + error_msg="Server failed to start within 120 seconds", ): sys.exit(1) @@ -747,18 +949,22 @@ def run_script_mode(args, cmd_info): sys.exit(rc) finally: - if server_start_lock is not None: + # Keep released ports reserved against new startup until worker cleanup ends. + if server_start_lock is None: + server_start_lock = open(server_start_lock_path, "w") + fcntl.flock(server_start_lock, fcntl.LOCK_EX) + try: + iii_output_active.clear() + time.sleep(0.5) + for worker_proc in worker_procs: + terminate_group(worker_proc) + terminate_group(proc) + stop_workspace_servers(workflow_dir, worker_url=iii_env["III_URL"]) + cleanup_config(config_path) + shutil.rmtree(state_dir) + finally: fcntl.flock(server_start_lock, fcntl.LOCK_UN) server_start_lock.close() - # Cleanup: stop iii, its process group, and any workspace worker it spawned. - iii_output_active.clear() - time.sleep(0.5) - for worker_proc in worker_procs: - terminate_group(worker_proc) - terminate_group(proc) - stop_workspace_servers(workflow_dir, worker_url=iii_env["III_URL"]) - cleanup_config(config_path) - shutil.rmtree(state_dir) if __name__ == "__main__": diff --git a/bbdev/mcp/common.py b/bbdev/mcp/common.py index f94e829b7..4371a17ce 100644 --- a/bbdev/mcp/common.py +++ b/bbdev/mcp/common.py @@ -17,6 +17,7 @@ from pathlib import Path from typing import Any, Dict, Optional + # bbdev/mcp -> repo root def repo_path() -> Path: return Path(__file__).resolve().parents[2] @@ -40,6 +41,8 @@ def log_path() -> Path: def state_dir() -> Path: return api_path() / "data" / "state_store.db" + + _proc: Optional[subprocess.Popen] = None _port: Optional[int] = None _log_fh = None @@ -123,7 +126,7 @@ def _http( def _ready(port: int) -> bool: try: - code, _ = _http("GET", f"http://127.0.0.1:{port}/compiler/build", timeout=2) + code, _ = _http("GET", f"http://127.0.0.1:{port}/stack/build", timeout=2) return code != 404 except Exception: return False @@ -133,7 +136,16 @@ def _stop() -> None: global _proc, _port, _log_fh if _port is not None and bbdev_path().is_file(): subprocess.run( - ["nix", "develop", "--command", str(bbdev_path()), "stop", "--server", "--port", str(_port)], + [ + "nix", + "develop", + "--command", + str(bbdev_path()), + "stop", + "--server", + "--port", + str(_port), + ], cwd=str(repo_path()), capture_output=True, text=True, @@ -186,7 +198,16 @@ def _ensure() -> int: log_path().parent.mkdir(parents=True, exist_ok=True) _log_fh = open(log_path(), "a", encoding="utf-8") _proc = subprocess.Popen( - ["nix", "develop", "--command", str(bbdev_path()), "start", "--server", "--port", str(port)], + [ + "nix", + "develop", + "--command", + str(bbdev_path()), + "start", + "--server", + "--port", + str(port), + ], cwd=str(repo_path()), stdout=_log_fh, stderr=_log_fh, @@ -212,9 +233,8 @@ def _ensure() -> int: except RuntimeError as e: msg = str(e) if ( - ("need" in msg and "Motia worker" in msg) - or "engine::workers::list failed" in msg - ): + "need" in msg and "Motia worker" in msg + ) or "engine::workers::list failed" in msg: time.sleep(1) continue _stop() @@ -352,162 +372,97 @@ def _load_toml(path: Path) -> Dict[str, Any]: return data -def _balldomain_path(chip: str, balldomain: Optional[str]) -> Path: +def _balldomain_path(design: str, balldomain: Optional[str]) -> Path: if balldomain: raw = Path(balldomain) if raw.is_absolute() or len(raw.parts) > 1: return (repo_path() / raw).resolve() - derived = repo_path() / "examples" / "chips" / chip / "configs/generated/config/derived.json" + derived = ( + repo_path() + / "examples" + / "designs" + / design + / "output/generated/config/derived.json" + ) cfg = json.loads(derived.read_text(encoding="utf-8")) - domains = {core["balldomain_base_dir"] for core in cfg["cores"]} + domains = {thread["balldomain_base_dir"] for thread in cfg["threads"]} if len(domains) != 1: - raise ValueError(f"chip {chip} has multiple BallDomains; provide an explicit TOML path") + raise ValueError( + f"design {design} has multiple BallDomains; provide an explicit TOML path" + ) directory = repo_path() / domains.pop() if balldomain: - return directory / (balldomain if balldomain.endswith(".toml") else f"{balldomain}.toml") - core_file = repo_path() / cfg["cores"][0]["config_path"] - include = _load_toml(core_file)["balldomain"] - return (core_file.parent / include).resolve() + return directory / ( + balldomain if balldomain.endswith(".toml") else f"{balldomain}.toml" + ) + thread_file = repo_path() / cfg["threads"][0]["config_path"] + include = _load_toml(thread_file)["balldomain"] + return (thread_file.parent / include).resolve() def _validate_domain(path: Path, cfg: Dict[str, Any]) -> Dict[str, Any]: - mappings = cfg.get("ballIdMappings", []) - if not isinstance(mappings, list): - raise ValueError(f"{path}: ballIdMappings must be an array") - isa = cfg.get("ballISA") - if not isinstance(isa, list): - raise ValueError(f"{path}: missing ballISA") - for i, e in enumerate(isa): - if not isinstance(e, dict): - raise ValueError(f"{path}: ballISA[{i}] must be a table") - if not isinstance(e.get("mnemonic"), str) or not e.get("mnemonic"): - raise ValueError(f"{path}: ballISA[{i}].mnemonic invalid") - if not isinstance(e.get("funct7"), int) or e.get("funct7") < 0: - raise ValueError(f"{path}: ballISA[{i}].funct7 invalid") - if not isinstance(e.get("bid"), int) or e.get("bid") < 0: - raise ValueError(f"{path}: ballISA[{i}].bid invalid") - - ids = [m.get("ballId") for m in mappings] - names = [m.get("ballName") for m in mappings] - id_set = set(ids) - - missing_config = [] - bad_bw = [] - for m in mappings: - name = m.get("ballName") - if not m.get("ballClass"): - bad_bw.append({"ballName": name, "error": "missing ballClass"}) - in_bw, out_bw = m.get("inBW"), m.get("outBW") - if ( - not isinstance(in_bw, int) - or in_bw <= 0 - or not isinstance(out_bw, int) - or out_bw <= 0 - ): - bad_bw.append({"ballName": name, "inBW": in_bw, "outBW": out_bw}) - config = m.get("config") - cfg_path = Path(config["_file"]) - if not cfg_path.is_file(): - missing_config.append({"ballName": name, "resolved": str(cfg_path)}) - funct7s = [e.get("funct7") for e in isa] - mnemonics = [e.get("mnemonic") for e in isa] - bids = [e.get("bid") for e in isa] - - orphan = sorted(id_set - set(bids)) - unknown = sorted(set(bids) - id_set) - - def dups(xs): - return sorted(x for x in set(xs) if xs.count(x) > 1) + from ball_normalize import normalize_ball_domain + domain = normalize_ball_domain({"balldomain": cfg}) + balls = domain["balls"] checks = { - "ballNum_matches_count": { - "pass": cfg.get("ballNum") == len(mappings), - "expected": len(mappings), - "actual": cfg.get("ballNum"), - }, - "ballId_strict_increment": {"pass": ids == list(range(len(ids))), "ids": ids}, - "ballId_no_duplicates": { - "pass": len(ids) == len(set(ids)), - "duplicates": dups(ids), - }, - "ballName_no_duplicates": { - "pass": len(names) == len(set(names)), - "duplicates": dups(names), - }, - "funct7_no_duplicates": { - "pass": len(funct7s) == len(set(funct7s)), - "duplicates": dups(funct7s), - }, - "mnemonic_no_duplicates": { - "pass": len(mnemonics) == len(set(mnemonics)), - "duplicates": dups(mnemonics), + "instance_contract": {"pass": True, "count": len(balls)}, + "every_ball_has_instructions": { + "pass": all(ball["instructions"] for ball in balls), + "empty": [ball["id"] for ball in balls if not ball["instructions"]], }, - "isa_bid_in_mappings": {"pass": not unknown, "unknown_bids": unknown}, - "every_ball_has_isa": {"pass": not orphan, "ballIds_without_isa": orphan}, - "ball_config_files_exist": { - "pass": not missing_config, - "missing": missing_config, - }, - "bandwidth_contract": {"pass": not bad_bw, "invalid": bad_bw}, } - - id_to_isa: Dict[Any, list] = {} - for e in isa: - id_to_isa.setdefault(e.get("bid"), []).append(e) - balls = [ - { - "ballId": m.get("ballId"), - "ballName": m.get("ballName"), - "ballClass": m.get("ballClass"), - "inBW": m.get("inBW"), - "outBW": m.get("outBW"), - "config": m.get("config"), - "isa": [ - {"mnemonic": e.get("mnemonic"), "funct7": e.get("funct7")} - for e in id_to_isa.get(m.get("ballId"), []) - ], - } - for m in mappings - ] - return { - "passed": all(c["pass"] for c in checks.values()), - "chip_balldomain": str(path.relative_to(repo_path())), + "passed": all(check["pass"] for check in checks.values()), + "design_balldomain": str(path.relative_to(repo_path())), "checks": checks, "balls": balls, } -def _validate_chip(chip: str, balldomain: Optional[str]) -> Dict[str, Any]: +def _validate_design(design: str, balldomain: Optional[str]) -> Dict[str, Any]: scripts = repo_path() / "bbdev/api/steps/config/scripts" sys.path.insert(0, str(scripts)) from ball_normalize import normalize_ball_domain - spec = importlib.util.spec_from_file_location("ball_validation_topology", scripts / "2_parameter_derivation.py") + + spec = importlib.util.spec_from_file_location( + "ball_validation_topology", scripts / "2_parameter_derivation.py" + ) topology = importlib.util.module_from_spec(spec) spec.loader.exec_module(topology) - expanded = repo_path() / "examples/chips" / chip / "configs/generated/config/config.json" + expanded = ( + repo_path() + / "examples/designs" + / design + / "output/generated/config/config.json" + ) config = json.loads(expanded.read_text(encoding="utf-8")) - selected = _balldomain_path(chip, balldomain) if balldomain is not None else None + selected = _balldomain_path(design, balldomain) if balldomain is not None else None results = [] - for index, core in enumerate(topology.iter_cores(config["designs"])): - raw = core.get("balldomain") + for index, thread in enumerate(topology.iter_threads(config["device"])): + raw = thread.get("balldomain") if not isinstance(raw, dict) or not raw.get("_file"): - raise ValueError(f"Core {index}: missing expanded BallDomain") + raise ValueError(f"Thread {index}: missing expanded BallDomain") path = Path(raw["_file"]).resolve() if selected is not None and path != selected: continue try: - domain = normalize_ball_domain(core) + domain = normalize_ball_domain(thread) if domain is None: raise ValueError("missing effective BallDomain") result = _validate_domain(path, domain) except ValueError as error: result = {"passed": False, "error": str(error)} - results.append({"core_index": index, "core_config": core["_file"], **result}) + results.append( + {"thread_index": index, "thread_config": thread["_file"], **result} + ) if not results: - raise ValueError(f"No expanded core matches BallDomain {selected}") - return {"chip": chip, "passed": all(result["passed"] for result in results), "cores": results} - + raise ValueError(f"No expanded thread matches BallDomain {selected}") + return { + "design": design, + "passed": all(result["passed"] for result in results), + "threads": results, + } atexit.register(_stop) @@ -520,4 +475,4 @@ def _validate_chip(chip: str, balldomain: Optional[str]) -> Dict[str, Any]: need = _need opt = _opt balldomain_path = _balldomain_path -validate_balldomains = _validate_chip +validate_balldomains = _validate_design diff --git a/bbdev/mcp/tests/test_common.py b/bbdev/mcp/tests/test_common.py index f5e855ca0..b19b856bd 100644 --- a/bbdev/mcp/tests/test_common.py +++ b/bbdev/mcp/tests/test_common.py @@ -18,7 +18,7 @@ def setUp(self): def test_submit_returns_trace_without_reading_state(self, ensure, workers, http): http.return_value = 202, {"trace_id": "trace-1"} - result = common.submit("/workload/build", {"chip": "pebble"}) + result = common.submit("/workload/build", {"design": "pebble"}) self.assertEqual( result, @@ -34,7 +34,7 @@ def test_submit_returns_trace_without_reading_state(self, ensure, workers, http) http.assert_called_once_with( "POST", "http://127.0.0.1:5100/workload/build", - {"chip": "pebble"}, + {"design": "pebble"}, timeout=30, ) @@ -141,7 +141,7 @@ def test_task_status_reports_nested_processing(self, read_state): "processing": { "processing": True, "task": "bebop.p2e.buildbitstream", - "chip": "pebble", + "design": "pebble", } } @@ -150,7 +150,7 @@ def test_task_status_reports_nested_processing(self, read_state): { "processing": True, "task": "bebop.p2e.buildbitstream", - "chip": "pebble", + "design": "pebble", "accepted": True, "trace_id": "trace-1", }, diff --git a/bbdev/mcp/tools/__init__.py b/bbdev/mcp/tools/__init__.py index b43622d2e..d6ab9f625 100644 --- a/bbdev/mcp/tools/__init__.py +++ b/bbdev/mcp/tools/__init__.py @@ -1,97 +1,19 @@ -"""Register all MCP tools.""" +"""Discover MCP capabilities from their owning tool modules.""" -from __future__ import annotations +import importlib +import pkgutil -from . import validate -from . import task_status -from . import task_cancel -from . import compiler_build -from . import workload_clean -from . import workload_build -from . import model_build -from . import bemu_sim -from . import bemu_analysis -from . import bemu_batch -from . import bebop_verilator_clean -from . import bebop_verilator_verilog -from . import bebop_verilator_build -from . import bebop_verilator_sim -from . import bebop_verilator_run -from . import bebop_verilator_batch -from . import verilator_clean -from . import verilator_verilog -from . import verilator_build -from . import verilator_sim -from . import verilator_run -from . import uvm_build -from . import uvm_run -from . import workload_tohex -from . import bebop_p2e_clean -from . import bebop_p2e_verilog -from . import bebop_p2e_buildbitstream -from . import bebop_p2e_runworkload -from . import bebop_p2e_batch -from . import dc_verilog -from . import dc_area -from . import dc_power -from . import ip_generate -from . import ip_replace -from . import firesim_enumeratefpgas -from . import firesim_buildbitstream -from . import firesim_infrasetup -from . import firesim_runworkload -from . import kernel_build -from . import yosys_run -from . import yosys_verilog -from . import yosys_synth def modules(): return [ - validate, - task_status, - task_cancel, - compiler_build, - workload_clean, - workload_build, - model_build, - bemu_sim, - bemu_analysis, - bemu_batch, - bebop_verilator_clean, - bebop_verilator_verilog, - bebop_verilator_build, - bebop_verilator_sim, - bebop_verilator_run, - bebop_verilator_batch, - verilator_clean, - verilator_verilog, - verilator_build, - verilator_sim, - verilator_run, - uvm_build, - uvm_run, - workload_tohex, - bebop_p2e_clean, - bebop_p2e_verilog, - bebop_p2e_buildbitstream, - bebop_p2e_runworkload, - bebop_p2e_batch, - dc_verilog, - dc_area, - dc_power, - ip_generate, - ip_replace, - firesim_enumeratefpgas, - firesim_buildbitstream, - firesim_infrasetup, - firesim_runworkload, - kernel_build, - yosys_run, - yosys_verilog, - yosys_synth, + importlib.import_module(f"{__name__}.{entry.name}") + for entry in sorted( + pkgutil.iter_modules(__path__), key=lambda entry: entry.name + ) + if not entry.name.startswith("_") ] -def register_all(mcp) -> None: - for mod in modules(): - mod.register(mcp) +def register_all(mcp): + for module in modules(): + module.register(mcp) diff --git a/bbdev/mcp/tools/bebop_p2e_batch.py b/bbdev/mcp/tools/bebop_p2e_batch.py index 0224a8b88..f8f352d97 100644 --- a/bbdev/mcp/tools/bebop_p2e_batch.py +++ b/bbdev/mcp/tools/bebop_p2e_batch.py @@ -9,25 +9,25 @@ def register(mcp): @mcp.tool() def bbdev_bebop_p2e_batch( - chip: str, - bitstream: str, - test: str, - diff: bool = False, + design: str, bitstream: str, test: str, diff: bool = False, device: bool = False ) -> str: """Batch bebop-p2e regression. test: bare-tests|linux-tests. POST /bebop/p2e/batch.""" - for n, v in (("chip", chip), ("bitstream", bitstream)): + for n, v in (("design", design), ("bitstream", bitstream)): if e := need(n, v): return err(e) if test not in ("bare-tests", "linux-tests"): return err("test must be bare-tests or linux-tests") + request = { + "design": design, + "bitstream": bitstream, + "test": test, + "diff": diff, + } + if device: + request["device"] = True return fmt( submit( "/bebop/p2e/batch", - { - "chip": chip, - "bitstream": bitstream, - "test": test, - "diff": diff, - }, + request, ) ) diff --git a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py index df6341896..d21004947 100644 --- a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py +++ b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py @@ -10,25 +10,32 @@ def register(mcp): @mcp.tool() def bbdev_bebop_p2e_buildbitstream( - chip: str, + design: str, diff: bool = False, itrace: bool = False, mtrace: bool = False, vsrc_dir: Optional[str] = None, output_dir: Optional[str] = None, stop_after: Optional[str] = None, + device: bool = False, + core: str | None = None, ) -> str: """Build bebop-p2e bitstream. POST /bebop/p2e/buildbitstream.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) + request = opt( + {"design": design, "diff": diff, "itrace": itrace, "mtrace": mtrace}, + vsrc_dir=vsrc_dir, + output_dir=output_dir, + stop_after=stop_after, + ) + if device: + request["device"] = True + if core is not None: + request["core"] = core return fmt( submit( "/bebop/p2e/buildbitstream", - opt( - {"chip": chip, "diff": diff, "itrace": itrace, "mtrace": mtrace}, - vsrc_dir=vsrc_dir, - output_dir=output_dir, - stop_after=stop_after, - ), + request, ) ) diff --git a/bbdev/mcp/tools/bebop_p2e_clean.py b/bbdev/mcp/tools/bebop_p2e_clean.py index 63e8148a8..58c07f1f0 100644 --- a/bbdev/mcp/tools/bebop_p2e_clean.py +++ b/bbdev/mcp/tools/bebop_p2e_clean.py @@ -8,8 +8,15 @@ def register(mcp): @mcp.tool() - def bbdev_bebop_p2e_clean(chip: str) -> str: + def bbdev_bebop_p2e_clean( + design: str, device: bool = False, core: str | None = None + ) -> str: """Clean bebop-p2e build. POST /bebop/p2e/clean.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) - return fmt(submit("/bebop/p2e/clean", {"chip": chip})) + request = {"design": design} + if device: + request["device"] = True + if core is not None: + request["core"] = core + return fmt(submit("/bebop/p2e/clean", request)) diff --git a/bbdev/mcp/tools/bebop_p2e_runworkload.py b/bbdev/mcp/tools/bebop_p2e_runworkload.py index 59e52b296..9df2cddba 100644 --- a/bbdev/mcp/tools/bebop_p2e_runworkload.py +++ b/bbdev/mcp/tools/bebop_p2e_runworkload.py @@ -10,7 +10,7 @@ def register(mcp): @mcp.tool() def bbdev_bebop_p2e_runworkload( - chip: str, + design: str, image: str = "", bitstream: str = "", itrace: bool = False, @@ -19,20 +19,18 @@ def bbdev_bebop_p2e_runworkload( ctrace: bool = False, reuse_runtime: bool = False, diff: bool = False, - load_manifest: str = "", + device: bool = False, + core: str | None = None, ) -> str: - """Cold-load one hex image or a validated binary load manifest on P2E.""" - if bool(image) == bool(load_manifest): - return err("exactly one of image and load_manifest is required") - if load_manifest and diff: - return err("load_manifest cannot use single-ELF diff") - for n, v in (("chip", chip), ("bitstream", bitstream)): + """Cold-load one hex image on P2E.""" + for n, v in (("design", design), ("image", image), ("bitstream", bitstream)): if e := need(n, v): return err(e) - params: Dict[str, Any] = {"chip": chip, "image": image, "bitstream": bitstream} - if load_manifest: - params.pop("image") - params["load-manifest"] = load_manifest + params: Dict[str, Any] = { + "design": design, + "image": image, + "bitstream": bitstream, + } if itrace: params["itrace"] = True if mtrace: @@ -45,4 +43,8 @@ def bbdev_bebop_p2e_runworkload( params["ctrace"] = True if diff: params["diff"] = True + if device: + params["device"] = True + if core is not None: + params["core"] = core return fmt(submit("/bebop/p2e/runworkload", params)) diff --git a/bbdev/mcp/tools/bebop_p2e_verilog.py b/bbdev/mcp/tools/bebop_p2e_verilog.py index 4037f547b..b1ec46f6b 100644 --- a/bbdev/mcp/tools/bebop_p2e_verilog.py +++ b/bbdev/mcp/tools/bebop_p2e_verilog.py @@ -10,14 +10,23 @@ def register(mcp): @mcp.tool() def bbdev_bebop_p2e_verilog( - chip: str, output_dir: Optional[str] = None, diff: bool = False + design: str, + output_dir: Optional[str] = None, + diff: bool = False, + device: bool = False, + core: str | None = None, ) -> str: """Generate Verilog for bebop-p2e. POST /bebop/p2e/verilog.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) + request = opt({"design": design, "diff": diff}, output_dir=output_dir) + if device: + request["device"] = True + if core is not None: + request["core"] = core return fmt( submit( "/bebop/p2e/verilog", - opt({"chip": chip, "diff": diff}, output_dir=output_dir), + request, ) ) diff --git a/bbdev/mcp/tools/bebop_verilator_batch.py b/bbdev/mcp/tools/bebop_verilator_batch.py index 747fbb179..cf17ee0dc 100644 --- a/bbdev/mcp/tools/bebop_verilator_batch.py +++ b/bbdev/mcp/tools/bebop_verilator_batch.py @@ -8,21 +8,30 @@ def register(mcp): @mcp.tool() def bbdev_bebop_verilator_batch( - chip: str, test: str, clean_before: bool = False, diff: bool = False + design: str, + test: str, + clean_before: bool = False, + diff: bool = False, + junit: str | None = None, + device: bool = False, ) -> str: """Batch bebop-verilator regression. POST /bebop/verilator/batch.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) if test not in ("bare-tests", "linux-tests"): return err("test must be bare-tests or linux-tests") + request = { + "design": design, + "test": test, + "clean-before": clean_before, + "diff": diff, + "junit": junit, + } + if device: + request["device"] = True return fmt( submit( "/bebop/verilator/batch", - { - "chip": chip, - "test": test, - "clean-before": clean_before, - "diff": diff, - }, + request, ) ) diff --git a/bbdev/mcp/tools/bebop_verilator_build.py b/bbdev/mcp/tools/bebop_verilator_build.py index b61e1a35d..dcd093b90 100644 --- a/bbdev/mcp/tools/bebop_verilator_build.py +++ b/bbdev/mcp/tools/bebop_verilator_build.py @@ -9,14 +9,23 @@ def register(mcp): @mcp.tool() def bbdev_bebop_verilator_build( - chip: str, jobs: int = 16, diff: bool = False + design: str, + jobs: int = 16, + diff: bool = False, + device: bool = False, + core: str | None = None, ) -> str: """Build bebop verilator binary. POST /bebop/verilator/build.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) + request = {"design": design, "jobs": jobs, "diff": diff} + if device: + request["device"] = True + if core is not None: + request["core"] = core return fmt( submit( "/bebop/verilator/build", - {"chip": chip, "jobs": jobs, "diff": diff}, + request, ) ) diff --git a/bbdev/mcp/tools/bebop_verilator_clean.py b/bbdev/mcp/tools/bebop_verilator_clean.py index 7660ae14e..72014b031 100644 --- a/bbdev/mcp/tools/bebop_verilator_clean.py +++ b/bbdev/mcp/tools/bebop_verilator_clean.py @@ -8,8 +8,15 @@ def register(mcp): @mcp.tool() - def bbdev_bebop_verilator_clean(chip: str) -> str: + def bbdev_bebop_verilator_clean( + design: str, device: bool = False, core: str | None = None + ) -> str: """Clean bebop-verilator build. POST /bebop/verilator/clean.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) - return fmt(submit("/bebop/verilator/clean", {"chip": chip})) + request = {"design": design} + if device: + request["device"] = True + if core is not None: + request["core"] = core + return fmt(submit("/bebop/verilator/clean", request)) diff --git a/bbdev/mcp/tools/bebop_verilator_run.py b/bbdev/mcp/tools/bebop_verilator_run.py index 122bc3be2..f657bcc5e 100644 --- a/bbdev/mcp/tools/bebop_verilator_run.py +++ b/bbdev/mcp/tools/bebop_verilator_run.py @@ -9,7 +9,7 @@ def register(mcp): @mcp.tool() def bbdev_bebop_verilator_run( binary: str, - chip: str, + design: str, jobs: int = 16, itrace: bool = False, mtrace: bool = False, @@ -18,25 +18,32 @@ def bbdev_bebop_verilator_run( banktrace: bool = False, no_wave: bool = False, diff: bool = False, + device: bool = False, + core: str | None = None, ) -> str: """Full bebop-verilator flow. POST /bebop/verilator/run.""" - for n, v in (("binary", binary), ("chip", chip)): + for n, v in (("binary", binary), ("design", design)): if e := need(n, v): return err(e) + request = { + "binary": binary, + "design": design, + "jobs": jobs, + "itrace": itrace, + "mtrace": mtrace, + "pmctrace": pmctrace, + "ctrace": ctrace, + "banktrace": banktrace, + "no-wave": no_wave, + "diff": diff, + } + if device: + request["device"] = True + if core is not None: + request["core"] = core return fmt( submit( "/bebop/verilator/run", - { - "binary": binary, - "chip": chip, - "jobs": jobs, - "itrace": itrace, - "mtrace": mtrace, - "pmctrace": pmctrace, - "ctrace": ctrace, - "banktrace": banktrace, - "no-wave": no_wave, - "diff": diff, - }, + request, ) ) diff --git a/bbdev/mcp/tools/bebop_verilator_sim.py b/bbdev/mcp/tools/bebop_verilator_sim.py index 3eed4cead..fb3810dc9 100644 --- a/bbdev/mcp/tools/bebop_verilator_sim.py +++ b/bbdev/mcp/tools/bebop_verilator_sim.py @@ -10,7 +10,7 @@ def register(mcp): @mcp.tool() def bbdev_bebop_verilator_sim( binary: str, - chip: str, + design: str, itrace: bool = False, mtrace: bool = False, pmctrace: bool = False, @@ -18,14 +18,16 @@ def bbdev_bebop_verilator_sim( banktrace: bool = False, no_wave: bool = False, diff: bool = False, + device: bool = False, + core: str | None = None, ) -> str: """Run one workload on bebop-verilator. POST /bebop/verilator/sim.""" - for n, v in (("binary", binary), ("chip", chip)): + for n, v in (("binary", binary), ("design", design)): if e := need(n, v): return err(e) params: Dict[str, Any] = { "binary": binary, - "chip": chip, + "design": design, "itrace": itrace, "mtrace": mtrace, "pmctrace": pmctrace, @@ -34,4 +36,8 @@ def bbdev_bebop_verilator_sim( "no-wave": no_wave, "diff": diff, } + if device: + params["device"] = True + if core is not None: + params["core"] = core return fmt(submit("/bebop/verilator/sim", params)) diff --git a/bbdev/mcp/tools/bebop_verilator_verilog.py b/bbdev/mcp/tools/bebop_verilator_verilog.py index 9358583d6..c0ecb2780 100644 --- a/bbdev/mcp/tools/bebop_verilator_verilog.py +++ b/bbdev/mcp/tools/bebop_verilator_verilog.py @@ -8,8 +8,15 @@ def register(mcp): @mcp.tool() - def bbdev_bebop_verilator_verilog(chip: str, diff: bool = False, main_only: bool = False) -> str: + def bbdev_bebop_verilator_verilog( + design: str, diff: bool = False, device: bool = False, core: str | None = None + ) -> str: """Generate Verilog for bebop-verilator. POST /bebop/verilator/verilog.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) - return fmt(submit("/bebop/verilator/verilog", {"chip": chip, "diff": diff, "main-only": main_only})) + request = {"design": design, "diff": diff} + if device: + request["device"] = True + if core is not None: + request["core"] = core + return fmt(submit("/bebop/verilator/verilog", request)) diff --git a/bbdev/mcp/tools/bemu_analysis.py b/bbdev/mcp/tools/bemu_analysis.py index 7189fb0dd..f1529d824 100644 --- a/bbdev/mcp/tools/bemu_analysis.py +++ b/bbdev/mcp/tools/bemu_analysis.py @@ -2,36 +2,39 @@ from __future__ import annotations -from pathlib import Path - from common import submit, err, fmt, need def register(mcp): @mcp.tool() def bbdev_bemu_analysis( - chip: str, - log_dir: str, + design: str, + run: str, itrace: bool = False, mtrace: bool = False, + device: bool = False, + core: str | None = None, ) -> str: - """Analyze existing bebop-bemu traces. POST /bebop/bemu/analysis.""" - if e := need("chip", chip): + """Analyze a run directory under the repository log/ directory. POST /bebop/bemu/analysis.""" + if e := need("design", design): return err(e) - if e := need("log_dir", log_dir): + if e := need("run", run): return err(e) - if not Path(log_dir).is_absolute(): - return err(f"log_dir must be an absolute path: {log_dir}") if not itrace and not mtrace: return err("need itrace and/or mtrace") + request = { + "design": design, + "run": run, + "itrace": itrace, + "mtrace": mtrace, + } + if device: + request["device"] = True + if core is not None: + request["core"] = core return fmt( submit( "/bebop/bemu/analysis", - { - "chip": chip, - "log-dir": log_dir, - "itrace": itrace, - "mtrace": mtrace, - }, + request, ) ) diff --git a/bbdev/mcp/tools/bemu_batch.py b/bbdev/mcp/tools/bemu_batch.py index eafcd3171..63c2a038d 100644 --- a/bbdev/mcp/tools/bemu_batch.py +++ b/bbdev/mcp/tools/bemu_batch.py @@ -8,24 +8,30 @@ def register(mcp): @mcp.tool() def bbdev_bemu_batch( - chip: str, + design: str, test: str, clean_before: bool = False, jobs: int = 1, + junit: str | None = None, + device: bool = False, ) -> str: """Batch bemu regression. test: bare-tests|linux-tests. POST /bebop/bemu/batch.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) if test not in ("bare-tests", "linux-tests"): return err("test must be bare-tests or linux-tests") + request = { + "design": design, + "test": test, + "clean-before": clean_before, + "jobs": jobs, + "junit": junit, + } + if device: + request["device"] = True return fmt( submit( "/bebop/bemu/batch", - { - "chip": chip, - "test": test, - "clean-before": clean_before, - "jobs": jobs, - }, + request, ) ) diff --git a/bbdev/mcp/tools/bemu_sim.py b/bbdev/mcp/tools/bemu_sim.py index 898dd1338..2b7f93800 100644 --- a/bbdev/mcp/tools/bemu_sim.py +++ b/bbdev/mcp/tools/bemu_sim.py @@ -9,7 +9,7 @@ def register(mcp): @mcp.tool() def bbdev_bemu_sim( - chip: str, + design: str, binary: str | None = None, dtb: str | None = None, initrd: str | None = None, @@ -18,54 +18,20 @@ def bbdev_bemu_sim( tool_profile: bool = False, itrace: bool = False, mtrace: bool = False, - core_index: int | None = None, - arguments: list[str] | None = None, - model: str | None = None, - reuse_simulator: bool = False, + device: bool = False, + core: str | None = None, ) -> str: - """Run a workload, kernel ELF, or compiled model on BEMU. - - Model runs use running-param.toml. POST /bebop/bemu/sim. - """ - if model is not None: - for n, v in (("chip", chip), ("model", model)): - if e := need(n, v): - return err(e) - if ( - binary is not None - or dtb - or initrd - or memory_mib is not None - or disasm - or tool_profile - or itrace - or mtrace - or core_index is not None - or arguments - ): - return err( - "model runs use running-param.toml; ELF options cannot be supplied" - ) - return fmt( - submit( - "/bebop/bemu/sim", - {"chip": chip, "model": model, "reuse-simulator": reuse_simulator}, - ) - ) - if reuse_simulator: - return err("reuse-simulator is supported for model runs only") - for n, v in (("chip", chip), ("binary", binary)): + """Run a workload or kernel ELF on BEMU. POST /bebop/bemu/sim.""" + for n, v in (("design", design), ("binary", binary)): if e := need(n, v): return err(e) params: Dict[str, Any] = { - "chip": chip, + "design": design, "binary": binary, "disasm": disasm, "tool-profile": tool_profile, "itrace": itrace, "mtrace": mtrace, - "core_index": core_index, - "arguments": arguments if arguments is not None else [], } for key, value in ( ("dtb", dtb), @@ -74,4 +40,8 @@ def bbdev_bemu_sim( ): if value is not None: params[key] = value + if device: + params["device"] = True + if core is not None: + params["core"] = core return fmt(submit("/bebop/bemu/sim", params)) diff --git a/bbdev/mcp/tools/compiler_build.py b/bbdev/mcp/tools/compiler_build.py deleted file mode 100644 index 7c5a43470..000000000 --- a/bbdev/mcp/tools/compiler_build.py +++ /dev/null @@ -1,13 +0,0 @@ -"""MCP tool: bbdev_compiler_build.""" - -from __future__ import annotations - - -from common import submit, fmt - - -def register(mcp): - @mcp.tool() - def bbdev_compiler_build(chip: str) -> str: - """Build buddy-mlir compiler for one chip. POST /compiler/build.""" - return fmt(submit("/compiler/build", {"chip": chip})) diff --git a/bbdev/mcp/tools/dc_area.py b/bbdev/mcp/tools/dc_area.py index cef17d4ed..f86d8f7e4 100644 --- a/bbdev/mcp/tools/dc_area.py +++ b/bbdev/mcp/tools/dc_area.py @@ -9,8 +9,18 @@ def register(mcp): @mcp.tool() - def bbdev_dc_area(chip: Optional[str] = None, top: Optional[str] = None) -> str: + def bbdev_dc_area( + design: Optional[str] = None, + top: Optional[str] = None, + device: bool = False, + core: str | None = None, + ) -> str: """Run DC synthesis and area reporting. Defaults to DigitalTop.""" - if not chip: - return err("chip is required") - return fmt(submit("/dc/area", opt({}, chip=chip, top=top))) + if not design: + return err("design is required") + request = opt({}, design=design, top=top) + if device: + request["device"] = True + if core is not None: + request["core"] = core + return fmt(submit("/dc/area", request)) diff --git a/bbdev/mcp/tools/dc_power.py b/bbdev/mcp/tools/dc_power.py index 961cac1e1..cfc3fb4c7 100644 --- a/bbdev/mcp/tools/dc_power.py +++ b/bbdev/mcp/tools/dc_power.py @@ -10,7 +10,7 @@ def register(mcp): @mcp.tool() def bbdev_dc_power( - chip: Optional[str] = None, + design: Optional[str] = None, top: Optional[str] = None, activity: Optional[str] = None, activity_format: Optional[str] = None, @@ -18,29 +18,36 @@ def bbdev_dc_power( workload: Optional[str] = None, start_ns: Optional[str] = None, end_ns: Optional[str] = None, + device: bool = False, + core: str | None = None, ) -> str: - """Run DC synthesis, chip-owned activity simulation, and PrimeTime PX. + """Run DC synthesis, design-owned activity simulation, and PrimeTime PX. Omit activity/activity_format for the normal rerun-and-measure flow. Supply both only as a debugging override. """ - if not chip: - return err("chip is required") + if not design: + return err("design is required") if bool(activity) != bool(activity_format): return err("activity and activity_format must be provided together") + request = opt( + {}, + design=design, + top=top, + activity=activity, + format=activity_format, + strip_path=strip_path, + workload=workload, + start_ns=start_ns, + end_ns=end_ns, + ) + if device: + request["device"] = True + if core is not None: + request["core"] = core return fmt( submit( "/dc/power", - opt( - {}, - chip=chip, - top=top, - activity=activity, - format=activity_format, - strip_path=strip_path, - workload=workload, - start_ns=start_ns, - end_ns=end_ns, - ), + request, ) ) diff --git a/bbdev/mcp/tools/dc_verilog.py b/bbdev/mcp/tools/dc_verilog.py index df93fe7b8..3b82546a6 100644 --- a/bbdev/mcp/tools/dc_verilog.py +++ b/bbdev/mcp/tools/dc_verilog.py @@ -10,15 +10,22 @@ def register(mcp): @mcp.tool() def bbdev_dc_verilog( - chip: Optional[str] = None, + design: Optional[str] = None, top: Optional[str] = None, + device: bool = False, + core: str | None = None, ) -> str: """Generate top-scoped DC RTL collateral. Defaults to DigitalTop.""" - if not chip: - return err("chip is required") + if not design: + return err("design is required") + request = opt({}, design=design, top=top) + if device: + request["device"] = True + if core is not None: + request["core"] = core return fmt( submit( "/dc/verilog", - opt({}, chip=chip, top=top), + request, ) ) diff --git a/bbdev/mcp/tools/firesim_buildbitstream.py b/bbdev/mcp/tools/firesim_buildbitstream.py index eaced3979..6e7198d63 100644 --- a/bbdev/mcp/tools/firesim_buildbitstream.py +++ b/bbdev/mcp/tools/firesim_buildbitstream.py @@ -7,8 +7,15 @@ def register(mcp): @mcp.tool() - def bbdev_firesim_buildbitstream(chip: str) -> str: + def bbdev_firesim_buildbitstream( + design: str, device: bool = False, core: str | None = None + ) -> str: """Build FireSim bitstream. POST /firesim/buildbitstream.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) - return fmt(submit("/firesim/buildbitstream", {"chip": chip})) + request = {"design": design} + if device: + request["device"] = True + if core is not None: + request["core"] = core + return fmt(submit("/firesim/buildbitstream", request)) diff --git a/bbdev/mcp/tools/firesim_enumeratefpgas.py b/bbdev/mcp/tools/firesim_enumeratefpgas.py index 66a060ea8..4f7253723 100644 --- a/bbdev/mcp/tools/firesim_enumeratefpgas.py +++ b/bbdev/mcp/tools/firesim_enumeratefpgas.py @@ -7,8 +7,15 @@ def register(mcp): @mcp.tool() - def bbdev_firesim_enumeratefpgas(chip: str) -> str: + def bbdev_firesim_enumeratefpgas( + design: str, device: bool = False, core: str | None = None + ) -> str: """Enumerate FireSim FPGAs. POST /firesim/enumeratefpgas.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) - return fmt(submit("/firesim/enumeratefpgas", {"chip": chip})) + request = {"design": design} + if device: + request["device"] = True + if core is not None: + request["core"] = core + return fmt(submit("/firesim/enumeratefpgas", request)) diff --git a/bbdev/mcp/tools/firesim_infrasetup.py b/bbdev/mcp/tools/firesim_infrasetup.py index d6caca6fb..f9feff958 100644 --- a/bbdev/mcp/tools/firesim_infrasetup.py +++ b/bbdev/mcp/tools/firesim_infrasetup.py @@ -7,8 +7,15 @@ def register(mcp): @mcp.tool() - def bbdev_firesim_infrasetup(chip: str) -> str: + def bbdev_firesim_infrasetup( + design: str, device: bool = False, core: str | None = None + ) -> str: """FireSim infrasetup. POST /firesim/infrasetup.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) - return fmt(submit("/firesim/infrasetup", {"chip": chip})) + request = {"design": design} + if device: + request["device"] = True + if core is not None: + request["core"] = core + return fmt(submit("/firesim/infrasetup", request)) diff --git a/bbdev/mcp/tools/firesim_runworkload.py b/bbdev/mcp/tools/firesim_runworkload.py index 2c4ef4d1b..dede134d5 100644 --- a/bbdev/mcp/tools/firesim_runworkload.py +++ b/bbdev/mcp/tools/firesim_runworkload.py @@ -7,8 +7,15 @@ def register(mcp): @mcp.tool() - def bbdev_firesim_runworkload(chip: str) -> str: + def bbdev_firesim_runworkload( + design: str, device: bool = False, core: str | None = None + ) -> str: """Run FireSim workload. POST /firesim/runworkload.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) - return fmt(submit("/firesim/runworkload", {"chip": chip})) + request = {"design": design} + if device: + request["device"] = True + if core is not None: + request["core"] = core + return fmt(submit("/firesim/runworkload", request)) diff --git a/bbdev/mcp/tools/ip_generate.py b/bbdev/mcp/tools/ip_generate.py index be8bd5404..31247a50c 100644 --- a/bbdev/mcp/tools/ip_generate.py +++ b/bbdev/mcp/tools/ip_generate.py @@ -10,24 +10,31 @@ def register(mcp): @mcp.tool() def bbdev_ip_generate( - chip: str, + design: str, consumer: Optional[str] = None, top: Optional[str] = None, output_dir: Optional[str] = None, next_topic: Optional[str] = None, + device: bool = False, + core: str | None = None, ) -> str: """Generate SRAM macros from mems.conf. POST /ip/generate.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) + request = opt( + {"design": design}, + consumer=consumer, + top=top, + output_dir=output_dir, + next_topic=next_topic, + ) + if device: + request["device"] = True + if core is not None: + request["core"] = core return fmt( submit( "/ip/generate", - opt( - {"chip": chip}, - consumer=consumer, - top=top, - output_dir=output_dir, - next_topic=next_topic, - ), + request, ) ) diff --git a/bbdev/mcp/tools/ip_replace.py b/bbdev/mcp/tools/ip_replace.py index dccafa35e..7f270e7d9 100644 --- a/bbdev/mcp/tools/ip_replace.py +++ b/bbdev/mcp/tools/ip_replace.py @@ -10,24 +10,31 @@ def register(mcp): @mcp.tool() def bbdev_ip_replace( - chip: str, + design: str, consumer: Optional[str] = None, top: Optional[str] = None, output_dir: Optional[str] = None, next_topic: Optional[str] = None, + device: bool = False, + core: str | None = None, ) -> str: """Assemble synthesis source list plus SRAM macros. POST /ip/replace.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) + request = opt( + {"design": design}, + consumer=consumer, + top=top, + output_dir=output_dir, + next_topic=next_topic, + ) + if device: + request["device"] = True + if core is not None: + request["core"] = core return fmt( submit( "/ip/replace", - opt( - {"chip": chip}, - consumer=consumer, - top=top, - output_dir=output_dir, - next_topic=next_topic, - ), + request, ) ) diff --git a/bbdev/mcp/tools/kernel_build.py b/bbdev/mcp/tools/kernel_build.py index 322e2d496..453f554fb 100644 --- a/bbdev/mcp/tools/kernel_build.py +++ b/bbdev/mcp/tools/kernel_build.py @@ -2,34 +2,36 @@ from __future__ import annotations -from typing import Any, Dict, Optional +from typing import Any, Dict -from common import submit, fmt, opt +from common import submit, fmt, need, err def register(mcp): @mcp.tool() def bbdev_kernel_build( - chip: Optional[str] = None, - model: Optional[str] = None, + design: str, interactive: bool = False, - guest_memory_mib: int = 512, - model_storage: str = "initramfs", - hart_count: Optional[int] = None, + device: bool = False, + model: str | None = None, ) -> str: """Build RISC-V kernel + rootfs. POST /kernel/build. - With --chip only: packs programs listed in examples/chips//kernel/workloads.toml + With --design only: packs the programs listed in + examples/designs//sims/device/workloads/kernel/CMakeLists.txt and runs them under Linux using the shared workload /init. - With --model: requires --chip; packs a native model artifact from - stack/models/build///artifact into fw_payload-. With --interactive: keep shared /init shell and do not auto-run. + With model: include the model image in the DDR image. """ - params: Dict[str, Any] = {"guest-memory-mib": guest_memory_mib, "model-storage": model_storage} - opt(params, chip=chip, model=model) + if e := need("design", design): + return err(e) + params: Dict[str, Any] = { + "design": design, + } if interactive: params["interactive"] = True - if hart_count is not None: - params["hart-count"] = hart_count + if device: + params["device"] = True + if model is not None: + params["model"] = model return fmt(submit("/kernel/build", params)) - diff --git a/bbdev/mcp/tools/model_build.py b/bbdev/mcp/tools/model_build.py index 647688499..5dae3f44e 100644 --- a/bbdev/mcp/tools/model_build.py +++ b/bbdev/mcp/tools/model_build.py @@ -5,11 +5,11 @@ def register(mcp): @mcp.tool() - def bbdev_model_build(chip: str, model: str | None = None, ctrace: bool = False, dtrace: bool = False) -> str: - """Build one model recipe, or all recipes for the chip when model is omitted. POST /model/build.""" - if error := need("chip", chip): + def bbdev_model_build(design: str, model: str | None = None) -> str: + """Build one model recipe, or all recipes for the design when model is omitted. POST /model/build.""" + if error := need("design", design): return err(error) - params = {"chip": chip, "ctrace": ctrace, "dtrace": dtrace} + params = {"design": design} if model is not None: params["model"] = model return fmt(submit("/model/build", params)) diff --git a/bbdev/mcp/tools/stack_build.py b/bbdev/mcp/tools/stack_build.py new file mode 100644 index 000000000..524e66a11 --- /dev/null +++ b/bbdev/mcp/tools/stack_build.py @@ -0,0 +1,20 @@ +"""MCP tool: bbdev_stack_build.""" + +from __future__ import annotations + + +from common import submit, fmt + + +def register(mcp): + @mcp.tool() + def bbdev_stack_build( + design: str, core: str | None = None, device: bool = False + ) -> str: + """Build a registered device or core simulation stack. POST /stack/build.""" + body = {"design": design} + if core is not None: + body["core"] = core + if device: + body["device"] = True + return fmt(submit("/stack/build", body)) diff --git a/bbdev/mcp/tools/uvm_build.py b/bbdev/mcp/tools/uvm_build.py index f5b85095f..097aa3bce 100644 --- a/bbdev/mcp/tools/uvm_build.py +++ b/bbdev/mcp/tools/uvm_build.py @@ -4,26 +4,27 @@ from typing import Optional -from common import submit, err, fmt, need +from common import submit, err, fmt def register(mcp): @mcp.tool() def bbdev_uvm_build( - chip: str, ball: Optional[str] = None, ip: Optional[str] = None, target: Optional[str] = None + design: Optional[str] = None, + ball: Optional[str] = None, + ip: Optional[str] = None, + device: bool = False, ) -> str: """Build a Ball or IP UVM simulation. POST /uvm/build.""" - if e := need("chip", chip): - return err(e) - params = {"chip": chip} + if design is None and not ip: + return err("Standalone UVM requires ip") + params = {} if design is None else {"design": design} if ball and ip: return err("Parameters --ball and --ip are mutually exclusive") - if target is not None: - if not ip or not target.strip(): - return err("Parameter --target requires --ip and a target name") - params["target"] = target if ball: params["ball"] = ball if ip: params["ip"] = ip + if device: + params["device"] = True return fmt(submit("/uvm/build", params)) diff --git a/bbdev/mcp/tools/uvm_run.py b/bbdev/mcp/tools/uvm_run.py index 5c40e49d8..b75331f59 100644 --- a/bbdev/mcp/tools/uvm_run.py +++ b/bbdev/mcp/tools/uvm_run.py @@ -4,26 +4,27 @@ from typing import Optional -from common import submit, err, fmt, need +from common import submit, err, fmt def register(mcp): @mcp.tool() def bbdev_uvm_run( - chip: str, ball: Optional[str] = None, ip: Optional[str] = None, target: Optional[str] = None + design: Optional[str] = None, + ball: Optional[str] = None, + ip: Optional[str] = None, + device: bool = False, ) -> str: """Build and run a Ball or IP UVM simulation. POST /uvm/run.""" - if e := need("chip", chip): - return err(e) - params = {"chip": chip} + if design is None and not ip: + return err("Standalone UVM requires ip") + params = {} if design is None else {"design": design} if ball and ip: return err("Parameters --ball and --ip are mutually exclusive") - if target is not None: - if not ip or not target.strip(): - return err("Parameter --target requires --ip and a target name") - params["target"] = target if ball: params["ball"] = ball if ip: params["ip"] = ip + if device: + params["device"] = True return fmt(submit("/uvm/run", params)) diff --git a/bbdev/mcp/tools/validate.py b/bbdev/mcp/tools/validate.py index c14bada76..902d09a78 100644 --- a/bbdev/mcp/tools/validate.py +++ b/bbdev/mcp/tools/validate.py @@ -9,12 +9,12 @@ def register(mcp): @mcp.tool() - def validate(chip: str = "toy", balldomain: Optional[str] = None) -> str: - """Validate the compiler Core's BallDomain registration for a Chip.""" - if e := need("chip", chip): + def validate(design: str = "toy", balldomain: Optional[str] = None) -> str: + """Validate the compiler Thread's BallDomain registration for a Design.""" + if e := need("design", design): return err(e) try: - return fmt(validate_balldomains(chip, balldomain)) + return fmt(validate_balldomains(design, balldomain)) except Exception as ex: return fmt( {"passed": False, "success": False, "failure": True, "error": str(ex)} diff --git a/bbdev/mcp/tools/verilator_build.py b/bbdev/mcp/tools/verilator_build.py index 6aa35d3ed..77a1999d7 100644 --- a/bbdev/mcp/tools/verilator_build.py +++ b/bbdev/mcp/tools/verilator_build.py @@ -8,14 +8,20 @@ def register(mcp): @mcp.tool() def bbdev_verilator_build( - chip: str, + design: str, jobs: int = 16, coverage: bool = False, + device: bool = False, + core: str | None = None, ) -> str: """Build non-bebop Verilator sim binary. POST /verilator/build.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) - params = {"chip": chip, "jobs": jobs} + params = {"design": design, "jobs": jobs} if coverage: params["coverage"] = True + if device: + params["device"] = True + if core is not None: + params["core"] = core return fmt(submit("/verilator/build", params)) diff --git a/bbdev/mcp/tools/verilator_clean.py b/bbdev/mcp/tools/verilator_clean.py index 89fada8e0..52ab1dc93 100644 --- a/bbdev/mcp/tools/verilator_clean.py +++ b/bbdev/mcp/tools/verilator_clean.py @@ -7,8 +7,15 @@ def register(mcp): @mcp.tool() - def bbdev_verilator_clean(chip: str) -> str: + def bbdev_verilator_clean( + design: str, device: bool = False, core: str | None = None + ) -> str: """Clean non-bebop Verilator build. POST /verilator/clean.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) - return fmt(submit("/verilator/clean", {"chip": chip})) + request = {"design": design} + if device: + request["device"] = True + if core is not None: + request["core"] = core + return fmt(submit("/verilator/clean", request)) diff --git a/bbdev/mcp/tools/verilator_run.py b/bbdev/mcp/tools/verilator_run.py index 4aa75920a..76016948b 100644 --- a/bbdev/mcp/tools/verilator_run.py +++ b/bbdev/mcp/tools/verilator_run.py @@ -11,7 +11,7 @@ def register(mcp): @mcp.tool() def bbdev_verilator_run( binary: str, - chip: str, + design: str, batch: bool = False, coverage: bool = False, no_wave: bool = False, @@ -21,15 +21,17 @@ def bbdev_verilator_run( pmctrace: bool = False, ctrace: bool = False, banktrace: bool = False, + device: bool = False, + core: str | None = None, ) -> str: """Full non-bebop Verilator flow. POST /verilator/run.""" if e := need("binary", binary): return err(e) - if e := need("chip", chip): + if e := need("design", design): return err(e) params: Dict[str, Any] = { "binary": binary, - "chip": chip, + "design": design, "batch": batch, "coverage": coverage, "no-wave": no_wave, @@ -41,4 +43,8 @@ def bbdev_verilator_run( } if jobs is not None: params["jobs"] = jobs + if device: + params["device"] = True + if core is not None: + params["core"] = core return fmt(submit("/verilator/run", params)) diff --git a/bbdev/mcp/tools/verilator_sim.py b/bbdev/mcp/tools/verilator_sim.py index ad2df6817..d168c063e 100644 --- a/bbdev/mcp/tools/verilator_sim.py +++ b/bbdev/mcp/tools/verilator_sim.py @@ -11,7 +11,7 @@ def register(mcp): @mcp.tool() def bbdev_verilator_sim( binary: str, - chip: str, + design: str, batch: bool = False, coverage: bool = False, no_wave: bool = False, @@ -20,14 +20,16 @@ def bbdev_verilator_sim( pmctrace: bool = False, ctrace: bool = False, banktrace: bool = False, + device: bool = False, + core: str | None = None, ) -> str: """Run one workload on non-bebop Verilator. POST /verilator/sim.""" - for n, v in (("binary", binary), ("chip", chip)): + for n, v in (("binary", binary), ("design", design)): if e := need(n, v): return err(e) params: Dict[str, Any] = { "binary": binary, - "chip": chip, + "design": design, "batch": batch, "no-wave": no_wave, "itrace": itrace, @@ -38,4 +40,8 @@ def bbdev_verilator_sim( } if coverage: params["coverage"] = True + if device: + params["device"] = True + if core is not None: + params["core"] = core return fmt(submit("/verilator/sim", params)) diff --git a/bbdev/mcp/tools/verilator_verilog.py b/bbdev/mcp/tools/verilator_verilog.py index 5ce8bdcd1..7468dc342 100644 --- a/bbdev/mcp/tools/verilator_verilog.py +++ b/bbdev/mcp/tools/verilator_verilog.py @@ -7,8 +7,15 @@ def register(mcp): @mcp.tool() - def bbdev_verilator_verilog(chip: str) -> str: + def bbdev_verilator_verilog( + design: str, device: bool = False, core: str | None = None + ) -> str: """Generate Verilog for non-bebop Verilator. POST /verilator/verilog.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) - return fmt(submit("/verilator/verilog", {"chip": chip})) + request = {"design": design} + if device: + request["device"] = True + if core is not None: + request["core"] = core + return fmt(submit("/verilator/verilog", request)) diff --git a/bbdev/mcp/tools/workload_build.py b/bbdev/mcp/tools/workload_build.py index ecff3e8e6..c4979f417 100644 --- a/bbdev/mcp/tools/workload_build.py +++ b/bbdev/mcp/tools/workload_build.py @@ -9,18 +9,24 @@ def register(mcp): @mcp.tool() def bbdev_workload_build( - chip: str, + design: str, ctest: bool = False, mlirtest: bool = False, + device: bool = False, + core: str | None = None, ) -> str: - """Build workloads for a chip. POST /workload/build.""" - if e := need("chip", chip): + """Build workloads for a design. POST /workload/build.""" + if e := need("design", design): return err(e) - params: Dict[str, Any] = {"chip": chip} + params: Dict[str, Any] = {"design": design} if ctest and mlirtest: return err("ctest and mlirtest cannot be used together") if ctest: params["ctest"] = True if mlirtest: params["mlirtest"] = True + if device: + params["device"] = True + if core is not None: + params["core"] = core return fmt(submit("/workload/build", params)) diff --git a/bbdev/mcp/tools/workload_clean.py b/bbdev/mcp/tools/workload_clean.py index 35000f2af..7a322948f 100644 --- a/bbdev/mcp/tools/workload_clean.py +++ b/bbdev/mcp/tools/workload_clean.py @@ -7,6 +7,13 @@ def register(mcp): @mcp.tool() - def bbdev_workload_clean(chip: str) -> str: - """Clean workload artifacts for one chip. POST /workload/clean.""" - return fmt(submit("/workload/clean", {"chip": chip})) + def bbdev_workload_clean( + design: str, device: bool = False, core: str | None = None + ) -> str: + """Clean workload artifacts for one design. POST /workload/clean.""" + request = {"design": design} + if device: + request["device"] = True + if core is not None: + request["core"] = core + return fmt(submit("/workload/clean", request)) diff --git a/bbdev/mcp/tools/workload_tohex.py b/bbdev/mcp/tools/workload_tohex.py index 1aab7a5db..779355899 100644 --- a/bbdev/mcp/tools/workload_tohex.py +++ b/bbdev/mcp/tools/workload_tohex.py @@ -7,6 +7,13 @@ def register(mcp): @mcp.tool() - def bbdev_workload_tohex(chip: str) -> str: - """Convert chip workload ELF to hex. POST /workload/tohex.""" - return fmt(submit("/workload/tohex", {"chip": chip})) + def bbdev_workload_tohex( + design: str, device: bool = False, core: str | None = None + ) -> str: + """Convert design workload ELF to hex. POST /workload/tohex.""" + request = {"design": design} + if device: + request["device"] = True + if core is not None: + request["core"] = core + return fmt(submit("/workload/tohex", request)) diff --git a/bbdev/mcp/tools/yosys_run.py b/bbdev/mcp/tools/yosys_run.py index c53a9f1c6..b38d15bed 100644 --- a/bbdev/mcp/tools/yosys_run.py +++ b/bbdev/mcp/tools/yosys_run.py @@ -10,24 +10,29 @@ def register(mcp): @mcp.tool() def bbdev_yosys_run( - chip: str, + design: str, top: Optional[str] = None, output_dir: Optional[str] = None, - log_dir: Optional[str] = None, vcd: Optional[str] = None, + device: bool = False, + core: str | None = None, ) -> str: """Full yosys flow. POST /yosys/run.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) + request = opt( + {"design": design}, + top=top, + output_dir=output_dir, + vcd=vcd, + ) + if device: + request["device"] = True + if core is not None: + request["core"] = core return fmt( submit( "/yosys/run", - opt( - {"chip": chip}, - top=top, - output_dir=output_dir, - log_dir=log_dir, - vcd=vcd, - ), + request, ) ) diff --git a/bbdev/mcp/tools/yosys_synth.py b/bbdev/mcp/tools/yosys_synth.py index 944fdb2fc..c02cd1886 100644 --- a/bbdev/mcp/tools/yosys_synth.py +++ b/bbdev/mcp/tools/yosys_synth.py @@ -10,24 +10,29 @@ def register(mcp): @mcp.tool() def bbdev_yosys_synth( - chip: str, + design: str, top: Optional[str] = None, output_dir: Optional[str] = None, - log_dir: Optional[str] = None, vcd: Optional[str] = None, + device: bool = False, + core: str | None = None, ) -> str: """Yosys synthesis + OpenSTA. POST /yosys/synth.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) + request = opt( + {"design": design}, + top=top, + output_dir=output_dir, + vcd=vcd, + ) + if device: + request["device"] = True + if core is not None: + request["core"] = core return fmt( submit( "/yosys/synth", - opt( - {"chip": chip}, - top=top, - output_dir=output_dir, - log_dir=log_dir, - vcd=vcd, - ), + request, ) ) diff --git a/bbdev/mcp/tools/yosys_verilog.py b/bbdev/mcp/tools/yosys_verilog.py index d6d926f5f..2387c6caa 100644 --- a/bbdev/mcp/tools/yosys_verilog.py +++ b/bbdev/mcp/tools/yosys_verilog.py @@ -10,22 +10,27 @@ def register(mcp): @mcp.tool() def bbdev_yosys_verilog( - chip: str, + design: str, top: Optional[str] = None, output_dir: Optional[str] = None, - log_dir: Optional[str] = None, + device: bool = False, + core: str | None = None, ) -> str: """Generate Verilog for yosys flow. POST /yosys/verilog.""" - if e := need("chip", chip): + if e := need("design", design): return err(e) + request = opt( + {"design": design}, + top=top, + output_dir=output_dir, + ) + if device: + request["device"] = True + if core is not None: + request["core"] = core return fmt( submit( "/yosys/verilog", - opt( - {"chip": chip}, - top=top, - output_dir=output_dir, - log_dir=log_dir, - ), + request, ) ) diff --git a/bebop b/bebop index 9c245acd1..4f1c04400 160000 --- a/bebop +++ b/bebop @@ -1 +1 @@ -Subproject commit 9c245acd1ef8863ea10e93a496ab81d2dfd7b45b +Subproject commit 4f1c04400865feb36bfefe074f0768cfc0cd7fae diff --git a/docs b/docs index 454b8f083..b3bb81823 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 454b8f083f12c5dbeb3905baab14bbe05a5b635b +Subproject commit b3bb818239b383b650f7bb6c2971290016777584 diff --git a/examples/.gitignore b/examples/.gitignore index 6a9039890..d2a87887b 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -3,3 +3,4 @@ build/ test-artifacts/ generated/ +output/ diff --git a/examples/balls/f32add/arch/src/main/scala/F32AddBall.scala b/examples/balls/f32add/arch/src/main/scala/F32AddBall.scala deleted file mode 100644 index fa9020471..000000000 --- a/examples/balls/f32add/arch/src/main/scala/F32AddBall.scala +++ /dev/null @@ -1,124 +0,0 @@ -package examples.balls.f32add - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.balldomain.rs.BallRsIssue -import framework.top.GlobalConfig - -@instantiable -class F32AddBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - private val mapping = b.ballDomain.ballIdMappings.find(_.ballName == "F32AddBall") - .getOrElse(throw new IllegalArgumentException("F32AddBall not found in config")) - private val funct = b.ballDomain.ballISA.find(_.mnemonic == "F32ADD").map(_.funct7) - .getOrElse(throw new IllegalArgumentException("F32ADD not found in ballISA")) - require(mapping.inBW == 2 && mapping.outBW == 1) - require(funct == 76) - require(b.memDomain.bankWidth == 128 && b.memDomain.bankMaskLen == 16) - - @public val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - def blink: BlinkIO = io - def status: BallStatus = io.status - - private val idle :: channels :: reading :: calculate :: writing :: acknowledge :: complete :: Nil = Enum(7) - private val state = RegInit(idle) - private val issue = Reg(new BallRsIssue(b)) - private val row = RegInit(0.U(16.W)) - private val requested = RegInit(VecInit(Seq.fill(2)(false.B))) - private val received = RegInit(VecInit(Seq.fill(2)(false.B))) - private val words = Reg(Vec(2, UInt(128.W))) - private val output = Reg(UInt(128.W)) - private val first = issue.cmd.rs2(5) - private val sourceGroup = issue.cmd.rs2(4, 0) - private val add = Instantiate(new F32AddRow) - add.io.source := words(0) - add.io.accumulator := Mux(first, 0.U(128.W), words(1)) - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := issue.rob_id - io.cmdResp.bits.is_sub := issue.is_sub - io.cmdResp.bits.sub_rob_id := issue.sub_rob_id - io.status.idle := state === idle - io.status.running := state =/= idle - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) - - for (port <- 0 until 2) { - val read = io.bankRead(port) - val active = if (port == 0) true.B else !first - read.rob_id := issue.rob_id - read.ball_id := 0.U - read.bank_id := (if (port == 0) issue.cmd.op1_bank else issue.cmd.op2_bank) - read.group_id := (if (port == 0) sourceGroup else 0.U) - read.io.req.valid := state === reading && active && !requested(port) - read.io.req.bits.addr := row - read.io.resp.ready := state === reading && active && requested(port) && !received(port) - when(read.io.req.fire)(requested(port) := true.B) - when(read.io.resp.fire) { - words(port) := read.io.resp.bits.data - received(port) := true.B - } - } - - private val write = io.bankWrite(0) - write.rob_id := issue.rob_id - write.ball_id := 0.U - write.bank_id := issue.cmd.wr_bank - write.group_id := 0.U - write.io.req.valid := state === writing - write.io.req.bits.addr := row - write.io.req.bits.data := output - write.io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) - write.io.resp.ready := state === acknowledge - - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - assert(cmd.funct7 === funct.U, "F32ADD invalid funct7") - assert(cmd.op1_en && cmd.op2_en && cmd.wr_spad_en, "F32ADD requires two inputs and one output") - assert(cmd.rs2(63, 6) === 0.U, "F32ADD reserved rs2 bits") - assert( - cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, - "F32ADD banks must differ" - ) - assert( - cmd.rs2(4, 0) < cmd.op1_col && cmd.op2_col === 1.U && cmd.wr_col === 1.U, - "F32ADD source group must exist and accumulators must have one group" - ) - for (bank <- Seq(cmd.op1_bank, cmd.op2_bank, cmd.wr_bank)) { - assert(bank < b.memDomain.virtualBankCount.U, "F32ADD invalid virtual bank") - val shared = bank >= b.frontend.shared_bank_id_base.U - assert(!shared || b.memDomain.sharedEnable.B, "F32ADD shared memory is disabled") - val depth = Mux(shared, b.memDomain.sharedBankEntries.U, b.memDomain.bankEntries.U) - assert(cmd.iter =/= 0.U && cmd.iter <= depth, "F32ADD row span exceeds bank") - } - issue := io.cmdReq.bits - row := 0.U - requested.foreach(_ := false.B) - received.foreach(_ := false.B) - state := channels - } - when(state === channels && io.channelReady)(state := reading) - when(state === reading && received(0) && (first || received(1)))(state := calculate) - when(state === calculate) { - output := add.io.sum - state := writing - } - when(write.io.req.fire)(state := acknowledge) - when(write.io.resp.fire) { - assert(write.io.resp.bits.ok, "F32ADD bank write failed") - when(row +& 1.U === issue.cmd.iter) { - state := complete - }.otherwise { - row := row + 1.U - requested.foreach(_ := false.B) - received.foreach(_ := false.B) - state := reading - } - } - when(io.cmdResp.fire)(state := idle) -} diff --git a/examples/balls/f32add/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/f32add/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 101d392d7..000000000 --- a/examples/balls/f32add/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class BankF32AddPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(BankF32AddOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getA(), op.getB(), op.getC(), - op.getRowsAttr(), op.getGroupAttr(), - op.getFirstAttr()); - rewriter.replaceOp(op, op.getC()); - return success(); - } -}; -} // namespace - -namespace mlir::buddy { -void populateF32AddBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &) { - patterns.add(patterns.getContext()); -} -} // namespace mlir::buddy diff --git a/examples/balls/f32add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/f32add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 1176c2946..000000000 --- a/examples/balls/f32add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -class F32AddLowering : public ConvertOpToLLVMPattern { -public: - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - LogicalResult - matchAndRewrite(F32AddOp op, OpAdaptor operands, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("F32AddBall"); - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || op.getRows() > target.bankDepth) - return op.emitError( - "F32ADD rows exceed the target 128-bit bank capacity"); - Location loc = op.getLoc(); - Value rs1 = - packRs1BanksIter(rewriter, loc, operands.getA(), operands.getB(), - operands.getC(), cstI64(rewriter, loc, op.getRows())); - Value rs2 = cstI64(rewriter, loc, op.getGroup() | (op.getFirst() ? 32 : 0)); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("F32ADD"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateF32AddBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { - patterns.add(converter); -} - -void configureF32AddBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/f32add/configs/default.toml b/examples/balls/f32add/configs/default.toml deleted file mode 100644 index c61b39cb6..000000000 --- a/examples/balls/f32add/configs/default.toml +++ /dev/null @@ -1 +0,0 @@ -[ball] diff --git a/examples/balls/f32add/emu/src/dispatch.rs b/examples/balls/f32add/emu/src/dispatch.rs deleted file mode 100644 index 4cc8f7b9e..000000000 --- a/examples/balls/f32add/emu/src/dispatch.rs +++ /dev/null @@ -1,26 +0,0 @@ -use crate::inst::instruction::ExecContext; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != "examples.balls.f32add.F32AddBall" { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("F32ADD") => Some(super::add::execute(xs1, xs2, ctx)), - _ => None, - } -} -pub fn cycles_after_issue(ball_class: &str, funct: u32, _xs1: u64, _xs2: u64) -> Option { - if ball_class != "examples.balls.f32add.F32AddBall" { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("F32ADD") => Some(1), - _ => None, - } -} diff --git a/examples/balls/f32add/emu/src/lib.rs b/examples/balls/f32add/emu/src/lib.rs deleted file mode 100644 index 0b9abb860..000000000 --- a/examples/balls/f32add/emu/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod add; -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/f32add/workloads/ctests/CMakeLists.txt b/examples/balls/f32add/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 7ddbd9531..000000000 --- a/examples/balls/f32add/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_buckyball_ctests(f32add_groups.c) diff --git a/examples/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir b/examples/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir deleted file mode 100644 index ee2f15544..000000000 --- a/examples/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir +++ /dev/null @@ -1,17 +0,0 @@ -// RUN: buddy-opt %s --target=main -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @reduce_groups -// CHECK-DAG: %[[AC:[^ ]+]] = arith.constant 2183169056 : i64 -// CHECK-DAG: %[[CA:[^ ]+]] = arith.constant 2182121504 : i64 -// CHECK-DAG: %[[FIRST:[^ ]+]] = arith.constant 32 : i64 -// CHECK: buckyball.intr.custom %[[AC]], %[[FIRST]] -// CHECK: buckyball.intr.custom %[[CA]], {{.*}} -// CHECK: buckyball.intr.custom %[[AC]], {{.*}} -func.func @reduce_groups() -> i64 attributes {llvm.emit_c_interface} { - %source = arith.constant 32 : i64 - %accumulator = arith.constant 33 : i64 - %output = arith.constant 34 : i64 - buckyball.f32add %source, %accumulator, %output : i64 - buckyball.f32add %source, %output, %accumulator : i64 - buckyball.f32add %source, %accumulator, %output : i64 - return %output : i64 -} diff --git a/examples/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir b/examples/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir deleted file mode 100644 index 8b05ee7b4..000000000 --- a/examples/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir +++ /dev/null @@ -1,36 +0,0 @@ -// RUN: buddy-opt %s --target=main -split-input-file -verify-diagnostics -func.func @zero_rows(%a: i64, %b: i64, %c: i64) { - // expected-error @+1 {{rows must be in [1, 17179869183]}} - buckyball.f32add %a, %b, %c : i64 - return -} -// ----- -func.func @wide_group(%a: i64, %b: i64, %c: i64) { - // expected-error @+1 {{group must be in [0, 31]}} - buckyball.f32add %a, %b, %c : i64 - return -} -// ----- -func.func @wide_rows(%a: i64, %b: i64, %c: i64) { - // expected-error @+1 {{rows must be in [1, 17179869183]}} - buckyball.f32add %a, %b, %c : i64 - return -} -// ----- -func.func @equal_bank_constants() { - %a = arith.constant 32 : i64 - %b = arith.constant 33 : i64 - %c = arith.constant 32 : i64 - // expected-error @+1 {{source, accumulator and output banks must differ}} - buckyball.f32add %a, %b, %c : i64 - return -} -// ----- -func.func @wide_bank() { - %a = arith.constant 1024 : i64 - %b = arith.constant 33 : i64 - %c = arith.constant 34 : i64 - // expected-error @+1 {{bank ID must be in [0, 1023]}} - buckyball.f32add %a, %b, %c : i64 - return -} diff --git a/examples/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir b/examples/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir deleted file mode 100644 index f51cb20ad..000000000 --- a/examples/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir +++ /dev/null @@ -1,21 +0,0 @@ -// RUN: buddy-opt %s --target=main -assign-physical-banks | FileCheck %s --check-prefix=BANK -// BANK-LABEL: func.func @reduce_groups -// BANK-COUNT-3: buckyball.f32add -// BANK-NOT: buckyball.bank_f32add -// RUN: buddy-opt %s --target=main -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @reduce_groups -// CHECK-DAG: %[[AC:[^ ]+]] = arith.constant 2183169056 : i64 -// CHECK-DAG: %[[CA:[^ ]+]] = arith.constant 2182121504 : i64 -// CHECK-DAG: %[[FIRST:[^ ]+]] = arith.constant 32 : i64 -// CHECK: buckyball.intr.custom %[[AC]], %[[FIRST]] -// CHECK: buckyball.intr.custom %[[CA]], {{.*}} -// CHECK: buckyball.intr.custom %[[AC]], {{.*}} -func.func @reduce_groups() -> i64 attributes {llvm.emit_c_interface} { - %source = arith.constant 32 : i64 - %accumulator = arith.constant 33 : i64 - %output = arith.constant 34 : i64 - %first = buckyball.bank_f32add %source %accumulator %output : i64 i64 i64 - %second = buckyball.bank_f32add %source %first %accumulator : i64 i64 i64 - %third = buckyball.bank_f32add %source %second %output : i64 i64 i64 - return %third : i64 -} diff --git a/examples/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp b/examples/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp deleted file mode 100644 index 43da2ae85..000000000 --- a/examples/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include -#include -#include -#include - -extern "C" int64_t _mlir_ciface_reduce_groups(); -static_assert(BB_SHARED_BANK_BASE == 32); - -static int checkF32Add() { - alignas(64) static const uint32_t input[3][8] = { - {0x3f800000, 0xc0200000, 0x7f800000, 0xff800000, 0, 0x80000000, - 0x7f7fffff, 1}, - {0x40000000, 0x40400000, 0xff800000, 0x7f800000, 0x80000000, 0, - 0xff7fffff, 1}, - {0xbf800000, 0x3f000000, 0x40a00000, 0xc0a00000, 0, 0x80000000, - 0x3f800000, 0x80000001}}; - alignas(64) static const uint32_t poison[8] = { - 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001, - 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001}; - alignas(64) static uint32_t output[8]; - bb_mem_alloc(32, 1, 3); - bb_mem_alloc(33, 1, 1); - bb_mem_alloc(34, 1, 1); - bb_mvin((uintptr_t)poison, 33, 2, 1); - for (unsigned group = 0; group < 3; ++group) - bb_mvin_group((uintptr_t)input[group], 32, group, 2, 1); - if (_mlir_ciface_reduce_groups() != 34) - return 1; - bb_mvout((uintptr_t)output, 34, 2, 1); - for (unsigned i = 0; i < 8; ++i) { - union { - uint32_t bits; - float value; - } expected = {0}, source; - for (unsigned group = 0; group < 3; ++group) { - source.bits = input[group][i]; - asm volatile("fadd.s %0, %1, %2, rne" - : "=f"(expected.value) - : "f"(expected.value), "f"(source.value)); - } - if (output[i] != expected.bits) - return 2; - } - bb_mem_release(32); - bb_mem_release(33); - bb_mem_release(34); - return 0; -} - -int main() { - int result = checkF32Add(); - if (!result) - puts("F32ADD Bank MLIR grouped/RNE/NaN/first/SSA output PASS"); - return result; -} diff --git a/examples/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir b/examples/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir deleted file mode 100644 index 90d633d65..000000000 --- a/examples/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir +++ /dev/null @@ -1,27 +0,0 @@ -// RUN: buddy-opt %s --target=main -split-input-file -verify-diagnostics -func.func @same_ssa_bank(%a: i64, %b: i64) { - // expected-error @+1 {{source, accumulator and output banks must differ}} - %out = buckyball.bank_f32add %a %b %b : i64 i64 i64 - return -} -// ----- -func.func @zero_rows(%a: i64, %b: i64, %c: i64) { - // expected-error @+1 {{rows must be in [1, 17179869183]}} - %out = buckyball.bank_f32add %a %b %c : i64 i64 i64 - return -} -// ----- -func.func @negative_group(%a: i64, %b: i64, %c: i64) { - // expected-error @+1 {{group must be in [0, 31]}} - %out = buckyball.bank_f32add %a %b %c : i64 i64 i64 - return -} -// ----- -func.func @negative_bank() { - %a = arith.constant -1 : i64 - %b = arith.constant 33 : i64 - %c = arith.constant 34 : i64 - // expected-error @+1 {{bank ID must be in [0, 1023]}} - %out = buckyball.bank_f32add %a %b %c : i64 i64 i64 - return -} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiBall.scala b/examples/balls/gemmini/arch/src/main/scala/GemminiBall.scala deleted file mode 100644 index e76a09705..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiBall.scala +++ /dev/null @@ -1,295 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink} -import framework.balldomain.blink.mmio.MmioRead -import framework.balldomain.rs.BallRsComplete -import framework.top.GlobalConfig - -@instantiable -class GemminiBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - require(b.memDomain.bankEntries <= 1024, "GemminiBall rs2 row-base fields support at most 1024 bank entries") - require( - b.frontend.sub_rob_enable, - "GemminiBall requires frontend.sub_rob_enable=true because loop instructions emit SubROB rows" - ) - - val ballCommonConfig = b.ballDomain.ballIdMappings - .find(_.ballName == "GemminiBall") - .getOrElse( - throw new IllegalArgumentException("GemminiBall not found in config") - ) - - val inBW = ballCommonConfig.inBW - val outBW = ballCommonConfig.outBW - - @public - val io = IO(new BlinkIO(b, inBW, outBW)) - - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - // ========================================================================= - // Sub-modules - // ========================================================================= - val exCtrl: Instance[GemminiExCtrl] = Instantiate(new GemminiExCtrl(b)) - - val matmulUnroller: Instance[LoopMatmulUnroller] = Instantiate( - new LoopMatmulUnroller(b) - ) - - val convUnroller: Instance[LoopConvUnroller] = Instantiate( - new LoopConvUnroller(b) - ) - - val encoder: Instance[LoopCmdEncoder] = Instantiate(new LoopCmdEncoder(b)) - - // ========================================================================= - // Config registers for Loop modes - // ========================================================================= - val loopWsConfig = Reg(new LoopWsConfig(b)) - val loopConvConfig = Reg(new LoopConvWsConfig(b)) - - // rob_id tracking for bank metadata - val rob_id_reg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - when(io.cmdReq.fire) { - rob_id_reg := io.cmdReq.bits.rob_id - } - - // ========================================================================= - // Instruction routing by funct7 - // ========================================================================= - val funct7 = io.cmdReq.bits.cmd.funct7 - - def ballFunct(mnemonic: String): UInt = - b.ballDomain.ballISA - .find(_.mnemonic == mnemonic) - .map(_.funct7.U(7.W)) - .getOrElse(throw new IllegalArgumentException(s"$mnemonic not found in ballISA")) - - val configFunct = ballFunct("GEMMINI_CONFIG") - val preloadFunct = ballFunct("GEMMINI_PRELOAD") - val computePreFunct = ballFunct("GEMMINI_COMPUTE_PRELOADED") - val computeAccFunct = ballFunct("GEMMINI_COMPUTE_ACCUMULATED") - val flushFunct = ballFunct("GEMMINI_FLUSH") - - val loopWsConfigFuncts = Seq( - ballFunct("GEMMINI_LOOP_WS_CONFIG_BOUNDS"), - ballFunct("GEMMINI_LOOP_WS_CONFIG_ADDR_A"), - ballFunct("GEMMINI_LOOP_WS_CONFIG_ADDR_B"), - ballFunct("GEMMINI_LOOP_WS_CONFIG_ADDR_D"), - ballFunct("GEMMINI_LOOP_WS_CONFIG_ADDR_C"), - ballFunct("GEMMINI_LOOP_WS_CONFIG_STRIDES_AB"), - ballFunct("GEMMINI_LOOP_WS_CONFIG_STRIDES_DC") - ) - - val loopWsTriggerFunct = ballFunct("GEMMINI_LOOP_WS") - - val loopConvConfigFuncts = Seq( - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_1"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_2"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_3"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_4"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_5"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_6"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_7"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_8"), - ballFunct("GEMMINI_LOOP_CONV_WS_CONFIG_9") - ) - - val loopConvTriggerFunct = ballFunct("GEMMINI_LOOP_CONV_WS") - - val rs2Data = io.cmdReq.bits.cmd.special - - val isConfig = funct7 === configFunct - val isPreload = funct7 === preloadFunct - val isComputePre = funct7 === computePreFunct - val isComputeAcc = funct7 === computeAccFunct - val isFlush = funct7 === flushFunct - val isExUnit = - isConfig || isPreload || isComputePre || isComputeAcc || isFlush - - val isLoopWsConfig = loopWsConfigFuncts.map(funct7 === _).reduce(_ || _) - val isLoopWsTrigger = funct7 === loopWsTriggerFunct - val isLoopConvConfig = loopConvConfigFuncts.map(funct7 === _).reduce(_ || _) - val isLoopConvTrigger = funct7 === loopConvTriggerFunct - - // ========================================================================= - // ExUnit path (non-Loop: CONFIG/PRELOAD/COMPUTE/FLUSH) - // ========================================================================= - exCtrl.exio.cmdReq.valid := io.cmdReq.valid && isExUnit - exCtrl.exio.cmdReq.bits := io.cmdReq.bits - - // ========================================================================= - // Config latch path (immediate cmdResp) - // ========================================================================= - val configRespValid = RegInit(false.B) - val configRespBits = Reg(new BallRsComplete(b)) - configRespValid := false.B // default: pulse - - when(io.cmdReq.fire && isLoopWsConfig) { - configRespValid := true.B - configRespBits.rob_id := io.cmdReq.bits.rob_id - configRespBits.is_sub := io.cmdReq.bits.is_sub - configRespBits.sub_rob_id := io.cmdReq.bits.sub_rob_id - switch(funct7) { - is(loopWsConfigFuncts(0)) { - loopWsConfig.max_k := rs2Data(15, 0) - loopWsConfig.max_j := rs2Data(31, 16) - loopWsConfig.max_i := rs2Data(47, 32) - } - is(loopWsConfigFuncts(1))(loopWsConfig.dram_addr_a := rs2Data(38, 0)) - is(loopWsConfigFuncts(2))(loopWsConfig.dram_addr_b := rs2Data(38, 0)) - is(loopWsConfigFuncts(3))(loopWsConfig.dram_addr_d := rs2Data(38, 0)) - is(loopWsConfigFuncts(4))(loopWsConfig.dram_addr_c := rs2Data(38, 0)) - is(loopWsConfigFuncts(5)) { - loopWsConfig.stride_a := rs2Data(31, 0) - loopWsConfig.stride_b := rs2Data(63, 32) - } - is(loopWsConfigFuncts(6)) { - loopWsConfig.stride_d := rs2Data(31, 0) - loopWsConfig.stride_c := rs2Data(63, 32) - } - } - } - - when(io.cmdReq.fire && isLoopConvConfig) { - configRespValid := true.B - configRespBits.rob_id := io.cmdReq.bits.rob_id - configRespBits.is_sub := io.cmdReq.bits.is_sub - configRespBits.sub_rob_id := io.cmdReq.bits.sub_rob_id - switch(funct7) { - is(loopConvConfigFuncts(0)) { - loopConvConfig.batch_size := rs2Data(15, 0) - loopConvConfig.in_dim := rs2Data(31, 16) - loopConvConfig.in_channels := rs2Data(47, 32) - } - is(loopConvConfigFuncts(1)) { - loopConvConfig.out_channels := rs2Data(15, 0) - loopConvConfig.out_dim := rs2Data(31, 16) - loopConvConfig.stride := rs2Data(39, 32) - loopConvConfig.padding := rs2Data(47, 40) - } - is(loopConvConfigFuncts(2)) { - loopConvConfig.kernel_dim := rs2Data(7, 0) - loopConvConfig.pool_size := rs2Data(15, 8) - loopConvConfig.pool_stride := rs2Data(23, 16) - loopConvConfig.pool_padding := rs2Data(31, 24) - } - is(loopConvConfigFuncts(3))(loopConvConfig.dram_addr_bias := rs2Data(38, 0)) - is(loopConvConfigFuncts(4))(loopConvConfig.dram_addr_input := rs2Data(38, 0)) - is(loopConvConfigFuncts(5))(loopConvConfig.dram_addr_weight := rs2Data(38, 0)) - is(loopConvConfigFuncts(6))(loopConvConfig.dram_addr_output := rs2Data(38, 0)) - is(loopConvConfigFuncts(7)) { - loopConvConfig.input_stride := rs2Data(31, 0) - loopConvConfig.weight_stride := rs2Data(63, 32) - } - is(loopConvConfigFuncts(8))(loopConvConfig.output_stride := rs2Data(31, 0)) - } - } - - // ========================================================================= - // Loop trigger: latch bank IDs and start unroller (no cmdResp) - // Bank values come from rs2Data in the trigger instruction, but loopWsConfig - // Reg won't update until next edge. Override start.bits combinationally. - // ========================================================================= - matmulUnroller.io.start.valid := false.B - matmulUnroller.io.start.bits := loopWsConfig - - when(io.cmdReq.fire && isLoopWsTrigger) { - loopWsConfig.bank_a := rs2Data(9, 0) - loopWsConfig.bank_b := rs2Data(19, 10) - loopWsConfig.bank_c := rs2Data(29, 20) - loopWsConfig.low_d := rs2Data(30) - matmulUnroller.io.start.valid := true.B - matmulUnroller.io.start.bits.bank_a := rs2Data(9, 0) - matmulUnroller.io.start.bits.bank_b := rs2Data(19, 10) - matmulUnroller.io.start.bits.bank_c := rs2Data(29, 20) - matmulUnroller.io.start.bits.low_d := rs2Data(30) - } - - convUnroller.io.start.valid := false.B - convUnroller.io.start.bits := loopConvConfig - - when(io.cmdReq.fire && isLoopConvTrigger) { - loopConvConfig.bank_input := rs2Data(9, 0) - loopConvConfig.bank_weight := rs2Data(19, 10) - loopConvConfig.bank_output := rs2Data(29, 20) - loopConvConfig.no_bias := rs2Data(30) - convUnroller.io.start.valid := true.B - convUnroller.io.start.bits.bank_input := rs2Data(9, 0) - convUnroller.io.start.bits.bank_weight := rs2Data(19, 10) - convUnroller.io.start.bits.bank_output := rs2Data(29, 20) - convUnroller.io.start.bits.no_bias := rs2Data(30) - } - - // ========================================================================= - // LoopUnrollers → Arbiter → LoopCmdEncoder → io.subRobReq - // ========================================================================= - val cmdArb = Module(new Arbiter(new LoopCmd(b), 2)) - cmdArb.io.in(0) <> matmulUnroller.io.cmd - cmdArb.io.in(1) <> convUnroller.io.cmd - encoder.io.cmd <> cmdArb.io.out - encoder.io.subRobRow <> io.subRobReq - encoder.io.ballId := ballCommonConfig.ballId.U - encoder.io.masterRobId := rob_id_reg - - // ========================================================================= - // cmdReq.ready: route to correct consumer - // ========================================================================= - val idleReady = - exCtrl.exio.cmdReq.ready && !matmulUnroller.io.busy && !convUnroller.io.busy - io.cmdReq.ready := Mux( - isExUnit, - exCtrl.exio.cmdReq.ready, - Mux( - isLoopWsConfig || isLoopConvConfig, - idleReady, - Mux( - isLoopWsTrigger, - !matmulUnroller.io.busy, - Mux(isLoopConvTrigger, !convUnroller.io.busy, idleReady) - ) - ) - ) - - // ========================================================================= - // cmdResp: mux between exUnit and config immediate response - // ========================================================================= - io.cmdResp <> exCtrl.exio.cmdResp - when(configRespValid) { - io.cmdResp.valid := true.B - io.cmdResp.bits := configRespBits - } - - // ========================================================================= - // Bank connections (unchanged from original) - // ========================================================================= - for (i <- 0 until inBW) { - io.bankRead(i).io.req <> exCtrl.exio.bankReadReq(i) - exCtrl.exio.bankReadResp(i) <> io.bankRead(i).io.resp - io.bankRead(i).rob_id := rob_id_reg - io.bankRead(i).ball_id := 0.U - io.bankRead(i).group_id := 0.U - } - io.bankRead(0).bank_id := exCtrl.exio.op1_bank_o - io.bankRead(0).group_id := exCtrl.exio.op1_group_o - if (inBW > 1) { - io.bankRead(1).bank_id := exCtrl.exio.op2_bank_o - } - - for (i <- 0 until outBW) { - io.bankWrite(i).io <> exCtrl.exio.bankWrite(i) - io.bankWrite(i).bank_id := exCtrl.exio.wr_bank_o - io.bankWrite(i).rob_id := rob_id_reg - io.bankWrite(i).ball_id := 0.U - io.bankWrite(i).group_id := i.U - } - - io.status <> exCtrl.exio.status - - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrl.scala b/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrl.scala deleted file mode 100644 index 8b8a5cc05..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrl.scala +++ /dev/null @@ -1,26 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig - -@instantiable -class GemminiExCtrl(val b: GlobalConfig) - extends Module - with GemminiExCtrlDefs - with GemminiExCtrlDefaults - with GemminiExCtrlCmdStates - with GemminiExCtrlPreloadStates - with GemminiExCtrlComputeReadState - with GemminiExCtrlComputeFeedState - with GemminiExCtrlStoreOps - with GemminiExCtrlAccumulateState - with GemminiExCtrlFsm { - @public val exio = io - - applyDefaults() - runFsm() - - io.status.idle := state === sIdle - io.status.running := state =/= sIdle -} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlCmdStates.scala b/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlCmdStates.scala deleted file mode 100644 index 4657da2fc..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlCmdStates.scala +++ /dev/null @@ -1,105 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ -import gemmini._ - -/** Sub-command encoding within the special field */ -object GemminiSubCmd { - val CONFIG = 0.U(4.W) - val PRELOAD = 1.U(4.W) - val COMPUTE_PRELOADED = 2.U(4.W) - val COMPUTE_ACCUMULATED = 3.U(4.W) - val FLUSH = 4.U(4.W) -} - -trait GemminiExCtrlCmdStates { this: GemminiExCtrl => - - protected def handleIdleState(): Unit = { - io.cmdReq.ready := true.B - when(io.cmdReq.fire) { - rob_id_reg := io.cmdReq.bits.rob_id - is_sub_reg := io.cmdReq.bits.is_sub - sub_rob_id_reg := io.cmdReq.bits.sub_rob_id - op1_bank := io.cmdReq.bits.cmd.op1_bank - op2_bank := io.cmdReq.bits.cmd.op2_bank - wr_bank := io.cmdReq.bits.cmd.wr_bank - op1_base := io.cmdReq.bits.cmd.special(15, 6) - op2_base := io.cmdReq.bits.cmd.special(25, 16) - wr_base := io.cmdReq.bits.cmd.special(35, 26) - total_rows := Mux( - io.cmdReq.bits.cmd.iter === 0.U, - DIM.U, - io.cmdReq.bits.cmd.iter - ) - zero_op2 := io.cmdReq.bits.cmd.special(4) - zero_op1_tail := io.cmdReq.bits.cmd.special(5) - accumulated_cmd := sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED - reset_acc_mesh := sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED - acc_read_group := 0.U - acc_read_pending := false.B - - when(sub_cmd === GemminiSubCmd.CONFIG) { - cfg_dataflow := io.cmdReq.bits.cmd.special(4) - cfg_a_transpose := io.cmdReq.bits.cmd.special(7) - cfg_bd_transpose := io.cmdReq.bits.cmd.special(8) - cfg_in_shift := io.cmdReq.bits.cmd.special( - log2Up(config.accWidth) + 8, - 9 - ) - io.cmdResp.valid := true.B - state := sCommit - }.elsewhen(sub_cmd === GemminiSubCmd.PRELOAD) { - read_row_cnt := 0.U - feed_row_cnt := 0.U - req_sent := false.B - xpose_ready := false.B - xpose_row_cnt := 0.U - state := sPreloadRead - }.elsewhen( - sub_cmd === GemminiSubCmd.COMPUTE_PRELOADED || sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED - ) { - read_row_cnt := 0.U - feed_row_cnt := 0.U - outBufRows := 0.U - outBufCollected := 0.U - req_sent := false.B - xpose_ready := false.B - xpose_row_cnt := 0.U - read_done.foreach(_ := false.B) - state := Mux(sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED, sPreloadFeed, sComputeRead) - }.elsewhen(sub_cmd === GemminiSubCmd.FLUSH) { - state := sFlush - } - } - } - - protected def handleCommitState(): Unit = { - io.cmdResp.valid := true.B - when(io.cmdResp.fire) { - state := sIdle - } - } - - protected def handleFlushState(): Unit = { - mesh.io.req.valid := true.B - mesh.io.req.bits.flush := 2.U - mesh.io.req.bits.total_rows := DIM.U - mesh.io.req.bits.tag.rob := robIdAsTag8(rob_id_reg) - - mesh.io.a.valid := true.B - mesh.io.a.bits := 0.U.asTypeOf(mesh.A_TYPE) - mesh.io.b.valid := true.B - mesh.io.b.bits := 0.U.asTypeOf(mesh.B_TYPE) - mesh.io.d.valid := true.B - mesh.io.d.bits := 0.U.asTypeOf(mesh.D_TYPE) - - when(mesh.io.req.ready) { - io.cmdResp.valid := true.B - when(io.cmdResp.fire) { - state := sIdle - } - } - } - -} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlDefaults.scala b/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlDefaults.scala deleted file mode 100644 index 932af6393..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlDefaults.scala +++ /dev/null @@ -1,43 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ - -trait GemminiExCtrlDefaults { this: GemminiExCtrl => - - protected def applyDefaults(): Unit = { - io.cmdReq.ready := state === sIdle - - io.cmdResp.valid := false.B - io.cmdResp.bits.rob_id := rob_id_reg - io.cmdResp.bits.is_sub := is_sub_reg - io.cmdResp.bits.sub_rob_id := sub_rob_id_reg - - for (i <- 0 until inBW) { - io.bankReadReq(i).valid := false.B - io.bankReadReq(i).bits.addr := 0.U - } - - rdQueue0.io.deq.ready := false.B - rdQueue1.io.deq.ready := false.B - - mesh.io.a.valid := false.B - mesh.io.a.bits := 0.U.asTypeOf(mesh.A_TYPE) - mesh.io.b.valid := false.B - mesh.io.b.bits := 0.U.asTypeOf(mesh.B_TYPE) - mesh.io.d.valid := false.B - mesh.io.d.bits := 0.U.asTypeOf(mesh.D_TYPE) - mesh.io.req.valid := false.B - mesh.io.req.bits := 0.U.asTypeOf(mesh.io.req.bits) - // mesh.io.resp is Valid-only (no ready); MeshWithDelays assumes immediate observation when valid - - io.bankWrite.foreach { bw => - bw.req.valid := false.B - bw.req.bits.addr := 0.U - bw.req.bits.data := 0.U - bw.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(false.B)) - bw.resp.ready := true.B - } - } - -} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlDefs.scala b/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlDefs.scala deleted file mode 100644 index e705d55f8..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlDefs.scala +++ /dev/null @@ -1,138 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ -import framework.balldomain.blink.BallStatus -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.memdomain.backend.banks.{SramReadReq, SramReadResp, SramWriteIO} -import examples.balls.gemmini.configs.GemminiBallParam -import gemmini._ -import gemmini.Util._ - -/** Minimal tag for MeshWithDelays that satisfies TagQueueTag */ -class SimpleTag extends Bundle with gemmini.TagQueueTag { - val rob = UInt(8.W) - override def make_this_garbage(dummy: Int = 0): Unit = - rob := 0xff.U -} - -trait GemminiExCtrlDefs { this: GemminiExCtrl => - val config = GemminiBallParam(b) - val DIM = config.blockSize - - val ballMapping = b.ballDomain.ballIdMappings - .find(_.ballName == "GemminiBall") - .getOrElse( - throw new IllegalArgumentException("GemminiBall not found in config") - ) - - val inBW = ballMapping.inBW - val outBW = ballMapping.outBW - - val inputType = SInt(config.inputWidth.W) - val accType = SInt(config.accWidth.W) - val meshOutputType = SInt(config.spatialOutputWidth.W) - - val ctrlIo = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankReadReq = Vec(inBW, Decoupled(new SramReadReq(b))) - val bankReadResp = Vec(inBW, Flipped(Decoupled(new SramReadResp(b)))) - val bankWrite = Vec(outBW, Flipped(new SramWriteIO(b))) - val op1_bank_o = Output(UInt(b.memDomain.vbankIdWidth.W)) - val op1_group_o = Output(UInt(b.memDomain.groupIdWidth.W)) - val op2_bank_o = Output(UInt(b.memDomain.vbankIdWidth.W)) - val wr_bank_o = Output(UInt(b.memDomain.vbankIdWidth.W)) - val status = new BallStatus - }) - - val io = ctrlIo - - val mesh = Module( - new MeshWithDelays( - inputType = inputType, - weightType = inputType, - outputType = meshOutputType, - accType = accType, - tagType = new SimpleTag, - df = Dataflow.BOTH, - tree_reduction = false, - tile_latency = config.tileLatency, - output_delay = config.outputDelay, - tileRows = config.tileRows, - tileColumns = config.tileColumns, - meshRows = config.meshRows, - meshColumns = config.meshColumns, - leftBanks = 1, - upBanks = 1 - ) - ) - - protected def widenMeshToAcc(src: Vec[Vec[SInt]]): Vec[Vec[SInt]] = - VecInit(src.map(col => VecInit(col.map(_.asTypeOf(accType))))) - - val cfg_dataflow = RegInit(0.U(1.W)) - val cfg_in_shift = RegInit(0.U(log2Up(config.accWidth).W)) - val cfg_a_transpose = RegInit(false.B) - val cfg_bd_transpose = RegInit(false.B) - val zero_op2 = RegInit(false.B) - val zero_op1_tail = RegInit(false.B) - val accumulated_cmd = RegInit(false.B) - val reset_acc_mesh = RegInit(false.B) - val acc_read_group = RegInit(0.U(log2Up(outBW).W)) - val acc_read_pending = RegInit(false.B) - val execution_dataflow = Mux(accumulated_cmd, Dataflow.OS.id.U, cfg_dataflow) - val execution_shift = Mux(accumulated_cmd, 0.U, cfg_in_shift) - - val sIdle :: sPreloadRead :: sPreloadFeed :: sComputeRead :: sComputeFeed :: sComputeFlush :: sFlush :: sDrain :: sReadAccum :: sStore :: sCommit :: Nil = - Enum(11) - val state = RegInit(sIdle) - - val rob_id_reg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val is_sub_reg = RegInit(false.B) - val sub_rob_id_reg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - - /** Match SimpleTag.rob (8b); must match mesh req tag bits */ - protected def robIdAsTag8(x: UInt): UInt = { - val w = x.getWidth - if (w >= 8) x(7, 0) else Cat(0.U((8 - w).W), x) - } - - val op1_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val op2_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val wr_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val op1_base = RegInit(0.U(10.W)) - val op2_base = RegInit(0.U(10.W)) - val wr_base = RegInit(0.U(10.W)) - io.op1_bank_o := Mux(state === sReadAccum, wr_bank, op1_bank) - io.op1_group_o := Mux(state === sReadAccum, acc_read_group, 0.U) - io.op2_bank_o := op2_bank - io.wr_bank_o := wr_bank - - val read_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) - val feed_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) - val store_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) - val total_rows = RegInit(DIM.U(log2Up(DIM + 1).W)) - val req_sent = RegInit(false.B) - val read_done = RegInit(VecInit(Seq.fill(inBW)(false.B))) - val xpose_ready = RegInit(false.B) - val xpose_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) - - val sub_cmd = io.cmdReq.bits.cmd.special(3, 0) - - val rdQueue0 = Module(new Queue(new SramReadResp(b), entries = DIM)) - val rdQueue1 = Module(new Queue(new SramReadResp(b), entries = DIM)) - rdQueue0.io.enq <> io.bankReadResp(0) - rdQueue1.io.enq <> io.bankReadResp(1) - - val outBuf = Reg( - Vec(DIM, Vec(config.meshColumns, Vec(config.tileColumns, accType))) - ) - - val outBufRows = RegInit(0.U(log2Up(DIM + 1).W)) - val outBufCollected = RegInit(0.U(log2Up(DIM + 1).W)) - val op1Buf = Reg(Vec(DIM, Vec(DIM, inputType))) - val op2Buf = Reg(Vec(DIM, Vec(DIM, inputType))) - - val port_written = RegInit(VecInit(Seq.fill(outBW)(false.B))) -} diff --git a/examples/balls/gemmini/arch/src/main/scala/LoopCmd.scala b/examples/balls/gemmini/arch/src/main/scala/LoopCmd.scala deleted file mode 100644 index 9e1285a71..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/LoopCmd.scala +++ /dev/null @@ -1,77 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ -import framework.top.GlobalConfig - -object LoopSubCmdType { - val MSET_ALLOC = 0.U(3.W) - val MSET_FREE = 1.U(3.W) - val MVIN = 2.U(3.W) - val MVOUT = 3.U(3.W) - val PRELOAD = 4.U(3.W) - val COMPUTE = 5.U(3.W) -} - -class LoopSubCmd(val b: GlobalConfig) extends Bundle { - val cmdType = UInt(3.W) - val bank_id = UInt(b.memDomain.vbankIdWidth.W) - val dram_addr = UInt(b.memDomain.memAddrLen.W) - val stride = UInt(19.W) - val iter = UInt(b.frontend.iter_len.W) - val bank_row = UInt(5.W) - val bank_col = UInt(5.W) - val op1_bank = UInt(b.memDomain.vbankIdWidth.W) - val op2_bank = UInt(b.memDomain.vbankIdWidth.W) - val wr_bank = UInt(b.memDomain.vbankIdWidth.W) - val compute_mode = UInt(2.W) - val zero_op2 = Bool() - val zero_op1_tail = Bool() -} - -class LoopCmd(val b: GlobalConfig) extends Bundle { - val slots = Vec(4, Valid(new LoopSubCmd(b))) -} - -class LoopWsConfig(val b: GlobalConfig) extends Bundle { - val max_i = UInt(16.W) - val max_j = UInt(16.W) - val max_k = UInt(16.W) - val dram_addr_a = UInt(b.memDomain.memAddrLen.W) - val dram_addr_b = UInt(b.memDomain.memAddrLen.W) - val dram_addr_d = UInt(b.memDomain.memAddrLen.W) - val dram_addr_c = UInt(b.memDomain.memAddrLen.W) - val stride_a = UInt(32.W) - val stride_b = UInt(32.W) - val stride_d = UInt(32.W) - val stride_c = UInt(32.W) - val bank_a = UInt(b.memDomain.vbankIdWidth.W) - val bank_b = UInt(b.memDomain.vbankIdWidth.W) - val bank_c = UInt(b.memDomain.vbankIdWidth.W) - val low_d = Bool() -} - -class LoopConvWsConfig(val b: GlobalConfig) extends Bundle { - val batch_size = UInt(16.W) - val in_dim = UInt(16.W) - val in_channels = UInt(16.W) - val out_channels = UInt(16.W) - val out_dim = UInt(16.W) - val stride = UInt(8.W) - val padding = UInt(8.W) - val kernel_dim = UInt(8.W) - val pool_size = UInt(8.W) - val pool_stride = UInt(8.W) - val pool_padding = UInt(8.W) - val dram_addr_bias = UInt(b.memDomain.memAddrLen.W) - val dram_addr_input = UInt(b.memDomain.memAddrLen.W) - val dram_addr_weight = UInt(b.memDomain.memAddrLen.W) - val dram_addr_output = UInt(b.memDomain.memAddrLen.W) - val input_stride = UInt(32.W) - val weight_stride = UInt(32.W) - val output_stride = UInt(32.W) - val bank_input = UInt(b.memDomain.vbankIdWidth.W) - val bank_weight = UInt(b.memDomain.vbankIdWidth.W) - val bank_output = UInt(b.memDomain.vbankIdWidth.W) - val no_bias = Bool() -} diff --git a/examples/balls/gemmini/arch/src/main/scala/LoopCmdEncoder.scala b/examples/balls/gemmini/arch/src/main/scala/LoopCmdEncoder.scala deleted file mode 100644 index 964498e4e..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/LoopCmdEncoder.scala +++ /dev/null @@ -1,143 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig -import framework.balldomain.blink.SubRobRow -import framework.frontend.scoreboard.BankAccessInfo -import framework.frontend.decoder.DomainId - -/** Converts LoopCmd (logical) → SubRobRow (hardware PostGDCmd) for SubROB. */ -@instantiable -class LoopCmdEncoder(val b: GlobalConfig) extends Module { - val bankIdLen = b.memDomain.vbankIdWidth - - def ballFunct(mnemonic: String): UInt = - b.ballDomain.ballISA - .find(_.mnemonic == mnemonic) - .map(_.funct7.U(7.W)) - .getOrElse(throw new IllegalArgumentException(s"$mnemonic not found in ballISA")) - - val preloadFunct = ballFunct("GEMMINI_PRELOAD") - val computePreFunct = ballFunct("GEMMINI_COMPUTE_PRELOADED") - val computeAccFunct = ballFunct("GEMMINI_COMPUTE_ACCUMULATED") - - @public - val io = IO(new Bundle { - val cmd = Flipped(Decoupled(new LoopCmd(b))) - val subRobRow = Decoupled(new SubRobRow(b)) - val ballId = Input(UInt(log2Up(b.ballDomain.ballNum).W)) - val masterRobId = Input(UInt(log2Up(b.frontend.rob_entries).W)) - }) - - // Passthrough handshake - io.subRobRow.valid := io.cmd.valid - io.cmd.ready := io.subRobRow.ready - - io.subRobRow.bits.ball_id := io.ballId - io.subRobRow.bits.master_rob_id := io.masterRobId - - for (i <- 0 until 4) { - val slot = io.subRobRow.bits.slots(i) - val lsub = io.cmd.bits.slots(i) - slot.valid := lsub.valid - - // Defaults — all unused RoCCCommandBB fields zero - slot.cmd.domain_id := 0.U - slot.cmd.ball_bid := io.ballId - slot.cmd.cmd.raw_inst := 0.U - slot.cmd.cmd.pc := 0.U - slot.cmd.cmd.funct := 0.U - slot.cmd.cmd.funct3 := 0.U - slot.cmd.cmd.rs2 := 0.U - slot.cmd.cmd.rs1 := 0.U - slot.cmd.cmd.xd := false.B - slot.cmd.cmd.xs1 := false.B - slot.cmd.cmd.xs2 := false.B - slot.cmd.cmd.rd := 0.U - slot.cmd.cmd.opcode := 0.U - slot.cmd.cmd.rs1Data := 0.U - slot.cmd.cmd.rs2Data := 0.U - slot.cmd.bankAccess := BankAccessInfo.none(bankIdLen) - slot.cmd.op1_col := 0.U - slot.cmd.op2_col := 0.U - slot.cmd.wr_col := 0.U - slot.cmd.isBarrier := false.B - - when(lsub.valid) { - switch(lsub.bits.cmdType) { - is(LoopSubCmdType.MSET_ALLOC) { - slot.cmd.domain_id := DomainId.MEM - slot.cmd.cmd.funct := 0x20.U // MSET (enable=010, opcode=0) - slot.cmd.cmd.rs1Data := lsub.bits.bank_id - slot.cmd.cmd.rs2Data := lsub.bits.bank_row.asUInt | - (lsub.bits.bank_col.asUInt << 5) | - (1.U << 10) // alloc=1 - slot.cmd.bankAccess.wr_bank_valid := true.B - slot.cmd.bankAccess.wr_bank_id := lsub.bits.bank_id - } - is(LoopSubCmdType.MSET_FREE) { - slot.cmd.domain_id := DomainId.MEM - slot.cmd.cmd.funct := 0x20.U // MSET - slot.cmd.cmd.rs1Data := lsub.bits.bank_id - slot.cmd.cmd.rs2Data := 0.U // alloc=0 - slot.cmd.bankAccess.wr_bank_valid := true.B - slot.cmd.bankAccess.wr_bank_id := lsub.bits.bank_id - } - is(LoopSubCmdType.MVIN) { - slot.cmd.domain_id := DomainId.MEM - slot.cmd.cmd.funct := 0x21.U // MVIN (enable=010, opcode=1) - slot.cmd.cmd.rs1Data := (lsub.bits.bank_id << 20) | (lsub.bits.iter << 30) - slot.cmd.cmd.rs2Data := lsub.bits.dram_addr | (lsub.bits.stride << 39) - slot.cmd.bankAccess.wr_bank_valid := true.B - slot.cmd.bankAccess.wr_bank_id := lsub.bits.bank_id - } - is(LoopSubCmdType.MVOUT) { - slot.cmd.domain_id := DomainId.MEM - slot.cmd.cmd.funct := 0x10.U // MVOUT (enable=001, opcode=0) - slot.cmd.cmd.rs1Data := lsub.bits.bank_id | (lsub.bits.iter << 30) - slot.cmd.cmd.rs2Data := lsub.bits.dram_addr | (lsub.bits.stride << 39) - slot.cmd.bankAccess.rd_bank_0_valid := true.B - slot.cmd.bankAccess.rd_bank_0_id := lsub.bits.bank_id - } - is(LoopSubCmdType.PRELOAD) { - slot.cmd.domain_id := DomainId.BALL - slot.cmd.cmd.funct := preloadFunct - slot.cmd.cmd.rs1Data := lsub.bits.op1_bank | - (lsub.bits.wr_bank << 20) | - (lsub.bits.iter << 30) - slot.cmd.cmd.rs2Data := 1.U - slot.cmd.bankAccess.rd_bank_0_valid := true.B - slot.cmd.bankAccess.rd_bank_0_id := lsub.bits.op1_bank - slot.cmd.bankAccess.wr_bank_valid := true.B - slot.cmd.bankAccess.wr_bank_id := lsub.bits.wr_bank - } - is(LoopSubCmdType.COMPUTE) { - slot.cmd.domain_id := DomainId.BALL - slot.cmd.cmd.funct := Mux( - lsub.bits.compute_mode === 0.U, - computePreFunct, - computeAccFunct - ) - slot.cmd.cmd.rs1Data := lsub.bits.op1_bank | - (lsub.bits.op2_bank << 10) | - (lsub.bits.wr_bank << 20) | - (lsub.bits.iter << 30) - slot.cmd.cmd.rs2Data := Mux( - lsub.bits.compute_mode === 0.U, - 2.U, - 3.U - ) | (lsub.bits.zero_op2 << 4) | - (lsub.bits.zero_op1_tail << 5) - slot.cmd.bankAccess.rd_bank_0_valid := true.B - slot.cmd.bankAccess.rd_bank_0_id := lsub.bits.op1_bank - slot.cmd.bankAccess.rd_bank_1_valid := !lsub.bits.zero_op2 - slot.cmd.bankAccess.rd_bank_1_id := lsub.bits.op2_bank - slot.cmd.bankAccess.wr_bank_valid := true.B - slot.cmd.bankAccess.wr_bank_id := lsub.bits.wr_bank - } - } - } - } -} diff --git a/examples/balls/gemmini/arch/src/main/scala/LoopConvUnroller.scala b/examples/balls/gemmini/arch/src/main/scala/LoopConvUnroller.scala deleted file mode 100644 index 778f50306..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/LoopConvUnroller.scala +++ /dev/null @@ -1,301 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.top.GlobalConfig -import examples.balls.gemmini.configs.GemminiBallParam - -/** - * LoopConvUnroller — expands LOOP_CONV_WS into LoopCmd rows. - * - * Simplified iteration: - * batch → orow → ocol → och → krow → kcol → kch (innermost) - * - * Each (orow, ocol, och, krow, kcol, kch) tile generates: - * MVIN input + MVIN weight → PRELOAD + COMPUTE → (on last k-iter) MVOUT - * - * First implementation: no pooling, no im2col tiling, single DIM×DIM tiles. - */ -@instantiable -class LoopConvUnroller(val b: GlobalConfig) extends Module { - val config = GemminiBallParam(b) - val DIM = config.blockSize - val elemSize = config.inputWidth / 8 - val bankBytes = b.memDomain.bankWidth / 8 - - @public - val io = IO(new Bundle { - val start = Flipped(Decoupled(new LoopConvWsConfig(b))) - val cmd = Decoupled(new LoopCmd(b)) - val busy = Output(Bool()) - }) - - val addrGen: Instance[LoopConvAddrGen] = Instantiate(new LoopConvAddrGen(b)) - - // FSM states - val sIdle :: sAlloc :: sMvinInput :: sMvinWeight :: sCompute :: sMvout :: sFree :: sDone :: Nil = - Enum(8) - val state = RegInit(sIdle) - - val cfg = Reg(new LoopConvWsConfig(b)) - - // Iterator registers - val batch_reg = RegInit(0.U(16.W)) - val orow_reg = RegInit(0.U(16.W)) - val ocol_reg = RegInit(0.U(16.W)) - val och_reg = RegInit(0.U(16.W)) - val krow_reg = RegInit(0.U(16.W)) - val kcol_reg = RegInit(0.U(16.W)) - val kch_reg = RegInit(0.U(16.W)) - - // Is this the first k-iteration for this (orow, ocol, och)? - val isFirstK = krow_reg === 0.U && kcol_reg === 0.U && kch_reg === 0.U - - // Is this the last k-iteration? - val isLastK = (krow_reg === cfg.kernel_dim - 1.U) && - (kcol_reg === cfg.kernel_dim - 1.U) && - (kch_reg + DIM.U >= cfg.in_channels) // Tile boundary - - // Connect address generator - addrGen.io.cfg := cfg - addrGen.io.batch := batch_reg - addrGen.io.orow := orow_reg - addrGen.io.ocol := ocol_reg - addrGen.io.och := och_reg - addrGen.io.krow := krow_reg - addrGen.io.kcol := kcol_reg - addrGen.io.kch := kch_reg - - // Defaults - io.start.ready := state === sIdle - io.cmd.valid := false.B - io.cmd.bits := 0.U.asTypeOf(new LoopCmd(b)) - io.busy := state =/= sIdle - - def emptySlot(): Valid[LoopSubCmd] = { - val v = Wire(Valid(new LoopSubCmd(b))) - v.valid := false.B - v.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - v - } - - def msetSlot( - bankId: UInt, - alloc: Boolean, - row: Int, - col: Int - ): Valid[LoopSubCmd] = { - val v = Wire(Valid(new LoopSubCmd(b))) - v.valid := true.B - v.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - v.bits.cmdType := (if (alloc) LoopSubCmdType.MSET_ALLOC - else LoopSubCmdType.MSET_FREE) - v.bits.bank_id := bankId - v.bits.bank_row := row.U - v.bits.bank_col := col.U - v - } - - def mvinSlot( - bankId: UInt, - addr: UInt, - iter: UInt, - stride: UInt - ): Valid[LoopSubCmd] = { - val v = Wire(Valid(new LoopSubCmd(b))) - v.valid := true.B - v.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - v.bits.cmdType := LoopSubCmdType.MVIN - v.bits.bank_id := bankId - v.bits.dram_addr := addr - v.bits.stride := stride - v.bits.iter := iter - v - } - - def outRows(): UInt = 1.U - - def inMemStride(bytes: UInt): UInt = - (bytes / bankBytes.U).pad(19)(18, 0) - - def outMemStride(bytes: UInt): UInt = - (bytes / (bankBytes * 4).U).pad(19)(18, 0) - - def weightTileStride(): UInt = - cfg.out_channels * elemSize.U - - // Advance k-iterators (kch → kcol → krow), then output iterators (och → ocol → orow → batch) - def advanceIter(): Unit = { - when(kch_reg + DIM.U < cfg.in_channels) { - kch_reg := kch_reg + DIM.U - }.elsewhen(kcol_reg + 1.U < cfg.kernel_dim) { - kch_reg := 0.U; kcol_reg := kcol_reg + 1.U - }.elsewhen(krow_reg + 1.U < cfg.kernel_dim) { - kch_reg := 0.U; kcol_reg := 0.U; krow_reg := krow_reg + 1.U - }.elsewhen(och_reg + DIM.U < cfg.out_channels) { - kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; - och_reg := och_reg + DIM.U - }.elsewhen(ocol_reg + 1.U < cfg.out_dim) { - kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; och_reg := 0.U - ocol_reg := ocol_reg + 1.U - }.elsewhen(orow_reg + 1.U < cfg.out_dim) { - kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; och_reg := 0.U; - ocol_reg := 0.U - orow_reg := orow_reg + 1.U - }.elsewhen(batch_reg + 1.U < cfg.batch_size) { - kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; och_reg := 0.U; - ocol_reg := 0.U; orow_reg := 0.U - batch_reg := batch_reg + 1.U - }.otherwise { - state := sFree // All iterations done - } - } - - // Track whether all iterations are done (used to decide next state) - val allDone = (batch_reg === cfg.batch_size - 1.U) && - (orow_reg === cfg.out_dim - 1.U) && (ocol_reg === cfg.out_dim - 1.U) && - (och_reg + DIM.U >= cfg.out_channels) && isLastK - - switch(state) { - is(sIdle) { - io.start.ready := true.B - when(io.start.fire) { - cfg := io.start.bits - batch_reg := 0.U; orow_reg := 0.U; ocol_reg := 0.U; och_reg := 0.U - krow_reg := 0.U; kcol_reg := 0.U; kch_reg := 0.U - state := sAlloc - } - } - - // [MSET_alloc(input), MSET_alloc(weight), MSET_alloc(output), --] - is(sAlloc) { - io.cmd.valid := true.B - io.cmd.bits - .slots(0) := msetSlot(cfg.bank_input, alloc = true, row = 1, col = 1) - io.cmd.bits - .slots(1) := msetSlot(cfg.bank_weight, alloc = true, row = 1, col = 1) - io.cmd.bits - .slots(2) := msetSlot(cfg.bank_output, alloc = true, row = 1, col = 4) - io.cmd.bits.slots(3) := emptySlot() - when(io.cmd.fire) { - state := sMvinInput - } - } - - // [MVIN_input, MVIN_weight, --, --] - is(sMvinInput) { - io.cmd.valid := true.B - val inputSlot = Mux( - addrGen.io.isPadding, - emptySlot(), // Skip MVIN if padding (load zeros implicitly) - mvinSlot( - cfg.bank_input, - addrGen.io.inputAddr, - outRows(), - inMemStride(cfg.input_stride) - ) - ) - io.cmd.bits.slots(0) := inputSlot - io.cmd.bits.slots(1) := mvinSlot( - cfg.bank_weight, - addrGen.io.weightAddr, - DIM.U, - inMemStride(weightTileStride()) - ) - io.cmd.bits.slots(2) := emptySlot() - io.cmd.bits.slots(3) := emptySlot() - when(io.cmd.fire) { - state := sCompute - } - } - - // [PRELOAD, COMPUTE, --, --] - is(sCompute) { - io.cmd.valid := true.B - val preSlot = Wire(Valid(new LoopSubCmd(b))) - preSlot.valid := true.B - preSlot.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - preSlot.bits.cmdType := LoopSubCmdType.PRELOAD - preSlot.bits.op1_bank := cfg.bank_weight - preSlot.bits.wr_bank := cfg.bank_output - preSlot.bits.iter := DIM.U - - val compSlot = Wire(Valid(new LoopSubCmd(b))) - compSlot.valid := true.B - compSlot.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - compSlot.bits.cmdType := LoopSubCmdType.COMPUTE - compSlot.bits.op1_bank := cfg.bank_input - compSlot.bits.op2_bank := cfg.bank_weight - compSlot.bits.wr_bank := cfg.bank_output - compSlot.bits.compute_mode := Mux( - isFirstK, - 0.U, - 1.U - ) // PRELOADED first, then ACCUMULATED - compSlot.bits.iter := DIM.U - compSlot.bits.zero_op2 := cfg.no_bias - compSlot.bits.zero_op1_tail := true.B - - io.cmd.bits.slots(0) := preSlot - io.cmd.bits.slots(1) := compSlot - io.cmd.bits.slots(2) := emptySlot() - io.cmd.bits.slots(3) := emptySlot() - - when(io.cmd.fire) { - when(isLastK) { - state := sMvout // Need to output results after last k iteration - }.otherwise { - advanceIter() - when(state =/= sFree)(state := sMvinInput) - } - } - } - - // [MVOUT, --, --, --] - is(sMvout) { - io.cmd.valid := true.B - val mvSlot = Wire(Valid(new LoopSubCmd(b))) - mvSlot.valid := true.B - mvSlot.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - mvSlot.bits.cmdType := LoopSubCmdType.MVOUT - mvSlot.bits.bank_id := cfg.bank_output - mvSlot.bits.dram_addr := addrGen.io.outputAddr - mvSlot.bits.stride := outMemStride(cfg.output_stride) - mvSlot.bits.iter := outRows() - - io.cmd.bits.slots(0) := mvSlot - io.cmd.bits.slots(1) := emptySlot() - io.cmd.bits.slots(2) := emptySlot() - io.cmd.bits.slots(3) := emptySlot() - - when(io.cmd.fire) { - when(allDone) { - state := sFree - }.otherwise { - advanceIter() - when(state =/= sFree)(state := sMvinInput) - } - } - } - - // [MSET_free(input), MSET_free(weight), MSET_free(output), --] - is(sFree) { - io.cmd.valid := true.B - io.cmd.bits - .slots(0) := msetSlot(cfg.bank_input, alloc = false, row = 0, col = 0) - io.cmd.bits - .slots(1) := msetSlot(cfg.bank_weight, alloc = false, row = 0, col = 0) - io.cmd.bits - .slots(2) := msetSlot(cfg.bank_output, alloc = false, row = 0, col = 0) - io.cmd.bits.slots(3) := emptySlot() - when(io.cmd.fire) { - state := sDone - } - } - - is(sDone) { - state := sIdle - } - } -} diff --git a/examples/balls/gemmini/arch/src/main/scala/LoopMatmulUnroller.scala b/examples/balls/gemmini/arch/src/main/scala/LoopMatmulUnroller.scala deleted file mode 100644 index 14055ff0f..000000000 --- a/examples/balls/gemmini/arch/src/main/scala/LoopMatmulUnroller.scala +++ /dev/null @@ -1,321 +0,0 @@ -package examples.balls.gemmini - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig -import examples.balls.gemmini.configs.GemminiBallParam - -/** - * LoopMatmulUnroller — expands LOOP_WS into a stream of LoopCmd rows. - * - * FSM: sIdle → sAllocBanks → sPrimeLoad → sMainLoop (Row1/Row2) → sDrainLast → sFreeBanks → sDone - * - * Row layout per iteration: - * Row 1: [PRELOAD(cur), COMPUTE(cur)] - * Row 2: [MVOUT(cur), MVIN_A(next), MVIN_B(next)] - * Last iteration: only Row 1, then sDrainLast handles final MVOUT. - */ -@instantiable -class LoopMatmulUnroller(val b: GlobalConfig) extends Module { - val config = GemminiBallParam(b) - val DIM = config.blockSize - val elemSize = config.inputWidth / 8 - val accBytes = config.accWidth / 8 - val bankBytes = b.memDomain.bankWidth / 8 - - @public - val io = IO(new Bundle { - val start = Flipped(Decoupled(new LoopWsConfig(b))) - val cmd = Decoupled(new LoopCmd(b)) - val busy = Output(Bool()) - }) - - // FSM states - val sIdle :: sAllocBanks :: sPrimeLoad :: sMainRow1 :: sMainRow2 :: sDrainLast :: sFreeBanks :: sDone :: Nil = - Enum(8) - val state = RegInit(sIdle) - - // Config registers (latched at start) - val cfg = Reg(new LoopWsConfig(b)) - - // Loop iterator registers - val i_reg = RegInit(0.U(16.W)) - val j_reg = RegInit(0.U(16.W)) - val k_reg = RegInit(0.U(16.W)) - - // Total iterations and current iteration index - val totalIter = cfg.max_i * cfg.max_j * cfg.max_k - val curIter = RegInit(0.U(32.W)) - - // Whether this is the last iteration - val isLastIter = curIter === (totalIter - 1.U) - - // Defaults - io.start.ready := state === sIdle - io.cmd.valid := false.B - io.cmd.bits := 0.U.asTypeOf(new LoopCmd(b)) - io.busy := state =/= sIdle - - // Helper to build an invalid LoopSubCmd slot - def emptySlot(): Valid[LoopSubCmd] = { - val v = Wire(Valid(new LoopSubCmd(b))) - v.valid := false.B - v.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - v - } - - // Helper to build an MSET slot - def msetSlot( - bankId: UInt, - alloc: Boolean, - row: Int, - col: Int - ): Valid[LoopSubCmd] = { - val v = Wire(Valid(new LoopSubCmd(b))) - v.valid := true.B - v.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - v.bits.cmdType := (if (alloc) LoopSubCmdType.MSET_ALLOC - else LoopSubCmdType.MSET_FREE) - v.bits.bank_id := bankId - v.bits.bank_row := row.U - v.bits.bank_col := col.U - v - } - - // Helper to build an MVIN slot - def mvinSlot( - bankId: UInt, - addr: UInt, - iter: UInt, - stride: UInt - ): Valid[LoopSubCmd] = { - val v = Wire(Valid(new LoopSubCmd(b))) - v.valid := true.B - v.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - v.bits.cmdType := LoopSubCmdType.MVIN - v.bits.bank_id := bankId - v.bits.dram_addr := addr - v.bits.stride := stride - v.bits.iter := iter - v - } - - // Helper to build an MVOUT slot - def mvoutSlot( - bankId: UInt, - addr: UInt, - iter: UInt, - stride: UInt - ): Valid[LoopSubCmd] = { - val v = Wire(Valid(new LoopSubCmd(b))) - v.valid := true.B - v.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - v.bits.cmdType := LoopSubCmdType.MVOUT - v.bits.bank_id := bankId - v.bits.dram_addr := addr - v.bits.stride := stride - v.bits.iter := iter - v - } - - // Address computation - // addr_a(i,k) = dram_addr_a + i * stride_a + k * DIM * elemSize - // addr_b(k,j) = dram_addr_b + k * stride_b + j * DIM * elemSize - // addr_c(i,j) = dram_addr_c + i * stride_c + j * DIM * accBytes - def addrA(i: UInt, k: UInt): UInt = - cfg.dram_addr_a + i * cfg.stride_a + k * (DIM * elemSize).U - - def addrB(k: UInt, j: UInt): UInt = - cfg.dram_addr_b + k * cfg.stride_b + j * (DIM * elemSize).U - - def addrC(i: UInt, j: UInt): UInt = - cfg.dram_addr_c + i * cfg.stride_c + j * (DIM * accBytes).U - - def inMemStride(bytes: UInt): UInt = - (bytes / bankBytes.U)(18, 0) - - def outMemStride(bytes: UInt): UInt = - (bytes / (bankBytes * 4).U)(18, 0) - - // Next iterator values (advance k, then j, then i) - val next_k = Wire(UInt(16.W)) - val next_j = Wire(UInt(16.W)) - val next_i = Wire(UInt(16.W)) - - when(k_reg + 1.U < cfg.max_k) { - next_k := k_reg + 1.U - next_j := j_reg - next_i := i_reg - }.elsewhen(j_reg + 1.U < cfg.max_j) { - next_k := 0.U - next_j := j_reg + 1.U - next_i := i_reg - }.otherwise { - next_k := 0.U - next_j := 0.U - next_i := i_reg + 1.U - } - - // Compute mode: first k iteration uses PRELOADED (0), rest use ACCUMULATED (1) - val computeMode = Mux(k_reg === 0.U, 0.U(2.W), 1.U(2.W)) - - switch(state) { - is(sIdle) { - io.start.ready := true.B - when(io.start.fire) { - cfg := io.start.bits - i_reg := 0.U - j_reg := 0.U - k_reg := 0.U - curIter := 0.U - state := sAllocBanks - } - } - - // Row 0: [MSET_alloc(A), MSET_alloc(B), MSET_alloc(C), --] - is(sAllocBanks) { - io.cmd.valid := true.B - io.cmd.bits - .slots(0) := msetSlot(cfg.bank_a, alloc = true, row = 1, col = 1) - io.cmd.bits - .slots(1) := msetSlot(cfg.bank_b, alloc = true, row = 1, col = 1) - io.cmd.bits.slots(2) := msetSlot( - cfg.bank_c, - alloc = true, - row = 1, - col = 4 - ) // accWidth = 4x inputWidth - io.cmd.bits.slots(3) := emptySlot() - when(io.cmd.fire) { - state := sPrimeLoad - } - } - - // Row 1: [MVIN_A(0), MVIN_B(0), --, --] - is(sPrimeLoad) { - io.cmd.valid := true.B - io.cmd.bits.slots(0) := mvinSlot( - cfg.bank_a, - addrA(0.U, 0.U), - DIM.U, - inMemStride(cfg.stride_a) - ) - io.cmd.bits.slots(1) := mvinSlot( - cfg.bank_b, - addrB(0.U, 0.U), - DIM.U, - inMemStride(cfg.stride_b) - ) - io.cmd.bits.slots(2) := emptySlot() - io.cmd.bits.slots(3) := emptySlot() - when(io.cmd.fire) { - state := sMainRow1 - } - } - - // Main loop Row 1: [PRELOAD(cur), COMPUTE(cur), --, --] - is(sMainRow1) { - io.cmd.valid := true.B - val preSlot = Wire(Valid(new LoopSubCmd(b))) - preSlot.valid := true.B - preSlot.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - preSlot.bits.cmdType := LoopSubCmdType.PRELOAD - preSlot.bits.op1_bank := cfg.bank_a // preload A (activations) into matrix array - preSlot.bits.wr_bank := cfg.bank_c - preSlot.bits.iter := DIM.U - - val compSlot = Wire(Valid(new LoopSubCmd(b))) - compSlot.valid := true.B - compSlot.bits := 0.U.asTypeOf(new LoopSubCmd(b)) - compSlot.bits.cmdType := LoopSubCmdType.COMPUTE - compSlot.bits.op1_bank := cfg.bank_a - compSlot.bits.op2_bank := cfg.bank_b - compSlot.bits.wr_bank := cfg.bank_c - compSlot.bits.compute_mode := computeMode - compSlot.bits.iter := DIM.U - - io.cmd.bits.slots(0) := preSlot - io.cmd.bits.slots(1) := compSlot - io.cmd.bits.slots(2) := emptySlot() - io.cmd.bits.slots(3) := emptySlot() - - when(io.cmd.fire) { - when(isLastIter) { - state := sDrainLast // Skip Row 2 for last iteration - }.otherwise { - state := sMainRow2 - } - } - } - - // Main loop Row 2: [MVOUT(cur), MVIN_A(next), MVIN_B(next), --] - is(sMainRow2) { - io.cmd.valid := true.B - io.cmd.bits.slots(0) := mvoutSlot( - cfg.bank_c, - addrC(i_reg, j_reg), - DIM.U, - outMemStride(cfg.stride_c) - ) - io.cmd.bits.slots(1) := mvinSlot( - cfg.bank_a, - addrA(next_i, next_k), - DIM.U, - inMemStride(cfg.stride_a) - ) - io.cmd.bits.slots(2) := mvinSlot( - cfg.bank_b, - addrB(next_k, next_j), - DIM.U, - inMemStride(cfg.stride_b) - ) - io.cmd.bits.slots(3) := emptySlot() - - when(io.cmd.fire) { - // Advance iterators - i_reg := next_i - j_reg := next_j - k_reg := next_k - curIter := curIter + 1.U - state := sMainRow1 - } - } - - // Drain: emit final MVOUT for last iteration - is(sDrainLast) { - io.cmd.valid := true.B - io.cmd.bits.slots(0) := mvoutSlot( - cfg.bank_c, - addrC(i_reg, j_reg), - DIM.U, - outMemStride(cfg.stride_c) - ) - io.cmd.bits.slots(1) := emptySlot() - io.cmd.bits.slots(2) := emptySlot() - io.cmd.bits.slots(3) := emptySlot() - when(io.cmd.fire) { - state := sFreeBanks - } - } - - // Free: [MSET_free(A), MSET_free(B), MSET_free(C), --] - is(sFreeBanks) { - io.cmd.valid := true.B - io.cmd.bits - .slots(0) := msetSlot(cfg.bank_a, alloc = false, row = 0, col = 0) - io.cmd.bits - .slots(1) := msetSlot(cfg.bank_b, alloc = false, row = 0, col = 0) - io.cmd.bits - .slots(2) := msetSlot(cfg.bank_c, alloc = false, row = 0, col = 0) - io.cmd.bits.slots(3) := emptySlot() - when(io.cmd.fire) { - state := sDone - } - } - - is(sDone) { - state := sIdle - } - } -} diff --git a/examples/balls/gemmini/compiler/python/quant.py b/examples/balls/gemmini/compiler/python/quant.py deleted file mode 100644 index 298f42767..000000000 --- a/examples/balls/gemmini/compiler/python/quant.py +++ /dev/null @@ -1,197 +0,0 @@ -import numpy as np -import torch -from buddy.compiler.graph.operation import Op, OpType, Conv2dOp, AddMMOp, TOp, PermuteOp -from buddy.compiler.graph.type import TensorDType -from buddy.compiler.ops import tosa as ops -from buddy_mlir import ir -from buddy_mlir.dialects import tosa -from stack.compiler.quant.rax import QuantTensor, RaxQuantPackage, write_rax - - -class GemminiConvOp(Op): - def __init__(self): - super().__init__() - self._op_type = OpType.ReduceType - - -def lower(node, symbols): - input, weight, bias = [symbols[(name, 0)] for name in node.args] - shape = list(node.tensor_meta["shape"]) - linear = len(shape) == 2 - if linear: - m, k = ir.RankedTensorType(input.type).shape - input = tosa.ReshapeOp( - input, - ops._create_shape_operand([m, k, 1, 1]), - results=[ir.RankedTensorType.get([m, k, 1, 1], ir.F32Type.get())], - ).result - result_shape = [shape[0], shape[1], 1, 1] - else: - result_shape = shape - attributes = { - name: ir.IntegerAttr.get(ir.IntegerType.get_signless(64), value) - for name, value in zip(("kh", "kw", "stride", "padding"), node.geometry) - } - attributes.update( - { - name: ir.FloatAttr.get(ir.F32Type.get(), float(value)) - for name, value in zip(("inputScale", "weightScale"), node.scales) - } - ) - result = ir.Operation.create( - "buckyball.gemmini_conv", - operands=[input, weight, bias], - results=[ir.RankedTensorType.get(result_shape, ir.F32Type.get())], - attributes=attributes, - ).result - if linear: - result = tosa.ReshapeOp( - result, - ops._create_shape_operand(shape), - results=[ir.RankedTensorType.get(shape, ir.F32Type.get())], - ).result - return result - - -def apply( - graph, - params, - names, - output, - name, - calibration, - quantized, - packer, - *, - reorder, - pool_result, -): - parameters = list(graph.params) - inputs = [graph._body[i] for i in graph._inputs] - positions = {p.name: i for i, p in enumerate(parameters)} - originals = {p.name: list(p.tensor_meta["shape"]) for p in parameters} - removed, consumed = set(), set() - occurrences = {} - for node in list(graph._body): - if isinstance(node, Conv2dOp): - activation, weight, bias = node.args[:3] - if node.args[5] != [1, 1] or node.args[6] or node.args[8] != 1: - raise ValueError( - "Gemmini model convolution requires unit dilation and one group" - ) - if len(set(node.args[3])) != 1 or len(set(node.args[4])) != 1: - raise ValueError( - "Gemmini model convolution requires equal spatial strides/padding" - ) - geometry = (*originals[weight][2:], node.args[3][0], node.args[4][0]) - elif isinstance(node, AddMMOp): - bias, activation, transposed = node.args[:3] - transpose = graph.node_table[transposed] - if not ( - isinstance(transpose, TOp) - or isinstance(transpose, PermuteOp) - and list(transpose.args[1]) == [1, 0] - ): - raise ValueError( - "Gemmini linear requires a transposed weight parameter" - ) - weight = transpose._parents[0] - removed.add(transposed) - geometry = (1, 1, 1, 0) - else: - continue - index = positions[weight] - parameter_name = names[index] - codes, scale = quantized[parameter_name] - if weight not in consumed: - packed = reorder(codes) - params[index] = torch.from_numpy(packed.copy()) - parameters[index].tensor_meta.update( - shape=list(packed.shape), dtype=TensorDType.Int8 - ) - consumed.add(weight) - occurrence = occurrences.get(parameter_name, 0) - input_scale = calibration[parameter_name][occurrence][0] - occurrences[parameter_name] = occurrence + 1 - replacement = GemminiConvOp() - replacement._name = node.name - replacement._arguments = [activation, weight, bias] - replacement._parents = [activation, weight, bias] - replacement._children = list(node._children) - replacement._tensor_meta = dict(node.tensor_meta) - replacement.trace_meta = node.trace_meta - replacement.geometry = geometry - replacement.scales = (input_scale, scale) - graph._body[graph._body.index(node)] = replacement - graph.node_table[node.name] = replacement - if node.name not in parameters[index]._children: - parameters[index]._children.append(node.name) - for key in removed: - node = graph.node_table[key] - if any( - not isinstance(graph.node_table[child], GemminiConvOp) - for child in node._children - ): - raise ValueError("Weight transpose has another consumer") - graph.node_table[node._parents[0]]._children.remove(key) - graph._body.remove(node) - del graph.node_table[key] - if {names[positions[p]] for p in consumed} != set(quantized): - raise ValueError("Gemmini quantization decisions do not match graph weights") - graph._fake_params = [graph._body.index(p) for p in parameters] - graph._inputs = [graph._body.index(p) for p in inputs] - for group in graph.op_groups: - graph.op_groups[group] = [ - graph.node_table[n.name] - for n in graph.op_groups[group] - if n.name not in removed - ] - graph._ops_registry["GemminiConvOp"] = lower - graph._ops_registry["MaxPool2dOp"] = pool_result - tensors, weights, floats, scales = [], [], [], [] - wo = fo = so = 0 - for node, parameter, parameter_name in zip(parameters, params, names): - raw = parameter.detach().numpy().tobytes() - if node.name in consumed: - scale = np.asarray(quantized[parameter_name][1], dtype=np.float32).tobytes() - tensors.append( - QuantTensor( - parameter_name, - originals[node.name], - list(parameter.shape), - "i8", - [], - wo, - len(raw), - so, - len(scale), - ) - ) - weights.append(raw) - scales.append(scale) - wo += len(raw) - so += len(scale) - else: - tensors.append( - QuantTensor( - parameter_name, - originals[node.name], - originals[node.name], - "f32", - [], - fo, - len(raw), - 0, - 0, - ) - ) - floats.append(raw) - fo += len(raw) - package = RaxQuantPackage( - tensors, b"".join(weights), b"".join(floats), b"".join(scales), {} - ) - output.mkdir(parents=True, exist_ok=True) - write_rax(package, output / f"{name}.rax", packer, name) - (output / "weights.bin").write_bytes(package.weights) - (output / "params.f32").write_bytes(package.params_f32) - (output / "scales.bin").write_bytes(package.scales_f32) diff --git a/examples/balls/gemmini/compiler/runtime/conv.cpp b/examples/balls/gemmini/compiler/runtime/conv.cpp deleted file mode 100644 index a7c8cc751..000000000 --- a/examples/balls/gemmini/compiler/runtime/conv.cpp +++ /dev/null @@ -1,95 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -extern "C" void _mlir_ciface_gemmini_conv(StridedMemRefType *input, - StridedMemRefType *weight, - StridedMemRefType *bias, - StridedMemRefType *output, - int64_t kh, int64_t kw, - int64_t stride, int64_t padding, - float inputScale, float weightScale) { - const int64_t batches = input->sizes[0], channels = input->sizes[1], - height = input->sizes[2], width = input->sizes[3]; - const int64_t oh = output->sizes[2], ow = output->sizes[3], - n = weight->sizes[1], k = channels * kh * kw, - m = batches * oh * ow; - if (k != weight->sizes[0] || n != bias->sizes[0] || - output->sizes[0] != batches || output->sizes[1] != n || - oh != (height + 2 * padding - kh) / stride + 1 || - ow != (width + 2 * padding - kw) / stride + 1 || - k > INT32_MAX / (128 * 128) || !std::isfinite(inputScale) || - inputScale <= 0 || !std::isfinite(weightScale) || weightScale <= 0) { - fputs("invalid Gemmini convolution shape/scales\n", stderr); - abort(); - } - int8_t a[256] __attribute__((aligned(64))), - b[256] __attribute__((aligned(64))); - int32_t c[256] __attribute__((aligned(64))); - bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mem_alloc(2, 1, 1); - bb_mem_alloc(3, 1, 4); - bb_mset_clear(2, 1, 1); - bb_gemmini_config(1, 0, 0, 0, 0); - for (int64_t row = 0; row < m; row += 16) - for (int64_t col = 0; col < n; col += 16) { - for (int64_t inner = 0; inner < k; inner += 16) { - for (int i = 0; i < 16; ++i) - for (int j = 0; j < 16; ++j) { - int64_t index = row + i, channel = (inner + j) % channels, - ky = (inner + j) / (kw * channels), - kx = ((inner + j) / channels) % kw; - int64_t batch = index / (oh * ow), - y = (index / ow) % oh * stride + ky - padding, - x = index % ow * stride + kx - padding; - float value = 0; - if (index < m && inner + j < k && y >= 0 && y < height && x >= 0 && - x < width) - value = - input->data[input->offset + batch * input->strides[0] + - channel * input->strides[1] + - y * input->strides[2] + x * input->strides[3]]; - if (!std::isfinite(value)) { - fputs("non-finite Gemmini activation\n", stderr); - abort(); - } - a[i * 16 + j] = int8_t( - std::clamp(std::nearbyint(value / inputScale), -128.f, 127.f)); - b[i * 16 + j] = - inner + i < k && col + j < n - ? weight->data[weight->offset + - (inner + i) * weight->strides[0] + - (col + j) * weight->strides[1]] - : 0; - } - bb_mvin((uintptr_t)a, 1, 16, 1); - bb_mvin((uintptr_t)b, 0, 16, 1); - if (inner == 0) { - bb_gemmini_preload(0, 3, 16, 0, 0); - bb_gemmini_compute_preloaded(1, 2, 3, 16, 0, 0, 0); - } else - bb_gemmini_compute_accumulated(1, 0, 3, 16, 0, 0, 0); - } - bb_mvout((uintptr_t)c, 3, 16, 1); - for (int i = 0; i < 16 && row + i < m; ++i) - for (int j = 0; j < 16 && col + j < n; ++j) { - int64_t index = row + i, batch = index / (oh * ow), - y = (index / ow) % oh, x = index % ow; - output->data[output->offset + batch * output->strides[0] + - (col + j) * output->strides[1] + y * output->strides[2] + - x * output->strides[3]] = - float(c[i * 16 + j]) * (inputScale * weightScale) + - bias->data[bias->offset + (col + j) * bias->strides[0]]; - } - } - bb_mem_release(0); - bb_mem_release(1); - bb_mem_release(2); - bb_mem_release(3); -} diff --git a/examples/balls/gemmini/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/gemmini/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 4a8b83091..000000000 --- a/examples/balls/gemmini/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,92 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateGemminiBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} - -namespace { -Value shift(PatternRewriter &rewriter, Location loc, Value value, - int64_t bits) { - return rewriter.create( - loc, value, rewriter.create(loc, bits, 64)); -} - -Value rs1(PatternRewriter &rewriter, Location loc, Value a, Value b, Value c, - Value iter) { - Value ab = rewriter.create(loc, a, shift(rewriter, loc, b, 10)); - Value ci = rewriter.create(loc, shift(rewriter, loc, c, 20), - shift(rewriter, loc, iter, 30)); - return rewriter.create(loc, ab, ci); -} - -Value rs2(PatternRewriter &rewriter, Location loc, Value a, Value b, Value c, - int64_t subcommand) { - Value packed = rewriter.create(loc, subcommand, 64); - packed = - rewriter.create(loc, packed, shift(rewriter, loc, a, 6)); - packed = - rewriter.create(loc, packed, shift(rewriter, loc, b, 16)); - return rewriter.create(loc, packed, - shift(rewriter, loc, c, 26)); -} - -class BankGemminiPreloadPattern - : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(BankGemminiPreloadOp op, - PatternRewriter &rewriter) const override { - Value zero = rewriter.create(op.getLoc(), 0, 64); - rewriter.create( - op.getLoc(), rewriter.getStringAttr("GEMMINI_PRELOAD"), - rs1(rewriter, op.getLoc(), op.getInBank(), zero, op.getOutBank(), - op.getIter()), - rs2(rewriter, op.getLoc(), op.getInBase(), zero, op.getOutBase(), 1)); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; - -template -class BankGemminiComputePattern : public OpRewritePattern { -public: - BankGemminiComputePattern(MLIRContext *context, StringRef mnemonic, - int64_t subcommand) - : OpRewritePattern(context), mnemonic(mnemonic), - subcommand(subcommand) {} - LogicalResult matchAndRewrite(Op op, - PatternRewriter &rewriter) const override { - rewriter.create( - op.getLoc(), rewriter.getStringAttr(mnemonic), - rs1(rewriter, op.getLoc(), op.getABank(), op.getBBank(), - op.getOutBank(), op.getIter()), - rs2(rewriter, op.getLoc(), op.getABase(), op.getBBase(), - op.getOutBase(), subcommand)); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } - -private: - StringRef mnemonic; - int64_t subcommand; -}; -} // namespace - -void mlir::buddy::populateGemminiBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); - patterns.add>( - patterns.getContext(), "GEMMINI_COMPUTE_PRELOADED", 2); - patterns.add>( - patterns.getContext(), "GEMMINI_COMPUTE_ACCUMULATED", 3); -} diff --git a/examples/balls/gemmini/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp b/examples/balls/gemmini/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp deleted file mode 100644 index b41ec08c8..000000000 --- a/examples/balls/gemmini/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/IR/PatternMatch.h" -using namespace mlir; -using namespace buddy::buckyball; -namespace { -struct ConvLowering : OpRewritePattern { - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(GemminiMemConvOp op, - PatternRewriter &b) const override { - buckyball_target::requireBuckyballBall("GemminiBall"); - auto input = cast(op.getInput().getType()), - weight = cast(op.getWeight().getType()), - bias = cast(op.getBias().getType()), - output = cast(op.getOutput().getType()); - if (input.getRank() != 4 || weight.getRank() != 2 || bias.getRank() != 1 || - output.getRank() != 4 || !input.getElementType().isF32() || - !weight.getElementType().isInteger(8) || - !bias.getElementType().isF32() || !output.getElementType().isF32()) - return op.emitError("Gemmini model ABI expects F32 NCHW, I8 matrix " - "weights and F32 bias/output"); - auto dynamic = ShapedType::kDynamic; - auto matrix = [&](int rank, Type element) { - return MemRefType::get( - SmallVector(rank, dynamic), element, - StridedLayoutAttr::get(b.getContext(), dynamic, - SmallVector(rank, dynamic))); - }; - auto a = matrix(4, b.getF32Type()), w = matrix(2, b.getI8Type()), - d = matrix(1, b.getF32Type()); - auto module = op->getParentOfType(); - auto loc = op.getLoc(); - if (!module.lookupSymbol("gemmini_conv")) { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(module.getBody()); - auto fn = b.create( - loc, "gemmini_conv", - b.getFunctionType({a, w, d, a, b.getI64Type(), b.getI64Type(), - b.getI64Type(), b.getI64Type(), b.getF32Type(), - b.getF32Type()}, - {})); - fn.setPrivate(); - fn->setAttr("llvm.emit_c_interface", b.getUnitAttr()); - } - b.create( - loc, "gemmini_conv", TypeRange{}, - ValueRange{b.create(loc, a, op.getInput()), - b.create(loc, w, op.getWeight()), - b.create(loc, d, op.getBias()), - b.create(loc, a, op.getOutput()), - b.create(loc, op.getKh(), 64), - b.create(loc, op.getKw(), 64), - b.create(loc, op.getStride(), 64), - b.create(loc, op.getPadding(), 64), - b.create(loc, b.getF32Type(), - op.getInputScaleAttr()), - b.create(loc, b.getF32Type(), - op.getWeightScaleAttr())}); - b.eraseOp(op); - return success(); - } -}; -} // namespace -namespace mlir::buddy { -void populateGemminiBallLowerBuckyballToBankSSAPatterns( - RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} -} // namespace mlir::buddy diff --git a/examples/balls/gemmini/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/gemmini/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 15acde3d4..000000000 --- a/examples/balls/gemmini/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; - -namespace { -class GemminiInstructionLowering - : public ConvertOpToLLVMPattern { -public: - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(GemminiInstructionOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("GemminiBall"); - rewriter.replaceOpWithNewOp( - op, adaptor.getRs1(), adaptor.getRs2(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7(op.getMnemonic()))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateGemminiBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - (void)stable; - (void)bankDepth; - - patterns.add(converter); -} - -void configureGemminiBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/gemmini/configs/GemminiBallParam.scala b/examples/balls/gemmini/configs/GemminiBallParam.scala deleted file mode 100644 index e13343ecd..000000000 --- a/examples/balls/gemmini/configs/GemminiBallParam.scala +++ /dev/null @@ -1,40 +0,0 @@ -package examples.balls.gemmini.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig - -case class GemminiBallParam( - meshRows: Int, - meshColumns: Int, - tileRows: Int, - tileColumns: Int, - inputWidth: Int, - accWidth: Int, - spatialOutputWidth: Int, - tileLatency: Int, - outputDelay: Int) { - - val totalRows: Int = meshRows * tileRows - val totalColumns: Int = meshColumns * tileColumns - val blockSize: Int = totalRows -} - -object GemminiBallParam { - private val ballName = "GemminiBall" - - def apply(b: GlobalConfig): GemminiBallParam = { - val tbl = BallParamLoader.ballTable(b, ballName) - GemminiBallParam( - meshRows = BallParamLoader.int(tbl, "meshRows"), - meshColumns = BallParamLoader.int(tbl, "meshColumns"), - tileRows = BallParamLoader.int(tbl, "tileRows"), - tileColumns = BallParamLoader.int(tbl, "tileColumns"), - inputWidth = BallParamLoader.int(tbl, "inputWidth"), - accWidth = BallParamLoader.int(tbl, "accWidth"), - spatialOutputWidth = BallParamLoader.int(tbl, "spatialOutputWidth"), - tileLatency = BallParamLoader.int(tbl, "tileLatency"), - outputDelay = BallParamLoader.int(tbl, "outputDelay") - ) - } - -} diff --git a/examples/balls/gemmini/configs/default.toml b/examples/balls/gemmini/configs/default.toml deleted file mode 100644 index 5958be6b8..000000000 --- a/examples/balls/gemmini/configs/default.toml +++ /dev/null @@ -1,12 +0,0 @@ -# Gemmini Ball Configuration - -[ball] -meshRows = 16 -meshColumns = 16 -tileRows = 1 -tileColumns = 1 -inputWidth = 8 -accWidth = 32 -spatialOutputWidth = 20 -tileLatency = 0 -outputDelay = 1 diff --git a/examples/balls/gemmini/emu/src/02_gemmini_config.rs b/examples/balls/gemmini/emu/src/02_gemmini_config.rs deleted file mode 100644 index 1cd2b5e05..000000000 --- a/examples/balls/gemmini/emu/src/02_gemmini_config.rs +++ /dev/null @@ -1,21 +0,0 @@ -//===- 02_gemmini_config.rs - GEMMINI_CONFIG instruction -------------------===// - -use super::gemmini_state::gemini; -use super::instruction::{BallInstruction, ExecContext}; - -pub struct GemminiConfig; - -impl BallInstruction for GemminiConfig { - fn exec(_xs1: u64, xs2: u64, _ctx: &mut ExecContext) -> u64 { - let mut g = gemini().lock().unwrap(); - g.cfg.dataflow = ((xs2 >> 4) & 1) as u8; - g.cfg.a_transpose = ((xs2 >> 7) & 1) != 0; - g.cfg.b_transpose = ((xs2 >> 8) & 1) != 0; - g.cfg.in_shift = ((xs2 >> 9) & 0xFFFFFFFF) as u32; - 0 - } - - fn latency(_xs1: u64, _xs2: u64) -> u64 { - 1 - } -} diff --git a/examples/balls/gemmini/emu/src/03_gemmini_flush.rs b/examples/balls/gemmini/emu/src/03_gemmini_flush.rs deleted file mode 100644 index 91333d08f..000000000 --- a/examples/balls/gemmini/emu/src/03_gemmini_flush.rs +++ /dev/null @@ -1,15 +0,0 @@ -//===- 03_gemmini_flush.rs - GEMMINI_FLUSH instruction ---------------------===// - -use super::instruction::{BallInstruction, ExecContext}; - -pub struct GemminiFlush; - -impl BallInstruction for GemminiFlush { - fn exec(_xs1: u64, _xs2: u64, _ctx: &mut ExecContext) -> u64 { - 0 - } - - fn latency(_xs1: u64, _xs2: u64) -> u64 { - 1 - } -} diff --git a/examples/balls/gemmini/emu/src/53_gemmini_preload.rs b/examples/balls/gemmini/emu/src/53_gemmini_preload.rs deleted file mode 100644 index 891d2d172..000000000 --- a/examples/balls/gemmini/emu/src/53_gemmini_preload.rs +++ /dev/null @@ -1,42 +0,0 @@ -use super::bank_matrix::read_i8_nn_at; -use super::decode::{pbank, rs1_b0, rs1_b2, rs1_iter}; -use super::gemmini_state::gemini; -use super::instruction::{BallInstruction, ExecContext}; - -pub struct GemminiPreload; - -impl BallInstruction for GemminiPreload { - fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let op1 = rs1_b0(xs1); - let wr = rs1_b2(xs1); - let n = rs1_iter(xs1) as usize; - - if !ctx.config(op1).allocated || !ctx.config(wr).allocated { - panic!("gemmini_preload: bank not allocated"); - } - if n == 0 || n > 64 { - panic!("gemmini_preload: bad iter"); - } - - let p1 = pbank(ctx, op1); - let mut gm = gemini().lock().unwrap(); - - if gm.cfg.dataflow == 1 { - let b = read_i8_nn_at(&ctx.banks, p1, ((xs2 >> 6) & 0x3ff) as usize, n); - gm.ws_b = Some(if gm.cfg.b_transpose { - (0..n).map(|i| (0..n).map(|j| b[j][i]).collect()).collect() - } else { - b - }); - } else { - // RTL OS preload only primes the mesh and injects D=0. It does - // not write the destination SRAM bank; compute produces C. - } - 0 - } - - fn latency(xs1: u64, _xs2: u64) -> u64 { - let n = rs1_iter(xs1).clamp(1, 64); - n.saturating_mul(n) - } -} diff --git a/examples/balls/gemmini/emu/src/80_gemmini_loop_ws.rs b/examples/balls/gemmini/emu/src/80_gemmini_loop_ws.rs deleted file mode 100644 index 6b82fa3ae..000000000 --- a/examples/balls/gemmini/emu/src/80_gemmini_loop_ws.rs +++ /dev/null @@ -1,143 +0,0 @@ -//===- 80_gemmini_loop_ws.rs - GEMMINI_LOOP_WS instruction -----------------===// - -use super::super::bank::{bank_row_bytes, MATRIX_SIZE}; -use super::gemmini_state::gemini; -use super::instruction::ExecContext; -use super::loop_micro_ops::{ - alloc, checked_stride, compute, free_after_hash, mvin, mvout, preload, -}; - -// Shared implementation -fn exec_cfg_impl(mnemonic: &str, xs2: u64) -> u64 { - let mut g = gemini().lock().unwrap(); - match mnemonic { - "GEMMINI_LOOP_WS_CONFIG_BOUNDS" => { - g.loop_ws.max_k = xs2 & 0xffff; - g.loop_ws.max_j = (xs2 >> 16) & 0xffff; - g.loop_ws.max_i = (xs2 >> 32) & 0xffff; - } - "GEMMINI_LOOP_WS_CONFIG_ADDR_A" => g.loop_ws.addr_a = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_WS_CONFIG_ADDR_B" => g.loop_ws.addr_b = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_WS_CONFIG_ADDR_D" => g.loop_ws.addr_d = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_WS_CONFIG_ADDR_C" => g.loop_ws.addr_c = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_WS_CONFIG_STRIDES_AB" => { - g.loop_ws.stride_a = xs2 & 0xffff_ffff; - g.loop_ws.stride_b = xs2 >> 32; - } - "GEMMINI_LOOP_WS_CONFIG_STRIDES_DC" => { - g.loop_ws.stride_d = xs2 & 0xffff_ffff; - g.loop_ws.stride_c = xs2 >> 32; - } - _ => panic!("gemmini_loop_ws: unknown cfg mnemonic={mnemonic}"), - } - 0 -} - -fn checked_tile_addr( - base: u64, - outer: u64, - outer_stride: u64, - inner: u64, - inner_stride: u64, -) -> u64 { - let outer_offset = outer - .checked_mul(outer_stride) - .expect("gemmini_loop_ws: outer address offset overflow"); - let inner_offset = inner - .checked_mul(inner_stride) - .expect("gemmini_loop_ws: inner address offset overflow"); - base.checked_add(outer_offset) - .and_then(|addr| addr.checked_add(inner_offset)) - .expect("gemmini_loop_ws: address overflow") -} - -fn exec_loop_impl(xs2: u64, ctx: &mut ExecContext) -> u64 { - let g = gemini().lock().unwrap(); - let lw = g.loop_ws.clone(); - let dataflow = g.cfg.dataflow; - drop(g); - - let bank_a = xs2 & 0x3ff; - let bank_b = (xs2 >> 10) & 0x3ff; - let bank_c = (xs2 >> 20) & 0x3ff; - let low_d = ((xs2 >> 30) & 1) != 0; - assert!( - bank_a != bank_b && bank_a != bank_c && bank_b != bank_c, - "gemmini_loop_ws: banks must be distinct" - ); - assert!( - lw.max_i > 0 && lw.max_j > 0 && lw.max_k > 0, - "gemmini_loop_ws: bounds must be > 0" - ); - assert_eq!( - dataflow, 0, - "gemmini_loop_ws: current RTL CISC contract supports OS dataflow only" - ); - assert!( - low_d && lw.addr_d == 0 && lw.stride_d == 0, - "gemmini_loop_ws: only zero D is supported" - ); - - let dim = MATRIX_SIZE as u64; - let bank_bytes = bank_row_bytes() as u64; - let stride_a = checked_stride(lw.stride_a, bank_bytes, "gemmini_loop_ws stride_a"); - let stride_b = checked_stride(lw.stride_b, bank_bytes, "gemmini_loop_ws stride_b"); - let stride_c = checked_stride( - lw.stride_c, - bank_bytes - .checked_mul(4) - .expect("gemmini_loop_ws: output row width overflow"), - "gemmini_loop_ws stride_c", - ); - - alloc(ctx, bank_a, 1); - alloc(ctx, bank_b, 1); - alloc(ctx, bank_c, 4); - - mvin(ctx, bank_a, lw.addr_a, dim, stride_a); - mvin(ctx, bank_b, lw.addr_b, dim, stride_b); - - for i in 0..lw.max_i { - for j in 0..lw.max_j { - for k in 0..lw.max_k { - preload(ctx, bank_a, bank_c, dim); - compute(ctx, k != 0, bank_a, bank_b, bank_c, dim, false, false); - - let addr_c = checked_tile_addr(lw.addr_c, i, lw.stride_c, j, dim * 4); - mvout(ctx, bank_c, addr_c, dim, stride_c); - - let next = if k + 1 < lw.max_k { - Some((i, j, k + 1)) - } else if j + 1 < lw.max_j { - Some((i, j + 1, 0)) - } else if i + 1 < lw.max_i { - Some((i + 1, 0, 0)) - } else { - None - }; - if let Some((next_i, next_j, next_k)) = next { - let addr_a = checked_tile_addr(lw.addr_a, next_i, lw.stride_a, next_k, dim); - let addr_b = checked_tile_addr(lw.addr_b, next_k, lw.stride_b, next_j, dim); - mvin(ctx, bank_a, addr_a, dim, stride_a); - mvin(ctx, bank_b, addr_b, dim, stride_b); - } - } - } - } - - free_after_hash(ctx, bank_a); - free_after_hash(ctx, bank_b); - free_after_hash(ctx, bank_c); - 0 -} - -pub fn execute(mnemonic: &str, xs2: u64, ctx: &mut ExecContext) -> u64 { - match mnemonic { - "GEMMINI_LOOP_WS" => exec_loop_impl(xs2, ctx), - _ => exec_cfg_impl(mnemonic, xs2), - } -} - -pub fn latency(mnemonic: &str) -> u64 { - if mnemonic == "GEMMINI_LOOP_WS" { 256 } else { 1 } -} diff --git a/examples/balls/gemmini/emu/src/dispatch.rs b/examples/balls/gemmini/emu/src/dispatch.rs deleted file mode 100644 index 13aa22526..000000000 --- a/examples/balls/gemmini/emu/src/dispatch.rs +++ /dev/null @@ -1,58 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::{ - f02_gemmini_config, f03_gemmini_flush, f53_gemmini_preload, f66_gemmini_compute_preloaded, - f67_gemmini_compute_accumulated, f80_gemmini_loop_ws, f96_gemmini_loop_conv_ws, -}; - -const BALL_CLASS: &str = "examples.balls.gemmini.GemminiBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - let mnemonic = crate::config::ball_domain::mnemonic_for_funct(funct)?; - let result = match mnemonic.as_str() { - "GEMMINI_CONFIG" => f02_gemmini_config::GemminiConfig::exec(xs1, xs2, ctx), - "GEMMINI_FLUSH" => f03_gemmini_flush::GemminiFlush::exec(xs1, xs2, ctx), - "GEMMINI_PRELOAD" => f53_gemmini_preload::GemminiPreload::exec(xs1, xs2, ctx), - "GEMMINI_COMPUTE_PRELOADED" => { - f66_gemmini_compute_preloaded::GemminiComputePreloaded::exec(xs1, xs2, ctx) - } - "GEMMINI_COMPUTE_ACCUMULATED" => { - f67_gemmini_compute_accumulated::GemminiComputeAccumulated::exec(xs1, xs2, ctx) - } - name if name.starts_with("GEMMINI_LOOP_WS_") || name == "GEMMINI_LOOP_WS" => { - f80_gemmini_loop_ws::execute(name, xs2, ctx) - } - name if name.starts_with("GEMMINI_LOOP_CONV_WS_") || name == "GEMMINI_LOOP_CONV_WS" => { - f96_gemmini_loop_conv_ws::execute(name, xs2, ctx) - } - _ => return None, - }; - Some(result) -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - let mnemonic = crate::config::ball_domain::mnemonic_for_funct(funct)?; - let latency = match mnemonic.as_str() { - "GEMMINI_CONFIG" => f02_gemmini_config::GemminiConfig::latency(xs1, xs2), - "GEMMINI_FLUSH" => f03_gemmini_flush::GemminiFlush::latency(xs1, xs2), - "GEMMINI_PRELOAD" => f53_gemmini_preload::GemminiPreload::latency(xs1, xs2), - "GEMMINI_COMPUTE_PRELOADED" => f66_gemmini_compute_preloaded::GemminiComputePreloaded::latency(xs1, xs2), - "GEMMINI_COMPUTE_ACCUMULATED" => f67_gemmini_compute_accumulated::GemminiComputeAccumulated::latency(xs1, xs2), - name if name.starts_with("GEMMINI_LOOP_WS_") || name == "GEMMINI_LOOP_WS" => f80_gemmini_loop_ws::latency(name), - name if name.starts_with("GEMMINI_LOOP_CONV_WS_") || name == "GEMMINI_LOOP_CONV_WS" => f96_gemmini_loop_conv_ws::latency(name), - _ => return None, - }; - Some(latency) -} diff --git a/examples/balls/gemmini/emu/src/lib.rs b/examples/balls/gemmini/emu/src/lib.rs deleted file mode 100644 index 9ee62e0fb..000000000 --- a/examples/balls/gemmini/emu/src/lib.rs +++ /dev/null @@ -1,22 +0,0 @@ -pub(crate) use crate::inst::{bank_matrix, decode, instruction}; - -#[path = "02_gemmini_config.rs"] -mod f02_gemmini_config; -#[path = "03_gemmini_flush.rs"] -mod f03_gemmini_flush; -#[path = "53_gemmini_preload.rs"] -mod f53_gemmini_preload; -#[path = "66_gemmini_compute_preloaded.rs"] -mod f66_gemmini_compute_preloaded; -#[path = "67_gemmini_compute_accumulated.rs"] -mod f67_gemmini_compute_accumulated; -#[path = "80_gemmini_loop_ws.rs"] -mod f80_gemmini_loop_ws; -#[path = "96_gemmini_loop_conv_ws.rs"] -mod f96_gemmini_loop_conv_ws; -mod gemmini_state; -mod loop_micro_ops; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/gemmini/verify/src/tb_top.sv b/examples/balls/gemmini/verify/src/tb_top.sv deleted file mode 100644 index be6204609..000000000 --- a/examples/balls/gemmini/verify/src/tb_top.sv +++ /dev/null @@ -1,81 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import gemmini_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - GemminiBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_channelReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_bankWrite_1_io_req_ready(intf.bank_write_req_ready[1]), - .io_bankWrite_1_io_req_valid(intf.bank_write_req_valid[1]), - .io_bankWrite_1_io_resp_ready(intf.bank_write_resp_ready[1]), - .io_bankWrite_1_io_resp_valid(intf.bank_write_resp_valid[1]), - .io_bankWrite_1_io_resp_bits_ok(intf.bank_write_resp_ok[1]), - .io_bankWrite_2_io_req_ready(intf.bank_write_req_ready[2]), - .io_bankWrite_2_io_req_valid(intf.bank_write_req_valid[2]), - .io_bankWrite_2_io_resp_ready(intf.bank_write_resp_ready[2]), - .io_bankWrite_2_io_resp_valid(intf.bank_write_resp_valid[2]), - .io_bankWrite_2_io_resp_bits_ok(intf.bank_write_resp_ok[2]), - .io_bankWrite_3_io_req_ready(intf.bank_write_req_ready[3]), - .io_bankWrite_3_io_req_valid(intf.bank_write_req_valid[3]), - .io_bankWrite_3_io_resp_ready(intf.bank_write_resp_ready[3]), - .io_bankWrite_3_io_resp_valid(intf.bank_write_resp_valid[3]), - .io_bankWrite_3_io_resp_bits_ok(intf.bank_write_resp_ok[3]), - .io_subRobReq_ready(intf.sub_rob_req_ready) - ); - - initial begin - intf.clock = 1'b0; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("gemmini_ball_test"); - end -endmodule diff --git a/examples/balls/gemmini/workloads/ctests/CMakeLists.txt b/examples/balls/gemmini/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 229898a07..000000000 --- a/examples/balls/gemmini/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -add_buckyball_ctests( - gemmini_accumulated_16x16_test.c - gemmini_os_cisc_atranspose_test.c - gemmini_os_cisc_basic_test.c - gemmini_os_cisc_btranspose_test.c - gemmini_os_cisc_shift_test.c - gemmini_os_risc_abtranspose_test.c - gemmini_os_risc_atranspose_test.c - gemmini_os_risc_basic_test.c - gemmini_os_risc_btranspose_test.c - gemmini_os_risc_shift_test.c - gemmini_ws_cisc_conv_test.c - gemmini_ws_risc_basic_test.c - gemmini_ws_risc_btranspose_test.c - gemmini_ws_risc_shift_test.c -) diff --git a/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir b/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir deleted file mode 100644 index 496db7704..000000000 --- a/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir +++ /dev/null @@ -1,68 +0,0 @@ -func.func private @check_result(memref<16x16xi32>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %one_i8 = arith.constant 1 : i8 - %initial = arith.constant 7 : i32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %sixteen = arith.constant 16 : index - %depth16 = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %a = memref.alloc() : memref<16x16xi8> - %b = memref.alloc() : memref<16x16xi8> - %c = memref.alloc() : memref<16x16xi32> - linalg.fill ins(%initial : i32) outs(%c : memref<16x16xi32>) - scf.for %row = %zero to %sixteen step %one { - scf.for %col = %zero to %sixteen step %one { - %r3 = arith.muli %row, %three : index - %c5 = arith.muli %col, %five : index - %asum = arith.addi %r3, %c5 : index - %arem = arith.remui %asum, %seven : index - %aval = arith.subi %arem, %three : index - %av = arith.index_cast %aval : index to i8 - %r2 = arith.muli %row, %two : index - %c3 = arith.muli %col, %three : index - %bsum = arith.addi %r2, %c3 : index - %brem = arith.remui %bsum, %five : index - %bval = arith.subi %brem, %two : index - %bv = arith.index_cast %bval : index to i8 - memref.store %av, %a[%row, %col] : memref<16x16xi8> - memref.store %bv, %b[%row, %col] : memref<16x16xi8> - } - } - %a_bank = arith.constant 0 : i64 - %b_bank = arith.constant 1 : i64 - %c_bank = arith.constant 3 : i64 - buckyball.mset %a_bank : i64 - buckyball.mset %b_bank : i64 - buckyball.mset %c_bank : i64 - buckyball.mvin %a %a_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mvin %b %b_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mvin %c %c_bank %depth16 %stride : memref<16x16xi32> i64 i64 i64 - %config = arith.constant 2064 : i64 - %zero64 = arith.constant 0 : i64 - %shift10 = arith.constant 10 : i64 - %shift20 = arith.constant 20 : i64 - %shift30 = arith.constant 30 : i64 - %iter_bits = arith.shli %depth16, %shift30 : i64 - %c_bits = arith.shli %c_bank, %shift20 : i64 - %b_bits = arith.shli %b_bank, %shift10 : i64 - %compute_rs1 = arith.ori %iter_bits, %b_bits : i64 - %compute_rs1_final = arith.ori %compute_rs1, %c_bits : i64 - %compute_rs2 = arith.constant 3 : i64 - buckyball.gemmini "GEMMINI_CONFIG" %zero64, %config : i64 - buckyball.gemmini "GEMMINI_COMPUTE_ACCUMULATED" %compute_rs1_final, %compute_rs2 : i64 - buckyball.mvout %c %c_bank %depth16 %stride : memref<16x16xi32> i64 i64 i64 - func.call @check_result(%c) : (memref<16x16xi32>) -> () - buckyball.mset %a_bank : i64 - buckyball.mset %b_bank : i64 - buckyball.mset %c_bank : i64 - memref.dealloc %a : memref<16x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %c : memref<16x16xi32> - return %zero_i8 : i8 -} diff --git a/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir b/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir deleted file mode 100644 index 00f54ed03..000000000 --- a/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir +++ /dev/null @@ -1,77 +0,0 @@ -func.func private @check_result(memref<16x16xi32>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %one_i8 = arith.constant 1 : i8 - %zero_i32 = arith.constant 0 : i32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %sixteen = arith.constant 16 : index - %depth16 = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %a = memref.alloc() : memref<16x16xi8> - %b = memref.alloc() : memref<16x16xi8> - %d = memref.alloc() : memref<16x16xi8> - %c = memref.alloc() : memref<16x16xi32> - linalg.fill ins(%zero_i8 : i8) outs(%d : memref<16x16xi8>) - linalg.fill ins(%zero_i32 : i32) outs(%c : memref<16x16xi32>) - scf.for %row = %zero to %sixteen step %one { - scf.for %col = %zero to %sixteen step %one { - %r3 = arith.muli %row, %three : index - %c5 = arith.muli %col, %five : index - %asum = arith.addi %r3, %c5 : index - %arem = arith.remui %asum, %seven : index - %aval = arith.subi %arem, %three : index - %av = arith.index_cast %aval : index to i8 - %r2 = arith.muli %row, %two : index - %c3 = arith.muli %col, %three : index - %bsum = arith.addi %r2, %c3 : index - %brem = arith.remui %bsum, %five : index - %bval = arith.subi %brem, %two : index - %bv = arith.index_cast %bval : index to i8 - memref.store %av, %a[%row, %col] : memref<16x16xi8> - memref.store %bv, %b[%row, %col] : memref<16x16xi8> - } - } - %a_bank = arith.constant 0 : i64 - %b_bank = arith.constant 1 : i64 - %d_bank = arith.constant 2 : i64 - %c_bank = arith.constant 3 : i64 - buckyball.mset %a_bank : i64 - buckyball.mset %b_bank : i64 - buckyball.mset %d_bank : i64 - buckyball.mset %c_bank : i64 - buckyball.mvin %a %a_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mvin %b %b_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mvin %d %d_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - %config = arith.constant 0 : i64 - %shift10 = arith.constant 10 : i64 - %shift20 = arith.constant 20 : i64 - %shift30 = arith.constant 30 : i64 - %iter_bits = arith.shli %depth16, %shift30 : i64 - %c_bits = arith.shli %c_bank, %shift20 : i64 - %preload_rs1_base = arith.ori %iter_bits, %d_bank : i64 - %preload_rs1 = arith.ori %preload_rs1_base, %c_bits : i64 - %b_bits = arith.shli %b_bank, %shift10 : i64 - %compute_rs1 = arith.ori %iter_bits, %b_bits : i64 - %compute_rs1_final = arith.ori %compute_rs1, %c_bits : i64 - %preload_rs2 = arith.constant 1 : i64 - %compute_rs2 = arith.constant 2 : i64 - buckyball.gemmini "GEMMINI_CONFIG" %config, %config : i64 - buckyball.gemmini "GEMMINI_PRELOAD" %preload_rs1, %preload_rs2 : i64 - buckyball.gemmini "GEMMINI_COMPUTE_PRELOADED" %compute_rs1_final, %compute_rs2 : i64 - buckyball.mvout %c %c_bank %depth16 %stride : memref<16x16xi32> i64 i64 i64 - func.call @check_result(%c) : (memref<16x16xi32>) -> () - buckyball.mset %a_bank : i64 - buckyball.mset %b_bank : i64 - buckyball.mset %d_bank : i64 - buckyball.mset %c_bank : i64 - memref.dealloc %a : memref<16x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %d : memref<16x16xi8> - memref.dealloc %c : memref<16x16xi32> - return %zero_i8 : i8 -} diff --git a/examples/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index a0c200267..000000000 --- a/examples/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) -buckyball_require_min_bank_lines(${BUCKYBALL_MLIR_ACTIVE_TARGET} 16 Gemmini) -add_buckyball_mlir_test(gemmini_compute_preloaded_bank_tiles TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) -add_buckyball_mlir_test(gemmini_compute_accumulated_bank_tiles TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir.in b/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir.in deleted file mode 100644 index d5f6071b1..000000000 --- a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir.in +++ /dev/null @@ -1,75 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x16xi32>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %o8 = arith.constant 1 : i8 - %z32 = arith.constant 0 : i32 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %dim = arith.constant 16 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %d16 = arith.constant 16 : i64 - %rows64 = arith.constant @BB_BANK_LINES@ : i64 - %zero64 = arith.constant 0 : i64 - %stride = arith.constant 1 : i64 - buckyball.gemmini "GEMMINI_CONFIG" %zero64, %zero64 : i64 - %a = memref.alloc() : memref<@BB_BANK_LINES@x16xi8> - %b = memref.alloc() : memref<16x16xi8> - %d = memref.alloc() : memref<16x16xi8> - %c = memref.alloc() : memref<@BB_BANK_LINES@x16xi32> - linalg.fill ins(%z8 : i8) outs(%d : memref<16x16xi8>) - linalg.fill ins(%z32 : i32) outs(%c : memref<@BB_BANK_LINES@x16xi32>) - scf.for %tile = %z to %rows step %dim { - scf.for %row = %z to %dim step %one { - %arow = arith.addi %tile, %row : index - scf.for %k = %z to %dim step %one { - %r3 = arith.muli %row, %three : index - %k5 = arith.muli %k, %five : index - %sum = arith.addi %r3, %k5 : index - %rem = arith.remui %sum, %seven : index - %value = arith.subi %rem, %three : index - %v8 = arith.index_cast %value : index to i8 - memref.store %v8, %a[%arow, %k] : memref<@BB_BANK_LINES@x16xi8> - } - } - } - scf.for %row = %z to %dim step %one { - scf.for %col = %z to %dim step %one { - %r2 = arith.muli %row, %two : index - %c3 = arith.muli %col, %three : index - %sum = arith.addi %r2, %c3 : index - %rem = arith.remui %sum, %five : index - %value = arith.subi %rem, %two : index - %v8 = arith.index_cast %value : index to i8 - memref.store %v8, %b[%row, %col] : memref<16x16xi8> - } - } - %ab = buckyball.bank_alloc - %bb = buckyball.bank_alloc - %db = buckyball.bank_alloc - %cb = buckyball.bank_alloc - %al = buckyball.bank_mvin %a %ab %rows64 %stride : memref<@BB_BANK_LINES@x16xi8> i64 i64 i64 - %bl = buckyball.bank_mvin %b %bb %d16 %stride : memref<16x16xi8> i64 i64 i64 - %dl = buckyball.bank_mvin %d %db %d16 %stride : memref<16x16xi8> i64 i64 i64 - %result = scf.for %base = %z to %rows step %dim iter_args(%c0 = %cb) -> (i64) { - %base64 = arith.index_cast %base : index to i64 - %p = buckyball.bank_gemmini_preload %dl %c0 %d16 %base64 %base64 : i64 i64 i64 i64 i64 - %first = buckyball.bank_gemmini_compute_preloaded %al %bl %p %d16 %base64 %zero64 %base64 : i64 i64 i64 i64 i64 i64 i64 - %sum = buckyball.bank_gemmini_compute_accumulated %al %bl %first %d16 %base64 %zero64 %base64 : i64 i64 i64 i64 i64 i64 i64 - scf.yield %sum : i64 - } - %stored = buckyball.bank_mvout %c %result %rows64 %stride : memref<@BB_BANK_LINES@x16xi32> i64 i64 i64 - func.call @check_result(%c) : (memref<@BB_BANK_LINES@x16xi32>) -> () - buckyball.bank_release %al : i64 - buckyball.bank_release %bl : i64 - buckyball.bank_release %dl : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %a : memref<@BB_BANK_LINES@x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %d : memref<16x16xi8> - memref.dealloc %c : memref<@BB_BANK_LINES@x16xi32> - return %z8 : i8 -} diff --git a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir.in b/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir.in deleted file mode 100644 index 1db91f60b..000000000 --- a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir.in +++ /dev/null @@ -1,74 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x16xi32>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %o8 = arith.constant 1 : i8 - %z32 = arith.constant 0 : i32 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %dim = arith.constant 16 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %rows64 = arith.constant @BB_BANK_LINES@ : i64 - %d16 = arith.constant 16 : i64 - %zero64 = arith.constant 0 : i64 - %stride = arith.constant 1 : i64 - buckyball.gemmini "GEMMINI_CONFIG" %zero64, %zero64 : i64 - %a = memref.alloc() : memref<@BB_BANK_LINES@x16xi8> - %b = memref.alloc() : memref<16x16xi8> - %d = memref.alloc() : memref<16x16xi8> - %c = memref.alloc() : memref<@BB_BANK_LINES@x16xi32> - linalg.fill ins(%z8 : i8) outs(%d : memref<16x16xi8>) - linalg.fill ins(%z32 : i32) outs(%c : memref<@BB_BANK_LINES@x16xi32>) - scf.for %tile = %z to %rows step %dim { - scf.for %row = %z to %dim step %one { - %arow = arith.addi %tile, %row : index - scf.for %k = %z to %dim step %one { - %r3 = arith.muli %row, %three : index - %k5 = arith.muli %k, %five : index - %sum = arith.addi %r3, %k5 : index - %rem = arith.remui %sum, %seven : index - %value = arith.subi %rem, %three : index - %v8 = arith.index_cast %value : index to i8 - memref.store %v8, %a[%arow, %k] : memref<@BB_BANK_LINES@x16xi8> - } - } - } - scf.for %row = %z to %dim step %one { - scf.for %col = %z to %dim step %one { - %r2 = arith.muli %row, %two : index - %c3 = arith.muli %col, %three : index - %sum = arith.addi %r2, %c3 : index - %rem = arith.remui %sum, %five : index - %value = arith.subi %rem, %two : index - %v8 = arith.index_cast %value : index to i8 - memref.store %v8, %b[%row, %col] : memref<16x16xi8> - } - } - %ab = buckyball.bank_alloc - %bb = buckyball.bank_alloc - %db = buckyball.bank_alloc - %cb = buckyball.bank_alloc - %al = buckyball.bank_mvin %a %ab %rows64 %stride : memref<@BB_BANK_LINES@x16xi8> i64 i64 i64 - %bl = buckyball.bank_mvin %b %bb %d16 %stride : memref<16x16xi8> i64 i64 i64 - %dl = buckyball.bank_mvin %d %db %d16 %stride : memref<16x16xi8> i64 i64 i64 - %result = scf.for %base = %z to %rows step %dim iter_args(%c0 = %cb) -> (i64) { - %base64 = arith.index_cast %base : index to i64 - %p = buckyball.bank_gemmini_preload %dl %c0 %d16 %base64 %base64 : i64 i64 i64 i64 i64 - %r = buckyball.bank_gemmini_compute_preloaded %al %bl %p %d16 %base64 %zero64 %base64 : i64 i64 i64 i64 i64 i64 i64 - scf.yield %r : i64 - } - %stored = buckyball.bank_mvout %c %result %rows64 %stride : memref<@BB_BANK_LINES@x16xi32> i64 i64 i64 - func.call @check_result(%c) : (memref<@BB_BANK_LINES@x16xi32>) -> () - buckyball.bank_release %al : i64 - buckyball.bank_release %bl : i64 - buckyball.bank_release %dl : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %a : memref<@BB_BANK_LINES@x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %d : memref<16x16xi8> - memref.dealloc %c : memref<@BB_BANK_LINES@x16xi32> - return %z8 : i8 -} diff --git a/examples/balls/im2col/arch/src/main/scala/Im2col.scala b/examples/balls/im2col/arch/src/main/scala/Im2col.scala deleted file mode 100644 index 6c411d08e..000000000 --- a/examples/balls/im2col/arch/src/main/scala/Im2col.scala +++ /dev/null @@ -1,223 +0,0 @@ -package examples.balls.im2col - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import examples.balls.im2col.configs.Im2colBallParam -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.top.GlobalConfig - -@instantiable -class Im2col(val b: GlobalConfig) extends Module { - private val ballCfg = Im2colBallParam(b) - private val maxIter = ballCfg.maxIter - private val maxKSize = ballCfg.maxKSize - private val maxPad = ballCfg.maxPadding - private val tile = 16 - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "Im2colBall") - .getOrElse(throw new IllegalArgumentException("Im2colBall not found in config")) - - private val im2colFunct = b.ballDomain.ballISA - .find(_.mnemonic == "IM2COL") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("IM2COL not found in ballISA")) - - require(mapping.inBW == 1, "Im2colBall requires inBW=1") - require(mapping.outBW == 1, "Im2colBall requires outBW=1") - require(b.memDomain.bankWidth == 128, "Im2colBall requires 128-bit bank rows") - require(isPow2(b.memDomain.bankEntries), "Im2colBall requires power-of-two bank depth") - require(maxIter + 2 * maxPad <= 255, "Im2colBall dimensions must fit the divider") - require((im2colFunct >> 4) == 3, "IM2COL must encode one read and one write") - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(mapping.inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(mapping.outBW, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - private val cfg = Module(new Im2colConfigRegs(b, maxIter, maxKSize, maxPad)) - private val win = Module(new Im2colWindow(maxKSize)) - private val div = Module(new RestoringDiv(16, 8)) - private val lineBuf: Instance[LineBufferManager] = Instantiate(new LineBufferManager(b)) - private val writer: Instance[StreamWriter] = Instantiate(new StreamWriter(b)) - - private val cIdle :: cDivRow :: cDivCol :: cCheck :: cDivStart :: cLaunch :: Nil = Enum(6) - private val calc = RegInit(cIdle) - - private val running = RegInit(false.B) - private val inputReady = RegInit(false.B) - private val respPending = RegInit(false.B) - private val armed = RegInit(false.B) - private val outRowsReg = RegInit(0.U(16.W)) - private val outColsReg = RegInit(0.U(16.W)) - private val startOutRow = RegInit(0.U(16.W)) - private val startOutCol = RegInit(0.U(16.W)) - private val localWindow = RegInit(0.U(12.W)) - private val elemAge = RegInit(0.U(2.W)) - private val launch = WireDefault(false.B) - - cfg.io.cmd := io.cmdReq.bits - cfg.io.load := io.cmdReq.fire - - io.cmdReq.ready := !running && !respPending && !armed && calc === cIdle - io.cmdResp.valid := respPending - io.cmdResp.bits.rob_id := cfg.io.robId - io.cmdResp.bits.is_sub := cfg.io.isSub - io.cmdResp.bits.sub_rob_id := cfg.io.subRobId - io.status.idle := !running && !respPending && !armed && calc === cIdle - io.status.running := running || armed || calc =/= cIdle - - div.io.start := false.B - div.io.a := 0.U - div.io.b := 0.U - - when(io.cmdReq.fire) { - val command = io.cmdReq.bits.cmd - assert(command.funct7 === im2colFunct.U, "Im2colBall received an unknown funct7") - assert(command.rs1(9, 0) < b.memDomain.virtualBankCount.U, "IM2COL input bank is invalid") - assert(command.rs1(19, 10) === 0.U, "IM2COL reserves input bank 1") - assert(command.rs1(29, 20) < b.memDomain.virtualBankCount.U, "IM2COL output bank is invalid") - assert(command.rs2(63) === 0.U, "IM2COL reserves rs2[63]") - assert(command.iter(b.frontend.iter_len - 1, 16) === 0.U, "IM2COL input size exceeds 16 bits") - assert( - command.op1_col === 1.U && command.op2_col === 0.U && command.wr_col === 1.U, - "IM2COL requires one physical input bank and one physical output bank" - ) - assert(command.rs2(7, 0) > 0.U && command.rs2(7, 0) <= maxKSize.U, "IM2COL kernel must be in the configured range") - armed := true.B - } - - when(armed) { - armed := false.B - assert(!cfg.io.invalid, "IM2COL command has an illegal shape or bank assignment") - val padded = cfg.io.inputSize +& (cfg.io.padding << 1) - div.io.start := true.B - div.io.a := (padded - cfg.io.kernel - cfg.io.startRow)(15, 0) - div.io.b := cfg.io.stride - calc := cDivRow - } - - when(calc === cDivRow && div.io.done) { - assert(div.io.r === 0.U, "IM2COL output row shape must be integral") - outRowsReg := div.io.q + 1.U - val padded = cfg.io.inputSize +& (cfg.io.padding << 1) - div.io.start := true.B - div.io.a := (padded - cfg.io.kernel - cfg.io.startCol)(15, 0) - div.io.b := cfg.io.stride - calc := cDivCol - } - - when(calc === cDivCol && div.io.done) { - assert(div.io.r === 0.U, "IM2COL output column shape must be integral") - outColsReg := div.io.q + 1.U - calc := cCheck - } - - when(calc === cCheck) { - val inputValues = cfg.io.inputSize * cfg.io.inputSize - val inputRows = (inputValues +& (tile - 1).U) >> log2Ceil(tile) - val windows = outRowsReg * outColsReg - val windowEnd = cfg.io.windowStart +& cfg.io.windowCount - val kernelElems = cfg.io.kernel * cfg.io.kernel - val mTiles = (cfg.io.windowCount +& (tile - 1).U) >> log2Ceil(tile) - val kTiles = (kernelElems +& (tile - 1).U) >> log2Ceil(tile) - val outputRows = mTiles * kTiles * tile.U - - assert(cfg.io.windowStart < windows && windowEnd <= windows, "IM2COL selected window range exceeds the output shape") - assert(inputRows <= b.memDomain.bankEntries.U, "IM2COL input footprint exceeds bank depth") - assert(outputRows <= b.memDomain.bankEntries.U, "IM2COL output footprint exceeds bank depth") - - div.io.start := true.B - div.io.a := cfg.io.windowStart - div.io.b := outColsReg(7, 0) - calc := cDivStart - } - - when(calc === cDivStart && div.io.done) { - assert(div.io.q < outRowsReg && div.io.r < outColsReg, "IM2COL window start decoding exceeded the output shape") - startOutRow := div.io.q - startOutCol := div.io.r - calc := cLaunch - } - - when(calc === cLaunch) { - launch := true.B - running := true.B - inputReady := false.B - localWindow := 0.U - elemAge := 0.U - calc := cIdle - } - - when(io.cmdResp.fire) { - respPending := false.B - } - - win.io.init := launch - win.io.next := false.B - win.io.kernel := cfg.io.kernel - win.io.outputCols := outColsReg - win.io.startRow := startOutRow - win.io.startCol := startOutCol - win.io.windowCount := cfg.io.windowCount - private val canEmitElem = running && inputReady && elemAge === 3.U - win.io.elemFire := canEmitElem && writer.io.elemIn.ready - - lineBuf.io.bankRead(0) <> io.bankRead(0) - lineBuf.io.start := launch - lineBuf.io.inRows := cfg.io.inputSize - lineBuf.io.inCols := cfg.io.inputSize - lineBuf.io.rowStride := cfg.io.stride - lineBuf.io.colStride := cfg.io.stride - lineBuf.io.padding := cfg.io.padding - lineBuf.io.startRow := cfg.io.startRow - lineBuf.io.startCol := cfg.io.startCol - lineBuf.io.inputBase := cfg.io.inputBase - lineBuf.io.lane := cfg.io.lane - lineBuf.io.outRow := win.io.outRow - lineBuf.io.outCol := win.io.outCol - lineBuf.io.kRowIdx := win.io.kRowIdx - lineBuf.io.kColIdx := win.io.kColIdx - lineBuf.io.rBankId := cfg.io.rBank - lineBuf.io.robId := cfg.io.robId - - writer.io.bankWrite(0) <> io.bankWrite(0) - writer.io.init := launch - writer.io.wBankId := cfg.io.wBank - writer.io.robId := cfg.io.robId - writer.io.elemIn.valid := canEmitElem - writer.io.elemIn.bits := lineBuf.io.elemData - writer.io.elemLast := win.io.elemLast - writer.io.finalWindow := win.io.last - writer.io.kRows := cfg.io.kernel - writer.io.kCols := cfg.io.kernel - writer.io.windowIdx := localWindow - - when(launch) { - inputReady := false.B - elemAge := 0.U - }.elsewhen(running && !inputReady && lineBuf.io.loadDone) { - inputReady := true.B - elemAge := 0.U - }.elsewhen(writer.io.elemIn.fire || writer.io.windowComplete) { - elemAge := 0.U - }.elsewhen(running && inputReady && elemAge =/= 3.U) { - elemAge := elemAge + 1.U - } - - when(writer.io.opComplete) { - inputReady := false.B - running := false.B - respPending := true.B - }.elsewhen(writer.io.windowComplete) { - win.io.next := true.B - localWindow := localWindow + 1.U - } - -} diff --git a/examples/balls/im2col/arch/src/main/scala/Im2colBall.scala b/examples/balls/im2col/arch/src/main/scala/Im2colBall.scala deleted file mode 100644 index 5e71a1593..000000000 --- a/examples/balls/im2col/arch/src/main/scala/Im2colBall.scala +++ /dev/null @@ -1,57 +0,0 @@ -package examples.balls.im2col - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BlinkIO, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.MmioRead -import examples.balls.im2col.Im2col -import framework.top.GlobalConfig - -/** - * Im2colBall - An Im2col computation Ball that complies with the Blink protocol - */ -@instantiable -class Im2colBall(val b: GlobalConfig) extends Module with HasBlink { - - val ballCommonConfig = b.ballDomain.ballIdMappings - .find(_.ballName == "Im2colBall") - .getOrElse( - throw new IllegalArgumentException("Im2colBall not found in config") - ) - - val inBW = ballCommonConfig.inBW - val outBW = ballCommonConfig.outBW - - @public - val io = IO(new BlinkIO(b, inBW, outBW)) - - def blink: BlinkIO = io - dontTouch(io) - - // Instantiate Im2col - val im2colUnit: Instance[Im2col] = Instantiate(new Im2col(b)) - - // Connect command interface - im2colUnit.io.cmdReq <> io.cmdReq - im2colUnit.io.cmdResp <> io.cmdResp - - for (i <- 0 until inBW) { - im2colUnit.io.bankRead(i) <> io.bankRead(i) - } - - // Connect SRAM write interface - Im2col needs to write to scratchpad - for (i <- 0 until outBW) { - im2colUnit.io.bankWrite(i) <> io.bankWrite(i) - } - - // Connect Status signals - directly obtained from internal unit - io.status <> im2colUnit.io.status - - // Ball does not use SubROB: tie off subRobReq - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - - // MMIO: this Ball does not consume MMIO metadata - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/im2col/arch/src/main/scala/Im2colConfigRegs.scala b/examples/balls/im2col/arch/src/main/scala/Im2colConfigRegs.scala deleted file mode 100644 index 1a5e29f09..000000000 --- a/examples/balls/im2col/arch/src/main/scala/Im2colConfigRegs.scala +++ /dev/null @@ -1,95 +0,0 @@ -package examples.balls.im2col - -import chisel3._ -import chisel3.util._ -import framework.balldomain.rs.BallRsIssue -import framework.top.GlobalConfig - -class Im2colConfigRegs( - val b: GlobalConfig, - maxIter: Int, - maxKSize: Int, - maxPadding: Int) - extends Module { - - private val kW = log2Ceil(maxKSize + 1) - - val io = IO(new Bundle { - val cmd = Input(new BallRsIssue(b)) - val load = Input(Bool()) - val invalid = Output(Bool()) - val robId = Output(UInt(log2Up(b.frontend.rob_entries).W)) - val isSub = Output(Bool()) - val subRobId = Output(UInt(log2Up(b.frontend.sub_rob_depth * 4).W)) - val rBank = Output(UInt(b.memDomain.vbankIdWidth.W)) - val wBank = Output(UInt(b.memDomain.vbankIdWidth.W)) - val inputSize = Output(UInt(16.W)) - val kernel = Output(UInt(kW.W)) - val stride = Output(UInt(8.W)) - val padding = Output(UInt(8.W)) - val startRow = Output(UInt(8.W)) - val startCol = Output(UInt(8.W)) - val inputBase = Output(UInt(6.W)) - val lane = Output(UInt(4.W)) - val windowStart = Output(UInt(6.W)) - val windowCount = Output(UInt(7.W)) - }) - - private val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSub = RegInit(false.B) - private val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val rBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val wBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val inputSize = RegInit(0.U(16.W)) - private val kernel = RegInit(0.U(kW.W)) - private val stride = RegInit(0.U(8.W)) - private val padding = RegInit(0.U(8.W)) - private val startRow = RegInit(0.U(8.W)) - private val startCol = RegInit(0.U(8.W)) - private val inputBase = RegInit(0.U(6.W)) - private val lane = RegInit(0.U(4.W)) - private val windowStart = RegInit(0.U(6.W)) - private val windowCount = RegInit(0.U(7.W)) - - when(io.load) { - robId := io.cmd.rob_id - isSub := io.cmd.is_sub - subRobId := io.cmd.sub_rob_id - rBank := io.cmd.cmd.op1_bank - wBank := io.cmd.cmd.wr_bank - inputSize := io.cmd.cmd.iter - kernel := io.cmd.cmd.rs2(7, 0)(kW - 1, 0) - stride := io.cmd.cmd.rs2(15, 8) - padding := io.cmd.cmd.rs2(23, 16) - startCol := io.cmd.cmd.rs2(31, 24) - startRow := io.cmd.cmd.rs2(39, 32) - inputBase := io.cmd.cmd.rs2(45, 40) - lane := io.cmd.cmd.rs2(49, 46) - windowStart := io.cmd.cmd.rs2(55, 50) - windowCount := io.cmd.cmd.rs2(62, 56) - } - - private val padded = inputSize +& (padding << 1) - io.invalid := inputSize === 0.U || inputSize > maxIter.U || - kernel === 0.U || kernel > maxKSize.U || stride === 0.U || - padding > maxPadding.U || startRow > padding || startCol > padding || - inputBase +& inputSize * inputSize > b.memDomain.bankEntries.U || - padded < kernel + startRow || padded < kernel + startCol || - windowCount === 0.U || rBank === wBank - - io.robId := robId - io.isSub := isSub - io.subRobId := subRobId - io.rBank := rBank - io.wBank := wBank - io.inputSize := inputSize - io.kernel := kernel - io.stride := stride - io.padding := padding - io.startRow := startRow - io.startCol := startCol - io.inputBase := inputBase - io.lane := lane - io.windowStart := windowStart - io.windowCount := windowCount -} diff --git a/examples/balls/im2col/arch/src/main/scala/Im2colWindow.scala b/examples/balls/im2col/arch/src/main/scala/Im2colWindow.scala deleted file mode 100644 index 2187d4a11..000000000 --- a/examples/balls/im2col/arch/src/main/scala/Im2colWindow.scala +++ /dev/null @@ -1,68 +0,0 @@ -package examples.balls.im2col - -import chisel3._ -import chisel3.util._ - -class Im2colWindow(maxK: Int) extends Module { - private val kW = log2Ceil(maxK + 1) - - val io = IO(new Bundle { - val init = Input(Bool()) - val next = Input(Bool()) - val elemFire = Input(Bool()) - val kernel = Input(UInt(kW.W)) - val outputCols = Input(UInt(16.W)) - val startRow = Input(UInt(16.W)) - val startCol = Input(UInt(16.W)) - val windowCount = Input(UInt(12.W)) - val outRow = Output(UInt(16.W)) - val outCol = Output(UInt(16.W)) - val kRowIdx = Output(UInt(kW.W)) - val kColIdx = Output(UInt(kW.W)) - val elemLast = Output(Bool()) - val last = Output(Bool()) - }) - - private val outRowReg = RegInit(0.U(16.W)) - private val outColReg = RegInit(0.U(16.W)) - private val kRowReg = RegInit(0.U(kW.W)) - private val kColReg = RegInit(0.U(kW.W)) - private val colsReg = RegInit(0.U(16.W)) - private val remaining = RegInit(0.U(12.W)) - - private val elemLast = kRowReg === io.kernel - 1.U && kColReg === io.kernel - 1.U - - when(io.init) { - outRowReg := io.startRow - outColReg := io.startCol - kRowReg := 0.U - kColReg := 0.U - colsReg := io.outputCols - remaining := io.windowCount - }.elsewhen(io.next) { - assert(remaining > 1.U, "Im2colWindow cannot advance past the selected range") - kRowReg := 0.U - kColReg := 0.U - remaining := remaining - 1.U - when(outColReg + 1.U === colsReg) { - outRowReg := outRowReg + 1.U - outColReg := 0.U - }.otherwise { - outColReg := outColReg + 1.U - } - }.elsewhen(io.elemFire && !elemLast) { - when(kColReg === io.kernel - 1.U) { - kColReg := 0.U - kRowReg := kRowReg + 1.U - }.otherwise { - kColReg := kColReg + 1.U - } - } - - io.outRow := outRowReg - io.outCol := outColReg - io.kRowIdx := kRowReg - io.kColIdx := kColReg - io.elemLast := elemLast - io.last := remaining === 1.U -} diff --git a/examples/balls/im2col/arch/src/main/scala/LineBufferManager.scala b/examples/balls/im2col/arch/src/main/scala/LineBufferManager.scala deleted file mode 100644 index f22ef3c8c..000000000 --- a/examples/balls/im2col/arch/src/main/scala/LineBufferManager.scala +++ /dev/null @@ -1,160 +0,0 @@ -package examples.balls.im2col - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.balldomain.blink.BankRead -import examples.balls.im2col.configs.Im2colBallParam -import framework.top.GlobalConfig - -@instantiable -class LineBufferManager(val b: GlobalConfig) extends Module { - private val ballCfg = Im2colBallParam(b) - private val maxIter = ballCfg.maxIter - private val maxKSize = ballCfg.maxKSize - private val elemWidth = ballCfg.inputWidth - private val bankWidth = b.memDomain.bankWidth - private val lanesPerBeat = bankWidth / elemWidth - require(isPow2(lanesPerBeat), "LineBuffer lanesPerBeat must be power of 2") - private val maxBeats = maxIter * maxIter - require(isPow2(maxBeats) && maxBeats >= 2, "LineBuffer depth must be an even power of two") - private val halfBeats = maxBeats / 2 - // FIXME: This is a local line-buffer address, sized for maxIter^2; it must not use - // the external bank address width. Each command's external footprint is - // checked separately against bankEntries in Im2colConfigRegs. - private val memoryAddrW = log2Ceil(maxBeats) - private val kW = log2Ceil(maxKSize + 1) - private val addrW = log2Ceil(b.memDomain.bankEntries) - - private val map = b.ballDomain.ballIdMappings - .find(_.ballName == "Im2colBall") - .getOrElse( - throw new IllegalArgumentException("Im2colBall not found in config") - ) - - private val inBW = map.inBW - - @public - val io = IO(new Bundle { - val bankRead = Vec(inBW, Flipped(new BankRead(b))) - val start = Input(Bool()) - val inRows = Input(UInt(16.W)) - val inCols = Input(UInt(16.W)) - val rowStride = Input(UInt(8.W)) - val colStride = Input(UInt(8.W)) - val padding = Input(UInt(8.W)) - val startRow = Input(UInt(8.W)) - val startCol = Input(UInt(8.W)) - val inputBase = Input(UInt(6.W)) - val lane = Input(UInt(log2Ceil(lanesPerBeat).W)) - val outRow = Input(UInt(16.W)) - val outCol = Input(UInt(16.W)) - val kRowIdx = Input(UInt(kW.W)) - val kColIdx = Input(UInt(kW.W)) - val rBankId = Input(UInt(b.memDomain.vbankIdWidth.W)) - val robId = Input(UInt(log2Up(b.frontend.rob_entries).W)) - val loadDone = Output(Bool()) - val elemData = Output(UInt(elemWidth.W)) - }) - - private val bufLo = SyncReadMem(halfBeats, UInt(bankWidth.W)) - private val bufHi = SyncReadMem(halfBeats, UInt(bankWidth.W)) - private val active = RegInit(false.B) - private val pending = RegInit(false.B) - private val beat = RegInit(0.U(addrW.W)) - private val dimW = log2Ceil(maxIter + 1) - private val beatsReg = RegInit(0.U((addrW + 1).W)) - - for (i <- 0 until inBW) { - io.bankRead(i).io.req.valid := false.B - io.bankRead(i).io.req.bits.addr := 0.U - io.bankRead(i).io.resp.ready := false.B - io.bankRead(i).bank_id := io.rBankId - io.bankRead(i).rob_id := io.robId - io.bankRead(i).ball_id := 0.U - io.bankRead(i).group_id := 0.U - } - - io.bankRead(0).io.req.valid := active && !pending - io.bankRead(0).io.req.bits.addr := (io.inputBase +& beat).asTypeOf(UInt(addrW.W)) - io.bankRead(0).io.resp.ready := pending - io.loadDone := !active - - when(io.start) { - val rows = io.inRows(dimW - 1, 0) - val cols = io.inCols(dimW - 1, 0) - beatsReg := rows * cols - active := true.B - pending := false.B - beat := 0.U - }.elsewhen(io.bankRead(0).io.req.fire) { - pending := true.B - } - - when(io.bankRead(0).io.resp.fire) { - pending := false.B - when(beat +& 1.U === beatsReg) { - active := false.B - }.otherwise { - beat := beat + 1.U - } - } - - private val paddedRow = io.startRow +& (io.outRow(dimW - 1, 0) * io.rowStride) +& io.kRowIdx - private val paddedCol = io.startCol +& (io.outCol(dimW - 1, 0) * io.colStride) +& io.kColIdx - private val rowValid = - paddedRow >= io.padding && paddedRow < io.padding + io.inRows - private val colValid = - paddedCol >= io.padding && paddedCol < io.padding + io.inCols - private val inBound = rowValid && colValid - private val srcRow = Mux(inBound, paddedRow - io.padding, 0.U) - private val srcCol = Mux(inBound, paddedCol - io.padding, 0.U) - private val elemIndex = srcRow(dimW - 1, 0) * io.inCols(dimW - 1, 0) + srcCol(dimW - 1, 0) - private val beatIndex = elemIndex - private val laneIndex = io.lane - private val memoryWrite = io.bankRead(0).io.resp.fire - private val memoryRead = !active - - private val memoryAddress = Mux( - memoryWrite, - beat.pad(memoryAddrW), - beatIndex(memoryAddrW - 1, 0) - ) - - private val memoryBank = memoryAddress(memoryAddrW - 1) - private val memoryRow = memoryAddress(log2Ceil(halfBeats) - 1, 0) - private val memoryEnable = memoryWrite || memoryRead - - private val loData = bufLo.readWrite( - memoryRow, - io.bankRead(0).io.resp.bits.data.asUInt, - memoryEnable && !memoryBank, - memoryWrite - ) - - private val hiData = bufHi.readWrite( - memoryRow, - io.bankRead(0).io.resp.bits.data.asUInt, - memoryEnable && memoryBank, - memoryWrite - ) - - private val readBank = RegEnable(memoryBank, false.B, memoryRead) - private val memoryData = Mux(readBank, hiData, loData) - private val laneR = RegInit(0.U(log2Ceil(lanesPerBeat).W)) - private val boundR = RegInit(false.B) - private val beatData = RegInit(0.U(bankWidth.W)) - private val laneD = RegInit(0.U(log2Ceil(lanesPerBeat).W)) - private val boundD = RegInit(false.B) - private val elemHold = RegInit(0.U(elemWidth.W)) - laneR := laneIndex - boundR := inBound - beatData := memoryData - laneD := laneR - boundD := boundR - private val lanes = - beatData.asTypeOf(Vec(lanesPerBeat, UInt(elemWidth.W))) - elemHold := Mux(boundD, lanes(laneD), 0.U) - - io.elemData := elemHold -} diff --git a/examples/balls/im2col/arch/src/main/scala/StreamWriter.scala b/examples/balls/im2col/arch/src/main/scala/StreamWriter.scala deleted file mode 100644 index f2c698eaf..000000000 --- a/examples/balls/im2col/arch/src/main/scala/StreamWriter.scala +++ /dev/null @@ -1,229 +0,0 @@ -package examples.balls.im2col - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -import framework.balldomain.blink.BankWrite -import framework.top.GlobalConfig -import examples.balls.im2col.configs.Im2colBallParam - -/** - * Packs im2col K tiles into 128-bit bank rows (SMatMulBall A layout). - * - * ((w / 16) * ceil(kernelElems / 16) + kt) * 16 + (w % 16) - * - * Partial final K-tile lanes and partial final M-tile rows are zero-filled. - */ -@instantiable -class StreamWriter(val b: GlobalConfig) extends Module { - private val ballCfg = Im2colBallParam(b) - private val maxKSize = ballCfg.maxKSize - private val elemWidth = ballCfg.inputWidth - private val bankWidth = b.memDomain.bankWidth - private val bankEntries = b.memDomain.bankEntries - private val lanesPerBeat = bankWidth / elemWidth - private val laneIdxWidth = log2Ceil(lanesPerBeat) - private val kW = log2Ceil(maxKSize + 1) - private val bankAddrW = log2Ceil(bankEntries) - private val maxKTiles = - (maxKSize * maxKSize + lanesPerBeat - 1) / lanesPerBeat - private val kTilesW = log2Ceil(maxKTiles + 1) - require( - isPow2(bankEntries), - "StreamWriter requires power-of-two bank depth" - ) - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "Im2colBall") - .getOrElse( - throw new IllegalArgumentException("Im2colBall not found in config") - ) - - private val outBW = mapping.outBW - - require(outBW == 1, "StreamWriter requires outBW=1") - require(bankWidth % elemWidth == 0) - require(lanesPerBeat == 16, "Im2col SMatMulBall layout requires 16 int8 lanes") - - @public - val io = IO(new Bundle { - val bankWrite = Vec(outBW, Flipped(new BankWrite(b))) - - val elemIn = Flipped(Decoupled(UInt(elemWidth.W))) - val elemLast = Input(Bool()) - val finalWindow = Input(Bool()) - val init = Input(Bool()) - val windowIdx = Input(UInt(32.W)) - val kRows = Input(UInt(kW.W)) - val kCols = Input(UInt(kW.W)) - val wBankId = Input(UInt(b.memDomain.vbankIdWidth.W)) - val robId = Input(UInt(log2Up(b.frontend.rob_entries).W)) - - val busy = Output(Bool()) - val windowComplete = Output(Bool()) - val opComplete = Output(Bool()) - }) - - private val zeros = VecInit(Seq.fill(lanesPerBeat)(0.U(elemWidth.W))) - - private val packCntReg = RegInit(0.U(log2Ceil(lanesPerBeat + 1).W)) - - private val packReg = RegInit( - VecInit(Seq.fill(lanesPerBeat)(0.U(elemWidth.W))) - ) - - private val chunkIdxReg = RegInit(0.U(kTilesW.W)) - private val wrPendingReg = RegInit(false.B) - private val issued = RegInit(false.B) - private val endWindowReg = RegInit(false.B) - private val finalWinReg = RegInit(false.B) - private val padActive = RegInit(false.B) - private val padMTile = RegInit(0.U(32.W)) - private val padKTiles = RegInit(0.U(kTilesW.W)) - private val padK = RegInit(0.U(kTilesW.W)) - private val padRow = RegInit(0.U(laneIdxWidth.W)) - private val padRow0 = RegInit(0.U(laneIdxWidth.W)) - private val wAddrReg = RegInit(0.U(bankAddrW.W)) - - private val kernelElems = io.kRows * io.kCols - private val kTilesReg = RegInit(0.U(kTilesW.W)) - - private val mTile = io.windowIdx >> laneIdxWidth - private val mRow = io.windowIdx(laneIdxWidth - 1, 0) - - private def tileOff(tile: UInt, kTiles: UInt): UInt = { - MuxLookup(kTiles, 0.U)( - Seq( - 1.U -> tile, - 2.U -> (tile << 1), - 3.U -> ((tile << 1) + tile), - 4.U -> (tile << 2) - ) - ) - } - - private val targetAddr = - ((tileOff(mTile, kTilesReg) + chunkIdxReg) << laneIdxWidth) + mRow - private val padAddr = - ((tileOff(padMTile, padKTiles) + padK) << laneIdxWidth) + padRow - private val writeFire = io.bankWrite(0).io.req.fire - private val writeResp = io.bankWrite(0).io.resp.fire - - io.busy := wrPendingReg || padActive - io.windowComplete := wrPendingReg && endWindowReg && writeResp && !finalWinReg - io.opComplete := Mux( - padActive, - wrPendingReg && writeResp && - (padK + 1.U === padKTiles) && (padRow === (lanesPerBeat - 1).U), - wrPendingReg && endWindowReg && writeResp && finalWinReg && - (mRow === (lanesPerBeat - 1).U) - ) - - for (i <- 0 until outBW) { - io.bankWrite(i).io.req.valid := false.B - io.bankWrite(i).io.req.bits.addr := 0.U - io.bankWrite(i).io.req.bits.data := 0.U - io.bankWrite(i).io.req.bits.mask := - VecInit(Seq.fill(b.memDomain.bankMaskLen)(false.B)) - io.bankWrite(i).io.resp.ready := false.B - io.bankWrite(i).bank_id := 0.U - io.bankWrite(i).rob_id := 0.U - io.bankWrite(i).ball_id := 0.U - io.bankWrite(i).group_id := 0.U - } - - io.bankWrite(0).io.req.valid := wrPendingReg && !issued - io.bankWrite(0).io.req.bits.addr := wAddrReg - io.bankWrite(0).io.req.bits.data := Cat(packReg.reverse) - io.bankWrite(0).io.req.bits.mask := - VecInit(Seq.fill(b.memDomain.bankMaskLen)(true.B)) - io.bankWrite(0).io.resp.ready := true.B - io.bankWrite(0).bank_id := io.wBankId - io.bankWrite(0).rob_id := io.robId - io.bankWrite(0).group_id := 0.U - - io.elemIn.ready := !wrPendingReg && !padActive - - when(io.init) { - packCntReg := 0.U - packReg := zeros - chunkIdxReg := 0.U - wrPendingReg := false.B - issued := false.B - endWindowReg := false.B - finalWinReg := false.B - wAddrReg := 0.U - padActive := false.B - padMTile := 0.U - padKTiles := 0.U - padK := 0.U - padRow := 0.U - padRow0 := 0.U - kTilesReg := ((kernelElems +& (lanesPerBeat - 1).U) >> laneIdxWidth) - .asTypeOf(UInt(kTilesW.W)) - assert( - ((kernelElems +& (lanesPerBeat - 1).U) >> laneIdxWidth) >= 1.U && - ((kernelElems +& (lanesPerBeat - 1).U) >> laneIdxWidth) <= 4.U, - "StreamWriter kTiles must be 1..4" - ) - }.otherwise { - when(writeFire) { - issued := true.B - } - when(writeResp) { - packCntReg := 0.U - packReg := zeros - wrPendingReg := false.B - issued := false.B - endWindowReg := false.B - when(padActive) { - when(padRow === (lanesPerBeat - 1).U) { - when(padK + 1.U === padKTiles) { - padActive := false.B - padK := 0.U - padRow := 0.U - }.otherwise { - padK := padK + 1.U - padRow := padRow0 - } - }.otherwise { - padRow := padRow + 1.U - } - }.elsewhen(endWindowReg) { - chunkIdxReg := 0.U - when(finalWinReg && (mRow =/= (lanesPerBeat - 1).U)) { - padActive := true.B - padMTile := mTile - padKTiles := kTilesReg - padK := 0.U - padRow := mRow + 1.U - padRow0 := mRow + 1.U - } - finalWinReg := false.B - }.otherwise { - chunkIdxReg := chunkIdxReg + 1.U - } - } - - when(padActive && !wrPendingReg) { - wrPendingReg := true.B - endWindowReg := false.B - finalWinReg := false.B - packReg := zeros - wAddrReg := padAddr.asTypeOf(UInt(bankAddrW.W)) - } - - when(io.elemIn.fire) { - packReg(packCntReg(laneIdxWidth - 1, 0)) := io.elemIn.bits - val nextCnt = packCntReg + 1.U - packCntReg := nextCnt - when(nextCnt === lanesPerBeat.U || io.elemLast) { - wrPendingReg := true.B - endWindowReg := io.elemLast - finalWinReg := io.elemLast && io.finalWindow - wAddrReg := targetAddr.asTypeOf(UInt(bankAddrW.W)) - } - } - } -} diff --git a/examples/balls/im2col/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/im2col/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 8f11d9c92..000000000 --- a/examples/balls/im2col/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,41 +0,0 @@ -//===- AssignPhysicalBankPatterns.cpp - Im2col bank assignment patterns ---===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateIm2colBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { - -class BankIm2colPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankIm2colOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), - op.getIter(), op.getKsize(), op.getStride(), - op.getPadding(), op.getInputBase(), op.getLane(), - op.getStartRowAttr(), op.getStartColAttr(), - op.getWindowStartAttr(), op.getWindowCountAttr()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateIm2colBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/im2col/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/im2col/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index a38a3c5e5..000000000 --- a/examples/balls/im2col/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,234 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -static LogicalResult -validateIm2colEncoding(Operation *op, Value iterValue, Value ksizeValue, - Value strideValue, Value paddingValue, - Value inputBaseValue, Value laneValue, int64_t startRow, - int64_t startCol, int64_t windowStart, - int64_t windowCount, int64_t bankDepth) { - auto getConst = [](Value value) -> std::optional { - if (auto constant = value.getDefiningOp()) - if (auto attr = dyn_cast(constant.getValue())) - return attr.getInt(); - return std::nullopt; - }; - auto iter = getConst(iterValue); - auto ksize = getConst(ksizeValue); - auto stride = getConst(strideValue); - auto padding = getConst(paddingValue); - auto inputBase = getConst(inputBaseValue); - auto lane = getConst(laneValue); - if (!iter || !ksize || !stride || !padding) - return op->emitError( - "Im2col instruction requires constant iter/ksize/stride/padding"); - if (*iter <= 0 || *ksize <= 0 || *ksize > 255 || *stride <= 0 || - *stride > 255 || *padding < 0 || *padding > 255 || startRow < 0 || - startRow > 255 || startCol < 0 || startCol > 255 || - (inputBase && (*inputBase < 0 || *inputBase >= 64)) || - (lane && (*lane < 0 || *lane >= 16))) - return op->emitError("Im2col instruction field out of range"); - int64_t inputRows = *iter * *iter; - if (inputBase && *inputBase + inputRows > bankDepth) - return op->emitError("Im2col input range exceeds the physical bank depth"); - if (startRow > *padding || startCol > *padding) - return op->emitError("Im2col startRow/startCol must not exceed padding"); - if (windowStart < 0 || windowStart >= 64 || windowCount <= 0 || - windowCount > 64) - return op->emitError( - "Im2col requires windowStart in [0, 63] and windowCount in [1, 64]"); - - int64_t padded = *iter + 2 * *padding; - if (padded < *ksize + startRow || padded < *ksize + startCol || - (padded - *ksize - startRow) % *stride || - (padded - *ksize - startCol) % *stride) - return op->emitError("Im2col shape does not produce integral windows"); - int64_t rows = (padded - *ksize - startRow) / *stride + 1; - int64_t cols = (padded - *ksize - startCol) / *stride + 1; - if (windowStart + windowCount > rows * cols) - return op->emitError("Im2col window range exceeds output shape"); - int64_t kernelTiles = (*ksize * *ksize + 15) / 16; - int64_t outputRows = (windowCount + 15) / 16 * kernelTiles * 16; - if (outputRows > bankDepth) - return op->emitError("Im2col output range exceeds the physical bank depth"); - return success(); -} - -struct Im2colLowering : public ConvertOpToLLVMPattern { - Im2colLowering(LLVMTypeConverter &converter, bool stable, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), stable(stable), - bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(Im2colOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("Im2colBall"); - if (failed(validateIm2colEncoding( - op, op.getIter(), op.getKsize(), op.getStride(), op.getPadding(), - op.getInputBase(), op.getLane(), op.getStartRow(), op.getStartCol(), - op.getWindowStart(), op.getWindowCount(), bankDepth))) - return failure(); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - - Value rs2 = adaptor.getKsize(); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getStride(), - cstI64(rewriter, loc, 8))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getPadding(), - cstI64(rewriter, loc, 16))); - Value startCol = cstI64(rewriter, loc, op.getStartCol()); - Value startRow = cstI64(rewriter, loc, op.getStartRow()); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, startCol, - cstI64(rewriter, loc, 24))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, startRow, - cstI64(rewriter, loc, 32))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getInputBase(), - cstI64(rewriter, loc, 40))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getLane(), - cstI64(rewriter, loc, 46))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create( - loc, cstI64(rewriter, loc, op.getWindowStart()), - cstI64(rewriter, loc, 50))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create( - loc, cstI64(rewriter, loc, op.getWindowCount()), - cstI64(rewriter, loc, 56))); - - if (stable) { - rewriter.replaceOpWithNewOp(op, rs1, rs2); - return success(); - } - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("IM2COL"))); - return success(); - } - -private: - bool stable = false; - int64_t bankDepth; -}; -struct BankIm2colLowering : public ConvertOpToLLVMPattern { - BankIm2colLowering(LLVMTypeConverter &converter, bool stable, - int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), stable(stable), - bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(BankIm2colOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("Im2colBall"); - if (failed(validateIm2colEncoding( - op, op.getIter(), op.getKsize(), op.getStride(), op.getPadding(), - op.getInputBase(), op.getLane(), op.getStartRow(), op.getStartCol(), - op.getWindowStart(), op.getWindowCount(), bankDepth))) - return failure(); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInBank(), - cstI64(rewriter, loc, 0), adaptor.getOutBank(), - adaptor.getIter()); - - Value rs2 = adaptor.getKsize(); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getStride(), - cstI64(rewriter, loc, 8))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getPadding(), - cstI64(rewriter, loc, 16))); - Value startCol = cstI64(rewriter, loc, op.getStartCol()); - Value startRow = cstI64(rewriter, loc, op.getStartRow()); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, startCol, - cstI64(rewriter, loc, 24))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, startRow, - cstI64(rewriter, loc, 32))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getInputBase(), - cstI64(rewriter, loc, 40))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getLane(), - cstI64(rewriter, loc, 46))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create( - loc, cstI64(rewriter, loc, op.getWindowStart()), - cstI64(rewriter, loc, 50))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create( - loc, cstI64(rewriter, loc, op.getWindowCount()), - cstI64(rewriter, loc, 56))); - - if (stable) { - rewriter.replaceOpWithNewOp(op, rs1, rs2); - return success(); - } - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("IM2COL"))); - return success(); - } - -private: - bool stable = false; - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateIm2colBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - patterns.add(converter, stable, bankDepth); - patterns.add(converter, stable, bankDepth); -} - -void configureIm2colBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - if (stable) - target.addLegalOp(); - else - target.addIllegalOp(); - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/im2col/configs/Im2colBallParam.scala b/examples/balls/im2col/configs/Im2colBallParam.scala deleted file mode 100644 index 82b8b58ae..000000000 --- a/examples/balls/im2col/configs/Im2colBallParam.scala +++ /dev/null @@ -1,25 +0,0 @@ -package examples.balls.im2col.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig - -case class Im2colBallParam( - maxIter: Int, - maxKSize: Int, - maxPadding: Int, - inputWidth: Int) - -object Im2colBallParam { - private val ballName = "Im2colBall" - - def apply(b: GlobalConfig): Im2colBallParam = { - val tbl = BallParamLoader.ballTable(b, ballName) - Im2colBallParam( - maxIter = BallParamLoader.int(tbl, "maxIter"), - maxKSize = BallParamLoader.int(tbl, "maxKSize"), - maxPadding = BallParamLoader.int(tbl, "maxPadding"), - inputWidth = BallParamLoader.int(tbl, "inputWidth") - ) - } - -} diff --git a/examples/balls/im2col/configs/default.toml b/examples/balls/im2col/configs/default.toml deleted file mode 100644 index 1316acf13..000000000 --- a/examples/balls/im2col/configs/default.toml +++ /dev/null @@ -1,29 +0,0 @@ -# Im2col Ball Configuration -# -# Legal command range (enforced fail-hard in bemu/RTL): -# iter in 1 ..= maxIter -# ksize in 1 ..= maxKSize # covers 1x1 / 3x3 / 5x5 / 7x7 -# stride >= 1 -# padding in 0 ..= maxPadding -# start_row / start_col in 0 ..= padding (asym pad: pad=max(lo,hi), start=pad-lo) -# paddedSize = iter + 2*padding >= ksize + start -# op1_bank != wr_bank -# inputBase + iter*iter <= bankEntries -# lane in 0 .. 15 -# outputDim = floor((iter + 2*padding - ksize - start) / stride) + 1 -# M = outputDim * outputDim -# K = ksize * ksize -# outputRows = ceil(M/16) * ceil(K/16) * 16 -# outputRows <= bankEntries # bankEntries from chip memdomain -# -# Partial final K-tile lanes and partial final M-tile rows are zero. -# One bank row is one spatial position with 16 INT8 channels. - -[ball] -inputWidth = 8 -# Resident Conv lowering can request a 4x4 output tile from a stride-2, -# kernel-7 producer, which needs a 13x13 local input. Keep this a power of two -# because the RTL line buffer splits maxIter^2 beats into two equal banks. -maxIter = 16 -maxKSize = 7 -maxPadding = 7 diff --git a/examples/balls/im2col/emu/src/dispatch.rs b/examples/balls/im2col/emu/src/dispatch.rs deleted file mode 100644 index 4128672a0..000000000 --- a/examples/balls/im2col/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::ExecContext; - -use super::im2col; - -const BALL_CLASS: &str = "examples.balls.im2col.Im2colBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("IM2COL") => Some(im2col::exec_im2col(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("IM2COL") => Some(im2col::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/im2col/emu/src/lib.rs b/examples/balls/im2col/emu/src/lib.rs deleted file mode 100644 index 88380c622..000000000 --- a/examples/balls/im2col/emu/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod im2col; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/im2col/verify/README.md b/examples/balls/im2col/verify/README.md deleted file mode 100644 index 273f1579c..000000000 --- a/examples/balls/im2col/verify/README.md +++ /dev/null @@ -1,102 +0,0 @@ -# Im2colBall UVM Verification - -Verifies the generated `Im2colBall` module directly using the shared Blink UVM framework. - -## Structure - -- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM transaction items and base env -- `src/common/im2col_defs.svh`: DPI imports and constants -- `src/common/im2col_items.svh`: `im2col_cmd_item` with `load_rust_case` -- `src/seq/im2col_sequences.svh`: `im2col_basic_seq` driving one case per run -- `src/cov/im2col_cov.svh`: ksize/stride/pad/iter coverage -- `src/env/im2col_scoreboard.svh`: preloads mem model, compares writes vs DPI dst words -- `src/env/im2col_env.svh`: extends `bb_blink_env#(1,1)` with scoreboard and cov -- `src/tests/im2col_test.svh`: directed cases 0,1 then random 2..21 with fixed seed -- `src/pkg/im2col_pkg.sv`: package include entry -- `src/tb_top.sv`: instantiates `bb_blink_if#(1,1)` and `Im2colBall` -- `filelists/im2col_ball.f`: VCS/Verilator filelist - -## Test plan - -`+UVM_TESTNAME=im2col_ball_test` runs: - -- case 0: directed k3 6x6 pad0 (ctest `im2col_k3_test.c` data) -- case 1: directed k3 6x6 pad1 (ctest `im2col_k3_pad_test.c` data) -- cases 2..21: random with fixed seed `0xCAFE_BABE`, iter in {3,4,5,6,7}, - ksize in {1,3,5}, stride=1, pad in {0,1}, distinct banks 0..7 - -Each case: reset, preload src words into `mem_model`, drive one command, wait for -`scb.done()` or timeout at 100000 cycles. Scoreboard compares observed writes -(data/addr/mask/bank/rob) with DPI `im2col_case_dst_word_*`. `im2col_cov` -requires ksize/stride/pad/iter bins hit across the whole test, else -`check_phase` fatal. - -Blink command fields (not ISA rs packing): - -- `iter` = cmd.iter -- `ksize` = cmd.special[7:0] -- `stride` = cmd.special[15:8] -- `padding` = cmd.special[23:16] -- banks = op1_bank / wr_bank; funct7 is injected from the selected Core ballISA -- TB: op1_col=1, wr_col=1, rs1=0, rs2=0 - -## BID (required) - -`+BID=` is mandatory. Missing it fatals at test start. There is no default bid. - -| Config | Plusarg | -|--------|---------| -| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=2` | -| `sims.verilator.BuckyballToyVerilatorConfig` | `+BID=3` | - -```console -./simv +UVM_TESTNAME=im2col_ball_test +BID=2 -``` - -## Build - -Enter the shared UVM/VCS environment: - -```console -nix develop ../../../../verify -``` - -Build the DPI reference library: - -```console -cargo build --manifest-path casegen/Cargo.toml -``` - -Compile from this directory (resolve `@UVM@` / `@RTL@` like bbdev): - -```console -# Do not pass -hsopt=off: VCS W-2024.09 segfaults on Im2colBall+UVM with it. -vcs -full64 -sverilog -timescale=1ns/1ps \ - $VCS_UVM_ARGS \ - -sv_lib casegen/target/debug/libim2col_casegen \ - -f build//im2col_ball.f -``` - -Run: - -```console -./simv +UVM_TESTNAME=im2col_ball_test +BID=2 -``` - -Or via bbdev: - -```console -bbdev uvm --build '--ball=im2col --config sims.verilator.BuckyballPebbleVerilatorConfig' -bbdev uvm --run '--ball=im2col' -- '+BID=2' -``` - -## Acceptance (2026-08-04) - -- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` -- [x] casegen pure im2col model (no ExecContext); whole-case DPI API -- [x] directed k3 pad0/pad1 scoreboard pass -- [x] deterministic random idx 2..21 pass -- [x] protocol + semantic cover enforced (`check_phase` fatal) -- [x] pebble: `+BID=2` — 22/22 -- [x] toy: `+BID=3` — 22/22 -- [x] Note: omit `-hsopt=off` for Im2col (VCS segfault); bbdev skips it for this ball diff --git a/examples/balls/im2col/verify/casegen/src/casegen.rs b/examples/balls/im2col/verify/casegen/src/casegen.rs deleted file mode 100644 index a482ed4f9..000000000 --- a/examples/balls/im2col/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,312 +0,0 @@ -use crate::model; - -pub const BANK_ROW_BYTES: usize = model::BANK_ROW_BYTES; -pub const MAX_WORDS: usize = 128; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Im2colCmd { - pub bid: u32, - pub iter: u32, - pub ksize: u32, - pub stride: u32, - pub padding: u32, - pub window_count: u32, - pub op1_bank: u32, - pub wr_bank: u32, - pub op1_col: u32, - pub wr_col: u32, - pub rob_id: u32, - pub num_src_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Im2colCase { - pub cmd: Im2colCmd, - pub src: Vec, - pub dst: Vec, -} - -impl Im2colCase { - pub fn src_word_lo(&self, word_index: usize) -> u64 { - word_lo(&self.src, word_index) - } - pub fn src_word_hi(&self, word_index: usize) -> u64 { - word_hi(&self.src, word_index) - } - pub fn dst_word_lo(&self, word_index: usize) -> u64 { - word_lo(&self.dst, word_index) - } - pub fn dst_word_hi(&self, word_index: usize) -> u64 { - word_hi(&self.dst, word_index) - } -} - -fn word_lo(data: &[u8], word_index: usize) -> u64 { - let off = word_index * BANK_ROW_BYTES; - if off + 8 > data.len() { - panic!("word_lo: word_index {word_index} out of range"); - } - let mut buf = [0u8; 8]; - buf.copy_from_slice(&data[off..off + 8]); - u64::from_le_bytes(buf) -} - -fn word_hi(data: &[u8], word_index: usize) -> u64 { - let off = word_index * BANK_ROW_BYTES + 8; - if off + 8 > data.len() { - panic!("word_hi: word_index {word_index} out of range"); - } - let mut buf = [0u8; 8]; - buf.copy_from_slice(&data[off..off + 8]); - u64::from_le_bytes(buf) -} - -pub fn gen_case(seed: u32, index: u32, bid: u32) -> Im2colCase { - match index { - 0 => directed_k3_pad0(bid), - 1 => directed_k3_pad1(bid), - _ => random_case(seed, index, bid), - } -} - -fn build_case( - bid: u32, - iter: usize, - ksize: usize, - stride: usize, - padding: usize, - op1_bank: u32, - wr_bank: u32, - rob_id: u32, - flat: &[u8], -) -> Im2colCase { - if op1_bank == wr_bank { - panic!("build_case: op1_bank and wr_bank must differ"); - } - if op1_bank > 7 || wr_bank > 7 { - panic!("build_case: banks must be in 0..7"); - } - if flat.len() != iter * iter { - panic!("build_case: flat len mismatch"); - } - let rows = model::output_rows(iter, ksize, stride, padding); - let windows = ((iter + 2 * padding - ksize) / stride + 1).pow(2); - if rows > model::bank_entries() { - panic!("build_case: output_rows={rows} exceeds bank depth"); - } - let dst_flat = model::im2col(flat, iter, ksize, stride, padding); - let src = model::pack_bank_words(flat); - let nsrc = model::num_words(src.len()); - let ndst = model::num_words(dst_flat.len()); - if nsrc > model::bank_entries() || ndst > model::bank_entries() || nsrc > MAX_WORDS || ndst > MAX_WORDS { - panic!("build_case: word count out of range src={nsrc} dst={ndst}"); - } - Im2colCase { - cmd: Im2colCmd { - bid, - iter: iter as u32, - ksize: ksize as u32, - stride: stride as u32, - padding: padding as u32, - window_count: windows as u32, - op1_bank, - wr_bank, - op1_col: 1, - wr_col: 1, - rob_id, - num_src_words: nsrc as u32, - num_dst_words: ndst as u32, - }, - src, - dst: dst_flat, - } -} - -fn directed_k3_pad0(bid: u32) -> Im2colCase { - let iter = 6usize; - let k = 3usize; - let flat: [i8; 36] = [ - -1, 2, 3, 4, -5, 6, 0, 8, 9, -10, 11, 12, 13, 0, -15, 16, 17, 18, 19, -20, 0, 22, 23, -24, - -25, 26, 27, 0, -29, 30, 31, 32, 33, -34, 0, 36, - ]; - let bytes: Vec = flat.iter().map(|&x| x as u8).collect(); - build_case(bid, iter, k, 1, 0, 0, 1, 3, &bytes) -} - -fn directed_k3_pad1(bid: u32) -> Im2colCase { - let iter = 6usize; - let k = 3usize; - let flat: [i8; 36] = [ - 7, -2, 3, 0, 5, -6, 1, 8, -9, 10, 11, 0, -4, 12, 13, 14, 0, 16, 17, 0, 19, -20, 21, 22, 0, - 24, 25, 26, -27, 28, 29, -30, 0, 32, 33, 34, - ]; - let bytes: Vec = flat.iter().map(|&x| x as u8).collect(); - build_case(bid, iter, k, 1, 1, 0, 1, 5, &bytes) -} - -fn shape_ok(iter: usize, ksize: usize, stride: usize, padding: usize) -> bool { - if iter == 0 || ksize == 0 || stride == 0 { - return false; - } - let padded = iter + 2 * padding; - if padded < ksize { - return false; - } - let rows = model::output_rows(iter, ksize, stride, padding); - rows <= model::bank_entries() -} - -fn random_case(seed: u32, index: u32, bid: u32) -> Im2colCase { - let mut rng = Rng::new(seed, index); - let iter_pool = [3usize, 4, 5, 6, 7]; - let ksize_pool = [1usize, 3, 5]; - let pad_pool = [0usize, 1]; - let base = (index - 2) as usize; - - for attempt in 0..10_000 { - let iter = iter_pool[(base + attempt) % iter_pool.len()]; - let ksize = ksize_pool[((base / iter_pool.len()) + attempt) % ksize_pool.len()]; - let stride = 1usize; - let padding = pad_pool[(base + attempt / 3) % pad_pool.len()]; - if !shape_ok(iter, ksize, stride, padding) { - continue; - } - let op1_bank = rng.range(0, 7); - let mut wr_bank = rng.range(0, 7); - if wr_bank == op1_bank { - wr_bank = (wr_bank + 1) & 7; - } - let mut flat = vec![0u8; iter * iter]; - for b in flat.iter_mut() { - *b = (rng.next() & 0xff) as u8; - } - return build_case( - bid, - iter, - ksize, - stride, - padding, - op1_bank, - wr_bank, - rng.range(0, 15), - &flat, - ); - } - panic!("random_case: failed to find legal shape for index={index}"); -} - -struct Rng { - state: u64, -} - -impl Rng { - fn new(seed: u32, index: u32) -> Self { - let state = (u64::from(seed) << 32) ^ u64::from(index) ^ 0x9E37_79B9_7F4A_7C15; - Self { state } - } - fn next(&mut self) -> u32 { - self.state ^= self.state >> 12; - self.state ^= self.state << 25; - self.state ^= self.state >> 27; - ((self.state.wrapping_mul(0x2545_F491_4F6C_DD1D)) >> 32) as u32 - } - fn range(&mut self, lo: u32, hi: u32) -> u32 { - lo + (self.next() % (hi - lo + 1)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn directed_k3_pad0_matches_ctest() { - let case = gen_case(0, 0, 2); - assert_eq!(case.cmd.iter, 6); - assert_eq!(case.cmd.ksize, 3); - assert_eq!(case.cmd.stride, 1); - assert_eq!(case.cmd.padding, 0); - assert_eq!(case.cmd.op1_bank, 0); - assert_eq!(case.cmd.wr_bank, 1); - assert_eq!(case.cmd.op1_col, 1); - assert_eq!(case.cmd.wr_col, 1); - assert_eq!(case.cmd.num_src_words, 36); - assert_eq!(case.cmd.num_dst_words, 16); - assert_eq!(case.src.len(), 36 * 16); - assert_eq!(case.dst.len(), 16 * 16); - assert_eq!(case.src[0], (-1i8) as u8); - assert_eq!(case.src[35 * 16], 36); - } - - #[test] - fn directed_k3_pad1_shape() { - let case = gen_case(0, 1, 2); - assert_eq!(case.cmd.iter, 6); - assert_eq!(case.cmd.ksize, 3); - assert_eq!(case.cmd.padding, 1); - assert_eq!(case.cmd.num_dst_words, 48); - } - - #[test] - fn random_deterministic_and_legal() { - let a = gen_case(0xCAFE_BABE, 2, 2); - let b = gen_case(0xCAFE_BABE, 2, 2); - assert_eq!(a, b); - assert!([3u32, 4, 5, 6, 7].contains(&a.cmd.iter)); - assert!([1u32, 3, 5].contains(&a.cmd.ksize)); - assert_eq!(a.cmd.stride, 1); - assert!(a.cmd.padding == 0 || a.cmd.padding == 1); - assert_ne!(a.cmd.op1_bank, a.cmd.wr_bank); - assert!(a.cmd.num_dst_words <= model::bank_entries() as u32); - } - - #[test] - fn coverage_bins_hit_across_suite() { - let mut saw_iter = [false; 8]; - let mut saw_k = [false; 6]; - let mut saw_pad = [false; 2]; - for i in 0..=21 { - let c = gen_case(0xCAFE_BABE, i, 2); - saw_iter[c.cmd.iter as usize] = true; - saw_k[c.cmd.ksize as usize] = true; - saw_pad[c.cmd.padding as usize] = true; - assert_eq!(c.cmd.stride, 1); - } - for it in [3usize, 4, 5, 6, 7] { - assert!(saw_iter[it], "missing iter bin {it}"); - } - for k in [1usize, 3, 5] { - assert!(saw_k[k], "missing ksize bin {k}"); - } - assert!(saw_pad[0] && saw_pad[1], "missing pad bins"); - } - - #[test] - fn bid_required_arg() { - assert_eq!(gen_case(0, 0, 2).cmd.bid, 2); - assert_eq!(gen_case(0, 0, 3).cmd.bid, 3); - } - - #[test] - fn word_lo_hi_round_trip() { - let case = gen_case(0, 0, 2); - for i in 0..case.cmd.num_src_words as usize { - let lo = case.src_word_lo(i); - let hi = case.src_word_hi(i); - let off = i * BANK_ROW_BYTES; - let expect_lo = u64::from_le_bytes(case.src[off..off + 8].try_into().unwrap()); - let expect_hi = u64::from_le_bytes(case.src[off + 8..off + 16].try_into().unwrap()); - assert_eq!(lo, expect_lo); - assert_eq!(hi, expect_hi); - } - } - - #[test] - #[should_panic] - fn word_lo_out_of_range_panics() { - let case = gen_case(0, 0, 2); - let _ = case.src_word_lo(MAX_WORDS); - } -} diff --git a/examples/balls/im2col/verify/casegen/src/lib.rs b/examples/balls/im2col/verify/casegen/src/lib.rs deleted file mode 100644 index 492750020..000000000 --- a/examples/balls/im2col/verify/casegen/src/lib.rs +++ /dev/null @@ -1,112 +0,0 @@ -mod casegen; -mod model; - -use std::cell::RefCell; - -use casegen::{Im2colCase, Im2colCmd, MAX_WORDS}; - -thread_local! { - static CURRENT: RefCell> = RefCell::new(None); -} - -#[no_mangle] -pub extern "C" fn im2col_case_load(seed: u32, index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(seed, index, bid); - let nsrc = case.cmd.num_src_words as usize; - let ndst = case.cmd.num_dst_words as usize; - if nsrc > MAX_WORDS || ndst > MAX_WORDS { - panic!("im2col_case_load: word count out of range src={nsrc} dst={ndst}"); - } - CURRENT.with(|c| *c.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|c| match c.borrow().as_ref() { - Some(case) => f(case), - None => panic!("im2col_case: no case loaded; call im2col_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn im2col_case_cmd(out_ptr: *mut Im2colCmd) { - current_case(|case| unsafe { - if out_ptr.is_null() { - panic!("im2col_case_cmd: null out_ptr"); - } - *out_ptr = case.cmd; - }); -} - -#[no_mangle] -pub extern "C" fn im2col_case_src_word_lo(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("im2col_case_src_word_lo: word_index out of range"); - } - current_case(|case| case.src_word_lo(word_index as usize)) -} - -#[no_mangle] -pub extern "C" fn im2col_case_src_word_hi(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("im2col_case_src_word_hi: word_index out of range"); - } - current_case(|case| case.src_word_hi(word_index as usize)) -} - -#[no_mangle] -pub extern "C" fn im2col_case_dst_word_lo(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("im2col_case_dst_word_lo: word_index out of range"); - } - current_case(|case| case.dst_word_lo(word_index as usize)) -} - -#[no_mangle] -pub extern "C" fn im2col_case_dst_word_hi(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("im2col_case_dst_word_hi: word_index out of range"); - } - current_case(|case| case.dst_word_hi(word_index as usize)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dpi_load_then_cmd_and_words() { - assert_eq!(im2col_case_load(0x1234, 0, 2), 0); - let mut cmd = Im2colCmd { - bid: 0, - iter: 0, - ksize: 0, - stride: 0, - padding: 0, - window_count: 0, - op1_bank: 0, - wr_bank: 0, - op1_col: 0, - wr_col: 0, - rob_id: 0, - num_src_words: 0, - num_dst_words: 0, - }; - im2col_case_cmd(&mut cmd as *mut Im2colCmd); - assert_eq!(cmd.iter, 6); - assert_eq!(cmd.ksize, 3); - assert_eq!(cmd.num_src_words, 36); - assert_eq!(cmd.num_dst_words, 16); - let _lo = im2col_case_src_word_lo(0); - let _hi = im2col_case_src_word_hi(0); - let _dlo = im2col_case_dst_word_lo(0); - let _dhi = im2col_case_dst_word_hi(0); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/im2col/verify/casegen/src/model.rs b/examples/balls/im2col/verify/casegen/src/model.rs deleted file mode 100644 index 35955b18b..000000000 --- a/examples/balls/im2col/verify/casegen/src/model.rs +++ /dev/null @@ -1,103 +0,0 @@ -pub const TILE: usize = 16; -pub const BANK_ROW_BYTES: usize = TILE; - -pub fn bank_entries() -> usize { - let config = match std::env::var("BB_VERIFY_CONFIG") { - Ok(path) => path, - Err(_) if cfg!(test) => return 64, - Err(_) => panic!("BB_VERIFY_CONFIG is required"), - }; - let text = std::fs::read_to_string(config).expect("failed to read BB_VERIFY_CONFIG"); - text.lines() - .find_map(|line| line.strip_prefix("bank_entries=")) - .unwrap_or_else(|| panic!("bank_entries missing from BB_VERIFY_CONFIG")) - .parse() - .expect("invalid bank_entries in BB_VERIFY_CONFIG") -} - -pub fn output_rows(iter: usize, ksize: usize, stride: usize, padding: usize) -> usize { - if iter == 0 || ksize == 0 || stride == 0 { - panic!("im2col model: iter/ksize/stride must be >= 1"); - } - let padded = iter + 2 * padding; - if padded < ksize { - panic!("im2col model: kernel larger than padded input"); - } - let out_dim = (padded - ksize) / stride + 1; - let windows = out_dim * out_dim; - let kernel = ksize * ksize; - let m_tiles = windows.div_ceil(TILE); - let k_tiles = kernel.div_ceil(TILE); - m_tiles * k_tiles * TILE -} - -pub fn im2col(src: &[u8], iter: usize, ksize: usize, stride: usize, padding: usize) -> Vec { - if src.len() != iter * iter { - panic!( - "im2col model: src len {} != iter*iter {}", - src.len(), - iter * iter - ); - } - let padded = iter + 2 * padding; - if padded < ksize { - panic!("im2col model: kernel larger than padded input"); - } - if stride == 0 { - panic!("im2col model: stride must be >= 1"); - } - let out_dim = (padded - ksize) / stride + 1; - let windows = out_dim * out_dim; - let kernel_elems = ksize * ksize; - let m_tiles = windows.div_ceil(TILE); - let k_tiles = kernel_elems.div_ceil(TILE); - let rows = m_tiles * k_tiles * TILE; - let mut output = vec![0u8; rows * TILE]; - - let mut window = 0usize; - for orow in 0..out_dim { - for ocol in 0..out_dim { - for kr in 0..ksize { - for kc in 0..ksize { - let kernel = kr * ksize + kc; - let m_tile = window / TILE; - let m_row = window % TILE; - let k_tile = kernel / TILE; - let lane = kernel % TILE; - let bank_row = (m_tile * k_tiles + k_tile) * TILE + m_row; - let out = bank_row * TILE + lane; - if out >= output.len() { - panic!("im2col model: output range out={out}"); - } - let input_row = (orow * stride + kr) as isize - padding as isize; - let input_col = (ocol * stride + kc) as isize - padding as isize; - if input_row >= 0 - && input_row < iter as isize - && input_col >= 0 - && input_col < iter as isize - { - let src_i = input_row as usize * iter + input_col as usize; - output[out] = src[src_i]; - } - } - } - window += 1; - } - } - output -} - -pub fn pack_bank_words(flat: &[u8]) -> Vec { - let mut out = vec![0u8; flat.len() * BANK_ROW_BYTES]; - for (index, value) in flat.iter().enumerate() { - out[index * BANK_ROW_BYTES] = *value; - } - out -} - -pub fn num_words(total_bytes: usize) -> usize { - if total_bytes % BANK_ROW_BYTES != 0 { - panic!("num_words: total_bytes {total_bytes} not a multiple of bank row"); - } - total_bytes / BANK_ROW_BYTES -} diff --git a/examples/balls/im2col/verify/src/tb_top.sv b/examples/balls/im2col/verify/src/tb_top.sv deleted file mode 100644 index fd043ab42..000000000 --- a/examples/balls/im2col/verify/src/tb_top.sv +++ /dev/null @@ -1,91 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import im2col_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - Im2colBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank[3:0]), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank[3:0]), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank[3:0]), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank[3:0]), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0][3:0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_group_id(intf.bank_read_group_id[0][3:0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0][3:0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0][3:0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_bankRead_0_ball_id(), - .io_bankWrite_0_ball_id() - ); - - initial begin - intf.clock = 1'b0; - intf.reset = 1'b1; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("im2col_ball_test"); - end -endmodule diff --git a/examples/balls/im2col/workloads/ctests/CMakeLists.txt b/examples/balls/im2col/workloads/ctests/CMakeLists.txt deleted file mode 100644 index eca3b8d08..000000000 --- a/examples/balls/im2col/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -add_buckyball_ctests( - im2col_window_test.c - im2col_k1_test.c - im2col_k3_test.c - im2col_k3_pad_test.c - im2col_k3_asym_pad_test.c - im2col_k3_partial_test.c - im2col_k5_test.c - im2col_k7_test.c -) diff --git a/examples/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir b/examples/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir deleted file mode 100644 index 31c376a82..000000000 --- a/examples/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir +++ /dev/null @@ -1,59 +0,0 @@ -// Already-lowered ball Op: buckyball.im2col (physical-bank form). -// Same 6x6 k3 shape / golden as bank_im2col. - -func.func private @check_result(memref<16x16xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c5 = arith.constant 5 : index - %c6 = arith.constant 6 : index - %c7 = arith.constant 7 : index - - %input = memref.alloc() alignment = 64 : memref<64x16xi8> - %output = memref.alloc() alignment = 64 : memref<16x16xi8> - linalg.fill ins(%zero_i8 : i8) outs(%input : memref<64x16xi8>) - linalg.fill ins(%zero_i8 : i8) outs(%output : memref<16x16xi8>) - - scf.for %r = %c0 to %c6 step %c1 { - scf.for %col = %c0 to %c6 step %c1 { - %tmp = arith.muli %r, %c6 : index - %idx = arith.addi %tmp, %col : index - %bank_row = arith.addi %c5, %idx : index - %ri = arith.index_cast %r : index to i32 - %ci = arith.index_cast %col : index to i32 - %v32 = arith.addi %ri, %ci : i32 - %v8 = arith.trunci %v32 : i32 to i8 - memref.store %v8, %input[%bank_row, %c7] : memref<64x16xi8> - } - } - - %iter = arith.constant 6 : i64 - %ksize = arith.constant 3 : i64 - %stride = arith.constant 1 : i64 - %padding = arith.constant 0 : i64 - %din = arith.constant 64 : i64 - %dout = arith.constant 16 : i64 - %s = arith.constant 1 : i64 - - %in = buckyball.bank_alloc - %out = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %in %din %s - : memref<64x16xi8> i64 i64 i64 - %inputBase = arith.constant 5 : i64 - %lane = arith.constant 7 : i64 - buckyball.im2col %loaded, %out, %iter, %ksize, %stride, %padding, - %inputBase, %lane - : i64 - %stored = buckyball.bank_mvout %output %out %dout %s - : memref<16x16xi8> i64 i64 i64 - buckyball.bank_release %loaded : i64 - buckyball.bank_release %stored : i64 - - func.call @check_result(%output) : (memref<16x16xi8>) -> () - memref.dealloc %input : memref<64x16xi8> - memref.dealloc %output : memref<16x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows.mlir.in b/examples/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows.mlir.in deleted file mode 100644 index 7db8852e9..000000000 --- a/examples/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows.mlir.in +++ /dev/null @@ -1,41 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %side = arith.constant 8 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %windows = arith.constant 64 : index - %lane = arith.constant 7 : index - %input_depth = arith.constant 64 : i64 - %output_depth = arith.constant 64 : i64 - %side64 = arith.constant 8 : i64 - %one64 = arith.constant 1 : i64 - %three64 = arith.constant 3 : i64 - %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - linalg.fill ins(%zero_i8 : i8) outs(%input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) - scf.for %index = %zero to %rows step %one { - %value32 = arith.index_cast %index : index to i32 - %value = arith.trunci %value32 : i32 to i8 - memref.store %value, %input[%index, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - } - %input_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %zero64 = arith.constant 0 : i64 - %lane64 = arith.constant 7 : i64 - %final:2 = scf.for %base = %zero to %rows step %windows iter_args(%ib = %input_bank, %ob = %output_bank) -> (i64, i64) { - %in = memref.subview %input[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> - %out = memref.subview %output[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> - %loaded = buckyball.bank_mvin %in %ib %input_depth %one64 : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> i64 i64 i64 - %result = buckyball.bank_im2col %loaded %ob %side64 %three64 %one64 %one64 %zero64 %lane64 : i64 i64 i64 i64 i64 i64 i64 i64 - %stored = buckyball.bank_mvout %out %result %output_depth %one64 : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> i64 i64 i64 - scf.yield %loaded, %stored : i64, i64 - } - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %final#0 : i64 - buckyball.bank_release %final#1 : i64 - memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/int2fp/arch/src/main/scala/Int2Fp.scala b/examples/balls/int2fp/arch/src/main/scala/Int2Fp.scala deleted file mode 100644 index 8d4b54bd7..000000000 --- a/examples/balls/int2fp/arch/src/main/scala/Int2Fp.scala +++ /dev/null @@ -1,205 +0,0 @@ -package examples.balls.int2fp - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.top.GlobalConfig - -@instantiable -class Int2Fp(val b: GlobalConfig) extends Module { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "Int2FpBall") - .getOrElse(throw new IllegalArgumentException("Int2FpBall not found in config")) - - private val funct = b.ballDomain.ballISA - .find(_.mnemonic == "INT32_TO_FP32") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("INT32_TO_FP32 not found in ballISA")) - - require(b.memDomain.bankWidth == 128, "Int2FpBall requires 128-bit bank rows") - require(mapping.inBW == 2, "Int2FpBall requires inBW=2") - require(mapping.outBW == 1, "Int2FpBall requires outBW=1") - require((funct >> 4) == 4, "INT32_TO_FP32 must encode two reads and one write") - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(mapping.inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(mapping.outBW, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - private val idle :: readReq :: readResp :: convert :: multiply :: writeReq :: writeResp :: complete :: Nil = - Enum(8) - private val state = RegInit(idle) - - private val robIdReg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSubReg = RegInit(false.B) - private val subRobIdReg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val inputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val scaleBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val outputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val iterReg = RegInit(0.U(b.frontend.iter_len.W)) - private val rowReg = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val reluReg = RegInit(false.B) - private val inputData = Reg(UInt(128.W)) - private val scaleData = Reg(UInt(128.W)) - private val fpOutputs = Reg(Vec(4, UInt(32.W))) - private val mulStarted = RegInit(false.B) - private val laneReg = RegInit(0.U(2.W)) - private val writeData = Reg(UInt(128.W)) - private val multiplier = Module(new Fp32Mul) - - private def int32ToFp32(value: UInt): UInt = { - val sign = value(31) - val abs = Mux(sign, (~value).asUInt + 1.U, value) - val zero = abs === 0.U - val leading = 31.U - PriorityEncoder(Reverse(abs)) - val shift = Mux(leading > 23.U, leading - 23.U, 0.U) - val truncated = abs >> shift - val half = Mux(shift === 0.U, 0.U(32.W), 1.U(32.W) << (shift - 1.U)) - val remainder = Mux(shift === 0.U, 0.U(32.W), abs & ((1.U(32.W) << shift) - 1.U)) - val roundUp = remainder > half || (remainder === half && truncated(0)) - val rounded = truncated +& roundUp.asUInt - val carry = rounded(24) - val significand = Mux(leading > 23.U, rounded(22, 0), (abs << (23.U - leading))(22, 0)) - val exponent = (leading +& 127.U +& carry)(7, 0) - Mux(zero, 0.U, Cat(sign, exponent, Mux(carry, 0.U(23.W), significand))) - } - - private def positiveFinite(value: UInt): Bool = - !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U - - for (i <- 0 until mapping.inBW) { - io.bankRead(i).rob_id := robIdReg - io.bankRead(i).ball_id := 0.U - io.bankRead(i).bank_id := Mux(i.U === 0.U, inputBank, scaleBank) - io.bankRead(i).group_id := 0.U - io.bankRead(i).io.req.valid := false.B - io.bankRead(i).io.req.bits.addr := Mux(i.U === 0.U, rowReg, rowReg(1, 0)) - io.bankRead(i).io.resp.ready := false.B - } - io.bankWrite(0).rob_id := robIdReg - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := outputBank - io.bankWrite(0).group_id := 0.U - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := rowReg - io.bankWrite(0).io.req.bits.data := writeData - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - - private val inputValue = (inputData >> (laneReg << 5))(31, 0) - private val scaleValue = (scaleData >> (laneReg << 5))(31, 0) - multiplier.io.start := false.B - multiplier.io.a := int32ToFp32(Mux(reluReg && inputValue(31), 0.U, inputValue)) - multiplier.io.b := scaleValue - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robIdReg - io.cmdResp.bits.is_sub := isSubReg - io.cmdResp.bits.sub_rob_id := subRobIdReg - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - assert(cmd.funct7 === funct.U, "Int2FpBall received an unknown funct7") - assert(cmd.iter > 0.U && cmd.iter(1, 0) === 0.U, "INT32_TO_FP32 iter must be a positive multiple of four") - assert(cmd.iter <= b.memDomain.bankEntries.U, "INT32_TO_FP32 input exceeds bank depth") - assert(cmd.op1_bank < b.memDomain.virtualBankCount.U, "INT32_TO_FP32 input bank is invalid") - assert(cmd.op2_bank < b.memDomain.virtualBankCount.U, "INT32_TO_FP32 scale bank is invalid") - assert(cmd.wr_bank < b.memDomain.virtualBankCount.U, "INT32_TO_FP32 output bank is invalid") - assert( - cmd.op1_col === 1.U && cmd.op2_col === 1.U && cmd.wr_col === 1.U, - "INT32_TO_FP32 operands must each occupy one bank" - ) - assert( - cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, - "INT32_TO_FP32 banks must be distinct" - ) - assert(cmd.rs2(63, 1) === 0.U, "INT32_TO_FP32 reserves rs2[63:1]") - - robIdReg := io.cmdReq.bits.rob_id - isSubReg := io.cmdReq.bits.is_sub - subRobIdReg := io.cmdReq.bits.sub_rob_id - inputBank := cmd.op1_bank - scaleBank := cmd.op2_bank - outputBank := cmd.wr_bank - iterReg := cmd.iter - rowReg := 0.U - reluReg := cmd.rs2(0) - state := readReq - } - } - is(readReq) { - for (i <- 0 until mapping.inBW) io.bankRead(i).io.req.valid := true.B - when(io.bankRead.map(_.io.req.fire).reduce(_ && _)) { - state := readResp - } - } - is(readResp) { - for (i <- 0 until mapping.inBW) io.bankRead(i).io.resp.ready := true.B - when(io.bankRead.map(_.io.resp.fire).reduce(_ && _)) { - inputData := io.bankRead(0).io.resp.bits.data - scaleData := io.bankRead(1).io.resp.bits.data - state := convert - } - } - is(convert) { - for (i <- 0 until 4) { - val scale = scaleData(32 * i + 31, 32 * i) - assert(positiveFinite(scale), "INT32_TO_FP32 scales must be finite and positive") - } - mulStarted := false.B - laneReg := 0.U - state := multiply - } - is(multiply) { - when(!mulStarted) { - multiplier.io.start := true.B - mulStarted := true.B - } - when(multiplier.io.done) { - fpOutputs(laneReg) := multiplier.io.result - when(laneReg === 3.U) { - writeData := Cat(multiplier.io.result, fpOutputs(2), fpOutputs(1), fpOutputs(0)) - state := writeReq - }.otherwise { - laneReg := laneReg + 1.U - mulStarted := false.B - } - } - } - is(writeReq) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire) { - state := writeResp - } - } - is(writeResp) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - when(rowReg === iterReg - 1.U) { - state := complete - }.otherwise { - rowReg := rowReg + 1.U - state := readReq - } - } - } - is(complete) { - when(io.cmdResp.fire) { - state := idle - } - } - } -} diff --git a/examples/balls/int2fp/arch/src/main/scala/Int2FpBall.scala b/examples/balls/int2fp/arch/src/main/scala/Int2FpBall.scala deleted file mode 100644 index cb4d34922..000000000 --- a/examples/balls/int2fp/arch/src/main/scala/Int2FpBall.scala +++ /dev/null @@ -1,34 +0,0 @@ -package examples.balls.int2fp - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} - -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.top.GlobalConfig - -@instantiable -class Int2FpBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "Int2FpBall") - .getOrElse(throw new IllegalArgumentException("Int2FpBall not found in config")) - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - private val unit: Instance[Int2Fp] = Instantiate(new Int2Fp(b)) - unit.io.cmdReq <> io.cmdReq - unit.io.cmdResp <> io.cmdResp - for (i <- 0 until mapping.inBW) unit.io.bankRead(i) <> io.bankRead(i) - for (i <- 0 until mapping.outBW) unit.io.bankWrite(i) <> io.bankWrite(i) - io.status <> unit.io.status - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) -} diff --git a/examples/balls/int2fp/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/int2fp/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index bdd086410..000000000 --- a/examples/balls/int2fp/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- AssignPhysicalBankPatterns.cpp - Int2Fp bank assignment patterns ---===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateInt2FpBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { - -class BankInt32ToFp32Pattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankInt32ToFp32Op op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getInBank(), - op.getScaleBank(), op.getOutBank(), - op.getIter(), op.getReluAttr()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateInt2FpBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 98a4ca959..000000000 --- a/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct Int32ToFp32Lowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(Int32ToFp32Op op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("Int2FpBall"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - adaptor.getScaleBankId(), - adaptor.getOutputBankId(), adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, cstI64(rewriter, loc, op.getRelu() ? 1 : 0), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("INT32_TO_FP32"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateInt2FpBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t) { - (void)stable; - patterns.add(converter); -} - -void configureInt2FpBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/int2fp/configs/Int2FpBallParam.scala b/examples/balls/int2fp/configs/Int2FpBallParam.scala deleted file mode 100644 index 2b61dd2d1..000000000 --- a/examples/balls/int2fp/configs/Int2FpBallParam.scala +++ /dev/null @@ -1,21 +0,0 @@ -package examples.balls.int2fp.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig - -case class Int2FpBallParam( - inputType: String, - outputType: String) - -object Int2FpBallParam { - private val ballName = "Int2FpBall" - - def apply(b: GlobalConfig): Int2FpBallParam = { - val tbl = BallParamLoader.ballTable(b, ballName) - Int2FpBallParam( - inputType = BallParamLoader.str(tbl, "inputType"), - outputType = BallParamLoader.str(tbl, "outputType") - ) - } - -} diff --git a/examples/balls/int2fp/configs/default.toml b/examples/balls/int2fp/configs/default.toml deleted file mode 100644 index cd0c70328..000000000 --- a/examples/balls/int2fp/configs/default.toml +++ /dev/null @@ -1,5 +0,0 @@ -# Int2Fp Ball Configuration - -[ball] -inputType = "INT32" -outputType = "FP32" diff --git a/examples/balls/int2fp/emu/src/dispatch.rs b/examples/balls/int2fp/emu/src/dispatch.rs deleted file mode 100644 index 8d0b6a128..000000000 --- a/examples/balls/int2fp/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::ExecContext; - -use super::int2fp; - -const BALL_CLASS: &str = "examples.balls.int2fp.Int2FpBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("INT32_TO_FP32") => Some(int2fp::execute(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("INT32_TO_FP32") => Some(int2fp::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/int2fp/emu/src/lib.rs b/examples/balls/int2fp/emu/src/lib.rs deleted file mode 100644 index 3f8080cde..000000000 --- a/examples/balls/int2fp/emu/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod int2fp; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/int2fp/emu/src/model.rs b/examples/balls/int2fp/emu/src/model.rs deleted file mode 100644 index a58c6d01a..000000000 --- a/examples/balls/int2fp/emu/src/model.rs +++ /dev/null @@ -1,117 +0,0 @@ -pub fn int2fp_fp32_bits(value: i32, dw_bits: u32) -> u32 { - fp32_multiply(int32_to_fp32(value), dw_bits) -} - -pub fn int2fp_dequant_bits(value: i32, da_bits: u32, dw_bits: u32) -> u32 { - int2fp_fp32_bits(value, fp32_multiply(da_bits, dw_bits)) -} - -pub fn int32_to_fp32(value: i32) -> u32 { - if value == 0 { - return 0; - } - - let sign = ((value as u32) >> 31) & 1; - let abs = if value == i32::MIN { - 0x8000_0000u32 - } else { - value.unsigned_abs() - }; - - let leading_one = 31u32 - abs.leading_zeros(); - let mut exponent = leading_one + 127; - let significand = if leading_one > 23 { - let right_shift = leading_one - 23; - let abs_wide = u64::from(abs); - let truncated = abs_wide >> right_shift; - let half = 1u64 << (right_shift - 1); - let remainder = abs_wide & ((1u64 << right_shift) - 1); - let round_up = remainder > half || (remainder == half && (truncated & 1) != 0); - let rounded = truncated + u64::from(round_up); - if ((rounded >> 24) & 1) != 0 { - exponent = leading_one + 128; - (rounded >> 1) as u32 - } else { - rounded as u32 - } - } else { - abs << (23 - leading_one) - }; - - (sign << 31) | ((exponent & 0xff) << 23) | (significand & 0x7f_ffff) -} - -fn fp32_multiply(a: u32, b: u32) -> u32 { - let a_sign = (a >> 31) & 1; - let b_sign = (b >> 31) & 1; - let a_exp = (a >> 23) & 0xff; - let b_exp = (b >> 23) & 0xff; - let a_frac = a & 0x7f_ffff; - let b_frac = b & 0x7f_ffff; - let a_mant = (1u64 << 23) | u64::from(a_frac); - let b_mant = (1u64 << 23) | u64::from(b_frac); - let a_zero = a_exp == 0 && a_frac == 0; - let b_zero = b_exp == 0 && b_frac == 0; - let prod = a_mant * b_mant; - let (sig, guard, round, sticky, exp_adjust) = if ((prod >> 47) & 1) != 0 { - ( - (prod >> 24) as u32, - ((prod >> 23) & 1) != 0, - ((prod >> 22) & 1) != 0, - (prod & ((1u64 << 22) - 1)) != 0, - 1u32, - ) - } else { - ( - (prod >> 23) as u32, - ((prod >> 22) & 1) != 0, - ((prod >> 21) & 1) != 0, - (prod & ((1u64 << 21) - 1)) != 0, - 0u32, - ) - }; - let increment = guard && (round || sticky || (sig & 1) != 0); - let rounded = u64::from(sig) + u64::from(increment); - let carry = ((rounded >> 24) & 1) as u32; - let final_sig = if carry != 0 { rounded >> 1 } else { rounded } as u32; - let exp_wide = (a_exp + b_exp + exp_adjust + carry).wrapping_sub(127) & 0x3ff; - - if a_zero || b_zero { - 0 - } else if (exp_wide & 0x200) != 0 { - 0 - } else if (exp_wide & 0x100) != 0 { - ((a_sign ^ b_sign) << 31) | (0xff << 23) - } else { - ((a_sign ^ b_sign) << 31) | ((exp_wide & 0xff) << 23) | (final_sig & 0x7f_ffff) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn int_to_fp_basic() { - let scale = 0x3F80_0000; - assert_eq!(int2fp_fp32_bits(1, scale), 0x3F80_0000); - assert_eq!(int2fp_fp32_bits(-1, scale), 0xBF80_0000); - assert_eq!(int2fp_fp32_bits(0, scale), 0); - assert_eq!(int2fp_fp32_bits(i32::MIN, scale), 0xCF00_0000); - } - - #[test] - fn int8_to_fp_scale() { - let scale = 0x3E80_0000; // 0.25 - assert_eq!(int2fp_fp32_bits(-128, scale), (-32.0f32).to_bits()); - assert_eq!(int2fp_fp32_bits(127, scale), (31.75f32).to_bits()); - } - - #[test] - fn dequant_uses_both_scales() { - assert_eq!( - int2fp_dequant_bits(8, 0x3f00_0000, 0x3e80_0000), - 1.0f32.to_bits() - ); - } -} diff --git a/examples/balls/int2fp/verify/README.md b/examples/balls/int2fp/verify/README.md deleted file mode 100644 index 8e3ec940c..000000000 --- a/examples/balls/int2fp/verify/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# Int2FpBall UVM Verification - -Verifies the generated `Int2FpBall` module using the shared Blink UVM framework. - -RTL contract: one mnemonic `INT32_TO_FP32`, `inBW=2` (input + scale banks), -`outBW=1`, `rs2[0]=relu`, `rs2[63:1]=0`, iter a positive multiple of 4. -Scales are bank rows, not MMIO. - -## Structure - -- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM agents and base env -- `src/common/int2fp_defs.svh`: DPI imports, `INT32_TO_FP32_FUNCT7`, timeouts -- `src/common/int2fp_items.svh`: `int2fp_cmd_item` with `load_rust_case` -- `src/seq/int2fp_sequences.svh`: one-case sequence -- `src/cov/int2fp_cov.svh`: relu `{0,1}` and iter `{4,8,16}`, 100% or fatal -- `src/env/int2fp_scoreboard.svh`: preload src/scale banks, compare writes vs DPI dst -- `src/env/int2fp_env.svh`: extends `bb_blink_env #(BB_IN_BW, BB_OUT_BW)` -- `src/tests/int2fp_test.svh`: `int2fp_ball_test` loops cases 0..5 -- `src/pkg/int2fp_pkg.sv`: package entry -- `src/tb_top.sv`: `bb_blink_if #(BB_IN_BW, BB_OUT_BW)` + `Int2FpBall` (2 read, 1 write) -- `filelists/int2fp_ball.f`: VCS filelist (`@UVM@` / `@RTL@`) - -## Test plan - -`+UVM_TESTNAME=int2fp_ball_test` runs six directed cases: - -| Index | iter | relu | Intent | -|-------|------|------|--------| -| 0 | 4 | 0 | MLIR-scale values | -| 1 | 4 | 1 | relu clamps negatives | -| 2 | 8 | 0 | eight-row, no relu | -| 3 | 8 | 1 | eight-row, relu | -| 4 | 16 | 0 | sixteen-row, no relu | -| 5 | 16 | 1 | sixteen-row, relu | - -Each case: reset, preload 16-byte i32 input rows and f32 scale rows into -`mem_model`, drive one `INT32_TO_FP32` command, wait for `scb.done()` or -timeout at 20000 cycles. Expected dst = `f32(i32) * scale` with relu clamp -on the integer before convert. `int2fp_cov` fatals unless relu and iter bins -are 100%. - -## BID (required) - -`+BID=` is mandatory. Missing it fatals at test start. No default bid. - -| Config | Plusarg | -|--------|---------| -| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=4` | - -```console -./simv +UVM_TESTNAME=int2fp_ball_test +BID=4 -``` - -## Build / Run - -Enter the shared UVM/VCS environment and build the DPI reference library: - -```console -nix develop ../../../../verify -cargo build --manifest-path casegen/Cargo.toml -``` - -Human: - -```console -bbdev uvm --run '--chip pebble' -bbdev uvm --run '--chip pebble --ball int2fp' -``` - -Agent (MCP): - -```text -bbdev_uvm_run(chip="pebble", ball="int2fp") -``` - -bbdev injects `INT32_TO_FP32_FUNCT7`, `BB_IN_BW`, `BB_OUT_BW`, `+BID`, and -`+UVM_TESTNAME=int2fp_ball_test` from pebble chip.pb. diff --git a/examples/balls/int2fp/verify/casegen/src/casegen.rs b/examples/balls/int2fp/verify/casegen/src/casegen.rs deleted file mode 100644 index be3725102..000000000 --- a/examples/balls/int2fp/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,346 +0,0 @@ -use crate::model::int2fp_fp32_bits; - -pub const BANK_NUM: u32 = 24; -pub const BANK_ENTRIES: u32 = 64; -pub const LANES: usize = 4; -pub const SCALE_ROWS: usize = 4; -pub const MAX_SRC: usize = 16; -pub const MAX_SCALE: usize = 4; -pub const MAX_DST: usize = 16; -pub const NUM_CASES: u32 = 6; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Int2FpCmd { - pub bid: u32, - pub iter: u32, - pub relu: u32, - pub op1_bank: u32, - pub op2_bank: u32, - pub wr_bank: u32, - pub op1_col: u32, - pub op2_col: u32, - pub wr_col: u32, - pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, - pub rs2_lo: u32, - pub rs2_hi: u32, - pub num_src_words: u32, - pub num_scale_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Int2FpCase { - pub cmd: Int2FpCmd, - pub src_words: [u128; MAX_SRC], - pub scale_words: [u128; MAX_SCALE], - pub dst_words: [u128; MAX_DST], -} - -impl Int2FpCase { - pub fn src_lo(&self, index: usize) -> u64 { - self.src_words[index] as u64 - } - - pub fn src_hi(&self, index: usize) -> u64 { - (self.src_words[index] >> 64) as u64 - } - - pub fn scale_lo(&self, index: usize) -> u64 { - self.scale_words[index] as u64 - } - - pub fn scale_hi(&self, index: usize) -> u64 { - (self.scale_words[index] >> 64) as u64 - } - - pub fn dst_lo(&self, index: usize) -> u64 { - self.dst_words[index] as u64 - } - - pub fn dst_hi(&self, index: usize) -> u64 { - (self.dst_words[index] >> 64) as u64 - } -} - -pub fn gen_case(index: u32, bid: u32) -> Int2FpCase { - if index >= NUM_CASES { - panic!("int2fp: unsupported directed case {index}"); - } - match index { - 0 => directed(bid, 2, 4, false, &MLIR_INPUT, &[0.125; 16]), - 1 => directed(bid, 3, 4, true, &RELU_INPUT, &[1.0; 16]), - 2 => directed(bid, 4, 8, false, &ITER8, &SCALE16), - 3 => directed(bid, 5, 8, true, &ITER8, &SCALE16), - 4 => directed(bid, 6, 16, false, &ITER16, &SCALE16), - 5 => directed(bid, 7, 16, true, &ITER16, &SCALE16), - _ => panic!("int2fp: unsupported directed case {index}"), - } -} - -const MLIR_INPUT: [i32; 16] = [1, 2, 3, -1, -2, 0, 4, 5, 10, -10, 7, 100, -100, 8, 16, -8]; -const RELU_INPUT: [i32; 16] = [-8, -1, 0, 1, 2, -3, 4, -5, 6, 7, -8, 9, -10, 11, 12, -13]; -const ITER8: [i32; 32] = [ - 8, -4, 16, 0, 1, -2, 3, -8, 10, -10, 7, 100, -100, 8, 16, -8, 4, -1, 0, 2, -16, 32, -32, 5, 9, - -9, 12, -12, 6, -6, 11, -11, -]; -const ITER16: [i32; 64] = [ - 1, 2, 3, -1, -2, 0, 4, 5, 10, -10, 7, 100, -100, 8, 16, -8, 8, -4, 16, 0, 1, -2, 3, -8, 12, -12, - 6, -6, 9, -9, 11, -11, 20, -20, 24, -24, 15, -15, 18, -18, 30, -30, 40, -40, 25, -25, 28, -28, - 50, -50, 60, -60, 7, -7, 13, -13, 14, -14, 17, -17, 19, -19, 21, -21, -]; -const SCALE16: [f32; 16] = [ - 0.125, 0.25, 0.5, 1.0, 0.25, 0.5, 1.0, 2.0, 0.5, 1.0, 2.0, 0.125, 1.0, 0.125, 0.25, 0.5, -]; - -fn pack_u32s(vals: &[u32]) -> Vec { - if vals.len() % LANES != 0 { - panic!("int2fp: value count {} is not a multiple of four", vals.len()); - } - vals.chunks(LANES) - .map(|chunk| { - let mut word = 0u128; - for (lane, value) in chunk.iter().enumerate() { - word |= u128::from(*value) << (lane * 32); - } - word - }) - .collect() -} - -fn copy_words(src: &[u128]) -> [u128; N] { - if src.len() > N { - panic!("int2fp: {} words exceed buffer {N}", src.len()); - } - let mut out = [0u128; N]; - out[..src.len()].copy_from_slice(src); - out -} - -fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { - u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) -} - -fn split_rs(value: u64) -> (u32, u32) { - (value as u32, (value >> 32) as u32) -} - -fn positive_finite(bits: u32) -> bool { - bits >> 31 == 0 && ((bits >> 23) & 0xff) != 0xff && (bits & 0x7fff_ffff) != 0 -} - -pub(crate) fn validate( - op1_bank: u32, - op2_bank: u32, - wr_bank: u32, - op1_col: u32, - op2_col: u32, - wr_col: u32, - iter: u32, - rs1: u64, - rs2: u64, - scales: &[u128], -) { - if iter == 0 || iter % 4 != 0 { - panic!("INT32_TO_FP32 iter must be a positive multiple of four"); - } - if iter > BANK_ENTRIES { - panic!("INT32_TO_FP32 input exceeds bank depth"); - } - if op1_bank >= BANK_NUM || op2_bank >= BANK_NUM || wr_bank >= BANK_NUM { - panic!("INT32_TO_FP32 bank id is invalid"); - } - if op1_col != 1 || op2_col != 1 || wr_col != 1 { - panic!("INT32_TO_FP32 operands must each occupy one bank"); - } - if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { - panic!("INT32_TO_FP32 banks must be distinct"); - } - if (rs1 & 0x3ff) != u64::from(op1_bank) - || ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) - || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) - { - panic!("int2fp: rs1 bank fields mismatch"); - } - if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { - panic!("int2fp: rs1 iter mismatch"); - } - if rs2 >> 1 != 0 { - panic!("INT32_TO_FP32 reserves rs2[63:1]"); - } - if scales.len() != SCALE_ROWS { - panic!("INT32_TO_FP32 must emit four scale rows"); - } - for word in scales { - for lane in 0..LANES { - let bits = (*word >> (lane * 32)) as u32; - if !positive_finite(bits) { - panic!("INT32_TO_FP32 scales must be finite and positive"); - } - } - } -} - -fn dst_words(src: &[u128], scales: &[u128], relu: bool) -> Vec { - src.iter() - .enumerate() - .map(|(row, word)| { - let scale = scales[row % SCALE_ROWS]; - let mut packed = 0u128; - for lane in 0..LANES { - let raw = (*word >> (lane * 32)) as u32 as i32; - let value = if relu { raw.max(0) } else { raw }; - let bits = int2fp_fp32_bits(value, (scale >> (lane * 32)) as u32); - packed |= u128::from(bits) << (lane * 32); - } - packed - }) - .collect() -} - -fn directed(bid: u32, rob_id: u32, iter: u32, relu: bool, values: &[i32], scales: &[f32]) -> Int2FpCase { - if values.len() != iter as usize * LANES { - panic!( - "int2fp: {} values do not match iter {iter} * {LANES}", - values.len() - ); - } - if scales.len() != SCALE_ROWS * LANES { - panic!("int2fp: {} scales, expected {}", scales.len(), SCALE_ROWS * LANES); - } - let src = pack_u32s(&values.iter().map(|v| *v as u32).collect::>()); - let scale = pack_u32s(&scales.iter().map(|v| v.to_bits()).collect::>()); - let op1_bank = 0; - let op2_bank = 1; - let wr_bank = 2; - let op1_col = 1; - let op2_col = 1; - let wr_col = 1; - let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); - let rs2 = u64::from(relu); - validate( - op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs1, rs2, &scale, - ); - if src.len() != iter as usize { - panic!("int2fp: src rows {} != iter {iter}", src.len()); - } - let dst = dst_words(&src, &scale, relu); - if dst.len() != iter as usize { - panic!("int2fp: dst rows {} != iter {iter}", dst.len()); - } - let (rs1_lo, rs1_hi) = split_rs(rs1); - let (rs2_lo, rs2_hi) = split_rs(rs2); - Int2FpCase { - cmd: Int2FpCmd { - bid, - iter, - relu: u32::from(relu), - op1_bank, - op2_bank, - wr_bank, - op1_col, - op2_col, - wr_col, - rob_id, - rs1_lo, - rs1_hi, - rs2_lo, - rs2_hi, - num_src_words: src.len() as u32, - num_scale_words: scale.len() as u32, - num_dst_words: dst.len() as u32, - }, - src_words: copy_words(&src), - scale_words: copy_words(&scale), - dst_words: copy_words(&dst), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cases_hit_relu_and_iter_bins() { - let mut relu = [false; 2]; - let mut iter = [false; 3]; - for index in 0..NUM_CASES { - let case = gen_case(index, 4); - assert_eq!(case.cmd.bid, 4); - assert_eq!(case.cmd.op1_col, 1); - assert_eq!(case.cmd.op2_col, 1); - assert_eq!(case.cmd.wr_col, 1); - assert_eq!(case.cmd.num_src_words, case.cmd.iter); - assert_eq!(case.cmd.num_dst_words, case.cmd.iter); - assert_eq!(case.cmd.num_scale_words, SCALE_ROWS as u32); - assert_eq!(case.cmd.rs2_lo & 1, case.cmd.relu); - assert_eq!(case.cmd.rs2_lo >> 1, 0); - assert_eq!(case.cmd.rs2_hi, 0); - relu[case.cmd.relu as usize] = true; - match case.cmd.iter { - 4 => iter[0] = true, - 8 => iter[1] = true, - 16 => iter[2] = true, - other => panic!("unexpected iter {other}"), - } - } - assert!(relu.iter().all(|hit| *hit)); - assert!(iter.iter().all(|hit| *hit)); - } - - #[test] - fn case_zero_matches_mlir_scale() { - let case = gen_case(0, 4); - assert_eq!(case.cmd.iter, 4); - assert_eq!(case.cmd.relu, 0); - assert_eq!(case.dst_words[0] as u32, 0.125f32.to_bits()); - assert_eq!((case.dst_words[0] >> 32) as u32, 0.25f32.to_bits()); - assert_eq!((case.dst_words[0] >> 64) as u32, 0.375f32.to_bits()); - assert_eq!((case.dst_words[0] >> 96) as u32, (-0.125f32).to_bits()); - } - - #[test] - fn case_one_relu_clamps_negatives() { - let case = gen_case(1, 4); - assert_eq!(case.cmd.iter, 4); - assert_eq!(case.cmd.relu, 1); - assert_eq!(case.dst_words[0] as u32, 0); - assert_eq!((case.dst_words[0] >> 32) as u32, 0); - assert_eq!((case.dst_words[0] >> 64) as u32, 0); - assert_eq!((case.dst_words[0] >> 96) as u32, 1.0f32.to_bits()); - } - - #[test] - #[should_panic(expected = "unsupported directed case")] - fn unknown_index_panics() { - let _ = gen_case(NUM_CASES, 4); - } - - #[test] - #[should_panic(expected = "positive multiple of four")] - fn illegal_iter_panics() { - validate(0, 1, 2, 1, 1, 1, 3, encode_rs1(0, 1, 2, 3), 0, &[0; 4]); - } - - #[test] - #[should_panic(expected = "banks must be distinct")] - fn overlapping_banks_panics() { - validate(0, 0, 2, 1, 1, 1, 4, encode_rs1(0, 0, 2, 4), 0, &[0; 4]); - } - - #[test] - #[should_panic(expected = "reserves rs2[63:1]")] - fn reserved_rs2_panics() { - let scale = pack_u32s(&[1.0f32.to_bits(); 16]); - validate(0, 1, 2, 1, 1, 1, 4, encode_rs1(0, 1, 2, 4), 2, &scale); - } - - #[test] - #[should_panic(expected = "finite and positive")] - fn nonpositive_scale_panics() { - let scale = pack_u32s(&[0u32; 16]); - validate(0, 1, 2, 1, 1, 1, 4, encode_rs1(0, 1, 2, 4), 0, &scale); - } -} diff --git a/examples/balls/int2fp/verify/casegen/src/lib.rs b/examples/balls/int2fp/verify/casegen/src/lib.rs deleted file mode 100644 index ad3308513..000000000 --- a/examples/balls/int2fp/verify/casegen/src/lib.rs +++ /dev/null @@ -1,158 +0,0 @@ -mod casegen; -#[path = "../../../emu/src/model.rs"] -mod model; - -use std::cell::RefCell; - -use casegen::{Int2FpCase, Int2FpCmd, MAX_DST, MAX_SCALE, MAX_SRC}; - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -#[no_mangle] -pub extern "C" fn int2fp_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid); - CURRENT.with(|current| *current.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|current| match current.borrow().as_ref() { - Some(case) => f(case), - None => panic!("int2fp_case: no case loaded; call int2fp_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn int2fp_case_cmd(out_ptr: *mut Int2FpCmd) { - if out_ptr.is_null() { - panic!("int2fp_case_cmd: null out_ptr"); - } - current_case(|case| unsafe { - *out_ptr = case.cmd; - }); -} - -fn src_index(index: u32, limit: u32, what: &str) -> usize { - if index >= limit { - panic!("int2fp_case_{what}: index {index} out of range {limit}"); - } - index as usize -} - -#[no_mangle] -pub extern "C" fn int2fp_case_src_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_src_words, "src_word_lo"); - if i >= MAX_SRC { - panic!("int2fp_case_src_word_lo: word_index out of range"); - } - case.src_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn int2fp_case_src_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_src_words, "src_word_hi"); - if i >= MAX_SRC { - panic!("int2fp_case_src_word_hi: word_index out of range"); - } - case.src_hi(i) - }) -} - -#[no_mangle] -pub extern "C" fn int2fp_case_scale_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_lo"); - if i >= MAX_SCALE { - panic!("int2fp_case_scale_word_lo: word_index out of range"); - } - case.scale_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn int2fp_case_scale_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_hi"); - if i >= MAX_SCALE { - panic!("int2fp_case_scale_word_hi: word_index out of range"); - } - case.scale_hi(i) - }) -} - -#[no_mangle] -pub extern "C" fn int2fp_case_dst_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_lo"); - if i >= MAX_DST { - panic!("int2fp_case_dst_word_lo: word_index out of range"); - } - case.dst_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn int2fp_case_dst_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_hi"); - if i >= MAX_DST { - panic!("int2fp_case_dst_word_hi: word_index out of range"); - } - case.dst_hi(i) - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dpi_load_then_cmd_and_words() { - assert_eq!(int2fp_case_load(0, 4), 0); - let mut cmd = Int2FpCmd { - bid: 0, - iter: 0, - relu: 0, - op1_bank: 0, - op2_bank: 0, - wr_bank: 0, - op1_col: 0, - op2_col: 0, - wr_col: 0, - rob_id: 0, - rs1_lo: 0, - rs1_hi: 0, - rs2_lo: 0, - rs2_hi: 0, - num_src_words: 0, - num_scale_words: 0, - num_dst_words: 0, - }; - int2fp_case_cmd(&mut cmd as *mut Int2FpCmd); - assert_eq!(cmd.bid, 4); - assert_eq!(cmd.iter, 4); - assert_eq!(cmd.relu, 0); - assert_eq!(cmd.op1_col, 1); - assert_eq!(cmd.op2_col, 1); - assert_eq!(cmd.wr_col, 1); - assert_eq!(cmd.num_src_words, 4); - assert_eq!(cmd.num_scale_words, 4); - assert_eq!(cmd.num_dst_words, 4); - let _lo = int2fp_case_src_word_lo(0); - let _hi = int2fp_case_src_word_hi(0); - let _slo = int2fp_case_scale_word_lo(0); - let _dlo = int2fp_case_dst_word_lo(0); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/int2fp/verify/src/tb_top.sv b/examples/balls/int2fp/verify/src/tb_top.sv deleted file mode 100644 index 961f4a179..000000000 --- a/examples/balls/int2fp/verify/src/tb_top.sv +++ /dev/null @@ -1,101 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import int2fp_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - Int2FpBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) - ); - - initial begin - intf.clock = 1'b0; - intf.reset = 1'b1; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("int2fp_ball_test"); - end -endmodule diff --git a/examples/balls/int2fp/workloads/ctests/CMakeLists.txt b/examples/balls/int2fp/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 27a32c4b6..000000000 --- a/examples/balls/int2fp/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_buckyball_ctests( - int2fp_tensor_values_test.c - int2fp_tensor_groups_test.c - int2fp_tensor_rows_test.c - int2fp_channel_lanes_test.c - int2fp_channel_base_test.c - int2fp_channel_two_rows_test.c -) diff --git a/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir b/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir deleted file mode 100644 index cb6610e77..000000000 --- a/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir +++ /dev/null @@ -1,34 +0,0 @@ -func.func private @check_result(memref<4x4xf32>) -> () - -memref.global "private" constant @input : memref<4x4xi32> = dense<[ - [1, 2, 3, -1], [-2, 0, 4, 5], - [10, -10, 7, 100], [-100, 8, 16, -8] -]> -memref.global "private" constant @scales : memref<4x4xf32> = - dense<1.250000e-01> - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %depth = arith.constant 4 : i64 - %stride = arith.constant 1 : i64 - %input = memref.get_global @input : memref<4x4xi32> - %scales = memref.get_global @scales : memref<4x4xf32> - %output = memref.alloc() alignment = 64 : memref<4x4xf32> - %input_bank = buckyball.bank_alloc - %scale_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride - : memref<4x4xi32> i64 i64 i64 - %loaded_scales = buckyball.bank_mvin %scales %scale_bank %depth %stride - : memref<4x4xf32> i64 i64 i64 - buckyball.int32_to_fp32 - %loaded_input, %loaded_scales, %output_bank, %depth : i64 - %stored = buckyball.bank_mvout %output %output_bank %depth %stride - : memref<4x4xf32> i64 i64 i64 - func.call @check_result(%output) : (memref<4x4xf32>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_scales : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %output : memref<4x4xf32> - return %zero_i8 : i8 -} diff --git a/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir b/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir deleted file mode 100644 index 862113577..000000000 --- a/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir +++ /dev/null @@ -1,33 +0,0 @@ -func.func private @check_result(memref<4x4xf32>) -> () - -memref.global "private" constant @input : memref<4x4xi32> = dense<[ - [-8, -1, 0, 1], [2, -3, 4, -5], - [6, 7, -8, 9], [-10, 11, 12, -13] -]> -memref.global "private" constant @scales : memref<4x4xf32> = dense<1.0> - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %depth = arith.constant 4 : i64 - %stride = arith.constant 1 : i64 - %input = memref.get_global @input : memref<4x4xi32> - %scales = memref.get_global @scales : memref<4x4xf32> - %output = memref.alloc() alignment = 64 : memref<4x4xf32> - %input_bank = buckyball.bank_alloc - %scale_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride - : memref<4x4xi32> i64 i64 i64 - %loaded_scales = buckyball.bank_mvin %scales %scale_bank %depth %stride - : memref<4x4xf32> i64 i64 i64 - buckyball.int32_to_fp32 - %loaded_input, %loaded_scales, %output_bank, %depth : i64 - %stored = buckyball.bank_mvout %output %output_bank %depth %stride - : memref<4x4xf32> i64 i64 i64 - func.call @check_result(%output) : (memref<4x4xf32>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_scales : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %output : memref<4x4xf32> - return %zero_i8 : i8 -} diff --git a/examples/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index 732c284a2..000000000 --- a/examples/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) - -add_buckyball_mlir_test(int2fp_unit_scale TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir b/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir deleted file mode 100644 index 5a79915c1..000000000 --- a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir +++ /dev/null @@ -1,35 +0,0 @@ -func.func private @check_result(memref<4x4xf32>) -> () - -memref.global "private" constant @input : memref<4x4xi32> = dense<[ - [1, 2, 3, -1], [-2, 0, 4, 5], - [10, -10, 7, 100], [-100, 8, 16, -8] -]> -memref.global "private" constant @scales : memref<4x4xf32> = - dense<1.250000e-01> - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %depth = arith.constant 4 : i64 - %stride = arith.constant 1 : i64 - %input = memref.get_global @input : memref<4x4xi32> - %scales = memref.get_global @scales : memref<4x4xf32> - %output = memref.alloc() alignment = 64 : memref<4x4xf32> - %input_bank = buckyball.bank_alloc - %scale_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride - : memref<4x4xi32> i64 i64 i64 - %loaded_scales = buckyball.bank_mvin %scales %scale_bank %depth %stride - : memref<4x4xf32> i64 i64 i64 - %result = buckyball.bank_int32_to_fp32 - %loaded_input %loaded_scales %output_bank %depth - : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %depth %stride - : memref<4x4xf32> i64 i64 i64 - func.call @check_result(%output) : (memref<4x4xf32>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_scales : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %output : memref<4x4xf32> - return %zero_i8 : i8 -} diff --git a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir b/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir deleted file mode 100644 index f9b8ab927..000000000 --- a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir +++ /dev/null @@ -1,34 +0,0 @@ -func.func private @check_result(memref<4x4xf32>) -> () - -memref.global "private" constant @input : memref<4x4xi32> = dense<[ - [-8, -1, 0, 1], [2, -3, 4, -5], - [6, 7, -8, 9], [-10, 11, 12, -13] -]> -memref.global "private" constant @scales : memref<4x4xf32> = dense<1.0> - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %depth = arith.constant 4 : i64 - %stride = arith.constant 1 : i64 - %input = memref.get_global @input : memref<4x4xi32> - %scales = memref.get_global @scales : memref<4x4xf32> - %output = memref.alloc() alignment = 64 : memref<4x4xf32> - %input_bank = buckyball.bank_alloc - %scale_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride - : memref<4x4xi32> i64 i64 i64 - %loaded_scales = buckyball.bank_mvin %scales %scale_bank %depth %stride - : memref<4x4xf32> i64 i64 i64 - %result = buckyball.bank_int32_to_fp32 - %loaded_input %loaded_scales %output_bank %depth - : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %depth %stride - : memref<4x4xf32> i64 i64 i64 - func.call @check_result(%output) : (memref<4x4xf32>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_scales : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %output : memref<4x4xf32> - return %zero_i8 : i8 -} diff --git a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in b/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in deleted file mode 100644 index 19be15e57..000000000 --- a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in +++ /dev/null @@ -1,39 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %lanes = arith.constant @BB_BANK_I32_LANES@ : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %depth4 = arith.constant 4 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - %scale = memref.alloc() : memref<4x4xf32> - %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32> - %one_f32 = arith.constant 1.0 : f32 - linalg.fill ins(%one_f32 : f32) outs(%scale : memref<4x4xf32>) - scf.for %row = %zero to %rows step %one { - scf.for %lane = %zero to %lanes step %one { - %r = arith.index_cast %row : index to i32 - %l = arith.index_cast %lane : index to i32 - %value = arith.subi %r, %l : i32 - memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - } - } - %input_bank = buckyball.bank_alloc - %scale_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> i64 i64 i64 - %loaded_scale = buckyball.bank_mvin %scale %scale_bank %depth4 %stride : memref<4x4xf32> i64 i64 i64 - %result = buckyball.bank_int32_to_fp32 %loaded_input %loaded_scale %output_bank %depth : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_scale : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - memref.dealloc %scale : memref<4x4xf32> - memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32> - return %zero_i8 : i8 -} diff --git a/examples/balls/int8add/arch/src/main/scala/Int8AddBall.scala b/examples/balls/int8add/arch/src/main/scala/Int8AddBall.scala deleted file mode 100644 index e580601ab..000000000 --- a/examples/balls/int8add/arch/src/main/scala/Int8AddBall.scala +++ /dev/null @@ -1,278 +0,0 @@ -package examples.balls.int8add - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import hardfloat.{rawFloatFromRecFN, recFNFromFN, RecFNToIN, RoundRawFNToRecFN} -import hardfloat.consts.{round_near_even, tininess_afterRounding} -import freechips.rocketchip.tile.{MulAddRecFNPipe, PipelinedINToRecFN} - -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.top.GlobalConfig - -@instantiable -class Int8AddBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "Int8AddBall") - .getOrElse(throw new IllegalArgumentException("Int8AddBall not found in config")) - - private val addFunct = b.ballDomain.ballISA - .find(_.mnemonic == "INT8ADD") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("INT8ADD not found in ballISA")) - - private val reluFunct = b.ballDomain.ballISA - .find(_.mnemonic == "INT8ADD_RELU") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("INT8ADD_RELU not found in ballISA")) - - require(mapping.inBW == 2, "Int8AddBall requires inBW=2") - require(mapping.outBW == 1, "Int8AddBall requires outBW=1") - require(b.memDomain.bankWidth == 128, "Int8AddBall requires 128-bit bank rows") - require((addFunct >> 4) == 4 && (reluFunct >> 4) == 4, "Int8AddBall instructions must encode two reads and one write") - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - private val idle :: waitForChannels :: readRequest :: readResponse :: calculate :: calculateDrain :: writeRequest :: writeResponse :: complete :: Nil = - Enum(9) - private val state = RegInit(idle) - private val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSub = RegInit(false.B) - private val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val lhsBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val rhsBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val outputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val groups = RegInit(0.U(5.W)) - private val group = RegInit(0.U(5.W)) - private val iter = RegInit(0.U(b.frontend.iter_len.W)) - private val row = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val lhsRatio = Reg(UInt(32.W)) - private val rhsRatio = Reg(UInt(32.W)) - private val relu = RegInit(false.B) - private val lhsWord = Reg(UInt(128.W)) - private val rhsWord = Reg(UInt(128.W)) - private val lhsRequested = RegInit(false.B) - private val rhsRequested = RegInit(false.B) - private val lhsReceived = RegInit(false.B) - private val rhsReceived = RegInit(false.B) - private val lane = RegInit(0.U(4.W)) - private val completed = RegInit(0.U(5.W)) - private val outputWord = Reg(Vec(16, UInt(8.W))) - - private def positiveFinite(value: UInt): Bool = - !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U - - private val lhsToFloat = Module(new PipelinedINToRecFN(8, 8, 24)) - private val rhsToFloat = Module(new PipelinedINToRecFN(8, 8, 24)) - private val lhsMul = Module(new MulAddRecFNPipe(3, 8, 24)) - private val rhsMul = Module(new MulAddRecFNPipe(3, 8, 24)) - private val add = Module(new PipelinedAddRawFN(8, 24)) - private val round = Module(new RoundRawFNToRecFN(8, 24, 0)) - private val toInt = Module(new RecFNToIN(8, 24, 8)) - private val lhsValue = (lhsWord >> (lane << 3))(7, 0) - private val rhsValue = (rhsWord >> (lane << 3))(7, 0) - private val calculateIssue = state === calculate - - private val ratioStage0 = RegEnable(lhsRatio, calculateIssue) - private val ratioStage1 = RegEnable(ratioStage0, RegNext(calculateIssue, false.B)) - private val rhsRatioStage0 = RegEnable(rhsRatio, calculateIssue) - private val rhsRatioStage1 = RegEnable(rhsRatioStage0, RegNext(calculateIssue, false.B)) - private val laneStage0 = RegEnable(lane, calculateIssue) - private val laneStage1 = RegEnable(laneStage0, RegNext(calculateIssue, false.B)) - - private val conversionValid = lhsToFloat.io.validout - private val productLane = Pipe(conversionValid, laneStage1, 3) - - private val addResult = add.io.rawOut - private val addLane = Pipe(lhsMul.io.validout, productLane.bits, 4) - private val addValid = add.io.validout - private val roundedResult = RegEnable(round.io.out, addValid) - private val roundedLane = RegEnable(addLane, addValid) - private val resultValid = RegNext(addValid, false.B) - - lhsToFloat.io.validin := calculateIssue - lhsToFloat.io.signedIn := true.B - lhsToFloat.io.in := lhsValue - lhsToFloat.io.bypassIn := lhsValue - lhsToFloat.io.roundingMode := round_near_even - lhsToFloat.io.detectTininess := tininess_afterRounding - lhsToFloat.io.typeTagIn := 0.U - lhsToFloat.io.wflagsIn := false.B - rhsToFloat.io.validin := calculateIssue - rhsToFloat.io.signedIn := true.B - rhsToFloat.io.in := rhsValue - rhsToFloat.io.bypassIn := rhsValue - rhsToFloat.io.roundingMode := round_near_even - rhsToFloat.io.detectTininess := tininess_afterRounding - rhsToFloat.io.typeTagIn := 0.U - rhsToFloat.io.wflagsIn := false.B - - lhsMul.io.validin := conversionValid - lhsMul.io.op := 0.U - lhsMul.io.a := lhsToFloat.io.out - lhsMul.io.b := recFNFromFN(8, 24, ratioStage1) - lhsMul.io.c := recFNFromFN(8, 24, 0.U) - lhsMul.io.roundingMode := round_near_even - lhsMul.io.detectTininess := tininess_afterRounding.asUInt - rhsMul.io.validin := conversionValid - rhsMul.io.op := 0.U - rhsMul.io.a := rhsToFloat.io.out - rhsMul.io.b := recFNFromFN(8, 24, rhsRatioStage1) - rhsMul.io.c := recFNFromFN(8, 24, 0.U) - rhsMul.io.roundingMode := round_near_even - rhsMul.io.detectTininess := tininess_afterRounding.asUInt - - add.io.validin := lhsMul.io.validout - add.io.subOp := false.B - add.io.a := rawFloatFromRecFN(8, 24, lhsMul.io.out) - add.io.b := rawFloatFromRecFN(8, 24, rhsMul.io.out) - add.io.roundingMode := round_near_even - round.io.invalidExc := add.io.invalidExc - round.io.infiniteExc := false.B - round.io.in := addResult - round.io.roundingMode := round_near_even - round.io.detectTininess := tininess_afterRounding - toInt.io.in := roundedResult - toInt.io.roundingMode := round_near_even - toInt.io.signedOut := true.B - private val result = Mux(relu && toInt.io.out(7), 0.U, toInt.io.out) - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRobId - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - for (port <- 0 until 2) { - io.bankRead(port).rob_id := robId - io.bankRead(port).ball_id := 0.U - io.bankRead(port).bank_id := Mux(port.U === 0.U, lhsBank, rhsBank) - io.bankRead(port).group_id := group - io.bankRead(port).io.req.valid := false.B - io.bankRead(port).io.req.bits.addr := row - io.bankRead(port).io.resp.ready := false.B - } - io.bankWrite(0).rob_id := robId - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := outputBank - io.bankWrite(0).group_id := group - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := row - io.bankWrite(0).io.req.bits.data := Cat(outputWord.reverse) - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - assert(cmd.funct7 === addFunct.U || cmd.funct7 === reluFunct.U, "Int8AddBall received an unknown funct7") - assert( - positiveFinite(cmd.rs2(31, 0)) && positiveFinite(cmd.rs2(63, 32)), - "Int8AddBall ratios must be finite and positive" - ) - assert( - cmd.op1_col =/= 0.U && cmd.op1_col === cmd.op2_col && cmd.op1_col === cmd.wr_col, - "Int8AddBall bank groups must match" - ) - assert(cmd.op1_col <= b.memDomain.bankNum.U, "Int8AddBall bank groups exceed physical banks") - assert( - cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, - "Int8AddBall banks must be distinct" - ) - assert(cmd.iter =/= 0.U && cmd.iter <= b.memDomain.bankEntries.U, "Int8AddBall iter must fit one physical bank") - robId := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRobId := io.cmdReq.bits.sub_rob_id - lhsBank := cmd.op1_bank - rhsBank := cmd.op2_bank - outputBank := cmd.wr_bank - groups := cmd.op1_col - group := 0.U - iter := cmd.iter - row := 0.U - lhsRatio := cmd.rs2(31, 0) - rhsRatio := cmd.rs2(63, 32) - relu := cmd.funct7 === reluFunct.U - lhsRequested := false.B - rhsRequested := false.B - lhsReceived := false.B - rhsReceived := false.B - lane := 0.U - state := waitForChannels - } - } - is(waitForChannels) { - when(io.channelReady)(state := readRequest) - } - is(readRequest) { - io.bankRead(0).io.req.valid := !lhsRequested - io.bankRead(1).io.req.valid := !rhsRequested - when(io.bankRead(0).io.req.fire)(lhsRequested := true.B) - when(io.bankRead(1).io.req.fire)(rhsRequested := true.B) - when((lhsRequested || io.bankRead(0).io.req.fire) && (rhsRequested || io.bankRead(1).io.req.fire)) { - state := readResponse - } - } - is(readResponse) { - io.bankRead(0).io.resp.ready := !lhsReceived - io.bankRead(1).io.resp.ready := !rhsReceived - when(io.bankRead(0).io.resp.fire) { lhsWord := io.bankRead(0).io.resp.bits.data; lhsReceived := true.B } - when(io.bankRead(1).io.resp.fire) { rhsWord := io.bankRead(1).io.resp.bits.data; rhsReceived := true.B } - when((lhsReceived || io.bankRead(0).io.resp.fire) && (rhsReceived || io.bankRead(1).io.resp.fire)) { - lane := 0.U - completed := 0.U - state := calculate - } - } - is(calculate) { - when(lane === 15.U) { - state := calculateDrain - }.otherwise { - lane := lane + 1.U - } - } - is(calculateDrain) {} - is(writeRequest) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire)(state := writeResponse) - } - is(writeResponse) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - lhsRequested := false.B - rhsRequested := false.B - lhsReceived := false.B - rhsReceived := false.B - when(row +& 1.U === iter) { - row := 0.U - when(group +& 1.U === groups)(state := complete) - .otherwise { group := group + 1.U; state := readRequest } - }.otherwise { row := row + 1.U; state := readRequest } - } - } - is(complete) { - when(io.cmdResp.fire)(state := idle) - } - } - - when((state === calculate || state === calculateDrain) && resultValid) { - outputWord(roundedLane.bits) := result - completed := completed + 1.U - when(completed === 15.U) { - state := writeRequest - } - } -} diff --git a/examples/balls/int8add/arch/src/main/scala/PipelinedAddRawFN.scala b/examples/balls/int8add/arch/src/main/scala/PipelinedAddRawFN.scala deleted file mode 100644 index ac30ed8df..000000000 --- a/examples/balls/int8add/arch/src/main/scala/PipelinedAddRawFN.scala +++ /dev/null @@ -1,167 +0,0 @@ -package examples.balls.int8add - -import chisel3._ -import chisel3.util._ -import hardfloat.{countLeadingZeros, isSigNaNRawFloat, lowMask, orReduceBy2, orReduceBy4, RawFloat} -import hardfloat.consts.round_min - -/** - * Throughput-oriented raw floating-point adder used by Int8AddBall. - * - * The Berkeley HardFloat AddRawFN implementation is combinational. This - * wrapper keeps the same raw-float equations but registers the alignment, - * normalization, and final select boundaries. It accepts one operand pair - * per cycle and reports the result four cycles later. - */ -class PipelinedAddRawFN(expWidth: Int, sigWidth: Int) extends Module { - - val io = IO(new Bundle { - val validin = Input(Bool()) - val subOp = Input(Bool()) - val a = Input(new RawFloat(expWidth, sigWidth)) - val b = Input(new RawFloat(expWidth, sigWidth)) - val roundingMode = Input(UInt(3.W)) - val validout = Output(Bool()) - val invalidExc = Output(Bool()) - val rawOut = Output(new RawFloat(expWidth, sigWidth + 2)) - }) - - private val alignDistWidth = log2Ceil(sigWidth) - - // Stage 0: register the raw operands and exponent comparison. - private val a0 = RegNext(io.a) - private val b0 = RegNext(io.b) - private val effSignB0 = RegNext(io.b.sign ^ io.subOp) - private val eqSigns0 = RegNext(io.a.sign === (io.b.sign ^ io.subOp)) - private val sDiffExps0 = RegNext(io.a.sExp - io.b.sExp) - private val roundingMode0 = RegNext(io.roundingMode) - - // Stage 1: prepare close-path subtraction and far-path operand selection. - private val modNatAlignDist1 = Mux(sDiffExps0 < 0.S, -sDiffExps0, sDiffExps0)(alignDistWidth - 1, 0) - - private val isMaxAlign1 = - (sDiffExps0 >> alignDistWidth) =/= 0.S && - ((sDiffExps0 >> alignDistWidth) =/= -1.S || sDiffExps0(alignDistWidth - 1, 0) === 0.U) - - private val alignDist1 = Mux(isMaxAlign1, ((BigInt(1) << alignDistWidth) - 1).U, modNatAlignDist1) - private val closeSubMags1 = !eqSigns0 && !isMaxAlign1 && (modNatAlignDist1 <= 1.U) - - private val closeAlignedSigA1 = - Mux((0.S <= sDiffExps0) && sDiffExps0(0), a0.sig << 2, 0.U) | - Mux((0.S <= sDiffExps0) && !sDiffExps0(0), a0.sig << 1, 0.U) | - Mux(sDiffExps0 < 0.S, a0.sig, 0.U) - - private val closeSSigSum1 = Wire(SInt((sigWidth + 3).W)) - closeSSigSum1 := closeAlignedSigA1.asSInt - (b0.sig << 1).asSInt - private val closeSigSum1 = Wire(UInt((sigWidth + 2).W)) - closeSigSum1 := Mux(closeSSigSum1 < 0.S, -closeSSigSum1, closeSSigSum1)(sigWidth + 1, 0) - private val closeAdjustedSigSum1 = Wire(UInt((sigWidth + 2).W)) - closeAdjustedSigSum1 := closeSigSum1 << (sigWidth & 1) - private val closeReduced2SigSum1 = orReduceBy2(closeAdjustedSigSum1) - - private val farSignOut1 = Mux(sDiffExps0 < 0.S, effSignB0, a0.sign) - private val farSigLarger1 = Wire(UInt(sigWidth.W)) - farSigLarger1 := Mux(sDiffExps0 < 0.S, b0.sig, a0.sig)(sigWidth - 1, 0) - private val farSigSmaller1 = Wire(UInt(sigWidth.W)) - farSigSmaller1 := Mux(sDiffExps0 < 0.S, a0.sig, b0.sig)(sigWidth - 1, 0) - - private val a1 = RegNext(a0) - private val b1 = RegNext(b0) - private val effSignB1 = RegNext(effSignB0) - private val eqSigns1 = RegNext(eqSigns0) - private val sDiffExps1 = RegNext(sDiffExps0) - private val roundingMode1 = RegNext(roundingMode0) - private val alignDist1r = RegInit(0.U(alignDistWidth.W)) - alignDist1r := alignDist1 - private val closeSubMags1r = RegInit(false.B) - closeSubMags1r := closeSubMags1 - private val closeSSigSum1r = RegInit(0.S((sigWidth + 3).W)) - closeSSigSum1r := closeSSigSum1 - private val closeSigSum1r = RegInit(0.U((sigWidth + 2).W)) - closeSigSum1r := closeSigSum1 - private val closeReduced2SigSum1r = RegInit(0.U(((sigWidth + 3) / 2).W)) - closeReduced2SigSum1r := closeReduced2SigSum1 - private val farSignOut1r = RegNext(farSignOut1) - private val farSigLarger1r = RegInit(0.U(sigWidth.W)) - farSigLarger1r := farSigLarger1 - private val farSigSmaller1r = RegInit(0.U(sigWidth.W)) - farSigSmaller1r := farSigSmaller1 - - // Stage 2: leading-zero normalization and far-path alignment. - private val closeNormDistReduced2_2 = countLeadingZeros(closeReduced2SigSum1r) - private val closeNearNormDist2 = (closeNormDistReduced2_2 << 1)(alignDistWidth - 1, 0) - private val closeSigOut2 = ((closeSigSum1r << closeNearNormDist2) << 1)(sigWidth + 2, 0) - private val closeTotalCancellation2 = !(closeSigOut2(sigWidth + 2, sigWidth + 1).orR) - private val closeNotTotalCancellationSignOut2 = a1.sign ^ (closeSSigSum1r < 0.S) - - private val farMainAlignedSigSmaller2 = Wire(UInt((sigWidth + 5).W)) - farMainAlignedSigSmaller2 := (farSigSmaller1r << 5) >> alignDist1r - private val farReduced4SigSmaller2 = Wire(UInt(((sigWidth + 5) / 4).W)) - farReduced4SigSmaller2 := orReduceBy4(Cat(farSigSmaller1r, 0.U(2.W))) - private val farRoundExtraMask2 = lowMask(alignDist1r(alignDistWidth - 1, 2), (sigWidth + 5) / 4, 0) - - private val farAlignedSigSmaller2 = Cat( - farMainAlignedSigSmaller2 >> 3, - farMainAlignedSigSmaller2(2, 0).orR || (farReduced4SigSmaller2 & farRoundExtraMask2).orR - ) - - private val farSubMags2 = !eqSigns1 - - private val a2 = RegNext(a1) - private val b2 = RegNext(b1) - private val effSignB2 = RegNext(effSignB1) - private val eqSigns2 = RegNext(eqSigns1) - private val sDiffExps2 = RegNext(sDiffExps1) - private val roundingMode2 = RegNext(roundingMode1) - private val closeSubMags2r = RegNext(closeSubMags1r) - private val closeNearNormDist2r = RegNext(closeNearNormDist2) - private val closeSigOut2r = RegNext(closeSigOut2) - private val closeTotalCancellation2r = RegNext(closeTotalCancellation2) - private val closeNotTotalCancellationSignOut2r = RegNext(closeNotTotalCancellationSignOut2) - private val farSignOut2r = RegNext(farSignOut1r) - private val farSigLarger2r = RegNext(farSigLarger1r) - private val farAlignedSigSmaller2r = RegNext(farAlignedSigSmaller2) - private val farSubMags2r = RegNext(farSubMags2) - - // Stage 3: significand add/subtract and special-case selection. - private val farNegAlignedSigSmaller3 = Mux( - farSubMags2r, - Cat(1.U, ~farAlignedSigSmaller2r), - farAlignedSigSmaller2r - ) - - private val farSigSum3 = (farSigLarger2r << 3) + farNegAlignedSigSmaller3 + farSubMags2r - private val farSigOut3 = Mux(farSubMags2r, farSigSum3, (farSigSum3 >> 1) | farSigSum3(0))(sigWidth + 2, 0) - - private val notSigNaNInvalidExc3 = a2.isInf && b2.isInf && !eqSigns2 - private val notNaNIsInfOut3 = a2.isInf || b2.isInf - private val addZeros3 = a2.isZero && b2.isZero - private val notNaNSpecialCase3 = notNaNIsInfOut3 || addZeros3 - private val notNaNIsZeroOut3 = addZeros3 || (!notNaNIsInfOut3 && closeSubMags2r && closeTotalCancellation2r) - - private val notNaNSignOut3 = - (eqSigns2 && a2.sign) || - (a2.isInf && a2.sign) || - (b2.isInf && effSignB2) || - (notNaNIsZeroOut3 && !eqSigns2 && roundingMode2 === round_min) || - (!notNaNSpecialCase3 && closeSubMags2r && !closeTotalCancellation2r && closeNotTotalCancellationSignOut2r) || - (!notNaNSpecialCase3 && !closeSubMags2r && farSignOut2r) - - private val commonSExp3 = - Mux(closeSubMags2r || (sDiffExps2 < 0.S), b2.sExp, a2.sExp) - - Mux(closeSubMags2r, closeNearNormDist2r, farSubMags2r).zext - - private val commonSigOut3 = Mux(closeSubMags2r, closeSigOut2r, farSigOut3) - - private val rawOutReg = RegInit(0.U.asTypeOf(new RawFloat(expWidth, sigWidth + 2))) - rawOutReg.isNaN := a2.isNaN || b2.isNaN - rawOutReg.isInf := notNaNIsInfOut3 - rawOutReg.isZero := notNaNIsZeroOut3 - rawOutReg.sExp := commonSExp3 - rawOutReg.sign := notNaNSignOut3 - rawOutReg.sig := commonSigOut3 - - io.rawOut := rawOutReg - io.invalidExc := RegNext(isSigNaNRawFloat(a2) || isSigNaNRawFloat(b2) || notSigNaNInvalidExc3) - io.validout := ShiftRegister(io.validin, 4) -} diff --git a/examples/balls/int8add/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/int8add/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 27db06fe8..000000000 --- a/examples/balls/int8add/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateInt8AddBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} - -namespace { -class BankInt8AddPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankInt8AddOp op, - PatternRewriter &rewriter) const override { - rewriter.create( - op.getLoc(), op.getLhsBank(), op.getRhsBank(), op.getOutputBank(), - op.getIter(), op.getLhsRatio(), op.getRhsRatio(), op.getReluAttr()); - rewriter.replaceOp(op, op.getOutputBank()); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateInt8AddBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/int8add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/int8add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 090933fe4..000000000 --- a/examples/balls/int8add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct Int8AddLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(Int8AddOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("Int8AddBall"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getLhsBankId(), - adaptor.getRhsBankId(), - adaptor.getOutputBankId(), adaptor.getIter()); - Value lhsBits = rewriter.create( - loc, rewriter.getI32Type(), adaptor.getLhsRatio()); - Value rhsBits = rewriter.create( - loc, rewriter.getI32Type(), adaptor.getRhsRatio()); - Value lhs64 = - rewriter.create(loc, rewriter.getI64Type(), lhsBits); - Value rhs64 = - rewriter.create(loc, rewriter.getI64Type(), rhsBits); - Value rs2 = rewriter.create( - loc, lhs64, - rewriter.create(loc, rhs64, cstI64(rewriter, loc, 32))); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr(buckyball_target::getBuckyballFunct7( - op.getRelu() ? "INT8ADD_RELU" : "INT8ADD"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateInt8AddBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { - patterns.add(converter); -} - -void configureInt8AddBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/int8add/configs/default.toml b/examples/balls/int8add/configs/default.toml deleted file mode 100644 index 15ef6060d..000000000 --- a/examples/balls/int8add/configs/default.toml +++ /dev/null @@ -1,3 +0,0 @@ -# Int8AddBall has no private configuration. - -[ball] diff --git a/examples/balls/int8add/emu/src/dispatch.rs b/examples/balls/int8add/emu/src/dispatch.rs deleted file mode 100644 index ede132ed1..000000000 --- a/examples/balls/int8add/emu/src/dispatch.rs +++ /dev/null @@ -1,32 +0,0 @@ -use crate::inst::instruction::ExecContext; - -use super::int8add; - -const BALL_CLASS: &str = "examples.balls.int8add.Int8AddBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("INT8ADD") => Some(int8add::Int8Add::exec(xs1, xs2, ctx, false)), - Some("INT8ADD_RELU") => Some(int8add::Int8Add::exec(xs1, xs2, ctx, true)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("INT8ADD") | Some("INT8ADD_RELU") => Some(int8add::Int8Add::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/int8add/emu/src/lib.rs b/examples/balls/int8add/emu/src/lib.rs deleted file mode 100644 index b1ebd2d62..000000000 --- a/examples/balls/int8add/emu/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod int8add; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/int8add/verify/casegen/src/casegen.rs b/examples/balls/int8add/verify/casegen/src/casegen.rs deleted file mode 100644 index c5c4a0cd6..000000000 --- a/examples/balls/int8add/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,339 +0,0 @@ -pub const ROW_BYTES: usize = 16; -pub const BANK_ENTRIES: u32 = 64; -pub const MAX_ROWS: usize = 64; -pub const NUM_CASES: u32 = 2; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Int8AddCmd { - pub relu: u32, - pub bid: u32, - pub iter: u32, - pub op1_bank: u32, - pub op2_bank: u32, - pub wr_bank: u32, - pub op1_col: u32, - pub op2_col: u32, - pub wr_col: u32, - pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, - pub rs2_lo: u32, - pub rs2_hi: u32, - pub num_lhs_words: u32, - pub num_rhs_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Int8AddCase { - pub cmd: Int8AddCmd, - pub lhs_words: [u128; MAX_ROWS], - pub rhs_words: [u128; MAX_ROWS], - pub dst_words: [u128; MAX_ROWS], -} - -impl Int8AddCase { - pub fn lhs_lo(&self, index: usize) -> u64 { - self.lhs_words[index] as u64 - } - - pub fn lhs_hi(&self, index: usize) -> u64 { - (self.lhs_words[index] >> 64) as u64 - } - - pub fn rhs_lo(&self, index: usize) -> u64 { - self.rhs_words[index] as u64 - } - - pub fn rhs_hi(&self, index: usize) -> u64 { - (self.rhs_words[index] >> 64) as u64 - } - - pub fn dst_lo(&self, index: usize) -> u64 { - self.dst_words[index] as u64 - } - - pub fn dst_hi(&self, index: usize) -> u64 { - (self.dst_words[index] >> 64) as u64 - } -} - -pub fn gen_case(index: u32, bid: u32) -> Int8AddCase { - if index >= NUM_CASES { - panic!("int8add: unsupported directed case {index}"); - } - match index { - 0 => directed(bid, 2, false), - 1 => directed(bid, 3, true), - _ => panic!("int8add: unsupported directed case {index}"), - } -} - -fn compute(lhs: i8, rhs: i8, lhs_ratio: f32, rhs_ratio: f32, relu: bool) -> i8 { - let value = (lhs as f32) * lhs_ratio + (rhs as f32) * rhs_ratio; - let rounded = value.round_ties_even(); - let clamped = if relu { - rounded.max(0.0) - } else { - rounded - } - .clamp(-128.0, 127.0); - clamped as i8 -} - -fn positive_finite(value: f32) -> bool { - value.is_finite() && value > 0.0 -} - -fn ctest_values(rows: usize) -> (Vec, Vec) { - let count = rows * ROW_BYTES; - let mut lhs = vec![0u8; count]; - let mut rhs = vec![0u8; count]; - for i in 0..count { - lhs[i] = ((i as i32 * 37 & 255) - 128) as i8 as u8; - rhs[i] = ((i as i32 * 19 & 255) - 128) as i8 as u8; - } - (pack_bytes(&lhs), pack_bytes(&rhs)) -} - -fn pack_bytes(bytes: &[u8]) -> Vec { - if bytes.len() % ROW_BYTES != 0 { - panic!( - "int8add: {} bytes is not a multiple of {ROW_BYTES}", - bytes.len() - ); - } - bytes - .chunks(ROW_BYTES) - .map(|chunk| { - let mut word = 0u128; - for (lane, byte) in chunk.iter().enumerate() { - word |= u128::from(*byte) << (lane * 8); - } - word - }) - .collect() -} - -fn copy_words(src: &[u128]) -> [u128; N] { - if src.len() > N { - panic!("int8add: {} words exceed buffer {N}", src.len()); - } - let mut out = [0u128; N]; - out[..src.len()].copy_from_slice(src); - out -} - -fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { - u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) -} - -fn encode_rs2(lhs_ratio: f32, rhs_ratio: f32) -> u64 { - u64::from(lhs_ratio.to_bits()) | (u64::from(rhs_ratio.to_bits()) << 32) -} - -fn split_rs(value: u64) -> (u32, u32) { - (value as u32, (value >> 32) as u32) -} - -pub(crate) fn validate( - op1_bank: u32, - op2_bank: u32, - wr_bank: u32, - op1_col: u32, - op2_col: u32, - wr_col: u32, - iter: u32, - rs1: u64, - rs2: u64, - lhs_ratio: f32, - rhs_ratio: f32, -) { - if iter == 0 || iter > BANK_ENTRIES { - panic!("int8add: iter must fit in one physical bank"); - } - if op1_col != 1 || op2_col != 1 || wr_col != 1 { - panic!("int8add: bank groups must match"); - } - if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { - panic!("int8add: banks must be distinct"); - } - if !positive_finite(lhs_ratio) || !positive_finite(rhs_ratio) { - panic!("int8add: ratios must be finite and positive"); - } - if (rs1 & 0x3ff) != u64::from(op1_bank) - || ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) - || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) - { - panic!("int8add: rs1 bank fields mismatch"); - } - if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { - panic!("int8add: rs1 iter mismatch"); - } - if rs2 != encode_rs2(lhs_ratio, rhs_ratio) { - panic!("int8add: rs2 ratio fields mismatch"); - } -} - -fn dst_words( - lhs: &[u128], - rhs: &[u128], - iter: u32, - lhs_ratio: f32, - rhs_ratio: f32, - relu: bool, -) -> Vec { - let mut out = Vec::with_capacity(iter as usize); - for row in 0..iter as usize { - let mut packed = 0u128; - for lane in 0..ROW_BYTES { - let l = ((lhs[row] >> (lane * 8)) & 0xff) as u8 as i8; - let r = ((rhs[row] >> (lane * 8)) & 0xff) as u8 as i8; - let q = compute(l, r, lhs_ratio, rhs_ratio, relu); - packed |= u128::from(q as u8) << (lane * 8); - } - out.push(packed); - } - out -} - -fn directed(bid: u32, rob_id: u32, relu: bool) -> Int8AddCase { - let iter = 7u32; - let lhs_ratio = 0.5f32; - let rhs_ratio = 0.25f32; - let (lhs, rhs) = ctest_values(iter as usize); - let op1_bank = 0; - let op2_bank = 1; - let wr_bank = 2; - let op1_col = 1; - let op2_col = 1; - let wr_col = 1; - let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); - let rs2 = encode_rs2(lhs_ratio, rhs_ratio); - validate( - op1_bank, - op2_bank, - wr_bank, - op1_col, - op2_col, - wr_col, - iter, - rs1, - rs2, - lhs_ratio, - rhs_ratio, - ); - if lhs.len() != iter as usize || rhs.len() != iter as usize { - panic!("int8add: operand rows != iter {iter}"); - } - let dst = dst_words(&lhs, &rhs, iter, lhs_ratio, rhs_ratio, relu); - let (rs1_lo, rs1_hi) = split_rs(rs1); - let (rs2_lo, rs2_hi) = split_rs(rs2); - Int8AddCase { - cmd: Int8AddCmd { - relu: u32::from(relu), - bid, - iter, - op1_bank, - op2_bank, - wr_bank, - op1_col, - op2_col, - wr_col, - rob_id, - rs1_lo, - rs1_hi, - rs2_lo, - rs2_hi, - num_lhs_words: lhs.len() as u32, - num_rhs_words: rhs.len() as u32, - num_dst_words: dst.len() as u32, - }, - lhs_words: copy_words(&lhs), - rhs_words: copy_words(&rhs), - dst_words: copy_words(&dst), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn ctest_expected(a: i8, b: i8, relu: bool) -> i8 { - compute(a, b, 0.5, 0.25, relu) - } - - #[test] - fn case_zero_matches_ctest() { - let case = gen_case(0, 7); - assert_eq!(case.cmd.bid, 7); - assert_eq!(case.cmd.relu, 0); - assert_eq!(case.cmd.iter, 7); - assert_eq!(case.cmd.op1_bank, 0); - assert_eq!(case.cmd.op2_bank, 1); - assert_eq!(case.cmd.wr_bank, 2); - assert_eq!(case.cmd.num_lhs_words, 7); - assert_eq!(case.cmd.num_rhs_words, 7); - assert_eq!(case.cmd.num_dst_words, 7); - let lhs0 = ((case.lhs_words[0] >> 0) & 0xff) as u8 as i8; - let rhs0 = ((case.rhs_words[0] >> 0) & 0xff) as u8 as i8; - assert_eq!(lhs0, -128); - assert_eq!(rhs0, -128); - let got0 = (case.dst_words[0] & 0xff) as u8 as i8; - assert_eq!(got0, ctest_expected(lhs0, rhs0, false)); - } - - #[test] - fn cases_hit_funct7_bins() { - let mut hit = [false; 2]; - for index in 0..NUM_CASES { - let case = gen_case(index, 7); - assert_eq!(case.cmd.bid, 7); - assert_eq!(case.cmd.op1_col, 1); - assert_eq!(case.cmd.op2_col, 1); - assert_eq!(case.cmd.wr_col, 1); - assert_eq!(case.cmd.num_lhs_words, case.cmd.iter); - assert_eq!(case.cmd.num_rhs_words, case.cmd.iter); - assert_eq!(case.cmd.num_dst_words, case.cmd.iter); - hit[case.cmd.relu as usize] = true; - } - assert!(hit.iter().all(|h| *h)); - } - - #[test] - fn relu_floors_negatives() { - let case = gen_case(1, 7); - assert_eq!(case.cmd.relu, 1); - for row in 0..case.cmd.iter as usize { - for lane in 0..ROW_BYTES { - let got = ((case.dst_words[row] >> (lane * 8)) & 0xff) as u8 as i8; - assert!(got >= 0); - } - } - } - - #[test] - #[should_panic(expected = "unsupported directed case")] - fn unknown_index_panics() { - let _ = gen_case(NUM_CASES, 7); - } - - #[test] - #[should_panic(expected = "iter must fit")] - fn illegal_iter_panics() { - validate(0, 1, 2, 1, 1, 1, 0, encode_rs1(0, 1, 2, 0), encode_rs2(0.5, 0.25), 0.5, 0.25); - } - - #[test] - #[should_panic(expected = "banks must be distinct")] - fn overlapping_banks_panics() { - validate(0, 0, 2, 1, 1, 1, 7, encode_rs1(0, 0, 2, 7), encode_rs2(0.5, 0.25), 0.5, 0.25); - } - - #[test] - #[should_panic(expected = "finite and positive")] - fn nonpositive_ratio_panics() { - validate(0, 1, 2, 1, 1, 1, 7, encode_rs1(0, 1, 2, 7), encode_rs2(0.0, 0.25), 0.0, 0.25); - } -} diff --git a/examples/balls/int8add/verify/casegen/src/lib.rs b/examples/balls/int8add/verify/casegen/src/lib.rs deleted file mode 100644 index d14ad09c7..000000000 --- a/examples/balls/int8add/verify/casegen/src/lib.rs +++ /dev/null @@ -1,117 +0,0 @@ -mod casegen; - -use std::cell::RefCell; - -use casegen::{Int8AddCase, Int8AddCmd}; - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -#[no_mangle] -pub extern "C" fn int8add_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid); - CURRENT.with(|current| *current.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|current| match current.borrow().as_ref() { - Some(case) => f(case), - None => panic!("int8add_case: no case loaded; call int8add_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn int8add_case_cmd(out_ptr: *mut Int8AddCmd) { - if out_ptr.is_null() { - panic!("int8add_case_cmd: null out_ptr"); - } - current_case(|case| unsafe { - *out_ptr = case.cmd; - }); -} - -fn idx(index: u32, limit: u32, what: &str) -> usize { - if index >= limit { - panic!("int8add_case_{what}: index {index} out of range {limit}"); - } - index as usize -} - -#[no_mangle] -pub extern "C" fn int8add_case_lhs_word_lo(word_index: u32) -> u64 { - current_case(|case| case.lhs_lo(idx(word_index, case.cmd.num_lhs_words, "lhs_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn int8add_case_lhs_word_hi(word_index: u32) -> u64 { - current_case(|case| case.lhs_hi(idx(word_index, case.cmd.num_lhs_words, "lhs_word_hi"))) -} - -#[no_mangle] -pub extern "C" fn int8add_case_rhs_word_lo(word_index: u32) -> u64 { - current_case(|case| case.rhs_lo(idx(word_index, case.cmd.num_rhs_words, "rhs_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn int8add_case_rhs_word_hi(word_index: u32) -> u64 { - current_case(|case| case.rhs_hi(idx(word_index, case.cmd.num_rhs_words, "rhs_word_hi"))) -} - -#[no_mangle] -pub extern "C" fn int8add_case_dst_word_lo(word_index: u32) -> u64 { - current_case(|case| case.dst_lo(idx(word_index, case.cmd.num_dst_words, "dst_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn int8add_case_dst_word_hi(word_index: u32) -> u64 { - current_case(|case| case.dst_hi(idx(word_index, case.cmd.num_dst_words, "dst_word_hi"))) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dpi_load_then_cmd_and_words() { - assert_eq!(int8add_case_load(0, 7), 0); - let mut cmd = Int8AddCmd { - relu: 0, - bid: 0, - iter: 0, - op1_bank: 0, - op2_bank: 0, - wr_bank: 0, - op1_col: 0, - op2_col: 0, - wr_col: 0, - rob_id: 0, - rs1_lo: 0, - rs1_hi: 0, - rs2_lo: 0, - rs2_hi: 0, - num_lhs_words: 0, - num_rhs_words: 0, - num_dst_words: 0, - }; - int8add_case_cmd(&mut cmd as *mut Int8AddCmd); - assert_eq!(cmd.bid, 7); - assert_eq!(cmd.iter, 7); - assert_eq!(cmd.relu, 0); - assert_eq!(cmd.num_lhs_words, 7); - assert_eq!(cmd.num_rhs_words, 7); - assert_eq!(cmd.num_dst_words, 7); - let _ = int8add_case_lhs_word_lo(0); - let _ = int8add_case_rhs_word_lo(0); - let _ = int8add_case_dst_word_lo(0); - assert_eq!(casegen::NUM_CASES, 2); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/int8add/verify/src/tb_top.sv b/examples/balls/int8add/verify/src/tb_top.sv deleted file mode 100644 index 250ef5dc8..000000000 --- a/examples/balls/int8add/verify/src/tb_top.sv +++ /dev/null @@ -1,101 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import int8add_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - Int8AddBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) - ); - - initial begin - intf.clock = 1'b0; - intf.reset = 1'b1; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("int8add_ball_test"); - end -endmodule diff --git a/examples/balls/int8add/workloads/ctests/CMakeLists.txt b/examples/balls/int8add/workloads/ctests/CMakeLists.txt deleted file mode 100644 index ce68547cc..000000000 --- a/examples/balls/int8add/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_buckyball_ctests(int8add_scaled_test.c int8add_relu_test.c) diff --git a/examples/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir b/examples/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir deleted file mode 100644 index 57575b219..000000000 --- a/examples/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir +++ /dev/null @@ -1,42 +0,0 @@ -func.func private @check_result(memref<4x16xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %depth = arith.constant 4 : i64 - %stride = arith.constant 1 : i64 - %ratio = arith.constant 1.0 : f32 - %a = memref.alloc() : memref<4x16xi8> - %b = memref.alloc() : memref<4x16xi8> - %out = memref.alloc() : memref<4x16xi8> - scf.for %row = %zero to %four step %one { - scf.for %lane = %zero to %sixteen step %one { - %r = arith.index_cast %row : index to i32 - %l = arith.index_cast %lane : index to i32 - %a32 = arith.subi %l, %r : i32 - %b32 = arith.subi %r, %l : i32 - %a8 = arith.trunci %a32 : i32 to i8 - %b8 = arith.trunci %b32 : i32 to i8 - memref.store %a8, %a[%row, %lane] : memref<4x16xi8> - memref.store %b8, %b[%row, %lane] : memref<4x16xi8> - } - } - %a_bank = buckyball.bank_alloc - %b_bank = buckyball.bank_alloc - %out_bank = buckyball.bank_alloc - %a_loaded = buckyball.bank_mvin %a %a_bank %depth %stride : memref<4x16xi8> i64 i64 i64 - %b_loaded = buckyball.bank_mvin %b %b_bank %depth %stride : memref<4x16xi8> i64 i64 i64 - buckyball.int8add %a_loaded, %b_loaded, %out_bank, %depth, %ratio, %ratio : i64 - %stored = buckyball.bank_mvout %out %out_bank %depth %stride : memref<4x16xi8> i64 i64 i64 - func.call @check_result(%out) : (memref<4x16xi8>) -> () - buckyball.bank_release %a_loaded : i64 - buckyball.bank_release %b_loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %a : memref<4x16xi8> - memref.dealloc %b : memref<4x16xi8> - memref.dealloc %out : memref<4x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in b/examples/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in deleted file mode 100644 index 8abc7defa..000000000 --- a/examples/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in +++ /dev/null @@ -1,44 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %mod = arith.constant 31 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %lanes = arith.constant @BB_BANK_I8_LANES@ : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %stride = arith.constant 1 : i64 - %ratio = arith.constant 1.0 : f32 - %a = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %b = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %out = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - scf.for %row = %zero to %rows step %one { - scf.for %lane = %zero to %lanes step %one { - %row_mod = arith.remui %row, %mod : index - %r = arith.index_cast %row_mod : index to i32 - %l = arith.index_cast %lane : index to i32 - %a32 = arith.subi %l, %r : i32 - %b32 = arith.subi %r, %l : i32 - %a8 = arith.trunci %a32 : i32 to i8 - %b8 = arith.trunci %b32 : i32 to i8 - memref.store %a8, %a[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.store %b8, %b[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - } - } - %a_bank = buckyball.bank_alloc - %b_bank = buckyball.bank_alloc - %out_bank = buckyball.bank_alloc - %a_loaded = buckyball.bank_mvin %a %a_bank %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - %b_loaded = buckyball.bank_mvin %b %b_bank %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - %result = buckyball.bank_int8add %a_loaded %b_loaded %out_bank %depth %ratio %ratio : i64 i64 i64 i64 f32 - %stored = buckyball.bank_mvout %out %result %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - func.call @check_result(%out) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %a_loaded : i64 - buckyball.bank_release %b_loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %a : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.dealloc %b : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.dealloc %out : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/int8mul/arch/src/main/scala/Int8MulBall.scala b/examples/balls/int8mul/arch/src/main/scala/Int8MulBall.scala deleted file mode 100644 index 825deca55..000000000 --- a/examples/balls/int8mul/arch/src/main/scala/Int8MulBall.scala +++ /dev/null @@ -1,238 +0,0 @@ -package examples.balls.int8mul - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import hardfloat.{recFNFromFN, INToRecFN, RecFNToIN} -import hardfloat.consts.{round_near_even, tininess_afterRounding} -import freechips.rocketchip.tile.MulAddRecFNPipe - -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.top.GlobalConfig - -@instantiable -class Int8MulBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "Int8MulBall") - .getOrElse(throw new IllegalArgumentException("Int8MulBall not found in config")) - - private val funct = b.ballDomain.ballISA - .find(_.mnemonic == "INT8MUL") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("INT8MUL not found in ballISA")) - - require(mapping.inBW == 2, "Int8MulBall requires inBW=2") - require(mapping.outBW == 1, "Int8MulBall requires outBW=1") - require(b.memDomain.bankWidth == 128, "Int8MulBall requires 128-bit bank rows") - private val bankRowBits = log2Ceil(b.memDomain.bankEntries) - require(bankRowBits <= 32, "Int8MulBall gate_row must fit rs2[63:32]") - require((funct >> 4) == 4, "INT8MUL must encode two reads and one write") - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - private val idle :: waitForChannels :: gateRequest :: gateResponse :: inputRequest :: inputResponse :: calculate :: calculateDrain :: writeRequest :: writeResponse :: complete :: Nil = - Enum(11) - private val state = RegInit(idle) - private val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSub = RegInit(false.B) - private val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val gateBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val inputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val outputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val groups = RegInit(0.U(5.W)) - private val group = RegInit(0.U(5.W)) - private val iter = RegInit(0.U(b.frontend.iter_len.W)) - private val gateRow = RegInit(0.U(bankRowBits.W)) - private val inputRow = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val ratio = Reg(UInt(32.W)) - private val gateWord = Reg(UInt(128.W)) - private val inputWord = Reg(UInt(128.W)) - private val lane = RegInit(0.U(4.W)) - private val outputWord = Reg(Vec(16, UInt(8.W))) - - private val productReg = Reg(UInt(16.W)) - private val productLaneReg = Reg(UInt(4.W)) - private val productValid = RegInit(false.B) - private val productFloatReg = Reg(UInt(33.W)) - private val floatLaneReg = Reg(UInt(4.W)) - private val floatValid = RegInit(false.B) - private val scaledReg = Reg(UInt(33.W)) - private val scaledLaneReg = Reg(UInt(4.W)) - private val scaledValid = RegInit(false.B) - - private def positiveFinite(value: UInt): Bool = - !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U - - private val gateValue = (gateWord >> (lane << 3))(7, 0).asSInt - private val inputValue = (inputWord >> (lane << 3))(7, 0).asSInt - private val integerProduct = (gateValue * inputValue).asUInt - private val productToFloat = Module(new INToRecFN(16, 8, 24)) - private val scale = Module(new MulAddRecFNPipe(3, 8, 24)) - private val toInt8 = Module(new RecFNToIN(8, 24, 8)) - - productToFloat.io.signedIn := true.B - productToFloat.io.in := productReg - productToFloat.io.roundingMode := round_near_even - productToFloat.io.detectTininess := tininess_afterRounding - scale.io.validin := floatValid - scale.io.op := 0.U - scale.io.a := productFloatReg - scale.io.b := recFNFromFN(8, 24, ratio) - scale.io.c := recFNFromFN(8, 24, 0.U) - scale.io.roundingMode := round_near_even - scale.io.detectTininess := tininess_afterRounding.asUInt - toInt8.io.in := scaledReg - toInt8.io.roundingMode := round_near_even - toInt8.io.signedOut := true.B - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRobId - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - for (port <- 0 until 2) { - io.bankRead(port).rob_id := robId - io.bankRead(port).ball_id := 0.U - io.bankRead(port).bank_id := Mux(port.U === 0.U, gateBank, inputBank) - io.bankRead(port).group_id := group - io.bankRead(port).io.req.valid := false.B - io.bankRead(port).io.req.bits.addr := Mux(port.U === 0.U, gateRow, inputRow) - io.bankRead(port).io.resp.ready := false.B - } - - io.bankWrite(0).rob_id := robId - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := outputBank - io.bankWrite(0).group_id := group - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := inputRow - io.bankWrite(0).io.req.bits.data := Cat(outputWord.reverse) - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - assert(cmd.funct7 === funct.U, "Int8MulBall funct7 must be INT8MUL") - if (bankRowBits < 32) - assert( - cmd.rs2(63, 32 + bankRowBits) === 0.U, - s"Int8MulBall reserves rs2 bits 63:${32 + bankRowBits}" - ) - assert(positiveFinite(cmd.rs2(31, 0)), "Int8MulBall ratio must be finite and positive") - assert( - cmd.rs2(31 + bankRowBits, 32) < b.memDomain.bankEntries.U, - "Int8MulBall gate_row must fit one physical bank" - ) - assert( - cmd.op1_col =/= 0.U && cmd.op1_col === cmd.op2_col && cmd.op1_col === cmd.wr_col, - "Int8MulBall bank groups must match" - ) - assert(cmd.op1_col <= b.memDomain.bankNum.U, "Int8MulBall bank groups exceed physical banks") - assert( - cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, - "Int8MulBall banks must be distinct" - ) - assert(cmd.iter =/= 0.U && cmd.iter <= b.memDomain.bankEntries.U, "Int8MulBall iter must fit one physical bank") - robId := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRobId := io.cmdReq.bits.sub_rob_id - gateBank := cmd.op1_bank - inputBank := cmd.op2_bank - outputBank := cmd.wr_bank - groups := cmd.op1_col - group := 0.U - iter := cmd.iter - gateRow := cmd.rs2(31 + bankRowBits, 32) - inputRow := 0.U - ratio := cmd.rs2(31, 0) - lane := 0.U - state := waitForChannels - } - } - is(waitForChannels) { - when(io.channelReady)(state := gateRequest) - } - is(gateRequest) { - io.bankRead(0).io.req.valid := true.B - when(io.bankRead(0).io.req.fire)(state := gateResponse) - } - is(gateResponse) { - io.bankRead(0).io.resp.ready := true.B - when(io.bankRead(0).io.resp.fire) { - gateWord := io.bankRead(0).io.resp.bits.data - state := inputRequest - } - } - is(inputRequest) { - io.bankRead(1).io.req.valid := true.B - when(io.bankRead(1).io.req.fire)(state := inputResponse) - } - is(inputResponse) { - io.bankRead(1).io.resp.ready := true.B - when(io.bankRead(1).io.resp.fire) { - inputWord := io.bankRead(1).io.resp.bits.data - lane := 0.U - state := calculate - } - } - is(calculate) { - productReg := integerProduct - productLaneReg := lane - productValid := true.B - when(lane === 15.U)(state := calculateDrain) - .otherwise(lane := lane + 1.U) - } - is(calculateDrain) { - when(scaledValid && scaledLaneReg === 15.U)(state := writeRequest) - } - is(writeRequest) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire)(state := writeResponse) - } - is(writeResponse) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - when(inputRow +& 1.U === iter) { - inputRow := 0.U - when(group +& 1.U === groups)(state := complete) - .otherwise { group := group + 1.U; state := gateRequest } - }.otherwise { inputRow := inputRow + 1.U; state := inputRequest } - } - } - is(complete) { - when(io.cmdResp.fire)(state := idle) - } - } - - when(productValid) { - productFloatReg := productToFloat.io.out - floatLaneReg := productLaneReg - } - when(scale.io.validout) { - scaledReg := scale.io.out - scaledLaneReg := Pipe(floatValid, floatLaneReg, 3).bits - } - when(scaledValid) { - outputWord(scaledLaneReg) := toInt8.io.out - } - - productValid := state === calculate - floatValid := productValid - scaledValid := scale.io.validout -} diff --git a/examples/balls/int8mul/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/int8mul/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index da185c78b..000000000 --- a/examples/balls/int8mul/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateInt8MulBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} - -namespace { -class BankInt8MulPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankInt8MulOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getGateBank(), op.getInputBank(), - op.getOutputBank(), op.getIter(), op.getRatio(), - op.getGateRow()); - rewriter.replaceOp(op, op.getOutputBank()); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateInt8MulBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/int8mul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/int8mul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index cbcfd4377..000000000 --- a/examples/balls/int8mul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct Int8MulLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(Int8MulOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("Int8MulBall"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getGateBankId(), - adaptor.getInputBankId(), - adaptor.getOutputBankId(), adaptor.getIter()); - Value ratioBits = rewriter.create( - loc, rewriter.getI32Type(), adaptor.getRatio()); - Value ratio = - rewriter.create(loc, rewriter.getI64Type(), ratioBits); - Value gateRow = rewriter.create(loc, adaptor.getGateRow(), - cstI64(rewriter, loc, 32)); - Value rs2 = rewriter.create(loc, ratio, gateRow); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("INT8MUL"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateInt8MulBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { - patterns.add(converter); -} - -void configureInt8MulBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/int8mul/configs/default.toml b/examples/balls/int8mul/configs/default.toml deleted file mode 100644 index b7bdf2941..000000000 --- a/examples/balls/int8mul/configs/default.toml +++ /dev/null @@ -1,3 +0,0 @@ -# Int8MulBall has no private configuration. - -[ball] diff --git a/examples/balls/int8mul/emu/src/dispatch.rs b/examples/balls/int8mul/emu/src/dispatch.rs deleted file mode 100644 index f6d4f818c..000000000 --- a/examples/balls/int8mul/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::ExecContext; - -use super::int8mul; - -const BALL_CLASS: &str = "examples.balls.int8mul.Int8MulBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("INT8MUL") => Some(int8mul::Int8Mul::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("INT8MUL") => Some(int8mul::Int8Mul::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/int8mul/emu/src/lib.rs b/examples/balls/int8mul/emu/src/lib.rs deleted file mode 100644 index 7a691a4aa..000000000 --- a/examples/balls/int8mul/emu/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod int8mul; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/int8mul/verify/casegen/src/casegen.rs b/examples/balls/int8mul/verify/casegen/src/casegen.rs deleted file mode 100644 index c42ddd88a..000000000 --- a/examples/balls/int8mul/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,456 +0,0 @@ -pub const ROW_BYTES: usize = 16; -pub const MAX_ROWS: usize = 64; -pub const NUM_CASES: u32 = 3; -pub const GATE_ROW: u32 = 1; -pub const RATIO: f32 = 0.25; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Int8MulCmd { - pub bid: u32, - pub iter: u32, - pub gate_bank: u32, - pub input_bank: u32, - pub output_bank: u32, - pub op1_col: u32, - pub op2_col: u32, - pub wr_col: u32, - pub gate_row: u32, - pub bank_entries: u32, - pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, - pub rs2_lo: u32, - pub rs2_hi: u32, - pub num_gate_words: u32, - pub num_input_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Int8MulCase { - pub cmd: Int8MulCmd, - pub gate_words: [u128; MAX_ROWS], - pub input_words: [u128; MAX_ROWS], - pub dst_words: [u128; MAX_ROWS], -} - -impl Int8MulCase { - pub fn gate_lo(&self, index: usize) -> u64 { - self.gate_words[index] as u64 - } - - pub fn gate_hi(&self, index: usize) -> u64 { - (self.gate_words[index] >> 64) as u64 - } - - pub fn input_lo(&self, index: usize) -> u64 { - self.input_words[index] as u64 - } - - pub fn input_hi(&self, index: usize) -> u64 { - (self.input_words[index] >> 64) as u64 - } - - pub fn dst_lo(&self, index: usize) -> u64 { - self.dst_words[index] as u64 - } - - pub fn dst_hi(&self, index: usize) -> u64 { - (self.dst_words[index] >> 64) as u64 - } -} - -pub fn gen_case(index: u32, bid: u32, bank_entries: u32) -> Int8MulCase { - if bank_entries == 0 { - panic!("int8mul: bank entries must be positive"); - } - if index >= NUM_CASES { - panic!("int8mul: unsupported directed case {index}"); - } - match index { - 0 => directed(bid, 2, 1, GATE_ROW, bank_entries), - 1 => directed(bid, 3, 4, GATE_ROW, bank_entries), - 2 => directed(bid, 4, 1, bank_entries - 1, bank_entries), - _ => panic!("int8mul: unsupported directed case {index}"), - } -} - -fn compute(gate: i8, input: i8, ratio: f32) -> i8 { - let value = (gate as f32) * (input as f32) * ratio; - value.round_ties_even().clamp(-128.0, 127.0) as i8 -} - -fn positive_finite(value: f32) -> bool { - value.is_finite() && value > 0.0 -} - -fn ctest_gate() -> u128 { - let mut row = [0u8; ROW_BYTES]; - for lane in 0..ROW_BYTES { - row[lane] = (lane as i8 - 8) as u8; - } - pack_bytes(&row)[0] -} - -fn ctest_input(rows: usize) -> Vec { - let count = rows * ROW_BYTES; - let bytes: Vec = (0..count) - .map(|i| ((i as i32 * 37 & 255) - 128) as i8 as u8) - .collect(); - pack_bytes(&bytes) -} - -fn pack_bytes(bytes: &[u8]) -> Vec { - if bytes.len() % ROW_BYTES != 0 { - panic!( - "int8mul: {} bytes is not a multiple of {ROW_BYTES}", - bytes.len() - ); - } - bytes - .chunks(ROW_BYTES) - .map(|chunk| { - let mut word = 0u128; - for (lane, byte) in chunk.iter().enumerate() { - word |= u128::from(*byte) << (lane * 8); - } - word - }) - .collect() -} - -fn copy_words(src: &[u128]) -> [u128; N] { - if src.len() > N { - panic!("int8mul: {} words exceed buffer {N}", src.len()); - } - let mut out = [0u128; N]; - out[..src.len()].copy_from_slice(src); - out -} - -fn encode_rs1(gate: u32, input: u32, output: u32, iter: u32) -> u64 { - u64::from(gate) | (u64::from(input) << 10) | (u64::from(output) << 20) | (u64::from(iter) << 30) -} - -fn encode_rs2(ratio: f32, gate_row: u32) -> u64 { - u64::from(ratio.to_bits()) | (u64::from(gate_row) << 32) -} - -fn split_rs(value: u64) -> (u32, u32) { - (value as u32, (value >> 32) as u32) -} - -pub(crate) fn validate( - gate_bank: u32, - input_bank: u32, - output_bank: u32, - op1_col: u32, - op2_col: u32, - wr_col: u32, - iter: u32, - gate_row: u32, - rs1: u64, - rs2: u64, - ratio: f32, - bank_entries: u32, -) { - if bank_entries == 0 { - panic!("int8mul: bank entries must be positive"); - } - if iter == 0 || iter > bank_entries { - panic!("int8mul: iter must fit one physical bank"); - } - if gate_row >= bank_entries { - panic!("int8mul: gate row must fit one physical bank"); - } - if op1_col != 1 || op2_col != 1 || wr_col != 1 { - panic!("int8mul: bank groups must match"); - } - if gate_bank == input_bank || gate_bank == output_bank || input_bank == output_bank { - panic!("int8mul: banks must be distinct"); - } - if !positive_finite(ratio) { - panic!("int8mul: ratio must be finite and positive"); - } - let gate_row_bits = u32::BITS - (bank_entries - 1).leading_zeros(); - if rs2 >> (32 + gate_row_bits) != 0 { - panic!("int8mul: reserved rs2 bits must be zero"); - } - if (rs1 & 0x3ff) != u64::from(gate_bank) - || ((rs1 >> 10) & 0x3ff) != u64::from(input_bank) - || ((rs1 >> 20) & 0x3ff) != u64::from(output_bank) - { - panic!("int8mul: rs1 bank fields mismatch"); - } - if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { - panic!("int8mul: rs1 iter mismatch"); - } - if rs2 != encode_rs2(ratio, gate_row) { - panic!("int8mul: rs2 ratio/gate_row fields mismatch"); - } -} - -fn dst_words(gate_word: u128, input: &[u128], iter: u32, ratio: f32) -> Vec { - let mut out = Vec::with_capacity(iter as usize); - for row in 0..iter as usize { - let mut packed = 0u128; - for lane in 0..ROW_BYTES { - let g = ((gate_word >> (lane * 8)) & 0xff) as u8 as i8; - let inp = ((input[row] >> (lane * 8)) & 0xff) as u8 as i8; - let q = compute(g, inp, ratio); - packed |= u128::from(q as u8) << (lane * 8); - } - out.push(packed); - } - out -} - -fn directed(bid: u32, rob_id: u32, iter: u32, gate_row: u32, bank_entries: u32) -> Int8MulCase { - let gate_bank = 0; - let input_bank = 1; - let output_bank = 2; - let op1_col = 1; - let op2_col = 1; - let wr_col = 1; - let ratio = RATIO; - let gate = ctest_gate(); - let input = ctest_input(iter as usize); - let rs1 = encode_rs1(gate_bank, input_bank, output_bank, iter); - let rs2 = encode_rs2(ratio, gate_row); - validate( - gate_bank, - input_bank, - output_bank, - op1_col, - op2_col, - wr_col, - iter, - gate_row, - rs1, - rs2, - ratio, - bank_entries, - ); - if input.len() != iter as usize { - panic!("int8mul: input rows {} != iter {iter}", input.len()); - } - let dst = dst_words(gate, &input, iter, ratio); - let (rs1_lo, rs1_hi) = split_rs(rs1); - let (rs2_lo, rs2_hi) = split_rs(rs2); - Int8MulCase { - cmd: Int8MulCmd { - bid, - iter, - gate_bank, - input_bank, - output_bank, - op1_col, - op2_col, - wr_col, - gate_row, - bank_entries, - rob_id, - rs1_lo, - rs1_hi, - rs2_lo, - rs2_hi, - num_gate_words: 1, - num_input_words: input.len() as u32, - num_dst_words: dst.len() as u32, - }, - gate_words: copy_words(&[gate]), - input_words: copy_words(&input), - dst_words: copy_words(&dst), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - const TEST_BANK_ENTRIES: u32 = 4096; - - fn ctest_expected(gate: i8, input: i8) -> i8 { - compute(gate, input, RATIO) - } - - #[test] - fn case_zero_matches_ctest() { - let case = gen_case(0, 8, TEST_BANK_ENTRIES); - assert_eq!(case.cmd.bid, 8); - assert_eq!(case.cmd.iter, 1); - assert_eq!(case.cmd.gate_bank, 0); - assert_eq!(case.cmd.input_bank, 1); - assert_eq!(case.cmd.output_bank, 2); - assert_eq!(case.cmd.gate_row, GATE_ROW); - assert_eq!(case.cmd.bank_entries, TEST_BANK_ENTRIES); - assert_eq!(case.cmd.num_gate_words, 1); - assert_eq!(case.cmd.num_input_words, 1); - assert_eq!(case.cmd.num_dst_words, 1); - let gate_lane0 = (case.gate_words[0] & 0xff) as u8 as i8; - let input0 = ((case.input_words[0] >> 0) & 0xff) as u8 as i8; - assert_eq!(gate_lane0, -8); - assert_eq!(input0, -128); - let got0 = (case.dst_words[0] & 0xff) as u8 as i8; - assert_eq!(got0, ctest_expected(gate_lane0, input0)); - } - - #[test] - fn cases_hit_iter_bins() { - let mut hit = [false; 2]; - for index in 0..NUM_CASES { - let case = gen_case(index, 8, TEST_BANK_ENTRIES); - assert_eq!(case.cmd.bid, 8); - assert_eq!(case.cmd.op1_col, 1); - assert_eq!(case.cmd.op2_col, 1); - assert_eq!(case.cmd.wr_col, 1); - assert_eq!(case.cmd.num_input_words, case.cmd.iter); - assert_eq!(case.cmd.num_dst_words, case.cmd.iter); - let rs2 = u64::from(case.cmd.rs2_lo) | (u64::from(case.cmd.rs2_hi) << 32); - assert_eq!(rs2 >> (32 + TEST_BANK_ENTRIES.ilog2()), 0); - match case.cmd.iter { - 1 => hit[0] = true, - 4 => hit[1] = true, - other => panic!("unexpected iter {other}"), - } - } - assert!(hit.iter().all(|h| *h)); - } - - #[test] - fn final_gate_row_tracks_bank_entries() { - let small = gen_case(2, 8, 64); - let non_power_of_two = gen_case(2, 8, 65); - let large = gen_case(2, 8, TEST_BANK_ENTRIES); - assert_eq!(small.cmd.gate_row, 63); - assert_eq!(non_power_of_two.cmd.gate_row, 64); - assert_eq!(large.cmd.gate_row, TEST_BANK_ENTRIES - 1); - let small_rs2 = u64::from(small.cmd.rs2_lo) | (u64::from(small.cmd.rs2_hi) << 32); - let non_power_rs2 = - u64::from(non_power_of_two.cmd.rs2_lo) | (u64::from(non_power_of_two.cmd.rs2_hi) << 32); - let large_rs2 = u64::from(large.cmd.rs2_lo) | (u64::from(large.cmd.rs2_hi) << 32); - assert_eq!(small_rs2 >> 38, 0); - assert_eq!(non_power_rs2 >> 39, 0); - assert_eq!(large_rs2 >> 44, 0); - } - - #[test] - #[should_panic(expected = "unsupported directed case")] - fn unknown_index_panics() { - let _ = gen_case(NUM_CASES, 8, TEST_BANK_ENTRIES); - } - - #[test] - #[should_panic(expected = "iter must fit")] - fn illegal_iter_panics() { - validate( - 0, - 1, - 2, - 1, - 1, - 1, - 0, - GATE_ROW, - encode_rs1(0, 1, 2, 0), - encode_rs2(RATIO, GATE_ROW), - RATIO, - TEST_BANK_ENTRIES, - ); - } - - #[test] - #[should_panic(expected = "gate row must fit")] - fn illegal_gate_row_panics() { - validate( - 0, - 1, - 2, - 1, - 1, - 1, - 1, - TEST_BANK_ENTRIES, - encode_rs1(0, 1, 2, 1), - encode_rs2(RATIO, TEST_BANK_ENTRIES), - RATIO, - TEST_BANK_ENTRIES, - ); - } - - #[test] - #[should_panic(expected = "banks must be distinct")] - fn overlapping_banks_panics() { - validate( - 0, - 0, - 2, - 1, - 1, - 1, - 1, - GATE_ROW, - encode_rs1(0, 0, 2, 1), - encode_rs2(RATIO, GATE_ROW), - RATIO, - TEST_BANK_ENTRIES, - ); - } - - #[test] - #[should_panic(expected = "finite and positive")] - fn nonpositive_ratio_panics() { - validate( - 0, - 1, - 2, - 1, - 1, - 1, - 1, - GATE_ROW, - encode_rs1(0, 1, 2, 1), - encode_rs2(0.0, GATE_ROW), - 0.0, - TEST_BANK_ENTRIES, - ); - } - - #[test] - fn gate_row_1024_uses_configured_address_bits() { - validate( - 0, - 1, - 2, - 1, - 1, - 1, - 1, - 1024, - encode_rs1(0, 1, 2, 1), - encode_rs2(RATIO, 1024), - RATIO, - TEST_BANK_ENTRIES, - ); - } - - #[test] - #[should_panic(expected = "reserved rs2 bits")] - fn reserved_rs2_panics() { - validate( - 0, - 1, - 2, - 1, - 1, - 1, - 1, - GATE_ROW, - encode_rs1(0, 1, 2, 1), - 1_u64 << (32 + TEST_BANK_ENTRIES.ilog2()), - RATIO, - TEST_BANK_ENTRIES, - ); - } -} diff --git a/examples/balls/int8mul/verify/casegen/src/lib.rs b/examples/balls/int8mul/verify/casegen/src/lib.rs deleted file mode 100644 index 9432937af..000000000 --- a/examples/balls/int8mul/verify/casegen/src/lib.rs +++ /dev/null @@ -1,130 +0,0 @@ -mod casegen; -mod config; - -use std::cell::RefCell; - -use casegen::{Int8MulCase, Int8MulCmd}; - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -#[no_mangle] -pub extern "C" fn int8mul_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid, config::bank_entries()); - CURRENT.with(|current| *current.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|current| match current.borrow().as_ref() { - Some(case) => f(case), - None => panic!("int8mul_case: no case loaded; call int8mul_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn int8mul_case_cmd(out_ptr: *mut Int8MulCmd) { - if out_ptr.is_null() { - panic!("int8mul_case_cmd: null out_ptr"); - } - current_case(|case| unsafe { - *out_ptr = case.cmd; - }); -} - -fn idx(index: u32, limit: u32, what: &str) -> usize { - if index >= limit { - panic!("int8mul_case_{what}: index {index} out of range {limit}"); - } - index as usize -} - -#[no_mangle] -pub extern "C" fn int8mul_case_gate_word_lo(word_index: u32) -> u64 { - current_case(|case| case.gate_lo(idx(word_index, case.cmd.num_gate_words, "gate_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn int8mul_case_gate_word_hi(word_index: u32) -> u64 { - current_case(|case| case.gate_hi(idx(word_index, case.cmd.num_gate_words, "gate_word_hi"))) -} - -#[no_mangle] -pub extern "C" fn int8mul_case_input_word_lo(word_index: u32) -> u64 { - current_case(|case| case.input_lo(idx(word_index, case.cmd.num_input_words, "input_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn int8mul_case_input_word_hi(word_index: u32) -> u64 { - current_case(|case| case.input_hi(idx(word_index, case.cmd.num_input_words, "input_word_hi"))) -} - -#[no_mangle] -pub extern "C" fn int8mul_case_dst_word_lo(word_index: u32) -> u64 { - current_case(|case| case.dst_lo(idx(word_index, case.cmd.num_dst_words, "dst_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn int8mul_case_dst_word_hi(word_index: u32) -> u64 { - current_case(|case| case.dst_hi(idx(word_index, case.cmd.num_dst_words, "dst_word_hi"))) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn configure_bank_entries(entries: u32) { - let path = std::env::temp_dir().join(format!( - "int8mul-verify-config-{}-{entries}", - std::process::id() - )); - std::fs::write(&path, format!("bank_entries={entries}\n")).unwrap(); - std::env::set_var("BB_VERIFY_CONFIG", path); - } - - #[test] - fn dpi_load_then_cmd_and_words() { - configure_bank_entries(4096); - assert_eq!(int8mul_case_load(0, 8), 0); - let mut cmd = Int8MulCmd { - bid: 0, - iter: 0, - gate_bank: 0, - input_bank: 0, - output_bank: 0, - op1_col: 0, - op2_col: 0, - wr_col: 0, - gate_row: 0, - bank_entries: 0, - rob_id: 0, - rs1_lo: 0, - rs1_hi: 0, - rs2_lo: 0, - rs2_hi: 0, - num_gate_words: 0, - num_input_words: 0, - num_dst_words: 0, - }; - int8mul_case_cmd(&mut cmd as *mut Int8MulCmd); - assert_eq!(cmd.bid, 8); - assert_eq!(cmd.iter, 1); - assert_eq!(cmd.gate_row, 1); - assert_eq!(cmd.bank_entries, 4096); - assert_eq!(cmd.num_gate_words, 1); - assert_eq!(cmd.num_input_words, 1); - assert_eq!(cmd.num_dst_words, 1); - let _ = int8mul_case_gate_word_lo(0); - let _ = int8mul_case_input_word_lo(0); - let _ = int8mul_case_dst_word_lo(0); - assert_eq!(casegen::NUM_CASES, 3); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/int8mul/verify/src/tb_top.sv b/examples/balls/int8mul/verify/src/tb_top.sv deleted file mode 100644 index 385256808..000000000 --- a/examples/balls/int8mul/verify/src/tb_top.sv +++ /dev/null @@ -1,101 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import int8mul_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - Int8MulBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) - ); - - initial begin - intf.clock = 1'b0; - intf.reset = 1'b1; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("int8mul_ball_test"); - end -endmodule diff --git a/examples/balls/int8mul/workloads/ctests/CMakeLists.txt b/examples/balls/int8mul/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 627f04523..000000000 --- a/examples/balls/int8mul/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -add_buckyball_ctests( - int8mul_per_channel_gate_test.c - int8mul_large_gate_row_test.c -) diff --git a/examples/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir b/examples/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir deleted file mode 100644 index d00b72e90..000000000 --- a/examples/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir +++ /dev/null @@ -1,41 +0,0 @@ -func.func private @check_result(memref<4x16xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %one_i8 = arith.constant 1 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %depth4 = arith.constant 4 : i64 - %depth1 = arith.constant 1 : i64 - %stride = arith.constant 1 : i64 - %ratio = arith.constant 1.0 : f32 - %gate_row = arith.constant 0 : i64 - %gate = memref.alloc() : memref<1x16xi8> - %input = memref.alloc() : memref<4x16xi8> - %output = memref.alloc() : memref<4x16xi8> - scf.for %lane = %zero to %sixteen step %one { - memref.store %one_i8, %gate[%zero, %lane] : memref<1x16xi8> - } - scf.for %row = %zero to %four step %one { - scf.for %lane = %zero to %sixteen step %one { - %value = arith.index_cast %lane : index to i8 - memref.store %value, %input[%row, %lane] : memref<4x16xi8> - } - } - %gate_bank = buckyball.bank_alloc - %input_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_gate = buckyball.bank_mvin %gate %gate_bank %depth1 %stride : memref<1x16xi8> i64 i64 i64 - %loaded_input = buckyball.bank_mvin %input %input_bank %depth4 %stride : memref<4x16xi8> i64 i64 i64 - buckyball.int8mul %loaded_gate, %loaded_input, %output_bank, %depth4, %ratio, %gate_row : i64 - %stored = buckyball.bank_mvout %output %output_bank %depth4 %stride : memref<4x16xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<4x16xi8>) -> () - buckyball.bank_release %loaded_gate : i64 - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %gate : memref<1x16xi8> - memref.dealloc %input : memref<4x16xi8> - memref.dealloc %output : memref<4x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in b/examples/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in deleted file mode 100644 index 8310fb5a6..000000000 --- a/examples/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in +++ /dev/null @@ -1,41 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %one_i8 = arith.constant 1 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %lanes = arith.constant @BB_BANK_I8_LANES@ : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %depth1 = arith.constant 1 : i64 - %stride = arith.constant 1 : i64 - %ratio = arith.constant 1.0 : f32 - %gate_row = arith.constant 0 : i64 - %gate = memref.alloc() : memref<1x@BB_BANK_I8_LANES@xi8> - %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - scf.for %lane = %zero to %lanes step %one { - memref.store %one_i8, %gate[%zero, %lane] : memref<1x@BB_BANK_I8_LANES@xi8> - } - scf.for %row = %zero to %rows step %one { - scf.for %lane = %zero to %lanes step %one { - %value = arith.index_cast %lane : index to i8 - memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - } - } - %gate_bank = buckyball.bank_alloc - %input_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_gate = buckyball.bank_mvin %gate %gate_bank %depth1 %stride : memref<1x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - %result = buckyball.bank_int8mul %loaded_gate %loaded_input %output_bank %depth %ratio %gate_row : i64 i64 i64 i64 f32 i64 - %stored = buckyball.bank_mvout %output %result %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %loaded_gate : i64 - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %gate : memref<1x@BB_BANK_I8_LANES@xi8> - memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/lut/arch/src/main/scala/LutBall.scala b/examples/balls/lut/arch/src/main/scala/LutBall.scala deleted file mode 100644 index aa7b395a4..000000000 --- a/examples/balls/lut/arch/src/main/scala/LutBall.scala +++ /dev/null @@ -1,165 +0,0 @@ -package examples.balls.lut - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.top.GlobalConfig - -@instantiable -class LutBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "LutBall") - .getOrElse(throw new IllegalArgumentException("LutBall not found in config")) - - private val funct = b.ballDomain.ballISA - .find(_.mnemonic == "LUT") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("LUT not found in ballISA")) - - require(b.memDomain.bankWidth == 128, "LutBall requires 128-bit bank rows") - require(b.memDomain.bankEntries >= 16, "LutBall requires sixteen LUT rows") - require(mapping.inBW == 2, "LutBall requires inBW=2") - require(mapping.outBW == 1, "LutBall requires outBW=1") - require((funct >> 4) == 4, "LUT must encode two reads and one write") - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - private val idle :: inputReq :: inputResp :: lutReq :: lutResp :: writeReq :: writeResp :: complete :: Nil = - Enum(8) - private val state = RegInit(idle) - private val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSub = RegInit(false.B) - private val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val inputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val lutBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val outputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val iter = RegInit(0.U(b.frontend.iter_len.W)) - private val inputRow = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val inputData = Reg(UInt(128.W)) - private val laneTable = RegInit(false.B) - private val lookupLane = RegInit(0.U(4.W)) - private val outputWord = Reg(Vec(16, UInt(8.W))) - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRobId - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - for (port <- 0 until 2) { - io.bankRead(port).rob_id := robId - io.bankRead(port).ball_id := 0.U - io.bankRead(port).group_id := 0.U - io.bankRead(port).io.req.valid := false.B - io.bankRead(port).io.req.bits.addr := 0.U - io.bankRead(port).io.resp.ready := false.B - } - io.bankRead(0).bank_id := inputBank - io.bankRead(1).bank_id := lutBank - io.bankRead(1).group_id := Mux(laneTable, lookupLane >> 2, 0.U) - - io.bankWrite(0).rob_id := robId - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := outputBank - io.bankWrite(0).group_id := 0.U - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := inputRow - io.bankWrite(0).io.req.bits.data := Cat(outputWord.reverse) - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - assert(cmd.funct7 === funct.U, "LutBall funct7 must be LUT") - assert(cmd.rs2 === 0.U, "LutBall reserves rs2") - assert(cmd.iter > 0.U && cmd.iter <= b.memDomain.bankEntries.U, "LutBall iter must fit in one bank") - assert(cmd.op1_bank < b.memDomain.virtualBankCount.U, "LutBall input bank is invalid") - assert(cmd.op2_bank < b.memDomain.virtualBankCount.U, "LutBall table bank is invalid") - assert(cmd.wr_bank < b.memDomain.virtualBankCount.U, "LutBall output bank is invalid") - assert( - cmd.op1_col === 1.U && (cmd.op2_col === 1.U || cmd.op2_col === 4.U) && cmd.wr_col === 1.U, - "LutBall requires col=1 input/output and col=1 or col=4 table" - ) - assert( - cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && - cmd.op2_bank =/= cmd.wr_bank, - "LutBall banks must be distinct" - ) - robId := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRobId := io.cmdReq.bits.sub_rob_id - inputBank := cmd.op1_bank - lutBank := cmd.op2_bank - outputBank := cmd.wr_bank - iter := cmd.iter - inputRow := 0.U - laneTable := cmd.op2_col === 4.U - lookupLane := 0.U - state := inputReq - } - } - is(inputReq) { - io.bankRead(0).io.req.valid := true.B - io.bankRead(0).io.req.bits.addr := inputRow - when(io.bankRead(0).io.req.fire)(state := inputResp) - } - is(inputResp) { - io.bankRead(0).io.resp.ready := true.B - when(io.bankRead(0).io.resp.fire) { - inputData := io.bankRead(0).io.resp.bits.data - lookupLane := 0.U - state := lutReq - } - } - is(lutReq) { - val inputByte = (inputData >> (lookupLane << 3))(7, 0) - io.bankRead(1).io.req.valid := true.B - io.bankRead(1).io.req.bits.addr := Mux( - laneTable, - Cat(lookupLane(1, 0), inputByte(7, 4)), - inputByte(7, 4) - ) - when(io.bankRead(1).io.req.fire)(state := lutResp) - } - is(lutResp) { - val inputByte = (inputData >> (lookupLane << 3))(7, 0) - io.bankRead(1).io.resp.ready := true.B - when(io.bankRead(1).io.resp.fire) { - outputWord(lookupLane) := - (io.bankRead(1).io.resp.bits.data >> (inputByte(3, 0) << 3))(7, 0) - when(lookupLane === 15.U)(state := writeReq) - .otherwise { lookupLane := lookupLane + 1.U; state := lutReq } - } - } - is(writeReq) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire)(state := writeResp) - } - is(writeResp) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - when(inputRow === iter - 1.U)(state := complete) - .otherwise { inputRow := inputRow + 1.U; state := inputReq } - } - } - is(complete) { - when(io.cmdResp.fire)(state := idle) - } - } -} diff --git a/examples/balls/lut/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/lut/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 818bc823c..000000000 --- a/examples/balls/lut/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,54 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace mlir::buddy; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateLutBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { -class BankLutPattern : public OpRewritePattern { -public: - BankLutPattern(MLIRContext *context, PhysicalBankState &state) - : OpRewritePattern(context), state(state) {} - - LogicalResult matchAndRewrite(BankLutOp op, - PatternRewriter &rewriter) const override { - auto input = state.getSlot(op.getInBank()); - auto lut = state.getSlot(op.getLutBank()); - auto output = state.getSlot(op.getOutBank()); - if (!input || !lut || !output) - return failure(); - if (input->row != 1 || input->col != 1 || output->row != 1 || - output->col != 1 || lut->row != 1 || (lut->col != 1 && lut->col != 4)) - return op.emitError( - "LUT requires col=1 input/output and col=1 or col=4 table"); - auto overlaps = [](const BankSlot &lhs, const BankSlot &rhs) { - return lhs.base < rhs.base + rhs.row * rhs.col && - rhs.base < lhs.base + lhs.row * lhs.col; - }; - if (overlaps(*input, *lut) || overlaps(*input, *output) || - overlaps(*lut, *output)) - return op.emitError("LUT bank groups must not overlap"); - rewriter.create(op.getLoc(), op.getInBank(), op.getLutBank(), - op.getOutBank(), op.getIter()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } - -private: - PhysicalBankState &state; -}; -} // namespace - -void mlir::buddy::populateLutBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, PhysicalBankState &state) { - patterns.add(patterns.getContext(), state); -} diff --git a/examples/balls/lut/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/lut/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index d45794738..000000000 --- a/examples/balls/lut/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,60 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct LutLowering : public ConvertOpToLLVMPattern { - LutLowering(LLVMTypeConverter &converter, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(LutOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("LutBall"); - auto iterOp = op.getIter().getDefiningOp(); - auto iterAttr = - iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); - if (!iterAttr || iterAttr.getInt() <= 0 || iterAttr.getInt() > bankDepth) - return op.emitError( - "LUT iter must be a positive constant within one bank"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - adaptor.getLutBankId(), - adaptor.getOutputBankId(), adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, cstI64(rewriter, loc, 0), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("LUT"))); - return success(); - } - -private: - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateLutBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns, - bool stable, - int64_t bankDepth) { - (void)stable; - - patterns.add(converter, bankDepth); -} - -void configureLutBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/lut/configs/default.toml b/examples/balls/lut/configs/default.toml deleted file mode 100644 index 289d8f7e3..000000000 --- a/examples/balls/lut/configs/default.toml +++ /dev/null @@ -1,3 +0,0 @@ -# LutBall has no private configuration parameters. - -[ball] diff --git a/examples/balls/lut/emu/src/dispatch.rs b/examples/balls/lut/emu/src/dispatch.rs deleted file mode 100644 index 4f2d604b7..000000000 --- a/examples/balls/lut/emu/src/dispatch.rs +++ /dev/null @@ -1,36 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::lut; - -const BALL_CLASS: &str = "examples.balls.lut.LutBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("LUT") => Some(lut::Lut::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("LUT") => Some(lut::Lut::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/lut/emu/src/lib.rs b/examples/balls/lut/emu/src/lib.rs deleted file mode 100644 index c774116e0..000000000 --- a/examples/balls/lut/emu/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod lut; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/lut/verify/casegen/src/casegen.rs b/examples/balls/lut/verify/casegen/src/casegen.rs deleted file mode 100644 index e6c9426db..000000000 --- a/examples/balls/lut/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,353 +0,0 @@ -pub const BYTES_PER_ROW: usize = 16; -pub const SHARED_TABLE_BYTES: usize = 256; -pub const LANE_TABLE_BYTES: usize = 16 * SHARED_TABLE_BYTES; -pub const MAX_SRC: usize = 8; -pub const MAX_LUT: usize = LANE_TABLE_BYTES / BYTES_PER_ROW; -pub const MAX_DST: usize = 8; -pub const NUM_CASES: u32 = 4; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct LutCmd { - pub bid: u32, - pub iter: u32, - pub op1_bank: u32, - pub op2_bank: u32, - pub wr_bank: u32, - pub op1_col: u32, - pub op2_col: u32, - pub wr_col: u32, - pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, - pub rs2_lo: u32, - pub rs2_hi: u32, - pub num_src_words: u32, - pub num_lut_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct LutCase { - pub cmd: LutCmd, - pub src_words: [u128; MAX_SRC], - pub lut_words: [u128; MAX_LUT], - pub dst_words: [u128; MAX_DST], -} - -impl LutCase { - pub fn src_lo(&self, index: usize) -> u64 { - self.src_words[index] as u64 - } - - pub fn src_hi(&self, index: usize) -> u64 { - (self.src_words[index] >> 64) as u64 - } - - pub fn lut_lo(&self, index: usize) -> u64 { - self.lut_words[index] as u64 - } - - pub fn lut_hi(&self, index: usize) -> u64 { - (self.lut_words[index] >> 64) as u64 - } - - pub fn dst_lo(&self, index: usize) -> u64 { - self.dst_words[index] as u64 - } - - pub fn dst_hi(&self, index: usize) -> u64 { - (self.dst_words[index] >> 64) as u64 - } -} - -pub fn gen_case(index: u32, bid: u32) -> LutCase { - if index >= NUM_CASES { - panic!("lut: unsupported directed case {index}"); - } - match index { - 0 => directed(bid, 2, 4, false), - 1 => directed(bid, 3, 1, false), - 2 => directed(bid, 4, 8, false), - 3 => directed(bid, 5, 4, true), - _ => panic!("lut: unsupported directed case {index}"), - } -} - -fn shared_table() -> Vec { - let mut table = vec![0u8; SHARED_TABLE_BYTES]; - for (i, slot) in table.iter_mut().enumerate() { - *slot = (((i * 73 + 19) ^ 0xa5) & 0xff) as u8; - } - table -} - -fn lane_table() -> Vec { - let mut table = vec![0u8; LANE_TABLE_BYTES]; - for channel in 0..16 { - for input in 0..256 { - table[channel * 256 + input] = ((input as i32 + channel as i32 * 17 - 128) as i8) as u8; - } - } - table -} - -fn pack_lane_table(table: &[u8]) -> Vec { - if table.len() != LANE_TABLE_BYTES { - panic!("lut: invalid lane table size {}", table.len()); - } - let mut packed = vec![0u8; LANE_TABLE_BYTES]; - for row in 0..64 { - for group in 0..4 { - for byte in 0..16 { - packed[group * 64 * 16 + row * 16 + byte] = - table[(group * 4 + row / 16) * 256 + (row % 16) * 16 + byte]; - } - } - } - packed -} - -fn ctest_input(nbytes: usize) -> Vec { - (0..nbytes) - .map(|i| (i as i32 * 29 - 128) as i8 as u8) - .collect() -} - -fn pack_bytes(bytes: &[u8]) -> Vec { - if bytes.len() % BYTES_PER_ROW != 0 { - panic!( - "lut: {} bytes is not a multiple of {BYTES_PER_ROW}", - bytes.len() - ); - } - bytes - .chunks(BYTES_PER_ROW) - .map(|chunk| { - let mut word = 0u128; - for (lane, byte) in chunk.iter().enumerate() { - word |= u128::from(*byte) << (lane * 8); - } - word - }) - .collect() -} - -fn copy_words(src: &[u128]) -> [u128; N] { - if src.len() > N { - panic!("lut: {} words exceed buffer {N}", src.len()); - } - let mut out = [0u128; N]; - out[..src.len()].copy_from_slice(src); - out -} - -fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { - u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) -} - -fn split_rs(value: u64) -> (u32, u32) { - (value as u32, (value >> 32) as u32) -} - -pub(crate) fn validate( - op1_bank: u32, - op2_bank: u32, - wr_bank: u32, - op1_col: u32, - op2_col: u32, - wr_col: u32, - iter: u32, - rs1: u64, - rs2: u64, -) { - if iter == 0 { - panic!("lut: iter must be positive"); - } - if op1_col != 1 || (op2_col != 1 && op2_col != 4) || wr_col != 1 { - panic!("lut: requires col=1 input/output and col=1 or col=4 table"); - } - if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { - panic!("lut: banks must be distinct"); - } - if (rs1 & 0x3ff) != u64::from(op1_bank) - || ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) - || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) - { - panic!("lut: rs1 bank fields mismatch"); - } - if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { - panic!("lut: rs1 iter mismatch"); - } - if rs2 != 0 { - panic!("lut: rs2 must be zero"); - } -} - -fn dst_words(table: &[u8], input: &[u8], lane_table: bool) -> Vec { - let mapped: Vec = input - .iter() - .enumerate() - .map(|(index, &byte)| { - let channel = index % 16; - table[(if lane_table { channel * 256 } else { 0 }) + byte as usize] - }) - .collect(); - pack_bytes(&mapped) -} - -fn directed(bid: u32, rob_id: u32, iter: u32, per_lane: bool) -> LutCase { - let nbytes = iter as usize * BYTES_PER_ROW; - let table = if per_lane { - lane_table() - } else { - shared_table() - }; - let input = ctest_input(nbytes); - let src = pack_bytes(&input); - let lut = if per_lane { - pack_bytes(&pack_lane_table(&table)) - } else { - pack_bytes(&table) - }; - let dst = dst_words(&table, &input, per_lane); - let op1_bank = 0; - let op2_bank = 1; - let wr_bank = 2; - let op1_col = 1; - let op2_col = if per_lane { 4 } else { 1 }; - let wr_col = 1; - let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); - let rs2 = 0u64; - validate( - op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs1, rs2, - ); - if src.len() != iter as usize { - panic!("lut: src rows {} != iter {iter}", src.len()); - } - let expected_lut_rows = if per_lane { MAX_LUT } else { 16 }; - if lut.len() != expected_lut_rows { - panic!("lut: table rows {} != {expected_lut_rows}", lut.len()); - } - if dst.len() != iter as usize { - panic!("lut: dst rows {} != iter {iter}", dst.len()); - } - let (rs1_lo, rs1_hi) = split_rs(rs1); - let (rs2_lo, rs2_hi) = split_rs(rs2); - LutCase { - cmd: LutCmd { - bid, - iter, - op1_bank, - op2_bank, - wr_bank, - op1_col, - op2_col, - wr_col, - rob_id, - rs1_lo, - rs1_hi, - rs2_lo, - rs2_hi, - num_src_words: src.len() as u32, - num_lut_words: lut.len() as u32, - num_dst_words: dst.len() as u32, - }, - src_words: copy_words(&src), - lut_words: copy_words(&lut), - dst_words: copy_words(&dst), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn case_zero_matches_ctest() { - let case = gen_case(0, 5); - assert_eq!(case.cmd.iter, 4); - assert_eq!(case.cmd.bid, 5); - assert_eq!(case.cmd.num_src_words, 4); - assert_eq!(case.cmd.num_lut_words, 16); - assert_eq!(case.cmd.num_dst_words, 4); - let table = shared_table(); - let input = ctest_input(64); - assert_eq!(table[0], ((19 ^ 0xa5) & 0xff) as u8); - assert_eq!(input[0], (-128i8) as u8); - let exp0 = table[input[0] as usize]; - assert_eq!(case.dst_words[0] as u8, exp0); - for i in 0..16 { - let got = ((case.src_words[0] >> (i * 8)) & 0xff) as u8; - assert_eq!(got, input[i]); - } - for i in 0..16 { - let got = ((case.lut_words[0] >> (i * 8)) & 0xff) as u8; - assert_eq!(got, table[i]); - } - } - - #[test] - fn cases_hit_iter_bins() { - let mut hit = [false; 3]; - for index in 0..NUM_CASES { - let case = gen_case(index, 5); - assert_eq!(case.cmd.bid, 5); - assert_eq!(case.cmd.op1_col, 1); - assert!(case.cmd.op2_col == 1 || case.cmd.op2_col == 4); - assert_eq!(case.cmd.wr_col, 1); - assert_eq!(case.cmd.rs2_lo, 0); - assert_eq!(case.cmd.rs2_hi, 0); - assert_eq!(case.cmd.num_src_words, case.cmd.iter); - assert_eq!(case.cmd.num_dst_words, case.cmd.iter); - assert_eq!( - case.cmd.num_lut_words, - if case.cmd.op2_col == 4 { 256 } else { 16 } - ); - match case.cmd.iter { - 1 => hit[0] = true, - 4 => hit[1] = true, - 8 => hit[2] = true, - other => panic!("unexpected iter {other}"), - } - } - assert!(hit.iter().all(|h| *h)); - } - - #[test] - fn lane_case_selects_channel_table() { - let case = gen_case(3, 5); - let input = ctest_input(64); - let table = lane_table(); - assert_eq!(case.cmd.op2_col, 4); - assert_eq!(case.cmd.num_lut_words, 256); - for lane in 0..16 { - let got = ((case.dst_words[0] >> (lane * 8)) & 0xff) as u8; - assert_eq!(got, table[lane * 256 + input[lane] as usize]); - } - } - - #[test] - #[should_panic(expected = "unsupported directed case")] - fn unknown_index_panics() { - let _ = gen_case(NUM_CASES, 5); - } - - #[test] - #[should_panic(expected = "iter must be positive")] - fn illegal_iter_panics() { - validate(0, 1, 2, 1, 1, 1, 0, encode_rs1(0, 1, 2, 0), 0); - } - - #[test] - #[should_panic(expected = "banks must be distinct")] - fn overlapping_banks_panics() { - validate(0, 0, 2, 1, 1, 1, 4, encode_rs1(0, 0, 2, 4), 0); - } - - #[test] - #[should_panic(expected = "rs2 must be zero")] - fn reserved_rs2_panics() { - validate(0, 1, 2, 1, 1, 1, 4, encode_rs1(0, 1, 2, 4), 1); - } -} diff --git a/examples/balls/lut/verify/casegen/src/lib.rs b/examples/balls/lut/verify/casegen/src/lib.rs deleted file mode 100644 index 257e15998..000000000 --- a/examples/balls/lut/verify/casegen/src/lib.rs +++ /dev/null @@ -1,115 +0,0 @@ -mod casegen; - -use std::cell::RefCell; - -use casegen::{LutCase, LutCmd}; - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -#[no_mangle] -pub extern "C" fn lut_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid); - CURRENT.with(|current| *current.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|current| match current.borrow().as_ref() { - Some(case) => f(case), - None => panic!("lut_case: no case loaded; call lut_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn lut_case_cmd(out_ptr: *mut LutCmd) { - if out_ptr.is_null() { - panic!("lut_case_cmd: null out_ptr"); - } - current_case(|case| unsafe { - *out_ptr = case.cmd; - }); -} - -fn src_index(index: u32, limit: u32, what: &str) -> usize { - if index >= limit { - panic!("lut_case_{what}: index {index} out of range {limit}"); - } - index as usize -} - -#[no_mangle] -pub extern "C" fn lut_case_src_word_lo(word_index: u32) -> u64 { - current_case(|case| case.src_lo(src_index(word_index, case.cmd.num_src_words, "src_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn lut_case_src_word_hi(word_index: u32) -> u64 { - current_case(|case| case.src_hi(src_index(word_index, case.cmd.num_src_words, "src_word_hi"))) -} - -#[no_mangle] -pub extern "C" fn lut_case_lut_word_lo(word_index: u32) -> u64 { - current_case(|case| case.lut_lo(src_index(word_index, case.cmd.num_lut_words, "lut_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn lut_case_lut_word_hi(word_index: u32) -> u64 { - current_case(|case| case.lut_hi(src_index(word_index, case.cmd.num_lut_words, "lut_word_hi"))) -} - -#[no_mangle] -pub extern "C" fn lut_case_dst_word_lo(word_index: u32) -> u64 { - current_case(|case| case.dst_lo(src_index(word_index, case.cmd.num_dst_words, "dst_word_lo"))) -} - -#[no_mangle] -pub extern "C" fn lut_case_dst_word_hi(word_index: u32) -> u64 { - current_case(|case| case.dst_hi(src_index(word_index, case.cmd.num_dst_words, "dst_word_hi"))) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dpi_load_then_cmd_and_words() { - assert_eq!(lut_case_load(0, 5), 0); - let mut cmd = LutCmd { - bid: 0, - iter: 0, - op1_bank: 0, - op2_bank: 0, - wr_bank: 0, - op1_col: 0, - op2_col: 0, - wr_col: 0, - rob_id: 0, - rs1_lo: 0, - rs1_hi: 0, - rs2_lo: 0, - rs2_hi: 0, - num_src_words: 0, - num_lut_words: 0, - num_dst_words: 0, - }; - lut_case_cmd(&mut cmd as *mut LutCmd); - assert_eq!(cmd.bid, 5); - assert_eq!(cmd.iter, 4); - assert_eq!(cmd.num_src_words, 4); - assert_eq!(cmd.num_lut_words, 16); - assert_eq!(cmd.num_dst_words, 4); - let _ = lut_case_src_word_lo(0); - let _ = lut_case_lut_word_lo(0); - let _ = lut_case_dst_word_lo(0); - assert_eq!(casegen::NUM_CASES, 4); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/lut/verify/src/tb_top.sv b/examples/balls/lut/verify/src/tb_top.sv deleted file mode 100644 index decec36b3..000000000 --- a/examples/balls/lut/verify/src/tb_top.sv +++ /dev/null @@ -1,101 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import lut_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - LutBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) - ); - - initial begin - intf.clock = 1'b0; - intf.reset = 1'b1; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("lut_ball_test"); - end -endmodule diff --git a/examples/balls/lut/workloads/ctests/CMakeLists.txt b/examples/balls/lut/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 9c489f467..000000000 --- a/examples/balls/lut/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_buckyball_ctests(lut_uniform_lookup_test.c lut_channel_table_lookup_test.c) diff --git a/examples/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir b/examples/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir deleted file mode 100644 index f5a1d4bd4..000000000 --- a/examples/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir +++ /dev/null @@ -1,51 +0,0 @@ -func.func private @check_result(memref<4x16xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %depth4 = arith.constant 4 : i64 - %depth16 = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %sixteen_i32 = arith.constant 16 : i32 - %input = memref.alloc() : memref<4x16xi8> - %table = memref.alloc() : memref<16x16xi8> - %output = memref.alloc() : memref<4x16xi8> - scf.for %row = %zero to %sixteen step %one { - scf.for %lane = %zero to %sixteen step %one { - %row32 = arith.index_cast %row : index to i32 - %lane32 = arith.index_cast %lane : index to i32 - %base = arith.muli %row32, %sixteen_i32 : i32 - %index = arith.addi %base, %lane32 : i32 - %value = arith.trunci %index : i32 to i8 - memref.store %value, %table[%row, %lane] : memref<16x16xi8> - } - } - scf.for %row = %zero to %four step %one { - scf.for %lane = %zero to %sixteen step %one { - %row32 = arith.index_cast %row : index to i32 - %lane32 = arith.index_cast %lane : index to i32 - %base = arith.muli %row32, %sixteen_i32 : i32 - %index = arith.addi %base, %lane32 : i32 - %value = arith.trunci %index : i32 to i8 - memref.store %value, %input[%row, %lane] : memref<4x16xi8> - } - } - %input_bank = buckyball.bank_alloc - %table_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth4 %stride : memref<4x16xi8> i64 i64 i64 - %loaded_table = buckyball.bank_mvin %table %table_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - buckyball.lut %loaded_input, %loaded_table, %output_bank, %depth4 : i64 - %stored = buckyball.bank_mvout %output %output_bank %depth4 %stride : memref<4x16xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<4x16xi8>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_table : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<4x16xi8> - memref.dealloc %table : memref<16x16xi8> - memref.dealloc %output : memref<4x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in b/examples/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in deleted file mode 100644 index dc1c8a007..000000000 --- a/examples/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in +++ /dev/null @@ -1,53 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %lanes = arith.constant @BB_BANK_I8_LANES@ : index - %sixteen = arith.constant 16 : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %table_depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %lanes_i32 = arith.constant @BB_BANK_I8_LANES@ : i32 - %sixteen_i32 = arith.constant 16 : i32 - %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %table = memref.alloc() : memref<16x16xi8> - %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - scf.for %row = %zero to %sixteen step %one { - scf.for %lane = %zero to %sixteen step %one { - %row32 = arith.index_cast %row : index to i32 - %lane32 = arith.index_cast %lane : index to i32 - %base = arith.muli %row32, %sixteen_i32 : i32 - %index = arith.addi %base, %lane32 : i32 - %value = arith.trunci %index : i32 to i8 - memref.store %value, %table[%row, %lane] : memref<16x16xi8> - } - } - scf.for %row = %zero to %rows step %one { - scf.for %lane = %zero to %lanes step %one { - %row32 = arith.index_cast %row : index to i32 - %lane32 = arith.index_cast %lane : index to i32 - %base = arith.muli %row32, %lanes_i32 : i32 - %index = arith.addi %base, %lane32 : i32 - %value = arith.trunci %index : i32 to i8 - memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - } - } - %input_bank = buckyball.bank_alloc - %table_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - %loaded_table = buckyball.bank_mvin %table %table_bank %table_depth %stride : memref<16x16xi8> i64 i64 i64 - %result = buckyball.bank_lut %loaded_input %loaded_table %output_bank %depth : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_table : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.dealloc %table : memref<16x16xi8> - memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/matadd/arch/src/main/scala/MatAdd.scala b/examples/balls/matadd/arch/src/main/scala/MatAdd.scala deleted file mode 100644 index c6c0d678c..000000000 --- a/examples/balls/matadd/arch/src/main/scala/MatAdd.scala +++ /dev/null @@ -1,202 +0,0 @@ -package examples.balls.matadd - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.top.GlobalConfig - -@instantiable -class MatAdd(val b: GlobalConfig) extends Module { - private val addressWidth = log2Up(b.memDomain.bankEntries) - private val countWidth = log2Up(b.memDomain.bankEntries + 1) - private val bankIdWidth = b.memDomain.vbankIdWidth - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "MatAddBall") - .getOrElse(throw new IllegalArgumentException("MatAddBall not found in config")) - - private val funct = b.ballDomain.ballISA - .find(_.mnemonic == "MATADD") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("MATADD not found in ballISA")) - - require(mapping.inBW == 2, "MatAddBall requires two SRAM read ports") - require(mapping.outBW == 1, "MatAddBall requires one SRAM write port") - require(b.memDomain.bankWidth == 128, "MatAddBall requires 128-bit SRAM rows") - require(b.memDomain.bankMaskLen == 16, "MatAddBall requires sixteen byte enables") - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val channelReady = Input(Bool()) - val bankRead = Vec(2, Flipped(new BankRead(b))) - val bankWrite = Vec(1, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - val Seq(idle, waitForChannels, readRequest, readResponse, writeRequest, writeResponse, complete) = Enum(7) - val state = RegInit(idle) - - val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val isSub = RegInit(false.B) - val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - val aBank = RegInit(0.U(bankIdWidth.W)) - val bBank = RegInit(0.U(bankIdWidth.W)) - val cBank = RegInit(0.U(bankIdWidth.W)) - val groups = RegInit(0.U(5.W)) - val group = RegInit(0.U(5.W)) - val iter = RegInit(0.U(countWidth.W)) - val line = RegInit(0.U(addressWidth.W)) - val aWord = Reg(UInt(128.W)) - val bWord = Reg(UInt(128.W)) - val aRequestSent = RegInit(false.B) - val bRequestSent = RegInit(false.B) - val aReceived = RegInit(false.B) - val bReceived = RegInit(false.B) - - val sum = Wire(Vec(4, UInt(32.W))) - for (lane <- 0 until 4) { - sum(lane) := (aWord(32 * lane + 31, 32 * lane) +& - bWord(32 * lane + 31, 32 * lane))(31, 0) - } - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRobId - - for (port <- 0 until 2) { - io.bankRead(port).rob_id := robId - io.bankRead(port).ball_id := 0.U - io.bankRead(port).bank_id := (if (port == 0) aBank else bBank) - io.bankRead(port).group_id := group - io.bankRead(port).io.req.valid := false.B - io.bankRead(port).io.req.bits.addr := line - io.bankRead(port).io.resp.ready := false.B - } - - io.bankWrite(0).rob_id := robId - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := cBank - io.bankWrite(0).group_id := group - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := line - io.bankWrite(0).io.req.bits.data := Cat(sum.reverse) - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val command = io.cmdReq.bits.cmd - assert(command.funct7 === funct.U(7.W), "MatAddBall funct7 must be MATADD") - assert(command.op1_en && command.op2_en && command.wr_spad_en, "MatAddBall requires two inputs and one output") - assert(command.op1_bank < b.memDomain.virtualBankCount.U, "MatAddBall input bank 0 is invalid") - assert(command.op2_bank < b.memDomain.virtualBankCount.U, "MatAddBall input bank 1 is invalid") - assert(command.wr_bank < b.memDomain.virtualBankCount.U, "MatAddBall output bank is invalid") - assert( - command.op1_bank =/= command.op2_bank && command.op1_bank =/= command.wr_bank && - command.op2_bank =/= command.wr_bank, - "MatAddBall banks must be distinct" - ) - assert( - command.op1_col =/= 0.U && command.op1_col === command.op2_col && - command.op1_col === command.wr_col, - "MatAddBall bank groups must match" - ) - assert(command.op1_col <= b.memDomain.bankNum.U, "MatAddBall bank groups must fit in physical banks") - assert( - command.iter =/= 0.U && command.iter <= b.memDomain.bankEntries.U, - "MatAddBall iter must fit in one physical bank" - ) - assert(command.special === 0.U, "MatAddBall reserves rs2") - - robId := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRobId := io.cmdReq.bits.sub_rob_id - aBank := command.op1_bank - bBank := command.op2_bank - cBank := command.wr_bank - groups := command.op1_col - group := 0.U - iter := command.iter - line := 0.U - aRequestSent := false.B - bRequestSent := false.B - aReceived := false.B - bReceived := false.B - state := waitForChannels - } - } - - is(waitForChannels) { - when(io.channelReady)(state := readRequest) - } - - is(readRequest) { - io.bankRead(0).io.req.valid := !aRequestSent - io.bankRead(1).io.req.valid := !bRequestSent - when(io.bankRead(0).io.req.fire)(aRequestSent := true.B) - when(io.bankRead(1).io.req.fire)(bRequestSent := true.B) - when((aRequestSent || io.bankRead(0).io.req.fire) && - (bRequestSent || io.bankRead(1).io.req.fire)) { - state := readResponse - } - } - - is(readResponse) { - io.bankRead(0).io.resp.ready := !aReceived - io.bankRead(1).io.resp.ready := !bReceived - when(io.bankRead(0).io.resp.fire) { - aWord := io.bankRead(0).io.resp.bits.data - aReceived := true.B - } - when(io.bankRead(1).io.resp.fire) { - bWord := io.bankRead(1).io.resp.bits.data - bReceived := true.B - } - when((aReceived || io.bankRead(0).io.resp.fire) && - (bReceived || io.bankRead(1).io.resp.fire)) { - state := writeRequest - } - } - - is(writeRequest) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire)(state := writeResponse) - } - - is(writeResponse) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - aRequestSent := false.B - bRequestSent := false.B - aReceived := false.B - bReceived := false.B - when(line +& 1.U === iter) { - line := 0.U - when(group +& 1.U === groups) { - state := complete - }.otherwise { - group := group + 1.U - state := readRequest - } - }.otherwise { - line := line + 1.U - state := readRequest - } - } - } - - is(complete) { - when(io.cmdResp.fire)(state := idle) - } - } - - io.status.idle := state === idle - io.status.running := state =/= idle -} diff --git a/examples/balls/matadd/arch/src/main/scala/MatAddBall.scala b/examples/balls/matadd/arch/src/main/scala/MatAddBall.scala deleted file mode 100644 index 4193186da..000000000 --- a/examples/balls/matadd/arch/src/main/scala/MatAddBall.scala +++ /dev/null @@ -1,34 +0,0 @@ -package examples.balls.matadd - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.MmioRead -import framework.top.GlobalConfig - -@instantiable -class MatAddBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "MatAddBall") - .getOrElse(throw new IllegalArgumentException("MatAddBall not found in config")) - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - - def blink: BlinkIO = io - def status = io.status - dontTouch(io) - - val unit: Instance[MatAdd] = Instantiate(new MatAdd(b)) - unit.io.cmdReq <> io.cmdReq - unit.io.cmdResp <> io.cmdResp - unit.io.channelReady := io.channelReady - unit.io.bankRead(0) <> io.bankRead(0) - unit.io.bankRead(1) <> io.bankRead(1) - unit.io.bankWrite(0) <> io.bankWrite(0) - io.status <> unit.io.status - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/matadd/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/matadd/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 6040e0a1f..000000000 --- a/examples/balls/matadd/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateMatAddBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} - -namespace { -class BankMatAddPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankMatAddOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getABank(), op.getBBank(), - op.getCBank(), op.getIter()); - rewriter.replaceOp(op, op.getCBank()); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateMatAddBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/matadd/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/matadd/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 8029495ce..000000000 --- a/examples/balls/matadd/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct MatAddLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(MatAddOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("MatAddBall"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getABankId(), - adaptor.getBBankId(), adaptor.getCBankId(), - adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, cstI64(rewriter, loc, 0), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MATADD"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateMatAddBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { - patterns.add(converter); -} - -void configureMatAddBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/matadd/configs/default.toml b/examples/balls/matadd/configs/default.toml deleted file mode 100644 index c61b39cb6..000000000 --- a/examples/balls/matadd/configs/default.toml +++ /dev/null @@ -1 +0,0 @@ -[ball] diff --git a/examples/balls/matadd/emu/src/dispatch.rs b/examples/balls/matadd/emu/src/dispatch.rs deleted file mode 100644 index 434c7cccd..000000000 --- a/examples/balls/matadd/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::f64_matadd; - -const BALL_CLASS: &str = "examples.balls.matadd.MatAddBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MATADD") => Some(f64_matadd::MatAdd::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MATADD") => Some(f64_matadd::MatAdd::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/matadd/emu/src/lib.rs b/examples/balls/matadd/emu/src/lib.rs deleted file mode 100644 index d4db1c64c..000000000 --- a/examples/balls/matadd/emu/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -#[path = "64_matadd.rs"] -mod f64_matadd; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/matadd/verify/src/common/matadd_defs.svh b/examples/balls/matadd/verify/src/common/matadd_defs.svh deleted file mode 100644 index 6b7f36f39..000000000 --- a/examples/balls/matadd/verify/src/common/matadd_defs.svh +++ /dev/null @@ -1,15 +0,0 @@ -`ifndef MATADD_FUNCT7 -`error "MATADD_FUNCT7 must be provided by the selected Core ballISA" -`endif - -localparam int MATADD_CORE_FUNCT7 = `MATADD_FUNCT7; -localparam int MATADD_MAX_LINES = 64; -localparam int MATADD_TIMEOUT_CYCLES = 100000; - -function automatic int unsigned matadd_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/balls/matadd/verify/src/tb_top.sv b/examples/balls/matadd/verify/src/tb_top.sv deleted file mode 100644 index 677b20a9e..000000000 --- a/examples/balls/matadd/verify/src/tb_top.sv +++ /dev/null @@ -1,96 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import matadd_pkg::*; - - bb_blink_if #(2, 1) intf (); - - MatAddBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_channelReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank[3:0]), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank[3:0]), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank[3:0]), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank[3:0]), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0][3:0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_group_id(intf.bank_read_group_id[0][3:0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1][3:0]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_group_id(intf.bank_read_group_id[1][3:0]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0][3:0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0][3:0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) - ); - - initial begin - intf.clock = 1'b0; - intf.reset = 1'b1; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(2, 1))::set(null, "*", "vif", intf); - run_test("matadd_ball_test"); - end -endmodule diff --git a/examples/balls/matadd/workloads/ctests/CMakeLists.txt b/examples/balls/matadd/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 2fc0cad9f..000000000 --- a/examples/balls/matadd/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_buckyball_ctests(matadd_i32_16row_test.c) diff --git a/examples/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir b/examples/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir deleted file mode 100644 index a3b560048..000000000 --- a/examples/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir +++ /dev/null @@ -1,42 +0,0 @@ -func.func private @check_result(memref<16x4xi32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %a = memref.alloc() : memref<16x4xi32> - %b = memref.alloc() : memref<16x4xi32> - %out = memref.alloc() : memref<16x4xi32> - scf.for %row = %zero to %sixteen step %one { - scf.for %lane = %zero to %four step %one { - %r = arith.index_cast %row : index to i32 - %l = arith.index_cast %lane : index to i32 - %x = arith.addi %r, %l : i32 - %y = arith.subi %l, %r : i32 - memref.store %x, %a[%row, %lane] : memref<16x4xi32> - memref.store %y, %b[%row, %lane] : memref<16x4xi32> - } - } - %a_bank = buckyball.bank_alloc - %b_bank = buckyball.bank_alloc - %out_bank = buckyball.bank_alloc - %a_loaded = buckyball.bank_mvin %a %a_bank %depth %stride - : memref<16x4xi32> i64 i64 i64 - %b_loaded = buckyball.bank_mvin %b %b_bank %depth %stride - : memref<16x4xi32> i64 i64 i64 - buckyball.matadd %a_loaded, %b_loaded, %out_bank, %depth : i64 - %stored = buckyball.bank_mvout %out %out_bank %depth %stride - : memref<16x4xi32> i64 i64 i64 - func.call @check_result(%out) : (memref<16x4xi32>) -> () - buckyball.bank_release %a_loaded : i64 - buckyball.bank_release %b_loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %a : memref<16x4xi32> - memref.dealloc %b : memref<16x4xi32> - memref.dealloc %out : memref<16x4xi32> - return %zero_i8 : i8 -} diff --git a/examples/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in b/examples/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in deleted file mode 100644 index 0ae0c975d..000000000 --- a/examples/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in +++ /dev/null @@ -1,43 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %lanes = arith.constant @BB_BANK_I32_LANES@ : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %stride = arith.constant 1 : i64 - %a = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - %b = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - %out = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - scf.for %row = %zero to %rows step %one { - scf.for %lane = %zero to %lanes step %one { - %r = arith.index_cast %row : index to i32 - %l = arith.index_cast %lane : index to i32 - %x = arith.addi %r, %l : i32 - %y = arith.subi %l, %r : i32 - memref.store %x, %a[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - memref.store %y, %b[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - } - } - %a_bank = buckyball.bank_alloc - %b_bank = buckyball.bank_alloc - %out_bank = buckyball.bank_alloc - %a_loaded = buckyball.bank_mvin %a %a_bank %depth %stride - : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> i64 i64 i64 - %b_loaded = buckyball.bank_mvin %b %b_bank %depth %stride - : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> i64 i64 i64 - %result = buckyball.bank_matadd %a_loaded %b_loaded %out_bank %depth - : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %out %result %depth %stride - : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> i64 i64 i64 - func.call @check_result(%out) : (memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () - buckyball.bank_release %a_loaded : i64 - buckyball.bank_release %b_loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %a : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - memref.dealloc %b : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - memref.dealloc %out : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - return %zero_i8 : i8 -} diff --git a/examples/balls/maxpool/arch/src/main/scala/MaxPoolBall.scala b/examples/balls/maxpool/arch/src/main/scala/MaxPoolBall.scala deleted file mode 100644 index c2595c9aa..000000000 --- a/examples/balls/maxpool/arch/src/main/scala/MaxPoolBall.scala +++ /dev/null @@ -1,258 +0,0 @@ -package examples.balls.maxpool - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.top.GlobalConfig - -@instantiable -class MaxPoolBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "MaxPoolBall") - .getOrElse(throw new IllegalArgumentException("MaxPoolBall not found in config")) - - private val funct = b.ballDomain.ballISA - .find(_.mnemonic == "MAXPOOL") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("MAXPOOL not found in ballISA")) - - require(b.memDomain.bankWidth == 128, "MaxPoolBall requires 128-bit bank rows") - require(mapping.inBW == 1, "MaxPoolBall requires inBW=1") - require(mapping.outBW == 1, "MaxPoolBall requires outBW=1") - require((funct >> 4) == 3, "MAXPOOL must encode one read and one write") - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - private val idle :: scan :: readResp :: writeReq :: writeResp :: complete :: Nil = Enum(6) - private val state = RegInit(idle) - private val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSub = RegInit(false.B) - private val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val inputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val outputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val iter = RegInit(0.U(b.frontend.iter_len.W)) - private val inputSide = RegInit(0.U(8.W)) - private val outputSide = RegInit(0.U(8.W)) - private val kernel = RegInit(0.U(8.W)) - private val stride = RegInit(0.U(8.W)) - private val padding = RegInit(0.U(8.W)) - private val inputBase = RegInit(0.U(6.W)) - private val outputBase = RegInit(0.U(6.W)) - private val outputStride = RegInit(0.U(6.W)) - private val startRow = RegInit(0.U(4.W)) - private val startCol = RegInit(0.U(4.W)) - private val outputRow = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val outputY = RegInit(0.U(8.W)) - private val outputX = RegInit(0.U(8.W)) - private val kernelY = RegInit(0.U(8.W)) - private val kernelX = RegInit(0.U(8.W)) - private val maximum = RegInit(VecInit(Seq.fill(16)("h80".U(8.W)))) - private val writeData = Reg(UInt(128.W)) - - private val sourceY = outputY * stride +& kernelY +& startRow - private val sourceX = outputX * stride +& kernelX +& startCol - private val sourceValid = sourceY >= padding && sourceX >= padding && - sourceY < padding +& inputSide && sourceX < padding +& inputSide - private val inputY = sourceY - padding - private val inputX = sourceX - padding - private val inputAddress = inputBase +& inputY * inputSide +& inputX - private val outputAddress = outputBase +& outputY * outputStride +& outputX - private val lastKernel = kernelY +& 1.U === kernel && kernelX +& 1.U === kernel - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRobId - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - io.bankRead(0).rob_id := robId - io.bankRead(0).ball_id := 0.U - io.bankRead(0).bank_id := inputBank - io.bankRead(0).group_id := 0.U - io.bankRead(0).io.req.valid := false.B - io.bankRead(0).io.req.bits.addr := inputAddress - io.bankRead(0).io.resp.ready := false.B - - io.bankWrite(0).rob_id := robId - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := outputBank - io.bankWrite(0).group_id := 0.U - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := outputAddress - io.bankWrite(0).io.req.bits.data := writeData - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - val inSide = cmd.rs2(3, 0) - val outSide = cmd.rs2(7, 4) - val kernelSize = cmd.rs2(11, 8) - val poolStride = cmd.rs2(15, 12) - val poolPadding = cmd.rs2(19, 16) - val inBase = cmd.rs2(25, 20) - val outBase = cmd.rs2(31, 26) - val outStride = cmd.rs2(37, 32) - val poolStartRow = cmd.rs2(41, 38) - val poolStartCol = cmd.rs2(45, 42) - assert(cmd.rs2(63, 46) === 0.U, "MAXPOOL reserved rs2 bits must be zero") - assert(cmd.funct7 === funct.U, "MaxPoolBall funct7 must be MAXPOOL") - assert(cmd.rs1(19, 10) === 0.U, "MAXPOOL reserves input bank 1") - assert( - cmd.rs1(9, 0) < b.memDomain.virtualBankCount.U && - cmd.rs1(29, 20) < b.memDomain.virtualBankCount.U, - "MAXPOOL bank id is invalid" - ) - assert( - cmd.op1_col === 1.U && cmd.op2_col === 0.U && cmd.wr_col === 1.U, - "MAXPOOL requires one input bank and one output bank" - ) - assert(cmd.op1_bank =/= cmd.wr_bank, "MAXPOOL input and output banks must differ") - assert( - inSide > 0.U && outSide > 0.U && kernelSize > 0.U && poolStride > 0.U, - "MAXPOOL dimensions must be positive" - ) - assert(outStride >= outSide, "MAXPOOL output stride is too small") - val lastSourceRow = poolStartRow +& (outSide - 1.U) * poolStride +& (kernelSize - 1.U) - val lastSourceCol = poolStartCol +& (outSide - 1.U) * poolStride +& (kernelSize - 1.U) - val inputEnd = poolPadding +& inSide - val readsInput = poolStartRow < inputEnd && poolStartCol < inputEnd && - lastSourceRow >= poolPadding && lastSourceCol >= poolPadding - val maxInputRow = Mux(lastSourceRow < inputEnd, lastSourceRow - poolPadding, inSide - 1.U) - val maxInputCol = Mux(lastSourceCol < inputEnd, lastSourceCol - poolPadding, inSide - 1.U) - assert( - !readsInput || inBase +& maxInputRow * inSide +& maxInputCol < b.memDomain.bankEntries.U, - "MAXPOOL input window exceeds bank depth" - ) - assert( - outSide * outSide === cmd.iter && - outBase +& (outSide - 1.U) * outStride +& outSide <= b.memDomain.bankEntries.U, - "MAXPOOL output tile exceeds bank depth" - ) - assert( - poolStartRow +& (outSide - 1.U) * poolStride +& kernelSize <= - inSide +& (poolPadding << 1) && - poolStartCol +& (outSide - 1.U) * poolStride +& kernelSize <= - inSide +& (poolPadding << 1), - "MAXPOOL pooling geometry is inconsistent" - ) - - robId := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRobId := io.cmdReq.bits.sub_rob_id - inputBank := cmd.op1_bank - outputBank := cmd.wr_bank - iter := cmd.iter - inputSide := inSide - outputSide := outSide - kernel := kernelSize - stride := poolStride - padding := poolPadding - inputBase := inBase - outputBase := outBase - outputStride := outStride - startRow := poolStartRow - startCol := poolStartCol - outputRow := 0.U - outputY := 0.U - outputX := 0.U - kernelY := 0.U - kernelX := 0.U - maximum := VecInit(Seq.fill(16)("h80".U(8.W))) - state := scan - } - } - - is(scan) { - when(sourceValid) { - io.bankRead(0).io.req.valid := true.B - when(io.bankRead(0).io.req.fire) { - state := readResp - } - }.elsewhen(lastKernel) { - writeData := Cat(maximum.reverse) - state := writeReq - }.otherwise { - when(kernelX +& 1.U === kernel) { - kernelX := 0.U - kernelY := kernelY + 1.U - }.otherwise { - kernelX := kernelX + 1.U - } - } - } - - is(readResp) { - io.bankRead(0).io.resp.ready := true.B - when(io.bankRead(0).io.resp.fire) { - val data = io.bankRead(0).io.resp.bits.data.asTypeOf(Vec(16, UInt(8.W))) - val nextMaximum = Wire(Vec(16, UInt(8.W))) - for (lane <- 0 until 16) - nextMaximum(lane) := Mux(data(lane).asSInt > maximum(lane).asSInt, data(lane), maximum(lane)) - maximum := nextMaximum - when(lastKernel) { - writeData := Cat(nextMaximum.reverse) - state := writeReq - }.otherwise { - when(kernelX +& 1.U === kernel) { - kernelX := 0.U - kernelY := kernelY + 1.U - }.otherwise { - kernelX := kernelX + 1.U - } - state := scan - } - } - } - - is(writeReq) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire) { - state := writeResp - } - } - - is(writeResp) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - when(outputRow +& 1.U === iter) { - state := complete - }.otherwise { - outputRow := outputRow + 1.U - when(outputX +& 1.U === outputSide) { - outputX := 0.U - outputY := outputY + 1.U - }.otherwise { - outputX := outputX + 1.U - } - kernelY := 0.U - kernelX := 0.U - maximum := VecInit(Seq.fill(16)("h80".U(8.W))) - state := scan - } - } - } - - is(complete) { - when(io.cmdResp.fire) { - state := idle - } - } - } -} diff --git a/examples/balls/maxpool/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/maxpool/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index ff9f77a4d..000000000 --- a/examples/balls/maxpool/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateMaxPoolBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { -class BankMaxPoolPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankMaxPoolOp op, - PatternRewriter &rewriter) const override { - rewriter.create( - op.getLoc(), op.getInBank(), op.getOutBank(), op.getIter(), - op.getInputSideAttr(), op.getOutputSideAttr(), op.getKernelAttr(), - op.getStrideAttr(), op.getPaddingAttr(), op.getInputBase(), - op.getOutputBase(), op.getOutputStride(), op.getStartRowAttr(), - op.getStartColAttr()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateMaxPoolBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/maxpool/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/maxpool/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 332d2d7c2..000000000 --- a/examples/balls/maxpool/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,109 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct MaxPoolLowering : public ConvertOpToLLVMPattern { - MaxPoolLowering(LLVMTypeConverter &converter, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(MaxPoolOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("MaxPoolBall"); - int64_t inputSide = op.getInputSide(); - int64_t outputSide = op.getOutputSide(); - int64_t kernel = op.getKernel(); - int64_t stride = op.getStride(); - int64_t padding = op.getPadding(); - int64_t startRow = op.getStartRow(); - int64_t startCol = op.getStartCol(); - auto iterOp = op.getIter().getDefiningOp(); - auto iterAttr = - iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); - if (!iterAttr || inputSide <= 0 || inputSide > 15 || outputSide <= 0 || - outputSide > 15 || kernel <= 0 || kernel > 15 || stride <= 0 || - stride > 15 || padding < 0 || padding > 15 || startRow < 0 || - startRow > 15 || startCol < 0 || startCol > 15) - return op.emitError("MaxPool shape does not fit one physical bank tile"); - - int64_t lastSourceRow = startRow + (outputSide - 1) * stride + kernel - 1; - int64_t lastSourceCol = startCol + (outputSide - 1) * stride + kernel - 1; - int64_t inputEnd = padding + inputSide; - bool readsInput = startRow < inputEnd && startCol < inputEnd && - lastSourceRow >= padding && lastSourceCol >= padding; - int64_t inputFootprint = 0; - if (readsInput) { - int64_t maxInputRow = std::min(lastSourceRow, inputEnd - 1) - padding; - int64_t maxInputCol = std::min(lastSourceCol, inputEnd - 1) - padding; - inputFootprint = maxInputRow * inputSide + maxInputCol + 1; - } - if (inputFootprint > bankDepth || - outputSide * outputSide != iterAttr.getInt() || - inputSide + 2 * padding < kernel + startRow || - inputSide + 2 * padding < kernel + startCol || - startRow + (outputSide - 1) * stride + kernel > - inputSide + 2 * padding || - startCol + (outputSide - 1) * stride + kernel > inputSide + 2 * padding) - return op.emitError("MaxPool shape does not fit one physical bank tile"); - - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - uint64_t geometry = static_cast(inputSide) | - (static_cast(outputSide) << 4) | - (static_cast(kernel) << 8) | - (static_cast(stride) << 12) | - (static_cast(padding) << 16) | - (static_cast(startRow) << 38) | - (static_cast(startCol) << 42); - Value rs2 = cstI64(rewriter, loc, geometry); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getInputBase(), - cstI64(rewriter, loc, 20))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getOutputBase(), - cstI64(rewriter, loc, 26))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getOutputStride(), - cstI64(rewriter, loc, 32))); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MAXPOOL"))); - return success(); - } - -private: - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateMaxPoolBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - (void)stable; - - patterns.add(converter, bankDepth); -} - -void configureMaxPoolBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/maxpool/configs/default.toml b/examples/balls/maxpool/configs/default.toml deleted file mode 100644 index c632a36cb..000000000 --- a/examples/balls/maxpool/configs/default.toml +++ /dev/null @@ -1,3 +0,0 @@ -# MaxPoolBall has no private configuration parameters. - -[ball] diff --git a/examples/balls/maxpool/emu/src/dispatch.rs b/examples/balls/maxpool/emu/src/dispatch.rs deleted file mode 100644 index c1ec8769d..000000000 --- a/examples/balls/maxpool/emu/src/dispatch.rs +++ /dev/null @@ -1,36 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::maxpool; - -const BALL_CLASS: &str = "examples.balls.maxpool.MaxPoolBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MAXPOOL") => Some(maxpool::MaxPool::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MAXPOOL") => Some(maxpool::MaxPool::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/maxpool/emu/src/lib.rs b/examples/balls/maxpool/emu/src/lib.rs deleted file mode 100644 index c1c40e7f1..000000000 --- a/examples/balls/maxpool/emu/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod maxpool; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/maxpool/verify/casegen/src/casegen.rs b/examples/balls/maxpool/verify/casegen/src/casegen.rs deleted file mode 100644 index 64b54d950..000000000 --- a/examples/balls/maxpool/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,565 +0,0 @@ -use crate::model; - -pub const MAX_INPUT_WORDS: usize = 64; -pub const MAX_DST: usize = 64; -pub const NUM_CASES: u32 = 9; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct MaxPoolCmd { - pub bid: u32, - pub iter: u32, - pub op1_bank: u32, - pub wr_bank: u32, - pub op1_col: u32, - pub wr_col: u32, - pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, - pub rs2_lo: u32, - pub rs2_hi: u32, - pub input_base: u32, - pub output_base: u32, - pub output_stride: u32, - pub input_side: u32, - pub output_side: u32, - pub kernel: u32, - pub stride: u32, - pub padding: u32, - pub start_row: u32, - pub start_col: u32, - pub num_input_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct MaxPoolCase { - pub cmd: MaxPoolCmd, - pub input_words: [u128; MAX_INPUT_WORDS], - pub input_addr: [u32; MAX_INPUT_WORDS], - pub dst_words: [u128; MAX_DST], - pub dst_addr: [u32; MAX_DST], -} - -impl MaxPoolCase { - pub fn input_lo(&self, index: usize) -> u64 { - self.input_words[index] as u64 - } - - pub fn input_hi(&self, index: usize) -> u64 { - (self.input_words[index] >> 64) as u64 - } - - pub fn dst_lo(&self, index: usize) -> u64 { - self.dst_words[index] as u64 - } - - pub fn dst_hi(&self, index: usize) -> u64 { - (self.dst_words[index] >> 64) as u64 - } -} - -#[derive(Clone, Copy)] -struct CaseSpec { - geo: model::Geo, - op1_bank: u32, - wr_bank: u32, - rob_id: u32, - ctest_input: bool, - seed: u32, -} - -fn field(val: u32, lo: u32, hi: u32) -> u64 { - let width = hi - lo + 1; - let mask = if width == 64 { - u64::MAX - } else { - (1u64 << width) - 1 - }; - (u64::from(val) & mask) << lo -} - -fn encode_rs1(input_bank: u32, output_bank: u32, iter: u32) -> u64 { - field(input_bank, 0, 9) | field(0, 10, 19) | field(output_bank, 20, 29) | field(iter, 30, 63) -} - -fn encode_rs2(geo: &model::Geo) -> u64 { - field(geo.input_side as u32, 0, 3) - | field(geo.output_side as u32, 4, 7) - | field(geo.kernel as u32, 8, 11) - | field(geo.stride as u32, 12, 15) - | field(geo.padding as u32, 16, 19) - | field(geo.input_base as u32, 20, 25) - | field(geo.output_base as u32, 26, 31) - | field(geo.output_stride as u32, 32, 37) - | field(geo.start_row as u32, 38, 41) - | field(geo.start_col as u32, 42, 45) -} - -fn split_rs(value: u64) -> (u32, u32) { - (value as u32, (value >> 32) as u32) -} - -fn validate_cmd(op1_bank: u32, wr_bank: u32, iter: u32, rs1: u64, rs2: u64) { - if iter == 0 { - panic!("maxpool: iter must be positive"); - } - if op1_bank == wr_bank { - panic!("maxpool: input and output banks must differ"); - } - if (rs1 & 0x3ff) != u64::from(op1_bank) { - panic!("maxpool: rs1 bank0 mismatch"); - } - if ((rs1 >> 10) & 0x3ff) != 0 { - panic!("maxpool: input bank 1 must be zero"); - } - if ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) { - panic!("maxpool: rs1 bank2 mismatch"); - } - if ((rs1 >> 30) & 0x3fff_ffff) != u64::from(iter) { - panic!("maxpool: rs1 iter mismatch"); - } - if (rs2 >> 46) != 0 { - panic!("maxpool: reserved rs2 bits must be zero"); - } -} - -fn ctest_tile(geo: &model::Geo) -> Vec { - let tile = geo.input_side * geo.input_side; - let mut bytes = vec![0u8; tile * model::BANK_ROW_BYTES]; - for position in 0..tile { - for channel in 0..model::BANK_ROW_BYTES { - bytes[position * model::BANK_ROW_BYTES + channel] = - (((position * 37 + channel * 19) & 255) as i32 - 128) as u8; - } - } - bytes -} - -fn seeded_tile(geo: &model::Geo, seed: u32) -> Vec { - let tile = geo.input_side * geo.input_side; - let mut bytes = vec![0u8; tile * model::BANK_ROW_BYTES]; - for position in 0..tile { - for channel in 0..model::BANK_ROW_BYTES { - let mix = seed - .wrapping_mul(0x9E37_79B9) - .wrapping_add(position as u32 * 131) - .wrapping_add(channel as u32 * 17); - bytes[position * model::BANK_ROW_BYTES + channel] = ((mix & 255) as i32 - 128) as u8; - } - } - bytes -} - -fn copy_words(src: &[u128]) -> [u128; N] { - if src.len() > N { - panic!("maxpool: {} words exceed buffer {N}", src.len()); - } - let mut out = [0u128; N]; - out[..src.len()].copy_from_slice(src); - out -} - -fn copy_addrs(src: &[u32]) -> [u32; N] { - if src.len() > N { - panic!("maxpool: {} addrs exceed buffer {N}", src.len()); - } - let mut out = [0u32; N]; - out[..src.len()].copy_from_slice(src); - out -} - -fn build_case(spec: CaseSpec, bid: u32) -> MaxPoolCase { - model::validate(&spec.geo); - let iter = model::iter_for(&spec.geo); - let input_bytes = if spec.ctest_input { - ctest_tile(&spec.geo) - } else { - seeded_tile(&spec.geo, spec.seed) - }; - - let mut referenced = vec![false; spec.geo.input_side * spec.geo.input_side]; - for output_y in 0..spec.geo.output_side { - for output_x in 0..spec.geo.output_side { - for kernel_y in 0..spec.geo.kernel { - for kernel_x in 0..spec.geo.kernel { - let y = output_y * spec.geo.stride + kernel_y + spec.geo.start_row; - let x = output_x * spec.geo.stride + kernel_x + spec.geo.start_col; - if y >= spec.geo.padding - && x >= spec.geo.padding - && y < spec.geo.padding + spec.geo.input_side - && x < spec.geo.padding + spec.geo.input_side - { - referenced[(y - spec.geo.padding) * spec.geo.input_side - + (x - spec.geo.padding)] = true; - } - } - } - } - } - - let mut input_addrs = Vec::new(); - let mut input_words = Vec::new(); - for y in 0..spec.geo.input_side { - for x in 0..spec.geo.input_side { - if !referenced[y * spec.geo.input_side + x] { - continue; - } - let addr = (spec.geo.input_base + y * spec.geo.input_side + x) as u32; - let row_off = (y * spec.geo.input_side + x) * model::BANK_ROW_BYTES; - let mut row = [0u8; model::BANK_ROW_BYTES]; - row.copy_from_slice(&input_bytes[row_off..row_off + model::BANK_ROW_BYTES]); - input_addrs.push(addr); - input_words.push(model::pack_row(&row)); - } - } - - let gold = model::maxpool_rows(&input_bytes, &spec.geo); - let dst_addrs: Vec = gold.iter().map(|(addr, _)| *addr).collect(); - let dst_words: Vec = gold.iter().map(|(_, row)| model::pack_row(row)).collect(); - - if dst_addrs.len() as u32 != iter { - panic!("maxpool: gold rows {} != iter {iter}", dst_addrs.len()); - } - - let rs1 = encode_rs1(spec.op1_bank, spec.wr_bank, iter); - let rs2 = encode_rs2(&spec.geo); - validate_cmd(spec.op1_bank, spec.wr_bank, iter, rs1, rs2); - - let (rs1_lo, rs1_hi) = split_rs(rs1); - let (rs2_lo, rs2_hi) = split_rs(rs2); - - MaxPoolCase { - cmd: MaxPoolCmd { - bid, - iter, - op1_bank: spec.op1_bank, - wr_bank: spec.wr_bank, - op1_col: 1, - wr_col: 1, - rob_id: spec.rob_id, - rs1_lo, - rs1_hi, - rs2_lo, - rs2_hi, - input_base: spec.geo.input_base as u32, - output_base: spec.geo.output_base as u32, - output_stride: spec.geo.output_stride as u32, - input_side: spec.geo.input_side as u32, - output_side: spec.geo.output_side as u32, - kernel: spec.geo.kernel as u32, - stride: spec.geo.stride as u32, - padding: spec.geo.padding as u32, - start_row: spec.geo.start_row as u32, - start_col: spec.geo.start_col as u32, - num_input_words: input_words.len() as u32, - num_dst_words: iter, - }, - input_words: copy_words(&input_words), - input_addr: copy_addrs::(&input_addrs), - dst_words: copy_words(&dst_words), - dst_addr: copy_addrs::(&dst_addrs), - } -} - -fn spec_for(index: u32) -> CaseSpec { - match index { - 0 => CaseSpec { - geo: model::Geo { - input_side: 6, - output_side: 3, - kernel: 2, - stride: 2, - padding: 0, - input_base: 7, - output_base: 11, - output_stride: 7, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 2, - ctest_input: true, - seed: 0, - }, - 1 => CaseSpec { - geo: model::Geo { - input_side: 4, - output_side: 3, - kernel: 2, - stride: 1, - padding: 0, - input_base: 0, - output_base: 20, - output_stride: 4, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 3, - ctest_input: false, - seed: 1, - }, - 2 => CaseSpec { - geo: model::Geo { - input_side: 4, - output_side: 3, - kernel: 2, - stride: 1, - padding: 1, - input_base: 0, - output_base: 30, - output_stride: 4, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 4, - ctest_input: false, - seed: 2, - }, - 3 => CaseSpec { - geo: model::Geo { - input_side: 6, - output_side: 3, - kernel: 2, - stride: 2, - padding: 1, - input_base: 0, - output_base: 40, - output_stride: 4, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 5, - ctest_input: false, - seed: 3, - }, - 4 => CaseSpec { - geo: model::Geo { - input_side: 5, - output_side: 3, - kernel: 3, - stride: 1, - padding: 0, - input_base: 0, - output_base: 50, - output_stride: 4, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 6, - ctest_input: false, - seed: 4, - }, - 5 => CaseSpec { - geo: model::Geo { - input_side: 7, - output_side: 3, - kernel: 3, - stride: 2, - padding: 0, - input_base: 0, - output_base: 0, - output_stride: 7, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 7, - ctest_input: false, - seed: 5, - }, - 6 => CaseSpec { - geo: model::Geo { - input_side: 5, - output_side: 3, - kernel: 3, - stride: 1, - padding: 1, - input_base: 0, - output_base: 15, - output_stride: 4, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 8, - ctest_input: false, - seed: 6, - }, - 7 => CaseSpec { - geo: model::Geo { - input_side: 7, - output_side: 3, - kernel: 3, - stride: 2, - padding: 1, - input_base: 0, - output_base: 25, - output_stride: 7, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 9, - ctest_input: false, - seed: 7, - }, - 8 => CaseSpec { - geo: model::Geo { - input_side: 7, - output_side: 1, - kernel: 1, - stride: 1, - padding: 0, - input_base: 63, - output_base: 0, - output_stride: 1, - start_row: 0, - start_col: 0, - }, - op1_bank: 0, - wr_bank: 1, - rob_id: 10, - ctest_input: false, - seed: 8, - }, - _ => panic!("maxpool: unsupported directed case {index}"), - } -} - -pub fn gen_case(index: u32, bid: u32) -> MaxPoolCase { - if index >= NUM_CASES { - panic!("maxpool: unsupported directed case {index}"); - } - build_case(spec_for(index), bid) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn case_zero_matches_ctest() { - let case = gen_case(0, 6); - assert_eq!(case.cmd.bid, 6); - assert_eq!(case.cmd.iter, 9); - assert_eq!(case.cmd.input_side, 6); - assert_eq!(case.cmd.output_side, 3); - assert_eq!(case.cmd.kernel, 2); - assert_eq!(case.cmd.stride, 2); - assert_eq!(case.cmd.padding, 0); - assert_eq!(case.cmd.input_base, 7); - assert_eq!(case.cmd.output_base, 11); - assert_eq!(case.cmd.output_stride, 7); - assert_eq!(case.cmd.op1_bank, 0); - assert_eq!(case.cmd.wr_bank, 1); - assert_eq!(case.cmd.num_input_words, 36); - assert_eq!(case.cmd.num_dst_words, 9); - let rs2 = u64::from(case.cmd.rs2_lo) | (u64::from(case.cmd.rs2_hi) << 32); - assert_eq!(rs2 >> 46, 0); - let rs1 = u64::from(case.cmd.rs1_lo) | (u64::from(case.cmd.rs1_hi) << 32); - assert_eq!(rs1 & 0x3ff, 0); - assert_eq!((rs1 >> 10) & 0x3ff, 0); - assert_eq!((rs1 >> 20) & 0x3ff, 1); - assert_eq!((rs1 >> 30) & 0xffff_ffff, 9); - } - - #[test] - fn cases_hit_cover_bins() { - let mut k2 = false; - let mut k3 = false; - let mut k1 = false; - let mut s1 = false; - let mut s2 = false; - let mut p0 = false; - let mut p1 = false; - for index in 0..NUM_CASES { - let case = gen_case(index, 6); - assert_eq!(case.cmd.bid, 6); - assert_eq!(case.cmd.op1_col, 1); - assert_eq!(case.cmd.wr_col, 1); - assert_ne!(case.cmd.op1_bank, case.cmd.wr_bank); - assert_eq!(case.cmd.num_dst_words, case.cmd.iter); - assert_eq!(case.cmd.iter, case.cmd.output_side * case.cmd.output_side); - match case.cmd.kernel { - 1 => k1 = true, - 2 => k2 = true, - 3 => k3 = true, - other => panic!("unexpected kernel {other}"), - } - match case.cmd.stride { - 1 => s1 = true, - 2 => s2 = true, - other => panic!("unexpected stride {other}"), - } - match case.cmd.padding { - 0 => p0 = true, - 1 => p1 = true, - other => panic!("unexpected padding {other}"), - } - } - assert!(k1 && k2 && k3 && s1 && s2 && p0 && p1); - } - - #[test] - #[should_panic(expected = "unsupported directed case")] - fn unknown_index_panics() { - let _ = gen_case(NUM_CASES, 6); - } - - #[test] - #[should_panic(expected = "illegal square pooling geometry")] - fn illegal_geometry_panics() { - model::validate(&model::Geo { - input_side: 2, - output_side: 3, - kernel: 2, - stride: 1, - padding: 0, - input_base: 0, - output_base: 0, - output_stride: 4, - start_row: 0, - start_col: 0, - }); - } - - #[test] - fn gold_matches_manual_window() { - let geo = model::Geo { - input_side: 2, - output_side: 1, - kernel: 2, - stride: 1, - padding: 0, - input_base: 0, - output_base: 5, - output_stride: 1, - start_row: 0, - start_col: 0, - }; - let mut input = vec![0u8; 2 * 2 * model::BANK_ROW_BYTES]; - for lane in 0..model::BANK_ROW_BYTES { - input[lane] = 1; - input[model::BANK_ROW_BYTES + lane] = 3; - input[2 * model::BANK_ROW_BYTES + lane] = 5; - input[3 * model::BANK_ROW_BYTES + lane] = 2; - } - let gold = model::maxpool_rows(&input, &geo); - assert_eq!(gold.len(), 1); - assert_eq!(gold[0].0, 5); - assert_eq!(gold[0].1[0], 5); - } -} diff --git a/examples/balls/maxpool/verify/casegen/src/lib.rs b/examples/balls/maxpool/verify/casegen/src/lib.rs deleted file mode 100644 index fd6b84ef8..000000000 --- a/examples/balls/maxpool/verify/casegen/src/lib.rs +++ /dev/null @@ -1,160 +0,0 @@ -mod casegen; -mod model; - -use std::cell::RefCell; - -use casegen::{MaxPoolCase, MaxPoolCmd, MAX_DST, MAX_INPUT_WORDS}; - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -#[no_mangle] -pub extern "C" fn maxpool_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid); - CURRENT.with(|current| *current.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|current| match current.borrow().as_ref() { - Some(case) => f(case), - None => panic!("maxpool_case: no case loaded; call maxpool_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn maxpool_case_cmd(out_ptr: *mut MaxPoolCmd) { - if out_ptr.is_null() { - panic!("maxpool_case_cmd: null out_ptr"); - } - current_case(|case| unsafe { - *out_ptr = case.cmd; - }); -} - -fn idx(index: u32, limit: u32, what: &str) -> usize { - if index >= limit { - panic!("maxpool_case_{what}: index {index} out of range {limit}"); - } - index as usize -} - -#[no_mangle] -pub extern "C" fn maxpool_case_input_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = idx(word_index, case.cmd.num_input_words, "input_word_lo"); - if i >= MAX_INPUT_WORDS { - panic!("maxpool_case_input_word_lo: word_index out of range"); - } - case.input_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn maxpool_case_input_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = idx(word_index, case.cmd.num_input_words, "input_word_hi"); - if i >= MAX_INPUT_WORDS { - panic!("maxpool_case_input_word_hi: word_index out of range"); - } - case.input_hi(i) - }) -} - -#[no_mangle] -pub extern "C" fn maxpool_case_input_addr(word_index: u32) -> u32 { - current_case(|case| { - let i = idx(word_index, case.cmd.num_input_words, "input_addr"); - if i >= MAX_INPUT_WORDS { - panic!("maxpool_case_input_addr: word_index out of range"); - } - case.input_addr[i] - }) -} - -#[no_mangle] -pub extern "C" fn maxpool_case_dst_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = idx(word_index, case.cmd.num_dst_words, "dst_word_lo"); - if i >= MAX_DST { - panic!("maxpool_case_dst_word_lo: word_index out of range"); - } - case.dst_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn maxpool_case_dst_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = idx(word_index, case.cmd.num_dst_words, "dst_word_hi"); - if i >= MAX_DST { - panic!("maxpool_case_dst_word_hi: word_index out of range"); - } - case.dst_hi(i) - }) -} - -#[no_mangle] -pub extern "C" fn maxpool_case_dst_addr(word_index: u32) -> u32 { - current_case(|case| { - let i = idx(word_index, case.cmd.num_dst_words, "dst_addr"); - if i >= MAX_DST { - panic!("maxpool_case_dst_addr: word_index out of range"); - } - case.dst_addr[i] - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dpi_load_then_cmd_and_words() { - assert_eq!(maxpool_case_load(0, 6), 0); - let mut cmd = MaxPoolCmd { - bid: 0, - iter: 0, - op1_bank: 0, - wr_bank: 0, - op1_col: 0, - wr_col: 0, - rob_id: 0, - rs1_lo: 0, - rs1_hi: 0, - rs2_lo: 0, - rs2_hi: 0, - input_base: 0, - output_base: 0, - output_stride: 0, - input_side: 0, - output_side: 0, - kernel: 0, - stride: 0, - padding: 0, - start_row: 0, - start_col: 0, - num_input_words: 0, - num_dst_words: 0, - }; - maxpool_case_cmd(&mut cmd as *mut MaxPoolCmd); - assert_eq!(cmd.bid, 6); - assert_eq!(cmd.iter, 9); - assert_eq!(cmd.num_input_words, 36); - assert_eq!(cmd.num_dst_words, 9); - let _ = maxpool_case_input_word_lo(0); - let _ = maxpool_case_input_word_hi(0); - let _ = maxpool_case_input_addr(0); - let _ = maxpool_case_dst_word_lo(0); - let _ = maxpool_case_dst_addr(0); - assert_eq!(casegen::NUM_CASES, 9); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/maxpool/verify/casegen/src/model.rs b/examples/balls/maxpool/verify/casegen/src/model.rs deleted file mode 100644 index e9a5fe01b..000000000 --- a/examples/balls/maxpool/verify/casegen/src/model.rs +++ /dev/null @@ -1,104 +0,0 @@ -pub const BANK_LINES: usize = 64; -pub const BANK_ROW_BYTES: usize = 16; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Geo { - pub input_side: usize, - pub output_side: usize, - pub kernel: usize, - pub stride: usize, - pub padding: usize, - pub input_base: usize, - pub output_base: usize, - pub output_stride: usize, - pub start_row: usize, - pub start_col: usize, -} - -pub fn validate(geo: &Geo) { - let g = geo; - let dimensions_valid = - g.input_side != 0 && g.output_side != 0 && g.kernel != 0 && g.stride != 0; - if !dimensions_valid { - panic!("maxpool: illegal square pooling geometry"); - } - let last_row = g.start_row + (g.output_side - 1) * g.stride + g.kernel - 1; - let last_col = g.start_col + (g.output_side - 1) * g.stride + g.kernel - 1; - let input_end = g.padding + g.input_side; - let reads_input = g.start_row < input_end - && g.start_col < input_end - && last_row >= g.padding - && last_col >= g.padding; - let input_fits = !reads_input - || g.input_base - + (last_row.min(input_end - 1) - g.padding) * g.input_side - + (last_col.min(input_end - 1) - g.padding) - < BANK_LINES; - if g.output_stride < g.output_side - || !input_fits - || g.output_base + (g.output_side - 1) * g.output_stride + g.output_side > BANK_LINES - || g.input_side + 2 * g.padding < g.kernel + g.start_row - || g.input_side + 2 * g.padding < g.kernel + g.start_col - || g.start_row + (g.output_side - 1) * g.stride + g.kernel > g.input_side + 2 * g.padding - || g.start_col + (g.output_side - 1) * g.stride + g.kernel > g.input_side + 2 * g.padding - { - panic!("maxpool: illegal square pooling geometry"); - } -} - -pub fn iter_for(geo: &Geo) -> u32 { - let iter = geo.output_side * geo.output_side; - if iter == 0 { - panic!("maxpool: iter must be positive"); - } - iter as u32 -} - -pub fn maxpool_rows(input: &[u8], geo: &Geo) -> Vec<(u32, [u8; BANK_ROW_BYTES])> { - validate(geo); - let tile = geo.input_side * geo.input_side; - if input.len() != tile * BANK_ROW_BYTES { - panic!("maxpool: input bytes {} != tile {} rows", input.len(), tile); - } - - let mut out = Vec::with_capacity(geo.output_side * geo.output_side); - for output_y in 0..geo.output_side { - for output_x in 0..geo.output_side { - let mut maximum = [0x80u8; BANK_ROW_BYTES]; - for kernel_y in 0..geo.kernel { - for kernel_x in 0..geo.kernel { - let input_y = output_y * geo.stride + kernel_y + geo.start_row; - let input_x = output_x * geo.stride + kernel_x + geo.start_col; - let sy = input_y as isize - geo.padding as isize; - let sx = input_x as isize - geo.padding as isize; - if sy < 0 - || sx < 0 - || sy >= geo.input_side as isize - || sx >= geo.input_side as isize - { - continue; - } - let row = (sy as usize) * geo.input_side + sx as usize; - let off = row * BANK_ROW_BYTES; - for lane in 0..BANK_ROW_BYTES { - let v = input[off + lane] as i8; - if v > maximum[lane] as i8 { - maximum[lane] = v as u8; - } - } - } - } - let addr = (geo.output_base + output_y * geo.output_stride + output_x) as u32; - out.push((addr, maximum)); - } - } - out -} - -pub fn pack_row(bytes: &[u8; BANK_ROW_BYTES]) -> u128 { - let mut word = 0u128; - for (lane, byte) in bytes.iter().enumerate() { - word |= u128::from(*byte) << (lane * 8); - } - word -} diff --git a/examples/balls/maxpool/workloads/ctests/CMakeLists.txt b/examples/balls/maxpool/workloads/ctests/CMakeLists.txt deleted file mode 100644 index b86255542..000000000 --- a/examples/balls/maxpool/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_buckyball_ctests(maxpool_window_test.c maxpool_offset_test.c) diff --git a/examples/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir b/examples/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir deleted file mode 100644 index b010bbb0a..000000000 --- a/examples/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir +++ /dev/null @@ -1,32 +0,0 @@ -func.func private @check_result(memref<9x16xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %thirty_six = arith.constant 36 : index - %sixteen = arith.constant 16 : index - %depth36 = arith.constant 36 : i64 - %depth9 = arith.constant 9 : i64 - %zero64 = arith.constant 0 : i64 - %three64 = arith.constant 3 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() : memref<36x16xi8> - %output = memref.alloc() : memref<9x16xi8> - scf.for %row = %zero to %thirty_six step %one { - %value = arith.index_cast %row : index to i8 - scf.for %lane = %zero to %sixteen step %one { - memref.store %value, %input[%row, %lane] : memref<36x16xi8> - } - } - %in_bank = buckyball.bank_alloc - %out_bank = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %in_bank %depth36 %stride : memref<36x16xi8> i64 i64 i64 - buckyball.maxpool %loaded, %out_bank, %depth9, %zero64, %zero64, %three64 : i64 i64 i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %out_bank %depth9 %stride : memref<9x16xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<9x16xi8>) -> () - buckyball.bank_release %loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<36x16xi8> - memref.dealloc %output : memref<9x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in b/examples/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in deleted file mode 100644 index f7ce8a64d..000000000 --- a/examples/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in +++ /dev/null @@ -1,41 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %batch_rows = arith.constant 64 : index - %lanes = arith.constant @BB_BANK_I8_LANES@ : index - %mod = arith.constant 31 : index - %bias = arith.constant 15 : index - %depth = arith.constant 64 : i64 - %zero64 = arith.constant 0 : i64 - %side64 = arith.constant 8 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - scf.for %row = %zero to %rows step %one { - %wrapped = arith.remui %row, %mod : index - %signed = arith.subi %wrapped, %bias : index - %value = arith.index_cast %signed : index to i8 - scf.for %lane = %zero to %lanes step %one { - memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.store %zero_i8, %output[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - } - } - %in_bank = buckyball.bank_alloc - %out_bank = buckyball.bank_alloc - %final:2 = scf.for %base = %zero to %rows step %batch_rows iter_args(%ib = %in_bank, %ob = %out_bank) -> (i64, i64) { - %in = memref.subview %input[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> - %out = memref.subview %output[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> - %loaded_input = buckyball.bank_mvin %in %ib %depth %stride : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> i64 i64 i64 - %loaded_output = buckyball.bank_mvin %out %ob %depth %stride : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> i64 i64 i64 - %result = buckyball.bank_maxpool %loaded_input %loaded_output %depth %zero64 %zero64 %side64 : i64 i64 i64 i64 i64 i64 - %stored = buckyball.bank_mvout %out %result %depth %stride : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> i64 i64 i64 - scf.yield %loaded_input, %stored : i64, i64 - } - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %final#0 : i64 - buckyball.bank_release %final#1 : i64 - return %zero_i8 : i8 -} diff --git a/examples/balls/mxfp2int/arch/src/main/scala/Mxfp2Int.scala b/examples/balls/mxfp2int/arch/src/main/scala/Mxfp2Int.scala deleted file mode 100644 index 42dca6482..000000000 --- a/examples/balls/mxfp2int/arch/src/main/scala/Mxfp2Int.scala +++ /dev/null @@ -1,292 +0,0 @@ -package examples.balls.mxfp2int - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.balldomain.blink.mmio.MmioRead -import framework.top.GlobalConfig -import examples.balls.mxfp2int.configs.Mxfp2IntBallParam - -/** - * PipelinedMxfp2Int - MXFP to INT8 dequantizer with MMIO per-block scale. - * - * Supports MXFP4, MXFP6, MXFP8 formats (32 elements per block). - * - * Input format (main bank, per block): - * MXFP4: 32 × 4-bit = 128 bits = 1 word - * MXFP6: 32 × 6-bit = 192 bits, padded to 2 words (256 bits, high 64 bits = 0) - * MXFP8: 32 × 8-bit = 256 bits = 2 words - * - * Scale (MMIO bank): per-block E8M0 (8-bit biased exponent, bias=127). - * Read MMIO byte at rel_addr = block_idx with meta_bank = wr_bank. - * - * FP4/6/8 decoding (v1, simplified): signed_val = raw - (1 << (bits-1)) - * FP4: raw - 8 (range -8..+7) - * FP6: raw - 32 (range -32..+31) - * FP8: raw - 128 (range -128..+127) - * - * Output (per block, 2 SRAM words = 32 bytes): - * 32 × INT8 = saturate(signed_val << (scale - 127)). - * Lower 16 INT8 to row (2 * block_idx), upper 16 to row (2 * block_idx + 1). - * - * iter encodes the number of MXFP blocks. - */ -@instantiable -class PipelinedMxfp2Int(val b: GlobalConfig) extends Module { - val ballConfig = Mxfp2IntBallParam(b) - val decoder = MxfpDecoder.fromFormat(ballConfig.mxfpFormat) - val elemBits = decoder.elemBits - - val bankWidth = b.memDomain.bankWidth - require( - bankWidth == 128, - s"Mxfp2IntBall requires bankWidth = 128, got $bankWidth" - ) - - val wordsPerBlock = ballConfig.mxfpFormat match { - case "MXFP4" => 1 - case "MXFP6" => - 2 // padded: 192 bits data + 64 bits padding = 256 bits = 2 words - case "MXFP8" => 2 - case _ => - throw new IllegalArgumentException( - s"Unsupported mxfpFormat: ${ballConfig.mxfpFormat}" - ) - } - - val FP_PER_BLOCK = 32 - val INT8_PER_WORD = bankWidth / 8 // 16 - - val ballMapping = b.ballDomain.ballIdMappings - .find(_.ballName == "Mxfp2IntBall") - .getOrElse( - throw new IllegalArgumentException("Mxfp2IntBall not found in config") - ) - - val inBW = ballMapping.inBW - val outBW = ballMapping.outBW - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(outBW, Flipped(new BankWrite(b))) - val status = new BallStatus - val mmioRead = Flipped(new MmioRead(b)) - }) - - // ROB bookkeeping - val rob_id_reg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val is_sub_reg = RegInit(false.B) - val sub_rob_id_reg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - - when(io.cmdReq.fire) { - rob_id_reg := io.cmdReq.bits.rob_id - is_sub_reg := io.cmdReq.bits.is_sub - sub_rob_id_reg := io.cmdReq.bits.sub_rob_id - } - - // FSM - val idle :: sReadReq :: sReadResp :: sMmioReq :: sMmioResp :: sWriteLoReq :: sWriteLoResp :: sWriteHiReq :: sWriteHiResp :: sComplete :: Nil = - Enum(10) - val state = RegInit(idle) - - // Per-instruction registers - val raddr_reg = RegInit(0.U(b.frontend.iter_len.W)) - val waddr_reg = RegInit(0.U(b.frontend.iter_len.W)) - val rbank_reg = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val wbank_reg = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val iter_reg = RegInit(0.U(b.frontend.iter_len.W)) - val block_idx_reg = RegInit(0.U(b.frontend.iter_len.W)) - val word_idx_reg = RegInit(0.U(2.W)) // 0..wordsPerBlock-1 - - val inputWordsReg = RegInit( - VecInit(Seq.fill(2)(0.U(bankWidth.W))) - ) // max 2 words for MXFP6/8 - val scaleReg = RegInit(0.U(8.W)) - - // Extract 32 FP elements from inputWordsReg based on format - val fpElements = VecInit(Seq.tabulate(FP_PER_BLOCK) { i => - val bitOffset = i * elemBits - val wordIdx = bitOffset / bankWidth - val bitInWord = bitOffset % bankWidth - if (wordIdx < wordsPerBlock && bitInWord + elemBits <= bankWidth) { - inputWordsReg(wordIdx)(bitInWord + elemBits - 1, bitInWord) - } else if (wordIdx < wordsPerBlock - 1 && bitInWord + elemBits > bankWidth) { - // Cross-word boundary (only for MXFP6 if packed, but we use padded so this won't happen) - val lowBits = bankWidth - bitInWord - val highBits = elemBits - lowBits - Cat( - inputWordsReg(wordIdx + 1)(highBits - 1, 0), - inputWordsReg(wordIdx)(bankWidth - 1, bitInWord) - ) - } else { - 0.U(elemBits.W) - } - }) - - // Combinationally compute all 32 INT8 values from the latched elements + scale - val int8Vec = VecInit( - fpElements.map(raw => decoder.dequant(raw, scaleReg).asUInt) - ) - - val outWordLo = Cat((0 until INT8_PER_WORD).map(i => int8Vec(i)).reverse) - - val outWordHi = Cat( - (INT8_PER_WORD until FP_PER_BLOCK).map(i => int8Vec(i)).reverse - ) - - // === Default IO === - for (i <- 0 until inBW) { - io.bankRead(i).io.req.valid := false.B - io.bankRead(i).io.req.bits.addr := 0.U - io.bankRead(i).io.resp.ready := false.B - io.bankRead(i).rob_id := rob_id_reg - io.bankRead(i).ball_id := 0.U - io.bankRead(i).bank_id := rbank_reg - io.bankRead(i).group_id := 0.U - } - for (i <- 0 until outBW) { - io.bankWrite(i).io.req.valid := false.B - io.bankWrite(i).io.req.bits.addr := 0.U - io.bankWrite(i).io.req.bits.data := 0.U - io.bankWrite(i).io.req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(0.U(1.W)) - ) - io.bankWrite(i).io.resp.ready := false.B - io.bankWrite(i).rob_id := rob_id_reg - io.bankWrite(i).ball_id := 0.U - io.bankWrite(i).bank_id := wbank_reg - io.bankWrite(i).group_id := 0.U - } - - io.mmioRead.req.valid := false.B - io.mmioRead.req.bits.addr := 0.U - io.mmioRead.resp.ready := false.B - io.mmioRead.rob_id := rob_id_reg - io.mmioRead.ball_id := 0.U - io.cmdReq.ready := state === idle - io.cmdResp.valid := false.B - io.cmdResp.bits.rob_id := rob_id_reg - io.cmdResp.bits.is_sub := is_sub_reg - io.cmdResp.bits.sub_rob_id := sub_rob_id_reg - - // === FSM === - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - rbank_reg := io.cmdReq.bits.cmd.op1_bank - wbank_reg := io.cmdReq.bits.cmd.wr_bank - iter_reg := io.cmdReq.bits.cmd.iter - block_idx_reg := 0.U - word_idx_reg := 0.U - raddr_reg := 0.U - waddr_reg := 0.U - when(io.cmdReq.bits.cmd.iter === 0.U) { - state := sComplete - }.otherwise { - state := sReadReq - } - } - } - - is(sReadReq) { - io.bankRead(0).io.req.valid := true.B - io.bankRead(0) - .io - .req - .bits - .addr := raddr_reg + block_idx_reg * wordsPerBlock.U + word_idx_reg - when(io.bankRead(0).io.req.fire) { - state := sReadResp - } - } - - is(sReadResp) { - io.bankRead(0).io.resp.ready := true.B - when(io.bankRead(0).io.resp.fire) { - inputWordsReg(word_idx_reg) := io.bankRead(0).io.resp.bits.data - when(word_idx_reg === (wordsPerBlock - 1).U) { - word_idx_reg := 0.U - state := sMmioReq - }.otherwise { - word_idx_reg := word_idx_reg + 1.U - state := sReadReq - } - } - } - - is(sMmioReq) { - io.mmioRead.req.valid := true.B - io.mmioRead.req.bits.addr := block_idx_reg - when(io.mmioRead.req.fire) { - state := sMmioResp - } - } - - is(sMmioResp) { - io.mmioRead.resp.ready := true.B - when(io.mmioRead.resp.fire) { - scaleReg := io.mmioRead.resp.bits.data(7, 0) - state := sWriteLoReq - } - } - - is(sWriteLoReq) { - io.bankWrite(0).io.req.valid := true.B - io.bankWrite(0).io.req.bits.addr := waddr_reg + (block_idx_reg << 1) - io.bankWrite(0).io.req.bits.data := outWordLo - io.bankWrite(0).io.req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(1.U(1.W)) - ) - when(io.bankWrite(0).io.req.fire) { - state := sWriteLoResp - } - } - - is(sWriteLoResp) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - state := sWriteHiReq - } - } - - is(sWriteHiReq) { - io.bankWrite(0).io.req.valid := true.B - io.bankWrite(0).io.req.bits.addr := waddr_reg + (block_idx_reg << 1) + 1.U - io.bankWrite(0).io.req.bits.data := outWordHi - io.bankWrite(0).io.req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(1.U(1.W)) - ) - when(io.bankWrite(0).io.req.fire) { - state := sWriteHiResp - } - } - - is(sWriteHiResp) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - when(block_idx_reg === iter_reg - 1.U) { - state := sComplete - }.otherwise { - block_idx_reg := block_idx_reg + 1.U - state := sReadReq - } - } - } - - is(sComplete) { - io.cmdResp.valid := true.B - when(io.cmdResp.fire) { - state := idle - } - } - } - - io.status.idle := state === idle - io.status.running := state =/= idle -} diff --git a/examples/balls/mxfp2int/arch/src/main/scala/Mxfp2IntBall.scala b/examples/balls/mxfp2int/arch/src/main/scala/Mxfp2IntBall.scala deleted file mode 100644 index 637ca9c66..000000000 --- a/examples/balls/mxfp2int/arch/src/main/scala/Mxfp2IntBall.scala +++ /dev/null @@ -1,57 +0,0 @@ -package examples.balls.mxfp2int - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} - -import framework.balldomain.blink.{BlinkIO, HasBlink, SubRobRow} -import framework.top.GlobalConfig - -/** - * Mxfp2IntBall - MXFP4 to INT8 dequantizer. - * - * Wraps PipelinedMxfp2Int and connects to the Blink protocol, - * including the MMIO read channel for per-block E8M0 scales. - */ -@instantiable -class Mxfp2IntBall(val b: GlobalConfig) extends Module with HasBlink { - - val ballCommonConfig = b.ballDomain.ballIdMappings - .find(_.ballName == "Mxfp2IntBall") - .getOrElse( - throw new IllegalArgumentException("Mxfp2IntBall not found in config") - ) - - val inBW = ballCommonConfig.inBW - val outBW = ballCommonConfig.outBW - val mmioReadBW = ballCommonConfig.mmioReadBW - require(mmioReadBW == 1, "Mxfp2IntBall requires exactly one MMIO read line") - - @public - val io = IO(new BlinkIO(b, inBW, outBW, mmioReadBW)) - - def blink: BlinkIO = io - dontTouch(io) - - val mxfp2intUnit: Instance[PipelinedMxfp2Int] = Instantiate( - new PipelinedMxfp2Int(b) - ) - - mxfp2intUnit.io.cmdReq <> io.cmdReq - mxfp2intUnit.io.cmdResp <> io.cmdResp - - for (i <- 0 until inBW) { - mxfp2intUnit.io.bankRead(i) <> io.bankRead(i) - } - - for (i <- 0 until outBW) { - mxfp2intUnit.io.bankWrite(i) <> io.bankWrite(i) - } - - io.status <> mxfp2intUnit.io.status - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - - io.mmioRead(0) <> mxfp2intUnit.io.mmioRead -} diff --git a/examples/balls/mxfp2int/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/mxfp2int/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 4a5c3f6d8..000000000 --- a/examples/balls/mxfp2int/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateMxfp2IntBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { -class BankMxfp2IntPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankMxfp2IntOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), - op.getIter(), op.getSpecial()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateMxfp2IntBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/mxfp2int/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/mxfp2int/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 86ea32318..000000000 --- a/examples/balls/mxfp2int/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -class Mxfp2IntLowering : public ConvertOpToLLVMPattern { -public: - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(Mxfp2IntOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("Mxfp2IntBall"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getSpecial(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MXFP2INT"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateMxfp2IntBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - (void)stable; - (void)bankDepth; - - patterns.add(converter); -} - -void configureMxfp2IntBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/mxfp2int/configs/Mxfp2IntBallParam.scala b/examples/balls/mxfp2int/configs/Mxfp2IntBallParam.scala deleted file mode 100644 index b8d2a71e7..000000000 --- a/examples/balls/mxfp2int/configs/Mxfp2IntBallParam.scala +++ /dev/null @@ -1,17 +0,0 @@ -package examples.balls.mxfp2int.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig - -case class Mxfp2IntBallParam( - mxfpFormat: String) - -object Mxfp2IntBallParam { - private val ballName = "Mxfp2IntBall" - - def apply(b: GlobalConfig): Mxfp2IntBallParam = { - val tbl = BallParamLoader.ballTable(b, ballName) - Mxfp2IntBallParam(mxfpFormat = BallParamLoader.str(tbl, "mxfpFormat")) - } - -} diff --git a/examples/balls/mxfp2int/configs/default.toml b/examples/balls/mxfp2int/configs/default.toml deleted file mode 100644 index 63342a459..000000000 --- a/examples/balls/mxfp2int/configs/default.toml +++ /dev/null @@ -1,4 +0,0 @@ -# Mxfp2Int Ball Configuration - -[ball] -mxfpFormat = "MXFP4" diff --git a/examples/balls/mxfp2int/emu/src/dispatch.rs b/examples/balls/mxfp2int/emu/src/dispatch.rs deleted file mode 100644 index a38a68451..000000000 --- a/examples/balls/mxfp2int/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::f55_mxfp2int; - -const BALL_CLASS: &str = "examples.balls.mxfp2int.Mxfp2IntBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MXFP2INT") => Some(f55_mxfp2int::Mxfp2Int::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MXFP2INT") => Some(f55_mxfp2int::Mxfp2Int::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/mxfp2int/emu/src/lib.rs b/examples/balls/mxfp2int/emu/src/lib.rs deleted file mode 100644 index b35b3155f..000000000 --- a/examples/balls/mxfp2int/emu/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -#[path = "55_mxfp2int.rs"] -mod f55_mxfp2int; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/mxfp2int/workloads/ctests/CMakeLists.txt b/examples/balls/mxfp2int/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 635814a36..000000000 --- a/examples/balls/mxfp2int/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_buckyball_ctests( - mxfp4_e8m0_two_block_test.c -) diff --git a/examples/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index e44f42b29..000000000 --- a/examples/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) -buckyball_require_mxfp2int_bank_shape(${BUCKYBALL_MLIR_ACTIVE_TARGET}) -add_buckyball_mlir_test(mxfp4_unit_scale_bank TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/mxmm/arch/src/main/scala/Array.scala b/examples/balls/mxmm/arch/src/main/scala/Array.scala deleted file mode 100644 index d539c6847..000000000 --- a/examples/balls/mxmm/arch/src/main/scala/Array.scala +++ /dev/null @@ -1,51 +0,0 @@ -package examples.balls.mxmm - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import examples.balls.mxmm.configs.MxmmBallParam -import hardfloat._ - -@instantiable -class Array(p: MxmmBallParam) extends Module { - - @public - val io = IO(new Bundle { - val valid = Input(Bool()) - val context = Input(UInt(log2Ceil(p.contexts).W)) - val separate = Input(Bool()) - val vector = Input(Bool()) - val a = Input(Vec(p.tileRows, UInt(32.W))) - val b = Input(Vec(p.tileCols, UInt(32.W))) - val load = Input(Bool()) - val row = Input(UInt(4.W)) - val rowData = Input(UInt(512.W)) - val rowOut = Output(UInt(512.W)) - val completed = Output(Bool()) - }) - - val accumulators = Reg(Vec(p.contexts, Vec(p.tileRows, Vec(p.tileCols, UInt(33.W))))) - when(io.load) { - for (col <- 0 until p.tileCols) { - accumulators(io.context)(io.row)(col) := recFNFromFN(8, 24, io.rowData(32 * col + 31, 32 * col)) - } - } - io.rowOut := Cat((0 until p.tileCols).reverse.map(col => fNFromRecFN(8, 24, accumulators(io.context)(io.row)(col)))) - val fusedTag = Pipe(io.valid && !io.separate, io.context, p.arithmeticLatency) - val separateTag = Pipe(io.valid && io.separate, io.context, 2 * p.arithmeticLatency) - io.completed := fusedTag.valid || separateTag.valid - for { - row <- 0 until p.tileRows - col <- 0 until p.tileCols - } { - val pe = Instantiate(new PE(p.arithmeticLatency)) - pe.io.valid := io.valid && (if (row == 0) true.B else !io.vector) - pe.io.separate := io.separate - pe.io.a := io.a(row) - pe.io.b := io.b(col) - pe.io.c := accumulators(io.context)(row)(col) - when(pe.io.outValid) { - accumulators(Mux(separateTag.valid, separateTag.bits, fusedTag.bits))(row)(col) := pe.io.out - } - } -} diff --git a/examples/balls/mxmm/arch/src/main/scala/Emit.scala b/examples/balls/mxmm/arch/src/main/scala/Emit.scala deleted file mode 100644 index be7271ad2..000000000 --- a/examples/balls/mxmm/arch/src/main/scala/Emit.scala +++ /dev/null @@ -1,36 +0,0 @@ -package examples.balls.mxmm - -import framework.top.GlobalConfig -import framework.balldomain.configs.{BallDomainParam, BallISAEntry, BallIdMapping} - -object Emit extends App { - require(args.length == 2, "Emit requires OUTPUT_DIRECTORY BALL_CONFIG") - val defaults = GlobalConfig() - - val b = defaults.copy( - memDomain = defaults.memDomain.copy( - bankNum = 8, - bankWidth = 128, - bankEntries = 1024, - bankMaskLen = 16, - virtualBankCount = 8, - sharedBankNum = 8, - mmioBankNum = 8, - mmioBankEntries = 1024, - mmioBankWidth = 128, - mmioReadWidth = 16 - ), - frontend = defaults.frontend.copy(rob_entries = 16, sub_rob_depth = 8, iter_len = 34), - ballDomain = BallDomainParam( - 1, - Seq(BallIdMapping(0, "MxmmBall", "examples.balls.mxmm.MxmmBall", Some(args(1)), 2, 1)), - Seq(BallISAEntry("MXMM_MXFP8", 71, 0), BallISAEntry("MXMM_FMA32", 72, 0), BallISAEntry("MXMM_F32", 73, 0)) - ) - ) - - _root_.circt.stage.ChiselStage.emitSystemVerilogFile( - new MxmmBall(b), - args = _root_.scala.Array("--target-dir", args(0), "--split-verilog"), - firtoolOpts = _root_.scala.Array("--disable-annotation-unknown", "--strip-debug-info") - ) -} diff --git a/examples/balls/mxmm/arch/src/main/scala/InputWindow.scala b/examples/balls/mxmm/arch/src/main/scala/InputWindow.scala deleted file mode 100644 index bc509cec8..000000000 --- a/examples/balls/mxmm/arch/src/main/scala/InputWindow.scala +++ /dev/null @@ -1,82 +0,0 @@ -package examples.balls.mxmm - -import chisel3._ -import chisel3.util._ -import framework.memdomain.backend.banks.{SramReadReq, SramReadResp} -import framework.top.GlobalConfig - -class InputWindow(b: GlobalConfig) extends Module { - private val addressWidth = log2Ceil(b.memDomain.bankEntries) - - val io = IO(new Bundle { - val start = Input(Bool()) - val enable = Input(Bool()) - val rows = Input(UInt(12.W)) - val count = Input(UInt(34.W)) - val fullK = Input(UInt(16.W)) - val startK = Input(UInt(16.W)) - val request = Decoupled(new SramReadReq(b)) - val response = Flipped(Decoupled(new SramReadResp(b))) - val done = Output(Bool()) - val write = Output(Bool()) - val scaleWrite = Output(Bool()) - val lane = Output(UInt(4.W)) - val address = Output(UInt(addressWidth.W)) - val word = Output(UInt(128.W)) - val scale = Output(UInt(8.W)) - }) - - private val request :: response :: scales :: done :: Nil = Enum(4) - private val state = RegInit(done) - private val rows = Reg(UInt(12.W)) - private val count = Reg(UInt(34.W)) - private val fullK = Reg(UInt(16.W)) - private val startK = Reg(UInt(16.W)) - private val row = Reg(UInt(12.W)) - private val offset = Reg(UInt(addressWidth.W)) - private val scaleMode = Reg(Bool()) - private val word = Reg(UInt(128.W)) - private val scaleAddress = rows * fullK + row * (fullK >> 5) + (startK >> 5) + offset - private val byte = scaleAddress(3, 0) - io.request.valid := io.enable && state === request - io.request.bits.addr := Mux(scaleMode, scaleAddress >> 4, row * (fullK >> 4) + (startK >> 4) + offset) - io.response.ready := io.enable && state === response - io.done := state === done - io.write := io.response.fire && !scaleMode - io.scaleWrite := io.enable && state === scales - io.lane := row(3, 0) - io.address := (row >> 4) * Mux(scaleMode, count >> 5, count >> 4) + offset - io.word := io.response.bits.data - io.scale := (word >> (byte << 3))(7, 0) - when(io.start) { - rows := io.rows - count := io.count - fullK := io.fullK - startK := io.startK - row := 0.U - offset := 0.U - scaleMode := false.B - state := request - } - when(io.request.fire)(state := response) - when(io.response.fire) { - when(scaleMode) { word := io.response.bits.data; state := scales }.otherwise { - when(offset + 1.U === (count >> 4)) { - offset := 0.U - when(row + 1.U === rows) { row := 0.U; scaleMode := true.B } - .otherwise(row := row + 1.U) - }.otherwise(offset := offset + 1.U) - state := request - } - } - when(io.enable && state === scales) { - when(offset + 1.U === (count >> 5)) { - offset := 0.U - when(row + 1.U === rows)(state := done) - .otherwise { row := row + 1.U; state := request } - }.otherwise { - offset := offset + 1.U - when(byte === 15.U)(state := request) - } - } -} diff --git a/examples/balls/mxmm/arch/src/main/scala/MxmmBall.scala b/examples/balls/mxmm/arch/src/main/scala/MxmmBall.scala deleted file mode 100644 index 847866b8b..000000000 --- a/examples/balls/mxmm/arch/src/main/scala/MxmmBall.scala +++ /dev/null @@ -1,26 +0,0 @@ -package examples.balls.mxmm - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.top.GlobalConfig -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} - -@instantiable -class MxmmBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - @public - val io = IO(new BlinkIO(b, 2, 1)) - def blink: BlinkIO = io - def status: BallStatus = io.status - val unit = Instantiate(new Unit(b)) - unit.io.cmdReq <> io.cmdReq - unit.io.cmdResp <> io.cmdResp - unit.io.channelReady := io.channelReady - unit.io.bankRead <> io.bankRead - unit.io.bankWrite <> io.bankWrite - io.status <> unit.io.status - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) -} diff --git a/examples/balls/mxmm/arch/src/main/scala/PE.scala b/examples/balls/mxmm/arch/src/main/scala/PE.scala deleted file mode 100644 index ca89bc1c6..000000000 --- a/examples/balls/mxmm/arch/src/main/scala/PE.scala +++ /dev/null @@ -1,48 +0,0 @@ -package examples.balls.mxmm - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import freechips.rocketchip.tile.MulAddRecFNPipe -import hardfloat._ - -@instantiable -class PE(latency: Int) extends Module { - require(latency == 3) - - @public - val io = IO(new Bundle { - val valid = Input(Bool()) - val separate = Input(Bool()) - val a = Input(UInt(32.W)) - val b = Input(UInt(32.W)) - val c = Input(UInt(33.W)) - val out = Output(UInt(33.W)) - val outValid = Output(Bool()) - }) - - val multiply = Module(new MulAddRecFNPipe(latency, 8, 24)) - multiply.io.validin := io.valid - multiply.io.op := 0.U - multiply.io.a := recFNFromFN(8, 24, io.a) - multiply.io.b := recFNFromFN(8, 24, io.b) - multiply.io.c := Mux(io.separate, Cat(io.a(31) ^ io.b(31), 0.U(32.W)), io.c) - multiply.io.roundingMode := consts.round_near_even - multiply.io.detectTininess := consts.tininess_afterRounding.asUInt - - val productMode = Pipe(io.valid, io.separate, latency) - val previous = Pipe(io.valid && io.separate, io.c, latency) - val add = Instantiate(new AddPipe) - add.io.valid := multiply.io.validout && productMode.bits - add.io.a := multiply.io.out - add.io.b := previous.bits - when(add.io.valid) { - assert(fNFromRecFN(8, 24, multiply.io.out)(30, 23) =/= 255.U, "MXMM_F32 product must remain finite") - } - val fusedValid = multiply.io.validout && !productMode.bits - io.outValid := fusedValid || add.io.outValid - io.out := Mux(add.io.outValid, add.io.out, multiply.io.out) - when(io.outValid) { - assert(fNFromRecFN(8, 24, io.out)(30, 23) =/= 255.U, "MATMUL accumulation must remain finite") - } -} diff --git a/examples/balls/mxmm/arch/src/main/scala/Unit.scala b/examples/balls/mxmm/arch/src/main/scala/Unit.scala deleted file mode 100644 index 1174774ee..000000000 --- a/examples/balls/mxmm/arch/src/main/scala/Unit.scala +++ /dev/null @@ -1,377 +0,0 @@ -package examples.balls.mxmm - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.top.GlobalConfig -import examples.balls.mxmm.configs.MxmmBallParam - -@instantiable -class Unit(b: GlobalConfig) extends Module { - private val p = MxmmBallParam(b) - private val entries = b.memDomain.bankEntries - private val addressWidth = log2Ceil(entries) - require(b.memDomain.bankWidth == 128 && b.memDomain.bankMaskLen == 16) - require(entries >= 64 && entries % 32 == 0) - private val mapping = b.ballDomain.ballIdMappings.find(_.ballName == "MxmmBall").get - require(mapping.inBW == 2 && mapping.outBW == 1) - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(2, Flipped(new BankRead(b))) - val bankWrite = Vec(1, Flipped(new BankWrite(b))) - val channelReady = Input(Bool()) - val status = new BallStatus - }) - - private val Seq( - idle, - waitChannels, - loadInputs, - initRequest, - initResponse, - compute, - drain, - store, - outputRequest, - outputResponse, - outputWrite, - outputAck, - complete - ) = Enum(13) - - private val state = RegInit(idle) - private val command = Reg(new BallRsIssue(b)) - private val m = Reg(UInt(12.W)) - private val n = Reg(UInt(12.W)) - private val k = Reg(UInt(34.W)) - private val mx = Reg(Bool()) - private val windowed = Reg(Bool()) - private val separate = Reg(Bool()) - private val first = Reg(Bool()) - private val last = Reg(Bool()) - private val base = Reg(UInt(6.W)) - private val chain = RegInit(false.B) - private val chainM = Reg(UInt(12.W)) - private val chainN = Reg(UInt(12.W)) - private val chainFormat = Reg(UInt(7.W)) - private val chainBank = Reg(UInt(b.memDomain.vbankIdWidth.W)) - private val chainBase = Reg(UInt(6.W)) - - private val nGroups = n >> 4 - private val panelHeight = Mux(m === 1.U, 1.U, 16.U) - private val totalPanels = Mux(m === 1.U, 1.U, m >> 4) * nGroups - private val batch = Reg(UInt(addressWidth.W)) - private val count = Mux(totalPanels - batch > p.contexts.U, p.contexts.U, totalPanels - batch) - private val minimumPeriod = Mux(separate, (2 * p.arithmeticLatency + 1).U, (p.arithmeticLatency + 1).U) - private val period = Mux(count > minimumPeriod, count, minimumPeriod) - private val context = Reg(UInt(log2Ceil(p.contexts).W)) - private val row = Reg(UInt(4.W)) - private val reduction = Reg(UInt(34.W)) - private val completed = Reg(UInt(48.W)) - private val panel = batch + context - private val rowGroup = panel / nGroups - private val columnGroup = panel % nGroups - private val accAddress = rowGroup * 16.U * nGroups + row * nGroups + columnGroup - private val accumulator = SyncReadMem(entries / 4, UInt(512.W)) - private val savedRow = Reg(UInt(512.W)) - private val outputLine = Reg(UInt(addressWidth.W)) - private val outputQuarter = Reg(UInt(2.W)) - - private val panels = Instantiate(new Panels(entries)) - private val array = Instantiate(new Array(p)) - private val readValid = state === compute && context < count - panels.io.read := readValid - panels.io.rowGroup := rowGroup - panels.io.columnGroup := columnGroup - panels.io.k := reduction - panels.io.reduction := k - panels.io.mxfp8 := mx - panels.io.vector := m === 1.U - array.io.valid := panels.io.valid - array.io.a := panels.io.a - array.io.b := panels.io.b - array.io.separate := separate - array.io.vector := m === 1.U - val computeContext = RegNext(context) - array.io.context := Mux(state === compute || state === drain, computeContext, context) - array.io.row := row - array.io.load := false.B - array.io.rowData := 0.U - - private val accReadEnable = state === outputRequest || (state === initRequest && !first && row < panelHeight) - private val accWriteEnable = state === store && row < panelHeight - - private val accRead = accumulator.readWrite( - Mux(state === outputRequest, outputLine, accAddress), - array.io.rowOut, - accReadEnable || accWriteEnable, - accWriteEnable - ) - - assert(!(accReadEnable && accWriteEnable)) - - private val inputState = Reg(Vec(2, UInt(2.W))) - private val inputWord = Reg(Vec(2, UInt(128.W))) - private val inputIndex = Reg(Vec(2, UInt(addressWidth.W))) - private val inputRow = Reg(Vec(2, UInt(12.W))) - private val rowOffset = Reg(Vec(2, UInt(addressWidth.W))) - private val scaleMode = Reg(Vec(2, Bool())) - private val scaleByte = Reg(Vec(2, UInt(4.W))) - private val scaleCount = Reg(Vec(2, UInt((addressWidth + 4).W))) - private val rowWords = Mux(mx, k >> 4, k >> 2) - private val window = Module(new InputWindow(b)) - window.io.start := io.cmdReq.fire && io.cmdReq.bits.cmd.funct7 === 75.U - window.io.enable := state === loadInputs && windowed - window.io.rows := io.cmdReq.bits.cmd.rs2(11, 0) - window.io.count := io.cmdReq.bits.cmd.rs1(63, 30) - window.io.fullK := io.cmdReq.bits.cmd.rs2(47, 32) - window.io.startK := io.cmdReq.bits.cmd.rs2(63, 48) - window.io.request.ready := io.bankRead(0).io.req.ready && windowed - window.io.response.valid := io.bankRead(0).io.resp.valid && windowed - window.io.response.bits := io.bankRead(0).io.resp.bits - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := command.rob_id - io.cmdResp.bits.is_sub := command.is_sub - io.cmdResp.bits.sub_rob_id := command.sub_rob_id - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - for (operand <- 0 until 2) { - val rows = if (operand == 0) m else n - val codeWords = rows * rowWords - val scales = rows * (k >> 5) - val port = io.bankRead(operand) - port.rob_id := command.rob_id - port.ball_id := 0.U - port.group_id := 0.U - port.bank_id := (if (operand == 0) command.cmd.op1_bank else command.cmd.op2_bank) - port.io.req.valid := state === loadInputs && inputState(operand) === 0.U - port.io.req.bits.addr := inputIndex(operand) - port.io.resp.ready := state === loadInputs && inputState(operand) === 1.U - panels.io.write(operand) := port.io.resp.fire && !scaleMode(operand) - panels.io.scaleWrite(operand) := state === loadInputs && inputState(operand) === 2.U - panels.io.lane(operand) := inputRow(operand)(3, 0) - panels.io.address(operand) := (inputRow(operand) >> 4) * Mux(scaleMode(operand), k >> 5, rowWords) + rowOffset( - operand - ) - panels.io.word(operand) := port.io.resp.bits.data - panels.io.scale(operand) := (inputWord(operand) >> (scaleByte(operand) << 3))(7, 0) - val legacy = if (operand == 0) !windowed else true.B - when(port.io.req.fire && legacy)(inputState(operand) := 1.U) - when(port.io.resp.fire && legacy) { - when(scaleMode(operand)) { - inputWord(operand) := port.io.resp.bits.data - scaleByte(operand) := 0.U - inputState(operand) := 2.U - }.otherwise { - inputIndex(operand) := inputIndex(operand) + 1.U - when(rowOffset(operand) + 1.U === rowWords) { - rowOffset(operand) := 0.U - inputRow(operand) := inputRow(operand) + 1.U - }.otherwise(rowOffset(operand) := rowOffset(operand) + 1.U) - when(inputIndex(operand) +& 1.U === codeWords) { - when(mx) { - scaleMode(operand) := true.B - inputRow(operand) := 0.U - rowOffset(operand) := 0.U - inputState(operand) := 0.U - }.otherwise(inputState(operand) := 3.U) - }.otherwise(inputState(operand) := 0.U) - } - } - when(state === loadInputs && inputState(operand) === 2.U && legacy) { - scaleCount(operand) := scaleCount(operand) + 1.U - scaleByte(operand) := scaleByte(operand) + 1.U - when(rowOffset(operand) + 1.U === (k >> 5)) { - rowOffset(operand) := 0.U - inputRow(operand) := inputRow(operand) + 1.U - }.otherwise(rowOffset(operand) := rowOffset(operand) + 1.U) - when(scaleCount(operand) + 1.U === scales) { - inputState(operand) := 3.U - }.elsewhen(scaleByte(operand) === 15.U) { - inputIndex(operand) := inputIndex(operand) + 1.U - inputState(operand) := 0.U - } - } - } - - when(windowed) { - io.bankRead(0).io.req.valid := window.io.request.valid - io.bankRead(0).io.req.bits.addr := window.io.request.bits.addr - io.bankRead(0).io.resp.ready := window.io.response.ready - panels.io.write(0) := window.io.write - panels.io.scaleWrite(0) := window.io.scaleWrite - panels.io.lane(0) := window.io.lane - panels.io.address(0) := window.io.address - panels.io.word(0) := window.io.word - panels.io.scale(0) := window.io.scale - } - - private val writePort = io.bankWrite(0) - writePort.rob_id := command.rob_id - writePort.ball_id := 0.U - writePort.group_id := 0.U - writePort.bank_id := command.cmd.wr_bank - writePort.io.req.valid := state === outputWrite - writePort.io.req.bits.addr := base + (outputLine << 2) + outputQuarter - writePort.io.req.bits.data := (savedRow >> (outputQuarter << 7))(127, 0) - writePort.io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) - writePort.io.resp.ready := state === outputAck - - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - val rows = cmd.rs2(11, 0) - val cols = cmd.rs2(23, 12) - val inputK = cmd.rs1(63, 30) - val isWindow = cmd.funct7 === 75.U - val isMx = cmd.funct7 === 71.U || isWindow - assert(isMx || cmd.funct7 === 72.U || cmd.funct7 === 73.U, "MATMUL format is invalid") - assert(rows === 1.U || (rows =/= 0.U && rows(3, 0) === 0.U), "MATMUL M must be one or a positive multiple of 16") - assert(cols =/= 0.U && cols(3, 0) === 0.U, "MATMUL N must be a positive multiple of 16") - assert(inputK =/= 0.U && Mux(isMx, inputK(4, 0) === 0.U, inputK(1, 0) === 0.U), "MATMUL K alignment is invalid") - when(isWindow) { - val full = cmd.rs2(47, 32) - val start = cmd.rs2(63, 48) - assert(full =/= 0.U && full(4, 0) === 0.U && start(4, 0) === 0.U, "MATMUL A window alignment is invalid") - assert(start +& inputK <= full, "MATMUL A window exceeds fullK") - }.otherwise(assert(cmd.rs2(63, 32) === 0.U, "MATMUL reserves rs2[63:32]")) - assert( - cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, - "MATMUL requires distinct banks" - ) - assert( - cmd.op1_col === 1.U && cmd.op2_col === 1.U && cmd.wr_col === 1.U, - "MATMUL requires one physical bank per operand" - ) - assert(cmd.rs1(9, 0) < b.memDomain.virtualBankCount.U && cmd.rs1(19, 10) < b.memDomain.virtualBankCount.U && cmd.rs1( - 29, - 20 - ) < b.memDomain.virtualBankCount.U) - val aElements = rows * Mux(isWindow, cmd.rs2(47, 32), inputK) - val bElements = cols * inputK - assert(Mux(isMx, aElements + (aElements >> 5), aElements << 2) <= (entries * 16).U, "MATMUL A exceeds bank capacity") - assert(Mux(isMx, bElements + (bElements >> 5), bElements << 2) <= (entries * 16).U, "MATMUL B exceeds bank capacity") - assert((cmd.rs2(31, 26) << 4) +& ((rows * cols) << 2) <= (entries * 16).U, "MATMUL C exceeds bank capacity") - when(cmd.rs2(24)) { - assert(!chain, "MATMUL first cannot replace a live chain") - chain := true.B - chainM := rows - chainN := cols - chainFormat := cmd.funct7 - chainBank := cmd.wr_bank - chainBase := cmd.rs2(31, 26) - }.otherwise { - assert( - chain && chainM === rows && chainN === cols && chainFormat === cmd.funct7 && chainBank === cmd.wr_bank && chainBase === cmd.rs2( - 31, - 26 - ), - "MATMUL continuation changed its chain" - ) - } - command := io.cmdReq.bits - m := rows - n := cols - k := inputK - mx := isMx - windowed := isWindow - separate := cmd.funct7 === 73.U - first := cmd.rs2(24) - last := cmd.rs2(25) - base := cmd.rs2(31, 26) - batch := 0.U - context := 0.U - row := 0.U - reduction := 0.U - completed := 0.U - for (operand <- 0 until 2) { - inputState(operand) := 0.U - inputIndex(operand) := 0.U - inputRow(operand) := 0.U - rowOffset(operand) := 0.U - scaleMode(operand) := false.B - scaleCount(operand) := 0.U - } - state := waitChannels - } - - switch(state) { - is(waitChannels)(when(io.channelReady)(state := loadInputs)) - is(loadInputs)( - when(Mux(windowed, window.io.done, inputState(0) === 3.U) && inputState(1) === 3.U)(state := initRequest) - ) - is(initRequest) { - when(first || row >= panelHeight) { - array.io.load := true.B - when(row === 15.U) { - row := 0.U - when(context + 1.U === count) { context := 0.U; state := compute } - .otherwise(context := context + 1.U) - }.otherwise(row := row + 1.U) - }.otherwise(state := initResponse) - } - is(initResponse) { - array.io.load := true.B - array.io.rowData := accRead - when(row === 15.U) { - row := 0.U - when(context + 1.U === count) { context := 0.U; state := compute } - .otherwise { context := context + 1.U; state := initRequest } - }.otherwise { row := row + 1.U; state := initRequest } - } - is(compute) { - when(context + 1.U === period) { - context := 0.U - when(reduction + 1.U === k)(state := drain) - .otherwise(reduction := reduction + 1.U) - }.otherwise(context := context + 1.U) - } - is(store) { - when(row === 15.U) { - row := 0.U - when(context + 1.U === count) { - context := 0.U - when(batch + count === totalPanels) { - when(last) { outputLine := 0.U; outputQuarter := 0.U; state := outputRequest } - .otherwise(state := complete) - }.otherwise { - batch := batch + count - reduction := 0.U - completed := 0.U - state := initRequest - } - }.otherwise(context := context + 1.U) - }.otherwise(row := row + 1.U) - } - is(outputRequest)(state := outputResponse) - is(outputResponse) { savedRow := accRead; state := outputWrite } - is(outputWrite)(when(writePort.io.req.fire)(state := outputAck)) - is(outputAck) { - when(writePort.io.resp.fire) { - assert(writePort.io.resp.bits.ok, "MATMUL bank write failed") - when(outputQuarter === 3.U) { - outputQuarter := 0.U - when(outputLine + 1.U === m * nGroups) { chain := false.B; state := complete } - .otherwise { outputLine := outputLine + 1.U; state := outputRequest } - }.otherwise { outputQuarter := outputQuarter + 1.U; state := outputWrite } - } - } - is(complete)(when(io.cmdResp.fire)(state := idle)) - } - when(array.io.completed) { - assert(state === compute || state === drain) - completed := completed + 1.U - when(completed + 1.U === count * k) { - context := 0.U - row := 0.U - state := store - } - } -} diff --git a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 02ac7257f..000000000 --- a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "mlir/IR/PatternMatch.h" -using namespace mlir; -using namespace ::buddy::buckyball; -namespace { -class BankMXFP8Pattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankMXFP8Op op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), - op.getWrBank(), op.getConfig(), op.getFirst(), - op.getLast(), op.getOutputBase()); - rewriter.replaceOp(op, op.getWrBank()); - return success(); - } -}; - -class BankMXFP8WindowPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(BankMXFP8WindowOp op, - PatternRewriter &rewriter) const override { - rewriter.create( - op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), op.getWrBank(), - op.getRows(), op.getCols(), op.getCount(), op.getFullK(), - op.getStartK(), op.getFirst(), op.getLast(), op.getOutputBase()); - rewriter.replaceOp(op, op.getWrBank()); - return success(); - } -}; - -class BankFP32Pattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankFP32Op op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), - op.getWrBank(), op.getConfig(), op.getFirst(), - op.getLast(), op.getOutputBase(), - op.getFusedAttr()); - rewriter.replaceOp(op, op.getWrBank()); - return success(); - } -}; - -} // namespace -namespace mlir::buddy { -void populateMxmmBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &) { - patterns.add( - patterns.getContext()); -} - -} // namespace mlir::buddy diff --git a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/FP32Patterns.inc b/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/FP32Patterns.inc deleted file mode 100644 index b77d73a1d..000000000 --- a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/FP32Patterns.inc +++ /dev/null @@ -1,337 +0,0 @@ -class FP32ToBanks : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(FP32MemMatmulOp op, - PatternRewriter &b) const override { - auto a = cast(op.getLhs().getType()), - w = cast(op.getRhs().getType()), - out = cast(op.getOutput().getType()); - int rank = a.getRank(); - if ((rank != 2 && rank != 3) || w.getRank() != rank || - out.getRank() != rank || !a.hasStaticShape() || !w.hasStaticShape() || - !out.hasStaticShape() || !a.getElementType().isF32() || - !w.getElementType().isF32() || !out.getElementType().isF32()) - return op.emitError( - "FP32 matmul requires static rank-2 or rank-3 FP32 operands"); - int64_t M = a.getShape()[rank - 2], K = a.getShape()[rank - 1], - N = w.getShape()[rank - 1], - batches = rank == 3 ? a.getShape()[0] : 1; - if (M <= 0 || N <= 0 || K <= 0 || K % 4 || w.getShape()[rank - 2] != K || - out.getShape()[rank - 2] != M || out.getShape()[rank - 1] != N || - (rank == 3 && - (w.getShape()[0] != batches || out.getShape()[0] != batches))) - return op.emitError( - "FP32 matmul shape mismatch or K not divisible by four"); - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128) - return op.emitError("FP32 matmul requires 128-bit bank rows"); - int64_t bytes = target.bankDepth * 16, rows = 0, columns = 0, chunk = 0; - std::tuple best{INT64_MAX, 0, 0, 0}; - for (int64_t m = M == 1 ? 1 : 16; m <= (M == 1 ? 1 : (M + 15) / 16 * 16); - m += 16) - for (int64_t n = 16; n <= (N + 15) / 16 * 16; n += 16) { - if (m * n * 4 > bytes) - continue; - int64_t k = std::min(K, bytes / (4 * std::max(m, n))) / 4 * 4; - if (!k) - continue; - auto score = std::tuple{((M + m - 1) / m) * ((N + n - 1) / n) * - ((K + k - 1) / k), - -k, -m, -n}; - if (score < best) { - best = score; - rows = m; - columns = n; - chunk = k; - } - } - if (!rows) - return op.emitError("FP32 matmul cannot fit a bank block"); - SmallVector rhsStrides; - int64_t rhsOffset; - bool bankPack = - target.bankNum >= 5 && target.rvvEnabled && - succeeded(w.getStridesAndOffset(rhsStrides, rhsOffset)) && - rhsStrides[rank - 2] == N && rhsStrides.back() == 1 && - K * N * 4 <= bytes && chunk == K && rows * columns * 4 >= 64; - Location loc = op.getLoc(); - auto idx = [&](int64_t v) -> Value { - return b.create(loc, v); - }; - Value zero = idx(0), one = idx(1), four = idx(4); - Value ab = allocBank(b, loc, 1, 1), wb = allocBank(b, loc, 1, 1), - cb = allocBank(b, loc, 1, 1); - auto packType = MemRefType::get({bytes / 16, 4}, b.getF32Type()); - Value ap = b.create(loc, packType), - bp = b.create(loc, packType), - cp = b.create(loc, packType); - for (Value buffer : {ap, bp, cp}) - buffer.getDefiningOp()->setAttr("alignment", b.getI64IntegerAttr(bytes)); - SmallVector banks{ab, wb, cb}; - if (bankPack) { - Value input = allocBank(b, loc, 1, 1); - Value descriptor = allocBank(b, loc, 1, 1); - Value read = b.create(loc, b.getI64Type(), ab, input); - Value write = b.create(loc, b.getI64Type(), wb, descriptor); - write = b.create(loc, b.getI64Type(), cb, write); - auto loaded = b.create( - loc, b.getI64Type(), b.getI64Type(), read, write, - FlatSymbolRefAttr::get(b.getContext(), "rvv_packmm"), ValueRange{ - createI64Const(b, loc, 0), createI64Const(b, loc, K), - createI64Const(b, loc, N), createI64Const(b, loc, 0), - createI64Const(b, loc, 0), createI64Const(b, loc, K), - createI64Const(b, loc, columns), createI64Const(b, loc, rows), - createI64Const(b, loc, 1), createI64Const(b, loc, 1), - createI64Const(b, loc, op.getFusedAttr().getValue())}); - banks = {loaded.getReadOut(), loaded.getWriteOut()}; - } - auto batchLoop = b.create(loc, zero, idx(batches), one, banks); - b.setInsertionPointToStart(batchLoop.getBody()); - Value batch = batchLoop.getInductionVar(); - SmallVector batchBanks(batchLoop.getRegionIterArgs()); - if (bankPack) { - SmallVector offsets(rank, b.getIndexAttr(0)); - SmallVector sizes; - for (int64_t dim : w.getShape()) - sizes.push_back(b.getIndexAttr(dim)); - SmallVector strides(rank, b.getIndexAttr(1)); - if (rank == 3) { - offsets[0] = batch; - sizes[0] = b.getIndexAttr(1); - } - auto rowType = memref::SubViewOp::inferRankReducedResultType( - {K, N}, w, offsets, sizes, strides); - Value source = b.create(loc, rowType, op.getRhs(), - offsets, sizes, strides); - Value flat = b.create( - loc, source, ArrayRef{{0, 1}}); - Value packed = b.create( - loc, ArrayRef{K * N / 4, 4}, flat, - ArrayRef{{0, 1}}); - batchBanks[0] = b.create( - loc, b.getI64Type(), packed, batchBanks[0], - createI64Const(b, loc, K * N / 4), createI64Const(b, loc, 1), - b.getI64IntegerAttr(0)); - } - auto rowLoop = - b.create(loc, zero, idx(M), idx(rows), batchBanks); - b.setInsertionPointToStart(rowLoop.getBody()); - Value row = rowLoop.getInductionVar(); - auto colLoop = b.create(loc, zero, idx(N), idx(columns), - rowLoop.getRegionIterArgs()); - b.setInsertionPointToStart(colLoop.getBody()); - Value column = colLoop.getInductionVar(); - ab = colLoop.getRegionIterArgs()[0]; - wb = colLoop.getRegionIterArgs()[1]; - cb = bankPack ? wb : colLoop.getRegionIterArgs()[2]; - for (int64_t begin = 0; begin < K; begin += chunk) { - int64_t count = std::min(chunk, K - begin); - for (int operand = 0; operand < (bankPack ? 1 : 2); ++operand) { - Value source = operand ? op.getRhs() : op.getLhs(); - Value packed = operand ? bp : ap; - auto sourceType = cast(source.getType()); - int axis = operand ? rank - 2 : rank - 1; - SmallVector strides; - int64_t offset; - if (failed(sourceType.getStridesAndOffset(strides, offset))) - return op.emitError("FP32 matmul requires strided operands"); - int64_t blockRows = operand ? columns : rows; - int64_t logicalRows = operand ? N : M; - Value base = operand ? column : row; - Value validRows = b.create( - loc, idx(blockRows), - b.create(loc, idx(logicalRows), base)); - auto flatType = MemRefType::get({bytes / 4}, b.getF32Type()); - Value flatBuffer = b.create( - loc, flatType, packed, ArrayRef{{0, 1}}); - auto rLoop = b.create(loc, zero, validRows, one); - b.setInsertionPointToStart(rLoop.getBody()); - Value r = rLoop.getInductionVar(); - Value global = b.create(loc, base, r); - Value flat = b.create(loc, r, idx(count)); - if (strides[axis] == 1) { - SmallVector sourceOffsets(rank, b.getIndexAttr(0)); - SmallVector sourceSizes(rank, b.getIndexAttr(1)); - SmallVector sourceStrides(rank, b.getIndexAttr(1)); - if (rank == 3) - sourceOffsets[0] = batch; - sourceOffsets[rank - 2] = operand - ? OpFoldResult(b.getIndexAttr(begin)) - : OpFoldResult(global); - sourceOffsets[rank - 1] = operand - ? OpFoldResult(global) - : OpFoldResult(b.getIndexAttr(begin)); - sourceSizes[axis] = b.getIndexAttr(count); - auto sourceRowType = memref::SubViewOp::inferRankReducedResultType( - {count}, sourceType, sourceOffsets, sourceSizes, sourceStrides); - Value sourceRow = b.create( - loc, sourceRowType, source, sourceOffsets, sourceSizes, - sourceStrides); - SmallVector targetOffsets{flat}; - SmallVector targetSizes{b.getIndexAttr(count)}; - SmallVector targetStrides{b.getIndexAttr(1)}; - auto targetRowType = memref::SubViewOp::inferRankReducedResultType( - {count}, flatType, targetOffsets, targetSizes, targetStrides); - Value targetRow = b.create( - loc, targetRowType, flatBuffer, targetOffsets, targetSizes, - targetStrides); - b.create(loc, sourceRow, targetRow); - } else { - auto kLoop = b.create(loc, zero, idx(count), one); - b.setInsertionPointToStart(kLoop.getBody()); - Value k = kLoop.getInductionVar(); - Value kk = b.create(loc, k, idx(begin)); - SmallVector indices; - if (rank == 3) - indices.push_back(batch); - indices.push_back(operand ? kk : global); - indices.push_back(operand ? global : kk); - Value value = b.create(loc, source, indices); - Value destination = b.create(loc, flat, k); - b.create(loc, value, flatBuffer, - ValueRange{destination}); - } - b.setInsertionPointAfter(rLoop); - if (logicalRows % blockRows) { - Value start = b.create(loc, validRows, idx(count * 4)); - Value pointer = - b.create(loc, packed); - Value address = b.create(loc, pointer, start); - Value integer = - b.create(loc, b.getI64Type(), address); - Value destination = b.create( - loc, LLVM::LLVMPointerType::get(b.getContext()), integer); - Value length = - b.create(loc, idx(blockRows * count * 4), start); - Value byteLength = - b.create(loc, b.getI64Type(), length); - Value byteZero = b.create(loc, 0, 8); - b.create(loc, destination, byteZero, byteLength, - false); - } - } - if (bankPack) { - ab = b.create( - loc, b.getI64Type(), ap, ab, - createI64Const(b, loc, rows * count / 4), - createI64Const(b, loc, 1), b.getI64IntegerAttr(1)); - auto packed = b.create( - loc, b.getI64Type(), b.getI64Type(), ab, wb, - FlatSymbolRefAttr::get(b.getContext(), "rvv_packmm"), ValueRange{ - createI64Const(b, loc, 1), createI64Const(b, loc, K), - createI64Const(b, loc, N), createI64Const(b, loc, begin), - b.create(loc, b.getI64Type(), column), - createI64Const(b, loc, count), createI64Const(b, loc, columns), - createI64Const(b, loc, rows), createI64Const(b, loc, begin == 0), - createI64Const(b, loc, begin + count == K), - createI64Const(b, loc, op.getFusedAttr().getValue())}); - ab = packed.getReadOut(); - wb = cb = packed.getWriteOut(); - } else { - ab = mvinBank(b, loc, ap, ab, rows * count / 4); - wb = mvinBank(b, loc, bp, wb, columns * count / 4); - cb = b.create(loc, b.getI64Type(), ab, wb, cb, - createI64Const(b, loc, - packBits(rows, 0, 11) | - packBits(columns, 12, 23) | - packBits(count, 24, 35)), - createI1Const(b, loc, begin == 0), - createI1Const(b, loc, begin + count == K), - createI64Const(b, loc, 0), op.getFusedAttr()) - .getWrBankOut(); - } - } - if (bankPack) - cb = b.create( - loc, b.getI64Type(), cp, cb, - createI64Const(b, loc, rows * columns / 4), - createI64Const(b, loc, 1), b.getI64IntegerAttr(2)); - else - cb = mvoutBank(b, loc, cp, cb, rows * columns / 4); - SmallVector outputStrides; - int64_t outputOffset; - if (failed(out.getStridesAndOffset(outputStrides, outputOffset))) - return op.emitError("FP32 matmul requires a strided output"); - Value validColumns = b.create( - loc, idx(columns), b.create(loc, idx(N), column)); - auto rr = b.create( - loc, zero, - b.create(loc, idx(rows), - b.create(loc, idx(M), row)), - one); - b.setInsertionPointToStart(rr.getBody()); - Value r = rr.getInductionVar(); - if (outputStrides.back() == 1) { - auto flatType = MemRefType::get({bytes / 4}, b.getF32Type()); - Value flat = b.create( - loc, flatType, cp, ArrayRef{{0, 1}}); - SmallVector offsets{ - b.create(loc, r, idx(columns)).getResult()}; - SmallVector sizes{validColumns}, strides{b.getIndexAttr(1)}; - auto sourceType = memref::SubViewOp::inferRankReducedResultType( - {ShapedType::kDynamic}, flatType, offsets, sizes, strides); - Value source = b.create(loc, sourceType, flat, offsets, - sizes, strides); - SmallVector outOffsets(rank, b.getIndexAttr(0)); - SmallVector outSizes(rank, b.getIndexAttr(1)); - SmallVector outSteps(rank, b.getIndexAttr(1)); - if (rank == 3) - outOffsets[0] = batch; - outOffsets[rank - 2] = b.create(loc, row, r).getResult(); - outOffsets[rank - 1] = column; - outSizes.back() = validColumns; - auto rowType = memref::SubViewOp::inferRankReducedResultType( - {ShapedType::kDynamic}, out, outOffsets, outSizes, outSteps); - Value destination = b.create( - loc, rowType, op.getOutput(), outOffsets, outSizes, outSteps); - b.create(loc, source, destination); - } else { - auto cc = b.create(loc, zero, validColumns, one); - b.setInsertionPointToStart(cc.getBody()); - Value c = cc.getInductionVar(); - Value flat = b.create( - loc, b.create(loc, r, idx(columns)), c); - Value result = b.create( - loc, cp, - ValueRange{b.create(loc, flat, four), - b.create(loc, flat, four)}); - SmallVector outputIndices; - if (rank == 3) - outputIndices.push_back(batch); - outputIndices.push_back(b.create(loc, row, r)); - outputIndices.push_back(b.create(loc, column, c)); - b.create(loc, result, op.getOutput(), outputIndices); - } - b.setInsertionPointAfter(rr); - SmallVector nextBanks{ab, wb, cb}; - if (bankPack) - nextBanks = {ab, cb}; - b.create(loc, nextBanks); - b.setInsertionPointAfter(colLoop); - b.create(loc, colLoop.getResults()); - b.setInsertionPointAfter(rowLoop); - b.create(loc, rowLoop.getResults()); - b.setInsertionPointAfter(batchLoop); - if (bankPack) { - auto finished = b.create( - loc, b.getI64Type(), b.getI64Type(), batchLoop.getResult(0), - batchLoop.getResult(1), - FlatSymbolRefAttr::get(b.getContext(), "rvv_packmm"), ValueRange{ - createI64Const(b, loc, 2), createI64Const(b, loc, K), - createI64Const(b, loc, N), createI64Const(b, loc, 0), - createI64Const(b, loc, 0), createI64Const(b, loc, K), - createI64Const(b, loc, columns), createI64Const(b, loc, rows), - createI64Const(b, loc, 1), createI64Const(b, loc, 1), - createI64Const(b, loc, op.getFusedAttr().getValue())}); - releaseBank(b, loc, finished.getReadOut()); - releaseBank(b, loc, finished.getWriteOut()); - } else { - for (Value bank : batchLoop.getResults()) - releaseBank(b, loc, bank); - } - for (Value buffer : {ap, bp, cp}) - b.create(loc, buffer); - b.eraseOp(op); - return success(); - } -}; diff --git a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp b/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp deleted file mode 100644 index 9a2219a17..000000000 --- a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include -#include -#include -using namespace mlir; -using namespace ::buddy::buckyball; -namespace { -#include "FP32Patterns.inc" -#include "MXFP8Patterns.inc" -#include "SharedPanelPatterns.inc" -#include "SharedWindowPatterns.inc" -} // namespace -namespace mlir::buddy { -void populateMxmmBallLowerBuckyballToBankSSAPatterns( - RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -} // namespace mlir::buddy diff --git a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/MXFP8Patterns.inc b/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/MXFP8Patterns.inc deleted file mode 100644 index b886f9b11..000000000 --- a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/MXFP8Patterns.inc +++ /dev/null @@ -1,222 +0,0 @@ -class MXFP8ToBanks : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(MXFP8MemMatmulOp op, - PatternRewriter &b) const override { - auto aTy = cast(op.getInput().getType()); - auto wTy = cast(op.getWeight().getType()); - auto oTy = cast(op.getOutput().getType()); - if (!aTy.hasStaticShape() || !wTy.hasStaticShape() || - !oTy.hasStaticShape() || aTy.getRank() != 1 || wTy.getRank() != 1 || - oTy.getRank() != 2 || !aTy.getElementType().isInteger(8) || - !wTy.getElementType().isInteger(8) || !oTy.getElementType().isF32()) - return op.emitError( - "MXFP8 requires static packed activation/weight bytes " - "and an FP32 output matrix"); - if (aTy.getStridesAndOffset().first[0] != 1 || - wTy.getStridesAndOffset().first[0] != 1) - return op.emitError("MXFP8 packed tensors must have unit byte stride"); - int64_t M = oTy.getShape()[0], K = op.getReductionK(), - N = oTy.getShape()[1]; - int64_t rows = M == 1 ? 1 : op.getTileM(), columns = op.getTileN(); - int64_t chunk = op.getTileK(), activationChunk = op.getActivationTileK(); - bool windowInput = activationChunk != chunk; - if (rows <= 0 || (rows != 1 && rows % 16) || columns <= 0 || columns % 16 || - chunk <= 0 || chunk % 32 || chunk >= 4096) - return op.emitError( - "MXFP8 tile_k must be a positive multiple of 32 below 4096"); - int64_t chunks = (K + chunk - 1) / chunk; - int64_t stride = op.getBankBytes(); - if (windowInput && (activationChunk != K || K > 65535 || - (rows * K * 33 / 32 + 15) / 16 * 16 > stride)) - return op.emitError("MXFP8 activation window must hold the complete row"); - int64_t activationChunks = windowInput ? 1 : chunks; - if (M <= 0 || K <= 0 || N <= 0 || K % 32 || - aTy.getShape()[0] != - ((M + rows - 1) / rows) * activationChunks * stride || - wTy.getShape()[0] != ((N + columns - 1) / columns) * chunks * stride) - return op.emitError("MXFP8 matrix or packed weight shape mismatch"); - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth * 16 != stride || - (rows != 1 && (rows <= 0 || rows % 16)) || columns <= 0 || - columns % 16 || (rows * chunk * 33 / 32 + 15) / 16 * 16 > stride || - columns * chunk * 33 / 32 > stride || rows * columns * 4 > stride) - return op.emitError( - "MXFP8 packed K block exceeds the 128-bit bank capacity"); - Location loc = op.getLoc(); - Value zero = b.create(loc, 0), - one = b.create(loc, 1); - Value packed = op.getInput(); - auto dynamic = ShapedType::kDynamic; - auto bankView = [&](Value source, Value offset, int64_t size) -> Value { - Value slice = b.create( - loc, source, SmallVector{offset}, - SmallVector{b.getIndexAttr(stride)}, - SmallVector{b.getIndexAttr(1)}); - auto ty = MemRefType::get( - {stride / 16, 16}, b.getI8Type(), - StridedLayoutAttr::get(b.getContext(), dynamic, {16, 1})); - return b.create( - loc, ty, slice, ArrayRef{{0, 1}}); - }; - Value aBank = allocBank(b, loc, 1, 1), wBank = allocBank(b, loc, 1, 1), - cBank = allocBank(b, loc, 1, 1); - bool residentInput = activationChunks + 2 <= target.bankNum; - SmallVector banks{aBank}; - if (residentInput) - for (int64_t part = 1; part < activationChunks; ++part) - banks.push_back(allocBank(b, loc, 1, 1)); - banks.push_back(wBank); - banks.push_back(cBank); - auto outPackTy = MemRefType::get({stride / 16, 4}, b.getF32Type()); - Value outPack = b.create(loc, outPackTy); - outPack.getDefiningOp()->setAttr("alignment", b.getI64IntegerAttr(stride)); - Value limit = b.create(loc, N), - step = b.create(loc, columns); - Value rowLimit = b.create(loc, M), - rowStep = b.create(loc, rows); - auto rowBlocks = b.create(loc, zero, rowLimit, rowStep, banks); - b.setInsertionPointToStart(rowBlocks.getBody()); - Value m = rowBlocks.getInductionVar(); - SmallVector inputBanks(rowBlocks.getRegionIterArgs().drop_back(2)); - aBank = inputBanks.front(); - wBank = rowBlocks.getRegionIterArgs()[banks.size() - 2]; - cBank = rowBlocks.getRegionIterArgs()[banks.size() - 1]; - Value rowPanel = b.create(loc, m, rowStep); - if (residentInput) { - for (int64_t part = 0; part < activationChunks; ++part) { - Value offset = b.create( - loc, - b.create( - loc, - b.create( - loc, rowPanel, - b.create(loc, activationChunks)), - b.create(loc, part)), - b.create(loc, stride)); - int64_t count = windowInput ? K : std::min(chunk, K - part * chunk); - int64_t aBytes = (rows * count * 33 / 32 + 15) / 16 * 16; - inputBanks[part] = mvinBank(b, loc, bankView(packed, offset, stride), - inputBanks[part], aBytes / 16); - } - aBank = inputBanks.front(); - } - auto panels = b.create(loc, zero, limit, step, - ValueRange{aBank, wBank, cBank}); - b.setInsertionPointToStart(panels.getBody()); - Value n = panels.getInductionVar(); - aBank = panels.getRegionIterArgs()[0]; - wBank = panels.getRegionIterArgs()[1]; - cBank = panels.getRegionIterArgs()[2]; - Value state = cBank; - for (int64_t part = 0; part < chunks; ++part) { - int64_t count = std::min(chunk, K - part * chunk); - int64_t aBytes = (rows * count * 33 / 32 + 15) / 16 * 16, - bBytes = columns * count * 33 / 32; - Value aOffset = b.create( - loc, - b.create( - loc, - b.create( - loc, rowPanel, b.create(loc, chunks)), - b.create(loc, part)), - b.create(loc, stride)); - Value panel = b.create(loc, n, step); - Value wOffset = b.create( - loc, - b.create( - loc, - b.create( - loc, panel, b.create(loc, chunks)), - b.create(loc, part)), - b.create(loc, stride)); - if (!residentInput) - aBank = mvinBank(b, loc, bankView(packed, aOffset, aBytes), aBank, - aBytes / 16); - wBank = mvinBank(b, loc, bankView(op.getWeight(), wOffset, bBytes), wBank, - bBytes / 16); - if (windowInput) - state = b.create( - loc, b.getI64Type(), aBank, wBank, state, - createI64Const(b, loc, rows), createI64Const(b, loc, columns), - createI64Const(b, loc, count), createI64Const(b, loc, K), - createI64Const(b, loc, part * chunk), - createI1Const(b, loc, part == 0), - createI1Const(b, loc, part == chunks - 1), - createI64Const(b, loc, 0)); - else - state = b.create( - loc, b.getI64Type(), residentInput ? inputBanks[part] : aBank, - wBank, state, - createI64Const(b, loc, matrixRs2(rows, columns, count)), - createI1Const(b, loc, part == 0), - createI1Const(b, loc, part == chunks - 1), - createI64Const(b, loc, 0)); - } - cBank = mvoutBank(b, loc, outPack, state, rows * columns / 4); - Value end = b.create( - loc, step, b.create(loc, limit, n)); - SmallVector outputStrides; - int64_t outputOffset; - if (failed(oTy.getStridesAndOffset(outputStrides, outputOffset))) - return op.emitError("MXFP8 matmul requires a strided output"); - auto rr = b.create( - loc, zero, - b.create(loc, rowStep, - b.create(loc, rowLimit, m)), - one); - b.setInsertionPointToStart(rr.getBody()); - Value r = rr.getInductionVar(); - if (outputStrides.back() == 1) { - auto flatType = MemRefType::get({stride / 4}, b.getF32Type()); - Value flat = b.create( - loc, flatType, outPack, ArrayRef{{0, 1}}); - SmallVector offsets{ - b.create( - loc, r, b.create(loc, columns)) - .getResult()}; - SmallVector sizes{end}, steps{b.getIndexAttr(1)}; - auto sourceType = memref::SubViewOp::inferRankReducedResultType( - {ShapedType::kDynamic}, flatType, offsets, sizes, steps); - Value source = b.create(loc, sourceType, flat, offsets, - sizes, steps); - SmallVector outOffsets{ - b.create(loc, m, r).getResult(), n}; - SmallVector outSizes{b.getIndexAttr(1), end}; - SmallVector outSteps(2, b.getIndexAttr(1)); - auto rowType = memref::SubViewOp::inferRankReducedResultType( - {ShapedType::kDynamic}, oTy, outOffsets, outSizes, outSteps); - Value destination = b.create( - loc, rowType, op.getOutput(), outOffsets, outSizes, outSteps); - b.create(loc, source, destination); - } else { - auto cc = b.create(loc, zero, end, one); - b.setInsertionPointToStart(cc.getBody()); - Value c = cc.getInductionVar(), - four = b.create(loc, 4); - Value pr = b.create( - loc, - b.create( - loc, r, b.create(loc, columns / 4)), - b.create(loc, c, four)); - Value pc = b.create(loc, c, four); - Value value = b.create(loc, outPack, ValueRange{pr, pc}); - b.create(loc, value, op.getOutput(), - ValueRange{b.create(loc, r, m), - b.create(loc, n, c)}); - } - b.setInsertionPointAfter(rr); - b.create(loc, ValueRange{aBank, wBank, cBank}); - b.setInsertionPointAfter(panels); - inputBanks[0] = panels.getResult(0); - inputBanks.push_back(panels.getResult(1)); - inputBanks.push_back(panels.getResult(2)); - b.create(loc, inputBanks); - b.setInsertionPointAfter(rowBlocks); - for (Value bank : rowBlocks.getResults()) - releaseBank(b, loc, bank); - b.create(loc, outPack); - b.eraseOp(op); - return success(); - } -}; diff --git a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/SharedPanelPatterns.inc b/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/SharedPanelPatterns.inc deleted file mode 100644 index 4233049f5..000000000 --- a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/SharedPanelPatterns.inc +++ /dev/null @@ -1,63 +0,0 @@ -class MXFP8SharedPanelToBanks : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(MXFP8SharedPanelOp op, - PatternRewriter &b) const override { - auto a = cast(op.getInput().getType()); - auto w = cast(op.getWeight().getType()); - int64_t rows = op.getRows(), columns = op.getColumns(); - int64_t k = op.getReductionK(), chunk = op.getTileK(); - int64_t bytes = op.getBankBytes(); - int64_t sharedBytes = op.getSharedBytesAttr().getInt(); - const auto &target = buckyball_target::getBuckyballTarget(); - if ((rows != 1 && (rows <= 0 || rows % 16)) || columns <= 0 || - columns % 16 || rows > 4095 || columns > 4095 || k <= 0 || k % 32 || - chunk <= 0 || chunk % 32 || chunk >= 4096 || - target.bankWidthBits != 128 || target.bankDepth * 16 != bytes || - sharedBytes <= 0 || sharedBytes % 16 || - rows * columns * 4 > sharedBytes || - (rows * chunk * 33 / 32 + 15) / 16 * 16 > bytes || - columns * chunk * 33 / 32 > bytes) - return op.emitError( - "shared MXFP8 panel exceeds the original bank layout"); - int64_t chunks = (k + chunk - 1) / chunk; - for (MemRefType type : {a, w}) - if (!type.hasStaticShape() || type.getRank() != 1 || - !type.getElementType().isInteger(8) || - type.getShape()[0] != chunks * bytes || - type.getStridesAndOffset().first[0] != 1) - return op.emitError( - "shared MXFP8 panel requires one packed row/N panel"); - Location loc = op.getLoc(); - Value ab = allocBank(b, loc, 1, 1), wb = allocBank(b, loc, 1, 1); - Value output = op.getBank(); - for (int64_t part = 0; part < chunks; ++part) { - int64_t count = std::min(chunk, k - part * chunk); - auto view = [&](Value packed) -> Value { - Value slice = b.create( - loc, packed, - SmallVector{b.getIndexAttr(part * bytes)}, - SmallVector{b.getIndexAttr(bytes)}, - SmallVector{b.getIndexAttr(1)}); - auto type = *memref::ExpandShapeOp::computeExpandedType( - cast(slice.getType()), {bytes / 16, 16}, - ArrayRef{{0, 1}}); - return b.create( - loc, type, slice, ArrayRef{{0, 1}}); - }; - ab = mvinBank(b, loc, view(op.getInput()), ab, - (rows * count * 33 / 32 + 15) / 16); - wb = mvinBank(b, loc, view(op.getWeight()), wb, - columns * count * 33 / 32 / 16); - output = b.create( - loc, b.getI64Type(), ab, wb, output, - createI64Const(b, loc, matrixRs2(rows, columns, count)), - createI1Const(b, loc, part == 0), - createI1Const(b, loc, part == chunks - 1), createI64Const(b, loc, 0)); - } - releaseBank(b, loc, ab); - releaseBank(b, loc, wb); - b.replaceOp(op, output); - return success(); - } -}; diff --git a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/SharedWindowPatterns.inc b/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/SharedWindowPatterns.inc deleted file mode 100644 index 792866dbb..000000000 --- a/examples/balls/mxmm/compiler/src/Conversion/LowerBuckyball/SharedWindowPatterns.inc +++ /dev/null @@ -1,62 +0,0 @@ -class MXFP8SharedWindowToBanks : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(MXFP8SharedWindowOp op, - PatternRewriter &b) const override { - auto a = cast(op.getInput().getType()); - auto w = cast(op.getWeight().getType()); - int64_t rows = op.getRows(), columns = op.getColumns(); - int64_t k = op.getReductionK(), chunk = op.getTileK(); - int64_t bytes = op.getBankBytes(); - int64_t sharedBytes = op.getSharedBytesAttr().getInt(); - const auto &target = buckyball_target::getBuckyballTarget(); - if ((rows != 1 && (rows <= 0 || rows % 16)) || rows > 4095 || - columns <= 0 || columns > 4095 || columns % 16 || k <= 0 || k > 65535 || - k % 32 || chunk <= 0 || chunk >= 4096 || chunk % 32 || - target.bankWidthBits != 128 || target.bankDepth * 16 != bytes || - sharedBytes <= 0 || sharedBytes % 16 || - rows * columns * 4 > sharedBytes || - (rows * k * 33 / 32 + 15) / 16 * 16 > bytes || - columns * chunk * 33 / 32 > bytes) - return op.emitError( - "shared MXFP8 window exceeds its activation or output bank"); - int64_t chunks = (k + chunk - 1) / chunk; - if (!a.hasStaticShape() || !w.hasStaticShape() || - a.getShape()[0] != bytes || w.getShape()[0] != chunks * bytes || - a.getStridesAndOffset().first[0] != 1 || - w.getStridesAndOffset().first[0] != 1) - return op.emitError( - "shared MXFP8 window requires a complete activation and one N panel"); - Location loc = op.getLoc(); - auto view = [&](Value source, int64_t offset) -> Value { - Value slice = b.create( - loc, source, SmallVector{b.getIndexAttr(offset)}, - SmallVector{b.getIndexAttr(bytes)}, - SmallVector{b.getIndexAttr(1)}); - auto type = *memref::ExpandShapeOp::computeExpandedType( - cast(slice.getType()), {bytes / 16, 16}, - ArrayRef{{0, 1}}); - return b.create( - loc, type, slice, ArrayRef{{0, 1}}); - }; - Value ab = allocBank(b, loc, 1, 1), wb = allocBank(b, loc, 1, 1); - ab = mvinBank(b, loc, view(op.getInput(), 0), ab, - (rows * k * 33 / 32 + 15) / 16); - Value output = op.getBank(); - for (int64_t part = 0; part < chunks; ++part) { - int64_t count = std::min(chunk, k - part * chunk); - wb = mvinBank(b, loc, view(op.getWeight(), part * bytes), wb, - columns * count * 33 / 32 / 16); - output = b.create( - loc, b.getI64Type(), ab, wb, output, createI64Const(b, loc, rows), - createI64Const(b, loc, columns), createI64Const(b, loc, count), - createI64Const(b, loc, k), createI64Const(b, loc, part * chunk), - createI1Const(b, loc, part == 0), - createI1Const(b, loc, part == chunks - 1), createI64Const(b, loc, 0)); - } - releaseBank(b, loc, ab); - releaseBank(b, loc, wb); - b.replaceOp(op, output); - return success(); - } -}; diff --git a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/mxmm/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index d9d7968ae..000000000 --- a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,157 +0,0 @@ -#include -//===- LegalizeForLLVMExport.cpp - MxmmBall LLVM lowering --------------===// - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include "llvm/Support/ErrorHandling.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -uint64_t matrixCfg(uint64_t rows, uint64_t cols, bool first, bool last) { - return fieldBits(rows, 0, 11) | fieldBits(cols, 12, 23) | - (uint64_t(first) << 24) | (uint64_t(last) << 25); -} -struct MXFP8Lowering : public ConvertOpToLLVMPattern { - MXFP8Lowering(LLVMTypeConverter &converter, bool, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(MXFP8Op op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("MxmmBall"); - Location loc = op.getLoc(); - auto config = op.getConfig().getDefiningOp(); - if (!config) - return op.emitError("mxmm config must be a constant"); - auto configAttr = dyn_cast(config.getValue()); - if (!configAttr) - return op.emitError("mxmm config must be an integer constant"); - uint64_t cfg = configAttr.getValue().getZExtValue(); - uint64_t rows = cfg & 0xfff; - uint64_t cols = (cfg >> 12) & 0xfff; - uint64_t k = (cfg >> 24) & 0xfff; - if (cfg >> 36 || (rows != 1 && (rows == 0 || rows % 16)) || cols == 0 || - cols % 16 || k == 0 || k % 32 || - (rows * k * 33 / 32 + 15) / 16 > static_cast(bankDepth) || - cols * k * 33 / 32 / 16 > static_cast(bankDepth) || - rows * cols / 4 > static_cast(bankDepth)) - return op.emitError("Mxmm matrix shape or C footprint is invalid"); - auto base = adaptor.getOutputBase().getDefiningOp(); - auto baseAttr = - base ? dyn_cast(base.getValue()) : IntegerAttr(); - if (!baseAttr || baseAttr.getInt() < 0 || baseAttr.getInt() > 63 || - baseAttr.getInt() + rows * cols / 4 > bankDepth) - return op.emitError("Mxmm outputBase must be constant and fit C in bank"); - Value rs1 = packRs1BanksIter( - rewriter, loc, adaptor.getOp1BankId(), adaptor.getOp2BankId(), - adaptor.getResultBankId(), cstI64(rewriter, loc, k)); - Value rs2 = cstI64(rewriter, loc, matrixCfg(rows, cols, false, false)); - Value first = rewriter.create(loc, rewriter.getI64Type(), - adaptor.getFirst()); - Value last = rewriter.create(loc, rewriter.getI64Type(), - adaptor.getLast()); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, first, cstI64(rewriter, loc, 24))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, last, cstI64(rewriter, loc, 25))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getOutputBase(), - cstI64(rewriter, loc, 26))); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MXMM_MXFP8"))); - return success(); - } - -private: - int64_t bankDepth; -}; - -struct FP32Lowering : public ConvertOpToLLVMPattern { - FP32Lowering(LLVMTypeConverter &converter, bool, int64_t depth) - : ConvertOpToLLVMPattern(converter), depth(depth) {} - - LogicalResult matchAndRewrite(FP32Op op, OpAdaptor adaptor, - ConversionPatternRewriter &b) const override { - buckyball_target::requireBuckyballBall("MxmmBall"); - auto constant = op.getConfig().getDefiningOp(); - auto shape = - constant ? dyn_cast(constant.getValue()) : IntegerAttr(); - if (!shape) - return op.emitError("FP32 matmul requires a constant shape"); - uint64_t cfg = shape.getValue().getZExtValue(); - uint64_t rows = cfg & 0xfff, cols = (cfg >> 12) & 0xfff, - k = (cfg >> 24) & 0xfff; - if (cfg >> 36 || (rows != 1 && (rows == 0 || rows % 16)) || cols == 0 || - cols % 16 || !k || k % 4 || rows * k / 4 > uint64_t(depth) || - cols * k / 4 > uint64_t(depth) || - buckyball_target::getBuckyballTarget().bankWidthBits != 128) - return op.emitError( - "FP32 matmul shape does not fit the configured banks"); - auto baseOp = op.getOutputBase().getDefiningOp(); - auto base = - baseOp ? dyn_cast(baseOp.getValue()) : IntegerAttr(); - if (!base || base.getInt() < 0 || base.getInt() > 63 || - base.getInt() + rows * cols / 4 > uint64_t(depth)) - return op.emitError("FP32 output base does not fit the configured bank"); - Location loc = op.getLoc(); - Value rs1 = - packRs1BanksIter(b, loc, adaptor.getOp1BankId(), adaptor.getOp2BankId(), - adaptor.getResultBankId(), cstI64(b, loc, k)); - Value rs2 = cstI64(b, loc, matrixCfg(rows, cols, false, false)); - Value first = - b.create(loc, b.getI64Type(), adaptor.getFirst()); - Value last = - b.create(loc, b.getI64Type(), adaptor.getLast()); - rs2 = b.create( - loc, rs2, b.create(loc, first, cstI64(b, loc, 24))); - rs2 = b.create( - loc, rs2, b.create(loc, last, cstI64(b, loc, 25))); - rs2 = b.create( - loc, rs2, - b.create(loc, adaptor.getOutputBase(), - cstI64(b, loc, 26))); - b.replaceOpWithNewOp( - op, rs1, rs2, - b.getI32IntegerAttr(buckyball_target::getBuckyballFunct7( - op.getFused() ? "MXMM_FMA32" : "MXMM_F32"))); - return success(); - } - int64_t depth; -}; - -#include "WindowPatterns.inc" - -} // namespace -namespace mlir::buddy::buckyball { -void populateMxmmBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns, - bool stable, int64_t depth) { - patterns.add(converter, - stable, depth); -} -void configureMxmmBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/mxmm/compiler/src/Transforms/Passes.cpp b/examples/balls/mxmm/compiler/src/Transforms/Passes.cpp deleted file mode 100644 index 33e983145..000000000 --- a/examples/balls/mxmm/compiler/src/Transforms/Passes.cpp +++ /dev/null @@ -1,4 +0,0 @@ -namespace mlir::buddy { -void registerNormQuantWindowPass(); -void registerMxmmBallPasses() { registerNormQuantWindowPass(); } -} // namespace mlir::buddy diff --git a/examples/balls/mxmm/configs/MxmmBallParam.scala b/examples/balls/mxmm/configs/MxmmBallParam.scala deleted file mode 100644 index c28e1fdce..000000000 --- a/examples/balls/mxmm/configs/MxmmBallParam.scala +++ /dev/null @@ -1,19 +0,0 @@ -package examples.balls.mxmm.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig - -case class MxmmBallParam(tileRows: Int, tileCols: Int) { - require(tileRows == 16 && tileCols == 16) - val arithmeticLatency = 3 - val contexts = arithmeticLatency * 2 + 1 -} - -object MxmmBallParam { - - def apply(b: GlobalConfig): MxmmBallParam = { - val table = BallParamLoader.ballTable(b, "MxmmBall") - MxmmBallParam(BallParamLoader.int(table, "tileRows"), BallParamLoader.int(table, "tileCols")) - } - -} diff --git a/examples/balls/mxmm/configs/default.toml b/examples/balls/mxmm/configs/default.toml deleted file mode 100644 index d71354112..000000000 --- a/examples/balls/mxmm/configs/default.toml +++ /dev/null @@ -1,3 +0,0 @@ -[ball] -tileRows = 16 -tileCols = 16 diff --git a/examples/balls/mxmm/emu/src/accumulator.rs b/examples/balls/mxmm/emu/src/accumulator.rs deleted file mode 100644 index cc496f008..000000000 --- a/examples/balls/mxmm/emu/src/accumulator.rs +++ /dev/null @@ -1,22 +0,0 @@ -use std::{cell::RefCell, collections::HashMap}; - -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum Format { - Mxfp8, - Mxfp8Window, - Fma32, - F32, -} - -pub(crate) struct Chain { - pub(crate) format: Format, - pub(crate) rows: usize, - pub(crate) cols: usize, - pub(crate) bank: u64, - pub(crate) base: usize, - pub(crate) values: Vec, -} - -thread_local! { - pub(crate) static CHAINS: RefCell> = RefCell::new(HashMap::new()); -} diff --git a/examples/balls/mxmm/emu/src/dispatch.rs b/examples/balls/mxmm/emu/src/dispatch.rs deleted file mode 100644 index 361cc955d..000000000 --- a/examples/balls/mxmm/emu/src/dispatch.rs +++ /dev/null @@ -1,33 +0,0 @@ -use super::{fp32, mxfp8}; -use crate::inst::instruction::ExecContext; - -const BALL_CLASS: &str = "examples.balls.mxmm.MxmmBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MXMM_MXFP8") => Some(mxfp8::execute(xs1, xs2, false, ctx)), - Some("MXMM_MXFP8_WINDOW") => Some(mxfp8::execute(xs1, xs2, true, ctx)), - Some("MXMM_FMA32") => Some(fp32::execute(xs1, xs2, true, ctx)), - Some("MXMM_F32") => Some(fp32::execute(xs1, xs2, false, ctx)), - _ => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, _xs1: u64, _xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MXMM_MXFP8" | "MXMM_MXFP8_WINDOW" | "MXMM_FMA32" | "MXMM_F32") => Some(1), - _ => None, - } -} diff --git a/examples/balls/mxmm/emu/src/fp32.rs b/examples/balls/mxmm/emu/src/fp32.rs deleted file mode 100644 index bd2588c86..000000000 --- a/examples/balls/mxmm/emu/src/fp32.rs +++ /dev/null @@ -1,121 +0,0 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; -use super::accumulator::{Chain, Format, CHAINS}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::ExecContext; - -pub(crate) fn execute(xs1: u64, xs2: u64, fused: bool, ctx: &mut ExecContext) -> u64 { - let format = if fused { Format::Fma32 } else { Format::F32 }; - let rows = (xs2 & 0xfff) as usize; - let cols = ((xs2 >> 12) & 0xfff) as usize; - let k = rs1_iter(xs1) as usize; - let first = (xs2 >> 24) & 1 != 0; - let last = (xs2 >> 25) & 1 != 0; - let base = ((xs2 >> 26) & 63) as usize; - let a = rs1_b0(xs1); - let b = rs1_b1(xs1); - let c = rs1_b2(xs1); - assert!( - xs2 >> 32 == 0 && (rows == 1 || (rows > 0 && rows % 16 == 0)) && cols > 0 && cols % 16 == 0 && k > 0 && k % 4 == 0, - "f32 matmul: requires M=1/multiple of 16, N multiple of 16, positive K multiple of four and zero reserved bits" - ); - assert!(a != b && b != c && a != c, "f32 matmul: banks must differ"); - for bank in [a, b, c] { - assert!( - ctx.config(bank).allocated && ctx.config(bank).cols == 1, - "f32 matmul: banks must be allocated with one column" - ); - } - let stride = bank_row_bytes(); - assert!( - stride == 16 - && rows * k * 4 <= bank_lines() * stride - && cols * k * 4 <= bank_lines() * stride - && base * stride + rows * cols * 4 <= bank_lines() * stride, - "f32 matmul: bank footprint exceeds depth" - ); - let pa = pbank(ctx, a); - let pb = pbank(ctx, b); - let av = &ctx.banks[pa][..rows * k * 4]; - let bv = &ctx.banks[pb][..cols * k * 4]; - assert!( - av.chunks_exact(4) - .chain(bv.chunks_exact(4)) - .all(|x| f32::from_le_bytes(x.try_into().unwrap()).is_finite()), - "f32 matmul: non-finite operand" - ); - let mut chain = CHAINS.with(|chains| { - let mut chains = chains.borrow_mut(); - if first { - assert!( - !chains.contains_key(&ctx.hart_id), - "f32 matmul: first while chain is live" - ); - Chain { - format, - rows, - cols, - bank: c, - base, - values: vec![0f32; rows * cols], - } - } else { - chains - .remove(&ctx.hart_id) - .expect("f32 matmul: continuation without first") - } - }); - assert!( - chain.format == format - && chain.rows == rows - && chain.cols == cols - && chain.bank == c - && chain.base == base, - "f32 matmul: continuation changed destination" - ); - let weights: Vec<&[u8]> = (0..cols) - .map(|col| &bv[col * k * 4..(col + 1) * k * 4]) - .collect(); - for row in 0..rows { - if av[row * k * 4..(row + 1) * k * 4] - .chunks_exact(4) - .all(|x| u32::from_le_bytes(x.try_into().unwrap()) & 0x7fffffff == 0) - && chain.values[row * cols..(row + 1) * cols] - .iter() - .all(|x| x.to_bits() == 0) - { - continue; - } - let av = &av[row * k * 4..(row + 1) * k * 4]; - let dst = &mut chain.values[row * cols..(row + 1) * cols]; - let mut sums = dst.to_vec(); - for inner in 0..k { - let a = f32::from_le_bytes(av[inner * 4..inner * 4 + 4].try_into().unwrap()); - for col in 0..cols { - let b = - f32::from_le_bytes(weights[col][inner * 4..inner * 4 + 4].try_into().unwrap()); - sums[col] = if fused { - a.mul_add(b, sums[col]) - } else { - (a * b) + sums[col] - }; - } - } - assert!( - sums.iter().all(|x| x.is_finite()), - "f32 matmul: accumulator overflow" - ); - dst.copy_from_slice(&sums); - } - if last { - let pc = pbank(ctx, c); - for (index, value) in chain.values.iter().enumerate() { - let offset = base * stride + index * 4; - ctx.banks[pc][offset..offset + 4].copy_from_slice(&value.to_le_bytes()); - } - } else { - CHAINS.with(|chains| { - chains.borrow_mut().insert(ctx.hart_id, chain); - }); - } - 0 -} diff --git a/examples/balls/mxmm/emu/src/lib.rs b/examples/balls/mxmm/emu/src/lib.rs deleted file mode 100644 index e9d96b625..000000000 --- a/examples/balls/mxmm/emu/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod accumulator; -mod fp32; -mod mxfp8; -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/mxmm/emu/src/mxfp8.rs b/examples/balls/mxmm/emu/src/mxfp8.rs deleted file mode 100644 index c5e5c75fa..000000000 --- a/examples/balls/mxmm/emu/src/mxfp8.rs +++ /dev/null @@ -1,176 +0,0 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; -use super::accumulator::{Chain, Format, CHAINS}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::ExecContext; - -fn value(code: u8, factor: f32) -> f32 { - let exponent = (code >> 3) & 15; - let mantissa = (code & 7) as u32; - let normal = ((exponent as u32 + 120) << 23) | (mantissa << 20); - let subnormal = (mantissa as f32 / 512.0).to_bits(); - let element = f32::from_bits(if exponent == 0 { subnormal } else { normal }); - let result = f32::from_bits((element * factor).to_bits() | ((code as u32 & 128) << 24)); - result -} - -pub(crate) fn execute(xs1: u64, xs2: u64, window: bool, ctx: &mut ExecContext) -> u64 { - let rows = (xs2 & 0xfff) as usize; - let cols = ((xs2 >> 12) & 0xfff) as usize; - let k = rs1_iter(xs1) as usize; - let first = (xs2 >> 24) & 1 != 0; - let last = (xs2 >> 25) & 1 != 0; - let base = ((xs2 >> 26) & 63) as usize; - let a = rs1_b0(xs1); - let b = rs1_b1(xs1); - let c = rs1_b2(xs1); - let full_k = if window { - ((xs2 >> 32) & 0xffff) as usize - } else { - k - }; - let start_k = if window { (xs2 >> 48) as usize } else { 0 }; - let format = if window { - Format::Mxfp8Window - } else { - Format::Mxfp8 - }; - assert!( - (window || xs2 >> 32 == 0) && (rows == 1 || (rows > 0 && rows % 16 == 0)) && cols > 0 && cols % 16 == 0 && k > 0 && k % 32 == 0, - "mxfp8: requires M=1/multiple of 16, N multiple of 16, K multiple of 32 and zero reserved bits" - ); - assert!( - full_k > 0 && full_k % 32 == 0 && start_k % 32 == 0 && start_k + k <= full_k, - "mxfp8: invalid activation window" - ); - assert!(a != b && b != c && a != c, "mxfp8: banks must differ"); - for bank in [a, b, c] { - assert!( - ctx.config(bank).allocated && ctx.config(bank).cols == 1, - "mxfp8: banks must be allocated with one column" - ); - } - let stride = bank_row_bytes(); - assert!(stride == 16, "mxfp8: requires 16-byte bank rows"); - assert!( - rows * full_k + rows * full_k / 32 <= bank_lines() * stride - && cols * k + cols * k / 32 <= bank_lines() * stride - && base * stride + rows * cols * 4 <= bank_lines() * stride, - "mxfp8: bank footprint exceeds depth" - ); - let pa = pbank(ctx, a); - let pb = pbank(ctx, b); - let a_bytes = &ctx.banks[pa][..rows * full_k + rows * full_k / 32]; - let b_bytes = &ctx.banks[pb][..cols * k + cols * k / 32]; - assert!( - b_bytes[..cols * k] - .iter() - .fold(true, |valid, code| valid & (code & 0x7f != 0x7f)), - "mxfp8: NaN encoding" - ); - let mut av = vec![0f32; rows * k]; - let mut bv = vec![0f32; cols * k]; - for (block, dst) in av.chunks_exact_mut(32).enumerate() { - let row = block / (k / 32); - let inner = block % (k / 32); - let code_offset = row * full_k + start_k + inner * 32; - let scale = a_bytes[rows * full_k + row * (full_k / 32) + start_k / 32 + inner]; - assert!(scale != 255, "mxfp8: NaN encoding"); - let factor = if scale == 0 { - f32::from_bits(1 << 22) - } else { - f32::from_bits((scale as u32) << 23) - }; - for (index, decoded) in dst.iter_mut().enumerate() { - let code = a_bytes[code_offset + index]; - assert!(code & 0x7f != 0x7f, "mxfp8: NaN encoding"); - *decoded = value(code, factor); - } - } - for block in 0..k / 32 { - for col in 0..cols { - let scale = b_bytes[cols * k + col * (k / 32) + block]; - assert!(scale != 255, "mxfp8: NaN encoding"); - let factor = if scale == 0 { - f32::from_bits(1 << 22) - } else { - f32::from_bits((scale as u32) << 23) - }; - let codes = &b_bytes[col * k + block * 32..col * k + (block + 1) * 32]; - let mut decoded = [0f32; 32]; - for (decoded, code) in decoded.iter_mut().zip(codes.iter()) { - *decoded = value(*code, factor); - } - for (index, decoded) in decoded.iter().enumerate() { - bv[(block * 32 + index) * cols + col] = *decoded; - } - } - } - assert!( - av.iter() - .chain(bv.iter()) - .fold(true, |valid, value| valid & value.is_finite()), - "mxfp8: decoded value overflows FP32" - ); - let mut chain = CHAINS.with(|chains| { - let mut chains = chains.borrow_mut(); - if first { - assert!( - !chains.contains_key(&ctx.hart_id), - "mxfp8: first while chain is live" - ); - Chain { - format, - rows, - cols, - bank: c, - base, - values: vec![0f32; rows * cols], - } - } else { - chains - .remove(&ctx.hart_id) - .expect("mxfp8: continuation without first") - } - }); - assert!( - chain.format == format - && chain.rows == rows - && chain.cols == cols - && chain.bank == c - && chain.base == base, - "mxfp8: continuation changed destination" - ); - for row in 0..rows { - if av[row * k..(row + 1) * k].iter().all(|value| *value == 0.0) - && chain.values[row * cols..(row + 1) * cols] - .iter() - .all(|value| value.to_bits() == 0) - { - continue; - } - let av = &av[row * k..(row + 1) * k]; - let dst = &mut chain.values[row * cols..(row + 1) * cols]; - let mut sums = dst.to_vec(); - for inner in 0..k { - let a = av[inner]; - let weights = &bv[inner * cols..(inner + 1) * cols]; - for col in 0..cols { - sums[col] = a.mul_add(weights[col], sums[col]); - } - } - assert!(sums.iter().all(|v| v.is_finite()), "accumulator overflow"); - dst.copy_from_slice(&sums); - } - if last { - let pc = pbank(ctx, c); - for (i, value) in chain.values.iter().enumerate() { - ctx.banks[pc][base * stride + i * 4..base * stride + i * 4 + 4] - .copy_from_slice(&value.to_le_bytes()); - } - } else { - CHAINS.with(|chains| { - chains.borrow_mut().insert(ctx.hart_id, chain); - }); - } - 0 -} diff --git a/examples/balls/mxmm/workloads/ctests/CMakeLists.txt b/examples/balls/mxmm/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 7fdb334ba..000000000 --- a/examples/balls/mxmm/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -add_buckyball_ctests(mxmm_fp32_test.c mxmm_mxfp8_test.c mxmm_bank_capacity_test.c) - -file(READ "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h" MXMM_PARAMS) -string(REGEX MATCH "#define BANK_LINES ([0-9]+)" _ "${MXMM_PARAMS}") -set(MXMM_BANK_LINES ${CMAKE_MATCH_1}) -string(REGEX MATCH "#define BANK_WIDTH ([0-9]+)" _ "${MXMM_PARAMS}") -math(EXPR MXMM_BANK_BYTES "${MXMM_BANK_LINES} * ${CMAKE_MATCH_1} / 8") -if(MXMM_BANK_BYTES LESS 15840) - message(FATAL_ERROR "Mxmm A-window row test requires 15840-byte banks; actual ${MXMM_BANK_BYTES} bytes") -endif() -add_buckyball_ctests(mxmm_window_test.c mxmm_window_rows_test.c mxmm_window_chain_test.c) diff --git a/examples/balls/mxmm/workloads/kernels/CMakeLists.txt b/examples/balls/mxmm/workloads/kernels/CMakeLists.txt deleted file mode 100644 index c8a67eea4..000000000 --- a/examples/balls/mxmm/workloads/kernels/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../../../.." ABSOLUTE) -set(RVV_SOURCE "${REPO_ROOT}/bb-tests/workloads/lib/bbsw/kernels/rvv") -set(IMAGES "${CMAKE_CURRENT_BINARY_DIR}/norm_window_image.h" - "${CMAKE_CURRENT_BINARY_DIR}/packmm_image.h") -add_custom_command(OUTPUT ${IMAGES} - COMMAND "${PYTHON}" "${CMAKE_CURRENT_SOURCE_DIR}/build_image.py" - --output "${CMAKE_CURRENT_BINARY_DIR}" --isa-dir "${RUNTIME_PARAMS}" - DEPENDS build_image.py norm_window.cpp window.h packmm.cpp packmm.h - "${RUNTIME_PARAMS}/ballISA.h" "${RVV_SOURCE}/build_images.py" - "${RVV_SOURCE}/norm.cpp" "${RVV_SOURCE}/pack.cpp" "${RVV_SOURCE}/kernels.h" "${RVV_SOURCE}/kernel.ld" - BYPRODUCTS norm_window.o norm.o norm_window.elf packmm.o pack.o packmm.elf - VERBATIM) -add_custom_target(mxmm-images DEPENDS ${IMAGES}) -add_library(mxmm_kernels STATIC dispatch.cpp pack_dispatch.cpp ${IMAGES}) -target_link_libraries(mxmm_kernels PRIVATE runtime) -target_compile_options(mxmm_kernels PRIVATE -O2) -target_compile_features(mxmm_kernels PRIVATE cxx_std_20) -target_include_directories(mxmm_kernels PRIVATE "${CMAKE_CURRENT_BINARY_DIR}" - "${RUNTIME_PARAMS}" "${REPO_ROOT}/bb-tests/workloads/lib") - -add_library(mxmm_kernels_ant STATIC EXCLUDE_FROM_ALL dispatch.cpp pack_dispatch.cpp ${IMAGES}) -target_link_libraries(mxmm_kernels_ant PRIVATE runtime_ant_tasks) -target_compile_options(mxmm_kernels_ant PRIVATE -O2) -target_compile_features(mxmm_kernels_ant PRIVATE cxx_std_20) -target_include_directories(mxmm_kernels_ant PRIVATE "${CMAKE_CURRENT_BINARY_DIR}" - "${RUNTIME_PARAMS}" "${REPO_ROOT}/bb-tests/workloads/lib") diff --git a/examples/balls/mxmm/workloads/kernels/build_image.py b/examples/balls/mxmm/workloads/kernels/build_image.py deleted file mode 100644 index 907420906..000000000 --- a/examples/balls/mxmm/workloads/kernels/build_image.py +++ /dev/null @@ -1,93 +0,0 @@ -import argparse -from pathlib import Path -import struct -import subprocess -import sys - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--isa-dir", type=Path, required=True) - args = parser.parse_args() - source = Path(__file__).resolve().parent - root = source.parents[4] - support = root / "bb-tests/workloads/lib/bbsw/kernels/rvv" - sys.path.insert(0, str(support)) - from build_images import array, read_segments - - tools = root / "result/bin" - args.output.mkdir(parents=True, exist_ok=True) - objects = [] - for name, path in ( - ("norm_window", source / "norm_window.cpp"), - ("norm", support / "norm.cpp"), - ("packmm", source / "packmm.cpp"), - ("pack", support / "pack.cpp"), - ): - obj = args.output / f"{name}.o" - subprocess.run( - [ - str(tools / "clang++"), - "--target=riscv64", - "-march=rv64imf_zve32f_zvl128b", - "-mabi=lp64f", - "-mcmodel=medany", - "-ffreestanding", - "-O2", - "-ffp-contract=off", - "-ffunction-sections", - "-fdata-sections", - f"-I{root / 'bb-tests/workloads/lib'}", - f"-I{args.isa_dir}", - "-c", - str(path), - "-o", - str(obj), - ], - check=True, - ) - objects.append(obj) - for name in ("norm_window", "packmm"): - elf = args.output / f"{name}.elf" - subprocess.run( - [ - str(tools / "riscv64-unknown-elf-ld"), - "-m", - "elf64lriscv", - "--gc-sections", - "-e", - name, - "-T", - str(support / "kernel.ld"), - *map(str, objects), - "-o", - str(elf), - ], - check=True, - ) - text, data, entry = read_segments(elf) - image = ( - struct.pack( - "<6I", - 0x31564B52, - len(text), - entry, - 0x40000000, - len(data), - 0, - ) - + text - + data - ) - (args.output / f"{name}_image.h").write_text( - "#pragma once\n#include \n" - + array(f"{name}_image", image) - + f"constexpr uint32_t {name}_entry = {entry};\n" - + f"constexpr uint32_t {name}_text = {len(text)};\n" - ) - print(f"{name}: image={len(image)} text={len(text)} data={len(data)}") - - -if __name__ == "__main__": - main() diff --git a/examples/balls/mxmm/workloads/mlir_tests/CMakeLists.txt b/examples/balls/mxmm/workloads/mlir_tests/CMakeLists.txt deleted file mode 100644 index 1987af32c..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_subdirectory(ball) -add_subdirectory(bank) -add_subdirectory(tensor) diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/CMakeLists.txt b/examples/balls/mxmm/workloads/mlir_tests/ball/CMakeLists.txt deleted file mode 100644 index 099ee8c64..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX ball) -foreach(MODE mxfp8 fma32 f32) - add_buckyball_mlir_test(mxmm_${MODE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) - buckyball_mlirtest_name(TEST_NAME ${BUCKYBALL_MLIR_ACTIVE_TARGET} ball_mxmm_${MODE}) - foreach(KIND baremetal linux) - add_custom_command(OUTPUT ${TEST_NAME}-${KIND} APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_test.h - ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_steps.h) - endforeach() -endforeach() -foreach(CASE mxmm_window mxmm_window_rows mxmm_window_chain) - add_buckyball_mlir_test(${CASE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) - buckyball_mlirtest_name(TEST_NAME ${BUCKYBALL_MLIR_ACTIVE_TARGET} ball_${CASE}) - foreach(KIND baremetal linux) - add_custom_command(OUTPUT ${TEST_NAME}-${KIND} APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../isa/mxmm.h) - endforeach() -endforeach() diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window.mlir b/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window.mlir deleted file mode 100644 index 8d8dd6f12..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window.mlir +++ /dev/null @@ -1,26 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK: buckyball.intr.custom {{.*}} -// CHECK-LABEL: func.func @window_small -// CHECK-DAG: %[[RS1:[^ ]+]] = arith.constant 34367084547 : i64 -// CHECK-DAG: %[[RS2:[^ ]+]] = arith.constant 9007474250153985 : i64 -// CHECK: buckyball.intr.custom %[[RS1]], %[[RS2]] -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 - return -} -func.func @window_small() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 - %b = arith.constant 6 : i64 - %c = arith.constant 7 : i64 - %rows = arith.constant 1 : i64 - %cols = arith.constant 16 : i64 - %count = arith.constant 32 : i64 - %full = arith.constant 64 : i64 - %start = arith.constant 32 : i64 - %first = arith.constant true - %last = arith.constant true - %base = arith.constant 1 : i64 - buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain.mlir b/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain.mlir deleted file mode 100644 index d1047488a..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain.mlir +++ /dev/null @@ -1,7 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK: buckyball.intr.custom {{.*}} -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows.mlir b/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows.mlir deleted file mode 100644 index d1047488a..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows.mlir +++ /dev/null @@ -1,7 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK: buckyball.intr.custom {{.*}} -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/mxmm/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index 8932020ab..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) -foreach(MODE mxfp8 fma32 f32) - add_buckyball_mlir_test(mxmm_${MODE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) - buckyball_mlirtest_name(TEST_NAME ${BUCKYBALL_MLIR_ACTIVE_TARGET} bank_mxmm_${MODE}) - foreach(KIND baremetal linux) - add_custom_command(OUTPUT ${TEST_NAME}-${KIND} APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_test.h - ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_steps.h) - endforeach() -endforeach() -foreach(CASE mxmm_window mxmm_window_rows mxmm_window_chain) - add_buckyball_mlir_test(${CASE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) - buckyball_mlirtest_name(TEST_NAME ${BUCKYBALL_MLIR_ACTIVE_TARGET} bank_${CASE}) - foreach(KIND baremetal linux) - add_custom_command(OUTPUT ${TEST_NAME}-${KIND} APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../isa/mxmm.h) - endforeach() -endforeach() diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window.mlir b/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window.mlir deleted file mode 100644 index baf9ba037..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window.mlir +++ /dev/null @@ -1,28 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks | FileCheck %s --check-prefix=BANK -// BANK: buckyball.mxfp8_window -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK: buckyball.intr.custom {{.*}} -// CHECK-LABEL: func.func @window_small -// CHECK-DAG: %[[RS1:[^ ]+]] = arith.constant 34367084547 : i64 -// CHECK-DAG: %[[RS2:[^ ]+]] = arith.constant 9007474250153985 : i64 -// CHECK: buckyball.intr.custom %[[RS1]], %[[RS2]] -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 - return -} -func.func @window_small() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 - %b = arith.constant 6 : i64 - %c = arith.constant 7 : i64 - %rows = arith.constant 1 : i64 - %cols = arith.constant 16 : i64 - %count = arith.constant 32 : i64 - %full = arith.constant 64 : i64 - %start = arith.constant 32 : i64 - %first = arith.constant true - %last = arith.constant true - %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain.mlir b/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain.mlir deleted file mode 100644 index 6008bc01a..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain.mlir +++ /dev/null @@ -1,9 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks | FileCheck %s --check-prefix=BANK -// BANK: buckyball.mxfp8_window -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK: buckyball.intr.custom {{.*}} -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows.mlir b/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows.mlir deleted file mode 100644 index 6008bc01a..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows.mlir +++ /dev/null @@ -1,9 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks | FileCheck %s --check-prefix=BANK -// BANK: buckyball.mxfp8_window -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK: buckyball.intr.custom {{.*}} -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 - return -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/CMakeLists.txt b/examples/balls/mxmm/workloads/mlir_tests/tensor/CMakeLists.txt deleted file mode 100644 index efaf53511..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/CMakeLists.txt +++ /dev/null @@ -1,93 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX tensor) - -set(PYTHON ${Python3_EXECUTABLE}) -set(RUNTIME_PARAMS ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}) -if(NOT TARGET runtime) - add_subdirectory(${BUCKYBALL_REPO_ROOT}/stack/runtime ${CMAKE_CURRENT_BINARY_DIR}/runtime) -endif() -if(NOT TARGET rvv_kernels) - set(RUNTIME_PARAMS ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}) - add_subdirectory(${BUCKYBALL_REPO_ROOT}/bb-tests/workloads/lib/bbsw/kernels/rvv - ${CMAKE_CURRENT_BINARY_DIR}/rvv) -endif() -if(NOT TARGET mxfp8_pack) - add_subdirectory(${BUCKYBALL_REPO_ROOT}/bb-tests/workloads/lib/bbsw/kernels/mxfp8 - ${CMAKE_CURRENT_BINARY_DIR}/mxfp8) -endif() -if(NOT TARGET mxmm_kernels) - add_subdirectory(${BUCKYBALL_REPO_ROOT}/examples/balls/mxmm/workloads/kernels - ${CMAKE_CURRENT_BINARY_DIR}/mxmm) -endif() -get_target_property(MXMM_IMAGE_DIR mxmm_kernels BINARY_DIR) -set(MXMM_KERNEL_SOURCE ${BUCKYBALL_REPO_ROOT}/examples/balls/mxmm/workloads/kernels) -file(STRINGS ${RUNTIME_PARAMS}/params.h ROWS_DEFINE REGEX "^#define BANK_LINES ") -file(STRINGS ${RUNTIME_PARAMS}/params.h WIDTH_DEFINE REGEX "^#define BANK_WIDTH ") -string(REGEX REPLACE ".* " "" BANK_ROWS "${ROWS_DEFINE}") -string(REGEX REPLACE ".* " "" ROW_BITS "${WIDTH_DEFINE}") -math(EXPR PACKED_BYTES "${BANK_ROWS} * ${ROW_BITS} / 8") -math(EXPR DOUBLE_PACKED_BYTES "${PACKED_BYTES} * 2") -math(EXPR HALF_PACKED_BYTES "${PACKED_BYTES} / 2") -set(BARE_RVV_ARCHIVE ${CMAKE_CURRENT_BINARY_DIR}/libquant_rvv_baremetal.a) -set(BARE_RVV_TARGET mxmm_quant_rvv_baremetal_${BUCKYBALL_MLIR_ACTIVE_TARGET}) -set(RVV_SOURCE ${BUCKYBALL_REPO_ROOT}/bb-tests/workloads/lib/bbsw/kernels/rvv) -get_target_property(RVV_IMAGE_DIR rvv_images BINARY_DIR) -set(BARE_RVV_FLAGS -O2 -march=rv64gc -mcmodel=medany -std=c++20 - -I${RUNTIME_PARAMS} -I${RVV_IMAGE_DIR} -I${MXMM_IMAGE_DIR} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include - -I${BUCKYBALL_REPO_ROOT}/bb-tests/workloads/lib - -I${BUCKYBALL_REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine) -add_custom_command(OUTPUT ${BARE_RVV_ARCHIVE} - COMMAND ${ELF_CC} ${BARE_RVV_FLAGS} -c ${RVV_SOURCE}/quant_dispatch.cpp -o quant_dispatch_bare.o - COMMAND ${ELF_CC} ${BARE_RVV_FLAGS} -c ${RVV_SOURCE}/../mxfp8/pack.cpp -o quant_pack_bare.o - COMMAND ${ELF_CC} ${BARE_RVV_FLAGS} -c ${RVV_IMAGE_DIR}/images.cpp -o quant_images_bare.o - COMMAND ${ELF_CC} ${BARE_RVV_FLAGS} -c ${MXMM_KERNEL_SOURCE}/pack_dispatch.cpp -o packmm_dispatch_bare.o - COMMAND ${ELF_CC} ${BARE_RVV_FLAGS} -c ${MXMM_KERNEL_SOURCE}/dispatch.cpp -o norm_window_dispatch_bare.o - COMMAND ${CMAKE_AR} rcs ${BARE_RVV_ARCHIVE} quant_dispatch_bare.o quant_pack_bare.o quant_images_bare.o packmm_dispatch_bare.o norm_window_dispatch_bare.o - DEPENDS rvv-images mxmm-images ${MXMM_KERNEL_SOURCE}/pack_dispatch.cpp - ${MXMM_KERNEL_SOURCE}/dispatch.cpp ${MXMM_KERNEL_SOURCE}/window.h - ${MXMM_KERNEL_SOURCE}/packmm.h ${RVV_SOURCE}/quant_dispatch.cpp ${RVV_SOURCE}/../mxfp8/pack.cpp - ${RVV_SOURCE}/../mxfp8/quant.h - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} VERBATIM) -add_custom_target(${BARE_RVV_TARGET}-build DEPENDS ${BARE_RVV_ARCHIVE}) -add_library(${BARE_RVV_TARGET} STATIC IMPORTED) -set_target_properties(${BARE_RVV_TARGET} PROPERTIES IMPORTED_LOCATION ${BARE_RVV_ARCHIVE}) -add_dependencies(${BARE_RVV_TARGET} ${BARE_RVV_TARGET}-build) -add_buckyball_mlir_test(mxmm_fp32 TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET} PASSES - -one-shot-bufferize=bufferize-function-boundaries - -lower-buckyball-to-bank-ssa -assign-physical-banks - -llvm-request-c-wrappers -lower-bank-ssa-to-intrinsics - -expand-strided-metadata -convert-linalg-to-loops -lower-buckyball - -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm - -convert-func-to-llvm -finalize-memref-to-llvm -reconcile-unrealized-casts - LIBRARIES rvv_kernels rvv_images mxmm_kernels BAREMETAL_LIBRARIES ${BARE_RVV_TARGET}) - -set(QUANT_PASSES - -canonicalize -cse -one-shot-bufferize=bufferize-function-boundaries - -buffer-deallocation-pipeline=private-function-dynamic-ownership=false - -lower-buckyball-to-bank-ssa -assign-physical-banks - -llvm-request-c-wrappers -lower-bank-ssa-to-intrinsics - -expand-strided-metadata -convert-linalg-to-loops -lower-buckyball - -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm - -convert-func-to-llvm -finalize-memref-to-llvm -reconcile-unrealized-casts) -foreach(BEHAVIOR shared layout input branch) - add_buckyball_mlir_test(mxmm_quant_${BEHAVIOR} - TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET} PASSES ${QUANT_PASSES} - LIBRARIES mxfp8_pack rvv_quant_backend rvv_images BAREMETAL_LIBRARIES ${BARE_RVV_TARGET}) - set(SOURCE ${CMAKE_CURRENT_BINARY_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}-mxmm_quant_${BEHAVIOR}.mlir) - set(CHECKED ${CMAKE_CURRENT_BINARY_DIR}/mxmm_quant_${BEHAVIOR}-checked.mlir) - add_custom_command(OUTPUT ${CHECKED} - COMMAND ${BUDDY_OPT} ${SOURCE} --target=${BUCKYBALL_MLIR_ACTIVE_TARGET} - -canonicalize -cse -one-shot-bufferize=bufferize-function-boundaries - -buffer-deallocation-pipeline=private-function-dynamic-ownership=false -o ${CHECKED} - COMMAND ${BUCKYBALL_REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/build/bin/FileCheck - ${SOURCE} --input-file=${CHECKED} - DEPENDS ${SOURCE} ${BUDDY_OPT} VERBATIM) - buckyball_mlirtest_name(TEST ${BUCKYBALL_MLIR_ACTIVE_TARGET} tensor_mxmm_quant_${BEHAVIOR}) - add_custom_target(${TEST}-ir-check DEPENDS ${CHECKED}) - add_dependencies(${TEST}-linux-build ${TEST}-ir-check) - add_dependencies(${TEST}-baremetal-build ${TEST}-ir-check) - add_dependencies(${BUCKYBALL_MLIR_GROUP_TARGET} ${TEST}-ir-check) -endforeach() diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_fp32.mlir b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_fp32.mlir deleted file mode 100644 index 679e25cfa..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_fp32.mlir +++ /dev/null @@ -1,8 +0,0 @@ -func.func @tensor_unfused(%a: tensor<8x64xf32>, %b: tensor<64x48xf32>) -> tensor<8x48xf32> { - %0 = buckyball.fp32_matmul %a, %b : tensor<8x64xf32>, tensor<64x48xf32> -> tensor<8x48xf32> - return %0 : tensor<8x48xf32> -} -func.func @tensor_fma(%a: tensor<8x64xf32>, %b: tensor<64x48xf32>) -> tensor<8x48xf32> { - %0 = buckyball.fp32_matmul %a, %b : tensor<8x64xf32>, tensor<64x48xf32> -> tensor<8x48xf32> - return %0 : tensor<8x48xf32> -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_fp32_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_fp32_main.cpp deleted file mode 100644 index 0f2963c30..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_fp32_main.cpp +++ /dev/null @@ -1,39 +0,0 @@ -#include -#include -#include -#include -#include -extern "C" void _mlir_ciface_tensor_fma(StridedMemRefType *, - StridedMemRefType *, - StridedMemRefType *); -extern "C" void _mlir_ciface_tensor_unfused(StridedMemRefType *, - StridedMemRefType *, - StridedMemRefType *); -int main() { - float a[8 * 64]{}, b[64 * 48]{}; - for (int r = 0; r < 8; r++) { - uint32_t x = 0xbf800000, y = 0x3f800001; - memcpy(a + r * 64, &x, 4); - memcpy(a + r * 64 + 1, &y, 4); - } - for (int c = 0; c < 48; c++) { - uint32_t x = 0x3f800000, y = 0x3f7ffffe; - memcpy(b + c, &x, 4); - memcpy(b + 48 + c, &y, 4); - } - StridedMemRefType lhs{a, a, 0, {8, 64}, {64, 1}}, - rhs{b, b, 0, {64, 48}, {48, 1}}, result; - for (int fused = 0; fused < 2; fused++) { - (fused ? _mlir_ciface_tensor_fma : _mlir_ciface_tensor_unfused)(&result, - &lhs, &rhs); - for (int i = 0; i < 384; i++) { - uint32_t bits; - memcpy(&bits, result.data + i, 4); - if (bits != (fused ? 0xa8800000u : 0)) - return 1; - } - free(result.basePtr); - } - puts("Mxmm FP32 tensor M8/N48 fused/unfused PASS"); - return 0; -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_branch.mlir.in b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_branch.mlir.in deleted file mode 100644 index 2e2836d33..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_branch.mlir.in +++ /dev/null @@ -1,26 +0,0 @@ -// CHECK-LABEL: func.func @quant_branch -// CHECK: %[[PACK:.*]] = memref.alloc() alignment -// CHECK: call @mxfp8_quant -// CHECK: scf.if -// CHECK-NOT: memref.dealloc %[[PACK]] -// CHECK: buckyball.mxfp8_mem_matmul %[[PACK]], -// CHECK-NOT: memref.dealloc %[[PACK]] -// CHECK: } else { -// CHECK-NOT: memref.dealloc %[[PACK]] -// CHECK: buckyball.mxfp8_mem_matmul %[[PACK]], -// CHECK: buckyball.mxfp8_mem_matmul %[[PACK]], -// CHECK: memref.dealloc %[[PACK]] -// CHECK-NOT: memref.dealloc %[[PACK]] -// CHECK: return -func.func @quant_branch(%x: tensor<16x64xf32>,%w0: tensor<@PACKED_BYTES@xi8> {bufferization.buffer_layout=strided<[1],offset:?>},%w1: tensor<@PACKED_BYTES@xi8> {bufferization.buffer_layout=strided<[1],offset:?>},%condition:i1) -> (tensor<16x16xf32>,tensor<16x16xf32>) { - %q = buckyball.mxfp8_quant %x : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - %first = scf.if %condition -> (tensor<16x16xf32>) { - %a = buckyball.mxfp8_matmul %q,%w0 : tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8> -> tensor<16x16xf32> - scf.yield %a : tensor<16x16xf32> - } else { - %a = buckyball.mxfp8_matmul %q,%w1 : tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8> -> tensor<16x16xf32> - scf.yield %a : tensor<16x16xf32> - } - %last = buckyball.mxfp8_matmul %q,%w1 : tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8> -> tensor<16x16xf32> - return %first,%last : tensor<16x16xf32>,tensor<16x16xf32> -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_branch_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_branch_main.cpp deleted file mode 100644 index 083c943b4..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_branch_main.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include -#include -#include -#include -#include -#include -struct Results { - StridedMemRefType first, last; -}; -extern "C" void _mlir_ciface_quant_branch(Results *, - StridedMemRefType *, - StridedMemRefType *, - StridedMemRefType *, bool); -int main() { - enum { Bytes = BANK_LINES * (BANK_WIDTH / 8) }; - alignas(64) float input[16 * 64]; - for (auto &value : input) - value = 1.0f; - alignas(64) int8_t a[Bytes]{}, b[Bytes]{}; - memset(a, 0x38, 16 * 64); - memset(b, 0x40, 16 * 64); - memset(a + 16 * 64, 127, 16 * 2); - memset(b + 16 * 64, 127, 16 * 2); - StridedMemRefType x{input, input, 0, {16, 64}, {64, 1}}; - StridedMemRefType wa{a, a, 0, {Bytes}, {1}}, - wb{b, b, 0, {Bytes}, {1}}; - for (bool condition : {false, true}) { - Results result; - _mlir_ciface_quant_branch(&result, &x, &wa, &wb, condition); - for (int row = 0; row < 16; row++) - for (int col = 0; col < 16; col++) { - if (result.first - .data[result.first.offset + row * result.first.strides[0] + - col * result.first.strides[1]] != - (condition ? 64.0f : 128.0f) || - result.last.data[result.last.offset + row * result.last.strides[0] + - col * result.last.strides[1]] != 128.0f) - return 1; - } - free(result.first.basePtr); - free(result.last.basePtr); - } - puts("Mxmm shared packed lifetime across both branch paths PASS"); - return 0; -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_input.mlir.in b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_input.mlir.in deleted file mode 100644 index 652306906..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_input.mlir.in +++ /dev/null @@ -1,9 +0,0 @@ -// CHECK-LABEL: func.func @quant_input -// CHECK-COUNT-2: call @mxfp8_quant -// CHECK-NOT: call @mxfp8_quant -// CHECK: return -func.func @quant_input(%a: tensor<16x64xf32>,%b: tensor<16x64xf32>) -> (tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8>) { - %q0 = buckyball.mxfp8_quant %a : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - %q1 = buckyball.mxfp8_quant %b : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - return %q0,%q1 : tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8> -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_input_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_input_main.cpp deleted file mode 100644 index 93f1e85db..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_input_main.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include -#include -#include -#include -#include -#include -struct Results { - StridedMemRefType a, b; -}; -extern "C" void _mlir_ciface_quant_input(Results *, - StridedMemRefType *, - StridedMemRefType *); -int main() { - alignas(64) float a[16 * 64], b[16 * 64]; - for (auto &value : a) - value = 1.0f; - for (auto &value : b) - value = 2.0f; - StridedMemRefType x{a, a, 0, {16, 64}, {64, 1}}, - y{b, b, 0, {16, 64}, {64, 1}}; - Results result; - _mlir_ciface_quant_input(&result, &x, &y); - StridedMemRefType *buffers[] = {&result.a, &result.b}; - for (int which = 0; which < 2; which++) { - auto &buffer = *buffers[which]; - for (int i = 0; i < 16 * 64; i++) - if (uint8_t(buffer.data[buffer.offset + i * buffer.strides[0]]) != 0x78) - return 1; - for (int i = 0; i < 16 * 2; i++) - if (uint8_t( - buffer.data[buffer.offset + (16 * 64 + i) * buffer.strides[0]]) != - 119 + which) - return 2; - free(buffer.basePtr); - } - puts("Mxmm distinct activation SSA inputs PASS"); - return 0; -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_layout.mlir.in b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_layout.mlir.in deleted file mode 100644 index 5a4f32900..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_layout.mlir.in +++ /dev/null @@ -1,11 +0,0 @@ -// CHECK-LABEL: func.func @quant_layout -// CHECK-COUNT-4: call @mxfp8_quant -// CHECK-NOT: call @mxfp8_quant -// CHECK: return -func.func @quant_layout(%x: tensor<16x64xf32>) -> (tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8>,tensor<@DOUBLE_PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8>) { - %q0 = buckyball.mxfp8_quant %x : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - %q1 = buckyball.mxfp8_quant %x : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - %q2 = buckyball.mxfp8_quant %x : tensor<16x64xf32> -> tensor<@DOUBLE_PACKED_BYTES@xi8> - %q3 = buckyball.mxfp8_quant %x : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - return %q0,%q1,%q2,%q3 : tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8>,tensor<@DOUBLE_PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8> -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_layout_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_layout_main.cpp deleted file mode 100644 index 4afb041c2..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_layout_main.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include -#include -#include -#include -#include -#include -struct Results { - StridedMemRefType row16, row32, k32, stride; -}; -extern "C" void _mlir_ciface_quant_layout(Results *, - StridedMemRefType *); -int main() { - alignas(64) float input[16 * 64]; - for (auto &value : input) - value = 1.0f; - StridedMemRefType x{input, input, 0, {16, 64}, {64, 1}}; - Results result; - _mlir_ciface_quant_layout(&result, &x); - enum { Bytes = BANK_LINES * (BANK_WIDTH / 8) }; - StridedMemRefType *buffers[] = {&result.row16, &result.row32, - &result.k32, &result.stride}; - int rows[] = {16, 32, 16, 16}, ks[] = {64, 64, 32, 32}, - strides[] = {Bytes, Bytes, Bytes, Bytes / 2}; - for (int which = 0; which < 4; which++) { - auto &buffer = *buffers[which]; - for (int panel = 0; panel < 64 / ks[which]; panel++) - for (int row = 0; row < rows[which]; row++) { - int base = panel * strides[which], width = ks[which]; - for (int k = 0; k < width; k++) - if (uint8_t(buffer.data[buffer.offset + (base + row * width + k) * - buffer.strides[0]]) != - (row < 16 ? 0x78 : 0)) - return 1; - for (int k = 0; k < width / 32; k++) - if (uint8_t(buffer.data[buffer.offset + (base + rows[which] * width + - row * (width / 32) + k) * - buffer.strides[0]]) != - (row < 16 ? 119 : 127)) - return 2; - } - free(buffer.basePtr); - } - puts("Mxmm distinct rows/K/stride layouts: defined payload PASS"); - return 0; -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_shared.mlir.in b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_shared.mlir.in deleted file mode 100644 index 75cd802ee..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_shared.mlir.in +++ /dev/null @@ -1,16 +0,0 @@ -// CHECK-LABEL: func.func @quant_shared -// CHECK: %[[PACK:.*]] = memref.alloc() alignment -// CHECK: call @mxfp8_quant -// CHECK-NOT: call @mxfp8_quant -// CHECK: buckyball.mxfp8_mem_matmul %[[PACK]], -// CHECK: buckyball.mxfp8_mem_matmul %[[PACK]], -// CHECK: memref.dealloc %[[PACK]] -// CHECK-NOT: memref.dealloc %[[PACK]] -// CHECK: return -func.func @quant_shared(%x: tensor<16x64xf32>, %gate: tensor<@PACKED_BYTES@xi8> {bufferization.buffer_layout=strided<[1],offset:?>}, %up: tensor<@PACKED_BYTES@xi8> {bufferization.buffer_layout=strided<[1],offset:?>}) -> (tensor<16x16xf32>,tensor<16x16xf32>) { - %q0 = buckyball.mxfp8_quant %x : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - %a = buckyball.mxfp8_matmul %q0,%gate : tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8> -> tensor<16x16xf32> - %q1 = buckyball.mxfp8_quant %x : tensor<16x64xf32> -> tensor<@PACKED_BYTES@xi8> - %b = buckyball.mxfp8_matmul %q1,%up : tensor<@PACKED_BYTES@xi8>,tensor<@PACKED_BYTES@xi8> -> tensor<16x16xf32> - return %a,%b : tensor<16x16xf32>,tensor<16x16xf32> -} diff --git a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_shared_main.cpp b/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_shared_main.cpp deleted file mode 100644 index 823137b3c..000000000 --- a/examples/balls/mxmm/workloads/mlir_tests/tensor/mxmm_quant_shared_main.cpp +++ /dev/null @@ -1,41 +0,0 @@ -#include -#include -#include -#include -#include -#include -struct Results { - StridedMemRefType gate, up; -}; -extern "C" void _mlir_ciface_quant_shared(Results *, - StridedMemRefType *, - StridedMemRefType *, - StridedMemRefType *); -int main() { - enum { Bytes = BANK_LINES * (BANK_WIDTH / 8) }; - alignas(64) float input[16 * 64]; - alignas(64) int8_t gate[Bytes]{}, up[Bytes]{}; - for (auto &value : input) - value = 1.0f; - memset(gate, 0x38, 16 * 64); - memset(up, 0x40, 16 * 64); - memset(gate + 16 * 64, 127, 16 * 2); - memset(up + 16 * 64, 127, 16 * 2); - StridedMemRefType x{input, input, 0, {16, 64}, {64, 1}}; - StridedMemRefType a{gate, gate, 0, {Bytes}, {1}}, - b{up, up, 0, {Bytes}, {1}}; - Results result; - _mlir_ciface_quant_shared(&result, &x, &a, &b); - for (int row = 0; row < 16; row++) - for (int col = 0; col < 16; col++) { - if (result.gate.data[result.gate.offset + row * result.gate.strides[0] + - col * result.gate.strides[1]] != 64.0f || - result.up.data[result.up.offset + row * result.up.strides[0] + - col * result.up.strides[1]] != 128.0f) - return 1; - } - free(result.gate.basePtr); - free(result.up.basePtr); - puts("Mxmm shared activation: two matrix consumers PASS"); - return 0; -} diff --git a/examples/balls/mxquant/arch/src/main/scala/MxquantBall.scala b/examples/balls/mxquant/arch/src/main/scala/MxquantBall.scala deleted file mode 100644 index a5b180db5..000000000 --- a/examples/balls/mxquant/arch/src/main/scala/MxquantBall.scala +++ /dev/null @@ -1,158 +0,0 @@ -package examples.balls.mxquant - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.top.GlobalConfig - -@instantiable -class MxquantBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - private val mapping = b.ballDomain.ballIdMappings.find(_.ballName == "MxquantBall").get - private val funct = b.ballDomain.ballISA.find(_.mnemonic == "MXQUANT").get.funct7 - require( - b.memDomain.bankWidth == 128 && b.memDomain.bankMaskLen == 16, - "MxquantBall requires 128-bit bank rows and byte write masks" - ) - require(b.memDomain.bankEntries >= 8, "MxquantBall requires eight input rows") - require(mapping.inBW == 1 && mapping.outBW == 1, "MxquantBall requires one read and one write") - require((funct >> 4) == 3, "MxquantBall funct7 must encode one read and one write") - - @public - val io = IO(new BlinkIO(b, 1, 1)) - def blink: BlinkIO = io - def status: BallStatus = io.status - - private val idle :: readReq :: readResp :: exponent :: encode :: codeReq :: codeResp :: scaleReq :: scaleResp :: complete :: Nil = - Enum(10) - private val state = RegInit(idle) - private val robId = Reg(UInt(log2Up(b.frontend.rob_entries).W)) - private val subId = Reg(UInt(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val isSub = Reg(Bool()) - private val source = Reg(UInt(b.memDomain.vbankIdWidth.W)) - private val destination = Reg(UInt(b.memDomain.vbankIdWidth.W)) - private val count = Reg(UInt(b.frontend.iter_len.W)) - private val group = RegInit(0.U(b.frontend.iter_len.W)) - private val readRow = RegInit(0.U(3.W)) - private val lane = RegInit(0.U(5.W)) - private val codeRow = RegInit(0.U(1.W)) - private val maximum = RegInit(0.U(31.W)) - private val blockExponent = Reg(SInt(10.W)) - private val scale = Reg(UInt(8.W)) - private val values = Reg(Vec(32, UInt(32.W))) - private val codes = Reg(Vec(32, UInt(8.W))) - private val math = Module(new Encode) - math.io.input := values(lane) - math.io.blockExponent := blockExponent - - private val active = !reset.asBool - io.cmdReq.ready := state === idle && active - io.cmdResp.valid := state === complete && active - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subId - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) - - val read = io.bankRead(0) - read.rob_id := robId - read.ball_id := 0.U - read.bank_id := source - read.group_id := 0.U - read.io.req.valid := state === readReq && active - read.io.req.bits.addr := (group << 3) + readRow - read.io.resp.ready := state === readResp && active - - val write = io.bankWrite(0) - write.rob_id := robId - write.ball_id := 0.U - write.bank_id := destination - write.group_id := 0.U - write.io.req.valid := (state === codeReq || state === scaleReq) && active - write.io.req.bits.addr := Mux(state === scaleReq, (count >> 4) + (group >> 4), (group << 1) + codeRow) - val lowCodes = Cat((15 to 0 by -1).map(codes(_))) - val highCodes = Cat((31 to 16 by -1).map(codes(_))) - write.io.req.bits.data := Mux( - state === scaleReq, - (scale.pad(128) << (group(3, 0) << 3))(127, 0), - Mux(codeRow === 0.U, lowCodes, highCodes) - ) - write.io.req.bits.mask := VecInit((0 until 16).map(i => state =/= scaleReq || group(3, 0) === i.U)) - write.io.resp.ready := (state === codeResp || state === scaleResp) && active - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - assert(cmd.funct7 === funct.U, "MxquantBall unknown funct7") - assert(cmd.iter > 0.U && cmd.iter(4, 0) === 0.U, "MxquantBall count must be positive and divisible by 32") - assert(cmd.iter <= (b.memDomain.bankEntries * 4).U, "MxquantBall input exceeds capacity") - assert(cmd.iter +& (cmd.iter >> 5) <= (b.memDomain.bankEntries * 16).U, "MxquantBall output exceeds capacity") - assert(cmd.rs1(19, 10) === 0.U && cmd.rs2 === 0.U, "MxquantBall reserved fields must be zero") - assert( - cmd.rs1(9, 0) < b.memDomain.virtualBankCount.U && cmd.rs1(29, 20) < b.memDomain.virtualBankCount.U, - "MxquantBall invalid bank" - ) - assert(cmd.op1_col === 1.U && cmd.wr_col === 1.U, "MxquantBall operands must occupy one bank") - assert(cmd.op1_bank =/= cmd.wr_bank, "MxquantBall banks must differ") - robId := io.cmdReq.bits.rob_id - subId := io.cmdReq.bits.sub_rob_id - isSub := io.cmdReq.bits.is_sub - source := cmd.op1_bank - destination := cmd.wr_bank - count := cmd.iter - group := 0.U - readRow := 0.U - maximum := 0.U - state := readReq - } - } - is(readReq)(when(read.io.req.fire)(state := readResp)) - is(readResp) { - when(read.io.resp.fire) { - val words = (0 until 4).map(i => read.io.resp.bits.data(32 * i + 31, 32 * i)) - for (i <- 0 until 4) { - assert(words(i)(30, 23) =/= 255.U, "MxquantBall input must be finite") - values(Cat(readRow, i.U(2.W))) := words(i) - } - maximum := (Seq(maximum) ++ words.map(_(30, 0))).reduce((a, c) => Mux(a > c, a, c)) - when(readRow === 7.U)(state := exponent).otherwise { readRow := readRow + 1.U; state := readReq } - } - } - is(exponent) { - val raw = maximum(30, 23).zext - 135.S(10.W) - val selected = Mux(maximum === 0.U, 0.S(10.W), Mux(raw < -127.S, -127.S(10.W), raw)) - blockExponent := selected - scale := (selected + 127.S).asUInt - lane := 0.U - state := encode - } - is(encode) { - codes(lane) := math.io.code - when(lane === 31.U) { codeRow := 0.U; state := codeReq }.otherwise(lane := lane + 1.U) - } - is(codeReq)(when(write.io.req.fire)(state := codeResp)) - is(codeResp) { - when(write.io.resp.fire) { - when(codeRow === 0.U) { codeRow := 1.U; state := codeReq }.otherwise(state := scaleReq) - } - } - is(scaleReq)(when(write.io.req.fire)(state := scaleResp)) - is(scaleResp) { - when(write.io.resp.fire) { - when(group === (count >> 5) - 1.U)(state := complete).otherwise { - group := group + 1.U - readRow := 0.U - maximum := 0.U - state := readReq - } - } - } - is(complete)(when(io.cmdResp.fire)(state := idle)) - } -} diff --git a/examples/balls/mxquant/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/mxquant/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 90aae51ad..000000000 --- a/examples/balls/mxquant/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "mlir/IR/PatternMatch.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateMxquantBallAssignPhysicalBankPatterns(RewritePatternSet &, - PhysicalBankState &); -} - -namespace { -class BankMxquantPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(BankMxquantOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), - op.getIter()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateMxquantBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, PhysicalBankState &) { - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/mxquant/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/mxquant/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 473dea024..000000000 --- a/examples/balls/mxquant/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct MxquantLowering : public ConvertOpToLLVMPattern { - MxquantLowering(LLVMTypeConverter &converter, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(MxquantOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("MxquantBall"); - if (buckyball_target::getBuckyballTarget().bankWidthBits != 128) - return op.emitError("mxquant: requires 16-byte bank rows"); - if (auto constant = op.getIter().getDefiningOp()) { - int64_t count = cast(constant.getValue()).getInt(); - if (count <= 0 || count % 32) - return op.emitError("mxquant: count must be a positive multiple of 32"); - if (count > bankDepth * 4 || count + count / 32 > bankDepth * 16) - return op.emitError("mxquant: bank footprint exceeds capacity"); - } - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, cstI64(rewriter, loc, 0), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MXQUANT"))); - return success(); - } - -private: - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateMxquantBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, - int64_t bankDepth) { - patterns.add(converter, bankDepth); -} -void configureMxquantBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/mxquant/configs/default.toml b/examples/balls/mxquant/configs/default.toml deleted file mode 100644 index c61b39cb6..000000000 --- a/examples/balls/mxquant/configs/default.toml +++ /dev/null @@ -1 +0,0 @@ -[ball] diff --git a/examples/balls/mxquant/emu/src/dispatch.rs b/examples/balls/mxquant/emu/src/dispatch.rs deleted file mode 100644 index afa7b38aa..000000000 --- a/examples/balls/mxquant/emu/src/dispatch.rs +++ /dev/null @@ -1,30 +0,0 @@ -use super::quant; -use crate::inst::instruction::ExecContext; - -const BALL_CLASS: &str = "examples.balls.mxquant.MxquantBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MXQUANT") => Some(quant::execute(xs1, xs2, ctx)), - _ => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, _xs1: u64, _xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("MXQUANT") => Some(1), - _ => None, - } -} diff --git a/examples/balls/mxquant/emu/src/lib.rs b/examples/balls/mxquant/emu/src/lib.rs deleted file mode 100644 index 647db3f0c..000000000 --- a/examples/balls/mxquant/emu/src/lib.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod dispatch; -mod quant; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/mxquant/emu/src/quant.rs b/examples/balls/mxquant/emu/src/quant.rs deleted file mode 100644 index a657a420b..000000000 --- a/examples/balls/mxquant/emu/src/quant.rs +++ /dev/null @@ -1,91 +0,0 @@ -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::ExecContext; -use crate::bank::{bank_lines, bank_row_bytes}; - -fn rounded_shift(value: u32, shift: u32) -> u32 { - if shift > 31 { - return 0; - } - let half = 1 << (shift - 1); - (value + half - 1 + ((value >> shift) & 1)) >> shift -} - -fn encode(bits: u32, block_exponent: i32) -> u8 { - let sign = ((bits >> 24) & 128) as u8; - if bits & 0x7fffffff == 0 { - return sign; - } - let mut exponent = ((bits >> 23) & 255) as i32; - let mut fraction = bits & 0x7fffff; - if exponent == 0 { - let shift = fraction.leading_zeros() - 8; - fraction = (fraction << shift) & 0x7fffff; - exponent = 1 - shift as i32; - } - exponent -= block_exponent + 120; - let code = if exponent <= 0 { - rounded_shift(fraction | 0x800000, (21 - exponent) as u32) - } else { - exponent as u32 * 8 + rounded_shift(fraction, 20) - }; - sign | code.min(126) as u8 -} - -pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let input = rs1_b0(xs1); - let output = rs1_b2(xs1); - let count = rs1_iter(xs1) as usize; - assert!( - xs2 == 0 && rs1_b1(xs1) == 0, - "mxquant: reserved fields must be zero" - ); - assert!( - count > 0 && count % 32 == 0, - "mxquant: count must be a positive multiple of 32" - ); - assert!( - input != output, - "mxquant: input and output banks must differ" - ); - for bank in [input, output] { - assert!( - ctx.config(bank).allocated && ctx.config(bank).cols == 1, - "mxquant: banks must be allocated with one column" - ); - } - let bytes = bank_lines() * bank_row_bytes(); - assert!( - bank_row_bytes() == 16 && count <= bytes / 4 && count + count / 32 <= bytes, - "mxquant: bank footprint exceeds capacity" - ); - let source = &ctx.banks[pbank(ctx, input)][..count * 4]; - assert!( - source - .chunks_exact(4) - .all( - |element| u32::from_le_bytes(element.try_into().unwrap()) & 0x7fffffff < 0x7f800000 - ), - "mxquant: non-finite activation" - ); - let mut packed = vec![0u8; count + count / 32]; - for (block, values) in source.chunks_exact(128).enumerate() { - let maximum = values - .chunks_exact(4) - .map(|element| u32::from_le_bytes(element.try_into().unwrap()) & 0x7fffffff) - .max() - .unwrap(); - let exponent = if maximum == 0 { - 0 - } else { - ((maximum >> 23) as i32 - 135).max(-127) - }; - packed[count + block] = (exponent + 127) as u8; - for (index, element) in values.chunks_exact(4).enumerate() { - packed[block * 32 + index] = - encode(u32::from_le_bytes(element.try_into().unwrap()), exponent); - } - } - let destination = pbank(ctx, output); - ctx.banks[destination][..packed.len()].copy_from_slice(&packed); - 0 -} diff --git a/examples/balls/mxquant/workloads/ctests/CMakeLists.txt b/examples/balls/mxquant/workloads/ctests/CMakeLists.txt deleted file mode 100644 index b4967e1cd..000000000 --- a/examples/balls/mxquant/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -file(READ "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h" MXQUANT_PARAMS) -string(REGEX MATCH "#define BANK_LINES ([0-9]+)" _ "${MXQUANT_PARAMS}") -set(MXQUANT_BANK_LINES ${CMAKE_MATCH_1}) -string(REGEX MATCH "#define BANK_WIDTH ([0-9]+)" _ "${MXQUANT_PARAMS}") -math(EXPR MXQUANT_BANK_BYTES "${MXQUANT_BANK_LINES} * ${CMAKE_MATCH_1} / 8") -math(EXPR MXQUANT_BANK_TAIL "${MXQUANT_BANK_BYTES} % 128") -if(MXQUANT_BANK_BYTES LESS 3840 OR MXQUANT_BANK_TAIL) - message(FATAL_ERROR - "Mxquant CTests require banks >= 3840 bytes and a 32-element FP32 capacity; actual ${MXQUANT_BANK_BYTES} bytes") -endif() -add_buckyball_ctests(mxquant_values_test.c mxquant_capacity_test.c) -buckyball_ctest_name(TEST_NAME ${BUCKYBALL_CTEST_TARGET} mxquant_values_test) -add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}-baremetal APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../reference.h) diff --git a/examples/balls/mxquant/workloads/mlir_tests/ball/CMakeLists.txt b/examples/balls/mxquant/workloads/mlir_tests/ball/CMakeLists.txt deleted file mode 100644 index d94a22636..000000000 --- a/examples/balls/mxquant/workloads/mlir_tests/ball/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(BUCKYBALL_MLIR_TEST_PREFIX ball) -add_buckyball_mlir_test(mxquant TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) -buckyball_mlirtest_name(TEST_NAME ${BUCKYBALL_MLIR_ACTIVE_TARGET} ball_mxquant) -foreach(KIND baremetal linux) - add_custom_command(OUTPUT ${TEST_NAME}-${KIND} APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../reference.h) -endforeach() diff --git a/examples/balls/mxquant/workloads/mlir_tests/ball/mxquant.mlir b/examples/balls/mxquant/workloads/mlir_tests/ball/mxquant.mlir deleted file mode 100644 index 6b9a62331..000000000 --- a/examples/balls/mxquant/workloads/mlir_tests/ball/mxquant.mlir +++ /dev/null @@ -1,20 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @quant_block_32 -// CHECK: %[[ZERO:[^ ]+]] = arith.constant 0 : i64 -// CHECK: %[[PACKED:[^ ]+]] = arith.constant 34363932675 : i64 -// CHECK: buckyball.intr.custom %[[PACKED]], %[[ZERO]] -// CHECK-LABEL: func.func @quant_dynamic -// CHECK: buckyball.intr.custom {{.*}} -func.func @quant_block_32() attributes {llvm.emit_c_interface} { - %input = arith.constant 3 : i64 - %output = arith.constant 4 : i64 - %count = arith.constant 32 : i64 - buckyball.mxquant %input, %output, %count : i64 - return -} -func.func @quant_dynamic(%count: i64) attributes {llvm.emit_c_interface} { - %input = arith.constant 3 : i64 - %output = arith.constant 4 : i64 - buckyball.mxquant %input, %output, %count : i64 - return -} diff --git a/examples/balls/mxquant/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/mxquant/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index b486e6c26..000000000 --- a/examples/balls/mxquant/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(BUCKYBALL_MLIR_TEST_PREFIX bank) -add_buckyball_mlir_test(mxquant TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) -buckyball_mlirtest_name(TEST_NAME ${BUCKYBALL_MLIR_ACTIVE_TARGET} bank_mxquant) -foreach(KIND baremetal linux) - add_custom_command(OUTPUT ${TEST_NAME}-${KIND} APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../reference.h) -endforeach() diff --git a/examples/balls/mxquant/workloads/mlir_tests/bank/mxquant.mlir b/examples/balls/mxquant/workloads/mlir_tests/bank/mxquant.mlir deleted file mode 100644 index da2203912..000000000 --- a/examples/balls/mxquant/workloads/mlir_tests/bank/mxquant.mlir +++ /dev/null @@ -1,25 +0,0 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks | FileCheck %s --check-prefix=BANK -// BANK-LABEL: func.func @quant_block_32 -// BANK: buckyball.mxquant -// BANK-LABEL: func.func @quant_dynamic -// BANK: buckyball.mxquant -// RUN: buddy-opt %s --target=toy -assign-physical-banks -lower-bank-ssa-to-intrinsics -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @quant_block_32 -// CHECK: %[[ZERO:[^ ]+]] = arith.constant 0 : i64 -// CHECK: %[[PACKED:[^ ]+]] = arith.constant 34363932675 : i64 -// CHECK: buckyball.intr.custom %[[PACKED]], %[[ZERO]] -// CHECK-LABEL: func.func @quant_dynamic -// CHECK: buckyball.intr.custom {{.*}} -func.func @quant_block_32() attributes {llvm.emit_c_interface} { - %input = arith.constant 3 : i64 - %output = arith.constant 4 : i64 - %count = arith.constant 32 : i64 - %result = buckyball.bank_mxquant %input %output %count : i64 i64 i64 - return -} -func.func @quant_dynamic(%count: i64) attributes {llvm.emit_c_interface} { - %input = arith.constant 3 : i64 - %output = arith.constant 4 : i64 - %result = buckyball.bank_mxquant %input %output %count : i64 i64 i64 - return -} diff --git a/examples/balls/relu/README.md b/examples/balls/relu/README.md deleted file mode 100644 index f4b81faa7..000000000 --- a/examples/balls/relu/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# ReluBall - -`ReluBall` computes signed INT32 `max(x, 0)` in place in one physical pbank. -It has no internal SRAM: a 128-bit register holds one pbank line between its -read and write requests. - -## Command - -`RELU(vbank, group, iter, stride)` selects physical pbank `group` in virtual -bank `vbank`. `group` is a pbank number, never a data-layout term. - -`iter` is the number of 128-bit physical lines processed in each segment. -`stride` is the distance between segments. Both are positive; `iter` is a -multiple of 16, `iter <= stride`, and `stride` divides the pbank depth. The -command processes `iter` lines at every `stride` boundary until it reaches the -end of the selected pbank. - -For every line the Ball performs one single-port sequence: - -```text -pbank read -> 4 signed int32 ReLUs -> pbank write -``` - -The selected group must be allocated. There is no output vbank, no fallback -layout, and no tail path. Tile lowering pads logical matrices to 16-aligned -shapes; compiler packing fills the final pbank chunk before issuing ReLU. diff --git a/examples/balls/relu/arch/src/main/scala/Relu.scala b/examples/balls/relu/arch/src/main/scala/Relu.scala deleted file mode 100644 index 4cfbf250c..000000000 --- a/examples/balls/relu/arch/src/main/scala/Relu.scala +++ /dev/null @@ -1,164 +0,0 @@ -package examples.balls.relu - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} -import chisel3.util._ -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.top.GlobalConfig - -@instantiable -class Relu(val b: GlobalConfig) extends Module { - private val addressWidth = log2Up(b.memDomain.bankEntries) - private val countWidth = log2Up(b.memDomain.bankEntries + 1) - private val bankIdWidth = b.memDomain.vbankIdWidth - private val groupWidth = log2Up(b.memDomain.bankNum) - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "ReluBall") - .getOrElse(throw new IllegalArgumentException("ReluBall not found in config")) - - private val funct = b.ballDomain.ballISA - .find(_.mnemonic == "RELU") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("RELU not found in ballISA")) - - require(mapping.inBW == 1, "ReluBall requires one SRAM read port") - require(mapping.outBW == 1, "ReluBall requires one SRAM write port") - require(b.memDomain.bankWidth == 128, "ReluBall requires 128-bit SRAM rows") - require(b.memDomain.bankMaskLen == 16, "ReluBall requires sixteen byte enables") - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val channelReady = Input(Bool()) - val bankRead = Vec(1, Flipped(new BankRead(b))) - val bankWrite = Vec(1, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - val Seq(idle, waitForChannels, readRequest, readResponse, writeRequest, writeResponse, complete) = Enum(7) - val state = RegInit(idle) - - val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val isSub = RegInit(false.B) - val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - val bank = RegInit(0.U(bankIdWidth.W)) - val group = RegInit(0.U(groupWidth.W)) - val iter = RegInit(0.U(countWidth.W)) - val stride = RegInit(0.U(countWidth.W)) - val segment = RegInit(0.U(addressWidth.W)) - val line = RegInit(0.U(addressWidth.W)) - val word = Reg(UInt(128.W)) - - val relu = Wire(Vec(4, UInt(32.W))) - for (lane <- 0 until 4) { - val value = word(32 * lane + 31, 32 * lane).asSInt - relu(lane) := Mux(value < 0.S, 0.U, value.asUInt) - } - val address = segment + line - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRobId - - io.bankRead(0).rob_id := robId - io.bankRead(0).ball_id := 0.U - io.bankRead(0).bank_id := bank - io.bankRead(0).group_id := group - io.bankRead(0).io.req.valid := false.B - io.bankRead(0).io.req.bits.addr := address - io.bankRead(0).io.resp.ready := false.B - - io.bankWrite(0).rob_id := robId - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := bank - io.bankWrite(0).group_id := group - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := address - io.bankWrite(0).io.req.bits.data := Cat(relu.reverse) - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val command = io.cmdReq.bits.cmd - assert(command.funct7 === funct.U(7.W), "ReluBall funct7 must be RELU") - assert(command.wr_bank === 0.U, "ReluBall bank2 must be zero") - assert(command.op1_col =/= 0.U && command.op2_bank < command.op1_col, "ReluBall group is not allocated in bank") - assert(command.iter =/= 0.U && command.iter(3, 0) === 0.U, "ReluBall iter must be a positive multiple of 16") - assert( - command.special =/= 0.U && command.special <= b.memDomain.bankEntries.U, - "ReluBall stride must fit in one physical bank" - ) - assert( - b.memDomain.bankEntries.U % command.special === 0.U, - "ReluBall stride must divide the physical bank depth" - ) - assert(command.iter <= command.special, "ReluBall iter must not exceed stride") - - robId := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRobId := io.cmdReq.bits.sub_rob_id - bank := command.op1_bank - group := command.op2_bank - iter := command.iter - stride := command.special - segment := 0.U - line := 0.U - state := waitForChannels - } - } - - is(waitForChannels) { - when(io.channelReady)(state := readRequest) - } - - is(readRequest) { - io.bankRead(0).io.req.valid := true.B - when(io.bankRead(0).io.req.fire)(state := readResponse) - } - - is(readResponse) { - io.bankRead(0).io.resp.ready := true.B - when(io.bankRead(0).io.resp.fire) { - word := io.bankRead(0).io.resp.bits.data - state := writeRequest - } - } - - is(writeRequest) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire)(state := writeResponse) - } - - is(writeResponse) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - when(line +& 1.U === iter) { - when(segment +& stride === b.memDomain.bankEntries.U) { - state := complete - }.otherwise { - segment := segment + stride - line := 0.U - state := readRequest - } - }.otherwise { - line := line + 1.U - state := readRequest - } - } - } - - is(complete) { - when(io.cmdResp.fire)(state := idle) - } - } - - io.status.idle := state === idle - io.status.running := state =/= idle -} diff --git a/examples/balls/relu/arch/src/main/scala/ReluBall.scala b/examples/balls/relu/arch/src/main/scala/ReluBall.scala deleted file mode 100644 index 91024b258..000000000 --- a/examples/balls/relu/arch/src/main/scala/ReluBall.scala +++ /dev/null @@ -1,33 +0,0 @@ -package examples.balls.relu - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.MmioRead -import framework.top.GlobalConfig - -@instantiable -class ReluBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "ReluBall") - .getOrElse(throw new IllegalArgumentException("ReluBall not found in config")) - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - - def blink: BlinkIO = io - def status = io.status - dontTouch(io) - - val unit: Instance[Relu] = Instantiate(new Relu(b)) - unit.io.cmdReq <> io.cmdReq - unit.io.cmdResp <> io.cmdResp - unit.io.channelReady := io.channelReady - unit.io.bankRead(0) <> io.bankRead(0) - unit.io.bankWrite(0) <> io.bankWrite(0) - io.status <> unit.io.status - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/relu/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/relu/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 8ecc276ff..000000000 --- a/examples/balls/relu/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateReluBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} - -namespace { -class BankReluPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankReluOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getBank(), op.getGroup(), - op.getIter(), op.getStride()); - rewriter.replaceOp(op, op.getBank()); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateReluBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/relu/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp b/examples/balls/relu/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp deleted file mode 100644 index c88dde897..000000000 --- a/examples/balls/relu/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp +++ /dev/null @@ -1,112 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -constexpr int64_t kValuesPerLine = 4; - -class ReluMatrixToBankSSAPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(ReluMatrixOp op, - PatternRewriter &b) const override { - auto inputType = dyn_cast(op.getInput().getType()); - auto outputType = dyn_cast(op.getOutput().getType()); - if (!inputType || !outputType || !inputType.hasStaticShape() || - inputType != outputType || !inputType.getElementType().isInteger(32)) - return op.emitError("requires matching static memref"); - - int64_t rows = inputType.getShape()[0]; - int64_t columns = inputType.getShape()[1]; - if (rows <= 0 || columns <= 0 || rows % 16 || columns % 16) - return op.emitError("requires positive 16-aligned dimensions"); - - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth <= 0 || - target.bankDepth % 16) - return op.emitError( - "ReluBall requires a 128-bit bank with 16-aligned depth"); - - Location loc = op.getLoc(); - int64_t lines = target.bankDepth; - int64_t values = lines * kValuesPerLine; - int64_t total = rows * columns; - Value bank = allocBank(b, loc, 1, 1); - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value four = b.create(loc, kValuesPerLine); - Value columnsValue = b.create(loc, columns); - auto packType = MemRefType::get({lines, kValuesPerLine}, b.getI32Type()); - - for (int64_t base = 0; base < total; base += values) { - int64_t count = std::min(values, total - base); - Value inputPack = b.create(loc, packType); - Value outputPack = b.create(loc, packType); - Value zeroValue = b.create(loc, 0, 32); - b.create(loc, zeroValue, inputPack); - - Value countValue = b.create(loc, count); - Value baseValue = b.create(loc, base); - auto copyIn = b.create(loc, zero, countValue, one); - b.setInsertionPointToStart(copyIn.getBody()); - Value index = copyIn.getInductionVar(); - Value source = b.create(loc, baseValue, index); - Value row = b.create(loc, source, columnsValue); - Value column = b.create(loc, source, columnsValue); - Value line = b.create(loc, index, four); - Value lane = b.create(loc, index, four); - Value value = - b.create(loc, op.getInput(), ValueRange{row, column}); - b.create(loc, value, inputPack, ValueRange{line, lane}); - b.setInsertionPointAfter(copyIn); - - Value loaded = mvinBank(b, loc, inputPack, bank, lines); - Value result = b.create( - loc, bank.getType(), loaded, createI64Const(b, loc, 0), - createI64Const(b, loc, lines), createI64Const(b, loc, lines)); - bank = mvoutBank(b, loc, outputPack, result, lines); - - auto copyOut = b.create(loc, zero, countValue, one); - b.setInsertionPointToStart(copyOut.getBody()); - index = copyOut.getInductionVar(); - source = b.create(loc, baseValue, index); - row = b.create(loc, source, columnsValue); - column = b.create(loc, source, columnsValue); - line = b.create(loc, index, four); - lane = b.create(loc, index, four); - value = b.create(loc, outputPack, ValueRange{line, lane}); - b.create(loc, value, op.getOutput(), - ValueRange{row, column}); - b.setInsertionPointAfter(copyOut); - b.create(loc, inputPack); - b.create(loc, outputPack); - } - - releaseBank(b, loc, bank); - b.eraseOp(op); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateReluBallLowerBuckyballToBankSSAPatterns( - RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/relu/compiler/src/Conversion/LowerTileToBuckyball/TileReluPatterns.cpp b/examples/balls/relu/compiler/src/Conversion/LowerTileToBuckyball/TileReluPatterns.cpp deleted file mode 100644 index 9c49d9ec5..000000000 --- a/examples/balls/relu/compiler/src/Conversion/LowerTileToBuckyball/TileReluPatterns.cpp +++ /dev/null @@ -1,91 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Tile/TileOps.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -namespace tile = ::buddy::tile; - -namespace { -class TileReluLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(tile::TileReluOp op, - PatternRewriter &b) const override { - Value input = op.getInput(); - Value output = op.getOutput(); - auto inputType = dyn_cast(input.getType()); - auto outputType = dyn_cast(output.getType()); - if (!inputType || !outputType || !inputType.hasStaticShape() || - inputType != outputType || inputType.getRank() != 2 || - !inputType.getElementType().isInteger(32)) - return op.emitError("requires matching static memref"); - - int64_t rows = inputType.getShape()[0]; - int64_t columns = inputType.getShape()[1]; - if (rows <= 0 || columns <= 0) - return op.emitError("requires positive dimensions"); - - int64_t paddedRows = llvm::alignTo(rows, 16); - int64_t paddedColumns = llvm::alignTo(columns, 16); - Location loc = op.getLoc(); - if (paddedRows == rows && paddedColumns == columns) { - b.create(loc, input, output); - b.eraseOp(op); - return success(); - } - - auto paddedType = - MemRefType::get({paddedRows, paddedColumns}, b.getI32Type()); - Value paddedInput = b.create(loc, paddedType); - Value paddedOutput = b.create(loc, paddedType); - Value zeroValue = b.create(loc, 0, 32); - b.create(loc, zeroValue, paddedInput); - - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value rowLimit = b.create(loc, rows); - Value columnLimit = b.create(loc, columns); - auto copyInputRows = b.create(loc, zero, rowLimit, one); - b.setInsertionPointToStart(copyInputRows.getBody()); - Value row = copyInputRows.getInductionVar(); - auto copyInputColumns = b.create(loc, zero, columnLimit, one); - b.setInsertionPointToStart(copyInputColumns.getBody()); - Value column = copyInputColumns.getInductionVar(); - Value value = b.create(loc, input, ValueRange{row, column}); - b.create(loc, value, paddedInput, ValueRange{row, column}); - b.setInsertionPointAfter(copyInputRows); - - b.create(loc, paddedInput, paddedOutput); - - auto copyOutputRows = b.create(loc, zero, rowLimit, one); - b.setInsertionPointToStart(copyOutputRows.getBody()); - row = copyOutputRows.getInductionVar(); - auto copyOutputColumns = b.create(loc, zero, columnLimit, one); - b.setInsertionPointToStart(copyOutputColumns.getBody()); - column = copyOutputColumns.getInductionVar(); - value = - b.create(loc, paddedOutput, ValueRange{row, column}); - b.create(loc, value, output, ValueRange{row, column}); - b.setInsertionPointAfter(copyOutputRows); - - b.create(loc, paddedInput); - b.create(loc, paddedOutput); - b.eraseOp(op); - return success(); - } -}; -} // namespace - -namespace mlir::buddy { -void populateReluBallTileLoweringPatterns(RewritePatternSet &patterns, int64_t, - int64_t, int64_t) { - patterns.add(patterns.getContext()); -} -} // namespace mlir::buddy diff --git a/examples/balls/relu/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/relu/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 36dec7d68..000000000 --- a/examples/balls/relu/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct ReluLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(ReluOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("ReluBall"); - Location loc = op.getLoc(); - Value rs1 = - packRs1BanksIter(rewriter, loc, adaptor.getBankId(), adaptor.getGroup(), - cstI64(rewriter, loc, 0), adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getStride(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("RELU"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateReluBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns, - bool, int64_t) { - patterns.add(converter); -} - -void configureReluBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/relu/configs/default.toml b/examples/balls/relu/configs/default.toml deleted file mode 100644 index 80321fad9..000000000 --- a/examples/balls/relu/configs/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -# ReluBall has no implementation-specific parameters. -[ball] diff --git a/examples/balls/relu/emu/src/dispatch.rs b/examples/balls/relu/emu/src/dispatch.rs deleted file mode 100644 index aca809580..000000000 --- a/examples/balls/relu/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::f50_relu; - -const BALL_CLASS: &str = "examples.balls.relu.ReluBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("RELU") => Some(f50_relu::Relu::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("RELU") => Some(f50_relu::Relu::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/relu/emu/src/lib.rs b/examples/balls/relu/emu/src/lib.rs deleted file mode 100644 index c691f0d9e..000000000 --- a/examples/balls/relu/emu/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -#[path = "50_relu.rs"] -mod f50_relu; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/relu/workloads/ctests/CMakeLists.txt b/examples/balls/relu/workloads/ctests/CMakeLists.txt deleted file mode 100644 index cfdbcc1df..000000000 --- a/examples/balls/relu/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_buckyball_ctests( - relu_i32_16x16_test.c -) diff --git a/examples/balls/relu/workloads/isa/relu.h b/examples/balls/relu/workloads/isa/relu.h deleted file mode 100644 index 7f6589b67..000000000 --- a/examples/balls/relu/workloads/isa/relu.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BB_RELU_H_ -#define _BB_RELU_H_ - -#include -#include - -#define bb_relu(bank_id, group, iter, stride) \ - BUCKYBALL_INSTRUCTION_R_R( \ - (BB_BANK0(bank_id) | BB_BANK1(group) | BB_ITER(iter)), (stride), \ - BB_FUNC7(RELU)) - -#endif // _BB_RELU_H_ diff --git a/examples/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir b/examples/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir deleted file mode 100644 index 496be2993..000000000 --- a/examples/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir +++ /dev/null @@ -1,34 +0,0 @@ -func.func private @check_result(memref<16x4xi32>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %four_i32 = arith.constant 4 : i32 - %sixteen_i32 = arith.constant 16 : i32 - %iter = arith.constant 16 : i64 - %group = arith.constant 0 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() : memref<16x4xi32> - %output = memref.alloc() : memref<16x4xi32> - scf.for %row = %zero to %sixteen step %one { - scf.for %lane = %zero to %four step %one { - %r = arith.index_cast %row : index to i32 - %l = arith.index_cast %lane : index to i32 - %base = arith.muli %r, %four_i32 : i32 - %value = arith.addi %base, %l : i32 - %signed = arith.subi %value, %sixteen_i32 : i32 - memref.store %signed, %input[%row, %lane] : memref<16x4xi32> - } - } - %bank = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %bank %iter %stride : memref<16x4xi32> i64 i64 i64 - buckyball.relu %loaded, %group, %iter, %iter : i64 - %stored = buckyball.bank_mvout %output %loaded %iter %stride : memref<16x4xi32> i64 i64 i64 - func.call @check_result(%output) : (memref<16x4xi32>) -> () - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<16x4xi32> - memref.dealloc %output : memref<16x4xi32> - return %zero_i8 : i8 -} diff --git a/examples/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in b/examples/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in deleted file mode 100644 index f6de71d79..000000000 --- a/examples/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in +++ /dev/null @@ -1,43 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () - -func.func @main() -> i8 { - %zeroI8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %group = arith.constant 0 : i64 - %lines = arith.constant @BB_BANK_LINES@ : i64 - %stride = arith.constant 1 : i64 - %limit = arith.constant @BB_BANK_LINES@ : index - %lanes = arith.constant @BB_BANK_I32_LANES@ : index - %lane_count = arith.constant @BB_BANK_I32_LANES@ : i32 - %seventeen = arith.constant 17 : i32 - %bias = arith.constant 8 : i32 - - %input = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - %output = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - - scf.for %line = %zero to %limit step %one { - scf.for %lane = %zero to %lanes step %one { - %line_i32 = arith.index_cast %line : index to i32 - %lane_i32 = arith.index_cast %lane : index to i32 - %base = arith.muli %line_i32, %lane_count : i32 - %index = arith.addi %base, %lane_i32 : i32 - %remainder = arith.remui %index, %seventeen : i32 - %signed = arith.subi %remainder, %bias : i32 - memref.store %signed, %input[%line, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - } - } - - %bank = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %bank %lines %stride - : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> i64 i64 i64 - %result = buckyball.bank_relu %loaded %group %lines %lines : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %lines %stride - : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> i64 i64 i64 - buckyball.bank_release %stored : i64 - - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () - memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - return %zeroI8 : i8 -} diff --git a/examples/balls/smatmul/arch/src/main/scala/Array.scala b/examples/balls/smatmul/arch/src/main/scala/Array.scala deleted file mode 100644 index 77d458342..000000000 --- a/examples/balls/smatmul/arch/src/main/scala/Array.scala +++ /dev/null @@ -1,83 +0,0 @@ -package examples.balls.smatmul - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} -import examples.balls.smatmul.configs.SMatMulBallParam -import framework.top.GlobalConfig - -@instantiable -class Array(val b: GlobalConfig) extends Module { - private val param = SMatMulBallParam(b) - private val rows = param.tileRows - private val cols = param.tileCols - - @public - val io = IO(new Bundle { - val start = Input(Bool()) - val aRows = Input(Vec(rows, UInt(128.W))) - val bRows = Input(Vec(cols, UInt(128.W))) - val done = Output(Bool()) - val result = Output(Vec(rows, UInt(512.W))) - }) - - val running = RegInit(false.B) - val cycle = RegInit(0.U(6.W)) - val pe = Seq.tabulate(rows, cols)((_, _) => Instantiate(new PE)) - val aShift = Reg(Vec(rows, UInt(128.W))) - // The PE product register adds one cycle between the systolic operands and - // the accumulator. Keep the final drain cycle explicit so the last tile - // product is included before done is asserted. - val lastCyc = (rows + cols + 15).U - - when(io.start) { - assert(!running, "Array: start while array is running") - running := true.B - cycle := 0.U - aShift := io.aRows - }.elsewhen(running) { - when(cycle === lastCyc) { - running := false.B - }.otherwise { - cycle := cycle + 1.U - } - for (row <- 0 until rows) { - when(cycle >= row.U && cycle < (row + cols).U) { - aShift(row) := aShift(row) >> 8 - } - } - } - - for (row <- 0 until rows) { - val aWord = aShift(row)(7, 0).asSInt - val aValid = running && cycle >= row.U && cycle < (row + cols).U - - for (column <- 0 until cols) { - pe(row)(column).io.clear := io.start - if (column == 0) { - pe(row)(column).io.aIn := aWord - pe(row)(column).io.aInValid := aValid - } else { - pe(row)(column).io.aIn := pe(row)(column - 1).io.aOut - pe(row)(column).io.aInValid := pe(row)(column - 1).io.aOutValid - } - - val bCycle = cycle - column.U - val bRow = bCycle(3, 0) - val bWord = io.bRows(bRow)(8 * column + 7, 8 * column).asSInt - val bValid = running && cycle >= column.U && cycle < (column + cols).U - if (row == 0) { - pe(row)(column).io.bIn := bWord - pe(row)(column).io.bInValid := bValid - } else { - pe(row)(column).io.bIn := pe(row - 1)(column).io.bOut - pe(row)(column).io.bInValid := pe(row - 1)(column).io.bOutValid - } - } - } - - for (row <- 0 until rows) { - io.result(row) := Cat((0 until cols).reverse.map(column => pe(row)(column).io.sum.asUInt)) - } - io.done := running && cycle === lastCyc -} diff --git a/examples/balls/smatmul/arch/src/main/scala/PE.scala b/examples/balls/smatmul/arch/src/main/scala/PE.scala deleted file mode 100644 index b01cae5b5..000000000 --- a/examples/balls/smatmul/arch/src/main/scala/PE.scala +++ /dev/null @@ -1,60 +0,0 @@ -package examples.balls.smatmul - -import chisel3._ -import chisel3.experimental.hierarchy.{instantiable, public} - -@instantiable -class PE extends Module { - - // One tile accumulates sixteen signed 8x8 products. Twenty bits cover the - // full range [-262144, 258064]; the unit-level accumulator remains 32-bit - // for reduction across tiles. - private val tileSumWidth = 20 - - @public - val io = IO(new Bundle { - val aIn = Input(SInt(8.W)) - val aInValid = Input(Bool()) - val bIn = Input(SInt(8.W)) - val bInValid = Input(Bool()) - val clear = Input(Bool()) - val aOut = Output(SInt(8.W)) - val aOutValid = Output(Bool()) - val bOut = Output(SInt(8.W)) - val bOutValid = Output(Bool()) - val sum = Output(SInt(32.W)) - }) - - val accumulator = RegInit(0.S(tileSumWidth.W)) - val aPipe = RegInit(0.S(8.W)) - val aPipeValid = RegInit(false.B) - val bPipe = RegInit(0.S(8.W)) - val bPipeValid = RegInit(false.B) - val product = RegInit(0.S(16.W)) - val productValid = RegInit(false.B) - - when(io.clear) { - accumulator := 0.S - aPipeValid := false.B - bPipeValid := false.B - productValid := false.B - }.elsewhen(productValid) { - accumulator := accumulator + product - } - when(!io.clear) { - product := aPipe * bPipe - productValid := aPipeValid && bPipeValid - } - when(!io.clear) { - aPipe := io.aIn - aPipeValid := io.aInValid - bPipe := io.bIn - bPipeValid := io.bInValid - } - - io.aOut := aPipe - io.aOutValid := aPipeValid - io.bOut := bPipe - io.bOutValid := bPipeValid - io.sum := accumulator.pad(32) -} diff --git a/examples/balls/smatmul/arch/src/main/scala/SMatMulBall.scala b/examples/balls/smatmul/arch/src/main/scala/SMatMulBall.scala deleted file mode 100644 index d4b4970e5..000000000 --- a/examples/balls/smatmul/arch/src/main/scala/SMatMulBall.scala +++ /dev/null @@ -1,50 +0,0 @@ -package examples.balls.smatmul - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.MmioRead -import framework.top.GlobalConfig - -@instantiable -class SMatMulBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - val ballCommonConfig = b.ballDomain.ballIdMappings - .find(_.ballName == "SMatMulBall") - .getOrElse( - throw new IllegalArgumentException("SMatMulBall not found in config") - ) - - val inBW = ballCommonConfig.inBW - val outBW = ballCommonConfig.outBW - - @public - val io = IO(new BlinkIO(b, inBW, outBW)) - - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - val smatmulArrayUnit: Instance[SMatMulUnit] = Instantiate(new SMatMulUnit(b)) - - smatmulArrayUnit.io.cmdReq <> io.cmdReq - smatmulArrayUnit.io.cmdResp <> io.cmdResp - smatmulArrayUnit.io.channelReady := io.channelReady - - for (i <- 0 until inBW) { - smatmulArrayUnit.io.bankRead(i) <> io.bankRead(i) - } - - for (i <- 0 until outBW) { - smatmulArrayUnit.io.bankWrite(i) <> io.bankWrite(i) - } - - io.status <> smatmulArrayUnit.io.status - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - - // MMIO: this Ball does not consume MMIO metadata - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/smatmul/arch/src/main/scala/SMatMulUnit.scala b/examples/balls/smatmul/arch/src/main/scala/SMatMulUnit.scala deleted file mode 100644 index 8304e912f..000000000 --- a/examples/balls/smatmul/arch/src/main/scala/SMatMulUnit.scala +++ /dev/null @@ -1,431 +0,0 @@ -package examples.balls.smatmul - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.top.GlobalConfig -import examples.balls.smatmul.configs.SMatMulBallParam - -@instantiable -class SMatMulUnit(val b: GlobalConfig) extends Module { - private val param = SMatMulBallParam(b) - private val rows = param.tileRows - private val cols = param.tileCols - private val resultWords = 4 - private val addressWidth = log2Ceil(b.memDomain.bankEntries) - private val bankIdWidth = b.memDomain.vbankIdWidth - private val maxOutputRows = b.memDomain.bankEntries / resultWords - private val accDepth = maxOutputRows.max(32) - private val accAddrWidth = log2Ceil(accDepth) - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "SMatMulBall") - .getOrElse(throw new IllegalArgumentException("SMatMulBall not found in config")) - - private val osFunct = b.ballDomain.ballISA - .find(_.mnemonic == "SMATMUL_OS") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("SMATMUL_OS not found in ballISA")) - - private val biasFunct = b.ballDomain.ballISA - .find(_.mnemonic == "SMATMUL_BIAS") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("SMATMUL_BIAS not found in ballISA")) - - require(mapping.inBW == 2, "SMatMulBall requires inBW=2") - require(mapping.outBW == 1, "SMatMulBall requires outBW=1") - require(b.memDomain.bankWidth == 128, "SMatMulBall requires 128-bit bank rows") - require(b.memDomain.bankMaskLen == 16, "SMatMulBall requires sixteen byte enables") - require(b.memDomain.bankEntries % resultWords == 0, "SMatMulBall bank depth must be divisible by four") - require(maxOutputRows >= rows, "SMatMulBall accumulator must hold at least one tile") - require((osFunct >> 4) == 4, "SMATMUL_OS must encode two reads and one write") - require((biasFunct >> 4) == 1, "SMATMUL_BIAS must encode one read") - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(mapping.inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(mapping.outBW, Flipped(new BankWrite(b))) - val channelReady = Input(Bool()) - val status = new BallStatus - }) - - private val Seq( - idle, - waitForChannels, - biasReadReq, - biasReadResp, - initAccumulator, - loadTile, - runArray, - readAccumulator, - writeAccumulator, - readResult, - holdResult, - writeResult, - waitForCWrite, - complete - ) = Enum(14) - - private val state = RegInit(idle) - - private val robId = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSub = RegInit(false.B) - private val subRobId = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - - private val biasCommand = RegInit(false.B) - private val firstBlock = RegInit(false.B) - private val lastBlock = RegInit(false.B) - private val vectorMode = RegInit(false.B) - private val biasValid = RegInit(false.B) - private val chainLive = RegInit(false.B) - private val biasBank = RegInit(0.U(bankIdWidth.W)) - private val biasBase = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val biasRow = RegInit(0.U(2.W)) - private val biasWords = Reg(Vec(resultWords, UInt(128.W))) - - private val aBank = RegInit(0.U(bankIdWidth.W)) - private val bBank = RegInit(0.U(bankIdWidth.W)) - private val cBank = RegInit(0.U(bankIdWidth.W)) - private val cBase = RegInit(0.U(addressWidth.W)) - private val chainRows = RegInit(0.U(12.W)) - private val chainCBank = RegInit(0.U(bankIdWidth.W)) - private val chainCBase = RegInit(0.U(addressWidth.W)) - - private val outputTileCount = RegInit(0.U(8.W)) - private val reductionTileCount = RegInit(0.U(8.W)) - private val outputTile = RegInit(0.U(8.W)) - private val reductionTile = RegInit(0.U(8.W)) - private val accumulatorRow = RegInit(0.U(4.W)) - private val resultRow = RegInit(0.U(4.W)) - private val outputWord = RegInit(0.U(2.W)) - private val aRowsRequested = RegInit(0.U(5.W)) - private val aRowsStored = RegInit(0.U(5.W)) - private val bRowsRequested = RegInit(0.U(5.W)) - private val bRowsStored = RegInit(0.U(5.W)) - private val cRowData = Reg(UInt(512.W)) - - private val aRows = Reg(Vec(rows, UInt(128.W))) - private val bRows = Reg(Vec(cols, UInt(128.W))) - private val accumulator = SyncReadMem(accDepth, UInt(512.W)) - private val array: Instance[Array] = Instantiate(new Array(b)) - - private val computeGlobalRow = outputTile * rows.U + accumulatorRow - private val resultGlobalRow = outputTile * rows.U + resultRow - private val accumulatorRead = state === readAccumulator || state === readResult - private val accumulatorWrite = state === initAccumulator || state === writeAccumulator - - private val accumulatorAddress = Mux( - state === readResult, - resultGlobalRow(accAddrWidth - 1, 0), - computeGlobalRow(accAddrWidth - 1, 0) - ) - - private val biasData = Cat(biasWords.reverse) - private val accumulatorWriteData = Wire(UInt(512.W)) - - private val accumulatorData = accumulator.readWrite( - accumulatorAddress, - accumulatorWriteData, - accumulatorRead || accumulatorWrite, - accumulatorWrite - ) - - private val accumulatedResult = Wire(Vec(cols, UInt(32.W))) - for (column <- 0 until cols) { - val oldValue = accumulatorData(32 * column + 31, 32 * column).asSInt - val newValue = array.io.result(accumulatorRow)(32 * column + 31, 32 * column).asSInt - accumulatedResult(column) := (oldValue + newValue).asUInt - } - accumulatorWriteData := Mux(state === initAccumulator, biasData, Cat(accumulatedResult.reverse)) - assert(!(accumulatorRead && accumulatorWrite), "SMatMulBall accumulator SRAM is single-port") - - private val aTileLine = Mux( - vectorMode, - reductionTile, - (outputTile * reductionTileCount + reductionTile) * rows.U - ) - - private val bTileLine = reductionTile * cols.U - private val cLine = cBase +& (resultGlobalRow << 2) +& outputWord - - for (port <- 0 until mapping.inBW) { - io.bankRead(port).rob_id := robId - io.bankRead(port).ball_id := 0.U - io.bankRead(port).bank_id := Mux(port.U === 0.U, aBank, bBank) - io.bankRead(port).group_id := 0.U - io.bankRead(port).io.req.valid := false.B - io.bankRead(port).io.req.bits.addr := 0.U - io.bankRead(port).io.resp.ready := false.B - } - io.bankRead(0).bank_id := Mux(state === biasReadReq || state === biasReadResp, biasBank, aBank) - io.bankRead(0).io.req.valid := Mux( - state === biasReadReq, - true.B, - state === loadTile && aRowsRequested < Mux(vectorMode, 1.U, rows.U) - ) - io.bankRead(0).io.req.bits.addr := Mux( - state === biasReadReq, - (biasBase +& biasRow)(addressWidth - 1, 0), - (aTileLine + aRowsRequested)(addressWidth - 1, 0) - ) - io.bankRead(0).io.resp.ready := state === biasReadResp || (state === loadTile && aRowsStored < rows.U) - io.bankRead(1).io.req.valid := state === loadTile && bRowsRequested < cols.U - io.bankRead(1).io.req.bits.addr := (bTileLine + bRowsRequested)(addressWidth - 1, 0) - io.bankRead(1).io.resp.ready := state === loadTile && bRowsStored < cols.U - - private val cWords = cRowData.asTypeOf(Vec(resultWords, UInt(128.W))) - io.bankWrite(0).rob_id := robId - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := cBank - io.bankWrite(0).group_id := 0.U - io.bankWrite(0).io.req.valid := state === writeResult - io.bankWrite(0).io.req.bits.addr := cLine(addressWidth - 1, 0) - io.bankWrite(0).io.req.bits.data := cWords(outputWord) - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) - io.bankWrite(0).io.resp.ready := state === waitForCWrite - - array.io.start := state === loadTile && - aRowsStored === Mux(vectorMode, 1.U, rows.U) && bRowsStored === cols.U - array.io.aRows := aRows - array.io.bRows := bRows - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robId - io.cmdResp.bits.is_sub := isSub - io.cmdResp.bits.sub_rob_id := subRobId - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - when(io.cmdReq.fire) { - val command = io.cmdReq.bits.cmd - val isBias = command.funct7 === biasFunct.U - val isMatmul = command.funct7 === osFunct.U - val rows = command.rs2(11, 0) - val columns = command.rs2(23, 12) - val reduction = command.iter - val first = command.rs2(24) - val last = command.rs2(25) - - assert(isBias || isMatmul, "SMatMulBall received an unknown funct7") - assert(command.rs1(9, 0) < b.memDomain.virtualBankCount.U, "SMatMulBall input bank 0 is invalid") - - robId := io.cmdReq.bits.rob_id - isSub := io.cmdReq.bits.is_sub - subRobId := io.cmdReq.bits.sub_rob_id - biasCommand := isBias - - when(isBias) { - assert(command.rs1(29, 10) === 0.U, "SMATMUL_BIAS reserves bank1 and bank2") - assert(command.iter === 4.U, "SMATMUL_BIAS iter must be four rows") - assert(command.rs2(63, 6) === 0.U, "SMATMUL_BIAS reserves rs2[63:6]") - assert( - command.rs2(5, 0) +& 4.U <= b.memDomain.bankEntries.U, - "SMATMUL_BIAS inputBase plus four rows must fit one bank" - ) - assert( - command.op1_col === 1.U && command.op2_col === 0.U && command.wr_col === 0.U, - "SMATMUL_BIAS requires exactly one input bank" - ) - assert(!chainLive, "SMATMUL_BIAS cannot replace bias during an accumulation chain") - biasBank := command.op1_bank - biasBase := command.rs2(5, 0) - biasRow := 0.U - }.otherwise { - assert(command.rs1(19, 10) < b.memDomain.virtualBankCount.U, "SMATMUL_OS input bank 1 is invalid") - assert(command.rs1(29, 20) < b.memDomain.virtualBankCount.U, "SMATMUL_OS output bank is invalid") - assert( - command.op1_bank =/= command.op2_bank && command.op1_bank =/= command.wr_bank && - command.op2_bank =/= command.wr_bank, - "SMATMUL_OS requires distinct A, B, and C banks" - ) - assert( - command.op1_col === 1.U && command.op2_col === 1.U && command.wr_col === 1.U, - "SMATMUL_OS requires one physical bank per operand" - ) - assert( - rows === 1.U || (rows =/= 0.U && rows(2, 0) === 0.U), - "SMATMUL_OS M must be one or a positive multiple of 8" - ) - assert(columns === cols.U, "SMATMUL_OS N must be 16") - assert(reduction =/= 0.U && reduction(3, 0) === 0.U, "SMATMUL_OS K must be a positive multiple of 16") - assert(command.rs2(63, 32) === 0.U, "SMATMUL_OS reserves rs2[63:32]") - assert( - Mux(rows === 1.U, reduction >> 4, (rows * reduction) >> 4) <= b.memDomain.bankEntries.U, - "SMATMUL_OS A footprint exceeds bank depth" - ) - assert(reduction <= b.memDomain.bankEntries.U, "SMATMUL_OS B footprint exceeds bank depth") - assert( - command.rs2(31, 26) +& rows * resultWords.U <= b.memDomain.bankEntries.U, - "SMATMUL_OS C footprint exceeds bank depth" - ) - when(first) { - assert(!chainLive, "SMATMUL_OS first block cannot start while a chain is live") - assert(biasValid, "SMATMUL_OS first block requires a bias preload") - chainLive := true.B - chainRows := rows - chainCBank := command.wr_bank - chainCBase := command.rs2(31, 26) - }.otherwise { - assert(chainLive, "SMATMUL_OS continuation requires a live chain") - assert( - rows === chainRows && command.wr_bank === chainCBank && - command.rs2(31, 26) === chainCBase, - "SMATMUL_OS continuation changed M or C destination/base" - ) - } - - firstBlock := first - lastBlock := last - vectorMode := rows === 1.U - aBank := command.op1_bank - bBank := command.op2_bank - cBank := command.wr_bank - cBase := command.rs2(31, 26) - outputTileCount := Mux(rows === 1.U, 1.U, rows / param.tileRows.U) - reductionTileCount := reduction >> 4 - outputTile := 0.U - reductionTile := 0.U - accumulatorRow := 0.U - resultRow := 0.U - outputWord := 0.U - aRowsRequested := 0.U - aRowsStored := 0.U - bRowsRequested := 0.U - bRowsStored := 0.U - for (row <- 1 until 8) { - aRows(row) := 0.U - } - } - state := waitForChannels - } - - when(state === waitForChannels && io.channelReady) { - state := Mux(biasCommand, biasReadReq, Mux(firstBlock, initAccumulator, loadTile)) - } - - when(state === biasReadReq && io.bankRead(0).io.req.fire) { - state := biasReadResp - } - when(state === biasReadResp && io.bankRead(0).io.resp.fire) { - biasWords(biasRow) := io.bankRead(0).io.resp.bits.data - when(biasRow === (resultWords - 1).U) { - biasValid := true.B - state := complete - }.otherwise { - biasRow := biasRow + 1.U - state := biasReadReq - } - } - - when(state === initAccumulator) { - when(accumulatorRow === Mux(vectorMode, 0.U, (rows - 1).U)) { - accumulatorRow := 0.U - when(outputTile + 1.U === outputTileCount) { - outputTile := 0.U - reductionTile := 0.U - aRowsRequested := 0.U - aRowsStored := 0.U - bRowsRequested := 0.U - bRowsStored := 0.U - state := loadTile - }.otherwise { - outputTile := outputTile + 1.U - } - }.otherwise { - accumulatorRow := accumulatorRow + 1.U - } - } - - when(state === loadTile) { - when(io.bankRead(0).io.req.fire)(aRowsRequested := aRowsRequested + 1.U) - when(io.bankRead(1).io.req.fire)(bRowsRequested := bRowsRequested + 1.U) - when(io.bankRead(0).io.resp.fire) { - aRows(aRowsStored(3, 0)) := io.bankRead(0).io.resp.bits.data - aRowsStored := aRowsStored + 1.U - } - when(io.bankRead(1).io.resp.fire) { - bRows(bRowsStored(3, 0)) := io.bankRead(1).io.resp.bits.data - bRowsStored := bRowsStored + 1.U - } - when(aRowsStored === Mux(vectorMode, 1.U, rows.U) && bRowsStored === cols.U) { - state := runArray - } - } - - when(state === runArray && array.io.done) { - accumulatorRow := 0.U - state := readAccumulator - } - when(state === readAccumulator) { - state := writeAccumulator - } - when(state === writeAccumulator) { - when(accumulatorRow === Mux(vectorMode, 0.U, (rows - 1).U)) { - accumulatorRow := 0.U - when(reductionTile + 1.U < reductionTileCount) { - reductionTile := reductionTile + 1.U - aRowsRequested := 0.U - aRowsStored := 0.U - bRowsRequested := 0.U - bRowsStored := 0.U - state := loadTile - }.elsewhen(outputTile + 1.U < outputTileCount) { - outputTile := outputTile + 1.U - reductionTile := 0.U - aRowsRequested := 0.U - aRowsStored := 0.U - bRowsRequested := 0.U - bRowsStored := 0.U - state := loadTile - }.otherwise { - when(lastBlock) { - outputTile := 0.U - resultRow := 0.U - state := readResult - }.otherwise { - state := complete - } - } - }.otherwise { - accumulatorRow := accumulatorRow + 1.U - state := readAccumulator - } - } - - when(state === readResult) { - outputWord := 0.U - state := holdResult - } - when(state === holdResult) { - cRowData := accumulatorData - state := writeResult - } - when(state === writeResult && io.bankWrite(0).io.req.fire) { - state := waitForCWrite - } - when(state === waitForCWrite && io.bankWrite(0).io.resp.fire) { - when(outputWord =/= (resultWords - 1).U) { - outputWord := outputWord + 1.U - state := writeResult - }.elsewhen(resultRow =/= Mux(vectorMode, 0.U, (rows - 1).U)) { - resultRow := resultRow + 1.U - state := readResult - }.elsewhen(outputTile + 1.U < outputTileCount) { - outputTile := outputTile + 1.U - resultRow := 0.U - state := readResult - }.otherwise { - chainLive := false.B - state := complete - } - } - - when(state === complete && io.cmdResp.fire) { - state := idle - } -} diff --git a/examples/balls/smatmul/compiler/runtime/fp32.cpp b/examples/balls/smatmul/compiler/runtime/fp32.cpp deleted file mode 100644 index 73c7c7768..000000000 --- a/examples/balls/smatmul/compiler/runtime/fp32.cpp +++ /dev/null @@ -1,99 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -extern "C" void _mlir_ciface_fp32_matmul(UnrankedMemRefType *lhs, - UnrankedMemRefType *rhs, - UnrankedMemRefType *result) { - DynamicMemRefType a(*lhs), b(*rhs), c(*result); - const int rank = a.rank; - if ((rank != 2 && rank != 3) || b.rank != rank || c.rank != rank) { - fputs("FP32 matmul requires matrices or matched batches\n", stderr); - abort(); - } - const int64_t batches = rank == 3 ? a.sizes[0] : 1; - const int64_t m = a.sizes[rank - 2], k = a.sizes[rank - 1], - n = b.sizes[rank - 1]; - if (batches <= 0 || m <= 0 || k <= 0 || n <= 0 || k != b.sizes[rank - 2] || - c.sizes[rank - 2] != m || c.sizes[rank - 1] != n || - (rank == 3 && (b.sizes[0] != batches || c.sizes[0] != batches))) { - fputs("FP32 matmul shape mismatch\n", stderr); - abort(); - } - constexpr int block = BANK_LINES * (BANK_WIDTH / 8) / (16 * sizeof(float)); - static_assert(block > 0 && block % 4 == 0 && block < 4096 && - BANK_WIDTH == 128); - alignas(64) float ap[16 * block], bp[16 * block], cp[256]; - for (int bank = 0; bank < 3; ++bank) - bb_mem_alloc(bank, 1, 1); - for (int64_t batch = 0; batch < batches; ++batch) { - const float *av = - a.data + a.offset + (rank == 3 ? batch * a.strides[0] : 0); - const float *bv = - b.data + b.offset + (rank == 3 ? batch * b.strides[0] : 0); - float *cv = c.data + c.offset + (rank == 3 ? batch * c.strides[0] : 0); - for (int64_t row = 0; row < m; row += 16) - for (int64_t column = 0; column < n; column += 16) { - int rows = m - row == 1 ? 1 : 16; - for (int64_t inner = 0; inner < k; inner += block) { - int count = int((std::min(block, k - inner) + 3) / 4 * 4); - const float *as = - av + row * a.strides[rank - 2] + inner * a.strides[rank - 1]; - const float *bs = - bv + inner * b.strides[rank - 2] + column * b.strides[rank - 1]; - bool direct_a = row + rows <= m && inner + count <= k && - a.strides[rank - 1] == 1 && - (rows == 1 || a.strides[rank - 2] == count); - bool direct_b = column + 16 <= n && inner + count <= k && - b.strides[rank - 2] == 1 && - b.strides[rank - 1] == count; - if (!direct_a) - for (int i = 0; i < rows; ++i) - if (row + i < m && a.strides[rank - 1] == 1) { - int valid = int(std::min(count, k - inner)); - std::memcpy(ap + i * count, as + i * a.strides[rank - 2], - valid * sizeof(float)); - std::fill_n(ap + i * count + valid, count - valid, 0.0f); - } else - for (int j = 0; j < count; ++j) - ap[i * count + j] = row + i < m && inner + j < k - ? as[i * a.strides[rank - 2] + - j * a.strides[rank - 1]] - : 0; - if (!direct_b) - for (int i = 0; i < 16; ++i) - if (column + i < n && b.strides[rank - 2] == 1) { - int valid = int(std::min(count, k - inner)); - std::memcpy(bp + i * count, bs + i * b.strides[rank - 1], - valid * sizeof(float)); - std::fill_n(bp + i * count + valid, count - valid, 0.0f); - } else - for (int j = 0; j < count; ++j) - bp[i * count + j] = column + i < n && inner + j < k - ? bs[j * b.strides[rank - 2] + - i * b.strides[rank - 1]] - : 0; - bb_mvin((uintptr_t)(direct_a ? as : ap), 0, rows * count / 4, 1); - bb_mvin((uintptr_t)(direct_b ? bs : bp), 1, 16 * count / 4, 1); - bb_smatmul_f32(0, 1, 2, rows, 16, count, inner == 0, - inner + count >= k, 0); - } - bb_mvout((uintptr_t)cp, 2, rows * 4, 1); - for (int i = 0; i < rows && row + i < m; ++i) - if (c.strides[rank - 1] == 1) - std::memcpy(cv + (row + i) * c.strides[rank - 2] + column, - cp + i * 16, - std::min(16, n - column) * sizeof(float)); - else - for (int j = 0; j < 16 && column + j < n; ++j) - cv[(row + i) * c.strides[rank - 2] + - (column + j) * c.strides[rank - 1]] = cp[i * 16 + j]; - } - } - for (int bank = 0; bank < 3; ++bank) - bb_mem_release(bank); -} diff --git a/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 62dcf68a9..000000000 --- a/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,53 +0,0 @@ -//===- AssignPhysicalBankPatterns.cpp - Matrix bank assignment patterns ---===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateSMatMulBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { - -class BankSMatMulPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankSMatMulOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), - op.getWrBank(), op.getConfig(), op.getFirst(), - op.getLast(), op.getOutputBase()); - rewriter.replaceOp(op, op.getWrBank()); - return success(); - } -}; - -class BankSMatMulBiasPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankSMatMulBiasOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getBiasBank(), - op.getInputBase()); - rewriter.replaceOp(op, op.getBiasBank()); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateSMatMulBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add( - patterns.getContext()); -} diff --git a/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp b/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp deleted file mode 100644 index dfeeed1e5..000000000 --- a/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp +++ /dev/null @@ -1,357 +0,0 @@ -//===- LowerBuckyballToBankSSAPatterns.cpp - Mega MatMul to banks --------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -constexpr int64_t kTile = 16; -constexpr int64_t kInt32RowsPerTile = 64; - -class MegaMatmulToBankSSAPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(MegaMatmulOp op, - PatternRewriter &b) const override { - Location loc = op.getLoc(); - auto aTy = dyn_cast(op.getInput().getType()); - auto wTy = dyn_cast(op.getWeight().getType()); - auto biasTy = dyn_cast(op.getBias().getType()); - auto scaleTy = dyn_cast(op.getScale().getType()); - auto lutTy = dyn_cast(op.getLut().getType()); - auto outTy = dyn_cast(op.getOutput().getType()); - auto kernel = op->getParentOfType(); - if (!kernel) - return op.emitError( - "Mega MatMul must be nested in buckyball.mega_kernel"); - bool finalOutput = op.getOutput() == kernel.getOutput(); - if (!aTy || !wTy || !biasTy || !scaleTy || !lutTy || !outTy || - !aTy.hasStaticShape() || !wTy.hasStaticShape() || - !biasTy.hasStaticShape() || !scaleTy.hasStaticShape() || - !lutTy.hasStaticShape() || !outTy.hasStaticShape()) - return op.emitError("Mega MatMul requires static memrefs"); - if (!aTy.getElementType().isInteger(8) || - !wTy.getElementType().isInteger(8) || - !biasTy.getElementType().isInteger(32) || - !scaleTy.getElementType().isF32() || - !lutTy.getElementType().isInteger(8) || lutTy.getRank() != 1 || - lutTy.getShape()[0] != (op.getActivation() == 2 ? 256 : 1) || - op.getActivation() < 0 || op.getActivation() > 2 || - (finalOutput && op.getActivation() == 2) || - (finalOutput ? !outTy.getElementType().isF32() - : !outTy.getElementType().isInteger(8))) - return op.emitError( - "Mega MatMul input/weight must be INT8, bias INT32, scale FP32, " - "and only the last stage may output FP32"); - - int64_t M = aTy.getShape()[0]; - int64_t K = aTy.getShape()[1]; - int64_t N = wTy.getShape()[1]; - if (M <= 0 || K <= 0 || N <= 0 || wTy.getShape()[0] != K || - biasTy.getShape()[0] != N || scaleTy.getShape()[0] != N || - outTy.getShape()[0] != M || outTy.getShape()[1] != N) - return op.emitError("Mega MatMul shape mismatch"); - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth < kInt32RowsPerTile || - target.bankDepth % kInt32RowsPerTile) - return op.emitError( - "Mega MatMul requires 128-bit banks whose depth is a positive " - "multiple of 64 rows"); - if (buckyball_target::getBuckyballBallMapping("SMatMulBall").outBW != 1) - return op.emitError("Mega MatMul requires SMatMulBall outBW=1"); - - const int64_t hardwareRows = - buckyball_target::getBuckyballBallParam("SMatMulBall", "tileRows"); - if (hardwareRows <= 0 || kTile % hardwareRows) - return op.emitError( - "SMatMul tileRows must divide the 16-row software tile"); - const int64_t tileRows = M == 1 ? 1 : kTile; - const int64_t resultRows = tileRows * 4; - int64_t paddedM = (M + tileRows - 1) / tileRows * tileRows; - int64_t paddedK = (K + kTile - 1) / kTile * kTile; - int64_t kChunk = target.bankDepth / kTile * kTile; - if (kChunk <= 0) - return op.emitError("bank depth cannot hold one K tile"); - // outputBase is a six-bit row index; one 16x16 result uses 64 rows. - constexpr int64_t tilesPerResultBank = 1; - - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value tileSize = b.create(loc, kTile); - Value columns = b.create(loc, N); - Value zeroI8 = - b.create(loc, b.getI8Type(), b.getI8IntegerAttr(0)); - auto matrixRowLoop = b.create( - loc, zero, b.create(loc, paddedM), - b.create(loc, tileRows)); - b.setInsertionPointToStart(matrixRowLoop.getBody()); - Value m0 = matrixRowLoop.getInductionVar(); - Value validRows = b.create( - loc, b.create(loc, tileRows), - b.create(loc, b.create(loc, M), - m0)); - SmallVector inputPacks; - { - for (int64_t k0 = 0; k0 < paddedK; k0 += kChunk) { - int64_t thisK = std::min(kChunk, paddedK - k0); - int64_t validK = std::min(thisK, K - k0); - auto aPackTy = - MemRefType::get({thisK / kTile * tileRows, kTile}, b.getI8Type()); - Value aPack = b.create(loc, aPackTy); - inputPacks.push_back(aPack); - b.create(loc, zeroI8, aPack); - - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value rowEnd = validRows; - Value kTileEnd = - b.create(loc, (validK + kTile - 1) / kTile); - auto ktLoop = b.create(loc, zero, kTileEnd, one); - b.setInsertionPointToStart(ktLoop.getBody()); - Value kt = ktLoop.getInductionVar(); - auto rowLoop = b.create(loc, zero, rowEnd, one); - b.setInsertionPointToStart(rowLoop.getBody()); - Value row = rowLoop.getInductionVar(); - Value remaining = b.create( - loc, b.create(loc, validK), - b.create( - loc, kt, b.create(loc, kTile))); - Value columnEnd = b.create( - loc, remaining, b.create(loc, kTile)); - auto columnLoop = b.create(loc, zero, columnEnd, one); - b.setInsertionPointToStart(columnLoop.getBody()); - Value column = columnLoop.getInductionVar(); - Value sourceRow = b.create(loc, m0, row); - Value sourceColumn = b.create( - loc, b.create(loc, k0), - b.create( - loc, - b.create( - loc, kt, b.create(loc, kTile)), - column)); - Value hardwareRowsValue = b.create( - loc, std::min(hardwareRows, tileRows)); - Value rowTile = b.create(loc, row, hardwareRowsValue); - Value rowInTile = b.create(loc, row, hardwareRowsValue); - Value tileIndex = b.create( - loc, - b.create( - loc, rowTile, - b.create(loc, thisK / kTile)), - kt); - Value packedRow = b.create( - loc, b.create(loc, tileIndex, hardwareRowsValue), - rowInTile); - Value value = b.create( - loc, op.getInput(), ValueRange{sourceRow, sourceColumn}); - b.create(loc, value, aPack, - ValueRange{packedRow, column}); - b.setInsertionPointAfter(ktLoop); - } - } - auto panelLoop = b.create(loc, zero, columns, tileSize); - b.setInsertionPointToStart(panelLoop.getBody()); - { - Value n0 = panelLoop.getInductionVar(); - Value thisN = b.create( - loc, tileSize, b.create(loc, columns, n0)); - Value aBank = allocBank(b, loc, 1, 1); - Value wBank = allocBank(b, loc, 1, 1); - auto biasPackTy = MemRefType::get({4, 4}, b.getI32Type()); - auto scalePackTy = MemRefType::get({4, 4}, b.getF32Type()); - Value biasPack = b.create(loc, biasPackTy); - Value scalePack = b.create(loc, scalePackTy); - Value zeroI32 = b.create(loc, b.getI32Type(), - b.getI32IntegerAttr(0)); - Value oneF32 = b.create(loc, b.getF32Type(), - b.getF32FloatAttr(1.0)); - b.create(loc, zeroI32, biasPack); - b.create(loc, oneF32, scalePack); - auto channelLoop = b.create(loc, zero, thisN, one); - b.setInsertionPointToStart(channelLoop.getBody()); - { - Value i = channelLoop.getInductionVar(); - Value four = b.create(loc, 4); - Value source = b.create(loc, n0, i); - Value row = b.create(loc, i, four); - Value column = b.create(loc, i, four); - Value bias = b.create(loc, op.getBias(), source); - Value scale = b.create(loc, op.getScale(), source); - b.create(loc, bias, biasPack, ValueRange{row, column}); - b.create(loc, scale, scalePack, - ValueRange{row, column}); - } - b.setInsertionPointAfter(channelLoop); - - Value biasBank = allocBank(b, loc, 1, 1); - Value biasLoaded = mvinBank(b, loc, biasPack, biasBank, 4); - Value biasState = b.create( - loc, biasLoaded.getType(), biasLoaded, createI64Const(b, loc, 0)); - Value scaleBank = allocBank(b, loc, 1, 1); - Value scaleLoaded = mvinBank(b, loc, scalePack, scaleBank, 4); - - { - int64_t tileCount = tilesPerResultBank; - Value resultBank = allocBank(b, loc, 1, 1); - Value resultState = resultBank; - SmallVector hostPacks; - - for (int64_t tile = 0; tile < tileCount; ++tile) { - for (int64_t k0 = 0; k0 < paddedK; k0 += kChunk) { - int64_t thisK = std::min(kChunk, paddedK - k0); - int64_t validK = std::min(thisK, std::max(0, K - k0)); - Value aPack = inputPacks[k0 / kChunk]; - Value aLoaded = - mvinBank(b, loc, aPack, aBank, thisK / kTile * tileRows); - Value wLoaded; - auto [weightStrides, weightOffset] = wTy.getStridesAndOffset(); - if (validK == thisK && N % kTile == 0 && weightStrides[1] == 1 && - weightStrides[0] > 0 && weightStrides[0] % kTile == 0) { - Value panel = b.create( - loc, op.getWeight(), - SmallVector{b.getIndexAttr(k0), n0}, - SmallVector{b.getIndexAttr(thisK), - b.getIndexAttr(kTile)}, - SmallVector{b.getIndexAttr(1), - b.getIndexAttr(1)}); - wLoaded = mvinBank(b, loc, panel, wBank, thisK, - weightStrides[0] / kTile); - } else { - auto wPackTy = MemRefType::get({thisK, kTile}, b.getI8Type()); - Value wPack = b.create(loc, wPackTy); - hostPacks.push_back(wPack); - b.create(loc, zeroI8, wPack); - Value validKEnd = b.create(loc, validK); - Value validNEnd = thisN; - auto weightRowLoop = - b.create(loc, zero, validKEnd, one); - b.setInsertionPointToStart(weightRowLoop.getBody()); - Value weightRow = weightRowLoop.getInductionVar(); - auto weightColumnLoop = - b.create(loc, zero, validNEnd, one); - b.setInsertionPointToStart(weightColumnLoop.getBody()); - Value weightColumn = weightColumnLoop.getInductionVar(); - Value sourceWeightRow = b.create( - loc, b.create(loc, k0), weightRow); - Value sourceWeightColumn = - b.create(loc, n0, weightColumn); - Value weightValue = b.create( - loc, op.getWeight(), - ValueRange{sourceWeightRow, sourceWeightColumn}); - b.create(loc, weightValue, wPack, - ValueRange{weightRow, weightColumn}); - b.setInsertionPointAfter(weightRowLoop); - - wLoaded = mvinBank(b, loc, wPack, wBank, thisK); - } - uint64_t cfg = matrixRs2(tileRows, kTile, thisK); - auto smatmul = b.create( - loc, resultState.getType(), aLoaded, wLoaded, resultState, - createI64Const(b, loc, static_cast(cfg)), - createI1Const(b, loc, k0 == 0), - createI1Const(b, loc, k0 + thisK == paddedK), - createI64Const(b, loc, tile * resultRows)); - resultState = smatmul.getWrBankOut(); - aBank = aLoaded; - wBank = wLoaded; - } - } - - Value outputBank = allocBank(b, loc, 1, 1); - Value converted = - finalOutput - ? b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, - createI64Const(b, loc, tileCount * resultRows), - b.getBoolAttr(op.getActivation() == 1)) - .getOutBankOut() - : b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, - createI64Const(b, loc, tileCount * resultRows), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - b.getI64IntegerAttr(tileCount * tileRows), - b.getI64IntegerAttr(1), - b.getI64IntegerAttr(tileCount * tileRows), - b.getBoolAttr(op.getActivation() == 1)) - .getOutBankOut(); - releaseBank(b, loc, resultState); - - Type outputElementType = - finalOutput ? Type(b.getF32Type()) : Type(b.getI8Type()); - auto packedOutTy = - MemRefType::get({tileCount * tileRows, kTile}, outputElementType); - Value packedOut = b.create(loc, packedOutTy); - Value stored = mvoutBank(b, loc, packedOut, converted, - finalOutput ? tileCount * resultRows - : tileCount * tileRows); - for (Value pack : hostPacks) - b.create(loc, pack); - - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value rowEnd = validRows; - Value columnEnd = thisN; - auto rowLoop = b.create(loc, zero, rowEnd, one); - b.setInsertionPointToStart(rowLoop.getBody()); - Value row = rowLoop.getInductionVar(); - auto columnLoop = b.create(loc, zero, columnEnd, one); - b.setInsertionPointToStart(columnLoop.getBody()); - Value column = columnLoop.getInductionVar(); - Value value = - b.create(loc, packedOut, ValueRange{row, column}); - Value outputRow = b.create(loc, m0, row); - Value outputColumn = b.create(loc, n0, column); - b.create(loc, value, op.getOutput(), - ValueRange{outputRow, outputColumn}); - b.setInsertionPointAfter(rowLoop); - releaseBank(b, loc, stored); - b.create(loc, packedOut); - } - - releaseBank(b, loc, biasState); - releaseBank(b, loc, scaleLoaded); - b.create(loc, biasPack); - b.create(loc, scalePack); - releaseBank(b, loc, aBank); - releaseBank(b, loc, wBank); - } - b.setInsertionPointAfter(panelLoop); - for (Value pack : inputPacks) - b.create(loc, pack); - b.setInsertionPointAfter(matrixRowLoop); - b.eraseOp(op); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateSMatMulBallLowerBuckyballToBankSSAPatterns( - RewritePatternSet &patterns) { - populateMatmulRegionToBankSSAPatterns(patterns, false, 0, -1); -} - -LogicalResult mlir::buddy::lowerMatmulToBanks(MegaMatmulOp op, - PatternRewriter &rewriter) { - return MegaMatmulToBankSSAPattern(op.getContext()) - .matchAndRewrite(op, rewriter); -} diff --git a/examples/balls/smatmul/compiler/src/Conversion/LowerTileToBuckyball/TileMatMulPatterns.cpp b/examples/balls/smatmul/compiler/src/Conversion/LowerTileToBuckyball/TileMatMulPatterns.cpp deleted file mode 100644 index abb98389d..000000000 --- a/examples/balls/smatmul/compiler/src/Conversion/LowerTileToBuckyball/TileMatMulPatterns.cpp +++ /dev/null @@ -1,324 +0,0 @@ -//===- TileMatMulPatterns.cpp - tile.matmul -> buckyball.smatmul_matmul -//----===// -// -// Non-quantized matmul is split into bank-sized panels. Quantized matmul -// retains its full N dimension so bank lowering can quantize one activation -// chunk and hand it to every output panel. -// -//===----------------------------------------------------------------------===// - -#include "Conversion/LowerTileToBuckyball/LowerTileToBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Tile/TileOps.h" - -#include - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::tile; -using mlir::buddy::aMvinDepthLines; -using mlir::buddy::bMvinDepthLines; -using mlir::buddy::ceilDiv; -using mlir::buddy::cMvoutDepthLines; -using mlir::buddy::kBankLane; - -namespace { - -class TileMatMulLowering : public OpRewritePattern { -public: - explicit TileMatMulLowering(MLIRContext *context, int64_t /*bankWidthBytes*/, - int64_t bankDepth, int64_t /*bankNum*/) - : OpRewritePattern(context), bankDepth(bankDepth) {} - - LogicalResult matchAndRewrite(TileMatMulOp tileMatMulOp, - PatternRewriter &rewriter) const override { - Location loc = tileMatMulOp.getLoc(); - - Value aMemArray = tileMatMulOp.getAMemArray(); - Value bMemArray = tileMatMulOp.getBMemArray(); - Value cMemArray = tileMatMulOp.getCMemArray(); - - auto aType = cast(aMemArray.getType()); - auto bType = cast(bMemArray.getType()); - auto cType = cast(cMemArray.getType()); - - auto aShape = aType.getShape(); - auto bShape = bType.getShape(); - auto cShape = cType.getShape(); - size_t M = aShape[aShape.size() - 2]; - size_t K = aShape[aShape.size() - 1]; - size_t N = bShape[bShape.size() - 1]; - - if (bShape[bShape.size() - 2] != (int64_t)K || - cShape[cShape.size() - 2] != (int64_t)M || - cShape[cShape.size() - 1] != (int64_t)N) - return tileMatMulOp.emitError("matmul input/output shapes mismatch"); - - const bool isQuantized = aType.getElementType().isF32() && - bType.getElementType().isInteger(8) && - cType.getElementType().isF32(); - const bool isWideFloat = - (aType.getElementType().isF32() || aType.getElementType().isBF16()) && - bType.getElementType() == aType.getElementType() && - cType.getElementType() == aType.getElementType(); - const bool isInt8 = aType.getElementType().isInteger(8) && - bType.getElementType().isInteger(8) && - cType.getElementType().isInteger(32); - if (!isQuantized && !isWideFloat && !isInt8) - return tileMatMulOp.emitError( - "requires matching floating-point types, FP32 x INT8 -> FP32, " - "or INT8 x INT8 -> INT32"); - auto dwAddrAttr = tileMatMulOp->getAttrOfType("dw_addr"); - auto dwBytesAttr = tileMatMulOp->getAttrOfType("dw_bytes"); - auto perChannelAttr = tileMatMulOp->getAttrOfType("per_channel"); - if (isQuantized && (!dwAddrAttr || !dwBytesAttr || !perChannelAttr || - dwAddrAttr.getInt() < 16)) - return tileMatMulOp.emitError( - "quantized matmul requires RAX Dw metadata"); - - constexpr size_t mTileSize = 8; - size_t M_pad = ceilDiv(M, mTileSize) * mTileSize; - size_t K_pad = ceilDiv(K, kBankLane) * kBankLane; - size_t N_pad = ceilDiv(N, kBankLane) * kBankLane; - bool needPadding = (M_pad != M) || (K_pad != K) || (N_pad != N); - - Value aMemArrayPadded = aMemArray; - Value bMemArrayPadded = bMemArray; - Value cMemArrayPadded = cMemArray; - - if (needPadding) { - auto aPadType = MemRefType::get({(int64_t)M_pad, (int64_t)K_pad}, - aType.getElementType()); - auto bPadType = MemRefType::get({(int64_t)K_pad, (int64_t)N_pad}, - bType.getElementType()); - auto cPadType = MemRefType::get({(int64_t)M_pad, (int64_t)N_pad}, - cType.getElementType()); - - aMemArrayPadded = rewriter.create(loc, aPadType); - bMemArrayPadded = rewriter.create(loc, bPadType); - cMemArrayPadded = rewriter.create(loc, cPadType); - - Value aZero = rewriter.create( - loc, aType.getElementType(), - rewriter.getZeroAttr(aType.getElementType())); - Value bZero = rewriter.create( - loc, bType.getElementType(), - rewriter.getZeroAttr(bType.getElementType())); - Value cZero = rewriter.create( - loc, cType.getElementType(), - rewriter.getZeroAttr(cType.getElementType())); - rewriter.create(loc, aZero, aMemArrayPadded); - rewriter.create(loc, bZero, bMemArrayPadded); - rewriter.create(loc, cZero, cMemArrayPadded); - - Value aView = rewriter.create( - loc, aMemArrayPadded, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(M), - rewriter.getIndexAttr(K)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - rewriter.create(loc, aMemArray, aView); - - Value bView = rewriter.create( - loc, bMemArrayPadded, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(K), - rewriter.getIndexAttr(N)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - rewriter.create(loc, bMemArray, bView); - } - - size_t M_tiling = needPadding ? M_pad : M; - size_t K_tiling = needPadding ? K_pad : K; - size_t N_tiling = needPadding ? N_pad : N; - - if (bankDepth <= 0 || (size_t)bankDepth % kBankLane != 0) - return tileMatMulOp.emitError( - "bankDepth must be a positive multiple of bank lane"); - if (M_tiling % mTileSize || K_tiling % kBankLane || N_tiling % kBankLane) - return tileMatMulOp.emitError( - "M must be a multiple of 8 and K/N must be multiples of bank lane"); - if (isQuantized) { - const int64_t dwRequired = perChannelAttr.getValue() ? N_tiling * 4 : 4; - if (dwBytesAttr.getInt() < dwRequired) - return tileMatMulOp.emitError("Dw scale image does not cover padded N"); - } - - // Bank unit on M (depth) and N (lane). Quantized matmul carries one - // logical K into SMatMul so FP2INT computes one activation scale. - const size_t nTileSize = kBankLane; - const size_t kTileSize = - (isWideFloat || isQuantized) ? K_tiling : kBankLane; - if (M_tiling % mTileSize != 0) - return tileMatMulOp.emitError("M does not split into bank-depth units"); - - if (!isWideFloat && !isQuantized) { - if (aMvinDepthLines(mTileSize, kTileSize) > (size_t)bankDepth || - bMvinDepthLines(kTileSize, nTileSize) > (size_t)bankDepth || - cMvoutDepthLines(mTileSize, nTileSize) > (size_t)bankDepth) - return tileMatMulOp.emitError( - "bank-unit matmul exceeds 8-bit mvin or 32-bit mvout depth"); - } - - const size_t kTileNum = ceilDiv(K_tiling, kTileSize); - - OpBuilder::InsertionGuard guard(rewriter); - Value zeroIdx = rewriter.create(loc, 0); - Value mStepVal = rewriter.create(loc, mTileSize); - Value kStepVal = rewriter.create(loc, kTileSize); - Value mUpperVal = rewriter.create(loc, M_tiling); - Value kUpperVal = rewriter.create(loc, K_tiling); - - auto mLoop = rewriter.create(loc, zeroIdx, mUpperVal, mStepVal); - rewriter.setInsertionPointToStart(mLoop.getBody()); - Value mIv = mLoop.getInductionVar(); - - if (isQuantized) { - Value aTile = rewriter.create( - loc, aMemArrayPadded, - SmallVector{mIv, rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(mTileSize), - rewriter.getIndexAttr(K_tiling)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - Value bTile = rewriter.create( - loc, bMemArrayPadded, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(K_tiling), - rewriter.getIndexAttr(N_tiling)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - Value cTile = rewriter.create( - loc, cMemArrayPadded, - SmallVector{mIv, rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(mTileSize), - rewriter.getIndexAttr(N_tiling)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - Value zero = rewriter.create( - loc, cType.getElementType(), - rewriter.getZeroAttr(cType.getElementType())); - rewriter.create(loc, zero, cTile); - auto smatmul = rewriter.create(loc, aTile, bTile, cTile); - smatmul->setAttr("dwAddr", dwAddrAttr); - smatmul->setAttr("dwBytes", dwBytesAttr); - smatmul->setAttr("perChannel", perChannelAttr); - } else { - // N is static. Non-quantized matmul remains one physical N panel per op. - for (size_t n0 = 0; n0 < N_tiling; n0 += nTileSize) { - Value cTile = rewriter.create( - loc, cMemArrayPadded, - SmallVector{mIv, rewriter.getIndexAttr(n0)}, - SmallVector{rewriter.getIndexAttr(mTileSize), - rewriter.getIndexAttr(nTileSize)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - - auto cElemType = cType.getElementType(); - Value zero = rewriter.create( - loc, cElemType, rewriter.getZeroAttr(cElemType)); - rewriter.create(loc, zero, cTile); - - if (kTileNum == 1) { - Value aTile = rewriter.create( - loc, aMemArrayPadded, - SmallVector{mIv, rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(mTileSize), - rewriter.getIndexAttr(kTileSize)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - Value bTile = rewriter.create( - loc, bMemArrayPadded, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(n0)}, - SmallVector{rewriter.getIndexAttr(kTileSize), - rewriter.getIndexAttr(nTileSize)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - auto smatmul = - rewriter.create(loc, aTile, bTile, cTile); - } else { - auto kLoop = - rewriter.create(loc, zeroIdx, kUpperVal, kStepVal); - rewriter.setInsertionPointToStart(kLoop.getBody()); - Value kIv = kLoop.getInductionVar(); - Value aTile = rewriter.create( - loc, aMemArrayPadded, SmallVector{mIv, kIv}, - SmallVector{rewriter.getIndexAttr(mTileSize), - rewriter.getIndexAttr(kTileSize)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - Value bTile = rewriter.create( - loc, bMemArrayPadded, - SmallVector{kIv, rewriter.getIndexAttr(n0)}, - SmallVector{rewriter.getIndexAttr(kTileSize), - rewriter.getIndexAttr(nTileSize)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - if (isWideFloat) - return tileMatMulOp.emitError( - "FP32 matmul must pass full K in one smatmul_matmul"); - // 8-bit legalize overwrites C; accumulate via host partial. - auto partialTy = MemRefType::get( - {(int64_t)mTileSize, (int64_t)nTileSize}, cElemType); - Value partial = rewriter.create(loc, partialTy); - rewriter.create(loc, zero, partial); - rewriter.create(loc, aTile, bTile, partial); - rewriter.create( - loc, ValueRange{cTile, partial}, ValueRange{cTile}, - linalg::ElementwiseKind::add); - rewriter.create(loc, partial); - rewriter.setInsertionPointAfter(kLoop); - } - } - } - - rewriter.setInsertionPointAfter(mLoop); - - if (needPadding) { - Value cView = rewriter.create( - loc, cMemArrayPadded, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(M), - rewriter.getIndexAttr(N)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - rewriter.create(loc, cView, cMemArray); - rewriter.create(loc, aMemArrayPadded); - rewriter.create(loc, bMemArrayPadded); - rewriter.create(loc, cMemArrayPadded); - } - - rewriter.eraseOp(tileMatMulOp); - return success(); - } - -private: - int64_t bankDepth; -}; - -} // namespace - -namespace mlir::buddy { -void populateSMatMulBallTileLoweringPatterns(RewritePatternSet &patterns, - int64_t bankWidthBytes, - int64_t bankDepth, - int64_t bankNum) { - patterns.add(patterns.getContext(), bankWidthBytes, - bankDepth, bankNum); -} -} // namespace mlir::buddy diff --git a/examples/balls/smatmul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/smatmul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index a7df304f8..000000000 --- a/examples/balls/smatmul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,263 +0,0 @@ -//===- LegalizeForLLVMExport.cpp - SMatMulBall LLVM lowering --------------===// - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include "llvm/Support/ErrorHandling.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { - -uint64_t matrixCfg(uint64_t rows, uint64_t cols, bool first = true, - bool last = true) { - if (rows == 0 || rows > 0xfff || (rows != 1 && rows % 8) || cols != 16) - llvm::report_fatal_error( - "matrix cfg: rows must be 1 or a multiple of 8 and cols must be 16"); - return fieldBits(rows, 0, 11) | fieldBits(cols, 12, 23) | - (uint64_t(first) << 24) | (uint64_t(last) << 25); -} - -struct SMatMulMatmulLowering : public ConvertOpToLLVMPattern { - SMatMulMatmulLowering(LLVMTypeConverter &converter, bool /*stable*/) - : ConvertOpToLLVMPattern(converter) {} - - LogicalResult - matchAndRewrite(SMatMulMatmulOp op, OpAdaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("SMatMulBall"); - Location loc = op.getLoc(); - Value aMem = op.getAMemArray(); - Value bMem = op.getBMemArray(); - Value cMem = op.getCMemArray(); - - auto aTy = dyn_cast(aMem.getType()); - auto bTy = dyn_cast(bMem.getType()); - auto cTy = dyn_cast(cMem.getType()); - if (!aTy || !bTy || !cTy || !aTy.hasStaticShape() || - !bTy.hasStaticShape() || !cTy.hasStaticShape()) - return rewriter.notifyMatchFailure( - op, "buckyball.smatmul_matmul requires static memref shapes"); - - uint64_t m = aTy.getShape()[0]; - uint64_t k = aTy.getShape()[1]; - uint64_t kb = bTy.getShape()[0]; - uint64_t n = bTy.getShape()[1]; - if (k != kb || cTy.getShape()[0] != (int64_t)m || - cTy.getShape()[1] != (int64_t)n) - return rewriter.notifyMatchFailure(op, "matmul shapes mismatch"); - const int64_t bankDepth = buckyball_target::getBuckyballTarget().bankDepth; - if (bankDepth <= 0) - return op.emitError("smatmul lowering requires target bankDepth > 0"); - // A occupies m lines, B occupies k lines, and packed int32 C occupies 2m. - if (m == 0 || n != 16 || k != 16 || m % 8 != 0 || - m > static_cast(bankDepth / 2) || - k > static_cast(bankDepth)) - return rewriter.notifyMatchFailure( - op, "SMatMul requires K/N exactly 16 and A/B/C to fit their banks"); - if (!aTy.getElementType().isInteger(8) || - !bTy.getElementType().isInteger(8) || - !cTy.getElementType().isInteger(32)) - return rewriter.notifyMatchFailure( - op, "SMatMul Ball supports only 8-bit A/B with 32-bit C"); - - const uint64_t aBank = 0; - const uint64_t bBank = 1; - const uint64_t cBank = 2; - uint64_t depthA = m; // K <= 16 => one bank row per M row - uint64_t depthB = k; - uint64_t depthC = 2 * m; - - emitMset(rewriter, loc, aBank, 1, 1, 1); - emitMset(rewriter, loc, bBank, 1, 1, 1); - emitMset(rewriter, loc, cBank, 1, 2, 1); - - Value aPtr = extractPtr(rewriter, loc, aMem); - Value bPtr = extractPtr(rewriter, loc, bMem); - auto packedType = MemRefType::get({static_cast(depthC), 8}, - cTy.getElementType()); - Value packed = rewriter.create(loc, packedType); - Value packedPtr = extractPtr(rewriter, loc, packed); - - Value rs1A = packRs1BankIter(rewriter, loc, cstI64(rewriter, loc, aBank), - cstI64(rewriter, loc, depthA)); - Value rs2A = - packRs2MemStride(rewriter, loc, aPtr, cstI64(rewriter, loc, 1)); - - rewriter.create(loc, rs1A, rs2A); - - Value rs1B = packRs1BankIter(rewriter, loc, cstI64(rewriter, loc, bBank), - cstI64(rewriter, loc, depthB)); - Value rs2B = - packRs2MemStride(rewriter, loc, bPtr, cstI64(rewriter, loc, 1)); - - rewriter.create(loc, rs1B, rs2B); - - rewriter.create( - loc, - packRs1BanksIter(rewriter, loc, cstI64(rewriter, loc, aBank), - cstI64(rewriter, loc, bBank), - cstI64(rewriter, loc, cBank), - cstI64(rewriter, loc, k)), - cstI64(rewriter, loc, matrixCfg(m, n)), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("SMATMUL_OS"))); - - Value rs1C = packRs1BankIter(rewriter, loc, cstI64(rewriter, loc, cBank), - cstI64(rewriter, loc, depthC)); - Value rs2C = - packRs2MemStride(rewriter, loc, packedPtr, cstI64(rewriter, loc, 1)); - - rewriter.create(loc, rs1C, rs2C); - - Value indexZero = rewriter.create(loc, 0); - Value indexOne = rewriter.create(loc, 1); - Value indexTwo = rewriter.create(loc, 2); - Value indexEight = rewriter.create(loc, 8); - Value indexM = rewriter.create(loc, m); - Value indexN = rewriter.create(loc, n); - auto rowLoop = - rewriter.create(loc, indexZero, indexM, indexOne); - rewriter.setInsertionPointToStart(rowLoop.getBody()); - Value row = rowLoop.getInductionVar(); - auto columnLoop = - rewriter.create(loc, indexZero, indexN, indexOne); - rewriter.setInsertionPointToStart(columnLoop.getBody()); - Value column = columnLoop.getInductionVar(); - Value packedRow = rewriter.create( - loc, rewriter.create(loc, row, indexTwo), - rewriter.create(loc, column, indexEight)); - Value packedColumn = - rewriter.create(loc, column, indexEight); - Value value = rewriter.create( - loc, packed, ValueRange{packedRow, packedColumn}); - rewriter.create(loc, value, cMem, ValueRange{row, column}); - rewriter.setInsertionPointAfter(rowLoop); - rewriter.create(loc, packed); - - emitMset(rewriter, loc, aBank, 0, 0, 0); - emitMset(rewriter, loc, bBank, 0, 0, 0); - emitMset(rewriter, loc, cBank, 0, 0, 0); - - rewriter.eraseOp(op); - return success(); - } -}; - -struct SMatMulLowering : public ConvertOpToLLVMPattern { - SMatMulLowering(LLVMTypeConverter &converter, bool, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(SMatMulOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("SMatMulBall"); - Location loc = op.getLoc(); - auto config = op.getConfig().getDefiningOp(); - if (!config) - return op.emitError("smatmul config must be a constant"); - auto configAttr = dyn_cast(config.getValue()); - if (!configAttr) - return op.emitError("smatmul config must be an integer constant"); - uint64_t cfg = configAttr.getValue().getZExtValue(); - uint64_t rows = cfg & 0xfff; - uint64_t cols = (cfg >> 12) & 0xfff; - uint64_t k = (cfg >> 24) & 0xfff; - if (rows == 0 || (rows != 1 && rows % 8) || cols != 16 || k == 0 || - k % 16 || rows * 4 > static_cast(bankDepth)) - return op.emitError("SMatMul matrix shape or C footprint is invalid"); - auto base = adaptor.getOutputBase().getDefiningOp(); - auto baseAttr = - base ? dyn_cast(base.getValue()) : IntegerAttr(); - if (!baseAttr || baseAttr.getInt() < 0 || baseAttr.getInt() > 63 || - baseAttr.getInt() + rows * 4 > bankDepth) - return op.emitError( - "SMatMul outputBase must be constant and fit C in bank"); - Value rs1 = packRs1BanksIter( - rewriter, loc, adaptor.getOp1BankId(), adaptor.getOp2BankId(), - adaptor.getResultBankId(), cstI64(rewriter, loc, k)); - Value rs2 = cstI64(rewriter, loc, matrixCfg(rows, cols, false, false)); - Value first = rewriter.create(loc, rewriter.getI64Type(), - adaptor.getFirst()); - Value last = rewriter.create(loc, rewriter.getI64Type(), - adaptor.getLast()); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, first, cstI64(rewriter, loc, 24))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, last, cstI64(rewriter, loc, 25))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getOutputBase(), - cstI64(rewriter, loc, 26))); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("SMATMUL_OS"))); - return success(); - } - -private: - int64_t bankDepth; -}; - -struct SMatMulBiasLowering : public ConvertOpToLLVMPattern { - SMatMulBiasLowering(LLVMTypeConverter &converter, bool) - : ConvertOpToLLVMPattern(converter) {} - - LogicalResult - matchAndRewrite(SMatMulBiasOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("SMatMulBall"); - Location loc = op.getLoc(); - auto base = adaptor.getInputBase().getDefiningOp(); - auto baseAttr = - base ? dyn_cast(base.getValue()) : IntegerAttr(); - if (!baseAttr || baseAttr.getInt() < 0 || - baseAttr.getInt() + 4 > - static_cast( - buckyball_target::getBuckyballTarget().bankDepth)) - return op.emitError( - "SMatMul bias inputBase must be constant and fit in bank"); - Value rs1 = packRs1BankIter(rewriter, loc, adaptor.getBiasBankId(), - cstI64(rewriter, loc, 4)); - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getInputBase(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("SMATMUL_BIAS"))); - return success(); - } -}; - -} // namespace - -namespace mlir::buddy::buckyball { -void populateSMatMulBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - patterns.add(converter, stable); - patterns.add(converter, stable, bankDepth); - patterns.add(converter, stable); -} - -void configureSMatMulBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool /*stable*/) { - target.addIllegalOp(); - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/smatmul/configs/SMatMulBallParam.scala b/examples/balls/smatmul/configs/SMatMulBallParam.scala deleted file mode 100644 index c103f17ae..000000000 --- a/examples/balls/smatmul/configs/SMatMulBallParam.scala +++ /dev/null @@ -1,21 +0,0 @@ -package examples.balls.smatmul.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig - -case class SMatMulBallParam( - tileRows: Int, - tileCols: Int) - -object SMatMulBallParam { - private val ballName = "SMatMulBall" - - def apply(b: GlobalConfig): SMatMulBallParam = { - val tbl = BallParamLoader.ballTable(b, ballName) - SMatMulBallParam( - tileRows = BallParamLoader.int(tbl, "tileRows"), - tileCols = BallParamLoader.int(tbl, "tileCols") - ) - } - -} diff --git a/examples/balls/smatmul/configs/default.toml b/examples/balls/smatmul/configs/default.toml deleted file mode 100644 index 8aa081cae..000000000 --- a/examples/balls/smatmul/configs/default.toml +++ /dev/null @@ -1,5 +0,0 @@ -# SMatMul Ball Configuration - -[ball] -tileRows = 16 -tileCols = 16 diff --git a/examples/balls/smatmul/configs/lane16.toml b/examples/balls/smatmul/configs/lane16.toml deleted file mode 100644 index 8e0df9360..000000000 --- a/examples/balls/smatmul/configs/lane16.toml +++ /dev/null @@ -1,5 +0,0 @@ -# SMatMul Ball Configuration - -[ball] -tileRows = 8 -tileCols = 16 diff --git a/examples/balls/smatmul/emu/src/dispatch.rs b/examples/balls/smatmul/emu/src/dispatch.rs deleted file mode 100644 index 6c85e5dec..000000000 --- a/examples/balls/smatmul/emu/src/dispatch.rs +++ /dev/null @@ -1,38 +0,0 @@ -use crate::inst::decode; -use crate::inst::instruction::ExecContext; - -use super::{fp32, mxfp8, smatmul}; - -const BALL_CLASS: &str = "examples.balls.smatmul.SMatMulBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("SMATMUL_OS") => Some(smatmul::exec_smatmul(xs1, xs2, ctx)), - Some("SMATMUL_MXFP8") => Some(mxfp8::execute(xs1, xs2, ctx)), - Some("SMATMUL_F32") => Some(fp32::execute(xs1, xs2, ctx)), - Some("SMATMUL_BIAS") => Some(smatmul::exec_bias(xs1, xs2, ctx)), - _ => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("SMATMUL_OS") => Some(smatmul::latency(xs1, xs2)), - Some("SMATMUL_MXFP8") => Some(smatmul::latency(xs1, xs2)), - Some("SMATMUL_F32") => Some(decode::rs1_iter(xs1) as u64 + (xs2 & 0xfff) + ((xs2 >> 12) & 0xfff)), - Some("SMATMUL_BIAS") => Some(smatmul::bias_latency(xs1, xs2)), - _ => None, - } -} diff --git a/examples/balls/smatmul/emu/src/fp32.rs b/examples/balls/smatmul/emu/src/fp32.rs deleted file mode 100644 index 02ba5e3aa..000000000 --- a/examples/balls/smatmul/emu/src/fp32.rs +++ /dev/null @@ -1,119 +0,0 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::ExecContext; -use std::{cell::RefCell, collections::HashMap}; - -struct Chain { - rows: usize, - bank: u64, - base: usize, - values: Vec, -} - -thread_local! { - static CHAINS: RefCell> = RefCell::new(HashMap::new()); -} - -pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let rows = (xs2 & 0xfff) as usize; - let cols = ((xs2 >> 12) & 0xfff) as usize; - let k = rs1_iter(xs1) as usize; - let first = (xs2 >> 24) & 1 != 0; - let last = (xs2 >> 25) & 1 != 0; - let base = ((xs2 >> 26) & 63) as usize; - let a = rs1_b0(xs1); - let b = rs1_b1(xs1); - let c = rs1_b2(xs1); - assert!( - xs2 >> 32 == 0 && (rows == 1 || rows == 16) && cols == 16 && k > 0 && k % 4 == 0, - "f32 matmul: requires M=1/16, N=16, positive K multiple of four and zero reserved bits" - ); - assert!(a != b && b != c && a != c, "f32 matmul: banks must differ"); - for bank in [a, b, c] { - assert!( - ctx.config(bank).allocated && ctx.config(bank).cols == 1, - "f32 matmul: banks must be allocated with one column" - ); - } - let stride = bank_row_bytes(); - assert!( - stride == 16 - && rows * k * 4 <= bank_lines() * stride - && cols * k * 4 <= bank_lines() * stride - && base * stride + rows * cols * 4 <= bank_lines() * stride, - "f32 matmul: bank footprint exceeds depth" - ); - let pa = pbank(ctx, a); - let pb = pbank(ctx, b); - let av = &ctx.banks[pa][..rows * k * 4]; - let bv = &ctx.banks[pb][..cols * k * 4]; - assert!( - av.chunks_exact(4) - .chain(bv.chunks_exact(4)) - .all(|x| f32::from_le_bytes(x.try_into().unwrap()).is_finite()), - "f32 matmul: non-finite operand" - ); - let mut chain = CHAINS.with(|chains| { - let mut chains = chains.borrow_mut(); - if first { - assert!( - !chains.contains_key(&ctx.hart_id), - "f32 matmul: first while chain is live" - ); - Chain { - rows, - bank: c, - base, - values: vec![0f32; rows * cols], - } - } else { - chains - .remove(&ctx.hart_id) - .expect("f32 matmul: continuation without first") - } - }); - assert!( - chain.rows == rows && chain.bank == c && chain.base == base, - "f32 matmul: continuation changed destination" - ); - let weights: [&[u8]; 16] = std::array::from_fn(|col| &bv[col * k * 4..(col + 1) * k * 4]); - for row in 0..rows { - if av[row * k * 4..(row + 1) * k * 4] - .chunks_exact(4) - .all(|x| u32::from_le_bytes(x.try_into().unwrap()) & 0x7fffffff == 0) - && chain.values[row * cols..(row + 1) * cols] - .iter() - .all(|x| x.to_bits() == 0) - { - continue; - } - let av = &av[row * k * 4..(row + 1) * k * 4]; - let dst = &mut chain.values[row * cols..(row + 1) * cols]; - let mut sums: [f32; 16] = dst.try_into().unwrap(); - for inner in 0..k { - let a = f32::from_le_bytes(av[inner * 4..inner * 4 + 4].try_into().unwrap()); - for col in 0..16 { - let b = - f32::from_le_bytes(weights[col][inner * 4..inner * 4 + 4].try_into().unwrap()); - sums[col] = a.mul_add(b, sums[col]); - } - } - assert!( - sums.iter().all(|x| x.is_finite()), - "f32 matmul: accumulator overflow" - ); - dst.copy_from_slice(&sums); - } - if last { - let pc = pbank(ctx, c); - for (index, value) in chain.values.iter().enumerate() { - let offset = base * stride + index * 4; - ctx.banks[pc][offset..offset + 4].copy_from_slice(&value.to_le_bytes()); - } - } else { - CHAINS.with(|chains| { - chains.borrow_mut().insert(ctx.hart_id, chain); - }); - } - 0 -} diff --git a/examples/balls/smatmul/emu/src/lib.rs b/examples/balls/smatmul/emu/src/lib.rs deleted file mode 100644 index 864e215a8..000000000 --- a/examples/balls/smatmul/emu/src/lib.rs +++ /dev/null @@ -1,9 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod fp32; -mod mxfp8; -mod smatmul; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/smatmul/emu/src/mxfp8.rs b/examples/balls/smatmul/emu/src/mxfp8.rs deleted file mode 100644 index fc1a388a5..000000000 --- a/examples/balls/smatmul/emu/src/mxfp8.rs +++ /dev/null @@ -1,165 +0,0 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::ExecContext; -use std::cell::RefCell; -use std::collections::HashMap; - -struct Chain { - rows: usize, - cols: usize, - bank: u64, - base: usize, - values: Vec, -} - -thread_local! { - static CHAINS: RefCell> = RefCell::new(HashMap::new()); -} - -fn value(code: u8, factor: f32) -> f32 { - let exponent = (code >> 3) & 15; - let mantissa = (code & 7) as u32; - let normal = ((exponent as u32 + 120) << 23) | (mantissa << 20); - let subnormal = (mantissa as f32 / 512.0).to_bits(); - let element = f32::from_bits(if exponent == 0 { subnormal } else { normal }); - let result = f32::from_bits((element * factor).to_bits() | ((code as u32 & 128) << 24)); - result -} - -pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let rows = (xs2 & 0xfff) as usize; - let cols = ((xs2 >> 12) & 0xfff) as usize; - let k = rs1_iter(xs1) as usize; - let first = (xs2 >> 24) & 1 != 0; - let last = (xs2 >> 25) & 1 != 0; - let base = ((xs2 >> 26) & 63) as usize; - let a = rs1_b0(xs1); - let b = rs1_b1(xs1); - let c = rs1_b2(xs1); - assert!( - xs2 >> 32 == 0 && (rows == 1 || rows == 16) && cols == 16 && k > 0 && k % 32 == 0, - "mxfp8: requires M=1/16, N=16, K multiple of 32 and zero reserved bits" - ); - assert!(a != b && b != c && a != c, "mxfp8: banks must differ"); - for bank in [a, b, c] { - assert!( - ctx.config(bank).allocated && ctx.config(bank).cols == 1, - "mxfp8: banks must be allocated with one column" - ); - } - let stride = bank_row_bytes(); - assert!(stride == 16, "mxfp8: requires 16-byte bank rows"); - assert!( - rows * k + rows * k / 32 <= bank_lines() * stride - && cols * k + cols * k / 32 <= bank_lines() * stride - && base * stride + rows * cols * 4 <= bank_lines() * stride, - "mxfp8: bank footprint exceeds depth" - ); - let pa = pbank(ctx, a); - let pb = pbank(ctx, b); - let a_bytes = &ctx.banks[pa][..rows * k + rows * k / 32]; - let b_bytes = &ctx.banks[pb][..cols * k + cols * k / 32]; - assert!( - a_bytes[..rows * k] - .iter() - .chain(b_bytes[..cols * k].iter()) - .fold(true, |valid, code| valid & (code & 0x7f != 0x7f)), - "mxfp8: NaN encoding" - ); - let mut av = vec![0f32; rows * k]; - let mut bv = vec![0f32; cols * k]; - for (block, dst) in av.chunks_exact_mut(32).enumerate() { - let scale = a_bytes[rows * k + block]; - assert!(scale != 255, "mxfp8: NaN encoding"); - let factor = if scale == 0 { - f32::from_bits(1 << 22) - } else { - f32::from_bits((scale as u32) << 23) - }; - for (index, decoded) in dst.iter_mut().enumerate() { - *decoded = value(a_bytes[block * 32 + index], factor); - } - } - for col in 0..cols { - for block in 0..k / 32 { - let scale = b_bytes[cols * k + col * (k / 32) + block]; - assert!(scale != 255, "mxfp8: NaN encoding"); - let factor = if scale == 0 { - f32::from_bits(1 << 22) - } else { - f32::from_bits((scale as u32) << 23) - }; - let codes = &b_bytes[col * k + block * 32..col * k + (block + 1) * 32]; - let mut decoded = [0f32; 32]; - for (decoded, code) in decoded.iter_mut().zip(codes.iter()) { - *decoded = value(*code, factor); - } - for (index, decoded) in decoded.iter().enumerate() { - bv[(block * 32 + index) * cols + col] = *decoded; - } - } - } - assert!( - av.iter() - .chain(bv.iter()) - .fold(true, |valid, value| valid & value.is_finite()), - "mxfp8: decoded value overflows FP32" - ); - let mut chain = CHAINS.with(|chains| { - let mut chains = chains.borrow_mut(); - if first { - assert!( - !chains.contains_key(&ctx.hart_id), - "mxfp8: first while chain is live" - ); - Chain { - rows, - cols, - bank: c, - base, - values: vec![0f32; rows * cols], - } - } else { - chains - .remove(&ctx.hart_id) - .expect("mxfp8: continuation without first") - } - }); - assert!( - chain.rows == rows && chain.cols == cols && chain.bank == c && chain.base == base, - "mxfp8: continuation changed destination" - ); - for row in 0..rows { - if av[row * k..(row + 1) * k].iter().all(|value| *value == 0.0) - && chain.values[row * cols..(row + 1) * cols] - .iter() - .all(|value| value.to_bits() == 0) - { - continue; - } - let av = &av[row * k..(row + 1) * k]; - let dst = &mut chain.values[row * cols..(row + 1) * cols]; - let mut sums: [f32; 16] = dst.try_into().unwrap(); - for inner in 0..k { - let a = av[inner]; - let weights = &bv[inner * 16..(inner + 1) * 16]; - for col in 0..16 { - sums[col] = a.mul_add(weights[col], sums[col]); - } - } - assert!(sums.iter().all(|v| v.is_finite()), "accumulator overflow"); - dst.copy_from_slice(&sums); - } - if last { - let pc = pbank(ctx, c); - for (i, value) in chain.values.iter().enumerate() { - ctx.banks[pc][base * stride + i * 4..base * stride + i * 4 + 4] - .copy_from_slice(&value.to_le_bytes()); - } - } else { - CHAINS.with(|chains| { - chains.borrow_mut().insert(ctx.hart_id, chain); - }); - } - 0 -} diff --git a/examples/balls/smatmul/verify/README.md b/examples/balls/smatmul/verify/README.md deleted file mode 100644 index 0dea2ea0e..000000000 --- a/examples/balls/smatmul/verify/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# SMatMulBall UVM Verification - -Verifies the generated `SMatMulBall` module using the shared Blink UVM framework. - -## Structure - -- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM agents and base env -- `src/common/smatmul_defs.svh`: DPI imports, `matrix_require_bid`, timeouts -- `src/common/smatmul_items.svh`: `smatmul_cmd_item` with `load_rust_case` -- `src/seq/smatmul_sequences.svh`: one-case sequence -- `src/cov/smatmul_cov.svh`: M/N/K shape bins `{1,2,4,8,16}` -- `src/env/smatmul_scoreboard.svh`: preload A/B, compare writes vs DPI expected list -- `src/env/smatmul_env.svh`: extends `bb_blink_env#(2,4)` -- `src/tests/smatmul_test.svh`: directed 0..3, random 4..23 -- `src/pkg/smatmul_pkg.sv`: package entry -- `src/tb_top.sv`: `bb_blink_if#(2,2)` + `SMatMulBall` (2 read, 2 write ports) -- `filelists/smatmul_ball.f`: VCS filelist - -## Test plan - -`+UVM_TESTNAME=smatmul_ball_test` runs: - -- case 0: 4x4x4 OS (banks 0,1,2) -- case 1: 5x7x3 OS tail -- case 2: 16x16x16 OS -- case 3: 32x16x16 WS -- cases 4..23: random M in {1,2,4,8,16,32}, N/K in {1,2,4,8,16}, distinct banks 0..7 - -Each case: reset, preload A/B into mem model, drive one command, wait for -`scb.done()` or timeout at 100000 cycles. Scoreboard compares writes -(group/addr/data/mask) in DPI emission order. - -## BID (required) - -`+BID=` is mandatory. Missing it fatals at test start. No default bid. - -| Config | Plusarg | -|--------|---------| -| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=1` | -| `sims.verilator.BuckyballToyVerilatorConfig` | `+BID=4` | - -```console -./simv +UVM_TESTNAME=smatmul_ball_test +BID=4 -``` - -## Build - -```console -nix develop ../../../../verify -cargo build --manifest-path casegen/Cargo.toml -``` - -Compile from this directory: - -```console -vcs -full64 -sverilog -timescale=1ns/1ps \ - $VCS_UVM_ARGS \ - -sv_lib casegen/target/debug/libmatrix_casegen \ - -f filelists/smatmul_ball.f -``` - -Run: - -```console -./simv +UVM_TESTNAME=smatmul_ball_test +BID=4 -``` - -Or via bbdev: - -```console -bbdev uvm --build '--ball=matrix' -bbdev uvm --run '--ball=matrix' -- '+BID=4' -``` - -## Acceptance - -- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` -- [x] casegen whole-case DPI API; `cargo test` in casegen -- [x] VCS build with `smatmul_ball.f` (no segfault on Instantiate DUT) -- [x] `+BID=` required; pebble `+BID=1` / toy `+BID=4` -- [ ] 23/23 functional pass diff --git a/examples/balls/smatmul/verify/casegen/src/casegen.rs b/examples/balls/smatmul/verify/casegen/src/casegen.rs deleted file mode 100644 index d776fb6f1..000000000 --- a/examples/balls/smatmul/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,156 +0,0 @@ -use crate::model; - -pub const BIAS_BANK: u32 = 0; -pub const C_BANK: u32 = 5; - -#[repr(C)] -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct MatrixCmd { - pub kind: u32, - pub bid: u32, - pub rob_id: u32, - pub op1_words: u32, - pub op2_words: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, - pub rs2_lo: u32, - pub rs2_hi: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct MatrixCase { - pub commands: Vec, - pub bias: Vec, - pub a: Vec>, - pub b: Vec>, - pub writes: Vec, -} - -pub fn gen_case(seed: u32, index: u32, bid: u32, tile_rows: usize) -> MatrixCase { - match index { - 0 => build_case(seed, bid, 16, 1, tile_rows), - 1 => build_case(seed, bid, 16, 2, tile_rows), - 2 => build_case(seed ^ index, bid, 16, 1, tile_rows), - _ => { - let mut rng = Rng::new(seed, index); - let blocks = 1 + (rng.next() % 2) as usize; - build_case(rng.next(), bid, 16, blocks, tile_rows) - } - } -} - -fn build_case(seed: u32, bid: u32, rows: usize, blocks: usize, tile_rows: usize) -> MatrixCase { - assert!(matches!(rows, 16 | 32 | 64)); - assert!(matches!(blocks, 1 | 2)); - assert!(rows * model::RESULT_WORDS <= model::BANK_DEPTH); - let mut rng = Rng::new(seed, rows as u32 ^ blocks as u32); - let bias_values: Vec = (0..model::TILE).map(|_| rng.value() as i32).collect(); - let mut bias = Vec::with_capacity(4 * model::ROW_BYTES); - for chunk in bias_values.chunks(4) { - for value in chunk { - bias.extend_from_slice(&value.to_le_bytes()); - } - } - - let mut a = Vec::with_capacity(blocks); - let mut b = Vec::with_capacity(blocks); - let mut result = vec![bias_values.clone(); rows]; - let mut commands = Vec::with_capacity(blocks + 1); - commands.push(MatrixCmd { - kind: 0, - bid, - rob_id: 0, - op1_words: 4, - rs1_lo: model::encode_rs1(BIAS_BANK, 0, 0, 4) as u32, - rs1_hi: (model::encode_rs1(BIAS_BANK, 0, 0, 4) >> 32) as u32, - ..MatrixCmd::default() - }); - - for block in 0..blocks { - let a_bank = 1 + 2 * block as u32; - let b_bank = 2 + 2 * block as u32; - let a_values: Vec = (0..rows * model::TILE).map(|_| rng.value()).collect(); - let b_values: Vec = (0..model::TILE * model::TILE) - .map(|_| rng.value()) - .collect(); - let a_packed = model::pack_a(&a_values, rows); - let b_packed = model::pack_b(&b_values); - a.push(a_packed); - b.push(b_packed); - for row in 0..rows { - for column in 0..model::TILE { - for k in 0..model::TILE { - result[row][column] = result[row][column].wrapping_add( - a_values[row * model::TILE + k] as i32 - * b_values[k * model::TILE + column] as i32, - ); - } - } - } - let rs1 = model::encode_rs1(a_bank, b_bank, C_BANK, model::TILE as u64); - let rs2 = model::encode_rs2(rows, block == 0, block + 1 == blocks); - commands.push(MatrixCmd { - kind: 1, - bid, - rob_id: (block + 1) as u32, - op1_words: rows as u32, - op2_words: (model::TILE * rows.div_ceil(tile_rows)) as u32, - rs1_lo: rs1 as u32, - rs1_hi: (rs1 >> 32) as u32, - rs2_lo: rs2 as u32, - rs2_hi: (rs2 >> 32) as u32, - }); - } - - MatrixCase { - commands, - bias, - a, - b, - writes: model::emit_writes(&result), - } -} - -struct Rng { - state: u64, -} - -impl Rng { - fn new(seed: u32, salt: u32) -> Self { - Self { - state: u64::from(seed) << 32 | u64::from(salt), - } - } - fn next(&mut self) -> u32 { - self.state ^= self.state << 13; - self.state ^= self.state >> 7; - self.state ^= self.state << 17; - self.state as u32 - } - fn value(&mut self) -> i8 { - (self.next() % 17) as i8 - 8 - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn single_block_fills_one_result_bank() { - let case = gen_case(1, 2, 1, 16); - assert_eq!(case.commands.len(), 2); - assert_eq!(case.writes.len(), 64); - assert_eq!(case.writes.last().unwrap().addr, 63); - } - - #[test] - fn two_blocks_encode_one_live_chain() { - let case = gen_case(1, 1, 1, 16); - assert_eq!(case.commands.len(), 3); - assert_eq!((case.commands[1].rs2_lo >> 24) & 3, 1); - assert_eq!((case.commands[2].rs2_lo >> 24) & 3, 2); - assert_eq!(case.commands[1].rs1_lo, 1 | (2 << 10) | (5 << 20)); - assert_eq!(case.commands[2].rs1_lo, 3 | (4 << 10) | (5 << 20)); - } -} diff --git a/examples/balls/smatmul/verify/casegen/src/model.rs b/examples/balls/smatmul/verify/casegen/src/model.rs deleted file mode 100644 index 48b7e9118..000000000 --- a/examples/balls/smatmul/verify/casegen/src/model.rs +++ /dev/null @@ -1,58 +0,0 @@ -pub const TILE: usize = 16; -pub const ROW_BYTES: usize = 16; -pub const RESULT_WORDS: usize = 4; -pub const BANK_DEPTH: usize = 64; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct WriteExp { - pub addr: u32, - pub data: [u8; ROW_BYTES], -} - -pub fn pack_a(src: &[i8], rows: usize) -> Vec { - assert_eq!(src.len(), rows * TILE); - src.iter().map(|value| *value as u8).collect() -} - -pub fn pack_b(src: &[i8]) -> Vec { - assert_eq!(src.len(), TILE * TILE); - src.iter().map(|value| *value as u8).collect() -} - -pub fn emit_writes(c: &[Vec]) -> Vec { - let mut writes = Vec::with_capacity(c.len() * RESULT_WORDS); - for (row, values) in c.iter().enumerate() { - assert_eq!(values.len(), TILE); - for word in 0..RESULT_WORDS { - let mut data = [0; ROW_BYTES]; - for lane in 0..4 { - let value = values[word * 4 + lane]; - data[lane * 4..lane * 4 + 4].copy_from_slice(&value.to_le_bytes()); - } - writes.push(WriteExp { - addr: (row * RESULT_WORDS + word) as u32, - data, - }); - } - } - writes -} - -pub fn word(data: &[u8], index: usize) -> [u8; ROW_BYTES] { - data[index * ROW_BYTES..(index + 1) * ROW_BYTES] - .try_into() - .unwrap() -} - -pub fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u64) -> u64 { - assert!(op1 < 1024 && op2 < 1024 && wr < 1024 && iter < (1_u64 << 34)); - u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (iter << 30) -} - -pub fn encode_rs2(rows: usize, first: bool, last: bool) -> u64 { - assert!(matches!(rows, 16 | 32 | 64)); - rows as u64 - | ((TILE as u64) << 12) - | ((first as u64) << 24) - | ((last as u64) << 25) -} diff --git a/examples/balls/smatmul/verify/src/tb_top.sv b/examples/balls/smatmul/verify/src/tb_top.sv deleted file mode 100644 index a69d8e3f6..000000000 --- a/examples/balls/smatmul/verify/src/tb_top.sv +++ /dev/null @@ -1,101 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import smatmul_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - SMatMulBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) - ); - - initial begin - intf.clock = 1'b0; - intf.reset = 1'b1; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("smatmul_ball_test"); - end -endmodule diff --git a/examples/balls/smatmul/workloads/ctests/CMakeLists.txt b/examples/balls/smatmul/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 8c9dad8d2..000000000 --- a/examples/balls/smatmul/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -add_buckyball_ctests( - smatmul_bias_accumulate_test.c - smatmul_1x16x64_test.c - smatmul_8x16_test.c - smatmul_16x16_test.c - smatmul_16x16x32_test.c - smatmul_nondefault_banks_test.c -) -file(STRINGS "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/ballISA.h" F32_ISA - REGEX "^#define BB_FUNC7_SMATMUL_F32 ") -if(F32_ISA) - add_buckyball_ctest(smatmul_f32_test.c) -endif() diff --git a/examples/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir b/examples/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir deleted file mode 100644 index 520c8c27d..000000000 --- a/examples/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir +++ /dev/null @@ -1,86 +0,0 @@ -func.func private @check_result(memref<64x4xi32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_i32 = arith.constant 0 : i32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %eight = arith.constant 8 : index - %sixteen = arith.constant 16 : index - %depth4 = arith.constant 4 : i64 - %depth16 = arith.constant 16 : i64 - %depth64 = arith.constant 64 : i64 - %stride = arith.constant 1 : i64 - %config = arith.constant 268501008 : i64 - %true = arith.constant true - %false = arith.constant false - %zero64 = arith.constant 0 : i64 - - %input0 = memref.alloc() alignment = 64 : memref<16x16xi8> - %input1 = memref.alloc() alignment = 64 : memref<16x16xi8> - %weight0 = memref.alloc() alignment = 64 : memref<16x16xi8> - %weight1 = memref.alloc() alignment = 64 : memref<16x16xi8> - %bias = memref.alloc() alignment = 64 : memref<4x4xi32> - %output = memref.alloc() alignment = 64 : memref<64x4xi32> - linalg.fill ins(%zero_i32 : i32) outs(%output : memref<64x4xi32>) - - scf.for %row = %zero to %sixteen step %one { - scf.for %column = %zero to %sixteen step %one { - %row_i32 = arith.index_cast %row : index to i32 - %column_i32 = arith.index_cast %column : index to i32 - %sum = arith.addi %row_i32, %column_i32 : i32 - %difference = arith.subi %row_i32, %column_i32 : i32 - %twice_row = arith.addi %row_i32, %row_i32 : i32 - %second = arith.subi %twice_row, %column_i32 : i32 - %sum_i8 = arith.trunci %sum : i32 to i8 - %difference_i8 = arith.trunci %difference : i32 to i8 - %second_i8 = arith.trunci %second : i32 to i8 - memref.store %difference_i8, %input0[%row, %column] : memref<16x16xi8> - memref.store %sum_i8, %weight0[%row, %column] : memref<16x16xi8> - memref.store %sum_i8, %input1[%row, %column] : memref<16x16xi8> - memref.store %second_i8, %weight1[%row, %column] : memref<16x16xi8> - } - %bias_value = arith.subi %row, %eight : index - %bias_i32 = arith.index_cast %bias_value : index to i32 - %bias_row = arith.divui %row, %four : index - %bias_lane = arith.remui %row, %four : index - memref.store %bias_i32, %bias[%bias_row, %bias_lane] : memref<4x4xi32> - } - - %bias_bank = buckyball.bank_alloc - %a0_bank = buckyball.bank_alloc - %b0_bank = buckyball.bank_alloc - %a1_bank = buckyball.bank_alloc - %b1_bank = buckyball.bank_alloc - %result_bank = buckyball.bank_alloc - %loaded_bias = buckyball.bank_mvin %bias %bias_bank %depth4 %stride - : memref<4x4xi32> i64 i64 i64 - %loaded_a0 = buckyball.bank_mvin %input0 %a0_bank %depth16 %stride - : memref<16x16xi8> i64 i64 i64 - %loaded_b0 = buckyball.bank_mvin %weight0 %b0_bank %depth16 %stride - : memref<16x16xi8> i64 i64 i64 - %loaded_a1 = buckyball.bank_mvin %input1 %a1_bank %depth16 %stride - : memref<16x16xi8> i64 i64 i64 - %loaded_b1 = buckyball.bank_mvin %weight1 %b1_bank %depth16 %stride - : memref<16x16xi8> i64 i64 i64 - buckyball.smatmul_bias %loaded_bias, %zero64 : i64 - buckyball.smatmul %loaded_a0, %loaded_b0, %result_bank, %config, %true, %false, %zero64 : i64 - buckyball.smatmul %loaded_a1, %loaded_b1, %result_bank, %config, %false, %true, %zero64 : i64 - %stored = buckyball.bank_mvout %output %result_bank %depth64 %stride - : memref<64x4xi32> i64 i64 i64 - func.call @check_result(%output) : (memref<64x4xi32>) -> () - buckyball.bank_release %loaded_bias : i64 - buckyball.bank_release %loaded_a0 : i64 - buckyball.bank_release %loaded_b0 : i64 - buckyball.bank_release %loaded_a1 : i64 - buckyball.bank_release %loaded_b1 : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input0 : memref<16x16xi8> - memref.dealloc %input1 : memref<16x16xi8> - memref.dealloc %weight0 : memref<16x16xi8> - memref.dealloc %weight1 : memref<16x16xi8> - memref.dealloc %bias : memref<4x4xi32> - memref.dealloc %output : memref<64x4xi32> - return %zero_i8 : i8 -} diff --git a/examples/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index 51bcb8fda..000000000 --- a/examples/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) - -buckyball_require_smatmul_bank_shape(${BUCKYBALL_MLIR_ACTIVE_TARGET}) -add_buckyball_mlir_test(smatmul_bias_two_terms TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in b/examples/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in deleted file mode 100644 index 8cdd1861c..000000000 --- a/examples/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in +++ /dev/null @@ -1,80 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x4xi32>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_i32 = arith.constant 0 : i32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %eight = arith.constant 8 : index - %sixteen = arith.constant 16 : index - %m = arith.constant @BB_SMM_M@ : index - %depth4 = arith.constant 4 : i64 - %depth16 = arith.constant 16 : i64 - %depth_out = arith.constant @BB_BANK_LINES@ : i64 - %m_i64 = arith.constant @BB_SMM_M@ : i64 - %stride = arith.constant 1 : i64 - %config = arith.constant @BB_SMM_CONFIG@ : i64 - %true = arith.constant true - %false = arith.constant false - %zero64 = arith.constant 0 : i64 - %a = memref.alloc() : memref<@BB_SMM_M@x16xi8> - %b0 = memref.alloc() : memref<16x16xi8> - %b1 = memref.alloc() : memref<16x16xi8> - %bias = memref.alloc() : memref<4x4xi32> - %output = memref.alloc() : memref<@BB_BANK_LINES@x4xi32> - linalg.fill ins(%zero_i32 : i32) outs(%output : memref<@BB_BANK_LINES@x4xi32>) - scf.for %row = %zero to %m step %one { - %slot = arith.remui %row, %sixteen : index - scf.for %col = %zero to %sixteen step %one { - %v = arith.subi %slot, %col : index - %v8 = arith.index_cast %v : index to i8 - memref.store %v8, %a[%row, %col] : memref<@BB_SMM_M@x16xi8> - } - } - scf.for %row = %zero to %sixteen step %one { - %bias_value = arith.subi %row, %eight : index - %bias_i32 = arith.index_cast %bias_value : index to i32 - %bias_row = arith.divui %row, %four : index - %bias_lane = arith.remui %row, %four : index - memref.store %bias_i32, %bias[%bias_row, %bias_lane] : memref<4x4xi32> - scf.for %col = %zero to %sixteen step %one { - %r32 = arith.index_cast %row : index to i32 - %c32 = arith.index_cast %col : index to i32 - %plus32 = arith.addi %r32, %c32 : i32 - %twice32 = arith.addi %r32, %r32 : i32 - %minus32 = arith.subi %twice32, %c32 : i32 - %plus = arith.trunci %plus32 : i32 to i8 - %minus = arith.trunci %minus32 : i32 to i8 - memref.store %plus, %b0[%row, %col] : memref<16x16xi8> - memref.store %minus, %b1[%row, %col] : memref<16x16xi8> - } - } - %bias_bank = buckyball.bank_alloc - %a0_bank = buckyball.bank_alloc - %b0_bank = buckyball.bank_alloc - %a1_bank = buckyball.bank_alloc - %b1_bank = buckyball.bank_alloc - %result_bank = buckyball.bank_alloc - %loaded_bias = buckyball.bank_mvin %bias %bias_bank %depth4 %stride : memref<4x4xi32> i64 i64 i64 - %loaded_a0 = buckyball.bank_mvin %a %a0_bank %m_i64 %stride : memref<@BB_SMM_M@x16xi8> i64 i64 i64 - %loaded_b0 = buckyball.bank_mvin %b0 %b0_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - %loaded_a1 = buckyball.bank_mvin %a %a1_bank %m_i64 %stride : memref<@BB_SMM_M@x16xi8> i64 i64 i64 - %loaded_b1 = buckyball.bank_mvin %b1 %b1_bank %depth16 %stride : memref<16x16xi8> i64 i64 i64 - %bias_state = buckyball.bank_smatmul_bias %loaded_bias %zero64 : i64 i64 - %partial = buckyball.bank_smatmul %loaded_a0 %loaded_b0 %result_bank %config %true %false %zero64 : i64 i64 i64 i64 - %result = buckyball.bank_smatmul %loaded_a1 %loaded_b1 %partial %config %false %true %zero64 : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %depth_out %stride : memref<@BB_BANK_LINES@x4xi32> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x4xi32>) -> () - buckyball.bank_release %bias_state : i64 - buckyball.bank_release %loaded_a0 : i64 - buckyball.bank_release %loaded_b0 : i64 - buckyball.bank_release %loaded_a1 : i64 - buckyball.bank_release %loaded_b1 : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %a : memref<@BB_SMM_M@x16xi8> - memref.dealloc %b0 : memref<16x16xi8> - memref.dealloc %b1 : memref<16x16xi8> - memref.dealloc %bias : memref<4x4xi32> - memref.dealloc %output : memref<@BB_BANK_LINES@x4xi32> - return %zero_i8 : i8 -} diff --git a/examples/balls/toint8/arch/src/main/scala/ToInt8Ball.scala b/examples/balls/toint8/arch/src/main/scala/ToInt8Ball.scala deleted file mode 100644 index 0f32431a5..000000000 --- a/examples/balls/toint8/arch/src/main/scala/ToInt8Ball.scala +++ /dev/null @@ -1,325 +0,0 @@ -package examples.balls.toint8 - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import hardfloat.{recFNFromFN, INToRecFN, RecFNToIN} -import hardfloat.consts.{round_near_even, tininess_afterRounding} -import freechips.rocketchip.tile.MulAddRecFNPipe - -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.{MmioRead, MmioWrite} -import framework.top.GlobalConfig - -@instantiable -class ToInt8Ball(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - private val mapping = b.ballDomain.ballIdMappings - .find(_.ballName == "ToInt8Ball") - .getOrElse(throw new IllegalArgumentException("ToInt8Ball not found in config")) - - private val f32Funct = b.ballDomain.ballISA - .find(_.mnemonic == "QUANT_F32_TO_I8") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("QUANT_F32_TO_I8 not found in ballISA")) - - private val i32Funct = b.ballDomain.ballISA - .find(_.mnemonic == "QUANT_I32_TO_I8") - .map(_.funct7) - .getOrElse(throw new IllegalArgumentException("QUANT_I32_TO_I8 not found in ballISA")) - - require(b.memDomain.bankWidth == 128, "ToInt8Ball requires 128-bit bank rows") - require(b.memDomain.bankEntries >= 4, "ToInt8Ball requires at least four bank rows") - require(mapping.inBW == 1, "ToInt8Ball requires inBW=1") - require(mapping.outBW == 1, "ToInt8Ball requires outBW=1") - require((f32Funct >> 4) == 3, "QUANT_F32_TO_I8 must encode one read and one write") - require((i32Funct >> 4) == 4, "QUANT_I32_TO_I8 must encode two reads and one write") - - @public - val io = IO(new BlinkIO(b, mapping.inBW, mapping.outBW)) - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - private val idle :: scaleReq :: scaleResp :: inputReq :: inputResp :: pack :: packDrain :: writeReq :: writeResp :: complete :: Nil = - Enum(10) - private val state = RegInit(idle) - - private val robIdReg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - private val isSubReg = RegInit(false.B) - private val subRobIdReg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - private val inputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val scaleBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val outputBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - private val iterReg = RegInit(0.U(b.frontend.iter_len.W)) - private val inputBaseReg = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - // Relative row within the command. inputBaseReg is the absolute bank row. - private val relativeRow = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val outputRow = RegInit(0.U(log2Ceil(b.memDomain.bankEntries).W)) - private val outputWidth = RegInit(0.U(7.W)) - private val outputStride = RegInit(0.U(7.W)) - private val outputColumn = RegInit(0.U(7.W)) - private val scaleRow = RegInit(0.U(2.W)) - private val i32Mode = RegInit(false.B) - private val reluReg = RegInit(false.B) - private val tensorScale = RegInit(0.U(32.W)) - private val scales = Reg(Vec(16, UInt(32.W))) - private val inputData = Reg(UInt(128.W)) - private val outputData = RegInit(0.U(128.W)) - private val writeData = Reg(UInt(128.W)) - private val lane = RegInit(0.U(2.W)) - private val outputBytes = Reg(Vec(4, UInt(8.W))) - - private def positiveFinite(value: UInt): Bool = - !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U - - private val intToFloat = Module(new INToRecFN(32, 8, 24)) - // The unpipelined recoded-float multiplier was the critical Buckyball path - // (about 8.5 ns before clock uncertainty). FMA mode with c=0 preserves the - // multiplication result while splitting the datapath into three stages. - private val multiply = Module(new MulAddRecFNPipe(3, 8, 24)) - private val floatToInt = Module(new RecFNToIN(8, 24, 8)) - private val input = (inputData >> (lane << 5))(31, 0) - private val scale = Mux(i32Mode, scales(Cat(relativeRow(1, 0), lane)), tensorScale) - private val packIssue = state === pack - - private val inputStage0 = RegEnable(input, packIssue) - private val scaleStage0 = RegEnable(scale, packIssue) - private val laneStage0 = RegEnable(lane, packIssue) - private val convertValid = RegNext(packIssue, false.B) - - private val multiplyInput = RegEnable( - Mux(i32Mode, intToFloat.io.out, recFNFromFN(8, 24, inputStage0)), - convertValid - ) - - private val multiplyScale = RegEnable(recFNFromFN(8, 24, scaleStage0), convertValid) - private val laneStage1 = RegEnable(laneStage0, convertValid) - private val multiplyValid = RegNext(convertValid, false.B) - - private val multiplyResult = RegEnable(multiply.io.out, multiply.io.validout) - private val multiplyLane = Pipe(multiplyValid, laneStage1, 3) - private val laneStage2 = RegEnable(multiplyLane.bits, multiply.io.validout) - private val toIntValid = RegNext(multiply.io.validout, false.B) - - private val quantizedResult = RegEnable( - Mux(reluReg && floatToInt.io.out(7), 0.U, floatToInt.io.out), - toIntValid - ) - - private val quantizedLane = RegEnable(laneStage2, toIntValid) - private val quantizedValid = RegNext(toIntValid, false.B) - - intToFloat.io.signedIn := true.B - intToFloat.io.in := inputStage0 - intToFloat.io.roundingMode := round_near_even - intToFloat.io.detectTininess := tininess_afterRounding - - multiply.io.a := multiplyInput - multiply.io.b := multiplyScale - multiply.io.c := recFNFromFN(8, 24, 0.U) - multiply.io.op := 0.U - multiply.io.validin := multiplyValid - multiply.io.roundingMode := round_near_even - multiply.io.detectTininess := tininess_afterRounding.asUInt - - floatToInt.io.in := multiplyResult - floatToInt.io.roundingMode := round_near_even - floatToInt.io.signedOut := true.B - - io.cmdReq.ready := state === idle - io.cmdResp.valid := state === complete - io.cmdResp.bits.rob_id := robIdReg - io.cmdResp.bits.is_sub := isSubReg - io.cmdResp.bits.sub_rob_id := subRobIdReg - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete - - io.bankRead(0).rob_id := robIdReg - io.bankRead(0).ball_id := 0.U - io.bankRead(0).bank_id := Mux(state === scaleReq || state === scaleResp, scaleBank, inputBank) - io.bankRead(0).group_id := 0.U - io.bankRead(0).io.req.valid := false.B - io.bankRead(0).io.req.bits.addr := 0.U - io.bankRead(0).io.resp.ready := false.B - - io.bankWrite(0).rob_id := robIdReg - io.bankWrite(0).ball_id := 0.U - io.bankWrite(0).bank_id := outputBank - io.bankWrite(0).group_id := 0.U - io.bankWrite(0).io.req.valid := false.B - io.bankWrite(0).io.req.bits.addr := outputRow - io.bankWrite(0).io.req.bits.data := writeData - io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(true.B)) - io.bankWrite(0).io.resp.ready := false.B - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - MmioRead.tieOff(io.mmioRead) - MmioWrite.tieOff(io.mmioWrite) - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - val isF32 = cmd.funct7 === f32Funct.U - val isI32 = cmd.funct7 === i32Funct.U - - assert(isF32 || isI32, "ToInt8Ball received an unknown funct7") - assert(cmd.iter > 0.U && cmd.iter(1, 0) === 0.U, "ToInt8Ball iter must be a positive multiple of four") - assert(cmd.iter <= b.memDomain.bankEntries.U, "ToInt8Ball input exceeds bank depth") - assert(cmd.rs1(9, 0) < b.memDomain.virtualBankCount.U, "ToInt8Ball input bank is invalid") - assert(cmd.rs1(29, 20) < b.memDomain.virtualBankCount.U, "ToInt8Ball output bank is invalid") - assert(cmd.op1_col === 1.U && cmd.wr_col === 1.U, "ToInt8Ball input and output must each occupy one bank") - assert(cmd.op1_bank =/= cmd.wr_bank, "ToInt8Ball input and output banks must differ") - when(isF32) { - assert(cmd.rs1(19, 10) === 0.U, "QUANT_F32_TO_I8 reserves input bank 1") - assert(cmd.rs2(63, 32) === 0.U, "QUANT_F32_TO_I8 reserves rs2[63:32]") - assert(positiveFinite(cmd.rs2(31, 0)), "QUANT_F32_TO_I8 scale must be finite and positive") - }.otherwise { - assert(cmd.rs1(19, 10) < b.memDomain.virtualBankCount.U, "QUANT_I32_TO_I8 scale bank is invalid") - assert(cmd.op2_col === 1.U, "QUANT_I32_TO_I8 scale must occupy one bank") - assert(cmd.op1_bank =/= cmd.op2_bank, "QUANT_I32_TO_I8 input and scale banks must differ") - assert(cmd.op2_bank =/= cmd.wr_bank, "QUANT_I32_TO_I8 scale and output banks must differ") - val outputBase = cmd.rs2(7, 1) - val width = cmd.rs2(14, 8) - val height = cmd.rs2(21, 15) - val stride = cmd.rs2(28, 22) - val outputRows = cmd.iter >> 2 - val outputEnd = outputBase +& ((height - 1.U) * stride) +& width - val inputBase = cmd.rs2(34, 29) - assert(cmd.rs2(63, 35) === 0.U, "QUANT_I32_TO_I8 reserves rs2[63:35]") - assert(width > 0.U && height > 0.U && stride >= width, "QUANT_I32_TO_I8 output geometry is invalid") - assert(width * height === outputRows, "QUANT_I32_TO_I8 iter does not match output geometry") - assert(outputEnd <= b.memDomain.bankEntries.U, "QUANT_I32_TO_I8 output exceeds bank depth") - assert(inputBase +& cmd.iter <= b.memDomain.bankEntries.U, "QUANT_I32_TO_I8 input exceeds bank depth") - } - - robIdReg := io.cmdReq.bits.rob_id - isSubReg := io.cmdReq.bits.is_sub - subRobIdReg := io.cmdReq.bits.sub_rob_id - inputBank := cmd.op1_bank - scaleBank := cmd.op2_bank - outputBank := cmd.wr_bank - iterReg := cmd.iter - inputBaseReg := Mux(isI32, cmd.rs2(34, 29), 0.U) - relativeRow := 0.U - outputRow := Mux(isI32, cmd.rs2(7, 1), 0.U) - outputWidth := Mux(isI32, cmd.rs2(14, 8), 0.U) - outputStride := Mux(isI32, cmd.rs2(28, 22), 0.U) - outputColumn := 0.U - scaleRow := 0.U - i32Mode := isI32 - reluReg := isI32 && cmd.rs2(0) - tensorScale := cmd.rs2(31, 0) - outputData := 0.U - lane := 0.U - state := Mux(isI32, scaleReq, inputReq) - } - } - - is(scaleReq) { - io.bankRead(0).io.req.valid := true.B - io.bankRead(0).io.req.bits.addr := scaleRow - when(io.bankRead(0).io.req.fire) { - state := scaleResp - } - } - - is(scaleResp) { - io.bankRead(0).io.resp.ready := true.B - when(io.bankRead(0).io.resp.fire) { - for (lane <- 0 until 4) { - val value = io.bankRead(0).io.resp.bits.data(32 * lane + 31, 32 * lane) - assert(positiveFinite(value), "QUANT_I32_TO_I8 scales must be finite and positive") - scales(Cat(scaleRow, lane.U(2.W))) := value - } - when(scaleRow === 3.U) { - state := inputReq - }.otherwise { - scaleRow := scaleRow + 1.U - state := scaleReq - } - } - } - - is(inputReq) { - io.bankRead(0).io.req.valid := true.B - io.bankRead(0).io.req.bits.addr := inputBaseReg +& relativeRow - when(io.bankRead(0).io.req.fire) { - state := inputResp - } - } - - is(inputResp) { - io.bankRead(0).io.resp.ready := true.B - when(io.bankRead(0).io.resp.fire) { - inputData := io.bankRead(0).io.resp.bits.data - lane := 0.U - state := pack - } - } - - is(pack) { - when(!i32Mode) { - assert(input(30, 23) =/= 255.U, "QUANT_F32_TO_I8 input must be finite") - } - when(lane === 3.U) { - state := packDrain - }.otherwise { - lane := lane + 1.U - } - } - - is(packDrain) {} - - is(writeReq) { - io.bankWrite(0).io.req.valid := true.B - when(io.bankWrite(0).io.req.fire) { - state := writeResp - } - } - - is(writeResp) { - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.resp.fire) { - when(relativeRow === iterReg - 1.U) { - state := complete - }.otherwise { - relativeRow := relativeRow + 1.U - when(outputColumn === outputWidth - 1.U) { - outputRow := outputRow + outputStride - outputWidth + 1.U - outputColumn := 0.U - }.otherwise { - outputRow := outputRow + 1.U - outputColumn := outputColumn + 1.U - } - state := inputReq - } - } - } - - is(complete) { - when(io.cmdResp.fire) { - state := idle - } - } - } - - when((state === pack || state === packDrain) && quantizedValid) { - outputBytes(quantizedLane) := quantizedResult - when(quantizedLane === 3.U) { - val bytes = Cat(quantizedResult, outputBytes(2), outputBytes(1), outputBytes(0)) - val nextWord = outputData | (bytes << (relativeRow(1, 0) << 5)) - when(relativeRow(1, 0) === 3.U) { - writeData := nextWord - outputData := 0.U - state := writeReq - }.otherwise { - outputData := nextWord - relativeRow := relativeRow + 1.U - state := inputReq - } - } - } -} diff --git a/examples/balls/toint8/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/toint8/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 35ff7f000..000000000 --- a/examples/balls/toint8/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,56 +0,0 @@ -//===- AssignPhysicalBankPatterns.cpp - ToInt8 bank assignment patterns ---===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateToInt8BallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { - -class BankQuantF32ToI8Pattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankQuantF32ToI8Op op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getInBank(), - op.getOutBank(), op.getIter(), - op.getScale()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; - -class BankQuantI32ToI8Pattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankQuantI32ToI8Op op, - PatternRewriter &rewriter) const override { - rewriter.create( - op.getLoc(), op.getInBank(), op.getScaleBank(), op.getOutBank(), - op.getIter(), op.getOutputBase(), op.getInputBase(), - op.getOutputWidthAttr(), op.getOutputHeightAttr(), - op.getOutputStrideAttr(), op.getReluAttr()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateToInt8BallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add( - patterns.getContext()); -} diff --git a/examples/balls/toint8/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/toint8/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index a83e6e77f..000000000 --- a/examples/balls/toint8/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,122 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct QuantF32ToI8Lowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(QuantF32ToI8Op op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("ToInt8Ball"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - Value scaleBits = rewriter.create( - loc, rewriter.getI32Type(), adaptor.getScale()); - Value rs2 = - rewriter.create(loc, rewriter.getI64Type(), scaleBits); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("QUANT_F32_TO_I8"))); - return success(); - } -}; - -struct QuantI32ToI8Lowering : public ConvertOpToLLVMPattern { - QuantI32ToI8Lowering(LLVMTypeConverter &converter, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), - bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(QuantI32ToI8Op op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("ToInt8Ball"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - adaptor.getScaleBankId(), - adaptor.getOutputBankId(), adaptor.getIter()); - auto iterOp = op.getIter().getDefiningOp(); - auto iterAttr = - iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); - if (!iterAttr || iterAttr.getInt() <= 0) - return op.emitError("INT32 to INT8 iter must be a positive constant"); - if (iterAttr.getInt() % 4 != 0 || iterAttr.getInt() > bankDepth) - return op.emitError( - "INT32 to INT8 iter must be a multiple of four within bank depth"); - int64_t outputRows = iterAttr.getInt() / 4; - int64_t outputWidth = op.getOutputWidthAttr().getInt(); - int64_t outputHeight = op.getOutputHeightAttr().getInt(); - int64_t outputStride = op.getOutputStrideAttr().getInt(); - auto outputBaseOp = - adaptor.getOutputBase().getDefiningOp(); - auto inputBaseOp = - adaptor.getInputBase().getDefiningOp(); - auto outputBaseAttr = outputBaseOp - ? dyn_cast(outputBaseOp.getValue()) - : IntegerAttr(); - auto inputBaseAttr = inputBaseOp - ? dyn_cast(inputBaseOp.getValue()) - : IntegerAttr(); - if (outputWidth <= 0 || outputHeight <= 0 || outputStride < outputWidth || - outputWidth * outputHeight != outputRows || outputWidth >= 128 || - outputHeight >= 128 || outputStride >= 128 || !outputBaseAttr || - !inputBaseAttr || outputBaseAttr.getInt() < 0 || - inputBaseAttr.getInt() < 0 || - outputBaseAttr.getInt() + (outputHeight - 1) * outputStride + - outputWidth > - bankDepth || - inputBaseAttr.getInt() + iterAttr.getInt() > bankDepth) - return op.emitError( - "INT32 to INT8 output tile is invalid for the physical bank"); - Value rs2 = rewriter.create(loc, adaptor.getOutputBase(), - cstI64(rewriter, loc, 1)); - rs2 = rewriter.create( - loc, rs2, - cstI64(rewriter, loc, - (outputWidth << 8) | (outputHeight << 15) | - (outputStride << 22) | (op.getRelu() ? 1 : 0))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getInputBase(), - cstI64(rewriter, loc, 29))); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("QUANT_I32_TO_I8"))); - return success(); - } - -private: - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateToInt8BallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - (void)stable; - patterns.add(converter); - patterns.add(converter, bankDepth); -} - -void configureToInt8BallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/toint8/configs/default.toml b/examples/balls/toint8/configs/default.toml deleted file mode 100644 index 83a5aded0..000000000 --- a/examples/balls/toint8/configs/default.toml +++ /dev/null @@ -1,3 +0,0 @@ -# ToInt8Ball has no private configuration parameters. - -[ball] diff --git a/examples/balls/toint8/emu/src/dispatch.rs b/examples/balls/toint8/emu/src/dispatch.rs deleted file mode 100644 index c85c832ff..000000000 --- a/examples/balls/toint8/emu/src/dispatch.rs +++ /dev/null @@ -1,33 +0,0 @@ -use crate::inst::instruction::ExecContext; - -use super::quant; - -const BALL_CLASS: &str = "examples.balls.toint8.ToInt8Ball"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("QUANT_F32_TO_I8") => Some(quant::exec_f32_to_i8(xs1, xs2, ctx)), - Some("QUANT_I32_TO_I8") => Some(quant::exec_i32_to_i8(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("QUANT_F32_TO_I8") => Some(quant::f32_to_i8_latency(xs1, xs2)), - Some("QUANT_I32_TO_I8") => Some(quant::i32_to_i8_latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/toint8/emu/src/lib.rs b/examples/balls/toint8/emu/src/lib.rs deleted file mode 100644 index 253402123..000000000 --- a/examples/balls/toint8/emu/src/lib.rs +++ /dev/null @@ -1,12 +0,0 @@ -pub(crate) use crate::inst::decode; -pub(crate) use crate::inst::instruction; - -mod model; -mod quant; - -#[cfg(test)] -mod tests; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/toint8/emu/src/quant.rs b/examples/balls/toint8/emu/src/quant.rs deleted file mode 100644 index 314adec29..000000000 --- a/examples/balls/toint8/emu/src/quant.rs +++ /dev/null @@ -1,166 +0,0 @@ -use super::super::bank::bank_lines; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::ExecContext; - -pub(crate) fn exec_f32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let src = rs1_b0(xs1); - let unused = rs1_b1(xs1); - let dst = rs1_b2(xs1); - let iter = rs1_iter(xs1) as usize; - if unused != 0 { - panic!("quant_f32_to_i8: input bank 1 must be zero"); - } - if src == dst { - panic!("quant_f32_to_i8: input and output banks must differ"); - } - if !ctx.config(src).allocated || !ctx.config(dst).allocated { - panic!("quant_f32_to_i8: bank not allocated"); - } - if ctx.config(src).cols != 1 || ctx.config(dst).cols != 1 { - panic!("quant_f32_to_i8: banks must have one column"); - } - if iter == 0 || iter % 4 != 0 || iter > bank_lines() { - panic!("quant_f32_to_i8: iter must be a positive multiple of four within bank depth"); - } - if xs2 >> 32 != 0 { - panic!("quant_f32_to_i8: rs2[63:32] must be zero"); - } - let scale = f32::from_bits(xs2 as u32); - if !scale.is_finite() || scale <= 0.0 { - panic!("quant_f32_to_i8: scale must be finite and positive"); - } - - let ps = pbank(ctx, src); - let pd = pbank(ctx, dst); - let (input, output) = ctx.banks.read_write(ps, pd); - for output_row in 0..iter / 4 { - let mut packed = [0u8; 16]; - for group in 0..4 { - let input_row = output_row * 4 + group; - let base = input_row * 16; - for lane in 0..4 { - let offset = base + lane * 4; - let bits = u32::from_le_bytes(input[offset..offset + 4].try_into().unwrap()); - packed[group * 4 + lane] = - super::model::quantize_f32(f32::from_bits(bits), scale, false) as u8; - } - } - let base = output_row * 16; - output[base..base + 16].copy_from_slice(&packed); - } - 0 -} - -pub(crate) fn exec_i32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let src = rs1_b0(xs1); - let scale_bank = rs1_b1(xs1); - let dst = rs1_b2(xs1); - let iter = rs1_iter(xs1) as usize; - if src == scale_bank || src == dst || scale_bank == dst { - panic!("quant_i32_to_i8: input, scale, and output banks must differ"); - } - if !ctx.config(src).allocated || !ctx.config(scale_bank).allocated || !ctx.config(dst).allocated - { - panic!("quant_i32_to_i8: bank not allocated"); - } - if ctx.config(src).cols != 1 || ctx.config(scale_bank).cols != 1 || ctx.config(dst).cols != 1 { - panic!("quant_i32_to_i8: banks must have one column"); - } - if iter == 0 || iter % 4 != 0 || iter > bank_lines() { - panic!("quant_i32_to_i8: iter must be a positive multiple of four within bank depth"); - } - if xs2 >> 35 != 0 { - panic!("quant_i32_to_i8: rs2[63:35] must be zero"); - } - let input_base = ((xs2 >> 29) & 0x3f) as usize; - let output_base = ((xs2 >> 1) & 0x7f) as usize; - let output_width = ((xs2 >> 8) & 0x7f) as usize; - let output_height = ((xs2 >> 15) & 0x7f) as usize; - let output_stride = ((xs2 >> 22) & 0x7f) as usize; - let output_rows = iter / 4; - if output_width == 0 - || output_height == 0 - || output_stride < output_width - || output_width.checked_mul(output_height) != Some(output_rows) - || output_base - .checked_add((output_height - 1) * output_stride + output_width) - .is_none_or(|end| end > bank_lines()) - || input_base + iter > bank_lines() - { - panic!("quant_i32_to_i8: invalid output tile"); - } - let relu = xs2 & 1 != 0; - - let pscale = pbank(ctx, scale_bank); - let mut scales = [0.0f32; 16]; - for group in 0..4 { - let base = group * 16; - for lane in 0..4 { - let offset = base + lane * 4; - scales[group * 4 + lane] = f32::from_bits(u32::from_le_bytes( - ctx.banks[pscale][offset..offset + 4].try_into().unwrap(), - )); - if !scales[group * 4 + lane].is_finite() || scales[group * 4 + lane] <= 0.0 { - panic!("quant_i32_to_i8: scales must be finite and positive"); - } - } - } - - let ps = pbank(ctx, src); - let pd = pbank(ctx, dst); - let (input, output) = ctx.banks.read_write(ps, pd); - for output_row in 0..iter / 4 { - let mut packed = [0u8; 16]; - for group in 0..4 { - let input_row = input_base + output_row * 4 + group; - let base = input_row * 16; - for lane in 0..4 { - let offset = base + lane * 4; - let value = i32::from_le_bytes(input[offset..offset + 4].try_into().unwrap()); - let channel = group * 4 + lane; - packed[channel] = super::model::quantize_i32(value, scales[channel], relu) as u8; - } - } - let row = - output_base + (output_row / output_width) * output_stride + output_row % output_width; - let base = row * 16; - output[base..base + 16].copy_from_slice(&packed); - } - 0 -} - -pub(crate) fn f32_to_i8_latency(xs1: u64, xs2: u64) -> u64 { - let iter = rs1_iter(xs1); - if iter == 0 || iter % 4 != 0 || xs2 >> 32 != 0 { - panic!("quant_f32_to_i8: illegal encoding"); - } - let scale = f32::from_bits(xs2 as u32); - if !scale.is_finite() || scale <= 0.0 { - panic!("quant_f32_to_i8: scale must be finite and positive"); - } - iter -} - -pub(crate) fn i32_to_i8_latency(xs1: u64, xs2: u64) -> u64 { - let iter = rs1_iter(xs1); - let input_base = (xs2 >> 29) & 0x3f; - if xs2 >> 35 != 0 { - panic!("quant_i32_to_i8: illegal encoding"); - } - let output_base = (xs2 >> 1) & 0x7f; - let output_width = (xs2 >> 8) & 0x7f; - let output_height = (xs2 >> 15) & 0x7f; - let output_stride = (xs2 >> 22) & 0x7f; - if iter == 0 - || iter % 4 != 0 - || output_width == 0 - || output_height == 0 - || output_stride < output_width - || output_width * output_height != iter / 4 - || output_base + (output_height - 1) * output_stride + output_width > bank_lines() as u64 - || input_base + iter > bank_lines() as u64 - { - panic!("quant_i32_to_i8: illegal encoding"); - } - iter + 4 -} diff --git a/examples/balls/toint8/emu/src/tests/activation_scale.rs b/examples/balls/toint8/emu/src/tests/activation_scale.rs deleted file mode 100644 index 60af3f030..000000000 --- a/examples/balls/toint8/emu/src/tests/activation_scale.rs +++ /dev/null @@ -1,11 +0,0 @@ -use super::super::model::{toint8_da_from_max_abs_bits, fp32_divide}; - -#[test] -fn computes_activation_scale_with_fp32_division() { - assert_eq!( - fp32_divide(2.0f32.to_bits(), 127.0f32.to_bits()), - 0x3c81_0204 - ); - assert_eq!(toint8_da_from_max_abs_bits(0), 1.0f32.to_bits()); - assert_eq!(toint8_da_from_max_abs_bits(63.5f32.to_bits()), 0.5f32.to_bits()); -} diff --git a/examples/balls/toint8/emu/src/tests/int32_rounding.rs b/examples/balls/toint8/emu/src/tests/int32_rounding.rs deleted file mode 100644 index c08859048..000000000 --- a/examples/balls/toint8/emu/src/tests/int32_rounding.rs +++ /dev/null @@ -1,21 +0,0 @@ -use super::super::model::toint8_i32_bits; - -#[test] -fn rounds_to_nearest_even() { - let scale = 0x3f80_0000; - let cases = [ - (0x3f80_0000, 1), - (0x4000_0000, 2), - (0xbf80_0000, -1), - (0x3f00_0000, 0), - (0xbf00_0000, 0), - (0x3fc0_0000, 2), - (0xbfc0_0000, -2), - (0x4020_0000, 2), - (0xc020_0000, -2), - ]; - - for (input, expected) in cases { - assert_eq!(toint8_i32_bits(input, scale), expected); - } -} diff --git a/examples/balls/toint8/emu/src/tests/int8_saturation.rs b/examples/balls/toint8/emu/src/tests/int8_saturation.rs deleted file mode 100644 index 47ac8554b..000000000 --- a/examples/balls/toint8/emu/src/tests/int8_saturation.rs +++ /dev/null @@ -1,9 +0,0 @@ -use super::super::model::toint8_i8_bits; - -#[test] -fn saturates_beyond_int8_range() { - let scale = 0x3f80_0000; - - assert_eq!(toint8_i8_bits(0x4300_0000, scale), 127); - assert_eq!(toint8_i8_bits(0xc300_0000, scale), -128); -} diff --git a/examples/balls/toint8/emu/src/tests/mod.rs b/examples/balls/toint8/emu/src/tests/mod.rs deleted file mode 100644 index 80ad99feb..000000000 --- a/examples/balls/toint8/emu/src/tests/mod.rs +++ /dev/null @@ -1 +0,0 @@ -mod quant; diff --git a/examples/balls/toint8/emu/src/tests/quant.rs b/examples/balls/toint8/emu/src/tests/quant.rs deleted file mode 100644 index da07539ed..000000000 --- a/examples/balls/toint8/emu/src/tests/quant.rs +++ /dev/null @@ -1,10 +0,0 @@ -use super::super::model::{quantize_f32, quantize_i32}; - -#[test] -fn rounds_ties_to_even_and_saturates() { - assert_eq!(quantize_f32(2.5, 1.0, false), 2); - assert_eq!(quantize_f32(3.5, 1.0, false), 4); - assert_eq!(quantize_i32(1000, 1.0, false), 127); - assert_eq!(quantize_i32(-1000, 1.0, false), -128); - assert_eq!(quantize_i32(-7, 1.0, true), 0); -} diff --git a/examples/balls/toint8/emu/src/tests/workload_vectors.rs b/examples/balls/toint8/emu/src/tests/workload_vectors.rs deleted file mode 100644 index dca0d4869..000000000 --- a/examples/balls/toint8/emu/src/tests/workload_vectors.rs +++ /dev/null @@ -1,19 +0,0 @@ -use super::super::model::toint8_i8_bits; - -#[test] -fn quantizes_workload_vectors() { - let scale = 2.0f32.to_bits(); - let input = [ - 0.125f32, -0.125, 0.25, -0.25, 0.75, -0.75, 1.25, -1.25, 1.75, -1.75, 63.25, 63.75, - -63.75, -64.75, 0.0, -0.0, 2.25, -2.25, 2.75, -2.75, 3.25, -3.25, 3.75, -3.75, 10.125, - -10.125, 20.25, -20.25, 0.375, -0.375, 64.25, -65.25, - ]; - let expected = [ - 0i8, 0, 0, 0, 2, -2, 2, -2, 4, -4, 126, 127, -128, -128, 0, 0, 4, -4, 6, -6, 6, -6, 8, - -8, 20, -20, 40, -40, 1, -1, 127, -128, - ]; - - for (value, expected) in input.into_iter().zip(expected) { - assert_eq!(toint8_i8_bits(value.to_bits(), scale), expected, "input={value}"); - } -} diff --git a/examples/balls/toint8/verify/README.md b/examples/balls/toint8/verify/README.md deleted file mode 100644 index 122824def..000000000 --- a/examples/balls/toint8/verify/README.md +++ /dev/null @@ -1,93 +0,0 @@ -# ToInt8Ball UVM Verification - -Verifies the generated `ToInt8Ball` module using the shared Blink UVM framework. - -## Structure - -- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM agents and base env -- `src/common/toint8_defs.svh`: DPI imports, `toint8_require_bid`, timeouts -- `src/common/toint8_items.svh`: `toint8_cmd_item` with `load_rust_case` (Da address in `special[12:0]`) -- `src/seq/toint8_sequences.svh`: one-case sequence -- `src/cov/toint8_cov.svh`: online INT8 layout and iter 1 coverage -- `src/env/toint8_scoreboard.svh`: preload mem model (group-aware for i8), compare writes vs DPI ref -- `src/env/toint8_env.svh`: extends `bb_blink_env#(1,1)` -- `src/tests/toint8_*_test.svh`: one directed case per UVM test -- `src/pkg/toint8_pkg.sv`: package entry -- `src/tb_top.sv`: `bb_blink_if#(1,1)` + `ToInt8Ball` -- `filelists/toint8_ball.f`: VCS filelist - -## Test plan - -The test names are: - -- `toint8_signed_test`: signed values across all four source groups -- `toint8_zero_test`: all-zero activation (`Da=1.0`) -- `toint8_rounding_test`: ties-to-even quantization boundaries -- `toint8_rows_test`: two-row scan and packed output traversal -- `toint8_scale_rows_test`: maximum value is in the second row - -Each case: reset, preload src words into `mem_model` (INT8 uses `group_id` 0..3), -drive one command, wait for `scb.done()` or timeout at 400 cycles. Scoreboard -compares observed writes with DPI `toint8_ref_i8`. `toint8_cov` requires the online -INT8 layout and one-row iteration bin, else `check_phase` fatal. - -## BID (required) - -`+BID=` is mandatory. Missing it fatals at test start. No default bid. - -| Config | Plusarg | -|--------|---------| -| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=3` | -| `sims.verilator.BuckyballToyVerilatorConfig` (full.toml) | `+BID=5` | - -```console -./simv +UVM_TESTNAME=toint8_scale_rows_test +BID=3 -``` - -## Build - -Enter the shared UVM/VCS environment: - -```console -nix develop ../../../../verify -``` - -Build the DPI reference library: - -```console -cargo build --manifest-path casegen/Cargo.toml -``` - -Compile from this directory: - -```console -vcs -full64 -sverilog -timescale=1ns/1ps \ - $VCS_UVM_ARGS \ - -sv_lib casegen/target/debug/libtoint8_casegen \ - -f filelists/toint8_ball.f -``` - -Run: - -```console -./simv +UVM_TESTNAME=toint8_scale_rows_test +BID=3 -``` - -Or via bbdev: - -```console -bbdev uvm --build '--ball=toint8' \ - --config sims.verilator.BuckyballPebbleVerilatorConfig \ - --core-config examples/cores/pebble/configs/default.toml -bbdev uvm --run '--ball=toint8 --plusargs +BID=3' -``` - -## Acceptance (2026-08-04) - -- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` -- [x] casegen reuses toint8 model; whole-case DPI API with injected bid -- [x] directed online INT8 scoreboard pass -- [x] signed, zero, rounding, and row-traversal directed cases pass -- [x] protocol + semantic cover enforced (`check_phase` fatal) -- [x] pebble: `+BID=3` config `sims.verilator.BuckyballPebbleVerilatorConfig` -- [ ] toy full: `+BID=5` config `sims.verilator.BuckyballToyVerilatorConfig` diff --git a/examples/balls/toint8/verify/casegen/src/casegen.rs b/examples/balls/toint8/verify/casegen/src/casegen.rs deleted file mode 100644 index 0491fc909..000000000 --- a/examples/balls/toint8/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,593 +0,0 @@ -use crate::model::{quantize_f32, quantize_i32}; - -pub const KIND_F32: u32 = 0; -pub const KIND_I32: u32 = 1; -pub const BANK_NUM: u32 = 24; -pub const BANK_ENTRIES: u32 = 64; -pub const SCALE_ROWS: usize = 4; -pub const LANES: usize = 4; -pub const MAX_SRC: usize = 16; -pub const MAX_SCALE: usize = 4; -pub const MAX_DST: usize = 4; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ToInt8Cmd { - pub kind: u32, - pub bid: u32, - pub iter: u32, - pub op1_bank: u32, - pub op2_bank: u32, - pub wr_bank: u32, - pub op1_col: u32, - pub op2_col: u32, - pub wr_col: u32, - pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, - pub rs2_lo: u32, - pub rs2_hi: u32, - pub input_base: u32, - pub num_src_words: u32, - pub num_scale_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ToInt8Case { - pub cmd: ToInt8Cmd, - pub src_words: [u128; MAX_SRC], - pub scale_words: [u128; MAX_SCALE], - pub dst_words: [u128; MAX_DST], - pub dst_addr: [u32; MAX_DST], -} - -impl ToInt8Case { - pub fn src_lo(&self, index: usize) -> u64 { - self.src_words[index] as u64 - } - - pub fn src_hi(&self, index: usize) -> u64 { - (self.src_words[index] >> 64) as u64 - } - - pub fn scale_lo(&self, index: usize) -> u64 { - self.scale_words[index] as u64 - } - - pub fn scale_hi(&self, index: usize) -> u64 { - (self.scale_words[index] >> 64) as u64 - } - - pub fn dst_lo(&self, index: usize) -> u64 { - self.dst_words[index] as u64 - } - - pub fn dst_hi(&self, index: usize) -> u64 { - (self.dst_words[index] >> 64) as u64 - } -} - -pub fn gen_case(index: u32, bid: u32) -> ToInt8Case { - match index { - 0 => f32_signed(bid), - 1 => f32_zero(bid), - 2 => f32_rounding(bid), - 3 => f32_rows(bid), - 4 => i32_scale_rows(bid), - 5 => i32_relu_tile(bid), - 6 => i32_signed(bid), - 7 => i32_offset(bid), - _ => panic!("toint8: unsupported directed case {index}"), - } -} - -fn pack_u32s(vals: &[u32]) -> Vec { - if vals.len() % LANES != 0 { - panic!("toint8: value count {} is not a multiple of four", vals.len()); - } - vals.chunks(LANES) - .map(|chunk| { - let mut word = 0u128; - for (lane, value) in chunk.iter().enumerate() { - word |= u128::from(*value) << (lane * 32); - } - word - }) - .collect() -} - -fn copy_words(src: &[u128]) -> [u128; N] { - if src.len() > N { - panic!("toint8: {} words exceed buffer {N}", src.len()); - } - let mut out = [0u128; N]; - out[..src.len()].copy_from_slice(src); - out -} - -fn copy_addrs(src: &[u32]) -> [u32; MAX_DST] { - if src.len() > MAX_DST { - panic!("toint8: {} dst rows exceed buffer", src.len()); - } - let mut out = [0u32; MAX_DST]; - out[..src.len()].copy_from_slice(src); - out -} - -fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { - u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) -} - -fn encode_i32_rs2( - relu: bool, - out_base: u32, - width: u32, - height: u32, - stride: u32, - in_base: u32, -) -> u64 { - u64::from(u32::from(relu)) - | (u64::from(out_base) << 1) - | (u64::from(width) << 8) - | (u64::from(height) << 15) - | (u64::from(stride) << 22) - | (u64::from(in_base) << 29) -} - -fn positive_finite(bits: u32) -> bool { - let value = f32::from_bits(bits); - value.is_finite() && value > 0.0 -} - -fn split_rs(value: u64) -> (u32, u32) { - (value as u32, (value >> 32) as u32) -} - -pub(crate) fn validate( - kind: u32, - op1_bank: u32, - op2_bank: u32, - wr_bank: u32, - op1_col: u32, - op2_col: u32, - wr_col: u32, - iter: u32, - rs1: u64, - rs2: u64, -) { - if kind != KIND_F32 && kind != KIND_I32 { - panic!("toint8: unknown kind {kind}"); - } - if iter == 0 || iter % 4 != 0 { - panic!("ToInt8Ball iter must be a positive multiple of four"); - } - if iter > BANK_ENTRIES { - panic!("ToInt8Ball input exceeds bank depth"); - } - if op1_bank >= BANK_NUM { - panic!("ToInt8Ball input bank is invalid"); - } - if wr_bank >= BANK_NUM { - panic!("ToInt8Ball output bank is invalid"); - } - if op1_col != 1 || wr_col != 1 { - panic!("ToInt8Ball input and output must each occupy one bank"); - } - if op1_bank == wr_bank { - panic!("ToInt8Ball input and output banks must differ"); - } - if (rs1 & 0x3ff) != u64::from(op1_bank) || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) { - panic!("toint8: rs1 bank fields mismatch"); - } - if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { - panic!("toint8: rs1 iter mismatch"); - } - if kind == KIND_F32 { - if ((rs1 >> 10) & 0x3ff) != 0 || op2_bank != 0 { - panic!("QUANT_F32_TO_I8 reserves input bank 1"); - } - if rs2 >> 32 != 0 { - panic!("QUANT_F32_TO_I8 reserves rs2[63:32]"); - } - if !positive_finite(rs2 as u32) { - panic!("QUANT_F32_TO_I8 scale must be finite and positive"); - } - } else { - if op2_bank >= BANK_NUM { - panic!("QUANT_I32_TO_I8 scale bank is invalid"); - } - if ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) { - panic!("toint8: rs1 scale bank mismatch"); - } - if op2_col != 1 { - panic!("QUANT_I32_TO_I8 scale must occupy one bank"); - } - if op1_bank == op2_bank { - panic!("QUANT_I32_TO_I8 input and scale banks must differ"); - } - if op2_bank == wr_bank { - panic!("QUANT_I32_TO_I8 scale and output banks must differ"); - } - if rs2 >> 35 != 0 { - panic!("QUANT_I32_TO_I8 reserves rs2[63:35]"); - } - let out_base = ((rs2 >> 1) & 0x7f) as u32; - let width = ((rs2 >> 8) & 0x7f) as u32; - let height = ((rs2 >> 15) & 0x7f) as u32; - let stride = ((rs2 >> 22) & 0x7f) as u32; - let in_base = ((rs2 >> 29) & 0x3f) as u32; - let output_rows = iter / 4; - if width == 0 || height == 0 || stride < width { - panic!("QUANT_I32_TO_I8 output geometry is invalid"); - } - if width.checked_mul(height) != Some(output_rows) { - panic!("QUANT_I32_TO_I8 iter does not match output geometry"); - } - let output_end = out_base - .checked_add((height - 1).checked_mul(stride).expect("stride overflow")) - .and_then(|value| value.checked_add(width)); - match output_end { - Some(end) if end <= BANK_ENTRIES => {} - _ => panic!("QUANT_I32_TO_I8 output exceeds bank depth"), - } - if in_base.checked_add(iter).is_none_or(|end| end > BANK_ENTRIES) { - panic!("QUANT_I32_TO_I8 input exceeds bank depth"); - } - } -} - -fn f32_dst(src: &[u128], scale: f32) -> Vec { - if src.len() % 4 != 0 { - panic!("QUANT_F32_TO_I8 src rows must be a multiple of four"); - } - src.chunks(4) - .map(|group| { - let mut packed = 0u128; - for (g, word) in group.iter().enumerate() { - for lane in 0..LANES { - let bits = (*word >> (lane * 32)) as u32; - if (bits >> 23) & 0xff == 0xff { - panic!("QUANT_F32_TO_I8 input must be finite"); - } - let q = quantize_f32(f32::from_bits(bits), scale, false) as u8; - packed |= u128::from(q) << ((g * LANES + lane) * 8); - } - } - packed - }) - .collect() -} - -fn i32_dst(src: &[u128], scales: &[u128], relu: bool) -> Vec { - if src.len() % 4 != 0 { - panic!("QUANT_I32_TO_I8 src rows must be a multiple of four"); - } - if scales.len() != SCALE_ROWS { - panic!("QUANT_I32_TO_I8 requires four scale rows"); - } - let mut scale_vals = [0.0f32; 16]; - for (row, word) in scales.iter().enumerate() { - for lane in 0..LANES { - let bits = (*word >> (lane * 32)) as u32; - if !positive_finite(bits) { - panic!("QUANT_I32_TO_I8 scales must be finite and positive"); - } - scale_vals[row * LANES + lane] = f32::from_bits(bits); - } - } - src.chunks(4) - .map(|group| { - let mut packed = 0u128; - for (g, word) in group.iter().enumerate() { - for lane in 0..LANES { - let bits = (*word >> (lane * 32)) as u32; - let channel = g * LANES + lane; - let q = quantize_i32(bits as i32, scale_vals[channel], relu) as u8; - packed |= u128::from(q) << (channel * 8); - } - } - packed - }) - .collect() -} - -fn i32_addrs(iter: u32, out_base: u32, width: u32, stride: u32) -> Vec { - let output_rows = iter / 4; - (0..output_rows) - .map(|row| out_base + (row / width) * stride + row % width) - .collect() -} - -fn finish( - kind: u32, - bid: u32, - iter: u32, - op1_bank: u32, - op2_bank: u32, - wr_bank: u32, - op2_col: u32, - rob_id: u32, - rs2: u64, - input_base: u32, - src: Vec, - scale: Vec, - dst: Vec, - dst_addr: Vec, -) -> ToInt8Case { - let op1_col = 1; - let wr_col = 1; - let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); - validate( - kind, op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs1, rs2, - ); - if src.len() != iter as usize { - panic!("toint8: src rows {} != iter {iter}", src.len()); - } - if dst.len() != (iter / 4) as usize || dst_addr.len() != dst.len() { - panic!("toint8: dst rows do not match iter/4"); - } - if kind == KIND_F32 { - if !scale.is_empty() { - panic!("QUANT_F32_TO_I8 must not emit scale rows"); - } - } else if scale.len() != SCALE_ROWS { - panic!("QUANT_I32_TO_I8 must emit four scale rows"); - } - let (rs1_lo, rs1_hi) = split_rs(rs1); - let (rs2_lo, rs2_hi) = split_rs(rs2); - ToInt8Case { - cmd: ToInt8Cmd { - kind, - bid, - iter, - op1_bank, - op2_bank, - wr_bank, - op1_col, - op2_col, - wr_col, - rob_id, - rs1_lo, - rs1_hi, - rs2_lo, - rs2_hi, - input_base, - num_src_words: src.len() as u32, - num_scale_words: scale.len() as u32, - num_dst_words: dst.len() as u32, - }, - src_words: copy_words(&src), - scale_words: copy_words(&scale), - dst_words: copy_words(&dst), - dst_addr: copy_addrs(&dst_addr), - } -} - -fn f32_case(bid: u32, rob_id: u32, scale: f32, values: &[f32]) -> ToInt8Case { - let bits: Vec = values.iter().map(|v| v.to_bits()).collect(); - let src = pack_u32s(&bits); - let iter = src.len() as u32; - let dst = f32_dst(&src, scale); - let dst_addr: Vec = (0..dst.len() as u32).collect(); - finish( - KIND_F32, - bid, - iter, - 0, - 0, - 1, - 0, - rob_id, - u64::from(scale.to_bits()), - 0, - src, - Vec::new(), - dst, - dst_addr, - ) -} - -fn i32_case( - bid: u32, - rob_id: u32, - op1_bank: u32, - op2_bank: u32, - wr_bank: u32, - relu: bool, - out_base: u32, - width: u32, - height: u32, - stride: u32, - in_base: u32, - values: &[i32], - scales: &[f32], -) -> ToInt8Case { - let src = pack_u32s(&values.iter().map(|v| *v as u32).collect::>()); - let scale = pack_u32s(&scales.iter().map(|v| v.to_bits()).collect::>()); - let iter = src.len() as u32; - let rs2 = encode_i32_rs2(relu, out_base, width, height, stride, in_base); - let dst = i32_dst(&src, &scale, relu); - let dst_addr = i32_addrs(iter, out_base, width, stride); - finish( - KIND_I32, - bid, - iter, - op1_bank, - op2_bank, - wr_bank, - 1, - rob_id, - rs2, - in_base, - src, - scale, - dst, - dst_addr, - ) -} - -fn f32_signed(bid: u32) -> ToInt8Case { - f32_case( - bid, - 2, - 0.5, - &[ - -300.0, -255.0, -5.0, -3.0, -1.0, 0.0, 1.0, 3.0, 5.0, 254.0, 255.0, 300.0, 2.0, 6.0, - 10.0, 14.0, - ], - ) -} - -fn f32_zero(bid: u32) -> ToInt8Case { - f32_case(bid, 3, 1.0, &[0.0; 16]) -} - -fn f32_rounding(bid: u32) -> ToInt8Case { - f32_case( - bid, - 4, - 1.0, - &[ - 2.5, -2.5, 3.5, -3.5, 0.5, -0.5, 1.5, -1.5, 63.5, -63.5, 127.5, -128.5, 0.0, 1.0, 2.0, - 3.0, - ], - ) -} - -fn f32_rows(bid: u32) -> ToInt8Case { - f32_case( - bid, - 5, - 2.0, - &[ - 0.125, -0.125, 0.25, -0.25, 0.75, -0.75, 1.25, -1.25, 1.75, -1.75, 63.25, 63.75, - -63.75, -64.75, 0.0, -0.0, 2.25, -2.25, 2.75, -2.75, 3.25, -3.25, 3.75, -3.75, 10.125, - -10.125, 20.25, -20.25, 0.375, -0.375, 64.25, -65.25, - ], - ) -} - -fn i32_scale_rows(bid: u32) -> ToInt8Case { - let mut values = [0i32; 16]; - let mut scales = [0.0f32; 16]; - for i in 0..16 { - values[i] = i as i32 - 8; - scales[i] = 0.5 + (i as f32) * 0.25; - } - i32_case(bid, 6, 0, 1, 2, false, 0, 1, 1, 1, 0, &values, &scales) -} - -fn i32_relu_tile(bid: u32) -> ToInt8Case { - let mut values = [0i32; 64]; - for position in 0..4 { - for channel in 0..16 { - values[position * 16 + channel] = position as i32 * 20 + channel as i32 - 8; - } - } - i32_case(bid, 7, 0, 1, 2, true, 3, 2, 2, 4, 0, &values, &[1.0; 16]) -} - -fn i32_signed(bid: u32) -> ToInt8Case { - let values: [i32; 16] = [ - -300, -128, -5, -1, 0, 1, 5, 127, 128, 200, -64, 64, -7, 7, 3, -3, - ]; - i32_case(bid, 8, 0, 1, 2, false, 0, 1, 1, 1, 0, &values, &[1.0; 16]) -} - -fn i32_offset(bid: u32) -> ToInt8Case { - let mut values = [0i32; 32]; - for i in 0..32 { - values[i] = i as i32 - 16; - } - i32_case(bid, 9, 0, 1, 2, true, 1, 2, 1, 2, 4, &values, &[0.5; 16]) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cases_cover_both_quant_kinds() { - let f32 = gen_case(0, 3); - let i32 = gen_case(5, 3); - assert_eq!(f32.cmd.kind, KIND_F32); - assert_eq!(i32.cmd.kind, KIND_I32); - assert_eq!(f32.cmd.iter % 4, 0); - assert_eq!(f32.cmd.op1_col, 1); - assert_eq!(f32.cmd.wr_col, 1); - assert_eq!(i32.cmd.op2_col, 1); - assert_eq!(i32.cmd.rs2_lo & 1, 1); - assert_eq!(i32.cmd.num_scale_words, 4); - assert_eq!(f32.cmd.num_scale_words, 0); - assert_eq!(f32.cmd.bid, 3); - } - - #[test] - fn former_indices_are_legal() { - for index in 0..8 { - let case = gen_case(index, 3); - assert!(case.cmd.kind == KIND_F32 || case.cmd.kind == KIND_I32); - assert_eq!(case.cmd.num_src_words, case.cmd.iter); - assert_eq!(case.cmd.num_dst_words, case.cmd.iter / 4); - } - } - - #[test] - fn f32_ctest_vector_saturates() { - let case = gen_case(0, 3); - let packed = case.dst_words[0]; - let bytes: [i8; 16] = [ - -128, -128, -2, -2, 0, 0, 0, 2, 2, 127, 127, 127, 1, 3, 5, 7, - ]; - for (i, expected) in bytes.iter().enumerate() { - assert_eq!(((packed >> (i * 8)) as u8) as i8, *expected); - } - } - - #[test] - #[should_panic(expected = "unsupported directed case")] - fn unknown_index_panics() { - let _ = gen_case(8, 3); - } - - #[test] - #[should_panic(expected = "positive multiple of four")] - fn illegal_iter_panics() { - validate(KIND_F32, 0, 0, 1, 1, 0, 1, 3, encode_rs1(0, 0, 1, 3), u64::from(1.0f32.to_bits())); - } - - #[test] - #[should_panic(expected = "reserves rs2[63:32]")] - fn f32_high_rs2_panics() { - validate( - KIND_F32, - 0, - 0, - 1, - 1, - 0, - 1, - 4, - encode_rs1(0, 0, 1, 4), - u64::from(1.0f32.to_bits()) | (1u64 << 32), - ); - } - - #[test] - #[should_panic(expected = "output geometry is invalid")] - fn i32_zero_width_panics() { - validate( - KIND_I32, - 0, - 1, - 2, - 1, - 1, - 1, - 4, - encode_rs1(0, 1, 2, 4), - encode_i32_rs2(false, 0, 0, 1, 1, 0), - ); - } -} diff --git a/examples/balls/toint8/verify/casegen/src/lib.rs b/examples/balls/toint8/verify/casegen/src/lib.rs deleted file mode 100644 index ead22d824..000000000 --- a/examples/balls/toint8/verify/casegen/src/lib.rs +++ /dev/null @@ -1,165 +0,0 @@ -mod casegen; -#[path = "../../../emu/src/model.rs"] -mod model; - -use std::cell::RefCell; - -use casegen::{ToInt8Case, ToInt8Cmd, MAX_DST, MAX_SCALE, MAX_SRC}; - -thread_local! { - static CURRENT: RefCell> = const { RefCell::new(None) }; -} - -#[no_mangle] -pub extern "C" fn toint8_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid); - CURRENT.with(|current| *current.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|current| match current.borrow().as_ref() { - Some(case) => f(case), - None => panic!("toint8_case: no case loaded; call toint8_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn toint8_case_cmd(out_ptr: *mut ToInt8Cmd) { - if out_ptr.is_null() { - panic!("toint8_case_cmd: null out_ptr"); - } - current_case(|case| unsafe { - *out_ptr = case.cmd; - }); -} - -fn src_index(index: u32, limit: u32, what: &str) -> usize { - if index >= limit { - panic!("toint8_case_{what}: index {index} out of range {limit}"); - } - index as usize -} - -#[no_mangle] -pub extern "C" fn toint8_case_src_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_src_words, "src_word_lo"); - if i >= MAX_SRC { - panic!("toint8_case_src_word_lo: word_index out of range"); - } - case.src_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn toint8_case_src_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_src_words, "src_word_hi"); - if i >= MAX_SRC { - panic!("toint8_case_src_word_hi: word_index out of range"); - } - case.src_hi(i) - }) -} - -#[no_mangle] -pub extern "C" fn toint8_case_scale_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_lo"); - if i >= MAX_SCALE { - panic!("toint8_case_scale_word_lo: word_index out of range"); - } - case.scale_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn toint8_case_scale_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_hi"); - if i >= MAX_SCALE { - panic!("toint8_case_scale_word_hi: word_index out of range"); - } - case.scale_hi(i) - }) -} - -#[no_mangle] -pub extern "C" fn toint8_case_dst_word_lo(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_lo"); - if i >= MAX_DST { - panic!("toint8_case_dst_word_lo: word_index out of range"); - } - case.dst_lo(i) - }) -} - -#[no_mangle] -pub extern "C" fn toint8_case_dst_word_hi(word_index: u32) -> u64 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_hi"); - if i >= MAX_DST { - panic!("toint8_case_dst_word_hi: word_index out of range"); - } - case.dst_hi(i) - }) -} - -#[no_mangle] -pub extern "C" fn toint8_case_dst_addr(word_index: u32) -> u32 { - current_case(|case| { - let i = src_index(word_index, case.cmd.num_dst_words, "dst_addr"); - if i >= MAX_DST { - panic!("toint8_case_dst_addr: word_index out of range"); - } - case.dst_addr[i] - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dpi_load_then_cmd_and_words() { - assert_eq!(toint8_case_load(0, 3), 0); - let mut cmd = ToInt8Cmd { - kind: 99, - bid: 0, - iter: 0, - op1_bank: 0, - op2_bank: 0, - wr_bank: 0, - op1_col: 0, - op2_col: 0, - wr_col: 0, - rob_id: 0, - rs1_lo: 0, - rs1_hi: 0, - rs2_lo: 0, - rs2_hi: 0, - input_base: 0, - num_src_words: 0, - num_scale_words: 0, - num_dst_words: 0, - }; - toint8_case_cmd(&mut cmd as *mut ToInt8Cmd); - assert_eq!(cmd.bid, 3); - assert_eq!(cmd.kind, casegen::KIND_F32); - assert_eq!(cmd.iter, 4); - assert_eq!(cmd.op1_col, 1); - assert_eq!(cmd.num_src_words, 4); - let _lo = toint8_case_src_word_lo(0); - let _hi = toint8_case_src_word_hi(0); - let _addr = toint8_case_dst_addr(0); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/toint8/verify/src/tb_top.sv b/examples/balls/toint8/verify/src/tb_top.sv deleted file mode 100644 index 402dae56a..000000000 --- a/examples/balls/toint8/verify/src/tb_top.sv +++ /dev/null @@ -1,90 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import toint8_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - ToInt8Ball dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_bankRead_0_ball_id(), - .io_bankWrite_0_ball_id() - ); - - initial begin - intf.clock = 1'b0; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("toint8_ball_test"); - end -endmodule diff --git a/examples/balls/toint8/workloads/ctests/CMakeLists.txt b/examples/balls/toint8/workloads/ctests/CMakeLists.txt deleted file mode 100644 index f9035c051..000000000 --- a/examples/balls/toint8/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_buckyball_ctests( - quant_f32_to_i8_test.c - quant_i32_to_i8_test.c - toint8_i8_rounding_test.c - toint8_i8_rows_test.c - toint8_i8_scale_rows_test.c - toint8_i8_signed_test.c -) diff --git a/examples/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir b/examples/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir deleted file mode 100644 index 1142af638..000000000 --- a/examples/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir +++ /dev/null @@ -1,30 +0,0 @@ -func.func private @check_result(memref<1x16xi8>) -> () - -memref.global "private" constant @input : memref<4x4xf32> = dense<[ - [-3.000000e+02, -2.550000e+02, -5.000000e+00, -3.000000e+00], - [-1.000000e+00, 0.000000e+00, 1.000000e+00, 3.000000e+00], - [5.000000e+00, 2.540000e+02, 2.550000e+02, 3.000000e+02], - [2.000000e+00, 6.000000e+00, 1.000000e+01, 1.400000e+01] -]> - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %depth4 = arith.constant 4 : i64 - %depth1 = arith.constant 1 : i64 - %stride = arith.constant 1 : i64 - %scale = arith.constant 5.000000e-01 : f32 - %input = memref.get_global @input : memref<4x4xf32> - %output = memref.alloc() alignment = 64 : memref<1x16xi8> - %input_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %input_bank %depth4 %stride - : memref<4x4xf32> i64 i64 i64 - buckyball.quant_f32_to_i8 %loaded, %output_bank, %depth4, %scale : i64 - %stored = buckyball.bank_mvout %output %output_bank %depth1 %stride - : memref<1x16xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<1x16xi8>) -> () - buckyball.bank_release %loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %output : memref<1x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir b/examples/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir deleted file mode 100644 index 7b65d1f11..000000000 --- a/examples/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir +++ /dev/null @@ -1,62 +0,0 @@ -func.func private @check_result(memref<12x16xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %one_f32 = arith.constant 1.0 : f32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %twenty = arith.constant 20 : index - %eight = arith.constant 8 : index - %depth16 = arith.constant 16 : i64 - %depth4 = arith.constant 4 : i64 - %depth12 = arith.constant 12 : i64 - %output_base = arith.constant 3 : i64 - %zero64 = arith.constant 0 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() alignment = 64 : memref<16x4xi32> - %scales = memref.alloc() alignment = 64 : memref<4x4xf32> - %output_seed = memref.alloc() alignment = 64 : memref<12x16xi8> - %output = memref.alloc() alignment = 64 : memref<12x16xi8> - linalg.fill ins(%one_f32 : f32) outs(%scales : memref<4x4xf32>) - linalg.fill ins(%zero_i8 : i8) outs(%output_seed : memref<12x16xi8>) - linalg.fill ins(%zero_i8 : i8) outs(%output : memref<12x16xi8>) - scf.for %position = %zero to %four step %one { - scf.for %channel = %zero to %sixteen step %one { - %position_base = arith.muli %position, %twenty : index - %with_channel = arith.addi %position_base, %channel : index - %value = arith.subi %with_channel, %eight : index - %value_i32 = arith.index_cast %value : index to i32 - %channel_row = arith.divui %channel, %four : index - %row_base = arith.muli %position, %four : index - %row = arith.addi %row_base, %channel_row : index - %lane = arith.remui %channel, %four : index - memref.store %value_i32, %input[%row, %lane] : memref<16x4xi32> - } - } - %input_bank = buckyball.bank_alloc - %scale_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth16 %stride - : memref<16x4xi32> i64 i64 i64 - %loaded_scales = buckyball.bank_mvin %scales %scale_bank %depth4 %stride - : memref<4x4xf32> i64 i64 i64 - %loaded_output = buckyball.bank_mvin %output_seed %output_bank %depth12 %stride - : memref<12x16xi8> i64 i64 i64 - buckyball.quant_i32_to_i8 - %loaded_input, %loaded_scales, %loaded_output, %depth16, %output_base, %zero64 - : i64 i64 i64 - %stored = buckyball.bank_mvout %output %loaded_output %depth12 %stride - : memref<12x16xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<12x16xi8>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_scales : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<16x4xi32> - memref.dealloc %scales : memref<4x4xf32> - memref.dealloc %output_seed : memref<12x16xi8> - memref.dealloc %output : memref<12x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt b/examples/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index 653a5c340..000000000 --- a/examples/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) - -buckyball_require_quant_i32_bank_shape(${BUCKYBALL_MLIR_ACTIVE_TARGET}) -add_buckyball_mlir_test(quant_i32_pack16 TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) -add_buckyball_mlir_test(quant_f32_pack16 TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in b/examples/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in deleted file mode 100644 index 118f0d8c6..000000000 --- a/examples/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in +++ /dev/null @@ -1,39 +0,0 @@ -func.func private @check_result(memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %rows = arith.constant @BB_QUANT_I32_OUTPUT_ROWS@ : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %out_depth = arith.constant @BB_QUANT_I32_OUTPUT_ROWS@ : i64 - %stride = arith.constant 1 : i64 - %scale = arith.constant 1.0 : f32 - %input = memref.alloc() : memref<@BB_BANK_LINES@x4xf32> - %output = memref.alloc() : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> - scf.for %position = %zero to %rows step %one { - scf.for %channel = %zero to %sixteen step %one { - %base = arith.muli %position, %sixteen : index - %linear = arith.addi %base, %channel : index - %value_i32 = arith.index_cast %linear : index to i32 - %value = arith.sitofp %value_i32 : i32 to f32 - %row_base = arith.muli %position, %four : index - %channel_row = arith.divui %channel, %four : index - %row = arith.addi %row_base, %channel_row : index - %lane = arith.remui %channel, %four : index - memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x4xf32> - } - } - %input_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride : memref<@BB_BANK_LINES@x4xf32> i64 i64 i64 - %result = buckyball.bank_quant_f32_to_i8 %loaded_input %output_bank %depth %scale : i64 i64 i64 f32 - %stored = buckyball.bank_mvout %output %result %out_depth %stride : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<@BB_BANK_LINES@x4xf32> - memref.dealloc %output : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in b/examples/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in deleted file mode 100644 index cffa6f5c8..000000000 --- a/examples/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in +++ /dev/null @@ -1,50 +0,0 @@ -func.func private @check_result(memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %one_f32 = arith.constant 1.0 : f32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %sixteen = arith.constant 16 : index - %rows = arith.constant @BB_QUANT_I32_OUTPUT_ROWS@ : index - %depth = arith.constant @BB_QUANT_I32_INPUT_ROWS@ : i64 - %scale_depth = arith.constant 4 : i64 - %out_depth = arith.constant @BB_QUANT_I32_OUTPUT_ROWS@ : i64 - %zero64 = arith.constant 0 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> - %scale = memref.alloc() : memref<4x4xf32> - %seed = memref.alloc() : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> - %output = memref.alloc() : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> - linalg.fill ins(%one_f32 : f32) outs(%scale : memref<4x4xf32>) - linalg.fill ins(%zero_i8 : i8) outs(%seed : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) - scf.for %position = %zero to %rows step %one { - scf.for %channel = %zero to %sixteen step %one { - %base = arith.muli %position, %sixteen : index - %linear = arith.addi %base, %channel : index - %value = arith.index_cast %linear : index to i32 - %row_base = arith.muli %position, %four : index - %channel_row = arith.divui %channel, %four : index - %row = arith.addi %row_base, %channel_row : index - %lane = arith.remui %channel, %four : index - memref.store %value, %input[%row, %lane] : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> - } - } - %input_bank = buckyball.bank_alloc - %scale_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded_input = buckyball.bank_mvin %input %input_bank %depth %stride : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> i64 i64 i64 - %loaded_scale = buckyball.bank_mvin %scale %scale_bank %scale_depth %stride : memref<4x4xf32> i64 i64 i64 - %loaded_output = buckyball.bank_mvin %seed %output_bank %out_depth %stride : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - %result = buckyball.bank_quant_i32_to_i8 %loaded_input %loaded_scale %loaded_output %depth %zero64 %zero64 : i64 i64 i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %out_depth %stride : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %loaded_input : i64 - buckyball.bank_release %loaded_scale : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> - memref.dealloc %scale : memref<4x4xf32> - memref.dealloc %seed : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> - memref.dealloc %output : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/trace/arch/src/main/scala/Trace.scala b/examples/balls/trace/arch/src/main/scala/Trace.scala deleted file mode 100644 index 823615095..000000000 --- a/examples/balls/trace/arch/src/main/scala/Trace.scala +++ /dev/null @@ -1,200 +0,0 @@ -package examples.balls.trace - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.top.GlobalConfig - -/** - * Trace — TraceBall inner processing unit. - * - * Handles cycle counter management (START/STOP/READ). - */ -@instantiable -class Trace(val b: GlobalConfig) extends Module { - - val ballMapping = b.ballDomain.ballIdMappings - .find(_.ballName == "TraceBall") - .getOrElse( - throw new IllegalArgumentException("TraceBall not found in config") - ) - - val inBW = ballMapping.inBW - val outBW = ballMapping.outBW - - val bankWidth = b.memDomain.bankWidth - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(outBW, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - // ============================================================ - // Constants - // ============================================================ - val CTR_START = 0.U(4.W) - val CTR_STOP = 1.U(4.W) - val CTR_READ = 2.U(4.W) - - val NUM_COUNTERS = 16 - - // ============================================================ - // State machine - // ============================================================ - val idle :: sCounter :: complete :: Nil = Enum(3) - val state = RegInit(idle) - - // ============================================================ - // Registers - // ============================================================ - val rob_id_reg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val is_sub_reg = RegInit(false.B) - val sub_rob_id_reg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - - // Counter-specific registers - val subcmd_reg = RegInit(0.U(4.W)) - val ctr_id_reg = RegInit(0.U(4.W)) - val payload_reg = RegInit(0.U(56.W)) - - // Cycle counters: 16 independent counters - val cycleCounter = RegInit(0.U(64.W)) - cycleCounter := cycleCounter + 1.U - - val ctrStartCycle = RegInit(VecInit(Seq.fill(NUM_COUNTERS)(0.U(64.W)))) - val ctrTag = RegInit(VecInit(Seq.fill(NUM_COUNTERS)(0.U(56.W)))) - val ctrActive = RegInit(VecInit(Seq.fill(NUM_COUNTERS)(false.B))) - - // ============================================================ - // DPI-C modules - // ============================================================ - val ctraceDpi = Module(new CTraceDPI) - ctraceDpi.io.clock := clock - ctraceDpi.io.reset := reset.asBool - ctraceDpi.io.subcmd := 0.U - ctraceDpi.io.ctr_id := 0.U - ctraceDpi.io.tag := 0.U - ctraceDpi.io.elapsed := 0.U - ctraceDpi.io.cycle := cycleCounter - ctraceDpi.io.enable := false.B - - // ============================================================ - // External bank port defaults - // ============================================================ - for (i <- 0 until inBW) { - io.bankRead(i).rob_id := rob_id_reg - io.bankRead(i).ball_id := 0.U - io.bankRead(i).bank_id := 0.U - io.bankRead(i).group_id := 0.U - io.bankRead(i).io.req.valid := false.B - io.bankRead(i).io.req.bits.addr := 0.U - io.bankRead(i).io.resp.ready := false.B - } - - for (i <- 0 until outBW) { - io.bankWrite(i).rob_id := rob_id_reg - io.bankWrite(i).ball_id := 0.U - io.bankWrite(i).bank_id := 0.U - io.bankWrite(i).group_id := 0.U - io.bankWrite(i).io.req.valid := false.B - io.bankWrite(i).io.req.bits.addr := 0.U - io.bankWrite(i).io.req.bits.data := 0.U - io.bankWrite(i).io.req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(0.U(1.W)) - ) - io.bankWrite(i).io.resp.ready := false.B - } - - // ============================================================ - // Command interface - // ============================================================ - io.cmdReq.ready := state === idle - io.cmdResp.valid := false.B - io.cmdResp.bits.rob_id := rob_id_reg - io.cmdResp.bits.is_sub := is_sub_reg - io.cmdResp.bits.sub_rob_id := sub_rob_id_reg - - // ============================================================ - // State machine - // ============================================================ - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - rob_id_reg := io.cmdReq.bits.rob_id - is_sub_reg := io.cmdReq.bits.is_sub - sub_rob_id_reg := io.cmdReq.bits.sub_rob_id - - val cmd = io.cmdReq.bits.cmd - val rs2 = cmd.rs2 - - // Counter fields from rs2 - subcmd_reg := rs2(3, 0) - ctr_id_reg := rs2(7, 4) - payload_reg := rs2(63, 8) - - state := sCounter - } - } - - // ---------------------------------------------------------- - // Counter instruction: execute and complete in 1 cycle - // ---------------------------------------------------------- - is(sCounter) { - val cid = ctr_id_reg - - ctraceDpi.io.subcmd := subcmd_reg - ctraceDpi.io.ctr_id := cid - - switch(subcmd_reg) { - is(CTR_START) { - ctrStartCycle(cid) := cycleCounter - ctrTag(cid) := payload_reg - ctrActive(cid) := true.B - - ctraceDpi.io.tag := payload_reg - ctraceDpi.io.elapsed := 0.U - ctraceDpi.io.enable := true.B - } - is(CTR_STOP) { - val elapsed = cycleCounter - ctrStartCycle(cid) - ctrActive(cid) := false.B - - ctraceDpi.io.tag := ctrTag(cid) - ctraceDpi.io.elapsed := elapsed - ctraceDpi.io.enable := true.B - } - is(CTR_READ) { - val current = cycleCounter - ctrStartCycle(cid) - - ctraceDpi.io.tag := ctrTag(cid) - ctraceDpi.io.elapsed := current - ctraceDpi.io.enable := true.B - } - } - - state := complete - } - - // ---------------------------------------------------------- - // Complete: fire cmdResp - // ---------------------------------------------------------- - is(complete) { - io.cmdResp.valid := true.B - io.cmdResp.bits.rob_id := rob_id_reg - when(io.cmdResp.fire) { - state := idle - } - } - } - - // ============================================================ - // Status - // ============================================================ - io.status.idle := state === idle - io.status.running := state =/= idle && state =/= complete -} diff --git a/examples/balls/trace/arch/src/main/scala/TraceBall.scala b/examples/balls/trace/arch/src/main/scala/TraceBall.scala deleted file mode 100644 index 53c48cf51..000000000 --- a/examples/balls/trace/arch/src/main/scala/TraceBall.scala +++ /dev/null @@ -1,53 +0,0 @@ -package examples.balls.trace - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BlinkIO, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.MmioRead -import framework.top.GlobalConfig - -/** - * TraceBall — Debug trace Ball providing cycle counters. - * - * The Core ballISA assigns the funct7 encoding for bdb_counter. - */ -@instantiable -class TraceBall(val b: GlobalConfig) extends Module with HasBlink { - - val ballCommonConfig = b.ballDomain.ballIdMappings - .find(_.ballName == "TraceBall") - .getOrElse( - throw new IllegalArgumentException("TraceBall not found in config") - ) - - val inBW = ballCommonConfig.inBW - val outBW = ballCommonConfig.outBW - - @public - val io = IO(new BlinkIO(b, inBW, outBW)) - - def blink: BlinkIO = io - dontTouch(io) - - val traceUnit: Instance[Trace] = Instantiate(new Trace(b)) - - traceUnit.io.cmdReq <> io.cmdReq - traceUnit.io.cmdResp <> io.cmdResp - - for (i <- 0 until inBW) { - traceUnit.io.bankRead(i) <> io.bankRead(i) - } - - for (i <- 0 until outBW) { - traceUnit.io.bankWrite(i) <> io.bankWrite(i) - } - - io.status <> traceUnit.io.status - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - - // MMIO: this Ball does not consume MMIO metadata - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/trace/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/trace/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 475ecd5ab..000000000 --- a/examples/balls/trace/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -class BdbCounterLowering : public ConvertOpToLLVMPattern { -public: - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(BdbCounterOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("TraceBall"); - Location loc = op.getLoc(); - Value counter = rewriter.create(loc, adaptor.getCounterId(), - cstI64(rewriter, loc, 4)); - Value payload = rewriter.create(loc, adaptor.getPayload(), - cstI64(rewriter, loc, 8)); - Value rs2 = - rewriter.create(loc, adaptor.getSubcommand(), counter); - rs2 = rewriter.create(loc, rs2, payload); - rewriter.replaceOpWithNewOp( - op, cstI64(rewriter, loc, 0), rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("BDB_COUNTER"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateTraceBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - (void)stable; - (void)bankDepth; - - patterns.add(converter); -} - -void configureTraceBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/trace/configs/default.toml b/examples/balls/trace/configs/default.toml deleted file mode 100644 index 6f2c952cd..000000000 --- a/examples/balls/trace/configs/default.toml +++ /dev/null @@ -1,3 +0,0 @@ -# Trace Ball Configuration - -[ball] diff --git a/examples/balls/trace/emu/src/04_bdb_counter.rs b/examples/balls/trace/emu/src/04_bdb_counter.rs deleted file mode 100644 index 4fd1f9437..000000000 --- a/examples/balls/trace/emu/src/04_bdb_counter.rs +++ /dev/null @@ -1,15 +0,0 @@ -//===- 04_bdb_counter.rs - BDB_COUNTER instruction -------------------------===// - -use super::instruction::{BallInstruction, ExecContext}; - -pub struct BdbCounter; - -impl BallInstruction for BdbCounter { - fn exec(_xs1: u64, _xs2: u64, _ctx: &mut ExecContext) -> u64 { - 0 - } - - fn latency(_xs1: u64, _xs2: u64) -> u64 { - 1 - } -} diff --git a/examples/balls/trace/emu/src/dispatch.rs b/examples/balls/trace/emu/src/dispatch.rs deleted file mode 100644 index a50ec9e07..000000000 --- a/examples/balls/trace/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::f04_bdb_counter; - -const BALL_CLASS: &str = "examples.balls.trace.TraceBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("BDB_COUNTER") => Some(f04_bdb_counter::BdbCounter::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("BDB_COUNTER") => Some(f04_bdb_counter::BdbCounter::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/trace/emu/src/lib.rs b/examples/balls/trace/emu/src/lib.rs deleted file mode 100644 index fe1e94f18..000000000 --- a/examples/balls/trace/emu/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub(crate) use crate::inst::instruction; - -#[path = "04_bdb_counter.rs"] -mod f04_bdb_counter; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/trace/workloads/ctests/CMakeLists.txt b/examples/balls/trace/workloads/ctests/CMakeLists.txt deleted file mode 100644 index fbfa8b478..000000000 --- a/examples/balls/trace/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -add_buckyball_ctests( - bdb_counter_start_stop_test.c - bdb_counter_read_active_test.c - bdb_counter_nested_pair_test.c - bdb_counter_multi_id_test.c -) diff --git a/examples/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir b/examples/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir deleted file mode 100644 index 923de4009..000000000 --- a/examples/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir +++ /dev/null @@ -1,12 +0,0 @@ -func.func @main() -> i8 { - %zero8 = arith.constant 0 : i8 - %start = arith.constant 0 : i64 - %stop = arith.constant 1 : i64 - %read = arith.constant 2 : i64 - %counter = arith.constant 3 : i64 - %tag = arith.constant 48879 : i64 - buckyball.bdb_counter %start, %counter, %tag : i64 - buckyball.bdb_counter %read, %counter, %start : i64 - buckyball.bdb_counter %stop, %counter, %start : i64 - return %zero8 : i8 -} diff --git a/examples/balls/transpose/arch/src/main/scala/README.md b/examples/balls/transpose/arch/src/main/scala/README.md deleted file mode 100644 index 3c01b53d7..000000000 --- a/examples/balls/transpose/arch/src/main/scala/README.md +++ /dev/null @@ -1,115 +0,0 @@ -# Matrix Transpose Accelerator - -## Overview - -This directory implements Buckyball's matrix transpose accelerator for matrix transpose operations. Located at `arch/src/main/scala/prototype/transpose`, it serves as a matrix transpose accelerator supporting pipelined transpose operations. - -Core components: -- **Transpose.scala**: Pipelined transposer implementation - -## Code Structure - -``` -transpose/ -└── Transpose.scala - Pipelined transposer -``` - -### Module Responsibilities - -**Transpose.scala** (Transpose implementation layer) -- Implements PipelinedTransposer module -- Manages matrix data read, transpose, and write-back -- Provides Ball domain command interface - -## Module Description - -### Transpose.scala - -**Main functionality**: Implements pipelined matrix transpose operation - -**State machine definition**: -```scala -val idle :: sRead :: sWrite :: complete :: Nil = Enum(4) -val state = RegInit(idle) -``` - -**Storage structure**: -```scala -// Matrix storage register (veclane x veclane) -val regArray = Reg(Vec(b.veclane, Vec(b.veclane, UInt(b.inputType.getWidth.W)))) -``` - -**Counter management**: -```scala -val readCounter = RegInit(0.U(log2Ceil(b.veclane + 1).W)) -val respCounter = RegInit(0.U(log2Ceil(b.veclane + 1).W)) -val writeCounter = RegInit(0.U(log2Ceil(b.veclane + 1).W)) -``` - -**Instruction registers**: -```scala -val robid_reg = RegInit(0.U(10.W)) // ROB ID -val waddr_reg = RegInit(0.U(10.W)) // Write address -val wbank_reg = RegInit(0.U(log2Up(b.sp_banks).W)) // Write bank -val raddr_reg = RegInit(0.U(10.W)) // Read address -val rbank_reg = RegInit(0.U(log2Up(b.sp_banks).W)) // Read bank -val iter_reg = RegInit(0.U(10.W)) // Iteration count -``` - -**Interface definition**: -```scala -val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue)) - val cmdResp = Decoupled(new BallRsComplete) - val sramRead = Vec(b.sp_banks, Flipped(new SramReadIO(...))) - val sramWrite = Vec(b.sp_banks, Flipped(new SramWriteIO(...))) -}) -``` - -**Processing flow**: -1. **idle**: Wait for command, parse transpose parameters -2. **sRead**: Read matrix data row by row into register array -3. **sWrite**: Write transposed data column by column -4. **complete**: Send completion signal - -**Transpose algorithm**: -- Uses veclane×veclane register array to store matrix -- Reads row-wise, writes column-wise to implement transpose -- Supports block-wise transpose for matrices of arbitrary size - -## Usage - -### Implementation Details - -**State machine**: -```scala -val idle :: sRead :: sWrite :: complete :: Nil = Enum(4) -``` -- `idle`: Wait for instruction -- `sRead`: Read matrix data -- `sWrite`: Write transpose result -- `complete`: Complete and respond - -**Register array**: -```scala -val regArray = Reg(Vec(b.veclane, Vec(b.veclane, UInt(b.inputType.getWidth.W)))) -``` -Uses veclane×veclane register array to cache matrix data. - -**Transpose operation**: -- Read phase: Read data row by row into `regArray(row)(col)` -- Write phase: Read `regArray(i)(col)` column by column to form new rows for writing - -### Configuration Parameters - -**Matrix size**: Determined by b.veclane parameter -**Data width**: Determined by b.inputType.getWidth -**Bank configuration**: Supports multi-bank SRAM access - -### Notes - -1. **Matrix size limitation**: Maximum support for veclane×veclane matrices -2. **Memory bandwidth**: Transpose operation has high memory bandwidth requirements -3. **Register overhead**: Requires veclane² registers to store matrix -4. **Address calculation**: Transposed address calculation needs to be handled correctly -5. **Pipeline control**: Read/write counters need to be synchronized correctly diff --git a/examples/balls/transpose/arch/src/main/scala/Transpose.scala b/examples/balls/transpose/arch/src/main/scala/Transpose.scala deleted file mode 100644 index b4a6230dc..000000000 --- a/examples/balls/transpose/arch/src/main/scala/Transpose.scala +++ /dev/null @@ -1,260 +0,0 @@ -package examples.balls.transpose - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.top.GlobalConfig -import examples.balls.transpose.configs.TransposeBallParam - -@instantiable -class Transpose(val b: GlobalConfig) extends Module { - val ballConfig = TransposeBallParam(b) - val bankWidth = b.memDomain.bankWidth - val rowBytes = bankWidth / 8 - - val ballMapping = b.ballDomain.ballIdMappings - .find(_.ballName == "TransposeBall") - .getOrElse( - throw new IllegalArgumentException("TransposeBall not found in config") - ) - - val inBW = ballMapping.inBW - val outBW = ballMapping.outBW - require(inBW == outBW, "TransposeBall requires inBW == outBW") - require(inBW == 1, "TransposeBall gather/scatter path requires inBW == 1") - require(bankWidth % 8 == 0, "bankWidth must be byte-aligned") - require( - ballConfig.InputNum * ballConfig.inputWidth == bankWidth, - "TransposeBall InputNum*inputWidth must equal bankWidth" - ) - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(outBW, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - val rob_id_reg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val is_sub_reg = RegInit(false.B) - val sub_rob_id_reg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - - val rbank_reg = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val wbank_reg = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val ncol_reg = RegInit(0.U(log2Up(b.memDomain.bankNum + 1).W)) - val iter_reg = RegInit(0.U(b.frontend.iter_len.W)) - val elem_reg = RegInit(0.U(8.W)) - - val idle :: sRead :: sWrite :: complete :: Nil = Enum(4) - val state = RegInit(idle) - - // Walk destination dense index 0 .. iter*W-1, filling write beats. - val pending = RegInit(false.B) - val wrBytes = Reg(Vec(rowBytes, UInt(8.W))) - val wrMask = RegInit(VecInit(Seq.fill(b.memDomain.bankMaskLen)(0.U(1.W)))) - val wrGroup = RegInit(0.U(log2Up(b.memDomain.bankNum + 1).W)) - val wrAddr = RegInit(0.U(b.frontend.iter_len.W)) - val wrValid = RegInit(false.B) - val beatFill = RegInit(0.U(8.W)) - - val cached = RegInit(false.B) - val cacheAddr = RegInit(0.U(b.frontend.iter_len.W)) - val cacheGroup = RegInit(0.U(log2Up(b.memDomain.bankNum + 1).W)) - val cacheData = RegInit(0.U(bankWidth.W)) - - val srcR = RegInit(0.U(b.frontend.iter_len.W)) - val srcC = RegInit(0.U(32.W)) - val dstVirtRow = RegInit(0.U(b.frontend.iter_len.W)) - val dstVirtCol = RegInit(0.U(32.W)) - val epgReg = RegInit(0.U(8.W)) - val wElemsReg = RegInit(0.U(32.W)) - val elemBytesReg = RegInit(0.U(8.W)) - val lastBeat = RegInit(false.B) - - for (i <- 0 until inBW) { - io.bankRead(i).rob_id := rob_id_reg - io.bankRead(i).ball_id := 0.U - io.bankRead(i).bank_id := rbank_reg - io.bankRead(i).group_id := 0.U - io.bankRead(i).io.req.valid := false.B - io.bankRead(i).io.req.bits.addr := 0.U - io.bankRead(i).io.resp.ready := false.B - } - for (i <- 0 until outBW) { - io.bankWrite(i).rob_id := rob_id_reg - io.bankWrite(i).ball_id := 0.U - io.bankWrite(i).bank_id := wbank_reg - io.bankWrite(i).group_id := 0.U - io.bankWrite(i).io.req.valid := false.B - io.bankWrite(i).io.req.bits.addr := 0.U - io.bankWrite(i).io.req.bits.data := 0.U - io.bankWrite(i).io.req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(0.U(1.W)) - ) - io.bankWrite(i).io.resp.ready := (state =/= idle) - } - - io.cmdReq.ready := (state === idle) - io.cmdResp.valid := false.B - io.cmdResp.bits.rob_id := rob_id_reg - io.cmdResp.bits.is_sub := is_sub_reg - io.cmdResp.bits.sub_rob_id := sub_rob_id_reg - - require(isPow2(rowBytes), "Transpose rowBytes must be power of 2") - - val laneBitsI8 = log2Ceil(rowBytes) - val laneBitsI32 = log2Ceil(rowBytes / 4) - - val isI8 = elemBytesReg === 1.U - val srcLane = Mux(isI8, srcC(laneBitsI8 - 1, 0), srcC(laneBitsI32 - 1, 0)) - val srcGroup = Mux(isI8, srcC >> laneBitsI8.U, srcC >> laneBitsI32.U) - .asTypeOf(UInt(log2Up(b.memDomain.bankNum + 1).W)) - val srcAddr = srcR - - val dstGroup = Mux(isI8, dstVirtCol >> laneBitsI8.U, dstVirtCol >> laneBitsI32.U) - .asTypeOf(UInt(log2Up(b.memDomain.bankNum + 1).W)) - val bytes = cacheData.asTypeOf(Vec(rowBytes, UInt(8.W))) - val words = cacheData.asTypeOf(Vec(rowBytes / 4, UInt(32.W))) - - val needRead = !cached || cacheAddr =/= srcAddr || cacheGroup =/= srcGroup - - switch(state) { - is(idle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - rob_id_reg := io.cmdReq.bits.rob_id - is_sub_reg := io.cmdReq.bits.is_sub - sub_rob_id_reg := io.cmdReq.bits.sub_rob_id - rbank_reg := cmd.op1_bank - wbank_reg := cmd.wr_bank - ncol_reg := cmd.op1_col - iter_reg := cmd.iter - elem_reg := cmd.rs2(7, 0) - pending := false.B - wrValid := false.B - beatFill := 0.U - cached := false.B - srcR := 0.U - srcC := 0.U - dstVirtRow := 0.U - dstVirtCol := 0.U - lastBeat := false.B - val i8 = cmd.rs2(7, 0) === 8.U - elemBytesReg := Mux(i8, 1.U, 4.U) - epgReg := Mux(i8, rowBytes.U, (rowBytes / 4).U) - wElemsReg := Mux(i8, cmd.op1_col << laneBitsI8.U, cmd.op1_col << laneBitsI32.U) - assert(cmd.iter > 0.U, "Transpose iter must be > 0") - assert(cmd.op1_bank < b.memDomain.virtualBankCount.U, "Transpose input bank is invalid") - assert(cmd.wr_bank < b.memDomain.virtualBankCount.U, "Transpose output bank is invalid") - assert(cmd.op1_bank =/= cmd.wr_bank, "Transpose op1 and wr must differ") - assert( - cmd.op1_col === cmd.wr_col && cmd.op1_col =/= 0.U, - "Transpose cols mismatch" - ) - assert(cmd.rs2(63, 8) === 0.U, "Transpose rs2[63:8] must be 0") - assert( - cmd.rs2(7, 0) === 8.U || cmd.rs2(7, 0) === 32.U, - "Transpose elem_bits" - ) - assert( - bankWidth.U % cmd.rs2(7, 0) === 0.U, - "Transpose bankWidth not divisible by elem_bits" - ) - state := sRead - } - } - - is(sRead) { - io.bankRead(0).group_id := srcGroup - io.bankRead(0).io.resp.ready := pending - io.bankRead(0).io.req.valid := needRead && !pending && !wrValid - io.bankRead(0).io.req.bits.addr := srcAddr - - when(io.bankRead(0).io.req.fire) { - pending := true.B - } - when(io.bankRead(0).io.resp.fire) { - cacheData := io.bankRead(0).io.resp.bits.data - cacheAddr := srcAddr - cacheGroup := srcGroup - cached := true.B - pending := false.B - } - - when(cached && !needRead && !wrValid) { - when(beatFill === 0.U) { - wrGroup := dstGroup - wrAddr := dstVirtRow - wrBytes := VecInit(Seq.fill(rowBytes)(0.U(8.W))) - } - when(isI8) { - wrBytes(beatFill) := bytes(srcLane) - }.otherwise { - val w = words(srcLane) - wrBytes(beatFill << 2) := w(7, 0) - wrBytes((beatFill << 2) + 1.U) := w(15, 8) - wrBytes((beatFill << 2) + 2.U) := w(23, 16) - wrBytes((beatFill << 2) + 3.U) := w(31, 24) - } - beatFill := beatFill + 1.U - - val lastElem = (srcR === iter_reg - 1.U) && (srcC === wElemsReg - 1.U) - when(srcR === iter_reg - 1.U) { - srcR := 0.U - srcC := srcC + 1.U - }.otherwise { - srcR := srcR + 1.U - } - when(dstVirtCol === wElemsReg - 1.U) { - dstVirtCol := 0.U - dstVirtRow := dstVirtRow + 1.U - }.otherwise { - dstVirtCol := dstVirtCol + 1.U - } - - val beatDone = (beatFill + 1.U === epgReg) || lastElem - when(beatDone) { - wrMask.foreach(_ := 1.U) - wrValid := true.B - lastBeat := lastElem - state := sWrite - } - } - } - - is(sWrite) { - io.bankWrite(0).group_id := wrGroup - io.bankWrite(0).io.req.valid := wrValid - io.bankWrite(0).io.req.bits.addr := wrAddr - io.bankWrite(0).io.req.bits.data := wrBytes.asUInt - io.bankWrite(0).io.req.bits.mask := wrMask - io.bankWrite(0).io.resp.ready := true.B - when(io.bankWrite(0).io.req.fire) { - wrValid := false.B - beatFill := 0.U - } - when(io.bankWrite(0).io.resp.fire) { - when(lastBeat) { - state := complete - }.otherwise { - state := sRead - } - } - } - - is(complete) { - io.cmdResp.valid := true.B - when(io.cmdResp.fire) { - state := idle - } - } - } - - io.status.idle := (state === idle) - io.status.running := (state =/= idle) -} diff --git a/examples/balls/transpose/arch/src/main/scala/TransposeBall.scala b/examples/balls/transpose/arch/src/main/scala/TransposeBall.scala deleted file mode 100644 index 109968b94..000000000 --- a/examples/balls/transpose/arch/src/main/scala/TransposeBall.scala +++ /dev/null @@ -1,49 +0,0 @@ -package examples.balls.transpose - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BlinkIO, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.MmioRead -import examples.balls.transpose.Transpose -import framework.top.GlobalConfig - -@instantiable -class TransposeBall(val b: GlobalConfig) extends Module with HasBlink { - - val ballCommonConfig = b.ballDomain.ballIdMappings - .find(_.ballName == "TransposeBall") - .getOrElse( - throw new IllegalArgumentException("TransposeBall not found in config") - ) - - val inBW = ballCommonConfig.inBW - val outBW = ballCommonConfig.outBW - - @public - val io = IO(new BlinkIO(b, inBW, outBW)) - - def blink: BlinkIO = io - dontTouch(io) - - val transposeUnit: Instance[Transpose] = Instantiate(new Transpose(b)) - - transposeUnit.io.cmdReq <> io.cmdReq - transposeUnit.io.cmdResp <> io.cmdResp - - for (i <- 0 until inBW) { - transposeUnit.io.bankRead(i) <> io.bankRead(i) - } - - for (i <- 0 until outBW) { - transposeUnit.io.bankWrite(i) <> io.bankWrite(i) - } - - io.status <> transposeUnit.io.status - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - - // MMIO: this Ball does not consume MMIO metadata - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/transpose/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/transpose/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 20819a12c..000000000 --- a/examples/balls/transpose/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,38 +0,0 @@ -//===- AssignPhysicalBankPatterns.cpp - Transpose bank assignment patterns ===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateTransposeBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { - -class BankTransposePattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankTransposeOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), - op.getIter(), op.getElemBits()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateTransposeBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/transpose/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp b/examples/balls/transpose/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp deleted file mode 100644 index 880276a9a..000000000 --- a/examples/balls/transpose/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Utils/StructuredOpsUtils.h" -#include "mlir/IR/AffineMap.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateTransposeBallLowerBuckyballToBankSSAPatterns( - RewritePatternSet &patterns); -} // namespace mlir::buddy - -namespace { -class MemTransposeToLinalgPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(MemTransposeOp op, - PatternRewriter &rewriter) const override { - auto input = dyn_cast(op.getInput().getType()); - auto output = dyn_cast(op.getOutput().getType()); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 2 || - output.getRank() != 2 || output.getShape()[0] != input.getShape()[1] || - output.getShape()[1] != input.getShape()[0]) - return op.emitError("requires matching static rank-2 transpose memrefs"); - MLIRContext *context = rewriter.getContext(); - AffineMap identity = AffineMap::getMultiDimIdentityMap(2, context); - AffineMap transpose = AffineMap::get( - 2, 0, {rewriter.getAffineDimExpr(1), rewriter.getAffineDimExpr(0)}, - context); - SmallVector iterators(2, - utils::IteratorType::parallel); - rewriter.create( - op.getLoc(), TypeRange{}, ValueRange{op.getInput()}, - ValueRange{op.getOutput()}, ArrayRef{identity, transpose}, - iterators, [](OpBuilder &builder, Location location, ValueRange args) { - builder.create(location, args[0]); - }); - rewriter.eraseOp(op); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateTransposeBallLowerBuckyballToBankSSAPatterns( - RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/transpose/compiler/src/Conversion/LowerTileToBuckyball/TileTransposePatterns.cpp b/examples/balls/transpose/compiler/src/Conversion/LowerTileToBuckyball/TileTransposePatterns.cpp deleted file mode 100644 index 8b0a81c2a..000000000 --- a/examples/balls/transpose/compiler/src/Conversion/LowerTileToBuckyball/TileTransposePatterns.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Tile/TileOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -namespace tile = ::buddy::tile; - -namespace mlir::buddy { -void populateTransposeBallTileLoweringPatterns(RewritePatternSet &patterns, - int64_t bankWidthBytes, - int64_t bankDepth, - int64_t bankNum); -} // namespace mlir::buddy - -namespace { -class TileTransposeLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(tile::TileTransposeOp op, - PatternRewriter &rewriter) const override { - auto inputType = dyn_cast(op.getAMemArray().getType()); - auto outputType = dyn_cast(op.getBMemArray().getType()); - if (!inputType || !outputType || !inputType.hasStaticShape() || - !outputType.hasStaticShape() || inputType.getRank() != 2 || - outputType.getRank() != 2) - return op.emitError("requires static rank-2 input and output memrefs"); - if (outputType.getShape()[0] != inputType.getShape()[1] || - outputType.getShape()[1] != inputType.getShape()[0] || - inputType.getElementType() != outputType.getElementType()) - return op.emitError("requires a matching transposed output memref"); - rewriter.create(op.getLoc(), op.getAMemArray(), - op.getBMemArray()); - rewriter.eraseOp(op); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateTransposeBallTileLoweringPatterns( - RewritePatternSet &patterns, int64_t bankWidthBytes, int64_t bankDepth, - int64_t bankNum) { - (void)bankWidthBytes; - (void)bankDepth; - (void)bankNum; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/transpose/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/transpose/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 1d6dfa1b3..000000000 --- a/examples/balls/transpose/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,115 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -static LogicalResult validateTranspose(Operation *op, Value elemBits, - Value iter, int64_t bankDepth) { - auto constant = elemBits.getDefiningOp(); - auto value = - constant ? dyn_cast(constant.getValue()) : IntegerAttr(); - if (!value || (value.getInt() != 8 && value.getInt() != 32)) - return op->emitError("Transpose elemBits must be constant 8 or 32"); - auto iterOp = iter.getDefiningOp(); - auto iterAttr = - iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); - if (!iterAttr || iterAttr.getInt() <= 0 || iterAttr.getInt() > bankDepth) - return op->emitError( - "Transpose iter must be a positive constant within the bank depth"); - return success(); -} - -struct TransposeLowering : public ConvertOpToLLVMPattern { - TransposeLowering(LLVMTypeConverter &converter, bool stable, - int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), stable(stable), - bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(TransposeOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("TransposeBall"); - Location loc = op.getLoc(); - if (failed( - validateTranspose(op, op.getElemBits(), op.getIter(), bankDepth))) - return failure(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - if (stable) { - rewriter.replaceOpWithNewOp(op, rs1, - adaptor.getElemBits()); - return success(); - } - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getElemBits(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("TRANSPOSE"))); - return success(); - } - -private: - bool stable = false; - int64_t bankDepth; -}; -struct BankTransposeLowering : public ConvertOpToLLVMPattern { - BankTransposeLowering(LLVMTypeConverter &converter, bool stable, - int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), stable(stable), - bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(BankTransposeOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("TransposeBall"); - Location loc = op.getLoc(); - if (failed( - validateTranspose(op, op.getElemBits(), op.getIter(), bankDepth))) - return failure(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInBank(), - cstI64(rewriter, loc, 0), adaptor.getOutBank(), - adaptor.getIter()); - if (stable) { - rewriter.replaceOpWithNewOp(op, rs1, - adaptor.getElemBits()); - return success(); - } - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getElemBits(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("TRANSPOSE"))); - return success(); - } - -private: - bool stable = false; - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateTransposeBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - patterns.add(converter, stable, bankDepth); - patterns.add(converter, stable, bankDepth); -} - -void configureTransposeBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - if (stable) - target.addLegalOp(); - else - target.addIllegalOp(); - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/transpose/configs/TransposeBallParam.scala b/examples/balls/transpose/configs/TransposeBallParam.scala deleted file mode 100644 index 1bb46a731..000000000 --- a/examples/balls/transpose/configs/TransposeBallParam.scala +++ /dev/null @@ -1,21 +0,0 @@ -package examples.balls.transpose.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig - -case class TransposeBallParam( - InputNum: Int, - inputWidth: Int) - -object TransposeBallParam { - private val ballName = "TransposeBall" - - def apply(b: GlobalConfig): TransposeBallParam = { - val tbl = BallParamLoader.ballTable(b, ballName) - TransposeBallParam( - InputNum = BallParamLoader.int(tbl, "InputNum"), - inputWidth = BallParamLoader.int(tbl, "inputWidth") - ) - } - -} diff --git a/examples/balls/transpose/configs/default.toml b/examples/balls/transpose/configs/default.toml deleted file mode 100644 index 5c957d025..000000000 --- a/examples/balls/transpose/configs/default.toml +++ /dev/null @@ -1,5 +0,0 @@ -# Transpose Ball Configuration - -[ball] -InputNum = 16 -inputWidth = 8 diff --git a/examples/balls/transpose/emu/src/dispatch.rs b/examples/balls/transpose/emu/src/dispatch.rs deleted file mode 100644 index 208951e45..000000000 --- a/examples/balls/transpose/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::ExecContext; - -use super::transpose; - -const BALL_CLASS: &str = "examples.balls.transpose.TransposeBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("TRANSPOSE") => Some(transpose::exec_transpose(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("TRANSPOSE") => Some(transpose::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/transpose/emu/src/lib.rs b/examples/balls/transpose/emu/src/lib.rs deleted file mode 100644 index 38f0fa485..000000000 --- a/examples/balls/transpose/emu/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub(crate) use crate::inst::{decode, instruction}; - -mod transpose; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/transpose/verify/README.md b/examples/balls/transpose/verify/README.md deleted file mode 100644 index abd6c90c9..000000000 --- a/examples/balls/transpose/verify/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# TransposeBall UVM Verification - -Verifies the generated `TransposeBall` module directly using the shared Blink UVM framework. - -## Structure - -- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM transaction items and base env -- `src/common/transpose_defs.svh`: DPI imports and constants -- `src/common/transpose_items.svh`: `transpose_cmd_item` with `load_rust_case` -- `src/seq/transpose_sequences.svh`: `transpose_basic_seq` driving one case per run -- `src/cov/transpose_cov.svh`: `elem_bits` {8,32} and `iter` {1,2,4,8,16} coverage -- `src/env/transpose_scoreboard.svh`: preloads mem model, compares writes vs DPI dst words -- `src/env/transpose_env.svh`: extends `bb_blink_env#(1,1)` with scoreboard and cov -- `src/tests/transpose_test.svh`: directed cases 0,1 then random 2..21 with fixed seed -- `src/pkg/transpose_pkg.sv`: package include entry -- `src/tb_top.sv`: instantiates `bb_blink_if#(1,1)` and `TransposeBall` -- `filelists/transpose_ball.f`: VCS/Verilator filelist - -## Test plan - -`+UVM_TESTNAME=transpose_ball_test` runs: - -- case 0: directed i8, iter=16 -- case 1: directed i32, iter=8 -- cases 2..21: random with fixed seed `0xCAFE_BABE`, elem_bits in {8,32}, iter in {1,2,4,8,16} - -Each case: reset, preload src words into `mem_model`, drive one command, wait for -`scb.done()` or timeout at 2000 cycles. Scoreboard compares observed writes -(data/addr/mask/bank/rob) with DPI `transpose_case_dst_word_*`. `transpose_cov` -requires both `elem_bits` bins and all `iter` bins hit across the whole test, else -`check_phase` fatal. - -## BID (required) - -`+BID=` is mandatory. Missing it fatals at test start. There is no default bid. - -| Config | Plusarg | -|--------|---------| -| `sims.verilator.BuckyballToyVerilatorConfig` | `+BID=2` | -| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=0` | - -```console -./simv +UVM_TESTNAME=transpose_ball_test +BID=2 -``` - -## Build - -Enter the shared UVM/VCS environment: - -```console -nix develop ../../../../verify -``` - -Build the DPI reference library: - -```console -cargo build --manifest-path casegen/Cargo.toml -``` - -Compile from this directory: - -```console -vcs -full64 -sverilog -timescale=1ns/1ps \ - $VCS_UVM_ARGS \ - -sv_lib casegen/target/debug/libtranspose_casegen \ - -f filelists/transpose_ball.f -``` - -Run: - -```console -./simv +UVM_TESTNAME=transpose_ball_test +BID=2 -``` - -Or via bbdev: - -```console -bbdev uvm --build '--ball=transpose' -bbdev uvm --run '--ball=transpose' -- '+BID=2' -``` - - -## Acceptance (2026-08-03) - -- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` -- [x] casegen reuses transpose model; whole-case DPI API -- [x] directed i8/i32 scoreboard pass -- [x] deterministic random idx 2..21 pass -- [x] protocol + semantic cover enforced (`check_phase` fatal) -- [x] toy: `+BID=2` config `sims.verilator.BuckyballToyVerilatorConfig` — 22/22 -- [x] pebble: `+BID=0` config `sims.verilator.BuckyballPebbleVerilatorConfig` — 22/22 diff --git a/examples/balls/transpose/verify/casegen/src/casegen.rs b/examples/balls/transpose/verify/casegen/src/casegen.rs deleted file mode 100644 index 38018e500..000000000 --- a/examples/balls/transpose/verify/casegen/src/casegen.rs +++ /dev/null @@ -1,303 +0,0 @@ -use crate::model; - -pub const BANK_ROW_BYTES: usize = 16; -pub const MAX_ITER: usize = 16; -pub const MAX_WORDS: usize = MAX_ITER; - -#[repr(C)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct TransposeCmd { - pub bid: u32, - pub iter: u32, - pub op1_bank: u32, - pub wr_bank: u32, - pub op1_col: u32, - pub wr_col: u32, - pub rob_id: u32, - pub elem_bits: u32, - pub num_src_words: u32, - pub num_dst_words: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TransposeCase { - pub cmd: TransposeCmd, - pub src: Vec, - pub dst: Vec, -} - -impl TransposeCase { - pub fn src_word_lo(&self, word_index: usize) -> u64 { - word_lo(&self.src, word_index) - } - pub fn src_word_hi(&self, word_index: usize) -> u64 { - word_hi(&self.src, word_index) - } - pub fn dst_word_lo(&self, word_index: usize) -> u64 { - word_lo(&self.dst, word_index) - } - pub fn dst_word_hi(&self, word_index: usize) -> u64 { - word_hi(&self.dst, word_index) - } -} - -fn word_lo(data: &[u8], word_index: usize) -> u64 { - let off = word_index * BANK_ROW_BYTES; - let mut buf = [0u8; 8]; - buf.copy_from_slice(&data[off..off + 8]); - u64::from_le_bytes(buf) -} - -fn word_hi(data: &[u8], word_index: usize) -> u64 { - let off = word_index * BANK_ROW_BYTES + 8; - let mut buf = [0u8; 8]; - buf.copy_from_slice(&data[off..off + 8]); - u64::from_le_bytes(buf) -} - -pub fn gen_case(seed: u32, index: u32, bid: u32) -> TransposeCase { - match index { - 0 => directed_i8(bid), - 1 => directed_i32(bid), - _ => random_case(seed, index, bid), - } -} - -fn directed_i8(bid: u32) -> TransposeCase { - let iter = 16usize; - let elem_bytes = 1usize; - let cols = 1usize; - let w = cols * (BANK_ROW_BYTES / elem_bytes); - let total = iter * w; - let src: Vec = (0..total).map(|i| i as u8).collect(); - let dst = model::transpose_bytes(&src, iter, w, elem_bytes); - let nwords = model::num_words(total * elem_bytes) as u32; - TransposeCase { - cmd: TransposeCmd { - bid, - iter: iter as u32, - op1_bank: 0, - wr_bank: 1, - op1_col: cols as u32, - wr_col: cols as u32, - rob_id: 3, - elem_bits: (elem_bytes * 8) as u32, - num_src_words: nwords, - num_dst_words: nwords, - }, - src, - dst, - } -} - -fn directed_i32(bid: u32) -> TransposeCase { - let iter = 8usize; - let elem_bytes = 4usize; - let cols = 1usize; - let w = cols * (BANK_ROW_BYTES / elem_bytes); - let total = iter * w; - let src: Vec = (0..total) - .map(|i| i as u32) - .flat_map(|v| v.to_le_bytes()) - .collect(); - let dst = model::transpose_bytes(&src, iter, w, elem_bytes); - let nwords = model::num_words(total * elem_bytes) as u32; - TransposeCase { - cmd: TransposeCmd { - bid, - iter: iter as u32, - op1_bank: 0, - wr_bank: 1, - op1_col: cols as u32, - wr_col: cols as u32, - rob_id: 5, - elem_bits: (elem_bytes * 8) as u32, - num_src_words: nwords, - num_dst_words: nwords, - }, - src, - dst, - } -} - -fn random_case(seed: u32, index: u32, bid: u32) -> TransposeCase { - let mut rng = Rng::new(seed, index); - let elem_bits = if rng.next() & 1 == 0 { 8u32 } else { 32u32 }; - let elem_bytes = (elem_bits / 8) as usize; - let iter_pool = [1u32, 2, 4, 8, 16]; - let iter = iter_pool[(rng.next() as usize) % iter_pool.len()] as usize; - let cols = 1usize; - let w = cols * (BANK_ROW_BYTES / elem_bytes); - - // Keep banks in 0..7 so the same cases work on configs with bankNum=8 or 16. - let op1_bank = rng.range(0, 7); - let mut wr_bank = rng.range(0, 7); - if wr_bank == op1_bank { - wr_bank = (wr_bank + 1) & 7; - } - - let total = iter * w; - let mut src = vec![0u8; total * elem_bytes]; - for b in src.iter_mut() { - *b = (rng.next() & 0xFF) as u8; - } - let dst = model::transpose_bytes(&src, iter, w, elem_bytes); - let nwords = model::num_words(total * elem_bytes) as u32; - - TransposeCase { - cmd: TransposeCmd { - bid, - iter: iter as u32, - op1_bank, - wr_bank, - op1_col: cols as u32, - wr_col: cols as u32, - rob_id: rng.range(0, 15), - elem_bits, - num_src_words: nwords, - num_dst_words: nwords, - }, - src, - dst, - } -} - -struct Rng { - state: u64, -} - -impl Rng { - fn new(seed: u32, index: u32) -> Self { - let state = (u64::from(seed) << 32) ^ u64::from(index) ^ 0x9E37_79B9_7F4A_7C15; - Self { state } - } - fn next(&mut self) -> u32 { - self.state ^= self.state >> 12; - self.state ^= self.state << 25; - self.state ^= self.state >> 27; - ((self.state.wrapping_mul(0x2545_F491_4F6C_DD1D)) >> 32) as u32 - } - fn range(&mut self, lo: u32, hi: u32) -> u32 { - lo + (self.next() % (hi - lo + 1)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn model_transpose_i8_roundtrip() { - let iter = 16; - let w = 16; - let src: Vec = (0..iter * w).map(|i| i as u8).collect(); - let dst = model::transpose_bytes(&src, iter, w, 1); - for r in 0..iter { - for c in 0..w { - assert_eq!(dst[c * iter + r], src[r * w + c]); - } - } - } - - #[test] - fn model_transpose_i32_roundtrip() { - let iter = 8; - let w = 4; - let src: Vec = (0..iter * w) - .map(|i| i as u32) - .flat_map(|v| v.to_le_bytes()) - .collect(); - let dst = model::transpose_bytes(&src, iter, w, 4); - for r in 0..iter { - for c in 0..w { - let s = (r * w + c) * 4; - let d = (c * iter + r) * 4; - assert_eq!(&dst[d..d + 4], &src[s..s + 4]); - } - } - } - - #[test] - fn directed_i8_matches_ctest_shape() { - let case = gen_case(0x1234, 0, 0); - assert_eq!(case.cmd.bid, 0); - assert_eq!(case.cmd.iter, 16); - assert_eq!(case.cmd.elem_bits, 8); - assert_eq!(case.cmd.op1_bank, 0); - assert_eq!(case.cmd.wr_bank, 1); - assert_eq!(case.cmd.op1_col, 1); - assert_eq!(case.cmd.wr_col, 1); - assert_eq!(case.cmd.num_src_words, 16); - assert_eq!(case.cmd.num_dst_words, 16); - assert_eq!(case.src.len(), 256); - assert_eq!(case.dst.len(), 256); - assert_eq!(case.src[0], 0); - assert_eq!(case.src[255], 255); - for r in 0..16 { - for c in 0..16 { - assert_eq!(case.dst[c * 16 + r], case.src[r * 16 + c]); - } - } - } - - #[test] - fn directed_i32_matches_ctest_shape() { - let case = gen_case(0, 1, 0); - assert_eq!(case.cmd.iter, 8); - assert_eq!(case.cmd.elem_bits, 32); - assert_eq!(case.cmd.op1_bank, 0); - assert_eq!(case.cmd.wr_bank, 1); - assert_eq!(case.cmd.op1_col, 1); - assert_eq!(case.cmd.wr_col, 1); - assert_eq!(case.cmd.num_src_words, 8); - assert_eq!(case.cmd.num_dst_words, 8); - assert_eq!(case.src.len(), 128); - assert_eq!(case.dst.len(), 128); - } - - #[test] - fn random_cases_deterministic_and_legal() { - let a = gen_case(0xCAFE_BABE, 2, 0); - let b = gen_case(0xCAFE_BABE, 2, 0); - assert_eq!(a, b); - assert!(a.cmd.elem_bits == 8 || a.cmd.elem_bits == 32); - assert!([1u32, 2, 4, 8, 16].contains(&a.cmd.iter)); - assert_eq!(a.cmd.op1_col, 1); - assert_eq!(a.cmd.wr_col, 1); - assert_ne!(a.cmd.op1_bank, a.cmd.wr_bank); - assert!(a.cmd.op1_bank < 32); - assert!(a.cmd.wr_bank < 32); - assert!(a.cmd.rob_id < 16); - assert_eq!(a.cmd.num_src_words, a.cmd.num_dst_words); - assert_eq!(a.src.len(), a.dst.len()); - } - - #[test] - fn bid_is_required_arg() { - let case = gen_case(0, 0, 2); - assert_eq!(case.cmd.bid, 2); - let case = gen_case(0, 0, 5); - assert_eq!(case.cmd.bid, 5); - } - - #[test] - fn word_lo_hi_round_trip() { - let case = gen_case(0x1234, 0, 0); - for i in 0..case.cmd.num_src_words as usize { - let lo = case.src_word_lo(i); - let hi = case.src_word_hi(i); - let off = i * BANK_ROW_BYTES; - let expect_lo = u64::from_le_bytes(case.src[off..off + 8].try_into().unwrap()); - let expect_hi = u64::from_le_bytes(case.src[off + 8..off + 16].try_into().unwrap()); - assert_eq!(lo, expect_lo); - assert_eq!(hi, expect_hi); - } - } - - #[test] - #[should_panic] - fn word_lo_out_of_range_panics() { - let case = gen_case(0x1234, 0, 0); - let _ = case.src_word_lo(MAX_WORDS); - } -} diff --git a/examples/balls/transpose/verify/casegen/src/lib.rs b/examples/balls/transpose/verify/casegen/src/lib.rs deleted file mode 100644 index 7f09fb927..000000000 --- a/examples/balls/transpose/verify/casegen/src/lib.rs +++ /dev/null @@ -1,107 +0,0 @@ -mod casegen; -mod model; - -use std::cell::RefCell; - -use casegen::{TransposeCase, TransposeCmd, MAX_WORDS}; - -thread_local! { - static CURRENT: RefCell> = RefCell::new(None); -} - -#[no_mangle] -pub extern "C" fn transpose_case_load(seed: u32, index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(seed, index, bid); - let nsrc = case.cmd.num_src_words as usize; - let ndst = case.cmd.num_dst_words as usize; - if nsrc > MAX_WORDS || ndst > MAX_WORDS { - panic!("transpose_case_load: word count out of range src={nsrc} dst={ndst}"); - } - CURRENT.with(|c| *c.borrow_mut() = Some(case)); - 0 -} - -fn current_case R, R>(f: F) -> R { - CURRENT.with(|c| match c.borrow().as_ref() { - Some(case) => f(case), - None => panic!("transpose_case: no case loaded; call transpose_case_load first"), - }) -} - -#[no_mangle] -pub extern "C" fn transpose_case_cmd(out_ptr: *mut TransposeCmd) { - current_case(|case| unsafe { - if out_ptr.is_null() { - panic!("transpose_case_cmd: null out_ptr"); - } - *out_ptr = case.cmd; - }); -} - -#[no_mangle] -pub extern "C" fn transpose_case_src_word_lo(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("transpose_case_src_word_lo: word_index out of range"); - } - current_case(|case| case.src_word_lo(word_index as usize)) -} - -#[no_mangle] -pub extern "C" fn transpose_case_src_word_hi(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("transpose_case_src_word_hi: word_index out of range"); - } - current_case(|case| case.src_word_hi(word_index as usize)) -} - -#[no_mangle] -pub extern "C" fn transpose_case_dst_word_lo(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("transpose_case_dst_word_lo: word_index out of range"); - } - current_case(|case| case.dst_word_lo(word_index as usize)) -} - -#[no_mangle] -pub extern "C" fn transpose_case_dst_word_hi(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("transpose_case_dst_word_hi: word_index out of range"); - } - current_case(|case| case.dst_word_hi(word_index as usize)) -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn dpi_load_then_cmd_and_words() { - assert_eq!(transpose_case_load(0x1234, 0, 0), 0); - let mut cmd = TransposeCmd { - bid: 0, - iter: 0, - op1_bank: 0, - wr_bank: 0, - op1_col: 0, - wr_col: 0, - rob_id: 0, - elem_bits: 0, - num_src_words: 0, - num_dst_words: 0, - }; - transpose_case_cmd(&mut cmd as *mut TransposeCmd); - assert_eq!(cmd.iter, 16); - assert_eq!(cmd.elem_bits, 8); - assert_eq!(cmd.num_src_words, 16); - let _lo = transpose_case_src_word_lo(0); - let _hi = transpose_case_src_word_hi(0); - let _dlo = transpose_case_dst_word_lo(0); - let _dhi = transpose_case_dst_word_hi(0); - } - - #[test] - #[should_panic(expected = "no case loaded")] - fn current_case_panics_without_load() { - CURRENT.with(|c| *c.borrow_mut() = None); - current_case(|_| ()); - } -} diff --git a/examples/balls/transpose/verify/src/tb_top.sv b/examples/balls/transpose/verify/src/tb_top.sv deleted file mode 100644 index 14deae43f..000000000 --- a/examples/balls/transpose/verify/src/tb_top.sv +++ /dev/null @@ -1,90 +0,0 @@ -module tb_top; - import uvm_pkg::*; - import transpose_pkg::*; - - bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); - - TransposeBall dut ( - .clock(intf.clock), - .reset(intf.reset), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready), - .io_status_idle(), - .io_status_running(), - .io_channelReady(1'b1), - .io_bankRead_0_ball_id(), - .io_bankWrite_0_ball_id() - ); - - initial begin - intf.clock = 1'b0; - forever #5 intf.clock = ~intf.clock; - end - - initial begin - uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); - run_test("transpose_ball_test"); - end -endmodule diff --git a/examples/balls/transpose/workloads/ctests/CMakeLists.txt b/examples/balls/transpose/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 9212a8e5b..000000000 --- a/examples/balls/transpose/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -add_buckyball_ctests( - transpose_i8_16x16_test.c - transpose_i8_16x32_test.c - transpose_i32_8x16_test.c -) diff --git a/examples/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir b/examples/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir deleted file mode 100644 index a03588615..000000000 --- a/examples/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir +++ /dev/null @@ -1,44 +0,0 @@ -// Already-lowered ball Op: buckyball.transpose (physical-bank form). - -func.func private @check_result(memref<16x16xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %n = arith.constant 16 : index - %c16_i32 = arith.constant 16 : i32 - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %iter = arith.constant 16 : i64 - %elem_bits = arith.constant 8 : i64 - - %input = memref.alloc() alignment = 64 : memref<16x16xi8> - %output = memref.alloc() alignment = 64 : memref<16x16xi8> - - scf.for %i = %c0 to %n step %c1 { - scf.for %j = %c0 to %n step %c1 { - %ii = arith.index_cast %i : index to i32 - %jj = arith.index_cast %j : index to i32 - %mul = arith.muli %ii, %c16_i32 : i32 - %val = arith.addi %mul, %jj : i32 - %v8 = arith.trunci %val : i32 to i8 - memref.store %v8, %input[%i, %j] : memref<16x16xi8> - } - } - - %bin = buckyball.bank_alloc - %bout = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %bin %depth %stride - : memref<16x16xi8> i64 i64 i64 - buckyball.transpose %loaded, %bout, %iter, %elem_bits : i64 - %stored = buckyball.bank_mvout %output %bout %depth %stride - : memref<16x16xi8> i64 i64 i64 - buckyball.bank_release %loaded : i64 - buckyball.bank_release %stored : i64 - - func.call @check_result(%output) : (memref<16x16xi8>) -> () - memref.dealloc %input : memref<16x16xi8> - memref.dealloc %output : memref<16x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in b/examples/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in deleted file mode 100644 index d7c57a39d..000000000 --- a/examples/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in +++ /dev/null @@ -1,35 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %lanes = arith.constant @BB_BANK_I8_LANES@ : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %stride = arith.constant 1 : i64 - %elem_bits = arith.constant 8 : i64 - %lanes_i32 = arith.constant @BB_BANK_I8_LANES@ : i32 - %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - scf.for %row = %zero to %rows step %one { - scf.for %lane = %zero to %lanes step %one { - %r = arith.index_cast %row : index to i32 - %l = arith.index_cast %lane : index to i32 - %base = arith.muli %r, %lanes_i32 : i32 - %index = arith.addi %base, %l : i32 - %value = arith.trunci %index : i32 to i8 - memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - } - } - %input_bank = buckyball.bank_alloc - %output_bank = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %input_bank %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - %result = buckyball.bank_transpose %loaded %output_bank %depth %elem_bits : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %depth %stride : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - buckyball.bank_release %loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - return %zero_i8 : i8 -} diff --git a/examples/balls/vector/arch/src/main/scala/VecBall.scala b/examples/balls/vector/arch/src/main/scala/VecBall.scala deleted file mode 100644 index 7cf6bc243..000000000 --- a/examples/balls/vector/arch/src/main/scala/VecBall.scala +++ /dev/null @@ -1,53 +0,0 @@ -package examples.balls.vector - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.balldomain.blink.{BallStatus, BlinkIO, HasBallStatus, HasBlink, SubRobRow} -import framework.balldomain.blink.mmio.MmioRead -import examples.balls.vector.VecUnit -import framework.top.GlobalConfig - -/** - * VecBall - */ -@instantiable -class VecBall(val b: GlobalConfig) extends Module with HasBlink with HasBallStatus { - - val ballCommonConfig = b.ballDomain.ballIdMappings - .find(_.ballName == "VecBall") - .getOrElse( - throw new IllegalArgumentException("VecBall not found in config") - ) - - val inBW = ballCommonConfig.inBW - val outBW = ballCommonConfig.outBW - - @public - val io = IO(new BlinkIO(b, inBW, outBW)) - - def blink: BlinkIO = io - def status: BallStatus = io.status - dontTouch(io) - - val vecUnit: Instance[VecUnit] = Instantiate(new VecUnit(b)) - - vecUnit.io.cmdReq <> io.cmdReq - vecUnit.io.cmdResp <> io.cmdResp - - for (i <- 0 until inBW) { - vecUnit.io.bankRead(i) <> io.bankRead(i) - } - - for (i <- 0 until outBW) { - vecUnit.io.bankWrite(i) <> io.bankWrite(i) - } - - io.status <> vecUnit.io.status - - io.subRobReq.valid := false.B - io.subRobReq.bits := SubRobRow.tieOff(b) - - // MMIO: this Ball does not consume MMIO metadata - MmioRead.tieOff(io.mmioRead) -} diff --git a/examples/balls/vector/arch/src/main/scala/VecCtrlUnit.scala b/examples/balls/vector/arch/src/main/scala/VecCtrlUnit.scala deleted file mode 100644 index e43433b51..000000000 --- a/examples/balls/vector/arch/src/main/scala/VecCtrlUnit.scala +++ /dev/null @@ -1,131 +0,0 @@ -package examples.balls.vector - -import chisel3._ -import chisel3.util._ -import chisel3.stage._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.top.GlobalConfig - -@instantiable -class VecCtrlUnit(val b: GlobalConfig) extends Module { - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp_o = Decoupled(new BallRsComplete(b)) - - val ctrl_ld_o = Decoupled(new ctrl_ld_req(b)) - val ctrl_st_o = Decoupled(new ctrl_st_req(b)) - val ctrl_ex_o = Decoupled(new ctrl_ex_req(b)) - - val cmdResp_i = Flipped(Valid(new Bundle { - val commit = Bool() - })) // from store unit - }) - - val rob_id_reg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val is_sub_reg = RegInit(false.B) - val sub_rob_id_reg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - val iter = RegInit(0.U(b.frontend.iter_len.W)) - val op1_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - - val op1_bank_addr = RegInit( - 0.U(12.W) - ) // New ISA: always 0, but keep for compatibility - val op2_bank_addr = RegInit( - 0.U(12.W) - ) // New ISA: always 0, but keep for compatibility - val op2_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val wr_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - - val wr_bank_addr = RegInit( - 0.U(12.W) - ) // New ISA: always 0, but keep for compatibility - val has_send = RegInit(false.B) - val mode = RegInit(0.U(1.W)) - - val idle :: busy :: Nil = Enum(2) - val state = RegInit(idle) - -// ----------------------------------------------------------------------------- -// Set registers when EX instruction arrives -// ----------------------------------------------------------------------------- - - when(io.cmdReq.fire) { - iter := io.cmdReq.bits.cmd.iter - rob_id_reg := io.cmdReq.bits.rob_id - is_sub_reg := io.cmdReq.bits.is_sub - sub_rob_id_reg := io.cmdReq.bits.sub_rob_id - op1_bank := io.cmdReq.bits.cmd.op1_bank - op1_bank_addr := 0.U // New ISA: all operations start from row 0 - op2_bank := io.cmdReq.bits.cmd.op2_bank - op2_bank_addr := 0.U // New ISA: all operations start from row 0 - wr_bank := io.cmdReq.bits.cmd.wr_bank - wr_bank_addr := 0.U // New ISA: all operations start from row 0 - mode := io.cmdReq.bits.cmd.special(0) - - state := busy - } - -// ----------------------------------------------------------------------------- -// Send control signals to VecUnit's load/store/ex units -// ----------------------------------------------------------------------------- - - when(state === busy && !has_send) { - io.ctrl_ld_o.valid := true.B - io.ctrl_ld_o.bits.op1_bank := op1_bank - io.ctrl_ld_o.bits.op1_bank_addr := op1_bank_addr - io.ctrl_ld_o.bits.op2_bank := op2_bank - io.ctrl_ld_o.bits.op2_bank_addr := op2_bank_addr - io.ctrl_ld_o.bits.iter := iter - io.ctrl_ld_o.bits.mode := mode - - io.ctrl_ex_o.valid := true.B - io.ctrl_ex_o.bits.iter := iter - - io.ctrl_st_o.valid := true.B - io.ctrl_st_o.bits.wr_bank := wr_bank - io.ctrl_st_o.bits.wr_bank_addr := wr_bank_addr - io.ctrl_st_o.bits.iter := iter - - has_send := true.B - }.otherwise { - io.ctrl_ld_o.valid := false.B - io.ctrl_ld_o.bits.op1_bank := 0.U - io.ctrl_ld_o.bits.op1_bank_addr := 0.U - io.ctrl_ld_o.bits.op2_bank := 0.U - io.ctrl_ld_o.bits.op2_bank_addr := 0.U - io.ctrl_ld_o.bits.iter := 0.U - io.ctrl_ld_o.bits.mode := 0.U - - io.ctrl_ex_o.valid := false.B - io.ctrl_ex_o.bits.iter := 0.U - - io.ctrl_st_o.valid := false.B - io.ctrl_st_o.bits.wr_bank := 0.U - io.ctrl_st_o.bits.wr_bank_addr := 0.U - io.ctrl_st_o.bits.iter := 0.U - } - -// ----------------------------------------------------------------------------- -// Wait for VecUnit's final write-back to complete -// ----------------------------------------------------------------------------- - - when(io.cmdResp_i.valid) { - io.cmdResp_o.valid := true.B - io.cmdResp_o.bits.rob_id := rob_id_reg - io.cmdResp_o.bits.is_sub := is_sub_reg - io.cmdResp_o.bits.sub_rob_id := sub_rob_id_reg - state := idle - has_send := false.B - }.otherwise { - io.cmdResp_o.valid := false.B - io.cmdResp_o.bits.rob_id := 0.U - io.cmdResp_o.bits.is_sub := false.B - io.cmdResp_o.bits.sub_rob_id := 0.U - } - - io.cmdReq.ready := state === idle - -} diff --git a/examples/balls/vector/arch/src/main/scala/VecLoadUnit.scala b/examples/balls/vector/arch/src/main/scala/VecLoadUnit.scala deleted file mode 100644 index c343e9b89..000000000 --- a/examples/balls/vector/arch/src/main/scala/VecLoadUnit.scala +++ /dev/null @@ -1,178 +0,0 @@ -package examples.balls.vector - -import chisel3._ -import chisel3.util._ -import chisel3.stage._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.memdomain.backend.banks.{SramReadReq, SramReadResp} -import framework.top.GlobalConfig -import examples.balls.vector.configs.VectorBallParam - -class ctrl_ld_req(b: GlobalConfig) extends Bundle { - val op1_bank = UInt(b.memDomain.vbankIdWidth.W) - val op1_bank_addr = UInt(log2Up(b.memDomain.bankEntries).W) - val op2_bank = UInt(b.memDomain.vbankIdWidth.W) - val op2_bank_addr = UInt(log2Up(b.memDomain.bankEntries).W) - val iter = UInt(b.frontend.iter_len.W) - val mode = UInt(1.W) -} - -@instantiable -class VecLoadUnit(val b: GlobalConfig) extends Module { - val config = VectorBallParam(b) - val InputNum = config.lane - val inputWidth = config.inputWidth - val bankWidth = b.memDomain.bankWidth - val rob_id_width = log2Up(b.frontend.rob_entries) - - // Get bandwidth from config (use first VecBall mapping) - val ballMapping = b.ballDomain.ballIdMappings - .find(_.ballName == "VecBall") - .getOrElse( - throw new IllegalArgumentException("VecBall not found in config") - ) - - val inBW = ballMapping.inBW - - @public - val io = IO(new Bundle { - val bankReadReq = Vec(inBW, Decoupled(new SramReadReq(b))) - val bankReadResp = Vec(inBW, Flipped(Decoupled(new SramReadResp(b)))) - val ctrl_ld_i = Flipped(Decoupled(new ctrl_ld_req(b))) - val ld_ex_o = Decoupled(new ld_ex_req(b)) - val op1_bank_o = Output(UInt(b.memDomain.vbankIdWidth.W)) - val op2_bank_o = Output(UInt(b.memDomain.vbankIdWidth.W)) - }) - - val op1_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val op2_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val op1_addr = RegInit(0.U(log2Up(b.memDomain.bankEntries).W)) - val op2_addr = RegInit(0.U(log2Up(b.memDomain.bankEntries).W)) - val iter = RegInit(0.U(b.frontend.iter_len.W)) - val op1_iter_counter = RegInit(0.U(b.frontend.iter_len.W)) - val op2_iter_counter = RegInit(0.U(b.frontend.iter_len.W)) - val idle :: busy :: Nil = Enum(2) - val state = RegInit(idle) - val ld_ex_op1_reg = Reg(Vec(InputNum, UInt(inputWidth.W))) - val ld_ex_op2_reg = Reg(Vec(InputNum, UInt(inputWidth.W))) - val ld_ex_iter_reg = RegInit(0.U(b.frontend.iter_len.W)) - val wait1_reg = RegInit(false.B) - val wait2_reg = RegInit(false.B) - val wait1_cnt = RegInit(0.U(7.W)) - val wait2_cnt = RegInit(0.U(7.W)) - - val bankRespQueue0 = Module(new Queue(new SramReadResp(b), entries = 8)) - val bankRespQueue1 = Module(new Queue(new SramReadResp(b), entries = 8)) - - for (i <- 0 until inBW) { - io.bankReadReq(i).valid := false.B - io.bankReadReq(i).bits.addr := 0.U - } - - io.op1_bank_o := op1_bank - io.op2_bank_o := op2_bank - io.ctrl_ld_i.ready := state === idle - - bankRespQueue0.io.enq <> io.bankReadResp(0) - bankRespQueue1.io.enq <> io.bankReadResp(1) - -// ----------------------------------------------------------------------------- -// Set registers when Ctrl instruction arrives -// ----------------------------------------------------------------------------- - when(io.ctrl_ld_i.fire) { - op1_bank := io.ctrl_ld_i.bits.op1_bank - op2_bank := io.ctrl_ld_i.bits.op2_bank - op1_addr := io.ctrl_ld_i.bits.op1_bank_addr - op2_addr := io.ctrl_ld_i.bits.op2_bank_addr - iter := io.ctrl_ld_i.bits.iter - op1_iter_counter := 0.U - op2_iter_counter := 0.U - state := busy - assert(io.ctrl_ld_i.bits.iter > 0.U, "iter should be greater than 0") - } - -// ----------------------------------------------------------------------------- -// Send SRAM read request -// ----------------------------------------------------------------------------- - //wait1_reg := Mux(io.run , 0.U, wait1_reg) - - //wait2_reg := Mux(io.run , 0.U, wait2_reg) - - when(state === busy && io.ld_ex_o.ready && !wait1_reg) { - io.bankReadReq(0).valid := op1_iter_counter < iter - io.bankReadReq(0).bits.addr := op1_addr + op1_iter_counter - op1_iter_counter := Mux( - io.bankReadReq(0).ready, - op1_iter_counter + 1.U, - op1_iter_counter - ) - wait1_reg := Mux((op1_iter_counter + 1.U) % 16.U === 0.U, 1.U, 0.U) - when(io.bankReadReq(0).fire) {} - } - - when(state === busy && io.ld_ex_o.ready && !wait2_reg) { - io.bankReadReq(1).valid := op2_iter_counter < iter - io.bankReadReq(1).bits.addr := op2_addr + op2_iter_counter - op2_iter_counter := Mux( - io.bankReadReq(1).ready, - op2_iter_counter + 1.U, - op2_iter_counter - ) - wait2_reg := Mux((op2_iter_counter + 1.U) % 16.U === 0.U, 1.U, 0.U) - when(io.bankReadReq(1).fire) {} - } - - when(wait1_reg) { - wait1_cnt := wait1_cnt + 1.U - when(wait1_cnt === 32.U) { - wait1_reg := false.B - wait1_cnt := 0.U - } - } - - when(wait2_reg) { - wait2_cnt := wait2_cnt + 1.U - when(wait2_cnt === 32.U) { - wait2_reg := false.B - wait2_cnt := 0.U - } - } - -// ----------------------------------------------------------------------------- -// SRAM returns data and passes to EX unit -// ----------------------------------------------------------------------------- - val both_valid = bankRespQueue0.io.deq.valid && bankRespQueue1.io.deq.valid - - io.ld_ex_o.valid := both_valid - when(both_valid) { - io.ld_ex_o.bits.op1 := bankRespQueue0.io.deq.bits.data - .asTypeOf(Vec(InputNum, UInt(inputWidth.W))) - io.ld_ex_o.bits.op2 := bankRespQueue1.io.deq.bits.data - .asTypeOf(Vec(InputNum, UInt(inputWidth.W))) - io.ld_ex_o.bits.iter := ld_ex_iter_reg - }.otherwise { - io.ld_ex_o.bits.iter := 0.U - io.ld_ex_o.bits.op1 := VecInit(Seq.fill(InputNum)(0.U(inputWidth.W))) - io.ld_ex_o.bits.op2 := VecInit(Seq.fill(InputNum)(0.U(inputWidth.W))) - } - - // Only dequeue and advance iter counter on successful handshake - bankRespQueue0.io.deq.ready := io.ld_ex_o.fire - bankRespQueue1.io.deq.ready := io.ld_ex_o.fire - - when(io.ld_ex_o.fire) { - ld_ex_iter_reg := ld_ex_iter_reg + 1.U - } - -// ----------------------------------------------------------------------------- -// Reset op1_iter_counter and return to idle state -// ----------------------------------------------------------------------------- - - when(state === busy && ld_ex_iter_reg === iter) { - state := idle - op1_iter_counter := 0.U - op2_iter_counter := 0.U - ld_ex_iter_reg := 0.U - } - -} diff --git a/examples/balls/vector/arch/src/main/scala/VecMat16.scala b/examples/balls/vector/arch/src/main/scala/VecMat16.scala deleted file mode 100644 index 75c28c12e..000000000 --- a/examples/balls/vector/arch/src/main/scala/VecMat16.scala +++ /dev/null @@ -1,190 +0,0 @@ -package examples.balls.vector - -import chisel3._ -import chisel3.util._ -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.top.GlobalConfig - -class VecMat16( - val b: GlobalConfig, - lane: Int, - inBW: Int, - outBW: Int, - inW: Int, - accW: Int) - extends Module { - require(lane == 16, "VecMat16 requires lane=16") - require(inW == 8, "VecMat16 requires int8 inputs") - require(accW == 32, "VecMat16 requires int32 accumulators") - require(inBW >= 2, "VecMat16 requires two read ports") - require(outBW >= 4, "VecMat16 requires four write ports") - - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(outBW, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - val sIdle :: sLoadAcc :: sReadReq :: sReadResp :: sWrite :: sDone :: Nil = - Enum(6) - val state = RegInit(sIdle) - - val robIdReg = RegInit(0.U(log2Up(b.frontend.rob_entries).W)) - val isSubReg = RegInit(false.B) - val subRobIdReg = RegInit(0.U(log2Up(b.frontend.sub_rob_depth * 4).W)) - val op1Bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val op2Bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val wrBank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val iterReg = RegInit(0.U(b.frontend.iter_len.W)) - val readRow = RegInit(0.U(b.frontend.iter_len.W)) - val aReqDone = RegInit(false.B) - val bReqDone = RegInit(false.B) - val aRespDone = RegInit(false.B) - val bRespDone = RegInit(false.B) - val aWord = RegInit(0.U(b.memDomain.bankWidth.W)) - val bWord = RegInit(0.U(b.memDomain.bankWidth.W)) - val acc = Reg(Vec(lane, Vec(lane, UInt(accW.W)))) - - val accLoad = Module(new VecMat16AccLoad(b, lane, accW)) - val store = Module(new VecMat16Store(b, lane, accW)) - accLoad.io.start := false.B - accLoad.io.wrBank := wrBank - accLoad.io.robId := robIdReg - store.io.start := false.B - store.io.wrBank := wrBank - store.io.robId := robIdReg - store.io.acc := acc - for (p <- 0 until 2) { - accLoad.io.bankRead(p).io.req.ready := io.bankRead(p).io.req.ready - accLoad.io.bankRead(p).io.resp.valid := io.bankRead(p).io.resp.valid - accLoad.io.bankRead(p).io.resp.bits := io.bankRead(p).io.resp.bits - } - for (g <- 0 until 4) { - store.io.bankWrite(g).io.req.ready := io.bankWrite(g).io.req.ready - store.io.bankWrite(g).io.resp.valid := io.bankWrite(g).io.resp.valid - store.io.bankWrite(g).io.resp.bits := io.bankWrite(g).io.resp.bits - } - - io.cmdReq.ready := state === sIdle - io.cmdResp.valid := state === sDone - io.cmdResp.bits.rob_id := robIdReg - io.cmdResp.bits.is_sub := isSubReg - io.cmdResp.bits.sub_rob_id := subRobIdReg - io.status.idle := state === sIdle - io.status.running := state =/= sIdle && state =/= sDone - VecMat16Tie.offRead(io.bankRead, robIdReg) - VecMat16Tie.offWrite(io.bankWrite, robIdReg, wrBank, b) - - switch(state) { - is(sIdle) { - when(io.cmdReq.fire) { - val cmd = io.cmdReq.bits.cmd - robIdReg := io.cmdReq.bits.rob_id - isSubReg := io.cmdReq.bits.is_sub - subRobIdReg := io.cmdReq.bits.sub_rob_id - op1Bank := cmd.op1_bank - op2Bank := cmd.op2_bank - wrBank := cmd.wr_bank - iterReg := cmd.iter - readRow := 0.U - aReqDone := false.B - bReqDone := false.B - aRespDone := false.B - bRespDone := false.B - assert(cmd.iter > 0.U, "VecMat16 iter must be > 0") - assert( - cmd.op1_col === 1.U && cmd.op2_col === 1.U && cmd.wr_col === 4.U, - "VecMat16 unsupported bank layout" - ) - accLoad.io.start := true.B - state := sLoadAcc - } - } - - is(sLoadAcc) { - for (p <- 0 until 2) { - VecMat16Tie.takeRead(io.bankRead(p), accLoad.io.bankRead(p)) - } - when(accLoad.io.done) { - acc := accLoad.io.acc - aReqDone := false.B - bReqDone := false.B - state := sReadReq - } - } - - is(sReadReq) { - io.bankRead(0).bank_id := op1Bank - io.bankRead(0).io.req.valid := !aReqDone - io.bankRead(0).io.req.bits.addr := readRow - io.bankRead(1).bank_id := op2Bank - io.bankRead(1).io.req.valid := !bReqDone - io.bankRead(1).io.req.bits.addr := readRow - when(io.bankRead(0).io.req.fire)(aReqDone := true.B) - when(io.bankRead(1).io.req.fire)(bReqDone := true.B) - when( - (aReqDone || io - .bankRead(0) - .io - .req - .fire) && (bReqDone || io.bankRead(1).io.req.fire) - ) { - aRespDone := false.B - bRespDone := false.B - state := sReadResp - } - } - - is(sReadResp) { - io.bankRead(0).bank_id := op1Bank - io.bankRead(1).bank_id := op2Bank - io.bankRead(0).io.resp.ready := !aRespDone - io.bankRead(1).io.resp.ready := !bRespDone - val aFire = io.bankRead(0).io.resp.fire - val bFire = io.bankRead(1).io.resp.fire - val nextA = Mux(aFire, io.bankRead(0).io.resp.bits.data, aWord) - val nextB = Mux(bFire, io.bankRead(1).io.resp.bits.data, bWord) - val haveA = aRespDone || aFire - val haveB = bRespDone || bFire - when(aFire)(aWord := io.bankRead(0).io.resp.bits.data) - when(bFire)(bWord := io.bankRead(1).io.resp.bits.data) - when(haveA)(aRespDone := true.B) - when(haveB)(bRespDone := true.B) - when(haveA && haveB) { - val aElems = nextA.asTypeOf(Vec(lane, UInt(inW.W))) - val bElems = nextB.asTypeOf(Vec(lane, UInt(inW.W))) - for { - i <- 0 until lane - j <- 0 until lane - } { - acc(i)(j) := (acc(i)(j).asSInt + aElems(i).asSInt * bElems( - j - ).asSInt).asUInt - } - when(readRow === iterReg - 1.U) { - store.io.start := true.B - state := sWrite - }.otherwise { - readRow := readRow + 1.U - aReqDone := false.B - bReqDone := false.B - state := sReadReq - } - } - } - - is(sWrite) { - for (g <- 0 until 4) { - VecMat16Tie.takeWrite(io.bankWrite(g), store.io.bankWrite(g)) - } - when(store.io.done)(state := sDone) - } - - is(sDone) { - when(io.cmdResp.fire)(state := sIdle) - } - } -} diff --git a/examples/balls/vector/arch/src/main/scala/VecMat16Store.scala b/examples/balls/vector/arch/src/main/scala/VecMat16Store.scala deleted file mode 100644 index 40f4e3614..000000000 --- a/examples/balls/vector/arch/src/main/scala/VecMat16Store.scala +++ /dev/null @@ -1,112 +0,0 @@ -package examples.balls.vector - -import chisel3._ -import chisel3.util._ -import framework.balldomain.blink.BankWrite -import framework.top.GlobalConfig - -class VecMat16Store(val b: GlobalConfig, lane: Int, accWidth: Int) extends Module { - - val io = IO(new Bundle { - val start = Input(Bool()) - val wrBank = Input(UInt(b.memDomain.vbankIdWidth.W)) - val robId = Input(UInt(log2Up(b.frontend.rob_entries).W)) - val acc = Input(Vec(lane, Vec(lane, UInt(accWidth.W)))) - val bankWrite = Vec(4, Flipped(new BankWrite(b))) - val done = Output(Bool()) - val busy = Output(Bool()) - }) - - val sIdle :: sReq :: sResp :: Nil = Enum(3) - val state = RegInit(sIdle) - - val writeRow = RegInit(0.U(log2Ceil(lane + 1).W)) - val writeIssued = RegInit(VecInit(Seq.fill(4)(false.B))) - val writeAcked = RegInit(VecInit(Seq.fill(4)(false.B))) - - def writeData(group: Int): UInt = { - val row = writeRow(log2Ceil(lane) - 1, 0) - val elems = (0 until 4).map(i => io.acc(row)(group * 4 + i)) - Cat(elems.reverse) - } - - def clr4() = VecInit(Seq.fill(4)(false.B)) - - io.done := false.B - io.busy := state =/= sIdle - - for (i <- 0 until 4) { - io.bankWrite(i).rob_id := io.robId - io.bankWrite(i).ball_id := 0.U - io.bankWrite(i).bank_id := io.wrBank - io.bankWrite(i).group_id := i.U - io.bankWrite(i).io.req.valid := false.B - io.bankWrite(i).io.req.bits.addr := 0.U - io.bankWrite(i).io.req.bits.data := 0.U - io.bankWrite(i).io.req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(false.B) - ) - io.bankWrite(i).io.resp.ready := false.B - } - - switch(state) { - is(sIdle) { - when(io.start) { - writeRow := 0.U - writeIssued := clr4() - writeAcked := clr4() - state := sReq - } - } - - is(sReq) { - for (g <- 0 until 4) { - io.bankWrite(g).bank_id := io.wrBank - io.bankWrite(g).group_id := g.U - io.bankWrite(g).io.req.valid := !writeIssued(g) - io.bankWrite(g).io.req.bits.addr := writeRow - io.bankWrite(g).io.req.bits.data := writeData(g) - io.bankWrite(g).io.req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(true.B) - ) - when(io.bankWrite(g).io.req.fire) { - writeIssued(g) := true.B - } - } - when( - (0 until 4) - .map(g => writeIssued(g) || io.bankWrite(g).io.req.fire) - .reduce(_ && _) - ) { - writeAcked := clr4() - state := sResp - } - } - - is(sResp) { - for (g <- 0 until 4) { - io.bankWrite(g).bank_id := io.wrBank - io.bankWrite(g).group_id := g.U - io.bankWrite(g).io.resp.ready := !writeAcked(g) - when(io.bankWrite(g).io.resp.fire) { - writeAcked(g) := true.B - } - } - when( - (0 until 4) - .map(g => writeAcked(g) || io.bankWrite(g).io.resp.fire) - .reduce(_ && _) - ) { - when(writeRow === (lane - 1).U) { - io.done := true.B - state := sIdle - }.otherwise { - writeRow := writeRow + 1.U - writeIssued := clr4() - writeAcked := clr4() - state := sReq - } - } - } - } -} diff --git a/examples/balls/vector/arch/src/main/scala/VecStoreUnit.scala b/examples/balls/vector/arch/src/main/scala/VecStoreUnit.scala deleted file mode 100644 index b38b1733d..000000000 --- a/examples/balls/vector/arch/src/main/scala/VecStoreUnit.scala +++ /dev/null @@ -1,147 +0,0 @@ -package examples.balls.vector - -import chisel3._ -import chisel3.util._ -import chisel3.stage._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.memdomain.backend.banks.SramWriteIO -import framework.top.GlobalConfig -import examples.balls.vector.configs.VectorBallParam - -class ctrl_st_req(b: GlobalConfig) extends Bundle { - val wr_bank = UInt(b.memDomain.vbankIdWidth.W) - val wr_bank_addr = UInt(log2Up(b.memDomain.bankEntries).W) - val iter = UInt(b.frontend.iter_len.W) -} - -class ex_st_req(b: GlobalConfig) extends Bundle { - val config = VectorBallParam(b) - val InputNum = config.lane - val accWidth = config.outputWidth - val rst = Vec(InputNum, UInt(accWidth.W)) - val iter = UInt(b.frontend.iter_len.W) -} - -class BankWriteEntry(b: GlobalConfig) extends Bundle { - val addr = UInt(log2Ceil(b.memDomain.bankEntries).W) - val data = UInt(b.memDomain.bankWidth.W) - val mask = Vec(b.memDomain.bankMaskLen, Bool()) -} - -@instantiable -class VecStoreUnit(val b: GlobalConfig) extends Module { - val config = VectorBallParam(b) - val InputNum = config.lane - val accWidth = config.outputWidth - - // Get bandwidth from config (use first VecBall mapping) - val ballMapping = b.ballDomain.ballIdMappings - .find(_.ballName == "VecBall") - .getOrElse( - throw new IllegalArgumentException("VecBall not found in config") - ) - - val outBW = ballMapping.outBW - - @public - val io = IO(new Bundle { - val ctrl_st_i = Flipped(Decoupled(new ctrl_st_req(b))) - val ex_st_i = Flipped(Decoupled(new ex_st_req(b))) - val bankWrite = Vec(outBW, Flipped(new SramWriteIO(b))) - val wr_bank_o = Output(UInt(b.memDomain.vbankIdWidth.W)) - val cmdResp_o = Valid(new Bundle { val commit = Bool() }) - }) - - val wr_bank = RegInit(0.U(b.memDomain.vbankIdWidth.W)) - val wr_bank_addr = RegInit(0.U(log2Up(b.memDomain.bankEntries).W)) - val iter = RegInit(0.U(b.frontend.iter_len.W)) - val iter_counter = RegInit(0.U(b.frontend.iter_len.W)) - val idle :: busy :: Nil = Enum(2) - val state = RegInit(idle) - - val writeQueues = VecInit( - Seq.fill(outBW)(Module(new Queue(new BankWriteEntry(b), 16)).io) - ) - -// ----------------------------------------------------------------------------- -// Set registers when Ctrl instruction arrives -// ----------------------------------------------------------------------------- - io.ctrl_st_i.ready := state === idle - - when(io.ctrl_st_i.fire) { - wr_bank := io.ctrl_st_i.bits.wr_bank - wr_bank_addr := io.ctrl_st_i.bits.wr_bank_addr - iter := (io.ctrl_st_i.bits.iter + 15.U) & (~15.U(b.frontend.iter_len.W)) - iter_counter := 0.U - state := busy - } - -// ----------------------------------------------------------------------------- -// Accept computation results from EX unit and push to write queues -// ----------------------------------------------------------------------------- - io.ex_st_i.ready := state === busy && writeQueues.forall(_.enq.ready) - - for (i <- 0 until outBW) { - writeQueues(i).enq.valid := false.B - writeQueues(i).enq.bits := DontCare - } - - when(io.ex_st_i.fire) { - for (i <- 0 until outBW) { - val elementsPerChannel = InputNum / outBW - val startIdx = i * elementsPerChannel - val endIdx = startIdx + elementsPerChannel - 1 - - val entry = Wire(new BankWriteEntry(b)) - entry.addr := wr_bank_addr + iter_counter(log2Ceil(InputNum) - 1, 0) - entry.data := Cat(io.ex_st_i.bits.rst.slice(startIdx, endIdx + 1).reverse) - entry.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(true.B)) - - writeQueues(i).enq.valid := true.B - writeQueues(i).enq.bits := entry - } - iter_counter := iter_counter + 1.U - } - -// ----------------------------------------------------------------------------- -// Drain write queues to bankWrite interface -// ----------------------------------------------------------------------------- - io.bankWrite.foreach { acc => - acc.req.valid := false.B - acc.req.bits.addr := 0.U - acc.req.bits.data := Cat(Seq.fill(accWidth / 8)(0.U(8.W))) - acc.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(false.B)) - acc.resp.ready := false.B - } - - for (i <- 0 until outBW) { - writeQueues(i).deq.ready := false.B - - when(writeQueues(i).deq.valid) { - io.bankWrite(i).req.valid := true.B - io.bankWrite(i).req.bits.addr := writeQueues(i).deq.bits.addr - io.bankWrite(i).req.bits.data := writeQueues(i).deq.bits.data - io.bankWrite(i).req.bits.mask := writeQueues(i).deq.bits.mask - writeQueues(i).deq.ready := io.bankWrite(i).req.ready - } - } - - // Output wr_bank for bank_id setting - io.wr_bank_o := wr_bank - -// ----------------------------------------------------------------------------- -// Reset iter counter, commit cmdResp, return to idle state -// ----------------------------------------------------------------------------- - val allQueuesEmpty = writeQueues.forall(q => !q.deq.valid) - val allDataEnqueued = state === busy && iter_counter >= iter - - when(allDataEnqueued && allQueuesEmpty) { - state := idle - io.cmdResp_o.valid := true.B - io.cmdResp_o.bits.commit := true.B - }.otherwise { - io.cmdResp_o.valid := false.B - io.cmdResp_o.bits.commit := false.B - } - -} diff --git a/examples/balls/vector/arch/src/main/scala/VecUnit.scala b/examples/balls/vector/arch/src/main/scala/VecUnit.scala deleted file mode 100644 index ebdb188ff..000000000 --- a/examples/balls/vector/arch/src/main/scala/VecUnit.scala +++ /dev/null @@ -1,49 +0,0 @@ -package examples.balls.vector - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -import framework.balldomain.blink.{BallStatus, BankRead, BankWrite} -import examples.balls.vector.configs.VectorBallParam -import framework.balldomain.rs.{BallRsComplete, BallRsIssue} -import framework.top.GlobalConfig - -@instantiable -class VecUnit(val b: GlobalConfig) extends Module { - val cfg = VectorBallParam(b) - val lane = cfg.lane - val inW = cfg.inputWidth - val accW = cfg.outputWidth - - require(lane == 16, "VecUnit vecmat16 requires lane=16") - require(inW == 8, "VecUnit vecmat16 requires int8 inputs") - require(accW == 32, "VecUnit vecmat16 requires int32 accumulators") - - val map = b.ballDomain.ballIdMappings - .find(_.ballName == "VecBall") - .getOrElse( - throw new IllegalArgumentException("VecBall not found in config") - ) - - val inBW = map.inBW - val outBW = map.outBW - require(inBW >= 2, "VecUnit vecmat16 requires two read ports") - require(outBW >= 4, "VecUnit vecmat16 requires four write ports") - - @public - val io = IO(new Bundle { - val cmdReq = Flipped(Decoupled(new BallRsIssue(b))) - val cmdResp = Decoupled(new BallRsComplete(b)) - val bankRead = Vec(inBW, Flipped(new BankRead(b))) - val bankWrite = Vec(outBW, Flipped(new BankWrite(b))) - val status = new BallStatus - }) - - val core = Module(new VecMat16(b, lane, inBW, outBW, inW, accW)) - core.io.cmdReq <> io.cmdReq - core.io.cmdResp <> io.cmdResp - for (i <- 0 until inBW) { core.io.bankRead(i) <> io.bankRead(i) } - for (i <- 0 until outBW) { core.io.bankWrite(i) <> io.bankWrite(i) } - io.status <> core.io.status -} diff --git a/examples/balls/vector/arch/src/main/scala/bond/vvv.scala b/examples/balls/vector/arch/src/main/scala/bond/vvv.scala deleted file mode 100644 index d63bb25ff..000000000 --- a/examples/balls/vector/arch/src/main/scala/bond/vvv.scala +++ /dev/null @@ -1,33 +0,0 @@ -//===----------------------------------------------------------------------===// -// VVV Bond: -// Input: Vec, Vec -// Output: Vec -//===----------------------------------------------------------------------===// - -package examples.balls.vector.bond - -import chisel3._ -import chisel3.util._ -import examples.balls.vector.configs.VectorBallParam -import chisel3.experimental.hierarchy.{instantiable, public} - -@instantiable -class VVV( - val config: VectorBallParam, - val inputWidth: Int, - val outputWidth: Int) - extends Bundle { - val lane = config.lane - - @public - val in = Flipped(Decoupled(new Bundle { - val in1 = Vec(lane, UInt(inputWidth.W)) - val in2 = Vec(lane, UInt(inputWidth.W)) - })) - - @public - val out = Decoupled(new Bundle { - val out = Vec(lane, UInt(outputWidth.W)) - }) - -} diff --git a/examples/balls/vector/arch/src/main/scala/thread/BaseThread.scala b/examples/balls/vector/arch/src/main/scala/thread/BaseThread.scala deleted file mode 100644 index f417786a1..000000000 --- a/examples/balls/vector/arch/src/main/scala/thread/BaseThread.scala +++ /dev/null @@ -1,13 +0,0 @@ -//===- BaseThread.scala - Level 1: Thread ---===// -package examples.balls.vector.thread - -import chisel3._ -import examples.balls.vector.configs.VectorBallParam - -//===----------------------------------------------------------------------===// -// BaseThread base class -//===----------------------------------------------------------------------===// -class BaseThread(val config: VectorBallParam, val opType: String) extends Module { - val io = IO(new Bundle {}) - val lane = config.lane -} diff --git a/examples/balls/vector/arch/src/main/scala/thread/CasThread.scala b/examples/balls/vector/arch/src/main/scala/thread/CasThread.scala deleted file mode 100644 index d2c4a6926..000000000 --- a/examples/balls/vector/arch/src/main/scala/thread/CasThread.scala +++ /dev/null @@ -1,19 +0,0 @@ -package examples.balls.vector.thread - -import chisel3._ -import chisel3.util._ -import examples.balls.vector.configs.VectorBallParam -import examples.balls.vector.bond.VVV -import examples.balls.vector.op.CascadeOp - -class CasThread(config: VectorBallParam) extends BaseThread(config, "cascade") { - val cascadeOp = Module(new CascadeOp(this.config)) - - val vvvBond = IO( - new VVV(this.config, this.config.outputWidth, this.config.outputWidth) - ) - - // Connect CascadeOp and VVVBond - cascadeOp.io.in <> vvvBond.in - cascadeOp.io.out <> vvvBond.out -} diff --git a/examples/balls/vector/arch/src/main/scala/thread/MulThread.scala b/examples/balls/vector/arch/src/main/scala/thread/MulThread.scala deleted file mode 100644 index 3ef6dd26f..000000000 --- a/examples/balls/vector/arch/src/main/scala/thread/MulThread.scala +++ /dev/null @@ -1,19 +0,0 @@ -package examples.balls.vector.thread - -import chisel3._ -import chisel3.util._ -import examples.balls.vector.configs.VectorBallParam -import examples.balls.vector.bond.VVV -import examples.balls.vector.op.MulOp - -class MulThread(config: VectorBallParam) extends BaseThread(config, "mul") { - val mulOp = Module(new MulOp(this.config)) - - val vvvBond = IO( - new VVV(this.config, this.config.inputWidth, this.config.outputWidth) - ) - - // Connect MulOp and VVVBond - mulOp.io.in <> vvvBond.in - mulOp.io.out <> vvvBond.out -} diff --git a/examples/balls/vector/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp b/examples/balls/vector/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp deleted file mode 100644 index 42b048874..000000000 --- a/examples/balls/vector/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,38 +0,0 @@ -//===- AssignPhysicalBankPatterns.cpp - Vector bank assignment patterns ---===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -void populateVecBallAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -} // namespace mlir::buddy - -namespace { - -class BankVecMat16Pattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankVecMat16Op op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), - op.getWrBank(), op.getIter(), op.getMode()); - rewriter.replaceOp(op, op.getWrBank()); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateVecBallAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)state; - patterns.add(patterns.getContext()); -} diff --git a/examples/balls/vector/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/balls/vector/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 7930ab3bd..000000000 --- a/examples/balls/vector/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball; -using namespace buddy::buckyball::legalize; - -namespace { -struct VecMat16Lowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(VecMat16Op op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - buckyball_target::requireBuckyballBall("VecBall"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getOp1BankId(), - adaptor.getOp2BankId(), adaptor.getWrBankId(), - adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getMode(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("VECMAT16"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateVecBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns, - bool stable, int64_t) { - (void)stable; - patterns.add(converter); -} - -void configureVecBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/examples/balls/vector/configs/VectorBallParam.scala b/examples/balls/vector/configs/VectorBallParam.scala deleted file mode 100644 index 0eaf9d325..000000000 --- a/examples/balls/vector/configs/VectorBallParam.scala +++ /dev/null @@ -1,32 +0,0 @@ -package examples.balls.vector.configs - -import framework.balldomain.configs.BallParamLoader -import framework.top.GlobalConfig -import upickle.default._ - -case class VectorBallParam( - InputNum: Int, - inputWidth: Int, - lane: Int, - outputWidth: Int, - numMulThreads: Int, - numCasThreads: Int) - -object VectorBallParam { - implicit val rw: ReadWriter[VectorBallParam] = macroRW - - private val ballName = "VecBall" - - def apply(b: GlobalConfig): VectorBallParam = { - val tbl = BallParamLoader.ballTable(b, ballName) - VectorBallParam( - InputNum = BallParamLoader.int(tbl, "InputNum"), - inputWidth = BallParamLoader.int(tbl, "inputWidth"), - lane = BallParamLoader.int(tbl, "lane"), - outputWidth = BallParamLoader.int(tbl, "outputWidth"), - numMulThreads = BallParamLoader.int(tbl, "numMulThreads"), - numCasThreads = BallParamLoader.int(tbl, "numCasThreads") - ) - } - -} diff --git a/examples/balls/vector/configs/default.toml b/examples/balls/vector/configs/default.toml deleted file mode 100644 index dd0404912..000000000 --- a/examples/balls/vector/configs/default.toml +++ /dev/null @@ -1,9 +0,0 @@ -# Vector Ball Configuration - -[ball] -InputNum = 16 -inputWidth = 8 -lane = 16 -outputWidth = 32 -numMulThreads = 16 -numCasThreads = 16 diff --git a/examples/balls/vector/emu/src/dispatch.rs b/examples/balls/vector/emu/src/dispatch.rs deleted file mode 100644 index 4b4eb6896..000000000 --- a/examples/balls/vector/emu/src/dispatch.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::inst::instruction::{BallInstruction, ExecContext}; - -use super::f64_vecmat16; - -const BALL_CLASS: &str = "examples.balls.vector.VecBall"; - -pub fn execute_known( - ball_class: &str, - funct: u32, - xs1: u64, - xs2: u64, - ctx: &mut ExecContext, -) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("VECMAT16") => Some(f64_vecmat16::VecMat16::exec(xs1, xs2, ctx)), - Some(_) | None => None, - } -} - -pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { - if ball_class != BALL_CLASS { - return None; - } - match crate::config::ball_domain::mnemonic_for_funct(funct).as_deref() { - Some("VECMAT16") => Some(f64_vecmat16::VecMat16::latency(xs1, xs2)), - Some(_) | None => None, - } -} diff --git a/examples/balls/vector/emu/src/lib.rs b/examples/balls/vector/emu/src/lib.rs deleted file mode 100644 index cd0540104..000000000 --- a/examples/balls/vector/emu/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub(crate) use crate::inst::{bank_matrix, decode, instruction}; - -#[path = "64_vecmat16.rs"] -mod f64_vecmat16; - -mod dispatch; - -pub use dispatch::{cycles_after_issue, execute_known}; diff --git a/examples/balls/vector/workloads/ctests/CMakeLists.txt b/examples/balls/vector/workloads/ctests/CMakeLists.txt deleted file mode 100644 index f1659a019..000000000 --- a/examples/balls/vector/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_buckyball_ctests( - vecmat16_atranspose_16x16_test.c -) diff --git a/examples/build.sc b/examples/build.sc new file mode 100644 index 000000000..d7baddb10 --- /dev/null +++ b/examples/build.sc @@ -0,0 +1,41 @@ +import mill._ +import mill.scalalib._ +import $file.^.arch.src.main.scala.framework.{build => framework_build} +import $file.^.arch.thirdparty.{build => thirdparty_build} +import $file.components.balls.{build => ball_components} +import $file.components.threads.{build => thread_components} + +import $file.^.arch.src.main.scala.framework.acc.balldomain.src.main.scala.bbus.{build => bbus_build} +import $file.^.arch.src.main.scala.framework.core.tbus.{build => tbus_build} + +val examplesRoot: os.Path = millSourcePath + +object library extends SbtModule { + override def scalaVersion = "2.13.16" + + override def moduleDeps = Seq( + framework_build.library, + thirdparty_build.hardfloat, + thirdparty_build.midas_target_utils, + thirdparty_build.gemmini + ) + + override def ivyDeps = Agg( + ivy"org.chipsalliance::chisel:6.7.0", + ivy"com.lihaoyi::upickle:3.3.1", + ivy"com.lihaoyi::mainargs:0.5.0", + ivy"org.json4s::json4s-jackson:4.0.5", + ivy"org.scala-graph::graph-core:1.13.5" + ) + + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") + + override def generatedSources = T { + bbus_build.registry(ball_components.sources(), examplesRoot / "components" / "balls", T.dest) ++ + tbus_build.registry(thread_components.sources(), examplesRoot / "components" / "threads", T.dest) + } + + override def sources = + T.sources(ball_components.sources() ++ thread_components.sources()) +} diff --git a/examples/chips/goban/arch/src/main/scala/sims/p2e/TargetConfigs.scala b/examples/chips/goban/arch/src/main/scala/sims/p2e/TargetConfigs.scala deleted file mode 100644 index 29e647378..000000000 --- a/examples/chips/goban/arch/src/main/scala/sims/p2e/TargetConfigs.scala +++ /dev/null @@ -1,10 +0,0 @@ -package sims.p2e - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -/** Goban on the P2E board: Linux scheduler core 0 and four compute cores. */ -class GobanP2ETarget extends P2ETarget("../examples/chips/goban/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) -} diff --git a/examples/chips/goban/arch/src/main/scala/sims/verilator/TargetConfigs.scala b/examples/chips/goban/arch/src/main/scala/sims/verilator/TargetConfigs.scala deleted file mode 100644 index 5d1d7a79a..000000000 --- a/examples/chips/goban/arch/src/main/scala/sims/verilator/TargetConfigs.scala +++ /dev/null @@ -1,12 +0,0 @@ -package sims.verilator - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -import sims.soc.SystemTarget - -/** Goban on the explicit System: Linux scheduler core 0 and four compute cores. */ -class GobanVerilatorTarget extends SystemTarget("../examples/chips/goban/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) -} diff --git a/examples/chips/goban/configs/chip.toml b/examples/chips/goban/configs/chip.toml deleted file mode 100644 index 30313c6a3..000000000 --- a/examples/chips/goban/configs/chip.toml +++ /dev/null @@ -1,10 +0,0 @@ -[designs] -include = "designs/goban.toml" - -[sims] -verilator = "sims.verilator.GobanVerilatorTarget" -p2e = "sims.p2e.GobanP2ETarget" - -[uvm] -balls = ["smatmul", "toint8", "int2fp", "transpose"] -ips = [] diff --git a/examples/chips/goban/configs/designs/goban.toml b/examples/chips/goban/configs/designs/goban.toml deleted file mode 100644 index aaa885c8e..000000000 --- a/examples/chips/goban/configs/designs/goban.toml +++ /dev/null @@ -1,4 +0,0 @@ -# Goban: a main tile only. - -[main_tile] -include = "tiles/default.toml" diff --git a/examples/chips/goban/configs/designs/tiles/default.toml b/examples/chips/goban/configs/designs/tiles/default.toml deleted file mode 100644 index 08bfefebf..000000000 --- a/examples/chips/goban/configs/designs/tiles/default.toml +++ /dev/null @@ -1,46 +0,0 @@ -# Goban main tile: five Linux-visible harts, with Buckyball on harts 1 through 4. - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -memBallChannelNum = 6 - -[sharedMem] -bankWidth = 128 -bankEntries = 1024 -enable = true -virtualBankCount = 64 -entries = 16384 -inputChannels = 28 -defaultGroupCount = 1 -memCycles = 10 - -[[cores]] -core_id = 0 -name = "scheduler" -include = "../../../../../cores/rocket/configs/default.toml" - -[[cores]] -core_id = 1 -name = "goban" -include = "../../../../../cores/goban/configs/default.toml" - -[[cores]] -core_id = 2 -name = "goban" -include = "../../../../../cores/goban/configs/default.toml" - -[[cores]] -core_id = 3 -name = "goban" -include = "../../../../../cores/goban/configs/default.toml" - -[[cores]] -core_id = 4 -name = "goban" -include = "../../../../../cores/goban/configs/default.toml" diff --git a/examples/chips/goban/kernel/workloads.toml b/examples/chips/goban/kernel/workloads.toml deleted file mode 100644 index 54401fcc3..000000000 --- a/examples/chips/goban/kernel/workloads.toml +++ /dev/null @@ -1,11 +0,0 @@ -[workloads] -tests = [ - "goban-goban-ctest-rvv_kernel-linux", - "goban-goban-ctest-rvv_silu-linux", - "goban-goban-ctest-rvv_swiglu-linux", - "goban-goban-ctest-rvv_snake-linux", - "goban-goban-ctest-rvv_mxfp8_quant-linux", - "goban-goban-ctest-rvv_matmul-linux", - "goban-goban-ctest-mvin_mvout_test-linux", - "goban-goban-ctest-mvin_mvout_stride_test-linux", -] diff --git a/examples/chips/goban/regression/batch/bemu/workloads-bare.toml b/examples/chips/goban/regression/batch/bemu/workloads-bare.toml deleted file mode 100644 index aef752407..000000000 --- a/examples/chips/goban/regression/batch/bemu/workloads-bare.toml +++ /dev/null @@ -1,16 +0,0 @@ -[workloads] -tests = [ - "goban-goban-ctest-rvv_kernel-baremetal", - "goban-goban-ctest-rvv_silu-baremetal", - "goban-goban-ctest-rvv_swiglu-baremetal", - "goban-goban-ctest-rvv_snake-baremetal", - "goban-goban-ctest-rvv_mxfp8_quant-baremetal", - "goban-goban-ctest-rvv_matmul-baremetal", - "goban-goban-ctest-mvin_mvout_test-baremetal", - "goban-goban-ctest-mvin_mvout_stride_test-baremetal", - "goban-goban-ctest-chip-mvover_rows-multicore-baremetal", - "goban-goban-ctest-core-transpose_smatmul_quant_pipeline-multicore-baremetal", - "goban-scheduler-soctest-clint_timer_interrupt-baremetal", - "goban-scheduler-soctest-clint_software_interrupt-baremetal", - "goban-scheduler-soctest-plic_register_access-baremetal", -] diff --git a/examples/chips/goban/regression/batch/bemu/workloads-linux.toml b/examples/chips/goban/regression/batch/bemu/workloads-linux.toml deleted file mode 100644 index d949632c3..000000000 --- a/examples/chips/goban/regression/batch/bemu/workloads-linux.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "fw_payload-h5-goban-linux.elf", -] diff --git a/examples/chips/goban/regression/batch/p2e/workloads-linux.toml b/examples/chips/goban/regression/batch/p2e/workloads-linux.toml deleted file mode 100644 index f63c2dc2f..000000000 --- a/examples/chips/goban/regression/batch/p2e/workloads-linux.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "fw_payload-h5-goban-linux.hex", -] diff --git a/examples/chips/goban/regression/batch/verilator/workloads-bare.toml b/examples/chips/goban/regression/batch/verilator/workloads-bare.toml deleted file mode 100644 index aef752407..000000000 --- a/examples/chips/goban/regression/batch/verilator/workloads-bare.toml +++ /dev/null @@ -1,16 +0,0 @@ -[workloads] -tests = [ - "goban-goban-ctest-rvv_kernel-baremetal", - "goban-goban-ctest-rvv_silu-baremetal", - "goban-goban-ctest-rvv_swiglu-baremetal", - "goban-goban-ctest-rvv_snake-baremetal", - "goban-goban-ctest-rvv_mxfp8_quant-baremetal", - "goban-goban-ctest-rvv_matmul-baremetal", - "goban-goban-ctest-mvin_mvout_test-baremetal", - "goban-goban-ctest-mvin_mvout_stride_test-baremetal", - "goban-goban-ctest-chip-mvover_rows-multicore-baremetal", - "goban-goban-ctest-core-transpose_smatmul_quant_pipeline-multicore-baremetal", - "goban-scheduler-soctest-clint_timer_interrupt-baremetal", - "goban-scheduler-soctest-clint_software_interrupt-baremetal", - "goban-scheduler-soctest-plic_register_access-baremetal", -] diff --git a/examples/chips/goban/workloads/CMakeLists.txt b/examples/chips/goban/workloads/CMakeLists.txt deleted file mode 100644 index 2b27cb711..000000000 --- a/examples/chips/goban/workloads/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -add_custom_target(chip-ctests-build) - -add_subdirectory(ctests) - -set(BUCKYBALL_CTEST_TARGET goban) -set(BUCKYBALL_CTEST_GROUP_TARGET chip-ctests-build) -add_subdirectory( - ${BUCKYBALL_REPO_ROOT}/examples/balls/smatmul/workloads/ctests - ${CMAKE_CURRENT_BINARY_DIR}/smatmul-ctests) - -add_custom_target(chip-workloads-build ALL - DEPENDS - chip-ctests-build - mlirtest-workloads-build - soctests-build - COMMENT "Building goban workloads" - VERBATIM) diff --git a/examples/chips/goban/workloads/ctests/CMakeLists.txt b/examples/chips/goban/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 0a030fec3..000000000 --- a/examples/chips/goban/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -buckyball_ctest_name(MVOVER_NAME goban chip-mvover_rows) -set(MVOVER_ELF ${MVOVER_NAME}-multicore-baremetal) -add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${MVOVER_ELF} - COMMAND ${RISCV_GNU_TOOLCHAIN}/bin/riscv64-unknown-elf-g++ - -O2 -static -march=rv64gc -mcmodel=medany -fno-builtin-printf - -specs=nano.specs -specs=nosys.specs -nostartfiles -Wl,-T,${BBSIM_LD} - -I${WORKLOAD_LIB_DIR} -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include - -I${WORKLOAD_LIB_DIR}/bbsw/baremetal/multicore - -I${BUCKYBALL_ISA_DIR}/chip -I${BUCKYBALL_ISA_DIR}/goban - -o ${MVOVER_ELF} ${WORKLOAD_LIB_DIR}/bbsw/baremetal/start.S - ${CMAKE_CURRENT_SOURCE_DIR}/mvover_rows.c - ${WORKLOAD_LIB_DIR}/buckyball.c ${BBHW_MEM_C} - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/mvover_rows.c - ${WORKLOAD_LIB_DIR}/bbsw/baremetal/start.S ${BBHW_MEM_C} - ${WORKLOAD_LIB_DIR}/bbhw/isa/13_mvover.c - ${BUCKYBALL_ISA_DIR}/goban/params.h ${BUCKYBALL_ISA_DIR}/chip/topology.h - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) -add_custom_target(${MVOVER_NAME}-ctest-build DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${MVOVER_ELF}) -add_dependencies(chip-ctests-build ${MVOVER_NAME}-ctest-build) diff --git a/examples/chips/multi-rocket/arch/src/main/scala/sims/verilator/MultiRocketVerilatorTarget.scala b/examples/chips/multi-rocket/arch/src/main/scala/sims/verilator/MultiRocketVerilatorTarget.scala deleted file mode 100644 index 84a0fe21e..000000000 --- a/examples/chips/multi-rocket/arch/src/main/scala/sims/verilator/MultiRocketVerilatorTarget.scala +++ /dev/null @@ -1,11 +0,0 @@ -package sims.verilator - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -import sims.soc.SystemTarget - -class MultiRocketVerilatorTarget extends SystemTarget("../examples/chips/multi-rocket/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) -} diff --git a/examples/chips/multi-rocket/configs/chip.toml b/examples/chips/multi-rocket/configs/chip.toml deleted file mode 100644 index bbade4444..000000000 --- a/examples/chips/multi-rocket/configs/chip.toml +++ /dev/null @@ -1,9 +0,0 @@ -[designs] -include = "designs/multi-rocket.toml" - -[sims] -verilator = "sims.verilator.MultiRocketVerilatorTarget" - -[uvm] -balls = [] -ips = ["axis"] diff --git a/examples/chips/multi-rocket/configs/designs/multi-rocket.toml b/examples/chips/multi-rocket/configs/designs/multi-rocket.toml deleted file mode 100644 index 2eb85b68b..000000000 --- a/examples/chips/multi-rocket/configs/designs/multi-rocket.toml +++ /dev/null @@ -1,4 +0,0 @@ -# Multi-Rocket: a main tile only. - -[main_tile] -include = "tiles/32core.toml" diff --git a/examples/chips/multi-rocket/configs/designs/tiles/32core.toml b/examples/chips/multi-rocket/configs/designs/tiles/32core.toml deleted file mode 100644 index c55ecc9a4..000000000 --- a/examples/chips/multi-rocket/configs/designs/tiles/32core.toml +++ /dev/null @@ -1,23 +0,0 @@ -# multi-rocket Tile (32 Rocket-only cores) - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -[sharedMem] -bankWidth = 128 -bankEntries = 16 -enable = false -entries = 16384 -inputChannels = 32 -defaultGroupCount = 1 -memCycles = 10 - -[coreTemplate] -core_ids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] -include = "../../../../../cores/rocket/configs/default.toml" -count = 32 diff --git a/examples/chips/multi-rocket/kernel/workloads.toml b/examples/chips/multi-rocket/kernel/workloads.toml deleted file mode 100644 index 12a9b1c7c..000000000 --- a/examples/chips/multi-rocket/kernel/workloads.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "multi-rocket-rocket-ctest-cpu_arithmetic-linux", -] diff --git a/examples/chips/multi-rocket/regression/batch/bemu/workloads-bare.toml b/examples/chips/multi-rocket/regression/batch/bemu/workloads-bare.toml deleted file mode 100644 index a11146aed..000000000 --- a/examples/chips/multi-rocket/regression/batch/bemu/workloads-bare.toml +++ /dev/null @@ -1,24 +0,0 @@ -[workloads] -tests = [ - "multi-rocket-rocket-ctest-check-embench_sglib-combined-baremetal", - "multi-rocket-rocket-ctest-check-embench_matmult-int-baremetal", - "multi-rocket-rocket-ctest-cpu_arithmetic-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_aha-mont64-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_crc32-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_edn-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_huffbench-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_matmult-int-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_minver-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nbody-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nettle-aes-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nettle-sha256-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nsichneu-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_picojpeg-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_qrduino-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_sglib-combined-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_slre-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_st-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_statemate-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_ud-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_wikisort-baremetal", -] diff --git a/examples/chips/multi-rocket/regression/batch/p2e/workloads-bare.toml b/examples/chips/multi-rocket/regression/batch/p2e/workloads-bare.toml deleted file mode 100644 index eb3873881..000000000 --- a/examples/chips/multi-rocket/regression/batch/p2e/workloads-bare.toml +++ /dev/null @@ -1,21 +0,0 @@ -[workloads] -tests = [ - "multi-rocket-rocket-ctest-benchmark-embench_aha-mont64-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_crc32-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_edn-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_huffbench-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_matmult-int-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_minver-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_nbody-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_nettle-aes-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_nettle-sha256-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_nsichneu-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_picojpeg-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_qrduino-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_sglib-combined-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_slre-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_st-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_statemate-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_ud-baremetal.hex", - "multi-rocket-rocket-ctest-benchmark-embench_wikisort-baremetal.hex", -] diff --git a/examples/chips/multi-rocket/regression/batch/verilator/workloads-bare.toml b/examples/chips/multi-rocket/regression/batch/verilator/workloads-bare.toml deleted file mode 100644 index 8dd5e23e6..000000000 --- a/examples/chips/multi-rocket/regression/batch/verilator/workloads-bare.toml +++ /dev/null @@ -1,18 +0,0 @@ -[workloads] -tests = [ - "multi-rocket-rocket-ctest-cpu_arithmetic-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_aha-mont64-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_crc32-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_edn-baremetal", - "multi-rocket-rocket-ctest-check-embench_matmult-int-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_minver-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nbody-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nettle-aes-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nettle-sha256-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_nsichneu-baremetal", - "multi-rocket-rocket-ctest-check-embench_sglib-combined-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_slre-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_st-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_statemate-baremetal", - "multi-rocket-rocket-ctest-benchmark-embench_ud-baremetal", -] diff --git a/examples/chips/multi-rocket/workloads/CMakeLists.txt b/examples/chips/multi-rocket/workloads/CMakeLists.txt deleted file mode 100644 index d8327a6da..000000000 --- a/examples/chips/multi-rocket/workloads/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -add_custom_target(chip-ctests-build) -set(BUCKYBALL_CTEST_GROUP_TARGET chip-ctests-build) -set(BUCKYBALL_CTEST_TARGET rocket) -add_buckyball_ctest(cpu_arithmetic.c) - -add_custom_target(chip-workloads-build ALL - DEPENDS - chip-ctests-build - mlirtest-workloads-build - soctests-build - COMMENT "Building multi-rocket workloads" - VERBATIM) diff --git a/examples/chips/omni/arch/src/main/scala/omni_tile/OmniTile.scala b/examples/chips/omni/arch/src/main/scala/omni_tile/OmniTile.scala deleted file mode 100644 index 56a3a530b..000000000 --- a/examples/chips/omni/arch/src/main/scala/omni_tile/OmniTile.scala +++ /dev/null @@ -1,21 +0,0 @@ -package examples.chips.omni.omni_tile - -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.system.GlobalConfigOps._ -import framework.system.core.CoreFactory -import framework.system.tile.{ComputeTilePlatform, TileConnection, TileEndpoint, TileFactory} - -@instantiable -class OmniTile(b: GlobalConfig) extends TileEndpoint(b.tileLinkParams, false) { - val p = b.tileParams - val platform = Instantiate(new ComputeTilePlatform(p)) - val controllerIndex = p.controller.get - val controller = CoreFactory.rocket(b.forCore(controllerIndex)) - val workers = p.cores.indices.filterNot(_ == controllerIndex).map(i => CoreFactory.ant(b.forCore(i))) - TileConnection.compute(platform, tlink, controller.clink, workers.map(_.clink)) -} - -object OmniTile extends TileFactory { - def instantiate(b: GlobalConfig): Instance[TileEndpoint] = Instantiate(new OmniTile(b)) -} diff --git a/examples/chips/omni/arch/src/main/scala/poly_tile/PolyTile.scala b/examples/chips/omni/arch/src/main/scala/poly_tile/PolyTile.scala deleted file mode 100644 index aff20b7e2..000000000 --- a/examples/chips/omni/arch/src/main/scala/poly_tile/PolyTile.scala +++ /dev/null @@ -1,21 +0,0 @@ -package examples.chips.omni.poly_tile - -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.system.GlobalConfigOps._ -import framework.system.core.CoreFactory -import framework.system.tile.{ComputeTilePlatform, TileConnection, TileEndpoint, TileFactory} - -@instantiable -class PolyTile(b: GlobalConfig) extends TileEndpoint(b.tileLinkParams, false) { - val p = b.tileParams - val platform = Instantiate(new ComputeTilePlatform(p)) - val controllerIndex = p.controller.get - val controller = CoreFactory.rocket(b.forCore(controllerIndex)) - val workers = p.cores.indices.filterNot(_ == controllerIndex).map(i => CoreFactory.ant(b.forCore(i))) - TileConnection.compute(platform, tlink, controller.clink, workers.map(_.clink)) -} - -object PolyTile extends TileFactory { - def instantiate(b: GlobalConfig): Instance[TileEndpoint] = Instantiate(new PolyTile(b)) -} diff --git a/examples/chips/omni/arch/src/main/scala/sims/OmniVerilatorTarget.scala b/examples/chips/omni/arch/src/main/scala/sims/OmniVerilatorTarget.scala deleted file mode 100644 index 8a17738ca..000000000 --- a/examples/chips/omni/arch/src/main/scala/sims/OmniVerilatorTarget.scala +++ /dev/null @@ -1,9 +0,0 @@ -package sims.verilator - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{Device, System, SystemParams} -import sims.soc.SystemTarget - -class OmniVerilatorTarget extends SystemTarget("../examples/chips/omni/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = Instantiate(new Device(p.toGlobalConfig)) -} diff --git a/examples/chips/omni/configs/chip.toml b/examples/chips/omni/configs/chip.toml deleted file mode 100644 index ca92f2852..000000000 --- a/examples/chips/omni/configs/chip.toml +++ /dev/null @@ -1,9 +0,0 @@ -[designs] -include = "designs/omni.toml" - -[sims] -verilator = "sims.verilator.OmniVerilatorTarget" - -[uvm] -balls = ["smatmul", "toint8", "int2fp", "transpose"] -ips = [] diff --git a/examples/chips/omni/configs/designs/omni.toml b/examples/chips/omni/configs/designs/omni.toml deleted file mode 100644 index a57ce721c..000000000 --- a/examples/chips/omni/configs/designs/omni.toml +++ /dev/null @@ -1,11 +0,0 @@ -# Fixed main position, four Poly tiles and one Goban-operator Omni tile. -[main_tile] -include = "tiles/main.toml" - -[[tiles]] -include = "tiles/poly.toml" -count = 4 - -[[tiles]] -include = "tiles/omni.toml" -count = 1 diff --git a/examples/chips/omni/configs/designs/tiles/main.toml b/examples/chips/omni/configs/designs/tiles/main.toml deleted file mode 100644 index ba059a857..000000000 --- a/examples/chips/omni/configs/designs/tiles/main.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Omni main tile: one Linux core. Add cores, Buckyball and sharedMem here as the -# main tile grows (for example cross-tile all-reduce). - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -[sharedMem] -bankWidth = 128 -bankEntries = 16 -enable = false -entries = 16384 -inputChannels = 1 -defaultGroupCount = 1 - -[[cores]] -core_id = 0 -name = "main" -include = "../../../../../cores/rocket/configs/default.toml" diff --git a/examples/chips/omni/configs/designs/tiles/omni.toml b/examples/chips/omni/configs/designs/tiles/omni.toml deleted file mode 100644 index ca7c1c0e8..000000000 --- a/examples/chips/omni/configs/designs/tiles/omni.toml +++ /dev/null @@ -1,53 +0,0 @@ -controllerCore = 0 -factory = "examples.chips.omni.omni_tile.OmniTile" -# One system controller and four tile-local Ant contexts with Goban operators. - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -memBallChannelNum = 6 - -[tss] -base = 0x20000 -bytes = 16384 -dataBits = 128 - -[sharedMem] -bankWidth = 128 -bankEntries = 1024 -enable = true -virtualBankCount = 64 -entries = 16384 -inputChannels = 28 -defaultGroupCount = 1 -memCycles = 10 - -[[cores]] -core_id = 0 -name = "controller" -include = "../../../../../cores/rocket/configs/default.toml" - -[[cores]] -core_id = 1 -name = "goban" -include = "../../../../../cores/goban/configs/ant.toml" - -[[cores]] -core_id = 2 -name = "goban" -include = "../../../../../cores/goban/configs/ant.toml" - -[[cores]] -core_id = 3 -name = "goban" -include = "../../../../../cores/goban/configs/ant.toml" - -[[cores]] -core_id = 4 -name = "goban" -include = "../../../../../cores/goban/configs/ant.toml" diff --git a/examples/chips/omni/configs/designs/tiles/poly.toml b/examples/chips/omni/configs/designs/tiles/poly.toml deleted file mode 100644 index 08c25906d..000000000 --- a/examples/chips/omni/configs/designs/tiles/poly.toml +++ /dev/null @@ -1,83 +0,0 @@ -controllerCore = 0 -factory = "examples.chips.omni.poly_tile.PolyTile" -# Omni Poly tile: controller core 0, two Attention cores and eight FFN cores. - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -memBallChannelNum = 11 - -[tss] -base = 0x20000 -bytes = 16384 -dataBits = 128 - -[sharedMem] -bankWidth = 128 -bankEntries = 1024 -enable = true -virtualBankCount = 40 -entries = 16384 -inputChannels = 10 -defaultGroupCount = 1 -memCycles = 10 - -[[cores]] -core_id = 0 -name = "controller" -include = "../../../../../cores/rocket/configs/default.toml" - -[[cores]] -core_id = 1 -name = "attention" -include = "../../../../../cores/attention/configs/default.toml" - -[[cores]] -core_id = 2 -name = "attention" -include = "../../../../../cores/attention/configs/default.toml" - -[[cores]] -core_id = 3 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 4 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 5 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 6 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 7 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 8 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 9 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 10 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" diff --git a/examples/chips/omni/configs/system.toml b/examples/chips/omni/configs/system.toml deleted file mode 100644 index ec9dc02e4..000000000 --- a/examples/chips/omni/configs/system.toml +++ /dev/null @@ -1,44 +0,0 @@ -model = "Qwen/Qwen3-Omni-30B-A3B-Instruct" -revision = "26291f793822fb6be9555850f06dfe95f2d7e695" - -[thinker] -tensor_tiles = [0, 1, 2, 3] - -[media] -vision_tile = 0 -audio_tile = 1 -talker_tile = 2 -code2wav_tile = 3 - -[vm] -chip = 4 -enabled = true - -[[chips]] -id = 0 -role = "thinker" -layers = [0, 12] -memory_mib = 16384 - -[[chips]] -id = 1 -role = "thinker" -layers = [12, 24] -memory_mib = 16384 - -[[chips]] -id = 2 -role = "thinker" -layers = [24, 36] -memory_mib = 16384 - -[[chips]] -id = 3 -role = "thinker" -layers = [36, 48] -memory_mib = 16384 - -[[chips]] -id = 4 -role = "media" -memory_mib = 16384 diff --git a/examples/chips/omni/emu/src/main.rs b/examples/chips/omni/emu/src/main.rs deleted file mode 100644 index 13cc7de63..000000000 --- a/examples/chips/omni/emu/src/main.rs +++ /dev/null @@ -1,41 +0,0 @@ -use bebop_bemu::root::{system, tile::run::Args}; -use clap::{CommandFactory, FromArgMatches, Parser}; - -#[derive(Parser)] -struct SystemArgs { - #[command(flatten)] - core: Args, - #[arg(long, requires = "pk")] - host_io: bool, -} - -fn main() { - let config: toml::Value = - toml::from_str(include_str!("../../configs/system.toml")).expect("Omni system config"); - let chips = config["chips"].as_array().expect("Omni chips"); - let memory = chips[0]["memory_mib"].as_integer().expect("DDR capacity"); - for (id, chip) in chips.iter().enumerate() { - assert_eq!(chip["id"].as_integer(), Some(id as i64)); - assert_eq!(chip["memory_mib"].as_integer(), Some(memory)); - } - let memory = Box::leak(memory.to_string().into_boxed_str()); - let matches = SystemArgs::command() - .mut_arg("memory_mib", |arg| arg.default_value(&*memory)) - .get_matches(); - let args = SystemArgs::from_arg_matches(&matches).expect("Omni arguments"); - let vm_chip = if config["vm"]["enabled"].as_bool().expect("VM enabled") { - Some( - config["vm"]["chip"] - .as_integer() - .expect("VM chip") - .try_into() - .expect("VM chip range"), - ) - } else { - None - }; - if let Err(error) = system::run(args.core, chips.len(), vm_chip, args.host_io) { - eprintln!("error: {error}"); - std::process::exit(1); - } -} diff --git a/examples/chips/omni/kernel/workloads.toml b/examples/chips/omni/kernel/workloads.toml deleted file mode 100644 index 394c9a2a7..000000000 --- a/examples/chips/omni/kernel/workloads.toml +++ /dev/null @@ -1,8 +0,0 @@ -[workloads] -tests = [ - "omni-attention-ctest-chip-mailbox-linux", - "omni-attention-ctest-chip-process-linux", - "omni-ffn-mlirtest-chip-moe-linux", - "omni-ffn-mlirtest-chip-moe-batch-linux", - "omni-attention-mlirtest-chip-attention-linux", -] diff --git a/examples/chips/omni/regression/batch/bemu/workloads-bare.toml b/examples/chips/omni/regression/batch/bemu/workloads-bare.toml deleted file mode 100644 index 433443b17..000000000 --- a/examples/chips/omni/regression/batch/bemu/workloads-bare.toml +++ /dev/null @@ -1,6 +0,0 @@ -[workloads] -tests = [ - "omni-attention-ctest-chip-interconnect-baremetal", - "omni-attention-ctest-chip-reservation-baremetal", - "omni-attention-ctest-chip-backpressure-baremetal", -] diff --git a/examples/chips/omni/regression/batch/bemu/workloads-linux.toml b/examples/chips/omni/regression/batch/bemu/workloads-linux.toml deleted file mode 100644 index 18b570832..000000000 --- a/examples/chips/omni/regression/batch/bemu/workloads-linux.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "fw_payload-h5-omni-linux.elf", -] diff --git a/examples/chips/omni/regression/batch/p2e/workloads-linux.toml b/examples/chips/omni/regression/batch/p2e/workloads-linux.toml deleted file mode 100644 index d0e89cac8..000000000 --- a/examples/chips/omni/regression/batch/p2e/workloads-linux.toml +++ /dev/null @@ -1,4 +0,0 @@ -[workloads] -tests = [ - "fw_payload-h5-omni-linux.hex", -] diff --git a/examples/chips/omni/tools/weights.py b/examples/chips/omni/tools/weights.py deleted file mode 100644 index 8941cb669..000000000 --- a/examples/chips/omni/tools/weights.py +++ /dev/null @@ -1,200 +0,0 @@ -import argparse -import dataclasses -import json -import tomllib -from collections import defaultdict -from pathlib import Path - -from huggingface_hub import HfApi - -ROOT = Path(__file__).resolve().parents[1] - - -def plan(metadata, config): - chips = config["chips"] - stages = [chip for chip in chips if chip["role"] == "thinker"] - media = [chip for chip in chips if chip["role"] == "media"] - tiles = config["thinker"]["tensor_tiles"] - if not stages or len(media) != 1 or tiles != [0, 1, 2, 3]: - raise ValueError("Qwen3 Omni placement requires TP4 tiles and one media chip") - names = [ - name for info in metadata["files_metadata"].values() for name in info["tensors"] - ] - layers = ( - max( - int(name.split(".")[3]) - for name in names - if name.startswith("thinker.model.layers.") - ) - + 1 - ) - covered = [layer for chip in stages for layer in range(*chip["layers"])] - if covered != list(range(layers)): - raise ValueError("Thinker chips must cover every layer exactly once in order") - totals = [ - { - "chip": c["id"], - "capacity_bytes": c["memory_mib"] << 20, - "checkpoint_bytes": 0, - "mxfp8_k128_estimate_bytes": 0, - "mxfp8_k512_estimate_bytes": 0, - } - for c in chips - ] - components = defaultdict(int) - tensors = [] - for filename, info in sorted(metadata["files_metadata"].items()): - for name, tensor in sorted(info["tensors"].items()): - shape = tensor["shape"] - original_bytes = tensor["data_offsets"][1] - tensor["data_offsets"][0] - component = ".".join(name.split(".")[:2]) - components[component] += original_bytes - entry = { - "name": name, - "file": filename, - "dtype": tensor["dtype"], - "shape": shape, - "source_bytes": original_bytes, - "placements": [], - } - if name.startswith(("thinker.model.", "thinker.lm_head.")): - if name.startswith("thinker.model.layers."): - layer = int(name.split(".")[3]) - chip = next( - chip - for chip in stages - if chip["layers"][0] <= layer < chip["layers"][1] - ) - elif name == "thinker.model.embed_tokens.weight": - chip = stages[0] - elif name in ("thinker.model.norm.weight", "thinker.lm_head.weight"): - chip = stages[-1] - else: - raise ValueError(f"No layer placement for {name}") - selected = [(chip, tile, rank) for rank, tile in enumerate(tiles)] - if name.endswith( - ( - "q_proj.weight", - "k_proj.weight", - "v_proj.weight", - "gate_proj.weight", - "up_proj.weight", - "embed_tokens.weight", - "lm_head.weight", - ) - ): - axis = 0 - elif name.endswith(("o_proj.weight", "down_proj.weight")): - axis = 1 - elif len(shape) == 1 or name.endswith("mlp.gate.weight"): - axis = None - else: - raise ValueError(f"No partition rule for {name}: {shape}") - elif name.startswith( - ("thinker.audio_tower.", "thinker.visual.", "talker.", "code2wav.") - ): - if name.startswith("thinker.audio_tower."): - tile = config["media"]["audio_tile"] - elif name.startswith("thinker.visual."): - tile = config["media"]["vision_tile"] - elif name.startswith("talker."): - tile = config["media"]["talker_tile"] - else: - tile = config["media"]["code2wav_tile"] - selected, axis = [(media[0], tile, 0)], None - else: - raise ValueError(f"Unknown Qwen3 Omni component: {name}") - for chip, tile, rank in selected: - shard_shape = list(shape) - start = 0 - if axis is not None: - if shape[axis] % len(tiles): - raise ValueError(f"Cannot split {name} along axis {axis}") - shard_shape[axis] //= len(tiles) - start = rank * shard_shape[axis] - size = original_bytes if axis is None else original_bytes // len(tiles) - amounts = { - "checkpoint_bytes": size, - "mxfp8_k128_estimate_bytes": size, - "mxfp8_k512_estimate_bytes": size, - } - # Estimate only Thinker projection/expert matrices. Encoders, routing, - # embeddings, Talker and waveform decode retain checkpoint precision. - quantized = axis is not None and not name.endswith( - ("embed_tokens.weight", "lm_head.weight") - ) - if quantized: - if len(shard_shape) != 2: - raise ValueError(f"Expected linear matrix: {name}") - rows, width = shard_shape - for tile_k in (128, 512): - elements = ((rows + 15) // 16 * 16) * ( - (width + tile_k - 1) // tile_k * tile_k - ) - amounts[f"mxfp8_k{tile_k}_estimate_bytes"] = ( - elements + elements // 32 - ) - totals[chip["id"]]["checkpoint_bytes"] += size - for tile_k in (128, 512): - key = f"mxfp8_k{tile_k}_estimate_bytes" - totals[chip["id"]][key] += amounts[key] - entry["placements"].append( - { - "chip": chip["id"], - "tile": tile, - "axis": axis, - "start": start, - "shape": shard_shape, - **amounts, - } - ) - tensors.append(entry) - for chip in totals: - for kind in ("checkpoint", "mxfp8_k128_estimate", "mxfp8_k512_estimate"): - chip[f"{kind}_remaining_bytes"] = ( - chip["capacity_bytes"] - chip[f"{kind}_bytes"] - ) - return { - "model": config["model"], - "revision": config["revision"], - "source_bytes": sum(components.values()), - "components": dict(components), - "chips": totals, - "tensors": tensors, - } - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--metadata", type=Path) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - with (ROOT / "configs/system.toml").open("rb") as source: - config = tomllib.load(source) - if args.metadata: - data = json.loads(args.metadata.read_text()) - if data["repo"] != config["model"] or data["revision"] != config["revision"]: - raise ValueError("Metadata does not match the pinned checkpoint") - metadata = data["metadata"] - else: - metadata = dataclasses.asdict( - HfApi().get_safetensors_metadata( - config["model"], revision=config["revision"] - ) - ) - result = plan(metadata, config) - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(result, indent=2) + "\n") - print( - f"Checkpoint tensors: {len(result['tensors'])}, bytes: {result['source_bytes']:,}" - ) - for chip in result["chips"]: - print( - f"chip {chip['chip']}: checkpoint {chip['checkpoint_bytes'] / 2**30:.3f} GiB; " - f"MXFP8 K128 estimate {chip['mxfp8_k128_estimate_bytes'] / 2**30:.3f} GiB; " - f"K512 estimate {chip['mxfp8_k512_estimate_bytes'] / 2**30:.3f} GiB" - ) - - -if __name__ == "__main__": - main() diff --git a/examples/chips/omni/workloads/CMakeLists.txt b/examples/chips/omni/workloads/CMakeLists.txt deleted file mode 100644 index d61bb11d1..000000000 --- a/examples/chips/omni/workloads/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_subdirectory(ctests) -add_custom_target(chip-ctests-build DEPENDS omni-interconnect-build) -add_custom_target(chip-workloads-build ALL - DEPENDS - chip-ctests-build - mlirtest-workloads-build - soctests-build -) diff --git a/examples/chips/omni/workloads/ctests/CMakeLists.txt b/examples/chips/omni/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 279ac2917..000000000 --- a/examples/chips/omni/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,52 +0,0 @@ -set(STARTUP ${WORKLOAD_LIB_DIR}/bbsw/baremetal/crt0.S) -set(SYSCALLS ${WORKLOAD_LIB_DIR}/bbsw/baremetal/syscalls.c) -set(LINKER_SCRIPT ${WORKLOAD_LIB_DIR}/bbsw/baremetal/bbsim.ld) -add_custom_target(omni-interconnect-build) -set(BUCKYBALL_CTEST_TARGET attention) -set(BUCKYBALL_CTEST_GROUP_TARGET omni-interconnect-build) -foreach(TEST interconnect reservation backpressure bounds peripherals) - buckyball_ctest_name(NAME attention chip-${TEST}) - set(BINARY ${NAME}-baremetal) - add_custom_command(OUTPUT ${BINARY} - COMMAND ${ELF_CC} -std=gnu11 -O2 -static -march=rv64gc -mcmodel=medany - -specs=nano.specs -specs=nosys.specs -nostartfiles - -Wl,-T,${LINKER_SCRIPT} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include - -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} - ${STARTUP} ${CMAKE_CURRENT_SOURCE_DIR}/${TEST}.c ${SYSCALLS} -o ${BINARY} - DEPENDS ${TEST}.c ${STARTUP} ${SYSCALLS} ${LINKER_SCRIPT} - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h - ${BUCKYBALL_REPO_ROOT}/stack/runtime/include/interconnect.h - VERBATIM) - add_custom_target(${NAME}-build DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${BINARY}) - add_dependencies(omni-interconnect-build ${NAME}-build) -endforeach() - -buckyball_ctest_name(NAME attention chip-process) -set(BINARY ${NAME}-linux) -add_custom_command(OUTPUT ${BINARY} - COMMAND ${LINUX_CC} -std=gnu11 -O2 -static -march=rv64gc - ${CMAKE_CURRENT_SOURCE_DIR}/process.c -o ${BINARY} - DEPENDS process.c - VERBATIM) -add_custom_target(${NAME}-build DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${BINARY}) -add_dependencies(omni-interconnect-build ${NAME}-build) - -buckyball_ctest_name(NAME attention chip-host-io) -set(BINARY ${NAME}-linux) -add_custom_command(OUTPUT ${BINARY} - COMMAND ${LINUX_CC} -std=gnu11 -O2 -static -march=rv64gc - ${CMAKE_CURRENT_SOURCE_DIR}/host_io.c -o ${BINARY} - DEPENDS host_io.c VERBATIM) -add_custom_target(${NAME}-build DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${BINARY}) -add_dependencies(omni-interconnect-build ${NAME}-build) - -buckyball_ctest_name(NAME attention chip-mailbox) -set(BINARY ${NAME}-linux) -add_custom_command(OUTPUT ${BINARY} - COMMAND ${LINUX_CC} -std=gnu11 -O2 -static -march=rv64gc - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include - ${CMAKE_CURRENT_SOURCE_DIR}/mailbox.c -o ${BINARY} - DEPENDS mailbox.c ${BUCKYBALL_REPO_ROOT}/stack/runtime/include/interconnect.h VERBATIM) -add_custom_target(${NAME}-build DEPENDS ${BINARY}) -add_dependencies(omni-interconnect-build ${NAME}-build) diff --git a/examples/chips/omni/workloads/ctests/backpressure.c b/examples/chips/omni/workloads/ctests/backpressure.c deleted file mode 100644 index 95c0a8149..000000000 --- a/examples/chips/omni/workloads/ctests/backpressure.c +++ /dev/null @@ -1,37 +0,0 @@ -#include -#include - -int main(void) { - uint64_t chip = link_read(LINK_CHIP_ID); - uint64_t memory = link_read(LINK_MEMORY_BYTES); - volatile uint64_t *source = (volatile uint64_t *)(0x80000000ULL + memory / 2); - volatile uint64_t *target = - (volatile uint64_t *)(0x80000000ULL + memory * 3 / 4); - uint64_t capacity = link_read(LINK_EVENT_CAPACITY); - if (chip > 1) - return 0; - if (chip == 0) { - for (uint64_t index = 0; index <= capacity; ++index) { - *source = 100 + index; - if (chip_send(1, (uintptr_t)source, (uintptr_t)(target + index), 8, - index)) - return 1; - } - } else { - // Fill the completion queue before allowing any consumption. - while (link_read(LINK_EVENT_COUNT) < capacity) { - } - if (link_read(LINK_EVENT_COUNT) != capacity) - return 2; - for (uint64_t index = 0; index <= capacity; ++index) { - while (!link_read(LINK_EVENT_COUNT)) { - } - if (link_read(LINK_EVENT_SOURCE) != 0 || - link_read(LINK_EVENT_TAG) != index || target[index] != 100 + index) - return 3; - link_write(LINK_EVENT_ACK, 1); - } - puts("completion queue backpressure and ordered delivery PASSED"); - } - return 0; -} diff --git a/examples/chips/omni/workloads/ctests/bounds.c b/examples/chips/omni/workloads/ctests/bounds.c deleted file mode 100644 index cde80d5a3..000000000 --- a/examples/chips/omni/workloads/ctests/bounds.c +++ /dev/null @@ -1,10 +0,0 @@ -#include - -int main(void) { - if (link_read(LINK_CHIP_ID) != 0) - return 0; - uint64_t bytes = link_read(LINK_MEMORY_BYTES); - // This descriptor must fail rather than writing past chip 1's DDR. - chip_send(1, 0x80000000ULL, 0x80000000ULL + bytes - 8, 16, 99); - return 1; -} diff --git a/examples/chips/omni/workloads/ctests/host_io.c b/examples/chips/omni/workloads/ctests/host_io.c deleted file mode 100644 index ded6f6349..000000000 --- a/examples/chips/omni/workloads/ctests/host_io.c +++ /dev/null @@ -1,28 +0,0 @@ -#include -#include - -int main(void) { - for (;;) { - uint64_t input; - size_t offset = 0; - while (offset < sizeof(input)) { - ssize_t count = read(0, (char *)&input + offset, sizeof(input) - offset); - if (count == 0) - return offset == 0 ? 0 : 1; - if (count < 0) - return 2; - offset += count; - } - if (input == UINT64_MAX) - return 11; - uint64_t output = input * input + 7; - offset = 0; - while (offset < sizeof(output)) { - ssize_t count = - write(1, (char *)&output + offset, sizeof(output) - offset); - if (count <= 0) - return 3; - offset += count; - } - } -} diff --git a/examples/chips/omni/workloads/ctests/interconnect.c b/examples/chips/omni/workloads/ctests/interconnect.c deleted file mode 100644 index 233c2aa43..000000000 --- a/examples/chips/omni/workloads/ctests/interconnect.c +++ /dev/null @@ -1,81 +0,0 @@ -#include -#include -#include - -#define CHECK(condition) \ - do { \ - if (!(condition)) { \ - fprintf(stderr, "chip %u line %d FAILED\n", (unsigned)chip, __LINE__); \ - return 1; \ - } \ - } while (0) - -int main(void) { - uint64_t chip = link_read(LINK_CHIP_ID); - uint64_t count = link_read(LINK_CHIP_COUNT); - CHECK(count > 1 && count <= 32); - const uint64_t memory = link_read(LINK_MEMORY_BYTES); - CHECK(memory > (4ULL << 30)); - // Same physical addresses in distinct DDRs, both above a 32-bit address - // range. - volatile uint64_t *source = (volatile uint64_t *)(0x80000000ULL + memory / 2); - volatile uint64_t *result = - (volatile uint64_t *)(0x80000000ULL + memory * 3 / 4); - for (unsigned i = 0; i < 32; ++i) - source[i] = 1000 * chip + i; - CHECK(chip_send((chip + 1) % count, (uintptr_t)source, (uintptr_t)result, 256, - 10 + chip) == 0); - // Thinker ranks publish disjoint partials into the last chip's DDR. - if (chip < count - 1) { - while (!link_read(LINK_EVENT_COUNT)) { - } - uint64_t previous = (chip + count - 1) % count; - CHECK(link_read(LINK_EVENT_SOURCE) == previous); - CHECK(link_read(LINK_EVENT_TAG) == 10 + previous); - CHECK(link_read(LINK_EVENT_BYTES) == 256); - __asm__ volatile("fence rw,rw" ::: "memory"); - for (unsigned i = 0; i < 32; ++i) - CHECK(result[i] == 1000 * previous + i); - link_write(LINK_EVENT_ACK, 1); - source[0] = chip + 1; - CHECK(chip_send(count - 1, (uintptr_t)source, - (uintptr_t)(result + 64 + chip), 8, 20 + chip) == 0); - while (!link_read(LINK_EVENT_COUNT)) { - } - CHECK(link_read(LINK_EVENT_SOURCE) == count - 1); - CHECK(link_read(LINK_EVENT_TAG) == 30); - CHECK(result[128] == count * (count - 1) / 2); - link_write(LINK_EVENT_ACK, 1); - } else { - unsigned received = 0; - unsigned ring_seen = 0; - for (unsigned i = 0; i < count; ++i) { - while (!link_read(LINK_EVENT_COUNT)) { - } - uint64_t peer = link_read(LINK_EVENT_SOURCE); - if (peer == count - 2 && link_read(LINK_EVENT_TAG) == 10 + count - 2) { - CHECK(peer == count - 2 && !ring_seen); - for (unsigned j = 0; j < 32; ++j) - CHECK(result[j] == 1000 * (count - 2) + j); - ring_seen = 1; - link_write(LINK_EVENT_ACK, 1); - continue; - } - CHECK(peer < count - 1 && !(received & (1u << peer))); - CHECK(link_read(LINK_EVENT_TAG) == 20 + peer); - received |= 1u << peer; - link_write(LINK_EVENT_ACK, 1); - } - CHECK(ring_seen && received == (1u << (count - 1)) - 1); - source[0] = 0; - for (unsigned peer = 0; peer < count - 1; ++peer) - source[0] += result[64 + peer]; - CHECK(source[0] == count * (count - 1) / 2); - for (unsigned peer = 0; peer < count - 1; ++peer) - CHECK(chip_send(peer, (uintptr_t)source, (uintptr_t)(result + 128), 8, - 30) == 0); - } - printf("chip %u: isolated DDR, ring DMA, gather and broadcast PASSED\n", - (unsigned)chip); - return 0; -} diff --git a/examples/chips/omni/workloads/ctests/mailbox.c b/examples/chips/omni/workloads/ctests/mailbox.c deleted file mode 100644 index bfc0b3487..000000000 --- a/examples/chips/omni/workloads/ctests/mailbox.c +++ /dev/null @@ -1,34 +0,0 @@ -#include -#include -#include -#include - -int main(void) { - _Atomic uint64_t *owner = (_Atomic uint64_t *)CHIP_LINK_BUFFER; - uint64_t zero = 0; - if (!atomic_compare_exchange_strong(owner, &zero, 1)) - return 0; - uint64_t chip = link_read(LINK_CHIP_ID), count = link_read(LINK_CHIP_COUNT); - uint64_t physical = link_read(LINK_BUFFER_ADDRESS); - if (count < 2 || link_read(LINK_BUFFER_BYTES) < 12288) - return 1; - volatile uint64_t *source = (volatile uint64_t *)(CHIP_LINK_BUFFER + 4096); - volatile uint64_t *destination = - (volatile uint64_t *)(CHIP_LINK_BUFFER + 8192); - for (unsigned i = 0; i < 32; ++i) - source[i] = chip * 100 + i; - if (chip_send((chip + 1) % count, physical + 4096, physical + 8192, 256, 42)) - return 2; - while (!link_read(LINK_EVENT_COUNT)) { - } - if (link_read(LINK_EVENT_SOURCE) != (chip + count - 1) % count || - link_read(LINK_EVENT_TAG) != 42 || link_read(LINK_EVENT_BYTES) != 256) - return 3; - __asm__ volatile("fence rw,rw" ::: "memory"); - for (unsigned i = 0; i < 32; ++i) - if (destination[i] != ((chip + count - 1) % count) * 100 + i) - return 4; - link_write(LINK_EVENT_ACK, 1); - printf("chip %lu: PK DDR INTERCONNECT PASSED\n", chip); - return 0; -} diff --git a/examples/chips/omni/workloads/ctests/peripherals.c b/examples/chips/omni/workloads/ctests/peripherals.c deleted file mode 100644 index 491d78a6a..000000000 --- a/examples/chips/omni/workloads/ctests/peripherals.c +++ /dev/null @@ -1,32 +0,0 @@ -#include -#include -#include - -int main(void) { - if (link_read(LINK_CHIP_ID) != 4) - return 0; - volatile uint8_t *uart = (volatile uint8_t *)0x60020000ULL; - puts("OMNI VM READY"); - while (!(uart[5] & 1)) { - } - if (uart[4] != 'g') - return 1; - volatile uint32_t *vga = (volatile uint32_t *)0x11000000ULL; - volatile uint32_t *pixels = (volatile uint32_t *)0x11001000ULL; - unsigned width = vga[0], height = vga[1]; - for (unsigned y = 0; y < height; ++y) - for (unsigned x = 0; x < width; ++x) - pixels[y * width + x] = - ((x * 255 / width) << 16) | ((y * 255 / height) << 8) | 0x40; - vga[3] = 1; - volatile uint32_t *speaker = (volatile uint32_t *)0x10002000ULL; - volatile int16_t *sample = (volatile int16_t *)0x10002004ULL; - for (unsigned i = 0; i < 256; ++i) - *sample = ((int)(i % 64) - 32) * 256; - speaker[3] = 1; - while (speaker[0]) { - } - speaker[3] = 0; - puts("OMNI VM TEXT FRAME AUDIO PASSED"); - return 0; -} diff --git a/examples/chips/omni/workloads/ctests/process.c b/examples/chips/omni/workloads/ctests/process.c deleted file mode 100644 index a43f22d01..000000000 --- a/examples/chips/omni/workloads/ctests/process.c +++ /dev/null @@ -1,39 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -int main(void) { - if (write(1, (void *)0x80000000ULL, 1) != -1 || errno != EFAULT) - return 7; - uint32_t seed; - if (getrandom(&seed, sizeof(seed), 0) != sizeof(seed)) - return 1; - for (unsigned iteration = 0; iteration < 32; ++iteration) { - size_t padding = (1 + ((seed >> (iteration % 24)) & 255)) * 4096; - unsigned char *allocation = malloc(padding + 8192); - if (!allocation) - return 2; - unsigned char *data = allocation + padding; - for (unsigned i = 0; i < 8192; ++i) - data[i] = (seed + iteration + i) & 255; - int fd = open("process.bin", O_CREAT | O_TRUNC | O_RDWR, 0600); - if (fd < 0 || write(fd, data, 8192) != 8192) - return 3; - memset(data, 0, 8192); - if (lseek(fd, 0, SEEK_SET) != 0 || read(fd, data, 8192) != 8192) - return 4; - for (unsigned i = 0; i < 8192; ++i) - if (data[i] != ((seed + iteration + i) & 255)) - return 5; - if (close(fd)) - return 6; - free(allocation); - } - puts("OMNI PROCESS MEMORY AND FILE ISOLATION PASSED"); - return 0; -} diff --git a/examples/chips/omni/workloads/ctests/reservation.c b/examples/chips/omni/workloads/ctests/reservation.c deleted file mode 100644 index c5880b467..000000000 --- a/examples/chips/omni/workloads/ctests/reservation.c +++ /dev/null @@ -1,47 +0,0 @@ -#include -#include - -int main(void) { - uint64_t chip = link_read(LINK_CHIP_ID); - const uint64_t memory = link_read(LINK_MEMORY_BYTES); - volatile uint64_t *source = (volatile uint64_t *)(0x80000000ULL + memory / 2); - volatile uint64_t *target = - (volatile uint64_t *)(0x80000000ULL + memory * 3 / 4); - if (chip > 1) - return 0; - if (chip == 0) { - uint64_t value, status; - *target = 10; - __asm__ volatile("lr.d %0, (%1)" : "=r"(value) : "r"(target) : "memory"); - __asm__ volatile("sc.d %0, %2, (%1)" - : "=r"(status) - : "r"(target), "r"(11ULL) - : "memory"); - if (value != 10 || status != 0 || *target != 11) - return 1; - __asm__ volatile("lr.d %0, (%1)" : "=r"(value) : "r"(target) : "memory"); - *source = 1; - if (chip_send(1, (uintptr_t)source, (uintptr_t)target, 8, 1)) - return 2; - while (!link_read(LINK_EVENT_COUNT)) { - } - if (link_read(LINK_EVENT_SOURCE) != 1) - return 3; - __asm__ volatile("sc.d %0, %2, (%1)" - : "=r"(status) - : "r"(target), "r"(99ULL) - : "memory"); - if (status == 0 || *target != 20) - return 4; - link_write(LINK_EVENT_ACK, 1); - puts("cross-chip DMA invalidates LR/SC reservation PASSED"); - } else { - while (!link_read(LINK_EVENT_COUNT)) { - } - link_write(LINK_EVENT_ACK, 1); - *source = 20; - if (chip_send(0, (uintptr_t)source, (uintptr_t)target, 8, 2)) - return 5; - } - return 0; -} diff --git a/examples/chips/omni/workloads/mlir_tests/CMakeLists.txt b/examples/chips/omni/workloads/mlir_tests/CMakeLists.txt deleted file mode 100644 index e52326652..000000000 --- a/examples/chips/omni/workloads/mlir_tests/CMakeLists.txt +++ /dev/null @@ -1,109 +0,0 @@ -set(REPO ${BUCKYBALL_REPO_ROOT}) -set(BUDDY ${REPO}/stack/compiler/thirdparty/buddy-mlir) -set(COMPILER ${BUDDY}/build/omni) -set(MODEL_DIR ${REPO}/stack/models/archs/buckyball/omni/qwen3-omni-30b-a3b-instruct) -set(DATA ${CMAKE_CURRENT_BINARY_DIR}/moe) -set(ATTENTION_DATA ${CMAKE_CURRENT_BINARY_DIR}/attention) -set(PYTHON ${REPO}/result/bin/python3) -set(MODEL_CONFIG ${REPO}/stack/models/models/qwen3-omni-30b-a3b-instruct/configs/model.toml) -set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${MODEL_CONFIG}) -set(CHECKPOINT_FILE ${CMAKE_CURRENT_BINARY_DIR}/checkpoint.txt) -add_custom_command(OUTPUT ${CHECKPOINT_FILE} - COMMAND ${PYTHON} -c - "import sys, tomllib; from pathlib import Path; from huggingface_hub import snapshot_download; p=tomllib.loads(Path(sys.argv[1]).read_text())['parameters']; Path(sys.argv[2]).write_text(snapshot_download(p['checkpoint'], revision=p['revision'], local_files_only=True))" - ${MODEL_CONFIG} ${CHECKPOINT_FILE} - DEPENDS ${MODEL_CONFIG} - WORKING_DIRECTORY ${REPO} VERBATIM) -file(GLOB_RECURSE MODEL_SOURCES CONFIGURE_DEPENDS ${MODEL_DIR}/trace/*.py ${MODEL_DIR}/permute/*.py ${MODEL_DIR}/quant/*.py) -list(APPEND MODEL_SOURCES ${BUDDY}/frontend/Python/ops/tosa.py - ${COMPILER}/python_packages/buddy/compiler/ops/tosa.py) -list(APPEND MODEL_SOURCES ${REPO}/examples/balls/mxmm/compiler/python/fp32.py) -add_custom_command(OUTPUT ${DATA}/router.mlir ${DATA}/router-forward.mlir - ${DATA}/expert_prefill.mlir ${DATA}/expert_prefill-forward.mlir - ${DATA}/expert.mlir ${DATA}/expert-forward.mlir ${DATA}/moe-parameters.h - COMMAND bash -o pipefail -c - "'${PYTHON}' -m stack.models.archs.buckyball.omni.qwen3-omni-30b-a3b-instruct.trace.export --checkpoint \"$(cat '${CHECKPOINT_FILE}')\" --compiler-build '${COMPILER}' --output '${DATA}'" - DEPENDS ${CHECKPOINT_FILE} ${MODEL_SOURCES} ${REPO}/stack/models/models/qwen3-omni-30b-a3b-instruct/moe.py - WORKING_DIRECTORY ${REPO} VERBATIM) -add_custom_command(OUTPUT ${ATTENTION_DATA}/prefill.mlir ${ATTENTION_DATA}/prefill-forward.mlir - ${ATTENTION_DATA}/decode.mlir ${ATTENTION_DATA}/decode-forward.mlir - ${ATTENTION_DATA}/attention-parameters.h - COMMAND bash -o pipefail -c - "'${PYTHON}' -m stack.models.archs.buckyball.omni.qwen3-omni-30b-a3b-instruct.trace.attention --checkpoint \"$(cat '${CHECKPOINT_FILE}')\" --compiler-build '${COMPILER}' --output '${ATTENTION_DATA}'" - DEPENDS ${CHECKPOINT_FILE} ${MODEL_SOURCES} ${REPO}/stack/models/models/qwen3-omni-30b-a3b-instruct/attention.py - WORKING_DIRECTORY ${REPO} VERBATIM) -set(FORWARD_PASSES - -buffer-deallocation-simplification -convert-linalg-to-loops -eliminate-empty-tensors - -llvm-request-c-wrappers -convert-math-to-llvm -convert-math-to-libm -lower-affine - -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -expand-strided-metadata - -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts - -canonicalize -cse -lower-tile-runtime) -set(KERNEL_PASSES - -eliminate-empty-tensors -convert-elementwise-to-linalg -convert-tensor-to-linalg - -canonicalize -cse -one-shot-bufferize=bufferize-function-boundaries - -buffer-deallocation-pipeline=private-function-dynamic-ownership=false -convert-linalg-to-tile=quantized-only - -convert-tile-to-buckyball -lower-buckyball-to-bank-ssa -assign-physical-banks - -llvm-request-c-wrappers -lower-bank-ssa-to-intrinsics -expand-strided-metadata - -convert-linalg-to-loops -lower-buckyball -convert-vector-to-scf=target-rank=0 - -lower-affine -canonicalize -cse -convert-scf-to-cf -convert-cf-to-llvm - -convert-vector-to-llvm -buffer-deallocation-simplification -bufferization-lower-deallocations - -convert-math-to-llvm -convert-math-to-libm -convert-arith-to-llvm -convert-func-to-llvm - -finalize-memref-to-llvm -reconcile-unrealized-casts -canonicalize -cse -lower-tile-runtime) -foreach(STAGE router expert expert_prefill prefill decode) - set(TARGET attention) - if(STAGE MATCHES "^expert") - set(TARGET ffn) - endif() - if(STAGE STREQUAL "router" OR STAGE MATCHES "^expert") - set(STAGE_DATA ${DATA}) - set(GROUP MOE) - else() - set(STAGE_DATA ${ATTENTION_DATA}) - set(GROUP ATTENTION) - endif() - foreach(KIND kernel forward) - if(KIND STREQUAL "kernel") - set(NAME ${STAGE}) - string(JOIN " " PASSES ${KERNEL_PASSES}) - set(LOWER "'${COMPILER}/bin/buddy-opt' '${STAGE_DATA}/${NAME}.mlir' -pass-pipeline 'builtin.module(func.func(tosa-to-linalg-named,tosa-to-linalg,tosa-to-tensor,tosa-to-arith))' | '${COMPILER}/bin/buddy-opt' --target=${TARGET} ${PASSES}") - else() - set(NAME ${STAGE}-forward) - string(JOIN " " PASSES ${FORWARD_PASSES}) - set(LOWER "'${COMPILER}/bin/buddy-opt' '${STAGE_DATA}/${NAME}.mlir' --target=${TARGET} ${PASSES}") - endif() - add_custom_command(OUTPUT ${NAME}.o - COMMAND bash -o pipefail -c "${LOWER} | '${COMPILER}/bin/buddy-translate' --buddy-to-llvmir | '${BUDDY}/llvm/build/bin/opt' -passes='default' | '${COMPILER}/bin/buddy-llc' -filetype=obj -mtriple=riscv64 -O2 -mattr=+xbuckyball,+m,+D -float-abi=hard -o '${NAME}.o'" - DEPENDS ${STAGE_DATA}/${NAME}.mlir ${COMPILER}/bin/buddy-opt VERBATIM) - if(STAGE STREQUAL "expert_prefill") - list(APPEND BATCH_OBJECTS ${NAME}.o) - else() - list(APPEND ${GROUP}_OBJECTS ${NAME}.o) - endif() - endforeach() -endforeach() -set(RUNTIME_PARAMS ${BUCKYBALL_ISA_DIR}/attention) -# Ball tensor tests may already provide the shared runtime and RVV kernels. -if(NOT TARGET runtime) - add_subdirectory(${REPO}/stack/runtime runtime) -endif() -if(NOT TARGET rvv_kernels) - add_subdirectory("${REPO}/bb-tests/workloads/lib/bbsw/kernels/rvv" rvv) -endif() -if(NOT TARGET mxfp8_pack) - add_subdirectory("${REPO}/bb-tests/workloads/lib/bbsw/kernels/mxfp8" mxfp8) -endif() -buckyball_mlirtest_name(NAME ffn chip-moe) -add_executable(${NAME}-linux moe_main.cpp ${MOE_OBJECTS} ${DATA}/moe-parameters.h) -target_include_directories(${NAME}-linux PRIVATE ${DATA} ${BUDDY}/frontend/Interfaces) -target_link_libraries(${NAME}-linux PRIVATE runtime rvv_kernels mxfp8_pack rvv_quant_backend -static) -add_dependencies(chip-mlirtests-build ${NAME}-linux) -buckyball_mlirtest_name(NAME ffn chip-moe-batch) -add_executable(${NAME}-linux moe_batch_main.cpp ${BATCH_OBJECTS} expert.o expert-forward.o ${DATA}/moe-parameters.h) -target_include_directories(${NAME}-linux PRIVATE ${DATA} ${BUDDY}/frontend/Interfaces) -target_link_libraries(${NAME}-linux PRIVATE runtime rvv_kernels mxfp8_pack rvv_quant_backend -static) -add_dependencies(chip-mlirtests-build ${NAME}-linux) -buckyball_mlirtest_name(NAME attention chip-attention) -add_executable(${NAME}-linux attention_main.cpp ${ATTENTION_OBJECTS} ${ATTENTION_DATA}/attention-parameters.h) -target_include_directories(${NAME}-linux PRIVATE ${ATTENTION_DATA} ${BUDDY}/frontend/Interfaces) -target_link_libraries(${NAME}-linux PRIVATE runtime rvv_kernels mxfp8_pack rvv_quant_backend -static) -add_dependencies(chip-mlirtests-build ${NAME}-linux) diff --git a/examples/chips/omni/workloads/mlir_tests/attention_main.cpp b/examples/chips/omni/workloads/mlir_tests/attention_main.cpp deleted file mode 100644 index d5d06ec77..000000000 --- a/examples/chips/omni/workloads/mlir_tests/attention_main.cpp +++ /dev/null @@ -1,107 +0,0 @@ -#include "attention-parameters.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using Hidden = MemRef; -using Cache = MemRef; -using Slots = MemRef; -using Positions = MemRef; -using Floats = MemRef; -using Bytes = MemRef; -struct Result { - Hidden hidden; - Cache keys, values; -}; -extern "C" void _mlir_ciface_forward_prefill(Result *, Floats *, Bytes *, - Hidden *, Cache *, Cache *, - Slots *, Positions *); -extern "C" void _mlir_ciface_forward_decode(Result *, Floats *, Bytes *, - Hidden *, Cache *, Cache *, Slots *, - Positions *); - -template -void load(const std::string &path, MemRef &tensor) { - std::ifstream stream; - stream.exceptions(std::ios::failbit | std::ios::badbit); - stream.open(path, std::ios::binary); - stream.read(reinterpret_cast(tensor.getData()), - tensor.getSize() * sizeof(T)); -} - -template -void compare(const std::string &path, MemRef &actual) { - auto name = std::filesystem::path(path); - std::ofstream saved("actual-" + name.parent_path().filename().string() + "-" + - name.filename().string(), - std::ios::binary); - saved.exceptions(std::ios::failbit | std::ios::badbit); - saved.write(reinterpret_cast(actual.getData()), - actual.getSize() * sizeof(float)); - saved.close(); - std::vector shape(actual.getSizes(), actual.getSizes() + Rank); - MemRef expected(shape); - load(path, expected); - for (size_t i = 0; i < actual.getSize(); ++i) - if (!std::isfinite(actual[i]) || std::abs(actual[i] - expected[i]) > - 2e-5f + 2e-4f * std::abs(expected[i])) - throw std::runtime_error("attention mismatch: " + path + - " index=" + std::to_string(i) + - " actual=" + std::to_string(actual[i]) + - " expected=" + std::to_string(expected[i])); -} - -int main(int argc, char **argv) { - const auto fixtures = - std::filesystem::path(argv[0]).parent_path() / "attention"; - runtime_init(1024 * 1024); - void *workspace = aligned_alloc(64, 64 * 1024 * 1024); - if (!workspace) - throw std::bad_alloc(); - workspace_init(workspace, 64 * 1024 * 1024); - const std::vector cacheShape{1, kvHeads, capacity, headSize}; - Cache keys(cacheShape, 0.0f), values(cacheShape, 0.0f); - for (bool prefill : {true, false}) { - workspace_begin(workspace, 64 * 1024 * 1024); - size_t count = prefill ? 3 : 1; - std::string path = (fixtures / (prefill ? "prefill" : "decode")).string(); - Hidden hidden({1, count, hiddenSize}); - Slots slots({count}); - Positions positions({3, count}); - Floats floats({prefill ? prefillFloats : decodeFloats}); - Bytes bytes({prefill ? prefillBytes : decodeBytes}); - load(path + "/hidden.bin", hidden); - load(path + "/cache_positions.bin", slots); - load(path + "/positions.bin", positions); - load(path + "/params.f32", floats); - load(path + "/weights.bin", bytes); - if (!prefill) { - compare((fixtures / "prefill/expected-keys.bin").string(), keys); - compare((fixtures / "prefill/expected-values.bin").string(), values); - } - Result result{Hidden({1, count, hiddenSize}, false, 0), - Cache(cacheShape, false, 0), Cache(cacheShape, false, 0)}; - auto run = - prefill ? _mlir_ciface_forward_prefill : _mlir_ciface_forward_decode; - run(&result, &floats, &bytes, &hidden, &keys, &values, &slots, &positions); - compare(path + "/expected-keys.bin", result.keys); - compare(path + "/expected-values.bin", result.values); - compare(path + "/expected-hidden.bin", result.hidden); - std::copy_n(result.keys.getData(), keys.getSize(), keys.getData()); - std::copy_n(result.values.getData(), values.getSize(), values.getData()); - workspace_free(result.hidden.release()); - workspace_free(result.keys.release()); - workspace_free(result.values.release()); - std::cout << "OMNI MXFP8 ATTENTION " << (prefill ? "PREFILL" : "DECODE") - << " PASSED\n"; - } - free(workspace); -} diff --git a/examples/chips/omni/workloads/mlir_tests/moe_batch_main.cpp b/examples/chips/omni/workloads/mlir_tests/moe_batch_main.cpp deleted file mode 100644 index 4dac2e898..000000000 --- a/examples/chips/omni/workloads/mlir_tests/moe_batch_main.cpp +++ /dev/null @@ -1,61 +0,0 @@ -#include "moe-parameters.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using Matrix = MemRef; -using Bytes = MemRef; -extern "C" void _mlir_ciface_forward_expert(Matrix *, Bytes *, Matrix *); -extern "C" void _mlir_ciface_forward_expert_prefill(Matrix *, Bytes *, - Matrix *); - -template -void load(const std::filesystem::path &path, MemRef &value) { - std::ifstream stream; - stream.exceptions(std::ios::failbit | std::ios::badbit); - stream.open(path, std::ios::binary); - stream.read(reinterpret_cast(value.getData()), - value.getSize() * sizeof(T)); -} - -int main(int argc, char **argv) { - auto path = std::filesystem::path(argv[0]).parent_path() / "moe"; - runtime_init(1024 * 1024); - void *workspace = aligned_alloc(64, 64 * 1024 * 1024); - if (!workspace) - throw std::bad_alloc(); - workspace_init(workspace, 64 * 1024 * 1024); - workspace_begin(workspace, 64 * 1024 * 1024); - Matrix input({batchSize, hiddenSize}), expected({batchSize, hiddenSize}); - Bytes weights({expertBytes}); - load(path / "batch-input.f32", input); - load(path / "expert/weights.bin", weights); - load(path / "expert_prefill/expected.f32", expected); - Matrix batch({batchSize, hiddenSize}, false, 0); - _mlir_ciface_forward_expert_prefill(&batch, &weights, &input); - Matrix row({1, hiddenSize}); - for (size_t token = 0; token < batchSize; ++token) { - std::copy_n(input.getData() + token * hiddenSize, hiddenSize, - row.getData()); - Matrix single({1, hiddenSize}, false, 0); - _mlir_ciface_forward_expert(&single, &weights, &row); - for (size_t i = 0; i < hiddenSize; ++i) { - size_t index = token * hiddenSize + i; - if (!std::isfinite(batch[index]) || batch[index] != single[i] || - std::abs(batch[index] - expected[index]) > - 1e-5f + 1e-4f * std::abs(expected[index])) - throw std::runtime_error( - "MXFP8 batch differs from scalar or reference"); - } - workspace_free(single.release()); - } - workspace_free(batch.release()); - free(workspace); - std::cout << "OMNI MXFP8 EXPERT BATCH PASSED\n"; -} diff --git a/examples/chips/omni/workloads/mlir_tests/moe_main.cpp b/examples/chips/omni/workloads/mlir_tests/moe_main.cpp deleted file mode 100644 index 138254807..000000000 --- a/examples/chips/omni/workloads/mlir_tests/moe_main.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include "moe-parameters.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using Hidden = MemRef; -using Indices = MemRef; -using Floats = MemRef; -using Bytes = MemRef; -struct RouterResult { - Hidden normalized; - Indices indices; - Hidden scores; -}; -extern "C" void _mlir_ciface_forward_router(RouterResult *, Floats *, Hidden *); -extern "C" void _mlir_ciface_forward_expert(Hidden *, Bytes *, Hidden *); - -template -void load(const std::string &path, MemRef &tensor) { - std::ifstream stream; - stream.exceptions(std::ios::failbit | std::ios::badbit); - stream.open(path, std::ios::binary); - stream.read(reinterpret_cast(tensor.getData()), - tensor.getSize() * sizeof(T)); -} - -int main(int argc, char **argv) { - const std::string path = - (std::filesystem::path(argv[0]).parent_path() / "moe").string(); - runtime_init(1024 * 1024); - void *workspace = aligned_alloc(64, 64 * 1024 * 1024); - if (!workspace) - throw std::bad_alloc(); - workspace_init(workspace, 64 * 1024 * 1024); - workspace_begin(workspace, 64 * 1024 * 1024); - Hidden input({1, hiddenSize}), expected({1, hiddenSize}), scores({1, topK}); - Indices indices({1, topK}); - Floats floats({routerFloats}); - Bytes bytes({expertBytes}); - load(path + "/input.f32", input); - load(path + "/router/params.f32", floats); - load(path + "/expert/weights.bin", bytes); - RouterResult routing{Hidden({1, hiddenSize}, false, 0), - Indices({1, topK}, false, 0), - Hidden({1, topK}, false, 0)}; - _mlir_ciface_forward_router(&routing, &floats, &input); - load(path + "/router/expected.f32", expected); - load(path + "/router/indices.i64", indices); - load(path + "/router/scores.f32", scores); - for (size_t i = 0; i < hiddenSize; ++i) - if (!std::isfinite(routing.normalized[i]) || - std::abs(routing.normalized[i] - expected[i]) > 1e-5f) - throw std::runtime_error("router normalization mismatch"); - for (size_t i = 0; i < topK; ++i) - if (routing.indices[i] != indices[i] || !std::isfinite(routing.scores[i]) || - std::abs(routing.scores[i] - scores[i]) > 1e-5f) - throw std::runtime_error("router top-k mismatch"); - Hidden result({1, hiddenSize}, false, 0); - _mlir_ciface_forward_expert(&result, &bytes, &input); - load(path + "/expert/expected.f32", expected); - float maximum = 0; - for (size_t i = 0; i < hiddenSize; ++i) { - float error = std::abs(result[i] - expected[i]); - if (!std::isfinite(result[i]) || - error > 1e-5f + 1e-4f * std::abs(expected[i])) - throw std::runtime_error("MXFP8 expert mismatch"); - maximum = std::max(maximum, error); - } - std::cout << "OMNI ROUTER AND MXFP8 EXPERT PASSED max_error=" << maximum - << '\n'; - workspace_free(routing.normalized.release()); - workspace_free(routing.indices.release()); - workspace_free(routing.scores.release()); - workspace_free(result.release()); - free(workspace); -} diff --git a/examples/chips/pebble/arch/src/main/scala/sims/p2e/TargetConfigs.scala b/examples/chips/pebble/arch/src/main/scala/sims/p2e/TargetConfigs.scala deleted file mode 100644 index 9981eb52d..000000000 --- a/examples/chips/pebble/arch/src/main/scala/sims/p2e/TargetConfigs.scala +++ /dev/null @@ -1,10 +0,0 @@ -package sims.p2e - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -/** Pebble on the P2E board: one compute core with private banks over the DDR4 macro. */ -class PebbleP2ETarget extends P2ETarget("../examples/chips/pebble/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) -} diff --git a/examples/chips/pebble/arch/src/main/scala/sims/verilator/TargetConfigs.scala b/examples/chips/pebble/arch/src/main/scala/sims/verilator/TargetConfigs.scala deleted file mode 100644 index 9a09e967a..000000000 --- a/examples/chips/pebble/arch/src/main/scala/sims/verilator/TargetConfigs.scala +++ /dev/null @@ -1,12 +0,0 @@ -package sims.verilator - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -import sims.soc.SystemTarget - -/** Pebble on the explicit System: one compute core with private banks and no task controller. */ -class PebbleVerilatorTarget extends SystemTarget("../examples/chips/pebble/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) -} diff --git a/examples/chips/pebble/configs/chip.toml b/examples/chips/pebble/configs/chip.toml deleted file mode 100644 index 268aa3106..000000000 --- a/examples/chips/pebble/configs/chip.toml +++ /dev/null @@ -1,10 +0,0 @@ -[designs] -include = "designs/pebble.toml" - -[sims] -verilator = "sims.verilator.PebbleVerilatorTarget" -p2e = "sims.p2e.PebbleP2ETarget" - -[uvm] -balls = ["transpose", "smatmul", "im2col", "toint8", "int2fp", "lut", "maxpool", "int8add", "int8mul", "matadd"] -ips = [] diff --git a/examples/chips/pebble/configs/designs/pebble.toml b/examples/chips/pebble/configs/designs/pebble.toml deleted file mode 100644 index fb742d1fe..000000000 --- a/examples/chips/pebble/configs/designs/pebble.toml +++ /dev/null @@ -1,4 +0,0 @@ -# Pebble: a main tile only. - -[main_tile] -include = "tiles/default.toml" diff --git a/examples/chips/pebble/configs/designs/tiles/default.toml b/examples/chips/pebble/configs/designs/tiles/default.toml deleted file mode 100644 index d1ef6a2e3..000000000 --- a/examples/chips/pebble/configs/designs/tiles/default.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Pebble tile configuration - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -memBallChannelNum = 6 - -[sharedMem] -bankWidth = 128 -bankEntries = 512 -enable = false -entries = 16384 -inputChannels = 7 -defaultGroupCount = 1 - -[[cores]] -core_id = 0 -include = "../../../../../cores/pebble/configs/default.toml" diff --git a/examples/chips/pebble/kernel/workloads.toml b/examples/chips/pebble/kernel/workloads.toml deleted file mode 100644 index aeb45f9a9..000000000 --- a/examples/chips/pebble/kernel/workloads.toml +++ /dev/null @@ -1,36 +0,0 @@ -[workloads] -tests = [ - "pebble-pebble-mlirtest-tile_mega_conv2d_k5_tail-linux", - "pebble-pebble-mlirtest-tile_mega_matmul_m17_k65_n17-linux", - "pebble-pebble-ctest-mvin_2d_test-linux", - "pebble-pebble-ctest-mvin_2d_pixel_stride_test-linux", - "pebble-pebble-ctest-mvin_mvout_acc_test-linux", - "pebble-pebble-ctest-mvin_mvout_test-linux", - "pebble-pebble-ctest-mvin_mvout_stride_test-linux", - "pebble-pebble-ctest-mvin_mvout_unaligned_test-linux", - "pebble-pebble-ctest-mvin_unaligned_test-linux", - "pebble-pebble-mlirtest-linalg_mega_conv2d_k3_c16-linux", - "pebble-pebble-mlirtest-tile_mega_conv2d_s2_depthwise_s2-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv_exact-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv2_exact-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_7x7_c288_exact-linux", - "pebble-pebble-mlirtest-tile_mega_se_gate_conv-linux", - "pebble-pebble-ctest-mega_conv_pipeline_test-linux", - "pebble-pebble-ctest-residual_add_lut_test-linux", - "pebble-pebble-ctest-maxpool_im2col_test-linux", - "pebble-pebble-ctest-se_gate_lut_test-linux", - "pebble-pebble-ctest-int8mul_large_gate_row_test-linux", - "pebble-pebble-ctest-dual_smatmul_matadd_test-linux", - "pebble-pebble-ctest-dual_conv_add_int2fp_test-linux", - "pebble-pebble-mlirtest-bank_matadd_i32_elementwise-linux", - "pebble-pebble-mlirtest-bank_int8add_identity_scale-linux", - "pebble-pebble-mlirtest-bank_lut_identity_table-linux", - "pebble-pebble-mlirtest-bank_int8mul_unit_gate-linux", - "pebble-pebble-mlirtest-bank_maxpool_k3_padded_spatial-linux", - "pebble-pebble-mlirtest-bank_smatmul_bias_two_terms-linux", - "pebble-pebble-mlirtest-bank_int2fp_unit_scale-linux", - "pebble-pebble-mlirtest-bank_quant_i32_pack16-linux", - "pebble-pebble-mlirtest-bank_quant_f32_pack16-linux", - "pebble-pebble-mlirtest-bank_im2col_k3_padded_bank_windows-linux", - "pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-linux", -] diff --git a/examples/chips/pebble/regression/batch/bemu/workloads-bare.toml b/examples/chips/pebble/regression/batch/bemu/workloads-bare.toml deleted file mode 100644 index 8e1016606..000000000 --- a/examples/chips/pebble/regression/batch/bemu/workloads-bare.toml +++ /dev/null @@ -1,91 +0,0 @@ -[workloads] -tests = [ - "pebble-pebble-ctest-mvin_2d_test-baremetal", - "pebble-pebble-ctest-mvin_2d_pixel_stride_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_acc_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_stride_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_unaligned_test-baremetal", - "pebble-pebble-ctest-mvin_unaligned_test-baremetal", - "pebble-pebble-mlirtest-linalg_mega_conv2d_k3_c16-baremetal", - "pebble-pebble-mlirtest-tile_mega_conv2d_s2_depthwise_s2-baremetal", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv_exact-baremetal", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv2_exact-baremetal", - "pebble-pebble-mlirtest-tile_mega_global_avg_7x7_c288_exact-baremetal", - "pebble-pebble-mlirtest-tile_mega_se_gate_conv-baremetal", - "pebble-pebble-ctest-mega_conv_pipeline_test-baremetal", - "pebble-pebble-ctest-residual_add_lut_test-baremetal", - "pebble-pebble-ctest-maxpool_im2col_test-baremetal", - "pebble-pebble-ctest-se_gate_lut_test-baremetal", - "pebble-pebble-ctest-int8mul_large_gate_row_test-baremetal", - "pebble-pebble-ctest-dual_smatmul_matadd_test-baremetal", - "pebble-pebble-ctest-dual_conv_add_int2fp_test-baremetal", - "pebble-pebble-ctest-quant_smatmul_int2fp_test-baremetal", - "pebble-pebble-ctest-smatmul_mvout_bank_reuse_test-baremetal", - "pebble-pebble-mlirtest-bank_matadd_i32_elementwise-baremetal", - "pebble-pebble-mlirtest-bank_int8add_identity_scale-baremetal", - "pebble-pebble-mlirtest-bank_lut_identity_table-baremetal", - "pebble-pebble-mlirtest-bank_int8mul_unit_gate-baremetal", - "pebble-pebble-mlirtest-bank_maxpool_k3_padded_spatial-baremetal", - "pebble-pebble-mlirtest-bank_smatmul_bias_two_terms-baremetal", - "pebble-pebble-mlirtest-bank_int2fp_unit_scale-baremetal", - "pebble-pebble-mlirtest-bank_quant_i32_pack16-baremetal", - "pebble-pebble-mlirtest-bank_quant_f32_pack16-baremetal", - "pebble-pebble-mlirtest-bank_im2col_k3_padded_bank_windows-baremetal", - "pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-baremetal", - "pebble-pebble-ctest-tlb_bare_mode_test-baremetal", - "pebble-pebble-ctest-tlb_repeated_sfence_test-baremetal", - "pebble-pebble-ctest-tlb_sfence_flush_test-baremetal", - "pebble-pebble-ctest-tlb_sfence_remap_test-baremetal", - "pebble-pebble-ctest-tlb_sv39_4k_alias_test-baremetal", - "pebble-pebble-ctest-tlb_sv39_identity_test-baremetal", - "pebble-pebble-ctest-tlb_sv39_nonidentity_test-baremetal", - "pebble-pebble-ctest-transpose_i8_16x16_test-baremetal", - "pebble-pebble-ctest-transpose_i8_16x32_test-baremetal", - "pebble-pebble-ctest-transpose_i32_8x16_test-baremetal", - "pebble-pebble-mlirtest-ball_transpose_16x16_i8-baremetal", - "pebble-pebble-ctest-smatmul_bias_accumulate_test-baremetal", - "pebble-pebble-ctest-smatmul_1x16x64_test-baremetal", - "pebble-pebble-ctest-smatmul_16x16_test-baremetal", - "pebble-pebble-ctest-smatmul_16x16x32_test-baremetal", - "pebble-pebble-ctest-smatmul_nondefault_banks_test-baremetal", - "pebble-pebble-mlirtest-ball_smatmul_bias_accumulate-baremetal", - "pebble-pebble-ctest-quant_f32_to_i8_test-baremetal", - "pebble-pebble-ctest-quant_i32_to_i8_test-baremetal", - "pebble-pebble-ctest-toint8_i8_rounding_test-baremetal", - "pebble-pebble-ctest-toint8_i8_rows_test-baremetal", - "pebble-pebble-ctest-toint8_i8_scale_rows_test-baremetal", - "pebble-pebble-ctest-toint8_i8_signed_test-baremetal", - "pebble-pebble-mlirtest-ball_quant_f32_to_i8-baremetal", - "pebble-pebble-mlirtest-ball_quant_i32_to_i8-baremetal", - "pebble-pebble-ctest-im2col_window_test-baremetal", - "pebble-pebble-ctest-im2col_k1_test-baremetal", - "pebble-pebble-ctest-im2col_k3_test-baremetal", - "pebble-pebble-ctest-im2col_k3_pad_test-baremetal", - "pebble-pebble-ctest-im2col_k3_asym_pad_test-baremetal", - "pebble-pebble-ctest-im2col_k3_partial_test-baremetal", - "pebble-pebble-ctest-im2col_k5_test-baremetal", - "pebble-pebble-ctest-im2col_k7_test-baremetal", - "pebble-pebble-mlirtest-ball_im2col_6x6_k3-baremetal", - "pebble-pebble-ctest-int2fp_tensor_values_test-baremetal", - "pebble-pebble-ctest-int2fp_tensor_groups_test-baremetal", - "pebble-pebble-ctest-int2fp_tensor_rows_test-baremetal", - "pebble-pebble-ctest-int2fp_channel_lanes_test-baremetal", - "pebble-pebble-ctest-int2fp_channel_base_test-baremetal", - "pebble-pebble-ctest-int2fp_channel_two_rows_test-baremetal", - "pebble-pebble-mlirtest-ball_int2fp_4x4-baremetal", - "pebble-pebble-mlirtest-ball_int2fp_relu_4x4-baremetal", - "pebble-pebble-ctest-lut_uniform_lookup_test-baremetal", - "pebble-pebble-ctest-lut_channel_table_lookup_test-baremetal", - "pebble-pebble-mlirtest-ball_lut_identity-baremetal", - "pebble-pebble-ctest-maxpool_window_test-baremetal", - "pebble-pebble-ctest-maxpool_offset_test-baremetal", - "pebble-pebble-mlirtest-ball_maxpool_6x6_s2-baremetal", - "pebble-pebble-ctest-int8add_scaled_test-baremetal", - "pebble-pebble-ctest-int8add_relu_test-baremetal", - "pebble-pebble-mlirtest-ball_int8add_relu-baremetal", - "pebble-pebble-ctest-int8mul_per_channel_gate_test-baremetal", - "pebble-pebble-mlirtest-ball_int8mul_gate-baremetal", - "pebble-pebble-ctest-matadd_i32_16row_test-baremetal", - "pebble-pebble-mlirtest-ball_matadd_i32-baremetal", -] diff --git a/examples/chips/pebble/regression/batch/bemu/workloads-linux.toml b/examples/chips/pebble/regression/batch/bemu/workloads-linux.toml deleted file mode 100644 index ba17ce202..000000000 --- a/examples/chips/pebble/regression/batch/bemu/workloads-linux.toml +++ /dev/null @@ -1,34 +0,0 @@ -[workloads] -tests = [ - "pebble-pebble-ctest-mvin_2d_test-linux", - "pebble-pebble-ctest-mvin_2d_pixel_stride_test-linux", - "pebble-pebble-ctest-mvin_mvout_acc_test-linux", - "pebble-pebble-ctest-mvin_mvout_test-linux", - "pebble-pebble-ctest-mvin_mvout_stride_test-linux", - "pebble-pebble-ctest-mvin_mvout_unaligned_test-linux", - "pebble-pebble-ctest-mvin_unaligned_test-linux", - "pebble-pebble-mlirtest-linalg_mega_conv2d_k3_c16-linux", - "pebble-pebble-mlirtest-tile_mega_conv2d_s2_depthwise_s2-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv_exact-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv2_exact-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_7x7_c288_exact-linux", - "pebble-pebble-mlirtest-tile_mega_se_gate_conv-linux", - "pebble-pebble-ctest-mega_conv_pipeline_test-linux", - "pebble-pebble-ctest-residual_add_lut_test-linux", - "pebble-pebble-ctest-maxpool_im2col_test-linux", - "pebble-pebble-ctest-se_gate_lut_test-linux", - "pebble-pebble-ctest-int8mul_large_gate_row_test-linux", - "pebble-pebble-ctest-dual_smatmul_matadd_test-linux", - "pebble-pebble-ctest-dual_conv_add_int2fp_test-linux", - "pebble-pebble-mlirtest-bank_matadd_i32_elementwise-linux", - "pebble-pebble-mlirtest-bank_int8add_identity_scale-linux", - "pebble-pebble-mlirtest-bank_lut_identity_table-linux", - "pebble-pebble-mlirtest-bank_int8mul_unit_gate-linux", - "pebble-pebble-mlirtest-bank_maxpool_k3_padded_spatial-linux", - "pebble-pebble-mlirtest-bank_smatmul_bias_two_terms-linux", - "pebble-pebble-mlirtest-bank_int2fp_unit_scale-linux", - "pebble-pebble-mlirtest-bank_quant_i32_pack16-linux", - "pebble-pebble-mlirtest-bank_quant_f32_pack16-linux", - "pebble-pebble-mlirtest-bank_im2col_k3_padded_bank_windows-linux", - "pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-linux", -] diff --git a/examples/chips/pebble/regression/batch/p2e/workloads-bare.toml b/examples/chips/pebble/regression/batch/p2e/workloads-bare.toml deleted file mode 100644 index 9dabc3631..000000000 --- a/examples/chips/pebble/regression/batch/p2e/workloads-bare.toml +++ /dev/null @@ -1,25 +0,0 @@ -[workloads] -tests = [ - "pebble-pebble-ctest-mvin_2d_pixel_stride_test-baremetal.hex", - "pebble-pebble-ctest-mega_conv_pipeline_test-baremetal.hex", - "pebble-pebble-mlirtest-tile_mega_conv2d_s2_depthwise_s2-baremetal.hex", - "pebble-pebble-mlirtest-tile_mega_se_gate_conv-baremetal.hex", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv_exact-baremetal.hex", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv2_exact-baremetal.hex", - "pebble-pebble-mlirtest-tile_mega_global_avg_7x7_c288_exact-baremetal.hex", - "pebble-pebble-ctest-residual_add_lut_test-baremetal.hex", - "pebble-pebble-ctest-maxpool_im2col_test-baremetal.hex", - "pebble-pebble-ctest-se_gate_lut_test-baremetal.hex", - "pebble-pebble-ctest-dual_conv_add_int2fp_test-baremetal.hex", - "pebble-pebble-mlirtest-bank_matadd_i32_elementwise-baremetal.hex", - "pebble-pebble-mlirtest-bank_int8add_identity_scale-baremetal.hex", - "pebble-pebble-mlirtest-bank_lut_identity_table-baremetal.hex", - "pebble-pebble-mlirtest-bank_int8mul_unit_gate-baremetal.hex", - "pebble-pebble-mlirtest-bank_maxpool_k3_padded_spatial-baremetal.hex", - "pebble-pebble-mlirtest-bank_smatmul_bias_two_terms-baremetal.hex", - "pebble-pebble-mlirtest-bank_int2fp_unit_scale-baremetal.hex", - "pebble-pebble-mlirtest-bank_quant_i32_pack16-baremetal.hex", - "pebble-pebble-mlirtest-bank_quant_f32_pack16-baremetal.hex", - "pebble-pebble-mlirtest-bank_im2col_k3_padded_bank_windows-baremetal.hex", - "pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-baremetal.hex", -] diff --git a/examples/chips/pebble/regression/batch/verilator/workloads-bare.toml b/examples/chips/pebble/regression/batch/verilator/workloads-bare.toml deleted file mode 100644 index 50f5fc9ab..000000000 --- a/examples/chips/pebble/regression/batch/verilator/workloads-bare.toml +++ /dev/null @@ -1,80 +0,0 @@ -[workloads] -tests = [ - "pebble-pebble-ctest-mvin_2d_test-baremetal", - "pebble-pebble-ctest-mvin_2d_pixel_stride_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_acc_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_stride_test-baremetal", - "pebble-pebble-ctest-mvin_mvout_unaligned_test-baremetal", - "pebble-pebble-ctest-mvin_unaligned_test-baremetal", - "pebble-pebble-ctest-tlb_bare_mode_test-baremetal", - "pebble-pebble-ctest-tlb_repeated_sfence_test-baremetal", - "pebble-pebble-ctest-tlb_sfence_flush_test-baremetal", - "pebble-pebble-ctest-tlb_sfence_remap_test-baremetal", - "pebble-pebble-ctest-tlb_sv39_4k_alias_test-baremetal", - "pebble-pebble-ctest-tlb_sv39_identity_test-baremetal", - "pebble-pebble-ctest-tlb_sv39_nonidentity_test-baremetal", - "pebble-pebble-mlirtest-linalg_mega_conv2d_k3_c16-baremetal", - "pebble-pebble-mlirtest-tile_mega_conv2d_s2_depthwise_s2-baremetal", - "pebble-pebble-ctest-transpose_i8_16x16_test-baremetal", - "pebble-pebble-ctest-transpose_i8_16x32_test-baremetal", - "pebble-pebble-ctest-transpose_i32_8x16_test-baremetal", - "pebble-pebble-mlirtest-ball_transpose_16x16_i8-baremetal", - "pebble-pebble-ctest-smatmul_bias_accumulate_test-baremetal", - "pebble-pebble-ctest-smatmul_1x16x64_test-baremetal", - "pebble-pebble-ctest-smatmul_16x16_test-baremetal", - "pebble-pebble-ctest-smatmul_16x16x32_test-baremetal", - "pebble-pebble-ctest-smatmul_nondefault_banks_test-baremetal", - "pebble-pebble-ctest-smatmul_mvout_bank_reuse_test-baremetal", - "pebble-pebble-mlirtest-ball_smatmul_bias_accumulate-baremetal", - "pebble-pebble-ctest-quant_f32_to_i8_test-baremetal", - "pebble-pebble-ctest-quant_i32_to_i8_test-baremetal", - "pebble-pebble-ctest-toint8_i8_rounding_test-baremetal", - "pebble-pebble-ctest-toint8_i8_rows_test-baremetal", - "pebble-pebble-ctest-toint8_i8_scale_rows_test-baremetal", - "pebble-pebble-ctest-toint8_i8_signed_test-baremetal", - "pebble-pebble-mlirtest-ball_quant_f32_to_i8-baremetal", - "pebble-pebble-mlirtest-ball_quant_i32_to_i8-baremetal", - "pebble-pebble-ctest-im2col_window_test-baremetal", - "pebble-pebble-ctest-im2col_k1_test-baremetal", - "pebble-pebble-ctest-im2col_k3_test-baremetal", - "pebble-pebble-ctest-im2col_k3_pad_test-baremetal", - "pebble-pebble-ctest-im2col_k3_asym_pad_test-baremetal", - "pebble-pebble-ctest-im2col_k3_partial_test-baremetal", - "pebble-pebble-ctest-im2col_k5_test-baremetal", - "pebble-pebble-ctest-im2col_k7_test-baremetal", - "pebble-pebble-mlirtest-ball_im2col_6x6_k3-baremetal", - "pebble-pebble-ctest-int2fp_tensor_values_test-baremetal", - "pebble-pebble-ctest-int2fp_tensor_groups_test-baremetal", - "pebble-pebble-ctest-int2fp_tensor_rows_test-baremetal", - "pebble-pebble-ctest-int2fp_channel_lanes_test-baremetal", - "pebble-pebble-ctest-int2fp_channel_base_test-baremetal", - "pebble-pebble-ctest-int2fp_channel_two_rows_test-baremetal", - "pebble-pebble-mlirtest-ball_int2fp_4x4-baremetal", - "pebble-pebble-mlirtest-ball_int2fp_relu_4x4-baremetal", - "pebble-pebble-ctest-lut_uniform_lookup_test-baremetal", - "pebble-pebble-ctest-lut_channel_table_lookup_test-baremetal", - "pebble-pebble-mlirtest-ball_lut_identity-baremetal", - "pebble-pebble-ctest-maxpool_window_test-baremetal", - "pebble-pebble-ctest-maxpool_offset_test-baremetal", - "pebble-pebble-mlirtest-ball_maxpool_6x6_s2-baremetal", - "pebble-pebble-ctest-int8add_scaled_test-baremetal", - "pebble-pebble-ctest-int8add_relu_test-baremetal", - "pebble-pebble-mlirtest-ball_int8add_relu-baremetal", - "pebble-pebble-ctest-int8mul_per_channel_gate_test-baremetal", - "pebble-pebble-ctest-int8mul_large_gate_row_test-baremetal", - "pebble-pebble-mlirtest-ball_int8mul_gate-baremetal", - "pebble-pebble-ctest-matadd_i32_16row_test-baremetal", - "pebble-pebble-mlirtest-ball_matadd_i32-baremetal", - "pebble-pebble-ctest-mega_conv_pipeline_test-baremetal", - "pebble-pebble-mlirtest-tile_mega_se_gate_conv-baremetal", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv_exact-baremetal", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv2_exact-baremetal", - "pebble-pebble-mlirtest-tile_mega_global_avg_7x7_c288_exact-baremetal", - "pebble-pebble-ctest-residual_add_lut_test-baremetal", - "pebble-pebble-ctest-maxpool_im2col_test-baremetal", - "pebble-pebble-ctest-se_gate_lut_test-baremetal", - "pebble-pebble-ctest-dual_smatmul_matadd_test-baremetal", - "pebble-pebble-ctest-dual_conv_add_int2fp_test-baremetal", - "pebble-pebble-ctest-quant_smatmul_int2fp_test-baremetal", -] diff --git a/examples/chips/pebble/regression/batch/verilator/workloads-linux.toml b/examples/chips/pebble/regression/batch/verilator/workloads-linux.toml deleted file mode 100644 index 1d5d6ac34..000000000 --- a/examples/chips/pebble/regression/batch/verilator/workloads-linux.toml +++ /dev/null @@ -1,69 +0,0 @@ -[workloads] -tests = [ - "pebble-pebble-ctest-mvin_2d_test-linux", - "pebble-pebble-ctest-mvin_2d_pixel_stride_test-linux", - "pebble-pebble-ctest-mvin_mvout_acc_test-linux", - "pebble-pebble-ctest-mvin_mvout_test-linux", - "pebble-pebble-ctest-mvin_mvout_stride_test-linux", - "pebble-pebble-ctest-mvin_mvout_unaligned_test-linux", - "pebble-pebble-ctest-mvin_unaligned_test-linux", - "pebble-pebble-ctest-transpose_i8_16x16_test-linux", - "pebble-pebble-ctest-transpose_i8_16x32_test-linux", - "pebble-pebble-ctest-transpose_i32_8x16_test-linux", - "pebble-pebble-mlirtest-ball_transpose_16x16_i8-linux", - "pebble-pebble-ctest-smatmul_bias_accumulate_test-linux", - "pebble-pebble-ctest-smatmul_1x16x64_test-linux", - "pebble-pebble-ctest-smatmul_16x16_test-linux", - "pebble-pebble-ctest-smatmul_16x16x32_test-linux", - "pebble-pebble-ctest-smatmul_nondefault_banks_test-linux", - "pebble-pebble-mlirtest-ball_smatmul_bias_accumulate-linux", - "pebble-pebble-ctest-quant_f32_to_i8_test-linux", - "pebble-pebble-ctest-quant_i32_to_i8_test-linux", - "pebble-pebble-ctest-toint8_i8_rounding_test-linux", - "pebble-pebble-ctest-toint8_i8_rows_test-linux", - "pebble-pebble-ctest-toint8_i8_scale_rows_test-linux", - "pebble-pebble-ctest-toint8_i8_signed_test-linux", - "pebble-pebble-mlirtest-ball_quant_f32_to_i8-linux", - "pebble-pebble-mlirtest-ball_quant_i32_to_i8-linux", - "pebble-pebble-ctest-im2col_window_test-linux", - "pebble-pebble-ctest-im2col_k1_test-linux", - "pebble-pebble-ctest-im2col_k3_test-linux", - "pebble-pebble-ctest-im2col_k3_pad_test-linux", - "pebble-pebble-ctest-im2col_k3_asym_pad_test-linux", - "pebble-pebble-ctest-im2col_k3_partial_test-linux", - "pebble-pebble-ctest-im2col_k5_test-linux", - "pebble-pebble-ctest-im2col_k7_test-linux", - "pebble-pebble-mlirtest-ball_im2col_6x6_k3-linux", - "pebble-pebble-ctest-int2fp_tensor_values_test-linux", - "pebble-pebble-ctest-int2fp_tensor_groups_test-linux", - "pebble-pebble-ctest-int2fp_tensor_rows_test-linux", - "pebble-pebble-ctest-int2fp_channel_lanes_test-linux", - "pebble-pebble-ctest-int2fp_channel_base_test-linux", - "pebble-pebble-ctest-int2fp_channel_two_rows_test-linux", - "pebble-pebble-mlirtest-ball_int2fp_4x4-linux", - "pebble-pebble-mlirtest-ball_int2fp_relu_4x4-linux", - "pebble-pebble-ctest-lut_uniform_lookup_test-linux", - "pebble-pebble-ctest-lut_channel_table_lookup_test-linux", - "pebble-pebble-mlirtest-ball_lut_identity-linux", - "pebble-pebble-ctest-maxpool_window_test-linux", - "pebble-pebble-ctest-maxpool_offset_test-linux", - "pebble-pebble-mlirtest-ball_maxpool_6x6_s2-linux", - "pebble-pebble-ctest-int8add_scaled_test-linux", - "pebble-pebble-ctest-int8add_relu_test-linux", - "pebble-pebble-mlirtest-ball_int8add_relu-linux", - "pebble-pebble-ctest-int8mul_per_channel_gate_test-linux", - "pebble-pebble-mlirtest-ball_int8mul_gate-linux", - "pebble-pebble-ctest-matadd_i32_16row_test-linux", - "pebble-pebble-mlirtest-ball_matadd_i32-linux", - "pebble-pebble-ctest-mega_conv_pipeline_test-linux", - "pebble-pebble-mlirtest-tile_mega_conv2d_s2_depthwise_s2-linux", - "pebble-pebble-mlirtest-tile_mega_se_gate_conv-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv_exact-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_conv2_exact-linux", - "pebble-pebble-mlirtest-tile_mega_global_avg_7x7_c288_exact-linux", - "pebble-pebble-ctest-residual_add_lut_test-linux", - "pebble-pebble-ctest-maxpool_im2col_test-linux", - "pebble-pebble-ctest-se_gate_lut_test-linux", - "pebble-pebble-ctest-dual_smatmul_matadd_test-linux", - "pebble-pebble-ctest-dual_conv_add_int2fp_test-linux", -] diff --git a/examples/chips/pebble/regression/soc/README.md b/examples/chips/pebble/regression/soc/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/chips/pebble/tapeout/README.md b/examples/chips/pebble/tapeout/README.md deleted file mode 100644 index 2b9f1a5b8..000000000 --- a/examples/chips/pebble/tapeout/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Pebble tapeout contract - -Layout: -- `config.toml` — chip contract (scripts, libs, power window) -- `area/` — DC (`dc.tcl`, `constraints.sdc`) -- `power/` — PTPX + gate sim (`power.tcl`, `power_sim.sh`) -- `ip/` — SRAM MDF - -`bbdev dc` resolves paths from `config.toml` and injects `RUN_*`. Timing truth is -`area/constraints.sdc`. Pebble main clock is 100 MHz (`10 ns`). diff --git a/examples/chips/pebble/tapeout/area/dc.tcl b/examples/chips/pebble/tapeout/area/dc.tcl deleted file mode 100755 index c4fbc56ac..000000000 --- a/examples/chips/pebble/tapeout/area/dc.tcl +++ /dev/null @@ -1,65 +0,0 @@ -# Chip-owned DC flow. Timing comes from constraints.sdc (RUN_SDC). -# bbdev injects RUN_* via -x {set RUN_CONFIG }. -if {![info exists RUN_CONFIG] || $RUN_CONFIG eq ""} { - error "bbdev must pass -x {set RUN_CONFIG }" -} -source [file normalize $RUN_CONFIG] - -proc bbdev_read_filelist {path} { - set fh [open $path r] - set files [list] - while {[gets $fh line] >= 0} { - set line [string trim $line] - if {$line eq "" || [string match "#*" $line]} { continue } - lappend files [file normalize $line] - } - close $fh - if {[llength $files] == 0} { error "empty DC source list: $path" } - return $files -} - -set top $RUN_TOP -set output_dir [file normalize $RUN_OUTPUT_DIR] -set report_dir [file normalize $RUN_REPORT_DIR] -file mkdir $output_dir $report_dir [file join $report_dir work] -set target_library [list $RUN_TARGET_LIBRARY] -set synthetic_library $RUN_SYNTHETIC_LIBRARY -set link_library [concat [list *] $target_library $synthetic_library $RUN_LINK_LIBRARY] -set_host_options -max_cores $RUN_MAX_CORES -define_design_lib WORK -path [file join $report_dir work] -set search_path [list .] -set_app_var verilogout_no_tri true -set_app_var verilogout_equation false -set_app_var compile_seqmap_propagate_constants false -set_app_var hdlin_ff_always_sync_set_reset true -analyze -format sverilog -define {SYNTHESIS DC_SYN} [bbdev_read_filelist $RUN_SOURCE_LIST] -elaborate $top -current_design $top -link -set sram_cells [get_cells -hierarchical -quiet -filter {ref_name =~ "chipyard_sram_*"}] -if {[sizeof_collection $sram_cells] == 0} { - error "no chipyard_sram_* cells after link; SRAM .db link_library missing" -} -set_dont_touch $sram_cells -set sram_designs [get_designs -quiet chipyard_sram_*] -if {[sizeof_collection $sram_designs] > 0} { - set_dont_touch $sram_designs -} -read_sdc [file normalize $RUN_SDC] -if {[sizeof_collection [all_clocks]] == 0} { - error "SDC created no clocks" -} -set_load 2.0 [all_outputs] -set_app_var compile_seqmap_honor_sync_set_reset true -compile_ultra -area_high_effort_script -no_autoungroup -no_boundary_optimization -set_fix_multiple_port_nets -all -buffer_constants -change_names -hierarchy -rules verilog -write -format ddc -hierarchy -output [file join $output_dir ${top}.ddc] -write -format verilog -hierarchy -output [file join $output_dir ${top}.v] -write_sdc [file join $output_dir ${top}.sdc] -report_constraint -all_violators > [file join $report_dir constraint.rpt] -report_timing -delay max -max_paths 50 > [file join $report_dir timing_max.rpt] -report_timing -delay min -max_paths 50 > [file join $report_dir timing_min.rpt] -report_area -hierarchy > [file join $report_dir area.rpt] -report_reference > [file join $report_dir reference.rpt] -exit diff --git a/examples/chips/pebble/tapeout/config.toml b/examples/chips/pebble/tapeout/config.toml deleted file mode 100644 index a1bb97799..000000000 --- a/examples/chips/pebble/tapeout/config.toml +++ /dev/null @@ -1,117 +0,0 @@ -[tapeout] -top = "DigitalTop" -sram_process = "smic180" -sram_mdf = "ip/smic180_sram_library.mdf.json" -constraints_sdc = "area/constraints.sdc" -target_library = "/home/bb-runner/Code/eda/D_libraries/SMIC180/SCC018UG_UHD_RVT_V0.4a/SCC018UG_UHD_RVT_V0p4a/liberty/1.8v/scc018ug_uhd_rvt_ss_v1p62_125c_ccs.db" -synthetic_library = [ - "/home/bb-runner/Code/eda/A_packages/tools/syn/R-2020.09-SP5/libraries/syn/dw_foundation.sldb", - "/home/bb-runner/Code/eda/A_packages/tools/syn/R-2020.09-SP5/libraries/syn/standard.sldb", -] -link_library = [] -max_cores = 16 -power_format = "vcd" -power_start_ns = "1000000" -power_end_ns = "5000000" -dc_script = "area/dc.tcl" -power_script = "power/power.tcl" -power_sim_script = "power/power_sim.sh" -power_strip_path = "GatePowerHarness.dut" - -sram_corner = "ss" -lc_shell = "/home/bb-runner/Code/eda/A_packages/tools/lc/R-2020.09-SP5/bin/lc_shell" - -[[tapeout.sram]] -name = "chipyard_sram_32x128" -words = 32 -mux = 4 -bits = 128 - -[[tapeout.sram]] -name = "chipyard_sram_32x22" -words = 32 -mux = 4 -bits = 22 - -[[tapeout.sram]] -name = "chipyard_sram_64x21" -words = 64 -mux = 8 -bits = 21 - -[[tapeout.sram]] -name = "chipyard_sram_64x22" -words = 64 -mux = 8 -bits = 22 - -[[tapeout.sram]] -name = "chipyard_sram_512x8" -words = 512 -mux = 8 -bits = 8 - -[[tapeout.sram]] -name = "chipyard_sram_512x32" -words = 512 -mux = 8 -bits = 32 - -[[tapeout.sram]] -name = "chipyard_sram_512x64" -words = 512 -mux = 8 -bits = 64 - -[[tapeout.sram]] -name = "chipyard_sram_1024x8" -words = 1024 -mux = 8 -bits = 8 - -[[tapeout.sram]] -name = "chipyard_sram_32x21" -words = 32 -mux = 4 -bits = 21 - -[[tapeout.sram]] -name = "chipyard_sram_32x23" -words = 32 -mux = 4 -bits = 23 - -[[tapeout.sram]] -name = "chipyard_sram_64x8" -words = 64 -mux = 8 -bits = 8 - -[[tapeout.sram]] -name = "chipyard_sram_64x128" -words = 64 -# SMIC180 has only six metal layers, so this macro cannot be treated as a -# routable-over cell. mux=4 changes only the internal column organization: -# it keeps the 64x128 1RW + BWEN interface but shrinks the generated macro -# width from about 2013 um (mux=8) to about 1045 um, avoiding long routing walls. -mux = 4 -bits = 128 -bitwrite = true - -[[tapeout.sram]] -name = "chipyard_sram_128x8" -words = 128 -mux = 8 -bits = 8 - -[[tapeout.sram]] -name = "chipyard_sram_256x32" -words = 256 -mux = 8 -bits = 32 - -[[tapeout.sram]] -name = "chipyard_sram_256x64" -words = 256 -mux = 8 -bits = 64 diff --git a/examples/chips/pebble/workloads/CMakeLists.txt b/examples/chips/pebble/workloads/CMakeLists.txt deleted file mode 100644 index c743dec9c..000000000 --- a/examples/chips/pebble/workloads/CMakeLists.txt +++ /dev/null @@ -1,25 +0,0 @@ -get_filename_component(BUCKYBALL_REPO_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../.. ABSOLUTE) - -set(BUCKYBALL_CHIP_COMMON_SOURCES - ${CMAKE_CURRENT_SOURCE_DIR}/common/pebble.c) -set(BUCKYBALL_CHIP_COMMON_HEADERS - ${CMAKE_CURRENT_SOURCE_DIR}/common/pebble.h) -set(BUCKYBALL_CHIP_COMMON_INCLUDE_DIRS - ${CMAKE_CURRENT_SOURCE_DIR}/common) -set(BUCKYBALL_CTEST_TARGET pebble) - -add_custom_target(pebble-ctests-build) -set(BUCKYBALL_CTEST_GROUP_TARGET pebble-ctests-build) - -add_subdirectory(ctests) -buckyball_add_ball_workload_subdirs(ctests) - -add_custom_target(chip-ctests-build DEPENDS pebble-ctests-build) - -add_custom_target(chip-workloads-build ALL - DEPENDS - chip-ctests-build - mlirtest-workloads-build - soctests-build - COMMENT "Building pebble workloads" - VERBATIM) diff --git a/examples/chips/pebble/workloads/ctests/CMakeLists.txt b/examples/chips/pebble/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 575003eea..000000000 --- a/examples/chips/pebble/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -add_buckyball_ctests( - mega_conv_pipeline_test.c - mega_conv_k5_pipeline_test.c - residual_add_lut_test.c - maxpool_im2col_test.c - se_gate_lut_test.c - dual_smatmul_matadd_test.c - dual_conv_add_int2fp_test.c - quant_smatmul_int2fp_test.c - smatmul_mvout_bank_reuse_test.c - mvin_bank78_stress_test.c - im2col_mvin_smatmul_hazard_test.c -) diff --git a/examples/chips/pebble/workloads/ctests/dual_conv_add_int2fp_test.c b/examples/chips/pebble/workloads/ctests/dual_conv_add_int2fp_test.c deleted file mode 100644 index 199edd879..000000000 --- a/examples/chips/pebble/workloads/ctests/dual_conv_add_int2fp_test.c +++ /dev/null @@ -1,80 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -enum { - TILE = 16, - IN = 4, - K = 3, - OUT = 2, - WINDOWS = 4, - PACKED_K = 16, - RESULT_ROWS = TILE * 2 -}; -static int8_t images[2][IN * IN * TILE] __attribute__((aligned(64))); -static int8_t weights[2][PACKED_K * TILE] __attribute__((aligned(64))); -static float scale[TILE] __attribute__((aligned(64))); -static int32_t bias[TILE] __attribute__((aligned(64))); -static float output[TILE * TILE] __attribute__((aligned(64))); - -static int8_t image_value(int branch, int row, int column) { - return (int8_t)(branch * 7 + row * IN + column - 8); -} - -static int32_t expected(int row, int column) { - int32_t sum = 0; - for (int branch = 0; branch < 2; ++branch) - for (int ky = 0; ky < K; ++ky) - for (int kx = 0; kx < K; ++kx) - sum += image_value(branch, row / OUT + ky, row % OUT + kx) * - weights[branch][(ky * K + kx) * TILE + column]; - return sum; -} - -static void run_branch(int branch, int out_bank) { - bb_mvin((uintptr_t)images[branch], 0, IN * IN, 1); - bb_im2col(0, 1, IN, K, 1, 0, 0, 0, 0, 0, 0, WINDOWS); - bb_mvin((uintptr_t)weights[branch], 0, PACKED_K, 1); - bb_smatmul_os(1, 0, out_bank, TILE, TILE, PACKED_K, 1, 1, 0); -} - -int main(void) { - for (int branch = 0; branch < 2; ++branch) { - for (int row = 0; row < IN; ++row) - for (int column = 0; column < IN; ++column) - images[branch][(row * IN + column) * TILE] = - image_value(branch, row, column); - for (int i = 0; i < PACKED_K * TILE; ++i) - weights[branch][i] = (int8_t)((i / TILE * 3 + i % TILE) % 41 - 20); - } - for (int lane = 0; lane < TILE; ++lane) - scale[lane] = 0.05f; - - for (int bank = 0; bank < 8; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)scale, 3, 4, 1); - bb_mvin((uintptr_t)bias, 7, 4, 1); - bb_smatmul_bias(7, 0); - run_branch(0, 2); - run_branch(1, 4); - bb_matadd(2, 4, 6, RESULT_ROWS); - bb_int32_to_fp32(6, 3, 5, RESULT_ROWS, 0); - bb_mvout((uintptr_t)output, 5, RESULT_ROWS, 1); - - for (int row = 0; row < WINDOWS; ++row) - for (int column = 0; column < TILE; ++column) - if (fabsf(output[row * TILE + column] - - expected(row, column) * scale[column]) > 1e-3f) { - printf("dual conv add mismatch row=%d column=%d\n", row, column); - return 1; - } - printf("dual conv add int2fp PASS\n"); - return 0; -} diff --git a/examples/chips/pebble/workloads/ctests/mega_conv_k5_pipeline_test.c b/examples/chips/pebble/workloads/ctests/mega_conv_k5_pipeline_test.c deleted file mode 100644 index 400d2a7dd..000000000 --- a/examples/chips/pebble/workloads/ctests/mega_conv_k5_pipeline_test.c +++ /dev/null @@ -1,70 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include -#include -#include -#include -#include - -enum { INPUT_SIDE = 8, OUTPUT_SIDE = 4, KERNEL = 5, TILE = 16, PADDED_K = 32 }; -static int8_t input[INPUT_SIDE * INPUT_SIDE * TILE] - __attribute__((aligned(64))); -static int8_t weight[PADDED_K * TILE] __attribute__((aligned(64))); -static int32_t bias[TILE] __attribute__((aligned(64))); -static float scale[TILE] __attribute__((aligned(64))); -static int8_t output[OUTPUT_SIDE * OUTPUT_SIDE * TILE] - __attribute__((aligned(64))); - -int main(void) { - for (int pixel = 0; pixel < INPUT_SIDE * INPUT_SIDE; ++pixel) - input[pixel * TILE] = pixel % 5 - 2; - for (int k = 0; k < KERNEL * KERNEL; ++k) - for (int channel = 0; channel < TILE; ++channel) - weight[k * TILE + channel] = (k + channel) % 3 - 1; - for (int channel = 0; channel < TILE; ++channel) - scale[channel] = 1.0f; - - for (int bank = 0; bank < 8; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)input, 0, INPUT_SIDE * INPUT_SIDE, 1); - bb_mvin((uintptr_t)weight, 2, PADDED_K, 1); - bb_mvin((uintptr_t)bias, 3, 4, 1); - bb_mvin((uintptr_t)scale, 5, 4, 1); - bb_im2col(0, 1, INPUT_SIDE, KERNEL, 1, 0, 0, 0, 0, 0, 0, - OUTPUT_SIDE * OUTPUT_SIDE); - for (int row_tile = 0; row_tile < 2; ++row_tile) - for (int k_tile = 0; k_tile < 2; ++k_tile) - for (int row = 0; row < 8; ++row) { - int input_row = k_tile * 16 + row_tile * 8 + row; - int output_row = (row_tile * 2 + k_tile) * 8 + row; - bb_maxpool(1, 7, 1, 1, 1, 1, 0, input_row, output_row, 1, 0, 0); - } - bb_smatmul_bias(3, 0); - bb_smatmul_os(7, 2, 4, OUTPUT_SIDE * OUTPUT_SIDE, TILE, PADDED_K, 1, 1, 0); - bb_quant_i32_to_i8(4, 5, 6, OUTPUT_SIDE * OUTPUT_SIDE * 4, 0, 0, OUTPUT_SIDE, - OUTPUT_SIDE, OUTPUT_SIDE, 0); - bb_mvout((uintptr_t)output, 6, OUTPUT_SIDE * OUTPUT_SIDE, 1); - - for (int y = 0; y < OUTPUT_SIDE; ++y) - for (int x = 0; x < OUTPUT_SIDE; ++x) - for (int channel = 0; channel < TILE; ++channel) { - int expected = 0; - for (int ky = 0; ky < KERNEL; ++ky) - for (int kx = 0; kx < KERNEL; ++kx) { - int pixel = (y + ky) * INPUT_SIDE + x + kx; - int k = ky * KERNEL + kx; - expected += input[pixel * TILE] * weight[k * TILE + channel]; - } - int index = (y * OUTPUT_SIDE + x) * TILE + channel; - if (output[index] != expected) { - printf( - "mega_conv_k5 FAIL y=%d x=%d channel=%d expected=%d actual=%d\n", - y, x, channel, expected, output[index]); - return 1; - } - } - printf("mega_conv_k5 PASS\n"); - return 0; -} diff --git a/examples/chips/pebble/workloads/mlir_tests/CMakeLists.txt b/examples/chips/pebble/workloads/mlir_tests/CMakeLists.txt deleted file mode 100644 index ffedb96f5..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -add_custom_target(pebble-mlir-tests-build) - -add_subdirectory(bank) -add_subdirectory(tile) -add_subdirectory(linalg) - -add_dependencies(chip-mlirtests-build pebble-mlir-tests-build) diff --git a/examples/chips/pebble/workloads/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir b/examples/chips/pebble/workloads/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir deleted file mode 100644 index 53bc9a0d4..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir +++ /dev/null @@ -1,46 +0,0 @@ -// Chip integration: f32-to-i8 quantization -> im2col k=1 -> mvout. - -func.func private @check_result(memref<16x16xi8>) -> () - -memref.global "private" constant @fp_in : memref<1x16xf32> = - dense<[[1.0, 2.0, 3.0, 4.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]> - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %depth_in = arith.constant 1 : i64 - %depth_out = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %quant_iter = arith.constant 4 : i64 - %input_size = arith.constant 1 : i64 - %scale = arith.constant 1.0 : f32 - %da_addr = arith.constant 0 : i64 - %ksize = arith.constant 1 : i64 - %pad = arith.constant 0 : i64 - - %input = memref.get_global @fp_in : memref<1x16xf32> - %output = memref.alloc() alignment = 64 : memref<16x16xi8> - - %bin = buckyball.bank_alloc - %bq = buckyball.bank_alloc - %bout = buckyball.bank_alloc - %loaded = buckyball.bank_mvin %input %bin %depth_in %stride - : memref<1x16xf32> i64 i64 i64 - %q = buckyball.bank_quant_f32_to_i8 %loaded %bq %quant_iter %scale - : i64 i64 i64 f32 - %base = arith.constant 0 : i64 - %lane = arith.constant 0 : i64 - %im = buckyball.bank_im2col %q %bout %input_size %ksize %stride %pad %base - %lane - : i64 i64 i64 i64 i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %im %depth_out %stride - : memref<16x16xi8> i64 i64 i64 - buckyball.bank_release %loaded : i64 - buckyball.bank_release %q : i64 - buckyball.bank_release %stored : i64 - - func.call @check_result(%output) : (memref<16x16xi8>) -> () - memref.dealloc %output : memref<16x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/CMakeLists.txt b/examples/chips/pebble/workloads/mlir_tests/linalg/CMakeLists.txt deleted file mode 100644 index 73893c006..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/CMakeLists.txt +++ /dev/null @@ -1,33 +0,0 @@ -set(BUCKYBALL_MLIR_GROUP_TARGET pebble-mlir-tests-build) -set(BUCKYBALL_MLIR_TEST_PREFIX linalg) - -set(PEBBLE_LINALG_FUNCTIONAL_PASSES - -convert-linalg-to-tile - -convert-tile-to-buckyball - -lower-buckyball-to-bank-ssa - -assign-physical-banks - ${BUCKYBALL_LOWER_BANK_SSA_TO_INTRINSICS} - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - -foreach(test - linalg_relu_513x17_i32 - linalg_matmul_16x16_i8 - linalg_transpose_16x16_i8 - linalg_transpose_detect_1x84x100_f32 - linalg_transpose_nchw_nhwc_1x3x8x8_f32 - linalg_transpose_oihw_ohwi_4x8x3x3_f32 - linalg_mega_conv2d_k3_c16) - add_buckyball_mlir_test(${test} - TARGET pebble - PASSES ${PEBBLE_LINALG_FUNCTIONAL_PASSES}) -endforeach() diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_matmul_16x16_i8.mlir b/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_matmul_16x16_i8.mlir deleted file mode 100644 index e92cd41c6..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_matmul_16x16_i8.mlir +++ /dev/null @@ -1,44 +0,0 @@ -func.func private @check_result(memref<16x16xi32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_i32 = arith.constant 0 : i32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %dim = arith.constant 16 : index - %a = memref.alloc() alignment = 64 : memref<16x16xi8> - %b = memref.alloc() alignment = 64 : memref<16x16xi8> - %c = memref.alloc() alignment = 64 : memref<16x16xi32> - - scf.for %row = %zero to %dim step %one { - scf.for %column = %zero to %dim step %one { - %r3 = arith.muli %row, %three : index - %c5 = arith.muli %column, %five : index - %asum = arith.addi %r3, %c5 : index - %arem = arith.remui %asum, %seven : index - %aval = arith.subi %arem, %three : index - %av = arith.index_cast %aval : index to i8 - %r2 = arith.muli %row, %two : index - %c3 = arith.muli %column, %three : index - %bsum = arith.addi %r2, %c3 : index - %brem = arith.remui %bsum, %five : index - %bval = arith.subi %brem, %two : index - %bv = arith.index_cast %bval : index to i8 - memref.store %av, %a[%row, %column] : memref<16x16xi8> - memref.store %bv, %b[%row, %column] : memref<16x16xi8> - } - } - linalg.fill ins(%zero_i32 : i32) outs(%c : memref<16x16xi32>) - linalg.matmul ins(%a, %b : memref<16x16xi8>, memref<16x16xi8>) - outs(%c : memref<16x16xi32>) - - func.call @check_result(%c) : (memref<16x16xi32>) -> () - memref.dealloc %a : memref<16x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %c : memref<16x16xi32> - return %zero_i8 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_mega_conv2d_k3_c16.mlir b/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_mega_conv2d_k3_c16.mlir deleted file mode 100644 index 0e5d203e7..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_mega_conv2d_k3_c16.mlir +++ /dev/null @@ -1,84 +0,0 @@ -#input = affine_map<(n, h, w, c) -> (n, h, w, c)> -#weight = affine_map<(n, h, w, c) -> (n, c, 0, 0)> -#channel = affine_map<(n, h, w, c) -> (c)> -#scalar = affine_map<(n, h, w, c) -> (0)> -#output = affine_map<(n, h, w, c) -> (n, c, 0, 0)> - -func.func private @check_result(memref<1x16x1x1xf32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_i32 = arith.constant 0 : i32 - %zero_f32 = arith.constant 0.0 : f32 - %one_i8 = arith.constant 1 : i8 - %one_f32 = arith.constant 1.0 : f32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %eight = arith.constant 8 : index - %nine = arith.constant 9 : index - %sixteen = arith.constant 16 : index - %input = memref.alloc() : memref<1x3x3x16xi8> - %weight = memref.alloc() : memref<1x16x16x16xi8> - %bias = memref.alloc() : memref<16xi32> - %scale = memref.alloc() : memref<16xf32> - %lut = memref.alloc() : memref<1xi8> - %output = memref.alloc() : memref<1x16x1x1xf32> - linalg.fill ins(%zero_i8 : i8) outs(%input : memref<1x3x3x16xi8>) - linalg.fill ins(%zero_i8 : i8) outs(%weight : memref<1x16x16x16xi8>) - linalg.fill ins(%one_f32 : f32) outs(%scale : memref<16xf32>) - linalg.fill ins(%zero_i8 : i8) outs(%lut : memref<1xi8>) - linalg.fill ins(%zero_f32 : f32) outs(%output : memref<1x16x1x1xf32>) - scf.for %position = %zero to %nine step %one { - %p2 = arith.muli %position, %two : index - %h = arith.divui %position, %three : index - %w = arith.remui %position, %three : index - scf.for %channel = %zero to %sixteen step %one { - %c3 = arith.muli %channel, %three : index - %isum = arith.addi %p2, %c3 : index - %irem = arith.remui %isum, %five : index - %ival = arith.subi %irem, %two : index - %iv = arith.index_cast %ival : index to i8 - memref.store %iv, %input[%zero, %h, %w, %channel] : memref<1x3x3x16xi8> - scf.for %out = %zero to %sixteen step %one { - %wsum0 = arith.addi %c3, %p2 : index - %wsum = arith.addi %wsum0, %out : index - %wrem = arith.remui %wsum, %five : index - %wval = arith.subi %wrem, %two : index - %wv = arith.index_cast %wval : index to i8 - memref.store %wv, %weight[%zero, %channel, %position, %out] : memref<1x16x16x16xi8> - } - } - } - scf.for %out = %zero to %sixteen step %one { - %bval = arith.subi %out, %eight : index - %bv = arith.index_cast %bval : index to i32 - memref.store %bv, %bias[%out] : memref<16xi32> - } - linalg.generic { - indexing_maps = [#input, #weight, #channel, #channel, #scalar, #output], - iterator_types = ["parallel", "reduction", "reduction", "parallel"]} - ins(%input, %weight, %bias, %scale, %lut - : memref<1x3x3x16xi8>, memref<1x16x16x16xi8>, memref<16xi32>, - memref<16xf32>, memref<1xi8>) - outs(%output : memref<1x16x1x1xf32>) - attrs = {activation = 0 : i64, buckyball.mega_conv2d = true, - buckyball.mega_kernel = true, final_output = true, - kernel = 3 : i64, mega_kernel_id = "conv_k3_c16", - mega_kernel_size = 1 : i64, mega_kernel_stage = 0 : i64, - output_scale = 1.0 : f32, pad_high = 0 : i64, - pad_low = 0 : i64, stride = 1 : i64} { - ^bb0(%in: i8, %w: i8, %b: i32, %s: f32, %l: i8, %out: f32): - linalg.yield %out : f32 - } - func.call @check_result(%output) : (memref<1x16x1x1xf32>) -> () - memref.dealloc %input : memref<1x3x3x16xi8> - memref.dealloc %weight : memref<1x16x16x16xi8> - memref.dealloc %bias : memref<16xi32> - memref.dealloc %scale : memref<16xf32> - memref.dealloc %lut : memref<1xi8> - memref.dealloc %output : memref<1x16x1x1xf32> - return %zero_i8 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_16x16_i8.mlir b/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_16x16_i8.mlir deleted file mode 100644 index 14c31826a..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_16x16_i8.mlir +++ /dev/null @@ -1,28 +0,0 @@ -func.func private @check_result(memref<16x16xi8>) -> () - -func.func @main() -> i8 { - %zero = arith.constant 0 : i8 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c3 = arith.constant 3 : i8 - %c5 = arith.constant 5 : i8 - %c16 = arith.constant 16 : index - %input = memref.alloc() alignment = 64 : memref<16x16xi8> - %output = memref.alloc() alignment = 64 : memref<16x16xi8> - scf.for %i = %c0 to %c16 step %c1 { - scf.for %j = %c0 to %c16 step %c1 { - %ii = arith.index_cast %i : index to i8 - %jj = arith.index_cast %j : index to i8 - %iv = arith.muli %ii, %c3 : i8 - %jv = arith.muli %jj, %c5 : i8 - %v = arith.addi %iv, %jv : i8 - memref.store %v, %input[%i, %j] : memref<16x16xi8> - } - } - linalg.transpose ins(%input : memref<16x16xi8>) - outs(%output : memref<16x16xi8>) permutation = [1, 0] - func.call @check_result(%output) : (memref<16x16xi8>) -> () - memref.dealloc %input : memref<16x16xi8> - memref.dealloc %output : memref<16x16xi8> - return %zero : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/CMakeLists.txt b/examples/chips/pebble/workloads/mlir_tests/tile/CMakeLists.txt deleted file mode 100644 index 1a3c8e258..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/CMakeLists.txt +++ /dev/null @@ -1,75 +0,0 @@ -set(BUCKYBALL_MLIR_GROUP_TARGET pebble-mlir-tests-build) -set(BUCKYBALL_MLIR_TEST_PREFIX tile) - -set(PEBBLE_TILE_FUNCTIONAL_PASSES - -convert-tile-to-buckyball - -lower-buckyball-to-bank-ssa - -assign-physical-banks - ${BUCKYBALL_LOWER_BANK_SSA_TO_INTRINSICS} - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - -add_buckyball_mlir_test(mega_matmul_m17_k65_n17 - TARGET pebble PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) - -add_buckyball_mlir_test(mega_conv2d - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) - -add_buckyball_mlir_test(mega_conv2d_k5_tail - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) - -add_buckyball_mlir_test(mega_conv2d_s2_depthwise_s2 - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) - -file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv_exact_input.i8 - PEBBLE_GAP_CONV_INPUT_HEX HEX) -file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_7x7_c288_input.i8 - PEBBLE_GAP_C288_INPUT_HEX HEX) -file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv_exact_weight.i8 - PEBBLE_GAP_CONV_WEIGHT_HEX HEX) -file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv2_exact_weight2.i8 - PEBBLE_GAP_CONV_WEIGHT2_HEX HEX) -file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv2_exact_weight3.i8 - PEBBLE_GAP_CONV_WEIGHT3_HEX HEX) -add_buckyball_mlir_test(mega_global_avg_conv_exact - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) -add_buckyball_mlir_test(mega_global_avg_7x7_c288_exact - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) -add_buckyball_mlir_test(mega_global_avg_conv2_exact - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) -unset(PEBBLE_GAP_CONV_INPUT_HEX) -unset(PEBBLE_GAP_C288_INPUT_HEX) -unset(PEBBLE_GAP_CONV_WEIGHT_HEX) -unset(PEBBLE_GAP_CONV_WEIGHT2_HEX) -unset(PEBBLE_GAP_CONV_WEIGHT3_HEX) - -add_buckyball_mlir_test(mega_se_gate_conv - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) - -add_buckyball_mlir_test(mega_matmul - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) - -foreach(test - tile_matmul_16x16_i8 - tile_transpose_16x16_i8) - add_buckyball_mlir_test(${test} - TARGET pebble - PASSES ${PEBBLE_TILE_FUNCTIONAL_PASSES}) -endforeach() diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d.mlir deleted file mode 100644 index 8560e7d7b..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d.mlir +++ /dev/null @@ -1,100 +0,0 @@ -func.func private @check_result(memref<1x16x2x2xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_i32 = arith.constant 0 : i32 - %one_f32 = arith.constant 1.0 : f32 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c3 = arith.constant 3 : index - %c4 = arith.constant 4 : index - %c5 = arith.constant 5 : index - %c7 = arith.constant 7 : index - %c16 = arith.constant 16 : index - %input = memref.alloc() alignment = 64 : memref<1x4x4x2xi8> - %conv_weight = memref.alloc() alignment = 64 : memref<1x2x16x16xi8> - %conv_bias = memref.alloc() alignment = 64 : memref<16xi32> - %conv_scale = memref.alloc() alignment = 64 : memref<16xf32> - %lut = memref.alloc() alignment = 64 : memref<1xi8> - %intermediate = memref.alloc() alignment = 64 : memref<1x4x4x16xi8> - %depthwise_weight = memref.alloc() alignment = 64 : memref<3x3x16x1xi8> - %depthwise_bias = memref.alloc() alignment = 64 : memref<16xi32> - %depthwise_scale = memref.alloc() alignment = 64 : memref<16xf32> - %depthwise_output = memref.alloc() alignment = 64 : memref<1x4x4x16xi8> - %output = memref.alloc() alignment = 64 : memref<1x16x2x2xi8> - scf.for %h = %c0 to %c4 step %c1 { - scf.for %w = %c0 to %c4 step %c1 { - scf.for %ic = %c0 to %c2 step %c1 { - %h3 = arith.muli %h, %c3 : index - %w2 = arith.muli %w, %c2 : index - %sum0 = arith.addi %h3, %w2 : index - %sum = arith.addi %sum0, %ic : index - %rem = arith.remui %sum, %c7 : index - %signed = arith.subi %rem, %c3 : index - %value = arith.index_cast %signed : index to i8 - memref.store %value, %input[%c0, %h, %w, %ic] : memref<1x4x4x2xi8> - } - } - } - scf.for %ic = %c0 to %c2 step %c1 { - scf.for %p = %c0 to %c16 step %c1 { - scf.for %oc = %c0 to %c16 step %c1 { - %oc3 = arith.muli %oc, %c3 : index - %sum0 = arith.addi %p, %ic : index - %sum = arith.addi %sum0, %oc3 : index - %rem = arith.remui %sum, %c5 : index - %signed = arith.subi %rem, %c2 : index - %value = arith.index_cast %signed : index to i8 - memref.store %value, %conv_weight[%c0, %ic, %p, %oc] : memref<1x2x16x16xi8> - } - } - } - scf.for %oc = %c0 to %c16 step %c1 { - %signed = arith.subi %oc, %c4 : index - %value = arith.index_cast %signed : index to i32 - memref.store %value, %conv_bias[%oc] : memref<16xi32> - } - linalg.fill ins(%one_f32 : f32) outs(%conv_scale : memref<16xf32>) - linalg.fill ins(%zero_i8 : i8) outs(%lut : memref<1xi8>) - linalg.fill ins(%zero_i8 : i8) outs(%intermediate : memref<1x4x4x16xi8>) - scf.for %kh = %c0 to %c3 step %c1 { - scf.for %kw = %c0 to %c3 step %c1 { - scf.for %oc = %c0 to %c16 step %c1 { - %kw2 = arith.muli %kw, %c2 : index - %sum0 = arith.addi %kh, %kw2 : index - %sum = arith.addi %sum0, %oc : index - %rem = arith.remui %sum, %c5 : index - %signed = arith.subi %rem, %c2 : index - %value = arith.index_cast %signed : index to i8 - memref.store %value, %depthwise_weight[%kh, %kw, %oc, %c0] : memref<3x3x16x1xi8> - } - } - } - linalg.fill ins(%zero_i32 : i32) outs(%depthwise_bias : memref<16xi32>) - linalg.fill ins(%one_f32 : f32) outs(%depthwise_scale : memref<16xf32>) - linalg.fill ins(%zero_i8 : i8) outs(%depthwise_output : memref<1x4x4x16xi8>) - linalg.fill ins(%zero_i8 : i8) outs(%output : memref<1x16x2x2xi8>) - tile.mega_kernel %input %output - : memref<1x4x4x2xi8> memref<1x16x2x2xi8> { - tile.mega_conv2d %input %conv_weight %conv_bias %conv_scale %lut - %intermediate - : memref<1x4x4x2xi8> memref<1x2x16x16xi8> memref<16xi32> - memref<16xf32> memref<1xi8> memref<1x4x4x16xi8> - tile.mega_conv2d_depthwise %intermediate %depthwise_weight - %depthwise_bias %depthwise_scale %lut %depthwise_output - - : memref<1x4x4x16xi8> memref<3x3x16x1xi8> memref<16xi32> - memref<16xf32> memref<1xi8> memref<1x4x4x16xi8> - tile.mega_max_pool2d %depthwise_output %output - - : memref<1x4x4x16xi8> memref<1x16x2x2xi8> - } - func.call @check_result(%output) : (memref<1x16x2x2xi8>) -> () - return %zero_i8 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_k5_tail.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_k5_tail.mlir deleted file mode 100644 index ea42e3f63..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_k5_tail.mlir +++ /dev/null @@ -1,62 +0,0 @@ -func.func private @check_result(memref<1x16x5x5xf32>) -func.func @main() -> i8 { - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c3 = arith.constant 3 : index - %c5 = arith.constant 5 : index - %c7 = arith.constant 7 : index - %c16 = arith.constant 16 : index - %c25 = arith.constant 25 : index - %zi8 = arith.constant 0 : i8 - %zi32 = arith.constant 0 : i32 - %zf32 = arith.constant 0.0 : f32 - %one = arith.constant 1.0 : f32 - %input = memref.alloc() alignment = 64 : memref<1x5x5x2xi8> - %weight = memref.alloc() alignment = 64 : memref<1x2x32x16xi8> - %bias = memref.alloc() alignment = 64 : memref<16xi32> - %scale = memref.alloc() alignment = 64 : memref<16xf32> - %lut = memref.alloc() alignment = 64 : memref<1xi8> - %output = memref.alloc() alignment = 64 : memref<1x16x5x5xf32> - linalg.fill ins(%zi8 : i8) outs(%weight : memref<1x2x32x16xi8>) - linalg.fill ins(%zi32 : i32) outs(%bias : memref<16xi32>) - linalg.fill ins(%one : f32) outs(%scale : memref<16xf32>) - linalg.fill ins(%zi8 : i8) outs(%lut : memref<1xi8>) - linalg.fill ins(%zf32 : f32) outs(%output : memref<1x16x5x5xf32>) - scf.for %y = %c0 to %c5 step %c1 { - scf.for %x = %c0 to %c5 step %c1 { - scf.for %c = %c0 to %c2 step %c1 { - %y3 = arith.muli %y, %c3 : index - %x2 = arith.muli %x, %c2 : index - %s0 = arith.addi %y3, %x2 : index - %s = arith.addi %s0, %c : index - %r = arith.remui %s, %c7 : index - %v = arith.subi %r, %c3 : index - %i = arith.index_cast %v : index to i8 - memref.store %i, %input[%c0, %y, %x, %c] : memref<1x5x5x2xi8> - } - } - } - scf.for %c = %c0 to %c2 step %c1 { - scf.for %k = %c0 to %c25 step %c1 { - scf.for %o = %c0 to %c16 step %c1 { - %s0 = arith.addi %k, %c : index - %s = arith.addi %s0, %o : index - %r = arith.remui %s, %c3 : index - %v = arith.subi %r, %c1 : index - %i = arith.index_cast %v : index to i8 - memref.store %i, %weight[%c0, %c, %k, %o] : memref<1x2x32x16xi8> - } - } - } - tile.mega_kernel %input %output : memref<1x5x5x2xi8> memref<1x16x5x5xf32> { - tile.mega_conv2d %input %weight %bias %scale %lut %output - - : memref<1x5x5x2xi8> memref<1x2x32x16xi8> memref<16xi32> - memref<16xf32> memref<1xi8> memref<1x16x5x5xf32> - } - func.call @check_result(%output) : (memref<1x16x5x5xf32>) -> () - %ok = arith.constant 0 : i8 - return %ok : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_s2_depthwise_s2.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_s2_depthwise_s2.mlir deleted file mode 100644 index f86aa1d80..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_s2_depthwise_s2.mlir +++ /dev/null @@ -1,91 +0,0 @@ -func.func private @check_result(memref<1x4x4x16xi8>) -> () - -func.func @main() -> i8 { - %i8_0 = arith.constant 0 : i8 - %i32_0 = arith.constant 0 : i32 - %f32_1 = arith.constant 1.0 : f32 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c3 = arith.constant 3 : index - %c4 = arith.constant 4 : index - %c5 = arith.constant 5 : index - %c7 = arith.constant 7 : index - %c9 = arith.constant 9 : index - %c16 = arith.constant 16 : index - %input = memref.alloc() alignment = 64 : memref<1x16x16x3xi8> - %weight0 = memref.alloc() alignment = 64 : memref<1x3x16x16xi8> - %bias0 = memref.alloc() alignment = 64 : memref<16xi32> - %scale0 = memref.alloc() alignment = 64 : memref<16xf32> - %lut = memref.alloc() alignment = 64 : memref<1xi8> - %middle = memref.alloc() alignment = 64 : memref<1x8x8x16xi8> - %weight1 = memref.alloc() alignment = 64 : memref<3x3x16x1xi8> - %bias1 = memref.alloc() alignment = 64 : memref<16xi32> - %scale1 = memref.alloc() alignment = 64 : memref<16xf32> - %output = memref.alloc() alignment = 64 : memref<1x4x4x16xi8> - scf.for %y = %c0 to %c16 step %c1 { - scf.for %x = %c0 to %c16 step %c1 { - scf.for %ic = %c0 to %c3 step %c1 { - %a = arith.muli %y, %c3 : index - %b = arith.muli %x, %c2 : index - %d = arith.addi %a, %b : index - %e = arith.addi %d, %ic : index - %r = arith.remui %e, %c7 : index - %s = arith.subi %r, %c3 : index - %v = arith.index_cast %s : index to i8 - memref.store %v, %input[%c0, %y, %x, %ic] : memref<1x16x16x3xi8> - } - } - } - scf.for %ic = %c0 to %c3 step %c1 { - scf.for %p = %c0 to %c16 step %c1 { - scf.for %oc = %c0 to %c16 step %c1 { - %oc3 = arith.muli %oc, %c3 : index - %a = arith.addi %p, %ic : index - %b = arith.addi %a, %oc3 : index - %r = arith.remui %b, %c5 : index - %s = arith.subi %r, %c2 : index - %v = arith.index_cast %s : index to i8 - memref.store %v, %weight0[%c0, %ic, %p, %oc] : memref<1x3x16x16xi8> - } - } - } - scf.for %oc = %c0 to %c16 step %c1 { - %s = arith.subi %oc, %c4 : index - %v = arith.index_cast %s : index to i32 - memref.store %v, %bias0[%oc] : memref<16xi32> - } - linalg.fill ins(%f32_1 : f32) outs(%scale0 : memref<16xf32>) - linalg.fill ins(%i8_0 : i8) outs(%lut : memref<1xi8>) - linalg.fill ins(%i8_0 : i8) outs(%middle : memref<1x8x8x16xi8>) - scf.for %ky = %c0 to %c3 step %c1 { - scf.for %kx = %c0 to %c3 step %c1 { - scf.for %c = %c0 to %c16 step %c1 { - %kx2 = arith.muli %kx, %c2 : index - %a = arith.addi %ky, %kx2 : index - %b = arith.addi %a, %c : index - %r = arith.remui %b, %c5 : index - %s = arith.subi %r, %c2 : index - %v = arith.index_cast %s : index to i8 - memref.store %v, %weight1[%ky, %kx, %c, %c0] : memref<3x3x16x1xi8> - } - } - } - linalg.fill ins(%i32_0 : i32) outs(%bias1 : memref<16xi32>) - linalg.fill ins(%f32_1 : f32) outs(%scale1 : memref<16xf32>) - linalg.fill ins(%i8_0 : i8) outs(%output : memref<1x4x4x16xi8>) - tile.mega_kernel %input %output : memref<1x16x16x3xi8> memref<1x4x4x16xi8> { - tile.mega_conv2d %input %weight0 %bias0 %scale0 %lut %middle - - : memref<1x16x16x3xi8> memref<1x3x16x16xi8> memref<16xi32> - memref<16xf32> memref<1xi8> memref<1x8x8x16xi8> - tile.mega_conv2d_depthwise %middle %weight1 %bias1 %scale1 %lut %output - - : memref<1x8x8x16xi8> memref<3x3x16x1xi8> memref<16xi32> - memref<16xf32> memref<1xi8> memref<1x4x4x16xi8> - } - func.call @check_result(%output) : (memref<1x4x4x16xi8>) -> () - return %i8_0 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_7x7_c288_exact.mlir.in b/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_7x7_c288_exact.mlir.in deleted file mode 100644 index f8336126d..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_7x7_c288_exact.mlir.in +++ /dev/null @@ -1,16 +0,0 @@ -memref.global "private" constant @input : memref<1x7x7x288xi8> = dense<"0x@PEBBLE_GAP_C288_INPUT_HEX@"> alignment = 64 - -func.func private @poison_reused_banks() -> () -func.func private @check_result(memref<1x1x1x288xi8>) -> () - -func.func @main() -> i8 { - %zero = arith.constant 0 : i8 - %input = memref.get_global @input : memref<1x7x7x288xi8> - %output = memref.alloc() alignment = 64 : memref<1x1x1x288xi8> - func.call @poison_reused_banks() : () -> () - tile.mega_kernel %input %output : memref<1x7x7x288xi8> memref<1x1x1x288xi8> { - tile.mega_global_avg_pool %input %output : memref<1x7x7x288xi8> memref<1x1x1x288xi8> - } - func.call @check_result(%output) : (memref<1x1x1x288xi8>) -> () - return %zero : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv2_exact.mlir.in b/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv2_exact.mlir.in deleted file mode 100644 index eac3dd086..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv2_exact.mlir.in +++ /dev/null @@ -1,58 +0,0 @@ -memref.global "private" constant @input : memref<1x14x14x96xi8> = dense<"0x@PEBBLE_GAP_CONV_INPUT_HEX@"> alignment = 64 -memref.global "private" constant @weight1raw : memref<49168xi8> = dense<"0x00000000000000000000000000000000@PEBBLE_GAP_CONV_WEIGHT_HEX@"> alignment = 64 -memref.global "private" constant @bias1 : memref<24xi32> = dense<[-1929, -3345, -826, -4425, -1950, -1378, -2544, 3397, -134, -2621, 385, -1868, -3754, -948, 275, -53, -1955, -3197, -1065, -298, 536, -2267, -3803, -48]> alignment = 64 -memref.global "private" constant @scale1 : memref<24xf32> = dense<[9.64902255E-11, 1.20455174E-10, 9.11374864E-11, 6.07460609E-11, 6.96993296E-11, 2.21286572E-10, 8.11237535E-11, 1.33313652E-10, 1.26243266E-10, 1.44089102E-10, 0.0341239199, 6.56752361E-11, 6.62821395E-11, 1.91158489E-9, 1.69707901E-10, 4.13074019E-10, 5.95885979E-11, 8.92385332E-11, 6.12655274E-11, 2.72214418E-10, 2.77576795E-10, 2.37361242E-10, 6.75774228E-11, 1.48867613E-10]> alignment = 64 -memref.global "private" constant @weight2raw : memref<36880xi8> = dense<"0x00000000000000000000000000000000@PEBBLE_GAP_CONV_WEIGHT2_HEX@"> alignment = 64 -memref.global "private" constant @bias2 : memref<96xi32> = dense<[7051, 3006, 26558, 11845, -12167, 14432, -194, 9249, -17237, 22963, 7410, 91984, 105591, 6346, 90458, 43212, 45789, 33855, 29981, 10245, 4370, -34, -712, 15905, 22004, 549465, -5252, -3669, 7224, 18985, 8518, 8198, 16536, 51867, 4243, 6598, 9711, 18037, 21720, 397303, -253, 10751, -10281, -10424, 1839, 25010, 6995, -15222, 1786, -3524, -15704, 13370, 8813, 27391, 8927, -12218, -4217, -7494, 4135, 12616, 9593, 1431, -5377, 9947, 370, 11072, 5546, 22002, 10029, -3861, 904, 95829, 4717, -14504, 52996, 33835, 12322, 3329, 21275, -872, 1465, 17484, 6331, 63424, -29681, 34866, 3915, -2807, 71222, 5504, -13650, 15420, 7960, -9852, -13797, 21711]> alignment = 64 -memref.global "private" constant @scale2 : memref<96xf32> = dense<[0.00181691104, 4.059140E-3, 0.00131794217, 0.00212815683, 0.00430351309, 0.00403421139, 5.176660E-3, 0.00188985514, 0.00558878435, 0.00152020913, 0.00459996238, 5.82394132E-4, 4.24394966E-4, 0.0019368825, 7.07225234E-4, 6.37308462E-4, 0.00137251348, 7.15450151E-4, 0.00140084175, 0.00317073381, 5.073430E-3, 0.00307706394, 0.00386786251, 0.00195057772, 0.00210299343, 1.08822038E-4, 0.00415666727, 0.00333561096, 0.00485164393, 7.86353368E-4, 0.00419554161, 0.00339963497, 0.00123977242, 0.00106979779, 0.00394831039, 0.00133779203, 0.00282861362, 0.00155300961, 0.00196025497, 5.82583889E-5, 0.0032499244, 0.00156923081, 0.00266220793, 0.00191042456, 0.00408306951, 9.02582774E-4, 0.00326385885, 0.00143681327, 0.00142906886, 0.0026714697, 0.00458595809, 0.00430518808, 2.272860E-3, 0.00113706058, 0.00272057392, 0.00554801896, 6.058620E-3, 0.00159710483, 0.00501794415, 0.00278559304, 0.00328800408, 0.00337952748, 0.00245677959, 0.00205389271, 0.00226871902, 0.00263371225, 9.1695087E-4, 0.00129066338, 0.00205622637, 2.126570E-3, 9.56189353E-4, 2.12495346E-4, 9.6136285E-4, 0.00436848169, 9.72485286E-4, 4.94753069E-4, 0.00348583842, 0.00738966605, 0.00289806188, 0.00333968201, 0.00305957114, 0.00181978801, 0.00184966961, 3.52834439E-4, 0.00117929338, 9.00093349E-4, 0.00424369285, 0.00269078952, 5.541500E-4, 0.0025951541, 0.00531184906, 0.00391697325, 0.00378050678, 0.00192236435, 0.00131825986, 0.00273139146]> alignment = 64 -memref.global "private" constant @lut : memref<256xi8> = dense<"0x191A1A1A1A1A1A1B1B1B1B1B1B1C1C1C1C1C1C1D1D1D1D1D1D1E1E1E1E1E1E1F1F1F1F1F1F2020202020202121212121212222222222222323232323232424242424242525252525252626262626262727272727272828282828282929292929292A2A2A2A2A2A2B2B2B2B2B2B2C2C2C2C2C2C2D2D2D2D2D2D2E2E2E2E2E2E2F0404040505050505050606060606060707070707070808080808080909090909090A0A0A0A0A0A0B0B0B0B0B0B0C0C0C0C0C0C0D0D0D0D0D0D0E0E0E0E0E0E0F0F0F0F0F0F1010101010101111111111111212121212121313131313131414141414141515151515151616161616161717171717171818181818181919191919"> alignment = 64 -memref.global "private" constant @weight3raw : memref<73744xi8> = dense<"0x00000000000000000000000000000000@PEBBLE_GAP_CONV_WEIGHT3_HEX@"> alignment = 64 -memref.global "private" constant @bias3 : memref<40xi32> = dense<[104, -13887, -3685, -19928, 5678, 8790, -6325, -4745, 15957, 3843, 4642, 11420, -2754, -19949, 17269, -673, 16732, -31364, 19420, 3384, -1286, -5401, 3250, -9546, 3517, 6568, 9818, -3768, -8763, 19005, 2689, -2236, -3490, 6369, -18573, -726, -16056, 10604, -4229, -7803]> alignment = 64 -memref.global "private" constant @scale3 : memref<40xf32> = dense<[0.007250275, 0.0052957274, 0.00507997582, 5.148440E-3, 0.00615183357, 0.00551314512, 0.00447647972, 0.00624849601, 4.808580E-3, 0.00621820614, 0.00767033594, 0.00502041401, 0.00771069526, 0.00713740429, 5.340260E-3, 0.00554999569, 0.00578600029, 0.00608736509, 0.00647644047, 0.00674922066, 0.00569664268, 0.00644684816, 0.00639614975, 0.00778135518, 0.00677069137, 0.00366282626, 0.00660506124, 0.00643686252, 0.00655329926, 0.00661863247, 0.00624936819, 0.0104040103, 0.00504696835, 7.654780E-3, 0.00546469772, 5.341790E-3, 0.0050635729, 0.00579059403, 0.0084069958, 0.0039168736]> alignment = 64 -memref.global "private" constant @none : memref<1xi8> = dense<0> alignment = 64 -func.func private @check_result(memref<1x14x14x40xi8>) -> () func.func private @poison_reused_bank() -> () - -func.func @main() -> i8 { - %zero = arith.constant 0 : i8 - %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index - %c14 = arith.constant 14 : index - %c96 = arith.constant 96 : index - %input = memref.get_global @input : memref<1x14x14x96xi8> - %input_heap = memref.alloc() alignment = 64 : memref<1x14x14x96xi8> - scf.for %y = %c0 to %c14 step %c1 { - scf.for %x = %c0 to %c14 step %c1 { - scf.for %c = %c0 to %c96 step %c1 { - %v = memref.load %input[%c0, %y, %x, %c] : memref<1x14x14x96xi8> - memref.store %v, %input_heap[%c0, %y, %x, %c] : memref<1x14x14x96xi8> - } - } - } - %weight1raw = memref.get_global @weight1raw : memref<49168xi8> - %weight1 = memref.reinterpret_cast %weight1raw to offset: [16], sizes: [2, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<49168xi8> to memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> - %bias1 = memref.get_global @bias1 : memref<24xi32> - %scale1 = memref.get_global @scale1 : memref<24xf32> - %weight2raw = memref.get_global @weight2raw : memref<36880xi8> - %weight2 = memref.reinterpret_cast %weight2raw to offset: [16], sizes: [6, 24, 16, 16], strides: [6144, 256, 16, 1] : memref<36880xi8> to memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> - %bias2 = memref.get_global @bias2 : memref<96xi32> - %scale2 = memref.get_global @scale2 : memref<96xf32> - %lut = memref.get_global @lut : memref<256xi8> - %weight3raw = memref.get_global @weight3raw : memref<73744xi8> - %weight3 = memref.reinterpret_cast %weight3raw to offset: [16], sizes: [3, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<73744xi8> to memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> - %bias3 = memref.get_global @bias3 : memref<40xi32> %scale3 = memref.get_global @scale3 : memref<40xf32> - %none = memref.get_global @none : memref<1xi8> - func.call @poison_reused_bank() : () -> () - %gap = memref.alloc() alignment = 64 : memref<1x1x1x96xi8> - %middle = memref.alloc() alignment = 64 : memref<1x1x1x24xi8> - %gate = memref.alloc() alignment = 64 : memref<1x1x1x96xi8> - %mul = memref.alloc() alignment = 64 : memref<1x14x14x96xi8> - %output = memref.alloc() alignment = 64 : memref<1x14x14x40xi8> - tile.mega_kernel %input_heap %output : memref<1x14x14x96xi8> memref<1x14x14x40xi8> { - tile.mega_global_avg_pool %input_heap %gap : memref<1x14x14x96xi8> memref<1x1x1x96xi8> - tile.mega_conv2d %gap %weight1 %bias1 %scale1 %none %middle : memref<1x1x1x96xi8> memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> memref<24xi32> memref<24xf32> memref<1xi8> memref<1x1x1x24xi8> - tile.mega_conv2d %middle %weight2 %bias2 %scale2 %lut %gate : memref<1x1x1x24xi8> memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> memref<96xi32> memref<96xf32> memref<256xi8> memref<1x1x1x96xi8> - tile.mega_int8_mul %gate %input_heap %mul : memref<1x1x1x96xi8> memref<1x14x14x96xi8> memref<1x14x14x96xi8> - tile.mega_conv2d %mul %weight3 %bias3 %scale3 %none %output : memref<1x14x14x96xi8> memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> memref<40xi32> memref<40xf32> memref<1xi8> memref<1x14x14x40xi8> - } - func.call @check_result(%output) : (memref<1x14x14x40xi8>) -> () - return %zero : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv_exact.mlir.in b/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv_exact.mlir.in deleted file mode 100644 index 7a2d32dbb..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv_exact.mlir.in +++ /dev/null @@ -1,24 +0,0 @@ -memref.global "private" constant @input : memref<1x14x14x96xi8> = dense<"0x@PEBBLE_GAP_CONV_INPUT_HEX@"> alignment = 64 -memref.global "private" constant @weight : memref<2x96x16x16xi8> = dense<"0x@PEBBLE_GAP_CONV_WEIGHT_HEX@"> alignment = 64 -memref.global "private" constant @bias : memref<24xi32> = dense<[-1929, -3345, -826, -4425, -1950, -1378, -2544, 3397, -134, -2621, 385, -1868, -3754, -948, 275, -53, -1955, -3197, -1065, -298, 536, -2267, -3803, -48]> alignment = 64 -memref.global "private" constant @scale : memref<24xf32> = dense<[9.64902255E-11, 1.20455174E-10, 9.11374864E-11, 6.07460609E-11, 6.96993296E-11, 2.21286572E-10, 8.11237535E-11, 1.33313652E-10, 1.26243266E-10, 1.44089102E-10, 0.0341239199, 6.56752361E-11, 6.62821395E-11, 1.91158489E-9, 1.69707901E-10, 4.13074019E-10, 5.95885979E-11, 8.92385332E-11, 6.12655274E-11, 2.72214418E-10, 2.77576795E-10, 2.37361242E-10, 6.75774228E-11, 1.48867613E-10]> alignment = 64 -memref.global "private" constant @none : memref<1xi8> = dense<0> alignment = 64 - -func.func private @check_result(memref<1x1x1x24xi8>) -> () - -func.func @main() -> i8 { - %zero = arith.constant 0 : i8 - %input = memref.get_global @input : memref<1x14x14x96xi8> - %weight = memref.get_global @weight : memref<2x96x16x16xi8> - %bias = memref.get_global @bias : memref<24xi32> - %scale = memref.get_global @scale : memref<24xf32> - %none = memref.get_global @none : memref<1xi8> - %gap = memref.alloc() alignment = 64 : memref<1x1x1x96xi8> - %output = memref.alloc() alignment = 64 : memref<1x1x1x24xi8> - tile.mega_kernel %input %output : memref<1x14x14x96xi8> memref<1x1x1x24xi8> { - tile.mega_global_avg_pool %input %gap : memref<1x14x14x96xi8> memref<1x1x1x96xi8> - tile.mega_conv2d %gap %weight %bias %scale %none %output : memref<1x1x1x96xi8> memref<2x96x16x16xi8> memref<24xi32> memref<24xf32> memref<1xi8> memref<1x1x1x24xi8> - } - func.call @check_result(%output) : (memref<1x1x1x24xi8>) -> () - return %zero : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul.mlir deleted file mode 100644 index af0225966..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul.mlir +++ /dev/null @@ -1,57 +0,0 @@ -func.func private @check_result(memref<1x16xf32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_i32 = arith.constant 0 : i32 - %one_f32 = arith.constant 1.0 : f32 - %zero_f32 = arith.constant 0.0 : f32 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %eight = arith.constant 8 : index - %dim = arith.constant 16 : index - %input = memref.alloc() alignment = 64 : memref<1x16xi8> - %weight = memref.alloc() alignment = 64 : memref<16x16xi8> - %bias = memref.alloc() alignment = 64 : memref<16xi32> - %scale = memref.alloc() alignment = 64 : memref<16xf32> - %lut = memref.alloc() alignment = 64 : memref<1xi8> - %output = memref.alloc() alignment = 64 : memref<1x16xf32> - scf.for %k = %zero to %dim step %one { - %irem = arith.remui %k, %seven : index - %ival = arith.subi %irem, %three : index - %iv = arith.index_cast %ival : index to i8 - memref.store %iv, %input[%zero, %k] : memref<1x16xi8> - %biasv = arith.subi %k, %eight : index - %bias32 = arith.index_cast %biasv : index to i32 - memref.store %bias32, %bias[%k] : memref<16xi32> - scf.for %column = %zero to %dim step %one { - %k2 = arith.muli %k, %two : index - %c3 = arith.muli %column, %three : index - %sum = arith.addi %k2, %c3 : index - %rem = arith.remui %sum, %five : index - %value = arith.subi %rem, %two : index - %wv = arith.index_cast %value : index to i8 - memref.store %wv, %weight[%k, %column] : memref<16x16xi8> - } - } - linalg.fill ins(%one_f32 : f32) outs(%scale : memref<16xf32>) - linalg.fill ins(%zero_i8 : i8) outs(%lut : memref<1xi8>) - linalg.fill ins(%zero_f32 : f32) outs(%output : memref<1x16xf32>) - tile.mega_kernel %input %output : memref<1x16xi8> memref<1x16xf32> { - tile.mega_matmul %input %weight %bias %scale %lut %output - - : memref<1x16xi8> memref<16x16xi8> memref<16xi32> - memref<16xf32> memref<1xi8> memref<1x16xf32> - } - func.call @check_result(%output) : (memref<1x16xf32>) -> () - memref.dealloc %input : memref<1x16xi8> - memref.dealloc %weight : memref<16x16xi8> - memref.dealloc %bias : memref<16xi32> - memref.dealloc %scale : memref<16xf32> - memref.dealloc %lut : memref<1xi8> - memref.dealloc %output : memref<1x16xf32> - return %zero_i8 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul_m17_k65_n17.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul_m17_k65_n17.mlir deleted file mode 100644 index 778dbd28d..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul_m17_k65_n17.mlir +++ /dev/null @@ -1,61 +0,0 @@ -func.func private @check_result(memref<17x17xf32>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %zf = arith.constant 0.0 : f32 - %half = arith.constant 0.5 : f32 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %eight = arith.constant 8 : index - %dim = arith.constant 17 : index - %inner = arith.constant 65 : index - %input = memref.alloc() alignment = 64 : memref<17x65xi8> - %weight = memref.alloc() alignment = 64 : memref<65x17xi8> - %bias = memref.alloc() alignment = 64 : memref<17xi32> - %scale = memref.alloc() alignment = 64 : memref<17xf32> - %lut = memref.alloc() alignment = 64 : memref<1xi8> - %output = memref.alloc() alignment = 64 : memref<17x17xf32> - scf.for %r = %z to %dim step %one { - scf.for %k = %z to %inner step %one { - %sum = arith.addi %r, %k : index - %rem = arith.remui %sum, %seven : index - %v = arith.subi %rem, %three : index - %i8 = arith.index_cast %v : index to i8 - memref.store %i8, %input[%r, %k] : memref<17x65xi8> - } - %b = arith.subi %r, %eight : index - %b32 = arith.index_cast %b : index to i32 - memref.store %b32, %bias[%r] : memref<17xi32> - } - scf.for %k = %z to %inner step %one { - scf.for %c = %z to %dim step %one { - %k2 = arith.muli %k, %two : index - %c3 = arith.muli %c, %three : index - %sum = arith.addi %k2, %c3 : index - %rem = arith.remui %sum, %five : index - %v = arith.subi %rem, %two : index - %i8 = arith.index_cast %v : index to i8 - memref.store %i8, %weight[%k, %c] : memref<65x17xi8> - } - } - linalg.fill ins(%half : f32) outs(%scale : memref<17xf32>) - linalg.fill ins(%z8 : i8) outs(%lut : memref<1xi8>) - linalg.fill ins(%zf : f32) outs(%output : memref<17x17xf32>) - tile.mega_kernel %input %output : memref<17x65xi8> memref<17x17xf32> { - tile.mega_matmul %input %weight %bias %scale %lut %output - - : memref<17x65xi8> memref<65x17xi8> memref<17xi32> - memref<17xf32> memref<1xi8> memref<17x17xf32> - } - func.call @check_result(%output) : (memref<17x17xf32>) -> () - memref.dealloc %input : memref<17x65xi8> - memref.dealloc %weight : memref<65x17xi8> - memref.dealloc %bias : memref<17xi32> - memref.dealloc %scale : memref<17xf32> - memref.dealloc %lut : memref<1xi8> - memref.dealloc %output : memref<17x17xf32> - return %z8 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_se_gate_conv.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/mega_se_gate_conv.mlir deleted file mode 100644 index efde11309..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_se_gate_conv.mlir +++ /dev/null @@ -1,75 +0,0 @@ -func.func private @check_result(memref<1x6x6x40xi8>) -> () - -func.func @main() -> i8 { - %i8_0 = arith.constant 0 : i8 - %i8_1 = arith.constant 1 : i8 - %i32_0 = arith.constant 0 : i32 - %i32_1 = arith.constant 1 : i32 - %f32_001 = arith.constant 0.01 : f32 - %f32_01 = arith.constant 0.1 : f32 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c5 = arith.constant 5 : index - %c6 = arith.constant 6 : index - %c96 = arith.constant 96 : index - %c256 = arith.constant 256 : index - %input = memref.alloc() alignment = 64 : memref<1x6x6x96xi8> - %gap = memref.alloc() alignment = 64 : memref<1x1x1x96xi8> - %w1raw = memref.alloc() alignment = 64 : memref<49168xi8> - %w1 = memref.reinterpret_cast %w1raw to offset: [16], sizes: [2, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<49168xi8> to memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> - %b1 = memref.alloc() alignment = 64 : memref<24xi32> - %s1 = memref.alloc() alignment = 64 : memref<24xf32> - %h1 = memref.alloc() alignment = 64 : memref<1x1x1x24xi8> - %w2raw = memref.alloc() alignment = 64 : memref<36880xi8> - %w2 = memref.reinterpret_cast %w2raw to offset: [16], sizes: [6, 24, 16, 16], strides: [6144, 256, 16, 1] : memref<36880xi8> to memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> - %b2 = memref.alloc() alignment = 64 : memref<96xi32> - %s2 = memref.alloc() alignment = 64 : memref<96xf32> - %lut = memref.alloc() alignment = 64 : memref<256xi8> - %gate = memref.alloc() alignment = 64 : memref<1x1x1x96xi8> - %mul = memref.alloc() alignment = 64 : memref<1x6x6x96xi8> - %w3raw = memref.alloc() alignment = 64 : memref<73744xi8> - %w3 = memref.reinterpret_cast %w3raw to offset: [16], sizes: [3, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<73744xi8> to memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> - %b3 = memref.alloc() alignment = 64 : memref<40xi32> - %s3 = memref.alloc() alignment = 64 : memref<40xf32> - %none = memref.alloc() alignment = 64 : memref<1xi8> - %output = memref.alloc() alignment = 64 : memref<1x6x6x40xi8> - scf.for %y = %c0 to %c6 step %c1 { - scf.for %x = %c0 to %c6 step %c1 { - scf.for %c = %c0 to %c96 step %c1 { - %a = arith.addi %y, %x : index - %d = arith.addi %a, %c : index - %r = arith.remui %d, %c5 : index - %v = arith.addi %r, %c1 : index - %q = arith.index_cast %v : index to i8 - memref.store %q, %input[%c0, %y, %x, %c] : memref<1x6x6x96xi8> - } - } - } - linalg.fill ins(%i8_1 : i8) outs(%w1 : memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>>) - linalg.fill ins(%i8_1 : i8) outs(%w2 : memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>>) - linalg.fill ins(%i8_1 : i8) outs(%w3 : memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>>) - linalg.fill ins(%i32_1 : i32) outs(%b1 : memref<24xi32>) - linalg.fill ins(%i32_0 : i32) outs(%b2 : memref<96xi32>) - linalg.fill ins(%i32_0 : i32) outs(%b3 : memref<40xi32>) - linalg.fill ins(%f32_001 : f32) outs(%s1 : memref<24xf32>) - linalg.fill ins(%f32_01 : f32) outs(%s2 : memref<96xf32>) - linalg.fill ins(%f32_001 : f32) outs(%s3 : memref<40xf32>) - linalg.fill ins(%i8_0 : i8) outs(%none : memref<1xi8>) - scf.for %i = %c0 to %c256 step %c1 { - %v = arith.index_cast %i : index to i8 - memref.store %v, %lut[%i] : memref<256xi8> - } - linalg.fill ins(%i8_0 : i8) outs(%output : memref<1x6x6x40xi8>) - scf.for %iteration = %c0 to %c2 step %c1 { - tile.mega_kernel %input %output : memref<1x6x6x96xi8> memref<1x6x6x40xi8> { - tile.mega_global_avg_pool %input %gap : memref<1x6x6x96xi8> memref<1x1x1x96xi8> - tile.mega_conv2d %gap %w1 %b1 %s1 %none %h1 : memref<1x1x1x96xi8> memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> memref<24xi32> memref<24xf32> memref<1xi8> memref<1x1x1x24xi8> - tile.mega_conv2d %h1 %w2 %b2 %s2 %lut %gate : memref<1x1x1x24xi8> memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> memref<96xi32> memref<96xf32> memref<256xi8> memref<1x1x1x96xi8> - tile.mega_int8_mul %gate %input %mul : memref<1x1x1x96xi8> memref<1x6x6x96xi8> memref<1x6x6x96xi8> - tile.mega_conv2d %mul %w3 %b3 %s3 %none %output : memref<1x6x6x96xi8> memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> memref<40xi32> memref<40xf32> memref<1xi8> memref<1x6x6x40xi8> - } - } - func.call @check_result(%output) : (memref<1x6x6x40xi8>) -> () - return %i8_0 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/tile_matmul_16x16_i8.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/tile_matmul_16x16_i8.mlir deleted file mode 100644 index 484ee1bdd..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/tile_matmul_16x16_i8.mlir +++ /dev/null @@ -1,41 +0,0 @@ -func.func private @check_result(memref<16x16xi32>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero_i32 = arith.constant 0 : i32 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c2 = arith.constant 2 : index - %c3 = arith.constant 3 : index - %c5 = arith.constant 5 : index - %c7 = arith.constant 7 : index - %c16 = arith.constant 16 : index - %a = memref.alloc() alignment = 64 : memref<16x16xi8> - %b = memref.alloc() alignment = 64 : memref<16x16xi8> - %c = memref.alloc() alignment = 64 : memref<16x16xi32> - scf.for %i = %c0 to %c16 step %c1 { - scf.for %j = %c0 to %c16 step %c1 { - %i3 = arith.muli %i, %c3 : index - %j5 = arith.muli %j, %c5 : index - %as = arith.addi %i3, %j5 : index - %ar = arith.remui %as, %c7 : index - %av0 = arith.subi %ar, %c3 : index - %av = arith.index_cast %av0 : index to i8 - %i2 = arith.muli %i, %c2 : index - %j3 = arith.muli %j, %c3 : index - %bs = arith.addi %i2, %j3 : index - %br = arith.remui %bs, %c5 : index - %bv0 = arith.subi %br, %c2 : index - %bv = arith.index_cast %bv0 : index to i8 - memref.store %av, %a[%i, %j] : memref<16x16xi8> - memref.store %bv, %b[%i, %j] : memref<16x16xi8> - } - } - linalg.fill ins(%zero_i32 : i32) outs(%c : memref<16x16xi32>) - tile.tile_matmul %a %b %c : memref<16x16xi8> memref<16x16xi8> memref<16x16xi32> - func.call @check_result(%c) : (memref<16x16xi32>) -> () - memref.dealloc %a : memref<16x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %c : memref<16x16xi32> - return %zero_i8 : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/tile_matmul_16x16_i8_main.cpp b/examples/chips/pebble/workloads/mlir_tests/tile/tile_matmul_16x16_i8_main.cpp deleted file mode 100644 index 04b00f2a5..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/tile_matmul_16x16_i8_main.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include -#include -#include - -static void fail(void) { -#ifdef BAREMETAL - *(volatile uint32_t *)0x60000000 = 1; - while (1) { - } -#else - exit(1); -#endif -} - -extern "C" void check_result(int32_t *, int32_t *out, int64_t offset, - int64_t rows, int64_t columns, int64_t row_stride, - int64_t column_stride) { - if (rows != 16 || columns != 16 || row_stride != 16 || column_stride != 1) - fail(); - for (int i = 0; i < 16; ++i) - for (int j = 0; j < 16; ++j) { - int expected = 0; - for (int k = 0; k < 16; ++k) - expected += ((3 * i + 5 * k) % 7 - 3) * ((2 * k + 3 * j) % 5 - 2); - if (out[offset + i * row_stride + j] != expected) - fail(); - } - printf("PASSED: tile.tile_matmul 16x16 ordered operands\n"); -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/tile_transpose_16x16_i8.mlir b/examples/chips/pebble/workloads/mlir_tests/tile/tile_transpose_16x16_i8.mlir deleted file mode 100644 index 45febad14..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/tile_transpose_16x16_i8.mlir +++ /dev/null @@ -1,27 +0,0 @@ -func.func private @check_result(memref<16x16xi8>) -> () - -func.func @main() -> i8 { - %zero = arith.constant 0 : i8 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c3 = arith.constant 3 : i8 - %c5 = arith.constant 5 : i8 - %c16 = arith.constant 16 : index - %input = memref.alloc() alignment = 64 : memref<16x16xi8> - %output = memref.alloc() alignment = 64 : memref<16x16xi8> - scf.for %i = %c0 to %c16 step %c1 { - scf.for %j = %c0 to %c16 step %c1 { - %ii = arith.index_cast %i : index to i8 - %jj = arith.index_cast %j : index to i8 - %iv = arith.muli %ii, %c3 : i8 - %jv = arith.muli %jj, %c5 : i8 - %v = arith.addi %iv, %jv : i8 - memref.store %v, %input[%i, %j] : memref<16x16xi8> - } - } - tile.tile_transpose %input %output : memref<16x16xi8> memref<16x16xi8> - func.call @check_result(%output) : (memref<16x16xi8>) -> () - memref.dealloc %input : memref<16x16xi8> - memref.dealloc %output : memref<16x16xi8> - return %zero : i8 -} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/tile_transpose_16x16_i8_main.cpp b/examples/chips/pebble/workloads/mlir_tests/tile/tile_transpose_16x16_i8_main.cpp deleted file mode 100644 index c7d0cfa45..000000000 --- a/examples/chips/pebble/workloads/mlir_tests/tile/tile_transpose_16x16_i8_main.cpp +++ /dev/null @@ -1,25 +0,0 @@ -#include -#include -#include - -static void fail(void) { -#ifdef BAREMETAL - *(volatile uint32_t *)0x60000000 = 1; - while (1) { - } -#else - exit(1); -#endif -} - -extern "C" void check_result(int8_t *, int8_t *out, int64_t offset, - int64_t rows, int64_t columns, int64_t row_stride, - int64_t column_stride) { - if (rows != 16 || columns != 16 || row_stride != 16 || column_stride != 1) - fail(); - for (int i = 0; i < 16; ++i) - for (int j = 0; j < 16; ++j) - if (out[offset + i * row_stride + j] != (int8_t)(3 * j + 5 * i)) - fail(); - printf("PASSED: tile.tile_transpose 16x16 ordered input\n"); -} diff --git a/examples/chips/poly/arch/src/main/scala/compute_tile/ComputeTile.scala b/examples/chips/poly/arch/src/main/scala/compute_tile/ComputeTile.scala deleted file mode 100644 index eb6de3909..000000000 --- a/examples/chips/poly/arch/src/main/scala/compute_tile/ComputeTile.scala +++ /dev/null @@ -1,21 +0,0 @@ -package examples.chips.poly.compute_tile - -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.top.GlobalConfig -import framework.system.GlobalConfigOps._ -import framework.system.core.CoreFactory -import framework.system.tile.{ComputeTilePlatform, TileConnection, TileEndpoint, TileFactory} - -@instantiable -class ComputeTile(b: GlobalConfig) extends TileEndpoint(b.tileLinkParams, false) { - val p = b.tileParams - val platform = Instantiate(new ComputeTilePlatform(p)) - val controllerIndex = p.controller.get - val controller = CoreFactory.rocket(b.forCore(controllerIndex)) - val workers = p.cores.indices.filterNot(_ == controllerIndex).map(i => CoreFactory.ant(b.forCore(i))) - TileConnection.compute(platform, tlink, controller.clink, workers.map(_.clink)) -} - -object ComputeTile extends TileFactory { - def instantiate(b: GlobalConfig): Instance[TileEndpoint] = Instantiate(new ComputeTile(b)) -} diff --git a/examples/chips/poly/arch/src/main/scala/sims/p2e/TargetConfigs.scala b/examples/chips/poly/arch/src/main/scala/sims/p2e/TargetConfigs.scala deleted file mode 100644 index 283159298..000000000 --- a/examples/chips/poly/arch/src/main/scala/sims/p2e/TargetConfigs.scala +++ /dev/null @@ -1,10 +0,0 @@ -package sims.p2e - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -/** Poly on the P2E board: a CPU main tile and eight homogeneous compute tiles. */ -class PolyP2ETarget extends P2ETarget("../examples/chips/poly/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) -} diff --git a/examples/chips/poly/arch/src/main/scala/sims/verilator/TargetConfigs.scala b/examples/chips/poly/arch/src/main/scala/sims/verilator/TargetConfigs.scala deleted file mode 100644 index fadcf25bd..000000000 --- a/examples/chips/poly/arch/src/main/scala/sims/verilator/TargetConfigs.scala +++ /dev/null @@ -1,13 +0,0 @@ -package sims.verilator - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -import sims.soc.SystemTarget - -/** Poly on the explicit System: a CPU main tile and two homogeneous compute tiles. */ -class PolyVerilatorTarget extends SystemTarget("../examples/chips/poly/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) - override val dramBytes: BigInt = BigInt(16) << 30 -} diff --git a/examples/chips/poly/configs/chip.toml b/examples/chips/poly/configs/chip.toml deleted file mode 100644 index 9fa5b292a..000000000 --- a/examples/chips/poly/configs/chip.toml +++ /dev/null @@ -1,10 +0,0 @@ -[designs] -include = "designs/poly.toml" - -[sims] -verilator = "sims.verilator.PolyVerilatorTarget" -p2e = "sims.p2e.PolyP2ETarget" - -[uvm] -balls = ["smatmul", "toint8", "int2fp", "transpose"] -ips = [] diff --git a/examples/chips/poly/configs/designs/poly.toml b/examples/chips/poly/configs/designs/poly.toml deleted file mode 100644 index 8422b53f2..000000000 --- a/examples/chips/poly/configs/designs/poly.toml +++ /dev/null @@ -1,9 +0,0 @@ -# Poly: a CPU main tile and 8 homogeneous compute tiles, each -# 1 controller + 2 Attention + 3 FFN cores. - -[main_tile] -include = "tiles/main.toml" - -[[tiles]] -include = "tiles/compute.toml" -count = 8 diff --git a/examples/chips/poly/configs/designs/tiles/compute.toml b/examples/chips/poly/configs/designs/tiles/compute.toml deleted file mode 100644 index fd02dead1..000000000 --- a/examples/chips/poly/configs/designs/tiles/compute.toml +++ /dev/null @@ -1,58 +0,0 @@ -controllerCore = 0 -factory = "examples.chips.poly.compute_tile.ComputeTile" -# Poly compute tile: controller core 0, two Attention cores and three FFN cores. - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -memBallChannelNum = 6 - -[tss] -base = 0x20000 -bytes = 16384 -dataBits = 128 - -[sharedMem] -bankWidth = 128 -bankEntries = 1024 -enable = true -virtualBankCount = 40 -entries = 16384 -inputChannels = 5 -defaultGroupCount = 1 -memCycles = 10 - -[[cores]] -core_id = 0 -name = "controller" -include = "../../../../../cores/rocket/configs/default.toml" - -[[cores]] -core_id = 1 -name = "attention" -include = "../../../../../cores/attention/configs/default.toml" - -[[cores]] -core_id = 2 -name = "attention" -include = "../../../../../cores/attention/configs/default.toml" - -[[cores]] -core_id = 3 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 4 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" - -[[cores]] -core_id = 5 -name = "ffn" -include = "../../../../../cores/ffn/configs/default.toml" diff --git a/examples/chips/poly/configs/designs/tiles/main.toml b/examples/chips/poly/configs/designs/tiles/main.toml deleted file mode 100644 index b2fd30e3a..000000000 --- a/examples/chips/poly/configs/designs/tiles/main.toml +++ /dev/null @@ -1,24 +0,0 @@ -# Poly main tile: one Linux core and shared storage for cross-tile reduction. - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 -memBallChannelNum = 3 - -[sharedMem] -bankWidth = 128 -bankEntries = 1024 -enable = true -entries = 16384 -inputChannels = 3 -defaultGroupCount = 1 -virtualBankCount = 40 - -[[cores]] -core_id = 0 -name = "main" -include = "../../../../../cores/reduce/configs/default.toml" diff --git a/examples/chips/poly/kernel/workloads.toml b/examples/chips/poly/kernel/workloads.toml deleted file mode 100644 index 04d6b9c79..000000000 --- a/examples/chips/poly/kernel/workloads.toml +++ /dev/null @@ -1,10 +0,0 @@ -[workloads] -tests = [ - "poly-main-ctest-chip-controllers-linux", - "poly-main-ctest-f32add_groups-linux", - "poly-main-ctest-chip-ant_mxmm-linux", - "poly-main-ctest-chip-ant_shared-linux", - "poly-main-ctest-chip-ant_bank-linux", - "poly-main-ctest-chip-ant_parallel-linux", - "poly-main-ctest-chip-ant_stream-linux", -] diff --git a/examples/chips/poly/regression/batch/bemu/workloads-bare.toml b/examples/chips/poly/regression/batch/bemu/workloads-bare.toml deleted file mode 100644 index 70c1c5bce..000000000 --- a/examples/chips/poly/regression/batch/bemu/workloads-bare.toml +++ /dev/null @@ -1,11 +0,0 @@ -[workloads] -tests = [ - "poly-main-ctest-f32add_groups-baremetal", - "poly-main-ctest-tlink_lease-baremetal", - "poly-main-ctest-chip-controllers-multicore-baremetal", - "poly-main-ctest-chip-ant_mxmm-multicore-baremetal", - "poly-main-ctest-chip-ant_shared-multicore-baremetal", - "poly-main-ctest-chip-ant_bank-multicore-baremetal", - "poly-main-ctest-chip-ant_bank_rows-multicore-baremetal", - "poly-main-ctest-chip-ant_parallel-multicore-baremetal", -] diff --git a/examples/chips/poly/regression/batch/verilator/workloads-bare.toml b/examples/chips/poly/regression/batch/verilator/workloads-bare.toml deleted file mode 100644 index 38f366460..000000000 --- a/examples/chips/poly/regression/batch/verilator/workloads-bare.toml +++ /dev/null @@ -1,11 +0,0 @@ -[workloads] -tests = [ - "poly-main-ctest-f32add_groups-baremetal", - "poly-main-ctest-tlink_lease-baremetal", - "poly-main-ctest-chip-controllers-multicore-baremetal", - "poly-main-ctest-chip-ant_mesh_move-multicore-baremetal", - "poly-main-ctest-chip-ant_mxmm-multicore-baremetal", - "poly-main-ctest-chip-ant_shared-multicore-baremetal", - "poly-main-ctest-chip-ant_bank_rows-multicore-baremetal", - "poly-main-ctest-chip-ant_parallel-multicore-baremetal", -] diff --git a/examples/chips/poly/workloads/CMakeLists.txt b/examples/chips/poly/workloads/CMakeLists.txt deleted file mode 100644 index 8abc07554..000000000 --- a/examples/chips/poly/workloads/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_custom_target(poly-ctests-build) -set(BUCKYBALL_CTEST_GROUP_TARGET poly-ctests-build) -set(BUCKYBALL_CTEST_TARGET attention) -add_subdirectory(ctests) - - -add_custom_target(chip-ctests-build DEPENDS poly-ctests-build) - -add_custom_target(chip-workloads-build ALL - DEPENDS - chip-ctests-build - mlirtest-workloads-build - soctests-build - COMMENT "Building poly workloads" - VERBATIM) diff --git a/examples/chips/poly/workloads/ctests/CMakeLists.txt b/examples/chips/poly/workloads/ctests/CMakeLists.txt deleted file mode 100644 index c1eed72e1..000000000 --- a/examples/chips/poly/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,151 +0,0 @@ -# Cross-tile Linux control cores share one coherent LLC. -buckyball_ctest_name(POLY_CONTROL_NAME main chip-controllers) -set(POLY_CONTROL_ELF ${POLY_CONTROL_NAME}-multicore-baremetal) -buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/controllers.c") -add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${POLY_CONTROL_ELF} - COMMAND ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 - -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${BBSIM_LD} - -I${BUCKYBALL_ISA_DIR}/chip -I${WORKLOAD_LIB_DIR}/bbsw/baremetal - -o ${POLY_CONTROL_ELF} - ${CMAKE_CURRENT_SOURCE_DIR}/controllers_start.S - ${CMAKE_CURRENT_SOURCE_DIR}/controllers.c -lgcc - DEPENDS controllers_start.S controllers.c ${WORKLOAD_LIB_DIR}/bbsw/baremetal/start.S - ${BBSIM_LD} - ${BUCKYBALL_ISA_DIR}/chip/topology.h - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) -add_custom_target(${POLY_CONTROL_NAME}-ctest-build - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${POLY_CONTROL_ELF}) -add_dependencies(poly-ctests-build ${POLY_CONTROL_NAME}-ctest-build) - -set(BUCKYBALL_CTEST_TARGET main) -add_buckyball_linux_ctest(${POLY_CONTROL_NAME} ${CMAKE_CURRENT_SOURCE_DIR} controllers.c) -target_include_directories(${POLY_CONTROL_NAME}-linux PRIVATE ${BUCKYBALL_ISA_DIR}/chip) -target_link_libraries(${POLY_CONTROL_NAME}-linux PRIVATE pthread) -add_dependencies(poly-ctests-build ${POLY_CONTROL_NAME}-linux-build) - -foreach(POLY_ANT_CASE mxmm shared bank bank_rows parallel mesh_move highpa tlink tlink_bank tlink_reduce coherence) -set(POLY_ANT_CORE attention) -if(POLY_ANT_CASE STREQUAL "bank" OR POLY_ANT_CASE STREQUAL "bank_rows") - set(POLY_ANT_CORE ffn) -endif() -buckyball_ctest_name(POLY_ANT_NAME ${POLY_ANT_CORE} chip-ant_${POLY_ANT_CASE}) -foreach(_name BANK_WIDTH BANK_LINES BB_PRIVATE_VBANK_MAX) - file(STRINGS "${BUCKYBALL_ISA_DIR}/${POLY_ANT_CORE}/params.h" _line REGEX "^#define ${_name} ") - string(REGEX REPLACE ".* " "" _${_name} "${_line}") -endforeach() -if(NOT _BANK_WIDTH EQUAL 128 OR _BANK_LINES LESS 128 OR _BB_PRIVATE_VBANK_MAX LESS 5) - message(FATAL_ERROR - "Ant Mxmm needs 128-bit rows, >=128 rows/bank and private banks through 5; actual width=${_BANK_WIDTH}, rows=${_BANK_LINES}, privateMax=${_BB_PRIVATE_VBANK_MAX}") -endif() -set(POLY_ANT_IMAGE ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_NAME}-ant) -set(ANT_RUNTIME_DIR ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/ant) -buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/ant_${POLY_ANT_CASE}.c") -add_custom_command( - OUTPUT ${POLY_ANT_IMAGE}.elf ${POLY_ANT_IMAGE}.bin - COMMAND ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany - -ffreestanding -fno-builtin -fno-pic -fno-stack-protector - -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax - -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${ANT_RUNTIME_DIR}/code.ld - -I${WORKLOAD_LIB_DIR} -I${BUCKYBALL_ISA_DIR}/${POLY_ANT_CORE} - -I${BUCKYBALL_REPO_ROOT}/examples/balls/mxmm/workloads - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include - ${ANT_RUNTIME_DIR}/start.S ${CMAKE_CURRENT_SOURCE_DIR}/ant_${POLY_ANT_CASE}.c - -o ${POLY_ANT_IMAGE}.elf - COMMAND ${RISCV_GNU_TOOLCHAIN}/bin/riscv64-unknown-elf-objcopy - -O binary --only-section=.text ${POLY_ANT_IMAGE}.elf ${POLY_ANT_IMAGE}.bin - DEPENDS ant_${POLY_ANT_CASE}.c ant_${POLY_ANT_CASE}.h ${ANT_RUNTIME_DIR}/start.S ${ANT_RUNTIME_DIR}/code.ld - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_CORE}/params.h ${BUCKYBALL_ISA_DIR}/${POLY_ANT_CORE}/ballISA.h - ${BUCKYBALL_BALL_ISA_DEPS}) -add_custom_target(${POLY_ANT_NAME}-ant-build - DEPENDS ${POLY_ANT_IMAGE}.elf ${POLY_ANT_IMAGE}.bin) -add_dependencies(poly-ctests-build ${POLY_ANT_NAME}-ant-build) - -set(POLY_ANT_CONTROL_SOURCES) -if(POLY_ANT_CASE STREQUAL "bank" OR POLY_ANT_CASE STREQUAL "bank_rows" OR POLY_ANT_CASE STREQUAL "parallel") - list(APPEND POLY_ANT_CONTROL_SOURCES - ${CMAKE_CURRENT_SOURCE_DIR}/ant_${POLY_ANT_CASE}_exercise.c) -elseif(POLY_ANT_CASE STREQUAL "tlink_reduce") - list(APPEND POLY_ANT_CONTROL_SOURCES - ${CMAKE_CURRENT_SOURCE_DIR}/ant_tlink_reduce_control.c) -endif() -foreach(SOURCE IN LISTS POLY_ANT_CONTROL_SOURCES) - buckyball_enforce_ctest_line_limit("${SOURCE}") -endforeach() - -set(POLY_ANT_EMBED ${CMAKE_CURRENT_BINARY_DIR}/ant_${POLY_ANT_CASE}_image.S) -enable_language(ASM) -file(GENERATE OUTPUT ${POLY_ANT_EMBED} CONTENT -".section .rodata.ant,\"a\",@progbits\n.balign 8\n.global ant_image, ant_image_end\nant_image:\n.incbin \"${POLY_ANT_IMAGE}.bin\"\nant_image_end:\n.section .note.GNU-stack,\"\",@progbits\n") -set_source_files_properties(${POLY_ANT_EMBED} PROPERTIES - OBJECT_DEPENDS "${POLY_ANT_IMAGE}.bin") -set(POLY_ANT_MAIN ant_${POLY_ANT_CASE}_main.c) -if(POLY_ANT_CASE STREQUAL "mxmm") - set(POLY_ANT_MAIN ant_mxmm_main.cpp) -endif() -buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_MAIN}") -buckyball_ctest_name(POLY_ANT_HOST_NAME main chip-ant_${POLY_ANT_CASE}) -set(BUCKYBALL_CTEST_TARGET ${POLY_ANT_CORE}) -if(NOT POLY_ANT_CASE STREQUAL "highpa" AND NOT POLY_ANT_CASE MATCHES "^tlink" AND - NOT POLY_ANT_CASE STREQUAL "coherence" AND NOT POLY_ANT_CASE STREQUAL "mesh_move") -add_buckyball_linux_ctest(${POLY_ANT_HOST_NAME} ${CMAKE_CURRENT_SOURCE_DIR} ${POLY_ANT_MAIN}) -target_sources(${POLY_ANT_HOST_NAME}-linux PRIVATE ${POLY_ANT_EMBED} ${POLY_ANT_CONTROL_SOURCES}) -target_include_directories(${POLY_ANT_HOST_NAME}-linux PRIVATE ${BUCKYBALL_ISA_DIR}/chip) -target_link_libraries(${POLY_ANT_HOST_NAME}-linux PRIVATE runtime_ant pthread) -add_dependencies(${POLY_ANT_HOST_NAME}-linux ${POLY_ANT_NAME}-ant-build) -add_dependencies(poly-ctests-build ${POLY_ANT_HOST_NAME}-linux-build) - -endif() - -set(POLY_ANT_HOST_CORE ${POLY_ANT_CORE}) -set(POLY_ANT_CONTROL_OBJECTS) -if(POLY_ANT_CASE STREQUAL "tlink_reduce") - set(POLY_ANT_HOST_CORE main) - set(POLY_ANT_CONTROL_OBJECTS ${CMAKE_CURRENT_BINARY_DIR}/ant_tlink_reduce_control.o) - add_custom_command(OUTPUT ${POLY_ANT_CONTROL_OBJECTS} - COMMAND ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 -fno-builtin - -I${BUCKYBALL_ISA_DIR}/${POLY_ANT_CORE} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include - -c ${POLY_ANT_CONTROL_SOURCES} -o ${POLY_ANT_CONTROL_OBJECTS} - DEPENDS ${POLY_ANT_CONTROL_SOURCES} ant_tlink_reduce.h - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_CORE}/params.h) - set(POLY_ANT_CONTROL_SOURCES) -endif() - -set(POLY_ANT_HOST_ELF ${POLY_ANT_HOST_NAME}-multicore-baremetal) -add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_HOST_ELF} - COMMAND ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 - -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${BBSIM_LD} - -I${BUCKYBALL_ISA_DIR}/chip -I${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_CORE} - -I${WORKLOAD_LIB_DIR} - -I${BUCKYBALL_REPO_ROOT}/examples/balls/f32add/workloads - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR}/bbsw/baremetal - ${CMAKE_CURRENT_SOURCE_DIR}/controllers_start.S - ${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_MAIN} ${POLY_ANT_CONTROL_SOURCES} ${POLY_ANT_CONTROL_OBJECTS} ${POLY_ANT_EMBED} - ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/ant.c - ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/tlink.c -lgcc -o ${POLY_ANT_HOST_ELF} - DEPENDS ${POLY_ANT_MAIN} ant_${POLY_ANT_CASE}.h controllers_start.S ${POLY_ANT_EMBED} - ${BBSW_BAREMETAL_DIR}/start.S ${BBSIM_LD} - ${POLY_ANT_CONTROL_SOURCES} ${POLY_ANT_CONTROL_OBJECTS} - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_CORE}/params.h - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_CORE}/ballISA.h - ${POLY_ANT_IMAGE}.bin ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/ant.c - ${BUCKYBALL_REPO_ROOT}/stack/runtime/include/ant.h - ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/tlink.c ${BUCKYBALL_REPO_ROOT}/stack/runtime/include/tlink.h - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_CORE}/params.h ${BUCKYBALL_ISA_DIR}/chip/topology.h - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) -add_custom_target(${POLY_ANT_HOST_NAME}-ctest-build - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_HOST_ELF}) -add_dependencies(poly-ctests-build ${POLY_ANT_HOST_NAME}-ctest-build) -endforeach() - -buckyball_ctest_name(POLY_STREAM_NAME main chip-ant_stream) -buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/ant_stream_main.c") -add_executable(${POLY_STREAM_NAME}-linux ant_stream_main.c) -target_include_directories(${POLY_STREAM_NAME}-linux PRIVATE - ${BUCKYBALL_ISA_DIR}/chip ${BUCKYBALL_ISA_DIR}/attention ${WORKLOAD_LIB_DIR} - ${BUCKYBALL_REPO_ROOT}/examples/balls/mxmm/workloads) -target_link_libraries(${POLY_STREAM_NAME}-linux PRIVATE runtime_ant_tasks) -target_include_directories(runtime_ant_tasks PRIVATE ${BUCKYBALL_ISA_DIR}/attention) -add_dependencies(poly-ctests-build ${POLY_STREAM_NAME}-linux) diff --git a/examples/chips/poly/workloads/ctests/ant_bank_exercise.c b/examples/chips/poly/workloads/ctests/ant_bank_exercise.c deleted file mode 100644 index 1c3a098a7..000000000 --- a/examples/chips/poly/workloads/ctests/ant_bank_exercise.c +++ /dev/null @@ -1,60 +0,0 @@ -#include "ant_bank.h" -#include -#include -#include -extern const unsigned char ant_image[], ant_image_end[]; -enum { - WORDS = BANK_LINES * BANK_WIDTH / 64, - SHARED_WORDS = BB_SHARED_BANK_LINES * BANK_WIDTH / 64 -}; -static uint64_t input[BB_COMPUTE_TILES][WORDS] - __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t private_out[BB_COMPUTE_TILES][WORDS] - __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t shared_out[BB_COMPUTE_TILES][SHARED_WORDS] - __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t tail_out[BB_COMPUTE_TILES][8] - __attribute__((section(".data"), aligned(4096))) = {0}; -unsigned bank_done[BB_COMPUTE_TILES]; -void bank_exercise(unsigned tile) { - const unsigned context = BB_ANT_CONTEXT; - for (unsigned i = 0; i < WORDS; ++i) - input[tile][i] = (uint64_t)(tile + 1) << 32 | (i + 1); - for (unsigned i = 0; i < WORDS; ++i) - private_out[tile][i] = 0; - for (unsigned i = 0; i < SHARED_WORDS; ++i) - shared_out[tile][i] = 0; - for (unsigned i = 0; i < 8; ++i) - tail_out[tile][i] = ~0ULL; - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(context, ANT_CODE_BYTES) || - ant_query(context, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - struct ant_bank_args args = {(uintptr_t)input[tile], - (uintptr_t)private_out[tile], - (uintptr_t)shared_out[tile], - (uintptr_t)tail_out[tile], BB_SHARED_BANK_LINES}; - ant_write(context, ANT_CODE, 0, ant_image, bytes); - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - uint64_t base = ant_query(context, ANT_TLS_BASE); - struct ant_task task = { - tile + 1, 0, bytes, base, base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_acquire(); - ant_start(context, &task); - int cancelled; - if (ant_wait(context, task.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); - for (unsigned i = 0; i < WORDS; ++i) - if (private_out[tile][i] != (((uint64_t)(tile + 1) << 32) | (i + 1))) - __builtin_trap(); - for (unsigned i = 0; i < SHARED_WORDS; ++i) - if (shared_out[tile][i] != (((uint64_t)(tile + 1) << 32) | (i + 1))) - __builtin_trap(); - for (unsigned i = 0; i < 8; ++i) - if (tail_out[tile][i] != - (i % 2 ? 0 : (((uint64_t)(tile + 1) << 32) | (i + 1)))) - __builtin_trap(); - __atomic_store_n(&bank_done[tile], 1, __ATOMIC_RELEASE); -} diff --git a/examples/chips/poly/workloads/ctests/ant_bank_main.c b/examples/chips/poly/workloads/ctests/ant_bank_main.c deleted file mode 100644 index 87507e4f9..000000000 --- a/examples/chips/poly/workloads/ctests/ant_bank_main.c +++ /dev/null @@ -1,49 +0,0 @@ -#define _GNU_SOURCE -#include "ant_bank.h" -#include -#include -#include -#ifdef __linux__ -#include -#include -#include -#include -#include -#endif -#ifdef __linux__ -static void *worker(void *argument) { - uintptr_t tile = (uintptr_t)argument; - cpu_set_t mask; - CPU_ZERO(&mask); - CPU_SET(BB_MAIN_CORES + tile, &mask); - if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) - abort(); - bank_exercise(tile); - return NULL; -} -#endif -int main(void) { -#ifdef __linux__ - if (mlockall(MCL_CURRENT | MCL_FUTURE)) - abort(); - pthread_t threads[BB_COMPUTE_TILES]; - for (uintptr_t tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_create(&threads[tile], NULL, worker, (void *)tile)) - abort(); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_join(threads[tile], NULL)) - abort(); - printf("Ant bank boundary PASS tiles=%u private=%u shared=%u\n", - (unsigned)BB_COMPUTE_TILES, BANK_LINES, BB_SHARED_BANK_LINES); -#else - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart >= BB_MAIN_CORES) - bank_exercise(hart - BB_MAIN_CORES); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&bank_done[tile], __ATOMIC_ACQUIRE)) { - } -#endif - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_bank_rows_exercise.c b/examples/chips/poly/workloads/ctests/ant_bank_rows_exercise.c deleted file mode 100644 index bd6eaa0ee..000000000 --- a/examples/chips/poly/workloads/ctests/ant_bank_rows_exercise.c +++ /dev/null @@ -1,59 +0,0 @@ -#include "ant_bank_rows.h" -#include -#include -#include -extern const unsigned char ant_image[], ant_image_end[]; -enum { - WORDS = ANT_BANK_ROWS * BANK_WIDTH / 64, - SHARED_WORDS = ANT_BANK_ROWS * BANK_WIDTH / 64 -}; -static uint64_t input[BB_COMPUTE_TILES][WORDS] - __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t private_out[BB_COMPUTE_TILES][WORDS] - __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t shared_out[BB_COMPUTE_TILES][SHARED_WORDS] - __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t tail_out[BB_COMPUTE_TILES][8] - __attribute__((section(".data"), aligned(4096))) = {0}; -unsigned bank_rows_done[BB_COMPUTE_TILES]; -void bank_rows_exercise(unsigned tile) { - const unsigned context = BB_ANT_CONTEXT; - for (unsigned i = 0; i < WORDS; ++i) - input[tile][i] = (uint64_t)(tile + 1) << 32 | (i + 1); - for (unsigned i = 0; i < WORDS; ++i) - private_out[tile][i] = 0; - for (unsigned i = 0; i < SHARED_WORDS; ++i) - shared_out[tile][i] = 0; - for (unsigned i = 0; i < 8; ++i) - tail_out[tile][i] = ~0ULL; - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(context, ANT_CODE_BYTES) || - ant_query(context, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - struct ant_bank_rows_args args = { - (uintptr_t)input[tile], (uintptr_t)private_out[tile], - (uintptr_t)shared_out[tile], (uintptr_t)tail_out[tile], ANT_BANK_ROWS}; - ant_write(context, ANT_CODE, 0, ant_image, bytes); - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - uint64_t base = ant_query(context, ANT_TLS_BASE); - struct ant_task task = { - tile + 1, 0, bytes, base, base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_acquire(); - ant_start(context, &task); - int cancelled; - if (ant_wait(context, task.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); - for (unsigned i = 0; i < WORDS; ++i) - if (private_out[tile][i] != (((uint64_t)(tile + 1) << 32) | (i + 1))) - __builtin_trap(); - for (unsigned i = 0; i < SHARED_WORDS; ++i) - if (shared_out[tile][i] != (((uint64_t)(tile + 1) << 32) | (i + 1))) - __builtin_trap(); - for (unsigned i = 0; i < 8; ++i) - if (tail_out[tile][i] != - (i % 2 ? 0 : (((uint64_t)(tile + 1) << 32) | (i + 1)))) - __builtin_trap(); - __atomic_store_n(&bank_rows_done[tile], 1, __ATOMIC_RELEASE); -} diff --git a/examples/chips/poly/workloads/ctests/ant_bank_rows_main.c b/examples/chips/poly/workloads/ctests/ant_bank_rows_main.c deleted file mode 100644 index 800d52376..000000000 --- a/examples/chips/poly/workloads/ctests/ant_bank_rows_main.c +++ /dev/null @@ -1,49 +0,0 @@ -#define _GNU_SOURCE -#include "ant_bank_rows.h" -#include -#include -#include -#ifdef __linux__ -#include -#include -#include -#include -#include -#endif -#ifdef __linux__ -static void *worker(void *argument) { - uintptr_t tile = (uintptr_t)argument; - cpu_set_t mask; - CPU_ZERO(&mask); - CPU_SET(BB_MAIN_CORES + tile, &mask); - if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) - abort(); - bank_rows_exercise(tile); - return NULL; -} -#endif -int main(void) { -#ifdef __linux__ - if (mlockall(MCL_CURRENT | MCL_FUTURE)) - abort(); - pthread_t threads[BB_COMPUTE_TILES]; - for (uintptr_t tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_create(&threads[tile], NULL, worker, (void *)tile)) - abort(); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_join(threads[tile], NULL)) - abort(); - printf("Ant bank boundary PASS tiles=%u private=%u shared=%u\n", - (unsigned)BB_COMPUTE_TILES, BANK_LINES, BB_SHARED_BANK_LINES); -#else - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart >= BB_MAIN_CORES) - bank_rows_exercise(hart - BB_MAIN_CORES); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&bank_rows_done[tile], __ATOMIC_ACQUIRE)) { - } -#endif - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_coherence_main.c b/examples/chips/poly/workloads/ctests/ant_coherence_main.c deleted file mode 100644 index 10bb5933a..000000000 --- a/examples/chips/poly/workloads/ctests/ant_coherence_main.c +++ /dev/null @@ -1,89 +0,0 @@ -#include "ant_coherence.h" -#include -#include -#include -_Static_assert(BB_MAIN_CORES == 1 && BB_COMPUTE_TILES >= 2 && - !(BB_COMPUTE_TILES & (BB_COMPUTE_TILES - 1)), - "coherence gate requires a power-of-two compute tile count"); -extern const unsigned char ant_image[], ant_image_end[]; -// Separate 64-byte lines/sets for data and synchronization; avoid main's -// top-of-page stack sets. -enum { - INPUT = 8, - OUTPUT = INPUT + BB_COMPUTE_TILES, - READY = OUTPUT + BB_COMPUTE_TILES, - DONE = READY + BB_COMPUTE_TILES, - ROUND = DONE + BB_COMPUTE_TILES -}; -static volatile uint64_t lines[ROUND + 1][8] - __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t pattern(unsigned round, unsigned tile, unsigned word) { - return ((uint64_t)round << 48) | ((uint64_t)(tile + 1) << 32) | (word + 17); -} -static void task(unsigned tile, unsigned phase, unsigned id) { - const unsigned context = BB_ANT_CONTEXT; - struct ant_coherence_args args = {phase, (uintptr_t)lines[INPUT + tile], - (uintptr_t)lines[OUTPUT + tile]}; - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - uint64_t base = ant_query(context, ANT_TLS_BASE); - struct ant_task command = {id, - 0, - ant_image_end - ant_image, - base, - base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_acquire(); - ant_start(context, &command); - int cancelled; - if (ant_wait(context, id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); -} -static void controller(unsigned tile) { - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(BB_ANT_CONTEXT, ANT_CODE_BYTES) || - ant_query(BB_ANT_CONTEXT, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - ant_write(BB_ANT_CONTEXT, ANT_CODE, 0, ant_image, bytes); - task(tile, 0, 1); - __atomic_store_n(&lines[READY + tile][0], 1, __ATOMIC_RELEASE); - for (unsigned round = 1; round <= 2; ++round) { - while (__atomic_load_n(&lines[ROUND][0], __ATOMIC_ACQUIRE) != round) { - } - task(tile, 1, round + 1); - __atomic_store_n(&lines[DONE + tile][0], round, __ATOMIC_RELEASE); - } - task(tile, 2, 4); - __atomic_store_n(&lines[DONE + tile][0], 3, __ATOMIC_RELEASE); -} -int main(void) { - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart) - controller(hart - 1); - else { - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&lines[READY + tile][0], __ATOMIC_ACQUIRE)) { - } - for (unsigned round = 1; round <= 2; ++round) { - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - for (unsigned word = 0; word < 2; ++word) { - lines[INPUT + tile][word] = pattern(round, tile, word); - lines[OUTPUT + tile][word] = ~pattern(round, tile, word); - } - __atomic_store_n(&lines[ROUND][0], round, __ATOMIC_RELEASE); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (__atomic_load_n(&lines[DONE + tile][0], __ATOMIC_ACQUIRE) < - round) { - } - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - for (unsigned word = 0; word < 2; ++word) - if (lines[OUTPUT + tile][word] != pattern(round, tile, word)) - __builtin_trap(); - } - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (__atomic_load_n(&lines[DONE + tile][0], __ATOMIC_ACQUIRE) != 3) { - } - } - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_highpa_main.c b/examples/chips/poly/workloads/ctests/ant_highpa_main.c deleted file mode 100644 index 78b736a8c..000000000 --- a/examples/chips/poly/workloads/ctests/ant_highpa_main.c +++ /dev/null @@ -1,64 +0,0 @@ -#include "ant_highpa.h" -#include -#include -#include -extern const unsigned char ant_image[], ant_image_end[]; -static unsigned done[BB_COMPUTE_TILES]; -static void exercise(unsigned tile) { - const unsigned context = BB_ANT_CONTEXT; - uintptr_t high = 0x180200000ULL + tile * 0x20000ULL; - volatile uint64_t *input = (void *)high, *output = (void *)(high + 0x1000); - volatile uint64_t *low = (void *)(high & 0xffffffffULL); - volatile uint64_t *sentinel = (void *)((high + 0x1000) & 0xffffffffULL); - const uint64_t marker = 0xdeadface12345678ULL; - uint64_t first = 0x1234567800000000ULL | ((uint64_t)tile << 16); - uint64_t second = 0xfedcba9800000000ULL | ((uint64_t)tile << 16); - for (unsigned i = 0; i < 4; ++i) { - low[i] = second + i; - sentinel[i] = marker; - input[i] = first + i; - output[i] = 0; - } - for (unsigned i = 0; i < 4; ++i) - if (input[i] != first + i || low[i] != second + i || sentinel[i] != marker) - __builtin_trap(); - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(context, ANT_CODE_BYTES) || - ant_query(context, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - ant_write(context, ANT_CODE, 0, ant_image, bytes); - uint64_t base = ant_query(context, ANT_TLS_BASE); - for (unsigned phase = 0; phase < 2; ++phase) { - struct ant_highpa_args args = {phase ? (uintptr_t)low : high, - high + 0x1000}; - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - struct ant_task task = {tile * 2 + phase + 1, - 0, - bytes, - base, - base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_acquire(); - ant_start(context, &task); - int cancelled; - if (ant_wait(context, task.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); - for (unsigned i = 0; i < 4; ++i) - if (output[i] != (phase ? second : first) + i || sentinel[i] != marker || - input[i] != first + i || low[i] != second + i) - __builtin_trap(); - } - __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); -} -int main(void) { - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart >= BB_MAIN_CORES) - exercise(hart - BB_MAIN_CORES); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { - } - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_mesh_move_main.c b/examples/chips/poly/workloads/ctests/ant_mesh_move_main.c deleted file mode 100644 index 221653a3e..000000000 --- a/examples/chips/poly/workloads/ctests/ant_mesh_move_main.c +++ /dev/null @@ -1,89 +0,0 @@ -#define _GNU_SOURCE -#include "ant_mesh_move.h" -#include -#include -#include - -extern const unsigned char ant_image[], ant_image_end[]; -enum { ROW_WORDS = BANK_WIDTH / 64, ROWS = 2 }; -static uint64_t input[BB_COMPUTE_TILES][ROWS * ROW_WORDS] - __attribute__((section(".data"), aligned(4096))); -static uint64_t output[BB_COMPUTE_TILES][ROWS * ROW_WORDS] - __attribute__((section(".data"), aligned(4096))); -static unsigned done[BB_COMPUTE_TILES]; - -static void run(unsigned context, unsigned id, - struct ant_mesh_move_args *args) { - uint64_t base = ant_query(context, ANT_TLS_BASE); - ant_write(context, ANT_TLS, 0, args, sizeof(*args)); - struct ant_task task = {id, - 0, - ant_image_end - ant_image, - base, - base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_start(context, &task); - int cancelled; - if (ant_wait(context, id, &cancelled) || cancelled) - __builtin_trap(); -} - -static struct ant_task prepare(unsigned context, unsigned id, - struct ant_mesh_move_args *args) { - uint64_t base = ant_query(context, ANT_TLS_BASE); - ant_write(context, ANT_TLS, 0, args, sizeof(*args)); - return (struct ant_task){id, - 0, - ant_image_end - ant_image, - base, - base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; -} - -static void exercise(unsigned tile) { - size_t bytes = ant_image_end - ant_image; - for (unsigned context = 0; context < 2; ++context) { - if (bytes > ant_query(context, ANT_CODE_BYTES) || - ant_query(context, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - ant_write(context, ANT_CODE, 0, ant_image, bytes); - } - for (unsigned i = 0; i < ROWS * ROW_WORDS; ++i) { - input[tile][i] = ((uint64_t)(tile + 1) << 56) | (0x100 + i); - output[tile][i] = 0; - } - uint64_t zero[2] = {0}; - ant_write(0, ANT_TSS, 0, zero, sizeof(zero)); - struct ant_mesh_move_args args = {(uintptr_t)input[tile], - (uintptr_t)output[tile], - ant_query(0, ANT_TSS_BASE), MESH_FILL}; - ant_acquire(); - run(0, tile * 3 + 1, &args); - args.role = MESH_MOVE; - struct ant_task move = prepare(0, tile * 3 + 2, &args); - args.role = MESH_CONSUME; - struct ant_task consume = prepare(1, tile * 3 + 3, &args); - ant_start(1, &consume); - ant_start(0, &move); - int cancelled; - if (ant_wait(0, move.id, &cancelled) || cancelled || - ant_wait(1, consume.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); - for (unsigned i = 0; i < ROWS * ROW_WORDS; ++i) - if (output[tile][i] != input[tile][i]) - __builtin_trap(); - __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); -} - -int main(void) { - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart >= BB_MAIN_CORES) - exercise(hart - BB_MAIN_CORES); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { - } - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_mxmm_main.cpp b/examples/chips/poly/workloads/ctests/ant_mxmm_main.cpp deleted file mode 100644 index 79fa6aa54..000000000 --- a/examples/chips/poly/workloads/ctests/ant_mxmm_main.cpp +++ /dev/null @@ -1,96 +0,0 @@ -#define _GNU_SOURCE -#include "ant_mxmm.h" -#include -#include -#include -#ifdef __linux__ -#include -#include -#include -#include -#include -#endif -extern const unsigned char ant_image[], ant_image_end[]; -struct matrices { - float a[BB_COMPUTE_TILES][ANT_M * ANT_K]; - float b[BB_COMPUTE_TILES][ANT_N * ANT_K]; - float out[BB_COMPUTE_TILES][ANT_M * ANT_N]; -}; -static constexpr matrices prepare() { - matrices data{}; - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) { - for (unsigned row = 0; row < ANT_M; ++row) - for (unsigned k = 0; k < ANT_K; ++k) - data.a[tile][row * ANT_K + k] = row + tile + 1; - for (unsigned col = 0; col < ANT_N; ++col) - for (unsigned k = 0; k < ANT_K; ++k) - data.b[tile][col * ANT_K + k] = col + 1; - for (unsigned i = 0; i < ANT_M * ANT_N; ++i) - data.out[tile][i] = -1; - } - return data; -} -alignas(64) static matrices data = prepare(); -static unsigned done[BB_COMPUTE_TILES]; -static void exercise(unsigned tile) { - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(0, ANT_CODE_BYTES) || - ant_query(0, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - struct ant_mxmm_args args = {(uintptr_t)data.a[tile], (uintptr_t)data.b[tile], - (uintptr_t)data.out[tile]}; - ant_write(0, ANT_CODE, 0, ant_image, bytes); - ant_write(0, ANT_TLS, 0, &args, sizeof(args)); - uint64_t base = ant_query(0, ANT_TLS_BASE), - capacity = ant_query(0, ANT_TLS_BYTES); - struct ant_task task = {tile + 1, 0, bytes, base, base + capacity, - CORE_SIGNATURE}; - ant_acquire(); - ant_start(0, &task); - int cancelled; - if (ant_wait(0, task.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); - for (unsigned row = 0; row < ANT_M; ++row) - for (unsigned col = 0; col < ANT_N; ++col) - if (data.out[tile][row * ANT_N + col] != - (float)(ANT_K * (row + tile + 1) * (col + 1))) - __builtin_trap(); - __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); -} -#ifdef __linux__ -static void *worker(void *argument) { - uintptr_t tile = (uintptr_t)argument; - cpu_set_t mask; - CPU_ZERO(&mask); - CPU_SET(BB_MAIN_CORES + tile, &mask); - if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) - abort(); - exercise(tile); - return NULL; -} -#endif -int main(void) { -#ifdef __linux__ - if (mlockall(MCL_CURRENT | MCL_FUTURE)) - abort(); - pthread_t threads[BB_COMPUTE_TILES]; - for (uintptr_t tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_create(&threads[tile], NULL, worker, (void *)tile)) - abort(); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_join(threads[tile], NULL)) - abort(); - printf("Ant Mxmm PASS tiles=%u\n", (unsigned)BB_COMPUTE_TILES); -#else - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart >= BB_MAIN_CORES) - exercise(hart - BB_MAIN_CORES); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { - } -#endif - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_parallel_exercise.c b/examples/chips/poly/workloads/ctests/ant_parallel_exercise.c deleted file mode 100644 index 0b693c103..000000000 --- a/examples/chips/poly/workloads/ctests/ant_parallel_exercise.c +++ /dev/null @@ -1,68 +0,0 @@ -#include "ant_parallel.h" -#include -#include -#include -extern const unsigned char ant_image[], ant_image_end[]; -static uint64_t input[BB_COMPUTE_TILES][2][512] - __attribute__((aligned(4096), section(".noinit"))); -static uint64_t output[BB_COMPUTE_TILES][2][512] - __attribute__((aligned(4096), section(".noinit"))); -unsigned parallel_done[BB_COMPUTE_TILES]; -static unsigned reached[3] __attribute__((aligned(64))); -static void barrier(unsigned phase) { - __atomic_fetch_add(&reached[phase], 1, __ATOMIC_ACQ_REL); - while (__atomic_load_n(&reached[phase], __ATOMIC_ACQUIRE) != - BB_COMPUTE_TILES) { - } -} -void parallel_exercise(unsigned tile) { - size_t bytes = ant_image_end - ant_image; - uint64_t shared = ant_query(0, ANT_TSS_BASE), zero = 0; - struct ant_task tasks[2]; - for (unsigned context = 0; context < 2; ++context) { - for (unsigned word = 0; word < 2; ++word) { - input[tile][context][word] = - ((uint64_t)(tile + 1) << 32) | (context * 2 + word + 1); - output[tile][context][word] = ~0ULL; - } - uint64_t base = ant_query(context, ANT_TLS_BASE); - if (bytes > ant_query(context, ANT_CODE_BYTES) || - ant_query(context, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - struct ant_parallel_args args = {shared, context, - (uintptr_t)input[tile][context], - (uintptr_t)output[tile][context]}; - ant_write(context, ANT_CODE, 0, ant_image, bytes); - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - tasks[context] = (struct ant_task){ - context + 1, 0, bytes, base, base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - } - ant_write(0, ANT_TSS, 0, &zero, sizeof(zero)); - barrier(0); - unsigned peer = (tile + 1) % BB_COMPUTE_TILES; - for (unsigned context = 0; context < 2; ++context) - for (unsigned word = 0; word < 2; ++word) - if (*(volatile uint64_t *)&output[peer][context][word] != ~0ULL) - __builtin_trap(); - barrier(1); - ant_acquire(); - ant_start(0, &tasks[0]); - ant_start(1, &tasks[1]); - for (unsigned context = 0; context < 2; ++context) { - int cancelled; - if (ant_wait(context, tasks[context].id, &cancelled) || cancelled) - __builtin_trap(); - for (unsigned word = 0; word < 2; ++word) - if (output[tile][context][word] != input[tile][context][word]) - __builtin_trap(); - } - ant_release(); - barrier(2); - for (unsigned context = 0; context < 2; ++context) - for (unsigned word = 0; word < 2; ++word) - if (*(volatile uint64_t *)&output[peer][context][word] != - input[peer][context][word]) - __builtin_trap(); - __atomic_store_n(¶llel_done[tile], 1, __ATOMIC_RELEASE); -} diff --git a/examples/chips/poly/workloads/ctests/ant_parallel_main.c b/examples/chips/poly/workloads/ctests/ant_parallel_main.c deleted file mode 100644 index 7df2096b9..000000000 --- a/examples/chips/poly/workloads/ctests/ant_parallel_main.c +++ /dev/null @@ -1,49 +0,0 @@ -#define _GNU_SOURCE -#include "ant_parallel.h" -#include -#include -#include -#ifdef __linux__ -#include -#include -#include -#include -#include -#endif -#ifdef __linux__ -static void *worker(void *argument) { - uintptr_t tile = (uintptr_t)argument; - cpu_set_t mask; - CPU_ZERO(&mask); - CPU_SET(BB_MAIN_CORES + tile, &mask); - if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) - abort(); - parallel_exercise(tile); - return NULL; -} -#endif -int main(void) { -#ifdef __linux__ - if (mlockall(MCL_CURRENT | MCL_FUTURE)) - abort(); - pthread_t threads[BB_COMPUTE_TILES]; - for (uintptr_t tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_create(&threads[tile], NULL, worker, (void *)tile)) - abort(); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_join(threads[tile], NULL)) - abort(); - printf("Ant concurrent DMA/cache sharing PASS tiles=%u contexts=2\n", - (unsigned)BB_COMPUTE_TILES); -#else - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart >= BB_MAIN_CORES) - parallel_exercise(hart - BB_MAIN_CORES); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(¶llel_done[tile], __ATOMIC_ACQUIRE)) { - } -#endif - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_shared.h b/examples/chips/poly/workloads/ctests/ant_shared.h deleted file mode 100644 index 2b08f1465..000000000 --- a/examples/chips/poly/workloads/ctests/ant_shared.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once -#include - -struct ant_shared_args { - uint64_t shared, producer, value; -}; diff --git a/examples/chips/poly/workloads/ctests/ant_shared_main.c b/examples/chips/poly/workloads/ctests/ant_shared_main.c deleted file mode 100644 index 8c72e74fe..000000000 --- a/examples/chips/poly/workloads/ctests/ant_shared_main.c +++ /dev/null @@ -1,95 +0,0 @@ -#define _GNU_SOURCE -#include "ant_shared.h" -#include -#include -#include -#ifdef __linux__ -#include -#include -#include -#include -#endif -extern const unsigned char ant_image[], ant_image_end[]; -static unsigned done[BB_COMPUTE_TILES]; -static void exercise(unsigned tile) { - size_t bytes = ant_image_end - ant_image; - uint64_t value = 100 + tile, shared = ant_query(0, ANT_TSS_BASE), - zero[2] = {0}; - struct ant_task tasks[2]; - for (unsigned context = 0; context < 2; ++context) { - uint64_t base = ant_query(context, ANT_TLS_BASE); - if (bytes > ant_query(context, ANT_CODE_BYTES) || - ant_query(context, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - tasks[context] = (struct ant_task){ - context + 1, 0, bytes, base, base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - struct ant_shared_args args = {shared, context == 0, context ? 0 : value}; - ant_write(context, ANT_CODE, 0, ant_image, bytes); - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - } - ant_write(0, ANT_TSS, 0, zero, sizeof(zero)); - ant_acquire(); - ant_start(1, &tasks[1]); - ant_start(0, &tasks[0]); - for (unsigned context = 0; context < 2; ++context) { - int cancelled; - if (ant_wait(context, tasks[context].id, &cancelled) != value || cancelled) - __builtin_trap(); - if (ant_read(context, ANT_TLS, 16) != value) - __builtin_trap(); - } - ant_release(); - ant_write(0, ANT_TSS, 0, zero, sizeof(zero)); - ant_acquire(); - ant_start(1, &tasks[1]); - ant_cancel(1); - int cancelled; - ant_wait(1, tasks[1].id, &cancelled); - if (!cancelled) - __builtin_trap(); - ant_release(); - uint64_t ready[2] = {value + 7, 1}; - ant_write(0, ANT_TSS, 0, ready, sizeof(ready)); - ++tasks[1].id; - ant_acquire(); - ant_start(1, &tasks[1]); - if (ant_wait(1, tasks[1].id, &cancelled) != value + 7 || cancelled) - __builtin_trap(); - ant_release(); - __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); -} -#ifdef __linux__ -static void *worker(void *argument) { - uintptr_t tile = (uintptr_t)argument; - cpu_set_t mask; - CPU_ZERO(&mask); - CPU_SET(BB_MAIN_CORES + tile, &mask); - if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) - abort(); - exercise(tile); - return NULL; -} -#endif -int main(void) { -#ifdef __linux__ - pthread_t threads[BB_COMPUTE_TILES]; - for (uintptr_t tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_create(&threads[tile], NULL, worker, (void *)tile)) - abort(); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - if (pthread_join(threads[tile], NULL)) - abort(); - printf("Ant shared/cancel PASS tiles=%u\n", (unsigned)BB_COMPUTE_TILES); -#else - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart >= BB_MAIN_CORES) - exercise(hart - BB_MAIN_CORES); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { - } -#endif - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_stream_main.c b/examples/chips/poly/workloads/ctests/ant_stream_main.c deleted file mode 100644 index 2c122871b..000000000 --- a/examples/chips/poly/workloads/ctests/ant_stream_main.c +++ /dev/null @@ -1,57 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -enum { M = 16, N = 16, K = 32 }; -static float a[M * K] __attribute__((aligned(4096))); -static float b[N * K] __attribute__((aligned(4096))); -static float output[M * N] __attribute__((aligned(4096))); -static void job(void *argument) { - (void)argument; - for (unsigned bank = 3; bank <= 5; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a, 3, sizeof(a) / 16, 1); - // A DMA call has completed before the CPU is allowed to reuse its source. - for (unsigned i = 0; i < M * K; ++i) - a[i] = -1; - bb_mvin((uintptr_t)b, 4, sizeof(b) / 16, 1); - bb_mxmm_f32(3, 4, 5, M, N, K, 1, 1, 0); - bb_mvout((uintptr_t)output, 5, sizeof(output) / 16, 1); - for (unsigned row = 0; row < M; ++row) - for (unsigned col = 0; col < N; ++col) - if (output[row * N + col] != (float)(K * (row + 1) * (col + 1))) - abort(); - for (unsigned bank = 3; bank <= 5; ++bank) - bb_mem_release(bank); -} -int main(void) { - cpu_set_t affinity; - CPU_ZERO(&affinity); - CPU_SET(BB_MAIN_CORES, &affinity); - if (sched_setaffinity(0, sizeof(affinity), &affinity)) - abort(); - runtime_init(1024 * 1024); - void *workspace = aligned_alloc(64, 1024 * 1024); - if (!workspace) - abort(); - workspace_init(workspace, 1024 * 1024); - workspace_begin(workspace, 1024 * 1024); - for (unsigned row = 0; row < M; ++row) - for (unsigned k = 0; k < K; ++k) - a[row * K + k] = row + 1; - for (unsigned col = 0; col < N; ++col) - for (unsigned k = 0; k < K; ++k) - b[col * K + k] = col + 1; - for (unsigned i = 0; i < M * N; ++i) - output[i] = -1; - task_run(CORE_SIGNATURE, job, NULL); - puts("Ant stream DMA lifetime/Mxmm PASS"); - free(workspace); - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_tlink.c b/examples/chips/poly/workloads/ctests/ant_tlink.c deleted file mode 100644 index a019d6f95..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink.c +++ /dev/null @@ -1,16 +0,0 @@ -#include "ant_tlink.h" -#include -uint64_t ant_main(const struct ant_tlink_args *args) { - if (args->phase == 0) { - bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); - bb_mset(BB_SHARED_BANK_BASE + 1, 1, 1, 1); - bb_mvin(args->input, BB_SHARED_BANK_BASE, TLINK_TEST_ROWS, 1); - } else if (args->phase == 1) { - bb_mvout(args->output, BB_SHARED_BANK_BASE + 1, TLINK_TEST_ROWS, 1); - } else if (args->phase == 2) { - bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); - bb_mset(BB_SHARED_BANK_BASE + 1, 0, 0, 0); - } else - __builtin_trap(); - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_tlink.h b/examples/chips/poly/workloads/ctests/ant_tlink.h deleted file mode 100644 index 8f01c1347..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once -#include -enum { TLINK_TEST_BYTES = 4160, TLINK_TEST_ROWS = TLINK_TEST_BYTES / 16 }; -struct ant_tlink_args { - uint64_t phase, input, output; -}; diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_bank.c b/examples/chips/poly/workloads/ctests/ant_tlink_bank.c deleted file mode 100644 index 9776e457d..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_bank.c +++ /dev/null @@ -1,16 +0,0 @@ -#include "ant_tlink_bank.h" -#include -uint64_t ant_main(const struct ant_tlink_bank_args *args) { - if (args->phase == 0) { - bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); - bb_mset(BB_SHARED_BANK_BASE + 1, 1, 1, 1); - bb_mvin(args->input, BB_SHARED_BANK_BASE, TLINK_CROSS_ROWS, 1); - } else if (args->phase == 1) { - bb_mvout(args->output, BB_SHARED_BANK_BASE, TLINK_CROSS_ROWS, 1); - } else if (args->phase == 2) { - bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); - bb_mset(BB_SHARED_BANK_BASE + 1, 0, 0, 0); - } else - __builtin_trap(); - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_bank.h b/examples/chips/poly/workloads/ctests/ant_tlink_bank.h deleted file mode 100644 index 13b587e46..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_bank.h +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once -#include -enum { TLINK_CROSS_BYTES = 192, TLINK_CROSS_ROWS = TLINK_CROSS_BYTES / 16 }; -struct ant_tlink_bank_args { - uint64_t phase, input, output; -}; diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_bank_main.c b/examples/chips/poly/workloads/ctests/ant_tlink_bank_main.c deleted file mode 100644 index 58ae1fb9e..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_bank_main.c +++ /dev/null @@ -1,92 +0,0 @@ -#include "ant_tlink_bank.h" -#include -#include -#include -#include -_Static_assert(BB_MAIN_CORES == 1 && BB_COMPUTE_TILES >= 2 && - !(BB_COMPUTE_TILES & (BB_COMPUTE_TILES - 1)), - "TLink gate requires a power-of-two compute tile count"); -extern const unsigned char ant_image[], ant_image_end[]; -static uint64_t input[BB_COMPUTE_TILES][TLINK_CROSS_BYTES / 8] - __attribute__((aligned(64))); -static uint64_t output[BB_COMPUTE_TILES][TLINK_CROSS_BYTES / 8] - __attribute__((aligned(64))); -static unsigned ready[BB_COMPUTE_TILES], done[BB_COMPUTE_TILES]; -static unsigned arrived[2]; -static void barrier(unsigned phase) { - __atomic_fetch_add(&arrived[phase], 1, __ATOMIC_ACQ_REL); - while (__atomic_load_n(&arrived[phase], __ATOMIC_ACQUIRE) != - BB_COMPUTE_TILES) { - } -} -static unsigned pattern(unsigned tile, unsigned i) { - return (i * 17 + tile * 23 + 3) & 255; -} -static void phase(unsigned tile, unsigned operation) { - const unsigned context = BB_ANT_CONTEXT; - size_t bytes = ant_image_end - ant_image; - struct ant_tlink_bank_args args = {operation, (uintptr_t)input[tile], - (uintptr_t)output[tile]}; - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - uint64_t base = ant_query(context, ANT_TLS_BASE); - struct ant_task task = {tile * 3 + operation + 1, - 0, - bytes, - base, - base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_acquire(); - ant_start(context, &task); - int cancelled; - if (ant_wait(context, task.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); -} -static void exercise(unsigned tile) { - uint64_t bank_bytes = tlink_query(TLINK_BANK_BYTES); - if (tlink_query(TLINK_TILE_ID) != tile + 1 || - tlink_query(TLINK_TILE_COUNT) != BB_MAIN_CORES + BB_COMPUTE_TILES || - bank_bytes < TLINK_CROSS_BYTES || - tlink_query(TLINK_SHARED_BYTES) < bank_bytes * 2) - __builtin_trap(); - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(BB_ANT_CONTEXT, ANT_CODE_BYTES) || - ant_query(BB_ANT_CONTEXT, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - ant_write(BB_ANT_CONTEXT, ANT_CODE, 0, ant_image, bytes); - for (unsigned i = 0; i < TLINK_CROSS_BYTES / 8; ++i) { - uint64_t value = 0; - for (unsigned j = 0; j < 8; ++j) - value |= (uint64_t)pattern(tile, i * 8 + j) << (8 * j); - input[tile][i] = value; - } - phase(tile, 0); - barrier(0); - unsigned next = (tile + 1) % BB_COMPUTE_TILES; - unsigned previous = (tile + BB_COMPUTE_TILES - 1) % BB_COMPUTE_TILES; - tlink_transfer(0, next + 1, bank_bytes - 16, 37); - barrier(1); - tlink_transfer(bank_bytes - 16, previous + 1, 128, 37); - __atomic_store_n(&ready[tile], 1, __ATOMIC_RELEASE); - while (!__atomic_load_n(&ready[next], __ATOMIC_ACQUIRE)) { - } - phase(tile, 1); - for (unsigned i = 0; i < TLINK_CROSS_BYTES; ++i) { - unsigned expected = pattern(tile, i >= 128 && i < 165 ? i - 128 : i); - if (((unsigned char *)output[tile])[i] != expected) - __builtin_trap(); - } - phase(tile, 2); - __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); -} -int main(void) { - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart) - exercise(hart - 1); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { - } - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_main.c b/examples/chips/poly/workloads/ctests/ant_tlink_main.c deleted file mode 100644 index f09c51ee0..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_main.c +++ /dev/null @@ -1,88 +0,0 @@ -#include "ant_tlink.h" -#include -#include -#include -#include -_Static_assert(BB_MAIN_CORES == 1 && BB_COMPUTE_TILES >= 2 && - !(BB_COMPUTE_TILES & (BB_COMPUTE_TILES - 1)), - "TLink gate requires a power-of-two compute tile count"); -extern const unsigned char ant_image[], ant_image_end[]; -static uint64_t input[BB_COMPUTE_TILES][TLINK_TEST_BYTES / 8] - __attribute__((aligned(64))); -static uint64_t output[BB_COMPUTE_TILES][TLINK_TEST_BYTES / 8] - __attribute__((aligned(64))); -static unsigned done[BB_COMPUTE_TILES]; -static unsigned arrived[2]; -static void barrier(unsigned phase) { - __atomic_fetch_add(&arrived[phase], 1, __ATOMIC_ACQ_REL); - while (__atomic_load_n(&arrived[phase], __ATOMIC_ACQUIRE) != - BB_COMPUTE_TILES) { - } -} -static unsigned pattern(unsigned tile, unsigned i) { - return (i * 17 + tile * 23 + 3) & 255; -} -static void phase(unsigned tile, unsigned operation) { - const unsigned context = BB_ANT_CONTEXT; - size_t bytes = ant_image_end - ant_image; - struct ant_tlink_args args = {operation, (uintptr_t)input[tile], - (uintptr_t)output[tile]}; - ant_write(context, ANT_TLS, 0, &args, sizeof(args)); - uint64_t base = ant_query(context, ANT_TLS_BASE); - struct ant_task task = {tile * 3 + operation + 1, - 0, - bytes, - base, - base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_acquire(); - ant_start(context, &task); - int cancelled; - if (ant_wait(context, task.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); -} -static void exercise(unsigned tile) { - uint64_t bank_bytes = tlink_query(TLINK_BANK_BYTES); - if (tlink_query(TLINK_TILE_ID) != tile + 1 || - tlink_query(TLINK_TILE_COUNT) != BB_MAIN_CORES + BB_COMPUTE_TILES || - tlink_query(TLINK_BANK_BYTES) < TLINK_TEST_BYTES || - tlink_query(TLINK_SHARED_BYTES) < bank_bytes * 2) - __builtin_trap(); - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(BB_ANT_CONTEXT, ANT_CODE_BYTES) || - ant_query(BB_ANT_CONTEXT, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - ant_write(BB_ANT_CONTEXT, ANT_CODE, 0, ant_image, bytes); - for (unsigned i = 0; i < TLINK_TEST_BYTES / 8; ++i) { - uint64_t value = 0; - for (unsigned j = 0; j < 8; ++j) - value |= (uint64_t)pattern(tile, i * 8 + j) << (8 * j); - input[tile][i] = value; - } - phase(tile, 0); - barrier(0); - unsigned next = (tile + 1) % BB_COMPUTE_TILES; - unsigned previous = (tile + BB_COMPUTE_TILES - 1) % BB_COMPUTE_TILES; - tlink_transfer(0, next + 1, bank_bytes + 16, 4101); - barrier(1); - phase(tile, 1); - for (unsigned i = 0; i < TLINK_TEST_BYTES; ++i) { - unsigned expected = i >= 16 && i < 4117 ? pattern(previous, i - 16) : 0; - if (((unsigned char *)output[tile])[i] != expected) - __builtin_trap(); - } - phase(tile, 2); - __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); -} -int main(void) { - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart) - exercise(hart - 1); - else - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { - } - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_reduce.c b/examples/chips/poly/workloads/ctests/ant_tlink_reduce.c deleted file mode 100644 index 4528083d2..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_reduce.c +++ /dev/null @@ -1,21 +0,0 @@ -#include "ant_tlink_reduce.h" -#include -#include -uint64_t ant_main(const struct ant_tlink_reduce_args *args) { - if (args->phase == 0) { - bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); - bb_mset(0, 1, 1, 1); - bb_mset(1, 1, 1, 1); - bb_mvin(args->input, 0, 4, 1); - bb_mvin(args->input + 16 * sizeof(float), 1, 64, 1); - bb_mxmm_fma32(0, 1, BB_SHARED_BANK_BASE, 1, 16, 16, 1, 1, 0); - bb_mset(0, 0, 0, 0); - bb_mset(1, 0, 0, 0); - } else if (args->phase == 1) { - bb_mvout(args->output, BB_SHARED_BANK_BASE, 2, 1); - } else if (args->phase == 2) { - bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); - } else - __builtin_trap(); - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_reduce.h b/examples/chips/poly/workloads/ctests/ant_tlink_reduce.h deleted file mode 100644 index 33fd9f396..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_reduce.h +++ /dev/null @@ -1,8 +0,0 @@ -#pragma once -#include -struct ant_tlink_reduce_args { - uint64_t phase, input, output; -}; -void ant_tlink_reduce_phase(unsigned tile, - const struct ant_tlink_reduce_args *args); -void ant_tlink_reduce_prepare(unsigned tile, float *input, float *output); diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_reduce_control.c b/examples/chips/poly/workloads/ctests/ant_tlink_reduce_control.c deleted file mode 100644 index 401a90079..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_reduce_control.c +++ /dev/null @@ -1,37 +0,0 @@ -#include "ant_tlink_reduce.h" -#include -#include -extern const unsigned char ant_image[], ant_image_end[]; -void ant_tlink_reduce_phase(unsigned tile, - const struct ant_tlink_reduce_args *args) { - unsigned context = BB_ANT_CONTEXT; - size_t bytes = ant_image_end - ant_image; - ant_write(context, ANT_TLS, 0, args, sizeof(*args)); - uint64_t base = ant_query(context, ANT_TLS_BASE); - struct ant_task task = {tile * 3 + args->phase + 1, - 0, - bytes, - base, - base + ant_query(context, ANT_TLS_BYTES), - CORE_SIGNATURE}; - ant_acquire(); - ant_start(context, &task); - int cancelled; - if (ant_wait(context, task.id, &cancelled) || cancelled) - __builtin_trap(); - ant_release(); -} -void ant_tlink_reduce_prepare(unsigned tile, float *input, float *output) { - size_t bytes = ant_image_end - ant_image; - if (bytes > ant_query(BB_ANT_CONTEXT, ANT_CODE_BYTES) || - ant_query(BB_ANT_CONTEXT, ANT_SIGNATURE) != CORE_SIGNATURE) - __builtin_trap(); - ant_write(BB_ANT_CONTEXT, ANT_CODE, 0, ant_image, bytes); - for (unsigned i = 0; i < 272; ++i) - input[i] = 0.0f; - input[0] = tile % 2 ? -(float)(tile + 1) * 0.5f : (float)(tile + 1) * 1.25f; - for (unsigned column = 0; column < 16; ++column) - input[16 + column * 16] = (float)(column + 1); - for (unsigned i = 0; i < 8; ++i) - output[i] = -77.0f; -} diff --git a/examples/chips/poly/workloads/ctests/ant_tlink_reduce_main.c b/examples/chips/poly/workloads/ctests/ant_tlink_reduce_main.c deleted file mode 100644 index 6acebe80d..000000000 --- a/examples/chips/poly/workloads/ctests/ant_tlink_reduce_main.c +++ /dev/null @@ -1,97 +0,0 @@ -#include "ant_tlink_reduce.h" -#include -#include -#include -#include -#include -#include -_Static_assert(BB_MAIN_CORES == 1 && BB_COMPUTE_TILES >= 2 && - (BB_COMPUTE_TILES & (BB_COMPUTE_TILES - 1)) == 0, - "Reduction needs main and a power-of-two compute tile count"); -static float input[BB_COMPUTE_TILES][272] __attribute__((aligned(64))); -static float output[BB_COMPUTE_TILES][16] __attribute__((aligned(64))); -static unsigned ready[BB_COMPUTE_TILES], broadcast, done[BB_COMPUTE_TILES]; -enum { WINDOW = 4 }; -static uint64_t shared_base[BB_COMPUTE_TILES], main_base[WINDOW]; -static unsigned main_ready, batch; -static float partial(unsigned tile, unsigned i) { - return (float)(i + 1) * - (tile % 2 ? -(float)(tile + 1) * 0.5f : (float)(tile + 1) * 1.25f); -} -static void controller(unsigned tile) { - ant_tlink_reduce_prepare(tile, input[tile], output[tile]); - struct ant_tlink_reduce_args args = {0, (uintptr_t)input[tile], - (uintptr_t)output[tile]}; - ant_tlink_reduce_phase(tile, &args); - shared_base[tile] = tlink_shared_export(0, BB_SHARED_BANK_BASE, 0); - while (!__atomic_load_n(&main_ready, __ATOMIC_ACQUIRE) || - __atomic_load_n(&batch, __ATOMIC_ACQUIRE) != tile / WINDOW) { - } - tlink_transfer(shared_base[tile], 0, main_base[tile % WINDOW], 16); - __atomic_store_n(&ready[tile], 1, __ATOMIC_RELEASE); - while (!__atomic_load_n(&broadcast, __ATOMIC_ACQUIRE)) { - } - args.phase = 1; - ant_tlink_reduce_phase(tile, &args); - for (unsigned i = 0; i < 4; ++i) { - float expected = 0.0f; - for (unsigned rank = 0; rank < BB_COMPUTE_TILES; ++rank) - expected += partial(rank, i); - if (output[tile][i] != partial(tile, i) || output[tile][i + 4] != expected) - __builtin_trap(); - } - tlink_shared_release(0, BB_SHARED_BANK_BASE, 0); - args.phase = 2; - ant_tlink_reduce_phase(tile, &args); - __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); -} -static void reduce(void) { - unsigned groups = BB_COMPUTE_TILES < WINDOW ? BB_COMPUTE_TILES : WINDOW; - unsigned source = 32, accumulator = 33, target = 34; - bb_mem_alloc(source, 1, groups); - bb_mem_alloc(accumulator, 1, 1); - bb_mem_alloc(target, 1, 1); - for (unsigned group = 0; group < groups; ++group) - main_base[group] = tlink_shared_export(0, source, group); - __atomic_store_n(&main_ready, 1, __ATOMIC_RELEASE); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) { - while (!__atomic_load_n(&ready[tile], __ATOMIC_ACQUIRE)) { - } - bb_f32add(source, accumulator, target, 1, tile % WINDOW, tile == 0); - unsigned previous = accumulator; - accumulator = target; - target = previous; - if (tile % WINDOW == WINDOW - 1) { - __atomic_store_n(&batch, tile / WINDOW + 1, __ATOMIC_RELEASE); - } - } - uint64_t result = tlink_shared_export(0, accumulator, 0); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - tlink_transfer(result, tile + 1, shared_base[tile] + 16, 16); - __atomic_store_n(&broadcast, 1, __ATOMIC_RELEASE); - for (unsigned tile = 0; tile < BB_COMPUTE_TILES; ++tile) - while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { - } - tlink_shared_release(0, accumulator, 0); - for (unsigned group = 0; group < groups; ++group) - tlink_shared_release(0, source, group); - bb_mem_release(source); - bb_mem_release(accumulator); - bb_mem_release(target); -} - -int main(void) { - uintptr_t hart; - __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart > BB_COMPUTE_TILES || tlink_query(TLINK_TILE_ID) != hart || - tlink_query(TLINK_TILE_COUNT) != BB_COMPUTE_TILES + 1 || - tlink_query(TLINK_BANK_BYTES) < 48 || - tlink_query(TLINK_SHARED_BYTES) < - tlink_query(TLINK_BANK_BYTES) + BB_COMPUTE_TILES * 16) - __builtin_trap(); - if (hart) - controller(hart - 1); - else - reduce(); - return 0; -} diff --git a/examples/chips/poly/workloads/ctests/controllers_start.S b/examples/chips/poly/workloads/ctests/controllers_start.S deleted file mode 100644 index dfa26d2d3..000000000 --- a/examples/chips/poly/workloads/ctests/controllers_start.S +++ /dev/null @@ -1,15 +0,0 @@ -#include -/* Only Linux control cores participate; data-plane workers remain parked. */ -.section .text.init,"ax",@progbits -.globl _start -_start: - csrr t0, mhartid - li t1, BB_MAIN_CORES + BB_COMPUTE_TILES - bltu t0, t1, controllers_start -1: - wfi - j 1b -#undef BB_REPORT_HART -#define BB_REPORT_HART 0 -#define _start controllers_start -#include "start.S" diff --git a/examples/chips/poly/workloads/ctests/mesh_shared_mem_test.c b/examples/chips/poly/workloads/ctests/mesh_shared_mem_test.c deleted file mode 100644 index 04cd3f76a..000000000 --- a/examples/chips/poly/workloads/ctests/mesh_shared_mem_test.c +++ /dev/null @@ -1,50 +0,0 @@ -#include -#include -#include - -#include - -#if BANK_WIDTH != 128 -#error "mesh_shared_mem_test requires 128-bit Bank rows" -#endif -#if BB_SHARED_PHYSICAL_BANK_NUM < 1 -#error "mesh_shared_mem_test requires at least one shared Bank" -#endif - -#if BB_VIRTUAL_BANK_NUM <= BB_SHARED_BANK_BASE -#error "mesh_shared_mem_test requires shared virtual Bank IDs" -#endif -#if BB_VIRTUAL_BANK_NUM - BB_SHARED_BANK_BASE > BB_SHARED_PHYSICAL_BANK_NUM -#error "mesh_shared_mem_test requires a physical Bank per shared virtual Bank" -#endif - -enum { - SHARED_TEST_BANKS = (BB_VIRTUAL_BANK_NUM - BB_SHARED_BANK_BASE), - ROW_BYTES = (BANK_WIDTH / 8) -}; - -static uint8_t source[SHARED_TEST_BANKS][ROW_BYTES] - __attribute__((aligned(64))); -static uint8_t result[SHARED_TEST_BANKS][ROW_BYTES] - __attribute__((aligned(64))); - -/* crt0 boots only the target's test hart. */ -int main(void) { - for (int bank = 0; bank < SHARED_TEST_BANKS; ++bank) { - bb_mem_alloc(BB_SHARED_BANK_BASE + bank, 1, 1); - for (int byte = 0; byte < ROW_BYTES; ++byte) - source[bank][byte] = (uint8_t)(bank * 17 + byte); - bb_mvin((uintptr_t)source[bank], BB_SHARED_BANK_BASE + bank, 1, 1); - } - - for (int bank = 0; bank < SHARED_TEST_BANKS; ++bank) - bb_mvout((uintptr_t)result[bank], BB_SHARED_BANK_BASE + bank, 1, 1); - - for (int bank = 0; bank < SHARED_TEST_BANKS; ++bank) { - for (int byte = 0; byte < ROW_BYTES; ++byte) - if (result[bank][byte] != source[bank][byte]) - return 1; - bb_mem_release(BB_SHARED_BANK_BASE + bank); - } - return 0; -} diff --git a/examples/chips/toy/arch/src/main/scala/sims/verilator/TargetConfigs.scala b/examples/chips/toy/arch/src/main/scala/sims/verilator/TargetConfigs.scala deleted file mode 100644 index f4f709879..000000000 --- a/examples/chips/toy/arch/src/main/scala/sims/verilator/TargetConfigs.scala +++ /dev/null @@ -1,11 +0,0 @@ -package sims.verilator - -import chisel3.experimental.hierarchy.{Instance, Instantiate} -import framework.system.{System, SystemParams} - -import sims.soc.SystemTarget - -class ToyVerilatorTarget extends SystemTarget("../examples/chips/toy/configs/generated/chip.pb") { - override def instantiate(p: SystemParams): Instance[System] = - Instantiate(new framework.system.Device(p.toGlobalConfig)) -} diff --git a/examples/chips/toy/configs/chip.toml b/examples/chips/toy/configs/chip.toml deleted file mode 100644 index 078e3f13d..000000000 --- a/examples/chips/toy/configs/chip.toml +++ /dev/null @@ -1,9 +0,0 @@ -[designs] -include = "designs/toy.toml" - -[sims] -verilator = "sims.verilator.ToyVerilatorTarget" - -[uvm] -balls = ["gemmini"] -ips = ["axis", "memdomain"] diff --git a/examples/chips/toy/configs/designs/tiles/default.toml b/examples/chips/toy/configs/designs/tiles/default.toml deleted file mode 100644 index d6b3c3dd6..000000000 --- a/examples/chips/toy/configs/designs/tiles/default.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Toy Tile Configuration (1 core) - -coreDataBytes = 64 -xLen = 64 -vaddrBits = 39 -paddrBits = 56 -pgIdxBits = 12 -pgLevels = 3 -nPMPs = 8 - -memBallChannelNum = 6 - -[sharedMem] -bankWidth = 128 -bankEntries = 1024 -enable = false -entries = 16384 -inputChannels = 7 -defaultGroupCount = 1 - -[[cores]] -core_id = 0 -include = "../../../../../cores/toy/configs/default.toml" diff --git a/examples/chips/toy/configs/designs/toy.toml b/examples/chips/toy/configs/designs/toy.toml deleted file mode 100644 index e8636c8ee..000000000 --- a/examples/chips/toy/configs/designs/toy.toml +++ /dev/null @@ -1,4 +0,0 @@ -# Toy: a main tile only. - -[main_tile] -include = "tiles/default.toml" diff --git a/examples/chips/toy/kernel/workloads.toml b/examples/chips/toy/kernel/workloads.toml deleted file mode 100644 index f93f96786..000000000 --- a/examples/chips/toy/kernel/workloads.toml +++ /dev/null @@ -1,44 +0,0 @@ -[workloads] -tests = [ - "toy-toy-mlirtest-base_dma_group-linux", - "toy-toy-mlirtest-base_mset_transfer-linux", - "toy-toy-ctest-mvin_mvout_group_test-linux", - "toy-toy-ctest-mset_transfer_test-linux", - "toy-toy-ctest-rvv_ball-linux", - "toy-toy-ctest-rvv_kernel-linux", - "toy-toy-ctest-gemmini_accumulated_16x16_test-linux", - "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-linux", - "toy-toy-ctest-alexnet_gemmini_round_test-linux", - "toy-toy-ctest-alexnet_gemmini_tail_test-linux", - "toy-toy-ctest-alexnet_gemmini_stride_test-linux", - "toy-toy-ctest-mvin_2d_test-linux", - "toy-toy-ctest-mvin_mvout_acc_test-linux", - "toy-toy-ctest-mvin_mvout_test-linux", - "toy-toy-ctest-mvin_mvout_stride_test-linux", - "toy-toy-ctest-mvin_mvout_unaligned_test-linux", - "toy-toy-ctest-mvin_unaligned_test-linux", - "toy-toy-ctest-bdb_counter_start_stop_test-linux", - "toy-toy-ctest-bdb_counter_read_active_test-linux", - "toy-toy-ctest-bdb_counter_nested_pair_test-linux", - "toy-toy-ctest-bdb_counter_multi_id_test-linux", - "toy-toy-mlirtest-ball_bdb_counter_start_read_stop-linux", - "toy-toy-ctest-gemmini_os_cisc_basic_test-linux", - "toy-toy-ctest-gemmini_os_cisc_atranspose_test-linux", - "toy-toy-ctest-gemmini_os_cisc_btranspose_test-linux", - "toy-toy-ctest-gemmini_os_cisc_shift_test-linux", - "toy-toy-ctest-gemmini_os_risc_atranspose_test-linux", - "toy-toy-ctest-gemmini_os_risc_btranspose_test-linux", - "toy-toy-ctest-gemmini_os_risc_abtranspose_test-linux", - "toy-toy-ctest-gemmini_os_risc_shift_test-linux", - "toy-toy-ctest-gemmini_ws_cisc_conv_test-linux", - "toy-toy-ctest-gemmini_ws_risc_basic_test-linux", - "toy-toy-ctest-mmio_minimal_test-linux", - "toy-toy-ctest-mxfp4_e8m0_two_block_test-linux", - "toy-toy-ctest-relu_i32_16x16_test-linux", - "toy-toy-mlirtest-ball_relu_i32_16row-linux", - "toy-toy-mlirtest-bank_relu_i32_mixed_sign-linux", - "toy-toy-mlirtest-bank_mxfp4_unit_scale_bank-linux", - "toy-toy-mlirtest-ball_mxfp4_unit_scale_2blocks-linux", - "toy-toy-mlirtest-bank_gemmini_compute_preloaded_bank_tiles-linux", - "toy-toy-mlirtest-bank_gemmini_compute_accumulated_bank_tiles-linux", -] diff --git a/examples/chips/toy/regression/batch/bemu/workloads-bare.toml b/examples/chips/toy/regression/batch/bemu/workloads-bare.toml deleted file mode 100644 index b2880a260..000000000 --- a/examples/chips/toy/regression/batch/bemu/workloads-bare.toml +++ /dev/null @@ -1,53 +0,0 @@ -[workloads] -tests = [ - "toy-toy-mlirtest-base_dma_group-baremetal", - "toy-toy-mlirtest-base_mset_transfer-baremetal", - "toy-toy-ctest-mvin_mvout_group_test-baremetal", - "toy-toy-ctest-mset_transfer_test-baremetal", - "toy-toy-ctest-rvv_ball-baremetal", - "toy-toy-ctest-rvv_kernel-baremetal", - "toy-toy-ctest-mvin_2d_pixel_stride_test-baremetal", - "toy-toy-ctest-gemmini_accumulated_16x16_test-baremetal", - "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-baremetal", - "toy-toy-ctest-alexnet_gemmini_round_test-baremetal", - "toy-toy-ctest-alexnet_gemmini_tail_test-baremetal", - "toy-toy-ctest-alexnet_gemmini_stride_test-baremetal", - "toy-toy-ctest-mvin_2d_test-baremetal", - "toy-toy-ctest-mvin_mvout_acc_test-baremetal", - "toy-toy-ctest-mvin_mvout_test-baremetal", - "toy-toy-ctest-mvin_mvout_stride_test-baremetal", - "toy-toy-ctest-mvin_mvout_unaligned_test-baremetal", - "toy-toy-ctest-mvin_unaligned_test-baremetal", - "toy-toy-ctest-bdb_counter_start_stop_test-baremetal", - "toy-toy-ctest-bdb_counter_read_active_test-baremetal", - "toy-toy-ctest-bdb_counter_nested_pair_test-baremetal", - "toy-toy-ctest-bdb_counter_multi_id_test-baremetal", - "toy-toy-mlirtest-ball_bdb_counter_start_read_stop-baremetal", - "toy-toy-ctest-gemmini_os_cisc_basic_test-baremetal", - "toy-toy-ctest-gemmini_os_cisc_atranspose_test-baremetal", - "toy-toy-ctest-gemmini_os_cisc_btranspose_test-baremetal", - "toy-toy-ctest-gemmini_os_cisc_shift_test-baremetal", - "toy-toy-ctest-gemmini_os_risc_atranspose_test-baremetal", - "toy-toy-ctest-gemmini_os_risc_btranspose_test-baremetal", - "toy-toy-ctest-gemmini_os_risc_abtranspose_test-baremetal", - "toy-toy-ctest-gemmini_os_risc_shift_test-baremetal", - "toy-toy-ctest-gemmini_ws_cisc_conv_test-baremetal", - "toy-toy-ctest-gemmini_ws_risc_basic_test-baremetal", - "toy-toy-ctest-mmio_minimal_test-baremetal", - "toy-toy-ctest-mxfp4_e8m0_two_block_test-baremetal", - "toy-toy-ctest-relu_i32_16x16_test-baremetal", - "toy-toy-mlirtest-ball_relu_i32_16row-baremetal", - "toy-toy-mlirtest-bank_relu_i32_mixed_sign-baremetal", - "toy-toy-mlirtest-bank_mxfp4_unit_scale_bank-baremetal", - "toy-toy-mlirtest-ball_mxfp4_unit_scale_2blocks-baremetal", - "toy-toy-mlirtest-bank_gemmini_compute_preloaded_bank_tiles-baremetal", - "toy-toy-mlirtest-bank_gemmini_compute_accumulated_bank_tiles-baremetal", - "toy-toy-mlirtest-ball_gemmini_preloaded_command_16x16-baremetal", - "toy-toy-ctest-tlb_bare_mode_test-baremetal", - "toy-toy-ctest-tlb_sv39_identity_test-baremetal", - "toy-toy-ctest-tlb_sfence_flush_test-baremetal", - "toy-toy-ctest-tlb_repeated_sfence_test-baremetal", - "toy-toy-ctest-tlb_sv39_nonidentity_test-baremetal", - "toy-toy-ctest-tlb_sv39_4k_alias_test-baremetal", - "toy-toy-ctest-tlb_sfence_remap_test-baremetal", -] diff --git a/examples/chips/toy/regression/batch/p2e/workloads-bare.toml b/examples/chips/toy/regression/batch/p2e/workloads-bare.toml deleted file mode 100644 index 59d07b1d1..000000000 --- a/examples/chips/toy/regression/batch/p2e/workloads-bare.toml +++ /dev/null @@ -1,26 +0,0 @@ -[workloads] -tests = [ - "toy-toy-ctest-gemmini_accumulated_16x16_test-baremetal.hex", - "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-baremetal.hex", - "toy-toy-ctest-alexnet_gemmini_tail_test-baremetal.hex", - "toy-toy-ctest-alexnet_gemmini_stride_test-baremetal.hex", - "toy-toy-ctest-alexnet_gemmini_round_test-baremetal.hex", - "toy-toy-ctest-mvin_mvout_test-baremetal.hex", - "toy-toy-ctest-mvin_mvout_stride_test-baremetal.hex", - "toy-toy-ctest-bdb_counter_start_stop_test-baremetal.hex", - "toy-toy-ctest-bdb_counter_read_active_test-baremetal.hex", - "toy-toy-ctest-bdb_counter_nested_pair_test-baremetal.hex", - "toy-toy-ctest-bdb_counter_multi_id_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_cisc_basic_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_cisc_atranspose_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_cisc_btranspose_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_cisc_shift_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_risc_atranspose_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_risc_btranspose_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_risc_abtranspose_test-baremetal.hex", - "toy-toy-ctest-gemmini_os_risc_shift_test-baremetal.hex", - "toy-toy-ctest-gemmini_ws_cisc_conv_test-baremetal.hex", - "toy-toy-ctest-gemmini_ws_risc_basic_test-baremetal.hex", - "toy-toy-ctest-mmio_minimal_test-baremetal.hex", - "toy-toy-ctest-mxfp4_e8m0_two_block_test-baremetal.hex", -] diff --git a/examples/chips/toy/regression/batch/verilator/workloads-bare.toml b/examples/chips/toy/regression/batch/verilator/workloads-bare.toml deleted file mode 100644 index d302c4692..000000000 --- a/examples/chips/toy/regression/batch/verilator/workloads-bare.toml +++ /dev/null @@ -1,39 +0,0 @@ -[workloads] -tests = [ - "toy-toy-ctest-rvv_kernel-baremetal", - "toy-toy-ctest-mvin_2d_pixel_stride_test-baremetal", - "toy-toy-ctest-gemmini_accumulated_16x16_test-baremetal", - "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-baremetal", - "toy-toy-ctest-alexnet_gemmini_round_test-baremetal", - "toy-toy-ctest-alexnet_gemmini_tail_test-baremetal", - "toy-toy-ctest-alexnet_gemmini_stride_test-baremetal", - "toy-toy-ctest-mvin_2d_test-baremetal", - "toy-toy-ctest-mvin_mvout_acc_test-baremetal", - "toy-toy-ctest-mvin_mvout_test-baremetal", - "toy-toy-ctest-mvin_mvout_unaligned_test-baremetal", - "toy-toy-ctest-mvin_unaligned_test-baremetal", - "toy-toy-ctest-bdb_counter_start_stop_test-baremetal", - "toy-toy-ctest-bdb_counter_read_active_test-baremetal", - "toy-toy-ctest-bdb_counter_nested_pair_test-baremetal", - "toy-toy-ctest-bdb_counter_multi_id_test-baremetal", - "toy-toy-mlirtest-ball_bdb_counter_start_read_stop-baremetal", - "toy-toy-ctest-gemmini_os_cisc_basic_test-baremetal", - "toy-toy-ctest-gemmini_os_cisc_atranspose_test-baremetal", - "toy-toy-ctest-gemmini_os_cisc_btranspose_test-baremetal", - "toy-toy-ctest-gemmini_os_cisc_shift_test-baremetal", - "toy-toy-ctest-gemmini_ws_cisc_conv_test-baremetal", - "toy-toy-ctest-mmio_minimal_test-baremetal", - "toy-toy-ctest-mxfp4_e8m0_two_block_test-baremetal", - "toy-toy-ctest-relu_i32_16x16_test-baremetal", - "toy-toy-mlirtest-ball_relu_i32_16row-baremetal", - "toy-toy-mlirtest-ball_mxfp4_unit_scale_2blocks-baremetal", - "toy-toy-mlirtest-ball_gemmini_preloaded_command_16x16-baremetal", - "toy-toy-ctest-tlb_bare_mode_test-baremetal", - "toy-toy-ctest-tlb_sv39_identity_test-baremetal", - "toy-toy-ctest-tlb_sfence_flush_test-baremetal", - "toy-toy-ctest-tlb_repeated_sfence_test-baremetal", - "toy-toy-ctest-tlb_sv39_nonidentity_test-baremetal", - "toy-toy-ctest-tlb_sv39_4k_alias_test-baremetal", - "toy-toy-ctest-tlb_sfence_remap_test-baremetal", - "toy-toy-ctest-mvin_mvout_stride_test-baremetal", -] diff --git a/examples/chips/toy/regression/batch/verilator/workloads-linux.toml b/examples/chips/toy/regression/batch/verilator/workloads-linux.toml deleted file mode 100644 index 6ae9a89f8..000000000 --- a/examples/chips/toy/regression/batch/verilator/workloads-linux.toml +++ /dev/null @@ -1,35 +0,0 @@ -[workloads] -tests = [ - "toy-toy-ctest-gemmini_accumulated_16x16_test-linux", - "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-linux", - "toy-toy-ctest-alexnet_gemmini_round_test-linux", - "toy-toy-ctest-alexnet_gemmini_tail_test-linux", - "toy-toy-ctest-alexnet_gemmini_stride_test-linux", - "toy-toy-ctest-mvin_2d_test-linux", - "toy-toy-ctest-mvin_mvout_acc_test-linux", - "toy-toy-ctest-mvin_mvout_test-linux", - "toy-toy-ctest-mvin_mvout_stride_test-linux", - "toy-toy-ctest-mvin_mvout_unaligned_test-linux", - "toy-toy-ctest-mvin_unaligned_test-linux", - "toy-toy-ctest-bdb_counter_start_stop_test-linux", - "toy-toy-ctest-bdb_counter_read_active_test-linux", - "toy-toy-ctest-bdb_counter_nested_pair_test-linux", - "toy-toy-ctest-bdb_counter_multi_id_test-linux", - "toy-toy-mlirtest-ball_bdb_counter_start_read_stop-linux", - "toy-toy-ctest-gemmini_os_cisc_basic_test-linux", - "toy-toy-ctest-gemmini_os_cisc_atranspose_test-linux", - "toy-toy-ctest-gemmini_os_cisc_btranspose_test-linux", - "toy-toy-ctest-gemmini_os_cisc_shift_test-linux", - "toy-toy-ctest-gemmini_os_risc_atranspose_test-linux", - "toy-toy-ctest-gemmini_os_risc_btranspose_test-linux", - "toy-toy-ctest-gemmini_os_risc_abtranspose_test-linux", - "toy-toy-ctest-gemmini_os_risc_shift_test-linux", - "toy-toy-ctest-gemmini_ws_cisc_conv_test-linux", - "toy-toy-ctest-gemmini_ws_risc_basic_test-linux", - "toy-toy-ctest-mmio_minimal_test-linux", - "toy-toy-ctest-mxfp4_e8m0_two_block_test-linux", - "toy-toy-ctest-relu_i32_16x16_test-linux", - "toy-toy-mlirtest-ball_relu_i32_16row-linux", - "toy-toy-mlirtest-ball_mxfp4_unit_scale_2blocks-linux", - "toy-toy-mlirtest-ball_gemmini_preloaded_command_16x16-linux", -] diff --git a/examples/chips/toy/regression/soc/README.md b/examples/chips/toy/regression/soc/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/chips/toy/workloads/CMakeLists.txt b/examples/chips/toy/workloads/CMakeLists.txt deleted file mode 100644 index 644aa54ae..000000000 --- a/examples/chips/toy/workloads/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -get_filename_component(BUCKYBALL_REPO_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../.. ABSOLUTE) - -add_custom_target(toy-ctests-build) -set(BUCKYBALL_CTEST_GROUP_TARGET toy-ctests-build) -set(BUCKYBALL_CTEST_TARGET toy) - -add_subdirectory(ctests) -buckyball_add_ball_workload_subdirs(ctests) - -add_custom_target(chip-ctests-build DEPENDS toy-ctests-build) - -add_custom_target(chip-workloads-build ALL - DEPENDS - chip-ctests-build - mlirtest-workloads-build - soctests-build - COMMENT "Building toy workloads" - VERBATIM) diff --git a/examples/chips/toy/workloads/ctests/CMakeLists.txt b/examples/chips/toy/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 8f2e8db5c..000000000 --- a/examples/chips/toy/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -add_buckyball_ctests( - mmio_minimal_test.c - scu_uart_rx_echo_test.c - alexnet_gemmini_tail_test.c - alexnet_gemmini_stride_test.c - alexnet_gemmini_round_test.c -) - -foreach(_id alexnet_gemmini_tail_test alexnet_gemmini_stride_test alexnet_gemmini_round_test) - buckyball_ctest_name(_test ${BUCKYBALL_CTEST_TARGET} ${_id}) - add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${_test}-baremetal APPEND - DEPENDS ${BUCKYBALL_REPO_ROOT}/examples/models/toy/alexnet/execute/conv.c - ${BUCKYBALL_REPO_ROOT}/examples/models/toy/alexnet/execute/conv.h) -endforeach() diff --git a/examples/chips/toy/workloads/ctests/alexnet_gemmini_round_test.c b/examples/chips/toy/workloads/ctests/alexnet_gemmini_round_test.c deleted file mode 100644 index 449dfc85d..000000000 --- a/examples/chips/toy/workloads/ctests/alexnet_gemmini_round_test.c +++ /dev/null @@ -1,22 +0,0 @@ -#include "../../../../models/toy/alexnet/execute/conv.c" -#include "buckyball.h" - -static float input[6] __attribute__((aligned(64))) = {0.125f, 0.375f, -0.125f, - -0.375f, 64.f, -64.f}; -static int8_t weights[6] __attribute__((aligned(64))) = {1, 2, 3, 4, 5, 6}; -static float bias[1], output[1]; - -int main(void) { - AlexnetF32_4 in = {input, input, 0, {1, 6, 1, 1}, {6, 1, 1, 1}}; - AlexnetI8_2 w = {weights, weights, 0, {6, 1}, {1, 1}}; - AlexnetF32_1 b = {bias, bias, 0, {1}, {1}}; - AlexnetF32_4 out = {output, output, 0, {1, 1, 1, 1}, {1, 1, 1, 1}}; - _mlir_ciface_gemmini_conv(&in, &w, &b, &out, 1, 1, 1, 0, 0.25f, 0.5f); - // RNE codes are {0, 2, 0, -2, 127, -128}; accumulation is -137. - if (output[0] != -17.125f) { - printf("AlexNet Gemmini rounding FAILED: %f\n", output[0]); - return 1; - } - printf("AlexNet Gemmini rounding PASSED\n"); - return 0; -} diff --git a/examples/chips/toy/workloads/ctests/alexnet_gemmini_stride_test.c b/examples/chips/toy/workloads/ctests/alexnet_gemmini_stride_test.c deleted file mode 100644 index 57d83b07f..000000000 --- a/examples/chips/toy/workloads/ctests/alexnet_gemmini_stride_test.c +++ /dev/null @@ -1,37 +0,0 @@ -#include "../../../../models/toy/alexnet/execute/conv.c" -#include "buckyball.h" - -static float input[4 * 4 * 4] __attribute__((aligned(64))); -static int8_t weights[16 * 32] __attribute__((aligned(64))); -static float bias[32], output[32 * 3 * 3]; - -int main(void) { - for (int i = 0; i < 64; ++i) - input[i] = (float)(i % 9 - 4) / 4; - for (int i = 0; i < 512; ++i) - weights[i] = i % 5 - 2; - for (int i = 0; i < 32; ++i) - bias[i] = (float)i / 2; - AlexnetF32_4 in = {input, input, 0, {1, 4, 4, 4}, {64, 16, 4, 1}}; - AlexnetI8_2 w = {weights, weights, 0, {16, 32}, {32, 1}}; - AlexnetF32_1 b = {bias, bias, 0, {32}, {1}}; - AlexnetF32_4 out = {output, output, 0, {1, 32, 3, 3}, {288, 9, 3, 1}}; - _mlir_ciface_gemmini_conv(&in, &w, &b, &out, 2, 2, 1, 0, 0.25f, 0.125f); - for (int n = 0; n < 32; ++n) - for (int y = 0; y < 3; ++y) - for (int x = 0; x < 3; ++x) { - int sum = 0; - for (int ky = 0; ky < 2; ++ky) - for (int kx = 0; kx < 2; ++kx) - for (int c = 0; c < 4; ++c) { - int code = (c * 16 + (y + ky) * 4 + x + kx) % 9 - 4; - sum += code * weights[((ky * 2 + kx) * 4 + c) * 32 + n]; - } - if (output[n * 9 + y * 3 + x] != (float)sum * 0.03125f + bias[n]) { - printf("AlexNet Gemmini stride FAILED n=%d y=%d x=%d\n", n, y, x); - return 1; - } - } - printf("AlexNet Gemmini stride PASSED\n"); - return 0; -} diff --git a/examples/chips/toy/workloads/ctests/alexnet_gemmini_tail_test.c b/examples/chips/toy/workloads/ctests/alexnet_gemmini_tail_test.c deleted file mode 100644 index a7561055b..000000000 --- a/examples/chips/toy/workloads/ctests/alexnet_gemmini_tail_test.c +++ /dev/null @@ -1,41 +0,0 @@ -#include "../../../../models/toy/alexnet/execute/conv.c" -#include "buckyball.h" - -static float input[3 * 5 * 5] __attribute__((aligned(64))); -static int8_t weights[27 * 5] __attribute__((aligned(64))); -static float bias[5], output[5 * 3 * 3]; - -int main(void) { - for (int i = 0; i < 75; ++i) - input[i] = (float)(i % 9 - 4) / 4; - for (int i = 0; i < 135; ++i) - weights[i] = i % 5 - 2; - for (int i = 0; i < 5; ++i) - bias[i] = (float)i / 2; - AlexnetF32_4 in = {input, input, 0, {1, 3, 5, 5}, {75, 25, 5, 1}}; - AlexnetI8_2 w = {weights, weights, 0, {27, 5}, {5, 1}}; - AlexnetF32_1 b = {bias, bias, 0, {5}, {1}}; - AlexnetF32_4 out = {output, output, 0, {1, 5, 3, 3}, {45, 9, 3, 1}}; - _mlir_ciface_gemmini_conv(&in, &w, &b, &out, 3, 3, 1, 0, 0.25f, 0.125f); - for (int n = 0; n < 5; ++n) - for (int y = 0; y < 3; ++y) - for (int x = 0; x < 3; ++x) { - int sum = 0; - for (int ky = 0; ky < 3; ++ky) - for (int kx = 0; kx < 3; ++kx) - for (int c = 0; c < 3; ++c) { - int code = (c * 25 + (y + ky) * 5 + x + kx) % 9 - 4; - sum += code * weights[((ky * 3 + kx) * 3 + c) * 5 + n]; - } - float expected = (float)sum * 0.03125f + bias[n]; - if (output[n * 9 + y * 3 + x] != expected) { - printf("AlexNet Gemmini tail FAILED n=%d y=%d x=%d got32=%ld " - "expected32=%ld\n", - n, y, x, (long)(output[n * 9 + y * 3 + x] * 32), - (long)(expected * 32)); - return 1; - } - } - printf("AlexNet Gemmini tail PASSED\n"); - return 0; -} diff --git a/examples/chips/toy/workloads/mlir_tests/CMakeLists.txt b/examples/chips/toy/workloads/mlir_tests/CMakeLists.txt deleted file mode 100644 index 478a635ed..000000000 --- a/examples/chips/toy/workloads/mlir_tests/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_custom_target(toy-mlir-tests-build) - -add_subdirectory(linalg) - -add_dependencies(toy-mlir-tests-build - linalg-mlir-tests-build) - -add_dependencies(chip-mlirtests-build toy-mlir-tests-build) diff --git a/examples/chips/toy/workloads/mlir_tests/linalg/CMakeLists.txt b/examples/chips/toy/workloads/mlir_tests/linalg/CMakeLists.txt deleted file mode 100644 index dafa9d014..000000000 --- a/examples/chips/toy/workloads/mlir_tests/linalg/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -set(BUCKYBALL_MLIR_GROUP_TARGET linalg-mlir-tests-build) -set(BUCKYBALL_MLIR_TEST_PREFIX linalg) - -add_custom_target(linalg-mlir-tests-build) - -set(TOY_LINALG_FUNCTIONAL_PASSES - -convert-linalg-to-tile - -convert-tile-to-buckyball - -lower-buckyball-to-bank-ssa - -assign-physical-banks - ${BUCKYBALL_LOWER_BANK_SSA_TO_INTRINSICS} - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - -foreach(test conv2d_nhwc_hwcf conv2d_nhwc_fhwc conv2d_nchw_fchw) - add_buckyball_mlir_test(${test} - TARGET toy - PASSES ${TOY_LINALG_FUNCTIONAL_PASSES}) -endforeach() diff --git a/examples/components/balls/build.sc b/examples/components/balls/build.sc new file mode 100644 index 000000000..f1e3aeac4 --- /dev/null +++ b/examples/components/balls/build.sc @@ -0,0 +1,12 @@ +import mill._ + +def sources = T.sources { + os.list(millSourcePath).filter(os.isDir).flatMap { root => + val source = root / "arch" + val configs = root / "configs" + (if (os.exists(source)) + os.walk(source).filter(p => p.ext == "scala" && !p.segments.exists(Set("out", "generated", "target", "build"))) + else Seq.empty) ++ + (if (os.exists(configs)) os.walk(configs).filter(_.ext == "scala") else Seq.empty) + }.map(PathRef(_)) +} diff --git a/examples/components/balls/f32add/arch/F32AddBall.scala b/examples/components/balls/f32add/arch/F32AddBall.scala new file mode 100644 index 000000000..17da110ea --- /dev/null +++ b/examples/components/balls/f32add/arch/F32AddBall.scala @@ -0,0 +1,139 @@ +package examples.balls.f32add + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.frontend.configs.FrontendParam +import buckyball.balls.f32add.{Params => ParametersProto} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.acc.balldomain.ball.blink.{BallCommand, BlinkIO, HasBlink} + +@instantiable +class F32AddBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + require(ball.inBW == 2 && ball.outBW == 1) + require( + mp.bankWidth == 128 && mp.bankMaskLen == 16 + ) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val idle :: channels :: reading :: calculate :: writing :: acknowledge :: complete :: Nil = Enum(7) + val state = RegInit(idle) + + val issue = Reg(new BallCommand(fp, mp)) + + val row = RegInit(0.U(16.W)) + val requested = RegInit(VecInit(Seq.fill(2)(false.B))) + val received = RegInit(VecInit(Seq.fill(2)(false.B))) + val words = Reg(Vec(2, UInt(128.W))) + val output = Reg(UInt(128.W)) + val first = issue.rs2(5) + val sourceGroup = issue.rs2(4, 0) + val add = Instantiate(new F32AddRow) + add.io.source := words(0) + add.io.accumulator := Mux(first, 0.U(128.W), words(1)) + + blink.cmdReq.ready := state === idle + blink.cmdResp.valid := state === complete + blink.cmdResp.bits.rob_id := issue.rob_id + blink.status.idle := state === idle + blink.status.running := state =/= idle + + for (port <- 0 until 2) { + val read = blink.bankRead(port) + val active = if (port == 0) true.B else !first + read.rob_id := issue.rob_id + read.bank_id := (if (port == 0) issue.op1_bank else issue.op2_bank) + read.group_id := (if (port == 0) issue.op1_group + sourceGroup else issue.op2_group) + read.io.req.valid := state === reading && active && !requested(port) + read.io.req.bits.addr := row + read.io.resp.ready := state === reading && active && requested(port) && !received(port) + when(read.io.req.fire)(requested(port) := true.B) + when(read.io.resp.fire) { + words(port) := read.io.resp.bits.data + received(port) := true.B + } + } + + val write = blink.bankWrite(0) + write.rob_id := issue.rob_id + write.bank_id := issue.wr_bank + write.group_id := issue.wr_group + write.io.req.valid := state === writing + write.io.req.bits.addr := row + write.io.req.bits.data := output + write.io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) + write.io.resp.ready := state === acknowledge + + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + assert(cmd.rs2(63, 6) === 0.U, "F32ADD reserved rs2 bits") + assert( + cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, + "F32ADD banks must differ" + ) + assert( + cmd.rs2(4, 0) < cmd.op1_col && cmd.op2_col === 1.U && cmd.wr_col === 1.U, + "F32ADD source group must exist and accumulators must have one group" + ) + for (bank <- Seq(cmd.op1_bank, cmd.op2_bank, cmd.wr_bank)) { + val shared = bank >= fp.shared_bank_id_base.U + assert( + !shared || mp.sharedEnable.B, + "F32ADD shared memory is disabled" + ) + val depth = Mux(shared, mp.sharedBankEntries.U, mp.bankEntries.U) + assert(cmd.iter =/= 0.U && cmd.iter <= depth, "F32ADD row span exceeds bank") + } + issue := blink.cmdReq.bits + row := 0.U + requested.foreach(_ := false.B) + received.foreach(_ := false.B) + state := channels + } + when(state === channels)(state := reading) + when(state === reading && received(0) && (first || received(1)))(state := calculate) + when(state === calculate) { + output := add.io.sum + state := writing + } + when(write.io.req.fire)(state := acknowledge) + when(write.io.resp.fire) { + assert(write.io.resp.bits.ok, "F32ADD bank write failed") + when(row +& 1.U === issue.iter) { + state := complete + }.otherwise { + row := row + 1.U + requested.foreach(_ := false.B) + received.foreach(_ := false.B) + state := reading + } + } + when(blink.cmdResp.fire)(state := idle) +} + +object F32AddBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new F32AddBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/balls/f32add/arch/src/main/scala/F32AddRow.scala b/examples/components/balls/f32add/arch/F32AddRow.scala similarity index 87% rename from examples/balls/f32add/arch/src/main/scala/F32AddRow.scala rename to examples/components/balls/f32add/arch/F32AddRow.scala index a582daec9..b1a762835 100644 --- a/examples/balls/f32add/arch/src/main/scala/F32AddRow.scala +++ b/examples/components/balls/f32add/arch/F32AddRow.scala @@ -1,5 +1,7 @@ package examples.balls.f32add +import chisel3.experimental.hierarchy.{Instance, Instantiate} + import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} @@ -9,7 +11,8 @@ import hardfloat.consts.{round_near_even, tininess_afterRounding} @instantiable class F32AddRow extends Module { - @public val io = IO(new Bundle { + @public + val io = IO(new Bundle { val source = Input(UInt(128.W)) val accumulator = Input(UInt(128.W)) val sum = Output(UInt(128.W)) @@ -17,7 +20,7 @@ class F32AddRow extends Module { val result = Wire(Vec(4, UInt(32.W))) for (lane <- 0 until 4) { - val add = Module(new AddRecFN(8, 24)) + val add = Instantiate(new AddRecFN(8, 24)) add.io.subOp := false.B add.io.a := recFNFromFN(8, 24, io.accumulator(32 * lane + 31, 32 * lane)) add.io.b := recFNFromFN(8, 24, io.source(32 * lane + 31, 32 * lane)) diff --git a/arch/src/main/scala/framework/mem-core/chi/src/main/resources/README.md b/examples/components/balls/f32add/configs/default.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/chi/src/main/resources/README.md rename to examples/components/balls/f32add/configs/default.toml diff --git a/examples/components/balls/f32add/configs/f32add_params.proto b/examples/components/balls/f32add/configs/f32add_params.proto new file mode 100644 index 000000000..150903ad5 --- /dev/null +++ b/examples/components/balls/f32add/configs/f32add_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.f32add; +option java_package = "buckyball.balls.f32add"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/balls/f32add/emu/src/add.rs b/examples/components/balls/f32add/emu/src/add.rs similarity index 92% rename from examples/balls/f32add/emu/src/add.rs rename to examples/components/balls/f32add/emu/src/add.rs index f3f252e5a..2da22b6bd 100644 --- a/examples/balls/f32add/emu/src/add.rs +++ b/examples/components/balls/f32add/emu/src/add.rs @@ -1,5 +1,5 @@ -use crate::inst::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use crate::inst::instruction::ExecContext; +use bebop_bemu::thread::instructions::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use bebop_bemu::ball::ExecContext; pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let a = rs1_b0(xs1); diff --git a/examples/components/balls/f32add/emu/src/dispatch.rs b/examples/components/balls/f32add/emu/src/dispatch.rs new file mode 100644 index 000000000..72bf0db76 --- /dev/null +++ b/examples/components/balls/f32add/emu/src/dispatch.rs @@ -0,0 +1,26 @@ +use bebop_bemu::ball::ExecContext; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != "examples.balls.f32add.F32AddBall" { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("F32ADD") => Some(super::add::execute(xs1, xs2, ctx)), + _ => None, + } +} +pub fn cycles_after_issue(ball_class: &str, funct: u32, _xs1: u64, _xs2: u64) -> Option { + if ball_class != "examples.balls.f32add.F32AddBall" { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("F32ADD") => Some(1), + _ => None, + } +} diff --git a/examples/components/balls/f32add/emu/src/lib.rs b/examples/components/balls/f32add/emu/src/lib.rs new file mode 100644 index 000000000..e75417d6e --- /dev/null +++ b/examples/components/balls/f32add/emu/src/lib.rs @@ -0,0 +1,8 @@ +mod add; +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/components/balls/f32add/stack/src/Lowering.cpp b/examples/components/balls/f32add/stack/src/Lowering.cpp new file mode 100644 index 000000000..332e986f9 --- /dev/null +++ b/examples/components/balls/f32add/stack/src/Lowering.cpp @@ -0,0 +1,55 @@ +//===- Lowering.cpp - F32AddBall in lower-buckyball -----------------------===// +// +// add_f32: C = B + A[group] over `rows` rows of four f32, RNE with canonical +// NaN. A may have several groups; B and C have one. With `first` the sum +// starts from +0 instead of reading B, which still names an allocated bank. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballOps.h" +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class AddF32Lowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &rewriter) const override { + if (op.getKind() != "add_f32") + return failure(); + Location loc = op.getLoc(); + int64_t rows = cast(op->getAttr("rows")).getInt(); + int64_t group = cast(op->getAttr("group")).getInt(); + bool first = cast(op->getAttr("first")).getValue(); + if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || + group < 0 || group > 31 || op.getSources().size() != 2 || + op.getSources()[0] == op.getSources()[1] || + llvm::is_contained(op.getSources(), op.getDestination())) + return op.emitOpError( + "add_f32 requires distinct banks, rows within a bank and a group " + "in [0, 31]"); + mlir::buddy::emitInstruction( + rewriter, loc, buckyball_target::getBuckyballFunct7("F32ADD"), + mlir::buddy::constantI64(rewriter, loc, + mlir::buddy::bankFields(op.getSources(), + op.getDestination(), + false, rows)), + mlir::buddy::constantI64(rewriter, loc, + mlir::buddy::field(group, 0, 4) | + mlir::buddy::field(first, 5, 5))); + rewriter.eraseOp(op); + return success(); + } +}; + +void populate(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} + +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallLowering(populate), true); +} // namespace diff --git a/examples/components/balls/f32add/workloads/ctests/CMakeLists.txt b/examples/components/balls/f32add/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..d51d3c3a5 --- /dev/null +++ b/examples/components/balls/f32add/workloads/ctests/CMakeLists.txt @@ -0,0 +1 @@ +add_buckyball_ctest(f32add_groups.c) diff --git a/examples/balls/f32add/workloads/ctests/f32add_groups.c b/examples/components/balls/f32add/workloads/ctests/f32add_groups.c similarity index 100% rename from examples/balls/f32add/workloads/ctests/f32add_groups.c rename to examples/components/balls/f32add/workloads/ctests/f32add_groups.c diff --git a/examples/balls/f32add/workloads/isa/f32add.h b/examples/components/balls/f32add/workloads/isa/f32add.h similarity index 100% rename from examples/balls/f32add/workloads/isa/f32add.h rename to examples/components/balls/f32add/workloads/isa/f32add.h diff --git a/examples/balls/f32add/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/f32add/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/f32add/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/f32add/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/f32add/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/f32add/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/f32add/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/f32add/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir b/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir new file mode 100644 index 000000000..16aab886f --- /dev/null +++ b/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir @@ -0,0 +1,16 @@ +// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @reduce_groups +// CHECK-DAG: %[[F76:.*]] = llvm.mlir.constant(76 : i32) : i32 +// CHECK-DAG: %[[AC:[^ ]+]] = arith.constant 2183169056 : i64 +// CHECK-DAG: %[[CA:[^ ]+]] = arith.constant 2182121504 : i64 +// CHECK-DAG: %[[FIRST:[^ ]+]] = arith.constant 32 : i64 +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[AC]], %[[FIRST]], %[[F76]]) +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[CA]], {{.*}}, %[[F76]]) +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[AC]], {{.*}}, %[[F76]]) +func.func @reduce_groups() -> i64 attributes {llvm.emit_c_interface} { + %output = arith.constant 34 : i64 + buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 2 : i64, group = 0 : i64, first = true} + buckyball.custom "add_f32" <{sources = array, destination = 33 : i64}> {rows = 2 : i64, group = 1 : i64, first = false} + buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 2 : i64, group = 2 : i64, first = false} + return %output : i64 +} diff --git a/examples/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp b/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp similarity index 100% rename from examples/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp rename to examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp diff --git a/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir b/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir new file mode 100644 index 000000000..98a8b0c4e --- /dev/null +++ b/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir @@ -0,0 +1,28 @@ +// RUN: buddy-opt %s -split-input-file -lower-buckyball -verify-diagnostics +func.func @zero_rows() { + // expected-error @+2 {{add_f32 requires distinct banks, rows within a bank and a group in [0, 31]}} + // expected-error @+1 {{has no lowering on this core}} + buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 0 : i64, group = 0 : i64, first = true} + return +} +// ----- +func.func @wide_group() { + // expected-error @+2 {{add_f32 requires distinct banks, rows within a bank and a group in [0, 31]}} + // expected-error @+1 {{has no lowering on this core}} + buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 1 : i64, group = 32 : i64, first = false} + return +} +// ----- +func.func @wide_rows() { + // expected-error @+2 {{add_f32 requires distinct banks, rows within a bank and a group in [0, 31]}} + // expected-error @+1 {{has no lowering on this core}} + buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 17179869184 : i64, group = 0 : i64, first = false} + return +} +// ----- +func.func @equal_banks() { + // expected-error @+2 {{add_f32 requires distinct banks, rows within a bank and a group in [0, 31]}} + // expected-error @+1 {{has no lowering on this core}} + buckyball.custom "add_f32" <{sources = array, destination = 32 : i64}> {rows = 1 : i64, group = 0 : i64, first = true} + return +} diff --git a/examples/balls/f32add/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/f32add/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/f32add/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/components/balls/f32add/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir b/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir new file mode 100644 index 000000000..4c6f162ca --- /dev/null +++ b/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir @@ -0,0 +1,23 @@ +// The reduce core has four banks: groups 0 and 1 share a two-group source +// bank, group 2 then reuses its banks. The first add_f32 starts from +0 +// instead of the poison in the accumulator. +func.func @reduce_groups(%pair: memref<2x2x4xi32>, %last: memref<2x4xi32>, + %poison: memref<2x4xi32>, %output: memref<2x4xi32>) + attributes {llvm.emit_c_interface} { + %source = buckyball.bank_alloc <{row = 1 : i64, col = 2 : i64}> + %accumulator = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %sum = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %groups = buckyball.bank_mvin %pair, %source <{depth = 2 : i64, stride = 1 : i64}> : memref<2x2x4xi32> + %poisoned = buckyball.bank_mvin %poison, %accumulator <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> + %first = buckyball.bank_custom "add_f32" (%groups, %poisoned) -> %sum {rows = 2 : i64, group = 0 : i64, first = true} + %second = buckyball.bank_custom "add_f32" (%groups, %first) -> %poisoned {rows = 2 : i64, group = 1 : i64, first = false} + buckyball.bank_release %groups + %single = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %group2 = buckyball.bank_mvin %last, %single <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> + %third = buckyball.bank_custom "add_f32" (%group2, %second) -> %first {rows = 2 : i64, group = 0 : i64, first = false} + %stored = buckyball.bank_mvout %output, %third <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> + buckyball.bank_release %group2 + buckyball.bank_release %second + buckyball.bank_release %stored + return +} diff --git a/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp b/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp new file mode 100644 index 000000000..719887296 --- /dev/null +++ b/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp @@ -0,0 +1,55 @@ +#include +#include +#include + +using Rows = StridedMemRefType; +extern "C" void _mlir_ciface_reduce_groups(StridedMemRefType *, + Rows *, Rows *, Rows *); + +static int checkF32Add() { + alignas(64) static const uint32_t input[3][8] = { + {0x3f800000, 0xc0200000, 0x7f800000, 0xff800000, 0, 0x80000000, + 0x7f7fffff, 1}, + {0x40000000, 0x40400000, 0xff800000, 0x7f800000, 0x80000000, 0, + 0xff7fffff, 1}, + {0xbf800000, 0x3f000000, 0x40a00000, 0xc0a00000, 0, 0x80000000, + 0x3f800000, 0x80000001}}; + alignas(64) static const uint32_t poison[8] = { + 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001, + 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001}; + alignas(64) static uint32_t output[8]; + // A whole-bank mvin reads row r of group g at (r * 2 + g) * 16 bytes. + alignas(64) static uint32_t pair[2][2][4]; + for (unsigned group = 0; group < 2; ++group) + for (unsigned i = 0; i < 8; ++i) + pair[i / 4][group][i % 4] = input[group][i]; + StridedMemRefType pairRef = { + &pair[0][0][0], &pair[0][0][0], 0, {2, 2, 4}, {8, 4, 1}}; + Rows lastRef = { + (uint32_t *)input[2], (uint32_t *)input[2], 0, {2, 4}, {4, 1}}; + Rows poisonRef = {(uint32_t *)poison, (uint32_t *)poison, 0, {2, 4}, {4, 1}}; + Rows outputRef = {output, output, 0, {2, 4}, {4, 1}}; + _mlir_ciface_reduce_groups(&pairRef, &lastRef, &poisonRef, &outputRef); + for (unsigned i = 0; i < 8; ++i) { + union { + uint32_t bits; + float value; + } expected = {0}, source; + for (unsigned group = 0; group < 3; ++group) { + source.bits = input[group][i]; + asm volatile("fadd.s %0, %1, %2, rne" + : "=f"(expected.value) + : "f"(expected.value), "f"(source.value)); + } + if (output[i] != expected.bits) + return 2; + } + return 0; +} + +int main() { + int result = checkF32Add(); + if (!result) + puts("F32ADD Bank MLIR grouped/RNE/NaN/first/SSA output PASS"); + return result; +} diff --git a/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir b/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir new file mode 100644 index 000000000..6bf585533 --- /dev/null +++ b/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir @@ -0,0 +1,40 @@ +// RUN: buddy-opt %s -split-input-file -assign-physical-banks -lower-buckyball -verify-diagnostics +func.func @same_bank(%input: memref<1x4xi32>) { + %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %la = buckyball.bank_mvin %input, %a <{depth = 1 : i64, stride = 1 : i64}> : memref<1x4xi32> + // expected-error @+2 {{add_f32 requires distinct banks, rows within a bank and a group in [0, 31]}} + // expected-error @+1 {{has no lowering on this core}} + %out = buckyball.bank_custom "add_f32" (%la, %b) -> %b {rows = 1 : i64, group = 0 : i64, first = true} + buckyball.bank_release %la + buckyball.bank_release %out + return +} +// ----- +func.func @zero_rows(%input: memref<1x4xi32>) { + %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %c = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %la = buckyball.bank_mvin %input, %a <{depth = 1 : i64, stride = 1 : i64}> : memref<1x4xi32> + // expected-error @+2 {{add_f32 requires distinct banks, rows within a bank and a group in [0, 31]}} + // expected-error @+1 {{has no lowering on this core}} + %out = buckyball.bank_custom "add_f32" (%la, %b) -> %c {rows = 0 : i64, group = 0 : i64, first = true} + buckyball.bank_release %la + buckyball.bank_release %b + buckyball.bank_release %out + return +} +// ----- +func.func @negative_group(%input: memref<1x4xi32>) { + %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %c = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %la = buckyball.bank_mvin %input, %a <{depth = 1 : i64, stride = 1 : i64}> : memref<1x4xi32> + // expected-error @+2 {{add_f32 requires distinct banks, rows within a bank and a group in [0, 31]}} + // expected-error @+1 {{has no lowering on this core}} + %out = buckyball.bank_custom "add_f32" (%la, %b) -> %c {rows = 1 : i64, group = -1 : i64, first = false} + buckyball.bank_release %la + buckyball.bank_release %b + buckyball.bank_release %out + return +} diff --git a/examples/components/balls/gemmini/arch/GemminiBall.scala b/examples/components/balls/gemmini/arch/GemminiBall.scala new file mode 100644 index 000000000..b5ceea348 --- /dev/null +++ b/examples/components/balls/gemmini/arch/GemminiBall.scala @@ -0,0 +1,87 @@ +package examples.balls.gemmini + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import framework.acc.balldomain.sub_rob.{Expansion, HasExpansion} +import buckyball.balls.gemmini.{Params => ParametersProto} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.gemmini.configs.GemminiBallParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import chisel3.util._ +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class GemminiBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink + with HasExpansion { + val ball = bp.balls.find(_.id == id).get + + val ballParam = GemminiBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + require( + mp.bankEntries <= 1024, + "GemminiBall rs2 row-base fields support at most 1024 bank entries" + ) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + @public + val expansion = IO(new Expansion(fp, mp, bp)) + + val loop: Instance[Loop] = Instantiate(new Loop(fp, mp, bp, id)) + expansion <> loop.expansion + + val execution: Instance[GemminiExCtrl] = + Instantiate(new GemminiExCtrl(fp, mp, bp, ball, ballParam)) + val robId = RegInit(0.U(log2Up(fp.rob_entries).W)) + when(blink.cmdReq.fire)(robId := blink.cmdReq.bits.rob_id) + + execution.exio.cmdReq <> blink.cmdReq + blink.cmdResp <> execution.exio.cmdResp + + for (port <- 0 until ball.inBW) { + blink.bankRead(port).io.req <> execution.exio.bankReadReq(port) + execution.exio.bankReadResp(port) <> blink.bankRead(port).io.resp + blink.bankRead(port).rob_id := robId + blink.bankRead(port).group_id := 0.U + } + blink.bankRead(0).bank_id := execution.exio.op1_bank_o + blink.bankRead(0).group_id := execution.exio.op1_group_o + if (ball.inBW > 1) { + blink.bankRead(1).bank_id := execution.exio.op2_bank_o + blink.bankRead(1).group_id := execution.exio.op2_group_o + } + + for (port <- 0 until ball.outBW) { + blink.bankWrite(port).io <> execution.exio.bankWrite(port) + blink.bankWrite(port).bank_id := execution.exio.wr_bank_o + blink.bankWrite(port).rob_id := robId + blink.bankWrite(port).group_id := execution.exio.wr_group_o + port.U + } + + blink.status <> execution.exio.status +} + +object GemminiBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = { + val ball = Instantiate(new GemminiBall(fp, mp, bp, id)) + BallPorts(ball.blink, Some(ball.expansion)) + } + +} diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrl.scala b/examples/components/balls/gemmini/arch/GemminiExCtrl.scala new file mode 100644 index 000000000..d5adba416 --- /dev/null +++ b/examples/components/balls/gemmini/arch/GemminiExCtrl.scala @@ -0,0 +1,40 @@ +package examples.balls.gemmini + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.gemmini.configs.GemminiBallParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class GemminiExCtrl( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: GemminiBallParam) + extends Module + with GemminiExCtrlDefs + with GemminiExCtrlDefaults + with GemminiExCtrlCmdStates + with GemminiExCtrlPreloadStates + with GemminiExCtrlComputeReadState + with GemminiExCtrlComputeFeedState + with GemminiExCtrlStoreOps + with GemminiExCtrlAccumulateState + with GemminiExCtrlFsm { + def commandDataBits: Int = frontendParam.commandDataBits + @public + val exio = io + + applyDefaults() + runFsm() + + io.status.idle := state === sIdle + io.status.running := state =/= sIdle +} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlAccumulateState.scala b/examples/components/balls/gemmini/arch/GemminiExCtrlAccumulateState.scala similarity index 82% rename from examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlAccumulateState.scala rename to examples/components/balls/gemmini/arch/GemminiExCtrlAccumulateState.scala index f6ae22c8c..57919e955 100644 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlAccumulateState.scala +++ b/examples/components/balls/gemmini/arch/GemminiExCtrlAccumulateState.scala @@ -1,5 +1,9 @@ package examples.balls.gemmini +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ @@ -15,7 +19,7 @@ trait GemminiExCtrlAccumulateState { this: GemminiExCtrl => }.otherwise { rdQueue0.io.deq.ready := true.B when(rdQueue0.io.deq.fire) { - val elemsPerPort = b.memDomain.bankWidth / config.accWidth + val elemsPerPort = memParam.bankWidth / config.accWidth val previous = rdQueue0.io.deq.bits.data.asTypeOf(Vec(elemsPerPort, accType)) for ((value, index) <- outBuf(store_row_cnt).flatten.zipWithIndex) { when(acc_read_group === (index / elemsPerPort).U) { diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrlCmdStates.scala b/examples/components/balls/gemmini/arch/GemminiExCtrlCmdStates.scala new file mode 100644 index 000000000..89bd21e07 --- /dev/null +++ b/examples/components/balls/gemmini/arch/GemminiExCtrlCmdStates.scala @@ -0,0 +1,110 @@ +package examples.balls.gemmini + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import gemmini._ + +/** Sub-command encoding within the special field */ +object GemminiSubCmd { + val CONFIG = 0.U(4.W) + val PRELOAD = 1.U(4.W) + val COMPUTE_PRELOADED = 2.U(4.W) + val COMPUTE_ACCUMULATED = 3.U(4.W) + val FLUSH = 4.U(4.W) +} + +trait GemminiExCtrlCmdStates { this: GemminiExCtrl => + + protected def handleIdleState(): Unit = { + io.cmdReq.ready := true.B + when(io.cmdReq.fire) { + rob_id_reg := io.cmdReq.bits.rob_id + op1_bank := io.cmdReq.bits.op1_bank + op2_bank := io.cmdReq.bits.op2_bank + wr_bank := io.cmdReq.bits.wr_bank + wr_read_bank := io.cmdReq.bits.wr_read_bank + op1_group := io.cmdReq.bits.op1_group + op2_group := io.cmdReq.bits.op2_group + wr_group := io.cmdReq.bits.wr_group + op1_base := io.cmdReq.bits.rs2(15, 6) + op2_base := io.cmdReq.bits.rs2(25, 16) + wr_base := io.cmdReq.bits.rs2(35, 26) + total_rows := Mux( + io.cmdReq.bits.iter === 0.U, + DIM.U, + io.cmdReq.bits.iter + ) + zero_op2 := io.cmdReq.bits.rs2(4) + zero_op1_tail := io.cmdReq.bits.rs2(5) + accumulated_cmd := sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED + reset_acc_mesh := sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED + acc_read_group := 0.U + acc_read_pending := false.B + + when(sub_cmd === GemminiSubCmd.CONFIG) { + cfg_dataflow := io.cmdReq.bits.rs2(4) + cfg_a_transpose := io.cmdReq.bits.rs2(7) + cfg_bd_transpose := io.cmdReq.bits.rs2(8) + cfg_in_shift := io.cmdReq.bits.rs2( + log2Up(config.accWidth) + 8, + 9 + ) + state := sCommit + }.elsewhen(sub_cmd === GemminiSubCmd.PRELOAD) { + read_row_cnt := 0.U + feed_row_cnt := 0.U + req_sent := false.B + xpose_ready := false.B + xpose_row_cnt := 0.U + state := sPreloadRead + }.elsewhen( + sub_cmd === GemminiSubCmd.COMPUTE_PRELOADED || sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED + ) { + read_row_cnt := 0.U + feed_row_cnt := 0.U + outBufRows := 0.U + outBufCollected := 0.U + req_sent := false.B + xpose_ready := false.B + xpose_row_cnt := 0.U + read_done.foreach(_ := false.B) + state := Mux(sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED, sPreloadFeed, sComputeRead) + }.elsewhen(sub_cmd === GemminiSubCmd.FLUSH) { + state := sFlush + } + } + } + + protected def handleCommitState(): Unit = { + io.cmdResp.valid := true.B + when(io.cmdResp.fire) { + state := sIdle + } + } + + protected def handleFlushState(): Unit = { + mesh.io.req.valid := true.B + mesh.io.req.bits.flush := 2.U + mesh.io.req.bits.total_rows := DIM.U + mesh.io.req.bits.tag.rob := robIdAsTag8(rob_id_reg) + + mesh.io.a.valid := true.B + mesh.io.a.bits := 0.U.asTypeOf(mesh.io.a.bits) + mesh.io.b.valid := true.B + mesh.io.b.bits := 0.U.asTypeOf(mesh.io.b.bits) + mesh.io.d.valid := true.B + mesh.io.d.bits := 0.U.asTypeOf(mesh.io.d.bits) + + when(mesh.io.req.ready) { + io.cmdResp.valid := true.B + when(io.cmdResp.fire) { + state := sIdle + } + } + } + +} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlComputeFeedState.scala b/examples/components/balls/gemmini/arch/GemminiExCtrlComputeFeedState.scala similarity index 92% rename from examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlComputeFeedState.scala rename to examples/components/balls/gemmini/arch/GemminiExCtrlComputeFeedState.scala index beaa64853..9fdc73f55 100644 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlComputeFeedState.scala +++ b/examples/components/balls/gemmini/arch/GemminiExCtrlComputeFeedState.scala @@ -1,5 +1,9 @@ package examples.balls.gemmini +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ import gemmini._ @@ -71,7 +75,7 @@ trait GemminiExCtrlComputeFeedState { this: GemminiExCtrl => mesh.io.a.valid := true.B mesh.io.a.bits := Mux( op1Zero, - 0.U.asTypeOf(mesh.A_TYPE), + 0.U.asTypeOf(mesh.io.a.bits), VecInit(a_row.grouped(config.tileRows).map(g => VecInit(g)).toSeq) ) when(execution_dataflow === Dataflow.OS.id.U) { @@ -81,15 +85,15 @@ trait GemminiExCtrlComputeFeedState { this: GemminiExCtrl => x_row.grouped(config.tileColumns).map(g => VecInit(g)).toSeq ) mesh.io.d.valid := true.B - mesh.io.d.bits := 0.U.asTypeOf(mesh.D_TYPE) + mesh.io.d.bits := 0.U.asTypeOf(mesh.io.d.bits) }.otherwise { // WS: stream A/D, B comes from preloaded weights mesh.io.b.valid := true.B - mesh.io.b.bits := 0.U.asTypeOf(mesh.B_TYPE) + mesh.io.b.bits := 0.U.asTypeOf(mesh.io.b.bits) mesh.io.d.valid := true.B mesh.io.d.bits := Mux( zero_op2, - 0.U.asTypeOf(mesh.D_TYPE), + 0.U.asTypeOf(mesh.io.d.bits), VecInit( x_row.grouped(config.tileColumns).map(g => VecInit(g)).toSeq ) diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlComputeReadState.scala b/examples/components/balls/gemmini/arch/GemminiExCtrlComputeReadState.scala similarity index 88% rename from examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlComputeReadState.scala rename to examples/components/balls/gemmini/arch/GemminiExCtrlComputeReadState.scala index 66901cd74..743437cfa 100644 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlComputeReadState.scala +++ b/examples/components/balls/gemmini/arch/GemminiExCtrlComputeReadState.scala @@ -1,5 +1,9 @@ package examples.balls.gemmini +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ import gemmini._ diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrlDefaults.scala b/examples/components/balls/gemmini/arch/GemminiExCtrlDefaults.scala new file mode 100644 index 000000000..5fa1c3b41 --- /dev/null +++ b/examples/components/balls/gemmini/arch/GemminiExCtrlDefaults.scala @@ -0,0 +1,45 @@ +package examples.balls.gemmini + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ + +trait GemminiExCtrlDefaults { this: GemminiExCtrl => + + protected def applyDefaults(): Unit = { + io.cmdReq.ready := state === sIdle + + io.cmdResp.valid := false.B + io.cmdResp.bits.rob_id := rob_id_reg + + for (i <- 0 until inBW) { + io.bankReadReq(i).valid := false.B + io.bankReadReq(i).bits.addr := 0.U + } + + rdQueue0.io.deq.ready := false.B + rdQueue1.io.deq.ready := false.B + + mesh.io.a.valid := false.B + mesh.io.a.bits := 0.U.asTypeOf(mesh.io.a.bits) + mesh.io.b.valid := false.B + mesh.io.b.bits := 0.U.asTypeOf(mesh.io.b.bits) + mesh.io.d.valid := false.B + mesh.io.d.bits := 0.U.asTypeOf(mesh.io.d.bits) + mesh.io.req.valid := false.B + mesh.io.req.bits := 0.U.asTypeOf(mesh.io.req.bits) + // mesh.io.resp is Valid-only (no ready); MeshWithDelays assumes immediate observation when valid + + io.bankWrite.foreach { bw => + bw.req.valid := false.B + bw.req.bits.addr := 0.U + bw.req.bits.data := 0.U + bw.req.bits.mask := VecInit(Seq.fill(memParam.bankMaskLen)(false.B)) + bw.resp.ready := true.B + } + } + +} diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrlDefs.scala b/examples/components/balls/gemmini/arch/GemminiExCtrlDefs.scala new file mode 100644 index 000000000..809ebf573 --- /dev/null +++ b/examples/components/balls/gemmini/arch/GemminiExCtrlDefs.scala @@ -0,0 +1,150 @@ +package examples.balls.gemmini + +import framework.lib.queue.Queue +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.ip.memory.bank.{SramReadIO, SramReadReq, SramReadResp, SramWriteIO, SramWriteReq, SramWriteResp} + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import framework.acc.balldomain.ball.blink.BallStatus +import framework.ip.memory.bank.{SramReadReq, SramReadResp, SramWriteIO} +import examples.balls.gemmini.configs.GemminiBallParam +import gemmini._ +import gemmini.Util._ + +/** Minimal tag for MeshWithDelays that satisfies TagQueueTag */ +class SimpleTag extends Bundle with gemmini.TagQueueTag { + val rob = UInt(8.W) + override def make_this_garbage(dummy: Int = 0): Unit = + rob := 0xff.U +} + +trait GemminiExCtrlDefs { this: GemminiExCtrl => + val config = ballParam + val DIM = config.blockSize + + val ballMapping = instance + + val inBW = ballMapping.inBW + val outBW = ballMapping.outBW + + val inputType = SInt(config.inputWidth.W) + val accType = SInt(config.accWidth.W) + val meshOutputType = SInt(config.spatialOutputWidth.W) + + val ctrlIo = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + val bankReadReq = Vec(inBW, Decoupled(new SramReadReq(memParam))) + val bankReadResp = + Vec(inBW, Flipped(Decoupled(new SramReadResp(memParam)))) + val bankWrite = Vec(outBW, Flipped(new SramWriteIO(memParam))) + val op1_bank_o = Output(UInt(frontendParam.bank_id_len.W)) + val op1_group_o = Output(UInt(memParam.groupIdWidth.W)) + val op2_bank_o = Output(UInt(frontendParam.bank_id_len.W)) + val op2_group_o = Output(UInt(memParam.groupIdWidth.W)) + val wr_bank_o = Output(UInt(frontendParam.bank_id_len.W)) + val wr_group_o = Output(UInt(memParam.groupIdWidth.W)) + val status = new BallStatus + }) + + val io = ctrlIo + + val mesh = Instantiate(new MeshWithDelays( + inputType, + inputType, + meshOutputType, + accType, + new SimpleTag, + Dataflow.BOTH, + false, + config.tileLatency, + config.outputDelay, + config.tileRows, + config.tileColumns, + config.meshRows, + config.meshColumns, + 1, + 1 + )) + + protected def widenMeshToAcc(src: Vec[Vec[SInt]]): Vec[Vec[SInt]] = + VecInit(src.map(col => VecInit(col.map(_.asTypeOf(accType))))) + + val cfg_dataflow = RegInit(0.U(1.W)) + val cfg_in_shift = RegInit(0.U(log2Up(config.accWidth).W)) + val cfg_a_transpose = RegInit(false.B) + val cfg_bd_transpose = RegInit(false.B) + val zero_op2 = RegInit(false.B) + val zero_op1_tail = RegInit(false.B) + val accumulated_cmd = RegInit(false.B) + val reset_acc_mesh = RegInit(false.B) + val acc_read_group = RegInit(0.U(log2Up(outBW).W)) + val acc_read_pending = RegInit(false.B) + val execution_dataflow = Mux(accumulated_cmd, Dataflow.OS.id.U, cfg_dataflow) + val execution_shift = Mux(accumulated_cmd, 0.U, cfg_in_shift) + + val sIdle :: sPreloadRead :: sPreloadFeed :: sComputeRead :: sComputeFeed :: sComputeFlush :: sFlush :: sDrain :: sReadAccum :: sStore :: sCommit :: Nil = + Enum(11) + val state = RegInit(sIdle) + + val rob_id_reg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + + /** Match SimpleTag.rob (8b); must match mesh req tag bits */ + protected def robIdAsTag8(x: UInt): UInt = { + val w = x.getWidth + if (w >= 8) x(7, 0) else Cat(0.U((8 - w).W), x) + } + + val op1_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val op2_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val wr_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val wr_read_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val op1_group = RegInit(0.U(memParam.groupIdWidth.W)) + val op2_group = RegInit(0.U(memParam.groupIdWidth.W)) + val wr_group = RegInit(0.U(memParam.groupIdWidth.W)) + val op1_base = RegInit(0.U(10.W)) + val op2_base = RegInit(0.U(10.W)) + val wr_base = RegInit(0.U(10.W)) + io.op1_bank_o := Mux(state === sReadAccum, wr_read_bank, op1_bank) + io.op1_group_o := Mux(state === sReadAccum, wr_group + acc_read_group, op1_group) + io.op2_bank_o := op2_bank + io.op2_group_o := op2_group + io.wr_bank_o := wr_bank + io.wr_group_o := wr_group + + val read_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) + val feed_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) + val store_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) + val total_rows = RegInit(DIM.U(log2Up(DIM + 1).W)) + val req_sent = RegInit(false.B) + val read_done = RegInit(VecInit(Seq.fill(inBW)(false.B))) + val xpose_ready = RegInit(false.B) + val xpose_row_cnt = RegInit(0.U(log2Up(DIM + 1).W)) + + val sub_cmd = io.cmdReq.bits.rs2(3, 0) + + val rdQueue0 = Instantiate(new Queue(new SramReadResp(memParam), DIM)) + val rdQueue1 = Instantiate(new Queue(new SramReadResp(memParam), DIM)) + rdQueue0.io.enq <> io.bankReadResp(0) + rdQueue1.io.enq <> io.bankReadResp(1) + + val outBuf = Reg( + Vec(DIM, Vec(config.meshColumns, Vec(config.tileColumns, accType))) + ) + + val outBufRows = RegInit(0.U(log2Up(DIM + 1).W)) + val outBufCollected = RegInit(0.U(log2Up(DIM + 1).W)) + val op1Buf = Reg(Vec(DIM, Vec(DIM, inputType))) + val op2Buf = Reg(Vec(DIM, Vec(DIM, inputType))) + + val port_written = RegInit(VecInit(Seq.fill(outBW)(false.B))) +} diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlFsm.scala b/examples/components/balls/gemmini/arch/GemminiExCtrlFsm.scala similarity index 83% rename from examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlFsm.scala rename to examples/components/balls/gemmini/arch/GemminiExCtrlFsm.scala index 376fb030b..c8562e68a 100644 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlFsm.scala +++ b/examples/components/balls/gemmini/arch/GemminiExCtrlFsm.scala @@ -1,5 +1,9 @@ package examples.balls.gemmini +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlPreloadStates.scala b/examples/components/balls/gemmini/arch/GemminiExCtrlPreloadStates.scala similarity index 93% rename from examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlPreloadStates.scala rename to examples/components/balls/gemmini/arch/GemminiExCtrlPreloadStates.scala index 4361571cc..0bbaa25d8 100644 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlPreloadStates.scala +++ b/examples/components/balls/gemmini/arch/GemminiExCtrlPreloadStates.scala @@ -1,5 +1,9 @@ package examples.balls.gemmini +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ import gemmini._ @@ -85,15 +89,15 @@ trait GemminiExCtrlPreloadStates { this: GemminiExCtrl => VecInit((0 until DIM).map(i => op2Buf(i)(transpose_col))) val weight_data = Mux(explicitBTranspose, transposed_data, row_data) mesh.io.a.valid := true.B - mesh.io.a.bits := 0.U.asTypeOf(mesh.A_TYPE) + mesh.io.a.bits := 0.U.asTypeOf(mesh.io.a.bits) mesh.io.b.valid := true.B - mesh.io.b.bits := 0.U.asTypeOf(mesh.B_TYPE) + mesh.io.b.bits := 0.U.asTypeOf(mesh.io.b.bits) mesh.io.d.valid := true.B // OS preload in Buckyball is used to prime pipeline state before compute. // Feed D=0 to avoid injecting bias-like data into the following matmul. mesh.io.d.bits := Mux( execution_dataflow === Dataflow.OS.id.U, - 0.U.asTypeOf(mesh.D_TYPE), + 0.U.asTypeOf(mesh.io.d.bits), VecInit( weight_data.grouped(config.tileColumns).map(g => VecInit(g)).toSeq ) diff --git a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlStoreOps.scala b/examples/components/balls/gemmini/arch/GemminiExCtrlStoreOps.scala similarity index 94% rename from examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlStoreOps.scala rename to examples/components/balls/gemmini/arch/GemminiExCtrlStoreOps.scala index 975d2ed5a..f9e1af016 100644 --- a/examples/balls/gemmini/arch/src/main/scala/GemminiExCtrlStoreOps.scala +++ b/examples/components/balls/gemmini/arch/GemminiExCtrlStoreOps.scala @@ -1,5 +1,9 @@ package examples.balls.gemmini +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ import gemmini._ @@ -95,7 +99,7 @@ trait GemminiExCtrlStoreOps { this: GemminiExCtrl => .reverse ) - val bitsPerPort = b.memDomain.bankWidth + val bitsPerPort = memParam.bankWidth for (i <- 0 until outBW) { when(!port_written(i)) { @@ -104,7 +108,7 @@ trait GemminiExCtrlStoreOps { this: GemminiExCtrl => io.bankWrite(i).req.bits.addr := wr_base + store_row_cnt io.bankWrite(i).req.bits.data := slice io.bankWrite(i).req.bits.mask := VecInit( - Seq.fill(b.memDomain.bankMaskLen)(true.B) + Seq.fill(memParam.bankMaskLen)(true.B) ) when(io.bankWrite(i).req.ready) { port_written(i) := true.B diff --git a/examples/balls/gemmini/arch/src/main/scala/LICENSE.gemmini b/examples/components/balls/gemmini/arch/LICENSE.gemmini similarity index 100% rename from examples/balls/gemmini/arch/src/main/scala/LICENSE.gemmini rename to examples/components/balls/gemmini/arch/LICENSE.gemmini diff --git a/examples/components/balls/gemmini/arch/Loop.scala b/examples/components/balls/gemmini/arch/Loop.scala new file mode 100644 index 000000000..d21e0feba --- /dev/null +++ b/examples/components/balls/gemmini/arch/Loop.scala @@ -0,0 +1,173 @@ +package examples.balls.gemmini + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.balls.gemmini.configs.GemminiBallParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.balldomain.sub_rob.{Expansion, HasExpansion} +import framework.acc.frontend.decoder.DomainId + +/** Loop configuration lives in BallDomain's metadata SRAM, not in a second register file. */ +@instantiable +class Loop( + f: FrontendParam, + m: MemDomainParam, + routing: BallDomainParam, + id: Int) + extends Module + with HasExpansion { + val ball = routing.balls.find(_.id == id).get + val p = GemminiBallParam.fromProto(ball.parameters.unpack(classOf[buckyball.balls.gemmini.Params])) + require(m.mmioTotalBytes >= 136) + def funct(name: String): UInt = ball.instructions.find(_.mnemonic == name).get.funct7.U(7.W) + val ws = Seq("BOUNDS", "ADDR_A", "ADDR_B", "ADDR_D", "ADDR_C", "STRIDES_AB", "STRIDES_DC") + .map(n => funct("GEMMINI_LOOP_WS_CONFIG_" + n)) + val conv = (1 to 9).map(n => funct(s"GEMMINI_LOOP_CONV_WS_CONFIG_$n")) + @public + val expansion = IO(new Expansion(f, m, routing)) + val idle :: storing :: configured :: reading :: receiving :: starting :: streaming :: Nil = Enum(7) + val state = RegInit(idle) + val master = Reg(UInt(log2Ceil(f.rob_entries).W)) + val operand = Reg(UInt(64.W)) + val byte = RegInit(0.U(7.W)) + val base = Reg(UInt(8.W)) + val convolution = Reg(Bool()) + val values = Reg(Vec(9, UInt(64.W))) + val fn = expansion.command.bits.funct + val wsConfig = VecInit(ws.map(fn === _)) + val convConfig = VecInit(conv.map(fn === _)) + val triggerWs = fn === funct("GEMMINI_LOOP_WS") + val triggerConv = fn === funct("GEMMINI_LOOP_CONV_WS") + expansion.accepts := wsConfig.asUInt.orR || convConfig.asUInt.orR || triggerWs || triggerConv + expansion.starts := triggerWs || triggerConv + expansion.command.ready := state === idle + when(expansion.command.fire) { + master := expansion.command.bits.rob_id + operand := expansion.command.bits.rs2 + byte := 0.U + convolution := triggerConv + when(expansion.starts) { + base := Mux(triggerConv, 64.U, 0.U) + state := reading + }.otherwise { + base := Mux(wsConfig.asUInt.orR, PriorityEncoder(wsConfig) << 3, 64.U + (PriorityEncoder(convConfig) << 3)) + state := storing + } + } + expansion.write.valid := state === storing + expansion.write.bits.addr := base + byte + expansion.write.bits.data := (operand >> (byte << 3))(7, 0) + when(expansion.write.fire) { + byte := byte + 1.U + when(byte === 7.U)(state := configured) + } + expansion.complete.valid := state === configured + expansion.complete.bits.rob_id := master + when(expansion.complete.fire)(state := idle) + expansion.read.req.valid := state === reading + expansion.read.req.bits.addr := base + byte + when(expansion.read.req.fire)(state := receiving) + expansion.read.resp.ready := state === receiving + when(expansion.read.resp.fire) { + val index = byte(6, 3) + values(index) := (values(index) >> 8) | (expansion.read.resp.bits.data.pad(64) << 56) + byte := byte + 1.U + state := Mux(byte === Mux(convolution, 71.U, 55.U), starting, reading) + } + val matmul: Instance[LoopMatmulUnroller] = Instantiate(new LoopMatmulUnroller(f, m, p)) + val convolve: Instance[LoopConvUnroller] = Instantiate(new LoopConvUnroller(f, m, p)) + matmul.io.start.valid := state === starting && !convolution + val mat = matmul.io.start.bits + mat.max_i := values(0)(47, 32); mat.max_j := values(0)(31, 16); mat.max_k := values(0)(15, 0) + mat.dram_addr_a := values(1); mat.dram_addr_b := values(2); mat.dram_addr_d := values(3); mat.dram_addr_c := values(4) + mat.stride_a := values(5)(31, 0); mat.stride_b := values(5)(63, 32) + mat.stride_d := values(6)(31, 0); mat.stride_c := values(6)(63, 32) + mat.bank_a := operand(9, 0); mat.bank_b := operand(19, 10); mat.bank_c := operand(29, 20); mat.low_d := operand(30) + convolve.io.start.valid := state === starting && convolution + val cv = convolve.io.start.bits + cv.batch_size := values(0)(15, 0); cv.in_dim := values(0)(31, 16); cv.in_channels := values(0)(47, 32) + cv.out_channels := values(1)(15, 0); cv.out_dim := values(1)(31, 16); cv.stride := values(1)(39, 32); + cv.padding := values(1)(47, 40) + cv.kernel_dim := values(2)(7, 0); cv.pool_size := values(2)(15, 8); cv.pool_stride := values(2)(23, 16); + cv.pool_padding := values(2)(31, 24) + cv.dram_addr_bias := values(3); cv.dram_addr_input := values(4); cv.dram_addr_weight := values(5); + cv.dram_addr_output := values(6) + cv.input_stride := values(7)(31, 0); cv.weight_stride := values(7)(63, 32); cv.output_stride := values(8)(31, 0) + cv.bank_input := operand(9, 0); cv.bank_weight := operand(19, 10); cv.bank_output := operand(29, 20); + cv.no_bias := operand(30) + when(matmul.io.start.fire || convolve.io.start.fire)(state := streaming) + val commands = Wire(Decoupled(new LoopCmd(f, m, p))) + commands.valid := Mux(convolution, convolve.io.cmd.valid, matmul.io.cmd.valid) + commands.bits := Mux(convolution, convolve.io.cmd.bits, matmul.io.cmd.bits) + matmul.io.cmd.ready := !convolution && commands.ready + convolve.io.cmd.ready := convolution && commands.ready + expansion.rows.valid := state === streaming && commands.valid + commands.ready := state === streaming && expansion.rows.ready + expansion.rows.bits := 0.U.asTypeOf(expansion.rows.bits) + expansion.rows.bits.last := commands.bits.last + expansion.rows.bits.ball_id := ball.id.U + expansion.rows.bits.master_rob_id := master + when(expansion.rows.fire && expansion.rows.bits.last)(state := idle) + for (i <- 0 until 4) { + val src = commands.bits.slots(i) + val out = expansion.rows.bits.slots(i) + val c = out.cmd + val t = src.bits + val alloc = t.cmdType === LoopSubCmdType.MSET_ALLOC + val free = t.cmdType === LoopSubCmdType.MSET_FREE + val load = t.cmdType === LoopSubCmdType.MVIN + val store = t.cmdType === LoopSubCmdType.MVOUT + val preload = t.cmdType === LoopSubCmdType.PRELOAD + val compute = t.cmdType === LoopSubCmdType.COMPUTE + val fn = MuxCase( + funct("GEMMINI_COMPUTE_ACCUMULATED"), + Seq( + (alloc || free) -> 32.U, + load -> 33.U, + store -> 16.U, + preload -> funct("GEMMINI_PRELOAD"), + (compute && t.compute_mode === 0.U) -> funct("GEMMINI_COMPUTE_PRELOADED") + ) + ) + out.valid := src.valid + c.domain_id := Mux(alloc || free || load || store, DomainId.MEM, DomainId.BALL) + c.cmd.raw_inst := (fn << 25) | "h307b".U + c.cmd.rs1Data := Mux( + alloc || free, + t.bank_id, + Mux( + load, + t.bank_id << 20, + Mux(store, t.bank_id, t.op1_bank | (t.op2_bank << 10) | (t.wr_bank << 20)) + ) | (t.iter << 30) + ) + c.cmd.rs2Data := Mux( + alloc, + t.bank_row | (t.bank_col << 5) | (1.U << 10), + Mux( + free, + 0.U, + Mux( + load || store, + t.dram_addr | (t.stride.pad(64) << 39), + Mux(preload, 1.U, Mux(t.compute_mode === 0.U, 2.U, 3.U)) | (t.zero_op2 << 4) | (t.zero_op1_tail << 5) + ) + ) + ) + c.iter := t.iter + c.bankAccess.rd_bank_0_valid := store || preload || compute + c.bankAccess.rd_bank_0_id := Mux(store, t.bank_id, t.op1_bank) + c.bankAccess.rd_bank_1_valid := compute && !t.zero_op2 + c.bankAccess.rd_bank_1_id := t.op2_bank + c.bankAccess.wr_bank_valid := alloc || free || load || compute + c.bankAccess.wr_bank_id := Mux(alloc || free || load, t.bank_id, t.wr_bank) + c.architecturalDestination := c.bankAccess.wr_bank_id + c.bankAccess.rd_wr_valid := compute && t.compute_mode =/= 0.U + c.bankAccess.rd_wr_id := t.wr_bank + c.wr_read_bank := t.wr_bank + c.op1_col := 1.U; c.op2_col := 1.U; c.wr_col := 4.U + } +} diff --git a/examples/components/balls/gemmini/arch/LoopCmd.scala b/examples/components/balls/gemmini/arch/LoopCmd.scala new file mode 100644 index 000000000..47240b530 --- /dev/null +++ b/examples/components/balls/gemmini/arch/LoopCmd.scala @@ -0,0 +1,86 @@ +package examples.balls.gemmini + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.gemmini.configs.GemminiBallParam + +import chisel3._ +import chisel3.util._ + +object LoopSubCmdType { + val MSET_ALLOC = 0.U(3.W) + val MSET_FREE = 1.U(3.W) + val MVIN = 2.U(3.W) + val MVOUT = 3.U(3.W) + val PRELOAD = 4.U(3.W) + val COMPUTE = 5.U(3.W) +} + +class LoopSubCmd(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: GemminiBallParam) + extends Bundle { + val cmdType = UInt(3.W) + val bank_id = UInt(frontendParam.bank_id_len.W) + val dram_addr = UInt(memParam.memAddrLen.W) + val stride = UInt(19.W) + val iter = UInt(frontendParam.iter_len.W) + val bank_row = UInt(5.W) + val bank_col = UInt(5.W) + val op1_bank = UInt(frontendParam.bank_id_len.W) + val op2_bank = UInt(frontendParam.bank_id_len.W) + val wr_bank = UInt(frontendParam.bank_id_len.W) + val compute_mode = UInt(2.W) + val zero_op2 = Bool() + val zero_op1_tail = Bool() +} + +class LoopCmd(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: GemminiBallParam) + extends Bundle { + val slots = Vec(4, Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + val last = Bool() +} + +class LoopWsConfig(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: GemminiBallParam) + extends Bundle { + val max_i = UInt(16.W) + val max_j = UInt(16.W) + val max_k = UInt(16.W) + val dram_addr_a = UInt(memParam.memAddrLen.W) + val dram_addr_b = UInt(memParam.memAddrLen.W) + val dram_addr_d = UInt(memParam.memAddrLen.W) + val dram_addr_c = UInt(memParam.memAddrLen.W) + val stride_a = UInt(32.W) + val stride_b = UInt(32.W) + val stride_d = UInt(32.W) + val stride_c = UInt(32.W) + val bank_a = UInt(frontendParam.bank_id_len.W) + val bank_b = UInt(frontendParam.bank_id_len.W) + val bank_c = UInt(frontendParam.bank_id_len.W) + val low_d = Bool() +} + +class LoopConvWsConfig(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: GemminiBallParam) + extends Bundle { + val batch_size = UInt(16.W) + val in_dim = UInt(16.W) + val in_channels = UInt(16.W) + val out_channels = UInt(16.W) + val out_dim = UInt(16.W) + val stride = UInt(8.W) + val padding = UInt(8.W) + val kernel_dim = UInt(8.W) + val pool_size = UInt(8.W) + val pool_stride = UInt(8.W) + val pool_padding = UInt(8.W) + val dram_addr_bias = UInt(memParam.memAddrLen.W) + val dram_addr_input = UInt(memParam.memAddrLen.W) + val dram_addr_weight = UInt(memParam.memAddrLen.W) + val dram_addr_output = UInt(memParam.memAddrLen.W) + val input_stride = UInt(32.W) + val weight_stride = UInt(32.W) + val output_stride = UInt(32.W) + val bank_input = UInt(frontendParam.bank_id_len.W) + val bank_weight = UInt(frontendParam.bank_id_len.W) + val bank_output = UInt(frontendParam.bank_id_len.W) + val no_bias = Bool() +} diff --git a/examples/balls/gemmini/arch/src/main/scala/LoopConvAddrGen.scala b/examples/components/balls/gemmini/arch/LoopConvAddrGen.scala similarity index 80% rename from examples/balls/gemmini/arch/src/main/scala/LoopConvAddrGen.scala rename to examples/components/balls/gemmini/arch/LoopConvAddrGen.scala index c0471a757..6cdcefe72 100644 --- a/examples/balls/gemmini/arch/src/main/scala/LoopConvAddrGen.scala +++ b/examples/components/balls/gemmini/arch/LoopConvAddrGen.scala @@ -1,9 +1,11 @@ package examples.balls.gemmini +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig import examples.balls.gemmini.configs.GemminiBallParam /** @@ -16,15 +18,16 @@ import examples.balls.gemmini.configs.GemminiBallParam * - Bias DRAM address */ @instantiable -class LoopConvAddrGen(val b: GlobalConfig) extends Module { - val config = GemminiBallParam(b) +class LoopConvAddrGen(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: GemminiBallParam) + extends Module { + val config = ballParam val elemSize = config.inputWidth / 8 val accBytes = config.accWidth / 8 @public val io = IO(new Bundle { // Conv config - val cfg = Input(new LoopConvWsConfig(b)) + val cfg = Input(new LoopConvWsConfig(frontendParam, memParam, ballParam)) // Loop indices val batch = Input(UInt(16.W)) val orow = Input(UInt(16.W)) @@ -34,10 +37,10 @@ class LoopConvAddrGen(val b: GlobalConfig) extends Module { val kcol = Input(UInt(16.W)) val kch = Input(UInt(16.W)) // Outputs - val inputAddr = Output(UInt(b.memDomain.memAddrLen.W)) - val weightAddr = Output(UInt(b.memDomain.memAddrLen.W)) - val outputAddr = Output(UInt(b.memDomain.memAddrLen.W)) - val biasAddr = Output(UInt(b.memDomain.memAddrLen.W)) + val inputAddr = Output(UInt(memParam.memAddrLen.W)) + val weightAddr = Output(UInt(memParam.memAddrLen.W)) + val outputAddr = Output(UInt(memParam.memAddrLen.W)) + val biasAddr = Output(UInt(memParam.memAddrLen.W)) val isPadding = Output(Bool()) }) diff --git a/examples/components/balls/gemmini/arch/LoopConvUnroller.scala b/examples/components/balls/gemmini/arch/LoopConvUnroller.scala new file mode 100644 index 000000000..2a756f642 --- /dev/null +++ b/examples/components/balls/gemmini/arch/LoopConvUnroller.scala @@ -0,0 +1,306 @@ +package examples.balls.gemmini + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.balls.gemmini.configs.GemminiBallParam + +/** + * LoopConvUnroller — expands LOOP_CONV_WS into LoopCmd rows. + * + * Simplified iteration: + * batch → orow → ocol → och → krow → kcol → kch (innermost) + * + * Each (orow, ocol, och, krow, kcol, kch) core generates: + * MVIN weight → PRELOAD + MVIN input → COMPUTE → (on last k-iter) MVOUT + * + * First implementation: no pooling, no im2col tiling, single DIM×DIM cores. + */ +@instantiable +class LoopConvUnroller(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: GemminiBallParam) + extends Module { + val config = ballParam + val DIM = config.blockSize + val elemSize = config.inputWidth / 8 + val bankBytes = memParam.bankWidth / 8 + + @public + val io = IO(new Bundle { + val start = Flipped(Decoupled(new LoopConvWsConfig(frontendParam, memParam, ballParam))) + val cmd = Decoupled(new LoopCmd(frontendParam, memParam, ballParam)) + val busy = Output(Bool()) + }) + + val addrGen: Instance[LoopConvAddrGen] = Instantiate(new LoopConvAddrGen(frontendParam, memParam, ballParam)) + + // FSM states + val sIdle :: sAlloc :: sMvinInput :: sMvinWeight :: sCompute :: sMvout :: sFree :: sDone :: Nil = + Enum(8) + val state = RegInit(sIdle) + + val cfg = Reg(new LoopConvWsConfig(frontendParam, memParam, ballParam)) + + // Iterator registers + val batch_reg = RegInit(0.U(16.W)) + val orow_reg = RegInit(0.U(16.W)) + val ocol_reg = RegInit(0.U(16.W)) + val och_reg = RegInit(0.U(16.W)) + val krow_reg = RegInit(0.U(16.W)) + val kcol_reg = RegInit(0.U(16.W)) + val kch_reg = RegInit(0.U(16.W)) + + // Is this the first k-iteration for this (orow, ocol, och)? + val isFirstK = krow_reg === 0.U && kcol_reg === 0.U && kch_reg === 0.U + + // Is this the last k-iteration? + val isLastK = (krow_reg === cfg.kernel_dim - 1.U) && + (kcol_reg === cfg.kernel_dim - 1.U) && + (kch_reg + DIM.U >= cfg.in_channels) // Core boundary + + // Connect address generator + addrGen.io.cfg := cfg + addrGen.io.batch := batch_reg + addrGen.io.orow := orow_reg + addrGen.io.ocol := ocol_reg + addrGen.io.och := och_reg + addrGen.io.krow := krow_reg + addrGen.io.kcol := kcol_reg + addrGen.io.kch := kch_reg + + // Defaults + io.start.ready := state === sIdle + io.cmd.valid := false.B + io.cmd.bits := 0.U.asTypeOf(new LoopCmd(frontendParam, memParam, ballParam)) + io.cmd.bits.last := state === sFree + io.busy := state =/= sIdle + + def emptySlot(): Valid[LoopSubCmd] = { + val v = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + v.valid := false.B + v.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + v + } + + def msetSlot( + bankId: UInt, + alloc: Boolean, + row: Int, + col: Int + ): Valid[LoopSubCmd] = { + val v = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + v.valid := true.B + v.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + v.bits.cmdType := (if (alloc) LoopSubCmdType.MSET_ALLOC + else LoopSubCmdType.MSET_FREE) + v.bits.bank_id := bankId + v.bits.bank_row := row.U + v.bits.bank_col := col.U + v + } + + def mvinSlot( + bankId: UInt, + addr: UInt, + iter: UInt, + stride: UInt + ): Valid[LoopSubCmd] = { + val v = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + v.valid := true.B + v.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + v.bits.cmdType := LoopSubCmdType.MVIN + v.bits.bank_id := bankId + v.bits.dram_addr := addr + v.bits.stride := stride + v.bits.iter := iter + v + } + + def outRows(): UInt = 1.U + + def inMemStride(bytes: UInt): UInt = + (bytes / bankBytes.U).pad(19)(18, 0) + + def outMemStride(bytes: UInt): UInt = + (bytes / (bankBytes * 4).U).pad(19)(18, 0) + + def weightTileStride(): UInt = + cfg.out_channels * elemSize.U + + // Advance k-iterators (kch → kcol → krow), then output iterators (och → ocol → orow → batch) + def advanceIter(): Unit = { + when(kch_reg + DIM.U < cfg.in_channels) { + kch_reg := kch_reg + DIM.U + }.elsewhen(kcol_reg + 1.U < cfg.kernel_dim) { + kch_reg := 0.U; kcol_reg := kcol_reg + 1.U + }.elsewhen(krow_reg + 1.U < cfg.kernel_dim) { + kch_reg := 0.U; kcol_reg := 0.U; krow_reg := krow_reg + 1.U + }.elsewhen(och_reg + DIM.U < cfg.out_channels) { + kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; + och_reg := och_reg + DIM.U + }.elsewhen(ocol_reg + 1.U < cfg.out_dim) { + kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; och_reg := 0.U + ocol_reg := ocol_reg + 1.U + }.elsewhen(orow_reg + 1.U < cfg.out_dim) { + kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; och_reg := 0.U; + ocol_reg := 0.U + orow_reg := orow_reg + 1.U + }.elsewhen(batch_reg + 1.U < cfg.batch_size) { + kch_reg := 0.U; kcol_reg := 0.U; krow_reg := 0.U; och_reg := 0.U; + ocol_reg := 0.U; orow_reg := 0.U + batch_reg := batch_reg + 1.U + }.otherwise { + state := sFree // All iterations done + } + } + + // Track whether all iterations are done (used to decide next state) + val allDone = (batch_reg === cfg.batch_size - 1.U) && + (orow_reg === cfg.out_dim - 1.U) && (ocol_reg === cfg.out_dim - 1.U) && + (och_reg + DIM.U >= cfg.out_channels) && isLastK + + switch(state) { + is(sIdle) { + io.start.ready := true.B + when(io.start.fire) { + cfg := io.start.bits + batch_reg := 0.U; orow_reg := 0.U; ocol_reg := 0.U; och_reg := 0.U + krow_reg := 0.U; kcol_reg := 0.U; kch_reg := 0.U + state := sAlloc + } + } + + // [MSET_alloc(input), MSET_alloc(weight), MSET_alloc(output), --] + is(sAlloc) { + io.cmd.valid := true.B + io.cmd.bits + .slots(0) := msetSlot(cfg.bank_input, alloc = true, row = 1, col = 1) + io.cmd.bits + .slots(1) := msetSlot(cfg.bank_weight, alloc = true, row = 1, col = 1) + io.cmd.bits + .slots(2) := msetSlot(cfg.bank_output, alloc = true, row = 1, col = 4) + io.cmd.bits.slots(3) := emptySlot() + when(io.cmd.fire) { + state := sMvinInput + } + } + + // Load weights before preloading them into the array. + is(sMvinInput) { + io.cmd.valid := true.B + io.cmd.bits.slots(0) := mvinSlot( + cfg.bank_weight, + addrGen.io.weightAddr, + DIM.U, + inMemStride(weightTileStride()) + ) + when(io.cmd.fire)(state := sMvinWeight) + } + + // Preload weights while loading independent input data. + is(sMvinWeight) { + io.cmd.valid := true.B + val preSlot = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + preSlot.valid := true.B + preSlot.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + preSlot.bits.cmdType := LoopSubCmdType.PRELOAD + preSlot.bits.op1_bank := cfg.bank_weight + preSlot.bits.wr_bank := cfg.bank_output + preSlot.bits.iter := DIM.U + io.cmd.bits.slots(0) := preSlot + io.cmd.bits.slots(1) := Mux( + addrGen.io.isPadding, + emptySlot(), + mvinSlot( + cfg.bank_input, + addrGen.io.inputAddr, + outRows(), + inMemStride(cfg.input_stride) + ) + ) + when(io.cmd.fire)(state := sCompute) + } + + // Compute consumes the preceding preload and DMA row. + is(sCompute) { + io.cmd.valid := true.B + val compSlot = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + compSlot.valid := true.B + compSlot.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + compSlot.bits.cmdType := LoopSubCmdType.COMPUTE + compSlot.bits.op1_bank := cfg.bank_input + compSlot.bits.op2_bank := cfg.bank_weight + compSlot.bits.wr_bank := cfg.bank_output + compSlot.bits.compute_mode := Mux( + isFirstK, + 0.U, + 1.U + ) // PRELOADED first, then ACCUMULATED + compSlot.bits.iter := DIM.U + compSlot.bits.zero_op2 := cfg.no_bias + compSlot.bits.zero_op1_tail := true.B + + io.cmd.bits.slots(0) := compSlot + io.cmd.bits.slots(1) := emptySlot() + io.cmd.bits.slots(2) := emptySlot() + io.cmd.bits.slots(3) := emptySlot() + + when(io.cmd.fire) { + when(isLastK) { + state := sMvout // Need to output results after last k iteration + }.otherwise { + advanceIter() + when(state =/= sFree)(state := sMvinInput) + } + } + } + + // [MVOUT, --, --, --] + is(sMvout) { + io.cmd.valid := true.B + val mvSlot = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + mvSlot.valid := true.B + mvSlot.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + mvSlot.bits.cmdType := LoopSubCmdType.MVOUT + mvSlot.bits.bank_id := cfg.bank_output + mvSlot.bits.dram_addr := addrGen.io.outputAddr + mvSlot.bits.stride := outMemStride(cfg.output_stride) + mvSlot.bits.iter := outRows() + + io.cmd.bits.slots(0) := mvSlot + io.cmd.bits.slots(1) := emptySlot() + io.cmd.bits.slots(2) := emptySlot() + io.cmd.bits.slots(3) := emptySlot() + + when(io.cmd.fire) { + when(allDone) { + state := sFree + }.otherwise { + advanceIter() + when(state =/= sFree)(state := sMvinInput) + } + } + } + + // [MSET_free(input), MSET_free(weight), MSET_free(output), --] + is(sFree) { + io.cmd.valid := true.B + io.cmd.bits + .slots(0) := msetSlot(cfg.bank_input, alloc = false, row = 0, col = 0) + io.cmd.bits + .slots(1) := msetSlot(cfg.bank_weight, alloc = false, row = 0, col = 0) + io.cmd.bits + .slots(2) := msetSlot(cfg.bank_output, alloc = false, row = 0, col = 0) + io.cmd.bits.slots(3) := emptySlot() + when(io.cmd.fire) { + state := sDone + } + } + + is(sDone) { + state := sIdle + } + } +} diff --git a/examples/components/balls/gemmini/arch/LoopMatmulUnroller.scala b/examples/components/balls/gemmini/arch/LoopMatmulUnroller.scala new file mode 100644 index 000000000..a13478b32 --- /dev/null +++ b/examples/components/balls/gemmini/arch/LoopMatmulUnroller.scala @@ -0,0 +1,340 @@ +package examples.balls.gemmini + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import examples.balls.gemmini.configs.GemminiBallParam + +/** + * LoopMatmulUnroller — expands LOOP_WS into a stream of LoopCmd rows. + * + * FSM: sIdle → sAllocBanks → sPrimeLoad → sMainLoop (Row1/Row2) → sDrainLast → sFreeBanks → sDone + * + * Row layout per iteration: + * Row 1: [PRELOAD(cur), MVIN_B(cur)] + * Row 2: [COMPUTE(cur)] + * Row 3: [MVOUT(cur), MVIN_A(next)] + * The final compute row is followed by a drain row and bank release. + */ +@instantiable +class LoopMatmulUnroller(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: GemminiBallParam) + extends Module { + val config = ballParam + val DIM = config.blockSize + val elemSize = config.inputWidth / 8 + val accBytes = config.accWidth / 8 + val bankBytes = memParam.bankWidth / 8 + + @public + val io = IO(new Bundle { + val start = Flipped(Decoupled(new LoopWsConfig(frontendParam, memParam, ballParam))) + val cmd = Decoupled(new LoopCmd(frontendParam, memParam, ballParam)) + val busy = Output(Bool()) + }) + + // FSM states + val sIdle :: sAllocBanks :: sPrimeLoad :: sMainRow1 :: sCompute :: sMainRow2 :: sDrainLast :: sFreeBanks :: sDone :: Nil = + Enum(9) + val state = RegInit(sIdle) + + // Config registers (latched at start) + val cfg = Reg(new LoopWsConfig(frontendParam, memParam, ballParam)) + + // Loop iterator registers + val i_reg = RegInit(0.U(16.W)) + val j_reg = RegInit(0.U(16.W)) + val k_reg = RegInit(0.U(16.W)) + + // Total iterations and current iteration index + val totalIter = cfg.max_i * cfg.max_j * cfg.max_k + val curIter = RegInit(0.U(32.W)) + + // Whether this is the last iteration + val isLastIter = curIter === (totalIter - 1.U) + + // Defaults + io.start.ready := state === sIdle + io.cmd.valid := false.B + io.cmd.bits := 0.U.asTypeOf(new LoopCmd(frontendParam, memParam, ballParam)) + io.cmd.bits.last := state === sFreeBanks + io.busy := state =/= sIdle + + // Helper to build an invalid LoopSubCmd slot + def emptySlot(): Valid[LoopSubCmd] = { + val v = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + v.valid := false.B + v.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + v + } + + // Helper to build an MSET slot + def msetSlot( + bankId: UInt, + alloc: Boolean, + row: Int, + col: Int + ): Valid[LoopSubCmd] = { + val v = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + v.valid := true.B + v.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + v.bits.cmdType := (if (alloc) LoopSubCmdType.MSET_ALLOC + else LoopSubCmdType.MSET_FREE) + v.bits.bank_id := bankId + v.bits.bank_row := row.U + v.bits.bank_col := col.U + v + } + + // Helper to build an MVIN slot + def mvinSlot( + bankId: UInt, + addr: UInt, + iter: UInt, + stride: UInt + ): Valid[LoopSubCmd] = { + val v = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + v.valid := true.B + v.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + v.bits.cmdType := LoopSubCmdType.MVIN + v.bits.bank_id := bankId + v.bits.dram_addr := addr + v.bits.stride := stride + v.bits.iter := iter + v + } + + // Helper to build an MVOUT slot + def mvoutSlot( + bankId: UInt, + addr: UInt, + iter: UInt, + stride: UInt + ): Valid[LoopSubCmd] = { + val v = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + v.valid := true.B + v.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + v.bits.cmdType := LoopSubCmdType.MVOUT + v.bits.bank_id := bankId + v.bits.dram_addr := addr + v.bits.stride := stride + v.bits.iter := iter + v + } + + // Address computation + // addr_a(i,k) = dram_addr_a + i * stride_a + k * DIM * elemSize + // addr_b(k,j) = dram_addr_b + k * stride_b + j * DIM * elemSize + // addr_c(i,j) = dram_addr_c + i * stride_c + j * DIM * accBytes + def addrA(i: UInt, k: UInt): UInt = + cfg.dram_addr_a + i * DIM.U * cfg.stride_a + k * (DIM * elemSize).U + + def addrB(k: UInt, j: UInt): UInt = + cfg.dram_addr_b + k * DIM.U * cfg.stride_b + j * (DIM * elemSize).U + + def addrC(i: UInt, j: UInt): UInt = + cfg.dram_addr_c + i * DIM.U * cfg.stride_c + j * (DIM * accBytes).U + + def inMemStride(bytes: UInt): UInt = + (bytes / bankBytes.U)(18, 0) + + def outMemStride(bytes: UInt): UInt = + (bytes / (bankBytes * 4).U)(18, 0) + + // Next iterator values (advance k, then j, then i) + val next_k = Wire(UInt(16.W)) + val next_j = Wire(UInt(16.W)) + val next_i = Wire(UInt(16.W)) + + when(k_reg + 1.U < cfg.max_k) { + next_k := k_reg + 1.U + next_j := j_reg + next_i := i_reg + }.elsewhen(j_reg + 1.U < cfg.max_j) { + next_k := 0.U + next_j := j_reg + 1.U + next_i := i_reg + }.otherwise { + next_k := 0.U + next_j := 0.U + next_i := i_reg + 1.U + } + + // Compute mode: first k iteration uses PRELOADED (0), rest use ACCUMULATED (1) + val computeMode = Mux(k_reg === 0.U, 0.U(2.W), 1.U(2.W)) + + switch(state) { + is(sIdle) { + io.start.ready := true.B + when(io.start.fire) { + assert(io.start.bits.max_i =/= 0.U && io.start.bits.max_j =/= 0.U && io.start.bits.max_k =/= 0.U) + assert( + io.start.bits.low_d && io.start.bits.dram_addr_d === 0.U, + "Loop matmul requires zero initial accumulation" + ) + assert( + io.start.bits.bank_a =/= io.start.bits.bank_b && io.start.bits.bank_a =/= io.start.bits.bank_c && io.start.bits.bank_b =/= io.start.bits.bank_c + ) + cfg := io.start.bits + i_reg := 0.U + j_reg := 0.U + k_reg := 0.U + curIter := 0.U + state := sAllocBanks + } + } + + // Row 0: [MSET_alloc(A), MSET_alloc(B), MSET_alloc(C), --] + is(sAllocBanks) { + io.cmd.valid := true.B + io.cmd.bits + .slots(0) := msetSlot(cfg.bank_a, alloc = true, row = 1, col = 1) + io.cmd.bits + .slots(1) := msetSlot(cfg.bank_b, alloc = true, row = 1, col = 1) + io.cmd.bits.slots(2) := msetSlot( + cfg.bank_c, + alloc = true, + row = 1, + col = 4 + ) // accWidth = 4x inputWidth + io.cmd.bits.slots(3) := emptySlot() + when(io.cmd.fire) { + state := sPrimeLoad + } + } + + // Load the activation core before preloading it. + is(sPrimeLoad) { + io.cmd.valid := true.B + io.cmd.bits.slots(0) := mvinSlot( + cfg.bank_a, + addrA(0.U, 0.U), + DIM.U, + inMemStride(cfg.stride_a) + ) + io.cmd.bits.slots(1) := emptySlot() + io.cmd.bits.slots(2) := emptySlot() + io.cmd.bits.slots(3) := emptySlot() + when(io.cmd.fire) { + state := sMainRow1 + } + } + + // Preload activations while loading independent weights. + is(sMainRow1) { + io.cmd.valid := true.B + val preSlot = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + preSlot.valid := k_reg === 0.U + preSlot.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + preSlot.bits.cmdType := LoopSubCmdType.PRELOAD + preSlot.bits.op1_bank := cfg.bank_a // preload A (activations) into matrix array + preSlot.bits.wr_bank := cfg.bank_c + preSlot.bits.iter := DIM.U + + io.cmd.bits.slots(0) := preSlot + io.cmd.bits.slots(1) := mvinSlot( + cfg.bank_b, + addrB(k_reg, j_reg), + DIM.U, + inMemStride(cfg.stride_b) + ) + io.cmd.bits.slots(2) := emptySlot() + io.cmd.bits.slots(3) := emptySlot() + when(io.cmd.fire)(state := sCompute) + } + + // Compute consumes the preceding preload and DMA row. + is(sCompute) { + io.cmd.valid := true.B + val compSlot = Wire(Valid(new LoopSubCmd(frontendParam, memParam, ballParam))) + compSlot.valid := true.B + compSlot.bits := 0.U.asTypeOf(new LoopSubCmd(frontendParam, memParam, ballParam)) + compSlot.bits.cmdType := LoopSubCmdType.COMPUTE + compSlot.bits.op1_bank := cfg.bank_a + compSlot.bits.op2_bank := cfg.bank_b + compSlot.bits.wr_bank := cfg.bank_c + compSlot.bits.compute_mode := computeMode + compSlot.bits.iter := DIM.U + + io.cmd.bits.slots(0) := compSlot + io.cmd.bits.slots(1) := emptySlot() + io.cmd.bits.slots(2) := emptySlot() + io.cmd.bits.slots(3) := emptySlot() + + when(io.cmd.fire) { + when(isLastIter) { + state := sDrainLast // Drain after the final compute. + }.otherwise { + state := sMainRow2 + } + } + } + + // Store the result while preparing the next activation core. + is(sMainRow2) { + io.cmd.valid := true.B + io.cmd.bits.slots(0) := mvoutSlot( + cfg.bank_c, + addrC(i_reg, j_reg), + DIM.U, + outMemStride(cfg.stride_c) + ) + io.cmd.bits.slots(0).valid := k_reg + 1.U === cfg.max_k + io.cmd.bits.slots(1) := mvinSlot( + cfg.bank_a, + addrA(next_i, next_k), + DIM.U, + inMemStride(cfg.stride_a) + ) + io.cmd.bits.slots(2) := emptySlot() + io.cmd.bits.slots(3) := emptySlot() + + when(io.cmd.fire) { + // Advance iterators + i_reg := next_i + j_reg := next_j + k_reg := next_k + curIter := curIter + 1.U + state := sMainRow1 + } + } + + // Drain: emit final MVOUT for last iteration + is(sDrainLast) { + io.cmd.valid := true.B + io.cmd.bits.slots(0) := mvoutSlot( + cfg.bank_c, + addrC(i_reg, j_reg), + DIM.U, + outMemStride(cfg.stride_c) + ) + io.cmd.bits.slots(1) := emptySlot() + io.cmd.bits.slots(2) := emptySlot() + io.cmd.bits.slots(3) := emptySlot() + when(io.cmd.fire) { + state := sFreeBanks + } + } + + // Free: [MSET_free(A), MSET_free(B), MSET_free(C), --] + is(sFreeBanks) { + io.cmd.valid := true.B + io.cmd.bits + .slots(0) := msetSlot(cfg.bank_a, alloc = false, row = 0, col = 0) + io.cmd.bits + .slots(1) := msetSlot(cfg.bank_b, alloc = false, row = 0, col = 0) + io.cmd.bits + .slots(2) := msetSlot(cfg.bank_c, alloc = false, row = 0, col = 0) + io.cmd.bits.slots(3) := emptySlot() + when(io.cmd.fire) { + state := sDone + } + } + + is(sDone) { + state := sIdle + } + } +} diff --git a/examples/components/balls/gemmini/configs/GemminiBallParam.scala b/examples/components/balls/gemmini/configs/GemminiBallParam.scala new file mode 100644 index 000000000..e02350d0b --- /dev/null +++ b/examples/components/balls/gemmini/configs/GemminiBallParam.scala @@ -0,0 +1,41 @@ +package examples.balls.gemmini.configs + +import buckyball.balls.gemmini.{Params => ParametersProto} + +import upickle.default.{macroRW, ReadWriter} + +case class GemminiBallParam( + meshRows: Int, + meshColumns: Int, + tileRows: Int, + tileColumns: Int, + inputWidth: Int, + accWidth: Int, + spatialOutputWidth: Int, + tileLatency: Int, + outputDelay: Int) { + + val totalRows: Int = meshRows * tileRows + val totalColumns: Int = meshColumns * tileColumns + val blockSize: Int = totalRows +} + +object GemminiBallParam { + + def fromProto(p: ParametersProto): GemminiBallParam = { + require(p.getAllFields.size == p.getDescriptorForType.getFields.size, "Gemmini parameters are required") + GemminiBallParam( + p.getMeshRows, + p.getMeshColumns, + p.getTileRows, + p.getTileColumns, + p.getInputWidth, + p.getAccWidth, + p.getSpatialOutputWidth, + p.getTileLatency, + p.getOutputDelay + ) + } + + implicit val rw: ReadWriter[GemminiBallParam] = macroRW +} diff --git a/examples/components/balls/gemmini/configs/default.toml b/examples/components/balls/gemmini/configs/default.toml new file mode 100644 index 000000000..6d1222892 --- /dev/null +++ b/examples/components/balls/gemmini/configs/default.toml @@ -0,0 +1,11 @@ +# Gemmini Ball Configuration + +meshRows = 16 +meshColumns = 16 +tileRows = 1 +tileColumns = 1 +inputWidth = 8 +accWidth = 32 +spatialOutputWidth = 20 +tileLatency = 0 +outputDelay = 1 diff --git a/examples/components/balls/gemmini/configs/gemmini_params.proto b/examples/components/balls/gemmini/configs/gemmini_params.proto new file mode 100644 index 000000000..2d6c66e95 --- /dev/null +++ b/examples/components/balls/gemmini/configs/gemmini_params.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; +package buckyball.balls.gemmini; +option java_package = "buckyball.balls.gemmini"; +option java_multiple_files = true; + +message Params { + optional int32 meshRows = 1; + optional int32 meshColumns = 2; + optional int32 tileRows = 3; + optional int32 tileColumns = 4; + optional int32 inputWidth = 5; + optional int32 accWidth = 6; + optional int32 spatialOutputWidth = 7; + optional int32 tileLatency = 8; + optional int32 outputDelay = 9; +} diff --git a/examples/components/balls/gemmini/emu/src/02_gemmini_config.rs b/examples/components/balls/gemmini/emu/src/02_gemmini_config.rs new file mode 100644 index 000000000..f59b81b76 --- /dev/null +++ b/examples/components/balls/gemmini/emu/src/02_gemmini_config.rs @@ -0,0 +1,22 @@ +//===- 02_gemmini_config.rs - GEMMINI_CONFIG instruction -------------------===// + +use super::gemmini_state::gemini; +use super::instruction::{BallModel, ExecContext}; + +pub struct GemminiConfig; + +impl BallModel for GemminiConfig { + fn exec(_xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + let state = gemini(ctx); + let mut g = state.lock().unwrap(); + g.cfg.dataflow = ((xs2 >> 4) & 1) as u8; + g.cfg.a_transpose = ((xs2 >> 7) & 1) != 0; + g.cfg.b_transpose = ((xs2 >> 8) & 1) != 0; + g.cfg.in_shift = ((xs2 >> 9) & 0xFFFFFFFF) as u32; + 0 + } + + fn latency(_xs1: u64, _xs2: u64) -> u64 { + 1 + } +} diff --git a/examples/components/balls/gemmini/emu/src/03_gemmini_flush.rs b/examples/components/balls/gemmini/emu/src/03_gemmini_flush.rs new file mode 100644 index 000000000..2fb2066f0 --- /dev/null +++ b/examples/components/balls/gemmini/emu/src/03_gemmini_flush.rs @@ -0,0 +1,15 @@ +//===- 03_gemmini_flush.rs - GEMMINI_FLUSH instruction ---------------------===// + +use super::instruction::{BallModel, ExecContext}; + +pub struct GemminiFlush; + +impl BallModel for GemminiFlush { + fn exec(_xs1: u64, _xs2: u64, _ctx: &mut ExecContext) -> u64 { + 0 + } + + fn latency(_xs1: u64, _xs2: u64) -> u64 { + 1 + } +} diff --git a/examples/components/balls/gemmini/emu/src/21_gemmini_preload.rs b/examples/components/balls/gemmini/emu/src/21_gemmini_preload.rs new file mode 100644 index 000000000..ef1ca1a1f --- /dev/null +++ b/examples/components/balls/gemmini/emu/src/21_gemmini_preload.rs @@ -0,0 +1,43 @@ +use super::bank_matrix::read_i8_nn_at; +use super::decode::{pbank, rs1_b0, rs1_b2, rs1_iter}; +use super::gemmini_state::gemini; +use super::instruction::{BallModel, ExecContext}; + +pub struct GemminiPreload; + +impl BallModel for GemminiPreload { + fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + let op1 = rs1_b0(xs1); + let wr = rs1_b2(xs1); + let n = rs1_iter(xs1) as usize; + + if !ctx.config(op1).allocated || !ctx.config(wr).allocated { + panic!("gemmini_preload: bank not allocated"); + } + if n == 0 || n > 64 { + panic!("gemmini_preload: bad iter"); + } + + let p1 = pbank(ctx, op1); + let state = gemini(ctx); + let mut gm = state.lock().unwrap(); + + if gm.cfg.dataflow == 1 { + let b = read_i8_nn_at(&ctx.banks, p1, ((xs2 >> 6) & 0x3ff) as usize, n); + gm.ws_b = Some(if gm.cfg.b_transpose { + (0..n).map(|i| (0..n).map(|j| b[j][i]).collect()).collect() + } else { + b + }); + } else { + // RTL OS preload only primes the mesh and injects D=0. It does + // not write the destination SRAM bank; compute produces C. + } + 0 + } + + fn latency(xs1: u64, _xs2: u64) -> u64 { + let n = rs1_iter(xs1).clamp(1, 64); + n.saturating_mul(n) + } +} diff --git a/examples/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs b/examples/components/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs similarity index 95% rename from examples/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs rename to examples/components/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs index 401f09714..afb8f7c4c 100644 --- a/examples/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs +++ b/examples/components/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs @@ -1,11 +1,11 @@ use super::bank_matrix::{read_i32_nn_at, read_i8_nn_at, write_i32_nn_groups_at}; use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::gemmini_state::{gemini, in_shift as apply_in_shift}; -use super::instruction::{BallInstruction, ExecContext}; +use super::instruction::{BallModel, ExecContext}; pub struct GemminiComputePreloaded; -impl BallInstruction for GemminiComputePreloaded { +impl BallModel for GemminiComputePreloaded { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let op_a = rs1_b0(xs1); let op_b = rs1_b1(xs1); @@ -24,7 +24,8 @@ impl BallInstruction for GemminiComputePreloaded { let pw: Vec<_> = (0..ctx.config(wr).cols) .map(|group| pbank_group(ctx, wr, group)) .collect(); - let gm = gemini().lock().unwrap(); + let state = gemini(ctx); + let gm = state.lock().unwrap(); let df = gm.cfg.dataflow; let a_transpose = gm.cfg.a_transpose; let b_transpose = gm.cfg.b_transpose; diff --git a/examples/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs b/examples/components/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs similarity index 93% rename from examples/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs rename to examples/components/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs index fee711ae2..3f31e93f4 100644 --- a/examples/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs +++ b/examples/components/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs @@ -1,11 +1,11 @@ use super::bank_matrix::{read_i32_nn_groups_at, read_i8_nn_at, write_i32_nn_groups_at}; use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::gemmini_state::{gemini, in_shift as apply_in_shift}; -use super::instruction::{BallInstruction, ExecContext}; +use super::instruction::{BallModel, ExecContext}; pub struct GemminiComputeAccumulated; -impl BallInstruction for GemminiComputeAccumulated { +impl BallModel for GemminiComputeAccumulated { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let op_a = rs1_b0(xs1); let op_b = rs1_b1(xs1); @@ -25,7 +25,8 @@ impl BallInstruction for GemminiComputeAccumulated { .map(|group| pbank_group(ctx, wr, group)) .collect(); - let gm = gemini().lock().unwrap(); + let state = gemini(ctx); + let gm = state.lock().unwrap(); let a_transpose = gm.cfg.a_transpose; let b_transpose = gm.cfg.b_transpose; let shift = gm.cfg.in_shift; diff --git a/examples/components/balls/gemmini/emu/src/80_gemmini_loop_ws.rs b/examples/components/balls/gemmini/emu/src/80_gemmini_loop_ws.rs new file mode 100644 index 000000000..81c100dde --- /dev/null +++ b/examples/components/balls/gemmini/emu/src/80_gemmini_loop_ws.rs @@ -0,0 +1,130 @@ +//===- 80_gemmini_loop_ws.rs - GEMMINI_LOOP_WS instruction -----------------===// + +use bebop_bemu::thread::banks::{bank_row_bytes, MATRIX_SIZE}; +use super::gemmini_state::gemini; +use super::instruction::ExecContext; +use super::loop_micro_ops::{ + alloc, checked_stride, compute, free_after_hash, mvin, mvout, preload, +}; + +// Shared implementation +fn exec_cfg_impl(mnemonic: &str, xs2: u64, ctx: &mut ExecContext) -> u64 { + let fields = ["BOUNDS", "ADDR_A", "ADDR_B", "ADDR_D", "ADDR_C", "STRIDES_AB", "STRIDES_DC"]; + let field = fields.iter().position(|field| mnemonic == format!("GEMMINI_LOOP_WS_CONFIG_{field}")).expect("loop configuration instruction"); + ctx.write_metadata_word(field * 8, xs2); + 0 +} + +fn checked_tile_addr( + base: u64, + outer: u64, + outer_stride: u64, + inner: u64, + inner_stride: u64, +) -> u64 { + let outer_offset = outer + .checked_mul(MATRIX_SIZE as u64).and_then(|row| row.checked_mul(outer_stride)) + .expect("gemmini_loop_ws: outer address offset overflow"); + let inner_offset = inner + .checked_mul(inner_stride) + .expect("gemmini_loop_ws: inner address offset overflow"); + base.checked_add(outer_offset) + .and_then(|addr| addr.checked_add(inner_offset)) + .expect("gemmini_loop_ws: address overflow") +} + +fn exec_loop_impl(xs2: u64, ctx: &mut ExecContext) -> u64 { + let bounds = ctx.metadata_word(0); + let ab = ctx.metadata_word(40); + let dc = ctx.metadata_word(48); + let lw = super::gemmini_state::LoopWsCfg { + max_i: bounds >> 32 & 0xffff, max_j: bounds >> 16 & 0xffff, max_k: bounds & 0xffff, + addr_a: ctx.metadata_word(8), addr_b: ctx.metadata_word(16), addr_d: ctx.metadata_word(24), addr_c: ctx.metadata_word(32), + stride_a: ab & 0xffff_ffff, stride_b: ab >> 32, stride_d: dc & 0xffff_ffff, stride_c: dc >> 32, + }; + let dataflow = gemini(ctx).lock().unwrap().cfg.dataflow; + + let bank_a = xs2 & 0x3ff; + let bank_b = (xs2 >> 10) & 0x3ff; + let bank_c = (xs2 >> 20) & 0x3ff; + let low_d = ((xs2 >> 30) & 1) != 0; + assert!( + bank_a != bank_b && bank_a != bank_c && bank_b != bank_c, + "gemmini_loop_ws: banks must be distinct" + ); + assert!( + lw.max_i > 0 && lw.max_j > 0 && lw.max_k > 0, + "gemmini_loop_ws: bounds must be > 0" + ); + assert_eq!( + dataflow, 0, + "gemmini_loop_ws: current RTL CISC contract supports OS dataflow only" + ); + assert!( + low_d && lw.addr_d == 0 && lw.stride_d == 0, + "gemmini_loop_ws: only zero D is supported" + ); + + let dim = MATRIX_SIZE as u64; + let bank_bytes = bank_row_bytes() as u64; + let stride_a = checked_stride(lw.stride_a, bank_bytes, "gemmini_loop_ws stride_a"); + let stride_b = checked_stride(lw.stride_b, bank_bytes, "gemmini_loop_ws stride_b"); + let stride_c = checked_stride( + lw.stride_c, + bank_bytes + .checked_mul(4) + .expect("gemmini_loop_ws: output row width overflow"), + "gemmini_loop_ws stride_c", + ); + + alloc(ctx, bank_a, 1); + alloc(ctx, bank_b, 1); + alloc(ctx, bank_c, 4); + + mvin(ctx, bank_a, lw.addr_a, dim, stride_a); + mvin(ctx, bank_b, lw.addr_b, dim, stride_b); + + for i in 0..lw.max_i { + for j in 0..lw.max_j { + for k in 0..lw.max_k { + preload(ctx, bank_a, bank_c, dim); + compute(ctx, k != 0, bank_a, bank_b, bank_c, dim, false, false); + + let addr_c = checked_tile_addr(lw.addr_c, i, lw.stride_c, j, dim * 4); + mvout(ctx, bank_c, addr_c, dim, stride_c); + + let next = if k + 1 < lw.max_k { + Some((i, j, k + 1)) + } else if j + 1 < lw.max_j { + Some((i, j + 1, 0)) + } else if i + 1 < lw.max_i { + Some((i + 1, 0, 0)) + } else { + None + }; + if let Some((next_i, next_j, next_k)) = next { + let addr_a = checked_tile_addr(lw.addr_a, next_i, lw.stride_a, next_k, dim); + let addr_b = checked_tile_addr(lw.addr_b, next_k, lw.stride_b, next_j, dim); + mvin(ctx, bank_a, addr_a, dim, stride_a); + mvin(ctx, bank_b, addr_b, dim, stride_b); + } + } + } + } + + free_after_hash(ctx, bank_a); + free_after_hash(ctx, bank_b); + free_after_hash(ctx, bank_c); + 0 +} + +pub fn execute(mnemonic: &str, xs2: u64, ctx: &mut ExecContext) -> u64 { + match mnemonic { + "GEMMINI_LOOP_WS" => exec_loop_impl(xs2, ctx), + _ => exec_cfg_impl(mnemonic, xs2, ctx), + } +} + +pub fn latency(mnemonic: &str) -> u64 { + if mnemonic == "GEMMINI_LOOP_WS" { 256 } else { 1 } +} diff --git a/examples/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs b/examples/components/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs similarity index 80% rename from examples/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs rename to examples/components/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs index 36d72f72b..43c2318fe 100644 --- a/examples/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs +++ b/examples/components/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs @@ -1,6 +1,6 @@ //===- 96_gemmini_loop_conv_ws.rs - GEMMINI_LOOP_CONV_WS instruction -------===// -use super::super::bank::{bank_row_bytes, MATRIX_SIZE}; +use bebop_bemu::thread::banks::{bank_row_bytes, MATRIX_SIZE}; use super::gemmini_state::gemini; use super::instruction::ExecContext; use super::loop_micro_ops::{ @@ -8,37 +8,10 @@ use super::loop_micro_ops::{ }; // Shared implementation -fn exec_cfg_impl(mnemonic: &str, xs2: u64) -> u64 { - let mut g = gemini().lock().unwrap(); - match mnemonic { - "GEMMINI_LOOP_CONV_WS_CONFIG_1" => { - g.loop_conv.batch = xs2 & 0xffff; - g.loop_conv.in_dim = (xs2 >> 16) & 0xffff; - g.loop_conv.in_ch = (xs2 >> 32) & 0xffff; - } - "GEMMINI_LOOP_CONV_WS_CONFIG_2" => { - g.loop_conv.out_ch = xs2 & 0xffff; - g.loop_conv.out_dim = (xs2 >> 16) & 0xffff; - g.loop_conv.stride = (xs2 >> 32) & 0xff; - g.loop_conv.padding = (xs2 >> 40) & 0xff; - } - "GEMMINI_LOOP_CONV_WS_CONFIG_3" => { - g.loop_conv.kernel_dim = xs2 & 0xff; - g.loop_conv.pool_size = (xs2 >> 8) & 0xff; - g.loop_conv.pool_stride = (xs2 >> 16) & 0xff; - g.loop_conv.pool_padding = (xs2 >> 24) & 0xff; - } - "GEMMINI_LOOP_CONV_WS_CONFIG_4" => g.loop_conv.addr_bias = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_CONV_WS_CONFIG_5" => g.loop_conv.addr_input = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_CONV_WS_CONFIG_6" => g.loop_conv.addr_weight = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_CONV_WS_CONFIG_7" => g.loop_conv.addr_output = xs2 & ((1u64 << 39) - 1), - "GEMMINI_LOOP_CONV_WS_CONFIG_8" => { - g.loop_conv.input_stride = xs2 & 0xffff_ffff; - g.loop_conv.weight_stride = xs2 >> 32; - } - "GEMMINI_LOOP_CONV_WS_CONFIG_9" => g.loop_conv.output_stride = xs2 & 0xffff_ffff, - _ => panic!("gemmini_loop_conv_ws: unknown cfg mnemonic={mnemonic}"), - } +fn exec_cfg_impl(mnemonic: &str, xs2: u64, ctx: &mut ExecContext) -> u64 { + let field: usize = mnemonic.strip_prefix("GEMMINI_LOOP_CONV_WS_CONFIG_").expect("convolution configuration instruction").parse().unwrap(); + assert!((1..=9).contains(&field)); + ctx.write_metadata_word(64 + (field - 1) * 8, xs2); 0 } @@ -75,7 +48,7 @@ fn input_addr( let element = checked_add( checked_mul(spatial, st.in_ch, "input channels"), kch, - "input channel tile", + "input channel core", ); checked_add(st.addr_input, element, "input address") } @@ -138,8 +111,17 @@ fn output_addr( } fn exec_loop_impl(xs2: u64, ctx: &mut ExecContext) -> u64 { - let st = gemini().lock().unwrap().loop_conv.clone(); - let dataflow = gemini().lock().unwrap().cfg.dataflow; + let words: Vec<_> = (0..9).map(|i| ctx.metadata_word(64+i*8)).collect(); + let st = super::gemmini_state::LoopConvCfg { + batch: words[0] & 0xffff, in_dim: words[0] >> 16 & 0xffff, in_ch: words[0] >> 32 & 0xffff, + out_ch: words[1] & 0xffff, out_dim: words[1] >> 16 & 0xffff, + stride: words[1] >> 32 & 0xff, padding: words[1] >> 40 & 0xff, + kernel_dim: words[2] & 0xff, pool_size: words[2] >> 8 & 0xff, + pool_stride: words[2] >> 16 & 0xff, pool_padding: words[2] >> 24 & 0xff, + addr_bias: words[3], addr_input: words[4], addr_weight: words[5], addr_output: words[6], + input_stride: words[7] & 0xffff_ffff, weight_stride: words[7] >> 32, output_stride: words[8] & 0xffff_ffff, + }; + let dataflow = gemini(ctx).lock().unwrap().cfg.dataflow; let bank_input = xs2 & 0x3ff; let bank_weight = (xs2 >> 10) & 0x3ff; let bank_output = (xs2 >> 20) & 0x3ff; @@ -194,7 +176,7 @@ fn exec_loop_impl(xs2: u64, ctx: &mut ExecContext) -> u64 { let weight_stride = checked_stride( st.out_ch, bank_bytes, - "gemmini_loop_conv_ws weight tile stride", + "gemmini_loop_conv_ws weight core stride", ); let output_stride = checked_stride( st.output_stride, @@ -275,7 +257,7 @@ fn exec_loop_impl(xs2: u64, ctx: &mut ExecContext) -> u64 { pub fn execute(mnemonic: &str, xs2: u64, ctx: &mut ExecContext) -> u64 { match mnemonic { "GEMMINI_LOOP_CONV_WS" => exec_loop_impl(xs2, ctx), - _ => exec_cfg_impl(mnemonic, xs2), + _ => exec_cfg_impl(mnemonic, xs2, ctx), } } diff --git a/examples/components/balls/gemmini/emu/src/dispatch.rs b/examples/components/balls/gemmini/emu/src/dispatch.rs new file mode 100644 index 000000000..d59051add --- /dev/null +++ b/examples/components/balls/gemmini/emu/src/dispatch.rs @@ -0,0 +1,58 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::{ + f02_gemmini_config, f03_gemmini_flush, f21_gemmini_preload, f66_gemmini_compute_preloaded, + f67_gemmini_compute_accumulated, f80_gemmini_loop_ws, f96_gemmini_loop_conv_ws, +}; + +const BALL_CLASS: &str = "examples.balls.gemmini.GemminiBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + let mnemonic = bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct)?; + let result = match mnemonic.as_str() { + "GEMMINI_CONFIG" => f02_gemmini_config::GemminiConfig::exec(xs1, xs2, ctx), + "GEMMINI_FLUSH" => f03_gemmini_flush::GemminiFlush::exec(xs1, xs2, ctx), + "GEMMINI_PRELOAD" => f21_gemmini_preload::GemminiPreload::exec(xs1, xs2, ctx), + "GEMMINI_COMPUTE_PRELOADED" => { + f66_gemmini_compute_preloaded::GemminiComputePreloaded::exec(xs1, xs2, ctx) + } + "GEMMINI_COMPUTE_ACCUMULATED" => { + f67_gemmini_compute_accumulated::GemminiComputeAccumulated::exec(xs1, xs2, ctx) + } + name if name.starts_with("GEMMINI_LOOP_WS_") || name == "GEMMINI_LOOP_WS" => { + f80_gemmini_loop_ws::execute(name, xs2, ctx) + } + name if name.starts_with("GEMMINI_LOOP_CONV_WS_") || name == "GEMMINI_LOOP_CONV_WS" => { + f96_gemmini_loop_conv_ws::execute(name, xs2, ctx) + } + _ => return None, + }; + Some(result) +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + let mnemonic = bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct)?; + let latency = match mnemonic.as_str() { + "GEMMINI_CONFIG" => f02_gemmini_config::GemminiConfig::latency(xs1, xs2), + "GEMMINI_FLUSH" => f03_gemmini_flush::GemminiFlush::latency(xs1, xs2), + "GEMMINI_PRELOAD" => f21_gemmini_preload::GemminiPreload::latency(xs1, xs2), + "GEMMINI_COMPUTE_PRELOADED" => f66_gemmini_compute_preloaded::GemminiComputePreloaded::latency(xs1, xs2), + "GEMMINI_COMPUTE_ACCUMULATED" => f67_gemmini_compute_accumulated::GemminiComputeAccumulated::latency(xs1, xs2), + name if name.starts_with("GEMMINI_LOOP_WS_") || name == "GEMMINI_LOOP_WS" => f80_gemmini_loop_ws::latency(name), + name if name.starts_with("GEMMINI_LOOP_CONV_WS_") || name == "GEMMINI_LOOP_CONV_WS" => f96_gemmini_loop_conv_ws::latency(name), + _ => return None, + }; + Some(latency) +} diff --git a/examples/balls/gemmini/emu/src/gemmini_state.rs b/examples/components/balls/gemmini/emu/src/gemmini_state.rs similarity index 85% rename from examples/balls/gemmini/emu/src/gemmini_state.rs rename to examples/components/balls/gemmini/emu/src/gemmini_state.rs index 9038fba23..615eae5bd 100644 --- a/examples/balls/gemmini/emu/src/gemmini_state.rs +++ b/examples/components/balls/gemmini/emu/src/gemmini_state.rs @@ -1,5 +1,6 @@ -//! Global config for Gemmini / loop instructions. -use std::sync::{Mutex, OnceLock}; +//! State owned by one Gemmini Ball model instance. +use std::sync::{Arc, Mutex}; +use super::instruction::ExecContext; #[derive(Clone, Default)] pub struct GemminiCfg { @@ -49,16 +50,12 @@ pub struct LoopConvCfg { #[derive(Default)] pub struct GemminiState { pub cfg: GemminiCfg, - pub loop_ws: LoopWsCfg, - pub loop_conv: LoopConvCfg, /// WS preload: B weights (iter × 16) i8 pub ws_b: Option>>, } -static GEMINI: OnceLock> = OnceLock::new(); - -pub fn gemini() -> &'static Mutex { - GEMINI.get_or_init(|| Mutex::new(GemminiState::default())) +pub fn gemini(ctx: &mut ExecContext) -> Arc> { + ctx.state::() } pub fn in_shift(v: i32, shift: u32) -> i32 { diff --git a/examples/components/balls/gemmini/emu/src/lib.rs b/examples/components/balls/gemmini/emu/src/lib.rs new file mode 100644 index 000000000..c0a788641 --- /dev/null +++ b/examples/components/balls/gemmini/emu/src/lib.rs @@ -0,0 +1,26 @@ +pub(crate) use bebop_bemu::{thread::{instructions::decode, bank_matrix}, ball as instruction}; + +#[path = "02_gemmini_config.rs"] +mod f02_gemmini_config; +#[path = "03_gemmini_flush.rs"] +mod f03_gemmini_flush; +#[path = "21_gemmini_preload.rs"] +mod f21_gemmini_preload; +#[path = "66_gemmini_compute_preloaded.rs"] +mod f66_gemmini_compute_preloaded; +#[path = "67_gemmini_compute_accumulated.rs"] +mod f67_gemmini_compute_accumulated; +#[path = "80_gemmini_loop_ws.rs"] +mod f80_gemmini_loop_ws; +#[path = "96_gemmini_loop_conv_ws.rs"] +mod f96_gemmini_loop_conv_ws; +mod gemmini_state; +mod loop_micro_ops; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/balls/gemmini/emu/src/loop_micro_ops.rs b/examples/components/balls/gemmini/emu/src/loop_micro_ops.rs similarity index 96% rename from examples/balls/gemmini/emu/src/loop_micro_ops.rs rename to examples/components/balls/gemmini/emu/src/loop_micro_ops.rs index 2363ff88f..90e5d78ce 100644 --- a/examples/balls/gemmini/emu/src/loop_micro_ops.rs +++ b/examples/components/balls/gemmini/emu/src/loop_micro_ops.rs @@ -1,8 +1,8 @@ use super::decode; -use super::f53_gemmini_preload::GemminiPreload; +use super::f21_gemmini_preload::GemminiPreload; use super::f66_gemmini_compute_preloaded::GemminiComputePreloaded; use super::f67_gemmini_compute_accumulated::GemminiComputeAccumulated; -use super::instruction::{BallInstruction, ExecContext}; +use super::instruction::{BallModel, ExecContext}; const FUNCT_MVOUT: u32 = 16; const FUNCT_MSET: u32 = 32; diff --git a/examples/components/balls/gemmini/stack/src/Lowering.cpp b/examples/components/balls/gemmini/stack/src/Lowering.cpp new file mode 100644 index 000000000..735b1b3ce --- /dev/null +++ b/examples/components/balls/gemmini/stack/src/Lowering.cpp @@ -0,0 +1,101 @@ +//===- Lowering.cpp - GemminiBall in lower-buckyball ----------------------===// +// +// matmul_i8_tile: C = A @ B for one 16x16 tile. A holds sixteen rows of +// sixteen i8 (M x K) at row a_base, B sixteen rows of sixteen i8 (K x N) at +// row b_base, and C, a bank of four columns, receives sixteen rows of sixteen +// i32 at row c_base. a_base, b_base and c_base are the i64 arguments. +// With read_destination C = in_shift(C + A @ B, shift), rounding to nearest +// even; without, C = A @ B on the output-stationary array. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballOps.h" +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" +#include "mlir/Dialect/Arith/IR/Arith.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class MatmulI8TileLowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &b) const override { + if (op.getKind() != "matmul_i8_tile") + return failure(); + Location loc = op.getLoc(); + bool accumulate = op.getReadDestination(); + int64_t shift = + accumulate ? cast(op->getAttr("shift")).getInt() : 0; + int64_t meshRows = buckyball_target::getBuckyballBallParam( + "GEMMINI_COMPUTE_PRELOADED", "meshRows"); + int64_t meshColumns = buckyball_target::getBuckyballBallParam( + "GEMMINI_COMPUTE_PRELOADED", "meshColumns"); + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || meshRows != 16 || meshColumns != 16 || + target.bankDepth < 16 || op.getSources().size() != 2 || + op.getArguments().size() != 3 || + op.getSources()[0] == op.getSources()[1] || + llvm::is_contained(op.getSources(), op.getDestination())) + return op.emitOpError("matmul_i8_tile requires distinct A, B and C " + "banks, three row bases and a 16x16 mesh"); + int64_t a = op.getSources()[0], bBank = op.getSources()[1]; + int64_t c = op.getDestination(); + // rs2: the subcommand in [3:0], then the A, B and C row bases. + auto rs2 = [&](uint64_t subcommand, Value aBase, Value bBase) { + Value word = mlir::buddy::constantI64(b, loc, subcommand); + for (auto [base, bit] : {std::pair{aBase, 6}, std::pair{bBase, 16}, + std::pair{op.getArguments()[2], 26}}) + if (base) + word = arith::OrIOp::create( + b, loc, word, + arith::ShLIOp::create(b, loc, base, + mlir::buddy::constantI64(b, loc, bit))); + return word; + }; + Value aBase = op.getArguments()[0], bBase = op.getArguments()[1]; + // CONFIG: weight-stationary for the accumulation, as the ISA tests use it, + // output-stationary for the plain product. + mlir::buddy::emitInstruction( + b, loc, buckyball_target::getBuckyballFunct7("GEMMINI_CONFIG"), + mlir::buddy::constantI64(b, loc, 0), + mlir::buddy::constantI64(b, loc, + mlir::buddy::field(accumulate, 4, 4) | + mlir::buddy::field(shift, 9, 40))); + if (accumulate) { + mlir::buddy::emitInstruction( + b, loc, + buckyball_target::getBuckyballFunct7("GEMMINI_COMPUTE_ACCUMULATED"), + mlir::buddy::constantI64( + b, loc, mlir::buddy::bankFields({a, bBank}, c, true, 16)), + rs2(3, aBase, bBase)); + b.eraseOp(op); + return success(); + } + // The output-stationary preload only primes the array with D = 0; it + // reads B, which is valid for sixteen rows. + mlir::buddy::emitInstruction( + b, loc, buckyball_target::getBuckyballFunct7("GEMMINI_PRELOAD"), + mlir::buddy::constantI64( + b, loc, mlir::buddy::bankFields({bBank}, c, false, 16)), + rs2(1, bBase, Value())); + mlir::buddy::emitInstruction( + b, loc, + buckyball_target::getBuckyballFunct7("GEMMINI_COMPUTE_PRELOADED"), + mlir::buddy::constantI64( + b, loc, mlir::buddy::bankFields({a, bBank}, c, false, 16)), + rs2(2, aBase, bBase)); + b.eraseOp(op); + return success(); + } +}; + +void populate(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} + +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallLowering(populate), true); +} // namespace diff --git a/examples/balls/gemmini/verify/casegen/Cargo.lock b/examples/components/balls/gemmini/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/gemmini/verify/casegen/Cargo.lock rename to examples/components/balls/gemmini/verify/casegen/Cargo.lock diff --git a/examples/balls/gemmini/verify/casegen/Cargo.toml b/examples/components/balls/gemmini/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/gemmini/verify/casegen/Cargo.toml rename to examples/components/balls/gemmini/verify/casegen/Cargo.toml diff --git a/examples/balls/gemmini/verify/casegen/src/lib.rs b/examples/components/balls/gemmini/verify/casegen/src/lib.rs similarity index 100% rename from examples/balls/gemmini/verify/casegen/src/lib.rs rename to examples/components/balls/gemmini/verify/casegen/src/lib.rs diff --git a/examples/balls/gemmini/verify/filelists/gemmini_ball.f b/examples/components/balls/gemmini/verify/filelists/gemmini_ball.f similarity index 97% rename from examples/balls/gemmini/verify/filelists/gemmini_ball.f rename to examples/components/balls/gemmini/verify/filelists/gemmini_ball.f index c4b12100e..187506b5e 100644 --- a/examples/balls/gemmini/verify/filelists/gemmini_ball.f +++ b/examples/components/balls/gemmini/verify/filelists/gemmini_ball.f @@ -9,7 +9,7 @@ @RTL@/AlwaysOutTransposer.sv @RTL@/MacUnit.sv @RTL@/PE_256.sv -@RTL@/Tile.sv +@RTL@/Core.sv @RTL@/Mesh.sv @RTL@/TagQueue.sv @RTL@/ram_6x16.sv diff --git a/examples/balls/gemmini/verify/src/common/gemmini_defs.svh b/examples/components/balls/gemmini/verify/src/common/gemmini_defs.svh similarity index 79% rename from examples/balls/gemmini/verify/src/common/gemmini_defs.svh rename to examples/components/balls/gemmini/verify/src/common/gemmini_defs.svh index c2e9a59da..bf2542620 100644 --- a/examples/balls/gemmini/verify/src/common/gemmini_defs.svh +++ b/examples/components/balls/gemmini/verify/src/common/gemmini_defs.svh @@ -1,3 +1,3 @@ `ifndef GEMMINI_LOOP_WS_CONFIG_BOUNDS_FUNCT7 -`error "GEMMINI_LOOP_WS_CONFIG_BOUNDS_FUNCT7 must be provided by the selected Core ballISA" +`error "GEMMINI_LOOP_WS_CONFIG_BOUNDS_FUNCT7 must be provided by the selected Thread ballISA" `endif diff --git a/examples/balls/gemmini/verify/src/pkg/gemmini_pkg.sv b/examples/components/balls/gemmini/verify/src/pkg/gemmini_pkg.sv similarity index 100% rename from examples/balls/gemmini/verify/src/pkg/gemmini_pkg.sv rename to examples/components/balls/gemmini/verify/src/pkg/gemmini_pkg.sv diff --git a/examples/components/balls/gemmini/verify/src/tb_top.sv b/examples/components/balls/gemmini/verify/src/tb_top.sv new file mode 100644 index 000000000..a4b30301e --- /dev/null +++ b/examples/components/balls/gemmini/verify/src/tb_top.sv @@ -0,0 +1,81 @@ +module tb_top; + import uvm_pkg::*; + import gemmini_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + GemminiBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .io_bankAccessReady(1'b1), + .io_cmdReq_ready(intf.cmd_req_ready), + .io_cmdReq_valid(intf.cmd_req_valid), + .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), + .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), + .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), + .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), + .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), + .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), + .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), + .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), + .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), + .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), + .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), + .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), + .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), + .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), + .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), + .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), + .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), + .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), + .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), + .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), + .io_cmdResp_ready(intf.cmd_resp_ready), + .io_cmdResp_valid(intf.cmd_resp_valid), + .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), + .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), + .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .io_bankWrite_1_io_req_ready(intf.bank_write_req_ready[1]), + .io_bankWrite_1_io_req_valid(intf.bank_write_req_valid[1]), + .io_bankWrite_1_io_resp_ready(intf.bank_write_resp_ready[1]), + .io_bankWrite_1_io_resp_valid(intf.bank_write_resp_valid[1]), + .io_bankWrite_1_io_resp_bits_ok(intf.bank_write_resp_ok[1]), + .io_bankWrite_2_io_req_ready(intf.bank_write_req_ready[2]), + .io_bankWrite_2_io_req_valid(intf.bank_write_req_valid[2]), + .io_bankWrite_2_io_resp_ready(intf.bank_write_resp_ready[2]), + .io_bankWrite_2_io_resp_valid(intf.bank_write_resp_valid[2]), + .io_bankWrite_2_io_resp_bits_ok(intf.bank_write_resp_ok[2]), + .io_bankWrite_3_io_req_ready(intf.bank_write_req_ready[3]), + .io_bankWrite_3_io_req_valid(intf.bank_write_req_valid[3]), + .io_bankWrite_3_io_resp_ready(intf.bank_write_resp_ready[3]), + .io_bankWrite_3_io_resp_valid(intf.bank_write_resp_valid[3]), + .io_bankWrite_3_io_resp_bits_ok(intf.bank_write_resp_ok[3]), + .io_subRobReq_ready(intf.sub_rob_req_ready) + ); + + initial begin + intf.clock = 1'b0; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("gemmini_ball_test"); + end +endmodule diff --git a/examples/balls/gemmini/verify/src/tests/gemmini_test.svh b/examples/components/balls/gemmini/verify/src/tests/gemmini_test.svh similarity index 100% rename from examples/balls/gemmini/verify/src/tests/gemmini_test.svh rename to examples/components/balls/gemmini/verify/src/tests/gemmini_test.svh diff --git a/examples/components/balls/gemmini/workloads/ctests/CMakeLists.txt b/examples/components/balls/gemmini/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..50f8d0748 --- /dev/null +++ b/examples/components/balls/gemmini/workloads/ctests/CMakeLists.txt @@ -0,0 +1,16 @@ +add_buckyball_ctest(gemmini_loop_subrob.c) +add_buckyball_ctest(gemmini_loop_mmio.c) +add_buckyball_ctest(gemmini_accumulated_16x16_test.c) +add_buckyball_ctest(gemmini_os_cisc_atranspose_test.c) +add_buckyball_ctest(gemmini_os_cisc_basic_test.c) +add_buckyball_ctest(gemmini_os_cisc_btranspose_test.c) +add_buckyball_ctest(gemmini_os_cisc_shift_test.c) +add_buckyball_ctest(gemmini_os_risc_abtranspose_test.c) +add_buckyball_ctest(gemmini_os_risc_atranspose_test.c) +add_buckyball_ctest(gemmini_os_risc_basic_test.c) +add_buckyball_ctest(gemmini_os_risc_btranspose_test.c) +add_buckyball_ctest(gemmini_os_risc_shift_test.c) +add_buckyball_ctest(gemmini_ws_cisc_conv_test.c) +add_buckyball_ctest(gemmini_ws_risc_basic_test.c) +add_buckyball_ctest(gemmini_ws_risc_btranspose_test.c) +add_buckyball_ctest(gemmini_ws_risc_shift_test.c) diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_accumulated_16x16_test.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_accumulated_16x16_test.c similarity index 100% rename from examples/balls/gemmini/workloads/ctests/gemmini_accumulated_16x16_test.c rename to examples/components/balls/gemmini/workloads/ctests/gemmini_accumulated_16x16_test.c diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_loop_mmio.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_loop_mmio.c new file mode 100644 index 000000000..2c5fb0910 --- /dev/null +++ b/examples/components/balls/gemmini/workloads/ctests/gemmini_loop_mmio.c @@ -0,0 +1,56 @@ +#include +#include +#include +#include + +#define DIM (BANK_WIDTH / 8) +#define M DIM +#define N (2 * DIM) +#define K (2 * DIM) +static int8_t a[M * K] __attribute__((aligned(64))); +static int8_t b[K * N] __attribute__((aligned(64))); +static int32_t c[M * N] __attribute__((aligned(64))); +static int32_t expected[9][7]; +_Static_assert(DIM == 16, + "Gemmini loop tests require the configured 16x16 array"); +static uint64_t descriptor[8] __attribute__((aligned(64))); + +int main(void) { + for (int i = 0; i < M; ++i) + for (int k = 0; k < K; ++k) + a[i * K + k] = (i + 3 * k) % 9 - 4; + for (int k = 0; k < K; ++k) + for (int j = 0; j < N; ++j) + b[k * N + j] = (2 * k + 5 * j) % 7 - 3; + for (int i = 0; i < M * N; ++i) + c[i] = 0x2468ace0; + descriptor[0] = + ((uint64_t)(M / DIM) << 32) | ((uint64_t)(N / DIM) << 16) | (K / DIM); + descriptor[1] = (uintptr_t)a; + descriptor[2] = (uintptr_t)b; + descriptor[3] = 0; + descriptor[4] = (uintptr_t)c; + descriptor[5] = ((uint64_t)N << 32) | K; + descriptor[6] = (uint64_t)(N * 4) << 32; + descriptor[7] = 0; + bb_gemmini_config(0, 0, 0, 0, 0); + bb_mvin_mmio((uintptr_t)descriptor, 0, sizeof(descriptor) / (BANK_WIDTH / 8), + BANK_WIDTH / 8); + bb_gemmini_loop_ws(0, 1, 2, 1); + bb_barrier(); + asm volatile("fence" ::: "memory"); + // Input rows/columns repeat with these periods; every output is still + // checked. + for (int i = 0; i < 9; ++i) + for (int j = 0; j < 7; ++j) { + int32_t sum = 0; + for (int k = 0; k < K; ++k) + sum += a[i * K + k] * b[k * N + j]; + expected[i][j] = sum; + } + for (int i = 0; i < M; ++i) + for (int j = 0; j < N; ++j) + if (c[i * N + j] != expected[i % 9][j % 7]) + return i * N + j + 1; + return 0; +} diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_loop_subrob.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_loop_subrob.c new file mode 100644 index 000000000..93aac9a74 --- /dev/null +++ b/examples/components/balls/gemmini/workloads/ctests/gemmini_loop_subrob.c @@ -0,0 +1,51 @@ +#include +#include +#include +#include + +#define DIM (BANK_WIDTH / 8) +#define M (2 * DIM) +#define N DIM +#define K (2 * DIM) +static int8_t a[M * K] __attribute__((aligned(64))); +static int8_t b[K * N] __attribute__((aligned(64))); +static int32_t c[M * N] __attribute__((aligned(64))); +static int32_t expected[11][13]; +_Static_assert(DIM == 16, + "Gemmini loop tests require the configured 16x16 array"); + +int main(void) { + for (int i = 0; i < M; ++i) + for (int k = 0; k < K; ++k) + a[i * K + k] = (i * 3 + k * 5) % 11 - 5; + for (int k = 0; k < K; ++k) + for (int j = 0; j < N; ++j) + b[k * N + j] = (k * 7 + j * 2) % 13 - 6; + for (int i = 0; i < M * N; ++i) + c[i] = 0x13579bdf; + bb_gemmini_config(0, 0, 0, 0, 0); + bb_gemmini_loop_ws_config_bounds(M / DIM, N / DIM, K / DIM); + bb_gemmini_loop_ws_config_addr_a((uintptr_t)a); + bb_gemmini_loop_ws_config_addr_b((uintptr_t)b); + bb_gemmini_loop_ws_config_addr_d(0); + bb_gemmini_loop_ws_config_addr_c((uintptr_t)c); + bb_gemmini_loop_ws_config_strides_ab(K, N); + bb_gemmini_loop_ws_config_strides_dc(0, N * 4); + bb_gemmini_loop_ws(0, 1, 2, 1); + bb_barrier(); + asm volatile("fence" ::: "memory"); + // Input rows/columns repeat with these periods; every output is still + // checked. + for (int i = 0; i < 11; ++i) + for (int j = 0; j < 13; ++j) { + int32_t sum = 0; + for (int k = 0; k < K; ++k) + sum += a[i * K + k] * b[k * N + j]; + expected[i][j] = sum; + } + for (int i = 0; i < M; ++i) + for (int j = 0; j < N; ++j) + if (c[i * N + j] != expected[i % 11][j % 13]) + return i * N + j + 1; + return 0; +} diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c rename to examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c index 386f7e5df..035f45a71 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c +++ b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS CISC a_transpose Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c similarity index 96% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c rename to examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c index dd06b8091..d0ec3420a 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c +++ b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c @@ -12,8 +12,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS CISC Basic Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c rename to examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c index 02e0192aa..9ec83f029 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c +++ b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS CISC b_transpose Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c similarity index 96% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c rename to examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c index 4e90e3b98..3dc388b81 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c +++ b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS CISC in_shift Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c rename to examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c index 7de942431..355e18581 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c +++ b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c @@ -14,8 +14,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS RISC ab_transpose Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c rename to examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c index 6d987dc52..c24289288 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c +++ b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS RISC a_transpose Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c rename to examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c index c5253f933..159916f7e 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c +++ b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS RISC Basic Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c rename to examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c index 3b89e5d70..9ca10346b 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c +++ b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS RISC b_transpose Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c rename to examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c index 04cb0de19..a2c1a11f7 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c +++ b/examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini OS RISC in_shift Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c similarity index 98% rename from examples/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c rename to examples/components/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c index d3d090cb4..1aa582aa3 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c +++ b/examples/components/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c @@ -29,8 +29,8 @@ static result_t expected[BATCH * OUT_DIM * OUT_DIM * OUT_CH] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini WS CISC Loop Conv Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c rename to examples/components/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c index 7d12fe127..f3a9174da 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c +++ b/examples/components/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c @@ -13,8 +13,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini WS RISC Basic Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c rename to examples/components/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c index 5e0404ee3..f964ab5da 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c +++ b/examples/components/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c @@ -14,8 +14,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini WS RISC b_transpose Test ===\n"); diff --git a/examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c b/examples/components/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c similarity index 97% rename from examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c rename to examples/components/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c index 2d84dbdb1..f53a09ab2 100644 --- a/examples/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c +++ b/examples/components/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c @@ -14,8 +14,8 @@ static result_t mat_c[DIM * DIM] __attribute__((aligned(64))); static result_t expected[DIM * DIM] __attribute__((aligned(64))); int main() { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Gemmini WS RISC in_shift Test ===\n"); diff --git a/examples/balls/gemmini/workloads/isa/gemmini.h b/examples/components/balls/gemmini/workloads/isa/gemmini.h similarity index 100% rename from examples/balls/gemmini/workloads/isa/gemmini.h rename to examples/components/balls/gemmini/workloads/isa/gemmini.h diff --git a/examples/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h b/examples/components/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h similarity index 93% rename from examples/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h rename to examples/components/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h index d3fda03ec..6d0c03a5e 100644 --- a/examples/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h +++ b/examples/components/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h @@ -15,7 +15,7 @@ iter, op1_base, op2_base, wr_base) \ BUCKYBALL_INSTRUCTION_R_R( \ (BB_BANK0(op1_bank_id) | BB_BANK1(op2_bank_id) | BB_BANK2(wr_bank_id) | \ - BB_ITER(iter)), \ + BB_ITER(iter) | BB_READ_WR_OLD), \ BB_GEMMINI_COMPUTE_ACCUMULATED_RS2 | ((uint64_t)(op1_base) << 6) | \ ((uint64_t)(op2_base) << 16) | ((uint64_t)(wr_base) << 26), \ BB_FUNC7(GEMMINI_COMPUTE_ACCUMULATED)) diff --git a/examples/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h b/examples/components/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h similarity index 100% rename from examples/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h rename to examples/components/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h diff --git a/examples/balls/gemmini/workloads/isa/gemmini_config.h b/examples/components/balls/gemmini/workloads/isa/gemmini_config.h similarity index 100% rename from examples/balls/gemmini/workloads/isa/gemmini_config.h rename to examples/components/balls/gemmini/workloads/isa/gemmini_config.h diff --git a/examples/balls/gemmini/workloads/isa/gemmini_flush.h b/examples/components/balls/gemmini/workloads/isa/gemmini_flush.h similarity index 100% rename from examples/balls/gemmini/workloads/isa/gemmini_flush.h rename to examples/components/balls/gemmini/workloads/isa/gemmini_flush.h diff --git a/examples/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h b/examples/components/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h similarity index 100% rename from examples/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h rename to examples/components/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h diff --git a/examples/balls/gemmini/workloads/isa/gemmini_loop_ws.h b/examples/components/balls/gemmini/workloads/isa/gemmini_loop_ws.h similarity index 100% rename from examples/balls/gemmini/workloads/isa/gemmini_loop_ws.h rename to examples/components/balls/gemmini/workloads/isa/gemmini_loop_ws.h diff --git a/examples/balls/gemmini/workloads/isa/gemmini_preload.h b/examples/components/balls/gemmini/workloads/isa/gemmini_preload.h similarity index 100% rename from examples/balls/gemmini/workloads/isa/gemmini_preload.h rename to examples/components/balls/gemmini/workloads/isa/gemmini_preload.h diff --git a/examples/balls/gemmini/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/gemmini/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/gemmini/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/gemmini/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/gemmini/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/gemmini/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/gemmini/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/gemmini/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir b/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir new file mode 100644 index 000000000..0dde99a71 --- /dev/null +++ b/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir @@ -0,0 +1,52 @@ +func.func private @check_result(memref<16x16xi32>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %initial = arith.constant 7 : i32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %sixteen = arith.constant 16 : index + %base = arith.constant 0 : i64 + %a = memref.alloc() : memref<16x16xi8> + %b = memref.alloc() : memref<16x16xi8> + %c = memref.alloc() : memref<16x16xi32> + linalg.fill ins(%initial : i32) outs(%c : memref<16x16xi32>) + scf.for %row = %zero to %sixteen step %one { + scf.for %col = %zero to %sixteen step %one { + %r3 = arith.muli %row, %three : index + %c5 = arith.muli %col, %five : index + %asum = arith.addi %r3, %c5 : index + %arem = arith.remui %asum, %seven : index + %aval = arith.subi %arem, %three : index + %av = arith.index_cast %aval : index to i8 + %r2 = arith.muli %row, %two : index + %c3 = arith.muli %col, %three : index + %bsum = arith.addi %r2, %c3 : index + %brem = arith.remui %bsum, %five : index + %bval = arith.subi %brem, %two : index + %bv = arith.index_cast %bval : index to i8 + memref.store %av, %a[%row, %col] : memref<16x16xi8> + memref.store %bv, %b[%row, %col] : memref<16x16xi8> + } + } + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 4 : i64}> + buckyball.mvin %a <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %b <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + // C = in_shift(C + A @ B, 4). + buckyball.custom "matmul_i8_tile" args(%base, %base, %base : i64, i64, i64) <{sources = array, destination = 3 : i64, read_destination}> {shift = 4 : i64} + buckyball.mvout %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + func.call @check_result(%c) : (memref<16x16xi32>) -> () + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %a : memref<16x16xi8> + memref.dealloc %b : memref<16x16xi8> + memref.dealloc %c : memref<16x16xi32> + return %zero_i8 : i8 +} diff --git a/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16_main.cpp b/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16_main.cpp similarity index 100% rename from examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16_main.cpp rename to examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16_main.cpp diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir b/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir new file mode 100644 index 000000000..b825b9083 --- /dev/null +++ b/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir @@ -0,0 +1,51 @@ +func.func private @check_result(memref<16x16xi32>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %sixteen = arith.constant 16 : index + %base = arith.constant 0 : i64 + %a = memref.alloc() : memref<16x16xi8> + %b = memref.alloc() : memref<16x16xi8> + %c = memref.alloc() : memref<16x16xi32> + linalg.fill ins(%zero_i32 : i32) outs(%c : memref<16x16xi32>) + scf.for %row = %zero to %sixteen step %one { + scf.for %col = %zero to %sixteen step %one { + %r3 = arith.muli %row, %three : index + %c5 = arith.muli %col, %five : index + %asum = arith.addi %r3, %c5 : index + %arem = arith.remui %asum, %seven : index + %aval = arith.subi %arem, %three : index + %av = arith.index_cast %aval : index to i8 + %r2 = arith.muli %row, %two : index + %c3 = arith.muli %col, %three : index + %bsum = arith.addi %r2, %c3 : index + %brem = arith.remui %bsum, %five : index + %bval = arith.subi %brem, %two : index + %bv = arith.index_cast %bval : index to i8 + memref.store %av, %a[%row, %col] : memref<16x16xi8> + memref.store %bv, %b[%row, %col] : memref<16x16xi8> + } + } + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 4 : i64}> + buckyball.mvin %a <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %b <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + // C = A @ B on the output-stationary array. + buckyball.custom "matmul_i8_tile" args(%base, %base, %base : i64, i64, i64) <{sources = array, destination = 3 : i64}> + buckyball.mvout %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + func.call @check_result(%c) : (memref<16x16xi32>) -> () + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %a : memref<16x16xi8> + memref.dealloc %b : memref<16x16xi8> + memref.dealloc %c : memref<16x16xi32> + return %zero_i8 : i8 +} diff --git a/examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16_main.cpp b/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16_main.cpp similarity index 100% rename from examples/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16_main.cpp rename to examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16_main.cpp diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..7bc2d80e5 --- /dev/null +++ b/examples/components/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,13 @@ +if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +endif() +set(BUCKYBALL_MLIR_TEST_PREFIX bank) +set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") +file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") +string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") +if(_lines LESS 16) + message(FATAL_ERROR + "Gemmini BankOp test requires BANK_LINES >= 16 (got ${_lines})") +endif() +add_buckyball_mlir_test(gemmini_compute_preloaded_bank_tiles TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) +add_buckyball_mlir_test(gemmini_compute_accumulated_bank_tiles TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir.in b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir.in new file mode 100644 index 000000000..75c15d6f6 --- /dev/null +++ b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir.in @@ -0,0 +1,63 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x16xi32>) -> () +func.func @main() -> i8 { + %z8 = arith.constant 0 : i8 + %z32 = arith.constant 0 : i32 + %z = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %dim = arith.constant 16 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %zero64 = arith.constant 0 : i64 + %a = memref.alloc() : memref<@BB_BANK_LINES@x16xi8> + %b = memref.alloc() : memref<16x16xi8> + %c = memref.alloc() : memref<@BB_BANK_LINES@x16xi32> + linalg.fill ins(%z32 : i32) outs(%c : memref<@BB_BANK_LINES@x16xi32>) + scf.for %tile = %z to %rows step %dim { + scf.for %row = %z to %dim step %one { + %arow = arith.addi %tile, %row : index + scf.for %k = %z to %dim step %one { + %r3 = arith.muli %row, %three : index + %k5 = arith.muli %k, %five : index + %sum = arith.addi %r3, %k5 : index + %rem = arith.remui %sum, %seven : index + %value = arith.subi %rem, %three : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %a[%arow, %k] : memref<@BB_BANK_LINES@x16xi8> + } + } + } + scf.for %row = %z to %dim step %one { + scf.for %col = %z to %dim step %one { + %r2 = arith.muli %row, %two : index + %c3 = arith.muli %col, %three : index + %sum = arith.addi %r2, %c3 : index + %rem = arith.remui %sum, %five : index + %value = arith.subi %rem, %two : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %b[%row, %col] : memref<16x16xi8> + } + } + %ab = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %bb = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %cb = buckyball.bank_alloc <{row = 1 : i64, col = 4 : i64}> + %al = buckyball.bank_mvin %a, %ab <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x16xi8> + %bl = buckyball.bank_mvin %b, %bb <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + // Tile t of A (rows 16t..16t+15) times B into rows 16t..16t+15 of C. + scf.for %base = %z to %rows step %dim { + %base64 = arith.index_cast %base : index to i64 + %r = buckyball.bank_custom "matmul_i8_tile" (%al, %bl) -> %cb args(%base64, %zero64, %base64 : i64, i64, i64) + %sum = buckyball.bank_custom "matmul_i8_tile" (%al, %bl) -> %r args(%base64, %zero64, %base64 : i64, i64, i64) <{read_destination}> {shift = 0 : i64} + } + %stored = buckyball.bank_mvout %c, %cb <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x16xi32> + func.call @check_result(%c) : (memref<@BB_BANK_LINES@x16xi32>) -> () + buckyball.bank_release %al + buckyball.bank_release %bl + buckyball.bank_release %stored + memref.dealloc %a : memref<@BB_BANK_LINES@x16xi8> + memref.dealloc %b : memref<16x16xi8> + memref.dealloc %c : memref<@BB_BANK_LINES@x16xi32> + return %z8 : i8 +} diff --git a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp similarity index 93% rename from examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp rename to examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp index 35d8446b8..225063d92 100644 --- a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp +++ b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp @@ -20,5 +20,5 @@ extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, exit(1); } } - printf("gemmini compute-accumulated bank tiles PASS rows=%d\n", BANK_LINES); + printf("gemmini compute-accumulated bank cores PASS rows=%d\n", BANK_LINES); } diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir.in b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir.in new file mode 100644 index 000000000..b191e7234 --- /dev/null +++ b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir.in @@ -0,0 +1,62 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x16xi32>) -> () +func.func @main() -> i8 { + %z8 = arith.constant 0 : i8 + %z32 = arith.constant 0 : i32 + %z = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %dim = arith.constant 16 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %zero64 = arith.constant 0 : i64 + %a = memref.alloc() : memref<@BB_BANK_LINES@x16xi8> + %b = memref.alloc() : memref<16x16xi8> + %c = memref.alloc() : memref<@BB_BANK_LINES@x16xi32> + linalg.fill ins(%z32 : i32) outs(%c : memref<@BB_BANK_LINES@x16xi32>) + scf.for %tile = %z to %rows step %dim { + scf.for %row = %z to %dim step %one { + %arow = arith.addi %tile, %row : index + scf.for %k = %z to %dim step %one { + %r3 = arith.muli %row, %three : index + %k5 = arith.muli %k, %five : index + %sum = arith.addi %r3, %k5 : index + %rem = arith.remui %sum, %seven : index + %value = arith.subi %rem, %three : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %a[%arow, %k] : memref<@BB_BANK_LINES@x16xi8> + } + } + } + scf.for %row = %z to %dim step %one { + scf.for %col = %z to %dim step %one { + %r2 = arith.muli %row, %two : index + %c3 = arith.muli %col, %three : index + %sum = arith.addi %r2, %c3 : index + %rem = arith.remui %sum, %five : index + %value = arith.subi %rem, %two : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %b[%row, %col] : memref<16x16xi8> + } + } + %ab = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %bb = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %cb = buckyball.bank_alloc <{row = 1 : i64, col = 4 : i64}> + %al = buckyball.bank_mvin %a, %ab <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x16xi8> + %bl = buckyball.bank_mvin %b, %bb <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + // Tile t of A (rows 16t..16t+15) times B into rows 16t..16t+15 of C. + scf.for %base = %z to %rows step %dim { + %base64 = arith.index_cast %base : index to i64 + %r = buckyball.bank_custom "matmul_i8_tile" (%al, %bl) -> %cb args(%base64, %zero64, %base64 : i64, i64, i64) + } + %stored = buckyball.bank_mvout %c, %cb <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x16xi32> + func.call @check_result(%c) : (memref<@BB_BANK_LINES@x16xi32>) -> () + buckyball.bank_release %al + buckyball.bank_release %bl + buckyball.bank_release %stored + memref.dealloc %a : memref<@BB_BANK_LINES@x16xi8> + memref.dealloc %b : memref<16x16xi8> + memref.dealloc %c : memref<@BB_BANK_LINES@x16xi32> + return %z8 : i8 +} diff --git a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp similarity index 93% rename from examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp rename to examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp index ce5a5fb20..fc21b551d 100644 --- a/examples/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp +++ b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp @@ -19,5 +19,5 @@ extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, exit(1); } } - printf("gemmini compute-preloaded bank tiles PASS rows=%d\n", BANK_LINES); + printf("gemmini compute-preloaded bank cores PASS rows=%d\n", BANK_LINES); } diff --git a/examples/components/balls/im2col/arch/Im2col.scala b/examples/components/balls/im2col/arch/Im2col.scala new file mode 100644 index 000000000..eed182542 --- /dev/null +++ b/examples/components/balls/im2col/arch/Im2col.scala @@ -0,0 +1,267 @@ +package examples.balls.im2col + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} +import examples.balls.im2col.configs.Im2colBallParam + +@instantiable +class Im2col( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: Im2colBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val ballCfg = ballParam + val maxIter = ballCfg.maxIter + val maxKSize = ballCfg.maxKSize + val maxPad = ballCfg.maxPadding + val core = 16 + + val mapping = instance + + require(mapping.inBW == 1, "Im2colBall requires inBW=1") + require(mapping.outBW == 1, "Im2colBall requires outBW=1") + require(memParam.bankWidth == 128, "Im2colBall requires 128-bit bank rows") + require( + isPow2(memParam.bankEntries), + "Im2colBall requires power-of-two bank depth" + ) + require(maxIter + 2 * maxPad <= 255, "Im2colBall dimensions must fit the divider") + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + mapping.inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + mapping.outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val cfg = + Instantiate(new Im2colConfigRegs(frontendParam, memParam, maxIter, maxKSize, maxPad, ballParam)) + + val win = + Instantiate(new Im2colWindow( + maxKSize, + ballParam + )) + + val div = Instantiate(new RestoringDiv(16, 8, ballParam)) + + val lineBuf: Instance[LineBufferManager] = + Instantiate(new LineBufferManager(frontendParam, memParam, routingParam, instance, ballParam)) + val writer: Instance[StreamWriter] = + Instantiate(new StreamWriter(frontendParam, memParam, routingParam, instance, ballParam)) + + val cIdle :: cDivRow :: cDivCol :: cCheck :: cDivStart :: cLaunch :: Nil = Enum(6) + val calc = RegInit(cIdle) + + val running = RegInit(false.B) + val inputReady = RegInit(false.B) + val respPending = RegInit(false.B) + val armed = RegInit(false.B) + val outRowsReg = RegInit(0.U(16.W)) + val outColsReg = RegInit(0.U(16.W)) + val startOutRow = RegInit(0.U(16.W)) + val startOutCol = RegInit(0.U(16.W)) + val localWindow = RegInit(0.U(12.W)) + val elemAge = RegInit(0.U(2.W)) + val launch = WireDefault(false.B) + + cfg.io.cmd := io.cmdReq.bits + cfg.io.load := io.cmdReq.fire + + io.cmdReq.ready := !running && !respPending && !armed && calc === cIdle + io.cmdResp.valid := respPending + io.cmdResp.bits.rob_id := cfg.io.robId + io.status.idle := !running && !respPending && !armed && calc === cIdle + io.status.running := running || armed || calc =/= cIdle + + div.io.start := false.B + div.io.a := 0.U + div.io.b := 0.U + + when(io.cmdReq.fire) { + val command = io.cmdReq.bits + + assert(command.rs2(63) === 0.U, "IM2COL reserves rs2[63]") + assert( + command.iter(frontendParam.iter_len - 1, 16) === 0.U, + "IM2COL input size exceeds 16 bits" + ) + assert( + command.op1_col === 1.U && command.op2_col === 0.U && command.wr_col === 1.U, + "IM2COL requires one physical input bank and one physical output bank" + ) + assert( + command.rs2(7, 0) > 0.U && command.rs2(7, 0) <= maxKSize.U, + "IM2COL kernel must be in the configured range" + ) + armed := true.B + } + + when(armed) { + armed := false.B + assert(!cfg.io.invalid, "IM2COL command has an illegal shape or bank assignment") + val padded = cfg.io.inputSize +& (cfg.io.padding << 1) + div.io.start := true.B + div.io.a := (padded - cfg.io.kernel - cfg.io.startRow)(15, 0) + div.io.b := cfg.io.stride + calc := cDivRow + } + + when(calc === cDivRow && div.io.done) { + assert(div.io.r === 0.U, "IM2COL output row shape must be integral") + outRowsReg := div.io.q + 1.U + val padded = cfg.io.inputSize +& (cfg.io.padding << 1) + div.io.start := true.B + div.io.a := (padded - cfg.io.kernel - cfg.io.startCol)(15, 0) + div.io.b := cfg.io.stride + calc := cDivCol + } + + when(calc === cDivCol && div.io.done) { + assert(div.io.r === 0.U, "IM2COL output column shape must be integral") + outColsReg := div.io.q + 1.U + calc := cCheck + } + + when(calc === cCheck) { + val inputValues = cfg.io.inputSize * cfg.io.inputSize + val inputRows = (inputValues +& (core - 1).U) >> log2Ceil(core) + val windows = outRowsReg * outColsReg + val windowEnd = cfg.io.windowStart +& cfg.io.windowCount + val kernelElems = cfg.io.kernel * cfg.io.kernel + val mTiles = (cfg.io.windowCount +& (core - 1).U) >> log2Ceil(core) + val kTiles = (kernelElems +& (core - 1).U) >> log2Ceil(core) + val outputRows = mTiles * kTiles * core.U + + assert(cfg.io.windowStart < windows && windowEnd <= windows, "IM2COL selected window range exceeds the output shape") + assert( + inputRows <= memParam.bankEntries.U, + "IM2COL input footprint exceeds bank depth" + ) + assert( + outputRows <= memParam.bankEntries.U, + "IM2COL output footprint exceeds bank depth" + ) + + div.io.start := true.B + div.io.a := cfg.io.windowStart + div.io.b := outColsReg(7, 0) + calc := cDivStart + } + + when(calc === cDivStart && div.io.done) { + assert(div.io.q < outRowsReg && div.io.r < outColsReg, "IM2COL window start decoding exceeded the output shape") + startOutRow := div.io.q + startOutCol := div.io.r + calc := cLaunch + } + + when(calc === cLaunch) { + launch := true.B + running := true.B + inputReady := false.B + localWindow := 0.U + elemAge := 0.U + calc := cIdle + } + + when(io.cmdResp.fire) { + respPending := false.B + } + + win.io.init := launch + win.io.next := false.B + win.io.kernel := cfg.io.kernel + win.io.outputCols := outColsReg + win.io.startRow := startOutRow + win.io.startCol := startOutCol + win.io.windowCount := cfg.io.windowCount + val canEmitElem = running && inputReady && elemAge === 3.U + win.io.elemFire := canEmitElem && writer.io.elemIn.ready + + lineBuf.io.bankRead(0) <> io.bankRead(0) + lineBuf.io.start := launch + lineBuf.io.inRows := cfg.io.inputSize + lineBuf.io.inCols := cfg.io.inputSize + lineBuf.io.rowStride := cfg.io.stride + lineBuf.io.colStride := cfg.io.stride + lineBuf.io.padding := cfg.io.padding + lineBuf.io.startRow := cfg.io.startRow + lineBuf.io.startCol := cfg.io.startCol + lineBuf.io.inputBase := cfg.io.inputBase + lineBuf.io.lane := cfg.io.lane + lineBuf.io.outRow := win.io.outRow + lineBuf.io.outCol := win.io.outCol + lineBuf.io.kRowIdx := win.io.kRowIdx + lineBuf.io.kColIdx := win.io.kColIdx + lineBuf.io.rBankId := cfg.io.rBank + lineBuf.io.rGroup := cfg.io.rGroup + lineBuf.io.robId := cfg.io.robId + + writer.io.bankWrite(0) <> io.bankWrite(0) + writer.io.init := launch + writer.io.wBankId := cfg.io.wBank + writer.io.wGroup := cfg.io.wGroup + writer.io.robId := cfg.io.robId + writer.io.elemIn.valid := canEmitElem + writer.io.elemIn.bits := lineBuf.io.elemData + writer.io.elemLast := win.io.elemLast + writer.io.finalWindow := win.io.last + writer.io.kRows := cfg.io.kernel + writer.io.kCols := cfg.io.kernel + writer.io.windowIdx := localWindow + + when(launch) { + inputReady := false.B + elemAge := 0.U + }.elsewhen(running && !inputReady && lineBuf.io.loadDone) { + inputReady := true.B + elemAge := 0.U + }.elsewhen(writer.io.elemIn.fire || writer.io.windowComplete) { + elemAge := 0.U + }.elsewhen(running && inputReady && elemAge =/= 3.U) { + elemAge := elemAge + 1.U + } + + when(writer.io.opComplete) { + inputReady := false.B + running := false.B + respPending := true.B + }.elsewhen(writer.io.windowComplete) { + win.io.next := true.B + localWindow := localWindow + 1.U + } + +} diff --git a/examples/components/balls/im2col/arch/Im2colBall.scala b/examples/components/balls/im2col/arch/Im2colBall.scala new file mode 100644 index 000000000..42754602c --- /dev/null +++ b/examples/components/balls/im2col/arch/Im2colBall.scala @@ -0,0 +1,74 @@ +package examples.balls.im2col + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.im2col.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.im2col.configs.Im2colBallParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.{BlinkIO, HasBlink} +import examples.balls.im2col.Im2col + +/** + * Im2colBall - An Im2col computation Ball that complies with the Blink protocol + */ +@instantiable +class Im2colBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + val ballParam = Im2colBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + val ballCommonConfig = ball + + val inBW = ballCommonConfig.inBW + val outBW = ballCommonConfig.outBW + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + // Instantiate Im2col + val im2colUnit: Instance[Im2col] = + Instantiate(new Im2col(fp, mp, bp, ball, ballParam)) + + // Connect command interface + im2colUnit.io.cmdReq <> blink.cmdReq + im2colUnit.io.cmdResp <> blink.cmdResp + + for (i <- 0 until inBW) { + im2colUnit.io.bankRead(i) <> blink.bankRead(i) + } + + // Connect SRAM write interface - Im2col needs to write to scratchpad + for (i <- 0 until outBW) { + im2colUnit.io.bankWrite(i) <> blink.bankWrite(i) + } + + // Connect Status signals - directly obtained from internal unit + blink.status <> im2colUnit.io.status + + // MMIO: this Ball does not consume MMIO metadata +} + +object Im2colBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new Im2colBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/components/balls/im2col/arch/Im2colConfigRegs.scala b/examples/components/balls/im2col/arch/Im2colConfigRegs.scala new file mode 100644 index 000000000..dd3baa96f --- /dev/null +++ b/examples/components/balls/im2col/arch/Im2colConfigRegs.scala @@ -0,0 +1,108 @@ +package examples.balls.im2col + +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.im2col.configs.Im2colBallParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.BallCommand +import chisel3.util._ + +@instantiable +class Im2colConfigRegs( + frontendParam: FrontendParam, + memParam: MemDomainParam, + val maxIter: Int, + val maxKSize: Int, + val maxPadding: Int, + val ballParam: Im2colBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + + val kW = log2Ceil(maxKSize + 1) + + @public + val io = IO(new Bundle { + + val cmd = Input(new BallCommand(frontendParam, memParam)) + + val load = Input(Bool()) + val invalid = Output(Bool()) + val robId = Output(UInt(log2Up(frontendParam.rob_entries).W)) + val rBank = Output(UInt(frontendParam.bank_id_len.W)) + val wBank = Output(UInt(frontendParam.bank_id_len.W)) + val rGroup = Output(UInt(memParam.groupIdWidth.W)) + val wGroup = Output(UInt(memParam.groupIdWidth.W)) + val inputSize = Output(UInt(16.W)) + val kernel = Output(UInt(kW.W)) + val stride = Output(UInt(8.W)) + val padding = Output(UInt(8.W)) + val startRow = Output(UInt(8.W)) + val startCol = Output(UInt(8.W)) + val inputBase = Output(UInt(6.W)) + val lane = Output(UInt(4.W)) + val windowStart = Output(UInt(6.W)) + val windowCount = Output(UInt(7.W)) + }) + + val robId = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + val rBank = RegInit(0.U(frontendParam.bank_id_len.W)) + val wBank = RegInit(0.U(frontendParam.bank_id_len.W)) + val rGroup = RegInit(0.U(memParam.groupIdWidth.W)) + val wGroup = RegInit(0.U(memParam.groupIdWidth.W)) + val inputSize = RegInit(0.U(16.W)) + val kernel = RegInit(0.U(kW.W)) + val stride = RegInit(0.U(8.W)) + val padding = RegInit(0.U(8.W)) + val startRow = RegInit(0.U(8.W)) + val startCol = RegInit(0.U(8.W)) + val inputBase = RegInit(0.U(6.W)) + val lane = RegInit(0.U(4.W)) + val windowStart = RegInit(0.U(6.W)) + val windowCount = RegInit(0.U(7.W)) + + when(io.load) { + robId := io.cmd.rob_id + rBank := io.cmd.op1_bank + wBank := io.cmd.wr_bank + rGroup := io.cmd.op1_group + wGroup := io.cmd.wr_group + inputSize := io.cmd.iter + kernel := io.cmd.rs2(7, 0)(kW - 1, 0) + stride := io.cmd.rs2(15, 8) + padding := io.cmd.rs2(23, 16) + startCol := io.cmd.rs2(31, 24) + startRow := io.cmd.rs2(39, 32) + inputBase := io.cmd.rs2(45, 40) + lane := io.cmd.rs2(49, 46) + windowStart := io.cmd.rs2(55, 50) + windowCount := io.cmd.rs2(62, 56) + } + + val padded = inputSize +& (padding << 1) + io.invalid := inputSize === 0.U || inputSize > maxIter.U || + kernel === 0.U || kernel > maxKSize.U || stride === 0.U || + padding > maxPadding.U || startRow > padding || startCol > padding || + inputBase +& inputSize * inputSize > memParam.bankEntries.U || + padded < kernel + startRow || padded < kernel + startCol || + windowCount === 0.U || rBank === wBank + + io.robId := robId + io.rBank := rBank + io.wBank := wBank + io.rGroup := rGroup + io.wGroup := wGroup + io.inputSize := inputSize + io.kernel := kernel + io.stride := stride + io.padding := padding + io.startRow := startRow + io.startCol := startCol + io.inputBase := inputBase + io.lane := lane + io.windowStart := windowStart + io.windowCount := windowCount +} diff --git a/examples/components/balls/im2col/arch/Im2colWindow.scala b/examples/components/balls/im2col/arch/Im2colWindow.scala new file mode 100644 index 000000000..36de20bd1 --- /dev/null +++ b/examples/components/balls/im2col/arch/Im2colWindow.scala @@ -0,0 +1,75 @@ +package examples.balls.im2col + +import chisel3.experimental.hierarchy.{instantiable, public} + +import examples.balls.im2col.configs.Im2colBallParam + +import chisel3._ +import chisel3.util._ + +@instantiable +class Im2colWindow(val maxK: Int, val ballParam: Im2colBallParam) extends Module { + + val kW = log2Ceil(maxK + 1) + + @public + val io = IO(new Bundle { + val init = Input(Bool()) + val next = Input(Bool()) + val elemFire = Input(Bool()) + val kernel = Input(UInt(kW.W)) + val outputCols = Input(UInt(16.W)) + val startRow = Input(UInt(16.W)) + val startCol = Input(UInt(16.W)) + val windowCount = Input(UInt(12.W)) + val outRow = Output(UInt(16.W)) + val outCol = Output(UInt(16.W)) + val kRowIdx = Output(UInt(kW.W)) + val kColIdx = Output(UInt(kW.W)) + val elemLast = Output(Bool()) + val last = Output(Bool()) + }) + + val outRowReg = RegInit(0.U(16.W)) + val outColReg = RegInit(0.U(16.W)) + val kRowReg = RegInit(0.U(kW.W)) + val kColReg = RegInit(0.U(kW.W)) + val colsReg = RegInit(0.U(16.W)) + val remaining = RegInit(0.U(12.W)) + + val elemLast = kRowReg === io.kernel - 1.U && kColReg === io.kernel - 1.U + + when(io.init) { + outRowReg := io.startRow + outColReg := io.startCol + kRowReg := 0.U + kColReg := 0.U + colsReg := io.outputCols + remaining := io.windowCount + }.elsewhen(io.next) { + assert(remaining > 1.U, "Im2colWindow cannot advance past the selected range") + kRowReg := 0.U + kColReg := 0.U + remaining := remaining - 1.U + when(outColReg + 1.U === colsReg) { + outRowReg := outRowReg + 1.U + outColReg := 0.U + }.otherwise { + outColReg := outColReg + 1.U + } + }.elsewhen(io.elemFire && !elemLast) { + when(kColReg === io.kernel - 1.U) { + kColReg := 0.U + kRowReg := kRowReg + 1.U + }.otherwise { + kColReg := kColReg + 1.U + } + } + + io.outRow := outRowReg + io.outCol := outColReg + io.kRowIdx := kRowReg + io.kColIdx := kColReg + io.elemLast := elemLast + io.last := remaining === 1.U +} diff --git a/examples/components/balls/im2col/arch/LineBufferManager.scala b/examples/components/balls/im2col/arch/LineBufferManager.scala new file mode 100644 index 000000000..cdc62d6fe --- /dev/null +++ b/examples/components/balls/im2col/arch/LineBufferManager.scala @@ -0,0 +1,175 @@ +package examples.balls.im2col + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.memdomain.BankRead +import examples.balls.im2col.configs.Im2colBallParam + +@instantiable +class LineBufferManager( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: Im2colBallParam) + extends Module { + val ballCfg = ballParam + val maxIter = ballCfg.maxIter + val maxKSize = ballCfg.maxKSize + val elemWidth = ballCfg.inputWidth + val bankWidth = memParam.bankWidth + val lanesPerBeat = bankWidth / elemWidth + require(isPow2(lanesPerBeat), "LineBuffer lanesPerBeat must be power of 2") + val maxBeats = maxIter * maxIter + require(isPow2(maxBeats) && maxBeats >= 2, "LineBuffer depth must be an even power of two") + val halfBeats = maxBeats / 2 + // FIXME: This is a local line-buffer address, sized for maxIter^2; it must not use + // the external bank address width. Each command's external footprint is + // checked separately against bankEntries in Im2colConfigRegs. + val memoryAddrW = log2Ceil(maxBeats) + val kW = log2Ceil(maxKSize + 1) + val addrW = log2Ceil(memParam.bankEntries) + + val map = instance + + val inBW = map.inBW + + @public + val io = IO(new Bundle { + + val bankRead = Vec( + inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val start = Input(Bool()) + val inRows = Input(UInt(16.W)) + val inCols = Input(UInt(16.W)) + val rowStride = Input(UInt(8.W)) + val colStride = Input(UInt(8.W)) + val padding = Input(UInt(8.W)) + val startRow = Input(UInt(8.W)) + val startCol = Input(UInt(8.W)) + val inputBase = Input(UInt(6.W)) + val lane = Input(UInt(log2Ceil(lanesPerBeat).W)) + val outRow = Input(UInt(16.W)) + val outCol = Input(UInt(16.W)) + val kRowIdx = Input(UInt(kW.W)) + val kColIdx = Input(UInt(kW.W)) + val rBankId = Input(UInt(frontendParam.bank_id_len.W)) + val rGroup = Input(UInt(memParam.groupIdWidth.W)) + val robId = Input(UInt(log2Up(frontendParam.rob_entries).W)) + val loadDone = Output(Bool()) + val elemData = Output(UInt(elemWidth.W)) + }) + + val bufLo = SyncReadMem(halfBeats, UInt(bankWidth.W)) + val bufHi = SyncReadMem(halfBeats, UInt(bankWidth.W)) + val active = RegInit(false.B) + val pending = RegInit(false.B) + val beat = RegInit(0.U(addrW.W)) + val dimW = log2Ceil(maxIter + 1) + val beatsReg = RegInit(0.U((addrW + 1).W)) + + for (i <- 0 until inBW) { + io.bankRead(i).io.req.valid := false.B + io.bankRead(i).io.req.bits.addr := 0.U + io.bankRead(i).io.resp.ready := false.B + io.bankRead(i).bank_id := io.rBankId + io.bankRead(i).rob_id := io.robId + io.bankRead(i).group_id := io.rGroup + } + + io.bankRead(0).io.req.valid := active && !pending + io.bankRead(0).io.req.bits.addr := (io.inputBase +& beat).asTypeOf(UInt(addrW.W)) + io.bankRead(0).io.resp.ready := pending + io.loadDone := !active + + when(io.start) { + val rows = io.inRows(dimW - 1, 0) + val cols = io.inCols(dimW - 1, 0) + beatsReg := rows * cols + active := true.B + pending := false.B + beat := 0.U + }.elsewhen(io.bankRead(0).io.req.fire) { + pending := true.B + } + + when(io.bankRead(0).io.resp.fire) { + pending := false.B + when(beat +& 1.U === beatsReg) { + active := false.B + }.otherwise { + beat := beat + 1.U + } + } + + val paddedRow = io.startRow +& (io.outRow(dimW - 1, 0) * io.rowStride) +& io.kRowIdx + val paddedCol = io.startCol +& (io.outCol(dimW - 1, 0) * io.colStride) +& io.kColIdx + val rowValid = + paddedRow >= io.padding && paddedRow < io.padding + io.inRows + val colValid = + paddedCol >= io.padding && paddedCol < io.padding + io.inCols + val inBound = rowValid && colValid + val srcRow = Mux(inBound, paddedRow - io.padding, 0.U) + val srcCol = Mux(inBound, paddedCol - io.padding, 0.U) + val elemIndex = srcRow(dimW - 1, 0) * io.inCols(dimW - 1, 0) + srcCol(dimW - 1, 0) + val beatIndex = elemIndex + val laneIndex = io.lane + val memoryWrite = io.bankRead(0).io.resp.fire + val memoryRead = !active + + val memoryAddress = Mux( + memoryWrite, + beat.pad(memoryAddrW), + beatIndex(memoryAddrW - 1, 0) + ) + + val memoryBank = memoryAddress(memoryAddrW - 1) + val memoryRow = memoryAddress(log2Ceil(halfBeats) - 1, 0) + val memoryEnable = memoryWrite || memoryRead + + val loData = bufLo.readWrite( + memoryRow, + io.bankRead(0).io.resp.bits.data.asUInt, + memoryEnable && !memoryBank, + memoryWrite + ) + + val hiData = bufHi.readWrite( + memoryRow, + io.bankRead(0).io.resp.bits.data.asUInt, + memoryEnable && memoryBank, + memoryWrite + ) + + val readBank = RegEnable(memoryBank, false.B, memoryRead) + val memoryData = Mux(readBank, hiData, loData) + val laneR = RegInit(0.U(log2Ceil(lanesPerBeat).W)) + val boundR = RegInit(false.B) + val beatData = RegInit(0.U(bankWidth.W)) + val laneD = RegInit(0.U(log2Ceil(lanesPerBeat).W)) + val boundD = RegInit(false.B) + val elemHold = RegInit(0.U(elemWidth.W)) + laneR := laneIndex + boundR := inBound + beatData := memoryData + laneD := laneR + boundD := boundR + val lanes = + beatData.asTypeOf(Vec(lanesPerBeat, UInt(elemWidth.W))) + elemHold := Mux(boundD, lanes(laneD), 0.U) + + io.elemData := elemHold +} diff --git a/examples/balls/im2col/arch/src/main/scala/README.md b/examples/components/balls/im2col/arch/README.md similarity index 100% rename from examples/balls/im2col/arch/src/main/scala/README.md rename to examples/components/balls/im2col/arch/README.md diff --git a/examples/balls/im2col/arch/src/main/scala/RestoringDiv.scala b/examples/components/balls/im2col/arch/RestoringDiv.scala similarity index 83% rename from examples/balls/im2col/arch/src/main/scala/RestoringDiv.scala rename to examples/components/balls/im2col/arch/RestoringDiv.scala index 5af0a851e..88d86fb06 100644 --- a/examples/balls/im2col/arch/src/main/scala/RestoringDiv.scala +++ b/examples/components/balls/im2col/arch/RestoringDiv.scala @@ -1,11 +1,18 @@ package examples.balls.im2col +import chisel3.experimental.hierarchy.{instantiable, public} + +import examples.balls.im2col.configs.Im2colBallParam + import chisel3._ import chisel3.util._ -class RestoringDiv(n: Int, d: Int) extends Module { +@instantiable +class RestoringDiv(val n: Int, val d: Int, val ballParam: Im2colBallParam) extends Module { + require(n >= 1 && d >= 1, "RestoringDiv widths must be >= 1") + @public val io = IO(new Bundle { val start = Input(Bool()) val a = Input(UInt(n.W)) diff --git a/examples/components/balls/im2col/arch/StreamWriter.scala b/examples/components/balls/im2col/arch/StreamWriter.scala new file mode 100644 index 000000000..cada34032 --- /dev/null +++ b/examples/components/balls/im2col/arch/StreamWriter.scala @@ -0,0 +1,243 @@ +package examples.balls.im2col + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.memdomain.BankWrite +import examples.balls.im2col.configs.Im2colBallParam + +/** + * Packs im2col K cores into 128-bit bank rows (SMatMulBall A layout). + * + * ((w / 16) * ceil(kernelElems / 16) + kt) * 16 + (w % 16) + * + * Partial final K-core lanes and partial final M-core rows are zero-filled. + */ +@instantiable +class StreamWriter( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: Im2colBallParam) + extends Module { + val ballCfg = ballParam + val maxKSize = ballCfg.maxKSize + val elemWidth = ballCfg.inputWidth + val bankWidth = memParam.bankWidth + val bankEntries = memParam.bankEntries + val lanesPerBeat = bankWidth / elemWidth + val laneIdxWidth = log2Ceil(lanesPerBeat) + val kW = log2Ceil(maxKSize + 1) + val bankAddrW = log2Ceil(bankEntries) + val maxKTiles = + (maxKSize * maxKSize + lanesPerBeat - 1) / lanesPerBeat + val kTilesW = log2Ceil(maxKTiles + 1) + require( + isPow2(bankEntries), + "StreamWriter requires power-of-two bank depth" + ) + + val mapping = instance + + val outBW = mapping.outBW + + require(outBW == 1, "StreamWriter requires outBW=1") + require(bankWidth % elemWidth == 0) + require(lanesPerBeat == 16, "Im2col SMatMulBall layout requires 16 int8 lanes") + + @public + val io = IO(new Bundle { + + val bankWrite = Vec( + outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val elemIn = Flipped(Decoupled(UInt(elemWidth.W))) + val elemLast = Input(Bool()) + val finalWindow = Input(Bool()) + val init = Input(Bool()) + val windowIdx = Input(UInt(32.W)) + val kRows = Input(UInt(kW.W)) + val kCols = Input(UInt(kW.W)) + val wBankId = Input(UInt(frontendParam.bank_id_len.W)) + val wGroup = Input(UInt(memParam.groupIdWidth.W)) + val robId = Input(UInt(log2Up(frontendParam.rob_entries).W)) + + val busy = Output(Bool()) + val windowComplete = Output(Bool()) + val opComplete = Output(Bool()) + }) + + val zeros = VecInit(Seq.fill(lanesPerBeat)(0.U(elemWidth.W))) + + val packCntReg = RegInit(0.U(log2Ceil(lanesPerBeat + 1).W)) + + val packReg = RegInit( + VecInit(Seq.fill(lanesPerBeat)(0.U(elemWidth.W))) + ) + + val chunkIdxReg = RegInit(0.U(kTilesW.W)) + val wrPendingReg = RegInit(false.B) + val issued = RegInit(false.B) + val endWindowReg = RegInit(false.B) + val finalWinReg = RegInit(false.B) + val padActive = RegInit(false.B) + val padMTile = RegInit(0.U(32.W)) + val padKTiles = RegInit(0.U(kTilesW.W)) + val padK = RegInit(0.U(kTilesW.W)) + val padRow = RegInit(0.U(laneIdxWidth.W)) + val padRow0 = RegInit(0.U(laneIdxWidth.W)) + val wAddrReg = RegInit(0.U(bankAddrW.W)) + + val kernelElems = io.kRows * io.kCols + val kTilesReg = RegInit(0.U(kTilesW.W)) + + val mTile = io.windowIdx >> laneIdxWidth + val mRow = io.windowIdx(laneIdxWidth - 1, 0) + + def tileOff(core: UInt, kTiles: UInt): UInt = { + MuxLookup(kTiles, 0.U)( + Seq( + 1.U -> core, + 2.U -> (core << 1), + 3.U -> ((core << 1) + core), + 4.U -> (core << 2) + ) + ) + } + + val targetAddr = + ((tileOff(mTile, kTilesReg) + chunkIdxReg) << laneIdxWidth) + mRow + val padAddr = + ((tileOff(padMTile, padKTiles) + padK) << laneIdxWidth) + padRow + val writeFire = io.bankWrite(0).io.req.fire + val writeResp = io.bankWrite(0).io.resp.fire + + io.busy := wrPendingReg || padActive + io.windowComplete := wrPendingReg && endWindowReg && writeResp && !finalWinReg + io.opComplete := Mux( + padActive, + wrPendingReg && writeResp && + (padK + 1.U === padKTiles) && (padRow === (lanesPerBeat - 1).U), + wrPendingReg && endWindowReg && writeResp && finalWinReg && + (mRow === (lanesPerBeat - 1).U) + ) + + for (i <- 0 until outBW) { + io.bankWrite(i).io.req.valid := false.B + io.bankWrite(i).io.req.bits.addr := 0.U + io.bankWrite(i).io.req.bits.data := 0.U + io.bankWrite(i).io.req.bits.mask := + VecInit(Seq.fill(memParam.bankMaskLen)(false.B)) + io.bankWrite(i).io.resp.ready := false.B + io.bankWrite(i).bank_id := 0.U + io.bankWrite(i).rob_id := 0.U + io.bankWrite(i).group_id := io.wGroup + } + + io.bankWrite(0).io.req.valid := wrPendingReg && !issued + io.bankWrite(0).io.req.bits.addr := wAddrReg + io.bankWrite(0).io.req.bits.data := Cat(packReg.reverse) + io.bankWrite(0).io.req.bits.mask := + VecInit(Seq.fill(memParam.bankMaskLen)(true.B)) + io.bankWrite(0).io.resp.ready := true.B + io.bankWrite(0).bank_id := io.wBankId + io.bankWrite(0).rob_id := io.robId + io.bankWrite(0).group_id := io.wGroup + + io.elemIn.ready := !wrPendingReg && !padActive + + when(io.init) { + packCntReg := 0.U + packReg := zeros + chunkIdxReg := 0.U + wrPendingReg := false.B + issued := false.B + endWindowReg := false.B + finalWinReg := false.B + wAddrReg := 0.U + padActive := false.B + padMTile := 0.U + padKTiles := 0.U + padK := 0.U + padRow := 0.U + padRow0 := 0.U + kTilesReg := ((kernelElems +& (lanesPerBeat - 1).U) >> laneIdxWidth) + .asTypeOf(UInt(kTilesW.W)) + assert( + ((kernelElems +& (lanesPerBeat - 1).U) >> laneIdxWidth) >= 1.U && + ((kernelElems +& (lanesPerBeat - 1).U) >> laneIdxWidth) <= 4.U, + "StreamWriter kTiles must be 1..4" + ) + }.otherwise { + when(writeFire) { + issued := true.B + } + when(writeResp) { + packCntReg := 0.U + packReg := zeros + wrPendingReg := false.B + issued := false.B + endWindowReg := false.B + when(padActive) { + when(padRow === (lanesPerBeat - 1).U) { + when(padK + 1.U === padKTiles) { + padActive := false.B + padK := 0.U + padRow := 0.U + }.otherwise { + padK := padK + 1.U + padRow := padRow0 + } + }.otherwise { + padRow := padRow + 1.U + } + }.elsewhen(endWindowReg) { + chunkIdxReg := 0.U + when(finalWinReg && (mRow =/= (lanesPerBeat - 1).U)) { + padActive := true.B + padMTile := mTile + padKTiles := kTilesReg + padK := 0.U + padRow := mRow + 1.U + padRow0 := mRow + 1.U + } + finalWinReg := false.B + }.otherwise { + chunkIdxReg := chunkIdxReg + 1.U + } + } + + when(padActive && !wrPendingReg) { + wrPendingReg := true.B + endWindowReg := false.B + finalWinReg := false.B + packReg := zeros + wAddrReg := padAddr.asTypeOf(UInt(bankAddrW.W)) + } + + when(io.elemIn.fire) { + packReg(packCntReg(laneIdxWidth - 1, 0)) := io.elemIn.bits + val nextCnt = packCntReg + 1.U + packCntReg := nextCnt + when(nextCnt === lanesPerBeat.U || io.elemLast) { + wrPendingReg := true.B + endWindowReg := io.elemLast + finalWinReg := io.elemLast && io.finalWindow + wAddrReg := targetAddr.asTypeOf(UInt(bankAddrW.W)) + } + } + } +} diff --git a/examples/components/balls/im2col/configs/Im2colBallParam.scala b/examples/components/balls/im2col/configs/Im2colBallParam.scala new file mode 100644 index 000000000..9ea577424 --- /dev/null +++ b/examples/components/balls/im2col/configs/Im2colBallParam.scala @@ -0,0 +1,22 @@ +package examples.balls.im2col.configs + +import buckyball.balls.im2col.{Params => ParametersProto} +import upickle.default.{macroRW, ReadWriter} + +case class Im2colBallParam( + maxIter: Int, + maxKSize: Int, + maxPadding: Int, + inputWidth: Int) + +object Im2colBallParam { + + def fromProto(source: ParametersProto): Im2colBallParam = { + val parameters = source + val param = + Im2colBallParam(parameters.getMaxIter, parameters.getMaxKSize, parameters.getMaxPadding, parameters.getInputWidth) + param + } + + implicit val rw: ReadWriter[Im2colBallParam] = macroRW +} diff --git a/examples/components/balls/im2col/configs/default.toml b/examples/components/balls/im2col/configs/default.toml new file mode 100644 index 000000000..863fa7b3f --- /dev/null +++ b/examples/components/balls/im2col/configs/default.toml @@ -0,0 +1,28 @@ +# Im2col Ball Configuration +# +# Legal command range (enforced fail-hard in bemu/RTL): +# iter in 1 ..= maxIter +# ksize in 1 ..= maxKSize # covers 1x1 / 3x3 / 5x5 / 7x7 +# stride >= 1 +# padding in 0 ..= maxPadding +# start_row / start_col in 0 ..= padding (asym pad: pad=max(lo,hi), start=pad-lo) +# paddedSize = iter + 2*padding >= ksize + start +# op1_bank != wr_bank +# inputBase + iter*iter <= bankEntries +# lane in 0 .. 15 +# outputDim = floor((iter + 2*padding - ksize - start) / stride) + 1 +# M = outputDim * outputDim +# K = ksize * ksize +# outputRows = ceil(M/16) * ceil(K/16) * 16 +# outputRows <= bankEntries # bankEntries from design memdomain +# +# Partial final K-core lanes and partial final M-core rows are zero. +# One bank row is one spatial position with 16 INT8 channels. + +inputWidth = 8 +# Resident Conv lowering can request a 4x4 output core from a stride-2, +# kernel-7 producer, which needs a 13x13 local input. Keep this a power of two +# because the RTL line buffer splits maxIter^2 beats into two equal banks. +maxIter = 16 +maxKSize = 7 +maxPadding = 7 diff --git a/examples/components/balls/im2col/configs/im2col_params.proto b/examples/components/balls/im2col/configs/im2col_params.proto new file mode 100644 index 000000000..203f6785a --- /dev/null +++ b/examples/components/balls/im2col/configs/im2col_params.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; +package buckyball.balls.im2col; +option java_package = "buckyball.balls.im2col"; +option java_multiple_files = true; + +message Params { + optional int32 inputWidth = 1; + optional int32 maxIter = 2; + optional int32 maxKSize = 3; + optional int32 maxPadding = 4; +} diff --git a/examples/components/balls/im2col/emu/src/dispatch.rs b/examples/components/balls/im2col/emu/src/dispatch.rs new file mode 100644 index 000000000..522677147 --- /dev/null +++ b/examples/components/balls/im2col/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::ExecContext; + +use super::im2col; + +const BALL_CLASS: &str = "examples.balls.im2col.Im2colBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("IM2COL") => Some(im2col::exec_im2col(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("IM2COL") => Some(im2col::latency(xs1, xs2, bebop_bemu::configs::ball_domain::instance_for_funct(funct).expect("Ball instance"))), + Some(_) | None => None, + } +} diff --git a/examples/balls/im2col/emu/src/im2col.rs b/examples/components/balls/im2col/emu/src/im2col.rs similarity index 76% rename from examples/balls/im2col/emu/src/im2col.rs rename to examples/components/balls/im2col/emu/src/im2col.rs index b70b5e008..8d54ae705 100644 --- a/examples/balls/im2col/emu/src/im2col.rs +++ b/examples/components/balls/im2col/emu/src/im2col.rs @@ -1,20 +1,19 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::instruction::ExecContext; -const TILE: usize = 16; -const BALL_CLASS: &str = "examples.balls.im2col.Im2colBall"; +const CORE: usize = 16; -fn max_iter() -> usize { - crate::config::ball_domain::param(BALL_CLASS, "maxIter") +fn max_iter(instance: u32) -> usize { + bebop_bemu::configs::ball_domain::parameters::("im2col", instance).max_iter.expect("maxIter") as usize } -fn max_kernel() -> usize { - crate::config::ball_domain::param(BALL_CLASS, "maxKSize") +fn max_kernel(instance: u32) -> usize { + bebop_bemu::configs::ball_domain::parameters::("im2col", instance).max_k_size.expect("maxKSize") as usize } -fn max_padding() -> usize { - crate::config::ball_domain::param(BALL_CLASS, "maxPadding") +fn max_padding(instance: u32) -> usize { + bebop_bemu::configs::ball_domain::parameters::("im2col", instance).max_padding.expect("maxPadding") as usize } #[derive(Clone, Copy)] @@ -46,13 +45,13 @@ fn decode_shape(xs1: u64, xs2: u64) -> Shape { } } -fn dimensions(shape: Shape) -> (usize, usize, usize) { +fn dimensions(shape: Shape, instance: u32) -> (usize, usize, usize) { if shape.input_size == 0 - || shape.input_size > max_iter() + || shape.input_size > max_iter(instance) || shape.kernel == 0 - || shape.kernel > max_kernel() + || shape.kernel > max_kernel(instance) || shape.stride == 0 - || shape.padding > max_padding() + || shape.padding > max_padding(instance) || shape.start_row > shape.padding || shape.start_col > shape.padding || shape.window_count == 0 @@ -91,7 +90,7 @@ fn footprints(shape: Shape) -> (usize, usize) { .unwrap_or_else(|| panic!("im2col: input footprint overflow")); let kernel_values = shape.kernel * shape.kernel; let input_rows = input_values; - let output_rows = shape.window_count.div_ceil(TILE) * kernel_values.div_ceil(TILE) * TILE; + let output_rows = shape.window_count.div_ceil(CORE) * kernel_values.div_ceil(CORE) * CORE; if shape.input_base + input_rows > bank_lines() || output_rows > bank_lines() { panic!("im2col: input or output footprint exceeds bank depth"); } @@ -118,10 +117,10 @@ pub(crate) fn exec_im2col(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { } let shape = decode_shape(xs1, xs2); - let (_, output_cols, _) = dimensions(shape); + let (_, output_cols, _) = dimensions(shape, ctx.ball_id.expect("Ball instance")); let (_, output_bank_rows) = footprints(shape); let kernel_values = shape.kernel * shape.kernel; - let kernel_tiles = kernel_values.div_ceil(TILE); + let kernel_tiles = kernel_values.div_ceil(CORE); let pi = pbank(ctx, input_bank); let po = pbank(ctx, output_bank); let (input, output) = ctx.banks.read_write(pi, po); @@ -147,20 +146,20 @@ pub(crate) fn exec_im2col(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { continue; } let kernel_element = kernel_row * shape.kernel + kernel_col; - let bank_row = ((local_window / TILE) * kernel_tiles + kernel_element / TILE) - * TILE - + local_window % TILE; - let lane = kernel_element % TILE; + let bank_row = ((local_window / CORE) * kernel_tiles + kernel_element / CORE) + * CORE + + local_window % CORE; + let lane = kernel_element % CORE; let source = source_row as usize * shape.input_size + source_col as usize; - output[bank_row * TILE + lane] = - input[(shape.input_base + source) * TILE + shape.lane]; + output[bank_row * CORE + lane] = + input[(shape.input_base + source) * CORE + shape.lane]; } } } 0 } -pub(crate) fn latency(xs1: u64, xs2: u64) -> u64 { +pub(crate) fn latency(xs1: u64, xs2: u64, instance: u32) -> u64 { if rs1_b1(xs1) != 0 { panic!("im2col: input bank 1 must be zero"); } @@ -168,7 +167,7 @@ pub(crate) fn latency(xs1: u64, xs2: u64) -> u64 { panic!("im2col: rs2[63] must be zero"); } let shape = decode_shape(xs1, xs2); - dimensions(shape); + dimensions(shape, instance); footprints(shape); (shape.window_count * shape.kernel * shape.kernel).max(16) as u64 } diff --git a/examples/components/balls/im2col/emu/src/lib.rs b/examples/components/balls/im2col/emu/src/lib.rs new file mode 100644 index 000000000..96b3d1aed --- /dev/null +++ b/examples/components/balls/im2col/emu/src/lib.rs @@ -0,0 +1,11 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod im2col; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/components/balls/im2col/stack/src/Lowering.cpp b/examples/components/balls/im2col/stack/src/Lowering.cpp new file mode 100644 index 000000000..c29680164 --- /dev/null +++ b/examples/components/balls/im2col/stack/src/Lowering.cpp @@ -0,0 +1,84 @@ +//===- Lowering.cpp - Im2colBall in lower-buckyball -----------------------===// + +#include "Buckyball/BuckyballOps.h" +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class Im2colI8Lowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &rewriter) const override { + if (op.getKind() != "im2col_i8") + return failure(); + if (op.getSources().size() != 1 || + op.getSources()[0] == op.getDestination()) + return op.emitOpError( + "im2col_i8 requires distinct input and output banks"); + int64_t side = cast(op->getAttr("input_side")).getInt(); + int64_t kernel = cast(op->getAttr("kernel")).getInt(); + int64_t stride = cast(op->getAttr("stride")).getInt(); + int64_t padding = cast(op->getAttr("padding")).getInt(); + int64_t startRow = cast(op->getAttr("start_row")).getInt(); + int64_t startCol = cast(op->getAttr("start_col")).getInt(); + int64_t base = cast(op->getAttr("op1_base")).getInt(); + int64_t lane = cast(op->getAttr("lane")).getInt(); + int64_t windowStart = + cast(op->getAttr("window_start")).getInt(); + int64_t windowCount = + cast(op->getAttr("window_count")).getInt(); + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || side <= 0 || side >= (1LL << 33) || + side > buckyball_target::getBuckyballBallParam("IM2COL", "maxIter") || + kernel <= 0 || kernel > 255 || + kernel > + buckyball_target::getBuckyballBallParam("IM2COL", "maxKSize") || + stride <= 0 || stride > 255 || padding < 0 || padding > 255 || + padding > + buckyball_target::getBuckyballBallParam("IM2COL", "maxPadding") || + startRow < 0 || startRow > padding || startCol < 0 || + startCol > padding || base < 0 || base > 63 || lane < 0 || lane > 15 || + windowStart < 0 || windowStart > 63 || windowCount <= 0 || + windowCount > 64) + return op.emitOpError("im2col_i8 geometry exceeds the ISA field ranges"); + int64_t rowNumerator = side + 2 * padding - kernel - startRow; + int64_t colNumerator = side + 2 * padding - kernel - startCol; + if (rowNumerator < 0 || colNumerator < 0 || rowNumerator % stride || + colNumerator % stride || + windowStart + windowCount > + (rowNumerator / stride + 1) * (colNumerator / stride + 1) || + base + side * side > target.bankDepth || + ((windowCount + 15) / 16) * ((kernel * kernel + 15) / 16) * 16 > + target.bankDepth) + return op.emitOpError("im2col_i8 window or bank footprint is invalid"); + uint64_t rs2 = + mlir::buddy::field(kernel, 0, 7) | mlir::buddy::field(stride, 8, 15) | + mlir::buddy::field(padding, 16, 23) | + mlir::buddy::field(startCol, 24, 31) | + mlir::buddy::field(startRow, 32, 39) | + mlir::buddy::field(base, 40, 45) | mlir::buddy::field(lane, 46, 49) | + mlir::buddy::field(windowStart, 50, 55) | + mlir::buddy::field(windowCount, 56, 62); + mlir::buddy::emitInstruction( + rewriter, op.getLoc(), buckyball_target::getBuckyballFunct7("IM2COL"), + mlir::buddy::constantI64(rewriter, op.getLoc(), + mlir::buddy::bankFields(op.getSources(), + op.getDestination(), + false, side)), + mlir::buddy::constantI64(rewriter, op.getLoc(), rs2)); + rewriter.eraseOp(op); + return success(); + } +}; + +void populate(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} + +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallLowering(populate), true); +} // namespace diff --git a/examples/components/balls/im2col/verify/README.md b/examples/components/balls/im2col/verify/README.md new file mode 100644 index 000000000..f2edcbf01 --- /dev/null +++ b/examples/components/balls/im2col/verify/README.md @@ -0,0 +1,102 @@ +# Im2colBall UVM Verification + +Verifies the generated `Im2colBall` module directly using the shared Blink UVM framework. + +## Structure + +- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM transaction items and base env +- `src/common/im2col_defs.svh`: DPI imports and constants +- `src/common/im2col_items.svh`: `im2col_cmd_item` with `load_rust_case` +- `src/seq/im2col_sequences.svh`: `im2col_basic_seq` driving one case per run +- `src/cov/im2col_cov.svh`: ksize/stride/pad/iter coverage +- `src/env/im2col_scoreboard.svh`: preloads mem model, compares writes vs DPI dst words +- `src/env/im2col_env.svh`: extends `bb_blink_env#(1,1)` with scoreboard and cov +- `src/tests/im2col_test.svh`: directed cases 0,1 then random 2..21 with fixed seed +- `src/pkg/im2col_pkg.sv`: package include entry +- `src/tb_top.sv`: instantiates `bb_blink_if#(1,1)` and `Im2colBall` +- `filelists/im2col_ball.f`: VCS/Verilator filelist + +## Test plan + +`+UVM_TESTNAME=im2col_ball_test` runs: + +- case 0: directed k3 6x6 pad0 (ctest `im2col_k3_test.c` data) +- case 1: directed k3 6x6 pad1 (ctest `im2col_k3_pad_test.c` data) +- cases 2..21: random with fixed seed `0xCAFE_BABE`, iter in {3,4,5,6,7}, + ksize in {1,3,5}, stride=1, pad in {0,1}, distinct banks 0..7 + +Each case: reset, preload src words into `mem_model`, drive one command, wait for +`scb.done()` or timeout at 100000 cycles. Scoreboard compares observed writes +(data/addr/mask/bank/rob) with DPI `im2col_case_dst_word_*`. `im2col_cov` +requires ksize/stride/pad/iter bins hit across the whole test, else +`check_phase` fatal. + +Blink command fields (not ISA rs packing): + +- `iter` = cmd.iter +- `ksize` = cmd.special[7:0] +- `stride` = cmd.special[15:8] +- `padding` = cmd.special[23:16] +- banks = op1_bank / wr_bank; funct7 is injected from the selected Core ballISA +- TB: op1_col=1, wr_col=1, rs1=0, rs2=0 + +## BID (required) + +`+BID=` is mandatory. Missing it fatals at test start. There is no default bid. + +| Config | Plusarg | +|--------|---------| +| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=2` | +| `sims.verilator.BuckyballToyVerilatorConfig` | `+BID=3` | + +```console +./simv +UVM_TESTNAME=im2col_ball_test +BID=2 +``` + +## Build + +Enter the shared UVM/VCS environment: + +```console +nix develop ../../../../verify +``` + +Build the DPI reference library: + +```console +cargo build --manifest-path casegen/Cargo.toml +``` + +Compile from this directory (resolve `@UVM@` / `@RTL@` like bbdev): + +```console +# Do not pass -hsopt=off: VCS W-2024.09 segfaults on Im2colBall+UVM with it. +vcs -full64 -sverilog -timescale=1ns/1ps \ + $VCS_UVM_ARGS \ + -sv_lib ../output/casegen/debug/libim2col_casegen \ + -f build//im2col_ball.f +``` + +Run: + +```console +./simv +UVM_TESTNAME=im2col_ball_test +BID=2 +``` + +Or via bbdev: + +```console +bbdev uvm --build '--ball=im2col --config sims.verilator.BuckyballPebbleVerilatorConfig' +bbdev uvm --run '--ball=im2col' -- '+BID=2' +``` + +## Acceptance (2026-08-04) + +- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` +- [x] casegen pure im2col model (no ExecContext); whole-case DPI API +- [x] directed k3 pad0/pad1 scoreboard pass +- [x] deterministic random idx 2..21 pass +- [x] protocol + semantic cover enforced (`check_phase` fatal) +- [x] pebble: `+BID=2` — 22/22 +- [x] toy: `+BID=3` — 22/22 +- [x] Note: omit `-hsopt=off` for Im2col (VCS segfault); bbdev skips it for this ball diff --git a/examples/balls/im2col/verify/casegen/Cargo.lock b/examples/components/balls/im2col/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/im2col/verify/casegen/Cargo.lock rename to examples/components/balls/im2col/verify/casegen/Cargo.lock diff --git a/examples/balls/im2col/verify/casegen/Cargo.toml b/examples/components/balls/im2col/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/im2col/verify/casegen/Cargo.toml rename to examples/components/balls/im2col/verify/casegen/Cargo.toml diff --git a/examples/components/balls/im2col/verify/casegen/src/casegen.rs b/examples/components/balls/im2col/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..ba16c6d90 --- /dev/null +++ b/examples/components/balls/im2col/verify/casegen/src/casegen.rs @@ -0,0 +1,218 @@ +use crate::model; + +pub const BANK_ROW_BYTES: usize = model::BANK_ROW_BYTES; +pub const MAX_WORDS: usize = 128; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Im2colCmd { + pub bid: u32, + pub iter: u32, + pub ksize: u32, + pub stride: u32, + pub padding: u32, + pub window_count: u32, + pub op1_bank: u32, + pub wr_bank: u32, + pub op1_col: u32, + pub wr_col: u32, + pub rob_id: u32, + pub num_src_words: u32, + pub num_dst_words: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Im2colCase { + pub cmd: Im2colCmd, + pub src: Vec, + pub dst: Vec, +} + +impl Im2colCase { + pub fn src_word_lo(&self, word_index: usize) -> u64 { + word_lo(&self.src, word_index) + } + pub fn src_word_hi(&self, word_index: usize) -> u64 { + word_hi(&self.src, word_index) + } + pub fn dst_word_lo(&self, word_index: usize) -> u64 { + word_lo(&self.dst, word_index) + } + pub fn dst_word_hi(&self, word_index: usize) -> u64 { + word_hi(&self.dst, word_index) + } +} + +fn word_lo(data: &[u8], word_index: usize) -> u64 { + let off = word_index * BANK_ROW_BYTES; + if off + 8 > data.len() { + panic!("word_lo: word_index {word_index} out of range"); + } + let mut buf = [0u8; 8]; + buf.copy_from_slice(&data[off..off + 8]); + u64::from_le_bytes(buf) +} + +fn word_hi(data: &[u8], word_index: usize) -> u64 { + let off = word_index * BANK_ROW_BYTES + 8; + if off + 8 > data.len() { + panic!("word_hi: word_index {word_index} out of range"); + } + let mut buf = [0u8; 8]; + buf.copy_from_slice(&data[off..off + 8]); + u64::from_le_bytes(buf) +} + +pub fn gen_case(seed: u32, index: u32, bid: u32) -> Im2colCase { + match index { + 0 => directed_k3_pad0(bid), + 1 => directed_k3_pad1(bid), + _ => random_case(seed, index, bid), + } +} + +fn build_case( + bid: u32, + iter: usize, + ksize: usize, + stride: usize, + padding: usize, + op1_bank: u32, + wr_bank: u32, + rob_id: u32, + flat: &[u8], +) -> Im2colCase { + if op1_bank == wr_bank { + panic!("build_case: op1_bank and wr_bank must differ"); + } + if op1_bank > 7 || wr_bank > 7 { + panic!("build_case: banks must be in 0..7"); + } + if flat.len() != iter * iter { + panic!("build_case: flat len mismatch"); + } + let rows = model::output_rows(iter, ksize, stride, padding); + let windows = ((iter + 2 * padding - ksize) / stride + 1).pow(2); + if rows > model::bank_entries() { + panic!("build_case: output_rows={rows} exceeds bank depth"); + } + let dst_flat = model::im2col(flat, iter, ksize, stride, padding); + let src = model::pack_bank_words(flat); + let nsrc = model::num_words(src.len()); + let ndst = model::num_words(dst_flat.len()); + if nsrc > model::bank_entries() || ndst > model::bank_entries() || nsrc > MAX_WORDS || ndst > MAX_WORDS { + panic!("build_case: word count out of range src={nsrc} dst={ndst}"); + } + Im2colCase { + cmd: Im2colCmd { + bid, + iter: iter as u32, + ksize: ksize as u32, + stride: stride as u32, + padding: padding as u32, + window_count: windows as u32, + op1_bank, + wr_bank, + op1_col: 1, + wr_col: 1, + rob_id, + num_src_words: nsrc as u32, + num_dst_words: ndst as u32, + }, + src, + dst: dst_flat, + } +} + +fn directed_k3_pad0(bid: u32) -> Im2colCase { + let iter = 6usize; + let k = 3usize; + let flat: [i8; 36] = [ + -1, 2, 3, 4, -5, 6, 0, 8, 9, -10, 11, 12, 13, 0, -15, 16, 17, 18, 19, -20, 0, 22, 23, -24, + -25, 26, 27, 0, -29, 30, 31, 32, 33, -34, 0, 36, + ]; + let bytes: Vec = flat.iter().map(|&x| x as u8).collect(); + build_case(bid, iter, k, 1, 0, 0, 1, 3, &bytes) +} + +fn directed_k3_pad1(bid: u32) -> Im2colCase { + let iter = 6usize; + let k = 3usize; + let flat: [i8; 36] = [ + 7, -2, 3, 0, 5, -6, 1, 8, -9, 10, 11, 0, -4, 12, 13, 14, 0, 16, 17, 0, 19, -20, 21, 22, 0, + 24, 25, 26, -27, 28, 29, -30, 0, 32, 33, 34, + ]; + let bytes: Vec = flat.iter().map(|&x| x as u8).collect(); + build_case(bid, iter, k, 1, 1, 0, 1, 5, &bytes) +} + +fn shape_ok(iter: usize, ksize: usize, stride: usize, padding: usize) -> bool { + if iter == 0 || ksize == 0 || stride == 0 { + return false; + } + let padded = iter + 2 * padding; + if padded < ksize { + return false; + } + let rows = model::output_rows(iter, ksize, stride, padding); + rows <= model::bank_entries() +} + +fn random_case(seed: u32, index: u32, bid: u32) -> Im2colCase { + let mut rng = Rng::new(seed, index); + let iter_pool = [3usize, 4, 5, 6, 7]; + let ksize_pool = [1usize, 3, 5]; + let pad_pool = [0usize, 1]; + let base = (index - 2) as usize; + + for attempt in 0..10_000 { + let iter = iter_pool[(base + attempt) % iter_pool.len()]; + let ksize = ksize_pool[((base / iter_pool.len()) + attempt) % ksize_pool.len()]; + let stride = 1usize; + let padding = pad_pool[(base + attempt / 3) % pad_pool.len()]; + if !shape_ok(iter, ksize, stride, padding) { + continue; + } + let op1_bank = rng.range(0, 7); + let mut wr_bank = rng.range(0, 7); + if wr_bank == op1_bank { + wr_bank = (wr_bank + 1) & 7; + } + let mut flat = vec![0u8; iter * iter]; + for b in flat.iter_mut() { + *b = (rng.next() & 0xff) as u8; + } + return build_case( + bid, + iter, + ksize, + stride, + padding, + op1_bank, + wr_bank, + rng.range(0, 15), + &flat, + ); + } + panic!("random_case: failed to find legal shape for index={index}"); +} + +struct Rng { + state: u64, +} + +impl Rng { + fn new(seed: u32, index: u32) -> Self { + let state = (u64::from(seed) << 32) ^ u64::from(index) ^ 0x9E37_79B9_7F4A_7C15; + Self { state } + } + fn next(&mut self) -> u32 { + self.state ^= self.state >> 12; + self.state ^= self.state << 25; + self.state ^= self.state >> 27; + ((self.state.wrapping_mul(0x2545_F491_4F6C_DD1D)) >> 32) as u32 + } + fn range(&mut self, lo: u32, hi: u32) -> u32 { + lo + (self.next() % (hi - lo + 1)) + } +} diff --git a/examples/components/balls/im2col/verify/casegen/src/lib.rs b/examples/components/balls/im2col/verify/casegen/src/lib.rs new file mode 100644 index 000000000..67176b88c --- /dev/null +++ b/examples/components/balls/im2col/verify/casegen/src/lib.rs @@ -0,0 +1,71 @@ +mod casegen; +mod model; + +use std::cell::RefCell; + +use casegen::{Im2colCase, Im2colCmd, MAX_WORDS}; + +thread_local! { + static CURRENT: RefCell> = RefCell::new(None); +} + +#[no_mangle] +pub extern "C" fn im2col_case_load(seed: u32, index: u32, bid: u32) -> i32 { + let case = casegen::gen_case(seed, index, bid); + let nsrc = case.cmd.num_src_words as usize; + let ndst = case.cmd.num_dst_words as usize; + if nsrc > MAX_WORDS || ndst > MAX_WORDS { + panic!("im2col_case_load: word count out of range src={nsrc} dst={ndst}"); + } + CURRENT.with(|c| *c.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|c| match c.borrow().as_ref() { + Some(case) => f(case), + None => panic!("im2col_case: no case loaded; call im2col_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn im2col_case_cmd(out_ptr: *mut Im2colCmd) { + current_case(|case| unsafe { + if out_ptr.is_null() { + panic!("im2col_case_cmd: null out_ptr"); + } + *out_ptr = case.cmd; + }); +} + +#[no_mangle] +pub extern "C" fn im2col_case_src_word_lo(word_index: u32) -> u64 { + if word_index as usize >= MAX_WORDS { + panic!("im2col_case_src_word_lo: word_index out of range"); + } + current_case(|case| case.src_word_lo(word_index as usize)) +} + +#[no_mangle] +pub extern "C" fn im2col_case_src_word_hi(word_index: u32) -> u64 { + if word_index as usize >= MAX_WORDS { + panic!("im2col_case_src_word_hi: word_index out of range"); + } + current_case(|case| case.src_word_hi(word_index as usize)) +} + +#[no_mangle] +pub extern "C" fn im2col_case_dst_word_lo(word_index: u32) -> u64 { + if word_index as usize >= MAX_WORDS { + panic!("im2col_case_dst_word_lo: word_index out of range"); + } + current_case(|case| case.dst_word_lo(word_index as usize)) +} + +#[no_mangle] +pub extern "C" fn im2col_case_dst_word_hi(word_index: u32) -> u64 { + if word_index as usize >= MAX_WORDS { + panic!("im2col_case_dst_word_hi: word_index out of range"); + } + current_case(|case| case.dst_word_hi(word_index as usize)) +} diff --git a/examples/components/balls/im2col/verify/casegen/src/model.rs b/examples/components/balls/im2col/verify/casegen/src/model.rs new file mode 100644 index 000000000..d4d374d2e --- /dev/null +++ b/examples/components/balls/im2col/verify/casegen/src/model.rs @@ -0,0 +1,103 @@ +pub const CORE: usize = 16; +pub const BANK_ROW_BYTES: usize = CORE; + +pub fn bank_entries() -> usize { + let config = match std::env::var("BB_VERIFY_CONFIG") { + Ok(path) => path, + Err(_) if cfg!(test) => return 64, + Err(_) => panic!("BB_VERIFY_CONFIG is required"), + }; + let text = std::fs::read_to_string(config).expect("failed to read BB_VERIFY_CONFIG"); + text.lines() + .find_map(|line| line.strip_prefix("bank_entries=")) + .unwrap_or_else(|| panic!("bank_entries missing from BB_VERIFY_CONFIG")) + .parse() + .expect("invalid bank_entries in BB_VERIFY_CONFIG") +} + +pub fn output_rows(iter: usize, ksize: usize, stride: usize, padding: usize) -> usize { + if iter == 0 || ksize == 0 || stride == 0 { + panic!("im2col model: iter/ksize/stride must be >= 1"); + } + let padded = iter + 2 * padding; + if padded < ksize { + panic!("im2col model: kernel larger than padded input"); + } + let out_dim = (padded - ksize) / stride + 1; + let windows = out_dim * out_dim; + let kernel = ksize * ksize; + let m_tiles = windows.div_ceil(CORE); + let k_tiles = kernel.div_ceil(CORE); + m_tiles * k_tiles * CORE +} + +pub fn im2col(src: &[u8], iter: usize, ksize: usize, stride: usize, padding: usize) -> Vec { + if src.len() != iter * iter { + panic!( + "im2col model: src len {} != iter*iter {}", + src.len(), + iter * iter + ); + } + let padded = iter + 2 * padding; + if padded < ksize { + panic!("im2col model: kernel larger than padded input"); + } + if stride == 0 { + panic!("im2col model: stride must be >= 1"); + } + let out_dim = (padded - ksize) / stride + 1; + let windows = out_dim * out_dim; + let kernel_elems = ksize * ksize; + let m_tiles = windows.div_ceil(CORE); + let k_tiles = kernel_elems.div_ceil(CORE); + let rows = m_tiles * k_tiles * CORE; + let mut output = vec![0u8; rows * CORE]; + + let mut window = 0usize; + for orow in 0..out_dim { + for ocol in 0..out_dim { + for kr in 0..ksize { + for kc in 0..ksize { + let kernel = kr * ksize + kc; + let m_tile = window / CORE; + let m_row = window % CORE; + let k_tile = kernel / CORE; + let lane = kernel % CORE; + let bank_row = (m_tile * k_tiles + k_tile) * CORE + m_row; + let out = bank_row * CORE + lane; + if out >= output.len() { + panic!("im2col model: output range out={out}"); + } + let input_row = (orow * stride + kr) as isize - padding as isize; + let input_col = (ocol * stride + kc) as isize - padding as isize; + if input_row >= 0 + && input_row < iter as isize + && input_col >= 0 + && input_col < iter as isize + { + let src_i = input_row as usize * iter + input_col as usize; + output[out] = src[src_i]; + } + } + } + window += 1; + } + } + output +} + +pub fn pack_bank_words(flat: &[u8]) -> Vec { + let mut out = vec![0u8; flat.len() * BANK_ROW_BYTES]; + for (index, value) in flat.iter().enumerate() { + out[index * BANK_ROW_BYTES] = *value; + } + out +} + +pub fn num_words(total_bytes: usize) -> usize { + if total_bytes % BANK_ROW_BYTES != 0 { + panic!("num_words: total_bytes {total_bytes} not a multiple of bank row"); + } + total_bytes / BANK_ROW_BYTES +} diff --git a/examples/balls/im2col/verify/filelists/im2col_ball.f b/examples/components/balls/im2col/verify/filelists/im2col_ball.f similarity index 100% rename from examples/balls/im2col/verify/filelists/im2col_ball.f rename to examples/components/balls/im2col/verify/filelists/im2col_ball.f diff --git a/examples/balls/im2col/verify/src/common/im2col_defs.svh b/examples/components/balls/im2col/verify/src/common/im2col_defs.svh similarity index 91% rename from examples/balls/im2col/verify/src/common/im2col_defs.svh rename to examples/components/balls/im2col/verify/src/common/im2col_defs.svh index 3a0ab79b0..4ce04ad4f 100644 --- a/examples/balls/im2col/verify/src/common/im2col_defs.svh +++ b/examples/components/balls/im2col/verify/src/common/im2col_defs.svh @@ -26,9 +26,9 @@ import "DPI-C" function longint unsigned im2col_case_dst_word_lo(input int unsig import "DPI-C" function longint unsigned im2col_case_dst_word_hi(input int unsigned word_index); `ifndef IM2COL_FUNCT7 -`error "IM2COL_FUNCT7 must be provided by the selected Core ballISA" +`error "IM2COL_FUNCT7 must be provided by the selected Thread ballISA" `endif -localparam int IM2COL_CORE_FUNCT7 = `IM2COL_FUNCT7; +localparam int IM2COL_THREAD_FUNCT7 = `IM2COL_FUNCT7; localparam int IM2COL_MAX_WORDS = 128; localparam int IM2COL_TIMEOUT_CYCLES = 100000; localparam int IM2COL_SEED = 32'hCAFE_BABE; diff --git a/examples/balls/im2col/verify/src/common/im2col_items.svh b/examples/components/balls/im2col/verify/src/common/im2col_items.svh similarity index 97% rename from examples/balls/im2col/verify/src/common/im2col_items.svh rename to examples/components/balls/im2col/verify/src/common/im2col_items.svh index 2dd71678a..1bd487bba 100644 --- a/examples/balls/im2col/verify/src/common/im2col_items.svh +++ b/examples/components/balls/im2col/verify/src/common/im2col_items.svh @@ -11,7 +11,7 @@ class im2col_cmd_item extends bb_blink_cmd_item; bit [127:0] dst_words[IM2COL_MAX_WORDS]; constraint legal_c { - funct7 == IM2COL_CORE_FUNCT7[6:0]; + funct7 == IM2COL_THREAD_FUNCT7[6:0]; op1_en == 1'b1; op2_en == 1'b0; wr_spad_en == 1'b1; @@ -44,7 +44,7 @@ class im2col_cmd_item extends bb_blink_cmd_item; im2col_case_cmd(cmd); bid = cmd.bid[4:0]; - funct7 = IM2COL_CORE_FUNCT7[6:0]; + funct7 = IM2COL_THREAD_FUNCT7[6:0]; iter = cmd.iter; ksize = cmd.ksize; stride = cmd.stride; diff --git a/examples/balls/im2col/verify/src/cov/im2col_cov.svh b/examples/components/balls/im2col/verify/src/cov/im2col_cov.svh similarity index 97% rename from examples/balls/im2col/verify/src/cov/im2col_cov.svh rename to examples/components/balls/im2col/verify/src/cov/im2col_cov.svh index bd512cffe..0ff8c5efa 100644 --- a/examples/balls/im2col/verify/src/cov/im2col_cov.svh +++ b/examples/components/balls/im2col/verify/src/cov/im2col_cov.svh @@ -34,6 +34,7 @@ class im2col_cov extends uvm_component; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") if (cmd_cg.get_coverage() != 100.0) begin diff --git a/examples/balls/im2col/verify/src/env/im2col_env.svh b/examples/components/balls/im2col/verify/src/env/im2col_env.svh similarity index 100% rename from examples/balls/im2col/verify/src/env/im2col_env.svh rename to examples/components/balls/im2col/verify/src/env/im2col_env.svh diff --git a/examples/balls/im2col/verify/src/env/im2col_scoreboard.svh b/examples/components/balls/im2col/verify/src/env/im2col_scoreboard.svh similarity index 98% rename from examples/balls/im2col/verify/src/env/im2col_scoreboard.svh rename to examples/components/balls/im2col/verify/src/env/im2col_scoreboard.svh index b79a5cb74..5067cc112 100644 --- a/examples/balls/im2col/verify/src/env/im2col_scoreboard.svh +++ b/examples/components/balls/im2col/verify/src/env/im2col_scoreboard.svh @@ -67,6 +67,7 @@ class im2col_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, im2col_cmd_item exp); + verification_pkg::evidence::checks++; if (got.bid !== exp.bid) `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) if (got.funct7 !== exp.funct7) @@ -160,6 +161,7 @@ class im2col_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) begin `uvm_fatal("SCB", $sformatf("incomplete: cmds=%0d reads=%0d writes=%0d/%0d resp=%0d", diff --git a/examples/balls/im2col/verify/src/pkg/im2col_pkg.sv b/examples/components/balls/im2col/verify/src/pkg/im2col_pkg.sv similarity index 100% rename from examples/balls/im2col/verify/src/pkg/im2col_pkg.sv rename to examples/components/balls/im2col/verify/src/pkg/im2col_pkg.sv diff --git a/examples/balls/im2col/verify/src/seq/im2col_sequences.svh b/examples/components/balls/im2col/verify/src/seq/im2col_sequences.svh similarity index 100% rename from examples/balls/im2col/verify/src/seq/im2col_sequences.svh rename to examples/components/balls/im2col/verify/src/seq/im2col_sequences.svh diff --git a/examples/components/balls/im2col/verify/src/tb_top.sv b/examples/components/balls/im2col/verify/src/tb_top.sv new file mode 100644 index 000000000..ce1c8fc12 --- /dev/null +++ b/examples/components/balls/im2col/verify/src/tb_top.sv @@ -0,0 +1,91 @@ +module tb_top; + import uvm_pkg::*; + import im2col_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + Im2colBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .io_cmdReq_ready(intf.cmd_req_ready), + .io_cmdReq_valid(intf.cmd_req_valid), + .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), + .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), + .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), + .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), + .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), + .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), + .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), + .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), + .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), + .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank[3:0]), + .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank[3:0]), + .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank[3:0]), + .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), + .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), + .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), + .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank[3:0]), + .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), + .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), + .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), + .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), + .io_cmdResp_ready(intf.cmd_resp_ready), + .io_cmdResp_valid(intf.cmd_resp_valid), + .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), + .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), + .io_bankRead_0_bank_id(intf.bank_read_bank_id[0][3:0]), + .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .io_bankRead_0_group_id(intf.bank_read_group_id[0][3:0]), + .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0][3:0]), + .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .io_bankWrite_0_group_id(intf.bank_write_group_id[0][3:0]), + .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .io_subRobReq_ready(intf.sub_rob_req_ready), + .io_status_idle(), + .io_status_running(), + .io_bankAccessReady(1'b1), + .io_bankRead_0_ball_id(), + .io_bankWrite_0_ball_id() + ); + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("im2col_ball_test"); + end +endmodule diff --git a/examples/balls/im2col/verify/src/tests/im2col_test.svh b/examples/components/balls/im2col/verify/src/tests/im2col_test.svh similarity index 100% rename from examples/balls/im2col/verify/src/tests/im2col_test.svh rename to examples/components/balls/im2col/verify/src/tests/im2col_test.svh diff --git a/examples/components/balls/im2col/workloads/ctests/CMakeLists.txt b/examples/components/balls/im2col/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..246b28cd1 --- /dev/null +++ b/examples/components/balls/im2col/workloads/ctests/CMakeLists.txt @@ -0,0 +1,8 @@ +add_buckyball_ctest(im2col_window_test.c) +add_buckyball_ctest(im2col_k1_test.c) +add_buckyball_ctest(im2col_k3_test.c) +add_buckyball_ctest(im2col_k3_pad_test.c) +add_buckyball_ctest(im2col_k3_asym_pad_test.c) +add_buckyball_ctest(im2col_k3_partial_test.c) +add_buckyball_ctest(im2col_k5_test.c) +add_buckyball_ctest(im2col_k7_test.c) diff --git a/examples/balls/im2col/workloads/ctests/im2col_k1_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_k1_test.c similarity index 97% rename from examples/balls/im2col/workloads/ctests/im2col_k1_test.c rename to examples/components/balls/im2col/workloads/ctests/im2col_k1_test.c index 1a8857d3e..2c28978c4 100644 --- a/examples/balls/im2col/workloads/ctests/im2col_k1_test.c +++ b/examples/components/balls/im2col/workloads/ctests/im2col_k1_test.c @@ -48,8 +48,8 @@ static void build_expected(void) { } int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif clear_i8_matrix(out, OUT_ROWS, LANES); build_expected(); diff --git a/examples/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c similarity index 97% rename from examples/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c rename to examples/components/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c index bcfb5385d..301e1d28c 100644 --- a/examples/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c +++ b/examples/components/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c @@ -47,8 +47,8 @@ static void build_expected(void) { } int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif clear_i8_matrix(out, OUT_ROWS, LANES); build_expected(); diff --git a/examples/balls/im2col/workloads/ctests/im2col_k3_pad_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_k3_pad_test.c similarity index 97% rename from examples/balls/im2col/workloads/ctests/im2col_k3_pad_test.c rename to examples/components/balls/im2col/workloads/ctests/im2col_k3_pad_test.c index 6552d46ce..16ed237b7 100644 --- a/examples/balls/im2col/workloads/ctests/im2col_k3_pad_test.c +++ b/examples/components/balls/im2col/workloads/ctests/im2col_k3_pad_test.c @@ -47,8 +47,8 @@ static void build_expected(void) { } int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif clear_i8_matrix(out, OUT_ROWS, LANES); build_expected(); diff --git a/examples/balls/im2col/workloads/ctests/im2col_k3_partial_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_k3_partial_test.c similarity index 95% rename from examples/balls/im2col/workloads/ctests/im2col_k3_partial_test.c rename to examples/components/balls/im2col/workloads/ctests/im2col_k3_partial_test.c index 2fc546064..55dec3a55 100644 --- a/examples/balls/im2col/workloads/ctests/im2col_k3_partial_test.c +++ b/examples/components/balls/im2col/workloads/ctests/im2col_k3_partial_test.c @@ -4,7 +4,7 @@ #include #include -/* partial M tile: 5x5 k3 -> 9 windows, unused m rows must be 0 */ +/* partial M core: 5x5 k3 -> 9 windows, unused m rows must be 0 */ enum { ITER = 5, K = 3, STRIDE = 1, PAD = 0, LANES = 16 }; enum { OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, @@ -47,8 +47,8 @@ static void build_expected(void) { } int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif clear_i8_matrix(out, OUT_ROWS, LANES); build_expected(); diff --git a/examples/balls/im2col/workloads/ctests/im2col_k3_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_k3_test.c similarity index 96% rename from examples/balls/im2col/workloads/ctests/im2col_k3_test.c rename to examples/components/balls/im2col/workloads/ctests/im2col_k3_test.c index 96897dc02..387f355a6 100644 --- a/examples/balls/im2col/workloads/ctests/im2col_k3_test.c +++ b/examples/components/balls/im2col/workloads/ctests/im2col_k3_test.c @@ -4,7 +4,7 @@ #include #include -/* exact M tile: 6x6 k3 -> 16 windows */ +/* exact M core: 6x6 k3 -> 16 windows */ enum { ITER = 6, K = 3, STRIDE = 1, PAD = 0, LANES = 16 }; enum { OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, @@ -48,8 +48,8 @@ static void build_expected(void) { } int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif clear_i8_matrix(out, OUT_ROWS, LANES); build_expected(); diff --git a/examples/balls/im2col/workloads/ctests/im2col_k5_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_k5_test.c similarity index 97% rename from examples/balls/im2col/workloads/ctests/im2col_k5_test.c rename to examples/components/balls/im2col/workloads/ctests/im2col_k5_test.c index 590978ade..7a14defa2 100644 --- a/examples/balls/im2col/workloads/ctests/im2col_k5_test.c +++ b/examples/components/balls/im2col/workloads/ctests/im2col_k5_test.c @@ -47,8 +47,8 @@ static void build_expected(void) { } int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif clear_i8_matrix(out, OUT_ROWS, LANES); build_expected(); diff --git a/examples/balls/im2col/workloads/ctests/im2col_k7_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_k7_test.c similarity index 97% rename from examples/balls/im2col/workloads/ctests/im2col_k7_test.c rename to examples/components/balls/im2col/workloads/ctests/im2col_k7_test.c index 11c9c13e4..0387975a5 100644 --- a/examples/balls/im2col/workloads/ctests/im2col_k7_test.c +++ b/examples/components/balls/im2col/workloads/ctests/im2col_k7_test.c @@ -47,8 +47,8 @@ static void build_expected(void) { } int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif clear_i8_matrix(out, OUT_ROWS, LANES); build_expected(); diff --git a/examples/balls/im2col/workloads/ctests/im2col_window_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_window_test.c similarity index 100% rename from examples/balls/im2col/workloads/ctests/im2col_window_test.c rename to examples/components/balls/im2col/workloads/ctests/im2col_window_test.c diff --git a/examples/balls/im2col/workloads/isa/im2col.h b/examples/components/balls/im2col/workloads/isa/im2col.h similarity index 100% rename from examples/balls/im2col/workloads/isa/im2col.h rename to examples/components/balls/im2col/workloads/isa/im2col.h diff --git a/examples/balls/im2col/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/im2col/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/im2col/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/im2col/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/im2col/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/im2col/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/im2col/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/im2col/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir b/examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir new file mode 100644 index 000000000..6b0e1fbf4 --- /dev/null +++ b/examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir @@ -0,0 +1,43 @@ +// Physical-bank im2col_i8: 6x6 input at rows 5.. lane 7, k3, 16 windows. + +func.func private @check_result(memref<16x16xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c5 = arith.constant 5 : index + %c6 = arith.constant 6 : index + %c7 = arith.constant 7 : index + + %input = memref.alloc() alignment = 64 : memref<64x16xi8> + %output = memref.alloc() alignment = 64 : memref<16x16xi8> + linalg.fill ins(%zero_i8 : i8) outs(%input : memref<64x16xi8>) + linalg.fill ins(%zero_i8 : i8) outs(%output : memref<16x16xi8>) + + scf.for %r = %c0 to %c6 step %c1 { + scf.for %col = %c0 to %c6 step %c1 { + %tmp = arith.muli %r, %c6 : index + %idx = arith.addi %tmp, %col : index + %bank_row = arith.addi %c5, %idx : index + %ri = arith.index_cast %r : index to i32 + %ci = arith.index_cast %col : index to i32 + %v32 = arith.addi %ri, %ci : i32 + %v8 = arith.trunci %v32 : i32 to i8 + memref.store %v8, %input[%bank_row, %c7] : memref<64x16xi8> + } + } + + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 0 : i64, depth = 64 : i64, stride = 1 : i64}> : memref<64x16xi8> + buckyball.custom "im2col_i8" <{sources = array, destination = 1 : i64}> {input_side = 6 : i64, kernel = 3 : i64, stride = 1 : i64, padding = 0 : i64, start_row = 0 : i64, start_col = 0 : i64, op1_base = 5 : i64, lane = 7 : i64, window_start = 0 : i64, window_count = 16 : i64} + buckyball.mvout %output <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + + func.call @check_result(%output) : (memref<16x16xi8>) -> () + memref.dealloc %input : memref<64x16xi8> + memref.dealloc %output : memref<16x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp b/examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp similarity index 100% rename from examples/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp rename to examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp diff --git a/examples/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/components/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows.mlir.in b/examples/components/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows.mlir.in new file mode 100644 index 000000000..62ae91620 --- /dev/null +++ b/examples/components/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows.mlir.in @@ -0,0 +1,32 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %windows = arith.constant 64 : index + %lane = arith.constant 7 : index + %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + linalg.fill ins(%zero_i8 : i8) outs(%input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) + scf.for %index = %zero to %rows step %one { + %value32 = arith.index_cast %index : index to i32 + %value = arith.trunci %value32 : i32 to i8 + memref.store %value, %input[%index, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + } + scf.for %base = %zero to %rows step %windows { + %in = memref.subview %input[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> + %out = memref.subview %output[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded = buckyball.bank_mvin %in, %input_bank <{depth = 64 : i64, stride = 1 : i64}> : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> + %result = buckyball.bank_custom "im2col_i8" (%loaded) -> %output_bank {input_side = 8 : i64, kernel = 3 : i64, stride = 1 : i64, padding = 1 : i64, start_row = 0 : i64, start_col = 0 : i64, op1_base = 0 : i64, lane = 7 : i64, window_start = 0 : i64, window_count = 64 : i64} + %stored = buckyball.bank_mvout %out, %result <{depth = 64 : i64, stride = 1 : i64}> : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> + buckyball.bank_release %loaded + buckyball.bank_release %stored + } + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () + memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows_main.cpp b/examples/components/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows_main.cpp similarity index 100% rename from examples/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows_main.cpp rename to examples/components/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows_main.cpp diff --git a/examples/balls/int2fp/arch/src/main/scala/Fp32Mul.scala b/examples/components/balls/int2fp/arch/Fp32Mul.scala similarity index 89% rename from examples/balls/int2fp/arch/src/main/scala/Fp32Mul.scala rename to examples/components/balls/int2fp/arch/Fp32Mul.scala index 94a899a4f..75467ff16 100644 --- a/examples/balls/int2fp/arch/src/main/scala/Fp32Mul.scala +++ b/examples/components/balls/int2fp/arch/Fp32Mul.scala @@ -1,11 +1,22 @@ package examples.balls.int2fp +import chisel3.experimental.hierarchy.public + +import chisel3.experimental.hierarchy.instantiable + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import examples.balls.int2fp.configs.Int2FpBallParam + import chisel3._ import chisel3.util._ -class ShiftAddMul(w: Int) extends Module { +@instantiable +class ShiftAddMul(val w: Int, val ballParam: Int2FpBallParam) extends Module { + require(w >= 1, "ShiftAddMul width must be >= 1") + @public val io = IO(new Bundle { val start = Input(Bool()) val a = Input(UInt(w.W)) @@ -74,8 +85,10 @@ class ShiftAddMul(w: Int) extends Module { } } -class Fp32Mul extends Module { +@instantiable +class Fp32Mul(val ballParam: Int2FpBallParam) extends Module { + @public val io = IO(new Bundle { val start = Input(Bool()) val a = Input(UInt(32.W)) @@ -84,7 +97,9 @@ class Fp32Mul extends Module { val done = Output(Bool()) }) - val umul = Module(new ShiftAddMul(24)) + val umul = + Instantiate(new ShiftAddMul(24, ballParam)) + val aReg = Reg(UInt(32.W)) val bReg = Reg(UInt(32.W)) val result = Reg(UInt(32.W)) diff --git a/examples/components/balls/int2fp/arch/Int2Fp.scala b/examples/components/balls/int2fp/arch/Int2Fp.scala new file mode 100644 index 000000000..31da0f780 --- /dev/null +++ b/examples/components/balls/int2fp/arch/Int2Fp.scala @@ -0,0 +1,239 @@ +package examples.balls.int2fp + +import framework.acc.balldomain.configs.BallRouteParam + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.int2fp.configs.Int2FpBallParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} + +@instantiable +class Int2Fp( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: Int2FpBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + + val mapping = instance + + require(memParam.bankWidth == 128, "Int2FpBall requires 128-bit bank rows") + require(mapping.inBW == 2, "Int2FpBall requires inBW=2") + require(mapping.outBW == 1, "Int2FpBall requires outBW=1") + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + mapping.inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + mapping.outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val idle :: readReq :: readResp :: convert :: multiply :: writeReq :: writeResp :: complete :: Nil = + Enum(8) + val state = RegInit(idle) + + val robIdReg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + val inputBank = RegInit(0.U(frontendParam.bank_id_len.W)) + val scaleBank = RegInit(0.U(frontendParam.bank_id_len.W)) + val outputBank = RegInit(0.U(frontendParam.bank_id_len.W)) + val bases = Reg(Vec(3, UInt(memParam.groupIdWidth.W))) + val iterReg = RegInit(0.U(frontendParam.iter_len.W)) + val rowReg = RegInit(0.U(log2Ceil(memParam.bankEntries).W)) + val reluReg = RegInit(false.B) + val inputData = Reg(UInt(128.W)) + val scaleData = Reg(UInt(128.W)) + val readSent = RegInit(VecInit(Seq.fill(mapping.inBW)(false.B))) + val readReceived = RegInit(VecInit(Seq.fill(mapping.inBW)(false.B))) + val fpOutputs = Reg(Vec(4, UInt(32.W))) + val mulStarted = RegInit(false.B) + val laneReg = RegInit(0.U(2.W)) + val writeData = Reg(UInt(128.W)) + val multiplier = Instantiate(new Fp32Mul(ballParam)) + + def int32ToFp32(value: UInt): UInt = { + val sign = value(31) + val abs = Mux(sign, (~value).asUInt + 1.U, value) + val zero = abs === 0.U + val leading = 31.U - PriorityEncoder(Reverse(abs)) + val shift = Mux(leading > 23.U, leading - 23.U, 0.U) + val truncated = abs >> shift + val half = Mux(shift === 0.U, 0.U(32.W), 1.U(32.W) << (shift - 1.U)) + val remainder = Mux(shift === 0.U, 0.U(32.W), abs & ((1.U(32.W) << shift) - 1.U)) + val roundUp = remainder > half || (remainder === half && truncated(0)) + val rounded = truncated +& roundUp.asUInt + val carry = rounded(24) + val significand = Mux(leading > 23.U, rounded(22, 0), (abs << (23.U - leading))(22, 0)) + val exponent = (leading +& 127.U +& carry)(7, 0) + Mux(zero, 0.U, Cat(sign, exponent, Mux(carry, 0.U(23.W), significand))) + } + + def positiveFinite(value: UInt): Bool = + !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U + + for (i <- 0 until mapping.inBW) { + io.bankRead(i).rob_id := robIdReg + io.bankRead(i).bank_id := Mux(i.U === 0.U, inputBank, scaleBank) + io.bankRead(i).group_id := bases(i) + io.bankRead(i).io.req.valid := false.B + io.bankRead(i).io.req.bits.addr := Mux(i.U === 0.U, rowReg, rowReg(1, 0)) + io.bankRead(i).io.resp.ready := false.B + } + io.bankWrite(0).rob_id := robIdReg + io.bankWrite(0).bank_id := outputBank + io.bankWrite(0).group_id := bases(2) + io.bankWrite(0).io.req.valid := false.B + io.bankWrite(0).io.req.bits.addr := rowReg + io.bankWrite(0).io.req.bits.data := writeData + io.bankWrite(0).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(true.B) + ) + io.bankWrite(0).io.resp.ready := false.B + + val inputValue = (inputData >> (laneReg << 5))(31, 0) + val scaleValue = (scaleData >> (laneReg << 5))(31, 0) + multiplier.io.start := false.B + multiplier.io.a := int32ToFp32(Mux(reluReg && inputValue(31), 0.U, inputValue)) + multiplier.io.b := scaleValue + + io.cmdReq.ready := state === idle + io.cmdResp.valid := state === complete + io.cmdResp.bits.rob_id := robIdReg + io.status.idle := state === idle + io.status.running := state =/= idle && state =/= complete + + switch(state) { + is(idle) { + when(io.cmdReq.fire) { + val cmd = io.cmdReq.bits + + assert(cmd.iter > 0.U && cmd.iter(1, 0) === 0.U, "INT32_TO_FP32 iter must be a positive multiple of four") + assert( + cmd.iter <= memParam.bankEntries.U, + "INT32_TO_FP32 input exceeds bank depth" + ) + assert( + cmd.op1_col === 1.U && cmd.op2_col === 1.U && cmd.wr_col === 1.U, + "INT32_TO_FP32 operands must each occupy one bank" + ) + assert( + cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, + "INT32_TO_FP32 banks must be distinct" + ) + assert(cmd.rs2(63, 1) === 0.U, "INT32_TO_FP32 reserves rs2[63:1]") + + robIdReg := io.cmdReq.bits.rob_id + inputBank := cmd.op1_bank + scaleBank := cmd.op2_bank + outputBank := cmd.wr_bank + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + iterReg := cmd.iter + rowReg := 0.U + reluReg := cmd.rs2(0) + readSent.foreach(_ := false.B) + readReceived.foreach(_ := false.B) + state := readReq + } + } + is(readReq) { + for (i <- 0 until mapping.inBW) { + io.bankRead(i).io.req.valid := !readSent(i) + when(io.bankRead(i).io.req.fire)(readSent(i) := true.B) + } + when(VecInit(io.bankRead.zipWithIndex.map { case (port, i) => readSent(i) || port.io.req.fire }).asUInt.andR) { + state := readResp + } + } + is(readResp) { + for (i <- 0 until mapping.inBW) { + io.bankRead(i).io.resp.ready := !readReceived(i) + when(io.bankRead(i).io.resp.fire)(readReceived(i) := true.B) + } + when(io.bankRead(0).io.resp.fire)(inputData := io.bankRead(0).io.resp.bits.data) + when(io.bankRead(1).io.resp.fire)(scaleData := io.bankRead(1).io.resp.bits.data) + when(VecInit(io.bankRead.zipWithIndex.map { case (port, i) => readReceived(i) || port.io.resp.fire }).asUInt.andR) { + state := convert + } + } + is(convert) { + for (i <- 0 until 4) { + val scale = scaleData(32 * i + 31, 32 * i) + assert(positiveFinite(scale), "INT32_TO_FP32 scales must be finite and positive") + } + mulStarted := false.B + laneReg := 0.U + state := multiply + } + is(multiply) { + when(!mulStarted) { + multiplier.io.start := true.B + mulStarted := true.B + } + when(multiplier.io.done) { + fpOutputs(laneReg) := multiplier.io.result + when(laneReg === 3.U) { + writeData := Cat(multiplier.io.result, fpOutputs(2), fpOutputs(1), fpOutputs(0)) + state := writeReq + }.otherwise { + laneReg := laneReg + 1.U + mulStarted := false.B + } + } + } + is(writeReq) { + io.bankWrite(0).io.req.valid := true.B + when(io.bankWrite(0).io.req.fire) { + state := writeResp + } + } + is(writeResp) { + io.bankWrite(0).io.resp.ready := true.B + when(io.bankWrite(0).io.resp.fire) { + when(rowReg === iterReg - 1.U) { + state := complete + }.otherwise { + rowReg := rowReg + 1.U + readSent.foreach(_ := false.B) + readReceived.foreach(_ := false.B) + state := readReq + } + } + } + is(complete) { + when(io.cmdResp.fire) { + state := idle + } + } + } +} diff --git a/examples/components/balls/int2fp/arch/Int2FpBall.scala b/examples/components/balls/int2fp/arch/Int2FpBall.scala new file mode 100644 index 000000000..c8f741996 --- /dev/null +++ b/examples/components/balls/int2fp/arch/Int2FpBall.scala @@ -0,0 +1,52 @@ +package examples.balls.int2fp + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.int2fp.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.int2fp.configs.Int2FpBallParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} + +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class Int2FpBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + val ballParam = Int2FpBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val unit: Instance[Int2Fp] = + Instantiate(new Int2Fp(fp, mp, bp, ball, ballParam)) + unit.io.cmdReq <> blink.cmdReq + unit.io.cmdResp <> blink.cmdResp + for (i <- 0 until ball.inBW) unit.io.bankRead(i) <> blink.bankRead(i) + for (i <- 0 until ball.outBW) unit.io.bankWrite(i) <> blink.bankWrite(i) + blink.status <> unit.io.status + +} + +object Int2FpBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new Int2FpBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/components/balls/int2fp/configs/Int2FpBallParam.scala b/examples/components/balls/int2fp/configs/Int2FpBallParam.scala new file mode 100644 index 000000000..9a20a54f8 --- /dev/null +++ b/examples/components/balls/int2fp/configs/Int2FpBallParam.scala @@ -0,0 +1,20 @@ +package examples.balls.int2fp.configs + +import buckyball.balls.int2fp.{Params => ParametersProto} +import upickle.default.{macroRW, ReadWriter} + +case class Int2FpBallParam( + inputType: String, + outputType: String) + +object Int2FpBallParam { + + def fromProto(source: ParametersProto): Int2FpBallParam = { + val parameters = source + require(parameters.hasInputType && parameters.hasOutputType) + val param = Int2FpBallParam(parameters.getInputType, parameters.getOutputType) + param + } + + implicit val rw: ReadWriter[Int2FpBallParam] = macroRW +} diff --git a/examples/components/balls/int2fp/configs/default.toml b/examples/components/balls/int2fp/configs/default.toml new file mode 100644 index 000000000..436aea20e --- /dev/null +++ b/examples/components/balls/int2fp/configs/default.toml @@ -0,0 +1,4 @@ +# Int2Fp Ball Configuration + +inputType = "INT32" +outputType = "FP32" diff --git a/examples/components/balls/int2fp/configs/int2fp_params.proto b/examples/components/balls/int2fp/configs/int2fp_params.proto new file mode 100644 index 000000000..cb4db0f16 --- /dev/null +++ b/examples/components/balls/int2fp/configs/int2fp_params.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; +package buckyball.balls.int2fp; +option java_package = "buckyball.balls.int2fp"; +option java_multiple_files = true; + +message Params { + optional string inputType = 1; + optional string outputType = 2; +} diff --git a/examples/components/balls/int2fp/emu/src/dispatch.rs b/examples/components/balls/int2fp/emu/src/dispatch.rs new file mode 100644 index 000000000..c0c9e8136 --- /dev/null +++ b/examples/components/balls/int2fp/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::ExecContext; + +use super::int2fp; + +const BALL_CLASS: &str = "examples.balls.int2fp.Int2FpBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("INT32_TO_FP32") => Some(int2fp::execute(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("INT32_TO_FP32") => Some(int2fp::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/balls/int2fp/emu/src/int2fp.rs b/examples/components/balls/int2fp/emu/src/int2fp.rs similarity index 98% rename from examples/balls/int2fp/emu/src/int2fp.rs rename to examples/components/balls/int2fp/emu/src/int2fp.rs index 35fd35d79..47ba26fad 100644 --- a/examples/balls/int2fp/emu/src/int2fp.rs +++ b/examples/components/balls/int2fp/emu/src/int2fp.rs @@ -1,4 +1,4 @@ -use super::super::bank::bank_lines; +use bebop_bemu::thread::banks::bank_lines; use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::instruction::ExecContext; diff --git a/examples/components/balls/int2fp/emu/src/lib.rs b/examples/components/balls/int2fp/emu/src/lib.rs new file mode 100644 index 000000000..4e14f52d0 --- /dev/null +++ b/examples/components/balls/int2fp/emu/src/lib.rs @@ -0,0 +1,11 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod int2fp; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/components/balls/int2fp/emu/src/model.rs b/examples/components/balls/int2fp/emu/src/model.rs new file mode 100644 index 000000000..34a59807d --- /dev/null +++ b/examples/components/balls/int2fp/emu/src/model.rs @@ -0,0 +1,88 @@ +pub fn int2fp_fp32_bits(value: i32, dw_bits: u32) -> u32 { + fp32_multiply(int32_to_fp32(value), dw_bits) +} + +pub fn int2fp_dequant_bits(value: i32, da_bits: u32, dw_bits: u32) -> u32 { + int2fp_fp32_bits(value, fp32_multiply(da_bits, dw_bits)) +} + +pub fn int32_to_fp32(value: i32) -> u32 { + if value == 0 { + return 0; + } + + let sign = ((value as u32) >> 31) & 1; + let abs = if value == i32::MIN { + 0x8000_0000u32 + } else { + value.unsigned_abs() + }; + + let leading_one = 31u32 - abs.leading_zeros(); + let mut exponent = leading_one + 127; + let significand = if leading_one > 23 { + let right_shift = leading_one - 23; + let abs_wide = u64::from(abs); + let truncated = abs_wide >> right_shift; + let half = 1u64 << (right_shift - 1); + let remainder = abs_wide & ((1u64 << right_shift) - 1); + let round_up = remainder > half || (remainder == half && (truncated & 1) != 0); + let rounded = truncated + u64::from(round_up); + if ((rounded >> 24) & 1) != 0 { + exponent = leading_one + 128; + (rounded >> 1) as u32 + } else { + rounded as u32 + } + } else { + abs << (23 - leading_one) + }; + + (sign << 31) | ((exponent & 0xff) << 23) | (significand & 0x7f_ffff) +} + +fn fp32_multiply(a: u32, b: u32) -> u32 { + let a_sign = (a >> 31) & 1; + let b_sign = (b >> 31) & 1; + let a_exp = (a >> 23) & 0xff; + let b_exp = (b >> 23) & 0xff; + let a_frac = a & 0x7f_ffff; + let b_frac = b & 0x7f_ffff; + let a_mant = (1u64 << 23) | u64::from(a_frac); + let b_mant = (1u64 << 23) | u64::from(b_frac); + let a_zero = a_exp == 0 && a_frac == 0; + let b_zero = b_exp == 0 && b_frac == 0; + let prod = a_mant * b_mant; + let (sig, guard, round, sticky, exp_adjust) = if ((prod >> 47) & 1) != 0 { + ( + (prod >> 24) as u32, + ((prod >> 23) & 1) != 0, + ((prod >> 22) & 1) != 0, + (prod & ((1u64 << 22) - 1)) != 0, + 1u32, + ) + } else { + ( + (prod >> 23) as u32, + ((prod >> 22) & 1) != 0, + ((prod >> 21) & 1) != 0, + (prod & ((1u64 << 21) - 1)) != 0, + 0u32, + ) + }; + let increment = guard && (round || sticky || (sig & 1) != 0); + let rounded = u64::from(sig) + u64::from(increment); + let carry = ((rounded >> 24) & 1) as u32; + let final_sig = if carry != 0 { rounded >> 1 } else { rounded } as u32; + let exp_wide = (a_exp + b_exp + exp_adjust + carry).wrapping_sub(127) & 0x3ff; + + if a_zero || b_zero { + 0 + } else if (exp_wide & 0x200) != 0 { + 0 + } else if (exp_wide & 0x100) != 0 { + ((a_sign ^ b_sign) << 31) | (0xff << 23) + } else { + ((a_sign ^ b_sign) << 31) | ((exp_wide & 0xff) << 23) | (final_sig & 0x7f_ffff) + } +} diff --git a/examples/components/balls/int2fp/stack/src/Lowering.cpp b/examples/components/balls/int2fp/stack/src/Lowering.cpp new file mode 100644 index 000000000..7269e4bde --- /dev/null +++ b/examples/components/balls/int2fp/stack/src/Lowering.cpp @@ -0,0 +1,49 @@ +//===- Lowering.cpp - Int2FpBall in lower-buckyball -----------------------===// +// +// int32_to_fp32: source 0 holds `rows` rows of sixteen i32 (four bank rows +// each), source 1 the sixteen f32 scales of the columns (four bank rows); the +// destination receives the f32 value * scale of every element. +// int32_to_fp32_relu: the same, with negative results replaced by zero. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballOps.h" +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class Int32ToFp32Lowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &rewriter) const override { + bool relu = op.getKind() == "int32_to_fp32_relu"; + if (op.getKind() != "int32_to_fp32" && !relu) + return failure(); + if (op.getSources().size() != 2) + return op.emitOpError("int32_to_fp32 reads the values and the scales"); + Location loc = op.getLoc(); + int64_t rows = cast(op->getAttr("rows")).getInt(); + mlir::buddy::emitInstruction( + rewriter, loc, buckyball_target::getBuckyballFunct7("INT32_TO_FP32"), + mlir::buddy::constantI64(rewriter, loc, + mlir::buddy::bankFields(op.getSources(), + op.getDestination(), + false, rows * 4)), + mlir::buddy::constantI64(rewriter, loc, + mlir::buddy::field(relu, 0, 0))); + rewriter.eraseOp(op); + return success(); + } +}; + +void populate(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} + +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallLowering(populate), true); +} // namespace diff --git a/examples/components/balls/int2fp/verify/README.md b/examples/components/balls/int2fp/verify/README.md new file mode 100644 index 000000000..1471bd59a --- /dev/null +++ b/examples/components/balls/int2fp/verify/README.md @@ -0,0 +1,77 @@ +# Int2FpBall UVM Verification + +Verifies the generated `Int2FpBall` module using the shared Blink UVM framework. + +RTL contract: one mnemonic `INT32_TO_FP32`, `inBW=2` (input + scale banks), +`outBW=1`, `rs2[0]=relu`, `rs2[63:1]=0`, iter a positive multiple of 4. +Scales are bank rows, not MMIO. + +## Structure + +- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM agents and base env +- `src/common/int2fp_defs.svh`: DPI imports, `INT32_TO_FP32_FUNCT7`, timeouts +- `src/common/int2fp_items.svh`: `int2fp_cmd_item` with `load_rust_case` +- `src/seq/int2fp_sequences.svh`: one-case sequence +- `src/cov/int2fp_cov.svh`: relu `{0,1}` and iter `{4,8,16}`, 100% or fatal +- `src/env/int2fp_scoreboard.svh`: preload src/scale banks, compare writes vs DPI dst +- `src/env/int2fp_env.svh`: extends `bb_blink_env #(BB_IN_BW, BB_OUT_BW)` +- `src/tests/int2fp_test.svh`: `int2fp_ball_test` loops cases 0..5 +- `src/pkg/int2fp_pkg.sv`: package entry +- `src/tb_top.sv`: `bb_blink_if #(BB_IN_BW, BB_OUT_BW)` + `Int2FpBall` (2 read, 1 write) +- `filelists/int2fp_ball.f`: VCS filelist (`@UVM@` / `@RTL@`) + +## Test plan + +`+UVM_TESTNAME=int2fp_ball_test` runs six directed cases: + +| Index | iter | relu | Intent | +|-------|------|------|--------| +| 0 | 4 | 0 | MLIR-scale values | +| 1 | 4 | 1 | relu clamps negatives | +| 2 | 8 | 0 | eight-row, no relu | +| 3 | 8 | 1 | eight-row, relu | +| 4 | 16 | 0 | sixteen-row, no relu | +| 5 | 16 | 1 | sixteen-row, relu | + +Each case: reset, preload 16-byte i32 input rows and f32 scale rows into +`mem_model`, drive one `INT32_TO_FP32` command, wait for `scb.done()` or +timeout at 20000 cycles. Expected dst = `f32(i32) * scale` with relu clamp +on the integer before convert. `int2fp_cov` fatals unless relu and iter bins +are 100%. + +## BID (required) + +`+BID=` is mandatory. Missing it fatals at test start. No default bid. + +| Config | Plusarg | +|--------|---------| +| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=4` | + +```console +./simv +UVM_TESTNAME=int2fp_ball_test +BID=4 +``` + +## Build / Run + +Enter the shared UVM/VCS environment and build the DPI reference library: + +```console +nix develop ../../../../verify +cargo build --manifest-path casegen/Cargo.toml +``` + +Human: + +```console +bbdev uvm --run '--design pebble' +bbdev uvm --run '--design pebble --ball int2fp' +``` + +Agent (MCP): + +```text +bbdev_uvm_run(design="pebble", ball="int2fp") +``` + +bbdev injects `INT32_TO_FP32_FUNCT7`, `BB_IN_BW`, `BB_OUT_BW`, `+BID`, and +`+UVM_TESTNAME=int2fp_ball_test` from pebble design.pb. diff --git a/examples/balls/int2fp/verify/casegen/Cargo.lock b/examples/components/balls/int2fp/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/int2fp/verify/casegen/Cargo.lock rename to examples/components/balls/int2fp/verify/casegen/Cargo.lock diff --git a/examples/balls/int2fp/verify/casegen/Cargo.toml b/examples/components/balls/int2fp/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/int2fp/verify/casegen/Cargo.toml rename to examples/components/balls/int2fp/verify/casegen/Cargo.toml diff --git a/examples/components/balls/int2fp/verify/casegen/src/casegen.rs b/examples/components/balls/int2fp/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..1573dc550 --- /dev/null +++ b/examples/components/balls/int2fp/verify/casegen/src/casegen.rs @@ -0,0 +1,259 @@ +use crate::model::int2fp_fp32_bits; + +pub const BANK_NUM: u32 = 24; +pub const BANK_ENTRIES: u32 = 64; +pub const LANES: usize = 4; +pub const SCALE_ROWS: usize = 4; +pub const MAX_SRC: usize = 16; +pub const MAX_SCALE: usize = 4; +pub const MAX_DST: usize = 16; +pub const NUM_CASES: u32 = 6; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Int2FpCmd { + pub bid: u32, + pub iter: u32, + pub relu: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, + pub rob_id: u32, + pub rs1_lo: u32, + pub rs1_hi: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub num_src_words: u32, + pub num_scale_words: u32, + pub num_dst_words: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Int2FpCase { + pub cmd: Int2FpCmd, + pub src_words: [u128; MAX_SRC], + pub scale_words: [u128; MAX_SCALE], + pub dst_words: [u128; MAX_DST], +} + +impl Int2FpCase { + pub fn src_lo(&self, index: usize) -> u64 { + self.src_words[index] as u64 + } + + pub fn src_hi(&self, index: usize) -> u64 { + (self.src_words[index] >> 64) as u64 + } + + pub fn scale_lo(&self, index: usize) -> u64 { + self.scale_words[index] as u64 + } + + pub fn scale_hi(&self, index: usize) -> u64 { + (self.scale_words[index] >> 64) as u64 + } + + pub fn dst_lo(&self, index: usize) -> u64 { + self.dst_words[index] as u64 + } + + pub fn dst_hi(&self, index: usize) -> u64 { + (self.dst_words[index] >> 64) as u64 + } +} + +pub fn gen_case(index: u32, bid: u32) -> Int2FpCase { + if index >= NUM_CASES { + panic!("int2fp: unsupported directed case {index}"); + } + match index { + 0 => directed(bid, 2, 4, false, &MLIR_INPUT, &[0.125; 16]), + 1 => directed(bid, 3, 4, true, &RELU_INPUT, &[1.0; 16]), + 2 => directed(bid, 4, 8, false, &ITER8, &SCALE16), + 3 => directed(bid, 5, 8, true, &ITER8, &SCALE16), + 4 => directed(bid, 6, 16, false, &ITER16, &SCALE16), + 5 => directed(bid, 7, 16, true, &ITER16, &SCALE16), + _ => panic!("int2fp: unsupported directed case {index}"), + } +} + +const MLIR_INPUT: [i32; 16] = [1, 2, 3, -1, -2, 0, 4, 5, 10, -10, 7, 100, -100, 8, 16, -8]; +const RELU_INPUT: [i32; 16] = [-8, -1, 0, 1, 2, -3, 4, -5, 6, 7, -8, 9, -10, 11, 12, -13]; +const ITER8: [i32; 32] = [ + 8, -4, 16, 0, 1, -2, 3, -8, 10, -10, 7, 100, -100, 8, 16, -8, 4, -1, 0, 2, -16, 32, -32, 5, 9, + -9, 12, -12, 6, -6, 11, -11, +]; +const ITER16: [i32; 64] = [ + 1, 2, 3, -1, -2, 0, 4, 5, 10, -10, 7, 100, -100, 8, 16, -8, 8, -4, 16, 0, 1, -2, 3, -8, 12, -12, + 6, -6, 9, -9, 11, -11, 20, -20, 24, -24, 15, -15, 18, -18, 30, -30, 40, -40, 25, -25, 28, -28, + 50, -50, 60, -60, 7, -7, 13, -13, 14, -14, 17, -17, 19, -19, 21, -21, +]; +const SCALE16: [f32; 16] = [ + 0.125, 0.25, 0.5, 1.0, 0.25, 0.5, 1.0, 2.0, 0.5, 1.0, 2.0, 0.125, 1.0, 0.125, 0.25, 0.5, +]; + +fn pack_u32s(vals: &[u32]) -> Vec { + if vals.len() % LANES != 0 { + panic!("int2fp: value count {} is not a multiple of four", vals.len()); + } + vals.chunks(LANES) + .map(|chunk| { + let mut word = 0u128; + for (lane, value) in chunk.iter().enumerate() { + word |= u128::from(*value) << (lane * 32); + } + word + }) + .collect() +} + +fn copy_words(src: &[u128]) -> [u128; N] { + if src.len() > N { + panic!("int2fp: {} words exceed buffer {N}", src.len()); + } + let mut out = [0u128; N]; + out[..src.len()].copy_from_slice(src); + out +} + +fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { + u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) +} + +fn split_rs(value: u64) -> (u32, u32) { + (value as u32, (value >> 32) as u32) +} + +fn positive_finite(bits: u32) -> bool { + bits >> 31 == 0 && ((bits >> 23) & 0xff) != 0xff && (bits & 0x7fff_ffff) != 0 +} + +pub(crate) fn validate( + op1_bank: u32, + op2_bank: u32, + wr_bank: u32, + op1_col: u32, + op2_col: u32, + wr_col: u32, + iter: u32, + rs1: u64, + rs2: u64, + scales: &[u128], +) { + if iter == 0 || iter % 4 != 0 { + panic!("INT32_TO_FP32 iter must be a positive multiple of four"); + } + if iter > BANK_ENTRIES { + panic!("INT32_TO_FP32 input exceeds bank depth"); + } + if op1_bank >= BANK_NUM || op2_bank >= BANK_NUM || wr_bank >= BANK_NUM { + panic!("INT32_TO_FP32 bank id is invalid"); + } + if op1_col != 1 || op2_col != 1 || wr_col != 1 { + panic!("INT32_TO_FP32 operands must each occupy one bank"); + } + if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { + panic!("INT32_TO_FP32 banks must be distinct"); + } + if (rs1 & 0x3ff) != u64::from(op1_bank) + || ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) + || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) + { + panic!("int2fp: rs1 bank fields mismatch"); + } + if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { + panic!("int2fp: rs1 iter mismatch"); + } + if rs2 >> 1 != 0 { + panic!("INT32_TO_FP32 reserves rs2[63:1]"); + } + if scales.len() != SCALE_ROWS { + panic!("INT32_TO_FP32 must emit four scale rows"); + } + for word in scales { + for lane in 0..LANES { + let bits = (*word >> (lane * 32)) as u32; + if !positive_finite(bits) { + panic!("INT32_TO_FP32 scales must be finite and positive"); + } + } + } +} + +fn dst_words(src: &[u128], scales: &[u128], relu: bool) -> Vec { + src.iter() + .enumerate() + .map(|(row, word)| { + let scale = scales[row % SCALE_ROWS]; + let mut packed = 0u128; + for lane in 0..LANES { + let raw = (*word >> (lane * 32)) as u32 as i32; + let value = if relu { raw.max(0) } else { raw }; + let bits = int2fp_fp32_bits(value, (scale >> (lane * 32)) as u32); + packed |= u128::from(bits) << (lane * 32); + } + packed + }) + .collect() +} + +fn directed(bid: u32, rob_id: u32, iter: u32, relu: bool, values: &[i32], scales: &[f32]) -> Int2FpCase { + if values.len() != iter as usize * LANES { + panic!( + "int2fp: {} values do not match iter {iter} * {LANES}", + values.len() + ); + } + if scales.len() != SCALE_ROWS * LANES { + panic!("int2fp: {} scales, expected {}", scales.len(), SCALE_ROWS * LANES); + } + let src = pack_u32s(&values.iter().map(|v| *v as u32).collect::>()); + let scale = pack_u32s(&scales.iter().map(|v| v.to_bits()).collect::>()); + let op1_bank = 0; + let op2_bank = 1; + let wr_bank = 2; + let op1_col = 1; + let op2_col = 1; + let wr_col = 1; + let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); + let rs2 = u64::from(relu); + validate( + op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs1, rs2, &scale, + ); + if src.len() != iter as usize { + panic!("int2fp: src rows {} != iter {iter}", src.len()); + } + let dst = dst_words(&src, &scale, relu); + if dst.len() != iter as usize { + panic!("int2fp: dst rows {} != iter {iter}", dst.len()); + } + let (rs1_lo, rs1_hi) = split_rs(rs1); + let (rs2_lo, rs2_hi) = split_rs(rs2); + Int2FpCase { + cmd: Int2FpCmd { + bid, + iter, + relu: u32::from(relu), + op1_bank, + op2_bank, + wr_bank, + op1_col, + op2_col, + wr_col, + rob_id, + rs1_lo, + rs1_hi, + rs2_lo, + rs2_hi, + num_src_words: src.len() as u32, + num_scale_words: scale.len() as u32, + num_dst_words: dst.len() as u32, + }, + src_words: copy_words(&src), + scale_words: copy_words(&scale), + dst_words: copy_words(&dst), + } +} diff --git a/examples/components/balls/int2fp/verify/casegen/src/lib.rs b/examples/components/balls/int2fp/verify/casegen/src/lib.rs new file mode 100644 index 000000000..bf1d33335 --- /dev/null +++ b/examples/components/balls/int2fp/verify/casegen/src/lib.rs @@ -0,0 +1,108 @@ +mod casegen; +#[path = "../../../emu/src/model.rs"] +mod model; + +use std::cell::RefCell; + +use casegen::{Int2FpCase, Int2FpCmd, MAX_DST, MAX_SCALE, MAX_SRC}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn int2fp_case_load(index: u32, bid: u32) -> i32 { + let case = casegen::gen_case(index, bid); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("int2fp_case: no case loaded; call int2fp_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn int2fp_case_cmd(out_ptr: *mut Int2FpCmd) { + if out_ptr.is_null() { + panic!("int2fp_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +fn src_index(index: u32, limit: u32, what: &str) -> usize { + if index >= limit { + panic!("int2fp_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn int2fp_case_src_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_src_words, "src_word_lo"); + if i >= MAX_SRC { + panic!("int2fp_case_src_word_lo: word_index out of range"); + } + case.src_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn int2fp_case_src_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_src_words, "src_word_hi"); + if i >= MAX_SRC { + panic!("int2fp_case_src_word_hi: word_index out of range"); + } + case.src_hi(i) + }) +} + +#[no_mangle] +pub extern "C" fn int2fp_case_scale_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_lo"); + if i >= MAX_SCALE { + panic!("int2fp_case_scale_word_lo: word_index out of range"); + } + case.scale_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn int2fp_case_scale_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_hi"); + if i >= MAX_SCALE { + panic!("int2fp_case_scale_word_hi: word_index out of range"); + } + case.scale_hi(i) + }) +} + +#[no_mangle] +pub extern "C" fn int2fp_case_dst_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_lo"); + if i >= MAX_DST { + panic!("int2fp_case_dst_word_lo: word_index out of range"); + } + case.dst_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn int2fp_case_dst_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_hi"); + if i >= MAX_DST { + panic!("int2fp_case_dst_word_hi: word_index out of range"); + } + case.dst_hi(i) + }) +} diff --git a/examples/balls/int2fp/verify/filelists/int2fp_ball.f b/examples/components/balls/int2fp/verify/filelists/int2fp_ball.f similarity index 100% rename from examples/balls/int2fp/verify/filelists/int2fp_ball.f rename to examples/components/balls/int2fp/verify/filelists/int2fp_ball.f diff --git a/examples/balls/int2fp/verify/src/common/int2fp_defs.svh b/examples/components/balls/int2fp/verify/src/common/int2fp_defs.svh similarity index 91% rename from examples/balls/int2fp/verify/src/common/int2fp_defs.svh rename to examples/components/balls/int2fp/verify/src/common/int2fp_defs.svh index 96a52851a..5ae0d67a8 100644 --- a/examples/balls/int2fp/verify/src/common/int2fp_defs.svh +++ b/examples/components/balls/int2fp/verify/src/common/int2fp_defs.svh @@ -31,10 +31,10 @@ import "DPI-C" function longint unsigned int2fp_case_dst_word_lo(input int unsig import "DPI-C" function longint unsigned int2fp_case_dst_word_hi(input int unsigned word_index); `ifndef INT32_TO_FP32_FUNCT7 -`error "INT32_TO_FP32_FUNCT7 must be provided by the selected Core ballISA" +`error "INT32_TO_FP32_FUNCT7 must be provided by the selected Thread ballISA" `endif -localparam int INT32_TO_FP32_CORE_FUNCT7 = `INT32_TO_FP32_FUNCT7; +localparam int INT32_TO_FP32_THREAD_FUNCT7 = `INT32_TO_FP32_FUNCT7; localparam int INT2FP_MAX_SRC_WORDS = 16; localparam int INT2FP_MAX_SCALE_WORDS = 4; localparam int INT2FP_MAX_DST_WORDS = 16; diff --git a/examples/balls/int2fp/verify/src/common/int2fp_items.svh b/examples/components/balls/int2fp/verify/src/common/int2fp_items.svh similarity index 96% rename from examples/balls/int2fp/verify/src/common/int2fp_items.svh rename to examples/components/balls/int2fp/verify/src/common/int2fp_items.svh index eafba5bce..12cccf89a 100644 --- a/examples/balls/int2fp/verify/src/common/int2fp_items.svh +++ b/examples/components/balls/int2fp/verify/src/common/int2fp_items.svh @@ -10,7 +10,7 @@ class int2fp_cmd_item extends bb_blink_cmd_item; bit [127:0] dst_words[INT2FP_MAX_DST_WORDS]; constraint legal_c { - funct7 == INT32_TO_FP32_CORE_FUNCT7[6:0]; + funct7 == INT32_TO_FP32_THREAD_FUNCT7[6:0]; op1_en == 1'b1; op2_en == 1'b1; wr_spad_en == 1'b1; @@ -57,7 +57,7 @@ class int2fp_cmd_item extends bb_blink_cmd_item; num_src_words = cmd.num_src_words; num_scale_words = cmd.num_scale_words; num_dst_words = cmd.num_dst_words; - funct7 = INT32_TO_FP32_CORE_FUNCT7[6:0]; + funct7 = INT32_TO_FP32_THREAD_FUNCT7[6:0]; special = 64'd0; op1_en = 1'b1; op2_en = 1'b1; @@ -88,7 +88,7 @@ class int2fp_cmd_item extends bb_blink_cmd_item; `uvm_fatal("CASE", $sformatf("num_src_words out of range: %0d", num_src_words)) if (num_dst_words == 0 || num_dst_words > INT2FP_MAX_DST_WORDS) `uvm_fatal("CASE", $sformatf("num_dst_words out of range: %0d", num_dst_words)) - if (funct7 != INT32_TO_FP32_CORE_FUNCT7[6:0]) + if (funct7 != INT32_TO_FP32_THREAD_FUNCT7[6:0]) `uvm_fatal("CASE", $sformatf("invalid funct7: %0d", funct7)) for (i = 0; i < num_src_words; i++) begin diff --git a/examples/balls/int2fp/verify/src/cov/int2fp_cov.svh b/examples/components/balls/int2fp/verify/src/cov/int2fp_cov.svh similarity index 91% rename from examples/balls/int2fp/verify/src/cov/int2fp_cov.svh rename to examples/components/balls/int2fp/verify/src/cov/int2fp_cov.svh index 8b0727983..ffa99a772 100644 --- a/examples/balls/int2fp/verify/src/cov/int2fp_cov.svh +++ b/examples/components/balls/int2fp/verify/src/cov/int2fp_cov.svh @@ -20,7 +20,7 @@ class int2fp_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != INT32_TO_FP32_CORE_FUNCT7[6:0]) + if (item.funct7 != INT32_TO_FP32_THREAD_FUNCT7[6:0]) `uvm_fatal("COV", $sformatf("unexpected funct7 %0d", item.funct7)) cur_relu = item.rs2[0]; cur_iter = item.iter; @@ -29,6 +29,7 @@ class int2fp_cov extends uvm_component; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") if (cmd_cg.get_coverage() != 100.0) diff --git a/examples/balls/int2fp/verify/src/env/int2fp_env.svh b/examples/components/balls/int2fp/verify/src/env/int2fp_env.svh similarity index 100% rename from examples/balls/int2fp/verify/src/env/int2fp_env.svh rename to examples/components/balls/int2fp/verify/src/env/int2fp_env.svh diff --git a/examples/balls/int2fp/verify/src/env/int2fp_scoreboard.svh b/examples/components/balls/int2fp/verify/src/env/int2fp_scoreboard.svh similarity index 98% rename from examples/balls/int2fp/verify/src/env/int2fp_scoreboard.svh rename to examples/components/balls/int2fp/verify/src/env/int2fp_scoreboard.svh index d21c506d6..15d1636ca 100644 --- a/examples/balls/int2fp/verify/src/env/int2fp_scoreboard.svh +++ b/examples/components/balls/int2fp/verify/src/env/int2fp_scoreboard.svh @@ -67,6 +67,7 @@ class int2fp_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, int2fp_cmd_item exp); + verification_pkg::evidence::checks++; if (got.bid !== exp.bid) `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) if (got.funct7 !== exp.funct7) @@ -180,6 +181,7 @@ class int2fp_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) begin `uvm_fatal("SCB", $sformatf( diff --git a/examples/balls/int2fp/verify/src/pkg/int2fp_pkg.sv b/examples/components/balls/int2fp/verify/src/pkg/int2fp_pkg.sv similarity index 100% rename from examples/balls/int2fp/verify/src/pkg/int2fp_pkg.sv rename to examples/components/balls/int2fp/verify/src/pkg/int2fp_pkg.sv diff --git a/examples/balls/int2fp/verify/src/seq/int2fp_sequences.svh b/examples/components/balls/int2fp/verify/src/seq/int2fp_sequences.svh similarity index 100% rename from examples/balls/int2fp/verify/src/seq/int2fp_sequences.svh rename to examples/components/balls/int2fp/verify/src/seq/int2fp_sequences.svh diff --git a/examples/components/balls/int2fp/verify/src/tb_top.sv b/examples/components/balls/int2fp/verify/src/tb_top.sv new file mode 100644 index 000000000..219f61c87 --- /dev/null +++ b/examples/components/balls/int2fp/verify/src/tb_top.sv @@ -0,0 +1,101 @@ +module tb_top; + import uvm_pkg::*; + import int2fp_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + Int2FpBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .io_status_idle(), + .io_status_running(), + .io_bankAccessReady(1'b1), + .io_cmdReq_ready(intf.cmd_req_ready), + .io_cmdReq_valid(intf.cmd_req_valid), + .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), + .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), + .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), + .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), + .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), + .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), + .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), + .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), + .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), + .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), + .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), + .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), + .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), + .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), + .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), + .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), + .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), + .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), + .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), + .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), + .io_cmdResp_ready(intf.cmd_resp_ready), + .io_cmdResp_valid(intf.cmd_resp_valid), + .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), + .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), + .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .io_bankRead_0_ball_id(), + .io_bankRead_0_group_id(intf.bank_read_group_id[0]), + .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .io_bankRead_1_ball_id(), + .io_bankRead_1_group_id(intf.bank_read_group_id[1]), + .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .io_bankWrite_0_ball_id(), + .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .io_subRobReq_ready(intf.sub_rob_req_ready) + ); + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("int2fp_ball_test"); + end +endmodule diff --git a/examples/balls/int2fp/verify/src/tests/int2fp_test.svh b/examples/components/balls/int2fp/verify/src/tests/int2fp_test.svh similarity index 100% rename from examples/balls/int2fp/verify/src/tests/int2fp_test.svh rename to examples/components/balls/int2fp/verify/src/tests/int2fp_test.svh diff --git a/examples/components/balls/int2fp/workloads/ctests/CMakeLists.txt b/examples/components/balls/int2fp/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..97f268d47 --- /dev/null +++ b/examples/components/balls/int2fp/workloads/ctests/CMakeLists.txt @@ -0,0 +1,6 @@ +add_buckyball_ctest(int2fp_tensor_values_test.c) +add_buckyball_ctest(int2fp_tensor_groups_test.c) +add_buckyball_ctest(int2fp_tensor_rows_test.c) +add_buckyball_ctest(int2fp_channel_lanes_test.c) +add_buckyball_ctest(int2fp_channel_base_test.c) +add_buckyball_ctest(int2fp_channel_two_rows_test.c) diff --git a/examples/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c b/examples/components/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c similarity index 96% rename from examples/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c rename to examples/components/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c index 44b9f7cf2..424c1ea30 100644 --- a/examples/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c +++ b/examples/components/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c @@ -18,8 +18,8 @@ static const float expected[16] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 4, 1); bb_mem_alloc(1, 4, 1); diff --git a/examples/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c b/examples/components/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c similarity index 96% rename from examples/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c rename to examples/components/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c index 438345cad..844242f82 100644 --- a/examples/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c +++ b/examples/components/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c @@ -18,8 +18,8 @@ static const float expected[16] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 4, 1); bb_mem_alloc(1, 4, 1); diff --git a/examples/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c b/examples/components/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c similarity index 96% rename from examples/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c rename to examples/components/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c index 972ee8735..32a70656c 100644 --- a/examples/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c +++ b/examples/components/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c @@ -20,8 +20,8 @@ static const float expected[32] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 8, 1); bb_mem_alloc(1, 4, 1); diff --git a/examples/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c b/examples/components/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c similarity index 96% rename from examples/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c rename to examples/components/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c index 69f38c09e..f35663ecd 100644 --- a/examples/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c +++ b/examples/components/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c @@ -18,8 +18,8 @@ static const float expected[16] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 4, 1); bb_mem_alloc(1, 4, 1); diff --git a/examples/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c b/examples/components/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c similarity index 96% rename from examples/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c rename to examples/components/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c index ee07f978e..573837d41 100644 --- a/examples/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c +++ b/examples/components/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c @@ -15,8 +15,8 @@ static const float expected[8] = {1.0f, -1.0f, 2.0f, -2.0f, 3.0f, -3.0f, 4.0f, -4.0f}; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 4, 1); bb_mem_alloc(1, 4, 1); diff --git a/examples/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c b/examples/components/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c similarity index 96% rename from examples/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c rename to examples/components/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c index eb3769194..eb63ed438 100644 --- a/examples/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c +++ b/examples/components/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c @@ -23,8 +23,8 @@ static const uint32_t expected[4] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 4, 1); bb_mem_alloc(1, 4, 1); diff --git a/examples/balls/int2fp/workloads/isa/int2fp.h b/examples/components/balls/int2fp/workloads/isa/int2fp.h similarity index 100% rename from examples/balls/int2fp/workloads/isa/int2fp.h rename to examples/components/balls/int2fp/workloads/isa/int2fp.h diff --git a/examples/balls/int2fp/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/int2fp/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/int2fp/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/int2fp/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/int2fp/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/int2fp/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/int2fp/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/int2fp/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir b/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir new file mode 100644 index 000000000..79ba711b9 --- /dev/null +++ b/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir @@ -0,0 +1,28 @@ +func.func private @check_result(memref<4x4xf32>) -> () + +memref.global "private" constant @input : memref<4x4xi32> = dense<[ + [1, 2, 3, -1], [-2, 0, 4, 5], + [10, -10, 7, 100], [-100, 8, 16, -8] +]> +memref.global "private" constant @scales : memref<4x4xf32> = + dense<1.250000e-01> + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %input = memref.get_global @input : memref<4x4xi32> + %scales = memref.get_global @scales : memref<4x4xf32> + %output = memref.alloc() alignment = 64 : memref<4x4xf32> + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 0 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> + buckyball.mvin %scales <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + buckyball.custom "int32_to_fp32" <{sources = array, destination = 2 : i64}> {rows = 1 : i64} + buckyball.mvout %output <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + func.call @check_result(%output) : (memref<4x4xf32>) -> () + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %output : memref<4x4xf32> + return %zero_i8 : i8 +} diff --git a/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4_main.cpp b/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4_main.cpp similarity index 100% rename from examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4_main.cpp rename to examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4_main.cpp diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir b/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir new file mode 100644 index 000000000..993c6e850 --- /dev/null +++ b/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir @@ -0,0 +1,27 @@ +func.func private @check_result(memref<4x4xf32>) -> () + +memref.global "private" constant @input : memref<4x4xi32> = dense<[ + [-8, -1, 0, 1], [2, -3, 4, -5], + [6, 7, -8, 9], [-10, 11, 12, -13] +]> +memref.global "private" constant @scales : memref<4x4xf32> = dense<1.0> + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %input = memref.get_global @input : memref<4x4xi32> + %scales = memref.get_global @scales : memref<4x4xf32> + %output = memref.alloc() alignment = 64 : memref<4x4xf32> + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 0 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> + buckyball.mvin %scales <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + buckyball.custom "int32_to_fp32_relu" <{sources = array, destination = 2 : i64}> {rows = 1 : i64} + buckyball.mvout %output <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + func.call @check_result(%output) : (memref<4x4xf32>) -> () + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %output : memref<4x4xf32> + return %zero_i8 : i8 +} diff --git a/examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4_main.cpp b/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4_main.cpp similarity index 100% rename from examples/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4_main.cpp rename to examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4_main.cpp diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..e977427a0 --- /dev/null +++ b/examples/components/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +endif() +set(BUCKYBALL_MLIR_TEST_PREFIX bank) + +set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") +file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") +string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") +# int32_to_fp32 rows hold sixteen i32, four bank rows each. +math(EXPR BB_INT2FP_ROWS "${_lines} / 4") +add_buckyball_mlir_test(int2fp_unit_scale TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir b/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir new file mode 100644 index 000000000..a4288842c --- /dev/null +++ b/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir @@ -0,0 +1,28 @@ +func.func private @check_result(memref<4x4xf32>) -> () + +memref.global "private" constant @input : memref<4x4xi32> = dense<[ + [1, 2, 3, -1], [-2, 0, 4, 5], + [10, -10, 7, 100], [-100, 8, 16, -8] +]> +memref.global "private" constant @scales : memref<4x4xf32> = + dense<1.250000e-01> + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %input = memref.get_global @input : memref<4x4xi32> + %scales = memref.get_global @scales : memref<4x4xf32> + %output = memref.alloc() alignment = 64 : memref<4x4xf32> + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %scale_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> + %loaded_scales = buckyball.bank_mvin %scales, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + %result = buckyball.bank_custom "int32_to_fp32" (%loaded_input, %loaded_scales) -> %output_bank {rows = 1 : i64} + %stored = buckyball.bank_mvout %output, %result <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + func.call @check_result(%output) : (memref<4x4xf32>) -> () + buckyball.bank_release %loaded_input + buckyball.bank_release %loaded_scales + buckyball.bank_release %stored + memref.dealloc %output : memref<4x4xf32> + return %zero_i8 : i8 +} diff --git a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4_main.cpp b/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4_main.cpp similarity index 100% rename from examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4_main.cpp rename to examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4_main.cpp diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir b/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir new file mode 100644 index 000000000..58ac1465d --- /dev/null +++ b/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir @@ -0,0 +1,27 @@ +func.func private @check_result(memref<4x4xf32>) -> () + +memref.global "private" constant @input : memref<4x4xi32> = dense<[ + [-8, -1, 0, 1], [2, -3, 4, -5], + [6, 7, -8, 9], [-10, 11, 12, -13] +]> +memref.global "private" constant @scales : memref<4x4xf32> = dense<1.0> + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %input = memref.get_global @input : memref<4x4xi32> + %scales = memref.get_global @scales : memref<4x4xf32> + %output = memref.alloc() alignment = 64 : memref<4x4xf32> + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %scale_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> + %loaded_scales = buckyball.bank_mvin %scales, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + %result = buckyball.bank_custom "int32_to_fp32_relu" (%loaded_input, %loaded_scales) -> %output_bank {rows = 1 : i64} + %stored = buckyball.bank_mvout %output, %result <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + func.call @check_result(%output) : (memref<4x4xf32>) -> () + buckyball.bank_release %loaded_input + buckyball.bank_release %loaded_scales + buckyball.bank_release %stored + memref.dealloc %output : memref<4x4xf32> + return %zero_i8 : i8 +} diff --git a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4_main.cpp b/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4_main.cpp similarity index 100% rename from examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4_main.cpp rename to examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4_main.cpp diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in b/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in new file mode 100644 index 000000000..26ca8c1e4 --- /dev/null +++ b/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in @@ -0,0 +1,36 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %lanes = arith.constant @BB_BANK_I32_LANES@ : index + %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %scale = memref.alloc() : memref<4x4xf32> + %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32> + %one_f32 = arith.constant 1.0 : f32 + linalg.fill ins(%one_f32 : f32) outs(%scale : memref<4x4xf32>) + scf.for %row = %zero to %rows step %one { + scf.for %lane = %zero to %lanes step %one { + %r = arith.index_cast %row : index to i32 + %l = arith.index_cast %lane : index to i32 + %value = arith.subi %r, %l : i32 + memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + } + } + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %scale_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %loaded_scale = buckyball.bank_mvin %scale, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + %result = buckyball.bank_custom "int32_to_fp32" (%loaded_input, %loaded_scale) -> %output_bank {rows = @BB_INT2FP_ROWS@ : i64} + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32> + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32>) -> () + buckyball.bank_release %loaded_input + buckyball.bank_release %loaded_scale + buckyball.bank_release %stored + memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + memref.dealloc %scale : memref<4x4xf32> + memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32> + return %zero_i8 : i8 +} diff --git a/examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp b/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp similarity index 100% rename from examples/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp rename to examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp diff --git a/examples/components/balls/int8add/arch/Int8AddBall.scala b/examples/components/balls/int8add/arch/Int8AddBall.scala new file mode 100644 index 000000000..11fab6a9b --- /dev/null +++ b/examples/components/balls/int8add/arch/Int8AddBall.scala @@ -0,0 +1,291 @@ +package examples.balls.int8add + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.int8add.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import chisel3.util._ +import hardfloat.{rawFloatFromRecFN, recFNFromFN, RecFNToIN, RoundRawFNToRecFN} +import hardfloat.consts.{round_near_even, tininess_afterRounding} +import framework.lib.arith.{Fma, IntToFloat} + +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class Int8AddBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + val addFunct = ball.instructions + .find(_.mnemonic == "INT8ADD").get.funct7 + + val reluFunct = ball.instructions + .find(_.mnemonic == "INT8ADD_RELU").get.funct7 + + require(ball.inBW == 2, "Int8AddBall requires inBW=2") + require(ball.outBW == 1, "Int8AddBall requires outBW=1") + require(mp.bankWidth == 128, "Int8AddBall requires 128-bit bank rows") + require((addFunct >> 4) == 4 && (reluFunct >> 4) == 4, "Int8AddBall instructions must encode two reads and one write") + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val idle :: waitForChannels :: readRequest :: readResponse :: calculate :: calculateDrain :: writeRequest :: writeResponse :: complete :: Nil = + Enum(9) + val state = RegInit(idle) + val robId = RegInit(0.U(log2Up(fp.rob_entries).W)) + val lhsBank = RegInit(0.U(fp.bank_id_len.W)) + val rhsBank = RegInit(0.U(fp.bank_id_len.W)) + val outputBank = RegInit(0.U(fp.bank_id_len.W)) + val groups = RegInit(0.U(5.W)) + val group = RegInit(0.U(5.W)) + val bases = Reg(Vec(3, UInt(mp.groupIdWidth.W))) + val iter = RegInit(0.U(fp.iter_len.W)) + val row = RegInit(0.U(log2Ceil(mp.bankEntries).W)) + val lhsRatio = Reg(UInt(32.W)) + val rhsRatio = Reg(UInt(32.W)) + val relu = RegInit(false.B) + val lhsWord = Reg(UInt(128.W)) + val rhsWord = Reg(UInt(128.W)) + val lhsRequested = RegInit(false.B) + val rhsRequested = RegInit(false.B) + val lhsReceived = RegInit(false.B) + val rhsReceived = RegInit(false.B) + val lane = RegInit(0.U(4.W)) + val completed = RegInit(0.U(5.W)) + val outputWord = Reg(Vec(16, UInt(8.W))) + + def positiveFinite(value: UInt): Bool = + !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U + + val lhsToFloat = Instantiate(new IntToFloat(8, 8, 24)) + + val rhsToFloat = Instantiate(new IntToFloat(8, 8, 24)) + + val lhsMul = Instantiate(new Fma(3, 8, 24)) + + val rhsMul = Instantiate(new Fma(3, 8, 24)) + + val add = Instantiate(new PipelinedAddRawFN(8, 24)) + + val round = Instantiate(new RoundRawFNToRecFN(8, 24, 0)) + val toInt = Instantiate(new RecFNToIN(8, 24, 8)) + val lhsValue = (lhsWord >> (lane << 3))(7, 0) + val rhsValue = (rhsWord >> (lane << 3))(7, 0) + val calculateIssue = state === calculate + + val ratioStage0 = RegEnable(lhsRatio, calculateIssue) + val ratioStage1 = RegEnable(ratioStage0, RegNext(calculateIssue, false.B)) + val rhsRatioStage0 = RegEnable(rhsRatio, calculateIssue) + val rhsRatioStage1 = RegEnable(rhsRatioStage0, RegNext(calculateIssue, false.B)) + val laneStage0 = RegEnable(lane, calculateIssue) + val laneStage1 = RegEnable(laneStage0, RegNext(calculateIssue, false.B)) + + val conversionValid = lhsToFloat.io.validout + val productLane = Pipe(conversionValid, laneStage1, 3) + + val addResult = add.io.rawOut + val addLane = Pipe(lhsMul.io.validout, productLane.bits, 4) + val addValid = add.io.validout + val roundedResult = RegEnable(round.io.out, addValid) + val roundedLane = RegEnable(addLane, addValid) + val resultValid = RegNext(addValid, false.B) + + lhsToFloat.io.validin := calculateIssue + lhsToFloat.io.signedIn := true.B + lhsToFloat.io.in := lhsValue + lhsToFloat.io.roundingMode := round_near_even + lhsToFloat.io.detectTininess := tininess_afterRounding + rhsToFloat.io.validin := calculateIssue + rhsToFloat.io.signedIn := true.B + rhsToFloat.io.in := rhsValue + rhsToFloat.io.roundingMode := round_near_even + rhsToFloat.io.detectTininess := tininess_afterRounding + + lhsMul.io.validin := conversionValid + lhsMul.io.op := 0.U + lhsMul.io.a := lhsToFloat.io.out + lhsMul.io.b := recFNFromFN(8, 24, ratioStage1) + lhsMul.io.c := recFNFromFN(8, 24, 0.U) + lhsMul.io.roundingMode := round_near_even + lhsMul.io.detectTininess := tininess_afterRounding.asUInt + rhsMul.io.validin := conversionValid + rhsMul.io.op := 0.U + rhsMul.io.a := rhsToFloat.io.out + rhsMul.io.b := recFNFromFN(8, 24, rhsRatioStage1) + rhsMul.io.c := recFNFromFN(8, 24, 0.U) + rhsMul.io.roundingMode := round_near_even + rhsMul.io.detectTininess := tininess_afterRounding.asUInt + + add.io.validin := lhsMul.io.validout + add.io.subOp := false.B + add.io.a := rawFloatFromRecFN(8, 24, lhsMul.io.out) + add.io.b := rawFloatFromRecFN(8, 24, rhsMul.io.out) + add.io.roundingMode := round_near_even + round.io.invalidExc := add.io.invalidExc + round.io.infiniteExc := false.B + round.io.in := addResult + round.io.roundingMode := round_near_even + round.io.detectTininess := tininess_afterRounding + toInt.io.in := roundedResult + toInt.io.roundingMode := round_near_even + toInt.io.signedOut := true.B + val result = Mux(relu && toInt.io.out(7), 0.U, toInt.io.out) + + blink.cmdReq.ready := state === idle + blink.cmdResp.valid := state === complete + blink.cmdResp.bits.rob_id := robId + blink.status.idle := state === idle + blink.status.running := state =/= idle && state =/= complete + + for (port <- 0 until 2) { + blink.bankRead(port).rob_id := robId + blink.bankRead(port).bank_id := Mux(port.U === 0.U, lhsBank, rhsBank) + blink.bankRead(port).group_id := bases(port) + group + blink.bankRead(port).io.req.valid := false.B + blink.bankRead(port).io.req.bits.addr := row + blink.bankRead(port).io.resp.ready := false.B + } + blink.bankWrite(0).rob_id := robId + blink.bankWrite(0).bank_id := outputBank + blink.bankWrite(0).group_id := bases(2) + group + blink.bankWrite(0).io.req.valid := false.B + blink.bankWrite(0).io.req.bits.addr := row + blink.bankWrite(0).io.req.bits.data := Cat(outputWord.reverse) + blink.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) + blink.bankWrite(0).io.resp.ready := false.B + + switch(state) { + is(idle) { + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + assert( + cmd.funct === addFunct.U || cmd.funct === reluFunct.U, + "Int8AddBall received an unknown funct7" + ) + assert( + positiveFinite(cmd.rs2(31, 0)) && positiveFinite(cmd.rs2(63, 32)), + "Int8AddBall ratios must be finite and positive" + ) + assert( + cmd.op1_col =/= 0.U && cmd.op1_col === cmd.op2_col && cmd.op1_col === cmd.wr_col, + "Int8AddBall bank groups must match" + ) + assert( + cmd.op1_col <= mp.bankNum.U, + "Int8AddBall bank groups exceed physical banks" + ) + assert( + cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, + "Int8AddBall banks must be distinct" + ) + assert( + cmd.iter =/= 0.U && cmd.iter <= mp.bankEntries.U, + "Int8AddBall iter must fit one physical bank" + ) + robId := blink.cmdReq.bits.rob_id + lhsBank := cmd.op1_bank + rhsBank := cmd.op2_bank + outputBank := cmd.wr_bank + groups := cmd.op1_col + group := 0.U + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + iter := cmd.iter + row := 0.U + lhsRatio := cmd.rs2(31, 0) + rhsRatio := cmd.rs2(63, 32) + relu := cmd.funct === reluFunct.U + lhsRequested := false.B + rhsRequested := false.B + lhsReceived := false.B + rhsReceived := false.B + lane := 0.U + state := waitForChannels + } + } + is(waitForChannels) { + state := readRequest + } + is(readRequest) { + blink.bankRead(0).io.req.valid := !lhsRequested + blink.bankRead(1).io.req.valid := !rhsRequested + when(blink.bankRead(0).io.req.fire)(lhsRequested := true.B) + when(blink.bankRead(1).io.req.fire)(rhsRequested := true.B) + when((lhsRequested || blink.bankRead(0).io.req.fire) && (rhsRequested || blink.bankRead(1).io.req.fire)) { + state := readResponse + } + } + is(readResponse) { + blink.bankRead(0).io.resp.ready := !lhsReceived + blink.bankRead(1).io.resp.ready := !rhsReceived + when(blink.bankRead(0).io.resp.fire) { lhsWord := blink.bankRead(0).io.resp.bits.data; lhsReceived := true.B } + when(blink.bankRead(1).io.resp.fire) { rhsWord := blink.bankRead(1).io.resp.bits.data; rhsReceived := true.B } + when((lhsReceived || blink.bankRead(0).io.resp.fire) && (rhsReceived || blink.bankRead(1).io.resp.fire)) { + lane := 0.U + completed := 0.U + state := calculate + } + } + is(calculate) { + when(lane === 15.U) { + state := calculateDrain + }.otherwise { + lane := lane + 1.U + } + } + is(calculateDrain) {} + is(writeRequest) { + blink.bankWrite(0).io.req.valid := true.B + when(blink.bankWrite(0).io.req.fire)(state := writeResponse) + } + is(writeResponse) { + blink.bankWrite(0).io.resp.ready := true.B + when(blink.bankWrite(0).io.resp.fire) { + lhsRequested := false.B + rhsRequested := false.B + lhsReceived := false.B + rhsReceived := false.B + when(row +& 1.U === iter) { + row := 0.U + when(group +& 1.U === groups)(state := complete) + .otherwise { group := group + 1.U; state := readRequest } + }.otherwise { row := row + 1.U; state := readRequest } + } + } + is(complete) { + when(blink.cmdResp.fire)(state := idle) + } + } + + when((state === calculate || state === calculateDrain) && resultValid) { + outputWord(roundedLane.bits) := result + completed := completed + 1.U + when(completed === 15.U) { + state := writeRequest + } + } +} + +object Int8AddBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new Int8AddBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/components/balls/int8add/arch/PipelinedAddRawFN.scala b/examples/components/balls/int8add/arch/PipelinedAddRawFN.scala new file mode 100644 index 000000000..afab4d310 --- /dev/null +++ b/examples/components/balls/int8add/arch/PipelinedAddRawFN.scala @@ -0,0 +1,171 @@ +package examples.balls.int8add + +import chisel3.experimental.hierarchy.{instantiable, public} + +import chisel3._ +import chisel3.util._ +import hardfloat.{countLeadingZeros, isSigNaNRawFloat, lowMask, orReduceBy2, orReduceBy4, RawFloat} +import hardfloat.consts.round_min + +/** + * Throughput-oriented raw floating-point adder used by Int8AddBall. + * + * The Berkeley HardFloat AddRawFN implementation is combinational. This + * wrapper keeps the same raw-float equations but registers the alignment, + * normalization, and final select boundaries. It accepts one operand pair + * per cycle and reports the result four cycles later. + */ +@instantiable +class PipelinedAddRawFN(val expWidth: Int, val sigWidth: Int) extends Module { + + @public + val io = IO(new Bundle { + val validin = Input(Bool()) + val subOp = Input(Bool()) + val a = Input(new RawFloat(expWidth, sigWidth)) + val b = Input(new RawFloat(expWidth, sigWidth)) + val roundingMode = Input(UInt(3.W)) + val validout = Output(Bool()) + val invalidExc = Output(Bool()) + val rawOut = Output(new RawFloat(expWidth, sigWidth + 2)) + }) + + val alignDistWidth = log2Ceil(sigWidth) + + // Stage 0: register the raw operands and exponent comparison. + val a0 = RegNext(io.a) + val b0 = RegNext(io.b) + val effSignB0 = RegNext(io.b.sign ^ io.subOp) + val eqSigns0 = RegNext(io.a.sign === (io.b.sign ^ io.subOp)) + val sDiffExps0 = RegNext(io.a.sExp - io.b.sExp) + val roundingMode0 = RegNext(io.roundingMode) + + // Stage 1: prepare close-path subtraction and far-path operand selection. + val modNatAlignDist1 = Mux(sDiffExps0 < 0.S, -sDiffExps0, sDiffExps0)(alignDistWidth - 1, 0) + + val isMaxAlign1 = + (sDiffExps0 >> alignDistWidth) =/= 0.S && + ((sDiffExps0 >> alignDistWidth) =/= -1.S || sDiffExps0(alignDistWidth - 1, 0) === 0.U) + + val alignDist1 = Mux(isMaxAlign1, ((BigInt(1) << alignDistWidth) - 1).U, modNatAlignDist1) + val closeSubMags1 = !eqSigns0 && !isMaxAlign1 && (modNatAlignDist1 <= 1.U) + + val closeAlignedSigA1 = + Mux((0.S <= sDiffExps0) && sDiffExps0(0), a0.sig << 2, 0.U) | + Mux((0.S <= sDiffExps0) && !sDiffExps0(0), a0.sig << 1, 0.U) | + Mux(sDiffExps0 < 0.S, a0.sig, 0.U) + + val closeSSigSum1 = Wire(SInt((sigWidth + 3).W)) + closeSSigSum1 := closeAlignedSigA1.asSInt - (b0.sig << 1).asSInt + val closeSigSum1 = Wire(UInt((sigWidth + 2).W)) + closeSigSum1 := Mux(closeSSigSum1 < 0.S, -closeSSigSum1, closeSSigSum1)(sigWidth + 1, 0) + val closeAdjustedSigSum1 = Wire(UInt((sigWidth + 2).W)) + closeAdjustedSigSum1 := closeSigSum1 << (sigWidth & 1) + val closeReduced2SigSum1 = orReduceBy2(closeAdjustedSigSum1) + + val farSignOut1 = Mux(sDiffExps0 < 0.S, effSignB0, a0.sign) + val farSigLarger1 = Wire(UInt(sigWidth.W)) + farSigLarger1 := Mux(sDiffExps0 < 0.S, b0.sig, a0.sig)(sigWidth - 1, 0) + val farSigSmaller1 = Wire(UInt(sigWidth.W)) + farSigSmaller1 := Mux(sDiffExps0 < 0.S, a0.sig, b0.sig)(sigWidth - 1, 0) + + val a1 = RegNext(a0) + val b1 = RegNext(b0) + val effSignB1 = RegNext(effSignB0) + val eqSigns1 = RegNext(eqSigns0) + val sDiffExps1 = RegNext(sDiffExps0) + val roundingMode1 = RegNext(roundingMode0) + val alignDist1r = RegInit(0.U(alignDistWidth.W)) + alignDist1r := alignDist1 + val closeSubMags1r = RegInit(false.B) + closeSubMags1r := closeSubMags1 + val closeSSigSum1r = RegInit(0.S((sigWidth + 3).W)) + closeSSigSum1r := closeSSigSum1 + val closeSigSum1r = RegInit(0.U((sigWidth + 2).W)) + closeSigSum1r := closeSigSum1 + val closeReduced2SigSum1r = RegInit(0.U(((sigWidth + 3) / 2).W)) + closeReduced2SigSum1r := closeReduced2SigSum1 + val farSignOut1r = RegNext(farSignOut1) + val farSigLarger1r = RegInit(0.U(sigWidth.W)) + farSigLarger1r := farSigLarger1 + val farSigSmaller1r = RegInit(0.U(sigWidth.W)) + farSigSmaller1r := farSigSmaller1 + + // Stage 2: leading-zero normalization and far-path alignment. + val closeNormDistReduced2_2 = countLeadingZeros(closeReduced2SigSum1r) + val closeNearNormDist2 = (closeNormDistReduced2_2 << 1)(alignDistWidth - 1, 0) + val closeSigOut2 = ((closeSigSum1r << closeNearNormDist2) << 1)(sigWidth + 2, 0) + val closeTotalCancellation2 = !(closeSigOut2(sigWidth + 2, sigWidth + 1).orR) + val closeNotTotalCancellationSignOut2 = a1.sign ^ (closeSSigSum1r < 0.S) + + val farMainAlignedSigSmaller2 = Wire(UInt((sigWidth + 5).W)) + farMainAlignedSigSmaller2 := (farSigSmaller1r << 5) >> alignDist1r + val farReduced4SigSmaller2 = Wire(UInt(((sigWidth + 5) / 4).W)) + farReduced4SigSmaller2 := orReduceBy4(Cat(farSigSmaller1r, 0.U(2.W))) + val farRoundExtraMask2 = lowMask(alignDist1r(alignDistWidth - 1, 2), (sigWidth + 5) / 4, 0) + + val farAlignedSigSmaller2 = Cat( + farMainAlignedSigSmaller2 >> 3, + farMainAlignedSigSmaller2(2, 0).orR || (farReduced4SigSmaller2 & farRoundExtraMask2).orR + ) + + val farSubMags2 = !eqSigns1 + + val a2 = RegNext(a1) + val b2 = RegNext(b1) + val effSignB2 = RegNext(effSignB1) + val eqSigns2 = RegNext(eqSigns1) + val sDiffExps2 = RegNext(sDiffExps1) + val roundingMode2 = RegNext(roundingMode1) + val closeSubMags2r = RegNext(closeSubMags1r) + val closeNearNormDist2r = RegNext(closeNearNormDist2) + val closeSigOut2r = RegNext(closeSigOut2) + val closeTotalCancellation2r = RegNext(closeTotalCancellation2) + val closeNotTotalCancellationSignOut2r = RegNext(closeNotTotalCancellationSignOut2) + val farSignOut2r = RegNext(farSignOut1r) + val farSigLarger2r = RegNext(farSigLarger1r) + val farAlignedSigSmaller2r = RegNext(farAlignedSigSmaller2) + val farSubMags2r = RegNext(farSubMags2) + + // Stage 3: significand add/subtract and special-case selection. + val farNegAlignedSigSmaller3 = Mux( + farSubMags2r, + Cat(1.U, ~farAlignedSigSmaller2r), + farAlignedSigSmaller2r + ) + + val farSigSum3 = (farSigLarger2r << 3) + farNegAlignedSigSmaller3 + farSubMags2r + val farSigOut3 = Mux(farSubMags2r, farSigSum3, (farSigSum3 >> 1) | farSigSum3(0))(sigWidth + 2, 0) + + val notSigNaNInvalidExc3 = a2.isInf && b2.isInf && !eqSigns2 + val notNaNIsInfOut3 = a2.isInf || b2.isInf + val addZeros3 = a2.isZero && b2.isZero + val notNaNSpecialCase3 = notNaNIsInfOut3 || addZeros3 + val notNaNIsZeroOut3 = addZeros3 || (!notNaNIsInfOut3 && closeSubMags2r && closeTotalCancellation2r) + + val notNaNSignOut3 = + (eqSigns2 && a2.sign) || + (a2.isInf && a2.sign) || + (b2.isInf && effSignB2) || + (notNaNIsZeroOut3 && !eqSigns2 && roundingMode2 === round_min) || + (!notNaNSpecialCase3 && closeSubMags2r && !closeTotalCancellation2r && closeNotTotalCancellationSignOut2r) || + (!notNaNSpecialCase3 && !closeSubMags2r && farSignOut2r) + + val commonSExp3 = + Mux(closeSubMags2r || (sDiffExps2 < 0.S), b2.sExp, a2.sExp) - + Mux(closeSubMags2r, closeNearNormDist2r, farSubMags2r).zext + + val commonSigOut3 = Mux(closeSubMags2r, closeSigOut2r, farSigOut3) + + val rawOutReg = RegInit(0.U.asTypeOf(new RawFloat(expWidth, sigWidth + 2))) + rawOutReg.isNaN := a2.isNaN || b2.isNaN + rawOutReg.isInf := notNaNIsInfOut3 + rawOutReg.isZero := notNaNIsZeroOut3 + rawOutReg.sExp := commonSExp3 + rawOutReg.sign := notNaNSignOut3 + rawOutReg.sig := commonSigOut3 + + io.rawOut := rawOutReg + io.invalidExc := RegNext(isSigNaNRawFloat(a2) || isSigNaNRawFloat(b2) || notSigNaNInvalidExc3) + io.validout := ShiftRegister(io.validin, 4) +} diff --git a/examples/components/balls/int8add/configs/default.toml b/examples/components/balls/int8add/configs/default.toml new file mode 100644 index 000000000..2cf302111 --- /dev/null +++ b/examples/components/balls/int8add/configs/default.toml @@ -0,0 +1 @@ +# Int8AddBall has no private configuration. diff --git a/examples/components/balls/int8add/configs/int8add_params.proto b/examples/components/balls/int8add/configs/int8add_params.proto new file mode 100644 index 000000000..0cb47dc6f --- /dev/null +++ b/examples/components/balls/int8add/configs/int8add_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.int8add; +option java_package = "buckyball.balls.int8add"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/components/balls/int8add/emu/src/dispatch.rs b/examples/components/balls/int8add/emu/src/dispatch.rs new file mode 100644 index 000000000..6aa07d004 --- /dev/null +++ b/examples/components/balls/int8add/emu/src/dispatch.rs @@ -0,0 +1,32 @@ +use bebop_bemu::ball::ExecContext; + +use super::int8add; + +const BALL_CLASS: &str = "examples.balls.int8add.Int8AddBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("INT8ADD") => Some(int8add::Int8Add::exec(xs1, xs2, ctx, false)), + Some("INT8ADD_RELU") => Some(int8add::Int8Add::exec(xs1, xs2, ctx, true)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("INT8ADD") | Some("INT8ADD_RELU") => Some(int8add::Int8Add::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/balls/int8add/emu/src/int8add.rs b/examples/components/balls/int8add/emu/src/int8add.rs similarity index 97% rename from examples/balls/int8add/emu/src/int8add.rs rename to examples/components/balls/int8add/emu/src/int8add.rs index 72416c6b0..70379adac 100644 --- a/examples/balls/int8add/emu/src/int8add.rs +++ b/examples/components/balls/int8add/emu/src/int8add.rs @@ -1,4 +1,4 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::instruction::ExecContext; diff --git a/examples/components/balls/int8add/emu/src/lib.rs b/examples/components/balls/int8add/emu/src/lib.rs new file mode 100644 index 000000000..b7c2d06e7 --- /dev/null +++ b/examples/components/balls/int8add/emu/src/lib.rs @@ -0,0 +1,11 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod int8add; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/components/balls/int8add/stack/src/Lowering.cpp b/examples/components/balls/int8add/stack/src/Lowering.cpp new file mode 100644 index 000000000..c170e1449 --- /dev/null +++ b/examples/components/balls/int8add/stack/src/Lowering.cpp @@ -0,0 +1,66 @@ +//===- Lowering.cpp - Int8AddBall in lower-buckyball ----------------------===// +// +// add_i8: the destination receives `rows` rows of sixteen i8, +// round_half_even(A * lhs_ratio + B * rhs_ratio) saturated to i8, and with +// `relu` clamped at zero. Arguments 0 and 1 are the f32 lhs_ratio and +// rhs_ratio, both finite and positive. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballOps.h" +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class AddI8Lowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &rewriter) const override { + if (op.getKind() != "add_i8") + return failure(); + Location loc = op.getLoc(); + int64_t rows = cast(op->getAttr("rows")).getInt(); + bool relu = cast(op->getAttr("relu")).getValue(); + if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || + op.getSources().size() != 2 || + op.getSources()[0] == op.getSources()[1] || + llvm::is_contained(op.getSources(), op.getDestination())) + return op.emitOpError( + "add_i8 requires distinct banks and rows within a bank"); + // rs2: lhs_ratio bits in [31:0], rhs_ratio bits in [63:32]. + auto ratioBits = [&](Value ratio) -> Value { + Value bits = + arith::BitcastOp::create(rewriter, loc, rewriter.getI32Type(), ratio); + return arith::ExtUIOp::create(rewriter, loc, rewriter.getI64Type(), bits); + }; + Value rhs = + arith::ShLIOp::create(rewriter, loc, ratioBits(op.getArguments()[1]), + mlir::buddy::constantI64(rewriter, loc, 32)); + Value rs2 = arith::OrIOp::create(rewriter, loc, + ratioBits(op.getArguments()[0]), rhs); + mlir::buddy::emitInstruction( + rewriter, loc, + buckyball_target::getBuckyballFunct7(relu ? "INT8ADD_RELU" : "INT8ADD"), + mlir::buddy::constantI64(rewriter, loc, + mlir::buddy::bankFields(op.getSources(), + op.getDestination(), + false, rows)), + rs2); + rewriter.eraseOp(op); + return success(); + } +}; + +void populate(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} + +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallLowering(populate), true); +} // namespace diff --git a/examples/balls/int8add/verify/casegen/Cargo.lock b/examples/components/balls/int8add/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/int8add/verify/casegen/Cargo.lock rename to examples/components/balls/int8add/verify/casegen/Cargo.lock diff --git a/examples/balls/int8add/verify/casegen/Cargo.toml b/examples/components/balls/int8add/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/int8add/verify/casegen/Cargo.toml rename to examples/components/balls/int8add/verify/casegen/Cargo.toml diff --git a/examples/components/balls/int8add/verify/casegen/src/casegen.rs b/examples/components/balls/int8add/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..95ccc3b69 --- /dev/null +++ b/examples/components/balls/int8add/verify/casegen/src/casegen.rs @@ -0,0 +1,257 @@ +pub const ROW_BYTES: usize = 16; +pub const BANK_ENTRIES: u32 = 64; +pub const MAX_ROWS: usize = 64; +pub const NUM_CASES: u32 = 2; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Int8AddCmd { + pub relu: u32, + pub bid: u32, + pub iter: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, + pub rob_id: u32, + pub rs1_lo: u32, + pub rs1_hi: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub num_lhs_words: u32, + pub num_rhs_words: u32, + pub num_dst_words: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Int8AddCase { + pub cmd: Int8AddCmd, + pub lhs_words: [u128; MAX_ROWS], + pub rhs_words: [u128; MAX_ROWS], + pub dst_words: [u128; MAX_ROWS], +} + +impl Int8AddCase { + pub fn lhs_lo(&self, index: usize) -> u64 { + self.lhs_words[index] as u64 + } + + pub fn lhs_hi(&self, index: usize) -> u64 { + (self.lhs_words[index] >> 64) as u64 + } + + pub fn rhs_lo(&self, index: usize) -> u64 { + self.rhs_words[index] as u64 + } + + pub fn rhs_hi(&self, index: usize) -> u64 { + (self.rhs_words[index] >> 64) as u64 + } + + pub fn dst_lo(&self, index: usize) -> u64 { + self.dst_words[index] as u64 + } + + pub fn dst_hi(&self, index: usize) -> u64 { + (self.dst_words[index] >> 64) as u64 + } +} + +pub fn gen_case(index: u32, bid: u32) -> Int8AddCase { + if index >= NUM_CASES { + panic!("int8add: unsupported directed case {index}"); + } + match index { + 0 => directed(bid, 2, false), + 1 => directed(bid, 3, true), + _ => panic!("int8add: unsupported directed case {index}"), + } +} + +fn compute(lhs: i8, rhs: i8, lhs_ratio: f32, rhs_ratio: f32, relu: bool) -> i8 { + let value = (lhs as f32) * lhs_ratio + (rhs as f32) * rhs_ratio; + let rounded = value.round_ties_even(); + let clamped = if relu { + rounded.max(0.0) + } else { + rounded + } + .clamp(-128.0, 127.0); + clamped as i8 +} + +fn positive_finite(value: f32) -> bool { + value.is_finite() && value > 0.0 +} + +fn ctest_values(rows: usize) -> (Vec, Vec) { + let count = rows * ROW_BYTES; + let mut lhs = vec![0u8; count]; + let mut rhs = vec![0u8; count]; + for i in 0..count { + lhs[i] = ((i as i32 * 37 & 255) - 128) as i8 as u8; + rhs[i] = ((i as i32 * 19 & 255) - 128) as i8 as u8; + } + (pack_bytes(&lhs), pack_bytes(&rhs)) +} + +fn pack_bytes(bytes: &[u8]) -> Vec { + if bytes.len() % ROW_BYTES != 0 { + panic!( + "int8add: {} bytes is not a multiple of {ROW_BYTES}", + bytes.len() + ); + } + bytes + .chunks(ROW_BYTES) + .map(|chunk| { + let mut word = 0u128; + for (lane, byte) in chunk.iter().enumerate() { + word |= u128::from(*byte) << (lane * 8); + } + word + }) + .collect() +} + +fn copy_words(src: &[u128]) -> [u128; N] { + if src.len() > N { + panic!("int8add: {} words exceed buffer {N}", src.len()); + } + let mut out = [0u128; N]; + out[..src.len()].copy_from_slice(src); + out +} + +fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { + u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) +} + +fn encode_rs2(lhs_ratio: f32, rhs_ratio: f32) -> u64 { + u64::from(lhs_ratio.to_bits()) | (u64::from(rhs_ratio.to_bits()) << 32) +} + +fn split_rs(value: u64) -> (u32, u32) { + (value as u32, (value >> 32) as u32) +} + +pub(crate) fn validate( + op1_bank: u32, + op2_bank: u32, + wr_bank: u32, + op1_col: u32, + op2_col: u32, + wr_col: u32, + iter: u32, + rs1: u64, + rs2: u64, + lhs_ratio: f32, + rhs_ratio: f32, +) { + if iter == 0 || iter > BANK_ENTRIES { + panic!("int8add: iter must fit in one physical bank"); + } + if op1_col != 1 || op2_col != 1 || wr_col != 1 { + panic!("int8add: bank groups must match"); + } + if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { + panic!("int8add: banks must be distinct"); + } + if !positive_finite(lhs_ratio) || !positive_finite(rhs_ratio) { + panic!("int8add: ratios must be finite and positive"); + } + if (rs1 & 0x3ff) != u64::from(op1_bank) + || ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) + || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) + { + panic!("int8add: rs1 bank fields mismatch"); + } + if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { + panic!("int8add: rs1 iter mismatch"); + } + if rs2 != encode_rs2(lhs_ratio, rhs_ratio) { + panic!("int8add: rs2 ratio fields mismatch"); + } +} + +fn dst_words( + lhs: &[u128], + rhs: &[u128], + iter: u32, + lhs_ratio: f32, + rhs_ratio: f32, + relu: bool, +) -> Vec { + let mut out = Vec::with_capacity(iter as usize); + for row in 0..iter as usize { + let mut packed = 0u128; + for lane in 0..ROW_BYTES { + let l = ((lhs[row] >> (lane * 8)) & 0xff) as u8 as i8; + let r = ((rhs[row] >> (lane * 8)) & 0xff) as u8 as i8; + let q = compute(l, r, lhs_ratio, rhs_ratio, relu); + packed |= u128::from(q as u8) << (lane * 8); + } + out.push(packed); + } + out +} + +fn directed(bid: u32, rob_id: u32, relu: bool) -> Int8AddCase { + let iter = 7u32; + let lhs_ratio = 0.5f32; + let rhs_ratio = 0.25f32; + let (lhs, rhs) = ctest_values(iter as usize); + let op1_bank = 0; + let op2_bank = 1; + let wr_bank = 2; + let op1_col = 1; + let op2_col = 1; + let wr_col = 1; + let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); + let rs2 = encode_rs2(lhs_ratio, rhs_ratio); + validate( + op1_bank, + op2_bank, + wr_bank, + op1_col, + op2_col, + wr_col, + iter, + rs1, + rs2, + lhs_ratio, + rhs_ratio, + ); + if lhs.len() != iter as usize || rhs.len() != iter as usize { + panic!("int8add: operand rows != iter {iter}"); + } + let dst = dst_words(&lhs, &rhs, iter, lhs_ratio, rhs_ratio, relu); + let (rs1_lo, rs1_hi) = split_rs(rs1); + let (rs2_lo, rs2_hi) = split_rs(rs2); + Int8AddCase { + cmd: Int8AddCmd { + relu: u32::from(relu), + bid, + iter, + op1_bank, + op2_bank, + wr_bank, + op1_col, + op2_col, + wr_col, + rob_id, + rs1_lo, + rs1_hi, + rs2_lo, + rs2_hi, + num_lhs_words: lhs.len() as u32, + num_rhs_words: rhs.len() as u32, + num_dst_words: dst.len() as u32, + }, + lhs_words: copy_words(&lhs), + rhs_words: copy_words(&rhs), + dst_words: copy_words(&dst), + } +} diff --git a/examples/components/balls/int8add/verify/casegen/src/lib.rs b/examples/components/balls/int8add/verify/casegen/src/lib.rs new file mode 100644 index 000000000..e1c679c8a --- /dev/null +++ b/examples/components/balls/int8add/verify/casegen/src/lib.rs @@ -0,0 +1,70 @@ +mod casegen; + +use std::cell::RefCell; + +use casegen::{Int8AddCase, Int8AddCmd}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn int8add_case_load(index: u32, bid: u32) -> i32 { + let case = casegen::gen_case(index, bid); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("int8add_case: no case loaded; call int8add_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn int8add_case_cmd(out_ptr: *mut Int8AddCmd) { + if out_ptr.is_null() { + panic!("int8add_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +fn idx(index: u32, limit: u32, what: &str) -> usize { + if index >= limit { + panic!("int8add_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn int8add_case_lhs_word_lo(word_index: u32) -> u64 { + current_case(|case| case.lhs_lo(idx(word_index, case.cmd.num_lhs_words, "lhs_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn int8add_case_lhs_word_hi(word_index: u32) -> u64 { + current_case(|case| case.lhs_hi(idx(word_index, case.cmd.num_lhs_words, "lhs_word_hi"))) +} + +#[no_mangle] +pub extern "C" fn int8add_case_rhs_word_lo(word_index: u32) -> u64 { + current_case(|case| case.rhs_lo(idx(word_index, case.cmd.num_rhs_words, "rhs_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn int8add_case_rhs_word_hi(word_index: u32) -> u64 { + current_case(|case| case.rhs_hi(idx(word_index, case.cmd.num_rhs_words, "rhs_word_hi"))) +} + +#[no_mangle] +pub extern "C" fn int8add_case_dst_word_lo(word_index: u32) -> u64 { + current_case(|case| case.dst_lo(idx(word_index, case.cmd.num_dst_words, "dst_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn int8add_case_dst_word_hi(word_index: u32) -> u64 { + current_case(|case| case.dst_hi(idx(word_index, case.cmd.num_dst_words, "dst_word_hi"))) +} diff --git a/examples/balls/int8add/verify/filelists/int8add_ball.f b/examples/components/balls/int8add/verify/filelists/int8add_ball.f similarity index 100% rename from examples/balls/int8add/verify/filelists/int8add_ball.f rename to examples/components/balls/int8add/verify/filelists/int8add_ball.f diff --git a/examples/balls/int8add/verify/src/common/int8add_defs.svh b/examples/components/balls/int8add/verify/src/common/int8add_defs.svh similarity index 85% rename from examples/balls/int8add/verify/src/common/int8add_defs.svh rename to examples/components/balls/int8add/verify/src/common/int8add_defs.svh index 908c50e71..b39b98dc6 100644 --- a/examples/balls/int8add/verify/src/common/int8add_defs.svh +++ b/examples/components/balls/int8add/verify/src/common/int8add_defs.svh @@ -31,14 +31,14 @@ import "DPI-C" function longint unsigned int8add_case_dst_word_lo(input int unsi import "DPI-C" function longint unsigned int8add_case_dst_word_hi(input int unsigned word_index); `ifndef INT8ADD_FUNCT7 -`error "INT8ADD_FUNCT7 must be provided by the selected Core ballISA" +`error "INT8ADD_FUNCT7 must be provided by the selected Thread ballISA" `endif `ifndef INT8ADD_RELU_FUNCT7 -`error "INT8ADD_RELU_FUNCT7 must be provided by the selected Core ballISA" +`error "INT8ADD_RELU_FUNCT7 must be provided by the selected Thread ballISA" `endif -localparam int INT8ADD_CORE_FUNCT7 = `INT8ADD_FUNCT7; -localparam int INT8ADD_RELU_CORE_FUNCT7 = `INT8ADD_RELU_FUNCT7; +localparam int INT8ADD_THREAD_FUNCT7 = `INT8ADD_FUNCT7; +localparam int INT8ADD_RELU_THREAD_FUNCT7 = `INT8ADD_RELU_FUNCT7; localparam int INT8ADD_MAX_ROWS = 64; localparam int INT8ADD_NUM_CASES = 2; localparam int INT8ADD_TIMEOUT_CYCLES = 20000; diff --git a/examples/balls/int8add/verify/src/common/int8add_items.svh b/examples/components/balls/int8add/verify/src/common/int8add_items.svh similarity index 93% rename from examples/balls/int8add/verify/src/common/int8add_items.svh rename to examples/components/balls/int8add/verify/src/common/int8add_items.svh index eb3b2d3a8..6fd58d963 100644 --- a/examples/balls/int8add/verify/src/common/int8add_items.svh +++ b/examples/components/balls/int8add/verify/src/common/int8add_items.svh @@ -10,7 +10,7 @@ class int8add_cmd_item extends bb_blink_cmd_item; bit [127:0] dst_words[INT8ADD_MAX_ROWS]; constraint legal_c { - funct7 inside {INT8ADD_CORE_FUNCT7[6:0], INT8ADD_RELU_CORE_FUNCT7[6:0]}; + funct7 inside {INT8ADD_THREAD_FUNCT7[6:0], INT8ADD_RELU_THREAD_FUNCT7[6:0]}; op1_en == 1'b1; op2_en == 1'b1; wr_spad_en == 1'b1; @@ -71,8 +71,8 @@ class int8add_cmd_item extends bb_blink_cmd_item; is_sub = 1'b0; sub_rob_id = 8'h00; - if (relu == 0) funct7 = INT8ADD_CORE_FUNCT7[6:0]; - else funct7 = INT8ADD_RELU_CORE_FUNCT7[6:0]; + if (relu == 0) funct7 = INT8ADD_THREAD_FUNCT7[6:0]; + else funct7 = INT8ADD_RELU_THREAD_FUNCT7[6:0]; if (iter == 0) `uvm_fatal("CASE", "Int8AddBall iter must be positive") if (op1_col != 1 || op2_col != 1 || wr_col != 1) @@ -85,7 +85,7 @@ class int8add_cmd_item extends bb_blink_cmd_item; `uvm_fatal("CASE", $sformatf("num_rhs_words %0d != iter %0d", num_rhs_words, iter)) if (num_dst_words != iter) `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter %0d", num_dst_words, iter)) - if (funct7 != INT8ADD_CORE_FUNCT7[6:0] && funct7 != INT8ADD_RELU_CORE_FUNCT7[6:0]) + if (funct7 != INT8ADD_THREAD_FUNCT7[6:0] && funct7 != INT8ADD_RELU_THREAD_FUNCT7[6:0]) `uvm_fatal("CASE", $sformatf("invalid funct7: %0d", funct7)) for (i = 0; i < num_lhs_words; i++) begin diff --git a/examples/balls/int8add/verify/src/cov/int8add_cov.svh b/examples/components/balls/int8add/verify/src/cov/int8add_cov.svh similarity index 79% rename from examples/balls/int8add/verify/src/cov/int8add_cov.svh rename to examples/components/balls/int8add/verify/src/cov/int8add_cov.svh index 1c756c3a5..53938ff62 100644 --- a/examples/balls/int8add/verify/src/cov/int8add_cov.svh +++ b/examples/components/balls/int8add/verify/src/cov/int8add_cov.svh @@ -7,7 +7,7 @@ class int8add_cov extends uvm_component; covergroup cmd_cg; coverpoint cur_funct7 { - bins add = {INT8ADD_CORE_FUNCT7[6:0]}; bins relu = {INT8ADD_RELU_CORE_FUNCT7[6:0]}; + bins add = {INT8ADD_THREAD_FUNCT7[6:0]}; bins relu = {INT8ADD_RELU_THREAD_FUNCT7[6:0]}; } endgroup @@ -20,7 +20,7 @@ class int8add_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != INT8ADD_CORE_FUNCT7[6:0] && item.funct7 != INT8ADD_RELU_CORE_FUNCT7[6:0]) + if (item.funct7 != INT8ADD_THREAD_FUNCT7[6:0] && item.funct7 != INT8ADD_RELU_THREAD_FUNCT7[6:0]) `uvm_fatal("COV", $sformatf("unexpected funct7 %0d", item.funct7)) cur_funct7 = item.funct7; cmd_count++; @@ -28,6 +28,7 @@ class int8add_cov extends uvm_component; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") if (cmd_cg.get_coverage() != 100.0) diff --git a/examples/balls/int8add/verify/src/env/int8add_env.svh b/examples/components/balls/int8add/verify/src/env/int8add_env.svh similarity index 100% rename from examples/balls/int8add/verify/src/env/int8add_env.svh rename to examples/components/balls/int8add/verify/src/env/int8add_env.svh diff --git a/examples/balls/int8add/verify/src/env/int8add_scoreboard.svh b/examples/components/balls/int8add/verify/src/env/int8add_scoreboard.svh similarity index 98% rename from examples/balls/int8add/verify/src/env/int8add_scoreboard.svh rename to examples/components/balls/int8add/verify/src/env/int8add_scoreboard.svh index e27527ad6..e5f118865 100644 --- a/examples/balls/int8add/verify/src/env/int8add_scoreboard.svh +++ b/examples/components/balls/int8add/verify/src/env/int8add_scoreboard.svh @@ -67,6 +67,7 @@ class int8add_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, int8add_cmd_item exp); + verification_pkg::evidence::checks++; if (got.bid !== exp.bid) `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) if (got.funct7 !== exp.funct7) @@ -178,6 +179,7 @@ class int8add_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) begin `uvm_fatal("SCB", $sformatf( diff --git a/examples/balls/int8add/verify/src/pkg/int8add_pkg.sv b/examples/components/balls/int8add/verify/src/pkg/int8add_pkg.sv similarity index 100% rename from examples/balls/int8add/verify/src/pkg/int8add_pkg.sv rename to examples/components/balls/int8add/verify/src/pkg/int8add_pkg.sv diff --git a/examples/balls/int8add/verify/src/seq/int8add_sequences.svh b/examples/components/balls/int8add/verify/src/seq/int8add_sequences.svh similarity index 100% rename from examples/balls/int8add/verify/src/seq/int8add_sequences.svh rename to examples/components/balls/int8add/verify/src/seq/int8add_sequences.svh diff --git a/examples/components/balls/int8add/verify/src/tb_top.sv b/examples/components/balls/int8add/verify/src/tb_top.sv new file mode 100644 index 000000000..cbd6b5183 --- /dev/null +++ b/examples/components/balls/int8add/verify/src/tb_top.sv @@ -0,0 +1,101 @@ +module tb_top; + import uvm_pkg::*; + import int8add_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + Int8AddBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .io_status_idle(), + .io_status_running(), + .io_bankAccessReady(1'b1), + .io_cmdReq_ready(intf.cmd_req_ready), + .io_cmdReq_valid(intf.cmd_req_valid), + .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), + .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), + .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), + .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), + .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), + .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), + .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), + .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), + .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), + .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), + .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), + .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), + .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), + .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), + .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), + .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), + .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), + .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), + .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), + .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), + .io_cmdResp_ready(intf.cmd_resp_ready), + .io_cmdResp_valid(intf.cmd_resp_valid), + .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), + .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), + .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .io_bankRead_0_ball_id(), + .io_bankRead_0_group_id(intf.bank_read_group_id[0]), + .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .io_bankRead_1_ball_id(), + .io_bankRead_1_group_id(intf.bank_read_group_id[1]), + .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .io_bankWrite_0_ball_id(), + .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .io_subRobReq_ready(intf.sub_rob_req_ready) + ); + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("int8add_ball_test"); + end +endmodule diff --git a/examples/balls/int8add/verify/src/tests/int8add_test.svh b/examples/components/balls/int8add/verify/src/tests/int8add_test.svh similarity index 100% rename from examples/balls/int8add/verify/src/tests/int8add_test.svh rename to examples/components/balls/int8add/verify/src/tests/int8add_test.svh diff --git a/examples/components/balls/int8add/workloads/ctests/CMakeLists.txt b/examples/components/balls/int8add/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..cb3eecb06 --- /dev/null +++ b/examples/components/balls/int8add/workloads/ctests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_buckyball_ctest(int8add_scaled_test.c) +add_buckyball_ctest(int8add_relu_test.c) diff --git a/examples/balls/int8add/workloads/ctests/int8add_relu_test.c b/examples/components/balls/int8add/workloads/ctests/int8add_relu_test.c similarity index 100% rename from examples/balls/int8add/workloads/ctests/int8add_relu_test.c rename to examples/components/balls/int8add/workloads/ctests/int8add_relu_test.c diff --git a/examples/balls/int8add/workloads/ctests/int8add_scaled_test.c b/examples/components/balls/int8add/workloads/ctests/int8add_scaled_test.c similarity index 100% rename from examples/balls/int8add/workloads/ctests/int8add_scaled_test.c rename to examples/components/balls/int8add/workloads/ctests/int8add_scaled_test.c diff --git a/examples/balls/int8add/workloads/isa/int8add.h b/examples/components/balls/int8add/workloads/isa/int8add.h similarity index 100% rename from examples/balls/int8add/workloads/isa/int8add.h rename to examples/components/balls/int8add/workloads/isa/int8add.h diff --git a/examples/balls/int8add/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/int8add/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/int8add/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/int8add/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/int8add/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/int8add/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/int8add/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/int8add/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir b/examples/components/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir new file mode 100644 index 000000000..b2494a279 --- /dev/null +++ b/examples/components/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir @@ -0,0 +1,40 @@ +func.func private @check_result(memref<4x16xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %ratio = arith.constant 1.0 : f32 + %a = memref.alloc() : memref<4x16xi8> + %b = memref.alloc() : memref<4x16xi8> + %out = memref.alloc() : memref<4x16xi8> + scf.for %row = %zero to %four step %one { + scf.for %lane = %zero to %sixteen step %one { + %r = arith.index_cast %row : index to i32 + %l = arith.index_cast %lane : index to i32 + %a32 = arith.subi %l, %r : i32 + %b32 = arith.subi %r, %l : i32 + %a8 = arith.trunci %a32 : i32 to i8 + %b8 = arith.trunci %b32 : i32 to i8 + memref.store %a8, %a[%row, %lane] : memref<4x16xi8> + memref.store %b8, %b[%row, %lane] : memref<4x16xi8> + } + } + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %a <{bank = 0 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.mvin %b <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.custom "add_i8" args(%ratio, %ratio : f32, f32) <{sources = array, destination = 2 : i64}> {rows = 4 : i64, relu = true} + buckyball.mvout %out <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + func.call @check_result(%out) : (memref<4x16xi8>) -> () + memref.dealloc %a : memref<4x16xi8> + memref.dealloc %b : memref<4x16xi8> + memref.dealloc %out : memref<4x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/int8add/workloads/mlir_tests/ball/int8add_relu_main.cpp b/examples/components/balls/int8add/workloads/mlir_tests/ball/int8add_relu_main.cpp similarity index 100% rename from examples/balls/int8add/workloads/mlir_tests/ball/int8add_relu_main.cpp rename to examples/components/balls/int8add/workloads/mlir_tests/ball/int8add_relu_main.cpp diff --git a/examples/balls/int8add/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/int8add/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/int8add/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/components/balls/int8add/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in b/examples/components/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in new file mode 100644 index 000000000..904ac8775 --- /dev/null +++ b/examples/components/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in @@ -0,0 +1,42 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %mod = arith.constant 31 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %lanes = arith.constant @BB_BANK_I8_LANES@ : index + %ratio = arith.constant 1.0 : f32 + %a = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %b = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %out = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + scf.for %row = %zero to %rows step %one { + scf.for %lane = %zero to %lanes step %one { + %row_mod = arith.remui %row, %mod : index + %r = arith.index_cast %row_mod : index to i32 + %l = arith.index_cast %lane : index to i32 + %a32 = arith.subi %l, %r : i32 + %b32 = arith.subi %r, %l : i32 + %a8 = arith.trunci %a32 : i32 to i8 + %b8 = arith.trunci %b32 : i32 to i8 + memref.store %a8, %a[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.store %b8, %b[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + } + } + %a_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %out_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %a_loaded = buckyball.bank_mvin %a, %a_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %b_loaded = buckyball.bank_mvin %b, %b_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %result = buckyball.bank_custom "add_i8" (%a_loaded, %b_loaded) -> %out_bank args(%ratio, %ratio : f32, f32) {rows = @BB_BANK_LINES@ : i64, relu = false} + %stored = buckyball.bank_mvout %out, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + buckyball.bank_release %a_loaded + buckyball.bank_release %b_loaded + buckyball.bank_release %stored + func.call @check_result(%out) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () + memref.dealloc %a : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.dealloc %b : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.dealloc %out : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp b/examples/components/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp similarity index 100% rename from examples/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp rename to examples/components/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp diff --git a/examples/components/balls/int8mul/arch/Int8MulBall.scala b/examples/components/balls/int8mul/arch/Int8MulBall.scala new file mode 100644 index 000000000..7c065936d --- /dev/null +++ b/examples/components/balls/int8mul/arch/Int8MulBall.scala @@ -0,0 +1,248 @@ +package examples.balls.int8mul + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.int8mul.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import chisel3.util._ +import hardfloat.{recFNFromFN, INToRecFN, RecFNToIN} +import hardfloat.consts.{round_near_even, tininess_afterRounding} +import framework.lib.arith.Fma + +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class Int8MulBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + require(ball.inBW == 2, "Int8MulBall requires inBW=2") + require(ball.outBW == 1, "Int8MulBall requires outBW=1") + require(mp.bankWidth == 128, "Int8MulBall requires 128-bit bank rows") + val bankRowBits = log2Ceil(mp.bankEntries) + require(bankRowBits <= 32, "Int8MulBall gate_row must fit rs2[63:32]") + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val idle :: waitForChannels :: gateRequest :: gateResponse :: inputRequest :: inputResponse :: calculate :: calculateDrain :: writeRequest :: writeResponse :: complete :: Nil = + Enum(11) + val state = RegInit(idle) + val robId = RegInit(0.U(log2Up(fp.rob_entries).W)) + val gateBank = RegInit(0.U(fp.bank_id_len.W)) + val inputBank = RegInit(0.U(fp.bank_id_len.W)) + val outputBank = RegInit(0.U(fp.bank_id_len.W)) + val groups = RegInit(0.U(5.W)) + val group = RegInit(0.U(5.W)) + val bases = Reg(Vec(3, UInt(mp.groupIdWidth.W))) + val iter = RegInit(0.U(fp.iter_len.W)) + val gateRow = RegInit(0.U(bankRowBits.W)) + val inputRow = RegInit(0.U(log2Ceil(mp.bankEntries).W)) + val ratio = Reg(UInt(32.W)) + val gateWord = Reg(UInt(128.W)) + val inputWord = Reg(UInt(128.W)) + val lane = RegInit(0.U(4.W)) + val outputWord = Reg(Vec(16, UInt(8.W))) + + val productReg = Reg(UInt(16.W)) + val productLaneReg = Reg(UInt(4.W)) + val productValid = RegInit(false.B) + val productFloatReg = Reg(UInt(33.W)) + val floatLaneReg = Reg(UInt(4.W)) + val floatValid = RegInit(false.B) + val scaledReg = Reg(UInt(33.W)) + val scaledLaneReg = Reg(UInt(4.W)) + val scaledValid = RegInit(false.B) + + def positiveFinite(value: UInt): Bool = + !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U + + val gateValue = (gateWord >> (lane << 3))(7, 0).asSInt + val inputValue = (inputWord >> (lane << 3))(7, 0).asSInt + val integerProduct = (gateValue * inputValue).asUInt + val productToFloat = Instantiate(new INToRecFN(16, 8, 24)) + + val scale = Instantiate(new Fma(3, 8, 24)) + + val toInt8 = Instantiate(new RecFNToIN(8, 24, 8)) + + productToFloat.io.signedIn := true.B + productToFloat.io.in := productReg + productToFloat.io.roundingMode := round_near_even + productToFloat.io.detectTininess := tininess_afterRounding + scale.io.validin := floatValid + scale.io.op := 0.U + scale.io.a := productFloatReg + scale.io.b := recFNFromFN(8, 24, ratio) + scale.io.c := recFNFromFN(8, 24, 0.U) + scale.io.roundingMode := round_near_even + scale.io.detectTininess := tininess_afterRounding.asUInt + toInt8.io.in := scaledReg + toInt8.io.roundingMode := round_near_even + toInt8.io.signedOut := true.B + + blink.cmdReq.ready := state === idle + blink.cmdResp.valid := state === complete + blink.cmdResp.bits.rob_id := robId + blink.status.idle := state === idle + blink.status.running := state =/= idle && state =/= complete + + for (port <- 0 until 2) { + blink.bankRead(port).rob_id := robId + blink.bankRead(port).bank_id := Mux(port.U === 0.U, gateBank, inputBank) + blink.bankRead(port).group_id := bases(port) + group + blink.bankRead(port).io.req.valid := false.B + blink.bankRead(port).io.req.bits.addr := Mux(port.U === 0.U, gateRow, inputRow) + blink.bankRead(port).io.resp.ready := false.B + } + + blink.bankWrite(0).rob_id := robId + blink.bankWrite(0).bank_id := outputBank + blink.bankWrite(0).group_id := bases(2) + group + blink.bankWrite(0).io.req.valid := false.B + blink.bankWrite(0).io.req.bits.addr := inputRow + blink.bankWrite(0).io.req.bits.data := Cat(outputWord.reverse) + blink.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) + blink.bankWrite(0).io.resp.ready := false.B + + switch(state) { + is(idle) { + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + + if (bankRowBits < 32) + assert( + cmd.rs2(63, 32 + bankRowBits) === 0.U, + s"Int8MulBall reserves rs2 bits 63:${32 + bankRowBits}" + ) + assert(positiveFinite(cmd.rs2(31, 0)), "Int8MulBall ratio must be finite and positive") + assert( + cmd.rs2(31 + bankRowBits, 32) < mp.bankEntries.U, + "Int8MulBall gate_row must fit one physical bank" + ) + assert( + cmd.op1_col =/= 0.U && cmd.op1_col === cmd.op2_col && cmd.op1_col === cmd.wr_col, + "Int8MulBall bank groups must match" + ) + assert( + cmd.op1_col <= mp.bankNum.U, + "Int8MulBall bank groups exceed physical banks" + ) + assert( + cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, + "Int8MulBall banks must be distinct" + ) + assert( + cmd.iter =/= 0.U && cmd.iter <= mp.bankEntries.U, + "Int8MulBall iter must fit one physical bank" + ) + robId := blink.cmdReq.bits.rob_id + gateBank := cmd.op1_bank + inputBank := cmd.op2_bank + outputBank := cmd.wr_bank + groups := cmd.op1_col + group := 0.U + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + iter := cmd.iter + gateRow := cmd.rs2(31 + bankRowBits, 32) + inputRow := 0.U + ratio := cmd.rs2(31, 0) + lane := 0.U + state := waitForChannels + } + } + is(waitForChannels) { + state := gateRequest + } + is(gateRequest) { + blink.bankRead(0).io.req.valid := true.B + when(blink.bankRead(0).io.req.fire)(state := gateResponse) + } + is(gateResponse) { + blink.bankRead(0).io.resp.ready := true.B + when(blink.bankRead(0).io.resp.fire) { + gateWord := blink.bankRead(0).io.resp.bits.data + state := inputRequest + } + } + is(inputRequest) { + blink.bankRead(1).io.req.valid := true.B + when(blink.bankRead(1).io.req.fire)(state := inputResponse) + } + is(inputResponse) { + blink.bankRead(1).io.resp.ready := true.B + when(blink.bankRead(1).io.resp.fire) { + inputWord := blink.bankRead(1).io.resp.bits.data + lane := 0.U + state := calculate + } + } + is(calculate) { + productReg := integerProduct + productLaneReg := lane + productValid := true.B + when(lane === 15.U)(state := calculateDrain) + .otherwise(lane := lane + 1.U) + } + is(calculateDrain) { + when(scaledValid && scaledLaneReg === 15.U)(state := writeRequest) + } + is(writeRequest) { + blink.bankWrite(0).io.req.valid := true.B + when(blink.bankWrite(0).io.req.fire)(state := writeResponse) + } + is(writeResponse) { + blink.bankWrite(0).io.resp.ready := true.B + when(blink.bankWrite(0).io.resp.fire) { + when(inputRow +& 1.U === iter) { + inputRow := 0.U + when(group +& 1.U === groups)(state := complete) + .otherwise { group := group + 1.U; state := gateRequest } + }.otherwise { inputRow := inputRow + 1.U; state := inputRequest } + } + } + is(complete) { + when(blink.cmdResp.fire)(state := idle) + } + } + + when(productValid) { + productFloatReg := productToFloat.io.out + floatLaneReg := productLaneReg + } + when(scale.io.validout) { + scaledReg := scale.io.out + scaledLaneReg := Pipe(floatValid, floatLaneReg, 3).bits + } + when(scaledValid) { + outputWord(scaledLaneReg) := toInt8.io.out + } + + productValid := state === calculate + floatValid := productValid + scaledValid := scale.io.validout +} + +object Int8MulBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new Int8MulBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/components/balls/int8mul/configs/default.toml b/examples/components/balls/int8mul/configs/default.toml new file mode 100644 index 000000000..5db85e8e7 --- /dev/null +++ b/examples/components/balls/int8mul/configs/default.toml @@ -0,0 +1 @@ +# Int8MulBall has no private configuration. diff --git a/examples/components/balls/int8mul/configs/int8mul_params.proto b/examples/components/balls/int8mul/configs/int8mul_params.proto new file mode 100644 index 000000000..daba4d0ae --- /dev/null +++ b/examples/components/balls/int8mul/configs/int8mul_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.int8mul; +option java_package = "buckyball.balls.int8mul"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/components/balls/int8mul/emu/src/dispatch.rs b/examples/components/balls/int8mul/emu/src/dispatch.rs new file mode 100644 index 000000000..19e733f52 --- /dev/null +++ b/examples/components/balls/int8mul/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::ExecContext; + +use super::int8mul; + +const BALL_CLASS: &str = "examples.balls.int8mul.Int8MulBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("INT8MUL") => Some(int8mul::Int8Mul::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("INT8MUL") => Some(int8mul::Int8Mul::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/balls/int8mul/emu/src/int8mul.rs b/examples/components/balls/int8mul/emu/src/int8mul.rs similarity index 98% rename from examples/balls/int8mul/emu/src/int8mul.rs rename to examples/components/balls/int8mul/emu/src/int8mul.rs index 92b073cd1..00efae535 100644 --- a/examples/balls/int8mul/emu/src/int8mul.rs +++ b/examples/components/balls/int8mul/emu/src/int8mul.rs @@ -1,4 +1,4 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::instruction::ExecContext; diff --git a/examples/components/balls/int8mul/emu/src/lib.rs b/examples/components/balls/int8mul/emu/src/lib.rs new file mode 100644 index 000000000..3949cf522 --- /dev/null +++ b/examples/components/balls/int8mul/emu/src/lib.rs @@ -0,0 +1,11 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod int8mul; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/components/balls/int8mul/stack/src/Lowering.cpp b/examples/components/balls/int8mul/stack/src/Lowering.cpp new file mode 100644 index 000000000..0957b9c32 --- /dev/null +++ b/examples/components/balls/int8mul/stack/src/Lowering.cpp @@ -0,0 +1,48 @@ +//===- Lowering.cpp - Int8MulBall in lower-buckyball +//-----------------------===// +#include "Buckyball/BuckyballOps.h" +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +using namespace mlir; +using namespace ::buddy::buckyball; +namespace { +class GateMulI8Lowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &b) const override { + if (op.getKind() != "gate_mul_i8") + return failure(); + int64_t rows = cast(op->getAttr("rows")).getInt(); + int64_t gateRow = cast(op->getAttr("gate_row")).getInt(); + int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; + if (rows <= 0 || rows > depth || gateRow < 0 || gateRow >= depth || + op.getSources().size() != 2 || + op.getSources()[0] == op.getSources()[1] || + llvm::is_contained(op.getSources(), op.getDestination())) + return op.emitOpError( + "gate_mul_i8 requires distinct banks and valid row addresses"); + Value bits = arith::BitcastOp::create(b, op.getLoc(), b.getI32Type(), + op.getArguments()[0]); + Value ratio = arith::ExtUIOp::create(b, op.getLoc(), b.getI64Type(), bits); + Value rs2 = arith::OrIOp::create( + b, op.getLoc(), ratio, + mlir::buddy::constantI64(b, op.getLoc(), uint64_t(gateRow) << 32)); + mlir::buddy::emitInstruction( + b, op.getLoc(), buckyball_target::getBuckyballFunct7("INT8MUL"), + mlir::buddy::constantI64(b, op.getLoc(), + mlir::buddy::bankFields(op.getSources(), + op.getDestination(), + false, rows)), + rs2); + b.eraseOp(op); + return success(); + } +}; +void populate(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallLowering(populate), true); +} // namespace diff --git a/examples/balls/int8mul/verify/casegen/Cargo.lock b/examples/components/balls/int8mul/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/int8mul/verify/casegen/Cargo.lock rename to examples/components/balls/int8mul/verify/casegen/Cargo.lock diff --git a/examples/balls/int8mul/verify/casegen/Cargo.toml b/examples/components/balls/int8mul/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/int8mul/verify/casegen/Cargo.toml rename to examples/components/balls/int8mul/verify/casegen/Cargo.toml diff --git a/examples/components/balls/int8mul/verify/casegen/src/casegen.rs b/examples/components/balls/int8mul/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..118e07b5a --- /dev/null +++ b/examples/components/balls/int8mul/verify/casegen/src/casegen.rs @@ -0,0 +1,266 @@ +pub const ROW_BYTES: usize = 16; +pub const MAX_ROWS: usize = 64; +pub const NUM_CASES: u32 = 3; +pub const GATE_ROW: u32 = 1; +pub const RATIO: f32 = 0.25; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Int8MulCmd { + pub bid: u32, + pub iter: u32, + pub gate_bank: u32, + pub input_bank: u32, + pub output_bank: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, + pub gate_row: u32, + pub bank_entries: u32, + pub rob_id: u32, + pub rs1_lo: u32, + pub rs1_hi: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub num_gate_words: u32, + pub num_input_words: u32, + pub num_dst_words: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Int8MulCase { + pub cmd: Int8MulCmd, + pub gate_words: [u128; MAX_ROWS], + pub input_words: [u128; MAX_ROWS], + pub dst_words: [u128; MAX_ROWS], +} + +impl Int8MulCase { + pub fn gate_lo(&self, index: usize) -> u64 { + self.gate_words[index] as u64 + } + + pub fn gate_hi(&self, index: usize) -> u64 { + (self.gate_words[index] >> 64) as u64 + } + + pub fn input_lo(&self, index: usize) -> u64 { + self.input_words[index] as u64 + } + + pub fn input_hi(&self, index: usize) -> u64 { + (self.input_words[index] >> 64) as u64 + } + + pub fn dst_lo(&self, index: usize) -> u64 { + self.dst_words[index] as u64 + } + + pub fn dst_hi(&self, index: usize) -> u64 { + (self.dst_words[index] >> 64) as u64 + } +} + +pub fn gen_case(index: u32, bid: u32, bank_entries: u32) -> Int8MulCase { + if bank_entries == 0 { + panic!("int8mul: bank entries must be positive"); + } + if index >= NUM_CASES { + panic!("int8mul: unsupported directed case {index}"); + } + match index { + 0 => directed(bid, 2, 1, GATE_ROW, bank_entries), + 1 => directed(bid, 3, 4, GATE_ROW, bank_entries), + 2 => directed(bid, 4, 1, bank_entries - 1, bank_entries), + _ => panic!("int8mul: unsupported directed case {index}"), + } +} + +fn compute(gate: i8, input: i8, ratio: f32) -> i8 { + let value = (gate as f32) * (input as f32) * ratio; + value.round_ties_even().clamp(-128.0, 127.0) as i8 +} + +fn positive_finite(value: f32) -> bool { + value.is_finite() && value > 0.0 +} + +fn ctest_gate() -> u128 { + let mut row = [0u8; ROW_BYTES]; + for lane in 0..ROW_BYTES { + row[lane] = (lane as i8 - 8) as u8; + } + pack_bytes(&row)[0] +} + +fn ctest_input(rows: usize) -> Vec { + let count = rows * ROW_BYTES; + let bytes: Vec = (0..count) + .map(|i| ((i as i32 * 37 & 255) - 128) as i8 as u8) + .collect(); + pack_bytes(&bytes) +} + +fn pack_bytes(bytes: &[u8]) -> Vec { + if bytes.len() % ROW_BYTES != 0 { + panic!( + "int8mul: {} bytes is not a multiple of {ROW_BYTES}", + bytes.len() + ); + } + bytes + .chunks(ROW_BYTES) + .map(|chunk| { + let mut word = 0u128; + for (lane, byte) in chunk.iter().enumerate() { + word |= u128::from(*byte) << (lane * 8); + } + word + }) + .collect() +} + +fn copy_words(src: &[u128]) -> [u128; N] { + if src.len() > N { + panic!("int8mul: {} words exceed buffer {N}", src.len()); + } + let mut out = [0u128; N]; + out[..src.len()].copy_from_slice(src); + out +} + +fn encode_rs1(gate: u32, input: u32, output: u32, iter: u32) -> u64 { + u64::from(gate) | (u64::from(input) << 10) | (u64::from(output) << 20) | (u64::from(iter) << 30) +} + +fn encode_rs2(ratio: f32, gate_row: u32) -> u64 { + u64::from(ratio.to_bits()) | (u64::from(gate_row) << 32) +} + +fn split_rs(value: u64) -> (u32, u32) { + (value as u32, (value >> 32) as u32) +} + +pub(crate) fn validate( + gate_bank: u32, + input_bank: u32, + output_bank: u32, + op1_col: u32, + op2_col: u32, + wr_col: u32, + iter: u32, + gate_row: u32, + rs1: u64, + rs2: u64, + ratio: f32, + bank_entries: u32, +) { + if bank_entries == 0 { + panic!("int8mul: bank entries must be positive"); + } + if iter == 0 || iter > bank_entries { + panic!("int8mul: iter must fit one physical bank"); + } + if gate_row >= bank_entries { + panic!("int8mul: gate row must fit one physical bank"); + } + if op1_col != 1 || op2_col != 1 || wr_col != 1 { + panic!("int8mul: bank groups must match"); + } + if gate_bank == input_bank || gate_bank == output_bank || input_bank == output_bank { + panic!("int8mul: banks must be distinct"); + } + if !positive_finite(ratio) { + panic!("int8mul: ratio must be finite and positive"); + } + let gate_row_bits = u32::BITS - (bank_entries - 1).leading_zeros(); + if rs2 >> (32 + gate_row_bits) != 0 { + panic!("int8mul: reserved rs2 bits must be zero"); + } + if (rs1 & 0x3ff) != u64::from(gate_bank) + || ((rs1 >> 10) & 0x3ff) != u64::from(input_bank) + || ((rs1 >> 20) & 0x3ff) != u64::from(output_bank) + { + panic!("int8mul: rs1 bank fields mismatch"); + } + if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { + panic!("int8mul: rs1 iter mismatch"); + } + if rs2 != encode_rs2(ratio, gate_row) { + panic!("int8mul: rs2 ratio/gate_row fields mismatch"); + } +} + +fn dst_words(gate_word: u128, input: &[u128], iter: u32, ratio: f32) -> Vec { + let mut out = Vec::with_capacity(iter as usize); + for row in 0..iter as usize { + let mut packed = 0u128; + for lane in 0..ROW_BYTES { + let g = ((gate_word >> (lane * 8)) & 0xff) as u8 as i8; + let inp = ((input[row] >> (lane * 8)) & 0xff) as u8 as i8; + let q = compute(g, inp, ratio); + packed |= u128::from(q as u8) << (lane * 8); + } + out.push(packed); + } + out +} + +fn directed(bid: u32, rob_id: u32, iter: u32, gate_row: u32, bank_entries: u32) -> Int8MulCase { + let gate_bank = 0; + let input_bank = 1; + let output_bank = 2; + let op1_col = 1; + let op2_col = 1; + let wr_col = 1; + let ratio = RATIO; + let gate = ctest_gate(); + let input = ctest_input(iter as usize); + let rs1 = encode_rs1(gate_bank, input_bank, output_bank, iter); + let rs2 = encode_rs2(ratio, gate_row); + validate( + gate_bank, + input_bank, + output_bank, + op1_col, + op2_col, + wr_col, + iter, + gate_row, + rs1, + rs2, + ratio, + bank_entries, + ); + if input.len() != iter as usize { + panic!("int8mul: input rows {} != iter {iter}", input.len()); + } + let dst = dst_words(gate, &input, iter, ratio); + let (rs1_lo, rs1_hi) = split_rs(rs1); + let (rs2_lo, rs2_hi) = split_rs(rs2); + Int8MulCase { + cmd: Int8MulCmd { + bid, + iter, + gate_bank, + input_bank, + output_bank, + op1_col, + op2_col, + wr_col, + gate_row, + bank_entries, + rob_id, + rs1_lo, + rs1_hi, + rs2_lo, + rs2_hi, + num_gate_words: 1, + num_input_words: input.len() as u32, + num_dst_words: dst.len() as u32, + }, + gate_words: copy_words(&[gate]), + input_words: copy_words(&input), + dst_words: copy_words(&dst), + } +} diff --git a/examples/balls/int8mul/verify/casegen/src/config.rs b/examples/components/balls/int8mul/verify/casegen/src/config.rs similarity index 100% rename from examples/balls/int8mul/verify/casegen/src/config.rs rename to examples/components/balls/int8mul/verify/casegen/src/config.rs diff --git a/examples/components/balls/int8mul/verify/casegen/src/lib.rs b/examples/components/balls/int8mul/verify/casegen/src/lib.rs new file mode 100644 index 000000000..41e8940f8 --- /dev/null +++ b/examples/components/balls/int8mul/verify/casegen/src/lib.rs @@ -0,0 +1,71 @@ +mod casegen; +mod config; + +use std::cell::RefCell; + +use casegen::{Int8MulCase, Int8MulCmd}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn int8mul_case_load(index: u32, bid: u32) -> i32 { + let case = casegen::gen_case(index, bid, config::bank_entries()); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("int8mul_case: no case loaded; call int8mul_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn int8mul_case_cmd(out_ptr: *mut Int8MulCmd) { + if out_ptr.is_null() { + panic!("int8mul_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +fn idx(index: u32, limit: u32, what: &str) -> usize { + if index >= limit { + panic!("int8mul_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn int8mul_case_gate_word_lo(word_index: u32) -> u64 { + current_case(|case| case.gate_lo(idx(word_index, case.cmd.num_gate_words, "gate_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn int8mul_case_gate_word_hi(word_index: u32) -> u64 { + current_case(|case| case.gate_hi(idx(word_index, case.cmd.num_gate_words, "gate_word_hi"))) +} + +#[no_mangle] +pub extern "C" fn int8mul_case_input_word_lo(word_index: u32) -> u64 { + current_case(|case| case.input_lo(idx(word_index, case.cmd.num_input_words, "input_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn int8mul_case_input_word_hi(word_index: u32) -> u64 { + current_case(|case| case.input_hi(idx(word_index, case.cmd.num_input_words, "input_word_hi"))) +} + +#[no_mangle] +pub extern "C" fn int8mul_case_dst_word_lo(word_index: u32) -> u64 { + current_case(|case| case.dst_lo(idx(word_index, case.cmd.num_dst_words, "dst_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn int8mul_case_dst_word_hi(word_index: u32) -> u64 { + current_case(|case| case.dst_hi(idx(word_index, case.cmd.num_dst_words, "dst_word_hi"))) +} diff --git a/examples/balls/int8mul/verify/filelists/int8mul_ball.f b/examples/components/balls/int8mul/verify/filelists/int8mul_ball.f similarity index 100% rename from examples/balls/int8mul/verify/filelists/int8mul_ball.f rename to examples/components/balls/int8mul/verify/filelists/int8mul_ball.f diff --git a/examples/balls/int8mul/verify/src/common/int8mul_defs.svh b/examples/components/balls/int8mul/verify/src/common/int8mul_defs.svh similarity index 92% rename from examples/balls/int8mul/verify/src/common/int8mul_defs.svh rename to examples/components/balls/int8mul/verify/src/common/int8mul_defs.svh index 8911278e8..a9b509633 100644 --- a/examples/balls/int8mul/verify/src/common/int8mul_defs.svh +++ b/examples/components/balls/int8mul/verify/src/common/int8mul_defs.svh @@ -32,10 +32,10 @@ import "DPI-C" function longint unsigned int8mul_case_dst_word_lo(input int unsi import "DPI-C" function longint unsigned int8mul_case_dst_word_hi(input int unsigned word_index); `ifndef INT8MUL_FUNCT7 -`error "INT8MUL_FUNCT7 must be provided by the selected Core ballISA" +`error "INT8MUL_FUNCT7 must be provided by the selected Thread ballISA" `endif -localparam int INT8MUL_CORE_FUNCT7 = `INT8MUL_FUNCT7; +localparam int INT8MUL_THREAD_FUNCT7 = `INT8MUL_FUNCT7; localparam int INT8MUL_MAX_ROWS = 64; localparam int INT8MUL_NUM_CASES = 3; localparam int INT8MUL_TIMEOUT_CYCLES = 20000; diff --git a/examples/balls/int8mul/verify/src/common/int8mul_items.svh b/examples/components/balls/int8mul/verify/src/common/int8mul_items.svh similarity index 96% rename from examples/balls/int8mul/verify/src/common/int8mul_items.svh rename to examples/components/balls/int8mul/verify/src/common/int8mul_items.svh index b9f3bd93f..017406d68 100644 --- a/examples/balls/int8mul/verify/src/common/int8mul_items.svh +++ b/examples/components/balls/int8mul/verify/src/common/int8mul_items.svh @@ -11,7 +11,7 @@ class int8mul_cmd_item extends bb_blink_cmd_item; bit [127:0] dst_words[INT8MUL_MAX_ROWS]; constraint legal_c { - funct7 == INT8MUL_CORE_FUNCT7[6:0]; + funct7 == INT8MUL_THREAD_FUNCT7[6:0]; op1_en == 1'b1; op2_en == 1'b1; wr_spad_en == 1'b1; @@ -59,7 +59,7 @@ class int8mul_cmd_item extends bb_blink_cmd_item; num_gate_words = cmd.num_gate_words; num_input_words = cmd.num_input_words; num_dst_words = cmd.num_dst_words; - funct7 = INT8MUL_CORE_FUNCT7[6:0]; + funct7 = INT8MUL_THREAD_FUNCT7[6:0]; special = 64'd0; op1_en = 1'b1; op2_en = 1'b1; @@ -86,7 +86,7 @@ class int8mul_cmd_item extends bb_blink_cmd_item; `uvm_fatal("CASE", $sformatf("num_input_words %0d != iter %0d", num_input_words, iter)) if (num_dst_words != iter) `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter %0d", num_dst_words, iter)) - if (funct7 != INT8MUL_CORE_FUNCT7[6:0]) + if (funct7 != INT8MUL_THREAD_FUNCT7[6:0]) `uvm_fatal("CASE", $sformatf("invalid funct7: %0d", funct7)) for (i = 0; i < num_gate_words; i++) begin diff --git a/examples/balls/int8mul/verify/src/cov/int8mul_cov.svh b/examples/components/balls/int8mul/verify/src/cov/int8mul_cov.svh similarity index 90% rename from examples/balls/int8mul/verify/src/cov/int8mul_cov.svh rename to examples/components/balls/int8mul/verify/src/cov/int8mul_cov.svh index 50034c541..027215c44 100644 --- a/examples/balls/int8mul/verify/src/cov/int8mul_cov.svh +++ b/examples/components/balls/int8mul/verify/src/cov/int8mul_cov.svh @@ -18,7 +18,7 @@ class int8mul_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != INT8MUL_CORE_FUNCT7[6:0]) + if (item.funct7 != INT8MUL_THREAD_FUNCT7[6:0]) `uvm_fatal("COV", $sformatf("unexpected funct7 %0d", item.funct7)) cur_iter = item.iter; cmd_count++; @@ -26,6 +26,7 @@ class int8mul_cov extends uvm_component; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") if (cmd_cg.get_coverage() != 100.0) diff --git a/examples/balls/int8mul/verify/src/env/int8mul_env.svh b/examples/components/balls/int8mul/verify/src/env/int8mul_env.svh similarity index 100% rename from examples/balls/int8mul/verify/src/env/int8mul_env.svh rename to examples/components/balls/int8mul/verify/src/env/int8mul_env.svh diff --git a/examples/balls/int8mul/verify/src/env/int8mul_scoreboard.svh b/examples/components/balls/int8mul/verify/src/env/int8mul_scoreboard.svh similarity index 98% rename from examples/balls/int8mul/verify/src/env/int8mul_scoreboard.svh rename to examples/components/balls/int8mul/verify/src/env/int8mul_scoreboard.svh index dedac48d8..7e043059f 100644 --- a/examples/balls/int8mul/verify/src/env/int8mul_scoreboard.svh +++ b/examples/components/balls/int8mul/verify/src/env/int8mul_scoreboard.svh @@ -67,6 +67,7 @@ class int8mul_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, int8mul_cmd_item exp); + verification_pkg::evidence::checks++; if (got.bid !== exp.bid) `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) if (got.funct7 !== exp.funct7) @@ -181,6 +182,7 @@ class int8mul_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) begin `uvm_fatal("SCB", $sformatf( diff --git a/examples/balls/int8mul/verify/src/pkg/int8mul_pkg.sv b/examples/components/balls/int8mul/verify/src/pkg/int8mul_pkg.sv similarity index 100% rename from examples/balls/int8mul/verify/src/pkg/int8mul_pkg.sv rename to examples/components/balls/int8mul/verify/src/pkg/int8mul_pkg.sv diff --git a/examples/balls/int8mul/verify/src/seq/int8mul_sequences.svh b/examples/components/balls/int8mul/verify/src/seq/int8mul_sequences.svh similarity index 100% rename from examples/balls/int8mul/verify/src/seq/int8mul_sequences.svh rename to examples/components/balls/int8mul/verify/src/seq/int8mul_sequences.svh diff --git a/examples/components/balls/int8mul/verify/src/tb_top.sv b/examples/components/balls/int8mul/verify/src/tb_top.sv new file mode 100644 index 000000000..cadcd55fe --- /dev/null +++ b/examples/components/balls/int8mul/verify/src/tb_top.sv @@ -0,0 +1,101 @@ +module tb_top; + import uvm_pkg::*; + import int8mul_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + Int8MulBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .io_status_idle(), + .io_status_running(), + .io_bankAccessReady(1'b1), + .io_cmdReq_ready(intf.cmd_req_ready), + .io_cmdReq_valid(intf.cmd_req_valid), + .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), + .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), + .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), + .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), + .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), + .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), + .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), + .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), + .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), + .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), + .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), + .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), + .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), + .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), + .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), + .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), + .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), + .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), + .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), + .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), + .io_cmdResp_ready(intf.cmd_resp_ready), + .io_cmdResp_valid(intf.cmd_resp_valid), + .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), + .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), + .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .io_bankRead_0_ball_id(), + .io_bankRead_0_group_id(intf.bank_read_group_id[0]), + .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .io_bankRead_1_ball_id(), + .io_bankRead_1_group_id(intf.bank_read_group_id[1]), + .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .io_bankWrite_0_ball_id(), + .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .io_subRobReq_ready(intf.sub_rob_req_ready) + ); + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("int8mul_ball_test"); + end +endmodule diff --git a/examples/balls/int8mul/verify/src/tests/int8mul_test.svh b/examples/components/balls/int8mul/verify/src/tests/int8mul_test.svh similarity index 100% rename from examples/balls/int8mul/verify/src/tests/int8mul_test.svh rename to examples/components/balls/int8mul/verify/src/tests/int8mul_test.svh diff --git a/examples/components/balls/int8mul/workloads/ctests/CMakeLists.txt b/examples/components/balls/int8mul/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..9b7b1acd2 --- /dev/null +++ b/examples/components/balls/int8mul/workloads/ctests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_buckyball_ctest(int8mul_per_channel_gate_test.c) +add_buckyball_ctest(int8mul_large_gate_row_test.c) diff --git a/examples/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c b/examples/components/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c similarity index 100% rename from examples/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c rename to examples/components/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c diff --git a/examples/balls/int8mul/workloads/ctests/int8mul_per_channel_gate_test.c b/examples/components/balls/int8mul/workloads/ctests/int8mul_per_channel_gate_test.c similarity index 100% rename from examples/balls/int8mul/workloads/ctests/int8mul_per_channel_gate_test.c rename to examples/components/balls/int8mul/workloads/ctests/int8mul_per_channel_gate_test.c diff --git a/examples/balls/int8mul/workloads/isa/int8mul.h b/examples/components/balls/int8mul/workloads/isa/int8mul.h similarity index 100% rename from examples/balls/int8mul/workloads/isa/int8mul.h rename to examples/components/balls/int8mul/workloads/isa/int8mul.h diff --git a/examples/balls/int8mul/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/int8mul/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/int8mul/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/int8mul/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/int8mul/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/int8mul/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/int8mul/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/int8mul/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir b/examples/components/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir new file mode 100644 index 000000000..82b8fd88f --- /dev/null +++ b/examples/components/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir @@ -0,0 +1,37 @@ +func.func private @check_result(memref<4x16xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %one_i8 = arith.constant 1 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %ratio = arith.constant 1.0 : f32 + %gate = memref.alloc() : memref<1x16xi8> + %input = memref.alloc() : memref<4x16xi8> + %output = memref.alloc() : memref<4x16xi8> + scf.for %lane = %zero to %sixteen step %one { + memref.store %one_i8, %gate[%zero, %lane] : memref<1x16xi8> + } + scf.for %row = %zero to %four step %one { + scf.for %lane = %zero to %sixteen step %one { + %value = arith.index_cast %lane : index to i8 + memref.store %value, %input[%row, %lane] : memref<4x16xi8> + } + } + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %gate <{bank = 0 : i64, depth = 1 : i64, stride = 1 : i64}> : memref<1x16xi8> + buckyball.mvin %input <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.custom "gate_mul_i8" args(%ratio : f32) <{sources = array, destination = 2 : i64}> {rows = 4 : i64, gate_row = 0 : i64} + buckyball.mvout %output <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + func.call @check_result(%output) : (memref<4x16xi8>) -> () + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %gate : memref<1x16xi8> + memref.dealloc %input : memref<4x16xi8> + memref.dealloc %output : memref<4x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate_main.cpp b/examples/components/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate_main.cpp similarity index 100% rename from examples/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate_main.cpp rename to examples/components/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate_main.cpp diff --git a/examples/balls/int8mul/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/int8mul/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/int8mul/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/components/balls/int8mul/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in b/examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in new file mode 100644 index 000000000..2cb2f331d --- /dev/null +++ b/examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in @@ -0,0 +1,37 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %one_i8 = arith.constant 1 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %lanes = arith.constant @BB_BANK_I8_LANES@ : index + %ratio = arith.constant 1.0 : f32 + %gate = memref.alloc() : memref<1x@BB_BANK_I8_LANES@xi8> + %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + scf.for %lane = %zero to %lanes step %one { + memref.store %one_i8, %gate[%zero, %lane] : memref<1x@BB_BANK_I8_LANES@xi8> + } + scf.for %row = %zero to %rows step %one { + scf.for %lane = %zero to %lanes step %one { + %value = arith.index_cast %lane : index to i8 + memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + } + } + %gate_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_gate = buckyball.bank_mvin %gate, %gate_bank <{depth = 1 : i64, stride = 1 : i64}> : memref<1x@BB_BANK_I8_LANES@xi8> + %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %result = buckyball.bank_custom "gate_mul_i8" (%loaded_gate, %loaded_input) -> %output_bank args(%ratio : f32) {rows = @BB_BANK_LINES@ : i64, gate_row = 0 : i64} + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () + buckyball.bank_release %loaded_gate + buckyball.bank_release %loaded_input + buckyball.bank_release %stored + memref.dealloc %gate : memref<1x@BB_BANK_I8_LANES@xi8> + memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp b/examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp similarity index 100% rename from examples/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp rename to examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp diff --git a/examples/components/balls/lut/arch/LutBall.scala b/examples/components/balls/lut/arch/LutBall.scala new file mode 100644 index 000000000..289838b20 --- /dev/null +++ b/examples/components/balls/lut/arch/LutBall.scala @@ -0,0 +1,169 @@ +package examples.balls.lut + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.lut.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import chisel3.util._ +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class LutBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + require(mp.bankWidth == 128, "LutBall requires 128-bit bank rows") + require(mp.bankEntries >= 16, "LutBall requires sixteen LUT rows") + require(ball.inBW == 2, "LutBall requires inBW=2") + require(ball.outBW == 1, "LutBall requires outBW=1") + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val idle :: inputReq :: inputResp :: lutReq :: lutResp :: writeReq :: writeResp :: complete :: Nil = + Enum(8) + val state = RegInit(idle) + val robId = RegInit(0.U(log2Up(fp.rob_entries).W)) + val inputBank = RegInit(0.U(fp.bank_id_len.W)) + val lutBank = RegInit(0.U(fp.bank_id_len.W)) + val outputBank = RegInit(0.U(fp.bank_id_len.W)) + val bases = Reg(Vec(3, UInt(mp.groupIdWidth.W))) + val iter = RegInit(0.U(fp.iter_len.W)) + val inputRow = RegInit(0.U(log2Ceil(mp.bankEntries).W)) + val inputData = Reg(UInt(128.W)) + val laneTable = RegInit(false.B) + val lookupLane = RegInit(0.U(4.W)) + val outputWord = Reg(Vec(16, UInt(8.W))) + + blink.cmdReq.ready := state === idle + blink.cmdResp.valid := state === complete + blink.cmdResp.bits.rob_id := robId + blink.status.idle := state === idle + blink.status.running := state =/= idle && state =/= complete + + for (port <- 0 until 2) { + blink.bankRead(port).rob_id := robId + blink.bankRead(port).group_id := bases(port) + blink.bankRead(port).io.req.valid := false.B + blink.bankRead(port).io.req.bits.addr := 0.U + blink.bankRead(port).io.resp.ready := false.B + } + blink.bankRead(0).bank_id := inputBank + blink.bankRead(1).bank_id := lutBank + blink.bankRead(1).group_id := bases(1) + Mux(laneTable, lookupLane >> 2, 0.U) + + blink.bankWrite(0).rob_id := robId + blink.bankWrite(0).bank_id := outputBank + blink.bankWrite(0).group_id := bases(2) + blink.bankWrite(0).io.req.valid := false.B + blink.bankWrite(0).io.req.bits.addr := inputRow + blink.bankWrite(0).io.req.bits.data := Cat(outputWord.reverse) + blink.bankWrite(0).io.req.bits.mask := VecInit( + Seq.fill(mp.bankMaskLen)(true.B) + ) + blink.bankWrite(0).io.resp.ready := false.B + + switch(state) { + is(idle) { + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + + assert(cmd.rs2 === 0.U, "LutBall reserves rs2") + assert( + cmd.iter > 0.U && cmd.iter <= mp.bankEntries.U, + "LutBall iter must fit in one bank" + ) + assert( + cmd.op1_col === 1.U && (cmd.op2_col === 1.U || cmd.op2_col === 4.U) && cmd.wr_col === 1.U, + "LutBall requires col=1 input/output and col=1 or col=4 table" + ) + assert( + cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && + cmd.op2_bank =/= cmd.wr_bank, + "LutBall banks must be distinct" + ) + robId := blink.cmdReq.bits.rob_id + inputBank := cmd.op1_bank + lutBank := cmd.op2_bank + outputBank := cmd.wr_bank + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + iter := cmd.iter + inputRow := 0.U + laneTable := cmd.op2_col === 4.U + lookupLane := 0.U + state := inputReq + } + } + is(inputReq) { + blink.bankRead(0).io.req.valid := true.B + blink.bankRead(0).io.req.bits.addr := inputRow + when(blink.bankRead(0).io.req.fire)(state := inputResp) + } + is(inputResp) { + blink.bankRead(0).io.resp.ready := true.B + when(blink.bankRead(0).io.resp.fire) { + inputData := blink.bankRead(0).io.resp.bits.data + lookupLane := 0.U + state := lutReq + } + } + is(lutReq) { + val inputByte = (inputData >> (lookupLane << 3))(7, 0) + blink.bankRead(1).io.req.valid := true.B + blink.bankRead(1).io.req.bits.addr := Mux( + laneTable, + Cat(lookupLane(1, 0), inputByte(7, 4)), + inputByte(7, 4) + ) + when(blink.bankRead(1).io.req.fire)(state := lutResp) + } + is(lutResp) { + val inputByte = (inputData >> (lookupLane << 3))(7, 0) + blink.bankRead(1).io.resp.ready := true.B + when(blink.bankRead(1).io.resp.fire) { + outputWord(lookupLane) := + (blink.bankRead(1).io.resp.bits.data >> (inputByte(3, 0) << 3))(7, 0) + when(lookupLane === 15.U)(state := writeReq) + .otherwise { lookupLane := lookupLane + 1.U; state := lutReq } + } + } + is(writeReq) { + blink.bankWrite(0).io.req.valid := true.B + when(blink.bankWrite(0).io.req.fire)(state := writeResp) + } + is(writeResp) { + blink.bankWrite(0).io.resp.ready := true.B + when(blink.bankWrite(0).io.resp.fire) { + when(inputRow === iter - 1.U)(state := complete) + .otherwise { inputRow := inputRow + 1.U; state := inputReq } + } + } + is(complete) { + when(blink.cmdResp.fire)(state := idle) + } + } +} + +object LutBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new LutBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/components/balls/lut/configs/default.toml b/examples/components/balls/lut/configs/default.toml new file mode 100644 index 000000000..dc430c153 --- /dev/null +++ b/examples/components/balls/lut/configs/default.toml @@ -0,0 +1 @@ +# LutBall has no private configuration parameters. diff --git a/examples/components/balls/lut/configs/lut_params.proto b/examples/components/balls/lut/configs/lut_params.proto new file mode 100644 index 000000000..453bc3579 --- /dev/null +++ b/examples/components/balls/lut/configs/lut_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.lut; +option java_package = "buckyball.balls.lut"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/components/balls/lut/emu/src/dispatch.rs b/examples/components/balls/lut/emu/src/dispatch.rs new file mode 100644 index 000000000..175fc37a9 --- /dev/null +++ b/examples/components/balls/lut/emu/src/dispatch.rs @@ -0,0 +1,36 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::lut; + +const BALL_CLASS: &str = "examples.balls.lut.LutBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("LUT") => Some(lut::Lut::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("LUT") => Some(lut::Lut::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/components/balls/lut/emu/src/lib.rs b/examples/components/balls/lut/emu/src/lib.rs new file mode 100644 index 000000000..80c7d3734 --- /dev/null +++ b/examples/components/balls/lut/emu/src/lib.rs @@ -0,0 +1,11 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod lut; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/balls/lut/emu/src/lut.rs b/examples/components/balls/lut/emu/src/lut.rs similarity index 94% rename from examples/balls/lut/emu/src/lut.rs rename to examples/components/balls/lut/emu/src/lut.rs index e52d7cc72..b253922ec 100644 --- a/examples/balls/lut/emu/src/lut.rs +++ b/examples/components/balls/lut/emu/src/lut.rs @@ -1,10 +1,10 @@ -use super::super::bank::bank_lines; +use bebop_bemu::thread::banks::bank_lines; use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::{BallInstruction, ExecContext}; +use super::instruction::{BallModel, ExecContext}; pub struct Lut; -impl BallInstruction for Lut { +impl BallModel for Lut { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let input_bank = rs1_b0(xs1); let lut_bank = rs1_b1(xs1); diff --git a/examples/components/balls/lut/stack/src/Lowering.cpp b/examples/components/balls/lut/stack/src/Lowering.cpp new file mode 100644 index 000000000..65ff0b717 --- /dev/null +++ b/examples/components/balls/lut/stack/src/Lowering.cpp @@ -0,0 +1,38 @@ +//===- Lowering.cpp - LutBall in lower-buckyball --------------------------===// +#include "Buckyball/BuckyballOps.h" +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" +using namespace mlir; +using namespace ::buddy::buckyball; +namespace { +class LookupI8Lowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &b) const override { + if (op.getKind() != "lookup_i8") + return failure(); + int64_t rows = cast(op->getAttr("rows")).getInt(); + if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || + op.getSources().size() != 2 || + op.getSources()[0] == op.getSources()[1] || + llvm::is_contained(op.getSources(), op.getDestination())) + return op.emitOpError( + "lookup_i8 requires distinct banks and rows within bank depth"); + mlir::buddy::emitInstruction( + b, op.getLoc(), buckyball_target::getBuckyballFunct7("LUT"), + mlir::buddy::constantI64(b, op.getLoc(), + mlir::buddy::bankFields(op.getSources(), + op.getDestination(), + false, rows)), + mlir::buddy::constantI64(b, op.getLoc(), 0)); + b.eraseOp(op); + return success(); + } +}; +void populate(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallLowering(populate), true); +} // namespace diff --git a/examples/balls/lut/verify/casegen/Cargo.lock b/examples/components/balls/lut/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/lut/verify/casegen/Cargo.lock rename to examples/components/balls/lut/verify/casegen/Cargo.lock diff --git a/examples/balls/lut/verify/casegen/Cargo.toml b/examples/components/balls/lut/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/lut/verify/casegen/Cargo.toml rename to examples/components/balls/lut/verify/casegen/Cargo.toml diff --git a/examples/components/balls/lut/verify/casegen/src/casegen.rs b/examples/components/balls/lut/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..631f250b1 --- /dev/null +++ b/examples/components/balls/lut/verify/casegen/src/casegen.rs @@ -0,0 +1,260 @@ +pub const BYTES_PER_ROW: usize = 16; +pub const SHARED_TABLE_BYTES: usize = 256; +pub const LANE_TABLE_BYTES: usize = 16 * SHARED_TABLE_BYTES; +pub const MAX_SRC: usize = 8; +pub const MAX_LUT: usize = LANE_TABLE_BYTES / BYTES_PER_ROW; +pub const MAX_DST: usize = 8; +pub const NUM_CASES: u32 = 4; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LutCmd { + pub bid: u32, + pub iter: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, + pub rob_id: u32, + pub rs1_lo: u32, + pub rs1_hi: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub num_src_words: u32, + pub num_lut_words: u32, + pub num_dst_words: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LutCase { + pub cmd: LutCmd, + pub src_words: [u128; MAX_SRC], + pub lut_words: [u128; MAX_LUT], + pub dst_words: [u128; MAX_DST], +} + +impl LutCase { + pub fn src_lo(&self, index: usize) -> u64 { + self.src_words[index] as u64 + } + + pub fn src_hi(&self, index: usize) -> u64 { + (self.src_words[index] >> 64) as u64 + } + + pub fn lut_lo(&self, index: usize) -> u64 { + self.lut_words[index] as u64 + } + + pub fn lut_hi(&self, index: usize) -> u64 { + (self.lut_words[index] >> 64) as u64 + } + + pub fn dst_lo(&self, index: usize) -> u64 { + self.dst_words[index] as u64 + } + + pub fn dst_hi(&self, index: usize) -> u64 { + (self.dst_words[index] >> 64) as u64 + } +} + +pub fn gen_case(index: u32, bid: u32) -> LutCase { + if index >= NUM_CASES { + panic!("lut: unsupported directed case {index}"); + } + match index { + 0 => directed(bid, 2, 4, false), + 1 => directed(bid, 3, 1, false), + 2 => directed(bid, 4, 8, false), + 3 => directed(bid, 5, 4, true), + _ => panic!("lut: unsupported directed case {index}"), + } +} + +fn shared_table() -> Vec { + let mut table = vec![0u8; SHARED_TABLE_BYTES]; + for (i, slot) in table.iter_mut().enumerate() { + *slot = (((i * 73 + 19) ^ 0xa5) & 0xff) as u8; + } + table +} + +fn lane_table() -> Vec { + let mut table = vec![0u8; LANE_TABLE_BYTES]; + for channel in 0..16 { + for input in 0..256 { + table[channel * 256 + input] = ((input as i32 + channel as i32 * 17 - 128) as i8) as u8; + } + } + table +} + +fn pack_lane_table(table: &[u8]) -> Vec { + if table.len() != LANE_TABLE_BYTES { + panic!("lut: invalid lane table size {}", table.len()); + } + let mut packed = vec![0u8; LANE_TABLE_BYTES]; + for row in 0..64 { + for group in 0..4 { + for byte in 0..16 { + packed[group * 64 * 16 + row * 16 + byte] = + table[(group * 4 + row / 16) * 256 + (row % 16) * 16 + byte]; + } + } + } + packed +} + +fn ctest_input(nbytes: usize) -> Vec { + (0..nbytes) + .map(|i| (i as i32 * 29 - 128) as i8 as u8) + .collect() +} + +fn pack_bytes(bytes: &[u8]) -> Vec { + if bytes.len() % BYTES_PER_ROW != 0 { + panic!( + "lut: {} bytes is not a multiple of {BYTES_PER_ROW}", + bytes.len() + ); + } + bytes + .chunks(BYTES_PER_ROW) + .map(|chunk| { + let mut word = 0u128; + for (lane, byte) in chunk.iter().enumerate() { + word |= u128::from(*byte) << (lane * 8); + } + word + }) + .collect() +} + +fn copy_words(src: &[u128]) -> [u128; N] { + if src.len() > N { + panic!("lut: {} words exceed buffer {N}", src.len()); + } + let mut out = [0u128; N]; + out[..src.len()].copy_from_slice(src); + out +} + +fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { + u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) +} + +fn split_rs(value: u64) -> (u32, u32) { + (value as u32, (value >> 32) as u32) +} + +pub(crate) fn validate( + op1_bank: u32, + op2_bank: u32, + wr_bank: u32, + op1_col: u32, + op2_col: u32, + wr_col: u32, + iter: u32, + rs1: u64, + rs2: u64, +) { + if iter == 0 { + panic!("lut: iter must be positive"); + } + if op1_col != 1 || (op2_col != 1 && op2_col != 4) || wr_col != 1 { + panic!("lut: requires col=1 input/output and col=1 or col=4 table"); + } + if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { + panic!("lut: banks must be distinct"); + } + if (rs1 & 0x3ff) != u64::from(op1_bank) + || ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) + || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) + { + panic!("lut: rs1 bank fields mismatch"); + } + if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { + panic!("lut: rs1 iter mismatch"); + } + if rs2 != 0 { + panic!("lut: rs2 must be zero"); + } +} + +fn dst_words(table: &[u8], input: &[u8], lane_table: bool) -> Vec { + let mapped: Vec = input + .iter() + .enumerate() + .map(|(index, &byte)| { + let channel = index % 16; + table[(if lane_table { channel * 256 } else { 0 }) + byte as usize] + }) + .collect(); + pack_bytes(&mapped) +} + +fn directed(bid: u32, rob_id: u32, iter: u32, per_lane: bool) -> LutCase { + let nbytes = iter as usize * BYTES_PER_ROW; + let table = if per_lane { + lane_table() + } else { + shared_table() + }; + let input = ctest_input(nbytes); + let src = pack_bytes(&input); + let lut = if per_lane { + pack_bytes(&pack_lane_table(&table)) + } else { + pack_bytes(&table) + }; + let dst = dst_words(&table, &input, per_lane); + let op1_bank = 0; + let op2_bank = 1; + let wr_bank = 2; + let op1_col = 1; + let op2_col = if per_lane { 4 } else { 1 }; + let wr_col = 1; + let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); + let rs2 = 0u64; + validate( + op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs1, rs2, + ); + if src.len() != iter as usize { + panic!("lut: src rows {} != iter {iter}", src.len()); + } + let expected_lut_rows = if per_lane { MAX_LUT } else { 16 }; + if lut.len() != expected_lut_rows { + panic!("lut: table rows {} != {expected_lut_rows}", lut.len()); + } + if dst.len() != iter as usize { + panic!("lut: dst rows {} != iter {iter}", dst.len()); + } + let (rs1_lo, rs1_hi) = split_rs(rs1); + let (rs2_lo, rs2_hi) = split_rs(rs2); + LutCase { + cmd: LutCmd { + bid, + iter, + op1_bank, + op2_bank, + wr_bank, + op1_col, + op2_col, + wr_col, + rob_id, + rs1_lo, + rs1_hi, + rs2_lo, + rs2_hi, + num_src_words: src.len() as u32, + num_lut_words: lut.len() as u32, + num_dst_words: dst.len() as u32, + }, + src_words: copy_words(&src), + lut_words: copy_words(&lut), + dst_words: copy_words(&dst), + } +} diff --git a/examples/components/balls/lut/verify/casegen/src/lib.rs b/examples/components/balls/lut/verify/casegen/src/lib.rs new file mode 100644 index 000000000..b1143b748 --- /dev/null +++ b/examples/components/balls/lut/verify/casegen/src/lib.rs @@ -0,0 +1,70 @@ +mod casegen; + +use std::cell::RefCell; + +use casegen::{LutCase, LutCmd}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn lut_case_load(index: u32, bid: u32) -> i32 { + let case = casegen::gen_case(index, bid); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("lut_case: no case loaded; call lut_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn lut_case_cmd(out_ptr: *mut LutCmd) { + if out_ptr.is_null() { + panic!("lut_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +fn src_index(index: u32, limit: u32, what: &str) -> usize { + if index >= limit { + panic!("lut_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn lut_case_src_word_lo(word_index: u32) -> u64 { + current_case(|case| case.src_lo(src_index(word_index, case.cmd.num_src_words, "src_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn lut_case_src_word_hi(word_index: u32) -> u64 { + current_case(|case| case.src_hi(src_index(word_index, case.cmd.num_src_words, "src_word_hi"))) +} + +#[no_mangle] +pub extern "C" fn lut_case_lut_word_lo(word_index: u32) -> u64 { + current_case(|case| case.lut_lo(src_index(word_index, case.cmd.num_lut_words, "lut_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn lut_case_lut_word_hi(word_index: u32) -> u64 { + current_case(|case| case.lut_hi(src_index(word_index, case.cmd.num_lut_words, "lut_word_hi"))) +} + +#[no_mangle] +pub extern "C" fn lut_case_dst_word_lo(word_index: u32) -> u64 { + current_case(|case| case.dst_lo(src_index(word_index, case.cmd.num_dst_words, "dst_word_lo"))) +} + +#[no_mangle] +pub extern "C" fn lut_case_dst_word_hi(word_index: u32) -> u64 { + current_case(|case| case.dst_hi(src_index(word_index, case.cmd.num_dst_words, "dst_word_hi"))) +} diff --git a/examples/balls/lut/verify/filelists/lut_ball.f b/examples/components/balls/lut/verify/filelists/lut_ball.f similarity index 100% rename from examples/balls/lut/verify/filelists/lut_ball.f rename to examples/components/balls/lut/verify/filelists/lut_ball.f diff --git a/examples/balls/lut/verify/src/common/lut_defs.svh b/examples/components/balls/lut/verify/src/common/lut_defs.svh similarity index 93% rename from examples/balls/lut/verify/src/common/lut_defs.svh rename to examples/components/balls/lut/verify/src/common/lut_defs.svh index 90758d450..90be94d89 100644 --- a/examples/balls/lut/verify/src/common/lut_defs.svh +++ b/examples/components/balls/lut/verify/src/common/lut_defs.svh @@ -30,10 +30,10 @@ import "DPI-C" function longint unsigned lut_case_dst_word_lo(input int unsigned import "DPI-C" function longint unsigned lut_case_dst_word_hi(input int unsigned word_index); `ifndef LUT_FUNCT7 -`error "LUT_FUNCT7 must be provided by the selected Core ballISA" +`error "LUT_FUNCT7 must be provided by the selected Thread ballISA" `endif -localparam int LUT_CORE_FUNCT7 = `LUT_FUNCT7; +localparam int LUT_THREAD_FUNCT7 = `LUT_FUNCT7; localparam int LUT_MAX_SRC_WORDS = 8; localparam int LUT_SHARED_TABLE_ROWS = 16; localparam int LUT_LANE_TABLE_ROWS = 256; diff --git a/examples/balls/lut/verify/src/common/lut_items.svh b/examples/components/balls/lut/verify/src/common/lut_items.svh similarity index 97% rename from examples/balls/lut/verify/src/common/lut_items.svh rename to examples/components/balls/lut/verify/src/common/lut_items.svh index 533ff69e9..39767cd5b 100644 --- a/examples/balls/lut/verify/src/common/lut_items.svh +++ b/examples/components/balls/lut/verify/src/common/lut_items.svh @@ -9,7 +9,7 @@ class lut_cmd_item extends bb_blink_cmd_item; bit [127:0] dst_words[LUT_MAX_DST_WORDS]; constraint legal_c { - funct7 == LUT_CORE_FUNCT7[6:0]; + funct7 == LUT_THREAD_FUNCT7[6:0]; op1_en == 1'b1; op2_en == 1'b1; wr_spad_en == 1'b1; @@ -56,7 +56,7 @@ class lut_cmd_item extends bb_blink_cmd_item; num_src_words = cmd.num_src_words; num_lut_words = cmd.num_lut_words; num_dst_words = cmd.num_dst_words; - funct7 = LUT_CORE_FUNCT7[6:0]; + funct7 = LUT_THREAD_FUNCT7[6:0]; special = 64'd0; op1_en = 1'b1; op2_en = 1'b1; diff --git a/examples/balls/lut/verify/src/cov/lut_cov.svh b/examples/components/balls/lut/verify/src/cov/lut_cov.svh similarity index 92% rename from examples/balls/lut/verify/src/cov/lut_cov.svh rename to examples/components/balls/lut/verify/src/cov/lut_cov.svh index 356973178..71c3ad503 100644 --- a/examples/balls/lut/verify/src/cov/lut_cov.svh +++ b/examples/components/balls/lut/verify/src/cov/lut_cov.svh @@ -20,7 +20,7 @@ class lut_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != LUT_CORE_FUNCT7[6:0]) + if (item.funct7 != LUT_THREAD_FUNCT7[6:0]) `uvm_fatal("COV", $sformatf("unexpected funct7 %0d", item.funct7)) cur_iter = item.iter; cur_lut_cols = item.op2_col; @@ -29,6 +29,7 @@ class lut_cov extends uvm_component; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") if (cmd_cg.get_coverage() != 100.0) diff --git a/examples/balls/lut/verify/src/env/lut_env.svh b/examples/components/balls/lut/verify/src/env/lut_env.svh similarity index 100% rename from examples/balls/lut/verify/src/env/lut_env.svh rename to examples/components/balls/lut/verify/src/env/lut_env.svh diff --git a/examples/balls/lut/verify/src/env/lut_scoreboard.svh b/examples/components/balls/lut/verify/src/env/lut_scoreboard.svh similarity index 98% rename from examples/balls/lut/verify/src/env/lut_scoreboard.svh rename to examples/components/balls/lut/verify/src/env/lut_scoreboard.svh index bfbf113f4..1a5704a31 100644 --- a/examples/balls/lut/verify/src/env/lut_scoreboard.svh +++ b/examples/components/balls/lut/verify/src/env/lut_scoreboard.svh @@ -67,6 +67,7 @@ class lut_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, lut_cmd_item exp); + verification_pkg::evidence::checks++; if (got.bid !== exp.bid) `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) if (got.funct7 !== exp.funct7) @@ -188,6 +189,7 @@ class lut_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) begin `uvm_fatal("SCB", $sformatf( diff --git a/examples/balls/lut/verify/src/pkg/lut_pkg.sv b/examples/components/balls/lut/verify/src/pkg/lut_pkg.sv similarity index 100% rename from examples/balls/lut/verify/src/pkg/lut_pkg.sv rename to examples/components/balls/lut/verify/src/pkg/lut_pkg.sv diff --git a/examples/balls/lut/verify/src/seq/lut_sequences.svh b/examples/components/balls/lut/verify/src/seq/lut_sequences.svh similarity index 100% rename from examples/balls/lut/verify/src/seq/lut_sequences.svh rename to examples/components/balls/lut/verify/src/seq/lut_sequences.svh diff --git a/examples/components/balls/lut/verify/src/tb_top.sv b/examples/components/balls/lut/verify/src/tb_top.sv new file mode 100644 index 000000000..468b257d5 --- /dev/null +++ b/examples/components/balls/lut/verify/src/tb_top.sv @@ -0,0 +1,101 @@ +module tb_top; + import uvm_pkg::*; + import lut_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + LutBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .io_status_idle(), + .io_status_running(), + .io_bankAccessReady(1'b1), + .io_cmdReq_ready(intf.cmd_req_ready), + .io_cmdReq_valid(intf.cmd_req_valid), + .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), + .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), + .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), + .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), + .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), + .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), + .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), + .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), + .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), + .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), + .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), + .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), + .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), + .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), + .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), + .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), + .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), + .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), + .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), + .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), + .io_cmdResp_ready(intf.cmd_resp_ready), + .io_cmdResp_valid(intf.cmd_resp_valid), + .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), + .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), + .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .io_bankRead_0_ball_id(), + .io_bankRead_0_group_id(intf.bank_read_group_id[0]), + .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .io_bankRead_1_ball_id(), + .io_bankRead_1_group_id(intf.bank_read_group_id[1]), + .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .io_bankWrite_0_ball_id(), + .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .io_subRobReq_ready(intf.sub_rob_req_ready) + ); + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("lut_ball_test"); + end +endmodule diff --git a/examples/balls/lut/verify/src/tests/lut_test.svh b/examples/components/balls/lut/verify/src/tests/lut_test.svh similarity index 100% rename from examples/balls/lut/verify/src/tests/lut_test.svh rename to examples/components/balls/lut/verify/src/tests/lut_test.svh diff --git a/examples/components/balls/lut/workloads/ctests/CMakeLists.txt b/examples/components/balls/lut/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..2ece5dc77 --- /dev/null +++ b/examples/components/balls/lut/workloads/ctests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_buckyball_ctest(lut_uniform_lookup_test.c) +add_buckyball_ctest(lut_channel_table_lookup_test.c) diff --git a/examples/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c b/examples/components/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c similarity index 100% rename from examples/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c rename to examples/components/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c diff --git a/examples/balls/lut/workloads/ctests/lut_uniform_lookup_test.c b/examples/components/balls/lut/workloads/ctests/lut_uniform_lookup_test.c similarity index 100% rename from examples/balls/lut/workloads/ctests/lut_uniform_lookup_test.c rename to examples/components/balls/lut/workloads/ctests/lut_uniform_lookup_test.c diff --git a/examples/balls/lut/workloads/isa/lut.h b/examples/components/balls/lut/workloads/isa/lut.h similarity index 100% rename from examples/balls/lut/workloads/isa/lut.h rename to examples/components/balls/lut/workloads/isa/lut.h diff --git a/examples/balls/lut/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/lut/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/lut/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/lut/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/lut/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/lut/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/lut/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/lut/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir b/examples/components/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir new file mode 100644 index 000000000..2887beb3a --- /dev/null +++ b/examples/components/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir @@ -0,0 +1,48 @@ +func.func private @check_result(memref<4x16xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %sixteen_i32 = arith.constant 16 : i32 + %input = memref.alloc() : memref<4x16xi8> + %table = memref.alloc() : memref<16x16xi8> + %output = memref.alloc() : memref<4x16xi8> + scf.for %row = %zero to %sixteen step %one { + scf.for %lane = %zero to %sixteen step %one { + %row32 = arith.index_cast %row : index to i32 + %lane32 = arith.index_cast %lane : index to i32 + %base = arith.muli %row32, %sixteen_i32 : i32 + %index = arith.addi %base, %lane32 : i32 + %value = arith.trunci %index : i32 to i8 + memref.store %value, %table[%row, %lane] : memref<16x16xi8> + } + } + scf.for %row = %zero to %four step %one { + scf.for %lane = %zero to %sixteen step %one { + %row32 = arith.index_cast %row : index to i32 + %lane32 = arith.index_cast %lane : index to i32 + %base = arith.muli %row32, %sixteen_i32 : i32 + %index = arith.addi %base, %lane32 : i32 + %value = arith.trunci %index : i32 to i8 + memref.store %value, %input[%row, %lane] : memref<4x16xi8> + } + } + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 0 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.mvin %table <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.custom "lookup_i8" <{sources = array, destination = 2 : i64}> {rows = 4 : i64} + buckyball.mvout %output <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + func.call @check_result(%output) : (memref<4x16xi8>) -> () + memref.dealloc %input : memref<4x16xi8> + memref.dealloc %table : memref<16x16xi8> + memref.dealloc %output : memref<4x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/lut/workloads/mlir_tests/ball/lut_identity_main.cpp b/examples/components/balls/lut/workloads/mlir_tests/ball/lut_identity_main.cpp similarity index 100% rename from examples/balls/lut/workloads/mlir_tests/ball/lut_identity_main.cpp rename to examples/components/balls/lut/workloads/mlir_tests/ball/lut_identity_main.cpp diff --git a/examples/balls/lut/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/lut/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/lut/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/components/balls/lut/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in b/examples/components/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in new file mode 100644 index 000000000..e0f7a147e --- /dev/null +++ b/examples/components/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in @@ -0,0 +1,50 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %lanes = arith.constant @BB_BANK_I8_LANES@ : index + %sixteen = arith.constant 16 : index + %lanes_i32 = arith.constant @BB_BANK_I8_LANES@ : i32 + %sixteen_i32 = arith.constant 16 : i32 + %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %table = memref.alloc() : memref<16x16xi8> + %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + scf.for %row = %zero to %sixteen step %one { + scf.for %lane = %zero to %sixteen step %one { + %row32 = arith.index_cast %row : index to i32 + %lane32 = arith.index_cast %lane : index to i32 + %base = arith.muli %row32, %sixteen_i32 : i32 + %index = arith.addi %base, %lane32 : i32 + %value = arith.trunci %index : i32 to i8 + memref.store %value, %table[%row, %lane] : memref<16x16xi8> + } + } + scf.for %row = %zero to %rows step %one { + scf.for %lane = %zero to %lanes step %one { + %row32 = arith.index_cast %row : index to i32 + %lane32 = arith.index_cast %lane : index to i32 + %base = arith.muli %row32, %lanes_i32 : i32 + %index = arith.addi %base, %lane32 : i32 + %value = arith.trunci %index : i32 to i8 + memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + } + } + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %table_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %loaded_table = buckyball.bank_mvin %table, %table_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + %result = buckyball.bank_custom "lookup_i8" (%loaded_input, %loaded_table) -> %output_bank {rows = @BB_BANK_LINES@ : i64} + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + buckyball.bank_release %loaded_input + buckyball.bank_release %loaded_table + buckyball.bank_release %stored + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () + memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.dealloc %table : memref<16x16xi8> + memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp b/examples/components/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp similarity index 100% rename from examples/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp rename to examples/components/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp diff --git a/examples/balls/matadd/README.md b/examples/components/balls/matadd/README.md similarity index 100% rename from examples/balls/matadd/README.md rename to examples/components/balls/matadd/README.md diff --git a/examples/components/balls/matadd/arch/MatAdd.scala b/examples/components/balls/matadd/arch/MatAdd.scala new file mode 100644 index 000000000..9e570d141 --- /dev/null +++ b/examples/components/balls/matadd/arch/MatAdd.scala @@ -0,0 +1,219 @@ +package examples.balls.matadd + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.experimental.hierarchy.{instantiable, public} +import chisel3.util._ +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} + +@instantiable +class MatAdd( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val addressWidth = log2Up(memParam.bankEntries) + val countWidth = log2Up(memParam.bankEntries + 1) + val bankIdWidth = frontendParam.bank_id_len + + val mapping = instance + + require(mapping.inBW == 2, "MatAddBall requires two SRAM read ports") + require(mapping.outBW == 1, "MatAddBall requires one SRAM write port") + require(memParam.bankWidth == 128, "MatAddBall requires 128-bit SRAM rows") + require( + memParam.bankMaskLen == 16, + "MatAddBall requires sixteen byte enables" + ) + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + 2, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + 1, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val Seq(idle, waitForChannels, readRequest, readResponse, writeRequest, writeResponse, complete) = Enum(7) + val state = RegInit(idle) + + val robId = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + val aBank = RegInit(0.U(bankIdWidth.W)) + val bBank = RegInit(0.U(bankIdWidth.W)) + val cBank = RegInit(0.U(bankIdWidth.W)) + val groups = RegInit(0.U(5.W)) + val group = RegInit(0.U(5.W)) + val bases = Reg(Vec(3, UInt(memParam.groupIdWidth.W))) + val iter = RegInit(0.U(countWidth.W)) + val line = RegInit(0.U(addressWidth.W)) + val aWord = Reg(UInt(128.W)) + val bWord = Reg(UInt(128.W)) + val aRequestSent = RegInit(false.B) + val bRequestSent = RegInit(false.B) + val aReceived = RegInit(false.B) + val bReceived = RegInit(false.B) + + val sum = Wire(Vec(4, UInt(32.W))) + for (lane <- 0 until 4) { + sum(lane) := (aWord(32 * lane + 31, 32 * lane) +& + bWord(32 * lane + 31, 32 * lane))(31, 0) + } + + io.cmdReq.ready := state === idle + io.cmdResp.valid := state === complete + io.cmdResp.bits.rob_id := robId + + for (port <- 0 until 2) { + io.bankRead(port).rob_id := robId + io.bankRead(port).bank_id := (if (port == 0) aBank else bBank) + io.bankRead(port).group_id := bases(port) + group + io.bankRead(port).io.req.valid := false.B + io.bankRead(port).io.req.bits.addr := line + io.bankRead(port).io.resp.ready := false.B + } + + io.bankWrite(0).rob_id := robId + io.bankWrite(0).bank_id := cBank + io.bankWrite(0).group_id := bases(2) + group + io.bankWrite(0).io.req.valid := false.B + io.bankWrite(0).io.req.bits.addr := line + io.bankWrite(0).io.req.bits.data := Cat(sum.reverse) + io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) + io.bankWrite(0).io.resp.ready := false.B + + switch(state) { + is(idle) { + when(io.cmdReq.fire) { + val command = io.cmdReq.bits + + assert( + command.op1_bank =/= command.op2_bank && command.op1_bank =/= command.wr_bank && + command.op2_bank =/= command.wr_bank, + "MatAddBall banks must be distinct" + ) + assert( + command.op1_col =/= 0.U && command.op1_col === command.op2_col && + command.op1_col === command.wr_col, + "MatAddBall bank groups must match" + ) + assert( + command.op1_col <= memParam.bankNum.U, + "MatAddBall bank groups must fit in physical banks" + ) + assert( + command.iter =/= 0.U && command.iter <= memParam.bankEntries.U, + "MatAddBall iter must fit in one physical bank" + ) + assert(command.rs2 === 0.U, "MatAddBall reserves rs2") + + robId := io.cmdReq.bits.rob_id + aBank := command.op1_bank + bBank := command.op2_bank + cBank := command.wr_bank + groups := command.op1_col + group := 0.U + bases := VecInit(command.op1_group, command.op2_group, command.wr_group) + iter := command.iter + line := 0.U + aRequestSent := false.B + bRequestSent := false.B + aReceived := false.B + bReceived := false.B + state := waitForChannels + } + } + + is(waitForChannels) { + state := readRequest + } + + is(readRequest) { + io.bankRead(0).io.req.valid := !aRequestSent + io.bankRead(1).io.req.valid := !bRequestSent + when(io.bankRead(0).io.req.fire)(aRequestSent := true.B) + when(io.bankRead(1).io.req.fire)(bRequestSent := true.B) + when((aRequestSent || io.bankRead(0).io.req.fire) && + (bRequestSent || io.bankRead(1).io.req.fire)) { + state := readResponse + } + } + + is(readResponse) { + io.bankRead(0).io.resp.ready := !aReceived + io.bankRead(1).io.resp.ready := !bReceived + when(io.bankRead(0).io.resp.fire) { + aWord := io.bankRead(0).io.resp.bits.data + aReceived := true.B + } + when(io.bankRead(1).io.resp.fire) { + bWord := io.bankRead(1).io.resp.bits.data + bReceived := true.B + } + when((aReceived || io.bankRead(0).io.resp.fire) && + (bReceived || io.bankRead(1).io.resp.fire)) { + state := writeRequest + } + } + + is(writeRequest) { + io.bankWrite(0).io.req.valid := true.B + when(io.bankWrite(0).io.req.fire)(state := writeResponse) + } + + is(writeResponse) { + io.bankWrite(0).io.resp.ready := true.B + when(io.bankWrite(0).io.resp.fire) { + aRequestSent := false.B + bRequestSent := false.B + aReceived := false.B + bReceived := false.B + when(line +& 1.U === iter) { + line := 0.U + when(group +& 1.U === groups) { + state := complete + }.otherwise { + group := group + 1.U + state := readRequest + } + }.otherwise { + line := line + 1.U + state := readRequest + } + } + } + + is(complete) { + when(io.cmdResp.fire)(state := idle) + } + } + + io.status.idle := state === idle + io.status.running := state =/= idle +} diff --git a/examples/components/balls/matadd/arch/MatAddBall.scala b/examples/components/balls/matadd/arch/MatAddBall.scala new file mode 100644 index 000000000..c6fc7517a --- /dev/null +++ b/examples/components/balls/matadd/arch/MatAddBall.scala @@ -0,0 +1,49 @@ +package examples.balls.matadd + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.matadd.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.{BlinkIO, HasBlink} + +@instantiable +class MatAddBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val unit: Instance[MatAdd] = + Instantiate(new MatAdd(fp, mp, bp, ball)) + unit.io.cmdReq <> blink.cmdReq + unit.io.cmdResp <> blink.cmdResp + unit.io.bankRead(0) <> blink.bankRead(0) + unit.io.bankRead(1) <> blink.bankRead(1) + unit.io.bankWrite(0) <> blink.bankWrite(0) + blink.status <> unit.io.status +} + +object MatAddBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new MatAddBall(fp, mp, bp, id)).blink, None) + +} diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/csrc/README.md b/examples/components/balls/matadd/configs/default.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/coherence/src/csrc/README.md rename to examples/components/balls/matadd/configs/default.toml diff --git a/examples/components/balls/matadd/configs/matadd_params.proto b/examples/components/balls/matadd/configs/matadd_params.proto new file mode 100644 index 000000000..4f6956443 --- /dev/null +++ b/examples/components/balls/matadd/configs/matadd_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.matadd; +option java_package = "buckyball.balls.matadd"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/balls/matadd/emu/src/64_matadd.rs b/examples/components/balls/matadd/emu/src/64_matadd.rs similarity index 94% rename from examples/balls/matadd/emu/src/64_matadd.rs rename to examples/components/balls/matadd/emu/src/64_matadd.rs index efee1a5cb..fdd71258e 100644 --- a/examples/balls/matadd/emu/src/64_matadd.rs +++ b/examples/components/balls/matadd/emu/src/64_matadd.rs @@ -1,10 +1,10 @@ -use super::super::bank::bank_size; +use bebop_bemu::thread::banks::bank_size; use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::{BallInstruction, ExecContext}; +use super::instruction::{BallModel, ExecContext}; pub struct MatAdd; -impl BallInstruction for MatAdd { +impl BallModel for MatAdd { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let a_bank = rs1_b0(xs1); let b_bank = rs1_b1(xs1); diff --git a/examples/components/balls/matadd/emu/src/dispatch.rs b/examples/components/balls/matadd/emu/src/dispatch.rs new file mode 100644 index 000000000..d090f7398 --- /dev/null +++ b/examples/components/balls/matadd/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::f64_matadd; + +const BALL_CLASS: &str = "examples.balls.matadd.MatAddBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MATADD") => Some(f64_matadd::MatAdd::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MATADD") => Some(f64_matadd::MatAdd::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/components/balls/matadd/emu/src/lib.rs b/examples/components/balls/matadd/emu/src/lib.rs new file mode 100644 index 000000000..6242a9b5a --- /dev/null +++ b/examples/components/balls/matadd/emu/src/lib.rs @@ -0,0 +1,12 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +#[path = "64_matadd.rs"] +mod f64_matadd; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/components/balls/matadd/stack/src/Lowering.cpp b/examples/components/balls/matadd/stack/src/Lowering.cpp new file mode 100644 index 000000000..6d79ce1a2 --- /dev/null +++ b/examples/components/balls/matadd/stack/src/Lowering.cpp @@ -0,0 +1,49 @@ +//===- Lowering.cpp - MatAddBall in lower-buckyball -----------------------===// +// +// add_i32: C = A + B modulo 2^32 over `rows` rows of four i32 in every group +// of the banks; A, B and C have the same number of groups. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballOps.h" +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class AddI32Lowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &rewriter) const override { + if (op.getKind() != "add_i32") + return failure(); + Location loc = op.getLoc(); + int64_t rows = cast(op->getAttr("rows")).getInt(); + if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || + op.getSources().size() != 2 || + op.getSources()[0] == op.getSources()[1] || + llvm::is_contained(op.getSources(), op.getDestination())) + return op.emitOpError( + "add_i32 requires distinct banks and rows within a bank"); + mlir::buddy::emitInstruction( + rewriter, loc, buckyball_target::getBuckyballFunct7("MATADD"), + mlir::buddy::constantI64(rewriter, loc, + mlir::buddy::bankFields(op.getSources(), + op.getDestination(), + false, rows)), + mlir::buddy::constantI64(rewriter, loc, 0)); + rewriter.eraseOp(op); + return success(); + } +}; + +void populate(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} + +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallLowering(populate), true); +} // namespace diff --git a/examples/balls/matadd/verify/casegen/Cargo.lock b/examples/components/balls/matadd/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/matadd/verify/casegen/Cargo.lock rename to examples/components/balls/matadd/verify/casegen/Cargo.lock diff --git a/examples/balls/matadd/verify/casegen/Cargo.toml b/examples/components/balls/matadd/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/matadd/verify/casegen/Cargo.toml rename to examples/components/balls/matadd/verify/casegen/Cargo.toml diff --git a/examples/balls/matadd/verify/casegen/src/lib.rs b/examples/components/balls/matadd/verify/casegen/src/lib.rs similarity index 100% rename from examples/balls/matadd/verify/casegen/src/lib.rs rename to examples/components/balls/matadd/verify/casegen/src/lib.rs diff --git a/examples/balls/matadd/verify/filelists/matadd_ball.f b/examples/components/balls/matadd/verify/filelists/matadd_ball.f similarity index 100% rename from examples/balls/matadd/verify/filelists/matadd_ball.f rename to examples/components/balls/matadd/verify/filelists/matadd_ball.f diff --git a/examples/components/balls/matadd/verify/src/common/matadd_defs.svh b/examples/components/balls/matadd/verify/src/common/matadd_defs.svh new file mode 100644 index 000000000..2510d839c --- /dev/null +++ b/examples/components/balls/matadd/verify/src/common/matadd_defs.svh @@ -0,0 +1,15 @@ +`ifndef MATADD_FUNCT7 +`error "MATADD_FUNCT7 must be provided by the selected Thread ballISA" +`endif + +localparam int MATADD_THREAD_FUNCT7 = `MATADD_FUNCT7; +localparam int MATADD_MAX_LINES = 64; +localparam int MATADD_TIMEOUT_CYCLES = 100000; + +function automatic int unsigned matadd_require_bid(); + int unsigned bid; + if (!$value$plusargs("BID=%d", bid)) begin + `uvm_fatal("BID", "missing required plusarg +BID=") + end + return bid; +endfunction diff --git a/examples/balls/matadd/verify/src/common/matadd_items.svh b/examples/components/balls/matadd/verify/src/common/matadd_items.svh similarity index 97% rename from examples/balls/matadd/verify/src/common/matadd_items.svh rename to examples/components/balls/matadd/verify/src/common/matadd_items.svh index 4ca129f43..f6c589262 100644 --- a/examples/balls/matadd/verify/src/common/matadd_items.svh +++ b/examples/components/balls/matadd/verify/src/common/matadd_items.svh @@ -6,7 +6,7 @@ class matadd_cmd_item extends bb_blink_cmd_item; int unsigned case_index; constraint legal_c { - funct7 == MATADD_CORE_FUNCT7[6:0]; + funct7 == MATADD_THREAD_FUNCT7[6:0]; op1_en == 1'b1; op2_en == 1'b1; wr_spad_en == 1'b1; @@ -28,7 +28,7 @@ class matadd_cmd_item extends bb_blink_cmd_item; if (index > 2) `uvm_fatal("CASE", $sformatf("unsupported case %0d", index)) case_index = index; this.bid = ball_id[4:0]; - funct7 = MATADD_CORE_FUNCT7[6:0]; + funct7 = MATADD_THREAD_FUNCT7[6:0]; iter = (index == 2) ? MATADD_MAX_LINES : 16; lines = iter; group_count = (index == 0) ? 1 : 2; diff --git a/examples/balls/matadd/verify/src/env/matadd_env.svh b/examples/components/balls/matadd/verify/src/env/matadd_env.svh similarity index 100% rename from examples/balls/matadd/verify/src/env/matadd_env.svh rename to examples/components/balls/matadd/verify/src/env/matadd_env.svh diff --git a/examples/balls/matadd/verify/src/env/matadd_scoreboard.svh b/examples/components/balls/matadd/verify/src/env/matadd_scoreboard.svh similarity index 99% rename from examples/balls/matadd/verify/src/env/matadd_scoreboard.svh rename to examples/components/balls/matadd/verify/src/env/matadd_scoreboard.svh index 5efeca2b3..14ddb898a 100644 --- a/examples/balls/matadd/verify/src/env/matadd_scoreboard.svh +++ b/examples/components/balls/matadd/verify/src/env/matadd_scoreboard.svh @@ -137,6 +137,7 @@ class matadd_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) `uvm_fatal("SCB", $sformatf( diff --git a/examples/balls/matadd/verify/src/pkg/matadd_pkg.sv b/examples/components/balls/matadd/verify/src/pkg/matadd_pkg.sv similarity index 100% rename from examples/balls/matadd/verify/src/pkg/matadd_pkg.sv rename to examples/components/balls/matadd/verify/src/pkg/matadd_pkg.sv diff --git a/examples/balls/matadd/verify/src/seq/matadd_sequences.svh b/examples/components/balls/matadd/verify/src/seq/matadd_sequences.svh similarity index 100% rename from examples/balls/matadd/verify/src/seq/matadd_sequences.svh rename to examples/components/balls/matadd/verify/src/seq/matadd_sequences.svh diff --git a/examples/components/balls/matadd/verify/src/tb_top.sv b/examples/components/balls/matadd/verify/src/tb_top.sv new file mode 100644 index 000000000..0edd42d94 --- /dev/null +++ b/examples/components/balls/matadd/verify/src/tb_top.sv @@ -0,0 +1,96 @@ +module tb_top; + import uvm_pkg::*; + import matadd_pkg::*; + + bb_blink_if #(2, 1) intf (); + + MatAddBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .io_bankAccessReady(1'b1), + .io_cmdReq_ready(intf.cmd_req_ready), + .io_cmdReq_valid(intf.cmd_req_valid), + .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), + .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), + .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), + .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), + .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), + .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), + .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), + .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), + .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), + .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank[3:0]), + .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank[3:0]), + .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank[3:0]), + .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), + .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), + .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), + .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank[3:0]), + .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), + .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), + .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), + .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), + .io_cmdResp_ready(intf.cmd_resp_ready), + .io_cmdResp_valid(intf.cmd_resp_valid), + .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), + .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), + .io_bankRead_0_bank_id(intf.bank_read_bank_id[0][3:0]), + .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .io_bankRead_0_group_id(intf.bank_read_group_id[0][3:0]), + .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .io_bankRead_1_bank_id(intf.bank_read_bank_id[1][3:0]), + .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .io_bankRead_1_group_id(intf.bank_read_group_id[1][3:0]), + .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0][3:0]), + .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .io_bankWrite_0_group_id(intf.bank_write_group_id[0][3:0]), + .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .io_subRobReq_ready(intf.sub_rob_req_ready) + ); + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(2, 1))::set(null, "*", "vif", intf); + run_test("matadd_ball_test"); + end +endmodule diff --git a/examples/balls/matadd/verify/src/tests/matadd_test.svh b/examples/components/balls/matadd/verify/src/tests/matadd_test.svh similarity index 100% rename from examples/balls/matadd/verify/src/tests/matadd_test.svh rename to examples/components/balls/matadd/verify/src/tests/matadd_test.svh diff --git a/examples/components/balls/matadd/workloads/ctests/CMakeLists.txt b/examples/components/balls/matadd/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..c1a53407d --- /dev/null +++ b/examples/components/balls/matadd/workloads/ctests/CMakeLists.txt @@ -0,0 +1 @@ +add_buckyball_ctest(matadd_i32_16row_test.c) diff --git a/examples/balls/matadd/workloads/ctests/matadd_i32_16row_test.c b/examples/components/balls/matadd/workloads/ctests/matadd_i32_16row_test.c similarity index 100% rename from examples/balls/matadd/workloads/ctests/matadd_i32_16row_test.c rename to examples/components/balls/matadd/workloads/ctests/matadd_i32_16row_test.c diff --git a/examples/balls/matadd/workloads/isa/matadd.h b/examples/components/balls/matadd/workloads/isa/matadd.h similarity index 100% rename from examples/balls/matadd/workloads/isa/matadd.h rename to examples/components/balls/matadd/workloads/isa/matadd.h diff --git a/examples/balls/matadd/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/matadd/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/matadd/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/matadd/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/matadd/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/matadd/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/matadd/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/matadd/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir b/examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir new file mode 100644 index 000000000..72535d490 --- /dev/null +++ b/examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir @@ -0,0 +1,37 @@ +func.func private @check_result(memref<16x4xi32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %a = memref.alloc() : memref<16x4xi32> + %b = memref.alloc() : memref<16x4xi32> + %out = memref.alloc() : memref<16x4xi32> + scf.for %row = %zero to %sixteen step %one { + scf.for %lane = %zero to %four step %one { + %r = arith.index_cast %row : index to i32 + %l = arith.index_cast %lane : index to i32 + %x = arith.addi %r, %l : i32 + %y = arith.subi %l, %r : i32 + memref.store %x, %a[%row, %lane] : memref<16x4xi32> + memref.store %y, %b[%row, %lane] : memref<16x4xi32> + } + } + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %a <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + buckyball.mvin %b <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + buckyball.custom "add_i32" <{sources = array, destination = 2 : i64}> {rows = 16 : i64} + buckyball.mvout %out <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + func.call @check_result(%out) : (memref<16x4xi32>) -> () + memref.dealloc %a : memref<16x4xi32> + memref.dealloc %b : memref<16x4xi32> + memref.dealloc %out : memref<16x4xi32> + return %zero_i8 : i8 +} diff --git a/examples/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp b/examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp similarity index 100% rename from examples/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp rename to examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp diff --git a/examples/balls/matadd/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/matadd/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/matadd/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/components/balls/matadd/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in b/examples/components/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in new file mode 100644 index 000000000..fad61675d --- /dev/null +++ b/examples/components/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in @@ -0,0 +1,40 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %lanes = arith.constant @BB_BANK_I32_LANES@ : index + %a = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %b = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %out = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + scf.for %row = %zero to %rows step %one { + scf.for %lane = %zero to %lanes step %one { + %r = arith.index_cast %row : index to i32 + %l = arith.index_cast %lane : index to i32 + %x = arith.addi %r, %l : i32 + %y = arith.subi %l, %r : i32 + memref.store %x, %a[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + memref.store %y, %b[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + } + } + %a_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %out_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %a_loaded = buckyball.bank_mvin %a, %a_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %b_loaded = buckyball.bank_mvin %b, %b_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %result = buckyball.bank_custom "add_i32" (%a_loaded, %b_loaded) -> %out_bank {rows = @BB_BANK_LINES@ : i64} + %stored = buckyball.bank_mvout %out, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + buckyball.bank_release %a_loaded + buckyball.bank_release %b_loaded + buckyball.bank_release %stored + func.call @check_result(%out) : (memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () + memref.dealloc %a : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + memref.dealloc %b : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + memref.dealloc %out : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + return %zero_i8 : i8 +} diff --git a/examples/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp b/examples/components/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp similarity index 100% rename from examples/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp rename to examples/components/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp diff --git a/examples/components/balls/maxpool/arch/MaxPoolBall.scala b/examples/components/balls/maxpool/arch/MaxPoolBall.scala new file mode 100644 index 000000000..00cfdcaf4 --- /dev/null +++ b/examples/components/balls/maxpool/arch/MaxPoolBall.scala @@ -0,0 +1,256 @@ +package examples.balls.maxpool + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.maxpool.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import chisel3.util._ +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class MaxPoolBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + require(mp.bankWidth == 128, "MaxPoolBall requires 128-bit bank rows") + require(ball.inBW == 1, "MaxPoolBall requires inBW=1") + require(ball.outBW == 1, "MaxPoolBall requires outBW=1") + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val idle :: scan :: readResp :: writeReq :: writeResp :: complete :: Nil = Enum(6) + val state = RegInit(idle) + val robId = RegInit(0.U(log2Up(fp.rob_entries).W)) + val inputBank = RegInit(0.U(fp.bank_id_len.W)) + val outputBank = RegInit(0.U(fp.bank_id_len.W)) + val bases = Reg(Vec(3, UInt(mp.groupIdWidth.W))) + val iter = RegInit(0.U(fp.iter_len.W)) + val inputSide = RegInit(0.U(8.W)) + val outputSide = RegInit(0.U(8.W)) + val kernel = RegInit(0.U(8.W)) + val stride = RegInit(0.U(8.W)) + val padding = RegInit(0.U(8.W)) + val inputBase = RegInit(0.U(6.W)) + val outputBase = RegInit(0.U(6.W)) + val outputStride = RegInit(0.U(6.W)) + val startRow = RegInit(0.U(4.W)) + val startCol = RegInit(0.U(4.W)) + val outputRow = RegInit(0.U(log2Ceil(mp.bankEntries).W)) + val outputY = RegInit(0.U(8.W)) + val outputX = RegInit(0.U(8.W)) + val kernelY = RegInit(0.U(8.W)) + val kernelX = RegInit(0.U(8.W)) + val maximum = RegInit(VecInit(Seq.fill(16)("h80".U(8.W)))) + val writeData = Reg(UInt(128.W)) + + val sourceY = outputY * stride +& kernelY +& startRow + val sourceX = outputX * stride +& kernelX +& startCol + val sourceValid = sourceY >= padding && sourceX >= padding && + sourceY < padding +& inputSide && sourceX < padding +& inputSide + val inputY = sourceY - padding + val inputX = sourceX - padding + val inputAddress = inputBase +& inputY * inputSide +& inputX + val outputAddress = outputBase +& outputY * outputStride +& outputX + val lastKernel = kernelY +& 1.U === kernel && kernelX +& 1.U === kernel + + blink.cmdReq.ready := state === idle + blink.cmdResp.valid := state === complete + blink.cmdResp.bits.rob_id := robId + blink.status.idle := state === idle + blink.status.running := state =/= idle && state =/= complete + + blink.bankRead(0).rob_id := robId + blink.bankRead(0).bank_id := inputBank + blink.bankRead(0).group_id := bases(0) + blink.bankRead(0).io.req.valid := false.B + blink.bankRead(0).io.req.bits.addr := inputAddress + blink.bankRead(0).io.resp.ready := false.B + + blink.bankWrite(0).rob_id := robId + blink.bankWrite(0).bank_id := outputBank + blink.bankWrite(0).group_id := bases(2) + blink.bankWrite(0).io.req.valid := false.B + blink.bankWrite(0).io.req.bits.addr := outputAddress + blink.bankWrite(0).io.req.bits.data := writeData + blink.bankWrite(0).io.req.bits.mask := VecInit( + Seq.fill(mp.bankMaskLen)(true.B) + ) + blink.bankWrite(0).io.resp.ready := false.B + + switch(state) { + is(idle) { + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + val inSide = cmd.rs2(3, 0) + val outSide = cmd.rs2(7, 4) + val kernelSize = cmd.rs2(11, 8) + val poolStride = cmd.rs2(15, 12) + val poolPadding = cmd.rs2(19, 16) + val inBase = cmd.rs2(25, 20) + val outBase = cmd.rs2(31, 26) + val outStride = cmd.rs2(37, 32) + val poolStartRow = cmd.rs2(41, 38) + val poolStartCol = cmd.rs2(45, 42) + assert(cmd.rs2(63, 46) === 0.U, "MAXPOOL reserved rs2 bits must be zero") + + assert( + cmd.op1_col === 1.U && cmd.op2_col === 0.U && cmd.wr_col === 1.U, + "MAXPOOL requires one input bank and one output bank" + ) + assert(cmd.op1_bank =/= cmd.wr_bank, "MAXPOOL input and output banks must differ") + assert( + inSide > 0.U && outSide > 0.U && kernelSize > 0.U && poolStride > 0.U, + "MAXPOOL dimensions must be positive" + ) + assert(outStride >= outSide, "MAXPOOL output stride is too small") + val lastSourceRow = poolStartRow +& (outSide - 1.U) * poolStride +& (kernelSize - 1.U) + val lastSourceCol = poolStartCol +& (outSide - 1.U) * poolStride +& (kernelSize - 1.U) + val inputEnd = poolPadding +& inSide + val readsInput = poolStartRow < inputEnd && poolStartCol < inputEnd && + lastSourceRow >= poolPadding && lastSourceCol >= poolPadding + val maxInputRow = Mux(lastSourceRow < inputEnd, lastSourceRow - poolPadding, inSide - 1.U) + val maxInputCol = Mux(lastSourceCol < inputEnd, lastSourceCol - poolPadding, inSide - 1.U) + assert( + !readsInput || inBase +& maxInputRow * inSide +& maxInputCol < mp.bankEntries.U, + "MAXPOOL input window exceeds bank depth" + ) + assert( + outSide * outSide === cmd.iter && + outBase +& (outSide - 1.U) * outStride +& outSide <= mp.bankEntries.U, + "MAXPOOL output core exceeds bank depth" + ) + assert( + poolStartRow +& (outSide - 1.U) * poolStride +& kernelSize <= + inSide +& (poolPadding << 1) && + poolStartCol +& (outSide - 1.U) * poolStride +& kernelSize <= + inSide +& (poolPadding << 1), + "MAXPOOL pooling geometry is inconsistent" + ) + + robId := blink.cmdReq.bits.rob_id + inputBank := cmd.op1_bank + outputBank := cmd.wr_bank + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + iter := cmd.iter + inputSide := inSide + outputSide := outSide + kernel := kernelSize + stride := poolStride + padding := poolPadding + inputBase := inBase + outputBase := outBase + outputStride := outStride + startRow := poolStartRow + startCol := poolStartCol + outputRow := 0.U + outputY := 0.U + outputX := 0.U + kernelY := 0.U + kernelX := 0.U + maximum := VecInit(Seq.fill(16)("h80".U(8.W))) + state := scan + } + } + + is(scan) { + when(sourceValid) { + blink.bankRead(0).io.req.valid := true.B + when(blink.bankRead(0).io.req.fire) { + state := readResp + } + }.elsewhen(lastKernel) { + writeData := Cat(maximum.reverse) + state := writeReq + }.otherwise { + when(kernelX +& 1.U === kernel) { + kernelX := 0.U + kernelY := kernelY + 1.U + }.otherwise { + kernelX := kernelX + 1.U + } + } + } + + is(readResp) { + blink.bankRead(0).io.resp.ready := true.B + when(blink.bankRead(0).io.resp.fire) { + val data = blink.bankRead(0).io.resp.bits.data.asTypeOf(Vec(16, UInt(8.W))) + val nextMaximum = Wire(Vec(16, UInt(8.W))) + for (lane <- 0 until 16) + nextMaximum(lane) := Mux(data(lane).asSInt > maximum(lane).asSInt, data(lane), maximum(lane)) + maximum := nextMaximum + when(lastKernel) { + writeData := Cat(nextMaximum.reverse) + state := writeReq + }.otherwise { + when(kernelX +& 1.U === kernel) { + kernelX := 0.U + kernelY := kernelY + 1.U + }.otherwise { + kernelX := kernelX + 1.U + } + state := scan + } + } + } + + is(writeReq) { + blink.bankWrite(0).io.req.valid := true.B + when(blink.bankWrite(0).io.req.fire) { + state := writeResp + } + } + + is(writeResp) { + blink.bankWrite(0).io.resp.ready := true.B + when(blink.bankWrite(0).io.resp.fire) { + when(outputRow +& 1.U === iter) { + state := complete + }.otherwise { + outputRow := outputRow + 1.U + when(outputX +& 1.U === outputSide) { + outputX := 0.U + outputY := outputY + 1.U + }.otherwise { + outputX := outputX + 1.U + } + kernelY := 0.U + kernelX := 0.U + maximum := VecInit(Seq.fill(16)("h80".U(8.W))) + state := scan + } + } + } + + is(complete) { + when(blink.cmdResp.fire) { + state := idle + } + } + } +} + +object MaxPoolBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new MaxPoolBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/components/balls/maxpool/configs/default.toml b/examples/components/balls/maxpool/configs/default.toml new file mode 100644 index 000000000..23c90c65a --- /dev/null +++ b/examples/components/balls/maxpool/configs/default.toml @@ -0,0 +1 @@ +# MaxPoolBall has no private configuration parameters. diff --git a/examples/components/balls/maxpool/configs/maxpool_params.proto b/examples/components/balls/maxpool/configs/maxpool_params.proto new file mode 100644 index 000000000..659427496 --- /dev/null +++ b/examples/components/balls/maxpool/configs/maxpool_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.maxpool; +option java_package = "buckyball.balls.maxpool"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/components/balls/maxpool/emu/src/dispatch.rs b/examples/components/balls/maxpool/emu/src/dispatch.rs new file mode 100644 index 000000000..1e6522985 --- /dev/null +++ b/examples/components/balls/maxpool/emu/src/dispatch.rs @@ -0,0 +1,36 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::maxpool; + +const BALL_CLASS: &str = "examples.balls.maxpool.MaxPoolBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MAXPOOL") => Some(maxpool::MaxPool::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MAXPOOL") => Some(maxpool::MaxPool::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/components/balls/maxpool/emu/src/lib.rs b/examples/components/balls/maxpool/emu/src/lib.rs new file mode 100644 index 000000000..6b4391d90 --- /dev/null +++ b/examples/components/balls/maxpool/emu/src/lib.rs @@ -0,0 +1,11 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod maxpool; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/balls/maxpool/emu/src/maxpool.rs b/examples/components/balls/maxpool/emu/src/maxpool.rs similarity index 97% rename from examples/balls/maxpool/emu/src/maxpool.rs rename to examples/components/balls/maxpool/emu/src/maxpool.rs index 4f5dbf2bb..af47d4477 100644 --- a/examples/balls/maxpool/emu/src/maxpool.rs +++ b/examples/components/balls/maxpool/emu/src/maxpool.rs @@ -1,6 +1,6 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::{BallInstruction, ExecContext}; +use super::instruction::{BallModel, ExecContext}; pub struct MaxPool; @@ -51,7 +51,7 @@ fn legal_geometry( input_base + max_row * input_side + max_col < bank_lines() as u64 } -impl BallInstruction for MaxPool { +impl BallModel for MaxPool { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let input_bank = rs1_b0(xs1); let output_bank = rs1_b2(xs1); diff --git a/examples/components/balls/maxpool/stack/src/Lowering.cpp b/examples/components/balls/maxpool/stack/src/Lowering.cpp new file mode 100644 index 000000000..8afa4ca45 --- /dev/null +++ b/examples/components/balls/maxpool/stack/src/Lowering.cpp @@ -0,0 +1,87 @@ +//===- Lowering.cpp - MaxPoolBall in lower-buckyball ----------------------===// + +#include "Buckyball/BuckyballOps.h" +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +#include + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class MaxpoolI8Lowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &rewriter) const override { + if (op.getKind() != "maxpool_i8") + return failure(); + if (op.getSources().size() != 1 || + op.getSources()[0] == op.getDestination()) + return op.emitOpError( + "maxpool_i8 requires distinct input and output banks"); + int64_t inputSide = cast(op->getAttr("input_side")).getInt(); + int64_t outputSide = cast(op->getAttr("output_side")).getInt(); + int64_t kernel = cast(op->getAttr("kernel")).getInt(); + int64_t stride = cast(op->getAttr("stride")).getInt(); + int64_t padding = cast(op->getAttr("padding")).getInt(); + int64_t inputBase = cast(op->getAttr("op1_base")).getInt(); + int64_t outputBase = cast(op->getAttr("wr_base")).getInt(); + int64_t outputStride = + cast(op->getAttr("output_stride")).getInt(); + int64_t startRow = cast(op->getAttr("start_row")).getInt(); + int64_t startCol = cast(op->getAttr("start_col")).getInt(); + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || inputSide <= 0 || inputSide > 15 || + outputSide <= 0 || outputSide > 15 || kernel <= 0 || kernel > 15 || + stride <= 0 || stride > 15 || padding < 0 || padding > 15 || + inputBase < 0 || inputBase > 63 || outputBase < 0 || outputBase > 63 || + outputStride < outputSide || outputStride > 63 || startRow < 0 || + startRow > 15 || startCol < 0 || startCol > 15) + return op.emitOpError("maxpool_i8 geometry exceeds the ISA field ranges"); + int64_t lastRow = startRow + (outputSide - 1) * stride + kernel - 1; + int64_t lastCol = startCol + (outputSide - 1) * stride + kernel - 1; + int64_t inputEnd = padding + inputSide; + if (lastRow >= inputSide + 2 * padding || + lastCol >= inputSide + 2 * padding || + outputBase + (outputSide - 1) * outputStride + outputSide > + target.bankDepth) + return op.emitOpError( + "maxpool_i8 window or output bank footprint is invalid"); + if (startRow < inputEnd && startCol < inputEnd && lastRow >= padding && + lastCol >= padding && + inputBase + (std::min(lastRow, inputEnd - 1) - padding) * inputSide + + std::min(lastCol, inputEnd - 1) - padding >= + target.bankDepth) + return op.emitOpError("maxpool_i8 input bank footprint is invalid"); + uint64_t rs2 = mlir::buddy::field(inputSide, 0, 3) | + mlir::buddy::field(outputSide, 4, 7) | + mlir::buddy::field(kernel, 8, 11) | + mlir::buddy::field(stride, 12, 15) | + mlir::buddy::field(padding, 16, 19) | + mlir::buddy::field(inputBase, 20, 25) | + mlir::buddy::field(outputBase, 26, 31) | + mlir::buddy::field(outputStride, 32, 37) | + mlir::buddy::field(startRow, 38, 41) | + mlir::buddy::field(startCol, 42, 45); + mlir::buddy::emitInstruction( + rewriter, op.getLoc(), buckyball_target::getBuckyballFunct7("MAXPOOL"), + mlir::buddy::constantI64( + rewriter, op.getLoc(), + mlir::buddy::bankFields(op.getSources(), op.getDestination(), false, + outputSide * outputSide) | + (1ULL << 63)), + mlir::buddy::constantI64(rewriter, op.getLoc(), rs2)); + rewriter.eraseOp(op); + return success(); + } +}; + +void populate(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} + +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallLowering(populate), true); +} // namespace diff --git a/examples/balls/maxpool/verify/casegen/Cargo.lock b/examples/components/balls/maxpool/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/maxpool/verify/casegen/Cargo.lock rename to examples/components/balls/maxpool/verify/casegen/Cargo.lock diff --git a/examples/balls/maxpool/verify/casegen/Cargo.toml b/examples/components/balls/maxpool/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/maxpool/verify/casegen/Cargo.toml rename to examples/components/balls/maxpool/verify/casegen/Cargo.toml diff --git a/examples/components/balls/maxpool/verify/casegen/src/casegen.rs b/examples/components/balls/maxpool/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..aeccf796c --- /dev/null +++ b/examples/components/balls/maxpool/verify/casegen/src/casegen.rs @@ -0,0 +1,447 @@ +use crate::model; + +pub const MAX_INPUT_WORDS: usize = 64; +pub const MAX_DST: usize = 64; +pub const NUM_CASES: u32 = 9; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MaxPoolCmd { + pub bid: u32, + pub iter: u32, + pub op1_bank: u32, + pub wr_bank: u32, + pub op1_col: u32, + pub wr_col: u32, + pub rob_id: u32, + pub rs1_lo: u32, + pub rs1_hi: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub input_base: u32, + pub output_base: u32, + pub output_stride: u32, + pub input_side: u32, + pub output_side: u32, + pub kernel: u32, + pub stride: u32, + pub padding: u32, + pub start_row: u32, + pub start_col: u32, + pub num_input_words: u32, + pub num_dst_words: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MaxPoolCase { + pub cmd: MaxPoolCmd, + pub input_words: [u128; MAX_INPUT_WORDS], + pub input_addr: [u32; MAX_INPUT_WORDS], + pub dst_words: [u128; MAX_DST], + pub dst_addr: [u32; MAX_DST], +} + +impl MaxPoolCase { + pub fn input_lo(&self, index: usize) -> u64 { + self.input_words[index] as u64 + } + + pub fn input_hi(&self, index: usize) -> u64 { + (self.input_words[index] >> 64) as u64 + } + + pub fn dst_lo(&self, index: usize) -> u64 { + self.dst_words[index] as u64 + } + + pub fn dst_hi(&self, index: usize) -> u64 { + (self.dst_words[index] >> 64) as u64 + } +} + +#[derive(Clone, Copy)] +struct CaseSpec { + geo: model::Geo, + op1_bank: u32, + wr_bank: u32, + rob_id: u32, + ctest_input: bool, + seed: u32, +} + +fn field(val: u32, lo: u32, hi: u32) -> u64 { + let width = hi - lo + 1; + let mask = if width == 64 { + u64::MAX + } else { + (1u64 << width) - 1 + }; + (u64::from(val) & mask) << lo +} + +fn encode_rs1(input_bank: u32, output_bank: u32, iter: u32) -> u64 { + field(input_bank, 0, 9) | field(0, 10, 19) | field(output_bank, 20, 29) | field(iter, 30, 63) +} + +fn encode_rs2(geo: &model::Geo) -> u64 { + field(geo.input_side as u32, 0, 3) + | field(geo.output_side as u32, 4, 7) + | field(geo.kernel as u32, 8, 11) + | field(geo.stride as u32, 12, 15) + | field(geo.padding as u32, 16, 19) + | field(geo.input_base as u32, 20, 25) + | field(geo.output_base as u32, 26, 31) + | field(geo.output_stride as u32, 32, 37) + | field(geo.start_row as u32, 38, 41) + | field(geo.start_col as u32, 42, 45) +} + +fn split_rs(value: u64) -> (u32, u32) { + (value as u32, (value >> 32) as u32) +} + +fn validate_cmd(op1_bank: u32, wr_bank: u32, iter: u32, rs1: u64, rs2: u64) { + if iter == 0 { + panic!("maxpool: iter must be positive"); + } + if op1_bank == wr_bank { + panic!("maxpool: input and output banks must differ"); + } + if (rs1 & 0x3ff) != u64::from(op1_bank) { + panic!("maxpool: rs1 bank0 mismatch"); + } + if ((rs1 >> 10) & 0x3ff) != 0 { + panic!("maxpool: input bank 1 must be zero"); + } + if ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) { + panic!("maxpool: rs1 bank2 mismatch"); + } + if ((rs1 >> 30) & 0x3fff_ffff) != u64::from(iter) { + panic!("maxpool: rs1 iter mismatch"); + } + if (rs2 >> 46) != 0 { + panic!("maxpool: reserved rs2 bits must be zero"); + } +} + +fn ctest_tile(geo: &model::Geo) -> Vec { + let core = geo.input_side * geo.input_side; + let mut bytes = vec![0u8; core * model::BANK_ROW_BYTES]; + for position in 0..core { + for channel in 0..model::BANK_ROW_BYTES { + bytes[position * model::BANK_ROW_BYTES + channel] = + (((position * 37 + channel * 19) & 255) as i32 - 128) as u8; + } + } + bytes +} + +fn seeded_tile(geo: &model::Geo, seed: u32) -> Vec { + let core = geo.input_side * geo.input_side; + let mut bytes = vec![0u8; core * model::BANK_ROW_BYTES]; + for position in 0..core { + for channel in 0..model::BANK_ROW_BYTES { + let mix = seed + .wrapping_mul(0x9E37_79B9) + .wrapping_add(position as u32 * 131) + .wrapping_add(channel as u32 * 17); + bytes[position * model::BANK_ROW_BYTES + channel] = ((mix & 255) as i32 - 128) as u8; + } + } + bytes +} + +fn copy_words(src: &[u128]) -> [u128; N] { + if src.len() > N { + panic!("maxpool: {} words exceed buffer {N}", src.len()); + } + let mut out = [0u128; N]; + out[..src.len()].copy_from_slice(src); + out +} + +fn copy_addrs(src: &[u32]) -> [u32; N] { + if src.len() > N { + panic!("maxpool: {} addrs exceed buffer {N}", src.len()); + } + let mut out = [0u32; N]; + out[..src.len()].copy_from_slice(src); + out +} + +fn build_case(spec: CaseSpec, bid: u32) -> MaxPoolCase { + model::validate(&spec.geo); + let iter = model::iter_for(&spec.geo); + let input_bytes = if spec.ctest_input { + ctest_tile(&spec.geo) + } else { + seeded_tile(&spec.geo, spec.seed) + }; + + let mut referenced = vec![false; spec.geo.input_side * spec.geo.input_side]; + for output_y in 0..spec.geo.output_side { + for output_x in 0..spec.geo.output_side { + for kernel_y in 0..spec.geo.kernel { + for kernel_x in 0..spec.geo.kernel { + let y = output_y * spec.geo.stride + kernel_y + spec.geo.start_row; + let x = output_x * spec.geo.stride + kernel_x + spec.geo.start_col; + if y >= spec.geo.padding + && x >= spec.geo.padding + && y < spec.geo.padding + spec.geo.input_side + && x < spec.geo.padding + spec.geo.input_side + { + referenced[(y - spec.geo.padding) * spec.geo.input_side + + (x - spec.geo.padding)] = true; + } + } + } + } + } + + let mut input_addrs = Vec::new(); + let mut input_words = Vec::new(); + for y in 0..spec.geo.input_side { + for x in 0..spec.geo.input_side { + if !referenced[y * spec.geo.input_side + x] { + continue; + } + let addr = (spec.geo.input_base + y * spec.geo.input_side + x) as u32; + let row_off = (y * spec.geo.input_side + x) * model::BANK_ROW_BYTES; + let mut row = [0u8; model::BANK_ROW_BYTES]; + row.copy_from_slice(&input_bytes[row_off..row_off + model::BANK_ROW_BYTES]); + input_addrs.push(addr); + input_words.push(model::pack_row(&row)); + } + } + + let gold = model::maxpool_rows(&input_bytes, &spec.geo); + let dst_addrs: Vec = gold.iter().map(|(addr, _)| *addr).collect(); + let dst_words: Vec = gold.iter().map(|(_, row)| model::pack_row(row)).collect(); + + if dst_addrs.len() as u32 != iter { + panic!("maxpool: gold rows {} != iter {iter}", dst_addrs.len()); + } + + let rs1 = encode_rs1(spec.op1_bank, spec.wr_bank, iter); + let rs2 = encode_rs2(&spec.geo); + validate_cmd(spec.op1_bank, spec.wr_bank, iter, rs1, rs2); + + let (rs1_lo, rs1_hi) = split_rs(rs1); + let (rs2_lo, rs2_hi) = split_rs(rs2); + + MaxPoolCase { + cmd: MaxPoolCmd { + bid, + iter, + op1_bank: spec.op1_bank, + wr_bank: spec.wr_bank, + op1_col: 1, + wr_col: 1, + rob_id: spec.rob_id, + rs1_lo, + rs1_hi, + rs2_lo, + rs2_hi, + input_base: spec.geo.input_base as u32, + output_base: spec.geo.output_base as u32, + output_stride: spec.geo.output_stride as u32, + input_side: spec.geo.input_side as u32, + output_side: spec.geo.output_side as u32, + kernel: spec.geo.kernel as u32, + stride: spec.geo.stride as u32, + padding: spec.geo.padding as u32, + start_row: spec.geo.start_row as u32, + start_col: spec.geo.start_col as u32, + num_input_words: input_words.len() as u32, + num_dst_words: iter, + }, + input_words: copy_words(&input_words), + input_addr: copy_addrs::(&input_addrs), + dst_words: copy_words(&dst_words), + dst_addr: copy_addrs::(&dst_addrs), + } +} + +fn spec_for(index: u32) -> CaseSpec { + match index { + 0 => CaseSpec { + geo: model::Geo { + input_side: 6, + output_side: 3, + kernel: 2, + stride: 2, + padding: 0, + input_base: 7, + output_base: 11, + output_stride: 7, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 2, + ctest_input: true, + seed: 0, + }, + 1 => CaseSpec { + geo: model::Geo { + input_side: 4, + output_side: 3, + kernel: 2, + stride: 1, + padding: 0, + input_base: 0, + output_base: 20, + output_stride: 4, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 3, + ctest_input: false, + seed: 1, + }, + 2 => CaseSpec { + geo: model::Geo { + input_side: 4, + output_side: 3, + kernel: 2, + stride: 1, + padding: 1, + input_base: 0, + output_base: 30, + output_stride: 4, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 4, + ctest_input: false, + seed: 2, + }, + 3 => CaseSpec { + geo: model::Geo { + input_side: 6, + output_side: 3, + kernel: 2, + stride: 2, + padding: 1, + input_base: 0, + output_base: 40, + output_stride: 4, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 5, + ctest_input: false, + seed: 3, + }, + 4 => CaseSpec { + geo: model::Geo { + input_side: 5, + output_side: 3, + kernel: 3, + stride: 1, + padding: 0, + input_base: 0, + output_base: 50, + output_stride: 4, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 6, + ctest_input: false, + seed: 4, + }, + 5 => CaseSpec { + geo: model::Geo { + input_side: 7, + output_side: 3, + kernel: 3, + stride: 2, + padding: 0, + input_base: 0, + output_base: 0, + output_stride: 7, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 7, + ctest_input: false, + seed: 5, + }, + 6 => CaseSpec { + geo: model::Geo { + input_side: 5, + output_side: 3, + kernel: 3, + stride: 1, + padding: 1, + input_base: 0, + output_base: 15, + output_stride: 4, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 8, + ctest_input: false, + seed: 6, + }, + 7 => CaseSpec { + geo: model::Geo { + input_side: 7, + output_side: 3, + kernel: 3, + stride: 2, + padding: 1, + input_base: 0, + output_base: 25, + output_stride: 7, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 9, + ctest_input: false, + seed: 7, + }, + 8 => CaseSpec { + geo: model::Geo { + input_side: 7, + output_side: 1, + kernel: 1, + stride: 1, + padding: 0, + input_base: 63, + output_base: 0, + output_stride: 1, + start_row: 0, + start_col: 0, + }, + op1_bank: 0, + wr_bank: 1, + rob_id: 10, + ctest_input: false, + seed: 8, + }, + _ => panic!("maxpool: unsupported directed case {index}"), + } +} + +pub fn gen_case(index: u32, bid: u32) -> MaxPoolCase { + if index >= NUM_CASES { + panic!("maxpool: unsupported directed case {index}"); + } + build_case(spec_for(index), bid) +} diff --git a/examples/components/balls/maxpool/verify/casegen/src/lib.rs b/examples/components/balls/maxpool/verify/casegen/src/lib.rs new file mode 100644 index 000000000..024a4fe8a --- /dev/null +++ b/examples/components/balls/maxpool/verify/casegen/src/lib.rs @@ -0,0 +1,107 @@ +mod casegen; +mod model; + +use std::cell::RefCell; + +use casegen::{MaxPoolCase, MaxPoolCmd, MAX_DST, MAX_INPUT_WORDS}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn maxpool_case_load(index: u32, bid: u32) -> i32 { + let case = casegen::gen_case(index, bid); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("maxpool_case: no case loaded; call maxpool_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn maxpool_case_cmd(out_ptr: *mut MaxPoolCmd) { + if out_ptr.is_null() { + panic!("maxpool_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +fn idx(index: u32, limit: u32, what: &str) -> usize { + if index >= limit { + panic!("maxpool_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn maxpool_case_input_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_input_words, "input_word_lo"); + if i >= MAX_INPUT_WORDS { + panic!("maxpool_case_input_word_lo: word_index out of range"); + } + case.input_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn maxpool_case_input_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_input_words, "input_word_hi"); + if i >= MAX_INPUT_WORDS { + panic!("maxpool_case_input_word_hi: word_index out of range"); + } + case.input_hi(i) + }) +} + +#[no_mangle] +pub extern "C" fn maxpool_case_input_addr(word_index: u32) -> u32 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_input_words, "input_addr"); + if i >= MAX_INPUT_WORDS { + panic!("maxpool_case_input_addr: word_index out of range"); + } + case.input_addr[i] + }) +} + +#[no_mangle] +pub extern "C" fn maxpool_case_dst_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_dst_words, "dst_word_lo"); + if i >= MAX_DST { + panic!("maxpool_case_dst_word_lo: word_index out of range"); + } + case.dst_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn maxpool_case_dst_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_dst_words, "dst_word_hi"); + if i >= MAX_DST { + panic!("maxpool_case_dst_word_hi: word_index out of range"); + } + case.dst_hi(i) + }) +} + +#[no_mangle] +pub extern "C" fn maxpool_case_dst_addr(word_index: u32) -> u32 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_dst_words, "dst_addr"); + if i >= MAX_DST { + panic!("maxpool_case_dst_addr: word_index out of range"); + } + case.dst_addr[i] + }) +} diff --git a/examples/components/balls/maxpool/verify/casegen/src/model.rs b/examples/components/balls/maxpool/verify/casegen/src/model.rs new file mode 100644 index 000000000..3780a70c7 --- /dev/null +++ b/examples/components/balls/maxpool/verify/casegen/src/model.rs @@ -0,0 +1,104 @@ +pub const BANK_LINES: usize = 64; +pub const BANK_ROW_BYTES: usize = 16; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Geo { + pub input_side: usize, + pub output_side: usize, + pub kernel: usize, + pub stride: usize, + pub padding: usize, + pub input_base: usize, + pub output_base: usize, + pub output_stride: usize, + pub start_row: usize, + pub start_col: usize, +} + +pub fn validate(geo: &Geo) { + let g = geo; + let dimensions_valid = + g.input_side != 0 && g.output_side != 0 && g.kernel != 0 && g.stride != 0; + if !dimensions_valid { + panic!("maxpool: illegal square pooling geometry"); + } + let last_row = g.start_row + (g.output_side - 1) * g.stride + g.kernel - 1; + let last_col = g.start_col + (g.output_side - 1) * g.stride + g.kernel - 1; + let input_end = g.padding + g.input_side; + let reads_input = g.start_row < input_end + && g.start_col < input_end + && last_row >= g.padding + && last_col >= g.padding; + let input_fits = !reads_input + || g.input_base + + (last_row.min(input_end - 1) - g.padding) * g.input_side + + (last_col.min(input_end - 1) - g.padding) + < BANK_LINES; + if g.output_stride < g.output_side + || !input_fits + || g.output_base + (g.output_side - 1) * g.output_stride + g.output_side > BANK_LINES + || g.input_side + 2 * g.padding < g.kernel + g.start_row + || g.input_side + 2 * g.padding < g.kernel + g.start_col + || g.start_row + (g.output_side - 1) * g.stride + g.kernel > g.input_side + 2 * g.padding + || g.start_col + (g.output_side - 1) * g.stride + g.kernel > g.input_side + 2 * g.padding + { + panic!("maxpool: illegal square pooling geometry"); + } +} + +pub fn iter_for(geo: &Geo) -> u32 { + let iter = geo.output_side * geo.output_side; + if iter == 0 { + panic!("maxpool: iter must be positive"); + } + iter as u32 +} + +pub fn maxpool_rows(input: &[u8], geo: &Geo) -> Vec<(u32, [u8; BANK_ROW_BYTES])> { + validate(geo); + let core = geo.input_side * geo.input_side; + if input.len() != core * BANK_ROW_BYTES { + panic!("maxpool: input bytes {} != core {} rows", input.len(), core); + } + + let mut out = Vec::with_capacity(geo.output_side * geo.output_side); + for output_y in 0..geo.output_side { + for output_x in 0..geo.output_side { + let mut maximum = [0x80u8; BANK_ROW_BYTES]; + for kernel_y in 0..geo.kernel { + for kernel_x in 0..geo.kernel { + let input_y = output_y * geo.stride + kernel_y + geo.start_row; + let input_x = output_x * geo.stride + kernel_x + geo.start_col; + let sy = input_y as isize - geo.padding as isize; + let sx = input_x as isize - geo.padding as isize; + if sy < 0 + || sx < 0 + || sy >= geo.input_side as isize + || sx >= geo.input_side as isize + { + continue; + } + let row = (sy as usize) * geo.input_side + sx as usize; + let off = row * BANK_ROW_BYTES; + for lane in 0..BANK_ROW_BYTES { + let v = input[off + lane] as i8; + if v > maximum[lane] as i8 { + maximum[lane] = v as u8; + } + } + } + } + let addr = (geo.output_base + output_y * geo.output_stride + output_x) as u32; + out.push((addr, maximum)); + } + } + out +} + +pub fn pack_row(bytes: &[u8; BANK_ROW_BYTES]) -> u128 { + let mut word = 0u128; + for (lane, byte) in bytes.iter().enumerate() { + word |= u128::from(*byte) << (lane * 8); + } + word +} diff --git a/examples/balls/maxpool/verify/filelists/maxpool_ball.f b/examples/components/balls/maxpool/verify/filelists/maxpool_ball.f similarity index 100% rename from examples/balls/maxpool/verify/filelists/maxpool_ball.f rename to examples/components/balls/maxpool/verify/filelists/maxpool_ball.f diff --git a/examples/balls/maxpool/verify/src/common/maxpool_defs.svh b/examples/components/balls/maxpool/verify/src/common/maxpool_defs.svh similarity index 93% rename from examples/balls/maxpool/verify/src/common/maxpool_defs.svh rename to examples/components/balls/maxpool/verify/src/common/maxpool_defs.svh index a08b4a7a9..4b00baf43 100644 --- a/examples/balls/maxpool/verify/src/common/maxpool_defs.svh +++ b/examples/components/balls/maxpool/verify/src/common/maxpool_defs.svh @@ -37,10 +37,10 @@ import "DPI-C" function longint unsigned maxpool_case_dst_word_hi(input int unsi import "DPI-C" function int unsigned maxpool_case_dst_addr(input int unsigned word_index); `ifndef MAXPOOL_FUNCT7 -`error "MAXPOOL_FUNCT7 must be provided by the selected Core ballISA" +`error "MAXPOOL_FUNCT7 must be provided by the selected Thread ballISA" `endif -localparam int MAXPOOL_CORE_FUNCT7 = `MAXPOOL_FUNCT7; +localparam int MAXPOOL_THREAD_FUNCT7 = `MAXPOOL_FUNCT7; localparam int MAXPOOL_MAX_INPUT_WORDS = 64; localparam int MAXPOOL_MAX_DST_WORDS = 64; localparam int MAXPOOL_NUM_CASES = 8; diff --git a/examples/balls/maxpool/verify/src/common/maxpool_items.svh b/examples/components/balls/maxpool/verify/src/common/maxpool_items.svh similarity index 98% rename from examples/balls/maxpool/verify/src/common/maxpool_items.svh rename to examples/components/balls/maxpool/verify/src/common/maxpool_items.svh index 1722971e5..51819a9c8 100644 --- a/examples/balls/maxpool/verify/src/common/maxpool_items.svh +++ b/examples/components/balls/maxpool/verify/src/common/maxpool_items.svh @@ -19,7 +19,7 @@ class maxpool_cmd_item extends bb_blink_cmd_item; bit [31:0] dst_addr[MAXPOOL_MAX_DST_WORDS]; constraint legal_c { - funct7 == MAXPOOL_CORE_FUNCT7[6:0]; + funct7 == MAXPOOL_THREAD_FUNCT7[6:0]; op1_en == 1'b1; op2_en == 1'b0; wr_spad_en == 1'b1; @@ -73,7 +73,7 @@ class maxpool_cmd_item extends bb_blink_cmd_item; start_col = cmd.start_col; num_input_words = cmd.num_input_words; num_dst_words = cmd.num_dst_words; - funct7 = MAXPOOL_CORE_FUNCT7[6:0]; + funct7 = MAXPOOL_THREAD_FUNCT7[6:0]; op1_en = 1'b1; op2_en = 1'b0; wr_spad_en = 1'b1; diff --git a/examples/balls/maxpool/verify/src/cov/maxpool_cov.svh b/examples/components/balls/maxpool/verify/src/cov/maxpool_cov.svh similarity index 93% rename from examples/balls/maxpool/verify/src/cov/maxpool_cov.svh rename to examples/components/balls/maxpool/verify/src/cov/maxpool_cov.svh index 319389202..c1f954c72 100644 --- a/examples/balls/maxpool/verify/src/cov/maxpool_cov.svh +++ b/examples/components/balls/maxpool/verify/src/cov/maxpool_cov.svh @@ -21,7 +21,7 @@ class maxpool_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != MAXPOOL_CORE_FUNCT7[6:0]) + if (item.funct7 != MAXPOOL_THREAD_FUNCT7[6:0]) `uvm_fatal("COV", $sformatf("unexpected funct7 %0d", item.funct7)) cur_kernel = int'(item.rs2[11:8]); cur_stride = int'(item.rs2[15:12]); @@ -31,6 +31,7 @@ class maxpool_cov extends uvm_component; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") if (cmd_cg.get_coverage() != 100.0) begin diff --git a/examples/balls/maxpool/verify/src/env/maxpool_env.svh b/examples/components/balls/maxpool/verify/src/env/maxpool_env.svh similarity index 100% rename from examples/balls/maxpool/verify/src/env/maxpool_env.svh rename to examples/components/balls/maxpool/verify/src/env/maxpool_env.svh diff --git a/examples/balls/maxpool/verify/src/env/maxpool_scoreboard.svh b/examples/components/balls/maxpool/verify/src/env/maxpool_scoreboard.svh similarity index 97% rename from examples/balls/maxpool/verify/src/env/maxpool_scoreboard.svh rename to examples/components/balls/maxpool/verify/src/env/maxpool_scoreboard.svh index 487a50ae3..69bd0b791 100644 --- a/examples/balls/maxpool/verify/src/env/maxpool_scoreboard.svh +++ b/examples/components/balls/maxpool/verify/src/env/maxpool_scoreboard.svh @@ -61,6 +61,7 @@ class maxpool_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, maxpool_cmd_item exp); + verification_pkg::evidence::checks++; if (got.bid !== exp.bid) `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) if (got.funct7 !== exp.funct7) @@ -106,7 +107,7 @@ class maxpool_scoreboard extends uvm_scoreboard; if (item.bank_id !== stim.op1_bank) `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.op1_bank)) if (!input_addr_legal(stim, item.addr)) - `uvm_fatal("READ", $sformatf("addr out of input tile: got %0d", item.addr)) + `uvm_fatal("READ", $sformatf("addr out of input core: got %0d", item.addr)) read_count++; endfunction @@ -173,6 +174,7 @@ class maxpool_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) begin `uvm_fatal("SCB", $sformatf("incomplete: cmds=%0d reads=%0d writes=%0d/%0d resp=%0d", diff --git a/examples/balls/maxpool/verify/src/pkg/maxpool_pkg.sv b/examples/components/balls/maxpool/verify/src/pkg/maxpool_pkg.sv similarity index 100% rename from examples/balls/maxpool/verify/src/pkg/maxpool_pkg.sv rename to examples/components/balls/maxpool/verify/src/pkg/maxpool_pkg.sv diff --git a/examples/balls/maxpool/verify/src/seq/maxpool_sequences.svh b/examples/components/balls/maxpool/verify/src/seq/maxpool_sequences.svh similarity index 100% rename from examples/balls/maxpool/verify/src/seq/maxpool_sequences.svh rename to examples/components/balls/maxpool/verify/src/seq/maxpool_sequences.svh diff --git a/examples/balls/maxpool/verify/src/tb_top.sv b/examples/components/balls/maxpool/verify/src/tb_top.sv similarity index 100% rename from examples/balls/maxpool/verify/src/tb_top.sv rename to examples/components/balls/maxpool/verify/src/tb_top.sv diff --git a/examples/balls/maxpool/verify/src/tests/maxpool_test.svh b/examples/components/balls/maxpool/verify/src/tests/maxpool_test.svh similarity index 100% rename from examples/balls/maxpool/verify/src/tests/maxpool_test.svh rename to examples/components/balls/maxpool/verify/src/tests/maxpool_test.svh diff --git a/examples/components/balls/maxpool/workloads/ctests/CMakeLists.txt b/examples/components/balls/maxpool/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..c2cd34b14 --- /dev/null +++ b/examples/components/balls/maxpool/workloads/ctests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_buckyball_ctest(maxpool_window_test.c) +add_buckyball_ctest(maxpool_offset_test.c) diff --git a/examples/balls/maxpool/workloads/ctests/maxpool_offset_test.c b/examples/components/balls/maxpool/workloads/ctests/maxpool_offset_test.c similarity index 100% rename from examples/balls/maxpool/workloads/ctests/maxpool_offset_test.c rename to examples/components/balls/maxpool/workloads/ctests/maxpool_offset_test.c diff --git a/examples/balls/maxpool/workloads/ctests/maxpool_window_test.c b/examples/components/balls/maxpool/workloads/ctests/maxpool_window_test.c similarity index 100% rename from examples/balls/maxpool/workloads/ctests/maxpool_window_test.c rename to examples/components/balls/maxpool/workloads/ctests/maxpool_window_test.c diff --git a/examples/balls/maxpool/workloads/isa/maxpool.h b/examples/components/balls/maxpool/workloads/isa/maxpool.h similarity index 93% rename from examples/balls/maxpool/workloads/isa/maxpool.h rename to examples/components/balls/maxpool/workloads/isa/maxpool.h index a9f4477eb..d0004b701 100644 --- a/examples/balls/maxpool/workloads/isa/maxpool.h +++ b/examples/components/balls/maxpool/workloads/isa/maxpool.h @@ -9,7 +9,7 @@ start_row, start_col) \ BUCKYBALL_INSTRUCTION_R_R( \ BB_BANK0(input_bank) | BB_BANK2(output_bank) | \ - BB_ITER((output_side) * (output_side)), \ + BB_ITER((output_side) * (output_side)) | BB_UPDATE_WR, \ FIELD(input_side, 0, 3) | FIELD(output_side, 4, 7) | \ FIELD(kernel, 8, 11) | FIELD(stride, 12, 15) | \ FIELD(padding, 16, 19) | FIELD(input_base, 20, 25) | \ diff --git a/examples/balls/maxpool/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/maxpool/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/maxpool/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/maxpool/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/maxpool/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/maxpool/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/maxpool/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/maxpool/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir b/examples/components/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir new file mode 100644 index 000000000..b3c877818 --- /dev/null +++ b/examples/components/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir @@ -0,0 +1,27 @@ +func.func private @check_result(memref<9x16xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %thirty_six = arith.constant 36 : index + %sixteen = arith.constant 16 : index + %input = memref.alloc() : memref<36x16xi8> + %output = memref.alloc() : memref<9x16xi8> + scf.for %row = %zero to %thirty_six step %one { + %value = arith.index_cast %row : index to i8 + scf.for %lane = %zero to %sixteen step %one { + memref.store %value, %input[%row, %lane] : memref<36x16xi8> + } + } + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 0 : i64, depth = 36 : i64, stride = 1 : i64}> : memref<36x16xi8> + buckyball.custom "maxpool_i8" <{sources = array, destination = 1 : i64}> {input_side = 6 : i64, output_side = 3 : i64, kernel = 2 : i64, stride = 2 : i64, padding = 0 : i64, op1_base = 0 : i64, wr_base = 0 : i64, output_stride = 3 : i64, start_row = 0 : i64, start_col = 0 : i64} + buckyball.mvout %output <{bank = 1 : i64, depth = 9 : i64, stride = 1 : i64}> : memref<9x16xi8> + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + func.call @check_result(%output) : (memref<9x16xi8>) -> () + memref.dealloc %input : memref<36x16xi8> + memref.dealloc %output : memref<9x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2_main.cpp b/examples/components/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2_main.cpp similarity index 100% rename from examples/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2_main.cpp rename to examples/components/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2_main.cpp diff --git a/examples/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/components/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in b/examples/components/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in new file mode 100644 index 000000000..415ab022b --- /dev/null +++ b/examples/components/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in @@ -0,0 +1,37 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %batch_rows = arith.constant 64 : index + %lanes = arith.constant @BB_BANK_I8_LANES@ : index + %mod = arith.constant 31 : index + %bias = arith.constant 15 : index + %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + scf.for %row = %zero to %rows step %one { + %wrapped = arith.remui %row, %mod : index + %signed = arith.subi %wrapped, %bias : index + %value = arith.index_cast %signed : index to i8 + scf.for %lane = %zero to %lanes step %one { + memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.store %zero_i8, %output[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + } + } + scf.for %base = %zero to %rows step %batch_rows { + %in = memref.subview %input[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> + %out = memref.subview %output[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> + %in_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %out_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded = buckyball.bank_mvin %in, %in_bank <{depth = 64 : i64, stride = 1 : i64}> : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> + %result = buckyball.bank_custom "maxpool_i8" (%loaded) -> %out_bank {input_side = 8 : i64, output_side = 8 : i64, kernel = 3 : i64, stride = 1 : i64, padding = 1 : i64, op1_base = 0 : i64, wr_base = 0 : i64, output_stride = 8 : i64, start_row = 0 : i64, start_col = 0 : i64} + %stored = buckyball.bank_mvout %out, %result <{depth = 64 : i64, stride = 1 : i64}> : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> + buckyball.bank_release %loaded + buckyball.bank_release %stored + } + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () + memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp b/examples/components/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp similarity index 100% rename from examples/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp rename to examples/components/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp diff --git a/examples/components/balls/mxfp2int/arch/Mxfp2Int.scala b/examples/components/balls/mxfp2int/arch/Mxfp2Int.scala new file mode 100644 index 000000000..4ae592bcb --- /dev/null +++ b/examples/components/balls/mxfp2int/arch/Mxfp2Int.scala @@ -0,0 +1,309 @@ +package examples.balls.mxfp2int + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} +import framework.acc.balldomain.mmio_bank.MmioRead +import examples.balls.mxfp2int.configs.Mxfp2IntBallParam + +/** + * PipelinedMxfp2Int - MXFP to INT8 dequantizer with MMIO per-block scale. + * + * Supports MXFP4, MXFP6, MXFP8 formats (32 elements per block). + * + * Input format (main bank, per block): + * MXFP4: 32 × 4-bit = 128 bits = 1 word + * MXFP6: 32 × 6-bit = 192 bits, padded to 2 words (256 bits, high 64 bits = 0) + * MXFP8: 32 × 8-bit = 256 bits = 2 words + * + * Scale (MMIO bank): per-block E8M0 (8-bit biased exponent, bias=127). + * + * FP4/6/8 decoding (v1, simplified): signed_val = raw - (1 << (bits-1)) + * FP4: raw - 8 (range -8..+7) + * FP6: raw - 32 (range -32..+31) + * FP8: raw - 128 (range -128..+127) + * + * Output (per block, 2 SRAM words = 32 bytes): + * 32 × INT8 = saturate(signed_val << (scale - 127)). + * Lower 16 INT8 to row (2 * block_idx), upper 16 to row (2 * block_idx + 1). + * + * iter encodes the number of MXFP blocks. + */ +@instantiable +class PipelinedMxfp2Int( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: Mxfp2IntBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val ballConfig = ballParam + val decoder = MxfpDecoder.fromFormat(ballConfig.mxfpFormat) + val elemBits = decoder.elemBits + + val bankWidth = memParam.bankWidth + require( + bankWidth == 128, + s"Mxfp2IntBall requires bankWidth = 128, got $bankWidth" + ) + + val wordsPerBlock = ballConfig.mxfpFormat match { + case "MXFP4" => 1 + case "MXFP6" => + 2 // padded: 192 bits data + 64 bits padding = 256 bits = 2 words + case "MXFP8" => 2 + case _ => + throw new IllegalArgumentException( + s"Unsupported mxfpFormat: ${ballConfig.mxfpFormat}" + ) + } + + val FP_PER_BLOCK = 32 + val INT8_PER_WORD = bankWidth / 8 // 16 + + val ballMapping = instance + + val inBW = ballMapping.inBW + val outBW = ballMapping.outBW + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + val mmioRead = Flipped(new MmioRead(memParam)) + }) + + // ROB bookkeeping + val rob_id_reg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + + when(io.cmdReq.fire) { + rob_id_reg := io.cmdReq.bits.rob_id + } + + // FSM + val idle :: sReadReq :: sReadResp :: sMmioReq :: sMmioResp :: sWriteLoReq :: sWriteLoResp :: sWriteHiReq :: sWriteHiResp :: sComplete :: Nil = + Enum(10) + val state = RegInit(idle) + + // Per-instruction registers + val raddr_reg = RegInit(0.U(frontendParam.iter_len.W)) + val waddr_reg = RegInit(0.U(frontendParam.iter_len.W)) + val rbank_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + val wbank_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + val bases = Reg(Vec(3, UInt(memParam.groupIdWidth.W))) + val iter_reg = RegInit(0.U(frontendParam.iter_len.W)) + val block_idx_reg = RegInit(0.U(frontendParam.iter_len.W)) + val word_idx_reg = RegInit(0.U(2.W)) // 0..wordsPerBlock-1 + + val inputWordsReg = RegInit( + VecInit(Seq.fill(2)(0.U(bankWidth.W))) + ) // max 2 words for MXFP6/8 + val scaleReg = RegInit(0.U(8.W)) + + // Extract 32 FP elements from inputWordsReg based on format + val fpElements = VecInit(Seq.tabulate(FP_PER_BLOCK) { i => + val bitOffset = i * elemBits + val wordIdx = bitOffset / bankWidth + val bitInWord = bitOffset % bankWidth + if (wordIdx < wordsPerBlock && bitInWord + elemBits <= bankWidth) { + inputWordsReg(wordIdx)(bitInWord + elemBits - 1, bitInWord) + } else if (wordIdx < wordsPerBlock - 1 && bitInWord + elemBits > bankWidth) { + // Cross-word boundary (only for MXFP6 if packed, but we use padded so this won't happen) + val lowBits = bankWidth - bitInWord + val highBits = elemBits - lowBits + Cat( + inputWordsReg(wordIdx + 1)(highBits - 1, 0), + inputWordsReg(wordIdx)(bankWidth - 1, bitInWord) + ) + } else { + 0.U(elemBits.W) + } + }) + + // Combinationally compute all 32 INT8 values from the latched elements + scale + val int8Vec = VecInit( + fpElements.map(raw => decoder.dequant(raw, scaleReg).asUInt) + ) + + val outWordLo = Cat((0 until INT8_PER_WORD).map(i => int8Vec(i)).reverse) + + val outWordHi = Cat( + (INT8_PER_WORD until FP_PER_BLOCK).map(i => int8Vec(i)).reverse + ) + + // === Default IO === + for (i <- 0 until inBW) { + io.bankRead(i).io.req.valid := false.B + io.bankRead(i).io.req.bits.addr := 0.U + io.bankRead(i).io.resp.ready := false.B + io.bankRead(i).rob_id := rob_id_reg + io.bankRead(i).bank_id := rbank_reg + io.bankRead(i).group_id := bases(0) + } + for (i <- 0 until outBW) { + io.bankWrite(i).io.req.valid := false.B + io.bankWrite(i).io.req.bits.addr := 0.U + io.bankWrite(i).io.req.bits.data := 0.U + io.bankWrite(i).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(0.U(1.W)) + ) + io.bankWrite(i).io.resp.ready := false.B + io.bankWrite(i).rob_id := rob_id_reg + io.bankWrite(i).bank_id := wbank_reg + io.bankWrite(i).group_id := bases(2) + } + + io.mmioRead.req.valid := false.B + io.mmioRead.req.bits.addr := 0.U + io.mmioRead.resp.ready := false.B + io.cmdReq.ready := state === idle + io.cmdResp.valid := false.B + io.cmdResp.bits.rob_id := rob_id_reg + + // === FSM === + switch(state) { + is(idle) { + when(io.cmdReq.fire) { + rbank_reg := io.cmdReq.bits.op1_bank + bases := VecInit(io.cmdReq.bits.op1_group, io.cmdReq.bits.op2_group, io.cmdReq.bits.wr_group) + wbank_reg := io.cmdReq.bits.wr_bank + iter_reg := io.cmdReq.bits.iter + block_idx_reg := 0.U + word_idx_reg := 0.U + raddr_reg := 0.U + waddr_reg := 0.U + when(io.cmdReq.bits.iter === 0.U) { + state := sComplete + }.otherwise { + state := sReadReq + } + } + } + + is(sReadReq) { + io.bankRead(0).io.req.valid := true.B + io.bankRead(0) + .io + .req + .bits + .addr := raddr_reg + block_idx_reg * wordsPerBlock.U + word_idx_reg + when(io.bankRead(0).io.req.fire) { + state := sReadResp + } + } + + is(sReadResp) { + io.bankRead(0).io.resp.ready := true.B + when(io.bankRead(0).io.resp.fire) { + inputWordsReg(word_idx_reg) := io.bankRead(0).io.resp.bits.data + when(word_idx_reg === (wordsPerBlock - 1).U) { + word_idx_reg := 0.U + state := sMmioReq + }.otherwise { + word_idx_reg := word_idx_reg + 1.U + state := sReadReq + } + } + } + + is(sMmioReq) { + io.mmioRead.req.valid := true.B + io.mmioRead.req.bits.addr := block_idx_reg + when(io.mmioRead.req.fire) { + state := sMmioResp + } + } + + is(sMmioResp) { + io.mmioRead.resp.ready := true.B + when(io.mmioRead.resp.fire) { + scaleReg := io.mmioRead.resp.bits.data(7, 0) + state := sWriteLoReq + } + } + + is(sWriteLoReq) { + io.bankWrite(0).io.req.valid := true.B + io.bankWrite(0).io.req.bits.addr := waddr_reg + (block_idx_reg << 1) + io.bankWrite(0).io.req.bits.data := outWordLo + io.bankWrite(0).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(1.U(1.W)) + ) + when(io.bankWrite(0).io.req.fire) { + state := sWriteLoResp + } + } + + is(sWriteLoResp) { + io.bankWrite(0).io.resp.ready := true.B + when(io.bankWrite(0).io.resp.fire) { + state := sWriteHiReq + } + } + + is(sWriteHiReq) { + io.bankWrite(0).io.req.valid := true.B + io.bankWrite(0).io.req.bits.addr := waddr_reg + (block_idx_reg << 1) + 1.U + io.bankWrite(0).io.req.bits.data := outWordHi + io.bankWrite(0).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(1.U(1.W)) + ) + when(io.bankWrite(0).io.req.fire) { + state := sWriteHiResp + } + } + + is(sWriteHiResp) { + io.bankWrite(0).io.resp.ready := true.B + when(io.bankWrite(0).io.resp.fire) { + when(block_idx_reg === iter_reg - 1.U) { + state := sComplete + }.otherwise { + block_idx_reg := block_idx_reg + 1.U + state := sReadReq + } + } + } + + is(sComplete) { + io.cmdResp.valid := true.B + when(io.cmdResp.fire) { + state := idle + } + } + } + + io.status.idle := state === idle + io.status.running := state =/= idle +} diff --git a/examples/components/balls/mxfp2int/arch/Mxfp2IntBall.scala b/examples/components/balls/mxfp2int/arch/Mxfp2IntBall.scala new file mode 100644 index 000000000..0728eb398 --- /dev/null +++ b/examples/components/balls/mxfp2int/arch/Mxfp2IntBall.scala @@ -0,0 +1,75 @@ +package examples.balls.mxfp2int + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.mxfp2int.{Params => ParametersProto} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.mxfp2int.configs.Mxfp2IntBallParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} + +import framework.acc.balldomain.ball.blink.{BlinkIO, HasBlink} + +/** + * Mxfp2IntBall - MXFP4 to INT8 dequantizer. + * + * Wraps PipelinedMxfp2Int and connects to the Blink protocol, + * including the MMIO read channel for per-block E8M0 scales. + */ +@instantiable +class Mxfp2IntBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + val ballParam = Mxfp2IntBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + val ballCommonConfig = ball + + val inBW = ballCommonConfig.inBW + val outBW = ballCommonConfig.outBW + + require(ballCommonConfig.mmioReadBW == 1 && ballCommonConfig.mmioWriteBW == 0) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val mxfp2intUnit: Instance[PipelinedMxfp2Int] = Instantiate( + new PipelinedMxfp2Int(fp, mp, bp, ball, ballParam) + ) + + mxfp2intUnit.io.cmdReq <> blink.cmdReq + mxfp2intUnit.io.cmdResp <> blink.cmdResp + + for (i <- 0 until inBW) { + mxfp2intUnit.io.bankRead(i) <> blink.bankRead(i) + } + + for (i <- 0 until outBW) { + mxfp2intUnit.io.bankWrite(i) <> blink.bankWrite(i) + } + + blink.status <> mxfp2intUnit.io.status + + blink.mmioRead(0) <> mxfp2intUnit.io.mmioRead +} + +object Mxfp2IntBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new Mxfp2IntBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/balls/mxfp2int/arch/src/main/scala/Mxfp4Decoder.scala b/examples/components/balls/mxfp2int/arch/Mxfp4Decoder.scala similarity index 100% rename from examples/balls/mxfp2int/arch/src/main/scala/Mxfp4Decoder.scala rename to examples/components/balls/mxfp2int/arch/Mxfp4Decoder.scala diff --git a/examples/balls/mxfp2int/arch/src/main/scala/Mxfp6Decoder.scala b/examples/components/balls/mxfp2int/arch/Mxfp6Decoder.scala similarity index 100% rename from examples/balls/mxfp2int/arch/src/main/scala/Mxfp6Decoder.scala rename to examples/components/balls/mxfp2int/arch/Mxfp6Decoder.scala diff --git a/examples/balls/mxfp2int/arch/src/main/scala/Mxfp8Decoder.scala b/examples/components/balls/mxfp2int/arch/Mxfp8Decoder.scala similarity index 100% rename from examples/balls/mxfp2int/arch/src/main/scala/Mxfp8Decoder.scala rename to examples/components/balls/mxfp2int/arch/Mxfp8Decoder.scala diff --git a/examples/balls/mxfp2int/arch/src/main/scala/MxfpDecoder.scala b/examples/components/balls/mxfp2int/arch/MxfpDecoder.scala similarity index 100% rename from examples/balls/mxfp2int/arch/src/main/scala/MxfpDecoder.scala rename to examples/components/balls/mxfp2int/arch/MxfpDecoder.scala diff --git a/examples/components/balls/mxfp2int/configs/Mxfp2IntBallParam.scala b/examples/components/balls/mxfp2int/configs/Mxfp2IntBallParam.scala new file mode 100644 index 000000000..867aa5e22 --- /dev/null +++ b/examples/components/balls/mxfp2int/configs/Mxfp2IntBallParam.scala @@ -0,0 +1,17 @@ +package examples.balls.mxfp2int.configs + +import buckyball.balls.mxfp2int.{Params => ParametersProto} +import upickle.default.{macroRW, ReadWriter} + +case class Mxfp2IntBallParam( + mxfpFormat: String) + +object Mxfp2IntBallParam { + + def fromProto(source: ParametersProto): Mxfp2IntBallParam = { + require(source.hasMxfpFormat) + Mxfp2IntBallParam(source.getMxfpFormat) + } + + implicit val rw: ReadWriter[Mxfp2IntBallParam] = macroRW +} diff --git a/examples/components/balls/mxfp2int/configs/default.toml b/examples/components/balls/mxfp2int/configs/default.toml new file mode 100644 index 000000000..77ef4658d --- /dev/null +++ b/examples/components/balls/mxfp2int/configs/default.toml @@ -0,0 +1,3 @@ +# Mxfp2Int Ball Configuration + +mxfpFormat = "MXFP4" diff --git a/examples/components/balls/mxfp2int/configs/mxfp2int_params.proto b/examples/components/balls/mxfp2int/configs/mxfp2int_params.proto new file mode 100644 index 000000000..1e78606da --- /dev/null +++ b/examples/components/balls/mxfp2int/configs/mxfp2int_params.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; +package buckyball.balls.mxfp2int; +option java_package = "buckyball.balls.mxfp2int"; +option java_multiple_files = true; + +message Params { + optional string mxfpFormat = 1; +} diff --git a/examples/balls/mxfp2int/emu/src/55_mxfp2int.rs b/examples/components/balls/mxfp2int/emu/src/55_mxfp2int.rs similarity index 96% rename from examples/balls/mxfp2int/emu/src/55_mxfp2int.rs rename to examples/components/balls/mxfp2int/emu/src/55_mxfp2int.rs index d7e5ecfa4..30744d9b8 100644 --- a/examples/balls/mxfp2int/emu/src/55_mxfp2int.rs +++ b/examples/components/balls/mxfp2int/emu/src/55_mxfp2int.rs @@ -11,13 +11,13 @@ // //===-----------------------------------------------------------------===// -use super::super::bank::{bank_width, mmio_read_byte}; +use bebop_bemu::thread::banks::{bank_width, mmio_read_byte}; use super::decode::{rs1_b0, rs1_b2, rs1_iter}; -use super::instruction::{BallInstruction, ExecContext}; +use super::instruction::{BallModel, ExecContext}; pub struct Mxfp2Int; -impl BallInstruction for Mxfp2Int { +impl BallModel for Mxfp2Int { fn exec(xs1: u64, _xs2: u64, ctx: &mut ExecContext) -> u64 { let in_bank = rs1_b0(xs1) as u32; let out_bank = rs1_b2(xs1) as u32; diff --git a/examples/components/balls/mxfp2int/emu/src/dispatch.rs b/examples/components/balls/mxfp2int/emu/src/dispatch.rs new file mode 100644 index 000000000..672066ef9 --- /dev/null +++ b/examples/components/balls/mxfp2int/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::f55_mxfp2int; + +const BALL_CLASS: &str = "examples.balls.mxfp2int.Mxfp2IntBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MXFP2INT") => Some(f55_mxfp2int::Mxfp2Int::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MXFP2INT") => Some(f55_mxfp2int::Mxfp2Int::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/components/balls/mxfp2int/emu/src/lib.rs b/examples/components/balls/mxfp2int/emu/src/lib.rs new file mode 100644 index 000000000..fd1b1d219 --- /dev/null +++ b/examples/components/balls/mxfp2int/emu/src/lib.rs @@ -0,0 +1,12 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +#[path = "55_mxfp2int.rs"] +mod f55_mxfp2int; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/components/balls/mxfp2int/workloads/ctests/CMakeLists.txt b/examples/components/balls/mxfp2int/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..211320815 --- /dev/null +++ b/examples/components/balls/mxfp2int/workloads/ctests/CMakeLists.txt @@ -0,0 +1 @@ +add_buckyball_ctest(mxfp4_e8m0_two_block_test.c) diff --git a/examples/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c b/examples/components/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c similarity index 98% rename from examples/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c rename to examples/components/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c index c94f9199c..708c9b18a 100644 --- a/examples/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c +++ b/examples/components/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c @@ -33,8 +33,8 @@ static int8_t golden(int block, int elem) { } int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif for (int b = 0; b < NUM_BLOCKS; b++) { diff --git a/examples/balls/mxfp2int/workloads/isa/mxfp2int.h b/examples/components/balls/mxfp2int/workloads/isa/mxfp2int.h similarity index 100% rename from examples/balls/mxfp2int/workloads/isa/mxfp2int.h rename to examples/components/balls/mxfp2int/workloads/isa/mxfp2int.h diff --git a/examples/balls/mxfp2int/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/mxfp2int/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/mxfp2int/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/mxfp2int/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks.mlir b/examples/components/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks.mlir similarity index 100% rename from examples/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks.mlir rename to examples/components/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks.mlir diff --git a/examples/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks_main.cpp b/examples/components/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks_main.cpp similarity index 100% rename from examples/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks_main.cpp rename to examples/components/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks_main.cpp diff --git a/examples/components/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..0abb14ed4 --- /dev/null +++ b/examples/components/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,16 @@ +if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +endif() +set(BUCKYBALL_MLIR_TEST_PREFIX bank) +set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") +file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") +string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") +math(EXPR _remainder "${_lines} % 32") +if(_lines LESS 32 OR NOT (_remainder EQUAL 0)) + message(FATAL_ERROR + "Mxfp2Int BankOp test requires BANK_LINES divisible by 32 " + "(got lines=${_lines})") +endif() +math(EXPR BB_MXFP_BLOCKS "${_lines} / 2") +math(EXPR BB_MXFP_MMIO_ROWS "${BB_MXFP_BLOCKS} / 16") +add_buckyball_mlir_test(mxfp4_unit_scale_bank TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank.mlir.in b/examples/components/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank.mlir.in similarity index 100% rename from examples/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank.mlir.in rename to examples/components/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank.mlir.in diff --git a/examples/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank_main.cpp b/examples/components/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank_main.cpp similarity index 100% rename from examples/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank_main.cpp rename to examples/components/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank_main.cpp diff --git a/examples/balls/mxmm/arch/src/main/scala/AddPipe.scala b/examples/components/balls/mxmm/arch/AddPipe.scala similarity index 87% rename from examples/balls/mxmm/arch/src/main/scala/AddPipe.scala rename to examples/components/balls/mxmm/arch/AddPipe.scala index d49ffea94..83ac251f4 100644 --- a/examples/balls/mxmm/arch/src/main/scala/AddPipe.scala +++ b/examples/components/balls/mxmm/arch/AddPipe.scala @@ -1,5 +1,7 @@ package examples.balls.mxmm +import chisel3.experimental.hierarchy.{Instance, Instantiate} + import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public} @@ -20,7 +22,7 @@ class AddPipe extends Module { val decodedA = RegEnable(rawFloatFromRecFN(8, 24, io.a), io.valid) val decodedB = RegEnable(rawFloatFromRecFN(8, 24, io.b), io.valid) val decodedValid = RegNext(io.valid, false.B) - val add = Module(new AddRawFN(8, 24)) + val add = Instantiate(new AddRawFN(8, 24)) add.io.subOp := false.B add.io.a := decodedA add.io.b := decodedB @@ -29,7 +31,7 @@ class AddPipe extends Module { val sum = RegEnable(add.io.rawOut, decodedValid) val invalid = RegEnable(add.io.invalidExc, decodedValid) val sumValid = RegNext(decodedValid, false.B) - val round = Module(new RoundRawFNToRecFN(8, 24, 0)) + val round = Instantiate(new RoundRawFNToRecFN(8, 24, 0)) round.io.in := sum round.io.invalidExc := invalid round.io.infiniteExc := false.B diff --git a/examples/components/balls/mxmm/arch/Array.scala b/examples/components/balls/mxmm/arch/Array.scala new file mode 100644 index 000000000..8e9f9f8e7 --- /dev/null +++ b/examples/components/balls/mxmm/arch/Array.scala @@ -0,0 +1,53 @@ +package examples.balls.mxmm + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import examples.balls.mxmm.configs.MxmmBallParam +import hardfloat._ + +@instantiable +class Array(val ballParam: MxmmBallParam) extends Module { + val p = ballParam + + @public + val io = IO(new Bundle { + val valid = Input(Bool()) + val context = Input(UInt(log2Ceil(p.contexts).W)) + val separate = Input(Bool()) + val vector = Input(Bool()) + val a = Input(Vec(p.tileRows, UInt(32.W))) + val b = Input(Vec(p.tileCols, UInt(32.W))) + val load = Input(Bool()) + val row = Input(UInt(4.W)) + val rowData = Input(UInt(512.W)) + val rowOut = Output(UInt(512.W)) + val completed = Output(Bool()) + }) + + val accumulators = Reg(Vec(p.contexts, Vec(p.tileRows, Vec(p.tileCols, UInt(33.W))))) + when(io.load) { + for (col <- 0 until p.tileCols) { + accumulators(io.context)(io.row)(col) := recFNFromFN(8, 24, io.rowData(32 * col + 31, 32 * col)) + } + } + io.rowOut := Cat((0 until p.tileCols).reverse.map(col => fNFromRecFN(8, 24, accumulators(io.context)(io.row)(col)))) + val fusedTag = Pipe(io.valid && !io.separate, io.context, p.arithmeticLatency) + val separateTag = Pipe(io.valid && io.separate, io.context, 2 * p.arithmeticLatency) + io.completed := fusedTag.valid || separateTag.valid + for { + row <- 0 until p.tileRows + col <- 0 until p.tileCols + } { + val pe = + Instantiate(new PE(p.arithmeticLatency, ballParam)) + pe.io.valid := io.valid && (if (row == 0) true.B else !io.vector) + pe.io.separate := io.separate + pe.io.a := io.a(row) + pe.io.b := io.b(col) + pe.io.c := accumulators(io.context)(row)(col) + when(pe.io.outValid) { + accumulators(Mux(separateTag.valid, separateTag.bits, fusedTag.bits))(row)(col) := pe.io.out + } + } +} diff --git a/examples/components/balls/mxmm/arch/InputWindow.scala b/examples/components/balls/mxmm/arch/InputWindow.scala new file mode 100644 index 000000000..3e2435026 --- /dev/null +++ b/examples/components/balls/mxmm/arch/InputWindow.scala @@ -0,0 +1,91 @@ +package examples.balls.mxmm + +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.mxmm.configs.MxmmBallParam + +import framework.ip.memory.bank.{SramReadIO, SramReadReq, SramReadResp, SramWriteIO, SramWriteReq, SramWriteResp} + +import chisel3._ +import chisel3.util._ +import framework.ip.memory.bank.{SramReadReq, SramReadResp} + +@instantiable +class InputWindow(memParam: MemDomainParam, val ballParam: MxmmBallParam) extends Module { + val addressWidth = log2Ceil(memParam.bankEntries) + + @public + val io = IO(new Bundle { + val start = Input(Bool()) + val enable = Input(Bool()) + val rows = Input(UInt(12.W)) + val count = Input(UInt(34.W)) + val fullK = Input(UInt(16.W)) + val startK = Input(UInt(16.W)) + val request = Decoupled(new SramReadReq(memParam)) + val response = Flipped(Decoupled(new SramReadResp(memParam))) + val done = Output(Bool()) + val write = Output(Bool()) + val scaleWrite = Output(Bool()) + val lane = Output(UInt(4.W)) + val address = Output(UInt(addressWidth.W)) + val word = Output(UInt(128.W)) + val scale = Output(UInt(8.W)) + }) + + val request :: response :: scales :: done :: Nil = Enum(4) + val state = RegInit(done) + val rows = Reg(UInt(12.W)) + val count = Reg(UInt(34.W)) + val fullK = Reg(UInt(16.W)) + val startK = Reg(UInt(16.W)) + val row = Reg(UInt(12.W)) + val offset = Reg(UInt(addressWidth.W)) + val scaleMode = Reg(Bool()) + val word = Reg(UInt(128.W)) + val scaleAddress = rows * fullK + row * (fullK >> 5) + (startK >> 5) + offset + val byte = scaleAddress(3, 0) + io.request.valid := io.enable && state === request + io.request.bits.addr := Mux(scaleMode, scaleAddress >> 4, row * (fullK >> 4) + (startK >> 4) + offset) + io.response.ready := io.enable && state === response + io.done := state === done + io.write := io.response.fire && !scaleMode + io.scaleWrite := io.enable && state === scales + io.lane := row(3, 0) + io.address := (row >> 4) * Mux(scaleMode, count >> 5, count >> 4) + offset + io.word := io.response.bits.data + io.scale := (word >> (byte << 3))(7, 0) + when(io.start) { + rows := io.rows + count := io.count + fullK := io.fullK + startK := io.startK + row := 0.U + offset := 0.U + scaleMode := false.B + state := request + } + when(io.request.fire)(state := response) + when(io.response.fire) { + when(scaleMode) { word := io.response.bits.data; state := scales }.otherwise { + when(offset + 1.U === (count >> 4)) { + offset := 0.U + when(row + 1.U === rows) { row := 0.U; scaleMode := true.B } + .otherwise(row := row + 1.U) + }.otherwise(offset := offset + 1.U) + state := request + } + } + when(io.enable && state === scales) { + when(offset + 1.U === (count >> 5)) { + offset := 0.U + when(row + 1.U === rows)(state := done) + .otherwise { row := row + 1.U; state := request } + }.otherwise { + offset := offset + 1.U + when(byte === 15.U)(state := request) + } + } +} diff --git a/examples/balls/mxmm/arch/src/main/scala/Mxfp8Decode.scala b/examples/components/balls/mxmm/arch/Mxfp8Decode.scala similarity index 100% rename from examples/balls/mxmm/arch/src/main/scala/Mxfp8Decode.scala rename to examples/components/balls/mxmm/arch/Mxfp8Decode.scala diff --git a/examples/components/balls/mxmm/arch/MxmmBall.scala b/examples/components/balls/mxmm/arch/MxmmBall.scala new file mode 100644 index 000000000..758eaab41 --- /dev/null +++ b/examples/components/balls/mxmm/arch/MxmmBall.scala @@ -0,0 +1,49 @@ +package examples.balls.mxmm + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.mxmm.{Params => ParametersProto} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.mxmm.configs.MxmmBallParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class MxmmBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + val ballParam = MxmmBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val unit = + Instantiate(new Unit(fp, mp, bp, ball, ballParam)) + unit.io.cmdReq <> blink.cmdReq + unit.io.cmdResp <> blink.cmdResp + unit.io.bankRead <> blink.bankRead + unit.io.bankWrite <> blink.bankWrite + blink.status <> unit.io.status +} + +object MxmmBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new MxmmBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/components/balls/mxmm/arch/PE.scala b/examples/components/balls/mxmm/arch/PE.scala new file mode 100644 index 000000000..48c500369 --- /dev/null +++ b/examples/components/balls/mxmm/arch/PE.scala @@ -0,0 +1,52 @@ +package examples.balls.mxmm + +import examples.balls.mxmm.configs.MxmmBallParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.lib.arith.Fma +import hardfloat._ + +@instantiable +class PE(val latency: Int, val ballParam: MxmmBallParam) extends Module { + + require(latency == 3) + + @public + val io = IO(new Bundle { + val valid = Input(Bool()) + val separate = Input(Bool()) + val a = Input(UInt(32.W)) + val b = Input(UInt(32.W)) + val c = Input(UInt(33.W)) + val out = Output(UInt(33.W)) + val outValid = Output(Bool()) + }) + + val multiply = Instantiate(new Fma(latency, 8, 24)) + + multiply.io.validin := io.valid + multiply.io.op := 0.U + multiply.io.a := recFNFromFN(8, 24, io.a) + multiply.io.b := recFNFromFN(8, 24, io.b) + multiply.io.c := Mux(io.separate, Cat(io.a(31) ^ io.b(31), 0.U(32.W)), io.c) + multiply.io.roundingMode := consts.round_near_even + multiply.io.detectTininess := consts.tininess_afterRounding.asUInt + + val productMode = Pipe(io.valid, io.separate, latency) + val previous = Pipe(io.valid && io.separate, io.c, latency) + val add = Instantiate(new AddPipe) + add.io.valid := multiply.io.validout && productMode.bits + add.io.a := multiply.io.out + add.io.b := previous.bits + when(add.io.valid) { + assert(fNFromRecFN(8, 24, multiply.io.out)(30, 23) =/= 255.U, "MXMM_F32 product must remain finite") + } + val fusedValid = multiply.io.validout && !productMode.bits + io.outValid := fusedValid || add.io.outValid + io.out := Mux(add.io.outValid, add.io.out, multiply.io.out) + when(io.outValid) { + assert(fNFromRecFN(8, 24, io.out)(30, 23) =/= 255.U, "MATMUL accumulation must remain finite") + } +} diff --git a/examples/balls/mxmm/arch/src/main/scala/Panels.scala b/examples/components/balls/mxmm/arch/Panels.scala similarity index 95% rename from examples/balls/mxmm/arch/src/main/scala/Panels.scala rename to examples/components/balls/mxmm/arch/Panels.scala index 49e4a168c..fb09f862c 100644 --- a/examples/balls/mxmm/arch/src/main/scala/Panels.scala +++ b/examples/components/balls/mxmm/arch/Panels.scala @@ -1,11 +1,13 @@ package examples.balls.mxmm +import examples.balls.mxmm.configs.MxmmBallParam + import chisel3._ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} @instantiable -class Panels(bankEntries: Int) extends Module { +class Panels(val bankEntries: Int, val ballParam: MxmmBallParam) extends Module { @public val io = IO(new Bundle { diff --git a/examples/components/balls/mxmm/arch/Unit.scala b/examples/components/balls/mxmm/arch/Unit.scala new file mode 100644 index 000000000..9c8d74b7b --- /dev/null +++ b/examples/components/balls/mxmm/arch/Unit.scala @@ -0,0 +1,411 @@ +package examples.balls.mxmm + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} +import examples.balls.mxmm.configs.MxmmBallParam + +@instantiable +class Unit( + frontendParam: FrontendParam, + memParam: MemDomainParam, + routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: MxmmBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val p = ballParam + val entries = memParam.bankEntries + val addressWidth = log2Ceil(entries) + require( + memParam.bankWidth == 128 && memParam.bankMaskLen == 16 + ) + require(entries >= 64 && entries % 32 == 0) + val mapping = instance + val mxFunct = instance.instructions.find(_.mnemonic == "MXMM_MXFP8").get.funct7 + val windowFunct = instance.instructions.find(_.mnemonic == "MXMM_MXFP8_WINDOW").get.funct7 + val fmaFunct = instance.instructions.find(_.mnemonic == "MXMM_FMA32").get.funct7 + val f32Funct = instance.instructions.find(_.mnemonic == "MXMM_F32").get.funct7 + require(mapping.inBW == 2 && mapping.outBW == 1) + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + 2, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + 1, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val Seq( + idle, + waitChannels, + loadInputs, + initRequest, + initResponse, + compute, + drain, + store, + outputRequest, + outputResponse, + outputWrite, + outputAck, + complete + ) = Enum(13) + + val state = RegInit(idle) + + val command = Reg(new BallCommand(frontendParam, memParam)) + + val m = Reg(UInt(12.W)) + val n = Reg(UInt(12.W)) + val k = Reg(UInt(34.W)) + val mx = Reg(Bool()) + val windowed = Reg(Bool()) + val separate = Reg(Bool()) + val first = Reg(Bool()) + val last = Reg(Bool()) + val base = Reg(UInt(6.W)) + val chain = RegInit(false.B) + val chainM = Reg(UInt(12.W)) + val chainN = Reg(UInt(12.W)) + val chainFormat = Reg(UInt(7.W)) + val chainBank = Reg(UInt(frontendParam.bank_id_len.W)) + val chainBase = Reg(UInt(6.W)) + + val nGroups = n >> 4 + val panelHeight = Mux(m === 1.U, 1.U, 16.U) + val totalPanels = Mux(m === 1.U, 1.U, m >> 4) * nGroups + val batch = Reg(UInt(addressWidth.W)) + val count = Mux(totalPanels - batch > p.contexts.U, p.contexts.U, totalPanels - batch) + val minimumPeriod = Mux(separate, (2 * p.arithmeticLatency + 1).U, (p.arithmeticLatency + 1).U) + val period = Mux(count > minimumPeriod, count, minimumPeriod) + val context = Reg(UInt(log2Ceil(p.contexts).W)) + val row = Reg(UInt(4.W)) + val reduction = Reg(UInt(34.W)) + val completed = Reg(UInt(48.W)) + val panel = batch + context + val rowGroup = panel / nGroups + val columnGroup = panel % nGroups + val accAddress = rowGroup * 16.U * nGroups + row * nGroups + columnGroup + val accumulator = SyncReadMem(entries / 4, UInt(512.W)) + val savedRow = Reg(UInt(512.W)) + val outputLine = Reg(UInt(addressWidth.W)) + val outputQuarter = Reg(UInt(2.W)) + + val panels = + Instantiate(new Panels(entries, ballParam)) + + val array = Instantiate(new Array(ballParam)) + val readValid = state === compute && context < count + panels.io.read := readValid + panels.io.rowGroup := rowGroup + panels.io.columnGroup := columnGroup + panels.io.k := reduction + panels.io.reduction := k + panels.io.mxfp8 := mx + panels.io.vector := m === 1.U + array.io.valid := panels.io.valid + array.io.a := panels.io.a + array.io.b := panels.io.b + array.io.separate := separate + array.io.vector := m === 1.U + val computeContext = RegNext(context) + array.io.context := Mux(state === compute || state === drain, computeContext, context) + array.io.row := row + array.io.load := false.B + array.io.rowData := 0.U + + val accReadEnable = state === outputRequest || (state === initRequest && !first && row < panelHeight) + val accWriteEnable = state === store && row < panelHeight + + val accRead = accumulator.readWrite( + Mux(state === outputRequest, outputLine, accAddress), + array.io.rowOut, + accReadEnable || accWriteEnable, + accWriteEnable + ) + + assert(!(accReadEnable && accWriteEnable)) + + val inputState = Reg(Vec(2, UInt(2.W))) + val inputWord = Reg(Vec(2, UInt(128.W))) + val inputIndex = Reg(Vec(2, UInt(addressWidth.W))) + val inputRow = Reg(Vec(2, UInt(12.W))) + val rowOffset = Reg(Vec(2, UInt(addressWidth.W))) + val scaleMode = Reg(Vec(2, Bool())) + val scaleByte = Reg(Vec(2, UInt(4.W))) + val scaleCount = Reg(Vec(2, UInt((addressWidth + 4).W))) + val rowWords = Mux(mx, k >> 4, k >> 2) + val window = Instantiate(new InputWindow(memParam, ballParam)) + window.io.start := io.cmdReq.fire && io.cmdReq.bits.funct === windowFunct.U + window.io.enable := state === loadInputs && windowed + window.io.rows := io.cmdReq.bits.rs2(11, 0) + window.io.count := io.cmdReq.bits.iter + window.io.fullK := io.cmdReq.bits.rs2(47, 32) + window.io.startK := io.cmdReq.bits.rs2(63, 48) + window.io.request.ready := io.bankRead(0).io.req.ready && windowed + window.io.response.valid := io.bankRead(0).io.resp.valid && windowed + window.io.response.bits := io.bankRead(0).io.resp.bits + + io.cmdReq.ready := state === idle + io.cmdResp.valid := state === complete + io.cmdResp.bits.rob_id := command.rob_id + io.status.idle := state === idle + io.status.running := state =/= idle && state =/= complete + + for (operand <- 0 until 2) { + val rows = if (operand == 0) m else n + val codeWords = rows * rowWords + val scales = rows * (k >> 5) + val port = io.bankRead(operand) + port.rob_id := command.rob_id + port.group_id := (if (operand == 0) command.op1_group else command.op2_group) + port.bank_id := (if (operand == 0) command.op1_bank else command.op2_bank) + port.io.req.valid := state === loadInputs && inputState(operand) === 0.U + port.io.req.bits.addr := inputIndex(operand) + port.io.resp.ready := state === loadInputs && inputState(operand) === 1.U + panels.io.write(operand) := port.io.resp.fire && !scaleMode(operand) + panels.io.scaleWrite(operand) := state === loadInputs && inputState(operand) === 2.U + panels.io.lane(operand) := inputRow(operand)(3, 0) + panels.io.address(operand) := (inputRow(operand) >> 4) * Mux(scaleMode(operand), k >> 5, rowWords) + rowOffset( + operand + ) + panels.io.word(operand) := port.io.resp.bits.data + panels.io.scale(operand) := (inputWord(operand) >> (scaleByte(operand) << 3))(7, 0) + val legacy = if (operand == 0) !windowed else true.B + when(port.io.req.fire && legacy)(inputState(operand) := 1.U) + when(port.io.resp.fire && legacy) { + when(scaleMode(operand)) { + inputWord(operand) := port.io.resp.bits.data + scaleByte(operand) := 0.U + inputState(operand) := 2.U + }.otherwise { + inputIndex(operand) := inputIndex(operand) + 1.U + when(rowOffset(operand) + 1.U === rowWords) { + rowOffset(operand) := 0.U + inputRow(operand) := inputRow(operand) + 1.U + }.otherwise(rowOffset(operand) := rowOffset(operand) + 1.U) + when(inputIndex(operand) +& 1.U === codeWords) { + when(mx) { + scaleMode(operand) := true.B + inputRow(operand) := 0.U + rowOffset(operand) := 0.U + inputState(operand) := 0.U + }.otherwise(inputState(operand) := 3.U) + }.otherwise(inputState(operand) := 0.U) + } + } + when(state === loadInputs && inputState(operand) === 2.U && legacy) { + scaleCount(operand) := scaleCount(operand) + 1.U + scaleByte(operand) := scaleByte(operand) + 1.U + when(rowOffset(operand) + 1.U === (k >> 5)) { + rowOffset(operand) := 0.U + inputRow(operand) := inputRow(operand) + 1.U + }.otherwise(rowOffset(operand) := rowOffset(operand) + 1.U) + when(scaleCount(operand) + 1.U === scales) { + inputState(operand) := 3.U + }.elsewhen(scaleByte(operand) === 15.U) { + inputIndex(operand) := inputIndex(operand) + 1.U + inputState(operand) := 0.U + } + } + } + + when(windowed) { + io.bankRead(0).io.req.valid := window.io.request.valid + io.bankRead(0).io.req.bits.addr := window.io.request.bits.addr + io.bankRead(0).io.resp.ready := window.io.response.ready + panels.io.write(0) := window.io.write + panels.io.scaleWrite(0) := window.io.scaleWrite + panels.io.lane(0) := window.io.lane + panels.io.address(0) := window.io.address + panels.io.word(0) := window.io.word + panels.io.scale(0) := window.io.scale + } + + val writePort = io.bankWrite(0) + writePort.rob_id := command.rob_id + writePort.group_id := command.wr_group + writePort.bank_id := command.wr_bank + writePort.io.req.valid := state === outputWrite + writePort.io.req.bits.addr := base + (outputLine << 2) + outputQuarter + writePort.io.req.bits.data := (savedRow >> (outputQuarter << 7))(127, 0) + writePort.io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) + writePort.io.resp.ready := state === outputAck + + when(io.cmdReq.fire) { + val cmd = io.cmdReq.bits + val rows = cmd.rs2(11, 0) + val cols = cmd.rs2(23, 12) + val inputK = cmd.iter + val isWindow = cmd.funct === windowFunct.U + val isMx = cmd.funct === mxFunct.U || isWindow + assert( + isMx || cmd.funct === fmaFunct.U || cmd.funct === f32Funct.U, + "MATMUL format is invalid" + ) + assert(rows === 1.U || (rows =/= 0.U && rows(3, 0) === 0.U), "MATMUL M must be one or a positive multiple of 16") + assert(cols =/= 0.U && cols(3, 0) === 0.U, "MATMUL N must be a positive multiple of 16") + assert(inputK =/= 0.U && Mux(isMx, inputK(4, 0) === 0.U, inputK(1, 0) === 0.U), "MATMUL K alignment is invalid") + when(isWindow) { + val full = cmd.rs2(47, 32) + val start = cmd.rs2(63, 48) + assert(full =/= 0.U && full(4, 0) === 0.U && start(4, 0) === 0.U, "MATMUL A window alignment is invalid") + assert(start +& inputK <= full, "MATMUL A window exceeds fullK") + }.otherwise(assert(cmd.rs2(63, 32) === 0.U, "MATMUL reserves rs2[63:32]")) + assert( + cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, + "MATMUL requires distinct banks" + ) + assert( + cmd.op1_col === 1.U && cmd.op2_col === 1.U && cmd.wr_col === 1.U, + "MATMUL requires one physical bank per operand" + ) + val aElements = rows * Mux(isWindow, cmd.rs2(47, 32), inputK) + val bElements = cols * inputK + assert(Mux(isMx, aElements + (aElements >> 5), aElements << 2) <= (entries * 16).U, "MATMUL A exceeds bank capacity") + assert(Mux(isMx, bElements + (bElements >> 5), bElements << 2) <= (entries * 16).U, "MATMUL B exceeds bank capacity") + assert((cmd.rs2(31, 26) << 4) +& ((rows * cols) << 2) <= (entries * 16).U, "MATMUL C exceeds bank capacity") + when(cmd.rs2(24)) { + assert(!chain, "MATMUL first cannot replace a live chain") + chain := true.B + chainM := rows + chainN := cols + chainFormat := cmd.funct + chainBank := cmd.wr_bank + chainBase := cmd.rs2(31, 26) + }.otherwise { + assert( + chain && chainM === rows && chainN === cols && chainFormat === cmd.funct && chainBank === cmd.wr_bank && chainBase === cmd.rs2( + 31, + 26 + ), + "MATMUL continuation changed its chain" + ) + } + command := io.cmdReq.bits + m := rows + n := cols + k := inputK + mx := isMx + windowed := isWindow + separate := cmd.funct === f32Funct.U + first := cmd.rs2(24) + last := cmd.rs2(25) + base := cmd.rs2(31, 26) + batch := 0.U + context := 0.U + row := 0.U + reduction := 0.U + completed := 0.U + for (operand <- 0 until 2) { + inputState(operand) := 0.U + inputIndex(operand) := 0.U + inputRow(operand) := 0.U + rowOffset(operand) := 0.U + scaleMode(operand) := false.B + scaleCount(operand) := 0.U + } + state := waitChannels + } + + switch(state) { + is(waitChannels)(state := loadInputs) + is(loadInputs)( + when(Mux(windowed, window.io.done, inputState(0) === 3.U) && inputState(1) === 3.U)(state := initRequest) + ) + is(initRequest) { + when(first || row >= panelHeight) { + array.io.load := true.B + when(row === 15.U) { + row := 0.U + when(context + 1.U === count) { context := 0.U; state := compute } + .otherwise(context := context + 1.U) + }.otherwise(row := row + 1.U) + }.otherwise(state := initResponse) + } + is(initResponse) { + array.io.load := true.B + array.io.rowData := accRead + when(row === 15.U) { + row := 0.U + when(context + 1.U === count) { context := 0.U; state := compute } + .otherwise { context := context + 1.U; state := initRequest } + }.otherwise { row := row + 1.U; state := initRequest } + } + is(compute) { + when(context + 1.U === period) { + context := 0.U + when(reduction + 1.U === k)(state := drain) + .otherwise(reduction := reduction + 1.U) + }.otherwise(context := context + 1.U) + } + is(store) { + when(row === 15.U) { + row := 0.U + when(context + 1.U === count) { + context := 0.U + when(batch + count === totalPanels) { + when(last) { outputLine := 0.U; outputQuarter := 0.U; state := outputRequest } + .otherwise(state := complete) + }.otherwise { + batch := batch + count + reduction := 0.U + completed := 0.U + state := initRequest + } + }.otherwise(context := context + 1.U) + }.otherwise(row := row + 1.U) + } + is(outputRequest)(state := outputResponse) + is(outputResponse) { savedRow := accRead; state := outputWrite } + is(outputWrite)(when(writePort.io.req.fire)(state := outputAck)) + is(outputAck) { + when(writePort.io.resp.fire) { + assert(writePort.io.resp.bits.ok, "MATMUL bank write failed") + when(outputQuarter === 3.U) { + outputQuarter := 0.U + when(outputLine + 1.U === m * nGroups) { chain := false.B; state := complete } + .otherwise { outputLine := outputLine + 1.U; state := outputRequest } + }.otherwise { outputQuarter := outputQuarter + 1.U; state := outputWrite } + } + } + is(complete)(when(io.cmdResp.fire)(state := idle)) + } + when(array.io.completed) { + assert(state === compute || state === drain) + completed := completed + 1.U + when(completed + 1.U === count * k) { + context := 0.U + row := 0.U + state := store + } + } +} diff --git a/examples/components/balls/mxmm/configs/MxmmBallParam.scala b/examples/components/balls/mxmm/configs/MxmmBallParam.scala new file mode 100644 index 000000000..2d1993d42 --- /dev/null +++ b/examples/components/balls/mxmm/configs/MxmmBallParam.scala @@ -0,0 +1,22 @@ +package examples.balls.mxmm.configs + +import buckyball.balls.mxmm.{Params => ParametersProto} +import upickle.default.{macroRW, ReadWriter} + +case class MxmmBallParam(tileRows: Int, tileCols: Int) { + require(tileRows == 16 && tileCols == 16) + val arithmeticLatency = 3 + val contexts = arithmeticLatency * 2 + 1 +} + +object MxmmBallParam { + + def fromProto(source: ParametersProto): MxmmBallParam = { + val p = source + require(p.hasTileRows && p.hasTileCols) + val param = MxmmBallParam(p.getTileRows, p.getTileCols) + param + } + + implicit val rw: ReadWriter[MxmmBallParam] = macroRW +} diff --git a/examples/components/balls/mxmm/configs/default.toml b/examples/components/balls/mxmm/configs/default.toml new file mode 100644 index 000000000..4fb96e5be --- /dev/null +++ b/examples/components/balls/mxmm/configs/default.toml @@ -0,0 +1,2 @@ +tileRows = 16 +tileCols = 16 diff --git a/examples/components/balls/mxmm/configs/mxmm_params.proto b/examples/components/balls/mxmm/configs/mxmm_params.proto new file mode 100644 index 000000000..cc716d469 --- /dev/null +++ b/examples/components/balls/mxmm/configs/mxmm_params.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; +package buckyball.balls.mxmm; +option java_package = "buckyball.balls.mxmm"; +option java_multiple_files = true; + +message Params { + optional int32 tileRows = 1; + optional int32 tileCols = 2; +} diff --git a/examples/components/balls/mxmm/emu/src/accumulator.rs b/examples/components/balls/mxmm/emu/src/accumulator.rs new file mode 100644 index 000000000..3dbdb8b01 --- /dev/null +++ b/examples/components/balls/mxmm/emu/src/accumulator.rs @@ -0,0 +1,17 @@ + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Format { + Mxfp8, + Mxfp8Window, + Fma32, + F32, +} + +pub(crate) struct Chain { + pub(crate) format: Format, + pub(crate) rows: usize, + pub(crate) cols: usize, + pub(crate) bank: u64, + pub(crate) base: usize, + pub(crate) values: Vec, +} diff --git a/examples/components/balls/mxmm/emu/src/dispatch.rs b/examples/components/balls/mxmm/emu/src/dispatch.rs new file mode 100644 index 000000000..3fced61c8 --- /dev/null +++ b/examples/components/balls/mxmm/emu/src/dispatch.rs @@ -0,0 +1,33 @@ +use super::{fp32, mxfp8}; +use bebop_bemu::ball::ExecContext; + +const BALL_CLASS: &str = "examples.balls.mxmm.MxmmBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MXMM_MXFP8") => Some(mxfp8::execute(xs1, xs2, false, ctx)), + Some("MXMM_MXFP8_WINDOW") => Some(mxfp8::execute(xs1, xs2, true, ctx)), + Some("MXMM_FMA32") => Some(fp32::execute(xs1, xs2, true, ctx)), + Some("MXMM_F32") => Some(fp32::execute(xs1, xs2, false, ctx)), + _ => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, _xs1: u64, _xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MXMM_MXFP8" | "MXMM_MXFP8_WINDOW" | "MXMM_FMA32" | "MXMM_F32") => Some(1), + _ => None, + } +} diff --git a/examples/components/balls/mxmm/emu/src/fp32.rs b/examples/components/balls/mxmm/emu/src/fp32.rs new file mode 100644 index 000000000..458ae3593 --- /dev/null +++ b/examples/components/balls/mxmm/emu/src/fp32.rs @@ -0,0 +1,123 @@ +use std::collections::HashMap; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; +use super::accumulator::{Chain, Format}; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::instruction::ExecContext; + +pub(crate) fn execute(xs1: u64, xs2: u64, fused: bool, ctx: &mut ExecContext) -> u64 { + let ball_state = ctx.state::>(); + let format = if fused { Format::Fma32 } else { Format::F32 }; + let rows = (xs2 & 0xfff) as usize; + let cols = ((xs2 >> 12) & 0xfff) as usize; + let k = rs1_iter(xs1) as usize; + let first = (xs2 >> 24) & 1 != 0; + let last = (xs2 >> 25) & 1 != 0; + let base = ((xs2 >> 26) & 63) as usize; + let a = rs1_b0(xs1); + let b = rs1_b1(xs1); + let c = rs1_b2(xs1); + assert!( + xs2 >> 32 == 0 && (rows == 1 || (rows > 0 && rows % 16 == 0)) && cols > 0 && cols % 16 == 0 && k > 0 && k % 4 == 0, + "f32 matmul: requires M=1/multiple of 16, N multiple of 16, positive K multiple of four and zero reserved bits" + ); + assert!(a != b && b != c && a != c, "f32 matmul: banks must differ"); + for bank in [a, b, c] { + assert!( + ctx.config(bank).allocated && ctx.config(bank).cols == 1, + "f32 matmul: banks must be allocated with one column" + ); + } + let stride = bank_row_bytes(); + assert!( + stride == 16 + && rows * k * 4 <= bank_lines() * stride + && cols * k * 4 <= bank_lines() * stride + && base * stride + rows * cols * 4 <= bank_lines() * stride, + "f32 matmul: bank footprint exceeds depth" + ); + let pa = pbank(ctx, a); + let pb = pbank(ctx, b); + let av = &ctx.banks[pa][..rows * k * 4]; + let bv = &ctx.banks[pb][..cols * k * 4]; + assert!( + av.chunks_exact(4) + .chain(bv.chunks_exact(4)) + .all(|x| f32::from_le_bytes(x.try_into().unwrap()).is_finite()), + "f32 matmul: non-finite operand" + ); + let mut chain = { + let mut chains = ball_state.lock().unwrap(); + if first { + assert!( + !chains.contains_key(&ctx.hart_id), + "f32 matmul: first while chain is live" + ); + Chain { + format, + rows, + cols, + bank: c, + base, + values: vec![0f32; rows * cols], + } + } else { + chains + .remove(&ctx.hart_id) + .expect("f32 matmul: continuation without first") + } + }; + assert!( + chain.format == format + && chain.rows == rows + && chain.cols == cols + && chain.bank == c + && chain.base == base, + "f32 matmul: continuation changed destination" + ); + let weights: Vec<&[u8]> = (0..cols) + .map(|col| &bv[col * k * 4..(col + 1) * k * 4]) + .collect(); + for row in 0..rows { + if av[row * k * 4..(row + 1) * k * 4] + .chunks_exact(4) + .all(|x| u32::from_le_bytes(x.try_into().unwrap()) & 0x7fffffff == 0) + && chain.values[row * cols..(row + 1) * cols] + .iter() + .all(|x| x.to_bits() == 0) + { + continue; + } + let av = &av[row * k * 4..(row + 1) * k * 4]; + let dst = &mut chain.values[row * cols..(row + 1) * cols]; + let mut sums = dst.to_vec(); + for inner in 0..k { + let a = f32::from_le_bytes(av[inner * 4..inner * 4 + 4].try_into().unwrap()); + for col in 0..cols { + let b = + f32::from_le_bytes(weights[col][inner * 4..inner * 4 + 4].try_into().unwrap()); + sums[col] = if fused { + a.mul_add(b, sums[col]) + } else { + (a * b) + sums[col] + }; + } + } + assert!( + sums.iter().all(|x| x.is_finite()), + "f32 matmul: accumulator overflow" + ); + dst.copy_from_slice(&sums); + } + if last { + let pc = pbank(ctx, c); + for (index, value) in chain.values.iter().enumerate() { + let offset = base * stride + index * 4; + ctx.banks[pc][offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + } else { + { + ball_state.lock().unwrap().insert(ctx.hart_id, chain); + }; + } + 0 +} diff --git a/examples/components/balls/mxmm/emu/src/lib.rs b/examples/components/balls/mxmm/emu/src/lib.rs new file mode 100644 index 000000000..c74a71243 --- /dev/null +++ b/examples/components/balls/mxmm/emu/src/lib.rs @@ -0,0 +1,12 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod accumulator; +mod fp32; +mod mxfp8; +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/components/balls/mxmm/emu/src/mxfp8.rs b/examples/components/balls/mxmm/emu/src/mxfp8.rs new file mode 100644 index 000000000..6debeb4d6 --- /dev/null +++ b/examples/components/balls/mxmm/emu/src/mxfp8.rs @@ -0,0 +1,178 @@ +use std::collections::HashMap; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; +use super::accumulator::{Chain, Format}; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::instruction::ExecContext; + +fn value(code: u8, factor: f32) -> f32 { + let exponent = (code >> 3) & 15; + let mantissa = (code & 7) as u32; + let normal = ((exponent as u32 + 120) << 23) | (mantissa << 20); + let subnormal = (mantissa as f32 / 512.0).to_bits(); + let element = f32::from_bits(if exponent == 0 { subnormal } else { normal }); + let result = f32::from_bits((element * factor).to_bits() | ((code as u32 & 128) << 24)); + result +} + +pub(crate) fn execute(xs1: u64, xs2: u64, window: bool, ctx: &mut ExecContext) -> u64 { + let ball_state = ctx.state::>(); + let rows = (xs2 & 0xfff) as usize; + let cols = ((xs2 >> 12) & 0xfff) as usize; + let k = rs1_iter(xs1) as usize; + let first = (xs2 >> 24) & 1 != 0; + let last = (xs2 >> 25) & 1 != 0; + let base = ((xs2 >> 26) & 63) as usize; + let a = rs1_b0(xs1); + let b = rs1_b1(xs1); + let c = rs1_b2(xs1); + let full_k = if window { + ((xs2 >> 32) & 0xffff) as usize + } else { + k + }; + let start_k = if window { (xs2 >> 48) as usize } else { 0 }; + let format = if window { + Format::Mxfp8Window + } else { + Format::Mxfp8 + }; + assert!( + (window || xs2 >> 32 == 0) && (rows == 1 || (rows > 0 && rows % 16 == 0)) && cols > 0 && cols % 16 == 0 && k > 0 && k % 32 == 0, + "mxfp8: requires M=1/multiple of 16, N multiple of 16, K multiple of 32 and zero reserved bits" + ); + assert!( + full_k > 0 && full_k % 32 == 0 && start_k % 32 == 0 && start_k + k <= full_k, + "mxfp8: invalid activation window" + ); + assert!(a != b && b != c && a != c, "mxfp8: banks must differ"); + for bank in [a, b, c] { + assert!( + ctx.config(bank).allocated && ctx.config(bank).cols == 1, + "mxfp8: banks must be allocated with one column" + ); + } + let stride = bank_row_bytes(); + assert!(stride == 16, "mxfp8: requires 16-byte bank rows"); + assert!( + rows * full_k + rows * full_k / 32 <= bank_lines() * stride + && cols * k + cols * k / 32 <= bank_lines() * stride + && base * stride + rows * cols * 4 <= bank_lines() * stride, + "mxfp8: bank footprint exceeds depth" + ); + let pa = pbank(ctx, a); + let pb = pbank(ctx, b); + let a_bytes = &ctx.banks[pa][..rows * full_k + rows * full_k / 32]; + let b_bytes = &ctx.banks[pb][..cols * k + cols * k / 32]; + assert!( + b_bytes[..cols * k] + .iter() + .fold(true, |valid, code| valid & (code & 0x7f != 0x7f)), + "mxfp8: NaN encoding" + ); + let mut av = vec![0f32; rows * k]; + let mut bv = vec![0f32; cols * k]; + for (block, dst) in av.chunks_exact_mut(32).enumerate() { + let row = block / (k / 32); + let inner = block % (k / 32); + let code_offset = row * full_k + start_k + inner * 32; + let scale = a_bytes[rows * full_k + row * (full_k / 32) + start_k / 32 + inner]; + assert!(scale != 255, "mxfp8: NaN encoding"); + let factor = if scale == 0 { + f32::from_bits(1 << 22) + } else { + f32::from_bits((scale as u32) << 23) + }; + for (index, decoded) in dst.iter_mut().enumerate() { + let code = a_bytes[code_offset + index]; + assert!(code & 0x7f != 0x7f, "mxfp8: NaN encoding"); + *decoded = value(code, factor); + } + } + for block in 0..k / 32 { + for col in 0..cols { + let scale = b_bytes[cols * k + col * (k / 32) + block]; + assert!(scale != 255, "mxfp8: NaN encoding"); + let factor = if scale == 0 { + f32::from_bits(1 << 22) + } else { + f32::from_bits((scale as u32) << 23) + }; + let codes = &b_bytes[col * k + block * 32..col * k + (block + 1) * 32]; + let mut decoded = [0f32; 32]; + for (decoded, code) in decoded.iter_mut().zip(codes.iter()) { + *decoded = value(*code, factor); + } + for (index, decoded) in decoded.iter().enumerate() { + bv[(block * 32 + index) * cols + col] = *decoded; + } + } + } + assert!( + av.iter() + .chain(bv.iter()) + .fold(true, |valid, value| valid & value.is_finite()), + "mxfp8: decoded value overflows FP32" + ); + let mut chain = { + let mut chains = ball_state.lock().unwrap(); + if first { + assert!( + !chains.contains_key(&ctx.hart_id), + "mxfp8: first while chain is live" + ); + Chain { + format, + rows, + cols, + bank: c, + base, + values: vec![0f32; rows * cols], + } + } else { + chains + .remove(&ctx.hart_id) + .expect("mxfp8: continuation without first") + } + }; + assert!( + chain.format == format + && chain.rows == rows + && chain.cols == cols + && chain.bank == c + && chain.base == base, + "mxfp8: continuation changed destination" + ); + for row in 0..rows { + if av[row * k..(row + 1) * k].iter().all(|value| *value == 0.0) + && chain.values[row * cols..(row + 1) * cols] + .iter() + .all(|value| value.to_bits() == 0) + { + continue; + } + let av = &av[row * k..(row + 1) * k]; + let dst = &mut chain.values[row * cols..(row + 1) * cols]; + let mut sums = dst.to_vec(); + for inner in 0..k { + let a = av[inner]; + let weights = &bv[inner * cols..(inner + 1) * cols]; + for col in 0..cols { + sums[col] = a.mul_add(weights[col], sums[col]); + } + } + assert!(sums.iter().all(|v| v.is_finite()), "accumulator overflow"); + dst.copy_from_slice(&sums); + } + if last { + let pc = pbank(ctx, c); + for (i, value) in chain.values.iter().enumerate() { + ctx.banks[pc][base * stride + i * 4..base * stride + i * 4 + 4] + .copy_from_slice(&value.to_le_bytes()); + } + } else { + { + ball_state.lock().unwrap().insert(ctx.hart_id, chain); + }; + } + 0 +} diff --git a/examples/components/balls/mxmm/stack/kernels/CMakeLists.txt b/examples/components/balls/mxmm/stack/kernels/CMakeLists.txt new file mode 100644 index 000000000..ac40de7bd --- /dev/null +++ b/examples/components/balls/mxmm/stack/kernels/CMakeLists.txt @@ -0,0 +1,45 @@ +# ===----------------------------------------------------------------------=== +# MXMM kernels: two RVV kernels built against the thread's Ball ISA, and the +# host code that launches them. Added by the stack of a thread that has this +# Ball, which compiles KERNEL_SOURCES and exports KERNEL_EXPORTS. +# ===----------------------------------------------------------------------=== +set(RVV "${DESIGN_REPO}/stack/kernels/rvv") +set(MXMM_OBJECTS) +foreach(SOURCE + "${CMAKE_CURRENT_SOURCE_DIR}/norm_window.cpp" + "${RVV}/norm.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/packmm.cpp" + "${RVV}/pack.cpp") + get_filename_component(NAME "${SOURCE}" NAME_WE) + add_custom_command( + OUTPUT ${NAME}.o + COMMAND "${RVV_CXX}" --target=riscv64 -march=rv64imf_zve32f_zvl128b -mabi=lp64f + -mcmodel=medany -ffreestanding -O2 -ffp-contract=off -ffunction-sections -fdata-sections + "-I${DESIGN_REPO}/stack/kernels" "-I${DESIGN_REPO}/bb-tests/workloads/lib" "-I${ISA}" + -MD -MF ${NAME}.d -c "${SOURCE}" -o ${NAME}.o + DEPENDS "${SOURCE}" + DEPFILE ${NAME}.d + VERBATIM) + list(APPEND MXMM_OBJECTS "${CMAKE_CURRENT_BINARY_DIR}/${NAME}.o") +endforeach() +foreach(NAME norm_window packmm) + add_custom_command( + OUTPUT ${NAME}.elf + COMMAND "${RVV_LD}" -m elf64lriscv --gc-sections -e ${NAME} --defsym=kernel_entry=${NAME} + -T "${RVV}/kernel.ld" ${MXMM_OBJECTS} -o ${NAME}.elf + DEPENDS ${MXMM_OBJECTS} "${RVV}/kernel.ld" + VERBATIM) + add_custom_command( + OUTPUT ${NAME}.image + COMMAND "${RVV_OBJCOPY}" -O binary -j .header -j .text -j .rodata ${NAME}.elf ${NAME}.image + DEPENDS ${NAME}.elf + VERBATIM) +endforeach() +add_custom_target(mxmm-images DEPENDS + "${CMAKE_CURRENT_BINARY_DIR}/norm_window.image" "${CMAKE_CURRENT_BINARY_DIR}/packmm.image") + +set(KERNEL_SOURCES ${KERNEL_SOURCES} + "${CMAKE_CURRENT_SOURCE_DIR}/dispatch.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/pack_dispatch.cpp" PARENT_SCOPE) +set(KERNEL_EMBED_DIRS ${KERNEL_EMBED_DIRS} "${CMAKE_CURRENT_BINARY_DIR}" PARENT_SCOPE) +set(KERNEL_DEPENDS ${KERNEL_DEPENDS} mxmm-images PARENT_SCOPE) +set(KERNEL_EXPORTS ${KERNEL_EXPORTS} _mlir_ciface_rvv_norm_window _mlir_ciface_rvv_packmm PARENT_SCOPE) diff --git a/examples/balls/mxmm/workloads/kernels/dispatch.cpp b/examples/components/balls/mxmm/stack/kernels/dispatch.cpp similarity index 100% rename from examples/balls/mxmm/workloads/kernels/dispatch.cpp rename to examples/components/balls/mxmm/stack/kernels/dispatch.cpp diff --git a/examples/balls/mxmm/workloads/kernels/norm_window.cpp b/examples/components/balls/mxmm/stack/kernels/norm_window.cpp similarity index 97% rename from examples/balls/mxmm/workloads/kernels/norm_window.cpp rename to examples/components/balls/mxmm/stack/kernels/norm_window.cpp index f759124a3..fab5e461a 100644 --- a/examples/balls/mxmm/workloads/kernels/norm_window.cpp +++ b/examples/components/balls/mxmm/stack/kernels/norm_window.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include diff --git a/examples/components/balls/mxmm/stack/kernels/norm_window_image.h b/examples/components/balls/mxmm/stack/kernels/norm_window_image.h new file mode 100644 index 000000000..3328faa6f --- /dev/null +++ b/examples/components/balls/mxmm/stack/kernels/norm_window_image.h @@ -0,0 +1,15 @@ +#pragma once +// The norm_window RVV kernel image, embedded from this directory's build +// output. +#include + +alignas(4) constexpr uint8_t norm_window_image[] = { +#embed "norm_window.image" +}; +// Header words, see stack/kernels/rvv/kernel.ld: 1 = code bytes, 2 = entry. +constexpr uint32_t norm_window_entry = + norm_window_image[8] | norm_window_image[9] << 8 | + norm_window_image[10] << 16 | uint32_t(norm_window_image[11]) << 24; +constexpr uint32_t norm_window_text = + norm_window_image[4] | norm_window_image[5] << 8 | + norm_window_image[6] << 16 | uint32_t(norm_window_image[7]) << 24; diff --git a/examples/balls/mxmm/workloads/kernels/pack_dispatch.cpp b/examples/components/balls/mxmm/stack/kernels/pack_dispatch.cpp similarity index 100% rename from examples/balls/mxmm/workloads/kernels/pack_dispatch.cpp rename to examples/components/balls/mxmm/stack/kernels/pack_dispatch.cpp diff --git a/examples/balls/mxmm/workloads/kernels/packmm.cpp b/examples/components/balls/mxmm/stack/kernels/packmm.cpp similarity index 97% rename from examples/balls/mxmm/workloads/kernels/packmm.cpp rename to examples/components/balls/mxmm/stack/kernels/packmm.cpp index d04526669..ab8a6a026 100644 --- a/examples/balls/mxmm/workloads/kernels/packmm.cpp +++ b/examples/components/balls/mxmm/stack/kernels/packmm.cpp @@ -1,6 +1,6 @@ #include "packmm.h" #include -#include +#include extern "C" void rvv_pack(uint32_t *, const uint32_t *, size_t, size_t, size_t, size_t, size_t, size_t); diff --git a/examples/balls/mxmm/workloads/kernels/packmm.h b/examples/components/balls/mxmm/stack/kernels/packmm.h similarity index 100% rename from examples/balls/mxmm/workloads/kernels/packmm.h rename to examples/components/balls/mxmm/stack/kernels/packmm.h diff --git a/examples/components/balls/mxmm/stack/kernels/packmm_image.h b/examples/components/balls/mxmm/stack/kernels/packmm_image.h new file mode 100644 index 000000000..46742a11c --- /dev/null +++ b/examples/components/balls/mxmm/stack/kernels/packmm_image.h @@ -0,0 +1,14 @@ +#pragma once +// The packmm RVV kernel image, embedded from this directory's build output. +#include + +alignas(4) constexpr uint8_t packmm_image[] = { +#embed "packmm.image" +}; +// Header words, see stack/kernels/rvv/kernel.ld: 1 = code bytes, 2 = entry. +constexpr uint32_t packmm_entry = packmm_image[8] | packmm_image[9] << 8 | + packmm_image[10] << 16 | + uint32_t(packmm_image[11]) << 24; +constexpr uint32_t packmm_text = packmm_image[4] | packmm_image[5] << 8 | + packmm_image[6] << 16 | + uint32_t(packmm_image[7]) << 24; diff --git a/examples/balls/mxmm/workloads/kernels/window.h b/examples/components/balls/mxmm/stack/kernels/window.h similarity index 100% rename from examples/balls/mxmm/workloads/kernels/window.h rename to examples/components/balls/mxmm/stack/kernels/window.h diff --git a/examples/components/balls/mxmm/workloads/ctests/CMakeLists.txt b/examples/components/balls/mxmm/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..f01d58c48 --- /dev/null +++ b/examples/components/balls/mxmm/workloads/ctests/CMakeLists.txt @@ -0,0 +1,18 @@ +add_buckyball_ctest(mxmm_fp32_test.c) +add_buckyball_ctest(mxmm_mxfp8_test.c) +add_buckyball_ctest(mxmm_bank_capacity_test.c) + +file(READ "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h" MXMM_PARAMS) +string(REGEX MATCH "#define BANK_LINES ([0-9]+)" _ "${MXMM_PARAMS}") +set(MXMM_BANK_LINES ${CMAKE_MATCH_1}) +string(REGEX MATCH "#define BANK_WIDTH ([0-9]+)" _ "${MXMM_PARAMS}") +math(EXPR MXMM_BANK_BYTES "${MXMM_BANK_LINES} * ${CMAKE_MATCH_1} / 8") +if(MXMM_BANK_BYTES LESS 15840) + message( + FATAL_ERROR + "Mxmm A-window row test requires 15840-byte banks; actual ${MXMM_BANK_BYTES} bytes" + ) +endif() +add_buckyball_ctest(mxmm_window_test.c) +add_buckyball_ctest(mxmm_window_rows_test.c) +add_buckyball_ctest(mxmm_window_chain_test.c) diff --git a/examples/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c b/examples/components/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c similarity index 100% rename from examples/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c rename to examples/components/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c diff --git a/examples/balls/mxmm/workloads/ctests/mxmm_fp32_test.c b/examples/components/balls/mxmm/workloads/ctests/mxmm_fp32_test.c similarity index 100% rename from examples/balls/mxmm/workloads/ctests/mxmm_fp32_test.c rename to examples/components/balls/mxmm/workloads/ctests/mxmm_fp32_test.c diff --git a/examples/balls/mxmm/workloads/ctests/mxmm_mxfp8_test.c b/examples/components/balls/mxmm/workloads/ctests/mxmm_mxfp8_test.c similarity index 100% rename from examples/balls/mxmm/workloads/ctests/mxmm_mxfp8_test.c rename to examples/components/balls/mxmm/workloads/ctests/mxmm_mxfp8_test.c diff --git a/examples/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c b/examples/components/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c similarity index 100% rename from examples/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c rename to examples/components/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c diff --git a/examples/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c b/examples/components/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c similarity index 100% rename from examples/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c rename to examples/components/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c diff --git a/examples/balls/mxmm/workloads/ctests/mxmm_window_test.c b/examples/components/balls/mxmm/workloads/ctests/mxmm_window_test.c similarity index 100% rename from examples/balls/mxmm/workloads/ctests/mxmm_window_test.c rename to examples/components/balls/mxmm/workloads/ctests/mxmm_window_test.c diff --git a/examples/balls/mxmm/workloads/isa/mxmm.h b/examples/components/balls/mxmm/workloads/isa/mxmm.h similarity index 100% rename from examples/balls/mxmm/workloads/isa/mxmm.h rename to examples/components/balls/mxmm/workloads/isa/mxmm.h diff --git a/examples/balls/relu/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/mxmm/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/relu/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/mxmm/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/mxmm/workloads/mlir_tests/ball/CMakeLists.txt new file mode 100644 index 000000000..5a66bfbab --- /dev/null +++ b/examples/components/balls/mxmm/workloads/mlir_tests/ball/CMakeLists.txt @@ -0,0 +1,32 @@ +if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +endif() +set(BUCKYBALL_MLIR_TEST_PREFIX ball) +foreach(MODE mxfp8 fma32 f32) + add_buckyball_mlir_test(mxmm_${MODE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) + set(TEST_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_MLIR_ACTIVE_TARGET}-mlirtest-ball_mxmm_${MODE}" + ) + foreach(KIND baremetal linux) + add_custom_command( + OUTPUT ${TEST_NAME}-${KIND} + APPEND + DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_test.h + ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_steps.h + ) + endforeach() +endforeach() +foreach(CASE mxmm_window mxmm_window_rows mxmm_window_chain) + add_buckyball_mlir_test(${CASE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) + set(TEST_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_MLIR_ACTIVE_TARGET}-mlirtest-ball_${CASE}" + ) + foreach(KIND baremetal linux) + add_custom_command( + OUTPUT ${TEST_NAME}-${KIND} + APPEND + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../isa/mxmm.h + ) + endforeach() +endforeach() diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir rename to examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32_main.cpp similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32_main.cpp rename to examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32_main.cpp diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir rename to examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32_main.cpp similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32_main.cpp rename to examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32_main.cpp diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8.mlir similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8.mlir rename to examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8.mlir diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_main.cpp similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_main.cpp rename to examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_main.cpp diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window.mlir new file mode 100644 index 000000000..07f60fb63 --- /dev/null +++ b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window.mlir @@ -0,0 +1,28 @@ +// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @window +// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F75]]) +// CHECK-LABEL: func.func @window_small +// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 +// CHECK-DAG: %[[RS1:[^ ]+]] = arith.constant 34367084547 : i64 +// CHECK-DAG: %[[RS2:[^ ]+]] = arith.constant 9007474250153985 : i64 +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[RS1]], %[[RS2]], %[[F75]]) +func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { + buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 + return +} +func.func @window_small() attributes {llvm.emit_c_interface} { + %a = arith.constant 3 : i64 + %b = arith.constant 6 : i64 + %c = arith.constant 7 : i64 + %rows = arith.constant 1 : i64 + %cols = arith.constant 16 : i64 + %count = arith.constant 32 : i64 + %full = arith.constant 64 : i64 + %start = arith.constant 32 : i64 + %first = arith.constant true + %last = arith.constant true + %base = arith.constant 1 : i64 + buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 + return +} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain.mlir new file mode 100644 index 000000000..b8d641136 --- /dev/null +++ b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain.mlir @@ -0,0 +1,8 @@ +// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @window +// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F75]]) +func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { + buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 + return +} diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain_main.cpp similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain_main.cpp rename to examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain_main.cpp diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_main.cpp similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_main.cpp rename to examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_main.cpp diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows.mlir new file mode 100644 index 000000000..b8d641136 --- /dev/null +++ b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows.mlir @@ -0,0 +1,8 @@ +// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @window +// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F75]]) +func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { + buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 + return +} diff --git a/examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows_main.cpp similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows_main.cpp rename to examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows_main.cpp diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/mxmm/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..d055d2169 --- /dev/null +++ b/examples/components/balls/mxmm/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,32 @@ +if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +endif() +set(BUCKYBALL_MLIR_TEST_PREFIX bank) +foreach(MODE mxfp8 fma32 f32) + add_buckyball_mlir_test(mxmm_${MODE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) + set(TEST_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_MLIR_ACTIVE_TARGET}-mlirtest-bank_mxmm_${MODE}" + ) + foreach(KIND baremetal linux) + add_custom_command( + OUTPUT ${TEST_NAME}-${KIND} + APPEND + DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_test.h + ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_steps.h + ) + endforeach() +endforeach() +foreach(CASE mxmm_window mxmm_window_rows mxmm_window_chain) + add_buckyball_mlir_test(${CASE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) + set(TEST_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_MLIR_ACTIVE_TARGET}-mlirtest-bank_${CASE}" + ) + foreach(KIND baremetal linux) + add_custom_command( + OUTPUT ${TEST_NAME}-${KIND} + APPEND + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../isa/mxmm.h + ) + endforeach() +endforeach() diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir rename to examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32_main.cpp similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32_main.cpp rename to examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32_main.cpp diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir rename to examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32_main.cpp similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32_main.cpp rename to examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32_main.cpp diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8.mlir similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8.mlir rename to examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8.mlir diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_main.cpp similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_main.cpp rename to examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_main.cpp diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window.mlir new file mode 100644 index 000000000..47a6b18b8 --- /dev/null +++ b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window.mlir @@ -0,0 +1,30 @@ +// RUN: buddy-opt %s -assign-physical-banks | FileCheck %s --check-prefix=BANK +// BANK: buckyball.mxfp8_window +// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @window +// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F75]]) +// CHECK-LABEL: func.func @window_small +// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 +// CHECK-DAG: %[[RS1:[^ ]+]] = arith.constant 34367084547 : i64 +// CHECK-DAG: %[[RS2:[^ ]+]] = arith.constant 9007474250153985 : i64 +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[RS1]], %[[RS2]], %[[F75]]) +func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { + %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 + return +} +func.func @window_small() attributes {llvm.emit_c_interface} { + %a = arith.constant 3 : i64 + %b = arith.constant 6 : i64 + %c = arith.constant 7 : i64 + %rows = arith.constant 1 : i64 + %cols = arith.constant 16 : i64 + %count = arith.constant 32 : i64 + %full = arith.constant 64 : i64 + %start = arith.constant 32 : i64 + %first = arith.constant true + %last = arith.constant true + %base = arith.constant 1 : i64 + %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 + return +} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain.mlir new file mode 100644 index 000000000..52000bec5 --- /dev/null +++ b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain.mlir @@ -0,0 +1,10 @@ +// RUN: buddy-opt %s -assign-physical-banks | FileCheck %s --check-prefix=BANK +// BANK: buckyball.mxfp8_window +// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @window +// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F75]]) +func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { + %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 + return +} diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain_main.cpp similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain_main.cpp rename to examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain_main.cpp diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_main.cpp similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_main.cpp rename to examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_main.cpp diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows.mlir new file mode 100644 index 000000000..52000bec5 --- /dev/null +++ b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows.mlir @@ -0,0 +1,10 @@ +// RUN: buddy-opt %s -assign-physical-banks | FileCheck %s --check-prefix=BANK +// BANK: buckyball.mxfp8_window +// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @window +// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F75]]) +func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { + %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 + return +} diff --git a/examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows_main.cpp similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows_main.cpp rename to examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows_main.cpp diff --git a/examples/balls/mxmm/workloads/mlir_tests/mxmm_native_steps.h b/examples/components/balls/mxmm/workloads/mlir_tests/mxmm_native_steps.h similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/mxmm_native_steps.h rename to examples/components/balls/mxmm/workloads/mlir_tests/mxmm_native_steps.h diff --git a/examples/balls/mxmm/workloads/mlir_tests/mxmm_native_test.h b/examples/components/balls/mxmm/workloads/mlir_tests/mxmm_native_test.h similarity index 100% rename from examples/balls/mxmm/workloads/mlir_tests/mxmm_native_test.h rename to examples/components/balls/mxmm/workloads/mlir_tests/mxmm_native_test.h diff --git a/examples/balls/mxquant/arch/src/main/scala/Encode.scala b/examples/components/balls/mxquant/arch/Encode.scala similarity index 94% rename from examples/balls/mxquant/arch/src/main/scala/Encode.scala rename to examples/components/balls/mxquant/arch/Encode.scala index 9c5a76196..4b185f9f2 100644 --- a/examples/balls/mxquant/arch/src/main/scala/Encode.scala +++ b/examples/components/balls/mxquant/arch/Encode.scala @@ -1,10 +1,14 @@ package examples.balls.mxquant +import chisel3.experimental.hierarchy.{instantiable, public} + import chisel3._ import chisel3.util._ +@instantiable class Encode extends Module { + @public val io = IO(new Bundle { val input = Input(UInt(32.W)) val blockExponent = Input(SInt(10.W)) diff --git a/examples/components/balls/mxquant/arch/MxquantBall.scala b/examples/components/balls/mxquant/arch/MxquantBall.scala new file mode 100644 index 000000000..d7ad256e0 --- /dev/null +++ b/examples/components/balls/mxquant/arch/MxquantBall.scala @@ -0,0 +1,173 @@ +package examples.balls.mxquant + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.mxquant.{Params => ParametersProto} +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class MxquantBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + require( + mp.bankWidth == 128 && mp.bankMaskLen == 16, + "MxquantBall requires 128-bit bank rows and byte write masks" + ) + require(mp.bankEntries >= 8, "MxquantBall requires eight input rows") + require(ball.inBW == 1 && ball.outBW == 1, "MxquantBall requires one read and one write") + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val idle :: readReq :: readResp :: exponent :: encode :: codeReq :: codeResp :: scaleReq :: scaleResp :: complete :: Nil = + Enum(10) + val state = RegInit(idle) + val robId = Reg(UInt(log2Up(fp.rob_entries).W)) + val source = Reg(UInt(fp.bank_id_len.W)) + val destination = Reg(UInt(fp.bank_id_len.W)) + val bases = Reg(Vec(3, UInt(mp.groupIdWidth.W))) + val count = Reg(UInt(fp.iter_len.W)) + val group = RegInit(0.U(fp.iter_len.W)) + val readRow = RegInit(0.U(3.W)) + val lane = RegInit(0.U(5.W)) + val codeRow = RegInit(0.U(1.W)) + val maximum = RegInit(0.U(31.W)) + val blockExponent = Reg(SInt(10.W)) + val scale = Reg(UInt(8.W)) + val values = Reg(Vec(32, UInt(32.W))) + val codes = Reg(Vec(32, UInt(8.W))) + val math = Instantiate(new Encode) + math.io.input := values(lane) + math.io.blockExponent := blockExponent + + val active = !reset.asBool + blink.cmdReq.ready := state === idle && active + blink.cmdResp.valid := state === complete && active + blink.cmdResp.bits.rob_id := robId + blink.status.idle := state === idle + blink.status.running := state =/= idle && state =/= complete + + val read = blink.bankRead(0) + read.rob_id := robId + read.bank_id := source + read.group_id := bases(0) + read.io.req.valid := state === readReq && active + read.io.req.bits.addr := (group << 3) + readRow + read.io.resp.ready := state === readResp && active + + val write = blink.bankWrite(0) + write.rob_id := robId + write.bank_id := destination + write.group_id := bases(2) + write.io.req.valid := (state === codeReq || state === scaleReq) && active + write.io.req.bits.addr := Mux(state === scaleReq, (count >> 4) + (group >> 4), (group << 1) + codeRow) + val lowCodes = Cat((15 to 0 by -1).map(codes(_))) + val highCodes = Cat((31 to 16 by -1).map(codes(_))) + write.io.req.bits.data := Mux( + state === scaleReq, + (scale.pad(128) << (group(3, 0) << 3))(127, 0), + Mux(codeRow === 0.U, lowCodes, highCodes) + ) + write.io.req.bits.mask := VecInit((0 until 16).map(i => state =/= scaleReq || group(3, 0) === i.U)) + write.io.resp.ready := (state === codeResp || state === scaleResp) && active + + switch(state) { + is(idle) { + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + + assert(cmd.iter > 0.U && cmd.iter(4, 0) === 0.U, "MxquantBall count must be positive and divisible by 32") + assert( + cmd.iter <= (mp.bankEntries * 4).U, + "MxquantBall input exceeds capacity" + ) + assert( + cmd.iter +& (cmd.iter >> 5) <= (mp.bankEntries * 16).U, + "MxquantBall output exceeds capacity" + ) + assert(cmd.rs2 === 0.U, "MxquantBall reserved fields must be zero") + assert(cmd.op1_col === 1.U && cmd.wr_col === 1.U, "MxquantBall operands must occupy one bank") + assert(cmd.op1_bank =/= cmd.wr_bank, "MxquantBall banks must differ") + robId := blink.cmdReq.bits.rob_id + source := cmd.op1_bank + destination := cmd.wr_bank + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + count := cmd.iter + group := 0.U + readRow := 0.U + maximum := 0.U + state := readReq + } + } + is(readReq)(when(read.io.req.fire)(state := readResp)) + is(readResp) { + when(read.io.resp.fire) { + val words = (0 until 4).map(i => read.io.resp.bits.data(32 * i + 31, 32 * i)) + for (i <- 0 until 4) { + assert(words(i)(30, 23) =/= 255.U, "MxquantBall input must be finite") + values(Cat(readRow, i.U(2.W))) := words(i) + } + maximum := (Seq(maximum) ++ words.map(_(30, 0))).reduce((a, c) => Mux(a > c, a, c)) + when(readRow === 7.U)(state := exponent).otherwise { readRow := readRow + 1.U; state := readReq } + } + } + is(exponent) { + val raw = maximum(30, 23).zext - 135.S(10.W) + val selected = Mux(maximum === 0.U, 0.S(10.W), Mux(raw < -127.S, -127.S(10.W), raw)) + blockExponent := selected + scale := (selected + 127.S).asUInt + lane := 0.U + state := encode + } + is(encode) { + codes(lane) := math.io.code + when(lane === 31.U) { codeRow := 0.U; state := codeReq }.otherwise(lane := lane + 1.U) + } + is(codeReq)(when(write.io.req.fire)(state := codeResp)) + is(codeResp) { + when(write.io.resp.fire) { + when(codeRow === 0.U) { codeRow := 1.U; state := codeReq }.otherwise(state := scaleReq) + } + } + is(scaleReq)(when(write.io.req.fire)(state := scaleResp)) + is(scaleResp) { + when(write.io.resp.fire) { + when(group === (count >> 5) - 1.U)(state := complete).otherwise { + group := group + 1.U + readRow := 0.U + maximum := 0.U + state := readReq + } + } + } + is(complete)(when(blink.cmdResp.fire)(state := idle)) + } +} + +object MxquantBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new MxquantBall(fp, mp, bp, id)).blink, None) + +} diff --git a/arch/src/main/scala/framework/mem-core/coherence/src/main/resources/README.md b/examples/components/balls/mxquant/configs/default.toml similarity index 100% rename from arch/src/main/scala/framework/mem-core/coherence/src/main/resources/README.md rename to examples/components/balls/mxquant/configs/default.toml diff --git a/examples/components/balls/mxquant/configs/mxquant_params.proto b/examples/components/balls/mxquant/configs/mxquant_params.proto new file mode 100644 index 000000000..2d178b8a4 --- /dev/null +++ b/examples/components/balls/mxquant/configs/mxquant_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.mxquant; +option java_package = "buckyball.balls.mxquant"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/components/balls/mxquant/emu/src/dispatch.rs b/examples/components/balls/mxquant/emu/src/dispatch.rs new file mode 100644 index 000000000..3cec59c5b --- /dev/null +++ b/examples/components/balls/mxquant/emu/src/dispatch.rs @@ -0,0 +1,30 @@ +use super::quant; +use bebop_bemu::ball::ExecContext; + +const BALL_CLASS: &str = "examples.balls.mxquant.MxquantBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MXQUANT") => Some(quant::execute(xs1, xs2, ctx)), + _ => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, _xs1: u64, _xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("MXQUANT") => Some(1), + _ => None, + } +} diff --git a/examples/components/balls/mxquant/emu/src/lib.rs b/examples/components/balls/mxquant/emu/src/lib.rs new file mode 100644 index 000000000..276b3f599 --- /dev/null +++ b/examples/components/balls/mxquant/emu/src/lib.rs @@ -0,0 +1,10 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod dispatch; +mod quant; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/components/balls/mxquant/emu/src/quant.rs b/examples/components/balls/mxquant/emu/src/quant.rs new file mode 100644 index 000000000..eb638d8d1 --- /dev/null +++ b/examples/components/balls/mxquant/emu/src/quant.rs @@ -0,0 +1,91 @@ +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::instruction::ExecContext; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; + +fn rounded_shift(value: u32, shift: u32) -> u32 { + if shift > 31 { + return 0; + } + let half = 1 << (shift - 1); + (value + half - 1 + ((value >> shift) & 1)) >> shift +} + +fn encode(bits: u32, block_exponent: i32) -> u8 { + let sign = ((bits >> 24) & 128) as u8; + if bits & 0x7fffffff == 0 { + return sign; + } + let mut exponent = ((bits >> 23) & 255) as i32; + let mut fraction = bits & 0x7fffff; + if exponent == 0 { + let shift = fraction.leading_zeros() - 8; + fraction = (fraction << shift) & 0x7fffff; + exponent = 1 - shift as i32; + } + exponent -= block_exponent + 120; + let code = if exponent <= 0 { + rounded_shift(fraction | 0x800000, (21 - exponent) as u32) + } else { + exponent as u32 * 8 + rounded_shift(fraction, 20) + }; + sign | code.min(126) as u8 +} + +pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + let input = rs1_b0(xs1); + let output = rs1_b2(xs1); + let count = rs1_iter(xs1) as usize; + assert!( + xs2 == 0 && rs1_b1(xs1) == 0, + "mxquant: reserved fields must be zero" + ); + assert!( + count > 0 && count % 32 == 0, + "mxquant: count must be a positive multiple of 32" + ); + assert!( + input != output, + "mxquant: input and output banks must differ" + ); + for bank in [input, output] { + assert!( + ctx.config(bank).allocated && ctx.config(bank).cols == 1, + "mxquant: banks must be allocated with one column" + ); + } + let bytes = bank_lines() * bank_row_bytes(); + assert!( + bank_row_bytes() == 16 && count <= bytes / 4 && count + count / 32 <= bytes, + "mxquant: bank footprint exceeds capacity" + ); + let source = &ctx.banks[pbank(ctx, input)][..count * 4]; + assert!( + source + .chunks_exact(4) + .all( + |element| u32::from_le_bytes(element.try_into().unwrap()) & 0x7fffffff < 0x7f800000 + ), + "mxquant: non-finite activation" + ); + let mut packed = vec![0u8; count + count / 32]; + for (block, values) in source.chunks_exact(128).enumerate() { + let maximum = values + .chunks_exact(4) + .map(|element| u32::from_le_bytes(element.try_into().unwrap()) & 0x7fffffff) + .max() + .unwrap(); + let exponent = if maximum == 0 { + 0 + } else { + ((maximum >> 23) as i32 - 135).max(-127) + }; + packed[count + block] = (exponent + 127) as u8; + for (index, element) in values.chunks_exact(4).enumerate() { + packed[block * 32 + index] = + encode(u32::from_le_bytes(element.try_into().unwrap()), exponent); + } + } + let destination = pbank(ctx, output); + ctx.banks[destination][..packed.len()].copy_from_slice(&packed); + 0 +} diff --git a/examples/components/balls/mxquant/stack/kernels/CMakeLists.txt b/examples/components/balls/mxquant/stack/kernels/CMakeLists.txt new file mode 100644 index 000000000..aa8104386 --- /dev/null +++ b/examples/components/balls/mxquant/stack/kernels/CMakeLists.txt @@ -0,0 +1,10 @@ +# ===----------------------------------------------------------------------=== +# MXQuant kernels: the host code that drives the Ball. Added by the stack of a +# thread that has this Ball, which compiles KERNEL_SOURCES and exports +# KERNEL_EXPORTS. +# ===----------------------------------------------------------------------=== +set(KERNEL_SOURCES ${KERNEL_SOURCES} + "${CMAKE_CURRENT_SOURCE_DIR}/dispatch.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/pack.cpp" PARENT_SCOPE) +set(KERNEL_INCLUDES ${KERNEL_INCLUDES} + "${DESIGN_REPO}/examples/components/balls/mxquant/workloads" PARENT_SCOPE) +set(KERNEL_EXPORTS ${KERNEL_EXPORTS} mxfp8_quant _mlir_ciface_mxfp8_quant PARENT_SCOPE) diff --git a/examples/components/balls/mxquant/stack/kernels/dispatch.cpp b/examples/components/balls/mxquant/stack/kernels/dispatch.cpp new file mode 100644 index 000000000..541933d7e --- /dev/null +++ b/examples/components/balls/mxquant/stack/kernels/dispatch.cpp @@ -0,0 +1,14 @@ +#include +#include + +extern "C" void mxfp8_quant(const uint32_t *input, uint8_t *output, + uint32_t count) { + constexpr unsigned inputBank = 0, outputBank = 1; + bb_mem_alloc(inputBank, 1, 1); + bb_mem_alloc(outputBank, 1, 1); + bb_mvin((uintptr_t)input, inputBank, count / 4, 1); + bb_mxquant(inputBank, outputBank, count); + bb_mvout((uintptr_t)output, outputBank, (count + count / 32 + 15) / 16, 1); + bb_mem_release(inputBank); + bb_mem_release(outputBank); +} diff --git a/examples/components/balls/mxquant/stack/kernels/pack.cpp b/examples/components/balls/mxquant/stack/kernels/pack.cpp new file mode 100644 index 000000000..20bc8b53c --- /dev/null +++ b/examples/components/balls/mxquant/stack/kernels/pack.cpp @@ -0,0 +1,70 @@ +#include +#include +#include +#include +#include +#include + +extern "C" void mxfp8_quant(const uint32_t *input, uint8_t *output, + uint32_t count); + +extern "C" void _mlir_ciface_mxfp8_quant(StridedMemRefType *input, + StridedMemRefType *output, + int64_t tileRows, int64_t tileK, + int64_t panelStride) { + int64_t m = input->sizes[0], k = input->sizes[1]; + int64_t mt = (m + tileRows - 1) / tileRows, kt = (k + tileK - 1) / tileK; + int64_t bytes = panelStride; + if (k % 32 || tileK % 32 || output->sizes[0] != mt * kt * bytes) { + fprintf(stderr, "mxfp8: invalid activation packing shape\n"); + abort(); + } + auto *source = input->data + input->offset; + auto *dest = output->data + output->offset; + const int64_t rowStride = input->strides[0], columnStride = input->strides[1]; + const int64_t outputStride = output->strides[0]; + for (int64_t mr = 0; mr < mt; ++mr) + for (int64_t kr = 0; kr < kt; ++kr) + for (int64_t row = 0; row < tileRows; ++row) { + int64_t width = std::min(tileK, k - kr * tileK); + int64_t base = (mr * kt + kr) * bytes; + if (mr * tileRows + row >= m && outputStride == 1) { + std::memset(dest + base + row * width, 0, width); + std::memset(dest + base + tileRows * width + row * (width / 32), 127, + width / 32); + continue; + } + alignas(16) uint32_t staging[1024]; + alignas(16) uint8_t packed[1056]; + for (int64_t begin = 0; begin < width; begin += 1024) { + int64_t count = std::min(1024, width - begin); + int64_t r = mr * tileRows + row, c = kr * tileK + begin; + int64_t valid = + r < m ? std::max(0, std::min(count, k - c)) : 0; + const uint32_t *values; + if (valid == count && columnStride == 1) { + values = + reinterpret_cast(source + r * rowStride + c); + } else { + for (int64_t i = 0; i < valid; ++i) + staging[i] = __builtin_bit_cast( + uint32_t, source[r * rowStride + (c + i) * columnStride]); + std::fill(staging + valid, staging + count, 0); + values = staging; + } + mxfp8_quant(values, packed, count); + int64_t codes = base + row * width + begin; + int64_t scales = + base + tileRows * width + row * (width / 32) + begin / 32; + if (outputStride == 1) { + std::memcpy(dest + codes, packed, count); + std::memcpy(dest + scales, packed + count, count / 32); + } else { + for (int64_t i = 0; i < count; ++i) + dest[(codes + i) * outputStride] = packed[i]; + for (int64_t i = 0; i < count / 32; ++i) + dest[(scales + i) * outputStride] = packed[count + i]; + } + } + } +} diff --git a/examples/components/balls/mxquant/workloads/ctests/CMakeLists.txt b/examples/components/balls/mxquant/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..017bd44ca --- /dev/null +++ b/examples/components/balls/mxquant/workloads/ctests/CMakeLists.txt @@ -0,0 +1,25 @@ +file( + READ "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h" + MXQUANT_PARAMS +) +string(REGEX MATCH "#define BANK_LINES ([0-9]+)" _ "${MXQUANT_PARAMS}") +set(MXQUANT_BANK_LINES ${CMAKE_MATCH_1}) +string(REGEX MATCH "#define BANK_WIDTH ([0-9]+)" _ "${MXQUANT_PARAMS}") +math(EXPR MXQUANT_BANK_BYTES "${MXQUANT_BANK_LINES} * ${CMAKE_MATCH_1} / 8") +math(EXPR MXQUANT_BANK_TAIL "${MXQUANT_BANK_BYTES} % 128") +if(MXQUANT_BANK_BYTES LESS 3840 OR MXQUANT_BANK_TAIL) + message( + FATAL_ERROR + "Mxquant CTests require banks >= 3840 bytes and a 32-element FP32 capacity; actual ${MXQUANT_BANK_BYTES} bytes" + ) +endif() +add_buckyball_ctest(mxquant_values_test.c) +add_buckyball_ctest(mxquant_capacity_test.c) +set(TEST_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-mxquant_values_test" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}-baremetal + APPEND + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../reference.h +) diff --git a/examples/balls/mxquant/workloads/ctests/mxquant_capacity_test.c b/examples/components/balls/mxquant/workloads/ctests/mxquant_capacity_test.c similarity index 100% rename from examples/balls/mxquant/workloads/ctests/mxquant_capacity_test.c rename to examples/components/balls/mxquant/workloads/ctests/mxquant_capacity_test.c diff --git a/examples/balls/mxquant/workloads/ctests/mxquant_values_test.c b/examples/components/balls/mxquant/workloads/ctests/mxquant_values_test.c similarity index 100% rename from examples/balls/mxquant/workloads/ctests/mxquant_values_test.c rename to examples/components/balls/mxquant/workloads/ctests/mxquant_values_test.c diff --git a/examples/balls/mxquant/workloads/isa/mxquant.h b/examples/components/balls/mxquant/workloads/isa/mxquant.h similarity index 100% rename from examples/balls/mxquant/workloads/isa/mxquant.h rename to examples/components/balls/mxquant/workloads/isa/mxquant.h diff --git a/examples/balls/mxquant/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/mxquant/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/mxquant/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/mxquant/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/mxquant/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/mxquant/workloads/mlir_tests/ball/CMakeLists.txt new file mode 100644 index 000000000..e0f52cc49 --- /dev/null +++ b/examples/components/balls/mxquant/workloads/mlir_tests/ball/CMakeLists.txt @@ -0,0 +1,12 @@ +set(BUCKYBALL_MLIR_TEST_PREFIX ball) +add_buckyball_mlir_test(mxquant TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) +set(TEST_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_MLIR_ACTIVE_TARGET}-mlirtest-ball_mxquant" +) +foreach(KIND baremetal linux) + add_custom_command( + OUTPUT ${TEST_NAME}-${KIND} + APPEND + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../reference.h + ) +endforeach() diff --git a/examples/components/balls/mxquant/workloads/mlir_tests/ball/mxquant.mlir b/examples/components/balls/mxquant/workloads/mlir_tests/ball/mxquant.mlir new file mode 100644 index 000000000..67f4775d0 --- /dev/null +++ b/examples/components/balls/mxquant/workloads/mlir_tests/ball/mxquant.mlir @@ -0,0 +1,22 @@ +// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @quant_block_32 +// CHECK-DAG: %[[F54:.*]] = llvm.mlir.constant(54 : i32) : i32 +// CHECK: %[[ZERO:[^ ]+]] = arith.constant 0 : i64 +// CHECK: %[[PACKED:[^ ]+]] = arith.constant 34363932675 : i64 +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[PACKED]], %[[ZERO]], %[[F54]]) +// CHECK-LABEL: func.func @quant_dynamic +// CHECK-DAG: %[[F54:.*]] = llvm.mlir.constant(54 : i32) : i32 +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F54]]) +func.func @quant_block_32() attributes {llvm.emit_c_interface} { + %input = arith.constant 3 : i64 + %output = arith.constant 4 : i64 + %count = arith.constant 32 : i64 + buckyball.mxquant %input, %output, %count : i64 + return +} +func.func @quant_dynamic(%count: i64) attributes {llvm.emit_c_interface} { + %input = arith.constant 3 : i64 + %output = arith.constant 4 : i64 + buckyball.mxquant %input, %output, %count : i64 + return +} diff --git a/examples/balls/mxquant/workloads/mlir_tests/ball/mxquant_main.cpp b/examples/components/balls/mxquant/workloads/mlir_tests/ball/mxquant_main.cpp similarity index 100% rename from examples/balls/mxquant/workloads/mlir_tests/ball/mxquant_main.cpp rename to examples/components/balls/mxquant/workloads/mlir_tests/ball/mxquant_main.cpp diff --git a/examples/components/balls/mxquant/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/mxquant/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..a4a7d11f5 --- /dev/null +++ b/examples/components/balls/mxquant/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,12 @@ +set(BUCKYBALL_MLIR_TEST_PREFIX bank) +add_buckyball_mlir_test(mxquant TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) +set(TEST_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_MLIR_ACTIVE_TARGET}-mlirtest-bank_mxquant" +) +foreach(KIND baremetal linux) + add_custom_command( + OUTPUT ${TEST_NAME}-${KIND} + APPEND + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../reference.h + ) +endforeach() diff --git a/examples/components/balls/mxquant/workloads/mlir_tests/bank/mxquant.mlir b/examples/components/balls/mxquant/workloads/mlir_tests/bank/mxquant.mlir new file mode 100644 index 000000000..55cbb2106 --- /dev/null +++ b/examples/components/balls/mxquant/workloads/mlir_tests/bank/mxquant.mlir @@ -0,0 +1,27 @@ +// RUN: buddy-opt %s -assign-physical-banks | FileCheck %s --check-prefix=BANK +// BANK-LABEL: func.func @quant_block_32 +// BANK: buckyball.mxquant +// BANK-LABEL: func.func @quant_dynamic +// BANK: buckyball.mxquant +// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @quant_block_32 +// CHECK-DAG: %[[F54:.*]] = llvm.mlir.constant(54 : i32) : i32 +// CHECK: %[[ZERO:[^ ]+]] = arith.constant 0 : i64 +// CHECK: %[[PACKED:[^ ]+]] = arith.constant 34363932675 : i64 +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[PACKED]], %[[ZERO]], %[[F54]]) +// CHECK-LABEL: func.func @quant_dynamic +// CHECK-DAG: %[[F54:.*]] = llvm.mlir.constant(54 : i32) : i32 +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F54]]) +func.func @quant_block_32() attributes {llvm.emit_c_interface} { + %input = arith.constant 3 : i64 + %output = arith.constant 4 : i64 + %count = arith.constant 32 : i64 + %result = buckyball.bank_mxquant %input %output %count : i64 i64 i64 + return +} +func.func @quant_dynamic(%count: i64) attributes {llvm.emit_c_interface} { + %input = arith.constant 3 : i64 + %output = arith.constant 4 : i64 + %result = buckyball.bank_mxquant %input %output %count : i64 i64 i64 + return +} diff --git a/examples/balls/mxquant/workloads/mlir_tests/bank/mxquant_main.cpp b/examples/components/balls/mxquant/workloads/mlir_tests/bank/mxquant_main.cpp similarity index 100% rename from examples/balls/mxquant/workloads/mlir_tests/bank/mxquant_main.cpp rename to examples/components/balls/mxquant/workloads/mlir_tests/bank/mxquant_main.cpp diff --git a/examples/balls/mxquant/workloads/reference.h b/examples/components/balls/mxquant/workloads/reference.h similarity index 94% rename from examples/balls/mxquant/workloads/reference.h rename to examples/components/balls/mxquant/workloads/reference.h index 765b062b7..107098472 100644 --- a/examples/balls/mxquant/workloads/reference.h +++ b/examples/components/balls/mxquant/workloads/reference.h @@ -2,7 +2,7 @@ #define MXQUANT_REFERENCE_H #include -/* Integer reference from bbsw/kernels/mxfp8/quant.cpp; no FP rounding state. */ +/* Integer MXFP8 quantization reference; no FP rounding state. */ static inline uint32_t rounded_shift(uint32_t value, unsigned shift) { if (shift > 31) return 0; diff --git a/examples/components/balls/redmule/arch/RedmuleBall.scala b/examples/components/balls/redmule/arch/RedmuleBall.scala new file mode 100644 index 000000000..12de4f5c7 --- /dev/null +++ b/examples/components/balls/redmule/arch/RedmuleBall.scala @@ -0,0 +1,234 @@ +package examples.balls.redmule + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.redmule.{Params => ParametersProto} +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.balldomain.ball.blink.{BallCommand, BallStatus, BlinkIO, HasBlink} + +@instantiable +class RedmuleBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + require(ball.inBW == 2 && ball.outBW == 1) + require( + mp.bankWidth == 128 && mp.bankMaskLen == 16 + ) + require( + mp.bankEntries <= 65535 && mp.sharedBankEntries <= 65535 + ) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val Seq(idle, channels, configure, controlReply, running, complete) = Enum(6) + val state = RegInit(idle) + + val issue = Reg(new BallCommand(fp, mp)) + + val controlStep = RegInit(0.U(4.W)) + val finished = RegInit(false.B) + val rows = issue.rs2(11, 0) + val aBase = issue.rs2(21, 12) + val bBase = issue.rs2(31, 22) + val cBase = issue.rs2(41, 32) + val first = issue.rs2(42) + val native = Instantiate(new RedmuleNative) + native.io.clk_i := clock + native.io.rst_ni := !reset.asBool + native.io.ctrl_req_i := state === configure + native.io.ctrl_wen_i := controlStep === 0.U + native.io.ctrl_addr_i := VecInit(Seq(4, 44, 48, 52, 32, 36, 40, 60, 0).map(_.U(32.W)))(controlStep) + native.io.ctrl_data_i := VecInit(Seq( + 0.U(32.W), + 16777216.U | (aBase << 4), + 33554432.U | (bBase << 4), + 50331648.U | (cBase << 4), + (8.U(32.W) << 16) | rows, + 41943040.U(32.W) | ((!first).asUInt << 20) | issue.iter(15, 0), + 0.U(32.W), + 0.U(32.W), + 0.U(32.W) + ))(controlStep) + + val Seq(busIdle, readRequest, readReply, writeRequest, writeReply, advance, grant, response) = Enum(8) + val busState = RegInit(busIdle) + val address = Reg(UInt(32.W)) + val load = Reg(Bool()) + val data = Reg(UInt(256.W)) + val mask = Reg(UInt(32.W)) + val transactionId = Reg(UInt(8.W)) + val half = RegInit(false.B) + val responseData = Reg(UInt(256.W)) + val region = address(25, 24) + val bankRow = address(23, 4) +& half + val startRow = Mux(region === 1.U, aBase, Mux(region === 2.U, bBase, cBase)) + val span = Mux(region === 1.U, rows * (issue.iter(15, 0) >> 3), Mux(region === 2.U, issue.iter(15, 0), rows)) + val inWindow = bankRow >= startRow && bankRow < (startRow +& span) + val readPort = Mux(region === 2.U, 1.U, 0.U) + + val readBank = Mux( + region === 1.U, + issue.op1_bank, + Mux(region === 2.U, issue.op2_bank, issue.wr_bank) + ) + + val readGroup = Mux(region === 1.U, issue.op1_group, Mux(region === 2.U, issue.op2_group, issue.wr_group)) + + val writeMask = Mux(half, mask(31, 16), mask(15, 0)) + val writeData = Mux(half, data(255, 128), data(127, 0)) + native.io.mem_gnt_i := busState === grant + native.io.mem_valid_i := busState === response + native.io.mem_data_i := responseData + native.io.mem_id_i := transactionId + + for (port <- 0 until 2) { + val read = blink.bankRead(port) + read.rob_id := issue.rob_id + read.bank_id := readBank + read.group_id := readGroup + read.io.req.valid := busState === readRequest && readPort === port.U && inWindow + read.io.req.bits.addr := bankRow + read.io.resp.ready := busState === readReply && readPort === port.U + when(read.io.req.fire)(busState := readReply) + when(read.io.resp.fire) { + responseData := Mux( + half, + Cat(read.io.resp.bits.data, responseData(127, 0)), + Cat(responseData(255, 128), read.io.resp.bits.data) + ) + busState := advance + } + } + val write = blink.bankWrite(0) + write.rob_id := issue.rob_id + write.bank_id := issue.wr_bank + write.group_id := issue.wr_group + write.io.req.valid := busState === writeRequest && writeMask.orR && inWindow + write.io.req.bits.addr := bankRow + write.io.req.bits.data := writeData + write.io.req.bits.mask := VecInit((0 until 16).map(writeMask(_))) + write.io.resp.ready := busState === writeReply + when(write.io.req.fire) { + assert(region === 3.U && inWindow, "RedMulE store outside C window") + busState := writeReply + } + when(write.io.resp.fire) { + assert(write.io.resp.bits.ok, "RedMulE bank write failed") + busState := advance + } + when(busState === busIdle && state === running && native.io.mem_req_o) { + assert( + native.io.mem_addr_o(31, 26) === 0.U && native.io.mem_addr_o(25, 24) =/= 0.U && + native.io.mem_addr_o(3, 0) === 0.U, + "RedMulE address cannot map to a bank row" + ) + address := native.io.mem_addr_o + data := native.io.mem_data_o + mask := native.io.mem_be_o + load := native.io.mem_wen_o + transactionId := native.io.mem_id_o + half := false.B + responseData := 0.U + busState := Mux(native.io.mem_wen_o, readRequest, writeRequest) + } + // Unused lanes in the native 256-bit load are zero-filled at a tensor boundary. + when(busState === readRequest && !inWindow)(busState := advance) + when(busState === writeRequest && (!writeMask.orR || !inWindow))(busState := advance) + when(busState === advance) { + when(!half) { + half := true.B + busState := Mux(load, readRequest, writeRequest) + }.otherwise(busState := grant) + } + when(busState === grant) { + assert( + native.io.mem_req_o && native.io.mem_addr_o === address && native.io.mem_wen_o === load, + "RedMulE request changed before grant" + ) + busState := Mux(load, response, busIdle) + } + when(busState === response && native.io.mem_ready_o)(busState := busIdle) + + blink.cmdReq.ready := state === idle + blink.cmdResp.valid := state === complete + blink.cmdResp.bits.rob_id := issue.rob_id + blink.status.idle := state === idle + blink.status.running := state =/= idle + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + val m = cmd.rs2(11, 0) + assert( + cmd.op1_col === 1.U && cmd.op2_col === 1.U && cmd.wr_col === 1.U, + "REDMULE_F16 invalid bank groups" + ) + assert( + cmd.rs2(63, 43) === 0.U && m =/= 0.U && cmd.iter =/= 0.U && cmd.iter(2, 0) === 0.U, + "REDMULE_F16 invalid dimensions or reserved bits" + ) + assert( + cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, + "REDMULE_F16 banks must differ" + ) + for ( + (bank, footprint) <- Seq( + (cmd.op1_bank, cmd.rs2(21, 12) +& m * (cmd.iter >> 3)), + (cmd.op2_bank, cmd.rs2(31, 22) +& cmd.iter), + (cmd.wr_bank, cmd.rs2(41, 32) +& m) + ) + ) { + val shared = bank >= fp.shared_bank_id_base.U + val capacity = Mux( + shared, + mp.sharedBankEntries.U, + mp.bankEntries.U + ) + assert(footprint <= capacity, "REDMULE_F16 bank window exceeds capacity") + } + issue := blink.cmdReq.bits + finished := false.B + controlStep := 0.U + state := channels + } + when(state === channels)(state := configure) + when(state === configure && native.io.ctrl_gnt_o)(state := controlReply) + when(state === controlReply && native.io.ctrl_valid_o) { + when(controlStep === 0.U) { + assert(native.io.ctrl_data_o =/= "hffffffff".U, "RedMulE has no free job context") + } + when(controlStep === 8.U)(state := running).otherwise { + controlStep := controlStep + 1.U + state := configure + } + } + when(state === running && native.io.evt_o)(finished := true.B) + when(state === running && finished && !native.io.busy_o && busState === busIdle)(state := complete) + when(blink.cmdResp.fire)(state := idle) +} + +object RedmuleBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new RedmuleBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/components/balls/redmule/arch/RedmuleNative.scala b/examples/components/balls/redmule/arch/RedmuleNative.scala new file mode 100644 index 000000000..acaa36f24 --- /dev/null +++ b/examples/components/balls/redmule/arch/RedmuleNative.scala @@ -0,0 +1,96 @@ +package examples.balls.redmule + +import chisel3.experimental.hierarchy.{instantiable, public} + +import chisel3._ +import chisel3.util.HasBlackBoxInline + +@instantiable +class RedmuleNative extends BlackBox with HasBlackBoxInline { + + @public + val io = IO(new Bundle { + val clk_i = Input(Clock()) + val rst_ni = Input(Bool()) + val busy_o = Output(Bool()) + val evt_o = Output(Bool()) + val ctrl_req_i = Input(Bool()) + val ctrl_wen_i = Input(Bool()) + val ctrl_addr_i = Input(UInt(32.W)) + val ctrl_data_i = Input(UInt(32.W)) + val ctrl_gnt_o = Output(Bool()) + val ctrl_valid_o = Output(Bool()) + val ctrl_data_o = Output(UInt(32.W)) + val mem_req_o = Output(Bool()) + val mem_wen_o = Output(Bool()) + val mem_addr_o = Output(UInt(32.W)) + val mem_be_o = Output(UInt(32.W)) + val mem_data_o = Output(UInt(256.W)) + val mem_id_o = Output(UInt(8.W)) + val mem_gnt_i = Input(Bool()) + val mem_valid_i = Input(Bool()) + val mem_data_i = Input(UInt(256.W)) + val mem_id_i = Input(UInt(8.W)) + val mem_ready_o = Output(Bool()) + }) + + // Locate RTL from the source checkout, independent of the elaboration directory. + val sourceDir = os.Path(sourcecode.File()) / os.up + val repoRoot = sourceDir / os.up / os.up / os.up / os.up / os.up + val thirdparty = repoRoot / "arch" / "thirdparty" + val common = thirdparty / "common_cells" + val fp = thirdparty / "cvfpu" / "src" + val hci = thirdparty / "hci" / "rtl" + val stream = thirdparty / "hwpe-stream" / "rtl" + val ctrl = thirdparty / "hwpe-ctrl" / "rtl" + val native = thirdparty / "redmule" / "rtl" + for (name <- Seq("assertions.svh", "registers.svh")) { + setInline(s"common_cells/$name", os.read(common / "include" / "common_cells" / name)) + } + setInline("hci_helpers.svh", os.read(hci / "common" / "hci_helpers.svh")) + + val packages = Seq( + common / "src" / "cf_math_pkg.sv", + fp / "fpnew_pkg.sv", + stream / "hwpe_stream_package.sv", + ctrl / "hwpe_ctrl_package.sv", + hci / "common" / "hci_package.sv", + native / "redmule_pkg.sv", + native / "ctrl" / "regif" / "redmule_regif_pkg.sv", + stream / "hwpe_stream_interfaces.sv", + ctrl / "hwpe_ctrl_interfaces.sv", + hci / "common" / "hci_interfaces.sv" + ) + + val primitives = Seq( + common / "src" / "lzc.sv", + common / "src" / "edge_detect.sv", + common / "src" / "sync.sv", + common / "src" / "sync_wedge.sv", + common / "src" / "fifo_v3.sv", + thirdparty / "tech_cells_generic" / "src" / "rtl" / "tc_clk.sv", + thirdparty / "tech_cells_generic" / "src" / "deprecated" / "cluster_clk_cells.sv", + thirdparty / "tech_cells_generic" / "src" / "deprecated" / "pulp_clk_cells.sv" + ) ++ Seq("fpnew_classifier.sv", "fpnew_rounding.sv", "fpnew_cast_multi.sv", "lfsr_sr.sv").map(fp / _) + + val streamSources = + Seq("basic", "fifo", "streamer", "tcdm").flatMap(dir => os.list(stream / dir).filter(_.ext == "sv")) + val hciSources = + os.walk(hci / "core").filter(_.ext == "sv") ++ Seq("hci_ecc_enc.sv", "hci_ecc_dec.sv").map(hci / "ecc" / _) + val controlSources = Seq("hwpe_ctrl_target.sv", "hwpe_ctrl_seq_mult.sv", "hwpe_ctrl_partial_mult.sv").map(ctrl / _) + + val nativeSources = os.walk(native).filter(p => + p.ext == "sv" && !packages.contains(p) && + !Seq("redmule_xif_wrap.sv", "redmule_mm_wrap.sv", "redmule_inst_decoder.sv").contains(p.last) + ) + + val wrapper = sourceDir / "RedmuleNative.sv" + // Copy the upstream files verbatim; only the external wrapper is authored here. + setInline( + "RedmuleNative.sv", + "/* verilator lint_off ENUMVALUE */\n" + + (packages ++ primitives ++ streamSources ++ hciSources ++ + controlSources ++ nativeSources).map(os.read(_)).mkString("\n") + + "\n/* verilator lint_on ENUMVALUE */\n" + os.read(wrapper) + ) +} diff --git a/examples/components/balls/redmule/arch/RedmuleNative.sv b/examples/components/balls/redmule/arch/RedmuleNative.sv new file mode 100644 index 000000000..ab5fa615d --- /dev/null +++ b/examples/components/balls/redmule/arch/RedmuleNative.sv @@ -0,0 +1,117 @@ +module RedmuleNative ( + input logic clk_i, + rst_ni, + output logic busy_o, + evt_o, + input logic ctrl_req_i, + ctrl_wen_i, + input logic [31:0] ctrl_addr_i, + ctrl_data_i, + output logic ctrl_gnt_o, + ctrl_valid_o, + output logic [31:0] ctrl_data_o, + output logic mem_req_o, + mem_wen_o, + output logic [31:0] mem_addr_o, + mem_be_o, + output logic [255:0] mem_data_o, + output logic [7:0] mem_id_o, + input logic mem_gnt_i, + mem_valid_i, + input logic [255:0] mem_data_i, + input logic [7:0] mem_id_i, + output logic mem_ready_o +); + localparam hci_package::hci_size_parameter_t HCI_SIZE = '{ + DW: 256, + AW: 32, + BW: 8, + UW: 1, + IW: 8, + EW: 1, + EHW: 1 + }; + hci_core_intf #( + .DW (256), + .AW (32), + .BW (8), + .UW (1), + .IW (8), + .EW (1), + .EHW(1) + ) memory ( + clk_i + ); + hwpe_ctrl_intf_periph #(.ID_WIDTH(4)) control (clk_i); + hwpe_stream_intf_stream #(.DATA_WIDTH(256)) + x_in (clk_i), w_in (clk_i), x_out (clk_i), w_out (clk_i); + assign x_in.valid = 0; + assign x_in.data = 0; + assign x_in.strb = 0; + assign w_in.valid = 0; + assign w_in.data = 0; + assign w_in.strb = 0; + assign x_out.ready = 1; + assign w_out.ready = 1; + assign control.req = ctrl_req_i; + assign control.wen = ctrl_wen_i; + assign control.add = ctrl_addr_i; + assign control.data = ctrl_data_i; + assign control.be = 4'hf; + assign control.id = 0; + assign ctrl_gnt_o = control.gnt; + assign ctrl_valid_o = control.r_valid; + assign ctrl_data_o = control.r_data; + assign mem_req_o = memory.req; + assign mem_wen_o = memory.wen; + assign mem_addr_o = memory.add; + assign mem_be_o = memory.be; + assign mem_data_o = memory.data; + assign mem_id_o = memory.id; + assign mem_ready_o = memory.r_ready; + assign memory.gnt = mem_gnt_i; + assign memory.r_valid = mem_valid_i; + assign memory.r_data = mem_data_i; + assign memory.r_id = mem_id_i; + assign memory.r_user = 0; + assign memory.r_opc = 0; + assign memory.r_ecc = 0; + assign memory.egnt = {1{mem_gnt_i}}; + assign memory.r_evalid = {1{mem_valid_i}}; + redmule_top #( + .DataW(256), + .Height(8), + .Width(8), + .NumPipeRegs(1), + .FpFormat(fpnew_pkg::FP16), + .PipeConfig(fpnew_pkg::DISTRIBUTED), + .CtrlIntfConfig(redmule_pkg::HWPE_TARGET), + .HCI_SIZE_tcdm(HCI_SIZE) + ) native_i ( + .clk_i, + .rst_ni, + .test_mode_i(1'b0), + .busy_o, + .evt_o, + .w_stream_i(w_in), + .x_stream_i(x_in), + .w_stream_o(w_out), + .x_stream_o(x_out), + .x_issue_req_i(1'b0), + .x_issue_resp_o(), + .x_issue_valid_i(1'b0), + .x_issue_ready_o(), + .x_register_i(1'b0), + .x_register_valid_i(1'b0), + .x_register_ready_o(), + .x_commit_i(1'b0), + .x_commit_valid_i(1'b0), + .x_result_o(), + .x_result_valid_o(), + .x_result_ready_i(1'b0), + .sync_o(), + .sync_i(1'b0), + .tcdm(memory), + .target(control) + ); +endmodule diff --git a/examples/components/balls/redmule/configs/default.toml b/examples/components/balls/redmule/configs/default.toml new file mode 100644 index 000000000..5412bfcba --- /dev/null +++ b/examples/components/balls/redmule/configs/default.toml @@ -0,0 +1 @@ +lanes = 8 diff --git a/examples/components/balls/redmule/configs/redmule_params.proto b/examples/components/balls/redmule/configs/redmule_params.proto new file mode 100644 index 000000000..469d725d5 --- /dev/null +++ b/examples/components/balls/redmule/configs/redmule_params.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; +package buckyball.balls.redmule; +option java_package = "buckyball.balls.redmule"; +option java_multiple_files = true; + +message Params { + optional int32 lanes = 1; +} diff --git a/examples/components/balls/redmule/emu/src/fp16.rs b/examples/components/balls/redmule/emu/src/fp16.rs new file mode 100644 index 000000000..f476b5e3d --- /dev/null +++ b/examples/components/balls/redmule/emu/src/fp16.rs @@ -0,0 +1,71 @@ +// Finite FP16 values are exact signed integers in units of 2^-24. +fn finite(bits: u16) -> i128 { + let exponent = (bits >> 10) & 31; + let mantissa = (bits & 1023) as i128; + let value = if exponent == 0 { + mantissa + } else { + (1024 + mantissa) << (exponent - 1) + }; + if bits & 0x8000 != 0 { + -value + } else { + value + } +} + +pub(super) fn fma(a: u16, b: u16, c: u16) -> u16 { + let aa = a & 0x7fff; + let bb = b & 0x7fff; + let cc = c & 0x7fff; + let product_sign = (a ^ b) & 0x8000; + if aa > 0x7c00 + || bb > 0x7c00 + || cc > 0x7c00 + || (aa == 0x7c00 && bb == 0) + || (bb == 0x7c00 && aa == 0) + { + return 0x7e00; + } + if aa == 0x7c00 || bb == 0x7c00 { + return if cc == 0x7c00 && c & 0x8000 != product_sign { + 0x7e00 + } else { + product_sign | 0x7c00 + }; + } + if cc == 0x7c00 { + return c; + } + // The largest product uses fewer than 81 bits; i128 retains every bit. + let sum = finite(a) * finite(b) + (finite(c) << 24); + if sum == 0 { + return if (aa == 0 || bb == 0) && cc == 0 { + product_sign & c & 0x8000 + } else { + 0 + }; + } + let sign = if sum < 0 { 0x8000 } else { 0 }; + let magnitude = sum.unsigned_abs(); + let highest = 127 - magnitude.leading_zeros(); + let shift = if highest < 34 { 24 } else { highest - 10 }; + let mut significand = magnitude >> shift; + let remainder = magnitude & ((1u128 << shift) - 1); + let halfway = 1u128 << (shift - 1); + if remainder > halfway || (remainder == halfway && significand & 1 != 0) { + significand += 1; + } + if highest < 34 { + return sign | significand as u16; + } + let mut exponent = highest as i32 - 48 + 15; + if significand == 2048 { + significand = 1024; + exponent += 1; + } + if exponent >= 31 { + return sign | 0x7c00; + } + sign | (exponent as u16) << 10 | (significand as u16 & 1023) +} diff --git a/examples/components/balls/redmule/emu/src/lib.rs b/examples/components/balls/redmule/emu/src/lib.rs new file mode 100644 index 000000000..d59b6c020 --- /dev/null +++ b/examples/components/balls/redmule/emu/src/lib.rs @@ -0,0 +1,36 @@ +mod fp16; +mod matmul; + +use bebop_bemu::ball::ExecContext; + +const CLASS: &str = "examples.balls.redmule.RedmuleBall"; + +pub fn execute_known( + class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if class != CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("REDMULE_F16") => Some(matmul::execute(xs1, xs2, ctx)), + _ => None, + } +} + +pub fn cycles_after_issue(class: &str, funct: u32, _xs1: u64, _xs2: u64) -> Option { + if class != CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("REDMULE_F16") => Some(1), + _ => None, + } +} + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/components/balls/redmule/emu/src/matmul.rs b/examples/components/balls/redmule/emu/src/matmul.rs new file mode 100644 index 000000000..2855ef6c2 --- /dev/null +++ b/examples/components/balls/redmule/emu/src/matmul.rs @@ -0,0 +1,64 @@ +use bebop_bemu::thread::instructions::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use bebop_bemu::ball::ExecContext; +pub(super) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + let a = rs1_b0(xs1); + let b = rs1_b1(xs1); + let c = rs1_b2(xs1); + let rows = (xs2 & 4095) as usize; + let k = rs1_iter(xs1) as usize; + let a_base = ((xs2 >> 12) & 1023) as usize; + let b_base = ((xs2 >> 22) & 1023) as usize; + let c_base = ((xs2 >> 32) & 1023) as usize; + let first = xs2 & (1 << 42) != 0; + assert_eq!(xs2 >> 43, 0, "redmule: reserved bits must be zero"); + assert!( + rows > 0 && k > 0 && k % 8 == 0, + "redmule: M > 0, K must be a positive multiple of eight" + ); + assert!(a != b && a != c && b != c, "redmule: banks must differ"); + assert_eq!( + bebop_bemu::configs::bank_width(), + 128, + "redmule: bank rows must be 128 bits" + ); + assert_eq!(bebop_bemu::configs::ball_domain::parameters::("redmule", ctx.ball_id.expect("Ball instance")).lanes.expect("lanes") as usize, 8); + for bank in [a, b, c] { + assert!( + ctx.config(bank).allocated && ctx.config(bank).cols == 1, + "redmule: banks must be allocated with one group" + ); + } + let pa = pbank(ctx, a); + let pb = pbank(ctx, b); + let pc = pbank(ctx, c); + assert!( + a_base + rows * (k / 8) <= ctx.banks[pa].len() / 16 + && b_base + k <= ctx.banks[pb].len() / 16 + && c_base + rows <= ctx.banks[pc].len() / 16, + "redmule: bank window exceeds capacity" + ); + let mut sums = vec![0; rows * 8]; + if !first { + for (index, value) in sums.iter_mut().enumerate() { + let offset = c_base * 16 + index * 2; + *value = u16::from_le_bytes(ctx.banks[pc][offset..offset + 2].try_into().unwrap()); + } + } + for row in 0..rows { + for inner in 0..k { + let offset = a_base * 16 + (row * k + inner) * 2; + let lhs = u16::from_le_bytes(ctx.banks[pa][offset..offset + 2].try_into().unwrap()); + for col in 0..8 { + let offset = (b_base + inner) * 16 + col * 2; + let rhs = u16::from_le_bytes(ctx.banks[pb][offset..offset + 2].try_into().unwrap()); + let index = row * 8 + col; + sums[index] = super::fp16::fma(lhs, rhs, sums[index]); + } + } + } + for (index, value) in sums.iter().enumerate() { + let offset = c_base * 16 + index * 2; + ctx.banks[pc][offset..offset + 2].copy_from_slice(&value.to_le_bytes()); + } + 0 +} diff --git a/examples/components/balls/redmule/stack/src/Lowering.cpp b/examples/components/balls/redmule/stack/src/Lowering.cpp new file mode 100644 index 000000000..306e5fd90 --- /dev/null +++ b/examples/components/balls/redmule/stack/src/Lowering.cpp @@ -0,0 +1,68 @@ +//===- Lowering.cpp - RedmuleBall in lower-buckyball ----------------------===// +// +// matmul_f16: C = A @ B in FP16 with ordered FMAs, or C = C + A @ B with +// read_destination. A (source 0) holds `rows` rows of `k` f16 from bank row +// op1_base, k / 8 bank rows each; B (source 1) holds `k` rows of eight f16 +// from bank row op2_base; C receives `rows` rows of eight f16 at bank row +// wr_base. `k` is a positive multiple of eight. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballOps.h" +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class MatmulF16Lowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &rewriter) const override { + if (op.getKind() != "matmul_f16") + return failure(); + int64_t rows = cast(op->getAttr("rows")).getInt(); + int64_t k = cast(op->getAttr("k")).getInt(); + int64_t op1Base = cast(op->getAttr("op1_base")).getInt(); + int64_t op2Base = cast(op->getAttr("op2_base")).getInt(); + int64_t wrBase = cast(op->getAttr("wr_base")).getInt(); + int64_t c = op.getDestination(); + const auto &target = buckyball_target::getBuckyballTarget(); + if (op.getSources().size() != 2 || + op.getSources()[0] == op.getSources()[1] || + llvm::is_contained(op.getSources(), c)) + return op.emitOpError("FP16 matmul requires distinct A, B and C banks"); + if (target.bankWidthBits != 128 || rows <= 0 || rows > 4095 || k <= 0 || + k % 8 || k >= (1LL << 33) || op1Base < 0 || op1Base > 1023 || + op2Base < 0 || op2Base > 1023 || wrBase < 0 || wrBase > 1023 || + op1Base + rows * (k / 8) > target.bankDepth || + op2Base + k > target.bankDepth || wrBase + rows > target.bankDepth) + return op.emitOpError("FP16 matmul shape exceeds the bank or ISA " + "geometry"); + // The Ball reads the old C itself when not first; rs1 does not mark it. + uint64_t rs2 = mlir::buddy::field(rows, 0, 11) | + mlir::buddy::field(op1Base, 12, 21) | + mlir::buddy::field(op2Base, 22, 31) | + mlir::buddy::field(wrBase, 32, 41) | + mlir::buddy::field(!op.getReadDestination(), 42, 42); + mlir::buddy::emitInstruction( + rewriter, op.getLoc(), + buckyball_target::getBuckyballFunct7("REDMULE_F16"), + mlir::buddy::constantI64( + rewriter, op.getLoc(), + mlir::buddy::bankFields(op.getSources(), c, false, k)), + mlir::buddy::constantI64(rewriter, op.getLoc(), rs2)); + rewriter.eraseOp(op); + return success(); + } +}; + +void populate(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} + +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallLowering(populate), true); +} // namespace diff --git a/examples/components/balls/redmule/verify/casegen/Cargo.lock b/examples/components/balls/redmule/verify/casegen/Cargo.lock new file mode 100644 index 000000000..598b7251f --- /dev/null +++ b/examples/components/balls/redmule/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "redmule-casegen" +version = "0.1.0" diff --git a/examples/components/balls/redmule/verify/casegen/Cargo.toml b/examples/components/balls/redmule/verify/casegen/Cargo.toml new file mode 100644 index 000000000..363e11dec --- /dev/null +++ b/examples/components/balls/redmule/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "redmule-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/components/balls/redmule/verify/casegen/src/lib.rs b/examples/components/balls/redmule/verify/casegen/src/lib.rs new file mode 100644 index 000000000..b60a07e35 --- /dev/null +++ b/examples/components/balls/redmule/verify/casegen/src/lib.rs @@ -0,0 +1,118 @@ +#[path = "../../../emu/src/fp16.rs"] +mod fp16; +use std::cell::RefCell; + +struct Case { + rows: usize, + k: usize, + base: usize, + blocks: usize, + a: Vec, + b: Vec, + c: Vec, +} +thread_local! { + static CASE: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn redmule_case_load(index: u32, lanes: u32) { + assert_eq!(lanes, 8); + let rows = if index >= 64 { + [16, 16, 32][index as usize - 64] + } else { + index as usize % 3 + 1 + }; + let k = if index >= 64 { + [16, 32, 32][index as usize - 64] + } else if index & 1 == 0 { + 8 + } else { + 16 + }; + let base = if index & 1 == 0 { 0 } else { 2 }; + let blocks = if index == 2 { 3 } else { 1 }; + let mut a = vec![0x7e00; base * 8 + rows * k]; + let mut b = vec![0x7e00; (base + k) * 8]; + let mut rng = 0xcafe_babeu32 ^ index; + for value in a[base * 8..].iter_mut().chain(b[base * 8..].iter_mut()) { + rng ^= rng << 13; + rng ^= rng >> 17; + rng ^= rng << 5; + *value = if index < 3 || index >= 64 { + [0x3c00, 0xbc00, 0x3800, 0xb800][rng as usize & 3] + } else { + rng as u16 + }; + } + if index == 3 { + a[base * 8..].fill(0); + b[base * 8..].fill(0); + for row in 0..rows { + a[base * 8 + row * k] = 0x3c00; + } + b[base * 8..base * 8 + 8] + .copy_from_slice(&[0, 1, 0x3c00, 0x7c00, 0x7d00, 0xbc00, 0x7bff, 0x8001]); + } + let mut c = vec![0; rows * 8]; + let mut outputs = Vec::new(); + for _ in 0..blocks { + for row in 0..rows { + for inner in 0..k { + for col in 0..8 { + let i = row * 8 + col; + c[i] = fp16::fma( + a[base * 8 + row * k + inner], + b[(base + inner) * 8 + col], + c[i], + ); + } + } + } + outputs.extend_from_slice(&c); + } + CASE.with(|slot| { + *slot.borrow_mut() = Some(Case { + rows, + k, + base, + blocks, + a, + b, + c: outputs, + }) + }); +} + +// Return rows, K, row base, and command count in successive 16-bit fields. +#[no_mangle] +pub extern "C" fn redmule_case_shape() -> u64 { + CASE.with(|slot| { + let slot = slot.borrow(); + let case = slot.as_ref().expect("case not loaded"); + case.rows as u64 + | (case.k as u64) << 16 + | (case.base as u64) << 32 + | (case.blocks as u64) << 48 + }) +} + +#[no_mangle] +pub extern "C" fn redmule_case_word(bank: u32, row: u32, half: u32) -> u64 { + CASE.with(|slot| { + let slot = slot.borrow(); + let case = slot.as_ref().expect("case not loaded"); + let data = match bank { + 0 => &case.a, + 1 => &case.b, + 2 => &case.c, + _ => panic!("invalid case bank"), + }; + let offset = row as usize * 8 + half as usize * 4; + let mut word = 0; + for lane in 0..4 { + word |= (data[offset + lane] as u64) << (lane * 16); + } + word + }) +} diff --git a/examples/components/balls/redmule/verify/filelists/redmule_ball.f b/examples/components/balls/redmule/verify/filelists/redmule_ball.f new file mode 100644 index 000000000..3ebb60a92 --- /dev/null +++ b/examples/components/balls/redmule/verify/filelists/redmule_ball.f @@ -0,0 +1,13 @@ ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+@RTL@ +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv +-F @RTL@/filelist.f +src/reference/redmule_reference.sv +src/pkg/redmule_pkg.sv +src/tb_top.sv diff --git a/examples/components/balls/redmule/verify/parameters.toml b/examples/components/balls/redmule/verify/parameters.toml new file mode 100644 index 000000000..6cfae947e --- /dev/null +++ b/examples/components/balls/redmule/verify/parameters.toml @@ -0,0 +1,2 @@ +[defines] +REDMULE_LANES = "lanes" diff --git a/examples/components/balls/redmule/verify/src/pkg/redmule_pkg.sv b/examples/components/balls/redmule/verify/src/pkg/redmule_pkg.sv new file mode 100644 index 000000000..7de087b0f --- /dev/null +++ b/examples/components/balls/redmule/verify/src/pkg/redmule_pkg.sv @@ -0,0 +1,351 @@ +package redmule_uvm_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + import "DPI-C" function void redmule_case_load( + input int unsigned index, + lanes + ); + import "DPI-C" function longint unsigned redmule_case_shape(); + import "DPI-C" function longint unsigned redmule_case_word( + input int unsigned bank, + row, + half + ); + + class redmule_mem_model extends bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(redmule_mem_model) + int read_stalls[2], write_stalls, response_stalls; + bit inject_stalls = 1; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + // Both performance ports model one-cycle memory; only functional cases inject stalls. + task run_phase(uvm_phase phase); + forever begin + @(posedge vif.clock); + if (vif.reset) begin + for (int port = 0; port < 2; port++) begin + vif.bank_read_req_ready[port] <= 1; + vif.bank_read_resp_valid[port] <= 0; + end + vif.bank_write_req_ready[0] <= 1; + vif.bank_write_resp_valid[0] <= 0; + vif.bank_write_resp_ok[0] <= 1; + vif.cmd_resp_ready <= 1; + vif.sub_rob_req_ready <= 1; + end else begin + for (int port = 0; port < 2; port++) begin + if (vif.bank_read_resp_valid[port] && vif.bank_read_resp_ready[port]) + vif.bank_read_resp_valid[port] <= 0; + if (vif.bank_read_req_valid[port] && vif.bank_read_req_ready[port]) begin + if (!mem.exists( + vif.bank_read_bank_id[port] + ) || !mem[vif.bank_read_bank_id[port]][0].exists( + vif.bank_read_req_addr[port] + )) + `uvm_fatal("MEM", "bank read was not preloaded") + vif.bank_read_resp_data[port] <= mem[vif.bank_read_bank_id[port]][0][vif.bank_read_req_addr[port]]; + vif.bank_read_resp_valid[port] <= 1; + end + vif.bank_read_req_ready[port] <= (!vif.bank_read_resp_valid[port] || vif.bank_read_resp_ready[port]) && + (!inject_stalls || $urandom_range( + 0, 3 + ) != 0); + if (vif.bank_read_req_valid[port] && !vif.bank_read_req_ready[port]) + read_stalls[port]++; + end + if (vif.bank_write_resp_valid[0] && vif.bank_write_resp_ready[0]) + vif.bank_write_resp_valid[0] <= 0; + if (vif.bank_write_req_valid[0] && vif.bank_write_req_ready[0]) + vif.bank_write_resp_valid[0] <= 1; + vif.bank_write_req_ready[0] <= (!vif.bank_write_resp_valid[0] || vif.bank_write_resp_ready[0]) && + (!inject_stalls || $urandom_range( + 0, 3 + ) != 0); + vif.cmd_resp_ready <= (!inject_stalls || $urandom_range(0, 3) != 0); + if (vif.bank_write_req_valid[0] && !vif.bank_write_req_ready[0]) write_stalls++; + if (vif.cmd_resp_valid && !vif.cmd_resp_ready) response_stalls++; + end + end + endtask + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!read_stalls[0] || !read_stalls[1] || !write_stalls || !response_stalls) + `uvm_fatal("STALL", "missing read/write/completion backpressure coverage") + endfunction + endclass + + class redmule_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(redmule_seq) + int bid, rows, k, base, block_index, blocks; + function new(string name = "redmule_seq"); + super.new(name); + endfunction + task body(); + bb_blink_cmd_item item = bb_blink_cmd_item::type_id::create("item"); + start_item(item); + item.bid = bid; + item.funct7 = `REDMULE_F16_FUNCT7; + item.iter = k; + item.op1_en = 1; + item.op2_en = 1; + item.wr_spad_en = 1; + item.op1_from_spad = 1; + item.op2_from_spad = 1; + item.op1_bank = 0; + item.op2_bank = 1; + item.wr_bank = 2; + item.op1_col = 1; + item.op2_col = 1; + item.wr_col = 1; + item.rs1 = (longint'(k) << 30) | (2 << 20) | (1 << 10); + item.rs2 = longint'(rows) | (longint'(base) << 12) | + (longint'(base) << 22) | (longint'(base) << 32) | + (longint'(block_index == 0) << 42); + item.special = item.rs2; + item.rob_id = block_index; + finish_item(item); + endtask + endclass + + class redmule_scoreboard extends uvm_scoreboard; + `uvm_component_utils(redmule_scoreboard) + uvm_analysis_imp_cmd #(bb_blink_cmd_item, redmule_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, redmule_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, redmule_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, redmule_scoreboard) resp_imp; + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + int rows, k, base, blocks, block_index; + int reads[2], writes, commands, responses; + bit first; + bit a_seen[int], b_seen[int], c_seen[int]; + int m_bucket, k_bucket, base_bucket, number_class; + covergroup contract_cg; + coverpoint first {bins start = {1}; bins continuation = {0};} + coverpoint m_bucket {bins shapes[] = {[1 : 3]}; bins tile_rows[] = {16, 32};} + coverpoint k_bucket {bins short_k = {8}; bins long_k = {16}; bins large_k = {32};} + coverpoint base_bucket {bins zero = {0}; bins offset = {2};} + endgroup + covergroup number_cg; + coverpoint number_class {bins classes[] = {[0 : 4]};} + endgroup + function new(string name, uvm_component parent); + super.new(name, parent); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + contract_cg = new(); + number_cg = new(); + endfunction + function void write_cmd(bb_blink_cmd_item item); + if (item.funct7 != `REDMULE_F16_FUNCT7 || item.iter != k || + item.rs2[11:0] != rows || item.rob_id != block_index) + `uvm_fatal("CMD", "unexpected RedMulE command") + first = item.rs2[42]; + m_bucket = rows; + k_bucket = k; + base_bucket = base; + contract_cg.sample(); + commands++; + endfunction + function void write_read(bb_blink_read_item item); + int limit; + if (item.group_id != 0 || item.rob_id != block_index) + `uvm_fatal("READ", "unexpected read metadata") + if (item.port == 1 && item.bank_id == 1) begin + limit = base + k; + b_seen[item.addr] = 1; + end else if (item.port == 0 && item.bank_id == 0) begin + limit = base + rows * k / 8; + a_seen[item.addr] = 1; + end else if (item.port == 0 && item.bank_id == 2 && !first) begin + limit = base + rows; + c_seen[item.addr] = 1; + end else `uvm_fatal("READ", "unexpected read bank") + if (item.addr < base || item.addr >= limit) `uvm_fatal("READ", "read outside bank window") + reads[item.port]++; + endfunction + function void write_write(bb_blink_write_item item); + bit [127:0] expected; + bit [ 15:0] value; + if (writes >= rows) `uvm_fatal("WRITE", "unexpected early/extra write") + expected = { + redmule_case_word(2, block_index * rows + writes, 1), + redmule_case_word(2, block_index * rows + writes, 0) + }; + if (item.port != 0 || item.bank_id != 2 || item.addr != base + writes || + item.rob_id != block_index || item.group_id != 0 || item.mask != 16'hffff || + item.data !== expected) + `uvm_fatal("WRITE", $sformatf( + "row %0d got %032h expected %032h", item.addr, item.data, expected)) + for (int lane = 0; lane < 8; lane++) begin + value = item.data[lane*16+:16] & 16'h7fff; + if (value == 0) number_class = 0; + else if (value < 16'h0400) number_class = 1; + else if (value < 16'h7c00) number_class = 2; + else if (value == 16'h7c00) number_class = 3; + else number_class = 4; + number_cg.sample(); + end + mem_model.load_word(2, item.addr, item.data); + writes++; + endfunction + function void write_resp(bb_blink_resp_item item); + if (item.rob_id != block_index || item.is_sub != 0 || item.sub_rob_id != 0 || writes != rows) + `uvm_fatal("RESP", "completion preceded required reads/writes") + for (int row = base; row < base + rows * k / 8; row++) + if (!a_seen.exists(row)) `uvm_fatal("READ", "missing A row") + for (int row = base; row < base + k; row++) + if (!b_seen.exists(row)) `uvm_fatal("READ", "missing B row") + if (!first) + for (int row = base; row < base + rows; row++) + if (!c_seen.exists(row)) `uvm_fatal("READ", "missing C accumulator row") + responses++; + endfunction + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (commands != responses || commands == 0) `uvm_fatal("SCB", "missing command completion") + if (contract_cg.get_coverage() != 100 || number_cg.get_coverage() != 100) + `uvm_fatal( + "COV", $sformatf( + "contract %0.2f%% numbers %0.2f%%", contract_cg.get_coverage(), number_cg.get_coverage() + )) + endfunction + endclass + + class redmule_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(redmule_env) + redmule_scoreboard scb; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + bb_blink_mem_model#(`BB_IN_BW, `BB_OUT_BW)::type_id::set_type_override( + redmule_mem_model::get_type()); + super.build_phase(phase); + scb = redmule_scoreboard::type_id::create("scb", this); + endfunction + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + scb.mem_model = mem_model; + cmd_agent.cmd_ap.connect(scb.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction + endclass + + class redmule_ball_test extends uvm_test; + `uvm_component_utils(redmule_ball_test) + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + redmule_env env; + virtual redmule_reference_if reference; + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + if (!uvm_config_db#(virtual redmule_reference_if)::get(this, "", "reference", reference)) + `uvm_fatal("NOVIF", "native reference interface not found") + env = redmule_env::type_id::create("env", this); + endfunction + task run_phase(uvm_phase phase); + int bid, rows, k, base, blocks, before_response, cycles; + longint unsigned shape; + longint native_cycles, ball_started, ball_cycles, total_native = 0, total_ball = 0; + redmule_seq seq; + redmule_mem_model memory_agent; + if (!$cast(memory_agent, env.mem_model)) `uvm_fatal("MEM", "wrong memory model type") + if (!$value$plusargs("BID=%d", bid)) `uvm_fatal("BID", "missing BID") + phase.raise_objection(this); + for (int index = 0; index < 67; index++) begin + vif.reset = 1; + repeat (5) @(posedge vif.clock); + vif.reset = 0; + repeat (2) @(posedge vif.clock); + memory_agent.inject_stalls = index < 64; + redmule_case_load(index, `REDMULE_LANES); + shape = redmule_case_shape(); + rows = shape[15:0]; + k = shape[31:16]; + base = shape[47:32]; + blocks = shape[63:48]; + env.scb.rows = rows; + env.scb.k = k; + env.scb.base = base; + env.scb.blocks = blocks; + env.mem_model.clear_mem(); + for (int row = 0; row < base + rows * k / 8; row++) begin + env.mem_model.load_word(0, row, { + redmule_case_word(0, row, 1), redmule_case_word(0, row, 0)}); + reference.load_word(0, row, {redmule_case_word(0, row, 1), redmule_case_word(0, row, 0)}); + end + for (int row = 0; row < base + k; row++) begin + env.mem_model.load_word(1, row, { + redmule_case_word(1, row, 1), redmule_case_word(1, row, 0)}); + reference.load_word(1, row, {redmule_case_word(1, row, 1), redmule_case_word(1, row, 0)}); + end + for (int row = 0; row < base + rows; row++) begin + env.mem_model.load_word(2, row, 0); + reference.load_word(2, row, 0); + end + env.mem_model.arm(); + for (int block_index = 0; block_index < blocks; block_index++) begin + env.scb.block_index = block_index; + env.scb.reads = '{0, 0}; + env.scb.writes = 0; + env.scb.a_seen.delete(); + env.scb.b_seen.delete(); + env.scb.c_seen.delete(); + before_response = env.scb.responses; + seq = redmule_seq::type_id::create("seq"); + seq.bid = bid; + seq.rows = rows; + seq.k = k; + seq.base = base; + seq.block_index = block_index; + seq.blocks = blocks; + reference.execute(rows, k, base, block_index == 0, native_cycles); + for (int row = 0; row < rows; row++) + if (reference.read_word( + base + row + ) !== {redmule_case_word( + 2, block_index * rows + row, 1 + ), redmule_case_word( + 2, block_index * rows + row, 0 + )}) + `uvm_fatal("REFERENCE", "native engine disagrees with golden") + ball_started = reference.get_ticks(); + seq.start(env.cmd_agent.seqr); + cycles = 0; + while (env.scb.responses == before_response) begin + @(posedge vif.clock); + if (++cycles > 200000) `uvm_fatal("TIMEOUT", "RedMulE command timed out") + end + ball_cycles = reference.get_ticks() - ball_started; + total_native += native_cycles; + total_ball += ball_cycles; + `uvm_info("PERF", $sformatf( + "M=%0d K=%0d N=8 first=%0d native=%0d ball=%0d", + rows, + k, + block_index == 0, + native_cycles, + ball_cycles + ), UVM_LOW) + repeat (2) @(posedge vif.clock); + end + `uvm_info("REDMULE", $sformatf("case %0d passed", index), UVM_LOW) + end + `uvm_info("PERF_TOTAL", $sformatf("native=%0d ball=%0d", total_native, total_ball), UVM_LOW) + phase.drop_objection(this); + endtask + endclass +endpackage diff --git a/examples/components/balls/redmule/verify/src/reference/redmule_reference.sv b/examples/components/balls/redmule/verify/src/reference/redmule_reference.sv new file mode 100644 index 000000000..26a179f65 --- /dev/null +++ b/examples/components/balls/redmule/verify/src/reference/redmule_reference.sv @@ -0,0 +1,130 @@ +module redmule_reference ( + input logic clock, + reset +); + logic control_req = 0, control_wen; + logic [31:0] control_addr, control_data, control_result; + logic control_grant, control_valid, busy, event_done; + logic request, load, ready; + logic [31:0] address, enables; + logic [255:0] write_data, response_data; + logic [7:0] request_id, response_id; + logic response_valid = 0; + bit [127:0] memory[3][1 << `BB_BANK_ADDR_W]; + int rows, reduction, row_base; + longint ticks = 0; + + RedmuleNative original ( + .clk_i(clock), + .rst_ni(!reset), + .busy_o(busy), + .evt_o(event_done), + .ctrl_req_i(control_req), + .ctrl_wen_i(control_wen), + .ctrl_addr_i(control_addr), + .ctrl_data_i(control_data), + .ctrl_gnt_o(control_grant), + .ctrl_valid_o(control_valid), + .ctrl_data_o(control_result), + .mem_req_o(request), + .mem_wen_o(load), + .mem_addr_o(address), + .mem_be_o(enables), + .mem_data_o(write_data), + .mem_id_o(request_id), + .mem_gnt_i(1'b1), + .mem_valid_i(response_valid), + .mem_data_i(response_data), + .mem_id_i(response_id), + .mem_ready_o(ready) + ); + + function void load_word(int bank, int row, bit [127:0] data); + memory[bank][row] = data; + endfunction + + always @(posedge clock) begin + int bank, word_index, limit; + ticks <= ticks + 1; + response_valid <= 0; + if (!reset && request) begin + bank = int'(address[25:24]) - 1; + word_index = address[23:4]; + case (bank) + 0: limit = row_base + rows * reduction / 8; + 1: limit = row_base + reduction; + 2: limit = row_base + rows; + default: $fatal(1, "Native reference address has no bank region"); + endcase + if (load) begin + response_data <= { + word_index + 1 < limit ? memory[bank][word_index+1] : 128'b0, + word_index < limit ? memory[bank][word_index] : 128'b0 + }; + response_id <= request_id; + response_valid <= 1; + end else begin + if (bank != 2) $fatal(1, "Native reference store is outside C region"); + for (int byte_index = 0; byte_index < 32; byte_index++) + if (enables[byte_index] && word_index + byte_index / 16 < limit) + memory[2][word_index+byte_index/16][(byte_index%16)*8+:8] <= write_data[byte_index*8+:8]; + end + end + end + + task execute(int m, int k, int base, bit first, output longint elapsed); + longint started; + int register_addr[9] = '{4, 44, 48, 52, 32, 36, 40, 60, 0}; + int register_data[9]; + rows = m; + reduction = k; + row_base = base; + register_data = '{ + 0, + 32'h01000000 + (base << 4), + 32'h02000000 + (base << 4), + 32'h03000000 + (base << 4), + (8 << 16) | m, + 32'h02800000 | (!first << 20) | k, + 0, + 0, + 0 + }; + started = ticks; + for (int step = 0; step < 9; step++) begin + @(negedge clock); + control_addr = register_addr[step]; + control_data = register_data[step]; + control_wen = (step == 0); + control_req = 1; + do @(posedge clock); while (!control_grant); + @(negedge clock); + control_req = 0; + wait (control_valid); + if (step == 0 && control_result == 32'hffffffff) + $fatal(1, "Native reference has no job context"); + end + wait (event_done); + wait (!busy); + elapsed = ticks - started; + @(negedge clock); + endtask +endmodule + +interface redmule_reference_if ( + input logic clock, + reset +); + function void load_word(int bank, int row, bit [127:0] data); + tb_top.reference_model.load_word(bank, row, data); + endfunction + function bit [127:0] read_word(int row); + return tb_top.reference_model.memory[2][row]; + endfunction + function longint get_ticks(); + return tb_top.reference_model.ticks; + endfunction + task execute(int m, int k, int base, bit first, output longint elapsed); + tb_top.reference_model.execute(m, k, base, first, elapsed); + endtask +endinterface diff --git a/examples/components/balls/redmule/verify/src/tb_top.sv b/examples/components/balls/redmule/verify/src/tb_top.sv new file mode 100644 index 000000000..02293ac1c --- /dev/null +++ b/examples/components/balls/redmule/verify/src/tb_top.sv @@ -0,0 +1,111 @@ +module tb_top; + import uvm_pkg::*; + import redmule_uvm_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + redmule_reference reference_model ( + intf.clock, + intf.reset + ); + redmule_reference_if reference_interface ( + intf.clock, + intf.reset + ); + + RedmuleBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .io_status_idle(), + .io_status_running(), + .io_bankAccessReady(1'b1), + .io_cmdReq_ready(intf.cmd_req_ready), + .io_cmdReq_valid(intf.cmd_req_valid), + .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), + .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), + .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), + .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), + .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), + .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), + .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), + .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), + .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), + .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), + .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), + .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), + .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), + .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), + .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), + .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), + .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), + .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), + .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), + .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), + .io_cmdResp_ready(intf.cmd_resp_ready), + .io_cmdResp_valid(intf.cmd_resp_valid), + .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), + .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), + .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .io_bankRead_0_ball_id(), + .io_bankRead_0_group_id(intf.bank_read_group_id[0]), + .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .io_bankRead_1_ball_id(), + .io_bankRead_1_group_id(intf.bank_read_group_id[1]), + .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .io_bankWrite_0_ball_id(), + .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .io_subRobReq_ready(intf.sub_rob_req_ready) + ); + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + uvm_config_db#(virtual redmule_reference_if)::set(null, "*", "reference", reference_interface); + run_test("redmule_ball_test"); + end +endmodule diff --git a/examples/components/balls/redmule/workloads/ctests/CMakeLists.txt b/examples/components/balls/redmule/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..448016697 --- /dev/null +++ b/examples/components/balls/redmule/workloads/ctests/CMakeLists.txt @@ -0,0 +1,17 @@ +file( + READ "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h" + REDMULE_PARAMS +) +string(REGEX MATCH "#define BANK_WIDTH ([0-9]+)" _ "${REDMULE_PARAMS}") +set(REDMULE_BANK_WIDTH ${CMAKE_MATCH_1}) +string(REGEX MATCH "#define BANK_LINES ([0-9]+)" _ "${REDMULE_PARAMS}") +if(NOT REDMULE_BANK_WIDTH EQUAL 128 OR CMAKE_MATCH_1 LESS 18) + message( + FATAL_ERROR + "RedmuleBall tests require 128-bit rows and at least 18 bank rows, got ${REDMULE_BANK_WIDTH} bits and ${CMAKE_MATCH_1} rows" + ) +endif() +add_buckyball_ctest(redmule_f16_test.c) +add_buckyball_ctest(redmule_f16_chain_test.c) +add_buckyball_ctest(redmule_f16_edges_test.c) +add_buckyball_ctest(redmule_f16_fused_test.c) diff --git a/examples/components/balls/redmule/workloads/ctests/redmule_f16_chain_test.c b/examples/components/balls/redmule/workloads/ctests/redmule_f16_chain_test.c new file mode 100644 index 000000000..393ae7620 --- /dev/null +++ b/examples/components/balls/redmule/workloads/ctests/redmule_f16_chain_test.c @@ -0,0 +1,43 @@ +#include +#include +#include + +static uint16_t a[16] __attribute__((aligned(64))); +static uint16_t b[64] __attribute__((aligned(64))); +static uint16_t c[16] __attribute__((aligned(64))); + +int main(void) { + unsigned ab = VIRTUAL_BANK_NUM - 6, bb = ab + 1, cb = ab + 2; + for (unsigned i = 0; i < 16; ++i) { + a[i] = 0x3c00; + c[i] = 0x3555; + } + for (unsigned i = 0; i < 64; ++i) + b[i] = 0x3800; + bb_mem_alloc(ab, 1, 1); + bb_mem_alloc(bb, 1, 1); + bb_mem_alloc(cb, 1, 1); + bb_mvin((uintptr_t)a, ab, 2, 1); + bb_mvin((uintptr_t)b, bb, 8, 1); + bb_mvin((uintptr_t)c, cb, 2, 1); + bb_redmule(ab, bb, cb, 2, 8, 0, 0, 0, 1); + bb_mvout((uintptr_t)c, cb, 2, 1); + for (unsigned i = 0; i < 16; ++i) + if (c[i] != 0x4400) + __builtin_trap(); + bb_redmule(ab, bb, cb, 2, 8, 0, 0, 0, 0); + bb_mvout((uintptr_t)c, cb, 2, 1); + for (unsigned i = 0; i < 16; ++i) + if (c[i] != 0x4800) + __builtin_trap(); + bb_redmule(ab, bb, cb, 2, 8, 0, 0, 0, 1); + bb_mvout((uintptr_t)c, cb, 2, 1); + for (unsigned i = 0; i < 16; ++i) + if (c[i] != 0x4400) + __builtin_trap(); + bb_mem_release(ab); + bb_mem_release(bb); + bb_mem_release(cb); + puts("RedMulE FP16 chain and restart PASS"); + return 0; +} diff --git a/examples/components/balls/redmule/workloads/ctests/redmule_f16_edges_test.c b/examples/components/balls/redmule/workloads/ctests/redmule_f16_edges_test.c new file mode 100644 index 000000000..60ad0c7d8 --- /dev/null +++ b/examples/components/balls/redmule/workloads/ctests/redmule_f16_edges_test.c @@ -0,0 +1,35 @@ +#include +#include +#include + +static const uint16_t a[8] + __attribute__((aligned(64))) = {0x3c00, 0x1000, 0, 0, 0, 0, 0, 0}; +static const uint16_t b[64] __attribute__((aligned(64))) = { + 0x3c00, 0x3c01, 1, 0x7bff, 0x7c00, 0xfc00, 0x7d00, 0x8000, 0x3c00, 0x3c00, + 0, 0x3c00, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0}; +static const uint16_t expected[8] = {0x3c00, 0x3c02, 1, 0x7bff, + 0x7c00, 0xfc00, 0x7e00, 0}; +static uint16_t c[8] __attribute__((aligned(64))); + +int main(void) { + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 1); + bb_mem_alloc(2, 1, 1); + bb_mvin((uintptr_t)a, 0, 1, 1); + bb_mvin((uintptr_t)b, 1, 8, 1); + bb_redmule(0, 1, 2, 1, 8, 0, 0, 0, 1); + bb_mvout((uintptr_t)c, 2, 1, 1); + for (unsigned i = 0; i < 8; ++i) + if (c[i] != expected[i]) + __builtin_trap(); + bb_mem_release(0); + bb_mem_release(1); + bb_mem_release(2); + puts("RedMulE FP16 ties, subnormal, infinity and canonical NaN PASS"); + return 0; +} diff --git a/examples/components/balls/redmule/workloads/ctests/redmule_f16_fused_test.c b/examples/components/balls/redmule/workloads/ctests/redmule_f16_fused_test.c new file mode 100644 index 000000000..0d36b9b52 --- /dev/null +++ b/examples/components/balls/redmule/workloads/ctests/redmule_f16_fused_test.c @@ -0,0 +1,29 @@ +#include +#include +#include + +static const uint16_t a[8] + __attribute__((aligned(64))) = {0x3c00, 0x3c01, 0, 0, 0, 0, 0, 0}; +static const uint16_t b[64] __attribute__((aligned(64))) = { + 0xbc00, 0xbc00, 0xbc00, 0xbc00, 0xbc00, 0xbc00, 0xbc00, 0xbc00, + 0x3bfe, 0x3bfe, 0x3bfe, 0x3bfe, 0x3bfe, 0x3bfe, 0x3bfe, 0x3bfe}; +static uint16_t c[8] __attribute__((aligned(64))); + +int main(void) { + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 1); + bb_mem_alloc(2, 1, 1); + bb_mvin((uintptr_t)a, 0, 1, 1); + bb_mvin((uintptr_t)b, 1, 8, 1); + bb_redmule(0, 1, 2, 1, 8, 0, 0, 0, 1); + bb_mvout((uintptr_t)c, 2, 1, 1); + // (1 + 2^-10) * (1 - 2^-10) - 1 = -2^-20; separate multiply/add loses it. + for (unsigned i = 0; i < 8; ++i) + if (c[i] != 0x8010) + __builtin_trap(); + bb_mem_release(0); + bb_mem_release(1); + bb_mem_release(2); + puts("RedMulE FP16 fused cancellation PASS"); + return 0; +} diff --git a/examples/components/balls/redmule/workloads/ctests/redmule_f16_test.c b/examples/components/balls/redmule/workloads/ctests/redmule_f16_test.c new file mode 100644 index 000000000..ba1e3f0b0 --- /dev/null +++ b/examples/components/balls/redmule/workloads/ctests/redmule_f16_test.c @@ -0,0 +1,47 @@ +#include +#include +#include + +static uint16_t a[8 * 8] __attribute__((aligned(64))); +static uint16_t b[18 * 8] __attribute__((aligned(64))); +static uint16_t c[7 * 8] __attribute__((aligned(64))); + +int main(void) { + unsigned ab = VIRTUAL_BANK_NUM - 3, bb = ab + 1, cb = ab + 2; + for (unsigned i = 0; i < 8 * 8; ++i) + a[i] = 0x7e00; + for (unsigned i = 0; i < 18 * 8; ++i) + b[i] = 0x7e00; + for (unsigned i = 0; i < 7 * 8; ++i) + c[i] = 0x3555; + for (unsigned r = 0; r < 3; ++r) + for (unsigned k = 0; k < 16; ++k) + a[2 * 8 + r * 16 + k] = r == 1 ? 0xc000 : 0x3c00; + for (unsigned k = 0; k < 16; ++k) + for (unsigned n = 0; n < 8; ++n) + b[(2 + k) * 8 + n] = n & 1 ? 0xb800 : 0x3800; + bb_mem_alloc(ab, 1, 1); + bb_mem_alloc(bb, 1, 1); + bb_mem_alloc(cb, 1, 1); + bb_mvin((uintptr_t)a, ab, 8, 1); + bb_mvin((uintptr_t)b, bb, 18, 1); + bb_mvin((uintptr_t)c, cb, 7, 1); + bb_redmule(ab, bb, cb, 3, 16, 2, 2, 3, 1); + bb_mvout((uintptr_t)c, cb, 7, 1); + for (unsigned r = 0; r < 7; ++r) + for (unsigned n = 0; n < 8; ++n) { + uint16_t expected = 0x3555; + if (r >= 3 && r < 6) { + expected = r == 4 ? 0xcc00 : 0x4800; + if (n & 1) + expected ^= 0x8000; + } + if (c[r * 8 + n] != expected) + __builtin_trap(); + } + bb_mem_release(ab); + bb_mem_release(bb); + bb_mem_release(cb); + puts("RedMulE FP16 nonzero bases and guard rows PASS"); + return 0; +} diff --git a/examples/components/balls/redmule/workloads/isa/redmule.h b/examples/components/balls/redmule/workloads/isa/redmule.h new file mode 100644 index 000000000..e089fab71 --- /dev/null +++ b/examples/components/balls/redmule/workloads/isa/redmule.h @@ -0,0 +1,22 @@ +#ifndef REDMULE_H +#define REDMULE_H +#include +#include + +// FP16 A[M,K], B[K,8], C[M,8]; ordered FP16 FMA, RNE, canonical NaN. +// Bases are bank rows. First starts with positive zero; every call writes C. +static inline void bb_redmule(uint32_t a, uint32_t b, uint32_t c, uint32_t m, + uint32_t k, uint32_t op1_base, uint32_t op2_base, + uint32_t wr_base, uint32_t first) { + if (a > 1023 || b > 1023 || c > 1023 || a == b || a == c || b == c || !m || + m > 4095 || !k || k % 8 || op1_base > 1023 || op2_base > 1023 || + wr_base > 1023 || first > 1) + __builtin_trap(); + uint64_t special = (uint64_t)m | (uint64_t)op1_base << 12 | + (uint64_t)op2_base << 22 | (uint64_t)wr_base << 32 | + (uint64_t)first << 42; + BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(a) | BB_BANK1(b) | BB_BANK2(c) | + BB_ITER(k), + special, BB_FUNC7(REDMULE_F16)); +} +#endif diff --git a/examples/components/balls/redmule/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/redmule/workloads/mlir_tests/CMakeLists.txt new file mode 100644 index 000000000..cd1e18e7d --- /dev/null +++ b/examples/components/balls/redmule/workloads/mlir_tests/CMakeLists.txt @@ -0,0 +1,3 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +add_subdirectory(ball) +add_subdirectory(bank) diff --git a/examples/components/balls/redmule/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/redmule/workloads/mlir_tests/ball/CMakeLists.txt new file mode 100644 index 000000000..3933abe13 --- /dev/null +++ b/examples/components/balls/redmule/workloads/mlir_tests/ball/CMakeLists.txt @@ -0,0 +1,2 @@ +set(BUCKYBALL_MLIR_TEST_PREFIX ball) +add_buckyball_mlir_test(redmule_f16_chain TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir b/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir new file mode 100644 index 000000000..e75931747 --- /dev/null +++ b/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir @@ -0,0 +1,6 @@ +// The banks 0 (A), 1 (B) and 2 (C) are allocated and loaded by the caller. +func.func @redmule_chain() attributes {llvm.emit_c_interface} { + buckyball.custom "matmul_f16" <{sources = array, destination = 2 : i64}> {rows = 2 : i64, k = 8 : i64, op1_base = 0 : i64, op2_base = 0 : i64, wr_base = 0 : i64} + buckyball.custom "matmul_f16" <{sources = array, destination = 2 : i64, read_destination}> {rows = 2 : i64, k = 8 : i64, op1_base = 0 : i64, op2_base = 0 : i64, wr_base = 0 : i64} + return +} diff --git a/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp b/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp new file mode 100644 index 000000000..a54f71af8 --- /dev/null +++ b/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp @@ -0,0 +1,28 @@ +#include +#include +#include + +extern "C" void _mlir_ciface_redmule_chain(); +alignas(64) static uint16_t a[16], b[64], c[16]; + +int main() { + for (auto &value : a) + value = 0x3c00; + for (auto &value : b) + value = 0x3800; + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 1); + bb_mem_alloc(2, 1, 1); + bb_mvin((uintptr_t)a, 0, 2, 1); + bb_mvin((uintptr_t)b, 1, 8, 1); + _mlir_ciface_redmule_chain(); + bb_mvout((uintptr_t)c, 2, 2, 1); + for (auto value : c) + if (value != 0x4800) + __builtin_trap(); + bb_mem_release(0); + bb_mem_release(1); + bb_mem_release(2); + puts("RedMulE BallOp FP16 chain PASS"); + return 0; +} diff --git a/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir b/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir new file mode 100644 index 000000000..e558d5513 --- /dev/null +++ b/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir @@ -0,0 +1,7 @@ +// RUN: buddy-opt %s -lower-buckyball -verify-diagnostics +func.func @invalid_k() { + // expected-error @+2 {{FP16 matmul shape exceeds the bank or ISA geometry}} + // expected-error @+1 {{has no lowering on this core}} + buckyball.custom "matmul_f16" <{sources = array, destination = 2 : i64}> {rows = 1 : i64, k = 7 : i64, op1_base = 0 : i64, op2_base = 0 : i64, wr_base = 0 : i64} + return +} diff --git a/examples/components/balls/redmule/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/redmule/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..d11a83ac3 --- /dev/null +++ b/examples/components/balls/redmule/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,2 @@ +set(BUCKYBALL_MLIR_TEST_PREFIX bank) +add_buckyball_mlir_test(redmule_f16_rows TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir b/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir new file mode 100644 index 000000000..e7c709e50 --- /dev/null +++ b/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir @@ -0,0 +1,11 @@ +// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -verify-diagnostics +func.func @invalid_alias() { + %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + // expected-error @+2 {{FP16 matmul requires distinct A, B and C banks}} + // expected-error @+1 {{has no lowering on this core}} + %c = buckyball.bank_custom "matmul_f16" (%a, %b) -> %a {rows = 1 : i64, k = 8 : i64, op1_base = 0 : i64, op2_base = 0 : i64, wr_base = 0 : i64} + buckyball.bank_release %c + buckyball.bank_release %b + return +} diff --git a/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in b/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in new file mode 100644 index 000000000..e01fb0c17 --- /dev/null +++ b/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in @@ -0,0 +1,48 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x8xf16>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %eight = arith.constant 8 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %pos_one = arith.constant 1.0 : f16 + %neg_one = arith.constant -1.0 : f16 + %pos_half = arith.constant 0.5 : f16 + %neg_half = arith.constant -0.5 : f16 + %a = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x8xf16> + %b = memref.alloc() alignment = 64 : memref<8x8xf16> + %c = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x8xf16> + scf.for %row = %zero to %rows step %one { + %parity = arith.remui %row, %two : index + %odd = arith.cmpi eq, %parity, %one : index + %value = arith.select %odd, %neg_one, %pos_one : f16 + scf.for %lane = %zero to %eight step %one { + memref.store %value, %a[%row, %lane] : memref<@BB_BANK_LINES@x8xf16> + } + } + scf.for %k = %zero to %eight step %one { + scf.for %lane = %zero to %eight step %one { + %parity = arith.remui %lane, %two : index + %odd = arith.cmpi eq, %parity, %one : index + %value = arith.select %odd, %neg_half, %pos_half : f16 + memref.store %value, %b[%k, %lane] : memref<8x8xf16> + } + } + %a_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %c_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_a = buckyball.bank_mvin %a, %a_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x8xf16> + %loaded_b = buckyball.bank_mvin %b, %b_bank <{depth = 8 : i64, stride = 1 : i64}> : memref<8x8xf16> + %result = buckyball.bank_custom "matmul_f16" (%loaded_a, %loaded_b) -> %c_bank + {rows = @BB_BANK_LINES@ : i64, k = 8 : i64, op1_base = 0 : i64, op2_base = 0 : i64, wr_base = 0 : i64} + %stored = buckyball.bank_mvout %c, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x8xf16> + buckyball.bank_release %loaded_a + buckyball.bank_release %loaded_b + buckyball.bank_release %stored + func.call @check_result(%c) : (memref<@BB_BANK_LINES@x8xf16>) -> () + memref.dealloc %a : memref<@BB_BANK_LINES@x8xf16> + memref.dealloc %b : memref<8x8xf16> + memref.dealloc %c : memref<@BB_BANK_LINES@x8xf16> + return %zero_i8 : i8 +} diff --git a/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows_main.cpp b/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows_main.cpp new file mode 100644 index 000000000..7fd11260c --- /dev/null +++ b/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows_main.cpp @@ -0,0 +1,19 @@ +#include +#include +#include +#include + +extern "C" void check_result(uint16_t *, uint16_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != BANK_LINES || lanes != 8 || stride != 8 || lane_stride != 1) + exit(1); + for (int row = 0; row < rows; ++row) + for (int lane = 0; lane < lanes; ++lane) + if (data[offset + row * stride + lane] != + (uint16_t)(0x4400 | ((row ^ lane) & 1) << 15)) { + printf("RedMulE bank rows mismatch row=%d lane=%d\n", row, lane); + exit(1); + } + puts("RedMulE BankOp full bank rows PASS"); +} diff --git a/examples/components/balls/relu/README.md b/examples/components/balls/relu/README.md new file mode 100644 index 000000000..42b0326d1 --- /dev/null +++ b/examples/components/balls/relu/README.md @@ -0,0 +1,28 @@ +# ReluBall + +`ReluBall` computes signed INT32 `max(x, 0)` in place in one physical pbank. +It has no internal SRAM: a 128-bit register holds one pbank line between its +read and write requests. + +## Command + +`RELU(vbank, group, iter, stride)` selects physical pbank `group` in virtual +bank `vbank`. The instruction places `vbank` in both the source and destination +bank fields and sets the generic `BB_UPDATE_WR` flag for an in-place update. +`group` is a pbank number, never a data-layout term. + +`iter` is the number of 128-bit physical lines processed in each segment. +`stride` is the distance between segments. Both are positive; `iter` is a +multiple of 16, `iter <= stride`, and `stride` divides the pbank depth. The +command processes `iter` lines at every `stride` boundary until it reaches the +end of the selected pbank. + +For every line the Ball performs one single-port sequence: + +```text +pbank read -> 4 signed int32 ReLUs -> pbank write +``` + +The selected group must be allocated. There is no output vbank, no fallback +layout, and no tail path. Tile lowering pads logical matrices to 16-aligned +shapes; compiler packing fills the final pbank chunk before issuing ReLU. diff --git a/examples/components/balls/relu/arch/Relu.scala b/examples/components/balls/relu/arch/Relu.scala new file mode 100644 index 000000000..99ccb37a6 --- /dev/null +++ b/examples/components/balls/relu/arch/Relu.scala @@ -0,0 +1,184 @@ +package examples.balls.relu + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.experimental.hierarchy.{instantiable, public} +import chisel3.util._ +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} + +@instantiable +class Relu( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val addressWidth = log2Up(memParam.bankEntries) + val countWidth = log2Up(memParam.bankEntries + 1) + val bankIdWidth = frontendParam.bank_id_len + val groupWidth = log2Up(memParam.bankNum) + + val mapping = instance + + require(mapping.inBW == 1, "ReluBall requires one SRAM read port") + require(mapping.outBW == 1, "ReluBall requires one SRAM write port") + require(memParam.bankWidth == 128, "ReluBall requires 128-bit SRAM rows") + require(memParam.bankMaskLen == 16, "ReluBall requires sixteen byte enables") + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + 1, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + 1, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val Seq(idle, waitForChannels, readRequest, readResponse, writeRequest, writeResponse, complete) = Enum(7) + val state = RegInit(idle) + + val robId = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + val bank = RegInit(0.U(bankIdWidth.W)) + val group = RegInit(0.U(groupWidth.W)) + val iter = RegInit(0.U(countWidth.W)) + val stride = RegInit(0.U(countWidth.W)) + val segment = RegInit(0.U(addressWidth.W)) + val line = RegInit(0.U(addressWidth.W)) + val word = Reg(UInt(128.W)) + + val relu = Wire(Vec(4, UInt(32.W))) + for (lane <- 0 until 4) { + val value = word(32 * lane + 31, 32 * lane).asSInt + relu(lane) := Mux(value < 0.S, 0.U, value.asUInt) + } + val address = segment + line + + io.cmdReq.ready := state === idle + io.cmdResp.valid := state === complete + io.cmdResp.bits.rob_id := robId + + io.bankRead(0).rob_id := robId + io.bankRead(0).bank_id := bank + io.bankRead(0).group_id := group + io.bankRead(0).io.req.valid := false.B + io.bankRead(0).io.req.bits.addr := address + io.bankRead(0).io.resp.ready := false.B + + io.bankWrite(0).rob_id := robId + io.bankWrite(0).bank_id := bank + io.bankWrite(0).group_id := group + io.bankWrite(0).io.req.valid := false.B + io.bankWrite(0).io.req.bits.addr := address + io.bankWrite(0).io.req.bits.data := Cat(relu.reverse) + io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) + io.bankWrite(0).io.resp.ready := false.B + + switch(state) { + is(idle) { + when(io.cmdReq.fire) { + val command = io.cmdReq.bits + + assert( + command.op1_bank === command.wr_bank, + "ReluBall source and destination banks must match" + ) + assert( + command.op1_col =/= 0.U && command.op2_bank < command.op1_col, + "ReluBall group is not allocated in bank" + ) + assert(command.iter =/= 0.U && command.iter(3, 0) === 0.U, "ReluBall iter must be a positive multiple of 16") + assert( + command.rs2 =/= 0.U && command.rs2 <= memParam.bankEntries.U, + "ReluBall stride must fit in one physical bank" + ) + assert( + memParam.bankEntries.U % command.rs2 === 0.U, + "ReluBall stride must divide the physical bank depth" + ) + assert(command.iter <= command.rs2, "ReluBall iter must not exceed stride") + + robId := io.cmdReq.bits.rob_id + bank := command.op1_bank + // RELU names its group in the op2 bank field. + group := command.op1_group + command.op2_bank + iter := command.iter + stride := command.rs2 + segment := 0.U + line := 0.U + state := waitForChannels + } + } + + is(waitForChannels) { + state := readRequest + } + + is(readRequest) { + io.bankRead(0).io.req.valid := true.B + when(io.bankRead(0).io.req.fire)(state := readResponse) + } + + is(readResponse) { + io.bankRead(0).io.resp.ready := true.B + when(io.bankRead(0).io.resp.fire) { + word := io.bankRead(0).io.resp.bits.data + state := writeRequest + } + } + + is(writeRequest) { + io.bankWrite(0).io.req.valid := true.B + when(io.bankWrite(0).io.req.fire)(state := writeResponse) + } + + is(writeResponse) { + io.bankWrite(0).io.resp.ready := true.B + when(io.bankWrite(0).io.resp.fire) { + when(line +& 1.U === iter) { + when(segment +& stride === memParam.bankEntries.U) { + state := complete + }.otherwise { + segment := segment + stride + line := 0.U + state := readRequest + } + }.otherwise { + line := line + 1.U + state := readRequest + } + } + } + + is(complete) { + when(io.cmdResp.fire)(state := idle) + } + } + + io.status.idle := state === idle + io.status.running := state =/= idle +} diff --git a/examples/components/balls/relu/arch/ReluBall.scala b/examples/components/balls/relu/arch/ReluBall.scala new file mode 100644 index 000000000..e717cfb15 --- /dev/null +++ b/examples/components/balls/relu/arch/ReluBall.scala @@ -0,0 +1,46 @@ +package examples.balls.relu + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.relu.{Params => ParametersProto} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.{BlinkIO, HasBlink} + +@instantiable +class ReluBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val unit: Instance[Relu] = Instantiate(new Relu(fp, mp, bp, ball)) + unit.io.cmdReq <> blink.cmdReq + unit.io.cmdResp <> blink.cmdResp + unit.io.bankRead(0) <> blink.bankRead(0) + unit.io.bankWrite(0) <> blink.bankWrite(0) + blink.status <> unit.io.status +} + +object ReluBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new ReluBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/components/balls/relu/configs/default.toml b/examples/components/balls/relu/configs/default.toml new file mode 100644 index 000000000..0ae61cbda --- /dev/null +++ b/examples/components/balls/relu/configs/default.toml @@ -0,0 +1 @@ +# ReluBall has no implementation-specific parameters. diff --git a/examples/components/balls/relu/configs/relu_params.proto b/examples/components/balls/relu/configs/relu_params.proto new file mode 100644 index 000000000..caf020c5c --- /dev/null +++ b/examples/components/balls/relu/configs/relu_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.relu; +option java_package = "buckyball.balls.relu"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/balls/relu/emu/src/50_relu.rs b/examples/components/balls/relu/emu/src/50_relu.rs similarity index 89% rename from examples/balls/relu/emu/src/50_relu.rs rename to examples/components/balls/relu/emu/src/50_relu.rs index 1f1a4f7b7..68ee6c847 100644 --- a/examples/balls/relu/emu/src/50_relu.rs +++ b/examples/components/balls/relu/emu/src/50_relu.rs @@ -1,18 +1,18 @@ -use super::super::bank::bank_size; +use bebop_bemu::thread::banks::bank_size; use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::{BallInstruction, ExecContext}; +use super::instruction::{BallModel, ExecContext}; pub struct Relu; -impl BallInstruction for Relu { +impl BallModel for Relu { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let bank = rs1_b0(xs1); let group = rs1_b1(xs1); let iter = rs1_iter(xs1) as usize; let stride = xs2 as usize; - if rs1_b2(xs1) != 0 { - panic!("relu: bank2 must be zero"); + if rs1_b2(xs1) != bank { + panic!("relu: source and destination banks must match"); } let config = *ctx.config(bank); if !config.allocated { diff --git a/examples/components/balls/relu/emu/src/dispatch.rs b/examples/components/balls/relu/emu/src/dispatch.rs new file mode 100644 index 000000000..f8d176b31 --- /dev/null +++ b/examples/components/balls/relu/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::f50_relu; + +const BALL_CLASS: &str = "examples.balls.relu.ReluBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("RELU") => Some(f50_relu::Relu::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("RELU") => Some(f50_relu::Relu::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/components/balls/relu/emu/src/lib.rs b/examples/components/balls/relu/emu/src/lib.rs new file mode 100644 index 000000000..bb463b57e --- /dev/null +++ b/examples/components/balls/relu/emu/src/lib.rs @@ -0,0 +1,12 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +#[path = "50_relu.rs"] +mod f50_relu; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/components/balls/relu/stack/src/Lowering.cpp b/examples/components/balls/relu/stack/src/Lowering.cpp new file mode 100644 index 000000000..ff86e2073 --- /dev/null +++ b/examples/components/balls/relu/stack/src/Lowering.cpp @@ -0,0 +1,55 @@ +//===- Lowering.cpp - ReluBall in lower-buckyball -------------------------===// +// +// relu_i32: max(x, 0) on the i32 lanes of physical bank `group` of the bank, +// in place: the one source is the destination. In every `stride` rows of the +// physical bank the first `rows` rows are updated; `rows` is a multiple of +// sixteen no larger than `stride`, and `stride` divides the bank depth. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballOps.h" +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class ReluI32Lowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &rewriter) const override { + if (op.getKind() != "relu_i32") + return failure(); + int64_t rows = cast(op->getAttr("rows")).getInt(); + int64_t stride = cast(op->getAttr("stride")).getInt(); + int64_t group = cast(op->getAttr("group")).getInt(); + int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; + if (op.getSources().size() != 1 || + op.getSources()[0] != int64_t(op.getDestination())) + return op.emitOpError("relu_i32 updates its one source bank in place"); + if (rows <= 0 || rows % 16 || rows > stride || depth % stride || + group < 0 || group >= 1024) + return op.emitOpError("relu_i32 rows, stride or group exceed the ISA"); + // The bank1 field selects the physical bank; bit 63 marks the in-place + // update. + uint64_t rs1 = mlir::buddy::bankFields(op.getSources(), op.getDestination(), + false, rows) | + mlir::buddy::field(group, 10, 19) | (1ULL << 63); + mlir::buddy::emitInstruction( + rewriter, op.getLoc(), buckyball_target::getBuckyballFunct7("RELU"), + mlir::buddy::constantI64(rewriter, op.getLoc(), rs1), + mlir::buddy::constantI64(rewriter, op.getLoc(), stride)); + rewriter.eraseOp(op); + return success(); + } +}; + +void populate(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} + +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallLowering(populate), true); +} // namespace diff --git a/examples/components/balls/relu/workloads/ctests/CMakeLists.txt b/examples/components/balls/relu/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..28834bf05 --- /dev/null +++ b/examples/components/balls/relu/workloads/ctests/CMakeLists.txt @@ -0,0 +1 @@ +add_buckyball_ctest(relu_i32_16x16_test.c) diff --git a/examples/balls/relu/workloads/ctests/relu_i32_16x16_test.c b/examples/components/balls/relu/workloads/ctests/relu_i32_16x16_test.c similarity index 100% rename from examples/balls/relu/workloads/ctests/relu_i32_16x16_test.c rename to examples/components/balls/relu/workloads/ctests/relu_i32_16x16_test.c diff --git a/examples/components/balls/relu/workloads/isa/relu.h b/examples/components/balls/relu/workloads/isa/relu.h new file mode 100644 index 000000000..3cbf226eb --- /dev/null +++ b/examples/components/balls/relu/workloads/isa/relu.h @@ -0,0 +1,13 @@ +#ifndef _BB_RELU_H_ +#define _BB_RELU_H_ + +#include +#include + +#define bb_relu(bank_id, group, iter, stride) \ + BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(bank_id) | BB_BANK1(group) | \ + BB_BANK2(bank_id) | BB_ITER(iter) | \ + BB_UPDATE_WR), \ + (stride), BB_FUNC7(RELU)) + +#endif // _BB_RELU_H_ diff --git a/examples/components/balls/relu/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/relu/workloads/mlir_tests/CMakeLists.txt new file mode 100644 index 000000000..4b64da6aa --- /dev/null +++ b/examples/components/balls/relu/workloads/mlir_tests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(ball) +add_subdirectory(bank) diff --git a/examples/balls/relu/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/relu/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/relu/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/relu/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir b/examples/components/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir new file mode 100644 index 000000000..bccae13c8 --- /dev/null +++ b/examples/components/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir @@ -0,0 +1,31 @@ +func.func private @check_result(memref<16x4xi32>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %four_i32 = arith.constant 4 : i32 + %sixteen_i32 = arith.constant 16 : i32 + %input = memref.alloc() : memref<16x4xi32> + %output = memref.alloc() : memref<16x4xi32> + scf.for %row = %zero to %sixteen step %one { + scf.for %lane = %zero to %four step %one { + %r = arith.index_cast %row : index to i32 + %l = arith.index_cast %lane : index to i32 + %base = arith.muli %r, %four_i32 : i32 + %value = arith.addi %base, %l : i32 + %signed = arith.subi %value, %sixteen_i32 : i32 + memref.store %signed, %input[%row, %lane] : memref<16x4xi32> + } + } + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + buckyball.custom "relu_i32" <{sources = array, destination = 0 : i64}> {rows = 16 : i64, stride = 16 : i64, group = 0 : i64} + buckyball.mvout %output <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + func.call @check_result(%output) : (memref<16x4xi32>) -> () + memref.dealloc %input : memref<16x4xi32> + memref.dealloc %output : memref<16x4xi32> + return %zero_i8 : i8 +} diff --git a/examples/balls/relu/workloads/mlir_tests/ball/relu_i32_16row_main.cpp b/examples/components/balls/relu/workloads/mlir_tests/ball/relu_i32_16row_main.cpp similarity index 100% rename from examples/balls/relu/workloads/mlir_tests/ball/relu_i32_16row_main.cpp rename to examples/components/balls/relu/workloads/mlir_tests/ball/relu_i32_16row_main.cpp diff --git a/examples/balls/relu/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/relu/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/relu/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/components/balls/relu/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in b/examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in new file mode 100644 index 000000000..e03f00def --- /dev/null +++ b/examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in @@ -0,0 +1,41 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () + +func.func @main() -> i8 { + %zeroI8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %limit = arith.constant @BB_BANK_LINES@ : index + %lanes = arith.constant @BB_BANK_I32_LANES@ : index + %lane_count = arith.constant @BB_BANK_I32_LANES@ : i32 + %seventeen = arith.constant 17 : i32 + %bias = arith.constant 8 : i32 + + %input = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %output = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + + scf.for %line = %zero to %limit step %one { + scf.for %lane = %zero to %lanes step %one { + %line_i32 = arith.index_cast %line : index to i32 + %lane_i32 = arith.index_cast %lane : index to i32 + %base = arith.muli %line_i32, %lane_count : i32 + %index = arith.addi %base, %lane_i32 : i32 + %remainder = arith.remui %index, %seventeen : i32 + %signed = arith.subi %remainder, %bias : i32 + memref.store %signed, %input[%line, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + } + } + + %bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded = buckyball.bank_mvin %input, %bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + %result = buckyball.bank_custom "relu_i32" (%loaded) -> %loaded + {rows = @BB_BANK_LINES@ : i64, stride = @BB_BANK_LINES@ : i64, group = 0 : i64} + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + buckyball.bank_release %stored + + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32>) -> () + memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> + return %zeroI8 : i8 +} diff --git a/examples/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp b/examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp similarity index 100% rename from examples/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp rename to examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp diff --git a/examples/balls/smatmul/README.md b/examples/components/balls/smatmul/README.md similarity index 100% rename from examples/balls/smatmul/README.md rename to examples/components/balls/smatmul/README.md diff --git a/examples/components/balls/smatmul/arch/Array.scala b/examples/components/balls/smatmul/arch/Array.scala new file mode 100644 index 000000000..70b12649b --- /dev/null +++ b/examples/components/balls/smatmul/arch/Array.scala @@ -0,0 +1,82 @@ +package examples.balls.smatmul + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import examples.balls.smatmul.configs.SMatMulBallParam + +@instantiable +class Array(val ballParam: SMatMulBallParam) extends Module { + val param = ballParam + val rows = param.tileRows + val cols = param.tileCols + + @public + val io = IO(new Bundle { + val start = Input(Bool()) + val aRows = Input(Vec(rows, UInt(128.W))) + val bRows = Input(Vec(cols, UInt(128.W))) + val done = Output(Bool()) + val result = Output(Vec(rows, UInt(512.W))) + }) + + val running = RegInit(false.B) + val cycle = RegInit(0.U(6.W)) + val pe = Seq.tabulate(rows, cols)((_, _) => Instantiate(new PE)) + val aShift = Reg(Vec(rows, UInt(128.W))) + // The PE product register adds one cycle between the systolic operands and + // the accumulator. Keep the final drain cycle explicit so the last core + // product is included before done is asserted. + val lastCyc = (rows + cols + 15).U + + when(io.start) { + assert(!running, "Array: start while array is running") + running := true.B + cycle := 0.U + aShift := io.aRows + }.elsewhen(running) { + when(cycle === lastCyc) { + running := false.B + }.otherwise { + cycle := cycle + 1.U + } + for (row <- 0 until rows) { + when(cycle >= row.U && cycle < (row + cols).U) { + aShift(row) := aShift(row) >> 8 + } + } + } + + for (row <- 0 until rows) { + val aWord = aShift(row)(7, 0).asSInt + val aValid = running && cycle >= row.U && cycle < (row + cols).U + + for (column <- 0 until cols) { + pe(row)(column).io.clear := io.start + if (column == 0) { + pe(row)(column).io.aIn := aWord + pe(row)(column).io.aInValid := aValid + } else { + pe(row)(column).io.aIn := pe(row)(column - 1).io.aOut + pe(row)(column).io.aInValid := pe(row)(column - 1).io.aOutValid + } + + val bCycle = cycle - column.U + val bRow = bCycle(3, 0) + val bWord = io.bRows(bRow)(8 * column + 7, 8 * column).asSInt + val bValid = running && cycle >= column.U && cycle < (column + cols).U + if (row == 0) { + pe(row)(column).io.bIn := bWord + pe(row)(column).io.bInValid := bValid + } else { + pe(row)(column).io.bIn := pe(row - 1)(column).io.bOut + pe(row)(column).io.bInValid := pe(row - 1)(column).io.bOutValid + } + } + } + + for (row <- 0 until rows) { + io.result(row) := Cat((0 until cols).reverse.map(column => pe(row)(column).io.sum.asUInt)) + } + io.done := running && cycle === lastCyc +} diff --git a/examples/components/balls/smatmul/arch/PE.scala b/examples/components/balls/smatmul/arch/PE.scala new file mode 100644 index 000000000..017c5f501 --- /dev/null +++ b/examples/components/balls/smatmul/arch/PE.scala @@ -0,0 +1,60 @@ +package examples.balls.smatmul + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class PE extends Module { + + // One core accumulates sixteen signed 8x8 products. Twenty bits cover the + // full range [-262144, 258064]; the unit-level accumulator remains 32-bit + // for reduction across cores. + val tileSumWidth = 20 + + @public + val io = IO(new Bundle { + val aIn = Input(SInt(8.W)) + val aInValid = Input(Bool()) + val bIn = Input(SInt(8.W)) + val bInValid = Input(Bool()) + val clear = Input(Bool()) + val aOut = Output(SInt(8.W)) + val aOutValid = Output(Bool()) + val bOut = Output(SInt(8.W)) + val bOutValid = Output(Bool()) + val sum = Output(SInt(32.W)) + }) + + val accumulator = RegInit(0.S(tileSumWidth.W)) + val aPipe = RegInit(0.S(8.W)) + val aPipeValid = RegInit(false.B) + val bPipe = RegInit(0.S(8.W)) + val bPipeValid = RegInit(false.B) + val product = RegInit(0.S(16.W)) + val productValid = RegInit(false.B) + + when(io.clear) { + accumulator := 0.S + aPipeValid := false.B + bPipeValid := false.B + productValid := false.B + }.elsewhen(productValid) { + accumulator := accumulator + product + } + when(!io.clear) { + product := aPipe * bPipe + productValid := aPipeValid && bPipeValid + } + when(!io.clear) { + aPipe := io.aIn + aPipeValid := io.aInValid + bPipe := io.bIn + bPipeValid := io.bInValid + } + + io.aOut := aPipe + io.aOutValid := aPipeValid + io.bOut := bPipe + io.bOutValid := bPipeValid + io.sum := accumulator.pad(32) +} diff --git a/examples/components/balls/smatmul/arch/SMatMulBall.scala b/examples/components/balls/smatmul/arch/SMatMulBall.scala new file mode 100644 index 000000000..8e0928e14 --- /dev/null +++ b/examples/components/balls/smatmul/arch/SMatMulBall.scala @@ -0,0 +1,66 @@ +package examples.balls.smatmul + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.smatmul.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.smatmul.configs.SMatMulBallParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class SMatMulBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + val ballParam = SMatMulBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + val ballCommonConfig = ball + + val inBW = ballCommonConfig.inBW + val outBW = ballCommonConfig.outBW + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val smatmulArrayUnit: Instance[SMatMulUnit] = + Instantiate(new SMatMulUnit(fp, mp, bp, ball, ballParam)) + + smatmulArrayUnit.io.cmdReq <> blink.cmdReq + smatmulArrayUnit.io.cmdResp <> blink.cmdResp + + for (i <- 0 until inBW) { + smatmulArrayUnit.io.bankRead(i) <> blink.bankRead(i) + } + + for (i <- 0 until outBW) { + smatmulArrayUnit.io.bankWrite(i) <> blink.bankWrite(i) + } + + blink.status <> smatmulArrayUnit.io.status + + // MMIO: this Ball does not consume MMIO metadata +} + +object SMatMulBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new SMatMulBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/components/balls/smatmul/arch/SMatMulUnit.scala b/examples/components/balls/smatmul/arch/SMatMulUnit.scala new file mode 100644 index 000000000..89aa66478 --- /dev/null +++ b/examples/components/balls/smatmul/arch/SMatMulUnit.scala @@ -0,0 +1,461 @@ +package examples.balls.smatmul + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} +import examples.balls.smatmul.configs.SMatMulBallParam + +@instantiable +class SMatMulUnit( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: SMatMulBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val param = ballParam + val rows = param.tileRows + val cols = param.tileCols + val resultWords = 4 + val addressWidth = log2Ceil(memParam.bankEntries) + val bankIdWidth = frontendParam.bank_id_len + val maxOutputRows = memParam.bankEntries / resultWords + val accDepth = maxOutputRows.max(32) + val accAddrWidth = log2Ceil(accDepth) + + val mapping = instance + + val osFunct = instance.instructions + .find(_.mnemonic == "SMATMUL_OS").get.funct7 + + val biasFunct = instance.instructions + .find(_.mnemonic == "SMATMUL_BIAS").get.funct7 + + require(mapping.inBW == 2, "SMatMulBall requires inBW=2") + require(mapping.outBW == 1, "SMatMulBall requires outBW=1") + require(memParam.bankWidth == 128, "SMatMulBall requires 128-bit bank rows") + require( + memParam.bankMaskLen == 16, + "SMatMulBall requires sixteen byte enables" + ) + require( + memParam.bankEntries % resultWords == 0, + "SMatMulBall bank depth must be divisible by four" + ) + require(maxOutputRows >= rows, "SMatMulBall accumulator must hold at least one core") + require((osFunct >> 4) == 4, "SMATMUL_OS must encode two reads and one write") + require((biasFunct >> 4) == 1, "SMATMUL_BIAS must encode one read") + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + mapping.inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + mapping.outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val Seq( + idle, + waitForChannels, + biasReadReq, + biasReadResp, + initAccumulator, + loadTile, + runArray, + readAccumulator, + writeAccumulator, + readResult, + holdResult, + writeResult, + waitForCWrite, + complete + ) = Enum(14) + + val state = RegInit(idle) + + val robId = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + + val biasCommand = RegInit(false.B) + val firstBlock = RegInit(false.B) + val lastBlock = RegInit(false.B) + val vectorMode = RegInit(false.B) + val biasValid = RegInit(false.B) + val chainLive = RegInit(false.B) + val biasBank = RegInit(0.U(bankIdWidth.W)) + val biasBase = RegInit(0.U(log2Ceil(memParam.bankEntries).W)) + val biasRow = RegInit(0.U(2.W)) + val biasWords = Reg(Vec(resultWords, UInt(128.W))) + + val aBank = RegInit(0.U(bankIdWidth.W)) + val bBank = RegInit(0.U(bankIdWidth.W)) + val cBank = RegInit(0.U(bankIdWidth.W)) + val bases = Reg(Vec(3, UInt(memParam.groupIdWidth.W))) + val cBase = RegInit(0.U(addressWidth.W)) + val chainRows = RegInit(0.U(12.W)) + val chainCBank = RegInit(0.U(bankIdWidth.W)) + val chainCBase = RegInit(0.U(addressWidth.W)) + + val outputTileCount = RegInit(0.U(8.W)) + val reductionTileCount = RegInit(0.U(8.W)) + val outputTile = RegInit(0.U(8.W)) + val reductionTile = RegInit(0.U(8.W)) + val accumulatorRow = RegInit(0.U(4.W)) + val resultRow = RegInit(0.U(4.W)) + val outputWord = RegInit(0.U(2.W)) + val aRowsRequested = RegInit(0.U(5.W)) + val aRowsStored = RegInit(0.U(5.W)) + val bRowsRequested = RegInit(0.U(5.W)) + val bRowsStored = RegInit(0.U(5.W)) + val cRowData = Reg(UInt(512.W)) + + val aRows = Reg(Vec(rows, UInt(128.W))) + val bRows = Reg(Vec(cols, UInt(128.W))) + val accumulator = SyncReadMem(accDepth, UInt(512.W)) + val array: Instance[Array] = Instantiate(new Array(ballParam)) + + val computeGlobalRow = outputTile * rows.U + accumulatorRow + val resultGlobalRow = outputTile * rows.U + resultRow + val accumulatorRead = state === readAccumulator || state === readResult + val accumulatorWrite = state === initAccumulator || state === writeAccumulator + + val accumulatorAddress = Mux( + state === readResult, + resultGlobalRow(accAddrWidth - 1, 0), + computeGlobalRow(accAddrWidth - 1, 0) + ) + + val biasData = Cat(biasWords.reverse) + val accumulatorWriteData = Wire(UInt(512.W)) + + val accumulatorData = accumulator.readWrite( + accumulatorAddress, + accumulatorWriteData, + accumulatorRead || accumulatorWrite, + accumulatorWrite + ) + + val accumulatedResult = Wire(Vec(cols, UInt(32.W))) + for (column <- 0 until cols) { + val oldValue = accumulatorData(32 * column + 31, 32 * column).asSInt + val newValue = array.io.result(accumulatorRow)(32 * column + 31, 32 * column).asSInt + accumulatedResult(column) := (oldValue + newValue).asUInt + } + accumulatorWriteData := Mux(state === initAccumulator, biasData, Cat(accumulatedResult.reverse)) + assert(!(accumulatorRead && accumulatorWrite), "SMatMulBall accumulator SRAM is single-port") + + val aTileLine = Mux( + vectorMode, + reductionTile, + (outputTile * reductionTileCount + reductionTile) * rows.U + ) + + val bTileLine = reductionTile * cols.U + val cLine = cBase +& (resultGlobalRow << 2) +& outputWord + + for (port <- 0 until mapping.inBW) { + io.bankRead(port).rob_id := robId + io.bankRead(port).bank_id := Mux(port.U === 0.U, aBank, bBank) + io.bankRead(port).group_id := bases(port) + io.bankRead(port).io.req.valid := false.B + io.bankRead(port).io.req.bits.addr := 0.U + io.bankRead(port).io.resp.ready := false.B + } + io.bankRead(0).bank_id := Mux(state === biasReadReq || state === biasReadResp, biasBank, aBank) + io.bankRead(0).io.req.valid := Mux( + state === biasReadReq, + true.B, + state === loadTile && aRowsRequested < Mux(vectorMode, 1.U, rows.U) + ) + io.bankRead(0).io.req.bits.addr := Mux( + state === biasReadReq, + (biasBase +& biasRow)(addressWidth - 1, 0), + (aTileLine + aRowsRequested)(addressWidth - 1, 0) + ) + io.bankRead(0).io.resp.ready := state === biasReadResp || (state === loadTile && aRowsStored < rows.U) + io.bankRead(1).io.req.valid := state === loadTile && bRowsRequested < cols.U + io.bankRead(1).io.req.bits.addr := (bTileLine + bRowsRequested)(addressWidth - 1, 0) + io.bankRead(1).io.resp.ready := state === loadTile && bRowsStored < cols.U + + val cWords = cRowData.asTypeOf(Vec(resultWords, UInt(128.W))) + io.bankWrite(0).rob_id := robId + io.bankWrite(0).bank_id := cBank + io.bankWrite(0).group_id := bases(2) + io.bankWrite(0).io.req.valid := state === writeResult + io.bankWrite(0).io.req.bits.addr := cLine(addressWidth - 1, 0) + io.bankWrite(0).io.req.bits.data := cWords(outputWord) + io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(16)(true.B)) + io.bankWrite(0).io.resp.ready := state === waitForCWrite + + array.io.start := state === loadTile && + aRowsStored === Mux(vectorMode, 1.U, rows.U) && bRowsStored === cols.U + array.io.aRows := aRows + array.io.bRows := bRows + + io.cmdReq.ready := state === idle + io.cmdResp.valid := state === complete + io.cmdResp.bits.rob_id := robId + io.status.idle := state === idle + io.status.running := state =/= idle && state =/= complete + + when(io.cmdReq.fire) { + val command = io.cmdReq.bits + bases := VecInit(command.op1_group, command.op2_group, command.wr_group) + val isBias = command.funct === biasFunct.U + val isMatmul = command.funct === osFunct.U + val rows = command.rs2(11, 0) + val columns = command.rs2(23, 12) + val reduction = command.iter + val first = command.rs2(24) + val last = command.rs2(25) + + assert(isBias || isMatmul, "SMatMulBall received an unknown funct7") + + robId := io.cmdReq.bits.rob_id + biasCommand := isBias + + when(isBias) { + assert(command.iter === 4.U, "SMATMUL_BIAS iter must be four rows") + assert(command.rs2(63, 6) === 0.U, "SMATMUL_BIAS reserves rs2[63:6]") + assert( + command.rs2(5, 0) +& 4.U <= memParam.bankEntries.U, + "SMATMUL_BIAS inputBase plus four rows must fit one bank" + ) + assert( + command.op1_col === 1.U && command.op2_col === 0.U && command.wr_col === 0.U, + "SMATMUL_BIAS requires exactly one input bank" + ) + assert(!chainLive, "SMATMUL_BIAS cannot replace bias during an accumulation chain") + biasBank := command.op1_bank + biasBase := command.rs2(5, 0) + biasRow := 0.U + }.otherwise { + assert( + command.op1_bank =/= command.op2_bank && command.op1_bank =/= command.wr_bank && + command.op2_bank =/= command.wr_bank, + "SMATMUL_OS requires distinct A, B, and C banks" + ) + assert( + command.op1_col === 1.U && command.op2_col === 1.U && command.wr_col === 1.U, + "SMATMUL_OS requires one physical bank per operand" + ) + assert( + rows === 1.U || (rows =/= 0.U && rows(2, 0) === 0.U), + "SMATMUL_OS M must be one or a positive multiple of 8" + ) + assert(columns === cols.U, "SMATMUL_OS N must be 16") + assert(reduction =/= 0.U && reduction(3, 0) === 0.U, "SMATMUL_OS K must be a positive multiple of 16") + assert(command.rs2(63, 32) === 0.U, "SMATMUL_OS reserves rs2[63:32]") + assert( + Mux( + rows === 1.U, + reduction >> 4, + (rows * reduction) >> 4 + ) <= memParam.bankEntries.U, + "SMATMUL_OS A footprint exceeds bank depth" + ) + assert( + reduction <= memParam.bankEntries.U, + "SMATMUL_OS B footprint exceeds bank depth" + ) + assert( + command.rs2( + 31, + 26 + ) +& rows * resultWords.U <= memParam.bankEntries.U, + "SMATMUL_OS C footprint exceeds bank depth" + ) + when(first) { + assert(!chainLive, "SMATMUL_OS first block cannot start while a chain is live") + assert(biasValid, "SMATMUL_OS first block requires a bias preload") + chainLive := true.B + chainRows := rows + // A chain retains its architectural destination across bank-version renaming. + chainCBank := command.wr_arch_bank + chainCBase := command.rs2(31, 26) + }.otherwise { + assert(chainLive, "SMATMUL_OS continuation requires a live chain") + assert( + rows === chainRows && command.wr_arch_bank === chainCBank && + command.rs2(31, 26) === chainCBase, + "SMATMUL_OS continuation changed M or C destination/base" + ) + } + + firstBlock := first + lastBlock := last + vectorMode := rows === 1.U + aBank := command.op1_bank + bBank := command.op2_bank + cBank := command.wr_bank + cBase := command.rs2(31, 26) + outputTileCount := Mux(rows === 1.U, 1.U, rows / param.tileRows.U) + reductionTileCount := reduction >> 4 + outputTile := 0.U + reductionTile := 0.U + accumulatorRow := 0.U + resultRow := 0.U + outputWord := 0.U + aRowsRequested := 0.U + aRowsStored := 0.U + bRowsRequested := 0.U + bRowsStored := 0.U + for (row <- 1 until 8) { + aRows(row) := 0.U + } + } + state := waitForChannels + } + + when(state === waitForChannels) { + state := Mux(biasCommand, biasReadReq, Mux(firstBlock, initAccumulator, loadTile)) + } + + when(state === biasReadReq && io.bankRead(0).io.req.fire) { + state := biasReadResp + } + when(state === biasReadResp && io.bankRead(0).io.resp.fire) { + biasWords(biasRow) := io.bankRead(0).io.resp.bits.data + when(biasRow === (resultWords - 1).U) { + biasValid := true.B + state := complete + }.otherwise { + biasRow := biasRow + 1.U + state := biasReadReq + } + } + + when(state === initAccumulator) { + when(accumulatorRow === Mux(vectorMode, 0.U, (rows - 1).U)) { + accumulatorRow := 0.U + when(outputTile + 1.U === outputTileCount) { + outputTile := 0.U + reductionTile := 0.U + aRowsRequested := 0.U + aRowsStored := 0.U + bRowsRequested := 0.U + bRowsStored := 0.U + state := loadTile + }.otherwise { + outputTile := outputTile + 1.U + } + }.otherwise { + accumulatorRow := accumulatorRow + 1.U + } + } + + when(state === loadTile) { + when(io.bankRead(0).io.req.fire)(aRowsRequested := aRowsRequested + 1.U) + when(io.bankRead(1).io.req.fire)(bRowsRequested := bRowsRequested + 1.U) + when(io.bankRead(0).io.resp.fire) { + aRows(aRowsStored(3, 0)) := io.bankRead(0).io.resp.bits.data + aRowsStored := aRowsStored + 1.U + } + when(io.bankRead(1).io.resp.fire) { + bRows(bRowsStored(3, 0)) := io.bankRead(1).io.resp.bits.data + bRowsStored := bRowsStored + 1.U + } + when(aRowsStored === Mux(vectorMode, 1.U, rows.U) && bRowsStored === cols.U) { + state := runArray + } + } + + when(state === runArray && array.io.done) { + accumulatorRow := 0.U + state := readAccumulator + } + when(state === readAccumulator) { + state := writeAccumulator + } + when(state === writeAccumulator) { + when(accumulatorRow === Mux(vectorMode, 0.U, (rows - 1).U)) { + accumulatorRow := 0.U + when(reductionTile + 1.U < reductionTileCount) { + reductionTile := reductionTile + 1.U + aRowsRequested := 0.U + aRowsStored := 0.U + bRowsRequested := 0.U + bRowsStored := 0.U + state := loadTile + }.elsewhen(outputTile + 1.U < outputTileCount) { + outputTile := outputTile + 1.U + reductionTile := 0.U + aRowsRequested := 0.U + aRowsStored := 0.U + bRowsRequested := 0.U + bRowsStored := 0.U + state := loadTile + }.otherwise { + when(lastBlock) { + outputTile := 0.U + resultRow := 0.U + state := readResult + }.otherwise { + state := complete + } + } + }.otherwise { + accumulatorRow := accumulatorRow + 1.U + state := readAccumulator + } + } + + when(state === readResult) { + outputWord := 0.U + state := holdResult + } + when(state === holdResult) { + cRowData := accumulatorData + state := writeResult + } + when(state === writeResult && io.bankWrite(0).io.req.fire) { + state := waitForCWrite + } + when(state === waitForCWrite && io.bankWrite(0).io.resp.fire) { + when(outputWord =/= (resultWords - 1).U) { + outputWord := outputWord + 1.U + state := writeResult + }.elsewhen(resultRow =/= Mux(vectorMode, 0.U, (rows - 1).U)) { + resultRow := resultRow + 1.U + state := readResult + }.elsewhen(outputTile + 1.U < outputTileCount) { + outputTile := outputTile + 1.U + resultRow := 0.U + state := readResult + }.otherwise { + chainLive := false.B + state := complete + } + } + + when(state === complete && io.cmdResp.fire) { + state := idle + } +} diff --git a/examples/components/balls/smatmul/configs/SMatMulBallParam.scala b/examples/components/balls/smatmul/configs/SMatMulBallParam.scala new file mode 100644 index 000000000..84b1791c2 --- /dev/null +++ b/examples/components/balls/smatmul/configs/SMatMulBallParam.scala @@ -0,0 +1,20 @@ +package examples.balls.smatmul.configs + +import buckyball.balls.smatmul.{Params => ParametersProto} +import upickle.default.{macroRW, ReadWriter} + +case class SMatMulBallParam( + tileRows: Int, + tileCols: Int) + +object SMatMulBallParam { + + def fromProto(source: ParametersProto): SMatMulBallParam = { + val parameters = source + require(parameters.hasTileRows && parameters.hasTileCols) + val param = SMatMulBallParam(parameters.getTileRows, parameters.getTileCols) + param + } + + implicit val rw: ReadWriter[SMatMulBallParam] = macroRW +} diff --git a/examples/components/balls/smatmul/configs/default.toml b/examples/components/balls/smatmul/configs/default.toml new file mode 100644 index 000000000..db3557dc7 --- /dev/null +++ b/examples/components/balls/smatmul/configs/default.toml @@ -0,0 +1,4 @@ +# SMatMul Ball Configuration + +tileRows = 16 +tileCols = 16 diff --git a/examples/components/balls/smatmul/configs/lane16.toml b/examples/components/balls/smatmul/configs/lane16.toml new file mode 100644 index 000000000..d4cc1e6ea --- /dev/null +++ b/examples/components/balls/smatmul/configs/lane16.toml @@ -0,0 +1,4 @@ +# SMatMul Ball Configuration + +tileRows = 8 +tileCols = 16 diff --git a/examples/components/balls/smatmul/configs/smatmul_params.proto b/examples/components/balls/smatmul/configs/smatmul_params.proto new file mode 100644 index 000000000..299bbd328 --- /dev/null +++ b/examples/components/balls/smatmul/configs/smatmul_params.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; +package buckyball.balls.smatmul; +option java_package = "buckyball.balls.smatmul"; +option java_multiple_files = true; + +message Params { + optional int32 tileRows = 1; + optional int32 tileCols = 2; +} diff --git a/examples/components/balls/smatmul/emu/src/dispatch.rs b/examples/components/balls/smatmul/emu/src/dispatch.rs new file mode 100644 index 000000000..4d832ba86 --- /dev/null +++ b/examples/components/balls/smatmul/emu/src/dispatch.rs @@ -0,0 +1,38 @@ +use bebop_bemu::thread::instructions::decode; +use bebop_bemu::ball::ExecContext; + +use super::{fp32, mxfp8, smatmul}; + +const BALL_CLASS: &str = "examples.balls.smatmul.SMatMulBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("SMATMUL_OS") => Some(smatmul::exec_smatmul(xs1, xs2, ctx)), + Some("SMATMUL_MXFP8") => Some(mxfp8::execute(xs1, xs2, ctx)), + Some("SMATMUL_F32") => Some(fp32::execute(xs1, xs2, ctx)), + Some("SMATMUL_BIAS") => Some(smatmul::exec_bias(xs1, xs2, ctx)), + _ => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("SMATMUL_OS") => Some(smatmul::latency(xs1, xs2, bebop_bemu::configs::ball_domain::instance_for_funct(funct).expect("Ball instance"))), + Some("SMATMUL_MXFP8") => Some(smatmul::latency(xs1, xs2, bebop_bemu::configs::ball_domain::instance_for_funct(funct).expect("Ball instance"))), + Some("SMATMUL_F32") => Some(decode::rs1_iter(xs1) as u64 + (xs2 & 0xfff) + ((xs2 >> 12) & 0xfff)), + Some("SMATMUL_BIAS") => Some(smatmul::bias_latency(xs1, xs2)), + _ => None, + } +} diff --git a/examples/components/balls/smatmul/emu/src/fp32.rs b/examples/components/balls/smatmul/emu/src/fp32.rs new file mode 100644 index 000000000..354b010d4 --- /dev/null +++ b/examples/components/balls/smatmul/emu/src/fp32.rs @@ -0,0 +1,118 @@ +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::instruction::ExecContext; +use std::collections::HashMap; + +struct Chain { + rows: usize, + bank: u64, + base: usize, + values: Vec, +} + + + +pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + let ball_state = ctx.state::>(); + let rows = (xs2 & 0xfff) as usize; + let cols = ((xs2 >> 12) & 0xfff) as usize; + let k = rs1_iter(xs1) as usize; + let first = (xs2 >> 24) & 1 != 0; + let last = (xs2 >> 25) & 1 != 0; + let base = ((xs2 >> 26) & 63) as usize; + let a = rs1_b0(xs1); + let b = rs1_b1(xs1); + let c = rs1_b2(xs1); + assert!( + xs2 >> 32 == 0 && (rows == 1 || rows == 16) && cols == 16 && k > 0 && k % 4 == 0, + "f32 matmul: requires M=1/16, N=16, positive K multiple of four and zero reserved bits" + ); + assert!(a != b && b != c && a != c, "f32 matmul: banks must differ"); + for bank in [a, b, c] { + assert!( + ctx.config(bank).allocated && ctx.config(bank).cols == 1, + "f32 matmul: banks must be allocated with one column" + ); + } + let stride = bank_row_bytes(); + assert!( + stride == 16 + && rows * k * 4 <= bank_lines() * stride + && cols * k * 4 <= bank_lines() * stride + && base * stride + rows * cols * 4 <= bank_lines() * stride, + "f32 matmul: bank footprint exceeds depth" + ); + let pa = pbank(ctx, a); + let pb = pbank(ctx, b); + let av = &ctx.banks[pa][..rows * k * 4]; + let bv = &ctx.banks[pb][..cols * k * 4]; + assert!( + av.chunks_exact(4) + .chain(bv.chunks_exact(4)) + .all(|x| f32::from_le_bytes(x.try_into().unwrap()).is_finite()), + "f32 matmul: non-finite operand" + ); + let mut chain = { + let mut chains = ball_state.lock().unwrap(); + if first { + assert!( + !chains.contains_key(&ctx.hart_id), + "f32 matmul: first while chain is live" + ); + Chain { + rows, + bank: c, + base, + values: vec![0f32; rows * cols], + } + } else { + chains + .remove(&ctx.hart_id) + .expect("f32 matmul: continuation without first") + } + }; + assert!( + chain.rows == rows && chain.bank == c && chain.base == base, + "f32 matmul: continuation changed destination" + ); + let weights: [&[u8]; 16] = std::array::from_fn(|col| &bv[col * k * 4..(col + 1) * k * 4]); + for row in 0..rows { + if av[row * k * 4..(row + 1) * k * 4] + .chunks_exact(4) + .all(|x| u32::from_le_bytes(x.try_into().unwrap()) & 0x7fffffff == 0) + && chain.values[row * cols..(row + 1) * cols] + .iter() + .all(|x| x.to_bits() == 0) + { + continue; + } + let av = &av[row * k * 4..(row + 1) * k * 4]; + let dst = &mut chain.values[row * cols..(row + 1) * cols]; + let mut sums: [f32; 16] = dst.try_into().unwrap(); + for inner in 0..k { + let a = f32::from_le_bytes(av[inner * 4..inner * 4 + 4].try_into().unwrap()); + for col in 0..16 { + let b = + f32::from_le_bytes(weights[col][inner * 4..inner * 4 + 4].try_into().unwrap()); + sums[col] = a.mul_add(b, sums[col]); + } + } + assert!( + sums.iter().all(|x| x.is_finite()), + "f32 matmul: accumulator overflow" + ); + dst.copy_from_slice(&sums); + } + if last { + let pc = pbank(ctx, c); + for (index, value) in chain.values.iter().enumerate() { + let offset = base * stride + index * 4; + ctx.banks[pc][offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + } else { + { + ball_state.lock().unwrap().insert(ctx.hart_id, chain); + }; + } + 0 +} diff --git a/examples/components/balls/smatmul/emu/src/lib.rs b/examples/components/balls/smatmul/emu/src/lib.rs new file mode 100644 index 000000000..58a903abf --- /dev/null +++ b/examples/components/balls/smatmul/emu/src/lib.rs @@ -0,0 +1,13 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod fp32; +mod mxfp8; +mod smatmul; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/components/balls/smatmul/emu/src/mxfp8.rs b/examples/components/balls/smatmul/emu/src/mxfp8.rs new file mode 100644 index 000000000..8f39fed1e --- /dev/null +++ b/examples/components/balls/smatmul/emu/src/mxfp8.rs @@ -0,0 +1,163 @@ +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::instruction::ExecContext; +use std::collections::HashMap; + +struct Chain { + rows: usize, + cols: usize, + bank: u64, + base: usize, + values: Vec, +} + + + +fn value(code: u8, factor: f32) -> f32 { + let exponent = (code >> 3) & 15; + let mantissa = (code & 7) as u32; + let normal = ((exponent as u32 + 120) << 23) | (mantissa << 20); + let subnormal = (mantissa as f32 / 512.0).to_bits(); + let element = f32::from_bits(if exponent == 0 { subnormal } else { normal }); + let result = f32::from_bits((element * factor).to_bits() | ((code as u32 & 128) << 24)); + result +} + +pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + let ball_state = ctx.state::>(); + let rows = (xs2 & 0xfff) as usize; + let cols = ((xs2 >> 12) & 0xfff) as usize; + let k = rs1_iter(xs1) as usize; + let first = (xs2 >> 24) & 1 != 0; + let last = (xs2 >> 25) & 1 != 0; + let base = ((xs2 >> 26) & 63) as usize; + let a = rs1_b0(xs1); + let b = rs1_b1(xs1); + let c = rs1_b2(xs1); + assert!( + xs2 >> 32 == 0 && (rows == 1 || rows == 16) && cols == 16 && k > 0 && k % 32 == 0, + "mxfp8: requires M=1/16, N=16, K multiple of 32 and zero reserved bits" + ); + assert!(a != b && b != c && a != c, "mxfp8: banks must differ"); + for bank in [a, b, c] { + assert!( + ctx.config(bank).allocated && ctx.config(bank).cols == 1, + "mxfp8: banks must be allocated with one column" + ); + } + let stride = bank_row_bytes(); + assert!(stride == 16, "mxfp8: requires 16-byte bank rows"); + assert!( + rows * k + rows * k / 32 <= bank_lines() * stride + && cols * k + cols * k / 32 <= bank_lines() * stride + && base * stride + rows * cols * 4 <= bank_lines() * stride, + "mxfp8: bank footprint exceeds depth" + ); + let pa = pbank(ctx, a); + let pb = pbank(ctx, b); + let a_bytes = &ctx.banks[pa][..rows * k + rows * k / 32]; + let b_bytes = &ctx.banks[pb][..cols * k + cols * k / 32]; + assert!( + a_bytes[..rows * k] + .iter() + .chain(b_bytes[..cols * k].iter()) + .fold(true, |valid, code| valid & (code & 0x7f != 0x7f)), + "mxfp8: NaN encoding" + ); + let mut av = vec![0f32; rows * k]; + let mut bv = vec![0f32; cols * k]; + for (block, dst) in av.chunks_exact_mut(32).enumerate() { + let scale = a_bytes[rows * k + block]; + assert!(scale != 255, "mxfp8: NaN encoding"); + let factor = if scale == 0 { + f32::from_bits(1 << 22) + } else { + f32::from_bits((scale as u32) << 23) + }; + for (index, decoded) in dst.iter_mut().enumerate() { + *decoded = value(a_bytes[block * 32 + index], factor); + } + } + for col in 0..cols { + for block in 0..k / 32 { + let scale = b_bytes[cols * k + col * (k / 32) + block]; + assert!(scale != 255, "mxfp8: NaN encoding"); + let factor = if scale == 0 { + f32::from_bits(1 << 22) + } else { + f32::from_bits((scale as u32) << 23) + }; + let codes = &b_bytes[col * k + block * 32..col * k + (block + 1) * 32]; + let mut decoded = [0f32; 32]; + for (decoded, code) in decoded.iter_mut().zip(codes.iter()) { + *decoded = value(*code, factor); + } + for (index, decoded) in decoded.iter().enumerate() { + bv[(block * 32 + index) * cols + col] = *decoded; + } + } + } + assert!( + av.iter() + .chain(bv.iter()) + .fold(true, |valid, value| valid & value.is_finite()), + "mxfp8: decoded value overflows FP32" + ); + let mut chain = { + let mut chains = ball_state.lock().unwrap(); + if first { + assert!( + !chains.contains_key(&ctx.hart_id), + "mxfp8: first while chain is live" + ); + Chain { + rows, + cols, + bank: c, + base, + values: vec![0f32; rows * cols], + } + } else { + chains + .remove(&ctx.hart_id) + .expect("mxfp8: continuation without first") + } + }; + assert!( + chain.rows == rows && chain.cols == cols && chain.bank == c && chain.base == base, + "mxfp8: continuation changed destination" + ); + for row in 0..rows { + if av[row * k..(row + 1) * k].iter().all(|value| *value == 0.0) + && chain.values[row * cols..(row + 1) * cols] + .iter() + .all(|value| value.to_bits() == 0) + { + continue; + } + let av = &av[row * k..(row + 1) * k]; + let dst = &mut chain.values[row * cols..(row + 1) * cols]; + let mut sums: [f32; 16] = dst.try_into().unwrap(); + for inner in 0..k { + let a = av[inner]; + let weights = &bv[inner * 16..(inner + 1) * 16]; + for col in 0..16 { + sums[col] = a.mul_add(weights[col], sums[col]); + } + } + assert!(sums.iter().all(|v| v.is_finite()), "accumulator overflow"); + dst.copy_from_slice(&sums); + } + if last { + let pc = pbank(ctx, c); + for (i, value) in chain.values.iter().enumerate() { + ctx.banks[pc][base * stride + i * 4..base * stride + i * 4 + 4] + .copy_from_slice(&value.to_le_bytes()); + } + } else { + { + ball_state.lock().unwrap().insert(ctx.hart_id, chain); + }; + } + 0 +} diff --git a/examples/balls/smatmul/emu/src/smatmul.rs b/examples/components/balls/smatmul/emu/src/smatmul.rs similarity index 84% rename from examples/balls/smatmul/emu/src/smatmul.rs rename to examples/components/balls/smatmul/emu/src/smatmul.rs index e4152858b..5381f2257 100644 --- a/examples/balls/smatmul/emu/src/smatmul.rs +++ b/examples/components/balls/smatmul/emu/src/smatmul.rs @@ -1,13 +1,10 @@ -use super::super::bank::{bank_lines, bank_row_bytes}; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::instruction::ExecContext; -use std::cell::RefCell; use std::collections::HashMap; -const BALL_CLASS: &str = "examples.balls.smatmul.SMatMulBall"; - -fn tile_rows() -> usize { crate::config::ball_domain::param(BALL_CLASS, "tileRows") } -fn tile_cols() -> usize { crate::config::ball_domain::param(BALL_CLASS, "tileCols") } +fn tile_rows(instance: u32) -> usize { bebop_bemu::configs::ball_domain::parameters::("smatmul", instance).tile_rows.expect("tileRows") as usize } +fn tile_cols(instance: u32) -> usize { bebop_bemu::configs::ball_domain::parameters::("smatmul", instance).tile_cols.expect("tileCols") as usize } #[derive(Default)] struct State { @@ -23,9 +20,7 @@ struct Chain { accumulators: Vec, } -thread_local! { - static STATES: RefCell> = RefCell::new(HashMap::new()); -} + fn read_i32(bank: &[u8], row: usize, lane: usize) -> i32 { let offset = row * bank_row_bytes() + lane * 4; @@ -33,6 +28,7 @@ fn read_i32(bank: &[u8], row: usize, lane: usize) -> i32 { } pub(crate) fn exec_bias(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + let ball_state = ctx.state::>(); let bank = rs1_b0(xs1); let input_base = (xs2 & 0x3f) as usize; if rs1_b1(xs1) != 0 || rs1_b2(xs1) != 0 || rs1_iter(xs1) != 4 || xs2 >> 6 != 0 { @@ -45,27 +41,28 @@ pub(crate) fn exec_bias(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { panic!("smatmul_bias: inputBase plus four rows exceeds bank depth"); } let physical = pbank(ctx, bank); - let mut bias = vec![0i32; tile_cols()]; + let mut bias = vec![0i32; tile_cols(ctx.ball_id.expect("Ball instance"))]; for group in 0..4 { for lane in 0..4 { bias[group * 4 + lane] = read_i32(&ctx.banks[physical], input_base + group, lane); } } - STATES.with(|states| { - let mut states = states.borrow_mut(); + { + let mut states = ball_state.lock().unwrap(); let state = states.entry(ctx.hart_id).or_default(); if state.chain.is_some() { panic!("smatmul_bias: cannot replace bias during an accumulation chain"); } state.bias = Some(bias); - }); + }; 0 } pub(crate) fn exec_smatmul(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { - let m_tile = tile_rows(); - let k_tile = tile_cols(); - let n_tile = tile_cols(); + let ball_state = ctx.state::>(); + let m_tile = tile_rows(ctx.ball_id.expect("Ball instance")); + let k_tile = tile_cols(ctx.ball_id.expect("Ball instance")); + let n_tile = tile_cols(ctx.ball_id.expect("Ball instance")); let a_bank = rs1_b0(xs1); let b_bank = rs1_b1(xs1); let c_bank = rs1_b2(xs1); @@ -102,8 +99,8 @@ pub(crate) fn exec_smatmul(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { panic!("smatmul: bank footprint exceeds bank depth"); } - let mut chain = STATES.with(|states| { - let mut states = states.borrow_mut(); + let mut chain = { + let mut states = ball_state.lock().unwrap(); let state = states.entry(ctx.hart_id).or_default(); if first { if state.chain.is_some() { @@ -130,7 +127,7 @@ pub(crate) fn exec_smatmul(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { .take() .unwrap_or_else(|| panic!("smatmul: continuation has no live chain")) } - }); + }; if chain.rows != rows || chain.cols != cols || chain.output_bank != c_bank @@ -182,9 +179,9 @@ pub(crate) fn exec_smatmul(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { } } } else { - STATES.with(|states| { - states.borrow_mut().entry(ctx.hart_id).or_default().chain = Some(chain); - }); + { + ball_state.lock().unwrap().entry(ctx.hart_id).or_default().chain = Some(chain); + }; } 0 } @@ -202,10 +199,10 @@ pub(crate) fn bias_latency(xs1: u64, xs2: u64) -> u64 { 4 } -pub(crate) fn latency(xs1: u64, xs2: u64) -> u64 { - let m_tile = tile_rows(); - let k_tile = tile_cols(); - let n_tile = tile_cols(); +pub(crate) fn latency(xs1: u64, xs2: u64, instance: u32) -> u64 { + let m_tile = tile_rows(instance); + let k_tile = tile_cols(instance); + let n_tile = tile_cols(instance); let rows = xs2 & 0xfff; let cols = (xs2 >> 12) & 0xfff; let k = rs1_iter(xs1); diff --git a/examples/components/balls/smatmul/stack/src/Lowering.cpp b/examples/components/balls/smatmul/stack/src/Lowering.cpp new file mode 100644 index 000000000..0ec2620ca --- /dev/null +++ b/examples/components/balls/smatmul/stack/src/Lowering.cpp @@ -0,0 +1,79 @@ +//===- Lowering.cpp - SMatMulBall in lower-buckyball ----------------------===// +// +// matmul_i8 with read_destination: C = C + A @ B for `rows` rows (one or a +// multiple of eight) and sixteen columns, over `k` (a multiple of sixteen). +// C starts with one row of sixteen i32 (four bank rows), the bias of every +// row, and receives `rows` rows of sixteen i32 (four bank rows each). +// A is contiguous for rows=1, otherwise packed in tileRows x 16 blocks; +// B holds k rows of sixteen i8. +// matmul_i8_block extends the reduction across instructions: first loads +// the bias from C, and last writes the completed accumulation to C. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballOps.h" +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class MatmulI8Lowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &rewriter) const override { + bool block = op.getKind() == "matmul_i8_block"; + if (op.getKind() != "matmul_i8" && !block) + return failure(); + if (!op.getReadDestination() || op.getSources().size() != 2) + return op.emitOpError("INT8 matmul reads A, B and the old C"); + Location loc = op.getLoc(); + int64_t rows = cast(op->getAttr("rows")).getInt(); + int64_t k = cast(op->getAttr("k")).getInt(); + int64_t c = op.getDestination(); + bool first = !block || cast(op->getAttr("first")).getValue(); + bool last = !block || cast(op->getAttr("last")).getValue(); + const auto &target = buckyball_target::getBuckyballTarget(); + int64_t tileRows = + buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); + int64_t tileCols = + buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileCols"); + if (target.bankWidthBits != 128 || tileCols != 16 || tileRows <= 0 || + rows <= 0 || rows > 4095 || (rows != 1 && rows % tileRows) || k <= 0 || + k >= (1LL << 33) || k > target.bankDepth || k % 16 || + rows * (k / 16) > target.bankDepth || rows * 4 > target.bankDepth) + return op.emitOpError( + "INT8 matmul shape exceeds the bank or ISA geometry"); + if (op.getSources()[0] == op.getSources()[1] || op.getSources()[0] == c || + op.getSources()[1] == c) + return op.emitOpError("INT8 matmul requires distinct A, B and C banks"); + // The bias of the accumulation: the four rows at the base of C. + if (first) + mlir::buddy::emitInstruction( + rewriter, loc, buckyball_target::getBuckyballFunct7("SMATMUL_BIAS"), + mlir::buddy::constantI64(rewriter, loc, + mlir::buddy::bankFields({c}, 0, false, 4)), + mlir::buddy::constantI64(rewriter, loc, 0)); + uint64_t rs2 = + mlir::buddy::field(rows, 0, 11) | mlir::buddy::field(16, 12, 23) | + mlir::buddy::field(first, 24, 24) | mlir::buddy::field(last, 25, 25); + mlir::buddy::emitInstruction( + rewriter, loc, buckyball_target::getBuckyballFunct7("SMATMUL_OS"), + mlir::buddy::constantI64( + rewriter, loc, + mlir::buddy::bankFields(op.getSources(), c, false, k)), + mlir::buddy::constantI64(rewriter, loc, rs2)); + rewriter.eraseOp(op); + return success(); + } +}; + +void populate(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} + +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallLowering(populate), true); +} // namespace diff --git a/examples/components/balls/smatmul/verify/README.md b/examples/components/balls/smatmul/verify/README.md new file mode 100644 index 000000000..1687af818 --- /dev/null +++ b/examples/components/balls/smatmul/verify/README.md @@ -0,0 +1,81 @@ +# SMatMulBall UVM Verification + +Verifies the generated `SMatMulBall` module using the shared Blink UVM framework. + +## Structure + +- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM agents and base env +- `src/common/smatmul_defs.svh`: DPI imports, `matrix_require_bid`, timeouts +- `src/common/smatmul_items.svh`: `smatmul_cmd_item` with `load_rust_case` +- `src/seq/smatmul_sequences.svh`: one-case sequence +- `src/cov/smatmul_cov.svh`: M/N/K shape bins `{1,2,4,8,16}` +- `src/env/smatmul_scoreboard.svh`: preload A/B, compare writes vs DPI expected list +- `src/env/smatmul_env.svh`: extends `bb_blink_env#(2,4)` +- `src/tests/smatmul_test.svh`: directed 0..3, random 4..23 +- `src/pkg/smatmul_pkg.sv`: package entry +- `src/tb_top.sv`: `bb_blink_if#(2,2)` + `SMatMulBall` (2 read, 2 write ports) +- `filelists/smatmul_ball.f`: VCS filelist + +## Test plan + +`+UVM_TESTNAME=smatmul_ball_test` runs: + +- case 0: 4x4x4 OS (banks 0,1,2) +- case 1: 5x7x3 OS tail +- case 2: 16x16x16 OS +- case 3: 32x16x16 WS +- cases 4..23: random M in {1,2,4,8,16,32}, N/K in {1,2,4,8,16}, distinct banks 0..7 + +Each case: reset, preload A/B into mem model, drive one command, wait for +`scb.done()` or timeout at 100000 cycles. Scoreboard compares writes +(group/addr/data/mask) in DPI emission order. + +## BID (required) + +`+BID=` is mandatory. Missing it fatals at test start. No default bid. + +| Config | Plusarg | +|--------|---------| +| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=1` | +| `sims.verilator.BuckyballToyVerilatorConfig` | `+BID=4` | + +```console +./simv +UVM_TESTNAME=smatmul_ball_test +BID=4 +``` + +## Build + +```console +nix develop ../../../../verify +cargo build --manifest-path casegen/Cargo.toml +``` + +Compile from this directory: + +```console +vcs -full64 -sverilog -timescale=1ns/1ps \ + $VCS_UVM_ARGS \ + -sv_lib ../output/casegen/debug/libmatrix_casegen \ + -f filelists/smatmul_ball.f +``` + +Run: + +```console +./simv +UVM_TESTNAME=smatmul_ball_test +BID=4 +``` + +Or via bbdev: + +```console +bbdev uvm --build '--ball=matrix' +bbdev uvm --run '--ball=matrix' -- '+BID=4' +``` + +## Acceptance + +- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` +- [x] casegen whole-case DPI API; `cargo test` in casegen +- [x] VCS build with `smatmul_ball.f` (no segfault on Instantiate DUT) +- [x] `+BID=` required; pebble `+BID=1` / toy `+BID=4` +- [ ] 23/23 functional pass diff --git a/examples/balls/smatmul/verify/casegen/Cargo.lock b/examples/components/balls/smatmul/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/smatmul/verify/casegen/Cargo.lock rename to examples/components/balls/smatmul/verify/casegen/Cargo.lock diff --git a/examples/balls/smatmul/verify/casegen/Cargo.toml b/examples/components/balls/smatmul/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/smatmul/verify/casegen/Cargo.toml rename to examples/components/balls/smatmul/verify/casegen/Cargo.toml diff --git a/examples/components/balls/smatmul/verify/casegen/src/casegen.rs b/examples/components/balls/smatmul/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..9d8891b8c --- /dev/null +++ b/examples/components/balls/smatmul/verify/casegen/src/casegen.rs @@ -0,0 +1,133 @@ +use crate::model; + +pub const BIAS_BANK: u32 = 0; +pub const C_BANK: u32 = 5; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MatrixCmd { + pub kind: u32, + pub bid: u32, + pub rob_id: u32, + pub op1_words: u32, + pub op2_words: u32, + pub rs1_lo: u32, + pub rs1_hi: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MatrixCase { + pub commands: Vec, + pub bias: Vec, + pub a: Vec>, + pub b: Vec>, + pub writes: Vec, +} + +pub fn gen_case(seed: u32, index: u32, bid: u32, tile_rows: usize) -> MatrixCase { + match index { + 0 => build_case(seed, bid, 16, 1, tile_rows), + 1 => build_case(seed, bid, 16, 2, tile_rows), + 2 => build_case(seed ^ index, bid, 16, 1, tile_rows), + _ => { + let mut rng = Rng::new(seed, index); + let blocks = 1 + (rng.next() % 2) as usize; + build_case(rng.next(), bid, 16, blocks, tile_rows) + } + } +} + +fn build_case(seed: u32, bid: u32, rows: usize, blocks: usize, tile_rows: usize) -> MatrixCase { + assert!(matches!(rows, 16 | 32 | 64)); + assert!(matches!(blocks, 1 | 2)); + assert!(rows * model::RESULT_WORDS <= model::BANK_DEPTH); + let mut rng = Rng::new(seed, rows as u32 ^ blocks as u32); + let bias_values: Vec = (0..model::CORE).map(|_| rng.value() as i32).collect(); + let mut bias = Vec::with_capacity(4 * model::ROW_BYTES); + for chunk in bias_values.chunks(4) { + for value in chunk { + bias.extend_from_slice(&value.to_le_bytes()); + } + } + + let mut a = Vec::with_capacity(blocks); + let mut b = Vec::with_capacity(blocks); + let mut result = vec![bias_values.clone(); rows]; + let mut commands = Vec::with_capacity(blocks + 1); + commands.push(MatrixCmd { + kind: 0, + bid, + rob_id: 0, + op1_words: 4, + rs1_lo: model::encode_rs1(BIAS_BANK, 0, 0, 4) as u32, + rs1_hi: (model::encode_rs1(BIAS_BANK, 0, 0, 4) >> 32) as u32, + ..MatrixCmd::default() + }); + + for block in 0..blocks { + let a_bank = 1 + 2 * block as u32; + let b_bank = 2 + 2 * block as u32; + let a_values: Vec = (0..rows * model::CORE).map(|_| rng.value()).collect(); + let b_values: Vec = (0..model::CORE * model::CORE) + .map(|_| rng.value()) + .collect(); + let a_packed = model::pack_a(&a_values, rows); + let b_packed = model::pack_b(&b_values); + a.push(a_packed); + b.push(b_packed); + for row in 0..rows { + for column in 0..model::CORE { + for k in 0..model::CORE { + result[row][column] = result[row][column].wrapping_add( + a_values[row * model::CORE + k] as i32 + * b_values[k * model::CORE + column] as i32, + ); + } + } + } + let rs1 = model::encode_rs1(a_bank, b_bank, C_BANK, model::CORE as u64); + let rs2 = model::encode_rs2(rows, block == 0, block + 1 == blocks); + commands.push(MatrixCmd { + kind: 1, + bid, + rob_id: (block + 1) as u32, + op1_words: rows as u32, + op2_words: (model::CORE * rows.div_ceil(tile_rows)) as u32, + rs1_lo: rs1 as u32, + rs1_hi: (rs1 >> 32) as u32, + rs2_lo: rs2 as u32, + rs2_hi: (rs2 >> 32) as u32, + }); + } + + MatrixCase { + commands, + bias, + a, + b, + writes: model::emit_writes(&result), + } +} + +struct Rng { + state: u64, +} + +impl Rng { + fn new(seed: u32, salt: u32) -> Self { + Self { + state: u64::from(seed) << 32 | u64::from(salt), + } + } + fn next(&mut self) -> u32 { + self.state ^= self.state << 13; + self.state ^= self.state >> 7; + self.state ^= self.state << 17; + self.state as u32 + } + fn value(&mut self) -> i8 { + (self.next() % 17) as i8 - 8 + } +} diff --git a/examples/balls/smatmul/verify/casegen/src/lib.rs b/examples/components/balls/smatmul/verify/casegen/src/lib.rs similarity index 100% rename from examples/balls/smatmul/verify/casegen/src/lib.rs rename to examples/components/balls/smatmul/verify/casegen/src/lib.rs diff --git a/examples/components/balls/smatmul/verify/casegen/src/model.rs b/examples/components/balls/smatmul/verify/casegen/src/model.rs new file mode 100644 index 000000000..1e5557110 --- /dev/null +++ b/examples/components/balls/smatmul/verify/casegen/src/model.rs @@ -0,0 +1,58 @@ +pub const CORE: usize = 16; +pub const ROW_BYTES: usize = 16; +pub const RESULT_WORDS: usize = 4; +pub const BANK_DEPTH: usize = 64; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WriteExp { + pub addr: u32, + pub data: [u8; ROW_BYTES], +} + +pub fn pack_a(src: &[i8], rows: usize) -> Vec { + assert_eq!(src.len(), rows * CORE); + src.iter().map(|value| *value as u8).collect() +} + +pub fn pack_b(src: &[i8]) -> Vec { + assert_eq!(src.len(), CORE * CORE); + src.iter().map(|value| *value as u8).collect() +} + +pub fn emit_writes(c: &[Vec]) -> Vec { + let mut writes = Vec::with_capacity(c.len() * RESULT_WORDS); + for (row, values) in c.iter().enumerate() { + assert_eq!(values.len(), CORE); + for word in 0..RESULT_WORDS { + let mut data = [0; ROW_BYTES]; + for lane in 0..4 { + let value = values[word * 4 + lane]; + data[lane * 4..lane * 4 + 4].copy_from_slice(&value.to_le_bytes()); + } + writes.push(WriteExp { + addr: (row * RESULT_WORDS + word) as u32, + data, + }); + } + } + writes +} + +pub fn word(data: &[u8], index: usize) -> [u8; ROW_BYTES] { + data[index * ROW_BYTES..(index + 1) * ROW_BYTES] + .try_into() + .unwrap() +} + +pub fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u64) -> u64 { + assert!(op1 < 1024 && op2 < 1024 && wr < 1024 && iter < (1_u64 << 34)); + u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (iter << 30) +} + +pub fn encode_rs2(rows: usize, first: bool, last: bool) -> u64 { + assert!(matches!(rows, 16 | 32 | 64)); + rows as u64 + | ((CORE as u64) << 12) + | ((first as u64) << 24) + | ((last as u64) << 25) +} diff --git a/examples/balls/smatmul/verify/filelists/smatmul_ball.f b/examples/components/balls/smatmul/verify/filelists/smatmul_ball.f similarity index 100% rename from examples/balls/smatmul/verify/filelists/smatmul_ball.f rename to examples/components/balls/smatmul/verify/filelists/smatmul_ball.f diff --git a/examples/balls/smatmul/verify/parameters.toml b/examples/components/balls/smatmul/verify/parameters.toml similarity index 100% rename from examples/balls/smatmul/verify/parameters.toml rename to examples/components/balls/smatmul/verify/parameters.toml diff --git a/examples/balls/smatmul/verify/src/common/smatmul_defs.svh b/examples/components/balls/smatmul/verify/src/common/smatmul_defs.svh similarity index 89% rename from examples/balls/smatmul/verify/src/common/smatmul_defs.svh rename to examples/components/balls/smatmul/verify/src/common/smatmul_defs.svh index 4617c8277..3b7badf38 100644 --- a/examples/balls/smatmul/verify/src/common/smatmul_defs.svh +++ b/examples/components/balls/smatmul/verify/src/common/smatmul_defs.svh @@ -40,14 +40,14 @@ import "DPI-C" function longint unsigned smatmul_case_write_data_lo(input int un import "DPI-C" function longint unsigned smatmul_case_write_data_hi(input int unsigned index); `ifndef SMATMUL_OS_FUNCT7 -`error "SMATMUL_OS_FUNCT7 must be provided by the selected Core ballISA" +`error "SMATMUL_OS_FUNCT7 must be provided by the selected Thread ballISA" `endif `ifndef SMATMUL_BIAS_FUNCT7 -`error "SMATMUL_BIAS_FUNCT7 must be provided by the selected Core ballISA" +`error "SMATMUL_BIAS_FUNCT7 must be provided by the selected Thread ballISA" `endif -localparam int SMATMUL_OS_CORE_FUNCT7 = `SMATMUL_OS_FUNCT7; -localparam int SMATMUL_BIAS_CORE_FUNCT7 = `SMATMUL_BIAS_FUNCT7; +localparam int SMATMUL_OS_THREAD_FUNCT7 = `SMATMUL_OS_FUNCT7; +localparam int SMATMUL_BIAS_THREAD_FUNCT7 = `SMATMUL_BIAS_FUNCT7; localparam int MATRIX_TIMEOUT_CYCLES = 200000; localparam int MATRIX_SEED = 32'hCAFE_BABE; diff --git a/examples/balls/smatmul/verify/src/common/smatmul_items.svh b/examples/components/balls/smatmul/verify/src/common/smatmul_items.svh similarity index 95% rename from examples/balls/smatmul/verify/src/common/smatmul_items.svh rename to examples/components/balls/smatmul/verify/src/common/smatmul_items.svh index 6dc1f82f3..6d46030e1 100644 --- a/examples/balls/smatmul/verify/src/common/smatmul_items.svh +++ b/examples/components/balls/smatmul/verify/src/common/smatmul_items.svh @@ -37,7 +37,7 @@ class smatmul_cmd_item extends bb_blink_cmd_item; sub_rob_id = 8'd0; if (kind == 0) begin - funct7 = SMATMUL_BIAS_CORE_FUNCT7[6:0]; + funct7 = SMATMUL_BIAS_THREAD_FUNCT7[6:0]; op1_en = 1'b1; op2_en = 1'b0; wr_spad_en = 1'b0; @@ -49,7 +49,7 @@ class smatmul_cmd_item extends bb_blink_cmd_item; if (rs1[29:10] != 0 || iter != 4 || rs2 != 0 || op1_words != 4) `uvm_fatal("CASE", "invalid bias command") end else begin - funct7 = SMATMUL_OS_CORE_FUNCT7[6:0]; + funct7 = SMATMUL_OS_THREAD_FUNCT7[6:0]; op1_en = 1'b1; op2_en = 1'b1; wr_spad_en = 1'b1; diff --git a/examples/balls/smatmul/verify/src/cov/smatmul_cov.svh b/examples/components/balls/smatmul/verify/src/cov/smatmul_cov.svh similarity index 89% rename from examples/balls/smatmul/verify/src/cov/smatmul_cov.svh rename to examples/components/balls/smatmul/verify/src/cov/smatmul_cov.svh index 59a4d05bc..b2e6887df 100644 --- a/examples/balls/smatmul/verify/src/cov/smatmul_cov.svh +++ b/examples/components/balls/smatmul/verify/src/cov/smatmul_cov.svh @@ -18,13 +18,14 @@ class smatmul_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - is_bias = (item.funct7 == SMATMUL_BIAS_CORE_FUNCT7[6:0]); + is_bias = (item.funct7 == SMATMUL_BIAS_THREAD_FUNCT7[6:0]); is_first = is_bias ? 0 : item.rs2[24]; cmd_count++; cmd_cg.sample(); endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") if (cmd_cg.get_coverage() != 100.0) diff --git a/examples/balls/smatmul/verify/src/env/smatmul_env.svh b/examples/components/balls/smatmul/verify/src/env/smatmul_env.svh similarity index 100% rename from examples/balls/smatmul/verify/src/env/smatmul_env.svh rename to examples/components/balls/smatmul/verify/src/env/smatmul_env.svh diff --git a/examples/balls/smatmul/verify/src/env/smatmul_scoreboard.svh b/examples/components/balls/smatmul/verify/src/env/smatmul_scoreboard.svh similarity index 99% rename from examples/balls/smatmul/verify/src/env/smatmul_scoreboard.svh rename to examples/components/balls/smatmul/verify/src/env/smatmul_scoreboard.svh index 8ee7362c5..063423230 100644 --- a/examples/balls/smatmul/verify/src/env/smatmul_scoreboard.svh +++ b/examples/components/balls/smatmul/verify/src/env/smatmul_scoreboard.svh @@ -178,6 +178,7 @@ class smatmul_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) `uvm_fatal("SCB", $sformatf( diff --git a/examples/balls/smatmul/verify/src/pkg/smatmul_pkg.sv b/examples/components/balls/smatmul/verify/src/pkg/smatmul_pkg.sv similarity index 100% rename from examples/balls/smatmul/verify/src/pkg/smatmul_pkg.sv rename to examples/components/balls/smatmul/verify/src/pkg/smatmul_pkg.sv diff --git a/examples/balls/smatmul/verify/src/seq/smatmul_sequences.svh b/examples/components/balls/smatmul/verify/src/seq/smatmul_sequences.svh similarity index 100% rename from examples/balls/smatmul/verify/src/seq/smatmul_sequences.svh rename to examples/components/balls/smatmul/verify/src/seq/smatmul_sequences.svh diff --git a/examples/components/balls/smatmul/verify/src/tb_top.sv b/examples/components/balls/smatmul/verify/src/tb_top.sv new file mode 100644 index 000000000..d29dae93e --- /dev/null +++ b/examples/components/balls/smatmul/verify/src/tb_top.sv @@ -0,0 +1,101 @@ +module tb_top; + import uvm_pkg::*; + import smatmul_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + SMatMulBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .io_status_idle(), + .io_status_running(), + .io_bankAccessReady(1'b1), + .io_cmdReq_ready(intf.cmd_req_ready), + .io_cmdReq_valid(intf.cmd_req_valid), + .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), + .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), + .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), + .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), + .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), + .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), + .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), + .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), + .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), + .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), + .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), + .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), + .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), + .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), + .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), + .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), + .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), + .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), + .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), + .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), + .io_cmdResp_ready(intf.cmd_resp_ready), + .io_cmdResp_valid(intf.cmd_resp_valid), + .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), + .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), + .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .io_bankRead_0_ball_id(), + .io_bankRead_0_group_id(intf.bank_read_group_id[0]), + .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .io_bankRead_1_ball_id(), + .io_bankRead_1_group_id(intf.bank_read_group_id[1]), + .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .io_bankWrite_0_ball_id(), + .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .io_subRobReq_ready(intf.sub_rob_req_ready) + ); + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("smatmul_ball_test"); + end +endmodule diff --git a/examples/balls/smatmul/verify/src/tests/smatmul_test.svh b/examples/components/balls/smatmul/verify/src/tests/smatmul_test.svh similarity index 100% rename from examples/balls/smatmul/verify/src/tests/smatmul_test.svh rename to examples/components/balls/smatmul/verify/src/tests/smatmul_test.svh diff --git a/examples/components/balls/smatmul/workloads/ctests/CMakeLists.txt b/examples/components/balls/smatmul/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..3040cbea3 --- /dev/null +++ b/examples/components/balls/smatmul/workloads/ctests/CMakeLists.txt @@ -0,0 +1,14 @@ +add_buckyball_ctest(smatmul_bias_accumulate_test.c) +add_buckyball_ctest(smatmul_1x16x64_test.c) +add_buckyball_ctest(smatmul_8x16_test.c) +add_buckyball_ctest(smatmul_16x16_test.c) +add_buckyball_ctest(smatmul_16x16x32_test.c) +add_buckyball_ctest(smatmul_nondefault_banks_test.c) +file( + STRINGS "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/ballISA.h" + F32_ISA + REGEX "^#define BB_FUNC7_SMATMUL_F32 " +) +if(F32_ISA) + add_buckyball_ctest(smatmul_f32_test.c) +endif() diff --git a/examples/balls/smatmul/workloads/ctests/smatmul_16x16_test.c b/examples/components/balls/smatmul/workloads/ctests/smatmul_16x16_test.c similarity index 100% rename from examples/balls/smatmul/workloads/ctests/smatmul_16x16_test.c rename to examples/components/balls/smatmul/workloads/ctests/smatmul_16x16_test.c diff --git a/examples/balls/smatmul/workloads/ctests/smatmul_16x16x32_test.c b/examples/components/balls/smatmul/workloads/ctests/smatmul_16x16x32_test.c similarity index 100% rename from examples/balls/smatmul/workloads/ctests/smatmul_16x16x32_test.c rename to examples/components/balls/smatmul/workloads/ctests/smatmul_16x16x32_test.c diff --git a/examples/balls/smatmul/workloads/ctests/smatmul_1x16x64_test.c b/examples/components/balls/smatmul/workloads/ctests/smatmul_1x16x64_test.c similarity index 100% rename from examples/balls/smatmul/workloads/ctests/smatmul_1x16x64_test.c rename to examples/components/balls/smatmul/workloads/ctests/smatmul_1x16x64_test.c diff --git a/examples/balls/smatmul/workloads/ctests/smatmul_8x16_test.c b/examples/components/balls/smatmul/workloads/ctests/smatmul_8x16_test.c similarity index 100% rename from examples/balls/smatmul/workloads/ctests/smatmul_8x16_test.c rename to examples/components/balls/smatmul/workloads/ctests/smatmul_8x16_test.c diff --git a/examples/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c b/examples/components/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c similarity index 94% rename from examples/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c rename to examples/components/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c index de17c59bc..53dfec7d4 100644 --- a/examples/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c +++ b/examples/components/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c @@ -30,6 +30,8 @@ int main(void) { bb_mvin((uintptr_t)weight0, 2, 16, 1); bb_mvin((uintptr_t)input1, 3, 16, 1); bb_mvin((uintptr_t)weight1, 4, 16, 1); + // An unfinished block defines a new destination version without bank writes. + bb_mvin((uintptr_t)bias, 5, 4, 1); bb_smatmul_bias(0, 0); bb_smatmul_os(1, 2, 5, 16, 16, 16, 1, 0, 0); bb_smatmul_os(3, 4, 5, 16, 16, 16, 0, 1, 0); diff --git a/examples/balls/smatmul/workloads/ctests/smatmul_f32_test.c b/examples/components/balls/smatmul/workloads/ctests/smatmul_f32_test.c similarity index 100% rename from examples/balls/smatmul/workloads/ctests/smatmul_f32_test.c rename to examples/components/balls/smatmul/workloads/ctests/smatmul_f32_test.c diff --git a/examples/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c b/examples/components/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c similarity index 100% rename from examples/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c rename to examples/components/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c diff --git a/examples/balls/smatmul/workloads/isa/smatmul.h b/examples/components/balls/smatmul/workloads/isa/smatmul.h similarity index 100% rename from examples/balls/smatmul/workloads/isa/smatmul.h rename to examples/components/balls/smatmul/workloads/isa/smatmul.h diff --git a/examples/balls/smatmul/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/smatmul/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/smatmul/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/smatmul/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/smatmul/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/smatmul/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/smatmul/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/smatmul/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir b/examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir new file mode 100644 index 000000000..bbdf49461 --- /dev/null +++ b/examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir @@ -0,0 +1,70 @@ +func.func private @check_result(memref<64x4xi32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %eight = arith.constant 8 : index + %sixteen = arith.constant 16 : index + + %input0 = memref.alloc() alignment = 64 : memref<16x16xi8> + %input1 = memref.alloc() alignment = 64 : memref<16x16xi8> + %weight0 = memref.alloc() alignment = 64 : memref<16x16xi8> + %weight1 = memref.alloc() alignment = 64 : memref<16x16xi8> + %bias = memref.alloc() alignment = 64 : memref<4x4xi32> + %output = memref.alloc() alignment = 64 : memref<64x4xi32> + linalg.fill ins(%zero_i32 : i32) outs(%output : memref<64x4xi32>) + + scf.for %row = %zero to %sixteen step %one { + scf.for %column = %zero to %sixteen step %one { + %row_i32 = arith.index_cast %row : index to i32 + %column_i32 = arith.index_cast %column : index to i32 + %sum = arith.addi %row_i32, %column_i32 : i32 + %difference = arith.subi %row_i32, %column_i32 : i32 + %twice_row = arith.addi %row_i32, %row_i32 : i32 + %second = arith.subi %twice_row, %column_i32 : i32 + %sum_i8 = arith.trunci %sum : i32 to i8 + %difference_i8 = arith.trunci %difference : i32 to i8 + %second_i8 = arith.trunci %second : i32 to i8 + memref.store %difference_i8, %input0[%row, %column] : memref<16x16xi8> + memref.store %sum_i8, %weight0[%row, %column] : memref<16x16xi8> + memref.store %sum_i8, %input1[%row, %column] : memref<16x16xi8> + memref.store %second_i8, %weight1[%row, %column] : memref<16x16xi8> + } + %bias_value = arith.subi %row, %eight : index + %bias_i32 = arith.index_cast %bias_value : index to i32 + %bias_row = arith.divui %row, %four : index + %bias_lane = arith.remui %row, %four : index + memref.store %bias_i32, %bias[%bias_row, %bias_lane] : memref<4x4xi32> + } + + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 4 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input0 <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %weight0 <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %input1 <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %weight1 <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + // The bias occupies the first four rows of C. + buckyball.mvin %bias <{bank = 4 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> + buckyball.custom "matmul_i8_block" <{sources = array, destination = 4 : i64, read_destination}> {rows = 16 : i64, k = 16 : i64, first = true, last = false} + buckyball.custom "matmul_i8_block" <{sources = array, destination = 4 : i64, read_destination}> {rows = 16 : i64, k = 16 : i64, first = false, last = true} + buckyball.mvout %output <{bank = 4 : i64, depth = 64 : i64, stride = 1 : i64}> : memref<64x4xi32> + func.call @check_result(%output) : (memref<64x4xi32>) -> () + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 4 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %input0 : memref<16x16xi8> + memref.dealloc %input1 : memref<16x16xi8> + memref.dealloc %weight0 : memref<16x16xi8> + memref.dealloc %weight1 : memref<16x16xi8> + memref.dealloc %bias : memref<4x4xi32> + memref.dealloc %output : memref<64x4xi32> + return %zero_i8 : i8 +} diff --git a/examples/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp b/examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp similarity index 100% rename from examples/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp rename to examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp diff --git a/examples/components/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..67abe72b7 --- /dev/null +++ b/examples/components/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,19 @@ +if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +endif() +set(BUCKYBALL_MLIR_TEST_PREFIX bank) + +set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") +file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") +file(STRINGS "${_params}" _bank_width REGEX "^#define BANK_WIDTH ") +string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") +string(REGEX REPLACE ".* " "" _width "${_bank_width}") +math(EXPR _line_remainder "${_lines} % 64") +if(NOT (_width EQUAL 128) OR _lines LESS 64 OR + NOT (_line_remainder EQUAL 0)) + message(FATAL_ERROR + "SMatMul BankOp test requires BANK_WIDTH=128 and BANK_LINES a multiple " + "of 64 (got width=${_width}, lines=${_lines})") +endif() +math(EXPR BB_SMM_M "${_lines} / 4") +add_buckyball_mlir_test(smatmul_bias_two_terms TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/components/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in b/examples/components/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in new file mode 100644 index 000000000..2876766dd --- /dev/null +++ b/examples/components/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in @@ -0,0 +1,69 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x4xi32>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %eight = arith.constant 8 : index + %sixteen = arith.constant 16 : index + %m = arith.constant @BB_SMM_M@ : index + %a = memref.alloc() : memref<@BB_SMM_M@x16xi8> + %w0 = memref.alloc() : memref<16x16xi8> + %w1 = memref.alloc() : memref<16x16xi8> + %bias = memref.alloc() : memref<4x4xi32> + %output = memref.alloc() : memref<@BB_BANK_LINES@x4xi32> + linalg.fill ins(%zero_i32 : i32) outs(%output : memref<@BB_BANK_LINES@x4xi32>) + scf.for %row = %zero to %m step %one { + %slot = arith.remui %row, %sixteen : index + scf.for %col = %zero to %sixteen step %one { + %v = arith.subi %slot, %col : index + %v8 = arith.index_cast %v : index to i8 + memref.store %v8, %a[%row, %col] : memref<@BB_SMM_M@x16xi8> + } + } + scf.for %row = %zero to %sixteen step %one { + %bias_value = arith.subi %row, %eight : index + %bias_i32 = arith.index_cast %bias_value : index to i32 + %bias_row = arith.divui %row, %four : index + %bias_lane = arith.remui %row, %four : index + memref.store %bias_i32, %bias[%bias_row, %bias_lane] : memref<4x4xi32> + scf.for %col = %zero to %sixteen step %one { + %r32 = arith.index_cast %row : index to i32 + %c32 = arith.index_cast %col : index to i32 + %plus32 = arith.addi %r32, %c32 : i32 + %twice32 = arith.addi %r32, %r32 : i32 + %minus32 = arith.subi %twice32, %c32 : i32 + %plus = arith.trunci %plus32 : i32 to i8 + %minus = arith.trunci %minus32 : i32 to i8 + memref.store %plus, %w0[%row, %col] : memref<16x16xi8> + memref.store %minus, %w1[%row, %col] : memref<16x16xi8> + } + } + %a0_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b0_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %a1_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b1_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %c_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %a0 = buckyball.bank_mvin %a, %a0_bank <{depth = @BB_SMM_M@ : i64, stride = 1 : i64}> : memref<@BB_SMM_M@x16xi8> + %b0 = buckyball.bank_mvin %w0, %b0_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + %a1 = buckyball.bank_mvin %a, %a1_bank <{depth = @BB_SMM_M@ : i64, stride = 1 : i64}> : memref<@BB_SMM_M@x16xi8> + %b1 = buckyball.bank_mvin %w1, %b1_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + // The bias occupies the first four rows of C. + %c = buckyball.bank_mvin %bias, %c_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> + %partial = buckyball.bank_custom "matmul_i8_block" (%a0, %b0) -> %c <{read_destination}> {rows = @BB_SMM_M@ : i64, k = 16 : i64, first = true, last = false} + %result = buckyball.bank_custom "matmul_i8_block" (%a1, %b1) -> %partial <{read_destination}> {rows = @BB_SMM_M@ : i64, k = 16 : i64, first = false, last = true} + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x4xi32> + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x4xi32>) -> () + buckyball.bank_release %a0 + buckyball.bank_release %b0 + buckyball.bank_release %a1 + buckyball.bank_release %b1 + buckyball.bank_release %stored + memref.dealloc %a : memref<@BB_SMM_M@x16xi8> + memref.dealloc %w0 : memref<16x16xi8> + memref.dealloc %w1 : memref<16x16xi8> + memref.dealloc %bias : memref<4x4xi32> + memref.dealloc %output : memref<@BB_BANK_LINES@x4xi32> + return %zero_i8 : i8 +} diff --git a/examples/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp b/examples/components/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp similarity index 100% rename from examples/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp rename to examples/components/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp diff --git a/examples/components/balls/toint8/arch/ToInt8Ball.scala b/examples/components/balls/toint8/arch/ToInt8Ball.scala new file mode 100644 index 000000000..0e570d6f5 --- /dev/null +++ b/examples/components/balls/toint8/arch/ToInt8Ball.scala @@ -0,0 +1,350 @@ +package examples.balls.toint8 + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.toint8.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import hardfloat.{recFNFromFN, INToRecFN, RecFNToIN} +import hardfloat.consts.{round_near_even, tininess_afterRounding} +import framework.lib.arith.Fma + +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} + +@instantiable +class ToInt8Ball( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + val f32Funct = ball.instructions + .find(_.mnemonic == "QUANT_F32_TO_I8").get.funct7 + + val i32Funct = ball.instructions + .find(_.mnemonic == "QUANT_I32_TO_I8").get.funct7 + + require(mp.bankWidth == 128, "ToInt8Ball requires 128-bit bank rows") + require( + mp.bankEntries >= 4, + "ToInt8Ball requires at least four bank rows" + ) + require(ball.inBW == 1, "ToInt8Ball requires inBW=1") + require(ball.outBW == 1, "ToInt8Ball requires outBW=1") + require((f32Funct >> 4) == 3, "QUANT_F32_TO_I8 must encode one read and one write") + require((i32Funct >> 4) == 4, "QUANT_I32_TO_I8 must encode two reads and one write") + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val idle :: scaleReq :: scaleResp :: inputReq :: inputResp :: pack :: packDrain :: writeReq :: writeResp :: complete :: Nil = + Enum(10) + val state = RegInit(idle) + + val robIdReg = RegInit(0.U(log2Up(fp.rob_entries).W)) + val inputBank = RegInit(0.U(fp.bank_id_len.W)) + val scaleBank = RegInit(0.U(fp.bank_id_len.W)) + val outputBank = RegInit(0.U(fp.bank_id_len.W)) + val bases = Reg(Vec(3, UInt(mp.groupIdWidth.W))) + val iterReg = RegInit(0.U(fp.iter_len.W)) + val inputBaseReg = RegInit(0.U(log2Ceil(mp.bankEntries).W)) + // Relative row within the command. inputBaseReg is the absolute bank row. + val relativeRow = RegInit(0.U(log2Ceil(mp.bankEntries).W)) + val outputRow = RegInit(0.U(log2Ceil(mp.bankEntries).W)) + val outputWidth = RegInit(0.U(7.W)) + val outputStride = RegInit(0.U(7.W)) + val outputColumn = RegInit(0.U(7.W)) + val scaleRow = RegInit(0.U(2.W)) + val i32Mode = RegInit(false.B) + val reluReg = RegInit(false.B) + val tensorScale = RegInit(0.U(32.W)) + val scales = Reg(Vec(16, UInt(32.W))) + val inputData = Reg(UInt(128.W)) + val outputData = RegInit(0.U(128.W)) + val writeData = Reg(UInt(128.W)) + val lane = RegInit(0.U(2.W)) + val outputBytes = Reg(Vec(4, UInt(8.W))) + + def positiveFinite(value: UInt): Bool = + !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U + + val intToFloat = Instantiate(new INToRecFN(32, 8, 24)) + + // The unpipelined recoded-float multiplier was the critical Buckyball path + // (about 8.5 ns before clock uncertainty). FMA mode with c=0 preserves the + // multiplication result while splitting the datapath into three stages. + val multiply = Instantiate(new Fma(3, 8, 24)) + + val floatToInt = Instantiate(new RecFNToIN(8, 24, 8)) + val input = (inputData >> (lane << 5))(31, 0) + val scale = Mux(i32Mode, scales(Cat(relativeRow(1, 0), lane)), tensorScale) + val packIssue = state === pack + + val inputStage0 = RegEnable(input, packIssue) + val scaleStage0 = RegEnable(scale, packIssue) + val laneStage0 = RegEnable(lane, packIssue) + val convertValid = RegNext(packIssue, false.B) + + val multiplyInput = RegEnable( + Mux(i32Mode, intToFloat.io.out, recFNFromFN(8, 24, inputStage0)), + convertValid + ) + + val multiplyScale = RegEnable(recFNFromFN(8, 24, scaleStage0), convertValid) + val laneStage1 = RegEnable(laneStage0, convertValid) + val multiplyValid = RegNext(convertValid, false.B) + + val multiplyResult = RegEnable(multiply.io.out, multiply.io.validout) + val multiplyLane = Pipe(multiplyValid, laneStage1, 3) + val laneStage2 = RegEnable(multiplyLane.bits, multiply.io.validout) + val toIntValid = RegNext(multiply.io.validout, false.B) + + val quantizedResult = RegEnable( + Mux(reluReg && floatToInt.io.out(7), 0.U, floatToInt.io.out), + toIntValid + ) + + val quantizedLane = RegEnable(laneStage2, toIntValid) + val quantizedValid = RegNext(toIntValid, false.B) + + intToFloat.io.signedIn := true.B + intToFloat.io.in := inputStage0 + intToFloat.io.roundingMode := round_near_even + intToFloat.io.detectTininess := tininess_afterRounding + + multiply.io.a := multiplyInput + multiply.io.b := multiplyScale + multiply.io.c := recFNFromFN(8, 24, 0.U) + multiply.io.op := 0.U + multiply.io.validin := multiplyValid + multiply.io.roundingMode := round_near_even + multiply.io.detectTininess := tininess_afterRounding.asUInt + + floatToInt.io.in := multiplyResult + floatToInt.io.roundingMode := round_near_even + floatToInt.io.signedOut := true.B + + blink.cmdReq.ready := state === idle + blink.cmdResp.valid := state === complete + blink.cmdResp.bits.rob_id := robIdReg + blink.status.idle := state === idle + blink.status.running := state =/= idle && state =/= complete + + blink.bankRead(0).rob_id := robIdReg + blink.bankRead(0).bank_id := Mux(state === scaleReq || state === scaleResp, scaleBank, inputBank) + blink.bankRead(0).group_id := Mux(state === scaleReq || state === scaleResp, bases(1), bases(0)) + blink.bankRead(0).io.req.valid := false.B + blink.bankRead(0).io.req.bits.addr := 0.U + blink.bankRead(0).io.resp.ready := false.B + + blink.bankWrite(0).rob_id := robIdReg + blink.bankWrite(0).bank_id := outputBank + blink.bankWrite(0).group_id := bases(2) + blink.bankWrite(0).io.req.valid := false.B + blink.bankWrite(0).io.req.bits.addr := outputRow + blink.bankWrite(0).io.req.bits.data := writeData + blink.bankWrite(0).io.req.bits.mask := VecInit( + Seq.fill(mp.bankMaskLen)(true.B) + ) + blink.bankWrite(0).io.resp.ready := false.B + + switch(state) { + is(idle) { + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + val isF32 = cmd.funct === f32Funct.U + val isI32 = cmd.funct === i32Funct.U + + assert(isF32 || isI32, "ToInt8Ball received an unknown funct7") + assert(cmd.iter > 0.U && cmd.iter(1, 0) === 0.U, "ToInt8Ball iter must be a positive multiple of four") + assert( + cmd.iter <= mp.bankEntries.U, + "ToInt8Ball input exceeds bank depth" + ) + assert(cmd.op1_col === 1.U && cmd.wr_col === 1.U, "ToInt8Ball input and output must each occupy one bank") + assert( + cmd.op1_bank =/= cmd.wr_bank, + "ToInt8Ball input and output banks must differ" + ) + when(isF32) { + assert(cmd.rs2(63, 32) === 0.U, "QUANT_F32_TO_I8 reserves rs2[63:32]") + assert(positiveFinite(cmd.rs2(31, 0)), "QUANT_F32_TO_I8 scale must be finite and positive") + }.otherwise { + assert(cmd.op2_col === 1.U, "QUANT_I32_TO_I8 scale must occupy one bank") + assert( + cmd.op1_bank =/= cmd.op2_bank, + "QUANT_I32_TO_I8 input and scale banks must differ" + ) + assert( + cmd.op2_bank =/= cmd.wr_bank, + "QUANT_I32_TO_I8 scale and output banks must differ" + ) + val outputBase = cmd.rs2(7, 1) + val width = cmd.rs2(14, 8) + val height = cmd.rs2(21, 15) + val stride = cmd.rs2(28, 22) + val outputRows = cmd.iter >> 2 + val outputEnd = outputBase +& ((height - 1.U) * stride) +& width + val inputBase = cmd.rs2(34, 29) + assert(cmd.rs2(63, 35) === 0.U, "QUANT_I32_TO_I8 reserves rs2[63:35]") + assert(width > 0.U && height > 0.U && stride >= width, "QUANT_I32_TO_I8 output geometry is invalid") + assert(width * height === outputRows, "QUANT_I32_TO_I8 iter does not match output geometry") + assert( + outputEnd <= mp.bankEntries.U, + "QUANT_I32_TO_I8 output exceeds bank depth" + ) + assert( + inputBase +& cmd.iter <= mp.bankEntries.U, + "QUANT_I32_TO_I8 input exceeds bank depth" + ) + } + + robIdReg := blink.cmdReq.bits.rob_id + inputBank := cmd.op1_bank + scaleBank := cmd.op2_bank + outputBank := cmd.wr_bank + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + iterReg := cmd.iter + inputBaseReg := Mux(isI32, cmd.rs2(34, 29), 0.U) + relativeRow := 0.U + outputRow := Mux(isI32, cmd.rs2(7, 1), 0.U) + outputWidth := Mux(isI32, cmd.rs2(14, 8), 0.U) + outputStride := Mux(isI32, cmd.rs2(28, 22), 0.U) + outputColumn := 0.U + scaleRow := 0.U + i32Mode := isI32 + reluReg := isI32 && cmd.rs2(0) + tensorScale := cmd.rs2(31, 0) + outputData := 0.U + lane := 0.U + state := Mux(isI32, scaleReq, inputReq) + } + } + + is(scaleReq) { + blink.bankRead(0).io.req.valid := true.B + blink.bankRead(0).io.req.bits.addr := scaleRow + when(blink.bankRead(0).io.req.fire) { + state := scaleResp + } + } + + is(scaleResp) { + blink.bankRead(0).io.resp.ready := true.B + when(blink.bankRead(0).io.resp.fire) { + for (lane <- 0 until 4) { + val value = blink.bankRead(0).io.resp.bits.data(32 * lane + 31, 32 * lane) + assert(positiveFinite(value), "QUANT_I32_TO_I8 scales must be finite and positive") + scales(Cat(scaleRow, lane.U(2.W))) := value + } + when(scaleRow === 3.U) { + state := inputReq + }.otherwise { + scaleRow := scaleRow + 1.U + state := scaleReq + } + } + } + + is(inputReq) { + blink.bankRead(0).io.req.valid := true.B + blink.bankRead(0).io.req.bits.addr := inputBaseReg +& relativeRow + when(blink.bankRead(0).io.req.fire) { + state := inputResp + } + } + + is(inputResp) { + blink.bankRead(0).io.resp.ready := true.B + when(blink.bankRead(0).io.resp.fire) { + inputData := blink.bankRead(0).io.resp.bits.data + lane := 0.U + state := pack + } + } + + is(pack) { + when(!i32Mode) { + assert(input(30, 23) =/= 255.U, "QUANT_F32_TO_I8 input must be finite") + } + when(lane === 3.U) { + state := packDrain + }.otherwise { + lane := lane + 1.U + } + } + + is(packDrain) {} + + is(writeReq) { + blink.bankWrite(0).io.req.valid := true.B + when(blink.bankWrite(0).io.req.fire) { + state := writeResp + } + } + + is(writeResp) { + blink.bankWrite(0).io.resp.ready := true.B + when(blink.bankWrite(0).io.resp.fire) { + when(relativeRow === iterReg - 1.U) { + state := complete + }.otherwise { + relativeRow := relativeRow + 1.U + when(outputColumn === outputWidth - 1.U) { + outputRow := outputRow + outputStride - outputWidth + 1.U + outputColumn := 0.U + }.otherwise { + outputRow := outputRow + 1.U + outputColumn := outputColumn + 1.U + } + state := inputReq + } + } + } + + is(complete) { + when(blink.cmdResp.fire) { + state := idle + } + } + } + + when((state === pack || state === packDrain) && quantizedValid) { + outputBytes(quantizedLane) := quantizedResult + when(quantizedLane === 3.U) { + val bytes = Cat(quantizedResult, outputBytes(2), outputBytes(1), outputBytes(0)) + val nextWord = outputData | (bytes << (relativeRow(1, 0) << 5)) + when(relativeRow(1, 0) === 3.U) { + writeData := nextWord + outputData := 0.U + state := writeReq + }.otherwise { + outputData := nextWord + relativeRow := relativeRow + 1.U + state := inputReq + } + } + } +} + +object ToInt8Ball extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new ToInt8Ball(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/components/balls/toint8/configs/default.toml b/examples/components/balls/toint8/configs/default.toml new file mode 100644 index 000000000..8166ca7f3 --- /dev/null +++ b/examples/components/balls/toint8/configs/default.toml @@ -0,0 +1 @@ +# ToInt8Ball has no private configuration parameters. diff --git a/examples/components/balls/toint8/configs/toint8_params.proto b/examples/components/balls/toint8/configs/toint8_params.proto new file mode 100644 index 000000000..5287e6689 --- /dev/null +++ b/examples/components/balls/toint8/configs/toint8_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.toint8; +option java_package = "buckyball.balls.toint8"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/components/balls/toint8/emu/src/dispatch.rs b/examples/components/balls/toint8/emu/src/dispatch.rs new file mode 100644 index 000000000..d0d2f3b57 --- /dev/null +++ b/examples/components/balls/toint8/emu/src/dispatch.rs @@ -0,0 +1,33 @@ +use bebop_bemu::ball::ExecContext; + +use super::quant; + +const BALL_CLASS: &str = "examples.balls.toint8.ToInt8Ball"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("QUANT_F32_TO_I8") => Some(quant::exec_f32_to_i8(xs1, xs2, ctx)), + Some("QUANT_I32_TO_I8") => Some(quant::exec_i32_to_i8(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("QUANT_F32_TO_I8") => Some(quant::f32_to_i8_latency(xs1, xs2)), + Some("QUANT_I32_TO_I8") => Some(quant::i32_to_i8_latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/components/balls/toint8/emu/src/lib.rs b/examples/components/balls/toint8/emu/src/lib.rs new file mode 100644 index 000000000..4de11c376 --- /dev/null +++ b/examples/components/balls/toint8/emu/src/lib.rs @@ -0,0 +1,13 @@ +pub(crate) use bebop_bemu::thread::instructions::decode; +pub(crate) use bebop_bemu::ball as instruction; + +mod model; +mod quant; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/balls/toint8/emu/src/model.rs b/examples/components/balls/toint8/emu/src/model.rs similarity index 100% rename from examples/balls/toint8/emu/src/model.rs rename to examples/components/balls/toint8/emu/src/model.rs diff --git a/examples/components/balls/toint8/emu/src/quant.rs b/examples/components/balls/toint8/emu/src/quant.rs new file mode 100644 index 000000000..787628299 --- /dev/null +++ b/examples/components/balls/toint8/emu/src/quant.rs @@ -0,0 +1,166 @@ +use bebop_bemu::thread::banks::bank_lines; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::instruction::ExecContext; + +pub(crate) fn exec_f32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + let src = rs1_b0(xs1); + let unused = rs1_b1(xs1); + let dst = rs1_b2(xs1); + let iter = rs1_iter(xs1) as usize; + if unused != 0 { + panic!("quant_f32_to_i8: input bank 1 must be zero"); + } + if src == dst { + panic!("quant_f32_to_i8: input and output banks must differ"); + } + if !ctx.config(src).allocated || !ctx.config(dst).allocated { + panic!("quant_f32_to_i8: bank not allocated"); + } + if ctx.config(src).cols != 1 || ctx.config(dst).cols != 1 { + panic!("quant_f32_to_i8: banks must have one column"); + } + if iter == 0 || iter % 4 != 0 || iter > bank_lines() { + panic!("quant_f32_to_i8: iter must be a positive multiple of four within bank depth"); + } + if xs2 >> 32 != 0 { + panic!("quant_f32_to_i8: rs2[63:32] must be zero"); + } + let scale = f32::from_bits(xs2 as u32); + if !scale.is_finite() || scale <= 0.0 { + panic!("quant_f32_to_i8: scale must be finite and positive"); + } + + let ps = pbank(ctx, src); + let pd = pbank(ctx, dst); + let (input, output) = ctx.banks.read_write(ps, pd); + for output_row in 0..iter / 4 { + let mut packed = [0u8; 16]; + for group in 0..4 { + let input_row = output_row * 4 + group; + let base = input_row * 16; + for lane in 0..4 { + let offset = base + lane * 4; + let bits = u32::from_le_bytes(input[offset..offset + 4].try_into().unwrap()); + packed[group * 4 + lane] = + super::model::quantize_f32(f32::from_bits(bits), scale, false) as u8; + } + } + let base = output_row * 16; + output[base..base + 16].copy_from_slice(&packed); + } + 0 +} + +pub(crate) fn exec_i32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + let src = rs1_b0(xs1); + let scale_bank = rs1_b1(xs1); + let dst = rs1_b2(xs1); + let iter = rs1_iter(xs1) as usize; + if src == scale_bank || src == dst || scale_bank == dst { + panic!("quant_i32_to_i8: input, scale, and output banks must differ"); + } + if !ctx.config(src).allocated || !ctx.config(scale_bank).allocated || !ctx.config(dst).allocated + { + panic!("quant_i32_to_i8: bank not allocated"); + } + if ctx.config(src).cols != 1 || ctx.config(scale_bank).cols != 1 || ctx.config(dst).cols != 1 { + panic!("quant_i32_to_i8: banks must have one column"); + } + if iter == 0 || iter % 4 != 0 || iter > bank_lines() { + panic!("quant_i32_to_i8: iter must be a positive multiple of four within bank depth"); + } + if xs2 >> 35 != 0 { + panic!("quant_i32_to_i8: rs2[63:35] must be zero"); + } + let input_base = ((xs2 >> 29) & 0x3f) as usize; + let output_base = ((xs2 >> 1) & 0x7f) as usize; + let output_width = ((xs2 >> 8) & 0x7f) as usize; + let output_height = ((xs2 >> 15) & 0x7f) as usize; + let output_stride = ((xs2 >> 22) & 0x7f) as usize; + let output_rows = iter / 4; + if output_width == 0 + || output_height == 0 + || output_stride < output_width + || output_width.checked_mul(output_height) != Some(output_rows) + || output_base + .checked_add((output_height - 1) * output_stride + output_width) + .is_none_or(|end| end > bank_lines()) + || input_base + iter > bank_lines() + { + panic!("quant_i32_to_i8: invalid output core"); + } + let relu = xs2 & 1 != 0; + + let pscale = pbank(ctx, scale_bank); + let mut scales = [0.0f32; 16]; + for group in 0..4 { + let base = group * 16; + for lane in 0..4 { + let offset = base + lane * 4; + scales[group * 4 + lane] = f32::from_bits(u32::from_le_bytes( + ctx.banks[pscale][offset..offset + 4].try_into().unwrap(), + )); + if !scales[group * 4 + lane].is_finite() || scales[group * 4 + lane] <= 0.0 { + panic!("quant_i32_to_i8: scales must be finite and positive"); + } + } + } + + let ps = pbank(ctx, src); + let pd = pbank(ctx, dst); + let (input, output) = ctx.banks.read_write(ps, pd); + for output_row in 0..iter / 4 { + let mut packed = [0u8; 16]; + for group in 0..4 { + let input_row = input_base + output_row * 4 + group; + let base = input_row * 16; + for lane in 0..4 { + let offset = base + lane * 4; + let value = i32::from_le_bytes(input[offset..offset + 4].try_into().unwrap()); + let channel = group * 4 + lane; + packed[channel] = super::model::quantize_i32(value, scales[channel], relu) as u8; + } + } + let row = + output_base + (output_row / output_width) * output_stride + output_row % output_width; + let base = row * 16; + output[base..base + 16].copy_from_slice(&packed); + } + 0 +} + +pub(crate) fn f32_to_i8_latency(xs1: u64, xs2: u64) -> u64 { + let iter = rs1_iter(xs1); + if iter == 0 || iter % 4 != 0 || xs2 >> 32 != 0 { + panic!("quant_f32_to_i8: illegal encoding"); + } + let scale = f32::from_bits(xs2 as u32); + if !scale.is_finite() || scale <= 0.0 { + panic!("quant_f32_to_i8: scale must be finite and positive"); + } + iter +} + +pub(crate) fn i32_to_i8_latency(xs1: u64, xs2: u64) -> u64 { + let iter = rs1_iter(xs1); + let input_base = (xs2 >> 29) & 0x3f; + if xs2 >> 35 != 0 { + panic!("quant_i32_to_i8: illegal encoding"); + } + let output_base = (xs2 >> 1) & 0x7f; + let output_width = (xs2 >> 8) & 0x7f; + let output_height = (xs2 >> 15) & 0x7f; + let output_stride = (xs2 >> 22) & 0x7f; + if iter == 0 + || iter % 4 != 0 + || output_width == 0 + || output_height == 0 + || output_stride < output_width + || output_width * output_height != iter / 4 + || output_base + (output_height - 1) * output_stride + output_width > bank_lines() as u64 + || input_base + iter > bank_lines() as u64 + { + panic!("quant_i32_to_i8: illegal encoding"); + } + iter + 4 +} diff --git a/examples/components/balls/toint8/stack/src/Lowering.cpp b/examples/components/balls/toint8/stack/src/Lowering.cpp new file mode 100644 index 000000000..eb41ae2d7 --- /dev/null +++ b/examples/components/balls/toint8/stack/src/Lowering.cpp @@ -0,0 +1,81 @@ +//===- Lowering.cpp - ToInt8Ball in lower-buckyball -----------------------===// +// +// fp32_to_int8: the source bank holds `rows` rows of four f32, a multiple of +// four; the destination receives rows / 4 rows of sixteen i8, +// clamp(round_half_even(x * multiplier), -128, 127). Argument 0 is the f32 +// multiplier. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballOps.h" +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class Fp32ToInt8Lowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &rewriter) const override { + if (op.getKind() != "fp32_to_int8") + return failure(); + Location loc = op.getLoc(); + auto rows = cast(op->getAttr("rows")).getInt(); + uint64_t rs1 = mlir::buddy::bankFields(op.getSources(), op.getDestination(), + false, rows); + Value bits = arith::BitcastOp::create(rewriter, loc, rewriter.getI32Type(), + op.getArguments()[0]); + Value rs2 = + arith::ExtUIOp::create(rewriter, loc, rewriter.getI64Type(), bits); + mlir::buddy::emitInstruction( + rewriter, loc, buckyball_target::getBuckyballFunct7("QUANT_F32_TO_I8"), + mlir::buddy::constantI64(rewriter, loc, rs1), rs2); + rewriter.eraseOp(op); + return success(); + } +}; + +class Int32ToInt8Lowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &rewriter) const override { + if (op.getKind() != "int32_to_int8") + return failure(); + auto rows = cast(op->getAttr("rows")).getInt(); + auto base = cast(op->getAttr("wr_base")).getInt(); + bool relu = cast(op->getAttr("relu")).getValue(); + int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; + if (rows <= 0 || rows > 127 || rows * 4 > depth || base < 0 || base > 127 || + base + rows > depth) + return op.emitOpError( + "INT32-to-INT8 rows or wr_base exceed the ISA range"); + // One row contains sixteen output bytes. A partial update retains the + // destination version; reading an old destination alone does not do that. + uint64_t rs1 = mlir::buddy::bankFields(op.getSources(), op.getDestination(), + false, rows * 4) | + (uint64_t(op.getReadDestination()) << 63); + uint64_t rs2 = uint64_t(relu) | (uint64_t(base) << 1) | (1ULL << 8) | + (uint64_t(rows) << 15) | (1ULL << 22); + mlir::buddy::emitInstruction( + rewriter, op.getLoc(), + buckyball_target::getBuckyballFunct7("QUANT_I32_TO_I8"), + mlir::buddy::constantI64(rewriter, op.getLoc(), rs1), + mlir::buddy::constantI64(rewriter, op.getLoc(), rs2)); + rewriter.eraseOp(op); + return success(); + } +}; + +void populate(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} + +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallLowering(populate), true); +} // namespace diff --git a/examples/components/balls/toint8/verify/README.md b/examples/components/balls/toint8/verify/README.md new file mode 100644 index 000000000..b68e5d7e1 --- /dev/null +++ b/examples/components/balls/toint8/verify/README.md @@ -0,0 +1,93 @@ +# ToInt8Ball UVM Verification + +Verifies the generated `ToInt8Ball` module using the shared Blink UVM framework. + +## Structure + +- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM agents and base env +- `src/common/toint8_defs.svh`: DPI imports, `toint8_require_bid`, timeouts +- `src/common/toint8_items.svh`: `toint8_cmd_item` with `load_rust_case` (Da address in `special[12:0]`) +- `src/seq/toint8_sequences.svh`: one-case sequence +- `src/cov/toint8_cov.svh`: online INT8 layout and iter 1 coverage +- `src/env/toint8_scoreboard.svh`: preload mem model (group-aware for i8), compare writes vs DPI ref +- `src/env/toint8_env.svh`: extends `bb_blink_env#(1,1)` +- `src/tests/toint8_*_test.svh`: one directed case per UVM test +- `src/pkg/toint8_pkg.sv`: package entry +- `src/tb_top.sv`: `bb_blink_if#(1,1)` + `ToInt8Ball` +- `filelists/toint8_ball.f`: VCS filelist + +## Test plan + +The test names are: + +- `toint8_signed_test`: signed values across all four source groups +- `toint8_zero_test`: all-zero activation (`Da=1.0`) +- `toint8_rounding_test`: ties-to-even quantization boundaries +- `toint8_rows_test`: two-row scan and packed output traversal +- `toint8_scale_rows_test`: maximum value is in the second row + +Each case: reset, preload src words into `mem_model` (INT8 uses `group_id` 0..3), +drive one command, wait for `scb.done()` or timeout at 400 cycles. Scoreboard +compares observed writes with DPI `toint8_ref_i8`. `toint8_cov` requires the online +INT8 layout and one-row iteration bin, else `check_phase` fatal. + +## BID (required) + +`+BID=` is mandatory. Missing it fatals at test start. No default bid. + +| Config | Plusarg | +|--------|---------| +| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=3` | +| `sims.verilator.BuckyballToyVerilatorConfig` (full.toml) | `+BID=5` | + +```console +./simv +UVM_TESTNAME=toint8_scale_rows_test +BID=3 +``` + +## Build + +Enter the shared UVM/VCS environment: + +```console +nix develop ../../../../verify +``` + +Build the DPI reference library: + +```console +cargo build --manifest-path casegen/Cargo.toml +``` + +Compile from this directory: + +```console +vcs -full64 -sverilog -timescale=1ns/1ps \ + $VCS_UVM_ARGS \ + -sv_lib ../output/casegen/debug/libtoint8_casegen \ + -f filelists/toint8_ball.f +``` + +Run: + +```console +./simv +UVM_TESTNAME=toint8_scale_rows_test +BID=3 +``` + +Or via bbdev: + +```console +bbdev uvm --build '--ball=toint8' \ + --config sims.verilator.BuckyballPebbleVerilatorConfig \ + --core-config examples/cores/pebble/configs/default.toml +bbdev uvm --run '--ball=toint8 --plusargs +BID=3' +``` + +## Acceptance (2026-08-04) + +- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` +- [x] casegen reuses toint8 model; whole-case DPI API with injected bid +- [x] directed online INT8 scoreboard pass +- [x] signed, zero, rounding, and row-traversal directed cases pass +- [x] protocol + semantic cover enforced (`check_phase` fatal) +- [x] pebble: `+BID=3` config `sims.verilator.BuckyballPebbleVerilatorConfig` +- [ ] toy full: `+BID=5` config `sims.verilator.BuckyballToyVerilatorConfig` diff --git a/examples/balls/toint8/verify/casegen/Cargo.lock b/examples/components/balls/toint8/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/toint8/verify/casegen/Cargo.lock rename to examples/components/balls/toint8/verify/casegen/Cargo.lock diff --git a/examples/balls/toint8/verify/casegen/Cargo.toml b/examples/components/balls/toint8/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/toint8/verify/casegen/Cargo.toml rename to examples/components/balls/toint8/verify/casegen/Cargo.toml diff --git a/examples/components/balls/toint8/verify/casegen/src/casegen.rs b/examples/components/balls/toint8/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..1b0a05907 --- /dev/null +++ b/examples/components/balls/toint8/verify/casegen/src/casegen.rs @@ -0,0 +1,504 @@ +use crate::model::{quantize_f32, quantize_i32}; + +pub const KIND_F32: u32 = 0; +pub const KIND_I32: u32 = 1; +pub const BANK_NUM: u32 = 24; +pub const BANK_ENTRIES: u32 = 64; +pub const SCALE_ROWS: usize = 4; +pub const LANES: usize = 4; +pub const MAX_SRC: usize = 16; +pub const MAX_SCALE: usize = 4; +pub const MAX_DST: usize = 4; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ToInt8Cmd { + pub kind: u32, + pub bid: u32, + pub iter: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, + pub rob_id: u32, + pub rs1_lo: u32, + pub rs1_hi: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub input_base: u32, + pub num_src_words: u32, + pub num_scale_words: u32, + pub num_dst_words: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ToInt8Case { + pub cmd: ToInt8Cmd, + pub src_words: [u128; MAX_SRC], + pub scale_words: [u128; MAX_SCALE], + pub dst_words: [u128; MAX_DST], + pub dst_addr: [u32; MAX_DST], +} + +impl ToInt8Case { + pub fn src_lo(&self, index: usize) -> u64 { + self.src_words[index] as u64 + } + + pub fn src_hi(&self, index: usize) -> u64 { + (self.src_words[index] >> 64) as u64 + } + + pub fn scale_lo(&self, index: usize) -> u64 { + self.scale_words[index] as u64 + } + + pub fn scale_hi(&self, index: usize) -> u64 { + (self.scale_words[index] >> 64) as u64 + } + + pub fn dst_lo(&self, index: usize) -> u64 { + self.dst_words[index] as u64 + } + + pub fn dst_hi(&self, index: usize) -> u64 { + (self.dst_words[index] >> 64) as u64 + } +} + +pub fn gen_case(index: u32, bid: u32) -> ToInt8Case { + match index { + 0 => f32_signed(bid), + 1 => f32_zero(bid), + 2 => f32_rounding(bid), + 3 => f32_rows(bid), + 4 => i32_scale_rows(bid), + 5 => i32_relu_tile(bid), + 6 => i32_signed(bid), + 7 => i32_offset(bid), + _ => panic!("toint8: unsupported directed case {index}"), + } +} + +fn pack_u32s(vals: &[u32]) -> Vec { + if vals.len() % LANES != 0 { + panic!("toint8: value count {} is not a multiple of four", vals.len()); + } + vals.chunks(LANES) + .map(|chunk| { + let mut word = 0u128; + for (lane, value) in chunk.iter().enumerate() { + word |= u128::from(*value) << (lane * 32); + } + word + }) + .collect() +} + +fn copy_words(src: &[u128]) -> [u128; N] { + if src.len() > N { + panic!("toint8: {} words exceed buffer {N}", src.len()); + } + let mut out = [0u128; N]; + out[..src.len()].copy_from_slice(src); + out +} + +fn copy_addrs(src: &[u32]) -> [u32; MAX_DST] { + if src.len() > MAX_DST { + panic!("toint8: {} dst rows exceed buffer", src.len()); + } + let mut out = [0u32; MAX_DST]; + out[..src.len()].copy_from_slice(src); + out +} + +fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { + u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) +} + +fn encode_i32_rs2( + relu: bool, + out_base: u32, + width: u32, + height: u32, + stride: u32, + in_base: u32, +) -> u64 { + u64::from(u32::from(relu)) + | (u64::from(out_base) << 1) + | (u64::from(width) << 8) + | (u64::from(height) << 15) + | (u64::from(stride) << 22) + | (u64::from(in_base) << 29) +} + +fn positive_finite(bits: u32) -> bool { + let value = f32::from_bits(bits); + value.is_finite() && value > 0.0 +} + +fn split_rs(value: u64) -> (u32, u32) { + (value as u32, (value >> 32) as u32) +} + +pub(crate) fn validate( + kind: u32, + op1_bank: u32, + op2_bank: u32, + wr_bank: u32, + op1_col: u32, + op2_col: u32, + wr_col: u32, + iter: u32, + rs1: u64, + rs2: u64, +) { + if kind != KIND_F32 && kind != KIND_I32 { + panic!("toint8: unknown kind {kind}"); + } + if iter == 0 || iter % 4 != 0 { + panic!("ToInt8Ball iter must be a positive multiple of four"); + } + if iter > BANK_ENTRIES { + panic!("ToInt8Ball input exceeds bank depth"); + } + if op1_bank >= BANK_NUM { + panic!("ToInt8Ball input bank is invalid"); + } + if wr_bank >= BANK_NUM { + panic!("ToInt8Ball output bank is invalid"); + } + if op1_col != 1 || wr_col != 1 { + panic!("ToInt8Ball input and output must each occupy one bank"); + } + if op1_bank == wr_bank { + panic!("ToInt8Ball input and output banks must differ"); + } + if (rs1 & 0x3ff) != u64::from(op1_bank) || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) { + panic!("toint8: rs1 bank fields mismatch"); + } + if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { + panic!("toint8: rs1 iter mismatch"); + } + if kind == KIND_F32 { + if ((rs1 >> 10) & 0x3ff) != 0 || op2_bank != 0 { + panic!("QUANT_F32_TO_I8 reserves input bank 1"); + } + if rs2 >> 32 != 0 { + panic!("QUANT_F32_TO_I8 reserves rs2[63:32]"); + } + if !positive_finite(rs2 as u32) { + panic!("QUANT_F32_TO_I8 scale must be finite and positive"); + } + } else { + if op2_bank >= BANK_NUM { + panic!("QUANT_I32_TO_I8 scale bank is invalid"); + } + if ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) { + panic!("toint8: rs1 scale bank mismatch"); + } + if op2_col != 1 { + panic!("QUANT_I32_TO_I8 scale must occupy one bank"); + } + if op1_bank == op2_bank { + panic!("QUANT_I32_TO_I8 input and scale banks must differ"); + } + if op2_bank == wr_bank { + panic!("QUANT_I32_TO_I8 scale and output banks must differ"); + } + if rs2 >> 35 != 0 { + panic!("QUANT_I32_TO_I8 reserves rs2[63:35]"); + } + let out_base = ((rs2 >> 1) & 0x7f) as u32; + let width = ((rs2 >> 8) & 0x7f) as u32; + let height = ((rs2 >> 15) & 0x7f) as u32; + let stride = ((rs2 >> 22) & 0x7f) as u32; + let in_base = ((rs2 >> 29) & 0x3f) as u32; + let output_rows = iter / 4; + if width == 0 || height == 0 || stride < width { + panic!("QUANT_I32_TO_I8 output geometry is invalid"); + } + if width.checked_mul(height) != Some(output_rows) { + panic!("QUANT_I32_TO_I8 iter does not match output geometry"); + } + let output_end = out_base + .checked_add((height - 1).checked_mul(stride).expect("stride overflow")) + .and_then(|value| value.checked_add(width)); + match output_end { + Some(end) if end <= BANK_ENTRIES => {} + _ => panic!("QUANT_I32_TO_I8 output exceeds bank depth"), + } + if in_base.checked_add(iter).is_none_or(|end| end > BANK_ENTRIES) { + panic!("QUANT_I32_TO_I8 input exceeds bank depth"); + } + } +} + +fn f32_dst(src: &[u128], scale: f32) -> Vec { + if src.len() % 4 != 0 { + panic!("QUANT_F32_TO_I8 src rows must be a multiple of four"); + } + src.chunks(4) + .map(|group| { + let mut packed = 0u128; + for (g, word) in group.iter().enumerate() { + for lane in 0..LANES { + let bits = (*word >> (lane * 32)) as u32; + if (bits >> 23) & 0xff == 0xff { + panic!("QUANT_F32_TO_I8 input must be finite"); + } + let q = quantize_f32(f32::from_bits(bits), scale, false) as u8; + packed |= u128::from(q) << ((g * LANES + lane) * 8); + } + } + packed + }) + .collect() +} + +fn i32_dst(src: &[u128], scales: &[u128], relu: bool) -> Vec { + if src.len() % 4 != 0 { + panic!("QUANT_I32_TO_I8 src rows must be a multiple of four"); + } + if scales.len() != SCALE_ROWS { + panic!("QUANT_I32_TO_I8 requires four scale rows"); + } + let mut scale_vals = [0.0f32; 16]; + for (row, word) in scales.iter().enumerate() { + for lane in 0..LANES { + let bits = (*word >> (lane * 32)) as u32; + if !positive_finite(bits) { + panic!("QUANT_I32_TO_I8 scales must be finite and positive"); + } + scale_vals[row * LANES + lane] = f32::from_bits(bits); + } + } + src.chunks(4) + .map(|group| { + let mut packed = 0u128; + for (g, word) in group.iter().enumerate() { + for lane in 0..LANES { + let bits = (*word >> (lane * 32)) as u32; + let channel = g * LANES + lane; + let q = quantize_i32(bits as i32, scale_vals[channel], relu) as u8; + packed |= u128::from(q) << (channel * 8); + } + } + packed + }) + .collect() +} + +fn i32_addrs(iter: u32, out_base: u32, width: u32, stride: u32) -> Vec { + let output_rows = iter / 4; + (0..output_rows) + .map(|row| out_base + (row / width) * stride + row % width) + .collect() +} + +fn finish( + kind: u32, + bid: u32, + iter: u32, + op1_bank: u32, + op2_bank: u32, + wr_bank: u32, + op2_col: u32, + rob_id: u32, + rs2: u64, + input_base: u32, + src: Vec, + scale: Vec, + dst: Vec, + dst_addr: Vec, +) -> ToInt8Case { + let op1_col = 1; + let wr_col = 1; + let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); + validate( + kind, op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs1, rs2, + ); + if src.len() != iter as usize { + panic!("toint8: src rows {} != iter {iter}", src.len()); + } + if dst.len() != (iter / 4) as usize || dst_addr.len() != dst.len() { + panic!("toint8: dst rows do not match iter/4"); + } + if kind == KIND_F32 { + if !scale.is_empty() { + panic!("QUANT_F32_TO_I8 must not emit scale rows"); + } + } else if scale.len() != SCALE_ROWS { + panic!("QUANT_I32_TO_I8 must emit four scale rows"); + } + let (rs1_lo, rs1_hi) = split_rs(rs1); + let (rs2_lo, rs2_hi) = split_rs(rs2); + ToInt8Case { + cmd: ToInt8Cmd { + kind, + bid, + iter, + op1_bank, + op2_bank, + wr_bank, + op1_col, + op2_col, + wr_col, + rob_id, + rs1_lo, + rs1_hi, + rs2_lo, + rs2_hi, + input_base, + num_src_words: src.len() as u32, + num_scale_words: scale.len() as u32, + num_dst_words: dst.len() as u32, + }, + src_words: copy_words(&src), + scale_words: copy_words(&scale), + dst_words: copy_words(&dst), + dst_addr: copy_addrs(&dst_addr), + } +} + +fn f32_case(bid: u32, rob_id: u32, scale: f32, values: &[f32]) -> ToInt8Case { + let bits: Vec = values.iter().map(|v| v.to_bits()).collect(); + let src = pack_u32s(&bits); + let iter = src.len() as u32; + let dst = f32_dst(&src, scale); + let dst_addr: Vec = (0..dst.len() as u32).collect(); + finish( + KIND_F32, + bid, + iter, + 0, + 0, + 1, + 0, + rob_id, + u64::from(scale.to_bits()), + 0, + src, + Vec::new(), + dst, + dst_addr, + ) +} + +fn i32_case( + bid: u32, + rob_id: u32, + op1_bank: u32, + op2_bank: u32, + wr_bank: u32, + relu: bool, + out_base: u32, + width: u32, + height: u32, + stride: u32, + in_base: u32, + values: &[i32], + scales: &[f32], +) -> ToInt8Case { + let src = pack_u32s(&values.iter().map(|v| *v as u32).collect::>()); + let scale = pack_u32s(&scales.iter().map(|v| v.to_bits()).collect::>()); + let iter = src.len() as u32; + let rs2 = encode_i32_rs2(relu, out_base, width, height, stride, in_base); + let dst = i32_dst(&src, &scale, relu); + let dst_addr = i32_addrs(iter, out_base, width, stride); + finish( + KIND_I32, + bid, + iter, + op1_bank, + op2_bank, + wr_bank, + 1, + rob_id, + rs2, + in_base, + src, + scale, + dst, + dst_addr, + ) +} + +fn f32_signed(bid: u32) -> ToInt8Case { + f32_case( + bid, + 2, + 0.5, + &[ + -300.0, -255.0, -5.0, -3.0, -1.0, 0.0, 1.0, 3.0, 5.0, 254.0, 255.0, 300.0, 2.0, 6.0, + 10.0, 14.0, + ], + ) +} + +fn f32_zero(bid: u32) -> ToInt8Case { + f32_case(bid, 3, 1.0, &[0.0; 16]) +} + +fn f32_rounding(bid: u32) -> ToInt8Case { + f32_case( + bid, + 4, + 1.0, + &[ + 2.5, -2.5, 3.5, -3.5, 0.5, -0.5, 1.5, -1.5, 63.5, -63.5, 127.5, -128.5, 0.0, 1.0, 2.0, + 3.0, + ], + ) +} + +fn f32_rows(bid: u32) -> ToInt8Case { + f32_case( + bid, + 5, + 2.0, + &[ + 0.125, -0.125, 0.25, -0.25, 0.75, -0.75, 1.25, -1.25, 1.75, -1.75, 63.25, 63.75, + -63.75, -64.75, 0.0, -0.0, 2.25, -2.25, 2.75, -2.75, 3.25, -3.25, 3.75, -3.75, 10.125, + -10.125, 20.25, -20.25, 0.375, -0.375, 64.25, -65.25, + ], + ) +} + +fn i32_scale_rows(bid: u32) -> ToInt8Case { + let mut values = [0i32; 16]; + let mut scales = [0.0f32; 16]; + for i in 0..16 { + values[i] = i as i32 - 8; + scales[i] = 0.5 + (i as f32) * 0.25; + } + i32_case(bid, 6, 0, 1, 2, false, 0, 1, 1, 1, 0, &values, &scales) +} + +fn i32_relu_tile(bid: u32) -> ToInt8Case { + let mut values = [0i32; 64]; + for position in 0..4 { + for channel in 0..16 { + values[position * 16 + channel] = position as i32 * 20 + channel as i32 - 8; + } + } + i32_case(bid, 7, 0, 1, 2, true, 3, 2, 2, 4, 0, &values, &[1.0; 16]) +} + +fn i32_signed(bid: u32) -> ToInt8Case { + let values: [i32; 16] = [ + -300, -128, -5, -1, 0, 1, 5, 127, 128, 200, -64, 64, -7, 7, 3, -3, + ]; + i32_case(bid, 8, 0, 1, 2, false, 0, 1, 1, 1, 0, &values, &[1.0; 16]) +} + +fn i32_offset(bid: u32) -> ToInt8Case { + let mut values = [0i32; 32]; + for i in 0..32 { + values[i] = i as i32 - 16; + } + i32_case(bid, 9, 0, 1, 2, true, 1, 2, 1, 2, 4, &values, &[0.5; 16]) +} diff --git a/examples/components/balls/toint8/verify/casegen/src/lib.rs b/examples/components/balls/toint8/verify/casegen/src/lib.rs new file mode 100644 index 000000000..51878c5fa --- /dev/null +++ b/examples/components/balls/toint8/verify/casegen/src/lib.rs @@ -0,0 +1,119 @@ +mod casegen; +#[path = "../../../emu/src/model.rs"] +mod model; + +use std::cell::RefCell; + +use casegen::{ToInt8Case, ToInt8Cmd, MAX_DST, MAX_SCALE, MAX_SRC}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn toint8_case_load(index: u32, bid: u32) -> i32 { + let case = casegen::gen_case(index, bid); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("toint8_case: no case loaded; call toint8_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn toint8_case_cmd(out_ptr: *mut ToInt8Cmd) { + if out_ptr.is_null() { + panic!("toint8_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +fn src_index(index: u32, limit: u32, what: &str) -> usize { + if index >= limit { + panic!("toint8_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn toint8_case_src_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_src_words, "src_word_lo"); + if i >= MAX_SRC { + panic!("toint8_case_src_word_lo: word_index out of range"); + } + case.src_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn toint8_case_src_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_src_words, "src_word_hi"); + if i >= MAX_SRC { + panic!("toint8_case_src_word_hi: word_index out of range"); + } + case.src_hi(i) + }) +} + +#[no_mangle] +pub extern "C" fn toint8_case_scale_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_lo"); + if i >= MAX_SCALE { + panic!("toint8_case_scale_word_lo: word_index out of range"); + } + case.scale_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn toint8_case_scale_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_scale_words, "scale_word_hi"); + if i >= MAX_SCALE { + panic!("toint8_case_scale_word_hi: word_index out of range"); + } + case.scale_hi(i) + }) +} + +#[no_mangle] +pub extern "C" fn toint8_case_dst_word_lo(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_lo"); + if i >= MAX_DST { + panic!("toint8_case_dst_word_lo: word_index out of range"); + } + case.dst_lo(i) + }) +} + +#[no_mangle] +pub extern "C" fn toint8_case_dst_word_hi(word_index: u32) -> u64 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_dst_words, "dst_word_hi"); + if i >= MAX_DST { + panic!("toint8_case_dst_word_hi: word_index out of range"); + } + case.dst_hi(i) + }) +} + +#[no_mangle] +pub extern "C" fn toint8_case_dst_addr(word_index: u32) -> u32 { + current_case(|case| { + let i = src_index(word_index, case.cmd.num_dst_words, "dst_addr"); + if i >= MAX_DST { + panic!("toint8_case_dst_addr: word_index out of range"); + } + case.dst_addr[i] + }) +} diff --git a/examples/balls/toint8/verify/filelists/toint8_ball.f b/examples/components/balls/toint8/verify/filelists/toint8_ball.f similarity index 100% rename from examples/balls/toint8/verify/filelists/toint8_ball.f rename to examples/components/balls/toint8/verify/filelists/toint8_ball.f diff --git a/examples/balls/toint8/verify/src/common/toint8_defs.svh b/examples/components/balls/toint8/verify/src/common/toint8_defs.svh similarity index 92% rename from examples/balls/toint8/verify/src/common/toint8_defs.svh rename to examples/components/balls/toint8/verify/src/common/toint8_defs.svh index fc68e0d95..41d20e9d6 100644 --- a/examples/balls/toint8/verify/src/common/toint8_defs.svh +++ b/examples/components/balls/toint8/verify/src/common/toint8_defs.svh @@ -33,14 +33,14 @@ import "DPI-C" function longint unsigned toint8_case_dst_word_hi(input int unsig import "DPI-C" function int unsigned toint8_case_dst_addr(input int unsigned word_index); `ifndef QUANT_F32_TO_I8_FUNCT7 -`error "QUANT_F32_TO_I8_FUNCT7 must be provided by the selected Core ballISA" +`error "QUANT_F32_TO_I8_FUNCT7 must be provided by the selected Thread ballISA" `endif `ifndef QUANT_I32_TO_I8_FUNCT7 -`error "QUANT_I32_TO_I8_FUNCT7 must be provided by the selected Core ballISA" +`error "QUANT_I32_TO_I8_FUNCT7 must be provided by the selected Thread ballISA" `endif -localparam int QUANT_F32_TO_I8_CORE_FUNCT7 = `QUANT_F32_TO_I8_FUNCT7; -localparam int QUANT_I32_TO_I8_CORE_FUNCT7 = `QUANT_I32_TO_I8_FUNCT7; +localparam int QUANT_F32_TO_I8_THREAD_FUNCT7 = `QUANT_F32_TO_I8_FUNCT7; +localparam int QUANT_I32_TO_I8_THREAD_FUNCT7 = `QUANT_I32_TO_I8_FUNCT7; localparam int TOINT8_KIND_F32 = 0; localparam int TOINT8_KIND_I32 = 1; localparam int TOINT8_MAX_SRC_WORDS = 16; diff --git a/examples/balls/toint8/verify/src/common/toint8_items.svh b/examples/components/balls/toint8/verify/src/common/toint8_items.svh similarity index 94% rename from examples/balls/toint8/verify/src/common/toint8_items.svh rename to examples/components/balls/toint8/verify/src/common/toint8_items.svh index d345b8e2f..3f2bfc7b8 100644 --- a/examples/balls/toint8/verify/src/common/toint8_items.svh +++ b/examples/components/balls/toint8/verify/src/common/toint8_items.svh @@ -12,7 +12,7 @@ class toint8_cmd_item extends bb_blink_cmd_item; bit [31:0] dst_addr[TOINT8_MAX_DST_WORDS]; constraint legal_c { - funct7 inside {QUANT_F32_TO_I8_CORE_FUNCT7[6:0], QUANT_I32_TO_I8_CORE_FUNCT7[6:0]}; + funct7 inside {QUANT_F32_TO_I8_THREAD_FUNCT7[6:0], QUANT_I32_TO_I8_THREAD_FUNCT7[6:0]}; op1_en == 1'b1; wr_spad_en == 1'b1; op1_from_spad == 1'b1; @@ -70,7 +70,7 @@ class toint8_cmd_item extends bb_blink_cmd_item; sub_rob_id = 8'h00; if (kind == TOINT8_KIND_F32) begin - funct7 = QUANT_F32_TO_I8_CORE_FUNCT7[6:0]; + funct7 = QUANT_F32_TO_I8_THREAD_FUNCT7[6:0]; op2_en = 1'b0; op2_from_spad = 1'b0; if (op2_bank != 0 || op2_col != 0 || rs1[19:10] != 0) @@ -79,7 +79,7 @@ class toint8_cmd_item extends bb_blink_cmd_item; if (num_scale_words != 0) `uvm_fatal("CASE", "QUANT_F32_TO_I8 must not emit scale rows") if (input_base != 0) `uvm_fatal("CASE", "QUANT_F32_TO_I8 input_base must be 0") end else begin - funct7 = QUANT_I32_TO_I8_CORE_FUNCT7[6:0]; + funct7 = QUANT_I32_TO_I8_THREAD_FUNCT7[6:0]; op2_en = 1'b1; op2_from_spad = 1'b1; if (op2_col != 1) `uvm_fatal("CASE", "QUANT_I32_TO_I8 scale must occupy one bank") @@ -103,7 +103,7 @@ class toint8_cmd_item extends bb_blink_cmd_item; `uvm_fatal("CASE", $sformatf("num_src_words out of range: %0d", num_src_words)) if (num_dst_words == 0 || num_dst_words > TOINT8_MAX_DST_WORDS) `uvm_fatal("CASE", $sformatf("num_dst_words out of range: %0d", num_dst_words)) - if (funct7 != QUANT_F32_TO_I8_CORE_FUNCT7[6:0] && funct7 != QUANT_I32_TO_I8_CORE_FUNCT7[6:0]) + if (funct7 != QUANT_F32_TO_I8_THREAD_FUNCT7[6:0] && funct7 != QUANT_I32_TO_I8_THREAD_FUNCT7[6:0]) `uvm_fatal("CASE", $sformatf("invalid funct7: %0d", funct7)) for (i = 0; i < num_src_words; i++) begin diff --git a/examples/balls/toint8/verify/src/cov/toint8_cov.svh b/examples/components/balls/toint8/verify/src/cov/toint8_cov.svh similarity index 84% rename from examples/balls/toint8/verify/src/cov/toint8_cov.svh rename to examples/components/balls/toint8/verify/src/cov/toint8_cov.svh index f18079d10..7113d9669 100644 --- a/examples/balls/toint8/verify/src/cov/toint8_cov.svh +++ b/examples/components/balls/toint8/verify/src/cov/toint8_cov.svh @@ -7,7 +7,8 @@ class toint8_cov extends uvm_component; covergroup cmd_cg; coverpoint cur_funct7 { - bins f32 = {QUANT_F32_TO_I8_CORE_FUNCT7[6:0]}; bins i32 = {QUANT_I32_TO_I8_CORE_FUNCT7[6:0]}; + bins f32 = {QUANT_F32_TO_I8_THREAD_FUNCT7[6:0]}; + bins i32 = {QUANT_I32_TO_I8_THREAD_FUNCT7[6:0]}; } endgroup @@ -26,6 +27,7 @@ class toint8_cov extends uvm_component; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") if (cmd_cg.get_coverage() != 100.0) diff --git a/examples/balls/toint8/verify/src/env/toint8_env.svh b/examples/components/balls/toint8/verify/src/env/toint8_env.svh similarity index 100% rename from examples/balls/toint8/verify/src/env/toint8_env.svh rename to examples/components/balls/toint8/verify/src/env/toint8_env.svh diff --git a/examples/balls/toint8/verify/src/env/toint8_scoreboard.svh b/examples/components/balls/toint8/verify/src/env/toint8_scoreboard.svh similarity index 98% rename from examples/balls/toint8/verify/src/env/toint8_scoreboard.svh rename to examples/components/balls/toint8/verify/src/env/toint8_scoreboard.svh index cc0de35bd..a6b932840 100644 --- a/examples/balls/toint8/verify/src/env/toint8_scoreboard.svh +++ b/examples/components/balls/toint8/verify/src/env/toint8_scoreboard.svh @@ -66,6 +66,7 @@ class toint8_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, toint8_cmd_item exp); + verification_pkg::evidence::checks++; if (got.bid !== exp.bid) `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) if (got.funct7 !== exp.funct7) @@ -178,6 +179,7 @@ class toint8_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) begin `uvm_fatal("SCB", $sformatf("incomplete: cmds=%0d reads=%0d/%0d writes=%0d/%0d resp=%0d", diff --git a/examples/balls/toint8/verify/src/pkg/toint8_pkg.sv b/examples/components/balls/toint8/verify/src/pkg/toint8_pkg.sv similarity index 100% rename from examples/balls/toint8/verify/src/pkg/toint8_pkg.sv rename to examples/components/balls/toint8/verify/src/pkg/toint8_pkg.sv diff --git a/examples/balls/toint8/verify/src/seq/toint8_sequences.svh b/examples/components/balls/toint8/verify/src/seq/toint8_sequences.svh similarity index 100% rename from examples/balls/toint8/verify/src/seq/toint8_sequences.svh rename to examples/components/balls/toint8/verify/src/seq/toint8_sequences.svh diff --git a/examples/components/balls/toint8/verify/src/tb_top.sv b/examples/components/balls/toint8/verify/src/tb_top.sv new file mode 100644 index 000000000..f1947f8b7 --- /dev/null +++ b/examples/components/balls/toint8/verify/src/tb_top.sv @@ -0,0 +1,90 @@ +module tb_top; + import uvm_pkg::*; + import toint8_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + ToInt8Ball dut ( + .clock(intf.clock), + .reset(intf.reset), + .io_cmdReq_ready(intf.cmd_req_ready), + .io_cmdReq_valid(intf.cmd_req_valid), + .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), + .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), + .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), + .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), + .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), + .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), + .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), + .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), + .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), + .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), + .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), + .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), + .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), + .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), + .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), + .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), + .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), + .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), + .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), + .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), + .io_cmdResp_ready(intf.cmd_resp_ready), + .io_cmdResp_valid(intf.cmd_resp_valid), + .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), + .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), + .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .io_bankRead_0_group_id(intf.bank_read_group_id[0]), + .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .io_subRobReq_ready(intf.sub_rob_req_ready), + .io_status_idle(), + .io_status_running(), + .io_bankAccessReady(1'b1), + .io_bankRead_0_ball_id(), + .io_bankWrite_0_ball_id() + ); + + initial begin + intf.clock = 1'b0; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("toint8_ball_test"); + end +endmodule diff --git a/examples/balls/toint8/verify/src/tests/toint8_test.svh b/examples/components/balls/toint8/verify/src/tests/toint8_test.svh similarity index 100% rename from examples/balls/toint8/verify/src/tests/toint8_test.svh rename to examples/components/balls/toint8/verify/src/tests/toint8_test.svh diff --git a/examples/components/balls/toint8/workloads/ctests/CMakeLists.txt b/examples/components/balls/toint8/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..ddb7a1f5c --- /dev/null +++ b/examples/components/balls/toint8/workloads/ctests/CMakeLists.txt @@ -0,0 +1,6 @@ +add_buckyball_ctest(quant_f32_to_i8_test.c) +add_buckyball_ctest(quant_i32_to_i8_test.c) +add_buckyball_ctest(toint8_i8_rounding_test.c) +add_buckyball_ctest(toint8_i8_rows_test.c) +add_buckyball_ctest(toint8_i8_scale_rows_test.c) +add_buckyball_ctest(toint8_i8_signed_test.c) diff --git a/examples/balls/toint8/workloads/ctests/quant_f32_to_i8_test.c b/examples/components/balls/toint8/workloads/ctests/quant_f32_to_i8_test.c similarity index 100% rename from examples/balls/toint8/workloads/ctests/quant_f32_to_i8_test.c rename to examples/components/balls/toint8/workloads/ctests/quant_f32_to_i8_test.c diff --git a/examples/balls/toint8/workloads/ctests/quant_i32_to_i8_test.c b/examples/components/balls/toint8/workloads/ctests/quant_i32_to_i8_test.c similarity index 100% rename from examples/balls/toint8/workloads/ctests/quant_i32_to_i8_test.c rename to examples/components/balls/toint8/workloads/ctests/quant_i32_to_i8_test.c diff --git a/examples/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c b/examples/components/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c similarity index 95% rename from examples/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c rename to examples/components/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c index 5837167b4..2fed43afb 100644 --- a/examples/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c +++ b/examples/components/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c @@ -15,8 +15,8 @@ static const int8_t expected[16] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 1, 1); bb_mem_alloc(1, 1, 1); diff --git a/examples/balls/toint8/workloads/ctests/toint8_i8_rows_test.c b/examples/components/balls/toint8/workloads/ctests/toint8_i8_rows_test.c similarity index 95% rename from examples/balls/toint8/workloads/ctests/toint8_i8_rows_test.c rename to examples/components/balls/toint8/workloads/ctests/toint8_i8_rows_test.c index d1000ae18..d6f1f2aa7 100644 --- a/examples/balls/toint8/workloads/ctests/toint8_i8_rows_test.c +++ b/examples/components/balls/toint8/workloads/ctests/toint8_i8_rows_test.c @@ -18,8 +18,8 @@ static const int8_t expected[32] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 1, 1); bb_mem_alloc(1, 1, 1); diff --git a/examples/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c b/examples/components/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c similarity index 95% rename from examples/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c rename to examples/components/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c index 526cbd310..a7e80f446 100644 --- a/examples/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c +++ b/examples/components/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c @@ -16,8 +16,8 @@ static const int8_t expected[32] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 1, 1); bb_mem_alloc(1, 1, 1); diff --git a/examples/balls/toint8/workloads/ctests/toint8_i8_signed_test.c b/examples/components/balls/toint8/workloads/ctests/toint8_i8_signed_test.c similarity index 95% rename from examples/balls/toint8/workloads/ctests/toint8_i8_signed_test.c rename to examples/components/balls/toint8/workloads/ctests/toint8_i8_signed_test.c index 65a518077..c9ffdfc3a 100644 --- a/examples/balls/toint8/workloads/ctests/toint8_i8_signed_test.c +++ b/examples/components/balls/toint8/workloads/ctests/toint8_i8_signed_test.c @@ -15,8 +15,8 @@ static const int8_t expected[16] = { }; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif bb_mem_alloc(0, 1, 1); bb_mem_alloc(1, 1, 1); diff --git a/examples/balls/toint8/workloads/isa/quant.h b/examples/components/balls/toint8/workloads/isa/quant.h similarity index 100% rename from examples/balls/toint8/workloads/isa/quant.h rename to examples/components/balls/toint8/workloads/isa/quant.h diff --git a/examples/balls/toint8/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/toint8/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/toint8/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/toint8/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/toint8/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/toint8/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/toint8/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/toint8/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir b/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir new file mode 100644 index 000000000..aec1588b8 --- /dev/null +++ b/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir @@ -0,0 +1,25 @@ +func.func private @check_result(memref<1x16xi8>) -> () + +memref.global "private" constant @input : memref<4x4xf32> = dense<[ + [-3.000000e+02, -2.550000e+02, -5.000000e+00, -3.000000e+00], + [-1.000000e+00, 0.000000e+00, 1.000000e+00, 3.000000e+00], + [5.000000e+00, 2.540000e+02, 2.550000e+02, 3.000000e+02], + [2.000000e+00, 6.000000e+00, 1.000000e+01, 1.400000e+01] +]> + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %scale = arith.constant 5.000000e-01 : f32 + %input = memref.get_global @input : memref<4x4xf32> + %output = memref.alloc() alignment = 64 : memref<1x16xi8> + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 0 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + buckyball.custom "fp32_to_int8" args(%scale : f32) <{sources = array, destination = 1 : i64}> {rows = 4 : i64} + buckyball.mvout %output <{bank = 1 : i64, depth = 1 : i64, stride = 1 : i64}> : memref<1x16xi8> + func.call @check_result(%output) : (memref<1x16xi8>) -> () + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %output : memref<1x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8_main.cpp b/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8_main.cpp similarity index 100% rename from examples/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8_main.cpp rename to examples/components/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8_main.cpp diff --git a/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir b/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir new file mode 100644 index 000000000..8e36bd460 --- /dev/null +++ b/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir @@ -0,0 +1,50 @@ +func.func private @check_result(memref<12x16xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %one_f32 = arith.constant 1.0 : f32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %twenty = arith.constant 20 : index + %eight = arith.constant 8 : index + %input = memref.alloc() alignment = 64 : memref<16x4xi32> + %scales = memref.alloc() alignment = 64 : memref<4x4xf32> + %output_seed = memref.alloc() alignment = 64 : memref<12x16xi8> + %output = memref.alloc() alignment = 64 : memref<12x16xi8> + linalg.fill ins(%one_f32 : f32) outs(%scales : memref<4x4xf32>) + linalg.fill ins(%zero_i8 : i8) outs(%output_seed : memref<12x16xi8>) + linalg.fill ins(%zero_i8 : i8) outs(%output : memref<12x16xi8>) + scf.for %position = %zero to %four step %one { + scf.for %channel = %zero to %sixteen step %one { + %position_base = arith.muli %position, %twenty : index + %with_channel = arith.addi %position_base, %channel : index + %value = arith.subi %with_channel, %eight : index + %value_i32 = arith.index_cast %value : index to i32 + %channel_row = arith.divui %channel, %four : index + %row_base = arith.muli %position, %four : index + %row = arith.addi %row_base, %channel_row : index + %lane = arith.remui %channel, %four : index + memref.store %value_i32, %input[%row, %lane] : memref<16x4xi32> + } + } + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + buckyball.mvin %scales <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + buckyball.mvin %output_seed <{bank = 2 : i64, depth = 12 : i64, stride = 1 : i64}> : memref<12x16xi8> + // Four positions to rows 3..6; the other rows keep the seed. + buckyball.custom "int32_to_int8" <{sources = array, destination = 2 : i64, read_destination}> {rows = 4 : i64, wr_base = 3 : i64, relu = true} + buckyball.mvout %output <{bank = 2 : i64, depth = 12 : i64, stride = 1 : i64}> : memref<12x16xi8> + func.call @check_result(%output) : (memref<12x16xi8>) -> () + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + memref.dealloc %input : memref<16x4xi32> + memref.dealloc %scales : memref<4x4xf32> + memref.dealloc %output_seed : memref<12x16xi8> + memref.dealloc %output : memref<12x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp b/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp similarity index 80% rename from examples/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp rename to examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp index 85eb15724..cc1c33451 100644 --- a/examples/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp +++ b/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp @@ -21,11 +21,7 @@ extern "C" void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, int8_t *output = aligned + offset; for (int row = 0; row < 12; ++row) { for (int channel = 0; channel < 16; ++channel) { - int position = row == 3 ? 0 - : row == 4 ? 1 - : row == 7 ? 2 - : row == 8 ? 3 - : -1; + int position = row >= 3 && row < 7 ? row - 3 : -1; int expected = position < 0 ? 0 : position * 20 + channel - 8; if (expected < 0) expected = 0; @@ -38,5 +34,5 @@ extern "C" void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, } } } - printf("PASSED: quant_i32_to_i8 relu and strided output\n"); + printf("PASSED: quant_i32_to_i8 relu and output base\n"); } diff --git a/examples/components/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..fc5a32057 --- /dev/null +++ b/examples/components/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,22 @@ +if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +endif() +set(BUCKYBALL_MLIR_TEST_PREFIX bank) + +set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") +file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") +string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") +math(EXPR _remainder "${_lines} % 4") +if(_lines LESS 4 OR NOT (_remainder EQUAL 0)) + message(FATAL_ERROR + "ToInt8 INT32 BankOp test requires BANK_LINES divisible by four " + "(got lines=${_lines})") +endif() +# One QUANT_I32_TO_I8 instruction accepts at most 508 packed INT32 rows. +set(BB_QUANT_I32_INPUT_ROWS ${_lines}) +if(BB_QUANT_I32_INPUT_ROWS GREATER 508) + set(BB_QUANT_I32_INPUT_ROWS 508) +endif() +math(EXPR BB_QUANT_I32_OUTPUT_ROWS "${BB_QUANT_I32_INPUT_ROWS} / 4") +add_buckyball_mlir_test(quant_i32_pack16 TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) +add_buckyball_mlir_test(quant_f32_pack16 TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in b/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in new file mode 100644 index 000000000..481456bcd --- /dev/null +++ b/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in @@ -0,0 +1,36 @@ +func.func private @check_result(memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %rows = arith.constant @BB_QUANT_I32_OUTPUT_ROWS@ : index + %scale = arith.constant 1.0 : f32 + %input = memref.alloc() : memref<@BB_BANK_LINES@x4xf32> + %output = memref.alloc() : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> + scf.for %position = %zero to %rows step %one { + scf.for %channel = %zero to %sixteen step %one { + %base = arith.muli %position, %sixteen : index + %linear = arith.addi %base, %channel : index + %value_i32 = arith.index_cast %linear : index to i32 + %value = arith.sitofp %value_i32 : i32 to f32 + %row_base = arith.muli %position, %four : index + %channel_row = arith.divui %channel, %four : index + %row = arith.addi %row_base, %channel_row : index + %lane = arith.remui %channel, %four : index + memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x4xf32> + } + } + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x4xf32> + %result = buckyball.bank_custom "fp32_to_int8" (%loaded_input) -> %output_bank args(%scale : f32) {rows = @BB_BANK_LINES@ : i64} + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_QUANT_I32_OUTPUT_ROWS@ : i64, stride = 1 : i64}> : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> + func.call @check_result(%output) : (memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () + buckyball.bank_release %loaded_input + buckyball.bank_release %stored + memref.dealloc %input : memref<@BB_BANK_LINES@x4xf32> + memref.dealloc %output : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp b/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp similarity index 100% rename from examples/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp rename to examples/components/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp diff --git a/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in b/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in new file mode 100644 index 000000000..f41cb4fed --- /dev/null +++ b/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in @@ -0,0 +1,41 @@ +func.func private @check_result(memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %one_f32 = arith.constant 1.0 : f32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %four = arith.constant 4 : index + %sixteen = arith.constant 16 : index + %rows = arith.constant @BB_QUANT_I32_OUTPUT_ROWS@ : index + %input = memref.alloc() : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> + %scale = memref.alloc() : memref<4x4xf32> + %output = memref.alloc() : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> + linalg.fill ins(%one_f32 : f32) outs(%scale : memref<4x4xf32>) + scf.for %position = %zero to %rows step %one { + scf.for %channel = %zero to %sixteen step %one { + %base = arith.muli %position, %sixteen : index + %linear = arith.addi %base, %channel : index + %value = arith.index_cast %linear : index to i32 + %row_base = arith.muli %position, %four : index + %channel_row = arith.divui %channel, %four : index + %row = arith.addi %row_base, %channel_row : index + %lane = arith.remui %channel, %four : index + memref.store %value, %input[%row, %lane] : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> + } + } + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %scale_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_QUANT_I32_INPUT_ROWS@ : i64, stride = 1 : i64}> : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> + %loaded_scale = buckyball.bank_mvin %scale, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + %result = buckyball.bank_custom "int32_to_int8" (%loaded_input, %loaded_scale) -> %output_bank {rows = @BB_QUANT_I32_OUTPUT_ROWS@ : i64, wr_base = 0 : i64, relu = false} + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_QUANT_I32_OUTPUT_ROWS@ : i64, stride = 1 : i64}> : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> + func.call @check_result(%output) : (memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () + buckyball.bank_release %loaded_input + buckyball.bank_release %loaded_scale + buckyball.bank_release %stored + memref.dealloc %input : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> + memref.dealloc %scale : memref<4x4xf32> + memref.dealloc %output : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp b/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp similarity index 100% rename from examples/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp rename to examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp diff --git a/examples/balls/trace/arch/src/main/scala/README.md b/examples/components/balls/trace/arch/README.md similarity index 100% rename from examples/balls/trace/arch/src/main/scala/README.md rename to examples/components/balls/trace/arch/README.md diff --git a/examples/components/balls/trace/arch/Trace.scala b/examples/components/balls/trace/arch/Trace.scala new file mode 100644 index 000000000..6c297f762 --- /dev/null +++ b/examples/components/balls/trace/arch/Trace.scala @@ -0,0 +1,219 @@ +package examples.balls.trace + +import framework.acc.balldomain.configs.BallRouteParam + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} + +/** + * Trace — TraceBall inner processing unit. + * + * Handles cycle counter management (START/STOP/READ). + */ +@instantiable +class Trace( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + + val ballMapping = instance + + val inBW = ballMapping.inBW + val outBW = ballMapping.outBW + + val bankWidth = memParam.bankWidth + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + // ============================================================ + // Constants + // ============================================================ + val CTR_START = 0.U(4.W) + val CTR_STOP = 1.U(4.W) + val CTR_READ = 2.U(4.W) + + val NUM_COUNTERS = 16 + + // ============================================================ + // State machine + // ============================================================ + val idle :: sCounter :: complete :: Nil = Enum(3) + val state = RegInit(idle) + + // ============================================================ + // Registers + // ============================================================ + val rob_id_reg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + + // Counter-specific registers + val subcmd_reg = RegInit(0.U(4.W)) + val ctr_id_reg = RegInit(0.U(4.W)) + val payload_reg = RegInit(0.U(56.W)) + + // Cycle counters: 16 independent counters + val cycleCounter = RegInit(0.U(64.W)) + cycleCounter := cycleCounter + 1.U + + val ctrStartCycle = RegInit(VecInit(Seq.fill(NUM_COUNTERS)(0.U(64.W)))) + val ctrTag = RegInit(VecInit(Seq.fill(NUM_COUNTERS)(0.U(56.W)))) + val ctrActive = RegInit(VecInit(Seq.fill(NUM_COUNTERS)(false.B))) + + // ============================================================ + // DPI-C modules + // ============================================================ + val ctraceDpi = Instantiate(new CTraceDPI) + ctraceDpi.io.clock := clock + ctraceDpi.io.reset := reset.asBool + ctraceDpi.io.subcmd := 0.U + ctraceDpi.io.ctr_id := 0.U + ctraceDpi.io.tag := 0.U + ctraceDpi.io.elapsed := 0.U + ctraceDpi.io.cycle := cycleCounter + ctraceDpi.io.enable := false.B + + // ============================================================ + // External bank port defaults + // ============================================================ + for (i <- 0 until inBW) { + io.bankRead(i).rob_id := rob_id_reg + io.bankRead(i).bank_id := 0.U + io.bankRead(i).group_id := 0.U + io.bankRead(i).io.req.valid := false.B + io.bankRead(i).io.req.bits.addr := 0.U + io.bankRead(i).io.resp.ready := false.B + } + + for (i <- 0 until outBW) { + io.bankWrite(i).rob_id := rob_id_reg + io.bankWrite(i).bank_id := 0.U + io.bankWrite(i).group_id := 0.U + io.bankWrite(i).io.req.valid := false.B + io.bankWrite(i).io.req.bits.addr := 0.U + io.bankWrite(i).io.req.bits.data := 0.U + io.bankWrite(i).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(0.U(1.W)) + ) + io.bankWrite(i).io.resp.ready := false.B + } + + // ============================================================ + // Command interface + // ============================================================ + io.cmdReq.ready := state === idle + io.cmdResp.valid := false.B + io.cmdResp.bits.rob_id := rob_id_reg + + // ============================================================ + // State machine + // ============================================================ + switch(state) { + is(idle) { + when(io.cmdReq.fire) { + rob_id_reg := io.cmdReq.bits.rob_id + + val cmd = io.cmdReq.bits + val rs2 = cmd.rs2 + + // Counter fields from rs2 + subcmd_reg := rs2(3, 0) + ctr_id_reg := rs2(7, 4) + payload_reg := rs2(63, 8) + + state := sCounter + } + } + + // ---------------------------------------------------------- + // Counter instruction: execute and complete in 1 cycle + // ---------------------------------------------------------- + is(sCounter) { + val cid = ctr_id_reg + + ctraceDpi.io.subcmd := subcmd_reg + ctraceDpi.io.ctr_id := cid + + switch(subcmd_reg) { + is(CTR_START) { + ctrStartCycle(cid) := cycleCounter + ctrTag(cid) := payload_reg + ctrActive(cid) := true.B + + ctraceDpi.io.tag := payload_reg + ctraceDpi.io.elapsed := 0.U + ctraceDpi.io.enable := true.B + } + is(CTR_STOP) { + val elapsed = cycleCounter - ctrStartCycle(cid) + ctrActive(cid) := false.B + + ctraceDpi.io.tag := ctrTag(cid) + ctraceDpi.io.elapsed := elapsed + ctraceDpi.io.enable := true.B + } + is(CTR_READ) { + val current = cycleCounter - ctrStartCycle(cid) + + ctraceDpi.io.tag := ctrTag(cid) + ctraceDpi.io.elapsed := current + ctraceDpi.io.enable := true.B + } + } + + state := complete + } + + // ---------------------------------------------------------- + // Complete: fire cmdResp + // ---------------------------------------------------------- + is(complete) { + io.cmdResp.valid := true.B + io.cmdResp.bits.rob_id := rob_id_reg + when(io.cmdResp.fire) { + state := idle + } + } + } + + // ============================================================ + // Status + // ============================================================ + io.status.idle := state === idle + io.status.running := state =/= idle && state =/= complete +} diff --git a/examples/components/balls/trace/arch/TraceBall.scala b/examples/components/balls/trace/arch/TraceBall.scala new file mode 100644 index 000000000..921381767 --- /dev/null +++ b/examples/components/balls/trace/arch/TraceBall.scala @@ -0,0 +1,68 @@ +package examples.balls.trace + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.trace.{Params => ParametersProto} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.{BlinkIO, HasBlink} + +/** + * TraceBall — Debug trace Ball providing cycle counters. + * + * The Thread ballISA assigns the funct7 encoding for bdb_counter. + */ +@instantiable +class TraceBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + ball.parameters.unpack(classOf[ParametersProto]) + + val ballCommonConfig = ball + + val inBW = ballCommonConfig.inBW + val outBW = ballCommonConfig.outBW + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val traceUnit: Instance[Trace] = + Instantiate(new Trace(fp, mp, bp, ball)) + + traceUnit.io.cmdReq <> blink.cmdReq + traceUnit.io.cmdResp <> blink.cmdResp + + for (i <- 0 until inBW) { + traceUnit.io.bankRead(i) <> blink.bankRead(i) + } + + for (i <- 0 until outBW) { + traceUnit.io.bankWrite(i) <> blink.bankWrite(i) + } + + blink.status <> traceUnit.io.status + + // MMIO: this Ball does not consume MMIO metadata +} + +object TraceBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new TraceBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/balls/trace/arch/src/main/scala/TraceDPI.scala b/examples/components/balls/trace/arch/TraceDPI.scala similarity index 96% rename from examples/balls/trace/arch/src/main/scala/TraceDPI.scala rename to examples/components/balls/trace/arch/TraceDPI.scala index a7d1e94dd..d6cae579b 100644 --- a/examples/balls/trace/arch/src/main/scala/TraceDPI.scala +++ b/examples/components/balls/trace/arch/TraceDPI.scala @@ -1,11 +1,15 @@ package examples.balls.trace +import chisel3.experimental.hierarchy.{instantiable, public} + import chisel3._ import chisel3.util._ -import framework.dpi.DpiGuard +import framework.lib.dpi.DpiGuard +@instantiable class CTraceDPI extends BlackBox with HasBlackBoxInline { + @public val io = IO(new Bundle { val clock = Input(Clock()) val reset = Input(Bool()) diff --git a/examples/components/balls/trace/configs/default.toml b/examples/components/balls/trace/configs/default.toml new file mode 100644 index 000000000..014d280eb --- /dev/null +++ b/examples/components/balls/trace/configs/default.toml @@ -0,0 +1 @@ +# Trace Ball Configuration diff --git a/examples/components/balls/trace/configs/trace_params.proto b/examples/components/balls/trace/configs/trace_params.proto new file mode 100644 index 000000000..28fd8d1c2 --- /dev/null +++ b/examples/components/balls/trace/configs/trace_params.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; +package buckyball.balls.trace; +option java_package = "buckyball.balls.trace"; +option java_multiple_files = true; + +message Params { +} diff --git a/examples/components/balls/trace/emu/src/04_bdb_counter.rs b/examples/components/balls/trace/emu/src/04_bdb_counter.rs new file mode 100644 index 000000000..5ab37932e --- /dev/null +++ b/examples/components/balls/trace/emu/src/04_bdb_counter.rs @@ -0,0 +1,15 @@ +//===- 04_bdb_counter.rs - BDB_COUNTER instruction -------------------------===// + +use super::instruction::{BallModel, ExecContext}; + +pub struct BdbCounter; + +impl BallModel for BdbCounter { + fn exec(_xs1: u64, _xs2: u64, _ctx: &mut ExecContext) -> u64 { + 0 + } + + fn latency(_xs1: u64, _xs2: u64) -> u64 { + 1 + } +} diff --git a/examples/components/balls/trace/emu/src/dispatch.rs b/examples/components/balls/trace/emu/src/dispatch.rs new file mode 100644 index 000000000..cfca2fbd7 --- /dev/null +++ b/examples/components/balls/trace/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::f04_bdb_counter; + +const BALL_CLASS: &str = "examples.balls.trace.TraceBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("BDB_COUNTER") => Some(f04_bdb_counter::BdbCounter::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("BDB_COUNTER") => Some(f04_bdb_counter::BdbCounter::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/components/balls/trace/emu/src/lib.rs b/examples/components/balls/trace/emu/src/lib.rs new file mode 100644 index 000000000..61ae12d8d --- /dev/null +++ b/examples/components/balls/trace/emu/src/lib.rs @@ -0,0 +1,12 @@ +pub(crate) use bebop_bemu::ball as instruction; + +#[path = "04_bdb_counter.rs"] +mod f04_bdb_counter; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/components/balls/trace/stack/src/Lowering.cpp b/examples/components/balls/trace/stack/src/Lowering.cpp new file mode 100644 index 000000000..900fc5929 --- /dev/null +++ b/examples/components/balls/trace/stack/src/Lowering.cpp @@ -0,0 +1,66 @@ +//===- Lowering.cpp - TraceBall in lower-buckyball ------------------------===// +// +// The BDB counter touches no bank: its operations have no sources and +// destination 0, and rs1 is zero. `counter` (0 to 15) selects the counter. +// counter_start starts it with the i64 tag of argument 0; counter_read traces +// its current value; counter_stop traces the elapsed cycles and stops it. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballOps.h" +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class CounterLowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &rewriter) const override { + uint64_t subcommand; + if (op.getKind() == "counter_start") + subcommand = 0; + else if (op.getKind() == "counter_stop") + subcommand = 1; + else if (op.getKind() == "counter_read") + subcommand = 2; + else + return failure(); + Location loc = op.getLoc(); + int64_t counter = cast(op->getAttr("counter")).getInt(); + if (!op.getSources().empty() || op.getDestination() != 0 || + op.getReadDestination()) + return op.emitOpError("BDB counter touches no bank"); + if (counter < 0 || counter > 15 || + op.getArguments().size() != (subcommand == 0 ? 1 : 0)) + return op.emitOpError("BDB counter needs a counter in [0, 15] and a " + "tag only to start"); + Value rs2 = mlir::buddy::constantI64(rewriter, loc, + mlir::buddy::field(subcommand, 0, 3) | + mlir::buddy::field(counter, 4, 7)); + if (subcommand == 0) { + Value tag = + arith::ShLIOp::create(rewriter, loc, op.getArguments()[0], + mlir::buddy::constantI64(rewriter, loc, 8)); + rs2 = arith::OrIOp::create(rewriter, loc, rs2, tag); + } + mlir::buddy::emitInstruction( + rewriter, loc, buckyball_target::getBuckyballFunct7("BDB_COUNTER"), + mlir::buddy::constantI64(rewriter, loc, 0), rs2); + rewriter.eraseOp(op); + return success(); + } +}; + +void populate(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} + +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallLowering(populate), true); +} // namespace diff --git a/examples/components/balls/trace/workloads/ctests/CMakeLists.txt b/examples/components/balls/trace/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..970b38aec --- /dev/null +++ b/examples/components/balls/trace/workloads/ctests/CMakeLists.txt @@ -0,0 +1,4 @@ +add_buckyball_ctest(bdb_counter_start_stop_test.c) +add_buckyball_ctest(bdb_counter_read_active_test.c) +add_buckyball_ctest(bdb_counter_nested_pair_test.c) +add_buckyball_ctest(bdb_counter_multi_id_test.c) diff --git a/examples/balls/trace/workloads/ctests/bdb_counter_multi_id_test.c b/examples/components/balls/trace/workloads/ctests/bdb_counter_multi_id_test.c similarity index 100% rename from examples/balls/trace/workloads/ctests/bdb_counter_multi_id_test.c rename to examples/components/balls/trace/workloads/ctests/bdb_counter_multi_id_test.c diff --git a/examples/balls/trace/workloads/ctests/bdb_counter_nested_pair_test.c b/examples/components/balls/trace/workloads/ctests/bdb_counter_nested_pair_test.c similarity index 100% rename from examples/balls/trace/workloads/ctests/bdb_counter_nested_pair_test.c rename to examples/components/balls/trace/workloads/ctests/bdb_counter_nested_pair_test.c diff --git a/examples/balls/trace/workloads/ctests/bdb_counter_read_active_test.c b/examples/components/balls/trace/workloads/ctests/bdb_counter_read_active_test.c similarity index 100% rename from examples/balls/trace/workloads/ctests/bdb_counter_read_active_test.c rename to examples/components/balls/trace/workloads/ctests/bdb_counter_read_active_test.c diff --git a/examples/balls/trace/workloads/ctests/bdb_counter_start_stop_test.c b/examples/components/balls/trace/workloads/ctests/bdb_counter_start_stop_test.c similarity index 100% rename from examples/balls/trace/workloads/ctests/bdb_counter_start_stop_test.c rename to examples/components/balls/trace/workloads/ctests/bdb_counter_start_stop_test.c diff --git a/examples/balls/trace/workloads/isa/bdb_counter.h b/examples/components/balls/trace/workloads/isa/bdb_counter.h similarity index 100% rename from examples/balls/trace/workloads/isa/bdb_counter.h rename to examples/components/balls/trace/workloads/isa/bdb_counter.h diff --git a/examples/balls/trace/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/trace/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/trace/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/trace/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/trace/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/trace/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/trace/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/trace/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir b/examples/components/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir new file mode 100644 index 000000000..479ff040c --- /dev/null +++ b/examples/components/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir @@ -0,0 +1,8 @@ +func.func @main() -> i8 { + %zero8 = arith.constant 0 : i8 + %tag = arith.constant 48879 : i64 + buckyball.custom "counter_start" args(%tag : i64) <{sources = array, destination = 0 : i64}> {counter = 3 : i64} + buckyball.custom "counter_read" <{sources = array, destination = 0 : i64}> {counter = 3 : i64} + buckyball.custom "counter_stop" <{sources = array, destination = 0 : i64}> {counter = 3 : i64} + return %zero8 : i8 +} diff --git a/examples/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop_main.cpp b/examples/components/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop_main.cpp similarity index 100% rename from examples/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop_main.cpp rename to examples/components/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop_main.cpp diff --git a/examples/components/balls/transpose/arch/README.md b/examples/components/balls/transpose/arch/README.md new file mode 100644 index 000000000..4d0628d1a --- /dev/null +++ b/examples/components/balls/transpose/arch/README.md @@ -0,0 +1,115 @@ +# Matrix Transpose Accelerator + +## Overview + +This directory implements Buckyball's matrix transpose accelerator for matrix transpose operations. Located at `examples/components/balls/transpose/arch`, it serves as a matrix transpose accelerator supporting pipelined transpose operations. + +Core components: +- **Transpose.scala**: Pipelined transposer implementation + +## Code Structure + +``` +transpose/ +└── Transpose.scala - Pipelined transposer +``` + +### Module Responsibilities + +**Transpose.scala** (Transpose implementation layer) +- Implements PipelinedTransposer module +- Manages matrix data read, transpose, and write-back +- Provides Ball domain command interface + +## Module Description + +### Transpose.scala + +**Main functionality**: Implements pipelined matrix transpose operation + +**State machine definition**: +```scala +val idle :: sRead :: sWrite :: complete :: Nil = Enum(4) +val state = RegInit(idle) +``` + +**Storage structure**: +```scala +// Matrix storage register (veclane x veclane) +val regArray = Reg(Vec(b.veclane, Vec(b.veclane, UInt(b.inputType.getWidth.W)))) +``` + +**Counter management**: +```scala +val readCounter = RegInit(0.U(log2Ceil(b.veclane + 1).W)) +val respCounter = RegInit(0.U(log2Ceil(b.veclane + 1).W)) +val writeCounter = RegInit(0.U(log2Ceil(b.veclane + 1).W)) +``` + +**Instruction registers**: +```scala +val robid_reg = RegInit(0.U(10.W)) // ROB ID +val waddr_reg = RegInit(0.U(10.W)) // Write address +val wbank_reg = RegInit(0.U(log2Up(b.sp_banks).W)) // Write bank +val raddr_reg = RegInit(0.U(10.W)) // Read address +val rbank_reg = RegInit(0.U(log2Up(b.sp_banks).W)) // Read bank +val iter_reg = RegInit(0.U(10.W)) // Iteration count +``` + +**Interface definition**: +```scala +val io = IO(new Bundle { + val cmdReq = Flipped(Decoupled(new BallRsIssue)) + val cmdResp = Decoupled(new BallRsComplete) + val sramRead = Vec(b.sp_banks, Flipped(new SramReadIO(...))) + val sramWrite = Vec(b.sp_banks, Flipped(new SramWriteIO(...))) +}) +``` + +**Processing flow**: +1. **idle**: Wait for command, parse transpose parameters +2. **sRead**: Read matrix data row by row into register array +3. **sWrite**: Write transposed data column by column +4. **complete**: Send completion signal + +**Transpose algorithm**: +- Uses veclane×veclane register array to store matrix +- Reads row-wise, writes column-wise to implement transpose +- Supports block-wise transpose for matrices of arbitrary size + +## Usage + +### Implementation Details + +**State machine**: +```scala +val idle :: sRead :: sWrite :: complete :: Nil = Enum(4) +``` +- `idle`: Wait for instruction +- `sRead`: Read matrix data +- `sWrite`: Write transpose result +- `complete`: Complete and respond + +**Register array**: +```scala +val regArray = Reg(Vec(b.veclane, Vec(b.veclane, UInt(b.inputType.getWidth.W)))) +``` +Uses veclane×veclane register array to cache matrix data. + +**Transpose operation**: +- Read phase: Read data row by row into `regArray(row)(col)` +- Write phase: Read `regArray(i)(col)` column by column to form new rows for writing + +### Configuration Parameters + +**Matrix size**: Determined by b.veclane parameter +**Data width**: Determined by b.inputType.getWidth +**Bank configuration**: Supports multi-bank SRAM access + +### Notes + +1. **Matrix size limitation**: Maximum support for veclane×veclane matrices +2. **Memory bandwidth**: Transpose operation has high memory bandwidth requirements +3. **Register overhead**: Requires veclane² registers to store matrix +4. **Address calculation**: Transposed address calculation needs to be handled correctly +5. **Pipeline control**: Read/write counters need to be synchronized correctly diff --git a/examples/components/balls/transpose/arch/Transpose.scala b/examples/components/balls/transpose/arch/Transpose.scala new file mode 100644 index 000000000..a9d572018 --- /dev/null +++ b/examples/components/balls/transpose/arch/Transpose.scala @@ -0,0 +1,278 @@ +package examples.balls.transpose + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} +import examples.balls.transpose.configs.TransposeBallParam + +@instantiable +class Transpose( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: TransposeBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val ballConfig = ballParam + val bankWidth = memParam.bankWidth + val rowBytes = bankWidth / 8 + + val ballMapping = instance + + val inBW = ballMapping.inBW + val outBW = ballMapping.outBW + require(inBW == outBW, "TransposeBall requires inBW == outBW") + require(inBW == 1, "TransposeBall gather/scatter path requires inBW == 1") + require(bankWidth % 8 == 0, "bankWidth must be byte-aligned") + require( + ballConfig.InputNum * ballConfig.inputWidth == bankWidth, + "TransposeBall InputNum*inputWidth must equal bankWidth" + ) + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val rob_id_reg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + + val rbank_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + val wbank_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + val ncol_reg = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) + val iter_reg = RegInit(0.U(frontendParam.iter_len.W)) + val elem_reg = RegInit(0.U(8.W)) + + val idle :: sRead :: sWrite :: complete :: Nil = Enum(4) + val state = RegInit(idle) + + // Walk destination dense index 0 .. iter*W-1, filling write beats. + val pending = RegInit(false.B) + val wrBytes = Reg(Vec(rowBytes, UInt(8.W))) + val wrMask = RegInit(VecInit(Seq.fill(memParam.bankMaskLen)(0.U(1.W)))) + val wrGroup = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) + val wrAddr = RegInit(0.U(frontendParam.iter_len.W)) + val wrValid = RegInit(false.B) + val beatFill = RegInit(0.U(8.W)) + + val cached = RegInit(false.B) + val cacheAddr = RegInit(0.U(frontendParam.iter_len.W)) + val cacheGroup = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) + val bases = Reg(Vec(3, UInt(memParam.groupIdWidth.W))) + val cacheData = RegInit(0.U(bankWidth.W)) + + val srcR = RegInit(0.U(frontendParam.iter_len.W)) + val srcC = RegInit(0.U(32.W)) + val dstVirtRow = RegInit(0.U(frontendParam.iter_len.W)) + val dstVirtCol = RegInit(0.U(32.W)) + val epgReg = RegInit(0.U(8.W)) + val wElemsReg = RegInit(0.U(32.W)) + val elemBytesReg = RegInit(0.U(8.W)) + val lastBeat = RegInit(false.B) + + for (i <- 0 until inBW) { + io.bankRead(i).rob_id := rob_id_reg + io.bankRead(i).bank_id := rbank_reg + io.bankRead(i).group_id := bases(0) + io.bankRead(i).io.req.valid := false.B + io.bankRead(i).io.req.bits.addr := 0.U + io.bankRead(i).io.resp.ready := false.B + } + for (i <- 0 until outBW) { + io.bankWrite(i).rob_id := rob_id_reg + io.bankWrite(i).bank_id := wbank_reg + io.bankWrite(i).group_id := bases(2) + io.bankWrite(i).io.req.valid := false.B + io.bankWrite(i).io.req.bits.addr := 0.U + io.bankWrite(i).io.req.bits.data := 0.U + io.bankWrite(i).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(0.U(1.W)) + ) + io.bankWrite(i).io.resp.ready := (state =/= idle) + } + + io.cmdReq.ready := (state === idle) + io.cmdResp.valid := false.B + io.cmdResp.bits.rob_id := rob_id_reg + + require(isPow2(rowBytes), "Transpose rowBytes must be power of 2") + + val laneBitsI8 = log2Ceil(rowBytes) + val laneBitsI32 = log2Ceil(rowBytes / 4) + + val isI8 = elemBytesReg === 1.U + val srcLane = Mux(isI8, srcC(laneBitsI8 - 1, 0), srcC(laneBitsI32 - 1, 0)) + val srcGroup = Mux(isI8, srcC >> laneBitsI8.U, srcC >> laneBitsI32.U) + .asTypeOf(UInt(log2Up(memParam.bankNum + 1).W)) + val srcAddr = srcR + + val dstGroup = Mux(isI8, dstVirtCol >> laneBitsI8.U, dstVirtCol >> laneBitsI32.U) + .asTypeOf(UInt(log2Up(memParam.bankNum + 1).W)) + val bytes = cacheData.asTypeOf(Vec(rowBytes, UInt(8.W))) + val words = cacheData.asTypeOf(Vec(rowBytes / 4, UInt(32.W))) + + val needRead = !cached || cacheAddr =/= srcAddr || cacheGroup =/= srcGroup + + switch(state) { + is(idle) { + when(io.cmdReq.fire) { + val cmd = io.cmdReq.bits + rob_id_reg := io.cmdReq.bits.rob_id + rbank_reg := cmd.op1_bank + wbank_reg := cmd.wr_bank + bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + ncol_reg := cmd.op1_col + iter_reg := cmd.iter + elem_reg := cmd.rs2(7, 0) + pending := false.B + wrValid := false.B + beatFill := 0.U + cached := false.B + srcR := 0.U + srcC := 0.U + dstVirtRow := 0.U + dstVirtCol := 0.U + lastBeat := false.B + val i8 = cmd.rs2(7, 0) === 8.U + elemBytesReg := Mux(i8, 1.U, 4.U) + epgReg := Mux(i8, rowBytes.U, (rowBytes / 4).U) + wElemsReg := Mux(i8, cmd.op1_col << laneBitsI8.U, cmd.op1_col << laneBitsI32.U) + assert(cmd.iter > 0.U, "Transpose iter must be > 0") + assert(cmd.op1_bank =/= cmd.wr_bank, "Transpose op1 and wr must differ") + assert( + cmd.op1_col === cmd.wr_col && cmd.op1_col =/= 0.U, + "Transpose cols mismatch" + ) + assert(cmd.rs2(63, 8) === 0.U, "Transpose rs2[63:8] must be 0") + assert( + cmd.rs2(7, 0) === 8.U || cmd.rs2(7, 0) === 32.U, + "Transpose elem_bits" + ) + assert( + bankWidth.U % cmd.rs2(7, 0) === 0.U, + "Transpose bankWidth not divisible by elem_bits" + ) + state := sRead + } + } + + is(sRead) { + io.bankRead(0).group_id := bases(0) + srcGroup + io.bankRead(0).io.resp.ready := pending + io.bankRead(0).io.req.valid := needRead && !pending && !wrValid + io.bankRead(0).io.req.bits.addr := srcAddr + + when(io.bankRead(0).io.req.fire) { + pending := true.B + } + when(io.bankRead(0).io.resp.fire) { + cacheData := io.bankRead(0).io.resp.bits.data + cacheAddr := srcAddr + cacheGroup := srcGroup + cached := true.B + pending := false.B + } + + when(cached && !needRead && !wrValid) { + when(beatFill === 0.U) { + wrGroup := dstGroup + wrAddr := dstVirtRow + wrBytes := VecInit(Seq.fill(rowBytes)(0.U(8.W))) + } + when(isI8) { + wrBytes(beatFill) := bytes(srcLane) + }.otherwise { + val w = words(srcLane) + wrBytes(beatFill << 2) := w(7, 0) + wrBytes((beatFill << 2) + 1.U) := w(15, 8) + wrBytes((beatFill << 2) + 2.U) := w(23, 16) + wrBytes((beatFill << 2) + 3.U) := w(31, 24) + } + beatFill := beatFill + 1.U + + val lastElem = (srcR === iter_reg - 1.U) && (srcC === wElemsReg - 1.U) + when(srcR === iter_reg - 1.U) { + srcR := 0.U + srcC := srcC + 1.U + }.otherwise { + srcR := srcR + 1.U + } + when(dstVirtCol === wElemsReg - 1.U) { + dstVirtCol := 0.U + dstVirtRow := dstVirtRow + 1.U + }.otherwise { + dstVirtCol := dstVirtCol + 1.U + } + + val beatDone = (beatFill + 1.U === epgReg) || lastElem + when(beatDone) { + wrMask.foreach(_ := 1.U) + wrValid := true.B + lastBeat := lastElem + state := sWrite + } + } + } + + is(sWrite) { + io.bankWrite(0).group_id := bases(2) + wrGroup + io.bankWrite(0).io.req.valid := wrValid + io.bankWrite(0).io.req.bits.addr := wrAddr + io.bankWrite(0).io.req.bits.data := wrBytes.asUInt + io.bankWrite(0).io.req.bits.mask := wrMask + io.bankWrite(0).io.resp.ready := true.B + when(io.bankWrite(0).io.req.fire) { + wrValid := false.B + beatFill := 0.U + } + when(io.bankWrite(0).io.resp.fire) { + when(lastBeat) { + state := complete + }.otherwise { + state := sRead + } + } + } + + is(complete) { + io.cmdResp.valid := true.B + when(io.cmdResp.fire) { + state := idle + } + } + } + + io.status.idle := (state === idle) + io.status.running := (state =/= idle) +} diff --git a/examples/components/balls/transpose/arch/TransposeBall.scala b/examples/components/balls/transpose/arch/TransposeBall.scala new file mode 100644 index 000000000..c03f21dc9 --- /dev/null +++ b/examples/components/balls/transpose/arch/TransposeBall.scala @@ -0,0 +1,67 @@ +package examples.balls.transpose + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.transpose.{Params => ParametersProto} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.transpose.configs.TransposeBallParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.{BlinkIO, HasBlink} +import examples.balls.transpose.Transpose + +@instantiable +class TransposeBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + val ballParam = TransposeBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + val ballCommonConfig = ball + + val inBW = ballCommonConfig.inBW + val outBW = ballCommonConfig.outBW + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val transposeUnit: Instance[Transpose] = + Instantiate(new Transpose(fp, mp, bp, ball, ballParam)) + + transposeUnit.io.cmdReq <> blink.cmdReq + transposeUnit.io.cmdResp <> blink.cmdResp + + for (i <- 0 until inBW) { + transposeUnit.io.bankRead(i) <> blink.bankRead(i) + } + + for (i <- 0 until outBW) { + transposeUnit.io.bankWrite(i) <> blink.bankWrite(i) + } + + blink.status <> transposeUnit.io.status + + // MMIO: this Ball does not consume MMIO metadata +} + +object TransposeBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new TransposeBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/components/balls/transpose/configs/TransposeBallParam.scala b/examples/components/balls/transpose/configs/TransposeBallParam.scala new file mode 100644 index 000000000..a381e99d7 --- /dev/null +++ b/examples/components/balls/transpose/configs/TransposeBallParam.scala @@ -0,0 +1,20 @@ +package examples.balls.transpose.configs + +import buckyball.balls.transpose.{Params => ParametersProto} +import upickle.default.{macroRW, ReadWriter} + +case class TransposeBallParam( + InputNum: Int, + inputWidth: Int) + +object TransposeBallParam { + + def fromProto(source: ParametersProto): TransposeBallParam = { + val parameters = source + require(parameters.hasInputNum && parameters.hasInputWidth) + val param = TransposeBallParam(parameters.getInputNum, parameters.getInputWidth) + param + } + + implicit val rw: ReadWriter[TransposeBallParam] = macroRW +} diff --git a/examples/components/balls/transpose/configs/default.toml b/examples/components/balls/transpose/configs/default.toml new file mode 100644 index 000000000..6d4c32c7d --- /dev/null +++ b/examples/components/balls/transpose/configs/default.toml @@ -0,0 +1,4 @@ +# Transpose Ball Configuration + +InputNum = 16 +inputWidth = 8 diff --git a/examples/components/balls/transpose/configs/transpose_params.proto b/examples/components/balls/transpose/configs/transpose_params.proto new file mode 100644 index 000000000..423ba834a --- /dev/null +++ b/examples/components/balls/transpose/configs/transpose_params.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; +package buckyball.balls.transpose; +option java_package = "buckyball.balls.transpose"; +option java_multiple_files = true; + +message Params { + optional int32 InputNum = 1; + optional int32 inputWidth = 2; +} diff --git a/examples/components/balls/transpose/emu/src/dispatch.rs b/examples/components/balls/transpose/emu/src/dispatch.rs new file mode 100644 index 000000000..37dd8002f --- /dev/null +++ b/examples/components/balls/transpose/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::ExecContext; + +use super::transpose; + +const BALL_CLASS: &str = "examples.balls.transpose.TransposeBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("TRANSPOSE") => Some(transpose::exec_transpose(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("TRANSPOSE") => Some(transpose::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/components/balls/transpose/emu/src/lib.rs b/examples/components/balls/transpose/emu/src/lib.rs new file mode 100644 index 000000000..e268be558 --- /dev/null +++ b/examples/components/balls/transpose/emu/src/lib.rs @@ -0,0 +1,11 @@ +pub(crate) use bebop_bemu::{thread::instructions::decode, ball as instruction}; + +mod transpose; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/balls/transpose/emu/src/transpose.rs b/examples/components/balls/transpose/emu/src/transpose.rs similarity index 97% rename from examples/balls/transpose/emu/src/transpose.rs rename to examples/components/balls/transpose/emu/src/transpose.rs index cb948cb28..fe91ead13 100644 --- a/examples/balls/transpose/emu/src/transpose.rs +++ b/examples/components/balls/transpose/emu/src/transpose.rs @@ -1,4 +1,4 @@ -use super::super::bank::{bank_row_bytes, bank_size, bank_width}; +use bebop_bemu::thread::banks::{bank_row_bytes, bank_size, bank_width}; use super::decode::{pbank_group, rs1_b0, rs1_b2, rs1_iter}; use super::instruction::ExecContext; diff --git a/examples/components/balls/transpose/stack/src/Lowering.cpp b/examples/components/balls/transpose/stack/src/Lowering.cpp new file mode 100644 index 000000000..8fe1ee973 --- /dev/null +++ b/examples/components/balls/transpose/stack/src/Lowering.cpp @@ -0,0 +1,55 @@ +//===- Lowering.cpp - TransposeBall in lower-buckyball --------------------===// +// +// Both banks have cols / (bankWidthBits / 8) columns. The instruction packs +// the transposed bytes into the same bank geometry, with `rows` bank rows. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballOps.h" +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class TransposeI8Lowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(CustomOp op, + PatternRewriter &rewriter) const override { + if (op.getKind() != "transpose_i8") + return failure(); + if (op.getSources().size() != 1 || + op.getSources()[0] == op.getDestination()) + return op.emitOpError( + "transpose_i8 requires distinct input and output banks"); + int64_t rows = cast(op->getAttr("rows")).getInt(); + int64_t cols = cast(op->getAttr("cols")).getInt(); + const auto &target = buckyball_target::getBuckyballTarget(); + int64_t lanes = target.bankWidthBits / 8; + if (lanes <= 0 || target.bankWidthBits % 8 || rows <= 0 || + rows >= (1LL << 33) || rows > target.bankDepth || cols <= 0 || + cols % lanes || cols / lanes > 31) + return op.emitOpError( + "transpose_i8 shape exceeds the bank or ISA geometry"); + mlir::buddy::emitInstruction( + rewriter, op.getLoc(), + buckyball_target::getBuckyballFunct7("TRANSPOSE"), + mlir::buddy::constantI64(rewriter, op.getLoc(), + mlir::buddy::bankFields(op.getSources(), + op.getDestination(), + false, rows)), + mlir::buddy::constantI64(rewriter, op.getLoc(), 8)); + rewriter.eraseOp(op); + return success(); + } +}; + +void populate(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} + +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallLowering(populate), true); +} // namespace diff --git a/examples/components/balls/transpose/verify/README.md b/examples/components/balls/transpose/verify/README.md new file mode 100644 index 000000000..0556427c9 --- /dev/null +++ b/examples/components/balls/transpose/verify/README.md @@ -0,0 +1,91 @@ +# TransposeBall UVM Verification + +Verifies the generated `TransposeBall` module directly using the shared Blink UVM framework. + +## Structure + +- `../../../../verify/uvm/src/ball/bb_uvm_pkg.sv`: common Blink UVM transaction items and base env +- `src/common/transpose_defs.svh`: DPI imports and constants +- `src/common/transpose_items.svh`: `transpose_cmd_item` with `load_rust_case` +- `src/seq/transpose_sequences.svh`: `transpose_basic_seq` driving one case per run +- `src/cov/transpose_cov.svh`: `elem_bits` {8,32} and `iter` {1,2,4,8,16} coverage +- `src/env/transpose_scoreboard.svh`: preloads mem model, compares writes vs DPI dst words +- `src/env/transpose_env.svh`: extends `bb_blink_env#(1,1)` with scoreboard and cov +- `src/tests/transpose_test.svh`: directed cases 0,1 then random 2..21 with fixed seed +- `src/pkg/transpose_pkg.sv`: package include entry +- `src/tb_top.sv`: instantiates `bb_blink_if#(1,1)` and `TransposeBall` +- `filelists/transpose_ball.f`: VCS/Verilator filelist + +## Test plan + +`+UVM_TESTNAME=transpose_ball_test` runs: + +- case 0: directed i8, iter=16 +- case 1: directed i32, iter=8 +- cases 2..21: random with fixed seed `0xCAFE_BABE`, elem_bits in {8,32}, iter in {1,2,4,8,16} + +Each case: reset, preload src words into `mem_model`, drive one command, wait for +`scb.done()` or timeout at 2000 cycles. Scoreboard compares observed writes +(data/addr/mask/bank/rob) with DPI `transpose_case_dst_word_*`. `transpose_cov` +requires both `elem_bits` bins and all `iter` bins hit across the whole test, else +`check_phase` fatal. + +## BID (required) + +`+BID=` is mandatory. Missing it fatals at test start. There is no default bid. + +| Config | Plusarg | +|--------|---------| +| `sims.verilator.BuckyballToyVerilatorConfig` | `+BID=2` | +| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=0` | + +```console +./simv +UVM_TESTNAME=transpose_ball_test +BID=2 +``` + +## Build + +Enter the shared UVM/VCS environment: + +```console +nix develop ../../../../verify +``` + +Build the DPI reference library: + +```console +cargo build --manifest-path casegen/Cargo.toml +``` + +Compile from this directory: + +```console +vcs -full64 -sverilog -timescale=1ns/1ps \ + $VCS_UVM_ARGS \ + -sv_lib ../output/casegen/debug/libtranspose_casegen \ + -f filelists/transpose_ball.f +``` + +Run: + +```console +./simv +UVM_TESTNAME=transpose_ball_test +BID=2 +``` + +Or via bbdev: + +```console +bbdev uvm --build '--ball=transpose' +bbdev uvm --run '--ball=transpose' -- '+BID=2' +``` + + +## Acceptance (2026-08-03) + +- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` +- [x] casegen reuses transpose model; whole-case DPI API +- [x] directed i8/i32 scoreboard pass +- [x] deterministic random idx 2..21 pass +- [x] protocol + semantic cover enforced (`check_phase` fatal) +- [x] toy: `+BID=2` config `sims.verilator.BuckyballToyVerilatorConfig` — 22/22 +- [x] pebble: `+BID=0` config `sims.verilator.BuckyballPebbleVerilatorConfig` — 22/22 diff --git a/examples/balls/transpose/verify/casegen/Cargo.lock b/examples/components/balls/transpose/verify/casegen/Cargo.lock similarity index 100% rename from examples/balls/transpose/verify/casegen/Cargo.lock rename to examples/components/balls/transpose/verify/casegen/Cargo.lock diff --git a/examples/balls/transpose/verify/casegen/Cargo.toml b/examples/components/balls/transpose/verify/casegen/Cargo.toml similarity index 100% rename from examples/balls/transpose/verify/casegen/Cargo.toml rename to examples/components/balls/transpose/verify/casegen/Cargo.toml diff --git a/examples/components/balls/transpose/verify/casegen/src/casegen.rs b/examples/components/balls/transpose/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..5d5a00338 --- /dev/null +++ b/examples/components/balls/transpose/verify/casegen/src/casegen.rs @@ -0,0 +1,183 @@ +use crate::model; + +pub const BANK_ROW_BYTES: usize = 16; +pub const MAX_ITER: usize = 16; +pub const MAX_WORDS: usize = MAX_ITER; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TransposeCmd { + pub bid: u32, + pub iter: u32, + pub op1_bank: u32, + pub wr_bank: u32, + pub op1_col: u32, + pub wr_col: u32, + pub rob_id: u32, + pub elem_bits: u32, + pub num_src_words: u32, + pub num_dst_words: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TransposeCase { + pub cmd: TransposeCmd, + pub src: Vec, + pub dst: Vec, +} + +impl TransposeCase { + pub fn src_word_lo(&self, word_index: usize) -> u64 { + word_lo(&self.src, word_index) + } + pub fn src_word_hi(&self, word_index: usize) -> u64 { + word_hi(&self.src, word_index) + } + pub fn dst_word_lo(&self, word_index: usize) -> u64 { + word_lo(&self.dst, word_index) + } + pub fn dst_word_hi(&self, word_index: usize) -> u64 { + word_hi(&self.dst, word_index) + } +} + +fn word_lo(data: &[u8], word_index: usize) -> u64 { + let off = word_index * BANK_ROW_BYTES; + let mut buf = [0u8; 8]; + buf.copy_from_slice(&data[off..off + 8]); + u64::from_le_bytes(buf) +} + +fn word_hi(data: &[u8], word_index: usize) -> u64 { + let off = word_index * BANK_ROW_BYTES + 8; + let mut buf = [0u8; 8]; + buf.copy_from_slice(&data[off..off + 8]); + u64::from_le_bytes(buf) +} + +pub fn gen_case(seed: u32, index: u32, bid: u32) -> TransposeCase { + match index { + 0 => directed_i8(bid), + 1 => directed_i32(bid), + _ => random_case(seed, index, bid), + } +} + +fn directed_i8(bid: u32) -> TransposeCase { + let iter = 16usize; + let elem_bytes = 1usize; + let cols = 1usize; + let w = cols * (BANK_ROW_BYTES / elem_bytes); + let total = iter * w; + let src: Vec = (0..total).map(|i| i as u8).collect(); + let dst = model::transpose_bytes(&src, iter, w, elem_bytes); + let nwords = model::num_words(total * elem_bytes) as u32; + TransposeCase { + cmd: TransposeCmd { + bid, + iter: iter as u32, + op1_bank: 0, + wr_bank: 1, + op1_col: cols as u32, + wr_col: cols as u32, + rob_id: 3, + elem_bits: (elem_bytes * 8) as u32, + num_src_words: nwords, + num_dst_words: nwords, + }, + src, + dst, + } +} + +fn directed_i32(bid: u32) -> TransposeCase { + let iter = 8usize; + let elem_bytes = 4usize; + let cols = 1usize; + let w = cols * (BANK_ROW_BYTES / elem_bytes); + let total = iter * w; + let src: Vec = (0..total) + .map(|i| i as u32) + .flat_map(|v| v.to_le_bytes()) + .collect(); + let dst = model::transpose_bytes(&src, iter, w, elem_bytes); + let nwords = model::num_words(total * elem_bytes) as u32; + TransposeCase { + cmd: TransposeCmd { + bid, + iter: iter as u32, + op1_bank: 0, + wr_bank: 1, + op1_col: cols as u32, + wr_col: cols as u32, + rob_id: 5, + elem_bits: (elem_bytes * 8) as u32, + num_src_words: nwords, + num_dst_words: nwords, + }, + src, + dst, + } +} + +fn random_case(seed: u32, index: u32, bid: u32) -> TransposeCase { + let mut rng = Rng::new(seed, index); + let elem_bits = if rng.next() & 1 == 0 { 8u32 } else { 32u32 }; + let elem_bytes = (elem_bits / 8) as usize; + let iter_pool = [1u32, 2, 4, 8, 16]; + let iter = iter_pool[(rng.next() as usize) % iter_pool.len()] as usize; + let cols = 1usize; + let w = cols * (BANK_ROW_BYTES / elem_bytes); + + // Keep banks in 0..7 so the same cases work on configs with bankNum=8 or 16. + let op1_bank = rng.range(0, 7); + let mut wr_bank = rng.range(0, 7); + if wr_bank == op1_bank { + wr_bank = (wr_bank + 1) & 7; + } + + let total = iter * w; + let mut src = vec![0u8; total * elem_bytes]; + for b in src.iter_mut() { + *b = (rng.next() & 0xFF) as u8; + } + let dst = model::transpose_bytes(&src, iter, w, elem_bytes); + let nwords = model::num_words(total * elem_bytes) as u32; + + TransposeCase { + cmd: TransposeCmd { + bid, + iter: iter as u32, + op1_bank, + wr_bank, + op1_col: cols as u32, + wr_col: cols as u32, + rob_id: rng.range(0, 15), + elem_bits, + num_src_words: nwords, + num_dst_words: nwords, + }, + src, + dst, + } +} + +struct Rng { + state: u64, +} + +impl Rng { + fn new(seed: u32, index: u32) -> Self { + let state = (u64::from(seed) << 32) ^ u64::from(index) ^ 0x9E37_79B9_7F4A_7C15; + Self { state } + } + fn next(&mut self) -> u32 { + self.state ^= self.state >> 12; + self.state ^= self.state << 25; + self.state ^= self.state >> 27; + ((self.state.wrapping_mul(0x2545_F491_4F6C_DD1D)) >> 32) as u32 + } + fn range(&mut self, lo: u32, hi: u32) -> u32 { + lo + (self.next() % (hi - lo + 1)) + } +} diff --git a/examples/components/balls/transpose/verify/casegen/src/lib.rs b/examples/components/balls/transpose/verify/casegen/src/lib.rs new file mode 100644 index 000000000..fee2abd4b --- /dev/null +++ b/examples/components/balls/transpose/verify/casegen/src/lib.rs @@ -0,0 +1,71 @@ +mod casegen; +mod model; + +use std::cell::RefCell; + +use casegen::{TransposeCase, TransposeCmd, MAX_WORDS}; + +thread_local! { + static CURRENT: RefCell> = RefCell::new(None); +} + +#[no_mangle] +pub extern "C" fn transpose_case_load(seed: u32, index: u32, bid: u32) -> i32 { + let case = casegen::gen_case(seed, index, bid); + let nsrc = case.cmd.num_src_words as usize; + let ndst = case.cmd.num_dst_words as usize; + if nsrc > MAX_WORDS || ndst > MAX_WORDS { + panic!("transpose_case_load: word count out of range src={nsrc} dst={ndst}"); + } + CURRENT.with(|c| *c.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|c| match c.borrow().as_ref() { + Some(case) => f(case), + None => panic!("transpose_case: no case loaded; call transpose_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn transpose_case_cmd(out_ptr: *mut TransposeCmd) { + current_case(|case| unsafe { + if out_ptr.is_null() { + panic!("transpose_case_cmd: null out_ptr"); + } + *out_ptr = case.cmd; + }); +} + +#[no_mangle] +pub extern "C" fn transpose_case_src_word_lo(word_index: u32) -> u64 { + if word_index as usize >= MAX_WORDS { + panic!("transpose_case_src_word_lo: word_index out of range"); + } + current_case(|case| case.src_word_lo(word_index as usize)) +} + +#[no_mangle] +pub extern "C" fn transpose_case_src_word_hi(word_index: u32) -> u64 { + if word_index as usize >= MAX_WORDS { + panic!("transpose_case_src_word_hi: word_index out of range"); + } + current_case(|case| case.src_word_hi(word_index as usize)) +} + +#[no_mangle] +pub extern "C" fn transpose_case_dst_word_lo(word_index: u32) -> u64 { + if word_index as usize >= MAX_WORDS { + panic!("transpose_case_dst_word_lo: word_index out of range"); + } + current_case(|case| case.dst_word_lo(word_index as usize)) +} + +#[no_mangle] +pub extern "C" fn transpose_case_dst_word_hi(word_index: u32) -> u64 { + if word_index as usize >= MAX_WORDS { + panic!("transpose_case_dst_word_hi: word_index out of range"); + } + current_case(|case| case.dst_word_hi(word_index as usize)) +} diff --git a/examples/balls/transpose/verify/casegen/src/model.rs b/examples/components/balls/transpose/verify/casegen/src/model.rs similarity index 100% rename from examples/balls/transpose/verify/casegen/src/model.rs rename to examples/components/balls/transpose/verify/casegen/src/model.rs diff --git a/examples/balls/transpose/verify/filelists/transpose_ball.f b/examples/components/balls/transpose/verify/filelists/transpose_ball.f similarity index 100% rename from examples/balls/transpose/verify/filelists/transpose_ball.f rename to examples/components/balls/transpose/verify/filelists/transpose_ball.f diff --git a/examples/balls/transpose/verify/src/common/transpose_defs.svh b/examples/components/balls/transpose/verify/src/common/transpose_defs.svh similarity index 90% rename from examples/balls/transpose/verify/src/common/transpose_defs.svh rename to examples/components/balls/transpose/verify/src/common/transpose_defs.svh index 5b2313d78..efc2543f5 100644 --- a/examples/balls/transpose/verify/src/common/transpose_defs.svh +++ b/examples/components/balls/transpose/verify/src/common/transpose_defs.svh @@ -23,9 +23,9 @@ import "DPI-C" function longint unsigned transpose_case_dst_word_lo(input int un import "DPI-C" function longint unsigned transpose_case_dst_word_hi(input int unsigned word_index); `ifndef TRANSPOSE_FUNCT7 -`error "TRANSPOSE_FUNCT7 must be provided by the selected Core ballISA" +`error "TRANSPOSE_FUNCT7 must be provided by the selected Thread ballISA" `endif -localparam int TRANSPOSE_CORE_FUNCT7 = `TRANSPOSE_FUNCT7; +localparam int TRANSPOSE_THREAD_FUNCT7 = `TRANSPOSE_FUNCT7; localparam int TRANSPOSE_MAX_WORDS = 16; localparam int TRANSPOSE_TIMEOUT_CYCLES = 2000; localparam int TRANSPOSE_SEED = 32'hCAFE_BABE; diff --git a/examples/balls/transpose/verify/src/common/transpose_items.svh b/examples/components/balls/transpose/verify/src/common/transpose_items.svh similarity index 97% rename from examples/balls/transpose/verify/src/common/transpose_items.svh rename to examples/components/balls/transpose/verify/src/common/transpose_items.svh index 19f26044e..cfde7017c 100644 --- a/examples/balls/transpose/verify/src/common/transpose_items.svh +++ b/examples/components/balls/transpose/verify/src/common/transpose_items.svh @@ -8,7 +8,7 @@ class transpose_cmd_item extends bb_blink_cmd_item; bit [127:0] dst_words[TRANSPOSE_MAX_WORDS]; constraint legal_c { - funct7 == TRANSPOSE_CORE_FUNCT7[6:0]; + funct7 == TRANSPOSE_THREAD_FUNCT7[6:0]; op1_en == 1'b1; op2_en == 1'b0; wr_spad_en == 1'b1; @@ -41,7 +41,7 @@ class transpose_cmd_item extends bb_blink_cmd_item; transpose_case_cmd(cmd); bid = cmd.bid[4:0]; - funct7 = TRANSPOSE_CORE_FUNCT7[6:0]; + funct7 = TRANSPOSE_THREAD_FUNCT7[6:0]; iter = cmd.iter; op1_bank = cmd.op1_bank[4:0]; wr_bank = cmd.wr_bank[4:0]; diff --git a/examples/balls/transpose/verify/src/cov/transpose_cov.svh b/examples/components/balls/transpose/verify/src/cov/transpose_cov.svh similarity index 96% rename from examples/balls/transpose/verify/src/cov/transpose_cov.svh rename to examples/components/balls/transpose/verify/src/cov/transpose_cov.svh index 7913440ae..1d414dcb7 100644 --- a/examples/balls/transpose/verify/src/cov/transpose_cov.svh +++ b/examples/components/balls/transpose/verify/src/cov/transpose_cov.svh @@ -28,6 +28,7 @@ class transpose_cov extends uvm_component; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") if (cmd_cg.get_coverage() != 100.0) begin diff --git a/examples/balls/transpose/verify/src/env/transpose_env.svh b/examples/components/balls/transpose/verify/src/env/transpose_env.svh similarity index 100% rename from examples/balls/transpose/verify/src/env/transpose_env.svh rename to examples/components/balls/transpose/verify/src/env/transpose_env.svh diff --git a/examples/balls/transpose/verify/src/env/transpose_scoreboard.svh b/examples/components/balls/transpose/verify/src/env/transpose_scoreboard.svh similarity index 98% rename from examples/balls/transpose/verify/src/env/transpose_scoreboard.svh rename to examples/components/balls/transpose/verify/src/env/transpose_scoreboard.svh index a4da6eacb..552e478e4 100644 --- a/examples/balls/transpose/verify/src/env/transpose_scoreboard.svh +++ b/examples/components/balls/transpose/verify/src/env/transpose_scoreboard.svh @@ -67,6 +67,7 @@ class transpose_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, transpose_cmd_item exp); + verification_pkg::evidence::checks++; if (got.bid !== exp.bid) `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) if (got.funct7 !== exp.funct7) @@ -155,6 +156,7 @@ class transpose_scoreboard extends uvm_scoreboard; endfunction function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) begin `uvm_fatal("SCB", $sformatf("incomplete: cmds=%0d reads=%0d writes=%0d/%0d resp=%0d", diff --git a/examples/balls/transpose/verify/src/pkg/transpose_pkg.sv b/examples/components/balls/transpose/verify/src/pkg/transpose_pkg.sv similarity index 100% rename from examples/balls/transpose/verify/src/pkg/transpose_pkg.sv rename to examples/components/balls/transpose/verify/src/pkg/transpose_pkg.sv diff --git a/examples/balls/transpose/verify/src/seq/transpose_sequences.svh b/examples/components/balls/transpose/verify/src/seq/transpose_sequences.svh similarity index 100% rename from examples/balls/transpose/verify/src/seq/transpose_sequences.svh rename to examples/components/balls/transpose/verify/src/seq/transpose_sequences.svh diff --git a/examples/components/balls/transpose/verify/src/tb_top.sv b/examples/components/balls/transpose/verify/src/tb_top.sv new file mode 100644 index 000000000..c42fbafb0 --- /dev/null +++ b/examples/components/balls/transpose/verify/src/tb_top.sv @@ -0,0 +1,90 @@ +module tb_top; + import uvm_pkg::*; + import transpose_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + TransposeBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .io_cmdReq_ready(intf.cmd_req_ready), + .io_cmdReq_valid(intf.cmd_req_valid), + .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), + .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), + .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), + .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), + .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), + .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), + .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), + .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), + .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), + .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), + .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), + .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), + .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), + .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), + .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), + .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), + .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), + .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), + .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), + .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), + .io_cmdResp_ready(intf.cmd_resp_ready), + .io_cmdResp_valid(intf.cmd_resp_valid), + .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), + .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), + .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .io_bankRead_0_group_id(intf.bank_read_group_id[0]), + .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .io_subRobReq_ready(intf.sub_rob_req_ready), + .io_status_idle(), + .io_status_running(), + .io_bankAccessReady(1'b1), + .io_bankRead_0_ball_id(), + .io_bankWrite_0_ball_id() + ); + + initial begin + intf.clock = 1'b0; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("transpose_ball_test"); + end +endmodule diff --git a/examples/balls/transpose/verify/src/tests/transpose_test.svh b/examples/components/balls/transpose/verify/src/tests/transpose_test.svh similarity index 100% rename from examples/balls/transpose/verify/src/tests/transpose_test.svh rename to examples/components/balls/transpose/verify/src/tests/transpose_test.svh diff --git a/examples/components/balls/transpose/workloads/ctests/CMakeLists.txt b/examples/components/balls/transpose/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..acf4f2cba --- /dev/null +++ b/examples/components/balls/transpose/workloads/ctests/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_ctest(transpose_i8_16x16_test.c) +add_buckyball_ctest(transpose_i8_16x32_test.c) +add_buckyball_ctest(transpose_i32_8x16_test.c) diff --git a/examples/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c b/examples/components/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c similarity index 97% rename from examples/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c rename to examples/components/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c index e9c213e91..6b28564cb 100644 --- a/examples/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c +++ b/examples/components/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c @@ -27,8 +27,8 @@ static int32_t output_matrix[COLS * ROWS] __attribute__((aligned(64))); static int32_t expected_matrix[COLS * ROWS] __attribute__((aligned(64))); int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif for (int r = 0; r < ROWS; ++r) { diff --git a/examples/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c b/examples/components/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c similarity index 98% rename from examples/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c rename to examples/components/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c index ca16efb6b..55d6cf435 100644 --- a/examples/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c +++ b/examples/components/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c @@ -38,8 +38,8 @@ static elem_t output_matrix[COLS * ROWS] __attribute__((aligned(64))); static elem_t expected_matrix[COLS * ROWS] __attribute__((aligned(64))); int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif transpose_u8_matrix(input_matrix, expected_matrix, ROWS, COLS); diff --git a/examples/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c b/examples/components/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c similarity index 98% rename from examples/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c rename to examples/components/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c index b0dc629be..4aa21aab2 100644 --- a/examples/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c +++ b/examples/components/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c @@ -43,8 +43,8 @@ static elem_t output_matrix[COLS * ROWS] __attribute__((aligned(64))); static elem_t expected_matrix[COLS * ROWS] __attribute__((aligned(64))); int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif transpose_u8_matrix(input_matrix, expected_matrix, ROWS, COLS); diff --git a/examples/balls/transpose/workloads/isa/transpose.h b/examples/components/balls/transpose/workloads/isa/transpose.h similarity index 100% rename from examples/balls/transpose/workloads/isa/transpose.h rename to examples/components/balls/transpose/workloads/isa/transpose.h diff --git a/examples/balls/transpose/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/transpose/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/transpose/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/transpose/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/transpose/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/transpose/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/transpose/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/transpose/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir b/examples/components/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir new file mode 100644 index 000000000..2040907f4 --- /dev/null +++ b/examples/components/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir @@ -0,0 +1,38 @@ +// Physical-bank transpose_i8 of a 16x16 i8 matrix. + +func.func private @check_result(memref<16x16xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %n = arith.constant 16 : index + %c16_i32 = arith.constant 16 : i32 + + %input = memref.alloc() alignment = 64 : memref<16x16xi8> + %output = memref.alloc() alignment = 64 : memref<16x16xi8> + + scf.for %i = %c0 to %n step %c1 { + scf.for %j = %c0 to %n step %c1 { + %ii = arith.index_cast %i : index to i32 + %jj = arith.index_cast %j : index to i32 + %mul = arith.muli %ii, %c16_i32 : i32 + %val = arith.addi %mul, %jj : i32 + %v8 = arith.trunci %val : i32 to i8 + memref.store %v8, %input[%i, %j] : memref<16x16xi8> + } + } + + buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.custom "transpose_i8" <{sources = array, destination = 1 : i64}> {rows = 16 : i64, cols = 16 : i64} + buckyball.mvout %output <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + + func.call @check_result(%output) : (memref<16x16xi8>) -> () + memref.dealloc %input : memref<16x16xi8> + memref.dealloc %output : memref<16x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8_main.cpp b/examples/components/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8_main.cpp similarity index 100% rename from examples/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8_main.cpp rename to examples/components/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8_main.cpp diff --git a/examples/balls/transpose/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/transpose/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/transpose/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/components/balls/transpose/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in b/examples/components/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in new file mode 100644 index 000000000..fdeab0431 --- /dev/null +++ b/examples/components/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in @@ -0,0 +1,32 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %rows = arith.constant @BB_BANK_LINES@ : index + %lanes = arith.constant @BB_BANK_I8_LANES@ : index + %lanes_i32 = arith.constant @BB_BANK_I8_LANES@ : i32 + %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + scf.for %row = %zero to %rows step %one { + scf.for %lane = %zero to %lanes step %one { + %r = arith.index_cast %row : index to i32 + %l = arith.index_cast %lane : index to i32 + %base = arith.muli %r, %lanes_i32 : i32 + %index = arith.addi %base, %l : i32 + %value = arith.trunci %index : i32 to i8 + memref.store %value, %input[%row, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + } + } + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + %result = buckyball.bank_custom "transpose_i8" (%loaded) -> %output_bank {rows = @BB_BANK_LINES@ : i64, cols = @BB_BANK_I8_LANES@ : i64} + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + buckyball.bank_release %loaded + buckyball.bank_release %stored + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () + memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> + return %zero_i8 : i8 +} diff --git a/examples/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp b/examples/components/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp similarity index 100% rename from examples/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp rename to examples/components/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp diff --git a/examples/balls/vector/arch/src/main/scala/README.md b/examples/components/balls/vector/arch/README.md similarity index 100% rename from examples/balls/vector/arch/src/main/scala/README.md rename to examples/components/balls/vector/arch/README.md diff --git a/examples/components/balls/vector/arch/VecBall.scala b/examples/components/balls/vector/arch/VecBall.scala new file mode 100644 index 000000000..fd7c2af59 --- /dev/null +++ b/examples/components/balls/vector/arch/VecBall.scala @@ -0,0 +1,70 @@ +package examples.balls.vector + +import framework.acc.balldomain.bbus.{BallFactory, BallPorts} +import buckyball.balls.vector.{Params => ParametersProto} +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.vector.configs.VectorBallParam + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.balldomain.ball.blink.{BallStatus, BlinkIO, HasBlink} +import examples.balls.vector.VecUnit + +/** + * VecBall + */ +@instantiable +class VecBall( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int) + extends Module + with HasBlink { + val ball = bp.balls.find(_.id == id).get + + val ballParam = VectorBallParam.fromProto(ball.parameters.unpack(classOf[ParametersProto])) + + val inBW = ball.inBW + val outBW = ball.outBW + + @public + val blink = IO(new BlinkIO(fp, mp, bp, id)) + + val vecUnit: Instance[VecUnit] = + Instantiate(new VecUnit(fp, mp, bp, ball, ballParam)) + + vecUnit.io.cmdReq <> blink.cmdReq + when(blink.cmdReq.fire) { + val cmd = blink.cmdReq.bits + assert(cmd.op1_group === 0.U && cmd.op2_group === 0.U && cmd.wr_group === 0.U, "VecBall has no group offsets") + } + vecUnit.io.cmdResp <> blink.cmdResp + + for (i <- 0 until inBW) { + vecUnit.io.bankRead(i) <> blink.bankRead(i) + } + + for (i <- 0 until outBW) { + vecUnit.io.bankWrite(i) <> blink.bankWrite(i) + } + + blink.status <> vecUnit.io.status + +} + +object VecBall extends BallFactory { + + def instantiate( + fp: FrontendParam, + mp: MemDomainParam, + bp: BallDomainParam, + id: Int + ): BallPorts = + BallPorts(Instantiate(new VecBall(fp, mp, bp, id)).blink, None) + +} diff --git a/examples/components/balls/vector/arch/VecCtrlUnit.scala b/examples/components/balls/vector/arch/VecCtrlUnit.scala new file mode 100644 index 000000000..fe28b7ed0 --- /dev/null +++ b/examples/components/balls/vector/arch/VecCtrlUnit.scala @@ -0,0 +1,131 @@ +package examples.balls.vector + +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.vector.configs.VectorBallParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.stage._ +import chisel3.experimental.hierarchy.{instantiable, public} + +@instantiable +class VecCtrlUnit(val frontendParam: FrontendParam, val memParam: MemDomainParam, val ballParam: VectorBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp_o = Decoupled(new BallResponse(frontendParam)) + + val ctrl_ld_o = Decoupled(new ctrl_ld_req(frontendParam, memParam, ballParam)) + val ctrl_st_o = Decoupled(new ctrl_st_req(frontendParam, memParam, ballParam)) + val ctrl_ex_o = Decoupled(new ctrl_ex_req(frontendParam, ballParam)) + + val cmdResp_i = Flipped(Valid(new Bundle { + val commit = Bool() + })) // from store unit + }) + + val rob_id_reg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + val iter = RegInit(0.U(frontendParam.iter_len.W)) + val op1_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + + val op1_bank_addr = RegInit( + 0.U(12.W) + ) // New ISA: always 0, but keep for compatibility + val op2_bank_addr = RegInit( + 0.U(12.W) + ) // New ISA: always 0, but keep for compatibility + val op2_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val wr_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + + val wr_bank_addr = RegInit( + 0.U(12.W) + ) // New ISA: always 0, but keep for compatibility + val has_send = RegInit(false.B) + val mode = RegInit(0.U(1.W)) + + val idle :: busy :: Nil = Enum(2) + val state = RegInit(idle) + +// ----------------------------------------------------------------------------- +// Set registers when EX instruction arrives +// ----------------------------------------------------------------------------- + + when(io.cmdReq.fire) { + iter := io.cmdReq.bits.iter + rob_id_reg := io.cmdReq.bits.rob_id + op1_bank := io.cmdReq.bits.op1_bank + op1_bank_addr := 0.U // New ISA: all operations start from row 0 + op2_bank := io.cmdReq.bits.op2_bank + op2_bank_addr := 0.U // New ISA: all operations start from row 0 + wr_bank := io.cmdReq.bits.wr_bank + wr_bank_addr := 0.U // New ISA: all operations start from row 0 + mode := io.cmdReq.bits.rs2(0) + + state := busy + } + +// ----------------------------------------------------------------------------- +// Send control signals to VecUnit's load/store/ex units +// ----------------------------------------------------------------------------- + + when(state === busy && !has_send) { + io.ctrl_ld_o.valid := true.B + io.ctrl_ld_o.bits.op1_bank := op1_bank + io.ctrl_ld_o.bits.op1_bank_addr := op1_bank_addr + io.ctrl_ld_o.bits.op2_bank := op2_bank + io.ctrl_ld_o.bits.op2_bank_addr := op2_bank_addr + io.ctrl_ld_o.bits.iter := iter + io.ctrl_ld_o.bits.mode := mode + + io.ctrl_ex_o.valid := true.B + io.ctrl_ex_o.bits.iter := iter + + io.ctrl_st_o.valid := true.B + io.ctrl_st_o.bits.wr_bank := wr_bank + io.ctrl_st_o.bits.wr_bank_addr := wr_bank_addr + io.ctrl_st_o.bits.iter := iter + + has_send := true.B + }.otherwise { + io.ctrl_ld_o.valid := false.B + io.ctrl_ld_o.bits.op1_bank := 0.U + io.ctrl_ld_o.bits.op1_bank_addr := 0.U + io.ctrl_ld_o.bits.op2_bank := 0.U + io.ctrl_ld_o.bits.op2_bank_addr := 0.U + io.ctrl_ld_o.bits.iter := 0.U + io.ctrl_ld_o.bits.mode := 0.U + + io.ctrl_ex_o.valid := false.B + io.ctrl_ex_o.bits.iter := 0.U + + io.ctrl_st_o.valid := false.B + io.ctrl_st_o.bits.wr_bank := 0.U + io.ctrl_st_o.bits.wr_bank_addr := 0.U + io.ctrl_st_o.bits.iter := 0.U + } + +// ----------------------------------------------------------------------------- +// Wait for VecUnit's final write-back to complete +// ----------------------------------------------------------------------------- + + when(io.cmdResp_i.valid) { + io.cmdResp_o.valid := true.B + io.cmdResp_o.bits.rob_id := rob_id_reg + state := idle + has_send := false.B + }.otherwise { + io.cmdResp_o.valid := false.B + io.cmdResp_o.bits.rob_id := 0.U + } + + io.cmdReq.ready := state === idle + +} diff --git a/examples/balls/vector/arch/src/main/scala/VecEXUnit.scala b/examples/components/balls/vector/arch/VecEXUnit.scala similarity index 76% rename from examples/balls/vector/arch/src/main/scala/VecEXUnit.scala rename to examples/components/balls/vector/arch/VecEXUnit.scala index 484a3f940..f9eead981 100644 --- a/examples/balls/vector/arch/src/main/scala/VecEXUnit.scala +++ b/examples/components/balls/vector/arch/VecEXUnit.scala @@ -1,43 +1,46 @@ package examples.balls.vector +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.frontend.configs.FrontendParam + import chisel3._ import chisel3.util._ import chisel3.stage._ import chisel3.experimental.hierarchy.{instantiable, public} -import framework.top.GlobalConfig import examples.balls.vector.warp.MeshWarp import examples.balls.vector.configs.VectorBallParam -class ctrl_ex_req(b: GlobalConfig) extends Bundle { - val iter = UInt(b.frontend.iter_len.W) +class ctrl_ex_req(frontendParam: FrontendParam, val ballParam: VectorBallParam) extends Bundle { + val iter = UInt(frontendParam.iter_len.W) } -class ld_ex_req(b: GlobalConfig) extends Bundle { - val config = VectorBallParam(b) +class ld_ex_req(frontendParam: FrontendParam, val ballParam: VectorBallParam) extends Bundle { + val config = ballParam val op1 = Vec(config.lane, UInt(config.inputWidth.W)) val op2 = Vec(config.lane, UInt(config.inputWidth.W)) - val iter = UInt(b.frontend.iter_len.W) + val iter = UInt(frontendParam.iter_len.W) } @instantiable -class VecEXUnit(val b: GlobalConfig) extends Module { - val config = VectorBallParam(b) +class VecEXUnit(val frontendParam: FrontendParam, val ballParam: VectorBallParam) extends Module { + val config = ballParam val InputNum = config.lane val inputWidth = config.inputWidth val accWidth = config.outputWidth @public val io = IO(new Bundle { - val ctrl_ex_i = Flipped(Decoupled(new ctrl_ex_req(b))) - val ld_ex_i = Flipped(Decoupled(new ld_ex_req(b))) + val ctrl_ex_i = Flipped(Decoupled(new ctrl_ex_req(frontendParam, ballParam))) + val ld_ex_i = Flipped(Decoupled(new ld_ex_req(frontendParam, ballParam))) - val ex_st_o = Decoupled(new ex_st_req(b)) + val ex_st_o = Decoupled(new ex_st_req(frontendParam, ballParam)) }) val idle :: busy :: Nil = Enum(2) val state = RegInit(idle) - val meshWarp = Module(new MeshWarp(config)) + val meshWarp = Instantiate(new MeshWarp(ballParam)) // Thread ID for MeshWarp (always use thread 0 for now) val threadId = RegInit(0.U(10.W)) diff --git a/examples/components/balls/vector/arch/VecLoadUnit.scala b/examples/components/balls/vector/arch/VecLoadUnit.scala new file mode 100644 index 000000000..dc03a3a66 --- /dev/null +++ b/examples/components/balls/vector/arch/VecLoadUnit.scala @@ -0,0 +1,203 @@ +package examples.balls.vector + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.lib.queue.Queue +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.ip.memory.bank.{SramReadIO, SramReadReq, SramReadResp, SramWriteIO, SramWriteReq, SramWriteResp} + +import chisel3._ +import chisel3.util._ +import chisel3.stage._ +import chisel3.experimental.hierarchy.{instantiable, public} +import framework.ip.memory.bank.{SramReadReq, SramReadResp} +import examples.balls.vector.configs.VectorBallParam + +class ctrl_ld_req(frontendParam: FrontendParam, memParam: MemDomainParam, val ballParam: VectorBallParam) + extends Bundle { + val op1_bank = UInt(frontendParam.bank_id_len.W) + val op1_bank_addr = UInt(log2Up(memParam.bankEntries).W) + val op2_bank = UInt(frontendParam.bank_id_len.W) + val op2_bank_addr = UInt(log2Up(memParam.bankEntries).W) + val iter = UInt(frontendParam.iter_len.W) + val mode = UInt(1.W) +} + +@instantiable +class VecLoadUnit( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: VectorBallParam) + extends Module { + val config = ballParam + val InputNum = config.lane + val inputWidth = config.inputWidth + val bankWidth = memParam.bankWidth + val rob_id_width = log2Up(frontendParam.rob_entries) + + // Use this instance's configured bandwidth. + val ballMapping = instance + + val inBW = ballMapping.inBW + + @public + val io = IO(new Bundle { + val bankReadReq = Vec(inBW, Decoupled(new SramReadReq(memParam))) + val bankReadResp = + Vec(inBW, Flipped(Decoupled(new SramReadResp(memParam)))) + val ctrl_ld_i = Flipped(Decoupled(new ctrl_ld_req(frontendParam, memParam, ballParam))) + val ld_ex_o = Decoupled(new ld_ex_req(frontendParam, ballParam)) + val op1_bank_o = Output(UInt(frontendParam.bank_id_len.W)) + val op2_bank_o = Output(UInt(frontendParam.bank_id_len.W)) + }) + + val op1_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val op2_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val op1_addr = RegInit(0.U(log2Up(memParam.bankEntries).W)) + val op2_addr = RegInit(0.U(log2Up(memParam.bankEntries).W)) + val iter = RegInit(0.U(frontendParam.iter_len.W)) + val op1_iter_counter = RegInit(0.U(frontendParam.iter_len.W)) + val op2_iter_counter = RegInit(0.U(frontendParam.iter_len.W)) + val idle :: busy :: Nil = Enum(2) + val state = RegInit(idle) + val ld_ex_op1_reg = Reg(Vec(InputNum, UInt(inputWidth.W))) + val ld_ex_op2_reg = Reg(Vec(InputNum, UInt(inputWidth.W))) + val ld_ex_iter_reg = RegInit(0.U(frontendParam.iter_len.W)) + val wait1_reg = RegInit(false.B) + val wait2_reg = RegInit(false.B) + val wait1_cnt = RegInit(0.U(7.W)) + val wait2_cnt = RegInit(0.U(7.W)) + + val bankRespQueue0 = Instantiate(new Queue( + new SramReadResp(memParam), + 8, + false, + false + )) + + val bankRespQueue1 = Instantiate(new Queue( + new SramReadResp(memParam), + 8, + false, + false + )) + + for (i <- 0 until inBW) { + io.bankReadReq(i).valid := false.B + io.bankReadReq(i).bits.addr := 0.U + } + + io.op1_bank_o := op1_bank + io.op2_bank_o := op2_bank + io.ctrl_ld_i.ready := state === idle + + bankRespQueue0.io.enq <> io.bankReadResp(0) + bankRespQueue1.io.enq <> io.bankReadResp(1) + +// ----------------------------------------------------------------------------- +// Set registers when Ctrl instruction arrives +// ----------------------------------------------------------------------------- + when(io.ctrl_ld_i.fire) { + op1_bank := io.ctrl_ld_i.bits.op1_bank + op2_bank := io.ctrl_ld_i.bits.op2_bank + op1_addr := io.ctrl_ld_i.bits.op1_bank_addr + op2_addr := io.ctrl_ld_i.bits.op2_bank_addr + iter := io.ctrl_ld_i.bits.iter + op1_iter_counter := 0.U + op2_iter_counter := 0.U + state := busy + assert(io.ctrl_ld_i.bits.iter > 0.U, "iter should be greater than 0") + } + +// ----------------------------------------------------------------------------- +// Send SRAM read request +// ----------------------------------------------------------------------------- + //wait1_reg := Mux(io.run , 0.U, wait1_reg) + + //wait2_reg := Mux(io.run , 0.U, wait2_reg) + + when(state === busy && io.ld_ex_o.ready && !wait1_reg) { + io.bankReadReq(0).valid := op1_iter_counter < iter + io.bankReadReq(0).bits.addr := op1_addr + op1_iter_counter + op1_iter_counter := Mux( + io.bankReadReq(0).ready, + op1_iter_counter + 1.U, + op1_iter_counter + ) + wait1_reg := Mux((op1_iter_counter + 1.U) % 16.U === 0.U, 1.U, 0.U) + when(io.bankReadReq(0).fire) {} + } + + when(state === busy && io.ld_ex_o.ready && !wait2_reg) { + io.bankReadReq(1).valid := op2_iter_counter < iter + io.bankReadReq(1).bits.addr := op2_addr + op2_iter_counter + op2_iter_counter := Mux( + io.bankReadReq(1).ready, + op2_iter_counter + 1.U, + op2_iter_counter + ) + wait2_reg := Mux((op2_iter_counter + 1.U) % 16.U === 0.U, 1.U, 0.U) + when(io.bankReadReq(1).fire) {} + } + + when(wait1_reg) { + wait1_cnt := wait1_cnt + 1.U + when(wait1_cnt === 32.U) { + wait1_reg := false.B + wait1_cnt := 0.U + } + } + + when(wait2_reg) { + wait2_cnt := wait2_cnt + 1.U + when(wait2_cnt === 32.U) { + wait2_reg := false.B + wait2_cnt := 0.U + } + } + +// ----------------------------------------------------------------------------- +// SRAM returns data and passes to EX unit +// ----------------------------------------------------------------------------- + val both_valid = bankRespQueue0.io.deq.valid && bankRespQueue1.io.deq.valid + + io.ld_ex_o.valid := both_valid + when(both_valid) { + io.ld_ex_o.bits.op1 := bankRespQueue0.io.deq.bits.data + .asTypeOf(Vec(InputNum, UInt(inputWidth.W))) + io.ld_ex_o.bits.op2 := bankRespQueue1.io.deq.bits.data + .asTypeOf(Vec(InputNum, UInt(inputWidth.W))) + io.ld_ex_o.bits.iter := ld_ex_iter_reg + }.otherwise { + io.ld_ex_o.bits.iter := 0.U + io.ld_ex_o.bits.op1 := VecInit(Seq.fill(InputNum)(0.U(inputWidth.W))) + io.ld_ex_o.bits.op2 := VecInit(Seq.fill(InputNum)(0.U(inputWidth.W))) + } + + // Only dequeue and advance iter counter on successful handshake + bankRespQueue0.io.deq.ready := io.ld_ex_o.fire + bankRespQueue1.io.deq.ready := io.ld_ex_o.fire + + when(io.ld_ex_o.fire) { + ld_ex_iter_reg := ld_ex_iter_reg + 1.U + } + +// ----------------------------------------------------------------------------- +// Reset op1_iter_counter and return to idle state +// ----------------------------------------------------------------------------- + + when(state === busy && ld_ex_iter_reg === iter) { + state := idle + op1_iter_counter := 0.U + op2_iter_counter := 0.U + ld_ex_iter_reg := 0.U + } + +} diff --git a/examples/components/balls/vector/arch/VecMat16.scala b/examples/components/balls/vector/arch/VecMat16.scala new file mode 100644 index 000000000..490a6e713 --- /dev/null +++ b/examples/components/balls/vector/arch/VecMat16.scala @@ -0,0 +1,222 @@ +package examples.balls.vector + +import chisel3.experimental.hierarchy.public + +import chisel3.experimental.hierarchy.instantiable + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.vector.configs.VectorBallParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} + +@instantiable +class VecMat16( + frontendParam: FrontendParam, + memParam: MemDomainParam, + routingParam: BallDomainParam, + val inBW: Int, + val outBW: Int, + val inW: Int, + val accW: Int, + lane: Int, + val ballParam: VectorBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + + require(lane == 16, "VecMat16 requires lane=16") + require(inW == 8, "VecMat16 requires int8 inputs") + require(accW == 32, "VecMat16 requires int32 accumulators") + require(inBW >= 2, "VecMat16 requires two read ports") + require(outBW >= 4, "VecMat16 requires four write ports") + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val sIdle :: sLoadAcc :: sReadReq :: sReadResp :: sWrite :: sDone :: Nil = + Enum(6) + val state = RegInit(sIdle) + + val robIdReg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + val op1Bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val op2Bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val wrBank = RegInit(0.U(frontendParam.bank_id_len.W)) + val iterReg = RegInit(0.U(frontendParam.iter_len.W)) + val readRow = RegInit(0.U(frontendParam.iter_len.W)) + val aReqDone = RegInit(false.B) + val bReqDone = RegInit(false.B) + val aRespDone = RegInit(false.B) + val bRespDone = RegInit(false.B) + val aWord = RegInit(0.U(memParam.bankWidth.W)) + val bWord = RegInit(0.U(memParam.bankWidth.W)) + val acc = Reg(Vec(lane, Vec(lane, UInt(accW.W)))) + + val accLoad = Instantiate(new VecMat16AccLoad(frontendParam, memParam, routingParam, accW, lane, ballParam)) + + val store = Instantiate(new VecMat16Store(frontendParam, memParam, routingParam, accW, lane, ballParam)) + + accLoad.io.start := false.B + accLoad.io.wrBank := wrBank + accLoad.io.robId := robIdReg + store.io.start := false.B + store.io.wrBank := wrBank + store.io.robId := robIdReg + store.io.acc := acc + for (p <- 0 until 2) { + accLoad.io.bankRead(p).io.req.ready := io.bankRead(p).io.req.ready + accLoad.io.bankRead(p).io.resp.valid := io.bankRead(p).io.resp.valid + accLoad.io.bankRead(p).io.resp.bits := io.bankRead(p).io.resp.bits + } + for (g <- 0 until 4) { + store.io.bankWrite(g).io.req.ready := io.bankWrite(g).io.req.ready + store.io.bankWrite(g).io.resp.valid := io.bankWrite(g).io.resp.valid + store.io.bankWrite(g).io.resp.bits := io.bankWrite(g).io.resp.bits + } + + io.cmdReq.ready := state === sIdle + io.cmdResp.valid := state === sDone + io.cmdResp.bits.rob_id := robIdReg + io.status.idle := state === sIdle + io.status.running := state =/= sIdle && state =/= sDone + VecMat16Tie.offRead(io.bankRead, robIdReg) + VecMat16Tie.offWrite(io.bankWrite, robIdReg, wrBank, memParam) + + switch(state) { + is(sIdle) { + when(io.cmdReq.fire) { + val cmd = io.cmdReq.bits + robIdReg := io.cmdReq.bits.rob_id + op1Bank := cmd.op1_bank + op2Bank := cmd.op2_bank + wrBank := cmd.wr_bank + iterReg := cmd.iter + readRow := 0.U + aReqDone := false.B + bReqDone := false.B + aRespDone := false.B + bRespDone := false.B + assert(cmd.iter > 0.U, "VecMat16 iter must be > 0") + assert( + cmd.op1_col === 1.U && cmd.op2_col === 1.U && cmd.wr_col === 4.U, + "VecMat16 unsupported bank layout" + ) + accLoad.io.start := true.B + state := sLoadAcc + } + } + + is(sLoadAcc) { + for (p <- 0 until 2) { + VecMat16Tie.takeRead(io.bankRead(p), accLoad.io.bankRead(p)) + } + when(accLoad.io.done) { + acc := accLoad.io.acc + aReqDone := false.B + bReqDone := false.B + state := sReadReq + } + } + + is(sReadReq) { + io.bankRead(0).bank_id := op1Bank + io.bankRead(0).io.req.valid := !aReqDone + io.bankRead(0).io.req.bits.addr := readRow + io.bankRead(1).bank_id := op2Bank + io.bankRead(1).io.req.valid := !bReqDone + io.bankRead(1).io.req.bits.addr := readRow + when(io.bankRead(0).io.req.fire)(aReqDone := true.B) + when(io.bankRead(1).io.req.fire)(bReqDone := true.B) + when( + (aReqDone || io + .bankRead(0) + .io + .req + .fire) && (bReqDone || io.bankRead(1).io.req.fire) + ) { + aRespDone := false.B + bRespDone := false.B + state := sReadResp + } + } + + is(sReadResp) { + io.bankRead(0).bank_id := op1Bank + io.bankRead(1).bank_id := op2Bank + io.bankRead(0).io.resp.ready := !aRespDone + io.bankRead(1).io.resp.ready := !bRespDone + val aFire = io.bankRead(0).io.resp.fire + val bFire = io.bankRead(1).io.resp.fire + val nextA = Mux(aFire, io.bankRead(0).io.resp.bits.data, aWord) + val nextB = Mux(bFire, io.bankRead(1).io.resp.bits.data, bWord) + val haveA = aRespDone || aFire + val haveB = bRespDone || bFire + when(aFire)(aWord := io.bankRead(0).io.resp.bits.data) + when(bFire)(bWord := io.bankRead(1).io.resp.bits.data) + when(haveA)(aRespDone := true.B) + when(haveB)(bRespDone := true.B) + when(haveA && haveB) { + val aElems = nextA.asTypeOf(Vec(lane, UInt(inW.W))) + val bElems = nextB.asTypeOf(Vec(lane, UInt(inW.W))) + for { + i <- 0 until lane + j <- 0 until lane + } { + acc(i)(j) := (acc(i)(j).asSInt + aElems(i).asSInt * bElems( + j + ).asSInt).asUInt + } + when(readRow === iterReg - 1.U) { + store.io.start := true.B + state := sWrite + }.otherwise { + readRow := readRow + 1.U + aReqDone := false.B + bReqDone := false.B + state := sReadReq + } + } + } + + is(sWrite) { + for (g <- 0 until 4) { + VecMat16Tie.takeWrite(io.bankWrite(g), store.io.bankWrite(g)) + } + when(store.io.done)(state := sDone) + } + + is(sDone) { + when(io.cmdResp.fire)(state := sIdle) + } + } +} diff --git a/examples/balls/vector/arch/src/main/scala/VecMat16AccLoad.scala b/examples/components/balls/vector/arch/VecMat16AccLoad.scala similarity index 75% rename from examples/balls/vector/arch/src/main/scala/VecMat16AccLoad.scala rename to examples/components/balls/vector/arch/VecMat16AccLoad.scala index 16c8224d9..d2c148e09 100644 --- a/examples/balls/vector/arch/src/main/scala/VecMat16AccLoad.scala +++ b/examples/components/balls/vector/arch/VecMat16AccLoad.scala @@ -1,20 +1,44 @@ package examples.balls.vector +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.vector.configs.VectorBallParam + import chisel3._ import chisel3.util._ -import framework.balldomain.blink.BankRead -import framework.top.GlobalConfig +import framework.acc.memdomain.BankRead -class VecMat16AccLoad(val b: GlobalConfig, lane: Int, accWidth: Int) extends Module { +@instantiable +class VecMat16AccLoad( + frontendParam: FrontendParam, + memParam: MemDomainParam, + routingParam: BallDomainParam, + val accWidth: Int, + lane: Int, + val ballParam: VectorBallParam) + extends Module { + @public val io = IO(new Bundle { - val start = Input(Bool()) - val wrBank = Input(UInt(b.memDomain.vbankIdWidth.W)) - val robId = Input(UInt(log2Up(b.frontend.rob_entries).W)) - val bankRead = Vec(2, Flipped(new BankRead(b))) - val acc = Output(Vec(lane, Vec(lane, UInt(accWidth.W)))) - val done = Output(Bool()) - val busy = Output(Bool()) + val start = Input(Bool()) + val wrBank = Input(UInt(frontendParam.bank_id_len.W)) + val robId = Input(UInt(log2Up(frontendParam.rob_entries).W)) + + val bankRead = Vec( + 2, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val acc = Output(Vec(lane, Vec(lane, UInt(accWidth.W)))) + val done = Output(Bool()) + val busy = Output(Bool()) }) val sIdle :: sReq :: sResp :: sFinish :: Nil = Enum(4) @@ -35,7 +59,6 @@ class VecMat16AccLoad(val b: GlobalConfig, lane: Int, accWidth: Int) extends Mod for (i <- 0 until 2) { io.bankRead(i).rob_id := io.robId - io.bankRead(i).ball_id := 0.U io.bankRead(i).bank_id := 0.U io.bankRead(i).group_id := 0.U io.bankRead(i).io.req.valid := false.B diff --git a/examples/components/balls/vector/arch/VecMat16Store.scala b/examples/components/balls/vector/arch/VecMat16Store.scala new file mode 100644 index 000000000..f64010b60 --- /dev/null +++ b/examples/components/balls/vector/arch/VecMat16Store.scala @@ -0,0 +1,135 @@ +package examples.balls.vector + +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import examples.balls.vector.configs.VectorBallParam + +import chisel3._ +import chisel3.util._ +import framework.acc.memdomain.BankWrite + +@instantiable +class VecMat16Store( + frontendParam: FrontendParam, + memParam: MemDomainParam, + routingParam: BallDomainParam, + val accWidth: Int, + lane: Int, + val ballParam: VectorBallParam) + extends Module { + + @public + val io = IO(new Bundle { + val start = Input(Bool()) + val wrBank = Input(UInt(frontendParam.bank_id_len.W)) + val robId = Input(UInt(log2Up(frontendParam.rob_entries).W)) + val acc = Input(Vec(lane, Vec(lane, UInt(accWidth.W)))) + + val bankWrite = Vec( + 4, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val done = Output(Bool()) + val busy = Output(Bool()) + }) + + val sIdle :: sReq :: sResp :: Nil = Enum(3) + val state = RegInit(sIdle) + + val writeRow = RegInit(0.U(log2Ceil(lane + 1).W)) + val writeIssued = RegInit(VecInit(Seq.fill(4)(false.B))) + val writeAcked = RegInit(VecInit(Seq.fill(4)(false.B))) + + def writeData(group: Int): UInt = { + val row = writeRow(log2Ceil(lane) - 1, 0) + val elems = (0 until 4).map(i => io.acc(row)(group * 4 + i)) + Cat(elems.reverse) + } + + def clr4() = VecInit(Seq.fill(4)(false.B)) + + io.done := false.B + io.busy := state =/= sIdle + + for (i <- 0 until 4) { + io.bankWrite(i).rob_id := io.robId + io.bankWrite(i).bank_id := io.wrBank + io.bankWrite(i).group_id := i.U + io.bankWrite(i).io.req.valid := false.B + io.bankWrite(i).io.req.bits.addr := 0.U + io.bankWrite(i).io.req.bits.data := 0.U + io.bankWrite(i).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(false.B) + ) + io.bankWrite(i).io.resp.ready := false.B + } + + switch(state) { + is(sIdle) { + when(io.start) { + writeRow := 0.U + writeIssued := clr4() + writeAcked := clr4() + state := sReq + } + } + + is(sReq) { + for (g <- 0 until 4) { + io.bankWrite(g).bank_id := io.wrBank + io.bankWrite(g).group_id := g.U + io.bankWrite(g).io.req.valid := !writeIssued(g) + io.bankWrite(g).io.req.bits.addr := writeRow + io.bankWrite(g).io.req.bits.data := writeData(g) + io.bankWrite(g).io.req.bits.mask := VecInit( + Seq.fill(memParam.bankMaskLen)(true.B) + ) + when(io.bankWrite(g).io.req.fire) { + writeIssued(g) := true.B + } + } + when( + (0 until 4) + .map(g => writeIssued(g) || io.bankWrite(g).io.req.fire) + .reduce(_ && _) + ) { + writeAcked := clr4() + state := sResp + } + } + + is(sResp) { + for (g <- 0 until 4) { + io.bankWrite(g).bank_id := io.wrBank + io.bankWrite(g).group_id := g.U + io.bankWrite(g).io.resp.ready := !writeAcked(g) + when(io.bankWrite(g).io.resp.fire) { + writeAcked(g) := true.B + } + } + when( + (0 until 4) + .map(g => writeAcked(g) || io.bankWrite(g).io.resp.fire) + .reduce(_ && _) + ) { + when(writeRow === (lane - 1).U) { + io.done := true.B + state := sIdle + }.otherwise { + writeRow := writeRow + 1.U + writeIssued := clr4() + writeAcked := clr4() + state := sReq + } + } + } + } +} diff --git a/examples/balls/vector/arch/src/main/scala/VecMat16Tie.scala b/examples/components/balls/vector/arch/VecMat16Tie.scala similarity index 77% rename from examples/balls/vector/arch/src/main/scala/VecMat16Tie.scala rename to examples/components/balls/vector/arch/VecMat16Tie.scala index a4d0f4dae..430399f73 100644 --- a/examples/balls/vector/arch/src/main/scala/VecMat16Tie.scala +++ b/examples/components/balls/vector/arch/VecMat16Tie.scala @@ -1,16 +1,16 @@ package examples.balls.vector +import framework.acc.memdomain.configs.MemDomainParam + import chisel3._ import chisel3.util._ -import framework.balldomain.blink.{BankRead, BankWrite} -import framework.top.GlobalConfig +import framework.acc.memdomain.{BankRead, BankWrite} object VecMat16Tie { def offRead(ports: Vec[BankRead], robId: UInt): Unit = { for (p <- ports) { p.rob_id := robId - p.ball_id := 0.U p.bank_id := 0.U p.group_id := 0.U p.io.req.valid := false.B @@ -20,27 +20,25 @@ object VecMat16Tie { } def offWrite( - ports: Vec[BankWrite], - robId: UInt, - wrBank: UInt, - b: GlobalConfig + ports: Vec[BankWrite], + robId: UInt, + wrBank: UInt, + memParam: MemDomainParam ): Unit = { for ((p, i) <- ports.zipWithIndex) { p.rob_id := robId - p.ball_id := 0.U p.bank_id := wrBank p.group_id := i.U p.io.req.valid := false.B p.io.req.bits.addr := 0.U p.io.req.bits.data := 0.U - p.io.req.bits.mask := VecInit(Seq.fill(b.memDomain.bankMaskLen)(false.B)) + p.io.req.bits.mask := VecInit(Seq.fill(memParam.bankMaskLen)(false.B)) p.io.resp.ready := false.B } } def takeRead(parent: BankRead, child: BankRead): Unit = { parent.rob_id := child.rob_id - parent.ball_id := child.ball_id parent.bank_id := child.bank_id parent.group_id := child.group_id parent.io.req.valid := child.io.req.valid @@ -50,7 +48,6 @@ object VecMat16Tie { def takeWrite(parent: BankWrite, child: BankWrite): Unit = { parent.rob_id := child.rob_id - parent.ball_id := child.ball_id parent.bank_id := child.bank_id parent.group_id := child.group_id parent.io.req.valid := child.io.req.valid diff --git a/examples/components/balls/vector/arch/VecStoreUnit.scala b/examples/components/balls/vector/arch/VecStoreUnit.scala new file mode 100644 index 000000000..a7d14ed26 --- /dev/null +++ b/examples/components/balls/vector/arch/VecStoreUnit.scala @@ -0,0 +1,161 @@ +package examples.balls.vector + +import framework.acc.balldomain.configs.BallRouteParam + +import framework.lib.queue.Queue +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import framework.ip.memory.bank.{SramReadIO, SramReadReq, SramReadResp, SramWriteIO, SramWriteReq, SramWriteResp} + +import chisel3._ +import chisel3.util._ +import chisel3.stage._ +import chisel3.experimental.hierarchy.{instantiable, public} +import examples.balls.vector.configs.VectorBallParam + +class ctrl_st_req(frontendParam: FrontendParam, memParam: MemDomainParam, val ballParam: VectorBallParam) + extends Bundle { + val wr_bank = UInt(frontendParam.bank_id_len.W) + val wr_bank_addr = UInt(log2Up(memParam.bankEntries).W) + val iter = UInt(frontendParam.iter_len.W) +} + +class ex_st_req(frontendParam: FrontendParam, val ballParam: VectorBallParam) extends Bundle { + val config = ballParam + val InputNum = config.lane + val accWidth = config.outputWidth + val rst = Vec(InputNum, UInt(accWidth.W)) + val iter = UInt(frontendParam.iter_len.W) +} + +class BankWriteEntry(memParam: MemDomainParam, val ballParam: VectorBallParam) extends Bundle { + val addr = UInt(log2Ceil(memParam.bankEntries).W) + val data = UInt(memParam.bankWidth.W) + val mask = Vec(memParam.bankMaskLen, Bool()) +} + +@instantiable +class VecStoreUnit( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: VectorBallParam) + extends Module { + val config = ballParam + val InputNum = config.lane + val accWidth = config.outputWidth + + // Use this instance's configured bandwidth. + val ballMapping = instance + + val outBW = ballMapping.outBW + + @public + val io = IO(new Bundle { + val ctrl_st_i = Flipped(Decoupled(new ctrl_st_req(frontendParam, memParam, ballParam))) + val ex_st_i = Flipped(Decoupled(new ex_st_req(frontendParam, ballParam))) + val bankWrite = Vec(outBW, Flipped(new SramWriteIO(memParam))) + val wr_bank_o = Output(UInt(frontendParam.bank_id_len.W)) + val cmdResp_o = Valid(new Bundle { val commit = Bool() }) + }) + + val wr_bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val wr_bank_addr = RegInit(0.U(log2Up(memParam.bankEntries).W)) + val iter = RegInit(0.U(frontendParam.iter_len.W)) + val iter_counter = RegInit(0.U(frontendParam.iter_len.W)) + val idle :: busy :: Nil = Enum(2) + val state = RegInit(idle) + + val writeQueues = VecInit( + Seq.fill(outBW)( + Instantiate(new Queue(new BankWriteEntry(memParam, ballParam), 16, false, false)).io + ) + ) + +// ----------------------------------------------------------------------------- +// Set registers when Ctrl instruction arrives +// ----------------------------------------------------------------------------- + io.ctrl_st_i.ready := state === idle + + when(io.ctrl_st_i.fire) { + wr_bank := io.ctrl_st_i.bits.wr_bank + wr_bank_addr := io.ctrl_st_i.bits.wr_bank_addr + iter := (io.ctrl_st_i.bits.iter + 15.U) & (~15.U(frontendParam.iter_len.W)) + iter_counter := 0.U + state := busy + } + +// ----------------------------------------------------------------------------- +// Accept computation results from EX unit and push to write queues +// ----------------------------------------------------------------------------- + io.ex_st_i.ready := state === busy && writeQueues.forall(_.enq.ready) + + for (i <- 0 until outBW) { + writeQueues(i).enq.valid := false.B + writeQueues(i).enq.bits := DontCare + } + + when(io.ex_st_i.fire) { + for (i <- 0 until outBW) { + val elementsPerChannel = InputNum / outBW + val startIdx = i * elementsPerChannel + val endIdx = startIdx + elementsPerChannel - 1 + + val entry = Wire(new BankWriteEntry(memParam, ballParam)) + entry.addr := wr_bank_addr + iter_counter(log2Ceil(InputNum) - 1, 0) + entry.data := Cat(io.ex_st_i.bits.rst.slice(startIdx, endIdx + 1).reverse) + entry.mask := VecInit(Seq.fill(memParam.bankMaskLen)(true.B)) + + writeQueues(i).enq.valid := true.B + writeQueues(i).enq.bits := entry + } + iter_counter := iter_counter + 1.U + } + +// ----------------------------------------------------------------------------- +// Drain write queues to bankWrite interface +// ----------------------------------------------------------------------------- + io.bankWrite.foreach { acc => + acc.req.valid := false.B + acc.req.bits.addr := 0.U + acc.req.bits.data := Cat(Seq.fill(accWidth / 8)(0.U(8.W))) + acc.req.bits.mask := VecInit(Seq.fill(memParam.bankMaskLen)(false.B)) + acc.resp.ready := false.B + } + + for (i <- 0 until outBW) { + writeQueues(i).deq.ready := false.B + + when(writeQueues(i).deq.valid) { + io.bankWrite(i).req.valid := true.B + io.bankWrite(i).req.bits.addr := writeQueues(i).deq.bits.addr + io.bankWrite(i).req.bits.data := writeQueues(i).deq.bits.data + io.bankWrite(i).req.bits.mask := writeQueues(i).deq.bits.mask + writeQueues(i).deq.ready := io.bankWrite(i).req.ready + } + } + + // Output wr_bank for bank_id setting + io.wr_bank_o := wr_bank + +// ----------------------------------------------------------------------------- +// Reset iter counter, commit cmdResp, return to idle state +// ----------------------------------------------------------------------------- + val allQueuesEmpty = writeQueues.forall(q => !q.deq.valid) + val allDataEnqueued = state === busy && iter_counter >= iter + + when(allDataEnqueued && allQueuesEmpty) { + state := idle + io.cmdResp_o.valid := true.B + io.cmdResp_o.bits.commit := true.B + }.otherwise { + io.cmdResp_o.valid := false.B + io.cmdResp_o.bits.commit := false.B + } + +} diff --git a/examples/components/balls/vector/arch/VecUnit.scala b/examples/components/balls/vector/arch/VecUnit.scala new file mode 100644 index 000000000..cdd358197 --- /dev/null +++ b/examples/components/balls/vector/arch/VecUnit.scala @@ -0,0 +1,79 @@ +package examples.balls.vector + +import framework.acc.balldomain.configs.BallRouteParam + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam + +import chisel3._ +import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} + +import framework.acc.balldomain.ball.blink.BallStatus +import framework.acc.memdomain.{BankRead, BankWrite} +import examples.balls.vector.configs.VectorBallParam + +@instantiable +class VecUnit( + val frontendParam: FrontendParam, + val memParam: MemDomainParam, + val routingParam: BallDomainParam, + val instance: BallRouteParam, + val ballParam: VectorBallParam) + extends Module { + val commandDataBits = frontendParam.commandDataBits + val cfg = ballParam + val lane = cfg.lane + val inW = cfg.inputWidth + val accW = cfg.outputWidth + + require(lane == 16, "VecUnit vecmat16 requires lane=16") + require(inW == 8, "VecUnit vecmat16 requires int8 inputs") + require(accW == 32, "VecUnit vecmat16 requires int32 accumulators") + + val map = instance + + val inBW = map.inBW + val outBW = map.outBW + require(inBW >= 2, "VecUnit vecmat16 requires two read ports") + require(outBW >= 4, "VecUnit vecmat16 requires four write ports") + + @public + val io = IO(new Bundle { + + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + + val cmdResp = Decoupled(new BallResponse(frontendParam)) + + val bankRead = Vec( + inBW, + Flipped(new BankRead( + frontendParam, + memParam + )) + ) + + val bankWrite = Vec( + outBW, + Flipped(new BankWrite( + frontendParam, + memParam + )) + ) + + val status = new BallStatus + }) + + val thread = + Instantiate(new VecMat16(frontendParam, memParam, routingParam, inBW, outBW, inW, accW, lane, ballParam)) + + thread.io.cmdReq <> io.cmdReq + thread.io.cmdResp <> io.cmdResp + for (i <- 0 until inBW) { thread.io.bankRead(i) <> io.bankRead(i) } + for (i <- 0 until outBW) { thread.io.bankWrite(i) <> io.bankWrite(i) } + io.status <> thread.io.status +} diff --git a/examples/balls/vector/arch/src/main/scala/bond/BondWrapper.scala b/examples/components/balls/vector/arch/bond/BondWrapper.scala similarity index 100% rename from examples/balls/vector/arch/src/main/scala/bond/BondWrapper.scala rename to examples/components/balls/vector/arch/bond/BondWrapper.scala diff --git a/examples/balls/vector/arch/src/main/scala/bond/README.md b/examples/components/balls/vector/arch/bond/README.md similarity index 100% rename from examples/balls/vector/arch/src/main/scala/bond/README.md rename to examples/components/balls/vector/arch/bond/README.md diff --git a/examples/components/balls/vector/arch/bond/vvv.scala b/examples/components/balls/vector/arch/bond/vvv.scala new file mode 100644 index 000000000..111cfd174 --- /dev/null +++ b/examples/components/balls/vector/arch/bond/vvv.scala @@ -0,0 +1,27 @@ +//===----------------------------------------------------------------------===// +// VVV Bond: +// Input: Vec, Vec +// Output: Vec +//===----------------------------------------------------------------------===// + +package examples.balls.vector.bond + +import chisel3._ +import chisel3.util._ +import examples.balls.vector.configs.VectorBallParam + +class VVV(val inputWidth: Int, val outputWidth: Int, val ballParam: VectorBallParam) extends Bundle { + val config = ballParam + + val lane = config.lane + + val in = Flipped(Decoupled(new Bundle { + val in1 = Vec(lane, UInt(inputWidth.W)) + val in2 = Vec(lane, UInt(inputWidth.W)) + })) + + val out = Decoupled(new Bundle { + val out = Vec(lane, UInt(outputWidth.W)) + }) + +} diff --git a/examples/balls/vector/arch/src/main/scala/op/README.md b/examples/components/balls/vector/arch/op/README.md similarity index 100% rename from examples/balls/vector/arch/src/main/scala/op/README.md rename to examples/components/balls/vector/arch/op/README.md diff --git a/examples/balls/vector/arch/src/main/scala/op/cascade.scala b/examples/components/balls/vector/arch/op/cascade.scala similarity index 79% rename from examples/balls/vector/arch/src/main/scala/op/cascade.scala rename to examples/components/balls/vector/arch/op/cascade.scala index 46ff3dcbf..1d67185f5 100644 --- a/examples/balls/vector/arch/src/main/scala/op/cascade.scala +++ b/examples/components/balls/vector/arch/op/cascade.scala @@ -1,16 +1,26 @@ package examples.balls.vector.op +import chisel3.experimental.hierarchy.{instantiable, public} + import chisel3._ import chisel3.util._ import examples.balls.vector.configs.VectorBallParam import examples.balls.vector.bond.VVV -class CascadeOp(val config: VectorBallParam) extends Module { +@instantiable +class CascadeOp(val ballParam: VectorBallParam) extends Module { + val config = ballParam + val lane = config.lane val inputWidth = config.outputWidth // Cascade uses outputWidth as input val outputWidth = config.outputWidth - val io = IO(new VVV(config, inputWidth, outputWidth)) + @public + val io = IO(new VVV( + inputWidth, + outputWidth, + ballParam + )) val reg1 = RegInit(VecInit(Seq.fill(lane)(0.U(outputWidth.W)))) val reg2 = RegInit(VecInit(Seq.fill(lane)(0.U(outputWidth.W)))) diff --git a/examples/balls/vector/arch/src/main/scala/op/mul.scala b/examples/components/balls/vector/arch/op/mul.scala similarity index 78% rename from examples/balls/vector/arch/src/main/scala/op/mul.scala rename to examples/components/balls/vector/arch/op/mul.scala index cde3e0c6c..69c8237a6 100644 --- a/examples/balls/vector/arch/src/main/scala/op/mul.scala +++ b/examples/components/balls/vector/arch/op/mul.scala @@ -1,16 +1,27 @@ package examples.balls.vector.op +import chisel3.experimental.hierarchy.{instantiable, public} + import chisel3._ import chisel3.util._ import examples.balls.vector.configs.VectorBallParam import examples.balls.vector.bond.VVV -class MulOp(val config: VectorBallParam) extends Module { +@instantiable +class MulOp(val ballParam: VectorBallParam) extends Module { + val config = ballParam + val lane = config.lane val inputWidth = config.inputWidth val outputWidth = config.outputWidth - val io = IO(new VVV(config, inputWidth, outputWidth)) + @public + val io = IO(new VVV( + inputWidth, + outputWidth, + ballParam + )) + val reg1 = RegInit(VecInit(Seq.fill(lane)(0.U(inputWidth.W)))) val reg2 = RegInit(VecInit(Seq.fill(lane)(0.U(inputWidth.W)))) val cnt = RegInit(0.U(log2Ceil(lane).W)) diff --git a/examples/components/balls/vector/arch/thread/BaseThread.scala b/examples/components/balls/vector/arch/thread/BaseThread.scala new file mode 100644 index 000000000..9d29fc315 --- /dev/null +++ b/examples/components/balls/vector/arch/thread/BaseThread.scala @@ -0,0 +1,19 @@ +//===- BaseThread.scala - Level 1: Thread ---===// +package examples.balls.vector.thread + +import chisel3.experimental.hierarchy.{instantiable, public} + +import chisel3._ +import examples.balls.vector.configs.VectorBallParam + +//===----------------------------------------------------------------------===// +// BaseThread base class +//===----------------------------------------------------------------------===// +@instantiable +class BaseThread(val opType: String, ballParam: VectorBallParam) extends Module { + val config = ballParam + + @public + val io = IO(new Bundle {}) + val lane = config.lane +} diff --git a/examples/components/balls/vector/arch/thread/CasThread.scala b/examples/components/balls/vector/arch/thread/CasThread.scala new file mode 100644 index 000000000..429f91df8 --- /dev/null +++ b/examples/components/balls/vector/arch/thread/CasThread.scala @@ -0,0 +1,35 @@ +package examples.balls.vector.thread + +import chisel3.experimental.hierarchy.public + +import chisel3.experimental.hierarchy.instantiable + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import chisel3._ +import chisel3.util._ +import examples.balls.vector.configs.VectorBallParam +import examples.balls.vector.bond.VVV +import examples.balls.vector.op.CascadeOp + +@instantiable +class CasThread(val ballParam: VectorBallParam) + extends BaseThread( + "cascade", + ballParam + ) { + val cascadeOp = Instantiate(new CascadeOp(ballParam)) + + @public + val vvvBond = IO( + new VVV( + this.config.outputWidth, + this.config.outputWidth, + ballParam + ) + ) + + // Connect CascadeOp and VVVBond + cascadeOp.io.in <> vvvBond.in + cascadeOp.io.out <> vvvBond.out +} diff --git a/examples/components/balls/vector/arch/thread/MulThread.scala b/examples/components/balls/vector/arch/thread/MulThread.scala new file mode 100644 index 000000000..ad065f818 --- /dev/null +++ b/examples/components/balls/vector/arch/thread/MulThread.scala @@ -0,0 +1,35 @@ +package examples.balls.vector.thread + +import chisel3.experimental.hierarchy.public + +import chisel3.experimental.hierarchy.instantiable + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + +import chisel3._ +import chisel3.util._ +import examples.balls.vector.configs.VectorBallParam +import examples.balls.vector.bond.VVV +import examples.balls.vector.op.MulOp + +@instantiable +class MulThread(val ballParam: VectorBallParam) + extends BaseThread( + "mul", + ballParam + ) { + val mulOp = Instantiate(new MulOp(ballParam)) + + @public + val vvvBond = IO( + new VVV( + this.config.inputWidth, + this.config.outputWidth, + ballParam + ) + ) + + // Connect MulOp and VVVBond + mulOp.io.in <> vvvBond.in + mulOp.io.out <> vvvBond.out +} diff --git a/examples/balls/vector/arch/src/main/scala/thread/README.md b/examples/components/balls/vector/arch/thread/README.md similarity index 100% rename from examples/balls/vector/arch/src/main/scala/thread/README.md rename to examples/components/balls/vector/arch/thread/README.md diff --git a/examples/balls/vector/arch/src/main/scala/warp/MeshWarp.scala b/examples/components/balls/vector/arch/warp/MeshWarp.scala similarity index 80% rename from examples/balls/vector/arch/src/main/scala/warp/MeshWarp.scala rename to examples/components/balls/vector/arch/warp/MeshWarp.scala index 13ebaf4d9..605a70b14 100644 --- a/examples/balls/vector/arch/src/main/scala/warp/MeshWarp.scala +++ b/examples/components/balls/vector/arch/warp/MeshWarp.scala @@ -1,34 +1,50 @@ package examples.balls.vector.warp +import chisel3.experimental.hierarchy.public + +import chisel3.experimental.hierarchy.instantiable + +import chisel3.experimental.hierarchy.{Instance, Instantiate} + import chisel3._ import chisel3.util._ import chisel3.stage._ import examples.balls.vector.configs.VectorBallParam import examples.balls.vector.thread._ -class MeshWarpInput(val config: VectorBallParam) extends Bundle { +class MeshWarpInput(val ballParam: VectorBallParam) extends Bundle { + val config = ballParam + val op1 = Vec(config.lane, UInt(config.inputWidth.W)) val op2 = Vec(config.lane, UInt(config.inputWidth.W)) val thread_id = UInt(10.W) } -class MeshWarpOutput(val config: VectorBallParam) extends Bundle { +class MeshWarpOutput(val ballParam: VectorBallParam) extends Bundle { + val config = ballParam + val res = Vec(config.lane, UInt(config.outputWidth.W)) } -class MeshWarp(val config: VectorBallParam) extends Module { +@instantiable +class MeshWarp(val ballParam: VectorBallParam) extends Module { + val config = ballParam + @public val io = IO(new Bundle { - val in = Flipped(Decoupled(new MeshWarpInput(config))) - val out = Decoupled(new MeshWarpOutput(config)) + + val in = Flipped(Decoupled(new MeshWarpInput(ballParam))) + + val out = Decoupled(new MeshWarpOutput(ballParam)) + }) val mulThreads = (0 until config.numMulThreads).map { i => - Module(new MulThread(config)) + Instantiate(new MulThread(ballParam)) } val casThreads = (0 until config.numCasThreads).map { i => - Module(new CasThread(config)) + Instantiate(new CasThread(ballParam)) } io.in.ready := mulThreads(0).vvvBond.in.ready diff --git a/examples/balls/vector/arch/src/main/scala/warp/README.md b/examples/components/balls/vector/arch/warp/README.md similarity index 100% rename from examples/balls/vector/arch/src/main/scala/warp/README.md rename to examples/components/balls/vector/arch/warp/README.md diff --git a/examples/components/balls/vector/configs/VectorBallParam.scala b/examples/components/balls/vector/configs/VectorBallParam.scala new file mode 100644 index 000000000..1b95c2f42 --- /dev/null +++ b/examples/components/balls/vector/configs/VectorBallParam.scala @@ -0,0 +1,31 @@ +package examples.balls.vector.configs + +import buckyball.balls.vector.{Params => ParametersProto} +import upickle.default.{macroRW, ReadWriter} + +case class VectorBallParam( + InputNum: Int, + inputWidth: Int, + lane: Int, + outputWidth: Int, + numMulThreads: Int, + numCasThreads: Int) + +object VectorBallParam { + + def fromProto(source: ParametersProto): VectorBallParam = { + require( + source.hasInputNum && source.hasInputWidth && source.hasLane && source.hasOutputWidth && source.hasNumMulThreads && source.hasNumCasThreads + ) + VectorBallParam( + source.getInputNum, + source.getInputWidth, + source.getLane, + source.getOutputWidth, + source.getNumMulThreads, + source.getNumCasThreads + ) + } + + implicit val rw: ReadWriter[VectorBallParam] = macroRW +} diff --git a/examples/components/balls/vector/configs/default.toml b/examples/components/balls/vector/configs/default.toml new file mode 100644 index 000000000..b83bbd5d9 --- /dev/null +++ b/examples/components/balls/vector/configs/default.toml @@ -0,0 +1,8 @@ +# Vector Ball Configuration + +InputNum = 16 +inputWidth = 8 +lane = 16 +outputWidth = 32 +numMulThreads = 16 +numCasThreads = 16 diff --git a/examples/components/balls/vector/configs/vector_params.proto b/examples/components/balls/vector/configs/vector_params.proto new file mode 100644 index 000000000..94cd9c9a8 --- /dev/null +++ b/examples/components/balls/vector/configs/vector_params.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; +package buckyball.balls.vector; +option java_package = "buckyball.balls.vector"; +option java_multiple_files = true; + +message Params { + optional int32 InputNum = 1; + optional int32 inputWidth = 2; + optional int32 lane = 3; + optional int32 outputWidth = 4; + optional int32 numMulThreads = 5; + optional int32 numCasThreads = 6; +} diff --git a/examples/balls/vector/emu/src/64_vecmat16.rs b/examples/components/balls/vector/emu/src/64_vecmat16.rs similarity index 95% rename from examples/balls/vector/emu/src/64_vecmat16.rs rename to examples/components/balls/vector/emu/src/64_vecmat16.rs index a8bbda23d..eac7ff2da 100644 --- a/examples/balls/vector/emu/src/64_vecmat16.rs +++ b/examples/components/balls/vector/emu/src/64_vecmat16.rs @@ -2,14 +2,14 @@ use super::bank_matrix::{read_i32_nn_groups, read_i8_k_rows, write_i32_nn_groups}; use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; -use super::instruction::{BallInstruction, ExecContext}; +use super::instruction::{BallModel, ExecContext}; const WARP_M: usize = 16; const WARP_N: usize = 16; pub struct VecMat16; -impl BallInstruction for VecMat16 { +impl BallModel for VecMat16 { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let op1 = rs1_b0(xs1); let op2 = rs1_b1(xs1); diff --git a/examples/components/balls/vector/emu/src/dispatch.rs b/examples/components/balls/vector/emu/src/dispatch.rs new file mode 100644 index 000000000..3754eba43 --- /dev/null +++ b/examples/components/balls/vector/emu/src/dispatch.rs @@ -0,0 +1,31 @@ +use bebop_bemu::ball::{BallModel, ExecContext}; + +use super::f64_vecmat16; + +const BALL_CLASS: &str = "examples.balls.vector.VecBall"; + +pub fn execute_known( + ball_class: &str, + funct: u32, + xs1: u64, + xs2: u64, + ctx: &mut ExecContext, +) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("VECMAT16") => Some(f64_vecmat16::VecMat16::exec(xs1, xs2, ctx)), + Some(_) | None => None, + } +} + +pub fn cycles_after_issue(ball_class: &str, funct: u32, xs1: u64, xs2: u64) -> Option { + if ball_class != BALL_CLASS { + return None; + } + match bebop_bemu::configs::ball_domain::mnemonic_for_funct(funct).as_deref() { + Some("VECMAT16") => Some(f64_vecmat16::VecMat16::latency(xs1, xs2)), + Some(_) | None => None, + } +} diff --git a/examples/components/balls/vector/emu/src/lib.rs b/examples/components/balls/vector/emu/src/lib.rs new file mode 100644 index 000000000..30fdb8605 --- /dev/null +++ b/examples/components/balls/vector/emu/src/lib.rs @@ -0,0 +1,12 @@ +pub(crate) use bebop_bemu::{thread::{instructions::decode, bank_matrix}, ball as instruction}; + +#[path = "64_vecmat16.rs"] +mod f64_vecmat16; + +mod dispatch; + +pub use dispatch::{cycles_after_issue, execute_known}; + +#[bebop_bemu::ball::linkme::distributed_slice(bebop_bemu::BALLS)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static BALL: bebop_bemu::Ball = bebop_bemu::Ball { execute: execute_known, cycles: cycles_after_issue }; diff --git a/examples/components/balls/vector/workloads/ctests/CMakeLists.txt b/examples/components/balls/vector/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..c5de3fc66 --- /dev/null +++ b/examples/components/balls/vector/workloads/ctests/CMakeLists.txt @@ -0,0 +1 @@ +add_buckyball_ctest(vecmat16_atranspose_16x16_test.c) diff --git a/examples/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c b/examples/components/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c similarity index 100% rename from examples/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c rename to examples/components/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c diff --git a/examples/balls/vector/workloads/isa/vecmat16.h b/examples/components/balls/vector/workloads/isa/vecmat16.h similarity index 100% rename from examples/balls/vector/workloads/isa/vecmat16.h rename to examples/components/balls/vector/workloads/isa/vecmat16.h diff --git a/examples/balls/vector/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/vector/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/balls/vector/workloads/mlir_tests/CMakeLists.txt rename to examples/components/balls/vector/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/components/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16.mlir b/examples/components/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16.mlir similarity index 100% rename from examples/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16.mlir rename to examples/components/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16.mlir diff --git a/examples/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16_main.cpp b/examples/components/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16_main.cpp similarity index 100% rename from examples/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16_main.cpp rename to examples/components/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16_main.cpp diff --git a/examples/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/components/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth.mlir.in b/examples/components/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth.mlir.in similarity index 100% rename from examples/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth.mlir.in rename to examples/components/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth.mlir.in diff --git a/examples/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth_main.cpp b/examples/components/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth_main.cpp similarity index 100% rename from examples/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth_main.cpp rename to examples/components/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth_main.cpp diff --git a/examples/components/cores/boom/LICENSE b/examples/components/cores/boom/LICENSE new file mode 100644 index 000000000..f22c14f96 --- /dev/null +++ b/examples/components/cores/boom/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2012-2020, The Regents of the University of California (Regents) +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/examples/components/cores/boom/arch/Boom.scala b/examples/components/cores/boom/arch/Boom.scala new file mode 100644 index 000000000..da1074500 --- /dev/null +++ b/examples/components/cores/boom/arch/Boom.scala @@ -0,0 +1,202 @@ +package examples.core.boom + +import framework.acc.balldomain.bbus.BallFactory +import sims.configs.SimParam +import framework.acc.balldomain.ball.blink.BlinkIO + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.boom.configs.Params +import framework.core.clink.{CLinkIO, HasCLink} +import framework.core.clink.configs.{Params => CLinkParam} +import framework.ip.bus.axi4 + +import framework.acc.frontend.Frontend +import framework.acc.memdomain.MemDomain +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.ball.blink.HasBlink +import framework.acc.debug.Debug + +@instantiable +class Boom( + p: Params, + simp: SimParam, + factories: Map[String, BallFactory]) + extends Module + with HasCLink { + val diffTest = simp.diffTest + @public + val clink = IO(new CLinkIO(CLinkParam(1, false, Seq.empty, p.chi, p.axi, p.axi, None))) + val native = Instantiate(new Native) + native.io.clock := clock + native.io.reset := reset.asBool + native.io.io_resetVector := p.resetVector.U + native.io.io_interrupts(0) := clink.interrupts(0).software + native.io.io_interrupts(1) := clink.interrupts(0).timer + native.io.io_interrupts(2) := clink.interrupts(0).external + native.io.io_interrupts(3) := clink.interrupts(0).supervisorExternal + for ( + (up, down) <- Seq((native.io.memoryAXI(0), clink.memory.ddr), (native.io.peripheralsAXI(0), clink.memory.uncached)) + ) { + down.aw.valid := up.aw.valid + up.aw.ready := down.aw.ready + down.ar.valid := up.ar.valid + up.ar.ready := down.ar.ready + for ((dst, src) <- Seq((down.aw.bits, up.aw.bits), (down.ar.bits, up.ar.bits))) { + dst := 0.U.asTypeOf(dst) + dst.id := src.id; dst.addr := src.addr; dst.len := src.len; dst.size := src.size + dst.burst := src.burst; dst.lock := src.lock; dst.cache := src.cache; dst.prot := src.prot; dst.qos := src.qos + } + down.w <> up.w + up.b <> down.b + up.r <> down.r + } + p.acc match { + case Some(acc) => + val frontend = + Instantiate(new Frontend(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, p.permission, simp)) + val memDomain = Instantiate(new MemDomain( + acc.frontend, + acc.memDomain, + acc.gpDomain, + acc.transport, + p.permission, + p.tracking, + p.walker, + simp + )) + val domain = + Instantiate(new BallDomain(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, p.permission, factories)) + if (acc.gpDomain.enable) { + val gp = Instantiate(new GpDomain(acc.frontend, acc.memDomain, acc.gpDomain)) + gp.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> gp.io.complete + gp.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> gp.io.banks + memDomain.io.gpLayoutRequest.get <> gp.io.layoutRequest + gp.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + domain.io.kernel.get.request <> gp.io.ball.request + gp.io.ball.response <> domain.io.kernel.get.response + } + val command = native.io.io_commands.request + frontend.io.rocc.cmd.valid := command.valid + command.ready := frontend.io.rocc.cmd.ready + frontend.io.rocc.cmd.bits.instruction.raw_inst := command.bits.instruction + frontend.io.rocc.cmd.bits.instruction.pc := 0.U + frontend.io.rocc.cmd.bits.instruction.rs1Data := command.bits.rs1 + frontend.io.rocc.cmd.bits.instruction.rs2Data := command.bits.rs2 + frontend.io.rocc.cmd.bits.satp := native.io.io_context.satp + frontend.io.rocc.cmd.bits.effectivePrivilege := command.bits.privilege + frontend.io.rocc.cmd.bits.sum := command.bits.sum + frontend.io.rocc.cmd.bits.mxr := command.bits.mxr + for ((target, entry) <- frontend.io.rocc.cmd.bits.pmp.zip(native.io.io_context.pmp)) { + target.cfg := entry.cfg.asTypeOf(target.cfg) + target.addr := entry.addr + val encoded = ((entry.addr.pad(44) << 1) | entry.cfg(3).asUInt)(43, 0) + target.mask := (((encoded & ~(encoded + 1.U)) << 2) | 3.U)(43, 0) + } + frontend.io.rocc.resp.ready := true.B + val pending = frontend.io.rocc.busy + native.io.io_commands.busy := frontend.io.rocc.busy || command.valid + native.io.io_commands.fault := frontend.io.rocc.interrupt + + domain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> domain.io.complete + memDomain.io.ball <> domain.io.memory + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + frontend.io.barrier.release := frontend.io.barrier.arrive + memDomain.io.banks <> domain.io.banks + memDomain.io.ballLayoutRequest <> domain.io.layoutRequest + domain.io.layoutResponse <> memDomain.io.ballLayoutResponse + + // The native coherence hub snoops BOOM caches for DMA. The submitting core has already drained stores + // and cannot execute younger memory operations until this command completes. + memDomain.io.memory.cpuQuery := 0.U.asTypeOf(memDomain.io.memory.cpuQuery) + val maintained = RegInit(false.B) + val maintenanceTag = Reg(chiselTypeOf(memDomain.io.memory.maintenance.bits.tag)) + memDomain.io.memory.maintenance.ready := !maintained + when(memDomain.io.memory.maintenance.fire) { + assert(pending) + maintenanceTag := memDomain.io.memory.maintenance.bits.tag + maintained := true.B + } + memDomain.io.memory.maintained.valid := maintained + memDomain.io.memory.maintained.bits.tag := maintenanceTag + memDomain.io.memory.maintained.bits.ok := true.B + when(memDomain.io.memory.maintained.fire)(maintained := false.B) + + val access = Instantiate(new axi4.Interconnect(axi4.InterconnectParam(p.axi, 2, Some(16)))) + access.io.in(0) <> memDomain.io.ddr + val pte = access.io.in(1) + val ptePending = RegInit(false.B) + val highWord = Reg(Bool()) + pte.aw.valid := false.B + pte.aw.bits := 0.U.asTypeOf(pte.aw.bits) + pte.w.valid := false.B + pte.w.bits := 0.U.asTypeOf(pte.w.bits) + pte.b.ready := true.B + pte.ar.valid := memDomain.io.memory.pteRequest.valid && !ptePending + pte.ar.bits := 0.U.asTypeOf(pte.ar.bits) + pte.ar.bits.addr := memDomain.io.memory.pteRequest.bits.addr + pte.ar.bits.size := 3.U + pte.ar.bits.burst := 1.U + memDomain.io.memory.pteRequest.ready := pte.ar.ready && !ptePending + when(pte.ar.fire) { ptePending := true.B; highWord := pte.ar.bits.addr(3) } + memDomain.io.memory.pteResponse.valid := pte.r.valid && ptePending + memDomain.io.memory.pteResponse.bits.data := Mux(highWord, pte.r.bits.data(127, 64), pte.r.bits.data(63, 0)) + memDomain.io.memory.pteResponse.bits.error := pte.r.bits.resp =/= 0.U + pte.r.ready := memDomain.io.memory.pteResponse.ready && ptePending + when(pte.r.fire) { assert(pte.r.bits.last && pte.r.bits.id === 0.U); ptePending := false.B } + val dma = native.io.dmaAXI(0) + val upstream = access.io.out + dma.aw.valid := upstream.aw.valid + upstream.aw.ready := dma.aw.ready + dma.ar.valid := upstream.ar.valid + upstream.ar.ready := dma.ar.ready + for ((dst, src) <- Seq((dma.aw.bits, upstream.aw.bits), (dma.ar.bits, upstream.ar.bits))) { + dst.id := src.id; dst.addr := src.addr; dst.len := src.len; dst.size := src.size + dst.burst := src.burst; dst.lock := src.lock; dst.cache := 15.U; dst.prot := src.prot; dst.qos := src.qos + } + dma.w <> upstream.w + upstream.b <> dma.b + upstream.r <> dma.r + + val debug = Instantiate(new Debug(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, simp, 0, 0)) + debug.io.ant.valid := false.B + debug.io.ant.bits := 0.U.asTypeOf(debug.io.ant.bits) + debug.io.acc.frontend <> frontend.io.debug + debug.io.acc.trace.memory := memDomain.io.traceMemory + debug.io.acc.trace.ballStarted := domain.io.traceStarted + debug.io.acc.trace.ballCompleted := domain.io.traceCompleted + debug.io.acc.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, src) => sink := src } + for (port <- memDomain.io.shared.bank.requests) { + port.read.req.ready := false.B + port.read.resp.valid := false.B + port.read.resp.bits := 0.U.asTypeOf(port.read.resp.bits) + port.write.req.ready := false.B + port.write.resp.valid := false.B + port.write.resp.bits := 0.U.asTypeOf(port.write.resp.bits) + } + memDomain.io.shared.bank.config.ready := false.B + memDomain.io.shared.bank.mapping.ready := false.B + memDomain.io.shared.bank.local.request.valid := false.B + memDomain.io.shared.bank.local.request.bits := 0.U.asTypeOf(memDomain.io.shared.bank.local.request.bits) + memDomain.io.shared.bank.local.response.ready := false.B + memDomain.io.shared.move.command.ready := false.B + memDomain.io.shared.move.completion.valid := false.B + memDomain.io.shared.move.completion.bits := false.B + case None => + native.io.io_commands.request.ready := true.B + native.io.io_commands.busy := false.B + native.io.io_commands.fault := RegNext(native.io.io_commands.request.fire, false.B) + val dma = native.io.dmaAXI(0) + dma.aw.valid := false.B; dma.aw.bits := 0.U.asTypeOf(dma.aw.bits) + dma.w.valid := false.B; dma.w.bits := 0.U.asTypeOf(dma.w.bits) + dma.ar.valid := false.B; dma.ar.bits := 0.U.asTypeOf(dma.ar.bits) + dma.b.ready := true.B; dma.r.ready := true.B + } +} diff --git a/examples/components/cores/boom/arch/Native.scala b/examples/components/cores/boom/arch/Native.scala new file mode 100644 index 000000000..191fd322d --- /dev/null +++ b/examples/components/cores/boom/arch/Native.scala @@ -0,0 +1,75 @@ +package examples.core.boom + +import chisel3.experimental.hierarchy.{instantiable, public} + +import chisel3._ +import chisel3.util._ +import framework.ip.bus.axi4 +import java.nio.file.Paths +import scala.io.Source +import scala.util.Using + +object Native { + + class Address(bits: Int) extends Bundle { + val id = UInt(4.W) + val addr = UInt(bits.W) + val len = UInt(8.W) + val size = UInt(3.W) + val burst = UInt(2.W) + val lock = Bool() + val cache = UInt(4.W) + val prot = UInt(3.W) + val qos = UInt(4.W) + } + + class Bus(bits: Int) extends Bundle { + val aw = Decoupled(new Address(bits)) + val w = Decoupled(new axi4.WriteData(axi4.Params(bits, 128, 4))) + val b = Flipped(Decoupled(new axi4.WriteResponse(axi4.Params(bits, 128, 4)))) + val ar = Decoupled(new Address(bits)) + val r = Flipped(Decoupled(new axi4.ReadData(axi4.Params(bits, 128, 4)))) + } + +} + +@instantiable +class Native extends BlackBox with HasBlackBoxPath { + override def desiredName = "BoomNative" + + @public + val io = IO(new Bundle { + val clock = Input(Clock()) + val reset = Input(Bool()) + val io_resetVector = Input(UInt(44.W)) + val io_interrupts = Input(Vec(4, Bool())) + + val io_commands = new Bundle { + + val request = Decoupled(new Bundle { + val instruction = UInt(32.W) + val rs1 = UInt(64.W) + val rs2 = UInt(64.W) + val privilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + }) + + val busy = Input(Bool()) + val fault = Input(Bool()) + } + + val io_context = Output(new Bundle { + val satp = UInt(64.W) + val pmp = Vec(8, new Bundle { val cfg = UInt(8.W); val addr = UInt(42.W) }) + }) + + val memoryAXI = Vec(1, new Native.Bus(44)) + val peripheralsAXI = Vec(1, new Native.Bus(31)) + val dmaAXI = Flipped(Vec(1, new Native.Bus(44))) + }) + + val manifest = Paths.get(getClass.getResource("/boom/filelist.f").toURI) + val files = Using.resource(Source.fromFile(manifest.toFile))(_.getLines().filter(_.nonEmpty).toVector) + files.foreach(file => addPath(manifest.getParent.resolve(file.stripPrefix("./")).toString)) +} diff --git a/examples/components/cores/boom/arch/build.sc b/examples/components/cores/boom/arch/build.sc new file mode 100644 index 000000000..8e1c39f52 --- /dev/null +++ b/examples/components/cores/boom/arch/build.sc @@ -0,0 +1,46 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.^.arch.src.main.scala.framework.{build => framework} +val root = millSourcePath / os.up +val rc = root / "thirdparty" / "rocket-chip" +val boomRoot = root / "thirdparty" / "riscv-boom" +trait ScalaBuild extends SbtModule { + override def scalaVersion = "2.13.16" + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") +} +object cde extends ScalaBuild { override def sources = T.sources(rc / os.RelPath("dependencies/cde/cde/src")) } +object macros extends ScalaBuild { + override def sources = T.sources(rc / os.RelPath("macros/src/main/scala")) + override def ivyDeps = super.ivyDeps() ++ Agg(ivy"org.scala-lang:scala-reflect:2.13.16") +} +object diplomacy extends ScalaBuild { + override def sources = T.sources(rc / os.RelPath("dependencies/diplomacy/diplomacy/src")) + override def moduleDeps = Seq(cde) + override def ivyDeps = super.ivyDeps() ++ Agg(ivy"com.lihaoyi::sourcecode:0.3.1") +} +object hardfloat extends ScalaBuild { override def sources = T.sources(rc / os.RelPath("dependencies/hardfloat/hardfloat/src/main/scala")) } +object rocketchip extends ScalaBuild { + override def sources = T.sources(rc / os.RelPath("src/main/scala")) + override def resources = T.sources(rc / os.RelPath("src/main/resources")) + override def moduleDeps = Seq(cde, macros, diplomacy, hardfloat) + override def ivyDeps = super.ivyDeps() ++ Agg(ivy"com.lihaoyi::mainargs:0.5.0", ivy"org.json4s::json4s-jackson:4.0.5") +} +object native extends ScalaBuild { + override def sources = T.sources(boomRoot / os.RelPath("src/main/scala/v4"), boomRoot / os.RelPath("src/main/scala/buckyball")) + override def resources = T.sources(boomRoot / os.RelPath("src/main/resources")) + override def moduleDeps = Seq(rocketchip) + override def forkArgs = Seq("-Xmx32G", "-Xss64m") +} +def nativeRtl = T { + val cp = native.runClasspath().map(_.path) + os.proc("java", "-Xmx32G", "-Xss64m", "-cp", cp.mkString(java.io.File.pathSeparator), + "buckyball.Emit", "--target-dir", T.dest / "boom").call(cwd = root) + PathRef(T.dest) +} +object library extends ScalaBuild { + override def moduleDeps = Seq(framework.library) + override def sources = T.sources(root / "arch", root / "configs") + override def resources = T.sources(Seq(nativeRtl())) +} diff --git a/examples/components/cores/boom/configs/Params.scala b/examples/components/cores/boom/configs/Params.scala new file mode 100644 index 000000000..c718141c0 --- /dev/null +++ b/examples/components/cores/boom/configs/Params.scala @@ -0,0 +1,64 @@ +package examples.core.boom.configs + +import buckyball.config.CorePlacement +import framework.ip.bus.{axi4, chi => chiBus} + +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.acc.configs.AccParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} + +case class Params( + chi: chiBus.Params, + axi: axi4.Params, + hartId: Int, + resetVector: BigInt, + mtimeAddress: BigInt, + permission: PmpParams, + tracking: TrackingParam, + walker: WalkerParam, + acc: Option[AccParam]) { + require(chi.nodeIdBits == 7 && chi.addressBits == 44 && chi.dataBits == 256) + require(axi.addressBits == 44 && axi.dataBits == 128 && axi.idBits == 4) + require(hartId == 0) + require((mtimeAddress & 7) == 0) + require(permission.addressBits == 44 && permission.entries == 8) + acc.foreach(a => require(!a.memDomain.sharedEnable && a.transport.axi == axi)) +} + +object Params { + + def fromProto(core: CorePlacement): Params = { + require(core.getThreadIndicesCount == 0) + require(core.getImplementation.unpack(classOf[buckyball.components.boom.Params]).getProfile == "SmallBoomV4") + val h = core + val bus = chiBus.Params.fromProto(h.getMemory.getHome.getRequesters) + val acc = Option.when(core.hasAccelerator) { + val a = core.getAccelerator + AccParam( + FrontendParam.fromProto(a.getFrontend, core.getParam.getXLen, core.getHartId), + MemDomainParam.fromProto(a.getMem, core.getSharedMem, core.getVirtualBankCount, 1), + GpDomainParam.fromProto(a.getGpdomain), + BallDomainParam.fromProto(a.getBalldomain), + TransportParam.fromProto(a.getTransport) + ) + } + Params( + bus, + axi4.Params.fromProto(h.getMemory.getUncached.getAxi), + h.getHartId, + BigInt(java.lang.Long.toUnsignedString(h.getResetVector)), + BigInt(java.lang.Long.toUnsignedString(h.getMtimeAddress)), + PmpParams(44, 8), + TrackingParam.fromProto(h.getMemory.getDcache.getTracking), + WalkerParam(bus, h.getMemory.getAccess.getTranslationEntries), + acc + ) + } + +} diff --git a/examples/components/cores/boom/configs/boom_params.proto b/examples/components/cores/boom/configs/boom_params.proto new file mode 100644 index 000000000..1f475593f --- /dev/null +++ b/examples/components/cores/boom/configs/boom_params.proto @@ -0,0 +1,7 @@ +syntax = "proto2"; +package buckyball.components.boom; +option java_package = "buckyball.components.boom"; +option java_multiple_files = true; +message Params { + required string profile = 1; +} diff --git a/examples/components/cores/boom/stack/CMakeLists.txt b/examples/components/cores/boom/stack/CMakeLists.txt new file mode 100644 index 000000000..7d00c1146 --- /dev/null +++ b/examples/components/cores/boom/stack/CMakeLists.txt @@ -0,0 +1,25 @@ +# ===----------------------------------------------------------------------=== +# The boom core: a CPU core without hardware threads; jobs run on the core. +# +# bbdev stack --build "--design " +# +# builds it for each design that uses it, configured with DESIGN_CMAKE +# (generated from design.pb) and CORE. Products: +# execute.o runs jobs on the core itself; links into the device runtime +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(boom-core LANGUAGES NONE) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +add_custom_command( + OUTPUT execute.o + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" + -c "${RUNTIME}/execute/direct.c" -o execute.o + DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" + VERBATIM) + +add_custom_target(execute ALL DEPENDS execute.o) diff --git a/examples/components/cores/boom/workloads/ctests/CMakeLists.txt b/examples/components/cores/boom/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..416956421 --- /dev/null +++ b/examples/components/cores/boom/workloads/ctests/CMakeLists.txt @@ -0,0 +1 @@ +add_buckyball_ctest(memory_matrix.c) diff --git a/examples/components/cores/boom/workloads/ctests/memory_matrix.c b/examples/components/cores/boom/workloads/ctests/memory_matrix.c new file mode 100644 index 000000000..d8f294adb --- /dev/null +++ b/examples/components/cores/boom/workloads/ctests/memory_matrix.c @@ -0,0 +1,47 @@ +#include + +static volatile int32_t a[32][32], b[32][16], c[32][16]; +static volatile uint64_t atomic_word; +static volatile uint64_t upgrade_word[8] __attribute__((aligned(64))) = {7}; + +int main(void) { + for (int i = 0; i < 32; ++i) + for (int k = 0; k < 32; ++k) + a[i][k] = (i * 3 + k * 5) % 13 - 6; + for (int k = 0; k < 32; ++k) + for (int j = 0; j < 16; ++j) + b[k][j] = (k * 7 + j * 2) % 11 - 5; + for (int i = 0; i < 32; ++i) + for (int j = 0; j < 16; ++j) { + int32_t sum = 0; + for (int k = 0; k < 32; ++k) + sum += a[i][k] * b[k][j]; + c[i][j] = sum; + } + for (int i = 0; i < 32; ++i) + for (int j = 0; j < 16; ++j) { + int32_t expected = 0; + for (int k = 0; k < 32; ++k) + expected += ((i * 3 + k * 5) % 13 - 6) * ((k * 7 + j * 2) % 11 - 5); + if (c[i][j] != expected) + return 2; + } + uint64_t old, value = 19; + __asm__ volatile("amoadd.d.aqrl %0, %2, (%1)" + : "=r"(old) + : "r"(&atomic_word), "r"(value) + : "memory"); + if (old != 0 || atomic_word != 19) + return 3; + if (upgrade_word[0] != 7) + return 5; + upgrade_word[0] = 23; + if (upgrade_word[0] != 23) + return 6; + uint64_t before, after; + __asm__ volatile("rdcycle %0" : "=r"(before)); + __asm__ volatile("rdcycle %0" : "=r"(after)); + if (before == 0 || after <= before) + return 4; + return 0; +} diff --git a/examples/components/cores/reduce/arch/ReduceCore.scala b/examples/components/cores/reduce/arch/ReduceCore.scala new file mode 100644 index 000000000..bdf148040 --- /dev/null +++ b/examples/components/cores/reduce/arch/ReduceCore.scala @@ -0,0 +1,126 @@ +package examples.core.reduce + +import sims.configs.SimParam +import framework.device.cbus.CoreFactory +import buckyball.config.{CorePlacement, Design} + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.core.clink.{CLinkIO, HasCLink} +import framework.device.cbus.{C2CTransfer, Control, SharedAxiEndpoint} +import framework.core.controller.admission.DirectAdmission +import framework.core.tbus.SharedNetwork +import framework.core.tbus.configs.Layout +import framework.core.shm.SHM +import framework.acc.debug.Debug +import framework.ip.bus.chi.rnf.AxiCache + +/** Native execution core with a direct BBus accelerator and shared-bank C2C access. */ +@instantiable +class ReduceCore(corep: CoreParam, simp: SimParam) extends Module with HasCLink { + val diffTest = simp.diffTest + val accp = corep.acc.get + require(corep.threads.isEmpty && !corep.clink.backing && corep.clink.requesters == 3) + require(corep.clink.shared && accp.memDomain.sharedEnable && !accp.gpDomain.enable) + require(corep.transferLocalAccess && corep.transferLeaseAccess && corep.tbus.get.router.physicalPorts == 4) + @public + val clink = IO(new CLinkIO(corep.clink)) + + val loom: Instance[Loom] = Instantiate(new Loom(corep)) + val admission: Instance[DirectAdmission] = Instantiate(new DirectAdmission(corep.controller.commands)) + + val acc: Instance[Acc] = Instantiate(new Acc( + accp, + corep.privileged.permission, + corep.memory.dcache.tracking, + corep.memory.memoryAccess.walker, + simp + )) + + loom.io.interrupts := clink.interrupts(0) + admission.io.execution <> loom.io.commands + loom.io.drained := admission.io.drained + admission.io.accelerator <> acc.io.rocc + loom.io.memory <> acc.io.memory + clink.memory.uncached <> loom.io.uncached + clink.memory.coherent.take(2).zip(loom.io.coherent).foreach { case (port, cache) => port <> cache } + + val dma = Instantiate(new AxiCache(accp.transport.axi, corep.dmaCache.get)) + dma.io.axi <> acc.io.ddr + dma.io.axi.ar.bits.cache := 15.U + dma.io.axi.aw.bits.cache := 15.U + clink.memory.coherent(2) <> dma.io.chi + clink.memory.ddr.aw.valid := false.B + clink.memory.ddr.aw.bits := 0.U.asTypeOf(clink.memory.ddr.aw.bits) + clink.memory.ddr.w.valid := false.B + clink.memory.ddr.w.bits := 0.U.asTypeOf(clink.memory.ddr.w.bits) + clink.memory.ddr.ar.valid := false.B + clink.memory.ddr.ar.bits := 0.U.asTypeOf(clink.memory.ddr.ar.bits) + clink.memory.ddr.b.ready := false.B + clink.memory.ddr.r.ready := false.B + + val banks = + Instantiate(new SharedNetwork( + accp.frontend, + accp.memDomain, + corep.tbus.get, + Seq(corep.hartId), + Seq(corep.hartId), + simp + )) + + val shm = Instantiate(new SHM(accp.frontend, accp.memDomain, corep.tbus.get)) + shm.io <> banks.io.shm + banks.io.threads(0).bank <> acc.io.shm.bank + banks.io.threads(0).move <> acc.io.shm.move + banks.io.threads(0).barrierArrive := acc.io.shm.barrierArrive + acc.io.shm.barrierRelease := banks.io.threads(0).barrierRelease + acc.io.shared_bank_hashes.zip(banks.io.bankHashes).foreach { case (sink, source) => sink := source } + + val control = Instantiate(new Control(accp.memDomain, corep.c2c, corep.transferLeaseAccess, corep.transferLocalAccess)) + val transfer = Instantiate(new C2CTransfer(corep.c2c)) + val endpoint = Instantiate(new SharedAxiEndpoint(corep.c2c)) + control.io.coreId := corep.hartId.U + control.io.request <> admission.io.remote.request + admission.io.remote.reply <> control.io.reply + admission.io.remote.busy := !control.io.idle || endpoint.io.busy + transfer.io.command <> control.io.command + control.io.completion <> transfer.io.completion + banks.io.external.get.memory(0) <> transfer.io.source + banks.io.external.get.memory(1) <> endpoint.io.read + banks.io.external.get.memory(2) <> endpoint.io.write + banks.io.external.get.memory(3) <> control.io.local.get + banks.io.external.get.lease.get <> control.io.lease.get + clink.c2c.get.tx <> transfer.io.mem + endpoint.io.mem <> clink.c2c.get.rx + + val debug = Instantiate(new Debug( + accp.frontend, + accp.memDomain, + accp.gpDomain, + accp.ballDomain, + simp, + corep.hartId, + corep.hartId + )) + + debug.io.ant.valid := false.B + debug.io.ant.bits := 0.U.asTypeOf(debug.io.ant.bits) + debug.io.acc <> acc.io.debug +} + +object Reduce extends CoreFactory { + val component = "reduce" + def parameters(design: Design, source: CorePlacement): CoreParam = + CoreParam.fromProto(design, source, externalHome = true) + + def instantiate( + source: CorePlacement, + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int] + ): CLinkIO = + Instantiate(new ReduceCore(corep, simp)).clink + +} diff --git a/examples/components/cores/reduce/arch/acc.scala b/examples/components/cores/reduce/arch/acc.scala new file mode 100644 index 000000000..9f538a99c --- /dev/null +++ b/examples/components/cores/reduce/arch/acc.scala @@ -0,0 +1,119 @@ +package examples.core.reduce + +import framework.acc.balldomain.bbus.generated.BallComponents +import sims.configs.SimParam +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.configs.AccParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.acc.debug.AcceleratorDebugPort +import framework.acc.frontend.RoCCIO +import framework.acc.frontend.Frontend +import framework.acc.memdomain.{MemDomain, PhysicalBankHash} +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.bbus.BBus +import framework.core.memory.MemoryPort +import framework.thread.tlink.SharedLink +import framework.ip.bus.axi4 + +@instantiable +class Acc( + val accp: AccParam, + val permissionp: PmpParams, + val trackingp: TrackingParam, + val walkerp: WalkerParam, + val simp: SimParam) + extends Module { + val commandDataBits = accp.frontend.commandDataBits + val ownerId = accp.frontend.ownerId + val sharedPorts = accp.memDomain.sharedPorts + val diffTest = simp.diffTest + val frontp = accp.frontend + val memp = accp.memDomain + val gpp = accp.gpDomain + val ballp = accp.ballDomain + val transportp = accp.transport + + val sharedHashCount = if (memp.sharedEnable) memp.sharedBankNum else 0 + + @public + val io = IO(new Bundle { + val debug = new AcceleratorDebugPort(frontp, memp, gpp, ballp, simp) + + val rocc = new RoCCIO(commandDataBits, permissionp) + val ddr = new axi4.Port(transportp.axi) + val memory = new MemoryPort(trackingp, walkerp) + val shm = new SharedLink(frontp, memp) + + val shared_bank_hashes = Option.when(diffTest && memp.sharedEnable)( + Input(Vec(sharedHashCount, new PhysicalBankHash(frontp))) + ) + + }) + + // --- Instantiate domains --- + val frontend: Instance[Frontend] = Instantiate(new Frontend(frontp, memp, gpp, ballp, permissionp, simp)) + + val ballDomain: Instance[BallDomain] = + Instantiate(new BallDomain(frontp, memp, gpp, ballp, permissionp, BallComponents.factories)) + + val memDomain: Instance[MemDomain] = + Instantiate(new MemDomain(frontp, memp, gpp, transportp, permissionp, trackingp, walkerp, simp)) + + val gpDomain = + if (gpp.enable) Some(Instantiate(new GpDomain(frontp, memp, gpp))) + else None + + memDomain.io.shared_bank_hashes.foreach(_ := io.shared_bank_hashes.get) + + io.debug.frontend <> frontend.io.debug + io.debug.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, source) => sink := source } + io.debug.trace.memory := memDomain.io.traceMemory + io.debug.trace.ballStarted := ballDomain.io.traceStarted + io.debug.trace.ballCompleted := ballDomain.io.traceCompleted + + io.rocc <> frontend.io.rocc + io.memory <> memDomain.io.memory + io.ddr <> memDomain.io.ddr + + // --- Frontend -> BallDomain --- + memDomain.io.ball <> ballDomain.io.memory + ballDomain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> ballDomain.io.complete + + // --- Frontend -> MemDomain --- + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + + if (gpp.enable) { + val controller = gpDomain.get + controller.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> controller.io.complete + controller.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> controller.io.banks + memDomain.io.gpLayoutRequest.get <> controller.io.layoutRequest + controller.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + ballDomain.io.kernel.get <> controller.io.ball + } + + memDomain.io.banks <> ballDomain.io.banks + memDomain.io.ballLayoutRequest <> ballDomain.io.layoutRequest + ballDomain.io.layoutResponse <> memDomain.io.ballLayoutResponse + + // --- Shared memory passthrough --- + io.shm.bank.requests <> memDomain.io.shared.bank.requests + io.shm.move <> memDomain.io.shared.move + io.shm.bank.local <> memDomain.io.shared.bank.local + io.shm.bank.config <> memDomain.io.shared.bank.config + io.shm.bank.mapping <> memDomain.io.shared.bank.mapping + + // --- Barrier passthrough --- + io.shm.barrierArrive := frontend.io.barrier.arrive + frontend.io.barrier.release := io.shm.barrierRelease + +} diff --git a/examples/components/cores/reduce/arch/build.sc b/examples/components/cores/reduce/arch/build.sc new file mode 100644 index 000000000..a2650c0fb --- /dev/null +++ b/examples/components/cores/reduce/arch/build.sc @@ -0,0 +1,14 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.{build => examples} + +val componentRoot: os.Path = millSourcePath / os.up + +object library extends SbtModule { + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(examples.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(componentRoot / "arch") +} diff --git a/examples/components/cores/reduce/arch/loom.scala b/examples/components/cores/reduce/arch/loom.scala new file mode 100644 index 000000000..d4af11343 --- /dev/null +++ b/examples/components/cores/reduce/arch/loom.scala @@ -0,0 +1,38 @@ +package examples.core.reduce + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.core.execution.Execution +import framework.core.memory.{MemoryPort, MemorySystem} +import framework.core.controller.admission.Commands +import framework.core.clink.CoreInterrupts +import framework.ip.bus.{axi4, chi} + +@instantiable +class Loom(corep: CoreParam) extends Module { + + @public + val io = IO(new Bundle { + val interrupts = Input(new CoreInterrupts) + val commands = new Commands(corep.memory.dcache.tracking, corep.integer.xLen, corep.privileged.permission) + val drained = Input(Bool()) + val memory = Flipped(new MemoryPort(corep.memory.dcache.tracking, corep.memory.memoryAccess.walker)) + val coherent = Vec(2, new chi.RequesterPort(corep.memory.home.memory.chi)) + val uncached = new axi4.Port(corep.memory.uncached.axi) + }) + + val execution: Instance[Execution] = Instantiate(new Execution(corep.execution(directAccelerator = true), false)) + val memory: Instance[MemorySystem] = Instantiate(new MemorySystem(corep.memory, externalHome = true)) + execution.io.interrupts := io.interrupts + io.commands <> execution.io.controller.get + execution.io.workDrained := io.drained + memory.io.fetch <> execution.io.fetch + memory.io.memory <> execution.io.data + memory.io.timeRequest <> execution.io.timeRequest + execution.io.timeResponse <> memory.io.timeResponse + memory.io.commandAllowed := io.drained + memory.io.services <> io.memory + io.uncached <> memory.io.uncached + io.coherent <> memory.io.coherent +} diff --git a/examples/components/cores/reduce/emu/Cargo.lock b/examples/components/cores/reduce/emu/Cargo.lock new file mode 100644 index 000000000..a9fa2182d --- /dev/null +++ b/examples/components/cores/reduce/emu/Cargo.lock @@ -0,0 +1,822 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "bebop-vm", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "rvsim", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-core-reduce" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rvsim" +version = "0.1.0" +source = "git+https://github.com/DangoSys/riscv-cpu-sim?rev=32abeddb3f0d4c810e07d8d6745f5150e96c8c54#32abeddb3f0d4c810e07d8d6745f5150e96c8c54" +dependencies = [ + "rustc_apfloat", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/components/cores/reduce/emu/Cargo.toml b/examples/components/cores/reduce/emu/Cargo.toml new file mode 100644 index 000000000..15499692c --- /dev/null +++ b/examples/components/cores/reduce/emu/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "bebop-core-reduce" +version = "0.1.0" +edition = "2021" + +[dependencies] +bebop-bemu = { path = "../../../../../bebop/src/nodes/bemu" } diff --git a/examples/components/cores/reduce/emu/src/lib.rs b/examples/components/cores/reduce/emu/src/lib.rs new file mode 100644 index 000000000..b9e4b5a8c --- /dev/null +++ b/examples/components/cores/reduce/emu/src/lib.rs @@ -0,0 +1,39 @@ +use bebop_bemu::{ + ball::linkme, Accelerator, BemuProfileReport, CoreFactory, CoreModel, CoreParams, Cpu, DeviceModel, ModelOptions, + Whatever, CORES, +}; +use std::time::Duration; + +/// The reduce core: one RV64 hart with a Ball accelerator on its RoCC port. +pub struct ReduceCore { + cpu: Cpu, + acc: Accelerator, +} + +impl CoreModel for ReduceCore { + fn cpu(&self) -> &Cpu { + &self.cpu + } + fn cpu_mut(&mut self) -> &mut Cpu { + &mut self.cpu + } + fn step(&mut self, count: u64) -> Result<(), Whatever> { + self.cpu.step(count, &mut self.acc) + } + fn take_barrier(&mut self) -> bool { + self.acc.take_barrier() + } + fn profile_report(&self, total: Duration) -> Option { + self.acc.profile_report(total, &self.cpu) + } +} + +fn instantiate(device: &DeviceModel, params: &CoreParams, options: &ModelOptions) -> Result, String> { + let cpu = Cpu::new(device, params, options).map_err(|error| error.to_string())?; + let acc = Accelerator::attached(&cpu, options)?; + Ok(Box::new(ReduceCore { cpu, acc })) +} + +#[linkme::distributed_slice(CORES)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static REDUCE: CoreFactory = CoreFactory { component: "reduce", instantiate }; diff --git a/examples/components/cores/reduce/stack/CMakeLists.txt b/examples/components/cores/reduce/stack/CMakeLists.txt new file mode 100644 index 000000000..0d00827cb --- /dev/null +++ b/examples/components/cores/reduce/stack/CMakeLists.txt @@ -0,0 +1,41 @@ +# ===----------------------------------------------------------------------=== +# The reduce core: a CPU core with a reduction accelerator; jobs run on the core. +# +# bbdev stack --build "--design " +# +# builds it for each design that uses it, configured with DESIGN_CMAKE +# (generated from design.pb) and CORE. Products: +# execute.o runs jobs on the core itself; links into the device runtime +# lib-compiler.so buddy-opt plugin: Buckyball dialect + the Balls of +# this core's accelerator +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(reduce-core LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +set(TARGET ${DESIGN_CORE_TARGET_${CORE}}) +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# ===----------------------------------------------------------------------=== +# Execution: jobs run on the core itself. +# ===----------------------------------------------------------------------=== +add_custom_command( + OUTPUT execute.o + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" + -c "${RUNTIME}/execute/direct.c" -o execute.o + DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" + VERBATIM) +add_custom_target(execute ALL DEPENDS execute.o) + +# ===----------------------------------------------------------------------=== +# Compiler: the Buckyball dialect and passes with the Balls of the accelerator. +# ===----------------------------------------------------------------------=== +set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${TARGET}}) +set(BUCKYBALL_TARGET_TABLE "${DESIGN_ISA_DIR}/${TARGET}/target.inc") + +add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) +add_library(${TARGET}-compiler MODULE $) +set_target_properties(${TARGET}-compiler PROPERTIES + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") diff --git a/examples/components/cores/rocket/LICENSE.Berkeley b/examples/components/cores/rocket/LICENSE.Berkeley new file mode 100644 index 000000000..7cff15e40 --- /dev/null +++ b/examples/components/cores/rocket/LICENSE.Berkeley @@ -0,0 +1,24 @@ +Copyright (c) 2012-2014, The Regents of the University of California +(Regents). All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the Regents nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING +OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED +HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. diff --git a/examples/components/cores/rocket/LICENSE.SiFive b/examples/components/cores/rocket/LICENSE.SiFive new file mode 100644 index 000000000..7e7093375 --- /dev/null +++ b/examples/components/cores/rocket/LICENSE.SiFive @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2017 SiFive, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/examples/components/cores/rocket/arch/Execution.scala b/examples/components/cores/rocket/arch/Execution.scala new file mode 100644 index 000000000..73d9501c8 --- /dev/null +++ b/examples/components/cores/rocket/arch/Execution.scala @@ -0,0 +1,219 @@ +package examples.core.rocket + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.rocket.configs.Params +import examples.core.rocket.pipeline.{CpuParams, HasCpuParameters, RocketBB} +import freechips.rocketchip.rocket.{CSRs, PMP, PMPChecker, PRV} +import freechips.rocketchip.tile.{CustomCSR, FPU, FPUCoreIO} +import framework.core.clink.CoreInterrupts +import framework.core.execution.fetch.Fetch +import framework.core.memory.access.{UncachedRequest, UncachedResponse, VirtualPort} +import framework.acc.frontend.RoCCIO + +/** ACAL Rocket pipeline attached to the framework's memory and local accelerator protocols. */ +@instantiable +class Execution(config: Params, commandsEnabled: Boolean) extends Module with HasCpuParameters { + implicit lazy val cpuParams: CpuParams = config.cpu + val access = config.memory.memoryAccess + + @public + val io = IO(new Bundle { + val interrupts = Input(new CoreInterrupts) + val fetch = Flipped(new VirtualPort(access)) + val data = Flipped(new VirtualPort(access)) + val timeRequest = Decoupled(new UncachedRequest(access.memory)) + val timeResponse = Flipped(Decoupled(new UncachedResponse(access.memory))) + val rocc = Flipped(new RoCCIO(xLen, config.permission)) + }) + + val pipeline: Instance[RocketBB] = Instantiate(new RocketBB( + Seq(CustomCSR(CSRs.time, (BigInt(1) << 64) - 1, Some(BigInt(0)))), + commandsEnabled, + false, + false, + false, + false + )) + + val fetch: Instance[Fetch] = Instantiate(new Fetch(config.fetch)) + val lsu: Instance[Lsu] = Instantiate(new Lsu(access)) + val cpu = pipeline.io + cpu.hartid := config.hartId.U + cpu.reset_vector := config.resetVector.U + cpu.interrupts := 0.U.asTypeOf(cpu.interrupts) + cpu.interrupts.mtip := io.interrupts.timer + cpu.interrupts.msip := io.interrupts.software + cpu.interrupts.meip := io.interrupts.external + cpu.interrupts.seip.foreach(_ := io.interrupts.supervisorExternal) + cpu.traceStall := false.B + cpu.ptw.perf := 0.U.asTypeOf(cpu.ptw.perf) + cpu.ptw.clock_enabled := true.B + cpu.ptw.customCSRs.csrs.foreach { csr => + csr.stall := false.B + csr.set := false.B + csr.sdata := 0.U + } + val fpu = coreParams.fpu.map(cfg => Instantiate(new FPU(cfg))) + fpu match { + case Some(unit) => + cpu.fpu :<>= unit.io.waiveAs[FPUCoreIO](_.cp_req, _.cp_resp) + unit.io.cp_req.valid := false.B + unit.io.cp_req.bits := 0.U.asTypeOf(unit.io.cp_req.bits) + unit.io.cp_resp.ready := true.B + case None => + cpu.fpu.store_data := 0.U + cpu.fpu.toint_data := 0.U + cpu.fpu.fcsr_flags.valid := false.B + cpu.fpu.fcsr_flags.bits := 0.U + cpu.fpu.fcsr_rdy := true.B + cpu.fpu.nack_mem := false.B + cpu.fpu.illegal_rm := false.B + cpu.fpu.dec := 0.U.asTypeOf(cpu.fpu.dec) + cpu.fpu.sboard_set := false.B + cpu.fpu.sboard_clr := false.B + cpu.fpu.sboard_clra := 0.U + } + + val satp = if (usingVM) Cat(cpu.ptw.ptbr.mode, cpu.ptw.ptbr.asid.pad(16), cpu.ptw.ptbr.ppn.pad(44)) else 0.U(64.W) + lsu.io.cpu <> cpu.dmem + lsu.io.pc := cpu.dmemPc + lsu.io.pmp := cpu.ptw.pmp + lsu.io.context.privilege := cpu.ptw.status.dprv + lsu.io.context.satp := satp + lsu.io.context.sum := cpu.ptw.status.sum + lsu.io.context.mxr := cpu.ptw.status.mxr + // A younger memory operation cannot pass an older unaccepted accelerator command. + val blockFresh = cpu.roccOlderPending && lsu.io.idle + lsu.io.cpu.req.valid := cpu.dmem.req.valid && !blockFresh + cpu.dmem.req.ready := lsu.io.cpu.req.ready && !blockFresh + io.data.request <> lsu.io.request + lsu.io.response <> io.data.response + lsu.io.maintenance.ready := true.B + val maintenancePending = RegInit(false.B) + when(lsu.io.maintenance.fire)(maintenancePending := true.B) + when(lsu.io.maintained.fire)(maintenancePending := false.B) + lsu.io.maintained.valid := maintenancePending + lsu.io.maintained.bits := true.B + io.data.flushTranslation := lsu.io.maintenance.fire + io.fetch.flushTranslation := lsu.io.maintenance.fire + + fetch.io.resetVector := config.resetVector.U + fetch.io.context.privilege := cpu.ptw.status.prv + fetch.io.context.satp := Mux(cpu.ptw.status.prv === PRV.M.U, 0.U, satp) + fetch.io.context.sum := cpu.ptw.status.sum + fetch.io.context.mxr := cpu.ptw.status.mxr + val redirect = if (usingVM) cpu.imem.req.bits.pc.asSInt.pad(64).asUInt else cpu.imem.req.bits.pc.pad(64) + fetch.io.redirect.valid := RegNext(cpu.imem.req.valid, false.B) + fetch.io.redirect.bits := RegEnable(redirect, cpu.imem.req.valid) + fetch.io.flush := RegNext(cpu.imem.flush_icache, false.B) + fetch.io.invalidate := RegNext(cpu.imem.sfence.valid, false.B) + cpu.imem.resp.valid := fetch.io.packet.valid + fetch.io.packet.ready := cpu.imem.resp.ready + cpu.imem.resp.bits := 0.U.asTypeOf(cpu.imem.resp.bits) + cpu.imem.resp.bits.pc := fetch.io.packet.bits.pc + cpu.imem.resp.bits.data := fetch.io.packet.bits.data + cpu.imem.resp.bits.mask := fetch.io.packet.bits.mask + cpu.imem.resp.bits.xcpt.pf.inst := fetch.io.packet.bits.pageFault + cpu.imem.resp.bits.xcpt.ae.inst := fetch.io.packet.bits.accessFault + cpu.imem.npc := fetch.io.npc + cpu.imem.clock_enabled := true.B + cpu.imem.perf := 0.U.asTypeOf(cpu.imem.perf) + cpu.imem.gpa.valid := false.B + cpu.imem.gpa.bits := 0.U + cpu.imem.gpa_is_pte := false.B + io.fetch.request.valid := fetch.io.request.valid + fetch.io.request.ready := io.fetch.request.ready + io.fetch.request.bits := 0.U.asTypeOf(io.fetch.request.bits) + io.fetch.request.bits.vaddr := fetch.io.request.bits.addr + io.fetch.request.bits.size := 3.U + io.fetch.request.bits.execute := true.B + io.fetch.request.bits.privilege := fetch.io.request.bits.context.privilege + io.fetch.request.bits.sum := fetch.io.request.bits.context.sum + io.fetch.request.bits.mxr := fetch.io.request.bits.context.mxr + io.fetch.request.bits.satpMode := fetch.io.request.bits.context.satp(63, 60) + io.fetch.request.bits.rootPpn := fetch.io.request.bits.context.satp(43, 0) + fetch.io.response.valid := io.fetch.response.valid + io.fetch.response.ready := fetch.io.response.ready + fetch.io.response.bits.data := io.fetch.response.bits.data + fetch.io.response.bits.line := 0.U + fetch.io.response.bits.lineExecutable := false.B + fetch.io.response.bits.pageFault := io.fetch.response.bits.pageFault + fetch.io.response.bits.accessFault := io.fetch.response.bits.accessFault + // Coherent I/D caches observe stores through Home; fence.i also discards fetch packets. + fetch.io.maintenance.ready := lsu.io.idle && !io.rocc.busy + val fetchMaintaining = RegInit(false.B) + when(fetch.io.maintenance.fire)(fetchMaintaining := true.B) + when(fetch.io.maintained.fire)(fetchMaintaining := false.B) + fetch.io.maintained.valid := fetchMaintaining + fetch.io.maintained.bits := true.B + + val fetchPmp = Reg(Vec(nPMPs, new PMP)) + when(io.fetch.request.fire)(fetchPmp := cpu.ptw.pmp) + for ((port, pmps) <- Seq(io.fetch -> fetchPmp, io.data -> lsu.io.capturedPmp)) { + val checker = Instantiate(new PMPChecker(3)) + checker.io.prv := port.authorizationRequest.bits.privilege + checker.io.pmp := pmps + checker.io.addr := port.authorizationRequest.bits.paddr + checker.io.size := port.authorizationRequest.bits.size + val pending = RegInit(false.B) + val allowed = Reg(Bool()) + port.authorizationRequest.ready := !pending + port.authorizationResponse.valid := pending + port.authorizationResponse.bits := allowed + when(port.authorizationRequest.fire) { + val req = port.authorizationRequest.bits + allowed := !req.paddr(63, paddrBits).orR && + (!req.read || checker.io.r) && (!req.write || checker.io.w) && (!req.execute || checker.io.x) + pending := true.B + } + when(port.authorizationResponse.fire)(pending := false.B) + } + + // Freeze a custom command only on its architectural acceptance in Rocket's WB stage. + io.rocc.cmd.valid := cpu.rocc.cmd.valid && cpu.roccCommitEligible && lsu.io.idle + cpu.rocc.cmd.ready := io.rocc.cmd.ready && cpu.roccCommitEligible && lsu.io.idle + io.rocc.cmd.bits.instruction.raw_inst := cpu.rocc.cmd.bits.raw_inst + io.rocc.cmd.bits.instruction.pc := cpu.rocc.cmd.bits.pc + io.rocc.cmd.bits.instruction.rs1Data := cpu.rocc.cmd.bits.rs1Data + io.rocc.cmd.bits.instruction.rs2Data := cpu.rocc.cmd.bits.rs2Data + io.rocc.cmd.bits.satp := satp + io.rocc.cmd.bits.effectivePrivilege := cpu.ptw.status.dprv + io.rocc.cmd.bits.sum := cpu.ptw.status.sum + io.rocc.cmd.bits.mxr := cpu.ptw.status.mxr + for ((target, source) <- io.rocc.cmd.bits.pmp.zip(cpu.ptw.pmp)) { + target.cfg := source.cfg.asTypeOf(target.cfg) + target.addr := source.addr + target.mask := source.mask + } + cpu.rocc.busy := io.rocc.busy + cpu.rocc.interrupt := io.rocc.interrupt + cpu.rocc.resp.valid := io.rocc.resp.valid + cpu.rocc.resp.bits.rd := io.rocc.resp.bits.rd + cpu.rocc.resp.bits.data := io.rocc.resp.bits.data + io.rocc.resp.ready := cpu.rocc.resp.ready + + // Read the shared device timer on demand; counting never occupies the AXI bus. + val timeIdle :: timeIssue :: timeWait :: timeReady :: Nil = Enum(4) + val timeState = RegInit(timeIdle) + val time = cpu.rocc.csrs(0) + when(time.ren && timeState === timeIdle)(timeState := timeIssue) + io.timeRequest.valid := timeState === timeIssue + io.timeRequest.bits := 0.U.asTypeOf(io.timeRequest.bits) + io.timeRequest.bits.addr := config.mtimeAddress.U + io.timeRequest.bits.size := 3.U + when(io.timeRequest.fire)(timeState := timeWait) + io.timeResponse.ready := timeState === timeWait + time.stall := timeState =/= timeReady + time.set := io.timeResponse.fire + time.sdata := io.timeResponse.bits.data + when(io.timeResponse.fire) { + assert(!io.timeResponse.bits.error, "CLINT mtime read failed") + timeState := timeReady + } + val retired = cpu.trace.insns(0) + when(retired.valid && !retired.exception && retired.insn(6, 0) === "h73".U && + retired.insn(31, 20) === CSRs.time.U && timeState === timeReady)(timeState := timeIdle) + cpu.interruptReady := lsu.io.idle && timeState === timeIdle && !io.rocc.busy +} diff --git a/examples/components/cores/rocket/arch/Lsu.scala b/examples/components/cores/rocket/arch/Lsu.scala new file mode 100644 index 000000000..a411d6be5 --- /dev/null +++ b/examples/components/cores/rocket/arch/Lsu.scala @@ -0,0 +1,194 @@ +package examples.core.rocket + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public} +import examples.core.rocket.pipeline.{CpuParams, HasCpuParameters} +import freechips.rocketchip.rocket._ +import freechips.rocketchip.rocket.constants.MemoryOpConstants +import framework.ip.bus.chi.rnf.CacheAtomic +import framework.core.memory.access.{VirtualRequest, VirtualResponse} +import framework.core.memory.access.configs.{Params => AccessParam} +import framework.core.execution.fetch.Context + +/** Replay-based Rocket LSU. Effects start only after both pipeline cancellation windows. */ +@instantiable +class Lsu(cp: AccessParam)(implicit val cpuParams: CpuParams) + extends Module + with HasCpuParameters + with MemoryOpConstants { + + @public + val io = IO(new Bundle { + val cpu = Flipped(new HellaCacheIO) + val pc = Input(UInt(vaddrBitsExtended.W)) + val pmp = Input(Vec(nPMPs, new PMP)) + val context = Input(new Context) + val maintenance = Decoupled(new SFenceReq) + val maintained = Flipped(Decoupled(Bool())) + val capturedPmp = Output(Vec(nPMPs, new PMP)) + val request = Decoupled(new VirtualRequest(cp)) + val response = Flipped(Decoupled(new VirtualResponse(cp))) + val idle = Output(Bool()) + val cancelled = Output(Bool()) + }) + + require(!usingHypervisor && cp.memory.tagBits <= coreParams.dcacheReqTagBits) + val idle :: s1 :: s2 :: issue :: receive :: completed :: replay1 :: replay2 :: Nil = Enum(8) + val state = RegInit(idle) + val command = Reg(new HellaCacheReq) + val instructionPc = Reg(UInt(vaddrBitsExtended.W)) + val data = Reg(UInt(64.W)) + val pmps = Reg(Vec(nPMPs, new PMP)) + val context = Reg(new Context) + val maintaining = RegInit(false.B) + io.capturedPmp := pmps + val result = Reg(new VirtualResponse(cp)) + val matches = io.cpu.req.bits.tag === command.tag && io.cpu.req.bits.addr === command.addr && + io.cpu.req.bits.cmd === command.cmd && io.cpu.req.bits.size === command.size && io.pc === instructionPc + io.idle := state === idle + io.cancelled := (state === s1 || state === replay1) && io.cpu.s1_kill || + (state === s2 || state === replay2) && io.cpu.s2_kill + io.cpu.req.ready := state === idle || (state === completed && (!io.cpu.req.valid || matches)) + when(io.cpu.req.fire) { + when(state === idle) { + assert(!io.cpu.req.bits.dv && io.cpu.req.bits.dprv =/= 2.U, "LSU requires nonvirtual U, S or M privilege") + if (io.cpu.req.bits.tag.getWidth > cp.memory.tagBits) { + assert( + !io.cpu.req.bits.tag(io.cpu.req.bits.tag.getWidth - 1, cp.memory.tagBits).orR, + "LSU arbiter tag bits must be zero" + ) + } + command := io.cpu.req.bits + instructionPc := io.pc + pmps := io.pmp + context := io.context + state := s1 + }.otherwise { + assert(matches, "LSU replay changed instruction identity") + state := replay1 + } + } + when(state === s1) { + when(io.cpu.s1_kill)(state := idle) + .otherwise { data := io.cpu.s1_data.data; state := s2 } + } + + val writes = isWrite(command.cmd) + val reads = isRead(command.cmd) + val address = if (usingVM) command.addr.asSInt.pad(64).asUInt else command.addr.pad(64) + val sfence = command.cmd === M_SFENCE + val misaligned = (address & ((1.U(64.W) << command.size) - 1.U)).orR + + when(state === s2) { + when(io.cpu.s2_kill)(state := idle).otherwise { + when(misaligned && !sfence) { + result := 0.U.asTypeOf(result) + result.tag := command.tag(cp.memory.tagBits - 1, 0) + result.misaligned := misaligned + result.accessFault := false.B + state := completed + }.otherwise(state := issue) + } + } + + val atom = MuxLookup(command.cmd, CacheAtomic.None.U)(Seq( + M_XA_SWAP -> CacheAtomic.Swap.U, + M_XA_ADD -> CacheAtomic.Add.U, + M_XA_XOR -> CacheAtomic.Xor.U, + M_XA_AND -> CacheAtomic.And.U, + M_XA_OR -> CacheAtomic.Or.U, + M_XA_MIN -> CacheAtomic.Min.U, + M_XA_MAX -> CacheAtomic.Max.U, + M_XA_MINU -> CacheAtomic.MinU.U, + M_XA_MAXU -> CacheAtomic.MaxU.U, + M_XLR -> CacheAtomic.LR.U, + M_XSC -> CacheAtomic.SC.U + )) + + io.maintenance.valid := state === issue && sfence + io.maintenance.bits := 0.U.asTypeOf(io.maintenance.bits) + io.maintenance.bits.addr := command.addr + io.maintenance.bits.asid := data + io.maintenance.bits.rs1 := command.size(0) + io.maintenance.bits.rs2 := command.size(1) + io.maintained.ready := state === receive && maintaining + when(io.maintenance.fire) { maintaining := true.B; state := receive } + when(io.maintained.fire) { + assert(io.maintained.bits, "LSU maintenance must acknowledge completion") + result := 0.U.asTypeOf(result) + result.tag := command.tag(cp.memory.tagBits - 1, 0) + maintaining := false.B + state := completed + } + io.request.valid := state === issue && !sfence + io.request.bits := 0.U.asTypeOf(io.request.bits) + io.request.bits.vaddr := address + io.request.bits.tag := command.tag(cp.memory.tagBits - 1, 0) + io.request.bits.size := command.size + io.request.bits.write := command.cmd === M_XWR + io.request.bits.execute := false.B + io.request.bits.signed := command.signed + io.request.bits.data := data + io.request.bits.atomic := atom + io.request.bits.privilege := command.dprv + io.request.bits.sum := context.sum + io.request.bits.mxr := context.mxr + io.request.bits.satpMode := context.satp(63, 60) + io.request.bits.rootPpn := context.satp(43, 0) + when(io.request.fire) { + assert( + command.cmd === M_XRD || command.cmd === M_XWR || command.cmd === M_XLR || + command.cmd === M_XSC || isAMO(command.cmd), + "LSU unsupported memory operation" + ) + state := receive + } + io.response.ready := state === receive && !maintaining + when(io.response.fire) { + assert(io.response.bits.tag === command.tag(cp.memory.tagBits - 1, 0), "LSU virtual response tag mismatch") + result := io.response.bits + state := completed + } + when(state === replay1)(state := Mux(io.cpu.s1_kill, completed, replay2)) + when(state === replay2)(state := Mux(io.cpu.s2_kill, completed, idle)) + + val returns = state === replay2 && !io.cpu.s2_kill + val exception = result.misaligned || result.pageFault || result.accessFault + io.cpu.s2_nack := state === s2 && !io.cpu.s2_kill + io.cpu.s2_nack_cause_raw := false.B + io.cpu.s2_uncached := false.B + io.cpu.s2_paddr := command.addr(paddrBits - 1, 0) + io.cpu.s2_gpa := 0.U + io.cpu.s2_gpa_is_pte := false.B + io.cpu.s2_xcpt := 0.U.asTypeOf(io.cpu.s2_xcpt) + io.cpu.s2_xcpt.ma.ld := returns && !writes && result.misaligned + io.cpu.s2_xcpt.ma.st := returns && writes && result.misaligned + io.cpu.s2_xcpt.pf.ld := returns && !writes && result.pageFault + io.cpu.s2_xcpt.pf.st := returns && writes && result.pageFault + io.cpu.s2_xcpt.ae.ld := returns && !writes && result.accessFault + io.cpu.s2_xcpt.ae.st := returns && writes && result.accessFault + io.cpu.resp.valid := returns + io.cpu.resp.bits := 0.U.asTypeOf(io.cpu.resp.bits) + io.cpu.resp.bits.tag := command.tag + io.cpu.resp.bits.cmd := command.cmd + io.cpu.resp.bits.size := command.size + io.cpu.resp.bits.addr := command.addr + io.cpu.resp.bits.signed := command.signed + io.cpu.resp.bits.dprv := command.dprv + io.cpu.resp.bits.has_data := reads && !command.no_resp && !exception + io.cpu.resp.bits.data := result.data + io.cpu.resp.bits.data_word_bypass := result.data + io.cpu.resp.bits.data_raw := result.data + io.cpu.resp.bits.store_data := data + io.cpu.replay_next := false.B + // A buffered completion awaits instruction replay, not a memory transaction; release ordered decoding. + val memoryPending = state === s1 || state === s2 || state === issue || state === receive + io.cpu.ordered := !memoryPending + io.cpu.store_pending := memoryPending && writes + io.cpu.clock_enabled := true.B + io.cpu.perf := 0.U.asTypeOf(io.cpu.perf) + io.cpu.perf.grant := returns + io.cpu.perf.blocked := !io.cpu.req.ready + io.cpu.uncached_resp.foreach { port => port.valid := false.B; port.bits := 0.U.asTypeOf(port.bits) } +} diff --git a/examples/components/cores/rocket/arch/Rocket.scala b/examples/components/cores/rocket/arch/Rocket.scala new file mode 100644 index 000000000..b9c356189 --- /dev/null +++ b/examples/components/cores/rocket/arch/Rocket.scala @@ -0,0 +1,163 @@ +package examples.core.rocket + +import framework.acc.balldomain.bbus.BallFactory +import sims.configs.SimParam +import framework.core.configs.CoreParam +import framework.device.cbus.CoreFactory +import buckyball.config.{CorePlacement, Design} + +import framework.acc.balldomain.ball.blink.BlinkIO + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.rocket.configs.Params +import framework.core.clink.{CLinkIO, HasCLink} +import framework.core.memory.MemorySystem +import framework.acc.frontend.Frontend +import framework.acc.memdomain.MemDomain +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.ball.blink.HasBlink +import framework.acc.debug.Debug + +/** Rocket core with local Ball endpoints. No task threads or TBus are instantiated. */ +@instantiable +class Rocket( + config: Params, + simp: SimParam, + factories: Map[String, BallFactory]) + extends Module + with HasCLink { + val diffTest = simp.diffTest + require(config.acc.forall(a => !a.memDomain.sharedEnable)) + + @public + val clink = IO(new CLinkIO(config.clink)) + + val execution: Instance[Execution] = Instantiate(new Execution(config, config.acc.nonEmpty)) + val memory: Instance[MemorySystem] = Instantiate(new MemorySystem(config.memory, !config.clink.backing)) + execution.io.interrupts := clink.interrupts(0) + memory.io.fetch <> execution.io.fetch + memory.io.memory <> execution.io.data + memory.io.timeRequest <> execution.io.timeRequest + execution.io.timeResponse <> memory.io.timeResponse + clink.memory.uncached <> memory.io.uncached + clink.memory.backing.zip(memory.io.backing).foreach { case (port, source) => port <> source } + clink.memory.coherent <> memory.io.coherent + memory.io.commandAllowed := true.B + + config.acc match { + case Some(acc) => + val frontend: Instance[Frontend] = + Instantiate(new Frontend(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, config.permission, simp)) + val memDomain: Instance[MemDomain] = Instantiate(new MemDomain( + acc.frontend, + acc.memDomain, + acc.gpDomain, + acc.transport, + config.permission, + config.memory.dcache.tracking, + config.memory.memoryAccess.walker, + simp + )) + val domain: Instance[BallDomain] = + Instantiate(new BallDomain( + acc.frontend, + acc.memDomain, + acc.gpDomain, + acc.ballDomain, + config.permission, + factories + )) + if (acc.gpDomain.enable) { + val gp: Instance[GpDomain] = Instantiate(new GpDomain(acc.frontend, acc.memDomain, acc.gpDomain)) + gp.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> gp.io.complete + gp.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> gp.io.banks + memDomain.io.gpLayoutRequest.get <> gp.io.layoutRequest + gp.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + domain.io.kernel.get.request <> gp.io.ball.request + gp.io.ball.response <> domain.io.kernel.get.response + } + frontend.io.rocc <> execution.io.rocc + memory.io.commandAllowed := !frontend.io.rocc.busy + domain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> domain.io.complete + memDomain.io.ball <> domain.io.memory + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + frontend.io.barrier.release := frontend.io.barrier.arrive + memDomain.io.banks <> domain.io.banks + memDomain.io.ballLayoutRequest <> domain.io.layoutRequest + domain.io.layoutResponse <> memDomain.io.ballLayoutResponse + clink.memory.ddr <> memDomain.io.ddr + memory.io.services <> memDomain.io.memory + val debug: Instance[Debug] = + Instantiate(new Debug(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, simp, 0, 0)) + debug.io.ant.valid := false.B + debug.io.ant.bits := 0.U.asTypeOf(debug.io.ant.bits) + debug.io.acc.frontend <> frontend.io.debug + debug.io.acc.trace.memory := memDomain.io.traceMemory + debug.io.acc.trace.ballStarted := domain.io.traceStarted + debug.io.acc.trace.ballCompleted := domain.io.traceCompleted + debug.io.acc.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, source) => sink := source } + for (port <- memDomain.io.shared.bank.requests) { + port.read.req.ready := false.B + port.read.resp.valid := false.B + port.read.resp.bits := 0.U.asTypeOf(port.read.resp.bits) + port.write.req.ready := false.B + port.write.resp.valid := false.B + port.write.resp.bits := 0.U.asTypeOf(port.write.resp.bits) + } + memDomain.io.shared.bank.config.ready := false.B + memDomain.io.shared.bank.mapping.ready := false.B + memDomain.io.shared.bank.local.request.valid := false.B + memDomain.io.shared.bank.local.request.bits := 0.U.asTypeOf(memDomain.io.shared.bank.local.request.bits) + memDomain.io.shared.bank.local.response.ready := false.B + memDomain.io.shared.move.command.ready := false.B + memDomain.io.shared.move.completion.valid := false.B + memDomain.io.shared.move.completion.bits := false.B + case None => + memory.io.services.cpuAllow := true.B + memory.io.services.maintenance.valid := false.B + memory.io.services.maintenance.bits := 0.U.asTypeOf(memory.io.services.maintenance.bits) + memory.io.services.pteRequest.valid := false.B + memory.io.services.pteRequest.bits := 0.U.asTypeOf(memory.io.services.pteRequest.bits) + memory.io.services.maintained.ready := true.B + memory.io.services.pteResponse.ready := true.B + execution.io.rocc.cmd.ready := false.B + execution.io.rocc.resp.valid := false.B + execution.io.rocc.resp.bits := 0.U.asTypeOf(execution.io.rocc.resp.bits) + execution.io.rocc.busy := false.B + execution.io.rocc.interrupt := false.B + clink.memory.ddr.aw.valid := false.B + clink.memory.ddr.aw.bits := 0.U.asTypeOf(clink.memory.ddr.aw.bits) + clink.memory.ddr.w.valid := false.B + clink.memory.ddr.w.bits := 0.U.asTypeOf(clink.memory.ddr.w.bits) + clink.memory.ddr.ar.valid := false.B + clink.memory.ddr.ar.bits := 0.U.asTypeOf(clink.memory.ddr.ar.bits) + clink.memory.ddr.b.ready := true.B + clink.memory.ddr.r.ready := true.B + } +} + +object RocketComponent extends CoreFactory { + val component = "rocket" + + def parameters(design: Design, source: CorePlacement): CoreParam = { + val native = Params.fromProto(source); require(native.acc.isEmpty); + CoreParam.fromProto(source, Seq.empty).copy(clink = native.clink) + } + + def instantiate( + source: CorePlacement, + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int] + ): CLinkIO = + Instantiate(new Rocket(Params.fromProto(source), simp, Map.empty)).clink + +} diff --git a/examples/components/cores/rocket/arch/build.sc b/examples/components/cores/rocket/arch/build.sc new file mode 100644 index 000000000..bb3c69dfc --- /dev/null +++ b/examples/components/cores/rocket/arch/build.sc @@ -0,0 +1,42 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.^.arch.thirdparty.{build => thirdparty} +import $file.^.^.^.^.^.arch.src.main.scala.framework.{build => framework} + +val root: os.Path = millSourcePath / os.up +trait ScalaBuild extends SbtModule { + override def scalaVersion = "2.13.16" + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0", ivy"com.lihaoyi::upickle:3.3.1") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") +} +object cde extends ScalaBuild { + override def sources = T.sources(root / "thirdparty" / "rocket-chip" / "dependencies" / "cde" / "cde" / "src") +} +object pipeline extends ScalaBuild { + override def moduleDeps = Seq(cde, thirdparty.hardfloat, thirdparty.midas_target_utils, framework.library) + override def sources = T.sources { + val src = root / "thirdparty" / "rocket-chip" / "src" / "main" / "scala" + val files = Seq( + "rocket/ALU.scala", "rocket/AMOALU.scala", "rocket/BTB.scala", "rocket/Breakpoint.scala", + "rocket/CSR.scala", "rocket/Consts.scala", "rocket/CustomInstructions.scala", "rocket/DebugROB.scala", + "rocket/Decode.scala", "rocket/Events.scala", "rocket/IBuf.scala", "rocket/IDecode.scala", + "rocket/Instructions.scala", "rocket/Instructions32.scala", "rocket/Multiplier.scala", "rocket/PMP.scala", + "rocket/RVC.scala", "rocket/package.scala", "tile/FPU.scala", "tile/CustomCSRs.scala", + "util/Misc.scala", "util/ClockGate.scala", "util/CoreMonitor.scala", "util/Property.scala", + "util/Counters.scala", "util/ECC.scala", "util/Replacement.scala", "util/ShiftReg.scala", + "util/ShiftQueue.scala", "util/HellaQueue.scala", "util/LCG.scala", "util/PlusArg.scala", + "util/MuxLiteral.scala", "util/Arbiters.scala", "util/AsyncResetReg.scala", "util/Timer.scala", + "util/GeneratorUtils.scala", "unittest/UnitTest.scala" + ) + files.map(f => PathRef(src / os.RelPath(f))) ++ Seq(PathRef(root / "arch" / "pipeline")) + } + override def resources = T.sources(root / "thirdparty" / "rocket-chip" / "src" / "main" / "resources") +} +object library extends ScalaBuild { + override def moduleDeps = Seq(pipeline, framework.library) + override def sources = T.sources { + os.walk(root / "arch").filter(p => p.ext == "scala" && !p.toString.contains("/pipeline/")).map(PathRef(_)) ++ + Seq(PathRef(root / "configs")) + } +} diff --git a/examples/components/cores/rocket/arch/pipeline/Config.scala b/examples/components/cores/rocket/arch/pipeline/Config.scala new file mode 100644 index 000000000..91b5c6565 --- /dev/null +++ b/examples/components/cores/rocket/arch/pipeline/Config.scala @@ -0,0 +1,492 @@ +// See LICENSE.SiFive for license details. + +package freechips.rocketchip.tile { + + import chisel3._ + import chisel3.util.isPow2 + import org.chipsalliance.cde.config._ + import freechips.rocketchip.rocket._ + import freechips.rocketchip.util._ + + case object MaxHartIdBits extends Field[Int] + case object CorePAddrBits extends Field[Int] + +// These parameters can be varied per-core + trait CoreParams { + val bootFreqHz: BigInt + val useVM: Boolean + val useHypervisor: Boolean + val useUser: Boolean + val useSupervisor: Boolean + val useDebug: Boolean + val useAtomics: Boolean + val useAtomicsOnlyForIO: Boolean + val useCompressed: Boolean + val useVector: Boolean = false + val vectorUseDCache: Boolean = false + val useRVE: Boolean + val useConditionalZero: Boolean + val useZba: Boolean + val useZbb: Boolean + val useZbs: Boolean + val mulDiv: Option[MulDivParams] + val fpu: Option[FPUParams] + val fetchWidth: Int + val decodeWidth: Int + val retireWidth: Int + val instBits: Int + val nLocalInterrupts: Int + val useNMI: Boolean + val nPMPs: Int + val pmpGranularity: Int + val nBreakpoints: Int + val useBPWatch: Boolean + val mcontextWidth: Int + val scontextWidth: Int + val nPerfCounters: Int + val haveBasicCounters: Boolean + val haveFSDirty: Boolean + val misaWritable: Boolean + val haveCFlush: Boolean + val nL2TLBEntries: Int + val nL2TLBWays: Int + val nPTECacheEntries: Int + val mtvecInit: Option[BigInt] + val mtvecWritable: Boolean + val traceHasWdata: Boolean + val xLen: Int + val pgLevels: Int + def traceCustom: Option[Data] = None + def customIsaExt: Option[String] = None + def customCSRs(implicit p: Parameters): CustomCSRs = new CustomCSRs + + def hasSupervisorMode: Boolean = useSupervisor || useVM + def instBytes: Int = instBits / 8 + def fetchBytes: Int = fetchWidth * instBytes + def lrscCycles: Int + + def dcacheReqTagBits: Int = 6 + + def minFLen: Int = 32 + + def vLen: Int = 0 + def eLen: Int = 0 + def vfLen: Int = 0 + def vfh: Boolean = false + def vExts: Seq[String] = Nil + def hasV: Boolean = vLen >= 128 && eLen >= 64 && vfLen >= 64 + def vMemDataBits: Int = 0 + + def useBitmanip = useZba && useZbb && useZbs + } + + trait HasCoreParameters extends HasTileParameters { + + override lazy val paddrBits: Int = { + val bits = p(CorePAddrBits) + require(bits <= maxPAddrBits) + bits + } + + val coreParams: CoreParams = tileParams.core + + val minFLen = coreParams.fpu.map(_ => coreParams.minFLen).getOrElse(0) + val fLen = coreParams.fpu.map(_.fLen).getOrElse(0) + + val usingMulDiv = coreParams.mulDiv.nonEmpty + val usingFPU = coreParams.fpu.nonEmpty + val usingAtomics = coreParams.useAtomics + val usingAtomicsOnlyForIO = coreParams.useAtomicsOnlyForIO + val usingAtomicsInCache = usingAtomics && !usingAtomicsOnlyForIO + val usingCompressed = coreParams.useCompressed + val usingVector = coreParams.useVector + val usingNMI = coreParams.useNMI + val usingConditionalZero = coreParams.useConditionalZero + + val retireWidth = coreParams.retireWidth + val fetchWidth = coreParams.fetchWidth + val decodeWidth = coreParams.decodeWidth + + val fetchBytes = coreParams.fetchBytes + val coreInstBits = coreParams.instBits + val coreInstBytes = coreInstBits / 8 + val coreDataBits = xLen max fLen max vMemDataBits + val coreDataBytes = coreDataBits / 8 + def coreMaxAddrBits = paddrBits max vaddrBitsExtended + + val nBreakpoints = coreParams.nBreakpoints + val nPMPs = coreParams.nPMPs + val pmpGranularity = coreParams.pmpGranularity + val nPerfCounters = coreParams.nPerfCounters + val mtvecInit = coreParams.mtvecInit + val mtvecWritable = coreParams.mtvecWritable + val customIsaExt = coreParams.customIsaExt + val traceHasWdata = coreParams.traceHasWdata + + def vLen = coreParams.vLen + def eLen = coreParams.eLen + def vfLen = coreParams.vfLen + def vMemDataBits = if (usingVector) coreParams.vMemDataBits else 0 + def maxVLMax = vLen + + if (usingVector) { + require(isPow2(vLen), s"vLen ($vLen) must be a power of 2") + require(eLen >= 32 && vLen % eLen == 0, s"eLen must divide vLen ($vLen) and be no less than 32") + require(eLen == 32 || eLen == 64) + require(vfLen <= eLen) + require(!coreParams.vfh || (vfLen >= 32 && coreParams.minFLen <= 16)) + } + + if (coreParams.useVM) { + if (coreParams.xLen == 32) { + require(coreParams.pgLevels == 2) + } else { + require(coreParams.pgLevels >= 3) + } + } + + lazy val hartIdLen: Int = p(MaxHartIdBits) + + lazy val resetVectorLen: Int = { + val externalLen = paddrBits + require(externalLen <= xLen, s"External reset vector length ($externalLen) must be <= XLEN ($xLen)") + require( + externalLen <= vaddrBitsExtended, + s"External reset vector length ($externalLen) must be <= virtual address bit width ($vaddrBitsExtended)" + ) + externalLen + } + + // Print out log of committed instructions and their writeback values. + // Requires post-processing due to out-of-order writebacks. + val enableCommitLog = false + + } + + abstract class CoreModule(implicit val p: Parameters) extends Module with HasCoreParameters + + abstract class CoreBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) with HasCoreParameters + +// This is a raw commit trace from the core, not the TraceCoreInterface + class TraceBundle(implicit val p: Parameters) extends Bundle with HasCoreParameters { + val insns = Vec(coreParams.retireWidth, new TracedInstruction) + val time = UInt(64.W) + val custom = coreParams.traceCustom + } + +} + +package examples.core.rocket.pipeline { + + import freechips.rocketchip.rocket.{BTBParams, DCacheParams, ICacheParams, RocketCoreParams} + + /** Explicit CPU metadata; the upstream parameter conversion stays in the Rocket adapter. */ + case class CpuParams( + core: RocketCoreParams, + dcache: Option[DCacheParams], + icache: Option[ICacheParams], + btb: Option[BTBParams], + physicalAddressBits: Int, + hartIdBits: Int, + beatBytes: Int = 8, + blockBytes: Int = 64, + asIdBits: Int = 0) + +} + +package examples.core.rocket.pipeline { + + import org.chipsalliance.cde.config.Parameters + import freechips.rocketchip.tile.{ + CoreBeatBytes, + CorePAddrBits, + CpuTileParams, + HasCoreParameters, + MaxHartIdBits, + TileKey + } + import freechips.rocketchip.rocket.ASIdBits + import freechips.rocketchip.subsystem.CacheBlockBytes + + object RocketParameters { + + def apply(cpu: CpuParams): Parameters = Parameters.empty.alterPartial { + case TileKey => CpuTileParams(core = cpu.core, dcache = cpu.dcache, icache = cpu.icache, btb = cpu.btb) + case CorePAddrBits => cpu.physicalAddressBits + case MaxHartIdBits => cpu.hartIdBits + case CoreBeatBytes => cpu.beatBytes + case CacheBlockBytes => cpu.blockBytes + case ASIdBits => cpu.asIdBits + } + + } + + trait HasCpuParameters extends HasCoreParameters { + implicit val cpuParams: CpuParams + implicit lazy val p: Parameters = RocketParameters(cpuParams) + } + +} + +// See LICENSE.Berkeley and LICENSE.SiFive for Rocket CSR semantics. +package freechips.rocketchip.rocket { + + import chisel3._ + import org.chipsalliance.cde.config.Parameters + import freechips.rocketchip.tile._ + import freechips.rocketchip.util._ + + trait HasRocketCoreParameters extends HasCoreParameters { + lazy val rocketParams = tileParams.core.asInstanceOf[RocketCoreParams] + val fastLoadWord = rocketParams.fastLoadWord + val fastLoadByte = rocketParams.fastLoadByte + val mulDivParams = rocketParams.mulDiv.getOrElse(MulDivParams()) + require(!fastLoadByte || fastLoadWord) + require(!rocketParams.haveFSDirty) + } + + class RocketCustomCSRs(implicit p: Parameters) extends CustomCSRs with HasRocketCoreParameters { + override def bpmCSR = rocketParams.branchPredictionModeCSR.option(CustomCSR(bpmCSRId, 1, Some(BigInt(0)))) + + override def chickenCSR = { + val dc = tileParams.dcache.get + val mask = BigInt(dc.clockGate.toInt | (rocketParams.clockGate.toInt << 1) | + (rocketParams.clockGate.toInt << 2) | (1 << 3) | (dc.scratch.isEmpty.toInt << 9) | + (tileParams.icache.get.prefetch.toInt << 17)) + Some(CustomCSR(chickenCSRId, mask, Some(mask))) + } + + def disableICachePrefetch = getOrElse(chickenCSR, _.value(17), true.B) + def marchid = CustomCSR.constant(CSRs.marchid, BigInt(1)) + def mvendorid = CustomCSR.constant(CSRs.mvendorid, BigInt(rocketParams.mvendorid)) + def mimpid = CustomCSR.constant(CSRs.mimpid, BigInt(rocketParams.mimpid)) + override def decls = super.decls :+ marchid :+ mvendorid :+ mimpid + } + +} + +// See LICENSE.Berkeley and LICENSE.SiFive for Rocket interface definitions. +package freechips.rocketchip.tile { + + import org.chipsalliance.cde.config.{Field, Parameters} + import chisel3.util.{log2Ceil, log2Up} + import freechips.rocketchip.rocket._ + + case object TileKey extends Field[CpuTileParams] + case object CoreBeatBytes extends Field[Int] + case object CoreRoCCCount extends Field[Int](0) + + trait HasTileParameters { + implicit val p: Parameters + def tileParams: CpuTileParams = p(TileKey) + def usingVM = tileParams.core.useVM + def usingSupervisor = tileParams.core.hasSupervisorMode + def usingUser = tileParams.core.useUser || usingSupervisor + def usingHypervisor = usingVM && tileParams.core.useHypervisor + def usingDebug = tileParams.core.useDebug + def usingRoCC = p(CoreRoCCCount) > 0 + def usingBTB = tileParams.btb.exists(_.nEntries > 0) + def usingPTW = usingVM + def usingDataScratchpad = tileParams.dcache.flatMap(_.scratch).isDefined + def xLen = tileParams.core.xLen + def xBytes = xLen / 8 + def iLen = 32 + def pgIdxBits = 12 + def pgLevelBits = 10 - log2Ceil(xLen / 32) + def pgLevels = tileParams.core.pgLevels + def minPgLevels = { val levels = if (xLen == 32) 2 else 3; require(pgLevels >= levels); levels } + def maxSVAddrBits = pgIdxBits + pgLevels * pgLevelBits + def maxHypervisorExtraAddrBits = 2 + def hypervisorExtraAddrBits = if (usingHypervisor) maxHypervisorExtraAddrBits else 0 + def maxHVAddrBits = maxSVAddrBits + hypervisorExtraAddrBits + def asIdBits = p(ASIdBits) + def vmIdBits = p(VMIdBits) + + lazy val maxPAddrBits: Int = { + require(xLen == 32 || xLen == 64); if (!usingVM) xLen else if (xLen == 32) 34 else 56 + } + + lazy val paddrBits: Int = p(CorePAddrBits) + + def vaddrBits: Int = + if (usingVM) { + val bits = maxHVAddrBits + require(bits == xLen || xLen > bits && bits > paddrBits) + bits + } else (paddrBits + 1).min(xLen) + + def vpnBits = vaddrBits - pgIdxBits + def ppnBits = paddrBits - pgIdxBits + def vpnBitsExtended = vpnBits + (if (vaddrBits < xLen) 1 + (if (usingHypervisor) 1 else 0) else 0) + def vaddrBitsExtended = vpnBitsExtended + pgIdxBits + def cacheBlockBytes = p(freechips.rocketchip.subsystem.CacheBlockBytes) + def lgCacheBlockBytes = log2Up(cacheBlockBytes) + def masterPortBeatBytes = p(CoreBeatBytes) + def dcacheArbPorts = 1 + (if (usingVM) 1 else 0) + (if (usingDataScratchpad) 1 else 0) + p(CoreRoCCCount) + + (if (tileParams.core.useVector && tileParams.core.vectorUseDCache) 1 else 0) + } + +} + +package freechips.rocketchip.rocket { + import org.chipsalliance.cde.config.Field + case object ASIdBits extends Field[Int](0) + case object VMIdBits extends Field[Int](0) +} + +package freechips.rocketchip.subsystem { + import org.chipsalliance.cde.config.Field + case object CacheBlockBytes extends Field[Int] +} + +package freechips.rocketchip.devices.debug { + import org.chipsalliance.cde.config.Field + case object DebugModuleKey extends Field[Option[DebugModuleParams]](Some(DebugModuleParams())) +} + +package freechips.rocketchip.rocket { + import chisel3._ + import chisel3.util._ + import org.chipsalliance.cde.config.Parameters + import freechips.rocketchip.tile._ + + case class DCacheParams( + nSets: Int = 64, + nWays: Int = 4, + rowBits: Int = 64, + subWordBits: Option[Int] = None, + replacementPolicy: String = "random", + nTLBSets: Int = 1, + nTLBWays: Int = 32, + nTLBBasePageSectors: Int = 4, + nTLBSuperpages: Int = 4, + tagECC: Option[String] = None, + dataECC: Option[String] = None, + dataECCBytes: Int = 1, + nMSHRs: Int = 1, + nSDQ: Int = 17, + nRPQ: Int = 16, + nMMIOs: Int = 1, + blockBytes: Int = 64, + separateUncachedResp: Boolean = false, + acquireBeforeRelease: Boolean = false, + pipelineWayMux: Boolean = false, + clockGate: Boolean = false, + scratch: Option[BigInt] = None) { + require(scratch.isEmpty || nWays == 1) + require(scratch.isEmpty || nMSHRs == 0) + if (scratch.isEmpty) require(isPow2(nSets)) + } + + case class ICacheParams( + nSets: Int = 64, + nWays: Int = 4, + rowBits: Int = 128, + nTLBSets: Int = 1, + nTLBWays: Int = 32, + nTLBBasePageSectors: Int = 4, + nTLBSuperpages: Int = 4, + cacheIdBits: Int = 0, + tagECC: Option[String] = None, + dataECC: Option[String] = None, + itimAddr: Option[BigInt] = None, + prefetch: Boolean = false, + blockBytes: Int = 64, + latency: Int = 2, + fetchBytes: Int = 4) + + case class RocketCoreParams( + xLen: Int = 64, + pgLevels: Int = 3, + bootFreqHz: BigInt = 0, + useVM: Boolean = true, + useUser: Boolean = false, + useSupervisor: Boolean = false, + useHypervisor: Boolean = false, + useDebug: Boolean = true, + useAtomics: Boolean = true, + useAtomicsOnlyForIO: Boolean = false, + useCompressed: Boolean = true, + useRVE: Boolean = false, + useConditionalZero: Boolean = false, + useZba: Boolean = false, + useZbb: Boolean = false, + useZbs: Boolean = false, + nLocalInterrupts: Int = 0, + useNMI: Boolean = false, + nBreakpoints: Int = 1, + useBPWatch: Boolean = false, + mcontextWidth: Int = 0, + scontextWidth: Int = 0, + nPMPs: Int = 8, + nPerfCounters: Int = 0, + haveBasicCounters: Boolean = true, + haveCFlush: Boolean = false, + misaWritable: Boolean = true, + nL2TLBEntries: Int = 0, + nL2TLBWays: Int = 1, + nPTECacheEntries: Int = 8, + mtvecInit: Option[BigInt] = Some(BigInt(0)), + mtvecWritable: Boolean = true, + fastLoadWord: Boolean = true, + fastLoadByte: Boolean = false, + branchPredictionModeCSR: Boolean = false, + clockGate: Boolean = false, + mvendorid: Int = 0, + mimpid: Int = 0x20181004, + mulDiv: Option[MulDivParams] = Some(MulDivParams()), + fpu: Option[FPUParams] = Some(FPUParams()), + debugROB: Option[DebugROBParams] = None, + haveCease: Boolean = true, + haveSimTimeout: Boolean = true, + vector: Option[RocketCoreVectorParams] = None, + enableTraceCoreIngress: Boolean = false) + extends CoreParams { + val lgPauseCycles = 5 + val haveFSDirty = false + val pmpGranularity = if (useHypervisor) 4096 else 4 + val fetchWidth = if (useCompressed) 2 else 1 + val decodeWidth = fetchWidth / (if (useCompressed) 2 else 1) + val retireWidth = 1 + val instBits = if (useCompressed) 16 else 32 + val lrscCycles = 80 + val traceHasWdata = debugROB.isDefined + override def minFLen = fpu.map(_.minFLen).getOrElse(32) + override def customCSRs(implicit p: Parameters) = new RocketCustomCSRs + override val useVector = vector.isDefined + override val vectorUseDCache = vector.exists(_.useDCache) + override def vLen = vector.map(_.vLen).getOrElse(0) + override def eLen = vector.map(_.eLen).getOrElse(0) + override def vfLen = vector.map(_.vfLen).getOrElse(0) + override def vfh = vector.exists(_.vfh) + override def vMemDataBits = vector.map(_.vMemDataBits).getOrElse(0) + override def vExts = vector.map(_.vExts).getOrElse(Nil) + } + + case class RocketCoreVectorParams( + vLen: Int, + eLen: Int, + vfLen: Int, + vfh: Boolean, + vMemDataBits: Int, + decoder: Parameters => RocketVectorDecoder, + useDCache: Boolean, + issueVConfig: Boolean, + vExts: Seq[String]) + +} + +package freechips.rocketchip.tile { + import freechips.rocketchip.rocket._ + + case class CpuTileParams( + core: CoreParams, + dcache: Option[DCacheParams], + icache: Option[ICacheParams], + btb: Option[BTBParams]) + +} + +package freechips.rocketchip.devices.debug { + case class DebugModuleParams(nDscratch: Int = 1, debugEntry: BigInt = 0x800, debugException: BigInt = 0x808) +} diff --git a/examples/components/cores/rocket/arch/pipeline/Pipeline.scala b/examples/components/cores/rocket/arch/pipeline/Pipeline.scala new file mode 100644 index 000000000..cab1b82a1 --- /dev/null +++ b/examples/components/cores/rocket/arch/pipeline/Pipeline.scala @@ -0,0 +1,1810 @@ +// See LICENSE.Berkeley for license details. +// See LICENSE.SiFive for license details. + +package examples.core.rocket.pipeline { + + import chisel3.experimental.hierarchy.Definition + + import framework.lib.arbiter.{Arbiter} + import chisel3.experimental.hierarchy.{Instance, Instantiate} + + import chisel3._ + import chisel3.experimental.hierarchy.{instantiable, public} + import chisel3.util._ + import chisel3.withClock + import freechips.rocketchip.tile._ + import freechips.rocketchip.util._ + import freechips.rocketchip.util.property + import scala.collection.mutable.ArrayBuffer + import freechips.rocketchip.rocket._ + + /** RoCC core IO — used inside Rocket core. */ + class RoCCCoreIOBB(val nRoCCCSRs: Int = 0)(implicit val cpuParams: CpuParams) extends Bundle with HasCpuParameters { + val cmd = Flipped(Decoupled(new RoCCCommandBB)) + val resp = Decoupled(new RoCCResponseBB) + val busy = Output(Bool()) + val interrupt = Output(Bool()) + val exception = Input(Bool()) + val csrs = Flipped(Vec(nRoCCCSRs, new CustomCSRIO)) + } + + @instantiable + class RocketBB( + roccCSRs: Seq[CustomCSR], + coreHasBuckyball: Boolean, + hasTaskControl: Boolean, + val hasBusError: Boolean, + dcacheFlushOnFenceI: Boolean, + dcacheCanSupportCFlushLine: Boolean + )( + implicit val cpuParams: CpuParams) + extends Module + with HasRocketCoreParameters + with HasCpuParameters { + def nTotalRoCCCSRs = roccCSRs.size + + override val usingRoCC = coreHasBuckyball || hasTaskControl + + @public + val io = IO(new CoreBundle()(p) { + val hartid = Input(UInt(hartIdLen.W)) + val reset_vector = Input(UInt(resetVectorLen.W)) + val interrupts = Input(new CoreInterrupts(hasBusError)) + val interruptReady = Input(Bool()) + val imem = new FrontendIO + val dmem = new HellaCacheIO + val dmemPc = Output(UInt(vaddrBitsExtended.W)) + val roccOlderPending = Output(Bool()) + val roccCommitEligible = Output(Bool()) + val ptw = Flipped(new DatapathPTWIO()) + val fpu = Flipped(new FPUCoreIO()) + val rocc = Flipped(new RoCCCoreIOBB(nTotalRoCCCSRs)) + val trace = Output(new TraceBundle) + val bpwatch = Output(Vec(coreParams.nBreakpoints, new BPWatch(coreParams.retireWidth))) + val cease = Output(Bool()) + val wfi = Output(Bool()) + val traceStall = Input(Bool()) + val vector = if (usingVector) Some(Flipped(new VectorCoreIO)) else None + }) + + import ALU._ + + val clock_en_reg = RegInit(true.B) + val long_latency_stall = Reg(Bool()) + val id_reg_pause = Reg(Bool()) + val imem_might_request_reg = Reg(Bool()) + val clock_en = WireDefault(true.B) + + val gated_clock = + if (!rocketParams.clockGate) clock + else ClockGate(clock, clock_en, "rocket_clock_gate") + + class RocketImpl { // entering gated-clock domain + + // performance counters + def pipelineIDToWB[T <: Data](x: T): T = + RegEnable(RegEnable(RegEnable(x, !ctrl_killd), ex_pc_valid), mem_pc_valid) + + val perfEvents = new EventSets(Seq( + new EventSet( + (mask, hits) => Mux(wb_xcpt, mask(0), wb_valid && pipelineIDToWB((mask & hits).orR)), + Seq( + ("exception", () => false.B), + ("load", () => id_ctrl.mem && id_ctrl.mem_cmd === M_XRD && !id_ctrl.fp), + ("store", () => id_ctrl.mem && id_ctrl.mem_cmd === M_XWR && !id_ctrl.fp), + ( + "amo", + () => usingAtomics.B && id_ctrl.mem && (isAMO(id_ctrl.mem_cmd) || id_ctrl.mem_cmd.isOneOf(M_XLR, M_XSC)) + ), + ("system", () => id_ctrl.csr =/= CSR.N), + ( + "arith", + () => + id_ctrl.wxd && !(id_ctrl.jal || id_ctrl.jalr || id_ctrl.mem || id_ctrl.fp || id_ctrl.mul || id_ctrl.div || id_ctrl.csr =/= CSR.N) + ), + ("branch", () => id_ctrl.branch), + ("jal", () => id_ctrl.jal), + ("jalr", () => id_ctrl.jalr) + ) + ++ (if (!usingMulDiv) Seq() + else + Seq( + ("mul", () => if (pipelinedMul) id_ctrl.mul else id_ctrl.div && (id_ctrl.alu_fn & FN_DIV) =/= FN_DIV), + ("div", () => if (pipelinedMul) id_ctrl.div else id_ctrl.div && (id_ctrl.alu_fn & FN_DIV) === FN_DIV) + )) + ++ (if (!usingFPU) Seq() + else + Seq( + ("fp load", () => id_ctrl.fp && io.fpu.dec.ldst && io.fpu.dec.wen), + ("fp store", () => id_ctrl.fp && io.fpu.dec.ldst && !io.fpu.dec.wen), + ("fp add", () => id_ctrl.fp && io.fpu.dec.fma && io.fpu.dec.swap23), + ("fp mul", () => id_ctrl.fp && io.fpu.dec.fma && !io.fpu.dec.swap23 && !io.fpu.dec.ren3), + ("fp mul-add", () => id_ctrl.fp && io.fpu.dec.fma && io.fpu.dec.ren3), + ("fp div/sqrt", () => id_ctrl.fp && (io.fpu.dec.div || io.fpu.dec.sqrt)), + ( + "fp other", + () => id_ctrl.fp && !(io.fpu.dec.ldst || io.fpu.dec.fma || io.fpu.dec.div || io.fpu.dec.sqrt) + ) + )) + ), + new EventSet( + (mask, hits) => (mask & hits).orR, + Seq( + ( + "load-use interlock", + () => id_ex_hazard && ex_ctrl.mem || id_mem_hazard && mem_ctrl.mem || id_wb_hazard && wb_ctrl.mem + ), + ("long-latency interlock", () => id_sboard_hazard), + ( + "csr interlock", + () => + id_ex_hazard && ex_ctrl.csr =/= CSR.N || id_mem_hazard && mem_ctrl.csr =/= CSR.N || id_wb_hazard && wb_ctrl.csr =/= CSR.N + ), + ("I$ blocked", () => icache_blocked), + ("D$ blocked", () => id_ctrl.mem && dcache_blocked), + ("branch misprediction", () => take_pc_mem && mem_direction_misprediction), + ( + "control-flow target misprediction", + () => take_pc_mem && mem_misprediction && mem_cfi && !mem_direction_misprediction && !icache_blocked + ), + ("flush", () => wb_reg_flush_pipe), + ("replay", () => replay_wb) + ) + ++ (if (!usingMulDiv) Seq() + else + Seq( + ( + "mul/div interlock", + () => + id_ex_hazard && (ex_ctrl.mul || ex_ctrl.div) || id_mem_hazard && (mem_ctrl.mul || mem_ctrl.div) || id_wb_hazard && wb_ctrl.div + ) + )) + ++ (if (!usingFPU) Seq() + else + Seq( + ( + "fp interlock", + () => + id_ex_hazard && ex_ctrl.fp || id_mem_hazard && mem_ctrl.fp || id_wb_hazard && wb_ctrl.fp || id_ctrl.fp && id_stall_fpu + ) + )) + ), + new EventSet( + (mask, hits) => (mask & hits).orR, + Seq( + ("I$ miss", () => io.imem.perf.acquire), + ("D$ miss", () => io.dmem.perf.acquire), + ("D$ release", () => io.dmem.perf.release), + ("ITLB miss", () => io.imem.perf.tlbMiss), + ("DTLB miss", () => io.dmem.perf.tlbMiss), + ("L2 TLB miss", () => io.ptw.perf.l2miss) + ) + ) + )) + + val pipelinedMul = usingMulDiv && mulDivParams.mulUnroll == xLen + + val decode_table = { + (if (usingMulDiv) new MDecode(pipelinedMul) +: (xLen > 32).option(new M64Decode(pipelinedMul)).toSeq + else Nil) ++: + (if (usingAtomics) new ADecode +: (xLen > 32).option(new A64Decode).toSeq else Nil) ++: + (if (fLen >= 32) new FDecode +: (xLen > 32).option(new F64Decode).toSeq else Nil) ++: + (if (fLen >= 64) new DDecode +: (xLen > 32).option(new D64Decode).toSeq else Nil) ++: + (if (minFLen == 16) + new HDecode +: (xLen > 32).option(new H64Decode).toSeq ++: (fLen >= 64).option(new HDDecode).toSeq + else Nil) ++: + (usingRoCC.option(new RoCCDecode)) ++: + (if (xLen == 32) new I32Decode else new I64Decode) +: + (usingVM.option(new SVMDecode)) ++: + (usingSupervisor.option(new SDecode)) ++: + (usingHypervisor.option(new HypervisorDecode)) ++: + ((usingHypervisor && (xLen == 64)).option(new Hypervisor64Decode)) ++: + (usingDebug.option(new DebugDecode)) ++: + (usingNMI.option(new NMIDecode)) ++: + (usingConditionalZero.option(new ConditionalZeroDecode)) ++: + Seq(new FenceIDecode(dcacheFlushOnFenceI || coreParams.haveCFlush)) ++: + coreParams.haveCFlush.option(new CFlushDecode(dcacheCanSupportCFlushLine)) ++: + rocketParams.haveCease.option(new CeaseDecode) ++: + usingVector.option(new VCFGDecode) ++: + (if (coreParams.useZba) new ZbaDecode +: (xLen > 32).option(new Zba64Decode).toSeq else Nil) ++: + (if (coreParams.useZbb) Seq(new ZbbDecode, if (xLen == 32) new Zbb32Decode else new Zbb64Decode) else Nil) ++: + coreParams.useZbs.option(new ZbsDecode) ++: + Seq(new IDecode) + } flatMap (_.table) + + val ex_ctrl = Reg(new IntCtrlSigs) + val mem_ctrl = Reg(new IntCtrlSigs) + val wb_ctrl = Reg(new IntCtrlSigs) + + val ex_reg_xcpt_interrupt = Reg(Bool()) + val ex_reg_valid = Reg(Bool()) + val ex_reg_rvc = Reg(Bool()) + val ex_reg_btb_resp = Reg(new BTBResp) + val ex_reg_xcpt = Reg(Bool()) + val ex_reg_flush_pipe = Reg(Bool()) + val ex_reg_load_use = Reg(Bool()) + val ex_reg_cause = Reg(UInt()) + val ex_reg_replay = Reg(Bool()) + val ex_reg_pc = Reg(UInt()) + val ex_reg_mem_size = Reg(UInt()) + val ex_reg_hls = Reg(Bool()) + val ex_reg_inst = Reg(Bits()) + val ex_reg_raw_inst = Reg(UInt()) + val ex_reg_wphit = Reg(Vec(nBreakpoints, Bool())) + val ex_reg_set_vconfig = Reg(Bool()) + + val mem_reg_xcpt_interrupt = Reg(Bool()) + val mem_reg_valid = Reg(Bool()) + val mem_reg_rvc = Reg(Bool()) + val mem_reg_btb_resp = Reg(new BTBResp) + val mem_reg_xcpt = Reg(Bool()) + val mem_reg_replay = Reg(Bool()) + val mem_reg_flush_pipe = Reg(Bool()) + val mem_reg_cause = Reg(UInt()) + val mem_reg_slow_bypass = Reg(Bool()) + val mem_reg_load = Reg(Bool()) + val mem_reg_store = Reg(Bool()) + val mem_reg_set_vconfig = Reg(Bool()) + val mem_reg_sfence = Reg(Bool()) + val mem_reg_pc = Reg(UInt()) + val mem_reg_inst = Reg(Bits()) + val mem_reg_mem_size = Reg(UInt()) + val mem_reg_hls_or_dv = Reg(Bool()) + val mem_reg_raw_inst = Reg(UInt()) + val mem_reg_wdata = Reg(Bits()) + val mem_reg_rs2 = Reg(Bits()) + val mem_br_taken = Reg(Bool()) + val take_pc_mem = Wire(Bool()) + val mem_reg_wphit = Reg(Vec(nBreakpoints, Bool())) + + val wb_reg_valid = Reg(Bool()) + val wb_reg_xcpt = Reg(Bool()) + val wb_reg_replay = Reg(Bool()) + val wb_reg_flush_pipe = Reg(Bool()) + val wb_reg_cause = Reg(UInt()) + val wb_reg_set_vconfig = Reg(Bool()) + val wb_reg_sfence = Reg(Bool()) + val wb_reg_pc = Reg(UInt()) + val wb_reg_mem_size = Reg(UInt()) + val wb_reg_hls_or_dv = Reg(Bool()) + val wb_reg_hfence_v = Reg(Bool()) + val wb_reg_hfence_g = Reg(Bool()) + val wb_reg_inst = Reg(Bits()) + val wb_reg_raw_inst = Reg(UInt()) + val wb_reg_wdata = Reg(Bits()) + val wb_reg_rs2 = Reg(Bits()) + val take_pc_wb = Wire(Bool()) + val wb_reg_wphit = Reg(Vec(nBreakpoints, Bool())) + + val take_pc_mem_wb = take_pc_wb || take_pc_mem + val take_pc = take_pc_mem_wb + + // decode stage + val ibuf = Instantiate(new IBuf) + val id_expanded_inst = ibuf.io.inst.map(_.bits.inst) + val id_raw_inst = ibuf.io.inst.map(_.bits.raw) + val id_inst = id_expanded_inst.map(_.bits) + ibuf.io.imem <> io.imem.resp + ibuf.io.kill := take_pc + + require(decodeWidth == 1 /* TODO */ && retireWidth == decodeWidth) + require(!(coreParams.useRVE && coreParams.fpu.nonEmpty), "Can't select both RVE and floating-point") + require(!(coreParams.useRVE && coreParams.useHypervisor), "Can't select both RVE and Hypervisor") + val id_ctrl = Wire(new IntCtrlSigs).decode(id_inst(0), decode_table) + + val lgNXRegs = if (coreParams.useRVE) 4 else 5 + val regAddrMask = (1 << lgNXRegs) - 1 + + def decodeReg(x: UInt) = (x.extract(x.getWidth - 1, lgNXRegs).asBool, x(lgNXRegs - 1, 0)) + val (id_raddr3_illegal, id_raddr3) = decodeReg(id_expanded_inst(0).rs3) + val (id_raddr2_illegal, id_raddr2) = decodeReg(id_expanded_inst(0).rs2) + val (id_raddr1_illegal, id_raddr1) = decodeReg(id_expanded_inst(0).rs1) + val (id_waddr_illegal, id_waddr) = decodeReg(id_expanded_inst(0).rd) + + val id_load_use = Wire(Bool()) + val id_reg_fence = RegInit(false.B) + val id_ren = IndexedSeq(id_ctrl.rxs1, id_ctrl.rxs2) + val id_raddr = IndexedSeq(id_raddr1, id_raddr2) + val rf = new RegFile(regAddrMask, xLen) + val id_rs = id_raddr.map(rf.read _) + val ctrl_killd = Wire(Bool()) + + // A scheduler needs the custom ISA even without compute Balls. + val csrP = + if (usingRoCC) p + else p.alterPartial { case CoreRoCCCount => 0 } + + val csr = Instance(Definition(new CSRFile( + perfEvents, + coreParams.customCSRs.decls, + roccCSRs, + hasBusError + )(csrP) { + override def usingRoCC: Boolean = RocketBB.this.usingRoCC + })) + + val id_csr_en = id_ctrl.csr.isOneOf(CSR.S, CSR.C, CSR.W) + val id_system_insn = id_ctrl.csr === CSR.I + val id_csr_ren = id_ctrl.csr.isOneOf(CSR.S, CSR.C) && id_expanded_inst(0).rs1 === 0.U + val id_csr = Mux(id_system_insn && id_ctrl.mem, CSR.N, Mux(id_csr_ren, CSR.R, id_ctrl.csr)) + val id_csr_flush = id_system_insn || (id_csr_en && !id_csr_ren && csr.io.decode(0).write_flush) + val id_set_vconfig = + Seq(Instructions.VSETVLI, Instructions.VSETIVLI, Instructions.VSETVL).map(_ === id_inst(0)).orR && usingVector.B + + id_ctrl.vec := false.B + if (usingVector) { + val v_decode = rocketParams.vector.get.decoder(p) + v_decode.io.inst := id_inst(0) + v_decode.io.vconfig := csr.io.vector.get.vconfig + when(v_decode.io.legal) { + id_ctrl.legal := !csr.io.vector.get.vconfig.vtype.vill + id_ctrl.fp := v_decode.io.fp + id_ctrl.rocc := false.B + id_ctrl.branch := false.B + id_ctrl.jal := false.B + id_ctrl.jalr := false.B + id_ctrl.rxs2 := v_decode.io.read_rs2 + id_ctrl.rxs1 := v_decode.io.read_rs1 + id_ctrl.mem := false.B + id_ctrl.rfs1 := v_decode.io.read_frs1 + id_ctrl.rfs2 := false.B + id_ctrl.rfs3 := false.B + id_ctrl.wfd := v_decode.io.write_frd + id_ctrl.mul := false.B + id_ctrl.div := false.B + id_ctrl.wxd := v_decode.io.write_rd + id_ctrl.csr := CSR.N + id_ctrl.fence_i := false.B + id_ctrl.fence := false.B + id_ctrl.amo := false.B + id_ctrl.dp := false.B + id_ctrl.vec := true.B + } + } + + val id_illegal_insn = !id_ctrl.legal || + (id_ctrl.mul || id_ctrl.div) && !csr.io.status.isa('m' - 'a') || + id_ctrl.amo && !csr.io.status.isa('a' - 'a') || + id_ctrl.fp && (csr.io.decode(0).fp_illegal || (io.fpu.illegal_rm && !id_ctrl.vec)) || + (id_ctrl.vec) && (csr.io.decode(0).vector_illegal || csr.io.vector.map(_.vconfig.vtype.vill).getOrElse( + false.B + )) || + id_ctrl.dp && !csr.io.status.isa('d' - 'a') || + ibuf.io.inst(0).bits.rvc && !csr.io.status.isa('c' - 'a') || + id_raddr2_illegal && id_ctrl.rxs2 || + id_raddr1_illegal && id_ctrl.rxs1 || + id_waddr_illegal && id_ctrl.wxd || + id_ctrl.rocc && csr.io.decode(0).rocc_illegal || + id_csr_en && (csr.io.decode(0).read_illegal || !id_csr_ren && csr.io.decode(0).write_illegal) || + !ibuf.io.inst(0).bits.rvc && (id_system_insn && csr.io.decode(0).system_illegal) + + val id_virtual_insn = id_ctrl.legal && + ((id_csr_en && !(!id_csr_ren && csr.io.decode(0).write_illegal) && csr.io.decode(0).virtual_access_illegal) || + (!ibuf.io.inst(0).bits.rvc && id_system_insn && csr.io.decode(0).virtual_system_illegal)) + + // stall decode for fences (now, for AMO.rl; later, for AMO.aq and FENCE) + val id_amo_aq = id_inst(0)(26) + val id_amo_rl = id_inst(0)(25) + val id_fence_succ = id_inst(0)(23, 20) + val id_fence_next = id_ctrl.fence || id_ctrl.amo && id_amo_aq + val id_mem_busy = !io.dmem.ordered || io.dmem.req.valid + when(!id_mem_busy)(id_reg_fence := false.B) + + val id_rocc_busy = usingRoCC.B && + (io.rocc.busy || ex_reg_valid && ex_ctrl.rocc || + mem_reg_valid && mem_ctrl.rocc || wb_reg_valid && wb_ctrl.rocc) + + val id_csr_rocc_write = roccCSRs.map(_.id.U === id_inst(0)(31, 20)).orR && id_csr_en && !id_csr_ren + val id_vec_busy = io.vector.map(v => v.backend_busy || v.trap_check_busy).getOrElse(false.B) + + val id_do_fence = WireDefault(id_rocc_busy && (id_ctrl.fence || id_csr_rocc_write) || + id_vec_busy && id_ctrl.fence || + id_mem_busy && (id_ctrl.amo && id_amo_rl || id_ctrl.fence_i || id_reg_fence && (id_ctrl.mem || id_ctrl.rocc))) + + val bpu = Instantiate(new BreakpointUnit(nBreakpoints)) + bpu.io.status := csr.io.status + bpu.io.bp := csr.io.bp + bpu.io.pc := ibuf.io.pc + bpu.io.ea := mem_reg_wdata + bpu.io.mcontext := csr.io.mcontext + bpu.io.scontext := csr.io.scontext + + val id_xcpt0 = ibuf.io.inst(0).bits.xcpt0 + val id_xcpt1 = ibuf.io.inst(0).bits.xcpt1 + + val deliveredInterrupt = csr.io.interrupt && io.interruptReady + + val (id_xcpt, id_cause) = checkExceptions(List( + (deliveredInterrupt, csr.io.interrupt_cause), + (bpu.io.debug_if, CSR.debugTriggerCause.U), + (bpu.io.xcpt_if, Causes.breakpoint.U), + (id_xcpt0.pf.inst, Causes.fetch_page_fault.U), + (id_xcpt0.gf.inst, Causes.fetch_guest_page_fault.U), + (id_xcpt0.ae.inst, Causes.fetch_access.U), + (id_xcpt1.pf.inst, Causes.fetch_page_fault.U), + (id_xcpt1.gf.inst, Causes.fetch_guest_page_fault.U), + (id_xcpt1.ae.inst, Causes.fetch_access.U), + (id_virtual_insn, Causes.virtual_instruction.U), + (id_illegal_insn, Causes.illegal_instruction.U) + )) + + val idCoverCauses = List( + (CSR.debugTriggerCause, "DEBUG_TRIGGER"), + (Causes.breakpoint, "BREAKPOINT"), + (Causes.fetch_access, "FETCH_ACCESS"), + (Causes.illegal_instruction, "ILLEGAL_INSTRUCTION") + ) ++ (if (usingVM) + List( + (Causes.fetch_page_fault, "FETCH_PAGE_FAULT") + ) + else Nil) + + coverExceptions(id_xcpt, id_cause, "DECODE", idCoverCauses) + + val dcache_bypass_data = + if (fastLoadByte) io.dmem.resp.bits.data(xLen - 1, 0) + else if (fastLoadWord) io.dmem.resp.bits.data_word_bypass(xLen - 1, 0) + else wb_reg_wdata + + // detect bypass opportunities + val ex_waddr = ex_reg_inst(11, 7) & regAddrMask.U + val mem_waddr = mem_reg_inst(11, 7) & regAddrMask.U + val wb_waddr = wb_reg_inst(11, 7) & regAddrMask.U + + val bypass_sources = IndexedSeq( + (true.B, 0.U, 0.U), // treat reading x0 as a bypass + (ex_reg_valid && ex_ctrl.wxd, ex_waddr, mem_reg_wdata), + (mem_reg_valid && mem_ctrl.wxd && !mem_ctrl.mem, mem_waddr, wb_reg_wdata), + (mem_reg_valid && mem_ctrl.wxd, mem_waddr, dcache_bypass_data) + ) + + val id_bypass_src = id_raddr.map(raddr => bypass_sources.map(s => s._1 && s._2 === raddr)) + + // execute stage + val bypass_mux = bypass_sources.map(_._3) + val ex_reg_rs_bypass = Reg(Vec(id_raddr.size, Bool())) + val ex_reg_rs_lsb = Reg(Vec(id_raddr.size, UInt(log2Ceil(bypass_sources.size).W))) + val ex_reg_rs_msb = Reg(Vec(id_raddr.size, UInt())) + + val ex_rs = + for (i <- 0 until id_raddr.size) + yield Mux(ex_reg_rs_bypass(i), bypass_mux(ex_reg_rs_lsb(i)), Cat(ex_reg_rs_msb(i), ex_reg_rs_lsb(i))) + + val ex_imm = ImmGen(ex_ctrl.sel_imm, ex_reg_inst) + val ex_rs1shl = Mux(ex_reg_inst(3), ex_rs(0)(31, 0), ex_rs(0)) << ex_reg_inst(14, 13) + + val ex_op1 = MuxLookup(ex_ctrl.sel_alu1, 0.S)(Seq( + A1_RS1 -> ex_rs(0).asSInt, + A1_PC -> ex_reg_pc.asSInt, + A1_RS1SHL -> (if (rocketParams.useZba) ex_rs1shl.asSInt else 0.S) + )) + + val ex_op2_oh = + UIntToOH(Mux(ex_ctrl.sel_alu2(0), (ex_reg_inst >> 20).asUInt, ex_rs(1))(log2Ceil(xLen) - 1, 0)).asSInt + + val ex_op2 = MuxLookup(ex_ctrl.sel_alu2, 0.S)(Seq( + A2_RS2 -> ex_rs(1).asSInt, + A2_IMM -> ex_imm, + A2_SIZE -> Mux(ex_reg_rvc, 2.S, 4.S) + ) ++ (if (coreParams.useZbs) + Seq( + A2_RS2OH -> ex_op2_oh, + A2_IMMOH -> ex_op2_oh + ) + else Nil)) + + val (ex_new_vl, ex_new_vconfig) = + if (usingVector) { + val ex_new_vtype = VType.fromUInt(MuxCase( + ex_rs(1), + Seq( + ex_reg_inst(31, 30).andR -> ex_reg_inst(29, 20), + !ex_reg_inst(31) -> ex_reg_inst(30, 20) + ) + )) + val ex_avl = Mux( + ex_ctrl.rxs1, + Mux( + ex_reg_inst(19, 15) === 0.U, + Mux(ex_reg_inst(11, 7) === 0.U, csr.io.vector.get.vconfig.vl, ex_new_vtype.vlMax), + ex_rs(0) + ), + ex_reg_inst(19, 15) + ) + val ex_new_vl = ex_new_vtype.vl(ex_avl, csr.io.vector.get.vconfig.vl, false.B, false.B, false.B) + val ex_new_vconfig = Wire(new VConfig) + ex_new_vconfig.vtype := ex_new_vtype + ex_new_vconfig.vl := ex_new_vl + (Some(ex_new_vl), Some(ex_new_vconfig)) + } else { (None, None) } + + val alu = Instantiate(new ALU) + alu.io.dw := ex_ctrl.alu_dw + alu.io.fn := ex_ctrl.alu_fn + alu.io.in2 := ex_op2.asUInt + alu.io.in1 := ex_op1.asUInt + + // multiplier and divider + val div = Instantiate(new MulDiv(if (pipelinedMul) mulDivParams.copy(mulUnroll = 0) else mulDivParams, xLen)) + div.io.req.valid := ex_reg_valid && ex_ctrl.div + div.io.req.bits.dw := ex_ctrl.alu_dw + div.io.req.bits.fn := ex_ctrl.alu_fn + div.io.req.bits.in1 := ex_rs(0) + div.io.req.bits.in2 := ex_rs(1) + div.io.req.bits.tag := ex_waddr + + val mul = pipelinedMul.option { + val m = Instantiate(new PipelinedMultiplier(xLen, 2)) + m.io.req.valid := ex_reg_valid && ex_ctrl.mul + m.io.req.bits := div.io.req.bits + m + } + + ex_reg_valid := !ctrl_killd + ex_reg_replay := !take_pc && ibuf.io.inst(0).valid && ibuf.io.inst(0).bits.replay + ex_reg_xcpt := !ctrl_killd && id_xcpt + ex_reg_xcpt_interrupt := !take_pc && ibuf.io.inst(0).valid && deliveredInterrupt + + when(!ctrl_killd) { + ex_ctrl := id_ctrl + ex_reg_rvc := ibuf.io.inst(0).bits.rvc + ex_ctrl.csr := id_csr + when(id_ctrl.fence && id_fence_succ === 0.U)(id_reg_pause := true.B) + when(id_fence_next)(id_reg_fence := true.B) + when(id_xcpt) { // pass PC down ALU writeback pipeline for badaddr + ex_ctrl.alu_fn := FN_ADD + ex_ctrl.alu_dw := DW_XPR + ex_ctrl.sel_alu1 := A1_RS1 // badaddr := instruction + ex_ctrl.sel_alu2 := A2_ZERO + when(id_xcpt1.asUInt.orR) { // badaddr := PC+2 + ex_ctrl.sel_alu1 := A1_PC + ex_ctrl.sel_alu2 := A2_SIZE + ex_reg_rvc := true.B + } + when(bpu.io.xcpt_if || id_xcpt0.asUInt.orR) { // badaddr := PC + ex_ctrl.sel_alu1 := A1_PC + ex_ctrl.sel_alu2 := A2_ZERO + } + } + ex_reg_flush_pipe := id_ctrl.fence_i || id_csr_flush + ex_reg_load_use := id_load_use + ex_reg_hls := usingHypervisor.B && id_system_insn && id_ctrl.mem_cmd.isOneOf(M_XRD, M_XWR, M_HLVX) + ex_reg_mem_size := Mux(usingHypervisor.B && id_system_insn, id_inst(0)(27, 26), id_inst(0)(13, 12)) + when(id_ctrl.mem_cmd.isOneOf(M_SFENCE, M_HFENCEV, M_HFENCEG, M_FLUSH_ALL)) { + ex_reg_mem_size := Cat(id_raddr2 =/= 0.U, id_raddr1 =/= 0.U) + } + when(id_ctrl.mem_cmd === M_SFENCE && csr.io.status.v) { + ex_ctrl.mem_cmd := M_HFENCEV + } + if (dcacheFlushOnFenceI || coreParams.haveCFlush) { + when(id_ctrl.fence_i) { + ex_reg_mem_size := 0.U + } + } + + for (i <- 0 until id_raddr.size) { + val do_bypass = id_bypass_src(i).reduce(_ || _) + val bypass_src = PriorityEncoder(id_bypass_src(i)) + ex_reg_rs_bypass(i) := do_bypass + ex_reg_rs_lsb(i) := bypass_src + when(id_ren(i) && !do_bypass) { + ex_reg_rs_lsb(i) := id_rs(i)(log2Ceil(bypass_sources.size) - 1, 0) + ex_reg_rs_msb(i) := id_rs(i) >> log2Ceil(bypass_sources.size) + } + } + when(id_illegal_insn || id_virtual_insn) { + val inst = Mux(ibuf.io.inst(0).bits.rvc, id_raw_inst(0)(15, 0), id_raw_inst(0)) + ex_reg_rs_bypass(0) := false.B + ex_reg_rs_lsb(0) := inst(log2Ceil(bypass_sources.size) - 1, 0) + ex_reg_rs_msb(0) := inst >> log2Ceil(bypass_sources.size) + } + } + when(!ctrl_killd || deliveredInterrupt || ibuf.io.inst(0).bits.replay) { + ex_reg_cause := id_cause + ex_reg_inst := id_inst(0) + ex_reg_raw_inst := id_raw_inst(0) + ex_reg_pc := ibuf.io.pc + ex_reg_btb_resp := ibuf.io.btb_resp + ex_reg_wphit := bpu.io.bpwatch.map(bpw => bpw.ivalid(0)) + ex_reg_set_vconfig := id_set_vconfig && !id_xcpt + } + + // replay inst in ex stage? + val ex_pc_valid = ex_reg_valid || ex_reg_replay || ex_reg_xcpt_interrupt + val wb_dcache_miss = wb_ctrl.mem && !io.dmem.resp.valid + + val replay_ex_structural = ex_ctrl.mem && !io.dmem.req.ready || + ex_ctrl.div && !div.io.req.ready || + ex_ctrl.vec && !io.vector.map(_.ex.ready).getOrElse(true.B) + + val replay_ex_load_use = wb_dcache_miss && ex_reg_load_use + val replay_ex = ex_reg_replay || (ex_reg_valid && (replay_ex_structural || replay_ex_load_use)) + val ctrl_killx = take_pc_mem_wb || replay_ex || !ex_reg_valid + // detect 2-cycle load-use delay for LB/LH/SC + val ex_slow_bypass = ex_ctrl.mem_cmd === M_XSC || ex_reg_mem_size < 2.U + val ex_sfence = + usingVM.B && ex_ctrl.mem && (ex_ctrl.mem_cmd === M_SFENCE || ex_ctrl.mem_cmd === M_HFENCEV || ex_ctrl.mem_cmd === M_HFENCEG) + + val (ex_xcpt, ex_cause) = checkExceptions(List( + (ex_reg_xcpt_interrupt || ex_reg_xcpt, ex_reg_cause) + )) + + val exCoverCauses = idCoverCauses + coverExceptions(ex_xcpt, ex_cause, "EXECUTE", exCoverCauses) + + // memory stage + val mem_pc_valid = mem_reg_valid || mem_reg_replay || mem_reg_xcpt_interrupt + + val mem_br_target = mem_reg_pc.asSInt + + Mux( + mem_ctrl.branch && mem_br_taken, + ImmGen(IMM_SB, mem_reg_inst), + Mux(mem_ctrl.jal, ImmGen(IMM_UJ, mem_reg_inst), Mux(mem_reg_rvc, 2.S, 4.S)) + ) + + val mem_npc = (Mux( + mem_ctrl.jalr || mem_reg_sfence, + encodeVirtualAddress(mem_reg_wdata, mem_reg_wdata).asSInt, + mem_br_target + ) & -2.S).asUInt + + val mem_wrong_npc = + Mux( + ex_pc_valid, + mem_npc =/= ex_reg_pc, + Mux(ibuf.io.inst(0).valid || ibuf.io.imem.valid, mem_npc =/= ibuf.io.pc, true.B) + ) + + val mem_npc_misaligned = !csr.io.status.isa('c' - 'a') && mem_npc(1) && !mem_reg_sfence + val mem_int_wdata = + Mux(!mem_reg_xcpt && (mem_ctrl.jalr ^ mem_npc_misaligned), mem_br_target, mem_reg_wdata.asSInt).asUInt + val mem_cfi = mem_ctrl.branch || mem_ctrl.jalr || mem_ctrl.jal + val mem_cfi_taken = (mem_ctrl.branch && mem_br_taken) || mem_ctrl.jalr || mem_ctrl.jal + val mem_direction_misprediction = mem_ctrl.branch && mem_br_taken =/= (usingBTB.B && mem_reg_btb_resp.taken) + val mem_misprediction = if (usingBTB) mem_wrong_npc else mem_cfi_taken + take_pc_mem := mem_reg_valid && !mem_reg_xcpt && (mem_misprediction || mem_reg_sfence) + + mem_reg_valid := !ctrl_killx + mem_reg_replay := !take_pc_mem_wb && replay_ex + mem_reg_xcpt := !ctrl_killx && ex_xcpt + mem_reg_xcpt_interrupt := !take_pc_mem_wb && ex_reg_xcpt_interrupt + + // on pipeline flushes, cause mem_npc to hold the sequential npc, which + // will drive the W-stage npc mux + when(mem_reg_valid && mem_reg_flush_pipe) { + mem_reg_sfence := false.B + }.elsewhen(ex_pc_valid) { + mem_ctrl := ex_ctrl + mem_reg_rvc := ex_reg_rvc + mem_reg_load := ex_ctrl.mem && isRead(ex_ctrl.mem_cmd) + mem_reg_store := ex_ctrl.mem && isWrite(ex_ctrl.mem_cmd) + mem_reg_sfence := ex_sfence + mem_reg_btb_resp := ex_reg_btb_resp + mem_reg_flush_pipe := ex_reg_flush_pipe + mem_reg_slow_bypass := ex_slow_bypass + mem_reg_wphit := ex_reg_wphit + mem_reg_set_vconfig := ex_reg_set_vconfig + + mem_reg_cause := ex_cause + mem_reg_inst := ex_reg_inst + mem_reg_raw_inst := ex_reg_raw_inst + mem_reg_mem_size := ex_reg_mem_size + mem_reg_hls_or_dv := io.dmem.req.bits.dv + mem_reg_pc := ex_reg_pc + // IDecode ensured they are 1H + mem_reg_wdata := Mux(ex_reg_set_vconfig, ex_new_vl.getOrElse(alu.io.out), alu.io.out) + mem_br_taken := alu.io.cmp_out + + when(ex_ctrl.rxs2 && (ex_ctrl.mem || ex_ctrl.rocc || ex_sfence)) { + val size = Mux(ex_ctrl.rocc, log2Ceil(xLen / 8).U, ex_reg_mem_size) + mem_reg_rs2 := new StoreGen(size, 0.U, ex_rs(1), coreDataBytes).data + } + if (usingVector) { + when(ex_reg_set_vconfig) { + mem_reg_rs2 := ex_new_vconfig.get.asUInt + } + } + when(ex_ctrl.jalr && csr.io.status.debug) { + // flush I$ on D-mode JALR to effect uncached fetch without D$ flush + mem_ctrl.fence_i := true.B + mem_reg_flush_pipe := true.B + } + } + + val mem_breakpoint = (mem_reg_load && bpu.io.xcpt_ld) || (mem_reg_store && bpu.io.xcpt_st) + val mem_debug_breakpoint = (mem_reg_load && bpu.io.debug_ld) || (mem_reg_store && bpu.io.debug_st) + + val (mem_ldst_xcpt, mem_ldst_cause) = checkExceptions(List( + (mem_debug_breakpoint, CSR.debugTriggerCause.U), + (mem_breakpoint, Causes.breakpoint.U) + )) + + val (mem_xcpt, mem_cause) = checkExceptions(List( + (mem_reg_xcpt_interrupt || mem_reg_xcpt, mem_reg_cause), + (mem_reg_valid && mem_npc_misaligned, Causes.misaligned_fetch.U), + (mem_reg_valid && mem_ldst_xcpt, mem_ldst_cause) + )) + + val memCoverCauses = (exCoverCauses ++ List( + (CSR.debugTriggerCause, "DEBUG_TRIGGER"), + (Causes.breakpoint, "BREAKPOINT"), + (Causes.misaligned_fetch, "MISALIGNED_FETCH") + )).distinct + + coverExceptions(mem_xcpt, mem_cause, "MEMORY", memCoverCauses) + + val dcache_kill_mem = mem_reg_valid && mem_ctrl.wxd && io.dmem.replay_next // structural hazard on writeback port + val fpu_kill_mem = mem_reg_valid && mem_ctrl.fp && io.fpu.nack_mem + val vec_kill_mem = mem_reg_valid && mem_ctrl.mem && io.vector.map(_.mem.block_mem).getOrElse(false.B) + val vec_kill_all = mem_reg_valid && io.vector.map(_.mem.block_all).getOrElse(false.B) + val replay_mem = dcache_kill_mem || mem_reg_replay || fpu_kill_mem || vec_kill_mem || vec_kill_all + val killm_common = dcache_kill_mem || take_pc_wb || mem_reg_xcpt || !mem_reg_valid + div.io.kill := killm_common && RegNext(div.io.req.fire) + val ctrl_killm = killm_common || mem_xcpt || fpu_kill_mem || vec_kill_mem + + // writeback stage + wb_reg_valid := !ctrl_killm + wb_reg_replay := replay_mem && !take_pc_wb + wb_reg_xcpt := mem_xcpt && !take_pc_wb && !io.vector.map(_.mem.block_all).getOrElse(false.B) + wb_reg_flush_pipe := !ctrl_killm && mem_reg_flush_pipe + when(mem_pc_valid) { + wb_ctrl := mem_ctrl + wb_reg_sfence := mem_reg_sfence + wb_reg_wdata := Mux(!mem_reg_xcpt && mem_ctrl.fp && mem_ctrl.wxd, io.fpu.toint_data, mem_int_wdata) + when(mem_ctrl.rocc || mem_reg_sfence || mem_reg_set_vconfig) { + wb_reg_rs2 := mem_reg_rs2 + } + wb_reg_cause := mem_cause + wb_reg_inst := mem_reg_inst + wb_reg_raw_inst := mem_reg_raw_inst + wb_reg_mem_size := mem_reg_mem_size + wb_reg_hls_or_dv := mem_reg_hls_or_dv + wb_reg_hfence_v := mem_ctrl.mem_cmd === M_HFENCEV + wb_reg_hfence_g := mem_ctrl.mem_cmd === M_HFENCEG + wb_reg_pc := mem_reg_pc + wb_reg_wphit := mem_reg_wphit | bpu.io.bpwatch.map { bpw => + (bpw.rvalid(0) && mem_reg_load) || (bpw.wvalid(0) && mem_reg_store) + } + wb_reg_set_vconfig := mem_reg_set_vconfig + } + + val (wb_xcpt, wb_cause) = checkExceptions(List( + (wb_reg_xcpt, wb_reg_cause), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.pf.st, Causes.store_page_fault.U), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.pf.ld, Causes.load_page_fault.U), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.gf.st, Causes.store_guest_page_fault.U), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.gf.ld, Causes.load_guest_page_fault.U), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ae.st, Causes.store_access.U), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ae.ld, Causes.load_access.U), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ma.st, Causes.misaligned_store.U), + (wb_reg_valid && wb_ctrl.mem && io.dmem.s2_xcpt.ma.ld, Causes.misaligned_load.U) + )) + + val wbCoverCauses = List( + (Causes.misaligned_store, "MISALIGNED_STORE"), + (Causes.misaligned_load, "MISALIGNED_LOAD"), + (Causes.store_access, "STORE_ACCESS"), + (Causes.load_access, "LOAD_ACCESS") + ) ++ (if (usingVM) + List( + (Causes.store_page_fault, "STORE_PAGE_FAULT"), + (Causes.load_page_fault, "LOAD_PAGE_FAULT") + ) + else Nil) ++ (if (usingHypervisor) + List( + (Causes.store_guest_page_fault, "STORE_GUEST_PAGE_FAULT"), + (Causes.load_guest_page_fault, "LOAD_GUEST_PAGE_FAULT") + ) + else Nil) + + coverExceptions(wb_xcpt, wb_cause, "WRITEBACK", wbCoverCauses) + + val wb_pc_valid = wb_reg_valid || wb_reg_replay || wb_reg_xcpt + val wb_wxd = wb_reg_valid && wb_ctrl.wxd + val wb_set_sboard = wb_ctrl.div || wb_dcache_miss || wb_ctrl.rocc || wb_ctrl.vec + val replay_wb_common = io.dmem.s2_nack || wb_reg_replay + val replay_wb_rocc = wb_reg_valid && wb_ctrl.rocc && !io.rocc.cmd.ready + val replay_wb_csr: Bool = wb_reg_valid && csr.io.rw_stall + val replay_wb_vec = wb_reg_valid && io.vector.map(_.wb.replay).getOrElse(false.B) + val replay_wb = replay_wb_common || replay_wb_rocc || replay_wb_csr || replay_wb_vec + take_pc_wb := replay_wb || wb_xcpt || csr.io.eret || wb_reg_flush_pipe + + // writeback arbitration + val dmem_resp_xpu = !io.dmem.resp.bits.tag(0).asBool + val dmem_resp_fpu = io.dmem.resp.bits.tag(0).asBool + val dmem_resp_waddr = io.dmem.resp.bits.tag(5, 1) + val dmem_resp_valid = io.dmem.resp.valid && io.dmem.resp.bits.has_data + val dmem_resp_replay = dmem_resp_valid && io.dmem.resp.bits.replay + + class LLWB extends Bundle { + val data = UInt(xLen.W) + val tag = UInt(5.W) + } + + val ll_arb = Instantiate(new Arbiter(new LLWB, 3)) // div, rocc, vec + ll_arb.io.in.foreach(_.valid := false.B) + ll_arb.io.in.foreach(_.bits := DontCare) + val ll_wdata = WireInit(ll_arb.io.out.bits.data) + val ll_waddr = WireInit(ll_arb.io.out.bits.tag) + val ll_wen = WireInit(ll_arb.io.out.fire) + ll_arb.io.out.ready := !wb_wxd + + div.io.resp.ready := ll_arb.io.in(0).ready + ll_arb.io.in(0).valid := div.io.resp.valid + ll_arb.io.in(0).bits.data := div.io.resp.bits.data + ll_arb.io.in(0).bits.tag := div.io.resp.bits.tag + + if (usingRoCC) { + io.rocc.resp.ready := ll_arb.io.in(1).ready + ll_arb.io.in(1).valid := io.rocc.resp.valid + ll_arb.io.in(1).bits.data := io.rocc.resp.bits.data + ll_arb.io.in(1).bits.tag := io.rocc.resp.bits.rd + } else { + // tie off RoCC + io.rocc.resp.ready := false.B + } + + io.vector.map { v => + v.resp.ready := Mux(v.resp.bits.fp, !(dmem_resp_valid && dmem_resp_fpu), ll_arb.io.in(2).ready) + ll_arb.io.in(2).valid := v.resp.valid && !v.resp.bits.fp + ll_arb.io.in(2).bits.data := v.resp.bits.data + ll_arb.io.in(2).bits.tag := v.resp.bits.rd + } + + when(dmem_resp_replay && dmem_resp_xpu) { + ll_arb.io.out.ready := false.B + ll_waddr := dmem_resp_waddr + ll_wen := true.B + } + + val wb_valid = wb_reg_valid && !replay_wb && !wb_xcpt + val wb_wen = wb_valid && wb_ctrl.wxd + val rf_wen = wb_wen || ll_wen + val rf_waddr = Mux(ll_wen, ll_waddr, wb_waddr) + + val rf_wdata = Mux( + dmem_resp_valid && dmem_resp_xpu, + io.dmem.resp.bits.data(xLen - 1, 0), + Mux( + ll_wen, + ll_wdata, + Mux( + wb_ctrl.csr =/= CSR.N, + csr.io.rw.rdata, + Mux(wb_ctrl.mul, mul.map(_.io.resp.bits.data).getOrElse(wb_reg_wdata), wb_reg_wdata) + ) + ) + ) + + when(rf_wen)(rf.write(rf_waddr, rf_wdata)) + + // hook up control/status regfile + csr.io.ungated_clock := clock + csr.io.decode(0).inst := id_inst(0) + csr.io.exception := wb_xcpt + csr.io.cause := wb_cause + csr.io.retire := wb_valid + csr.io.inst(0) := (if (usingCompressed) + Cat(Mux(wb_reg_raw_inst(1, 0).andR, wb_reg_inst >> 16, 0.U), wb_reg_raw_inst(15, 0)) + else wb_reg_inst) + csr.io.interrupts := io.interrupts + csr.io.hartid := io.hartid + io.fpu.fcsr_rm := csr.io.fcsr_rm + val vector_fcsr_flags = io.vector.map(_.set_fflags.bits).getOrElse(0.U(5.W)) + val vector_fcsr_flags_valid = io.vector.map(_.set_fflags.valid).getOrElse(false.B) + csr.io.fcsr_flags.valid := io.fpu.fcsr_flags.valid | vector_fcsr_flags_valid + csr.io.fcsr_flags.bits := (io.fpu.fcsr_flags.bits & Fill(5, io.fpu.fcsr_flags.valid)) | (vector_fcsr_flags & Fill( + 5, + vector_fcsr_flags_valid + )) + io.fpu.time := csr.io.time(31, 0) + io.fpu.hartid := io.hartid + csr.io.rocc_interrupt := io.rocc.interrupt + csr.io.pc := wb_reg_pc + + val tval_dmem_addr = !wb_reg_xcpt + + val tval_any_addr = tval_dmem_addr || + wb_reg_cause.isOneOf( + Causes.breakpoint.U, + Causes.fetch_access.U, + Causes.fetch_page_fault.U, + Causes.fetch_guest_page_fault.U + ) + + val tval_inst = wb_reg_cause === Causes.illegal_instruction.U + val tval_valid = wb_xcpt && (tval_any_addr || tval_inst) + csr.io.gva := wb_xcpt && (tval_any_addr && csr.io.status.v || tval_dmem_addr && wb_reg_hls_or_dv) + csr.io.tval := Mux(tval_valid, encodeVirtualAddress(wb_reg_wdata, wb_reg_wdata), 0.U) + + val (htval, mhtinst_read_pseudo) = { + val htval_valid_imem = wb_reg_xcpt && wb_reg_cause === Causes.fetch_guest_page_fault.U + val htval_imem = Mux(htval_valid_imem, io.imem.gpa.bits, 0.U) + assert(!htval_valid_imem || io.imem.gpa.valid) + + val htval_valid_dmem = + wb_xcpt && tval_dmem_addr && io.dmem.s2_xcpt.gf.asUInt.orR && !io.dmem.s2_xcpt.pf.asUInt.orR + val htval_dmem = Mux(htval_valid_dmem, io.dmem.s2_gpa, 0.U) + + val htval = (htval_dmem | htval_imem) >> hypervisorExtraAddrBits + // read pseudoinstruction if a guest-page fault is caused by an implicit memory access for VS-stage address + // translation + val mhtinst_read_pseudo = (io.imem.gpa_is_pte && htval_valid_imem) || (io.dmem.s2_gpa_is_pte && htval_valid_dmem) + (htval, mhtinst_read_pseudo) + } + + csr.io.vector.foreach { v => + v.set_vconfig.valid := wb_reg_set_vconfig && wb_reg_valid + v.set_vconfig.bits := wb_reg_rs2.asTypeOf(new VConfig) + v.set_vs_dirty := wb_valid && wb_ctrl.vec + v.set_vstart.valid := wb_valid && wb_reg_set_vconfig + v.set_vstart.bits := 0.U + } + + io.vector.foreach { v => + when(v.wb.retire || v.wb.xcpt || wb_ctrl.vec) { + csr.io.pc := v.wb.pc + csr.io.retire := v.wb.retire + csr.io.inst(0) := v.wb.inst + when(v.wb.xcpt && !wb_reg_xcpt) { + wb_xcpt := true.B + wb_cause := v.wb.cause + csr.io.tval := v.wb.tval + } + } + v.wb.store_pending := io.dmem.store_pending + v.wb.vxrm := csr.io.vector.get.vxrm + v.wb.frm := csr.io.fcsr_rm + csr.io.vector.get.set_vxsat := v.set_vxsat + when(v.set_vconfig.valid) { + csr.io.vector.get.set_vconfig.valid := true.B + csr.io.vector.get.set_vconfig.bits := v.set_vconfig.bits + } + when(v.set_vstart.valid) { + csr.io.vector.get.set_vstart.valid := true.B + csr.io.vector.get.set_vstart.bits := v.set_vstart.bits + } + } + + csr.io.htval := htval + csr.io.mhtinst_read_pseudo := mhtinst_read_pseudo + io.ptw.ptbr := csr.io.ptbr + io.ptw.hgatp := csr.io.hgatp + io.ptw.vsatp := csr.io.vsatp + (io.ptw.customCSRs.csrs zip csr.io.customCSRs).map { case (lhs, rhs) => lhs <> rhs } + io.ptw.status := csr.io.status + io.ptw.hstatus := csr.io.hstatus + io.ptw.gstatus := csr.io.gstatus + io.ptw.pmp := csr.io.pmp + csr.io.rw.addr := wb_reg_inst(31, 20) + csr.io.rw.cmd := CSR.maskCmd(wb_reg_valid, wb_ctrl.csr) + csr.io.rw.wdata := wb_reg_wdata + + io.rocc.csrs <> csr.io.roccCSRs + for ((port, source) <- io.rocc.csrs.zip(csr.io.roccCSRs)) { + port.ren := source.ren && wb_reg_valid && !wb_xcpt && !replay_wb_common + } + io.trace.time := csr.io.time + io.trace.insns := csr.io.trace + if (rocketParams.debugROB.isDefined) { + val sz = rocketParams.debugROB.get.size + if (sz < 1) { // use unsynthesizable ROB + val csr_trace_with_wdata = WireInit(csr.io.trace(0)) + csr_trace_with_wdata.wdata.get := rf_wdata + val should_wb = WireInit((wb_ctrl.wfd || (wb_ctrl.wxd && wb_waddr =/= 0.U)) && !csr.io.trace(0).exception) + val has_wb = WireInit(wb_ctrl.wxd && wb_wen && !wb_set_sboard) + val wb_addr = WireInit(wb_waddr + Mux(wb_ctrl.wfd, 32.U, 0.U)) + + io.vector.foreach { v => + when(v.wb.retire) { + should_wb := v.wb.rob_should_wb + has_wb := false.B + wb_addr := Cat(v.wb.rob_should_wb_fp, csr_trace_with_wdata.insn(11, 7)) + } + } + + DebugROB.pushTrace(clock, reset, io.hartid, csr_trace_with_wdata, should_wb, has_wb, wb_addr) + + io.trace.insns(0) := DebugROB.popTrace(clock, reset, io.hartid) + + DebugROB.pushWb(clock, reset, io.hartid, ll_wen, rf_waddr, rf_wdata) + } else { // synthesizable ROB (no FPRs) + require(!usingVector, "Synthesizable ROB does not support vector implementations") + val csr_trace_with_wdata = WireInit(csr.io.trace(0)) + csr_trace_with_wdata.wdata.get := rf_wdata + + val debug_rob = Instantiate(new HardDebugROB(sz, 32)) + debug_rob.io.i_insn := csr_trace_with_wdata + debug_rob.io.should_wb := (wb_ctrl.wfd || (wb_ctrl.wxd && wb_waddr =/= 0.U)) && + !csr.io.trace(0).exception + debug_rob.io.has_wb := wb_ctrl.wxd && wb_wen && !wb_set_sboard + debug_rob.io.tag := wb_waddr + Mux(wb_ctrl.wfd, 32.U, 0.U) + + debug_rob.io.wb_val := ll_wen + debug_rob.io.wb_tag := rf_waddr + debug_rob.io.wb_data := rf_wdata + + io.trace.insns(0) := debug_rob.io.o_insn + } + } else { + io.trace.insns := csr.io.trace + } + for (((iobpw, wphit), bp) <- io.bpwatch zip wb_reg_wphit zip csr.io.bp) { + iobpw.valid(0) := wphit + iobpw.action := bp.control.action + // tie off bpwatch valids + iobpw.rvalid.foreach(_ := false.B) + iobpw.wvalid.foreach(_ := false.B) + iobpw.ivalid.foreach(_ := false.B) + } + + val hazard_targets = Seq( + (id_ctrl.rxs1 && id_raddr1 =/= 0.U, id_raddr1), + (id_ctrl.rxs2 && id_raddr2 =/= 0.U, id_raddr2), + (id_ctrl.wxd && id_waddr =/= 0.U, id_waddr) + ) + + val fp_hazard_targets = Seq( + (io.fpu.dec.ren1, id_raddr1), + (io.fpu.dec.ren2, id_raddr2), + (io.fpu.dec.ren3, id_raddr3), + (io.fpu.dec.wen, id_waddr) + ) + + val sboard = new Scoreboard(32, true) + sboard.clear(ll_wen, ll_waddr) + + def id_sboard_clear_bypass(r: UInt) = + // ll_waddr arrives late when D$ has ECC, so reshuffle the hazard check + if (!tileParams.dcache.get.dataECC.isDefined) ll_wen && ll_waddr === r + else + div.io.resp.fire && div.io.resp.bits.tag === r || dmem_resp_replay && dmem_resp_xpu && dmem_resp_waddr === r + + val id_sboard_hazard = checkHazards(hazard_targets, rd => sboard.read(rd) && !id_sboard_clear_bypass(rd)) + sboard.set(wb_set_sboard && wb_wen, wb_waddr) + + // stall for RAW/WAW hazards on CSRs, loads, AMOs, and mul/div in execute stage. + val ex_cannot_bypass = + ex_ctrl.csr =/= CSR.N || ex_ctrl.jalr || ex_ctrl.mem || ex_ctrl.mul || ex_ctrl.div || ex_ctrl.fp || ex_ctrl.rocc || ex_ctrl.vec + val data_hazard_ex = ex_ctrl.wxd && checkHazards(hazard_targets, _ === ex_waddr) + val fp_data_hazard_ex = id_ctrl.fp && ex_ctrl.wfd && checkHazards(fp_hazard_targets, _ === ex_waddr) + val id_ex_hazard = ex_reg_valid && (data_hazard_ex && ex_cannot_bypass || fp_data_hazard_ex) + + // stall for RAW/WAW hazards on CSRs, LB/LH, and mul/div in memory stage. + val mem_mem_cmd_bh = + if (fastLoadWord) (!fastLoadByte).B && mem_reg_slow_bypass + else true.B + + val mem_cannot_bypass = + mem_ctrl.csr =/= CSR.N || mem_ctrl.mem && mem_mem_cmd_bh || mem_ctrl.mul || mem_ctrl.div || mem_ctrl.fp || mem_ctrl.rocc || mem_ctrl.vec + val data_hazard_mem = mem_ctrl.wxd && checkHazards(hazard_targets, _ === mem_waddr) + val fp_data_hazard_mem = id_ctrl.fp && mem_ctrl.wfd && checkHazards(fp_hazard_targets, _ === mem_waddr) + val id_mem_hazard = mem_reg_valid && (data_hazard_mem && mem_cannot_bypass || fp_data_hazard_mem) + id_load_use := mem_reg_valid && data_hazard_mem && mem_ctrl.mem + + val id_vconfig_hazard = id_ctrl.vec && ( + (ex_reg_valid && ex_reg_set_vconfig) || + (mem_reg_valid && mem_reg_set_vconfig) || + (wb_reg_valid && wb_reg_set_vconfig) + ) + + // stall for RAW/WAW hazards on load/AMO misses and mul/div in writeback. + val data_hazard_wb = wb_ctrl.wxd && checkHazards(hazard_targets, _ === wb_waddr) + val fp_data_hazard_wb = id_ctrl.fp && wb_ctrl.wfd && checkHazards(fp_hazard_targets, _ === wb_waddr) + val id_wb_hazard = wb_reg_valid && (data_hazard_wb && wb_set_sboard || fp_data_hazard_wb) + + val id_stall_fpu = + if (usingFPU) { + val fp_sboard = new Scoreboard(32) + fp_sboard.set(((wb_dcache_miss || wb_ctrl.vec) && wb_ctrl.wfd || io.fpu.sboard_set) && wb_valid, wb_waddr) + val v_ll = io.vector.map(v => v.resp.fire && v.resp.bits.fp).getOrElse(false.B) + fp_sboard.clear((dmem_resp_replay && dmem_resp_fpu) || v_ll, io.fpu.ll_resp_tag) + fp_sboard.clear(io.fpu.sboard_clr, io.fpu.sboard_clra) + + checkHazards(fp_hazard_targets, fp_sboard.read _) + } else false.B + + val dcache_blocked = { + // speculate that a blocked D$ will unblock the cycle after a Grant + val blocked = Reg(Bool()) + blocked := !io.dmem.req.ready && io.dmem.clock_enabled && !io.dmem.perf.grant && (blocked || io.dmem.req.valid || io.dmem.s2_nack) + blocked && !io.dmem.perf.grant + } + + val rocc_blocked = RegInit(false.B) + rocc_blocked := !wb_xcpt && !io.rocc.cmd.ready && (io.rocc.cmd.valid || rocc_blocked) + + val ctrl_stalld = + id_ex_hazard || id_mem_hazard || id_wb_hazard || id_sboard_hazard || + id_vconfig_hazard || + csr.io.singleStep && (ex_reg_valid || mem_reg_valid || wb_reg_valid) || + id_csr_en && csr.io.decode(0).fp_csr && !io.fpu.fcsr_rdy || + id_csr_en && csr.io.decode(0).vector_csr && id_vec_busy || + id_ctrl.fp && id_stall_fpu || + id_ctrl.mem && (dcache_blocked || id_rocc_busy) || // data accesses follow posted coprocessor work + id_ctrl.rocc && (rocc_blocked || id_mem_busy) || // prior CPU memory requests complete before submission + id_ctrl.div && (!(div.io.req.ready || (div.io.resp.valid && !wb_wxd)) || div.io.req.valid) || // reduce odds of replay + !clock_en || + id_do_fence || + csr.io.csr_stall || + id_reg_pause || + io.traceStall + + ctrl_killd := !ibuf.io.inst(0).valid || ibuf.io.inst( + 0 + ).bits.replay || take_pc_mem_wb || ctrl_stalld || deliveredInterrupt + + io.imem.req.valid := take_pc + io.imem.req.bits.speculative := !take_pc_wb + io.imem.req.bits.pc := + Mux( + wb_xcpt || csr.io.eret, + csr.io.evec, // exception or [m|s]ret + Mux( + replay_wb, + wb_reg_pc, // replay + mem_npc + ) + ) // flush or branch misprediction + io.imem.flush_icache := wb_reg_valid && wb_ctrl.fence_i && !io.dmem.s2_nack + io.imem.might_request := { + imem_might_request_reg := ex_pc_valid || mem_pc_valid || io.ptw.customCSRs.disableICacheClockGate || io.vector.map( + _.trap_check_busy + ).getOrElse(false.B) + imem_might_request_reg + } + io.imem.progress := RegNext(wb_reg_valid && !replay_wb_common) + io.imem.sfence.valid := wb_reg_valid && wb_reg_sfence + io.imem.sfence.bits.rs1 := wb_reg_mem_size(0) + io.imem.sfence.bits.rs2 := wb_reg_mem_size(1) + io.imem.sfence.bits.addr := wb_reg_wdata + io.imem.sfence.bits.asid := wb_reg_rs2 + io.imem.sfence.bits.hv := wb_reg_hfence_v + io.imem.sfence.bits.hg := wb_reg_hfence_g + io.ptw.sfence := io.imem.sfence + + ibuf.io.inst(0).ready := !ctrl_stalld + + io.imem.btb_update.valid := mem_reg_valid && !take_pc_wb && mem_wrong_npc && (!mem_cfi || mem_cfi_taken) + io.imem.btb_update.bits.isValid := mem_cfi + io.imem.btb_update.bits.cfiType := + Mux( + (mem_ctrl.jal || mem_ctrl.jalr) && mem_waddr(0), + CFIType.call, + Mux( + mem_ctrl.jalr && (mem_reg_inst(19, 15) & regAddrMask.U) === BitPat("b00?01"), + CFIType.ret, + Mux(mem_ctrl.jal || mem_ctrl.jalr, CFIType.jump, CFIType.branch) + ) + ) + io.imem.btb_update.bits.target := io.imem.req.bits.pc + io.imem.btb_update.bits.br_pc := (if (usingCompressed) mem_reg_pc + Mux(mem_reg_rvc, 0.U, 2.U) else mem_reg_pc) + io.imem.btb_update.bits.pc := ~(~io.imem.btb_update.bits.br_pc | (coreInstBytes * fetchWidth - 1).U) + io.imem.btb_update.bits.prediction := mem_reg_btb_resp + io.imem.btb_update.bits.taken := DontCare + + io.imem.bht_update.valid := mem_reg_valid && !take_pc_wb + io.imem.bht_update.bits.pc := io.imem.btb_update.bits.pc + io.imem.bht_update.bits.taken := mem_br_taken + io.imem.bht_update.bits.mispredict := mem_wrong_npc + io.imem.bht_update.bits.branch := mem_ctrl.branch + io.imem.bht_update.bits.prediction := mem_reg_btb_resp.bht + + // Connect RAS in Frontend + io.imem.ras_update := DontCare + + io.fpu.valid := !ctrl_killd && id_ctrl.fp + io.fpu.killx := ctrl_killx + io.fpu.killm := killm_common + io.fpu.inst := id_inst(0) + io.fpu.fromint_data := ex_rs(0) + io.fpu.ll_resp_val := dmem_resp_valid && dmem_resp_fpu + io.fpu.ll_resp_data := (if (minFLen == 32) io.dmem.resp.bits.data_word_bypass else io.dmem.resp.bits.data) + io.fpu.ll_resp_type := io.dmem.resp.bits.size + io.fpu.ll_resp_tag := dmem_resp_waddr + io.fpu.keep_clock_enabled := io.ptw.customCSRs.disableCoreClockGate + + io.fpu.v_sew := csr.io.vector.map(_.vconfig.vtype.vsew).getOrElse(0.U) + + io.vector.map { v => + when(!(dmem_resp_valid && dmem_resp_fpu)) { + io.fpu.ll_resp_val := v.resp.valid && v.resp.bits.fp + io.fpu.ll_resp_data := v.resp.bits.data + io.fpu.ll_resp_type := v.resp.bits.size + io.fpu.ll_resp_tag := v.resp.bits.rd + } + } + + io.vector.foreach { v => + v.ex.valid := ex_reg_valid && (ex_ctrl.vec || rocketParams.vector.get.issueVConfig.B && ex_reg_set_vconfig) && !ctrl_killx + v.ex.inst := ex_reg_inst + v.ex.vconfig := csr.io.vector.get.vconfig + v.ex.vstart := Mux(mem_reg_valid && mem_ctrl.vec || wb_reg_valid && wb_ctrl.vec, 0.U, csr.io.vector.get.vstart) + v.ex.rs1 := ex_rs(0) + v.ex.rs2 := ex_rs(1) + v.ex.pc := ex_reg_pc + v.mem.frs1 := io.fpu.store_data + v.killm := killm_common + v.status := csr.io.status + } + + io.dmemPc := ex_reg_pc + io.dmem.req.valid := ex_reg_valid && ex_ctrl.mem + val ex_dcache_tag = Cat(ex_waddr, ex_ctrl.fp) + require(coreParams.dcacheReqTagBits >= ex_dcache_tag.getWidth) + io.dmem.req.bits.tag := ex_dcache_tag + io.dmem.req.bits.cmd := ex_ctrl.mem_cmd + io.dmem.req.bits.size := ex_reg_mem_size + io.dmem.req.bits.signed := !Mux(ex_reg_hls, ex_reg_inst(20), ex_reg_inst(14)) + io.dmem.req.bits.phys := false.B + io.dmem.req.bits.addr := encodeVirtualAddress(ex_rs(0), alu.io.adder_out) + io.dmem.req.bits.idx.foreach(_ := io.dmem.req.bits.addr) + io.dmem.req.bits.dprv := Mux(ex_reg_hls, csr.io.hstatus.spvp, csr.io.status.dprv) + io.dmem.req.bits.dv := ex_reg_hls || csr.io.status.dv + io.dmem.req.bits.no_resp := !isRead(ex_ctrl.mem_cmd) || (!ex_ctrl.fp && ex_waddr === 0.U) + io.dmem.req.bits.no_alloc := DontCare + io.dmem.req.bits.no_xcpt := DontCare + io.dmem.req.bits.data := DontCare + io.dmem.req.bits.mask := DontCare + + io.dmem.s1_data.data := (if (fLen == 0) mem_reg_rs2 + else Mux(mem_ctrl.fp, Fill(coreDataBits / fLen, io.fpu.store_data), mem_reg_rs2)) + io.dmem.s1_data.mask := DontCare + + io.dmem.s1_kill := killm_common || mem_ldst_xcpt || fpu_kill_mem || vec_kill_mem + io.dmem.s2_kill := false.B + // don't let D$ go to sleep if we're probably going to use it soon + io.dmem.keep_clock_enabled := ibuf.io.inst(0).valid && id_ctrl.mem && !csr.io.csr_stall + + // Pipeline classification, not a second accelerator decoder. Readiness is deliberately absent. + io.roccOlderPending := usingRoCC.B && + ((mem_reg_valid && mem_ctrl.rocc) || (wb_reg_valid && wb_ctrl.rocc)) + io.roccCommitEligible := usingRoCC.B && wb_reg_valid && wb_ctrl.rocc && !replay_wb_common && !wb_xcpt + io.rocc.cmd.valid := wb_reg_valid && wb_ctrl.rocc && !replay_wb_common + io.rocc.exception := wb_xcpt && csr.io.status.xs.orR + io.rocc.cmd.bits.raw_inst := wb_reg_inst + io.rocc.cmd.bits.funct := wb_reg_inst(31, 25) + io.rocc.cmd.bits.funct3 := wb_reg_inst(14, 12) + io.rocc.cmd.bits.rs2 := wb_reg_inst(24, 20) + io.rocc.cmd.bits.rs1 := wb_reg_inst(19, 15) + io.rocc.cmd.bits.pc := wb_reg_pc + io.rocc.cmd.bits.xd := wb_reg_inst(14) + io.rocc.cmd.bits.xs1 := wb_reg_inst(13) + io.rocc.cmd.bits.xs2 := wb_reg_inst(12) + io.rocc.cmd.bits.rd := wb_reg_inst(11, 7) + io.rocc.cmd.bits.opcode := wb_reg_inst(6, 0) + io.rocc.cmd.bits.rs1Data := wb_reg_wdata + io.rocc.cmd.bits.rs2Data := wb_reg_rs2 + + // gate the clock + val unpause = + csr.io.time( + rocketParams.lgPauseCycles - 1, + 0 + ) === 0.U || csr.io.inhibit_cycle || io.dmem.perf.release || take_pc + + when(unpause)(id_reg_pause := false.B) + io.cease := csr.io.status.cease && !clock_en_reg + io.wfi := csr.io.status.wfi + if (rocketParams.clockGate) { + long_latency_stall := csr.io.csr_stall || io.dmem.perf.blocked || id_reg_pause && !unpause + clock_en := clock_en_reg || ex_pc_valid || (!long_latency_stall && io.imem.resp.valid) + clock_en_reg := + ex_pc_valid || mem_pc_valid || wb_pc_valid || // instruction in flight + io.ptw.customCSRs.disableCoreClockGate || // chicken bit + !div.io.req.ready || // mul/div in flight + usingFPU.B && !io.fpu.fcsr_rdy || // long-latency FPU in flight + io.dmem.replay_next || // long-latency load replaying + (!long_latency_stall && (ibuf.io.inst(0).valid || io.imem.resp.valid)) // instruction pending + + assert(!(ex_pc_valid || mem_pc_valid || wb_pc_valid) || clock_en) + } + + // evaluate performance counters + val icache_blocked = !(io.imem.resp.valid || RegNext(io.imem.resp.valid)) + csr.io.counters foreach { c => c.inc := RegNext(perfEvents.evaluate(c.eventSel)) } + + val coreMonitorBundle = Wire(new CoreMonitorBundle(xLen, fLen)) + + coreMonitorBundle.clock := clock + coreMonitorBundle.reset := reset + coreMonitorBundle.hartid := io.hartid + coreMonitorBundle.timer := csr.io.time(31, 0) + coreMonitorBundle.valid := csr.io.trace(0).valid && !csr.io.trace(0).exception + coreMonitorBundle.pc := csr.io.trace(0).iaddr(vaddrBitsExtended - 1, 0).sextTo(xLen) + coreMonitorBundle.wrenx := wb_wen && !wb_set_sboard + coreMonitorBundle.wrenf := false.B + coreMonitorBundle.wrdst := wb_waddr + coreMonitorBundle.wrdata := rf_wdata + coreMonitorBundle.rd0src := wb_reg_inst(19, 15) + coreMonitorBundle.rd0val := RegNext(RegNext(ex_rs(0))) + coreMonitorBundle.rd1src := wb_reg_inst(24, 20) + coreMonitorBundle.rd1val := RegNext(RegNext(ex_rs(1))) + coreMonitorBundle.inst := csr.io.trace(0).insn + coreMonitorBundle.excpt := csr.io.trace(0).exception + coreMonitorBundle.priv_mode := csr.io.trace(0).priv + + if (enableCommitLog) { + val t = csr.io.trace(0) + val rd = wb_waddr + val wfd = wb_ctrl.wfd + val wxd = wb_ctrl.wxd + val has_data = wb_wen && !wb_set_sboard + + when(t.valid && !t.exception) { + when(wfd) { + printf("%d 0x%x (0x%x) f%d p%d 0xXXXXXXXXXXXXXXXX\n", t.priv, t.iaddr, t.insn, rd, rd + 32.U) + } + .elsewhen(wxd && rd =/= 0.U && has_data) { + printf("%d 0x%x (0x%x) x%d 0x%x\n", t.priv, t.iaddr, t.insn, rd, rf_wdata) + } + .elsewhen(wxd && rd =/= 0.U && !has_data) { + printf("%d 0x%x (0x%x) x%d p%d 0xXXXXXXXXXXXXXXXX\n", t.priv, t.iaddr, t.insn, rd, rd) + } + .otherwise { + printf("%d 0x%x (0x%x)\n", t.priv, t.iaddr, t.insn) + } + } + + when(ll_wen && rf_waddr =/= 0.U) { + printf("x%d p%d 0x%x\n", rf_waddr, rf_waddr, rf_wdata) + } + } else { + when(csr.io.trace(0).valid) { + printf( + "C%d: %d [%d] pc=[%x] W[r%d=%x][%d] R[r%d=%x] R[r%d=%x] inst=[%x] DASM(%x) wb_xcpt:%d\n", + io.hartid, + coreMonitorBundle.timer, + coreMonitorBundle.valid, + coreMonitorBundle.pc, + Mux(wb_ctrl.wxd || wb_ctrl.wfd, coreMonitorBundle.wrdst, 0.U), + Mux(coreMonitorBundle.wrenx, coreMonitorBundle.wrdata, 0.U), + coreMonitorBundle.wrenx, + Mux(wb_ctrl.rxs1 || wb_ctrl.rfs1, coreMonitorBundle.rd0src, 0.U), + Mux(wb_ctrl.rxs1 || wb_ctrl.rfs1, coreMonitorBundle.rd0val, 0.U), + Mux(wb_ctrl.rxs2 || wb_ctrl.rfs2, coreMonitorBundle.rd1src, 0.U), + Mux(wb_ctrl.rxs2 || wb_ctrl.rfs2, coreMonitorBundle.rd1val, 0.U), + coreMonitorBundle.inst, + coreMonitorBundle.inst, + wb_xcpt + ) + } + } + + // CoreMonitorBundle for late latency writes + val xrfWriteBundle = Wire(new CoreMonitorBundle(xLen, fLen)) + + xrfWriteBundle.clock := clock + xrfWriteBundle.reset := reset + xrfWriteBundle.hartid := io.hartid + xrfWriteBundle.timer := csr.io.time(31, 0) + xrfWriteBundle.valid := false.B + xrfWriteBundle.pc := 0.U + xrfWriteBundle.wrdst := rf_waddr + xrfWriteBundle.wrenx := rf_wen && !(csr.io.trace(0).valid && wb_wen && (wb_waddr === rf_waddr)) + xrfWriteBundle.wrenf := false.B + xrfWriteBundle.wrdata := rf_wdata + xrfWriteBundle.rd0src := 0.U + xrfWriteBundle.rd0val := 0.U + xrfWriteBundle.rd1src := 0.U + xrfWriteBundle.rd1val := 0.U + xrfWriteBundle.inst := 0.U + xrfWriteBundle.excpt := false.B + xrfWriteBundle.priv_mode := csr.io.trace(0).priv + + if (rocketParams.haveSimTimeout) + PlusArg.timeout( + name = "max_core_cycles", + docstring = "Kill the emulation after INT rdtime cycles. Off if 0." + )(csr.io.time) + + } // leaving gated-clock domain + val rocketImpl = withClock(gated_clock)(new RocketImpl) + + def checkExceptions(x: Seq[(Bool, UInt)]) = + (WireInit(x.map(_._1).reduce(_ || _)), WireInit(PriorityMux(x))) + + def coverExceptions( + exceptionValid: Bool, + cause: UInt, + labelPrefix: String, + coverCausesLabels: Seq[(Int, String)] + ): Unit = { + for ((coverCause, label) <- coverCausesLabels) { + property.cover(exceptionValid && (cause === coverCause.U), s"${labelPrefix}_$label") + } + } + + def checkHazards(targets: Seq[(Bool, UInt)], cond: UInt => Bool) = + targets.map(h => h._1 && cond(h._2)).reduce(_ || _) + + def encodeVirtualAddress(a0: UInt, ea: UInt) = + if (vaddrBitsExtended == vaddrBits) ea + else { + // efficient means to compress 64-bit VA into vaddrBits+1 bits + // (VA is bad if VA(vaddrBits) != VA(vaddrBits-1)) + val b = vaddrBitsExtended - 1 + val a = (a0 >> b).asSInt + val msb = Mux(a === 0.S || a === -1.S, ea(b), !ea(b - 1)) + Cat(msb, ea(b - 1, 0)) + } + + class Scoreboard(n: Int, zero: Boolean = false) { + def set(en: Bool, addr: UInt): Unit = update(en, _next | mask(en, addr)) + def clear(en: Bool, addr: UInt): Unit = update(en, _next & ~mask(en, addr)) + def read(addr: UInt): Bool = r(addr) + def readBypassed(addr: UInt): Bool = _next(addr) + + val _r = RegInit(0.U(n.W)) + val r = if (zero) (_r >> 1 << 1) else _r + private var _next = r + private var ens = false.B + def mask(en: Bool, addr: UInt) = Mux(en, 1.U << addr, 0.U) + + def update(en: Bool, update: UInt) = { + _next = update + ens = ens || en + when(ens)(_r := _next) + } + + } + + } + + class RegFile(n: Int, w: Int, zero: Boolean = false) { + val rf = Reg(Vec(n, UInt(w.W))) + def access(addr: UInt) = rf(~addr(log2Up(n) - 1, 0)) + val reads = ArrayBuffer[(UInt, UInt)]() + private var canRead = true + + def read(addr: UInt) = { + require(canRead) + reads += addr -> Wire(UInt()) + reads.last._2 := Mux(zero.B && addr === 0.U, 0.U, access(addr)) + reads.last._2 + } + + def write(addr: UInt, data: UInt) = { + canRead = false + when(addr =/= 0.U) { + access(addr) := data + for ((raddr, rdata) <- reads) + when(addr === raddr)(rdata := data) + } + } + + } + + object ImmGen { + + def apply(sel: UInt, inst: UInt) = { + val sign = Mux(sel === IMM_Z, 0.S, inst(31).asSInt) + val b30_20 = Mux(sel === IMM_U, inst(30, 20).asSInt, sign) + val b19_12 = Mux(sel =/= IMM_U && sel =/= IMM_UJ, sign, inst(19, 12).asSInt) + val b11 = Mux( + sel === IMM_U || sel === IMM_Z, + 0.S, + Mux(sel === IMM_UJ, inst(20).asSInt, Mux(sel === IMM_SB, inst(7).asSInt, sign)) + ) + val b10_5 = Mux(sel === IMM_U || sel === IMM_Z, 0.U, inst(30, 25)) + val b4_1 = Mux( + sel === IMM_U, + 0.U, + Mux(sel === IMM_S || sel === IMM_SB, inst(11, 8), Mux(sel === IMM_Z, inst(19, 16), inst(24, 21))) + ) + val b0 = Mux(sel === IMM_S, inst(7), Mux(sel === IMM_I, inst(20), Mux(sel === IMM_Z, inst(15), 0.U))) + + Cat(sign, b30_20, b19_12, b11, b10_5, b4_1, b0).asSInt + } + + } + +} + +// See LICENSE.Berkeley and LICENSE.SiFive for the Rocket interface definitions. +package freechips.rocketchip.rocket { + + import chisel3._ + import chisel3.util._ + import org.chipsalliance.cde.config.Parameters + import freechips.rocketchip.tile.{CoreBundle, HasCoreParameters, TileInterrupts} + + /** CPU-facing interfaces, independent of the upstream cache and TileLink implementations. */ + class FrontendReq(implicit p: Parameters) extends CoreBundle()(p) { + val pc = UInt(vaddrBitsExtended.W) + val speculative = Bool() + } + + class FrontendExceptions extends Bundle { + val pf = new Bundle { val inst = Bool() } + val gf = new Bundle { val inst = Bool() } + val ae = new Bundle { val inst = Bool() } + } + + class FrontendResp(implicit p: Parameters) extends CoreBundle()(p) { + val btb = new BTBResp + val pc = UInt(vaddrBitsExtended.W) + val data = UInt((fetchWidth * coreInstBits).W) + val mask = Bits(fetchWidth.W) + val xcpt = new FrontendExceptions + val replay = Bool() + } + + class FrontendPerfEvents extends Bundle { + val acquire = Bool() + val tlbMiss = Bool() + } + + class SFenceReq(implicit p: Parameters) extends CoreBundle()(p) { + val rs1 = Bool() + val rs2 = Bool() + val addr = UInt(vaddrBits.W) + val asid = UInt((asIdBits max 1).W) + val hv = Bool() + val hg = Bool() + } + + class FrontendIO(implicit p: Parameters) extends CoreBundle()(p) { + val might_request = Output(Bool()) + val clock_enabled = Input(Bool()) + val req = Valid(new FrontendReq) + val sfence = Valid(new SFenceReq) + val resp = Flipped(Decoupled(new FrontendResp)) + val gpa = Flipped(Valid(UInt(vaddrBitsExtended.W))) + val gpa_is_pte = Input(Bool()) + val btb_update = Valid(new BTBUpdate) + val bht_update = Valid(new BHTUpdate) + val ras_update = Valid(new RASUpdate) + val flush_icache = Output(Bool()) + val npc = Input(UInt(vaddrBitsExtended.W)) + val perf = Input(new FrontendPerfEvents) + val progress = Output(Bool()) + } + + trait HasCoreMemOp extends HasCoreParameters { + val addr = UInt(coreMaxAddrBits.W) + + val idx = Option.when(usingVM && lgCacheBlockBytes + log2Up(tileParams.dcache.get.nSets) > pgIdxBits)( + UInt(coreMaxAddrBits.W) + ) + + val tag = UInt((coreParams.dcacheReqTagBits + log2Ceil(dcacheArbPorts)).W) + val cmd = UInt(M_SZ.W) + val size = UInt(log2Ceil(log2Ceil(coreDataBytes) + 1).W) + val signed = Bool() + val dprv = UInt(PRV.SZ.W) + val dv = Bool() + } + + trait HasCoreData extends HasCoreParameters { + val data = UInt(coreDataBits.W) + val mask = UInt(coreDataBytes.W) + } + + class HellaCacheReqInternal(implicit p: Parameters) extends CoreBundle()(p) with HasCoreMemOp { + val phys = Bool() + val no_resp = Bool() + val no_alloc = Bool() + val no_xcpt = Bool() + } + + class HellaCacheReq(implicit p: Parameters) extends HellaCacheReqInternal()(p) with HasCoreData + + class HellaCacheResp(implicit p: Parameters) extends CoreBundle()(p) with HasCoreMemOp with HasCoreData { + val replay = Bool() + val has_data = Bool() + val data_word_bypass = UInt(coreDataBits.W) + val data_raw = UInt(coreDataBits.W) + val store_data = UInt(coreDataBits.W) + } + + class AlignmentExceptions extends Bundle { + val ld = Bool() + val st = Bool() + } + + class HellaCacheExceptions extends Bundle { + val ma = new AlignmentExceptions + val pf = new AlignmentExceptions + val gf = new AlignmentExceptions + val ae = new AlignmentExceptions + } + + class HellaCacheWriteData(implicit p: Parameters) extends CoreBundle()(p) with HasCoreData + + class HellaCachePerfEvents extends Bundle { + val acquire = Bool() + val release = Bool() + val grant = Bool() + val tlbMiss = Bool() + val blocked = Bool() + val canAcceptStoreThenLoad = Bool() + val canAcceptStoreThenRMW = Bool() + val canAcceptLoadThenLoad = Bool() + val storeBufferEmptyAfterLoad = Bool() + val storeBufferEmptyAfterStore = Bool() + } + + class HellaCacheIO(implicit p: Parameters) extends CoreBundle()(p) { + val req = Decoupled(new HellaCacheReq) + val s1_kill = Output(Bool()) + val s1_data = Output(new HellaCacheWriteData) + val s2_nack = Input(Bool()) + val s2_nack_cause_raw = Input(Bool()) + val s2_kill = Output(Bool()) + val s2_uncached = Input(Bool()) + val s2_paddr = Input(UInt(paddrBits.W)) + val resp = Flipped(Valid(new HellaCacheResp)) + val replay_next = Input(Bool()) + val s2_xcpt = Input(new HellaCacheExceptions) + val s2_gpa = Input(UInt(vaddrBitsExtended.W)) + val s2_gpa_is_pte = Input(Bool()) + val uncached_resp = Option.when(tileParams.dcache.get.separateUncachedResp)(Flipped(Decoupled(new HellaCacheResp))) + val ordered = Input(Bool()) + val store_pending = Input(Bool()) + val perf = Input(new HellaCachePerfEvents) + val keep_clock_enabled = Output(Bool()) + val clock_enabled = Input(Bool()) + } + + class PTWPerfEvents extends Bundle { + val l2miss = Bool() + val l2hit = Bool() + val pte_miss = Bool() + val pte_hit = Bool() + } + + class DatapathPTWIO(implicit p: Parameters) extends CoreBundle()(p) { + val ptbr = Input(new PTBR) + val hgatp = Input(new PTBR) + val vsatp = Input(new PTBR) + val sfence = Flipped(Valid(new SFenceReq)) + val status = Input(new MStatus) + val hstatus = Input(new HStatus) + val gstatus = Input(new MStatus) + val pmp = Input(Vec(nPMPs, new PMP)) + val perf = Output(new PTWPerfEvents) + val customCSRs = Flipped(coreParams.customCSRs) + val clock_enabled = Output(Bool()) + } + + class CoreInterrupts(val hasBeu: Boolean)(implicit p: Parameters) extends TileInterrupts()(p) { + val buserror = Option.when(hasBeu)(Bool()) + } + +} + +package examples.core.rocket.pipeline { + + import chisel3._ + import chisel3.util._ + + /** RoCC response bundle */ + class RoCCResponseBB(xLen: Int = 64) extends Bundle { + val rd = Bits(5.W) + val data = Bits(xLen.W) + } + + /** RoCC interface between a core and an accelerator. */ + class RoCCIO(xLen: Int = 64) extends Bundle { + val cmd = Flipped(Decoupled(new RoCCCommandBB(xLen))) + val resp = Decoupled(new RoCCResponseBB(xLen)) + val busy = Output(Bool()) + val interrupt = Output(Bool()) + val exception = Input(Bool()) + } + + /** RoCC command bundle */ + class RoCCCommandBB(xLen: Int = 64) extends Bundle { + val raw_inst = UInt(32.W) + val pc = UInt(xLen.W) + val funct = UInt(7.W) + val funct3 = UInt(3.W) + val rs2 = Bits(5.W) + val rs1 = Bits(5.W) + val xd = Bool() + val xs1 = Bool() + val xs2 = Bool() + val rd = Bits(5.W) + val opcode = UInt(7.W) + val rs1Data = UInt(xLen.W) + val rs2Data = UInt(xLen.W) + } + +} + +// See LICENSE.SiFive for the Rocket interface definitions. +package freechips.rocketchip.tile { + + import chisel3._ + import org.chipsalliance.cde.config.Parameters + + class NMI(val w: Int) extends Bundle { + val rnmi = Bool() + val rnmi_interrupt_vector = UInt(w.W) + val rnmi_exception_vector = UInt(w.W) + } + + class TileInterrupts(implicit p: Parameters) extends CoreBundle()(p) { + val debug = Bool() + val mtip = Bool() + val msip = Bool() + val meip = Bool() + val seip = Option.when(usingSupervisor)(Bool()) + val lip = Vec(coreParams.nLocalInterrupts, Bool()) + val nmi = Option.when(usingNMI)(new NMI(resetVectorLen)) + } + +} + +// See LICENSE.Berkeley for Rocket's CPU/vector interface contract. +package freechips.rocketchip.rocket { + + import chisel3._ + import chisel3.util._ + import org.chipsalliance.cde.config.Parameters + import freechips.rocketchip.tile._ + + class VectorCoreIO(implicit p: Parameters) extends CoreBundle { + val status = Input(new MStatus) + + val ex = new Bundle { + val valid = Input(Bool()); val ready = Output(Bool()); val inst = Input(UInt(32.W)) + val pc = Input(UInt(vaddrBitsExtended.W)); val vconfig = Input(new VConfig) + val vstart = Input(UInt(log2Ceil(maxVLMax).W)); val rs1 = Input(UInt(xLen.W)); val rs2 = Input(UInt(xLen.W)) + } + + val killm = Input(Bool()) + + val mem = new Bundle { + val frs1 = Input(UInt(fLen.W)); val block_mem = Output(Bool()); val block_all = Output(Bool()) + } + + val wb = new Bundle { + val store_pending = Input(Bool()); val replay = Output(Bool()); val retire = Output(Bool()) + val inst = Output(UInt(32.W)); val rob_should_wb = Output(Bool()); val rob_should_wb_fp = Output(Bool()) + val pc = Output(UInt(vaddrBitsExtended.W)); val xcpt = Output(Bool()) + val cause = Output(UInt(log2Ceil(Causes.all.max).W)); val tval = Output(UInt(coreMaxAddrBits.W)) + val vxrm = Input(UInt(2.W)); val frm = Input(UInt(3.W)) + } + + val resp = Decoupled(new Bundle { + val fp = Bool(); val size = UInt(2.W); val rd = UInt(5.W); val data = UInt((xLen max fLen).W) + }) + + val set_vstart = Valid(UInt(log2Ceil(maxVLMax).W)) + val set_vxsat = Output(Bool()) + val set_vconfig = Valid(new VConfig) + val set_fflags = Valid(UInt(5.W)) + val trap_check_busy = Output(Bool()) + val backend_busy = Output(Bool()) + } + + abstract class RocketVectorDecoder(implicit p: Parameters) extends CoreModule { + + val io = IO(new Bundle { + val inst = Input(UInt(32.W)); val vconfig = Input(new VConfig) + val vector = Output(Bool()); val legal = Output(Bool()); val fp = Output(Bool()) + val read_rs1 = Output(Bool()); val read_rs2 = Output(Bool()); val read_frs1 = Output(Bool()) + val write_rd = Output(Bool()); val write_frd = Output(Bool()) + }) + + } + +} + +package examples.core.rocket.pipeline { + + /** + * Elaboration-time representation of one Buckyball custom instruction. + * Domain ISA builders own the funct7 and operand encodings; boot only streams + * these already-formed commands. + */ + case class BuckyballCommand(funct: Int, rs1: BigInt = 0, rs2: BigInt = 0) + + object BuckyballCommand { + val Custom3Opcode = 0x7b + val Custom3Funct3 = 3 + } + +} diff --git a/arch/src/main/scala/framework/system/core/rocket/Utility.scala b/examples/components/cores/rocket/arch/pipeline/Utility.scala similarity index 100% rename from arch/src/main/scala/framework/system/core/rocket/Utility.scala rename to examples/components/cores/rocket/arch/pipeline/Utility.scala diff --git a/examples/components/cores/rocket/configs/Params.scala b/examples/components/cores/rocket/configs/Params.scala new file mode 100644 index 000000000..e74dbcac7 --- /dev/null +++ b/examples/components/cores/rocket/configs/Params.scala @@ -0,0 +1,107 @@ +package examples.core.rocket.configs + +import examples.core.rocket.pipeline.CpuParams +import framework.core.clink.configs.{Params => CLinkParam} +import framework.core.execution.fetch.configs.{Params => FetchParam} +import framework.core.memory.configs.MemoryParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.acc.configs.AccParam + +import buckyball.config.CorePlacement +import buckyball.components.rocket.{Params => RocketProto} +import scala.jdk.CollectionConverters._ +import framework.ip.bus.{axi4, chi} +import framework.ip.bus.chi.rnf.RnfParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.ip.memory.coherence.configs.CoherenceParams +import framework.core.memory.home.configs.{Params => HomeParam} +import framework.core.memory.access.cache.configs.{Params => CacheParam} +import framework.core.memory.access.configs.{Params => AccessParam} +import framework.core.memory.access.{AccessParams, PhysicalRegion} +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} +import freechips.rocketchip.rocket.{DCacheParams, ICacheParams, MulDivParams, RocketCoreParams} +import freechips.rocketchip.tile.FPUParams + +case class Params( + cpu: CpuParams, + memory: MemoryParam, + fetch: FetchParam, + clink: CLinkParam, + permission: PmpParams, + hartId: Int, + resetVector: BigInt, + mtimeAddress: BigInt, + acc: Option[AccParam]) { + require(clink.harts == 1 && !clink.shared) + require(clink.requesters == (if (clink.backing) 0 else 2)) + require(cpu.core.xLen == 64 && !cpu.core.useHypervisor && cpu.core.useCompressed) + require(cpu.core.nPMPs == permission.entries && cpu.physicalAddressBits == permission.addressBits) + require(memory.memoryAccess.lineBits == 0) + require(memory.threadPorts == 0) +} + +object Params { + + def fromProto(core: CorePlacement): Params = { + require(core.hasMemory && core.hasImplementation && core.hasSharedMem) + val r = core.getImplementation.unpack(classOf[RocketProto]) + val bus = chi.Params.fromProto(core.getMemory.getHome.getRequesters) + val axi = axi4.Params.fromProto(core.getMemory.getUncached.getAxi) + val acc = Option.when(core.hasAccelerator) { + val a = core.getAccelerator + AccParam( + FrontendParam.fromProto(a.getFrontend, core.getParam.getXLen, core.getHartId), + MemDomainParam.fromProto(a.getMem, core.getSharedMem, core.getVirtualBankCount, 1), + GpDomainParam.fromProto(a.getGpdomain), + BallDomainParam.fromProto(a.getBalldomain), + TransportParam.fromProto(a.getTransport) + ) + } + Params( + CpuParams( + RocketCoreParams( + xLen = 64, + pgLevels = r.getPageLevels, + nPMPs = r.getPmpEntries, + useVM = r.getVirtualMemory, + useSupervisor = r.getSupervisor, + useUser = r.getUser, + useHypervisor = false, + useDebug = false, + useAtomics = r.getAtomics, + useCompressed = true, + clockGate = false, + mulDiv = Option.when(r.getMultiplyDivide)(MulDivParams()), + fpu = Option.when(r.getFloatingPoint)(FPUParams()) + ), + Some(DCacheParams()), + Some(ICacheParams()), + None, + r.getPhysicalAddressBits, + r.getHartIdBits + ), + MemoryParam.fromProto(core.getMemory, 0), + FetchParam(core.getFetch.getInstructionBytes), + CLinkParam( + 1, + !r.getExternalHome, + if (r.getExternalHome) Seq(core.getMemory.getIcache.getNodeId, core.getMemory.getDcache.getCache.getNodeId) + else Seq.empty, + bus, + axi, + acc.map(_.transport.axi).getOrElse(axi4.Params.fromProto(core.getInterconnect.getAxi)), + None + ), + PmpParams(r.getPhysicalAddressBits, r.getPmpEntries), + core.getHartId, + BigInt(java.lang.Long.toUnsignedString(core.getResetVector)), + BigInt(java.lang.Long.toUnsignedString(core.getMtimeAddress)), + acc + ) + } + +} diff --git a/examples/components/cores/rocket/configs/rocket_params.proto b/examples/components/cores/rocket/configs/rocket_params.proto new file mode 100644 index 000000000..16f8fcbc3 --- /dev/null +++ b/examples/components/cores/rocket/configs/rocket_params.proto @@ -0,0 +1,17 @@ +syntax = "proto2"; +package buckyball.components.rocket; +option java_package = "buckyball.components.rocket"; +option java_multiple_files = true; +message Params { + required uint32 physical_address_bits = 1; + required uint32 hart_id_bits = 2; + required uint32 page_levels = 3; + required uint32 pmp_entries = 4; + required bool virtual_memory = 5; + required bool supervisor = 6; + required bool user = 7; + required bool floating_point = 8; + required bool atomics = 9; + required bool multiply_divide = 10; + required bool external_home = 11; +} diff --git a/examples/components/cores/rocket/emu/Cargo.lock b/examples/components/cores/rocket/emu/Cargo.lock new file mode 100644 index 000000000..3eb3cb3c1 --- /dev/null +++ b/examples/components/cores/rocket/emu/Cargo.lock @@ -0,0 +1,822 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "bebop-vm", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "rvsim", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-core-rocket" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rvsim" +version = "0.1.0" +source = "git+https://github.com/DangoSys/riscv-cpu-sim?rev=32abeddb3f0d4c810e07d8d6745f5150e96c8c54#32abeddb3f0d4c810e07d8d6745f5150e96c8c54" +dependencies = [ + "rustc_apfloat", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/components/cores/rocket/emu/Cargo.toml b/examples/components/cores/rocket/emu/Cargo.toml new file mode 100644 index 000000000..1c6a30e92 --- /dev/null +++ b/examples/components/cores/rocket/emu/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "bebop-core-rocket" +version = "0.1.0" +edition = "2021" + +[dependencies] +bebop-bemu = { path = "../../../../../bebop/src/nodes/bemu" } diff --git a/examples/components/cores/rocket/emu/src/lib.rs b/examples/components/cores/rocket/emu/src/lib.rs new file mode 100644 index 000000000..ba2cfba41 --- /dev/null +++ b/examples/components/cores/rocket/emu/src/lib.rs @@ -0,0 +1,27 @@ +use bebop_bemu::{ball::linkme, CoreFactory, CoreModel, CoreParams, Cpu, DeviceModel, ModelOptions, NoRocc, Whatever, CORES}; + +/// The Rocket core: one RV64GC hart with nothing on its RoCC port. +pub struct RocketCore { + cpu: Cpu, +} + +impl CoreModel for RocketCore { + fn cpu(&self) -> &Cpu { + &self.cpu + } + fn cpu_mut(&mut self) -> &mut Cpu { + &mut self.cpu + } + fn step(&mut self, count: u64) -> Result<(), Whatever> { + self.cpu.step(count, &mut NoRocc) + } +} + +fn instantiate(device: &DeviceModel, params: &CoreParams, options: &ModelOptions) -> Result, String> { + let cpu = Cpu::new(device, params, options).map_err(|error| error.to_string())?; + Ok(Box::new(RocketCore { cpu })) +} + +#[linkme::distributed_slice(CORES)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static ROCKET: CoreFactory = CoreFactory { component: "rocket", instantiate }; diff --git a/examples/components/cores/rocket/stack/CMakeLists.txt b/examples/components/cores/rocket/stack/CMakeLists.txt new file mode 100644 index 000000000..389428cd5 --- /dev/null +++ b/examples/components/cores/rocket/stack/CMakeLists.txt @@ -0,0 +1,25 @@ +# ===----------------------------------------------------------------------=== +# The rocket core: a CPU core without hardware threads; jobs run on the core. +# +# bbdev stack --build "--design " +# +# builds it for each design that uses it, configured with DESIGN_CMAKE +# (generated from design.pb) and CORE. Products: +# execute.o runs jobs on the core itself; links into the device runtime +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(rocket-core LANGUAGES NONE) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +add_custom_command( + OUTPUT execute.o + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" + -c "${RUNTIME}/execute/direct.c" -o execute.o + DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" + VERBATIM) + +add_custom_target(execute ALL DEPENDS execute.o) diff --git a/examples/components/cores/xiangshan/LICENSE b/examples/components/cores/xiangshan/LICENSE new file mode 100644 index 000000000..ebf13f89a --- /dev/null +++ b/examples/components/cores/xiangshan/LICENSE @@ -0,0 +1,127 @@ + 木兰宽松许可证, 第2版 + + 木兰宽松许可证， 第2版 + 2020年1月 http://license.coscl.org.cn/MulanPSL2 + + + 您对“软件”的复制、使用、修改及分发受木兰宽松许可证，第2版（“本许可证”）的如下条款的约束： + + 0. 定义 + + “软件”是指由“贡献”构成的许可在“本许可证”下的程序和相关文档的集合。 + + “贡献”是指由任一“贡献者”许可在“本许可证”下的受版权法保护的作品。 + + “贡献者”是指将受版权法保护的作品许可在“本许可证”下的自然人或“法人实体”。 + + “法人实体”是指提交贡献的机构及其“关联实体”。 + + “关联实体”是指，对“本许可证”下的行为方而言，控制、受控制或与其共同受控制的机构，此处的控制是指有受控方或共同受控方至少50%直接或间接的投票权、资金或其他有价证券。 + + 1. 授予版权许可 + + 每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的版权许可，您可以复制、使用、修改、分发其“贡献”，不论修改与否。 + + 2. 授予专利许可 + + 每个“贡献者”根据“本许可证”授予您永久性的、全球性的、免费的、非独占的、不可撤销的（根据本条规定撤销除外）专利许可，供您制造、委托制造、使用、许诺销售、销售、进口其“贡献”或以其他方式转移其“贡献”。前述专利许可仅限于“贡献者”现在或将来拥有或控制的其“贡献”本身或其“贡献”与许可“贡献”时的“软件”结合而将必然会侵犯的专利权利要求，不包括对“贡献”的修改或包含“贡献”的其他结合。如果您或您的“关联实体”直接或间接地，就“软件”或其中的“贡献”对任何人发起专利侵权诉讼（包括反诉或交叉诉讼）或其他专利维权行动，指控其侵犯专利权，则“本许可证”授予您对“软件”的专利许可自您提起诉讼或发起维权行动之日终止。 + + 3. 无商标许可 + + “本许可证”不提供对“贡献者”的商品名称、商标、服务标志或产品名称的商标许可，但您为满足第4条规定的声明义务而必须使用除外。 + + 4. 分发限制 + + 您可以在任何媒介中将“软件”以源程序形式或可执行形式重新分发，不论修改与否，但您必须向接收者提供“本许可证”的副本，并保留“软件”中的版权、商标、专利及免责声明。 + + 5. 免责声明与责任限制 + + “软件”及其中的“贡献”在提供时不带任何明示或默示的担保。在任何情况下，“贡献者”或版权所有者不对任何人因使用“软件”或其中的“贡献”而引发的任何直接或间接损失承担责任，不论因何种原因导致或者基于何种法律理论，即使其曾被建议有此种损失的可能性。 + + 6. 语言 + “本许可证”以中英文双语表述，中英文版本具有同等法律效力。如果中英文版本存在任何冲突不一致，以中文版为准。 + + 条款结束 + + 如何将木兰宽松许可证，第2版，应用到您的软件 + + 如果您希望将木兰宽松许可证，第2版，应用到您的新软件，为了方便接收者查阅，建议您完成如下三步： + + 1， 请您补充如下声明中的空白，包括软件名、软件的首次发表年份以及您作为版权人的名字； + + 2， 请您在软件包的一级目录下创建以“LICENSE”为名的文件，将整个许可证文本放入该文件中； + + 3， 请将如下声明文本放入每个源文件的头部注释中。 + + Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences + XiangShan is licensed under Mulan PSL v2. + You can use this software according to the terms and conditions of the Mulan PSL v2. + You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 + THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + See the Mulan PSL v2 for more details. + + + Mulan Permissive Software License，Version 2 + + Mulan Permissive Software License，Version 2 (Mulan PSL v2) + January 2020 http://license.coscl.org.cn/MulanPSL2 + + Your reproduction, use, modification and distribution of the Software shall be subject to Mulan PSL v2 (this License) with the following terms and conditions: + + 0. Definition + + Software means the program and related documents which are licensed under this License and comprise all Contribution(s). + + Contribution means the copyrightable work licensed by a particular Contributor under this License. + + Contributor means the Individual or Legal Entity who licenses its copyrightable work under this License. + + Legal Entity means the entity making a Contribution and all its Affiliates. + + Affiliates means entities that control, are controlled by, or are under common control with the acting entity under this License, ‘control’ means direct or indirect ownership of at least fifty percent (50%) of the voting power, capital or other securities of controlled or commonly controlled entity. + + 1. Grant of Copyright License + + Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable copyright license to reproduce, use, modify, or distribute its Contribution, with modification or not. + + 2. Grant of Patent License + + Subject to the terms and conditions of this License, each Contributor hereby grants to you a perpetual, worldwide, royalty-free, non-exclusive, irrevocable (except for revocation under this Section) patent license to make, have made, use, offer for sale, sell, import or otherwise transfer its Contribution, where such patent license is only limited to the patent claims owned or controlled by such Contributor now or in future which will be necessarily infringed by its Contribution alone, or by combination of the Contribution with the Software to which the Contribution was contributed. The patent license shall not apply to any modification of the Contribution, and any other combination which includes the Contribution. If you or your Affiliates directly or indirectly institute patent litigation (including a cross claim or counterclaim in a litigation) or other patent enforcement activities against any individual or entity by alleging that the Software or any Contribution in it infringes patents, then any patent license granted to you under this License for the Software shall terminate as of the date such litigation or activity is filed or taken. + + 3. No Trademark License + + No trademark license is granted to use the trade names, trademarks, service marks, or product names of Contributor, except as required to fulfill notice requirements in Section 4. + + 4. Distribution Restriction + + You may distribute the Software in any medium with or without modification, whether in source or executable forms, provided that you provide recipients with a copy of this License and retain copyright, patent, trademark and disclaimer statements in the Software. + + 5. Disclaimer of Warranty and Limitation of Liability + + THE SOFTWARE AND CONTRIBUTION IN IT ARE PROVIDED WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED. IN NO EVENT SHALL ANY CONTRIBUTOR OR COPYRIGHT HOLDER BE LIABLE TO YOU FOR ANY DAMAGES, INCLUDING, BUT NOT LIMITED TO ANY DIRECT, OR INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING FROM YOUR USE OR INABILITY TO USE THE SOFTWARE OR THE CONTRIBUTION IN IT, NO MATTER HOW IT’S CAUSED OR BASED ON WHICH LEGAL THEORY, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + 6. Language + + THIS LICENSE IS WRITTEN IN BOTH CHINESE AND ENGLISH, AND THE CHINESE VERSION AND ENGLISH VERSION SHALL HAVE THE SAME LEGAL EFFECT. IN THE CASE OF DIVERGENCE BETWEEN THE CHINESE AND ENGLISH VERSIONS, THE CHINESE VERSION SHALL PREVAIL. + + END OF THE TERMS AND CONDITIONS + + How to Apply the Mulan Permissive Software License，Version 2 (Mulan PSL v2) to Your Software + + To apply the Mulan PSL v2 to your work, for easy identification by recipients, you are suggested to complete following three steps: + + i Fill in the blanks in following statement, including insert your software name, the year of the first publication of your software, and your name identified as the copyright owner; + + ii Create a file named “LICENSE” which contains the whole context of this License in the first directory of your software package; + + iii Attach the statement to the appropriate annotated syntax at the beginning of each source file. + + + Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences + XiangShan is licensed under Mulan PSL v2. + You can use this software according to the terms and conditions of the Mulan PSL v2. + You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 + THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + See the Mulan PSL v2 for more details. diff --git a/examples/components/cores/xiangshan/arch/Chi.scala b/examples/components/cores/xiangshan/arch/Chi.scala new file mode 100644 index 000000000..72252d02b --- /dev/null +++ b/examples/components/cores/xiangshan/arch/Chi.scala @@ -0,0 +1,192 @@ +package examples.core.xiangshan + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.ip.bus.chi + +/** Credit-accounted native CHI B link and explicit framework flit translation. */ +@instantiable +class Chi(p: chi.Params, homeId: Int) extends Module { + require(p.nodeIdBits == 7 && p.addressBits == 48 && p.dataBits == 256) + + @public + val io = IO(new Bundle { + val native = Flipped(new Native.Link) + val requester = new chi.RequesterPort(p) + }) + + io.native.tx.linkactiveack := RegNext(io.native.tx.linkactivereq, false.B) + io.native.rx.linkactivereq := !reset.asBool + io.native.rxsactive := !reset.asBool + io.native.syscoack := RegNext(io.native.syscoreq, false.B) + val receiving = io.native.tx.linkactivereq && io.native.tx.linkactiveack + val sending = io.native.rx.linkactivereq && io.native.rx.linkactiveack + + val requestRx = Instantiate(new chi.Rx((new Native.Request).getWidth, 4)) + requestRx.io.active := receiving + requestRx.io.link <> io.native.tx.req + val request = Wire(new Native.Request) + request.unpack(requestRx.io.out.bits) + io.requester.req.valid := requestRx.io.out.valid + requestRx.io.out.ready := io.requester.req.ready + io.requester.req.bits := 0.U.asTypeOf(io.requester.req.bits) + io.requester.req.bits.qos := request.qos + io.requester.req.bits.srcId := request.srcId + io.requester.req.bits.txnId := request.txnId + io.requester.req.bits.returnNid := request.returnNid + io.requester.req.bits.stashNidValidEndian := request.stashNidValidEndian + io.requester.req.bits.returnTxnId := request.returnTxnId + io.requester.req.bits.opcode := request.opcode + io.requester.req.bits.size := request.size + io.requester.req.bits.addr := request.addr + io.requester.req.bits.likelyShared := request.likelyShared + io.requester.req.bits.allowRetry := request.allowRetry + io.requester.req.bits.order := request.order + io.requester.req.bits.pCrdType := request.pCrdType + io.requester.req.bits.memAttr := request.memAttr + io.requester.req.bits.snpAttr := request.snpAttr + io.requester.req.bits.lpid := request.lpid + io.requester.req.bits.exclSnoopMe := request.exclSnoopMe + io.requester.req.bits.expCompAck := request.expCompAck + io.requester.req.bits.traceTag := request.traceTag + io.requester.req.bits.tgtId := homeId.U + io.requester.req.bits.pas := request.ns.asUInt + when(requestRx.io.out.valid) { + assert(request.tgtId === 0.U || request.tgtId === homeId.U) + assert(request.rsvdc === 0.U) + } + + val responseRx = Instantiate(new chi.Rx((new Native.Response).getWidth, 4)) + responseRx.io.active := receiving + responseRx.io.link <> io.native.tx.rsp + val response = Wire(new Native.Response) + response.unpack(responseRx.io.out.bits) + io.requester.txRsp.valid := responseRx.io.out.valid + responseRx.io.out.ready := io.requester.txRsp.ready + io.requester.txRsp.bits := 0.U.asTypeOf(io.requester.txRsp.bits) + io.requester.txRsp.bits.qos := response.qos + io.requester.txRsp.bits.tgtId := response.tgtId + io.requester.txRsp.bits.srcId := response.srcId + io.requester.txRsp.bits.txnId := response.txnId + io.requester.txRsp.bits.opcode := response.opcode + io.requester.txRsp.bits.respErr := response.respErr + io.requester.txRsp.bits.resp := response.resp + io.requester.txRsp.bits.fwdState := response.fwdState + io.requester.txRsp.bits.dbid := response.dbid + io.requester.txRsp.bits.pCrdType := response.pCrdType + io.requester.txRsp.bits.traceTag := response.traceTag + + val dataRx = Instantiate(new chi.Rx((new Native.Data).getWidth, 4)) + dataRx.io.active := receiving + dataRx.io.link <> io.native.tx.dat + val data = Wire(new Native.Data) + data.unpack(dataRx.io.out.bits) + io.requester.txDat.valid := dataRx.io.out.valid + dataRx.io.out.ready := io.requester.txDat.ready + io.requester.txDat.bits := 0.U.asTypeOf(io.requester.txDat.bits) + io.requester.txDat.bits.qos := data.qos + io.requester.txDat.bits.tgtId := data.tgtId + io.requester.txDat.bits.srcId := data.srcId + io.requester.txDat.bits.txnId := data.txnId + io.requester.txDat.bits.homeNid := data.homeNid + io.requester.txDat.bits.opcode := data.opcode + io.requester.txDat.bits.respErr := data.respErr + io.requester.txDat.bits.resp := data.resp + io.requester.txDat.bits.dataSource := data.dataSource + io.requester.txDat.bits.dbid := data.dbid + io.requester.txDat.bits.ccid := data.ccid + io.requester.txDat.bits.dataId := data.dataId + io.requester.txDat.bits.traceTag := data.traceTag + io.requester.txDat.bits.be := data.be + io.requester.txDat.bits.data := data.data + when(dataRx.io.out.valid)(assert(data.rsvdc === 0.U)) + + val incomingResponseTx = Instantiate(new chi.Tx((new Native.Response).getWidth, 15)) + incomingResponseTx.io.active := sending + io.native.rx.rsp <> incomingResponseTx.io.link + val incomingResponse = Wire(new Native.Response) + incomingResponse := 0.U.asTypeOf(incomingResponse) + incomingResponse.qos := io.requester.rxRsp.bits.qos + incomingResponse.tgtId := io.requester.rxRsp.bits.tgtId + incomingResponse.srcId := io.requester.rxRsp.bits.srcId + incomingResponse.txnId := io.requester.rxRsp.bits.txnId(7, 0) + incomingResponse.opcode := io.requester.rxRsp.bits.opcode(3, 0) + incomingResponse.respErr := io.requester.rxRsp.bits.respErr + incomingResponse.resp := io.requester.rxRsp.bits.resp + incomingResponse.fwdState := io.requester.rxRsp.bits.fwdState + incomingResponse.dbid := io.requester.rxRsp.bits.dbid(7, 0) + incomingResponse.pCrdType := io.requester.rxRsp.bits.pCrdType + incomingResponse.traceTag := io.requester.rxRsp.bits.traceTag.asBool + incomingResponseTx.io.in.valid := io.requester.rxRsp.valid + incomingResponseTx.io.in.bits := incomingResponse.packed + io.requester.rxRsp.ready := incomingResponseTx.io.in.ready + when(io.requester.rxRsp.valid) { + assert(io.requester.rxRsp.bits.txnId < 256.U) + assert(io.requester.rxRsp.bits.dbid < 256.U && io.requester.rxRsp.bits.opcode < 16.U) + assert( + io.requester.rxRsp.bits.cBusy === 0.U && io.requester.rxRsp.bits.tagOp === 0.U && io.requester.rxRsp.bits.cacheLineId === 0.U + ) + } + + val incomingDataTx = Instantiate(new chi.Tx((new Native.Data).getWidth, 15)) + incomingDataTx.io.active := sending + io.native.rx.dat <> incomingDataTx.io.link + val incomingData = Wire(new Native.Data) + incomingData := 0.U.asTypeOf(incomingData) + incomingData.qos := io.requester.rxDat.bits.qos + incomingData.tgtId := io.requester.rxDat.bits.tgtId + incomingData.srcId := io.requester.rxDat.bits.srcId + incomingData.txnId := io.requester.rxDat.bits.txnId(7, 0) + incomingData.homeNid := io.requester.rxDat.bits.homeNid + incomingData.opcode := io.requester.rxDat.bits.opcode(2, 0) + incomingData.respErr := io.requester.rxDat.bits.respErr + incomingData.resp := io.requester.rxDat.bits.resp + incomingData.dataSource := io.requester.rxDat.bits.dataSource(2, 0) + incomingData.dbid := io.requester.rxDat.bits.dbid(7, 0) + incomingData.ccid := io.requester.rxDat.bits.ccid + incomingData.dataId := io.requester.rxDat.bits.dataId + incomingData.traceTag := io.requester.rxDat.bits.traceTag.asBool + incomingData.be := io.requester.rxDat.bits.be + incomingData.data := io.requester.rxDat.bits.data + incomingDataTx.io.in.valid := io.requester.rxDat.valid + incomingDataTx.io.in.bits := incomingData.packed + io.requester.rxDat.ready := incomingDataTx.io.in.ready + when(io.requester.rxDat.valid) { + assert(io.requester.rxDat.bits.txnId < 256.U) + assert( + io.requester.rxDat.bits.dbid < 256.U && io.requester.rxDat.bits.opcode < 8.U && io.requester.rxDat.bits.dataSource < 8.U + ) + assert(io.requester.rxDat.bits.cBusy === 0.U && io.requester.rxDat.bits.tagOp === 0.U) + assert(io.requester.rxDat.bits.tag === 0.U && io.requester.rxDat.bits.tagUpdate === 0.U) + assert( + io.requester.rxDat.bits.copyAtHome === 0.U && io.requester.rxDat.bits.numDat === 0.U && io.requester.rxDat.bits.replicate === 0.U + ) + assert(io.requester.rxDat.bits.cacheLineId === 0.U && io.requester.rxDat.bits.dataPull === 0.U) + } + + val snoopTx = Instantiate(new chi.Tx((new Native.Snoop).getWidth, 4)) + snoopTx.io.active := sending + io.native.rx.snp <> snoopTx.io.link + val snoop = Wire(new Native.Snoop) + snoop := 0.U.asTypeOf(snoop) + snoop.qos := io.requester.snp.bits.qos + snoop.srcId := io.requester.snp.bits.srcId + snoop.txnId := io.requester.snp.bits.txnId(7, 0) + snoop.fwdNid := io.requester.snp.bits.fwdNid + snoop.fwdTxnId := io.requester.snp.bits.fwdTxnId(7, 0) + snoop.opcode := io.requester.snp.bits.opcode(4, 0) + snoop.addr := io.requester.snp.bits.addr + snoop.doNotGoToSd := io.requester.snp.bits.doNotGoToSd.asBool + snoop.retToSrc := io.requester.snp.bits.retToSrc.asBool + snoop.traceTag := io.requester.snp.bits.traceTag.asBool + snoop.ns := io.requester.snp.bits.pas(0) + snoopTx.io.in.valid := io.requester.snp.valid + snoopTx.io.in.bits := snoop.packed + io.requester.snp.ready := snoopTx.io.in.ready + when(io.requester.snp.valid) { + assert(io.requester.snp.bits.txnId < 256.U) + assert(io.requester.snp.bits.fwdTxnId < 256.U && io.requester.snp.bits.pas < 2.U) + } + +} diff --git a/examples/components/cores/xiangshan/arch/Native.scala b/examples/components/cores/xiangshan/arch/Native.scala new file mode 100644 index 000000000..7fc589e37 --- /dev/null +++ b/examples/components/cores/xiangshan/arch/Native.scala @@ -0,0 +1,239 @@ +package examples.core.xiangshan + +import chisel3.experimental.hierarchy.{instantiable, public} + +import chisel3._ +import java.nio.file.Paths +import scala.io.Source +import scala.util.Using +import chisel3.util._ +import framework.ip.bus.{axi4, chi} + +/** Issue B layout at the pinned upstream RTL boundary. Fields are declared LSB first. */ +object Native { + + class Request extends chi.Flit { + val qos = UInt(4.W) + val tgtId = UInt(7.W) + val srcId = UInt(7.W) + val txnId = UInt(8.W) + val returnNid = UInt(7.W) + val stashNidValidEndian = Bool() + val returnTxnId = UInt(8.W) + val opcode = UInt(6.W) + val size = UInt(3.W) + val addr = UInt(48.W) + val ns = Bool() + val likelyShared = Bool() + val allowRetry = Bool() + val order = UInt(2.W) + val pCrdType = UInt(4.W) + val memAttr = UInt(4.W) + val snpAttr = Bool() + val lpid = UInt(5.W) + val exclSnoopMe = Bool() + val expCompAck = Bool() + val traceTag = Bool() + val rsvdc = UInt(4.W) + + def fieldsLSB = Seq( + qos, + tgtId, + srcId, + txnId, + returnNid, + stashNidValidEndian, + returnTxnId, + opcode, + size, + addr, + ns, + likelyShared, + allowRetry, + order, + pCrdType, + memAttr, + snpAttr, + lpid, + exclSnoopMe, + expCompAck, + traceTag, + rsvdc + ) + + } + + class Response extends chi.Flit { + val qos = UInt(4.W) + val tgtId = UInt(7.W) + val srcId = UInt(7.W) + val txnId = UInt(8.W) + val opcode = UInt(4.W) + val respErr = UInt(2.W) + val resp = UInt(3.W) + val fwdState = UInt(3.W) + val dbid = UInt(8.W) + val pCrdType = UInt(4.W) + val traceTag = Bool() + def fieldsLSB = Seq(qos, tgtId, srcId, txnId, opcode, respErr, resp, fwdState, dbid, pCrdType, traceTag) + } + + class Data extends chi.Flit { + val qos = UInt(4.W) + val tgtId = UInt(7.W) + val srcId = UInt(7.W) + val txnId = UInt(8.W) + val homeNid = UInt(7.W) + val opcode = UInt(3.W) + val respErr = UInt(2.W) + val resp = UInt(3.W) + val dataSource = UInt(3.W) + val dbid = UInt(8.W) + val ccid = UInt(2.W) + val dataId = UInt(2.W) + val traceTag = Bool() + val rsvdc = UInt(4.W) + val be = UInt(32.W) + val data = UInt(256.W) + + def fieldsLSB = Seq( + qos, + tgtId, + srcId, + txnId, + homeNid, + opcode, + respErr, + resp, + dataSource, + dbid, + ccid, + dataId, + traceTag, + rsvdc, + be, + data + ) + + } + + class Snoop extends chi.Flit { + val qos = UInt(4.W) + val srcId = UInt(7.W) + val txnId = UInt(8.W) + val fwdNid = UInt(7.W) + val fwdTxnId = UInt(8.W) + val opcode = UInt(5.W) + val addr = UInt(45.W) + val ns = Bool() + val doNotGoToSd = Bool() + val retToSrc = Bool() + val traceTag = Bool() + def fieldsLSB = Seq(qos, srcId, txnId, fwdNid, fwdTxnId, opcode, addr, ns, doNotGoToSd, retToSrc, traceTag) + } + + class Link extends Bundle { + val txsactive = Output(Bool()) + val rxsactive = Input(Bool()) + val syscoreq = Output(Bool()) + val syscoack = Input(Bool()) + + val tx = new Bundle { + val linkactivereq = Output(Bool()) + val linkactiveack = Input(Bool()) + val req = new chi.Channel((new Request).getWidth) + val rsp = new chi.Channel((new Response).getWidth) + val dat = new chi.Channel((new Data).getWidth) + } + + val rx = Flipped(new Bundle { + val linkactivereq = Output(Bool()) + val linkactiveack = Input(Bool()) + val rsp = new chi.Channel((new Response).getWidth) + val dat = new chi.Channel((new Data).getWidth) + val snp = new chi.Channel((new Snoop).getWidth) + }) + + } + + class Commands extends Bundle { + + val request = Decoupled(new Bundle { + val instruction = UInt(32.W) + val pc = UInt(64.W) + val rs1 = UInt(64.W) + val rs2 = UInt(64.W) + + val context = new Bundle { + val satp = UInt(64.W) + val privilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + val virt = Bool() + + val pmp = Vec( + 64, + new Bundle { + val cfg = UInt(8.W) + val addr = UInt(46.W) + } + ) + + } + + }) + + val complete = Flipped(Valid(Bool())) + } + + class Address(bits: Int) extends Bundle { + val id = UInt(4.W) + val addr = UInt(bits.W) + val len = UInt(8.W) + val size = UInt(3.W) + val burst = UInt(2.W) + val lock = Bool() + val cache = UInt(4.W) + val prot = UInt(3.W) + val qos = UInt(4.W) + } + + class Bus(addressBits: Int) extends Bundle { + val aw = Decoupled(new Address(addressBits)) + val w = Decoupled(new axi4.WriteData(axi4.Params(addressBits, 128, 4))) + val b = Flipped(Decoupled(new axi4.WriteResponse(axi4.Params(addressBits, 128, 4)))) + val ar = Decoupled(new Address(addressBits)) + val r = Flipped(Decoupled(new axi4.ReadData(axi4.Params(addressBits, 128, 4)))) + } + +} + +/** Only this internal leaf knows the native RTL's pin names. */ +@instantiable +class Native extends BlackBox with HasBlackBoxPath { + override def desiredName = "XiangShanNative" + + @public + val io = IO(new Bundle { + val clock = Input(Clock()) + val reset = Input(AsyncReset()) + val io_hartId = Input(UInt(6.W)) + val io_resetVector = Input(UInt(48.W)) + val io_nodeId = Input(UInt(7.W)) + val io_interrupts = Input(Vec(4, Bool())) + val io_commands = new Native.Commands + val io_timeRequest = Decoupled(Bool()) + val io_time = Flipped(Valid(UInt(64.W))) + val io_chi = new Native.Link + val uncachedAXI = Vec(1, new Native.Bus(31)) + }) + + val manifest = Paths.get(getClass.getResource("/xiangshan/filelist.f").toURI) + + val files = Using.resource(Source.fromFile(manifest.toFile))( + _.getLines().filter(_.nonEmpty).toVector + ) + + files.foreach(file => addPath(manifest.getParent.resolve(file.stripPrefix("./")).toString)) + +} diff --git a/examples/components/cores/xiangshan/arch/XiangShan.scala b/examples/components/cores/xiangshan/arch/XiangShan.scala new file mode 100644 index 000000000..0f108cf40 --- /dev/null +++ b/examples/components/cores/xiangshan/arch/XiangShan.scala @@ -0,0 +1,255 @@ +package examples.core.xiangshan + +import framework.acc.balldomain.bbus.BallFactory +import sims.configs.SimParam +import framework.acc.balldomain.ball.blink.BlinkIO + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.xiangshan.configs.Params +import framework.core.clink.{CLinkIO, HasCLink} +import framework.core.clink.configs.{Params => CLinkParam} +import framework.ip.bus.axi4 +import framework.ip.bus.chi.rnf.AxiCache + +import framework.acc.frontend.Frontend +import framework.acc.memdomain.MemDomain +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.ball.blink.HasBlink +import framework.acc.debug.Debug + +@instantiable +class XiangShan( + p: Params, + simp: SimParam, + factories: Map[String, BallFactory]) + extends Module + with HasCLink { + val diffTest = simp.diffTest + + @public + val clink = IO(new CLinkIO(CLinkParam( + 1, + false, + Seq(p.nodeId) ++ p.acc.toSeq.map(_ => p.dmaCache.nodeId), + p.chi, + p.axi, + p.axi, + None + ))) + + val native = Instantiate(new Native) + native.io.clock := clock + native.io.reset := reset.asAsyncReset + native.io.io_hartId := p.hartId.U + native.io.io_nodeId := p.nodeId.U + native.io.io_resetVector := p.resetVector.U + native.io.io_interrupts(0) := clink.interrupts(0).software + native.io.io_interrupts(1) := clink.interrupts(0).timer + native.io.io_interrupts(2) := clink.interrupts(0).external + native.io.io_interrupts(3) := clink.interrupts(0).supervisorExternal + + val coherent = Instantiate(new Chi(p.chi, p.homeId)) + coherent.io.native <> native.io.io_chi + clink.memory.coherent(0) <> coherent.io.requester + + val peripherals = Instantiate(new axi4.Interconnect(axi4.InterconnectParam(p.axi, 2, Some(16)))) + val upstream = native.io.uncachedAXI(0) + val downstream = peripherals.io.in(0) + downstream.aw.valid := upstream.aw.valid + upstream.aw.ready := downstream.aw.ready + downstream.aw.bits := 0.U.asTypeOf(downstream.aw.bits) + downstream.aw.bits.id := upstream.aw.bits.id + downstream.aw.bits.addr := upstream.aw.bits.addr + downstream.aw.bits.len := upstream.aw.bits.len + downstream.aw.bits.size := upstream.aw.bits.size + downstream.aw.bits.burst := upstream.aw.bits.burst + downstream.aw.bits.lock := upstream.aw.bits.lock + downstream.aw.bits.cache := upstream.aw.bits.cache + downstream.aw.bits.prot := upstream.aw.bits.prot + downstream.aw.bits.qos := upstream.aw.bits.qos + downstream.ar.valid := upstream.ar.valid + upstream.ar.ready := downstream.ar.ready + downstream.ar.bits := 0.U.asTypeOf(downstream.ar.bits) + downstream.ar.bits.id := upstream.ar.bits.id + downstream.ar.bits.addr := upstream.ar.bits.addr + downstream.ar.bits.len := upstream.ar.bits.len + downstream.ar.bits.size := upstream.ar.bits.size + downstream.ar.bits.burst := upstream.ar.bits.burst + downstream.ar.bits.lock := upstream.ar.bits.lock + downstream.ar.bits.cache := upstream.ar.bits.cache + downstream.ar.bits.prot := upstream.ar.bits.prot + downstream.ar.bits.qos := upstream.ar.bits.qos + downstream.w <> upstream.w + upstream.b <> downstream.b + upstream.r <> downstream.r + clink.memory.uncached <> peripherals.io.out + + val time = peripherals.io.in(1) + val timePending = RegInit(false.B) + time.aw.valid := false.B + time.aw.bits := 0.U.asTypeOf(time.aw.bits) + time.w.valid := false.B + time.w.bits := 0.U.asTypeOf(time.w.bits) + time.b.ready := true.B + time.ar.valid := native.io.io_timeRequest.valid && !timePending + time.ar.bits := 0.U.asTypeOf(time.ar.bits) + time.ar.bits.addr := p.mtimeAddress.U + time.ar.bits.size := 3.U + time.ar.bits.burst := 1.U + native.io.io_timeRequest.ready := time.ar.ready && !timePending + when(time.ar.fire)(timePending := true.B) + time.r.ready := timePending + native.io.io_time.valid := time.r.fire + native.io.io_time.bits := (time.r.bits.data >> ((p.mtimeAddress & 15).toInt * 8))(63, 0) + when(time.r.fire) { + assert(time.r.bits.last && time.r.bits.resp === 0.U && time.r.bits.id === 0.U) + timePending := false.B + } + + p.acc match { + case Some(acc) => + val frontend = + Instantiate(new Frontend(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, p.permission, simp)) + val memDomain = Instantiate(new MemDomain( + acc.frontend, + acc.memDomain, + acc.gpDomain, + acc.transport, + p.permission, + p.tracking, + p.walker, + simp + )) + val domain = + Instantiate(new BallDomain(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, p.permission, factories)) + if (acc.gpDomain.enable) { + val gp = Instantiate(new GpDomain(acc.frontend, acc.memDomain, acc.gpDomain)) + gp.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> gp.io.complete + gp.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> gp.io.banks + memDomain.io.gpLayoutRequest.get <> gp.io.layoutRequest + gp.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + domain.io.kernel.get.request <> gp.io.ball.request + gp.io.ball.response <> domain.io.kernel.get.response + } + val command = native.io.io_commands.request + frontend.io.rocc.cmd.valid := command.valid + command.ready := frontend.io.rocc.cmd.ready + frontend.io.rocc.cmd.bits.instruction.raw_inst := command.bits.instruction + frontend.io.rocc.cmd.bits.instruction.pc := command.bits.pc + frontend.io.rocc.cmd.bits.instruction.rs1Data := command.bits.rs1 + frontend.io.rocc.cmd.bits.instruction.rs2Data := command.bits.rs2 + frontend.io.rocc.cmd.bits.satp := command.bits.context.satp + frontend.io.rocc.cmd.bits.effectivePrivilege := command.bits.context.privilege + frontend.io.rocc.cmd.bits.sum := command.bits.context.sum + frontend.io.rocc.cmd.bits.mxr := command.bits.context.mxr + for ((target, entry) <- frontend.io.rocc.cmd.bits.pmp.zip(command.bits.context.pmp)) { + target.cfg := entry.cfg.asTypeOf(target.cfg) + target.addr := entry.addr + val encoded = ((entry.addr.pad(48) << 1) | entry.cfg(3).asUInt | 1023.U)(47, 0) + target.mask := (((encoded & ~(encoded + 1.U)) << 2) | 3.U)(47, 0) + } + frontend.io.rocc.resp.ready := true.B + val pending = RegInit(false.B) + when(command.fire)(pending := true.B) + native.io.io_commands.complete.valid := pending && (!frontend.io.rocc.busy || frontend.io.rocc.interrupt) + native.io.io_commands.complete.bits := frontend.io.rocc.interrupt + when(native.io.io_commands.complete.valid)(pending := false.B) + + domain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> domain.io.complete + memDomain.io.ball <> domain.io.memory + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + frontend.io.barrier.release := frontend.io.barrier.arrive + memDomain.io.banks <> domain.io.banks + memDomain.io.ballLayoutRequest <> domain.io.layoutRequest + domain.io.layoutResponse <> memDomain.io.ballLayoutResponse + + // DMA is a coherent requester alongside the native core. The submitting core has already drained stores + // and cannot execute younger memory operations until this command completes. + memDomain.io.memory.cpuQuery := 0.U.asTypeOf(memDomain.io.memory.cpuQuery) + val maintained = RegInit(false.B) + val maintenanceTag = Reg(chiselTypeOf(memDomain.io.memory.maintenance.bits.tag)) + memDomain.io.memory.maintenance.ready := !maintained + when(memDomain.io.memory.maintenance.fire) { + assert(pending) + maintenanceTag := memDomain.io.memory.maintenance.bits.tag + maintained := true.B + } + memDomain.io.memory.maintained.valid := maintained + memDomain.io.memory.maintained.bits.tag := maintenanceTag + memDomain.io.memory.maintained.bits.ok := true.B + when(memDomain.io.memory.maintained.fire)(maintained := false.B) + + val access = Instantiate(new axi4.Interconnect(axi4.InterconnectParam(p.axi, 2, Some(16)))) + access.io.in(0) <> memDomain.io.ddr + val pte = access.io.in(1) + val ptePending = RegInit(false.B) + val highWord = Reg(Bool()) + pte.aw.valid := false.B + pte.aw.bits := 0.U.asTypeOf(pte.aw.bits) + pte.w.valid := false.B + pte.w.bits := 0.U.asTypeOf(pte.w.bits) + pte.b.ready := true.B + pte.ar.valid := memDomain.io.memory.pteRequest.valid && !ptePending + pte.ar.bits := 0.U.asTypeOf(pte.ar.bits) + pte.ar.bits.addr := memDomain.io.memory.pteRequest.bits.addr + pte.ar.bits.size := 3.U + pte.ar.bits.burst := 1.U + memDomain.io.memory.pteRequest.ready := pte.ar.ready && !ptePending + when(pte.ar.fire) { ptePending := true.B; highWord := pte.ar.bits.addr(3) } + memDomain.io.memory.pteResponse.valid := pte.r.valid && ptePending + memDomain.io.memory.pteResponse.bits.data := Mux(highWord, pte.r.bits.data(127, 64), pte.r.bits.data(63, 0)) + memDomain.io.memory.pteResponse.bits.error := pte.r.bits.resp =/= 0.U + pte.r.ready := memDomain.io.memory.pteResponse.ready && ptePending + when(pte.r.fire) { assert(pte.r.bits.last && pte.r.bits.id === 0.U); ptePending := false.B } + val dma = Instantiate(new AxiCache(p.axi, p.dmaCache)) + dma.io.axi <> access.io.out + dma.io.axi.ar.bits.cache := 15.U + dma.io.axi.aw.bits.cache := 15.U + clink.memory.coherent(1) <> dma.io.chi + + val debug = Instantiate(new Debug(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, simp, 0, 0)) + debug.io.ant.valid := false.B + debug.io.ant.bits := 0.U.asTypeOf(debug.io.ant.bits) + debug.io.acc.frontend <> frontend.io.debug + debug.io.acc.trace.memory := memDomain.io.traceMemory + debug.io.acc.trace.ballStarted := domain.io.traceStarted + debug.io.acc.trace.ballCompleted := domain.io.traceCompleted + debug.io.acc.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, src) => sink := src } + for (port <- memDomain.io.shared.bank.requests) { + port.read.req.ready := false.B + port.read.resp.valid := false.B + port.read.resp.bits := 0.U.asTypeOf(port.read.resp.bits) + port.write.req.ready := false.B + port.write.resp.valid := false.B + port.write.resp.bits := 0.U.asTypeOf(port.write.resp.bits) + } + memDomain.io.shared.bank.config.ready := false.B + memDomain.io.shared.bank.mapping.ready := false.B + memDomain.io.shared.bank.local.request.valid := false.B + memDomain.io.shared.bank.local.request.bits := 0.U.asTypeOf(memDomain.io.shared.bank.local.request.bits) + memDomain.io.shared.bank.local.response.ready := false.B + memDomain.io.shared.move.command.ready := false.B + memDomain.io.shared.move.completion.valid := false.B + memDomain.io.shared.move.completion.bits := false.B + case None => + native.io.io_commands.request.ready := true.B + native.io.io_commands.complete.valid := RegNext(native.io.io_commands.request.fire, false.B) + native.io.io_commands.complete.bits := true.B + } + clink.memory.ddr.aw.valid := false.B + clink.memory.ddr.aw.bits := 0.U.asTypeOf(clink.memory.ddr.aw.bits) + clink.memory.ddr.w.valid := false.B + clink.memory.ddr.w.bits := 0.U.asTypeOf(clink.memory.ddr.w.bits) + clink.memory.ddr.ar.valid := false.B + clink.memory.ddr.ar.bits := 0.U.asTypeOf(clink.memory.ddr.ar.bits) + clink.memory.ddr.b.ready := true.B + clink.memory.ddr.r.ready := true.B +} diff --git a/examples/components/cores/xiangshan/arch/build.sc b/examples/components/cores/xiangshan/arch/build.sc new file mode 100644 index 000000000..2de8ecf46 --- /dev/null +++ b/examples/components/cores/xiangshan/arch/build.sc @@ -0,0 +1,41 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.^.arch.src.main.scala.framework.{build => framework} + +val componentRoot: os.Path = millSourcePath / os.up +val vendorRoot: os.Path = componentRoot / "thirdparty" / "xiangshan" +val vendorHead: os.Path = os.Path(os.proc("git", "rev-parse", "--git-path", "HEAD").call(cwd = vendorRoot).out.text().trim, vendorRoot) + +def nativeInputs = T.sources { + val paths = Seq( + "src", "rocket-chip/src", "rocket-chip/macros/src", "rocket-chip/cde/cde/src", + "rocket-chip/hardfloat/hardfloat/src", "difftest/src", "utility/src", "huancun/src", + "fudian/src", "yunsuan/src", "coupledL2/src", "openLLC/src", "openLLC/openNCB/src", + "ChiselAIA/src", "ChiselIOPMP/src", "build.sc", ".mill-version", + "rocket-chip/common.sc", "rocket-chip/cde/common.sc", "rocket-chip/hardfloat/common.sc", + "huancun/common.sc", "coupledL2/common.sc", "openLLC/common.sc" + ) + paths.map(path => PathRef(vendorRoot / os.RelPath(path))) :+ PathRef(vendorHead) +} + +def nativeRtl = T { + nativeInputs() + val output = T.dest / "xiangshan" + os.proc("mill", "--no-server", "native.runMain", "buckyball.Emit", + "--target-dir", output, "--target", "systemverilog").call(cwd = vendorRoot) + os.move(output / "extern_modules.sv", output / "XiangShanExtern.sv") + os.write.over(output / "filelist.f", os.read(output / "filelist.f").replace("extern_modules.sv", "XiangShanExtern.sv")) + PathRef(T.dest) +} + +trait ScalaBuild extends SbtModule { + override def scalaVersion = "2.13.16" + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") +} +object library extends ScalaBuild { + override def moduleDeps = Seq(framework.library) + override def sources = T.sources(componentRoot / "arch", componentRoot / "configs") + override def resources = T.sources(Seq(nativeRtl())) +} diff --git a/examples/components/cores/xiangshan/configs/Params.scala b/examples/components/cores/xiangshan/configs/Params.scala new file mode 100644 index 000000000..297468b90 --- /dev/null +++ b/examples/components/cores/xiangshan/configs/Params.scala @@ -0,0 +1,73 @@ +package examples.core.xiangshan.configs + +import buckyball.config.CorePlacement +import buckyball.components.xiangshan.{Params => NativeParams} +import framework.ip.bus.{axi4, chi => chiBus} +import framework.ip.bus.chi.rnf.RnfParams + +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.acc.configs.AccParam +import framework.acc.frontend.configs.FrontendParam +import framework.acc.memdomain.configs.MemDomainParam +import framework.acc.gpdomain.configs.GpDomainParam +import framework.acc.balldomain.configs.BallDomainParam +import framework.acc.memdomain.tomem.configs.{Params => TransportParam} + +case class Params( + chi: chiBus.Params, + axi: axi4.Params, + hartId: Int, + nodeId: Int, + homeId: Int, + resetVector: BigInt, + mtimeAddress: BigInt, + dmaCache: RnfParams, + permission: PmpParams, + tracking: TrackingParam, + walker: WalkerParam, + acc: Option[AccParam]) { + require(chi.nodeIdBits == 7 && chi.addressBits == 48 && chi.dataBits == 256) + require(axi.addressBits == 48 && axi.dataBits == 128 && axi.idBits == 4) + require(hartId >= 0 && hartId < 64 && nodeId > 0 && nodeId < homeId && homeId < 128) + require((mtimeAddress & 7) == 0) + require(permission.addressBits == 48 && permission.entries == 64) + require(dmaCache.nodeId != nodeId && dmaCache.chi == chi && dmaCache.homeId == homeId) + acc.foreach(a => require(!a.memDomain.sharedEnable && a.transport.axi == axi)) +} + +object Params { + + def fromProto(core: CorePlacement): Params = { + require(core.getThreadIndicesCount == 0) + val h = core + val native = core.getImplementation.unpack(classOf[NativeParams]) + val bus = chiBus.Params.fromProto(h.getMemory.getHome.getRequesters) + val acc = Option.when(core.hasAccelerator) { + val a = core.getAccelerator + AccParam( + FrontendParam.fromProto(a.getFrontend, core.getParam.getXLen, core.getHartId), + MemDomainParam.fromProto(a.getMem, core.getSharedMem, core.getVirtualBankCount, 1), + GpDomainParam.fromProto(a.getGpdomain), + BallDomainParam.fromProto(a.getBalldomain), + TransportParam.fromProto(a.getTransport) + ) + } + Params( + bus, + axi4.Params.fromProto(h.getMemory.getUncached.getAxi), + h.getHartId, + native.getNodeId, + native.getHomeId, + BigInt(java.lang.Long.toUnsignedString(h.getResetVector)), + BigInt(java.lang.Long.toUnsignedString(h.getMtimeAddress)), + RnfParams.fromProto(native.getDmaCache, bus), + PmpParams(48, 64), + TrackingParam.fromProto(h.getMemory.getDcache.getTracking), + WalkerParam(bus, h.getMemory.getAccess.getTranslationEntries), + acc + ) + } + +} diff --git a/examples/components/cores/xiangshan/configs/xiangshan_params.proto b/examples/components/cores/xiangshan/configs/xiangshan_params.proto new file mode 100644 index 000000000..01caec49e --- /dev/null +++ b/examples/components/cores/xiangshan/configs/xiangshan_params.proto @@ -0,0 +1,10 @@ +syntax = "proto2"; +import "hardware.proto"; +package buckyball.components.xiangshan; +option java_package = "buckyball.components.xiangshan"; +option java_multiple_files = true; +message Params { + required uint32 node_id = 1; + required uint32 home_id = 2; + required buckyball.config.RnfHardware dma_cache = 3; +} diff --git a/examples/components/cores/xiangshan/stack/CMakeLists.txt b/examples/components/cores/xiangshan/stack/CMakeLists.txt new file mode 100644 index 000000000..bb4434ab2 --- /dev/null +++ b/examples/components/cores/xiangshan/stack/CMakeLists.txt @@ -0,0 +1,25 @@ +# ===----------------------------------------------------------------------=== +# The xiangshan core: a CPU core without hardware threads; jobs run on the core. +# +# bbdev stack --build "--design " +# +# builds it for each design that uses it, configured with DESIGN_CMAKE +# (generated from design.pb) and CORE. Products: +# execute.o runs jobs on the core itself; links into the device runtime +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(xiangshan-core LANGUAGES NONE) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +add_custom_command( + OUTPUT execute.o + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" + -c "${RUNTIME}/execute/direct.c" -o execute.o + DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" + VERBATIM) + +add_custom_target(execute ALL DEPENDS execute.o) diff --git a/examples/components/cores/xiangshan/workloads/ctests/CMakeLists.txt b/examples/components/cores/xiangshan/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..416956421 --- /dev/null +++ b/examples/components/cores/xiangshan/workloads/ctests/CMakeLists.txt @@ -0,0 +1 @@ +add_buckyball_ctest(memory_matrix.c) diff --git a/examples/components/cores/xiangshan/workloads/ctests/memory_matrix.c b/examples/components/cores/xiangshan/workloads/ctests/memory_matrix.c new file mode 100644 index 000000000..1b1d8e04b --- /dev/null +++ b/examples/components/cores/xiangshan/workloads/ctests/memory_matrix.c @@ -0,0 +1,47 @@ +#include + +static volatile int32_t a[32][32], b[32][16], c[32][16]; +static volatile uint64_t atomic_word; +static volatile uint64_t upgrade_word[8] __attribute__((aligned(64))) = {7}; + +int main(void) { + for (int i = 0; i < 32; ++i) + for (int k = 0; k < 32; ++k) + a[i][k] = (i * 3 + k * 5) % 13 - 6; + for (int k = 0; k < 32; ++k) + for (int j = 0; j < 16; ++j) + b[k][j] = (k * 7 + j * 2) % 11 - 5; + for (int i = 0; i < 32; ++i) + for (int j = 0; j < 16; ++j) { + int32_t sum = 0; + for (int k = 0; k < 32; ++k) + sum += a[i][k] * b[k][j]; + c[i][j] = sum; + } + for (int i = 0; i < 32; ++i) + for (int j = 0; j < 16; ++j) { + int32_t expected = 0; + for (int k = 0; k < 32; ++k) + expected += ((i * 3 + k * 5) % 13 - 6) * ((k * 7 + j * 2) % 11 - 5); + if (c[i][j] != expected) + return 2; + } + uint64_t old, value = 19; + __asm__ volatile("amoadd.d.aqrl %0, %2, (%1)" + : "=r"(old) + : "r"(&atomic_word), "r"(value) + : "memory"); + if (old != 0 || atomic_word != 19) + return 3; + if (upgrade_word[0] != 7) + return 5; + upgrade_word[0] = 23; + if (upgrade_word[0] != 23) + return 6; + uint64_t before, after; + __asm__ volatile("rdtime %0" : "=r"(before)); + __asm__ volatile("rdtime %0" : "=r"(after)); + if (before == 0 || after <= before) + return 4; + return 0; +} diff --git a/examples/components/threads/attention/arch/AttentionThread.scala b/examples/components/threads/attention/arch/AttentionThread.scala new file mode 100644 index 000000000..7c5b92bfd --- /dev/null +++ b/examples/components/threads/attention/arch/AttentionThread.scala @@ -0,0 +1,77 @@ +package examples.threads.attention + +import framework.thread.ant.Ant +import framework.core.tbus.ThreadFactory +import framework.core.configs.CoreParam +import sims.configs.SimParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.thread.configs.ThreadParam +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} + +import framework.acc.debug.Debug + +import chisel3.experimental.hierarchy.Instance + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.thread.tlink.{HasTLink, TLinkIO} + +@instantiable +class AttentionThread( + threadp: ThreadParam, + permissionp: PmpParams, + trackingp: TrackingParam, + walkerp: WalkerParam, + simp: SimParam, + traceId: Int, + sharedOwnerId: Int) + extends Module + with HasTLink { + val sharedPorts = threadp.acc.get.memDomain.sharedPorts + val diffTest = simp.diffTest + val accp = threadp.acc.get + val antp = threadp.ant.get + + val ant: Instance[Ant] = + Instantiate(new Ant(antp, permissionp)) + + val acc: Instance[Acc] = Instantiate(new Acc(accp, permissionp, trackingp, walkerp, simp)) + + val debug: Instance[Debug] = + Instantiate(new Debug(accp.frontend, accp.memDomain, accp.gpDomain, accp.ballDomain, simp, traceId, sharedOwnerId)) + + debug.io.ant := ant.io.trace + debug.io.acc <> acc.io.debug + + @public + val tlink = IO(new TLinkIO(threadp, permissionp, trackingp, walkerp, simp)) + + ant.io.task <> tlink.task + tlink.tss <> ant.io.shared + ant.io.rocc <> acc.io.rocc + tlink.memory <> acc.io.memory + tlink.ddr <> acc.io.ddr + tlink.shm <> acc.io.shm + acc.io.shared_bank_hashes.zip(tlink.sharedHashes).foreach { case (port, hashes) => port := hashes } +} + +object AttentionThread extends ThreadFactory { + + def instantiate( + corep: CoreParam, + threadp: ThreadParam, + simp: SimParam, + traceId: Int + ): TLinkIO = + Instantiate(new AttentionThread( + threadp, + corep.privileged.permission, + corep.memory.dcache.tracking, + corep.memory.memoryAccess.walker, + simp, + traceId, + corep.hartId + )).tlink + +} diff --git a/examples/components/threads/attention/arch/acc.scala b/examples/components/threads/attention/arch/acc.scala new file mode 100644 index 000000000..161174faa --- /dev/null +++ b/examples/components/threads/attention/arch/acc.scala @@ -0,0 +1,119 @@ +package examples.threads.attention + +import framework.acc.balldomain.bbus.generated.BallComponents +import sims.configs.SimParam +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.configs.AccParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.acc.debug.AcceleratorDebugPort +import framework.acc.frontend.RoCCIO +import framework.acc.frontend.Frontend +import framework.acc.memdomain.{MemDomain, PhysicalBankHash} +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.bbus.BBus +import framework.core.memory.MemoryPort +import framework.thread.tlink.SharedLink +import framework.ip.bus.axi4 + +@instantiable +class Acc( + val accp: AccParam, + val permissionp: PmpParams, + val trackingp: TrackingParam, + val walkerp: WalkerParam, + val simp: SimParam) + extends Module { + val commandDataBits = accp.frontend.commandDataBits + val ownerId = accp.frontend.ownerId + val sharedPorts = accp.memDomain.sharedPorts + val diffTest = simp.diffTest + val frontp = accp.frontend + val memp = accp.memDomain + val gpp = accp.gpDomain + val ballp = accp.ballDomain + val transportp = accp.transport + + val sharedHashCount = if (memp.sharedEnable) memp.sharedBankNum else 0 + + @public + val io = IO(new Bundle { + val debug = new AcceleratorDebugPort(frontp, memp, gpp, ballp, simp) + + val rocc = new RoCCIO(commandDataBits, permissionp) + val ddr = new axi4.Port(transportp.axi) + val memory = new MemoryPort(trackingp, walkerp) + val shm = new SharedLink(frontp, memp) + + val shared_bank_hashes = Option.when(diffTest && memp.sharedEnable)( + Input(Vec(sharedHashCount, new PhysicalBankHash(frontp))) + ) + + }) + + // --- Instantiate domains --- + val frontend: Instance[Frontend] = Instantiate(new Frontend(frontp, memp, gpp, ballp, permissionp, simp)) + + val ballDomain: Instance[BallDomain] = + Instantiate(new BallDomain(frontp, memp, gpp, ballp, permissionp, BallComponents.factories)) + + val memDomain: Instance[MemDomain] = + Instantiate(new MemDomain(frontp, memp, gpp, transportp, permissionp, trackingp, walkerp, simp)) + + val gpDomain = + if (gpp.enable) Some(Instantiate(new GpDomain(frontp, memp, gpp))) + else None + + memDomain.io.shared_bank_hashes.foreach(_ := io.shared_bank_hashes.get) + + io.debug.frontend <> frontend.io.debug + io.debug.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, source) => sink := source } + io.debug.trace.memory := memDomain.io.traceMemory + io.debug.trace.ballStarted := ballDomain.io.traceStarted + io.debug.trace.ballCompleted := ballDomain.io.traceCompleted + + io.rocc <> frontend.io.rocc + io.memory <> memDomain.io.memory + io.ddr <> memDomain.io.ddr + + // --- Frontend -> BallDomain --- + memDomain.io.ball <> ballDomain.io.memory + ballDomain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> ballDomain.io.complete + + // --- Frontend -> MemDomain --- + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + + if (gpp.enable) { + val controller = gpDomain.get + controller.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> controller.io.complete + controller.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> controller.io.banks + memDomain.io.gpLayoutRequest.get <> controller.io.layoutRequest + controller.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + ballDomain.io.kernel.get <> controller.io.ball + } + + memDomain.io.banks <> ballDomain.io.banks + memDomain.io.ballLayoutRequest <> ballDomain.io.layoutRequest + ballDomain.io.layoutResponse <> memDomain.io.ballLayoutResponse + + // --- Shared memory passthrough --- + io.shm.bank.requests <> memDomain.io.shared.bank.requests + io.shm.move <> memDomain.io.shared.move + io.shm.bank.local <> memDomain.io.shared.bank.local + io.shm.bank.config <> memDomain.io.shared.bank.config + io.shm.bank.mapping <> memDomain.io.shared.bank.mapping + + // --- Barrier passthrough --- + io.shm.barrierArrive := frontend.io.barrier.arrive + frontend.io.barrier.release := io.shm.barrierRelease + +} diff --git a/examples/components/threads/attention/configs/balldomains/default.toml b/examples/components/threads/attention/configs/balldomains/default.toml new file mode 100644 index 000000000..cc8647054 --- /dev/null +++ b/examples/components/threads/attention/configs/balldomains/default.toml @@ -0,0 +1,53 @@ +[[balls]] +id = 0 +name = "MxmmBall" +component = "mxmm" +param = "../../../../balls/mxmm/configs/default.toml" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "MXMM_MXFP8", funct7 = 71 }, + { mnemonic = "MXMM_MXFP8_WINDOW", funct7 = 75 }, + { mnemonic = "MXMM_FMA32", funct7 = 72 }, + { mnemonic = "MXMM_F32", funct7 = 73 }, +] + +[[balls]] +id = 1 +name = "ToInt8Ball" +component = "toint8" +param = "../../../../balls/toint8/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "QUANT_F32_TO_I8", funct7 = 51 }, + { mnemonic = "QUANT_I32_TO_I8", funct7 = 67 }, +] + +[[balls]] +id = 2 +name = "Int2FpBall" +component = "int2fp" +param = "../../../../balls/int2fp/configs/default.toml" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "INT32_TO_FP32", funct7 = 68 }, +] + +[[balls]] +id = 3 +name = "TransposeBall" +component = "transpose" +param = "../../../../balls/transpose/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "TRANSPOSE", funct7 = 49 }, +] + +[[balls]] +id = 4 +name = "MxquantBall" +component = "mxquant" +param = "../../../../balls/mxquant/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "MXQUANT", funct7 = 54 }, +] diff --git a/examples/components/threads/attention/configs/cpu/default.toml b/examples/components/threads/attention/configs/cpu/default.toml new file mode 100644 index 000000000..c752c9500 --- /dev/null +++ b/examples/components/threads/attention/configs/cpu/default.toml @@ -0,0 +1,2 @@ +kind = "ant" +config = "../../../../../../arch/src/main/scala/framework/thread/ant/configs/default.toml" diff --git a/examples/components/threads/attention/configs/default.toml b/examples/components/threads/attention/configs/default.toml new file mode 100644 index 000000000..c081f963f --- /dev/null +++ b/examples/components/threads/attention/configs/default.toml @@ -0,0 +1,12 @@ +source = ".." +# Attention Thread Configuration (default) + +balldomain = "balldomains/default.toml" +memdomain = "memdomains/default.toml" +frontend = "frontends/default.toml" +gpdomain = "gpdomain/default.toml" +cpu = "cpu/default.toml" + +[compiler] +name = "attention" +component = "attention" diff --git a/examples/components/threads/attention/configs/frontends/default.toml b/examples/components/threads/attention/configs/frontends/default.toml new file mode 100644 index 000000000..5c58b1cb2 --- /dev/null +++ b/examples/components/threads/attention/configs/frontends/default.toml @@ -0,0 +1,9 @@ +bankAliasEnable = true +robEntries = 16 +rsOutOfOrderResponse = true +bankIdLen = 10 +vbankIdUpperBound = 31 +sharedBankIdBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/examples/cores/attention/configs/rvv/default.toml b/examples/components/threads/attention/configs/gpdomain/default.toml similarity index 100% rename from examples/cores/attention/configs/rvv/default.toml rename to examples/components/threads/attention/configs/gpdomain/default.toml diff --git a/examples/components/threads/attention/configs/memdomains/default.toml b/examples/components/threads/attention/configs/memdomains/default.toml new file mode 100644 index 000000000..0f187340e --- /dev/null +++ b/examples/components/threads/attention/configs/memdomains/default.toml @@ -0,0 +1,23 @@ +# Attention Thread Memory Domain Configuration (default) +# Private (per-thread) memory parameters. Shared memory lives in the core TOML. + +[bank] +layoutEnable = true +num = 20 +width = 128 +entries = 512 +maskLen = 16 +requestPorts = 7 + +[dma] +busWidth = 128 + +[mmio] +enable = false +bankNum = 16 +bankEntries = 64 +bankWidth = 128 +readWidth = 8 + +[mem] +addrLen = 39 diff --git a/examples/components/threads/attention/stack/CMakeLists.txt b/examples/components/threads/attention/stack/CMakeLists.txt new file mode 100644 index 000000000..086ec8497 --- /dev/null +++ b/examples/components/threads/attention/stack/CMakeLists.txt @@ -0,0 +1,157 @@ +# ===----------------------------------------------------------------------=== +# The attention thread: a hardware thread that executes the Ball instruction streams +# its core issues (the engine's stream interpreter). +# +# bbdev stack --build "--design " +# +# builds it for each design that uses it, configured with DESIGN_CMAKE +# (generated from design.pb) and THREAD, the +# compiler target of this thread instance. Products, in the build directory: +# lib-compiler.so buddy-opt plugin: Buckyball dialect + this thread's Balls +# program.elf, program.bin the stream interpreter the hardware thread runs +# loader.o loads program.bin and one instruction batch into a +# thread; links into the engine +# libkernels.a, libkernels-stream.a, kernels.rename +# the kernels of this thread, see Kernels below +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(attention-thread LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(ENGINE "${DESIGN_REPO}/stack/engine") +set(ISA "${DESIGN_ISA_DIR}/${THREAD}") +find_program(ELF_CC riscv64-unknown-elf-gcc REQUIRED) +find_program(ELF_OBJCOPY riscv64-unknown-elf-objcopy REQUIRED) +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# ===----------------------------------------------------------------------=== +# Compiler: the Buckyball dialect and passes with the Balls of this thread. +# ===----------------------------------------------------------------------=== +set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${THREAD}}) +set(BUCKYBALL_TARGET_TABLE "${ISA}/target.inc") +add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) +add_library(${THREAD}-compiler MODULE $) +set_target_properties(${THREAD}-compiler PROPERTIES + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") + +# ===----------------------------------------------------------------------=== +# Program: the engine's stream interpreter, freestanding at the start of the +# thread's code memory. +# ===----------------------------------------------------------------------=== +add_custom_command( + OUTPUT program.elf + COMMAND "${ELF_CC}" -O2 -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector -fno-jump-tables + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax "-Wl,-T,${ENGINE}/program/code.ld" + "-I${ENGINE}/include" "-I${ISA}" + "${ENGINE}/program/start.S" "${ENGINE}/program/stream.c" -o program.elf + DEPENDS "${ENGINE}/program/stream.c" "${ENGINE}/program/instructions.inc" + "${ENGINE}/program/start.S" "${ENGINE}/program/code.ld" "${ISA}/params.h" + VERBATIM) +add_custom_command( + OUTPUT program.bin + COMMAND "${ELF_OBJCOPY}" -O binary --only-section=.text program.elf program.bin + DEPENDS program.elf + VERBATIM) + +# ===----------------------------------------------------------------------=== +# Loader: stream_prepare.c embeds program.bin and registers as THREAD_SIGNATURE. +# ===----------------------------------------------------------------------=== +add_custom_command( + OUTPUT loader.o + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "--embed-dir=${CMAKE_CURRENT_BINARY_DIR}" "-I${ENGINE}/include" "-I${ISA}" + -c "${ENGINE}/program/stream_prepare.c" -o loader.o + DEPENDS "${ENGINE}/program/stream_prepare.c" program.bin "${ISA}/params.h" + VERBATIM) + +add_custom_target(program ALL DEPENDS program.bin loader.o) + +# ===----------------------------------------------------------------------=== +# Kernels: the RVV kernels of the GP domain and the kernels of this thread's +# Balls, with the host code that launches them, built against this thread's +# ISA. Every exported symbol gets the suffix __${THREAD}, so the kernels of +# several threads link into one program; models apply kernels.rename to the +# code they compile for this thread. +# libkernels.a Ball instructions issued by the calling core +# libkernels-stream.a Ball instructions sent to the thread through the engine +# ===----------------------------------------------------------------------=== +find_program(LINUX_CXX riscv64-unknown-linux-gnu-g++ REQUIRED) +find_program(LINUX_AR riscv64-unknown-linux-gnu-ar REQUIRED) +find_program(LINUX_OBJCOPY riscv64-unknown-linux-gnu-objcopy REQUIRED) +set(RVV "${DESIGN_REPO}/stack/kernels/rvv") +add_subdirectory("${RVV}" rvv) + +set(KERNEL_SOURCES + "${RVV}/dispatch.cpp" + "${RVV}/matmul_dispatch.cpp" + "${RVV}/cache_dispatch.cpp" + "${RVV}/normalization_dispatch.cpp" + "${RVV}/pointwise_dispatch.cpp" + "${RVV}/activation_dispatch.cpp" + "${RVV}/flash_attention_dispatch.cpp" + "${RVV}/flash_attention_mxfp8_dispatch.cpp") +set(KERNEL_INCLUDES + "${ISA}" + "${DESIGN_ISA_DIR}/design" + "${DESIGN_REPO}/bb-tests/workloads/lib" + "${DESIGN_REPO}/stack/kernels" + "${DESIGN_REPO}/stack/runtime/scheduler" + "${DESIGN_REPO}/stack/runtime/services" + "${ENGINE}/include" + "${DESIGN_REPO}/stack/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") +set(KERNEL_EMBED_DIRS "${CMAKE_CURRENT_BINARY_DIR}/rvv") +set(KERNEL_DEPENDS rvv-images) +set(KERNEL_EXPORTS + _mlir_ciface_rvv_attention_softmax _mlir_ciface_rvv_binary _mlir_ciface_rvv_cos + _mlir_ciface_rvv_flash_attention _mlir_ciface_rvv_flash_attention_mxfp8 _mlir_ciface_rvv_gelu + _mlir_ciface_rvv_layernorm _mlir_ciface_rvv_layernorm_no_bias _mlir_ciface_rvv_logsumexp_softmax + _mlir_ciface_rvv_matmul _mlir_ciface_rvv_matmul_broadcast_rhs + _mlir_ciface_rvv_matmul_broadcast_transpose_rhs _mlir_ciface_rvv_matmul_transpose_rhs + _mlir_ciface_rvv_mxfp8_decode _mlir_ciface_rvv_mxfp8_encode _mlir_ciface_rvv_norm + _mlir_ciface_rvv_norm_no_weight _mlir_ciface_rvv_rope _mlir_ciface_rvv_silu _mlir_ciface_rvv_sin + _mlir_ciface_rvv_snake _mlir_ciface_rvv_softmax _mlir_ciface_rvv_swiglu _mlir_ciface_rvv_tanh + _mlir_ciface_rvv_unary) +# The Balls of this thread add their kernels to the lists above. +foreach(DIRECTORY IN LISTS DESIGN_TARGET_KERNELS_${THREAD}) + get_filename_component(BALL "${DIRECTORY}/../.." ABSOLUTE) + get_filename_component(BALL "${BALL}" NAME) + add_subdirectory("${DIRECTORY}" balls/${BALL}) +endforeach() + +set(RENAME "") +foreach(SYMBOL IN LISTS KERNEL_EXPORTS) + string(APPEND RENAME "${SYMBOL} ${SYMBOL}__${THREAD}\n") +endforeach() +file(CONFIGURE OUTPUT kernels.rename CONTENT "${RENAME}") +list(TRANSFORM KERNEL_INCLUDES PREPEND "-I") +list(TRANSFORM KERNEL_EMBED_DIRS PREPEND "--embed-dir=") + +foreach(LIBRARY kernels kernels-stream) + set(DEFINES) + if(LIBRARY STREQUAL "kernels-stream") + set(DEFINES -DBUCKYBALL_STREAM) + endif() + set(OBJECTS) + foreach(SOURCE IN LISTS KERNEL_SOURCES) + file(RELATIVE_PATH OBJECT "${DESIGN_REPO}" "${SOURCE}.o") + set(OBJECT "${LIBRARY}/${OBJECT}") + add_custom_command( + OUTPUT ${OBJECT} + COMMAND "${LINUX_CXX}" -std=c++20 -O2 -fPIC -ffp-contract=off ${DEFINES} + ${KERNEL_INCLUDES} ${KERNEL_EMBED_DIRS} -MD -MF ${OBJECT}.d -c "${SOURCE}" -o ${OBJECT} + DEPENDS "${SOURCE}" ${KERNEL_DEPENDS} + DEPFILE ${OBJECT}.d + VERBATIM) + list(APPEND OBJECTS "${CMAKE_CURRENT_BINARY_DIR}/${OBJECT}") + endforeach() + add_custom_command( + OUTPUT lib${LIBRARY}.a + COMMAND "${CMAKE_COMMAND}" -E rm -f lib${LIBRARY}.a + COMMAND "${LINUX_AR}" rcs lib${LIBRARY}.a ${OBJECTS} + COMMAND "${LINUX_OBJCOPY}" --redefine-syms=kernels.rename lib${LIBRARY}.a + DEPENDS ${OBJECTS} "${CMAKE_CURRENT_BINARY_DIR}/kernels.rename" + VERBATIM) +endforeach() +add_custom_target(kernels ALL DEPENDS libkernels.a libkernels-stream.a) diff --git a/examples/components/threads/build.sc b/examples/components/threads/build.sc new file mode 100644 index 000000000..932a84d4b --- /dev/null +++ b/examples/components/threads/build.sc @@ -0,0 +1,10 @@ +import mill._ + +def sources = T.sources { + os.list(millSourcePath).filter(os.isDir).flatMap { root => + val source = root / "arch" + val configs = root / "configs" + (if (os.exists(source)) os.walk(source).filter(p => p.ext == "scala" && !p.toString.contains("/out/") && !p.toString.contains("/build/")) else Seq.empty) ++ + (if (os.exists(configs)) os.walk(configs).filter(_.ext == "scala") else Seq.empty) + }.map(PathRef(_)) +} diff --git a/examples/components/threads/ffn/arch/FfnThread.scala b/examples/components/threads/ffn/arch/FfnThread.scala new file mode 100644 index 000000000..49cff8a1c --- /dev/null +++ b/examples/components/threads/ffn/arch/FfnThread.scala @@ -0,0 +1,77 @@ +package examples.threads.ffn + +import framework.thread.ant.Ant +import framework.core.tbus.ThreadFactory +import framework.core.configs.CoreParam +import sims.configs.SimParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.thread.configs.ThreadParam +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} + +import framework.acc.debug.Debug + +import chisel3.experimental.hierarchy.Instance + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} +import framework.thread.tlink.{HasTLink, TLinkIO} + +@instantiable +class FfnThread( + threadp: ThreadParam, + permissionp: PmpParams, + trackingp: TrackingParam, + walkerp: WalkerParam, + simp: SimParam, + traceId: Int, + sharedOwnerId: Int) + extends Module + with HasTLink { + val sharedPorts = threadp.acc.get.memDomain.sharedPorts + val diffTest = simp.diffTest + val accp = threadp.acc.get + val antp = threadp.ant.get + + val ant: Instance[Ant] = + Instantiate(new Ant(antp, permissionp)) + + val acc: Instance[Acc] = Instantiate(new Acc(accp, permissionp, trackingp, walkerp, simp)) + + val debug: Instance[Debug] = + Instantiate(new Debug(accp.frontend, accp.memDomain, accp.gpDomain, accp.ballDomain, simp, traceId, sharedOwnerId)) + + debug.io.ant := ant.io.trace + debug.io.acc <> acc.io.debug + + @public + val tlink = IO(new TLinkIO(threadp, permissionp, trackingp, walkerp, simp)) + + ant.io.task <> tlink.task + tlink.tss <> ant.io.shared + ant.io.rocc <> acc.io.rocc + tlink.memory <> acc.io.memory + tlink.ddr <> acc.io.ddr + tlink.shm <> acc.io.shm + acc.io.shared_bank_hashes.zip(tlink.sharedHashes).foreach { case (port, hashes) => port := hashes } +} + +object FfnThread extends ThreadFactory { + + def instantiate( + corep: CoreParam, + threadp: ThreadParam, + simp: SimParam, + traceId: Int + ): TLinkIO = + Instantiate(new FfnThread( + threadp, + corep.privileged.permission, + corep.memory.dcache.tracking, + corep.memory.memoryAccess.walker, + simp, + traceId, + corep.hartId + )).tlink + +} diff --git a/examples/components/threads/ffn/arch/acc.scala b/examples/components/threads/ffn/arch/acc.scala new file mode 100644 index 000000000..dbf0a7a4f --- /dev/null +++ b/examples/components/threads/ffn/arch/acc.scala @@ -0,0 +1,119 @@ +package examples.threads.ffn + +import framework.acc.balldomain.bbus.generated.BallComponents +import sims.configs.SimParam +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.configs.AccParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.acc.debug.AcceleratorDebugPort +import framework.acc.frontend.RoCCIO +import framework.acc.frontend.Frontend +import framework.acc.memdomain.{MemDomain, PhysicalBankHash} +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.bbus.BBus +import framework.core.memory.MemoryPort +import framework.thread.tlink.SharedLink +import framework.ip.bus.axi4 + +@instantiable +class Acc( + val accp: AccParam, + val permissionp: PmpParams, + val trackingp: TrackingParam, + val walkerp: WalkerParam, + val simp: SimParam) + extends Module { + val commandDataBits = accp.frontend.commandDataBits + val ownerId = accp.frontend.ownerId + val sharedPorts = accp.memDomain.sharedPorts + val diffTest = simp.diffTest + val frontp = accp.frontend + val memp = accp.memDomain + val gpp = accp.gpDomain + val ballp = accp.ballDomain + val transportp = accp.transport + + val sharedHashCount = if (memp.sharedEnable) memp.sharedBankNum else 0 + + @public + val io = IO(new Bundle { + val debug = new AcceleratorDebugPort(frontp, memp, gpp, ballp, simp) + + val rocc = new RoCCIO(commandDataBits, permissionp) + val ddr = new axi4.Port(transportp.axi) + val memory = new MemoryPort(trackingp, walkerp) + val shm = new SharedLink(frontp, memp) + + val shared_bank_hashes = Option.when(diffTest && memp.sharedEnable)( + Input(Vec(sharedHashCount, new PhysicalBankHash(frontp))) + ) + + }) + + // --- Instantiate domains --- + val frontend: Instance[Frontend] = Instantiate(new Frontend(frontp, memp, gpp, ballp, permissionp, simp)) + + val ballDomain: Instance[BallDomain] = + Instantiate(new BallDomain(frontp, memp, gpp, ballp, permissionp, BallComponents.factories)) + + val memDomain: Instance[MemDomain] = + Instantiate(new MemDomain(frontp, memp, gpp, transportp, permissionp, trackingp, walkerp, simp)) + + val gpDomain = + if (gpp.enable) Some(Instantiate(new GpDomain(frontp, memp, gpp))) + else None + + memDomain.io.shared_bank_hashes.foreach(_ := io.shared_bank_hashes.get) + + io.debug.frontend <> frontend.io.debug + io.debug.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, source) => sink := source } + io.debug.trace.memory := memDomain.io.traceMemory + io.debug.trace.ballStarted := ballDomain.io.traceStarted + io.debug.trace.ballCompleted := ballDomain.io.traceCompleted + + io.rocc <> frontend.io.rocc + io.memory <> memDomain.io.memory + io.ddr <> memDomain.io.ddr + + // --- Frontend -> BallDomain --- + memDomain.io.ball <> ballDomain.io.memory + ballDomain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> ballDomain.io.complete + + // --- Frontend -> MemDomain --- + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + + if (gpp.enable) { + val controller = gpDomain.get + controller.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> controller.io.complete + controller.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> controller.io.banks + memDomain.io.gpLayoutRequest.get <> controller.io.layoutRequest + controller.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + ballDomain.io.kernel.get <> controller.io.ball + } + + memDomain.io.banks <> ballDomain.io.banks + memDomain.io.ballLayoutRequest <> ballDomain.io.layoutRequest + ballDomain.io.layoutResponse <> memDomain.io.ballLayoutResponse + + // --- Shared memory passthrough --- + io.shm.bank.requests <> memDomain.io.shared.bank.requests + io.shm.move <> memDomain.io.shared.move + io.shm.bank.local <> memDomain.io.shared.bank.local + io.shm.bank.config <> memDomain.io.shared.bank.config + io.shm.bank.mapping <> memDomain.io.shared.bank.mapping + + // --- Barrier passthrough --- + io.shm.barrierArrive := frontend.io.barrier.arrive + frontend.io.barrier.release := io.shm.barrierRelease + +} diff --git a/examples/components/threads/ffn/configs/balldomains/default.toml b/examples/components/threads/ffn/configs/balldomains/default.toml new file mode 100644 index 000000000..cc8647054 --- /dev/null +++ b/examples/components/threads/ffn/configs/balldomains/default.toml @@ -0,0 +1,53 @@ +[[balls]] +id = 0 +name = "MxmmBall" +component = "mxmm" +param = "../../../../balls/mxmm/configs/default.toml" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "MXMM_MXFP8", funct7 = 71 }, + { mnemonic = "MXMM_MXFP8_WINDOW", funct7 = 75 }, + { mnemonic = "MXMM_FMA32", funct7 = 72 }, + { mnemonic = "MXMM_F32", funct7 = 73 }, +] + +[[balls]] +id = 1 +name = "ToInt8Ball" +component = "toint8" +param = "../../../../balls/toint8/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "QUANT_F32_TO_I8", funct7 = 51 }, + { mnemonic = "QUANT_I32_TO_I8", funct7 = 67 }, +] + +[[balls]] +id = 2 +name = "Int2FpBall" +component = "int2fp" +param = "../../../../balls/int2fp/configs/default.toml" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "INT32_TO_FP32", funct7 = 68 }, +] + +[[balls]] +id = 3 +name = "TransposeBall" +component = "transpose" +param = "../../../../balls/transpose/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "TRANSPOSE", funct7 = 49 }, +] + +[[balls]] +id = 4 +name = "MxquantBall" +component = "mxquant" +param = "../../../../balls/mxquant/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "MXQUANT", funct7 = 54 }, +] diff --git a/examples/components/threads/ffn/configs/cpu/default.toml b/examples/components/threads/ffn/configs/cpu/default.toml new file mode 100644 index 000000000..c752c9500 --- /dev/null +++ b/examples/components/threads/ffn/configs/cpu/default.toml @@ -0,0 +1,2 @@ +kind = "ant" +config = "../../../../../../arch/src/main/scala/framework/thread/ant/configs/default.toml" diff --git a/examples/components/threads/ffn/configs/default.toml b/examples/components/threads/ffn/configs/default.toml new file mode 100644 index 000000000..79b6fb436 --- /dev/null +++ b/examples/components/threads/ffn/configs/default.toml @@ -0,0 +1,12 @@ +source = ".." +# FFN Thread Configuration (default) + +balldomain = "balldomains/default.toml" +memdomain = "memdomains/default.toml" +frontend = "frontends/default.toml" +gpdomain = "gpdomain/default.toml" +cpu = "cpu/default.toml" + +[compiler] +name = "ffn" +component = "ffn" diff --git a/examples/components/threads/ffn/configs/frontends/default.toml b/examples/components/threads/ffn/configs/frontends/default.toml new file mode 100644 index 000000000..5c58b1cb2 --- /dev/null +++ b/examples/components/threads/ffn/configs/frontends/default.toml @@ -0,0 +1,9 @@ +bankAliasEnable = true +robEntries = 16 +rsOutOfOrderResponse = true +bankIdLen = 10 +vbankIdUpperBound = 31 +sharedBankIdBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/examples/cores/ffn/configs/rvv/default.toml b/examples/components/threads/ffn/configs/gpdomain/default.toml similarity index 100% rename from examples/cores/ffn/configs/rvv/default.toml rename to examples/components/threads/ffn/configs/gpdomain/default.toml diff --git a/examples/components/threads/ffn/configs/memdomains/default.toml b/examples/components/threads/ffn/configs/memdomains/default.toml new file mode 100644 index 000000000..ff0bddb0e --- /dev/null +++ b/examples/components/threads/ffn/configs/memdomains/default.toml @@ -0,0 +1,23 @@ +# FFN Thread Memory Domain Configuration (default) +# Private (per-thread) memory parameters. Shared memory lives in the core TOML. + +[bank] +layoutEnable = true +num = 32 +width = 128 +entries = 512 +maskLen = 16 +requestPorts = 7 + +[dma] +busWidth = 128 + +[mmio] +enable = false +bankNum = 16 +bankEntries = 64 +bankWidth = 128 +readWidth = 8 + +[mem] +addrLen = 39 diff --git a/examples/components/threads/ffn/stack/CMakeLists.txt b/examples/components/threads/ffn/stack/CMakeLists.txt new file mode 100644 index 000000000..042abb44d --- /dev/null +++ b/examples/components/threads/ffn/stack/CMakeLists.txt @@ -0,0 +1,157 @@ +# ===----------------------------------------------------------------------=== +# The ffn thread: a hardware thread that executes the Ball instruction streams +# its core issues (the engine's stream interpreter). +# +# bbdev stack --build "--design " +# +# builds it for each design that uses it, configured with DESIGN_CMAKE +# (generated from design.pb) and THREAD, the +# compiler target of this thread instance. Products, in the build directory: +# lib-compiler.so buddy-opt plugin: Buckyball dialect + this thread's Balls +# program.elf, program.bin the stream interpreter the hardware thread runs +# loader.o loads program.bin and one instruction batch into a +# thread; links into the engine +# libkernels.a, libkernels-stream.a, kernels.rename +# the kernels of this thread, see Kernels below +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(ffn-thread LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(ENGINE "${DESIGN_REPO}/stack/engine") +set(ISA "${DESIGN_ISA_DIR}/${THREAD}") +find_program(ELF_CC riscv64-unknown-elf-gcc REQUIRED) +find_program(ELF_OBJCOPY riscv64-unknown-elf-objcopy REQUIRED) +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# ===----------------------------------------------------------------------=== +# Compiler: the Buckyball dialect and passes with the Balls of this thread. +# ===----------------------------------------------------------------------=== +set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${THREAD}}) +set(BUCKYBALL_TARGET_TABLE "${ISA}/target.inc") +add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) +add_library(${THREAD}-compiler MODULE $) +set_target_properties(${THREAD}-compiler PROPERTIES + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") + +# ===----------------------------------------------------------------------=== +# Program: the engine's stream interpreter, freestanding at the start of the +# thread's code memory. +# ===----------------------------------------------------------------------=== +add_custom_command( + OUTPUT program.elf + COMMAND "${ELF_CC}" -O2 -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector -fno-jump-tables + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax "-Wl,-T,${ENGINE}/program/code.ld" + "-I${ENGINE}/include" "-I${ISA}" + "${ENGINE}/program/start.S" "${ENGINE}/program/stream.c" -o program.elf + DEPENDS "${ENGINE}/program/stream.c" "${ENGINE}/program/instructions.inc" + "${ENGINE}/program/start.S" "${ENGINE}/program/code.ld" "${ISA}/params.h" + VERBATIM) +add_custom_command( + OUTPUT program.bin + COMMAND "${ELF_OBJCOPY}" -O binary --only-section=.text program.elf program.bin + DEPENDS program.elf + VERBATIM) + +# ===----------------------------------------------------------------------=== +# Loader: stream_prepare.c embeds program.bin and registers as THREAD_SIGNATURE. +# ===----------------------------------------------------------------------=== +add_custom_command( + OUTPUT loader.o + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "--embed-dir=${CMAKE_CURRENT_BINARY_DIR}" "-I${ENGINE}/include" "-I${ISA}" + -c "${ENGINE}/program/stream_prepare.c" -o loader.o + DEPENDS "${ENGINE}/program/stream_prepare.c" program.bin "${ISA}/params.h" + VERBATIM) + +add_custom_target(program ALL DEPENDS program.bin loader.o) + +# ===----------------------------------------------------------------------=== +# Kernels: the RVV kernels of the GP domain and the kernels of this thread's +# Balls, with the host code that launches them, built against this thread's +# ISA. Every exported symbol gets the suffix __${THREAD}, so the kernels of +# several threads link into one program; models apply kernels.rename to the +# code they compile for this thread. +# libkernels.a Ball instructions issued by the calling core +# libkernels-stream.a Ball instructions sent to the thread through the engine +# ===----------------------------------------------------------------------=== +find_program(LINUX_CXX riscv64-unknown-linux-gnu-g++ REQUIRED) +find_program(LINUX_AR riscv64-unknown-linux-gnu-ar REQUIRED) +find_program(LINUX_OBJCOPY riscv64-unknown-linux-gnu-objcopy REQUIRED) +set(RVV "${DESIGN_REPO}/stack/kernels/rvv") +add_subdirectory("${RVV}" rvv) + +set(KERNEL_SOURCES + "${RVV}/dispatch.cpp" + "${RVV}/matmul_dispatch.cpp" + "${RVV}/cache_dispatch.cpp" + "${RVV}/normalization_dispatch.cpp" + "${RVV}/pointwise_dispatch.cpp" + "${RVV}/activation_dispatch.cpp" + "${RVV}/flash_attention_dispatch.cpp" + "${RVV}/flash_attention_mxfp8_dispatch.cpp") +set(KERNEL_INCLUDES + "${ISA}" + "${DESIGN_ISA_DIR}/design" + "${DESIGN_REPO}/bb-tests/workloads/lib" + "${DESIGN_REPO}/stack/kernels" + "${DESIGN_REPO}/stack/runtime/scheduler" + "${DESIGN_REPO}/stack/runtime/services" + "${ENGINE}/include" + "${DESIGN_REPO}/stack/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") +set(KERNEL_EMBED_DIRS "${CMAKE_CURRENT_BINARY_DIR}/rvv") +set(KERNEL_DEPENDS rvv-images) +set(KERNEL_EXPORTS + _mlir_ciface_rvv_attention_softmax _mlir_ciface_rvv_binary _mlir_ciface_rvv_cos + _mlir_ciface_rvv_flash_attention _mlir_ciface_rvv_flash_attention_mxfp8 _mlir_ciface_rvv_gelu + _mlir_ciface_rvv_layernorm _mlir_ciface_rvv_layernorm_no_bias _mlir_ciface_rvv_logsumexp_softmax + _mlir_ciface_rvv_matmul _mlir_ciface_rvv_matmul_broadcast_rhs + _mlir_ciface_rvv_matmul_broadcast_transpose_rhs _mlir_ciface_rvv_matmul_transpose_rhs + _mlir_ciface_rvv_mxfp8_decode _mlir_ciface_rvv_mxfp8_encode _mlir_ciface_rvv_norm + _mlir_ciface_rvv_norm_no_weight _mlir_ciface_rvv_rope _mlir_ciface_rvv_silu _mlir_ciface_rvv_sin + _mlir_ciface_rvv_snake _mlir_ciface_rvv_softmax _mlir_ciface_rvv_swiglu _mlir_ciface_rvv_tanh + _mlir_ciface_rvv_unary) +# The Balls of this thread add their kernels to the lists above. +foreach(DIRECTORY IN LISTS DESIGN_TARGET_KERNELS_${THREAD}) + get_filename_component(BALL "${DIRECTORY}/../.." ABSOLUTE) + get_filename_component(BALL "${BALL}" NAME) + add_subdirectory("${DIRECTORY}" balls/${BALL}) +endforeach() + +set(RENAME "") +foreach(SYMBOL IN LISTS KERNEL_EXPORTS) + string(APPEND RENAME "${SYMBOL} ${SYMBOL}__${THREAD}\n") +endforeach() +file(CONFIGURE OUTPUT kernels.rename CONTENT "${RENAME}") +list(TRANSFORM KERNEL_INCLUDES PREPEND "-I") +list(TRANSFORM KERNEL_EMBED_DIRS PREPEND "--embed-dir=") + +foreach(LIBRARY kernels kernels-stream) + set(DEFINES) + if(LIBRARY STREQUAL "kernels-stream") + set(DEFINES -DBUCKYBALL_STREAM) + endif() + set(OBJECTS) + foreach(SOURCE IN LISTS KERNEL_SOURCES) + file(RELATIVE_PATH OBJECT "${DESIGN_REPO}" "${SOURCE}.o") + set(OBJECT "${LIBRARY}/${OBJECT}") + add_custom_command( + OUTPUT ${OBJECT} + COMMAND "${LINUX_CXX}" -std=c++20 -O2 -fPIC -ffp-contract=off ${DEFINES} + ${KERNEL_INCLUDES} ${KERNEL_EMBED_DIRS} -MD -MF ${OBJECT}.d -c "${SOURCE}" -o ${OBJECT} + DEPENDS "${SOURCE}" ${KERNEL_DEPENDS} + DEPFILE ${OBJECT}.d + VERBATIM) + list(APPEND OBJECTS "${CMAKE_CURRENT_BINARY_DIR}/${OBJECT}") + endforeach() + add_custom_command( + OUTPUT lib${LIBRARY}.a + COMMAND "${CMAKE_COMMAND}" -E rm -f lib${LIBRARY}.a + COMMAND "${LINUX_AR}" rcs lib${LIBRARY}.a ${OBJECTS} + COMMAND "${LINUX_OBJCOPY}" --redefine-syms=kernels.rename lib${LIBRARY}.a + DEPENDS ${OBJECTS} "${CMAKE_CURRENT_BINARY_DIR}/kernels.rename" + VERBATIM) +endforeach() +add_custom_target(kernels ALL DEPENDS libkernels.a libkernels-stream.a) diff --git a/examples/components/threads/matmul/arch/MatMulThread.scala b/examples/components/threads/matmul/arch/MatMulThread.scala new file mode 100644 index 000000000..63dac307e --- /dev/null +++ b/examples/components/threads/matmul/arch/MatMulThread.scala @@ -0,0 +1,75 @@ +package examples.threads.matmul + +import framework.thread.ant.Ant +import framework.core.tbus.ThreadFactory +import framework.core.configs.CoreParam +import sims.configs.SimParam +import framework.thread.configs.ThreadParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} + +import framework.acc.debug.Debug + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.thread.tlink.{HasTLink, TLinkIO} + +@instantiable +class MatMulThread( + threadp: ThreadParam, + permissionp: PmpParams, + trackingp: TrackingParam, + walkerp: WalkerParam, + simp: SimParam, + traceId: Int, + sharedOwnerId: Int) + extends Module + with HasTLink { + val sharedPorts = threadp.acc.get.memDomain.sharedPorts + val diffTest = simp.diffTest + val accp = threadp.acc.get + val antp = threadp.ant.get + + val ant: Instance[Ant] = + Instantiate(new Ant(antp, permissionp)) + + val acc: Instance[Acc] = Instantiate(new Acc(accp, permissionp, trackingp, walkerp, simp)) + + val debug: Instance[Debug] = + Instantiate(new Debug(accp.frontend, accp.memDomain, accp.gpDomain, accp.ballDomain, simp, traceId, sharedOwnerId)) + + debug.io.ant := ant.io.trace + debug.io.acc <> acc.io.debug + + @public + val tlink = IO(new TLinkIO(threadp, permissionp, trackingp, walkerp, simp)) + + ant.io.task <> tlink.task + tlink.tss <> ant.io.shared + ant.io.rocc <> acc.io.rocc + tlink.memory <> acc.io.memory + tlink.ddr <> acc.io.ddr + tlink.shm <> acc.io.shm + acc.io.shared_bank_hashes.zip(tlink.sharedHashes).foreach { case (port, hashes) => port := hashes } +} + +object MatMulThread extends ThreadFactory { + + def instantiate( + corep: CoreParam, + threadp: ThreadParam, + simp: SimParam, + traceId: Int + ): TLinkIO = + Instantiate(new MatMulThread( + threadp, + corep.privileged.permission, + corep.memory.dcache.tracking, + corep.memory.memoryAccess.walker, + simp, + traceId, + corep.hartId + )).tlink + +} diff --git a/examples/components/threads/matmul/arch/acc.scala b/examples/components/threads/matmul/arch/acc.scala new file mode 100644 index 000000000..26c96c938 --- /dev/null +++ b/examples/components/threads/matmul/arch/acc.scala @@ -0,0 +1,119 @@ +package examples.threads.matmul + +import framework.acc.balldomain.bbus.generated.BallComponents +import sims.configs.SimParam +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.acc.configs.AccParam +import framework.core.memory.access.preflight.configs.PmpParams +import framework.ip.memory.interlock.{Params => TrackingParam} +import framework.core.memory.access.mmu.configs.{Params => WalkerParam} +import framework.acc.debug.AcceleratorDebugPort +import framework.acc.frontend.RoCCIO +import framework.acc.frontend.Frontend +import framework.acc.memdomain.{MemDomain, PhysicalBankHash} +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.bbus.BBus +import framework.core.memory.MemoryPort +import framework.thread.tlink.SharedLink +import framework.ip.bus.axi4 + +@instantiable +class Acc( + val accp: AccParam, + val permissionp: PmpParams, + val trackingp: TrackingParam, + val walkerp: WalkerParam, + val simp: SimParam) + extends Module { + val commandDataBits = accp.frontend.commandDataBits + val ownerId = accp.frontend.ownerId + val sharedPorts = accp.memDomain.sharedPorts + val diffTest = simp.diffTest + val frontp = accp.frontend + val memp = accp.memDomain + val gpp = accp.gpDomain + val ballp = accp.ballDomain + val transportp = accp.transport + + val sharedHashCount = if (memp.sharedEnable) memp.sharedBankNum else 0 + + @public + val io = IO(new Bundle { + val debug = new AcceleratorDebugPort(frontp, memp, gpp, ballp, simp) + + val rocc = new RoCCIO(commandDataBits, permissionp) + val ddr = new axi4.Port(transportp.axi) + val memory = new MemoryPort(trackingp, walkerp) + val shm = new SharedLink(frontp, memp) + + val shared_bank_hashes = Option.when(diffTest && memp.sharedEnable)( + Input(Vec(sharedHashCount, new PhysicalBankHash(frontp))) + ) + + }) + + // --- Instantiate domains --- + val frontend: Instance[Frontend] = Instantiate(new Frontend(frontp, memp, gpp, ballp, permissionp, simp)) + + val ballDomain: Instance[BallDomain] = + Instantiate(new BallDomain(frontp, memp, gpp, ballp, permissionp, BallComponents.factories)) + + val memDomain: Instance[MemDomain] = + Instantiate(new MemDomain(frontp, memp, gpp, transportp, permissionp, trackingp, walkerp, simp)) + + val gpDomain = + if (gpp.enable) Some(Instantiate(new GpDomain(frontp, memp, gpp))) + else None + + memDomain.io.shared_bank_hashes.foreach(_ := io.shared_bank_hashes.get) + + io.debug.frontend <> frontend.io.debug + io.debug.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, source) => sink := source } + io.debug.trace.memory := memDomain.io.traceMemory + io.debug.trace.ballStarted := ballDomain.io.traceStarted + io.debug.trace.ballCompleted := ballDomain.io.traceCompleted + + io.rocc <> frontend.io.rocc + io.memory <> memDomain.io.memory + io.ddr <> memDomain.io.ddr + + // --- Frontend -> BallDomain --- + memDomain.io.ball <> ballDomain.io.memory + ballDomain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> ballDomain.io.complete + + // --- Frontend -> MemDomain --- + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + + if (gpp.enable) { + val controller = gpDomain.get + controller.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> controller.io.complete + controller.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> controller.io.banks + memDomain.io.gpLayoutRequest.get <> controller.io.layoutRequest + controller.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + ballDomain.io.kernel.get <> controller.io.ball + } + + memDomain.io.banks <> ballDomain.io.banks + memDomain.io.ballLayoutRequest <> ballDomain.io.layoutRequest + ballDomain.io.layoutResponse <> memDomain.io.ballLayoutResponse + + // --- Shared memory passthrough --- + io.shm.bank.requests <> memDomain.io.shared.bank.requests + io.shm.move <> memDomain.io.shared.move + io.shm.bank.local <> memDomain.io.shared.bank.local + io.shm.bank.config <> memDomain.io.shared.bank.config + io.shm.bank.mapping <> memDomain.io.shared.bank.mapping + + // --- Barrier passthrough --- + io.shm.barrierArrive := frontend.io.barrier.arrive + frontend.io.barrier.release := io.shm.barrierRelease + +} diff --git a/examples/components/threads/matmul/configs/ant.toml b/examples/components/threads/matmul/configs/ant.toml new file mode 100644 index 000000000..6e8d3ad08 --- /dev/null +++ b/examples/components/threads/matmul/configs/ant.toml @@ -0,0 +1,5 @@ +source = ".." +balldomain = "balldomains/default.toml" +memdomain = "memdomains/default.toml" +frontend = "frontends/default.toml" +gpdomain = "gpdomain/default.toml" diff --git a/examples/components/threads/matmul/configs/balldomains/default.toml b/examples/components/threads/matmul/configs/balldomains/default.toml new file mode 100644 index 000000000..0bc2e08ad --- /dev/null +++ b/examples/components/threads/matmul/configs/balldomains/default.toml @@ -0,0 +1,41 @@ +[[balls]] +id = 0 +name = "SMatMulBall" +component = "smatmul" +param = "../../../../balls/smatmul/configs/default.toml" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "SMATMUL_OS", funct7 = 65 }, + { mnemonic = "SMATMUL_BIAS", funct7 = 17 }, +] + +[[balls]] +id = 1 +name = "ToInt8Ball" +component = "toint8" +param = "../../../../balls/toint8/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "QUANT_F32_TO_I8", funct7 = 51 }, + { mnemonic = "QUANT_I32_TO_I8", funct7 = 67 }, +] + +[[balls]] +id = 2 +name = "Int2FpBall" +component = "int2fp" +param = "../../../../balls/int2fp/configs/default.toml" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "INT32_TO_FP32", funct7 = 68 }, +] + +[[balls]] +id = 3 +name = "TransposeBall" +component = "transpose" +param = "../../../../balls/transpose/configs/default.toml" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "TRANSPOSE", funct7 = 49 }, +] diff --git a/examples/components/threads/matmul/configs/cpu/default.toml b/examples/components/threads/matmul/configs/cpu/default.toml new file mode 100644 index 000000000..c752c9500 --- /dev/null +++ b/examples/components/threads/matmul/configs/cpu/default.toml @@ -0,0 +1,2 @@ +kind = "ant" +config = "../../../../../../arch/src/main/scala/framework/thread/ant/configs/default.toml" diff --git a/examples/components/threads/matmul/configs/default.toml b/examples/components/threads/matmul/configs/default.toml new file mode 100644 index 000000000..1bd004769 --- /dev/null +++ b/examples/components/threads/matmul/configs/default.toml @@ -0,0 +1,13 @@ +source = ".." +# MatMul Thread Configuration (default) + +balldomain = "balldomains/default.toml" +memdomain = "memdomains/default.toml" +frontend = "frontends/default.toml" +gpdomain = "gpdomain/default.toml" +cpu = "cpu/default.toml" +transport = "default" + +[compiler] +name = "matmul" +component = "matmul" diff --git a/examples/components/threads/matmul/configs/frontends/default.toml b/examples/components/threads/matmul/configs/frontends/default.toml new file mode 100644 index 000000000..c3e3d5e71 --- /dev/null +++ b/examples/components/threads/matmul/configs/frontends/default.toml @@ -0,0 +1,9 @@ +bankAliasEnable = true +robEntries = 16 +rsOutOfOrderResponse = true +bankIdLen = 10 +vbankIdUpperBound = 15 +sharedBankIdBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/examples/cores/goban/configs/rvv/default.toml b/examples/components/threads/matmul/configs/gpdomain/default.toml similarity index 100% rename from examples/cores/goban/configs/rvv/default.toml rename to examples/components/threads/matmul/configs/gpdomain/default.toml diff --git a/examples/components/threads/matmul/configs/memdomains/default.toml b/examples/components/threads/matmul/configs/memdomains/default.toml new file mode 100644 index 000000000..94c9b9ef3 --- /dev/null +++ b/examples/components/threads/matmul/configs/memdomains/default.toml @@ -0,0 +1,23 @@ +# MatMul Thread Memory Domain Configuration (default) +# Private (per-thread) memory parameters. Shared memory lives in the core TOML. + +[bank] +layoutEnable = true +num = 20 +width = 128 +entries = 1024 +maskLen = 16 +requestPorts = 7 + +[dma] +busWidth = 128 + +[mmio] +enable = false +bankNum = 16 +bankEntries = 64 +bankWidth = 128 +readWidth = 8 + +[mem] +addrLen = 39 diff --git a/examples/components/threads/matmul/stack/CMakeLists.txt b/examples/components/threads/matmul/stack/CMakeLists.txt new file mode 100644 index 000000000..dbd540556 --- /dev/null +++ b/examples/components/threads/matmul/stack/CMakeLists.txt @@ -0,0 +1,157 @@ +# ===----------------------------------------------------------------------=== +# The matmul thread: a hardware thread that runs one fixed SMatMul program. +# +# bbdev stack --build "--design " +# +# builds it for each design that uses it, configured with DESIGN_CMAKE +# (generated from design.pb) and THREAD, the +# compiler target of this thread instance. Products, in the build directory: +# lib-compiler.so buddy-opt plugin: Buckyball dialect + this thread's Balls +# program.elf, program.bin the code the hardware thread runs +# loader.o loads program.bin into a thread; links into the engine +# libkernels.a, libkernels-stream.a, kernels.rename +# the kernels of this thread, see Kernels below +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(matmul-thread LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(ENGINE "${DESIGN_REPO}/stack/engine") +set(ISA "${DESIGN_ISA_DIR}/${THREAD}") +find_program(ELF_CC riscv64-unknown-elf-gcc REQUIRED) +find_program(ELF_OBJCOPY riscv64-unknown-elf-objcopy REQUIRED) +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# ===----------------------------------------------------------------------=== +# Compiler: the Buckyball dialect and passes with the Balls of this thread. +# ===----------------------------------------------------------------------=== +set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${THREAD}}) +set(BUCKYBALL_TARGET_TABLE "${ISA}/target.inc") +add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) +add_library(${THREAD}-compiler MODULE $) +set_target_properties(${THREAD}-compiler PROPERTIES + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") + +# ===----------------------------------------------------------------------=== +# Program: program.c, freestanding at the start of the thread's code memory. +# ===----------------------------------------------------------------------=== +add_custom_command( + OUTPUT program.elf + COMMAND "${ELF_CC}" -O2 -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector -fno-jump-tables + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax "-Wl,-T,${ENGINE}/program/code.ld" + "-I${ENGINE}/include" "-I${ISA}" "-I${DESIGN_REPO}/bb-tests/workloads/lib" + "-I${DESIGN_REPO}/examples/components/balls/smatmul/workloads" + "${ENGINE}/program/start.S" "${CMAKE_CURRENT_SOURCE_DIR}/program.c" + -o program.elf + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/program.c" "${CMAKE_CURRENT_SOURCE_DIR}/matmul.h" + "${ENGINE}/program/start.S" "${ENGINE}/program/code.ld" "${ISA}/params.h" + VERBATIM) +add_custom_command( + OUTPUT program.bin + COMMAND "${ELF_OBJCOPY}" -O binary --only-section=.text program.elf program.bin + DEPENDS program.elf + VERBATIM) + +# ===----------------------------------------------------------------------=== +# Loader: prepare.c embeds program.bin and registers as THREAD_SIGNATURE. +# ===----------------------------------------------------------------------=== +add_custom_command( + OUTPUT loader.o + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "--embed-dir=${CMAKE_CURRENT_BINARY_DIR}" "-I${ENGINE}/include" "-I${ISA}" + -c "${CMAKE_CURRENT_SOURCE_DIR}/prepare.c" -o loader.o + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/prepare.c" "${CMAKE_CURRENT_SOURCE_DIR}/matmul.h" + program.bin "${ISA}/params.h" + VERBATIM) + +add_custom_target(program ALL DEPENDS program.bin loader.o) + +# ===----------------------------------------------------------------------=== +# Kernels: the RVV kernels of the GP domain and the kernels of this thread's +# Balls, with the host code that launches them, built against this thread's +# ISA. Every exported symbol gets the suffix __${THREAD}, so the kernels of +# several threads link into one program; models apply kernels.rename to the +# code they compile for this thread. +# libkernels.a Ball instructions issued by the calling core +# libkernels-stream.a Ball instructions sent to the thread through the engine +# ===----------------------------------------------------------------------=== +find_program(LINUX_CXX riscv64-unknown-linux-gnu-g++ REQUIRED) +find_program(LINUX_AR riscv64-unknown-linux-gnu-ar REQUIRED) +find_program(LINUX_OBJCOPY riscv64-unknown-linux-gnu-objcopy REQUIRED) +set(RVV "${DESIGN_REPO}/stack/kernels/rvv") +add_subdirectory("${RVV}" rvv) + +set(KERNEL_SOURCES + "${RVV}/dispatch.cpp" + "${RVV}/matmul_dispatch.cpp" + "${RVV}/cache_dispatch.cpp" + "${RVV}/normalization_dispatch.cpp" + "${RVV}/pointwise_dispatch.cpp" + "${RVV}/activation_dispatch.cpp" + "${RVV}/flash_attention_dispatch.cpp" + "${RVV}/flash_attention_mxfp8_dispatch.cpp") +set(KERNEL_INCLUDES + "${ISA}" + "${DESIGN_ISA_DIR}/design" + "${DESIGN_REPO}/bb-tests/workloads/lib" + "${DESIGN_REPO}/stack/kernels" + "${DESIGN_REPO}/stack/runtime/scheduler" + "${DESIGN_REPO}/stack/runtime/services" + "${ENGINE}/include" + "${DESIGN_REPO}/stack/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") +set(KERNEL_EMBED_DIRS "${CMAKE_CURRENT_BINARY_DIR}/rvv") +set(KERNEL_DEPENDS rvv-images) +set(KERNEL_EXPORTS + _mlir_ciface_rvv_attention_softmax _mlir_ciface_rvv_binary _mlir_ciface_rvv_cos + _mlir_ciface_rvv_flash_attention _mlir_ciface_rvv_flash_attention_mxfp8 _mlir_ciface_rvv_gelu + _mlir_ciface_rvv_layernorm _mlir_ciface_rvv_layernorm_no_bias _mlir_ciface_rvv_logsumexp_softmax + _mlir_ciface_rvv_matmul _mlir_ciface_rvv_matmul_broadcast_rhs + _mlir_ciface_rvv_matmul_broadcast_transpose_rhs _mlir_ciface_rvv_matmul_transpose_rhs + _mlir_ciface_rvv_mxfp8_decode _mlir_ciface_rvv_mxfp8_encode _mlir_ciface_rvv_norm + _mlir_ciface_rvv_norm_no_weight _mlir_ciface_rvv_rope _mlir_ciface_rvv_silu _mlir_ciface_rvv_sin + _mlir_ciface_rvv_snake _mlir_ciface_rvv_softmax _mlir_ciface_rvv_swiglu _mlir_ciface_rvv_tanh + _mlir_ciface_rvv_unary) +# The Balls of this thread add their kernels to the lists above. +foreach(DIRECTORY IN LISTS DESIGN_TARGET_KERNELS_${THREAD}) + get_filename_component(BALL "${DIRECTORY}/../.." ABSOLUTE) + get_filename_component(BALL "${BALL}" NAME) + add_subdirectory("${DIRECTORY}" balls/${BALL}) +endforeach() + +set(RENAME "") +foreach(SYMBOL IN LISTS KERNEL_EXPORTS) + string(APPEND RENAME "${SYMBOL} ${SYMBOL}__${THREAD}\n") +endforeach() +file(CONFIGURE OUTPUT kernels.rename CONTENT "${RENAME}") +list(TRANSFORM KERNEL_INCLUDES PREPEND "-I") +list(TRANSFORM KERNEL_EMBED_DIRS PREPEND "--embed-dir=") + +foreach(LIBRARY kernels kernels-stream) + set(DEFINES) + if(LIBRARY STREQUAL "kernels-stream") + set(DEFINES -DBUCKYBALL_STREAM) + endif() + set(OBJECTS) + foreach(SOURCE IN LISTS KERNEL_SOURCES) + file(RELATIVE_PATH OBJECT "${DESIGN_REPO}" "${SOURCE}.o") + set(OBJECT "${LIBRARY}/${OBJECT}") + add_custom_command( + OUTPUT ${OBJECT} + COMMAND "${LINUX_CXX}" -std=c++20 -O2 -fPIC -ffp-contract=off ${DEFINES} + ${KERNEL_INCLUDES} ${KERNEL_EMBED_DIRS} -MD -MF ${OBJECT}.d -c "${SOURCE}" -o ${OBJECT} + DEPENDS "${SOURCE}" ${KERNEL_DEPENDS} + DEPFILE ${OBJECT}.d + VERBATIM) + list(APPEND OBJECTS "${CMAKE_CURRENT_BINARY_DIR}/${OBJECT}") + endforeach() + add_custom_command( + OUTPUT lib${LIBRARY}.a + COMMAND "${CMAKE_COMMAND}" -E rm -f lib${LIBRARY}.a + COMMAND "${LINUX_AR}" rcs lib${LIBRARY}.a ${OBJECTS} + COMMAND "${LINUX_OBJCOPY}" --redefine-syms=kernels.rename lib${LIBRARY}.a + DEPENDS ${OBJECTS} "${CMAKE_CURRENT_BINARY_DIR}/kernels.rename" + VERBATIM) +endforeach() +add_custom_target(kernels ALL DEPENDS libkernels.a libkernels-stream.a) diff --git a/examples/components/threads/matmul/stack/matmul.h b/examples/components/threads/matmul/stack/matmul.h new file mode 100644 index 000000000..6bbf8334c --- /dev/null +++ b/examples/components/threads/matmul/stack/matmul.h @@ -0,0 +1,5 @@ +#pragma once +#include +struct matmul_task { + uint64_t a, b, bias, output; +}; diff --git a/examples/components/threads/matmul/stack/prepare.c b/examples/components/threads/matmul/stack/prepare.c new file mode 100644 index 000000000..b2fd721dc --- /dev/null +++ b/examples/components/threads/matmul/stack/prepare.c @@ -0,0 +1,32 @@ +#include "matmul.h" +#include "params.h" +#include + +static const unsigned char image[] __attribute__((aligned(8))) = { +#embed "program.bin" +}; + +/* Loads the matmul program and its task descriptor into a thread. */ +static void prepare(uint32_t thread, void *opaque, struct thread_task *task) { + size_t code_bytes = sizeof(image); + size_t data_bytes = sizeof(struct matmul_task); + uint64_t tls_base = thread_query(thread, THREAD_FIELD_TLS_BASE); + uint64_t tls_bytes = thread_query(thread, THREAD_FIELD_TLS_BYTES); + if (code_bytes > thread_query(thread, THREAD_FIELD_CODE_BYTES) || + ((data_bytes + 7) & ~(size_t)7) + 256 > tls_bytes || + code_bytes > thread_query(thread, THREAD_FIELD_TSS_BYTES) || + data_bytes > thread_query(thread, THREAD_FIELD_TSS_BYTES)) + __builtin_trap(); + thread_tss_write(0, image, code_bytes); + thread_load(thread, THREAD_CODE, 0, 0, (code_bytes + 7) & ~(size_t)7); + thread_tss_write(0, opaque, data_bytes); + thread_load(thread, THREAD_TLS, 0, 0, (data_bytes + 7) & ~(size_t)7); + task->entry = 0; + task->code_end = code_bytes; + task->argument = tls_base; + task->stack = tls_base + tls_bytes; +} + +static const struct engine_binding binding + __attribute__((section("engine_bindings"), used)) = {THREAD_SIGNATURE, + prepare}; diff --git a/examples/components/threads/matmul/stack/program.c b/examples/components/threads/matmul/stack/program.c new file mode 100644 index 000000000..cad976c0e --- /dev/null +++ b/examples/components/threads/matmul/stack/program.c @@ -0,0 +1,17 @@ +#include "matmul.h" +#include +#include + +uint64_t entry(const struct matmul_task *args) { + for (unsigned bank = 0; bank < 4; ++bank) + bb_mset(bank, 1, 1, 1); + bb_mvin(args->bias, 0, 4, 1); + bb_mvin(args->a, 1, 16, 1); + bb_mvin(args->b, 2, 16, 1); + bb_smatmul_bias(0, 0); + bb_smatmul_os(1, 2, 3, 16, 16, 16, 1, 1, 0); + bb_mvout(args->output, 3, 64, 1); + for (unsigned bank = 0; bank < 4; ++bank) + bb_mset(bank, 0, 0, 0); + return 0; +} diff --git a/examples/components/threads/matmul/workloads/ctests/CMakeLists.txt b/examples/components/threads/matmul/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..609450e62 --- /dev/null +++ b/examples/components/threads/matmul/workloads/ctests/CMakeLists.txt @@ -0,0 +1,14 @@ +add_subdirectory(transpose_smatmul_quant_pipeline) +set(BUCKYBALL_CTEST_TARGET ${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}) +file( + RELATIVE_PATH + RVV_KERNEL_SOURCE + ${CMAKE_CURRENT_SOURCE_DIR} + ${WORKLOAD_SRC_DIR}/CTest/rvv/rvv_kernel.c +) +add_buckyball_ctest(${RVV_KERNEL_SOURCE}) +set(RVV_KERNEL_TEST + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_kernel" +) +add_dependencies(component-ctests-build ${RVV_KERNEL_TEST}-ctest-build) +add_subdirectory(rvv) diff --git a/examples/components/threads/matmul/workloads/ctests/rvv/CMakeLists.txt b/examples/components/threads/matmul/workloads/ctests/rvv/CMakeLists.txt new file mode 100644 index 000000000..d8f3d530e --- /dev/null +++ b/examples/components/threads/matmul/workloads/ctests/rvv/CMakeLists.txt @@ -0,0 +1,76 @@ +# Images: built by the thread stack (stack/kernels/rvv); dispatch code: built here. +set(KERNEL_DIR "${BUCKYBALL_REPO_ROOT}/stack/kernels/rvv") +set(IMAGE_DIR "${DESIGN_THREAD_BUILD_${BUCKYBALL_CTEST_TARGET}}/rvv") +file(READ "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h" PARAMS) +foreach(PARAM BANK_NUM BANK_WIDTH BANK_LINES) + string(REGEX MATCH "#define ${PARAM} ([0-9]+)" MATCH "${PARAMS}") + set(${PARAM} "${CMAKE_MATCH_1}") +endforeach() +if(BANK_NUM LESS 4 OR NOT BANK_WIDTH EQUAL 128 OR BANK_LINES LESS 32) + message( + FATAL_ERROR + "RVV kernel CTests require 4 banks, 128-bit rows and 512 bytes per bank; " + "${BUCKYBALL_CTEST_TARGET} has ${BANK_NUM} banks, ${BANK_WIDTH}-bit rows, ${BANK_LINES} lines" + ) +endif() +foreach( + KERNEL + silu + swiglu + snake + mxfp8_quant + matmul + norm +) + buckyball_enforce_ctest_line_limit("${KERNEL}.cpp") + set(NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_${KERNEL}" + ) + add_buckyball_linux_ctest(${NAME} ${CMAKE_CURRENT_SOURCE_DIR} ${KERNEL}.cpp) + if(KERNEL STREQUAL "matmul") + set(DISPATCH_SOURCE "${KERNEL_DIR}/matmul_dispatch.cpp") + elseif(KERNEL STREQUAL "mxfp8_quant") + set(DISPATCH_SOURCE "${KERNEL_DIR}/quant_dispatch.cpp") + else() + set(DISPATCH_SOURCE "${KERNEL_DIR}/dispatch.cpp") + endif() + target_sources(${NAME}-linux PRIVATE "${DISPATCH_SOURCE}") + target_compile_options(${NAME}-linux PRIVATE "--embed-dir=${IMAGE_DIR}") + target_include_directories( + ${NAME}-linux + PRIVATE + "${KERNEL_DIR}" + "${BUCKYBALL_REPO_ROOT}/stack/kernels" + "${BUCKYBALL_REPO_ROOT}/stack/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine" + ) + target_compile_options(${NAME}-linux PRIVATE -O2) + set_target_properties(${NAME}-linux PROPERTIES LINKER_LANGUAGE CXX) + target_link_libraries(${NAME}-linux PRIVATE m) + buckyball_ctest_deps(DEPS ${CMAKE_CURRENT_SOURCE_DIR} ${KERNEL}.cpp) + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${NAME}-baremetal" + COMMAND + ${ELF_CXX} ${BUCKYBALL_CTEST_C_FLAGS} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${WORKLOAD_LIB_DIR} -I${KERNEL_DIR} + -I${BUCKYBALL_REPO_ROOT}/stack/kernels --embed-dir=${IMAGE_DIR} + -I${BUCKYBALL_ISA_DIR}/design + -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} + -I${BUCKYBALL_REPO_ROOT}/stack/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine + -o "${NAME}-baremetal" ${BAREMETAL_DIR}/crt0.S -x c + ${WORKLOAD_LIB_DIR}/buckyball.c ${BBHW_MEM_C} + ${BUCKYBALL_DESIGN_COMMON_SOURCES} -x c++ + "${CMAKE_CURRENT_SOURCE_DIR}/${KERNEL}.cpp" "${DISPATCH_SOURCE}" -lm + DEPENDS ${DEPS} "${DISPATCH_SOURCE}" ${BAREMETAL_DIR}/crt0.S + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + VERBATIM + ) + add_custom_target( + ${NAME}-ctest-build + DEPENDS + ${NAME}-linux-build + "${CMAKE_CURRENT_BINARY_DIR}/${NAME}-baremetal" + ) + add_dependencies(component-ctests-build ${NAME}-ctest-build) +endforeach() diff --git a/examples/cores/goban/workloads/ctests/rvv/matmul.cpp b/examples/components/threads/matmul/workloads/ctests/rvv/matmul.cpp similarity index 100% rename from examples/cores/goban/workloads/ctests/rvv/matmul.cpp rename to examples/components/threads/matmul/workloads/ctests/rvv/matmul.cpp diff --git a/examples/cores/goban/workloads/ctests/rvv/mxfp8_quant.cpp b/examples/components/threads/matmul/workloads/ctests/rvv/mxfp8_quant.cpp similarity index 95% rename from examples/cores/goban/workloads/ctests/rvv/mxfp8_quant.cpp rename to examples/components/threads/matmul/workloads/ctests/rvv/mxfp8_quant.cpp index 07f9f04b1..fa3ccd486 100644 --- a/examples/cores/goban/workloads/ctests/rvv/mxfp8_quant.cpp +++ b/examples/components/threads/matmul/workloads/ctests/rvv/mxfp8_quant.cpp @@ -1,9 +1,11 @@ #include -#include #include #include #include +extern "C" void mxfp8_quant(const uint32_t *input, uint8_t *output, + uint32_t count); + alignas(16) static float input[128]; alignas(16) static uint8_t output[144]; static double candidates[127]; diff --git a/examples/components/threads/matmul/workloads/ctests/rvv/norm.cpp b/examples/components/threads/matmul/workloads/ctests/rvv/norm.cpp new file mode 100644 index 000000000..27f693da4 --- /dev/null +++ b/examples/components/threads/matmul/workloads/ctests/rvv/norm.cpp @@ -0,0 +1,81 @@ +#include +#include +#include +#include +#include + +extern "C" void _mlir_ciface_rvv_norm(UnrankedMemRefType *, + UnrankedMemRefType *, + UnrankedMemRefType *, uint32_t, + uint32_t); +extern "C" void _mlir_ciface_rvv_norm_no_weight(UnrankedMemRefType *, + UnrankedMemRefType *, + uint32_t, uint32_t); +constexpr int capacity = BANK_LINES * BANK_WIDTH / 32; +constexpr int maximum = 2 * capacity + 5; +alignas(16) static float input[2 * (maximum + 4) + 4]; +alignas(16) static float output[2 * maximum + 4], expected[2 * maximum]; +alignas(16) static float weights[maximum]; + +int main() { + for (int width : {capacity - 1, capacity, capacity + 3, maximum}) { + for (bool weighted : {false, true}) { + for (float &value : output) + value = -999.0f; + for (int col = 0; col < width; ++col) + weights[col] = 0.5f + (col % 7) * 0.125f; + for (int row = 0; row < 2; ++row) + for (int col = 0; col < width; ++col) + input[2 + row * (width + 4) + col] = + ((col + row * 3) % 31 - 15) * 0.125f; + float multiplier = 1.0f / width, epsilon = 1.0e-5f; + uint32_t meanBits, epsilonBits; + std::memcpy(&meanBits, &multiplier, 4); + std::memcpy(&epsilonBits, &epsilon, 4); + asm volatile("csrw fcsr, zero" ::: "memory"); + for (int row = 0; row < 2; ++row) { + volatile float sum = 0; + for (int col = 0; col < width; ++col) { + float value = input[2 + row * (width + 4) + col]; + volatile float square = value * value; + sum = sum + square; + } + volatile float mean = sum * multiplier; + float scale = 1.0f / std::sqrt(mean + epsilon); + for (int col = 0; col < width; ++col) { + volatile float value = input[2 + row * (width + 4) + col] * scale; + expected[row * width + col] = weighted ? weights[col] * value : value; + } + } + uint32_t expectedFlags; + asm volatile("csrr %0, fflags" : "=r"(expectedFlags)::"memory"); + StridedMemRefType in{ + input, input, 2, {2, width}, {width + 4, 1}}; + StridedMemRefType out{ + output, output, 2, {2, width}, {width, 1}}; + StridedMemRefType weight{weights, weights, 0, {width}, {1}}; + UnrankedMemRefType inRef{2, &in}, outRef{2, &out}, + wRef{1, &weight}; + asm volatile("csrw fcsr, zero" ::: "memory"); + if (weighted) + _mlir_ciface_rvv_norm(&outRef, &inRef, &wRef, meanBits, epsilonBits); + else + _mlir_ciface_rvv_norm_no_weight(&outRef, &inRef, meanBits, epsilonBits); + uint32_t flags; + asm volatile("csrr %0, fflags" : "=r"(flags)::"memory"); + if (flags != expectedFlags || + std::memcmp(output + 2, expected, 2 * width * sizeof(float))) { + std::printf( + "RMSNorm mismatch width=%d weighted=%d flags=%u expected=%u\n", + width, weighted, flags, expectedFlags); + return 1; + } + if (output[0] != -999 || output[1] != -999 || + output[2 * width + 2] != -999 || output[2 * width + 3] != -999) + return 1; + } + } + std::puts( + "RVV RMSNORM PASSED (multi-bank chunks, strided rows, tails, FP flags)"); + return 0; +} diff --git a/examples/cores/goban/workloads/ctests/rvv/silu.cpp b/examples/components/threads/matmul/workloads/ctests/rvv/silu.cpp similarity index 100% rename from examples/cores/goban/workloads/ctests/rvv/silu.cpp rename to examples/components/threads/matmul/workloads/ctests/rvv/silu.cpp diff --git a/examples/cores/goban/workloads/ctests/rvv/snake.cpp b/examples/components/threads/matmul/workloads/ctests/rvv/snake.cpp similarity index 100% rename from examples/cores/goban/workloads/ctests/rvv/snake.cpp rename to examples/components/threads/matmul/workloads/ctests/rvv/snake.cpp diff --git a/examples/cores/goban/workloads/ctests/rvv/swiglu.cpp b/examples/components/threads/matmul/workloads/ctests/rvv/swiglu.cpp similarity index 100% rename from examples/cores/goban/workloads/ctests/rvv/swiglu.cpp rename to examples/components/threads/matmul/workloads/ctests/rvv/swiglu.cpp diff --git a/examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt b/examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt new file mode 100644 index 000000000..e77cc0d0c --- /dev/null +++ b/examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(transpose_smatmul_quant_pipeline SOURCES + transpose_smatmul_quant_pipeline-main.cpp + goban.c) diff --git a/examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c b/examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c new file mode 100644 index 000000000..3c8f4afe8 --- /dev/null +++ b/examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c @@ -0,0 +1,53 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +#if BANK_WIDTH != 128 || BANK_LINES < 64 +#error "Goban thread pipeline requires 128-bit banks with at least 64 rows" +#endif + +enum { DIM = 16 }; +static int8_t input[BB_CORE_NUM][DIM * DIM] __attribute__((aligned(64))); +static int8_t weight[BB_CORE_NUM][DIM * DIM] __attribute__((aligned(64))); +static int32_t bias[BB_CORE_NUM][DIM] __attribute__((aligned(64))); +static float scale[BB_CORE_NUM][DIM] __attribute__((aligned(64))); +static int8_t output[BB_CORE_NUM][DIM * DIM] __attribute__((aligned(64))); + +#ifdef __cplusplus +extern "C" +#endif + int goban_pipeline(uint32_t instance) { + int slot = instance; + for (int row = 0; row < DIM; ++row) + for (int col = 0; col < DIM; ++col) { + input[slot][row * DIM + col] = (row * 3 + col * 5) % 9 - 4; + weight[slot][row * DIM + col] = (row + 2 * col) % 5 - 2; + } + for (int col = 0; col < DIM; ++col) + scale[slot][col] = 1.0f; + for (int bank = 0; bank < 10; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[slot], 0, DIM, 1); + bb_mvin((uintptr_t)weight[slot], 2, DIM, 1); + bb_mvin((uintptr_t)bias[slot], 3, 4, 1); + bb_mvin((uintptr_t)scale[slot], 8, 4, 1); + bb_transpose(0, 1, DIM, 8); + bb_smatmul_bias(3, 0); + bb_smatmul_os(1, 2, 4, DIM, DIM, DIM, 1, 1, 0); + bb_quant_i32_to_i8(4, 8, 9, DIM * 4, 0, 0, 4, 4, 4, 0); + bb_mvout((uintptr_t)output[slot], 9, DIM, 1); + for (int row = 0; row < DIM; ++row) + for (int col = 0; col < DIM; ++col) { + int expected = 0; + for (int k = 0; k < DIM; ++k) + expected += input[slot][k * DIM + row] * weight[slot][k * DIM + col]; + if (output[slot][row * DIM + col] != expected) + return 1; + } + return 0; +} diff --git a/examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp b/examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp new file mode 100644 index 000000000..32fd7c88a --- /dev/null +++ b/examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int goban_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, goban_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == BB_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/cores/README.md b/examples/cores/README.md deleted file mode 100644 index ffe742d6a..000000000 --- a/examples/cores/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Cores - -Each directory in this tree describes one concrete single-core design. A Core -owns its CPU/frontend parameters, BallDomain, MemDomain, and the -compiler package for its ISA. It does not own a tile topology, device model, -kernel, or regression suite. - -Each implemented Core has an `arch/` top implementing `HasCLink` and explicitly -instantiating its CPU and compute IPs. The framework connects CLink to shared -memory and control interfaces; concrete designs do not inherit another design. - -Every Core has a `configs/` tree and `isa/` (`ballISA.h` from balldomain). -Buckyball Cores also have `compiler/` (buddy-mlir dialect plugin; CMake is the -C++ recipe, not the product entry) and non-empty `ballISA`. Rocket-only -cores keep `configs/default.toml` without a `balldomain` key so tiles stay -Rocket-only; empty `configs/balldomains/` + `isa/` still exist for Bazel / -`compilerCore` consumers. BEMU lives in bebop; chip-level tile runners are -`examples/chips//emu/src/main.rs`. Ball functional models are -`examples/balls//emu/`. - -Chip topology files instantiate a concrete Core TOML through relative -`include` paths; that include is the config entry (not a fixed -`configs/default.toml`). Chips own tile-shared memory, runtime, and multi-core -behavior. diff --git a/examples/cores/ant/arch/src/main/scala/AntCore.scala b/examples/cores/ant/arch/src/main/scala/AntCore.scala deleted file mode 100644 index f66c99ac6..000000000 --- a/examples/cores/ant/arch/src/main/scala/AntCore.scala +++ /dev/null @@ -1,19 +0,0 @@ -package examples.cores.ant - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.ant.LocalExecution -import framework.system.core.{AntEndpoint, CoreConnection} -import framework.system.core.accelerator.{AntAdmission, BuckyballAccelerator} - -@instantiable -class AntCore(config: GlobalConfig) extends AntEndpoint(config) { - val execution = Instantiate(new LocalExecution(local)) - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball)) - val issuer = Instantiate(new AntAdmission(local, tracking, cpuParams.core.nPMPs, bus)) - CoreConnection.ant(clink, execution, accelerator, issuer, signature) -} - -object AntCore extends framework.system.core.AntCoreFactory { - def instantiate(b: GlobalConfig): Instance[AntEndpoint] = Instantiate(new AntCore(b)) -} diff --git a/examples/cores/ant/configs/cpu/ant.toml b/examples/cores/ant/configs/cpu/ant.toml deleted file mode 100644 index f658a9bc7..000000000 --- a/examples/cores/ant/configs/cpu/ant.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "ant" -config = "default.toml" diff --git a/examples/cores/ant/configs/cpu/default.toml b/examples/cores/ant/configs/cpu/default.toml deleted file mode 100644 index 19fe2e82e..000000000 --- a/examples/cores/ant/configs/cpu/default.toml +++ /dev/null @@ -1,7 +0,0 @@ -codeBytes = 16384 -taskBits = 32 - -[tls] -base = 0x10000 -bytes = 16384 -dataBits = 128 diff --git a/examples/cores/ant/configs/default.toml b/examples/cores/ant/configs/default.toml deleted file mode 100644 index 9eab27336..000000000 --- a/examples/cores/ant/configs/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -factory = "examples.cores.ant.AntCore" -cpu = "cpu/ant.toml" diff --git a/examples/cores/attention/arch/src/main/scala/AttentionCore.scala b/examples/cores/attention/arch/src/main/scala/AttentionCore.scala deleted file mode 100644 index 8afd2d445..000000000 --- a/examples/cores/attention/arch/src/main/scala/AttentionCore.scala +++ /dev/null @@ -1,19 +0,0 @@ -package examples.cores.attention - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.ant.LocalExecution -import framework.system.core.{AntEndpoint, CoreConnection} -import framework.system.core.accelerator.{AntAdmission, BuckyballAccelerator} - -@instantiable -class AttentionCore(config: GlobalConfig) extends AntEndpoint(config) { - val execution = Instantiate(new LocalExecution(local)) - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball)) - val issuer = Instantiate(new AntAdmission(local, tracking, cpuParams.core.nPMPs, bus)) - CoreConnection.ant(clink, execution, accelerator, issuer, signature) -} - -object AttentionCore extends framework.system.core.AntCoreFactory { - def instantiate(b: GlobalConfig): Instance[AntEndpoint] = Instantiate(new AttentionCore(b)) -} diff --git a/examples/cores/attention/compiler/CMakeLists.txt b/examples/cores/attention/compiler/CMakeLists.txt deleted file mode 100644 index 6b8bd19d0..000000000 --- a/examples/cores/attention/compiler/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -set(BB_CORE_NAME attention) -set(BB_CORE_TARGET_PREFIX "") - -set(BB_CORE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..) -get_filename_component(_BB_REPO_ROOT "${BB_CORE_DIR}/../../.." ABSOLUTE) -include_directories(${_BB_REPO_ROOT}/examples/balls) - -add_subdirectory(src) diff --git a/examples/cores/attention/compiler/src/CMakeLists.txt b/examples/cores/attention/compiler/src/CMakeLists.txt deleted file mode 100644 index 081d3ff4a..000000000 --- a/examples/cores/attention/compiler/src/CMakeLists.txt +++ /dev/null @@ -1,51 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -message(STATUS "Building Attention compiler Core package") -message(STATUS " Source: ${CMAKE_CURRENT_SOURCE_DIR}") -message(STATUS " Binary: ${CMAKE_CURRENT_BINARY_DIR}") - -get_filename_component(BB_REPO_ROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../.. ABSOLUTE) -set(BB_COMPILER_COMMON_DIR ${BB_REPO_ROOT}/stack/compiler/src) - -if(NOT EXISTS ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Buckyball.td) - message(FATAL_ERROR "Missing Buckyball compiler common dir: ${BB_COMPILER_COMMON_DIR}") -endif() - -include_directories(${BB_COMPILER_COMMON_DIR}) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/Dialect) -include_directories(${CMAKE_CURRENT_BINARY_DIR}/Dialect) -include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/Dialect/Buckyball) - -set(ATTENTION_BALL_COMPILER_DIALECT_DIRS - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball -) - -foreach(ATTENTION_BALL_COMPILER_DIALECT_DIR ${ATTENTION_BALL_COMPILER_DIALECT_DIRS}) - if(NOT EXISTS ${ATTENTION_BALL_COMPILER_DIALECT_DIR}) - message(FATAL_ERROR - "Missing attention ball compiler dialect dir: ${ATTENTION_BALL_COMPILER_DIALECT_DIR}") - endif() - include_directories(${ATTENTION_BALL_COMPILER_DIALECT_DIR}) -endforeach() - -add_subdirectory(Dialect) -add_subdirectory(Conversion) -add_subdirectory(Target) - -add_custom_target(buckyball-compiler-${BB_CORE_NAME} - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass -) diff --git a/examples/cores/attention/compiler/src/Conversion/CMakeLists.txt b/examples/cores/attention/compiler/src/Conversion/CMakeLists.txt deleted file mode 100644 index 9ca69b1a2..000000000 --- a/examples/cores/attention/compiler/src/Conversion/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_subdirectory(LowerBuckyball) -add_subdirectory(LowerTileToBuckyball) diff --git a/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt b/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt deleted file mode 100644 index eb782edc6..000000000 --- a/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/PhysicalBankState.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/ReportBankUsagePass.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp - ${BB_assign_SOURCES} - ${BB_bank_ssa_SOURCES} - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/QuantizeTensorToBankSSAPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/RuntimeLowering.cpp - LowerBuckyballPass.cpp - LowerBuckyballToBankSSAPass.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - MLIRLinalgDialect -) diff --git a/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp b/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp deleted file mode 100644 index c4989db92..000000000 --- a/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp +++ /dev/null @@ -1,127 +0,0 @@ -//===- LowerBuckyballPass.cpp - Attention Buckyball lowering pass -//-------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/TypeConverter.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Pass/Pass.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Buckyball/Transform.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -class LowerBuckyballToLLVMPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToLLVMPass) - LowerBuckyballToLLVMPass() = default; - LowerBuckyballToLLVMPass(const LowerBuckyballToLLVMPass &) {} - - StringRef getArgument() const final { return "lower-buckyball"; } - StringRef getDescription() const final { - return "Lower Attention Buckyball dialect ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -class LowerBankSSAToIntrinsicsPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBankSSAToIntrinsicsPass) - LowerBankSSAToIntrinsicsPass() = default; - LowerBankSSAToIntrinsicsPass(const LowerBankSSAToIntrinsicsPass &) {} - - StringRef getArgument() const final { return "lower-bank-ssa-to-intrinsics"; } - StringRef getDescription() const final { - return "Lower Attention bank-SSA and Buckyball ops to intrinsic ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballPass() { - registerTileRuntimePass(); - PassRegistration(); -} - -void mlir::buddy::registerLowerBankSSAToIntrinsicsPass() { - PassRegistration(); -} diff --git a/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp b/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp deleted file mode 100644 index 9450ff9b5..000000000 --- a/examples/cores/attention/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp +++ /dev/null @@ -1,108 +0,0 @@ -//===- LowerBuckyballToBankSSAPass.cpp - Buckyball bank-SSA lowering -//----------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Utils/StructuredOpsUtils.h" -#include "mlir/IR/AffineMap.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Trace/TraceDialect.h" - -using namespace mlir; - -namespace mlir::buddy { -#define BUCKYBALL_BANK_SSA_HOOK(BALL) \ - void populate##BALL##LowerBuckyballToBankSSAPatterns(RewritePatternSet &); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_BANK_SSA_HOOK -} // namespace mlir::buddy - -namespace { - -class MemTransposeToLinalgPattern - : public OpRewritePattern<::buddy::buckyball::MemTransposeOp> { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(::buddy::buckyball::MemTransposeOp op, - PatternRewriter &rewriter) const override { - auto input = dyn_cast(op.getInput().getType()); - auto output = dyn_cast(op.getOutput().getType()); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 2 || - output.getRank() != 2 || output.getShape()[0] != input.getShape()[1] || - output.getShape()[1] != input.getShape()[0]) - return op.emitError("requires matching static rank-2 transpose memrefs"); - - MLIRContext *context = rewriter.getContext(); - AffineMap identity = AffineMap::getMultiDimIdentityMap(2, context); - AffineMap transpose = AffineMap::get( - 2, 0, {rewriter.getAffineDimExpr(1), rewriter.getAffineDimExpr(0)}, - context); - SmallVector iterators(2, - utils::IteratorType::parallel); - rewriter.create( - op.getLoc(), TypeRange{}, ValueRange{op.getInput()}, - ValueRange{op.getOutput()}, ArrayRef{identity, transpose}, - iterators, [](OpBuilder &builder, Location location, ValueRange args) { - builder.create(location, args[0]); - }); - rewriter.eraseOp(op); - return success(); - } -}; - -} // namespace - -namespace { - -class LowerBuckyballToBankSSAPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToBankSSAPass) - - StringRef getArgument() const final { return "lower-buckyball-to-bank-ssa"; } - StringRef getDescription() const final { - return "Lower Buckyball Buckyball ops to explicit bank-SSA ops."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert<::buddy::trace::BuddyTraceDialect, arith::ArithDialect, - linalg::LinalgDialect, memref::MemRefDialect, - scf::SCFDialect, ::buddy::buckyball::BuckyballDialect>(); - } - - void runOnOperation() override { - RewritePatternSet patterns(&getContext()); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_BANK_SSA_HOOK(BALL) \ - if (ball == #BALL) \ - mlir::buddy::populate##BALL##LowerBuckyballToBankSSAPatterns(patterns); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_BANK_SSA_HOOK - } - mlir::buddy::populateQuantizeTensorToBankSSAPatterns(patterns); - patterns.add(&getContext()); - if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballToBankSSAPass() { - PassRegistration(); -} diff --git a/examples/cores/attention/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt b/examples/cores/attention/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt deleted file mode 100644 index c12d60c23..000000000 --- a/examples/cores/attention/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerTileToBuckyball/QuantizedKernelPatterns.cpp - LowerTileToBuckyball.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Conversion/LowerTileToBuckyball/TileMatMulPatterns.cpp - - LINK_LIBS PUBLIC - BuddyTile - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRLinalgDialect -) diff --git a/examples/cores/attention/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp b/examples/cores/attention/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp deleted file mode 100644 index cca771535..000000000 --- a/examples/cores/attention/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp +++ /dev/null @@ -1,104 +0,0 @@ -#include "Target/BuckyballTargetRegistry.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/DialectConversion.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerTileToBuckyball/LowerTileToBuckyball.h" -#include "Tile/TileDialect.h" -#include "Tile/TileOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -namespace tile = ::buddy::tile; - -namespace mlir::buddy { -#define BUCKYBALL_TILE_HOOK(BALL) \ - void populate##BALL##TileLoweringPatterns(RewritePatternSet &, int64_t, \ - int64_t, int64_t); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_TILE_HOOK -} // namespace mlir::buddy - -namespace { -class TileTransposeLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(tile::TileTransposeOp op, - PatternRewriter &rewriter) const override { - auto inputType = dyn_cast(op.getAMemArray().getType()); - auto outputType = dyn_cast(op.getBMemArray().getType()); - if (!inputType || !outputType || !inputType.hasStaticShape() || - !outputType.hasStaticShape() || inputType.getRank() != 2 || - outputType.getRank() != 2) - return op.emitError("requires static rank-2 input and output memrefs"); - if (outputType.getShape()[0] != inputType.getShape()[1] || - outputType.getShape()[1] != inputType.getShape()[0] || - inputType.getElementType() != outputType.getElementType()) - return op.emitError("requires a matching transposed output memref"); - - rewriter.create(op.getLoc(), op.getAMemArray(), - op.getBMemArray()); - rewriter.eraseOp(op); - return success(); - } -}; - -class LowerTileToBuckyballPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerTileToBuckyballPass) - - StringRef getArgument() const final { return "convert-tile-to-buckyball"; } - StringRef getDescription() const final { - return "Convert Tile operations for the selected Buckyball target."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ConversionTarget target(*context); - target.addLegalDialect<::buddy::buckyball::BuckyballDialect, - memref::MemRefDialect, arith::ArithDialect, - scf::SCFDialect, func::FuncDialect, - linalg::LinalgDialect>(); - target.addIllegalOp(); - - RewritePatternSet patterns(context); - mlir::buddy::populateQuantizedKernelTileLoweringPatterns(patterns); - patterns.add(context); - for (llvm::StringRef ball : targetConfig.balls) { -#define BUCKYBALL_TILE_HOOK(BALL) \ - if (ball == #BALL) \ - mlir::buddy::populate##BALL##TileLoweringPatterns( \ - patterns, targetConfig.bankWidthBits / 8, targetConfig.bankDepth, \ - targetConfig.bankNum); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_TILE_HOOK - } - if (failed(applyPartialConversion(getOperation(), target, - std::move(patterns)))) - signalPassFailure(); - } -}; -} // namespace - -void mlir::buddy::registerLowerTileToBuckyballPass() { - PassRegistration(); -} diff --git a/examples/cores/attention/compiler/src/Dialect/Buckyball/AttentionBuckyball.td b/examples/cores/attention/compiler/src/Dialect/Buckyball/AttentionBuckyball.td deleted file mode 100644 index e9290ae5c..000000000 --- a/examples/cores/attention/compiler/src/Dialect/Buckyball/AttentionBuckyball.td +++ /dev/null @@ -1,26 +0,0 @@ -//===-- AttentionBuckyball.td - Attention Buckyball dialect ops -*- tablegen -*-===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#ifndef ATTENTION_BUCKYBALL_OPS -#define ATTENTION_BUCKYBALL_OPS - -include "Buckyball.td" -include "ToInt8Ball.td" -include "Int2FpBall.td" -include "TransposeBall.td" -include "MxmmBall.td" - -#endif // ATTENTION_BUCKYBALL_OPS diff --git a/examples/cores/attention/compiler/src/Dialect/Buckyball/CMakeLists.txt b/examples/cores/attention/compiler/src/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 598c4434a..000000000 --- a/examples/cores/attention/compiler/src/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(Buckyball.h.inc -gen-op-decls) -mlir_tablegen(Buckyball.cpp.inc -gen-op-defs) -mlir_tablegen(BuckyballTypes.h.inc -gen-typedef-decls -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballTypes.cpp.inc -gen-typedef-defs -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballDialect.h.inc -gen-dialect-decls -dialect=buckyball) -mlir_tablegen(BuckyballDialect.cpp.inc -gen-dialect-defs -dialect=buckyball) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}MLIRBuckyballIncGen) -add_mlir_doc(AttentionBuckyball ${BB_CORE_TARGET_PREFIX}Buckyball Dialects/ - -gen-dialect-doc) - -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(BuckyballConversions.inc -gen-llvmir-conversions) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen) - -add_subdirectory(IR) -add_subdirectory(Transforms) diff --git a/examples/cores/attention/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt b/examples/cores/attention/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt deleted file mode 100644 index 42c7f8807..000000000 --- a/examples/cores/attention/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/IR/BuckyballDialect.cpp - ${BB_TARGET_REGISTRY_SOURCE} - ${BB_BALL_IR_SOURCES} - - LINK_LIBS PUBLIC - MLIRIR - MLIRLLVMDialect - MLIRBufferizationDialect - MLIRMemRefDialect -) - -target_include_directories(${BB_CORE_TARGET_PREFIX}BuddyBuckyball PUBLIC - ${BB_COMPILER_COMMON_DIR} - ${BB_COMPILER_COMMON_DIR}/Dialect - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball - ${CMAKE_CURRENT_BINARY_DIR}/.. - ${CMAKE_CURRENT_BINARY_DIR}/../../.. -) diff --git a/examples/cores/attention/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt b/examples/cores/attention/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt deleted file mode 100644 index 0d2cecf2e..000000000 --- a/examples/cores/attention/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - LegalizeForLLVMExport.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRIR - MLIRLLVMCommonConversion - MLIRLLVMDialect -) diff --git a/examples/cores/attention/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/cores/attention/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 0b1a5b224..000000000 --- a/examples/cores/attention/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "Buckyball/Transform.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball::legalize; - -namespace mlir::buddy::buckyball { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - void populate##BALL##LegalizeForLLVMExportPatterns( \ - LLVMTypeConverter &, RewritePatternSet &, bool, int64_t); \ - void configure##BALL##LegalizeForExportTarget(LLVMConversionTarget &, bool); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK -} // namespace mlir::buddy::buckyball - -void mlir::populateBuckyballLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, - int64_t bankWidthBytes, int64_t bankDepth, int64_t bankNum, - bool includeFuncOperandForwarding, bool stable) { - populateBaseLegalizeForLLVMExportPatterns(converter, patterns, - includeFuncOperandForwarding); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - if (ball == #BALL) \ - buddy::buckyball::populate##BALL##LegalizeForLLVMExportPatterns( \ - converter, patterns, stable, bankDepth); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK - } - (void)bankWidthBytes; - (void)bankNum; -} - -void mlir::configureBuckyballLegalizeForExportTarget( - LLVMConversionTarget &target, bool stable) { - configureBaseLegalizeForExportTarget(target); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - if (ball == #BALL) \ - buddy::buckyball::configure##BALL##LegalizeForExportTarget(target, stable); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK - } -} diff --git a/examples/cores/attention/compiler/src/Dialect/CMakeLists.txt b/examples/cores/attention/compiler/src/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/attention/compiler/src/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/attention/compiler/src/Target/CMakeLists.txt b/examples/cores/attention/compiler/src/Target/CMakeLists.txt deleted file mode 100644 index 39d31dc9b..000000000 --- a/examples/cores/attention/compiler/src/Target/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(LLVMIR) diff --git a/examples/cores/attention/compiler/src/Target/LLVMIR/CMakeLists.txt b/examples/cores/attention/compiler/src/Target/LLVMIR/CMakeLists.txt deleted file mode 100644 index 0ca0f41c5..000000000 --- a/examples/cores/attention/compiler/src/Target/LLVMIR/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Dialect) diff --git a/examples/cores/attention/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt b/examples/cores/attention/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 06b99171b..000000000 --- a/examples/cores/attention/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_mlir_translation_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_COMPILER_COMMON_DIR}/Target/LLVMIR/Dialect/Buckyball/BuckyballToLLVMIRTranslation.cpp - - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen - intrinsics_gen - - LINK_COMPONENTS - Core - - LINK_LIBS PUBLIC - MLIRIR - MLIRSupport - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball -) diff --git a/examples/cores/attention/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt b/examples/cores/attention/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/attention/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/attention/configs/balldomains/default.toml b/examples/cores/attention/configs/balldomains/default.toml deleted file mode 100644 index f9796a850..000000000 --- a/examples/cores/attention/configs/balldomains/default.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Attention Ball Domain Configuration - -ballNum = 5 - -ballIdMappings = [ - { ballId = 0, ballName = "MxmmBall", ballClass = "examples.balls.mxmm.MxmmBall", config = "../../../../balls/mxmm/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 1, ballName = "ToInt8Ball", ballClass = "examples.balls.toint8.ToInt8Ball", config = "../../../../balls/toint8/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 2, ballName = "Int2FpBall", ballClass = "examples.balls.int2fp.Int2FpBall", config = "../../../../balls/int2fp/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 3, ballName = "TransposeBall", ballClass = "examples.balls.transpose.TransposeBall", config = "../../../../balls/transpose/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 4, ballName = "MxquantBall", ballClass = "examples.balls.mxquant.MxquantBall", config = "../../../../balls/mxquant/configs/default.toml", inBW = 1, outBW = 1 } -] - -ballISA = [ - { mnemonic = "MXMM_MXFP8", funct7 = 71, bid = 0 }, - { mnemonic = "MXMM_MXFP8_WINDOW", funct7 = 75, bid = 0 }, - { mnemonic = "MXMM_FMA32", funct7 = 72, bid = 0 }, - { mnemonic = "MXMM_F32", funct7 = 73, bid = 0 }, - { mnemonic = "QUANT_F32_TO_I8", funct7 = 51, bid = 1 }, - { mnemonic = "QUANT_I32_TO_I8", funct7 = 67, bid = 1 }, - { mnemonic = "INT32_TO_FP32", funct7 = 68, bid = 2 }, - { mnemonic = "TRANSPOSE", funct7 = 49, bid = 3 }, - { mnemonic = "MXQUANT", funct7 = 54, bid = 4 } -] diff --git a/examples/cores/attention/configs/cpu/default.toml b/examples/cores/attention/configs/cpu/default.toml deleted file mode 100644 index 9cf6bf076..000000000 --- a/examples/cores/attention/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "ant" -config = "../../../ant/configs/cpu/default.toml" diff --git a/examples/cores/attention/configs/default.toml b/examples/cores/attention/configs/default.toml deleted file mode 100644 index 582c4a554..000000000 --- a/examples/cores/attention/configs/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -factory = "examples.cores.attention.AttentionCore" -# Attention Core Configuration (default) - -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/default.toml" -rvv = "rvv/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/attention/configs/frontends/default.toml b/examples/cores/attention/configs/frontends/default.toml deleted file mode 100644 index 418994d6b..000000000 --- a/examples/cores/attention/configs/frontends/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -robEntries = 16 -rsOutOfOrderResponse = true -bankIdLen = 10 -vbankIdUpperBound = 31 -sharedBankIdBase = 32 -iterLen = 34 -subRobEnable = false -subRobDepth = 64 diff --git a/examples/cores/attention/configs/memdomains/default.toml b/examples/cores/attention/configs/memdomains/default.toml deleted file mode 100644 index d128b5e50..000000000 --- a/examples/cores/attention/configs/memdomains/default.toml +++ /dev/null @@ -1,32 +0,0 @@ -# Attention Core Memory Domain Configuration (default) -# Private (per-core) memory parameters. Shared memory lives in the tile TOML. - -[bank] -num = 20 -width = 128 -entries = 1024 -maskLen = 16 -channel = 7 - -[dma] -nXacts = 8 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 4 - -[tma] -readChannel = 4 -writeChannel = 4 - -[mmio] -enable = false -bankNum = 16 -bankEntries = 64 -bankWidth = 128 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/boom/arch/src/main/scala/configs/BoomCpuParam.scala b/examples/cores/boom/arch/src/main/scala/configs/BoomCpuParam.scala deleted file mode 100644 index 32fd57a0f..000000000 --- a/examples/cores/boom/arch/src/main/scala/configs/BoomCpuParam.scala +++ /dev/null @@ -1,32 +0,0 @@ -package framework.system.core.boom.configs - -import upickle.default._ - -case class BoomDCacheParam( - nSets: Int = 64, - nWays: Int = 4, - nMSHRs: Int = 2) - -object BoomDCacheParam { - implicit val rw: ReadWriter[BoomDCacheParam] = macroRW -} - -case class BoomICacheParam( - nSets: Int = 64, - nWays: Int = 4) - -object BoomICacheParam { - implicit val rw: ReadWriter[BoomICacheParam] = macroRW -} - -/** BOOM configuration from the chip PB. The explicit Tile does not instantiate BOOM yet. */ -case class BoomCpuParam( - fetchWidth: Int = 4, - decodeWidth: Int = 1, - numRobEntries: Int = 32, - dcache: BoomDCacheParam = BoomDCacheParam(), - icache: BoomICacheParam = BoomICacheParam()) - -object BoomCpuParam { - implicit val rw: ReadWriter[BoomCpuParam] = macroRW -} diff --git a/examples/cores/boom/configs/balldomains/default.toml b/examples/cores/boom/configs/balldomains/default.toml deleted file mode 100644 index a2733b93c..000000000 --- a/examples/cores/boom/configs/balldomains/default.toml +++ /dev/null @@ -1,9 +0,0 @@ -# Boom-only core: no Buckyball balls. - -ballNum = 0 - -ballIdMappings = [ -] - -ballISA = [ -] diff --git a/examples/cores/boom/configs/cpu/boom/default.toml b/examples/cores/boom/configs/cpu/boom/default.toml deleted file mode 100644 index 51c752071..000000000 --- a/examples/cores/boom/configs/cpu/boom/default.toml +++ /dev/null @@ -1,12 +0,0 @@ -fetchWidth = 4 -decodeWidth = 1 -numRobEntries = 32 - -[dcache] -nSets = 64 -nWays = 4 -nMSHRs = 2 - -[icache] -nSets = 64 -nWays = 4 diff --git a/examples/cores/boom/configs/cpu/default.toml b/examples/cores/boom/configs/cpu/default.toml deleted file mode 100644 index 40b4f000e..000000000 --- a/examples/cores/boom/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "boom" -config = "./boom/default.toml" diff --git a/examples/cores/boom/configs/default.toml b/examples/cores/boom/configs/default.toml deleted file mode 100644 index c691e6a4a..000000000 --- a/examples/cores/boom/configs/default.toml +++ /dev/null @@ -1,5 +0,0 @@ -# Boom-only core configuration - -memdomain = "memdomains/default.toml" -balldomain = "balldomains/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/boom/configs/memdomains/default.toml b/examples/cores/boom/configs/memdomains/default.toml deleted file mode 100644 index 67f5570ea..000000000 --- a/examples/cores/boom/configs/memdomains/default.toml +++ /dev/null @@ -1,31 +0,0 @@ -# Boom-only stub memdomain for bank_params / chip select pairing. - -[bank] -num = 16 -width = 128 -entries = 16 -maskLen = 16 -channel = 1 - -[dma] -nXacts = 1 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 1 - -[tma] -readChannel = 1 -writeChannel = 1 - -[mmio] -enable = false -bankNum = 1 -bankEntries = 1 -bankWidth = 128 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/ffn/arch/src/main/scala/FfnCore.scala b/examples/cores/ffn/arch/src/main/scala/FfnCore.scala deleted file mode 100644 index b2e9273e7..000000000 --- a/examples/cores/ffn/arch/src/main/scala/FfnCore.scala +++ /dev/null @@ -1,19 +0,0 @@ -package examples.cores.ffn - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.ant.LocalExecution -import framework.system.core.{AntEndpoint, CoreConnection} -import framework.system.core.accelerator.{AntAdmission, BuckyballAccelerator} - -@instantiable -class FfnCore(config: GlobalConfig) extends AntEndpoint(config) { - val execution = Instantiate(new LocalExecution(local)) - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball)) - val issuer = Instantiate(new AntAdmission(local, tracking, cpuParams.core.nPMPs, bus)) - CoreConnection.ant(clink, execution, accelerator, issuer, signature) -} - -object FfnCore extends framework.system.core.AntCoreFactory { - def instantiate(b: GlobalConfig): Instance[AntEndpoint] = Instantiate(new FfnCore(b)) -} diff --git a/examples/cores/ffn/compiler/CMakeLists.txt b/examples/cores/ffn/compiler/CMakeLists.txt deleted file mode 100644 index 4f4cd14c7..000000000 --- a/examples/cores/ffn/compiler/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -set(BB_CORE_NAME ffn) -set(BB_CORE_TARGET_PREFIX "") - -set(BB_CORE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..) -get_filename_component(_BB_REPO_ROOT "${BB_CORE_DIR}/../../.." ABSOLUTE) -include_directories(${_BB_REPO_ROOT}/examples/balls) - -add_subdirectory(src) diff --git a/examples/cores/ffn/compiler/src/CMakeLists.txt b/examples/cores/ffn/compiler/src/CMakeLists.txt deleted file mode 100644 index 49dfefaae..000000000 --- a/examples/cores/ffn/compiler/src/CMakeLists.txt +++ /dev/null @@ -1,51 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -message(STATUS "Building FFN compiler Core package") -message(STATUS " Source: ${CMAKE_CURRENT_SOURCE_DIR}") -message(STATUS " Binary: ${CMAKE_CURRENT_BINARY_DIR}") - -get_filename_component(BB_REPO_ROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../.. ABSOLUTE) -set(BB_COMPILER_COMMON_DIR ${BB_REPO_ROOT}/stack/compiler/src) - -if(NOT EXISTS ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Buckyball.td) - message(FATAL_ERROR "Missing Buckyball compiler common dir: ${BB_COMPILER_COMMON_DIR}") -endif() - -include_directories(${BB_COMPILER_COMMON_DIR}) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/Dialect) -include_directories(${CMAKE_CURRENT_BINARY_DIR}/Dialect) -include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/Dialect/Buckyball) - -set(FFN_BALL_COMPILER_DIALECT_DIRS - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball -) - -foreach(FFN_BALL_COMPILER_DIALECT_DIR ${FFN_BALL_COMPILER_DIALECT_DIRS}) - if(NOT EXISTS ${FFN_BALL_COMPILER_DIALECT_DIR}) - message(FATAL_ERROR - "Missing ffn ball compiler dialect dir: ${FFN_BALL_COMPILER_DIALECT_DIR}") - endif() - include_directories(${FFN_BALL_COMPILER_DIALECT_DIR}) -endforeach() - -add_subdirectory(Dialect) -add_subdirectory(Conversion) -add_subdirectory(Target) - -add_custom_target(buckyball-compiler-${BB_CORE_NAME} - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass -) diff --git a/examples/cores/ffn/compiler/src/Conversion/CMakeLists.txt b/examples/cores/ffn/compiler/src/Conversion/CMakeLists.txt deleted file mode 100644 index 9ca69b1a2..000000000 --- a/examples/cores/ffn/compiler/src/Conversion/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_subdirectory(LowerBuckyball) -add_subdirectory(LowerTileToBuckyball) diff --git a/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt b/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt deleted file mode 100644 index eb782edc6..000000000 --- a/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/PhysicalBankState.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/ReportBankUsagePass.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp - ${BB_assign_SOURCES} - ${BB_bank_ssa_SOURCES} - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/QuantizeTensorToBankSSAPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/RuntimeLowering.cpp - LowerBuckyballPass.cpp - LowerBuckyballToBankSSAPass.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - MLIRLinalgDialect -) diff --git a/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp b/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp deleted file mode 100644 index 0a3f1e218..000000000 --- a/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp +++ /dev/null @@ -1,126 +0,0 @@ -//===- LowerBuckyballPass.cpp - FFN Buckyball lowering pass -------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/TypeConverter.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Pass/Pass.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Buckyball/Transform.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -class LowerBuckyballToLLVMPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToLLVMPass) - LowerBuckyballToLLVMPass() = default; - LowerBuckyballToLLVMPass(const LowerBuckyballToLLVMPass &) {} - - StringRef getArgument() const final { return "lower-buckyball"; } - StringRef getDescription() const final { - return "Lower FFN Buckyball dialect ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -class LowerBankSSAToIntrinsicsPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBankSSAToIntrinsicsPass) - LowerBankSSAToIntrinsicsPass() = default; - LowerBankSSAToIntrinsicsPass(const LowerBankSSAToIntrinsicsPass &) {} - - StringRef getArgument() const final { return "lower-bank-ssa-to-intrinsics"; } - StringRef getDescription() const final { - return "Lower FFN bank-SSA and Buckyball ops to intrinsic ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballPass() { - registerTileRuntimePass(); - PassRegistration(); -} - -void mlir::buddy::registerLowerBankSSAToIntrinsicsPass() { - PassRegistration(); -} diff --git a/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp b/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp deleted file mode 100644 index 9450ff9b5..000000000 --- a/examples/cores/ffn/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp +++ /dev/null @@ -1,108 +0,0 @@ -//===- LowerBuckyballToBankSSAPass.cpp - Buckyball bank-SSA lowering -//----------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Utils/StructuredOpsUtils.h" -#include "mlir/IR/AffineMap.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Trace/TraceDialect.h" - -using namespace mlir; - -namespace mlir::buddy { -#define BUCKYBALL_BANK_SSA_HOOK(BALL) \ - void populate##BALL##LowerBuckyballToBankSSAPatterns(RewritePatternSet &); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_BANK_SSA_HOOK -} // namespace mlir::buddy - -namespace { - -class MemTransposeToLinalgPattern - : public OpRewritePattern<::buddy::buckyball::MemTransposeOp> { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(::buddy::buckyball::MemTransposeOp op, - PatternRewriter &rewriter) const override { - auto input = dyn_cast(op.getInput().getType()); - auto output = dyn_cast(op.getOutput().getType()); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 2 || - output.getRank() != 2 || output.getShape()[0] != input.getShape()[1] || - output.getShape()[1] != input.getShape()[0]) - return op.emitError("requires matching static rank-2 transpose memrefs"); - - MLIRContext *context = rewriter.getContext(); - AffineMap identity = AffineMap::getMultiDimIdentityMap(2, context); - AffineMap transpose = AffineMap::get( - 2, 0, {rewriter.getAffineDimExpr(1), rewriter.getAffineDimExpr(0)}, - context); - SmallVector iterators(2, - utils::IteratorType::parallel); - rewriter.create( - op.getLoc(), TypeRange{}, ValueRange{op.getInput()}, - ValueRange{op.getOutput()}, ArrayRef{identity, transpose}, - iterators, [](OpBuilder &builder, Location location, ValueRange args) { - builder.create(location, args[0]); - }); - rewriter.eraseOp(op); - return success(); - } -}; - -} // namespace - -namespace { - -class LowerBuckyballToBankSSAPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToBankSSAPass) - - StringRef getArgument() const final { return "lower-buckyball-to-bank-ssa"; } - StringRef getDescription() const final { - return "Lower Buckyball Buckyball ops to explicit bank-SSA ops."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert<::buddy::trace::BuddyTraceDialect, arith::ArithDialect, - linalg::LinalgDialect, memref::MemRefDialect, - scf::SCFDialect, ::buddy::buckyball::BuckyballDialect>(); - } - - void runOnOperation() override { - RewritePatternSet patterns(&getContext()); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_BANK_SSA_HOOK(BALL) \ - if (ball == #BALL) \ - mlir::buddy::populate##BALL##LowerBuckyballToBankSSAPatterns(patterns); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_BANK_SSA_HOOK - } - mlir::buddy::populateQuantizeTensorToBankSSAPatterns(patterns); - patterns.add(&getContext()); - if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballToBankSSAPass() { - PassRegistration(); -} diff --git a/examples/cores/ffn/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt b/examples/cores/ffn/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt deleted file mode 100644 index c12d60c23..000000000 --- a/examples/cores/ffn/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerTileToBuckyball/QuantizedKernelPatterns.cpp - LowerTileToBuckyball.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Conversion/LowerTileToBuckyball/TileMatMulPatterns.cpp - - LINK_LIBS PUBLIC - BuddyTile - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRLinalgDialect -) diff --git a/examples/cores/ffn/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp b/examples/cores/ffn/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp deleted file mode 100644 index cca771535..000000000 --- a/examples/cores/ffn/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp +++ /dev/null @@ -1,104 +0,0 @@ -#include "Target/BuckyballTargetRegistry.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/DialectConversion.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerTileToBuckyball/LowerTileToBuckyball.h" -#include "Tile/TileDialect.h" -#include "Tile/TileOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -namespace tile = ::buddy::tile; - -namespace mlir::buddy { -#define BUCKYBALL_TILE_HOOK(BALL) \ - void populate##BALL##TileLoweringPatterns(RewritePatternSet &, int64_t, \ - int64_t, int64_t); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_TILE_HOOK -} // namespace mlir::buddy - -namespace { -class TileTransposeLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(tile::TileTransposeOp op, - PatternRewriter &rewriter) const override { - auto inputType = dyn_cast(op.getAMemArray().getType()); - auto outputType = dyn_cast(op.getBMemArray().getType()); - if (!inputType || !outputType || !inputType.hasStaticShape() || - !outputType.hasStaticShape() || inputType.getRank() != 2 || - outputType.getRank() != 2) - return op.emitError("requires static rank-2 input and output memrefs"); - if (outputType.getShape()[0] != inputType.getShape()[1] || - outputType.getShape()[1] != inputType.getShape()[0] || - inputType.getElementType() != outputType.getElementType()) - return op.emitError("requires a matching transposed output memref"); - - rewriter.create(op.getLoc(), op.getAMemArray(), - op.getBMemArray()); - rewriter.eraseOp(op); - return success(); - } -}; - -class LowerTileToBuckyballPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerTileToBuckyballPass) - - StringRef getArgument() const final { return "convert-tile-to-buckyball"; } - StringRef getDescription() const final { - return "Convert Tile operations for the selected Buckyball target."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ConversionTarget target(*context); - target.addLegalDialect<::buddy::buckyball::BuckyballDialect, - memref::MemRefDialect, arith::ArithDialect, - scf::SCFDialect, func::FuncDialect, - linalg::LinalgDialect>(); - target.addIllegalOp(); - - RewritePatternSet patterns(context); - mlir::buddy::populateQuantizedKernelTileLoweringPatterns(patterns); - patterns.add(context); - for (llvm::StringRef ball : targetConfig.balls) { -#define BUCKYBALL_TILE_HOOK(BALL) \ - if (ball == #BALL) \ - mlir::buddy::populate##BALL##TileLoweringPatterns( \ - patterns, targetConfig.bankWidthBits / 8, targetConfig.bankDepth, \ - targetConfig.bankNum); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_TILE_HOOK - } - if (failed(applyPartialConversion(getOperation(), target, - std::move(patterns)))) - signalPassFailure(); - } -}; -} // namespace - -void mlir::buddy::registerLowerTileToBuckyballPass() { - PassRegistration(); -} diff --git a/examples/cores/ffn/compiler/src/Dialect/Buckyball/CMakeLists.txt b/examples/cores/ffn/compiler/src/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 50c0c0f72..000000000 --- a/examples/cores/ffn/compiler/src/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(Buckyball.h.inc -gen-op-decls) -mlir_tablegen(Buckyball.cpp.inc -gen-op-defs) -mlir_tablegen(BuckyballTypes.h.inc -gen-typedef-decls -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballTypes.cpp.inc -gen-typedef-defs -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballDialect.h.inc -gen-dialect-decls -dialect=buckyball) -mlir_tablegen(BuckyballDialect.cpp.inc -gen-dialect-defs -dialect=buckyball) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}MLIRBuckyballIncGen) -add_mlir_doc(FFNBuckyball ${BB_CORE_TARGET_PREFIX}Buckyball Dialects/ - -gen-dialect-doc) - -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(BuckyballConversions.inc -gen-llvmir-conversions) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen) - -add_subdirectory(IR) -add_subdirectory(Transforms) diff --git a/examples/cores/ffn/compiler/src/Dialect/Buckyball/FFNBuckyball.td b/examples/cores/ffn/compiler/src/Dialect/Buckyball/FFNBuckyball.td deleted file mode 100644 index a11c7d6e0..000000000 --- a/examples/cores/ffn/compiler/src/Dialect/Buckyball/FFNBuckyball.td +++ /dev/null @@ -1,26 +0,0 @@ -//===-- FFNBuckyball.td - FFN Buckyball dialect ops -*- tablegen -*-===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#ifndef FFN_BUCKYBALL_OPS -#define FFN_BUCKYBALL_OPS - -include "Buckyball.td" -include "ToInt8Ball.td" -include "Int2FpBall.td" -include "TransposeBall.td" -include "MxmmBall.td" - -#endif // FFN_BUCKYBALL_OPS diff --git a/examples/cores/ffn/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt b/examples/cores/ffn/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt deleted file mode 100644 index 42c7f8807..000000000 --- a/examples/cores/ffn/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/IR/BuckyballDialect.cpp - ${BB_TARGET_REGISTRY_SOURCE} - ${BB_BALL_IR_SOURCES} - - LINK_LIBS PUBLIC - MLIRIR - MLIRLLVMDialect - MLIRBufferizationDialect - MLIRMemRefDialect -) - -target_include_directories(${BB_CORE_TARGET_PREFIX}BuddyBuckyball PUBLIC - ${BB_COMPILER_COMMON_DIR} - ${BB_COMPILER_COMMON_DIR}/Dialect - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball - ${CMAKE_CURRENT_BINARY_DIR}/.. - ${CMAKE_CURRENT_BINARY_DIR}/../../.. -) diff --git a/examples/cores/ffn/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt b/examples/cores/ffn/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt deleted file mode 100644 index 0d2cecf2e..000000000 --- a/examples/cores/ffn/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - LegalizeForLLVMExport.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRIR - MLIRLLVMCommonConversion - MLIRLLVMDialect -) diff --git a/examples/cores/ffn/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/cores/ffn/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 0b1a5b224..000000000 --- a/examples/cores/ffn/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "Buckyball/Transform.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball::legalize; - -namespace mlir::buddy::buckyball { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - void populate##BALL##LegalizeForLLVMExportPatterns( \ - LLVMTypeConverter &, RewritePatternSet &, bool, int64_t); \ - void configure##BALL##LegalizeForExportTarget(LLVMConversionTarget &, bool); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK -} // namespace mlir::buddy::buckyball - -void mlir::populateBuckyballLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, - int64_t bankWidthBytes, int64_t bankDepth, int64_t bankNum, - bool includeFuncOperandForwarding, bool stable) { - populateBaseLegalizeForLLVMExportPatterns(converter, patterns, - includeFuncOperandForwarding); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - if (ball == #BALL) \ - buddy::buckyball::populate##BALL##LegalizeForLLVMExportPatterns( \ - converter, patterns, stable, bankDepth); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK - } - (void)bankWidthBytes; - (void)bankNum; -} - -void mlir::configureBuckyballLegalizeForExportTarget( - LLVMConversionTarget &target, bool stable) { - configureBaseLegalizeForExportTarget(target); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - if (ball == #BALL) \ - buddy::buckyball::configure##BALL##LegalizeForExportTarget(target, stable); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK - } -} diff --git a/examples/cores/ffn/compiler/src/Dialect/CMakeLists.txt b/examples/cores/ffn/compiler/src/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/ffn/compiler/src/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/ffn/compiler/src/Target/CMakeLists.txt b/examples/cores/ffn/compiler/src/Target/CMakeLists.txt deleted file mode 100644 index 39d31dc9b..000000000 --- a/examples/cores/ffn/compiler/src/Target/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(LLVMIR) diff --git a/examples/cores/ffn/compiler/src/Target/LLVMIR/CMakeLists.txt b/examples/cores/ffn/compiler/src/Target/LLVMIR/CMakeLists.txt deleted file mode 100644 index 0ca0f41c5..000000000 --- a/examples/cores/ffn/compiler/src/Target/LLVMIR/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Dialect) diff --git a/examples/cores/ffn/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt b/examples/cores/ffn/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 06b99171b..000000000 --- a/examples/cores/ffn/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_mlir_translation_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_COMPILER_COMMON_DIR}/Target/LLVMIR/Dialect/Buckyball/BuckyballToLLVMIRTranslation.cpp - - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen - intrinsics_gen - - LINK_COMPONENTS - Core - - LINK_LIBS PUBLIC - MLIRIR - MLIRSupport - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball -) diff --git a/examples/cores/ffn/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt b/examples/cores/ffn/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/ffn/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/ffn/configs/balldomains/default.toml b/examples/cores/ffn/configs/balldomains/default.toml deleted file mode 100644 index b64240389..000000000 --- a/examples/cores/ffn/configs/balldomains/default.toml +++ /dev/null @@ -1,23 +0,0 @@ -# FFN Ball Domain Configuration - -ballNum = 5 - -ballIdMappings = [ - { ballId = 0, ballName = "MxmmBall", ballClass = "examples.balls.mxmm.MxmmBall", config = "../../../../balls/mxmm/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 1, ballName = "ToInt8Ball", ballClass = "examples.balls.toint8.ToInt8Ball", config = "../../../../balls/toint8/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 2, ballName = "Int2FpBall", ballClass = "examples.balls.int2fp.Int2FpBall", config = "../../../../balls/int2fp/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 3, ballName = "TransposeBall", ballClass = "examples.balls.transpose.TransposeBall", config = "../../../../balls/transpose/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 4, ballName = "MxquantBall", ballClass = "examples.balls.mxquant.MxquantBall", config = "../../../../balls/mxquant/configs/default.toml", inBW = 1, outBW = 1 } -] - -ballISA = [ - { mnemonic = "MXMM_MXFP8", funct7 = 71, bid = 0 }, - { mnemonic = "MXMM_MXFP8_WINDOW", funct7 = 75, bid = 0 }, - { mnemonic = "MXMM_FMA32", funct7 = 72, bid = 0 }, - { mnemonic = "MXMM_F32", funct7 = 73, bid = 0 }, - { mnemonic = "QUANT_F32_TO_I8", funct7 = 51, bid = 1 }, - { mnemonic = "QUANT_I32_TO_I8", funct7 = 67, bid = 1 }, - { mnemonic = "INT32_TO_FP32", funct7 = 68, bid = 2 }, - { mnemonic = "TRANSPOSE", funct7 = 49, bid = 3 }, - { mnemonic = "MXQUANT", funct7 = 54, bid = 4 } -] diff --git a/examples/cores/ffn/configs/cpu/default.toml b/examples/cores/ffn/configs/cpu/default.toml deleted file mode 100644 index 9cf6bf076..000000000 --- a/examples/cores/ffn/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "ant" -config = "../../../ant/configs/cpu/default.toml" diff --git a/examples/cores/ffn/configs/default.toml b/examples/cores/ffn/configs/default.toml deleted file mode 100644 index 60331493c..000000000 --- a/examples/cores/ffn/configs/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -factory = "examples.cores.ffn.FfnCore" -# FFN Core Configuration (default) - -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/default.toml" -rvv = "rvv/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/ffn/configs/frontends/default.toml b/examples/cores/ffn/configs/frontends/default.toml deleted file mode 100644 index 418994d6b..000000000 --- a/examples/cores/ffn/configs/frontends/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -robEntries = 16 -rsOutOfOrderResponse = true -bankIdLen = 10 -vbankIdUpperBound = 31 -sharedBankIdBase = 32 -iterLen = 34 -subRobEnable = false -subRobDepth = 64 diff --git a/examples/cores/ffn/configs/memdomains/default.toml b/examples/cores/ffn/configs/memdomains/default.toml deleted file mode 100644 index 1b39694e8..000000000 --- a/examples/cores/ffn/configs/memdomains/default.toml +++ /dev/null @@ -1,32 +0,0 @@ -# FFN Core Memory Domain Configuration (default) -# Private (per-core) memory parameters. Shared memory lives in the tile TOML. - -[bank] -num = 32 -width = 128 -entries = 2048 -maskLen = 16 -channel = 7 - -[dma] -nXacts = 8 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 4 - -[tma] -readChannel = 4 -writeChannel = 4 - -[mmio] -enable = false -bankNum = 16 -bankEntries = 64 -bankWidth = 128 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/goban/arch/src/main/scala/AntGobanCore.scala b/examples/cores/goban/arch/src/main/scala/AntGobanCore.scala deleted file mode 100644 index 6d521027f..000000000 --- a/examples/cores/goban/arch/src/main/scala/AntGobanCore.scala +++ /dev/null @@ -1,19 +0,0 @@ -package examples.cores.goban - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.ant.LocalExecution -import framework.system.core.{AntEndpoint, CoreConnection} -import framework.system.core.accelerator.{AntAdmission, BuckyballAccelerator} - -@instantiable -class AntGobanCore(config: GlobalConfig) extends AntEndpoint(config) { - val execution = Instantiate(new LocalExecution(local)) - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball)) - val issuer = Instantiate(new AntAdmission(local, tracking, cpuParams.core.nPMPs, bus)) - CoreConnection.ant(clink, execution, accelerator, issuer, signature) -} - -object AntGobanCore extends framework.system.core.AntCoreFactory { - def instantiate(b: GlobalConfig): Instance[AntEndpoint] = Instantiate(new AntGobanCore(b)) -} diff --git a/examples/cores/goban/arch/src/main/scala/GobanCore.scala b/examples/cores/goban/arch/src/main/scala/GobanCore.scala deleted file mode 100644 index b2de78793..000000000 --- a/examples/cores/goban/arch/src/main/scala/GobanCore.scala +++ /dev/null @@ -1,20 +0,0 @@ -package examples.cores.goban - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.system.core.{CoreConnection, RocketEndpoint} -import hier.core.rocket.Core -import framework.system.core.accelerator.BuckyballAccelerator - -@instantiable -class GobanCore(b: GlobalConfig) extends RocketEndpoint(b) { - require(p.buckyball.nonEmpty) - val cpu = Instantiate(new Core(p.data, p.instruction, p.regions, Some(p.commands))(p.cpu)) - clink.cpu <> cpu.clink - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball.get)) - CoreConnection.accelerator(clink.accelerator.get, accelerator) -} - -object GobanCore extends framework.system.core.RocketCoreFactory { - def instantiate(b: GlobalConfig): Instance[RocketEndpoint] = Instantiate(new GobanCore(b)) -} diff --git a/examples/cores/goban/compiler/CMakeLists.txt b/examples/cores/goban/compiler/CMakeLists.txt deleted file mode 100644 index cf5f19d4c..000000000 --- a/examples/cores/goban/compiler/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -set(BB_CORE_NAME goban) -set(BB_CORE_TARGET_PREFIX "") - -set(BB_CORE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..) -get_filename_component(_BB_REPO_ROOT "${BB_CORE_DIR}/../../.." ABSOLUTE) -include_directories(${_BB_REPO_ROOT}/examples/balls) - -add_subdirectory(src) diff --git a/examples/cores/goban/compiler/src/CMakeLists.txt b/examples/cores/goban/compiler/src/CMakeLists.txt deleted file mode 100644 index 561efedb1..000000000 --- a/examples/cores/goban/compiler/src/CMakeLists.txt +++ /dev/null @@ -1,51 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -message(STATUS "Building Goban compiler Core package") -message(STATUS " Source: ${CMAKE_CURRENT_SOURCE_DIR}") -message(STATUS " Binary: ${CMAKE_CURRENT_BINARY_DIR}") - -get_filename_component(BB_REPO_ROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../.. ABSOLUTE) -set(BB_COMPILER_COMMON_DIR ${BB_REPO_ROOT}/stack/compiler/src) - -if(NOT EXISTS ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Buckyball.td) - message(FATAL_ERROR "Missing Buckyball compiler common dir: ${BB_COMPILER_COMMON_DIR}") -endif() - -include_directories(${BB_COMPILER_COMMON_DIR}) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/Dialect) -include_directories(${CMAKE_CURRENT_BINARY_DIR}/Dialect) -include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/Dialect/Buckyball) - -set(GOBAN_BALL_COMPILER_DIALECT_DIRS - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball -) - -foreach(GOBAN_BALL_COMPILER_DIALECT_DIR ${GOBAN_BALL_COMPILER_DIALECT_DIRS}) - if(NOT EXISTS ${GOBAN_BALL_COMPILER_DIALECT_DIR}) - message(FATAL_ERROR - "Missing goban ball compiler dialect dir: ${GOBAN_BALL_COMPILER_DIALECT_DIR}") - endif() - include_directories(${GOBAN_BALL_COMPILER_DIALECT_DIR}) -endforeach() - -add_subdirectory(Dialect) -add_subdirectory(Conversion) -add_subdirectory(Target) - -add_custom_target(buckyball-compiler-${BB_CORE_NAME} - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass -) diff --git a/examples/cores/goban/compiler/src/Conversion/CMakeLists.txt b/examples/cores/goban/compiler/src/Conversion/CMakeLists.txt deleted file mode 100644 index 9ca69b1a2..000000000 --- a/examples/cores/goban/compiler/src/Conversion/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_subdirectory(LowerBuckyball) -add_subdirectory(LowerTileToBuckyball) diff --git a/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt b/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt deleted file mode 100644 index 12199e0c5..000000000 --- a/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/PhysicalBankState.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/ReportBankUsagePass.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp - ${BB_assign_SOURCES} - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/QuantizeTensorToBankSSAPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/RuntimeLowering.cpp - LowerBuckyballPass.cpp - LowerBuckyballToBankSSAPass.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - MLIRLinalgDialect -) diff --git a/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp b/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp deleted file mode 100644 index eeb8ecc1a..000000000 --- a/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp +++ /dev/null @@ -1,126 +0,0 @@ -//===- LowerBuckyballPass.cpp - Goban Buckyball lowering pass -------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/TypeConverter.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Pass/Pass.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Buckyball/Transform.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -class LowerBuckyballToLLVMPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToLLVMPass) - LowerBuckyballToLLVMPass() = default; - LowerBuckyballToLLVMPass(const LowerBuckyballToLLVMPass &) {} - - StringRef getArgument() const final { return "lower-buckyball"; } - StringRef getDescription() const final { - return "Lower Goban Buckyball dialect ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -class LowerBankSSAToIntrinsicsPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBankSSAToIntrinsicsPass) - LowerBankSSAToIntrinsicsPass() = default; - LowerBankSSAToIntrinsicsPass(const LowerBankSSAToIntrinsicsPass &) {} - - StringRef getArgument() const final { return "lower-bank-ssa-to-intrinsics"; } - StringRef getDescription() const final { - return "Lower Goban bank-SSA and Buckyball ops to intrinsic ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballPass() { - registerTileRuntimePass(); - PassRegistration(); -} - -void mlir::buddy::registerLowerBankSSAToIntrinsicsPass() { - PassRegistration(); -} diff --git a/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp b/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp deleted file mode 100644 index ed67fb60b..000000000 --- a/examples/cores/goban/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp +++ /dev/null @@ -1,93 +0,0 @@ -//===- LowerBuckyballToBankSSAPass.cpp - Goban bank-SSA lowering ----------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Utils/StructuredOpsUtils.h" -#include "mlir/IR/AffineMap.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Trace/TraceDialect.h" - -using namespace mlir; - -namespace { - -class MemTransposeToLinalgPattern - : public OpRewritePattern<::buddy::buckyball::MemTransposeOp> { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(::buddy::buckyball::MemTransposeOp op, - PatternRewriter &rewriter) const override { - auto input = dyn_cast(op.getInput().getType()); - auto output = dyn_cast(op.getOutput().getType()); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 2 || - output.getRank() != 2 || output.getShape()[0] != input.getShape()[1] || - output.getShape()[1] != input.getShape()[0]) - return op.emitError("requires matching static rank-2 transpose memrefs"); - - MLIRContext *context = rewriter.getContext(); - AffineMap identity = AffineMap::getMultiDimIdentityMap(2, context); - AffineMap transpose = AffineMap::get( - 2, 0, {rewriter.getAffineDimExpr(1), rewriter.getAffineDimExpr(0)}, - context); - SmallVector iterators(2, - utils::IteratorType::parallel); - rewriter.create( - op.getLoc(), TypeRange{}, ValueRange{op.getInput()}, - ValueRange{op.getOutput()}, ArrayRef{identity, transpose}, - iterators, [](OpBuilder &builder, Location location, ValueRange args) { - builder.create(location, args[0]); - }); - rewriter.eraseOp(op); - return success(); - } -}; - -} // namespace - -namespace { - -class LowerBuckyballToBankSSAPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToBankSSAPass) - - StringRef getArgument() const final { return "lower-buckyball-to-bank-ssa"; } - StringRef getDescription() const final { - return "Lower Goban Buckyball ops to explicit bank-SSA ops."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert<::buddy::trace::BuddyTraceDialect, arith::ArithDialect, - linalg::LinalgDialect, memref::MemRefDialect, - scf::SCFDialect, ::buddy::buckyball::BuckyballDialect>(); - } - - void runOnOperation() override { - RewritePatternSet patterns(&getContext()); - mlir::buddy::populateMatmulRegionToBankSSAPatterns(patterns, false, 0, -1); - mlir::buddy::populateQuantizeTensorToBankSSAPatterns(patterns); - patterns.add(&getContext()); - if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballToBankSSAPass() { - PassRegistration(); -} diff --git a/examples/cores/goban/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt b/examples/cores/goban/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt deleted file mode 100644 index c12d60c23..000000000 --- a/examples/cores/goban/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerTileToBuckyball/QuantizedKernelPatterns.cpp - LowerTileToBuckyball.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Conversion/LowerTileToBuckyball/TileMatMulPatterns.cpp - - LINK_LIBS PUBLIC - BuddyTile - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRLinalgDialect -) diff --git a/examples/cores/goban/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp b/examples/cores/goban/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp deleted file mode 100644 index f17989c35..000000000 --- a/examples/cores/goban/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp +++ /dev/null @@ -1,115 +0,0 @@ -//===- LowerTileToBuckyball.cpp - Goban tile pass registration ------------===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/DialectConversion.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerTileToBuckyball/LowerTileToBuckyball.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Tile/TileDialect.h" -#include "Tile/TileOps.h" -#include "Tile/Transform.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -namespace tile = ::buddy::tile; - -namespace mlir::buddy { -void populateSMatMulBallTileLoweringPatterns(RewritePatternSet &patterns, - int64_t bankWidthBytes, - int64_t bankDepth, - int64_t bankNum); -} // namespace mlir::buddy - -namespace { -class TileTransposeLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(tile::TileTransposeOp op, - PatternRewriter &rewriter) const override { - auto inputType = dyn_cast(op.getAMemArray().getType()); - auto outputType = dyn_cast(op.getBMemArray().getType()); - if (!inputType || !outputType || !inputType.hasStaticShape() || - !outputType.hasStaticShape() || inputType.getRank() != 2 || - outputType.getRank() != 2) - return op.emitError("requires static rank-2 input and output memrefs"); - if (outputType.getShape()[0] != inputType.getShape()[1] || - outputType.getShape()[1] != inputType.getShape()[0] || - inputType.getElementType() != outputType.getElementType()) - return op.emitError("requires a matching transposed output memref"); - - rewriter.create(op.getLoc(), op.getAMemArray(), - op.getBMemArray()); - rewriter.eraseOp(op); - return success(); - } -}; - -class LowerTileToBuckyballPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerTileToBuckyballPass) - StringRef getArgument() const final { return "convert-tile-to-buckyball"; } - StringRef getDescription() const final { - return "Convert Goban Tile operations to Buckyball operations"; - } - LowerTileToBuckyballPass() = default; - LowerTileToBuckyballPass(const LowerTileToBuckyballPass &) {} - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ConversionTarget target(*context); - target.addLegalDialect<::buddy::buckyball::BuckyballDialect, - memref::MemRefDialect, arith::ArithDialect, - scf::SCFDialect, func::FuncDialect, - linalg::LinalgDialect>(); - target.addIllegalOp(); - target.addIllegalOp(); - - RewritePatternSet patterns(context); - mlir::buddy::populateSMatMulBallTileLoweringPatterns( - patterns, targetConfig.bankWidthBits / 8, targetConfig.bankDepth, - targetConfig.bankNum); - mlir::buddy::populateQuantizedKernelTileLoweringPatterns(patterns); - patterns.add(context); - if (failed(applyPartialConversion(getOperation(), target, - std::move(patterns)))) - signalPassFailure(); - } -}; -} // namespace - -void mlir::buddy::registerLowerTileToBuckyballPass() { - PassRegistration(); -} diff --git a/examples/cores/goban/compiler/src/Dialect/Buckyball/CMakeLists.txt b/examples/cores/goban/compiler/src/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 004ad362c..000000000 --- a/examples/cores/goban/compiler/src/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(Buckyball.h.inc -gen-op-decls) -mlir_tablegen(Buckyball.cpp.inc -gen-op-defs) -mlir_tablegen(BuckyballTypes.h.inc -gen-typedef-decls -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballTypes.cpp.inc -gen-typedef-defs -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballDialect.h.inc -gen-dialect-decls -dialect=buckyball) -mlir_tablegen(BuckyballDialect.cpp.inc -gen-dialect-defs -dialect=buckyball) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}MLIRBuckyballIncGen) -add_mlir_doc(GobanBuckyball ${BB_CORE_TARGET_PREFIX}Buckyball Dialects/ - -gen-dialect-doc) - -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(BuckyballConversions.inc -gen-llvmir-conversions) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen) - -add_subdirectory(IR) -add_subdirectory(Transforms) diff --git a/examples/cores/goban/compiler/src/Dialect/Buckyball/GobanBuckyball.td b/examples/cores/goban/compiler/src/Dialect/Buckyball/GobanBuckyball.td deleted file mode 100644 index bc56af64c..000000000 --- a/examples/cores/goban/compiler/src/Dialect/Buckyball/GobanBuckyball.td +++ /dev/null @@ -1,26 +0,0 @@ -//===-- GobanBuckyball.td - Goban Buckyball dialect ops -*- tablegen -*-===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#ifndef GOBAN_BUCKYBALL_OPS -#define GOBAN_BUCKYBALL_OPS - -include "Buckyball.td" -include "ToInt8Ball.td" -include "Int2FpBall.td" -include "TransposeBall.td" -include "SMatMulBall.td" - -#endif // GOBAN_BUCKYBALL_OPS diff --git a/examples/cores/goban/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt b/examples/cores/goban/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt deleted file mode 100644 index 42c7f8807..000000000 --- a/examples/cores/goban/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/IR/BuckyballDialect.cpp - ${BB_TARGET_REGISTRY_SOURCE} - ${BB_BALL_IR_SOURCES} - - LINK_LIBS PUBLIC - MLIRIR - MLIRLLVMDialect - MLIRBufferizationDialect - MLIRMemRefDialect -) - -target_include_directories(${BB_CORE_TARGET_PREFIX}BuddyBuckyball PUBLIC - ${BB_COMPILER_COMMON_DIR} - ${BB_COMPILER_COMMON_DIR}/Dialect - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball - ${CMAKE_CURRENT_BINARY_DIR}/.. - ${CMAKE_CURRENT_BINARY_DIR}/../../.. -) diff --git a/examples/cores/goban/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt b/examples/cores/goban/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt deleted file mode 100644 index 0d2cecf2e..000000000 --- a/examples/cores/goban/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - LegalizeForLLVMExport.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRIR - MLIRLLVMCommonConversion - MLIRLLVMDialect -) diff --git a/examples/cores/goban/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/cores/goban/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 603fd0520..000000000 --- a/examples/cores/goban/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,73 +0,0 @@ -//===- LegalizeForLLVMExport.cpp - Goban Buckyball LLVM lowering ----------===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/Transform.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" - -using namespace mlir; -using namespace buddy::buckyball::legalize; - -namespace mlir::buddy::buckyball { -void populateTransposeBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, - bool, int64_t); -void configureTransposeBallLegalizeForExportTarget(LLVMConversionTarget &, - bool); -void populateSMatMulBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureSMatMulBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -void populateToInt8BallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureToInt8BallLegalizeForExportTarget(LLVMConversionTarget &, bool); -void populateInt2FpBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureInt2FpBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace mlir::buddy::buckyball - -void mlir::populateBuckyballLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, - int64_t bankWidthBytes, int64_t bankDepth, int64_t bankNum, - bool includeFuncOperandForwarding, bool stable) { - (void)bankWidthBytes; - (void)bankNum; - - populateBaseLegalizeForLLVMExportPatterns(converter, patterns, - includeFuncOperandForwarding); - mlir::buddy::buckyball::populateToInt8BallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateInt2FpBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateTransposeBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateSMatMulBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); -} - -void mlir::configureBuckyballLegalizeForExportTarget( - LLVMConversionTarget &target, bool stable) { - configureBaseLegalizeForExportTarget(target); - mlir::buddy::buckyball::configureToInt8BallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureInt2FpBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureTransposeBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureSMatMulBallLegalizeForExportTarget(target, - stable); -} diff --git a/examples/cores/goban/compiler/src/Dialect/CMakeLists.txt b/examples/cores/goban/compiler/src/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/goban/compiler/src/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/goban/compiler/src/Target/CMakeLists.txt b/examples/cores/goban/compiler/src/Target/CMakeLists.txt deleted file mode 100644 index 39d31dc9b..000000000 --- a/examples/cores/goban/compiler/src/Target/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(LLVMIR) diff --git a/examples/cores/goban/compiler/src/Target/LLVMIR/CMakeLists.txt b/examples/cores/goban/compiler/src/Target/LLVMIR/CMakeLists.txt deleted file mode 100644 index 0ca0f41c5..000000000 --- a/examples/cores/goban/compiler/src/Target/LLVMIR/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Dialect) diff --git a/examples/cores/goban/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt b/examples/cores/goban/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 06b99171b..000000000 --- a/examples/cores/goban/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_mlir_translation_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_COMPILER_COMMON_DIR}/Target/LLVMIR/Dialect/Buckyball/BuckyballToLLVMIRTranslation.cpp - - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen - intrinsics_gen - - LINK_COMPONENTS - Core - - LINK_LIBS PUBLIC - MLIRIR - MLIRSupport - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball -) diff --git a/examples/cores/goban/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt b/examples/cores/goban/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/goban/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/goban/configs/ant.toml b/examples/cores/goban/configs/ant.toml deleted file mode 100644 index 93cfc7804..000000000 --- a/examples/cores/goban/configs/ant.toml +++ /dev/null @@ -1,6 +0,0 @@ -factory = "examples.cores.goban.AntGobanCore" -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/default.toml" -rvv = "rvv/default.toml" -cpu = "cpu/ant.toml" diff --git a/examples/cores/goban/configs/balldomains/default.toml b/examples/cores/goban/configs/balldomains/default.toml deleted file mode 100644 index fb41a02ab..000000000 --- a/examples/cores/goban/configs/balldomains/default.toml +++ /dev/null @@ -1,19 +0,0 @@ -# Goban Ball Domain Configuration - -ballNum = 4 - -ballIdMappings = [ - { ballId = 0, ballName = "SMatMulBall", ballClass = "examples.balls.smatmul.SMatMulBall", config = "../../../../balls/smatmul/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 1, ballName = "ToInt8Ball", ballClass = "examples.balls.toint8.ToInt8Ball", config = "../../../../balls/toint8/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 2, ballName = "Int2FpBall", ballClass = "examples.balls.int2fp.Int2FpBall", config = "../../../../balls/int2fp/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 3, ballName = "TransposeBall", ballClass = "examples.balls.transpose.TransposeBall", config = "../../../../balls/transpose/configs/default.toml", inBW = 1, outBW = 1 } -] - -ballISA = [ - { mnemonic = "SMATMUL_OS", funct7 = 65, bid = 0 }, - { mnemonic = "SMATMUL_BIAS", funct7 = 17, bid = 0 }, - { mnemonic = "QUANT_F32_TO_I8", funct7 = 51, bid = 1 }, - { mnemonic = "QUANT_I32_TO_I8", funct7 = 67, bid = 1 }, - { mnemonic = "INT32_TO_FP32", funct7 = 68, bid = 2 }, - { mnemonic = "TRANSPOSE", funct7 = 49, bid = 3 } -] diff --git a/examples/cores/goban/configs/cpu/ant.toml b/examples/cores/goban/configs/cpu/ant.toml deleted file mode 100644 index 9cf6bf076..000000000 --- a/examples/cores/goban/configs/cpu/ant.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "ant" -config = "../../../ant/configs/cpu/default.toml" diff --git a/examples/cores/goban/configs/cpu/default.toml b/examples/cores/goban/configs/cpu/default.toml deleted file mode 100644 index 1bf2f46e8..000000000 --- a/examples/cores/goban/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "rocket" -config = "../../../rocket/configs/cpu/rocket/default.toml" diff --git a/examples/cores/goban/configs/default.toml b/examples/cores/goban/configs/default.toml deleted file mode 100644 index 672adbd91..000000000 --- a/examples/cores/goban/configs/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -factory = "examples.cores.goban.GobanCore" -# Goban Core Configuration (default) - -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/default.toml" -rvv = "rvv/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/goban/configs/frontends/default.toml b/examples/cores/goban/configs/frontends/default.toml deleted file mode 100644 index 41cd684cc..000000000 --- a/examples/cores/goban/configs/frontends/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -robEntries = 16 -rsOutOfOrderResponse = true -bankIdLen = 10 -vbankIdUpperBound = 15 -sharedBankIdBase = 32 -iterLen = 34 -subRobEnable = false -subRobDepth = 64 diff --git a/examples/cores/goban/configs/memdomains/default.toml b/examples/cores/goban/configs/memdomains/default.toml deleted file mode 100644 index ef9a70dbc..000000000 --- a/examples/cores/goban/configs/memdomains/default.toml +++ /dev/null @@ -1,32 +0,0 @@ -# Goban Core Memory Domain Configuration (default) -# Private (per-core) memory parameters. Shared memory lives in the tile TOML. - -[bank] -num = 20 -width = 128 -entries = 1024 -maskLen = 16 -channel = 7 - -[dma] -nXacts = 8 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 4 - -[tma] -readChannel = 4 -writeChannel = 4 - -[mmio] -enable = false -bankNum = 16 -bankEntries = 64 -bankWidth = 128 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/goban/workloads/ctests/CMakeLists.txt b/examples/cores/goban/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 9d2130123..000000000 --- a/examples/cores/goban/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_subdirectory(transpose_smatmul_quant_pipeline) -set(BUCKYBALL_CTEST_TARGET ${BUCKYBALL_CORE_WORKLOAD_TARGET}) -file(RELATIVE_PATH RVV_KERNEL_SOURCE ${CMAKE_CURRENT_SOURCE_DIR} - ${WORKLOAD_SRC_DIR}/CTest/rvv/rvv_kernel.c) -add_buckyball_ctest(${RVV_KERNEL_SOURCE}) -buckyball_ctest_name(RVV_KERNEL_TEST ${BUCKYBALL_CTEST_TARGET} rvv_kernel) -add_dependencies(core-ctests-build ${RVV_KERNEL_TEST}-ctest-build) -add_subdirectory(rvv) diff --git a/examples/cores/goban/workloads/ctests/rvv/CMakeLists.txt b/examples/cores/goban/workloads/ctests/rvv/CMakeLists.txt deleted file mode 100644 index dd57c1e63..000000000 --- a/examples/cores/goban/workloads/ctests/rvv/CMakeLists.txt +++ /dev/null @@ -1,67 +0,0 @@ -set(KERNEL_DIR "${WORKLOAD_LIB_DIR}/bbsw/kernels/rvv") -set(IMAGE_SOURCE "${CMAKE_CURRENT_BINARY_DIR}/images.cpp") -set(IMAGE_HEADER "${CMAKE_CURRENT_BINARY_DIR}/images.h") -file(READ "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h" PARAMS) -foreach(PARAM BANK_NUM BANK_WIDTH BANK_LINES) - string(REGEX MATCH "#define ${PARAM} ([0-9]+)" MATCH "${PARAMS}") - set(${PARAM} "${CMAKE_MATCH_1}") -endforeach() -if(BANK_NUM LESS 4 OR NOT BANK_WIDTH EQUAL 128 OR BANK_LINES LESS 32) - message(FATAL_ERROR - "RVV kernel CTests require 4 banks, 128-bit rows and 512 bytes per bank; " - "${BUCKYBALL_CTEST_TARGET} has ${BANK_NUM} banks, ${BANK_WIDTH}-bit rows, ${BANK_LINES} lines") -endif() -add_custom_command( - OUTPUT "${IMAGE_SOURCE}" "${IMAGE_HEADER}" - COMMAND "${Python3_EXECUTABLE}" "${KERNEL_DIR}/build_images.py" - --output "${CMAKE_CURRENT_BINARY_DIR}" - DEPENDS "${KERNEL_DIR}/build_images.py" "${KERNEL_DIR}/kernel.ld" - "${KERNEL_DIR}/kernels.h" "${KERNEL_DIR}/silu.cpp" - "${KERNEL_DIR}/snake.cpp" "${KERNEL_DIR}/quant.cpp" - "${KERNEL_DIR}/matmul.cpp" - "${KERNEL_DIR}/math/exp.cpp" "${KERNEL_DIR}/math/sin.cpp" - "${KERNEL_DIR}/math/math.h" "${KERNEL_DIR}/math/constants.h" - VERBATIM) -buckyball_ctest_name(IMAGE_TARGET ${BUCKYBALL_CTEST_TARGET} rvv_images) -add_custom_target(${IMAGE_TARGET} DEPENDS "${IMAGE_SOURCE}" "${IMAGE_HEADER}") - -foreach(KERNEL silu swiglu snake mxfp8_quant matmul) - buckyball_enforce_ctest_line_limit("${KERNEL}.cpp") - buckyball_ctest_name(NAME ${BUCKYBALL_CTEST_TARGET} rvv_${KERNEL}) - add_buckyball_linux_ctest(${NAME} ${CMAKE_CURRENT_SOURCE_DIR} ${KERNEL}.cpp) - if(KERNEL STREQUAL "matmul") - set(DISPATCH_SOURCE "${KERNEL_DIR}/matmul_dispatch.cpp") - elseif(KERNEL STREQUAL "mxfp8_quant") - set(DISPATCH_SOURCE "${KERNEL_DIR}/quant_dispatch.cpp") - else() - set(DISPATCH_SOURCE "${KERNEL_DIR}/dispatch.cpp") - endif() - target_sources(${NAME}-linux PRIVATE "${IMAGE_SOURCE}" "${DISPATCH_SOURCE}") - target_include_directories(${NAME}-linux PRIVATE "${CMAKE_CURRENT_BINARY_DIR}" - "${BUCKYBALL_REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine") - target_compile_options(${NAME}-linux PRIVATE -O2) - set_target_properties(${NAME}-linux PROPERTIES LINKER_LANGUAGE CXX) - target_link_libraries(${NAME}-linux PRIVATE m) - add_dependencies(${NAME}-linux ${IMAGE_TARGET}) - buckyball_ctest_deps(DEPS ${CMAKE_CURRENT_SOURCE_DIR} ${KERNEL}.cpp) - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${NAME}-baremetal" - COMMAND ${RISCV_GNU_TOOLCHAIN}/bin/riscv64-unknown-elf-g++ - ${BUCKYBALL_CTEST_C_FLAGS} - -I${BUCKYBALL_REPO_ROOT}/stack/runtime/include -I${WORKLOAD_LIB_DIR} - -I${BUCKYBALL_ISA_DIR}/chip -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} - -I${CMAKE_CURRENT_BINARY_DIR} - -I${BUCKYBALL_REPO_ROOT}/stack/compiler/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine - -o "${NAME}-baremetal" ${BBSW_BAREMETAL_DIR}/crt0.S - -x c ${WORKLOAD_LIB_DIR}/buckyball.c ${BBHW_MEM_C} - ${BUCKYBALL_CHIP_COMMON_SOURCES} - -x c++ "${CMAKE_CURRENT_SOURCE_DIR}/${KERNEL}.cpp" "${IMAGE_SOURCE}" "${DISPATCH_SOURCE}" -lm - DEPENDS ${DEPS} "${IMAGE_SOURCE}" "${IMAGE_HEADER}" "${DISPATCH_SOURCE}" - ${BBSW_BAREMETAL_DIR}/crt0.S - WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" - VERBATIM) - add_custom_target(${NAME}-ctest-build - DEPENDS ${NAME}-linux-build "${CMAKE_CURRENT_BINARY_DIR}/${NAME}-baremetal") - add_dependencies(${NAME}-ctest-build ${IMAGE_TARGET}) - add_dependencies(core-ctests-build ${NAME}-ctest-build) -endforeach() diff --git a/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt b/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt deleted file mode 100644 index 9676371fa..000000000 --- a/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_buckyball_core_workload(transpose_smatmul_quant_pipeline SOURCES - transpose_smatmul_quant_pipeline-main.cpp - goban.c) diff --git a/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c b/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c deleted file mode 100644 index fc4bfe290..000000000 --- a/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c +++ /dev/null @@ -1,53 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include -#include -#include -#include - -#if BANK_WIDTH != 128 || BANK_LINES < 64 -#error "Goban core pipeline requires 128-bit banks with at least 64 rows" -#endif - -enum { DIM = 16 }; -static int8_t input[BB_CORES_PER_TILE][DIM * DIM] __attribute__((aligned(64))); -static int8_t weight[BB_CORES_PER_TILE][DIM * DIM] __attribute__((aligned(64))); -static int32_t bias[BB_CORES_PER_TILE][DIM] __attribute__((aligned(64))); -static float scale[BB_CORES_PER_TILE][DIM] __attribute__((aligned(64))); -static int8_t output[BB_CORES_PER_TILE][DIM * DIM] __attribute__((aligned(64))); - -#ifdef __cplusplus -extern "C" -#endif - int goban_core(core_id_t id) { - int slot = id.core; - for (int row = 0; row < DIM; ++row) - for (int col = 0; col < DIM; ++col) { - input[slot][row * DIM + col] = (row * 3 + col * 5) % 9 - 4; - weight[slot][row * DIM + col] = (row + 2 * col) % 5 - 2; - } - for (int col = 0; col < DIM; ++col) - scale[slot][col] = 1.0f; - for (int bank = 0; bank < 10; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)input[slot], 0, DIM, 1); - bb_mvin((uintptr_t)weight[slot], 2, DIM, 1); - bb_mvin((uintptr_t)bias[slot], 3, 4, 1); - bb_mvin((uintptr_t)scale[slot], 8, 4, 1); - bb_transpose(0, 1, DIM, 8); - bb_smatmul_bias(3, 0); - bb_smatmul_os(1, 2, 4, DIM, DIM, DIM, 1, 1, 0); - bb_quant_i32_to_i8(4, 8, 9, DIM * 4, 0, 0, 4, 4, 4, 0); - bb_mvout((uintptr_t)output[slot], 9, DIM, 1); - for (int row = 0; row < DIM; ++row) - for (int col = 0; col < DIM; ++col) { - int expected = 0; - for (int k = 0; k < DIM; ++k) - expected += input[slot][k * DIM + row] * weight[slot][k * DIM + col]; - if (output[slot][row * DIM + col] != expected) - return 1; - } - return 0; -} diff --git a/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp b/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp deleted file mode 100644 index 0f33dbc42..000000000 --- a/examples/cores/goban/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include - -extern "C" int goban_core(core_id_t); -static volatile uint32_t completed; -static volatile uint32_t failed; - -int main() { - core_id_t id = bb_get_core_id(); - if (id.tile == BB_WORK_TILE && - bb_topology_core_profile(id) == BB_CORE_WORKLOAD_PROFILE) { - __atomic_fetch_or(&failed, goban_core(id) != 0, __ATOMIC_RELEASE); - __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); - } - if (id.tile == BB_WORK_TILE && id.core == 0) { - uint32_t expected = - bb_topology_profile_cores_per_tile(BB_CORE_WORKLOAD_PROFILE); - while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) - asm volatile("nop"); - return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; - } - for (;;) - asm volatile("wfi"); -} diff --git a/examples/cores/pebble/arch/src/main/scala/PebbleCore.scala b/examples/cores/pebble/arch/src/main/scala/PebbleCore.scala deleted file mode 100644 index 27685dd23..000000000 --- a/examples/cores/pebble/arch/src/main/scala/PebbleCore.scala +++ /dev/null @@ -1,20 +0,0 @@ -package examples.cores.pebble - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.system.core.{CoreConnection, RocketEndpoint} -import hier.core.rocket.Core -import framework.system.core.accelerator.BuckyballAccelerator - -@instantiable -class PebbleCore(b: GlobalConfig) extends RocketEndpoint(b) { - require(p.buckyball.nonEmpty) - val cpu = Instantiate(new Core(p.data, p.instruction, p.regions, Some(p.commands))(p.cpu)) - clink.cpu <> cpu.clink - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball.get)) - CoreConnection.accelerator(clink.accelerator.get, accelerator) -} - -object PebbleCore extends framework.system.core.RocketCoreFactory { - def instantiate(b: GlobalConfig): Instance[RocketEndpoint] = Instantiate(new PebbleCore(b)) -} diff --git a/examples/cores/pebble/compiler/CMakeLists.txt b/examples/cores/pebble/compiler/CMakeLists.txt deleted file mode 100644 index 2551caa37..000000000 --- a/examples/cores/pebble/compiler/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -set(BB_CORE_NAME pebble) -set(BB_CORE_TARGET_PREFIX "") - -set(BB_CORE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..) -get_filename_component(_BB_REPO_ROOT "${BB_CORE_DIR}/../../.." ABSOLUTE) -include_directories(${_BB_REPO_ROOT}/examples/balls) - -add_subdirectory(src) diff --git a/examples/cores/pebble/compiler/src/CMakeLists.txt b/examples/cores/pebble/compiler/src/CMakeLists.txt deleted file mode 100644 index 7dcd81bc3..000000000 --- a/examples/cores/pebble/compiler/src/CMakeLists.txt +++ /dev/null @@ -1,70 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -message(STATUS "Building Pebble compiler Core package") -message(STATUS " Source: ${CMAKE_CURRENT_SOURCE_DIR}") -message(STATUS " Binary: ${CMAKE_CURRENT_BINARY_DIR}") - -get_filename_component(BB_REPO_ROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../.. ABSOLUTE) -set(BB_COMPILER_COMMON_DIR ${BB_REPO_ROOT}/stack/compiler/src) - -if(NOT EXISTS ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Buckyball.td) - message(FATAL_ERROR "Missing Buckyball compiler common dir: ${BB_COMPILER_COMMON_DIR}") -endif() - -include_directories(${BB_COMPILER_COMMON_DIR}) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/Dialect) -include_directories(${CMAKE_CURRENT_BINARY_DIR}/Dialect) -include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/Dialect/Buckyball) - -set(PEBBLE_BALL_COMPILER_DIALECT_DIRS - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/im2col/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/lut/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/maxpool/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/int8add/compiler/src/Dialect/Buckyball - ${BB_REPO_ROOT}/examples/balls/int8mul/compiler/src/Dialect/Buckyball -) - -set(BB_CORE_BACKEND_INTRINSICS_DIR - ${CMAKE_CURRENT_SOURCE_DIR}/Backend/include - CACHE INTERNAL "Buckyball chip backend intrinsic directory") - -set(BB_BALL_BACKEND_INTRINSICS_DIRS) - -foreach(PEBBLE_BALL_COMPILER_DIALECT_DIR ${PEBBLE_BALL_COMPILER_DIALECT_DIRS}) - if(NOT EXISTS ${PEBBLE_BALL_COMPILER_DIALECT_DIR}) - message(FATAL_ERROR - "Missing pebble ball compiler dialect dir: ${PEBBLE_BALL_COMPILER_DIALECT_DIR}") - endif() - include_directories(${PEBBLE_BALL_COMPILER_DIALECT_DIR}) - get_filename_component(PEBBLE_BALL_COMPILER_DIR - ${PEBBLE_BALL_COMPILER_DIALECT_DIR}/../.. - ABSOLUTE) - list(APPEND BB_BALL_BACKEND_INTRINSICS_DIRS - ${PEBBLE_BALL_COMPILER_DIR}/Backend/include) -endforeach() - -set(BB_BALL_BACKEND_INTRINSICS_DIRS ${BB_BALL_BACKEND_INTRINSICS_DIRS} - CACHE INTERNAL "Buckyball ball backend intrinsic directories") - -add_subdirectory(Dialect) -add_subdirectory(Conversion) -add_subdirectory(Target) - -add_custom_target(buckyball-compiler-${BB_CORE_NAME} - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass -) diff --git a/examples/cores/pebble/compiler/src/Conversion/CMakeLists.txt b/examples/cores/pebble/compiler/src/Conversion/CMakeLists.txt deleted file mode 100644 index 9ca69b1a2..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_subdirectory(LowerBuckyball) -add_subdirectory(LowerTileToBuckyball) diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt b/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt deleted file mode 100644 index 9cafa2834..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt +++ /dev/null @@ -1,27 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/PhysicalBankState.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/ReportBankUsagePass.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp - ${BB_assign_SOURCES} - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp - CoreBankSSALowering.cpp - LowerBuckyballPass.cpp - LowerBuckyballToBankSSAPass.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp - ResidentConvRegionToBankSSAPatterns.cpp - MegaConv2dToBankSSAPatterns.cpp - MemTransposeToBankSSAPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/QuantizeTensorToBankSSAPatterns.cpp - - LINK_LIBS PUBLIC - MLIRControlFlowDialect - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass -) - -# CMake's archive rule appends members, so a chip switch can retain stale -# objects from the previously configured LowerBuckyballPass target. -add_custom_command(TARGET ${BB_CORE_TARGET_PREFIX}LowerBuckyballPass PRE_LINK - COMMAND ${CMAKE_COMMAND} -E rm -f - $) diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/CoreBankSSALowering.cpp b/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/CoreBankSSALowering.cpp deleted file mode 100644 index 1039379f3..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/CoreBankSSALowering.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -using namespace mlir; - -namespace mlir::buddy { -void populatePebbleMegaConv2dToBankSSAPatterns(RewritePatternSet &patterns); -void populatePebbleMemTransposeToBankSSAPatterns(RewritePatternSet &patterns); - -void populatePebbleCoreBankSSALoweringPatterns(RewritePatternSet &patterns, - bool traceMegaStages, - int64_t traceMegaStageStart, - int64_t traceMegaStageLimit) { - populateMatmulRegionToBankSSAPatterns( - patterns, traceMegaStages, traceMegaStageStart, traceMegaStageLimit); - populatePebbleMegaConv2dToBankSSAPatterns(patterns); - populatePebbleMemTransposeToBankSSAPatterns(patterns); - populateQuantizeTensorToBankSSAPatterns(patterns); -} -} // namespace mlir::buddy diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp b/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp deleted file mode 100644 index 0e9bdc690..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp +++ /dev/null @@ -1,127 +0,0 @@ -//===- LowerBuckyballPass.cpp - Pebble Buckyball lowering pass -//-------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/TypeConverter.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Pass/Pass.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Buckyball/Transform.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy {} - -namespace { - -class LowerBuckyballToLLVMPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToLLVMPass) - LowerBuckyballToLLVMPass() = default; - LowerBuckyballToLLVMPass(const LowerBuckyballToLLVMPass &) {} - - StringRef getArgument() const final { return "lower-buckyball"; } - StringRef getDescription() const final { - return "Lower Pebble Buckyball dialect ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -class LowerBankSSAToIntrinsicsPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBankSSAToIntrinsicsPass) - LowerBankSSAToIntrinsicsPass() = default; - LowerBankSSAToIntrinsicsPass(const LowerBankSSAToIntrinsicsPass &) {} - - StringRef getArgument() const final { return "lower-bank-ssa-to-intrinsics"; } - StringRef getDescription() const final { - return "Lower Pebble bank-SSA and Buckyball ops to intrinsic ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballPass() { - PassRegistration(); -} - -void mlir::buddy::registerLowerBankSSAToIntrinsicsPass() { - PassRegistration(); -} diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp b/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp deleted file mode 100644 index 8123157c3..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp +++ /dev/null @@ -1,125 +0,0 @@ -//===- LowerBuckyballToBankSSAPass.cpp - Pebble bank-SSA lowering -//----------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Trace/TraceDialect.h" - -using namespace mlir; - -namespace mlir::buddy { -void populatePebbleCoreBankSSALoweringPatterns(RewritePatternSet &patterns, - bool traceMegaStages, - int64_t traceMegaStageStart, - int64_t traceMegaStageLimit); -void populatePebbleResidentConvRegionToBankSSAPatterns( - RewritePatternSet &patterns, bool traceMegaStages, - int64_t traceMegaStageStart, int64_t traceMegaStageLimit, - int64_t traceMegaRegion, bool traceMegaReloadStages, - bool traceMegaInputBeforeRegion); -} // namespace mlir::buddy - -namespace { - -class LowerBuckyballToBankSSAPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToBankSSAPass) - LowerBuckyballToBankSSAPass() = default; - LowerBuckyballToBankSSAPass(const LowerBuckyballToBankSSAPass &) {} - - StringRef getArgument() const final { return "lower-buckyball-to-bank-ssa"; } - StringRef getDescription() const final { - return "Lower Pebble Buckyball ops to explicit bank-SSA ops."; - } - - Option traceMegaStages{ - *this, "trace-mega-stages", - llvm::cl::desc("Materialize and trace leading MegaKernel stages"), - llvm::cl::init(false)}; - Option traceMegaStageLimit{ - *this, "trace-mega-stage-limit", - llvm::cl::desc("Trace only the first N MegaKernel stages (-1 means all)"), - llvm::cl::init(-1)}; - Option traceMegaStageStart{ - *this, "trace-mega-stage-start", - llvm::cl::desc("Do not dump MegaKernel stages before this index"), - llvm::cl::init(0)}; - Option traceMegaRegion{ - *this, "trace-mega-region", - llvm::cl::desc( - "Trace only this resident MegaKernel region ID (-1 means all)"), - llvm::cl::init(-1)}; - Option traceMegaReloadStages{ - *this, "trace-mega-reload-stages", - llvm::cl::desc("Reload traced stages for downstream consumers"), - llvm::cl::init(true)}; - Option traceMegaInputBeforeRegion{ - *this, "trace-mega-input-before-region", - llvm::cl::desc( - "Trace the input tensor immediately before the selected region"), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert(); - } - - void runOnOperation() override { - RewritePatternSet residentPatterns(&getContext()); - mlir::buddy::populatePebbleResidentConvRegionToBankSSAPatterns( - residentPatterns, traceMegaStages, traceMegaStageStart, - traceMegaStageLimit, traceMegaRegion, traceMegaReloadStages, - traceMegaInputBeforeRegion); - if (failed(applyPatternsGreedily(getOperation(), - std::move(residentPatterns)))) { - signalPassFailure(); - return; - } - RewritePatternSet patterns(&getContext()); - mlir::buddy::populatePebbleCoreBankSSALoweringPatterns( - patterns, traceMegaStages && traceMegaRegion < 0, traceMegaStageStart, - traceMegaStageLimit); - if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) { - signalPassFailure(); - return; - } - bool illegalMega = false; - getOperation().walk([&](Operation *op) { - if (isa<::buddy::buckyball::MegaKernelOp>(op)) { - op->emitError( - "MegaKernel region-wide bank SSA lowering is not implemented"); - illegalMega = true; - } else if (isa<::buddy::buckyball::MegaMatmulOp, - ::buddy::buckyball::MegaConv2dOp, - ::buddy::buckyball::MegaConv2dDepthwiseOp>(op) && - !op->getParentOfType<::buddy::buckyball::MegaKernelOp>()) { - op->emitError( - "MegaKernel stage is only legal inside buckyball.mega_kernel"); - illegalMega = true; - } - }); - if (illegalMega) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballToBankSSAPass() { - PassRegistration(); -} diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt b/examples/cores/pebble/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt deleted file mode 100644 index 93d970513..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerTileToBuckyball/QuantizedKernelPatterns.cpp - CoreTileLowering.cpp - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Conversion/LowerTileToBuckyball/TileTransposePatterns.cpp - - LINK_LIBS PUBLIC - BuddyTile - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRLinalgDialect -) diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerTileToBuckyball/CoreTileLowering.cpp b/examples/cores/pebble/compiler/src/Conversion/LowerTileToBuckyball/CoreTileLowering.cpp deleted file mode 100644 index f428d492c..000000000 --- a/examples/cores/pebble/compiler/src/Conversion/LowerTileToBuckyball/CoreTileLowering.cpp +++ /dev/null @@ -1,88 +0,0 @@ -//===- CoreTileLowering.cpp - Pebble Tile to Buckyball lowering ----------===// - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/DialectConversion.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerTileToBuckyball/LowerTileToBuckyball.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Tile/TileDialect.h" -#include "Tile/TileOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -namespace tile = ::buddy::tile; - -namespace mlir::buddy { -void populateTransposeBallTileLoweringPatterns(RewritePatternSet &patterns, - int64_t bankWidthBytes, - int64_t bankDepth, - int64_t bankNum); -} // namespace mlir::buddy - -namespace { - -class CpuMatmulLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(tile::TileMatMulOp op, - PatternRewriter &rewriter) const override { - rewriter.create( - op.getLoc(), ValueRange{op.getAMemArray(), op.getBMemArray()}, - ValueRange{op.getCMemArray()}); - rewriter.eraseOp(op); - return success(); - } -}; - -class LowerTileToBuckyballPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerTileToBuckyballPass) - - StringRef getArgument() const final { return "convert-tile-to-buckyball"; } - StringRef getDescription() const final { - return "Convert explicit Pebble Tile kernels to Buckyball"; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - ConversionTarget target(getContext()); - target.addLegalDialect(); - target.addIllegalDialect(); - - RewritePatternSet patterns(&getContext()); - mlir::buddy::populateTransposeBallTileLoweringPatterns( - patterns, targetConfig.bankWidthBits / 8, targetConfig.bankDepth, - targetConfig.bankNum); - mlir::buddy::populateQuantizedKernelTileLoweringPatterns(patterns); - patterns.add(&getContext()); - if (failed(applyPartialConversion(getOperation(), target, - std::move(patterns)))) - signalPassFailure(); - } -}; - -} // namespace - -namespace mlir::buddy { -void registerLowerTileToBuckyballPass() { - PassRegistration(); -} -} // namespace mlir::buddy diff --git a/examples/cores/pebble/compiler/src/Dialect/Buckyball/CMakeLists.txt b/examples/cores/pebble/compiler/src/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 867a85b69..000000000 --- a/examples/cores/pebble/compiler/src/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(Buckyball.h.inc -gen-op-decls) -mlir_tablegen(Buckyball.cpp.inc -gen-op-defs) -mlir_tablegen(BuckyballTypes.h.inc -gen-typedef-decls -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballTypes.cpp.inc -gen-typedef-defs -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballDialect.h.inc -gen-dialect-decls -dialect=buckyball) -mlir_tablegen(BuckyballDialect.cpp.inc -gen-dialect-defs -dialect=buckyball) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}MLIRBuckyballIncGen) -add_mlir_doc(PebbleBuckyball ${BB_CORE_TARGET_PREFIX}Buckyball Dialects/ - -gen-dialect-doc) - -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(BuckyballConversions.inc -gen-llvmir-conversions) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen) - -add_subdirectory(IR) -add_subdirectory(Transforms) diff --git a/examples/cores/pebble/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt b/examples/cores/pebble/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt deleted file mode 100644 index 42c7f8807..000000000 --- a/examples/cores/pebble/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/IR/BuckyballDialect.cpp - ${BB_TARGET_REGISTRY_SOURCE} - ${BB_BALL_IR_SOURCES} - - LINK_LIBS PUBLIC - MLIRIR - MLIRLLVMDialect - MLIRBufferizationDialect - MLIRMemRefDialect -) - -target_include_directories(${BB_CORE_TARGET_PREFIX}BuddyBuckyball PUBLIC - ${BB_COMPILER_COMMON_DIR} - ${BB_COMPILER_COMMON_DIR}/Dialect - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball - ${CMAKE_CURRENT_BINARY_DIR}/.. - ${CMAKE_CURRENT_BINARY_DIR}/../../.. -) diff --git a/examples/cores/pebble/compiler/src/Dialect/Buckyball/PebbleBuckyball.td b/examples/cores/pebble/compiler/src/Dialect/Buckyball/PebbleBuckyball.td deleted file mode 100644 index 73a59c06a..000000000 --- a/examples/cores/pebble/compiler/src/Dialect/Buckyball/PebbleBuckyball.td +++ /dev/null @@ -1,31 +0,0 @@ -//===-- PebbleBuckyball.td - Pebble Buckyball dialect ops -*- tablegen -*-===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#ifndef PEBBLE_BUCKYBALL_OPS -#define PEBBLE_BUCKYBALL_OPS - -include "Buckyball.td" -include "TransposeBall.td" -include "SMatMulBall.td" -include "Im2colBall.td" -include "ToInt8Ball.td" -include "Int2FpBall.td" -include "LutBall.td" -include "MaxPoolBall.td" -include "Int8AddBall.td" -include "Int8MulBall.td" - -#endif // PEBBLE_BUCKYBALL_OPS diff --git a/examples/cores/pebble/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt b/examples/cores/pebble/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt deleted file mode 100644 index 747f807e1..000000000 --- a/examples/cores/pebble/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp - ${BB_REPO_ROOT}/examples/balls/transpose/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/smatmul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/im2col/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/toint8/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/lut/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/maxpool/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/int8add/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/int8mul/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - ${BB_REPO_ROOT}/examples/balls/matadd/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp - LegalizeForLLVMExport.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRIR - MLIRLLVMCommonConversion - MLIRLLVMDialect -) diff --git a/examples/cores/pebble/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/cores/pebble/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index dba15d1b9..000000000 --- a/examples/cores/pebble/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,131 +0,0 @@ -//===- LegalizeForLLVMExport.cpp - Pebble Buckyball LLVM lowering ---------===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/Transform.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" - -using namespace mlir; -using namespace buddy::buckyball::legalize; - -namespace mlir::buddy::buckyball { -void populateTransposeBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureTransposeBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateSMatMulBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureSMatMulBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateIm2colBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureIm2colBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateToInt8BallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureToInt8BallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateInt2FpBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureInt2FpBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateLutBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns, - bool stable, - int64_t bankDepth); -void configureLutBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateMaxPoolBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureMaxPoolBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateInt8AddBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureInt8AddBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateInt8MulBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureInt8MulBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -void populateMatAddBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth); -void configureMatAddBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable); -} // namespace mlir::buddy::buckyball - -void mlir::populateBuckyballLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, - int64_t bankWidthBytes, int64_t bankDepth, int64_t bankNum, - bool includeFuncOperandForwarding, bool stable) { - (void)bankWidthBytes; - (void)bankNum; - - populateBaseLegalizeForLLVMExportPatterns(converter, patterns, - includeFuncOperandForwarding); - mlir::buddy::buckyball::populateSMatMulBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateTransposeBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateIm2colBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateToInt8BallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateInt2FpBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateLutBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateMaxPoolBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateInt8AddBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateInt8MulBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::populateMatAddBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); -} - -void mlir::configureBuckyballLegalizeForExportTarget( - LLVMConversionTarget &target, bool stable) { - configureBaseLegalizeForExportTarget(target); - mlir::buddy::buckyball::configureSMatMulBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureTransposeBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureIm2colBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureToInt8BallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureInt2FpBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureLutBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureMaxPoolBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureInt8AddBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureInt8MulBallLegalizeForExportTarget(target, - stable); - mlir::buddy::buckyball::configureMatAddBallLegalizeForExportTarget(target, - stable); -} diff --git a/examples/cores/pebble/compiler/src/Dialect/CMakeLists.txt b/examples/cores/pebble/compiler/src/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/pebble/compiler/src/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/pebble/compiler/src/Target/CMakeLists.txt b/examples/cores/pebble/compiler/src/Target/CMakeLists.txt deleted file mode 100644 index 39d31dc9b..000000000 --- a/examples/cores/pebble/compiler/src/Target/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(LLVMIR) diff --git a/examples/cores/pebble/compiler/src/Target/LLVMIR/CMakeLists.txt b/examples/cores/pebble/compiler/src/Target/LLVMIR/CMakeLists.txt deleted file mode 100644 index 0ca0f41c5..000000000 --- a/examples/cores/pebble/compiler/src/Target/LLVMIR/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Dialect) diff --git a/examples/cores/pebble/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt b/examples/cores/pebble/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 06b99171b..000000000 --- a/examples/cores/pebble/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_mlir_translation_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_COMPILER_COMMON_DIR}/Target/LLVMIR/Dialect/Buckyball/BuckyballToLLVMIRTranslation.cpp - - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen - intrinsics_gen - - LINK_COMPONENTS - Core - - LINK_LIBS PUBLIC - MLIRIR - MLIRSupport - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball -) diff --git a/examples/cores/pebble/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt b/examples/cores/pebble/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt deleted file mode 100644 index ec181148e..000000000 --- a/examples/cores/pebble/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(Buckyball) diff --git a/examples/cores/pebble/configs/balldomains/default.toml b/examples/cores/pebble/configs/balldomains/default.toml deleted file mode 100644 index 9d2878f21..000000000 --- a/examples/cores/pebble/configs/balldomains/default.toml +++ /dev/null @@ -1,32 +0,0 @@ -# Pebble ball domain configuration - -ballNum = 10 - -ballIdMappings = [ - { ballId = 0, ballName = "TransposeBall", ballClass = "examples.balls.transpose.TransposeBall", config = "../../../../balls/transpose/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 1, ballName = "SMatMulBall", ballClass = "examples.balls.smatmul.SMatMulBall", config = "balls/smatmul/8x16.toml", inBW = 2, outBW = 1 }, - { ballId = 2, ballName = "Im2colBall", ballClass = "examples.balls.im2col.Im2colBall", config = "../../../../balls/im2col/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 3, ballName = "ToInt8Ball", ballClass = "examples.balls.toint8.ToInt8Ball", config = "../../../../balls/toint8/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 4, ballName = "Int2FpBall", ballClass = "examples.balls.int2fp.Int2FpBall", config = "../../../../balls/int2fp/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 5, ballName = "LutBall", ballClass = "examples.balls.lut.LutBall", config = "../../../../balls/lut/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 6, ballName = "MaxPoolBall", ballClass = "examples.balls.maxpool.MaxPoolBall", config = "../../../../balls/maxpool/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 7, ballName = "Int8AddBall", ballClass = "examples.balls.int8add.Int8AddBall", config = "../../../../balls/int8add/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 8, ballName = "Int8MulBall", ballClass = "examples.balls.int8mul.Int8MulBall", config = "../../../../balls/int8mul/configs/default.toml", inBW = 2, outBW = 1 }, - { ballId = 9, ballName = "MatAddBall", ballClass = "examples.balls.matadd.MatAddBall", config = "../../../../balls/matadd/configs/default.toml", inBW = 2, outBW = 1 } -] - -ballISA = [ - { mnemonic = "TRANSPOSE", funct7 = 49, bid = 0 }, - { mnemonic = "SMATMUL_OS", funct7 = 65, bid = 1 }, - { mnemonic = "SMATMUL_BIAS", funct7 = 17, bid = 1 }, - { mnemonic = "IM2COL", funct7 = 48, bid = 2 }, - { mnemonic = "QUANT_F32_TO_I8", funct7 = 51, bid = 3 }, - { mnemonic = "QUANT_I32_TO_I8", funct7 = 67, bid = 3 }, - { mnemonic = "INT32_TO_FP32", funct7 = 68, bid = 4 }, - { mnemonic = "LUT", funct7 = 66, bid = 5 }, - { mnemonic = "MAXPOOL", funct7 = 50, bid = 6 }, - { mnemonic = "INT8ADD", funct7 = 69, bid = 7 }, - { mnemonic = "INT8ADD_RELU", funct7 = 70, bid = 7 }, - { mnemonic = "INT8MUL", funct7 = 71, bid = 8 }, - { mnemonic = "MATADD", funct7 = 72, bid = 9 } -] diff --git a/examples/cores/pebble/configs/cpu/default.toml b/examples/cores/pebble/configs/cpu/default.toml deleted file mode 100644 index e651581c6..000000000 --- a/examples/cores/pebble/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "rocket" -config = "./rocket/default.toml" diff --git a/examples/cores/pebble/configs/cpu/rocket/default.toml b/examples/cores/pebble/configs/cpu/rocket/default.toml deleted file mode 100644 index c896503f3..000000000 --- a/examples/cores/pebble/configs/cpu/rocket/default.toml +++ /dev/null @@ -1,76 +0,0 @@ -useUser = true -useSupervisor = true -useHypervisor = false -useDebug = false -useAtomics = true -useAtomicsOnlyForIO = false -useCompressed = true -useRVE = false -useConditionalZero = false -nLocalInterrupts = 0 -useNMI = false -nBreakpoints = 1 -useBPWatch = false -mcontextWidth = 0 -scontextWidth = 0 -nPerfCounters = 0 -haveBasicCounters = true -misaWritable = true -mtvecInitEnable = true -mtvecInit = 0 -mtvecWritable = true -fastLoadWord = true -fastLoadByte = false -branchPredictionModeCSR = false -clockGate = false -mvendorid = 0 -mimpid = 538447876 -haveCease = false -haveSimTimeout = false -useVM = true -useZba = true -useZbb = true -useZbs = true -haveCFlush = false - -[mulDiv] -divUnroll = 1 -divEarlyOutGranularity = 1 -enable = true -mulUnroll = 8 -mulEarlyOut = true -divEarlyOut = true - -[fpu] -divSqrt = true -sfmaLatency = 5 -dfmaLatency = 6 -fpmuLatency = 2 -ifpuLatency = 4 -enable = true -minFLen = 16 -fLen = 64 - -[dcache] -clockGate = false -nSets = 32 -nWays = 4 -nMSHRs = 0 - -[icache] -prefetch = false -nSets = 64 -nWays = 8 - -[btb] -nMatchBits = 14 -nPages = 6 -updatesOutOfOrder = false -bhtEnable = true -bhtEntries = 512 -bhtCounterLength = 1 -bhtHistoryLength = 8 -bhtHistoryBits = 3 -enable = true -nEntries = 28 -nRAS = 6 diff --git a/examples/cores/pebble/configs/default.toml b/examples/cores/pebble/configs/default.toml deleted file mode 100644 index 0e77b20e9..000000000 --- a/examples/cores/pebble/configs/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -factory = "examples.cores.pebble.PebbleCore" -# Pebble Core Configuration (default) - -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/default.toml" -rvv = "rvv/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/pebble/configs/frontends/default.toml b/examples/cores/pebble/configs/frontends/default.toml deleted file mode 100644 index 11926b62b..000000000 --- a/examples/cores/pebble/configs/frontends/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -robEntries = 8 -rsOutOfOrderResponse = true -bankIdLen = 10 -vbankIdUpperBound = 23 -sharedBankIdBase = 32 -iterLen = 34 -subRobEnable = false -subRobDepth = 64 diff --git a/examples/cores/pebble/configs/memdomains/default.toml b/examples/cores/pebble/configs/memdomains/default.toml deleted file mode 100644 index 97ed89d7b..000000000 --- a/examples/cores/pebble/configs/memdomains/default.toml +++ /dev/null @@ -1,31 +0,0 @@ -# Pebble core memory domain configuration - -[bank] -num = 24 -width = 128 -entries = 512 -maskLen = 16 -channel = 13 - -[dma] -nXacts = 8 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 4 - -[tma] -readChannel = 4 -writeChannel = 4 - -[mmio] -enable = false -bankNum = 5 -bankEntries = 1024 -bankWidth = 8 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/pebble/workloads/ctests/CMakeLists.txt b/examples/cores/pebble/workloads/ctests/CMakeLists.txt deleted file mode 100644 index cb6a9e67b..000000000 --- a/examples/cores/pebble/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(im2col_smatmul_quant_pipeline) diff --git a/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/CMakeLists.txt b/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/CMakeLists.txt deleted file mode 100644 index bec14fba6..000000000 --- a/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_buckyball_core_workload(im2col_smatmul_quant_pipeline SOURCES - im2col_smatmul_quant_pipeline-main.cpp - pebble.c) diff --git a/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp b/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp deleted file mode 100644 index 20d2dab05..000000000 --- a/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include - -extern "C" int pebble_core(core_id_t); -static volatile uint32_t completed; -static volatile uint32_t failed; - -int main() { - core_id_t id = bb_get_core_id(); - if (id.tile == BB_WORK_TILE && - bb_topology_core_profile(id) == BB_CORE_WORKLOAD_PROFILE) { - __atomic_fetch_or(&failed, pebble_core(id) != 0, __ATOMIC_RELEASE); - __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); - } - if (id.tile == BB_WORK_TILE && id.core == 0) { - uint32_t expected = - bb_topology_profile_cores_per_tile(BB_CORE_WORKLOAD_PROFILE); - while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) - asm volatile("nop"); - return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; - } - for (;;) - asm volatile("wfi"); -} diff --git a/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/pebble.c b/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/pebble.c deleted file mode 100644 index 58b4da4de..000000000 --- a/examples/cores/pebble/workloads/ctests/im2col_smatmul_quant_pipeline/pebble.c +++ /dev/null @@ -1,63 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include -#include -#include -#include - -#if BANK_WIDTH != 128 || BANK_LINES < 64 -#error "Pebble core pipeline requires 128-bit banks with at least 64 rows" -#endif - -enum { LANES = BANK_WIDTH / 8, INPUT_ROWS = 36, WINDOWS = 16 }; -static int8_t input[BB_CORES_PER_TILE][INPUT_ROWS * LANES] - __attribute__((aligned(64))); -static int8_t weight[BB_CORES_PER_TILE][LANES * LANES] - __attribute__((aligned(64))); -static int32_t bias[BB_CORES_PER_TILE][LANES] __attribute__((aligned(64))); -static float scale[BB_CORES_PER_TILE][LANES] __attribute__((aligned(64))); -static int8_t output[BB_CORES_PER_TILE][WINDOWS * LANES] - __attribute__((aligned(64))); - -#ifdef __cplusplus -extern "C" -#endif - int pebble_core(core_id_t id) { - int core = id.core; - for (int row = 0; row < INPUT_ROWS; ++row) - input[core][row * LANES] = (3 * row + 1) % 7 - 3; - for (int channel = 0; channel < LANES; ++channel) { - bias[core][channel] = channel - 8; - scale[core][channel] = 1.0f; - for (int k = 0; k < LANES; ++k) - weight[core][k * LANES + channel] = - k < 9 ? (2 * k + 3 * channel) % 5 - 2 : 0; - } - for (int bank = 0; bank < 7; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)input[core], 0, INPUT_ROWS, 1); - bb_mvin((uintptr_t)weight[core], 2, LANES, 1); - bb_mvin((uintptr_t)bias[core], 3, LANES * 32 / BANK_WIDTH, 1); - bb_mvin((uintptr_t)scale[core], 5, LANES * 32 / BANK_WIDTH, 1); - bb_im2col(0, 1, 6, 3, 1, 0, 0, 0, 0, 0, 0, WINDOWS); - bb_smatmul_bias(3, 0); - bb_smatmul_os(1, 2, 4, WINDOWS, LANES, LANES, 1, 1, 0); - bb_quant_i32_to_i8(4, 5, 6, WINDOWS * 4, 0, 0, 4, 4, 4, 0); - bb_mvout((uintptr_t)output[core], 6, WINDOWS, 1); - for (int window = 0; window < WINDOWS; ++window) - for (int channel = 0; channel < LANES; ++channel) { - int expected = bias[core][channel]; - int y = window / 4, x = window % 4; - for (int ky = 0; ky < 3; ++ky) - for (int kx = 0; kx < 3; ++kx) { - int k = ky * 3 + kx, row = (y + ky) * 6 + x + kx; - expected += - input[core][row * LANES] * weight[core][k * LANES + channel]; - } - if (output[core][window * LANES + channel] != expected) - return 1; - } - return 0; -} diff --git a/examples/cores/reduce/arch/src/main/scala/ReduceCore.scala b/examples/cores/reduce/arch/src/main/scala/ReduceCore.scala deleted file mode 100644 index bf14a7750..000000000 --- a/examples/cores/reduce/arch/src/main/scala/ReduceCore.scala +++ /dev/null @@ -1,20 +0,0 @@ -package examples.cores.reduce - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.system.core.{CoreConnection, RocketEndpoint} -import hier.core.rocket.Core -import framework.system.core.accelerator.BuckyballAccelerator - -@instantiable -class ReduceCore(b: GlobalConfig) extends RocketEndpoint(b) { - require(p.buckyball.nonEmpty) - val cpu = Instantiate(new Core(p.data, p.instruction, p.regions, Some(p.commands))(p.cpu)) - clink.cpu <> cpu.clink - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball.get)) - CoreConnection.accelerator(clink.accelerator.get, accelerator) -} - -object ReduceCore extends framework.system.core.RocketCoreFactory { - def instantiate(b: GlobalConfig): Instance[RocketEndpoint] = Instantiate(new ReduceCore(b)) -} diff --git a/examples/cores/reduce/compiler/CMakeLists.txt b/examples/cores/reduce/compiler/CMakeLists.txt deleted file mode 100644 index 678fa8bd2..000000000 --- a/examples/cores/reduce/compiler/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -set(BB_CORE_NAME reduce) -set(BB_CORE_TARGET_PREFIX "") -get_filename_component(_BB_REPO_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../../.." ABSOLUTE) -set(_BB_F32ADD_DIALECT_DIR - "${_BB_REPO_ROOT}/examples/balls/f32add/compiler/src/Dialect/Buckyball") -set(_BB_F32ADD_LEGALIZE "${_BB_F32ADD_DIALECT_DIR}/Transforms/LegalizeForLLVMExport.cpp") -if(NOT _BB_F32ADD_LEGALIZE IN_LIST BB_legalize_SOURCES) - message(FATAL_ERROR "Reduce compiler requires the configured F32Add Ball lowering") -endif() -include_directories("${_BB_F32ADD_DIALECT_DIR}") - -# The base builds the generated dialect and selected Ball IR/assign/export hooks. -add_subdirectory("${_BB_REPO_ROOT}/examples/cores/toy/compiler/src" - "${CMAKE_CURRENT_BINARY_DIR}/base") diff --git a/examples/cores/reduce/configs/balldomains/default.toml b/examples/cores/reduce/configs/balldomains/default.toml deleted file mode 100644 index 810a63d1b..000000000 --- a/examples/cores/reduce/configs/balldomains/default.toml +++ /dev/null @@ -1,9 +0,0 @@ -ballNum = 1 - -ballIdMappings = [ - { ballId = 0, ballName = "F32AddBall", ballClass = "examples.balls.f32add.F32AddBall", config = "../../../../balls/f32add/configs/default.toml", inBW = 2, outBW = 1 } -] - -ballISA = [ - { mnemonic = "F32ADD", funct7 = 76, bid = 0 } -] diff --git a/examples/cores/reduce/configs/cpu/default.toml b/examples/cores/reduce/configs/cpu/default.toml deleted file mode 100644 index 1bf2f46e8..000000000 --- a/examples/cores/reduce/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "rocket" -config = "../../../rocket/configs/cpu/rocket/default.toml" diff --git a/examples/cores/reduce/configs/default.toml b/examples/cores/reduce/configs/default.toml deleted file mode 100644 index ec4b3d5d3..000000000 --- a/examples/cores/reduce/configs/default.toml +++ /dev/null @@ -1,6 +0,0 @@ -factory = "examples.cores.reduce.ReduceCore" -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/default.toml" -rvv = "rvv/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/reduce/configs/frontends/default.toml b/examples/cores/reduce/configs/frontends/default.toml deleted file mode 100644 index 41cd684cc..000000000 --- a/examples/cores/reduce/configs/frontends/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -robEntries = 16 -rsOutOfOrderResponse = true -bankIdLen = 10 -vbankIdUpperBound = 15 -sharedBankIdBase = 32 -iterLen = 34 -subRobEnable = false -subRobDepth = 64 diff --git a/examples/cores/reduce/configs/memdomains/default.toml b/examples/cores/reduce/configs/memdomains/default.toml deleted file mode 100644 index 37adae0ba..000000000 --- a/examples/cores/reduce/configs/memdomains/default.toml +++ /dev/null @@ -1,29 +0,0 @@ -[bank] -num = 4 -width = 128 -entries = 1024 -maskLen = 16 -channel = 3 - -[dma] -nXacts = 1 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 1 - -[tma] -readChannel = 2 -writeChannel = 1 - -[mmio] -enable = false -bankNum = 1 -bankEntries = 1 -bankWidth = 128 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/reduce/configs/rvv/default.toml b/examples/cores/reduce/configs/rvv/default.toml deleted file mode 100644 index e006b2055..000000000 --- a/examples/cores/reduce/configs/rvv/default.toml +++ /dev/null @@ -1,6 +0,0 @@ -enable = false -laneNumber = 4 -vLen = 1024 -eLen = 32 -iBufWords = 1024 -memoryPorts = 4 diff --git a/examples/cores/reduce/workloads/ctests/CMakeLists.txt b/examples/cores/reduce/workloads/ctests/CMakeLists.txt deleted file mode 100644 index abfd63910..000000000 --- a/examples/cores/reduce/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -set(BUCKYBALL_CTEST_TARGET ${BUCKYBALL_CORE_WORKLOAD_TARGET}) -set(BUCKYBALL_CTEST_GROUP_TARGET core-ctests-build) -add_subdirectory( - ${BUCKYBALL_REPO_ROOT}/examples/balls/f32add/workloads/ctests - f32add) - -list(APPEND BUCKYBALL_CHIP_COMMON_SOURCES - ${BUCKYBALL_REPO_ROOT}/stack/runtime/src/tlink.c) -add_buckyball_ctest(tlink_lease.c) diff --git a/examples/cores/reduce/workloads/ctests/tlink_lease.c b/examples/cores/reduce/workloads/ctests/tlink_lease.c deleted file mode 100644 index 529230188..000000000 --- a/examples/cores/reduce/workloads/ctests/tlink_lease.c +++ /dev/null @@ -1,36 +0,0 @@ -#include -#include -#include -#include -#include - -static const uint32_t input[2][4] __attribute__((aligned(64))) = { - {1, 2, 3, 4}, {0x12345678, 0xabcdef01, 7, 8}}; -static volatile uint32_t output[8] __attribute__((aligned(64))); - -int main(void) { - unsigned bank = BB_SHARED_BANK_BASE; - bb_mem_alloc(bank, 1, 2); - for (unsigned i = 0; i < 8; ++i) - output[i] = 0xaaaaaaaa; - for (unsigned group = 0; group < 2; ++group) - bb_mvin_group((uintptr_t)input[group], bank, group, 1, 1); - uint64_t address = tlink_shared_export(0, bank, 1); - if (tlink_shared_export(0, bank, 1) != address) - __builtin_trap(); - tlink_shared_release(0, bank, 1); - if (tlink_read64(address) != UINT64_C(0xabcdef0112345678)) - __builtin_trap(); - tlink_write64(address + 8, UINT64_C(0x76543210deadbeef)); - bb_mvout((uintptr_t)output, bank, 1, 1); - for (unsigned i = 0; i < 4; ++i) - if (output[i] != input[0][i]) - __builtin_trap(); - if (output[4] != input[1][0] || output[5] != input[1][1] || - output[6] != 0xdeadbeef || output[7] != 0x76543210) - __builtin_trap(); - tlink_shared_release(0, bank, 1); - bb_mem_release(bank); - puts("Main TLink lease and NPU shared storage PASS"); - return 0; -} diff --git a/examples/cores/rocket/arch/src/main/scala/RocketCore.scala b/examples/cores/rocket/arch/src/main/scala/RocketCore.scala deleted file mode 100644 index 57563e285..000000000 --- a/examples/cores/rocket/arch/src/main/scala/RocketCore.scala +++ /dev/null @@ -1,17 +0,0 @@ -package examples.cores.rocket - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.system.core.{CoreConnection, RocketEndpoint} -import hier.core.rocket.Core - -@instantiable -class RocketCore(b: GlobalConfig) extends RocketEndpoint(b) { - require(p.buckyball.isEmpty) - val cpu = Instantiate(new Core(p.data, p.instruction, p.regions, Some(p.commands))(p.cpu)) - clink.cpu <> cpu.clink -} - -object RocketCore extends framework.system.core.RocketCoreFactory { - def instantiate(b: GlobalConfig): Instance[RocketEndpoint] = Instantiate(new RocketCore(b)) -} diff --git a/examples/cores/rocket/arch/src/main/scala/cpu/AdmissionBridge.scala b/examples/cores/rocket/arch/src/main/scala/cpu/AdmissionBridge.scala deleted file mode 100644 index f0b4d7007..000000000 --- a/examples/cores/rocket/arch/src/main/scala/cpu/AdmissionBridge.scala +++ /dev/null @@ -1,164 +0,0 @@ -package hier.core.rocket - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import framework.system.core.rocket.RoCCCommandBB -import memcore.memory.interlock.{Params => TrackingParams} - -class RobFault(robEntries: Int) extends Bundle { - val rob_id = UInt(log2Ceil(robEntries).W) - val error = UInt(4.W) - val address = UInt(64.W) -} - -class AdmissionRetirement(tracking: TrackingParams, pmps: Int)(implicit val cpuParams: CpuParams) - extends Bundle - with HasCpuParameters { - val snapshot = new CommandSnapshot(tracking, pmps) - val error = UInt(4.W) - val address = UInt(64.W) -} - -/** Associates accepted CPU snapshots with actual external-command ROB allocations. */ -@instantiable -class AdmissionBridge( - robEntries: Int, - tracking: TrackingParams, - pmps: Int, - nonRobFuncts: Seq[Int] -)( - implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - require(robEntries >= 2 && tracking.entries == 4) - val robBits = log2Ceil(robEntries) - - @public - val io = IO(new Bundle { - val command = Flipped(Decoupled(new CommandSnapshot(tracking, pmps))) - val npuCommand = Decoupled(new RoCCCommandBB) - // bits is the current tail preview even when valid is low; valid means exact alloc.fire. - val allocation = Flipped(Valid(UInt(robBits.W))) - val retired = Input(UInt(robEntries.W)) - val fault = Flipped(Valid(new RobFault(robEntries))) - val unboundFault = Valid(new RobFault(robEntries)) - val faultTag = Valid(UInt(tracking.idBits.W)) - - val lookup = Vec( - 3, - new Bundle { - val robId = Input(UInt(robBits.W)) - val snapshot = Output(Valid(new CommandSnapshot(tracking, pmps))) - } - ) - - val lookupTag = new Bundle { - val tag = Input(UInt(tracking.idBits.W)) - val snapshot = Output(Valid(new CommandSnapshot(tracking, pmps))) - } - - // A ROB retirement notice alone does not establish DMA/cache completion. - val retirement = Decoupled(new AdmissionRetirement(tracking, pmps)) - }) - - val live = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) - val pending = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) - val hasRob = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) - val robIds = Reg(Vec(tracking.entries, UInt(robBits.W))) - val snapshots = Reg(Vec(tracking.entries, new CommandSnapshot(tracking, pmps))) - val errors = RegInit(VecInit(Seq.fill(tracking.entries)(0.U(4.W)))) - val addresses = RegInit(VecInit(Seq.fill(tracking.entries)(0.U(64.W)))) - val free = VecInit(live.map(!_)).asUInt - val slot = PriorityEncoder(free) - val nonRob = nonRobFuncts.map(f => io.command.bits.instruction.funct === f.U(7.W)).reduce(_ || _) - val duplicateTag = - (0 until tracking.entries).map(i => live(i) && snapshots(i).tag === io.command.bits.tag).reduce(_ || _) - val duplicateRob = - (0 until tracking.entries).map(i => live(i) && hasRob(i) && robIds(i) === io.allocation.bits).reduce(_ || _) - val eligible = free.orR && !duplicateTag && (nonRob || !duplicateRob) && !reset.asBool - io.npuCommand.valid := io.command.valid && eligible - io.npuCommand.bits := io.command.bits.instruction - io.command.ready := io.npuCommand.ready && eligible - - when(io.npuCommand.fire) { - assert(io.allocation.valid === !nonRob, "AdmissionBridge command/ROB allocation mismatch") - when(!nonRob)(assert(io.allocation.bits < robEntries.U, "AdmissionBridge ROB allocation out of range")) - live(slot) := true.B - pending(slot) := nonRob - hasRob(slot) := !nonRob - robIds(slot) := io.allocation.bits - snapshots(slot) := io.command.bits - errors(slot) := 0.U - addresses(slot) := 0.U - } - // Boot is reset-only in the actual Frontend: its allocations precede all external contexts. - when(io.allocation.valid && !io.npuCommand.fire) { - assert(!live.asUInt.orR, "AdmissionBridge boot allocation overlaps a live CPU context") - } - val heldCommand = RegNext(io.npuCommand.valid && !io.npuCommand.ready, false.B) - val previousCommand = RegEnable(io.npuCommand.bits.asUInt, io.npuCommand.valid) - when(heldCommand && !reset.asBool) { - assert( - io.npuCommand.valid && io.npuCommand.bits.asUInt === previousCommand, - "AdmissionBridge stalled NPU command changed" - ) - } - // Boot-only allocations have no external command fire and create no CPU context. - for (i <- 0 until tracking.entries) { - when(live(i) && hasRob(i) && io.retired(robIds(i)))(pending(i) := true.B) - } - for (port <- io.lookup) { - val hits = VecInit((0 until tracking.entries).map(i => - live(i) && hasRob(i) && !pending(i) && - !io.retired(robIds(i)) && robIds(i) === port.robId - )) - port.snapshot.valid := hits.asUInt.orR && !reset.asBool - port.snapshot.bits := Mux1H(hits, snapshots) - assert(PopCount(hits) <= 1.U, "AdmissionBridge duplicate live ROB binding") - } - - val tagHits = VecInit((0 until tracking.entries).map(i => - live(i) && hasRob(i) && !pending(i) && !io.retired(robIds(i)) && snapshots(i).tag === io.lookupTag.tag - )) - - io.lookupTag.snapshot.valid := tagHits.asUInt.orR && !reset.asBool - io.lookupTag.snapshot.bits := Mux1H(tagHits, snapshots) - assert(PopCount(tagHits) <= 1.U, "AdmissionBridge duplicate live admission tag") - - val noticeValid = RegInit(false.B) - val noticeSlot = RegInit(0.U(log2Ceil(tracking.entries).W)) - when(!noticeValid && pending.asUInt.orR) { - noticeValid := true.B - noticeSlot := PriorityEncoder(pending.asUInt) - } - io.retirement.valid := noticeValid && !reset.asBool - val faultHits = - VecInit((0 until tracking.entries).map(i => live(i) && hasRob(i) && robIds(i) === io.fault.bits.rob_id)) - io.unboundFault.valid := io.fault.valid && !faultHits.asUInt.orR && !reset.asBool - io.unboundFault.bits := io.fault.bits - io.faultTag.valid := io.fault.valid && faultHits.asUInt.orR && !reset.asBool - io.faultTag.bits := Mux1H(faultHits, snapshots.map(_.tag)) - when(io.fault.valid && io.fault.bits.error =/= 0.U) { - for (i <- 0 until tracking.entries) { - when(faultHits(i)) { - val alreadyOffered = noticeValid && noticeSlot === i.U - assert(!alreadyOffered, "AdmissionBridge fault arrived after retirement offer") - when(!alreadyOffered && errors(i) === 0.U) { - errors(i) := io.fault.bits.error - addresses(i) := io.fault.bits.address - } - } - } - } - io.retirement.bits.snapshot := snapshots(noticeSlot) - io.retirement.bits.error := errors(noticeSlot) - io.retirement.bits.address := addresses(noticeSlot) - when(io.retirement.fire) { - live(noticeSlot) := false.B - pending(noticeSlot) := false.B - hasRob(noticeSlot) := false.B - noticeValid := false.B - } -} diff --git a/examples/cores/rocket/arch/src/main/scala/cpu/Cache.scala b/examples/cores/rocket/arch/src/main/scala/cpu/Cache.scala deleted file mode 100644 index c394f6d7a..000000000 --- a/examples/cores/rocket/arch/src/main/scala/cpu/Cache.scala +++ /dev/null @@ -1,75 +0,0 @@ -package hier.core.rocket - -import memcore.memory.queue.Queue - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import memcore.bus.chi.{RequestFlit, RequesterPort} -import memcore.bus.chi.rnf.{BankedChiCache, CacheAccess, CacheProbe, CacheResult, RnfParams} -import memcore.memory.interlock.{Acknowledgement, Maintenance => Range, Params} - -/** A core's own L1 and range maintenance client share one explicit CHI requester. */ -@instantiable -class Cache(config: RnfParams, tracking: Params) extends Module { - override def desiredName: String = "CoreCache" - val c = config.chi - - @public - val io = IO(new Bundle { - val access = Flipped(Decoupled(new CacheAccess(c))) - val result = Decoupled(new CacheResult) - val probe = Option.when(config.probe)(new CacheProbe(c)) - val maintenance = Flipped(Decoupled(new Range(tracking))) - val maintained = Decoupled(new Acknowledgement(tracking)) - // Includes previously accepted virtual accesses, page walks and fetches outside this L1. - val olderRequestsDrained = Input(Bool()) - val chi = new RequesterPort(c) - val outstanding = Output(UInt(32.W)) - }) - - val cache: Instance[BankedChiCache] = Instantiate(new BankedChiCache(config)) - val maintenance: Instance[Maintenance] = Instantiate(new Maintenance(config, tracking)) - val drained = io.olderRequestsDrained && cache.io.outstanding === 0.U - maintenance.io.request.valid := io.maintenance.valid && drained - maintenance.io.request.bits := io.maintenance.bits - io.maintenance.ready := maintenance.io.request.ready && drained - // Existing owners may keep progressing until drained. At admission, a new CPU - // request cannot race the CMO; subsequent conflicts belong to the external interlock. - val admitMaintenance = io.maintenance.valid && io.maintenance.ready - cache.io.access.valid := io.access.valid && !admitMaintenance - cache.io.access.bits := io.access.bits - io.access.ready := cache.io.access.ready && !admitMaintenance - io.result <> cache.io.result - io.probe.foreach(_ <> cache.io.probe.get) - maintenance.io.drained := true.B - io.maintained <> maintenance.io.response - io.outstanding := cache.io.outstanding - - val requests = Module(new RRArbiter(new RequestFlit(c), 2) { - - override lazy val lastGrant = { - val previous = RegInit(0.U(1.W)) - when(io.out.fire)(previous := io.chosen) - previous - } - - }) - - requests.io.in(0) <> cache.io.chi.req - requests.io.in(1) <> maintenance.io.req - val requestQueue = Module(new Queue(new RequestFlit(c), 2)) - requestQueue.io.enq <> requests.io.out - io.chi.req <> requestQueue.io.deq - io.chi.txRsp <> cache.io.chi.txRsp - io.chi.txDat <> cache.io.chi.txDat - cache.io.chi.rxDat <> io.chi.rxDat - cache.io.chi.snp <> io.chi.snp - - val maintenanceResponse = io.chi.rxRsp.bits.txnId === config.banks.U - maintenance.io.rsp.valid := io.chi.rxRsp.valid && maintenanceResponse - maintenance.io.rsp.bits := io.chi.rxRsp.bits - cache.io.chi.rxRsp.valid := io.chi.rxRsp.valid && !maintenanceResponse - cache.io.chi.rxRsp.bits := io.chi.rxRsp.bits - io.chi.rxRsp.ready := Mux(maintenanceResponse, maintenance.io.rsp.ready, cache.io.chi.rxRsp.ready) -} diff --git a/examples/cores/rocket/arch/src/main/scala/cpu/Lsu.scala b/examples/cores/rocket/arch/src/main/scala/cpu/Lsu.scala deleted file mode 100644 index c846b5207..000000000 --- a/examples/cores/rocket/arch/src/main/scala/cpu/Lsu.scala +++ /dev/null @@ -1,252 +0,0 @@ -package hier.core.rocket - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import freechips.rocketchip.rocket._ -import freechips.rocketchip.rocket.constants.MemoryOpConstants -import memcore.bus.chi.rnf.CacheAtomic -import memcore.memory.cpu.{CpuMemParams, VirtualMemoryRequest, VirtualMemoryResponse} -import memcore.memory.fetch.Context - -/** A load or store offered to the L1 fast path by virtual address; the Core translates it through the data TLB. */ -class LsuProbeRequest extends Bundle { - val addr = UInt(64.W) - val size = UInt(2.W) - val write = Bool() - val data = UInt(64.W) - val privilege = UInt(2.W) - val satp = UInt(64.W) - val sum = Bool() - val mxr = Bool() -} - -class LsuProbeResult extends Bundle { - val hit = Bool() - val value = UInt(64.W) -} - -class LsuProbe extends Bundle { - val req = Decoupled(new LsuProbeRequest) - val complete = Flipped(Decoupled(new LsuProbeResult)) - val cancel = Output(Bool()) - val retire = Output(Bool()) - val active = Output(Bool()) - val cancelled = Input(Bool()) -} - -/** - * A probe hit completes in s2. Otherwise: fixed S1/S2 cancellation windows, then execute once - * and return only on the instruction's replay. - */ -@instantiable -class Lsu(cp: CpuMemParams)(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters - with MemoryOpConstants { - - @public - val io = IO(new Bundle { - val cpu = Flipped(new HellaCacheIO) - val pc = Input(UInt(vaddrBitsExtended.W)) - val pmp = Input(Vec(nPMPs, new PMP)) - val context = Input(new Context) - val maintenance = Decoupled(new SFenceReq) - val maintained = Flipped(Decoupled(Bool())) - val capturedPmp = Output(Vec(nPMPs, new PMP)) - val request = Decoupled(new VirtualMemoryRequest(cp)) - val response = Flipped(Decoupled(new VirtualMemoryResponse(cp))) - val probe = new LsuProbe - val idle = Output(Bool()) - val cancelled = Output(Bool()) - }) - - require(!usingHypervisor && cp.tagBits <= coreParams.dcacheReqTagBits) - val idle :: s1 :: s2 :: issue :: receive :: completed :: replay1 :: replay2 :: Nil = Enum(8) - val state = RegInit(idle) - val command = Reg(new HellaCacheReq) - val instructionPc = Reg(UInt(vaddrBitsExtended.W)) - val data = Reg(UInt(64.W)) - val pmps = Reg(Vec(nPMPs, new PMP)) - val context = Reg(new Context) - val maintaining = RegInit(false.B) - val probeAccepted = RegInit(false.B) - io.capturedPmp := pmps - val result = Reg(new VirtualMemoryResponse(cp)) - val matches = io.cpu.req.bits.tag === command.tag && io.cpu.req.bits.addr === command.addr && - io.cpu.req.bits.cmd === command.cmd && io.cpu.req.bits.size === command.size && io.pc === instructionPc - io.idle := state === idle - io.cancelled := (state === s1 || state === replay1) && io.cpu.s1_kill || - (state === s2 || state === replay2) && io.cpu.s2_kill - io.cpu.req.ready := state === idle || (state === completed && (!io.cpu.req.valid || matches)) - when(io.cpu.req.fire) { - when(state === idle) { - assert(!io.cpu.req.bits.dv && io.cpu.req.bits.dprv =/= 2.U, "LSU requires nonvirtual U, S or M privilege") - if (io.cpu.req.bits.tag.getWidth > cp.tagBits) { - assert( - !io.cpu.req.bits.tag(io.cpu.req.bits.tag.getWidth - 1, cp.tagBits).orR, - "LSU arbiter tag bits must be zero" - ) - } - command := io.cpu.req.bits - instructionPc := io.pc - pmps := io.pmp - context := io.context - state := s1 - }.otherwise { - assert(matches, "LSU replay changed instruction identity") - state := replay1 - } - } - when(state === s1) { - when(io.cpu.s1_kill)(state := idle) - .otherwise { data := io.cpu.s1_data.data; state := s2 } - } - - val writes = isWrite(command.cmd) - val reads = isRead(command.cmd) - val address = if (usingVM) command.addr.asSInt.pad(64).asUInt else command.addr.pad(64) - val sfence = command.cmd === M_SFENCE - val misaligned = (address & ((1.U(64.W) << command.size) - 1.U)).orR - - val probeEligible = !misaligned && (command.cmd === M_XRD || command.cmd === M_XWR) - io.probe.active := (state === s1 || state === s2) && probeEligible - io.probe.req.valid := state === s1 && !io.cpu.s1_kill && probeEligible - io.probe.req.bits.addr := address - io.probe.req.bits.size := command.size - io.probe.req.bits.write := command.cmd === M_XWR - io.probe.req.bits.data := io.cpu.s1_data.data - io.probe.req.bits.privilege := command.dprv - io.probe.req.bits.satp := context.satp - io.probe.req.bits.sum := context.sum - io.probe.req.bits.mxr := context.mxr - io.probe.cancel := state === s2 && io.cpu.s2_kill - io.probe.retire := state === s2 && probeAccepted && !io.cpu.s2_kill - io.probe.complete.ready := state === s2 && probeAccepted && !io.cpu.s2_kill - when(io.probe.req.fire)(probeAccepted := true.B) - when(state === s2)(probeAccepted := false.B) - when(state === s2 && probeAccepted && !io.cpu.s2_kill) { - assert(io.probe.complete.valid || io.probe.cancelled, "Accepted LSU probe did not complete in s2") - } - val probed = io.probe.complete.fire && io.probe.complete.bits.hit && !io.probe.cancelled - val probedRaw = io.probe.complete.bits.value >> (address(2, 0) << 3) - - val probedLoad = MuxLookup(command.size, probedRaw)(Seq( - 0.U -> Cat(Fill(56, command.signed && probedRaw(7)), probedRaw(7, 0)), - 1.U -> Cat(Fill(48, command.signed && probedRaw(15)), probedRaw(15, 0)), - 2.U -> Cat(Fill(32, command.signed && probedRaw(31)), probedRaw(31, 0)) - )) - - when(state === s2) { - when(io.cpu.s2_kill)(state := idle).elsewhen(probed)(state := idle).otherwise { - when(misaligned && !sfence) { - result := 0.U.asTypeOf(result) - result.tag := command.tag(cp.tagBits - 1, 0) - result.misaligned := misaligned - result.accessFault := false.B - state := completed - }.otherwise(state := issue) - } - } - - val atom = MuxLookup(command.cmd, CacheAtomic.None.U)(Seq( - M_XA_SWAP -> CacheAtomic.Swap.U, - M_XA_ADD -> CacheAtomic.Add.U, - M_XA_XOR -> CacheAtomic.Xor.U, - M_XA_AND -> CacheAtomic.And.U, - M_XA_OR -> CacheAtomic.Or.U, - M_XA_MIN -> CacheAtomic.Min.U, - M_XA_MAX -> CacheAtomic.Max.U, - M_XA_MINU -> CacheAtomic.MinU.U, - M_XA_MAXU -> CacheAtomic.MaxU.U, - M_XLR -> CacheAtomic.LR.U, - M_XSC -> CacheAtomic.SC.U - )) - - io.maintenance.valid := state === issue && sfence - io.maintenance.bits := 0.U.asTypeOf(io.maintenance.bits) - io.maintenance.bits.addr := command.addr - io.maintenance.bits.asid := data - io.maintenance.bits.rs1 := command.size(0) - io.maintenance.bits.rs2 := command.size(1) - io.maintained.ready := state === receive && maintaining - when(io.maintenance.fire) { maintaining := true.B; state := receive } - when(io.maintained.fire) { - assert(io.maintained.bits, "LSU maintenance must acknowledge completion") - result := 0.U.asTypeOf(result) - result.tag := command.tag(cp.tagBits - 1, 0) - maintaining := false.B - state := completed - } - io.request.valid := state === issue && !sfence - io.request.bits := 0.U.asTypeOf(io.request.bits) - io.request.bits.vaddr := address - io.request.bits.tag := command.tag(cp.tagBits - 1, 0) - io.request.bits.size := command.size - io.request.bits.write := command.cmd === M_XWR - io.request.bits.execute := false.B - io.request.bits.signed := command.signed - io.request.bits.data := data - io.request.bits.atomic := atom - io.request.bits.privilege := command.dprv - io.request.bits.sum := context.sum - io.request.bits.mxr := context.mxr - io.request.bits.satpMode := context.satp(63, 60) - io.request.bits.rootPpn := context.satp(43, 0) - when(io.request.fire) { - assert( - command.cmd === M_XRD || command.cmd === M_XWR || command.cmd === M_XLR || - command.cmd === M_XSC || isAMO(command.cmd), - "LSU unsupported memory operation" - ) - state := receive - } - io.response.ready := state === receive && !maintaining - when(io.response.fire) { - assert(io.response.bits.tag === command.tag(cp.tagBits - 1, 0), "LSU virtual response tag mismatch") - result := io.response.bits - state := completed - } - when(state === replay1)(state := Mux(io.cpu.s1_kill, completed, replay2)) - when(state === replay2)(state := Mux(io.cpu.s2_kill, completed, idle)) - - val returns = state === replay2 && !io.cpu.s2_kill - val exception = result.misaligned || result.pageFault || result.accessFault - io.cpu.s2_nack := state === s2 && !io.cpu.s2_kill && !probed - io.cpu.s2_nack_cause_raw := false.B - io.cpu.s2_uncached := false.B - io.cpu.s2_paddr := command.addr(paddrBits - 1, 0) - io.cpu.s2_gpa := 0.U - io.cpu.s2_gpa_is_pte := false.B - io.cpu.s2_xcpt := 0.U.asTypeOf(io.cpu.s2_xcpt) - io.cpu.s2_xcpt.ma.ld := returns && !writes && result.misaligned - io.cpu.s2_xcpt.ma.st := returns && writes && result.misaligned - io.cpu.s2_xcpt.pf.ld := returns && !writes && result.pageFault - io.cpu.s2_xcpt.pf.st := returns && writes && result.pageFault - io.cpu.s2_xcpt.ae.ld := returns && !writes && result.accessFault - io.cpu.s2_xcpt.ae.st := returns && writes && result.accessFault - io.cpu.resp.valid := returns || probed - io.cpu.resp.bits := 0.U.asTypeOf(io.cpu.resp.bits) - io.cpu.resp.bits.tag := command.tag - io.cpu.resp.bits.cmd := command.cmd - io.cpu.resp.bits.size := command.size - io.cpu.resp.bits.addr := command.addr - io.cpu.resp.bits.signed := command.signed - io.cpu.resp.bits.dprv := command.dprv - io.cpu.resp.bits.has_data := reads && !command.no_resp && (probed || !exception) - io.cpu.resp.bits.data := Mux(probed, probedLoad, result.data) - io.cpu.resp.bits.data_word_bypass := Mux(probed, probedLoad, result.data) - io.cpu.resp.bits.data_raw := Mux(probed, probedLoad, result.data) - io.cpu.resp.bits.store_data := data - io.cpu.replay_next := false.B - // A buffered completion awaits instruction replay, not a memory transaction; release ordered decoding. - val memoryPending = state === s1 || state === s2 || state === issue || state === receive - io.cpu.ordered := !memoryPending - io.cpu.store_pending := memoryPending && writes - io.cpu.clock_enabled := true.B - io.cpu.perf := 0.U.asTypeOf(io.cpu.perf) - io.cpu.perf.grant := returns || probed - io.cpu.perf.blocked := !io.cpu.req.ready - io.cpu.uncached_resp.foreach { port => port.valid := false.B; port.bits := 0.U.asTypeOf(port.bits) } -} diff --git a/examples/cores/rocket/arch/src/main/scala/cpu/PermissionCheck.scala b/examples/cores/rocket/arch/src/main/scala/cpu/PermissionCheck.scala deleted file mode 100644 index 407193b33..000000000 --- a/examples/cores/rocket/arch/src/main/scala/cpu/PermissionCheck.scala +++ /dev/null @@ -1,86 +0,0 @@ -package hier.core.rocket - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import freechips.rocketchip.rocket.{PMP, PMPChecker} -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.preflight.{Authorization, Permission, Params => PreparationParams} - -/** Checks the complete transport range against the submitting command's frozen PMP image. */ -@instantiable -class PermissionCheck(config: PreparationParams, regions: Seq[PhysicalRegion])(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - val physicalBits = math.min(config.bus.addressBits, paddrBits) - require(regions.nonEmpty) - regions.foreach(r => require(r.base >= 0 && r.bytes > 0 && r.base + r.bytes <= (BigInt(1) << physicalBits))) - - @public - val io = IO(new Bundle { - val request = Flipped(Decoupled(new Authorization(config))) - // Selected from the captured admission context using request.bits.id, never the live CPU CSRs. - val contextValid = Input(Bool()) - val contextId = Input(UInt(config.idBits.W)) - val pmp = Input(Vec(nPMPs, new PMP)) - val response = Decoupled(new Permission(config)) - }) - - val idle :: checking :: reporting :: Nil = Enum(3) - val state = RegInit(idle) - val request = Reg(new Authorization(config)) - val capturedPmp = Reg(Vec(nPMPs, new PMP)) - val address = Reg(UInt(config.bus.addressBits.W)) - val remaining = Reg(UInt(13.W)) - val allowed = Reg(Bool()) - io.request.ready := state === idle && !reset.asBool - io.response.valid := state === reporting && !reset.asBool - io.response.bits.id := request.id - io.response.bits.allow := allowed - - when(io.request.fire) { - val r = io.request.bits - val end = r.pa +& r.bytes - val shape = io.contextValid && io.contextId === r.id && r.bytes =/= 0.U && r.bytes <= 4096.U && - end <= (BigInt(1) << physicalBits).U && - (r.privilege === 0.U || r.privilege === 1.U || r.privilege === 3.U) && - (!r.isPte || (!r.write && r.bytes === 8.U && r.privilege === 1.U)) - val regionAllowed = regions.map { region => - val permission = Mux(r.write, region.writable.B, region.readable.B) - region.normal.B && r.pa >= region.base.U && end <= (region.base + region.bytes).U && permission - }.reduce(_ || _) - request := r - capturedPmp := io.pmp - address := r.pa - remaining := r.bytes - allowed := shape && regionAllowed - state := Mux(shape && regionAllowed, checking, reporting) - } - - // A page segment may cover different PMP entries. Check each actual transport - // transaction (at most one 64-byte line), never a widened page-sized access. - val size = (1 to 6).foldLeft(0.U(3.W)) { (selected, logBytes) => - Mux(remaining >= (1 << logBytes).U && address(logBytes - 1, 0) === 0.U, logBytes.U, selected) - } - - val bytes = (1.U(13.W) << size)(12, 0) - val checker = Module(new PMPChecker(6)) - checker.io.pmp := capturedPmp - checker.io.prv := request.privilege - checker.io.addr := address.pad(paddrBits)(paddrBits - 1, 0) - checker.io.size := size - val permitted = Mux(request.write, checker.io.w, checker.io.r) - when(state === checking) { - when(!permitted) { - allowed := false.B - state := reporting - }.elsewhen(remaining === bytes) { - state := reporting - }.otherwise { - address := address + bytes - remaining := remaining - bytes - } - } - when(io.response.fire)(state := idle) -} diff --git a/examples/cores/rocket/arch/src/main/scala/cpu/Preparation.scala b/examples/cores/rocket/arch/src/main/scala/cpu/Preparation.scala deleted file mode 100644 index dc57d1106..000000000 --- a/examples/cores/rocket/arch/src/main/scala/cpu/Preparation.scala +++ /dev/null @@ -1,77 +0,0 @@ -package hier.core.rocket - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import freechips.rocketchip.rocket.PMP -import memcore.bus.chi.rnf.{CacheAccess, CacheResult} -import memcore.memory.cpu.PhysicalRegion -import memcore.memory.preflight.{ - Command, - PreparedSegment, - Preflight, - PreparedMap, - MapQuery, - MapResult, - MapReady, - MapTag, - Params => PreparationParams -} - -/** Uses the admission snapshot and retains translations until DMA and cache completion. */ -@instantiable -class Preparation(config: PreparationParams, regions: Seq[PhysicalRegion])(implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters { - - @public - val io = IO(new Bundle { - val command = Flipped(Decoupled(new Command(config))) - // AdmissionBridge owns the frozen context. Query it by tag instead of duplicating its table. - val contextTag = Output(UInt(config.idBits.W)) - val contextValid = Input(Bool()) - val contextId = Input(UInt(config.idBits.W)) - val contextPmp = Input(Vec(nPMPs, new PMP)) - // Every successful segment is delivered both to the retained map and the physical interlock. - // A terminal error goes to the caller too, which must cancel its unsealed reservation. - val ranges = Decoupled(new PreparedSegment(config)) - val ready = Decoupled(new MapReady(config)) - val release = Flipped(Decoupled(new MapTag(config))) - val queries = Input(Vec(2, new MapQuery(config))) - val results = Output(Vec(2, new MapResult(config))) - val pteRequest = Decoupled(new CacheAccess(config.bus)) - val pteResponse = Flipped(Decoupled(new CacheResult)) - }) - - val preflight: Instance[Preflight] = Instantiate(new Preflight(config)) - val maps: Instance[PreparedMap] = Instantiate(new PreparedMap(config)) - val permission: Instance[PermissionCheck] = Instantiate(new PermissionCheck(config, regions)) - preflight.io.command.valid := io.command.valid && maps.io.reserve.ready && !reset.asBool - preflight.io.command.bits := io.command.bits - maps.io.reserve.valid := io.command.valid && preflight.io.command.ready && !reset.asBool - maps.io.reserve.bits.id := io.command.bits.id - io.command.ready := preflight.io.command.ready && maps.io.reserve.ready && !reset.asBool - assert(preflight.io.command.fire === io.command.fire, "Preparation command reservation is not atomic") - assert(maps.io.reserve.fire === io.command.fire, "Preparation map reservation is not atomic") - - val authorization = preflight.io.authorization - io.contextTag := authorization.bits.id - permission.io.request <> authorization - permission.io.contextValid := io.contextValid - permission.io.contextId := io.contextId - permission.io.pmp := io.contextPmp - preflight.io.permission <> permission.io.response - io.pteRequest <> preflight.io.pteRequest - preflight.io.pteResponse <> io.pteResponse - - maps.io.prepared.valid := preflight.io.prepared.valid && io.ranges.ready - maps.io.prepared.bits := preflight.io.prepared.bits - io.ranges.valid := preflight.io.prepared.valid && maps.io.prepared.ready - io.ranges.bits := preflight.io.prepared.bits - preflight.io.prepared.ready := maps.io.prepared.ready && io.ranges.ready - io.ready <> maps.io.ready - maps.io.queries := io.queries - io.results := maps.io.results - maps.io.release <> io.release -} diff --git a/examples/cores/rocket/arch/src/main/scala/cpu/Rocket.scala b/examples/cores/rocket/arch/src/main/scala/cpu/Rocket.scala deleted file mode 100644 index 8afa51712..000000000 --- a/examples/cores/rocket/arch/src/main/scala/cpu/Rocket.scala +++ /dev/null @@ -1,655 +0,0 @@ -package hier.core.rocket - -import chisel3._ -import chisel3.util._ -import framework.system.core.clink.{CLink, HasCLink} -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} -import framework.system.core.rocket.{CpuParams, HasCpuParameters} -import freechips.rocketchip.tile.{CustomCSR, FPU, FPUCoreIO, TraceBundle} -import freechips.rocketchip.rocket.{CSRs, PMPChecker, PRV} -import framework.system.core.rocket.{RoCCCommandBB, RoCCResponseBB, RocketBB} -import memcore.memory.fetch.{Fetch, Params => FetchParams} -import memcore.memory.cpu.{CpuMemParams, PhysicalRegion, VirtualMemory} -import memcore.bus.chi.RequesterPort -import memcore.bus.chi.rnf.{BankedChiCache, CacheAccess, CacheAtomic, CacheResult, RnfParams} -import memcore.memory.interlock.{ - CpuQuery, - Dispatch, - Tag, - Acknowledgement, - Maintenance => MaintenanceRange, - Params => TrackingParams -} - -case class Commands(compute: Boolean, scheduler: Boolean, tracking: TrackingParams = TrackingParams()) { - require(compute || scheduler) -} - -class CommandSnapshot(tracking: TrackingParams, pmps: Int)(implicit val cpuParams: CpuParams) - extends Bundle - with HasCpuParameters { - val tag = UInt(tracking.idBits.W) - val instruction = new RoCCCommandBB - val satp = UInt(64.W) - val effectivePrivilege = UInt(2.W) - val sum = Bool() - val mxr = Bool() - val pmp = Vec(pmps, new freechips.rocketchip.rocket.PMP) -} - -class AdmissionPorts(tracking: TrackingParams, pmps: Int, bus: memcore.bus.chi.Params)(implicit val cpuParams: CpuParams) - extends Bundle - with HasCpuParameters { - val reserve = Decoupled(new Dispatch(tracking)) - val command = Decoupled(new CommandSnapshot(tracking, pmps)) - val complete = Flipped(Decoupled(new Tag(tracking))) - val cancelled = Flipped(Decoupled(new Tag(tracking))) - val response = Flipped(Decoupled(new RoCCResponseBB)) - val interrupt = Input(Bool()) - val maintenance = Flipped(Decoupled(new MaintenanceRange(tracking))) - val maintained = Decoupled(new Acknowledgement(tracking)) - val cpuQuery = Output(new CpuQuery(tracking)) - val cpuAllow = Input(Bool()) - val cpuProbeAllow = Input(Bool()) - val pteRequest = Flipped(Decoupled(new CacheAccess(bus))) - val pteResponse = Decoupled(new CacheResult) - // Ownership registration only; TaskController must also join actual accelerator/DMA idle. - val outstanding = Output(UInt(log2Ceil(tracking.entries + 1).W)) -} - -class CpuCLink(config: RnfParams, commands: Option[Commands])(implicit val cpuParams: CpuParams) - extends CLink - with HasCpuParameters { - val cp = CpuMemParams(config.chi, tagBits = 6) - val resetVector = Input(UInt(64.W)) - val time = Input(UInt(64.W)) - val timerInterrupt = Input(Bool()) - val softwareInterrupt = Input(Bool()) - val externalInterrupt = Input(Bool()) - val supervisorExternalInterrupt = Input(Bool()) - val hartId = Input(UInt(hartIdLen.W)) - val chi = new RequesterPort(config.chi) - val instructionChi = new RequesterPort(config.chi) - val uncachedRequest = Decoupled(new memcore.memory.cpu.UncachedRequest(cp)) - val uncachedResponse = Flipped(Decoupled(new memcore.memory.cpu.UncachedResponse(cp))) - val trace = Output(new TraceBundle) - val retired = Output(Bool()) - val retiredPc = Output(UInt(64.W)) - val trapped = Output(Bool()) - val trapCause = Output(UInt(64.W)) - val trapValue = Output(UInt(64.W)) - val trapPc = Output(UInt(64.W)) - val cancelledData = Output(Bool()) - val admission = commands.map(mode => new AdmissionPorts(mode.tracking, nPMPs, config.chi)) -} - -/** One integer Rocket, own instruction/data memory, and explicit physical permission checks. */ -@instantiable -class Core( - config: RnfParams, - instruction: RnfParams, - regions: Seq[PhysicalRegion], - commands: Option[Commands] = None -)( - implicit val cpuParams: CpuParams) - extends Module - with HasCpuParameters - with HasCLink { - override def desiredName: String = "RocketCore" - require(!usingHypervisor && usingCompressed && fetchBytes == 4) - require(!usingFPU || fLen <= 64, "Core memory data path supports floating-point formats up to 64 bits") - require(instruction.chi == config.chi && instruction.nodeId != config.nodeId) - val cp = CpuMemParams(config.chi, tagBits = 6) - - @public val clink = IO(new CpuCLink(config, commands)) - - val cpu: Instance[RocketBB] = - Instantiate(new RocketBB( - Seq(CustomCSR(CSRs.time, (BigInt(1) << 64) - 1, Some(BigInt(0)))), - commands.exists(_.compute), - commands.exists(_.scheduler), - false, - false, - false - )) - - val fetch: Instance[Fetch] = Instantiate(new Fetch(FetchParams())) - val lsu: Instance[Lsu] = Instantiate(new Lsu(cp)) - val system: Instance[VirtualMemory] = Instantiate(new VirtualMemory(cp, regions, 0, 16)) - // Instruction words have their own translation, PMP check and coherent L1, apart from the data broker. - val instructionSystem: Instance[VirtualMemory] = - Instantiate(new VirtualMemory(cp, regions, 512, 16)) - val instructionCache: Instance[BankedChiCache] = Instantiate(new BankedChiCache(instruction.copy(lineResult = true))) - - val cacheTracking = commands match { - case Some(mode) => mode.tracking - case None => TrackingParams(addressBits = config.chi.addressBits) - } - - val cache: Instance[Cache] = Instantiate(new Cache(config.copy(probe = true), cacheTracking)) - val maintenanceActive = RegInit(false.B) - val maintenanceBlock = WireDefault(false.B) - val cacheOffer = RegInit(false.B) - val cacheOwner = RegInit(false.B) - val offerExternal = RegInit(false.B) - val ownerExternal = RegInit(false.B) - val offerPacket = Reg(new CacheAccess(config.chi)) - val preferExternal = RegInit(false.B) - val uncachedOffer = RegInit(false.B) - val uncachedOwner = RegInit(false.B) - val uncachedPacket = Reg(new memcore.memory.cpu.UncachedRequest(cp)) - val pteDenied = RegInit(false.B) - commands match { - case Some(_) => - val port = clink.admission.get - cache.io.maintenance <> port.maintenance - port.maintained <> cache.io.maintained - maintenanceBlock := port.maintenance.valid || maintenanceActive - when(port.maintenance.fire)(maintenanceActive := true.B) - when(port.maintained.fire)(maintenanceActive := false.B) - case None => - cache.io.maintenance.valid := false.B - cache.io.maintenance.bits := 0.U.asTypeOf(cache.io.maintenance.bits) - cache.io.maintained.ready := true.B - } - cache.io.access <> system.io.cacheRequest - system.io.cacheResponse <> cache.io.result - clink.chi <> cache.io.chi - system.io.active := true.B - // SFENCE.VMA invalidates the cached data and instruction translations. - system.io.flushTranslation := lsu.io.maintenance.fire - instructionSystem.io.flushTranslation := lsu.io.maintenance.fire - clink.uncachedRequest <> system.io.uncachedRequest - system.io.uncachedResponse <> clink.uncachedResponse - clink.trace := cpu.io.trace - clink.retired := cpu.io.trace.insns(0).valid && !cpu.io.trace.insns(0).exception - clink.retiredPc := cpu.io.trace.insns(0).iaddr - clink.trapped := cpu.io.trace.insns(0).valid && cpu.io.trace.insns(0).exception - clink.trapCause := cpu.io.trace.insns(0).cause - clink.trapValue := cpu.io.trace.insns(0).tval - clink.trapPc := cpu.io.trace.insns(0).iaddr - clink.cancelledData := lsu.io.cancelled - assert(clink.resetVector(63, paddrBits) === 0.U, "Core reset vector exceeds physical width") - cpu.io.hartid := clink.hartId - cpu.io.reset_vector := clink.resetVector(paddrBits - 1, 0) - cpu.io.interrupts := 0.U.asTypeOf(cpu.io.interrupts) - cpu.io.interrupts.mtip := clink.timerInterrupt - cpu.io.interrupts.msip := clink.softwareInterrupt - cpu.io.interrupts.meip := clink.externalInterrupt - cpu.io.interrupts.seip.foreach(_ := clink.supervisorExternalInterrupt) - cpu.io.traceStall := false.B - // TIME samples the chip's CLINT clock; CSRFile enforces its read-only address and counter permissions. - cpu.io.rocc.csrs(0).stall := false.B - cpu.io.rocc.csrs(0).set := true.B - cpu.io.rocc.csrs(0).sdata := clink.time - cpu.io.ptw.perf := 0.U.asTypeOf(cpu.io.ptw.perf) - cpu.io.ptw.clock_enabled := true.B - cpu.io.ptw.customCSRs.csrs.foreach { csr => - csr.stall := false.B - csr.set := false.B - csr.sdata := 0.U - } - val fpu = coreParams.fpu.map(cfg => Module(new FPU(cfg))) - fpu match { - case Some(unit) => - cpu.io.fpu :<>= unit.io.waiveAs[FPUCoreIO](_.cp_req, _.cp_resp) - unit.io.cp_req.valid := false.B - unit.io.cp_req.bits := DontCare - unit.io.cp_resp.ready := false.B - case None => - cpu.io.fpu.store_data := 0.U - cpu.io.fpu.toint_data := 0.U - cpu.io.fpu.fcsr_flags.valid := false.B - cpu.io.fpu.fcsr_flags.bits := 0.U - cpu.io.fpu.fcsr_rdy := true.B - cpu.io.fpu.nack_mem := false.B - cpu.io.fpu.illegal_rm := false.B - cpu.io.fpu.dec := 0.U.asTypeOf(cpu.io.fpu.dec) - cpu.io.fpu.sboard_set := false.B - cpu.io.fpu.sboard_clr := false.B - cpu.io.fpu.sboard_clra := 0.U - } - cpu.io.rocc.cmd.ready := true.B - cpu.io.rocc.resp.valid := false.B - cpu.io.rocc.resp.bits := 0.U.asTypeOf(cpu.io.rocc.resp.bits) - cpu.io.rocc.busy := false.B - cpu.io.rocc.interrupt := false.B - cpu.io.rocc.mem := DontCare - - lsu.io.cpu <> cpu.io.dmem - if (commands.nonEmpty) { - // Existing LSU owners, including the replay that retires them, must keep progressing. - val blockFresh = (cpu.io.roccOlderPending || maintenanceBlock) && lsu.io.idle - lsu.io.cpu.req.valid := cpu.io.dmem.req.valid && !blockFresh - cpu.io.dmem.req.ready := lsu.io.cpu.req.ready && !blockFresh - } - lsu.io.pc := cpu.io.dmemPc - lsu.io.pmp := cpu.io.ptw.pmp - val satp = - if (usingVM) Cat(cpu.io.ptw.ptbr.mode, cpu.io.ptw.ptbr.asid.pad(16), cpu.io.ptw.ptbr.ppn.pad(44)) else 0.U(64.W) - lsu.io.context.privilege := cpu.io.ptw.status.dprv - lsu.io.context.satp := satp - lsu.io.context.sum := cpu.io.ptw.status.sum - lsu.io.context.mxr := cpu.io.ptw.status.mxr - fetch.io.resetVector := clink.resetVector - fetch.io.context.privilege := cpu.io.ptw.status.prv - val instructionTranslation = usingVM.B && cpu.io.ptw.status.prv =/= PRV.M.U && satp(63, 60) =/= 0.U - fetch.io.context.satp := Mux(instructionTranslation, satp, 0.U) - fetch.io.context.sum := instructionTranslation && cpu.io.ptw.status.sum - fetch.io.context.mxr := instructionTranslation && cpu.io.ptw.status.mxr - - // Rocket kills its IBuf on the correction cycle. Fetch receives that captured - // correction one cycle later, breaking the BTB valid/redirect feedback path. - val redirectedPc = - if (usingVM) cpu.io.imem.req.bits.pc.asSInt.pad(64).asUInt - else cpu.io.imem.req.bits.pc.pad(64) - - fetch.io.redirect.valid := RegNext(cpu.io.imem.req.valid, false.B) - fetch.io.redirect.bits := RegEnable(redirectedPc, cpu.io.imem.req.valid) - fetch.io.flush := RegNext(cpu.io.imem.flush_icache, false.B) - cpu.io.imem.resp.valid := fetch.io.packet.valid - fetch.io.packet.ready := cpu.io.imem.resp.ready - cpu.io.imem.resp.bits := 0.U.asTypeOf(cpu.io.imem.resp.bits) - cpu.io.imem.resp.bits.pc := fetch.io.packet.bits.pc - cpu.io.imem.resp.bits.data := fetch.io.packet.bits.data - cpu.io.imem.resp.bits.mask := fetch.io.packet.bits.mask - cpu.io.imem.resp.bits.xcpt.pf.inst := fetch.io.packet.bits.pageFault - cpu.io.imem.resp.bits.xcpt.ae.inst := fetch.io.packet.bits.accessFault - cpu.io.imem.npc := fetch.io.npc - cpu.io.imem.clock_enabled := true.B - cpu.io.imem.perf := 0.U.asTypeOf(cpu.io.imem.perf) - cpu.io.imem.gpa.valid := false.B - cpu.io.imem.gpa.bits := 0.U - cpu.io.imem.gpa_is_pte := false.B - - // Fetch prepares its context one edge before its word offer becomes visible. - val effectivePmp = WireDefault(cpu.io.ptw.pmp) - for (i <- 0 until nPMPs) { - val active = cpu.io.ptw.pmp(i).cfg.a =/= 0.U - val torLowerBound = if (i + 1 < nPMPs) cpu.io.ptw.pmp(i + 1).tor else false.B - effectivePmp(i).cfg.res := 0.U - when(!active) { - effectivePmp(i).cfg.r := false.B - effectivePmp(i).cfg.w := false.B - effectivePmp(i).cfg.x := false.B - effectivePmp(i).mask := 0.U - } - when(!active && !torLowerBound)(effectivePmp(i).addr := 0.U) - } - val previousPmp = RegNext(effectivePmp) - fetch.io.invalidate := RegNext(cpu.io.imem.sfence.valid, false.B) || (effectivePmp.asUInt =/= previousPmp.asUInt) - val instructionLineExecutable = RegInit(false.B) - val fetchOffer = RegNext(fetch.io.request.valid, false.B) - val fetchPmp = Reg(Vec(nPMPs, new freechips.rocketchip.rocket.PMP)) - when(fetch.io.request.valid && !fetchOffer)(fetchPmp := previousPmp) - val wordPmp = Mux(fetchOffer, fetchPmp, previousPmp) - - instructionSystem.io.active := true.B - instructionSystem.io.request.valid := fetch.io.request.valid - instructionSystem.io.request.bits := 0.U.asTypeOf(instructionSystem.io.request.bits) - instructionSystem.io.request.bits.vaddr := fetch.io.request.bits.addr - instructionSystem.io.request.bits.size := 3.U - instructionSystem.io.request.bits.execute := true.B - instructionSystem.io.request.bits.privilege := fetch.io.request.bits.context.privilege - instructionSystem.io.request.bits.sum := fetch.io.request.bits.context.sum - instructionSystem.io.request.bits.mxr := fetch.io.request.bits.context.mxr - instructionSystem.io.request.bits.satpMode := fetch.io.request.bits.context.satp(63, 60) - instructionSystem.io.request.bits.rootPpn := fetch.io.request.bits.context.satp(43, 0) - fetch.io.request.ready := instructionSystem.io.request.ready - fetch.io.response.valid := instructionSystem.io.response.valid - fetch.io.response.bits.data := instructionSystem.io.response.bits.data - fetch.io.response.bits.line := instructionSystem.io.response.bits.line - fetch.io.response.bits.lineExecutable := instructionLineExecutable - fetch.io.response.bits.pageFault := instructionSystem.io.response.bits.pageFault - fetch.io.response.bits.accessFault := instructionSystem.io.response.bits.accessFault - instructionSystem.io.response.ready := fetch.io.response.ready - when(instructionSystem.io.response.valid) { - assert(!instructionSystem.io.response.bits.misaligned, "Aligned instruction word returned misalignment") - } - instructionCache.io.access <> instructionSystem.io.cacheRequest - instructionSystem.io.cacheResponse <> instructionCache.io.result - clink.instructionChi <> instructionCache.io.chi - - val instructionPmp = Reg(Vec(nPMPs, new freechips.rocketchip.rocket.PMP)) - val instructionAuthorization = Module(new PMPChecker(3)) - val instructionLineAuthorization = Module(new PMPChecker(6)) - instructionLineAuthorization.io.pmp := instructionPmp - instructionLineAuthorization.io.prv := instructionSystem.io.authorizationRequest.bits.privilege - val instructionPhysicalLine = (instructionSystem.io.authorizationRequest.bits.paddr >> 6) << 6 - instructionLineAuthorization.io.addr := instructionPhysicalLine(paddrBits - 1, 0) - instructionLineAuthorization.io.size := 6.U - - val instructionLineRegion = regions.filter(r => r.cacheable && r.normal && r.executable).map(r => - instructionPhysicalLine.pad(65) >= r.base.U(65.W) && - instructionPhysicalLine.pad(65) + 63.U < (r.base + r.bytes).U(65.W) - ).foldLeft(false.B)(_ || _) - - val instructionPending = RegInit(false.B) - val instructionAllowed = Reg(Bool()) - when(instructionSystem.io.request.fire)(instructionPmp := wordPmp) - instructionAuthorization.io.pmp := instructionPmp - instructionAuthorization.io.prv := instructionSystem.io.authorizationRequest.bits.privilege - instructionAuthorization.io.addr := instructionSystem.io.authorizationRequest.bits.paddr(paddrBits - 1, 0) - instructionAuthorization.io.size := instructionSystem.io.authorizationRequest.bits.size - instructionSystem.io.authorizationRequest.ready := !instructionPending - instructionSystem.io.authorizationResponse.valid := instructionPending - instructionSystem.io.authorizationResponse.bits := instructionAllowed - when(instructionSystem.io.authorizationRequest.fire) { - val r = instructionSystem.io.authorizationRequest.bits - instructionAllowed := !r.paddr(63, paddrBits).orR && - (!r.read || instructionAuthorization.io.r) && (!r.execute || instructionAuthorization.io.x) - instructionLineExecutable := !r.paddr( - 63, - paddrBits - ).orR && instructionLineAuthorization.io.x && instructionLineRegion - instructionPending := true.B - } - when(instructionSystem.io.authorizationResponse.fire)(instructionPending := false.B) - - // Instructions are fetched only through the cacheable L1; an uncached executable word faults. - val instructionUncached = RegInit(false.B) - val instructionUncachedTag = Reg(UInt(cp.tagBits.W)) - instructionSystem.io.uncachedRequest.ready := !instructionUncached - instructionSystem.io.uncachedResponse.valid := instructionUncached - instructionSystem.io.uncachedResponse.bits.tag := instructionUncachedTag - instructionSystem.io.uncachedResponse.bits.data := 0.U - instructionSystem.io.uncachedResponse.bits.error := true.B - when(instructionSystem.io.uncachedRequest.fire) { - instructionUncached := true.B - instructionUncachedTag := instructionSystem.io.uncachedRequest.bits.tag - } - when(instructionSystem.io.uncachedResponse.fire)(instructionUncached := false.B) - - val idle :: data :: barrier :: translationBarrier :: Nil = Enum(4) - val state = RegInit(idle) - cpu.io.interruptReady := state === idle && lsu.io.idle && !cpu.io.dmem.req.valid - val authorizationPmp = Reg(Vec(nPMPs, new freechips.rocketchip.rocket.PMP)) - val authorization = Module(new PMPChecker(3)) - authorization.io.pmp := authorizationPmp - authorization.io.prv := system.io.authorizationRequest.bits.privilege - authorization.io.addr := system.io.authorizationRequest.bits.paddr(paddrBits - 1, 0) - authorization.io.size := system.io.authorizationRequest.bits.size - val authorizationPending = RegInit(false.B) - val authorizationAllowed = Reg(Bool()) - system.io.authorizationRequest.ready := !authorizationPending - system.io.authorizationResponse.valid := authorizationPending - system.io.authorizationResponse.bits := authorizationAllowed - when(system.io.authorizationRequest.fire) { - val r = system.io.authorizationRequest.bits - authorizationAllowed := !r.paddr(63, paddrBits).orR && - (!r.read || authorization.io.r) && (!r.write || authorization.io.w) && - (!r.execute || authorization.io.x) - authorizationPending := true.B - } - when(system.io.authorizationResponse.fire)(authorizationPending := false.B) - - // Data fast path: the LSU's plain access completes in s2 when a data-TLB translation (or - // identity), permission, region, NPU interlock and an L1 hit all agree while the slow broker - // path is fully idle. A TLB miss or fault leaves the access to the broker, which walks. - val probe = lsu.io.probe - system.io.lookup.config.mode := probe.req.bits.satp(63, 60) - system.io.lookup.config.rootPpn := probe.req.bits.satp(43, 0) - system.io.lookup.req.vaddr := probe.req.bits.addr - system.io.lookup.req.write := probe.req.bits.write - system.io.lookup.req.execute := false.B - system.io.lookup.req.privilege := probe.req.bits.privilege - system.io.lookup.req.sum := probe.req.bits.sum - system.io.lookup.req.mxr := probe.req.bits.mxr - instructionSystem.io.lookup.config := 0.U.asTypeOf(instructionSystem.io.lookup.config) - instructionSystem.io.lookup.req := 0.U.asTypeOf(instructionSystem.io.lookup.req) - val probeAddr = system.io.lookup.paddr.pad(64) - val probeBytes = (1.U(4.W) << probe.req.bits.size)(3, 0) - val probeLast = probeAddr.pad(65) + probeBytes - 1.U - - def probeRegion( - write: Boolean - ): Bool = regions.filter(r => r.cacheable && r.normal && (if (write) r.writable else r.readable)).map(r => - probeAddr.pad(65) >= r.base.U(65.W) && probeLast < (r.base + r.bytes).U(65.W) - ) - .foldLeft(false.B)(_ || _) - - val probePmp = Module(new PMPChecker(3)) - probePmp.io.pmp := lsu.io.capturedPmp - probePmp.io.prv := probe.req.bits.privilege - probePmp.io.addr := probeAddr(paddrBits - 1, 0) - probePmp.io.size := probe.req.bits.size - val probePermitted = system.io.lookup.hit && !probeAddr(63, paddrBits).orR && - Mux(probe.req.bits.write, probePmp.io.w && probeRegion(true), probePmp.io.r && probeRegion(false)) - val slowIdle = state === idle && !authorizationPending && !cacheOffer && !cacheOwner && - !uncachedOffer && !uncachedOwner && !pteDenied && !maintenanceBlock - val probeQuery = probe.active && probePermitted && slowIdle - val probeAllow = WireDefault(true.B) - val l1Probe = cache.io.probe.get - val probeAllowed = probePermitted && slowIdle && probeAllow - l1Probe.req.valid := probe.req.valid && probeAllowed - l1Probe.req.bits.addr := probeAddr(config.chi.addressBits - 1, 0) - l1Probe.req.bits.write := probe.req.bits.write - l1Probe.req.bits.data := probe.req.bits.data << (probeAddr(2, 0) << 3) - l1Probe.req.bits.mask := (MuxLookup(probe.req.bits.size, 255.U(8.W))(Seq( - 0.U -> 1.U(8.W), - 1.U -> 3.U(8.W), - 2.U -> 15.U(8.W) - )) << - probeAddr(2, 0))(7, 0) - probe.req.ready := l1Probe.req.ready && probeAllowed - l1Probe.cancel := probe.cancel || (probe.active && !probeAllowed) - l1Probe.retire := probe.retire && probeAllowed && !probe.cancel - probe.cancelled := l1Probe.cancel - probe.complete.valid := l1Probe.complete.valid && probeAllowed && !probe.cancel - probe.complete.bits.hit := l1Probe.complete.bits.hit - probe.complete.bits.value := l1Probe.complete.bits.value - l1Probe.complete.ready := probe.complete.ready && probeAllowed && !probe.cancel - // Broker state covers the full accepted VM/PTE/data/fetch request through its response. - // The wrapper adds L1 outstanding==0; CMO activity/ACK must not gate its own drain. - val originalIdleDrain = state === idle && lsu.io.idle - val blockedPhysicalQuery = WireDefault(false.B) - cache.io.olderRequestsDrained := (originalIdleDrain || blockedPhysicalQuery) && !probe.active && - !authorizationPending && !cacheOffer && !cacheOwner && !uncachedOffer && !uncachedOwner && !pteDenied - commands.foreach { mode => - val port = clink.admission.get - val tracking = mode.tracking - require(tracking.addressBits == config.chi.addressBits) - require(tracking.idBits >= log2Ceil(tracking.entries)) - val live = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) - val sent = RegInit(VecInit(Seq.fill(tracking.entries)(false.B))) - val pending = RegInit(false.B) - val snapshot = Reg(new CommandSnapshot(tracking, nPMPs)) - val free = VecInit(live.map(!_)).asUInt - val selected = PriorityEncoder(free) - val olderDrained = state === idle && lsu.io.idle && !authorizationPending && - cache.io.outstanding === 0.U && !cacheOffer && !cacheOwner && !uncachedOffer && !uncachedOwner && !pteDenied - val capacity = free.orR && !pending && olderDrained && !maintenanceBlock - // Interlock ready is independent of valid. Pulse reserve only for an actual atomic WB acceptance; - // Rocket's pre-acceptance WB replay may withdraw its offer and must not create an orphan reservation. - cpu.io.rocc.cmd.ready := capacity && port.reserve.ready && - (!cpu.io.rocc.cmd.valid || cpu.io.roccCommitEligible) - port.reserve.valid := cpu.io.rocc.cmd.valid && cpu.io.roccCommitEligible && capacity && port.reserve.ready - port.reserve.bits.id := selected - assert(cpu.io.rocc.cmd.fire === port.reserve.fire, "Admission reservation must match Rocket command acceptance") - port.command.valid := pending - port.command.bits := snapshot - when(port.reserve.fire) { - snapshot.tag := selected - snapshot.instruction := cpu.io.rocc.cmd.bits - snapshot.satp := satp - // Like BEMU, an M-mode command translates through an enabled satp as S-mode. - snapshot.effectivePrivilege := Mux( - cpu.io.ptw.status.dprv === PRV.M.U && satp(63, 60) =/= 0.U, - PRV.S.U, - cpu.io.ptw.status.dprv - ) - snapshot.sum := cpu.io.ptw.status.sum - snapshot.mxr := cpu.io.ptw.status.mxr - snapshot.pmp := cpu.io.ptw.pmp - live(selected) := true.B - sent(selected) := false.B - pending := true.B - } - val tagIndexBits = log2Ceil(tracking.entries) - def index(tag: UInt): UInt = tag(tagIndexBits - 1, 0) - when(port.command.fire) { pending := false.B; sent(index(snapshot.tag)) := true.B } - val completeFound = port.complete.bits.tag < tracking.entries.U && live(index(port.complete.bits.tag)) && sent( - index(port.complete.bits.tag) - ) - val cancelledFound = port.cancelled.bits.tag < tracking.entries.U && live(index(port.cancelled.bits.tag)) && sent( - index(port.cancelled.bits.tag) - ) - port.complete.ready := completeFound - port.cancelled.ready := cancelledFound && !(port.complete.valid && port.complete.bits.tag === port.cancelled.bits.tag) - when(port.complete.valid)(assert(completeFound, "Admission completion must identify a delivered live command")) - when(port.cancelled.valid) { - assert(cancelledFound, "Admission cancellation must identify a delivered live command") - assert( - !port.complete.valid || port.complete.bits.tag =/= port.cancelled.bits.tag, - "Admission cannot complete and cancel the same command" - ) - } - when(port.complete.fire) { - live(index(port.complete.bits.tag)) := false.B; sent(index(port.complete.bits.tag)) := false.B - } - when(port.cancelled.fire) { - live(index(port.cancelled.bits.tag)) := false.B; sent(index(port.cancelled.bits.tag)) := false.B - } - port.outstanding := PopCount(live) - cpu.io.rocc.resp <> port.response - cpu.io.rocc.interrupt := port.interrupt - // Older accepted broker/PTW/LSU owners continue. Fresh EX memory and fresh word offers - // are stopped above/below until the MEM/WB command has reserved its identity. - val access = system.io.cacheRequest.bits - val uncached = system.io.uncachedRequest - port.cpuQuery.valid := (system.io.cacheRequest.valid && access.atomic =/= CacheAtomic.Fence.U) || - uncached.valid || probeQuery - port.cpuQuery.paddr := Mux( - probeQuery, - probeAddr(config.chi.addressBits - 1, 0), - Mux(uncached.valid, uncached.bits.addr(config.chi.addressBits - 1, 0), access.addr) - ) - port.cpuQuery.sizeLog2 := Mux( - probeQuery, - probe.req.bits.size, - Mux(uncached.valid, uncached.bits.size, Mux(access.atomicWord, 2.U, 3.U)) - ) - port.cpuQuery.write := Mux( - probeQuery, - probe.req.bits.write, - Mux( - uncached.valid, - uncached.bits.write, - access.write || access.atomic === CacheAtomic.SC.U || - (access.atomic >= CacheAtomic.Swap.U && access.atomic <= CacheAtomic.MaxU.U) - ) - ) - probeAllow := port.cpuProbeAllow - port.cpuQuery.olderDispatchPending := false.B - // Reservation itself drained all old owners; an unissued blocked physical - // query is younger and must not hold up the reservation's maintenance. - blockedPhysicalQuery := (system.io.cacheRequest.valid || uncached.valid) && !port.cpuAllow - val legalPte = !port.pteRequest.bits.write && port.pteRequest.bits.atomic === CacheAtomic.None.U && - !port.pteRequest.bits.atomicWord && port.pteRequest.bits.addr(2, 0) === 0.U && - port.pteRequest.bits.mask === 0.U - when(port.pteRequest.valid) { - assert(legalPte, "Core external PTE request must be an aligned read with the Walker mask") - } - val systemEligible = system.io.cacheRequest.valid && port.cpuAllow && - (!maintenanceBlock || state === data) - // Preparation already authorized this PA using its retained command context. - // Region classification additionally forbids device reads, including PTE reads. - val pteAddr = port.pteRequest.bits.addr.pad(65) - val pteCached = regions.filter(r => r.normal && r.readable && r.cacheable).map(r => - pteAddr >= r.base.U(65.W) && pteAddr + 8.U <= (r.base + r.bytes).U(65.W) - ).foldLeft(false.B)(_ || _) - val pteBusy = (cacheOffer && offerExternal) || (cacheOwner && ownerExternal) || pteDenied - val pteAvailable = port.pteRequest.valid && legalPte && !maintenanceBlock && !pteBusy - val pteEligible = pteAvailable && pteCached - val selectPte = pteEligible && (!systemEligible || preferExternal) - when(!cacheOffer && !cacheOwner && (systemEligible || pteEligible)) { - offerPacket := Mux(selectPte, port.pteRequest.bits, access) - offerExternal := selectPte - cacheOffer := true.B - } - cache.io.access.valid := cacheOffer - cache.io.access.bits := offerPacket - system.io.cacheRequest.ready := cacheOffer && !offerExternal && cache.io.access.ready - port.pteRequest.ready := cacheOffer && offerExternal && cache.io.access.ready - when(cache.io.access.fire) { - cacheOffer := false.B - cacheOwner := true.B - ownerExternal := offerExternal - preferExternal := !offerExternal - } - system.io.cacheResponse.valid := cacheOwner && !ownerExternal && cache.io.result.valid - system.io.cacheResponse.bits := cache.io.result.bits - port.pteResponse.valid := cacheOwner && ownerExternal && cache.io.result.valid - port.pteResponse.bits := cache.io.result.bits - cache.io.result.ready := cacheOwner && Mux(ownerExternal, port.pteResponse.ready, system.io.cacheResponse.ready) - when(cache.io.result.fire)(cacheOwner := false.B) - val uncachedCpuEligible = uncached.valid && port.cpuAllow && - (!maintenanceBlock || state === data) - when(!uncachedOffer && !uncachedOwner && uncachedCpuEligible) { - uncachedPacket := uncached.bits - uncachedOffer := true.B - } - clink.uncachedRequest.valid := uncachedOffer - clink.uncachedRequest.bits := uncachedPacket - uncached.ready := uncachedOffer && clink.uncachedRequest.ready - when(clink.uncachedRequest.fire) { - uncachedOffer := false.B - uncachedOwner := true.B - } - system.io.uncachedResponse.valid := uncachedOwner && clink.uncachedResponse.valid - system.io.uncachedResponse.bits := clink.uncachedResponse.bits - clink.uncachedResponse.ready := uncachedOwner && system.io.uncachedResponse.ready - when(clink.uncachedResponse.valid) { - assert(uncachedOwner, "Core uncached response requires an accepted owner") - assert(clink.uncachedResponse.bits.tag === uncachedPacket.tag, "Core uncached response tag must match its owner") - } - when(clink.uncachedResponse.fire)(uncachedOwner := false.B) - when(pteAvailable && !pteCached) { - port.pteRequest.ready := true.B - when(port.pteRequest.fire)(pteDenied := true.B) - } - when(pteDenied) { - port.pteResponse.valid := true.B - port.pteResponse.bits.data := 0.U - port.pteResponse.bits.error := true.B - when(port.pteResponse.fire)(pteDenied := false.B) - } - } - - lsu.io.maintenance.ready := false.B - lsu.io.maintained.valid := state === translationBarrier - lsu.io.maintained.bits := true.B - lsu.io.request.ready := false.B - lsu.io.response.valid := false.B - lsu.io.response.bits := system.io.response.bits - system.io.request.valid := false.B - system.io.request.bits := lsu.io.request.bits - system.io.response.ready := false.B - fetch.io.maintenance.ready := false.B - fetch.io.maintained.valid := state === barrier - fetch.io.maintained.bits := true.B - when(state === idle) { - when(lsu.io.maintenance.valid && cache.io.outstanding === 0.U) { - // No translation cache exists: draining the prior broker transaction is the SFENCE boundary. - lsu.io.maintenance.ready := true.B - when(lsu.io.maintenance.fire)(state := translationBarrier) - }.elsewhen(lsu.io.request.valid) { - system.io.request.valid := true.B - lsu.io.request.ready := system.io.request.ready - when(system.io.request.fire) { - authorizationPmp := lsu.io.capturedPmp - state := data - } - }.elsewhen(fetch.io.maintenance.valid && lsu.io.idle && cache.io.outstanding === 0.U) { - // Drained stores own their lines in the data L1; the coherent instruction L1 is snooped on those writes. - fetch.io.maintenance.ready := true.B - when(fetch.io.maintenance.fire)(state := barrier) - } - } - when(state === data) { - lsu.io.response.valid := system.io.response.valid - system.io.response.ready := lsu.io.response.ready - when(system.io.response.fire)(state := idle) - } - when(fetch.io.maintained.fire || lsu.io.maintained.fire)(state := idle) -} diff --git a/examples/cores/rocket/compiler/CMakeLists.txt b/examples/cores/rocket/compiler/CMakeLists.txt deleted file mode 100644 index 95412c193..000000000 --- a/examples/cores/rocket/compiler/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -# Rocket has no Ball lowering policy. Reuse the Ball-free compiler base rather -# than importing another Core's dialect or lowering implementation. -set(BB_CORE_NAME rocket) -set(BB_CORE_TARGET_PREFIX "") - -get_filename_component(_BB_REPO_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../../.." ABSOLUTE) -add_subdirectory("${_BB_REPO_ROOT}/examples/cores/toy/compiler/src" - "${CMAKE_CURRENT_BINARY_DIR}/base") diff --git a/examples/cores/rocket/configs/balldomains/default.toml b/examples/cores/rocket/configs/balldomains/default.toml deleted file mode 100644 index a23e23157..000000000 --- a/examples/cores/rocket/configs/balldomains/default.toml +++ /dev/null @@ -1,9 +0,0 @@ -# Rocket-only core: no Buckyball balls. - -ballNum = 0 - -ballIdMappings = [ -] - -ballISA = [ -] diff --git a/examples/cores/rocket/configs/cpu/default.toml b/examples/cores/rocket/configs/cpu/default.toml deleted file mode 100644 index e651581c6..000000000 --- a/examples/cores/rocket/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "rocket" -config = "./rocket/default.toml" diff --git a/examples/cores/rocket/configs/cpu/rocket/default.toml b/examples/cores/rocket/configs/cpu/rocket/default.toml deleted file mode 100644 index f423a2a88..000000000 --- a/examples/cores/rocket/configs/cpu/rocket/default.toml +++ /dev/null @@ -1,76 +0,0 @@ -useUser = true -useSupervisor = true -useHypervisor = false -useDebug = false -useAtomics = true -useAtomicsOnlyForIO = false -useCompressed = true -useRVE = false -useConditionalZero = false -nLocalInterrupts = 0 -useNMI = false -nBreakpoints = 1 -useBPWatch = false -mcontextWidth = 0 -scontextWidth = 0 -nPerfCounters = 0 -haveBasicCounters = true -misaWritable = true -mtvecInitEnable = true -mtvecInit = 0 -mtvecWritable = true -fastLoadWord = true -fastLoadByte = false -branchPredictionModeCSR = false -clockGate = false -mvendorid = 0 -mimpid = 538447876 -haveCease = false -haveSimTimeout = false -useVM = true -useZba = true -useZbb = true -useZbs = true -haveCFlush = false - -[mulDiv] -divUnroll = 1 -divEarlyOutGranularity = 1 -enable = true -mulUnroll = 8 -mulEarlyOut = true -divEarlyOut = true - -[fpu] -divSqrt = true -sfmaLatency = 5 -dfmaLatency = 6 -fpmuLatency = 2 -ifpuLatency = 4 -enable = true -minFLen = 16 -fLen = 64 - -[dcache] -clockGate = false -nSets = 32 -nWays = 4 -nMSHRs = 4 - -[icache] -prefetch = false -nSets = 64 -nWays = 8 - -[btb] -nMatchBits = 14 -nPages = 6 -updatesOutOfOrder = false -bhtEnable = true -bhtEntries = 512 -bhtCounterLength = 1 -bhtHistoryLength = 8 -bhtHistoryBits = 3 -enable = true -nEntries = 28 -nRAS = 6 diff --git a/examples/cores/rocket/configs/default.toml b/examples/cores/rocket/configs/default.toml deleted file mode 100644 index 9f121207b..000000000 --- a/examples/cores/rocket/configs/default.toml +++ /dev/null @@ -1,6 +0,0 @@ -factory = "examples.cores.rocket.RocketCore" -# Rocket-only core configuration - -memdomain = "memdomains/default.toml" -balldomain = "balldomains/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/rocket/configs/memdomains/default.toml b/examples/cores/rocket/configs/memdomains/default.toml deleted file mode 100644 index 717803c90..000000000 --- a/examples/cores/rocket/configs/memdomains/default.toml +++ /dev/null @@ -1,32 +0,0 @@ -# Rocket-only stub memdomain for bank_params / chip select pairing. -# Not referenced by rocket configs/default.toml (tiles stay rocket-only). - -[bank] -num = 16 -width = 128 -entries = 16 -maskLen = 16 -channel = 1 - -[dma] -nXacts = 1 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 1 - -[tma] -readChannel = 1 -writeChannel = 1 - -[mmio] -enable = false -bankNum = 1 -bankEntries = 1 -bankWidth = 128 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/toy/arch/src/main/scala/ToyCore.scala b/examples/cores/toy/arch/src/main/scala/ToyCore.scala deleted file mode 100644 index a2c91ea31..000000000 --- a/examples/cores/toy/arch/src/main/scala/ToyCore.scala +++ /dev/null @@ -1,20 +0,0 @@ -package examples.cores.toy - -import framework.top.GlobalConfig -import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} -import framework.system.core.{CoreConnection, RocketEndpoint} -import hier.core.rocket.Core -import framework.system.core.accelerator.BuckyballAccelerator - -@instantiable -class ToyCore(b: GlobalConfig) extends RocketEndpoint(b) { - require(p.buckyball.nonEmpty) - val cpu = Instantiate(new Core(p.data, p.instruction, p.regions, Some(p.commands))(p.cpu)) - clink.cpu <> cpu.clink - val accelerator = Instantiate(new BuckyballAccelerator(p.buckyball.get)) - CoreConnection.accelerator(clink.accelerator.get, accelerator) -} - -object ToyCore extends framework.system.core.RocketCoreFactory { - def instantiate(b: GlobalConfig): Instance[RocketEndpoint] = Instantiate(new ToyCore(b)) -} diff --git a/examples/cores/toy/compiler/CMakeLists.txt b/examples/cores/toy/compiler/CMakeLists.txt deleted file mode 100644 index 1893a2fc2..000000000 --- a/examples/cores/toy/compiler/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -set(BB_CORE_NAME toy) -set(BB_CORE_TARGET_PREFIX "") - -set(BB_CORE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..) -get_filename_component(_BB_REPO_ROOT "${BB_CORE_DIR}/../../.." ABSOLUTE) -include_directories(${_BB_REPO_ROOT}/examples/balls) - -add_subdirectory(src) diff --git a/examples/cores/toy/compiler/include/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h b/examples/cores/toy/compiler/include/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h deleted file mode 100644 index c07fc3ade..000000000 --- a/examples/cores/toy/compiler/include/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h +++ /dev/null @@ -1,22 +0,0 @@ -//===- ToyLowerBuckyballPatterns.h - Toy lowering pattern hooks -*- C++ -*-===// - -#ifndef TOY_CONVERSION_LOWER_BUCKYBALL_PATTERNS_H -#define TOY_CONVERSION_LOWER_BUCKYBALL_PATTERNS_H - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" - -namespace mlir { -namespace buddy { - -void populateToyAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); - -void populateToyLowerBuckyballToLLVMPatterns(RewritePatternSet &patterns); -void configureToyLowerBuckyballToLLVMTarget(LLVMConversionTarget &target); -void populateToyLowerBuckyballToBankSSAPatterns(RewritePatternSet &patterns); - -} // namespace buddy -} // namespace mlir - -#endif // TOY_CONVERSION_LOWER_BUCKYBALL_PATTERNS_H diff --git a/examples/cores/toy/compiler/src/Backend/include/IntrinsicsRISCVBuckyball.td b/examples/cores/toy/compiler/src/Backend/include/IntrinsicsRISCVBuckyball.td deleted file mode 100644 index dd695fb46..000000000 --- a/examples/cores/toy/compiler/src/Backend/include/IntrinsicsRISCVBuckyball.td +++ /dev/null @@ -1,3 +0,0 @@ -//===- IntrinsicsRISCVBuckyball.td - Buckyball intrinsics ----------------===// - -include "IntrinsicsRISCVBuckyballBase.td" diff --git a/examples/cores/toy/compiler/src/Backend/include/IntrinsicsRISCVBuckyballBase.td b/examples/cores/toy/compiler/src/Backend/include/IntrinsicsRISCVBuckyballBase.td deleted file mode 100644 index a8dc27e66..000000000 --- a/examples/cores/toy/compiler/src/Backend/include/IntrinsicsRISCVBuckyballBase.td +++ /dev/null @@ -1,10 +0,0 @@ -//===- IntrinsicsRISCVBuckyballBase.td - Base intrinsics -----------------===// - -let TargetPrefix = "riscv" in -def int_riscv_bb_mvin : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; - -let TargetPrefix = "riscv" in -def int_riscv_bb_mvout : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; - -let TargetPrefix = "riscv" in -def int_riscv_bb_mset : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; diff --git a/examples/cores/toy/compiler/src/CMakeLists.txt b/examples/cores/toy/compiler/src/CMakeLists.txt deleted file mode 100644 index 522ca6dd0..000000000 --- a/examples/cores/toy/compiler/src/CMakeLists.txt +++ /dev/null @@ -1,61 +0,0 @@ -if(NOT COMMAND add_mlir_library OR NOT COMMAND add_mlir_dialect) - message(FATAL_ERROR "MLIR CMake helpers must be loaded. Build as part of buddy-mlir.") -endif() - -message(STATUS "Building Toy compiler Core package") -message(STATUS " Source: ${CMAKE_CURRENT_SOURCE_DIR}") -message(STATUS " Binary: ${CMAKE_CURRENT_BINARY_DIR}") - -get_filename_component(BB_REPO_ROOT ${CMAKE_CURRENT_LIST_DIR}/../../../../.. ABSOLUTE) -set(BB_COMPILER_COMMON_DIR ${BB_REPO_ROOT}/stack/compiler/src) - -if(NOT EXISTS ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/BuckyballOps.h) - message(FATAL_ERROR "Missing Buckyball compiler common dir: ${BB_COMPILER_COMMON_DIR}") -endif() - -include_directories(${BB_COMPILER_COMMON_DIR}) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect) -include_directories(${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball) -include_directories(${CMAKE_CURRENT_LIST_DIR}/../include) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) - -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/Dialect) -include_directories(${CMAKE_CURRENT_BINARY_DIR}/Dialect) -include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/Dialect/Buckyball) - -set(TOY_BALL_COMPILER_DIALECT_DIRS) - -set(BB_CORE_BACKEND_INTRINSICS_DIR - ${CMAKE_CURRENT_SOURCE_DIR}/Backend/include - CACHE INTERNAL "Buckyball chip backend intrinsic directory") - -set(BB_BALL_BACKEND_INTRINSICS_DIRS) - -foreach(TOY_BALL_COMPILER_DIALECT_DIR ${TOY_BALL_COMPILER_DIALECT_DIRS}) - if(NOT EXISTS ${TOY_BALL_COMPILER_DIALECT_DIR}) - message(FATAL_ERROR "Missing toy ball compiler dialect dir: ${TOY_BALL_COMPILER_DIALECT_DIR}") - endif() - include_directories(${TOY_BALL_COMPILER_DIALECT_DIR}) - get_filename_component(TOY_BALL_COMPILER_DIR - ${TOY_BALL_COMPILER_DIALECT_DIR}/../.. - ABSOLUTE) - list(APPEND BB_BALL_BACKEND_INTRINSICS_DIRS - ${TOY_BALL_COMPILER_DIR}/Backend/include) -endforeach() - -set(BB_BALL_BACKEND_INTRINSICS_DIRS ${BB_BALL_BACKEND_INTRINSICS_DIRS} - CACHE INTERNAL "Buckyball ball backend intrinsic directories") - -add_subdirectory(Dialect) -add_subdirectory(Conversion) -add_subdirectory(Target) - -add_custom_target(buckyball-compiler-${BB_CORE_NAME} - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass -) diff --git a/examples/cores/toy/compiler/src/Conversion/CMakeLists.txt b/examples/cores/toy/compiler/src/Conversion/CMakeLists.txt deleted file mode 100644 index 5dd235cfa..000000000 --- a/examples/cores/toy/compiler/src/Conversion/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Buckyball Conversion Passes - -add_subdirectory(LowerBuckyball) -add_subdirectory(LowerTileToBuckyball) diff --git a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt b/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt deleted file mode 100644 index 7bc594c92..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerBuckyballPass - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/PhysicalBankState.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/ReportBankUsagePass.cpp - ${BB_COMPILER_COMMON_DIR}/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp - ${BB_assign_SOURCES} - ${BB_bank_ssa_SOURCES} - LowerBuckyballPass.cpp - LowerBuckyballToBankSSAPass.cpp - Patterns/ToyAssignPhysicalBankPatterns.cpp - Patterns/ToyLowerBuckyballToBankSSAPatterns.cpp - Patterns/ToyLowerBuckyballToLLVMPatterns.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ) - -add_custom_command(TARGET ${BB_CORE_TARGET_PREFIX}LowerBuckyballPass PRE_LINK - COMMAND ${CMAKE_COMMAND} -E rm -f - $) diff --git a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp b/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp deleted file mode 100644 index a8ab8d2cc..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp +++ /dev/null @@ -1,128 +0,0 @@ -//===- LowerBuckyballPass.cpp - Toy Buckyball lowering pass ---------------===// - -#include "Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h" - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/TypeConverter.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Pass/Pass.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Buckyball/Transform.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -class LowerBuckyballToLLVMPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToLLVMPass) - LowerBuckyballToLLVMPass() = default; - LowerBuckyballToLLVMPass(const LowerBuckyballToLLVMPass &) {} - - StringRef getArgument() const final { return "lower-buckyball"; } - StringRef getDescription() const final { - return "Lower Toy Buckyball dialect ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - mlir::buddy::configureToyLowerBuckyballToLLVMTarget(target); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - mlir::buddy::populateToyLowerBuckyballToLLVMPatterns(patterns); - - target.addLegalDialect(); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -class LowerBankSSAToIntrinsicsPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBankSSAToIntrinsicsPass) - LowerBankSSAToIntrinsicsPass() = default; - LowerBankSSAToIntrinsicsPass(const LowerBankSSAToIntrinsicsPass &) {} - - StringRef getArgument() const final { return "lower-bank-ssa-to-intrinsics"; } - StringRef getDescription() const final { - return "Lower Toy bank-SSA and Buckyball ops to intrinsic ops."; - } - - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - - configureBuckyballLegalizeForExportTarget(target, stable); - target.addLegalDialect(); - populateBuckyballLegalizeForLLVMExportPatterns( - converter, patterns, targetConfig.bankWidthBits / 8, - targetConfig.bankDepth, targetConfig.bankNum, - /*includeFuncOperandForwarding=*/false, stable); - - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(module, target, std::move(patterns), - config))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballPass() { - PassRegistration(); -} - -void mlir::buddy::registerLowerBankSSAToIntrinsicsPass() { - PassRegistration(); -} diff --git a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp b/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp deleted file mode 100644 index 9f5a82cbb..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp +++ /dev/null @@ -1,48 +0,0 @@ -//===- LowerBuckyballToBankSSAPass.cpp - Toy bank-SSA lowering ------------===// - -#include "Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -#include "Buckyball/BuckyballDialect.h" - -using namespace mlir; - -namespace { - -class LowerBuckyballToBankSSAPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToBankSSAPass) - - StringRef getArgument() const final { return "lower-buckyball-to-bank-ssa"; } - StringRef getDescription() const final { - return "Lower Toy Buckyball ops to explicit bank-SSA ops."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - RewritePatternSet patterns(&getContext()); - mlir::buddy::populateToyLowerBuckyballToBankSSAPatterns(patterns); - if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerBuckyballToBankSSAPass() { - PassRegistration(); -} diff --git a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyAssignPhysicalBankPatterns.cpp b/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyAssignPhysicalBankPatterns.cpp deleted file mode 100644 index 13bc120b1..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyAssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,11 +0,0 @@ -//===- ToyAssignPhysicalBankPatterns.cpp - Toy bank assignment hooks ------===// - -#include "Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h" - -using namespace mlir; - -void mlir::buddy::populateToyAssignPhysicalBankPatterns( - RewritePatternSet &patterns, mlir::buddy::PhysicalBankState &state) { - (void)patterns; - (void)state; -} diff --git a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballToBankSSAPatterns.cpp b/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballToBankSSAPatterns.cpp deleted file mode 100644 index 8c4f9835c..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballToBankSSAPatterns.cpp +++ /dev/null @@ -1,24 +0,0 @@ -//===- ToyLowerBuckyballToBankSSAPatterns.cpp - Toy bank SSA hooks --------===// - -#include "Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; - -namespace mlir::buddy { -#define BUCKYBALL_BANK_SSA_HOOK(BALL) \ - void populate##BALL##LowerBuckyballToBankSSAPatterns(RewritePatternSet &); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_BANK_SSA_HOOK -} // namespace mlir::buddy - -void mlir::buddy::populateToyLowerBuckyballToBankSSAPatterns( - RewritePatternSet &patterns) { - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_BANK_SSA_HOOK(BALL) \ - if (ball == #BALL) \ - populate##BALL##LowerBuckyballToBankSSAPatterns(patterns); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_BANK_SSA_HOOK - } -} diff --git a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballToLLVMPatterns.cpp b/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballToLLVMPatterns.cpp deleted file mode 100644 index f995205de..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballToLLVMPatterns.cpp +++ /dev/null @@ -1,172 +0,0 @@ -//===- ToyLowerBuckyballToLLVMPatterns.cpp - Toy LLVM patterns ------------===// - -#include "Conversion/LowerBuckyball/Patterns/ToyLowerBuckyballPatterns.h" - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -static LLVM::LLVMFunctionType getPrintfType(MLIRContext *context) { - auto i32 = IntegerType::get(context, 32); - auto ptr = LLVM::LLVMPointerType::get(context); - return LLVM::LLVMFunctionType::get(i32, ptr, true); -} - -static FlatSymbolRefAttr getOrInsertPrintf(PatternRewriter &rewriter, - ModuleOp module) { - auto *context = module.getContext(); - if (module.lookupSymbol("printf")) - return SymbolRefAttr::get(context, "printf"); - - PatternRewriter::InsertionGuard insertGuard(rewriter); - rewriter.setInsertionPointToStart(module.getBody()); - rewriter.create(module.getLoc(), "printf", - getPrintfType(context)); - return SymbolRefAttr::get(context, "printf"); -} - -static Value getOrCreateGlobalString(Location loc, OpBuilder &builder, - StringRef name, StringRef value, - ModuleOp module) { - LLVM::GlobalOp global; - if (!(global = module.lookupSymbol(name))) { - OpBuilder::InsertionGuard insertGuard(builder); - builder.setInsertionPointToStart(module.getBody()); - auto type = LLVM::LLVMArrayType::get( - IntegerType::get(builder.getContext(), 8), value.size()); - global = - builder.create(loc, type, true, LLVM::Linkage::Internal, - name, builder.getStringAttr(value), 0); - } - - Value globalPtr = builder.create(loc, global); - Value zero = builder.create(loc, builder.getI64Type(), - builder.getI64IntegerAttr(0)); - return builder.create( - loc, LLVM::LLVMPointerType::get(builder.getContext()), global.getType(), - globalPtr, ArrayRef({zero, zero})); -} - -class BBPrintMemRefOpLowering : public ConversionPattern { -public: - explicit BBPrintMemRefOpLowering(MLIRContext *context) - : ConversionPattern(PrintMemRefOp::getOperationName(), 1, context) {} - - LogicalResult - matchAndRewrite(Operation *op, ArrayRef operands, - ConversionPatternRewriter &rewriter) const override { - auto memRefType = cast(*op->operand_type_begin()); - auto shape = memRefType.getShape(); - Type elemTy = memRefType.getElementType(); - auto loc = op->getLoc(); - auto module = op->getParentOfType(); - auto printfRef = getOrInsertPrintf(rewriter, module); - - Value fmt; - if (elemTy == rewriter.getF32Type() || elemTy == rewriter.getF64Type()) { - fmt = getOrCreateGlobalString(loc, rewriter, "frmt_spec", - StringRef("%f \0", 4), module); - } else if (elemTy == rewriter.getI8Type() || - elemTy == rewriter.getI32Type()) { - fmt = getOrCreateGlobalString(loc, rewriter, "frmt_spec", - StringRef("%d \0", 4), module); - } else { - op->emitError("print_memref supports only i8, i32, f32 and f64"); - return failure(); - } - - Value nl = getOrCreateGlobalString(loc, rewriter, "nl", - StringRef("\n\0", 2), module); - SmallVector ivs; - for (unsigned i = 0, e = shape.size(); i != e; ++i) { - auto lb = rewriter.create(loc, 0); - auto ub = rewriter.create(loc, shape[i]); - auto step = rewriter.create(loc, 1); - auto loop = rewriter.create(loc, lb, ub, step); - for (Operation &nested : llvm::make_early_inc_range(*loop.getBody())) - rewriter.eraseOp(&nested); - ivs.push_back(loop.getInductionVar()); - - rewriter.setInsertionPointToEnd(loop.getBody()); - if (i != e - 1) - rewriter.create(loc, getPrintfType(rewriter.getContext()), - printfRef, nl); - rewriter.create(loc); - rewriter.setInsertionPointToStart(loop.getBody()); - } - - auto printOp = cast(op); - Value elem = rewriter.create(loc, printOp.getInput(), ivs); - if (elem.getType() == rewriter.getF32Type()) - elem = rewriter.create(loc, rewriter.getF64Type(), elem); - else if (elem.getType() == rewriter.getI8Type()) - elem = rewriter.create(loc, rewriter.getI32Type(), elem); - - rewriter.create(loc, getPrintfType(rewriter.getContext()), - printfRef, ArrayRef({fmt, elem})); - rewriter.eraseOp(op); - return success(); - } -}; - -class BBPrintScalarOpLowering : public ConversionPattern { -public: - explicit BBPrintScalarOpLowering(MLIRContext *context) - : ConversionPattern(PrintScalarOp::getOperationName(), 1, context) {} - - LogicalResult - matchAndRewrite(Operation *op, ArrayRef operands, - ConversionPatternRewriter &rewriter) const override { - auto loc = op->getLoc(); - auto module = op->getParentOfType(); - auto printfRef = getOrInsertPrintf(rewriter, module); - Type elemTy = op->getOperand(0).getType(); - - Value fmt; - if (elemTy == rewriter.getF32Type() || elemTy == rewriter.getF64Type()) { - fmt = getOrCreateGlobalString(loc, rewriter, "scalar_fmt", - StringRef("%f\n\0", 5), module); - } else if (elemTy == rewriter.getI8Type() || - elemTy == rewriter.getI32Type()) { - fmt = getOrCreateGlobalString(loc, rewriter, "scalar_fmt", - StringRef("%d\n\0", 5), module); - } else { - op->emitError("print_scalar supports only i8, i32, f32 and f64"); - return failure(); - } - - Value value = op->getOperand(0); - if (elemTy == rewriter.getF32Type()) - value = rewriter.create(loc, rewriter.getF64Type(), value); - else if (elemTy == rewriter.getI8Type()) - value = rewriter.create(loc, rewriter.getI32Type(), value); - - rewriter.create(loc, getPrintfType(rewriter.getContext()), - printfRef, ArrayRef({fmt, value})); - rewriter.eraseOp(op); - return success(); - } -}; - -} // namespace - -void mlir::buddy::populateToyLowerBuckyballToLLVMPatterns( - RewritePatternSet &patterns) { - patterns.add( - patterns.getContext()); -} - -void mlir::buddy::configureToyLowerBuckyballToLLVMTarget( - LLVMConversionTarget &target) { - target.addIllegalOp(); -} diff --git a/examples/cores/toy/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt b/examples/cores/toy/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt deleted file mode 100644 index 82b282ef1..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerTileToBuckyball/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -add_mlir_library(${BB_CORE_TARGET_PREFIX}LowerTileToBuckyballPass - LowerTileToBuckyball.cpp - - LINK_LIBS PUBLIC - BuddyTile - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRLinalgDialect -) diff --git a/examples/cores/toy/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp b/examples/cores/toy/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp deleted file mode 100644 index 6914fbdb4..000000000 --- a/examples/cores/toy/compiler/src/Conversion/LowerTileToBuckyball/LowerTileToBuckyball.cpp +++ /dev/null @@ -1,226 +0,0 @@ -//===- LowerTileToBuckyball.cpp - Toy tile->buckyball pass ---------------===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#include "Conversion/LowerTileToBuckyball/LowerTileToBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/PatternMatch.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/DialectConversion.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Tile/TileDialect.h" -#include "Tile/TileOps.h" -#include "Tile/Transform.h" - -using namespace mlir; -namespace tile = ::buddy::tile; -using mlir::buddy::kDefaultBankWidthBytes; - -namespace { - -static Value cstF32(OpBuilder &b, Location loc, float v) { - return b.create(loc, b.getF32Type(), b.getF32FloatAttr(v)); -} - -class TileConv2dLowering : public OpRewritePattern { -public: - explicit TileConv2dLowering(MLIRContext *context, int64_t bankWidthBytes, - int64_t bankDepth, int64_t /*bankNum*/) - : OpRewritePattern(context), bankWidthBytes(bankWidthBytes), - bankDepth(bankDepth) {} - - LogicalResult matchAndRewrite(tile::TileConv2dOp op, - PatternRewriter &rewriter) const override { - Location loc = op.getLoc(); - - Value input = op.getInput(); - Value filter = op.getFilter(); - Value output = op.getOutput(); - - auto inType = cast(input.getType()); - auto filterType = cast(filter.getType()); - auto outType = cast(output.getType()); - - auto inShape = inType.getShape(); - auto fShape = filterType.getShape(); - auto outShape = outType.getShape(); - - int64_t N = inShape[0], H = inShape[1], W = inShape[2], C = inShape[3]; - int64_t KH = fShape[0], KW = fShape[1], OC = fShape[3]; - int64_t OH = outShape[1], OW = outShape[2]; - - if (!inType.getElementType().isF32() || - !filterType.getElementType().isF32()) - return op.emitError("tile_conv2d lowering currently expects f32"); - if (N <= 0 || H <= 0 || W <= 0 || C <= 0 || KH <= 0 || KW <= 0 || OC <= 0 || - OH <= 0 || OW <= 0) - return op.emitError("tile_conv2d requires positive static shapes"); - Value zero = rewriter.create(loc, 0); - Value one = rewriter.create(loc, 1); - Value nUpper = rewriter.create(loc, N); - Value ohUpper = rewriter.create(loc, OH); - Value owUpper = rewriter.create(loc, OW); - Value ocUpper = rewriter.create(loc, OC); - Value khUpper = rewriter.create(loc, KH); - Value kwUpper = rewriter.create(loc, KW); - Value cUpper = rewriter.create(loc, C); - Value zeroF32 = cstF32(rewriter, loc, 0.0f); - - auto nLoop = rewriter.create(loc, zero, nUpper, one); - { - OpBuilder::InsertionGuard nGuard(rewriter); - rewriter.setInsertionPointToStart(nLoop.getBody()); - Value nIv = nLoop.getInductionVar(); - - auto ohLoop = rewriter.create(loc, zero, ohUpper, one); - { - OpBuilder::InsertionGuard ohGuard(rewriter); - rewriter.setInsertionPointToStart(ohLoop.getBody()); - Value ohIv = ohLoop.getInductionVar(); - - auto owLoop = rewriter.create(loc, zero, owUpper, one); - { - OpBuilder::InsertionGuard owGuard(rewriter); - rewriter.setInsertionPointToStart(owLoop.getBody()); - Value owIv = owLoop.getInductionVar(); - - auto ocLoop = rewriter.create(loc, zero, ocUpper, one); - { - OpBuilder::InsertionGuard ocGuard(rewriter); - rewriter.setInsertionPointToStart(ocLoop.getBody()); - Value ocIv = ocLoop.getInductionVar(); - - Value init = zeroF32; - auto khLoop = - rewriter.create(loc, zero, khUpper, one, init); - { - OpBuilder::InsertionGuard khGuard(rewriter); - rewriter.setInsertionPointToStart(khLoop.getBody()); - Value khIv = khLoop.getInductionVar(); - Value khAcc = khLoop.getRegionIterArgs().front(); - - auto kwLoop = - rewriter.create(loc, zero, kwUpper, one, khAcc); - { - OpBuilder::InsertionGuard kwGuard(rewriter); - rewriter.setInsertionPointToStart(kwLoop.getBody()); - Value kwIv = kwLoop.getInductionVar(); - Value kwAcc = kwLoop.getRegionIterArgs().front(); - - auto cLoop = - rewriter.create(loc, zero, cUpper, one, kwAcc); - { - OpBuilder::InsertionGuard cGuard(rewriter); - rewriter.setInsertionPointToStart(cLoop.getBody()); - Value cIv = cLoop.getInductionVar(); - Value cAcc = cLoop.getRegionIterArgs().front(); - Value inH = rewriter.create(loc, ohIv, khIv); - Value inW = rewriter.create(loc, owIv, kwIv); - Value inValue = rewriter.create( - loc, input, ValueRange{nIv, inH, inW, cIv}); - Value filterValue = rewriter.create( - loc, filter, ValueRange{khIv, kwIv, cIv, ocIv}); - Value product = - rewriter.create(loc, inValue, filterValue); - Value sum = - rewriter.create(loc, cAcc, product); - rewriter.create(loc, sum); - } - rewriter.setInsertionPointAfter(cLoop); - rewriter.create(loc, cLoop.getResult(0)); - } - rewriter.setInsertionPointAfter(kwLoop); - rewriter.create(loc, kwLoop.getResult(0)); - } - rewriter.setInsertionPointAfter(khLoop); - rewriter.create(loc, khLoop.getResult(0), output, - ValueRange{nIv, ohIv, owIv, ocIv}); - } - } - } - } - - rewriter.eraseOp(op); - return success(); - } - -private: - int64_t bankWidthBytes, bankDepth; -}; - -} // namespace - -void mlir::populateLowerTileToBuckyballConversionPatterns( - RewritePatternSet &patterns, int64_t bankWidthBytes, int64_t bankDepth, - int64_t bankNum) { - patterns.add(patterns.getContext(), bankWidthBytes, - bankDepth, bankNum); -} - -namespace { - -class LowerTileToBuckyballPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerTileToBuckyballPass) - StringRef getArgument() const final { return "convert-tile-to-buckyball"; } - StringRef getDescription() const final { - return "Convert Tile dialect to Buckyball dialect"; - } - LowerTileToBuckyballPass() = default; - LowerTileToBuckyballPass(const LowerTileToBuckyballPass &) {} - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert<::buddy::tile::TileDialect, - ::buddy::buckyball::BuckyballDialect, func::FuncDialect, - memref::MemRefDialect, arith::ArithDialect, scf::SCFDialect, - linalg::LinalgDialect>(); - } - - void runOnOperation() override { - const auto &targetConfig = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &getContext(); - ModuleOp module = getOperation(); - - ConversionTarget target(*context); - target.addLegalDialect<::buddy::buckyball::BuckyballDialect, - memref::MemRefDialect, arith::ArithDialect, - scf::SCFDialect, func::FuncDialect, - linalg::LinalgDialect>(); - target.addIllegalDialect<::buddy::tile::TileDialect>(); - - RewritePatternSet patterns(context); - populateLowerTileToBuckyballConversionPatterns( - patterns, targetConfig.bankWidthBits / 8, targetConfig.bankDepth, - targetConfig.bankNum); - - if (failed(applyPartialConversion(module, target, std::move(patterns)))) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::registerLowerTileToBuckyballPass() { - PassRegistration(); -} diff --git a/examples/cores/toy/compiler/src/Dialect/Buckyball/CMakeLists.txt b/examples/cores/toy/compiler/src/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index aa8da8d59..000000000 --- a/examples/cores/toy/compiler/src/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Generate dialect and ops from the Toy dialect assembly. -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(Buckyball.h.inc -gen-op-decls) -mlir_tablegen(Buckyball.cpp.inc -gen-op-defs) -mlir_tablegen(BuckyballTypes.h.inc -gen-typedef-decls -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballTypes.cpp.inc -gen-typedef-defs -typedefs-dialect=buckyball) -mlir_tablegen(BuckyballDialect.h.inc -gen-dialect-decls -dialect=buckyball) -mlir_tablegen(BuckyballDialect.cpp.inc -gen-dialect-defs -dialect=buckyball) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}MLIRBuckyballIncGen) -add_mlir_doc(ToyBuckyball ${BB_CORE_TARGET_PREFIX}Buckyball Dialects/ - -gen-dialect-doc) - -# Generate LLVM IR conversions -set(LLVM_TARGET_DEFINITIONS ${BB_BUCKYBALL_DIALECT_TD}) -mlir_tablegen(BuckyballConversions.inc -gen-llvmir-conversions) -add_public_tablegen_target(${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen) - -# Add subdirectories -add_subdirectory(IR) -add_subdirectory(Transforms) diff --git a/examples/cores/toy/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt b/examples/cores/toy/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt deleted file mode 100644 index 42c7f8807..000000000 --- a/examples/cores/toy/compiler/src/Dialect/Buckyball/IR/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/IR/BuckyballDialect.cpp - ${BB_TARGET_REGISTRY_SOURCE} - ${BB_BALL_IR_SOURCES} - - LINK_LIBS PUBLIC - MLIRIR - MLIRLLVMDialect - MLIRBufferizationDialect - MLIRMemRefDialect -) - -target_include_directories(${BB_CORE_TARGET_PREFIX}BuddyBuckyball PUBLIC - ${BB_COMPILER_COMMON_DIR} - ${BB_COMPILER_COMMON_DIR}/Dialect - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball - ${CMAKE_CURRENT_BINARY_DIR}/.. - ${CMAKE_CURRENT_BINARY_DIR}/../../.. -) diff --git a/examples/cores/toy/compiler/src/Dialect/Buckyball/ToyBuckyball.td b/examples/cores/toy/compiler/src/Dialect/Buckyball/ToyBuckyball.td deleted file mode 100644 index 303415d74..000000000 --- a/examples/cores/toy/compiler/src/Dialect/Buckyball/ToyBuckyball.td +++ /dev/null @@ -1,27 +0,0 @@ -//===-- ToyBuckyball.td - Toy Buckyball dialect operations -*- tablegen -*-===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// -// -// Toy mounts leftover balls: Gemmini, Trace, Mxfp2Int. -// Gemmini/Trace/Mxfp2Int lower outside this dialect (Gemmini dialect / custom). -// -//===----------------------------------------------------------------------===// - -#ifndef TOY_BUCKYBALL_OPS -#define TOY_BUCKYBALL_OPS - -include "Buckyball.td" - -#endif // TOY_BUCKYBALL_OPS diff --git a/examples/cores/toy/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt b/examples/cores/toy/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt deleted file mode 100644 index adb89436c..000000000 --- a/examples/cores/toy/compiler/src/Dialect/Buckyball/Transforms/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -add_mlir_dialect_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballTransforms - ${BB_COMPILER_COMMON_DIR}/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp - ${BB_legalize_SOURCES} - LegalizeForLLVMExport.cpp - - LINK_LIBS PUBLIC - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - MLIRIR - MLIRLLVMCommonConversion - MLIRLLVMDialect - ) diff --git a/examples/cores/toy/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/examples/cores/toy/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 7af964ed7..000000000 --- a/examples/cores/toy/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,51 +0,0 @@ -//===- LegalizeForLLVMExport.cpp - Toy Buckyball LLVM lowering ------------===// - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/Transform.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace buddy::buckyball::legalize; - -namespace mlir::buddy::buckyball { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - void populate##BALL##LegalizeForLLVMExportPatterns( \ - LLVMTypeConverter &, RewritePatternSet &, bool, int64_t); \ - void configure##BALL##LegalizeForExportTarget(LLVMConversionTarget &, bool); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK -} // namespace mlir::buddy::buckyball - -void mlir::populateBuckyballLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, - int64_t bankWidthBytes, int64_t bankDepth, int64_t bankNum, - bool includeFuncOperandForwarding, bool stable) { - populateBaseLegalizeForLLVMExportPatterns(converter, patterns, - includeFuncOperandForwarding); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - if (ball == #BALL) \ - buddy::buckyball::populate##BALL##LegalizeForLLVMExportPatterns( \ - converter, patterns, stable, bankDepth); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK - } - (void)bankWidthBytes; - (void)bankNum; -} - -void mlir::configureBuckyballLegalizeForExportTarget( - LLVMConversionTarget &target, bool stable) { - configureBaseLegalizeForExportTarget(target); - for (llvm::StringRef ball : buckyball_target::getBuckyballTarget().balls) { -#define BUCKYBALL_LEGALIZE_HOOK(BALL) \ - if (ball == #BALL) \ - buddy::buckyball::configure##BALL##LegalizeForExportTarget(target, stable); -#include "BuckyballBallLoweringHooks.inc" -#undef BUCKYBALL_LEGALIZE_HOOK - } -} diff --git a/examples/cores/toy/compiler/src/Dialect/CMakeLists.txt b/examples/cores/toy/compiler/src/Dialect/CMakeLists.txt deleted file mode 100644 index 33d2424f0..000000000 --- a/examples/cores/toy/compiler/src/Dialect/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Buckyball Dialects - -add_subdirectory(Buckyball) -# add_subdirectory(vecball) -# add_subdirectory(matrixball) diff --git a/examples/cores/toy/compiler/src/Target/CMakeLists.txt b/examples/cores/toy/compiler/src/Target/CMakeLists.txt deleted file mode 100644 index cf3f2b4c1..000000000 --- a/examples/cores/toy/compiler/src/Target/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Buckyball Target Support - -add_subdirectory(LLVMIR) diff --git a/examples/cores/toy/compiler/src/Target/LLVMIR/CMakeLists.txt b/examples/cores/toy/compiler/src/Target/LLVMIR/CMakeLists.txt deleted file mode 100644 index 486c68391..000000000 --- a/examples/cores/toy/compiler/src/Target/LLVMIR/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Buckyball LLVM IR Translation - -add_subdirectory(Dialect) diff --git a/examples/cores/toy/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt b/examples/cores/toy/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt deleted file mode 100644 index 4afc6820b..000000000 --- a/examples/cores/toy/compiler/src/Target/LLVMIR/Dialect/Buckyball/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -add_mlir_translation_library(${BB_CORE_TARGET_PREFIX}BuddyBuckyballToLLVMIRTranslation - ${BB_COMPILER_COMMON_DIR}/Target/LLVMIR/Dialect/Buckyball/BuckyballToLLVMIRTranslation.cpp - - DEPENDS - ${BB_CORE_TARGET_PREFIX}BuddyBuckyballConversionsIncGen - intrinsics_gen - - LINK_COMPONENTS - Core - - LINK_LIBS PUBLIC - MLIRIR - MLIRSupport - ${BB_CORE_TARGET_PREFIX}BuddyBuckyball - ) diff --git a/examples/cores/toy/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt b/examples/cores/toy/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt deleted file mode 100644 index 890a6514d..000000000 --- a/examples/cores/toy/compiler/src/Target/LLVMIR/Dialect/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Buckyball Dialect LLVM IR Translation - -add_subdirectory(Buckyball) diff --git a/examples/cores/toy/configs/balldomains/balls/smatmul/lane16.toml b/examples/cores/toy/configs/balldomains/balls/smatmul/lane16.toml deleted file mode 100644 index c0a935cc7..000000000 --- a/examples/cores/toy/configs/balldomains/balls/smatmul/lane16.toml +++ /dev/null @@ -1,8 +0,0 @@ -[ball] -InputNum = 16 -inputWidth = 8 -lane = 16 -wsReuseTiles = 2 -outputWidth = 32 -numMulThreads = 16 -numCasThreads = 16 diff --git a/examples/cores/toy/configs/balldomains/default.toml b/examples/cores/toy/configs/balldomains/default.toml deleted file mode 100644 index 791c5bd2f..000000000 --- a/examples/cores/toy/configs/balldomains/default.toml +++ /dev/null @@ -1,39 +0,0 @@ -# Toy: virtual single-core. Mounts leftover balls not used by pebble/poly/goban. - -ballNum = 4 - -ballIdMappings = [ - { ballId = 0, ballName = "GemminiBall", ballClass = "examples.balls.gemmini.GemminiBall", config = "../../../../balls/gemmini/configs/default.toml", inBW = 2, outBW = 4 }, - { ballId = 1, ballName = "TraceBall", ballClass = "examples.balls.trace.TraceBall", config = "../../../../balls/trace/configs/default.toml", inBW = 1, outBW = 1 }, - { ballId = 2, ballName = "Mxfp2IntBall", ballClass = "examples.balls.mxfp2int.Mxfp2IntBall", config = "../../../../balls/mxfp2int/configs/default.toml", inBW = 1, outBW = 1, mmioReadBW = 1 }, - { ballId = 3, ballName = "ReluBall", ballClass = "examples.balls.relu.ReluBall", config = "../../../../balls/relu/configs/default.toml", inBW = 1, outBW = 1 }, -] - -ballISA = [ - { mnemonic = "GEMMINI_CONFIG", funct7 = 2, bid = 0 }, - { mnemonic = "GEMMINI_FLUSH", funct7 = 3, bid = 0 }, - { mnemonic = "GEMMINI_PRELOAD", funct7 = 53, bid = 0 }, - { mnemonic = "GEMMINI_COMPUTE_PRELOADED", funct7 = 66, bid = 0 }, - { mnemonic = "GEMMINI_COMPUTE_ACCUMULATED", funct7 = 67, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS_CONFIG_BOUNDS", funct7 = 80, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_A", funct7 = 81, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_B", funct7 = 82, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_D", funct7 = 83, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_C", funct7 = 84, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS_CONFIG_STRIDES_AB", funct7 = 85, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS_CONFIG_STRIDES_DC", funct7 = 86, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_WS", funct7 = 87, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_1", funct7 = 96, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_2", funct7 = 97, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_3", funct7 = 98, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_4", funct7 = 99, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_5", funct7 = 100, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_6", funct7 = 101, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_7", funct7 = 102, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_8", funct7 = 103, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_9", funct7 = 104, bid = 0 }, - { mnemonic = "GEMMINI_LOOP_CONV_WS", funct7 = 105, bid = 0 }, - { mnemonic = "BDB_COUNTER", funct7 = 4, bid = 1 }, - { mnemonic = "MXFP2INT", funct7 = 55, bid = 2 }, - { mnemonic = "RELU", funct7 = 50, bid = 3 }, -] diff --git a/examples/cores/toy/configs/cpu/default.toml b/examples/cores/toy/configs/cpu/default.toml deleted file mode 100644 index 1bf2f46e8..000000000 --- a/examples/cores/toy/configs/cpu/default.toml +++ /dev/null @@ -1,2 +0,0 @@ -kind = "rocket" -config = "../../../rocket/configs/cpu/rocket/default.toml" diff --git a/examples/cores/toy/configs/default.toml b/examples/cores/toy/configs/default.toml deleted file mode 100644 index e40977c2a..000000000 --- a/examples/cores/toy/configs/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -factory = "examples.cores.toy.ToyCore" -# Toy Core Configuration (default) - -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/default.toml" -rvv = "rvv/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/toy/configs/demo.toml b/examples/cores/toy/configs/demo.toml deleted file mode 100644 index 49a49854a..000000000 --- a/examples/cores/toy/configs/demo.toml +++ /dev/null @@ -1,8 +0,0 @@ -factory = "examples.cores.toy.ToyCore" -# Toy Core Configuration (demo frontend) - -balldomain = "balldomains/default.toml" -memdomain = "memdomains/default.toml" -frontend = "frontends/demo.toml" -rvv = "rvv/default.toml" -cpu = "cpu/default.toml" diff --git a/examples/cores/toy/configs/frontends/default.toml b/examples/cores/toy/configs/frontends/default.toml deleted file mode 100644 index b17bc5c5d..000000000 --- a/examples/cores/toy/configs/frontends/default.toml +++ /dev/null @@ -1,8 +0,0 @@ -robEntries = 16 -rsOutOfOrderResponse = true -bankIdLen = 10 -vbankIdUpperBound = 15 -sharedBankIdBase = 32 -iterLen = 34 -subRobEnable = true -subRobDepth = 64 diff --git a/examples/cores/toy/configs/frontends/demo.toml b/examples/cores/toy/configs/frontends/demo.toml deleted file mode 100644 index 41cd684cc..000000000 --- a/examples/cores/toy/configs/frontends/demo.toml +++ /dev/null @@ -1,8 +0,0 @@ -robEntries = 16 -rsOutOfOrderResponse = true -bankIdLen = 10 -vbankIdUpperBound = 15 -sharedBankIdBase = 32 -iterLen = 34 -subRobEnable = false -subRobDepth = 64 diff --git a/examples/cores/toy/configs/memdomains/default.toml b/examples/cores/toy/configs/memdomains/default.toml deleted file mode 100644 index 5ee4d654e..000000000 --- a/examples/cores/toy/configs/memdomains/default.toml +++ /dev/null @@ -1,32 +0,0 @@ -# Toy Core Memory Domain Configuration (default) -# Private (per-core) memory parameters. Shared memory lives in the tile TOML. - -[bank] -num = 16 -width = 128 -entries = 1024 -maskLen = 16 -channel = 7 - -[dma] -nXacts = 8 -burstMaxBytes = 1024 -busWidth = 128 -maxInFlightMemReqs = 16 - -[tlb] -size = 4 - -[tma] -readChannel = 4 -writeChannel = 4 - -[mmio] -enable = true -bankNum = 16 -bankEntries = 64 -bankWidth = 8 -readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/cores/toy/workloads/ctests/CMakeLists.txt b/examples/cores/toy/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 931fea644..000000000 --- a/examples/cores/toy/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -add_subdirectory(gemmini_relu_trace_pipeline) -set(BUCKYBALL_CTEST_TARGET ${BUCKYBALL_CORE_WORKLOAD_TARGET}) -file(RELATIVE_PATH RVV_KERNEL_SOURCE ${CMAKE_CURRENT_SOURCE_DIR} - ${WORKLOAD_SRC_DIR}/CTest/rvv/rvv_kernel.c) -add_buckyball_ctest(${RVV_KERNEL_SOURCE}) -buckyball_ctest_name(RVV_KERNEL_TEST ${BUCKYBALL_CTEST_TARGET} rvv_kernel) -add_dependencies(core-ctests-build ${RVV_KERNEL_TEST}-ctest-build) - -add_buckyball_ctest(rvv_ball.c) -buckyball_ctest_name(RVV_BALL_TEST ${BUCKYBALL_CTEST_TARGET} rvv_ball) -add_dependencies(core-ctests-build ${RVV_BALL_TEST}-ctest-build) diff --git a/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/CMakeLists.txt b/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/CMakeLists.txt deleted file mode 100644 index 2da7121a8..000000000 --- a/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -add_buckyball_core_workload(gemmini_relu_trace_pipeline SOURCES - gemmini_relu_trace_pipeline-main.cpp - toy.c) diff --git a/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp b/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp deleted file mode 100644 index 217817cb9..000000000 --- a/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include - -extern "C" int toy_core(core_id_t); -static volatile uint32_t completed; -static volatile uint32_t failed; - -int main() { - core_id_t id = bb_get_core_id(); - if (id.tile == BB_WORK_TILE && - bb_topology_core_profile(id) == BB_CORE_WORKLOAD_PROFILE) { - __atomic_fetch_or(&failed, toy_core(id) != 0, __ATOMIC_RELEASE); - __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); - } - if (id.tile == BB_WORK_TILE && id.core == 0) { - uint32_t expected = - bb_topology_profile_cores_per_tile(BB_CORE_WORKLOAD_PROFILE); - while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) - asm volatile("nop"); - return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; - } - for (;;) - asm volatile("wfi"); -} diff --git a/examples/designs/goban/arch/build.sc b/examples/designs/goban/arch/build.sc new file mode 100644 index 000000000..ca8f7e2b1 --- /dev/null +++ b/examples/designs/goban/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.arch.src.main.scala.framework.device.cbus.{build => cbus_build} +import $file.^.^.^.{build => examples} +import $file.^.^.^.components.cores.rocket.arch.{build => rocket} + +val designRoot: os.Path = millSourcePath / os.up + +object design extends SbtModule { + override def generatedSources = T(cbus_build.registry(sources() ++ rocket.library.sources(), T.dest)) + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(examples.library, rocket.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(designRoot / "arch", designRoot / "configs") +} diff --git a/examples/designs/goban/arch/device/DeviceTop.scala b/examples/designs/goban/arch/device/DeviceTop.scala new file mode 100644 index 000000000..07854da55 --- /dev/null +++ b/examples/designs/goban/arch/device/DeviceTop.scala @@ -0,0 +1,77 @@ +package examples.designs.goban + +import framework.lib.arbiter.{RRArbiter} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} +import framework.device.configs.DeviceParam +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import buckyball.config.Design +import _root_.configs.GlobalConfig +import framework.core.clink.CLinkIO +import framework.core.memory.home.Home +import framework.device.ddr.Bridge +import framework.device.peripherals.Controller +import framework.device.peripherals.bootrom.BootRomLines +import framework.device.dlink.BaseDevice +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.LineResponse +import framework.lib.queue.{Queue => ResponseQueue} + +/** One Rocket hart schedules local MatMul threads through its core. */ +@instantiable +class DeviceTop(b: GlobalConfig, design: Design) extends BaseDevice(b.device.ddr.axi) { + val device = b.device + + val Seq(corep) = device.cores + val sharedHome = corep.memory.home + require(sharedHome.memory.agents == 3) + val bus = Instantiate(new CBus(b.device, CBus.generators(design, b.device, b.sim, CoreComponents.factories))) + + val home: Instance[Home] = Instantiate(new Home(sharedHome)) + home.io.active := true.B + home.io.blockRequesterRsp := false.B + home.io.requesters <> bus.coherent + + val peripherals = Instantiate(new Controller(device, device.ddr.axi, corep.memory.memoryAccess.memory)) + + val peripheralBus = Instantiate(new axi4.Decoder( + Seq( + (device.clint.base, device.clint.bytes), + (device.plic.base, device.plic.bytes), + (device.scu.baseAddress, device.scu.totalSizeBytes) + ), + device.ddr.axi + )) + + peripheralBus.io.in <> bus.uncached + peripherals.io.axi <> peripheralBus.io.matched + bus.external <> peripheralBus.io.other + peripherals.io.hardwareInterrupt.sources := 0.U + bus.interrupts := peripherals.io.hardwareInterrupt.harts + + val rom = Instantiate(new BootRomLines(device.bootrom, sharedHome.memory.chi)) + val bridge = Instantiate(new Bridge(device.ddr)) + require(device.ddr.clients == 1) + val request = home.io.memoryReq + val inRom = request.bits.addr >= device.bootrom.base.U && + request.bits.addr < (device.bootrom.base + device.bootrom.bytes).U + rom.io.request.valid := request.valid && inRom + rom.io.request.bits := request.bits + bridge.io.request(0).valid := request.valid && !inRom + bridge.io.request(0).bits := request.bits + request.ready := Mux(inRom, rom.io.request.ready, bridge.io.request(0).ready) + val responses = Instantiate(new RRArbiter(new LineResponse(sharedHome.memory.chi), 2)) + responses.io.in(0) <> rom.io.response + responses.io.in(1) <> bridge.io.response(0) + val response = Instantiate(new ResponseQueue(new LineResponse(sharedHome.memory.chi), 1, false, false)) + response.io.enq <> responses.io.out + home.io.memoryResp <> response.io.deq + + val memoryBus = Instantiate(new axi4.Interconnect(axi4.InterconnectParam(device.ddr.axi, 2, None))) + memoryBus.io.in(0) <> bus.dlink.axi + memoryBus.io.in(1) <> bridge.io.axi + dlink.axi <> memoryBus.io.out +} diff --git a/examples/designs/goban/arch/device/core/GobanCore.scala b/examples/designs/goban/arch/device/core/GobanCore.scala new file mode 100644 index 000000000..a9161b978 --- /dev/null +++ b/examples/designs/goban/arch/device/core/GobanCore.scala @@ -0,0 +1,88 @@ +package examples.designs.goban + +import framework.core.tbus.generated.ThreadComponents +import sims.configs.SimParam +import framework.device.cbus.CoreFactory +import buckyball.config.{CorePlacement, Design} + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.rocket.configs.{Params => RocketParam} +import framework.core.configs.CoreParam +import framework.core.controller.Controller +import framework.core.tbus.TBus +import framework.core.tss.Store +import framework.core.shm.SHM +import framework.core.clink.{CLinkIO, HasCLink} +import framework.ip.bus.chi.rnf.AxiCache + +@instantiable +class GobanCore( + corep: CoreParam, + rocketp: RocketParam, + simp: SimParam, + traceIds: Seq[Int]) + extends Module + with HasCLink { + val diffTest = simp.diffTest + require(corep.clink.harts == 1 && !corep.clink.backing && !corep.clink.shared) + require(corep.clink.requesters == 3 && corep.threads.nonEmpty) + + @public + val clink = IO(new CLinkIO(corep.clink)) + + val loom: Instance[Loom] = Instantiate(new Loom(corep, rocketp)) + val controller: Instance[Controller] = Instantiate(new Controller(corep.controller)) + + val tbus: Instance[TBus] = + Instantiate(new TBus(corep, simp, traceIds, TBus.generators(corep, simp, traceIds, ThreadComponents.factories))) + + val shm: Instance[SHM] = Instantiate(new SHM(corep.sharedFrontend, corep.sharedMemory, corep.tbus.get)) + val tss: Instance[Store] = Instantiate(new Store(corep.tss.get)) + val dma: Instance[AxiCache] = Instantiate(new AxiCache(corep.interconnect.bus, corep.dmaCache.get)) + + loom.io.interrupts := clink.interrupts(0) + controller.io.execution <> loom.io.commands + loom.io.drained := controller.io.drained + loom.io.memoryAllowed := controller.io.memoryAllowed + tbus.io.control.get <> controller.io.threads.get + tbus.io.memory.get <> loom.io.memory + tbus.io.move <> controller.io.move + shm.io <> tbus.io.shm.get + tss.io.clients <> tbus.io.tss.get + tss.io.host <> controller.io.tss.get.host + tss.io.inUse := controller.io.tss.get.inUse + tss.io.cancel := controller.io.tss.get.cancel + controller.io.tss.get.busy := tss.io.busy + controller.io.tss.get.hostBusy := tss.io.hostBusy + + clink.memory.uncached <> loom.io.uncached + clink.memory.coherent.take(2).zip(loom.io.coherent).foreach { case (port, cache) => port <> cache } + dma.io.axi <> tbus.io.ddr + dma.io.axi.ar.bits.cache := 15.U + dma.io.axi.aw.bits.cache := 15.U + clink.memory.coherent(2) <> dma.io.chi + clink.memory.ddr.aw.valid := false.B + clink.memory.ddr.aw.bits := 0.U.asTypeOf(clink.memory.ddr.aw.bits) + clink.memory.ddr.w.valid := false.B + clink.memory.ddr.w.bits := 0.U.asTypeOf(clink.memory.ddr.w.bits) + clink.memory.ddr.ar.valid := false.B + clink.memory.ddr.ar.bits := 0.U.asTypeOf(clink.memory.ddr.ar.bits) + clink.memory.ddr.b.ready := false.B + clink.memory.ddr.r.ready := false.B +} + +object Goban extends CoreFactory { + val component = "goban" + def parameters(design: Design, source: CorePlacement): CoreParam = + CoreParam.fromProto(design, source, externalHome = !RocketParam.fromProto(source).clink.backing) + + def instantiate( + source: CorePlacement, + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int] + ): CLinkIO = + Instantiate(new GobanCore(corep, RocketParam.fromProto(source), simp, traceIds)).clink + +} diff --git a/examples/designs/goban/arch/device/core/loom.scala b/examples/designs/goban/arch/device/core/loom.scala new file mode 100644 index 000000000..934ba4939 --- /dev/null +++ b/examples/designs/goban/arch/device/core/loom.scala @@ -0,0 +1,60 @@ +package examples.designs.goban + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.rocket.Execution +import examples.core.rocket.configs.{Params => RocketParam} +import framework.core.configs.CoreParam +import framework.core.memory.{Memory, MemoryPort} +import framework.core.controller.admission.Commands +import framework.core.clink.CoreInterrupts +import framework.ip.bus.{axi4, chi} + +/** Outer Loom execution core, implemented by Rocket; runs the engine that schedules task threads. */ +@instantiable +class Loom(corep: CoreParam, rocketp: RocketParam) extends Module { + require(!corep.clink.backing) + + @public + val io = IO(new Bundle { + val interrupts = Input(new CoreInterrupts) + val commands = new Commands(corep.memory.dcache.tracking, corep.integer.xLen, corep.privileged.permission) + val drained = Input(Bool()) + val memoryAllowed = Input(Bool()) + val memory = + Vec(corep.threads.size, Flipped(new MemoryPort(corep.memory.dcache.tracking, corep.memory.memoryAccess.walker))) + val coherent = Vec(2, new chi.RequesterPort(corep.memory.home.memory.chi)) + val uncached = new axi4.Port(corep.memory.uncached.axi) + }) + + val execution: Instance[Execution] = Instantiate(new Execution(rocketp, commandsEnabled = true)) + val memory: Instance[Memory] = Instantiate(new Memory(corep.memory, externalHome = true)) + execution.io.interrupts := io.interrupts + memory.io.fetch <> execution.io.fetch + memory.io.data <> execution.io.data + memory.io.timeRequest <> execution.io.timeRequest + execution.io.timeResponse <> memory.io.timeResponse + memory.io.commandAllowed := io.memoryAllowed + memory.io.threads.get <> io.memory + io.coherent <> memory.io.coherent + io.uncached <> memory.io.uncached + + val rocc = execution.io.rocc + val command = io.commands.command + val pending = RegInit(false.B) + command.valid := rocc.cmd.valid && !pending + rocc.cmd.ready := command.ready && !pending + command.bits.tag := 0.U + command.bits.instruction := rocc.cmd.bits.instruction + command.bits.satp := rocc.cmd.bits.satp + command.bits.effectivePrivilege := rocc.cmd.bits.effectivePrivilege + command.bits.sum := rocc.cmd.bits.sum + command.bits.mxr := rocc.cmd.bits.mxr + command.bits.pmp := rocc.cmd.bits.pmp + when(command.fire)(pending := true.B) + io.commands.complete.ready := true.B + when(io.commands.complete.fire)(pending := false.B) + rocc.resp <> io.commands.response + rocc.busy := pending || !io.drained + rocc.interrupt := false.B +} diff --git a/examples/designs/goban/configs/design.toml b/examples/designs/goban/configs/design.toml new file mode 100644 index 000000000..a55a275cc --- /dev/null +++ b/examples/designs/goban/configs/design.toml @@ -0,0 +1,4 @@ +[device] +include = "device/default.toml" +[sims] +include = "sims/default.toml" diff --git a/examples/designs/goban/configs/device/clint.toml b/examples/designs/goban/configs/device/clint.toml new file mode 100644 index 000000000..417a2788c --- /dev/null +++ b/examples/designs/goban/configs/device/clint.toml @@ -0,0 +1,4 @@ +base = 0x2000000 +bytes = 0x10000 +tickCycles = 1000 +hartIds = [0] diff --git a/examples/designs/goban/configs/device/core/c2c/goban.toml b/examples/designs/goban/configs/device/core/c2c/goban.toml new file mode 100644 index 000000000..1284ec5d8 --- /dev/null +++ b/examples/designs/goban/configs/device/core/c2c/goban.toml @@ -0,0 +1,3 @@ +enabled = false +peerCores = 1 +axi = "default" diff --git a/examples/designs/goban/configs/device/core/controller/goban.toml b/examples/designs/goban/configs/device/core/controller/goban.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/goban/configs/device/core/controller/goban.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/goban/configs/device/core/default.toml b/examples/designs/goban/configs/device/core/default.toml new file mode 100644 index 000000000..46ea1c98b --- /dev/null +++ b/examples/designs/goban/configs/device/core/default.toml @@ -0,0 +1,60 @@ +component = "goban" +implementation = "rocket.toml" +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +memory = "memory/goban.toml" +dmaCache = "memory/dma-cache/shared.toml" +tbus = "tbus/default.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/goban.toml" +c2c = "c2c/goban.toml" +controller = "controller/goban.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[sharedMem] +enable = true +entries = 16384 +inputChannels = 52 +defaultGroupCount = 1 +virtualBankCount = 64 +memCycles = 10 +bankEntries = 1024 +bankWidth = 128 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[[threads]] +id = 0 +name = "matmul0" +component = "matmul" +include = "../../../../../components/threads/matmul/configs/default.toml" + +[[threads]] +id = 1 +name = "matmul1" +component = "matmul" +include = "../../../../../components/threads/matmul/configs/default.toml" + +[[threads]] +id = 2 +name = "matmul2" +component = "matmul" +include = "../../../../../components/threads/matmul/configs/default.toml" + +[[threads]] +id = 3 +name = "matmul3" +component = "matmul" +include = "../../../../../components/threads/matmul/configs/default.toml" diff --git a/examples/designs/goban/configs/device/core/interconnect/goban.toml b/examples/designs/goban/configs/device/core/interconnect/goban.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/goban/configs/device/core/interconnect/goban.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/goban/configs/device/core/memory/access/goban.toml b/examples/designs/goban/configs/device/core/memory/access/goban.toml new file mode 100644 index 000000000..bb87f76a4 --- /dev/null +++ b/examples/designs/goban/configs/device/core/memory/access/goban.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +accessTagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x20000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/goban/configs/device/core/memory/dcache/goban.toml b/examples/designs/goban/configs/device/core/memory/dcache/goban.toml new file mode 100644 index 000000000..386bc4427 --- /dev/null +++ b/examples/designs/goban/configs/device/core/memory/dcache/goban.toml @@ -0,0 +1,7 @@ +cache = "shared.toml" +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 diff --git a/examples/designs/goban/configs/device/core/memory/dcache/shared.toml b/examples/designs/goban/configs/device/core/memory/dcache/shared.toml new file mode 100644 index 000000000..0f17666dd --- /dev/null +++ b/examples/designs/goban/configs/device/core/memory/dcache/shared.toml @@ -0,0 +1,7 @@ +nodeId = 2 +cacheLines = 64 +homeId = 32 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/goban/configs/device/core/memory/dma-cache/shared.toml b/examples/designs/goban/configs/device/core/memory/dma-cache/shared.toml new file mode 100644 index 000000000..9dd394051 --- /dev/null +++ b/examples/designs/goban/configs/device/core/memory/dma-cache/shared.toml @@ -0,0 +1,7 @@ +nodeId = 3 +cacheLines = 64 +homeId = 32 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/goban/configs/device/core/memory/goban.toml b/examples/designs/goban/configs/device/core/memory/goban.toml new file mode 100644 index 000000000..844bd730f --- /dev/null +++ b/examples/designs/goban/configs/device/core/memory/goban.toml @@ -0,0 +1,5 @@ +uncached = "default" +icache = "icache/shared.toml" +access = "access/goban.toml" +dcache = "dcache/goban.toml" +home = "home/goban.toml" diff --git a/examples/designs/goban/configs/device/core/memory/home/goban.toml b/examples/designs/goban/configs/device/core/memory/home/goban.toml new file mode 100644 index 000000000..0e966ccbe --- /dev/null +++ b/examples/designs/goban/configs/device/core/memory/home/goban.toml @@ -0,0 +1,13 @@ +agents = 3 +mshrEntries = 8 +homeId = 32 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/goban/configs/device/core/memory/icache/shared.toml b/examples/designs/goban/configs/device/core/memory/icache/shared.toml new file mode 100644 index 000000000..873c62547 --- /dev/null +++ b/examples/designs/goban/configs/device/core/memory/icache/shared.toml @@ -0,0 +1,7 @@ +nodeId = 1 +cacheLines = 64 +homeId = 32 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/goban/configs/device/core/rocket.toml b/examples/designs/goban/configs/device/core/rocket.toml new file mode 100644 index 000000000..4d6fb0712 --- /dev/null +++ b/examples/designs/goban/configs/device/core/rocket.toml @@ -0,0 +1,12 @@ +component = "rocket" +physical_address_bits = 36 +hart_id_bits = 3 +page_levels = 3 +pmp_entries = 16 +virtual_memory = true +supervisor = true +user = true +floating_point = true +atomics = true +multiply_divide = true +external_home = true diff --git a/examples/designs/goban/configs/device/core/tbus/default.toml b/examples/designs/goban/configs/device/core/tbus/default.toml new file mode 100644 index 000000000..9665aee43 --- /dev/null +++ b/examples/designs/goban/configs/device/core/tbus/default.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3] diff --git a/examples/designs/goban/configs/device/ddr.toml b/examples/designs/goban/configs/device/ddr.toml new file mode 100644 index 000000000..31815e9e1 --- /dev/null +++ b/examples/designs/goban/configs/device/ddr.toml @@ -0,0 +1,13 @@ +clients = 1 +slotsPerClient = 16 +dataBits = 128 +idBits = 4 +idBase = 0 +base = 0x80000000 +bytes = 0x20000000 +model = 0 + +[line] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 diff --git a/examples/designs/goban/configs/device/default.toml b/examples/designs/goban/configs/device/default.toml new file mode 100644 index 000000000..402388b16 --- /dev/null +++ b/examples/designs/goban/configs/device/default.toml @@ -0,0 +1,9 @@ +ddr = "ddr.toml" +bootrom = "default" +clint = "clint.toml" +plic = "plic.toml" +scu = "default" + +[[cores]] +include = "core/default.toml" +hartId = 0 diff --git a/examples/designs/goban/configs/device/plic.toml b/examples/designs/goban/configs/device/plic.toml new file mode 100644 index 000000000..dea72f94e --- /dev/null +++ b/examples/designs/goban/configs/device/plic.toml @@ -0,0 +1,5 @@ +base = 0xc000000 +bytes = 0x4000000 +sources = 1 +priorityBits = 3 +hartIds = [0] diff --git a/examples/designs/goban/configs/sims/default.toml b/examples/designs/goban/configs/sims/default.toml new file mode 100644 index 000000000..37e108378 --- /dev/null +++ b/examples/designs/goban/configs/sims/default.toml @@ -0,0 +1,14 @@ +diffTest = false + +[memory] +lineBytes = 64 +clockHz = 1000000000 + +[targets.device] +config = "examples.designs.goban.sims.device.DeviceConfig" +stack_name = "goban" +regression = "sims/device/regression" +kernel = "sims/device/workloads/kernel" + +[verification] +registry = "../../sims/ip/uvm/registry.toml" diff --git a/examples/designs/goban/emu/Cargo.lock b/examples/designs/goban/emu/Cargo.lock new file mode 100644 index 000000000..adfacbb8c --- /dev/null +++ b/examples/designs/goban/emu/Cargo.lock @@ -0,0 +1,821 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "bebop-vm", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "rvsim", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-goban" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rvsim" +version = "0.1.0" +dependencies = [ + "rustc_apfloat", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/goban/emu/Cargo.toml b/examples/designs/goban/emu/Cargo.toml new file mode 100644 index 000000000..dbd933223 --- /dev/null +++ b/examples/designs/goban/emu/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "bebop-design-goban" +version = "0.1.0" +edition = "2021" + +[dependencies] +bebop-bemu = { path = "../../../../bebop/src/nodes/bemu" } diff --git a/examples/designs/goban/emu/src/device/core/mod.rs b/examples/designs/goban/emu/src/device/core/mod.rs new file mode 100644 index 000000000..004047758 --- /dev/null +++ b/examples/designs/goban/emu/src/device/core/mod.rs @@ -0,0 +1,37 @@ +use bebop_bemu::{ + ball::linkme, CoreFactory, CoreModel, CoreParams, Cpu, DeviceModel, FromString, ModelOptions, NoRocc, ThreadGroup, + Whatever, CORES, +}; + +/// One RV64GC hart driving its matmul hardware threads (arch: GobanCore, behind TBus). +pub struct GobanCore { + cpu: Cpu, + threads: ThreadGroup, +} + +impl CoreModel for GobanCore { + fn cpu(&self) -> &Cpu { + &self.cpu + } + fn cpu_mut(&mut self) -> &mut Cpu { + &mut self.cpu + } + fn step(&mut self, count: u64) -> Result<(), Whatever> { + self.threads.check().map_err(Whatever::without_source)?; + self.cpu.step(count, &mut NoRocc)?; + if self.cpu.finished() { + self.threads.shutdown().map_err(Whatever::without_source)?; + } + Ok(()) + } +} + +fn instantiate(device: &DeviceModel, params: &CoreParams, options: &ModelOptions) -> Result, String> { + let cpu = Cpu::new(device, params, options).map_err(|error| error.to_string())?; + let threads = ThreadGroup::new(cpu.state(), options)?; + Ok(Box::new(GobanCore { cpu, threads })) +} + +#[linkme::distributed_slice(CORES)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static GOBAN: CoreFactory = CoreFactory { component: "goban", instantiate }; diff --git a/examples/designs/goban/emu/src/device/mod.rs b/examples/designs/goban/emu/src/device/mod.rs new file mode 100644 index 000000000..db0b63ed2 --- /dev/null +++ b/examples/designs/goban/emu/src/device/mod.rs @@ -0,0 +1,2 @@ +//! The goban device: its own Core model; the cores are instantiated by component on the CBus. +pub mod core; diff --git a/examples/designs/goban/emu/src/lib.rs b/examples/designs/goban/emu/src/lib.rs new file mode 100644 index 000000000..545892414 --- /dev/null +++ b/examples/designs/goban/emu/src/lib.rs @@ -0,0 +1 @@ +pub mod device; diff --git a/examples/designs/goban/sims/device/arch/DeviceConfig.scala b/examples/designs/goban/sims/device/arch/DeviceConfig.scala new file mode 100644 index 000000000..25ae9a744 --- /dev/null +++ b/examples/designs/goban/sims/device/arch/DeviceConfig.scala @@ -0,0 +1,18 @@ +package examples.designs.goban.sims.device + +import buckyball.config.Design +import chisel3.experimental.hierarchy.{Instance, Instantiate} +import examples.designs.goban.DeviceTop +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import framework.device.dlink.BaseDevice +import _root_.configs.GlobalConfig +import _root_.sims.soc.SystemTarget +import java.nio.file.{Files, Paths} + +class DeviceConfig(design: Design) + extends SystemTarget( + GlobalConfig.fromProto(design, CBus.parameters(design, CoreComponents.factories)) + ) { + def instantiate(b: GlobalConfig): Instance[BaseDevice] = Instantiate(new DeviceTop(b, design)) +} diff --git a/examples/designs/goban/sims/device/arch/build.sc b/examples/designs/goban/sims/device/arch/build.sc new file mode 100644 index 000000000..5b94111b8 --- /dev/null +++ b/examples/designs/goban/sims/device/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.arch.{build => design_build} +import $file.^.^.^.^.^.^.arch.src.main.scala.sims.{build => sims_build} + +val entryRoot: os.Path = millSourcePath + +object simulation extends SbtModule { + override def millSourcePath = entryRoot + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(design_build.design, sims_build.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(entryRoot / "DeviceConfig.scala") + override def mainClass = Some("sims.verilator.Elaborate") +} diff --git a/examples/designs/goban/sims/device/emu/Cargo.lock b/examples/designs/goban/sims/device/emu/Cargo.lock new file mode 100644 index 000000000..1cb8200cc --- /dev/null +++ b/examples/designs/goban/sims/device/emu/Cargo.lock @@ -0,0 +1,2008 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "assert_cmd" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aa3a22042e45de04255c7bf3626e239f450200fd0493c1e382263544b20aea6" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bebop" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-fd-redirect", + "bebop-p2e", + "bebop-report", + "bebop-rtl-trace", + "bebop-terminal", + "bebop-trace-perfetto", + "bebop-uart", + "bebop-vcs", + "bebop-verilator", + "bebop-vm", + "camino", + "clap", + "duct", + "env_logger", + "libc", + "log", + "nix", + "serde", + "serde_json", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-goban" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-fd-redirect" +version = "0.1.0" +dependencies = [ + "libc", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-p2e" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "bebop-rtl-trace", + "bebop-uart", + "duct", + "libc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bebop-report" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-rtl-trace" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", +] + +[[package]] +name = "bebop-sim-build" +version = "0.1.0" +dependencies = [ + "prost", + "prost-build", + "prost-types", +] + +[[package]] +name = "bebop-sim-goban-device" +version = "0.1.0" +dependencies = [ + "assert_cmd", + "bebop", + "bebop-bemu", + "bebop-design-goban", + "bebop-report", + "bebop-sim-build", + "bebop-verilator", + "chrono", + "clap", + "libtest-mimic", + "predicates", + "serde", + "toml", + "walkdir", +] + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-terminal" +version = "0.1.0" +dependencies = [ + "crossterm", + "ratatui", +] + +[[package]] +name = "bebop-trace-perfetto" +version = "0.1.0" +dependencies = [ + "clap", + "serde_json", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vcs" +version = "0.1.0" +dependencies = [ + "snafu", +] + +[[package]] +name = "bebop-verilator" +version = "0.1.0" +dependencies = [ + "bebop-dasm", + "bebop-fd-redirect", + "bebop-rtl-trace", + "bebop-uart", + "libc", + "snafu", +] + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" +dependencies = [ + "clap", + "font8x8", + "sdl2", + "vt100", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "camino" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbad30e4b4c14a39e3cc8aed085a12a327257c316619c93581e017bc52be591" + +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compact_str" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.13.2", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix 0.38.44", + "signal-hook 0.3.18", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "duct" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "font8x8" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875488b8711a968268c7cf5d139578713097ca4635a76044e8fe8eedf831d07e" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "indoc" +version = "2.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1" +dependencies = [ + "rustversion", +] + +[[package]] +name = "instability" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3b5acc1e2fd9375041a388da33d1eb8aed5f7a8c0dd3543e3ea2805adfbe20" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "os_pipe" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags 2.13.2", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags 2.13.2", + "smallvec", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdl2" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d42407afc6a8ab67e36f92e80b8ba34cbdc55aaeed05249efe9a2e8d0e9feef" +dependencies = [ + "bitflags 1.3.2", + "lazy_static", + "libc", + "sdl2-sys", +] + +[[package]] +name = "sdl2-sys" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff61407fc75d4b0bbc93dc7e4d6c196439965fbef8e4a4f003a36095823eac0" +dependencies = [ + "cfg-if", + "libc", + "pkg-config", + "version-compare", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "shared_child" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "607549934f6cc26b89cfecfdc46fa90f1e5d1536a68349b0c3a4f9d1c0d37959" +dependencies = [ + "libc", + "sigchld", + "windows-sys 0.61.2", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "sigchld" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f2b37f04360cd465089b87a9c3869c08220a2f3458463f0adf8badf5e77f2c" +dependencies = [ + "libc", + "os_pipe", + "signal-hook 0.4.5", +] + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b29ffa27d3d59a049bb328258de100f8012848d1b08d444e08bbc02ac468435e" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook 0.3.18", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix 1.1.5", + "windows-sys 0.61.2", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version-compare" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" + +[[package]] +name = "vt100" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de" +dependencies = [ + "itoa", + "log", + "unicode-width 0.1.14", + "vte", +] + +[[package]] +name = "vte" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/goban/sims/device/emu/Cargo.toml b/examples/designs/goban/sims/device/emu/Cargo.toml new file mode 100644 index 000000000..08691a836 --- /dev/null +++ b/examples/designs/goban/sims/device/emu/Cargo.toml @@ -0,0 +1,83 @@ +[workspace] + +[package] +name = "bebop-sim-goban-device" +version = "0.1.0" +edition = "2021" +autobins = false +autotests = false + +# The model of this simulation entry; the runners below are built from it. +[lib] +name = "simulation" + +[[bin]] +name = "bebop" +path = "../../../../../../bebop/src/main.rs" + +[[bin]] +name = "bebop-bemu" +path = "../../../../../../bebop/src/bin/bebop-bemu.rs" +required-features = ["bemu"] + +[[bin]] +name = "bebop-verilator" +path = "../../../../../../bebop/src/bin/bebop-verilator.rs" +required-features = ["verilator"] + +[[bin]] +name = "bebop-p2e" +path = "../../../../../../bebop/src/bin/bebop-p2e.rs" +required-features = ["p2e"] + +[[test]] +name = "test_bemu" +path = "../../../../../../bebop/tests/test_bemu.rs" +harness = false +required-features = ["bemu"] + +[[test]] +name = "test_verilator" +path = "../../../../../../bebop/tests/test_verilator.rs" +harness = false +required-features = ["verilator"] + +[[test]] +name = "test_p2e" +path = "../../../../../../bebop/tests/test_p2e.rs" +harness = false +required-features = ["p2e"] + +[[bin]] +name = "bebop-vcs" +path = "../../../../../../bebop/src/bin/bebop-vcs.rs" + +[[test]] +name = "test_vcs" +path = "../../../../../../bebop/tests/test_vcs.rs" +harness = false + +[features] +bemu = ["bebop/bemu"] +verilator = ["bebop/verilator", "dep:bebop-verilator"] +p2e = ["bebop/p2e"] + +[dependencies] +bebop = { path = "../../../../../../bebop" } +bebop-verilator = { path = "../../../../../../bebop/src/nodes/verilator", optional = true } +bebop-bemu = { path = "../../../../../../bebop/src/nodes/bemu" } +bebop-design-goban = { path = "../../../emu" } +clap = { version = "4", features = ["derive"] } + +[build-dependencies] +bebop-sim-build = { path = "../../../../../../bebop/src/nodes/lib/sim-build" } + +[dev-dependencies] +bebop-report = { path = "../../../../../../bebop/src/nodes/lib/report" } +libtest-mimic = "0.8" +assert_cmd = "2" +walkdir = "2" +predicates = "3" +serde = { version = "1", features = ["derive"] } +toml = "0.8" +chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/examples/designs/goban/sims/device/emu/build.rs b/examples/designs/goban/sims/device/emu/build.rs new file mode 100644 index 000000000..d634d5dba --- /dev/null +++ b/examples/designs/goban/sims/device/emu/build.rs @@ -0,0 +1,7 @@ +use std::{env, path::PathBuf}; + +fn main() { + let entry = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + bebop_sim_build::balls(&entry.join("../../../output/generated/design.pb")); + bebop_sim_build::runner_links(); +} diff --git a/examples/designs/goban/sims/device/emu/src/lib.rs b/examples/designs/goban/sims/device/emu/src/lib.rs new file mode 100644 index 000000000..ec91dc757 --- /dev/null +++ b/examples/designs/goban/sims/device/emu/src/lib.rs @@ -0,0 +1,11 @@ +//! The goban device simulation. +//! Its Core models register in `bebop_bemu::CORES`; the Balls its design.pb +//! names are compiled here and register in `bebop_bemu::BALLS`. +extern crate bebop_design_goban; + +include!(concat!(env!("OUT_DIR"), "/balls.rs")); + +pub const SIMULATION: bebop_bemu::Simulation = bebop_bemu::Simulation { + design: include_bytes!("../../../../output/generated/design.pb"), + instantiate: bebop_bemu::device::cbus::instantiate, +}; diff --git a/examples/designs/goban/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/goban/sims/device/regression/bemu/workloads-bare.toml new file mode 100644 index 000000000..7c481bca7 --- /dev/null +++ b/examples/designs/goban/sims/device/regression/bemu/workloads-bare.toml @@ -0,0 +1,7 @@ +[workloads] +tests = [ + "goban-matmul-ctest-design-matmul_threads-baremetal", + "goban-core0-soctest-clint_timer_interrupt-baremetal", + "goban-core0-soctest-clint_software_interrupt-baremetal", + "goban-core0-soctest-plic_register_access-baremetal", +] diff --git a/examples/designs/goban/sims/device/regression/bemu/workloads-linux.toml b/examples/designs/goban/sims/device/regression/bemu/workloads-linux.toml new file mode 100644 index 000000000..07563aaaf --- /dev/null +++ b/examples/designs/goban/sims/device/regression/bemu/workloads-linux.toml @@ -0,0 +1,4 @@ +[workloads] +tests = [ + "fw_payload-h1-goban-linux.elf", +] diff --git a/examples/designs/goban/sims/device/regression/p2e/workloads-linux.toml b/examples/designs/goban/sims/device/regression/p2e/workloads-linux.toml new file mode 100644 index 000000000..9a02148b5 --- /dev/null +++ b/examples/designs/goban/sims/device/regression/p2e/workloads-linux.toml @@ -0,0 +1,4 @@ +[workloads] +tests = [ + "fw_payload-h1-goban-linux.hex", +] diff --git a/examples/designs/goban/sims/device/regression/verilator/workloads-bare.toml b/examples/designs/goban/sims/device/regression/verilator/workloads-bare.toml new file mode 100644 index 000000000..7c481bca7 --- /dev/null +++ b/examples/designs/goban/sims/device/regression/verilator/workloads-bare.toml @@ -0,0 +1,7 @@ +[workloads] +tests = [ + "goban-matmul-ctest-design-matmul_threads-baremetal", + "goban-core0-soctest-clint_timer_interrupt-baremetal", + "goban-core0-soctest-clint_software_interrupt-baremetal", + "goban-core0-soctest-plic_register_access-baremetal", +] diff --git a/examples/designs/goban/sims/device/stack/CMakeLists.txt b/examples/designs/goban/sims/device/stack/CMakeLists.txt new file mode 100644 index 000000000..18f9e5345 --- /dev/null +++ b/examples/designs/goban/sims/device/stack/CMakeLists.txt @@ -0,0 +1,48 @@ +# ===----------------------------------------------------------------------=== +# The software stack of the goban device simulation: every level the device +# needs, built bottom up. +# +# bbdev stack --build "--design goban" +# +# configures this with DESIGN_CMAKE, generated from design.pb. Each level is a +# CMake build of its own: cores and threads build buddy-opt plugins for the +# host, the device builds RISC-V libraries. +# thread/matmul libmatmul-compiler.so, program.bin, loader.o, libkernels*.a +# core/goban libengine-compiler.so, execute.o +# device runtime, runtime-baremetal, runtime-linux, engine; device.cmake +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(goban-device-stack NONE) +include(ExternalProject) +include("${DESIGN_CMAKE}") + +# The matmul hardware thread. +ExternalProject_Add(thread-matmul + SOURCE_DIR "${DESIGN_THREAD_STACK_matmul}" + BINARY_DIR "${DESIGN_THREAD_BUILD_matmul}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DTHREAD=matmul + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# Goban's CPU core, which runs the matmul thread. +ExternalProject_Add(core-goban + SOURCE_DIR "${DESIGN_CORE_STACK_goban}" + BINARY_DIR "${DESIGN_CORE_BUILD_goban}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DCORE=goban + DEPENDS thread-matmul + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# The device: the runtime that schedules jobs onto the cores. +ExternalProject_Add(device + SOURCE_DIR "${DESIGN_DEVICE_STACK}" + BINARY_DIR "${DESIGN_DEVICE_BUILD}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" + -DCMAKE_C_COMPILER=riscv64-unknown-linux-gnu-gcc + -DCMAKE_CXX_COMPILER=riscv64-unknown-linux-gnu-g++ + DEPENDS thread-matmul core-goban + BUILD_ALWAYS ON + INSTALL_COMMAND "") diff --git a/examples/designs/goban/sims/device/stack/core/goban/CMakeLists.txt b/examples/designs/goban/sims/device/stack/core/goban/CMakeLists.txt new file mode 100644 index 000000000..1eab481d1 --- /dev/null +++ b/examples/designs/goban/sims/device/stack/core/goban/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.20) +project(goban-core-stack LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +add_subdirectory("${DESIGN_REPO}/examples/designs/goban/stack/engine" engine) +add_subdirectory("${DESIGN_REPO}/examples/designs/goban/stack/midend" midend) diff --git a/examples/designs/goban/sims/device/stack/runtime/CMakeLists.txt b/examples/designs/goban/sims/device/stack/runtime/CMakeLists.txt new file mode 100644 index 000000000..4408bd386 --- /dev/null +++ b/examples/designs/goban/sims/device/stack/runtime/CMakeLists.txt @@ -0,0 +1,37 @@ +# ===----------------------------------------------------------------------=== +# The goban device: its cores, their hardware threads and the runtime that +# schedules jobs onto the cores. +# +# bbdev stack --build "--design goban" +# +# builds every thread and core stack of the device first, then configures this +# with DESIGN_CMAKE, generated from design.pb, and the RISC-V Linux toolchain. +# Products, exported in device.cmake for the programs that run on the device: +# runtime, runtime-baremetal, runtime-linux job scheduling over the cores +# engine hardware threads and instruction streams +# bindings how each core runs jobs and each +# hardware thread is loaded +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(goban-device LANGUAGES C CXX) +include("${DESIGN_CMAKE}") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# How each core runs jobs, and how each hardware thread is loaded. Nothing +# references these bindings, so they are linked as objects rather than taken +# from an archive; as link options they come before the libraries they call. +add_library(bindings INTERFACE) +foreach(CORE IN LISTS DESIGN_CORES) + target_link_options(bindings INTERFACE "${DESIGN_CORE_BUILD_${CORE}}/execute.o") +endforeach() +foreach(THREAD IN LISTS DESIGN_THREADS) + target_link_options(bindings INTERFACE "${DESIGN_THREAD_BUILD_${THREAD}}/loader.o") +endforeach() +# DMA helpers are sized by the target with the fewest bank lines. +list(GET DESIGN_RUNTIME_TARGETS 0 PARAMS_TARGET) +set(RUNTIME_PARAMS "${DESIGN_ISA_DIR}/${PARAMS_TARGET}") + +add_subdirectory("${DESIGN_REPO}/stack/engine" engine) +add_subdirectory("${DESIGN_REPO}/stack/runtime" runtime) +export(TARGETS bindings runtime runtime-baremetal runtime-linux engine + runtime-shared runtime-linux-shared engine-shared FILE device.cmake) diff --git a/examples/designs/goban/sims/device/workloads/CMakeLists.txt b/examples/designs/goban/sims/device/workloads/CMakeLists.txt new file mode 100644 index 000000000..3d1f085d9 --- /dev/null +++ b/examples/designs/goban/sims/device/workloads/CMakeLists.txt @@ -0,0 +1,78 @@ +include("${DESIGN_DEVICE_BUILD}/device.cmake") + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/base" + "${CMAKE_BINARY_DIR}/src/CTest/base" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tlb" + "${CMAKE_BINARY_DIR}/src/CTest/tlb" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tutorial" + "${CMAKE_BINARY_DIR}/src/CTest/tutorial" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/embench" + "${CMAKE_BINARY_DIR}/src/CTest/embench" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/coremark" + "${CMAKE_BINARY_DIR}/src/CTest/coremark" +) +foreach(BUCKYBALL_COMPONENT_WORKLOAD_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(COMPONENT_WORKLOADS + "${DESIGN_TARGET_WORKLOADS_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" + ) + if( + COMPONENT_WORKLOADS + STREQUAL + "${BUCKYBALL_REPO_ROOT}/examples/designs/${BUCKYBALL_WORKLOAD_DESIGN}/workloads" + ) + continue() + endif() + set(BUCKYBALL_COMPONENT_WORKLOAD_PROFILE + "${DESIGN_TARGET_ID_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" + ) + set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}") + set(BUCKYBALL_CTEST_GROUP_TARGET component-ctests-build) + if(IS_DIRECTORY "${COMPONENT_WORKLOADS}/ctests") + add_subdirectory( + "${COMPONENT_WORKLOADS}/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/components/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" + ) + endif() +endforeach() +add_dependencies( + ctest-workloads-build + base-ctests-build + tlb-ctests-build + tutorial-ctests-build + component-ctests-build +) +add_dependencies(sync-native-bin embench-all coremark-all) + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/MLIRTest/base" + "${CMAKE_BINARY_DIR}/src/MLIRTest/mlir_tests/base" +) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_MLIR_ACTIVE_TARGET "${BUCKYBALL_TARGET}") + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_TARGET_BALLS_${BUCKYBALL_TARGET}) + set(BALL_MLIR_TESTS + "${BUCKYBALL_REPO_ROOT}/examples/components/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" + ) + if(IS_DIRECTORY "${BALL_MLIR_TESTS}") + add_subdirectory( + "${BALL_MLIR_TESTS}" + "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}" + ) + endif() + endforeach() +endforeach() +add_dependencies(MLIRTest-all base-mlirtest-build balls-mlir-tests-build) + +add_subdirectory( + "${BUCKYBALL_REPO_ROOT}/examples/designs/goban/workloads" + design +) diff --git a/examples/designs/goban/sims/device/workloads/kernel/CMakeLists.txt b/examples/designs/goban/sims/device/workloads/kernel/CMakeLists.txt new file mode 100644 index 000000000..368e8cd4b --- /dev/null +++ b/examples/designs/goban/sims/device/workloads/kernel/CMakeLists.txt @@ -0,0 +1,11 @@ +# Linux programs installed into the goban rootfs, each with the hart it runs on. +set(BUCKYBALL_KERNEL_PROGRAMS + 0 goban-matmul-ctest-rvv_kernel-linux + 0 goban-matmul-ctest-rvv_silu-linux + 0 goban-matmul-ctest-rvv_swiglu-linux + 0 goban-matmul-ctest-rvv_snake-linux + 0 goban-matmul-ctest-rvv_mxfp8_quant-linux + 0 goban-matmul-ctest-rvv_matmul-linux + 0 goban-matmul-ctest-mvin_mvout_test-linux + 0 goban-matmul-ctest-mvin_mvout_stride_test-linux + PARENT_SCOPE) diff --git a/examples/designs/goban/sims/ip/uvm/registry.toml b/examples/designs/goban/sims/ip/uvm/registry.toml new file mode 100644 index 000000000..8e8131449 --- /dev/null +++ b/examples/designs/goban/sims/ip/uvm/registry.toml @@ -0,0 +1 @@ +ips = [] diff --git a/examples/designs/goban/stack/engine/CMakeLists.txt b/examples/designs/goban/stack/engine/CMakeLists.txt new file mode 100644 index 000000000..21a7dac9a --- /dev/null +++ b/examples/designs/goban/stack/engine/CMakeLists.txt @@ -0,0 +1,14 @@ +# Core execution binding; instruction streams use the framework engine. +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +set(ENGINE "${DESIGN_REPO}/stack/engine") +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/execute.o" + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" + "-I${ENGINE}/include" -c "${RUNTIME}/execute/engine.c" -o "${CMAKE_BINARY_DIR}/execute.o" + DEPENDS "${RUNTIME}/execute/engine.c" "${RUNTIME}/scheduler/runtime.h" + "${ENGINE}/include/engine.h" + VERBATIM) +add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") diff --git a/examples/designs/goban/stack/midend/CMakeLists.txt b/examples/designs/goban/stack/midend/CMakeLists.txt new file mode 100644 index 000000000..f8202e654 --- /dev/null +++ b/examples/designs/goban/stack/midend/CMakeLists.txt @@ -0,0 +1,8 @@ +# Host compiler plugin for this design's core. +set(ENGINE "${DESIGN_REPO}/stack/engine") +set(TARGET ${DESIGN_CORE_TARGET_${CORE}}) + +add_subdirectory("${ENGINE}/compiler" engine-compiler) +add_library(engine-compiler-plugin MODULE $) +set_target_properties(engine-compiler-plugin PROPERTIES OUTPUT_NAME engine-compiler + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") diff --git a/examples/designs/goban/workloads/CMakeLists.txt b/examples/designs/goban/workloads/CMakeLists.txt new file mode 100644 index 000000000..6f05e9c42 --- /dev/null +++ b/examples/designs/goban/workloads/CMakeLists.txt @@ -0,0 +1,11 @@ +add_custom_target(design-ctests-build) + +add_subdirectory(device) + +add_custom_target(design-workloads-build ALL + DEPENDS + design-ctests-build + mlirtest-workloads-build + soctests-build + COMMENT "Building goban workloads" + VERBATIM) diff --git a/examples/designs/goban/workloads/configs/tests.toml b/examples/designs/goban/workloads/configs/tests.toml new file mode 100644 index 000000000..c1c7ceb33 --- /dev/null +++ b/examples/designs/goban/workloads/configs/tests.toml @@ -0,0 +1,2 @@ +controlHart = 0 +reportHart = 0 diff --git a/examples/designs/goban/workloads/device/CMakeLists.txt b/examples/designs/goban/workloads/device/CMakeLists.txt new file mode 100644 index 000000000..ad6d4787c --- /dev/null +++ b/examples/designs/goban/workloads/device/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(core) diff --git a/examples/designs/goban/workloads/device/core/CMakeLists.txt b/examples/designs/goban/workloads/device/core/CMakeLists.txt new file mode 100644 index 000000000..bd9914576 --- /dev/null +++ b/examples/designs/goban/workloads/device/core/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(engine) diff --git a/examples/designs/goban/workloads/device/core/engine/CMakeLists.txt b/examples/designs/goban/workloads/device/core/engine/CMakeLists.txt new file mode 100644 index 000000000..6d362d881 --- /dev/null +++ b/examples/designs/goban/workloads/device/core/engine/CMakeLists.txt @@ -0,0 +1,40 @@ +set(ENGINE_DIR ${BUCKYBALL_REPO_ROOT}/stack/engine) +set(MATMUL_TASK_DIR + ${BUCKYBALL_REPO_ROOT}/examples/components/threads/matmul/stack +) +set(MATMUL_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-matmul-ctest-design-matmul_threads" +) +set(MATMUL_ELF ${MATMUL_NAME}-baremetal) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${MATMUL_ELF} + COMMAND + ${ELF_CC} -O2 -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${BBSIM_LD} + -I${BUCKYBALL_ISA_DIR}/control -I${BUCKYBALL_ISA_DIR}/design + -I${ENGINE_DIR}/include -I${MATMUL_TASK_DIR} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services -I${BAREMETAL_DIR} + ${BAREMETAL_DIR}/start.S ${CMAKE_CURRENT_SOURCE_DIR}/matmul_threads.c + $ $ + # How the device's cores run jobs and its hardware threads are loaded. + $ $ + -lgcc -o ${MATMUL_ELF} + DEPENDS + matmul_threads.c + ${MATMUL_TASK_DIR}/matmul.h + runtime-baremetal + runtime + engine + ${ENGINE_DIR}/include/thread.h + ${ENGINE_DIR}/include/engine.h + ${BUCKYBALL_ISA_DIR}/control/params.h + ${BBSIM_LD} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMAND_EXPAND_LISTS +) +add_custom_target( + ${MATMUL_NAME}-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${MATMUL_ELF} +) +add_dependencies(design-ctests-build ${MATMUL_NAME}-ctest-build) diff --git a/examples/designs/goban/workloads/device/core/engine/matmul_threads.c b/examples/designs/goban/workloads/device/core/engine/matmul_threads.c new file mode 100644 index 000000000..715d4c067 --- /dev/null +++ b/examples/designs/goban/workloads/device/core/engine/matmul_threads.c @@ -0,0 +1,61 @@ +#include +#include +#include +#include +#include + +static int8_t a[4][256] __attribute__((aligned(64))); +static int8_t b[4][256] __attribute__((aligned(64))); +static int32_t bias[4][16] __attribute__((aligned(64))); +static int32_t output[4][256] __attribute__((aligned(64))); +static struct matmul_task args[4]; +static struct engine_job jobs[4]; +static struct engine_result results[4]; + +static int compute(void *context, void *argument) { + (void)argument; + struct engine *engine = context; + if (engine->thread_count != 4) + return 1; + for (unsigned tid = 0; tid < 4; ++tid) { + for (unsigned i = 0; i < 256; ++i) { + a[tid][i] = (i * 3 + tid) % 13 - 6; + b[tid][i] = (i * 5 + tid) % 11 - 5; + } + for (unsigned i = 0; i < 16; ++i) + bias[tid][i] = i - 8 + tid; + args[tid] = + (struct matmul_task){(uintptr_t)a[tid], (uintptr_t)b[tid], + (uintptr_t)bias[tid], (uintptr_t)output[tid]}; + jobs[tid] = (struct engine_job){ + .thread = tid, + .task = {.id = tid + 1, .signature = engine->slots[tid].signature}, + .opaque = &args[tid]}; + } + engine_run(engine, jobs, 4, results); + for (unsigned tid = 0; tid < 4; ++tid) { + if (results[tid].state != ENGINE_SUCCEEDED || results[tid].value) + return 2; + for (unsigned row = 0; row < 16; ++row) + for (unsigned col = 0; col < 16; ++col) { + int32_t expected = bias[tid][col]; + for (unsigned k = 0; k < 16; ++k) + expected += a[tid][row * 16 + k] * b[tid][k * 16 + col]; + if (output[tid][row * 16 + col] != expected) + return 3; + } + } + return 0; +} + +int main(void) { + struct runtime_core core = + runtime_core(0, bb_cores[bb_hart_slot(0)].signature, 0); + struct runtime_job job = { + .signature = core.signature, .core = 0, .entry = compute}; + struct runtime_result result; + struct runtime runtime; + runtime_init(&runtime, &core, 1, &job, 1, &result); + runtime_baremetal_run(&runtime, 0); + return result.state == RUNTIME_SUCCEEDED ? result.status : 4; +} diff --git a/examples/chips/goban/workloads/ctests/mvover_rows.c b/examples/designs/goban/workloads/device/core/engine/mvover_rows.c similarity index 82% rename from examples/chips/goban/workloads/ctests/mvover_rows.c rename to examples/designs/goban/workloads/device/core/engine/mvover_rows.c index e0362484c..0953ab0f6 100644 --- a/examples/chips/goban/workloads/ctests/mvover_rows.c +++ b/examples/designs/goban/workloads/device/core/engine/mvover_rows.c @@ -1,8 +1,8 @@ #include -#include +#include +#include #include #include -#include #define ROWS 8 #define ROW_BYTES (BANK_WIDTH / 8) @@ -11,23 +11,20 @@ #endif static uint8_t input[ROWS * ROW_BYTES] __attribute__((aligned(64))); static uint8_t output[ROWS * ROW_BYTES] __attribute__((aligned(64))); -#if BB_SHARED_PHYSICAL_BANK_NUM < 1 || BB_SHARED_BANK_BASE >= VIRTUAL_BANK_NUM +#if SHARED_BANK_NUM < 1 || BB_SHARED_BANK_BASE >= VIRTUAL_BANK_NUM #error "mvover_rows requires a shared bank for issuer/owner trace coverage" #endif static uint8_t shared_output[ROWS * ROW_BYTES] __attribute__((aligned(64))); static unsigned source_ready, target_ready, moved, stored; int main(void) { - core_id_t id = bb_get_core_id(); - if (id.tile != 0) - for (;;) - asm volatile("wfi"); - if (id.core == 1) { + const uint32_t core = bb_get_hart_id(); + if (core == 1) { for (unsigned i = 0; i < sizeof(input); ++i) input[i] = (uint8_t)(i * 17 + 3); bb_mem_alloc(1, 1, 1); bb_mem_alloc(BB_SHARED_BANK_BASE, 1, 1); bb_mvin((uintptr_t)input, 1, ROWS, 1); - // Hart1 issues the write; the shared storage belongs to this Tile's Core0. + // Hart 1 issues the write; the shared bank pool is owned by hart 0. bb_mvin((uintptr_t)input, BB_SHARED_BANK_BASE, ROWS, 1); bb_mvout((uintptr_t)shared_output, BB_SHARED_BANK_BASE, ROWS, 1); for (unsigned i = 0; i < sizeof(input); ++i) @@ -35,14 +32,14 @@ int main(void) { return 1; bb_mem_release(BB_SHARED_BANK_BASE); __atomic_store_n(&source_ready, 1, __ATOMIC_RELEASE); - } else if (id.core == 2) { + } else if (core == 2) { bb_mem_alloc(8, 1, 1); __atomic_store_n(&target_ready, 1, __ATOMIC_RELEASE); while (!__atomic_load_n(&moved, __ATOMIC_ACQUIRE)) { } bb_mvout((uintptr_t)output, 8, ROWS, 1); __atomic_store_n(&stored, 1, __ATOMIC_RELEASE); - } else if (id.core == 0) { + } else if (core == 0) { while (!__atomic_load_n(&source_ready, __ATOMIC_ACQUIRE) || !__atomic_load_n(&target_ready, __ATOMIC_ACQUIRE)) { } diff --git a/examples/designs/multi-rocket/arch/build.sc b/examples/designs/multi-rocket/arch/build.sc new file mode 100644 index 000000000..ca8f7e2b1 --- /dev/null +++ b/examples/designs/multi-rocket/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.arch.src.main.scala.framework.device.cbus.{build => cbus_build} +import $file.^.^.^.{build => examples} +import $file.^.^.^.components.cores.rocket.arch.{build => rocket} + +val designRoot: os.Path = millSourcePath / os.up + +object design extends SbtModule { + override def generatedSources = T(cbus_build.registry(sources() ++ rocket.library.sources(), T.dest)) + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(examples.library, rocket.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(designRoot / "arch", designRoot / "configs") +} diff --git a/examples/designs/multi-rocket/arch/device/DeviceTop.scala b/examples/designs/multi-rocket/arch/device/DeviceTop.scala new file mode 100644 index 000000000..0c67d0282 --- /dev/null +++ b/examples/designs/multi-rocket/arch/device/DeviceTop.scala @@ -0,0 +1,80 @@ +package examples.designs.multirocket + +import framework.lib.arbiter.{RRArbiter} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} +import framework.device.configs.DeviceParam +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import buckyball.config.Design +import _root_.configs.GlobalConfig +import framework.core.clink.CLinkIO +import framework.core.memory.home.Home +import framework.device.ddr.Bridge +import framework.device.peripherals.Controller +import framework.device.peripherals.bootrom.BootRomLines +import framework.device.dlink.BaseDevice +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.LineResponse +import framework.lib.queue.{Queue => ResponseQueue} + +/** Configured Rocket instances share one coherent Home and one device memory map. */ +@instantiable +class DeviceTop(b: GlobalConfig, design: Design) extends BaseDevice(b.device.ddr.axi) { + val device = b.device + val parameters = device.cores + + require(parameters.forall(p => !p.clink.backing && p.clink.requesters == 2 && p.acc.isEmpty)) + val sharedHome = parameters(0).memory.home + require(parameters.forall(_.memory.home == sharedHome)) + require(sharedHome.memory.agents == parameters.size * 2) + + val bus = Instantiate(new CBus(b.device, CBus.generators(design, b.device, b.sim, CoreComponents.factories))) + + val home: Instance[Home] = Instantiate(new Home(sharedHome)) + home.io.active := true.B + home.io.blockRequesterRsp := false.B + home.io.requesters <> bus.coherent + + val peripherals = Instantiate(new Controller(device, device.ddr.axi, parameters(0).memory.memoryAccess.memory)) + + val peripheralBus = Instantiate(new axi4.Decoder( + Seq( + (device.clint.base, device.clint.bytes), + (device.plic.base, device.plic.bytes), + (device.scu.baseAddress, device.scu.totalSizeBytes) + ), + device.ddr.axi + )) + + peripheralBus.io.in <> bus.uncached + peripherals.io.axi <> peripheralBus.io.matched + bus.external <> peripheralBus.io.other + peripherals.io.hardwareInterrupt.sources := 0.U + bus.interrupts := peripherals.io.hardwareInterrupt.harts + + val rom = Instantiate(new BootRomLines(device.bootrom, sharedHome.memory.chi)) + val bridge = Instantiate(new Bridge(device.ddr)) + require(device.ddr.clients == 1) + val request = home.io.memoryReq + val inRom = request.bits.addr >= device.bootrom.base.U && + request.bits.addr < (device.bootrom.base + device.bootrom.bytes).U + rom.io.request.valid := request.valid && inRom + rom.io.request.bits := request.bits + bridge.io.request(0).valid := request.valid && !inRom + bridge.io.request(0).bits := request.bits + request.ready := Mux(inRom, rom.io.request.ready, bridge.io.request(0).ready) + val responses = Instantiate(new RRArbiter(new LineResponse(sharedHome.memory.chi), 2)) + responses.io.in(0) <> rom.io.response + responses.io.in(1) <> bridge.io.response(0) + val response = Instantiate(new ResponseQueue(new LineResponse(sharedHome.memory.chi), 1, false, false)) + response.io.enq <> responses.io.out + home.io.memoryResp <> response.io.deq + + val memoryBus = Instantiate(new axi4.Interconnect(axi4.InterconnectParam(device.ddr.axi, 2, None))) + memoryBus.io.in(0) <> bus.dlink.axi + memoryBus.io.in(1) <> bridge.io.axi + dlink.axi <> memoryBus.io.out +} diff --git a/examples/designs/multi-rocket/configs/design.toml b/examples/designs/multi-rocket/configs/design.toml new file mode 100644 index 000000000..a55a275cc --- /dev/null +++ b/examples/designs/multi-rocket/configs/design.toml @@ -0,0 +1,4 @@ +[device] +include = "device/default.toml" +[sims] +include = "sims/default.toml" diff --git a/examples/designs/multi-rocket/configs/device/clint.toml b/examples/designs/multi-rocket/configs/device/clint.toml new file mode 100644 index 000000000..9d0ad86a0 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/clint.toml @@ -0,0 +1,4 @@ +base = 0x2000000 +bytes = 0x10000 +tickCycles = 1000 +hartIds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] diff --git a/examples/designs/multi-rocket/configs/device/core/c2c/core.toml b/examples/designs/multi-rocket/configs/device/core/c2c/core.toml new file mode 100644 index 000000000..ebf8171e4 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = false +peerCores = 32 +axi = "default" diff --git a/examples/designs/multi-rocket/configs/device/core/default.toml b/examples/designs/multi-rocket/configs/device/core/default.toml new file mode 100644 index 000000000..8f3cda39a --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/default.toml @@ -0,0 +1,30 @@ +component = "rocket" +implementation = "rocket.toml" +threads = [] +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +controller = "default" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[sharedMem] +enable = false +entries = 16384 +inputChannels = 1 +defaultGroupCount = 1 +virtualBankCount = 16 +memCycles = 10 +bankEntries = 1024 +bankWidth = 128 diff --git a/examples/designs/multi-rocket/configs/device/core/interconnect/core.toml b/examples/designs/multi-rocket/configs/device/core/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/multi-rocket/configs/device/core/memory/access/core.toml b/examples/designs/multi-rocket/configs/device/core/memory/access/core.toml new file mode 100644 index 000000000..bb87f76a4 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +accessTagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x20000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/multi-rocket/configs/device/core/memory/core.toml b/examples/designs/multi-rocket/configs/device/core/memory/core.toml new file mode 100644 index 000000000..985e1cf13 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/memory/core.toml @@ -0,0 +1,4 @@ +uncached = "default" +access = "access/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/multi-rocket/configs/device/core/memory/dcache/core.toml b/examples/designs/multi-rocket/configs/device/core/memory/dcache/core.toml new file mode 100644 index 000000000..88b00a13d --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/memory/dcache/core.toml @@ -0,0 +1,6 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 diff --git a/examples/designs/multi-rocket/configs/device/core/memory/dcache/shared.toml b/examples/designs/multi-rocket/configs/device/core/memory/dcache/shared.toml new file mode 100644 index 000000000..0396137af --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/memory/dcache/shared.toml @@ -0,0 +1,6 @@ +cacheLines = 64 +homeId = 96 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/multi-rocket/configs/device/core/memory/home/core.toml b/examples/designs/multi-rocket/configs/device/core/memory/home/core.toml new file mode 100644 index 000000000..bfc26d87b --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 64 +mshrEntries = 8 +homeId = 96 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/multi-rocket/configs/device/core/memory/icache/shared.toml b/examples/designs/multi-rocket/configs/device/core/memory/icache/shared.toml new file mode 100644 index 000000000..0396137af --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/memory/icache/shared.toml @@ -0,0 +1,6 @@ +cacheLines = 64 +homeId = 96 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/multi-rocket/configs/device/core/rocket.toml b/examples/designs/multi-rocket/configs/device/core/rocket.toml new file mode 100644 index 000000000..b94846c8d --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/core/rocket.toml @@ -0,0 +1,12 @@ +component = "rocket" +physical_address_bits = 36 +hart_id_bits = 5 +page_levels = 3 +pmp_entries = 16 +virtual_memory = true +supervisor = true +user = true +floating_point = true +atomics = true +multiply_divide = true +external_home = true diff --git a/examples/designs/multi-rocket/configs/device/ddr.toml b/examples/designs/multi-rocket/configs/device/ddr.toml new file mode 100644 index 000000000..31815e9e1 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/ddr.toml @@ -0,0 +1,13 @@ +clients = 1 +slotsPerClient = 16 +dataBits = 128 +idBits = 4 +idBase = 0 +base = 0x80000000 +bytes = 0x20000000 +model = 0 + +[line] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 diff --git a/examples/designs/multi-rocket/configs/device/default.toml b/examples/designs/multi-rocket/configs/device/default.toml new file mode 100644 index 000000000..de4966bb9 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/default.toml @@ -0,0 +1,517 @@ +ddr = "ddr.toml" +bootrom = "default" +clint = "clint.toml" +plic = "plic.toml" +scu = "default" + +[[cores]] +include = "core/default.toml" +hartId = 0 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 1 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 2 + +[[cores]] +include = "core/default.toml" +hartId = 1 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 3 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 4 + +[[cores]] +include = "core/default.toml" +hartId = 2 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 5 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 6 + +[[cores]] +include = "core/default.toml" +hartId = 3 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 7 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 8 + +[[cores]] +include = "core/default.toml" +hartId = 4 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 9 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 10 + +[[cores]] +include = "core/default.toml" +hartId = 5 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 11 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 12 + +[[cores]] +include = "core/default.toml" +hartId = 6 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 13 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 14 + +[[cores]] +include = "core/default.toml" +hartId = 7 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 15 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 16 + +[[cores]] +include = "core/default.toml" +hartId = 8 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 17 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 18 + +[[cores]] +include = "core/default.toml" +hartId = 9 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 19 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 20 + +[[cores]] +include = "core/default.toml" +hartId = 10 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 21 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 22 + +[[cores]] +include = "core/default.toml" +hartId = 11 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 23 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 24 + +[[cores]] +include = "core/default.toml" +hartId = 12 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 25 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 26 + +[[cores]] +include = "core/default.toml" +hartId = 13 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 27 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 28 + +[[cores]] +include = "core/default.toml" +hartId = 14 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 29 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 30 + +[[cores]] +include = "core/default.toml" +hartId = 15 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 31 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 32 + +[[cores]] +include = "core/default.toml" +hartId = 16 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 33 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 34 + +[[cores]] +include = "core/default.toml" +hartId = 17 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 35 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 36 + +[[cores]] +include = "core/default.toml" +hartId = 18 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 37 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 38 + +[[cores]] +include = "core/default.toml" +hartId = 19 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 39 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 40 + +[[cores]] +include = "core/default.toml" +hartId = 20 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 41 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 42 + +[[cores]] +include = "core/default.toml" +hartId = 21 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 43 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 44 + +[[cores]] +include = "core/default.toml" +hartId = 22 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 45 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 46 + +[[cores]] +include = "core/default.toml" +hartId = 23 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 47 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 48 + +[[cores]] +include = "core/default.toml" +hartId = 24 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 49 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 50 + +[[cores]] +include = "core/default.toml" +hartId = 25 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 51 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 52 + +[[cores]] +include = "core/default.toml" +hartId = 26 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 53 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 54 + +[[cores]] +include = "core/default.toml" +hartId = 27 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 55 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 56 + +[[cores]] +include = "core/default.toml" +hartId = 28 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 57 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 58 + +[[cores]] +include = "core/default.toml" +hartId = 29 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 59 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 60 + +[[cores]] +include = "core/default.toml" +hartId = 30 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 61 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 62 + +[[cores]] +include = "core/default.toml" +hartId = 31 + +[cores.memory] + +[cores.memory.icache] +include = "core/memory/icache/shared.toml" +nodeId = 63 + +[cores.memory.dcache] + +[cores.memory.dcache.cache] +include = "core/memory/dcache/shared.toml" +nodeId = 64 diff --git a/examples/designs/multi-rocket/configs/device/plic.toml b/examples/designs/multi-rocket/configs/device/plic.toml new file mode 100644 index 000000000..3ccf21de4 --- /dev/null +++ b/examples/designs/multi-rocket/configs/device/plic.toml @@ -0,0 +1,5 @@ +base = 0xc000000 +bytes = 0x4000000 +sources = 1 +priorityBits = 3 +hartIds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] diff --git a/examples/designs/multi-rocket/configs/sims/default.toml b/examples/designs/multi-rocket/configs/sims/default.toml new file mode 100644 index 000000000..0fc0560fc --- /dev/null +++ b/examples/designs/multi-rocket/configs/sims/default.toml @@ -0,0 +1,14 @@ +diffTest = false + +[memory] +lineBytes = 64 +clockHz = 1000000000 + +[targets.device] +config = "examples.designs.multirocket.sims.device.DeviceConfig" +stack_name = "multi-rocket" +regression = "sims/device/regression" +kernel = "sims/device/workloads/kernel" + +[verification] +registry = "../../sims/ip/uvm/registry.toml" diff --git a/examples/designs/multi-rocket/sims/device/arch/DeviceConfig.scala b/examples/designs/multi-rocket/sims/device/arch/DeviceConfig.scala new file mode 100644 index 000000000..9a9ef4f76 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/arch/DeviceConfig.scala @@ -0,0 +1,18 @@ +package examples.designs.multirocket.sims.device + +import buckyball.config.Design +import chisel3.experimental.hierarchy.{Instance, Instantiate} +import examples.designs.multirocket.DeviceTop +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import framework.device.dlink.BaseDevice +import _root_.configs.GlobalConfig +import _root_.sims.soc.SystemTarget +import java.nio.file.{Files, Paths} + +class DeviceConfig(design: Design) + extends SystemTarget( + GlobalConfig.fromProto(design, CBus.parameters(design, CoreComponents.factories)) + ) { + def instantiate(b: GlobalConfig): Instance[BaseDevice] = Instantiate(new DeviceTop(b, design)) +} diff --git a/examples/designs/multi-rocket/sims/device/arch/build.sc b/examples/designs/multi-rocket/sims/device/arch/build.sc new file mode 100644 index 000000000..5b94111b8 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.arch.{build => design_build} +import $file.^.^.^.^.^.^.arch.src.main.scala.sims.{build => sims_build} + +val entryRoot: os.Path = millSourcePath + +object simulation extends SbtModule { + override def millSourcePath = entryRoot + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(design_build.design, sims_build.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(entryRoot / "DeviceConfig.scala") + override def mainClass = Some("sims.verilator.Elaborate") +} diff --git a/examples/designs/multi-rocket/sims/device/emu/Cargo.lock b/examples/designs/multi-rocket/sims/device/emu/Cargo.lock new file mode 100644 index 000000000..e2f70bc76 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/emu/Cargo.lock @@ -0,0 +1,1971 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "assert_cmd" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aa3a22042e45de04255c7bf3626e239f450200fd0493c1e382263544b20aea6" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bebop" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-fd-redirect", + "bebop-p2e", + "bebop-report", + "bebop-rtl-trace", + "bebop-terminal", + "bebop-trace-perfetto", + "bebop-uart", + "bebop-vcs", + "bebop-verilator", + "bebop-vm", + "camino", + "clap", + "duct", + "env_logger", + "libc", + "log", + "nix", + "serde", + "serde_json", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-core-rocket" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-fd-redirect" +version = "0.1.0" +dependencies = [ + "libc", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-p2e" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "bebop-rtl-trace", + "bebop-uart", + "duct", + "libc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bebop-report" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-rtl-trace" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", +] + +[[package]] +name = "bebop-sim-build" +version = "0.1.0" +dependencies = [ + "prost", + "prost-build", + "prost-types", +] + +[[package]] +name = "bebop-sim-multi-rocket-device" +version = "0.1.0" +dependencies = [ + "assert_cmd", + "bebop", + "bebop-bemu", + "bebop-core-rocket", + "bebop-report", + "bebop-sim-build", + "bebop-verilator", + "chrono", + "clap", + "libtest-mimic", + "predicates", + "serde", + "toml", + "walkdir", +] + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-terminal" +version = "0.1.0" +dependencies = [ + "crossterm", + "ratatui", +] + +[[package]] +name = "bebop-trace-perfetto" +version = "0.1.0" +dependencies = [ + "clap", + "serde_json", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vcs" +version = "0.1.0" +dependencies = [ + "snafu", +] + +[[package]] +name = "bebop-verilator" +version = "0.1.0" +dependencies = [ + "bebop-dasm", + "bebop-fd-redirect", + "bebop-rtl-trace", + "bebop-uart", + "libc", + "snafu", +] + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" +dependencies = [ + "clap", + "font8x8", + "sdl2", + "vt100", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "camino" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbad30e4b4c14a39e3cc8aed085a12a327257c316619c93581e017bc52be591" + +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compact_str" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.13.2", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix 0.38.44", + "signal-hook 0.3.18", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "duct" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "font8x8" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875488b8711a968268c7cf5d139578713097ca4635a76044e8fe8eedf831d07e" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "indoc" +version = "2.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1" +dependencies = [ + "rustversion", +] + +[[package]] +name = "instability" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3b5acc1e2fd9375041a388da33d1eb8aed5f7a8c0dd3543e3ea2805adfbe20" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "os_pipe" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags 2.13.2", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags 2.13.2", + "smallvec", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdl2" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d42407afc6a8ab67e36f92e80b8ba34cbdc55aaeed05249efe9a2e8d0e9feef" +dependencies = [ + "bitflags 1.3.2", + "lazy_static", + "libc", + "sdl2-sys", +] + +[[package]] +name = "sdl2-sys" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff61407fc75d4b0bbc93dc7e4d6c196439965fbef8e4a4f003a36095823eac0" +dependencies = [ + "cfg-if", + "libc", + "pkg-config", + "version-compare", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "shared_child" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "607549934f6cc26b89cfecfdc46fa90f1e5d1536a68349b0c3a4f9d1c0d37959" +dependencies = [ + "libc", + "sigchld", + "windows-sys 0.61.2", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "sigchld" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f2b37f04360cd465089b87a9c3869c08220a2f3458463f0adf8badf5e77f2c" +dependencies = [ + "libc", + "os_pipe", + "signal-hook 0.4.5", +] + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b29ffa27d3d59a049bb328258de100f8012848d1b08d444e08bbc02ac468435e" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook 0.3.18", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix 1.1.5", + "windows-sys 0.61.2", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version-compare" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" + +[[package]] +name = "vt100" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de" +dependencies = [ + "itoa", + "log", + "unicode-width 0.1.14", + "vte", +] + +[[package]] +name = "vte" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/multi-rocket/sims/device/emu/Cargo.toml b/examples/designs/multi-rocket/sims/device/emu/Cargo.toml new file mode 100644 index 000000000..f6c743f8f --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/emu/Cargo.toml @@ -0,0 +1,83 @@ +[workspace] + +[package] +name = "bebop-sim-multi-rocket-device" +version = "0.1.0" +edition = "2021" +autobins = false +autotests = false + +# The model of this simulation entry; the runners below are built from it. +[lib] +name = "simulation" + +[[bin]] +name = "bebop" +path = "../../../../../../bebop/src/main.rs" + +[[bin]] +name = "bebop-bemu" +path = "../../../../../../bebop/src/bin/bebop-bemu.rs" +required-features = ["bemu"] + +[[bin]] +name = "bebop-verilator" +path = "../../../../../../bebop/src/bin/bebop-verilator.rs" +required-features = ["verilator"] + +[[bin]] +name = "bebop-p2e" +path = "../../../../../../bebop/src/bin/bebop-p2e.rs" +required-features = ["p2e"] + +[[test]] +name = "test_bemu" +path = "../../../../../../bebop/tests/test_bemu.rs" +harness = false +required-features = ["bemu"] + +[[test]] +name = "test_verilator" +path = "../../../../../../bebop/tests/test_verilator.rs" +harness = false +required-features = ["verilator"] + +[[test]] +name = "test_p2e" +path = "../../../../../../bebop/tests/test_p2e.rs" +harness = false +required-features = ["p2e"] + +[[bin]] +name = "bebop-vcs" +path = "../../../../../../bebop/src/bin/bebop-vcs.rs" + +[[test]] +name = "test_vcs" +path = "../../../../../../bebop/tests/test_vcs.rs" +harness = false + +[features] +bemu = ["bebop/bemu"] +verilator = ["bebop/verilator", "dep:bebop-verilator"] +p2e = ["bebop/p2e"] + +[dependencies] +bebop = { path = "../../../../../../bebop" } +bebop-verilator = { path = "../../../../../../bebop/src/nodes/verilator", optional = true } +bebop-bemu = { path = "../../../../../../bebop/src/nodes/bemu" } +bebop-core-rocket = { path = "../../../../../components/cores/rocket/emu" } +clap = { version = "4", features = ["derive"] } + +[build-dependencies] +bebop-sim-build = { path = "../../../../../../bebop/src/nodes/lib/sim-build" } + +[dev-dependencies] +bebop-report = { path = "../../../../../../bebop/src/nodes/lib/report" } +libtest-mimic = "0.8" +assert_cmd = "2" +walkdir = "2" +predicates = "3" +serde = { version = "1", features = ["derive"] } +toml = "0.8" +chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/examples/designs/multi-rocket/sims/device/emu/build.rs b/examples/designs/multi-rocket/sims/device/emu/build.rs new file mode 100644 index 000000000..d634d5dba --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/emu/build.rs @@ -0,0 +1,7 @@ +use std::{env, path::PathBuf}; + +fn main() { + let entry = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + bebop_sim_build::balls(&entry.join("../../../output/generated/design.pb")); + bebop_sim_build::runner_links(); +} diff --git a/examples/designs/multi-rocket/sims/device/emu/src/lib.rs b/examples/designs/multi-rocket/sims/device/emu/src/lib.rs new file mode 100644 index 000000000..c4c11131f --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/emu/src/lib.rs @@ -0,0 +1,11 @@ +//! The multi-rocket device simulation. +//! Its Core models register in `bebop_bemu::CORES`; the Balls its design.pb +//! names are compiled here and register in `bebop_bemu::BALLS`. +extern crate bebop_core_rocket; + +include!(concat!(env!("OUT_DIR"), "/balls.rs")); + +pub const SIMULATION: bebop_bemu::Simulation = bebop_bemu::Simulation { + design: include_bytes!("../../../../output/generated/design.pb"), + instantiate: bebop_bemu::device::cbus::instantiate, +}; diff --git a/examples/designs/multi-rocket/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/multi-rocket/sims/device/regression/bemu/workloads-bare.toml new file mode 100644 index 000000000..e70ffa6d5 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/regression/bemu/workloads-bare.toml @@ -0,0 +1,25 @@ +[workloads] +tests = [ + "multi-rocket-core0-ctest-design-coherent_exchange-multicore-baremetal", + "multi-rocket-core0-ctest-check-embench_sglib-combined-baremetal", + "multi-rocket-core0-ctest-check-embench_matmult-int-baremetal", + "multi-rocket-core0-ctest-cpu_arithmetic-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_aha-mont64-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_crc32-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_edn-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_huffbench-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_matmult-int-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_minver-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nbody-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nettle-aes-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nettle-sha256-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nsichneu-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_picojpeg-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_qrduino-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_sglib-combined-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_slre-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_st-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_statemate-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_ud-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_wikisort-baremetal", +] diff --git a/examples/chips/multi-rocket/regression/batch/bemu/workloads-linux.toml b/examples/designs/multi-rocket/sims/device/regression/bemu/workloads-linux.toml similarity index 100% rename from examples/chips/multi-rocket/regression/batch/bemu/workloads-linux.toml rename to examples/designs/multi-rocket/sims/device/regression/bemu/workloads-linux.toml diff --git a/examples/designs/multi-rocket/sims/device/regression/p2e/workloads-bare.toml b/examples/designs/multi-rocket/sims/device/regression/p2e/workloads-bare.toml new file mode 100644 index 000000000..73064b9f6 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/regression/p2e/workloads-bare.toml @@ -0,0 +1,21 @@ +[workloads] +tests = [ + "multi-rocket-core0-ctest-benchmark-embench_aha-mont64-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_crc32-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_edn-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_huffbench-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_matmult-int-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_minver-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_nbody-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_nettle-aes-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_nettle-sha256-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_nsichneu-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_picojpeg-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_qrduino-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_sglib-combined-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_slre-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_st-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_statemate-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_ud-baremetal.hex", + "multi-rocket-core0-ctest-benchmark-embench_wikisort-baremetal.hex", +] diff --git a/examples/chips/multi-rocket/regression/batch/p2e/workloads-linux.toml b/examples/designs/multi-rocket/sims/device/regression/p2e/workloads-linux.toml similarity index 100% rename from examples/chips/multi-rocket/regression/batch/p2e/workloads-linux.toml rename to examples/designs/multi-rocket/sims/device/regression/p2e/workloads-linux.toml diff --git a/examples/designs/multi-rocket/sims/device/regression/verilator/workloads-bare.toml b/examples/designs/multi-rocket/sims/device/regression/verilator/workloads-bare.toml new file mode 100644 index 000000000..024512457 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/regression/verilator/workloads-bare.toml @@ -0,0 +1,19 @@ +[workloads] +tests = [ + "multi-rocket-core0-ctest-design-coherent_exchange-multicore-baremetal", + "multi-rocket-core0-ctest-cpu_arithmetic-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_aha-mont64-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_crc32-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_edn-baremetal", + "multi-rocket-core0-ctest-check-embench_matmult-int-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_minver-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nbody-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nettle-aes-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nettle-sha256-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_nsichneu-baremetal", + "multi-rocket-core0-ctest-check-embench_sglib-combined-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_slre-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_st-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_statemate-baremetal", + "multi-rocket-core0-ctest-benchmark-embench_ud-baremetal", +] diff --git a/examples/designs/multi-rocket/sims/device/stack/CMakeLists.txt b/examples/designs/multi-rocket/sims/device/stack/CMakeLists.txt new file mode 100644 index 000000000..c73416325 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/stack/CMakeLists.txt @@ -0,0 +1,37 @@ +# ===----------------------------------------------------------------------=== +# The software stack of the multi-rocket device simulation: every level the device +# needs, built bottom up. +# +# bbdev stack --build "--design multi-rocket" +# +# configures this with DESIGN_CMAKE, generated from design.pb. Each level is a +# CMake build of its own: cores and threads build buddy-opt plugins for the +# host, the device builds RISC-V libraries. +# core/rocket execute.o +# device runtime, runtime-baremetal, runtime-linux, engine; device.cmake +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(multi-rocket-device-stack NONE) +include(ExternalProject) +include("${DESIGN_CMAKE}") + +# The Rocket core. +ExternalProject_Add(core-rocket + SOURCE_DIR "${DESIGN_CORE_STACK_rocket}" + BINARY_DIR "${DESIGN_CORE_BUILD_rocket}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DCORE=rocket + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# The device: the runtime that schedules jobs onto the cores. +ExternalProject_Add(device + SOURCE_DIR "${DESIGN_DEVICE_STACK}" + BINARY_DIR "${DESIGN_DEVICE_BUILD}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" + -DCMAKE_C_COMPILER=riscv64-unknown-linux-gnu-gcc + -DCMAKE_CXX_COMPILER=riscv64-unknown-linux-gnu-g++ + DEPENDS core-rocket + BUILD_ALWAYS ON + INSTALL_COMMAND "") diff --git a/examples/designs/multi-rocket/sims/device/stack/runtime/CMakeLists.txt b/examples/designs/multi-rocket/sims/device/stack/runtime/CMakeLists.txt new file mode 100644 index 000000000..be49a4d40 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/stack/runtime/CMakeLists.txt @@ -0,0 +1,32 @@ +# ===----------------------------------------------------------------------=== +# The multi-rocket device: Rocket cores without Balls or hardware threads, and +# the runtime that schedules jobs onto them. +# +# bbdev stack --build "--design multi-rocket" +# +# builds every core stack of the device first, then configures this with +# DESIGN_CMAKE, generated from design.pb, and the RISC-V Linux toolchain. +# Products, exported in device.cmake for the programs that run on the device: +# runtime, runtime-baremetal, runtime-linux job scheduling over the cores +# engine instruction streams (no threads here) +# bindings how each core runs jobs +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(multi-rocket-device LANGUAGES C CXX) +include("${DESIGN_CMAKE}") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# How each core runs jobs. Nothing references these bindings, so they are +# linked as objects rather than taken from an archive; as link options they +# come before the libraries they call. +add_library(bindings INTERFACE) +foreach(CORE IN LISTS DESIGN_CORES) + target_link_options(bindings INTERFACE "${DESIGN_CORE_BUILD_${CORE}}/execute.o") +endforeach() +# The device has no banks, so no DMA helpers. +set(RUNTIME_PARAMS) + +add_subdirectory("${DESIGN_REPO}/stack/engine" engine) +add_subdirectory("${DESIGN_REPO}/stack/runtime" runtime) +export(TARGETS bindings runtime runtime-baremetal runtime-linux engine + runtime-shared runtime-linux-shared engine-shared FILE device.cmake) diff --git a/examples/designs/multi-rocket/sims/device/workloads/CMakeLists.txt b/examples/designs/multi-rocket/sims/device/workloads/CMakeLists.txt new file mode 100644 index 000000000..65e317533 --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/workloads/CMakeLists.txt @@ -0,0 +1,78 @@ +include("${DESIGN_DEVICE_BUILD}/device.cmake") + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/base" + "${CMAKE_BINARY_DIR}/src/CTest/base" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tlb" + "${CMAKE_BINARY_DIR}/src/CTest/tlb" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tutorial" + "${CMAKE_BINARY_DIR}/src/CTest/tutorial" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/embench" + "${CMAKE_BINARY_DIR}/src/CTest/embench" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/coremark" + "${CMAKE_BINARY_DIR}/src/CTest/coremark" +) +foreach(BUCKYBALL_COMPONENT_WORKLOAD_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(COMPONENT_WORKLOADS + "${DESIGN_TARGET_WORKLOADS_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" + ) + if( + COMPONENT_WORKLOADS + STREQUAL + "${BUCKYBALL_REPO_ROOT}/examples/designs/${BUCKYBALL_WORKLOAD_DESIGN}/workloads" + ) + continue() + endif() + set(BUCKYBALL_COMPONENT_WORKLOAD_PROFILE + "${DESIGN_TARGET_ID_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" + ) + set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}") + set(BUCKYBALL_CTEST_GROUP_TARGET component-ctests-build) + if(IS_DIRECTORY "${COMPONENT_WORKLOADS}/ctests") + add_subdirectory( + "${COMPONENT_WORKLOADS}/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/components/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" + ) + endif() +endforeach() +add_dependencies( + ctest-workloads-build + base-ctests-build + tlb-ctests-build + tutorial-ctests-build + component-ctests-build +) +add_dependencies(sync-native-bin embench-all coremark-all) + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/MLIRTest/base" + "${CMAKE_BINARY_DIR}/src/MLIRTest/mlir_tests/base" +) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_MLIR_ACTIVE_TARGET "${BUCKYBALL_TARGET}") + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_TARGET_BALLS_${BUCKYBALL_TARGET}) + set(BALL_MLIR_TESTS + "${BUCKYBALL_REPO_ROOT}/examples/components/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" + ) + if(IS_DIRECTORY "${BALL_MLIR_TESTS}") + add_subdirectory( + "${BALL_MLIR_TESTS}" + "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}" + ) + endif() + endforeach() +endforeach() +add_dependencies(MLIRTest-all base-mlirtest-build balls-mlir-tests-build) + +add_subdirectory( + "${BUCKYBALL_REPO_ROOT}/examples/designs/multi-rocket/workloads" + design +) diff --git a/examples/designs/multi-rocket/sims/device/workloads/kernel/CMakeLists.txt b/examples/designs/multi-rocket/sims/device/workloads/kernel/CMakeLists.txt new file mode 100644 index 000000000..8bb5ab79d --- /dev/null +++ b/examples/designs/multi-rocket/sims/device/workloads/kernel/CMakeLists.txt @@ -0,0 +1,4 @@ +# Linux programs installed into the multi-rocket rootfs, each with the hart it runs on. +set(BUCKYBALL_KERNEL_PROGRAMS + 0 multi-rocket-core0-ctest-cpu_arithmetic-linux + PARENT_SCOPE) diff --git a/examples/designs/multi-rocket/sims/ip/uvm/registry.toml b/examples/designs/multi-rocket/sims/ip/uvm/registry.toml new file mode 100644 index 000000000..8e8131449 --- /dev/null +++ b/examples/designs/multi-rocket/sims/ip/uvm/registry.toml @@ -0,0 +1 @@ +ips = [] diff --git a/examples/designs/multi-rocket/workloads/CMakeLists.txt b/examples/designs/multi-rocket/workloads/CMakeLists.txt new file mode 100644 index 000000000..af0c9633a --- /dev/null +++ b/examples/designs/multi-rocket/workloads/CMakeLists.txt @@ -0,0 +1,5 @@ +add_custom_target(design-ctests-build) +add_subdirectory(device) + +add_custom_target(design-workloads-build ALL + DEPENDS design-ctests-build mlirtest-workloads-build soctests-build) diff --git a/examples/designs/multi-rocket/workloads/configs/tests.toml b/examples/designs/multi-rocket/workloads/configs/tests.toml new file mode 100644 index 000000000..c1c7ceb33 --- /dev/null +++ b/examples/designs/multi-rocket/workloads/configs/tests.toml @@ -0,0 +1,2 @@ +controlHart = 0 +reportHart = 0 diff --git a/examples/designs/multi-rocket/workloads/device/CMakeLists.txt b/examples/designs/multi-rocket/workloads/device/CMakeLists.txt new file mode 100644 index 000000000..679d879e4 --- /dev/null +++ b/examples/designs/multi-rocket/workloads/device/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(soc) +add_subdirectory(core) diff --git a/examples/designs/multi-rocket/workloads/device/core/CMakeLists.txt b/examples/designs/multi-rocket/workloads/device/core/CMakeLists.txt new file mode 100644 index 000000000..e90e2fc5a --- /dev/null +++ b/examples/designs/multi-rocket/workloads/device/core/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(ctests) diff --git a/examples/designs/multi-rocket/workloads/device/core/ctests/CMakeLists.txt b/examples/designs/multi-rocket/workloads/device/core/ctests/CMakeLists.txt new file mode 100644 index 000000000..a714819f0 --- /dev/null +++ b/examples/designs/multi-rocket/workloads/device/core/ctests/CMakeLists.txt @@ -0,0 +1,21 @@ +set(CPU_ARITHMETIC + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CONTROL_TARGET}-ctest-cpu_arithmetic" +) +set(CPU_ELF ${CMAKE_CURRENT_BINARY_DIR}/${CPU_ARITHMETIC}-baremetal) +add_custom_command( + OUTPUT ${CPU_ELF} + COMMAND + ${ELF_CC} ${BUCKYBALL_CTEST_C_FLAGS} -I${BUCKYBALL_ISA_DIR}/control + ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c + ${CMAKE_CURRENT_SOURCE_DIR}/cpu_arithmetic.c -o ${CPU_ELF} + DEPENDS + cpu_arithmetic.c + ${BAREMETAL_DIR}/crt0.S + ${BAREMETAL_DIR}/syscalls.c + ${BBSIM_LD} + ${BUCKYBALL_ISA_DIR}/control/params.h +) +add_custom_target(${CPU_ARITHMETIC}-baremetal-build DEPENDS ${CPU_ELF}) +add_dependencies(design-ctests-build ${CPU_ARITHMETIC}-baremetal-build) +add_executable(${CPU_ARITHMETIC}-linux cpu_arithmetic.c) +add_dependencies(design-ctests-build ${CPU_ARITHMETIC}-linux) diff --git a/examples/chips/multi-rocket/workloads/cpu_arithmetic.c b/examples/designs/multi-rocket/workloads/device/core/ctests/cpu_arithmetic.c similarity index 100% rename from examples/chips/multi-rocket/workloads/cpu_arithmetic.c rename to examples/designs/multi-rocket/workloads/device/core/ctests/cpu_arithmetic.c diff --git a/examples/designs/multi-rocket/workloads/device/soc/CMakeLists.txt b/examples/designs/multi-rocket/workloads/device/soc/CMakeLists.txt new file mode 100644 index 000000000..87ac0f726 --- /dev/null +++ b/examples/designs/multi-rocket/workloads/device/soc/CMakeLists.txt @@ -0,0 +1,12 @@ +set(COHERENT_ELF ${CMAKE_CURRENT_BINARY_DIR}/${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CONTROL_TARGET}-ctest-design-coherent_exchange-multicore-baremetal) +add_custom_command(OUTPUT ${COHERENT_ELF} + COMMAND ${ELF_CC} -O2 -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${BBSIM_LD} + -I${BUCKYBALL_ISA_DIR}/design + ${BAREMETAL_DIR}/start.S ${CMAKE_CURRENT_SOURCE_DIR}/coherent_exchange.c + -lgcc -o ${COHERENT_ELF} + DEPENDS coherent_exchange.c ${BAREMETAL_DIR}/start.S ${BBSIM_LD} + ${BUCKYBALL_ISA_DIR}/design/instances.h) +add_custom_target(coherent-exchange-build DEPENDS ${COHERENT_ELF}) +add_dependencies(design-ctests-build coherent-exchange-build) +add_dependencies(soctests-build coherent-exchange-build) diff --git a/examples/designs/multi-rocket/workloads/device/soc/coherent_exchange.c b/examples/designs/multi-rocket/workloads/device/soc/coherent_exchange.c new file mode 100644 index 000000000..3a3813344 --- /dev/null +++ b/examples/designs/multi-rocket/workloads/device/soc/coherent_exchange.c @@ -0,0 +1,30 @@ +#include +#include + +static uint64_t payload[BB_CORE_NUM][8] __attribute__((aligned(64))); +static unsigned arrived[2] __attribute__((aligned(64))); +static unsigned consumed[2] __attribute__((aligned(64))); +static unsigned failed; + +int main(void) { + uintptr_t hart; + asm volatile("csrr %0, mhartid" : "=r"(hart)); + unsigned slot = bb_hart_slot(hart); + for (unsigned round = 0; round < 2; ++round) { + uint64_t value = 0x5a000000ULL + round * 0x10000 + slot * 9; + payload[slot][0] = value; + payload[slot][7] = ~value; + __atomic_fetch_add(&arrived[round], 1, __ATOMIC_ACQ_REL); + while (__atomic_load_n(&arrived[round], __ATOMIC_ACQUIRE) != BB_CORE_NUM) { + } + for (unsigned peer = 0; peer < BB_CORE_NUM; ++peer) { + uint64_t expected = 0x5a000000ULL + round * 0x10000 + peer * 9; + if (payload[peer][0] != expected || payload[peer][7] != ~expected) + __atomic_fetch_or(&failed, 1, __ATOMIC_RELAXED); + } + __atomic_fetch_add(&consumed[round], 1, __ATOMIC_ACQ_REL); + while (__atomic_load_n(&consumed[round], __ATOMIC_ACQUIRE) != BB_CORE_NUM) { + } + } + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE); +} diff --git a/examples/designs/pebble/arch/build.sc b/examples/designs/pebble/arch/build.sc new file mode 100644 index 000000000..9c69932e4 --- /dev/null +++ b/examples/designs/pebble/arch/build.sc @@ -0,0 +1,18 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.arch.src.main.scala.framework.device.cbus.{build => cbus_build} +import $file.^.^.^.{build => examples} + +val designRoot: os.Path = millSourcePath / os.up + +object pebble extends SbtModule { + override def generatedSources = T(cbus_build.registry(sources(), T.dest)) + override def millSourcePath = designRoot + override def forkWorkingDir = T(designRoot) + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(examples.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(designRoot / "arch") +} diff --git a/examples/designs/pebble/arch/device/DeviceTop.scala b/examples/designs/pebble/arch/device/DeviceTop.scala new file mode 100644 index 000000000..25400c6ae --- /dev/null +++ b/examples/designs/pebble/arch/device/DeviceTop.scala @@ -0,0 +1,73 @@ +package examples.designs.pebble + +import framework.lib.arbiter.{RRArbiter} + +import _root_.sims.configs.SimParam +import chisel3._ +import chisel3.util._ +import framework.device.peripherals.bootrom.BootRomLines +import framework.device.peripherals.Controller +import framework.ip.bus.axi4.Decoder +import framework.ip.bus.chi.snf.LineResponse +import framework.lib.queue.{Queue => ResponseQueue} +import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} +import framework.device.configs.DeviceParam +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import buckyball.config.Design +import _root_.configs.GlobalConfig +import framework.core.clink.CLinkIO +import framework.device.dlink.BaseDevice + +@instantiable +class DeviceTop(b: GlobalConfig, design: Design) extends BaseDevice(b.device.ddr.axi) { + val devicep = b.device + + val Seq(corep) = devicep.cores + require(corep.mtimeAddress == devicep.clint.base + 0xbff8, "Core mtime address must match CLINT") + + val bus = Instantiate(new CBus(b.device, CBus.generators(design, b.device, b.sim, CoreComponents.factories))) + dlink <> bus.dlink + + val peripherals = + Instantiate(new Controller(devicep, devicep.ddr.axi, corep.memory.memoryAccess.memory)) + + val peripheralBus = Instantiate(new Decoder( + Seq( + (devicep.clint.base, devicep.clint.bytes), + (devicep.plic.base, devicep.plic.bytes), + (devicep.scu.baseAddress, devicep.scu.totalSizeBytes) + ), + devicep.ddr.axi + )) + + peripheralBus.io.in <> bus.uncached + peripherals.io.axi <> peripheralBus.io.matched + bus.external <> peripheralBus.io.other + peripherals.io.hardwareInterrupt.sources := 0.U + bus.interrupts := peripherals.io.hardwareInterrupt.harts + + val romParams = devicep.bootrom + val rom = Instantiate(new BootRomLines(devicep.bootrom, corep.memory.home.memory.chi)) + + val backing = bus.backing(0) + val ddr = bus.memory(0) + val inRom = backing.req.bits.addr >= romParams.base.U && + backing.req.bits.addr < (romParams.base + romParams.bytes).U + rom.io.request.valid := backing.req.valid && inRom + rom.io.request.bits := backing.req.bits + ddr.req.valid := backing.req.valid && !inRom + ddr.req.bits := backing.req.bits + backing.req.ready := Mux(inRom, rom.io.request.ready, ddr.req.ready) + + val responses = Instantiate(new RRArbiter(new LineResponse(corep.memory.home.memory.chi), 2)) + responses.io.in(0) <> rom.io.response + responses.io.in(1) <> ddr.resp + + // Buffer the selected answer so downstream backpressure cannot change its source. + val response = Instantiate(new ResponseQueue(new LineResponse(corep.memory.home.memory.chi), 1, false, false)) + + response.io.enq <> responses.io.out + backing.resp <> response.io.deq + +} diff --git a/examples/designs/pebble/arch/device/core/PebbleCore.scala b/examples/designs/pebble/arch/device/core/PebbleCore.scala new file mode 100644 index 000000000..329a3841d --- /dev/null +++ b/examples/designs/pebble/arch/device/core/PebbleCore.scala @@ -0,0 +1,46 @@ +package examples.designs.pebble + +import sims.configs.SimParam +import framework.device.cbus.CoreFactory +import buckyball.config.{CorePlacement, Design} + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.core.clink.{CLinkIO, HasCLink} + +/** One RV64GC hart and its local NPU, explicitly wired behind CLink. */ +@instantiable +class PebbleCore(corep: CoreParam, simp: SimParam) extends Module with HasCLink { + val diffTest = simp.diffTest + require(corep.threads.isEmpty && corep.acc.nonEmpty, "PebbleCore connects a local ACC without TBus") + require(corep.clink.harts == 1 && corep.clink.backing) + + @public + val clink = IO(new CLinkIO(corep.clink)) + + val loom: Instance[Loom] = Instantiate(new Loom(corep)) + val acc: Instance[Acc] = Instantiate(new Acc(corep, corep.acc.get, simp)) + + loom.io.rocc <> acc.io.rocc + loom.io.memory <> acc.io.memory + loom.io.interrupts := clink.interrupts(0) + clink.memory.uncached <> loom.io.uncached + clink.memory.backing.get <> loom.io.backing + clink.memory.ddr <> acc.io.ddr +} + +object Pebble extends CoreFactory { + val component = "pebble" + def parameters(design: Design, source: CorePlacement): CoreParam = + CoreParam.fromProto(design, source, externalHome = false) + + def instantiate( + source: CorePlacement, + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int] + ): CLinkIO = + Instantiate(new PebbleCore(corep, simp)).clink + +} diff --git a/examples/designs/pebble/arch/device/core/acc.scala b/examples/designs/pebble/arch/device/core/acc.scala new file mode 100644 index 000000000..28f64b8ac --- /dev/null +++ b/examples/designs/pebble/arch/device/core/acc.scala @@ -0,0 +1,113 @@ +package examples.designs.pebble + +import framework.acc.balldomain.bbus.generated.BallComponents +import sims.configs.SimParam +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.core.memory.MemoryPort +import framework.acc.frontend.RoCCIO +import framework.acc.configs.AccParam +import framework.acc.frontend.Frontend +import framework.acc.memdomain.MemDomain +import framework.acc.debug.Debug +import framework.ip.bus.axi4 +import framework.acc.balldomain.BallDomain +import framework.acc.balldomain.bbus.BBus + +/** Pebble accelerator domains, concrete Balls and their local debug wiring. */ +@instantiable +class Acc(corep: CoreParam, accp: AccParam, simp: SimParam) extends Module { + val diffTest = simp.diffTest + require(!accp.memDomain.sharedEnable && !accp.gpDomain.enable) + + @public + val io = IO(new Bundle { + val rocc = new RoCCIO(corep.integer.xLen, corep.privileged.permission) + val memory = new MemoryPort(corep.memory.dcache.tracking, corep.memory.memoryAccess.walker) + val ddr = new axi4.Port(accp.transport.axi) + }) + + val frontend: Instance[Frontend] = Instantiate(new Frontend( + accp.frontend, + accp.memDomain, + accp.gpDomain, + accp.ballDomain, + corep.privileged.permission, + simp + )) + + val memDomain: Instance[MemDomain] = Instantiate(new MemDomain( + accp.frontend, + accp.memDomain, + accp.gpDomain, + accp.transport, + corep.privileged.permission, + corep.memory.dcache.tracking, + corep.memory.memoryAccess.walker, + simp + )) + + val ballDomain: Instance[BallDomain] = Instantiate(new BallDomain( + accp.frontend, + accp.memDomain, + accp.gpDomain, + accp.ballDomain, + corep.privileged.permission, + BallComponents.factories + )) + + frontend.io.rocc <> io.rocc + io.memory <> memDomain.io.memory + + // Local NPU domains are visible here; + // BBUS owns concrete Ball instances. + val debug: Instance[Debug] = Instantiate(new Debug( + accp.frontend, + accp.memDomain, + accp.gpDomain, + accp.ballDomain, + simp, + traceId = corep.hartId, + sharedOwnerId = corep.hartId + )) + + debug.io.ant.valid := false.B + debug.io.ant.bits := 0.U.asTypeOf(debug.io.ant.bits) + debug.io.acc.frontend <> frontend.io.debug + debug.io.acc.trace.memory := memDomain.io.traceMemory + debug.io.acc.trace.ballStarted := ballDomain.io.traceStarted + debug.io.acc.trace.ballCompleted := ballDomain.io.traceCompleted + debug.io.acc.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, source) => sink := source } + + ballDomain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> ballDomain.io.complete + memDomain.io.ball <> ballDomain.io.memory + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.banks <> ballDomain.io.banks + memDomain.io.ballLayoutRequest <> ballDomain.io.layoutRequest + ballDomain.io.layoutResponse <> memDomain.io.ballLayoutResponse + memDomain.io.mapping <> frontend.io.mappingUpdate + frontend.io.barrier.release := frontend.io.barrier.arrive + io.ddr <> memDomain.io.ddr + + // Pebble has no shared-memory connection; + // its complete data path is local. + for (port <- memDomain.io.shared.bank.requests) { + port.read.req.ready := false.B + port.read.resp.valid := false.B + port.read.resp.bits := 0.U.asTypeOf(port.read.resp.bits) + port.write.req.ready := false.B + port.write.resp.valid := false.B + port.write.resp.bits := 0.U.asTypeOf(port.write.resp.bits) + } + memDomain.io.shared.bank.config.ready := false.B + memDomain.io.shared.bank.mapping.ready := false.B + memDomain.io.shared.bank.local.request.valid := false.B + memDomain.io.shared.bank.local.request.bits := 0.U.asTypeOf(memDomain.io.shared.bank.local.request.bits) + memDomain.io.shared.bank.local.response.ready := false.B + memDomain.io.shared.move.command.ready := false.B + memDomain.io.shared.move.completion.valid := false.B + memDomain.io.shared.move.completion.bits := false.B +} diff --git a/examples/designs/pebble/arch/device/core/loom.scala b/examples/designs/pebble/arch/device/core/loom.scala new file mode 100644 index 000000000..277912502 --- /dev/null +++ b/examples/designs/pebble/arch/device/core/loom.scala @@ -0,0 +1,51 @@ +package examples.designs.pebble + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.core.execution.Execution +import framework.core.memory.{MemoryPort, MemorySystem} +import framework.acc.frontend.RoCCIO +import framework.core.clink.CoreInterrupts +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} + +/** Outer Loom execution core and private memory hierarchy; directly controls the local ACC. */ +@instantiable +class Loom(corep: CoreParam) extends Module { + + @public + val io = IO(new Bundle { + val interrupts = Input(new CoreInterrupts) + val rocc = Flipped(new RoCCIO(corep.integer.xLen, corep.privileged.permission)) + val memory = Flipped(new MemoryPort(corep.memory.dcache.tracking, corep.memory.memoryAccess.walker)) + val uncached = new axi4.Port(corep.memory.uncached.axi) + + val backing = new Bundle { + val request = Decoupled(new LineRequest(corep.memory.home.memory.chi)) + val response = Flipped(Decoupled(new LineResponse(corep.memory.home.memory.chi))) + } + + }) + + val execution: Instance[Execution] = Instantiate(new Execution( + corep.execution(directAccelerator = true), + localRocc = true + )) + + val memory: Instance[MemorySystem] = Instantiate(new MemorySystem(corep.memory, externalHome = false)) + + execution.io.interrupts := io.interrupts + execution.io.rocc.get <> io.rocc + execution.io.workDrained := !io.rocc.busy + memory.io.fetch <> execution.io.fetch + memory.io.memory <> execution.io.data + memory.io.timeRequest <> execution.io.timeRequest + execution.io.timeResponse <> memory.io.timeResponse + memory.io.commandAllowed := !io.rocc.busy + memory.io.services <> io.memory + io.uncached <> memory.io.uncached + io.backing <> memory.io.backing.get + +} diff --git a/examples/designs/pebble/configs/design.toml b/examples/designs/pebble/configs/design.toml new file mode 100644 index 000000000..1d45ec7e4 --- /dev/null +++ b/examples/designs/pebble/configs/design.toml @@ -0,0 +1,5 @@ +[device] +include = "device/default.toml" + +[sims] +include = "sims/default.toml" diff --git a/examples/designs/pebble/configs/device/core/acc/ablation-bat0-blt0.toml b/examples/designs/pebble/configs/device/core/acc/ablation-bat0-blt0.toml new file mode 100644 index 000000000..269b93083 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/ablation-bat0-blt0.toml @@ -0,0 +1,8 @@ +source = "../../../.." +# Pebble accelerator profile + +balldomain = "balldomain/default.toml" +memdomain = "memdomain/blt-off.toml" +frontend = "frontend/bat-off-rob24.toml" +gpdomain = "default" +transport = "default" diff --git a/examples/designs/pebble/configs/device/core/acc/ablation-bat0-blt1.toml b/examples/designs/pebble/configs/device/core/acc/ablation-bat0-blt1.toml new file mode 100644 index 000000000..6f14a5454 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/ablation-bat0-blt1.toml @@ -0,0 +1,8 @@ +source = "../../../.." +# Pebble accelerator profile + +balldomain = "balldomain/default.toml" +memdomain = "default" +frontend = "frontend/bat-off-rob24.toml" +gpdomain = "default" +transport = "default" diff --git a/examples/designs/pebble/configs/device/core/acc/ablation-bat1-blt0.toml b/examples/designs/pebble/configs/device/core/acc/ablation-bat1-blt0.toml new file mode 100644 index 000000000..c746a0d40 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/ablation-bat1-blt0.toml @@ -0,0 +1,8 @@ +source = "../../../.." +# Pebble accelerator profile + +balldomain = "balldomain/default.toml" +memdomain = "memdomain/blt-off.toml" +frontend = "frontend/rob24.toml" +gpdomain = "default" +transport = "default" diff --git a/examples/cores/pebble/configs/balldomains/balls/smatmul/8x16.toml b/examples/designs/pebble/configs/device/core/acc/balldomain/balls/smatmul/8x16.toml similarity index 90% rename from examples/cores/pebble/configs/balldomains/balls/smatmul/8x16.toml rename to examples/designs/pebble/configs/device/core/acc/balldomain/balls/smatmul/8x16.toml index 93333ca8e..fa9a5342c 100644 --- a/examples/cores/pebble/configs/balldomains/balls/smatmul/8x16.toml +++ b/examples/designs/pebble/configs/device/core/acc/balldomain/balls/smatmul/8x16.toml @@ -1,5 +1,4 @@ # Pebble SMatMul Ball Configuration -[ball] tileRows = 8 tileCols = 16 diff --git a/examples/designs/pebble/configs/device/core/acc/balldomain/default.toml b/examples/designs/pebble/configs/device/core/acc/balldomain/default.toml new file mode 100644 index 000000000..f68c803b7 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/balldomain/default.toml @@ -0,0 +1,102 @@ +[[balls]] +id = 0 +name = "TransposeBall" +component = "transpose" +param = "default" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "TRANSPOSE", funct7 = 49 }, +] + +[[balls]] +id = 1 +name = "SMatMulBall" +component = "smatmul" +param = "balls/smatmul/8x16.toml" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "SMATMUL_OS", funct7 = 65 }, + { mnemonic = "SMATMUL_BIAS", funct7 = 17 }, +] + +[[balls]] +id = 2 +name = "Im2colBall" +component = "im2col" +param = "default" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "IM2COL", funct7 = 48 }, +] + +[[balls]] +id = 3 +name = "ToInt8Ball" +component = "toint8" +param = "default" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "QUANT_F32_TO_I8", funct7 = 51 }, + { mnemonic = "QUANT_I32_TO_I8", funct7 = 67 }, +] + +[[balls]] +id = 4 +name = "Int2FpBall" +component = "int2fp" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "INT32_TO_FP32", funct7 = 68 }, +] + +[[balls]] +id = 5 +name = "LutBall" +component = "lut" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "LUT", funct7 = 66 }, +] + +[[balls]] +id = 6 +name = "MaxPoolBall" +component = "maxpool" +param = "default" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "MAXPOOL", funct7 = 50 }, +] + +[[balls]] +id = 7 +name = "Int8AddBall" +component = "int8add" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "INT8ADD", funct7 = 69 }, + { mnemonic = "INT8ADD_RELU", funct7 = 70 }, +] + +[[balls]] +id = 8 +name = "Int8MulBall" +component = "int8mul" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "INT8MUL", funct7 = 71 }, +] + +[[balls]] +id = 9 +name = "MatAddBall" +component = "matadd" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "MATADD", funct7 = 72 }, +] diff --git a/examples/designs/pebble/configs/device/core/acc/default.toml b/examples/designs/pebble/configs/device/core/acc/default.toml new file mode 100644 index 000000000..24ccccdcc --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/default.toml @@ -0,0 +1,8 @@ +source = "../../../.." +# Pebble accelerator profile + +balldomain = "balldomain/default.toml" +memdomain = "default" +frontend = "default" +gpdomain = "default" +transport = "default" diff --git a/examples/designs/pebble/configs/device/core/acc/frontend/bat-off-rob24.toml b/examples/designs/pebble/configs/device/core/acc/frontend/bat-off-rob24.toml new file mode 100644 index 000000000..f196d4b84 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/frontend/bat-off-rob24.toml @@ -0,0 +1,9 @@ +bankAliasEnable = false +robEntries = 24 +rsOutOfOrderResponse = true +bankIdLen = 10 +vbankIdUpperBound = 23 +sharedBankIdBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/examples/designs/pebble/configs/device/core/acc/frontend/rob24.toml b/examples/designs/pebble/configs/device/core/acc/frontend/rob24.toml new file mode 100644 index 000000000..3964cea2e --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/frontend/rob24.toml @@ -0,0 +1,9 @@ +bankAliasEnable = true +robEntries = 24 +rsOutOfOrderResponse = true +bankIdLen = 10 +vbankIdUpperBound = 23 +sharedBankIdBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/examples/designs/pebble/configs/device/core/acc/memdomain/blt-off.toml b/examples/designs/pebble/configs/device/core/acc/memdomain/blt-off.toml new file mode 100644 index 000000000..82d606195 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/memdomain/blt-off.toml @@ -0,0 +1,22 @@ +# Pebble thread memory domain configuration + +[bank] +layoutEnable = false +num = 24 +width = 128 +entries = 512 +maskLen = 16 +requestPorts = 13 + +[dma] +busWidth = 128 + +[mmio] +enable = false +bankNum = 5 +bankEntries = 1024 +bankWidth = 8 +readWidth = 8 + +[mem] +addrLen = 39 diff --git a/examples/designs/pebble/configs/device/core/acc/rob24.toml b/examples/designs/pebble/configs/device/core/acc/rob24.toml new file mode 100644 index 000000000..183297aff --- /dev/null +++ b/examples/designs/pebble/configs/device/core/acc/rob24.toml @@ -0,0 +1,8 @@ +source = "../../../.." +# Pebble accelerator profile + +balldomain = "balldomain/default.toml" +memdomain = "default" +frontend = "frontend/rob24.toml" +gpdomain = "default" +transport = "default" diff --git a/examples/designs/pebble/configs/device/core/c2c/pebble.toml b/examples/designs/pebble/configs/device/core/c2c/pebble.toml new file mode 100644 index 000000000..00bd5f8b4 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/c2c/pebble.toml @@ -0,0 +1,3 @@ +enabled = false +peerCores = 9 +axi = "default" diff --git a/examples/designs/pebble/configs/device/core/default.toml b/examples/designs/pebble/configs/device/core/default.toml new file mode 100644 index 000000000..787135494 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/default.toml @@ -0,0 +1,35 @@ +component = "pebble" +accelerator = "acc/default.toml" +threads = [] +hartId = 0 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +memory = "memory/pebble.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/pebble.toml" +c2c = "c2c/pebble.toml" +controller = "default" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[sharedMem] +enable = false +bankWidth = 128 +bankEntries = 1024 +virtualBankCount = 24 +entries = 16384 +inputChannels = 1 +defaultGroupCount = 1 +memCycles = 10 + +[compiler] +name = "pebble" +component = "pebble" diff --git a/examples/designs/pebble/configs/device/core/interconnect/pebble.toml b/examples/designs/pebble/configs/device/core/interconnect/pebble.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/interconnect/pebble.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/pebble/configs/device/core/memory/access/pebble.toml b/examples/designs/pebble/configs/device/core/memory/access/pebble.toml new file mode 100644 index 000000000..bb87f76a4 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/memory/access/pebble.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +accessTagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x20000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/pebble/configs/device/core/memory/dcache/pebble.toml b/examples/designs/pebble/configs/device/core/memory/dcache/pebble.toml new file mode 100644 index 000000000..97b8e2bba --- /dev/null +++ b/examples/designs/pebble/configs/device/core/memory/dcache/pebble.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 2 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/pebble/configs/device/core/memory/home/pebble.toml b/examples/designs/pebble/configs/device/core/memory/home/pebble.toml new file mode 100644 index 000000000..3e0779cbc --- /dev/null +++ b/examples/designs/pebble/configs/device/core/memory/home/pebble.toml @@ -0,0 +1,13 @@ +agents = 2 +mshrEntries = 8 +homeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/pebble/configs/device/core/memory/icache/pebble.toml b/examples/designs/pebble/configs/device/core/memory/icache/pebble.toml new file mode 100644 index 000000000..75e065643 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/memory/icache/pebble.toml @@ -0,0 +1,7 @@ +nodeId = 1 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/pebble/configs/device/core/memory/pebble.toml b/examples/designs/pebble/configs/device/core/memory/pebble.toml new file mode 100644 index 000000000..fb90c6584 --- /dev/null +++ b/examples/designs/pebble/configs/device/core/memory/pebble.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/pebble.toml" +icache = "icache/pebble.toml" +dcache = "dcache/pebble.toml" +home = "home/pebble.toml" diff --git a/examples/designs/pebble/configs/device/ddr/pebble.toml b/examples/designs/pebble/configs/device/ddr/pebble.toml new file mode 100644 index 000000000..31815e9e1 --- /dev/null +++ b/examples/designs/pebble/configs/device/ddr/pebble.toml @@ -0,0 +1,13 @@ +clients = 1 +slotsPerClient = 16 +dataBits = 128 +idBits = 4 +idBase = 0 +base = 0x80000000 +bytes = 0x20000000 +model = 0 + +[line] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 diff --git a/examples/designs/pebble/configs/device/default.toml b/examples/designs/pebble/configs/device/default.toml new file mode 100644 index 000000000..b83f276f2 --- /dev/null +++ b/examples/designs/pebble/configs/device/default.toml @@ -0,0 +1,8 @@ +ddr = "ddr/pebble.toml" +bootrom = "default" +clint = "default" +plic = "default" +scu = "default" + +[[cores]] +include = "core/default.toml" diff --git a/examples/designs/pebble/configs/sims/default.toml b/examples/designs/pebble/configs/sims/default.toml new file mode 100644 index 000000000..70c1347aa --- /dev/null +++ b/examples/designs/pebble/configs/sims/default.toml @@ -0,0 +1,15 @@ +diffTest = false + +[memory] +lineBytes = 64 +clockHz = 1000000000 + +[targets.device] +config = "examples.designs.pebble.sims.device.DeviceConfig" +stack_name = "pebble" +regression = "sims/device/regression" +kernel = "sims/device/workloads/kernel" +tapeout = "sims/device/tapeout" + +[verification] +registry = "../../sims/ip/uvm/registry.toml" diff --git a/examples/designs/pebble/emu/Cargo.lock b/examples/designs/pebble/emu/Cargo.lock new file mode 100644 index 000000000..171dff7cd --- /dev/null +++ b/examples/designs/pebble/emu/Cargo.lock @@ -0,0 +1,821 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "bebop-vm", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "rvsim", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-pebble" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rvsim" +version = "0.1.0" +dependencies = [ + "rustc_apfloat", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/pebble/emu/Cargo.toml b/examples/designs/pebble/emu/Cargo.toml new file mode 100644 index 000000000..3d0a9c02a --- /dev/null +++ b/examples/designs/pebble/emu/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "bebop-design-pebble" +version = "0.1.0" +edition = "2021" + +[dependencies] +bebop-bemu = { path = "../../../../bebop/src/nodes/bemu" } diff --git a/examples/designs/pebble/emu/src/device/core/mod.rs b/examples/designs/pebble/emu/src/device/core/mod.rs new file mode 100644 index 000000000..fc633526e --- /dev/null +++ b/examples/designs/pebble/emu/src/device/core/mod.rs @@ -0,0 +1,39 @@ +use bebop_bemu::{ + ball::linkme, Accelerator, BemuProfileReport, CoreFactory, CoreModel, CoreParams, Cpu, DeviceModel, ModelOptions, + Whatever, CORES, +}; +use std::time::Duration; + +/// One RV64GC hart with its local NPU on the RoCC port (arch: PebbleCore, Loom and Acc). +pub struct PebbleCore { + cpu: Cpu, + acc: Accelerator, +} + +impl CoreModel for PebbleCore { + fn cpu(&self) -> &Cpu { + &self.cpu + } + fn cpu_mut(&mut self) -> &mut Cpu { + &mut self.cpu + } + fn step(&mut self, count: u64) -> Result<(), Whatever> { + self.cpu.step(count, &mut self.acc) + } + fn take_barrier(&mut self) -> bool { + self.acc.take_barrier() + } + fn profile_report(&self, total: Duration) -> Option { + self.acc.profile_report(total, &self.cpu) + } +} + +fn instantiate(device: &DeviceModel, params: &CoreParams, options: &ModelOptions) -> Result, String> { + let cpu = Cpu::new(device, params, options).map_err(|error| error.to_string())?; + let acc = Accelerator::attached(&cpu, options)?; + Ok(Box::new(PebbleCore { cpu, acc })) +} + +#[linkme::distributed_slice(CORES)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static PEBBLE: CoreFactory = CoreFactory { component: "pebble", instantiate }; diff --git a/examples/designs/pebble/emu/src/device/mod.rs b/examples/designs/pebble/emu/src/device/mod.rs new file mode 100644 index 000000000..aa4942065 --- /dev/null +++ b/examples/designs/pebble/emu/src/device/mod.rs @@ -0,0 +1,2 @@ +//! The pebble device: its own Core model; the cores are instantiated by component on the CBus. +pub mod core; diff --git a/examples/designs/pebble/emu/src/lib.rs b/examples/designs/pebble/emu/src/lib.rs new file mode 100644 index 000000000..545892414 --- /dev/null +++ b/examples/designs/pebble/emu/src/lib.rs @@ -0,0 +1 @@ +pub mod device; diff --git a/examples/designs/pebble/sims/device/arch/DeviceConfig.scala b/examples/designs/pebble/sims/device/arch/DeviceConfig.scala new file mode 100644 index 000000000..10af06ff8 --- /dev/null +++ b/examples/designs/pebble/sims/device/arch/DeviceConfig.scala @@ -0,0 +1,18 @@ +package examples.designs.pebble.sims.device + +import buckyball.config.Design +import chisel3.experimental.hierarchy.{Instance, Instantiate} +import examples.designs.pebble.DeviceTop +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import framework.device.dlink.BaseDevice +import _root_.configs.GlobalConfig +import _root_.sims.soc.SystemTarget +import java.nio.file.{Files, Paths} + +class DeviceConfig(design: Design) + extends SystemTarget( + GlobalConfig.fromProto(design, CBus.parameters(design, CoreComponents.factories)) + ) { + def instantiate(b: GlobalConfig): Instance[BaseDevice] = Instantiate(new DeviceTop(b, design)) +} diff --git a/examples/designs/pebble/sims/device/arch/build.sc b/examples/designs/pebble/sims/device/arch/build.sc new file mode 100644 index 000000000..d777182d1 --- /dev/null +++ b/examples/designs/pebble/sims/device/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.arch.{build => design_build} +import $file.^.^.^.^.^.^.arch.src.main.scala.sims.{build => sims_build} + +val entryRoot: os.Path = millSourcePath + +object simulation extends SbtModule { + override def millSourcePath = entryRoot + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(design_build.pebble, sims_build.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(entryRoot / "DeviceConfig.scala") + override def mainClass = Some("sims.verilator.Elaborate") +} diff --git a/examples/designs/pebble/sims/device/emu/Cargo.lock b/examples/designs/pebble/sims/device/emu/Cargo.lock new file mode 100644 index 000000000..a86e1261b --- /dev/null +++ b/examples/designs/pebble/sims/device/emu/Cargo.lock @@ -0,0 +1,2008 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "assert_cmd" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aa3a22042e45de04255c7bf3626e239f450200fd0493c1e382263544b20aea6" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bebop" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-fd-redirect", + "bebop-p2e", + "bebop-report", + "bebop-rtl-trace", + "bebop-terminal", + "bebop-trace-perfetto", + "bebop-uart", + "bebop-vcs", + "bebop-verilator", + "bebop-vm", + "camino", + "clap", + "duct", + "env_logger", + "libc", + "log", + "nix", + "serde", + "serde_json", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-pebble" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-fd-redirect" +version = "0.1.0" +dependencies = [ + "libc", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-p2e" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "bebop-rtl-trace", + "bebop-uart", + "duct", + "libc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bebop-report" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-rtl-trace" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", +] + +[[package]] +name = "bebop-sim-build" +version = "0.1.0" +dependencies = [ + "prost", + "prost-build", + "prost-types", +] + +[[package]] +name = "bebop-sim-pebble-device" +version = "0.1.0" +dependencies = [ + "assert_cmd", + "bebop", + "bebop-bemu", + "bebop-design-pebble", + "bebop-report", + "bebop-sim-build", + "bebop-verilator", + "chrono", + "clap", + "libtest-mimic", + "predicates", + "serde", + "toml", + "walkdir", +] + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-terminal" +version = "0.1.0" +dependencies = [ + "crossterm", + "ratatui", +] + +[[package]] +name = "bebop-trace-perfetto" +version = "0.1.0" +dependencies = [ + "clap", + "serde_json", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vcs" +version = "0.1.0" +dependencies = [ + "snafu", +] + +[[package]] +name = "bebop-verilator" +version = "0.1.0" +dependencies = [ + "bebop-dasm", + "bebop-fd-redirect", + "bebop-rtl-trace", + "bebop-uart", + "libc", + "snafu", +] + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" +dependencies = [ + "clap", + "font8x8", + "sdl2", + "vt100", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "camino" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbad30e4b4c14a39e3cc8aed085a12a327257c316619c93581e017bc52be591" + +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compact_str" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.13.2", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix 0.38.44", + "signal-hook 0.3.18", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "duct" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "font8x8" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875488b8711a968268c7cf5d139578713097ca4635a76044e8fe8eedf831d07e" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "indoc" +version = "2.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1" +dependencies = [ + "rustversion", +] + +[[package]] +name = "instability" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3b5acc1e2fd9375041a388da33d1eb8aed5f7a8c0dd3543e3ea2805adfbe20" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "os_pipe" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags 2.13.2", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags 2.13.2", + "smallvec", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdl2" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d42407afc6a8ab67e36f92e80b8ba34cbdc55aaeed05249efe9a2e8d0e9feef" +dependencies = [ + "bitflags 1.3.2", + "lazy_static", + "libc", + "sdl2-sys", +] + +[[package]] +name = "sdl2-sys" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff61407fc75d4b0bbc93dc7e4d6c196439965fbef8e4a4f003a36095823eac0" +dependencies = [ + "cfg-if", + "libc", + "pkg-config", + "version-compare", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "shared_child" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "607549934f6cc26b89cfecfdc46fa90f1e5d1536a68349b0c3a4f9d1c0d37959" +dependencies = [ + "libc", + "sigchld", + "windows-sys 0.61.2", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "sigchld" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f2b37f04360cd465089b87a9c3869c08220a2f3458463f0adf8badf5e77f2c" +dependencies = [ + "libc", + "os_pipe", + "signal-hook 0.4.5", +] + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b29ffa27d3d59a049bb328258de100f8012848d1b08d444e08bbc02ac468435e" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook 0.3.18", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix 1.1.5", + "windows-sys 0.61.2", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version-compare" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" + +[[package]] +name = "vt100" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de" +dependencies = [ + "itoa", + "log", + "unicode-width 0.1.14", + "vte", +] + +[[package]] +name = "vte" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/pebble/sims/device/emu/Cargo.toml b/examples/designs/pebble/sims/device/emu/Cargo.toml new file mode 100644 index 000000000..f07ef4f5f --- /dev/null +++ b/examples/designs/pebble/sims/device/emu/Cargo.toml @@ -0,0 +1,83 @@ +[workspace] + +[package] +name = "bebop-sim-pebble-device" +version = "0.1.0" +edition = "2021" +autobins = false +autotests = false + +# The model of this simulation entry; the runners below are built from it. +[lib] +name = "simulation" + +[[bin]] +name = "bebop" +path = "../../../../../../bebop/src/main.rs" + +[[bin]] +name = "bebop-bemu" +path = "../../../../../../bebop/src/bin/bebop-bemu.rs" +required-features = ["bemu"] + +[[bin]] +name = "bebop-verilator" +path = "../../../../../../bebop/src/bin/bebop-verilator.rs" +required-features = ["verilator"] + +[[bin]] +name = "bebop-p2e" +path = "../../../../../../bebop/src/bin/bebop-p2e.rs" +required-features = ["p2e"] + +[[test]] +name = "test_bemu" +path = "../../../../../../bebop/tests/test_bemu.rs" +harness = false +required-features = ["bemu"] + +[[test]] +name = "test_verilator" +path = "../../../../../../bebop/tests/test_verilator.rs" +harness = false +required-features = ["verilator"] + +[[test]] +name = "test_p2e" +path = "../../../../../../bebop/tests/test_p2e.rs" +harness = false +required-features = ["p2e"] + +[[bin]] +name = "bebop-vcs" +path = "../../../../../../bebop/src/bin/bebop-vcs.rs" + +[[test]] +name = "test_vcs" +path = "../../../../../../bebop/tests/test_vcs.rs" +harness = false + +[features] +bemu = ["bebop/bemu"] +verilator = ["bebop/verilator", "dep:bebop-verilator"] +p2e = ["bebop/p2e"] + +[dependencies] +bebop = { path = "../../../../../../bebop" } +bebop-verilator = { path = "../../../../../../bebop/src/nodes/verilator", optional = true } +bebop-bemu = { path = "../../../../../../bebop/src/nodes/bemu" } +bebop-design-pebble = { path = "../../../emu" } +clap = { version = "4", features = ["derive"] } + +[build-dependencies] +bebop-sim-build = { path = "../../../../../../bebop/src/nodes/lib/sim-build" } + +[dev-dependencies] +bebop-report = { path = "../../../../../../bebop/src/nodes/lib/report" } +libtest-mimic = "0.8" +assert_cmd = "2" +walkdir = "2" +predicates = "3" +serde = { version = "1", features = ["derive"] } +toml = "0.8" +chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/examples/designs/pebble/sims/device/emu/build.rs b/examples/designs/pebble/sims/device/emu/build.rs new file mode 100644 index 000000000..d634d5dba --- /dev/null +++ b/examples/designs/pebble/sims/device/emu/build.rs @@ -0,0 +1,7 @@ +use std::{env, path::PathBuf}; + +fn main() { + let entry = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + bebop_sim_build::balls(&entry.join("../../../output/generated/design.pb")); + bebop_sim_build::runner_links(); +} diff --git a/examples/designs/pebble/sims/device/emu/src/lib.rs b/examples/designs/pebble/sims/device/emu/src/lib.rs new file mode 100644 index 000000000..5d34a2311 --- /dev/null +++ b/examples/designs/pebble/sims/device/emu/src/lib.rs @@ -0,0 +1,11 @@ +//! The pebble device simulation. +//! Its Core models register in `bebop_bemu::CORES`; the Balls its design.pb +//! names are compiled here and register in `bebop_bemu::BALLS`. +extern crate bebop_design_pebble; + +include!(concat!(env!("OUT_DIR"), "/balls.rs")); + +pub const SIMULATION: bebop_bemu::Simulation = bebop_bemu::Simulation { + design: include_bytes!("../../../../output/generated/design.pb"), + instantiate: bebop_bemu::device::cbus::instantiate, +}; diff --git a/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml new file mode 100644 index 000000000..c49bf1ff5 --- /dev/null +++ b/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml @@ -0,0 +1,112 @@ +[workloads] +tests = [ + "pebble-pebble-mlirtest-core_mega_matmul_m17_k65_n17-baremetal", + "pebble-pebble-mlirtest-core_mega_matmul-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d_k5_tail-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d-baremetal", + "pebble-pebble-mlirtest-core_lookup_channel_i8-baremetal", + "pebble-pebble-mlirtest-core_generic_matmul_i8-baremetal", + "pebble-pebble-mlirtest-core_linear_int8-baremetal", + "pebble-pebble-mlirtest-core_linear_chain_i8-baremetal", + "pebble-pebble-ctest-bank_capacity_rename_test-baremetal", + "pebble-pebble-ctest-bank_transfer_recycle_test-baremetal", + "pebble-pebble-ctest-mset_transfer_test-baremetal", + "pebble-pebble-ctest-bank_recycle_test-baremetal", + "pebble-pebble-ctest-mvin_2d_test-baremetal", + "pebble-pebble-ctest-mvin_2d_pixel_stride_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_acc_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_stride_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_unaligned_test-baremetal", + "pebble-pebble-ctest-mvin_unaligned_test-baremetal", + "pebble-pebble-mlirtest-linalg_mega_conv2d_k3_c16-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d_s2_depthwise_s2-baremetal", + "pebble-pebble-mlirtest-core_mega_global_avg_conv_exact-baremetal", + "pebble-pebble-mlirtest-core_mega_global_avg_conv2_exact-baremetal", + "pebble-pebble-mlirtest-core_mega_global_avg_7x7_c288_exact-baremetal", + "pebble-pebble-mlirtest-core_mega_se_gate_conv-baremetal", + "pebble-pebble-ctest-mega_conv_pipeline_test-baremetal", + "pebble-pebble-ctest-residual_add_lut_test-baremetal", + "pebble-pebble-ctest-maxpool_im2col_test-baremetal", + "pebble-pebble-ctest-se_gate_lut_test-baremetal", + "pebble-pebble-ctest-int8mul_large_gate_row_test-baremetal", + "pebble-pebble-ctest-dual_smatmul_matadd_test-baremetal", + "pebble-pebble-ctest-dual_conv_add_int2fp_test-baremetal", + "pebble-pebble-ctest-quant_smatmul_int2fp_test-baremetal", + "pebble-pebble-ctest-smatmul_mvout_bank_reuse_test-baremetal", + "pebble-pebble-mlirtest-bank_matadd_i32_elementwise-baremetal", + "pebble-pebble-mlirtest-bank_int8add_identity_scale-baremetal", + "pebble-pebble-mlirtest-bank_lut_identity_table-baremetal", + "pebble-pebble-mlirtest-bank_int8mul_unit_gate-baremetal", + "pebble-pebble-mlirtest-bank_maxpool_k3_padded_spatial-baremetal", + "pebble-pebble-mlirtest-bank_smatmul_bias_two_terms-baremetal", + "pebble-pebble-mlirtest-bank_int2fp_unit_scale-baremetal", + "pebble-pebble-mlirtest-bank_quant_i32_pack16-baremetal", + "pebble-pebble-mlirtest-bank_quant_f32_pack16-baremetal", + "pebble-pebble-mlirtest-bank_im2col_k3_padded_bank_windows-baremetal", + "pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-baremetal", + "pebble-pebble-ctest-tlb_bare_mode_test-baremetal", + "pebble-pebble-ctest-tlb_repeated_sfence_test-baremetal", + "pebble-pebble-ctest-tlb_sfence_flush_test-baremetal", + "pebble-pebble-ctest-tlb_sfence_remap_test-baremetal", + "pebble-pebble-ctest-tlb_sv39_4k_alias_test-baremetal", + "pebble-pebble-ctest-tlb_sv39_identity_test-baremetal", + "pebble-pebble-ctest-tlb_sv39_nonidentity_test-baremetal", + "pebble-pebble-ctest-transpose_i8_16x16_test-baremetal", + "pebble-pebble-ctest-transpose_i8_16x32_test-baremetal", + "pebble-pebble-ctest-transpose_i32_8x16_test-baremetal", + "pebble-pebble-mlirtest-ball_transpose_16x16_i8-baremetal", + "pebble-pebble-ctest-smatmul_bias_accumulate_test-baremetal", + "pebble-pebble-ctest-smatmul_1x16x64_test-baremetal", + "pebble-pebble-ctest-smatmul_16x16_test-baremetal", + "pebble-pebble-ctest-smatmul_16x16x32_test-baremetal", + "pebble-pebble-ctest-smatmul_nondefault_banks_test-baremetal", + "pebble-pebble-mlirtest-ball_smatmul_bias_accumulate-baremetal", + "pebble-pebble-ctest-quant_f32_to_i8_test-baremetal", + "pebble-pebble-ctest-quant_i32_to_i8_test-baremetal", + "pebble-pebble-ctest-toint8_i8_rounding_test-baremetal", + "pebble-pebble-ctest-toint8_i8_rows_test-baremetal", + "pebble-pebble-ctest-toint8_i8_scale_rows_test-baremetal", + "pebble-pebble-ctest-toint8_i8_signed_test-baremetal", + "pebble-pebble-mlirtest-ball_quant_f32_to_i8-baremetal", + "pebble-pebble-mlirtest-ball_quant_i32_to_i8-baremetal", + "pebble-pebble-ctest-im2col_window_test-baremetal", + "pebble-pebble-ctest-im2col_k1_test-baremetal", + "pebble-pebble-ctest-im2col_k3_test-baremetal", + "pebble-pebble-ctest-im2col_k3_pad_test-baremetal", + "pebble-pebble-ctest-im2col_k3_asym_pad_test-baremetal", + "pebble-pebble-ctest-im2col_k3_partial_test-baremetal", + "pebble-pebble-ctest-im2col_k5_test-baremetal", + "pebble-pebble-ctest-im2col_k7_test-baremetal", + "pebble-pebble-mlirtest-ball_im2col_6x6_k3-baremetal", + "pebble-pebble-ctest-int2fp_tensor_values_test-baremetal", + "pebble-pebble-ctest-int2fp_tensor_groups_test-baremetal", + "pebble-pebble-ctest-int2fp_tensor_rows_test-baremetal", + "pebble-pebble-ctest-int2fp_channel_lanes_test-baremetal", + "pebble-pebble-ctest-int2fp_channel_base_test-baremetal", + "pebble-pebble-ctest-int2fp_channel_two_rows_test-baremetal", + "pebble-pebble-mlirtest-ball_int2fp_4x4-baremetal", + "pebble-pebble-mlirtest-ball_int2fp_relu_4x4-baremetal", + "pebble-pebble-ctest-lut_uniform_lookup_test-baremetal", + "pebble-pebble-ctest-lut_channel_table_lookup_test-baremetal", + "pebble-pebble-mlirtest-ball_lut_identity-baremetal", + "pebble-pebble-ctest-maxpool_window_test-baremetal", + "pebble-pebble-ctest-maxpool_offset_test-baremetal", + "pebble-pebble-mlirtest-ball_maxpool_6x6_s2-baremetal", + "pebble-pebble-ctest-int8add_scaled_test-baremetal", + "pebble-pebble-ctest-int8add_relu_test-baremetal", + "pebble-pebble-mlirtest-ball_int8add_relu-baremetal", + "pebble-pebble-ctest-int8mul_per_channel_gate_test-baremetal", + "pebble-pebble-mlirtest-ball_int8mul_gate-baremetal", + "pebble-pebble-ctest-matadd_i32_16row_test-baremetal", + "pebble-pebble-mlirtest-ball_matadd_i32-baremetal", + "pebble-pebble-ctest-pipeline-bank_dma_reuse-multicore-baremetal", + "pebble-pebble-ctest-pipeline-bank_rename_overlap-multicore-baremetal", + "pebble-pebble-ctest-pipeline-maxpool_partial_update-multicore-baremetal", + "pebble-pebble-ctest-pipeline-bank_dma_pingpong-multicore-baremetal", + "pebble-pebble-ctest-pipeline-read_bank_shared-multicore-baremetal", + "pebble-pebble-ctest-pipeline-read_bank_split-multicore-baremetal", + "pebble-pebble-ctest-pipeline-same_ball_independent-multicore-baremetal", + "pebble-pebble-ctest-pipeline-rob_window_probe-multicore-baremetal", + "pebble-pebble-ctest-pipeline-write_bank_versions-multicore-baremetal", +] diff --git a/examples/designs/pebble/sims/device/regression/bemu/workloads-linux.toml b/examples/designs/pebble/sims/device/regression/bemu/workloads-linux.toml new file mode 100644 index 000000000..3b03d2bd2 --- /dev/null +++ b/examples/designs/pebble/sims/device/regression/bemu/workloads-linux.toml @@ -0,0 +1,4 @@ +[workloads] +tests = [ + "fw_payload-h1-pebble-linux.elf", +] diff --git a/examples/designs/pebble/sims/device/regression/p2e/workloads-bare.toml b/examples/designs/pebble/sims/device/regression/p2e/workloads-bare.toml new file mode 100644 index 000000000..f483058b0 --- /dev/null +++ b/examples/designs/pebble/sims/device/regression/p2e/workloads-bare.toml @@ -0,0 +1,33 @@ +[workloads] +tests = [ + "pebble-pebble-ctest-mvin_2d_pixel_stride_test-baremetal.hex", + "pebble-pebble-ctest-mega_conv_pipeline_test-baremetal.hex", + "pebble-pebble-mlirtest-core_mega_conv2d_s2_depthwise_s2-baremetal.hex", + "pebble-pebble-mlirtest-core_mega_se_gate_conv-baremetal.hex", + "pebble-pebble-mlirtest-core_mega_global_avg_conv_exact-baremetal.hex", + "pebble-pebble-mlirtest-core_mega_global_avg_conv2_exact-baremetal.hex", + "pebble-pebble-mlirtest-core_mega_global_avg_7x7_c288_exact-baremetal.hex", + "pebble-pebble-ctest-residual_add_lut_test-baremetal.hex", + "pebble-pebble-ctest-maxpool_im2col_test-baremetal.hex", + "pebble-pebble-ctest-se_gate_lut_test-baremetal.hex", + "pebble-pebble-ctest-dual_conv_add_int2fp_test-baremetal.hex", + "pebble-pebble-mlirtest-bank_matadd_i32_elementwise-baremetal.hex", + "pebble-pebble-mlirtest-bank_int8add_identity_scale-baremetal.hex", + "pebble-pebble-mlirtest-bank_lut_identity_table-baremetal.hex", + "pebble-pebble-mlirtest-bank_int8mul_unit_gate-baremetal.hex", + "pebble-pebble-mlirtest-bank_maxpool_k3_padded_spatial-baremetal.hex", + "pebble-pebble-mlirtest-bank_smatmul_bias_two_terms-baremetal.hex", + "pebble-pebble-mlirtest-bank_int2fp_unit_scale-baremetal.hex", + "pebble-pebble-mlirtest-bank_quant_i32_pack16-baremetal.hex", + "pebble-pebble-mlirtest-bank_quant_f32_pack16-baremetal.hex", + "pebble-pebble-mlirtest-bank_im2col_k3_padded_bank_windows-baremetal.hex", + "pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-baremetal.hex", + "pebble-pebble-ctest-pipeline-bank_dma_reuse-multicore-baremetal", + "pebble-pebble-ctest-pipeline-maxpool_partial_update-multicore-baremetal", + "pebble-pebble-ctest-pipeline-bank_dma_pingpong-multicore-baremetal", + "pebble-pebble-ctest-pipeline-read_bank_shared-multicore-baremetal", + "pebble-pebble-ctest-pipeline-read_bank_split-multicore-baremetal", + "pebble-pebble-ctest-pipeline-same_ball_independent-multicore-baremetal", + "pebble-pebble-ctest-pipeline-rob_window_probe-multicore-baremetal", + "pebble-pebble-ctest-pipeline-write_bank_versions-multicore-baremetal", +] diff --git a/examples/chips/pebble/regression/batch/p2e/workloads-linux.toml b/examples/designs/pebble/sims/device/regression/p2e/workloads-linux.toml similarity index 100% rename from examples/chips/pebble/regression/batch/p2e/workloads-linux.toml rename to examples/designs/pebble/sims/device/regression/p2e/workloads-linux.toml diff --git a/examples/designs/pebble/sims/device/regression/vcs/workloads-bare.toml b/examples/designs/pebble/sims/device/regression/vcs/workloads-bare.toml new file mode 100644 index 000000000..5a932f0b2 --- /dev/null +++ b/examples/designs/pebble/sims/device/regression/vcs/workloads-bare.toml @@ -0,0 +1,11 @@ +[workloads] +tests = [ + "pebble-core0-soctest-clint_timer_interrupt-baremetal", + "pebble-core0-soctest-clint_software_interrupt-baremetal", + "pebble-core0-soctest-clint_supervisor_timer-baremetal", + "pebble-core0-soctest-time_csr_clock-baremetal", + "pebble-core0-soctest-time_csr_permissions-baremetal", + "pebble-core0-soctest-lrsc_word-baremetal", + "pebble-core0-soctest-lrsc_translated-baremetal", + "pebble-core0-soctest-plic_register_access-baremetal", +] diff --git a/examples/designs/pebble/sims/device/regression/verilator/workloads-bare.toml b/examples/designs/pebble/sims/device/regression/verilator/workloads-bare.toml new file mode 100644 index 000000000..c50847150 --- /dev/null +++ b/examples/designs/pebble/sims/device/regression/verilator/workloads-bare.toml @@ -0,0 +1,101 @@ +[workloads] +tests = [ + "pebble-pebble-mlirtest-core_mega_matmul_m17_k65_n17-baremetal", + "pebble-pebble-mlirtest-core_mega_matmul-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d_k5_tail-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d-baremetal", + "pebble-pebble-mlirtest-core_lookup_channel_i8-baremetal", + "pebble-pebble-mlirtest-core_generic_matmul_i8-baremetal", + "pebble-pebble-mlirtest-core_linear_int8-baremetal", + "pebble-pebble-mlirtest-core_linear_chain_i8-baremetal", + "pebble-pebble-ctest-bank_capacity_rename_test-baremetal", + "pebble-pebble-ctest-bank_transfer_recycle_test-baremetal", + "pebble-pebble-ctest-mset_transfer_test-baremetal", + "pebble-pebble-ctest-bank_recycle_test-baremetal", + "pebble-pebble-ctest-mvin_2d_test-baremetal", + "pebble-pebble-ctest-mvin_2d_pixel_stride_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_acc_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_stride_test-baremetal", + "pebble-pebble-ctest-mvin_mvout_unaligned_test-baremetal", + "pebble-pebble-ctest-mvin_unaligned_test-baremetal", + "pebble-pebble-ctest-tlb_bare_mode_test-baremetal", + "pebble-pebble-ctest-tlb_repeated_sfence_test-baremetal", + "pebble-pebble-ctest-tlb_sfence_flush_test-baremetal", + "pebble-pebble-ctest-tlb_sfence_remap_test-baremetal", + "pebble-pebble-ctest-tlb_sv39_4k_alias_test-baremetal", + "pebble-pebble-ctest-tlb_sv39_identity_test-baremetal", + "pebble-pebble-ctest-tlb_sv39_nonidentity_test-baremetal", + "pebble-pebble-mlirtest-linalg_mega_conv2d_k3_c16-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d_s2_depthwise_s2-baremetal", + "pebble-pebble-ctest-transpose_i8_16x16_test-baremetal", + "pebble-pebble-ctest-transpose_i8_16x32_test-baremetal", + "pebble-pebble-ctest-transpose_i32_8x16_test-baremetal", + "pebble-pebble-mlirtest-ball_transpose_16x16_i8-baremetal", + "pebble-pebble-ctest-smatmul_bias_accumulate_test-baremetal", + "pebble-pebble-ctest-smatmul_1x16x64_test-baremetal", + "pebble-pebble-ctest-smatmul_16x16_test-baremetal", + "pebble-pebble-ctest-smatmul_16x16x32_test-baremetal", + "pebble-pebble-ctest-smatmul_nondefault_banks_test-baremetal", + "pebble-pebble-ctest-smatmul_mvout_bank_reuse_test-baremetal", + "pebble-pebble-mlirtest-ball_smatmul_bias_accumulate-baremetal", + "pebble-pebble-ctest-quant_f32_to_i8_test-baremetal", + "pebble-pebble-ctest-quant_i32_to_i8_test-baremetal", + "pebble-pebble-ctest-toint8_i8_rounding_test-baremetal", + "pebble-pebble-ctest-toint8_i8_rows_test-baremetal", + "pebble-pebble-ctest-toint8_i8_scale_rows_test-baremetal", + "pebble-pebble-ctest-toint8_i8_signed_test-baremetal", + "pebble-pebble-mlirtest-ball_quant_f32_to_i8-baremetal", + "pebble-pebble-mlirtest-ball_quant_i32_to_i8-baremetal", + "pebble-pebble-ctest-im2col_window_test-baremetal", + "pebble-pebble-ctest-im2col_k1_test-baremetal", + "pebble-pebble-ctest-im2col_k3_test-baremetal", + "pebble-pebble-ctest-im2col_k3_pad_test-baremetal", + "pebble-pebble-ctest-im2col_k3_asym_pad_test-baremetal", + "pebble-pebble-ctest-im2col_k3_partial_test-baremetal", + "pebble-pebble-ctest-im2col_k5_test-baremetal", + "pebble-pebble-ctest-im2col_k7_test-baremetal", + "pebble-pebble-mlirtest-ball_im2col_6x6_k3-baremetal", + "pebble-pebble-ctest-int2fp_tensor_values_test-baremetal", + "pebble-pebble-ctest-int2fp_tensor_groups_test-baremetal", + "pebble-pebble-ctest-int2fp_tensor_rows_test-baremetal", + "pebble-pebble-ctest-int2fp_channel_lanes_test-baremetal", + "pebble-pebble-ctest-int2fp_channel_base_test-baremetal", + "pebble-pebble-ctest-int2fp_channel_two_rows_test-baremetal", + "pebble-pebble-mlirtest-ball_int2fp_4x4-baremetal", + "pebble-pebble-mlirtest-ball_int2fp_relu_4x4-baremetal", + "pebble-pebble-ctest-lut_uniform_lookup_test-baremetal", + "pebble-pebble-ctest-lut_channel_table_lookup_test-baremetal", + "pebble-pebble-mlirtest-ball_lut_identity-baremetal", + "pebble-pebble-ctest-maxpool_window_test-baremetal", + "pebble-pebble-ctest-maxpool_offset_test-baremetal", + "pebble-pebble-mlirtest-ball_maxpool_6x6_s2-baremetal", + "pebble-pebble-ctest-int8add_scaled_test-baremetal", + "pebble-pebble-ctest-int8add_relu_test-baremetal", + "pebble-pebble-mlirtest-ball_int8add_relu-baremetal", + "pebble-pebble-ctest-int8mul_per_channel_gate_test-baremetal", + "pebble-pebble-ctest-int8mul_large_gate_row_test-baremetal", + "pebble-pebble-mlirtest-ball_int8mul_gate-baremetal", + "pebble-pebble-ctest-matadd_i32_16row_test-baremetal", + "pebble-pebble-mlirtest-ball_matadd_i32-baremetal", + "pebble-pebble-ctest-mega_conv_pipeline_test-baremetal", + "pebble-pebble-mlirtest-core_mega_se_gate_conv-baremetal", + "pebble-pebble-mlirtest-core_mega_global_avg_conv_exact-baremetal", + "pebble-pebble-mlirtest-core_mega_global_avg_conv2_exact-baremetal", + "pebble-pebble-mlirtest-core_mega_global_avg_7x7_c288_exact-baremetal", + "pebble-pebble-ctest-residual_add_lut_test-baremetal", + "pebble-pebble-ctest-maxpool_im2col_test-baremetal", + "pebble-pebble-ctest-se_gate_lut_test-baremetal", + "pebble-pebble-ctest-dual_smatmul_matadd_test-baremetal", + "pebble-pebble-ctest-dual_conv_add_int2fp_test-baremetal", + "pebble-pebble-ctest-quant_smatmul_int2fp_test-baremetal", + "pebble-pebble-ctest-pipeline-bank_dma_reuse-multicore-baremetal", + "pebble-pebble-ctest-pipeline-bank_rename_overlap-multicore-baremetal", + "pebble-pebble-ctest-pipeline-maxpool_partial_update-multicore-baremetal", + "pebble-pebble-ctest-pipeline-bank_dma_pingpong-multicore-baremetal", + "pebble-pebble-ctest-pipeline-read_bank_shared-multicore-baremetal", + "pebble-pebble-ctest-pipeline-read_bank_split-multicore-baremetal", + "pebble-pebble-ctest-pipeline-same_ball_independent-multicore-baremetal", + "pebble-pebble-ctest-pipeline-rob_window_probe-multicore-baremetal", + "pebble-pebble-ctest-pipeline-write_bank_versions-multicore-baremetal", +] diff --git a/examples/designs/pebble/sims/device/stack/CMakeLists.txt b/examples/designs/pebble/sims/device/stack/CMakeLists.txt new file mode 100644 index 000000000..22b169301 --- /dev/null +++ b/examples/designs/pebble/sims/device/stack/CMakeLists.txt @@ -0,0 +1,37 @@ +# ===----------------------------------------------------------------------=== +# The software stack of the pebble device simulation: every level the device +# needs, built bottom up. +# +# bbdev stack --build "--design pebble" +# +# configures this with DESIGN_CMAKE, generated from design.pb. Each level is a +# CMake build of its own: cores and threads build buddy-opt plugins for the +# host, the device builds RISC-V libraries. +# core/pebble libpebble-compiler.so, execute.o +# device runtime, runtime-baremetal, runtime-linux, engine; device.cmake +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(pebble-device-stack NONE) +include(ExternalProject) +include("${DESIGN_CMAKE}") + +# Pebble's CPU core with its Ball accelerator. +ExternalProject_Add(core-pebble + SOURCE_DIR "${DESIGN_CORE_STACK_pebble}" + BINARY_DIR "${DESIGN_CORE_BUILD_pebble}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DCORE=pebble + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# The device: the runtime that schedules jobs onto the cores. +ExternalProject_Add(device + SOURCE_DIR "${DESIGN_DEVICE_STACK}" + BINARY_DIR "${DESIGN_DEVICE_BUILD}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" + -DCMAKE_C_COMPILER=riscv64-unknown-linux-gnu-gcc + -DCMAKE_CXX_COMPILER=riscv64-unknown-linux-gnu-g++ + DEPENDS core-pebble + BUILD_ALWAYS ON + INSTALL_COMMAND "") diff --git a/examples/designs/pebble/sims/device/stack/core/pebble/CMakeLists.txt b/examples/designs/pebble/sims/device/stack/core/pebble/CMakeLists.txt new file mode 100644 index 000000000..908456289 --- /dev/null +++ b/examples/designs/pebble/sims/device/stack/core/pebble/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.20) +project(pebble-core-stack LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +add_subdirectory("${DESIGN_REPO}/examples/designs/pebble/stack/engine" engine) +add_subdirectory("${DESIGN_REPO}/examples/designs/pebble/stack/midend" midend) diff --git a/examples/designs/pebble/sims/device/stack/runtime/CMakeLists.txt b/examples/designs/pebble/sims/device/stack/runtime/CMakeLists.txt new file mode 100644 index 000000000..1c3999e2b --- /dev/null +++ b/examples/designs/pebble/sims/device/stack/runtime/CMakeLists.txt @@ -0,0 +1,37 @@ +# ===----------------------------------------------------------------------=== +# The pebble device: its cores, their hardware threads and the runtime that +# schedules jobs onto the cores. +# +# bbdev stack --build "--design pebble" +# +# builds every thread and core stack of the device first, then configures this +# with DESIGN_CMAKE, generated from design.pb, and the RISC-V Linux toolchain. +# Products, exported in device.cmake for the programs that run on the device: +# runtime, runtime-baremetal, runtime-linux job scheduling over the cores +# engine hardware threads and instruction streams +# bindings how each core runs jobs and each +# hardware thread is loaded +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(pebble-device LANGUAGES C CXX) +include("${DESIGN_CMAKE}") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# How each core runs jobs, and how each hardware thread is loaded. Nothing +# references these bindings, so they are linked as objects rather than taken +# from an archive; as link options they come before the libraries they call. +add_library(bindings INTERFACE) +foreach(CORE IN LISTS DESIGN_CORES) + target_link_options(bindings INTERFACE "${DESIGN_CORE_BUILD_${CORE}}/execute.o") +endforeach() +foreach(THREAD IN LISTS DESIGN_THREADS) + target_link_options(bindings INTERFACE "${DESIGN_THREAD_BUILD_${THREAD}}/loader.o") +endforeach() +# DMA helpers are sized by the target with the fewest bank lines. +list(GET DESIGN_RUNTIME_TARGETS 0 PARAMS_TARGET) +set(RUNTIME_PARAMS "${DESIGN_ISA_DIR}/${PARAMS_TARGET}") + +add_subdirectory("${DESIGN_REPO}/stack/engine" engine) +add_subdirectory("${DESIGN_REPO}/stack/runtime" runtime) +export(TARGETS bindings runtime runtime-baremetal runtime-linux engine + runtime-shared runtime-linux-shared engine-shared FILE device.cmake) diff --git a/examples/designs/pebble/sims/device/tapeout/README.md b/examples/designs/pebble/sims/device/tapeout/README.md new file mode 100644 index 000000000..ee8f027b3 --- /dev/null +++ b/examples/designs/pebble/sims/device/tapeout/README.md @@ -0,0 +1,10 @@ +# Pebble tapeout contract + +Layout: +- `config.toml` — design contract (scripts, libs, power window) +- `area/` — DC (`dc.tcl`, `constraints.sdc`) +- `power/` — PTPX + gate sim (`power.tcl`, `power_sim.sh`) +- `ip/` — SRAM MDF + +`bbdev dc` resolves paths from `config.toml` and injects `RUN_*`. Timing truth is +`area/constraints.sdc`. Pebble main clock is 100 MHz (`10 ns`). diff --git a/examples/chips/pebble/tapeout/area/.gitignore b/examples/designs/pebble/sims/device/tapeout/area/.gitignore similarity index 100% rename from examples/chips/pebble/tapeout/area/.gitignore rename to examples/designs/pebble/sims/device/tapeout/area/.gitignore diff --git a/examples/chips/pebble/tapeout/area/constraints.sdc b/examples/designs/pebble/sims/device/tapeout/area/constraints.sdc similarity index 100% rename from examples/chips/pebble/tapeout/area/constraints.sdc rename to examples/designs/pebble/sims/device/tapeout/area/constraints.sdc diff --git a/examples/designs/pebble/sims/device/tapeout/area/dc.tcl b/examples/designs/pebble/sims/device/tapeout/area/dc.tcl new file mode 100755 index 000000000..f5d3b97a2 --- /dev/null +++ b/examples/designs/pebble/sims/device/tapeout/area/dc.tcl @@ -0,0 +1,65 @@ +# Design-owned DC flow. Timing comes from constraints.sdc (RUN_SDC). +# bbdev injects RUN_* via -x {set RUN_CONFIG }. +if {![info exists RUN_CONFIG] || $RUN_CONFIG eq ""} { + error "bbdev must pass -x {set RUN_CONFIG }" +} +source [file normalize $RUN_CONFIG] + +proc bbdev_read_filelist {path} { + set fh [open $path r] + set files [list] + while {[gets $fh line] >= 0} { + set line [string trim $line] + if {$line eq "" || [string match "#*" $line]} { continue } + lappend files [file normalize $line] + } + close $fh + if {[llength $files] == 0} { error "empty DC source list: $path" } + return $files +} + +set top $RUN_TOP +set output_dir [file normalize $RUN_OUTPUT_DIR] +set report_dir [file normalize $RUN_REPORT_DIR] +file mkdir $output_dir $report_dir [file join $report_dir work] +set target_library [list $RUN_TARGET_LIBRARY] +set synthetic_library $RUN_SYNTHETIC_LIBRARY +set link_library [concat [list *] $target_library $synthetic_library $RUN_LINK_LIBRARY] +set_host_options -max_cores $RUN_MAX_CORES +define_design_lib WORK -path [file join $report_dir work] +set search_path [list .] +set_app_var verilogout_no_tri true +set_app_var verilogout_equation false +set_app_var compile_seqmap_propagate_constants false +set_app_var hdlin_ff_always_sync_set_reset true +analyze -format sverilog -define {SYNTHESIS DC_SYN} [bbdev_read_filelist $RUN_SOURCE_LIST] +elaborate $top +current_design $top +link +set sram_cells [get_cells -hierarchical -quiet -filter {ref_name =~ "chipyard_sram_*"}] +if {[sizeof_collection $sram_cells] == 0} { + error "no chipyard_sram_* cells after link; SRAM .db link_library missing" +} +set_dont_touch $sram_cells +set sram_designs [get_designs -quiet chipyard_sram_*] +if {[sizeof_collection $sram_designs] > 0} { + set_dont_touch $sram_designs +} +read_sdc [file normalize $RUN_SDC] +if {[sizeof_collection [all_clocks]] == 0} { + error "SDC created no clocks" +} +set_load 2.0 [all_outputs] +set_app_var compile_seqmap_honor_sync_set_reset true +compile_ultra -area_high_effort_script -no_autoungroup -no_boundary_optimization +set_fix_multiple_port_nets -all -buffer_constants +change_names -hierarchy -rules verilog +write -format ddc -hierarchy -output [file join $output_dir ${top}.ddc] +write -format verilog -hierarchy -output [file join $output_dir ${top}.v] +write_sdc [file join $output_dir ${top}.sdc] +report_constraint -all_violators > [file join $report_dir constraint.rpt] +report_timing -delay max -max_paths 50 > [file join $report_dir timing_max.rpt] +report_timing -delay min -max_paths 50 > [file join $report_dir timing_min.rpt] +report_area -hierarchy > [file join $report_dir area.rpt] +report_reference > [file join $report_dir reference.rpt] +exit diff --git a/examples/designs/pebble/sims/device/tapeout/config.toml b/examples/designs/pebble/sims/device/tapeout/config.toml new file mode 100644 index 000000000..758d298bb --- /dev/null +++ b/examples/designs/pebble/sims/device/tapeout/config.toml @@ -0,0 +1,117 @@ +[tapeout] +top = "DigitalTop" +sram_process = "smic180" +sram_mdf = "ip/smic180_sram_library.mdf.json" +constraints_sdc = "area/constraints.sdc" +target_library = "/home/bb-runner/Code/eda/D_libraries/SMIC180/SCC018UG_UHD_RVT_V0.4a/SCC018UG_UHD_RVT_V0p4a/liberty/1.8v/scc018ug_uhd_rvt_ss_v1p62_125c_ccs.db" +synthetic_library = [ + "/home/bb-runner/Code/eda/A_packages/tools/syn/R-2020.09-SP5/libraries/syn/dw_foundation.sldb", + "/home/bb-runner/Code/eda/A_packages/tools/syn/R-2020.09-SP5/libraries/syn/standard.sldb", +] +link_library = [] +max_threads = 16 +power_format = "vcd" +power_start_ns = "1000000" +power_end_ns = "5000000" +dc_script = "area/dc.tcl" +power_script = "power/power.tcl" +power_sim_script = "power/power_sim.sh" +power_strip_path = "GatePowerHarness.dut" + +sram_corner = "ss" +lc_shell = "/home/bb-runner/Code/eda/A_packages/tools/lc/R-2020.09-SP5/bin/lc_shell" + +[[tapeout.sram]] +name = "chipyard_sram_32x128" +words = 32 +mux = 4 +bits = 128 + +[[tapeout.sram]] +name = "chipyard_sram_32x22" +words = 32 +mux = 4 +bits = 22 + +[[tapeout.sram]] +name = "chipyard_sram_64x21" +words = 64 +mux = 8 +bits = 21 + +[[tapeout.sram]] +name = "chipyard_sram_64x22" +words = 64 +mux = 8 +bits = 22 + +[[tapeout.sram]] +name = "chipyard_sram_512x8" +words = 512 +mux = 8 +bits = 8 + +[[tapeout.sram]] +name = "chipyard_sram_512x32" +words = 512 +mux = 8 +bits = 32 + +[[tapeout.sram]] +name = "chipyard_sram_512x64" +words = 512 +mux = 8 +bits = 64 + +[[tapeout.sram]] +name = "chipyard_sram_1024x8" +words = 1024 +mux = 8 +bits = 8 + +[[tapeout.sram]] +name = "chipyard_sram_32x21" +words = 32 +mux = 4 +bits = 21 + +[[tapeout.sram]] +name = "chipyard_sram_32x23" +words = 32 +mux = 4 +bits = 23 + +[[tapeout.sram]] +name = "chipyard_sram_64x8" +words = 64 +mux = 8 +bits = 8 + +[[tapeout.sram]] +name = "chipyard_sram_64x128" +words = 64 +# SMIC180 has only six metal layers, so this macro cannot be treated as a +# routable-over cell. mux=4 changes only the internal column organization: +# it keeps the 64x128 1RW + BWEN interface but shrinks the generated macro +# width from about 2013 um (mux=8) to about 1045 um, avoiding long routing walls. +mux = 4 +bits = 128 +bitwrite = true + +[[tapeout.sram]] +name = "chipyard_sram_128x8" +words = 128 +mux = 8 +bits = 8 + +[[tapeout.sram]] +name = "chipyard_sram_256x32" +words = 256 +mux = 8 +bits = 32 + +[[tapeout.sram]] +name = "chipyard_sram_256x64" +words = 256 +mux = 8 +bits = 64 diff --git a/examples/chips/pebble/tapeout/ip/smic180_sram_library.mdf.json b/examples/designs/pebble/sims/device/tapeout/ip/smic180_sram_library.mdf.json similarity index 100% rename from examples/chips/pebble/tapeout/ip/smic180_sram_library.mdf.json rename to examples/designs/pebble/sims/device/tapeout/ip/smic180_sram_library.mdf.json diff --git a/examples/chips/pebble/tapeout/ip/tsmc28_sram_library.mdf.json b/examples/designs/pebble/sims/device/tapeout/ip/tsmc28_sram_library.mdf.json similarity index 100% rename from examples/chips/pebble/tapeout/ip/tsmc28_sram_library.mdf.json rename to examples/designs/pebble/sims/device/tapeout/ip/tsmc28_sram_library.mdf.json diff --git a/examples/chips/pebble/tapeout/power/power.tcl b/examples/designs/pebble/sims/device/tapeout/power/power.tcl similarity index 100% rename from examples/chips/pebble/tapeout/power/power.tcl rename to examples/designs/pebble/sims/device/tapeout/power/power.tcl diff --git a/examples/chips/pebble/tapeout/power/power_sim.sh b/examples/designs/pebble/sims/device/tapeout/power/power_sim.sh similarity index 77% rename from examples/chips/pebble/tapeout/power/power_sim.sh rename to examples/designs/pebble/sims/device/tapeout/power/power_sim.sh index 3f3969706..01192a1aa 100755 --- a/examples/chips/pebble/tapeout/power/power_sim.sh +++ b/examples/designs/pebble/sims/device/tapeout/power/power_sim.sh @@ -13,7 +13,7 @@ if [[ "$ACTIVITY_FORMAT" != "vcd" ]]; then echo "Pebble gate power simulation currently emits VCD, got $ACTIVITY_FORMAT" >&2 exit 2 fi -ROOT=$(cd "$(dirname "$0")/../../../../.." && pwd) +ROOT=$(git -C "$(dirname "$0")" rev-parse --show-toplevel) RTL_DIR="$ROOT/arch/build/${CONFIG}" MEM_CONF="$RTL_DIR/mems.conf" BUILD="$OUTPUT_DIR/gate-vcs" @@ -55,7 +55,7 @@ module GatePowerHarness; wire [31:0] axi_aw_addr, axi_ar_addr; wire [7:0] axi_aw_len, axi_ar_len, axi_w_strb; wire [2:0] axi_aw_size, axi_aw_prot, axi_ar_size, axi_ar_prot; wire [3:0] axi_aw_cache, axi_aw_qos, axi_ar_cache, axi_ar_qos; wire [63:0] axi_w_data, axi_r_data; - BBSimDRAM #(.ADDR_BITS(32), .CHIP_ID(0), .CLOCK_HZ(100000000), .DATA_BITS(64), .ID_BITS(4), .LINE_SIZE(64), .MEM_BASE(40'd2147483648), .MEM_SIZE(268435456)) mem(.clock(clock), .reset(reset), .axi_aw_ready(axi_aw_ready), .axi_aw_valid(axi_aw_valid), .axi_aw_bits_id(axi_aw_id), .axi_aw_bits_addr(axi_aw_addr), .axi_aw_bits_len(axi_aw_len), .axi_aw_bits_size(axi_aw_size), .axi_aw_bits_burst(axi_aw_burst), .axi_aw_bits_lock(axi_aw_lock), .axi_aw_bits_cache(axi_aw_cache), .axi_aw_bits_prot(axi_aw_prot), .axi_aw_bits_qos(axi_aw_qos), .axi_w_ready(axi_w_ready), .axi_w_valid(axi_w_valid), .axi_w_bits_data(axi_w_data), .axi_w_bits_strb(axi_w_strb), .axi_w_bits_last(axi_w_last), .axi_b_ready(axi_b_ready), .axi_b_valid(axi_b_valid), .axi_b_bits_id(axi_b_id), .axi_b_bits_resp(axi_b_resp), .axi_ar_ready(axi_ar_ready), .axi_ar_valid(axi_ar_valid), .axi_ar_bits_id(axi_ar_id), .axi_ar_bits_addr(axi_ar_addr), .axi_ar_bits_len(axi_ar_len), .axi_ar_bits_size(axi_ar_size), .axi_ar_bits_burst(axi_ar_burst), .axi_ar_bits_lock(axi_ar_lock), .axi_ar_bits_cache(axi_ar_cache), .axi_ar_bits_prot(axi_ar_prot), .axi_ar_bits_qos(axi_ar_qos), .axi_r_ready(axi_r_ready), .axi_r_valid(axi_r_valid), .axi_r_bits_id(axi_r_id), .axi_r_bits_data(axi_r_data), .axi_r_bits_resp(axi_r_resp), .axi_r_bits_last(axi_r_last)); + BBSimDRAM #(.ADDR_BITS(32), .DEVICE_ID(0), .CLOCK_HZ(100000000), .DATA_BITS(64), .ID_BITS(4), .LINE_SIZE(64), .MEM_BASE(40'd2147483648), .MEM_SIZE(268435456)) mem(.clock(clock), .reset(reset), .axi_aw_ready(axi_aw_ready), .axi_aw_valid(axi_aw_valid), .axi_aw_bits_id(axi_aw_id), .axi_aw_bits_addr(axi_aw_addr), .axi_aw_bits_len(axi_aw_len), .axi_aw_bits_size(axi_aw_size), .axi_aw_bits_burst(axi_aw_burst), .axi_aw_bits_lock(axi_aw_lock), .axi_aw_bits_cache(axi_aw_cache), .axi_aw_bits_prot(axi_aw_prot), .axi_aw_bits_qos(axi_aw_qos), .axi_w_ready(axi_w_ready), .axi_w_valid(axi_w_valid), .axi_w_bits_data(axi_w_data), .axi_w_bits_strb(axi_w_strb), .axi_w_bits_last(axi_w_last), .axi_b_ready(axi_b_ready), .axi_b_valid(axi_b_valid), .axi_b_bits_id(axi_b_id), .axi_b_bits_resp(axi_b_resp), .axi_ar_ready(axi_ar_ready), .axi_ar_valid(axi_ar_valid), .axi_ar_bits_id(axi_ar_id), .axi_ar_bits_addr(axi_ar_addr), .axi_ar_bits_len(axi_ar_len), .axi_ar_bits_size(axi_ar_size), .axi_ar_bits_burst(axi_ar_burst), .axi_ar_bits_lock(axi_ar_lock), .axi_ar_bits_cache(axi_ar_cache), .axi_ar_bits_prot(axi_ar_prot), .axi_ar_bits_qos(axi_ar_qos), .axi_r_ready(axi_r_ready), .axi_r_valid(axi_r_valid), .axi_r_bits_id(axi_r_id), .axi_r_bits_data(axi_r_data), .axi_r_bits_resp(axi_r_resp), .axi_r_bits_last(axi_r_last)); always #5 clock = ~clock; initial begin longint timeout_ns; diff --git a/examples/designs/pebble/sims/device/workloads/CMakeLists.txt b/examples/designs/pebble/sims/device/workloads/CMakeLists.txt new file mode 100644 index 000000000..c3678bd81 --- /dev/null +++ b/examples/designs/pebble/sims/device/workloads/CMakeLists.txt @@ -0,0 +1,78 @@ +include("${DESIGN_DEVICE_BUILD}/device.cmake") + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/base" + "${CMAKE_BINARY_DIR}/src/CTest/base" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tlb" + "${CMAKE_BINARY_DIR}/src/CTest/tlb" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tutorial" + "${CMAKE_BINARY_DIR}/src/CTest/tutorial" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/embench" + "${CMAKE_BINARY_DIR}/src/CTest/embench" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/coremark" + "${CMAKE_BINARY_DIR}/src/CTest/coremark" +) +foreach(BUCKYBALL_COMPONENT_WORKLOAD_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(COMPONENT_WORKLOADS + "${DESIGN_TARGET_WORKLOADS_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" + ) + if( + COMPONENT_WORKLOADS + STREQUAL + "${BUCKYBALL_REPO_ROOT}/examples/designs/${BUCKYBALL_WORKLOAD_DESIGN}/workloads" + ) + continue() + endif() + set(BUCKYBALL_COMPONENT_WORKLOAD_PROFILE + "${DESIGN_TARGET_ID_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" + ) + set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}") + set(BUCKYBALL_CTEST_GROUP_TARGET component-ctests-build) + if(IS_DIRECTORY "${COMPONENT_WORKLOADS}/ctests") + add_subdirectory( + "${COMPONENT_WORKLOADS}/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/components/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" + ) + endif() +endforeach() +add_dependencies( + ctest-workloads-build + base-ctests-build + tlb-ctests-build + tutorial-ctests-build + component-ctests-build +) +add_dependencies(sync-native-bin embench-all coremark-all) + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/MLIRTest/base" + "${CMAKE_BINARY_DIR}/src/MLIRTest/mlir_tests/base" +) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_MLIR_ACTIVE_TARGET "${BUCKYBALL_TARGET}") + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_TARGET_BALLS_${BUCKYBALL_TARGET}) + set(BALL_MLIR_TESTS + "${BUCKYBALL_REPO_ROOT}/examples/components/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" + ) + if(IS_DIRECTORY "${BALL_MLIR_TESTS}") + add_subdirectory( + "${BALL_MLIR_TESTS}" + "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}" + ) + endif() + endforeach() +endforeach() +add_dependencies(MLIRTest-all base-mlirtest-build balls-mlir-tests-build) + +add_subdirectory( + "${BUCKYBALL_REPO_ROOT}/examples/designs/pebble/workloads" + design +) diff --git a/examples/designs/pebble/sims/device/workloads/kernel/CMakeLists.txt b/examples/designs/pebble/sims/device/workloads/kernel/CMakeLists.txt new file mode 100644 index 000000000..46341dc68 --- /dev/null +++ b/examples/designs/pebble/sims/device/workloads/kernel/CMakeLists.txt @@ -0,0 +1,42 @@ +# Linux programs installed into the pebble rootfs, each with the hart it runs on. +set(BUCKYBALL_KERNEL_PROGRAMS + 0 pebble-pebble-mlirtest-core_mega_matmul-linux + 0 pebble-pebble-mlirtest-core_mega_conv2d-linux + 0 pebble-pebble-mlirtest-core_lookup_channel_i8-linux + 0 pebble-pebble-mlirtest-core_generic_matmul_i8-linux + 0 pebble-pebble-mlirtest-core_linear_int8-linux + 0 pebble-pebble-mlirtest-core_linear_chain_i8-linux + 0 pebble-pebble-mlirtest-core_mega_conv2d_k5_tail-linux + 0 pebble-pebble-mlirtest-core_mega_matmul_m17_k65_n17-linux + 0 pebble-pebble-ctest-mvin_2d_test-linux + 0 pebble-pebble-ctest-mvin_2d_pixel_stride_test-linux + 0 pebble-pebble-ctest-mvin_mvout_acc_test-linux + 0 pebble-pebble-ctest-mvin_mvout_test-linux + 0 pebble-pebble-ctest-mvin_mvout_stride_test-linux + 0 pebble-pebble-ctest-mvin_mvout_unaligned_test-linux + 0 pebble-pebble-ctest-mvin_unaligned_test-linux + 0 pebble-pebble-mlirtest-linalg_mega_conv2d_k3_c16-linux + 0 pebble-pebble-mlirtest-core_mega_conv2d_s2_depthwise_s2-linux + 0 pebble-pebble-mlirtest-core_mega_global_avg_conv_exact-linux + 0 pebble-pebble-mlirtest-core_mega_global_avg_conv2_exact-linux + 0 pebble-pebble-mlirtest-core_mega_global_avg_7x7_c288_exact-linux + 0 pebble-pebble-mlirtest-core_mega_se_gate_conv-linux + 0 pebble-pebble-ctest-mega_conv_pipeline_test-linux + 0 pebble-pebble-ctest-residual_add_lut_test-linux + 0 pebble-pebble-ctest-maxpool_im2col_test-linux + 0 pebble-pebble-ctest-se_gate_lut_test-linux + 0 pebble-pebble-ctest-int8mul_large_gate_row_test-linux + 0 pebble-pebble-ctest-dual_smatmul_matadd_test-linux + 0 pebble-pebble-ctest-dual_conv_add_int2fp_test-linux + 0 pebble-pebble-mlirtest-bank_matadd_i32_elementwise-linux + 0 pebble-pebble-mlirtest-bank_int8add_identity_scale-linux + 0 pebble-pebble-mlirtest-bank_lut_identity_table-linux + 0 pebble-pebble-mlirtest-bank_int8mul_unit_gate-linux + 0 pebble-pebble-mlirtest-bank_maxpool_k3_padded_spatial-linux + 0 pebble-pebble-mlirtest-bank_smatmul_bias_two_terms-linux + 0 pebble-pebble-mlirtest-bank_int2fp_unit_scale-linux + 0 pebble-pebble-mlirtest-bank_quant_i32_pack16-linux + 0 pebble-pebble-mlirtest-bank_quant_f32_pack16-linux + 0 pebble-pebble-mlirtest-bank_im2col_k3_padded_bank_windows-linux + 0 pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-linux + PARENT_SCOPE) diff --git a/examples/designs/pebble/sims/ip/uvm/registry.toml b/examples/designs/pebble/sims/ip/uvm/registry.toml new file mode 100644 index 000000000..8e8131449 --- /dev/null +++ b/examples/designs/pebble/sims/ip/uvm/registry.toml @@ -0,0 +1 @@ +ips = [] diff --git a/examples/designs/pebble/stack/engine/CMakeLists.txt b/examples/designs/pebble/stack/engine/CMakeLists.txt new file mode 100644 index 000000000..686343604 --- /dev/null +++ b/examples/designs/pebble/stack/engine/CMakeLists.txt @@ -0,0 +1,13 @@ +# Core execution binding; instruction streams use the framework engine. +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +set(ENGINE "${DESIGN_REPO}/stack/engine") +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/execute.o" + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" + -c "${RUNTIME}/execute/direct.c" -o "${CMAKE_BINARY_DIR}/execute.o" + DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" + VERBATIM) +add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") diff --git a/examples/designs/pebble/stack/midend/CMakeLists.txt b/examples/designs/pebble/stack/midend/CMakeLists.txt new file mode 100644 index 000000000..f4006e66f --- /dev/null +++ b/examples/designs/pebble/stack/midend/CMakeLists.txt @@ -0,0 +1,32 @@ +# Host compiler plugin for this design's core. +set(ENGINE "${DESIGN_REPO}/stack/engine") +set(TARGET ${DESIGN_CORE_TARGET_${CORE}}) + +set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${TARGET}}) +set(BUCKYBALL_TARGET_TABLE "${DESIGN_ISA_DIR}/${TARGET}/target.inc") +# The core owns computation recognition and its conversion to bank dataflow. +file(GLOB_RECURSE BUCKYBALL_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/src/*.cpp") +add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) +add_library(${TARGET}-compiler MODULE $) +set_target_properties(${TARGET}-compiler PROPERTIES + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") + +set(CHECK "${DESIGN_MLIR}/../../../bin/FileCheck") +set(OPT "${DESIGN_COMPILER}/bin/buddy-opt --load-dialect-plugin=$ --load-pass-plugin=$") +set(TESTS "${CMAKE_CURRENT_SOURCE_DIR}/tests") +add_custom_target(${TARGET}-compiler-check + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-custom-ops.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${TESTS}/form-custom-ops.mlir --implicit-check-not=buckyball." + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-int32-bias.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${TESTS}/form-int32-bias.mlir --implicit-check-not=buckyball." + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-linear-chain.mlir -form-custom-ops | ${CHECK} ${TESTS}/form-linear-chain.mlir --check-prefix=FORM --implicit-check-not=custom.chain" + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-linear-chain.mlir -form-custom-ops -convert-linalg-to-buckyball | ${CHECK} ${TESTS}/form-linear-chain.mlir --check-prefix=BANK" + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/lower-frontend-types.mlir -allow-unregistered-dialect -lower-frontend -split-input-file -verify-diagnostics | ${CHECK} ${TESTS}/lower-frontend-types.mlir --check-prefix=VALID --implicit-check-not=frontend." + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/lower-custom-ops.mlir -convert-linalg-to-buckyball | ${CHECK} ${TESTS}/lower-custom-ops.mlir --implicit-check-not=custom.op" + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/unknown-custom-op.mlir -convert-linalg-to-buckyball -verify-diagnostics -o /dev/null" + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-quantized-chain.mlir -form-custom-ops | ${CHECK} ${TESTS}/form-quantized-chain.mlir --check-prefix=FORM" + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-quantized-chain.mlir -form-custom-ops -convert-linalg-to-buckyball | ${CHECK} ${TESTS}/form-quantized-chain.mlir --check-prefix=BANK --implicit-check-not=buckyball.bank_mvout" + COMMAND bash -o pipefail -c "${OPT} ${CMAKE_CURRENT_SOURCE_DIR}/../../workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${CMAKE_CURRENT_SOURCE_DIR}/../../workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir --check-prefix=FORM" + COMMAND bash -o pipefail -c "${OPT} ${CMAKE_CURRENT_SOURCE_DIR}/../../workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir -allow-unregistered-dialect -form-custom-ops -convert-linalg-to-buckyball | ${CHECK} ${CMAKE_CURRENT_SOURCE_DIR}/../../workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir --check-prefix=BANK --implicit-check-not=buckyball.bank_mvout" + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-depthwise.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${TESTS}/form-depthwise.mlir --implicit-check-not=linalg.depthwise" + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/lower-bank-updates.mlir -lower-buckyball | ${CHECK} ${TESTS}/lower-bank-updates.mlir" + DEPENDS ${TARGET}-compiler + VERBATIM) diff --git a/examples/designs/pebble/stack/midend/src/ConvertLinalgToBuckyball.cpp b/examples/designs/pebble/stack/midend/src/ConvertLinalgToBuckyball.cpp new file mode 100644 index 000000000..3458cd7da --- /dev/null +++ b/examples/designs/pebble/stack/midend/src/ConvertLinalgToBuckyball.cpp @@ -0,0 +1,328 @@ +//===- ConvertLinalgToBuckyball.cpp - Pebble core lowering ----------------===// + +#include "Buckyball/BuckyballDialect.h" +#include "Buckyball/BuckyballOps.h" +#include "Custom/CustomOps.h" +#include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/Pass/Pass.h" +#include "llvm/ADT/MapVector.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +// The recipe supplies contiguous tensors; offsets may point into packed +// weights. +Value pebble::tensorBuffer(OpBuilder &builder, Location loc, Value tensor) { + auto type = cast(tensor.getType()); + SmallVector strides(type.getRank()); + int64_t stride = 1; + for (int64_t dim = type.getRank() - 1; dim >= 0; --dim) { + strides[dim] = stride; + stride *= type.getDimSize(dim); + } + auto layout = StridedLayoutAttr::get(builder.getContext(), + ShapedType::kDynamic, strides); + auto memory = MemRefType::get(type.getShape(), type.getElementType(), layout); + return bufferization::ToBufferOp::create(builder, loc, memory, tensor, true); +} + +namespace { +LogicalResult lowerMatmul(::buddy::custom::CustomOp op, Value residentInput, + Value nextMultiplier, bool relu, Value &result) { + bool linear = op.getKind() == "linear_i8"; + if (op.getInputs().size() != (linear ? 6 : 3) || op.getNumResults() != 1) + return op.emitOpError("has the wrong number of inputs or results for ") + << op.getKind(); + ValueRange inputs = op.getInputs(); + auto a = cast(inputs[0].getType()); + auto b = cast(inputs[1].getType()); + auto resultType = cast(op.getResult(0).getType()); + if (a.hasStaticShape() && b.hasStaticShape() && a.getRank() == 2 && + b.getRank() == 2 && + (a.getDimSize(0) != 1 || a.getDimSize(1) % 16 || b.getDimSize(1) % 16)) { + if (residentInput || nextMultiplier) + return op.emitOpError("resident Linear requires one row and aligned K/N"); + return pebble::lowerLinearTiles(op, result); + } + const auto &target = buckyball_target::getBuckyballTarget(); + if (!a.hasStaticShape() || !b.hasStaticShape() || a.getRank() != 2 || + b.getRank() != 2 || a.getDimSize(0) != 1 || + a.getDimSize(1) != b.getDimSize(0) || a.getDimSize(1) <= 0 || + b.getDimSize(1) <= 0 || a.getDimSize(1) % 16 || b.getDimSize(1) % 16) + return op.emitOpError("requires 1xK and KxN tensors with positive K and N " + "multiples of sixteen"); + int64_t k = a.getDimSize(1), n = b.getDimSize(1); + if (target.bankWidthBits != 128 || k > target.bankDepth) + return op.emitOpError("operands do not fit the core bank geometry"); + if (!b.getElementType().isInteger(8) || + !(linear ? a.getElementType().isF32() + : a.getElementType().isInteger(8)) || + !(linear ? resultType.getElementType().isF32() + : resultType.getElementType().isInteger(32)) || + resultType.getShape() != ArrayRef({1, n})) + return op.emitOpError("has invalid operand or result types for ") + << op.getKind(); + const int64_t scalarDimensions[] = {1, 1}; + ArrayRef scalarShape(scalarDimensions); + if (linear && + (cast(inputs[3].getType()).getShape() != scalarShape || + cast(inputs[4].getType()).getShape() != scalarShape || + cast(inputs[5].getType()).getShape() != + ArrayRef({1, n}))) + return op.emitOpError("requires one activation scale and one weight scale " + "per output channel"); + auto initialType = cast(inputs[2].getType()); + if (!initialType.getElementType().isInteger(32) || + initialType.getShape() != resultType.getShape()) + return op.emitOpError( + "requires an INT32 initial accumulator matching the result"); + + if (nextMultiplier && (n / 16 > 128 || n / 16 > target.bankDepth)) + return op.emitOpError( + "resident output exceeds the quantizer row address range"); + + OpBuilder builder(op); + Location loc = op.getLoc(); + Type i64 = builder.getI64Type(); + Value zero = arith::ConstantIndexOp::create(builder, loc, 0); + Value weight = pebble::tensorBuffer(builder, loc, inputs[1]); + Value output; + if (!nextMultiplier) + output = memref::AllocOp::create( + builder, loc, MemRefType::get({1, n}, resultType.getElementType())); + Value residentOutput; + if (nextMultiplier) + residentOutput = BankAllocOp::create(builder, loc, i64, 1, 1); + Value initial = pebble::tensorBuffer(builder, loc, inputs[2]); + Value scales, inputStep, outputMultiplier; + Value codes = residentInput; + if (linear) { + inputStep = tensor::ExtractOp::create(builder, loc, inputs[4], + ValueRange{zero, zero}); + if (nextMultiplier) + outputMultiplier = tensor::ExtractOp::create(builder, loc, nextMultiplier, + ValueRange{zero, zero}); + scales = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getF32Type())); + } + if (!codes) { + Value input = pebble::tensorBuffer(builder, loc, inputs[0]); + codes = BankAllocOp::create(builder, loc, i64, 1, 1); + if (linear) { + Value multiplier = tensor::ExtractOp::create(builder, loc, inputs[3], + ValueRange{zero, zero}); + Value inputBank = BankAllocOp::create(builder, loc, i64, 1, 1); + inputBank = + BankMvinOp::create(builder, loc, i64, input, inputBank, k / 4, 1); + auto quantized = + BankCustomOp::create(builder, loc, i64, ValueRange{inputBank}, codes, + ValueRange{multiplier}, "fp32_to_int8"); + quantized->setAttr("rows", builder.getI64IntegerAttr(k / 4)); + codes = quantized.getDestinationOut(); + BankReleaseOp::create(builder, loc, inputBank); + } else { + codes = BankMvinOp::create(builder, loc, i64, input, codes, k / 16, 1); + } + } + + // Keep the activation in its bank while processing each sixteen-column slice. + for (int64_t column = 0; column < n; column += 16) { + for (int64_t lane = 0; linear && lane < 16; ++lane) { + Value index = arith::ConstantIndexOp::create(builder, loc, lane); + Value channel = + arith::ConstantIndexOp::create(builder, loc, column + lane); + Value step = tensor::ExtractOp::create(builder, loc, inputs[5], + ValueRange{zero, channel}); + Value product = arith::MulFOp::create(builder, loc, inputStep, step); + if (nextMultiplier) + product = + arith::MulFOp::create(builder, loc, product, outputMultiplier); + memref::StoreOp::create(builder, loc, product, scales, index); + } + Value weightSlice = memref::SubViewOp::create( + builder, loc, weight, ArrayRef{0, column}, + ArrayRef{k, 16}, ArrayRef{1, 1}); + Value weightBank = BankAllocOp::create(builder, loc, i64, 1, 1); + weightBank = BankMvinOp::create(builder, loc, i64, weightSlice, weightBank, + k, n / 16); + Value accumulator = BankAllocOp::create(builder, loc, i64, 1, 1); + Value initialSlice = memref::SubViewOp::create( + builder, loc, initial, ArrayRef{0, column}, + ArrayRef{1, 16}, ArrayRef{1, 1}); + accumulator = + BankMvinOp::create(builder, loc, i64, initialSlice, accumulator, 4, 1); + auto matmul = + BankCustomOp::create(builder, loc, i64, ValueRange{codes, weightBank}, + accumulator, ValueRange{}, "matmul_i8", true); + matmul->setAttr("rows", builder.getI64IntegerAttr(1)); + matmul->setAttr("k", builder.getI64IntegerAttr(k)); + accumulator = matmul.getDestinationOut(); + BankReleaseOp::create(builder, loc, weightBank); + + Value resultBank = accumulator; + if (linear) { + Value scaleBank = BankAllocOp::create(builder, loc, i64, 1, 1); + scaleBank = + BankMvinOp::create(builder, loc, i64, scales, scaleBank, 4, 1); + if (nextMultiplier) { + auto quantized = BankCustomOp::create( + builder, loc, i64, ValueRange{accumulator, scaleBank}, + residentOutput, ValueRange{}, "int32_to_int8", true); + quantized->setAttr("rows", builder.getI64IntegerAttr(1)); + quantized->setAttr("wr_base", builder.getI64IntegerAttr(column / 16)); + quantized->setAttr("relu", builder.getBoolAttr(relu)); + residentOutput = quantized.getDestinationOut(); + } else { + resultBank = BankAllocOp::create(builder, loc, i64, 1, 1); + auto restored = BankCustomOp::create( + builder, loc, i64, ValueRange{accumulator, scaleBank}, resultBank, + ValueRange{}, "int32_to_fp32"); + restored->setAttr("rows", builder.getI64IntegerAttr(1)); + resultBank = restored.getDestinationOut(); + } + BankReleaseOp::create(builder, loc, scaleBank); + } + if (!nextMultiplier) { + Value outputSlice = memref::SubViewOp::create( + builder, loc, output, ArrayRef{0, column}, + ArrayRef{1, 16}, ArrayRef{1, 1}); + resultBank = + BankMvoutOp::create(builder, loc, i64, outputSlice, resultBank, 4, 1); + if (linear) + BankReleaseOp::create(builder, loc, resultBank); + } + BankReleaseOp::create(builder, loc, accumulator); + } + BankReleaseOp::create(builder, loc, codes); + result = residentOutput; + if (!nextMultiplier) + result = bufferization::ToTensorOp::create( + builder, loc, op.getResult(0).getType(), output, true, true); + return success(); +} + +class ConvertLinalgToBuckyballPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ConvertLinalgToBuckyballPass) + StringRef getArgument() const final { return "convert-linalg-to-buckyball"; } + StringRef getDescription() const final { + return "Lower custom tensor computations to the core bank dataflow"; + } + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert(); + } + void runOnOperation() override { + if (failed(pebble::lowerQuantizedChains(getOperation()))) + return signalPassFailure(); + DenseMap residentVectors; + SmallVector spatial; + if (failed( + pebble::lowerSpatialOps(getOperation(), residentVectors, spatial))) + return signalPassFailure(); + llvm::MapVector> chains; + getOperation().walk([&](Operation *op) { + if (auto group = op->getAttrOfType("custom.chain")) + chains[group.getInt()].push_back(op); + }); + for (auto &[id, links] : chains) { + SmallVector<::buddy::custom::CustomOp> stages; + for (Operation *link : links) + if (auto op = dyn_cast<::buddy::custom::CustomOp>(link)) + if (op.getKind() == "linear_i8") + stages.push_back(op); + DenseMap resident; + Value input = residentVectors.lookup(stages.front().getInputs()[0]); + for (auto [index, stage] : llvm::enumerate(stages)) { + bool last = index + 1 == stages.size(); + Value multiplier = last ? Value() : stages[index + 1].getInputs()[3]; + bool relu = + !last && stage->getAttrOfType("custom.relu").getValue(); + Value lowered; + if (failed(lowerMatmul(stage, input, multiplier, relu, lowered))) + return signalPassFailure(); + if (last) + stage.getResult(0).replaceAllUsesWith(lowered); + else + resident[stage.getResult(0)] = input = lowered; + } + for (Operation *link : links) { + if (auto op = dyn_cast<::buddy::custom::CustomOp>(link)) { + if (op.getKind() == "relu") + resident[op.getResult(0)] = resident.at(op.getInputs()[0]); + } else { + // Cycle-only markers observe the resident bank without spilling it. + Value bank = resident.at(link->getOperand(0)); + resident[link->getResult(0)] = bank; + OpBuilder builder(link); + Operation *trace = builder.clone(*link); + trace->setOperand(0, bank); + trace->getResult(0).setType(bank.getType()); + trace->removeAttr("custom.chain"); + } + } + for (Operation *link : llvm::reverse(links)) + link->erase(); + } + SmallVector<::buddy::custom::CustomOp> computations; + getOperation().walk([&](::buddy::custom::CustomOp op) { + if (!llvm::is_contained(spatial, op.getOperation())) + computations.push_back(op); + }); + for (auto op : computations) { + if (op.getKind() == "global_avg_i8" || op.getKind() == "lookup_i8" || + op.getKind() == "gate_mul_i8") { + Value result; + LogicalResult status = + op.getKind() == "global_avg_i8" + ? pebble::lowerGlobalAverage(op, result, false) + : pebble::lowerChannelOp(op, result); + if (failed(status)) + return signalPassFailure(); + op.getResult(0).replaceAllUsesWith(result); + op.erase(); + continue; + } + if (op.getKind() == "transpose_i8" || + op.getKind() == "linear_i8_requant") { + Value result; + if (failed(op.getKind() == "transpose_i8" + ? pebble::lowerTranspose(op, result) + : pebble::lowerLinearTiles(op, result))) + return signalPassFailure(); + op.getResult(0).replaceAllUsesWith(result); + op.erase(); + continue; + } + if (op.getKind() != "linear_i8" && op.getKind() != "matmul_i8") { + op.emitOpError("has no bank lowering on this core for ") + << op.getKind(); + return signalPassFailure(); + } + Value result; + if (failed(lowerMatmul(op, residentVectors.lookup(op.getInputs()[0]), + Value(), false, result))) + return signalPassFailure(); + op.getResult(0).replaceAllUsesWith(result); + op.erase(); + } + for (Operation *op : llvm::reverse(spatial)) + op->erase(); + } +}; + +PassRegistration registration; +} // namespace diff --git a/examples/designs/pebble/stack/midend/src/FormCustomOps.cpp b/examples/designs/pebble/stack/midend/src/FormCustomOps.cpp new file mode 100644 index 000000000..3728268d2 --- /dev/null +++ b/examples/designs/pebble/stack/midend/src/FormCustomOps.cpp @@ -0,0 +1,295 @@ +//===- FormCustomOps.cpp - Recognize and fuse core computations -----------===// + +#include "Custom/CustomOps.h" +#include "Lowering.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/Linalg/Transforms/Transforms.h" +#include "mlir/IR/Dominance.h" +#include "mlir/IR/Matchers.h" +#include "mlir/IR/PatternMatch.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" +#include "mlir/Pass/Pass.h" + +using namespace mlir; +using ::buddy::custom::CustomOp; + +namespace { +Operation *frontend(Value value, StringRef name) { + Operation *op = value.getDefiningOp(); + return op && op->getName().getStringRef() == name ? op : nullptr; +} + +} // namespace + +bool pebble::elementwise(linalg::GenericOp op, StringRef name) { + if (!op || !op.hasPureTensorSemantics() || + !linalg::isaElemwiseSingleBinaryOpInterface(op) || + op.getBody()->front().getName().getStringRef() != name) + return false; + auto result = cast(op.getResult(0).getType()); + auto maps = op.getIndexingMapsArray(); + if (!result.hasStaticShape() || !maps.back().isIdentity()) + return false; + for (auto [index, input] : llvm::enumerate(op.getInputs())) { + auto type = cast(input.getType()); + if (!type.hasStaticShape() || type.getRank() != result.getRank()) + return false; + SmallVector dimensions; + for (int64_t dim = 0; dim < type.getRank(); ++dim) + dimensions.push_back(type.getDimSize(dim) == 1 && + result.getDimSize(dim) != 1 + ? getAffineConstantExpr(0, op.getContext()) + : getAffineDimExpr(dim, op.getContext())); + if (maps[index] != + AffineMap::get(type.getRank(), 0, dimensions, op.getContext())) + return false; + } + return true; +} + +namespace { +class FormCustomOpsPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(FormCustomOpsPass) + StringRef getArgument() const final { return "form-custom-ops"; } + StringRef getDescription() const final { + return "Recognize core computations and fuse them into custom tensor ops"; + } + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert<::buddy::custom::CustomDialect, linalg::LinalgDialect>(); + } + void runOnOperation() override { + // MLIR checks both indexing maps and scalar computations, including casts. + SmallVector generics; + getOperation().walk([&](linalg::GenericOp op) { generics.push_back(op); }); + IRRewriter rewriter(&getContext()); + for (auto op : generics) { + if (op.hasPureTensorSemantics() && + (linalg::isaContractionOpInterface(op) || + linalg::isaTransposeOpInterface(op))) { + rewriter.setInsertionPoint(op); + (void)linalg::specializeGenericOp(rewriter, op); + } + } + + SmallVector matmuls; + getOperation().walk([&](linalg::MatmulOp op) { matmuls.push_back(op); }); + DominanceInfo dominance(getOperation()); + for (auto op : matmuls) { + if (!op.hasPureTensorSemantics() || op.hasUserDefinedMaps() || + op.getCast() != linalg::TypeFn::cast_signed) + continue; + rewriter.setInsertionPoint(op); + auto a = cast(op.getInputs()[0].getType()); + auto b = cast(op.getInputs()[1].getType()); + auto result = cast(op.getResult(0).getType()); + if (a.getElementType().isInteger(8) && b.getElementType().isInteger(8) && + result.getElementType().isInteger(32)) { + // matmul_i8(A, B, C) = C + A @ B, with signed INT8 operands. + auto custom = CustomOp::create( + rewriter, op.getLoc(), TypeRange{result}, "matmul_i8", + ValueRange{op.getInputs()[0], op.getInputs()[1], + op.getOutputs()[0]}); + rewriter.replaceOp(op, custom.getOutputs()); + continue; + } + + Operation *lhs = frontend(op.getInputs()[0], "frontend.int8_to_fp32"); + Operation *rhs = frontend(op.getInputs()[1], "frontend.int8_to_fp32"); + if (!lhs || !rhs || !matchPattern(op.getOutputs()[0], m_AnyZeroFloat())) + continue; + Operation *quantize = + frontend(lhs->getOperand(0), "frontend.fp32_to_int8"); + if (!quantize) + continue; + // Match bias dequantized with exactly input_step * weight_step. + Value biasedResult = op.getResult(0); + Operation *trace = nullptr; + if (biasedResult.hasOneUse()) { + Operation *user = *biasedResult.getUsers().begin(); + auto tensorTrace = user->getAttrOfType("tensor"); + // Cycle markers follow the logical Linear. Tensor traces keep their + // observed boundary and therefore prevent folding bias across it. + if (user->getName().getStringRef() == "buddy_trace.end" && + tensorTrace && !tensorTrace.getValue()) { + trace = user; + biasedResult = user->getResult(0); + } + } + Value bias; + linalg::GenericOp biasAdd, biasScale; + Operation *biasDequant = nullptr; + if (biasedResult.hasOneUse()) { + auto add = + dyn_cast(*biasedResult.getUsers().begin()); + if (pebble::elementwise(add, "arith.addf") && + llvm::is_contained(add.getInputs(), biasedResult)) { + Value other = + add.getInputs()[add.getInputs()[0] == biasedResult ? 1 : 0]; + auto dequant = frontend(other, "frontend.int32_to_fp32"); + if (dequant) { + auto scale = + dequant->getOperand(1).getDefiningOp(); + auto biasType = + cast(dequant->getOperand(0).getType()); + if (pebble::elementwise(scale, "arith.mulf") && + biasType.getElementType().isInteger(32) && + biasType.getShape() == result.getShape() && + ((scale.getInputs()[0] == lhs->getOperand(1) && + scale.getInputs()[1] == rhs->getOperand(1)) || + (scale.getInputs()[1] == lhs->getOperand(1) && + scale.getInputs()[0] == rhs->getOperand(1))) && + dominance.properlyDominates(dequant->getOperand(0), op)) { + bias = dequant->getOperand(0); + biasAdd = add; + biasScale = scale; + biasDequant = dequant; + } + } + } + } + if (!bias) { + auto biasType = + RankedTensorType::get(result.getShape(), rewriter.getI32Type()); + bias = arith::ConstantOp::create(rewriter, op.getLoc(), biasType, + rewriter.getZeroAttr(biasType)); + } + // linear_i8(x, W, bias, multiplier, input_step, weight_step) accumulates + // signed INT8 products on INT32 bias, then restores FP32 values. + auto custom = CustomOp::create( + rewriter, op.getLoc(), TypeRange{result}, "linear_i8", + ValueRange{quantize->getOperand(0), rhs->getOperand(0), bias, + quantize->getOperand(1), lhs->getOperand(1), + rhs->getOperand(1)}); + rewriter.replaceOp(op, custom.getOutputs()); + if (biasAdd) { + rewriter.replaceOp(biasAdd, + trace ? trace->getResult(0) : custom.getResult(0)); + if (biasDequant->use_empty()) + rewriter.eraseOp(biasDequant); + if (biasScale->use_empty()) + rewriter.eraseOp(biasScale); + } + if (rhs != lhs && rhs->use_empty()) + rewriter.eraseOp(rhs); + if (lhs->use_empty()) + rewriter.eraseOp(lhs); + if (quantize->use_empty()) + rewriter.eraseOp(quantize); + } + + SmallVector transposes; + getOperation().walk( + [&](linalg::TransposeOp op) { transposes.push_back(op); }); + for (auto op : transposes) { + auto type = dyn_cast(op.getInput().getType()); + if (!type || type.getRank() != 2 || !type.getElementType().isInteger(8) || + op.getPermutation() != ArrayRef({1, 0})) + continue; + rewriter.setInsertionPoint(op); + auto custom = + CustomOp::create(rewriter, op.getLoc(), op->getResultTypes(), + "transpose_i8", ValueRange{op.getInput()}); + rewriter.replaceOp(op, custom.getOutputs()); + } + pebble::formSpatialOps(getOperation()); + pebble::formQuantizedChains(getOperation()); + + // Form single-consumer Linear chains. The links remain explicit custom + // operations; bank lowering consumes each group as one resident kernel. + SmallVector linears; + int64_t group = 0; + getOperation().walk([&](CustomOp op) { + if (op.getKind() == "linear_i8") + linears.push_back(op); + if (auto id = op->getAttrOfType("custom.chain")) + group = std::max(group, id.getInt() + 1); + }); + for (CustomOp root : linears) { + if (root->hasAttr("custom.chain")) + continue; + SmallVector stages{root}; + SmallVector links; + SmallVector activations; + while (true) { + Value value = stages.back().getResult(0); + SmallVector edge; + bool relu = false; + while (value.hasOneUse()) { + Operation *user = *value.getUsers().begin(); + if (user->getBlock() != root->getBlock()) + break; + if (user->getName().getStringRef() == "buddy_trace.end") { + auto tensor = user->getAttrOfType("tensor"); + if (!tensor || tensor.getValue()) + break; + } else { + auto maximum = dyn_cast(user); + if (relu || !pebble::elementwise(maximum, "arith.maximumf") || + !llvm::is_contained(maximum.getInputs(), value)) + break; + Value other = + maximum.getInputs()[maximum.getInputs()[0] == value ? 1 : 0]; + if (!matchPattern(other, m_AnyZeroFloat())) + break; + relu = true; + } + edge.push_back(user); + value = user->getResult(0); + } + if (!value.hasOneUse()) + break; + auto next = dyn_cast(*value.getUsers().begin()); + if (!next || next.getKind() != "linear_i8" || + next->getBlock() != root->getBlock() || + next.getInputs()[0] != value || next->hasAttr("custom.chain") || + !dominance.properlyDominates(next.getInputs()[3], stages.back())) + break; + bool barrier = false; + for (Operation *between = stages.back()->getNextNode(); between != next; + between = between->getNextNode()) { + if (!isMemoryEffectFree(between) && + between->getName().getDialectNamespace() != "buddy_trace") { + barrier = true; + break; + } + } + if (barrier) + break; + stages.push_back(next); + llvm::append_range(links, edge); + activations.push_back(relu); + } + if (stages.size() < 2) + continue; + auto id = rewriter.getI64IntegerAttr(group++); + for (auto [index, stage] : llvm::enumerate(stages)) { + stage->setAttr("custom.chain", id); + if (index < activations.size()) + stage->setAttr("custom.relu", + rewriter.getBoolAttr(activations[index])); + } + for (Operation *link : links) { + if (auto maximum = dyn_cast(link)) { + rewriter.setInsertionPoint(maximum); + Value input = matchPattern(maximum.getInputs()[0], m_AnyZeroFloat()) + ? maximum.getInputs()[1] + : maximum.getInputs()[0]; + auto activation = CustomOp::create(rewriter, maximum.getLoc(), + maximum->getResultTypes(), "relu", + ValueRange{input}); + activation->setAttr("custom.chain", id); + rewriter.replaceOp(maximum, activation.getOutputs()); + } else { + link->setAttr("custom.chain", id); + } + } + } + } +}; + +PassRegistration registration; +} // namespace diff --git a/examples/designs/pebble/stack/midend/src/FormSpatialOps.cpp b/examples/designs/pebble/stack/midend/src/FormSpatialOps.cpp new file mode 100644 index 000000000..8620fad01 --- /dev/null +++ b/examples/designs/pebble/stack/midend/src/FormSpatialOps.cpp @@ -0,0 +1,276 @@ +//===- FormSpatialOps.cpp - Recognize quantized convolution chains --------===// +#include "Custom/CustomOps.h" +#include "Lowering.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/IR/Dominance.h" +#include "mlir/IR/Matchers.h" +#include "mlir/IR/PatternMatch.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" +#include + +using namespace mlir; +using ::buddy::custom::CustomOp; + +namespace { +Operation *frontend(Value value, StringRef name) { + Operation *op = value.getDefiningOp(); + return op && op->getName().getStringRef() == name ? op : nullptr; +} +} // namespace + +void pebble::formSpatialOps(func::FuncOp function) { + IRRewriter rewriter(function.getContext()); + SmallVector convolutions; + function.walk([&](Operation *op) { + if (isa(op)) + convolutions.push_back(op); + }); + for (Operation *op : convolutions) { + if (!cast(op).hasPureTensorSemantics()) + continue; + bool depthwise = isa(op); + auto transpose = op->getOperand(1).getDefiningOp(); + if (!transpose || transpose.getPermutation() != + (depthwise ? ArrayRef({2, 3, 0, 1}) + : ArrayRef({0, 2, 3, 1}))) + continue; + Value input = op->getOperand(0); + auto pad = input.getDefiningOp(); + SmallVector padding{0, 0, 0, 0}; + if (pad) { + auto low = pad.getStaticLow(), high = pad.getStaticHigh(); + Value padValue = pad.getConstantPaddingValue(); + if (llvm::is_contained(low, ShapedType::kDynamic) || + llvm::is_contained(high, ShapedType::kDynamic) || low[0] || low[3] || + high[0] || high[3] || + (!padValue || !matchPattern(padValue, m_AnyZeroFloat()))) + continue; + padding = {low[1], high[1], low[2], high[2]}; + input = pad.getSource(); + } + auto lhs = frontend(input, "frontend.int8_to_fp32"); + auto rhs = frontend(transpose.getInput(), "frontend.int8_to_fp32"); + if (!lhs || !rhs) + continue; + auto quant = frontend(lhs->getOperand(0), "frontend.fp32_to_int8"); + Operation *end = op; + Value biasValue; + Operation *initial = op->getOperand(2).getDefiningOp(); + tensor::CollapseShapeOp collapsed; + if (depthwise) { + auto type = cast(op->getResult(0).getType()); + Value zero = op->getOperand(2); + if (auto fill = zero.getDefiningOp()) + zero = fill.getInputs()[0]; + if (type.getDimSize(4) != 1 || !matchPattern(zero, m_AnyZeroFloat()) || + !op->getResult(0).hasOneUse()) + continue; + collapsed = dyn_cast( + *op->getResult(0).getUsers().begin()); + if (!collapsed || !collapsed.getResult().hasOneUse() || + collapsed.getReassociationIndices() != + SmallVector{{0}, {1}, {2}, {3, 4}}) + continue; + auto add = dyn_cast( + *collapsed.getResult().getUsers().begin()); + if (!add || !linalg::isaElemwiseSingleBinaryOpInterface(add) || + !isa(add.getBody()->front())) + continue; + int biasIndex = add.getInputs()[0] == collapsed.getResult() ? 1 : 0; + auto maps = add.getIndexingMapsArray(); + if (maps[biasIndex] != + AffineMap::get(4, 0, rewriter.getAffineDimExpr(3)) || + !maps[1 - biasIndex].isIdentity() || !maps[2].isIdentity()) + continue; + biasValue = add.getInputs()[biasIndex]; + end = add; + } else { + auto broadcast = dyn_cast_or_null(initial); + if (!broadcast || broadcast.getInputs().size() != 1 || + broadcast.getBody()->getOperations().size() != 1) + continue; + auto maps = broadcast.getIndexingMapsArray(); + if (maps[0] != AffineMap::get(4, 0, rewriter.getAffineDimExpr(3)) || + !maps[1].isIdentity() || + broadcast.getBody()->getTerminator()->getOperand(0) != + broadcast.getBody()->getArgument(0)) + continue; + biasValue = broadcast.getInputs()[0]; + } + auto bias = frontend(biasValue, "frontend.int32_to_fp32"); + if (!quant || !bias) + continue; + auto reshape = bias->getOperand(1).getDefiningOp(); + auto scale = reshape ? reshape.getSrc().getDefiningOp() + : linalg::GenericOp(); + if (!scale || !pebble::elementwise(scale, "arith.mulf") || + !llvm::is_contained(scale.getInputs(), lhs->getOperand(1)) || + !llvm::is_contained(scale.getInputs(), rhs->getOperand(1))) + continue; + rewriter.setInsertionPoint(end); + auto custom = + CustomOp::create(rewriter, end->getLoc(), end->getResultTypes(), + depthwise ? "depthwise_conv2d_i8" : "conv2d_i8", + ValueRange{quant->getOperand(0), rhs->getOperand(0), + bias->getOperand(0), quant->getOperand(1), + lhs->getOperand(1), rhs->getOperand(1)}); + custom->setAttr("strides", + rewriter.getDenseI64ArrayAttr(llvm::to_vector( + cast(op->getAttr("strides")) + .getValues()))); + custom->setAttr("dilations", + rewriter.getDenseI64ArrayAttr(llvm::to_vector( + cast(op->getAttr("dilations")) + .getValues()))); + custom->setAttr("padding", rewriter.getDenseI64ArrayAttr(padding)); + rewriter.replaceOp(end, custom.getOutputs()); + if (depthwise) { + rewriter.eraseOp(collapsed); + rewriter.eraseOp(op); + } + if (pad && pad->use_empty()) + rewriter.eraseOp(pad); + for (Operation *dead : + {initial, bias, reshape.getOperation(), scale.getOperation(), + transpose.getOperation(), rhs, lhs, quant}) + if (dead->use_empty()) + rewriter.eraseOp(dead); + } + + // A spatial stage retains its ordinary tensor semantics until bank lowering. + // Only single-consumer links ending in another quantized computation fuse. + SmallVector roots; + function.walk([&](CustomOp op) { + if (op.getKind() == "conv2d_i8" || op.getKind() == "depthwise_conv2d_i8") + roots.push_back(op); + }); + DominanceInfo dominance(function); + int64_t group = 0; + for (auto root : roots) { + Value value = root.getResult(0); + SmallVector links; + bool relu = false, pool = false, flatten = false; + while (value.hasOneUse()) { + Operation *user = *value.getUsers().begin(); + if (user->getBlock() != root->getBlock()) + break; + if (user->getName().getStringRef() == "buddy_trace.end") { + if (cast(user->getAttr("tensor")).getValue()) + break; + } else if (auto maximum = dyn_cast(user)) { + if (relu || pool || + !linalg::isaElemwiseSingleBinaryOpInterface(maximum) || + !isa(maximum.getBody()->front()) || + !llvm::all_of(maximum.getIndexingMapsArray(), + [](AffineMap map) { return map.isIdentity(); }) || + !matchPattern( + maximum.getInputs()[maximum.getInputs()[0] == value ? 1 : 0], + m_AnyZeroFloat())) + break; + relu = true; + } else if (auto pooling = dyn_cast(user)) { + if (pool || flatten || + !llvm::all_of(pooling.getDilations().getValues(), + [](int64_t v) { return v == 1; })) + break; + Value initial = pooling.getOutputs()[0]; + if (auto fill = initial.getDefiningOp()) + initial = fill.getInputs()[0]; + APFloat minimum(0.0f); + bool negativeLimit = + matchPattern(initial, m_ConstantFloat(&minimum)) && + minimum.convertToFloat() <= -std::numeric_limits::max(); + if (!negativeLimit && + !(relu && matchPattern(initial, m_AnyZeroFloat()))) + break; + pool = true; + } else if (auto transpose = dyn_cast(user)) { + if (flatten || + transpose.getPermutation() != ArrayRef({0, 3, 1, 2}) || + !transpose->getResult(0).hasOneUse()) + break; + auto reshape = dyn_cast( + *transpose->getResult(0).getUsers().begin()); + if (!reshape || reshape.getReassociationIndices() != + SmallVector{{0}, {1, 2, 3}}) + break; + flatten = true; + links.push_back(transpose); + user = reshape; + } else { + break; + } + links.push_back(user); + value = user->getResult(0); + } + if (!value.hasOneUse()) + continue; + Operation *consumer = *value.getUsers().begin(); + auto next = dyn_cast(consumer); + bool terminalQuant = + consumer->getName().getStringRef() == "frontend.fp32_to_int8"; + if (!terminalQuant && (!next || + (next.getKind() != "conv2d_i8" && + next.getKind() != "depthwise_conv2d_i8" && + next.getKind() != "linear_i8") || + next.getInputs()[0] != value)) + continue; + Value multiplier = + terminalQuant ? consumer->getOperand(1) : next.getInputs()[3]; + if ((terminalQuant && flatten) || + !dominance.properlyDominates(multiplier, root)) + continue; + bool barrier = false; + for (Operation *between = root->getNextNode(); between != consumer; + between = between->getNextNode()) + if (!isMemoryEffectFree(between) && + between->getName().getDialectNamespace() != "buddy_trace") { + barrier = true; + break; + } + if (barrier) + continue; + if (terminalQuant) + links.push_back(consumer); + auto id = rewriter.getI64IntegerAttr(group++); + root->setAttr("custom.spatial_chain", id); + for (Operation *link : links) { + rewriter.setInsertionPoint(link); + if (auto maximum = dyn_cast(link)) { + Value input = matchPattern(maximum.getInputs()[0], m_AnyZeroFloat()) + ? maximum.getInputs()[1] + : maximum.getInputs()[0]; + auto custom = CustomOp::create(rewriter, maximum.getLoc(), + maximum->getResultTypes(), "relu", + ValueRange{input}); + custom->setAttr("custom.spatial_chain", id); + rewriter.replaceOp(maximum, custom.getOutputs()); + } else if (auto pooling = dyn_cast(link)) { + auto custom = CustomOp::create(rewriter, pooling.getLoc(), + pooling->getResultTypes(), "maxpool2d", + ValueRange{pooling.getInputs()[0]}); + custom->setAttr("kernel", rewriter.getDenseI64ArrayAttr( + cast( + pooling.getInputs()[1].getType()) + .getShape())); + custom->setAttr("strides", + rewriter.getDenseI64ArrayAttr(llvm::to_vector( + pooling.getStrides().getValues()))); + custom->setAttr("custom.spatial_chain", id); + rewriter.replaceOp(pooling, custom.getOutputs()); + } else if (auto reshape = dyn_cast(link)) { + auto transpose = reshape.getSrc().getDefiningOp(); + auto custom = CustomOp::create( + rewriter, reshape.getLoc(), reshape->getResultTypes(), + "flatten_nchw", ValueRange{transpose.getInput()}); + custom->setAttr("custom.spatial_chain", id); + rewriter.replaceOp(reshape, custom.getOutputs()); + rewriter.eraseOp(transpose); + } else if (!isa(link)) { + link->setAttr("custom.spatial_chain", id); + } + } + } +} diff --git a/examples/designs/pebble/stack/midend/src/LowerLinearTiles.cpp b/examples/designs/pebble/stack/midend/src/LowerLinearTiles.cpp new file mode 100644 index 000000000..64fc2a16a --- /dev/null +++ b/examples/designs/pebble/stack/midend/src/LowerLinearTiles.cpp @@ -0,0 +1,240 @@ +//===- LowerLinearTiles.cpp - Linear rows and reduction/output tails ------===// +#include "Buckyball/BuckyballOps.h" +#include "Custom/CustomOps.h" +#include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult pebble::lowerLinearTiles(::buddy::custom::CustomOp op, + Value &result) { + bool integer = op.getKind() == "matmul_i8"; + bool requant = op.getKind() == "linear_i8_requant"; + auto args = op.getInputs(); + auto a = cast(args[0].getType()); + auto w = cast(args[1].getType()); + auto biasType = cast(args[2].getType()); + auto resultType = cast(op.getResult(0).getType()); + if (!a.hasStaticShape() || !w.hasStaticShape() || a.getRank() != 2 || + w.getRank() != 2 || a.getDimSize(1) != w.getDimSize(0) || + !((requant || integer) ? a.getElementType().isInteger(8) + : a.getElementType().isF32()) || + !w.getElementType().isInteger(8) || + !(integer ? resultType.getElementType().isInteger(32) + : requant ? resultType.getElementType().isInteger(8) + : resultType.getElementType().isF32()) || + !biasType.getElementType().isInteger(32) || + biasType.getShape() != + ArrayRef({integer ? a.getDimSize(0) : 1, w.getDimSize(1)}) || + resultType.getShape() != + ArrayRef({a.getDimSize(0), w.getDimSize(1)})) + return op.emitOpError( + "requires matching MxK/KxN tensors and INT32 1xN bias"); + int64_t m = a.getDimSize(0), k = a.getDimSize(1), n = w.getDimSize(1); + const auto &target = buckyball_target::getBuckyballTarget(); + int64_t rows = integer ? 1 + : buckyball_target::getBuckyballBallParam("SMATMUL_OS", + "tileRows"); + int64_t maxK = + std::min(target.bankDepth, + target.bankDepth * ((requant || integer) ? 16 : 4) / rows) / + 16 * 16; + if (m <= 0 || k <= 0 || n <= 0 || target.bankWidthBits != 128 || maxK == 0 || + rows * 4 > target.bankDepth) + return op.emitOpError("Linear tile does not fit the configured banks"); + OpBuilder b(op); + Location loc = op.getLoc(); + auto index = [&](int64_t n) -> Value { + return arith::ConstantIndexOp::create(b, loc, n); + }; + Value zero = index(0), one = index(1); + Type i64 = b.getI64Type(); + Value input = tensorBuffer(b, loc, args[0]); + Value weight = tensorBuffer(b, loc, args[1]); + Value bias = tensorBuffer(b, loc, args[2]); + Value step; + if (!integer) + step = tensorBuffer(b, loc, args[requant ? 3 : 5]); + Value multiplier, inputStep; + if (!requant && !integer) { + multiplier = + tensor::ExtractOp::create(b, loc, args[3], ValueRange{zero, zero}); + inputStep = + tensor::ExtractOp::create(b, loc, args[4], ValueRange{zero, zero}); + } + Value output = memref::AllocOp::create( + b, loc, MemRefType::get({m, n}, resultType.getElementType())); + Value resultPack = memref::AllocOp::create( + b, loc, MemRefType::get({rows, 16}, resultType.getElementType())); + for (int64_t col = 0; col < n; col += 16) { + Value biasPack = + memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getI32Type())); + Value scalePack; + if (!integer) + scalePack = memref::AllocOp::create( + b, loc, MemRefType::get({16}, b.getF32Type())); + Value biasZero = arith::ConstantIntOp::create(b, loc, 0, 32); + linalg::FillOp::create(b, loc, biasZero, biasPack); + Value scales; + if (!integer) { + Value scaleOne = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(1)); + linalg::FillOp::create(b, loc, scaleOne, scalePack); + auto lanes = scf::ForOp::create( + b, loc, zero, index(std::min(16, n - col)), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value channel = arith::AddIOp::create(b, loc, index(col), lane); + Value value = + memref::LoadOp::create(b, loc, bias, ValueRange{zero, channel}); + memref::StoreOp::create(b, loc, value, biasPack, lane); + Value scale = + memref::LoadOp::create(b, loc, step, ValueRange{zero, channel}); + if (!requant) + scale = arith::MulFOp::create(b, loc, inputStep, scale); + memref::StoreOp::create(b, loc, scale, scalePack, lane); + } + scales = BankAllocOp::create(b, loc, i64, 1, 1); + scales = BankMvinOp::create(b, loc, i64, scalePack, scales, 4, 1); + } + for (int64_t row = 0; row < m; row += rows) { + if (integer) { + auto lanes = scf::ForOp::create( + b, loc, zero, index(std::min(16, n - col)), one); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value channel = arith::AddIOp::create(b, loc, index(col), lane); + Value value = memref::LoadOp::create(b, loc, bias, + ValueRange{index(row), channel}); + memref::StoreOp::create(b, loc, value, biasPack, lane); + } + Value accumulator = BankAllocOp::create(b, loc, i64, 1, 1); + accumulator = + BankMvinOp::create(b, loc, i64, biasPack, accumulator, 4, 1); + for (int64_t start = 0; start < k; start += maxK) { + int64_t valid = std::min(maxK, k - start), + reduction = (valid + 15) / 16 * 16; + Value inputPack = memref::AllocOp::create( + b, loc, + MemRefType::get({reduction / 16 * rows, 16}, a.getElementType())); + Value weightPack = memref::AllocOp::create( + b, loc, MemRefType::get({reduction, 16}, b.getI8Type())); + Value fzero = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(0)); + Value izero = arith::ConstantIntOp::create(b, loc, 0, 8); + linalg::FillOp::create(b, loc, (requant || integer) ? izero : fzero, + inputPack); + linalg::FillOp::create(b, loc, izero, weightPack); + auto inner = scf::ForOp::create(b, loc, zero, index(valid), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(inner.getBody()); + Value reductionIndex = inner.getInductionVar(); + Value sourceK = + arith::AddIOp::create(b, loc, reductionIndex, index(start)); + Value block = arith::MulIOp::create( + b, loc, arith::DivUIOp::create(b, loc, reductionIndex, index(16)), + index(rows)); + Value lane = + arith::RemUIOp::create(b, loc, reductionIndex, index(16)); + auto inputRows = scf::ForOp::create( + b, loc, zero, index(std::min(rows, m - row)), one); + { + OpBuilder::InsertionGuard rowGuard(b); + b.setInsertionPointToStart(inputRows.getBody()); + Value r = inputRows.getInductionVar(); + Value sourceM = arith::AddIOp::create(b, loc, r, index(row)); + Value destination = arith::AddIOp::create(b, loc, block, r); + Value value = memref::LoadOp::create(b, loc, input, + ValueRange{sourceM, sourceK}); + memref::StoreOp::create(b, loc, value, inputPack, + ValueRange{destination, lane}); + } + auto columns = scf::ForOp::create( + b, loc, zero, index(std::min(16, n - col)), one); + b.setInsertionPointToStart(columns.getBody()); + Value c = columns.getInductionVar(); + Value sourceN = arith::AddIOp::create(b, loc, c, index(col)); + Value value = memref::LoadOp::create(b, loc, weight, + ValueRange{sourceK, sourceN}); + memref::StoreOp::create(b, loc, value, weightPack, + ValueRange{reductionIndex, c}); + } + Value codes = BankAllocOp::create(b, loc, i64, 1, 1); + if (requant || integer) { + codes = BankMvinOp::create(b, loc, i64, inputPack, codes, + rows * reduction / 16, 1); + } else { + Value fp = BankAllocOp::create(b, loc, i64, 1, 1); + fp = BankMvinOp::create(b, loc, i64, inputPack, fp, + rows * reduction / 4, 1); + auto quant = + BankCustomOp::create(b, loc, i64, ValueRange{fp}, codes, + ValueRange{multiplier}, "fp32_to_int8"); + quant->setAttr("rows", b.getI64IntegerAttr(rows * reduction / 4)); + codes = quant.getDestinationOut(); + BankReleaseOp::create(b, loc, fp); + } + Value weights = BankAllocOp::create(b, loc, i64, 1, 1); + weights = + BankMvinOp::create(b, loc, i64, weightPack, weights, reduction, 1); + auto matmul = BankCustomOp::create( + b, loc, i64, ValueRange{codes, weights}, accumulator, ValueRange{}, + "matmul_i8_block", true); + matmul->setAttr("rows", b.getI64IntegerAttr(rows)); + matmul->setAttr("k", b.getI64IntegerAttr(reduction)); + matmul->setAttr("first", b.getBoolAttr(start == 0)); + matmul->setAttr("last", b.getBoolAttr(start + valid == k)); + accumulator = matmul.getDestinationOut(); + BankReleaseOp::create(b, loc, codes); + BankReleaseOp::create(b, loc, weights); + } + Value restored = accumulator; + if (!integer) { + restored = BankAllocOp::create(b, loc, i64, 1, 1); + auto dequant = BankCustomOp::create( + b, loc, i64, ValueRange{accumulator, scales}, restored, + ValueRange{}, requant ? "int32_to_int8" : "int32_to_fp32"); + dequant->setAttr("rows", b.getI64IntegerAttr(rows)); + if (requant) { + dequant->setAttr("wr_base", b.getI64IntegerAttr(0)); + dequant->setAttr("relu", op->getAttr("relu")); + } + restored = dequant.getDestinationOut(); + } + BankMvoutOp::create(b, loc, i64, resultPack, restored, + rows * (requant ? 1 : 4), 1); + BankReleaseOp::create(b, loc, accumulator); + if (!integer) + BankReleaseOp::create(b, loc, restored); + auto outputRows = + scf::ForOp::create(b, loc, zero, index(std::min(rows, m - row)), one); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(outputRows.getBody()); + Value r = outputRows.getInductionVar(); + auto columns = scf::ForOp::create( + b, loc, zero, index(std::min(16, n - col)), one); + b.setInsertionPointToStart(columns.getBody()); + Value c = columns.getInductionVar(); + Value value = + memref::LoadOp::create(b, loc, resultPack, ValueRange{r, c}); + memref::StoreOp::create( + b, loc, value, output, + ValueRange{arith::AddIOp::create(b, loc, r, index(row)), + arith::AddIOp::create(b, loc, c, index(col))}); + } + if (!integer) + BankReleaseOp::create(b, loc, scales); + } + result = + bufferization::ToTensorOp::create(b, loc, resultType, output, true, true); + return success(); +} diff --git a/examples/designs/pebble/stack/midend/src/LowerQuantizedOps.cpp b/examples/designs/pebble/stack/midend/src/LowerQuantizedOps.cpp new file mode 100644 index 000000000..4135be143 --- /dev/null +++ b/examples/designs/pebble/stack/midend/src/LowerQuantizedOps.cpp @@ -0,0 +1,326 @@ +//===- LowerQuantizedOps.cpp - INT8 reductions and channel operations -----===// +#include "Buckyball/BuckyballOps.h" +#include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult pebble::lowerGlobalAverage(::buddy::custom::CustomOp op, + Value &result, bool resident) { + auto type = cast(op.getInputs()[0].getType()); + auto resultType = cast(op.getResult(0).getType()); + if (!type.hasStaticShape() || type.getRank() != 4 || + type.getDimSize(0) != 1 || !type.getElementType().isInteger(8) || + !resultType.getElementType().isInteger(8) || + resultType.getShape() != ArrayRef({1, 1, 1, type.getDimSize(3)})) + return op.emitOpError( + "global_avg_i8 requires NHWC INT8 input and 1x1x1xC INT8 output"); + int64_t height = type.getDimSize(1), width = type.getDimSize(2), + channels = type.getDimSize(3); + int64_t pixels = height * width; + int64_t capacity = buckyball_target::getBuckyballTarget().bankDepth / 16 * 16; + if (pixels <= 0 || channels <= 0 || capacity == 0) + return op.emitOpError("global average does not fit the configured banks"); + OpBuilder b(op); + Location loc = op.getLoc(); + auto index = [&](int64_t n) -> Value { + return arith::ConstantIndexOp::create(b, loc, n); + }; + Value zero = index(0), one = index(1); + Type i64 = b.getI64Type(); + float inputScale = + cast(op->getAttr("input_scale")).getValueAsDouble(); + float outputScale = + cast(op->getAttr("output_scale")).getValueAsDouble(); + float ratio = inputScale / (pixels * outputScale); + Value input = tensorBuffer(b, loc, op.getInputs()[0]); + Value output, residentOutput; + if (resident) + residentOutput = BankAllocOp::create(b, loc, i64, 1, 1); + else + output = memref::AllocOp::create( + b, loc, MemRefType::get(resultType.getShape(), b.getI8Type())); + Value bias = + memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getI32Type())); + Value scales = + memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getF32Type())); + Value values = + memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getI8Type())); + Value izero = arith::ConstantIntOp::create(b, loc, 0, 8); + Value biasZero = arith::ConstantIntOp::create(b, loc, 0, 32); + Value iOne = arith::ConstantIntOp::create(b, loc, 1, 8); + Value scale = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(ratio)); + linalg::FillOp::create(b, loc, biasZero, bias); + linalg::FillOp::create(b, loc, scale, scales); + Value scaleBank = BankAllocOp::create(b, loc, i64, 1, 1); + scaleBank = BankMvinOp::create(b, loc, i64, scales, scaleBank, 4, 1); + for (int64_t c = 0; c < channels; c += 16) { + Value acc = BankAllocOp::create(b, loc, i64, 1, 1); + acc = BankMvinOp::create(b, loc, i64, bias, acc, 4, 1); + for (int64_t first = 0; first < pixels; first += capacity) { + int64_t valid = std::min(capacity, pixels - first), + k = (valid + 15) / 16 * 16; + Value pack = memref::AllocOp::create( + b, loc, MemRefType::get({k, 16}, b.getI8Type())); + Value ones = + memref::AllocOp::create(b, loc, MemRefType::get({k}, b.getI8Type())); + linalg::FillOp::create(b, loc, izero, pack); + linalg::FillOp::create(b, loc, iOne, ones); + auto spatial = scf::ForOp::create(b, loc, zero, index(valid), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(spatial.getBody()); + Value p = spatial.getInductionVar(); + Value source = arith::AddIOp::create(b, loc, p, index(first)); + Value y = arith::DivUIOp::create(b, loc, source, index(width)); + Value x = arith::RemUIOp::create(b, loc, source, index(width)); + auto lanes = scf::ForOp::create( + b, loc, zero, index(std::min(16, channels - c)), one); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value channel = arith::AddIOp::create(b, loc, lane, index(c)); + Value value = memref::LoadOp::create(b, loc, input, + ValueRange{zero, y, x, channel}); + memref::StoreOp::create(b, loc, value, pack, ValueRange{p, lane}); + } + Value a = BankAllocOp::create(b, loc, i64, 1, 1); + a = BankMvinOp::create(b, loc, i64, ones, a, k / 16, 1); + Value w = BankAllocOp::create(b, loc, i64, 1, 1); + w = BankMvinOp::create(b, loc, i64, pack, w, k, 1); + auto sum = BankCustomOp::create(b, loc, i64, ValueRange{a, w}, acc, + ValueRange{}, "matmul_i8_block", true); + sum->setAttr("rows", b.getI64IntegerAttr(1)); + sum->setAttr("k", b.getI64IntegerAttr(k)); + sum->setAttr("first", b.getBoolAttr(first == 0)); + sum->setAttr("last", b.getBoolAttr(first + valid == pixels)); + acc = sum.getDestinationOut(); + BankReleaseOp::create(b, loc, a); + BankReleaseOp::create(b, loc, w); + } + Value codes = residentOutput; + if (!resident) + codes = BankAllocOp::create(b, loc, i64, 1, 1); + auto quant = + BankCustomOp::create(b, loc, i64, ValueRange{acc, scaleBank}, codes, + ValueRange{}, "int32_to_int8", resident); + quant->setAttr("rows", b.getI64IntegerAttr(1)); + quant->setAttr("wr_base", b.getI64IntegerAttr(resident ? c / 16 : 0)); + quant->setAttr("relu", b.getBoolAttr(false)); + codes = quant.getDestinationOut(); + BankReleaseOp::create(b, loc, acc); + if (resident) { + residentOutput = codes; + continue; + } + BankMvoutOp::create(b, loc, i64, values, codes, 1, 1); + BankReleaseOp::create(b, loc, codes); + auto lanes = scf::ForOp::create( + b, loc, zero, index(std::min(16, channels - c)), one); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value value = memref::LoadOp::create(b, loc, values, lane); + memref::StoreOp::create( + b, loc, value, output, + ValueRange{zero, zero, zero, + arith::AddIOp::create(b, loc, lane, index(c))}); + } + BankReleaseOp::create(b, loc, scaleBank); + result = residentOutput; + if (!resident) + result = bufferization::ToTensorOp::create(b, loc, resultType, output, true, + true); + return success(); +} + +LogicalResult pebble::lowerChannelOp(::buddy::custom::CustomOp op, + Value &result) { + bool lookup = op.getKind() == "lookup_i8"; + auto type = cast(op.getInputs()[lookup ? 0 : 1].getType()); + auto otherType = + cast(op.getInputs()[lookup ? 1 : 0].getType()); + if (!type.hasStaticShape() || type.getRank() != 2 || + !type.getElementType().isInteger(8) || + !otherType.getElementType().isInteger(8) || + op.getResult(0).getType() != type || + (lookup ? (otherType.getRank() != 1 || (otherType.getDimSize(0) != 256 && + otherType.getDimSize(0) != 4096)) + : otherType.getShape() != + ArrayRef({1, type.getDimSize(1)}))) + return op.emitOpError( + "requires an INT8 matrix and an INT8 table or per-channel gate"); + int64_t rows = type.getDimSize(0), channels = type.getDimSize(1); + int64_t capacity = buckyball_target::getBuckyballTarget().bankDepth; + OpBuilder b(op); + Location loc = op.getLoc(); + auto index = [&](int64_t n) -> Value { + return arith::ConstantIndexOp::create(b, loc, n); + }; + Value zero = index(0), one = index(1); + Type i64 = b.getI64Type(); + Value input = tensorBuffer(b, loc, op.getInputs()[lookup ? 0 : 1]); + Value other = tensorBuffer(b, loc, op.getInputs()[lookup ? 1 : 0]); + Value output = memref::AllocOp::create( + b, loc, MemRefType::get(type.getShape(), b.getI8Type())); + Value izero = arith::ConstantIntOp::create(b, loc, 0, 8); + Value tableBank, ratio; + if (lookup) { + int64_t columns = otherType.getDimSize(0) == 256 ? 1 : 4; + if (columns == 4) { + Value packed = memref::AllocOp::create( + b, loc, MemRefType::get({64, 64}, b.getI8Type())); + auto rows = scf::ForOp::create(b, loc, zero, index(64), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(rows.getBody()); + Value row = rows.getInductionVar(); + auto cols = scf::ForOp::create(b, loc, zero, index(64), one); + b.setInsertionPointToStart(cols.getBody()); + Value col = cols.getInductionVar(); + Value channel = arith::AddIOp::create( + b, loc, + arith::MulIOp::create( + b, loc, arith::DivUIOp::create(b, loc, col, index(16)), + index(4)), + arith::DivUIOp::create(b, loc, row, index(16))); + Value entry = arith::AddIOp::create( + b, loc, + arith::MulIOp::create( + b, loc, arith::RemUIOp::create(b, loc, row, index(16)), + index(16)), + arith::RemUIOp::create(b, loc, col, index(16))); + Value offset = arith::AddIOp::create( + b, loc, arith::MulIOp::create(b, loc, channel, index(256)), entry); + Value value = memref::LoadOp::create(b, loc, other, offset); + memref::StoreOp::create(b, loc, value, packed, ValueRange{row, col}); + } + other = packed; + } + tableBank = BankAllocOp::create(b, loc, i64, 1, columns); + tableBank = BankMvinOp::create(b, loc, i64, other, tableBank, + otherType.getDimSize(0) / columns / 16, 1); + } else { + ratio = arith::ConstantOp::create(b, loc, + cast(op->getAttr("ratio"))); + } + for (int64_t c = 0; c < channels; c += 16) { + Value gateBank; + if (!lookup) { + Value gate = + memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getI8Type())); + linalg::FillOp::create(b, loc, izero, gate); + auto lanes = scf::ForOp::create( + b, loc, zero, index(std::min(16, channels - c)), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value value = memref::LoadOp::create( + b, loc, other, + ValueRange{zero, arith::AddIOp::create(b, loc, lane, index(c))}); + memref::StoreOp::create(b, loc, value, gate, lane); + } + gateBank = BankAllocOp::create(b, loc, i64, 1, 1); + gateBank = BankMvinOp::create(b, loc, i64, gate, gateBank, 1, 1); + } + for (int64_t start = 0; start < rows; start += capacity) { + int64_t count = std::min(capacity, rows - start); + Value pack = memref::AllocOp::create( + b, loc, MemRefType::get({count, 16}, b.getI8Type())); + linalg::FillOp::create(b, loc, izero, pack); + auto spatial = scf::ForOp::create(b, loc, zero, index(count), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(spatial.getBody()); + Value r = spatial.getInductionVar(); + auto lanes = scf::ForOp::create( + b, loc, zero, index(std::min(16, channels - c)), one); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value value = memref::LoadOp::create( + b, loc, input, + ValueRange{arith::AddIOp::create(b, loc, r, index(start)), + arith::AddIOp::create(b, loc, lane, index(c))}); + memref::StoreOp::create(b, loc, value, pack, ValueRange{r, lane}); + } + Value source = BankAllocOp::create(b, loc, i64, 1, 1); + source = BankMvinOp::create(b, loc, i64, pack, source, count, 1); + Value destination = BankAllocOp::create(b, loc, i64, 1, 1); + auto operation = BankCustomOp::create( + b, loc, i64, + lookup ? ValueRange{source, tableBank} : ValueRange{gateBank, source}, + destination, lookup ? ValueRange{} : ValueRange{ratio}, + lookup ? "lookup_i8" : "gate_mul_i8"); + operation->setAttr("rows", b.getI64IntegerAttr(count)); + if (!lookup) + operation->setAttr("gate_row", b.getI64IntegerAttr(0)); + destination = operation.getDestinationOut(); + BankMvoutOp::create(b, loc, i64, pack, destination, count, 1); + BankReleaseOp::create(b, loc, source); + BankReleaseOp::create(b, loc, destination); + auto outputRows = scf::ForOp::create(b, loc, zero, index(count), one); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(outputRows.getBody()); + Value r = outputRows.getInductionVar(); + auto lanes = scf::ForOp::create( + b, loc, zero, index(std::min(16, channels - c)), one); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value value = memref::LoadOp::create(b, loc, pack, ValueRange{r, lane}); + memref::StoreOp::create( + b, loc, value, output, + ValueRange{arith::AddIOp::create(b, loc, r, index(start)), + arith::AddIOp::create(b, loc, lane, index(c))}); + } + if (!lookup) + BankReleaseOp::create(b, loc, gateBank); + } + if (lookup) + BankReleaseOp::create(b, loc, tableBank); + result = bufferization::ToTensorOp::create(b, loc, type, output, true, true); + return success(); +} + +LogicalResult pebble::lowerTranspose(::buddy::custom::CustomOp op, + Value &result) { + auto type = cast(op.getInputs()[0].getType()); + auto outputType = cast(op.getResult(0).getType()); + const auto &target = buckyball_target::getBuckyballTarget(); + if (!type.hasStaticShape() || type.getRank() != 2 || + !type.getElementType().isInteger(8) || + outputType.getShape() != + ArrayRef({type.getDimSize(1), type.getDimSize(0)}) || + type.getDimSize(1) % 16 || type.getDimSize(0) > target.bankDepth) + return op.emitOpError( + "transpose_i8 requires a static matrix fitting complete bank columns"); + OpBuilder b(op); + Location loc = op.getLoc(); + Value input = tensorBuffer(b, loc, op.getInputs()[0]); + Value output = memref::AllocOp::create( + b, loc, MemRefType::get(outputType.getShape(), b.getI8Type())); + Value source = + BankAllocOp::create(b, loc, b.getI64Type(), 1, type.getDimSize(1) / 16); + Value destination = + BankAllocOp::create(b, loc, b.getI64Type(), 1, type.getDimSize(1) / 16); + source = BankMvinOp::create(b, loc, b.getI64Type(), input, source, + type.getDimSize(0), 1); + auto transpose = + BankCustomOp::create(b, loc, b.getI64Type(), ValueRange{source}, + destination, ValueRange{}, "transpose_i8"); + transpose->setAttr("rows", b.getI64IntegerAttr(type.getDimSize(0))); + transpose->setAttr("cols", b.getI64IntegerAttr(type.getDimSize(1))); + destination = transpose.getDestinationOut(); + BankMvoutOp::create(b, loc, b.getI64Type(), output, destination, + type.getDimSize(0), 1); + BankReleaseOp::create(b, loc, source); + BankReleaseOp::create(b, loc, destination); + result = + bufferization::ToTensorOp::create(b, loc, outputType, output, true, true); + return success(); +} diff --git a/examples/designs/pebble/stack/midend/src/LowerSpatialOps.cpp b/examples/designs/pebble/stack/midend/src/LowerSpatialOps.cpp new file mode 100644 index 000000000..553fadc47 --- /dev/null +++ b/examples/designs/pebble/stack/midend/src/LowerSpatialOps.cpp @@ -0,0 +1,525 @@ +//===- LowerSpatialOps.cpp - Convolution and pooling on resident banks ----===// +#include "Buckyball/BuckyballOps.h" +#include "Custom/CustomOps.h" +#include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using Computation = ::buddy::custom::CustomOp; + +namespace { +// Each spatial panel holds sixteen channels per pixel, with at most 64 pixels. +// MAXPOOL can address every panel origin with its six-bit bank row fields. +struct ImageBanks { + SmallVector banks; + int64_t height, width, channels, side; +}; + +BankCustomOp +compute(OpBuilder &b, Location loc, StringRef kind, ValueRange sources, + Value destination, bool read, + std::initializer_list> attrs) { + auto op = BankCustomOp::create(b, loc, b.getI64Type(), sources, destination, + ValueRange{}, kind, read); + for (auto [name, value] : attrs) + op->setAttr(name, b.getI64IntegerAttr(value)); + return op; +} + +LogicalResult convolution(Computation op, const ImageBanks *inputImage, + Value nextMultiplier, int64_t pool, bool relu, + ImageBanks &outputImage, Value &tensorOutput) { + bool depthwise = op.getKind() == "depthwise_conv2d_i8"; + auto args = op.getInputs(); + auto inputType = cast(args[0].getType()); + auto weightType = cast(args[1].getType()); + auto outputType = cast(op.getResult(0).getType()); + auto strides = cast(op->getAttr("strides")).asArrayRef(); + auto dilation = + cast(op->getAttr("dilations")).asArrayRef(); + auto padding = cast(op->getAttr("padding")).asArrayRef(); + if (args.size() != 6 || !inputType.hasStaticShape() || + inputType.getRank() != 4 || !weightType.hasStaticShape() || + weightType.getRank() != 4 || inputType.getDimSize(0) != 1 || + !inputType.getElementType().isF32() || + !weightType.getElementType().isInteger(8) || + !cast(args[2].getType()) + .getElementType() + .isInteger(32) || + strides[0] != strides[1] || dilation != ArrayRef({1, 1})) + return op.emitOpError("requires static NHWC/FCHW convolution with INT32 " + "bias and unit dilation"); + int64_t cin = inputType.getDimSize(3), cout = weightType.getDimSize(0); + int64_t kernel = weightType.getDimSize(2), stride = strides[0]; + int64_t height = outputType.getDimSize(1) / pool; + int64_t width = outputType.getDimSize(2) / pool; + if (kernel != weightType.getDimSize(3) || + (depthwise ? (weightType.getDimSize(1) != 1 || cout != cin) + : cin != weightType.getDimSize(1)) || + outputType.getDimSize(1) % pool || outputType.getDimSize(2) % pool || + kernel > buckyball_target::getBuckyballBallParam("IM2COL", "maxKSize")) + return op.emitOpError("has incompatible convolution or pooling geometry"); + int64_t tile = 4 / pool; + while (tile > 0 && (height % tile || width % tile || + ((tile * pool - 1) * stride + kernel) > 8)) + --tile; + if (!tile) + return op.emitOpError("convolution window exceeds addressable bank rows"); + int64_t convSide = tile * pool; + int64_t inputSide = (convSide - 1) * stride + kernel; + int64_t arrayRows = + buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); + int64_t rows = (convSide * convSide + arrayRows - 1) / arrayRows * arrayRows; + int64_t reduction = (kernel * kernel + 15) / 16 * 16; + int64_t side = std::min(8, std::min(height, width)); + while (side % tile) + --side; + int64_t panelsY = (height + side - 1) / side; + int64_t panelsX = (width + side - 1) / side; + const auto &target = buckyball_target::getBuckyballTarget(); + int64_t outputBanks = + nextMultiplier ? panelsY * panelsX * ((cout + 15) / 16) : 0; + int64_t inputBanks = inputImage ? inputImage->banks.size() : 0; + if (target.bankWidthBits != 128 || + inputSide * inputSide * 4 > target.bankDepth || + rows * 4 > target.bankDepth || reduction > target.bankDepth || + inputBanks + outputBanks + 7 > target.bankNum) + return op.emitOpError("spatial chain exceeds the configured bank capacity"); + + OpBuilder b(op); + Location loc = op.getLoc(); + Type i64 = b.getI64Type(); + auto index = [&](int64_t n) -> Value { + return arith::ConstantIndexOp::create(b, loc, n); + }; + Value zero = index(0), one = index(1); + Value fzero = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(0)); + Value izero = arith::ConstantIntOp::create(b, loc, 0, 8); + Value input; + if (!inputImage) + input = pebble::tensorBuffer(b, loc, args[0]); + Value weight = pebble::tensorBuffer(b, loc, args[1]); + Value bias = pebble::tensorBuffer(b, loc, args[2]); + Value weightStep = pebble::tensorBuffer(b, loc, args[5]); + Value step = tensor::ExtractOp::create(b, loc, args[4], + ValueRange{zero, zero, zero, zero}); + Value multiplier; + if (nextMultiplier) { + SmallVector indices( + cast(nextMultiplier.getType()).getRank(), zero); + multiplier = tensor::ExtractOp::create(b, loc, nextMultiplier, indices); + } + outputImage = {{}, height, width, cout, side}; + for (int64_t i = 0; i < outputBanks; ++i) + outputImage.banks.push_back(BankAllocOp::create(b, loc, i64, 1, 1)); + Value output; + if (!nextMultiplier) + output = memref::AllocOp::create( + b, loc, MemRefType::get(outputType.getShape(), b.getF32Type())); + Value zeroPanel = memref::AllocOp::create( + b, loc, MemRefType::get({inputSide * inputSide, 16}, b.getI8Type())); + linalg::FillOp::create(b, loc, izero, zeroPanel); + Value floatPanel; + if (!inputImage) + floatPanel = memref::AllocOp::create( + b, loc, MemRefType::get({inputSide * inputSide, 16}, b.getF32Type())); + Value outputPack; + if (!nextMultiplier) + outputPack = memref::AllocOp::create( + b, loc, MemRefType::get({rows, 16}, b.getF32Type())); + + for (int64_t oc = 0; oc < cout; oc += 16) { + int64_t terms = depthwise ? std::min(16, cout - oc) : cin; + Value weightPack = memref::AllocOp::create( + b, loc, MemRefType::get({terms, reduction, 16}, b.getI8Type())); + linalg::FillOp::create(b, loc, izero, weightPack); + Value biasPack = + memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getI32Type())); + Value scalePack = + memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getF32Type())); + linalg::FillOp::create( + b, loc, ValueRange{arith::ConstantIntOp::create(b, loc, 0, 32)}, + biasPack); + linalg::FillOp::create( + b, loc, + ValueRange{arith::ConstantOp::create(b, loc, b.getF32FloatAttr(1))}, + scalePack); + for (int64_t lane = 0; lane < std::min(16, cout - oc); ++lane) { + Value channel = index(oc + lane), laneIndex = index(lane); + Value biasValue = memref::LoadOp::create(b, loc, bias, channel); + memref::StoreOp::create(b, loc, biasValue, biasPack, laneIndex); + Value weightScale = memref::LoadOp::create( + b, loc, weightStep, ValueRange{channel, zero, zero, zero}); + Value scale = arith::MulFOp::create(b, loc, step, weightScale); + if (nextMultiplier) + scale = arith::MulFOp::create(b, loc, scale, multiplier); + memref::StoreOp::create(b, loc, scale, scalePack, laneIndex); + auto cLoop = scf::ForOp::create(b, loc, index(depthwise ? lane : 0), + index(depthwise ? lane + 1 : cin), one); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(cLoop.getBody()); + Value c = cLoop.getInductionVar(); + auto kLoop = + scf::ForOp::create(b, loc, zero, index(kernel * kernel), one); + b.setInsertionPointToStart(kLoop.getBody()); + Value k = kLoop.getInductionVar(); + Value ky = arith::DivUIOp::create(b, loc, k, index(kernel)); + Value kx = arith::RemUIOp::create(b, loc, k, index(kernel)); + Value w = memref::LoadOp::create( + b, loc, weight, ValueRange{channel, depthwise ? zero : c, ky, kx}); + memref::StoreOp::create(b, loc, w, weightPack, + ValueRange{c, k, laneIndex}); + } + Value scales = BankAllocOp::create(b, loc, i64, 1, 1); + scales = BankMvinOp::create(b, loc, i64, scalePack, scales, 4, 1); + for (int64_t y = 0; y < height; y += tile) { + for (int64_t x = 0; x < width; x += tile) { + Value acc = BankAllocOp::create(b, loc, i64, 1, 1); + acc = BankMvinOp::create(b, loc, i64, biasPack, acc, 4, 1); + Value source; + for (int64_t term = 0; term < terms; ++term) { + int64_t ic = depthwise ? oc + term : term; + if (ic % 16 == 0) { + source = BankAllocOp::create(b, loc, i64, 1, 1); + if (inputImage) { + source = BankMvinOp::create(b, loc, i64, zeroPanel, source, + inputSide * inputSide, 1); + int64_t inPanelsY = (inputImage->height + inputImage->side - 1) / + inputImage->side; + int64_t inPanelsX = + (inputImage->width + inputImage->side - 1) / inputImage->side; + for (int64_t iy = 0; iy < inputSide; ++iy) + for (int64_t ix = 0; ix < inputSide; ++ix) { + int64_t gy = y * pool * stride + iy - padding[0], + gx = x * pool * stride + ix - padding[2]; + if (gy < 0 || gx < 0 || gy >= inputImage->height || + gx >= inputImage->width) + continue; + int64_t panel = + (ic / 16 * inPanelsY + gy / inputImage->side) * + inPanelsX + + gx / inputImage->side; + int64_t base = gy % inputImage->side * inputImage->side + + gx % inputImage->side; + source = compute(b, loc, "maxpool_i8", + ValueRange{inputImage->banks[panel]}, source, + true, + {{"input_side", 1}, + {"output_side", 1}, + {"kernel", 1}, + {"stride", 1}, + {"padding", 0}, + {"op1_base", base}, + {"wr_base", iy * inputSide + ix}, + {"output_stride", 1}, + {"start_row", 0}, + {"start_col", 0}}) + .getDestinationOut(); + } + } else { + linalg::FillOp::create(b, loc, fzero, floatPanel); + auto pixels = scf::ForOp::create( + b, loc, zero, index(inputSide * inputSide), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(pixels.getBody()); + Value pixel = pixels.getInductionVar(); + Value iy = arith::AddIOp::create( + b, loc, + arith::DivUIOp::create(b, loc, pixel, index(inputSide)), + index(y * pool * stride - padding[0])); + Value ix = arith::AddIOp::create( + b, loc, + arith::RemUIOp::create(b, loc, pixel, index(inputSide)), + index(x * pool * stride - padding[2])); + auto channels = scf::ForOp::create( + b, loc, zero, index(std::min(16, cin - ic)), one); + b.setInsertionPointToStart(channels.getBody()); + Value lane = channels.getInductionVar(); + Value channel = arith::AddIOp::create(b, loc, lane, index(ic)); + Value inBounds = arith::AndIOp::create( + b, loc, + arith::AndIOp::create( + b, loc, + arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sge, + iy, zero), + arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt, + iy, + index(inputType.getDimSize(1)))), + arith::AndIOp::create( + b, loc, + arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sge, + ix, zero), + arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt, + ix, + index(inputType.getDimSize(2))))); + auto valid = scf::IfOp::create(b, loc, inBounds, false); + b.setInsertionPointToStart(&valid.getThenRegion().front()); + Value value = memref::LoadOp::create( + b, loc, input, ValueRange{zero, iy, ix, channel}); + memref::StoreOp::create(b, loc, value, floatPanel, + ValueRange{pixel, lane}); + } + Value fp = BankAllocOp::create(b, loc, i64, 1, 1); + fp = BankMvinOp::create(b, loc, i64, floatPanel, fp, + inputSide * inputSide * 4, 1); + Value inputMultiplier = tensor::ExtractOp::create( + b, loc, args[3], ValueRange{zero, zero, zero, zero}); + auto q = BankCustomOp::create(b, loc, i64, ValueRange{fp}, source, + ValueRange{inputMultiplier}, + "fp32_to_int8"); + q->setAttr("rows", + b.getI64IntegerAttr(inputSide * inputSide * 4)); + source = q.getDestinationOut(); + BankReleaseOp::create(b, loc, fp); + } + } + Value patch = BankAllocOp::create(b, loc, i64, 1, 1); + patch = compute(b, loc, "im2col_i8", ValueRange{source}, patch, false, + {{"input_side", inputSide}, + {"kernel", kernel}, + {"stride", stride}, + {"padding", 0}, + {"start_row", 0}, + {"start_col", 0}, + {"op1_base", 0}, + {"lane", ic % 16}, + {"window_start", 0}, + {"window_count", convSide * convSide}}) + .getDestinationOut(); + Value packed = BankAllocOp::create(b, loc, i64, 1, 1); + // Im2col packs M16/K16, while this array consumes tileRows/K16. + for (int64_t m = 0; m < rows; m += arrayRows) + for (int64_t k = 0; k < reduction / 16; ++k) + for (int64_t r = 0; r < arrayRows; r += 4) + packed = + compute(b, loc, "maxpool_i8", ValueRange{patch}, packed, + true, + {{"input_side", 2}, + {"output_side", 2}, + {"kernel", 1}, + {"stride", 1}, + {"padding", 0}, + {"op1_base", k * 16 + m + r}, + {"wr_base", (m / arrayRows * (reduction / 16) + + k) * arrayRows + + r}, + {"output_stride", 2}, + {"start_row", 0}, + {"start_col", 0}}) + .getDestinationOut(); + BankReleaseOp::create(b, loc, patch); + Value w = BankAllocOp::create(b, loc, i64, 1, 1); + Value slice = memref::SubViewOp::create( + b, loc, weightPack, ArrayRef{term, 0, 0}, + ArrayRef{1, reduction, 16}, ArrayRef{1, 1, 1}); + w = BankMvinOp::create(b, loc, i64, slice, w, reduction, 1); + auto matmul = + compute(b, loc, "matmul_i8_block", ValueRange{packed, w}, acc, + true, {{"rows", rows}, {"k", reduction}}); + matmul->setAttr("first", b.getBoolAttr(term == 0)); + matmul->setAttr("last", b.getBoolAttr(term + 1 == terms)); + acc = matmul.getDestinationOut(); + BankReleaseOp::create(b, loc, packed); + BankReleaseOp::create(b, loc, w); + if (ic % 16 == 15 || term + 1 == terms) + BankReleaseOp::create(b, loc, source); + } + Value restored = BankAllocOp::create(b, loc, i64, 1, 1); + if (nextMultiplier) { + auto q = compute(b, loc, "int32_to_int8", ValueRange{acc, scales}, + restored, false, {{"rows", rows}, {"wr_base", 0}}); + q->setAttr("relu", b.getBoolAttr(relu)); + restored = q.getDestinationOut(); + int64_t panel = (oc / 16 * panelsY + y / side) * panelsX + x / side; + outputImage.banks[panel] = + compute(b, loc, "maxpool_i8", ValueRange{restored}, + outputImage.banks[panel], true, + {{"input_side", convSide}, + {"output_side", tile}, + {"kernel", pool}, + {"stride", pool}, + {"padding", 0}, + {"op1_base", 0}, + {"wr_base", y % side * side + x % side}, + {"output_stride", side}, + {"start_row", 0}, + {"start_col", 0}}) + .getDestinationOut(); + } else { + restored = compute(b, loc, "int32_to_fp32", ValueRange{acc, scales}, + restored, false, {{"rows", rows}}) + .getDestinationOut(); + restored = BankMvoutOp::create(b, loc, i64, outputPack, restored, + rows * 4, 1); + for (int64_t iy = 0; iy < tile; ++iy) + for (int64_t ix = 0; ix < tile; ++ix) { + auto channels = scf::ForOp::create( + b, loc, zero, index(std::min(16, cout - oc)), one); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(channels.getBody()); + Value c = channels.getInductionVar(); + Value value = memref::LoadOp::create( + b, loc, outputPack, ValueRange{index(iy * tile + ix), c}); + memref::StoreOp::create( + b, loc, value, output, + ValueRange{zero, index(y + iy), index(x + ix), + arith::AddIOp::create(b, loc, c, index(oc))}); + } + } + BankReleaseOp::create(b, loc, restored); + BankReleaseOp::create(b, loc, acc); + } + } + BankReleaseOp::create(b, loc, scales); + } + if (inputImage) + for (Value bank : inputImage->banks) + BankReleaseOp::create(b, loc, bank); + if (!nextMultiplier) + tensorOutput = bufferization::ToTensorOp::create(b, loc, outputType, output, + true, true); + return success(); +} +} // namespace + +LogicalResult pebble::lowerSpatialOps(func::FuncOp function, + DenseMap &residentVectors, + SmallVector &consumed) { + SmallVector convolutions; + function.walk([&](Computation op) { + if (op.getKind() == "conv2d_i8" || op.getKind() == "depthwise_conv2d_i8") + convolutions.push_back(op); + }); + DenseMap images; + for (auto conv : convolutions) { + SmallVector links{conv}; + auto group = conv->getAttrOfType("custom.spatial_chain"); + if (group) + for (Operation *next = conv->getNextNode(); next; + next = next->getNextNode()) + if (next->getAttr("custom.spatial_chain") == group) + links.push_back(next); + Value output = links.back()->getResult(0); + Value multiplier; + int64_t pool = 1; + bool relu = false, flatten = false; + bool terminalQuant = + links.back()->getName().getStringRef() == "frontend.fp32_to_int8"; + if (group) { + if (terminalQuant) + multiplier = links.back()->getOperand(1); + else + multiplier = + cast(*output.getUsers().begin()).getInputs()[3]; + for (Operation *link : links) { + if (auto op = dyn_cast(link)) { + relu |= op.getKind() == "relu"; + flatten |= op.getKind() == "flatten_nchw"; + if (op.getKind() == "maxpool2d") { + auto kernel = + cast(op->getAttr("kernel")).asArrayRef(); + auto strides = + cast(op->getAttr("strides")).asArrayRef(); + if (kernel[0] != kernel[1] || strides != kernel) + return op.emitOpError("requires square non-overlapping pooling"); + pool = kernel[0]; + } + } + } + } + auto input = images.find(conv.getInputs()[0]); + ImageBanks result; + Value tensor; + if (failed(convolution(conv, + input == images.end() ? nullptr : &input->second, + multiplier, pool, relu, result, tensor))) + return failure(); + if (!group) { + conv.getResult(0).replaceAllUsesWith(tensor); + consumed.push_back(conv); + continue; + } + OpBuilder b(conv); + Value bank = result.banks.front(); + if (terminalQuant) { + Location loc = conv.getLoc(); + auto type = cast(output.getType()); + Value memory = memref::AllocOp::create( + b, loc, MemRefType::get(type.getShape(), b.getI8Type())); + Value pack = memref::AllocOp::create( + b, loc, + MemRefType::get({result.side * result.side, 16}, b.getI8Type())); + int64_t py = (result.height + result.side - 1) / result.side; + int64_t px = (result.width + result.side - 1) / result.side; + auto index = [&](int64_t n) -> Value { + return arith::ConstantIndexOp::create(b, loc, n); + }; + for (int64_t c = 0; c < result.channels; c += 16) + for (int64_t y = 0; y < result.height; y += result.side) + for (int64_t x = 0; x < result.width; x += result.side) { + Value source = result.banks[(c / 16 * py + y / result.side) * px + + x / result.side]; + BankMvoutOp::create(b, loc, b.getI64Type(), pack, source, + result.side * result.side, 1); + for (int64_t iy = 0; iy < std::min(result.side, result.height - y); + ++iy) + for (int64_t ix = 0; ix < std::min(result.side, result.width - x); + ++ix) { + auto channels = scf::ForOp::create( + b, loc, index(0), + index(std::min(16, result.channels - c)), + index(1)); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(channels.getBody()); + Value lane = channels.getInductionVar(); + Value value = memref::LoadOp::create( + b, loc, pack, + ValueRange{index(iy * result.side + ix), lane}); + memref::StoreOp::create( + b, loc, value, memory, + ValueRange{index(0), index(y + iy), index(x + ix), + arith::AddIOp::create(b, loc, lane, index(c))}); + } + } + tensor = + bufferization::ToTensorOp::create(b, loc, type, memory, true, true); + output.replaceAllUsesWith(tensor); + for (Value source : result.banks) + BankReleaseOp::create(b, loc, source); + } else if (flatten) { + if (result.banks.size() != 1 || result.height * result.width != 16 || + result.channels != 16 || result.side != result.width) + return conv.emitOpError( + "resident NCHW flatten requires a 16x16 INT8 panel"); + Value transposed = + BankAllocOp::create(b, conv.getLoc(), b.getI64Type(), 1, 1); + transposed = compute(b, conv.getLoc(), "transpose_i8", ValueRange{bank}, + transposed, false, {{"rows", 16}, {"cols", 16}}) + .getDestinationOut(); + BankReleaseOp::create(b, conv.getLoc(), bank); + bank = transposed; + residentVectors[output] = bank; + } else { + images[output] = result; + } + for (Operation *link : links) { + if (link->getName().getStringRef() == "buddy_trace.end") { + b.setInsertionPoint(link); + Operation *trace = b.clone(*link); + trace->setOperand(0, bank); + trace->getResult(0).setType(bank.getType()); + trace->removeAttr("custom.spatial_chain"); + } + consumed.push_back(link); + } + } + return success(); +} diff --git a/examples/designs/pebble/stack/midend/src/Lowering.h b/examples/designs/pebble/stack/midend/src/Lowering.h new file mode 100644 index 000000000..cbd2b7024 --- /dev/null +++ b/examples/designs/pebble/stack/midend/src/Lowering.h @@ -0,0 +1,33 @@ +//===- Lowering.h - Custom computations in the Pebble midend -------------===// +#ifndef PEBBLE_LOWERING_H +#define PEBBLE_LOWERING_H + +#include "Custom/CustomOps.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/IR/Builders.h" +#include "llvm/ADT/DenseMap.h" + +namespace mlir::linalg { +class GenericOp; +} +namespace pebble { +bool elementwise(mlir::linalg::GenericOp op, llvm::StringRef name); +mlir::Value tensorBuffer(mlir::OpBuilder &builder, mlir::Location loc, + mlir::Value tensor); +mlir::LogicalResult lowerLinearTiles(::buddy::custom::CustomOp op, + mlir::Value &result); +mlir::LogicalResult lowerGlobalAverage(::buddy::custom::CustomOp op, + mlir::Value &result, bool resident); +mlir::LogicalResult lowerChannelOp(::buddy::custom::CustomOp op, + mlir::Value &result); +mlir::LogicalResult lowerTranspose(::buddy::custom::CustomOp op, + mlir::Value &result); +void formQuantizedChains(mlir::func::FuncOp function); +mlir::LogicalResult lowerQuantizedChains(mlir::func::FuncOp function); +void formSpatialOps(mlir::func::FuncOp function); +mlir::LogicalResult +lowerSpatialOps(mlir::func::FuncOp function, + llvm::DenseMap &residentVectors, + llvm::SmallVector &consumed); +} // namespace pebble +#endif diff --git a/examples/designs/pebble/stack/midend/src/QuantizedChains.cpp b/examples/designs/pebble/stack/midend/src/QuantizedChains.cpp new file mode 100644 index 000000000..0a4a47779 --- /dev/null +++ b/examples/designs/pebble/stack/midend/src/QuantizedChains.cpp @@ -0,0 +1,453 @@ +//===- QuantizedChains.cpp - Resident INT8 vector and channel chains ------===// +#include "Buckyball/BuckyballOps.h" +#include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" +#include "llvm/ADT/MapVector.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using Computation = ::buddy::custom::CustomOp; + +void pebble::formQuantizedChains(func::FuncOp function) { + SmallVector roots; + function.walk([&](Computation op) { + if (op.getKind() == "global_avg_i8" || + (op.getKind() == "linear_i8_requant" && + cast(op.getResult(0).getType()).getDimSize(0) == 1)) + roots.push_back(op); + }); + int64_t id = 0; + for (auto root : roots) { + if (root->hasAttr("custom.quantized_chain")) + continue; + SmallVector chain{root}; + Value value = root.getResult(0); + int computations = 1; + while (value.hasOneUse()) { + Operation *user = *value.getUsers().begin(); + if (user->getBlock() != root->getBlock()) + break; + bool barrier = false; + for (Operation *between = chain.back()->getNextNode(); between != user; + between = between->getNextNode()) + if (!isMemoryEffectFree(between)) { + barrier = true; + break; + } + if (barrier) + break; + if (auto collapse = dyn_cast(user)) { + auto type = collapse.getResultType(); + if (type.getRank() != 2 || type.getDimSize(0) != 1) + break; + } else if (auto op = dyn_cast(user)) { + if (op.getInputs()[0] != value || op->hasAttr("custom.quantized_chain")) + break; + if (op.getKind() == "gate_mul_i8") { + if (!op.getResult(0).hasOneUse()) + break; + auto consumer = + dyn_cast(*op.getResult(0).getUsers().begin()); + auto type = cast(op.getInputs()[1].getType()); + if (!consumer || consumer.getKind() != "linear_i8_requant" || + consumer.getInputs()[0] != op.getResult(0) || + type.getDimSize(1) % 16 || + consumer->getBlock() != root->getBlock()) + break; + for (Operation *between = op->getNextNode(); between != consumer; + between = between->getNextNode()) + if (!isMemoryEffectFree(between)) { + barrier = true; + break; + } + if (barrier) + break; + chain.append({op, consumer}); + computations += 2; + break; + } + if (op.getKind() == "lookup_i8" && + cast(op.getInputs()[1].getType()) + .getNumElements() != 256) + break; + if ((op.getKind() != "linear_i8_requant" && + op.getKind() != "lookup_i8") || + cast(op.getResult(0).getType()).getDimSize(0) != + 1) + break; + ++computations; + } else { + break; + } + chain.push_back(user); + value = user->getResult(0); + } + if (computations < 2) + continue; + for (Operation *op : chain) + op->setAttr( + "custom.quantized_chain", + IntegerAttr::get(IntegerType::get(function.getContext(), 64), id)); + ++id; + } +} + +namespace { +LogicalResult vectorLinear(Computation op, Value input, Value &result) { + auto args = op.getInputs(); + auto weightType = cast(args[1].getType()); + int64_t k = weightType.getDimSize(0), n = weightType.getDimSize(1); + int64_t paddedK = (k + 15) / 16 * 16; + if (paddedK > buckyball_target::getBuckyballTarget().bankDepth || n > 2048) + return op.emitOpError( + "resident Linear exceeds its bank or quantizer row range"); + OpBuilder b(op); + Location loc = op.getLoc(); + auto index = [&](int64_t n) -> Value { + return arith::ConstantIndexOp::create(b, loc, n); + }; + Value zero = index(0), one = index(1); + Type i64 = b.getI64Type(); + Value weights = pebble::tensorBuffer(b, loc, args[1]); + Value biases = pebble::tensorBuffer(b, loc, args[2]); + Value scales = pebble::tensorBuffer(b, loc, args[3]); + result = BankAllocOp::create(b, loc, i64, 1, 1); + for (int64_t c = 0; c < n; c += 16) { + Value weightPack = memref::AllocOp::create( + b, loc, MemRefType::get({paddedK, 16}, b.getI8Type())); + Value biasPack = + memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getI32Type())); + Value scalePack = + memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getF32Type())); + Value zeroI8 = arith::ConstantIntOp::create(b, loc, 0, 8); + Value zeroI32 = arith::ConstantIntOp::create(b, loc, 0, 32); + Value oneF32 = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(1)); + linalg::FillOp::create(b, loc, zeroI8, weightPack); + linalg::FillOp::create(b, loc, zeroI32, biasPack); + linalg::FillOp::create(b, loc, oneF32, scalePack); + auto lanes = scf::ForOp::create(b, loc, zero, + index(std::min(16, n - c)), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value channel = arith::AddIOp::create(b, loc, lane, index(c)); + Value bias = + memref::LoadOp::create(b, loc, biases, ValueRange{zero, channel}); + Value scale = + memref::LoadOp::create(b, loc, scales, ValueRange{zero, channel}); + memref::StoreOp::create(b, loc, bias, biasPack, lane); + memref::StoreOp::create(b, loc, scale, scalePack, lane); + auto inner = scf::ForOp::create(b, loc, zero, index(k), one); + b.setInsertionPointToStart(inner.getBody()); + Value r = inner.getInductionVar(); + Value weight = + memref::LoadOp::create(b, loc, weights, ValueRange{r, channel}); + memref::StoreOp::create(b, loc, weight, weightPack, ValueRange{r, lane}); + } + Value w = BankAllocOp::create(b, loc, i64, 1, 1); + w = BankMvinOp::create(b, loc, i64, weightPack, w, paddedK, 1); + Value acc = BankAllocOp::create(b, loc, i64, 1, 1); + acc = BankMvinOp::create(b, loc, i64, biasPack, acc, 4, 1); + auto product = BankCustomOp::create(b, loc, i64, ValueRange{input, w}, acc, + ValueRange{}, "matmul_i8", true); + product->setAttr("rows", b.getI64IntegerAttr(1)); + product->setAttr("k", b.getI64IntegerAttr(paddedK)); + acc = product.getDestinationOut(); + BankReleaseOp::create(b, loc, w); + Value scale = BankAllocOp::create(b, loc, i64, 1, 1); + scale = BankMvinOp::create(b, loc, i64, scalePack, scale, 4, 1); + auto quant = + BankCustomOp::create(b, loc, i64, ValueRange{acc, scale}, result, + ValueRange{}, "int32_to_int8", true); + quant->setAttr("rows", b.getI64IntegerAttr(1)); + quant->setAttr("wr_base", b.getI64IntegerAttr(c / 16)); + quant->setAttr("relu", op->getAttr("relu")); + result = quant.getDestinationOut(); + BankReleaseOp::create(b, loc, acc); + BankReleaseOp::create(b, loc, scale); + } + BankReleaseOp::create(b, loc, input); + return success(); +} + +LogicalResult gatedLinear(Computation product, Computation multiply, Value gate, + Value &result) { + auto args = product.getInputs(); + auto inputType = cast(multiply.getInputs()[1].getType()); + auto resultType = cast(product.getResult(0).getType()); + int64_t m = inputType.getDimSize(0), k = inputType.getDimSize(1), + n = resultType.getDimSize(1); + int64_t rows = + buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); + int64_t reductionLimit = + std::min(64 * 16 / rows, + buckyball_target::getBuckyballTarget().bankDepth) / + 16 * 16; + OpBuilder b(product); + Location loc = product.getLoc(); + auto index = [&](int64_t n) -> Value { + return arith::ConstantIndexOp::create(b, loc, n); + }; + Value zero = index(0), one = index(1); + Type i64 = b.getI64Type(); + Value input = pebble::tensorBuffer(b, loc, multiply.getInputs()[1]); + Value weights = pebble::tensorBuffer(b, loc, args[1]); + Value biases = pebble::tensorBuffer(b, loc, args[2]); + Value scales = pebble::tensorBuffer(b, loc, args[3]); + Value ratio = arith::ConstantOp::create( + b, loc, cast(multiply->getAttr("ratio"))); + Value output = memref::AllocOp::create( + b, loc, MemRefType::get(resultType.getShape(), b.getI8Type())); + Value scratch = memref::AllocOp::create( + b, loc, MemRefType::get({rows, 16}, b.getI8Type())); + Value izero = arith::ConstantIntOp::create(b, loc, 0, 8); + for (int64_t c = 0; c < n; c += 16) { + Value biasPack = + memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getI32Type())); + Value scalePack = + memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getF32Type())); + Value zeroI32 = arith::ConstantIntOp::create(b, loc, 0, 32); + Value oneF32 = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(1)); + linalg::FillOp::create(b, loc, zeroI32, biasPack); + linalg::FillOp::create(b, loc, oneF32, scalePack); + auto lanes = scf::ForOp::create(b, loc, zero, + index(std::min(16, n - c)), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(), + channel = arith::AddIOp::create(b, loc, lane, index(c)); + Value bias = + memref::LoadOp::create(b, loc, biases, ValueRange{zero, channel}); + Value scale = + memref::LoadOp::create(b, loc, scales, ValueRange{zero, channel}); + memref::StoreOp::create(b, loc, bias, biasPack, lane); + memref::StoreOp::create(b, loc, scale, scalePack, lane); + } + Value scale = BankAllocOp::create(b, loc, i64, 1, 1); + scale = BankMvinOp::create(b, loc, i64, scalePack, scale, 4, 1); + for (int64_t r = 0; r < m; r += rows) { + Value acc = BankAllocOp::create(b, loc, i64, 1, 1); + acc = BankMvinOp::create(b, loc, i64, biasPack, acc, 4, 1); + for (int64_t start = 0; start < k; start += reductionLimit) { + int64_t count = std::min(reductionLimit, k - start); + Value a = BankAllocOp::create(b, loc, i64, 1, 1); + for (int64_t channel = start; channel < start + count; channel += 16) { + linalg::FillOp::create(b, loc, izero, scratch); + auto pixels = scf::ForOp::create(b, loc, zero, + index(std::min(rows, m - r)), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(pixels.getBody()); + Value row = pixels.getInductionVar(); + auto lanes = scf::ForOp::create(b, loc, zero, index(16), one); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value value = memref::LoadOp::create( + b, loc, input, + ValueRange{ + arith::AddIOp::create(b, loc, row, index(r)), + arith::AddIOp::create(b, loc, lane, index(channel))}); + memref::StoreOp::create(b, loc, value, scratch, + ValueRange{row, lane}); + } + Value source = BankAllocOp::create(b, loc, i64, 1, 1); + source = BankMvinOp::create(b, loc, i64, scratch, source, rows, 1); + Value gated = BankAllocOp::create(b, loc, i64, 1, 1); + auto mul = + BankCustomOp::create(b, loc, i64, ValueRange{gate, source}, gated, + ValueRange{ratio}, "gate_mul_i8"); + mul->setAttr("rows", b.getI64IntegerAttr(rows)); + mul->setAttr("gate_row", b.getI64IntegerAttr(channel / 16)); + gated = mul.getDestinationOut(); + BankReleaseOp::create(b, loc, source); + for (int64_t offset = 0; offset < rows; offset += 4) { + auto copy = BankCustomOp::create(b, loc, i64, ValueRange{gated}, a, + ValueRange{}, "maxpool_i8", true); + for (auto [name, value] : + std::initializer_list>{ + {"input_side", 2}, + {"output_side", 2}, + {"kernel", 1}, + {"stride", 1}, + {"padding", 0}, + {"op1_base", offset}, + {"wr_base", (channel - start) / 16 * rows + offset}, + {"output_stride", 2}, + {"start_row", 0}, + {"start_col", 0}}) + copy->setAttr(name, b.getI64IntegerAttr(value)); + a = copy.getDestinationOut(); + } + BankReleaseOp::create(b, loc, gated); + } + Value weightPack = memref::AllocOp::create( + b, loc, MemRefType::get({count, 16}, b.getI8Type())); + linalg::FillOp::create(b, loc, izero, weightPack); + auto inner = scf::ForOp::create(b, loc, zero, index(count), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(inner.getBody()); + Value row = inner.getInductionVar(); + auto lanes = scf::ForOp::create( + b, loc, zero, index(std::min(16, n - c)), one); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value value = memref::LoadOp::create( + b, loc, weights, + ValueRange{arith::AddIOp::create(b, loc, row, index(start)), + arith::AddIOp::create(b, loc, lane, index(c))}); + memref::StoreOp::create(b, loc, value, weightPack, + ValueRange{row, lane}); + } + Value w = BankAllocOp::create(b, loc, i64, 1, 1); + w = BankMvinOp::create(b, loc, i64, weightPack, w, count, 1); + auto matmul = + BankCustomOp::create(b, loc, i64, ValueRange{a, w}, acc, + ValueRange{}, "matmul_i8_block", true); + matmul->setAttr("rows", b.getI64IntegerAttr(rows)); + matmul->setAttr("k", b.getI64IntegerAttr(count)); + matmul->setAttr("first", b.getBoolAttr(start == 0)); + matmul->setAttr("last", b.getBoolAttr(start + count == k)); + acc = matmul.getDestinationOut(); + BankReleaseOp::create(b, loc, a); + BankReleaseOp::create(b, loc, w); + } + Value codes = BankAllocOp::create(b, loc, i64, 1, 1); + auto quant = BankCustomOp::create(b, loc, i64, ValueRange{acc, scale}, + codes, ValueRange{}, "int32_to_int8"); + quant->setAttr("rows", b.getI64IntegerAttr(rows)); + quant->setAttr("wr_base", b.getI64IntegerAttr(0)); + quant->setAttr("relu", product->getAttr("relu")); + codes = quant.getDestinationOut(); + BankMvoutOp::create(b, loc, i64, scratch, codes, rows, 1); + BankReleaseOp::create(b, loc, acc); + BankReleaseOp::create(b, loc, codes); + auto pixels = + scf::ForOp::create(b, loc, zero, index(std::min(rows, m - r)), one); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(pixels.getBody()); + Value row = pixels.getInductionVar(); + auto lanes = scf::ForOp::create(b, loc, zero, + index(std::min(16, n - c)), one); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value value = + memref::LoadOp::create(b, loc, scratch, ValueRange{row, lane}); + memref::StoreOp::create( + b, loc, value, output, + ValueRange{arith::AddIOp::create(b, loc, row, index(r)), + arith::AddIOp::create(b, loc, lane, index(c))}); + } + BankReleaseOp::create(b, loc, scale); + } + BankReleaseOp::create(b, loc, gate); + result = + bufferization::ToTensorOp::create(b, loc, resultType, output, true, true); + return success(); +} +} // namespace + +LogicalResult pebble::lowerQuantizedChains(func::FuncOp function) { + llvm::MapVector> groups; + function.walk([&](Operation *op) { + if (auto id = op->getAttrOfType("custom.quantized_chain")) + groups[id.getInt()].push_back(op); + }); + for (auto &[id, chain] : groups) { + Value bank, result; + Computation multiply; + for (Operation *link : chain) { + auto op = dyn_cast(link); + if (!op) + continue; + if (op.getKind() == "global_avg_i8") { + if (failed(lowerGlobalAverage(op, bank, true))) + return failure(); + } else if (op.getKind() == "linear_i8_requant") { + if (multiply) { + if (failed(gatedLinear(op, multiply, bank, result))) + return failure(); + bank = Value(); + } else { + if (!bank) { + OpBuilder b(op); + Location loc = op.getLoc(); + auto type = cast(op.getInputs()[0].getType()); + int64_t k = type.getDimSize(1), padded = (k + 15) / 16 * 16; + Value pack = memref::AllocOp::create( + b, loc, MemRefType::get({1, padded}, b.getI8Type())); + Value zero = arith::ConstantIntOp::create(b, loc, 0, 8); + linalg::FillOp::create(b, loc, zero, pack); + Value slice = memref::SubViewOp::create( + b, loc, pack, ArrayRef{0, 0}, ArrayRef{1, k}, + ArrayRef{1, 1}); + memref::CopyOp::create( + b, loc, tensorBuffer(b, loc, op.getInputs()[0]), slice); + bank = BankAllocOp::create(b, loc, b.getI64Type(), 1, 1); + bank = BankMvinOp::create(b, loc, b.getI64Type(), pack, bank, + padded / 16, 1); + } + Value next; + if (failed(vectorLinear(op, bank, next))) + return failure(); + bank = next; + } + } else if (op.getKind() == "lookup_i8") { + OpBuilder b(op); + Location loc = op.getLoc(); + auto tableType = cast(op.getInputs()[1].getType()); + if (tableType.getNumElements() != 256) + return op.emitOpError( + "resident lookup requires one shared 256-entry table"); + Value table = BankAllocOp::create(b, loc, b.getI64Type(), 1, 1); + table = BankMvinOp::create(b, loc, b.getI64Type(), + tensorBuffer(b, loc, op.getInputs()[1]), + table, 16, 1); + Value destination = BankAllocOp::create(b, loc, b.getI64Type(), 1, 1); + auto lookup = BankCustomOp::create(b, loc, b.getI64Type(), + ValueRange{bank, table}, destination, + ValueRange{}, "lookup_i8"); + int64_t channels = + cast(op.getResult(0).getType()).getDimSize(1); + lookup->setAttr("rows", b.getI64IntegerAttr((channels + 15) / 16)); + BankReleaseOp::create(b, loc, bank); + BankReleaseOp::create(b, loc, table); + bank = lookup.getDestinationOut(); + } else if (op.getKind() == "gate_mul_i8") { + multiply = op; + } + } + Operation *last = chain.back(); + if (bank) { + OpBuilder b(last); + Location loc = last->getLoc(); + auto type = cast(last->getResult(0).getType()); + int64_t channels = type.getDimSize(1), padded = (channels + 15) / 16 * 16; + Value pack = memref::AllocOp::create( + b, loc, MemRefType::get({1, padded}, b.getI8Type())); + BankMvoutOp::create(b, loc, b.getI64Type(), pack, bank, padded / 16, 1); + BankReleaseOp::create(b, loc, bank); + Value output = memref::AllocOp::create( + b, loc, MemRefType::get({1, channels}, b.getI8Type())); + Value slice = memref::SubViewOp::create( + b, loc, pack, ArrayRef{0, 0}, ArrayRef{1, channels}, + ArrayRef{1, 1}); + memref::CopyOp::create(b, loc, slice, output); + result = + bufferization::ToTensorOp::create(b, loc, type, output, true, true); + } + last->getResult(0).replaceAllUsesWith(result); + for (Operation *op : llvm::reverse(chain)) + op->erase(); + } + return success(); +} diff --git a/examples/designs/pebble/stack/midend/tests/form-custom-ops.mlir b/examples/designs/pebble/stack/midend/tests/form-custom-ops.mlir new file mode 100644 index 000000000..54dd3fe48 --- /dev/null +++ b/examples/designs/pebble/stack/midend/tests/form-custom-ops.mlir @@ -0,0 +1,83 @@ +// CHECK-LABEL: func.func @named +// CHECK-NOT: linalg.matmul +// CHECK: custom.op "matmul_i8"(%arg0, %arg1, %arg2) +// CHECK: return +func.func @named(%a: tensor<1x32xi8>, %b: tensor<32x32xi8>, %initial: tensor<1x32xi32>) -> tensor<1x32xi32> { + %product = linalg.matmul ins(%a, %b: tensor<1x32xi8>, tensor<32x32xi8>) outs(%initial: tensor<1x32xi32>) -> tensor<1x32xi32> + return %product : tensor<1x32xi32> +} + +// CHECK-LABEL: func.func @generic +// CHECK-NOT: linalg.generic +// CHECK: custom.op "matmul_i8"(%arg0, %arg1, %arg2) +// CHECK: return +func.func @generic(%a: tensor<1x32xi8>, %b: tensor<32x32xi8>, %initial: tensor<1x32xi32>) -> tensor<1x32xi32> { + %product = linalg.generic { + indexing_maps = [affine_map<(m,n,k)->(m,k)>, affine_map<(m,n,k)->(k,n)>, affine_map<(m,n,k)->(m,n)>], + iterator_types = ["parallel", "parallel", "reduction"] + } ins(%a, %b: tensor<1x32xi8>, tensor<32x32xi8>) outs(%initial: tensor<1x32xi32>) { + ^bb0(%lhs: i8, %rhs: i8, %acc: i32): + %x = arith.extsi %lhs : i8 to i32 + %y = arith.extsi %rhs : i8 to i32 + %p = arith.muli %x, %y : i32 + %s = arith.addi %acc, %p : i32 + linalg.yield %s : i32 + } -> tensor<1x32xi32> + return %product : tensor<1x32xi32> +} + +// CHECK-LABEL: func.func @linear +// CHECK: %[[Q:.*]] = "frontend.fp32_to_int8" +// CHECK: %[[D:.*]] = "frontend.int8_to_fp32"(%[[Q]], +// CHECK-NOT: linalg.generic +// CHECK: %[[R:.*]] = custom.op "linear_i8"(%arg0, %arg1, %{{.*}}, %arg2, %arg3, %arg4) +// CHECK: return %[[R]], %[[D]] +func.func @linear(%a: tensor<1x32xf32>, %w: tensor<32x32xi8>, %multiplier: tensor<1x1xf32>, %input_step: tensor<1x1xf32>, %weight_step: tensor<1x32xf32>) -> (tensor<1x32xf32>, tensor<1x32xf32>) { + %zero = arith.constant dense<0.0> : tensor<1x32xf32> + %q = "frontend.fp32_to_int8"(%a, %multiplier) : (tensor<1x32xf32>, tensor<1x1xf32>) -> tensor<1x32xi8> + %x = "frontend.int8_to_fp32"(%q, %input_step) : (tensor<1x32xi8>, tensor<1x1xf32>) -> tensor<1x32xf32> + %y = "frontend.int8_to_fp32"(%w, %weight_step) : (tensor<32x32xi8>, tensor<1x32xf32>) -> tensor<32x32xf32> + %product = linalg.generic { + indexing_maps = [affine_map<(m,n,k)->(m,k)>, affine_map<(m,n,k)->(k,n)>, affine_map<(m,n,k)->(m,n)>], + iterator_types = ["parallel", "parallel", "reduction"] + } ins(%x, %y: tensor<1x32xf32>, tensor<32x32xf32>) outs(%zero: tensor<1x32xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %acc: f32): + %p = arith.mulf %lhs, %rhs : f32 + %s = arith.addf %acc, %p : f32 + linalg.yield %s : f32 + } -> tensor<1x32xf32> + return %product, %x : tensor<1x32xf32>, tensor<1x32xf32> +} + +// CHECK-LABEL: func.func @different_body +// CHECK-NOT: custom.op +// CHECK: linalg.generic +// CHECK: arith.subi +// CHECK: return +func.func @different_body(%a: tensor<1x32xi8>, %b: tensor<32x32xi8>, %initial: tensor<1x32xi32>) -> tensor<1x32xi32> { + %product = linalg.generic { + indexing_maps = [affine_map<(m,n,k)->(m,k)>, affine_map<(m,n,k)->(k,n)>, affine_map<(m,n,k)->(m,n)>], + iterator_types = ["parallel", "parallel", "reduction"] + } ins(%a, %b: tensor<1x32xi8>, tensor<32x32xi8>) outs(%initial: tensor<1x32xi32>) { + ^bb0(%lhs: i8, %rhs: i8, %acc: i32): + %x = arith.extsi %lhs : i8 to i32 + %y = arith.extsi %rhs : i8 to i32 + %p = arith.muli %x, %y : i32 + %s = arith.subi %acc, %p : i32 + linalg.yield %s : i32 + } -> tensor<1x32xi32> + return %product : tensor<1x32xi32> +} + +// CHECK-LABEL: func.func @shared_producer +// CHECK: %[[CODES:.*]] = "frontend.fp32_to_int8" +// CHECK-NOT: "frontend.int8_to_fp32" +// CHECK: custom.op "linear_i8"(%arg0, %[[CODES]], %{{.*}}, %arg1, %arg2, %arg2) +// CHECK: return +func.func @shared_producer(%a: tensor<1x1xf32>, %multiplier: tensor<1x1xf32>, %step: tensor<1x1xf32>) -> tensor<1x1xf32> { + %zero = arith.constant dense<0.0> : tensor<1x1xf32> + %q = "frontend.fp32_to_int8"(%a, %multiplier) : (tensor<1x1xf32>, tensor<1x1xf32>) -> tensor<1x1xi8> + %x = "frontend.int8_to_fp32"(%q, %step) : (tensor<1x1xi8>, tensor<1x1xf32>) -> tensor<1x1xf32> + %product = linalg.matmul ins(%x, %x: tensor<1x1xf32>, tensor<1x1xf32>) outs(%zero: tensor<1x1xf32>) -> tensor<1x1xf32> + return %product : tensor<1x1xf32> +} diff --git a/examples/designs/pebble/stack/midend/tests/form-depthwise.mlir b/examples/designs/pebble/stack/midend/tests/form-depthwise.mlir new file mode 100644 index 000000000..2dc476017 --- /dev/null +++ b/examples/designs/pebble/stack/midend/tests/form-depthwise.mlir @@ -0,0 +1,33 @@ +// CHECK-LABEL: func.func @depthwise +// CHECK: custom.op "depthwise_conv2d_i8" +// CHECK-SAME: padding = array +// CHECK: return +func.func @depthwise(%x: tensor<1x4x4x16xf32>, %w: tensor<16x1x3x3xi8>, %bias_codes: tensor<16xi32>, %input_step: tensor<1x1x1x1xf32>, %multiplier: tensor<1x1x1x1xf32>, %weight_step: tensor<16x1x1x1xf32>) -> tensor<1x4x4x16xf32> { + %q = "frontend.fp32_to_int8"(%x,%multiplier) : (tensor<1x4x4x16xf32>,tensor<1x1x1x1xf32>) -> tensor<1x4x4x16xi8> + %values = "frontend.int8_to_fp32"(%q,%input_step) : (tensor<1x4x4x16xi8>,tensor<1x1x1x1xf32>) -> tensor<1x4x4x16xf32> + %zero = arith.constant 0.0 : f32 + %padded = tensor.pad %values low[0,1,1,0] high[0,1,1,0] { + ^bb0(%n: index, %h: index, %wi: index, %c: index): + tensor.yield %zero : f32 + } : tensor<1x4x4x16xf32> to tensor<1x6x6x16xf32> + %wf = "frontend.int8_to_fp32"(%w,%weight_step) : (tensor<16x1x3x3xi8>,tensor<16x1x1x1xf32>) -> tensor<16x1x3x3xf32> + %we = tensor.empty() : tensor<3x3x16x1xf32> + %weights = linalg.transpose ins(%wf: tensor<16x1x3x3xf32>) outs(%we: tensor<3x3x16x1xf32>) permutation = [2,3,0,1] + %scale = linalg.generic {indexing_maps = [affine_map<(f,c,h,w)->(0,c,h,w)>,affine_map<(f,c,h,w)->(f,c,h,w)>,affine_map<(f,c,h,w)->(f,c,h,w)>], iterator_types = ["parallel","parallel","parallel","parallel"]} ins(%input_step,%weight_step: tensor<1x1x1x1xf32>,tensor<16x1x1x1xf32>) outs(%weight_step: tensor<16x1x1x1xf32>) { + ^bb0(%a: f32,%b: f32,%o: f32): + %r = arith.mulf %a,%b : f32 + linalg.yield %r : f32 + } -> tensor<16x1x1x1xf32> + %bs = tensor.collapse_shape %scale [[0,1,2,3]] : tensor<16x1x1x1xf32> into tensor<16xf32> + %bias = "frontend.int32_to_fp32"(%bias_codes,%bs) : (tensor<16xi32>,tensor<16xf32>) -> tensor<16xf32> + %initial = arith.constant dense<0.0> : tensor<1x4x4x16x1xf32> + %convolution = linalg.depthwise_conv_2d_nhwc_hwcm {strides = dense<1> : tensor<2xi64>, dilations = dense<1> : tensor<2xi64>} ins(%padded,%weights: tensor<1x6x6x16xf32>,tensor<3x3x16x1xf32>) outs(%initial: tensor<1x4x4x16x1xf32>) -> tensor<1x4x4x16x1xf32> + %collapsed = tensor.collapse_shape %convolution [[0],[1],[2],[3,4]] : tensor<1x4x4x16x1xf32> into tensor<1x4x4x16xf32> + %empty = tensor.empty() : tensor<1x4x4x16xf32> + %result = linalg.generic {indexing_maps = [affine_map<(n,h,w,c)->(c)>,affine_map<(n,h,w,c)->(n,h,w,c)>,affine_map<(n,h,w,c)->(n,h,w,c)>], iterator_types = ["parallel","parallel","parallel","parallel"]} ins(%bias,%collapsed: tensor<16xf32>,tensor<1x4x4x16xf32>) outs(%empty: tensor<1x4x4x16xf32>) { + ^bb0(%a: f32,%b: f32,%o: f32): + %r = arith.addf %a,%b : f32 + linalg.yield %r : f32 + } -> tensor<1x4x4x16xf32> + return %result : tensor<1x4x4x16xf32> +} diff --git a/examples/designs/pebble/stack/midend/tests/form-int32-bias.mlir b/examples/designs/pebble/stack/midend/tests/form-int32-bias.mlir new file mode 100644 index 000000000..b3148f674 --- /dev/null +++ b/examples/designs/pebble/stack/midend/tests/form-int32-bias.mlir @@ -0,0 +1,61 @@ +// CHECK-LABEL: func.func @linear +// CHECK: buddy_trace.start +// CHECK: %[[R:.*]] = custom.op "linear_i8"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5) +// CHECK: %[[T:.*]] = buddy_trace.end %[[R]] +// CHECK-NOT: arith.addf +// CHECK-NOT: frontend.int32_to_fp32 +// CHECK: return %[[T]] +func.func @linear(%a: tensor<1x32xf32>, %w: tensor<32x32xi8>, %bias: tensor<1x32xi32>, %multiplier: tensor<1x1xf32>, %input_step: tensor<1x1xf32>, %weight_step: tensor<1x32xf32>) -> tensor<1x32xf32> { + %zero = arith.constant dense<0.0> : tensor<1x32xf32> + %q = "frontend.fp32_to_int8"(%a, %multiplier) : (tensor<1x32xf32>, tensor<1x1xf32>) -> tensor<1x32xi8> + %x = "frontend.int8_to_fp32"(%q, %input_step) : (tensor<1x32xi8>, tensor<1x1xf32>) -> tensor<1x32xf32> + %y = "frontend.int8_to_fp32"(%w, %weight_step) : (tensor<32x32xi8>, tensor<1x32xf32>) -> tensor<32x32xf32> + buddy_trace.start {cycle = true, id = 0 : i64, id_path = [0], tag = "linear", tensor = false} + %product = linalg.matmul ins(%x, %y: tensor<1x32xf32>, tensor<32x32xf32>) outs(%zero: tensor<1x32xf32>) -> tensor<1x32xf32> + %traced = buddy_trace.end %product {cycle = true, id = 0 : i64, id_path = [0], tag = "linear", tensor = false} : tensor<1x32xf32> + %empty = tensor.empty() : tensor<1x32xf32> + %scale = linalg.generic { + indexing_maps = [affine_map<(m,n)->(m,0)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], + iterator_types = ["parallel", "parallel"] + } ins(%input_step, %weight_step: tensor<1x1xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %unused: f32): + %s = arith.mulf %lhs, %rhs : f32 + linalg.yield %s : f32 + } -> tensor<1x32xf32> + %bias_f32 = "frontend.int32_to_fp32"(%bias, %scale) : (tensor<1x32xi32>, tensor<1x32xf32>) -> tensor<1x32xf32> + %result = linalg.generic { + indexing_maps = [affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], + iterator_types = ["parallel", "parallel"] + } ins(%traced, %bias_f32: tensor<1x32xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %unused: f32): + %s = arith.addf %lhs, %rhs : f32 + linalg.yield %s : f32 + } -> tensor<1x32xf32> + return %result : tensor<1x32xf32> +} + +// A bias in a different scale domain cannot initialize this accumulator. +// CHECK-LABEL: func.func @unrelated_scale +// CHECK: %[[ZERO:.*]] = arith.constant dense<0> : tensor<1x32xi32> +// CHECK: custom.op "linear_i8"(%arg0, %arg1, %[[ZERO]], %arg3, %arg4, %arg5) +// CHECK: "frontend.int32_to_fp32"(%arg2, %arg6) +// CHECK: arith.addf +// CHECK: return +func.func @unrelated_scale(%a: tensor<1x32xf32>, %w: tensor<32x32xi8>, %bias: tensor<1x32xi32>, %multiplier: tensor<1x1xf32>, %input_step: tensor<1x1xf32>, %weight_step: tensor<1x32xf32>, %bias_step: tensor<1x32xf32>) -> tensor<1x32xf32> { + %zero = arith.constant dense<0.0> : tensor<1x32xf32> + %q = "frontend.fp32_to_int8"(%a, %multiplier) : (tensor<1x32xf32>, tensor<1x1xf32>) -> tensor<1x32xi8> + %x = "frontend.int8_to_fp32"(%q, %input_step) : (tensor<1x32xi8>, tensor<1x1xf32>) -> tensor<1x32xf32> + %y = "frontend.int8_to_fp32"(%w, %weight_step) : (tensor<32x32xi8>, tensor<1x32xf32>) -> tensor<32x32xf32> + %product = linalg.matmul ins(%x, %y: tensor<1x32xf32>, tensor<32x32xf32>) outs(%zero: tensor<1x32xf32>) -> tensor<1x32xf32> + %bias_f32 = "frontend.int32_to_fp32"(%bias, %bias_step) : (tensor<1x32xi32>, tensor<1x32xf32>) -> tensor<1x32xf32> + %empty = tensor.empty() : tensor<1x32xf32> + %result = linalg.generic { + indexing_maps = [affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], + iterator_types = ["parallel", "parallel"] + } ins(%product, %bias_f32: tensor<1x32xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %unused: f32): + %s = arith.addf %lhs, %rhs : f32 + linalg.yield %s : f32 + } -> tensor<1x32xf32> + return %result : tensor<1x32xf32> +} diff --git a/examples/designs/pebble/stack/midend/tests/form-linear-chain.mlir b/examples/designs/pebble/stack/midend/tests/form-linear-chain.mlir new file mode 100644 index 000000000..0f5bbab58 --- /dev/null +++ b/examples/designs/pebble/stack/midend/tests/form-linear-chain.mlir @@ -0,0 +1,83 @@ +// FORM-LABEL: func.func @cycle_chain +// FORM: %[[FIRST:.*]] = custom.op "linear_i8"{{.*}}custom.chain = [[CHAIN:[0-9]+]] : i64, custom.relu = true +// FORM: %[[TRACE:.*]] = buddy_trace.end %[[FIRST]] {{.*}}custom.chain = [[CHAIN]] : i64{{.*}}tensor = false +// FORM: %[[RELU:.*]] = custom.op "relu"(%[[TRACE]]) {{.*}}custom.chain = [[CHAIN]] : i64 +// FORM: custom.op "linear_i8"(%[[RELU]], {{.*}}custom.chain = [[CHAIN]] : i64 +// FORM: return +// BANK-LABEL: func.func @cycle_chain +// BANK-NOT: {{fp32_to_int8|int32_to_int8|int32_to_fp32|bank_mvout}} +// BANK: buckyball.bank_custom "fp32_to_int8" +// BANK-NOT: {{fp32_to_int8|int32_to_int8|int32_to_fp32|bank_mvout}} +// BANK: buckyball.bank_custom "int32_to_int8"{{.*}}relu = true{{.*}}wr_base = 0 : i64 +// BANK-NOT: {{fp32_to_int8|int32_to_int8|int32_to_fp32|bank_mvout}} +// BANK: %[[RESIDENT:.*]] = buckyball.bank_custom "int32_to_int8"{{.*}}relu = true{{.*}}wr_base = 1 : i64 +// BANK-NOT: {{fp32_to_int8|int32_to_int8|int32_to_fp32|bank_mvout}} +// BANK: buddy_trace.end %[[RESIDENT]] {{.*}}tensor = false} : i64 +// BANK-NOT: {{fp32_to_int8|int32_to_int8|int32_to_fp32|bank_mvout}} +// BANK: buckyball.bank_custom "int32_to_fp32" +// BANK-NOT: {{fp32_to_int8|int32_to_int8|int32_to_fp32|bank_mvout}} +// BANK: buckyball.bank_mvout +// BANK-NOT: {{fp32_to_int8|int32_to_int8|int32_to_fp32|bank_mvout}} +// BANK: return +func.func @cycle_chain(%x: tensor<1x32xf32>, %w1: tensor<32x32xi8>, %b1: tensor<1x32xi32>, %s1: tensor<1x32xf32>, %w2: tensor<32x16xi8>, %b2: tensor<1x16xi32>, %s2: tensor<1x16xf32>, %multiplier: tensor<1x1xf32>, %step: tensor<1x1xf32>) -> tensor<1x16xf32> { + %zero = arith.constant dense<0.0> : tensor<1x32xf32> + %empty = tensor.empty() : tensor<1x32xf32> + %first = custom.op "linear_i8"(%x, %w1, %b1, %multiplier, %step, %s1) : (tensor<1x32xf32>, tensor<32x32xi8>, tensor<1x32xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x32xf32>) -> tensor<1x32xf32> + %trace = buddy_trace.end %first {cycle = true, id = 0 : i64, id_path = [0], tag = "first", tensor = false} : tensor<1x32xf32> + %relu = linalg.generic { + indexing_maps = [affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], + iterator_types = ["parallel", "parallel"] + } ins(%trace, %zero: tensor<1x32xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { + ^bb0(%value: f32, %zero_value: f32, %unused: f32): + %positive = arith.maximumf %value, %zero_value : f32 + linalg.yield %positive : f32 + } -> tensor<1x32xf32> + %second = custom.op "linear_i8"(%relu, %w2, %b2, %multiplier, %step, %s2) : (tensor<1x32xf32>, tensor<32x16xi8>, tensor<1x16xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x16xf32>) -> tensor<1x16xf32> + return %second : tensor<1x16xf32> +} + +// FORM-LABEL: func.func @tensor_trace +// FORM: custom.op "linear_i8" +// FORM: buddy_trace.end {{.*}}tensor = true} : tensor<1x32xf32> +// FORM: linalg.generic +// FORM: arith.maximumf +// FORM: custom.op "linear_i8" +// FORM: return +func.func @tensor_trace(%x: tensor<1x32xf32>, %w1: tensor<32x32xi8>, %b1: tensor<1x32xi32>, %s1: tensor<1x32xf32>, %w2: tensor<32x16xi8>, %b2: tensor<1x16xi32>, %s2: tensor<1x16xf32>, %multiplier: tensor<1x1xf32>, %step: tensor<1x1xf32>) -> tensor<1x16xf32> { + %zero = arith.constant dense<0.0> : tensor<1x32xf32> + %empty = tensor.empty() : tensor<1x32xf32> + %first = custom.op "linear_i8"(%x, %w1, %b1, %multiplier, %step, %s1) : (tensor<1x32xf32>, tensor<32x32xi8>, tensor<1x32xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x32xf32>) -> tensor<1x32xf32> + %trace = buddy_trace.end %first {cycle = true, id = 1 : i64, id_path = [1], tag = "first", tensor = true} : tensor<1x32xf32> + %relu = linalg.generic { + indexing_maps = [affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], + iterator_types = ["parallel", "parallel"] + } ins(%trace, %zero: tensor<1x32xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { + ^bb0(%value: f32, %zero_value: f32, %unused: f32): + %positive = arith.maximumf %value, %zero_value : f32 + linalg.yield %positive : f32 + } -> tensor<1x32xf32> + %second = custom.op "linear_i8"(%relu, %w2, %b2, %multiplier, %step, %s2) : (tensor<1x32xf32>, tensor<32x16xi8>, tensor<1x16xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x16xf32>) -> tensor<1x16xf32> + return %second : tensor<1x16xf32> +} + +// FORM-LABEL: func.func @multiple_consumers +// FORM: %[[OBSERVED:.*]] = custom.op "linear_i8" +// FORM: linalg.generic +// FORM: arith.maximumf +// FORM: custom.op "linear_i8" +// FORM: return {{.*}}, %[[OBSERVED]] +func.func @multiple_consumers(%x: tensor<1x32xf32>, %w1: tensor<32x32xi8>, %b1: tensor<1x32xi32>, %s1: tensor<1x32xf32>, %w2: tensor<32x16xi8>, %b2: tensor<1x16xi32>, %s2: tensor<1x16xf32>, %multiplier: tensor<1x1xf32>, %step: tensor<1x1xf32>) -> (tensor<1x16xf32>, tensor<1x32xf32>) { + %zero = arith.constant dense<0.0> : tensor<1x32xf32> + %empty = tensor.empty() : tensor<1x32xf32> + %first = custom.op "linear_i8"(%x, %w1, %b1, %multiplier, %step, %s1) : (tensor<1x32xf32>, tensor<32x32xi8>, tensor<1x32xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x32xf32>) -> tensor<1x32xf32> + %relu = linalg.generic { + indexing_maps = [affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], + iterator_types = ["parallel", "parallel"] + } ins(%first, %zero: tensor<1x32xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { + ^bb0(%value: f32, %zero_value: f32, %unused: f32): + %positive = arith.maximumf %value, %zero_value : f32 + linalg.yield %positive : f32 + } -> tensor<1x32xf32> + %second = custom.op "linear_i8"(%relu, %w2, %b2, %multiplier, %step, %s2) : (tensor<1x32xf32>, tensor<32x16xi8>, tensor<1x16xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x16xf32>) -> tensor<1x16xf32> + return %second, %first : tensor<1x16xf32>, tensor<1x32xf32> +} diff --git a/examples/designs/pebble/stack/midend/tests/form-quantized-chain.mlir b/examples/designs/pebble/stack/midend/tests/form-quantized-chain.mlir new file mode 100644 index 000000000..a1a263edb --- /dev/null +++ b/examples/designs/pebble/stack/midend/tests/form-quantized-chain.mlir @@ -0,0 +1,19 @@ +// FORM-LABEL: func.func @squeeze_excite +// FORM-COUNT-7: custom.quantized_chain = 0 : i64 +// BANK-LABEL: func.func @squeeze_excite +// BANK: buckyball.bank_custom "lookup_i8" +// BANK: buckyball.bank_custom "gate_mul_i8" +// BANK: buckyball.bank_custom "matmul_i8_block" +// BANK: buckyball.bank_mvout +// BANK: return +func.func @squeeze_excite(%x: tensor<1x2x4x32xi8>, %w1: tensor<32x16xi8>, %b1: tensor<1x16xi32>, %s1: tensor<1x16xf32>, %w2: tensor<16x32xi8>, %b2: tensor<1x32xi32>, %s2: tensor<1x32xf32>, %lut: tensor<256xi8>, %w3: tensor<32x16xi8>) -> tensor<8x16xi8> { + %source = tensor.collapse_shape %x [[0,1,2],[3]] : tensor<1x2x4x32xi8> into tensor<8x32xi8> + %avg = custom.op "global_avg_i8"(%x) {input_scale = 1.0 : f32, output_scale = 1.0 : f32} : (tensor<1x2x4x32xi8>) -> tensor<1x1x1x32xi8> + %vector = tensor.collapse_shape %avg [[0,1,2],[3]] : tensor<1x1x1x32xi8> into tensor<1x32xi8> + %first = custom.op "linear_i8_requant"(%vector,%w1,%b1,%s1) {relu = true} : (tensor<1x32xi8>,tensor<32x16xi8>,tensor<1x16xi32>,tensor<1x16xf32>) -> tensor<1x16xi8> + %second = custom.op "linear_i8_requant"(%first,%w2,%b2,%s2) {relu = false} : (tensor<1x16xi8>,tensor<16x32xi8>,tensor<1x32xi32>,tensor<1x32xf32>) -> tensor<1x32xi8> + %gate = custom.op "lookup_i8"(%second,%lut) : (tensor<1x32xi8>,tensor<256xi8>) -> tensor<1x32xi8> + %multiplied = custom.op "gate_mul_i8"(%gate,%source) {ratio = 0.5 : f32} : (tensor<1x32xi8>,tensor<8x32xi8>) -> tensor<8x32xi8> + %result = custom.op "linear_i8_requant"(%multiplied,%w3,%b1,%s1) {relu = false} : (tensor<8x32xi8>,tensor<32x16xi8>,tensor<1x16xi32>,tensor<1x16xf32>) -> tensor<8x16xi8> + return %result : tensor<8x16xi8> +} diff --git a/examples/designs/pebble/stack/midend/tests/lower-bank-updates.mlir b/examples/designs/pebble/stack/midend/tests/lower-bank-updates.mlir new file mode 100644 index 000000000..31c37f388 --- /dev/null +++ b/examples/designs/pebble/stack/midend/tests/lower-bank-updates.mlir @@ -0,0 +1,21 @@ +// CHECK-LABEL: func.func @copy_rows +// CHECK: arith.constant -9223372035778936831 : i64 +// CHECK: return +func.func @copy_rows() { + buckyball.custom "maxpool_i8" {input_side = 1 : i64, output_side = 1 : i64, kernel = 1 : i64, stride = 1 : i64, padding = 0 : i64, op1_base = 0 : i64, wr_base = 1 : i64, output_stride = 1 : i64, start_row = 0 : i64, start_col = 0 : i64} + return +} +// CHECK-LABEL: func.func @update_quantized_rows +// CHECK: arith.constant -9223372032556660735 : i64 +// CHECK: return +func.func @update_quantized_rows() { + buckyball.custom "int32_to_int8" {rows = 1 : i64, wr_base = 1 : i64, relu = false} + return +} +// CHECK-LABEL: func.func @define_quantized_rows +// CHECK: arith.constant 4298115073 : i64 +// CHECK: return +func.func @define_quantized_rows() { + buckyball.custom "int32_to_int8" {rows = 1 : i64, wr_base = 0 : i64, relu = false} + return +} diff --git a/examples/designs/pebble/stack/midend/tests/lower-custom-ops.mlir b/examples/designs/pebble/stack/midend/tests/lower-custom-ops.mlir new file mode 100644 index 000000000..788473a76 --- /dev/null +++ b/examples/designs/pebble/stack/midend/tests/lower-custom-ops.mlir @@ -0,0 +1,26 @@ +// CHECK-LABEL: func.func @named +// CHECK: buckyball.bank_mvin +// CHECK-NOT: "fp32_to_int8" +// CHECK: buckyball.bank_custom "matmul_i8" +// CHECK: buckyball.bank_mvout +// CHECK: buckyball.bank_custom "matmul_i8" +// CHECK: buckyball.bank_mvout +// CHECK: return +func.func @named(%a: tensor<1x32xi8>, %b: tensor<32x32xi8>, %initial: tensor<1x32xi32>) -> tensor<1x32xi32> { + %product = custom.op "matmul_i8"(%a, %b, %initial) : (tensor<1x32xi8>, tensor<32x32xi8>, tensor<1x32xi32>) -> tensor<1x32xi32> + return %product : tensor<1x32xi32> +} + +// CHECK-LABEL: func.func @linear +// CHECK: buckyball.bank_custom "fp32_to_int8" +// CHECK: buckyball.bank_custom "matmul_i8" +// CHECK: buckyball.bank_custom "int32_to_fp32" +// CHECK: buckyball.bank_mvout +// CHECK: buckyball.bank_custom "matmul_i8" +// CHECK: buckyball.bank_custom "int32_to_fp32" +// CHECK: buckyball.bank_mvout +// CHECK: return +func.func @linear(%a: tensor<1x32xf32>, %w: tensor<32x32xi8>, %bias: tensor<1x32xi32>, %multiplier: tensor<1x1xf32>, %input_step: tensor<1x1xf32>, %weight_step: tensor<1x32xf32>) -> tensor<1x32xf32> { + %product = custom.op "linear_i8"(%a, %w, %bias, %multiplier, %input_step, %weight_step) : (tensor<1x32xf32>, tensor<32x32xi8>, tensor<1x32xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x32xf32>) -> tensor<1x32xf32> + return %product : tensor<1x32xf32> +} diff --git a/examples/designs/pebble/stack/midend/tests/lower-frontend-types.mlir b/examples/designs/pebble/stack/midend/tests/lower-frontend-types.mlir new file mode 100644 index 000000000..b6ada3b79 --- /dev/null +++ b/examples/designs/pebble/stack/midend/tests/lower-frontend-types.mlir @@ -0,0 +1,27 @@ +func.func @i32_is_not_i8(%codes: tensor<1x16xi32>, %step: tensor<1x16xf32>) -> tensor<1x16xf32> { + // expected-error @+1 {{requires an INT8 input}} + %value = "frontend.int8_to_fp32"(%codes, %step) : (tensor<1x16xi32>, tensor<1x16xf32>) -> tensor<1x16xf32> + return %value : tensor<1x16xf32> +} + +// ----- + +func.func @i8_is_not_i32(%codes: tensor<1x16xi8>, %step: tensor<1x16xf32>) -> tensor<1x16xf32> { + // expected-error @+1 {{requires an INT32 input}} + %value = "frontend.int32_to_fp32"(%codes, %step) : (tensor<1x16xi8>, tensor<1x16xf32>) -> tensor<1x16xf32> + return %value : tensor<1x16xf32> +} + +// ----- + +// VALID-LABEL: func.func @valid_formats +// VALID: arith.sitofp {{.*}} : i8 to f32 +// VALID: arith.mulf +// VALID: arith.sitofp {{.*}} : i32 to f32 +// VALID: arith.mulf +// VALID: return +func.func @valid_formats(%i8_codes: tensor<1x16xi8>, %i32_codes: tensor<1x16xi32>, %step: tensor<1x16xf32>) -> (tensor<1x16xf32>, tensor<1x16xf32>) { + %first = "frontend.int8_to_fp32"(%i8_codes, %step) : (tensor<1x16xi8>, tensor<1x16xf32>) -> tensor<1x16xf32> + %second = "frontend.int32_to_fp32"(%i32_codes, %step) : (tensor<1x16xi32>, tensor<1x16xf32>) -> tensor<1x16xf32> + return %first, %second : tensor<1x16xf32>, tensor<1x16xf32> +} diff --git a/examples/designs/pebble/stack/midend/tests/unknown-custom-op.mlir b/examples/designs/pebble/stack/midend/tests/unknown-custom-op.mlir new file mode 100644 index 000000000..34ab31381 --- /dev/null +++ b/examples/designs/pebble/stack/midend/tests/unknown-custom-op.mlir @@ -0,0 +1,5 @@ +func.func @unknown(%input: tensor<1x32xf32>) -> tensor<1x32xf32> { + // expected-error @+1 {{has no bank lowering on this core for unknown}} + %result = custom.op "unknown"(%input) : (tensor<1x32xf32>) -> tensor<1x32xf32> + return %result : tensor<1x32xf32> +} diff --git a/examples/designs/pebble/workloads/CMakeLists.txt b/examples/designs/pebble/workloads/CMakeLists.txt new file mode 100644 index 000000000..1d19718bb --- /dev/null +++ b/examples/designs/pebble/workloads/CMakeLists.txt @@ -0,0 +1,24 @@ +get_filename_component(BUCKYBALL_REPO_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../.. ABSOLUTE) + +set(BUCKYBALL_DESIGN_COMMON_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/common/pebble.c) +set(BUCKYBALL_DESIGN_COMMON_HEADERS + ${CMAKE_CURRENT_SOURCE_DIR}/common/pebble.h) +set(BUCKYBALL_DESIGN_COMMON_INCLUDE_DIRS + ${CMAKE_CURRENT_SOURCE_DIR}/common) +set(BUCKYBALL_CTEST_TARGET pebble) + +add_custom_target(pebble-ctests-build) +set(BUCKYBALL_CTEST_GROUP_TARGET pebble-ctests-build) + +add_subdirectory(device) + +add_custom_target(design-ctests-build DEPENDS pebble-ctests-build) + +add_custom_target(design-workloads-build ALL + DEPENDS + design-ctests-build + mlirtest-workloads-build + soctests-build + COMMENT "Building pebble workloads" + VERBATIM) diff --git a/examples/chips/pebble/workloads/common/pebble.c b/examples/designs/pebble/workloads/common/pebble.c similarity index 100% rename from examples/chips/pebble/workloads/common/pebble.c rename to examples/designs/pebble/workloads/common/pebble.c diff --git a/examples/chips/pebble/workloads/common/pebble.h b/examples/designs/pebble/workloads/common/pebble.h similarity index 100% rename from examples/chips/pebble/workloads/common/pebble.h rename to examples/designs/pebble/workloads/common/pebble.h diff --git a/examples/designs/pebble/workloads/configs/tests.toml b/examples/designs/pebble/workloads/configs/tests.toml new file mode 100644 index 000000000..c1c7ceb33 --- /dev/null +++ b/examples/designs/pebble/workloads/configs/tests.toml @@ -0,0 +1,2 @@ +controlHart = 0 +reportHart = 0 diff --git a/examples/designs/pebble/workloads/device/CMakeLists.txt b/examples/designs/pebble/workloads/device/CMakeLists.txt new file mode 100644 index 000000000..ad6d4787c --- /dev/null +++ b/examples/designs/pebble/workloads/device/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(core) diff --git a/examples/designs/pebble/workloads/device/core/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/CMakeLists.txt new file mode 100644 index 000000000..2a6ad3b6a --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(acc) diff --git a/examples/designs/pebble/workloads/device/core/acc/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/CMakeLists.txt new file mode 100644 index 000000000..7b5637b41 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/CMakeLists.txt @@ -0,0 +1,10 @@ +add_subdirectory(ctests) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_TARGET}") + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_TARGET_BALLS_${BUCKYBALL_TARGET}) + add_subdirectory( + "${BUCKYBALL_REPO_ROOT}/examples/components/balls/${BUCKYBALL_BALL_DIR}/workloads/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}/ctests") + endforeach() +endforeach() +add_subdirectory(mlir_tests) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/CMakeLists.txt new file mode 100644 index 000000000..670217993 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/CMakeLists.txt @@ -0,0 +1,28 @@ +add_buckyball_ctest(bank_capacity_rename_test.c) +add_buckyball_ctest(bank_recycle_test.c) +add_buckyball_ctest(bank_transfer_recycle_test.c) +add_buckyball_ctest(mega_conv_pipeline_test.c) +add_buckyball_ctest(mega_conv_k5_pipeline_test.c) +add_buckyball_ctest(residual_add_lut_test.c) +add_buckyball_ctest(maxpool_im2col_test.c) +add_buckyball_ctest(se_gate_lut_test.c) +add_buckyball_ctest(dual_smatmul_matadd_test.c) +add_buckyball_ctest(dual_conv_add_int2fp_test.c) +add_buckyball_ctest(quant_smatmul_int2fp_test.c) +add_buckyball_ctest(smatmul_mvout_bank_reuse_test.c) +add_buckyball_ctest(mvin_bank78_stress_test.c) +add_buckyball_ctest(im2col_mvin_smatmul_hazard_test.c) + +add_subdirectory(im2col_smatmul_quant_pipeline) +add_subdirectory(maxpool_partial_update) +add_subdirectory(bank_dma_reuse) +add_subdirectory(bank_rename_overlap) +add_subdirectory(bank_dma_pingpong) +add_subdirectory(read_bank_shared) +add_subdirectory(read_bank_split) +add_subdirectory(same_ball_independent) +add_subdirectory(rob_window_probe) +add_subdirectory(write_bank_versions) +if(DESIGN_TARGET_SHARED_BANKS_${BUCKYBALL_CTEST_TARGET} GREATER 0) + add_subdirectory(shared_bank_versions) +endif() diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_capacity_rename_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_capacity_rename_test.c new file mode 100644 index 000000000..c94c66a8e --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_capacity_rename_test.c @@ -0,0 +1,36 @@ +#include +#include +#include +#include + +_Static_assert(VIRTUAL_BANK_NUM >= BANK_NUM, + "Test requires one virtual ID per physical bank"); +static uint8_t input[BANK_NUM][BANK_WIDTH / 8] __attribute__((aligned(64))); +static uint8_t output[BANK_NUM][BANK_WIDTH / 8] __attribute__((aligned(64))); + +int main(void) { + for (unsigned bank = 0; bank < BANK_NUM; ++bank) + bb_mem_alloc(bank, 1, 1); + for (unsigned round = 0; round < 3; ++round) { + for (unsigned bank = 0; bank < BANK_NUM; ++bank) + for (unsigned byte = 0; byte < BANK_WIDTH / 8; ++byte) { + input[bank][byte] = bank * 7 + byte + round * 31; + output[bank][byte] = 0; + } + for (unsigned bank = 0; bank < BANK_NUM; ++bank) + bb_mvin((uintptr_t)input[bank], bank, 1, 1); + for (unsigned bank = 0; bank < BANK_NUM; ++bank) + bb_mvout((uintptr_t)output[bank], bank, 1, 1); + for (unsigned bank = 0; bank < BANK_NUM; ++bank) + for (unsigned byte = 0; byte < BANK_WIDTH / 8; ++byte) + if (output[bank][byte] != input[bank][byte]) { + printf("capacity rename mismatch round=%u bank=%u byte=%u\n", round, + bank, byte); + return 1; + } + } + for (unsigned bank = 0; bank < BANK_NUM; ++bank) + bb_mem_release(bank); + puts("Bank capacity rename PASS"); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/CMakeLists.txt new file mode 100644 index 000000000..f789e46c0 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(bank_dma_pingpong SOURCES + bank_dma_pingpong-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/bank_dma_pingpong-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/bank_dma_pingpong-main.cpp new file mode 100644 index 000000000..b38e76558 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/bank_dma_pingpong-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == BB_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/pebble.c new file mode 100644 index 000000000..c70aff2d4 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/pebble.c @@ -0,0 +1,55 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { M = 8, N = 16, K = 64, LANES = BANK_WIDTH / 8 }; +static int8_t input[BB_CORE_NUM][M * K] __attribute__((aligned(64))); +static int8_t first[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +static int8_t second[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_CORE_NUM][N] __attribute__((aligned(64))); +static int32_t out0[BB_CORE_NUM][M * N] __attribute__((aligned(64))); +static int32_t out1[BB_CORE_NUM][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_CORE_NUM][N] __attribute__((aligned(64))); +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < M * K; ++i) + input[c][i] = 1; + for (int i = 0; i < K * N; ++i) { + first[c][i] = 2; + second[c][i] = 3; + } + for (int bank = 0; bank < 6; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], 0, M * K / LANES, 1); + bb_mvin((uintptr_t)first[c], 1, K * N / LANES, 1); + bb_mvin((uintptr_t)bias[c], 2, N * 4 / LANES, 1); + bb_smatmul_bias(2, 0); + bb_smatmul_os(0, 1, 3, 1, N, N, 1, 1, 0); + bb_mvout((uintptr_t)warmup[c], 3, N * 4 / LANES, 1); + if (warmup[c][0] != 2 * N) + return 1; + uint64_t start, end; + asm volatile("rdcycle %0" : "=r"(start)); + bb_smatmul_os(0, 1, 3, M, N, K, 1, 1, 0); + bb_mvin((uintptr_t)second[c], 5, K * N / LANES, 1); + bb_smatmul_os(0, 5, 4, M, N, K, 1, 1, 0); + bb_mvout((uintptr_t)out0[c], 3, M * N * 4 / LANES, 1); + bb_mvout((uintptr_t)out1[c], 4, M * N * 4 / LANES, 1); + for (int i = 0; i < M * N; ++i) + if (out0[c][i] != 2 * K || out1[c][i] != 3 * K) { + printf("bank_dma_pingpong FAIL index=%d actual=%d,%d\n", i, out0[c][i], + out1[c][i]); + return 1; + } + asm volatile("rdcycle %0" : "=r"(end)); + for (int bank = 0; bank < 6; ++bank) + bb_mem_release(bank); + printf("bank_dma_pingpong PASS cycles=%llu\n", + (unsigned long long)(end - start)); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/CMakeLists.txt new file mode 100644 index 000000000..c7565bbcf --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(bank_dma_reuse SOURCES + bank_dma_reuse-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/bank_dma_reuse-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/bank_dma_reuse-main.cpp new file mode 100644 index 000000000..b38e76558 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/bank_dma_reuse-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == BB_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/pebble.c new file mode 100644 index 000000000..bea6e5826 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/pebble.c @@ -0,0 +1,62 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { M = 8, N = 16, K = 64, LANES = BANK_WIDTH / 8 }; +static int8_t input[BB_CORE_NUM][M * K] __attribute__((aligned(64))); +static int8_t first[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +static int8_t second[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_CORE_NUM][N] __attribute__((aligned(64))); +static int32_t out0[BB_CORE_NUM][M * N] __attribute__((aligned(64))); +static int32_t out1[BB_CORE_NUM][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_CORE_NUM][N] __attribute__((aligned(64))); +static volatile int8_t moved[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < M * K; ++i) + input[c][i] = 1; + for (int i = 0; i < K * N; ++i) { + first[c][i] = 2; + second[c][i] = 3; + } + for (int bank = 0; bank < 6; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], 0, M * K / LANES, 1); + bb_mvin((uintptr_t)first[c], 1, K * N / LANES, 1); + bb_mvin((uintptr_t)bias[c], 2, N * 4 / LANES, 1); + bb_smatmul_bias(2, 0); + bb_smatmul_os(0, 1, 3, 1, N, N, 1, 1, 0); + bb_mvout((uintptr_t)warmup[c], 3, N * 4 / LANES, 1); + if (warmup[c][0] != 2 * N) + return 1; + uint64_t start, end; + asm volatile("rdcycle %0" : "=r"(start)); + bb_smatmul_os(0, 1, 3, M, N, K, 1, 1, 0); + bb_mvin((uintptr_t)second[c], 1, K * N / LANES, 1); + bb_smatmul_os(0, 1, 4, M, N, K, 1, 1, 0); + bb_mvout((uintptr_t)out0[c], 3, M * N * 4 / LANES, 1); + bb_mvout((uintptr_t)out1[c], 4, M * N * 4 / LANES, 1); + for (int i = 0; i < M * N; ++i) + if (out0[c][i] != 2 * K || out1[c][i] != 3 * K) { + printf("bank_dma_reuse FAIL index=%d actual=%d,%d\n", i, out0[c][i], + out1[c][i]); + return 1; + } + asm volatile("rdcycle %0" : "=r"(end)); + bb_mem_transfer(1, 6); + bb_mvout((uintptr_t)moved[c], 6, K * N / LANES, 1); + for (int i = 0; i < K * N; ++i) + if (moved[c][i] != 3) + return 1; + for (int bank = 0; bank < 7; ++bank) + if (bank != 1) + bb_mem_release(bank); + printf("bank_dma_reuse PASS cycles=%llu\n", + (unsigned long long)(end - start)); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_recycle_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_recycle_test.c new file mode 100644 index 000000000..76a9aef7f --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_recycle_test.c @@ -0,0 +1,25 @@ +#include +#include +#include +#include + +static uint64_t input[2] __attribute__((aligned(64))); +static uint64_t output[2] __attribute__((aligned(64))); + +int main(void) { + for (unsigned iteration = 0; iteration < 128; ++iteration) { + input[0] = 0x1234567800000000ULL + iteration; + input[1] = ~input[0]; + output[0] = output[1] = 0; + bb_mem_alloc(1, 1, 1); + bb_mvin((uintptr_t)input, 1, 1, 1); + bb_mvout((uintptr_t)output, 1, 1, 1); + if (output[0] != input[0] || output[1] != input[1]) { + printf("bank recycle mismatch iteration=%u\n", iteration); + return 1; + } + bb_mem_release(1); + } + puts("Bank recycle PASS"); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/CMakeLists.txt new file mode 100644 index 000000000..a38b3c993 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(bank_rename_overlap SOURCES + bank_rename_overlap-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/bank_rename_overlap-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/bank_rename_overlap-main.cpp new file mode 100644 index 000000000..b38e76558 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/bank_rename_overlap-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == BB_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/pebble.c new file mode 100644 index 000000000..c8d99e78c --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/pebble.c @@ -0,0 +1,54 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include + +enum { M = 8, N = 16, K = 64, LANES = BANK_WIDTH / 8 }; +static int8_t input[BB_CORE_NUM][M * K] __attribute__((aligned(64))); +static int8_t first[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +static int8_t second[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_CORE_NUM][N] __attribute__((aligned(64))); +static int32_t out0[BB_CORE_NUM][M * N] __attribute__((aligned(64))); +static int32_t out1[BB_CORE_NUM][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_CORE_NUM][N] __attribute__((aligned(64))); +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < M * K; ++i) + input[c][i] = 1; + for (int i = 0; i < K * N; ++i) { + first[c][i] = 2; + second[c][i] = 3; + } + for (int bank = 0; bank < 5; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], 0, M * K / LANES, 1); + bb_mvin((uintptr_t)first[c], 1, K * N / LANES, 1); + bb_mvin((uintptr_t)bias[c], 2, N * 4 / LANES, 1); + bb_smatmul_bias(2, 0); + bb_smatmul_os(0, 1, 3, 1, N, N, 1, 1, 0); + bb_mvout((uintptr_t)warmup[c], 3, N * 4 / LANES, 1); + if (warmup[c][0] != 2 * N) + return 1; + uint64_t start, end; + asm volatile("rdcycle %0" : "=r"(start)); + bb_smatmul_os(0, 1, 3, M, N, K, 1, 1, 0); + bb_mvin((uintptr_t)second[c], 1, K * N / LANES, 1); + bb_smatmul_os(0, 1, 4, M, N, K, 1, 1, 0); + bb_mvout((uintptr_t)out0[c], 3, M * N * 4 / LANES, 1); + bb_mvout((uintptr_t)out1[c], 4, M * N * 4 / LANES, 1); + for (int i = 0; i < M * N; ++i) + if (out0[c][i] != 2 * K || out1[c][i] != 3 * K) { + printf("bank_rename_overlap FAIL index=%d actual=%d,%d\n", i, out0[c][i], + out1[c][i]); + return 1; + } + asm volatile("rdcycle %0" : "=r"(end)); + for (int bank = 0; bank < 5; ++bank) + bb_mem_release(bank); + printf("bank_rename_overlap PASS cycles=%llu\n", + (unsigned long long)(end - start)); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_transfer_recycle_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_transfer_recycle_test.c new file mode 100644 index 000000000..e30241516 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_transfer_recycle_test.c @@ -0,0 +1,31 @@ +#include +#include +#include +#include + +static uint64_t source[2] __attribute__((aligned(64))); +static uint64_t target[2] __attribute__((aligned(64))); +static uint64_t output[4] __attribute__((aligned(64))); + +int main(void) { + for (unsigned iteration = 0; iteration < 128; ++iteration) { + source[0] = 0x1234000000000000ULL + iteration; + source[1] = ~source[0]; + target[0] = 0x5678000000000000ULL + iteration; + target[1] = ~target[0]; + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 1); + bb_mvin((uintptr_t)source, 0, 1, 1); + bb_mvin((uintptr_t)target, 1, 1, 1); + bb_mem_transfer(0, 1); + bb_mvout((uintptr_t)output, 1, 1, 1); + if (output[0] != target[0] || output[1] != target[1] || + output[2] != source[0] || output[3] != source[1]) { + printf("bank transfer recycle mismatch iteration=%u\n", iteration); + return 1; + } + bb_mem_release(1); + } + puts("Bank transfer recycle PASS"); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/dual_conv_add_int2fp_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/dual_conv_add_int2fp_test.c new file mode 100644 index 000000000..077463b26 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/dual_conv_add_int2fp_test.c @@ -0,0 +1,80 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + CORE = 16, + IN = 4, + K = 3, + OUT = 2, + WINDOWS = 4, + PACKED_K = 16, + RESULT_ROWS = CORE * 2 +}; +static int8_t images[2][IN * IN * CORE] __attribute__((aligned(64))); +static int8_t weights[2][PACKED_K * CORE] __attribute__((aligned(64))); +static float scale[CORE] __attribute__((aligned(64))); +static int32_t bias[CORE] __attribute__((aligned(64))); +static float output[CORE * CORE] __attribute__((aligned(64))); + +static int8_t image_value(int branch, int row, int column) { + return (int8_t)(branch * 7 + row * IN + column - 8); +} + +static int32_t expected(int row, int column) { + int32_t sum = 0; + for (int branch = 0; branch < 2; ++branch) + for (int ky = 0; ky < K; ++ky) + for (int kx = 0; kx < K; ++kx) + sum += image_value(branch, row / OUT + ky, row % OUT + kx) * + weights[branch][(ky * K + kx) * CORE + column]; + return sum; +} + +static void run_branch(int branch, int out_bank) { + bb_mvin((uintptr_t)images[branch], 0, IN * IN, 1); + bb_im2col(0, 1, IN, K, 1, 0, 0, 0, 0, 0, 0, WINDOWS); + bb_mvin((uintptr_t)weights[branch], 0, PACKED_K, 1); + bb_smatmul_os(1, 0, out_bank, CORE, CORE, PACKED_K, 1, 1, 0); +} + +int main(void) { + for (int branch = 0; branch < 2; ++branch) { + for (int row = 0; row < IN; ++row) + for (int column = 0; column < IN; ++column) + images[branch][(row * IN + column) * CORE] = + image_value(branch, row, column); + for (int i = 0; i < PACKED_K * CORE; ++i) + weights[branch][i] = (int8_t)((i / CORE * 3 + i % CORE) % 41 - 20); + } + for (int lane = 0; lane < CORE; ++lane) + scale[lane] = 0.05f; + + for (int bank = 0; bank < 8; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)scale, 3, 4, 1); + bb_mvin((uintptr_t)bias, 7, 4, 1); + bb_smatmul_bias(7, 0); + run_branch(0, 2); + run_branch(1, 4); + bb_matadd(2, 4, 6, RESULT_ROWS); + bb_int32_to_fp32(6, 3, 5, RESULT_ROWS, 0); + bb_mvout((uintptr_t)output, 5, RESULT_ROWS, 1); + + for (int row = 0; row < WINDOWS; ++row) + for (int column = 0; column < CORE; ++column) + if (fabsf(output[row * CORE + column] - + expected(row, column) * scale[column]) > 1e-3f) { + printf("dual conv add mismatch row=%d column=%d\n", row, column); + return 1; + } + printf("dual conv add int2fp PASS\n"); + return 0; +} diff --git a/examples/chips/pebble/workloads/ctests/dual_smatmul_matadd_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/dual_smatmul_matadd_test.c similarity index 100% rename from examples/chips/pebble/workloads/ctests/dual_smatmul_matadd_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/dual_smatmul_matadd_test.c diff --git a/examples/chips/pebble/workloads/ctests/im2col_mvin_smatmul_hazard_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_mvin_smatmul_hazard_test.c similarity index 100% rename from examples/chips/pebble/workloads/ctests/im2col_mvin_smatmul_hazard_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/im2col_mvin_smatmul_hazard_test.c diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/CMakeLists.txt new file mode 100644 index 000000000..ae4e87813 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(im2col_smatmul_quant_pipeline SOURCES + im2col_smatmul_quant_pipeline-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp new file mode 100644 index 000000000..b38e76558 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == BB_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/pebble.c new file mode 100644 index 000000000..e19dcd49b --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/pebble.c @@ -0,0 +1,61 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +#if BANK_WIDTH != 128 || BANK_LINES < 64 +#error "Pebble thread pipeline requires 128-bit banks with at least 64 rows" +#endif + +enum { LANES = BANK_WIDTH / 8, INPUT_ROWS = 36, WINDOWS = 16 }; +static int8_t input[BB_CORE_NUM][INPUT_ROWS * LANES] + __attribute__((aligned(64))); +static int8_t weight[BB_CORE_NUM][LANES * LANES] __attribute__((aligned(64))); +static int32_t bias[BB_CORE_NUM][LANES] __attribute__((aligned(64))); +static float scale[BB_CORE_NUM][LANES] __attribute__((aligned(64))); +static int8_t output[BB_CORE_NUM][WINDOWS * LANES] __attribute__((aligned(64))); + +#ifdef __cplusplus +extern "C" +#endif + int pebble_pipeline(uint32_t instance) { + int thread = instance; + for (int row = 0; row < INPUT_ROWS; ++row) + input[thread][row * LANES] = (3 * row + 1) % 7 - 3; + for (int channel = 0; channel < LANES; ++channel) { + bias[thread][channel] = channel - 8; + scale[thread][channel] = 1.0f; + for (int k = 0; k < LANES; ++k) + weight[thread][k * LANES + channel] = + k < 9 ? (2 * k + 3 * channel) % 5 - 2 : 0; + } + for (int bank = 0; bank < 7; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[thread], 0, INPUT_ROWS, 1); + bb_mvin((uintptr_t)weight[thread], 2, LANES, 1); + bb_mvin((uintptr_t)bias[thread], 3, LANES * 32 / BANK_WIDTH, 1); + bb_mvin((uintptr_t)scale[thread], 5, LANES * 32 / BANK_WIDTH, 1); + bb_im2col(0, 1, 6, 3, 1, 0, 0, 0, 0, 0, 0, WINDOWS); + bb_smatmul_bias(3, 0); + bb_smatmul_os(1, 2, 4, WINDOWS, LANES, LANES, 1, 1, 0); + bb_quant_i32_to_i8(4, 5, 6, WINDOWS * 4, 0, 0, 4, 4, 4, 0); + bb_mvout((uintptr_t)output[thread], 6, WINDOWS, 1); + for (int window = 0; window < WINDOWS; ++window) + for (int channel = 0; channel < LANES; ++channel) { + int expected = bias[thread][channel]; + int y = window / 4, x = window % 4; + for (int ky = 0; ky < 3; ++ky) + for (int kx = 0; kx < 3; ++kx) { + int k = ky * 3 + kx, row = (y + ky) * 6 + x + kx; + expected += + input[thread][row * LANES] * weight[thread][k * LANES + channel]; + } + if (output[thread][window * LANES + channel] != expected) + return 1; + } + return 0; +} diff --git a/examples/chips/pebble/workloads/ctests/maxpool_im2col_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_im2col_test.c similarity index 100% rename from examples/chips/pebble/workloads/ctests/maxpool_im2col_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_im2col_test.c diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/CMakeLists.txt new file mode 100644 index 000000000..5374aea3c --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(maxpool_partial_update SOURCES + maxpool_partial_update-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/maxpool_partial_update-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/maxpool_partial_update-main.cpp new file mode 100644 index 000000000..b38e76558 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/maxpool_partial_update-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == BB_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/pebble.c new file mode 100644 index 000000000..7a3caef34 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/pebble.c @@ -0,0 +1,40 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include + +enum { ROWS = 16, LANES = BANK_WIDTH / 8 }; +static int8_t input[BB_CORE_NUM][2 * LANES] __attribute__((aligned(64))); +static int8_t initial[BB_CORE_NUM][ROWS * LANES] __attribute__((aligned(64))); +static int8_t output[BB_CORE_NUM][ROWS * LANES] __attribute__((aligned(64))); + +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < 2 * LANES; ++i) + input[c][i] = i - LANES; + for (int i = 0; i < ROWS * LANES; ++i) + initial[c][i] = -37; + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 1); + bb_mvin((uintptr_t)input[c], 0, 2, 1); + bb_mvin((uintptr_t)initial[c], 1, ROWS, 1); + bb_maxpool(0, 1, 2, 1, 1, 1, 0, 0, 5, 1, 0, 0); + bb_maxpool(0, 1, 2, 1, 1, 1, 0, 1, 11, 1, 0, 0); + bb_mvout((uintptr_t)output[c], 1, ROWS, 1); + for (int row = 0; row < ROWS; ++row) + for (int lane = 0; lane < LANES; ++lane) { + int8_t expected = row == 5 ? input[c][lane] + : row == 11 ? input[c][LANES + lane] + : -37; + if (output[c][row * LANES + lane] != expected) { + printf("maxpool_partial_update FAIL row=%d lane=%d\n", row, lane); + return 1; + } + } + bb_mem_release(0); + bb_mem_release(1); + printf("maxpool_partial_update PASS\n"); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_k5_pipeline_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_k5_pipeline_test.c new file mode 100644 index 000000000..aa3183c70 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_k5_pipeline_test.c @@ -0,0 +1,70 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include +#include + +enum { INPUT_SIDE = 8, OUTPUT_SIDE = 4, KERNEL = 5, CORE = 16, PADDED_K = 32 }; +static int8_t input[INPUT_SIDE * INPUT_SIDE * CORE] + __attribute__((aligned(64))); +static int8_t weight[PADDED_K * CORE] __attribute__((aligned(64))); +static int32_t bias[CORE] __attribute__((aligned(64))); +static float scale[CORE] __attribute__((aligned(64))); +static int8_t output[OUTPUT_SIDE * OUTPUT_SIDE * CORE] + __attribute__((aligned(64))); + +int main(void) { + for (int pixel = 0; pixel < INPUT_SIDE * INPUT_SIDE; ++pixel) + input[pixel * CORE] = pixel % 5 - 2; + for (int k = 0; k < KERNEL * KERNEL; ++k) + for (int channel = 0; channel < CORE; ++channel) + weight[k * CORE + channel] = (k + channel) % 3 - 1; + for (int channel = 0; channel < CORE; ++channel) + scale[channel] = 1.0f; + + for (int bank = 0; bank < 8; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input, 0, INPUT_SIDE * INPUT_SIDE, 1); + bb_mvin((uintptr_t)weight, 2, PADDED_K, 1); + bb_mvin((uintptr_t)bias, 3, 4, 1); + bb_mvin((uintptr_t)scale, 5, 4, 1); + bb_im2col(0, 1, INPUT_SIDE, KERNEL, 1, 0, 0, 0, 0, 0, 0, + OUTPUT_SIDE * OUTPUT_SIDE); + for (int row_core = 0; row_core < 2; ++row_core) + for (int k_core = 0; k_core < 2; ++k_core) + for (int row = 0; row < 8; ++row) { + int input_row = k_core * 16 + row_core * 8 + row; + int output_row = (row_core * 2 + k_core) * 8 + row; + bb_maxpool(1, 7, 1, 1, 1, 1, 0, input_row, output_row, 1, 0, 0); + } + bb_smatmul_bias(3, 0); + bb_smatmul_os(7, 2, 4, OUTPUT_SIDE * OUTPUT_SIDE, CORE, PADDED_K, 1, 1, 0); + bb_quant_i32_to_i8(4, 5, 6, OUTPUT_SIDE * OUTPUT_SIDE * 4, 0, 0, OUTPUT_SIDE, + OUTPUT_SIDE, OUTPUT_SIDE, 0); + bb_mvout((uintptr_t)output, 6, OUTPUT_SIDE * OUTPUT_SIDE, 1); + + for (int y = 0; y < OUTPUT_SIDE; ++y) + for (int x = 0; x < OUTPUT_SIDE; ++x) + for (int channel = 0; channel < CORE; ++channel) { + int expected = 0; + for (int ky = 0; ky < KERNEL; ++ky) + for (int kx = 0; kx < KERNEL; ++kx) { + int pixel = (y + ky) * INPUT_SIDE + x + kx; + int k = ky * KERNEL + kx; + expected += input[pixel * CORE] * weight[k * CORE + channel]; + } + int index = (y * OUTPUT_SIDE + x) * CORE + channel; + if (output[index] != expected) { + printf( + "mega_conv_k5 FAIL y=%d x=%d channel=%d expected=%d actual=%d\n", + y, x, channel, expected, output[index]); + return 1; + } + } + printf("mega_conv_k5 PASS\n"); + return 0; +} diff --git a/examples/chips/pebble/workloads/ctests/mega_conv_pipeline_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_pipeline_test.c similarity index 100% rename from examples/chips/pebble/workloads/ctests/mega_conv_pipeline_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_pipeline_test.c diff --git a/examples/chips/pebble/workloads/ctests/mvin_bank78_stress_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/mvin_bank78_stress_test.c similarity index 100% rename from examples/chips/pebble/workloads/ctests/mvin_bank78_stress_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/mvin_bank78_stress_test.c diff --git a/examples/chips/pebble/workloads/ctests/quant_smatmul_int2fp_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/quant_smatmul_int2fp_test.c similarity index 100% rename from examples/chips/pebble/workloads/ctests/quant_smatmul_int2fp_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/quant_smatmul_int2fp_test.c diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/CMakeLists.txt new file mode 100644 index 000000000..04f041e79 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(read_bank_shared SOURCES + read_bank_shared-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/pebble.c new file mode 100644 index 000000000..461d5b99e --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/pebble.c @@ -0,0 +1,52 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { M = 8, N = 16, K = 16, LANES = BANK_WIDTH / 8 }; +static int8_t input[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +static int8_t weight[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_CORE_NUM][N] __attribute__((aligned(64))); +static int8_t transposed[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +static int32_t product[BB_CORE_NUM][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_CORE_NUM][N] __attribute__((aligned(64))); +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < K * N; ++i) { + input[c][i] = 1; + weight[c][i] = 2; + } + for (int bank = 0; bank < 6; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], 0, K * N / LANES, 1); + bb_mvin((uintptr_t)input[c], 4, K * N / LANES, 1); + bb_mvin((uintptr_t)weight[c], 2, K * N / LANES, 1); + bb_mvin((uintptr_t)bias[c], 5, N * 4 / LANES, 1); + bb_smatmul_bias(5, 0); + bb_smatmul_os(0, 2, 3, 1, N, N, 1, 1, 0); + bb_mvout((uintptr_t)warmup[c], 3, N * 4 / LANES, 1); + if (warmup[c][0] != 2 * N) + return 1; + uint64_t start, end; + asm volatile("rdcycle %0" : "=r"(start)); + bb_transpose(0, 1, K * N / LANES, 8); + bb_smatmul_os(0, 2, 3, M, N, K, 1, 1, 0); + bb_mvout((uintptr_t)transposed[c], 1, K * N / LANES, 1); + bb_mvout((uintptr_t)product[c], 3, M * N * 4 / LANES, 1); + for (int i = 0; i < K * N; ++i) + if (transposed[c][i] != 1) + return 1; + for (int i = 0; i < M * N; ++i) + if (product[c][i] != 2 * K) + return 1; + asm volatile("rdcycle %0" : "=r"(end)); + for (int bank = 0; bank < 6; ++bank) + bb_mem_release(bank); + printf("read_bank_shared PASS cycles=%llu\n", + (unsigned long long)(end - start)); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/read_bank_shared-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/read_bank_shared-main.cpp new file mode 100644 index 000000000..b38e76558 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/read_bank_shared-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == BB_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/CMakeLists.txt new file mode 100644 index 000000000..25c5eb017 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(read_bank_split SOURCES + read_bank_split-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/pebble.c new file mode 100644 index 000000000..2c154df41 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/pebble.c @@ -0,0 +1,52 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { M = 8, N = 16, K = 16, LANES = BANK_WIDTH / 8 }; +static int8_t input[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +static int8_t weight[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_CORE_NUM][N] __attribute__((aligned(64))); +static int8_t transposed[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +static int32_t product[BB_CORE_NUM][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_CORE_NUM][N] __attribute__((aligned(64))); +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < K * N; ++i) { + input[c][i] = 1; + weight[c][i] = 2; + } + for (int bank = 0; bank < 6; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], 0, K * N / LANES, 1); + bb_mvin((uintptr_t)input[c], 4, K * N / LANES, 1); + bb_mvin((uintptr_t)weight[c], 2, K * N / LANES, 1); + bb_mvin((uintptr_t)bias[c], 5, N * 4 / LANES, 1); + bb_smatmul_bias(5, 0); + bb_smatmul_os(0, 2, 3, 1, N, N, 1, 1, 0); + bb_mvout((uintptr_t)warmup[c], 3, N * 4 / LANES, 1); + if (warmup[c][0] != 2 * N) + return 1; + uint64_t start, end; + asm volatile("rdcycle %0" : "=r"(start)); + bb_transpose(0, 1, K * N / LANES, 8); + bb_smatmul_os(4, 2, 3, M, N, K, 1, 1, 0); + bb_mvout((uintptr_t)transposed[c], 1, K * N / LANES, 1); + bb_mvout((uintptr_t)product[c], 3, M * N * 4 / LANES, 1); + for (int i = 0; i < K * N; ++i) + if (transposed[c][i] != 1) + return 1; + for (int i = 0; i < M * N; ++i) + if (product[c][i] != 2 * K) + return 1; + asm volatile("rdcycle %0" : "=r"(end)); + for (int bank = 0; bank < 6; ++bank) + bb_mem_release(bank); + printf("read_bank_split PASS cycles=%llu\n", + (unsigned long long)(end - start)); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/read_bank_split-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/read_bank_split-main.cpp new file mode 100644 index 000000000..b38e76558 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/read_bank_split-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == BB_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/chips/pebble/workloads/ctests/residual_add_lut_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/residual_add_lut_test.c similarity index 100% rename from examples/chips/pebble/workloads/ctests/residual_add_lut_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/residual_add_lut_test.c diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/CMakeLists.txt new file mode 100644 index 000000000..401f7629c --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(rob_window_probe SOURCES + rob_window_probe-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/pebble.c new file mode 100644 index 000000000..0614a56e5 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/pebble.c @@ -0,0 +1,57 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { M = 8, N = 16, K = 64, LANES = BANK_WIDTH / 8, STORES = 16 }; +static int8_t input[BB_CORE_NUM][M * K] __attribute__((aligned(64))); +static int8_t weight[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +static int8_t core[BB_CORE_NUM][N * N] __attribute__((aligned(64))); +static int32_t bias[BB_CORE_NUM][N] __attribute__((aligned(64))); +static int32_t output[BB_CORE_NUM][STORES][M * N] __attribute__((aligned(64))); +static int8_t transposed[BB_CORE_NUM][N * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_CORE_NUM][N] __attribute__((aligned(64))); +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < M * K; ++i) + input[c][i] = 1; + for (int i = 0; i < K * N; ++i) + weight[c][i] = 2; + for (int i = 0; i < N * N; ++i) + core[c][i] = 1; + for (int bank = 0; bank < 6; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], 0, M * K / LANES, 1); + bb_mvin((uintptr_t)weight[c], 1, K * N / LANES, 1); + bb_mvin((uintptr_t)core[c], 3, N * N / LANES, 1); + bb_mvin((uintptr_t)bias[c], 5, N * 4 / LANES, 1); + bb_smatmul_bias(5, 0); + bb_smatmul_os(0, 1, 2, 1, N, N, 1, 1, 0); + bb_mvout((uintptr_t)warmup[c], 2, N * 4 / LANES, 1); + if (warmup[c][0] != 2 * N) + return 1; + uint64_t start, end; + asm volatile("rdcycle %0" : "=r"(start)); + bb_smatmul_os(0, 1, 2, M, N, K, 1, 1, 0); + for (int i = 0; i < STORES; ++i) + bb_mvout((uintptr_t)output[c][i], 2, M * N * 4 / LANES, 1); + bb_transpose(3, 4, N * N / LANES, 8); + bb_mvout((uintptr_t)transposed[c], 4, N * N / LANES, 1); + for (int s = 0; s < STORES; ++s) + for (int i = 0; i < M * N; ++i) + if (output[c][s][i] != 2 * K) + return 1; + for (int i = 0; i < N * N; ++i) + if (transposed[c][i] != 1) + return 1; + asm volatile("rdcycle %0" : "=r"(end)); + for (int bank = 0; bank < 6; ++bank) + bb_mem_release(bank); + printf("rob_window_probe PASS cycles=%llu\n", + (unsigned long long)(end - start)); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/rob_window_probe-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/rob_window_probe-main.cpp new file mode 100644 index 000000000..b38e76558 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/rob_window_probe-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == BB_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/runtime_job_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/runtime_job_test.c new file mode 100644 index 000000000..87e08e410 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/runtime_job_test.c @@ -0,0 +1,53 @@ +// Two runtime jobs on the core under test: the first moves data through a +// bank of its accelerator, the second depends on it and checks the data. +#include +#include +#include +#include +#include +#include + +static uint64_t input[2] __attribute__((aligned(64))); +static uint64_t output[2] __attribute__((aligned(64))); + +static int move(void *core, void *argument) { + (void)core; + (void)argument; + bb_mem_alloc(1, 1, 1); + bb_mvin((uintptr_t)input, 1, 1, 1); + bb_mvout((uintptr_t)output, 1, 1, 1); + bb_mem_release(1); + return 0; +} + +static int check(void *core, void *argument) { + (void)core; + (void)argument; + return output[0] != input[0] || output[1] != input[1]; +} + +int main(void) { + input[0] = 0x0123456789abcdefULL; + input[1] = ~input[0]; + const uint64_t signature = bb_cores[0].signature; + const struct runtime_core core = + runtime_core(bb_cores[0].hart, signature, NULL); + static const size_t after_move[] = {0}; + const struct runtime_job jobs[] = { + {signature, RUNTIME_ANY, move, NULL, NULL, 0}, + {signature, RUNTIME_ANY, check, NULL, after_move, 1}, + }; + struct runtime_result results[2]; + struct runtime runtime; + runtime_init(&runtime, &core, 1, jobs, 2, results); + runtime_baremetal_run(&runtime, 0); + for (unsigned job = 0; job < 2; ++job) { + if (results[job].state != RUNTIME_SUCCEEDED || results[job].status) { + printf("runtime job %u state=%d status=%d\n", job, results[job].state, + results[job].status); + return 1; + } + } + puts("Runtime job PASS"); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/CMakeLists.txt new file mode 100644 index 000000000..3adff7725 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(same_ball_independent SOURCES + same_ball_independent-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/pebble.c new file mode 100644 index 000000000..10face5b5 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/pebble.c @@ -0,0 +1,56 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { M = 8, N = 16, K = 64, LANES = BANK_WIDTH / 8 }; +static int8_t a0[BB_CORE_NUM][M * K] __attribute__((aligned(64))); +static int8_t b0[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +static int8_t a1[BB_CORE_NUM][M * K] __attribute__((aligned(64))); +static int8_t b1[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_CORE_NUM][N] __attribute__((aligned(64))); +static int32_t out0[BB_CORE_NUM][M * N] __attribute__((aligned(64))); +static int32_t out1[BB_CORE_NUM][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_CORE_NUM][N] __attribute__((aligned(64))); +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < M * K; ++i) { + a0[c][i] = 1; + a1[c][i] = 3; + } + for (int i = 0; i < K * N; ++i) { + b0[c][i] = 2; + b1[c][i] = 4; + } + for (int bank = 0; bank < 7; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)a0[c], 0, M * K / LANES, 1); + bb_mvin((uintptr_t)b0[c], 1, K * N / LANES, 1); + bb_mvin((uintptr_t)a1[c], 3, M * K / LANES, 1); + bb_mvin((uintptr_t)b1[c], 4, K * N / LANES, 1); + bb_mvin((uintptr_t)bias[c], 6, N * 4 / LANES, 1); + bb_smatmul_bias(6, 0); + bb_smatmul_os(0, 1, 2, 1, N, N, 1, 1, 0); + bb_mvout((uintptr_t)warmup[c], 2, N * 4 / LANES, 1); + if (warmup[c][0] != 2 * N) + return 1; + uint64_t start, end; + asm volatile("rdcycle %0" : "=r"(start)); + bb_smatmul_os(0, 1, 2, M, N, K, 1, 1, 0); + bb_smatmul_os(3, 4, 5, M, N, K, 1, 1, 0); + bb_mvout((uintptr_t)out0[c], 2, M * N * 4 / LANES, 1); + bb_mvout((uintptr_t)out1[c], 5, M * N * 4 / LANES, 1); + for (int i = 0; i < M * N; ++i) + if (out0[c][i] != 2 * K || out1[c][i] != 12 * K) + return 1; + asm volatile("rdcycle %0" : "=r"(end)); + for (int bank = 0; bank < 7; ++bank) + bb_mem_release(bank); + printf("same_ball_independent PASS cycles=%llu\n", + (unsigned long long)(end - start)); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/same_ball_independent-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/same_ball_independent-main.cpp new file mode 100644 index 000000000..b38e76558 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/same_ball_independent-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == BB_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/chips/pebble/workloads/ctests/se_gate_lut_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/se_gate_lut_test.c similarity index 100% rename from examples/chips/pebble/workloads/ctests/se_gate_lut_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/se_gate_lut_test.c diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/CMakeLists.txt new file mode 100644 index 000000000..bdcbda565 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(shared_bank_versions SOURCES + shared_bank_versions-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/pebble.c new file mode 100644 index 000000000..3f478dfbd --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/pebble.c @@ -0,0 +1,47 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { + ROWS = 16, + NEW_ROWS = 8, + BYTES = ROWS * BANK_WIDTH / 8, + BASE = BB_SHARED_BANK_BASE +}; +static int8_t input[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); +static int8_t left[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); +static int8_t right[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); +static volatile int8_t warmup[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); +static int8_t output[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); + +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < BYTES; ++i) { + input[c][i] = 1; + left[c][i] = 3; + right[c][i] = 4; + } + for (int bank = BASE; bank < BASE + 5; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], BASE, ROWS, 1); + bb_mvin((uintptr_t)left[c], BASE + 3, ROWS, 1); + bb_mvin((uintptr_t)right[c], BASE + 4, ROWS, 1); + bb_mvout((uintptr_t)warmup[c], BASE + 4, ROWS, 1); + if (warmup[c][0] != 4) + return 1; + bb_transpose(BASE, BASE + 2, ROWS, 8); + bb_int8add(BASE + 3, BASE + 4, BASE + 2, NEW_ROWS, 1.0f, 1.0f); + bb_mvout((uintptr_t)output[c], BASE + 2, NEW_ROWS, 1); + for (int i = 0; i < NEW_ROWS * BANK_WIDTH / 8; ++i) + if (output[c][i] != 7) + return 1; + for (int bank = BASE; bank < BASE + 5; ++bank) + bb_mem_release(bank); + printf("shared_bank_versions PASS\n"); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/shared_bank_versions-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/shared_bank_versions-main.cpp new file mode 100644 index 000000000..b38e76558 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/shared_bank_versions-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == BB_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/chips/pebble/workloads/ctests/smatmul_mvout_bank_reuse_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/smatmul_mvout_bank_reuse_test.c similarity index 100% rename from examples/chips/pebble/workloads/ctests/smatmul_mvout_bank_reuse_test.c rename to examples/designs/pebble/workloads/device/core/acc/ctests/smatmul_mvout_bank_reuse_test.c diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/CMakeLists.txt new file mode 100644 index 000000000..add409216 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(write_bank_versions SOURCES + write_bank_versions-main.cpp + pebble.c) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/pebble.c new file mode 100644 index 000000000..cd8d4571f --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/pebble.c @@ -0,0 +1,42 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include +#include +#include + +enum { ROWS = 16, NEW_ROWS = 8, BYTES = ROWS * BANK_WIDTH / 8 }; +static int8_t input[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); +static int8_t left[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); +static int8_t right[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); +static volatile int8_t warmup[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); +static int8_t output[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); + +extern "C" int pebble_pipeline(uint32_t instance) { + int c = instance; + for (int i = 0; i < BYTES; ++i) { + input[c][i] = 1; + left[c][i] = 3; + right[c][i] = 4; + } + for (int bank = 0; bank < 5; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)input[c], 0, ROWS, 1); + bb_mvin((uintptr_t)left[c], 3, ROWS, 1); + bb_mvin((uintptr_t)right[c], 4, ROWS, 1); + bb_mvout((uintptr_t)warmup[c], 4, ROWS, 1); + if (warmup[c][0] != 4) + return 1; + bb_transpose(0, 2, ROWS, 8); + bb_int8add(3, 4, 2, NEW_ROWS, 1.0f, 1.0f); + bb_mvout((uintptr_t)output[c], 2, NEW_ROWS, 1); + for (int i = 0; i < NEW_ROWS * BANK_WIDTH / 8; ++i) + if (output[c][i] != 7) + return 1; + for (int bank = 0; bank < 5; ++bank) + bb_mem_release(bank); + printf("write_bank_versions PASS\n"); + return 0; +} diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/write_bank_versions-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/write_bank_versions-main.cpp new file mode 100644 index 000000000..b38e76558 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/write_bank_versions-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int pebble_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, pebble_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == BB_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/CMakeLists.txt new file mode 100644 index 000000000..a0cb74653 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/CMakeLists.txt @@ -0,0 +1,32 @@ +add_custom_target(pebble-mlir-tests-build) + +set(PEBBLE_CUSTOM_PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_subdirectory(bank) +add_subdirectory(core) +add_subdirectory(linalg) + +add_dependencies(design-mlirtests-build pebble-mlir-tests-build) diff --git a/examples/chips/pebble/workloads/mlir_tests/bank/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/CMakeLists.txt diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir new file mode 100644 index 000000000..36d776314 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir @@ -0,0 +1,30 @@ +// Design integration: f32-to-i8 quantization -> im2col k=1 -> mvout. + +func.func private @check_result(memref<16x16xi8>) -> () + +memref.global "private" constant @fp_in : memref<4x4xf32> = + dense<[[1.0, 2.0, 3.0, 4.0], [0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]]> + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %scale = arith.constant 1.0 : f32 + + %input = memref.get_global @fp_in : memref<4x4xf32> + %output = memref.alloc() alignment = 64 : memref<16x16xi8> + + %bin = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %bq = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %bout = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %loaded = buckyball.bank_mvin %input, %bin <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + %q = buckyball.bank_custom "fp32_to_int8" (%loaded) -> %bq args(%scale : f32) {rows = 4 : i64} + %im = buckyball.bank_custom "im2col_i8" (%q) -> %bout {input_side = 1 : i64, kernel = 1 : i64, stride = 1 : i64, padding = 0 : i64, start_row = 0 : i64, start_col = 0 : i64, op1_base = 0 : i64, lane = 0 : i64, window_start = 0 : i64, window_count = 1 : i64} + %stored = buckyball.bank_mvout %output, %im <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.bank_release %loaded + buckyball.bank_release %q + buckyball.bank_release %stored + + func.call @check_result(%output) : (memref<16x16xi8>) -> () + memref.dealloc %output : memref<16x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/bank/quant_im2col_k1_single_pixel_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/bank/quant_im2col_k1_single_pixel_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel_main.cpp diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/CMakeLists.txt new file mode 100644 index 000000000..6ca7097fc --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/CMakeLists.txt @@ -0,0 +1,63 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET pebble-mlir-tests-build) +set(BUCKYBALL_MLIR_TEST_PREFIX core) + +add_buckyball_mlir_test(lookup_channel_i8 TARGET pebble PASSES ${PEBBLE_CUSTOM_PASSES}) +add_buckyball_mlir_test(linear_chain_i8 TARGET pebble PASSES ${PEBBLE_CUSTOM_PASSES}) +add_buckyball_mlir_test(linear_int8 TARGET pebble PASSES ${PEBBLE_CUSTOM_PASSES}) +add_buckyball_mlir_test(generic_matmul_i8 TARGET pebble PASSES ${PEBBLE_CUSTOM_PASSES}) + +add_buckyball_mlir_test(mega_matmul_m17_k65_n17 + TARGET pebble PASSES ${PEBBLE_CUSTOM_PASSES}) + +add_buckyball_mlir_test(mega_conv2d + TARGET pebble + PASSES ${PEBBLE_CUSTOM_PASSES}) + +add_buckyball_mlir_test(mega_conv2d_k5_tail + TARGET pebble + PASSES ${PEBBLE_CUSTOM_PASSES}) + +add_buckyball_mlir_test(mega_conv2d_s2_depthwise_s2 + TARGET pebble + PASSES ${PEBBLE_CUSTOM_PASSES}) + +file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv_exact_input.i8 + PEBBLE_GAP_CONV_INPUT_HEX HEX) +file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_7x7_c288_input.i8 + PEBBLE_GAP_C288_INPUT_HEX HEX) +file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv_exact_weight.i8 + PEBBLE_GAP_CONV_WEIGHT_HEX HEX) +file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv2_exact_weight2.i8 + PEBBLE_GAP_CONV_WEIGHT2_HEX HEX) +file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv2_exact_weight3.i8 + PEBBLE_GAP_CONV_WEIGHT3_HEX HEX) +add_buckyball_mlir_test(mega_global_avg_conv_exact + TARGET pebble + PASSES ${PEBBLE_CUSTOM_PASSES}) +add_buckyball_mlir_test(mega_global_avg_7x7_c288_exact + TARGET pebble + PASSES ${PEBBLE_CUSTOM_PASSES}) +add_buckyball_mlir_test(mega_global_avg_conv2_exact + TARGET pebble + PASSES ${PEBBLE_CUSTOM_PASSES}) +unset(PEBBLE_GAP_CONV_INPUT_HEX) +unset(PEBBLE_GAP_C288_INPUT_HEX) +unset(PEBBLE_GAP_CONV_WEIGHT_HEX) +unset(PEBBLE_GAP_CONV_WEIGHT2_HEX) +unset(PEBBLE_GAP_CONV_WEIGHT3_HEX) + +add_buckyball_mlir_test(mega_se_gate_conv + TARGET pebble + PASSES ${PEBBLE_CUSTOM_PASSES}) + +add_buckyball_mlir_test(mega_matmul + TARGET pebble + PASSES ${PEBBLE_CUSTOM_PASSES}) + +foreach(test + core_matmul_16x16_i8 + core_transpose_16x16_i8) + add_buckyball_mlir_test(${test} + TARGET pebble + PASSES ${PEBBLE_CUSTOM_PASSES}) +endforeach() diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_matmul_16x16_i8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_matmul_16x16_i8.mlir new file mode 100644 index 000000000..17b262bdd --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_matmul_16x16_i8.mlir @@ -0,0 +1,40 @@ +func.func private @check_result(memref<16x16xi32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c16 = arith.constant 16 : index + %a = memref.alloc() alignment = 64 : memref<16x16xi8> + %b = memref.alloc() alignment = 64 : memref<16x16xi8> + scf.for %i = %c0 to %c16 step %c1 { + scf.for %j = %c0 to %c16 step %c1 { + %i3 = arith.muli %i, %c3 : index + %j5 = arith.muli %j, %c5 : index + %as = arith.addi %i3, %j5 : index + %ar = arith.remui %as, %c7 : index + %av0 = arith.subi %ar, %c3 : index + %av = arith.index_cast %av0 : index to i8 + %i2 = arith.muli %i, %c2 : index + %j3 = arith.muli %j, %c3 : index + %bs = arith.addi %i2, %j3 : index + %br = arith.remui %bs, %c5 : index + %bv0 = arith.subi %br, %c2 : index + %bv = arith.index_cast %bv0 : index to i8 + memref.store %av, %a[%i, %j] : memref<16x16xi8> + memref.store %bv, %b[%i, %j] : memref<16x16xi8> + } + } + %at = bufferization.to_tensor %a restrict : memref<16x16xi8> to tensor<16x16xi8> + %bt = bufferization.to_tensor %b restrict : memref<16x16xi8> to tensor<16x16xi8> + %init = arith.constant dense<0> : tensor<16x16xi32> + %result = linalg.matmul ins(%at, %bt: tensor<16x16xi8>, tensor<16x16xi8>) outs(%init: tensor<16x16xi32>) -> tensor<16x16xi32> + %memory = bufferization.to_buffer %result : tensor<16x16xi32> to memref<16x16xi32> + func.call @check_result(%memory) : (memref<16x16xi32>) -> () + return %zero_i8 : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_matmul_16x16_i8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_matmul_16x16_i8_main.cpp new file mode 100644 index 000000000..156db9a22 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_matmul_16x16_i8_main.cpp @@ -0,0 +1,29 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +extern "C" void check_result(int32_t *, int32_t *out, int64_t offset, + int64_t rows, int64_t columns, int64_t row_stride, + int64_t column_stride) { + if (rows != 16 || columns != 16 || row_stride != 16 || column_stride != 1) + fail(); + for (int i = 0; i < 16; ++i) + for (int j = 0; j < 16; ++j) { + int expected = 0; + for (int k = 0; k < 16; ++k) + expected += ((3 * i + 5 * k) % 7 - 3) * ((2 * k + 3 * j) % 5 - 2); + if (out[offset + i * row_stride + j] != expected) + fail(); + } + printf("PASSED: core.core_matmul 16x16 ordered operands\n"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_transpose_16x16_i8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_transpose_16x16_i8.mlir new file mode 100644 index 000000000..622974fa5 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_transpose_16x16_i8.mlir @@ -0,0 +1,27 @@ +func.func private @check_result(memref<16x16xi8>) -> () + +func.func @main() -> i8 { + %zero = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : i8 + %c5 = arith.constant 5 : i8 + %c16 = arith.constant 16 : index + %input = memref.alloc() alignment = 64 : memref<16x16xi8> + scf.for %i = %c0 to %c16 step %c1 { + scf.for %j = %c0 to %c16 step %c1 { + %ii = arith.index_cast %i : index to i8 + %jj = arith.index_cast %j : index to i8 + %iv = arith.muli %ii, %c3 : i8 + %jv = arith.muli %jj, %c5 : i8 + %v = arith.addi %iv, %jv : i8 + memref.store %v, %input[%i, %j] : memref<16x16xi8> + } + } + %it = bufferization.to_tensor %input restrict : memref<16x16xi8> to tensor<16x16xi8> + %empty = tensor.empty() : tensor<16x16xi8> + %result = linalg.transpose ins(%it: tensor<16x16xi8>) outs(%empty: tensor<16x16xi8>) permutation = [1,0] + %memory = bufferization.to_buffer %result : tensor<16x16xi8> to memref<16x16xi8> + func.call @check_result(%memory) : (memref<16x16xi8>) -> () + return %zero : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_transpose_16x16_i8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_transpose_16x16_i8_main.cpp new file mode 100644 index 000000000..84b02b78b --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/core_transpose_16x16_i8_main.cpp @@ -0,0 +1,25 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +extern "C" void check_result(int8_t *, int8_t *out, int64_t offset, + int64_t rows, int64_t columns, int64_t row_stride, + int64_t column_stride) { + if (rows != 16 || columns != 16 || row_stride != 16 || column_stride != 1) + fail(); + for (int i = 0; i < 16; ++i) + for (int j = 0; j < 16; ++j) + if (out[offset + i * row_stride + j] != (int8_t)(3 * j + 5 * i)) + fail(); + printf("PASSED: core.core_transpose 16x16 ordered input\n"); +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_7x7_c288_input.i8 b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_7x7_c288_input.i8 similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_7x7_c288_input.i8 rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_7x7_c288_input.i8 diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv2_exact_weight2.i8 b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv2_exact_weight2.i8 similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv2_exact_weight2.i8 rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv2_exact_weight2.i8 diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv2_exact_weight3.i8 b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv2_exact_weight3.i8 similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv2_exact_weight3.i8 rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv2_exact_weight3.i8 diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv_exact_input.i8 b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv_exact_input.i8 similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv_exact_input.i8 rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv_exact_input.i8 diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv_exact_weight.i8 b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv_exact_weight.i8 similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/data/mega_global_avg_conv_exact_weight.i8 rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/data/mega_global_avg_conv_exact_weight.i8 diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/generic_matmul_i8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/generic_matmul_i8.mlir new file mode 100644 index 000000000..88373edac --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/generic_matmul_i8.mlir @@ -0,0 +1,52 @@ +func.func private @check_result(memref<1x32xi32>) + +func.func @main() -> i8 { + %exit = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %c16 = arith.constant 16 : index + %c32 = arith.constant 32 : index + %a = memref.alloc() alignment = 64 : memref<1x32xi8> + %b = memref.alloc() alignment = 64 : memref<32x32xi8> + %c = memref.alloc() alignment = 64 : memref<1x32xi32> + scf.for %k = %c0 to %c32 step %c1 { + %rem = arith.remui %k, %c7 : index + %value = arith.subi %rem, %c3 : index + %a8 = arith.index_cast %value : index to i8 + memref.store %a8, %a[%c0, %k] : memref<1x32xi8> + %initial = arith.subi %k, %c16 : index + %c32value = arith.index_cast %initial : index to i32 + memref.store %c32value, %c[%c0, %k] : memref<1x32xi32> + scf.for %column = %c0 to %c32 step %c1 { + %k3 = arith.muli %k, %c3 : index + %column5 = arith.muli %column, %c5 : index + %sum = arith.addi %k3, %column5 : index + %rem9 = arith.remui %sum, %c9 : index + %signed = arith.subi %rem9, %c4 : index + %b8 = arith.index_cast %signed : index to i8 + memref.store %b8, %b[%k, %column] : memref<32x32xi8> + } + } + %lhs = bufferization.to_tensor %a restrict : memref<1x32xi8> to tensor<1x32xi8> + %rhs = bufferization.to_tensor %b restrict : memref<32x32xi8> to tensor<32x32xi8> + %initial = bufferization.to_tensor %c restrict : memref<1x32xi32> to tensor<1x32xi32> + %product = linalg.generic { + indexing_maps = [affine_map<(m,n,k)->(m,k)>, affine_map<(m,n,k)->(k,n)>, affine_map<(m,n,k)->(m,n)>], + iterator_types = ["parallel", "parallel", "reduction"] + } ins(%lhs, %rhs: tensor<1x32xi8>, tensor<32x32xi8>) outs(%initial: tensor<1x32xi32>) { + ^bb0(%x: i8, %y: i8, %acc: i32): + %x32 = arith.extsi %x : i8 to i32 + %y32 = arith.extsi %y : i8 to i32 + %mul = arith.muli %x32, %y32 : i32 + %sum = arith.addi %acc, %mul : i32 + linalg.yield %sum : i32 + } -> tensor<1x32xi32> + %output = bufferization.to_buffer %product : tensor<1x32xi32> to memref<1x32xi32> + func.call @check_result(%output) : (memref<1x32xi32>) -> () + return %exit : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/generic_matmul_i8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/generic_matmul_i8_main.cpp new file mode 100644 index 000000000..07b79adfd --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/generic_matmul_i8_main.cpp @@ -0,0 +1,33 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +extern "C" void check_result(int32_t *, int32_t *output, int64_t offset, + int64_t rows, int64_t columns, int64_t row_stride, + int64_t column_stride) { + if (rows != 1 || columns != 32 || row_stride != 32 || column_stride != 1) + fail(); + for (int column = 0; column < 32; ++column) { + int32_t expected = column - 16; + for (int k = 0; k < 32; ++k) + expected += (k % 7 - 3) * ((3 * k + 5 * column) % 9 - 4); + int32_t actual = output[offset + column * column_stride]; + if (actual != expected) { + printf("FAILED: generic matmul column=%d expected=%d actual=%d\n", column, + expected, actual); + fail(); + } + } + printf("PASSED: generic matmul INT8 signed inputs and nonzero INT32 " + "accumulator\n"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8.mlir new file mode 100644 index 000000000..cefe53df5 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8.mlir @@ -0,0 +1,93 @@ +func.func private @check_result(memref<1x16xf32>) + +func.func @main() -> i8 { + %exit = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c8 = arith.constant 8 : index + %c9 = arith.constant 9 : index + %c16 = arith.constant 16 : index + %c32 = arith.constant 32 : index + %c99 = arith.constant 99 : i32 + %unit1 = arith.constant 0.015625 : f32 + %unit2 = arith.constant 0.0078125 : f32 + %multiplier = arith.constant dense<0.5> : tensor<1x1xf32> + %input_step = arith.constant dense<2.0> : tensor<1x1xf32> + %zero = arith.constant dense<0.0> : tensor<1x32xf32> + %x = memref.alloc() alignment = 64 : memref<1x32xf32> + %w1 = memref.alloc() alignment = 64 : memref<32x32xi8> + %b1 = memref.alloc() alignment = 64 : memref<1x32xi32> + %s1 = memref.alloc() alignment = 64 : memref<1x32xf32> + %w2 = memref.alloc() alignment = 64 : memref<32x16xi8> + %b2 = memref.alloc() alignment = 64 : memref<1x16xi32> + %s2 = memref.alloc() alignment = 64 : memref<1x16xf32> + scf.for %k = %c0 to %c32 step %c1 { + %rem = arith.remui %k, %c7 : index + %shift = arith.subi %rem, %c3 : index + %integer = arith.index_cast %shift : index to i32 + %scaled = arith.muli %integer, %c99 : i32 + %value = arith.sitofp %scaled : i32 to f32 + memref.store %value, %x[%c0, %k] : memref<1x32xf32> + %bias = arith.subi %k, %c16 : index + %bias32 = arith.index_cast %bias : index to i32 + memref.store %bias32, %b1[%c0, %k] : memref<1x32xi32> + %number = arith.addi %k, %c1 : index + %number32 = arith.index_cast %number : index to i32 + %numberf = arith.sitofp %number32 : i32 to f32 + %scale = arith.mulf %numberf, %unit1 : f32 + memref.store %scale, %s1[%c0, %k] : memref<1x32xf32> + scf.for %column = %c0 to %c32 step %c1 { + %k3 = arith.muli %k, %c3 : index + %column5 = arith.muli %column, %c5 : index + %sum = arith.addi %k3, %column5 : index + %mod = arith.remui %sum, %c9 : index + %signed = arith.subi %mod, %c4 : index + %weight = arith.index_cast %signed : index to i8 + memref.store %weight, %w1[%k, %column] : memref<32x32xi8> + } + scf.for %column = %c0 to %c16 step %c1 { + %k5 = arith.muli %k, %c5 : index + %column3 = arith.muli %column, %c3 : index + %sum = arith.addi %k5, %column3 : index + %mod = arith.remui %sum, %c7 : index + %signed = arith.subi %mod, %c3 : index + %weight = arith.index_cast %signed : index to i8 + memref.store %weight, %w2[%k, %column] : memref<32x16xi8> + } + } + scf.for %column = %c0 to %c16 step %c1 { + %bias = arith.subi %column, %c8 : index + %bias32 = arith.index_cast %bias : index to i32 + memref.store %bias32, %b2[%c0, %column] : memref<1x16xi32> + %number = arith.addi %column, %c1 : index + %number32 = arith.index_cast %number : index to i32 + %numberf = arith.sitofp %number32 : i32 to f32 + %scale = arith.mulf %numberf, %unit2 : f32 + memref.store %scale, %s2[%c0, %column] : memref<1x16xf32> + } + %input = bufferization.to_tensor %x restrict : memref<1x32xf32> to tensor<1x32xf32> + %weight1 = bufferization.to_tensor %w1 restrict : memref<32x32xi8> to tensor<32x32xi8> + %bias1 = bufferization.to_tensor %b1 restrict : memref<1x32xi32> to tensor<1x32xi32> + %step1 = bufferization.to_tensor %s1 restrict : memref<1x32xf32> to tensor<1x32xf32> + %weight2 = bufferization.to_tensor %w2 restrict : memref<32x16xi8> to tensor<32x16xi8> + %bias2 = bufferization.to_tensor %b2 restrict : memref<1x16xi32> to tensor<1x16xi32> + %step2 = bufferization.to_tensor %s2 restrict : memref<1x16xf32> to tensor<1x16xf32> + %first = custom.op "linear_i8"(%input, %weight1, %bias1, %multiplier, %input_step, %step1) : (tensor<1x32xf32>, tensor<32x32xi8>, tensor<1x32xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x32xf32>) -> tensor<1x32xf32> + %empty = tensor.empty() : tensor<1x32xf32> + %relu = linalg.generic { + indexing_maps = [affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], + iterator_types = ["parallel", "parallel"] + } ins(%first, %zero: tensor<1x32xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { + ^bb0(%value: f32, %zero_value: f32, %unused: f32): + %positive = arith.maximumf %value, %zero_value : f32 + linalg.yield %positive : f32 + } -> tensor<1x32xf32> + %second = custom.op "linear_i8"(%relu, %weight2, %bias2, %multiplier, %input_step, %step2) : (tensor<1x32xf32>, tensor<32x16xi8>, tensor<1x16xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x16xf32>) -> tensor<1x16xf32> + %output = bufferization.to_buffer %second : tensor<1x16xf32> to memref<1x16xf32> + func.call @check_result(%output) : (memref<1x16xf32>) -> () + return %exit : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8_main.cpp new file mode 100644 index 000000000..76c3315d9 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8_main.cpp @@ -0,0 +1,60 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +static int8_t quantize(float value) { + int rounded = (int)value; + float fraction = value - rounded; + if (fraction > 0.5f || (fraction == 0.5f && rounded % 2 != 0)) + ++rounded; + if (fraction < -0.5f || (fraction == -0.5f && rounded % 2 != 0)) + --rounded; + if (rounded < -128) + rounded = -128; + if (rounded > 127) + rounded = 127; + return (int8_t)rounded; +} + +extern "C" void check_result(float *, float *output, int64_t offset, + int64_t rows, int64_t columns, int64_t row_stride, + int64_t column_stride) { + if (rows != 1 || columns != 16 || row_stride != 16 || column_stride != 1) + fail(); + int8_t input[32], intermediate[32]; + for (int k = 0; k < 32; ++k) + input[k] = quantize((float)((k % 7 - 3) * 99) * 0.5f); + for (int column = 0; column < 32; ++column) { + int32_t accumulator = column - 16; + for (int k = 0; k < 32; ++k) + accumulator += input[k] * ((3 * k + 5 * column) % 9 - 4); + float scale = 2.0f * ((column + 1) / 64.0f) * 0.5f; + int8_t requantized = quantize((float)accumulator * scale); + intermediate[column] = requantized < 0 ? 0 : requantized; + } + for (int column = 0; column < 16; ++column) { + int32_t accumulator = column - 8; + for (int k = 0; k < 32; ++k) + accumulator += intermediate[k] * ((5 * k + 3 * column) % 7 - 3); + float expected = (float)accumulator * (2.0f * ((column + 1) / 128.0f)); + float actual = output[offset + column * column_stride]; + // Binary fraction scales keep this small integer reference exact. + if (actual != expected) { + printf("FAILED: linear_chain_i8 column=%d expected=%f actual=%f\n", + column, expected, actual); + fail(); + } + } + printf( + "PASSED: linear_chain_i8 two Linear layers with ReLU and INT8 rescale\n"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_int8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_int8.mlir new file mode 100644 index 000000000..34107df71 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_int8.mlir @@ -0,0 +1,73 @@ +func.func private @check_result(memref<1x32xf32>) + +func.func @main() -> i8 { + %exit = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c6 = arith.constant 6 : index + %c7 = arith.constant 7 : index + %c13 = arith.constant 13 : index + %c16 = arith.constant 16 : index + %c32 = arith.constant 32 : index + %scale_unit = arith.constant 0.0078125 : f32 + %input = arith.constant dense<[[ + -300.0, -255.0, -5.0, -3.0, -1.0, 0.0, 1.0, 3.0, + 5.0, 254.0, 255.0, 300.0, 2.0, 6.0, 10.0, 14.0, + -257.0, -253.0, -7.0, -9.0, -11.0, -13.0, -15.0, 7.0, + 9.0, 11.0, 13.0, 15.0, 253.0, 257.0, -254.0, -256.0 + ]]> : tensor<1x32xf32> + %multiplier = arith.constant dense<0.5> : tensor<1x1xf32> + %input_step = arith.constant dense<2.0> : tensor<1x1xf32> + %zero = arith.constant dense<0.0> : tensor<1x32xf32> + %weights = memref.alloc() alignment = 64 : memref<32x32xi8> + %steps = memref.alloc() alignment = 64 : memref<1x32xf32> + %bias = memref.alloc() alignment = 64 : memref<1x32xi32> + scf.for %column = %c0 to %c32 step %c1 { + %number = arith.addi %column, %c1 : index + %number_i32 = arith.index_cast %number : index to i32 + %number_f32 = arith.sitofp %number_i32 : i32 to f32 + %step = arith.mulf %number_f32, %scale_unit : f32 + memref.store %step, %steps[%c0, %column] : memref<1x32xf32> + %shifted = arith.subi %column, %c16 : index + %shifted_i32 = arith.index_cast %shifted : index to i32 + memref.store %shifted_i32, %bias[%c0, %column] : memref<1x32xi32> + scf.for %k = %c0 to %c32 step %c1 { + %k7 = arith.muli %k, %c7 : index + %column3 = arith.muli %column, %c3 : index + %sum = arith.addi %k7, %column3 : index + %mod = arith.remui %sum, %c13 : index + %signed = arith.subi %mod, %c6 : index + %weight = arith.index_cast %signed : index to i8 + memref.store %weight, %weights[%k, %column] : memref<32x32xi8> + } + } + %weight_codes = bufferization.to_tensor %weights restrict : memref<32x32xi8> to tensor<32x32xi8> + %weight_step = bufferization.to_tensor %steps restrict : memref<1x32xf32> to tensor<1x32xf32> + %bias_codes = bufferization.to_tensor %bias restrict : memref<1x32xi32> to tensor<1x32xi32> + %codes = "frontend.fp32_to_int8"(%input, %multiplier) : (tensor<1x32xf32>, tensor<1x1xf32>) -> tensor<1x32xi8> + %activation = "frontend.int8_to_fp32"(%codes, %input_step) : (tensor<1x32xi8>, tensor<1x1xf32>) -> tensor<1x32xf32> + %weight_values = "frontend.int8_to_fp32"(%weight_codes, %weight_step) : (tensor<32x32xi8>, tensor<1x32xf32>) -> tensor<32x32xf32> + %product = linalg.matmul ins(%activation, %weight_values : tensor<1x32xf32>, tensor<32x32xf32>) outs(%zero : tensor<1x32xf32>) -> tensor<1x32xf32> + %empty = tensor.empty() : tensor<1x32xf32> + %bias_step = linalg.generic { + indexing_maps = [affine_map<(m,n)->(m,0)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], + iterator_types = ["parallel", "parallel"] + } ins(%input_step, %weight_step: tensor<1x1xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { + ^bb0(%lhs: f32, %rhs: f32, %unused: f32): + %scale = arith.mulf %lhs, %rhs : f32 + linalg.yield %scale : f32 + } -> tensor<1x32xf32> + %bias_tensor = "frontend.int32_to_fp32"(%bias_codes, %bias_step) : (tensor<1x32xi32>, tensor<1x32xf32>) -> tensor<1x32xf32> + %result = linalg.generic { + indexing_maps = [affine_map<(m, n) -> (m, n)>, affine_map<(m, n) -> (m, n)>, affine_map<(m, n) -> (m, n)>], + iterator_types = ["parallel", "parallel"] + } ins(%product, %bias_tensor : tensor<1x32xf32>, tensor<1x32xf32>) outs(%empty : tensor<1x32xf32>) { + ^bb0(%value: f32, %addend: f32, %unused: f32): + %added = arith.addf %value, %addend : f32 + linalg.yield %added : f32 + } -> tensor<1x32xf32> + %output = bufferization.to_buffer %result : tensor<1x32xf32> to memref<1x32xf32> + func.call @check_result(%output) : (memref<1x32xf32>) -> () + return %exit : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_int8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_int8_main.cpp new file mode 100644 index 000000000..d7f23ac32 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_int8_main.cpp @@ -0,0 +1,58 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +extern "C" void check_result(float *, float *output, int64_t offset, + int64_t rows, int64_t columns, int64_t row_stride, + int64_t column_stride) { + if (rows != 1 || columns != 32 || row_stride != 32 || column_stride != 1) + fail(); + const float input[32] = { + -300, -255, -5, -3, -1, 0, 1, 3, 5, 254, 255, + 300, 2, 6, 10, 14, -257, -253, -7, -9, -11, -13, + -15, 7, 9, 11, 13, 15, 253, 257, -254, -256, + }; + int8_t codes[32]; + for (int k = 0; k < 32; ++k) { + float scaled = input[k] * 0.5f; + int rounded = (int)scaled; + float fraction = scaled - rounded; + if (fraction > 0.5f || (fraction == 0.5f && rounded % 2 != 0)) + ++rounded; + if (fraction < -0.5f || (fraction == -0.5f && rounded % 2 != 0)) + --rounded; + if (rounded < -128) + rounded = -128; + if (rounded > 127) + rounded = 127; + codes[k] = (int8_t)rounded; + } + for (int column = 0; column < 32; ++column) { + int32_t accumulator = column - 16; + for (int k = 0; k < 32; ++k) { + int8_t weight = (7 * k + 3 * column) % 13 - 6; + accumulator += codes[k] * weight; + } + float step = 2.0f * ((column + 1) / 128.0f); + float expected = (float)accumulator * step; + float actual = output[offset + column * column_stride]; + // The scales are binary fractions; these results are exact. + if (actual != expected) { + printf("FAILED: linear_int8 column=%d expected=%f actual=%f\n", column, + expected, actual); + fail(); + } + } + printf("PASSED: linear_int8 1x32x32 rounding, saturation, channel scales, " + "bias\n"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/lookup_channel_i8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/lookup_channel_i8.mlir new file mode 100644 index 000000000..e5235e3b7 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/lookup_channel_i8.mlir @@ -0,0 +1,40 @@ +func.func private @check_result(memref<4x16xi8>) +func.func @main() -> i8 { + %zero = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c4 = arith.constant 4 : index + %c16 = arith.constant 16 : index + %c17 = arith.constant 17 : index + %c29 = arith.constant 29 : index + %c128 = arith.constant 128 : index + %c256 = arith.constant 256 : index + %c4096 = arith.constant 4096 : index + %input = memref.alloc() : memref<4x16xi8> + %table = memref.alloc() : memref<4096xi8> + scf.for %row = %c0 to %c4 step %c1 { + scf.for %lane = %c0 to %c16 step %c1 { + %base = arith.muli %row, %c16 : index + %offset = arith.addi %base, %lane : index + %scaled = arith.muli %offset, %c29 : index + %shifted = arith.subi %scaled, %c128 : index + %value = arith.index_cast %shifted : index to i8 + memref.store %value, %input[%row, %lane] : memref<4x16xi8> + } + } + scf.for %offset = %c0 to %c4096 step %c1 { + %lane = arith.divui %offset, %c256 : index + %entry = arith.remui %offset, %c256 : index + %scaled = arith.muli %lane, %c17 : index + %sum = arith.addi %entry, %scaled : index + %shifted = arith.subi %sum, %c128 : index + %value = arith.index_cast %shifted : index to i8 + memref.store %value, %table[%offset] : memref<4096xi8> + } + %x = bufferization.to_tensor %input restrict : memref<4x16xi8> to tensor<4x16xi8> + %lut = bufferization.to_tensor %table restrict : memref<4096xi8> to tensor<4096xi8> + %result = custom.op "lookup_i8"(%x, %lut) : (tensor<4x16xi8>, tensor<4096xi8>) -> tensor<4x16xi8> + %memory = bufferization.to_buffer %result : tensor<4x16xi8> to memref<4x16xi8> + func.call @check_result(%memory) : (memref<4x16xi8>) -> () + return %zero : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/lookup_channel_i8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/lookup_channel_i8_main.cpp new file mode 100644 index 000000000..75cc70b73 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/lookup_channel_i8_main.cpp @@ -0,0 +1,20 @@ +#include +#include +#include + +extern "C" void check_result(int8_t *, int8_t *output, int64_t offset, + int64_t rows, int64_t columns, int64_t rs, + int64_t cs) { + if (rows != 4 || columns != 16) + abort(); + for (int row = 0; row < 4; ++row) + for (int lane = 0; lane < 16; ++lane) { + uint8_t input = (uint8_t)((row * 16 + lane) * 29 - 128); + int8_t expected = (int8_t)(input + lane * 17 - 128); + if (output[offset + row * rs + lane * cs] != expected) { + printf("FAILED: channel LUT row=%d lane=%d\n", row, lane); + abort(); + } + } + printf("PASSED: sixteen independent INT8 lookup tables\n"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir new file mode 100644 index 000000000..d065e7c45 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir @@ -0,0 +1,100 @@ +// FORM: custom.op "conv2d_i8"{{.*}}custom.spatial_chain +// FORM: custom.op "relu"{{.*}}custom.spatial_chain +// FORM: custom.op "depthwise_conv2d_i8"{{.*}}custom.spatial_chain +// FORM: custom.op "maxpool2d"{{.*}}custom.spatial_chain +// FORM: "frontend.fp32_to_int8"{{.*}}custom.spatial_chain +// BANK: buckyball.bank_mvout +// BANK: return +func.func private @check_result(memref<1x16x2x2xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %c16 = arith.constant 16 : index + %input = memref.alloc() alignment = 64 : memref<1x4x4x2xi8> + %conv_weight = memref.alloc() alignment = 64 : memref<16x2x3x3xi8> + %conv_bias = memref.alloc() alignment = 64 : memref<16xi32> + %depthwise_weight = memref.alloc() alignment = 64 : memref<16x1x3x3xi8> + %depthwise_bias = memref.alloc() alignment = 64 : memref<16xi32> + scf.for %h = %c0 to %c4 step %c1 { + scf.for %w = %c0 to %c4 step %c1 { + scf.for %ic = %c0 to %c2 step %c1 { + %h3 = arith.muli %h, %c3 : index + %w2 = arith.muli %w, %c2 : index + %sum0 = arith.addi %h3, %w2 : index + %sum = arith.addi %sum0, %ic : index + %rem = arith.remui %sum, %c7 : index + %signed = arith.subi %rem, %c3 : index + %value = arith.index_cast %signed : index to i8 + memref.store %value, %input[%c0, %h, %w, %ic] : memref<1x4x4x2xi8> + } + } + } + scf.for %ic = %c0 to %c2 step %c1 { + scf.for %p = %c0 to %c9 step %c1 { + scf.for %oc = %c0 to %c16 step %c1 { + %oc3 = arith.muli %oc, %c3 : index + %sum0 = arith.addi %p, %ic : index + %sum = arith.addi %sum0, %oc3 : index + %rem = arith.remui %sum, %c5 : index + %signed = arith.subi %rem, %c2 : index + %value = arith.index_cast %signed : index to i8 + %ky_index = arith.divui %p, %c3 : index + %kx_index = arith.remui %p, %c3 : index + memref.store %value, %conv_weight[%oc, %ic, %ky_index, %kx_index] : memref<16x2x3x3xi8> + } + } + } + scf.for %oc = %c0 to %c16 step %c1 { + %signed = arith.subi %oc, %c4 : index + %value = arith.index_cast %signed : index to i32 + memref.store %value, %conv_bias[%oc] : memref<16xi32> + } + scf.for %kh = %c0 to %c3 step %c1 { + scf.for %kw = %c0 to %c3 step %c1 { + scf.for %oc = %c0 to %c16 step %c1 { + %kw2 = arith.muli %kw, %c2 : index + %sum0 = arith.addi %kh, %kw2 : index + %sum = arith.addi %sum0, %oc : index + %rem = arith.remui %sum, %c5 : index + %signed = arith.subi %rem, %c2 : index + %value = arith.index_cast %signed : index to i8 + memref.store %value, %depthwise_weight[%oc, %c0, %kh, %kw] : memref<16x1x3x3xi8> + } + } + } + linalg.fill ins(%zero_i32 : i32) outs(%depthwise_bias : memref<16xi32>) + %unit = arith.constant dense<1.0> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<1.0> : tensor<16x1x1x1xf32> + %it = bufferization.to_tensor %input restrict : memref<1x4x4x2xi8> to tensor<1x4x4x2xi8> + %input_float = "frontend.int8_to_fp32"(%it, %unit) : (tensor<1x4x4x2xi8>, tensor<1x1x1x1xf32>) -> tensor<1x4x4x2xf32> + %wt = bufferization.to_tensor %conv_weight restrict : memref<16x2x3x3xi8> to tensor<16x2x3x3xi8> + %bt = bufferization.to_tensor %conv_bias restrict : memref<16xi32> to tensor<16xi32> + %dt = bufferization.to_tensor %depthwise_weight restrict : memref<16x1x3x3xi8> to tensor<16x1x3x3xi8> + %dbt = bufferization.to_tensor %depthwise_bias restrict : memref<16xi32> to tensor<16xi32> + %first = custom.op "conv2d_i8"(%input_float, %wt, %bt, %unit, %unit, %weight_step) {strides = array, dilations = array, padding = array} : (tensor<1x4x4x2xf32>, tensor<16x2x3x3xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x4x4x16xf32> + %zero = arith.constant dense<0.0> : tensor<1x4x4x16xf32> + %relu = linalg.generic {indexing_maps = [affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%first, %zero: tensor<1x4x4x16xf32>, tensor<1x4x4x16xf32>) outs(%zero: tensor<1x4x4x16xf32>) { + ^bb0(%a: f32, %b: f32, %out: f32): + %r = arith.maximumf %a, %b : f32 + linalg.yield %r : f32 + } -> tensor<1x4x4x16xf32> + %second = custom.op "depthwise_conv2d_i8"(%relu, %dt, %dbt, %unit, %unit, %weight_step) {strides = array, dilations = array, padding = array} : (tensor<1x4x4x16xf32>, tensor<16x1x3x3xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x4x4x16xf32> + %filter = tensor.empty() : tensor<2x2xf32> + %minimum = arith.constant dense<-3.40282347E+38> : tensor<1x2x2x16xf32> + %pooled = linalg.pooling_nhwc_max {strides = dense<2> : vector<2xi64>, dilations = dense<1> : vector<2xi64>} ins(%second, %filter: tensor<1x4x4x16xf32>, tensor<2x2xf32>) outs(%minimum: tensor<1x2x2x16xf32>) -> tensor<1x2x2x16xf32> + %q = "frontend.fp32_to_int8"(%pooled, %unit) : (tensor<1x2x2x16xf32>, tensor<1x1x1x1xf32>) -> tensor<1x2x2x16xi8> + %empty = tensor.empty() : tensor<1x16x2x2xi8> + %nchw = linalg.transpose ins(%q: tensor<1x2x2x16xi8>) outs(%empty: tensor<1x16x2x2xi8>) permutation = [0, 3, 1, 2] + %memory = bufferization.to_buffer %nchw : tensor<1x16x2x2xi8> to memref<1x16x2x2xi8> + func.call @check_result(%memory) : (memref<1x16x2x2xi8>) -> () + return %zero_i8 : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_k5_tail.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_k5_tail.mlir new file mode 100644 index 000000000..0fc47a463 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_k5_tail.mlir @@ -0,0 +1,59 @@ +func.func private @check_result(memref<1x16x5x5xf32>) +func.func @main() -> i8 { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c16 = arith.constant 16 : index + %c25 = arith.constant 25 : index + %zi8 = arith.constant 0 : i8 + %zi32 = arith.constant 0 : i32 + %input = memref.alloc() alignment = 64 : memref<1x5x5x2xi8> + %weight = memref.alloc() alignment = 64 : memref<16x2x5x5xi8> + %bias = memref.alloc() alignment = 64 : memref<16xi32> + linalg.fill ins(%zi8 : i8) outs(%weight : memref<16x2x5x5xi8>) + linalg.fill ins(%zi32 : i32) outs(%bias : memref<16xi32>) + scf.for %y = %c0 to %c5 step %c1 { + scf.for %x = %c0 to %c5 step %c1 { + scf.for %c = %c0 to %c2 step %c1 { + %y3 = arith.muli %y, %c3 : index + %x2 = arith.muli %x, %c2 : index + %s0 = arith.addi %y3, %x2 : index + %s = arith.addi %s0, %c : index + %r = arith.remui %s, %c7 : index + %v = arith.subi %r, %c3 : index + %i = arith.index_cast %v : index to i8 + memref.store %i, %input[%c0, %y, %x, %c] : memref<1x5x5x2xi8> + } + } + } + scf.for %c = %c0 to %c2 step %c1 { + scf.for %k = %c0 to %c25 step %c1 { + scf.for %o = %c0 to %c16 step %c1 { + %s0 = arith.addi %k, %c : index + %s = arith.addi %s0, %o : index + %r = arith.remui %s, %c3 : index + %v = arith.subi %r, %c1 : index + %i = arith.index_cast %v : index to i8 + %ky = arith.divui %k, %c5 : index + %kx = arith.remui %k, %c5 : index + memref.store %i, %weight[%o, %c, %ky, %kx] : memref<16x2x5x5xi8> + } + } + } + %input_tensor = bufferization.to_tensor %input restrict : memref<1x5x5x2xi8> to tensor<1x5x5x2xi8> + %input_step = arith.constant dense<1.0> : tensor<1x1x1x1xf32> + %input_float = "frontend.int8_to_fp32"(%input_tensor, %input_step) : (tensor<1x5x5x2xi8>, tensor<1x1x1x1xf32>) -> tensor<1x5x5x2xf32> + %weight_tensor = bufferization.to_tensor %weight restrict : memref<16x2x5x5xi8> to tensor<16x2x5x5xi8> + %bias_tensor = bufferization.to_tensor %bias restrict : memref<16xi32> to tensor<16xi32> + %weight_step = arith.constant dense<1.0> : tensor<16x1x1x1xf32> + %result = custom.op "conv2d_i8"(%input_float, %weight_tensor, %bias_tensor, %input_step, %input_step, %weight_step) {strides = array, dilations = array, padding = array} : (tensor<1x5x5x2xf32>, tensor<16x2x5x5xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x5x5x16xf32> + %empty = tensor.empty() : tensor<1x16x5x5xf32> + %nchw = linalg.transpose ins(%result: tensor<1x5x5x16xf32>) outs(%empty: tensor<1x16x5x5xf32>) permutation = [0, 3, 1, 2] + %memory = bufferization.to_buffer %nchw : tensor<1x16x5x5xf32> to memref<1x16x5x5xf32> + func.call @check_result(%memory) : (memref<1x16x5x5xf32>) -> () + %ok = arith.constant 0 : i8 + return %ok : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_k5_tail_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_k5_tail_main.cpp similarity index 95% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_k5_tail_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_k5_tail_main.cpp index 39bfae0a9..0aee33303 100644 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_k5_tail_main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_k5_tail_main.cpp @@ -41,5 +41,5 @@ extern "C" void check_result(float *, float *data, int64_t offset, int64_t n, abort(); } } - printf("PASS k5 Conv: 4x4 tiles, tail, 8x16 packing and FP32 row copies\n"); + printf("PASS k5 Conv: 4x4 cores, tail, 8x16 packing and FP32 row copies\n"); } diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_main.cpp diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_s2_depthwise_s2.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_s2_depthwise_s2.mlir new file mode 100644 index 000000000..136bf9e86 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_s2_depthwise_s2.mlir @@ -0,0 +1,88 @@ +func.func private @check_result(memref<1x4x4x16xi8>) -> () + +func.func @main() -> i8 { + %i8_0 = arith.constant 0 : i8 + %i32_0 = arith.constant 0 : i32 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %c16 = arith.constant 16 : index + %input = memref.alloc() alignment = 64 : memref<1x16x16x3xi8> + %weight0 = memref.alloc() alignment = 64 : memref<16x3x3x3xi8> + %bias0 = memref.alloc() alignment = 64 : memref<16xi32> + %weight1 = memref.alloc() alignment = 64 : memref<16x1x3x3xi8> + %bias1 = memref.alloc() alignment = 64 : memref<16xi32> + scf.for %y = %c0 to %c16 step %c1 { + scf.for %x = %c0 to %c16 step %c1 { + scf.for %ic = %c0 to %c3 step %c1 { + %a = arith.muli %y, %c3 : index + %b = arith.muli %x, %c2 : index + %d = arith.addi %a, %b : index + %e = arith.addi %d, %ic : index + %r = arith.remui %e, %c7 : index + %s = arith.subi %r, %c3 : index + %v = arith.index_cast %s : index to i8 + memref.store %v, %input[%c0, %y, %x, %ic] : memref<1x16x16x3xi8> + } + } + } + scf.for %ic = %c0 to %c3 step %c1 { + scf.for %p = %c0 to %c9 step %c1 { + scf.for %oc = %c0 to %c16 step %c1 { + %oc3 = arith.muli %oc, %c3 : index + %a = arith.addi %p, %ic : index + %b = arith.addi %a, %oc3 : index + %r = arith.remui %b, %c5 : index + %s = arith.subi %r, %c2 : index + %v = arith.index_cast %s : index to i8 + %ky_index = arith.divui %p, %c3 : index + %kx_index = arith.remui %p, %c3 : index + memref.store %v, %weight0[%oc, %ic, %ky_index, %kx_index] : memref<16x3x3x3xi8> + } + } + } + scf.for %oc = %c0 to %c16 step %c1 { + %s = arith.subi %oc, %c4 : index + %v = arith.index_cast %s : index to i32 + memref.store %v, %bias0[%oc] : memref<16xi32> + } + scf.for %ky = %c0 to %c3 step %c1 { + scf.for %kx = %c0 to %c3 step %c1 { + scf.for %c = %c0 to %c16 step %c1 { + %kx2 = arith.muli %kx, %c2 : index + %a = arith.addi %ky, %kx2 : index + %b = arith.addi %a, %c : index + %r = arith.remui %b, %c5 : index + %s = arith.subi %r, %c2 : index + %v = arith.index_cast %s : index to i8 + memref.store %v, %weight1[%c, %c0, %ky, %kx] : memref<16x1x3x3xi8> + } + } + } + linalg.fill ins(%i32_0 : i32) outs(%bias1 : memref<16xi32>) + %unit = arith.constant dense<1.0> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<1.0> : tensor<16x1x1x1xf32> + %it = bufferization.to_tensor %input restrict : memref<1x16x16x3xi8> to tensor<1x16x16x3xi8> + %input_float = "frontend.int8_to_fp32"(%it, %unit) : (tensor<1x16x16x3xi8>, tensor<1x1x1x1xf32>) -> tensor<1x16x16x3xf32> + %wt = bufferization.to_tensor %weight0 restrict : memref<16x3x3x3xi8> to tensor<16x3x3x3xi8> + %bt = bufferization.to_tensor %bias0 restrict : memref<16xi32> to tensor<16xi32> + %dt = bufferization.to_tensor %weight1 restrict : memref<16x1x3x3xi8> to tensor<16x1x3x3xi8> + %dbt = bufferization.to_tensor %bias1 restrict : memref<16xi32> to tensor<16xi32> + %first = custom.op "conv2d_i8"(%input_float, %wt, %bt, %unit, %unit, %weight_step) {strides = array, dilations = array, padding = array} : (tensor<1x16x16x3xf32>, tensor<16x3x3x3xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x8x8x16xf32> + %zero = arith.constant dense<0.0> : tensor<1x8x8x16xf32> + %relu = linalg.generic {indexing_maps = [affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%first, %zero: tensor<1x8x8x16xf32>, tensor<1x8x8x16xf32>) outs(%zero: tensor<1x8x8x16xf32>) { + ^bb0(%a: f32, %b: f32, %out: f32): + %r = arith.maximumf %a, %b : f32 + linalg.yield %r : f32 + } -> tensor<1x8x8x16xf32> + %second = custom.op "depthwise_conv2d_i8"(%relu, %dt, %dbt, %unit, %unit, %weight_step) {strides = array, dilations = array, padding = array} : (tensor<1x8x8x16xf32>, tensor<16x1x3x3xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x4x4x16xf32> + %q = "frontend.fp32_to_int8"(%second, %unit) : (tensor<1x4x4x16xf32>, tensor<1x1x1x1xf32>) -> tensor<1x4x4x16xi8> + %memory = bufferization.to_buffer %q : tensor<1x4x4x16xi8> to memref<1x4x4x16xi8> + func.call @check_result(%memory) : (memref<1x4x4x16xi8>) -> () + return %i8_0 : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_s2_depthwise_s2_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_s2_depthwise_s2_main.cpp similarity index 93% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_s2_depthwise_s2_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_s2_depthwise_s2_main.cpp index 8f9bf9202..b3609defb 100644 --- a/examples/chips/pebble/workloads/mlir_tests/tile/mega_conv2d_s2_depthwise_s2_main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_s2_depthwise_s2_main.cpp @@ -65,18 +65,18 @@ extern "C" void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, ++mismatch_count; } } - for (int tile_y = 0; tile_y < 2; ++tile_y) { - for (int tile_x = 0; tile_x < 2; ++tile_x) { + for (int core_y = 0; core_y < 2; ++core_y) { + for (int core_x = 0; core_x < 2; ++core_x) { uint32_t hash = 2166136261U; for (int dy = 0; dy < 2; ++dy) for (int dx = 0; dx < 2; ++dx) for (int c = 0; c < 16; ++c) { - int y = 2 * tile_y + dy, x = 2 * tile_x + dx; + int y = 2 * core_y + dy, x = 2 * core_x + dx; hash ^= (uint8_t)output[y * height_stride + x * width_stride + c]; hash *= 16777619U; } - printf("tile[%d,%d]=%08x%s", tile_y, tile_x, hash, - tile_x == 1 ? "\n" : " "); + printf("core[%d,%d]=%08x%s", core_y, core_x, hash, + core_x == 1 ? "\n" : " "); } } if (mismatch_count != 0) { diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact.mlir.in b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact.mlir.in new file mode 100644 index 000000000..344c5bf6f --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact.mlir.in @@ -0,0 +1,16 @@ +memref.global "private" constant @input : memref<1x7x7x288xi8> = dense<"0x@PEBBLE_GAP_C288_INPUT_HEX@"> alignment = 64 + +func.func private @poison_reused_banks() -> () +func.func private @check_result(memref<1x1x1x288xi8>) -> () + +func.func @main() -> i8 { + %zero = arith.constant 0 : i8 + %input = memref.get_global @input : memref<1x7x7x288xi8> + func.call @poison_reused_banks() : () -> () + %source_tensor = bufferization.to_tensor %input restrict : memref<1x7x7x288xi8> to tensor<1x7x7x288xi8> + %source_matrix = tensor.collapse_shape %source_tensor [[0, 1, 2], [3]] : tensor<1x7x7x288xi8> into tensor<49x288xi8> + %average = custom.op "global_avg_i8"(%source_tensor) {input_scale = 0.0578813739 : f32, output_scale = 0.0106952852 : f32} : (tensor<1x7x7x288xi8>) -> tensor<1x1x1x288xi8> + %memory = bufferization.to_buffer %average : tensor<1x1x1x288xi8> to memref<1x1x1x288xi8> + func.call @check_result(%memory) : (memref<1x1x1x288xi8>) -> () + return %zero : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_7x7_c288_exact_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_7x7_c288_exact_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact_main.cpp diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact.mlir.in b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact.mlir.in new file mode 100644 index 000000000..3e6462970 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact.mlir.in @@ -0,0 +1,84 @@ +memref.global "private" constant @input : memref<1x14x14x96xi8> = dense<"0x@PEBBLE_GAP_CONV_INPUT_HEX@"> alignment = 64 +memref.global "private" constant @weight1raw : memref<49168xi8> = dense<"0x00000000000000000000000000000000@PEBBLE_GAP_CONV_WEIGHT_HEX@"> alignment = 64 +memref.global "private" constant @bias1 : memref<24xi32> = dense<[-1929, -3345, -826, -4425, -1950, -1378, -2544, 3397, -134, -2621, 385, -1868, -3754, -948, 275, -53, -1955, -3197, -1065, -298, 536, -2267, -3803, -48]> alignment = 64 +memref.global "private" constant @scale1 : memref<24xf32> = dense<[9.64902255E-11, 1.20455174E-10, 9.11374864E-11, 6.07460609E-11, 6.96993296E-11, 2.21286572E-10, 8.11237535E-11, 1.33313652E-10, 1.26243266E-10, 1.44089102E-10, 0.0341239199, 6.56752361E-11, 6.62821395E-11, 1.91158489E-9, 1.69707901E-10, 4.13074019E-10, 5.95885979E-11, 8.92385332E-11, 6.12655274E-11, 2.72214418E-10, 2.77576795E-10, 2.37361242E-10, 6.75774228E-11, 1.48867613E-10]> alignment = 64 +memref.global "private" constant @weight2raw : memref<36880xi8> = dense<"0x00000000000000000000000000000000@PEBBLE_GAP_CONV_WEIGHT2_HEX@"> alignment = 64 +memref.global "private" constant @bias2 : memref<96xi32> = dense<[7051, 3006, 26558, 11845, -12167, 14432, -194, 9249, -17237, 22963, 7410, 91984, 105591, 6346, 90458, 43212, 45789, 33855, 29981, 10245, 4370, -34, -712, 15905, 22004, 549465, -5252, -3669, 7224, 18985, 8518, 8198, 16536, 51867, 4243, 6598, 9711, 18037, 21720, 397303, -253, 10751, -10281, -10424, 1839, 25010, 6995, -15222, 1786, -3524, -15704, 13370, 8813, 27391, 8927, -12218, -4217, -7494, 4135, 12616, 9593, 1431, -5377, 9947, 370, 11072, 5546, 22002, 10029, -3861, 904, 95829, 4717, -14504, 52996, 33835, 12322, 3329, 21275, -872, 1465, 17484, 6331, 63424, -29681, 34866, 3915, -2807, 71222, 5504, -13650, 15420, 7960, -9852, -13797, 21711]> alignment = 64 +memref.global "private" constant @scale2 : memref<96xf32> = dense<[0.00181691104, 4.059140E-3, 0.00131794217, 0.00212815683, 0.00430351309, 0.00403421139, 5.176660E-3, 0.00188985514, 0.00558878435, 0.00152020913, 0.00459996238, 5.82394132E-4, 4.24394966E-4, 0.0019368825, 7.07225234E-4, 6.37308462E-4, 0.00137251348, 7.15450151E-4, 0.00140084175, 0.00317073381, 5.073430E-3, 0.00307706394, 0.00386786251, 0.00195057772, 0.00210299343, 1.08822038E-4, 0.00415666727, 0.00333561096, 0.00485164393, 7.86353368E-4, 0.00419554161, 0.00339963497, 0.00123977242, 0.00106979779, 0.00394831039, 0.00133779203, 0.00282861362, 0.00155300961, 0.00196025497, 5.82583889E-5, 0.0032499244, 0.00156923081, 0.00266220793, 0.00191042456, 0.00408306951, 9.02582774E-4, 0.00326385885, 0.00143681327, 0.00142906886, 0.0026714697, 0.00458595809, 0.00430518808, 2.272860E-3, 0.00113706058, 0.00272057392, 0.00554801896, 6.058620E-3, 0.00159710483, 0.00501794415, 0.00278559304, 0.00328800408, 0.00337952748, 0.00245677959, 0.00205389271, 0.00226871902, 0.00263371225, 9.1695087E-4, 0.00129066338, 0.00205622637, 2.126570E-3, 9.56189353E-4, 2.12495346E-4, 9.6136285E-4, 0.00436848169, 9.72485286E-4, 4.94753069E-4, 0.00348583842, 0.00738966605, 0.00289806188, 0.00333968201, 0.00305957114, 0.00181978801, 0.00184966961, 3.52834439E-4, 0.00117929338, 9.00093349E-4, 0.00424369285, 0.00269078952, 5.541500E-4, 0.0025951541, 0.00531184906, 0.00391697325, 0.00378050678, 0.00192236435, 0.00131825986, 0.00273139146]> alignment = 64 +memref.global "private" constant @lut : memref<256xi8> = dense<"0x191A1A1A1A1A1A1B1B1B1B1B1B1C1C1C1C1C1C1D1D1D1D1D1D1E1E1E1E1E1E1F1F1F1F1F1F2020202020202121212121212222222222222323232323232424242424242525252525252626262626262727272727272828282828282929292929292A2A2A2A2A2A2B2B2B2B2B2B2C2C2C2C2C2C2D2D2D2D2D2D2E2E2E2E2E2E2F0404040505050505050606060606060707070707070808080808080909090909090A0A0A0A0A0A0B0B0B0B0B0B0C0C0C0C0C0C0D0D0D0D0D0D0E0E0E0E0E0E0F0F0F0F0F0F1010101010101111111111111212121212121313131313131414141414141515151515151616161616161717171717171818181818181919191919"> alignment = 64 +memref.global "private" constant @weight3raw : memref<73744xi8> = dense<"0x00000000000000000000000000000000@PEBBLE_GAP_CONV_WEIGHT3_HEX@"> alignment = 64 +memref.global "private" constant @bias3 : memref<40xi32> = dense<[104, -13887, -3685, -19928, 5678, 8790, -6325, -4745, 15957, 3843, 4642, 11420, -2754, -19949, 17269, -673, 16732, -31364, 19420, 3384, -1286, -5401, 3250, -9546, 3517, 6568, 9818, -3768, -8763, 19005, 2689, -2236, -3490, 6369, -18573, -726, -16056, 10604, -4229, -7803]> alignment = 64 +memref.global "private" constant @scale3 : memref<40xf32> = dense<[0.007250275, 0.0052957274, 0.00507997582, 5.148440E-3, 0.00615183357, 0.00551314512, 0.00447647972, 0.00624849601, 4.808580E-3, 0.00621820614, 0.00767033594, 0.00502041401, 0.00771069526, 0.00713740429, 5.340260E-3, 0.00554999569, 0.00578600029, 0.00608736509, 0.00647644047, 0.00674922066, 0.00569664268, 0.00644684816, 0.00639614975, 0.00778135518, 0.00677069137, 0.00366282626, 0.00660506124, 0.00643686252, 0.00655329926, 0.00661863247, 0.00624936819, 0.0104040103, 0.00504696835, 7.654780E-3, 0.00546469772, 5.341790E-3, 0.0050635729, 0.00579059403, 0.0084069958, 0.0039168736]> alignment = 64 +memref.global "private" constant @none : memref<1xi8> = dense<0> alignment = 64 +func.func private @check_result(memref<1x14x14x40xi8>) -> () func.func private @poison_reused_bank() -> () + +func.func @main() -> i8 { + %zero = arith.constant 0 : i8 + %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index + %c14 = arith.constant 14 : index + %c96 = arith.constant 96 : index + %input = memref.get_global @input : memref<1x14x14x96xi8> + %input_heap = memref.alloc() alignment = 64 : memref<1x14x14x96xi8> + scf.for %y = %c0 to %c14 step %c1 { + scf.for %x = %c0 to %c14 step %c1 { + scf.for %c = %c0 to %c96 step %c1 { + %v = memref.load %input[%c0, %y, %x, %c] : memref<1x14x14x96xi8> + memref.store %v, %input_heap[%c0, %y, %x, %c] : memref<1x14x14x96xi8> + } + } + } + %weight1raw = memref.get_global @weight1raw : memref<49168xi8> + %weight1 = memref.reinterpret_cast %weight1raw to offset: [16], sizes: [2, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<49168xi8> to memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> + %bias1 = memref.get_global @bias1 : memref<24xi32> + %scale1 = memref.get_global @scale1 : memref<24xf32> + %weight2raw = memref.get_global @weight2raw : memref<36880xi8> + %weight2 = memref.reinterpret_cast %weight2raw to offset: [16], sizes: [6, 24, 16, 16], strides: [6144, 256, 16, 1] : memref<36880xi8> to memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> + %bias2 = memref.get_global @bias2 : memref<96xi32> + %scale2 = memref.get_global @scale2 : memref<96xf32> + %lut = memref.get_global @lut : memref<256xi8> + %weight3raw = memref.get_global @weight3raw : memref<73744xi8> + %weight3 = memref.reinterpret_cast %weight3raw to offset: [16], sizes: [3, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<73744xi8> to memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> + %bias3 = memref.get_global @bias3 : memref<40xi32> %scale3 = memref.get_global @scale3 : memref<40xf32> + func.call @poison_reused_bank() : () -> () + %source_tensor = bufferization.to_tensor %input_heap restrict : memref<1x14x14x96xi8> to tensor<1x14x14x96xi8> + %source_matrix = tensor.collapse_shape %source_tensor [[0, 1, 2], [3]] : tensor<1x14x14x96xi8> into tensor<196x96xi8> + %w1_raw = bufferization.to_tensor %weight1 restrict : memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> to tensor<2x96x16x16xi8> + %w1_empty = tensor.empty() : tensor<96x24xi8> + %weights1 = linalg.generic {indexing_maps = [affine_map<(k,n)->(n floordiv 16,k,0,n mod 16)>, affine_map<(k,n)->(k,n)>], iterator_types = ["parallel", "parallel"]} ins(%w1_raw: tensor<2x96x16x16xi8>) outs(%w1_empty: tensor<96x24xi8>) { + ^bb0(%v: i8, %o: i8): linalg.yield %v : i8 + } -> tensor<96x24xi8> + %b1_raw = bufferization.to_tensor %bias1 restrict : memref<24xi32> to tensor<24xi32> + %bias_tensor1 = tensor.expand_shape %b1_raw [[0,1]] output_shape [1,24] : tensor<24xi32> into tensor<1x24xi32> + %s1_raw = bufferization.to_tensor %scale1 restrict : memref<24xf32> to tensor<24xf32> + %scale_tensor1 = tensor.expand_shape %s1_raw [[0,1]] output_shape [1,24] : tensor<24xf32> into tensor<1x24xf32> + %w2_raw = bufferization.to_tensor %weight2 restrict : memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> to tensor<6x24x16x16xi8> + %w2_empty = tensor.empty() : tensor<24x96xi8> + %weights2 = linalg.generic {indexing_maps = [affine_map<(k,n)->(n floordiv 16,k,0,n mod 16)>, affine_map<(k,n)->(k,n)>], iterator_types = ["parallel", "parallel"]} ins(%w2_raw: tensor<6x24x16x16xi8>) outs(%w2_empty: tensor<24x96xi8>) { + ^bb0(%v: i8, %o: i8): linalg.yield %v : i8 + } -> tensor<24x96xi8> + %b2_raw = bufferization.to_tensor %bias2 restrict : memref<96xi32> to tensor<96xi32> + %bias_tensor2 = tensor.expand_shape %b2_raw [[0,1]] output_shape [1,96] : tensor<96xi32> into tensor<1x96xi32> + %s2_raw = bufferization.to_tensor %scale2 restrict : memref<96xf32> to tensor<96xf32> + %scale_tensor2 = tensor.expand_shape %s2_raw [[0,1]] output_shape [1,96] : tensor<96xf32> into tensor<1x96xf32> + %w3_raw = bufferization.to_tensor %weight3 restrict : memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> to tensor<3x96x16x16xi8> + %w3_empty = tensor.empty() : tensor<96x40xi8> + %weights3 = linalg.generic {indexing_maps = [affine_map<(k,n)->(n floordiv 16,k,0,n mod 16)>, affine_map<(k,n)->(k,n)>], iterator_types = ["parallel", "parallel"]} ins(%w3_raw: tensor<3x96x16x16xi8>) outs(%w3_empty: tensor<96x40xi8>) { + ^bb0(%v: i8, %o: i8): linalg.yield %v : i8 + } -> tensor<96x40xi8> + %b3_raw = bufferization.to_tensor %bias3 restrict : memref<40xi32> to tensor<40xi32> + %bias_tensor3 = tensor.expand_shape %b3_raw [[0,1]] output_shape [1,40] : tensor<40xi32> into tensor<1x40xi32> + %s3_raw = bufferization.to_tensor %scale3 restrict : memref<40xf32> to tensor<40xf32> + %scale_tensor3 = tensor.expand_shape %s3_raw [[0,1]] output_shape [1,40] : tensor<40xf32> into tensor<1x40xf32> + %table = bufferization.to_tensor %lut restrict : memref<256xi8> to tensor<256xi8> + %average = custom.op "global_avg_i8"(%source_tensor) {input_scale = 0.043821115 : f32, output_scale = 0.023039883 : f32} : (tensor<1x14x14x96xi8>) -> tensor<1x1x1x96xi8> + %a1 = tensor.collapse_shape %average [[0,1,2],[3]] : tensor<1x1x1x96xi8> into tensor<1x96xi8> + %v1 = custom.op "linear_i8_requant"(%a1, %weights1, %bias_tensor1, %scale_tensor1) {relu = true} : (tensor<1x96xi8>, tensor<96x24xi8>, tensor<1x24xi32>, tensor<1x24xf32>) -> tensor<1x24xi8> + %v2 = custom.op "linear_i8_requant"(%v1, %weights2, %bias_tensor2, %scale_tensor2) {relu = false} : (tensor<1x24xi8>, tensor<24x96xi8>, tensor<1x96xi32>, tensor<1x96xf32>) -> tensor<1x96xi8> + %looked_up = custom.op "lookup_i8"(%v2, %table) : (tensor<1x96xi8>, tensor<256xi8>) -> tensor<1x96xi8> + %multiplied = custom.op "gate_mul_i8"(%looked_up, %source_matrix) {ratio = 3.404043242e-02 : f32} : (tensor<1x96xi8>, tensor<196x96xi8>) -> tensor<196x96xi8> + %v3 = custom.op "linear_i8_requant"(%multiplied, %weights3, %bias_tensor3, %scale_tensor3) {relu = false} : (tensor<196x96xi8>, tensor<96x40xi8>, tensor<1x40xi32>, tensor<1x40xf32>) -> tensor<196x40xi8> + %final = tensor.expand_shape %v3 [[0,1,2],[3]] output_shape [1,14,14,40] : tensor<196x40xi8> into tensor<1x14x14x40xi8> + %memory = bufferization.to_buffer %final : tensor<1x14x14x40xi8> to memref<1x14x14x40xi8> + func.call @check_result(%memory) : (memref<1x14x14x40xi8>) -> () + return %zero : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv2_exact_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv2_exact_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact_main.cpp diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv_exact.mlir.in b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv_exact.mlir.in new file mode 100644 index 000000000..1640fc8bb --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv_exact.mlir.in @@ -0,0 +1,33 @@ +memref.global "private" constant @input : memref<1x14x14x96xi8> = dense<"0x@PEBBLE_GAP_CONV_INPUT_HEX@"> alignment = 64 +memref.global "private" constant @weight : memref<2x96x16x16xi8> = dense<"0x@PEBBLE_GAP_CONV_WEIGHT_HEX@"> alignment = 64 +memref.global "private" constant @bias : memref<24xi32> = dense<[-1929, -3345, -826, -4425, -1950, -1378, -2544, 3397, -134, -2621, 385, -1868, -3754, -948, 275, -53, -1955, -3197, -1065, -298, 536, -2267, -3803, -48]> alignment = 64 +memref.global "private" constant @scale : memref<24xf32> = dense<[9.64902255E-11, 1.20455174E-10, 9.11374864E-11, 6.07460609E-11, 6.96993296E-11, 2.21286572E-10, 8.11237535E-11, 1.33313652E-10, 1.26243266E-10, 1.44089102E-10, 0.0341239199, 6.56752361E-11, 6.62821395E-11, 1.91158489E-9, 1.69707901E-10, 4.13074019E-10, 5.95885979E-11, 8.92385332E-11, 6.12655274E-11, 2.72214418E-10, 2.77576795E-10, 2.37361242E-10, 6.75774228E-11, 1.48867613E-10]> alignment = 64 +memref.global "private" constant @none : memref<1xi8> = dense<0> alignment = 64 + +func.func private @check_result(memref<1x1x1x24xi8>) -> () + +func.func @main() -> i8 { + %zero = arith.constant 0 : i8 + %input = memref.get_global @input : memref<1x14x14x96xi8> + %weight = memref.get_global @weight : memref<2x96x16x16xi8> + %bias = memref.get_global @bias : memref<24xi32> + %scale = memref.get_global @scale : memref<24xf32> + %source_tensor = bufferization.to_tensor %input restrict : memref<1x14x14x96xi8> to tensor<1x14x14x96xi8> + %source_matrix = tensor.collapse_shape %source_tensor [[0, 1, 2], [3]] : tensor<1x14x14x96xi8> into tensor<196x96xi8> + %w1_raw = bufferization.to_tensor %weight restrict : memref<2x96x16x16xi8> to tensor<2x96x16x16xi8> + %w1_empty = tensor.empty() : tensor<96x24xi8> + %weights1 = linalg.generic {indexing_maps = [affine_map<(k,n)->(n floordiv 16,k,0,n mod 16)>, affine_map<(k,n)->(k,n)>], iterator_types = ["parallel", "parallel"]} ins(%w1_raw: tensor<2x96x16x16xi8>) outs(%w1_empty: tensor<96x24xi8>) { + ^bb0(%v: i8, %o: i8): linalg.yield %v : i8 + } -> tensor<96x24xi8> + %b1_raw = bufferization.to_tensor %bias restrict : memref<24xi32> to tensor<24xi32> + %bias_tensor1 = tensor.expand_shape %b1_raw [[0,1]] output_shape [1,24] : tensor<24xi32> into tensor<1x24xi32> + %s1_raw = bufferization.to_tensor %scale restrict : memref<24xf32> to tensor<24xf32> + %scale_tensor1 = tensor.expand_shape %s1_raw [[0,1]] output_shape [1,24] : tensor<24xf32> into tensor<1x24xf32> + %average = custom.op "global_avg_i8"(%source_tensor) {input_scale = 0.043821115 : f32, output_scale = 0.023039883 : f32} : (tensor<1x14x14x96xi8>) -> tensor<1x1x1x96xi8> + %a1 = tensor.collapse_shape %average [[0,1,2],[3]] : tensor<1x1x1x96xi8> into tensor<1x96xi8> + %v1 = custom.op "linear_i8_requant"(%a1, %weights1, %bias_tensor1, %scale_tensor1) {relu = true} : (tensor<1x96xi8>, tensor<96x24xi8>, tensor<1x24xi32>, tensor<1x24xf32>) -> tensor<1x24xi8> + %final = tensor.expand_shape %v1 [[0,1,2],[3]] output_shape [1,1,1,24] : tensor<1x24xi8> into tensor<1x1x1x24xi8> + %memory = bufferization.to_buffer %final : tensor<1x1x1x24xi8> to memref<1x1x1x24xi8> + func.call @check_result(%memory) : (memref<1x1x1x24xi8>) -> () + return %zero : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv_exact_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv_exact_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_global_avg_conv_exact_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv_exact_main.cpp diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul.mlir new file mode 100644 index 000000000..e03d1eb8f --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul.mlir @@ -0,0 +1,45 @@ +func.func private @check_result(memref<1x16xf32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %eight = arith.constant 8 : index + %dim = arith.constant 16 : index + %input = memref.alloc() alignment = 64 : memref<1x16xi8> + %weight = memref.alloc() alignment = 64 : memref<16x16xi8> + %bias = memref.alloc() alignment = 64 : memref<16xi32> + scf.for %k = %zero to %dim step %one { + %irem = arith.remui %k, %seven : index + %ival = arith.subi %irem, %three : index + %iv = arith.index_cast %ival : index to i8 + memref.store %iv, %input[%zero, %k] : memref<1x16xi8> + %biasv = arith.subi %k, %eight : index + %bias32 = arith.index_cast %biasv : index to i32 + memref.store %bias32, %bias[%k] : memref<16xi32> + scf.for %column = %zero to %dim step %one { + %k2 = arith.muli %k, %two : index + %c3 = arith.muli %column, %three : index + %sum = arith.addi %k2, %c3 : index + %rem = arith.remui %sum, %five : index + %value = arith.subi %rem, %two : index + %wv = arith.index_cast %value : index to i8 + memref.store %wv, %weight[%k, %column] : memref<16x16xi8> + } + } + %it = bufferization.to_tensor %input restrict : memref<1x16xi8> to tensor<1x16xi8> + %wt = bufferization.to_tensor %weight restrict : memref<16x16xi8> to tensor<16x16xi8> + %bt = bufferization.to_tensor %bias restrict : memref<16xi32> to tensor<16xi32> + %bias_tensor = tensor.expand_shape %bt [[0, 1]] output_shape [1, 16] : tensor<16xi32> into tensor<1x16xi32> + %unit = arith.constant dense<1.0> : tensor<1x1xf32> + %weight_step = arith.constant dense<1.0> : tensor<1x16xf32> + %input_float = "frontend.int8_to_fp32"(%it, %unit) : (tensor<1x16xi8>, tensor<1x1xf32>) -> tensor<1x16xf32> + %result = custom.op "linear_i8"(%input_float, %wt, %bias_tensor, %unit, %unit, %weight_step) : (tensor<1x16xf32>, tensor<16x16xi8>, tensor<1x16xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x16xf32>) -> tensor<1x16xf32> + %memory = bufferization.to_buffer %result : tensor<1x16xf32> to memref<1x16xf32> + func.call @check_result(%memory) : (memref<1x16xf32>) -> () + return %zero_i8 : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul_m17_k65_n17.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul_m17_k65_n17.mlir new file mode 100644 index 000000000..aa5b49851 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul_m17_k65_n17.mlir @@ -0,0 +1,50 @@ +func.func private @check_result(memref<17x17xf32>) -> () +func.func @main() -> i8 { + %z8 = arith.constant 0 : i8 + %z = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %eight = arith.constant 8 : index + %dim = arith.constant 17 : index + %inner = arith.constant 65 : index + %input = memref.alloc() alignment = 64 : memref<17x65xi8> + %weight = memref.alloc() alignment = 64 : memref<65x17xi8> + %bias = memref.alloc() alignment = 64 : memref<17xi32> + scf.for %r = %z to %dim step %one { + scf.for %k = %z to %inner step %one { + %sum = arith.addi %r, %k : index + %rem = arith.remui %sum, %seven : index + %v = arith.subi %rem, %three : index + %i8 = arith.index_cast %v : index to i8 + memref.store %i8, %input[%r, %k] : memref<17x65xi8> + } + %b = arith.subi %r, %eight : index + %b32 = arith.index_cast %b : index to i32 + memref.store %b32, %bias[%r] : memref<17xi32> + } + scf.for %k = %z to %inner step %one { + scf.for %c = %z to %dim step %one { + %k2 = arith.muli %k, %two : index + %c3 = arith.muli %c, %three : index + %sum = arith.addi %k2, %c3 : index + %rem = arith.remui %sum, %five : index + %v = arith.subi %rem, %two : index + %i8 = arith.index_cast %v : index to i8 + memref.store %i8, %weight[%k, %c] : memref<65x17xi8> + } + } + %it = bufferization.to_tensor %input restrict : memref<17x65xi8> to tensor<17x65xi8> + %wt = bufferization.to_tensor %weight restrict : memref<65x17xi8> to tensor<65x17xi8> + %bt = bufferization.to_tensor %bias restrict : memref<17xi32> to tensor<17xi32> + %bias_tensor = tensor.expand_shape %bt [[0, 1]] output_shape [1, 17] : tensor<17xi32> into tensor<1x17xi32> + %unit = arith.constant dense<1.0> : tensor<1x1xf32> + %weight_step = arith.constant dense<0.5> : tensor<1x17xf32> + %input_float = "frontend.int8_to_fp32"(%it, %unit) : (tensor<17x65xi8>, tensor<1x1xf32>) -> tensor<17x65xf32> + %result = custom.op "linear_i8"(%input_float, %wt, %bias_tensor, %unit, %unit, %weight_step) : (tensor<17x65xf32>, tensor<65x17xi8>, tensor<1x17xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x17xf32>) -> tensor<17x17xf32> + %memory = bufferization.to_buffer %result : tensor<17x17xf32> to memref<17x17xf32> + func.call @check_result(%memory) : (memref<17x17xf32>) -> () + return %z8 : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul_m17_k65_n17_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul_m17_k65_n17_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul_m17_k65_n17_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul_m17_k65_n17_main.cpp diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_matmul_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_matmul_main.cpp diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_se_gate_conv.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_se_gate_conv.mlir new file mode 100644 index 000000000..37cce5e11 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_se_gate_conv.mlir @@ -0,0 +1,99 @@ +func.func private @check_result(memref<1x6x6x40xi8>) -> () + +func.func @main() -> i8 { + %i8_0 = arith.constant 0 : i8 + %i8_1 = arith.constant 1 : i8 + %i32_0 = arith.constant 0 : i32 + %i32_1 = arith.constant 1 : i32 + %f32_001 = arith.constant 0.01 : f32 + %f32_01 = arith.constant 0.1 : f32 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c5 = arith.constant 5 : index + %c6 = arith.constant 6 : index + %c96 = arith.constant 96 : index + %c256 = arith.constant 256 : index + %input = memref.alloc() alignment = 64 : memref<1x6x6x96xi8> + %w1raw = memref.alloc() alignment = 64 : memref<49168xi8> + %w1 = memref.reinterpret_cast %w1raw to offset: [16], sizes: [2, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<49168xi8> to memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> + %b1 = memref.alloc() alignment = 64 : memref<24xi32> + %s1 = memref.alloc() alignment = 64 : memref<24xf32> + %w2raw = memref.alloc() alignment = 64 : memref<36880xi8> + %w2 = memref.reinterpret_cast %w2raw to offset: [16], sizes: [6, 24, 16, 16], strides: [6144, 256, 16, 1] : memref<36880xi8> to memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> + %b2 = memref.alloc() alignment = 64 : memref<96xi32> + %s2 = memref.alloc() alignment = 64 : memref<96xf32> + %lut = memref.alloc() alignment = 64 : memref<256xi8> + %w3raw = memref.alloc() alignment = 64 : memref<73744xi8> + %w3 = memref.reinterpret_cast %w3raw to offset: [16], sizes: [3, 96, 16, 16], strides: [24576, 256, 16, 1] : memref<73744xi8> to memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> + %b3 = memref.alloc() alignment = 64 : memref<40xi32> + %s3 = memref.alloc() alignment = 64 : memref<40xf32> + scf.for %y = %c0 to %c6 step %c1 { + scf.for %x = %c0 to %c6 step %c1 { + scf.for %c = %c0 to %c96 step %c1 { + %a = arith.addi %y, %x : index + %d = arith.addi %a, %c : index + %r = arith.remui %d, %c5 : index + %v = arith.addi %r, %c1 : index + %q = arith.index_cast %v : index to i8 + memref.store %q, %input[%c0, %y, %x, %c] : memref<1x6x6x96xi8> + } + } + } + linalg.fill ins(%i8_1 : i8) outs(%w1 : memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>>) + linalg.fill ins(%i8_1 : i8) outs(%w2 : memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>>) + linalg.fill ins(%i8_1 : i8) outs(%w3 : memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>>) + linalg.fill ins(%i32_1 : i32) outs(%b1 : memref<24xi32>) + linalg.fill ins(%i32_0 : i32) outs(%b2 : memref<96xi32>) + linalg.fill ins(%i32_0 : i32) outs(%b3 : memref<40xi32>) + linalg.fill ins(%f32_001 : f32) outs(%s1 : memref<24xf32>) + linalg.fill ins(%f32_01 : f32) outs(%s2 : memref<96xf32>) + linalg.fill ins(%f32_001 : f32) outs(%s3 : memref<40xf32>) + scf.for %i = %c0 to %c256 step %c1 { + %v = arith.index_cast %i : index to i8 + memref.store %v, %lut[%i] : memref<256xi8> + } + %source_tensor = bufferization.to_tensor %input restrict : memref<1x6x6x96xi8> to tensor<1x6x6x96xi8> + %source_matrix = tensor.collapse_shape %source_tensor [[0, 1, 2], [3]] : tensor<1x6x6x96xi8> into tensor<36x96xi8> + %w1_raw = bufferization.to_tensor %w1 restrict : memref<2x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> to tensor<2x96x16x16xi8> + %w1_empty = tensor.empty() : tensor<96x24xi8> + %weights1 = linalg.generic {indexing_maps = [affine_map<(k,n)->(n floordiv 16,k,0,n mod 16)>, affine_map<(k,n)->(k,n)>], iterator_types = ["parallel", "parallel"]} ins(%w1_raw: tensor<2x96x16x16xi8>) outs(%w1_empty: tensor<96x24xi8>) { + ^bb0(%v: i8, %o: i8): linalg.yield %v : i8 + } -> tensor<96x24xi8> + %b1_raw = bufferization.to_tensor %b1 restrict : memref<24xi32> to tensor<24xi32> + %bias_tensor1 = tensor.expand_shape %b1_raw [[0,1]] output_shape [1,24] : tensor<24xi32> into tensor<1x24xi32> + %s1_raw = bufferization.to_tensor %s1 restrict : memref<24xf32> to tensor<24xf32> + %scale_tensor1 = tensor.expand_shape %s1_raw [[0,1]] output_shape [1,24] : tensor<24xf32> into tensor<1x24xf32> + %w2_raw = bufferization.to_tensor %w2 restrict : memref<6x24x16x16xi8, strided<[6144, 256, 16, 1], offset: 16>> to tensor<6x24x16x16xi8> + %w2_empty = tensor.empty() : tensor<24x96xi8> + %weights2 = linalg.generic {indexing_maps = [affine_map<(k,n)->(n floordiv 16,k,0,n mod 16)>, affine_map<(k,n)->(k,n)>], iterator_types = ["parallel", "parallel"]} ins(%w2_raw: tensor<6x24x16x16xi8>) outs(%w2_empty: tensor<24x96xi8>) { + ^bb0(%v: i8, %o: i8): linalg.yield %v : i8 + } -> tensor<24x96xi8> + %b2_raw = bufferization.to_tensor %b2 restrict : memref<96xi32> to tensor<96xi32> + %bias_tensor2 = tensor.expand_shape %b2_raw [[0,1]] output_shape [1,96] : tensor<96xi32> into tensor<1x96xi32> + %s2_raw = bufferization.to_tensor %s2 restrict : memref<96xf32> to tensor<96xf32> + %scale_tensor2 = tensor.expand_shape %s2_raw [[0,1]] output_shape [1,96] : tensor<96xf32> into tensor<1x96xf32> + %w3_raw = bufferization.to_tensor %w3 restrict : memref<3x96x16x16xi8, strided<[24576, 256, 16, 1], offset: 16>> to tensor<3x96x16x16xi8> + %w3_empty = tensor.empty() : tensor<96x40xi8> + %weights3 = linalg.generic {indexing_maps = [affine_map<(k,n)->(n floordiv 16,k,0,n mod 16)>, affine_map<(k,n)->(k,n)>], iterator_types = ["parallel", "parallel"]} ins(%w3_raw: tensor<3x96x16x16xi8>) outs(%w3_empty: tensor<96x40xi8>) { + ^bb0(%v: i8, %o: i8): linalg.yield %v : i8 + } -> tensor<96x40xi8> + %b3_raw = bufferization.to_tensor %b3 restrict : memref<40xi32> to tensor<40xi32> + %bias_tensor3 = tensor.expand_shape %b3_raw [[0,1]] output_shape [1,40] : tensor<40xi32> into tensor<1x40xi32> + %s3_raw = bufferization.to_tensor %s3 restrict : memref<40xf32> to tensor<40xf32> + %scale_tensor3 = tensor.expand_shape %s3_raw [[0,1]] output_shape [1,40] : tensor<40xf32> into tensor<1x40xf32> + %table = bufferization.to_tensor %lut restrict : memref<256xi8> to tensor<256xi8> + scf.for %iteration = %c0 to %c2 step %c1 { + %average = custom.op "global_avg_i8"(%source_tensor) {input_scale = 1.0 : f32, output_scale = 1.0 : f32} : (tensor<1x6x6x96xi8>) -> tensor<1x1x1x96xi8> + %a1 = tensor.collapse_shape %average [[0,1,2],[3]] : tensor<1x1x1x96xi8> into tensor<1x96xi8> + %v1 = custom.op "linear_i8_requant"(%a1, %weights1, %bias_tensor1, %scale_tensor1) {relu = true} : (tensor<1x96xi8>, tensor<96x24xi8>, tensor<1x24xi32>, tensor<1x24xf32>) -> tensor<1x24xi8> + %v2 = custom.op "linear_i8_requant"(%v1, %weights2, %bias_tensor2, %scale_tensor2) {relu = false} : (tensor<1x24xi8>, tensor<24x96xi8>, tensor<1x96xi32>, tensor<1x96xf32>) -> tensor<1x96xi8> + %looked_up = custom.op "lookup_i8"(%v2, %table) : (tensor<1x96xi8>, tensor<256xi8>) -> tensor<1x96xi8> + %multiplied = custom.op "gate_mul_i8"(%looked_up, %source_matrix) {ratio = 1.000000015e-01 : f32} : (tensor<1x96xi8>, tensor<36x96xi8>) -> tensor<36x96xi8> + %v3 = custom.op "linear_i8_requant"(%multiplied, %weights3, %bias_tensor3, %scale_tensor3) {relu = false} : (tensor<36x96xi8>, tensor<96x40xi8>, tensor<1x40xi32>, tensor<1x40xf32>) -> tensor<36x40xi8> + %final = tensor.expand_shape %v3 [[0,1,2],[3]] output_shape [1,6,6,40] : tensor<36x40xi8> into tensor<1x6x6x40xi8> + %memory = bufferization.to_buffer %final : tensor<1x6x6x40xi8> to memref<1x6x6x40xi8> + func.call @check_result(%memory) : (memref<1x6x6x40xi8>) -> () + } + return %i8_0 : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/tile/mega_se_gate_conv_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_se_gate_conv_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/tile/mega_se_gate_conv_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_se_gate_conv_main.cpp diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt new file mode 100644 index 000000000..fa20eb6a1 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt @@ -0,0 +1,15 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET pebble-mlir-tests-build) +set(BUCKYBALL_MLIR_TEST_PREFIX linalg) + +foreach(test + linalg_relu_513x17_i32 + linalg_matmul_16x16_i8 + linalg_transpose_16x16_i8 + linalg_transpose_detect_1x84x100_f32 + linalg_transpose_nchw_nhwc_1x3x8x8_f32 + linalg_transpose_oihw_ohwi_4x8x3x3_f32 + linalg_mega_conv2d_k3_c16) + add_buckyball_mlir_test(${test} + TARGET pebble + PASSES ${PEBBLE_CUSTOM_PASSES}) +endforeach() diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_matmul_16x16_i8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_matmul_16x16_i8.mlir new file mode 100644 index 000000000..4e8544c9a --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_matmul_16x16_i8.mlir @@ -0,0 +1,41 @@ +func.func private @check_result(memref<16x16xi32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %dim = arith.constant 16 : index + %a = memref.alloc() alignment = 64 : memref<16x16xi8> + %b = memref.alloc() alignment = 64 : memref<16x16xi8> + + scf.for %row = %zero to %dim step %one { + scf.for %column = %zero to %dim step %one { + %r3 = arith.muli %row, %three : index + %c5 = arith.muli %column, %five : index + %asum = arith.addi %r3, %c5 : index + %arem = arith.remui %asum, %seven : index + %aval = arith.subi %arem, %three : index + %av = arith.index_cast %aval : index to i8 + %r2 = arith.muli %row, %two : index + %c3 = arith.muli %column, %three : index + %bsum = arith.addi %r2, %c3 : index + %brem = arith.remui %bsum, %five : index + %bval = arith.subi %brem, %two : index + %bv = arith.index_cast %bval : index to i8 + memref.store %av, %a[%row, %column] : memref<16x16xi8> + memref.store %bv, %b[%row, %column] : memref<16x16xi8> + } + } + %at = bufferization.to_tensor %a restrict : memref<16x16xi8> to tensor<16x16xi8> + %bt = bufferization.to_tensor %b restrict : memref<16x16xi8> to tensor<16x16xi8> + %init = arith.constant dense<0> : tensor<16x16xi32> + %result = linalg.matmul ins(%at, %bt: tensor<16x16xi8>, tensor<16x16xi8>) outs(%init: tensor<16x16xi32>) -> tensor<16x16xi32> + %memory = bufferization.to_buffer %result : tensor<16x16xi32> to memref<16x16xi32> + func.call @check_result(%memory) : (memref<16x16xi32>) -> () + return %zero_i8 : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_matmul_16x16_i8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_matmul_16x16_i8_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_matmul_16x16_i8_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_matmul_16x16_i8_main.cpp diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_mega_conv2d_k3_c16.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_mega_conv2d_k3_c16.mlir new file mode 100644 index 000000000..9bc197835 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_mega_conv2d_k3_c16.mlir @@ -0,0 +1,76 @@ +func.func private @check_result(memref<1x16x1x1xf32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %zero_f32 = arith.constant 0.0 : f32 + %one_i8 = arith.constant 1 : i8 + %one_f32 = arith.constant 1.0 : f32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %eight = arith.constant 8 : index + %nine = arith.constant 9 : index + %sixteen = arith.constant 16 : index + %input = memref.alloc() : memref<1x3x3x16xi8> + %weight = memref.alloc() : memref<16x16x3x3xi8> + %bias = memref.alloc() : memref<16xi32> + linalg.fill ins(%zero_i8 : i8) outs(%input : memref<1x3x3x16xi8>) + linalg.fill ins(%zero_i8 : i8) outs(%weight : memref<16x16x3x3xi8>) + scf.for %position = %zero to %nine step %one { + %p2 = arith.muli %position, %two : index + %h = arith.divui %position, %three : index + %w = arith.remui %position, %three : index + scf.for %channel = %zero to %sixteen step %one { + %c3 = arith.muli %channel, %three : index + %isum = arith.addi %p2, %c3 : index + %irem = arith.remui %isum, %five : index + %ival = arith.subi %irem, %two : index + %iv = arith.index_cast %ival : index to i8 + memref.store %iv, %input[%zero, %h, %w, %channel] : memref<1x3x3x16xi8> + scf.for %out = %zero to %sixteen step %one { + %wsum0 = arith.addi %c3, %p2 : index + %wsum = arith.addi %wsum0, %out : index + %wrem = arith.remui %wsum, %five : index + %wval = arith.subi %wrem, %two : index + %wv = arith.index_cast %wval : index to i8 + memref.store %wv, %weight[%out, %channel, %h, %w] : memref<16x16x3x3xi8> + } + } + } + scf.for %out = %zero to %sixteen step %one { + %bval = arith.subi %out, %eight : index + %bv = arith.index_cast %bval : index to i32 + memref.store %bv, %bias[%out] : memref<16xi32> + } + %unit = arith.constant dense<1.0> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<1.0> : tensor<16x1x1x1xf32> + %it = bufferization.to_tensor %input restrict : memref<1x3x3x16xi8> to tensor<1x3x3x16xi8> + %wt = bufferization.to_tensor %weight restrict : memref<16x16x3x3xi8> to tensor<16x16x3x3xi8> + %bt = bufferization.to_tensor %bias restrict : memref<16xi32> to tensor<16xi32> + %input_float = "frontend.int8_to_fp32"(%it, %unit) : (tensor<1x3x3x16xi8>, tensor<1x1x1x1xf32>) -> tensor<1x3x3x16xf32> + %q = "frontend.fp32_to_int8"(%input_float, %unit) : (tensor<1x3x3x16xf32>, tensor<1x1x1x1xf32>) -> tensor<1x3x3x16xi8> + %x = "frontend.int8_to_fp32"(%q, %unit) : (tensor<1x3x3x16xi8>, tensor<1x1x1x1xf32>) -> tensor<1x3x3x16xf32> + %wf = "frontend.int8_to_fp32"(%wt, %weight_step) : (tensor<16x16x3x3xi8>, tensor<16x1x1x1xf32>) -> tensor<16x16x3x3xf32> + %we = tensor.empty() : tensor<16x3x3x16xf32> + %weights = linalg.transpose ins(%wf: tensor<16x16x3x3xf32>) outs(%we: tensor<16x3x3x16xf32>) permutation = [0,2,3,1] + %scale = linalg.generic {indexing_maps = [affine_map<(f,c,h,w)->(0,c,h,w)>, affine_map<(f,c,h,w)->(f,c,h,w)>, affine_map<(f,c,h,w)->(f,c,h,w)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%unit, %weight_step: tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) outs(%weight_step: tensor<16x1x1x1xf32>) { + ^bb0(%a: f32, %b: f32, %o: f32): + %r = arith.mulf %a, %b : f32 + linalg.yield %r : f32 + } -> tensor<16x1x1x1xf32> + %bs = tensor.collapse_shape %scale [[0,1,2,3]] : tensor<16x1x1x1xf32> into tensor<16xf32> + %bf = "frontend.int32_to_fp32"(%bt, %bs) : (tensor<16xi32>, tensor<16xf32>) -> tensor<16xf32> + %empty = tensor.empty() : tensor<1x1x1x16xf32> + %initial = linalg.generic {indexing_maps = [affine_map<(n,h,w,c)->(c)>, affine_map<(n,h,w,c)->(n,h,w,c)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%bf: tensor<16xf32>) outs(%empty: tensor<1x1x1x16xf32>) { + ^bb0(%a: f32, %o: f32): linalg.yield %a : f32 + } -> tensor<1x1x1x16xf32> + %result = linalg.conv_2d_nhwc_fhwc {strides = dense<1> : tensor<2xi64>, dilations = dense<1> : tensor<2xi64>} ins(%x, %weights: tensor<1x3x3x16xf32>, tensor<16x3x3x16xf32>) outs(%initial: tensor<1x1x1x16xf32>) -> tensor<1x1x1x16xf32> + %oe = tensor.empty() : tensor<1x16x1x1xf32> + %nchw = linalg.transpose ins(%result: tensor<1x1x1x16xf32>) outs(%oe: tensor<1x16x1x1xf32>) permutation = [0,3,1,2] + %memory = bufferization.to_buffer %nchw : tensor<1x16x1x1xf32> to memref<1x16x1x1xf32> + func.call @check_result(%memory) : (memref<1x16x1x1xf32>) -> () + return %zero_i8 : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_mega_conv2d_k3_c16_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_mega_conv2d_k3_c16_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_mega_conv2d_k3_c16_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_mega_conv2d_k3_c16_main.cpp diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_relu_513x17_i32.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_relu_513x17_i32.mlir similarity index 94% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_relu_513x17_i32.mlir rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_relu_513x17_i32.mlir index 6480fca6d..bc06544a2 100644 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_relu_513x17_i32.mlir +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_relu_513x17_i32.mlir @@ -38,7 +38,5 @@ func.func @main() -> i8 { } func.call @check_result(%output) : (memref<513x17xi32>) -> () - memref.dealloc %input : memref<513x17xi32> - memref.dealloc %output : memref<513x17xi32> return %zeroI8 : i8 } diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_relu_513x17_i32_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_relu_513x17_i32_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_relu_513x17_i32_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_relu_513x17_i32_main.cpp diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_16x16_i8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_16x16_i8.mlir new file mode 100644 index 000000000..622974fa5 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_16x16_i8.mlir @@ -0,0 +1,27 @@ +func.func private @check_result(memref<16x16xi8>) -> () + +func.func @main() -> i8 { + %zero = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : i8 + %c5 = arith.constant 5 : i8 + %c16 = arith.constant 16 : index + %input = memref.alloc() alignment = 64 : memref<16x16xi8> + scf.for %i = %c0 to %c16 step %c1 { + scf.for %j = %c0 to %c16 step %c1 { + %ii = arith.index_cast %i : index to i8 + %jj = arith.index_cast %j : index to i8 + %iv = arith.muli %ii, %c3 : i8 + %jv = arith.muli %jj, %c5 : i8 + %v = arith.addi %iv, %jv : i8 + memref.store %v, %input[%i, %j] : memref<16x16xi8> + } + } + %it = bufferization.to_tensor %input restrict : memref<16x16xi8> to tensor<16x16xi8> + %empty = tensor.empty() : tensor<16x16xi8> + %result = linalg.transpose ins(%it: tensor<16x16xi8>) outs(%empty: tensor<16x16xi8>) permutation = [1,0] + %memory = bufferization.to_buffer %result : tensor<16x16xi8> to memref<16x16xi8> + func.call @check_result(%memory) : (memref<16x16xi8>) -> () + return %zero : i8 +} diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_16x16_i8_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_16x16_i8_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_16x16_i8_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_16x16_i8_main.cpp diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32.mlir similarity index 91% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32.mlir rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32.mlir index 54fb4af68..3008f81eb 100644 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32.mlir +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32.mlir @@ -24,7 +24,5 @@ func.func @main() -> i8 { linalg.transpose ins(%in : memref<1x84x100xf32>) outs(%out : memref<1x100x84xf32>) permutation = [0, 2, 1] func.call @check_result(%out) : (memref<1x100x84xf32>) -> () - memref.dealloc %in : memref<1x84x100xf32> - memref.dealloc %out : memref<1x100x84xf32> return %z : i8 } diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_detect_1x84x100_f32_main.cpp diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32.mlir similarity index 92% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32.mlir rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32.mlir index 434d754d4..0a467830c 100644 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32.mlir +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32.mlir @@ -28,7 +28,5 @@ func.func @main() -> i8 { linalg.transpose ins(%in : memref<1x3x8x8xf32>) outs(%out : memref<1x8x8x3xf32>) permutation = [0, 2, 3, 1] func.call @check_result(%out) : (memref<1x8x8x3xf32>) -> () - memref.dealloc %in : memref<1x3x8x8xf32> - memref.dealloc %out : memref<1x8x8x3xf32> return %z : i8 } diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_nchw_nhwc_1x3x8x8_f32_main.cpp diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32.mlir similarity index 94% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32.mlir rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32.mlir index c9799b70d..1c5ceb748 100644 --- a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32.mlir +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32.mlir @@ -35,7 +35,5 @@ func.func @main() -> i8 { linalg.transpose ins(%in : memref<4x8x3x3xf32>) outs(%out : memref<4x3x3x8xf32>) permutation = [0, 2, 3, 1] func.call @check_result(%out) : (memref<4x3x3x8xf32>) -> () - memref.dealloc %in : memref<4x8x3x3xf32> - memref.dealloc %out : memref<4x3x3x8xf32> return %z : i8 } diff --git a/examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32_main.cpp similarity index 100% rename from examples/chips/pebble/workloads/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32_main.cpp rename to examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/linalg_transpose_oihw_ohwi_4x8x3x3_f32_main.cpp diff --git a/examples/designs/poly/arch/build.sc b/examples/designs/poly/arch/build.sc new file mode 100644 index 000000000..f64ac6e98 --- /dev/null +++ b/examples/designs/poly/arch/build.sc @@ -0,0 +1,19 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.arch.src.main.scala.framework.device.cbus.{build => cbus_build} +import $file.^.^.^.{build => examples} +import $file.^.^.^.components.cores.reduce.arch.{build => reduce} + +val designRoot: os.Path = millSourcePath / os.up + +object design extends SbtModule { + override def generatedSources = T(cbus_build.registry(sources() ++ reduce.library.sources(), T.dest)) + override def millSourcePath = designRoot + override def forkWorkingDir = T(designRoot) + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(examples.library, reduce.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(designRoot / "arch", designRoot / "configs") +} diff --git a/examples/designs/poly/arch/device/DeviceTop.scala b/examples/designs/poly/arch/device/DeviceTop.scala new file mode 100644 index 000000000..8f9bb9765 --- /dev/null +++ b/examples/designs/poly/arch/device/DeviceTop.scala @@ -0,0 +1,90 @@ +package examples.designs.poly + +import framework.lib.arbiter.{RRArbiter} + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.device.configs.DeviceParam +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import buckyball.config.Design +import _root_.configs.GlobalConfig +import framework.core.clink.CLinkIO +import framework.core.memory.home.Home +import framework.device.ddr.Bridge +import framework.device.peripherals.Controller +import framework.device.peripherals.bootrom.BootRomLines +import framework.device.dlink.BaseDevice +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.LineResponse +import framework.lib.queue.{Queue => ResponseQueue} + +/** Configured cores share the device coherence fabric. */ +@instantiable +class DeviceTop(b: GlobalConfig, design: Design) extends BaseDevice(b.device.ddr.axi) { + val device = b.device + val cores = device.cores + require(cores.forall(p => !p.clink.backing && p.clink.requesters == 3)) + val sharedHome = cores.head.memory.home + require( + cores.forall(_.memory.memoryAccess.memory.chi == sharedHome.memory.chi), + "Core coherent channels must match the shared Home" + ) + require(cores.forall(_.memory.home == sharedHome) && sharedHome.memory.agents == cores.size * 3) + + val bus: Instance[CBus] = + Instantiate(new CBus(b.device, CBus.generators(design, b.device, b.sim, CoreComponents.factories))) + val home: Instance[Home] = Instantiate(new Home(sharedHome)) + val peripherals: Instance[Controller] = + Instantiate(new Controller(device, device.ddr.axi, cores.head.memory.memoryAccess.memory)) + + val peripheralBus: Instance[axi4.Decoder] = Instantiate(new axi4.Decoder( + Seq( + (device.clint.base, device.clint.bytes), + (device.plic.base, device.plic.bytes), + (device.scu.baseAddress, device.scu.totalSizeBytes) + ), + device.ddr.axi + )) + + val rom: Instance[BootRomLines] = Instantiate(new BootRomLines(device.bootrom, sharedHome.memory.chi)) + val bridge: Instance[Bridge] = Instantiate(new Bridge(device.ddr)) + val responses: Instance[RRArbiter[LineResponse]] = + Instantiate(new RRArbiter(new LineResponse(sharedHome.memory.chi), 2)) + val response: Instance[ResponseQueue[LineResponse]] = + Instantiate(new ResponseQueue(new LineResponse(sharedHome.memory.chi), 1, false, false)) + val memoryBus: Instance[axi4.Interconnect] = + Instantiate(new axi4.Interconnect(axi4.InterconnectParam(device.ddr.axi, 2, None))) + + home.io.active := true.B + home.io.blockRequesterRsp := false.B + home.io.requesters <> bus.coherent + + peripheralBus.io.in <> bus.uncached + peripherals.io.axi <> peripheralBus.io.matched + bus.external <> peripheralBus.io.other + peripherals.io.hardwareInterrupt.sources := 0.U + bus.interrupts := peripherals.io.hardwareInterrupt.harts + + require(device.ddr.clients == 1) + val request = home.io.memoryReq + val inRom = request.bits.addr >= device.bootrom.base.U && + request.bits.addr < (device.bootrom.base + device.bootrom.bytes).U + rom.io.request.valid := request.valid && inRom + rom.io.request.bits := request.bits + bridge.io.request(0).valid := request.valid && !inRom + bridge.io.request(0).bits := request.bits + request.ready := Mux(inRom, rom.io.request.ready, bridge.io.request(0).ready) + + responses.io.in(0) <> rom.io.response + responses.io.in(1) <> bridge.io.response(0) + + response.io.enq <> responses.io.out + home.io.memoryResp <> response.io.deq + + memoryBus.io.in(0) <> bus.dlink.axi + memoryBus.io.in(1) <> bridge.io.axi + dlink.axi <> memoryBus.io.out +} diff --git a/examples/designs/poly/arch/device/core/PolyCore.scala b/examples/designs/poly/arch/device/core/PolyCore.scala new file mode 100644 index 000000000..27e1c6f24 --- /dev/null +++ b/examples/designs/poly/arch/device/core/PolyCore.scala @@ -0,0 +1,154 @@ +package examples.designs.poly + +import framework.core.tbus.generated.ThreadComponents +import sims.configs.SimParam +import framework.device.cbus.CoreFactory +import buckyball.config.{CorePlacement, Design} + +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.configs.CoreParam +import framework.core.tbus.TBus +import framework.ip.bus.chi.rnf.AxiCache +import framework.core.execution.Execution +import framework.core.memory.Memory +import framework.core.tss.Store +import framework.core.shm.SHM +import framework.core.controller.Controller +import framework.core.clink.{CLinkIO, HasCLink} + +@instantiable +class PolyCore(cp: CoreParam, simp: SimParam, threadTraceIds: Seq[Int]) extends Module with HasCLink { + + val diffTest = simp.diffTest + require( + cp.clink.harts == 1 && + ((cp.clink.backing && cp.clink.requesters == 0) || + (!cp.clink.backing && cp.clink.requesters == 2 + cp.dmaCache.size)), + "PolyCore requires one hart and a matching cache interface" + ) + if (!cp.clink.backing) { + require( + cp.dmaCache.nonEmpty == cp.threads.nonEmpty, + "Thread DMA requires its own coherent cache requester" + ) + } + if (cp.threads.nonEmpty) { + if (cp.clink.shared) + require(cp.clink.c2c.contains(cp.c2c.axi), "C2C protocol must match its CLink channel") + val tracking = cp.memory.dcache.tracking + cp.threads.foreach { thread => + val transport = thread.acc.get.transport + require(transport.axi == cp.interconnect.bus, "Thread DMA protocol must match the core DDR interconnect") + require( + transport.prepared.bus.addressBits == cp.memory.memoryAccess.memory.chi.addressBits, + "Thread page-table address width must match the core memory service" + ) + require( + ( + transport.interlock.addressBits, + transport.interlock.idBits, + transport.interlock.lineBytes, + transport.interlock.maxRanges + ) == + (tracking.addressBits, tracking.idBits, tracking.lineBytes, tracking.maxRanges), + "Thread cache-maintenance interface must match the core memory service" + ) + } + } + + @public + val clink = IO(new CLinkIO(cp.clink)) + + val execution: Instance[Execution] = + Instantiate(new Execution(cp.execution(directAccelerator = false), localRocc = false)) + val controller: Instance[Controller] = Instantiate(new Controller(cp.controller)) + val memory: Instance[Memory] = + Instantiate(new Memory(cp.memory, externalHome = !cp.clink.backing)) + + execution.io.interrupts := clink.interrupts(0) + clink.memory.backing.zip(memory.io.backing).foreach { case (port, backing) => port <> backing } + clink.memory.coherent.take(2).zip(memory.io.coherent).foreach { case (port, cache) => port <> cache } + clink.memory.uncached <> memory.io.uncached + execution.io.workDrained := controller.io.drained + controller.io.execution <> execution.io.controller.get + memory.io.fetch <> execution.io.fetch + memory.io.data <> execution.io.data + memory.io.timeRequest <> execution.io.timeRequest + execution.io.timeResponse <> memory.io.timeResponse + memory.io.commandAllowed := controller.io.memoryAllowed + + if (cp.threads.nonEmpty) { + val tbus: Instance[TBus] = + Instantiate(new TBus( + cp, + simp, + threadTraceIds, + TBus.generators(cp, simp, threadTraceIds, ThreadComponents.factories) + )) + tbus.io.shm.foreach { port => + val shm: Instance[SHM] = Instantiate(new SHM(cp.sharedFrontend, cp.sharedMemory, cp.tbus.get)) + shm.io <> port + } + tbus.io.tss.foreach { port => + val tss: Instance[Store] = Instantiate(new Store(cp.tss.get)) + tss.io.clients <> port + tss.io.host <> controller.io.tss.get.host + tss.io.inUse := controller.io.tss.get.inUse + tss.io.cancel := controller.io.tss.get.cancel + controller.io.tss.get.busy := tss.io.busy + controller.io.tss.get.hostBusy := tss.io.hostBusy + } + if (cp.clink.backing) { + require(cp.interconnect.bus == cp.clink.ddr, "Thread DDR interconnect must match CLink") + clink.memory.ddr <> tbus.io.ddr + } else { + val dma = Instantiate(new AxiCache(cp.interconnect.bus, cp.dmaCache.get)) + dma.io.axi <> tbus.io.ddr + dma.io.axi.ar.bits.cache := 15.U + dma.io.axi.aw.bits.cache := 15.U + clink.memory.coherent(2) <> dma.io.chi + clink.memory.ddr.aw.valid := false.B + clink.memory.ddr.aw.bits := 0.U.asTypeOf(clink.memory.ddr.aw.bits) + clink.memory.ddr.w.valid := false.B + clink.memory.ddr.w.bits := 0.U.asTypeOf(clink.memory.ddr.w.bits) + clink.memory.ddr.ar.valid := false.B + clink.memory.ddr.ar.bits := 0.U.asTypeOf(clink.memory.ddr.ar.bits) + clink.memory.ddr.b.ready := false.B + clink.memory.ddr.r.ready := false.B + } + clink.c2c.zip(tbus.io.c2c).foreach { case (link, port) => link <> port } + tbus.io.control.zip(controller.io.threads).foreach { case (port, control) => port <> control } + tbus.io.remote.zip(controller.io.remote).foreach { case (port, remote) => port <> remote } + tbus.io.move <> controller.io.move + tbus.io.memory.zip(memory.io.threads).foreach { case (port, service) => port <> service } + } else { + require(!cp.clink.shared, "A core without a shared bank network must not advertise a C2C endpoint") + clink.memory.ddr.aw.valid := false.B + clink.memory.ddr.aw.bits := 0.U.asTypeOf(clink.memory.ddr.aw.bits) + clink.memory.ddr.w.valid := false.B + clink.memory.ddr.w.bits := 0.U.asTypeOf(clink.memory.ddr.w.bits) + clink.memory.ddr.ar.valid := false.B + clink.memory.ddr.ar.bits := 0.U.asTypeOf(clink.memory.ddr.ar.bits) + clink.memory.ddr.b.ready := false.B + clink.memory.ddr.r.ready := false.B + controller.io.move.command.ready := false.B + controller.io.move.completion.valid := false.B + controller.io.move.completion.bits := false.B + } +} + +object Poly extends CoreFactory { + val component = "poly" + def parameters(design: Design, source: CorePlacement): CoreParam = + CoreParam.fromProto(design, source, externalHome = true) + + def instantiate( + source: CorePlacement, + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int] + ): CLinkIO = + Instantiate(new PolyCore(corep, simp, traceIds)).clink + +} diff --git a/examples/designs/poly/configs/design.toml b/examples/designs/poly/configs/design.toml new file mode 100644 index 000000000..1d45ec7e4 --- /dev/null +++ b/examples/designs/poly/configs/design.toml @@ -0,0 +1,5 @@ +[device] +include = "device/default.toml" + +[sims] +include = "sims/default.toml" diff --git a/examples/designs/poly/configs/device/clint.toml b/examples/designs/poly/configs/device/clint.toml new file mode 100644 index 000000000..c94fc18d8 --- /dev/null +++ b/examples/designs/poly/configs/device/clint.toml @@ -0,0 +1,4 @@ +base = 0x2000000 +bytes = 0x10000 +tickCycles = 1000 +hartIds = [0, 1, 2, 3, 4, 5, 6, 7, 8] diff --git a/examples/designs/poly/configs/device/cores/core0/acc/balldomain/default.toml b/examples/designs/poly/configs/device/cores/core0/acc/balldomain/default.toml new file mode 100644 index 000000000..ab2226811 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/acc/balldomain/default.toml @@ -0,0 +1,9 @@ +[[balls]] +id = 0 +name = "F32AddBall" +component = "f32add" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "F32ADD", funct7 = 76 }, +] diff --git a/examples/designs/poly/configs/device/cores/core0/acc/default.toml b/examples/designs/poly/configs/device/cores/core0/acc/default.toml new file mode 100644 index 000000000..0099d6438 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/acc/default.toml @@ -0,0 +1,6 @@ +source = "../../../../.." +balldomain = "balldomain/default.toml" +memdomain = "memdomain/poly.toml" +frontend = "frontend/poly.toml" +gpdomain = "default" +transport = "../../core1/transport.toml" diff --git a/examples/designs/poly/configs/device/cores/core0/acc/frontend/poly.toml b/examples/designs/poly/configs/device/cores/core0/acc/frontend/poly.toml new file mode 100644 index 000000000..c3e3d5e71 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/acc/frontend/poly.toml @@ -0,0 +1,9 @@ +bankAliasEnable = true +robEntries = 16 +rsOutOfOrderResponse = true +bankIdLen = 10 +vbankIdUpperBound = 15 +sharedBankIdBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/examples/designs/poly/configs/device/cores/core0/acc/memdomain/poly.toml b/examples/designs/poly/configs/device/cores/core0/acc/memdomain/poly.toml new file mode 100644 index 000000000..b502d7a33 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/acc/memdomain/poly.toml @@ -0,0 +1,20 @@ +[bank] +layoutEnable = true +num = 4 +width = 128 +entries = 1024 +maskLen = 16 +requestPorts = 3 + +[dma] +busWidth = 128 + +[mmio] +enable = false +bankNum = 1 +bankEntries = 1 +bankWidth = 128 +readWidth = 8 + +[mem] +addrLen = 39 diff --git a/examples/designs/poly/configs/device/cores/core0/c2c/core.toml b/examples/designs/poly/configs/device/cores/core0/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core0/controller/core.toml b/examples/designs/poly/configs/device/cores/core0/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core0/default.toml b/examples/designs/poly/configs/device/cores/core0/default.toml new file mode 100644 index 000000000..d1fb23449 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/default.toml @@ -0,0 +1,37 @@ +component = "reduce" +accelerator = "acc/default.toml" +threads = [] +hartId = 0 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 5 +defaultGroupCount = 1 +memCycles = 10 + +[compiler] +name = "main" +component = "reduce" diff --git a/examples/designs/poly/configs/device/cores/core0/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core0/dmaCache/core.toml new file mode 100644 index 000000000..309ac5a3d --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 3 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core0/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core0/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core0/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core0/memory/access/core.toml new file mode 100644 index 000000000..d961d54fc --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +accessTagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core0/memory/core.toml b/examples/designs/poly/configs/device/cores/core0/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core0/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core0/memory/dcache/core.toml new file mode 100644 index 000000000..97b8e2bba --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 2 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core0/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core0/memory/home/core.toml new file mode 100644 index 000000000..f20d607fe --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +homeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core0/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core0/memory/icache/core.toml new file mode 100644 index 000000000..75e065643 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 1 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core0/tbus/core.toml b/examples/designs/poly/configs/device/cores/core0/tbus/core.toml new file mode 100644 index 000000000..36bdb8cfc --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core0/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0] diff --git a/examples/designs/poly/configs/device/cores/core1/c2c/core.toml b/examples/designs/poly/configs/device/cores/core1/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core1/controller/core.toml b/examples/designs/poly/configs/device/cores/core1/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core1/default.toml b/examples/designs/poly/configs/device/cores/core1/default.toml new file mode 100644 index 000000000..fd8332ae3 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 1 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +memCycles = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core1/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core1/dmaCache/core.toml new file mode 100644 index 000000000..b23c6c6cd --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 6 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core1/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core1/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core1/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core1/memory/access/core.toml new file mode 100644 index 000000000..d961d54fc --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +accessTagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core1/memory/core.toml b/examples/designs/poly/configs/device/cores/core1/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core1/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core1/memory/dcache/core.toml new file mode 100644 index 000000000..1fa9b4c07 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 5 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core1/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core1/memory/home/core.toml new file mode 100644 index 000000000..f20d607fe --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +homeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core1/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core1/memory/icache/core.toml new file mode 100644 index 000000000..5a95e88f0 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 4 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core1/tbus/core.toml b/examples/designs/poly/configs/device/cores/core1/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/cores/core1/transport.toml b/examples/designs/poly/configs/device/cores/core1/transport.toml new file mode 100644 index 000000000..cb0b0cd17 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core1/transport.toml @@ -0,0 +1,82 @@ +# Per-thread DMA transport values follow the existing Pebble configuration. + +[tracking] +entries = 24 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + + +[prepared] +contexts = 24 +maxRanges = 8 +idBits = 8 +beatBytes = 16 + +[prepared.bus] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 + + + +[axi] +addressBits = 44 +dataBits = 128 +idBits = 4 + + +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core2/c2c/core.toml b/examples/designs/poly/configs/device/cores/core2/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core2/controller/core.toml b/examples/designs/poly/configs/device/cores/core2/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core2/default.toml b/examples/designs/poly/configs/device/cores/core2/default.toml new file mode 100644 index 000000000..f2caf5ff0 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 2 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +memCycles = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core2/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core2/dmaCache/core.toml new file mode 100644 index 000000000..714218f58 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 9 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core2/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core2/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core2/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core2/memory/access/core.toml new file mode 100644 index 000000000..d961d54fc --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +accessTagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core2/memory/core.toml b/examples/designs/poly/configs/device/cores/core2/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core2/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core2/memory/dcache/core.toml new file mode 100644 index 000000000..acc89ffe1 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 8 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core2/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core2/memory/home/core.toml new file mode 100644 index 000000000..f20d607fe --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +homeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core2/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core2/memory/icache/core.toml new file mode 100644 index 000000000..82e254beb --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 7 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core2/tbus/core.toml b/examples/designs/poly/configs/device/cores/core2/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core2/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/cores/core3/c2c/core.toml b/examples/designs/poly/configs/device/cores/core3/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core3/controller/core.toml b/examples/designs/poly/configs/device/cores/core3/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core3/default.toml b/examples/designs/poly/configs/device/cores/core3/default.toml new file mode 100644 index 000000000..8eb441d2d --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 3 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +memCycles = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core3/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core3/dmaCache/core.toml new file mode 100644 index 000000000..e37f02576 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 12 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core3/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core3/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core3/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core3/memory/access/core.toml new file mode 100644 index 000000000..d961d54fc --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +accessTagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core3/memory/core.toml b/examples/designs/poly/configs/device/cores/core3/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core3/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core3/memory/dcache/core.toml new file mode 100644 index 000000000..f71df002a --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 11 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core3/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core3/memory/home/core.toml new file mode 100644 index 000000000..f20d607fe --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +homeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core3/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core3/memory/icache/core.toml new file mode 100644 index 000000000..9b09d3745 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 10 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core3/tbus/core.toml b/examples/designs/poly/configs/device/cores/core3/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core3/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/cores/core4/c2c/core.toml b/examples/designs/poly/configs/device/cores/core4/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core4/controller/core.toml b/examples/designs/poly/configs/device/cores/core4/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core4/default.toml b/examples/designs/poly/configs/device/cores/core4/default.toml new file mode 100644 index 000000000..0da5398b1 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 4 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +memCycles = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core4/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core4/dmaCache/core.toml new file mode 100644 index 000000000..8c645ccb0 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 15 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core4/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core4/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core4/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core4/memory/access/core.toml new file mode 100644 index 000000000..d961d54fc --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +accessTagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core4/memory/core.toml b/examples/designs/poly/configs/device/cores/core4/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core4/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core4/memory/dcache/core.toml new file mode 100644 index 000000000..83545e2a1 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 14 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core4/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core4/memory/home/core.toml new file mode 100644 index 000000000..f20d607fe --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +homeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core4/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core4/memory/icache/core.toml new file mode 100644 index 000000000..382168405 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 13 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core4/tbus/core.toml b/examples/designs/poly/configs/device/cores/core4/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core4/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/cores/core5/c2c/core.toml b/examples/designs/poly/configs/device/cores/core5/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core5/controller/core.toml b/examples/designs/poly/configs/device/cores/core5/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core5/default.toml b/examples/designs/poly/configs/device/cores/core5/default.toml new file mode 100644 index 000000000..87531a19d --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 5 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +memCycles = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core5/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core5/dmaCache/core.toml new file mode 100644 index 000000000..755ae918e --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 18 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core5/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core5/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core5/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core5/memory/access/core.toml new file mode 100644 index 000000000..d961d54fc --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +accessTagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core5/memory/core.toml b/examples/designs/poly/configs/device/cores/core5/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core5/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core5/memory/dcache/core.toml new file mode 100644 index 000000000..cdc0c63e3 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 17 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core5/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core5/memory/home/core.toml new file mode 100644 index 000000000..f20d607fe --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +homeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core5/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core5/memory/icache/core.toml new file mode 100644 index 000000000..81af78919 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 16 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core5/tbus/core.toml b/examples/designs/poly/configs/device/cores/core5/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core5/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/cores/core6/c2c/core.toml b/examples/designs/poly/configs/device/cores/core6/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core6/controller/core.toml b/examples/designs/poly/configs/device/cores/core6/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core6/default.toml b/examples/designs/poly/configs/device/cores/core6/default.toml new file mode 100644 index 000000000..4fa2fad54 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 6 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +memCycles = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core6/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core6/dmaCache/core.toml new file mode 100644 index 000000000..413178519 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 21 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core6/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core6/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core6/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core6/memory/access/core.toml new file mode 100644 index 000000000..d961d54fc --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +accessTagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core6/memory/core.toml b/examples/designs/poly/configs/device/cores/core6/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core6/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core6/memory/dcache/core.toml new file mode 100644 index 000000000..92e94a216 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 20 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core6/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core6/memory/home/core.toml new file mode 100644 index 000000000..f20d607fe --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +homeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core6/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core6/memory/icache/core.toml new file mode 100644 index 000000000..9c2d9947f --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 19 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core6/tbus/core.toml b/examples/designs/poly/configs/device/cores/core6/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core6/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/cores/core7/c2c/core.toml b/examples/designs/poly/configs/device/cores/core7/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core7/controller/core.toml b/examples/designs/poly/configs/device/cores/core7/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core7/default.toml b/examples/designs/poly/configs/device/cores/core7/default.toml new file mode 100644 index 000000000..e53fe7834 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 7 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +memCycles = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core7/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core7/dmaCache/core.toml new file mode 100644 index 000000000..d315ca578 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 24 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core7/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core7/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core7/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core7/memory/access/core.toml new file mode 100644 index 000000000..d961d54fc --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +accessTagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core7/memory/core.toml b/examples/designs/poly/configs/device/cores/core7/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core7/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core7/memory/dcache/core.toml new file mode 100644 index 000000000..9b77b7ae8 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 23 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core7/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core7/memory/home/core.toml new file mode 100644 index 000000000..f20d607fe --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +homeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core7/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core7/memory/icache/core.toml new file mode 100644 index 000000000..b7600e54a --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 22 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core7/tbus/core.toml b/examples/designs/poly/configs/device/cores/core7/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core7/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/cores/core8/c2c/core.toml b/examples/designs/poly/configs/device/cores/core8/c2c/core.toml new file mode 100644 index 000000000..d34ac9d9b --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = true +peerCores = 9 +axi = "default" diff --git a/examples/designs/poly/configs/device/cores/core8/controller/core.toml b/examples/designs/poly/configs/device/cores/core8/controller/core.toml new file mode 100644 index 000000000..87f38f33c --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/controller/core.toml @@ -0,0 +1 @@ +moves = true diff --git a/examples/designs/poly/configs/device/cores/core8/default.toml b/examples/designs/poly/configs/device/cores/core8/default.toml new file mode 100644 index 000000000..b687187a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/default.toml @@ -0,0 +1,71 @@ +component = "poly" +hartId = 8 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +dmaCache = "dmaCache/core.toml" +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +tbus = "tbus/core.toml" +controller = "controller/core.toml" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[tss] +base = 0x20000 +bytes = 0x4000 +dataBits = 128 + +[sharedMem] +bankWidth = 128 +bankEntries = 1024 +enable = true +virtualBankCount = 40 +entries = 16384 +inputChannels = 65 +defaultGroupCount = 1 +memCycles = 10 + +[[threads]] +id = 0 +name = "attention0" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 1 +name = "attention1" +component = "attention" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/attention/configs/default.toml" + +[[threads]] +id = 2 +name = "ffn0" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 3 +name = "ffn1" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" + +[[threads]] +id = 4 +name = "ffn2" +component = "ffn" +transport = "../core1/transport.toml" +include = "../../../../../../components/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core8/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core8/dmaCache/core.toml new file mode 100644 index 000000000..f9fcaf866 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/dmaCache/core.toml @@ -0,0 +1,7 @@ +nodeId = 27 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = true +probe = false diff --git a/examples/designs/poly/configs/device/cores/core8/interconnect/core.toml b/examples/designs/poly/configs/device/cores/core8/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/poly/configs/device/cores/core8/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core8/memory/access/core.toml new file mode 100644 index 000000000..d961d54fc --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +accessTagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x400000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/poly/configs/device/cores/core8/memory/core.toml b/examples/designs/poly/configs/device/cores/core8/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/poly/configs/device/cores/core8/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core8/memory/dcache/core.toml new file mode 100644 index 000000000..7feda67d6 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 26 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core8/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core8/memory/home/core.toml new file mode 100644 index 000000000..f20d607fe --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 27 +mshrEntries = 8 +homeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/poly/configs/device/cores/core8/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core8/memory/icache/core.toml new file mode 100644 index 000000000..7bee95556 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 25 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/poly/configs/device/cores/core8/tbus/core.toml b/examples/designs/poly/configs/device/cores/core8/tbus/core.toml new file mode 100644 index 000000000..81037e7d9 --- /dev/null +++ b/examples/designs/poly/configs/device/cores/core8/tbus/core.toml @@ -0,0 +1 @@ +bankNodes = [0, 1, 2, 3, 4] diff --git a/examples/designs/poly/configs/device/ddr.toml b/examples/designs/poly/configs/device/ddr.toml new file mode 100644 index 000000000..ac8cb4ff3 --- /dev/null +++ b/examples/designs/poly/configs/device/ddr.toml @@ -0,0 +1,13 @@ +clients = 1 +slotsPerClient = 16 +dataBits = 128 +idBits = 4 +idBase = 0 +base = 0x80000000 +bytes = 0x400000000 +model = 0x300000000 + +[line] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 diff --git a/examples/designs/poly/configs/device/default.toml b/examples/designs/poly/configs/device/default.toml new file mode 100644 index 000000000..33cce1c5d --- /dev/null +++ b/examples/designs/poly/configs/device/default.toml @@ -0,0 +1,32 @@ +ddr = "ddr.toml" +bootrom = "default" +clint = "clint.toml" +plic = "plic.toml" +scu = "default" + +[[cores]] +include = "cores/core0/default.toml" + +[[cores]] +include = "cores/core1/default.toml" + +[[cores]] +include = "cores/core2/default.toml" + +[[cores]] +include = "cores/core3/default.toml" + +[[cores]] +include = "cores/core4/default.toml" + +[[cores]] +include = "cores/core5/default.toml" + +[[cores]] +include = "cores/core6/default.toml" + +[[cores]] +include = "cores/core7/default.toml" + +[[cores]] +include = "cores/core8/default.toml" diff --git a/examples/designs/poly/configs/device/plic.toml b/examples/designs/poly/configs/device/plic.toml new file mode 100644 index 000000000..0beb0dba9 --- /dev/null +++ b/examples/designs/poly/configs/device/plic.toml @@ -0,0 +1,5 @@ +base = 0xc000000 +bytes = 0x4000000 +sources = 1 +priorityBits = 3 +hartIds = [0, 1, 2, 3, 4, 5, 6, 7, 8] diff --git a/examples/designs/poly/configs/sims/default.toml b/examples/designs/poly/configs/sims/default.toml new file mode 100644 index 000000000..39c913e06 --- /dev/null +++ b/examples/designs/poly/configs/sims/default.toml @@ -0,0 +1,14 @@ +diffTest = false + +[memory] +lineBytes = 64 +clockHz = 1000000000 + +[targets.device] +config = "examples.designs.poly.sims.device.DeviceConfig" +stack_name = "poly" +regression = "sims/device/regression" +kernel = "sims/device/workloads/kernel" + +[verification] +registry = "../../sims/ip/uvm/registry.toml" diff --git a/examples/designs/poly/emu/Cargo.lock b/examples/designs/poly/emu/Cargo.lock new file mode 100644 index 000000000..545b9a687 --- /dev/null +++ b/examples/designs/poly/emu/Cargo.lock @@ -0,0 +1,821 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "bebop-vm", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "rvsim", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-poly" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rvsim" +version = "0.1.0" +dependencies = [ + "rustc_apfloat", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/poly/emu/Cargo.toml b/examples/designs/poly/emu/Cargo.toml new file mode 100644 index 000000000..b74706f10 --- /dev/null +++ b/examples/designs/poly/emu/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "bebop-design-poly" +version = "0.1.0" +edition = "2021" + +[dependencies] +bebop-bemu = { path = "../../../../bebop/src/nodes/bemu" } diff --git a/examples/designs/poly/emu/src/device/core/mod.rs b/examples/designs/poly/emu/src/device/core/mod.rs new file mode 100644 index 000000000..f4935f943 --- /dev/null +++ b/examples/designs/poly/emu/src/device/core/mod.rs @@ -0,0 +1,37 @@ +use bebop_bemu::{ + ball::linkme, CoreFactory, CoreModel, CoreParams, Cpu, DeviceModel, FromString, ModelOptions, NoRocc, ThreadGroup, + Whatever, CORES, +}; + +/// One RV64GC hart driving its attention and ffn hardware threads (arch: PolyCore, behind TBus). +pub struct PolyCore { + cpu: Cpu, + threads: ThreadGroup, +} + +impl CoreModel for PolyCore { + fn cpu(&self) -> &Cpu { + &self.cpu + } + fn cpu_mut(&mut self) -> &mut Cpu { + &mut self.cpu + } + fn step(&mut self, count: u64) -> Result<(), Whatever> { + self.threads.check().map_err(Whatever::without_source)?; + self.cpu.step(count, &mut NoRocc)?; + if self.cpu.finished() { + self.threads.shutdown().map_err(Whatever::without_source)?; + } + Ok(()) + } +} + +fn instantiate(device: &DeviceModel, params: &CoreParams, options: &ModelOptions) -> Result, String> { + let cpu = Cpu::new(device, params, options).map_err(|error| error.to_string())?; + let threads = ThreadGroup::new(cpu.state(), options)?; + Ok(Box::new(PolyCore { cpu, threads })) +} + +#[linkme::distributed_slice(CORES)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static POLY: CoreFactory = CoreFactory { component: "poly", instantiate }; diff --git a/examples/designs/poly/emu/src/device/mod.rs b/examples/designs/poly/emu/src/device/mod.rs new file mode 100644 index 000000000..c0ba35020 --- /dev/null +++ b/examples/designs/poly/emu/src/device/mod.rs @@ -0,0 +1,2 @@ +//! The poly device: its own Core model; the cores are instantiated by component on the CBus. +pub mod core; diff --git a/examples/designs/poly/emu/src/lib.rs b/examples/designs/poly/emu/src/lib.rs new file mode 100644 index 000000000..545892414 --- /dev/null +++ b/examples/designs/poly/emu/src/lib.rs @@ -0,0 +1 @@ +pub mod device; diff --git a/examples/designs/poly/sims/device/arch/DeviceConfig.scala b/examples/designs/poly/sims/device/arch/DeviceConfig.scala new file mode 100644 index 000000000..c8dc74f8e --- /dev/null +++ b/examples/designs/poly/sims/device/arch/DeviceConfig.scala @@ -0,0 +1,18 @@ +package examples.designs.poly.sims.device + +import buckyball.config.Design +import chisel3.experimental.hierarchy.{Instance, Instantiate} +import examples.designs.poly.DeviceTop +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import framework.device.dlink.BaseDevice +import _root_.configs.GlobalConfig +import _root_.sims.soc.SystemTarget +import java.nio.file.{Files, Paths} + +class DeviceConfig(design: Design) + extends SystemTarget( + GlobalConfig.fromProto(design, CBus.parameters(design, CoreComponents.factories)) + ) { + def instantiate(b: GlobalConfig): Instance[BaseDevice] = Instantiate(new DeviceTop(b, design)) +} diff --git a/examples/designs/poly/sims/device/arch/build.sc b/examples/designs/poly/sims/device/arch/build.sc new file mode 100644 index 000000000..5b94111b8 --- /dev/null +++ b/examples/designs/poly/sims/device/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.arch.{build => design_build} +import $file.^.^.^.^.^.^.arch.src.main.scala.sims.{build => sims_build} + +val entryRoot: os.Path = millSourcePath + +object simulation extends SbtModule { + override def millSourcePath = entryRoot + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(design_build.design, sims_build.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(entryRoot / "DeviceConfig.scala") + override def mainClass = Some("sims.verilator.Elaborate") +} diff --git a/examples/designs/poly/sims/device/emu/Cargo.lock b/examples/designs/poly/sims/device/emu/Cargo.lock new file mode 100644 index 000000000..686a331aa --- /dev/null +++ b/examples/designs/poly/sims/device/emu/Cargo.lock @@ -0,0 +1,1979 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "assert_cmd" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aa3a22042e45de04255c7bf3626e239f450200fd0493c1e382263544b20aea6" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bebop" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-fd-redirect", + "bebop-p2e", + "bebop-report", + "bebop-rtl-trace", + "bebop-terminal", + "bebop-trace-perfetto", + "bebop-uart", + "bebop-vcs", + "bebop-verilator", + "bebop-vm", + "camino", + "clap", + "duct", + "env_logger", + "libc", + "log", + "nix", + "serde", + "serde_json", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-core-reduce" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-poly" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-fd-redirect" +version = "0.1.0" +dependencies = [ + "libc", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-p2e" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "bebop-rtl-trace", + "bebop-uart", + "duct", + "libc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bebop-report" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-rtl-trace" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", +] + +[[package]] +name = "bebop-sim-build" +version = "0.1.0" +dependencies = [ + "prost", + "prost-build", + "prost-types", +] + +[[package]] +name = "bebop-sim-poly-device" +version = "0.1.0" +dependencies = [ + "assert_cmd", + "bebop", + "bebop-bemu", + "bebop-core-reduce", + "bebop-design-poly", + "bebop-report", + "bebop-sim-build", + "bebop-verilator", + "chrono", + "clap", + "libtest-mimic", + "predicates", + "serde", + "toml", + "walkdir", +] + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-terminal" +version = "0.1.0" +dependencies = [ + "crossterm", + "ratatui", +] + +[[package]] +name = "bebop-trace-perfetto" +version = "0.1.0" +dependencies = [ + "clap", + "serde_json", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vcs" +version = "0.1.0" +dependencies = [ + "snafu", +] + +[[package]] +name = "bebop-verilator" +version = "0.1.0" +dependencies = [ + "bebop-dasm", + "bebop-fd-redirect", + "bebop-rtl-trace", + "bebop-uart", + "libc", + "snafu", +] + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" +dependencies = [ + "clap", + "font8x8", + "sdl2", + "vt100", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "camino" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbad30e4b4c14a39e3cc8aed085a12a327257c316619c93581e017bc52be591" + +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compact_str" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.13.2", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix 0.38.44", + "signal-hook 0.3.18", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "duct" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "font8x8" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875488b8711a968268c7cf5d139578713097ca4635a76044e8fe8eedf831d07e" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "indoc" +version = "2.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1" +dependencies = [ + "rustversion", +] + +[[package]] +name = "instability" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3b5acc1e2fd9375041a388da33d1eb8aed5f7a8c0dd3543e3ea2805adfbe20" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "os_pipe" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags 2.13.2", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags 2.13.2", + "smallvec", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdl2" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d42407afc6a8ab67e36f92e80b8ba34cbdc55aaeed05249efe9a2e8d0e9feef" +dependencies = [ + "bitflags 1.3.2", + "lazy_static", + "libc", + "sdl2-sys", +] + +[[package]] +name = "sdl2-sys" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff61407fc75d4b0bbc93dc7e4d6c196439965fbef8e4a4f003a36095823eac0" +dependencies = [ + "cfg-if", + "libc", + "pkg-config", + "version-compare", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "shared_child" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "607549934f6cc26b89cfecfdc46fa90f1e5d1536a68349b0c3a4f9d1c0d37959" +dependencies = [ + "libc", + "sigchld", + "windows-sys 0.61.2", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "sigchld" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f2b37f04360cd465089b87a9c3869c08220a2f3458463f0adf8badf5e77f2c" +dependencies = [ + "libc", + "os_pipe", + "signal-hook 0.4.5", +] + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b29ffa27d3d59a049bb328258de100f8012848d1b08d444e08bbc02ac468435e" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook 0.3.18", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix 1.1.5", + "windows-sys 0.61.2", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version-compare" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" + +[[package]] +name = "vt100" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de" +dependencies = [ + "itoa", + "log", + "unicode-width 0.1.14", + "vte", +] + +[[package]] +name = "vte" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/poly/sims/device/emu/Cargo.toml b/examples/designs/poly/sims/device/emu/Cargo.toml new file mode 100644 index 000000000..d7ab45df4 --- /dev/null +++ b/examples/designs/poly/sims/device/emu/Cargo.toml @@ -0,0 +1,84 @@ +[workspace] + +[package] +name = "bebop-sim-poly-device" +version = "0.1.0" +edition = "2021" +autobins = false +autotests = false + +# The model of this simulation entry; the runners below are built from it. +[lib] +name = "simulation" + +[[bin]] +name = "bebop" +path = "../../../../../../bebop/src/main.rs" + +[[bin]] +name = "bebop-bemu" +path = "../../../../../../bebop/src/bin/bebop-bemu.rs" +required-features = ["bemu"] + +[[bin]] +name = "bebop-verilator" +path = "../../../../../../bebop/src/bin/bebop-verilator.rs" +required-features = ["verilator"] + +[[bin]] +name = "bebop-p2e" +path = "../../../../../../bebop/src/bin/bebop-p2e.rs" +required-features = ["p2e"] + +[[test]] +name = "test_bemu" +path = "../../../../../../bebop/tests/test_bemu.rs" +harness = false +required-features = ["bemu"] + +[[test]] +name = "test_verilator" +path = "../../../../../../bebop/tests/test_verilator.rs" +harness = false +required-features = ["verilator"] + +[[test]] +name = "test_p2e" +path = "../../../../../../bebop/tests/test_p2e.rs" +harness = false +required-features = ["p2e"] + +[[bin]] +name = "bebop-vcs" +path = "../../../../../../bebop/src/bin/bebop-vcs.rs" + +[[test]] +name = "test_vcs" +path = "../../../../../../bebop/tests/test_vcs.rs" +harness = false + +[features] +bemu = ["bebop/bemu"] +verilator = ["bebop/verilator", "dep:bebop-verilator"] +p2e = ["bebop/p2e"] + +[dependencies] +bebop = { path = "../../../../../../bebop" } +bebop-verilator = { path = "../../../../../../bebop/src/nodes/verilator", optional = true } +bebop-bemu = { path = "../../../../../../bebop/src/nodes/bemu" } +bebop-design-poly = { path = "../../../emu" } +bebop-core-reduce = { path = "../../../../../components/cores/reduce/emu" } +clap = { version = "4", features = ["derive"] } + +[build-dependencies] +bebop-sim-build = { path = "../../../../../../bebop/src/nodes/lib/sim-build" } + +[dev-dependencies] +bebop-report = { path = "../../../../../../bebop/src/nodes/lib/report" } +libtest-mimic = "0.8" +assert_cmd = "2" +walkdir = "2" +predicates = "3" +serde = { version = "1", features = ["derive"] } +toml = "0.8" +chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/examples/designs/poly/sims/device/emu/build.rs b/examples/designs/poly/sims/device/emu/build.rs new file mode 100644 index 000000000..d634d5dba --- /dev/null +++ b/examples/designs/poly/sims/device/emu/build.rs @@ -0,0 +1,7 @@ +use std::{env, path::PathBuf}; + +fn main() { + let entry = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + bebop_sim_build::balls(&entry.join("../../../output/generated/design.pb")); + bebop_sim_build::runner_links(); +} diff --git a/examples/designs/poly/sims/device/emu/src/lib.rs b/examples/designs/poly/sims/device/emu/src/lib.rs new file mode 100644 index 000000000..dc8107ed5 --- /dev/null +++ b/examples/designs/poly/sims/device/emu/src/lib.rs @@ -0,0 +1,12 @@ +//! The poly device simulation. +//! Its Core models register in `bebop_bemu::CORES`; the Balls its design.pb +//! names are compiled here and register in `bebop_bemu::BALLS`. +extern crate bebop_design_poly; +extern crate bebop_core_reduce; + +include!(concat!(env!("OUT_DIR"), "/balls.rs")); + +pub const SIMULATION: bebop_bemu::Simulation = bebop_bemu::Simulation { + design: include_bytes!("../../../../output/generated/design.pb"), + instantiate: bebop_bemu::device::cbus::instantiate, +}; diff --git a/examples/designs/poly/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/poly/sims/device/regression/bemu/workloads-bare.toml new file mode 100644 index 000000000..f4abb54a7 --- /dev/null +++ b/examples/designs/poly/sims/device/regression/bemu/workloads-bare.toml @@ -0,0 +1,19 @@ +[workloads] +tests = [ + "poly-main-ctest-design-ant_mesh_move-multicore-baremetal", + "poly-main-ctest-design-ant_mesh_private-multicore-baremetal", + "poly-main-ctest-design-ant_mesh_shared-multicore-baremetal", + "poly-main-ctest-f32add_groups-baremetal", + "poly-main-ctest-corelink_lease-baremetal", + "poly-main-ctest-design-controllers-multicore-baremetal", + "poly-main-ctest-design-ant_mxmm-multicore-baremetal", + "poly-main-ctest-design-ant_shared-multicore-baremetal", + "poly-main-ctest-design-ant_bank-multicore-baremetal", + "poly-main-ctest-design-ant_bank_rows-multicore-baremetal", + "poly-main-ctest-design-ant_parallel-multicore-baremetal", + "poly-main-ctest-design-ant_highpa-multicore-baremetal", + "poly-main-ctest-design-ant_corelink-multicore-baremetal", + "poly-main-ctest-design-ant_corelink_bank-multicore-baremetal", + "poly-main-ctest-design-ant_corelink_reduce-multicore-baremetal", + "poly-main-ctest-design-ant_coherence-multicore-baremetal", +] diff --git a/examples/chips/poly/regression/batch/bemu/workloads-linux.toml b/examples/designs/poly/sims/device/regression/bemu/workloads-linux.toml similarity index 100% rename from examples/chips/poly/regression/batch/bemu/workloads-linux.toml rename to examples/designs/poly/sims/device/regression/bemu/workloads-linux.toml diff --git a/examples/chips/poly/regression/batch/p2e/workloads-bare.toml b/examples/designs/poly/sims/device/regression/p2e/workloads-bare.toml similarity index 100% rename from examples/chips/poly/regression/batch/p2e/workloads-bare.toml rename to examples/designs/poly/sims/device/regression/p2e/workloads-bare.toml diff --git a/examples/chips/poly/regression/batch/p2e/workloads-linux.toml b/examples/designs/poly/sims/device/regression/p2e/workloads-linux.toml similarity index 100% rename from examples/chips/poly/regression/batch/p2e/workloads-linux.toml rename to examples/designs/poly/sims/device/regression/p2e/workloads-linux.toml diff --git a/examples/designs/poly/sims/device/regression/verilator/workloads-bare.toml b/examples/designs/poly/sims/device/regression/verilator/workloads-bare.toml new file mode 100644 index 000000000..7bc345182 --- /dev/null +++ b/examples/designs/poly/sims/device/regression/verilator/workloads-bare.toml @@ -0,0 +1,19 @@ +[workloads] +tests = [ + "poly-main-ctest-design-ant_mesh_move-multicore-baremetal", + "poly-main-ctest-design-ant_mesh_private-multicore-baremetal", + "poly-main-ctest-design-ant_mesh_shared-multicore-baremetal", + "poly-main-ctest-f32add_groups-baremetal", + "poly-main-ctest-corelink_lease-baremetal", + "poly-main-ctest-design-controllers-multicore-baremetal", + "poly-main-ctest-design-ant_mxmm-multicore-baremetal", + "poly-main-ctest-design-ant_shared-multicore-baremetal", + "poly-main-ctest-design-ant_bank_rows-multicore-baremetal", + "poly-main-ctest-design-ant_parallel-multicore-baremetal", + "poly-main-ctest-design-ant_bank-multicore-baremetal", + "poly-main-ctest-design-ant_highpa-multicore-baremetal", + "poly-main-ctest-design-ant_corelink-multicore-baremetal", + "poly-main-ctest-design-ant_corelink_bank-multicore-baremetal", + "poly-main-ctest-design-ant_corelink_reduce-multicore-baremetal", + "poly-main-ctest-design-ant_coherence-multicore-baremetal", +] diff --git a/examples/designs/poly/sims/device/stack/CMakeLists.txt b/examples/designs/poly/sims/device/stack/CMakeLists.txt new file mode 100644 index 000000000..383594aa1 --- /dev/null +++ b/examples/designs/poly/sims/device/stack/CMakeLists.txt @@ -0,0 +1,67 @@ +# ===----------------------------------------------------------------------=== +# The software stack of the poly device simulation: every level the device +# needs, built bottom up. +# +# bbdev stack --build "--design poly" +# +# configures this with DESIGN_CMAKE, generated from design.pb. Each level is a +# CMake build of its own: cores and threads build buddy-opt plugins for the +# host, the device builds RISC-V libraries. +# thread/attention, thread/ffn lib-compiler.so, program.bin, loader.o, +# libkernels*.a +# core/reduce, core/poly execute.o and their compiler plugins +# device runtime, runtime-baremetal, runtime-linux, engine; device.cmake +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(poly-device-stack NONE) +include(ExternalProject) +include("${DESIGN_CMAKE}") + +# The attention hardware thread. +ExternalProject_Add(thread-attention + SOURCE_DIR "${DESIGN_THREAD_STACK_attention}" + BINARY_DIR "${DESIGN_THREAD_BUILD_attention}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DTHREAD=attention + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# The ffn hardware thread. +ExternalProject_Add(thread-ffn + SOURCE_DIR "${DESIGN_THREAD_STACK_ffn}" + BINARY_DIR "${DESIGN_THREAD_BUILD_ffn}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DTHREAD=ffn + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# The reduce core. +ExternalProject_Add(core-reduce + SOURCE_DIR "${DESIGN_CORE_STACK_reduce}" + BINARY_DIR "${DESIGN_CORE_BUILD_reduce}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DCORE=reduce + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# Poly's CPU core, which runs the attention and ffn threads. +ExternalProject_Add(core-poly + SOURCE_DIR "${DESIGN_CORE_STACK_poly}" + BINARY_DIR "${DESIGN_CORE_BUILD_poly}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DCORE=poly + DEPENDS thread-attention thread-ffn + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# The device: the runtime that schedules jobs onto the cores. +ExternalProject_Add(device + SOURCE_DIR "${DESIGN_DEVICE_STACK}" + BINARY_DIR "${DESIGN_DEVICE_BUILD}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" + -DCMAKE_C_COMPILER=riscv64-unknown-linux-gnu-gcc + -DCMAKE_CXX_COMPILER=riscv64-unknown-linux-gnu-g++ + DEPENDS thread-attention thread-ffn core-reduce core-poly + BUILD_ALWAYS ON + INSTALL_COMMAND "") diff --git a/examples/designs/poly/sims/device/stack/core/poly/CMakeLists.txt b/examples/designs/poly/sims/device/stack/core/poly/CMakeLists.txt new file mode 100644 index 000000000..5898bd39d --- /dev/null +++ b/examples/designs/poly/sims/device/stack/core/poly/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.20) +project(poly-core-stack LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +add_subdirectory("${DESIGN_REPO}/examples/designs/poly/stack/engine" engine) +add_subdirectory("${DESIGN_REPO}/examples/designs/poly/stack/midend" midend) diff --git a/examples/designs/poly/sims/device/stack/runtime/CMakeLists.txt b/examples/designs/poly/sims/device/stack/runtime/CMakeLists.txt new file mode 100644 index 000000000..33cb18cf2 --- /dev/null +++ b/examples/designs/poly/sims/device/stack/runtime/CMakeLists.txt @@ -0,0 +1,38 @@ +# ===----------------------------------------------------------------------=== +# The poly device: its cores, their hardware threads and the runtime that +# schedules jobs onto the cores. +# +# bbdev stack --build "--design poly" +# +# builds every thread and core stack of the device first, then configures this +# with DESIGN_CMAKE, generated from design.pb, and the RISC-V Linux toolchain. +# Products, exported in device.cmake for the programs that run on the device: +# runtime, runtime-baremetal, runtime-linux job scheduling over the cores +# engine hardware threads and instruction streams +# bindings how each core runs jobs and each +# hardware thread is loaded +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(poly-device LANGUAGES C CXX) +include("${DESIGN_CMAKE}") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# How each core runs jobs, and how each hardware thread is loaded. Nothing +# references these bindings, so they are linked as objects rather than taken +# from an archive; as link options they come before the libraries they call. +add_library(bindings INTERFACE) +foreach(CORE IN LISTS DESIGN_CORES) + target_link_options(bindings INTERFACE "${DESIGN_CORE_BUILD_${CORE}}/execute.o") +endforeach() +foreach(THREAD IN LISTS DESIGN_THREADS) + target_link_options(bindings INTERFACE "${DESIGN_THREAD_BUILD_${THREAD}}/loader.o") +endforeach() +# DMA helpers are sized by the target with the fewest bank lines. +list(GET DESIGN_RUNTIME_TARGETS 0 PARAMS_TARGET) +set(RUNTIME_PARAMS "${DESIGN_ISA_DIR}/${PARAMS_TARGET}") + +add_subdirectory("${DESIGN_REPO}/stack/engine" engine) +add_subdirectory("${DESIGN_REPO}/stack/runtime" runtime) + +export(TARGETS bindings runtime runtime-baremetal runtime-linux engine + runtime-shared runtime-linux-shared engine-shared FILE device.cmake) diff --git a/examples/designs/poly/sims/device/workloads/CMakeLists.txt b/examples/designs/poly/sims/device/workloads/CMakeLists.txt new file mode 100644 index 000000000..096844c37 --- /dev/null +++ b/examples/designs/poly/sims/device/workloads/CMakeLists.txt @@ -0,0 +1,78 @@ +include("${DESIGN_DEVICE_BUILD}/device.cmake") + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/base" + "${CMAKE_BINARY_DIR}/src/CTest/base" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tlb" + "${CMAKE_BINARY_DIR}/src/CTest/tlb" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tutorial" + "${CMAKE_BINARY_DIR}/src/CTest/tutorial" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/embench" + "${CMAKE_BINARY_DIR}/src/CTest/embench" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/coremark" + "${CMAKE_BINARY_DIR}/src/CTest/coremark" +) +foreach(BUCKYBALL_COMPONENT_WORKLOAD_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(COMPONENT_WORKLOADS + "${DESIGN_TARGET_WORKLOADS_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" + ) + if( + COMPONENT_WORKLOADS + STREQUAL + "${BUCKYBALL_REPO_ROOT}/examples/designs/${BUCKYBALL_WORKLOAD_DESIGN}/workloads" + ) + continue() + endif() + set(BUCKYBALL_COMPONENT_WORKLOAD_PROFILE + "${DESIGN_TARGET_ID_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" + ) + set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}") + set(BUCKYBALL_CTEST_GROUP_TARGET component-ctests-build) + if(IS_DIRECTORY "${COMPONENT_WORKLOADS}/ctests") + add_subdirectory( + "${COMPONENT_WORKLOADS}/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/components/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" + ) + endif() +endforeach() +add_dependencies( + ctest-workloads-build + base-ctests-build + tlb-ctests-build + tutorial-ctests-build + component-ctests-build +) +add_dependencies(sync-native-bin embench-all coremark-all) + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/MLIRTest/base" + "${CMAKE_BINARY_DIR}/src/MLIRTest/mlir_tests/base" +) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_MLIR_ACTIVE_TARGET "${BUCKYBALL_TARGET}") + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_TARGET_BALLS_${BUCKYBALL_TARGET}) + set(BALL_MLIR_TESTS + "${BUCKYBALL_REPO_ROOT}/examples/components/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" + ) + if(IS_DIRECTORY "${BALL_MLIR_TESTS}") + add_subdirectory( + "${BALL_MLIR_TESTS}" + "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}" + ) + endif() + endforeach() +endforeach() +add_dependencies(MLIRTest-all base-mlirtest-build balls-mlir-tests-build) + +add_subdirectory( + "${BUCKYBALL_REPO_ROOT}/examples/designs/poly/workloads" + design +) diff --git a/examples/designs/poly/sims/device/workloads/kernel/CMakeLists.txt b/examples/designs/poly/sims/device/workloads/kernel/CMakeLists.txt new file mode 100644 index 000000000..2270721e8 --- /dev/null +++ b/examples/designs/poly/sims/device/workloads/kernel/CMakeLists.txt @@ -0,0 +1,10 @@ +# Linux programs installed into the poly rootfs, each with the hart it runs on. +set(BUCKYBALL_KERNEL_PROGRAMS + 0 poly-main-ctest-design-controllers-linux + 0 poly-main-ctest-f32add_groups-linux + 0 poly-main-ctest-design-ant_mxmm-linux + 0 poly-main-ctest-design-ant_shared-linux + 0 poly-main-ctest-design-ant_bank-linux + 0 poly-main-ctest-design-ant_parallel-linux + 0 poly-main-ctest-design-stream-linux + PARENT_SCOPE) diff --git a/examples/designs/poly/sims/ip/uvm/registry.toml b/examples/designs/poly/sims/ip/uvm/registry.toml new file mode 100644 index 000000000..8e8131449 --- /dev/null +++ b/examples/designs/poly/sims/ip/uvm/registry.toml @@ -0,0 +1 @@ +ips = [] diff --git a/examples/designs/poly/stack/engine/CMakeLists.txt b/examples/designs/poly/stack/engine/CMakeLists.txt new file mode 100644 index 000000000..2a03cfe3f --- /dev/null +++ b/examples/designs/poly/stack/engine/CMakeLists.txt @@ -0,0 +1,14 @@ +# Core execution binding; instruction streams use the framework engine. +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +set(ENGINE "${DESIGN_REPO}/stack/engine") +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/execute.o" + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" + "-I${ENGINE}/include" -c "${CMAKE_CURRENT_SOURCE_DIR}/execute.c" -o "${CMAKE_BINARY_DIR}/execute.o" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/execute.c" "${RUNTIME}/scheduler/runtime.h" + "${ENGINE}/include/engine.h" + VERBATIM) +add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") diff --git a/examples/designs/poly/stack/engine/execute.c b/examples/designs/poly/stack/engine/execute.c new file mode 100644 index 000000000..76b31878f --- /dev/null +++ b/examples/designs/poly/stack/engine/execute.c @@ -0,0 +1,23 @@ +#include "engine.h" +#include "runtime.h" + +/* Execution for a core with hardware threads: the job's entry receives the + * core's engine. */ +static int execute(void *context, int (*entry)(void *, void *), + void *argument) { + (void)context; + size_t count = thread_query(0, THREAD_FIELD_COUNT); + if (!count) + __builtin_trap(); + struct engine_slot slots[count]; + struct engine engine; + engine_init(&engine, slots, count); + int status = entry(&engine, argument); + if (engine.active) + __builtin_trap(); + return status; +} + +static const struct runtime_binding binding + __attribute__((section("runtime_bindings"), used)) = {CORE_SIGNATURE, + execute}; diff --git a/examples/designs/poly/stack/midend/CMakeLists.txt b/examples/designs/poly/stack/midend/CMakeLists.txt new file mode 100644 index 000000000..f8202e654 --- /dev/null +++ b/examples/designs/poly/stack/midend/CMakeLists.txt @@ -0,0 +1,8 @@ +# Host compiler plugin for this design's core. +set(ENGINE "${DESIGN_REPO}/stack/engine") +set(TARGET ${DESIGN_CORE_TARGET_${CORE}}) + +add_subdirectory("${ENGINE}/compiler" engine-compiler) +add_library(engine-compiler-plugin MODULE $) +set_target_properties(engine-compiler-plugin PROPERTIES OUTPUT_NAME engine-compiler + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") diff --git a/examples/designs/poly/workloads/CMakeLists.txt b/examples/designs/poly/workloads/CMakeLists.txt new file mode 100644 index 000000000..d1c990b8a --- /dev/null +++ b/examples/designs/poly/workloads/CMakeLists.txt @@ -0,0 +1,17 @@ +add_custom_target(poly-ctests-build) +set(BUCKYBALL_CTEST_GROUP_TARGET poly-ctests-build) +set(BUCKYBALL_CTEST_TARGET attention) +set(POLY_BOOT_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/device/common/controllers_start.S) +set(BUCKYBALL_DESIGN_COMMON_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/device/core/thread/ctests) +add_subdirectory(device) + + +add_custom_target(design-ctests-build DEPENDS poly-ctests-build) + +add_custom_target(design-workloads-build ALL + DEPENDS + design-ctests-build + mlirtest-workloads-build + soctests-build + COMMENT "Building poly workloads" + VERBATIM) diff --git a/examples/designs/poly/workloads/configs/tests.toml b/examples/designs/poly/workloads/configs/tests.toml new file mode 100644 index 000000000..c1c7ceb33 --- /dev/null +++ b/examples/designs/poly/workloads/configs/tests.toml @@ -0,0 +1,2 @@ +controlHart = 0 +reportHart = 0 diff --git a/examples/designs/poly/workloads/device/CMakeLists.txt b/examples/designs/poly/workloads/device/CMakeLists.txt new file mode 100644 index 000000000..19f4ef63a --- /dev/null +++ b/examples/designs/poly/workloads/device/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(core) +add_subdirectory(soc) +add_subdirectory(runtime) diff --git a/examples/designs/poly/workloads/device/common/controllers_start.S b/examples/designs/poly/workloads/device/common/controllers_start.S new file mode 100644 index 000000000..b7f41c631 --- /dev/null +++ b/examples/designs/poly/workloads/device/common/controllers_start.S @@ -0,0 +1 @@ +#include "start.S" diff --git a/examples/designs/poly/workloads/device/core/CMakeLists.txt b/examples/designs/poly/workloads/device/core/CMakeLists.txt new file mode 100644 index 000000000..d3db8c656 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/CMakeLists.txt @@ -0,0 +1,5 @@ +add_subdirectory(thread) +add_subdirectory(engine) +add_subdirectory(ball) +set(POLY_ANT_IMAGE_DIR "${POLY_ANT_IMAGE_DIR}" PARENT_SCOPE) +set(POLY_ANT_SOURCE_DIR "${POLY_ANT_SOURCE_DIR}" PARENT_SCOPE) diff --git a/examples/designs/poly/workloads/device/core/ball/CMakeLists.txt b/examples/designs/poly/workloads/device/core/ball/CMakeLists.txt new file mode 100644 index 000000000..0abc064dd --- /dev/null +++ b/examples/designs/poly/workloads/device/core/ball/CMakeLists.txt @@ -0,0 +1,5 @@ +set(BUCKYBALL_CTEST_TARGET main) +set(BUCKYBALL_CTEST_GROUP_TARGET poly-ctests-build) +add_subdirectory( + ${BUCKYBALL_REPO_ROOT}/examples/components/balls/f32add/workloads/ctests + f32add) diff --git a/examples/designs/poly/workloads/device/core/engine/CMakeLists.txt b/examples/designs/poly/workloads/device/core/engine/CMakeLists.txt new file mode 100644 index 000000000..6ae5e3818 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/CMakeLists.txt @@ -0,0 +1,122 @@ +foreach( + POLY_ANT_CASE + mxmm + shared + bank + bank_rows + parallel + highpa +) + set(POLY_ANT_THREAD attention) + set(POLY_ANT_HEADER ant_${POLY_ANT_CASE}.h) + set(POLY_ANT_HOST_DEFINITIONS) + if(POLY_ANT_CASE STREQUAL "bank" OR POLY_ANT_CASE STREQUAL "bank_rows") + set(POLY_ANT_THREAD ffn) + endif() + set(POLY_ANT_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${POLY_ANT_THREAD}-ctest-design-ant_${POLY_ANT_CASE}" + ) + set(POLY_ANT_IMAGE ${POLY_ANT_IMAGE_DIR}/${POLY_ANT_NAME}-ant) + set(POLY_ANT_CONTROL_SOURCES) + if( + POLY_ANT_CASE STREQUAL "bank" + OR POLY_ANT_CASE STREQUAL "bank_rows" + OR POLY_ANT_CASE STREQUAL "parallel" + OR POLY_ANT_CASE STREQUAL "shared" + ) + list( + APPEND POLY_ANT_CONTROL_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/ant_${POLY_ANT_CASE}_exercise.c + ) + endif() + foreach(SOURCE IN LISTS POLY_ANT_CONTROL_SOURCES) + buckyball_enforce_ctest_line_limit("${SOURCE}") + endforeach() + + set(POLY_ANT_EMBED ${CMAKE_CURRENT_BINARY_DIR}/ant_${POLY_ANT_CASE}_image.S) + enable_language(ASM) + file( + GENERATE OUTPUT ${POLY_ANT_EMBED} + CONTENT + ".section .rodata.ant,\"a\",@progbits\n.balign 8\n.global ant_image, ant_image_end\nant_image:\n.incbin \"${POLY_ANT_IMAGE}.bin\"\nant_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" + ) + set_source_files_properties( + ${POLY_ANT_EMBED} + PROPERTIES OBJECT_DEPENDS "${POLY_ANT_IMAGE}.bin" + ) + set(POLY_ANT_MAIN ant_${POLY_ANT_CASE}_main.c) + + if(POLY_ANT_CASE STREQUAL "mxmm") + set(POLY_ANT_MAIN ant_mxmm_main.cpp) + endif() + buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_MAIN}") + set(POLY_ANT_HOST_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-main-ctest-design-ant_${POLY_ANT_CASE}" + ) + set(BUCKYBALL_CTEST_TARGET ${POLY_ANT_THREAD}) + if(NOT POLY_ANT_CASE STREQUAL "highpa") + add_buckyball_linux_ctest(${POLY_ANT_HOST_NAME} ${CMAKE_CURRENT_SOURCE_DIR} ${POLY_ANT_MAIN}) + target_sources( + ${POLY_ANT_HOST_NAME}-linux + PRIVATE ${POLY_ANT_EMBED} ${POLY_ANT_CONTROL_SOURCES} + ) + target_include_directories( + ${POLY_ANT_HOST_NAME}-linux + PRIVATE ${BUCKYBALL_ISA_DIR}/design + ) + target_link_libraries( + ${POLY_ANT_HOST_NAME}-linux + PRIVATE engine runtime-linux pthread + ) + add_dependencies(${POLY_ANT_HOST_NAME}-linux ${POLY_ANT_NAME}-ant-build) + add_dependencies(poly-ctests-build ${POLY_ANT_HOST_NAME}-linux-build) + endif() + + set(POLY_ANT_HOST_THREAD ${POLY_ANT_THREAD}) + set(POLY_ANT_CONTROL_OBJECTS) + + set(POLY_ANT_HOST_ELF ${POLY_ANT_HOST_NAME}-multicore-baremetal) + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_HOST_ELF} + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} ${POLY_ANT_HOST_DEFINITIONS} + -I${POLY_ANT_SOURCE_DIR} + -I${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_THREAD} + -I${BUCKYBALL_ISA_DIR}/design -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/components/balls/f32add/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_MAIN} + ${POLY_ANT_CONTROL_SOURCES} ${POLY_ANT_CONTROL_OBJECTS} + ${POLY_ANT_EMBED} ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${POLY_ANT_HOST_ELF} + DEPENDS + ${POLY_ANT_MAIN} + ${POLY_ANT_SOURCE_DIR}/${POLY_ANT_HEADER} + ${POLY_BOOT_SOURCE} + ${POLY_ANT_EMBED} + ${BAREMETAL_DIR}/start.S + ${BBSIM_LD} + ${POLY_ANT_CONTROL_SOURCES} + ${POLY_ANT_CONTROL_OBJECTS} + ${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_THREAD}/params.h + ${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_THREAD}/ballISA.h + ${POLY_ANT_IMAGE}.bin + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.h + ${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD}/params.h + ${BUCKYBALL_ISA_DIR}/design/instances.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + add_custom_target( + ${POLY_ANT_HOST_NAME}-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_HOST_ELF} + ) + add_dependencies(poly-ctests-build ${POLY_ANT_HOST_NAME}-ctest-build) +endforeach() diff --git a/examples/designs/poly/workloads/device/core/engine/ant_bank_exercise.c b/examples/designs/poly/workloads/device/core/engine/ant_bank_exercise.c new file mode 100644 index 000000000..318efbdbf --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_bank_exercise.c @@ -0,0 +1,67 @@ +#include "ant_bank.h" +#include +#include +#include +extern const unsigned char ant_image[], ant_image_end[]; +enum { + WORDS = BANK_LINES * BANK_WIDTH / 64, + SHARED_WORDS = BB_SHARED_BANK_LINES * BANK_WIDTH / 64, + INPUT_WORDS = WORDS > SHARED_WORDS ? WORDS : SHARED_WORDS +}; +static uint64_t input[BB_THREAD_CORE_COUNT][INPUT_WORDS] + __attribute__((section(".data"), aligned(4096))) = {0}; +static uint64_t private_out[BB_THREAD_CORE_COUNT][WORDS] + __attribute__((section(".data"), aligned(4096))) = {0}; +static uint64_t shared_out[BB_THREAD_CORE_COUNT][SHARED_WORDS] + __attribute__((section(".data"), aligned(4096))) = {0}; +static uint64_t tail_out[BB_THREAD_CORE_COUNT][8] + __attribute__((section(".data"), aligned(4096))) = {0}; +unsigned bank_done[BB_THREAD_CORE_COUNT]; +void bank_exercise(unsigned core) { + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + for (unsigned i = 0; i < INPUT_WORDS; ++i) + input[core][i] = (uint64_t)(core + 1) << 32 | (i + 1); + for (unsigned i = 0; i < WORDS; ++i) + private_out[core][i] = 0; + for (unsigned i = 0; i < SHARED_WORDS; ++i) + shared_out[core][i] = 0; + for (unsigned i = 0; i < 8; ++i) + tail_out[core][i] = ~0ULL; + size_t bytes = ant_image_end - ant_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + struct ant_bank_args args = {(uintptr_t)input[core], + (uintptr_t)private_out[core], + (uintptr_t)shared_out[core], + (uintptr_t)tail_out[core], BB_SHARED_BANK_LINES}; + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + struct thread_task task = {core + 1, + 0, + bytes, + base, + base + + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); + for (unsigned i = 0; i < WORDS; ++i) + if (private_out[core][i] != (((uint64_t)(core + 1) << 32) | (i + 1))) + __builtin_trap(); + for (unsigned i = 0; i < SHARED_WORDS; ++i) + if (shared_out[core][i] != (((uint64_t)(core + 1) << 32) | (i + 1))) + __builtin_trap(); + for (unsigned i = 0; i < 8; ++i) + if (tail_out[core][i] != + (i % 2 ? 0 : (((uint64_t)(core + 1) << 32) | (i + 1)))) + __builtin_trap(); + __atomic_store_n(&bank_done[core], 1, __ATOMIC_RELEASE); +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_bank_main.c b/examples/designs/poly/workloads/device/core/engine/ant_bank_main.c new file mode 100644 index 000000000..bba2a6502 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_bank_main.c @@ -0,0 +1,50 @@ +#define _GNU_SOURCE +#include "ant_bank.h" +#include +#include +#include +#ifdef __linux__ +#include +#include +#include +#include +#include +#endif +#ifdef __linux__ +static void *worker(void *argument) { + uintptr_t core = (uintptr_t)argument; + cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET((core + 1), &mask); + if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) + abort(); + bank_exercise(core); + return NULL; +} +#endif +int main(void) { +#ifdef __linux__ + if (mlockall(MCL_CURRENT | MCL_FUTURE)) + abort(); + pthread_t threads[BB_THREAD_CORE_COUNT]; + for (uintptr_t core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (pthread_create(&threads[core], NULL, worker, (void *)core)) + abort(); + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (pthread_join(threads[core], NULL)) + abort(); + printf("Ant bank boundary PASS cores=%u private=%u shared=%u\n", + (unsigned)BB_THREAD_CORE_COUNT, BANK_LINES, BB_SHARED_BANK_LINES); +#else + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == BB_REPORT_HART) { + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + while (!__atomic_load_n(&bank_done[core], __ATOMIC_ACQUIRE)) { + } + } else { + bank_exercise(hart - 1); + } +#endif + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_exercise.c b/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_exercise.c new file mode 100644 index 000000000..44e572ef2 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_exercise.c @@ -0,0 +1,65 @@ +#include "ant_bank_rows.h" +#include +#include +#include +extern const unsigned char ant_image[], ant_image_end[]; +enum { + WORDS = ANT_BANK_ROWS * BANK_WIDTH / 64, + SHARED_WORDS = ANT_BANK_ROWS * BANK_WIDTH / 64 +}; +static uint64_t input[BB_THREAD_CORE_COUNT][WORDS] + __attribute__((section(".data"), aligned(4096))) = {0}; +static uint64_t private_out[BB_THREAD_CORE_COUNT][WORDS] + __attribute__((section(".data"), aligned(4096))) = {0}; +static uint64_t shared_out[BB_THREAD_CORE_COUNT][SHARED_WORDS] + __attribute__((section(".data"), aligned(4096))) = {0}; +static uint64_t tail_out[BB_THREAD_CORE_COUNT][8] + __attribute__((section(".data"), aligned(4096))) = {0}; +unsigned bank_rows_done[BB_THREAD_CORE_COUNT]; +void bank_rows_exercise(unsigned core) { + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + for (unsigned i = 0; i < WORDS; ++i) + input[core][i] = (uint64_t)(core + 1) << 32 | (i + 1); + for (unsigned i = 0; i < WORDS; ++i) + private_out[core][i] = 0; + for (unsigned i = 0; i < SHARED_WORDS; ++i) + shared_out[core][i] = 0; + for (unsigned i = 0; i < 8; ++i) + tail_out[core][i] = ~0ULL; + size_t bytes = ant_image_end - ant_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + struct ant_bank_rows_args args = { + (uintptr_t)input[core], (uintptr_t)private_out[core], + (uintptr_t)shared_out[core], (uintptr_t)tail_out[core], ANT_BANK_ROWS}; + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + struct thread_task task = {core + 1, + 0, + bytes, + base, + base + + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); + for (unsigned i = 0; i < WORDS; ++i) + if (private_out[core][i] != (((uint64_t)(core + 1) << 32) | (i + 1))) + __builtin_trap(); + for (unsigned i = 0; i < SHARED_WORDS; ++i) + if (shared_out[core][i] != (((uint64_t)(core + 1) << 32) | (i + 1))) + __builtin_trap(); + for (unsigned i = 0; i < 8; ++i) + if (tail_out[core][i] != + (i % 2 ? 0 : (((uint64_t)(core + 1) << 32) | (i + 1)))) + __builtin_trap(); + __atomic_store_n(&bank_rows_done[core], 1, __ATOMIC_RELEASE); +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_main.c b/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_main.c new file mode 100644 index 000000000..4769f77e4 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_main.c @@ -0,0 +1,50 @@ +#define _GNU_SOURCE +#include "ant_bank_rows.h" +#include +#include +#include +#ifdef __linux__ +#include +#include +#include +#include +#include +#endif +#ifdef __linux__ +static void *worker(void *argument) { + uintptr_t core = (uintptr_t)argument; + cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET((core + 1), &mask); + if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) + abort(); + bank_rows_exercise(core); + return NULL; +} +#endif +int main(void) { +#ifdef __linux__ + if (mlockall(MCL_CURRENT | MCL_FUTURE)) + abort(); + pthread_t threads[BB_THREAD_CORE_COUNT]; + for (uintptr_t core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (pthread_create(&threads[core], NULL, worker, (void *)core)) + abort(); + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (pthread_join(threads[core], NULL)) + abort(); + printf("Ant bank boundary PASS cores=%u private=%u shared=%u\n", + (unsigned)BB_THREAD_CORE_COUNT, BANK_LINES, BB_SHARED_BANK_LINES); +#else + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == BB_REPORT_HART) { + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + while (!__atomic_load_n(&bank_rows_done[core], __ATOMIC_ACQUIRE)) { + } + } else { + bank_rows_exercise(hart - 1); + } +#endif + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_highpa_main.c b/examples/designs/poly/workloads/device/core/engine/ant_highpa_main.c new file mode 100644 index 000000000..bcedfacf4 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_highpa_main.c @@ -0,0 +1,68 @@ +#include "ant_highpa.h" +#include +#include +#include +extern const unsigned char ant_image[], ant_image_end[]; +static unsigned done[BB_THREAD_CORE_COUNT]; +static void exercise(unsigned core) { + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + uintptr_t high = 0x180200000ULL + core * 0x20000ULL; + volatile uint64_t *input = (void *)high, *output = (void *)(high + 0x1000); + volatile uint64_t *low = (void *)(high & 0xffffffffULL); + volatile uint64_t *sentinel = (void *)((high + 0x1000) & 0xffffffffULL); + const uint64_t marker = 0xdeadface12345678ULL; + uint64_t first = 0x1234567800000000ULL | ((uint64_t)core << 16); + uint64_t second = 0xfedcba9800000000ULL | ((uint64_t)core << 16); + for (unsigned i = 0; i < 4; ++i) { + low[i] = second + i; + sentinel[i] = marker; + input[i] = first + i; + output[i] = 0; + } + for (unsigned i = 0; i < 4; ++i) + if (input[i] != first + i || low[i] != second + i || sentinel[i] != marker) + __builtin_trap(); + size_t bytes = ant_image_end - ant_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + for (unsigned phase = 0; phase < 2; ++phase) { + struct ant_highpa_args args = {phase ? (uintptr_t)low : high, + high + 0x1000}; + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + struct thread_task task = { + core * 2 + phase + 1, + 0, + bytes, + base, + base + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); + for (unsigned i = 0; i < 4; ++i) + if (output[i] != (phase ? second : first) + i || sentinel[i] != marker || + input[i] != first + i || low[i] != second + i) + __builtin_trap(); + } + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == BB_REPORT_HART) { + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_mxmm_main.cpp b/examples/designs/poly/workloads/device/core/engine/ant_mxmm_main.cpp new file mode 100644 index 000000000..b49e632da --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_mxmm_main.cpp @@ -0,0 +1,100 @@ +#define _GNU_SOURCE +#include "ant_mxmm.h" +#include +#include +#include +#ifdef __linux__ +#include +#include +#include +#include +#include +#endif +extern const unsigned char ant_image[], ant_image_end[]; +struct matrices { + float a[BB_THREAD_CORE_COUNT][ANT_M * ANT_K]; + float b[BB_THREAD_CORE_COUNT][ANT_N * ANT_K]; + float out[BB_THREAD_CORE_COUNT][ANT_M * ANT_N]; +}; +static constexpr matrices prepare() { + matrices data{}; + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) { + for (unsigned row = 0; row < ANT_M; ++row) + for (unsigned k = 0; k < ANT_K; ++k) + data.a[core][row * ANT_K + k] = row + core + 1; + for (unsigned col = 0; col < ANT_N; ++col) + for (unsigned k = 0; k < ANT_K; ++k) + data.b[core][col * ANT_K + k] = col + 1; + for (unsigned i = 0; i < ANT_M * ANT_N; ++i) + data.out[core][i] = -1; + } + return data; +} +alignas(64) static matrices data = prepare(); +static unsigned done[BB_THREAD_CORE_COUNT]; +static void exercise(unsigned core) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = ant_image_end - ant_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + struct ant_mxmm_args args = {(uintptr_t)data.a[core], (uintptr_t)data.b[core], + (uintptr_t)data.out[core]}; + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); + for (unsigned row = 0; row < ANT_M; ++row) + for (unsigned col = 0; col < ANT_N; ++col) + if (data.out[core][row * ANT_N + col] != + (float)(ANT_K * (row + core + 1) * (col + 1))) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} +#ifdef __linux__ +static void *worker(void *argument) { + uintptr_t core = (uintptr_t)argument; + cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET((core + 1), &mask); + if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) + abort(); + exercise(core); + return NULL; +} +#endif +int main(void) { +#ifdef __linux__ + if (mlockall(MCL_CURRENT | MCL_FUTURE)) + abort(); + pthread_t threads[BB_THREAD_CORE_COUNT]; + for (uintptr_t core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (pthread_create(&threads[core], NULL, worker, (void *)core)) + abort(); + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (pthread_join(threads[core], NULL)) + abort(); + printf("Ant Mxmm PASS cores=%u\n", (unsigned)BB_THREAD_CORE_COUNT); +#else + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == BB_REPORT_HART) { + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } +#endif + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_parallel_exercise.c b/examples/designs/poly/workloads/device/core/engine/ant_parallel_exercise.c new file mode 100644 index 000000000..8144fab77 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_parallel_exercise.c @@ -0,0 +1,82 @@ +#include "ant_parallel.h" +#include +#include +#include +extern const unsigned char ant_image[], ant_image_end[]; +static uint64_t input[BB_THREAD_CORE_COUNT][2][512] + __attribute__((aligned(4096), section(".noinit"))); +static uint64_t output[BB_THREAD_CORE_COUNT][2][512] + __attribute__((aligned(4096), section(".noinit"))); +unsigned parallel_done[BB_THREAD_CORE_COUNT]; +static unsigned reached[3] __attribute__((aligned(64))); +static void barrier(unsigned phase) { + __atomic_fetch_add(&reached[phase], 1, __ATOMIC_ACQ_REL); + while (__atomic_load_n(&reached[phase], __ATOMIC_ACQUIRE) != + BB_THREAD_CORE_COUNT) { + } +} +void parallel_exercise(unsigned core) { + const uint32_t contexts[] = {thread_find(THREAD_SIGNATURE, 0), + thread_find(THREAD_SIGNATURE, 1)}; + size_t bytes = ant_image_end - ant_image; + uint64_t shared = thread_query(contexts[0], THREAD_FIELD_TSS_BASE), zero = 0; + struct thread_task tasks[2]; + for (unsigned context = 0; context < 2; ++context) { + for (unsigned word = 0; word < 2; ++word) { + input[core][context][word] = + ((uint64_t)(core + 1) << 32) | (context * 2 + word + 1); + output[core][context][word] = ~0ULL; + } + uint64_t base = thread_query(contexts[context], THREAD_FIELD_TLS_BASE); + if (bytes > thread_query(contexts[context], THREAD_FIELD_CODE_BYTES) || + thread_query(contexts[context], THREAD_FIELD_SIGNATURE) != + THREAD_SIGNATURE) + __builtin_trap(); + struct ant_parallel_args args = {shared, context, + (uintptr_t)input[core][context], + (uintptr_t)output[core][context]}; + thread_tss_write(0, ant_image, bytes); + thread_load(contexts[context], THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, &args, sizeof(args)); + thread_load(contexts[context], THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + tasks[context] = (struct thread_task){ + context + 1, + 0, + bytes, + base, + base + thread_query(contexts[context], THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + } + thread_tss_write(0, &zero, sizeof(zero)); + barrier(0); + unsigned peer = (core + 1) % BB_THREAD_CORE_COUNT; + for (unsigned context = 0; context < 2; ++context) + for (unsigned word = 0; word < 2; ++word) + if (*(volatile uint64_t *)&output[peer][context][word] != ~0ULL) + __builtin_trap(); + barrier(1); + thread_acquire(); + thread_start(contexts[0], &tasks[0]); + struct ant_parallel_args pending = {shared, 1, (uintptr_t)input[core][1], + (uintptr_t)output[core][1]}; + thread_tss_write(64, &pending, sizeof(pending)); + thread_load(contexts[1], THREAD_TLS, 0, 64, sizeof(pending)); + thread_start(contexts[1], &tasks[1]); + for (unsigned context = 0; context < 2; ++context) { + int cancelled; + if (thread_wait(contexts[context], tasks[context].id, &cancelled) || + cancelled) + __builtin_trap(); + for (unsigned word = 0; word < 2; ++word) + if (output[core][context][word] != input[core][context][word]) + __builtin_trap(); + } + thread_release(); + barrier(2); + for (unsigned context = 0; context < 2; ++context) + for (unsigned word = 0; word < 2; ++word) + if (*(volatile uint64_t *)&output[peer][context][word] != + input[peer][context][word]) + __builtin_trap(); + __atomic_store_n(¶llel_done[core], 1, __ATOMIC_RELEASE); +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_parallel_main.c b/examples/designs/poly/workloads/device/core/engine/ant_parallel_main.c new file mode 100644 index 000000000..6929f72f6 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_parallel_main.c @@ -0,0 +1,50 @@ +#define _GNU_SOURCE +#include "ant_parallel.h" +#include +#include +#include +#ifdef __linux__ +#include +#include +#include +#include +#include +#endif +#ifdef __linux__ +static void *worker(void *argument) { + uintptr_t core = (uintptr_t)argument; + cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET((core + 1), &mask); + if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) + abort(); + parallel_exercise(core); + return NULL; +} +#endif +int main(void) { +#ifdef __linux__ + if (mlockall(MCL_CURRENT | MCL_FUTURE)) + abort(); + pthread_t threads[BB_THREAD_CORE_COUNT]; + for (uintptr_t core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (pthread_create(&threads[core], NULL, worker, (void *)core)) + abort(); + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (pthread_join(threads[core], NULL)) + abort(); + printf("Ant concurrent DMA/cache sharing PASS cores=%u contexts=2\n", + (unsigned)BB_THREAD_CORE_COUNT); +#else + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == BB_REPORT_HART) { + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + while (!__atomic_load_n(¶llel_done[core], __ATOMIC_ACQUIRE)) { + } + } else { + parallel_exercise(hart - 1); + } +#endif + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_shared_exercise.c b/examples/designs/poly/workloads/device/core/engine/ant_shared_exercise.c new file mode 100644 index 000000000..f01e8d09c --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_shared_exercise.c @@ -0,0 +1,70 @@ +#include "ant_shared.h" +#include +#include +#include +extern const unsigned char ant_image[], ant_image_end[]; +unsigned shared_done[BB_THREAD_CORE_COUNT]; +void shared_exercise(unsigned core) { + const uint32_t contexts[] = {thread_find(THREAD_SIGNATURE, 0), + thread_find(THREAD_SIGNATURE, 1)}; + size_t bytes = ant_image_end - ant_image; + uint64_t value = 100 + core, + shared = thread_query(contexts[0], THREAD_FIELD_TSS_BASE), + zero[2] = {0}; + struct thread_task tasks[2]; + for (unsigned context = 0; context < 2; ++context) { + uint64_t base = thread_query(contexts[context], THREAD_FIELD_TLS_BASE); + if (bytes > thread_query(contexts[context], THREAD_FIELD_CODE_BYTES) || + thread_query(contexts[context], THREAD_FIELD_SIGNATURE) != + THREAD_SIGNATURE) + __builtin_trap(); + tasks[context] = (struct thread_task){ + context + 1, + 0, + bytes, + base, + base + thread_query(contexts[context], THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + struct ant_shared_args args = {shared, context == 0, context ? 0 : value}; + thread_tss_write(0, ant_image, bytes); + thread_load(contexts[context], THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, &args, sizeof(args)); + thread_load(contexts[context], THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + } + thread_tss_write(0, zero, sizeof(zero)); + thread_acquire(); + thread_start(contexts[1], &tasks[1]); + thread_start(contexts[0], &tasks[0]); + for (unsigned context = 0; context < 2; ++context) { + int cancelled; + if (thread_wait(contexts[context], tasks[context].id, &cancelled) != + value || + cancelled) + __builtin_trap(); + } + thread_release(); + for (unsigned context = 0; context < 2; ++context) { + thread_store(contexts[context], THREAD_TLS, 16, 0, 8); + if (thread_tss_read(0) != value) + __builtin_trap(); + } + thread_tss_write(0, zero, sizeof(zero)); + thread_acquire(); + thread_start(contexts[1], &tasks[1]); + thread_cancel(contexts[1]); + int cancelled; + thread_wait(contexts[1], tasks[1].id, &cancelled); + if (!cancelled) + __builtin_trap(); + thread_release(); + uint64_t ready[2] = {value + 7, 1}; + thread_tss_write(0, ready, sizeof(ready)); + ++tasks[1].id; + thread_acquire(); + thread_start(contexts[1], &tasks[1]); + if (thread_wait(contexts[1], tasks[1].id, &cancelled) != value + 7 || + cancelled) + __builtin_trap(); + thread_release(); + __atomic_store_n(&shared_done[core], 1, __ATOMIC_RELEASE); +} diff --git a/examples/designs/poly/workloads/device/core/engine/ant_shared_main.c b/examples/designs/poly/workloads/device/core/engine/ant_shared_main.c new file mode 100644 index 000000000..56057ff11 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/ant_shared_main.c @@ -0,0 +1,45 @@ +#define _GNU_SOURCE +#include "ant_shared.h" +#include +#include +#include +#ifdef __linux__ +#include +#include +#include +#include +#endif +#ifdef __linux__ +static void *worker(void *argument) { + uintptr_t core = (uintptr_t)argument; + cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET((core + 1), &mask); + if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask)) + abort(); + shared_exercise(core); + return NULL; +} +#endif +int main(void) { +#ifdef __linux__ + pthread_t threads[BB_THREAD_CORE_COUNT]; + for (uintptr_t core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (pthread_create(&threads[core], NULL, worker, (void *)core)) + abort(); + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (pthread_join(threads[core], NULL)) + abort(); + printf("Ant shared/cancel PASS cores=%u\n", (unsigned)BB_THREAD_CORE_COUNT); +#else + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart != BB_REPORT_HART) + shared_exercise(hart - 1); + else + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + while (!__atomic_load_n(&shared_done[core], __ATOMIC_ACQUIRE)) { + } +#endif + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/CMakeLists.txt b/examples/designs/poly/workloads/device/core/thread/CMakeLists.txt new file mode 100644 index 000000000..833069e63 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(ctests) +set(POLY_ANT_IMAGE_DIR "${POLY_ANT_IMAGE_DIR}" PARENT_SCOPE) +set(POLY_ANT_SOURCE_DIR "${POLY_ANT_SOURCE_DIR}" PARENT_SCOPE) diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/CMakeLists.txt b/examples/designs/poly/workloads/device/core/thread/ctests/CMakeLists.txt new file mode 100644 index 000000000..4ce0d5d56 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/CMakeLists.txt @@ -0,0 +1,94 @@ +foreach( + POLY_ANT_CASE + mxmm + shared + bank + bank_rows + parallel + mesh_move + highpa + corelink + corelink_bank + corelink_reduce + coherence + mesh_private + mesh_shared +) + set(POLY_ANT_THREAD attention) + set(POLY_ANT_PAYLOAD ant_${POLY_ANT_CASE}.c) + set(POLY_ANT_HEADER ant_${POLY_ANT_CASE}.h) + set(POLY_ANT_HOST_DEFINITIONS) + if(POLY_ANT_CASE MATCHES "^mesh_(private|shared)$") + set(POLY_ANT_HEADER ant_mesh.h) + if(POLY_ANT_CASE STREQUAL "mesh_private") + set(POLY_ANT_PAYLOAD mesh_private_bank_move_test.c) + set(POLY_ANT_HOST_DEFINITIONS -DPOLY_MESH_PRIVATE=1) + else() + set(POLY_ANT_PAYLOAD mesh_shared_mem_test.c) + set(POLY_ANT_HOST_DEFINITIONS -DPOLY_MESH_PRIVATE=0) + endif() + endif() + if(POLY_ANT_CASE STREQUAL "bank" OR POLY_ANT_CASE STREQUAL "bank_rows") + set(POLY_ANT_THREAD ffn) + endif() + set(POLY_ANT_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${POLY_ANT_THREAD}-ctest-design-ant_${POLY_ANT_CASE}" + ) + foreach(_name BANK_WIDTH BANK_LINES BB_PRIVATE_VBANK_MAX) + file( + STRINGS "${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD}/params.h" + _line + REGEX "^#define ${_name} " + ) + string(REGEX REPLACE ".* " "" _${_name} "${_line}") + endforeach() + if( + NOT _BANK_WIDTH EQUAL 128 + OR _BANK_LINES LESS 128 + OR _BB_PRIVATE_VBANK_MAX LESS 5 + ) + message( + FATAL_ERROR + "Ant Mxmm needs 128-bit rows, >=128 rows/bank and private banks through 5; actual width=${_BANK_WIDTH}, rows=${_BANK_LINES}, privateMax=${_BB_PRIVATE_VBANK_MAX}" + ) + endif() + set(POLY_ANT_IMAGE ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_NAME}-ant) + set(ANT_RUNTIME_DIR ${BUCKYBALL_REPO_ROOT}/stack/engine/program) + buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_PAYLOAD}") + add_custom_command( + OUTPUT ${POLY_ANT_IMAGE}.elf ${POLY_ANT_IMAGE}.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${ANT_RUNTIME_DIR}/code.ld -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD} + -I${BUCKYBALL_REPO_ROOT}/examples/components/balls/mxmm/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include + ${ANT_RUNTIME_DIR}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_PAYLOAD} -o + ${POLY_ANT_IMAGE}.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${POLY_ANT_IMAGE}.elf + ${POLY_ANT_IMAGE}.bin + DEPENDS + ${POLY_ANT_PAYLOAD} + ${POLY_ANT_HEADER} + ${ANT_RUNTIME_DIR}/start.S + ${ANT_RUNTIME_DIR}/code.ld + ${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD}/params.h + ${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD}/ballISA.h + ${BUCKYBALL_BALL_ISA_DEPS} + ) + add_custom_target( + ${POLY_ANT_NAME}-ant-build + DEPENDS ${POLY_ANT_IMAGE}.elf ${POLY_ANT_IMAGE}.bin + ) + add_dependencies(poly-ctests-build ${POLY_ANT_NAME}-ant-build) +endforeach() + +set(POLY_ANT_IMAGE_DIR "${CMAKE_CURRENT_BINARY_DIR}" PARENT_SCOPE) +set(POLY_ANT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" PARENT_SCOPE) diff --git a/examples/chips/poly/workloads/ctests/ant_bank.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.c similarity index 88% rename from examples/chips/poly/workloads/ctests/ant_bank.c rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.c index 31a9026c2..730c1f791 100644 --- a/examples/chips/poly/workloads/ctests/ant_bank.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.c @@ -1,7 +1,8 @@ #include "ant_bank.h" #include +#include -uint64_t ant_main(const struct ant_bank_args *args) { +uint64_t entry(const struct ant_bank_args *args) { bb_mset(3, 1, 1, 1); bb_mvin(args->input, 3, BANK_LINES, 1); bb_mvout(args->private_out, 3, BANK_LINES, 1); diff --git a/examples/chips/poly/workloads/ctests/ant_bank.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.h similarity index 81% rename from examples/chips/poly/workloads/ctests/ant_bank.h rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.h index 0034d3669..d482aeab4 100644 --- a/examples/chips/poly/workloads/ctests/ant_bank.h +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.h @@ -6,4 +6,4 @@ struct ant_bank_args { }; extern unsigned bank_done[]; -void bank_exercise(unsigned tile); +void bank_exercise(unsigned core); diff --git a/examples/chips/poly/workloads/ctests/ant_bank_rows.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.c similarity index 88% rename from examples/chips/poly/workloads/ctests/ant_bank_rows.c rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.c index 17d8d191a..8d2753ae4 100644 --- a/examples/chips/poly/workloads/ctests/ant_bank_rows.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.c @@ -1,7 +1,8 @@ #include "ant_bank_rows.h" #include +#include -uint64_t ant_main(const struct ant_bank_rows_args *args) { +uint64_t entry(const struct ant_bank_rows_args *args) { bb_mset(3, 1, 1, 1); bb_mvin(args->input, 3, ANT_BANK_ROWS, 1); bb_mvout(args->private_out, 3, ANT_BANK_ROWS, 1); diff --git a/examples/chips/poly/workloads/ctests/ant_bank_rows.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.h similarity index 82% rename from examples/chips/poly/workloads/ctests/ant_bank_rows.h rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.h index d93667015..68825f29a 100644 --- a/examples/chips/poly/workloads/ctests/ant_bank_rows.h +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.h @@ -7,4 +7,4 @@ struct ant_bank_rows_args { }; extern unsigned bank_rows_done[]; -void bank_rows_exercise(unsigned tile); +void bank_rows_exercise(unsigned core); diff --git a/examples/chips/poly/workloads/ctests/ant_coherence.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_coherence.c similarity index 83% rename from examples/chips/poly/workloads/ctests/ant_coherence.c rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_coherence.c index 3be1a9d82..097b98308 100644 --- a/examples/chips/poly/workloads/ctests/ant_coherence.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_coherence.c @@ -1,7 +1,7 @@ #include "ant_coherence.h" #include -uint64_t ant_main(const struct ant_coherence_args *args) { +uint64_t entry(const struct ant_coherence_args *args) { if (args->phase == 0) bb_mset(3, 1, 1, 1); else if (args->phase == 1) { diff --git a/examples/chips/poly/workloads/ctests/ant_coherence.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_coherence.h similarity index 100% rename from examples/chips/poly/workloads/ctests/ant_coherence.h rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_coherence.h diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.c new file mode 100644 index 000000000..2e9f9e9b5 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.c @@ -0,0 +1,17 @@ +#include "ant_corelink.h" +#include +#include +uint64_t entry(const struct ant_corelink_args *args) { + if (args->phase == 0) { + bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); + bb_mset(BB_SHARED_BANK_BASE + 1, 1, 1, 1); + bb_mvin(args->input, BB_SHARED_BANK_BASE, CORELINK_TEST_ROWS, 1); + } else if (args->phase == 1) { + bb_mvout(args->output, BB_SHARED_BANK_BASE + 1, CORELINK_TEST_ROWS, 1); + } else if (args->phase == 2) { + bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); + bb_mset(BB_SHARED_BANK_BASE + 1, 0, 0, 0); + } else + __builtin_trap(); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.h new file mode 100644 index 000000000..d8341d49d --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.h @@ -0,0 +1,9 @@ +#pragma once +#include +enum { + CORELINK_TEST_BYTES = 4160, + CORELINK_TEST_ROWS = CORELINK_TEST_BYTES / 16 +}; +struct ant_corelink_args { + uint64_t phase, input, output; +}; diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.c new file mode 100644 index 000000000..76c1a2982 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.c @@ -0,0 +1,17 @@ +#include "ant_corelink_bank.h" +#include +#include +uint64_t entry(const struct ant_corelink_bank_args *args) { + if (args->phase == 0) { + bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); + bb_mset(BB_SHARED_BANK_BASE + 1, 1, 1, 1); + bb_mvin(args->input, BB_SHARED_BANK_BASE, CORELINK_CROSS_ROWS, 1); + } else if (args->phase == 1) { + bb_mvout(args->output, BB_SHARED_BANK_BASE, CORELINK_CROSS_ROWS, 1); + } else if (args->phase == 2) { + bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); + bb_mset(BB_SHARED_BANK_BASE + 1, 0, 0, 0); + } else + __builtin_trap(); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.h new file mode 100644 index 000000000..2c21dc932 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.h @@ -0,0 +1,9 @@ +#pragma once +#include +enum { + CORELINK_CROSS_BYTES = 192, + CORELINK_CROSS_ROWS = CORELINK_CROSS_BYTES / 16 +}; +struct ant_corelink_bank_args { + uint64_t phase, input, output; +}; diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.c new file mode 100644 index 000000000..5a59fedec --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.c @@ -0,0 +1,22 @@ +#include "ant_corelink_reduce.h" +#include +#include +#include +uint64_t entry(const struct ant_corelink_reduce_args *args) { + if (args->phase == 0) { + bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); + bb_mset(0, 1, 1, 1); + bb_mset(1, 1, 1, 1); + bb_mvin(args->input, 0, 4, 1); + bb_mvin(args->input + 16 * sizeof(float), 1, 64, 1); + bb_mxmm_fma32(0, 1, BB_SHARED_BANK_BASE, 1, 16, 16, 1, 1, 0); + bb_mset(0, 0, 0, 0); + bb_mset(1, 0, 0, 0); + } else if (args->phase == 1) { + bb_mvout(args->output, BB_SHARED_BANK_BASE, 2, 1); + } else if (args->phase == 2) { + bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); + } else + __builtin_trap(); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.h new file mode 100644 index 000000000..d930c66e8 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.h @@ -0,0 +1,8 @@ +#pragma once +#include +struct ant_corelink_reduce_args { + uint64_t phase, input, output; +}; +void ant_corelink_reduce_phase(unsigned core, + const struct ant_corelink_reduce_args *args); +void ant_corelink_reduce_prepare(unsigned core, float *input, float *output); diff --git a/examples/chips/poly/workloads/ctests/ant_highpa.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_highpa.c similarity index 76% rename from examples/chips/poly/workloads/ctests/ant_highpa.c rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_highpa.c index 7f790ae9c..1d10edf92 100644 --- a/examples/chips/poly/workloads/ctests/ant_highpa.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_highpa.c @@ -1,6 +1,6 @@ #include "ant_highpa.h" #include -uint64_t ant_main(const struct ant_highpa_args *args) { +uint64_t entry(const struct ant_highpa_args *args) { bb_mset(3, 1, 1, 1); bb_mvin(args->input, 3, 2, 1); bb_mvout(args->output, 3, 2, 1); diff --git a/examples/chips/poly/workloads/ctests/ant_highpa.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_highpa.h similarity index 100% rename from examples/chips/poly/workloads/ctests/ant_highpa.h rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_highpa.h diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh.h new file mode 100644 index 000000000..9c8408a36 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh.h @@ -0,0 +1,5 @@ +#pragma once +#include +struct ant_mesh_args { + uint64_t input, output, index; +}; diff --git a/examples/chips/poly/workloads/ctests/ant_mesh_move.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh_move.c similarity index 92% rename from examples/chips/poly/workloads/ctests/ant_mesh_move.c rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh_move.c index bbcb12493..86db5aaf6 100644 --- a/examples/chips/poly/workloads/ctests/ant_mesh_move.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh_move.c @@ -3,7 +3,7 @@ enum { ROWS = 2, SOURCE_BANK = 1, TARGET_BANK = 8 }; -uint64_t ant_main(const struct ant_mesh_move_args *args) { +uint64_t entry(const struct ant_mesh_move_args *args) { volatile uint64_t *sync = (volatile uint64_t *)args->sync; if (args->role == MESH_FILL) { bb_mset(SOURCE_BANK, 1, 1, 1); diff --git a/examples/chips/poly/workloads/ctests/ant_mesh_move.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh_move.h similarity index 100% rename from examples/chips/poly/workloads/ctests/ant_mesh_move.h rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_mesh_move.h diff --git a/examples/chips/poly/workloads/ctests/ant_mxmm.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mxmm.c similarity index 87% rename from examples/chips/poly/workloads/ctests/ant_mxmm.c rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_mxmm.c index 78bf63180..88aebd321 100644 --- a/examples/chips/poly/workloads/ctests/ant_mxmm.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mxmm.c @@ -1,8 +1,9 @@ #include "ant_mxmm.h" #include #include +#include -uint64_t ant_main(const struct ant_mxmm_args *args) { +uint64_t entry(const struct ant_mxmm_args *args) { for (unsigned bank = 3; bank <= 5; ++bank) bb_mset(bank, 1, 1, 1); bb_mvin(args->a, 3, ANT_M * ANT_K * 4 / (BANK_WIDTH / 8), 1); diff --git a/examples/chips/poly/workloads/ctests/ant_mxmm.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mxmm.h similarity index 100% rename from examples/chips/poly/workloads/ctests/ant_mxmm.h rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_mxmm.h diff --git a/examples/chips/poly/workloads/ctests/ant_parallel.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_parallel.c similarity index 86% rename from examples/chips/poly/workloads/ctests/ant_parallel.c rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_parallel.c index e35835ad5..561344a90 100644 --- a/examples/chips/poly/workloads/ctests/ant_parallel.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_parallel.c @@ -1,7 +1,7 @@ #include "ant_parallel.h" #include -uint64_t ant_main(const struct ant_parallel_args *args) { +uint64_t entry(const struct ant_parallel_args *args) { volatile uint64_t *ready = (volatile uint64_t *)args->shared; if (args->publish) *ready = 1; diff --git a/examples/chips/poly/workloads/ctests/ant_parallel.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_parallel.h similarity index 78% rename from examples/chips/poly/workloads/ctests/ant_parallel.h rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_parallel.h index cb8268558..a3281b687 100644 --- a/examples/chips/poly/workloads/ctests/ant_parallel.h +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_parallel.h @@ -6,4 +6,4 @@ struct ant_parallel_args { }; extern unsigned parallel_done[]; -void parallel_exercise(unsigned tile); +void parallel_exercise(unsigned core); diff --git a/examples/chips/poly/workloads/ctests/ant_shared.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_shared.c similarity index 87% rename from examples/chips/poly/workloads/ctests/ant_shared.c rename to examples/designs/poly/workloads/device/core/thread/ctests/ant_shared.c index 2ad249004..2fa2fed88 100644 --- a/examples/chips/poly/workloads/ctests/ant_shared.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_shared.c @@ -1,6 +1,6 @@ #include "ant_shared.h" -uint64_t ant_main(struct ant_shared_args *args) { +uint64_t entry(struct ant_shared_args *args) { volatile uint64_t *shared = (volatile uint64_t *)args->shared; if (args->producer) { shared[0] = args->value; diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_shared.h b/examples/designs/poly/workloads/device/core/thread/ctests/ant_shared.h new file mode 100644 index 000000000..7924df30d --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_shared.h @@ -0,0 +1,9 @@ +#pragma once +#include + +struct ant_shared_args { + uint64_t shared, producer, value; +}; + +extern unsigned shared_done[]; +void shared_exercise(unsigned core); diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/mesh_private_bank_move_test.c b/examples/designs/poly/workloads/device/core/thread/ctests/mesh_private_bank_move_test.c new file mode 100644 index 000000000..ab28881da --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/mesh_private_bank_move_test.c @@ -0,0 +1,55 @@ +#include "ant_mesh.h" +#include +#include + +uint64_t entry(const struct ant_mesh_args *args) { + if (args->index == 0) { + bb_mem_alloc(0, 1, 1); + bb_mvin(args->input, 0, 2, 1); + } + if (args->index == 1) { + bb_mem_alloc(1, 1, 1); + bb_mem_alloc(4, 1, 1); + } + if (args->index == 2) + bb_mem_alloc(3, 1, 1); + if (args->index == 3) + bb_mem_alloc(2, 1, 1); + if (args->index == 4) + bb_mem_alloc(1, 1, 1); + + /* All five Buckyball threads join each barrier. Each barrier drains the ROB. + */ + bb_barrier(); + if (args->index == 0) { + bb_mvover(0, 0, 1, 1, 1, 1, 1); + bb_mvover(0, 0, 1, 3, 2, 1, 1); + } + bb_barrier(); + + if (args->index == 1) + bb_mvout(args->output + 0, 1, 2, 1); + if (args->index == 3) + bb_mvout(args->output + 32, 2, 2, 1); + bb_barrier(); + + if (args->index == 3) + bb_mvover(3, 2, 1, 2, 3, 1, 1); + bb_barrier(); + if (args->index == 2) + bb_mvout(args->output + 64, 3, 2, 1); + bb_barrier(); + + if (args->index == 0) + bb_mvover(0, 0, 1, 4, 1, 1, 1); + if (args->index == 3) + bb_mvover(3, 2, 1, 1, 4, 1, 1); + bb_barrier(); + if (args->index == 4) + bb_mvout(args->output + 96, 1, 2, 1); + if (args->index == 1) + bb_mvout(args->output + 128, 4, 2, 1); + bb_barrier(); + + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/mesh_shared_mem_test.c b/examples/designs/poly/workloads/device/core/thread/ctests/mesh_shared_mem_test.c new file mode 100644 index 000000000..aa8b7bf69 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/ctests/mesh_shared_mem_test.c @@ -0,0 +1,22 @@ +#include "ant_mesh.h" +#include +#include + +enum { + BANKS = VIRTUAL_BANK_NUM - BB_SHARED_BANK_BASE, + ROW_BYTES = BANK_WIDTH / 8 +}; +_Static_assert( + BANK_WIDTH == 128 && BANKS > 0 && BANKS <= SHARED_BANK_NUM, + "Shared bank test requires one physical bank per shared virtual bank"); +uint64_t entry(const struct ant_mesh_args *args) { + for (unsigned bank = 0; bank < BANKS; ++bank) { + bb_mem_alloc(BB_SHARED_BANK_BASE + bank, 1, 1); + bb_mvin(args->input + bank * ROW_BYTES, BB_SHARED_BANK_BASE + bank, 1, 1); + } + for (unsigned bank = 0; bank < BANKS; ++bank) + bb_mvout(args->output + bank * ROW_BYTES, BB_SHARED_BANK_BASE + bank, 1, 1); + for (unsigned bank = 0; bank < BANKS; ++bank) + bb_mem_release(BB_SHARED_BANK_BASE + bank); + return 0; +} diff --git a/examples/chips/poly/workloads/ctests/mxmm_fp32_small.c b/examples/designs/poly/workloads/device/core/thread/ctests/mxmm_fp32_small.c similarity index 100% rename from examples/chips/poly/workloads/ctests/mxmm_fp32_small.c rename to examples/designs/poly/workloads/device/core/thread/ctests/mxmm_fp32_small.c diff --git a/examples/chips/poly/workloads/ctests/mxmm_mxfp8_small.c b/examples/designs/poly/workloads/device/core/thread/ctests/mxmm_mxfp8_small.c similarity index 100% rename from examples/chips/poly/workloads/ctests/mxmm_mxfp8_small.c rename to examples/designs/poly/workloads/device/core/thread/ctests/mxmm_mxfp8_small.c diff --git a/examples/designs/poly/workloads/device/runtime/CMakeLists.txt b/examples/designs/poly/workloads/device/runtime/CMakeLists.txt new file mode 100644 index 000000000..6a043b962 --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/CMakeLists.txt @@ -0,0 +1,147 @@ +foreach( + POLY_ANT_CASE + mesh_move + corelink + corelink_bank + corelink_reduce + coherence + mesh_private + mesh_shared +) + set(POLY_ANT_THREAD attention) + set(POLY_ANT_HEADER ant_${POLY_ANT_CASE}.h) + set(POLY_ANT_HOST_DEFINITIONS) + if(POLY_ANT_CASE MATCHES "^mesh_(private|shared)$") + set(POLY_ANT_HEADER ant_mesh.h) + if(POLY_ANT_CASE STREQUAL "mesh_private") + set(POLY_ANT_PAYLOAD mesh_private_bank_move_test.c) + set(POLY_ANT_HOST_DEFINITIONS -DPOLY_MESH_PRIVATE=1) + else() + set(POLY_ANT_PAYLOAD mesh_shared_mem_test.c) + set(POLY_ANT_HOST_DEFINITIONS -DPOLY_MESH_PRIVATE=0) + endif() + endif() + + set(POLY_ANT_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-${POLY_ANT_THREAD}-ctest-design-ant_${POLY_ANT_CASE}" + ) + set(POLY_ANT_IMAGE ${POLY_ANT_IMAGE_DIR}/${POLY_ANT_NAME}-ant) + set(POLY_ANT_CONTROL_SOURCES) + if(POLY_ANT_CASE STREQUAL "corelink_reduce") + list( + APPEND POLY_ANT_CONTROL_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/ant_corelink_reduce_control.c + ) + endif() + foreach(SOURCE IN LISTS POLY_ANT_CONTROL_SOURCES) + buckyball_enforce_ctest_line_limit("${SOURCE}") + endforeach() + + set(POLY_ANT_EMBED ${CMAKE_CURRENT_BINARY_DIR}/ant_${POLY_ANT_CASE}_image.S) + enable_language(ASM) + file( + GENERATE OUTPUT ${POLY_ANT_EMBED} + CONTENT + ".section .rodata.ant,\"a\",@progbits\n.balign 8\n.global ant_image, ant_image_end\nant_image:\n.incbin \"${POLY_ANT_IMAGE}.bin\"\nant_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" + ) + set_source_files_properties( + ${POLY_ANT_EMBED} + PROPERTIES OBJECT_DEPENDS "${POLY_ANT_IMAGE}.bin" + ) + set(POLY_ANT_MAIN ant_${POLY_ANT_CASE}_main.c) + if(POLY_ANT_CASE MATCHES "^mesh_(private|shared)$") + set(POLY_ANT_MAIN ant_mesh_main.c) + endif() + + buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_MAIN}") + set(POLY_ANT_HOST_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-main-ctest-design-ant_${POLY_ANT_CASE}" + ) + set(BUCKYBALL_CTEST_TARGET ${POLY_ANT_THREAD}) + + set(POLY_ANT_HOST_THREAD ${POLY_ANT_THREAD}) + set(POLY_ANT_CONTROL_OBJECTS) + if(POLY_ANT_CASE STREQUAL "corelink_reduce") + set(POLY_ANT_HOST_THREAD main) + set(POLY_ANT_CONTROL_OBJECTS + ${CMAKE_CURRENT_BINARY_DIR}/ant_corelink_reduce_control.o + ) + add_custom_command( + OUTPUT ${POLY_ANT_CONTROL_OBJECTS} + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany + -msmall-data-limit=0 -fno-builtin -I${POLY_ANT_SOURCE_DIR} + -I${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD} + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -c + ${POLY_ANT_CONTROL_SOURCES} -o ${POLY_ANT_CONTROL_OBJECTS} + DEPENDS + ${POLY_ANT_CONTROL_SOURCES} + ${POLY_ANT_SOURCE_DIR}/ant_corelink_reduce.h + ${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD}/params.h + ) + set(POLY_ANT_CONTROL_SOURCES) + endif() + + set(POLY_ANT_HOST_ELF ${POLY_ANT_HOST_NAME}-multicore-baremetal) + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_HOST_ELF} + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} ${POLY_ANT_HOST_DEFINITIONS} + -I${POLY_ANT_SOURCE_DIR} + -I${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_THREAD} + -I${BUCKYBALL_ISA_DIR}/design -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/components/balls/f32add/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_MAIN} + ${POLY_ANT_CONTROL_SOURCES} ${POLY_ANT_CONTROL_OBJECTS} + ${POLY_ANT_EMBED} ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${POLY_ANT_HOST_ELF} + DEPENDS + ${POLY_ANT_MAIN} + ${POLY_ANT_SOURCE_DIR}/${POLY_ANT_HEADER} + ${POLY_BOOT_SOURCE} + ${POLY_ANT_EMBED} + ${BAREMETAL_DIR}/start.S + ${BBSIM_LD} + ${POLY_ANT_CONTROL_SOURCES} + ${POLY_ANT_CONTROL_OBJECTS} + ${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_THREAD}/params.h + ${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_THREAD}/ballISA.h + ${POLY_ANT_IMAGE}.bin + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.h + ${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD}/params.h + ${BUCKYBALL_ISA_DIR}/design/instances.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + add_custom_target( + ${POLY_ANT_HOST_NAME}-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_HOST_ELF} + ) + add_dependencies(poly-ctests-build ${POLY_ANT_HOST_NAME}-ctest-build) +endforeach() + +set(POLY_STREAM_NAME "${BUCKYBALL_WORKLOAD_DESIGN}-main-ctest-design-stream") +buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/stream_main.c") +add_executable(${POLY_STREAM_NAME}-linux stream_main.c) +target_include_directories( + ${POLY_STREAM_NAME}-linux + PRIVATE + ${BUCKYBALL_ISA_DIR}/design + ${BUCKYBALL_ISA_DIR}/attention + ${WORKLOAD_LIB_DIR} + ${BUCKYBALL_REPO_ROOT}/examples/components/balls/mxmm/workloads +) +target_link_libraries(${POLY_STREAM_NAME}-linux PRIVATE engine runtime-linux) +add_dependencies(poly-ctests-build ${POLY_STREAM_NAME}-linux) + +add_subdirectory(ctests) diff --git a/examples/designs/poly/workloads/device/runtime/ant_coherence_main.c b/examples/designs/poly/workloads/device/runtime/ant_coherence_main.c new file mode 100644 index 000000000..dc1d30ab3 --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ant_coherence_main.c @@ -0,0 +1,94 @@ +#include "ant_coherence.h" +#include +#include +#include +_Static_assert( + BB_CORE_NUM == BB_THREAD_CORE_COUNT + 1 && BB_THREAD_CORE_COUNT >= 2 && + !(BB_THREAD_CORE_COUNT & (BB_THREAD_CORE_COUNT - 1)), + "coherence gate requires a power-of-two thread-bearing core count"); +extern const unsigned char ant_image[], ant_image_end[]; +// Separate 64-byte lines/sets for data and synchronization; avoid main's +// top-of-page stack sets. +enum { + INPUT = 8, + OUTPUT = INPUT + BB_THREAD_CORE_COUNT, + READY = OUTPUT + BB_THREAD_CORE_COUNT, + DONE = READY + BB_THREAD_CORE_COUNT, + ROUND = DONE + BB_THREAD_CORE_COUNT +}; +static volatile uint64_t lines[ROUND + 1][8] + __attribute__((section(".data"), aligned(4096))) = {0}; +static uint64_t pattern(unsigned round, unsigned core, unsigned word) { + return ((uint64_t)round << 48) | ((uint64_t)(core + 1) << 32) | (word + 17); +} +static void task(unsigned core, unsigned phase, unsigned id) { + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + struct ant_coherence_args args = {phase, (uintptr_t)lines[INPUT + core], + (uintptr_t)lines[OUTPUT + core]}; + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + struct thread_task command = { + id, + 0, + ant_image_end - ant_image, + base, + base + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &command); + int cancelled; + if (thread_wait(context, id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} +static void controller(unsigned core) { + size_t bytes = ant_image_end - ant_image; + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + task(core, 0, 1); + __atomic_store_n(&lines[READY + core][0], 1, __ATOMIC_RELEASE); + for (unsigned round = 1; round <= 2; ++round) { + while (__atomic_load_n(&lines[ROUND][0], __ATOMIC_ACQUIRE) != round) { + } + task(core, 1, round + 1); + __atomic_store_n(&lines[DONE + core][0], round, __ATOMIC_RELEASE); + } + task(core, 2, 4); + __atomic_store_n(&lines[DONE + core][0], 3, __ATOMIC_RELEASE); +} +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart) + controller(hart - 1); + else { + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + while (!__atomic_load_n(&lines[READY + core][0], __ATOMIC_ACQUIRE)) { + } + for (unsigned round = 1; round <= 2; ++round) { + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned word = 0; word < 2; ++word) { + lines[INPUT + core][word] = pattern(round, core, word); + lines[OUTPUT + core][word] = ~pattern(round, core, word); + } + __atomic_store_n(&lines[ROUND][0], round, __ATOMIC_RELEASE); + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + while (__atomic_load_n(&lines[DONE + core][0], __ATOMIC_ACQUIRE) < + round) { + } + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned word = 0; word < 2; ++word) + if (lines[OUTPUT + core][word] != pattern(round, core, word)) + __builtin_trap(); + } + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + while (__atomic_load_n(&lines[DONE + core][0], __ATOMIC_ACQUIRE) != 3) { + } + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/runtime/ant_corelink_bank_main.c b/examples/designs/poly/workloads/device/runtime/ant_corelink_bank_main.c new file mode 100644 index 000000000..aad49586d --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ant_corelink_bank_main.c @@ -0,0 +1,97 @@ +#include "ant_corelink_bank.h" +#include +#include +#include +#include +_Static_assert( + BB_CORE_NUM == BB_THREAD_CORE_COUNT + 1 && BB_THREAD_CORE_COUNT >= 2 && + !(BB_THREAD_CORE_COUNT & (BB_THREAD_CORE_COUNT - 1)), + "CoreLink gate requires a power-of-two thread-bearing core count"); +extern const unsigned char ant_image[], ant_image_end[]; +static uint64_t input[BB_THREAD_CORE_COUNT][CORELINK_CROSS_BYTES / 8] + __attribute__((aligned(64))); +static uint64_t output[BB_THREAD_CORE_COUNT][CORELINK_CROSS_BYTES / 8] + __attribute__((aligned(64))); +static unsigned ready[BB_THREAD_CORE_COUNT], done[BB_THREAD_CORE_COUNT]; +static unsigned arrived[2]; +static void barrier(unsigned phase) { + __atomic_fetch_add(&arrived[phase], 1, __ATOMIC_ACQ_REL); + while (__atomic_load_n(&arrived[phase], __ATOMIC_ACQUIRE) != + BB_THREAD_CORE_COUNT) { + } +} +static unsigned pattern(unsigned core, unsigned i) { + return (i * 17 + core * 23 + 3) & 255; +} +static void phase(unsigned core, unsigned operation) { + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = ant_image_end - ant_image; + struct ant_corelink_bank_args args = {operation, (uintptr_t)input[core], + (uintptr_t)output[core]}; + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + struct thread_task task = {core * 3 + operation + 1, + 0, + bytes, + base, + base + + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} +static void exercise(unsigned core) { + uint64_t bank_bytes = corelink_query(CORELINK_BANK_BYTES); + if (corelink_query(CORELINK_CORE_ID) != core + 1 || + corelink_query(CORELINK_CORE_COUNT) != BB_CORE_NUM || + bank_bytes < CORELINK_CROSS_BYTES || + corelink_query(CORELINK_SHARED_BYTES) < bank_bytes * 2) + __builtin_trap(); + size_t bytes = ant_image_end - ant_image; + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + for (unsigned i = 0; i < CORELINK_CROSS_BYTES / 8; ++i) { + uint64_t value = 0; + for (unsigned j = 0; j < 8; ++j) + value |= (uint64_t)pattern(core, i * 8 + j) << (8 * j); + input[core][i] = value; + } + phase(core, 0); + barrier(0); + unsigned next = (core + 1) % BB_THREAD_CORE_COUNT; + unsigned previous = (core + BB_THREAD_CORE_COUNT - 1) % BB_THREAD_CORE_COUNT; + corelink_transfer(0, next + 1, bank_bytes - 16, 37); + barrier(1); + corelink_transfer(bank_bytes - 16, previous + 1, 128, 37); + __atomic_store_n(&ready[core], 1, __ATOMIC_RELEASE); + while (!__atomic_load_n(&ready[next], __ATOMIC_ACQUIRE)) { + } + phase(core, 1); + for (unsigned i = 0; i < CORELINK_CROSS_BYTES; ++i) { + unsigned expected = pattern(core, i >= 128 && i < 165 ? i - 128 : i); + if (((unsigned char *)output[core])[i] != expected) + __builtin_trap(); + } + phase(core, 2); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart) + exercise(hart - 1); + else + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/runtime/ant_corelink_main.c b/examples/designs/poly/workloads/device/runtime/ant_corelink_main.c new file mode 100644 index 000000000..b7e2dc138 --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ant_corelink_main.c @@ -0,0 +1,93 @@ +#include "ant_corelink.h" +#include +#include +#include +#include +_Static_assert( + BB_CORE_NUM == BB_THREAD_CORE_COUNT + 1 && BB_THREAD_CORE_COUNT >= 2 && + !(BB_THREAD_CORE_COUNT & (BB_THREAD_CORE_COUNT - 1)), + "CoreLink gate requires a power-of-two thread-bearing core count"); +extern const unsigned char ant_image[], ant_image_end[]; +static uint64_t input[BB_THREAD_CORE_COUNT][CORELINK_TEST_BYTES / 8] + __attribute__((aligned(64))); +static uint64_t output[BB_THREAD_CORE_COUNT][CORELINK_TEST_BYTES / 8] + __attribute__((aligned(64))); +static unsigned done[BB_THREAD_CORE_COUNT]; +static unsigned arrived[2]; +static void barrier(unsigned phase) { + __atomic_fetch_add(&arrived[phase], 1, __ATOMIC_ACQ_REL); + while (__atomic_load_n(&arrived[phase], __ATOMIC_ACQUIRE) != + BB_THREAD_CORE_COUNT) { + } +} +static unsigned pattern(unsigned core, unsigned i) { + return (i * 17 + core * 23 + 3) & 255; +} +static void phase(unsigned core, unsigned operation) { + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = ant_image_end - ant_image; + struct ant_corelink_args args = {operation, (uintptr_t)input[core], + (uintptr_t)output[core]}; + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + struct thread_task task = {core * 3 + operation + 1, + 0, + bytes, + base, + base + + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} +static void exercise(unsigned core) { + uint64_t bank_bytes = corelink_query(CORELINK_BANK_BYTES); + if (corelink_query(CORELINK_CORE_ID) != core + 1 || + corelink_query(CORELINK_CORE_COUNT) != BB_CORE_NUM || + corelink_query(CORELINK_BANK_BYTES) < CORELINK_TEST_BYTES || + corelink_query(CORELINK_SHARED_BYTES) < bank_bytes * 2) + __builtin_trap(); + size_t bytes = ant_image_end - ant_image; + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + for (unsigned i = 0; i < CORELINK_TEST_BYTES / 8; ++i) { + uint64_t value = 0; + for (unsigned j = 0; j < 8; ++j) + value |= (uint64_t)pattern(core, i * 8 + j) << (8 * j); + input[core][i] = value; + } + phase(core, 0); + barrier(0); + unsigned next = (core + 1) % BB_THREAD_CORE_COUNT; + unsigned previous = (core + BB_THREAD_CORE_COUNT - 1) % BB_THREAD_CORE_COUNT; + corelink_transfer(0, next + 1, bank_bytes + 16, 4101); + barrier(1); + phase(core, 1); + for (unsigned i = 0; i < CORELINK_TEST_BYTES; ++i) { + unsigned expected = i >= 16 && i < 4117 ? pattern(previous, i - 16) : 0; + if (((unsigned char *)output[core])[i] != expected) + __builtin_trap(); + } + phase(core, 2); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart) + exercise(hart - 1); + else + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_control.c b/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_control.c new file mode 100644 index 000000000..e9d708b42 --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_control.c @@ -0,0 +1,41 @@ +#include "ant_corelink_reduce.h" +#include +#include +extern const unsigned char ant_image[], ant_image_end[]; +void ant_corelink_reduce_phase(unsigned core, + const struct ant_corelink_reduce_args *args) { + unsigned context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = ant_image_end - ant_image; + thread_tss_write(0, args, sizeof(*args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(*args) + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + struct thread_task task = {core * 3 + args->phase + 1, + 0, + bytes, + base, + base + + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} +void ant_corelink_reduce_prepare(unsigned core, float *input, float *output) { + size_t bytes = ant_image_end - ant_image; + const unsigned context = thread_find(THREAD_SIGNATURE, 0); + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + for (unsigned i = 0; i < 272; ++i) + input[i] = 0.0f; + input[0] = core % 2 ? -(float)(core + 1) * 0.5f : (float)(core + 1) * 1.25f; + for (unsigned column = 0; column < 16; ++column) + input[16 + column * 16] = (float)(column + 1); + for (unsigned i = 0; i < 8; ++i) + output[i] = -77.0f; +} diff --git a/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_main.c b/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_main.c new file mode 100644 index 000000000..b59a2e2f4 --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_main.c @@ -0,0 +1,99 @@ +#include "ant_corelink_reduce.h" +#include +#include +#include +#include +#include +#include +_Static_assert( + BB_CORE_NUM == BB_THREAD_CORE_COUNT + 1 && BB_THREAD_CORE_COUNT >= 2 && + (BB_THREAD_CORE_COUNT & (BB_THREAD_CORE_COUNT - 1)) == 0, + "Reduction needs main and a power-of-two thread-bearing core count"); +static float input[BB_THREAD_CORE_COUNT][272] __attribute__((aligned(64))); +static float output[BB_THREAD_CORE_COUNT][16] __attribute__((aligned(64))); +static unsigned ready[BB_THREAD_CORE_COUNT], broadcast, + done[BB_THREAD_CORE_COUNT]; +enum { WINDOW = 4 }; +static uint64_t shared_base[BB_THREAD_CORE_COUNT], main_base[WINDOW]; +static unsigned main_ready, batch; +static float partial(unsigned core, unsigned i) { + return (float)(i + 1) * + (core % 2 ? -(float)(core + 1) * 0.5f : (float)(core + 1) * 1.25f); +} +static void controller(unsigned core) { + ant_corelink_reduce_prepare(core, input[core], output[core]); + struct ant_corelink_reduce_args args = {0, (uintptr_t)input[core], + (uintptr_t)output[core]}; + ant_corelink_reduce_phase(core, &args); + shared_base[core] = corelink_shared_export(0, BB_SHARED_BANK_BASE, 0); + while (!__atomic_load_n(&main_ready, __ATOMIC_ACQUIRE) || + __atomic_load_n(&batch, __ATOMIC_ACQUIRE) != core / WINDOW) { + } + corelink_transfer(shared_base[core], 0, main_base[core % WINDOW], 16); + __atomic_store_n(&ready[core], 1, __ATOMIC_RELEASE); + while (!__atomic_load_n(&broadcast, __ATOMIC_ACQUIRE)) { + } + args.phase = 1; + ant_corelink_reduce_phase(core, &args); + for (unsigned i = 0; i < 4; ++i) { + float expected = 0.0f; + for (unsigned rank = 0; rank < BB_THREAD_CORE_COUNT; ++rank) + expected += partial(rank, i); + if (output[core][i] != partial(core, i) || output[core][i + 4] != expected) + __builtin_trap(); + } + corelink_shared_release(0, BB_SHARED_BANK_BASE, 0); + args.phase = 2; + ant_corelink_reduce_phase(core, &args); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} +static void reduce(void) { + unsigned groups = + BB_THREAD_CORE_COUNT < WINDOW ? BB_THREAD_CORE_COUNT : WINDOW; + unsigned source = 32, accumulator = 33, target = 34; + bb_mem_alloc(source, 1, groups); + bb_mem_alloc(accumulator, 1, 1); + bb_mem_alloc(target, 1, 1); + for (unsigned group = 0; group < groups; ++group) + main_base[group] = corelink_shared_export(0, source, group); + __atomic_store_n(&main_ready, 1, __ATOMIC_RELEASE); + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) { + while (!__atomic_load_n(&ready[core], __ATOMIC_ACQUIRE)) { + } + bb_f32add(source, accumulator, target, 1, core % WINDOW, core == 0); + unsigned previous = accumulator; + accumulator = target; + target = previous; + if (core % WINDOW == WINDOW - 1) { + __atomic_store_n(&batch, core / WINDOW + 1, __ATOMIC_RELEASE); + } + } + uint64_t result = corelink_shared_export(0, accumulator, 0); + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + corelink_transfer(result, core + 1, shared_base[core] + 16, 16); + __atomic_store_n(&broadcast, 1, __ATOMIC_RELEASE); + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + corelink_shared_release(0, accumulator, 0); + for (unsigned group = 0; group < groups; ++group) + corelink_shared_release(0, source, group); + bb_mem_release(source); + bb_mem_release(accumulator); + bb_mem_release(target); +} +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart > BB_THREAD_CORE_COUNT || corelink_query(CORELINK_CORE_ID) != hart || + corelink_query(CORELINK_CORE_COUNT) != BB_THREAD_CORE_COUNT + 1 || + corelink_query(CORELINK_BANK_BYTES) < 48 || + corelink_query(CORELINK_SHARED_BYTES) < + corelink_query(CORELINK_BANK_BYTES) + BB_THREAD_CORE_COUNT * 16) + __builtin_trap(); + if (hart) + controller(hart - 1); + else + reduce(); + return 0; +} diff --git a/examples/designs/poly/workloads/device/runtime/ant_mesh_main.c b/examples/designs/poly/workloads/device/runtime/ant_mesh_main.c new file mode 100644 index 000000000..84a7727b3 --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ant_mesh_main.c @@ -0,0 +1,81 @@ +#include "ant_mesh.h" +#include +#include +#include + +enum { + BANKS = VIRTUAL_BANK_NUM - BB_SHARED_BANK_BASE, + ROW_BYTES = BANK_WIDTH / 8 +}; +_Static_assert(BANKS >= 2 && BANK_WIDTH == 128, + "Mesh tests require two 128-bit input rows"); +static uint8_t input[BB_THREAD_CORE_COUNT][BANKS * ROW_BYTES] + __attribute__((aligned(64))); +static uint8_t output[BB_THREAD_CORE_COUNT] + [5 * 2 * ROW_BYTES + BANKS * ROW_BYTES] + __attribute__((aligned(64))); +static unsigned done[BB_THREAD_CORE_COUNT]; +extern const unsigned char ant_image[], ant_image_end[]; +static void exercise(unsigned core) { + if (thread_query(0, THREAD_FIELD_COUNT) != 5) + __builtin_trap(); + for (unsigned i = 0; i < sizeof(input[core]); ++i) + input[core][i] = POLY_MESH_PRIVATE + ? (i < ROW_BYTES ? 0x30 + i : 0xa0 + i - ROW_BYTES) + : (i / ROW_BYTES * 17 + i % ROW_BYTES); + unsigned count = POLY_MESH_PRIVATE ? 5 : 1; + size_t bytes = ant_image_end - ant_image; + struct thread_task tasks[5]; + // Both Attention and FFN execute the same common bank-ISA payload. + for (unsigned context = 0; context < count; ++context) { + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES)) + __builtin_trap(); + struct ant_mesh_args args = {(uintptr_t)input[core], + (uintptr_t)output[core], context}; + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, &args, sizeof(args)); + thread_load(context, THREAD_TLS, 0, 0, (sizeof(args) + 7) & ~7u); + tasks[context] = (struct thread_task){ + context + 1, + 0, + bytes, + base, + base + thread_query(context, THREAD_FIELD_TLS_BYTES), + thread_query(context, THREAD_FIELD_SIGNATURE)}; + } + thread_acquire(); + for (unsigned context = 0; context < count; ++context) + thread_start(context, &tasks[context]); + for (unsigned context = 0; context < count; ++context) { + int cancelled; + if (thread_wait(context, tasks[context].id, &cancelled) || cancelled) + __builtin_trap(); + } + thread_release(); + if (POLY_MESH_PRIVATE) { + for (unsigned target = 0; target < 5; ++target) + for (unsigned byte = 0; byte < ROW_BYTES; ++byte) + if (output[core][target * 2 * ROW_BYTES + ROW_BYTES + byte] != + input[core][ROW_BYTES + byte]) + __builtin_trap(); + } else { + for (unsigned i = 0; i < sizeof(input[core]); ++i) + if (output[core][i] != input[core][i]) + __builtin_trap(); + } + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == BB_REPORT_HART) { + for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/runtime/ant_mesh_move_main.c b/examples/designs/poly/workloads/device/runtime/ant_mesh_move_main.c new file mode 100644 index 000000000..1949c7c6d --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ant_mesh_move_main.c @@ -0,0 +1,94 @@ +#define _GNU_SOURCE +#include "ant_mesh_move.h" +#include +#include +#include + +extern const unsigned char ant_image[], ant_image_end[]; +enum { ROW_WORDS = BANK_WIDTH / 64, ROWS = 2 }; +static uint64_t input[BB_THREAD_CORE_COUNT][ROWS * ROW_WORDS] + __attribute__((section(".data"), aligned(4096))); +static uint64_t output[BB_THREAD_CORE_COUNT][ROWS * ROW_WORDS] + __attribute__((section(".data"), aligned(4096))); +static unsigned done[BB_THREAD_CORE_COUNT]; + +static void run(unsigned context, unsigned id, + struct ant_mesh_move_args *args) { + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + thread_tss_write(64, args, sizeof(*args)); + thread_load(context, THREAD_TLS, 0, 64, sizeof(*args)); + struct thread_task task = {id, + 0, + ant_image_end - ant_image, + base, + base + + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; + thread_start(context, &task); + int cancelled; + if (thread_wait(context, id, &cancelled) || cancelled) + __builtin_trap(); +} + +static struct thread_task prepare(unsigned context, unsigned id, + struct ant_mesh_move_args *args) { + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE); + thread_tss_write(64, args, sizeof(*args)); + thread_load(context, THREAD_TLS, 0, 64, sizeof(*args)); + return (struct thread_task){id, + 0, + ant_image_end - ant_image, + base, + base + + thread_query(context, THREAD_FIELD_TLS_BYTES), + THREAD_SIGNATURE}; +} + +static void exercise(unsigned tile) { + size_t bytes = ant_image_end - ant_image; + for (unsigned context = 0; context < 2; ++context) { + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, ant_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + } + for (unsigned i = 0; i < ROWS * ROW_WORDS; ++i) { + input[tile][i] = ((uint64_t)(tile + 1) << 56) | (0x100 + i); + output[tile][i] = 0; + } + uint64_t zero[2] = {0}; + thread_tss_write(0, zero, sizeof(zero)); + struct ant_mesh_move_args args = { + (uintptr_t)input[tile], (uintptr_t)output[tile], + thread_query(0, THREAD_FIELD_TSS_BASE), MESH_FILL}; + thread_acquire(); + run(0, tile * 3 + 1, &args); + args.role = MESH_MOVE; + struct thread_task move = prepare(0, tile * 3 + 2, &args); + args.role = MESH_CONSUME; + struct thread_task consume = prepare(1, tile * 3 + 3, &args); + thread_start(1, &consume); + thread_start(0, &move); + int cancelled; + if (thread_wait(0, move.id, &cancelled) || cancelled || + thread_wait(1, consume.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); + for (unsigned i = 0; i < ROWS * ROW_WORDS; ++i) + if (output[tile][i] != input[tile][i]) + __builtin_trap(); + __atomic_store_n(&done[tile], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart != BB_REPORT_HART) + exercise(hart - 1); + else + for (unsigned tile = 0; tile < BB_THREAD_CORE_COUNT; ++tile) + while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/runtime/ctests/CMakeLists.txt b/examples/designs/poly/workloads/device/runtime/ctests/CMakeLists.txt new file mode 100644 index 000000000..799aaa75f --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ctests/CMakeLists.txt @@ -0,0 +1,7 @@ +set(BUCKYBALL_CTEST_TARGET main) +set(BUCKYBALL_CTEST_GROUP_TARGET poly-ctests-build) + + +list(APPEND BUCKYBALL_DESIGN_COMMON_SOURCES + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c) +add_buckyball_ctest(corelink_lease.c) diff --git a/examples/designs/poly/workloads/device/runtime/ctests/corelink_lease.c b/examples/designs/poly/workloads/device/runtime/ctests/corelink_lease.c new file mode 100644 index 000000000..d20dc0b77 --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/ctests/corelink_lease.c @@ -0,0 +1,36 @@ +#include +#include +#include +#include +#include + +static const uint32_t input[2][4] __attribute__((aligned(64))) = { + {1, 2, 3, 4}, {0x12345678, 0xabcdef01, 7, 8}}; +static volatile uint32_t output[8] __attribute__((aligned(64))); + +int main(void) { + unsigned bank = BB_SHARED_BANK_BASE; + bb_mem_alloc(bank, 1, 2); + for (unsigned i = 0; i < 8; ++i) + output[i] = 0xaaaaaaaa; + for (unsigned group = 0; group < 2; ++group) + bb_mvin_group((uintptr_t)input[group], bank, group, 1, 1); + uint64_t address = corelink_shared_export(0, bank, 1); + if (corelink_shared_export(0, bank, 1) != address) + __builtin_trap(); + corelink_shared_release(0, bank, 1); + if (corelink_read64(address) != UINT64_C(0xabcdef0112345678)) + __builtin_trap(); + corelink_write64(address + 8, UINT64_C(0x76543210deadbeef)); + bb_mvout((uintptr_t)output, bank, 1, 1); + for (unsigned i = 0; i < 4; ++i) + if (output[i] != input[0][i]) + __builtin_trap(); + if (output[4] != input[1][0] || output[5] != input[1][1] || + output[6] != 0xdeadbeef || output[7] != 0x76543210) + __builtin_trap(); + corelink_shared_release(0, bank, 1); + bb_mem_release(bank); + puts("Main CoreLink lease and NPU shared storage PASS"); + return 0; +} diff --git a/examples/designs/poly/workloads/device/runtime/stream_main.c b/examples/designs/poly/workloads/device/runtime/stream_main.c new file mode 100644 index 000000000..bc431f5ca --- /dev/null +++ b/examples/designs/poly/workloads/device/runtime/stream_main.c @@ -0,0 +1,59 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +enum { M = 16, N = 16, K = 32 }; +static float a[M * K] __attribute__((aligned(4096))); +static float b[N * K] __attribute__((aligned(4096))); +static float output[M * N] __attribute__((aligned(4096))); +static void job(void *argument) { + (void)argument; + for (unsigned bank = 3; bank <= 5; ++bank) + bb_mem_alloc(bank, 1, 1); + bb_mvin((uintptr_t)a, 3, sizeof(a) / 16, 1); + // A DMA call has completed before the CPU is allowed to reuse its source. + for (unsigned i = 0; i < M * K; ++i) + a[i] = -1; + bb_mvin((uintptr_t)b, 4, sizeof(b) / 16, 1); + bb_mxmm_f32(3, 4, 5, M, N, K, 1, 1, 0); + bb_mvout((uintptr_t)output, 5, sizeof(output) / 16, 1); + for (unsigned row = 0; row < M; ++row) + for (unsigned col = 0; col < N; ++col) + if (output[row * N + col] != (float)(K * (row + 1) * (col + 1))) + abort(); + for (unsigned bank = 3; bank <= 5; ++bank) + bb_mem_release(bank); +} +// Runs on the core; the stream executes its Ball instructions on a hardware +// thread. +static int run(void *engine, void *argument) { + (void)engine; + (void)argument; + void *workspace = aligned_alloc(64, 1024 * 1024); + if (!workspace) + abort(); + workspace_init(workspace, 1024 * 1024); + workspace_begin(workspace, 1024 * 1024); + stream_run(THREAD_SIGNATURE, job, NULL); + free(workspace); + return 0; +} +int main(void) { + for (unsigned row = 0; row < M; ++row) + for (unsigned k = 0; k < K; ++k) + a[row * K + k] = row + 1; + for (unsigned col = 0; col < N; ++col) + for (unsigned k = 0; k < K; ++k) + b[col * K + k] = col + 1; + for (unsigned i = 0; i < M * N; ++i) + output[i] = -1; + if (runtime_linux_job(bb_cores[1].hart, bb_cores[1].signature, run, NULL)) + abort(); + puts("Stream DMA lifetime/Mxmm PASS"); + return 0; +} diff --git a/examples/designs/poly/workloads/device/soc/CMakeLists.txt b/examples/designs/poly/workloads/device/soc/CMakeLists.txt new file mode 100644 index 000000000..892630029 --- /dev/null +++ b/examples/designs/poly/workloads/device/soc/CMakeLists.txt @@ -0,0 +1,37 @@ +# Cross-core Linux control threads share one coherent LLC. +set(POLY_CONTROL_NAME + "${BUCKYBALL_WORKLOAD_DESIGN}-main-ctest-design-controllers" +) +set(POLY_CONTROL_ELF ${POLY_CONTROL_NAME}-multicore-baremetal) +buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/controllers.c") +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${POLY_CONTROL_ELF} + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${BBSIM_LD} + -I${BUCKYBALL_ISA_DIR}/design -I${BAREMETAL_DIR} -o ${POLY_CONTROL_ELF} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/controllers.c -lgcc + DEPENDS + ${POLY_BOOT_SOURCE} + controllers.c + ${BAREMETAL_DIR}/start.S + ${BBSIM_LD} + ${BUCKYBALL_ISA_DIR}/design/instances.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) +add_custom_target( + ${POLY_CONTROL_NAME}-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${POLY_CONTROL_ELF} +) +add_dependencies(poly-ctests-build ${POLY_CONTROL_NAME}-ctest-build) + +set(BUCKYBALL_CTEST_TARGET attention) +add_buckyball_linux_ctest(${POLY_CONTROL_NAME} ${CMAKE_CURRENT_SOURCE_DIR} controllers.c) +target_include_directories( + ${POLY_CONTROL_NAME}-linux + PRIVATE ${BUCKYBALL_ISA_DIR}/design +) +target_link_libraries(${POLY_CONTROL_NAME}-linux PRIVATE pthread) +add_dependencies(poly-ctests-build ${POLY_CONTROL_NAME}-linux-build) + +add_dependencies(soctests-build ${POLY_CONTROL_NAME}-ctest-build) diff --git a/examples/chips/poly/workloads/ctests/controllers.c b/examples/designs/poly/workloads/device/soc/controllers.c similarity index 92% rename from examples/chips/poly/workloads/ctests/controllers.c rename to examples/designs/poly/workloads/device/soc/controllers.c index 0779a2b9b..939c30c21 100644 --- a/examples/chips/poly/workloads/ctests/controllers.c +++ b/examples/designs/poly/workloads/device/soc/controllers.c @@ -1,6 +1,6 @@ #define _GNU_SOURCE +#include #include -#include #ifdef __linux__ #include #include @@ -9,7 +9,7 @@ #include #endif // Controllers publish cache-line payloads and contend on one atomic counter. -enum { CONTROLLERS = BB_MAIN_CORES + BB_COMPUTE_TILES, ROUNDS = 2 }; +enum { CONTROLLERS = BB_CORE_NUM, ROUNDS = 2 }; static uint64_t payload[CONTROLLERS][8] __attribute__((aligned(64))); static uint64_t counter __attribute__((aligned(64))); static uint64_t turn __attribute__((aligned(64))); @@ -70,11 +70,11 @@ int main(void) { for (unsigned hart = 0; hart < CONTROLLERS; ++hart) if (pthread_join(threads[hart], NULL)) return 6; - printf("Controllers SMP PASS cores=%u\n", (unsigned)CONTROLLERS); + printf("Controllers SMP PASS threads=%u\n", (unsigned)CONTROLLERS); return 0; #else uintptr_t hart; asm volatile("csrr %0, mhartid" : "=r"(hart)); - return exercise(hart); + return exercise(bb_hart_slot(hart)); #endif } diff --git a/examples/designs/toy/arch/build.sc b/examples/designs/toy/arch/build.sc new file mode 100644 index 000000000..1ef8185c8 --- /dev/null +++ b/examples/designs/toy/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.arch.src.main.scala.framework.device.cbus.{build => cbus_build} +import $file.^.^.^.{build => examples} +import $file.^.^.^.components.cores.rocket.arch.{build => rocket} +val designRoot: os.Path = millSourcePath / os.up + +object design extends SbtModule { + override def generatedSources = T(cbus_build.registry(sources() ++ rocket.library.sources(), T.dest)) + override def forkWorkingDir = T(designRoot / os.up / os.up / os.up / "arch") + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(examples.library, rocket.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(designRoot / "arch", designRoot / "configs") +} diff --git a/examples/designs/toy/arch/device/DeviceTop.scala b/examples/designs/toy/arch/device/DeviceTop.scala new file mode 100644 index 000000000..06901ebe8 --- /dev/null +++ b/examples/designs/toy/arch/device/DeviceTop.scala @@ -0,0 +1,72 @@ +package examples.designs.toy + +import framework.lib.arbiter.{RRArbiter} + +import chisel3._ +import chisel3.util._ +import framework.device.peripherals.bootrom.BootRomLines +import framework.device.peripherals.Controller +import framework.ip.bus.axi4.Decoder +import framework.ip.bus.chi.snf.LineResponse +import framework.lib.queue.{Queue => ResponseQueue} +import chisel3.experimental.hierarchy.{instantiable, Instance, Instantiate} +import framework.device.configs.DeviceParam +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import buckyball.config.Design +import _root_.configs.GlobalConfig +import framework.core.clink.CLinkIO +import framework.device.dlink.BaseDevice + +@instantiable +class DeviceTop(b: GlobalConfig, design: Design) extends BaseDevice(b.device.ddr.axi) { + val deviceParam = b.device + + val Seq(coreParam) = deviceParam.cores + + val bus = Instantiate(new CBus(b.device, CBus.generators(design, b.device, b.sim, CoreComponents.factories))) + dlink <> bus.dlink + + val p = deviceParam + val peripherals = + Instantiate(new Controller(deviceParam, deviceParam.ddr.axi, coreParam.memory.memoryAccess.memory)) + + val peripheralBus = Instantiate(new Decoder( + Seq( + (deviceParam.clint.base, deviceParam.clint.bytes), + (deviceParam.plic.base, deviceParam.plic.bytes), + (deviceParam.scu.baseAddress, deviceParam.scu.totalSizeBytes) + ), + deviceParam.ddr.axi + )) + + peripheralBus.io.in <> bus.uncached + peripherals.io.axi <> peripheralBus.io.matched + bus.external <> peripheralBus.io.other + peripherals.io.hardwareInterrupt.sources := 0.U + bus.interrupts := peripherals.io.hardwareInterrupt.harts + + val romParams = deviceParam.bootrom + val rom = Instantiate(new BootRomLines(deviceParam.bootrom, coreParam.memory.home.memory.chi)) + + val backing = bus.backing(0) + val ddr = bus.memory(0) + val inRom = backing.req.bits.addr >= romParams.base.U && + backing.req.bits.addr < (romParams.base + romParams.bytes).U + rom.io.request.valid := backing.req.valid && inRom + rom.io.request.bits := backing.req.bits + ddr.req.valid := backing.req.valid && !inRom + ddr.req.bits := backing.req.bits + backing.req.ready := Mux(inRom, rom.io.request.ready, ddr.req.ready) + + val responses = Instantiate(new RRArbiter(new LineResponse(coreParam.memory.home.memory.chi), 2)) + responses.io.in(0) <> rom.io.response + responses.io.in(1) <> ddr.resp + + // Buffer the selected answer so downstream backpressure cannot change its source. + val response = Instantiate(new ResponseQueue(new LineResponse(coreParam.memory.home.memory.chi), 1, false, false)) + + response.io.enq <> responses.io.out + backing.resp <> response.io.deq + +} diff --git a/examples/designs/toy/arch/device/core/ToyCore.scala b/examples/designs/toy/arch/device/core/ToyCore.scala new file mode 100644 index 000000000..8463667c5 --- /dev/null +++ b/examples/designs/toy/arch/device/core/ToyCore.scala @@ -0,0 +1,50 @@ +package examples.designs.toy + +import sims.configs.SimParam +import framework.core.configs.CoreParam +import framework.device.cbus.CoreFactory +import buckyball.config.{CorePlacement, Design} + +import examples.core.rocket.configs.Params +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import framework.core.clink.{CLinkIO, HasCLink} + +/** One RV64GC hart and its local NPU, explicitly wired behind CLink. */ +@instantiable +class ToyCore(config: Params, simp: SimParam) extends Module with HasCLink { + val diffTest = simp.diffTest + val corep = config + require(corep.acc.nonEmpty) + require(corep.clink.harts == 1 && corep.clink.backing) + + @public + val clink = IO(new CLinkIO(corep.clink)) + + val loom: Instance[Loom] = Instantiate(new Loom(corep)) + val acc: Instance[Acc] = Instantiate(new Acc(config, simp)) + + loom.io.rocc <> acc.io.rocc + loom.io.memory <> acc.io.memory + loom.io.interrupts := clink.interrupts(0) + clink.memory.uncached <> loom.io.uncached + clink.memory.backing.get <> loom.io.backing + clink.memory.ddr <> acc.io.ddr +} + +object Toy extends CoreFactory { + val component = "toy" + + def parameters(design: Design, source: CorePlacement): CoreParam = { + val native = Params.fromProto(source); CoreParam.fromProto(source, Seq.empty).copy(clink = native.clink) + } + + def instantiate( + source: CorePlacement, + corep: CoreParam, + simp: SimParam, + traceIds: Seq[Int] + ): CLinkIO = + Instantiate(new ToyCore(Params.fromProto(source), simp)).clink + +} diff --git a/examples/designs/toy/arch/device/core/acc.scala b/examples/designs/toy/arch/device/core/acc.scala new file mode 100644 index 000000000..9ae91cf6a --- /dev/null +++ b/examples/designs/toy/arch/device/core/acc.scala @@ -0,0 +1,114 @@ +package examples.designs.toy + +import framework.acc.balldomain.bbus.generated.BallComponents +import sims.configs.SimParam +import chisel3._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.rocket.configs.Params +import framework.core.memory.MemoryPort +import framework.acc.frontend.RoCCIO +import framework.ip.bus.axi4 +import framework.acc.balldomain.bbus.BBus +import framework.acc.frontend.Frontend +import framework.acc.memdomain.MemDomain +import framework.acc.gpdomain.GpDomain +import framework.acc.balldomain.BallDomain +import framework.acc.debug.Debug + +@instantiable +class Acc(config: Params, simp: SimParam) extends Module { + val diffTest = simp.diffTest + val corep = config + val acc = corep.acc.get + require(!acc.memDomain.sharedEnable) + + @public + val io = IO(new Bundle { + val rocc = new RoCCIO(corep.cpu.core.xLen, corep.permission) + val memory = new MemoryPort(corep.memory.dcache.tracking, corep.memory.memoryAccess.walker) + val ddr = new axi4.Port(acc.transport.axi) + }) + + val frontend: Instance[Frontend] = + Instantiate(new Frontend(acc.frontend, acc.memDomain, acc.gpDomain, acc.ballDomain, corep.permission, simp)) + + val memDomain: Instance[MemDomain] = Instantiate(new MemDomain( + acc.frontend, + acc.memDomain, + acc.gpDomain, + acc.transport, + corep.permission, + corep.memory.dcache.tracking, + corep.memory.memoryAccess.walker, + simp + )) + + val domain: Instance[BallDomain] = + Instantiate(new BallDomain( + acc.frontend, + acc.memDomain, + acc.gpDomain, + acc.ballDomain, + corep.permission, + BallComponents.factories + )) + + if (acc.gpDomain.enable) { + val gp: Instance[GpDomain] = Instantiate(new GpDomain(acc.frontend, acc.memDomain, acc.gpDomain)) + gp.io.command <> frontend.io.gpRequest.get + frontend.io.gpCompletion.get <> gp.io.complete + gp.io.image <> memDomain.io.image.get + memDomain.io.gpBanks.get <> gp.io.banks + memDomain.io.gpLayoutRequest.get <> gp.io.layoutRequest + gp.io.layoutResponse <> memDomain.io.gpLayoutResponse.get + domain.io.kernel.get.request <> gp.io.ball.request + gp.io.ball.response <> domain.io.kernel.get.response + } + frontend.io.rocc <> io.rocc + domain.io.issue <> frontend.io.ballRequest + frontend.io.ballCompletion <> domain.io.complete + memDomain.io.ball <> domain.io.memory + memDomain.io.issue <> frontend.io.memoryRequest + frontend.io.memoryCompletion <> memDomain.io.complete + memDomain.io.mapping <> frontend.io.mappingUpdate + frontend.io.barrier.release := frontend.io.barrier.arrive + memDomain.io.banks <> domain.io.banks + memDomain.io.ballLayoutRequest <> domain.io.layoutRequest + domain.io.layoutResponse <> memDomain.io.ballLayoutResponse + io.ddr <> memDomain.io.ddr + io.memory <> memDomain.io.memory + + val debug: Instance[Debug] = Instantiate(new Debug( + acc.frontend, + acc.memDomain, + acc.gpDomain, + acc.ballDomain, + simp, + traceId = corep.hartId, + sharedOwnerId = corep.hartId + )) + + debug.io.ant.valid := false.B + debug.io.ant.bits := 0.U.asTypeOf(debug.io.ant.bits) + debug.io.acc.frontend <> frontend.io.debug + debug.io.acc.trace.memory := memDomain.io.traceMemory + debug.io.acc.trace.ballStarted := domain.io.traceStarted + debug.io.acc.trace.ballCompleted := domain.io.traceCompleted + debug.io.acc.banks.zip(memDomain.io.bank_hashes).foreach { case (sink, source) => sink := source } + for (port <- memDomain.io.shared.bank.requests) { + port.read.req.ready := false.B + port.read.resp.valid := false.B + port.read.resp.bits := 0.U.asTypeOf(port.read.resp.bits) + port.write.req.ready := false.B + port.write.resp.valid := false.B + port.write.resp.bits := 0.U.asTypeOf(port.write.resp.bits) + } + memDomain.io.shared.bank.config.ready := false.B + memDomain.io.shared.bank.mapping.ready := false.B + memDomain.io.shared.bank.local.request.valid := false.B + memDomain.io.shared.bank.local.request.bits := 0.U.asTypeOf(memDomain.io.shared.bank.local.request.bits) + memDomain.io.shared.bank.local.response.ready := false.B + memDomain.io.shared.move.command.ready := false.B + memDomain.io.shared.move.completion.valid := false.B + memDomain.io.shared.move.completion.bits := false.B +} diff --git a/examples/designs/toy/arch/device/core/loom.scala b/examples/designs/toy/arch/device/core/loom.scala new file mode 100644 index 000000000..832e49cde --- /dev/null +++ b/examples/designs/toy/arch/device/core/loom.scala @@ -0,0 +1,48 @@ +package examples.designs.toy + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import examples.core.rocket.configs.{Params => RocketParam} +import examples.core.rocket.Execution +import framework.core.memory.{MemoryPort, MemorySystem} +import framework.acc.frontend.RoCCIO +import framework.core.clink.CoreInterrupts +import framework.ip.bus.axi4 +import framework.ip.bus.chi.snf.{LineRequest, LineResponse} + +/** Outer Loom execution core and private memory hierarchy; directly controls the local ACC. */ +@instantiable +class Loom(corep: RocketParam) extends Module { + require(corep.clink.backing) + + @public + val io = IO(new Bundle { + val interrupts = Input(new CoreInterrupts) + val rocc = Flipped(new RoCCIO(corep.cpu.core.xLen, corep.permission)) + val memory = Flipped(new MemoryPort(corep.memory.dcache.tracking, corep.memory.memoryAccess.walker)) + val uncached = new axi4.Port(corep.memory.uncached.axi) + + val backing = new Bundle { + val request = Decoupled(new LineRequest(corep.memory.home.memory.chi)) + val response = Flipped(Decoupled(new LineResponse(corep.memory.home.memory.chi))) + } + + }) + + val execution: Instance[Execution] = Instantiate(new Execution(corep, commandsEnabled = true)) + + val memory: Instance[MemorySystem] = Instantiate(new MemorySystem(corep.memory, externalHome = false)) + + execution.io.interrupts := io.interrupts + execution.io.rocc <> io.rocc + memory.io.fetch <> execution.io.fetch + memory.io.memory <> execution.io.data + memory.io.timeRequest <> execution.io.timeRequest + execution.io.timeResponse <> memory.io.timeResponse + memory.io.commandAllowed := !io.rocc.busy + memory.io.services <> io.memory + io.uncached <> memory.io.uncached + io.backing <> memory.io.backing.get + +} diff --git a/examples/designs/toy/configs/design.toml b/examples/designs/toy/configs/design.toml new file mode 100644 index 000000000..1d45ec7e4 --- /dev/null +++ b/examples/designs/toy/configs/design.toml @@ -0,0 +1,5 @@ +[device] +include = "device/default.toml" + +[sims] +include = "sims/default.toml" diff --git a/examples/designs/toy/configs/device/core/acc/balldomain/balls/smatmul/lane16.toml b/examples/designs/toy/configs/device/core/acc/balldomain/balls/smatmul/lane16.toml new file mode 100644 index 000000000..70527a327 --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/balldomain/balls/smatmul/lane16.toml @@ -0,0 +1,7 @@ +InputNum = 16 +inputWidth = 8 +lane = 16 +wsReuseCores = 2 +outputWidth = 32 +numMulThreads = 16 +numCasThreads = 16 diff --git a/examples/designs/toy/configs/device/core/acc/balldomain/default.toml b/examples/designs/toy/configs/device/core/acc/balldomain/default.toml new file mode 100644 index 000000000..4b5b830f8 --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/balldomain/default.toml @@ -0,0 +1,61 @@ +[[balls]] +id = 0 +name = "GemminiBall" +component = "gemmini" +param = "default" +port = { inBW = 2, outBW = 4, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "GEMMINI_LOOP_WS_CONFIG_BOUNDS", funct7 = 80 }, + { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_A", funct7 = 81 }, + { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_B", funct7 = 82 }, + { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_D", funct7 = 83 }, + { mnemonic = "GEMMINI_LOOP_WS_CONFIG_ADDR_C", funct7 = 84 }, + { mnemonic = "GEMMINI_LOOP_WS_CONFIG_STRIDES_AB", funct7 = 85 }, + { mnemonic = "GEMMINI_LOOP_WS_CONFIG_STRIDES_DC", funct7 = 86 }, + { mnemonic = "GEMMINI_LOOP_WS", funct7 = 87 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_1", funct7 = 96 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_2", funct7 = 97 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_3", funct7 = 98 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_4", funct7 = 99 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_5", funct7 = 100 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_6", funct7 = 101 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_7", funct7 = 102 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_8", funct7 = 103 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS_CONFIG_9", funct7 = 104 }, + { mnemonic = "GEMMINI_LOOP_CONV_WS", funct7 = 105 }, + { mnemonic = "GEMMINI_CONFIG", funct7 = 2 }, + { mnemonic = "GEMMINI_FLUSH", funct7 = 3 }, + { mnemonic = "GEMMINI_PRELOAD", funct7 = 21 }, + { mnemonic = "GEMMINI_COMPUTE_PRELOADED", funct7 = 66 }, + { mnemonic = "GEMMINI_COMPUTE_ACCUMULATED", funct7 = 67 }, +] + +[[balls]] +id = 1 +name = "TraceBall" +component = "trace" +param = "default" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "BDB_COUNTER", funct7 = 4 }, +] + +[[balls]] +id = 3 +name = "ReluBall" +component = "relu" +param = "default" +port = { inBW = 1, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "RELU", funct7 = 50 }, +] + +[[balls]] +id = 4 +name = "RedmuleBall" +component = "redmule" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "REDMULE_F16", funct7 = 68 }, +] diff --git a/examples/designs/toy/configs/device/core/acc/default.toml b/examples/designs/toy/configs/device/core/acc/default.toml new file mode 100644 index 000000000..5c3cf855f --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/default.toml @@ -0,0 +1,6 @@ +source = "../../../.." +balldomain = "balldomain/default.toml" +memdomain = "memdomain/toy.toml" +frontend = "frontend/toy.toml" +gpdomain = "gpdomain/toy.toml" +transport = "default" diff --git a/examples/designs/toy/configs/device/core/acc/demo.toml b/examples/designs/toy/configs/device/core/acc/demo.toml new file mode 100644 index 000000000..dc62522d8 --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/demo.toml @@ -0,0 +1,7 @@ +source = "../../../.." +# Toy ACC Configuration (demo frontend) + +balldomain = "balldomain/default.toml" +memdomain = "memdomain/toy.toml" +frontend = "frontend/demo.toml" +gpdomain = "gpdomain/toy.toml" diff --git a/examples/designs/toy/configs/device/core/acc/frontend/demo.toml b/examples/designs/toy/configs/device/core/acc/frontend/demo.toml new file mode 100644 index 000000000..c3e3d5e71 --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/frontend/demo.toml @@ -0,0 +1,9 @@ +bankAliasEnable = true +robEntries = 16 +rsOutOfOrderResponse = true +bankIdLen = 10 +vbankIdUpperBound = 15 +sharedBankIdBase = 32 +iterLen = 34 +subRobEnable = false +subRobDepth = 64 diff --git a/examples/designs/toy/configs/device/core/acc/frontend/toy.toml b/examples/designs/toy/configs/device/core/acc/frontend/toy.toml new file mode 100644 index 000000000..b5d20a710 --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/frontend/toy.toml @@ -0,0 +1,9 @@ +bankAliasEnable = false +robEntries = 16 +rsOutOfOrderResponse = true +bankIdLen = 10 +vbankIdUpperBound = 15 +sharedBankIdBase = 32 +iterLen = 34 +subRobEnable = true +subRobDepth = 4 diff --git a/examples/cores/toy/configs/rvv/default.toml b/examples/designs/toy/configs/device/core/acc/gpdomain/toy.toml similarity index 100% rename from examples/cores/toy/configs/rvv/default.toml rename to examples/designs/toy/configs/device/core/acc/gpdomain/toy.toml diff --git a/examples/designs/toy/configs/device/core/acc/memdomain/paper.toml b/examples/designs/toy/configs/device/core/acc/memdomain/paper.toml new file mode 100644 index 000000000..912791949 --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/memdomain/paper.toml @@ -0,0 +1,23 @@ +# Toy ACC Memory Domain Configuration (default) +# Private (per-thread) memory parameters. Shared memory lives in the core TOML. + +[bank] +layoutEnable = true +num = 16 +width = 128 +entries = 512 +maskLen = 16 +requestPorts = 7 + +[dma] +busWidth = 128 + +[mmio] +enable = false +bankNum = 16 +bankEntries = 64 +bankWidth = 8 +readWidth = 8 + +[mem] +addrLen = 39 diff --git a/examples/designs/toy/configs/device/core/acc/memdomain/toy.toml b/examples/designs/toy/configs/device/core/acc/memdomain/toy.toml new file mode 100644 index 000000000..69b64fc76 --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/memdomain/toy.toml @@ -0,0 +1,23 @@ +# Toy ACC Memory Domain Configuration (default) +# Private (per-thread) memory parameters. Shared memory lives in the core TOML. + +[bank] +layoutEnable = true +num = 16 +width = 128 +entries = 1024 +maskLen = 16 +requestPorts = 7 + +[dma] +busWidth = 128 + +[mmio] +enable = true +bankNum = 16 +bankEntries = 64 +bankWidth = 8 +readWidth = 8 + +[mem] +addrLen = 39 diff --git a/examples/designs/toy/configs/device/core/acc/paper.toml b/examples/designs/toy/configs/device/core/acc/paper.toml new file mode 100644 index 000000000..2d0e702c5 --- /dev/null +++ b/examples/designs/toy/configs/device/core/acc/paper.toml @@ -0,0 +1,7 @@ +source = "../../../.." +# Toy ACC Configuration (default) + +balldomain = "balldomain/default.toml" +memdomain = "memdomain/paper.toml" +frontend = "frontend/toy.toml" +gpdomain = "gpdomain/toy.toml" diff --git a/examples/designs/toy/configs/device/core/c2c/core.toml b/examples/designs/toy/configs/device/core/c2c/core.toml new file mode 100644 index 000000000..1284ec5d8 --- /dev/null +++ b/examples/designs/toy/configs/device/core/c2c/core.toml @@ -0,0 +1,3 @@ +enabled = false +peerCores = 1 +axi = "default" diff --git a/examples/designs/toy/configs/device/core/default.toml b/examples/designs/toy/configs/device/core/default.toml new file mode 100644 index 000000000..14ec3cb36 --- /dev/null +++ b/examples/designs/toy/configs/device/core/default.toml @@ -0,0 +1,39 @@ +component = "toy" +implementation = "rocket.toml" +accelerator = "acc/default.toml" +threads = [] +hartId = 0 +resetVector = 0x10000 +mtimeAddress = 0x200bff8 +memory = "memory/core.toml" +fetch = "default" +fpu = "default" +interconnect = "interconnect/core.toml" +c2c = "c2c/core.toml" +controller = "default" + +[param] +threadDataBytes = 64 +xLen = 64 +vaddrBits = 39 +paddrBits = 36 +pgIdxBits = 12 +pgLevels = 3 +nPMPs = 16 + +[sharedMem] +enable = false +entries = 16384 +inputChannels = 1 +defaultGroupCount = 1 +virtualBankCount = 16 +memCycles = 10 +bankEntries = 1024 +bankWidth = 128 + +[compiler] +name = "toy" +component = "toy" + +[build] +toy = "design" diff --git a/examples/designs/toy/configs/device/core/interconnect/core.toml b/examples/designs/toy/configs/device/core/interconnect/core.toml new file mode 100644 index 000000000..13d04a5a6 --- /dev/null +++ b/examples/designs/toy/configs/device/core/interconnect/core.toml @@ -0,0 +1,2 @@ +axi = "default" +slots = 16 diff --git a/examples/designs/toy/configs/device/core/memory/access/core.toml b/examples/designs/toy/configs/device/core/memory/access/core.toml new file mode 100644 index 000000000..bb87f76a4 --- /dev/null +++ b/examples/designs/toy/configs/device/core/memory/access/core.toml @@ -0,0 +1,52 @@ +translationEntries = 16 +accessTagBits = 6 +resultLineBits = 0 +[[regions]] +base = 0x80000000 +bytes = 0x20000000 +cacheable = true +executable = true +readable = true +writable = true +atomic = true +normal = true + +[[regions]] +base = 0x10000 +bytes = 0x10000 +cacheable = true +executable = true +readable = true +writable = false +atomic = false +normal = true + +[[regions]] +base = 0x2000000 +bytes = 0x10000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0xc000000 +bytes = 0x4000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false + +[[regions]] +base = 0x60000000 +bytes = 0x10000000 +cacheable = false +executable = false +readable = true +writable = true +atomic = false +normal = false diff --git a/examples/designs/toy/configs/device/core/memory/core.toml b/examples/designs/toy/configs/device/core/memory/core.toml new file mode 100644 index 000000000..3caf53579 --- /dev/null +++ b/examples/designs/toy/configs/device/core/memory/core.toml @@ -0,0 +1,5 @@ +uncached = "default" +access = "access/core.toml" +icache = "icache/core.toml" +dcache = "dcache/core.toml" +home = "home/core.toml" diff --git a/examples/designs/toy/configs/device/core/memory/dcache/core.toml b/examples/designs/toy/configs/device/core/memory/dcache/core.toml new file mode 100644 index 000000000..97b8e2bba --- /dev/null +++ b/examples/designs/toy/configs/device/core/memory/dcache/core.toml @@ -0,0 +1,15 @@ +[tracking] +entries = 8 +addressBits = 44 +idBits = 8 +lineBytes = 64 +maxRanges = 8 + +[cache] +nodeId = 2 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/toy/configs/device/core/memory/home/core.toml b/examples/designs/toy/configs/device/core/memory/home/core.toml new file mode 100644 index 000000000..3e0779cbc --- /dev/null +++ b/examples/designs/toy/configs/device/core/memory/home/core.toml @@ -0,0 +1,13 @@ +agents = 2 +mshrEntries = 8 +homeId = 64 +requesters = "default" + +[cache] +addressBits = 44 +lineBytes = 64 +sets = 64 +ways = 4 +idBits = 8 +metadataBits = 4 +responseDepth = 2 diff --git a/examples/designs/toy/configs/device/core/memory/icache/core.toml b/examples/designs/toy/configs/device/core/memory/icache/core.toml new file mode 100644 index 000000000..75e065643 --- /dev/null +++ b/examples/designs/toy/configs/device/core/memory/icache/core.toml @@ -0,0 +1,7 @@ +nodeId = 1 +cacheLines = 64 +homeId = 64 +homeCount = 1 +banks = 2 +lineResult = false +probe = false diff --git a/examples/designs/toy/configs/device/core/rocket.toml b/examples/designs/toy/configs/device/core/rocket.toml new file mode 100644 index 000000000..256ac66e0 --- /dev/null +++ b/examples/designs/toy/configs/device/core/rocket.toml @@ -0,0 +1,12 @@ +component = "rocket" +physical_address_bits = 36 +hart_id_bits = 1 +page_levels = 3 +pmp_entries = 16 +virtual_memory = true +supervisor = true +user = true +floating_point = true +atomics = true +multiply_divide = true +external_home = false diff --git a/examples/designs/toy/configs/device/ddr.toml b/examples/designs/toy/configs/device/ddr.toml new file mode 100644 index 000000000..31815e9e1 --- /dev/null +++ b/examples/designs/toy/configs/device/ddr.toml @@ -0,0 +1,13 @@ +clients = 1 +slotsPerClient = 16 +dataBits = 128 +idBits = 4 +idBase = 0 +base = 0x80000000 +bytes = 0x20000000 +model = 0 + +[line] +nodeIdBits = 7 +addressBits = 44 +dataBits = 256 diff --git a/examples/designs/toy/configs/device/default.toml b/examples/designs/toy/configs/device/default.toml new file mode 100644 index 000000000..7a1fb7525 --- /dev/null +++ b/examples/designs/toy/configs/device/default.toml @@ -0,0 +1,8 @@ +ddr = "ddr.toml" +bootrom = "default" +clint = "default" +plic = "default" +scu = "default" + +[[cores]] +include = "core/default.toml" diff --git a/examples/designs/toy/configs/sims/default.toml b/examples/designs/toy/configs/sims/default.toml new file mode 100644 index 000000000..0463ed6ae --- /dev/null +++ b/examples/designs/toy/configs/sims/default.toml @@ -0,0 +1,15 @@ +diffTest = false + +[memory] +lineBytes = 64 +clockHz = 1000000000 + +[targets.device] +config = "examples.designs.toy.sims.device.DeviceConfig" +stack_name = "toy" +regression = "sims/device/regression" +kernel = "sims/device/workloads/kernel" +tapeout = "sims/device/tapeout" + +[verification] +registry = "../../sims/ip/uvm/registry.toml" diff --git a/examples/designs/toy/emu/Cargo.lock b/examples/designs/toy/emu/Cargo.lock new file mode 100644 index 000000000..4469013c3 --- /dev/null +++ b/examples/designs/toy/emu/Cargo.lock @@ -0,0 +1,821 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "bebop-vm", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "rvsim", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-toy" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rvsim" +version = "0.1.0" +dependencies = [ + "rustc_apfloat", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/toy/emu/Cargo.toml b/examples/designs/toy/emu/Cargo.toml new file mode 100644 index 000000000..e2856f42b --- /dev/null +++ b/examples/designs/toy/emu/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] + +[package] +name = "bebop-design-toy" +version = "0.1.0" +edition = "2021" + +[dependencies] +bebop-bemu = { path = "../../../../bebop/src/nodes/bemu" } diff --git a/examples/designs/toy/emu/src/device/core/mod.rs b/examples/designs/toy/emu/src/device/core/mod.rs new file mode 100644 index 000000000..1658962cd --- /dev/null +++ b/examples/designs/toy/emu/src/device/core/mod.rs @@ -0,0 +1,39 @@ +use bebop_bemu::{ + ball::linkme, Accelerator, BemuProfileReport, CoreFactory, CoreModel, CoreParams, Cpu, DeviceModel, ModelOptions, + Whatever, CORES, +}; +use std::time::Duration; + +/// One RV64GC hart with its Ball accelerator on the RoCC port (arch: ToyCore). +pub struct ToyCore { + cpu: Cpu, + acc: Accelerator, +} + +impl CoreModel for ToyCore { + fn cpu(&self) -> &Cpu { + &self.cpu + } + fn cpu_mut(&mut self) -> &mut Cpu { + &mut self.cpu + } + fn step(&mut self, count: u64) -> Result<(), Whatever> { + self.cpu.step(count, &mut self.acc) + } + fn take_barrier(&mut self) -> bool { + self.acc.take_barrier() + } + fn profile_report(&self, total: Duration) -> Option { + self.acc.profile_report(total, &self.cpu) + } +} + +fn instantiate(device: &DeviceModel, params: &CoreParams, options: &ModelOptions) -> Result, String> { + let cpu = Cpu::new(device, params, options).map_err(|error| error.to_string())?; + let acc = Accelerator::attached(&cpu, options)?; + Ok(Box::new(ToyCore { cpu, acc })) +} + +#[linkme::distributed_slice(CORES)] +#[linkme(crate = bebop_bemu::ball::linkme)] +static TOY: CoreFactory = CoreFactory { component: "toy", instantiate }; diff --git a/examples/designs/toy/emu/src/device/mod.rs b/examples/designs/toy/emu/src/device/mod.rs new file mode 100644 index 000000000..a6ce9b1dc --- /dev/null +++ b/examples/designs/toy/emu/src/device/mod.rs @@ -0,0 +1,2 @@ +//! The toy device: its own Core model; the cores are instantiated by component on the CBus. +pub mod core; diff --git a/examples/designs/toy/emu/src/lib.rs b/examples/designs/toy/emu/src/lib.rs new file mode 100644 index 000000000..545892414 --- /dev/null +++ b/examples/designs/toy/emu/src/lib.rs @@ -0,0 +1 @@ +pub mod device; diff --git a/examples/designs/toy/sims/device/arch/DeviceConfig.scala b/examples/designs/toy/sims/device/arch/DeviceConfig.scala new file mode 100644 index 000000000..b8fc8ef57 --- /dev/null +++ b/examples/designs/toy/sims/device/arch/DeviceConfig.scala @@ -0,0 +1,18 @@ +package examples.designs.toy.sims.device + +import buckyball.config.Design +import chisel3.experimental.hierarchy.{Instance, Instantiate} +import examples.designs.toy.DeviceTop +import framework.device.cbus.CBus +import framework.device.cbus.generated.CoreComponents +import framework.device.dlink.BaseDevice +import _root_.configs.GlobalConfig +import _root_.sims.soc.SystemTarget +import java.nio.file.{Files, Paths} + +class DeviceConfig(design: Design) + extends SystemTarget( + GlobalConfig.fromProto(design, CBus.parameters(design, CoreComponents.factories)) + ) { + def instantiate(b: GlobalConfig): Instance[BaseDevice] = Instantiate(new DeviceTop(b, design)) +} diff --git a/examples/designs/toy/sims/device/arch/build.sc b/examples/designs/toy/sims/device/arch/build.sc new file mode 100644 index 000000000..5b94111b8 --- /dev/null +++ b/examples/designs/toy/sims/device/arch/build.sc @@ -0,0 +1,17 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.arch.{build => design_build} +import $file.^.^.^.^.^.^.arch.src.main.scala.sims.{build => sims_build} + +val entryRoot: os.Path = millSourcePath + +object simulation extends SbtModule { + override def millSourcePath = entryRoot + override def scalaVersion = "2.13.16" + override def moduleDeps = Seq(design_build.design, sims_build.library) + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") + override def sources = T.sources(entryRoot / "DeviceConfig.scala") + override def mainClass = Some("sims.verilator.Elaborate") +} diff --git a/examples/designs/toy/sims/device/emu/Cargo.lock b/examples/designs/toy/sims/device/emu/Cargo.lock new file mode 100644 index 000000000..eb97e3ba0 --- /dev/null +++ b/examples/designs/toy/sims/device/emu/Cargo.lock @@ -0,0 +1,1971 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "assert_cmd" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aa3a22042e45de04255c7bf3626e239f450200fd0493c1e382263544b20aea6" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bebop" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-fd-redirect", + "bebop-p2e", + "bebop-report", + "bebop-rtl-trace", + "bebop-terminal", + "bebop-trace-perfetto", + "bebop-uart", + "bebop-vcs", + "bebop-verilator", + "bebop-vm", + "camino", + "clap", + "duct", + "env_logger", + "libc", + "log", + "nix", + "serde", + "serde_json", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bank-hash" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "snafu", +] + +[[package]] +name = "bebop-bemu" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", + "bebop-bemu-profile", + "bebop-dasm", + "bebop-dtb", + "bebop-elf", + "bebop-keyboard", + "bebop-memory", + "bebop-microphone", + "bebop-rtc", + "bebop-speaker", + "bebop-syscall", + "bebop-uart", + "bebop-vga", + "clap", + "linkme", + "prost", + "prost-build", + "prost-types", + "rustc_apfloat", + "snafu", + "toml", +] + +[[package]] +name = "bebop-bemu-profile" +version = "0.1.0" + +[[package]] +name = "bebop-dasm" +version = "0.1.0" + +[[package]] +name = "bebop-design-toy" +version = "0.1.0" +dependencies = [ + "bebop-bemu", +] + +[[package]] +name = "bebop-dtb" +version = "0.1.0" + +[[package]] +name = "bebop-elf" +version = "0.1.0" +dependencies = [ + "bebop-memory", +] + +[[package]] +name = "bebop-fd-redirect" +version = "0.1.0" +dependencies = [ + "libc", +] + +[[package]] +name = "bebop-keyboard" +version = "0.1.0" + +[[package]] +name = "bebop-memory" +version = "0.1.0" + +[[package]] +name = "bebop-microphone" +version = "0.1.0" + +[[package]] +name = "bebop-p2e" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "bebop-rtl-trace", + "bebop-uart", + "duct", + "libc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bebop-report" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "bebop-rtc" +version = "0.1.0" + +[[package]] +name = "bebop-rtl-trace" +version = "0.1.0" +dependencies = [ + "bebop-bank-hash", +] + +[[package]] +name = "bebop-sim-build" +version = "0.1.0" +dependencies = [ + "prost", + "prost-build", + "prost-types", +] + +[[package]] +name = "bebop-sim-toy-device" +version = "0.1.0" +dependencies = [ + "assert_cmd", + "bebop", + "bebop-bemu", + "bebop-design-toy", + "bebop-report", + "bebop-sim-build", + "bebop-verilator", + "chrono", + "clap", + "libtest-mimic", + "predicates", + "serde", + "toml", + "walkdir", +] + +[[package]] +name = "bebop-speaker" +version = "0.1.0" + +[[package]] +name = "bebop-syscall" +version = "0.1.0" +dependencies = [ + "bebop-memory", + "once_cell", +] + +[[package]] +name = "bebop-terminal" +version = "0.1.0" +dependencies = [ + "crossterm", + "ratatui", +] + +[[package]] +name = "bebop-trace-perfetto" +version = "0.1.0" +dependencies = [ + "clap", + "serde_json", +] + +[[package]] +name = "bebop-uart" +version = "0.1.0" + +[[package]] +name = "bebop-vcs" +version = "0.1.0" +dependencies = [ + "snafu", +] + +[[package]] +name = "bebop-verilator" +version = "0.1.0" +dependencies = [ + "bebop-dasm", + "bebop-fd-redirect", + "bebop-rtl-trace", + "bebop-uart", + "libc", + "snafu", +] + +[[package]] +name = "bebop-vga" +version = "0.1.0" + +[[package]] +name = "bebop-vm" +version = "0.1.0" +dependencies = [ + "clap", + "font8x8", + "sdl2", + "vt100", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "camino" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbad30e4b4c14a39e3cc8aed085a12a327257c316619c93581e017bc52be591" + +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compact_str" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd622ebbb56a5b2ccb651b32b911cdeb2a9b4b11776b2473bf26a26a286244e" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.13.2", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix 0.38.44", + "signal-hook 0.3.18", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "duct" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "escape8259" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "font8x8" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875488b8711a968268c7cf5d139578713097ca4635a76044e8fe8eedf831d07e" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "indoc" +version = "2.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1" +dependencies = [ + "rustversion", +] + +[[package]] +name = "instability" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3b5acc1e2fd9375041a388da33d1eb8aed5f7a8c0dd3543e3ea2805adfbe20" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libtest-mimic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" +dependencies = [ + "anstream", + "anstyle", + "clap", + "escape8259", +] + +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "os_pipe" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "ratatui" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +dependencies = [ + "bitflags 2.13.2", + "cassowary", + "compact_str", + "crossterm", + "indoc", + "instability", + "itertools 0.13.0", + "lru", + "paste", + "strum", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.0", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags 2.13.2", + "smallvec", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdl2" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d42407afc6a8ab67e36f92e80b8ba34cbdc55aaeed05249efe9a2e8d0e9feef" +dependencies = [ + "bitflags 1.3.2", + "lazy_static", + "libc", + "sdl2-sys", +] + +[[package]] +name = "sdl2-sys" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff61407fc75d4b0bbc93dc7e4d6c196439965fbef8e4a4f003a36095823eac0" +dependencies = [ + "cfg-if", + "libc", + "pkg-config", + "version-compare", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "shared_child" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "607549934f6cc26b89cfecfdc46fa90f1e5d1536a68349b0c3a4f9d1c0d37959" +dependencies = [ + "libc", + "sigchld", + "windows-sys 0.61.2", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "sigchld" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f2b37f04360cd465089b87a9c3869c08220a2f3458463f0adf8badf5e77f2c" +dependencies = [ + "libc", + "os_pipe", + "signal-hook 0.4.5", +] + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b29ffa27d3d59a049bb328258de100f8012848d1b08d444e08bbc02ac468435e" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook 0.3.18", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix 1.1.5", + "windows-sys 0.61.2", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-truncate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.1.14", +] + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version-compare" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" + +[[package]] +name = "vt100" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de" +dependencies = [ + "itoa", + "log", + "unicode-width 0.1.14", + "vte", +] + +[[package]] +name = "vte" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +dependencies = [ + "arrayvec", + "utf8parse", + "vte_generate_state_changes", +] + +[[package]] +name = "vte_generate_state_changes" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/designs/toy/sims/device/emu/Cargo.toml b/examples/designs/toy/sims/device/emu/Cargo.toml new file mode 100644 index 000000000..c544bc315 --- /dev/null +++ b/examples/designs/toy/sims/device/emu/Cargo.toml @@ -0,0 +1,87 @@ +[workspace] + +[package] +name = "bebop-sim-toy-device" +version = "0.1.0" +edition = "2021" +autobins = false +autotests = false + +# The model of this simulation entry; the runners below are built from it. +[lib] +name = "simulation" + +[[bin]] +name = "bebop" +path = "../../../../../../bebop/src/main.rs" + +[[bin]] +name = "bebop-bemu" +path = "../../../../../../bebop/src/bin/bebop-bemu.rs" +required-features = ["bemu"] + +[[bin]] +name = "bebop-verilator" +path = "../../../../../../bebop/src/bin/bebop-verilator.rs" +required-features = ["verilator"] + +[[bin]] +name = "bebop-p2e" +path = "../../../../../../bebop/src/bin/bebop-p2e.rs" +required-features = ["p2e"] + +[[test]] +name = "test_bemu" +path = "../../../../../../bebop/tests/test_bemu.rs" +harness = false +required-features = ["bemu"] + +[[test]] +name = "test_verilator" +path = "../../../../../../bebop/tests/test_verilator.rs" +harness = false +required-features = ["verilator"] + +[[test]] +name = "test_p2e" +path = "../../../../../../bebop/tests/test_p2e.rs" +harness = false +required-features = ["p2e"] + +[[test]] +name = "instance_isolation" +path = "tests/instance_isolation.rs" + +[[bin]] +name = "bebop-vcs" +path = "../../../../../../bebop/src/bin/bebop-vcs.rs" + +[[test]] +name = "test_vcs" +path = "../../../../../../bebop/tests/test_vcs.rs" +harness = false + +[features] +bemu = ["bebop/bemu"] +verilator = ["bebop/verilator", "dep:bebop-verilator"] +p2e = ["bebop/p2e"] + +[dependencies] +bebop = { path = "../../../../../../bebop" } +bebop-verilator = { path = "../../../../../../bebop/src/nodes/verilator", optional = true } +bebop-bemu = { path = "../../../../../../bebop/src/nodes/bemu" } +bebop-design-toy = { path = "../../../emu" } +clap = { version = "4", features = ["derive"] } + +[build-dependencies] +bebop-sim-build = { path = "../../../../../../bebop/src/nodes/lib/sim-build" } + +[dev-dependencies] +bebop-report = { path = "../../../../../../bebop/src/nodes/lib/report" } +libtest-mimic = "0.8" +assert_cmd = "2" +walkdir = "2" +predicates = "3" +serde = { version = "1", features = ["derive"] } +toml = "0.8" +chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/examples/designs/toy/sims/device/emu/build.rs b/examples/designs/toy/sims/device/emu/build.rs new file mode 100644 index 000000000..d634d5dba --- /dev/null +++ b/examples/designs/toy/sims/device/emu/build.rs @@ -0,0 +1,7 @@ +use std::{env, path::PathBuf}; + +fn main() { + let entry = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + bebop_sim_build::balls(&entry.join("../../../output/generated/design.pb")); + bebop_sim_build::runner_links(); +} diff --git a/examples/designs/toy/sims/device/emu/src/lib.rs b/examples/designs/toy/sims/device/emu/src/lib.rs new file mode 100644 index 000000000..9684e5e32 --- /dev/null +++ b/examples/designs/toy/sims/device/emu/src/lib.rs @@ -0,0 +1,11 @@ +//! The toy device simulation. +//! Its Core models register in `bebop_bemu::CORES`; the Balls its design.pb +//! names are compiled here and register in `bebop_bemu::BALLS`. +extern crate bebop_design_toy; + +include!(concat!(env!("OUT_DIR"), "/balls.rs")); + +pub const SIMULATION: bebop_bemu::Simulation = bebop_bemu::Simulation { + design: include_bytes!("../../../../output/generated/design.pb"), + instantiate: bebop_bemu::device::cbus::instantiate, +}; diff --git a/examples/designs/toy/sims/device/emu/tests/instance_isolation.rs b/examples/designs/toy/sims/device/emu/tests/instance_isolation.rs new file mode 100644 index 000000000..866381f72 --- /dev/null +++ b/examples/designs/toy/sims/device/emu/tests/instance_isolation.rs @@ -0,0 +1,51 @@ +use bebop_bemu::{device::cbus, sim::run::Args, DeviceModel}; +use clap::Parser; +use std::{path::{Path, PathBuf}, process::Command}; + +fn run(command: &mut Command) { + assert!(command.status().expect("execute verification tool").success()); +} + +#[test] +fn independent_devices_keep_gemmini_configuration_and_memory_private() { + let manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let repo = manifest.ancestors().nth(6).expect("simulation entry location"); + let output = manifest.join("target/verification/instance-isolation"); + std::fs::create_dir_all(&output).unwrap(); + let isa = output.join("isa"); + run(Command::new("python3").arg(repo.join("stack/scripts/design.py")) + .arg("--design-pb").arg(repo.join("examples/designs/toy/output/generated/design.pb")) + .arg("--target").arg("device") + .arg("--out").arg(&output)); + let bare = repo.join("stack/os/baremetal"); + bebop_bemu::configs::load(simulation::SIMULATION.design); + let mut cpus = Vec::new(); + for name in ["gemmini_loop_subrob", "gemmini_loop_mmio"] { + let elf = output.join(format!("{name}.elf")); + run(Command::new("riscv64-none-elf-gcc") + .args(["-nostdlib", "-nostartfiles", "-O2", "-march=rv64gc", "-mabi=lp64d", + "-mcmodel=medany", "-msmall-data-limit=0", "-fno-builtin", "-Wl,--no-relax"]) + .arg("-T").arg(bare.join("bbsim.ld")) + .arg("-I").arg(isa.join("toy")) + .arg("-I").arg(repo.join("bb-tests/workloads/lib")) + .arg("-I").arg(repo.join("examples/components/balls/gemmini/workloads")) + .arg(bare.join("crt0.S")) + .arg(repo.join(format!("examples/components/balls/gemmini/workloads/ctests/{name}.c"))) + .arg("-o").arg(&elf)); + let log = output.join(name); + let args = Args::try_parse_from([Path::new("test"), Path::new("--elf"), &elf, + Path::new("--log-dir"), &log]).unwrap(); + let device = DeviceModel::new(512 * 1024 * 1024, bebop_bemu::hart_capacity()); + let mut instances = cbus::instantiate(&device, &args).unwrap(); + assert_eq!(instances.len(), 1); + let mut cpu = instances.pop().unwrap(); + cpu.cpu_mut().load_elf(&elf).unwrap(); + cpu.cpu_mut().init_firmware(None, None).unwrap(); + cpus.push(cpu); + } + for _ in 0..2_000_000 { + if cpus.iter().all(|cpu| cpu.cpu().finished()) { break; } + for cpu in &mut cpus { if !cpu.cpu().finished() { cpu.step(1).unwrap(); } } + } + for cpu in cpus { assert_eq!(cpu.cpu().exit_code(), Some(0)); } +} diff --git a/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml new file mode 100644 index 000000000..285d0e7a2 --- /dev/null +++ b/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml @@ -0,0 +1,56 @@ +[workloads] +tests = [ + "toy-toy-ctest-redmule_f16_test-baremetal", + "toy-toy-ctest-redmule_f16_chain_test-baremetal", + "toy-toy-ctest-redmule_f16_edges_test-baremetal", + "toy-toy-ctest-redmule_f16_fused_test-baremetal", + "toy-toy-mlirtest-ball_redmule_f16_chain-baremetal", + "toy-toy-mlirtest-bank_redmule_f16_rows-baremetal", + + "toy-toy-mlirtest-base_dma_group-baremetal", + "toy-toy-mlirtest-base_mset_transfer-baremetal", + "toy-toy-ctest-mvin_mvout_group_test-baremetal", + "toy-toy-ctest-mset_transfer_test-baremetal", + "toy-toy-ctest-rvv_ball-baremetal", + "toy-toy-ctest-rvv_kernel-baremetal", + "toy-toy-ctest-mvin_2d_pixel_stride_test-baremetal", + "toy-toy-ctest-gemmini_accumulated_16x16_test-baremetal", + "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-baremetal", + "toy-toy-mlirtest-bank_gemmini_conv_tail-baremetal", + "toy-toy-mlirtest-bank_gemmini_conv_direct_weight-baremetal", + "toy-toy-mlirtest-bank_gemmini_conv_round-baremetal", + "toy-toy-ctest-mvin_2d_test-baremetal", + "toy-toy-ctest-mvin_mvout_acc_test-baremetal", + "toy-toy-ctest-mvin_mvout_test-baremetal", + "toy-toy-ctest-mvin_mvout_stride_test-baremetal", + "toy-toy-ctest-mvin_mvout_unaligned_test-baremetal", + "toy-toy-ctest-mvin_unaligned_test-baremetal", + "toy-toy-ctest-bdb_counter_start_stop_test-baremetal", + "toy-toy-ctest-bdb_counter_read_active_test-baremetal", + "toy-toy-ctest-bdb_counter_nested_pair_test-baremetal", + "toy-toy-ctest-bdb_counter_multi_id_test-baremetal", + "toy-toy-mlirtest-ball_bdb_counter_start_read_stop-baremetal", + "toy-toy-ctest-gemmini_os_cisc_basic_test-baremetal", + "toy-toy-ctest-gemmini_os_cisc_atranspose_test-baremetal", + "toy-toy-ctest-gemmini_os_cisc_btranspose_test-baremetal", + "toy-toy-ctest-gemmini_os_cisc_shift_test-baremetal", + "toy-toy-ctest-gemmini_os_risc_atranspose_test-baremetal", + "toy-toy-ctest-gemmini_os_risc_btranspose_test-baremetal", + "toy-toy-ctest-gemmini_os_risc_abtranspose_test-baremetal", + "toy-toy-ctest-gemmini_os_risc_shift_test-baremetal", + "toy-toy-ctest-gemmini_ws_cisc_conv_test-baremetal", + "toy-toy-ctest-gemmini_ws_risc_basic_test-baremetal", + "toy-toy-ctest-relu_i32_16x16_test-baremetal", + "toy-toy-mlirtest-ball_relu_i32_16row-baremetal", + "toy-toy-mlirtest-bank_relu_i32_mixed_sign-baremetal", + "toy-toy-mlirtest-bank_gemmini_compute_preloaded_bank_tiles-baremetal", + "toy-toy-mlirtest-bank_gemmini_compute_accumulated_bank_tiles-baremetal", + "toy-toy-mlirtest-ball_gemmini_preloaded_command_16x16-baremetal", + "toy-toy-ctest-tlb_bare_mode_test-baremetal", + "toy-toy-ctest-tlb_sv39_identity_test-baremetal", + "toy-toy-ctest-tlb_sfence_flush_test-baremetal", + "toy-toy-ctest-tlb_repeated_sfence_test-baremetal", + "toy-toy-ctest-tlb_sv39_nonidentity_test-baremetal", + "toy-toy-ctest-tlb_sv39_4k_alias_test-baremetal", + "toy-toy-ctest-tlb_sfence_remap_test-baremetal", +] diff --git a/examples/chips/toy/regression/batch/bemu/workloads-linux.toml b/examples/designs/toy/sims/device/regression/bemu/workloads-linux.toml similarity index 100% rename from examples/chips/toy/regression/batch/bemu/workloads-linux.toml rename to examples/designs/toy/sims/device/regression/bemu/workloads-linux.toml diff --git a/examples/designs/toy/sims/device/regression/p2e/workloads-bare.toml b/examples/designs/toy/sims/device/regression/p2e/workloads-bare.toml new file mode 100644 index 000000000..7f6c30521 --- /dev/null +++ b/examples/designs/toy/sims/device/regression/p2e/workloads-bare.toml @@ -0,0 +1,24 @@ +[workloads] +tests = [ + "toy-toy-ctest-gemmini_accumulated_16x16_test-baremetal.hex", + "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-baremetal.hex", + "toy-toy-mlirtest-bank_gemmini_conv_tail-baremetal.hex", + "toy-toy-mlirtest-bank_gemmini_conv_direct_weight-baremetal.hex", + "toy-toy-mlirtest-bank_gemmini_conv_round-baremetal.hex", + "toy-toy-ctest-mvin_mvout_test-baremetal.hex", + "toy-toy-ctest-mvin_mvout_stride_test-baremetal.hex", + "toy-toy-ctest-bdb_counter_start_stop_test-baremetal.hex", + "toy-toy-ctest-bdb_counter_read_active_test-baremetal.hex", + "toy-toy-ctest-bdb_counter_nested_pair_test-baremetal.hex", + "toy-toy-ctest-bdb_counter_multi_id_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_cisc_basic_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_cisc_atranspose_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_cisc_btranspose_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_cisc_shift_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_risc_atranspose_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_risc_btranspose_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_risc_abtranspose_test-baremetal.hex", + "toy-toy-ctest-gemmini_os_risc_shift_test-baremetal.hex", + "toy-toy-ctest-gemmini_ws_cisc_conv_test-baremetal.hex", + "toy-toy-ctest-gemmini_ws_risc_basic_test-baremetal.hex", +] diff --git a/examples/chips/toy/regression/batch/p2e/workloads-linux.toml b/examples/designs/toy/sims/device/regression/p2e/workloads-linux.toml similarity index 100% rename from examples/chips/toy/regression/batch/p2e/workloads-linux.toml rename to examples/designs/toy/sims/device/regression/p2e/workloads-linux.toml diff --git a/arch/src/main/scala/framework/root/root-chip/src/csrc/README.md b/examples/designs/toy/sims/device/regression/soc/README.md similarity index 100% rename from arch/src/main/scala/framework/root/root-chip/src/csrc/README.md rename to examples/designs/toy/sims/device/regression/soc/README.md diff --git a/examples/designs/toy/sims/device/regression/verilator/workloads-bare.toml b/examples/designs/toy/sims/device/regression/verilator/workloads-bare.toml new file mode 100644 index 000000000..0073d1782 --- /dev/null +++ b/examples/designs/toy/sims/device/regression/verilator/workloads-bare.toml @@ -0,0 +1,42 @@ +[workloads] +tests = [ + "toy-toy-ctest-redmule_f16_test-baremetal", + "toy-toy-ctest-redmule_f16_chain_test-baremetal", + "toy-toy-ctest-redmule_f16_edges_test-baremetal", + "toy-toy-ctest-redmule_f16_fused_test-baremetal", + "toy-toy-mlirtest-ball_redmule_f16_chain-baremetal", + + "toy-toy-ctest-rvv_kernel-baremetal", + "toy-toy-ctest-mvin_2d_pixel_stride_test-baremetal", + "toy-toy-ctest-gemmini_accumulated_16x16_test-baremetal", + "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-baremetal", + "toy-toy-mlirtest-bank_gemmini_conv_tail-baremetal", + "toy-toy-mlirtest-bank_gemmini_conv_direct_weight-baremetal", + "toy-toy-mlirtest-bank_gemmini_conv_round-baremetal", + "toy-toy-ctest-mvin_2d_test-baremetal", + "toy-toy-ctest-mvin_mvout_acc_test-baremetal", + "toy-toy-ctest-mvin_mvout_test-baremetal", + "toy-toy-ctest-mvin_mvout_unaligned_test-baremetal", + "toy-toy-ctest-mvin_unaligned_test-baremetal", + "toy-toy-ctest-bdb_counter_start_stop_test-baremetal", + "toy-toy-ctest-bdb_counter_read_active_test-baremetal", + "toy-toy-ctest-bdb_counter_nested_pair_test-baremetal", + "toy-toy-ctest-bdb_counter_multi_id_test-baremetal", + "toy-toy-mlirtest-ball_bdb_counter_start_read_stop-baremetal", + "toy-toy-ctest-gemmini_os_cisc_basic_test-baremetal", + "toy-toy-ctest-gemmini_os_cisc_atranspose_test-baremetal", + "toy-toy-ctest-gemmini_os_cisc_btranspose_test-baremetal", + "toy-toy-ctest-gemmini_os_cisc_shift_test-baremetal", + "toy-toy-ctest-gemmini_ws_cisc_conv_test-baremetal", + "toy-toy-ctest-relu_i32_16x16_test-baremetal", + "toy-toy-mlirtest-ball_relu_i32_16row-baremetal", + "toy-toy-mlirtest-ball_gemmini_preloaded_command_16x16-baremetal", + "toy-toy-ctest-tlb_bare_mode_test-baremetal", + "toy-toy-ctest-tlb_sv39_identity_test-baremetal", + "toy-toy-ctest-tlb_sfence_flush_test-baremetal", + "toy-toy-ctest-tlb_repeated_sfence_test-baremetal", + "toy-toy-ctest-tlb_sv39_nonidentity_test-baremetal", + "toy-toy-ctest-tlb_sv39_4k_alias_test-baremetal", + "toy-toy-ctest-tlb_sfence_remap_test-baremetal", + "toy-toy-ctest-mvin_mvout_stride_test-baremetal", +] diff --git a/examples/designs/toy/sims/device/regression/verilator/workloads-linux.toml b/examples/designs/toy/sims/device/regression/verilator/workloads-linux.toml new file mode 100644 index 000000000..88c2741bc --- /dev/null +++ b/examples/designs/toy/sims/device/regression/verilator/workloads-linux.toml @@ -0,0 +1,32 @@ +[workloads] +tests = [ + "toy-toy-ctest-gemmini_accumulated_16x16_test-linux", + "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-linux", + "toy-toy-mlirtest-bank_gemmini_conv_tail-linux", + "toy-toy-mlirtest-bank_gemmini_conv_direct_weight-linux", + "toy-toy-mlirtest-bank_gemmini_conv_round-linux", + "toy-toy-ctest-mvin_2d_test-linux", + "toy-toy-ctest-mvin_mvout_acc_test-linux", + "toy-toy-ctest-mvin_mvout_test-linux", + "toy-toy-ctest-mvin_mvout_stride_test-linux", + "toy-toy-ctest-mvin_mvout_unaligned_test-linux", + "toy-toy-ctest-mvin_unaligned_test-linux", + "toy-toy-ctest-bdb_counter_start_stop_test-linux", + "toy-toy-ctest-bdb_counter_read_active_test-linux", + "toy-toy-ctest-bdb_counter_nested_pair_test-linux", + "toy-toy-ctest-bdb_counter_multi_id_test-linux", + "toy-toy-mlirtest-ball_bdb_counter_start_read_stop-linux", + "toy-toy-ctest-gemmini_os_cisc_basic_test-linux", + "toy-toy-ctest-gemmini_os_cisc_atranspose_test-linux", + "toy-toy-ctest-gemmini_os_cisc_btranspose_test-linux", + "toy-toy-ctest-gemmini_os_cisc_shift_test-linux", + "toy-toy-ctest-gemmini_os_risc_atranspose_test-linux", + "toy-toy-ctest-gemmini_os_risc_btranspose_test-linux", + "toy-toy-ctest-gemmini_os_risc_abtranspose_test-linux", + "toy-toy-ctest-gemmini_os_risc_shift_test-linux", + "toy-toy-ctest-gemmini_ws_cisc_conv_test-linux", + "toy-toy-ctest-gemmini_ws_risc_basic_test-linux", + "toy-toy-ctest-relu_i32_16x16_test-linux", + "toy-toy-mlirtest-ball_relu_i32_16row-linux", + "toy-toy-mlirtest-ball_gemmini_preloaded_command_16x16-linux", +] diff --git a/examples/designs/toy/sims/device/regression/workloads-verilator.toml b/examples/designs/toy/sims/device/regression/workloads-verilator.toml new file mode 100644 index 000000000..c90097753 --- /dev/null +++ b/examples/designs/toy/sims/device/regression/workloads-verilator.toml @@ -0,0 +1,5 @@ +[workloads] +tests = [ + "toy-rocket-ctest-gemmini_loop_subrob-baremetal", + "toy-rocket-ctest-gemmini_loop_mmio-baremetal", +] diff --git a/examples/designs/toy/sims/device/stack/CMakeLists.txt b/examples/designs/toy/sims/device/stack/CMakeLists.txt new file mode 100644 index 000000000..b077e0e8e --- /dev/null +++ b/examples/designs/toy/sims/device/stack/CMakeLists.txt @@ -0,0 +1,37 @@ +# ===----------------------------------------------------------------------=== +# The software stack of the toy device simulation: every level the device +# needs, built bottom up. +# +# bbdev stack --build "--design toy" +# +# configures this with DESIGN_CMAKE, generated from design.pb. Each level is a +# CMake build of its own: cores and threads build buddy-opt plugins for the +# host, the device builds RISC-V libraries. +# core/toy libtoy-compiler.so, execute.o +# device runtime, runtime-baremetal, runtime-linux, engine; device.cmake +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(toy-device-stack NONE) +include(ExternalProject) +include("${DESIGN_CMAKE}") + +# Toy's CPU core with its Ball accelerator. +ExternalProject_Add(core-toy + SOURCE_DIR "${DESIGN_CORE_STACK_toy}" + BINARY_DIR "${DESIGN_CORE_BUILD_toy}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" -DCORE=toy + BUILD_ALWAYS ON + INSTALL_COMMAND "") + +# The device: the runtime that schedules jobs onto the cores. +ExternalProject_Add(device + SOURCE_DIR "${DESIGN_DEVICE_STACK}" + BINARY_DIR "${DESIGN_DEVICE_BUILD}" + CMAKE_GENERATOR Ninja + CMAKE_ARGS "-DDESIGN_CMAKE=${DESIGN_CMAKE}" + -DCMAKE_C_COMPILER=riscv64-unknown-linux-gnu-gcc + -DCMAKE_CXX_COMPILER=riscv64-unknown-linux-gnu-g++ + DEPENDS core-toy + BUILD_ALWAYS ON + INSTALL_COMMAND "") diff --git a/examples/designs/toy/sims/device/stack/core/toy/CMakeLists.txt b/examples/designs/toy/sims/device/stack/core/toy/CMakeLists.txt new file mode 100644 index 000000000..da36ffb78 --- /dev/null +++ b/examples/designs/toy/sims/device/stack/core/toy/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.20) +project(toy-core-stack LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +add_subdirectory("${DESIGN_REPO}/examples/designs/toy/stack/engine" engine) +add_subdirectory("${DESIGN_REPO}/examples/designs/toy/stack/midend" midend) diff --git a/examples/designs/toy/sims/device/stack/runtime/CMakeLists.txt b/examples/designs/toy/sims/device/stack/runtime/CMakeLists.txt new file mode 100644 index 000000000..995b2c8bb --- /dev/null +++ b/examples/designs/toy/sims/device/stack/runtime/CMakeLists.txt @@ -0,0 +1,37 @@ +# ===----------------------------------------------------------------------=== +# The toy device: its cores, their hardware threads and the runtime that +# schedules jobs onto the cores. +# +# bbdev stack --build "--design toy" +# +# builds every thread and core stack of the device first, then configures this +# with DESIGN_CMAKE, generated from design.pb, and the RISC-V Linux toolchain. +# Products, exported in device.cmake for the programs that run on the device: +# runtime, runtime-baremetal, runtime-linux job scheduling over the cores +# engine hardware threads and instruction streams +# bindings how each core runs jobs and each +# hardware thread is loaded +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(toy-device LANGUAGES C CXX) +include("${DESIGN_CMAKE}") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# How each core runs jobs, and how each hardware thread is loaded. Nothing +# references these bindings, so they are linked as objects rather than taken +# from an archive; as link options they come before the libraries they call. +add_library(bindings INTERFACE) +foreach(CORE IN LISTS DESIGN_CORES) + target_link_options(bindings INTERFACE "${DESIGN_CORE_BUILD_${CORE}}/execute.o") +endforeach() +foreach(THREAD IN LISTS DESIGN_THREADS) + target_link_options(bindings INTERFACE "${DESIGN_THREAD_BUILD_${THREAD}}/loader.o") +endforeach() +# DMA helpers are sized by the target with the fewest bank lines. +list(GET DESIGN_RUNTIME_TARGETS 0 PARAMS_TARGET) +set(RUNTIME_PARAMS "${DESIGN_ISA_DIR}/${PARAMS_TARGET}") + +add_subdirectory("${DESIGN_REPO}/stack/engine" engine) +add_subdirectory("${DESIGN_REPO}/stack/runtime" runtime) +export(TARGETS bindings runtime runtime-baremetal runtime-linux engine + runtime-shared runtime-linux-shared engine-shared FILE device.cmake) diff --git a/examples/designs/toy/sims/device/tapeout/area/constraints.sdc b/examples/designs/toy/sims/device/tapeout/area/constraints.sdc new file mode 100644 index 000000000..b34468521 --- /dev/null +++ b/examples/designs/toy/sims/device/tapeout/area/constraints.sdc @@ -0,0 +1,8 @@ +set clock_port [get_ports {clock clk_i}] +if {[sizeof_collection $clock_port] != 1} { error "synthesis top must have one clock or clk_i input" } +create_clock -name core -period 10.0 $clock_port +set_clock_uncertainty -setup 1.0 [get_clocks core] +set_clock_transition 0.5 [get_clocks core] +set data_inputs [remove_from_collection [all_inputs] $clock_port] +set_input_delay 2.0 -clock core $data_inputs +set_output_delay 2.0 -clock core [all_outputs] diff --git a/examples/designs/toy/sims/device/tapeout/area/dc.tcl b/examples/designs/toy/sims/device/tapeout/area/dc.tcl new file mode 100644 index 000000000..7833680ca --- /dev/null +++ b/examples/designs/toy/sims/device/tapeout/area/dc.tcl @@ -0,0 +1,67 @@ +proc run_synthesis {} { + global RUN_CONFIG target_library synthetic_library link_library search_path +# Design-owned DC flow. Timing comes from constraints.sdc (RUN_SDC). +# bbdev injects RUN_* via -x {set RUN_CONFIG }. +if {![info exists RUN_CONFIG] || $RUN_CONFIG eq ""} { + error "bbdev must pass -x {set RUN_CONFIG }" +} +source [file normalize $RUN_CONFIG] + +proc bbdev_read_filelist {path} { + set fh [open $path r] + set files [list] + while {[gets $fh line] >= 0} { + set line [string trim $line] + if {$line eq "" || [string match "#*" $line]} { continue } + lappend files [file normalize $line] + } + close $fh + if {[llength $files] == 0} { error "empty DC source list: $path" } + return $files +} + +set top $RUN_TOP +set output_dir [file normalize $RUN_OUTPUT_DIR] +set report_dir [file normalize $RUN_REPORT_DIR] +file mkdir $output_dir $report_dir [file join $report_dir work] +set target_library [list $RUN_TARGET_LIBRARY] +set synthetic_library $RUN_SYNTHETIC_LIBRARY +set link_library [concat [list *] $target_library $synthetic_library $RUN_LINK_LIBRARY] +set_host_options -max_cores $RUN_MAX_CORES +define_design_lib WORK -path [file join $report_dir work] +set search_path [list . [file dirname $RUN_SOURCE_LIST] [file dirname [file dirname $RUN_SOURCE_LIST]] [file dirname [file dirname [file dirname $RUN_SOURCE_LIST]]]] +set_app_var verilogout_no_tri true +set_app_var verilogout_equation false +set_app_var compile_seqmap_propagate_constants false +set_app_var hdlin_ff_always_sync_set_reset true +analyze -format sverilog -define {SYNTHESIS DC_SYN} [bbdev_read_filelist $RUN_SOURCE_LIST] +elaborate $top +current_design $top +link +set sram_cells [get_cells -hierarchical -quiet -filter {ref_name =~ "chipyard_sram_*"}] +if {[sizeof_collection $sram_cells] > 0} { set_dont_touch $sram_cells } +set sram_designs [get_designs -quiet chipyard_sram_*] +if {[sizeof_collection $sram_designs] > 0} { + set_dont_touch $sram_designs +} +source [file normalize $RUN_SDC] +if {[sizeof_collection [all_clocks]] == 0} { + error "SDC created no clocks" +} +set_load 2.0 [all_outputs] +set_app_var compile_seqmap_honor_sync_set_reset true +compile_ultra -area_high_effort_script -no_autoungroup -no_boundary_optimization +set_fix_multiple_port_nets -all -buffer_constants +change_names -hierarchy -rules verilog +write -format ddc -hierarchy -output [file join $output_dir ${top}.ddc] +write -format verilog -hierarchy -output [file join $output_dir ${top}.v] +write_sdc [file join $output_dir ${top}.sdc] +report_constraint -all_violators > [file join $report_dir constraint.rpt] +report_timing -delay max -max_paths 50 > [file join $report_dir timing_max.rpt] +report_timing -delay min -max_paths 50 > [file join $report_dir timing_min.rpt] +report_area -hierarchy > [file join $report_dir area.rpt] +report_reference > [file join $report_dir reference.rpt] + +} +if {[catch {run_synthesis} reason]} { puts stderr $reason; exit 1 } +exit diff --git a/examples/designs/toy/sims/device/tapeout/config.toml b/examples/designs/toy/sims/device/tapeout/config.toml new file mode 100644 index 000000000..1eef9db6b --- /dev/null +++ b/examples/designs/toy/sims/device/tapeout/config.toml @@ -0,0 +1,24 @@ +[tapeout] +top = "RedmuleBall" +sram_process = "smic180" +sram_mdf = "ip/smic180_sram_library.mdf.json" +constraints_sdc = "area/constraints.sdc" +target_library = "/home/bb-runner/Code/eda/D_libraries/SMIC180/SCC018UG_UHD_RVT_V0.4a/SCC018UG_UHD_RVT_V0p4a/liberty/1.8v/scc018ug_uhd_rvt_ss_v1p62_125c_ccs.db" +synthetic_library = [ + "/home/bb-runner/Code/eda/A_packages/tools/syn/R-2020.09-SP5/libraries/syn/dw_foundation.sldb", + "/home/bb-runner/Code/eda/A_packages/tools/syn/R-2020.09-SP5/libraries/syn/standard.sldb", +] +link_library = [] +max_threads = 8 +power_format = "vcd" +power_start_ns = "1000000" +power_end_ns = "5000000" +dc_script = "area/dc.tcl" +power_script = "../../../../../bbdev/api/steps/dc/scripts/ptpx.tcl" +power_strip_path = "GatePowerHarness.dut" + +sram_corner = "ss" +lc_shell = "/home/bb-runner/Code/eda/A_packages/tools/lc/R-2020.09-SP5/bin/lc_shell" + + +sram = [] diff --git a/examples/designs/toy/sims/device/tapeout/ip/smic180_sram_library.mdf.json b/examples/designs/toy/sims/device/tapeout/ip/smic180_sram_library.mdf.json new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/examples/designs/toy/sims/device/tapeout/ip/smic180_sram_library.mdf.json @@ -0,0 +1 @@ +[] diff --git a/examples/designs/toy/sims/device/workloads/CMakeLists.txt b/examples/designs/toy/sims/device/workloads/CMakeLists.txt new file mode 100644 index 000000000..ebf202d4c --- /dev/null +++ b/examples/designs/toy/sims/device/workloads/CMakeLists.txt @@ -0,0 +1,75 @@ +include("${DESIGN_DEVICE_BUILD}/device.cmake") + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/base" + "${CMAKE_BINARY_DIR}/src/CTest/base" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tlb" + "${CMAKE_BINARY_DIR}/src/CTest/tlb" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/tutorial" + "${CMAKE_BINARY_DIR}/src/CTest/tutorial" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/embench" + "${CMAKE_BINARY_DIR}/src/CTest/embench" +) +add_subdirectory( + "${WORKLOAD_SRC_DIR}/CTest/coremark" + "${CMAKE_BINARY_DIR}/src/CTest/coremark" +) +foreach(BUCKYBALL_COMPONENT_WORKLOAD_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(COMPONENT_WORKLOADS + "${DESIGN_TARGET_WORKLOADS_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" + ) + if( + COMPONENT_WORKLOADS + STREQUAL + "${BUCKYBALL_REPO_ROOT}/examples/designs/${BUCKYBALL_WORKLOAD_DESIGN}/workloads" + ) + continue() + endif() + set(BUCKYBALL_COMPONENT_WORKLOAD_PROFILE + "${DESIGN_TARGET_ID_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" + ) + set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}") + set(BUCKYBALL_CTEST_GROUP_TARGET component-ctests-build) + if(IS_DIRECTORY "${COMPONENT_WORKLOADS}/ctests") + add_subdirectory( + "${COMPONENT_WORKLOADS}/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/components/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" + ) + endif() +endforeach() +add_dependencies( + ctest-workloads-build + base-ctests-build + tlb-ctests-build + tutorial-ctests-build + component-ctests-build +) +add_dependencies(sync-native-bin embench-all coremark-all) + +add_subdirectory( + "${WORKLOAD_SRC_DIR}/MLIRTest/base" + "${CMAKE_BINARY_DIR}/src/MLIRTest/mlir_tests/base" +) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_MLIR_ACTIVE_TARGET "${BUCKYBALL_TARGET}") + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_TARGET_BALLS_${BUCKYBALL_TARGET}) + set(BALL_MLIR_TESTS + "${BUCKYBALL_REPO_ROOT}/examples/components/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" + ) + if(IS_DIRECTORY "${BALL_MLIR_TESTS}") + add_subdirectory( + "${BALL_MLIR_TESTS}" + "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}" + ) + endif() + endforeach() +endforeach() +add_dependencies(MLIRTest-all base-mlirtest-build balls-mlir-tests-build) + +add_subdirectory("${BUCKYBALL_REPO_ROOT}/examples/designs/toy/workloads" design) diff --git a/examples/designs/toy/sims/device/workloads/kernel/CMakeLists.txt b/examples/designs/toy/sims/device/workloads/kernel/CMakeLists.txt new file mode 100644 index 000000000..5e7eb8f61 --- /dev/null +++ b/examples/designs/toy/sims/device/workloads/kernel/CMakeLists.txt @@ -0,0 +1,46 @@ +# Linux programs installed into the toy rootfs, each with the hart it runs on. +set(BUCKYBALL_KERNEL_PROGRAMS + 0 toy-toy-ctest-redmule_f16_test-linux + 0 toy-toy-ctest-redmule_f16_chain_test-linux + 0 toy-toy-ctest-redmule_f16_edges_test-linux + 0 toy-toy-ctest-redmule_f16_fused_test-linux + 0 toy-toy-mlirtest-ball_redmule_f16_chain-linux + 0 toy-toy-mlirtest-bank_redmule_f16_rows-linux + 0 toy-toy-mlirtest-base_dma_group-linux + 0 toy-toy-mlirtest-base_mset_transfer-linux + 0 toy-toy-ctest-mvin_mvout_group_test-linux + 0 toy-toy-ctest-mset_transfer_test-linux + 0 toy-toy-ctest-rvv_ball-linux + 0 toy-toy-ctest-rvv_kernel-linux + 0 toy-toy-ctest-gemmini_accumulated_16x16_test-linux + 0 toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-linux + 0 toy-toy-mlirtest-bank_gemmini_conv_tail-linux + 0 toy-toy-mlirtest-bank_gemmini_conv_direct_weight-linux + 0 toy-toy-mlirtest-bank_gemmini_conv_round-linux + 0 toy-toy-ctest-mvin_2d_test-linux + 0 toy-toy-ctest-mvin_mvout_acc_test-linux + 0 toy-toy-ctest-mvin_mvout_test-linux + 0 toy-toy-ctest-mvin_mvout_stride_test-linux + 0 toy-toy-ctest-mvin_mvout_unaligned_test-linux + 0 toy-toy-ctest-mvin_unaligned_test-linux + 0 toy-toy-ctest-bdb_counter_start_stop_test-linux + 0 toy-toy-ctest-bdb_counter_read_active_test-linux + 0 toy-toy-ctest-bdb_counter_nested_pair_test-linux + 0 toy-toy-ctest-bdb_counter_multi_id_test-linux + 0 toy-toy-mlirtest-ball_bdb_counter_start_read_stop-linux + 0 toy-toy-ctest-gemmini_os_cisc_basic_test-linux + 0 toy-toy-ctest-gemmini_os_cisc_atranspose_test-linux + 0 toy-toy-ctest-gemmini_os_cisc_btranspose_test-linux + 0 toy-toy-ctest-gemmini_os_cisc_shift_test-linux + 0 toy-toy-ctest-gemmini_os_risc_atranspose_test-linux + 0 toy-toy-ctest-gemmini_os_risc_btranspose_test-linux + 0 toy-toy-ctest-gemmini_os_risc_abtranspose_test-linux + 0 toy-toy-ctest-gemmini_os_risc_shift_test-linux + 0 toy-toy-ctest-gemmini_ws_cisc_conv_test-linux + 0 toy-toy-ctest-gemmini_ws_risc_basic_test-linux + 0 toy-toy-ctest-relu_i32_16x16_test-linux + 0 toy-toy-mlirtest-ball_relu_i32_16row-linux + 0 toy-toy-mlirtest-bank_relu_i32_mixed_sign-linux + 0 toy-toy-mlirtest-bank_gemmini_compute_preloaded_bank_tiles-linux + 0 toy-toy-mlirtest-bank_gemmini_compute_accumulated_bank_tiles-linux + PARENT_SCOPE) diff --git a/examples/designs/toy/sims/ip/uvm/configs/axis16.toml b/examples/designs/toy/sims/ip/uvm/configs/axis16.toml new file mode 100644 index 000000000..a5e2badc6 --- /dev/null +++ b/examples/designs/toy/sims/ip/uvm/configs/axis16.toml @@ -0,0 +1,6 @@ +[parameters] +dataBits = 16 +inputs = 2 + +[defines] +AXIS_DATA_BITS = "dataBits" diff --git a/examples/designs/toy/sims/ip/uvm/registry.toml b/examples/designs/toy/sims/ip/uvm/registry.toml new file mode 100644 index 000000000..6992c6ebe --- /dev/null +++ b/examples/designs/toy/sims/ip/uvm/registry.toml @@ -0,0 +1,7 @@ +[[ips]] +ip = "axis" +config = "default" + +[[ips]] +ip = "axis" +config = "configs/axis16.toml" diff --git a/examples/designs/toy/stack/engine/CMakeLists.txt b/examples/designs/toy/stack/engine/CMakeLists.txt new file mode 100644 index 000000000..686343604 --- /dev/null +++ b/examples/designs/toy/stack/engine/CMakeLists.txt @@ -0,0 +1,13 @@ +# Core execution binding; instruction streams use the framework engine. +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +set(ENGINE "${DESIGN_REPO}/stack/engine") +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/execute.o" + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" + -c "${RUNTIME}/execute/direct.c" -o "${CMAKE_BINARY_DIR}/execute.o" + DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" + VERBATIM) +add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") diff --git a/examples/designs/toy/stack/midend/CMakeLists.txt b/examples/designs/toy/stack/midend/CMakeLists.txt new file mode 100644 index 000000000..7d445ba9a --- /dev/null +++ b/examples/designs/toy/stack/midend/CMakeLists.txt @@ -0,0 +1,11 @@ +# Host compiler plugin for this design's core. +set(ENGINE "${DESIGN_REPO}/stack/engine") +set(TARGET ${DESIGN_CORE_TARGET_${CORE}}) + +set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${TARGET}}) +set(BUCKYBALL_TARGET_TABLE "${DESIGN_ISA_DIR}/${TARGET}/target.inc") + +add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) +add_library(${TARGET}-compiler MODULE $) +set_target_properties(${TARGET}-compiler PROPERTIES + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") diff --git a/examples/designs/toy/workloads/CMakeLists.txt b/examples/designs/toy/workloads/CMakeLists.txt new file mode 100644 index 000000000..a545bb368 --- /dev/null +++ b/examples/designs/toy/workloads/CMakeLists.txt @@ -0,0 +1,17 @@ +get_filename_component(BUCKYBALL_REPO_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../.. ABSOLUTE) + +add_custom_target(toy-ctests-build) +set(BUCKYBALL_CTEST_GROUP_TARGET toy-ctests-build) +set(BUCKYBALL_CTEST_TARGET toy) + +add_subdirectory(device) + +add_custom_target(design-ctests-build DEPENDS toy-ctests-build) + +add_custom_target(design-workloads-build ALL + DEPENDS + design-ctests-build + mlirtest-workloads-build + soctests-build + COMMENT "Building toy workloads" + VERBATIM) diff --git a/examples/designs/toy/workloads/configs/tests.toml b/examples/designs/toy/workloads/configs/tests.toml new file mode 100644 index 000000000..c1c7ceb33 --- /dev/null +++ b/examples/designs/toy/workloads/configs/tests.toml @@ -0,0 +1,2 @@ +controlHart = 0 +reportHart = 0 diff --git a/examples/designs/toy/workloads/device/CMakeLists.txt b/examples/designs/toy/workloads/device/CMakeLists.txt new file mode 100644 index 000000000..679d879e4 --- /dev/null +++ b/examples/designs/toy/workloads/device/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(soc) +add_subdirectory(core) diff --git a/examples/designs/toy/workloads/device/core/CMakeLists.txt b/examples/designs/toy/workloads/device/core/CMakeLists.txt new file mode 100644 index 000000000..2a6ad3b6a --- /dev/null +++ b/examples/designs/toy/workloads/device/core/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(acc) diff --git a/examples/designs/toy/workloads/device/core/acc/CMakeLists.txt b/examples/designs/toy/workloads/device/core/acc/CMakeLists.txt new file mode 100644 index 000000000..7b5637b41 --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/CMakeLists.txt @@ -0,0 +1,10 @@ +add_subdirectory(ctests) +foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) + set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_TARGET}") + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_TARGET_BALLS_${BUCKYBALL_TARGET}) + add_subdirectory( + "${BUCKYBALL_REPO_ROOT}/examples/components/balls/${BUCKYBALL_BALL_DIR}/workloads/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}/ctests") + endforeach() +endforeach() +add_subdirectory(mlir_tests) diff --git a/examples/designs/toy/workloads/device/core/acc/ctests/CMakeLists.txt b/examples/designs/toy/workloads/device/core/acc/ctests/CMakeLists.txt new file mode 100644 index 000000000..b89676936 --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/ctests/CMakeLists.txt @@ -0,0 +1,22 @@ +add_subdirectory(gemmini_relu_trace_pipeline) +set(BUCKYBALL_CTEST_TARGET toy) +set(BUCKYBALL_CTEST_GROUP_TARGET toy-ctests-build) +file( + RELATIVE_PATH + RVV_KERNEL_SOURCE + ${CMAKE_CURRENT_SOURCE_DIR} + ${WORKLOAD_SRC_DIR}/CTest/rvv/rvv_kernel.c +) +add_buckyball_ctest(${RVV_KERNEL_SOURCE}) +set(RVV_KERNEL_TEST + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_kernel" +) +add_dependencies(toy-ctests-build ${RVV_KERNEL_TEST}-ctest-build) + +add_buckyball_ctest(rvv_ball.c) +set(RVV_BALL_TEST + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_ball" +) +add_dependencies(toy-ctests-build ${RVV_BALL_TEST}-ctest-build) + +add_buckyball_ctest(mmio_minimal_test.c) diff --git a/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/CMakeLists.txt b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/CMakeLists.txt new file mode 100644 index 000000000..3686171da --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/CMakeLists.txt @@ -0,0 +1,3 @@ +add_buckyball_component_workload(gemmini_relu_trace_pipeline SOURCES + gemmini_relu_trace_pipeline-main.cpp + toy.c) diff --git a/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp new file mode 100644 index 000000000..1a553d265 --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp @@ -0,0 +1,22 @@ +#include +extern "C" int toy_pipeline(uint32_t instance); +static volatile uint32_t completed, failed; +int main() { + const uint32_t hart = bb_get_hart_id(); + const uint32_t expected = + bb_profile_core_count(BB_COMPONENT_WORKLOAD_PROFILE); + if (!expected) + return 1; + if (bb_hart_profile(hart) == BB_COMPONENT_WORKLOAD_PROFILE) { + __atomic_fetch_or(&failed, toy_pipeline(bb_hart_slot(hart)) != 0, + __ATOMIC_RELEASE); + __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); + } + if (hart == BB_REPORT_HART) { + while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) + asm volatile("nop"); + return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; + } + for (;;) + asm volatile("wfi"); +} diff --git a/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/toy.c b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/toy.c similarity index 90% rename from examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/toy.c rename to examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/toy.c index 4ce602537..ed565c654 100644 --- a/examples/cores/toy/workloads/ctests/gemmini_relu_trace_pipeline/toy.c +++ b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/toy.c @@ -4,11 +4,11 @@ #include #include #include -#include +#include #include #if BANK_WIDTH != 128 || BANK_LINES < 64 || BANK_LINES % 64 != 0 -#error "Toy core pipeline requires 128-bit banks and depth divisible by 64" +#error "Toy ACC pipeline requires 128-bit banks and depth divisible by 64" #endif enum { DIM = 16 }; @@ -20,8 +20,8 @@ static int32_t output[DIM * DIM] __attribute__((aligned(64))); #ifdef __cplusplus extern "C" #endif - int toy_core(core_id_t id) { - (void)id; + int toy_pipeline(uint32_t instance) { + (void)instance; for (int row = 0; row < DIM; ++row) for (int col = 0; col < DIM; ++col) { input[row * DIM + col] = (row * 3 + col * 5) % 15 - 7; diff --git a/examples/chips/toy/workloads/ctests/mmio_minimal_test.c b/examples/designs/toy/workloads/device/core/acc/ctests/mmio_minimal_test.c similarity index 94% rename from examples/chips/toy/workloads/ctests/mmio_minimal_test.c rename to examples/designs/toy/workloads/device/core/acc/ctests/mmio_minimal_test.c index 17ab11398..6a931afce 100644 --- a/examples/chips/toy/workloads/ctests/mmio_minimal_test.c +++ b/examples/designs/toy/workloads/device/core/acc/ctests/mmio_minimal_test.c @@ -16,8 +16,8 @@ static uint8_t test_data[16] __attribute__((aligned(16))) = { 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10}; int main(void) { -#ifdef MULTICORE - multicore(MULTICORE); +#ifdef MULTITHREAD + multithread(MULTITHREAD); #endif printf("=== Minimal MMIO Test ===\n"); @@ -46,7 +46,7 @@ int main(void) { printf("MMIO test PASSED\n"); -#ifdef MULTICORE +#ifdef MULTITHREAD exit(0); #endif diff --git a/examples/cores/toy/workloads/ctests/rvv_ball.c b/examples/designs/toy/workloads/device/core/acc/ctests/rvv_ball.c similarity index 78% rename from examples/cores/toy/workloads/ctests/rvv_ball.c rename to examples/designs/toy/workloads/device/core/acc/ctests/rvv_ball.c index a340ed361..c234c4378 100644 --- a/examples/cores/toy/workloads/ctests/rvv_ball.c +++ b/examples/designs/toy/workloads/device/core/acc/ctests/rvv_ball.c @@ -13,13 +13,13 @@ int main(void) { input[i] = ((int)i - 64) * 7; bb_mem_alloc(0, 1, 1); bb_mem_alloc(2, 1, 2); - struct kernel_launch call - __attribute__((aligned(16))) = {0, - 12, - 0x40002000, - {BB_BANK0(2) | BB_ITER(16), BANK_LINES, - BB_BANK0(3) | BB_ITER(16), BANK_LINES}, - 0}; + struct kernel_launch call __attribute__((aligned(16))) = { + 0, + 12, + 0x40002000, + {BB_BANK0(2) | BB_BANK2(2) | BB_ITER(16), BANK_LINES, + BB_BANK0(3) | BB_BANK2(3) | BB_ITER(16), BANK_LINES}, + 0}; bb_mvin((uintptr_t)input, 2, 16, 1); bb_mvin((uintptr_t)&call, 0, sizeof(call) / 16, 1); bb_mem_alloc(4, 1, 1); diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/CMakeLists.txt b/examples/designs/toy/workloads/device/core/acc/mlir_tests/CMakeLists.txt new file mode 100644 index 000000000..a859647b8 --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/CMakeLists.txt @@ -0,0 +1,8 @@ +add_custom_target(toy-mlir-tests-build) + +add_subdirectory(linalg) + +add_dependencies(toy-mlir-tests-build + linalg-mlir-tests-build) + +add_dependencies(design-mlirtests-build toy-mlir-tests-build) diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt new file mode 100644 index 000000000..f193aaaf9 --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt @@ -0,0 +1,27 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET linalg-mlir-tests-build) +set(BUCKYBALL_MLIR_TEST_PREFIX linalg) + +add_custom_target(linalg-mlir-tests-build) + +set(TOY_LINALG_FUNCTIONAL_PASSES + -convert-linalg-to-buckyball + -lower-buckyball-to-bank-ssa + -assign-physical-banks + ${BUCKYBALL_LOWER_BUCKYBALL} + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + ${BUCKYBALL_LOWER_BUCKYBALL} + -convert-arith-to-llvm + -convert-math-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +foreach(test conv2d_nhwc_hwcf conv2d_nhwc_fhwc conv2d_nchw_fchw) + add_buckyball_mlir_test(${test} + TARGET toy + PASSES ${TOY_LINALG_FUNCTIONAL_PASSES}) +endforeach() diff --git a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nchw_fchw.mlir b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw.mlir similarity index 100% rename from examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nchw_fchw.mlir rename to examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw.mlir diff --git a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nchw_fchw_main.cpp b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw_main.cpp similarity index 100% rename from examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nchw_fchw_main.cpp rename to examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw_main.cpp diff --git a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_fhwc.mlir b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc.mlir similarity index 100% rename from examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_fhwc.mlir rename to examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc.mlir diff --git a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_fhwc_main.cpp b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc_main.cpp similarity index 100% rename from examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_fhwc_main.cpp rename to examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc_main.cpp diff --git a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_hwcf.mlir b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf.mlir similarity index 100% rename from examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_hwcf.mlir rename to examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf.mlir diff --git a/examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_hwcf_main.cpp b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_main.cpp similarity index 100% rename from examples/chips/toy/workloads/mlir_tests/linalg/conv2d_nhwc_hwcf_main.cpp rename to examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_main.cpp diff --git a/examples/designs/toy/workloads/device/soc/CMakeLists.txt b/examples/designs/toy/workloads/device/soc/CMakeLists.txt new file mode 100644 index 000000000..f100f3903 --- /dev/null +++ b/examples/designs/toy/workloads/device/soc/CMakeLists.txt @@ -0,0 +1,5 @@ +add_buckyball_ctest(scu_uart_rx_echo_test.c) +set(SCU_UART_TEST + "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-scu_uart_rx_echo_test" +) +add_dependencies(soctests-build ${SCU_UART_TEST}-baremetal-build) diff --git a/examples/chips/toy/workloads/ctests/scu_uart_rx_echo_test.c b/examples/designs/toy/workloads/device/soc/scu_uart_rx_echo_test.c similarity index 100% rename from examples/chips/toy/workloads/ctests/scu_uart_rx_echo_test.c rename to examples/designs/toy/workloads/device/soc/scu_uart_rx_echo_test.c diff --git a/flake.nix b/flake.nix index 485d56454..6b11fed03 100644 --- a/flake.nix +++ b/flake.nix @@ -21,10 +21,12 @@ }; defaultShell = pkgs.mkShell { + nativeBuildInputs = [ pkgs.pkg-config ]; # this is needed to build LLVM/libc++, disable the nix injected hardening hardeningDisable = [ "libcxxhardeningfast" ]; buildInputs = with pkgs; [ + SDL2 clibs.zlib-dev clibs.zlib clibs.lz4-dev @@ -43,13 +45,16 @@ clibs.libdwarf compiler.flatbuffers + compiler.tomlplusplus compiler.numactl rustTools.cargoNextest - # protoc for bbdev config --install (chip.proto -> chip.pb) + # protoc for bbdev config --install (design.proto -> design.pb) pkgs.protobuf + tools.espresso pkgs.zsh + pkgs.gersemi pkgs.xorg-server pkgs.jdk8 @@ -74,9 +79,6 @@ # Verilator build acceleration: ccache via OBJCACHE export OBJCACHE=ccache - export CUDA_HOME="${pkgs.cuda.cudatoolkit}" - export CPATH="''${CUDA_HOME}/include''${CPATH:+:$CPATH}" - export CC="${pkgs.systemTools.clang}/bin/clang" export CXX="${pkgs.systemTools.clang}/bin/clang++" @@ -127,6 +129,7 @@ name = "buckyball-environment"; paths = with pkgs; [ tools.verilator + tools.espresso tools.dramsim3 tools.ccache tools.lld @@ -147,6 +150,7 @@ # python environment python.python3Packages pkgs."pre-commit" + pkgs.gersemi # clang-format for pre-commit (language: system) pkgs.clang-tools (pkgs.runCommand "clang-kernel" {} '' @@ -173,6 +177,7 @@ kernel.e2fsprogs # C libraries (headers + link libs) + pkgs.SDL2 clibs.zlib-dev clibs.zlib clibs.lz4-dev @@ -192,6 +197,7 @@ # Compiler tools compiler.flatbuffers + compiler.tomlplusplus compiler.numactl # Scala tools @@ -212,21 +218,12 @@ systemTools.git systemTools.pnpm - # CUDA toolkit (12.8, matches host driver) + host g++-13 - cuda.cudatoolkit - cuda.gcc13 ]; }; # nix develop devShells.default = defaultShell; - devShells.vm = pkgs.mkShell { - inputsFrom = [ defaultShell ]; - nativeBuildInputs = [ pkgs.pkg-config ]; - packages = [ pkgs.SDL2 ]; - }; - # default + EDA tools / sky130 devShells.full = pkgs.mkShell { inputsFrom = [ defaultShell ]; diff --git a/scripts/claude/README.md b/scripts/claude/README.md index 797d71d06..7b34a43aa 100644 --- a/scripts/claude/README.md +++ b/scripts/claude/README.md @@ -26,11 +26,11 @@ User / Agent host ## Daily path ```text -bbdev_compiler_build(chip) +bbdev_stack_build(design, thread, core, device) → bbdev_task_status(trace_id) until success - → bbdev_workload_build(chip) + → bbdev_workload_build(design) → bbdev_task_status(trace_id) until success - → bbdev_bemu_sim(chip, binary) + → bbdev_bemu_sim(design, binary) → bbdev_task_status(trace_id) until success → bbdev_bebop_verilator_run(binary, config) → bbdev_task_status(trace_id) until success @@ -43,13 +43,13 @@ UVM when needed: `bbdev_uvm_build` / `bbdev_uvm_run`. ### Validation | Tool | Purpose | |------|---------| -| `validate(chip, balldomain?)` | Chip balldomain TOML invariants (`ballIdMappings` / `ballISA`) | +| `validate(design, balldomain?)` | Design balldomain TOML invariants (`ballIdMappings` / `ballISA`) | ### bbdev wrappers (all POST APIs) | Tool | API | |------|-----| | `bbdev_config_install` | `/config/install` | -| `bbdev_compiler_build` | `/compiler/build` | +| `bbdev_stack_build` | `/stack/build` | | `bbdev_task_status` | State for a submitted `trace_id` | | `bbdev_workload_{clean,build,tohex}` | `/workload/{clean,build,tohex}` | | `bbdev_bemu_{sim,batch}` | `/bebop/bemu/{sim,batch}` | @@ -88,7 +88,7 @@ the current MCP server; any other unknown trace fails. | Trigger | Skill | |---------|-------| | `/ball-align` | `.agents/skills/ball-align` | -| `/chip-designer` | `.agents/skills/chip-designer` | +| `/design-designer` | `.agents/skills/design-designer` | | `/verify ` | `.agents/skills/verify` | | `/check` | `.agents/skills/check` | | `/waveform` | `.agents/skills/waveform` | diff --git a/scripts/nix/build-all.sh b/scripts/nix/build-all.sh index 2e0503646..093669b05 100755 --- a/scripts/nix/build-all.sh +++ b/scripts/nix/build-all.sh @@ -129,17 +129,17 @@ fi if run_step "2"; then begin_step "2" "Compiler installation" cd "$BBDIR" - bbdev compiler --build '--chip toy' + bbdev stack --build '--design toy' fi if run_step "3"; then begin_step "3" "RTL source pre-compile" - bbdev verilator --verilog '--chip toy' + bbdev verilator --verilog '--design toy' fi if run_step "4"; then begin_step "4" "bb-tests pre-compile sources" - bbdev workload --build '--chip toy' + bbdev workload --build '--design toy' fi if run_step "5"; then diff --git a/scripts/nix/build-env-compiler.nix b/scripts/nix/build-env-compiler.nix index 322c5124a..0d9d4d688 100644 --- a/scripts/nix/build-env-compiler.nix +++ b/scripts/nix/build-env-compiler.nix @@ -4,6 +4,9 @@ # FlatBuffers - serialization library and compiler flatbuffers = pkgs.flatbuffers; + # Header-only TOML library; device programs read their model description with it + tomlplusplus = pkgs.tomlplusplus; + # NUMA (Non-Uniform Memory Access) library numactl = pkgs.numactl; } diff --git a/scripts/nix/build-env-cuda.nix b/scripts/nix/build-env-cuda.nix deleted file mode 100644 index bce669723..000000000 --- a/scripts/nix/build-env-cuda.nix +++ /dev/null @@ -1,21 +0,0 @@ -{ pkgs }: - -let - # Host driver reports CUDA 12.8 (570.x); do not use default 12.9+. - cudaPkgs = pkgs.cudaPackages_12_8; -in -{ - cudatoolkit = cudaPkgs.cudatoolkit; - nvcc = cudaPkgs.cuda_nvcc; - cudart = cudaPkgs.cuda_cudart; - - # CUDA host compiler: default nix gcc is 15, toolkit wants <=13. - # Versioned names only — avoids colliding with default gcc on PATH. - gcc13 = pkgs.runCommand "gcc13-named" { } '' - mkdir -p $out/bin - for b in gcc g++ c++ cpp; do - [ -e ${pkgs.gcc13}/bin/$b ] || continue - ln -s ${pkgs.gcc13}/bin/$b $out/bin/$b-13 - done - ''; -} diff --git a/scripts/nix/build-env-python.nix b/scripts/nix/build-env-python.nix index acbcfb6a0..b889131f7 100644 --- a/scripts/nix/build-env-python.nix +++ b/scripts/nix/build-env-python.nix @@ -46,6 +46,7 @@ in pre-commit-hooks # compiler + tomli-w torch numpy transformers diff --git a/scripts/nix/build-env-riscv.nix b/scripts/nix/build-env-riscv.nix index d87cbdee5..64b2c58be 100644 --- a/scripts/nix/build-env-riscv.nix +++ b/scripts/nix/build-env-riscv.nix @@ -50,7 +50,8 @@ in ''; }; - # RISC-V Linux toolchain: inject static libc -L only for gcc/g++. Wrapping ar/nm/… + # RISC-V Linux toolchain: inject static libc -L only for gcc/g++ linking with + # -static; shared objects (models) link the shared glibc. Wrapping ar/nm/… # breaks Kbuild/busybox (ar must not see linker -L flags). riscv-linux-gcc = let cc = pkgs.pkgsCross.riscv64.stdenv.cc; @@ -63,7 +64,8 @@ in case "$name" in riscv64-unknown-linux-gnu-gcc|riscv64-unknown-linux-gnu-g++|riscv64-unknown-linux-gnu-c++) echo '#!${pkgs.stdenv.shell}' > $out/bin/$name - echo 'exec "'"$f"'" -L${libcStatic}/lib "$@"' >> $out/bin/$name + echo 'case " $* " in *" -static "*) exec "'"$f"'" -L${libcStatic}/lib "$@" ;; esac' >> $out/bin/$name + echo 'exec "'"$f"'" "$@"' >> $out/bin/$name chmod +x $out/bin/$name ;; *) diff --git a/scripts/nix/build-env-tools.nix b/scripts/nix/build-env-tools.nix index 8b77eee89..2d95d52fc 100644 --- a/scripts/nix/build-env-tools.nix +++ b/scripts/nix/build-env-tools.nix @@ -30,6 +30,8 @@ let cp libdramsim3.so $out/lib/ cp src/*.h ext/headers/*.h ext/headers/*.hpp $out/include/ cp -r configs $out/share/dramsim3/ + # VCS compiles these with the EDA host toolchain instead of linking the nix library. + cp -r src ext $out/share/dramsim3/ runHook postInstall ''; }; @@ -79,6 +81,7 @@ let in { verilator = pkgs.verilator; + espresso = pkgs.espresso; dramsim3 = dramsim3; diff --git a/scripts/nix/download.sh b/scripts/nix/download.sh index dad0ea613..acecd350b 100755 --- a/scripts/nix/download.sh +++ b/scripts/nix/download.sh @@ -24,9 +24,7 @@ begin_step "0-2" "submodules init" cd ${BBDIR} git submodule update --init --progress \ arch/thirdparty/gemmini \ - arch/thirdparty/rocket-chip \ arch/thirdparty/berkeley-hardfloat \ - bb-tests/workloads/lib/kernel \ bbdev \ bebop \ stack \ @@ -37,11 +35,10 @@ git submodule update --init --progress \ thirdparty/waveform-mcp \ .agents/skills \ .agents/mcps -git -C ${BBDIR}/arch/thirdparty/rocket-chip submodule sync -- dependencies/cde -git -C ${BBDIR}/arch/thirdparty/rocket-chip submodule update --init --progress dependencies/cde -git submodule update --init --depth 1 --single-branch --recommend-shallow --progress \ - bb-tests/thirdparty/linux \ - bb-tests/thirdparty/opensbi +git -C ${BBDIR}/stack submodule update --init --progress engine os/linux +git -C ${BBDIR}/stack submodule update --init --depth 1 --single-branch --recommend-shallow --progress \ + os/thirdparty/linux \ + os/thirdparty/opensbi begin_step "0-4" "buddy-mlir llvm init" -git -C ${BBDIR}/stack submodule update --init --progress compiler/thirdparty/buddy-mlir -git -C ${BBDIR}/stack/compiler/thirdparty/buddy-mlir submodule update --init --depth 1 --single-branch --recommend-shallow --progress llvm +git -C ${BBDIR}/stack submodule update --init --progress thirdparty/buddy-mlir +git -C ${BBDIR}/stack/thirdparty/buddy-mlir submodule update --init --depth 1 --single-branch --recommend-shallow --progress llvm diff --git a/scripts/nix/overlay.nix b/scripts/nix/overlay.nix index eca706954..be0f4f282 100644 --- a/scripts/nix/overlay.nix +++ b/scripts/nix/overlay.nix @@ -16,5 +16,4 @@ final: prev: opensta = final.tools.opensta; }; kernel = final.callPackage ./build-env-kernel.nix { }; - cuda = final.callPackage ./build-env-cuda.nix { }; } diff --git a/sourceme.sh b/sourceme.sh index c3adbc59c..9a6be2fa3 100644 --- a/sourceme.sh +++ b/sourceme.sh @@ -19,28 +19,13 @@ RESULT_PATH="${BBDIR}/result" #===----------------------------------------------------------------------------=== # Source Environment Variables #===----------------------------------------------------------------------------=== -export BUDDY_MLIR_BUILD_ROOT="${BBDIR}/stack/compiler/thirdparty/buddy-mlir/build" -export LLVM_MLIR_BUILD_DIR="${BBDIR}/stack/compiler/thirdparty/buddy-mlir/llvm/build" +export BUDDY_MLIR_BUILD_ROOT="${BBDIR}/stack/thirdparty/buddy-mlir/build" +export LLVM_MLIR_BUILD_DIR="${BBDIR}/stack/thirdparty/buddy-mlir/llvm/build" export PYTHONPATH="${LLVM_MLIR_BUILD_DIR}/tools/mlir/python_packages/mlir_core:${PYTHONPATH}" export RISCV="${BBDIR}/result" export PATH="${BBDIR}/thirdparty/libgloss/install/lib:$PATH" -# Per-chip buddy-opt lives at stack/compiler/thirdparty/buddy-mlir/build//bin export PATH="${BUDDY_MLIR_BUILD_ROOT}/bin:${PATH}" -# Optional interactive compiler session. Set before sourcing, e.g.: -# export BUCKYBALL_COMPILER_CHIP=toy -# bbdev compiler/workload inject BUDDY_MLIR_BUILD_DIR per subprocess. -if [ -n "${BUCKYBALL_COMPILER_CHIP:-}" ]; then - export BUDDY_MLIR_BUILD_DIR="${BUDDY_MLIR_BUILD_ROOT}/${BUCKYBALL_COMPILER_CHIP}" - export BUDDY_BINARY_DIR="${BUDDY_MLIR_BUILD_DIR}/bin" - if [ ! -d "${BUDDY_MLIR_BUILD_DIR}" ]; then - echo "ERROR: missing compiler build for chip ${BUCKYBALL_COMPILER_CHIP}: ${BUDDY_MLIR_BUILD_DIR}" >&2 - return 1 2>/dev/null || exit 1 - fi - export PYTHONPATH="${BUDDY_MLIR_BUILD_DIR}/python_packages:${PYTHONPATH}" - export PATH="${BUDDY_BINARY_DIR}:${PATH}" -fi - #===----------------------------------------------------------------------------=== # Export each submodule's PATH #===----------------------------------------------------------------------------=== diff --git a/stack b/stack index 81cced91c..256e889e1 160000 --- a/stack +++ b/stack @@ -1 +1 @@ -Subproject commit 81cced91c6b4138bf30033c1fda9a4c68a82d94a +Subproject commit 256e889e1b75d23201c41f7505c656c1a957ab8b diff --git a/verify b/verify index 893a197ed..b997b9687 160000 --- a/verify +++ b/verify @@ -1 +1 @@ -Subproject commit 893a197ede7f478911649aafb8db5fda0c75ac26 +Subproject commit b997b9687825cd5c6fcd0d12fd461faf72b082a1 diff --git a/bb-tests/workloads/src/MLIRTest/base/base_dma_group.mlir b/waste/bb-tests/workloads/src/MLIRTest/base/base_dma_group.mlir similarity index 100% rename from bb-tests/workloads/src/MLIRTest/base/base_dma_group.mlir rename to waste/bb-tests/workloads/src/MLIRTest/base/base_dma_group.mlir diff --git a/bb-tests/workloads/src/MLIRTest/base/base_dma_group_main.cpp b/waste/bb-tests/workloads/src/MLIRTest/base/base_dma_group_main.cpp similarity index 100% rename from bb-tests/workloads/src/MLIRTest/base/base_dma_group_main.cpp rename to waste/bb-tests/workloads/src/MLIRTest/base/base_dma_group_main.cpp diff --git a/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer.mlir b/waste/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer.mlir similarity index 94% rename from bb-tests/workloads/src/MLIRTest/base/base_mset_transfer.mlir rename to waste/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer.mlir index 5b054bdbf..2ca759221 100644 --- a/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer.mlir +++ b/waste/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer.mlir @@ -1,4 +1,4 @@ -// RUN: buddy-opt %s --target=toy -report-bank-usage="bank_num=4 verbose=true" 2>&1 | FileCheck %s --check-prefix=REPORT +// RUN: buddy-opt %s -report-bank-usage="bank_num=4 verbose=true" 2>&1 | FileCheck %s --check-prefix=REPORT // REPORT: transfer b1 -> b7 groups=1 cur=2/4 // REPORT: transfer b2 -> b7 groups=1 cur=2/4 // REPORT: peak=2/4 alloc=2 release=1 leaked=0 diff --git a/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer_main.cpp b/waste/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer_main.cpp similarity index 100% rename from bb-tests/workloads/src/MLIRTest/base/base_mset_transfer_main.cpp rename to waste/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer_main.cpp diff --git a/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_calls.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_calls.mlir similarity index 89% rename from bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_calls.mlir rename to waste/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_calls.mlir index 04e399b5d..55a69ffd3 100644 --- a/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_calls.mlir +++ b/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_calls.mlir @@ -1,4 +1,4 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks | FileCheck %s +// RUN: buddy-opt %s -assign-physical-banks | FileCheck %s // CHECK-LABEL: func.func @group_output // CHECK: call @mock // CHECK: buckyball.mvout {{.*}} diff --git a/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_invalid.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_invalid.mlir similarity index 94% rename from bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_invalid.mlir rename to waste/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_invalid.mlir index 2ec5537e4..952596fdc 100644 --- a/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_invalid.mlir +++ b/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_invalid.mlir @@ -1,4 +1,4 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks -split-input-file -verify-diagnostics +// RUN: buddy-opt %s -assign-physical-banks -split-input-file -verify-diagnostics module { func.func private @wrong_type(i64, i64, i32) func.func @mismatched() { diff --git a/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_cfg_invalid.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_cfg_invalid.mlir similarity index 93% rename from bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_cfg_invalid.mlir rename to waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_cfg_invalid.mlir index 3fa941841..d4c223a75 100644 --- a/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_cfg_invalid.mlir +++ b/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_cfg_invalid.mlir @@ -1,4 +1,4 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks -split-input-file -verify-diagnostics +// RUN: buddy-opt %s -assign-physical-banks -split-input-file -verify-diagnostics func.func @captured_allocation(%condition: i1) { %source = buckyball.bank_alloc diff --git a/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_invalid.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_invalid.mlir similarity index 95% rename from bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_invalid.mlir rename to waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_invalid.mlir index 80f41c0c9..a1aacf677 100644 --- a/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_invalid.mlir +++ b/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_invalid.mlir @@ -1,4 +1,4 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks -split-input-file -verify-diagnostics +// RUN: buddy-opt %s -assign-physical-banks -split-input-file -verify-diagnostics func.func @source_use_after_transfer() { %source = buckyball.bank_alloc diff --git a/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_ownership.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_ownership.mlir similarity index 97% rename from bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_ownership.mlir rename to waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_ownership.mlir index e96c053e0..7b9c1ec06 100644 --- a/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_ownership.mlir +++ b/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_ownership.mlir @@ -1,4 +1,4 @@ -// RUN: buddy-opt %s --target=toy -assign-physical-banks | FileCheck %s +// RUN: buddy-opt %s -assign-physical-banks | FileCheck %s // CHECK-LABEL: func.func @release_fragmented // CHECK: buckyball.mset {{.*}} {{.*}}col = 14 diff --git a/bb-tests/workloads/src/MLIRTest/compiler/contiguous_scatter_rows.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/contiguous_scatter_rows.mlir similarity index 100% rename from bb-tests/workloads/src/MLIRTest/compiler/contiguous_scatter_rows.mlir rename to waste/bb-tests/workloads/src/MLIRTest/compiler/contiguous_scatter_rows.mlir diff --git a/bb-tests/workloads/src/MLIRTest/compiler/dma_group_invalid.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/dma_group_invalid.mlir similarity index 100% rename from bb-tests/workloads/src/MLIRTest/compiler/dma_group_invalid.mlir rename to waste/bb-tests/workloads/src/MLIRTest/compiler/dma_group_invalid.mlir diff --git a/bb-tests/workloads/src/MLIRTest/compiler/dma_group_lowering.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/dma_group_lowering.mlir similarity index 78% rename from bb-tests/workloads/src/MLIRTest/compiler/dma_group_lowering.mlir rename to waste/bb-tests/workloads/src/MLIRTest/compiler/dma_group_lowering.mlir index d9d5d094d..5c65def9c 100644 --- a/bb-tests/workloads/src/MLIRTest/compiler/dma_group_lowering.mlir +++ b/waste/bb-tests/workloads/src/MLIRTest/compiler/dma_group_lowering.mlir @@ -1,11 +1,11 @@ -// RUN: buddy-opt %s --target=toy -lower-buckyball -canonicalize | FileCheck %s +// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s // CHECK-LABEL: func.func @selected // CHECK-DAG: %[[MODE:.*]] = arith.constant -8935140561191436288 : i64 // CHECK: arith.ori {{.*}}, %[[MODE]] -// CHECK: "buckyball.intr.mvin" +// CHECK: llvm.call_intrinsic "llvm.riscv.bb.mvin" // CHECK: llvm.call @dma_touch_mvout_group // CHECK: arith.ori {{.*}}, %[[MODE]] -// CHECK: "buckyball.intr.mvout" +// CHECK: llvm.call_intrinsic "llvm.riscv.bb.mvout" func.func @selected(%input: memref<16x32xi8>, %output: memref<16x32xi8>, %bank: i64) { %depth = arith.constant 16 : i64 %stride = arith.constant 2 : i64 @@ -16,9 +16,9 @@ func.func @selected(%input: memref<16x32xi8>, %output: memref<16x32xi8>, %bank: // CHECK-LABEL: func.func @whole // CHECK-NOT: dma_touch_mvout_group -// CHECK: "buckyball.intr.mvin" +// CHECK: llvm.call_intrinsic "llvm.riscv.bb.mvin" // CHECK: llvm.call @dma_touch_mvout -// CHECK: "buckyball.intr.mvout" +// CHECK: llvm.call_intrinsic "llvm.riscv.bb.mvout" func.func @whole(%input: memref<16x48xi8>, %output: memref<16x48xi8>, %bank: i64) { %depth = arith.constant 16 : i64 %stride = arith.constant 1 : i64 diff --git a/bb-tests/workloads/src/MLIRTest/compiler/fp32_packmm_mixed.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/fp32_packmm_mixed.mlir similarity index 81% rename from bb-tests/workloads/src/MLIRTest/compiler/fp32_packmm_mixed.mlir rename to waste/bb-tests/workloads/src/MLIRTest/compiler/fp32_packmm_mixed.mlir index b5e6fff29..2f4d4557a 100644 --- a/bb-tests/workloads/src/MLIRTest/compiler/fp32_packmm_mixed.mlir +++ b/waste/bb-tests/workloads/src/MLIRTest/compiler/fp32_packmm_mixed.mlir @@ -1,5 +1,5 @@ -// RUN: buddy-opt %s --target=attention -lower-buckyball-to-bank-ssa | FileCheck %s --check-prefix=SSA -// RUN: buddy-opt %s --target=attention -lower-buckyball-to-bank-ssa -assign-physical-banks | FileCheck %s --check-prefix=BANK +// RUN: buddy-opt %s -lower-buckyball-to-bank-ssa | FileCheck %s --check-prefix=SSA +// RUN: buddy-opt %s -lower-buckyball-to-bank-ssa -assign-physical-banks | FileCheck %s --check-prefix=BANK // SSA-LABEL: func.func @packed // SSA-COUNT-3: buckyball.bank_transfer // SSA: buckyball.bank_kernel @rvv_packmm diff --git a/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_invalid.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_invalid.mlir similarity index 100% rename from bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_invalid.mlir rename to waste/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_invalid.mlir diff --git a/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_lowering.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_lowering.mlir similarity index 77% rename from bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_lowering.mlir rename to waste/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_lowering.mlir index d4202266a..a7340b266 100644 --- a/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_lowering.mlir +++ b/waste/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_lowering.mlir @@ -1,10 +1,10 @@ -// RUN: buddy-opt %s --target=toy -lower-buckyball -canonicalize | FileCheck %s +// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s // CHECK-LABEL: func.func @dynamic // CHECK-SAME: %[[SRC:.*]]: i64, %[[DST:.*]]: i64 // CHECK: llvm.call @dma_bank_transfer(%[[SRC]], %[[DST]]) : (i64, i64) -> () // CHECK: arith.shli %[[DST]] // CHECK: arith.ori %[[SRC]] -// CHECK: "buckyball.intr.mset" +// CHECK: llvm.call_intrinsic "llvm.riscv.bb.mset" func.func @dynamic(%source: i64, %target: i64) { buckyball.mset_transfer %source %target : i64 i64 return @@ -14,7 +14,7 @@ func.func @dynamic(%source: i64, %target: i64) { // CHECK-DAG: %[[MODE:.*]] = arith.constant 4096 : i64 // CHECK-DAG: %[[PACKED:.*]] = arith.constant 17825793 : i64 // CHECK: llvm.call @dma_bank_transfer -// CHECK: "buckyball.intr.mset"(%[[PACKED]], %[[MODE]]) +// CHECK: llvm.call_intrinsic "llvm.riscv.bb.mset"(%[[PACKED]], %[[MODE]]) func.func @constant() { %source = arith.constant 1 : i64 %target = arith.constant 17 : i64 diff --git a/bb-tests/workloads/src/MLIRTest/compiler/norm_quant_window_mixed.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/norm_quant_window_mixed.mlir similarity index 92% rename from bb-tests/workloads/src/MLIRTest/compiler/norm_quant_window_mixed.mlir rename to waste/bb-tests/workloads/src/MLIRTest/compiler/norm_quant_window_mixed.mlir index fec444072..d4e483283 100644 --- a/bb-tests/workloads/src/MLIRTest/compiler/norm_quant_window_mixed.mlir +++ b/waste/bb-tests/workloads/src/MLIRTest/compiler/norm_quant_window_mixed.mlir @@ -1,5 +1,5 @@ -// RUN: buddy-opt %s --target=ffn -fuse-norm-quant-window | FileCheck %s --check-prefix=SSA -// RUN: buddy-opt %s --target=ffn -fuse-norm-quant-window -assign-physical-banks | FileCheck %s --check-prefix=BANK +// RUN: buddy-opt %s -fuse-norm-quant-window | FileCheck %s --check-prefix=SSA +// RUN: buddy-opt %s -fuse-norm-quant-window -assign-physical-banks | FileCheck %s --check-prefix=BANK // SSA-LABEL: func.func @mixed // SSA-COUNT-3: buckyball.bank_transfer diff --git a/bb-tests/workloads/src/MLIRTest/compiler/rvv_rms_suffix.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/rvv_rms_suffix.mlir similarity index 100% rename from bb-tests/workloads/src/MLIRTest/compiler/rvv_rms_suffix.mlir rename to waste/bb-tests/workloads/src/MLIRTest/compiler/rvv_rms_suffix.mlir diff --git a/bb-tests/workloads/src/MLIRTest/compiler/rvv_tanh.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/rvv_tanh.mlir similarity index 100% rename from bb-tests/workloads/src/MLIRTest/compiler/rvv_tanh.mlir rename to waste/bb-tests/workloads/src/MLIRTest/compiler/rvv_tanh.mlir diff --git a/waste/examples/components/balls/f32add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/f32add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..e2f15c56d --- /dev/null +++ b/waste/examples/components/balls/f32add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,16 @@ +#include "Buckyball/BuckyballOps.h" +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankF32AddOp::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankF32AddOp op = *this; + rewriter.create(op.getLoc(), op.getA(), op.getB(), op.getC(), + op.getRowsAttr(), op.getGroupAttr(), + op.getFirstAttr()); + rewriter.replaceOp(op, op.getC()); + return success(); +} diff --git a/examples/balls/f32add/compiler/src/Dialect/Buckyball/F32AddBall.td b/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/F32AddBall.td similarity index 88% rename from examples/balls/f32add/compiler/src/Dialect/Buckyball/F32AddBall.td rename to waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/F32AddBall.td index bcca4d6b3..eca07ca5d 100644 --- a/examples/balls/f32add/compiler/src/Dialect/Buckyball/F32AddBall.td +++ b/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/F32AddBall.td @@ -17,7 +17,7 @@ def F32AddOp : Buckyball_Op<"f32add", [MemoryEffects<[MemRead, MemWrite]>]> { let assemblyFormat = "$a `,` $b `,` $c prop-dict attr-dict `:` type($a)"; } -def BankF32AddOp : Buckyball_Op<"bank_f32add", [MemoryEffects<[MemRead, MemWrite]>]> { +def BankF32AddOp : Buckyball_Op<"bank_f32add", [DeclareOpInterfaceMethods, MemoryEffects<[MemRead, MemWrite]>]> { let summary = "FP32 grouped addition with an SSA output bank handle"; let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$group, BoolAttr:$first); diff --git a/examples/balls/f32add/compiler/src/Dialect/Buckyball/IR/F32Add.cpp b/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/IR/F32Add.cpp similarity index 96% rename from examples/balls/f32add/compiler/src/Dialect/Buckyball/IR/F32Add.cpp rename to waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/IR/F32Add.cpp index 30344af2b..bcd4bc844 100644 --- a/examples/balls/f32add/compiler/src/Dialect/Buckyball/IR/F32Add.cpp +++ b/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/IR/F32Add.cpp @@ -2,7 +2,7 @@ #include "mlir/Dialect/Arith/IR/Arith.h" using namespace mlir; -using namespace buddy::buckyball; +using namespace ::buddy::buckyball; namespace { template LogicalResult verifyAddition(Op op) { diff --git a/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..354fc9d80 --- /dev/null +++ b/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,46 @@ +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +class F32AddLowering : public ConvertOpToLLVMPattern { +public: + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + LogicalResult + matchAndRewrite(F32AddOp op, OpAdaptor operands, + ConversionPatternRewriter &rewriter) const override { + + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || op.getRows() > target.bankDepth) + return op.emitError( + "F32ADD rows exceed the target 128-bit bank capacity"); + Location loc = op.getLoc(); + Value rs1 = + packRs1BanksIter(rewriter, loc, operands.getA(), operands.getB(), + operands.getC(), cstI64(rewriter, loc, op.getRows())); + Value rs2 = cstI64(rewriter, loc, op.getGroup() | (op.getFirst() ? 32 : 0)); + rewriter.replaceOpWithNewOp( + op, rs1, rs2, + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("F32ADD"))); + return success(); + } +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateF32AddBallLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { + patterns.add(converter); +} + +void configureF32AddBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool) { + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/f32add/stack/src/Passes.cpp b/waste/examples/components/balls/f32add/stack/src/Passes.cpp new file mode 100644 index 000000000..a2b0e71aa --- /dev/null +++ b/waste/examples/components/balls/f32add/stack/src/Passes.cpp @@ -0,0 +1,39 @@ +//===- Passes.cpp - F32AddBall passes -------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +namespace buckyball { +void populateF32AddBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureF32AddBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LowerF32AddPass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerF32AddPass) + StringRef getArgument() const final { return "lower-f32add-ball"; } + StringRef getDescription() const final { + return "Lower F32AddBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateF32AddBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureF32AddBallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + return true; +}(); diff --git a/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..c7fbdcdcf --- /dev/null +++ b/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,76 @@ +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +Value shift(PatternRewriter &rewriter, Location loc, Value value, + int64_t bits) { + return rewriter.create( + loc, value, rewriter.create(loc, bits, 64)); +} + +Value rs1(PatternRewriter &rewriter, Location loc, Value a, Value b, Value c, + Value iter) { + Value ab = rewriter.create(loc, a, shift(rewriter, loc, b, 10)); + Value ci = rewriter.create(loc, shift(rewriter, loc, c, 20), + shift(rewriter, loc, iter, 30)); + return rewriter.create(loc, ab, ci); +} + +Value rs2(PatternRewriter &rewriter, Location loc, Value a, Value b, Value c, + int64_t subcommand) { + Value packed = rewriter.create(loc, subcommand, 64); + packed = + rewriter.create(loc, packed, shift(rewriter, loc, a, 6)); + packed = + rewriter.create(loc, packed, shift(rewriter, loc, b, 16)); + return rewriter.create(loc, packed, + shift(rewriter, loc, c, 26)); +} + +// Computes A x B into the output bank; accumulation sets rs1 bit 29. +template +LogicalResult compute(Op op, PatternRewriter &rewriter, StringRef mnemonic, + int64_t subcommand) { + Value operands = rs1(rewriter, op.getLoc(), op.getABank(), op.getBBank(), + op.getOutBank(), op.getIter()); + if (subcommand == 3) + operands = rewriter.create( + op.getLoc(), operands, + rewriter.create(op.getLoc(), 1LL << 29, 64)); + rewriter.create( + op.getLoc(), rewriter.getStringAttr(mnemonic), operands, + rs2(rewriter, op.getLoc(), op.getABase(), op.getBBase(), op.getOutBase(), + subcommand)); + rewriter.replaceOp(op, op.getOutBank()); + return success(); +} +} // namespace + +LogicalResult +BankGemminiPreloadOp::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + Value zero = rewriter.create(getLoc(), 0, 64); + rewriter.create( + getLoc(), rewriter.getStringAttr("GEMMINI_PRELOAD"), + rs1(rewriter, getLoc(), getInBank(), zero, getOutBank(), getIter()), + rs2(rewriter, getLoc(), getInBase(), zero, getOutBase(), 1)); + rewriter.replaceOp(*this, getOutBank()); + return success(); +} + +LogicalResult BankGemminiComputePreloadedOp::assignPhysicalBanks( + PatternRewriter &rewriter, mlir::buddy::PhysicalBankState &) { + return compute(*this, rewriter, "GEMMINI_COMPUTE_PRELOADED", 2); +} + +LogicalResult BankGemminiComputeAccumulatedOp::assignPhysicalBanks( + PatternRewriter &rewriter, mlir::buddy::PhysicalBankState &) { + return compute(*this, rewriter, "GEMMINI_COMPUTE_ACCUMULATED", 3); +} diff --git a/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp b/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp new file mode 100644 index 000000000..c2aac15d6 --- /dev/null +++ b/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp @@ -0,0 +1,307 @@ +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/Math/IR/Math.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Target/BuckyballTargetRegistry.h" +#include "Utils/BankUtils.h" + +#include + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult GemminiMemConvOp::lowerToBankSSA(PatternRewriter &b) { + GemminiMemConvOp op = *this; + auto in = cast(op.getInput().getType()); + auto w = cast(op.getWeight().getType()); + auto bias = cast(op.getBias().getType()); + auto out = cast(op.getOutput().getType()); + bool integer = in.getElementType().isInteger(8) && + bias.getElementType().isInteger(32) && + out.getElementType().isInteger(32); + bool floating = in.getElementType().isF32() && + bias.getElementType().isF32() && out.getElementType().isF32(); + if (in.getRank() != 4 || w.getRank() != 2 || bias.getRank() != 1 || + out.getRank() != 4 || !in.hasStaticShape() || !w.hasStaticShape() || + !bias.hasStaticShape() || !out.hasStaticShape() || + !w.getElementType().isInteger(8) || (!integer && !floating)) + return op.emitError( + "requires static NCHW input/output and I8 KxN weights; " + "use I8 input with I32 output/bias, or F32 input/output/bias"); + + int64_t C = in.getShape()[1], H = in.getShape()[2], W = in.getShape()[3]; + int64_t OH = out.getShape()[2], OW = out.getShape()[3]; + int64_t kh = op.getKh(), kw = op.getKw(), stride = op.getStride(); + int64_t pad = op.getPadding(); + int64_t M = in.getShape()[0] * OH * OW, K = C * kh * kw; + int64_t N = w.getShape()[1], kTiles = (K + 15) / 16; + if (w.getShape()[0] != K || bias.getShape()[0] != N || + out.getShape()[0] != in.getShape()[0] || out.getShape()[1] != N || + OH != (H + 2 * pad - kh) / stride + 1 || + OW != (W + 2 * pad - kw) / stride + 1) + return op.emitError("conv shape mismatch"); + float inScale = op.getInputScale().convertToFloat(); + float wScale = op.getWeightScale().convertToFloat(); + if (!std::isfinite(inScale) || inScale <= 0 || !std::isfinite(wScale) || + wScale <= 0) + return op.emitError("scales must be finite and positive"); + + if (integer && (inScale != 1.0f || wScale != 1.0f)) + return op.emitError("native INT8 convolution requires unit scales"); + + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || target.bankDepth < 16) + return op.emitError( + "GemminiBall requires a 128-bit bank with at least 16 lines"); + + Location loc = op.getLoc(); + auto idx = [&](int64_t v) -> Value { + return b.create(loc, v); + }; + using P = arith::CmpIPredicate; + Value zero = idx(0), one = idx(1), sixteen = idx(16); + Value zeroI8 = b.create(loc, 0, 8); + IntegerAttr align = b.getI64IntegerAttr(64); + + // Quantized im2col panels, one 16x16 tile per (M tile, K tile). + Value panels = b.create( + loc, MemRefType::get({(M + 15) / 16 * kTiles, 16, 16}, b.getI8Type()), + align); + b.create(loc, zeroI8, panels); + auto rows = b.create(loc, zero, idx(M), one); + b.setInsertionPointToStart(rows.getBody()); + Value r = rows.getInductionVar(); + Value y0 = b.create( + loc, + b.create( + loc, + b.create( + loc, b.create(loc, r, idx(OW)), idx(OH)), + idx(stride)), + idx(pad)); + Value x0 = b.create( + loc, + b.create(loc, b.create(loc, r, idx(OW)), + idx(stride)), + idx(pad)); + auto inners = b.create(loc, zero, idx(K), one); + b.setInsertionPointToStart(inners.getBody()); + Value i = inners.getInductionVar(); + Value y = b.create( + loc, y0, b.create(loc, i, idx(kw * C))); + Value x = b.create( + loc, x0, + b.create(loc, b.create(loc, i, idx(C)), + idx(kw))); + Value yIn = b.create( + loc, b.create(loc, P::sge, y, zero), + b.create(loc, P::slt, y, idx(H))); + Value xIn = b.create( + loc, b.create(loc, P::sge, x, zero), + b.create(loc, P::slt, x, idx(W))); + auto inside = + b.create(loc, b.create(loc, yIn, xIn)); + b.setInsertionPointToStart(&inside.getThenRegion().front()); + Value v = b.create( + loc, op.getInput(), + ValueRange{b.create(loc, r, idx(OH * OW)), + b.create(loc, i, idx(C)), y, x}); + if (!integer) { + v = b.create( + loc, v, b.create(loc, b.getF32FloatAttr(inScale))); + v = b.create( + loc, v, b.create(loc, b.getF32FloatAttr(-128))); + v = b.create( + loc, v, b.create(loc, b.getF32FloatAttr(127))); + v = b.create(loc, v); + v = b.create(loc, b.getI32Type(), v); + v = b.create(loc, b.getI8Type(), v); + } + Value panel = b.create( + loc, + b.create(loc, b.create(loc, r, sixteen), + idx(kTiles)), + b.create(loc, i, sixteen)); + b.create( + loc, v, panels, + ValueRange{panel, b.create(loc, r, sixteen), + b.create(loc, i, sixteen)}); + b.setInsertionPointAfter(rows); + + auto tileType = MemRefType::get({16, 16}, b.getI8Type()); + Value dTile = b.create(loc, tileType, align); + Value wTile = b.create(loc, tileType, align); + Value acc = b.create( + loc, MemRefType::get({16, 16}, b.getI32Type()), align); + b.create(loc, zeroI8, dTile); + auto meta = b.create(loc, op.getWeight()); + Value wStride = meta.getStrides()[0]; + Value wDirect = b.create( + loc, + b.create( + loc, P::eq, b.create(loc, wStride, sixteen), zero), + b.create(loc, P::eq, meta.getStrides()[1], one)); + Value wLines = b.create( + loc, b.getI64Type(), b.create(loc, wStride, sixteen)); + Value iter = createI64Const(b, loc, 16), base = createI64Const(b, loc, 0); + // Weight-stationary dataflow. + b.create(loc, b.getStringAttr("GEMMINI_CONFIG"), base, + createI64Const(b, loc, 16)); + Value ab = allocBank(b, loc, 1, 1), wb = allocBank(b, loc, 1, 1); + Value db = allocBank(b, loc, 1, 1), cb = allocBank(b, loc, 1, 4); + db = mvinBank(b, loc, dTile, db, 16); + + auto rowLoop = + b.create(loc, zero, idx(M), sixteen, ValueRange{ab, wb, cb}); + b.setInsertionPointToStart(rowLoop.getBody()); + Value row = rowLoop.getInductionVar(); + auto colLoop = b.create(loc, zero, idx(N), sixteen, + rowLoop.getRegionIterArgs()); + b.setInsertionPointToStart(colLoop.getBody()); + Value col = colLoop.getInductionVar(); + auto kLoop = b.create(loc, zero, idx(K), sixteen, + colLoop.getRegionIterArgs()); + b.setInsertionPointToStart(kLoop.getBody()); + Value k = kLoop.getInductionVar(); + ab = kLoop.getRegionIterArgs()[0]; + wb = kLoop.getRegionIterArgs()[1]; + cb = kLoop.getRegionIterArgs()[2]; + + panel = b.create( + loc, + b.create(loc, b.create(loc, row, sixteen), + idx(kTiles)), + b.create(loc, k, sixteen)); + SmallVector panelOffsets{panel, b.getIndexAttr(0), + b.getIndexAttr(0)}; + SmallVector panelSizes{b.getIndexAttr(1), b.getIndexAttr(16), + b.getIndexAttr(16)}; + SmallVector panelSteps(3, b.getIndexAttr(1)); + auto panelType = memref::SubViewOp::inferRankReducedResultType( + {16, 16}, cast(panels.getType()), panelOffsets, panelSizes, + panelSteps); + Value aTile = b.create( + loc, panelType, panels, panelOffsets, panelSizes, panelSteps); + ab = mvinBank(b, loc, aTile, ab, 16); + + Value full = b.create( + loc, wDirect, + b.create( + loc, + b.create( + loc, P::ule, b.create(loc, k, sixteen), idx(K)), + b.create(loc, P::ule, + b.create(loc, col, sixteen), + idx(N)))); + auto weight = b.create(loc, b.getI64Type(), full, true); + b.setInsertionPointToStart(&weight.getThenRegion().front()); + SmallVector wOffsets{k, col}; + SmallVector wSizes(2, b.getIndexAttr(16)); + SmallVector wSteps(2, b.getIndexAttr(1)); + Value wBlock = b.create(loc, op.getWeight(), wOffsets, + wSizes, wSteps); + b.create( + loc, ValueRange{b.create(loc, b.getI64Type(), wBlock, wb, + iter, wLines, IntegerAttr{})}); + b.setInsertionPointToStart(&weight.getElseRegion().front()); + auto wi = b.create(loc, zero, sixteen, one); + b.setInsertionPointToStart(wi.getBody()); + auto wj = b.create(loc, zero, sixteen, one); + b.setInsertionPointToStart(wj.getBody()); + Value wk = b.create(loc, k, wi.getInductionVar()); + Value wn = b.create(loc, col, wj.getInductionVar()); + auto pick = b.create( + loc, b.getI8Type(), + b.create(loc, + b.create(loc, P::ult, wk, idx(K)), + b.create(loc, P::ult, wn, idx(N))), + true); + b.setInsertionPointToStart(&pick.getThenRegion().front()); + b.create(loc, ValueRange{b.create( + loc, op.getWeight(), ValueRange{wk, wn})}); + b.setInsertionPointToStart(&pick.getElseRegion().front()); + b.create(loc, zeroI8); + b.setInsertionPointAfter(pick); + b.create( + loc, pick.getResult(0), wTile, + ValueRange{wi.getInductionVar(), wj.getInductionVar()}); + b.setInsertionPointAfter(wi); + b.create(loc, mvinBank(b, loc, wTile, wb, 16)); + b.setInsertionPointAfter(weight); + wb = weight.getResult(0); + + auto first = b.create(loc, P::eq, k, zero); + auto compute = b.create(loc, b.getI64Type(), first, true); + b.setInsertionPointToStart(&compute.getThenRegion().front()); + Value preloaded = b.create(loc, b.getI64Type(), wb, cb, + iter, base, base); + b.create( + loc, + ValueRange{b.create( + loc, b.getI64Type(), ab, db, preloaded, iter, base, base, base)}); + b.setInsertionPointToStart(&compute.getElseRegion().front()); + b.create( + loc, ValueRange{b.create( + loc, b.getI64Type(), ab, wb, cb, iter, base, base, base)}); + b.setInsertionPointAfter(compute); + b.create(loc, ValueRange{ab, wb, compute.getResult(0)}); + b.setInsertionPointAfter(kLoop); + + cb = mvoutBank(b, loc, acc, kLoop.getResult(2), 16); + auto ei = b.create( + loc, zero, + b.create(loc, sixteen, + b.create(loc, idx(M), row)), + one); + b.setInsertionPointToStart(ei.getBody()); + auto ej = b.create( + loc, zero, + b.create(loc, sixteen, + b.create(loc, idx(N), col)), + one); + b.setInsertionPointToStart(ej.getBody()); + Value ri = b.create(loc, row, ei.getInductionVar()); + Value cj = b.create(loc, col, ej.getInductionVar()); + Value sum = b.create( + loc, acc, ValueRange{ei.getInductionVar(), ej.getInductionVar()}); + Value result; + if (integer) { + result = b.create( + loc, sum, b.create(loc, op.getBias(), ValueRange{cj})); + } else { + result = b.create( + loc, + b.create( + loc, b.create(loc, b.getF32Type(), sum), + b.create(loc, + b.getF32FloatAttr(inScale * wScale))), + b.create(loc, op.getBias(), ValueRange{cj})); + } + b.create( + loc, result, op.getOutput(), + ValueRange{b.create(loc, ri, idx(OH * OW)), cj, + b.create( + loc, b.create(loc, ri, idx(OW)), idx(OH)), + b.create(loc, ri, idx(OW))}); + b.setInsertionPointAfter(ei); + b.create( + loc, ValueRange{kLoop.getResult(0), kLoop.getResult(1), cb}); + b.setInsertionPointAfter(colLoop); + b.create(loc, colLoop.getResults()); + b.setInsertionPointAfter(rowLoop); + + for (Value bank : rowLoop.getResults()) + releaseBank(b, loc, bank); + releaseBank(b, loc, db); + for (Value buffer : {panels, dTile, wTile, acc}) + b.create(loc, buffer); + b.eraseOp(op); + return success(); +} diff --git a/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerLinalgToBuckyball/Int8Contractions.cpp b/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerLinalgToBuckyball/Int8Contractions.cpp new file mode 100644 index 000000000..5b3d7a8e0 --- /dev/null +++ b/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerLinalgToBuckyball/Int8Contractions.cpp @@ -0,0 +1,132 @@ +//===- Int8Contractions.cpp - INT8 contractions for GemminiBall +//------------===// + +#include "Buckyball/BuckyballOps.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/IR/PatternMatch.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class Int8ContractionLowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(linalg::GenericOp op, + PatternRewriter &rewriter) const override { + auto convolution = op->getAttrOfType("buddy.int8_conv"); + bool matrix = op->hasAttr("buddy.int8_matmul"); + if (!convolution && !matrix) + return failure(); + if (op.getInputs().size() != 2 || op.getOutputs().size() != 1) + return op.emitError( + "INT8 contraction requires two inputs and one output"); + Value input = op.getInputs()[0], weight = op.getInputs()[1]; + Value output = op.getOutputs()[0]; + auto in = dyn_cast(input.getType()); + auto w = dyn_cast(weight.getType()); + auto out = dyn_cast(output.getType()); + if (!in || !w || !out || !in.hasStaticShape() || !w.hasStaticShape() || + !out.hasStaticShape() || !in.getElementType().isInteger(8) || + !w.getElementType().isInteger(8) || !out.getElementType().isInteger(32)) + return op.emitError( + "GemminiBall requires static INT8 operands and INT32 output"); + Location loc = op.getLoc(); + Value packedWeight; + SmallVector permutation; + if (matrix) { + auto map = op.getIndexingMapsArray()[1]; + if (map.getResult(0) == getAffineDimExpr(1, rewriter.getContext())) + permutation = {1, 0}; + } else { + auto layout = + cast(convolution.get("weight_permutation")) + .asArrayRef(); + if (layout == ArrayRef{0, 1, 2, 3}) + permutation = {2, 3, 1, 0}; + } + if (!permutation.empty()) { + SmallVector shape; + for (int64_t axis : permutation) + shape.push_back(w.getShape()[axis]); + packedWeight = memref::AllocOp::create( + rewriter, loc, MemRefType::get(shape, rewriter.getI8Type())); + SmallVector coordinates; + for (int64_t extent : w.getShape()) { + auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0); + auto end = arith::ConstantIndexOp::create(rewriter, loc, extent); + auto one = arith::ConstantIndexOp::create(rewriter, loc, 1); + auto loop = scf::ForOp::create(rewriter, loc, zero, end, one); + rewriter.setInsertionPointToStart(loop.getBody()); + coordinates.push_back(loop.getInductionVar()); + } + Value value = memref::LoadOp::create(rewriter, loc, weight, coordinates); + SmallVector indices; + for (int64_t axis : permutation) + indices.push_back(coordinates[axis]); + memref::StoreOp::create(rewriter, loc, value, packedWeight, indices); + rewriter.setInsertionPoint(op); + weight = packedWeight; + w = cast(weight.getType()); + } + int64_t kh = 1, kw = 1, stride = 1, channels; + if (matrix) { + if (in.getRank() != 2 || w.getRank() != 2 || out.getRank() != 2) + return op.emitError("GemminiBall matrix operands must have rank two"); + channels = out.getShape()[1]; + input = memref::ExpandShapeOp::create( + rewriter, loc, + ArrayRef{in.getShape()[0], in.getShape()[1], 1, 1}, input, + ArrayRef{{0}, {1, 2, 3}}); + output = memref::ExpandShapeOp::create( + rewriter, loc, ArrayRef{out.getShape()[0], channels, 1, 1}, + output, ArrayRef{{0}, {1, 2, 3}}); + } else { + auto steps = + dyn_cast_or_null(convolution.get("stride")); + auto dilation = + dyn_cast_or_null(convolution.get("dilation")); + auto permutation = dyn_cast_or_null( + convolution.get("weight_permutation")); + auto groups = dyn_cast_or_null(convolution.get("groups")); + if (in.getRank() != 4 || w.getRank() != 4 || out.getRank() != 4 || + !steps || steps.size() != 2 || steps[0] != steps[1] || !dilation || + dilation.asArrayRef() != ArrayRef{1, 1} || !groups || + groups.getInt() != 1 || !permutation || + (permutation.asArrayRef() != ArrayRef{2, 3, 1, 0} && + permutation.asArrayRef() != ArrayRef{0, 1, 2, 3})) + return op.emitError("GemminiBall requires ordinary NCHW convolution " + "with HWIO weights and unit dilation"); + kh = w.getShape()[0]; + kw = w.getShape()[1]; + stride = steps[0]; + channels = out.getShape()[1]; + weight = memref::CollapseShapeOp::create( + rewriter, loc, weight, + ArrayRef{{0, 1, 2}, {3}}); + } + auto biasType = MemRefType::get({channels}, rewriter.getI32Type()); + Value bias = memref::AllocOp::create(rewriter, loc, biasType); + Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32); + linalg::FillOp::create(rewriter, loc, zero, bias); + GemminiMemConvOp::create( + rewriter, loc, input, weight, bias, output, + rewriter.getI64IntegerAttr(kh), rewriter.getI64IntegerAttr(kw), + rewriter.getI64IntegerAttr(stride), rewriter.getI64IntegerAttr(0), + rewriter.getF32FloatAttr(1.0), rewriter.getF32FloatAttr(1.0)); + memref::DeallocOp::create(rewriter, loc, bias); + if (packedWeight) + memref::DeallocOp::create(rewriter, loc, packedWeight); + rewriter.eraseOp(op); + return success(); + } +}; +} // namespace +namespace mlir::buddy { +void populateGemminiBallLinalgPatterns(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} +} // namespace mlir::buddy diff --git a/examples/balls/gemmini/compiler/src/Dialect/Buckyball/GemminiBall.td b/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/GemminiBall.td similarity index 76% rename from examples/balls/gemmini/compiler/src/Dialect/Buckyball/GemminiBall.td rename to waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/GemminiBall.td index 944024194..9d8453483 100644 --- a/examples/balls/gemmini/compiler/src/Dialect/Buckyball/GemminiBall.td +++ b/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/GemminiBall.td @@ -19,7 +19,7 @@ def GemminiInstructionOp : Buckyball_Op<"gemmini"> { let assemblyFormat = "$mnemonic $rs1 `,` $rs2 prop-dict attr-dict `:` type($rs1)"; } -def BankGemminiPreloadOp : Buckyball_Op<"bank_gemmini_preload"> { +def BankGemminiPreloadOp : Buckyball_Op<"bank_gemmini_preload", [DeclareOpInterfaceMethods]> { let summary = "BankSSA Gemmini preload"; let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, I64:$inBase, I64:$outBase); @@ -28,7 +28,7 @@ def BankGemminiPreloadOp : Buckyball_Op<"bank_gemmini_preload"> { "type($inBank) type($outBank) type($iter) type($inBase) type($outBase)"; } -def BankGemminiComputePreloadedOp : Buckyball_Op<"bank_gemmini_compute_preloaded"> { +def BankGemminiComputePreloadedOp : Buckyball_Op<"bank_gemmini_compute_preloaded", [DeclareOpInterfaceMethods]> { let summary = "BankSSA Gemmini compute using preloaded state"; let arguments = (ins I64:$aBank, I64:$bBank, I64:$outBank, I64:$iter, I64:$aBase, I64:$bBase, I64:$outBase); @@ -37,7 +37,7 @@ def BankGemminiComputePreloadedOp : Buckyball_Op<"bank_gemmini_compute_preloaded "type($aBank) type($bBank) type($outBank) type($iter) type($aBase) type($bBase) type($outBase)"; } -def BankGemminiComputeAccumulatedOp : Buckyball_Op<"bank_gemmini_compute_accumulated"> { +def BankGemminiComputeAccumulatedOp : Buckyball_Op<"bank_gemmini_compute_accumulated", [DeclareOpInterfaceMethods]> { let summary = "BankSSA Gemmini compute accumulating the output bank"; let arguments = (ins I64:$aBank, I64:$bBank, I64:$outBank, I64:$iter, I64:$aBase, I64:$bBase, I64:$outBase); @@ -49,16 +49,23 @@ def BankGemminiComputeAccumulatedOp : Buckyball_Op<"bank_gemmini_compute_accumul def GemminiConvOp : Buckyball_Op<"gemmini_conv", [ DeclareOpInterfaceMethods]> { - let summary = "NCHW convolution with INT8 HWIO matrix weights and scalar quantization scales"; + let summary = "NCHW convolution with INT8 HWIO matrix weights"; + let description = [{ + FP32 inputs are quantized using the supplied scales and produce FP32 output. + INT8 inputs bypass quantization and produce INT32 output with unit scales. + }]; let arguments = (ins AnyRankedTensor:$input, AnyRankedTensor:$weight, AnyRankedTensor:$bias, I64Attr:$kh, I64Attr:$kw, I64Attr:$stride, I64Attr:$padding, F32Attr:$inputScale, F32Attr:$weightScale); let results = (outs AnyRankedTensor:$output); let assemblyFormat = "$input `,` $weight `,` $bias prop-dict attr-dict `:` type($input) `,` type($weight) `,` type($bias) `->` type($output)"; } -def GemminiMemConvOp : Buckyball_Op<"gemmini_mem_conv"> { - let arguments = (ins AnyMemRef:$input, AnyMemRef:$weight, AnyMemRef:$bias, - AnyMemRef:$output, I64Attr:$kh, I64Attr:$kw, I64Attr:$stride, +def GemminiMemConvOp : Buckyball_Op<"gemmini_mem_conv", [DeclareOpInterfaceMethods]> { + let arguments = (ins Arg]>:$input, + Arg]>:$weight, + Arg]>:$bias, + Arg]>:$output, + I64Attr:$kh, I64Attr:$kw, I64Attr:$stride, I64Attr:$padding, F32Attr:$inputScale, F32Attr:$weightScale); let assemblyFormat = "$input `,` $weight `,` $bias `,` $output prop-dict attr-dict `:` type($input) `,` type($weight) `,` type($bias) `,` type($output)"; } diff --git a/examples/balls/gemmini/compiler/src/Dialect/Buckyball/IR/GemminiConv.cpp b/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/IR/GemminiConv.cpp similarity index 97% rename from examples/balls/gemmini/compiler/src/Dialect/Buckyball/IR/GemminiConv.cpp rename to waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/IR/GemminiConv.cpp index e91d3ef9f..82caba124 100644 --- a/examples/balls/gemmini/compiler/src/Dialect/Buckyball/IR/GemminiConv.cpp +++ b/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/IR/GemminiConv.cpp @@ -2,7 +2,7 @@ #include "mlir/Dialect/MemRef/IR/MemRef.h" using namespace mlir; using namespace mlir::bufferization; -using namespace buddy::buckyball; +using namespace ::buddy::buckyball; bool GemminiConvOp::bufferizesToAllocation(Value) { return true; } bool GemminiConvOp::bufferizesToMemoryRead(OpOperand &, const AnalysisState &) { return true; diff --git a/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..5c44ca176 --- /dev/null +++ b/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,46 @@ +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class GemminiInstructionLowering + : public ConvertOpToLLVMPattern { +public: + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + LogicalResult + matchAndRewrite(GemminiInstructionOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + rewriter.replaceOpWithNewOp( + op, adaptor.getRs1(), adaptor.getRs2(), + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7(op.getMnemonic()))); + return success(); + } +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateGemminiBallLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, + int64_t bankDepth) { + (void)stable; + (void)bankDepth; + + patterns.add(converter); +} + +void configureGemminiBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool stable) { + (void)stable; + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/gemmini/stack/src/Passes.cpp b/waste/examples/components/balls/gemmini/stack/src/Passes.cpp new file mode 100644 index 000000000..673db551c --- /dev/null +++ b/waste/examples/components/balls/gemmini/stack/src/Passes.cpp @@ -0,0 +1,56 @@ +//===- Passes.cpp - GemminiBall passes ------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +void populateGemminiBallLinalgPatterns(RewritePatternSet &patterns); +namespace buckyball { +void populateGemminiBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureGemminiBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LinalgToGemminiPass + : public mlir::buddy::LinalgToBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LinalgToGemminiPass) + StringRef getArgument() const final { + return "convert-linalg-to-gemmini-ball"; + } + StringRef getDescription() const final { + return "Lower signed INT8 Linalg contractions to GemminiBall."; + } + void populate(RewritePatternSet &patterns, int64_t, int64_t, int64_t) const { + mlir::buddy::populateGemminiBallLinalgPatterns(patterns); + } +}; + +class LowerGemminiPass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerGemminiPass) + StringRef getArgument() const final { return "lower-gemmini-ball"; } + StringRef getDescription() const final { + return "Lower GemminiBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateGemminiBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureGemminiBallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + PassRegistration(); + return true; +}(); diff --git a/examples/balls/im2col/compiler/src/Backend/include/IntrinsicsRISCVIm2colBall.td b/waste/examples/components/balls/im2col/stack/src/Backend/include/IntrinsicsRISCVIm2colBall.td similarity index 100% rename from examples/balls/im2col/compiler/src/Backend/include/IntrinsicsRISCVIm2colBall.td rename to waste/examples/components/balls/im2col/stack/src/Backend/include/IntrinsicsRISCVIm2colBall.td diff --git a/waste/examples/components/balls/im2col/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/im2col/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..f237f4067 --- /dev/null +++ b/waste/examples/components/balls/im2col/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,23 @@ +//===- AssignPhysicalBanks.cpp - Im2col bank assignment -------------------===// + +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankIm2colOp::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankIm2colOp op = *this; + rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), + op.getIter(), op.getKsize(), op.getStride(), + op.getPadding(), op.getInputBase(), op.getLane(), + op.getStartRowAttr(), op.getStartColAttr(), + op.getWindowStartAttr(), op.getWindowCountAttr()); + rewriter.replaceOp(op, op.getOutBank()); + return success(); +} diff --git a/examples/balls/im2col/compiler/src/Dialect/Buckyball/Im2colBall.td b/waste/examples/components/balls/im2col/stack/src/Dialect/Buckyball/Im2colBall.td similarity index 97% rename from examples/balls/im2col/compiler/src/Dialect/Buckyball/Im2colBall.td rename to waste/examples/components/balls/im2col/stack/src/Dialect/Buckyball/Im2colBall.td index 450922bd5..112001c9f 100644 --- a/examples/balls/im2col/compiler/src/Dialect/Buckyball/Im2colBall.td +++ b/waste/examples/components/balls/im2col/stack/src/Dialect/Buckyball/Im2colBall.td @@ -13,7 +13,7 @@ include "Buckyball.td" // Im2colBall operations //===----------------------------------------------------------------------===// -// Memref-level im2col+gemm (like smatmul_matmul). Chip bank-ssa decides +// Memref-level im2col+gemm (like smatmul_matmul). Design bank-ssa decides // f32 quant / Bank* composition; this op carries no quant policy. def Im2colMatmulOp : Buckyball_Op<"im2col_matmul"> { let summary = "Packed-plane im2col then matrix gemm"; @@ -113,7 +113,7 @@ def Im2colOp : Buckyball_Op<"im2col"> { "type($inputBankId)"; } -def BankIm2colOp : Buckyball_Op<"bank_im2col"> { +def BankIm2colOp : Buckyball_Op<"bank_im2col", [DeclareOpInterfaceMethods]> { let summary = "Im2col using virtual bank handles"; let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, I64:$ksize, I64:$stride, I64:$padding, diff --git a/waste/examples/components/balls/im2col/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/im2col/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..603dbb6a7 --- /dev/null +++ b/waste/examples/components/balls/im2col/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,234 @@ +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/IR/PatternMatch.h" + +#include + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +static LogicalResult +validateIm2colEncoding(Operation *op, Value iterValue, Value ksizeValue, + Value strideValue, Value paddingValue, + Value inputBaseValue, Value laneValue, int64_t startRow, + int64_t startCol, int64_t windowStart, + int64_t windowCount, int64_t bankDepth) { + auto getConst = [](Value value) -> std::optional { + if (auto constant = value.getDefiningOp()) + if (auto attr = dyn_cast(constant.getValue())) + return attr.getInt(); + return std::nullopt; + }; + auto iter = getConst(iterValue); + auto ksize = getConst(ksizeValue); + auto stride = getConst(strideValue); + auto padding = getConst(paddingValue); + auto inputBase = getConst(inputBaseValue); + auto lane = getConst(laneValue); + if (!iter || !ksize || !stride || !padding) + return op->emitError( + "Im2col instruction requires constant iter/ksize/stride/padding"); + if (*iter <= 0 || *ksize <= 0 || *ksize > 255 || *stride <= 0 || + *stride > 255 || *padding < 0 || *padding > 255 || startRow < 0 || + startRow > 255 || startCol < 0 || startCol > 255 || + (inputBase && (*inputBase < 0 || *inputBase >= 64)) || + (lane && (*lane < 0 || *lane >= 16))) + return op->emitError("Im2col instruction field out of range"); + int64_t inputRows = *iter * *iter; + if (inputBase && *inputBase + inputRows > bankDepth) + return op->emitError("Im2col input range exceeds the physical bank depth"); + if (startRow > *padding || startCol > *padding) + return op->emitError("Im2col startRow/startCol must not exceed padding"); + if (windowStart < 0 || windowStart >= 64 || windowCount <= 0 || + windowCount > 64) + return op->emitError( + "Im2col requires windowStart in [0, 63] and windowCount in [1, 64]"); + + int64_t padded = *iter + 2 * *padding; + if (padded < *ksize + startRow || padded < *ksize + startCol || + (padded - *ksize - startRow) % *stride || + (padded - *ksize - startCol) % *stride) + return op->emitError("Im2col shape does not produce integral windows"); + int64_t rows = (padded - *ksize - startRow) / *stride + 1; + int64_t cols = (padded - *ksize - startCol) / *stride + 1; + if (windowStart + windowCount > rows * cols) + return op->emitError("Im2col window range exceeds output shape"); + int64_t kernelTiles = (*ksize * *ksize + 15) / 16; + int64_t outputRows = (windowCount + 15) / 16 * kernelTiles * 16; + if (outputRows > bankDepth) + return op->emitError("Im2col output range exceeds the physical bank depth"); + return success(); +} + +struct Im2colLowering : public ConvertOpToLLVMPattern { + Im2colLowering(LLVMTypeConverter &converter, bool stable, int64_t bankDepth) + : ConvertOpToLLVMPattern(converter), stable(stable), + bankDepth(bankDepth) {} + + LogicalResult + matchAndRewrite(Im2colOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + if (failed(validateIm2colEncoding( + op, op.getIter(), op.getKsize(), op.getStride(), op.getPadding(), + op.getInputBase(), op.getLane(), op.getStartRow(), op.getStartCol(), + op.getWindowStart(), op.getWindowCount(), bankDepth))) + return failure(); + Location loc = op.getLoc(); + Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), + cstI64(rewriter, loc, 0), + adaptor.getOutputBankId(), adaptor.getIter()); + + Value rs2 = adaptor.getKsize(); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getStride(), + cstI64(rewriter, loc, 8))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getPadding(), + cstI64(rewriter, loc, 16))); + Value startCol = cstI64(rewriter, loc, op.getStartCol()); + Value startRow = cstI64(rewriter, loc, op.getStartRow()); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, startCol, + cstI64(rewriter, loc, 24))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, startRow, + cstI64(rewriter, loc, 32))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getInputBase(), + cstI64(rewriter, loc, 40))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getLane(), + cstI64(rewriter, loc, 46))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create( + loc, cstI64(rewriter, loc, op.getWindowStart()), + cstI64(rewriter, loc, 50))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create( + loc, cstI64(rewriter, loc, op.getWindowCount()), + cstI64(rewriter, loc, 56))); + + if (stable) { + rewriter.replaceOpWithNewOp(op, rs1, rs2); + return success(); + } + rewriter.replaceOpWithNewOp( + op, rs1, rs2, + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("IM2COL"))); + return success(); + } + +private: + bool stable = false; + int64_t bankDepth; +}; +struct BankIm2colLowering : public ConvertOpToLLVMPattern { + BankIm2colLowering(LLVMTypeConverter &converter, bool stable, + int64_t bankDepth) + : ConvertOpToLLVMPattern(converter), stable(stable), + bankDepth(bankDepth) {} + + LogicalResult + matchAndRewrite(BankIm2colOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + if (failed(validateIm2colEncoding( + op, op.getIter(), op.getKsize(), op.getStride(), op.getPadding(), + op.getInputBase(), op.getLane(), op.getStartRow(), op.getStartCol(), + op.getWindowStart(), op.getWindowCount(), bankDepth))) + return failure(); + Location loc = op.getLoc(); + Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInBank(), + cstI64(rewriter, loc, 0), adaptor.getOutBank(), + adaptor.getIter()); + + Value rs2 = adaptor.getKsize(); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getStride(), + cstI64(rewriter, loc, 8))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getPadding(), + cstI64(rewriter, loc, 16))); + Value startCol = cstI64(rewriter, loc, op.getStartCol()); + Value startRow = cstI64(rewriter, loc, op.getStartRow()); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, startCol, + cstI64(rewriter, loc, 24))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, startRow, + cstI64(rewriter, loc, 32))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getInputBase(), + cstI64(rewriter, loc, 40))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getLane(), + cstI64(rewriter, loc, 46))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create( + loc, cstI64(rewriter, loc, op.getWindowStart()), + cstI64(rewriter, loc, 50))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create( + loc, cstI64(rewriter, loc, op.getWindowCount()), + cstI64(rewriter, loc, 56))); + + if (stable) { + rewriter.replaceOpWithNewOp(op, rs1, rs2); + return success(); + } + rewriter.replaceOpWithNewOp( + op, rs1, rs2, + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("IM2COL"))); + return success(); + } + +private: + bool stable = false; + int64_t bankDepth; +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateIm2colBallLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, + int64_t bankDepth) { + patterns.add(converter, stable, bankDepth); + patterns.add(converter, stable, bankDepth); +} + +void configureIm2colBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool stable) { + if (stable) + target.addLegalOp(); + else + target.addIllegalOp(); + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/im2col/stack/src/Passes.cpp b/waste/examples/components/balls/im2col/stack/src/Passes.cpp new file mode 100644 index 000000000..3234a5aac --- /dev/null +++ b/waste/examples/components/balls/im2col/stack/src/Passes.cpp @@ -0,0 +1,39 @@ +//===- Passes.cpp - Im2colBall passes -------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +namespace buckyball { +void populateIm2colBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureIm2colBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LowerIm2colPass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerIm2colPass) + StringRef getArgument() const final { return "lower-im2col-ball"; } + StringRef getDescription() const final { + return "Lower Im2colBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateIm2colBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureIm2colBallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + return true; +}(); diff --git a/waste/examples/components/balls/int2fp/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/int2fp/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..30ab5dd49 --- /dev/null +++ b/waste/examples/components/balls/int2fp/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,21 @@ +//===- AssignPhysicalBanks.cpp - Int2Fp bank assignment -------------------===// + +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankInt32ToFp32Op::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankInt32ToFp32Op op = *this; + rewriter.create(op.getLoc(), op.getInBank(), op.getScaleBank(), + op.getOutBank(), op.getIter(), + op.getReluAttr()); + rewriter.replaceOp(op, op.getOutBank()); + return success(); +} diff --git a/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Int2FpBall.td b/waste/examples/components/balls/int2fp/stack/src/Dialect/Buckyball/Int2FpBall.td similarity index 92% rename from examples/balls/int2fp/compiler/src/Dialect/Buckyball/Int2FpBall.td rename to waste/examples/components/balls/int2fp/stack/src/Dialect/Buckyball/Int2FpBall.td index c5c241cd9..34297d84a 100644 --- a/examples/balls/int2fp/compiler/src/Dialect/Buckyball/Int2FpBall.td +++ b/waste/examples/components/balls/int2fp/stack/src/Dialect/Buckyball/Int2FpBall.td @@ -27,7 +27,7 @@ def Int32ToFp32Op : Buckyball_Op<"int32_to_fp32"> { "type($inputBankId)"; } -def BankInt32ToFp32Op : Buckyball_Op<"bank_int32_to_fp32"> { +def BankInt32ToFp32Op : Buckyball_Op<"bank_int32_to_fp32", [DeclareOpInterfaceMethods]> { let summary = "INT32 to FP32 conversion with virtual bank handles"; let arguments = (ins I64:$inBank, I64:$scaleBank, I64:$outBank, I64:$iter, BoolAttr:$relu); diff --git a/waste/examples/components/balls/int2fp/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/int2fp/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..93a4f6e14 --- /dev/null +++ b/waste/examples/components/balls/int2fp/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,46 @@ +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +struct Int32ToFp32Lowering : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + LogicalResult + matchAndRewrite(Int32ToFp32Op op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), + adaptor.getScaleBankId(), + adaptor.getOutputBankId(), adaptor.getIter()); + rewriter.replaceOpWithNewOp( + op, rs1, cstI64(rewriter, loc, op.getRelu() ? 1 : 0), + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("INT32_TO_FP32"))); + return success(); + } +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateInt2FpBallLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, + int64_t) { + (void)stable; + patterns.add(converter); +} + +void configureInt2FpBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool stable) { + (void)stable; + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/int2fp/stack/src/Passes.cpp b/waste/examples/components/balls/int2fp/stack/src/Passes.cpp new file mode 100644 index 000000000..c3d5c1093 --- /dev/null +++ b/waste/examples/components/balls/int2fp/stack/src/Passes.cpp @@ -0,0 +1,39 @@ +//===- Passes.cpp - Int2FpBall passes -------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +namespace buckyball { +void populateInt2FpBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureInt2FpBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LowerInt2FpPass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerInt2FpPass) + StringRef getArgument() const final { return "lower-int2fp-ball"; } + StringRef getDescription() const final { + return "Lower Int2FpBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateInt2FpBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureInt2FpBallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + return true; +}(); diff --git a/examples/balls/int8add/compiler/src/Backend/include/IntrinsicsRISCVInt8AddBall.td b/waste/examples/components/balls/int8add/stack/src/Backend/include/IntrinsicsRISCVInt8AddBall.td similarity index 100% rename from examples/balls/int8add/compiler/src/Backend/include/IntrinsicsRISCVInt8AddBall.td rename to waste/examples/components/balls/int8add/stack/src/Backend/include/IntrinsicsRISCVInt8AddBall.td diff --git a/waste/examples/components/balls/int8add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/int8add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..fe667e078 --- /dev/null +++ b/waste/examples/components/balls/int8add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,19 @@ +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankInt8AddOp::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankInt8AddOp op = *this; + rewriter.create(op.getLoc(), op.getLhsBank(), op.getRhsBank(), + op.getOutputBank(), op.getIter(), op.getLhsRatio(), + op.getRhsRatio(), op.getReluAttr()); + rewriter.replaceOp(op, op.getOutputBank()); + return success(); +} diff --git a/examples/balls/int8add/compiler/src/Dialect/Buckyball/Int8AddBall.td b/waste/examples/components/balls/int8add/stack/src/Dialect/Buckyball/Int8AddBall.td similarity index 90% rename from examples/balls/int8add/compiler/src/Dialect/Buckyball/Int8AddBall.td rename to waste/examples/components/balls/int8add/stack/src/Dialect/Buckyball/Int8AddBall.td index 253a89e99..bcbca0cb1 100644 --- a/examples/balls/int8add/compiler/src/Dialect/Buckyball/Int8AddBall.td +++ b/waste/examples/components/balls/int8add/stack/src/Dialect/Buckyball/Int8AddBall.td @@ -12,7 +12,7 @@ def Int8AddOp : Buckyball_Op<"int8add"> { "$lhsRatio `,` $rhsRatio prop-dict attr-dict `:` type($lhsBankId)"; } -def BankInt8AddOp : Buckyball_Op<"bank_int8add"> { +def BankInt8AddOp : Buckyball_Op<"bank_int8add", [DeclareOpInterfaceMethods]> { let summary = "Scaled INT8 addition on virtual banks"; let arguments = (ins I64:$lhsBank, I64:$rhsBank, I64:$outputBank, I64:$iter, F32:$lhsRatio, F32:$rhsRatio, diff --git a/waste/examples/components/balls/int8add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/int8add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..aa924bb03 --- /dev/null +++ b/waste/examples/components/balls/int8add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,56 @@ +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +struct Int8AddLowering : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + LogicalResult + matchAndRewrite(Int8AddOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getLhsBankId(), + adaptor.getRhsBankId(), + adaptor.getOutputBankId(), adaptor.getIter()); + Value lhsBits = rewriter.create( + loc, rewriter.getI32Type(), adaptor.getLhsRatio()); + Value rhsBits = rewriter.create( + loc, rewriter.getI32Type(), adaptor.getRhsRatio()); + Value lhs64 = + rewriter.create(loc, rewriter.getI64Type(), lhsBits); + Value rhs64 = + rewriter.create(loc, rewriter.getI64Type(), rhsBits); + Value rs2 = rewriter.create( + loc, lhs64, + rewriter.create(loc, rhs64, cstI64(rewriter, loc, 32))); + rewriter.replaceOpWithNewOp( + op, rs1, rs2, + rewriter.getI32IntegerAttr(buckyball_target::getBuckyballFunct7( + op.getRelu() ? "INT8ADD_RELU" : "INT8ADD"))); + return success(); + } +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateInt8AddBallLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { + patterns.add(converter); +} + +void configureInt8AddBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool) { + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/int8add/stack/src/Passes.cpp b/waste/examples/components/balls/int8add/stack/src/Passes.cpp new file mode 100644 index 000000000..333431c85 --- /dev/null +++ b/waste/examples/components/balls/int8add/stack/src/Passes.cpp @@ -0,0 +1,39 @@ +//===- Passes.cpp - Int8AddBall passes ------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +namespace buckyball { +void populateInt8AddBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureInt8AddBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LowerInt8AddPass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerInt8AddPass) + StringRef getArgument() const final { return "lower-int8add-ball"; } + StringRef getDescription() const final { + return "Lower Int8AddBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateInt8AddBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureInt8AddBallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + return true; +}(); diff --git a/examples/balls/int8mul/compiler/src/Backend/include/IntrinsicsRISCVInt8MulBall.td b/waste/examples/components/balls/int8mul/stack/src/Backend/include/IntrinsicsRISCVInt8MulBall.td similarity index 100% rename from examples/balls/int8mul/compiler/src/Backend/include/IntrinsicsRISCVInt8MulBall.td rename to waste/examples/components/balls/int8mul/stack/src/Backend/include/IntrinsicsRISCVInt8MulBall.td diff --git a/waste/examples/components/balls/int8mul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/int8mul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..4b93ea121 --- /dev/null +++ b/waste/examples/components/balls/int8mul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,19 @@ +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankInt8MulOp::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankInt8MulOp op = *this; + rewriter.create(op.getLoc(), op.getGateBank(), op.getInputBank(), + op.getOutputBank(), op.getIter(), op.getRatio(), + op.getGateRow()); + rewriter.replaceOp(op, op.getOutputBank()); + return success(); +} diff --git a/examples/balls/int8mul/compiler/src/Dialect/Buckyball/Int8MulBall.td b/waste/examples/components/balls/int8mul/stack/src/Dialect/Buckyball/Int8MulBall.td similarity index 90% rename from examples/balls/int8mul/compiler/src/Dialect/Buckyball/Int8MulBall.td rename to waste/examples/components/balls/int8mul/stack/src/Dialect/Buckyball/Int8MulBall.td index 6295351a1..b4451aa67 100644 --- a/examples/balls/int8mul/compiler/src/Dialect/Buckyball/Int8MulBall.td +++ b/waste/examples/components/balls/int8mul/stack/src/Dialect/Buckyball/Int8MulBall.td @@ -12,7 +12,7 @@ def Int8MulOp : Buckyball_Op<"int8mul"> { "type($gateBankId)"; } -def BankInt8MulOp : Buckyball_Op<"bank_int8mul"> { +def BankInt8MulOp : Buckyball_Op<"bank_int8mul", [DeclareOpInterfaceMethods]> { let summary = "Scaled INT8 per-channel broadcast multiply on virtual banks"; let arguments = (ins I64:$gateBank, I64:$inputBank, I64:$outputBank, I64:$iter, F32:$ratio, I64:$gateRow); diff --git a/waste/examples/components/balls/int8mul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/int8mul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..bcde87217 --- /dev/null +++ b/waste/examples/components/balls/int8mul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,52 @@ +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +struct Int8MulLowering : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + LogicalResult + matchAndRewrite(Int8MulOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getGateBankId(), + adaptor.getInputBankId(), + adaptor.getOutputBankId(), adaptor.getIter()); + Value ratioBits = rewriter.create( + loc, rewriter.getI32Type(), adaptor.getRatio()); + Value ratio = + rewriter.create(loc, rewriter.getI64Type(), ratioBits); + Value gateRow = rewriter.create(loc, adaptor.getGateRow(), + cstI64(rewriter, loc, 32)); + Value rs2 = rewriter.create(loc, ratio, gateRow); + rewriter.replaceOpWithNewOp( + op, rs1, rs2, + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("INT8MUL"))); + return success(); + } +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateInt8MulBallLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { + patterns.add(converter); +} + +void configureInt8MulBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool) { + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/int8mul/stack/src/Passes.cpp b/waste/examples/components/balls/int8mul/stack/src/Passes.cpp new file mode 100644 index 000000000..8008e2d09 --- /dev/null +++ b/waste/examples/components/balls/int8mul/stack/src/Passes.cpp @@ -0,0 +1,39 @@ +//===- Passes.cpp - Int8MulBall passes ------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +namespace buckyball { +void populateInt8MulBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureInt8MulBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LowerInt8MulPass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerInt8MulPass) + StringRef getArgument() const final { return "lower-int8mul-ball"; } + StringRef getDescription() const final { + return "Lower Int8MulBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateInt8MulBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureInt8MulBallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + return true; +}(); diff --git a/waste/examples/components/balls/lut/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/lut/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..759859bf7 --- /dev/null +++ b/waste/examples/components/balls/lut/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,35 @@ +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace mlir::buddy; +using namespace ::buddy::buckyball; + +LogicalResult +BankLutOp::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &state) { + BankLutOp op = *this; + auto input = state.getSlot(op.getInBank()); + auto lut = state.getSlot(op.getLutBank()); + auto output = state.getSlot(op.getOutBank()); + if (!input || !lut || !output) + return failure(); + if (input->row != 1 || input->col != 1 || output->row != 1 || + output->col != 1 || lut->row != 1 || (lut->col != 1 && lut->col != 4)) + return op.emitError( + "LUT requires col=1 input/output and col=1 or col=4 table"); + auto overlaps = [](const BankSlot &lhs, const BankSlot &rhs) { + return lhs.base < rhs.base + rhs.row * rhs.col && + rhs.base < lhs.base + lhs.row * lhs.col; + }; + if (overlaps(*input, *lut) || overlaps(*input, *output) || + overlaps(*lut, *output)) + return op.emitError("LUT bank groups must not overlap"); + rewriter.create(op.getLoc(), op.getInBank(), op.getLutBank(), + op.getOutBank(), op.getIter()); + rewriter.replaceOp(op, op.getOutBank()); + return success(); +} diff --git a/examples/balls/lut/compiler/src/Dialect/Buckyball/LutBall.td b/waste/examples/components/balls/lut/stack/src/Dialect/Buckyball/LutBall.td similarity index 88% rename from examples/balls/lut/compiler/src/Dialect/Buckyball/LutBall.td rename to waste/examples/components/balls/lut/stack/src/Dialect/Buckyball/LutBall.td index 386b75259..d119ceadf 100644 --- a/examples/balls/lut/compiler/src/Dialect/Buckyball/LutBall.td +++ b/waste/examples/components/balls/lut/stack/src/Dialect/Buckyball/LutBall.td @@ -11,7 +11,7 @@ def LutOp : Buckyball_Op<"lut"> { "$iter attr-dict `:` type($inputBankId)"; } -def BankLutOp : Buckyball_Op<"bank_lut"> { +def BankLutOp : Buckyball_Op<"bank_lut", [DeclareOpInterfaceMethods]> { let summary = "Apply a shared or 16-lane INT8 lookup table with virtual banks"; let arguments = (ins I64:$inBank, I64:$lutBank, I64:$outBank, I64:$iter); let results = (outs I64:$outBankOut); diff --git a/waste/examples/components/balls/lut/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/lut/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..e56f4f021 --- /dev/null +++ b/waste/examples/components/balls/lut/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,60 @@ +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +struct LutLowering : public ConvertOpToLLVMPattern { + LutLowering(LLVMTypeConverter &converter, int64_t bankDepth) + : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} + + LogicalResult + matchAndRewrite(LutOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + auto iterOp = op.getIter().getDefiningOp(); + auto iterAttr = + iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); + if (!iterAttr || iterAttr.getInt() <= 0 || iterAttr.getInt() > bankDepth) + return op.emitError( + "LUT iter must be a positive constant within one bank"); + Location loc = op.getLoc(); + Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), + adaptor.getLutBankId(), + adaptor.getOutputBankId(), adaptor.getIter()); + rewriter.replaceOpWithNewOp( + op, rs1, cstI64(rewriter, loc, 0), + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("LUT"))); + return success(); + } + +private: + int64_t bankDepth; +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateLutBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, + RewritePatternSet &patterns, + bool stable, + int64_t bankDepth) { + (void)stable; + + patterns.add(converter, bankDepth); +} + +void configureLutBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool stable) { + (void)stable; + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/lut/stack/src/Passes.cpp b/waste/examples/components/balls/lut/stack/src/Passes.cpp new file mode 100644 index 000000000..30ac9667d --- /dev/null +++ b/waste/examples/components/balls/lut/stack/src/Passes.cpp @@ -0,0 +1,39 @@ +//===- Passes.cpp - LutBall passes ----------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +namespace buckyball { +void populateLutBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureLutBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LowerLutPass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerLutPass) + StringRef getArgument() const final { return "lower-lut-ball"; } + StringRef getDescription() const final { + return "Lower LutBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateLutBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureLutBallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + return true; +}(); diff --git a/examples/balls/matadd/compiler/src/Backend/include/IntrinsicsRISCVMatAddBall.td b/waste/examples/components/balls/matadd/stack/src/Backend/include/IntrinsicsRISCVMatAddBall.td similarity index 100% rename from examples/balls/matadd/compiler/src/Backend/include/IntrinsicsRISCVMatAddBall.td rename to waste/examples/components/balls/matadd/stack/src/Backend/include/IntrinsicsRISCVMatAddBall.td diff --git a/waste/examples/components/balls/matadd/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/matadd/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..7b4531ce9 --- /dev/null +++ b/waste/examples/components/balls/matadd/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,18 @@ +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankMatAddOp::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankMatAddOp op = *this; + rewriter.create(op.getLoc(), op.getABank(), op.getBBank(), + op.getCBank(), op.getIter()); + rewriter.replaceOp(op, op.getCBank()); + return success(); +} diff --git a/examples/balls/matadd/compiler/src/Dialect/Buckyball/MatAddBall.td b/waste/examples/components/balls/matadd/stack/src/Dialect/Buckyball/MatAddBall.td similarity index 86% rename from examples/balls/matadd/compiler/src/Dialect/Buckyball/MatAddBall.td rename to waste/examples/components/balls/matadd/stack/src/Dialect/Buckyball/MatAddBall.td index 62e34bd00..a59ad403c 100644 --- a/examples/balls/matadd/compiler/src/Dialect/Buckyball/MatAddBall.td +++ b/waste/examples/components/balls/matadd/stack/src/Dialect/Buckyball/MatAddBall.td @@ -10,7 +10,7 @@ def MatAddOp : Buckyball_Op<"matadd"> { "type($aBankId)"; } -def BankMatAddOp : Buckyball_Op<"bank_matadd"> { +def BankMatAddOp : Buckyball_Op<"bank_matadd", [DeclareOpInterfaceMethods]> { let summary = "Add matching INT32 virtual-bank lines"; let arguments = (ins I64:$aBank, I64:$bBank, I64:$cBank, I64:$iter); let results = (outs I64:$cBankOut); diff --git a/waste/examples/components/balls/matadd/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/matadd/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..1a4822e0d --- /dev/null +++ b/waste/examples/components/balls/matadd/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,44 @@ +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +struct MatAddLowering : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + LogicalResult + matchAndRewrite(MatAddOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getABankId(), + adaptor.getBBankId(), adaptor.getCBankId(), + adaptor.getIter()); + rewriter.replaceOpWithNewOp( + op, rs1, cstI64(rewriter, loc, 0), + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("MATADD"))); + return success(); + } +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateMatAddBallLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { + patterns.add(converter); +} + +void configureMatAddBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool) { + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/matadd/stack/src/Passes.cpp b/waste/examples/components/balls/matadd/stack/src/Passes.cpp new file mode 100644 index 000000000..f473d98b1 --- /dev/null +++ b/waste/examples/components/balls/matadd/stack/src/Passes.cpp @@ -0,0 +1,39 @@ +//===- Passes.cpp - MatAddBall passes -------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +namespace buckyball { +void populateMatAddBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureMatAddBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LowerMatAddPass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerMatAddPass) + StringRef getArgument() const final { return "lower-matadd-ball"; } + StringRef getDescription() const final { + return "Lower MatAddBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateMatAddBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureMatAddBallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + return true; +}(); diff --git a/waste/examples/components/balls/maxpool/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/maxpool/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..a2b949c60 --- /dev/null +++ b/waste/examples/components/balls/maxpool/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,22 @@ +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankMaxPoolOp::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankMaxPoolOp op = *this; + rewriter.create( + op.getLoc(), op.getInBank(), op.getOutBank(), op.getIter(), + op.getInputSideAttr(), op.getOutputSideAttr(), op.getKernelAttr(), + op.getStrideAttr(), op.getPaddingAttr(), op.getInputBase(), + op.getOutputBase(), op.getOutputStride(), op.getStartRowAttr(), + op.getStartColAttr()); + rewriter.replaceOp(op, op.getOutBank()); + return success(); +} diff --git a/examples/balls/maxpool/compiler/src/Dialect/Buckyball/MaxPoolBall.td b/waste/examples/components/balls/maxpool/stack/src/Dialect/Buckyball/MaxPoolBall.td similarity index 89% rename from examples/balls/maxpool/compiler/src/Dialect/Buckyball/MaxPoolBall.td rename to waste/examples/components/balls/maxpool/stack/src/Dialect/Buckyball/MaxPoolBall.td index 98ec6617e..ed3cef778 100644 --- a/examples/balls/maxpool/compiler/src/Dialect/Buckyball/MaxPoolBall.td +++ b/waste/examples/components/balls/maxpool/stack/src/Dialect/Buckyball/MaxPoolBall.td @@ -11,7 +11,8 @@ def MaxPoolOp : Buckyball_Op<"maxpool"> { Each bank row contains the 16 channels of one spatial position. The input and output tiles are square. inputBase selects the first input row. outputBase and outputStride place the result inside a larger resident - feature-map tile. iter is the number of output spatial positions. + feature-map tile, preserving destination rows outside the result. + iter is the number of output spatial positions. }]; let arguments = (ins I64:$inputBankId, I64:$outputBankId, I64:$iter, I64Attr:$inputSide, I64Attr:$outputSide, @@ -26,7 +27,7 @@ def MaxPoolOp : Buckyball_Op<"maxpool"> { "type($outputStride)"; } -def BankMaxPoolOp : Buckyball_Op<"bank_maxpool"> { +def BankMaxPoolOp : Buckyball_Op<"bank_maxpool", [DeclareOpInterfaceMethods]> { let summary = "Signed INT8 spatial max pooling with virtual banks"; let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, I64Attr:$inputSide, I64Attr:$outputSide, diff --git a/waste/examples/components/balls/maxpool/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/maxpool/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..56a78af40 --- /dev/null +++ b/waste/examples/components/balls/maxpool/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,113 @@ +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/IR/PatternMatch.h" +#include + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +struct MaxPoolLowering : public ConvertOpToLLVMPattern { + MaxPoolLowering(LLVMTypeConverter &converter, int64_t bankDepth) + : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} + + LogicalResult + matchAndRewrite(MaxPoolOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + int64_t inputSide = op.getInputSide(); + int64_t outputSide = op.getOutputSide(); + int64_t kernel = op.getKernel(); + int64_t stride = op.getStride(); + int64_t padding = op.getPadding(); + int64_t startRow = op.getStartRow(); + int64_t startCol = op.getStartCol(); + auto iterOp = op.getIter().getDefiningOp(); + auto iterAttr = + iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); + if (!iterAttr || inputSide <= 0 || inputSide > 15 || outputSide <= 0 || + outputSide > 15 || kernel <= 0 || kernel > 15 || stride <= 0 || + stride > 15 || padding < 0 || padding > 15 || startRow < 0 || + startRow > 15 || startCol < 0 || startCol > 15) + return op.emitError("MaxPool shape does not fit one physical bank tile"); + + int64_t lastSourceRow = startRow + (outputSide - 1) * stride + kernel - 1; + int64_t lastSourceCol = startCol + (outputSide - 1) * stride + kernel - 1; + int64_t inputEnd = padding + inputSide; + bool readsInput = startRow < inputEnd && startCol < inputEnd && + lastSourceRow >= padding && lastSourceCol >= padding; + int64_t inputFootprint = 0; + if (readsInput) { + int64_t maxInputRow = std::min(lastSourceRow, inputEnd - 1) - padding; + int64_t maxInputCol = std::min(lastSourceCol, inputEnd - 1) - padding; + inputFootprint = maxInputRow * inputSide + maxInputCol + 1; + } + if (inputFootprint > bankDepth || + outputSide * outputSide != iterAttr.getInt() || + inputSide + 2 * padding < kernel + startRow || + inputSide + 2 * padding < kernel + startCol || + startRow + (outputSide - 1) * stride + kernel > + inputSide + 2 * padding || + startCol + (outputSide - 1) * stride + kernel > inputSide + 2 * padding) + return op.emitError("MaxPool shape does not fit one physical bank tile"); + + Location loc = op.getLoc(); + Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), + cstI64(rewriter, loc, 0), + adaptor.getOutputBankId(), adaptor.getIter()); + rs1 = + rewriter.create(loc, rs1, cstI64(rewriter, loc, INT64_MIN)); + uint64_t geometry = static_cast(inputSide) | + (static_cast(outputSide) << 4) | + (static_cast(kernel) << 8) | + (static_cast(stride) << 12) | + (static_cast(padding) << 16) | + (static_cast(startRow) << 38) | + (static_cast(startCol) << 42); + Value rs2 = cstI64(rewriter, loc, geometry); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getInputBase(), + cstI64(rewriter, loc, 20))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getOutputBase(), + cstI64(rewriter, loc, 26))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getOutputStride(), + cstI64(rewriter, loc, 32))); + rewriter.replaceOpWithNewOp( + op, rs1, rs2, + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("MAXPOOL"))); + return success(); + } + +private: + int64_t bankDepth; +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateMaxPoolBallLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, + int64_t bankDepth) { + (void)stable; + + patterns.add(converter, bankDepth); +} + +void configureMaxPoolBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool stable) { + (void)stable; + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/maxpool/stack/src/Passes.cpp b/waste/examples/components/balls/maxpool/stack/src/Passes.cpp new file mode 100644 index 000000000..70136ea2d --- /dev/null +++ b/waste/examples/components/balls/maxpool/stack/src/Passes.cpp @@ -0,0 +1,39 @@ +//===- Passes.cpp - MaxPoolBall passes ------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +namespace buckyball { +void populateMaxPoolBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureMaxPoolBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LowerMaxPoolPass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerMaxPoolPass) + StringRef getArgument() const final { return "lower-maxpool-ball"; } + StringRef getDescription() const final { + return "Lower MaxPoolBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateMaxPoolBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureMaxPoolBallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + return true; +}(); diff --git a/waste/examples/components/balls/mxfp2int/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/mxfp2int/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..7c140d4dd --- /dev/null +++ b/waste/examples/components/balls/mxfp2int/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,18 @@ +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankMxfp2IntOp::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankMxfp2IntOp op = *this; + rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), + op.getIter(), op.getSpecial()); + rewriter.replaceOp(op, op.getOutBank()); + return success(); +} diff --git a/examples/balls/mxfp2int/compiler/src/Dialect/Buckyball/Mxfp2IntBall.td b/waste/examples/components/balls/mxfp2int/stack/src/Dialect/Buckyball/Mxfp2IntBall.td similarity index 89% rename from examples/balls/mxfp2int/compiler/src/Dialect/Buckyball/Mxfp2IntBall.td rename to waste/examples/components/balls/mxfp2int/stack/src/Dialect/Buckyball/Mxfp2IntBall.td index 75e4b813c..689f71f66 100644 --- a/examples/balls/mxfp2int/compiler/src/Dialect/Buckyball/Mxfp2IntBall.td +++ b/waste/examples/components/balls/mxfp2int/stack/src/Dialect/Buckyball/Mxfp2IntBall.td @@ -16,7 +16,7 @@ def Mxfp2IntOp : Buckyball_Op<"mxfp2int"> { let assemblyFormat = "$inputBankId `,` $outputBankId `,` $iter `,` $special attr-dict `:` type($inputBankId)"; } -def BankMxfp2IntOp : Buckyball_Op<"bank_mxfp2int"> { +def BankMxfp2IntOp : Buckyball_Op<"bank_mxfp2int", [DeclareOpInterfaceMethods]> { let summary = "MXFP to INT8 dequantization using bank-SSA handles"; let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, I64:$special); let results = (outs I64:$outBankOut); diff --git a/waste/examples/components/balls/mxfp2int/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/mxfp2int/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..9b9b21ea6 --- /dev/null +++ b/waste/examples/components/balls/mxfp2int/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,50 @@ +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +class Mxfp2IntLowering : public ConvertOpToLLVMPattern { +public: + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + LogicalResult + matchAndRewrite(Mxfp2IntOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), + cstI64(rewriter, loc, 0), + adaptor.getOutputBankId(), adaptor.getIter()); + rewriter.replaceOpWithNewOp( + op, rs1, adaptor.getSpecial(), + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("MXFP2INT"))); + return success(); + } +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateMxfp2IntBallLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, + int64_t bankDepth) { + (void)stable; + (void)bankDepth; + + patterns.add(converter); +} + +void configureMxfp2IntBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool stable) { + (void)stable; + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/mxfp2int/stack/src/Passes.cpp b/waste/examples/components/balls/mxfp2int/stack/src/Passes.cpp new file mode 100644 index 000000000..c1a285ac1 --- /dev/null +++ b/waste/examples/components/balls/mxfp2int/stack/src/Passes.cpp @@ -0,0 +1,39 @@ +//===- Passes.cpp - Mxfp2IntBall passes -----------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +namespace buckyball { +void populateMxfp2IntBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, + bool, int64_t); +void configureMxfp2IntBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LowerMxfp2IntPass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerMxfp2IntPass) + StringRef getArgument() const final { return "lower-mxfp2int-ball"; } + StringRef getDescription() const final { + return "Lower Mxfp2IntBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateMxfp2IntBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureMxfp2IntBallLegalizeForExportTarget( + target, stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + return true; +}(); diff --git a/examples/balls/mxmm/compiler/python/fp32.py b/waste/examples/components/balls/mxmm/stack/python/fp32.py similarity index 100% rename from examples/balls/mxmm/compiler/python/fp32.py rename to waste/examples/components/balls/mxmm/stack/python/fp32.py diff --git a/examples/balls/mxmm/compiler/python/grouped_attention.py b/waste/examples/components/balls/mxmm/stack/python/grouped_attention.py similarity index 100% rename from examples/balls/mxmm/compiler/python/grouped_attention.py rename to waste/examples/components/balls/mxmm/stack/python/grouped_attention.py diff --git a/examples/balls/mxmm/compiler/python/layout.py b/waste/examples/components/balls/mxmm/stack/python/layout.py similarity index 83% rename from examples/balls/mxmm/compiler/python/layout.py rename to waste/examples/components/balls/mxmm/stack/python/layout.py index f506ac778..a1f3559a2 100644 --- a/examples/balls/mxmm/compiler/python/layout.py +++ b/waste/examples/components/balls/mxmm/stack/python/layout.py @@ -75,24 +75,3 @@ def pack(codes, scales, layout): torch.nn.functional.pad(payload, (0, stride - payload.numel())) ) return torch.cat(panels) - - -def bank_bytes(compiler_build, target): - import re - from pathlib import Path - - text = ( - Path(compiler_build) / "external_dialects/BuckyballTargetRegistry.inc" - ).read_text() - configs = { - name: int(width) * int(depth) // 8 - for name, width, depth in re.findall( - r'\{"([^"]+)", "[^"]+", \d+, (\d+), (\d+), \d+, (?:true|false), llvm::ArrayRef', - text, - ) - } - if target not in configs: - raise ValueError( - f"Target {target} has no registered bank geometry in {compiler_build}" - ) - return configs[target] diff --git a/waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..58fb2fb38 --- /dev/null +++ b/waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,39 @@ +#include "Buckyball/BuckyballOps.h" +#include "Conversion/LowerBuckyball/LowerBuckyball.h" +#include "mlir/IR/PatternMatch.h" +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankMXFP8Op::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankMXFP8Op op = *this; + rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), + op.getWrBank(), op.getConfig(), op.getFirst(), + op.getLast(), op.getOutputBase()); + rewriter.replaceOp(op, op.getWrBank()); + return success(); +} + +LogicalResult +BankMXFP8WindowOp::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankMXFP8WindowOp op = *this; + rewriter.create( + op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), op.getWrBank(), + op.getRows(), op.getCols(), op.getCount(), op.getFullK(), op.getStartK(), + op.getFirst(), op.getLast(), op.getOutputBase()); + rewriter.replaceOp(op, op.getWrBank()); + return success(); +} + +LogicalResult +BankFP32Op::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankFP32Op op = *this; + rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), + op.getWrBank(), op.getConfig(), op.getFirst(), + op.getLast(), op.getOutputBase(), op.getFusedAttr()); + rewriter.replaceOp(op, op.getWrBank()); + return success(); +} diff --git a/waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp b/waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp new file mode 100644 index 000000000..a70ca1ad9 --- /dev/null +++ b/waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp @@ -0,0 +1,672 @@ +#include "Buckyball/BuckyballOps.h" +#include "Conversion/LowerBuckyball/LowerBuckyball.h" +#include "Target/BuckyballTargetRegistry.h" +#include "Utils/BankUtils.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include +#include +#include +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult FP32MemMatmulOp::lowerToBankSSA(PatternRewriter &b) { + FP32MemMatmulOp op = *this; + auto a = cast(op.getLhs().getType()), + w = cast(op.getRhs().getType()), + out = cast(op.getOutput().getType()); + int rank = a.getRank(); + if ((rank != 2 && rank != 3) || w.getRank() != rank || + out.getRank() != rank || !a.hasStaticShape() || !w.hasStaticShape() || + !out.hasStaticShape() || !a.getElementType().isF32() || + !w.getElementType().isF32() || !out.getElementType().isF32()) + return op.emitError( + "FP32 matmul requires static rank-2 or rank-3 FP32 operands"); + int64_t M = a.getShape()[rank - 2], K = a.getShape()[rank - 1], + N = w.getShape()[rank - 1], batches = rank == 3 ? a.getShape()[0] : 1; + if (M <= 0 || N <= 0 || K <= 0 || K % 4 || w.getShape()[rank - 2] != K || + out.getShape()[rank - 2] != M || out.getShape()[rank - 1] != N || + (rank == 3 && + (w.getShape()[0] != batches || out.getShape()[0] != batches))) + return op.emitError( + "FP32 matmul shape mismatch or K not divisible by four"); + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128) + return op.emitError("FP32 matmul requires 128-bit bank rows"); + int64_t bytes = target.bankDepth * 16, rows = 0, columns = 0, chunk = 0; + std::tuple best{INT64_MAX, 0, 0, 0}; + for (int64_t m = M == 1 ? 1 : 16; m <= (M == 1 ? 1 : (M + 15) / 16 * 16); + m += 16) + for (int64_t n = 16; n <= (N + 15) / 16 * 16; n += 16) { + if (m * n * 4 > bytes) + continue; + int64_t k = std::min(K, bytes / (4 * std::max(m, n))) / 4 * 4; + if (!k) + continue; + auto score = + std::tuple{((M + m - 1) / m) * ((N + n - 1) / n) * ((K + k - 1) / k), + -k, -m, -n}; + if (score < best) { + best = score; + rows = m; + columns = n; + chunk = k; + } + } + if (!rows) + return op.emitError("FP32 matmul cannot fit a bank block"); + SmallVector rhsStrides; + int64_t rhsOffset; + bool bankPack = target.bankNum >= 5 && target.rvvEnabled && + succeeded(w.getStridesAndOffset(rhsStrides, rhsOffset)) && + rhsStrides[rank - 2] == N && rhsStrides.back() == 1 && + K * N * 4 <= bytes && chunk == K && rows * columns * 4 >= 64; + Location loc = op.getLoc(); + auto idx = [&](int64_t v) -> Value { + return b.create(loc, v); + }; + Value zero = idx(0), one = idx(1), four = idx(4); + Value ab = allocBank(b, loc, 1, 1), wb = allocBank(b, loc, 1, 1), + cb = allocBank(b, loc, 1, 1); + auto packType = MemRefType::get({bytes / 16, 4}, b.getF32Type()); + Value ap = b.create(loc, packType), + bp = b.create(loc, packType), + cp = b.create(loc, packType); + for (Value buffer : {ap, bp, cp}) + buffer.getDefiningOp()->setAttr("alignment", b.getI64IntegerAttr(bytes)); + SmallVector banks{ab, wb, cb}; + if (bankPack) { + Value input = allocBank(b, loc, 1, 1); + Value descriptor = allocBank(b, loc, 1, 1); + Value read = b.create(loc, b.getI64Type(), ab, input); + Value write = b.create(loc, b.getI64Type(), wb, descriptor); + write = b.create(loc, b.getI64Type(), cb, write); + auto loaded = b.create( + loc, b.getI64Type(), b.getI64Type(), read, write, + FlatSymbolRefAttr::get(b.getContext(), "rvv_packmm"), + ValueRange{createI64Const(b, loc, 0), createI64Const(b, loc, K), + createI64Const(b, loc, N), createI64Const(b, loc, 0), + createI64Const(b, loc, 0), createI64Const(b, loc, K), + createI64Const(b, loc, columns), + createI64Const(b, loc, rows), createI64Const(b, loc, 1), + createI64Const(b, loc, 1), + createI64Const(b, loc, op.getFusedAttr().getValue())}); + banks = {loaded.getReadOut(), loaded.getWriteOut()}; + } + auto batchLoop = b.create(loc, zero, idx(batches), one, banks); + b.setInsertionPointToStart(batchLoop.getBody()); + Value batch = batchLoop.getInductionVar(); + SmallVector batchBanks(batchLoop.getRegionIterArgs()); + if (bankPack) { + SmallVector offsets(rank, b.getIndexAttr(0)); + SmallVector sizes; + for (int64_t dim : w.getShape()) + sizes.push_back(b.getIndexAttr(dim)); + SmallVector strides(rank, b.getIndexAttr(1)); + if (rank == 3) { + offsets[0] = batch; + sizes[0] = b.getIndexAttr(1); + } + auto rowType = memref::SubViewOp::inferRankReducedResultType( + {K, N}, w, offsets, sizes, strides); + Value source = b.create(loc, rowType, op.getRhs(), + offsets, sizes, strides); + Value flat = b.create( + loc, source, ArrayRef{{0, 1}}); + Value packed = b.create( + loc, ArrayRef{K * N / 4, 4}, flat, + ArrayRef{{0, 1}}); + batchBanks[0] = + b.create(loc, b.getI64Type(), packed, batchBanks[0], + createI64Const(b, loc, K * N / 4), + createI64Const(b, loc, 1), b.getI64IntegerAttr(0)); + } + auto rowLoop = b.create(loc, zero, idx(M), idx(rows), batchBanks); + b.setInsertionPointToStart(rowLoop.getBody()); + Value row = rowLoop.getInductionVar(); + auto colLoop = b.create(loc, zero, idx(N), idx(columns), + rowLoop.getRegionIterArgs()); + b.setInsertionPointToStart(colLoop.getBody()); + Value column = colLoop.getInductionVar(); + ab = colLoop.getRegionIterArgs()[0]; + wb = colLoop.getRegionIterArgs()[1]; + cb = bankPack ? wb : colLoop.getRegionIterArgs()[2]; + for (int64_t begin = 0; begin < K; begin += chunk) { + int64_t count = std::min(chunk, K - begin); + for (int operand = 0; operand < (bankPack ? 1 : 2); ++operand) { + Value source = operand ? op.getRhs() : op.getLhs(); + Value packed = operand ? bp : ap; + auto sourceType = cast(source.getType()); + int axis = operand ? rank - 2 : rank - 1; + SmallVector strides; + int64_t offset; + if (failed(sourceType.getStridesAndOffset(strides, offset))) + return op.emitError("FP32 matmul requires strided operands"); + int64_t blockRows = operand ? columns : rows; + int64_t logicalRows = operand ? N : M; + Value base = operand ? column : row; + Value validRows = b.create( + loc, idx(blockRows), + b.create(loc, idx(logicalRows), base)); + auto flatType = MemRefType::get({bytes / 4}, b.getF32Type()); + Value flatBuffer = b.create( + loc, flatType, packed, ArrayRef{{0, 1}}); + auto rLoop = b.create(loc, zero, validRows, one); + b.setInsertionPointToStart(rLoop.getBody()); + Value r = rLoop.getInductionVar(); + Value global = b.create(loc, base, r); + Value flat = b.create(loc, r, idx(count)); + if (strides[axis] == 1) { + SmallVector sourceOffsets(rank, b.getIndexAttr(0)); + SmallVector sourceSizes(rank, b.getIndexAttr(1)); + SmallVector sourceStrides(rank, b.getIndexAttr(1)); + if (rank == 3) + sourceOffsets[0] = batch; + sourceOffsets[rank - 2] = operand ? OpFoldResult(b.getIndexAttr(begin)) + : OpFoldResult(global); + sourceOffsets[rank - 1] = operand ? OpFoldResult(global) + : OpFoldResult(b.getIndexAttr(begin)); + sourceSizes[axis] = b.getIndexAttr(count); + auto sourceRowType = memref::SubViewOp::inferRankReducedResultType( + {count}, sourceType, sourceOffsets, sourceSizes, sourceStrides); + Value sourceRow = b.create( + loc, sourceRowType, source, sourceOffsets, sourceSizes, + sourceStrides); + SmallVector targetOffsets{flat}; + SmallVector targetSizes{b.getIndexAttr(count)}; + SmallVector targetStrides{b.getIndexAttr(1)}; + auto targetRowType = memref::SubViewOp::inferRankReducedResultType( + {count}, flatType, targetOffsets, targetSizes, targetStrides); + Value targetRow = b.create( + loc, targetRowType, flatBuffer, targetOffsets, targetSizes, + targetStrides); + b.create(loc, sourceRow, targetRow); + } else { + auto kLoop = b.create(loc, zero, idx(count), one); + b.setInsertionPointToStart(kLoop.getBody()); + Value k = kLoop.getInductionVar(); + Value kk = b.create(loc, k, idx(begin)); + SmallVector indices; + if (rank == 3) + indices.push_back(batch); + indices.push_back(operand ? kk : global); + indices.push_back(operand ? global : kk); + Value value = b.create(loc, source, indices); + Value destination = b.create(loc, flat, k); + b.create(loc, value, flatBuffer, + ValueRange{destination}); + } + b.setInsertionPointAfter(rLoop); + if (logicalRows % blockRows) { + Value start = b.create(loc, validRows, idx(count * 4)); + Value pointer = + b.create(loc, packed); + Value address = b.create(loc, pointer, start); + Value integer = + b.create(loc, b.getI64Type(), address); + Value destination = b.create( + loc, LLVM::LLVMPointerType::get(b.getContext()), integer); + Value length = + b.create(loc, idx(blockRows * count * 4), start); + Value byteLength = + b.create(loc, b.getI64Type(), length); + Value byteZero = b.create(loc, 0, 8); + b.create(loc, destination, byteZero, byteLength, false); + } + } + if (bankPack) { + ab = b.create( + loc, b.getI64Type(), ap, ab, createI64Const(b, loc, rows * count / 4), + createI64Const(b, loc, 1), b.getI64IntegerAttr(1)); + auto packed = b.create( + loc, b.getI64Type(), b.getI64Type(), ab, wb, + FlatSymbolRefAttr::get(b.getContext(), "rvv_packmm"), + ValueRange{ + createI64Const(b, loc, 1), createI64Const(b, loc, K), + createI64Const(b, loc, N), createI64Const(b, loc, begin), + b.create(loc, b.getI64Type(), column), + createI64Const(b, loc, count), createI64Const(b, loc, columns), + createI64Const(b, loc, rows), createI64Const(b, loc, begin == 0), + createI64Const(b, loc, begin + count == K), + createI64Const(b, loc, op.getFusedAttr().getValue())}); + ab = packed.getReadOut(); + wb = cb = packed.getWriteOut(); + } else { + ab = mvinBank(b, loc, ap, ab, rows * count / 4); + wb = mvinBank(b, loc, bp, wb, columns * count / 4); + cb = b.create(loc, b.getI64Type(), ab, wb, cb, + createI64Const(b, loc, + packBits(rows, 0, 11) | + packBits(columns, 12, 23) | + packBits(count, 24, 35)), + createI1Const(b, loc, begin == 0), + createI1Const(b, loc, begin + count == K), + createI64Const(b, loc, 0), op.getFusedAttr()) + .getWrBankOut(); + } + } + if (bankPack) + cb = b.create( + loc, b.getI64Type(), cp, cb, createI64Const(b, loc, rows * columns / 4), + createI64Const(b, loc, 1), b.getI64IntegerAttr(2)); + else + cb = mvoutBank(b, loc, cp, cb, rows * columns / 4); + SmallVector outputStrides; + int64_t outputOffset; + if (failed(out.getStridesAndOffset(outputStrides, outputOffset))) + return op.emitError("FP32 matmul requires a strided output"); + Value validColumns = b.create( + loc, idx(columns), b.create(loc, idx(N), column)); + auto rr = b.create( + loc, zero, + b.create(loc, idx(rows), + b.create(loc, idx(M), row)), + one); + b.setInsertionPointToStart(rr.getBody()); + Value r = rr.getInductionVar(); + if (outputStrides.back() == 1) { + auto flatType = MemRefType::get({bytes / 4}, b.getF32Type()); + Value flat = b.create( + loc, flatType, cp, ArrayRef{{0, 1}}); + SmallVector offsets{ + b.create(loc, r, idx(columns)).getResult()}; + SmallVector sizes{validColumns}, strides{b.getIndexAttr(1)}; + auto sourceType = memref::SubViewOp::inferRankReducedResultType( + {ShapedType::kDynamic}, flatType, offsets, sizes, strides); + Value source = b.create(loc, sourceType, flat, offsets, + sizes, strides); + SmallVector outOffsets(rank, b.getIndexAttr(0)); + SmallVector outSizes(rank, b.getIndexAttr(1)); + SmallVector outSteps(rank, b.getIndexAttr(1)); + if (rank == 3) + outOffsets[0] = batch; + outOffsets[rank - 2] = b.create(loc, row, r).getResult(); + outOffsets[rank - 1] = column; + outSizes.back() = validColumns; + auto rowType = memref::SubViewOp::inferRankReducedResultType( + {ShapedType::kDynamic}, out, outOffsets, outSizes, outSteps); + Value destination = b.create( + loc, rowType, op.getOutput(), outOffsets, outSizes, outSteps); + b.create(loc, source, destination); + } else { + auto cc = b.create(loc, zero, validColumns, one); + b.setInsertionPointToStart(cc.getBody()); + Value c = cc.getInductionVar(); + Value flat = b.create( + loc, b.create(loc, r, idx(columns)), c); + Value result = b.create( + loc, cp, + ValueRange{b.create(loc, flat, four), + b.create(loc, flat, four)}); + SmallVector outputIndices; + if (rank == 3) + outputIndices.push_back(batch); + outputIndices.push_back(b.create(loc, row, r)); + outputIndices.push_back(b.create(loc, column, c)); + b.create(loc, result, op.getOutput(), outputIndices); + } + b.setInsertionPointAfter(rr); + SmallVector nextBanks{ab, wb, cb}; + if (bankPack) + nextBanks = {ab, cb}; + b.create(loc, nextBanks); + b.setInsertionPointAfter(colLoop); + b.create(loc, colLoop.getResults()); + b.setInsertionPointAfter(rowLoop); + b.create(loc, rowLoop.getResults()); + b.setInsertionPointAfter(batchLoop); + if (bankPack) { + auto finished = b.create( + loc, b.getI64Type(), b.getI64Type(), batchLoop.getResult(0), + batchLoop.getResult(1), + FlatSymbolRefAttr::get(b.getContext(), "rvv_packmm"), + ValueRange{createI64Const(b, loc, 2), createI64Const(b, loc, K), + createI64Const(b, loc, N), createI64Const(b, loc, 0), + createI64Const(b, loc, 0), createI64Const(b, loc, K), + createI64Const(b, loc, columns), + createI64Const(b, loc, rows), createI64Const(b, loc, 1), + createI64Const(b, loc, 1), + createI64Const(b, loc, op.getFusedAttr().getValue())}); + releaseBank(b, loc, finished.getReadOut()); + releaseBank(b, loc, finished.getWriteOut()); + } else { + for (Value bank : batchLoop.getResults()) + releaseBank(b, loc, bank); + } + for (Value buffer : {ap, bp, cp}) + b.create(loc, buffer); + b.eraseOp(op); + return success(); +} + +LogicalResult MXFP8MemMatmulOp::lowerToBankSSA(PatternRewriter &b) { + MXFP8MemMatmulOp op = *this; + auto aTy = cast(op.getInput().getType()); + auto wTy = cast(op.getWeight().getType()); + auto oTy = cast(op.getOutput().getType()); + if (!aTy.hasStaticShape() || !wTy.hasStaticShape() || !oTy.hasStaticShape() || + aTy.getRank() != 1 || wTy.getRank() != 1 || oTy.getRank() != 2 || + !aTy.getElementType().isInteger(8) || + !wTy.getElementType().isInteger(8) || !oTy.getElementType().isF32()) + return op.emitError("MXFP8 requires static packed activation/weight bytes " + "and an FP32 output matrix"); + if (aTy.getStridesAndOffset().first[0] != 1 || + wTy.getStridesAndOffset().first[0] != 1) + return op.emitError("MXFP8 packed tensors must have unit byte stride"); + int64_t M = oTy.getShape()[0], K = op.getReductionK(), N = oTy.getShape()[1]; + int64_t rows = M == 1 ? 1 : op.getTileM(), columns = op.getTileN(); + int64_t chunk = op.getTileK(), activationChunk = op.getActivationTileK(); + bool windowInput = activationChunk != chunk; + if (rows <= 0 || (rows != 1 && rows % 16) || columns <= 0 || columns % 16 || + chunk <= 0 || chunk % 32 || chunk >= 4096) + return op.emitError( + "MXFP8 tile_k must be a positive multiple of 32 below 4096"); + int64_t chunks = (K + chunk - 1) / chunk; + int64_t stride = op.getBankBytes(); + if (windowInput && (activationChunk != K || K > 65535 || + (rows * K * 33 / 32 + 15) / 16 * 16 > stride)) + return op.emitError("MXFP8 activation window must hold the complete row"); + int64_t activationChunks = windowInput ? 1 : chunks; + if (M <= 0 || K <= 0 || N <= 0 || K % 32 || + aTy.getShape()[0] != + ((M + rows - 1) / rows) * activationChunks * stride || + wTy.getShape()[0] != ((N + columns - 1) / columns) * chunks * stride) + return op.emitError("MXFP8 matrix or packed weight shape mismatch"); + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || target.bankDepth * 16 != stride || + (rows != 1 && (rows <= 0 || rows % 16)) || columns <= 0 || columns % 16 || + (rows * chunk * 33 / 32 + 15) / 16 * 16 > stride || + columns * chunk * 33 / 32 > stride || rows * columns * 4 > stride) + return op.emitError( + "MXFP8 packed K block exceeds the 128-bit bank capacity"); + Location loc = op.getLoc(); + Value zero = b.create(loc, 0), + one = b.create(loc, 1); + Value packed = op.getInput(); + auto dynamic = ShapedType::kDynamic; + auto bankView = [&](Value source, Value offset, int64_t size) -> Value { + Value slice = b.create( + loc, source, SmallVector{offset}, + SmallVector{b.getIndexAttr(stride)}, + SmallVector{b.getIndexAttr(1)}); + auto ty = MemRefType::get( + {stride / 16, 16}, b.getI8Type(), + StridedLayoutAttr::get(b.getContext(), dynamic, {16, 1})); + return b.create( + loc, ty, slice, ArrayRef{{0, 1}}); + }; + Value aBank = allocBank(b, loc, 1, 1), wBank = allocBank(b, loc, 1, 1), + cBank = allocBank(b, loc, 1, 1); + bool residentInput = activationChunks + 2 <= target.bankNum; + SmallVector banks{aBank}; + if (residentInput) + for (int64_t part = 1; part < activationChunks; ++part) + banks.push_back(allocBank(b, loc, 1, 1)); + banks.push_back(wBank); + banks.push_back(cBank); + auto outPackTy = MemRefType::get({stride / 16, 4}, b.getF32Type()); + Value outPack = b.create(loc, outPackTy); + outPack.getDefiningOp()->setAttr("alignment", b.getI64IntegerAttr(stride)); + Value limit = b.create(loc, N), + step = b.create(loc, columns); + Value rowLimit = b.create(loc, M), + rowStep = b.create(loc, rows); + auto rowBlocks = b.create(loc, zero, rowLimit, rowStep, banks); + b.setInsertionPointToStart(rowBlocks.getBody()); + Value m = rowBlocks.getInductionVar(); + SmallVector inputBanks(rowBlocks.getRegionIterArgs().drop_back(2)); + aBank = inputBanks.front(); + wBank = rowBlocks.getRegionIterArgs()[banks.size() - 2]; + cBank = rowBlocks.getRegionIterArgs()[banks.size() - 1]; + Value rowPanel = b.create(loc, m, rowStep); + if (residentInput) { + for (int64_t part = 0; part < activationChunks; ++part) { + Value offset = b.create( + loc, + b.create( + loc, + b.create( + loc, rowPanel, + b.create(loc, activationChunks)), + b.create(loc, part)), + b.create(loc, stride)); + int64_t count = windowInput ? K : std::min(chunk, K - part * chunk); + int64_t aBytes = (rows * count * 33 / 32 + 15) / 16 * 16; + inputBanks[part] = mvinBank(b, loc, bankView(packed, offset, stride), + inputBanks[part], aBytes / 16); + } + aBank = inputBanks.front(); + } + auto panels = b.create(loc, zero, limit, step, + ValueRange{aBank, wBank, cBank}); + b.setInsertionPointToStart(panels.getBody()); + Value n = panels.getInductionVar(); + aBank = panels.getRegionIterArgs()[0]; + wBank = panels.getRegionIterArgs()[1]; + cBank = panels.getRegionIterArgs()[2]; + Value state = cBank; + for (int64_t part = 0; part < chunks; ++part) { + int64_t count = std::min(chunk, K - part * chunk); + int64_t aBytes = (rows * count * 33 / 32 + 15) / 16 * 16, + bBytes = columns * count * 33 / 32; + Value aOffset = b.create( + loc, + b.create( + loc, + b.create( + loc, rowPanel, b.create(loc, chunks)), + b.create(loc, part)), + b.create(loc, stride)); + Value panel = b.create(loc, n, step); + Value wOffset = b.create( + loc, + b.create( + loc, + b.create( + loc, panel, b.create(loc, chunks)), + b.create(loc, part)), + b.create(loc, stride)); + if (!residentInput) + aBank = mvinBank(b, loc, bankView(packed, aOffset, aBytes), aBank, + aBytes / 16); + wBank = mvinBank(b, loc, bankView(op.getWeight(), wOffset, bBytes), wBank, + bBytes / 16); + if (windowInput) + state = b.create( + loc, b.getI64Type(), aBank, wBank, state, + createI64Const(b, loc, rows), createI64Const(b, loc, columns), + createI64Const(b, loc, count), createI64Const(b, loc, K), + createI64Const(b, loc, part * chunk), + createI1Const(b, loc, part == 0), + createI1Const(b, loc, part == chunks - 1), createI64Const(b, loc, 0)); + else + state = b.create( + loc, b.getI64Type(), residentInput ? inputBanks[part] : aBank, wBank, + state, createI64Const(b, loc, matrixRs2(rows, columns, count)), + createI1Const(b, loc, part == 0), + createI1Const(b, loc, part == chunks - 1), createI64Const(b, loc, 0)); + } + cBank = mvoutBank(b, loc, outPack, state, rows * columns / 4); + Value end = b.create(loc, step, + b.create(loc, limit, n)); + SmallVector outputStrides; + int64_t outputOffset; + if (failed(oTy.getStridesAndOffset(outputStrides, outputOffset))) + return op.emitError("MXFP8 matmul requires a strided output"); + auto rr = b.create( + loc, zero, + b.create(loc, rowStep, + b.create(loc, rowLimit, m)), + one); + b.setInsertionPointToStart(rr.getBody()); + Value r = rr.getInductionVar(); + if (outputStrides.back() == 1) { + auto flatType = MemRefType::get({stride / 4}, b.getF32Type()); + Value flat = b.create( + loc, flatType, outPack, ArrayRef{{0, 1}}); + SmallVector offsets{ + b.create(loc, r, + b.create(loc, columns)) + .getResult()}; + SmallVector sizes{end}, steps{b.getIndexAttr(1)}; + auto sourceType = memref::SubViewOp::inferRankReducedResultType( + {ShapedType::kDynamic}, flatType, offsets, sizes, steps); + Value source = b.create(loc, sourceType, flat, offsets, + sizes, steps); + SmallVector outOffsets{ + b.create(loc, m, r).getResult(), n}; + SmallVector outSizes{b.getIndexAttr(1), end}; + SmallVector outSteps(2, b.getIndexAttr(1)); + auto rowType = memref::SubViewOp::inferRankReducedResultType( + {ShapedType::kDynamic}, oTy, outOffsets, outSizes, outSteps); + Value destination = b.create( + loc, rowType, op.getOutput(), outOffsets, outSizes, outSteps); + b.create(loc, source, destination); + } else { + auto cc = b.create(loc, zero, end, one); + b.setInsertionPointToStart(cc.getBody()); + Value c = cc.getInductionVar(), + four = b.create(loc, 4); + Value pr = b.create( + loc, + b.create( + loc, r, b.create(loc, columns / 4)), + b.create(loc, c, four)); + Value pc = b.create(loc, c, four); + Value value = b.create(loc, outPack, ValueRange{pr, pc}); + b.create(loc, value, op.getOutput(), + ValueRange{b.create(loc, r, m), + b.create(loc, n, c)}); + } + b.setInsertionPointAfter(rr); + b.create(loc, ValueRange{aBank, wBank, cBank}); + b.setInsertionPointAfter(panels); + inputBanks[0] = panels.getResult(0); + inputBanks.push_back(panels.getResult(1)); + inputBanks.push_back(panels.getResult(2)); + b.create(loc, inputBanks); + b.setInsertionPointAfter(rowBlocks); + for (Value bank : rowBlocks.getResults()) + releaseBank(b, loc, bank); + b.create(loc, outPack); + b.eraseOp(op); + return success(); +} + +LogicalResult MXFP8SharedPanelOp::lowerToBankSSA(PatternRewriter &b) { + MXFP8SharedPanelOp op = *this; + auto a = cast(op.getInput().getType()); + auto w = cast(op.getWeight().getType()); + int64_t rows = op.getRows(), columns = op.getColumns(); + int64_t k = op.getReductionK(), chunk = op.getTileK(); + int64_t bytes = op.getBankBytes(); + int64_t sharedBytes = op.getSharedBytesAttr().getInt(); + const auto &target = buckyball_target::getBuckyballTarget(); + if ((rows != 1 && (rows <= 0 || rows % 16)) || columns <= 0 || columns % 16 || + rows > 4095 || columns > 4095 || k <= 0 || k % 32 || chunk <= 0 || + chunk % 32 || chunk >= 4096 || target.bankWidthBits != 128 || + target.bankDepth * 16 != bytes || sharedBytes <= 0 || sharedBytes % 16 || + rows * columns * 4 > sharedBytes || + (rows * chunk * 33 / 32 + 15) / 16 * 16 > bytes || + columns * chunk * 33 / 32 > bytes) + return op.emitError("shared MXFP8 panel exceeds the original bank layout"); + int64_t chunks = (k + chunk - 1) / chunk; + for (MemRefType type : {a, w}) + if (!type.hasStaticShape() || type.getRank() != 1 || + !type.getElementType().isInteger(8) || + type.getShape()[0] != chunks * bytes || + type.getStridesAndOffset().first[0] != 1) + return op.emitError("shared MXFP8 panel requires one packed row/N panel"); + Location loc = op.getLoc(); + Value ab = allocBank(b, loc, 1, 1), wb = allocBank(b, loc, 1, 1); + Value output = op.getBank(); + for (int64_t part = 0; part < chunks; ++part) { + int64_t count = std::min(chunk, k - part * chunk); + auto view = [&](Value packed) -> Value { + Value slice = b.create( + loc, packed, SmallVector{b.getIndexAttr(part * bytes)}, + SmallVector{b.getIndexAttr(bytes)}, + SmallVector{b.getIndexAttr(1)}); + auto type = *memref::ExpandShapeOp::computeExpandedType( + cast(slice.getType()), {bytes / 16, 16}, + ArrayRef{{0, 1}}); + return b.create( + loc, type, slice, ArrayRef{{0, 1}}); + }; + ab = mvinBank(b, loc, view(op.getInput()), ab, + (rows * count * 33 / 32 + 15) / 16); + wb = mvinBank(b, loc, view(op.getWeight()), wb, + columns * count * 33 / 32 / 16); + output = b.create( + loc, b.getI64Type(), ab, wb, output, + createI64Const(b, loc, matrixRs2(rows, columns, count)), + createI1Const(b, loc, part == 0), + createI1Const(b, loc, part == chunks - 1), createI64Const(b, loc, 0)); + } + releaseBank(b, loc, ab); + releaseBank(b, loc, wb); + b.replaceOp(op, output); + return success(); +} + +LogicalResult MXFP8SharedWindowOp::lowerToBankSSA(PatternRewriter &b) { + MXFP8SharedWindowOp op = *this; + auto a = cast(op.getInput().getType()); + auto w = cast(op.getWeight().getType()); + int64_t rows = op.getRows(), columns = op.getColumns(); + int64_t k = op.getReductionK(), chunk = op.getTileK(); + int64_t bytes = op.getBankBytes(); + int64_t sharedBytes = op.getSharedBytesAttr().getInt(); + const auto &target = buckyball_target::getBuckyballTarget(); + if ((rows != 1 && (rows <= 0 || rows % 16)) || rows > 4095 || columns <= 0 || + columns > 4095 || columns % 16 || k <= 0 || k > 65535 || k % 32 || + chunk <= 0 || chunk >= 4096 || chunk % 32 || + target.bankWidthBits != 128 || target.bankDepth * 16 != bytes || + sharedBytes <= 0 || sharedBytes % 16 || + rows * columns * 4 > sharedBytes || + (rows * k * 33 / 32 + 15) / 16 * 16 > bytes || + columns * chunk * 33 / 32 > bytes) + return op.emitError( + "shared MXFP8 window exceeds its activation or output bank"); + int64_t chunks = (k + chunk - 1) / chunk; + if (!a.hasStaticShape() || !w.hasStaticShape() || a.getShape()[0] != bytes || + w.getShape()[0] != chunks * bytes || + a.getStridesAndOffset().first[0] != 1 || + w.getStridesAndOffset().first[0] != 1) + return op.emitError( + "shared MXFP8 window requires a complete activation and one N panel"); + Location loc = op.getLoc(); + auto view = [&](Value source, int64_t offset) -> Value { + Value slice = b.create( + loc, source, SmallVector{b.getIndexAttr(offset)}, + SmallVector{b.getIndexAttr(bytes)}, + SmallVector{b.getIndexAttr(1)}); + auto type = *memref::ExpandShapeOp::computeExpandedType( + cast(slice.getType()), {bytes / 16, 16}, + ArrayRef{{0, 1}}); + return b.create( + loc, type, slice, ArrayRef{{0, 1}}); + }; + Value ab = allocBank(b, loc, 1, 1), wb = allocBank(b, loc, 1, 1); + ab = mvinBank(b, loc, view(op.getInput(), 0), ab, + (rows * k * 33 / 32 + 15) / 16); + Value output = op.getBank(); + for (int64_t part = 0; part < chunks; ++part) { + int64_t count = std::min(chunk, k - part * chunk); + wb = mvinBank(b, loc, view(op.getWeight(), part * bytes), wb, + columns * count * 33 / 32 / 16); + output = b.create( + loc, b.getI64Type(), ab, wb, output, createI64Const(b, loc, rows), + createI64Const(b, loc, columns), createI64Const(b, loc, count), + createI64Const(b, loc, k), createI64Const(b, loc, part * chunk), + createI1Const(b, loc, part == 0), + createI1Const(b, loc, part == chunks - 1), createI64Const(b, loc, 0)); + } + releaseBank(b, loc, ab); + releaseBank(b, loc, wb); + b.replaceOp(op, output); + return success(); +} diff --git a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/IR/FP32.cpp b/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/IR/FP32.cpp similarity index 100% rename from examples/balls/mxmm/compiler/src/Dialect/Buckyball/IR/FP32.cpp rename to waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/IR/FP32.cpp diff --git a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/IR/MXFP8.cpp b/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/IR/MXFP8.cpp similarity index 100% rename from examples/balls/mxmm/compiler/src/Dialect/Buckyball/IR/MXFP8.cpp rename to waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/IR/MXFP8.cpp diff --git a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/MxmmBall.td b/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/MxmmBall.td similarity index 88% rename from examples/balls/mxmm/compiler/src/Dialect/Buckyball/MxmmBall.td rename to waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/MxmmBall.td index a7576cbfa..a12c70dd8 100644 --- a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/MxmmBall.td +++ b/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/MxmmBall.td @@ -30,13 +30,13 @@ def MXFP8MatmulOp : Buckyball_Op<"mxfp8_matmul", [ let hasVerifier = 1; let assemblyFormat = "$input `,` $weight prop-dict attr-dict `:` type($input) `,` type($weight) `->` type($output)"; } -def MXFP8MemMatmulOp : Buckyball_Op<"mxfp8_mem_matmul"> { +def MXFP8MemMatmulOp : Buckyball_Op<"mxfp8_mem_matmul", [DeclareOpInterfaceMethods]> { let arguments = (ins Arg]>:$input, Arg]>:$weight, Arg]>:$output, I64Attr:$reduction_k, I64Attr:$tile_k, I64Attr:$tile_m, I64Attr:$tile_n, I64Attr:$bank_bytes, I64Attr:$activation_tile_k); let assemblyFormat = "$input `,` $weight `,` $output prop-dict attr-dict `:` type($input) `,` type($weight) `,` type($output)"; } -def MXFP8SharedPanelOp : Buckyball_Op<"mxfp8_shared_panel", [MemoryEffects<[MemRead, MemWrite]>]> { +def MXFP8SharedPanelOp : Buckyball_Op<"mxfp8_shared_panel", [DeclareOpInterfaceMethods, MemoryEffects<[MemRead, MemWrite]>]> { let summary = "Compute one original MXFP8 output panel into a live shared bank"; let arguments = (ins MemRefRankOf<[I8], [1]>:$input, MemRefRankOf<[I8], [1]>:$weight, I64:$bank, @@ -46,7 +46,7 @@ def MXFP8SharedPanelOp : Buckyball_Op<"mxfp8_shared_panel", [MemoryEffects<[MemR let assemblyFormat = "$input `,` $weight `,` $bank prop-dict attr-dict `:` " "type($input) `,` type($weight) `,` type($bank)"; } -def MXFP8SharedWindowOp : Buckyball_Op<"mxfp8_shared_window", [MemoryEffects<[MemRead, MemWrite]>]> { +def MXFP8SharedWindowOp : Buckyball_Op<"mxfp8_shared_window", [DeclareOpInterfaceMethods, MemoryEffects<[MemRead, MemWrite]>]> { let summary = "Compute a shared output panel from one complete MXFP8 activation bank"; let arguments = (ins MemRefRankOf<[I8], [1]>:$input, MemRefRankOf<[I8], [1]>:$weight, I64:$bank, @@ -70,7 +70,7 @@ def MXFP8Op : Buckyball_Op<"mxfp8"> { "type($op1BankId)"; } -def BankMXFP8Op : Buckyball_Op<"bank_mxfp8"> { +def BankMXFP8Op : Buckyball_Op<"bank_mxfp8", [DeclareOpInterfaceMethods]> { let summary = "MxmmBall compute with virtual bank handles"; let arguments = (ins I64:$op1Bank, I64:$op2Bank, I64:$wrBank, I64:$config, I1:$first, I1:$last, I64:$outputBase); @@ -91,7 +91,7 @@ def MXFP8WindowOp : Buckyball_Op<"mxfp8_window"> { "$first `,` $last `,` $outputBase prop-dict attr-dict `:` type($op1BankId)"; } -def BankMXFP8WindowOp : Buckyball_Op<"bank_mxfp8_window", [MemoryEffects<[MemRead, MemWrite]>]> { +def BankMXFP8WindowOp : Buckyball_Op<"bank_mxfp8_window", [DeclareOpInterfaceMethods, MemoryEffects<[MemRead, MemWrite]>]> { let summary = "MXFP8 activation row window with virtual bank handles"; let arguments = (ins I64:$op1Bank, I64:$op2Bank, I64:$wrBank, I64:$rows, I64:$cols, I64:$count, I64:$fullK, I64:$startK, @@ -109,7 +109,7 @@ def FP32MatmulOp : Buckyball_Op<"fp32_matmul", [ let results = (outs AnyRankedTensor:$output); let assemblyFormat = "$lhs `,` $rhs prop-dict attr-dict `:` type($lhs) `,` type($rhs) `->` type($output)"; } -def FP32MemMatmulOp : Buckyball_Op<"fp32_mem_matmul"> { +def FP32MemMatmulOp : Buckyball_Op<"fp32_mem_matmul", [DeclareOpInterfaceMethods]> { let arguments = (ins Arg]>:$lhs, Arg]>:$rhs, Arg]>:$output, BoolAttr:$fused); @@ -129,7 +129,7 @@ def FP32Op : Buckyball_Op<"matmul_f32"> { "type($op1BankId)"; } -def BankFP32Op : Buckyball_Op<"bank_matmul_f32"> { +def BankFP32Op : Buckyball_Op<"bank_matmul_f32", [DeclareOpInterfaceMethods]> { let summary = "MxmmBall compute with virtual bank handles"; let arguments = (ins I64:$op1Bank, I64:$op2Bank, I64:$wrBank, I64:$config, I1:$first, I1:$last, I64:$outputBase, BoolAttr:$fused); diff --git a/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..ca982b05d --- /dev/null +++ b/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,157 @@ +#include +//===- LegalizeForLLVMExport.cpp - MxmmBall LLVM lowering --------------===// + +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" +#include "Utils/BankUtils.h" + +#include "llvm/Support/ErrorHandling.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +uint64_t matrixCfg(uint64_t rows, uint64_t cols, bool first, bool last) { + return fieldBits(rows, 0, 11) | fieldBits(cols, 12, 23) | + (uint64_t(first) << 24) | (uint64_t(last) << 25); +} +struct MXFP8Lowering : public ConvertOpToLLVMPattern { + MXFP8Lowering(LLVMTypeConverter &converter, bool, int64_t bankDepth) + : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} + + LogicalResult + matchAndRewrite(MXFP8Op op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + auto config = op.getConfig().getDefiningOp(); + if (!config) + return op.emitError("mxmm config must be a constant"); + auto configAttr = dyn_cast(config.getValue()); + if (!configAttr) + return op.emitError("mxmm config must be an integer constant"); + uint64_t cfg = configAttr.getValue().getZExtValue(); + uint64_t rows = cfg & 0xfff; + uint64_t cols = (cfg >> 12) & 0xfff; + uint64_t k = (cfg >> 24) & 0xfff; + if (cfg >> 36 || (rows != 1 && (rows == 0 || rows % 16)) || cols == 0 || + cols % 16 || k == 0 || k % 32 || + (rows * k * 33 / 32 + 15) / 16 > static_cast(bankDepth) || + cols * k * 33 / 32 / 16 > static_cast(bankDepth) || + rows * cols / 4 > static_cast(bankDepth)) + return op.emitError("Mxmm matrix shape or C footprint is invalid"); + auto base = adaptor.getOutputBase().getDefiningOp(); + auto baseAttr = + base ? dyn_cast(base.getValue()) : IntegerAttr(); + if (!baseAttr || baseAttr.getInt() < 0 || baseAttr.getInt() > 63 || + baseAttr.getInt() + rows * cols / 4 > bankDepth) + return op.emitError("Mxmm outputBase must be constant and fit C in bank"); + Value rs1 = packRs1BanksIter( + rewriter, loc, adaptor.getOp1BankId(), adaptor.getOp2BankId(), + adaptor.getResultBankId(), cstI64(rewriter, loc, k)); + Value rs2 = cstI64(rewriter, loc, matrixCfg(rows, cols, false, false)); + Value first = rewriter.create(loc, rewriter.getI64Type(), + adaptor.getFirst()); + Value last = rewriter.create(loc, rewriter.getI64Type(), + adaptor.getLast()); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, first, cstI64(rewriter, loc, 24))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, last, cstI64(rewriter, loc, 25))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getOutputBase(), + cstI64(rewriter, loc, 26))); + rewriter.replaceOpWithNewOp( + op, rs1, rs2, + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("MXMM_MXFP8"))); + return success(); + } + +private: + int64_t bankDepth; +}; + +struct FP32Lowering : public ConvertOpToLLVMPattern { + FP32Lowering(LLVMTypeConverter &converter, bool, int64_t depth) + : ConvertOpToLLVMPattern(converter), depth(depth) {} + + LogicalResult matchAndRewrite(FP32Op op, OpAdaptor adaptor, + ConversionPatternRewriter &b) const override { + + auto constant = op.getConfig().getDefiningOp(); + auto shape = + constant ? dyn_cast(constant.getValue()) : IntegerAttr(); + if (!shape) + return op.emitError("FP32 matmul requires a constant shape"); + uint64_t cfg = shape.getValue().getZExtValue(); + uint64_t rows = cfg & 0xfff, cols = (cfg >> 12) & 0xfff, + k = (cfg >> 24) & 0xfff; + if (cfg >> 36 || (rows != 1 && (rows == 0 || rows % 16)) || cols == 0 || + cols % 16 || !k || k % 4 || rows * k / 4 > uint64_t(depth) || + cols * k / 4 > uint64_t(depth) || + buckyball_target::getBuckyballTarget().bankWidthBits != 128) + return op.emitError( + "FP32 matmul shape does not fit the configured banks"); + auto baseOp = op.getOutputBase().getDefiningOp(); + auto base = + baseOp ? dyn_cast(baseOp.getValue()) : IntegerAttr(); + if (!base || base.getInt() < 0 || base.getInt() > 63 || + base.getInt() + rows * cols / 4 > uint64_t(depth)) + return op.emitError("FP32 output base does not fit the configured bank"); + Location loc = op.getLoc(); + Value rs1 = + packRs1BanksIter(b, loc, adaptor.getOp1BankId(), adaptor.getOp2BankId(), + adaptor.getResultBankId(), cstI64(b, loc, k)); + Value rs2 = cstI64(b, loc, matrixCfg(rows, cols, false, false)); + Value first = + b.create(loc, b.getI64Type(), adaptor.getFirst()); + Value last = + b.create(loc, b.getI64Type(), adaptor.getLast()); + rs2 = b.create( + loc, rs2, b.create(loc, first, cstI64(b, loc, 24))); + rs2 = b.create( + loc, rs2, b.create(loc, last, cstI64(b, loc, 25))); + rs2 = b.create( + loc, rs2, + b.create(loc, adaptor.getOutputBase(), + cstI64(b, loc, 26))); + b.replaceOpWithNewOp( + op, rs1, rs2, + b.getI32IntegerAttr(buckyball_target::getBuckyballFunct7( + op.getFused() ? "MXMM_FMA32" : "MXMM_F32"))); + return success(); + } + int64_t depth; +}; + +#include "WindowPatterns.inc" + +} // namespace +namespace mlir::buddy::buckyball { +void populateMxmmBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, + RewritePatternSet &patterns, + bool stable, int64_t depth) { + patterns.add(converter, + stable, depth); +} +void configureMxmmBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool) { + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/Transforms/WindowPatterns.inc b/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/Transforms/WindowPatterns.inc similarity index 98% rename from examples/balls/mxmm/compiler/src/Dialect/Buckyball/Transforms/WindowPatterns.inc rename to waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/Transforms/WindowPatterns.inc index 5fbd8d375..5ad880b4e 100644 --- a/examples/balls/mxmm/compiler/src/Dialect/Buckyball/Transforms/WindowPatterns.inc +++ b/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/Transforms/WindowPatterns.inc @@ -4,7 +4,6 @@ struct MXFP8WindowLowering : public ConvertOpToLLVMPattern { LogicalResult matchAndRewrite(MXFP8WindowOp op, OpAdaptor a, ConversionPatternRewriter &b) const override { - buckyball_target::requireBuckyballBall("MxmmBall"); if (buckyball_target::getBuckyballTarget().bankWidthBits != 128) return op.emitError("mxmm window: requires 16-byte bank rows"); auto integer = [](Value value) -> std::optional { diff --git a/waste/examples/components/balls/mxmm/stack/src/Passes.cpp b/waste/examples/components/balls/mxmm/stack/src/Passes.cpp new file mode 100644 index 000000000..8458a7ffc --- /dev/null +++ b/waste/examples/components/balls/mxmm/stack/src/Passes.cpp @@ -0,0 +1,41 @@ +//===- Passes.cpp - MxmmBall passes ---------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +void registerNormQuantWindowPass(); +namespace buckyball { +void populateMxmmBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureMxmmBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LowerMxmmPass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerMxmmPass) + StringRef getArgument() const final { return "lower-mxmm-ball"; } + StringRef getDescription() const final { + return "Lower MxmmBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateMxmmBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureMxmmBallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + mlir::buddy::registerNormQuantWindowPass(); + return true; +}(); diff --git a/examples/balls/mxmm/compiler/src/Transforms/NormQuantWindow.cpp b/waste/examples/components/balls/mxmm/stack/src/Transforms/NormQuantWindow.cpp similarity index 99% rename from examples/balls/mxmm/compiler/src/Transforms/NormQuantWindow.cpp rename to waste/examples/components/balls/mxmm/stack/src/Transforms/NormQuantWindow.cpp index a4eade8e7..05452306b 100644 --- a/examples/balls/mxmm/compiler/src/Transforms/NormQuantWindow.cpp +++ b/waste/examples/components/balls/mxmm/stack/src/Transforms/NormQuantWindow.cpp @@ -162,7 +162,9 @@ class FuseNormQuantWindow : public OpRewritePattern { const auto &target = buckyball_target::getBuckyballTarget(); if (!target.rvvEnabled || target.bankWidthBits != 128 || target.bankNum < 5 || - !llvm::is_contained(target.balls, "MxquantBall") || + !llvm::any_of( + target.isa, + [](const auto &entry) { return entry.mnemonic == "MXQUANT"; }) || !llvm::any_of( target.isa, [](const auto &entry) { diff --git a/waste/examples/components/balls/mxquant/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/mxquant/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..8a341eae1 --- /dev/null +++ b/waste/examples/components/balls/mxquant/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,16 @@ +#include "Buckyball/BuckyballOps.h" +#include "Conversion/LowerBuckyball/LowerBuckyball.h" +#include "mlir/IR/PatternMatch.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankMxquantOp::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankMxquantOp op = *this; + rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), + op.getIter()); + rewriter.replaceOp(op, op.getOutBank()); + return success(); +} diff --git a/examples/balls/mxquant/compiler/src/Dialect/Buckyball/MxquantBall.td b/waste/examples/components/balls/mxquant/stack/src/Dialect/Buckyball/MxquantBall.td similarity index 82% rename from examples/balls/mxquant/compiler/src/Dialect/Buckyball/MxquantBall.td rename to waste/examples/components/balls/mxquant/stack/src/Dialect/Buckyball/MxquantBall.td index 6e9cff07b..5ba8fbe88 100644 --- a/examples/balls/mxquant/compiler/src/Dialect/Buckyball/MxquantBall.td +++ b/waste/examples/components/balls/mxquant/stack/src/Dialect/Buckyball/MxquantBall.td @@ -10,7 +10,7 @@ def MxquantOp : Buckyball_Op<"mxquant"> { "prop-dict attr-dict `:` type($inputBankId)"; } -def BankMxquantOp : Buckyball_Op<"bank_mxquant", [MemoryEffects<[MemRead, MemWrite]>]> { +def BankMxquantOp : Buckyball_Op<"bank_mxquant", [DeclareOpInterfaceMethods, MemoryEffects<[MemRead, MemWrite]>]> { let summary = "FP32 to block32 MXFP8 quantization with virtual banks"; let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter); let results = (outs I64:$outBankOut); diff --git a/waste/examples/components/balls/mxquant/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/mxquant/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..47654dab4 --- /dev/null +++ b/waste/examples/components/balls/mxquant/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,56 @@ +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/Arith/IR/Arith.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +struct MxquantLowering : public ConvertOpToLLVMPattern { + MxquantLowering(LLVMTypeConverter &converter, int64_t bankDepth) + : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} + + LogicalResult + matchAndRewrite(MxquantOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + if (buckyball_target::getBuckyballTarget().bankWidthBits != 128) + return op.emitError("mxquant: requires 16-byte bank rows"); + if (auto constant = op.getIter().getDefiningOp()) { + int64_t count = cast(constant.getValue()).getInt(); + if (count <= 0 || count % 32) + return op.emitError("mxquant: count must be a positive multiple of 32"); + if (count > bankDepth * 4 || count + count / 32 > bankDepth * 16) + return op.emitError("mxquant: bank footprint exceeds capacity"); + } + Location loc = op.getLoc(); + Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), + cstI64(rewriter, loc, 0), + adaptor.getOutputBankId(), adaptor.getIter()); + rewriter.replaceOpWithNewOp( + op, rs1, cstI64(rewriter, loc, 0), + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("MXQUANT"))); + return success(); + } + +private: + int64_t bankDepth; +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateMxquantBallLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, + int64_t bankDepth) { + patterns.add(converter, bankDepth); +} +void configureMxquantBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool) { + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/mxquant/stack/src/Passes.cpp b/waste/examples/components/balls/mxquant/stack/src/Passes.cpp new file mode 100644 index 000000000..bcc56d4e0 --- /dev/null +++ b/waste/examples/components/balls/mxquant/stack/src/Passes.cpp @@ -0,0 +1,39 @@ +//===- Passes.cpp - MxquantBall passes ------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +namespace buckyball { +void populateMxquantBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureMxquantBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LowerMxquantPass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerMxquantPass) + StringRef getArgument() const final { return "lower-mxquant-ball"; } + StringRef getDescription() const final { + return "Lower MxquantBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateMxquantBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureMxquantBallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + return true; +}(); diff --git a/waste/examples/components/balls/redmule/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/redmule/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..8917a85d6 --- /dev/null +++ b/waste/examples/components/balls/redmule/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,17 @@ +#include "Buckyball/BuckyballOps.h" +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankRedmuleF16Op::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankRedmuleF16Op op = *this; + rewriter.create(op.getLoc(), op.getA(), op.getB(), op.getC(), + op.getRowsAttr(), op.getKAttr(), + op.getOp1BaseAttr(), op.getOp2BaseAttr(), + op.getWrBaseAttr(), op.getFirstAttr()); + rewriter.replaceOp(op, op.getC()); + return success(); +} diff --git a/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/IR/Redmule.cpp b/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/IR/Redmule.cpp new file mode 100644 index 000000000..a7697234b --- /dev/null +++ b/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/IR/Redmule.cpp @@ -0,0 +1,40 @@ +#include "Buckyball/BuckyballOps.h" +#include "Target/BuckyballTargetRegistry.h" +#include "mlir/Dialect/Arith/IR/Arith.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +template LogicalResult verifyMatmul(Op op) { + if (op.getRows() <= 0 || op.getRows() > 4095) + return op.emitOpError("rows must be in [1, 4095]"); + if (op.getK() <= 0 || op.getK() % 8 || op.getK() >= (int64_t(1) << 34)) + return op.emitOpError( + "K must be a positive multiple of eight within 34 bits"); + for (auto base : {op.getOp1Base(), op.getOp2Base(), op.getWrBase()}) + if (base < 0 || base > 1023) + return op.emitOpError("bank row bases must be in [0, 1023]"); + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || + op.getOp1Base() + op.getRows() * (op.getK() / 8) > target.bankDepth || + op.getOp2Base() + op.getK() > target.bankDepth || + op.getWrBase() + op.getRows() > target.bankDepth) + return op.emitOpError("matrix window exceeds target 128-bit bank capacity"); + auto banks = op.getOperands(); + for (unsigned i = 0; i < banks.size(); ++i) { + auto lhs = banks[i].template getDefiningOp(); + if (lhs && (lhs.value() < 0 || lhs.value() > 1023)) + return op.emitOpError("bank ID must be in [0, 1023]"); + for (unsigned j = 0; j < i; ++j) { + auto rhs = banks[j].template getDefiningOp(); + if (banks[i] == banks[j] || (lhs && rhs && lhs.value() == rhs.value())) + return op.emitOpError("A, B and C banks must differ"); + } + } + return success(); +} +} // namespace + +LogicalResult RedmuleF16Op::verify() { return verifyMatmul(*this); } +LogicalResult BankRedmuleF16Op::verify() { return verifyMatmul(*this); } diff --git a/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/RedmuleBall.td b/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/RedmuleBall.td new file mode 100644 index 000000000..2c6dbca5d --- /dev/null +++ b/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/RedmuleBall.td @@ -0,0 +1,23 @@ +#ifndef REDMULEBALL_OPS +#define REDMULEBALL_OPS +include "Buckyball.td" + +def RedmuleF16Op : Buckyball_Op<"redmule_f16", [MemoryEffects<[MemRead, MemWrite]>]> { + let summary = "Ordered FP16 FMA for A[M,K], B[K,8], C[M,8] bank windows"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$k, + I64Attr:$op1_base, I64Attr:$op2_base, I64Attr:$wr_base, + BoolAttr:$first); + let hasVerifier = 1; + let assemblyFormat = "$a `,` $b `,` $c prop-dict attr-dict `:` type($a)"; +} + +def BankRedmuleF16Op : Buckyball_Op<"bank_redmule_f16", [DeclareOpInterfaceMethods, MemoryEffects<[MemRead, MemWrite]>]> { + let summary = "FP16 bank matrix product with an SSA output bank handle"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$k, + I64Attr:$op1_base, I64Attr:$op2_base, I64Attr:$wr_base, + BoolAttr:$first); + let results = (outs I64:$cOut); + let hasVerifier = 1; + let assemblyFormat = "$a $b $c prop-dict attr-dict `:` type($a) type($b) type($c)"; +} +#endif diff --git a/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..180b99a6c --- /dev/null +++ b/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,44 @@ +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +class RedmuleLowering : public ConvertOpToLLVMPattern { +public: + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + LogicalResult + matchAndRewrite(RedmuleF16Op op, OpAdaptor operands, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + Value rs1 = + packRs1BanksIter(rewriter, loc, operands.getA(), operands.getB(), + operands.getC(), cstI64(rewriter, loc, op.getK())); + uint64_t special = + uint64_t(op.getRows()) | uint64_t(op.getOp1Base()) << 12 | + uint64_t(op.getOp2Base()) << 22 | uint64_t(op.getWrBase()) << 32 | + uint64_t(op.getFirst()) << 42; + rewriter.replaceOpWithNewOp( + op, rs1, cstI64(rewriter, loc, special), + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("REDMULE_F16"))); + return success(); + } +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateRedmuleBallLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { + patterns.add(converter); +} +void configureRedmuleBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool) { + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/redmule/stack/src/Passes.cpp b/waste/examples/components/balls/redmule/stack/src/Passes.cpp new file mode 100644 index 000000000..67b8443c8 --- /dev/null +++ b/waste/examples/components/balls/redmule/stack/src/Passes.cpp @@ -0,0 +1,39 @@ +//===- Passes.cpp - RedmuleBall passes ------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +namespace buckyball { +void populateRedmuleBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureRedmuleBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LowerRedmulePass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerRedmulePass) + StringRef getArgument() const final { return "lower-redmule-ball"; } + StringRef getDescription() const final { + return "Lower RedmuleBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateRedmuleBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureRedmuleBallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + return true; +}(); diff --git a/examples/balls/relu/compiler/src/Backend/include/IntrinsicsRISCVReluBall.td b/waste/examples/components/balls/relu/stack/src/Backend/include/IntrinsicsRISCVReluBall.td similarity index 100% rename from examples/balls/relu/compiler/src/Backend/include/IntrinsicsRISCVReluBall.td rename to waste/examples/components/balls/relu/stack/src/Backend/include/IntrinsicsRISCVReluBall.td diff --git a/waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..f7adc1f07 --- /dev/null +++ b/waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,18 @@ +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankReluOp::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankReluOp op = *this; + rewriter.create(op.getLoc(), op.getBank(), op.getGroup(), + op.getIter(), op.getStride()); + rewriter.replaceOp(op, op.getBank()); + return success(); +} diff --git a/waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp b/waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp new file mode 100644 index 000000000..2897421f8 --- /dev/null +++ b/waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp @@ -0,0 +1,102 @@ +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Target/BuckyballTargetRegistry.h" +#include "Utils/BankUtils.h" + +#include + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { + +constexpr int64_t kValuesPerLine = 4; + +} // namespace + +LogicalResult ReluMatrixOp::lowerToBankSSA(PatternRewriter &b) { + ReluMatrixOp op = *this; + auto inputType = dyn_cast(op.getInput().getType()); + auto outputType = dyn_cast(op.getOutput().getType()); + if (!inputType || !outputType || !inputType.hasStaticShape() || + inputType != outputType || !inputType.getElementType().isInteger(32)) + return op.emitError("requires matching static memref"); + + int64_t rows = inputType.getShape()[0]; + int64_t columns = inputType.getShape()[1]; + if (rows <= 0 || columns <= 0 || rows % 16 || columns % 16) + return op.emitError("requires positive 16-aligned dimensions"); + + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || target.bankDepth <= 0 || + target.bankDepth % 16) + return op.emitError( + "ReluBall requires a 128-bit bank with 16-aligned depth"); + + Location loc = op.getLoc(); + int64_t lines = target.bankDepth; + int64_t values = lines * kValuesPerLine; + int64_t total = rows * columns; + Value bank = allocBank(b, loc, 1, 1); + Value zero = b.create(loc, 0); + Value one = b.create(loc, 1); + Value four = b.create(loc, kValuesPerLine); + Value columnsValue = b.create(loc, columns); + auto packType = MemRefType::get({lines, kValuesPerLine}, b.getI32Type()); + + for (int64_t base = 0; base < total; base += values) { + int64_t count = std::min(values, total - base); + Value inputPack = b.create(loc, packType); + Value outputPack = b.create(loc, packType); + Value zeroValue = b.create(loc, 0, 32); + b.create(loc, zeroValue, inputPack); + + Value countValue = b.create(loc, count); + Value baseValue = b.create(loc, base); + auto copyIn = b.create(loc, zero, countValue, one); + b.setInsertionPointToStart(copyIn.getBody()); + Value index = copyIn.getInductionVar(); + Value source = b.create(loc, baseValue, index); + Value row = b.create(loc, source, columnsValue); + Value column = b.create(loc, source, columnsValue); + Value line = b.create(loc, index, four); + Value lane = b.create(loc, index, four); + Value value = + b.create(loc, op.getInput(), ValueRange{row, column}); + b.create(loc, value, inputPack, ValueRange{line, lane}); + b.setInsertionPointAfter(copyIn); + + Value loaded = mvinBank(b, loc, inputPack, bank, lines); + Value result = b.create( + loc, bank.getType(), loaded, createI64Const(b, loc, 0), + createI64Const(b, loc, lines), createI64Const(b, loc, lines)); + bank = mvoutBank(b, loc, outputPack, result, lines); + + auto copyOut = b.create(loc, zero, countValue, one); + b.setInsertionPointToStart(copyOut.getBody()); + index = copyOut.getInductionVar(); + source = b.create(loc, baseValue, index); + row = b.create(loc, source, columnsValue); + column = b.create(loc, source, columnsValue); + line = b.create(loc, index, four); + lane = b.create(loc, index, four); + value = b.create(loc, outputPack, ValueRange{line, lane}); + b.create(loc, value, op.getOutput(), + ValueRange{row, column}); + b.setInsertionPointAfter(copyOut); + b.create(loc, inputPack); + b.create(loc, outputPack); + } + + releaseBank(b, loc, bank); + b.eraseOp(op); + return success(); +} diff --git a/waste/examples/components/balls/relu/stack/src/Conversion/LowerLinalgToBuckyball/ReluPatterns.cpp b/waste/examples/components/balls/relu/stack/src/Conversion/LowerLinalgToBuckyball/ReluPatterns.cpp new file mode 100644 index 000000000..f001c6cfd --- /dev/null +++ b/waste/examples/components/balls/relu/stack/src/Conversion/LowerLinalgToBuckyball/ReluPatterns.cpp @@ -0,0 +1,118 @@ +//===- ReluPatterns.cpp - signed INT32 ReLU -> buckyball.relu_matrix -----===// +// +// Claims linalg.generic max(input, 0) over static memref. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballOps.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/PatternMatch.h" +#include "mlir/IR/SymbolTable.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +// The zero operand must be a constant zero splat global. +bool isZeroGlobal(Value value) { + auto global = value.getDefiningOp(); + if (!global) + return false; + auto constant = SymbolTable::lookupNearestSymbolFrom( + global, global.getNameAttr()); + auto values = + constant + ? dyn_cast_or_null(constant.getConstantInitValue()) + : nullptr; + return values && values.isSplat() && values.getSplatValue().isZero(); +} + +class ReluLowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(linalg::GenericOp op, + PatternRewriter &b) const override { + Block &body = op.getRegion().front(); + if (body.getOperations().size() != 2 || op.getInputs().size() != 2 || + op.getOutputs().size() != 1) + return failure(); + auto maximum = dyn_cast(body.front()); + ValueRange args = body.getArguments(); + if (!maximum || body.back().getOperand(0) != maximum.getResult() || + !((maximum.getLhs() == args[0] && maximum.getRhs() == args[1]) || + (maximum.getLhs() == args[1] && maximum.getRhs() == args[0]))) + return failure(); + Value input = op.getInputs()[0]; + Value output = op.getOutputs()[0]; + auto inputType = dyn_cast(input.getType()); + if (!inputType || !inputType.hasStaticShape() || inputType.getRank() != 2 || + !inputType.getElementType().isInteger(32) || + op.getInputs()[1].getType() != inputType || + output.getType() != inputType || !isZeroGlobal(op.getInputs()[1]) || + !llvm::all_of(op.getIndexingMapsArray(), + [](AffineMap map) { return map.isIdentity(); })) + return failure(); + + int64_t rows = inputType.getShape()[0]; + int64_t columns = inputType.getShape()[1]; + int64_t paddedRows = llvm::alignTo(rows, 16); + int64_t paddedColumns = llvm::alignTo(columns, 16); + Location loc = op.getLoc(); + if (paddedRows == rows && paddedColumns == columns) { + b.create(loc, input, output); + b.eraseOp(op); + return success(); + } + + auto paddedType = + MemRefType::get({paddedRows, paddedColumns}, b.getI32Type()); + Value paddedInput = b.create(loc, paddedType); + Value paddedOutput = b.create(loc, paddedType); + Value zeroValue = b.create(loc, 0, 32); + b.create(loc, zeroValue, paddedInput); + + Value zero = b.create(loc, 0); + Value one = b.create(loc, 1); + Value rowLimit = b.create(loc, rows); + Value columnLimit = b.create(loc, columns); + auto copyInputRows = b.create(loc, zero, rowLimit, one); + b.setInsertionPointToStart(copyInputRows.getBody()); + Value row = copyInputRows.getInductionVar(); + auto copyInputColumns = b.create(loc, zero, columnLimit, one); + b.setInsertionPointToStart(copyInputColumns.getBody()); + Value column = copyInputColumns.getInductionVar(); + Value value = b.create(loc, input, ValueRange{row, column}); + b.create(loc, value, paddedInput, ValueRange{row, column}); + b.setInsertionPointAfter(copyInputRows); + + b.create(loc, paddedInput, paddedOutput); + + auto copyOutputRows = b.create(loc, zero, rowLimit, one); + b.setInsertionPointToStart(copyOutputRows.getBody()); + row = copyOutputRows.getInductionVar(); + auto copyOutputColumns = b.create(loc, zero, columnLimit, one); + b.setInsertionPointToStart(copyOutputColumns.getBody()); + column = copyOutputColumns.getInductionVar(); + value = + b.create(loc, paddedOutput, ValueRange{row, column}); + b.create(loc, value, output, ValueRange{row, column}); + b.setInsertionPointAfter(copyOutputRows); + + b.create(loc, paddedInput); + b.create(loc, paddedOutput); + b.eraseOp(op); + return success(); + } +}; +} // namespace + +namespace mlir::buddy { +void populateReluBallLinalgPatterns(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} +} // namespace mlir::buddy diff --git a/examples/balls/relu/compiler/src/Dialect/Buckyball/ReluBall.td b/waste/examples/components/balls/relu/stack/src/Dialect/Buckyball/ReluBall.td similarity index 83% rename from examples/balls/relu/compiler/src/Dialect/Buckyball/ReluBall.td rename to waste/examples/components/balls/relu/stack/src/Dialect/Buckyball/ReluBall.td index 70fafee38..71934ff29 100644 --- a/examples/balls/relu/compiler/src/Dialect/Buckyball/ReluBall.td +++ b/waste/examples/components/balls/relu/stack/src/Dialect/Buckyball/ReluBall.td @@ -5,7 +5,7 @@ include "Buckyball.td" -def ReluMatrixOp : Buckyball_Op<"relu_matrix"> { +def ReluMatrixOp : Buckyball_Op<"relu_matrix", [DeclareOpInterfaceMethods]> { let summary = "Logical INT32 ReLU matrix tile"; let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, MemRefRankOf<[AnyType], [2]>:$output); @@ -20,7 +20,7 @@ def ReluOp : Buckyball_Op<"relu"> { "type($bankId)"; } -def BankReluOp : Buckyball_Op<"bank_relu"> { +def BankReluOp : Buckyball_Op<"bank_relu", [DeclareOpInterfaceMethods]> { let summary = "In-place INT32 ReLU with a virtual bank handle"; let arguments = (ins I64:$bank, I64:$group, I64:$iter, I64:$stride); let results = (outs I64:$bankOut); diff --git a/waste/examples/components/balls/relu/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/relu/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..d7dc345f5 --- /dev/null +++ b/waste/examples/components/balls/relu/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,50 @@ +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" + +#include "mlir/IR/PatternMatch.h" +#include + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +struct ReluLowering : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + LogicalResult + matchAndRewrite(ReluOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + Value rs1 = + packRs1BanksIter(rewriter, loc, adaptor.getBankId(), adaptor.getGroup(), + adaptor.getBankId(), adaptor.getIter()); + rs1 = + rewriter.create(loc, rs1, cstI64(rewriter, loc, INT64_MIN)); + rewriter.replaceOpWithNewOp( + op, rs1, adaptor.getStride(), + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("RELU"))); + return success(); + } +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateReluBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, + RewritePatternSet &patterns, + bool, int64_t) { + patterns.add(converter); +} + +void configureReluBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool) { + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/relu/stack/src/Passes.cpp b/waste/examples/components/balls/relu/stack/src/Passes.cpp new file mode 100644 index 000000000..972f95a5c --- /dev/null +++ b/waste/examples/components/balls/relu/stack/src/Passes.cpp @@ -0,0 +1,55 @@ +//===- Passes.cpp - ReluBall passes ---------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +void populateReluBallLinalgPatterns(RewritePatternSet &patterns); +namespace buckyball { +void populateReluBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureReluBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LinalgToReluPass + : public mlir::buddy::LinalgToBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LinalgToReluPass) + StringRef getArgument() const final { return "convert-linalg-to-relu-ball"; } + StringRef getDescription() const final { + return "Lower the Linalg operations ReluBall executes."; + } + void populate(RewritePatternSet &patterns, int64_t, int64_t bankDepth, + int64_t) const { + mlir::buddy::populateReluBallLinalgPatterns(patterns); + } +}; + +class LowerReluPass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerReluPass) + StringRef getArgument() const final { return "lower-relu-ball"; } + StringRef getDescription() const final { + return "Lower ReluBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateReluBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureReluBallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + PassRegistration(); + return true; +}(); diff --git a/examples/balls/smatmul/compiler/src/Backend/include/IntrinsicsRISCVSMatMulBall.td b/waste/examples/components/balls/smatmul/stack/src/Backend/include/IntrinsicsRISCVSMatMulBall.td similarity index 100% rename from examples/balls/smatmul/compiler/src/Backend/include/IntrinsicsRISCVSMatMulBall.td rename to waste/examples/components/balls/smatmul/stack/src/Backend/include/IntrinsicsRISCVSMatMulBall.td diff --git a/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..8e824b317 --- /dev/null +++ b/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,31 @@ +//===- AssignPhysicalBanks.cpp - Matrix bank assignment -------------------===// + +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankSMatMulOp::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankSMatMulOp op = *this; + rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), + op.getWrBank(), op.getConfig(), op.getFirst(), + op.getLast(), op.getOutputBase()); + rewriter.replaceOp(op, op.getWrBank()); + return success(); +} + +LogicalResult +BankSMatMulBiasOp::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankSMatMulBiasOp op = *this; + rewriter.create(op.getLoc(), op.getBiasBank(), + op.getInputBase()); + rewriter.replaceOp(op, op.getBiasBank()); + return success(); +} diff --git a/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp b/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp new file mode 100644 index 000000000..0672e722d --- /dev/null +++ b/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp @@ -0,0 +1,356 @@ +//===- LowerBuckyballToBankSSAPatterns.cpp - Mega MatMul to banks --------===// + +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Target/BuckyballTargetRegistry.h" +#include "Utils/BankUtils.h" + +#include + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace mlir::buddy { +LogicalResult lowerMatmulToBanks(MegaMatmulOp op, PatternRewriter &rewriter); +} // namespace mlir::buddy + +namespace { + +constexpr int64_t kTile = 16; +constexpr int64_t kInt32RowsPerTile = 64; + +class MegaMatmulToBankSSAPattern : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(MegaMatmulOp op, + PatternRewriter &b) const override { + Location loc = op.getLoc(); + auto aTy = dyn_cast(op.getInput().getType()); + auto wTy = dyn_cast(op.getWeight().getType()); + auto biasTy = dyn_cast(op.getBias().getType()); + auto scaleTy = dyn_cast(op.getScale().getType()); + auto lutTy = dyn_cast(op.getLut().getType()); + auto outTy = dyn_cast(op.getOutput().getType()); + auto kernel = op->getParentOfType(); + if (!kernel) + return op.emitError( + "Mega MatMul must be nested in buckyball.mega_kernel"); + bool finalOutput = op.getOutput() == kernel.getOutput(); + if (!aTy || !wTy || !biasTy || !scaleTy || !lutTy || !outTy || + !aTy.hasStaticShape() || !wTy.hasStaticShape() || + !biasTy.hasStaticShape() || !scaleTy.hasStaticShape() || + !lutTy.hasStaticShape() || !outTy.hasStaticShape()) + return op.emitError("Mega MatMul requires static memrefs"); + if (!aTy.getElementType().isInteger(8) || + !wTy.getElementType().isInteger(8) || + !biasTy.getElementType().isInteger(32) || + !scaleTy.getElementType().isF32() || + !lutTy.getElementType().isInteger(8) || lutTy.getRank() != 1 || + lutTy.getShape()[0] != (op.getActivation() == 2 ? 256 : 1) || + op.getActivation() < 0 || op.getActivation() > 2 || + (finalOutput && op.getActivation() == 2) || + (finalOutput ? !outTy.getElementType().isF32() + : !outTy.getElementType().isInteger(8))) + return op.emitError( + "Mega MatMul input/weight must be INT8, bias INT32, scale FP32, " + "and only the last stage may output FP32"); + + int64_t M = aTy.getShape()[0]; + int64_t K = aTy.getShape()[1]; + int64_t N = wTy.getShape()[1]; + if (M <= 0 || K <= 0 || N <= 0 || wTy.getShape()[0] != K || + biasTy.getShape()[0] != N || scaleTy.getShape()[0] != N || + outTy.getShape()[0] != M || outTy.getShape()[1] != N) + return op.emitError("Mega MatMul shape mismatch"); + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || target.bankDepth < kInt32RowsPerTile || + target.bankDepth % kInt32RowsPerTile) + return op.emitError( + "Mega MatMul requires 128-bit banks whose depth is a positive " + "multiple of 64 rows"); + if (buckyball_target::getBuckyballBallMapping("SMATMUL_OS").outBW != 1) + return op.emitError("Mega MatMul requires SMatMulBall outBW=1"); + + const int64_t hardwareRows = + buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); + if (hardwareRows <= 0 || kTile % hardwareRows) + return op.emitError( + "SMatMul tileRows must divide the 16-row software tile"); + const int64_t tileRows = M == 1 ? 1 : kTile; + const int64_t resultRows = tileRows * 4; + int64_t paddedM = (M + tileRows - 1) / tileRows * tileRows; + int64_t paddedK = (K + kTile - 1) / kTile * kTile; + int64_t kChunk = target.bankDepth / kTile * kTile; + if (kChunk <= 0) + return op.emitError("bank depth cannot hold one K tile"); + // outputBase is a six-bit row index; one 16x16 result uses 64 rows. + constexpr int64_t tilesPerResultBank = 1; + + Value zero = b.create(loc, 0); + Value one = b.create(loc, 1); + Value tileSize = b.create(loc, kTile); + Value columns = b.create(loc, N); + Value zeroI8 = + b.create(loc, b.getI8Type(), b.getI8IntegerAttr(0)); + auto matrixRowLoop = b.create( + loc, zero, b.create(loc, paddedM), + b.create(loc, tileRows)); + b.setInsertionPointToStart(matrixRowLoop.getBody()); + Value m0 = matrixRowLoop.getInductionVar(); + Value validRows = b.create( + loc, b.create(loc, tileRows), + b.create(loc, b.create(loc, M), + m0)); + SmallVector inputPacks; + { + for (int64_t k0 = 0; k0 < paddedK; k0 += kChunk) { + int64_t thisK = std::min(kChunk, paddedK - k0); + int64_t validK = std::min(thisK, K - k0); + auto aPackTy = + MemRefType::get({thisK / kTile * tileRows, kTile}, b.getI8Type()); + Value aPack = b.create(loc, aPackTy); + inputPacks.push_back(aPack); + b.create(loc, zeroI8, aPack); + + Value zero = b.create(loc, 0); + Value one = b.create(loc, 1); + Value rowEnd = validRows; + Value kTileEnd = + b.create(loc, (validK + kTile - 1) / kTile); + auto ktLoop = b.create(loc, zero, kTileEnd, one); + b.setInsertionPointToStart(ktLoop.getBody()); + Value kt = ktLoop.getInductionVar(); + auto rowLoop = b.create(loc, zero, rowEnd, one); + b.setInsertionPointToStart(rowLoop.getBody()); + Value row = rowLoop.getInductionVar(); + Value remaining = b.create( + loc, b.create(loc, validK), + b.create( + loc, kt, b.create(loc, kTile))); + Value columnEnd = b.create( + loc, remaining, b.create(loc, kTile)); + auto columnLoop = b.create(loc, zero, columnEnd, one); + b.setInsertionPointToStart(columnLoop.getBody()); + Value column = columnLoop.getInductionVar(); + Value sourceRow = b.create(loc, m0, row); + Value sourceColumn = b.create( + loc, b.create(loc, k0), + b.create( + loc, + b.create( + loc, kt, b.create(loc, kTile)), + column)); + Value hardwareRowsValue = b.create( + loc, std::min(hardwareRows, tileRows)); + Value rowTile = b.create(loc, row, hardwareRowsValue); + Value rowInTile = b.create(loc, row, hardwareRowsValue); + Value tileIndex = b.create( + loc, + b.create( + loc, rowTile, + b.create(loc, thisK / kTile)), + kt); + Value packedRow = b.create( + loc, b.create(loc, tileIndex, hardwareRowsValue), + rowInTile); + Value value = b.create( + loc, op.getInput(), ValueRange{sourceRow, sourceColumn}); + b.create(loc, value, aPack, + ValueRange{packedRow, column}); + b.setInsertionPointAfter(ktLoop); + } + } + auto panelLoop = b.create(loc, zero, columns, tileSize); + b.setInsertionPointToStart(panelLoop.getBody()); + { + Value n0 = panelLoop.getInductionVar(); + Value thisN = b.create( + loc, tileSize, b.create(loc, columns, n0)); + Value aBank = allocBank(b, loc, 1, 1); + Value wBank = allocBank(b, loc, 1, 1); + auto biasPackTy = MemRefType::get({4, 4}, b.getI32Type()); + auto scalePackTy = MemRefType::get({4, 4}, b.getF32Type()); + Value biasPack = b.create(loc, biasPackTy); + Value scalePack = b.create(loc, scalePackTy); + Value zeroI32 = b.create(loc, b.getI32Type(), + b.getI32IntegerAttr(0)); + Value oneF32 = b.create(loc, b.getF32Type(), + b.getF32FloatAttr(1.0)); + b.create(loc, zeroI32, biasPack); + b.create(loc, oneF32, scalePack); + auto channelLoop = b.create(loc, zero, thisN, one); + b.setInsertionPointToStart(channelLoop.getBody()); + { + Value i = channelLoop.getInductionVar(); + Value four = b.create(loc, 4); + Value source = b.create(loc, n0, i); + Value row = b.create(loc, i, four); + Value column = b.create(loc, i, four); + Value bias = b.create(loc, op.getBias(), source); + Value scale = b.create(loc, op.getScale(), source); + b.create(loc, bias, biasPack, ValueRange{row, column}); + b.create(loc, scale, scalePack, + ValueRange{row, column}); + } + b.setInsertionPointAfter(channelLoop); + + Value biasBank = allocBank(b, loc, 1, 1); + Value biasLoaded = mvinBank(b, loc, biasPack, biasBank, 4); + Value biasState = b.create( + loc, biasLoaded.getType(), biasLoaded, createI64Const(b, loc, 0)); + Value scaleBank = allocBank(b, loc, 1, 1); + Value scaleLoaded = mvinBank(b, loc, scalePack, scaleBank, 4); + + { + int64_t tileCount = tilesPerResultBank; + Value resultBank = allocBank(b, loc, 1, 1); + Value resultState = resultBank; + SmallVector hostPacks; + + for (int64_t tile = 0; tile < tileCount; ++tile) { + for (int64_t k0 = 0; k0 < paddedK; k0 += kChunk) { + int64_t thisK = std::min(kChunk, paddedK - k0); + int64_t validK = std::min(thisK, std::max(0, K - k0)); + Value aPack = inputPacks[k0 / kChunk]; + Value aLoaded = + mvinBank(b, loc, aPack, aBank, thisK / kTile * tileRows); + Value wLoaded; + auto [weightStrides, weightOffset] = wTy.getStridesAndOffset(); + if (validK == thisK && N % kTile == 0 && weightStrides[1] == 1 && + weightStrides[0] > 0 && weightStrides[0] % kTile == 0) { + Value panel = b.create( + loc, op.getWeight(), + SmallVector{b.getIndexAttr(k0), n0}, + SmallVector{b.getIndexAttr(thisK), + b.getIndexAttr(kTile)}, + SmallVector{b.getIndexAttr(1), + b.getIndexAttr(1)}); + wLoaded = mvinBank(b, loc, panel, wBank, thisK, + weightStrides[0] / kTile); + } else { + auto wPackTy = MemRefType::get({thisK, kTile}, b.getI8Type()); + Value wPack = b.create(loc, wPackTy); + hostPacks.push_back(wPack); + b.create(loc, zeroI8, wPack); + Value validKEnd = b.create(loc, validK); + Value validNEnd = thisN; + auto weightRowLoop = + b.create(loc, zero, validKEnd, one); + b.setInsertionPointToStart(weightRowLoop.getBody()); + Value weightRow = weightRowLoop.getInductionVar(); + auto weightColumnLoop = + b.create(loc, zero, validNEnd, one); + b.setInsertionPointToStart(weightColumnLoop.getBody()); + Value weightColumn = weightColumnLoop.getInductionVar(); + Value sourceWeightRow = b.create( + loc, b.create(loc, k0), weightRow); + Value sourceWeightColumn = + b.create(loc, n0, weightColumn); + Value weightValue = b.create( + loc, op.getWeight(), + ValueRange{sourceWeightRow, sourceWeightColumn}); + b.create(loc, weightValue, wPack, + ValueRange{weightRow, weightColumn}); + b.setInsertionPointAfter(weightRowLoop); + + wLoaded = mvinBank(b, loc, wPack, wBank, thisK); + } + uint64_t cfg = matrixRs2(tileRows, kTile, thisK); + auto smatmul = b.create( + loc, resultState.getType(), aLoaded, wLoaded, resultState, + createI64Const(b, loc, static_cast(cfg)), + createI1Const(b, loc, k0 == 0), + createI1Const(b, loc, k0 + thisK == paddedK), + createI64Const(b, loc, tile * resultRows)); + resultState = smatmul.getWrBankOut(); + aBank = aLoaded; + wBank = wLoaded; + } + } + + Value outputBank = allocBank(b, loc, 1, 1); + Value converted = + finalOutput + ? b.create( + loc, outputBank.getType(), resultState, scaleLoaded, + outputBank, + createI64Const(b, loc, tileCount * resultRows), + b.getBoolAttr(op.getActivation() == 1)) + .getOutBankOut() + : b.create( + loc, outputBank.getType(), resultState, scaleLoaded, + outputBank, + createI64Const(b, loc, tileCount * resultRows), + createI64Const(b, loc, 0), createI64Const(b, loc, 0), + b.getI64IntegerAttr(tileCount * tileRows), + b.getI64IntegerAttr(1), + b.getI64IntegerAttr(tileCount * tileRows), + b.getBoolAttr(op.getActivation() == 1)) + .getOutBankOut(); + releaseBank(b, loc, resultState); + + Type outputElementType = + finalOutput ? Type(b.getF32Type()) : Type(b.getI8Type()); + auto packedOutTy = + MemRefType::get({tileCount * tileRows, kTile}, outputElementType); + Value packedOut = b.create(loc, packedOutTy); + Value stored = mvoutBank(b, loc, packedOut, converted, + finalOutput ? tileCount * resultRows + : tileCount * tileRows); + for (Value pack : hostPacks) + b.create(loc, pack); + + Value zero = b.create(loc, 0); + Value one = b.create(loc, 1); + Value rowEnd = validRows; + Value columnEnd = thisN; + auto rowLoop = b.create(loc, zero, rowEnd, one); + b.setInsertionPointToStart(rowLoop.getBody()); + Value row = rowLoop.getInductionVar(); + auto columnLoop = b.create(loc, zero, columnEnd, one); + b.setInsertionPointToStart(columnLoop.getBody()); + Value column = columnLoop.getInductionVar(); + Value value = + b.create(loc, packedOut, ValueRange{row, column}); + Value outputRow = b.create(loc, m0, row); + Value outputColumn = b.create(loc, n0, column); + b.create(loc, value, op.getOutput(), + ValueRange{outputRow, outputColumn}); + b.setInsertionPointAfter(rowLoop); + releaseBank(b, loc, stored); + b.create(loc, packedOut); + } + + releaseBank(b, loc, biasState); + releaseBank(b, loc, scaleLoaded); + b.create(loc, biasPack); + b.create(loc, scalePack); + releaseBank(b, loc, aBank); + releaseBank(b, loc, wBank); + } + b.setInsertionPointAfter(panelLoop); + for (Value pack : inputPacks) + b.create(loc, pack); + b.setInsertionPointAfter(matrixRowLoop); + b.eraseOp(op); + return success(); + } +}; + +} // namespace + +LogicalResult mlir::buddy::lowerMatmulToBanks(MegaMatmulOp op, + PatternRewriter &rewriter) { + return MegaMatmulToBankSSAPattern(op.getContext()) + .matchAndRewrite(op, rewriter); +} diff --git a/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp b/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp similarity index 84% rename from examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp rename to waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp index 91c01b4bc..882b988bf 100644 --- a/examples/balls/smatmul/compiler/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp +++ b/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp @@ -13,6 +13,7 @@ #include "Target/BuckyballTargetRegistry.h" #include "Trace/TraceOps.h" #include "Utils/BankUtils.h" +#include "Utils/StageTrace.h" #include "llvm/Support/MathExtras.h" #include @@ -21,6 +22,10 @@ using namespace mlir; using namespace ::buddy::buckyball; +namespace mlir::buddy { +LogicalResult lowerMatmulToBanks(MegaMatmulOp op, PatternRewriter &rewriter); +} // namespace mlir::buddy + namespace { constexpr int64_t kTile = 16; @@ -28,13 +33,7 @@ constexpr int64_t kInt32Rows = 64; class MatmulMegaKernelToBankSSAPattern : public OpRewritePattern { public: - MatmulMegaKernelToBankSSAPattern(MLIRContext *context, bool traceMegaStages, - int64_t traceMegaStageStart, - int64_t traceMegaStageLimit) - : OpRewritePattern(context), - traceMegaStages(traceMegaStages), - traceMegaStageStart(traceMegaStageStart), - traceMegaStageLimit(traceMegaStageLimit) {} + using OpRewritePattern::OpRewritePattern; LogicalResult matchAndRewrite(MegaKernelOp kernel, PatternRewriter &b) const override { @@ -54,8 +53,12 @@ class MatmulMegaKernelToBankSSAPattern : public OpRewritePattern { if (!matmul) return kernel.emitError( "MatMul MegaKernel cannot contain non-MatMul stages"); - if (matmul.getActivation() == 2) - buckyball_target::requireBuckyballBall("LutBall"); + // The activation LUT runs on the LUT Ball. + if (matmul.getActivation() == 2 && + !llvm::any_of( + buckyball_target::getBuckyballTarget().isa, + [](const auto &entry) { return entry.mnemonic == "LUT"; })) + return matmul.emitError("MatMul MegaKernel LUT requires the LUT Ball"); stages.push_back(matmul); } @@ -65,14 +68,35 @@ class MatmulMegaKernelToBankSSAPattern : public OpRewritePattern { return kernel.emitError( "MatMul MegaKernel requires 128-bit power-of-two-depth banks with " "at least 64 rows"); - if (buckyball_target::getBuckyballBallMapping("SMatMulBall").outBW != 1) + if (buckyball_target::getBuckyballBallMapping("SMATMUL_OS").outBW != 1) return kernel.emitError("MatMul MegaKernel requires SMatMulBall outBW=1"); int64_t rows = cast(stages.front().getInput().getType()).getShape()[0]; if (stages.size() == 1) { + auto stage = stages.front(); + Value output = stage.getOutput(); + auto trace = stage->getAttrOfType("trace_node"); + b.setInsertionPoint(stage); + if (trace && cast(trace.get("cycle")).getValue()) { + auto start = b.create<::buddy::trace::StartOp>( + stage.getLoc(), cast(trace.get("id")), + b.getBoolAttr(true), b.getBoolAttr(false), + cast(trace.get("tag"))); + start->setAttr("id_path", trace.get("id_path")); + } if (failed(mlir::buddy::lowerMatmulToBanks(stages.front(), b))) return failure(); + b.setInsertionPoint(body.getTerminator()); + if (trace) { + auto end = b.create<::buddy::trace::EndOp>( + kernel.getLoc(), output.getType(), output, + cast(trace.get("id")), + cast(trace.get("cycle")), + cast(trace.get("tensor")), + cast(trace.get("tag"))); + end->setAttr("id_path", trace.get("id_path")); + } b.eraseOp(body.getTerminator()); b.inlineBlockBefore(&body, kernel); b.eraseOp(kernel); @@ -137,7 +161,7 @@ class MatmulMegaKernelToBankSSAPattern : public OpRewritePattern { loc, zero, b.create(loc, rows), one); b.setInsertionPointToStart(rowLoop.getBody()); Value inputRow = rowLoop.getInductionVar(); - SmallVector> tracedOutputs; + SmallVector> tracedOutputs; int64_t paddedFirstK = (firstK + kTile - 1) / kTile * kTile; for (int64_t k0 = 0; k0 < paddedFirstK; k0 += target.bankDepth) { int64_t thisK = std::min(target.bankDepth, paddedFirstK - k0); @@ -165,6 +189,17 @@ class MatmulMegaKernelToBankSSAPattern : public OpRewritePattern { for (auto [stageIndex, stage] : llvm::enumerate(stages)) { bool last = stageIndex + 1 == stages.size(); + auto traceNode = stage->getAttrOfType("trace_node"); + bool cycle = + traceNode && cast(traceNode.get("cycle")).getValue(); + bool tensor = + traceNode && cast(traceNode.get("tensor")).getValue(); + if (cycle) { + auto start = b.create<::buddy::trace::StartOp>( + loc, cast(traceNode.get("id")), b.getBoolAttr(true), + b.getBoolAttr(false), cast(traceNode.get("tag"))); + start->setAttr("id_path", traceNode.get("id_path")); + } int64_t logicalK = cast(stage.getInput().getType()).getShape()[1]; int64_t n = cast(stage.getWeight().getType()).getShape()[1]; @@ -350,11 +385,7 @@ class MatmulMegaKernelToBankSSAPattern : public OpRewritePattern { if (lutBank) releaseBank(b, loc, lutBank); - bool traceStage = - traceMegaStages && !last && - static_cast(stageIndex) >= traceMegaStageStart && - (traceMegaStageLimit < 0 || - static_cast(stageIndex) < traceMegaStageLimit); + bool traceStage = tensor && !last; if (traceStage) { int64_t channelBase = 0; for (auto [bankIndex, output] : llvm::enumerate(outputs)) { @@ -380,8 +411,7 @@ class MatmulMegaKernelToBankSSAPattern : public OpRewritePattern { b.setInsertionPointAfter(channelLoop); channelBase += outputK[bankIndex]; } - tracedOutputs.push_back( - {stage.getOutput(), b.getI64IntegerAttr(1000 + stageIndex), false}); + tracedOutputs.push_back({stage.getOutput(), traceNode, false}); } for (Value activation : activationBanks) @@ -405,14 +435,17 @@ class MatmulMegaKernelToBankSSAPattern : public OpRewritePattern { ValueRange{inputRow, outputN}); b.setInsertionPointAfter(outputLoop); } - if (traceMegaStages && - traceMegaStageStart <= static_cast(stageIndex) && - (traceMegaStageLimit < 0 || - static_cast(stageIndex) < traceMegaStageLimit)) { - tracedOutputs.push_back( - {stage.getOutput(), b.getI64IntegerAttr(1001), true}); + if (tensor) { + tracedOutputs.push_back({stage.getOutput(), traceNode, true}); } } + if (cycle) { + auto end = b.create<::buddy::trace::EndOp>( + loc, zero.getType(), zero, cast(traceNode.get("id")), + b.getBoolAttr(true), b.getBoolAttr(false), + cast(traceNode.get("tag"))); + end->setAttr("id_path", traceNode.get("id_path")); + } } for (Value bank : activationBanks) @@ -420,33 +453,28 @@ class MatmulMegaKernelToBankSSAPattern : public OpRewritePattern { for (Value pack : hostPacks) b.create(loc, pack); b.setInsertionPointAfter(rowLoop); - for (auto [output, id, last] : tracedOutputs) { - auto trace = b.create<::buddy::trace::EndOp>( - loc, output.getType(), output, id, - b.getStringAttr(last ? "classifier-output" : "mega-matmul-stage")); - trace->setAttr("id_path", b.getArrayAttr({id})); - if (!last) - trace->setAttr("buckyball.stage_trace", b.getUnitAttr()); + for (auto [output, meta, last] : tracedOutputs) { + if (!last && + cast(output.getType()).getElementType().isInteger(8)) { + if (failed(mlir::buddy::emitStageTensorTrace(b, loc, output, meta))) + return failure(); + } else { + auto trace = b.create<::buddy::trace::EndOp>( + loc, output.getType(), output, cast(meta.get("id")), + b.getBoolAttr(false), b.getBoolAttr(true), + cast(meta.get("tag"))); + trace->setAttr("id_path", meta.get("id_path")); + } } b.eraseOp(kernel); return success(); } - -private: - bool traceMegaStages; - int64_t traceMegaStageStart; - int64_t traceMegaStageLimit; }; } // namespace namespace mlir::buddy { -void populateMatmulRegionToBankSSAPatterns(RewritePatternSet &patterns, - bool traceMegaStages, - int64_t traceMegaStageStart, - int64_t traceMegaStageLimit) { - patterns.add( - patterns.getContext(), traceMegaStages, traceMegaStageStart, - traceMegaStageLimit); +void populateMatmulRegionToBankSSAPatterns(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); } } // namespace mlir::buddy diff --git a/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerLinalgToBuckyball/MatMulPatterns.cpp b/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerLinalgToBuckyball/MatMulPatterns.cpp new file mode 100644 index 000000000..5197d0f89 --- /dev/null +++ b/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerLinalgToBuckyball/MatMulPatterns.cpp @@ -0,0 +1,416 @@ +//===- MatMulPatterns.cpp - linalg matmul -> buckyball.smatmul_matmul ----===// +// +// Claims linalg.matmul (plain or transposed B) and linalg.batch_matmul, which +// splits into one matmul per batch. Non-quantized matmul is split into +// bank-sized panels. Quantized matmul retains its full N dimension so bank +// lowering can quantize one activation chunk and hand it to every output panel. +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +#include + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { + +constexpr size_t kBankLane = 16; + +size_t ceilDiv(size_t a, size_t b) { return (a + b - 1) / b; } +size_t aMvinDepthLines(size_t m, size_t k) { return m * (k / kBankLane); } +size_t bMvinDepthLines(size_t k, size_t n) { return k * (n / kBankLane); } +size_t cMvoutDepthLines(size_t m, size_t n) { return m * (n / kBankLane); } + +// Copies B [N, K] into a new [K, N] buffer. +Value transposeB(PatternRewriter &rewriter, Location loc, Value b) { + auto type = cast(b.getType()); + int64_t n = type.getShape()[0], k = type.getShape()[1]; + Value transposed = rewriter.create( + loc, MemRefType::get({k, n}, type.getElementType())); + Value zero = rewriter.create(loc, 0); + Value one = rewriter.create(loc, 1); + auto nLoop = rewriter.create( + loc, zero, rewriter.create(loc, n), one); + OpBuilder::InsertionGuard guard(rewriter); + rewriter.setInsertionPointToStart(nLoop.getBody()); + auto kLoop = rewriter.create( + loc, zero, rewriter.create(loc, k), one); + rewriter.setInsertionPointToStart(kLoop.getBody()); + Value nIv = nLoop.getInductionVar(), kIv = kLoop.getInductionVar(); + Value value = rewriter.create(loc, b, ValueRange{nIv, kIv}); + rewriter.create(loc, value, transposed, + ValueRange{kIv, nIv}); + return transposed; +} + +// Drops a leading unit dimension of a rank-3 memref. +Value collapseUnitBatch(PatternRewriter &rewriter, Location loc, Value value) { + auto type = cast(value.getType()); + if (type.getRank() != 3 || type.getShape()[0] != 1) + return value; + return rewriter.create( + loc, value, ArrayRef{{0, 1}, {2}}); +} + +class MatMulLowering : public OpRewritePattern { +public: + MatMulLowering(MLIRContext *context, int64_t bankDepth) + : OpRewritePattern(context), bankDepth(bankDepth) {} + + LogicalResult matchAndRewrite(linalg::MatmulOp op, + PatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + Attribute maps = op->getAttr("indexing_maps"); + bool isTransposeB = linalg::MatmulTransposeBOp::isDefaultIndexingMaps(maps); + if (!linalg::MatmulOp::isDefaultIndexingMaps(maps) && !isTransposeB) + return failure(); + for (Value operand : op->getOperands()) { + auto type = dyn_cast(operand.getType()); + if (!type || !type.hasStaticShape() || + (type.getRank() != 2 && + !(type.getRank() == 3 && type.getShape()[0] == 1))) + return failure(); + } + Type aElement = + cast(op.getInputs()[0].getType()).getElementType(); + Type bElement = + cast(op.getInputs()[1].getType()).getElementType(); + Type cElement = + cast(op.getOutputs()[0].getType()).getElementType(); + if (!(aElement.isF32() && bElement.isInteger(8) && cElement.isF32()) && + !((aElement.isF32() || aElement.isBF16()) && bElement == aElement && + cElement == aElement) && + !(aElement.isInteger(8) && bElement.isInteger(8) && + cElement.isInteger(32))) + return failure(); + + Value aMemArray = collapseUnitBatch(rewriter, loc, op.getInputs()[0]); + Value bMemArray = collapseUnitBatch(rewriter, loc, op.getInputs()[1]); + Value cMemArray = collapseUnitBatch(rewriter, loc, op.getOutputs()[0]); + if (isTransposeB) + bMemArray = transposeB(rewriter, loc, bMemArray); + + auto aType = cast(aMemArray.getType()); + auto bType = cast(bMemArray.getType()); + auto cType = cast(cMemArray.getType()); + + auto aShape = aType.getShape(); + auto bShape = bType.getShape(); + auto cShape = cType.getShape(); + size_t M = aShape[aShape.size() - 2]; + size_t K = aShape[aShape.size() - 1]; + size_t N = bShape[bShape.size() - 1]; + + if (bShape[bShape.size() - 2] != (int64_t)K || + cShape[cShape.size() - 2] != (int64_t)M || + cShape[cShape.size() - 1] != (int64_t)N) + return op.emitError("matmul input/output shapes mismatch"); + + const bool isQuantized = aType.getElementType().isF32() && + bType.getElementType().isInteger(8) && + cType.getElementType().isF32(); + const bool isWideFloat = + (aType.getElementType().isF32() || aType.getElementType().isBF16()) && + bType.getElementType() == aType.getElementType() && + cType.getElementType() == aType.getElementType(); + const bool isInt8 = aType.getElementType().isInteger(8) && + bType.getElementType().isInteger(8) && + cType.getElementType().isInteger(32); + if (!isQuantized && !isWideFloat && !isInt8) + return failure(); + auto dwAddrAttr = op->getAttrOfType("dw_addr"); + auto dwBytesAttr = op->getAttrOfType("dw_bytes"); + auto perChannelAttr = op->getAttrOfType("per_channel"); + if (isQuantized && (!dwAddrAttr || !dwBytesAttr || !perChannelAttr || + dwAddrAttr.getInt() < 16)) + return op.emitError("quantized matmul requires RAX Dw metadata"); + + constexpr size_t mTileSize = 8; + size_t M_pad = ceilDiv(M, mTileSize) * mTileSize; + size_t K_pad = ceilDiv(K, kBankLane) * kBankLane; + size_t N_pad = ceilDiv(N, kBankLane) * kBankLane; + bool needPadding = (M_pad != M) || (K_pad != K) || (N_pad != N); + + Value aMemArrayPadded = aMemArray; + Value bMemArrayPadded = bMemArray; + Value cMemArrayPadded = cMemArray; + + if (needPadding) { + auto aPadType = MemRefType::get({(int64_t)M_pad, (int64_t)K_pad}, + aType.getElementType()); + auto bPadType = MemRefType::get({(int64_t)K_pad, (int64_t)N_pad}, + bType.getElementType()); + auto cPadType = MemRefType::get({(int64_t)M_pad, (int64_t)N_pad}, + cType.getElementType()); + + aMemArrayPadded = rewriter.create(loc, aPadType); + bMemArrayPadded = rewriter.create(loc, bPadType); + cMemArrayPadded = rewriter.create(loc, cPadType); + + Value aZero = rewriter.create( + loc, aType.getElementType(), + rewriter.getZeroAttr(aType.getElementType())); + Value bZero = rewriter.create( + loc, bType.getElementType(), + rewriter.getZeroAttr(bType.getElementType())); + Value cZero = rewriter.create( + loc, cType.getElementType(), + rewriter.getZeroAttr(cType.getElementType())); + rewriter.create(loc, aZero, aMemArrayPadded); + rewriter.create(loc, bZero, bMemArrayPadded); + rewriter.create(loc, cZero, cMemArrayPadded); + + Value aView = rewriter.create( + loc, aMemArrayPadded, + SmallVector{rewriter.getIndexAttr(0), + rewriter.getIndexAttr(0)}, + SmallVector{rewriter.getIndexAttr(M), + rewriter.getIndexAttr(K)}, + SmallVector{rewriter.getIndexAttr(1), + rewriter.getIndexAttr(1)}); + rewriter.create(loc, aMemArray, aView); + + Value bView = rewriter.create( + loc, bMemArrayPadded, + SmallVector{rewriter.getIndexAttr(0), + rewriter.getIndexAttr(0)}, + SmallVector{rewriter.getIndexAttr(K), + rewriter.getIndexAttr(N)}, + SmallVector{rewriter.getIndexAttr(1), + rewriter.getIndexAttr(1)}); + rewriter.create(loc, bMemArray, bView); + } + + size_t M_tiling = needPadding ? M_pad : M; + size_t K_tiling = needPadding ? K_pad : K; + size_t N_tiling = needPadding ? N_pad : N; + + if (bankDepth <= 0 || (size_t)bankDepth % kBankLane != 0) + return op.emitError("bankDepth must be a positive multiple of bank lane"); + if (M_tiling % mTileSize || K_tiling % kBankLane || N_tiling % kBankLane) + return op.emitError( + "M must be a multiple of 8 and K/N must be multiples of bank lane"); + if (isQuantized) { + const int64_t dwRequired = perChannelAttr.getValue() ? N_tiling * 4 : 4; + if (dwBytesAttr.getInt() < dwRequired) + return op.emitError("Dw scale image does not cover padded N"); + } + + // Bank unit on M (depth) and N (lane). Quantized matmul carries one + // logical K into SMatMul so FP2INT computes one activation scale. + const size_t nTileSize = kBankLane; + const size_t kTileSize = + (isWideFloat || isQuantized) ? K_tiling : kBankLane; + if (M_tiling % mTileSize != 0) + return op.emitError("M does not split into bank-depth units"); + + if (!isWideFloat && !isQuantized) { + if (aMvinDepthLines(mTileSize, kTileSize) > (size_t)bankDepth || + bMvinDepthLines(kTileSize, nTileSize) > (size_t)bankDepth || + cMvoutDepthLines(mTileSize, nTileSize) > (size_t)bankDepth) + return op.emitError( + "bank-unit matmul exceeds 8-bit mvin or 32-bit mvout depth"); + } + + const size_t kTileNum = ceilDiv(K_tiling, kTileSize); + + OpBuilder::InsertionGuard guard(rewriter); + Value zeroIdx = rewriter.create(loc, 0); + Value mStepVal = rewriter.create(loc, mTileSize); + Value kStepVal = rewriter.create(loc, kTileSize); + Value mUpperVal = rewriter.create(loc, M_tiling); + Value kUpperVal = rewriter.create(loc, K_tiling); + + auto mLoop = rewriter.create(loc, zeroIdx, mUpperVal, mStepVal); + rewriter.setInsertionPointToStart(mLoop.getBody()); + Value mIv = mLoop.getInductionVar(); + + if (isQuantized) { + Value aTile = rewriter.create( + loc, aMemArrayPadded, + SmallVector{mIv, rewriter.getIndexAttr(0)}, + SmallVector{rewriter.getIndexAttr(mTileSize), + rewriter.getIndexAttr(K_tiling)}, + SmallVector{rewriter.getIndexAttr(1), + rewriter.getIndexAttr(1)}); + Value bTile = rewriter.create( + loc, bMemArrayPadded, + SmallVector{rewriter.getIndexAttr(0), + rewriter.getIndexAttr(0)}, + SmallVector{rewriter.getIndexAttr(K_tiling), + rewriter.getIndexAttr(N_tiling)}, + SmallVector{rewriter.getIndexAttr(1), + rewriter.getIndexAttr(1)}); + Value cTile = rewriter.create( + loc, cMemArrayPadded, + SmallVector{mIv, rewriter.getIndexAttr(0)}, + SmallVector{rewriter.getIndexAttr(mTileSize), + rewriter.getIndexAttr(N_tiling)}, + SmallVector{rewriter.getIndexAttr(1), + rewriter.getIndexAttr(1)}); + Value zero = rewriter.create( + loc, cType.getElementType(), + rewriter.getZeroAttr(cType.getElementType())); + rewriter.create(loc, zero, cTile); + auto smatmul = rewriter.create(loc, aTile, bTile, cTile); + smatmul->setAttr("dwAddr", dwAddrAttr); + smatmul->setAttr("dwBytes", dwBytesAttr); + smatmul->setAttr("perChannel", perChannelAttr); + } else { + // N is static. Non-quantized matmul remains one physical N panel per op. + for (size_t n0 = 0; n0 < N_tiling; n0 += nTileSize) { + Value cTile = rewriter.create( + loc, cMemArrayPadded, + SmallVector{mIv, rewriter.getIndexAttr(n0)}, + SmallVector{rewriter.getIndexAttr(mTileSize), + rewriter.getIndexAttr(nTileSize)}, + SmallVector{rewriter.getIndexAttr(1), + rewriter.getIndexAttr(1)}); + + auto cElemType = cType.getElementType(); + Value zero = rewriter.create( + loc, cElemType, rewriter.getZeroAttr(cElemType)); + rewriter.create(loc, zero, cTile); + + if (kTileNum == 1) { + Value aTile = rewriter.create( + loc, aMemArrayPadded, + SmallVector{mIv, rewriter.getIndexAttr(0)}, + SmallVector{rewriter.getIndexAttr(mTileSize), + rewriter.getIndexAttr(kTileSize)}, + SmallVector{rewriter.getIndexAttr(1), + rewriter.getIndexAttr(1)}); + Value bTile = rewriter.create( + loc, bMemArrayPadded, + SmallVector{rewriter.getIndexAttr(0), + rewriter.getIndexAttr(n0)}, + SmallVector{rewriter.getIndexAttr(kTileSize), + rewriter.getIndexAttr(nTileSize)}, + SmallVector{rewriter.getIndexAttr(1), + rewriter.getIndexAttr(1)}); + auto smatmul = + rewriter.create(loc, aTile, bTile, cTile); + } else { + auto kLoop = + rewriter.create(loc, zeroIdx, kUpperVal, kStepVal); + rewriter.setInsertionPointToStart(kLoop.getBody()); + Value kIv = kLoop.getInductionVar(); + Value aTile = rewriter.create( + loc, aMemArrayPadded, SmallVector{mIv, kIv}, + SmallVector{rewriter.getIndexAttr(mTileSize), + rewriter.getIndexAttr(kTileSize)}, + SmallVector{rewriter.getIndexAttr(1), + rewriter.getIndexAttr(1)}); + Value bTile = rewriter.create( + loc, bMemArrayPadded, + SmallVector{kIv, rewriter.getIndexAttr(n0)}, + SmallVector{rewriter.getIndexAttr(kTileSize), + rewriter.getIndexAttr(nTileSize)}, + SmallVector{rewriter.getIndexAttr(1), + rewriter.getIndexAttr(1)}); + if (isWideFloat) + return op.emitError( + "FP32 matmul must pass full K in one smatmul_matmul"); + // 8-bit legalize overwrites C; accumulate via host partial. + auto partialTy = MemRefType::get( + {(int64_t)mTileSize, (int64_t)nTileSize}, cElemType); + Value partial = rewriter.create(loc, partialTy); + rewriter.create(loc, zero, partial); + rewriter.create(loc, aTile, bTile, partial); + rewriter.create( + loc, ValueRange{cTile, partial}, ValueRange{cTile}, + linalg::ElementwiseKind::add); + rewriter.create(loc, partial); + rewriter.setInsertionPointAfter(kLoop); + } + } + } + + rewriter.setInsertionPointAfter(mLoop); + + if (needPadding) { + Value cView = rewriter.create( + loc, cMemArrayPadded, + SmallVector{rewriter.getIndexAttr(0), + rewriter.getIndexAttr(0)}, + SmallVector{rewriter.getIndexAttr(M), + rewriter.getIndexAttr(N)}, + SmallVector{rewriter.getIndexAttr(1), + rewriter.getIndexAttr(1)}); + rewriter.create(loc, cView, cMemArray); + rewriter.create(loc, aMemArrayPadded); + rewriter.create(loc, bMemArrayPadded); + rewriter.create(loc, cMemArrayPadded); + } + + if (isTransposeB) + rewriter.create(loc, bMemArray); + rewriter.eraseOp(op); + return success(); + } + +private: + int64_t bankDepth; +}; + +// One matmul per batch; each is claimed by MatMulLowering. +class BatchMatMulLowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(linalg::BatchMatmulOp op, + PatternRewriter &rewriter) const override { + Attribute maps = op->getAttr("indexing_maps"); + bool isTransposeB = + linalg::BatchMatmulTransposeBOp::isDefaultIndexingMaps(maps); + if (!linalg::BatchMatmulOp::isDefaultIndexingMaps(maps) && !isTransposeB) + return failure(); + Location loc = op.getLoc(); + Value a = op.getInputs()[0], b = op.getInputs()[1], c = op.getOutputs()[0]; + auto aType = dyn_cast(a.getType()); + auto bType = dyn_cast(b.getType()); + auto cType = dyn_cast(c.getType()); + if (!aType || !bType || !cType || !aType.hasStaticShape() || + !bType.hasStaticShape() || !cType.hasStaticShape()) + return failure(); + auto slice = [&](Value value, MemRefType type, int64_t batch) -> Value { + ArrayRef shape = type.getShape(); + Value sub = rewriter.create( + loc, value, ArrayRef{batch, 0, 0}, + ArrayRef{1, shape[1], shape[2]}, ArrayRef{1, 1, 1}); + return rewriter.create( + loc, sub, ArrayRef{{0, 1}, {2}}); + }; + for (int64_t batch = 0; batch < aType.getShape()[0]; ++batch) { + Value bBatch = slice(b, bType, batch); + if (isTransposeB) + bBatch = transposeB(rewriter, loc, bBatch); + rewriter.create( + loc, ValueRange{slice(a, aType, batch), bBatch}, + ValueRange{slice(c, cType, batch)}); + if (isTransposeB) + rewriter.create(loc, bBatch); + } + rewriter.eraseOp(op); + return success(); + } +}; + +} // namespace + +namespace mlir::buddy { +void populateSMatMulBallLinalgPatterns(RewritePatternSet &patterns, + int64_t bankDepth) { + patterns.add(patterns.getContext(), bankDepth); + patterns.add(patterns.getContext()); +} +} // namespace mlir::buddy diff --git a/examples/balls/smatmul/compiler/src/Dialect/Buckyball/SMatMulBall.td b/waste/examples/components/balls/smatmul/stack/src/Dialect/Buckyball/SMatMulBall.td similarity index 92% rename from examples/balls/smatmul/compiler/src/Dialect/Buckyball/SMatMulBall.td rename to waste/examples/components/balls/smatmul/stack/src/Dialect/Buckyball/SMatMulBall.td index 0d51988e1..d76555db9 100644 --- a/examples/balls/smatmul/compiler/src/Dialect/Buckyball/SMatMulBall.td +++ b/waste/examples/components/balls/smatmul/stack/src/Dialect/Buckyball/SMatMulBall.td @@ -48,7 +48,7 @@ def SMatMulBiasOp : Buckyball_Op<"smatmul_bias"> { "type($biasBankId)"; } -def BankSMatMulOp : Buckyball_Op<"bank_smatmul"> { +def BankSMatMulOp : Buckyball_Op<"bank_smatmul", [DeclareOpInterfaceMethods]> { let summary = "SMatMulBall compute with virtual bank handles"; let arguments = (ins I64:$op1Bank, I64:$op2Bank, I64:$wrBank, I64:$config, I1:$first, I1:$last, I64:$outputBase); @@ -59,7 +59,7 @@ def BankSMatMulOp : Buckyball_Op<"bank_smatmul"> { "type($op1Bank) type($op2Bank) type($wrBank) type($config)"; } -def BankSMatMulBiasOp : Buckyball_Op<"bank_smatmul_bias"> { +def BankSMatMulBiasOp : Buckyball_Op<"bank_smatmul_bias", [DeclareOpInterfaceMethods]> { let summary = "SMatMul bias preload with a virtual bank handle"; let arguments = (ins I64:$biasBank, I64:$inputBase); let results = (outs I64:$biasBankOut); diff --git a/waste/examples/components/balls/smatmul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/smatmul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..5ea5463c5 --- /dev/null +++ b/waste/examples/components/balls/smatmul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,263 @@ +//===- LegalizeForLLVMExport.cpp - SMatMulBall LLVM lowering --------------===// + +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" +#include "Utils/BankUtils.h" + +#include "llvm/Support/ErrorHandling.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { + +uint64_t matrixCfg(uint64_t rows, uint64_t cols, bool first = true, + bool last = true) { + if (rows == 0 || rows > 0xfff || (rows != 1 && rows % 8) || cols != 16) + llvm::report_fatal_error( + "matrix cfg: rows must be 1 or a multiple of 8 and cols must be 16"); + return fieldBits(rows, 0, 11) | fieldBits(cols, 12, 23) | + (uint64_t(first) << 24) | (uint64_t(last) << 25); +} + +struct SMatMulMatmulLowering : public ConvertOpToLLVMPattern { + SMatMulMatmulLowering(LLVMTypeConverter &converter, bool /*stable*/) + : ConvertOpToLLVMPattern(converter) {} + + LogicalResult + matchAndRewrite(SMatMulMatmulOp op, OpAdaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + Value aMem = op.getAMemArray(); + Value bMem = op.getBMemArray(); + Value cMem = op.getCMemArray(); + + auto aTy = dyn_cast(aMem.getType()); + auto bTy = dyn_cast(bMem.getType()); + auto cTy = dyn_cast(cMem.getType()); + if (!aTy || !bTy || !cTy || !aTy.hasStaticShape() || + !bTy.hasStaticShape() || !cTy.hasStaticShape()) + return rewriter.notifyMatchFailure( + op, "buckyball.smatmul_matmul requires static memref shapes"); + + uint64_t m = aTy.getShape()[0]; + uint64_t k = aTy.getShape()[1]; + uint64_t kb = bTy.getShape()[0]; + uint64_t n = bTy.getShape()[1]; + if (k != kb || cTy.getShape()[0] != (int64_t)m || + cTy.getShape()[1] != (int64_t)n) + return rewriter.notifyMatchFailure(op, "matmul shapes mismatch"); + const int64_t bankDepth = buckyball_target::getBuckyballTarget().bankDepth; + if (bankDepth <= 0) + return op.emitError("smatmul lowering requires target bankDepth > 0"); + // A occupies m lines, B occupies k lines, and packed int32 C occupies 2m. + if (m == 0 || n != 16 || k != 16 || m % 8 != 0 || + m > static_cast(bankDepth / 2) || + k > static_cast(bankDepth)) + return rewriter.notifyMatchFailure( + op, "SMatMul requires K/N exactly 16 and A/B/C to fit their banks"); + if (!aTy.getElementType().isInteger(8) || + !bTy.getElementType().isInteger(8) || + !cTy.getElementType().isInteger(32)) + return rewriter.notifyMatchFailure( + op, "SMatMul Ball supports only 8-bit A/B with 32-bit C"); + + const uint64_t aBank = 0; + const uint64_t bBank = 1; + const uint64_t cBank = 2; + uint64_t depthA = m; // K <= 16 => one bank row per M row + uint64_t depthB = k; + uint64_t depthC = 2 * m; + + emitMset(rewriter, loc, aBank, 1, 1, 1); + emitMset(rewriter, loc, bBank, 1, 1, 1); + emitMset(rewriter, loc, cBank, 1, 2, 1); + + Value aPtr = extractPtr(rewriter, loc, aMem); + Value bPtr = extractPtr(rewriter, loc, bMem); + auto packedType = MemRefType::get({static_cast(depthC), 8}, + cTy.getElementType()); + Value packed = rewriter.create(loc, packedType); + Value packedPtr = extractPtr(rewriter, loc, packed); + + Value rs1A = packRs1BankIter(rewriter, loc, cstI64(rewriter, loc, aBank), + cstI64(rewriter, loc, depthA)); + Value rs2A = + packRs2MemStride(rewriter, loc, aPtr, cstI64(rewriter, loc, 1)); + + rewriter.create(loc, rs1A, rs2A); + + Value rs1B = packRs1BankIter(rewriter, loc, cstI64(rewriter, loc, bBank), + cstI64(rewriter, loc, depthB)); + Value rs2B = + packRs2MemStride(rewriter, loc, bPtr, cstI64(rewriter, loc, 1)); + + rewriter.create(loc, rs1B, rs2B); + + rewriter.create( + loc, + packRs1BanksIter(rewriter, loc, cstI64(rewriter, loc, aBank), + cstI64(rewriter, loc, bBank), + cstI64(rewriter, loc, cBank), + cstI64(rewriter, loc, k)), + cstI64(rewriter, loc, matrixCfg(m, n)), + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("SMATMUL_OS"))); + + Value rs1C = packRs1BankIter(rewriter, loc, cstI64(rewriter, loc, cBank), + cstI64(rewriter, loc, depthC)); + Value rs2C = + packRs2MemStride(rewriter, loc, packedPtr, cstI64(rewriter, loc, 1)); + + rewriter.create(loc, rs1C, rs2C); + + Value indexZero = rewriter.create(loc, 0); + Value indexOne = rewriter.create(loc, 1); + Value indexTwo = rewriter.create(loc, 2); + Value indexEight = rewriter.create(loc, 8); + Value indexM = rewriter.create(loc, m); + Value indexN = rewriter.create(loc, n); + auto rowLoop = + rewriter.create(loc, indexZero, indexM, indexOne); + rewriter.setInsertionPointToStart(rowLoop.getBody()); + Value row = rowLoop.getInductionVar(); + auto columnLoop = + rewriter.create(loc, indexZero, indexN, indexOne); + rewriter.setInsertionPointToStart(columnLoop.getBody()); + Value column = columnLoop.getInductionVar(); + Value packedRow = rewriter.create( + loc, rewriter.create(loc, row, indexTwo), + rewriter.create(loc, column, indexEight)); + Value packedColumn = + rewriter.create(loc, column, indexEight); + Value value = rewriter.create( + loc, packed, ValueRange{packedRow, packedColumn}); + rewriter.create(loc, value, cMem, ValueRange{row, column}); + rewriter.setInsertionPointAfter(rowLoop); + rewriter.create(loc, packed); + + emitMset(rewriter, loc, aBank, 0, 0, 0); + emitMset(rewriter, loc, bBank, 0, 0, 0); + emitMset(rewriter, loc, cBank, 0, 0, 0); + + rewriter.eraseOp(op); + return success(); + } +}; + +struct SMatMulLowering : public ConvertOpToLLVMPattern { + SMatMulLowering(LLVMTypeConverter &converter, bool, int64_t bankDepth) + : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} + + LogicalResult + matchAndRewrite(SMatMulOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + auto config = op.getConfig().getDefiningOp(); + if (!config) + return op.emitError("smatmul config must be a constant"); + auto configAttr = dyn_cast(config.getValue()); + if (!configAttr) + return op.emitError("smatmul config must be an integer constant"); + uint64_t cfg = configAttr.getValue().getZExtValue(); + uint64_t rows = cfg & 0xfff; + uint64_t cols = (cfg >> 12) & 0xfff; + uint64_t k = (cfg >> 24) & 0xfff; + if (rows == 0 || (rows != 1 && rows % 8) || cols != 16 || k == 0 || + k % 16 || rows * 4 > static_cast(bankDepth)) + return op.emitError("SMatMul matrix shape or C footprint is invalid"); + auto base = adaptor.getOutputBase().getDefiningOp(); + auto baseAttr = + base ? dyn_cast(base.getValue()) : IntegerAttr(); + if (!baseAttr || baseAttr.getInt() < 0 || baseAttr.getInt() > 63 || + baseAttr.getInt() + rows * 4 > bankDepth) + return op.emitError( + "SMatMul outputBase must be constant and fit C in bank"); + Value rs1 = packRs1BanksIter( + rewriter, loc, adaptor.getOp1BankId(), adaptor.getOp2BankId(), + adaptor.getResultBankId(), cstI64(rewriter, loc, k)); + Value rs2 = cstI64(rewriter, loc, matrixCfg(rows, cols, false, false)); + Value first = rewriter.create(loc, rewriter.getI64Type(), + adaptor.getFirst()); + Value last = rewriter.create(loc, rewriter.getI64Type(), + adaptor.getLast()); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, first, cstI64(rewriter, loc, 24))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, last, cstI64(rewriter, loc, 25))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getOutputBase(), + cstI64(rewriter, loc, 26))); + rewriter.replaceOpWithNewOp( + op, rs1, rs2, + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("SMATMUL_OS"))); + return success(); + } + +private: + int64_t bankDepth; +}; + +struct SMatMulBiasLowering : public ConvertOpToLLVMPattern { + SMatMulBiasLowering(LLVMTypeConverter &converter, bool) + : ConvertOpToLLVMPattern(converter) {} + + LogicalResult + matchAndRewrite(SMatMulBiasOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + auto base = adaptor.getInputBase().getDefiningOp(); + auto baseAttr = + base ? dyn_cast(base.getValue()) : IntegerAttr(); + if (!baseAttr || baseAttr.getInt() < 0 || + baseAttr.getInt() + 4 > + static_cast( + buckyball_target::getBuckyballTarget().bankDepth)) + return op.emitError( + "SMatMul bias inputBase must be constant and fit in bank"); + Value rs1 = packRs1BankIter(rewriter, loc, adaptor.getBiasBankId(), + cstI64(rewriter, loc, 4)); + rewriter.replaceOpWithNewOp( + op, rs1, adaptor.getInputBase(), + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("SMATMUL_BIAS"))); + return success(); + } +}; + +} // namespace + +namespace mlir::buddy::buckyball { +void populateSMatMulBallLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, + int64_t bankDepth) { + patterns.add(converter, stable); + patterns.add(converter, stable, bankDepth); + patterns.add(converter, stable); +} + +void configureSMatMulBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool /*stable*/) { + target.addIllegalOp(); + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/smatmul/stack/src/Passes.cpp b/waste/examples/components/balls/smatmul/stack/src/Passes.cpp new file mode 100644 index 000000000..2ae76231f --- /dev/null +++ b/waste/examples/components/balls/smatmul/stack/src/Passes.cpp @@ -0,0 +1,86 @@ +//===- Passes.cpp - SMatMulBall passes ------------------------------------===// + +#include "Conversion/BallPasses.h" + +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" + +using namespace mlir; + +namespace mlir::buddy { +void populateSMatMulBallLinalgPatterns(RewritePatternSet &patterns, + int64_t bankDepth); +void populateMatmulRegionToBankSSAPatterns(RewritePatternSet &patterns); +namespace buckyball { +void populateSMatMulBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureSMatMulBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LinalgToSMatMulPass + : public mlir::buddy::LinalgToBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LinalgToSMatMulPass) + StringRef getArgument() const final { + return "convert-linalg-to-smatmul-ball"; + } + StringRef getDescription() const final { + return "Lower the Linalg operations SMatMulBall executes."; + } + void populate(RewritePatternSet &patterns, int64_t, int64_t bankDepth, + int64_t) const { + mlir::buddy::populateSMatMulBallLinalgPatterns(patterns, bankDepth); + } +}; + +class LowerSMatMulPass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerSMatMulPass) + StringRef getArgument() const final { return "lower-smatmul-ball"; } + StringRef getDescription() const final { + return "Lower SMatMulBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateSMatMulBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureSMatMulBallLegalizeForExportTarget(target, + stable); + } +}; + +// Lowers MegaKernels made of MatMul stages as one resident bank region. +class LowerMatmulRegionPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerMatmulRegionPass) + StringRef getArgument() const final { + return "lower-matmul-region-to-bank-ssa"; + } + StringRef getDescription() const final { + return "Lower MatMul MegaKernels to bank-SSA operations."; + } + void getDependentDialects(DialectRegistry ®istry) const override { + registry + .insert(); + } + void runOnOperation() override { + RewritePatternSet patterns(&getContext()); + mlir::buddy::populateMatmulRegionToBankSSAPatterns(patterns); + if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) + signalPassFailure(); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + PassRegistration(); + PassRegistration(); + return true; +}(); diff --git a/waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..b55db8d4e --- /dev/null +++ b/waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,33 @@ +//===- AssignPhysicalBanks.cpp - ToInt8 bank assignment -------------------===// + +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankQuantF32ToI8Op::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankQuantF32ToI8Op op = *this; + rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), + op.getIter(), op.getScale()); + rewriter.replaceOp(op, op.getOutBank()); + return success(); +} + +LogicalResult +BankQuantI32ToI8Op::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankQuantI32ToI8Op op = *this; + rewriter.create( + op.getLoc(), op.getInBank(), op.getScaleBank(), op.getOutBank(), + op.getIter(), op.getOutputBase(), op.getInputBase(), + op.getOutputWidthAttr(), op.getOutputHeightAttr(), + op.getOutputStrideAttr(), op.getReluAttr()); + rewriter.replaceOp(op, op.getOutBank()); + return success(); +} diff --git a/waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp b/waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp new file mode 100644 index 000000000..e1051330d --- /dev/null +++ b/waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp @@ -0,0 +1,156 @@ +//===- LowerToBankSSA.cpp - Input quantization ----------------------------===// + +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Target/BuckyballTargetRegistry.h" +#include "Utils/BankUtils.h" + +#include + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult QuantizeTensorF32ToI8Op::lowerToBankSSA(PatternRewriter &b) { + QuantizeTensorF32ToI8Op op = *this; + Location loc = op.getLoc(); + auto inputTy = dyn_cast(op.getInput().getType()); + auto outputTy = dyn_cast(op.getOutput().getType()); + if (!inputTy || !outputTy || !inputTy.hasStaticShape() || + !outputTy.hasStaticShape() || inputTy.getRank() != outputTy.getRank() || + (inputTy.getRank() != 2 && inputTy.getRank() != 4) || + !inputTy.getElementType().isF32() || + !outputTy.getElementType().isInteger(8)) + return op.emitError( + "input quantization requires static rank-2/rank-4 memrefs"); + ArrayRef inputShape = inputTy.getShape(); + ArrayRef outputShape = outputTy.getShape(); + bool nchwToNhwc = op.getNchwToNhwcAttr().getValue(); + if ((!nchwToNhwc && inputShape != outputShape) || + (nchwToNhwc && inputTy.getRank() != 4) || + (nchwToNhwc && + (inputShape[0] != outputShape[0] || inputShape[1] != outputShape[3] || + inputShape[2] != outputShape[1] || inputShape[3] != outputShape[2]))) + return op.emitError("input quantization layout shape mismatch"); + + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || target.bankDepth < 4 || + target.bankDepth % 4) + return op.emitError("input quantization requires 128-bit banks with " + "depth divisible by 4"); + int64_t elements = 1; + for (int64_t dim : outputShape) { + if (dim <= 0) + return op.emitError("input quantization requires positive dimensions"); + elements *= dim; + } + + Value inputBank = allocBank(b, loc, 1, 1); + Value outputBank = allocBank(b, loc, 1, 1); + Value scale = + b.create(loc, b.getF32Type(), op.getScaleAttr()); + Value zero = b.create(loc, 0); + Value one = b.create(loc, 1); + Value four = b.create(loc, 4); + Value sixteen = b.create(loc, 16); + Value zeroF32 = + b.create(loc, b.getF32Type(), b.getF32FloatAttr(0.0)); + int64_t maxElements = target.bankDepth * 4; + + Value elementCount = b.create(loc, elements); + Value chunkSize = b.create(loc, maxElements); + auto chunkLoop = b.create(loc, zero, elementCount, chunkSize, + ValueRange{inputBank, outputBank}); + b.setInsertionPointToStart(chunkLoop.getBody()); + Value offset = chunkLoop.getInductionVar(); + ValueRange bankStates = chunkLoop.getRegionIterArgs(); + auto inputPackTy = MemRefType::get({target.bankDepth, 4}, b.getF32Type()); + auto outputPackTy = + MemRefType::get({target.bankDepth / 4, 16}, b.getI8Type()); + Value inputPack = b.create(loc, inputPackTy); + Value outputPack = b.create(loc, outputPackTy); + b.create(loc, zeroF32, inputPack); + + auto copyIn = b.create(loc, zero, chunkSize, one); + b.setInsertionPointToStart(copyIn.getBody()); + Value i = copyIn.getInductionVar(); + Value linear = b.create(loc, offset, i); + auto inputValid = b.create( + loc, + b.create(loc, arith::CmpIPredicate::slt, linear, + elementCount), + false); + b.setInsertionPointToStart(&inputValid.getThenRegion().front()); + SmallVector outputIndices(outputTy.getRank()); + Value remaining = linear; + for (int64_t dim = outputTy.getRank() - 1; dim >= 0; --dim) { + Value extent = b.create(loc, outputShape[dim]); + outputIndices[dim] = b.create(loc, remaining, extent); + remaining = b.create(loc, remaining, extent); + } + SmallVector indices(outputTy.getRank()); + if (nchwToNhwc) { + indices[0] = outputIndices[0]; + indices[1] = outputIndices[3]; + indices[2] = outputIndices[1]; + indices[3] = outputIndices[2]; + } else { + indices = outputIndices; + } + Value value = b.create(loc, op.getInput(), indices); + Value packRow = b.create(loc, i, four); + Value packColumn = b.create(loc, i, four); + b.create(loc, value, inputPack, + ValueRange{packRow, packColumn}); + b.setInsertionPointAfter(inputValid); + b.setInsertionPointAfter(copyIn); + + Value inputNext = + mvinBank(b, loc, inputPack, bankStates[0], target.bankDepth); + Value outputNext = b.create( + loc, bankStates[1].getType(), inputNext, bankStates[1], + createI64Const(b, loc, target.bankDepth), scale); + outputNext = mvoutBank(b, loc, outputPack, outputNext, target.bankDepth / 4); + + auto copyOut = b.create(loc, zero, chunkSize, one); + b.setInsertionPointToStart(copyOut.getBody()); + i = copyOut.getInductionVar(); + linear = b.create(loc, offset, i); + auto outputValid = b.create( + loc, + b.create(loc, arith::CmpIPredicate::slt, linear, + elementCount), + false); + b.setInsertionPointToStart(&outputValid.getThenRegion().front()); + remaining = linear; + for (int64_t dim = outputTy.getRank() - 1; dim >= 0; --dim) { + Value extent = b.create(loc, outputShape[dim]); + indices[dim] = b.create(loc, remaining, extent); + remaining = b.create(loc, remaining, extent); + } + packRow = b.create(loc, i, sixteen); + packColumn = b.create(loc, i, sixteen); + value = b.create(loc, outputPack, + ValueRange{packRow, packColumn}); + b.create(loc, value, op.getOutput(), indices); + b.setInsertionPointAfter(outputValid); + b.setInsertionPointAfter(copyOut); + b.create(loc, inputPack); + b.create(loc, outputPack); + b.create(loc, ValueRange{inputNext, outputNext}); + b.setInsertionPointAfter(chunkLoop); + inputBank = chunkLoop.getResult(0); + outputBank = chunkLoop.getResult(1); + + releaseBank(b, loc, inputBank); + releaseBank(b, loc, outputBank); + b.eraseOp(op); + return success(); +} diff --git a/waste/examples/components/balls/toint8/stack/src/Conversion/LowerLinalgToBuckyball/QuantizePatterns.cpp b/waste/examples/components/balls/toint8/stack/src/Conversion/LowerLinalgToBuckyball/QuantizePatterns.cpp new file mode 100644 index 000000000..3e2c5f4f6 --- /dev/null +++ b/waste/examples/components/balls/toint8/stack/src/Conversion/LowerLinalgToBuckyball/QuantizePatterns.cpp @@ -0,0 +1,64 @@ +//===- QuantizePatterns.cpp - marked FP32-to-INT8 input quantization -----===// +// +// Claims the generics the frontend marked `buckyball.quant_f32_to_i8`. +// +//===----------------------------------------------------------------------===// + +#include + +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class QuantF32ToI8Lowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(linalg::GenericOp op, + PatternRewriter &rewriter) const override { + if (!op->hasAttr("buckyball.quant_f32_to_i8")) + return failure(); + if (op.getInputs().size() != 1 || op.getOutputs().size() != 1) + return op.emitError("FP32-to-INT8 quant requires one input and output"); + Value input = op.getInputs()[0]; + Value output = op.getOutputs()[0]; + auto inputType = dyn_cast(input.getType()); + auto outputType = dyn_cast(output.getType()); + auto nchwToNhwc = op->getAttrOfType("nchw_to_nhwc"); + auto scale = op->getAttrOfType("scale"); + if (!inputType || !outputType || !inputType.hasStaticShape() || + !outputType.hasStaticShape() || + inputType.getRank() != outputType.getRank() || + (inputType.getRank() != 2 && inputType.getRank() != 4) || + !inputType.getElementType().isF32() || + !outputType.getElementType().isInteger(8) || !nchwToNhwc || !scale || + !std::isfinite(scale.getValueAsDouble()) || + scale.getValueAsDouble() <= 0.0) + return op.emitError("FP32-to-INT8 quant requires matching static " + "rank-2/rank-4 memrefs and a positive scale"); + ArrayRef in = inputType.getShape(); + ArrayRef out = outputType.getShape(); + if (nchwToNhwc.getValue() + ? inputType.getRank() != 4 || in[0] != out[0] || in[1] != out[3] || + in[2] != out[1] || in[3] != out[2] + : in != out) + return op.emitError("FP32-to-INT8 quant layout shape mismatch"); + rewriter.create(op.getLoc(), input, output, scale, + nchwToNhwc); + rewriter.eraseOp(op); + return success(); + } +}; + +} // namespace + +namespace mlir::buddy { +void populateToInt8BallLinalgPatterns(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} +} // namespace mlir::buddy diff --git a/examples/balls/toint8/compiler/src/Dialect/Buckyball/ToInt8Ball.td b/waste/examples/components/balls/toint8/stack/src/Dialect/Buckyball/ToInt8Ball.td similarity index 77% rename from examples/balls/toint8/compiler/src/Dialect/Buckyball/ToInt8Ball.td rename to waste/examples/components/balls/toint8/stack/src/Dialect/Buckyball/ToInt8Ball.td index 0cc1ea566..86c927de6 100644 --- a/examples/balls/toint8/compiler/src/Dialect/Buckyball/ToInt8Ball.td +++ b/waste/examples/components/balls/toint8/stack/src/Dialect/Buckyball/ToInt8Ball.td @@ -17,7 +17,7 @@ def QuantF32ToI8Op : Buckyball_Op<"quant_f32_to_i8"> { "prop-dict attr-dict `:` type($inputBankId)"; } -def BankQuantF32ToI8Op : Buckyball_Op<"bank_quant_f32_to_i8"> { +def BankQuantF32ToI8Op : Buckyball_Op<"bank_quant_f32_to_i8", [DeclareOpInterfaceMethods]> { let summary = "Per-tensor FP32 to INT8 quantization with virtual banks"; let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, F32:$scale); let results = (outs I64:$outBankOut); @@ -42,7 +42,7 @@ def QuantI32ToI8Op : Buckyball_Op<"quant_i32_to_i8"> { "type($inputBankId) type($outputBase) type($inputBase)"; } -def BankQuantI32ToI8Op : Buckyball_Op<"bank_quant_i32_to_i8"> { +def BankQuantI32ToI8Op : Buckyball_Op<"bank_quant_i32_to_i8", [DeclareOpInterfaceMethods]> { let summary = "Per-channel INT32 to INT8 requantization with virtual banks"; let arguments = (ins I64:$inBank, I64:$scaleBank, I64:$outBank, I64:$iter, I64:$outputBase, @@ -56,4 +56,14 @@ def BankQuantI32ToI8Op : Buckyball_Op<"bank_quant_i32_to_i8"> { "type($inputBase)"; } +def QuantizeTensorF32ToI8Op : Buckyball_Op<"quantize_tensor_f32_to_i8", + [DeclareOpInterfaceMethods]> { + let summary = "Quantize one FP32 model input tensor to INT8"; + let arguments = (ins MemRefRankOf<[F32], [2, 4]>:$input, + MemRefRankOf<[I8], [2, 4]>:$output, + F32Attr:$scale, + BoolAttr:$nchwToNhwc); + let assemblyFormat = [{$input $output prop-dict attr-dict `:` type($input) type($output)}]; +} + #endif // TOINT8BALL_OPS diff --git a/waste/examples/components/balls/toint8/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/toint8/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..ad0541f1d --- /dev/null +++ b/waste/examples/components/balls/toint8/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,122 @@ +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +struct QuantF32ToI8Lowering : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + LogicalResult + matchAndRewrite(QuantF32ToI8Op op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), + cstI64(rewriter, loc, 0), + adaptor.getOutputBankId(), adaptor.getIter()); + Value scaleBits = rewriter.create( + loc, rewriter.getI32Type(), adaptor.getScale()); + Value rs2 = + rewriter.create(loc, rewriter.getI64Type(), scaleBits); + rewriter.replaceOpWithNewOp( + op, rs1, rs2, + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("QUANT_F32_TO_I8"))); + return success(); + } +}; + +struct QuantI32ToI8Lowering : public ConvertOpToLLVMPattern { + QuantI32ToI8Lowering(LLVMTypeConverter &converter, int64_t bankDepth) + : ConvertOpToLLVMPattern(converter), + bankDepth(bankDepth) {} + + LogicalResult + matchAndRewrite(QuantI32ToI8Op op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), + adaptor.getScaleBankId(), + adaptor.getOutputBankId(), adaptor.getIter()); + auto iterOp = op.getIter().getDefiningOp(); + auto iterAttr = + iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); + if (!iterAttr || iterAttr.getInt() <= 0) + return op.emitError("INT32 to INT8 iter must be a positive constant"); + if (iterAttr.getInt() % 4 != 0 || iterAttr.getInt() > bankDepth) + return op.emitError( + "INT32 to INT8 iter must be a multiple of four within bank depth"); + int64_t outputRows = iterAttr.getInt() / 4; + int64_t outputWidth = op.getOutputWidthAttr().getInt(); + int64_t outputHeight = op.getOutputHeightAttr().getInt(); + int64_t outputStride = op.getOutputStrideAttr().getInt(); + auto outputBaseOp = + adaptor.getOutputBase().getDefiningOp(); + auto inputBaseOp = + adaptor.getInputBase().getDefiningOp(); + auto outputBaseAttr = outputBaseOp + ? dyn_cast(outputBaseOp.getValue()) + : IntegerAttr(); + auto inputBaseAttr = inputBaseOp + ? dyn_cast(inputBaseOp.getValue()) + : IntegerAttr(); + if (outputWidth <= 0 || outputHeight <= 0 || outputStride < outputWidth || + outputWidth * outputHeight != outputRows || outputWidth >= 128 || + outputHeight >= 128 || outputStride >= 128 || !outputBaseAttr || + !inputBaseAttr || outputBaseAttr.getInt() < 0 || + inputBaseAttr.getInt() < 0 || + outputBaseAttr.getInt() + (outputHeight - 1) * outputStride + + outputWidth > + bankDepth || + inputBaseAttr.getInt() + iterAttr.getInt() > bankDepth) + return op.emitError( + "INT32 to INT8 output tile is invalid for the physical bank"); + Value rs2 = rewriter.create(loc, adaptor.getOutputBase(), + cstI64(rewriter, loc, 1)); + rs2 = rewriter.create( + loc, rs2, + cstI64(rewriter, loc, + (outputWidth << 8) | (outputHeight << 15) | + (outputStride << 22) | (op.getRelu() ? 1 : 0))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getInputBase(), + cstI64(rewriter, loc, 29))); + rewriter.replaceOpWithNewOp( + op, rs1, rs2, + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("QUANT_I32_TO_I8"))); + return success(); + } + +private: + int64_t bankDepth; +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateToInt8BallLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, + int64_t bankDepth) { + (void)stable; + patterns.add(converter); + patterns.add(converter, bankDepth); +} + +void configureToInt8BallLegalizeForExportTarget(LLVMConversionTarget &target, + bool stable) { + (void)stable; + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/toint8/stack/src/Passes.cpp b/waste/examples/components/balls/toint8/stack/src/Passes.cpp new file mode 100644 index 000000000..9290eed04 --- /dev/null +++ b/waste/examples/components/balls/toint8/stack/src/Passes.cpp @@ -0,0 +1,57 @@ +//===- Passes.cpp - ToInt8Ball passes -------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +void populateToInt8BallLinalgPatterns(RewritePatternSet &patterns); +namespace buckyball { +void populateToInt8BallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureToInt8BallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LinalgToToInt8Pass + : public mlir::buddy::LinalgToBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LinalgToToInt8Pass) + StringRef getArgument() const final { + return "convert-linalg-to-toint8-ball"; + } + StringRef getDescription() const final { + return "Lower the Linalg operations ToInt8Ball executes."; + } + void populate(RewritePatternSet &patterns, int64_t, int64_t bankDepth, + int64_t) const { + mlir::buddy::populateToInt8BallLinalgPatterns(patterns); + } +}; + +class LowerToInt8Pass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerToInt8Pass) + StringRef getArgument() const final { return "lower-toint8-ball"; } + StringRef getDescription() const final { + return "Lower ToInt8Ball operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateToInt8BallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureToInt8BallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + PassRegistration(); + return true; +}(); diff --git a/examples/balls/trace/compiler/src/Dialect/Buckyball/TraceBall.td b/waste/examples/components/balls/trace/stack/src/Dialect/Buckyball/TraceBall.td similarity index 100% rename from examples/balls/trace/compiler/src/Dialect/Buckyball/TraceBall.td rename to waste/examples/components/balls/trace/stack/src/Dialect/Buckyball/TraceBall.td diff --git a/waste/examples/components/balls/trace/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/trace/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..befba6731 --- /dev/null +++ b/waste/examples/components/balls/trace/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,55 @@ +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +class BdbCounterLowering : public ConvertOpToLLVMPattern { +public: + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + LogicalResult + matchAndRewrite(BdbCounterOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + Value counter = rewriter.create(loc, adaptor.getCounterId(), + cstI64(rewriter, loc, 4)); + Value payload = rewriter.create(loc, adaptor.getPayload(), + cstI64(rewriter, loc, 8)); + Value rs2 = + rewriter.create(loc, adaptor.getSubcommand(), counter); + rs2 = rewriter.create(loc, rs2, payload); + rewriter.replaceOpWithNewOp( + op, cstI64(rewriter, loc, 0), rs2, + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("BDB_COUNTER"))); + return success(); + } +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateTraceBallLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, + int64_t bankDepth) { + (void)stable; + (void)bankDepth; + + patterns.add(converter); +} + +void configureTraceBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool stable) { + (void)stable; + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/trace/stack/src/Passes.cpp b/waste/examples/components/balls/trace/stack/src/Passes.cpp new file mode 100644 index 000000000..b96f3b28e --- /dev/null +++ b/waste/examples/components/balls/trace/stack/src/Passes.cpp @@ -0,0 +1,39 @@ +//===- Passes.cpp - TraceBall passes --------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +namespace buckyball { +void populateTraceBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureTraceBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LowerTracePass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerTracePass) + StringRef getArgument() const final { return "lower-trace-ball"; } + StringRef getDescription() const final { + return "Lower TraceBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateTraceBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureTraceBallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + return true; +}(); diff --git a/examples/balls/transpose/compiler/src/Backend/include/IntrinsicsRISCVTransposeBall.td b/waste/examples/components/balls/transpose/stack/src/Backend/include/IntrinsicsRISCVTransposeBall.td similarity index 100% rename from examples/balls/transpose/compiler/src/Backend/include/IntrinsicsRISCVTransposeBall.td rename to waste/examples/components/balls/transpose/stack/src/Backend/include/IntrinsicsRISCVTransposeBall.td diff --git a/waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..7668e9666 --- /dev/null +++ b/waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,20 @@ +//===- AssignPhysicalBanks.cpp - Transpose bank assignment ----------------===// + +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankTransposeOp::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankTransposeOp op = *this; + rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), + op.getIter(), op.getElemBits()); + rewriter.replaceOp(op, op.getOutBank()); + return success(); +} diff --git a/waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp b/waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp new file mode 100644 index 000000000..f573e06b9 --- /dev/null +++ b/waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp @@ -0,0 +1,34 @@ +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/Utils/StructuredOpsUtils.h" +#include "mlir/IR/AffineMap.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult MemTransposeOp::lowerToBankSSA(PatternRewriter &rewriter) { + MemTransposeOp op = *this; + auto input = dyn_cast(op.getInput().getType()); + auto output = dyn_cast(op.getOutput().getType()); + if (!input || !output || !input.hasStaticShape() || + !output.hasStaticShape() || input.getRank() != 2 || + output.getRank() != 2 || output.getShape()[0] != input.getShape()[1] || + output.getShape()[1] != input.getShape()[0]) + return op.emitError("requires matching static rank-2 transpose memrefs"); + MLIRContext *context = rewriter.getContext(); + AffineMap identity = AffineMap::getMultiDimIdentityMap(2, context); + AffineMap transpose = AffineMap::get( + 2, 0, {rewriter.getAffineDimExpr(1), rewriter.getAffineDimExpr(0)}, + context); + SmallVector iterators(2, utils::IteratorType::parallel); + rewriter.create( + op.getLoc(), TypeRange{}, ValueRange{op.getInput()}, + ValueRange{op.getOutput()}, ArrayRef{identity, transpose}, + iterators, [](OpBuilder &builder, Location location, ValueRange args) { + builder.create(location, args[0]); + }); + rewriter.eraseOp(op); + return success(); +} diff --git a/waste/examples/components/balls/transpose/stack/src/Conversion/LowerLinalgToBuckyball/TransposePatterns.cpp b/waste/examples/components/balls/transpose/stack/src/Conversion/LowerLinalgToBuckyball/TransposePatterns.cpp new file mode 100644 index 000000000..ad3050a65 --- /dev/null +++ b/waste/examples/components/balls/transpose/stack/src/Conversion/LowerLinalgToBuckyball/TransposePatterns.cpp @@ -0,0 +1,263 @@ +//===- TransposePatterns.cpp - linalg.transpose -> buckyball.mem_transpose ===// +// +// Rank-2 transposes run on the Ball directly. Higher-rank permutations the +// Ball supports are decomposed into rank-2 transposes of contiguous slices; +// other permutations stay for the CPU. +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +class TransposeLowering : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(linalg::TransposeOp op, + PatternRewriter &rewriter) const override { + Value input = op.getInput(); + Value output = op.getInit(); + Location loc = op.getLoc(); + auto inT = dyn_cast(input.getType()); + auto outT = dyn_cast(output.getType()); + if (!inT || !outT || !inT.hasStaticShape() || !outT.hasStaticShape() || + inT.getElementType() != outT.getElementType()) + return failure(); + + ArrayRef perm = op.getPermutation(); + ArrayRef is = inT.getShape(); + ArrayRef os = outT.getShape(); + Type elem = inT.getElementType(); + SmallVector temps; + + // The Ball and collapse_shape need identity-layout buffers. + auto asContig = [&](Value v) -> Value { + auto t = cast(v.getType()); + if (t.getLayout().isIdentity()) + return v; + Value b = rewriter.create( + loc, MemRefType::get(t.getShape(), t.getElementType())); + rewriter.create(loc, v, b); + temps.push_back(b); + return b; + }; + auto dstContig = [&]() -> Value { + if (outT.getLayout().isIdentity()) + return output; + Value b = + rewriter.create(loc, MemRefType::get(os, elem)); + temps.push_back(b); + return b; + }; + auto collapse = [&](Value src, ArrayRef shape2, + ArrayRef reassoc) -> Value { + return rewriter.create( + loc, MemRefType::get(shape2, elem), src, reassoc); + }; + auto finish = [&](Value dst) { + if (dst != output) + rewriter.create(loc, dst, output); + for (Value t : llvm::reverse(temps)) + rewriter.create(loc, t); + rewriter.eraseOp(op); + return success(); + }; + auto idx = [&](int64_t v) -> Value { + return rewriter.create(loc, v); + }; + auto ofr = [&](Value v) { return OpFoldResult(v); }; + // Copies one slice of a rank-4 source into a buffer and collapses it to + // rank 2. Returns {collapsed, buffer}; the caller deallocates the buffer. + auto slice2D = + [&](Value src4, ArrayRef off, ArrayRef sz4, + ArrayRef shape2, + ArrayRef reassoc) -> std::pair { + SmallVector sz = {ofr(idx(sz4[0])), ofr(idx(sz4[1])), + ofr(idx(sz4[2])), ofr(idx(sz4[3]))}; + SmallVector str(4, ofr(idx(1))); + Value sub = rewriter.create(loc, src4, off, sz, str); + Value buf = + rewriter.create(loc, MemRefType::get(sz4, elem)); + rewriter.create(loc, sub, buf); + return {collapse(buf, shape2, reassoc), buf}; + }; + // Transposes a rank-2 slice and stores it into a slice of the destination. + auto transposeStore = [&](Value in2, Value dst4, ArrayRef off, + ArrayRef outSz4, + ArrayRef shape2, + ArrayRef reassoc) { + Value outBuf = + rewriter.create(loc, MemRefType::get(outSz4, elem)); + rewriter.create(loc, in2, + collapse(outBuf, shape2, reassoc)); + SmallVector sz = {ofr(idx(outSz4[0])), ofr(idx(outSz4[1])), + ofr(idx(outSz4[2])), ofr(idx(outSz4[3]))}; + SmallVector str(4, ofr(idx(1))); + rewriter.create( + loc, outBuf, + rewriter.create(loc, dst4, off, sz, str)); + rewriter.create(loc, outBuf); + }; + auto loop = [&](int64_t upper) { + auto forOp = rewriter.create(loc, idx(0), idx(upper), idx(1)); + rewriter.setInsertionPointToStart(forOp.getBody()); + return forOp; + }; + // One Ball transpose of the whole tensor viewed as [rows, cols]. + auto whole = [&](ArrayRef in2, ArrayRef inR, + ArrayRef out2, + ArrayRef outR) { + Value src = asContig(input), dst = dstContig(); + rewriter.create(loc, collapse(src, in2, inR), + collapse(dst, out2, outR)); + return finish(dst); + }; + + if (inT.getRank() == 2 && perm == ArrayRef({1, 0})) { + Value dst = dstContig(); + rewriter.create(loc, asContig(input), dst); + return finish(dst); + } + + if (inT.getRank() == 3 && perm == ArrayRef({0, 2, 1}) && + is[0] == 1) + return whole({is[1], is[2]}, {{0, 1}, {2}}, {os[1], os[2]}, + {{0, 1}, {2}}); + + if (inT.getRank() != 4) + return failure(); + Value c0 = idx(0); + + // Moving a channel across singleton spatial dimensions keeps byte order. + if (perm == ArrayRef({0, 3, 1, 2}) && is[0] == 1 && is[1] == 1 && + is[2] == 1) { + Value src = asContig(input), dst = dstContig(); + rewriter.create( + loc, collapse(src, {1, is[3]}, {{0, 1, 2}, {3}}), + collapse(dst, {1, is[3]}, {{0}, {1, 2, 3}})); + return finish(dst); + } + + // 1x1 OIHW->OHWI is layout-identical after collapsing to [O,I]. + if (perm == ArrayRef({0, 2, 3, 1}) && is[2] == 1 && is[3] == 1) { + Value src = asContig(input), dst = dstContig(); + rewriter.create( + loc, collapse(src, {is[0], is[1]}, {{0}, {1, 2, 3}}), + collapse(dst, {os[0], os[3]}, {{0}, {1, 2, 3}})); + return finish(dst); + } + + // N=1 NCHW->NHWC: [1,C,H,W] -> [1,H,W,C]. + if (perm == ArrayRef({0, 2, 3, 1}) && is[0] == 1) + return whole({is[1], is[2] * is[3]}, {{0, 1}, {2, 3}}, + {is[2] * is[3], is[1]}, {{0, 1, 2}, {3}}); + + // Weight OIHW->OHWI: one [I, H*W] transpose per O. + if (perm == ArrayRef({0, 2, 3, 1})) { + int64_t i = is[1], h = is[2], w = is[3]; + Value src = asContig(input), dst = dstContig(); + auto oLoop = loop(is[0]); + SmallVector off = {ofr(oLoop.getInductionVar()), ofr(c0), + ofr(c0), ofr(c0)}; + auto [in2, inBuf] = + slice2D(src, off, {1, i, h, w}, {i, h * w}, {{0, 1}, {2, 3}}); + transposeStore(in2, dst, off, {1, h, w, i}, {h * w, i}, {{0, 1, 2}, {3}}); + rewriter.create(loc, inBuf); + rewriter.setInsertionPointAfter(oLoop); + return finish(dst); + } + + // N=1 NHWC->NCHW: [1,A,B,C] -> [1,C,A,B]. + if (perm == ArrayRef({0, 3, 1, 2}) && is[0] == 1) + return whole({is[1] * is[2], is[3]}, {{0, 1, 2}, {3}}, + {is[3], is[1] * is[2]}, {{0, 1}, {2, 3}}); + + // [N,A,B,C] -> [N,C,B,A]. + if (perm == ArrayRef({0, 3, 2, 1})) { + int64_t n = is[0], a = is[1], b = is[2], c = is[3]; + if (n == 1) + return whole({a * b, c}, {{0, 1, 2}, {3}}, {c, a * b}, + {{0, 1}, {2, 3}}); + Value src = asContig(input), dst = dstContig(); + auto nLoop = loop(n); + Value ni = nLoop.getInductionVar(); + if (b <= a) { + auto bLoop = loop(b); + SmallVector off = {ofr(ni), ofr(c0), + ofr(bLoop.getInductionVar()), ofr(c0)}; + auto [in2, inBuf] = + slice2D(src, off, {1, a, 1, c}, {a, c}, {{0, 1}, {2, 3}}); + transposeStore(in2, dst, off, {1, c, 1, a}, {c, a}, {{0, 1}, {2, 3}}); + rewriter.create(loc, inBuf); + } else { + auto aLoop = loop(a); + Value ai = aLoop.getInductionVar(); + SmallVector inOff = {ofr(ni), ofr(ai), ofr(c0), ofr(c0)}; + SmallVector outOff = {ofr(ni), ofr(c0), ofr(c0), ofr(ai)}; + auto [in2, inBuf] = + slice2D(src, inOff, {1, 1, b, c}, {b, c}, {{0, 1, 2}, {3}}); + transposeStore(in2, dst, outOff, {1, c, b, 1}, {c, b}, + {{0, 1}, {2, 3}}); + rewriter.create(loc, inBuf); + } + rewriter.setInsertionPointAfter(nLoop); + return finish(dst); + } + + // [N,A,B,C] -> [N,B,A,C]: one [A,B] transpose per (n, c). + if (perm == ArrayRef({0, 2, 1, 3})) { + int64_t a = is[1], b = is[2]; + Value src = asContig(input), dst = dstContig(); + auto nLoop = loop(is[0]); + auto cLoop = loop(is[3]); + SmallVector off = {ofr(nLoop.getInductionVar()), ofr(c0), + ofr(c0), ofr(cLoop.getInductionVar())}; + auto [in2, inBuf] = + slice2D(src, off, {1, a, b, 1}, {a, b}, {{0, 1}, {2, 3}}); + transposeStore(in2, dst, off, {1, b, a, 1}, {b, a}, {{0, 1}, {2, 3}}); + rewriter.create(loc, inBuf); + rewriter.setInsertionPointAfter(nLoop); + return finish(dst); + } + + // [N,H,A,B] -> [N,H,B,A]: one [A,B] transpose per (n, h). + if (perm == ArrayRef({0, 1, 3, 2})) { + int64_t a = is[2], b = is[3]; + Value src = asContig(input), dst = dstContig(); + auto nLoop = loop(is[0]); + auto hLoop = loop(is[1]); + SmallVector off = {ofr(nLoop.getInductionVar()), + ofr(hLoop.getInductionVar()), ofr(c0), + ofr(c0)}; + auto [in2, inBuf] = + slice2D(src, off, {1, 1, a, b}, {a, b}, {{0, 1, 2}, {3}}); + transposeStore(in2, dst, off, {1, 1, b, a}, {b, a}, {{0, 1, 2}, {3}}); + rewriter.create(loc, inBuf); + rewriter.setInsertionPointAfter(nLoop); + return finish(dst); + } + + // Depthwise OIHW->HWIO: [C,1,k,k] -> [k,k,C,1]. + if (perm == ArrayRef({2, 3, 0, 1}) && is[1] == 1) + return whole({is[0], is[2] * is[3]}, {{0, 1}, {2, 3}}, + {is[2] * is[3], is[0]}, {{0, 1}, {2, 3}}); + + return failure(); + } +}; +} // namespace + +namespace mlir::buddy { +void populateTransposeBallLinalgPatterns(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} +} // namespace mlir::buddy diff --git a/waste/examples/components/balls/transpose/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/transpose/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..0a3281237 --- /dev/null +++ b/waste/examples/components/balls/transpose/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,115 @@ +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +static LogicalResult validateTranspose(Operation *op, Value elemBits, + Value iter, int64_t bankDepth) { + auto constant = elemBits.getDefiningOp(); + auto value = + constant ? dyn_cast(constant.getValue()) : IntegerAttr(); + if (!value || (value.getInt() != 8 && value.getInt() != 32)) + return op->emitError("Transpose elemBits must be constant 8 or 32"); + auto iterOp = iter.getDefiningOp(); + auto iterAttr = + iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); + if (!iterAttr || iterAttr.getInt() <= 0 || iterAttr.getInt() > bankDepth) + return op->emitError( + "Transpose iter must be a positive constant within the bank depth"); + return success(); +} + +struct TransposeLowering : public ConvertOpToLLVMPattern { + TransposeLowering(LLVMTypeConverter &converter, bool stable, + int64_t bankDepth) + : ConvertOpToLLVMPattern(converter), stable(stable), + bankDepth(bankDepth) {} + + LogicalResult + matchAndRewrite(TransposeOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + if (failed( + validateTranspose(op, op.getElemBits(), op.getIter(), bankDepth))) + return failure(); + Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), + cstI64(rewriter, loc, 0), + adaptor.getOutputBankId(), adaptor.getIter()); + if (stable) { + rewriter.replaceOpWithNewOp(op, rs1, + adaptor.getElemBits()); + return success(); + } + rewriter.replaceOpWithNewOp( + op, rs1, adaptor.getElemBits(), + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("TRANSPOSE"))); + return success(); + } + +private: + bool stable = false; + int64_t bankDepth; +}; +struct BankTransposeLowering : public ConvertOpToLLVMPattern { + BankTransposeLowering(LLVMTypeConverter &converter, bool stable, + int64_t bankDepth) + : ConvertOpToLLVMPattern(converter), stable(stable), + bankDepth(bankDepth) {} + + LogicalResult + matchAndRewrite(BankTransposeOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + if (failed( + validateTranspose(op, op.getElemBits(), op.getIter(), bankDepth))) + return failure(); + Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInBank(), + cstI64(rewriter, loc, 0), adaptor.getOutBank(), + adaptor.getIter()); + if (stable) { + rewriter.replaceOpWithNewOp(op, rs1, + adaptor.getElemBits()); + return success(); + } + rewriter.replaceOpWithNewOp( + op, rs1, adaptor.getElemBits(), + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("TRANSPOSE"))); + return success(); + } + +private: + bool stable = false; + int64_t bankDepth; +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateTransposeBallLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, + int64_t bankDepth) { + patterns.add(converter, stable, bankDepth); + patterns.add(converter, stable, bankDepth); +} + +void configureTransposeBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool stable) { + if (stable) + target.addLegalOp(); + else + target.addIllegalOp(); + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/examples/balls/transpose/compiler/src/Dialect/Buckyball/TransposeBall.td b/waste/examples/components/balls/transpose/stack/src/Dialect/Buckyball/TransposeBall.td similarity index 83% rename from examples/balls/transpose/compiler/src/Dialect/Buckyball/TransposeBall.td rename to waste/examples/components/balls/transpose/stack/src/Dialect/Buckyball/TransposeBall.td index 0cf5f4898..f7cdf2480 100644 --- a/examples/balls/transpose/compiler/src/Dialect/Buckyball/TransposeBall.td +++ b/waste/examples/components/balls/transpose/stack/src/Dialect/Buckyball/TransposeBall.td @@ -9,13 +9,13 @@ include "Buckyball.td" -// Memref-level transpose (like smatmul_matmul). Chip bank-ssa decides +// Memref-level transpose (like smatmul_matmul). Design bank-ssa decides // pad / tile / Bank* composition; this op carries no bank policy. -def MemTransposeOp : Buckyball_Op<"mem_transpose"> { +def MemTransposeOp : Buckyball_Op<"mem_transpose", [DeclareOpInterfaceMethods]> { let summary = "Logical matrix transpose on memrefs"; let description = [{ Transposes input[R,C] into output[C,R]. Element type and layout - constraints are enforced by the chip bank-ssa lowering. + constraints are enforced by the design bank-ssa lowering. }]; let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, MemRefRankOf<[AnyType], [2]>:$output); @@ -38,7 +38,7 @@ def TransposeOp : Buckyball_Op<"transpose"> { "type($inputBankId)"; } -def BankTransposeOp : Buckyball_Op<"bank_transpose"> { +def BankTransposeOp : Buckyball_Op<"bank_transpose", [DeclareOpInterfaceMethods]> { let summary = "Transpose using virtual bank handles"; let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, I64:$elemBits); let results = (outs I64:$outBankOut); diff --git a/waste/examples/components/balls/transpose/stack/src/Passes.cpp b/waste/examples/components/balls/transpose/stack/src/Passes.cpp new file mode 100644 index 000000000..08e941ee2 --- /dev/null +++ b/waste/examples/components/balls/transpose/stack/src/Passes.cpp @@ -0,0 +1,59 @@ +//===- Passes.cpp - TransposeBall passes ----------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +void populateTransposeBallLinalgPatterns(RewritePatternSet &patterns); +namespace buckyball { +void populateTransposeBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, + bool, int64_t); +void configureTransposeBallLegalizeForExportTarget(LLVMConversionTarget &, + bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LinalgToTransposePass + : public mlir::buddy::LinalgToBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LinalgToTransposePass) + StringRef getArgument() const final { + return "convert-linalg-to-transpose-ball"; + } + StringRef getDescription() const final { + return "Lower the Linalg operations TransposeBall executes."; + } + void populate(RewritePatternSet &patterns, int64_t, int64_t bankDepth, + int64_t) const { + mlir::buddy::populateTransposeBallLinalgPatterns(patterns); + } +}; + +class LowerTransposePass + : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerTransposePass) + StringRef getArgument() const final { return "lower-transpose-ball"; } + StringRef getDescription() const final { + return "Lower TransposeBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateTransposeBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureTransposeBallLegalizeForExportTarget( + target, stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + PassRegistration(); + return true; +}(); diff --git a/examples/balls/vector/compiler/src/Backend/include/IntrinsicsRISCVVecBall.td b/waste/examples/components/balls/vector/stack/src/Backend/include/IntrinsicsRISCVVecBall.td similarity index 100% rename from examples/balls/vector/compiler/src/Backend/include/IntrinsicsRISCVVecBall.td rename to waste/examples/components/balls/vector/stack/src/Backend/include/IntrinsicsRISCVVecBall.td diff --git a/waste/examples/components/balls/vector/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/vector/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp new file mode 100644 index 000000000..bd46db46f --- /dev/null +++ b/waste/examples/components/balls/vector/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp @@ -0,0 +1,20 @@ +//===- AssignPhysicalBanks.cpp - Vector bank assignment -------------------===// + +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult +BankVecMat16Op::assignPhysicalBanks(PatternRewriter &rewriter, + mlir::buddy::PhysicalBankState &) { + BankVecMat16Op op = *this; + rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), + op.getWrBank(), op.getIter(), op.getMode()); + rewriter.replaceOp(op, op.getWrBank()); + return success(); +} diff --git a/waste/examples/components/balls/vector/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/vector/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp new file mode 100644 index 000000000..27fb18385 --- /dev/null +++ b/waste/examples/components/balls/vector/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp @@ -0,0 +1,47 @@ +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using namespace ::buddy::buckyball::legalize; + +namespace { +struct VecMat16Lowering : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + LogicalResult + matchAndRewrite(VecMat16Op op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getOp1BankId(), + adaptor.getOp2BankId(), adaptor.getWrBankId(), + adaptor.getIter()); + rewriter.replaceOpWithNewOp( + op, rs1, adaptor.getMode(), + rewriter.getI32IntegerAttr( + buckyball_target::getBuckyballFunct7("VECMAT16"))); + return success(); + } +}; +} // namespace + +namespace mlir::buddy::buckyball { +void populateVecBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, + RewritePatternSet &patterns, + bool stable, int64_t) { + (void)stable; + patterns.add(converter); +} + +void configureVecBallLegalizeForExportTarget(LLVMConversionTarget &target, + bool stable) { + (void)stable; + target.addIllegalOp(); +} +} // namespace mlir::buddy::buckyball diff --git a/examples/balls/vector/compiler/src/Dialect/Buckyball/VecBall.td b/waste/examples/components/balls/vector/stack/src/Dialect/Buckyball/VecBall.td similarity index 92% rename from examples/balls/vector/compiler/src/Dialect/Buckyball/VecBall.td rename to waste/examples/components/balls/vector/stack/src/Dialect/Buckyball/VecBall.td index b3fdec455..10aa2da48 100644 --- a/examples/balls/vector/compiler/src/Dialect/Buckyball/VecBall.td +++ b/waste/examples/components/balls/vector/stack/src/Dialect/Buckyball/VecBall.td @@ -27,7 +27,7 @@ def VecMat16Op : Buckyball_Op<"vecmat16"> { "type($op1BankId)"; } -def BankVecMat16Op : Buckyball_Op<"bank_vecmat16"> { +def BankVecMat16Op : Buckyball_Op<"bank_vecmat16", [DeclareOpInterfaceMethods]> { let summary = "Warp-16 matmul using virtual bank handles"; let arguments = (ins I64:$op1Bank, I64:$op2Bank, I64:$wrBank, I64:$iter, I64:$mode); diff --git a/waste/examples/components/balls/vector/stack/src/Passes.cpp b/waste/examples/components/balls/vector/stack/src/Passes.cpp new file mode 100644 index 000000000..7d61a6486 --- /dev/null +++ b/waste/examples/components/balls/vector/stack/src/Passes.cpp @@ -0,0 +1,39 @@ +//===- Passes.cpp - VecBall passes ----------------------------------------===// + +#include "Conversion/BallPasses.h" + +using namespace mlir; + +namespace mlir::buddy { +namespace buckyball { +void populateVecBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, + RewritePatternSet &, bool, + int64_t); +void configureVecBallLegalizeForExportTarget(LLVMConversionTarget &, bool); +} // namespace buckyball +} // namespace mlir::buddy + +namespace { +class LowerVecPass : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerVecPass) + StringRef getArgument() const final { return "lower-vector-ball"; } + StringRef getDescription() const final { + return "Lower VecBall operations to LLVM intrinsics."; + } + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool stable, + int64_t bankDepth) const { + mlir::buddy::buckyball::populateVecBallLegalizeForLLVMExportPatterns( + converter, patterns, stable, bankDepth); + mlir::buddy::buckyball::configureVecBallLegalizeForExportTarget(target, + stable); + } +}; +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + mlir::buddy::registerBallLoweringPass(); + return true; +}(); diff --git a/waste/examples/designs/goban/stack/midend/src/FormMegaKernels.cpp b/waste/examples/designs/goban/stack/midend/src/FormMegaKernels.cpp new file mode 100644 index 000000000..6a6575584 --- /dev/null +++ b/waste/examples/designs/goban/stack/midend/src/FormMegaKernels.cpp @@ -0,0 +1,33 @@ +#include "Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Pass/Pass.h" + +using namespace mlir; +namespace { +class FormGobanMegaKernelsPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(FormGobanMegaKernelsPass) + StringRef getArgument() const final { return "form-goban-mega-kernels"; } + StringRef getDescription() const final { + return "Fuse quantized matrix operators within one Goban core"; + } + void runOnOperation() override { + DenseMap> blocks; + getOperation().walk([&](linalg::GenericOp op) { + if (op->hasAttr("buckyball.quantized_matmul")) + blocks[op->getBlock()].push_back(op); + }); + for (auto &entry : blocks) { + mlir::buddy::prepareQuantizedWeights(entry.second); + mlir::buddy::formMegaKernelRegions(entry.second); + } + } +}; +[[maybe_unused]] const bool registered = [] { + PassRegistration(); + return true; +}(); +} // namespace diff --git a/waste/examples/designs/pebble/stack/midend/ops.py b/waste/examples/designs/pebble/stack/midend/ops.py new file mode 100644 index 000000000..23d343b3b --- /dev/null +++ b/waste/examples/designs/pebble/stack/midend/ops.py @@ -0,0 +1,110 @@ +"""Pebble's banked INT8 operator emission.""" + +from copy import deepcopy + +from buddy_mlir import ir +from buddy_mlir.dialects import tosa as dialect + +from stack.compiler.frontend.operation import ( + Int8MatmulOp, + QuantizedDepthwiseConv2dOp, + QuantizedConv2dOp, + QuantizedMatmulOp, +) +from stack.compiler.frontend.layout.provenance import has_nhwc_layout +from stack.compiler.frontend.ops import linalg, tosa +from stack.compiler.frontend.ops.int8 import _rescale + + +def contraction(node, symbols): + linear = isinstance(node, Int8MatmulOp) + if linear: + activation, weight, activation_scale, weight_scale, bias = node.args + operation = QuantizedMatmulOp() + else: + activation, weight = node.args[:2] + ( + bias, + stride, + padding, + dilation, + transposed, + _, + groups, + activation_scale, + weight_scale, + ) = node.args[2:] + source = symbols[("__buddy_ops_by_name__", 0)][activation] + input_shape = list(source.tensor_meta["shape"]) + depthwise = ( + groups > 1 and groups == input_shape[1] == node.tensor_meta["shape"][1] + ) + if ( + transposed + or list(dilation) != [1, 1] + or stride[0] != stride[1] + or padding[0] != padding[1] + or (groups != 1 and not depthwise) + ): + raise ValueError( + "Pebble requires ordinary or depthwise convolution with unit dilation" + ) + operation = QuantizedDepthwiseConv2dOp() if depthwise else QuantizedConv2dOp() + operation._input_shape = input_shape + operation._weight_shape = list( + ir.RankedTensorType(symbols[(weight, 0)].type).shape + ) + operation._output_shape = list(node.tensor_meta["shape"]) + operation._output_nhwc = node.tensor_meta.get("activation_layout") == "nhwc" + operation._stride, operation._padding = stride[0], padding[0] + value = symbols[(activation, 0)] + if not has_nhwc_layout(value): + n, c, h, w = input_shape + value = dialect.TransposeOp( + ir.RankedTensorType.get([n, h, w, c], ir.IntegerType.get_signless(8)), + value, + tosa._create_permutation_attr([0, 2, 3, 1]), + ).result + symbols = {**symbols, (activation, 0): value} + operation.name = node.name + operation._arguments = [activation, weight] + operation.tensor_meta = dict(node.tensor_meta) + operation.trace_meta = deepcopy(node.trace_meta) + channels = node.tensor_meta["shape"][1] + operation._bias_i32 = [0] * channels + operation._dequant_scale = [1.0] * channels + operation._requant_scale = [1.0] * channels + operation._final_output = True + operation._activation = 0 + operation._output_scale = 1.0 + operation._lut_i8 = [0] + result = (linalg.quantized_matmul_op if linear else linalg.quantized_conv2d_op)( + operation, symbols + ) + return _rescale( + result, + activation_scale, + weight_scale, + bias, + symbols, + channel_axis=3 if not linear and operation._output_nhwc else 1, + ) + + +def permute(node, symbols): + # Preserve NHWC storage across a projection's logical NCHW boundary. + if list(node.args[1]) == [0, 3, 1, 2]: + value = symbols[(str(node.args[0]), 0)] + n, c, h, w = node.tensor_meta["shape"] + if list(ir.RankedTensorType(value.type).shape) != [n, h, w, c]: + raise ValueError("Pebble projection boundary requires NHWC storage") + return value + return tosa.permute_op(node, symbols) + + +ops_registry = { + **tosa.ops_registry, + "Int8Conv2dOp": contraction, + "Int8MatmulOp": contraction, + "PermuteOp": permute, +} diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/MegaConv2dToBankSSAPatterns.cpp b/waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/MegaConv2dToBankSSAPatterns.cpp similarity index 98% rename from examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/MegaConv2dToBankSSAPatterns.cpp rename to waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/MegaConv2dToBankSSAPatterns.cpp index b34cf74b3..7cdc7ae1f 100644 --- a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/MegaConv2dToBankSSAPatterns.cpp +++ b/waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/MegaConv2dToBankSSAPatterns.cpp @@ -38,6 +38,8 @@ class MegaConv2dToBankSSAPattern : public OpRewritePattern { if (op->template getParentOfType()) return failure(); bool finalOutput = true; + auto layout = op->template getAttrOfType("output_nhwc"); + bool nchwOutput = !(layout && layout.getValue()); if (!inputTy || !weightTy || !biasTy || !scaleTy || !lutTy || !outputTy || !inputTy.hasStaticShape() || !weightTy.hasStaticShape() || !biasTy.hasStaticShape() || !scaleTy.hasStaticShape() || @@ -66,7 +68,7 @@ class MegaConv2dToBankSSAPattern : public OpRewritePattern { int64_t cin = inputShape[3]; int64_t kernelSize = weightShape[0]; int64_t cout = kDepthwise ? cin : weightShape[3]; - int64_t outputSize = finalOutput ? outputShape[2] : outputShape[1]; + int64_t outputSize = nchwOutput ? outputShape[2] : outputShape[1]; int64_t stride = op.getStride(); int64_t padding = op.getPadLow(); bool weightShapeMatches = @@ -75,9 +77,9 @@ class MegaConv2dToBankSSAPattern : public OpRewritePattern { if (batch <= 0 || inputSize <= 0 || cin <= 0 || kernelSize <= 0 || cout <= 0 || stride <= 0 || padding < 0 || inputShape[2] != inputSize || !weightShapeMatches || outputShape[0] != batch || - (finalOutput ? outputShape[3] != outputSize - : outputShape[2] != outputSize) || - (finalOutput ? outputShape[1] != cout : outputShape[3] != cout) || + (nchwOutput ? outputShape[3] != outputSize + : outputShape[2] != outputSize) || + (nchwOutput ? outputShape[1] != cout : outputShape[3] != cout) || biasTy.getShape()[0] != cout || scaleTy.getShape()[0] != cout) return op.emitError("Mega Conv2D NHWC/HWCF shape mismatch"); if (kernelSize > 7) @@ -90,7 +92,7 @@ class MegaConv2dToBankSSAPattern : public OpRewritePattern { target.bankDepth % 4) return op.emitError( "Mega Conv2D requires 128-bit banks with depth divisible by 4"); - if (buckyball_target::getBuckyballBallMapping("SMatMulBall").outBW != 1) + if (buckyball_target::getBuckyballBallMapping("SMATMUL_OS").outBW != 1) return op.emitError("Mega Conv2D requires SMatMulBall outBW=1"); int64_t kernelRows = (kernelSize * kernelSize + kTile - 1) / kTile * kTile; int64_t windowCapacity = @@ -424,7 +426,7 @@ class MegaConv2dToBankSSAPattern : public OpRewritePattern { loc, arith::CmpIPredicate::slt, globalChannel, coutValue); auto storeOutput = b.create(loc, validChannel, false); b.setInsertionPointToStart(&storeOutput.getThenRegion().front()); - if (finalOutput) + if (nchwOutput) b.create( loc, outputValue, op.getOutput(), ValueRange{outputBatchValue, globalChannel, globalY, globalX}); diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/MemTransposeToBankSSAPatterns.cpp b/waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/MemTransposeToBankSSAPatterns.cpp similarity index 98% rename from examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/MemTransposeToBankSSAPatterns.cpp rename to waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/MemTransposeToBankSSAPatterns.cpp index 62f88c5d7..474594ddc 100644 --- a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/MemTransposeToBankSSAPatterns.cpp +++ b/waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/MemTransposeToBankSSAPatterns.cpp @@ -273,7 +273,10 @@ class MemTransposeToBankSSAPattern : public OpRewritePattern { namespace mlir::buddy { void populatePebbleMemTransposeToBankSSAPatterns(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); + // Pebble keeps transposes on banks; it takes precedence over the transpose + // Ball's own lowering. + patterns.add(patterns.getContext(), + /*benefit=*/2); } } // namespace mlir::buddy diff --git a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/ResidentConvRegionToBankSSAPatterns.cpp b/waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/ResidentConvRegionToBankSSAPatterns.cpp similarity index 96% rename from examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/ResidentConvRegionToBankSSAPatterns.cpp rename to waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/ResidentConvRegionToBankSSAPatterns.cpp index 6eb52e8cf..6c585b498 100644 --- a/examples/cores/pebble/compiler/src/Conversion/LowerBuckyball/ResidentConvRegionToBankSSAPatterns.cpp +++ b/waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/ResidentConvRegionToBankSSAPatterns.cpp @@ -14,6 +14,7 @@ #include "Target/BuckyballTargetRegistry.h" #include "Trace/TraceOps.h" #include "Utils/BankUtils.h" +#include "Utils/StageTrace.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/STLExtras.h" @@ -33,18 +34,8 @@ constexpr int64_t kMvin2dAddressableRows = 64; class ResidentConvRegionPattern : public OpRewritePattern { public: - ResidentConvRegionPattern(MLIRContext *context, bool traceMegaStages, - int64_t traceMegaStageStart, - int64_t traceMegaStageLimit, - int64_t traceMegaRegion, bool traceMegaReloadStages, - bool traceMegaInputBeforeRegion) - : OpRewritePattern(context, 4), - traceMegaStages(traceMegaStages), - traceMegaStageStart(traceMegaStageStart), - traceMegaStageLimit(traceMegaStageLimit), - traceMegaRegion(traceMegaRegion), - traceMegaReloadStages(traceMegaReloadStages), - traceMegaInputBeforeRegion(traceMegaInputBeforeRegion) {} + ResidentConvRegionPattern(MLIRContext *context) + : OpRewritePattern(context, 4) {} LogicalResult matchAndRewrite(MegaKernelOp kernel, PatternRewriter &b) const override { @@ -91,6 +82,7 @@ class ResidentConvRegionPattern : public OpRewritePattern { int64_t scaleH; int64_t scaleW; bool finalOutput; + bool outputNhwc = false; int64_t lutEntries; SmallVector lutOutputScales; }; @@ -111,6 +103,9 @@ class ResidentConvRegionPattern : public OpRewritePattern { // next kernel. Region/value identity alone does not mean this is // the FP32 boundary; the output element type is the contract. bool finalOutput = output && output.getElementType().isF32(); + auto layout = conv->getAttrOfType("output_nhwc"); + bool outputNhwc = layout && layout.getValue(); + bool nchwOutput = finalOutput && !outputNhwc; if (!input || !weight || !bias || !scale || !lut || !output || !input.hasStaticShape() || !weight.hasStaticShape() || !bias.hasStaticShape() || !scale.hasStaticShape() || @@ -133,8 +128,8 @@ class ResidentConvRegionPattern : public OpRewritePattern { auto out = output.getShape(); int64_t kernelSize = conv.getKernel(); int64_t outputChannels = bias.getShape()[0]; - int64_t outputHeight = finalOutput ? out[2] : out[1]; - int64_t outputWidth = finalOutput ? out[3] : out[2]; + int64_t outputHeight = nchwOutput ? out[2] : out[1]; + int64_t outputWidth = nchwOutput ? out[3] : out[2]; int64_t paddedKernel = ((kernelSize * kernelSize + kTile - 1) / kTile) * kTile; if (kernelSize <= 0 || kernelSize > 7 || @@ -153,8 +148,8 @@ class ResidentConvRegionPattern : public OpRewritePattern { (in[2] + 2 * conv.getPadLow() - kernelSize) / conv.getStride() + 1 != outputWidth || - (finalOutput ? (out[1] != outputChannels) - : (out[3] != outputChannels))) + (nchwOutput ? (out[1] != outputChannels) + : (out[3] != outputChannels))) return conv.emitError("Conv shape is inconsistent"); stage.input = conv.getInput(); stage.output = conv.getOutput(); @@ -174,6 +169,7 @@ class ResidentConvRegionPattern : public OpRewritePattern { stage.activation = conv.getActivation(); stage.outputScale = conv.getOutputScale().convertToFloat(); stage.finalOutput = finalOutput; + stage.outputNhwc = outputNhwc; stage.lutEntries = lut.getShape()[0]; if (auto scales = operation.getAttrOfType( "lane_output_scales")) @@ -192,6 +188,9 @@ class ResidentConvRegionPattern : public OpRewritePattern { auto lut = dyn_cast(conv.getLut().getType()); auto output = dyn_cast(conv.getOutput().getType()); bool finalOutput = output && output.getElementType().isF32(); + auto layout = conv->getAttrOfType("output_nhwc"); + bool outputNhwc = layout && layout.getValue(); + bool nchwOutput = finalOutput && !outputNhwc; if (!input || !weight || !bias || !scale || !lut || !output || !input.hasStaticShape() || !weight.hasStaticShape() || !bias.hasStaticShape() || !scale.hasStaticShape() || @@ -215,8 +214,8 @@ class ResidentConvRegionPattern : public OpRewritePattern { auto out = output.getShape(); int64_t kernelSize = conv.getKernel(); int64_t outputChannels = bias.getShape()[0]; - int64_t outputHeight = finalOutput ? out[2] : out[1]; - int64_t outputWidth = finalOutput ? out[3] : out[2]; + int64_t outputHeight = nchwOutput ? out[2] : out[1]; + int64_t outputWidth = nchwOutput ? out[3] : out[2]; if (kernelSize <= 0 || kernelSize > 7 || weights != ArrayRef({kernelSize, kernelSize, in[3], 1}) || bias.getShape() != ArrayRef({outputChannels}) || @@ -231,8 +230,8 @@ class ResidentConvRegionPattern : public OpRewritePattern { (in[2] + 2 * conv.getPadLow() - kernelSize) / conv.getStride() + 1 != outputWidth || - (finalOutput ? (out[1] != outputChannels) - : (out[3] != outputChannels))) + (nchwOutput ? (out[1] != outputChannels) + : (out[3] != outputChannels))) return conv.emitError("Depthwise Conv shape is inconsistent"); stage.input = conv.getInput(); stage.output = conv.getOutput(); @@ -253,6 +252,7 @@ class ResidentConvRegionPattern : public OpRewritePattern { stage.outputScale = conv.getOutputScale().convertToFloat(); stage.depthwise = true; stage.finalOutput = finalOutput; + stage.outputNhwc = outputNhwc; stage.lutEntries = lut.getShape()[0]; if (auto scales = operation.getAttrOfType( "lane_output_scales")) @@ -521,33 +521,19 @@ class ResidentConvRegionPattern : public OpRewritePattern { return kernel.emitError( "resident Conv region requires 128-bit rows, bankDepth divisible " "by 4, and a positive bank count"); - if (buckyball_target::getBuckyballBallMapping("SMatMulBall").outBW != 1) + if (buckyball_target::getBuckyballBallMapping("SMATMUL_OS").outBW != 1) return kernel.emitError("resident Conv region requires SMatMul outBW=1"); const int64_t smatmulRows = - buckyball_target::getBuckyballBallParam("SMatMulBall", "tileRows"); + buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); const int64_t smatmulCols = - buckyball_target::getBuckyballBallParam("SMatMulBall", "tileCols"); + buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileCols"); if (smatmulRows <= 0 || kTile % smatmulRows != 0 || smatmulCols != kTile) return kernel.emitError( "resident Conv region requires SMatMul tileRows to divide 16 and " "tileCols to equal 16"); Location loc = kernel.getLoc(); - const int64_t traceRegionId = - traceMegaStages ? nextTraceRegionId++ : int64_t{0}; - const bool traceThisRegion = - traceMegaStages && - (traceMegaRegion < 0 || traceRegionId == traceMegaRegion); b.setInsertionPoint(kernel); - if (traceThisRegion && traceMegaInputBeforeRegion) { - auto id = b.getI64IntegerAttr(4095); - auto trace = b.create<::buddy::trace::EndOp>( - loc, kernel.getInput().getType(), kernel.getInput(), id, - b.getStringAttr("mega-input")); - trace->setAttr("id_path", - b.getArrayAttr({b.getI64IntegerAttr(traceRegionId), id})); - trace->setAttr("buckyball.stage_trace", b.getUnitAttr()); - } Value zeroI8 = b.create(loc, b.getI8Type(), b.getI8IntegerAttr(0)); Value minI8 = b.create(loc, b.getI8Type(), @@ -766,6 +752,7 @@ class ResidentConvRegionPattern : public OpRewritePattern { // stages must consume the checked host tensor instead of recursively // recomputing it, otherwise a passing hash does not isolate a boundary. DenseSet traceMaterialized; + DenseSet tracingStages; struct TileBanks { SmallVector banks; @@ -1092,11 +1079,29 @@ class ResidentConvRegionPattern : public OpRewritePattern { Operation *insertionAnchor = b.create(loc, 0).getOperation(); b.setInsertionPoint(insertionAnchor); + Stage &stage = stages[stageIndex]; + auto trace = stage.op->getAttrOfType("trace_node"); + bool cycle = trace && cast(trace.get("cycle")).getValue() && + !tracingStages.contains(stageIndex); + if (cycle) { + tracingStages.insert(stageIndex); + auto start = b.create<::buddy::trace::StartOp>( + loc, cast(trace.get("id")), b.getBoolAttr(true), + b.getBoolAttr(false), cast(trace.get("tag"))); + start->setAttr("id_path", trace.get("id_path")); + } llvm::scope_exit eraseInsertionAnchor([&]() { b.setInsertionPointAfter(insertionAnchor); + if (cycle) { + auto end = b.create<::buddy::trace::EndOp>( + loc, zero.getType(), zero, cast(trace.get("id")), + b.getBoolAttr(true), b.getBoolAttr(false), + cast(trace.get("tag"))); + end->setAttr("id_path", trace.get("id_path")); + tracingStages.erase(stageIndex); + } b.eraseOp(insertionAnchor); }); - Stage &stage = stages[stageIndex]; const bool fp32Output = stage.finalOutput && !stage.pool; const bool externalInput = !producer.contains(stage.input); int64_t totalPanels = (stage.outputChannels + kTile - 1) / kTile; @@ -1107,7 +1112,7 @@ class ResidentConvRegionPattern : public OpRewritePattern { destination.panelRows) return stage.op->emitError("invalid resident tile request"); - if (traceMegaReloadStages && traceMaterialized.contains(stageIndex)) { + if (traceMaterialized.contains(stageIndex)) { FailureOr loaded = loadInt8Tile(stage.output, stage.outputHeight, stage.outputWidth, stage.outputChannels, y0, x0, firstPanel, panelCount, @@ -3083,7 +3088,7 @@ class ResidentConvRegionPattern : public OpRewritePattern { loc, lane, b.create(loc, 4))}); value = b.create(loc, valueF32, outputScale); } - if (stage.finalOutput) + if (stage.finalOutput && !stage.outputNhwc) b.create( loc, value, stage.output, ValueRange{zero, channel, globalY, globalX}); @@ -3125,21 +3130,21 @@ class ResidentConvRegionPattern : public OpRewritePattern { } b.setInsertionPointAfter(xLoop); b.setInsertionPointAfter(yLoop); - // A zero limit is the debug mode for the normal resident path: do not - // force any stage to materialize, but trace every stage that the normal - // scheduler materializes at a pool/add/final boundary. - if (traceThisRegion && stageIndex >= traceMegaStageStart && - (traceMegaStageLimit == 0 || traceMegaStageLimit < 0 || - stageIndex < traceMegaStageLimit || - stageIndex == static_cast(stages.size()) - 1)) { - auto id = b.getI64IntegerAttr(stageIndex); - auto trace = b.create<::buddy::trace::EndOp>( - loc, stage.output.getType(), stage.output, id, - b.getStringAttr("mega-stage")); - trace->setAttr( - "id_path", - b.getArrayAttr({b.getI64IntegerAttr(traceRegionId), id})); - trace->setAttr("buckyball.stage_trace", b.getUnitAttr()); + auto traceNode = stage.op->getAttrOfType("trace_node"); + if (traceNode && cast(traceNode.get("tensor")).getValue()) { + if (cast(stage.output.getType()) + .getElementType() + .isInteger(8)) { + if (failed(mlir::buddy::emitStageTensorTrace(b, loc, stage.output, + traceNode))) + return failure(); + } else { + auto trace = b.create<::buddy::trace::EndOp>( + loc, stage.output.getType(), stage.output, + cast(traceNode.get("id")), b.getBoolAttr(false), + b.getBoolAttr(true), cast(traceNode.get("tag"))); + trace->setAttr("id_path", traceNode.get("id_path")); + } } materialized.insert(stageIndex); return success(); @@ -3157,44 +3162,7 @@ class ResidentConvRegionPattern : public OpRewritePattern { } } - int64_t traceLimit = 0; - if (traceThisRegion) { - if (traceMegaStageStart < 0 || - traceMegaStageStart > static_cast(stages.size())) - return kernel.emitError("trace-mega-stage-start is out of range"); - traceLimit = traceMegaStageLimit < 0 - ? static_cast(stages.size()) - : std::min(traceMegaStageLimit, - static_cast(stages.size())); - if (traceLimit < 0) - return kernel.emitError("trace-mega-stage-limit must be non-negative"); - for (int64_t stageIndex = 0; stageIndex < traceLimit; ++stageIndex) { - Stage &stage = stages[stageIndex]; - if (stage.multiply) { - int64_t gateStage = producer.lookup(stage.rhs); - int64_t gatePanels = - (stages[gateStage].outputChannels + kTile - 1) / kTile; - TileBanks gate = allocateTile(gatePanels, 1, zeroI8); - if (gate.banks.size() != 1 || - failed(emitInto(gateStage, zero, zero, 1, 1, zero, gatePanels, - gate, 0, 1))) - return stage.op->emitError( - "INT8 Mul gate must fit one complete bank"); - gateCaches.try_emplace(gateStage, std::move(gate)); - } - if (failed(materializeStage(stageIndex))) - return failure(); - traceMaterialized.insert(stageIndex); - if (stage.multiply) { - for (auto &entry : gateCaches) - releaseTile(entry.second); - gateCaches.clear(); - } - } - } for (auto [stageIndex, stage] : llvm::enumerate(stages)) { - if (static_cast(stageIndex) < traceLimit) - continue; if (stage.multiply) { if (producer.contains(stage.input)) { int64_t inputStage = producer.lookup(stage.input); @@ -3218,6 +3186,13 @@ class ResidentConvRegionPattern : public OpRewritePattern { gateCaches.try_emplace(gateStage, std::move(gate)); } } + auto traceNode = stage.op->getAttrOfType("trace_node"); + if (traceNode && cast(traceNode.get("tensor")).getValue() && + !materialized.contains(stageIndex)) { + if (failed(materializeStage(stageIndex))) + return failure(); + traceMaterialized.insert(stageIndex); + } // Spatial kernels reuse overlapping producer windows; resize repeats // pixels. Shared DAG results also need one evaluation before consumers // request tiles. @@ -3263,28 +3238,13 @@ class ResidentConvRegionPattern : public OpRewritePattern { b.eraseOp(kernel); return success(); } - -private: - bool traceMegaStages; - int64_t traceMegaStageStart; - int64_t traceMegaStageLimit; - int64_t traceMegaRegion; - bool traceMegaReloadStages; - bool traceMegaInputBeforeRegion; - mutable int64_t nextTraceRegionId = 0; }; } // namespace namespace mlir::buddy { void populatePebbleResidentConvRegionToBankSSAPatterns( - RewritePatternSet &patterns, bool traceMegaStages, - int64_t traceMegaStageStart, int64_t traceMegaStageLimit, - int64_t traceMegaRegion, bool traceMegaReloadStages, - bool traceMegaInputBeforeRegion) { - patterns.add( - patterns.getContext(), traceMegaStages, traceMegaStageStart, - traceMegaStageLimit, traceMegaRegion, traceMegaReloadStages, - traceMegaInputBeforeRegion); + RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); } } // namespace mlir::buddy diff --git a/waste/examples/designs/pebble/stack/midend/src/LargeConvolution.cpp b/waste/examples/designs/pebble/stack/midend/src/LargeConvolution.cpp new file mode 100644 index 000000000..b94b810ee --- /dev/null +++ b/waste/examples/designs/pebble/stack/midend/src/LargeConvolution.cpp @@ -0,0 +1,158 @@ +#include "Buckyball/BuckyballOps.h" +#include "Trace/TraceOps.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/Math/IR/Math.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +LogicalResult lowerPebbleLargeConvolutions(func::FuncOp function) { + SmallVector convolutions; + function.walk([&](linalg::GenericOp op) { + if (op->hasAttr("buckyball.quantized_conv2d") && + op->getAttrOfType("kernel").getInt() > 7) + convolutions.push_back(op); + }); + for (auto op : convolutions) { + OpBuilder b(op); + Location loc = op.getLoc(); + Value input = op.getInputs()[0], weight = op.getInputs()[1]; + Value output = op.getOutputs()[0]; + auto in = cast(input.getType()).getShape(); + auto wt = cast(weight.getType()).getShape(); + auto outType = cast(output.getType()); + auto out = outType.getShape(); + int64_t kernel = op->getAttrOfType("kernel").getInt(); + int64_t stride = op->getAttrOfType("stride").getInt(); + int64_t padding = op->getAttrOfType("pad_low").getInt(); + bool nchw = outType.getElementType().isF32() && + !op->getAttrOfType("output_nhwc").getValue(); + int64_t height = out[nchw ? 2 : 1], width = out[nchw ? 3 : 2]; + int64_t rows = height * width, columns = wt[0]; + int64_t reduction = in[3] * kernel * kernel; + int64_t packedReduction = ((reduction + 15) / 16) * 16; + if (in[0] != 1 || wt[1] != in[3] || + op->getAttrOfType("activation").getInt() > 1) + return op.emitError("large Pebble convolution requires batch one and " + "linear/ReLU activation"); + auto idx = [&](int64_t n) -> Value { + return arith::ConstantIndexOp::create(b, loc, n); + }; + Value zero = idx(0), one = idx(1); + Value zeroByte = arith::ConstantIntOp::create(b, loc, 0, 8); + Value patches = memref::AllocOp::create( + b, loc, MemRefType::get({rows, packedReduction}, b.getI8Type())); + Value weights = memref::AllocOp::create( + b, loc, MemRefType::get({packedReduction, columns}, b.getI8Type())); + Value result = memref::AllocOp::create( + b, loc, MemRefType::get({rows, columns}, b.getF32Type())); + linalg::FillOp::create(b, loc, zeroByte, patches); + linalg::FillOp::create(b, loc, zeroByte, weights); + auto trace = op->getAttrOfType("trace_node"); + if (trace && cast(trace.get("cycle")).getValue()) { + auto start = ::buddy::trace::StartOp::create( + b, loc, cast(trace.get("id")), b.getBoolAttr(true), + b.getBoolAttr(false), cast(trace.get("tag"))); + start->setAttr("id_path", trace.get("id_path")); + } + auto rowLoop = scf::ForOp::create(b, loc, zero, idx(rows), one); + b.setInsertionPointToStart(rowLoop.getBody()); + auto kLoop = scf::ForOp::create(b, loc, zero, idx(reduction), one); + b.setInsertionPointToStart(kLoop.getBody()); + Value row = rowLoop.getInductionVar(), k = kLoop.getInductionVar(); + Value channel = arith::DivUIOp::create(b, loc, k, idx(kernel * kernel)); + Value ky = arith::RemUIOp::create( + b, loc, arith::DivUIOp::create(b, loc, k, idx(kernel)), idx(kernel)); + Value kx = arith::RemUIOp::create(b, loc, k, idx(kernel)); + Value y = arith::SubIOp::create( + b, loc, + arith::AddIOp::create( + b, loc, + arith::MulIOp::create( + b, loc, arith::DivUIOp::create(b, loc, row, idx(width)), + idx(stride)), + ky), + idx(padding)); + Value x = arith::SubIOp::create( + b, loc, + arith::AddIOp::create( + b, loc, + arith::MulIOp::create( + b, loc, arith::RemUIOp::create(b, loc, row, idx(width)), + idx(stride)), + kx), + idx(padding)); + Value valid = arith::AndIOp::create( + b, loc, + arith::CmpIOp::create(b, loc, arith::CmpIPredicate::ult, y, idx(in[1])), + arith::CmpIOp::create(b, loc, arith::CmpIPredicate::ult, x, + idx(in[2]))); + auto present = scf::IfOp::create(b, loc, valid, false); + b.setInsertionPointToStart(present.thenBlock()); + Value value = + memref::LoadOp::create(b, loc, input, ValueRange{zero, y, x, channel}); + memref::StoreOp::create(b, loc, value, patches, ValueRange{row, k}); + b.setInsertionPointAfter(rowLoop); + auto outputChannel = scf::ForOp::create(b, loc, zero, idx(columns), one); + b.setInsertionPointToStart(outputChannel.getBody()); + auto weightK = scf::ForOp::create(b, loc, zero, idx(reduction), one); + b.setInsertionPointToStart(weightK.getBody()); + k = weightK.getInductionVar(); + channel = arith::DivUIOp::create(b, loc, k, idx(kernel * kernel)); + ky = arith::RemUIOp::create( + b, loc, arith::DivUIOp::create(b, loc, k, idx(kernel)), idx(kernel)); + kx = arith::RemUIOp::create(b, loc, k, idx(kernel)); + Value co = outputChannel.getInductionVar(); + value = + memref::LoadOp::create(b, loc, weight, ValueRange{co, channel, ky, kx}); + memref::StoreOp::create(b, loc, value, weights, ValueRange{k, co}); + b.setInsertionPointAfter(outputChannel); + auto region = MegaKernelOp::create(b, loc, patches, result); + region.getBody().emplaceBlock(); + b.setInsertionPointToStart(®ion.getBody().front()); + MegaMatmulOp::create(b, loc, patches, weights, op.getInputs()[2], + op.getInputs()[3], op.getInputs()[4], result, + op->getAttrOfType("activation"), + op->getAttrOfType("output_scale")); + MegaYieldOp::create(b, loc); + b.setInsertionPointAfter(region); + auto copyRow = scf::ForOp::create(b, loc, zero, idx(rows), one); + b.setInsertionPointToStart(copyRow.getBody()); + auto copyChannel = scf::ForOp::create(b, loc, zero, idx(columns), one); + b.setInsertionPointToStart(copyChannel.getBody()); + row = copyRow.getInductionVar(); + co = copyChannel.getInductionVar(); + value = memref::LoadOp::create(b, loc, result, ValueRange{row, co}); + if (outType.getElementType().isInteger(8)) { + value = math::RoundEvenOp::create(b, loc, value); + Value low = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(-128.0)); + Value high = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(127.0)); + value = arith::MaximumFOp::create(b, loc, value, low); + value = arith::MinimumFOp::create(b, loc, value, high); + value = arith::FPToSIOp::create(b, loc, b.getI8Type(), value); + } + y = arith::DivUIOp::create(b, loc, row, idx(width)); + x = arith::RemUIOp::create(b, loc, row, idx(width)); + memref::StoreOp::create(b, loc, value, output, + nchw ? ValueRange{zero, co, y, x} + : ValueRange{zero, y, x, co}); + b.setInsertionPointAfter(copyRow); + if (trace) { + auto end = ::buddy::trace::EndOp::create( + b, loc, output.getType(), output, cast(trace.get("id")), + cast(trace.get("cycle")), + cast(trace.get("tensor")), + cast(trace.get("tag"))); + end->setAttr("id_path", trace.get("id_path")); + } + memref::DeallocOp::create(b, loc, patches); + memref::DeallocOp::create(b, loc, weights); + memref::DeallocOp::create(b, loc, result); + op.erase(); + } + return success(); +} diff --git a/waste/examples/designs/pebble/stack/midend/src/Passes.cpp b/waste/examples/designs/pebble/stack/midend/src/Passes.cpp new file mode 100644 index 000000000..2066ffdbc --- /dev/null +++ b/waste/examples/designs/pebble/stack/midend/src/Passes.cpp @@ -0,0 +1,140 @@ +//===- LowerPebbleToBankSSAPass.cpp - Pebble bank-SSA lowering ----------===// +// +// Pebble lowers its MegaKernels as resident regions and keeps transposes on +// banks, then lowers the remaining Ball operations through BankSSALowering. +// +//===----------------------------------------------------------------------===// + +#include "Conversion/LowerBuckyball/LowerBuckyball.h" +#include "Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/Math/IR/Math.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Pass/Pass.h" +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" + +#include "Buckyball/BuckyballDialect.h" +#include "Buckyball/BuckyballOps.h" +#include "Trace/TraceDialect.h" + +using namespace mlir; + +namespace mlir::buddy { +void populateMatmulRegionToBankSSAPatterns(RewritePatternSet &patterns); +void populatePebbleMegaConv2dToBankSSAPatterns(RewritePatternSet &patterns); +void populatePebbleMemTransposeToBankSSAPatterns(RewritePatternSet &patterns); +void populatePebbleResidentConvRegionToBankSSAPatterns( + RewritePatternSet &patterns); +} // namespace mlir::buddy + +LogicalResult lowerPebbleLargeConvolutions(func::FuncOp function); + +namespace { + +class FormPebbleMegaKernelsPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(FormPebbleMegaKernelsPass) + StringRef getArgument() const final { return "form-pebble-mega-kernels"; } + StringRef getDescription() const final { + return "Fuse quantized operators within one Pebble core"; + } + void getDependentDialects(DialectRegistry ®istry) const override { + registry + .insert(); + } + void runOnOperation() override { + if (failed(lowerPebbleLargeConvolutions(getOperation()))) { + signalPassFailure(); + return; + } + DenseMap> blocks; + getOperation().walk([&](linalg::GenericOp op) { + for (NamedAttribute attr : op->getAttrs()) + if (attr.getName().strref().starts_with("buckyball.quantized_")) { + blocks[op->getBlock()].push_back(op); + break; + } + }); + for (auto &entry : blocks) { + mlir::buddy::prepareQuantizedWeights(entry.second); + mlir::buddy::formMegaKernelRegions(entry.second); + } + } +}; + +class LowerPebbleToBankSSAPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerPebbleToBankSSAPass) + LowerPebbleToBankSSAPass() = default; + LowerPebbleToBankSSAPass(const LowerPebbleToBankSSAPass &) {} + + StringRef getArgument() const final { return "lower-pebble-to-bank-ssa"; } + StringRef getDescription() const final { + return "Lower Pebble Buckyball ops to explicit bank-SSA ops."; + } + + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert(); + } + + void runOnOperation() override { + RewritePatternSet residentPatterns(&getContext()); + mlir::buddy::populatePebbleResidentConvRegionToBankSSAPatterns( + residentPatterns); + if (failed(applyPatternsGreedily(getOperation(), + std::move(residentPatterns)))) { + signalPassFailure(); + return; + } + RewritePatternSet patterns(&getContext()); + mlir::buddy::populateMatmulRegionToBankSSAPatterns(patterns); + mlir::buddy::populatePebbleMegaConv2dToBankSSAPatterns(patterns); + mlir::buddy::populatePebbleMemTransposeToBankSSAPatterns(patterns); + mlir::buddy::populateBankSSALoweringPatterns(patterns); + if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) { + signalPassFailure(); + return; + } + bool illegalMega = false; + getOperation().walk([&](Operation *op) { + if (isa<::buddy::buckyball::MegaKernelOp>(op)) { + op->emitError( + "MegaKernel region-wide bank SSA lowering is not implemented"); + illegalMega = true; + } else if (isa<::buddy::buckyball::MegaMatmulOp, + ::buddy::buckyball::MegaConv2dOp, + ::buddy::buckyball::MegaConv2dDepthwiseOp>(op) && + !op->getParentOfType<::buddy::buckyball::MegaKernelOp>()) { + op->emitError( + "MegaKernel stage is only legal inside buckyball.mega_kernel"); + illegalMega = true; + } + }); + if (illegalMega) + signalPassFailure(); + } +}; + +} // namespace + +// Registered when the compiler plugin is loaded. +[[maybe_unused]] static const bool registered = [] { + PassRegistration(); + PassRegistration(); + return true; +}(); diff --git a/arch/src/main/scala/framework/root/root-chip/src/main/resources/README.md b/waste/stack/compiler/frontend/__init__.py similarity index 100% rename from arch/src/main/scala/framework/root/root-chip/src/main/resources/README.md rename to waste/stack/compiler/frontend/__init__.py diff --git a/arch/src/main/scala/framework/root/root-core/src/csrc/README.md b/waste/stack/compiler/frontend/layout/__init__.py similarity index 100% rename from arch/src/main/scala/framework/root/root-core/src/csrc/README.md rename to waste/stack/compiler/frontend/layout/__init__.py diff --git a/waste/stack/compiler/frontend/layout/provenance.py b/waste/stack/compiler/frontend/layout/provenance.py new file mode 100644 index 000000000..3d6d0f232 --- /dev/null +++ b/waste/stack/compiler/frontend/layout/provenance.py @@ -0,0 +1,62 @@ +from buddy_mlir import ir + + +def has_nhwc_layout(value): + """Follow operations that preserve TOSA convolution/pooling axis order.""" + if not isinstance(value, ir.OpResult): + return False + producer = value.owner + if isinstance(producer, ir.OpView): + producer = producer.operation + if "buddy.layout" in producer.attributes: + return ir.StringAttr(producer.attributes["buddy.layout"]).value == "nhwc" + if producer.name in { + "tosa.conv2d", + "tosa.depthwise_conv2d", + "tosa.transpose_conv2d", + "tosa.avg_pool2d", + "tosa.max_pool2d", + }: + return True + if producer.name == "linalg.generic": + maps = ir.ArrayAttr(producer.attributes["indexing_maps"]) + identity = ir.AffineMap.get_identity(len(ir.RankedTensorType(value.type).shape)) + if ir.AffineMapAttr(maps[len(maps) - 1]).value != identity: + return False + return any( + ir.AffineMapAttr(mapping).value == identity + and isinstance(operand.type, ir.RankedTensorType) + and len(ir.RankedTensorType(operand.type).shape) == 4 + and has_nhwc_layout(operand) + for operand, mapping in zip(producer.operands, maps, strict=True) + ) + if producer.name in { + "tosa.add", + "tosa.sub", + "tosa.mul", + "tosa.maximum", + "tosa.minimum", + "tosa.clamp", + "tosa.sigmoid", + "tosa.tanh", + "tosa.exp", + "tosa.log", + "tosa.negate", + "tosa.abs", + "tosa.reciprocal", + "tosa.rsqrt", + "tosa.cast", + "tosa.select", + "tosa.slice", + "tosa.pad", + "tensor.cast", + "math.roundeven", + "buddy_trace.end", + }: + return any( + isinstance(operand.type, ir.RankedTensorType) + and len(ir.RankedTensorType(operand.type).shape) == 4 + and has_nhwc_layout(operand) + for operand in producer.operands + ) + return False diff --git a/waste/stack/compiler/frontend/operation.py b/waste/stack/compiler/frontend/operation.py new file mode 100644 index 000000000..153dcff5a --- /dev/null +++ b/waste/stack/compiler/frontend/operation.py @@ -0,0 +1,107 @@ +# ===- operation.py ------------------------------------------------------------ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ===--------------------------------------------------------------------------- +# +# This is the operation structure of Buddy Compiler graph representation. +# +# ===--------------------------------------------------------------------------- + +from buddy.compiler.graph.operation import Conv2dOp, Op, OpType + + +class QuantizedOp(Op): + def __init__(self): + super().__init__() + self.quantization_trace_meta = None + + +class QuantizedMatmulOp(QuantizedOp): + def __init__(self) -> None: + super().__init__() + self._op_type = OpType.ReduceType + + +class QuantizedConv2dOp(QuantizedOp): + def __init__(self) -> None: + super().__init__() + self._op_type = OpType.ReduceType + self._output_nhwc = False + + +class QuantizedDepthwiseConv2dOp(QuantizedOp): + def __init__(self) -> None: + super().__init__() + self._op_type = OpType.ReduceType + self._output_nhwc = False + + +class QuantizedMaxPool2dOp(QuantizedOp): + def __init__(self) -> None: + super().__init__() + self._op_type = OpType.ReduceType + + +class QuantizedGlobalAvgPoolOp(QuantizedOp): + def __init__(self) -> None: + super().__init__() + self._op_type = OpType.ReduceType + + +class QuantizedChannelSliceOp(QuantizedOp): + def __init__(self) -> None: + super().__init__() + self._op_type = OpType.ReshapeType + + +class QuantizedChannelConcatOp(QuantizedOp): + def __init__(self) -> None: + super().__init__() + self._op_type = OpType.ConcatType + + +class QuantizedResizeNearestOp(QuantizedOp): + def __init__(self) -> None: + super().__init__() + self._op_type = OpType.ReshapeType + + +class QuantizedMulOp(QuantizedOp): + def __init__(self) -> None: + super().__init__() + self._op_type = OpType.ElementwiseType + + +class QuantizedAddOp(QuantizedOp): + def __init__(self) -> None: + super().__init__() + self._op_type = OpType.ElementwiseType + + +class Int8QuantizeOp(Op): + def __init__(self): + super().__init__() + self._op_type = OpType.ElementwiseType + + +class Int8Conv2dOp(Conv2dOp): + """Signed INT8 convolution with INT32 accumulation and FP32 rescaling.""" + + +class Int8MatmulOp(Op): + """Signed INT8 matrix product, scales and optional FP32 bias.""" + + def __init__(self) -> None: + super().__init__() + self._op_type = OpType.ReduceType diff --git a/arch/src/main/scala/framework/root/root-core/src/main/resources/README.md b/waste/stack/compiler/frontend/ops/__init__.py similarity index 100% rename from arch/src/main/scala/framework/root/root-core/src/main/resources/README.md rename to waste/stack/compiler/frontend/ops/__init__.py diff --git a/waste/stack/compiler/frontend/ops/expf.py b/waste/stack/compiler/frontend/ops/expf.py new file mode 100644 index 000000000..c335162fa --- /dev/null +++ b/waste/stack/compiler/frontend/ops/expf.py @@ -0,0 +1,83 @@ +# FP32 range reduction and coefficients follow SLEEF's expf. +# Copyright Naoki Shibata and contributors 2010 - 2025. +# Distributed under the Boost Software License, Version 1.0. +# https://github.com/shibatch/sleef/blob/3.9.0/src/libm/sleefsimdsp.c + +# Boost Software License - Version 1.0 - August 17th, 2003 +# +# Permission is hereby granted, free of charge, to any person or organization +# obtaining a copy of the software and accompanying documentation covered by +# this license (the "Software") to use, reproduce, display, distribute, +# execute, and transmit the Software, and to prepare derivative works of the +# Software, and to permit third-parties to whom the Software is furnished to +# do so, all subject to the following: +# +# The copyright notices in the Software and this entire statement, including +# the above license grant, this restriction and the following disclaimer, +# must be included in all copies of the Software, in whole or in part, and +# all derivative works of the Software, unless such copies or derivative +# works are solely in the form of machine-executable object code generated by +# a source language processor. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +# SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +# FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +from buddy_mlir import ir +from buddy_mlir.dialects import arith, math + + +def exp_f32(value): + """Emit a fixed FP32 exp approximation before activation rounding.""" + f32 = ir.F32Type.get() + i32 = ir.IntegerType.get_signless(32) + + def constant(number): + return arith.ConstantOp(f32, float(number)).result + + zero = constant(0.0) + lower = constant(-104.0) + upper = constant(100.0) + is_nan = arith.CmpFOp(arith.CmpFPredicate.UNO, value, value).result + underflow = arith.CmpFOp(arith.CmpFPredicate.OLT, value, lower).result + overflow = arith.CmpFOp(arith.CmpFPredicate.OGT, value, upper).result + # Bound the exponent conversion, including NaN and infinity inputs. + bounded = arith.SelectOp(underflow, lower, value).result + bounded = arith.SelectOp(overflow, upper, bounded).result + bounded = arith.SelectOp(is_nan, zero, bounded).result + rounded = math.RoundEvenOp( + arith.MulFOp(bounded, constant(1.4426950408889634)).result + ).result + exponent = arith.FPToSIOp(i32, rounded).result + residual = math.FmaOp(rounded, constant(-0.693145751953125), bounded).result + residual = math.FmaOp(rounded, constant(-1.428606765330187e-6), residual).result + polynomial = constant(0.00019852761761285365) + for coefficient in ( + 0.0013930435525253415, + 0.008333360776305199, + 0.041666485369205475, + 0.1666666716337204, + 0.5, + ): + polynomial = math.FmaOp(polynomial, residual, constant(coefficient)).result + square = arith.MulFOp(residual, residual).result + polynomial = arith.AddFOp( + constant(1.0), math.FmaOp(square, polynomial, residual).result + ).result + + one = arith.ConstantOp(i32, 1).result + bias = arith.ConstantOp(i32, 127).result + shift = arith.ConstantOp(i32, 23).result + half = arith.ShRSIOp(exponent, one).result + remainder = arith.SubIOp(exponent, half).result + for power in (half, remainder): + bits = arith.ShLIOp(arith.AddIOp(power, bias).result, shift).result + factor = arith.BitcastOp(f32, bits).result + polynomial = arith.MulFOp(polynomial, factor).result + result = arith.SelectOp(underflow, zero, polynomial).result + result = arith.SelectOp(overflow, constant(float("inf")), result).result + return arith.SelectOp(is_nan, value, result).result diff --git a/waste/stack/compiler/frontend/ops/int8.py b/waste/stack/compiler/frontend/ops/int8.py new file mode 100644 index 000000000..bfbb53b4a --- /dev/null +++ b/waste/stack/compiler/frontend/ops/int8.py @@ -0,0 +1,250 @@ +"""Native signed INT8 contractions with INT32 accumulators.""" + +import math as python_math + +from buddy_mlir import ir +from buddy_mlir.dialects import arith, linalg, math, tensor, tosa + + +def _maps(values): + return ir.ArrayAttr.get([ir.AffineMapAttr.get(value) for value in values]) + + +def _iterators(parallel, reduction): + return ir.ArrayAttr.get( + [ir.Attribute.parse("#linalg.iterator_type")] * parallel + + [ir.Attribute.parse("#linalg.iterator_type")] * reduction + ) + + +def _zero(shape): + dtype = ir.IntegerType.get_signless(32) + empty = tensor.EmptyOp(shape, dtype) + zero = arith.ConstantOp(dtype, 0).result + fill = linalg.FillOp([empty.result.type], [zero], [empty.result]) + linalg.fill_builtin_region(fill.operation) + return fill.result + + +def _multiply_accumulate(op): + i8 = ir.IntegerType.get_signless(8) + i32 = ir.IntegerType.get_signless(32) + body = ir.Block.create_at_start(op.region, [i8, i8, i32]) + with ir.InsertionPoint(body): + lhs = arith.ExtSIOp(i32, body.arguments[0]).result + rhs = arith.ExtSIOp(i32, body.arguments[1]).result + product = arith.MulIOp(lhs, rhs).result + linalg.YieldOp([arith.AddIOp(body.arguments[2], product).result]) + + +def _rescale( + accumulator, + activation_scale, + weight_scale, + bias, + symbols, + *, + channel_axis=1, +): + from buddy.compiler.ops.tosa import _create_shape_operand, _gen_arith_binary_op + + shape = list(ir.RankedTensorType(accumulator.type).shape) + rank = len(shape) + f32 = ir.F32Type.get() + if isinstance(activation_scale, str): + activation_scale = symbols[(activation_scale, 0)] + else: + ty = ir.RankedTensorType.get([1] * rank, f32) + activation_scale = tosa.ConstOp( + ir.DenseElementsAttr.get_splat(ty, ir.FloatAttr.get(f32, activation_scale)) + ).result + activation_shape = list(ir.RankedTensorType(activation_scale.type).shape) + if channel_axis == 3 and len(activation_shape) == 4 and activation_shape[1] != 1: + activation_scale = tosa.ReshapeOp( + activation_scale, + _create_shape_operand([1, 1, 1, activation_shape[1]]), + ).result + scale = symbols[(weight_scale, 0)] + count = python_math.prod(ir.RankedTensorType(scale.type).shape) + broadcast_shape = [1] * rank + broadcast_shape[channel_axis] = count + scale = tosa.ReshapeOp(scale, _create_shape_operand(broadcast_shape)).result + combined_scale = _gen_arith_binary_op(activation_scale, scale, tosa.MulOp).result + restored = ( + accumulator + if ir.RankedTensorType(accumulator.type).element_type == f32 + else tosa.CastOp(ir.RankedTensorType.get(shape, f32), accumulator).result + ) + result = _gen_arith_binary_op(restored, combined_scale, tosa.MulOp).result + if bias is not None: + value = symbols[(bias, 0)] + bias_shape = [1] * rank + bias_shape[channel_axis] = shape[channel_axis] + value = tosa.ReshapeOp(value, _create_shape_operand(bias_shape)).result + result = _gen_arith_binary_op(result, value, tosa.AddOp).result + return result + + +def conv2d(node, symbols): + from stack.compiler.frontend.layout.provenance import has_nhwc_layout + + value, weight = [symbols[(name, 0)] for name in node.args[:2]] + input_nhwc = has_nhwc_layout(value) + output_nhwc = node.tensor_meta.get("activation_layout") == "nhwc" + ( + bias, + stride, + padding, + dilation, + transposed, + _, + groups, + activation_scale, + weight_scale, + ) = node.args[2:] + if transposed: + raise NotImplementedError("INT8 transpose convolution is not supported") + input_shape = list(ir.RankedTensorType(value.type).shape) + stored_shape = list(ir.RankedTensorType(weight.type).shape) + permutation = node.tensor_meta.get("weight_permutation", [0, 1, 2, 3]) + weight_shape = node.tensor_meta.get( + "weight_shape", [stored_shape[permutation.index(i)] for i in range(4)] + ) + shape = list(node.tensor_meta["shape"]) + if output_nhwc: + shape = [shape[0], shape[2], shape[3], shape[1]] + ph, pw = padding + if ph or pw: + low = [0, ph, pw, 0] if input_nhwc else [0, 0, ph, pw] + padded_shape = [size + 2 * pad for size, pad in zip(input_shape, low)] + op = tensor.PadOp( + ir.RankedTensorType.get(padded_shape, ir.IntegerType.get_signless(8)), + value, + [], + [], + low, + low, + ) + block = ir.Block.create_at_start(op.region, [ir.IndexType.get()] * 4) + with ir.InsertionPoint(block): + tensor.YieldOp(arith.ConstantOp(ir.IntegerType.get_signless(8), 0).result) + value = op.result + n, f, h, w, c, kh, kw = [ir.AffineExpr.get_dim(i) for i in range(7)] + channel = ( + ir.AffineExpr.get_floor_div(f, weight_shape[0] // groups) * weight_shape[1] + c + ) + if node.tensor_meta.get("weight_layout") == "blocked_conv": + lanes = node.tensor_meta["output_lanes"] + d0, d1, d2, d3 = [ir.AffineExpr.get_dim(i) for i in range(4)] + indices = [ + ir.AffineExpr.get_floor_div(d0, lanes), + d1, + d2 * weight_shape[3] + d3, + ir.AffineExpr.get_mod(d0, lanes), + ] + unpacked = tensor.EmptyOp(weight_shape, ir.IntegerType.get_signless(8)) + unpack = linalg.GenericOp( + [unpacked.result.type], + [weight], + [unpacked.result], + _maps([ir.AffineMap.get(4, 0, indices), ir.AffineMap.get_identity(4)]), + _iterators(4, 0), + ) + block = ir.Block.create_at_start( + unpack.region, [ir.IntegerType.get_signless(8)] * 2 + ) + with ir.InsertionPoint(block): + linalg.YieldOp([block.arguments[0]]) + weight = unpack.result + permutation = [0, 1, 2, 3] + weight_indices = [[f, c, kh, kw][i] for i in permutation] + height = h * stride[0] + kh * dilation[0] + width = w * stride[1] + kw * dilation[1] + maps = [ + ir.AffineMap.get( + 7, + 0, + ( + [n, height, width, channel] + if input_nhwc + else [n, channel, height, width] + ), + ), + ir.AffineMap.get(7, 0, weight_indices), + ir.AffineMap.get(7, 0, [n, h, w, f] if output_nhwc else [n, f, h, w]), + ] + output = _zero(shape) + op = linalg.GenericOp( + [output.type], [value, weight], [output], _maps(maps), _iterators(4, 3) + ) + _multiply_accumulate(op) + op.operation.attributes["buddy.int8_conv"] = ir.DictAttr.get( + { + "stride": ir.DenseI64ArrayAttr.get(stride), + "dilation": ir.DenseI64ArrayAttr.get(dilation), + "groups": ir.IntegerAttr.get(ir.IntegerType.get_signless(64), groups), + "weight_permutation": ir.DenseI64ArrayAttr.get(permutation), + } + ) + result = _rescale( + op.result, + activation_scale, + weight_scale, + bias, + symbols, + channel_axis=3 if output_nhwc else 1, + ) + if output_nhwc: + result.owner.attributes["buddy.layout"] = ir.StringAttr.get("nhwc") + return result + + +def matmul(node, symbols): + activation, weight, activation_scale, weight_scale, bias = node.args + lhs, rhs = symbols[(activation, 0)], symbols[(weight, 0)] + shape = list(node.tensor_meta["shape"]) + m, n, k = [ir.AffineExpr.get_dim(i) for i in range(3)] + maps = [ + ir.AffineMap.get(3, 0, [m, k]), + ir.AffineMap.get( + 3, 0, [n, k] if node.tensor_meta.get("transpose_b") else [k, n] + ), + ir.AffineMap.get(3, 0, [m, n]), + ] + output = _zero(shape) + op = linalg.GenericOp( + [output.type], [lhs, rhs], [output], _maps(maps), _iterators(2, 1) + ) + _multiply_accumulate(op) + op.operation.attributes["buddy.int8_matmul"] = ir.UnitAttr.get() + return _rescale(op.result, activation_scale, weight_scale, bias, symbols) + + +def quantize_activation(node, symbols): + value = symbols[(node.args[0], 0)] + shape = list(ir.RankedTensorType(value.type).shape) + f32, i8 = ir.F32Type.get(), ir.IntegerType.get_signless(8) + output = tensor.EmptyOp(shape, i8) + identity = ir.AffineMap.get_identity(len(shape)) + op = linalg.GenericOp( + [output.result.type], + [value], + [output.result], + _maps([identity, identity]), + _iterators(len(shape), 0), + ) + multiplier = 1.0 / node.args[1] + op.operation.attributes["buckyball.quant_f32_to_i8"] = ir.BoolAttr.get(True) + op.operation.attributes["nchw_to_nhwc"] = ir.BoolAttr.get(False) + op.operation.attributes["scale"] = ir.FloatAttr.get(f32, multiplier) + block = ir.Block.create_at_start(op.region, [f32, i8]) + with ir.InsertionPoint(block): + scale = arith.ConstantOp(f32, ir.FloatAttr.get(f32, multiplier)).result + value = math.RoundEvenOp(arith.MulFOp(block.arguments[0], scale).result).result + minimum = arith.ConstantOp(f32, ir.FloatAttr.get(f32, -128.0)).result + maximum = arith.ConstantOp(f32, ir.FloatAttr.get(f32, 127.0)).result + value = arith.MinimumFOp( + arith.MaximumFOp(value, minimum).result, maximum + ).result + linalg.YieldOp([arith.FPToSIOp(i8, value).result]) + return op.result diff --git a/waste/stack/compiler/frontend/ops/linalg.py b/waste/stack/compiler/frontend/ops/linalg.py new file mode 100644 index 000000000..e0339f10c --- /dev/null +++ b/waste/stack/compiler/frontend/ops/linalg.py @@ -0,0 +1,1599 @@ +# ===- linalg.py --------------------------------------------------------------- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ===--------------------------------------------------------------------------- +# +# The registry of mappings from Buddy Graph to MLIR linalg dialect operations. +# +# ===--------------------------------------------------------------------------- + +import array +import copy +import buddy_mlir.ir as ir +import numpy +from buddy_mlir.dialects import ( + arith, + bufferization, + linalg, + math, + memref, + scf, + tensor, + tosa, + vector, +) +from buddy_mlir.dialects import ( + complex as complex_dialect, +) +from buddy.compiler.graph import * +from buddy.compiler.graph.graph import TensorDType +from buddy.compiler.graph.transform.trace.lowering import _trace_attrs, has_trace +from buddy.compiler.ops.utils import * +from stack.compiler.frontend.operation import * +from buddy.compiler.ops.linalg import ( + _as_value, + _const_shape_operand, + _max_pool_indices_to_offsets_3d, + _normalize_pool_param, + max_pool3d_op, +) + + +def _perm_attr(perm: list[int]) -> ir.Attribute: + return ir.DenseI32ArrayAttr.get([int(x) for x in perm]) + + +def _nhwc_to_nchw(value: ir.Value) -> ir.Value: + shape = [int(x) for x in ir.RankedTensorType(value.type).shape] + if len(shape) != 4: + raise RuntimeError(f"_nhwc_to_nchw expects rank-4, got {shape}") + n, h, w, c = shape + elem = ir.RankedTensorType(value.type).element_type + out_ty = ir.RankedTensorType.get([n, c, h, w], elem) + return tosa.TransposeOp(out_ty, value, _perm_attr([0, 3, 1, 2])).result + + +def _nchw_to_nhwc(value: ir.Value) -> ir.Value: + shape = [int(x) for x in ir.RankedTensorType(value.type).shape] + if len(shape) != 4: + raise RuntimeError(f"_nchw_to_nhwc expects rank-4, got {shape}") + n, c, h, w = shape + elem = ir.RankedTensorType(value.type).element_type + out_ty = ir.RankedTensorType.get([n, h, w, c], elem) + return tosa.TransposeOp(out_ty, value, _perm_attr([0, 2, 3, 1])).result + + +def _split_via_tosa_slice( + input_tensor: ir.Value, + input_shape: list[int], + dim: int, + split_sizes: list[int], +) -> list[ir.Value]: + # tosa.slice / extract_slice on non-dim1 (esp. NHWC C) lowers wrong; + # always slice on dim1 via transpose. + rank = len(input_shape) + if dim != 1: + perm = list(range(rank)) + perm[1], perm[dim] = perm[dim], perm[1] + inv = [0] * rank + for i, j in enumerate(perm): + inv[j] = i + elem = ir.RankedTensorType(input_tensor.type).element_type + t_shape = [input_shape[i] for i in perm] + t = tosa.TransposeOp( + ir.RankedTensorType.get(t_shape, elem), + input_tensor, + _perm_attr(perm), + ).result + parts = _split_via_tosa_slice(t, t_shape, 1, split_sizes) + out = [] + for p in parts: + ps = [int(x) for x in ir.RankedTensorType(p.type).shape] + o_shape = [ps[i] for i in inv] + out.append( + tosa.TransposeOp( + ir.RankedTensorType.get(o_shape, elem), + p, + _perm_attr(inv), + ).result + ) + return out + + elem = ir.RankedTensorType(input_tensor.type).element_type + results = [] + offset = 0 + for size in split_sizes: + start = [0] * rank + start[1] = offset + out_shape = list(input_shape) + out_shape[1] = size + out_ty = ir.RankedTensorType.get(out_shape, elem) + results.append( + tosa.SliceOp( + out_ty, + input_tensor, + _const_shape_operand(start), + _const_shape_operand(out_shape), + ).result + ) + offset += size + return results + + +def cat_op( + node: CatOp, + symbol_table: dict[tuple[str, int], ir.Operation], +): + """ + Import the tensor concate operation. + From buddy CatOp to MLIR tensor `insert_slice` + operation. + + Note: This op, concate multiple input tensors. + Args: + node: Containing information from the input graph node. + symbol_table: A dictionary mapping symbols to their corresponding + operations. + + Returns: + op: The operation return the tensor.insert_slice op. + """ + # Support both 2 args and variable number of inputs + inputs = node.args[0] # List of input tensors + # dim is optional, default to 0 if not provided + dim = int(node.args[1]) if len(node.args) > 1 else 0 + + # Get all input tensors + input_tensors = [] + input_shapes = [] + + for inp in inputs: + t = symbol_table.get((str(inp), 0)) + if t is None: + raise ValueError(f"missing cat input for {node.name}: {inp}") + input_tensors.append(t) + input_shapes.append(list(ir.RankedTensorType(t.type).shape)) + + if len(input_tensors) == 0: + raise ValueError(f"cat has no inputs: {node.name}") + + rank = len(input_shapes[0]) + if dim < 0: + dim += rank + + # Sticky NHWC: PyTorch cat dim is NCHW; remap when SSA is NHWC. + out_meta = [int(x) for x in node.tensor_meta["shape"]] + if rank == 4 and dim == 1 and len(out_meta) == 4: + n, out_c, h, w = out_meta + nchw_inputs = [] + channels = 0 + for value, shape in zip(input_tensors, input_shapes): + if shape[0] == n and shape[1] == h and shape[2] == w: + nchw_inputs.append(_nhwc_to_nchw(value)) + channels += shape[3] + elif shape[0] == n and shape[2] == h and shape[3] == w: + nchw_inputs.append(value) + channels += shape[1] + else: + raise RuntimeError( + f"channel cat input layout mismatch: shape={shape} meta={out_meta}" + ) + if channels != out_c: + raise RuntimeError( + f"channel cat count mismatch: inputs={input_shapes} meta={out_meta}" + ) + return tosa.ConcatOp(nchw_inputs, 1).result + + if ( + rank == 4 + and len(out_meta) == 4 + and input_shapes[0][1] == out_meta[2] + and input_shapes[0][2] == out_meta[3] + and sum(s[3] for s in input_shapes) == out_meta[1] + ): + dim = {0: 0, 1: 3, 2: 1, 3: 2}[dim] + + for s in input_shapes[1:]: + if len(s) != rank: + raise RuntimeError(f"cat rank mismatch: {input_shapes[0]} vs {s}") + for d in range(rank): + if d != dim and s[d] != input_shapes[0][d]: + raise RuntimeError( + f"cat non-concat dim{d} mismatch: shapes={input_shapes} " + f"dim={dim} meta={out_meta}" + ) + + output_shape = list(input_shapes[0]) + output_shape[dim] = sum(s[dim] for s in input_shapes) + + dtype = node.tensor_meta["dtype"] + mlir_dtype = mlir_element_type_get(dtype) + + # insert_slice on NHWC C lowers wrong; concat on NCHW axis=1 instead. + nhwc_c = ( + rank == 4 + and dim == 3 + and input_shapes[0][1] == input_shapes[0][2] + and input_shapes[0][3] < output_shape[3] + ) + if nhwc_c: + nchw_inputs = [_nhwc_to_nchw(t) for t in input_tensors] + cat_nchw = tosa.ConcatOp(nchw_inputs, 1).result + return cat_nchw + + output = tensor.EmptyOp(output_shape, mlir_dtype) + + offset = [0] * rank + stride_attr = ir._denseI64ArrayAttr([1] * rank, None) + + current_result = output.result + + for input_tensor, input_shape in zip(input_tensors, input_shapes): + offset_attr = ir._denseI64ArrayAttr(offset, None) + size_attr = ir._denseI64ArrayAttr(input_shape, None) + insert_op = tensor.InsertSliceOp( + input_tensor, + current_result, + [], + [], + [], + offset_attr, + size_attr, + stride_attr, + ) + current_result = insert_op.result + offset[dim] += input_shape[dim] + + return insert_op + + +def split_op(node: SplitOp, symbol_table): + """ + Split the input tensor into smaller tensors along the specified dimension. + + Args: + node (SplitOp): The split operation node with metadata. + symbol_table: Mapping of variable names to tensor references. + + Returns: + List[Tensor]: List of split tensors. + """ + # Get the input tensor and parameters + input_tensor = symbol_table.get((str(node.args[0]), 0), node.args[0]) + if not hasattr(input_tensor, "type"): + raise ValueError(f"missing split input for {node.name}: {node.args[0]}") + split_size = node.args[1] # Size of each split tensor + input_shape = [int(x) for x in input_tensor.type.shape] + dim = int(node.args[2]) # Dimension to split along (PyTorch NCHW) + if dim < 0: + dim += len(input_shape) + + # Sticky NHWC: remap channel/spatial dims from NCHW numbering. + meta_shapes = node.tensor_meta["shape"] + if ( + isinstance(meta_shapes, (list, tuple)) + and len(meta_shapes) > 0 + and hasattr(meta_shapes[0], "__len__") + and len(input_shape) == 4 + and len(meta_shapes[0]) == 4 + ): + m0 = [int(x) for x in meta_shapes[0]] + # Full NCHW concat along dim would match input meta; NHWC has C last. + if ( + input_shape[3] == sum(int(s[1]) for s in meta_shapes) + and input_shape[1] == m0[2] + ): + dim = {0: 0, 1: 3, 2: 1, 3: 2}[dim] + + split_count = (input_shape[dim] + split_size - 1) // split_size # Round up + if ( + isinstance(meta_shapes, (list, tuple)) + and len(meta_shapes) > 0 + and hasattr(meta_shapes[0], "__len__") + and split_count != len(meta_shapes) + ): + raise RuntimeError( + f"split count mismatch: ssa_dim={dim} count={split_count} " + f"meta_outs={len(meta_shapes)} shape={input_shape}" + ) + + split_sizes = [ + min(split_size, input_shape[dim] - i * split_size) for i in range(split_count) + ] + return _split_via_tosa_slice(input_tensor, input_shape, dim, split_sizes) + + +def _infer_pool2d_nhwc(shape: list[int]) -> bool: + if len(shape) != 4: + raise RuntimeError(f"expected rank-4 pool tensor, got {shape}") + _, a, b, c = shape + if a == b and c > a: + return True + if b == c and a > b: + return False + raise RuntimeError(f"cannot infer pool layout from shape {shape}") + + +def _pool2d_input_hw(input_shape: list[int]) -> list[int]: + if len(input_shape) != 4: + raise RuntimeError(f"expected rank-4 pool input, got {input_shape}") + _, a, b, c = input_shape + if a == b and c > a: + return [a, b] + if b == c and a > b: + return [b, c] + raise RuntimeError(f"cannot infer pool input_hw from shape {input_shape}") + + +def _emit_max_pool_offset_store( + flat_idx, + oh, + ow, + out_idx, + w_const, + kw_const, + sh_const, + sw_const, + ph_const, + pw_const, + dh_const, + dw_const, + i64, + i8, + offsets_memref, +): + ih = arith.DivSIOp(flat_idx, w_const).result + iw = arith.RemSIOp(flat_idx, w_const).result + + oh_i64 = arith.IndexCastOp(i64, oh).result + ow_i64 = arith.IndexCastOp(i64, ow).result + + hbase = arith.SubIOp(arith.MulIOp(oh_i64, sh_const).result, ph_const).result + wbase = arith.SubIOp(arith.MulIOp(ow_i64, sw_const).result, pw_const).result + + h_inc = arith.DivSIOp(arith.SubIOp(ih, hbase).result, dh_const).result + w_inc = arith.DivSIOp(arith.SubIOp(iw, wbase).result, dw_const).result + offset_i64 = arith.AddIOp(arith.MulIOp(h_inc, kw_const).result, w_inc).result + offset_i8 = arith.TruncIOp(i8, offset_i64).result + memref.StoreOp(offset_i8, offsets_memref.result, out_idx) + + +def _max_pool_indices_to_offsets_2d( + indices_tensor: ir.Value, + input_hw: list[int], + kernel_size: list[int], + stride: list[int], + padding: list[int], + dilation: list[int], +): + indices_tensor = _as_value(indices_tensor) + shape = [int(x) for x in ir.RankedTensorType(indices_tensor.type).shape] + nhwc = _infer_pool2d_nhwc(shape) + i64 = ir.IntegerType.get_signless(64) + i8 = ir.IntegerType.get_signless(8) + index_ty = ir.IndexType.get() + + offsets_memref_type = ir.MemRefType.get(shape, i8) + offsets_memref = memref.AllocOp(offsets_memref_type, [], []) + + lb = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, 0)).result + step = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, 1)).result + + n_ub = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, shape[0])).result + if nhwc: + oh_ub = arith.ConstantOp( + index_ty, ir.IntegerAttr.get(index_ty, shape[1]) + ).result + ow_ub = arith.ConstantOp( + index_ty, ir.IntegerAttr.get(index_ty, shape[2]) + ).result + c_ub = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, shape[3])).result + else: + c_ub = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, shape[1])).result + oh_ub = arith.ConstantOp( + index_ty, ir.IntegerAttr.get(index_ty, shape[2]) + ).result + ow_ub = arith.ConstantOp( + index_ty, ir.IntegerAttr.get(index_ty, shape[3]) + ).result + + w_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(input_hw[1]))).result + kw_const = arith.ConstantOp( + i64, ir.IntegerAttr.get(i64, int(kernel_size[1])) + ).result + sh_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(stride[0]))).result + sw_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(stride[1]))).result + ph_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(padding[0]))).result + pw_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(padding[1]))).result + dh_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(dilation[0]))).result + dw_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(dilation[1]))).result + + n_for = scf.ForOp(lb, n_ub, step) + with ir.InsertionPoint(n_for.body): + n = n_for.induction_variable + if nhwc: + oh_for = scf.ForOp(lb, oh_ub, step) + with ir.InsertionPoint(oh_for.body): + oh = oh_for.induction_variable + ow_for = scf.ForOp(lb, ow_ub, step) + with ir.InsertionPoint(ow_for.body): + ow = ow_for.induction_variable + c_for = scf.ForOp(lb, c_ub, step) + with ir.InsertionPoint(c_for.body): + c = c_for.induction_variable + flat_idx = tensor.ExtractOp( + indices_tensor, [n, oh, ow, c] + ).result + out_idx = [n, oh, ow, c] + _emit_max_pool_offset_store( + flat_idx, + oh, + ow, + out_idx, + w_const, + kw_const, + sh_const, + sw_const, + ph_const, + pw_const, + dh_const, + dw_const, + i64, + i8, + offsets_memref, + ) + scf.YieldOp([]) + scf.YieldOp([]) + scf.YieldOp([]) + scf.YieldOp([]) + else: + c_for = scf.ForOp(lb, c_ub, step) + with ir.InsertionPoint(c_for.body): + c = c_for.induction_variable + oh_for = scf.ForOp(lb, oh_ub, step) + with ir.InsertionPoint(oh_for.body): + oh = oh_for.induction_variable + ow_for = scf.ForOp(lb, ow_ub, step) + with ir.InsertionPoint(ow_for.body): + ow = ow_for.induction_variable + flat_idx = tensor.ExtractOp( + indices_tensor, [n, c, oh, ow] + ).result + out_idx = [n, c, oh, ow] + _emit_max_pool_offset_store( + flat_idx, + oh, + ow, + out_idx, + w_const, + kw_const, + sh_const, + sw_const, + ph_const, + pw_const, + dh_const, + dw_const, + i64, + i8, + offsets_memref, + ) + scf.YieldOp([]) + scf.YieldOp([]) + scf.YieldOp([]) + scf.YieldOp([]) + + offsets_type = ir.RankedTensorType.get(shape, i8) + return bufferization.ToTensorOp( + offsets_type, offsets_memref.result, restrict=True + ).result + + +def _max_pool_offsets_to_indices_2d( + offsets_tensor: ir.Value, + input_hw: list[int], + kernel_size: list[int], + stride: list[int], + padding: list[int], + dilation: list[int], +): + offsets_tensor = _as_value(offsets_tensor) + shape = [int(x) for x in ir.RankedTensorType(offsets_tensor.type).shape] + nhwc = _infer_pool2d_nhwc(shape) + i64 = ir.IntegerType.get_signless(64) + i8 = ir.IntegerType.get_signless(8) + index_ty = ir.IndexType.get() + + indices_memref_type = ir.MemRefType.get(shape, i64) + indices_memref = memref.AllocOp(indices_memref_type, [], []) + + lb = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, 0)).result + step = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, 1)).result + + n_ub = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, shape[0])).result + if nhwc: + oh_ub = arith.ConstantOp( + index_ty, ir.IntegerAttr.get(index_ty, shape[1]) + ).result + ow_ub = arith.ConstantOp( + index_ty, ir.IntegerAttr.get(index_ty, shape[2]) + ).result + c_ub = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, shape[3])).result + else: + c_ub = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, shape[1])).result + oh_ub = arith.ConstantOp( + index_ty, ir.IntegerAttr.get(index_ty, shape[2]) + ).result + ow_ub = arith.ConstantOp( + index_ty, ir.IntegerAttr.get(index_ty, shape[3]) + ).result + + in_w_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(input_hw[1]))).result + kw_const = arith.ConstantOp( + i64, ir.IntegerAttr.get(i64, int(kernel_size[1])) + ).result + sh_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(stride[0]))).result + sw_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(stride[1]))).result + ph_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(padding[0]))).result + pw_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(padding[1]))).result + dh_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(dilation[0]))).result + dw_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(dilation[1]))).result + + n_for = scf.ForOp(lb, n_ub, step) + with ir.InsertionPoint(n_for.body): + n = n_for.induction_variable + if nhwc: + oh_for = scf.ForOp(lb, oh_ub, step) + with ir.InsertionPoint(oh_for.body): + oh = oh_for.induction_variable + ow_for = scf.ForOp(lb, ow_ub, step) + with ir.InsertionPoint(ow_for.body): + ow = ow_for.induction_variable + c_for = scf.ForOp(lb, c_ub, step) + with ir.InsertionPoint(c_for.body): + c = c_for.induction_variable + off_i8 = tensor.ExtractOp(offsets_tensor, [n, oh, ow, c]).result + off_i64 = arith.ExtSIOp(i64, off_i8).result + + h_inc = arith.DivSIOp(off_i64, kw_const).result + w_inc = arith.RemSIOp(off_i64, kw_const).result + + oh_i64 = arith.IndexCastOp(i64, oh).result + ow_i64 = arith.IndexCastOp(i64, ow).result + + hbase = arith.SubIOp( + arith.MulIOp(oh_i64, sh_const).result, ph_const + ).result + wbase = arith.SubIOp( + arith.MulIOp(ow_i64, sw_const).result, pw_const + ).result + + ih = arith.AddIOp( + hbase, arith.MulIOp(h_inc, dh_const).result + ).result + iw = arith.AddIOp( + wbase, arith.MulIOp(w_inc, dw_const).result + ).result + + flat_idx = arith.AddIOp( + arith.MulIOp(ih, in_w_const).result, iw + ).result + memref.StoreOp(flat_idx, indices_memref.result, [n, oh, ow, c]) + scf.YieldOp([]) + scf.YieldOp([]) + scf.YieldOp([]) + scf.YieldOp([]) + else: + c_for = scf.ForOp(lb, c_ub, step) + with ir.InsertionPoint(c_for.body): + c = c_for.induction_variable + oh_for = scf.ForOp(lb, oh_ub, step) + with ir.InsertionPoint(oh_for.body): + oh = oh_for.induction_variable + ow_for = scf.ForOp(lb, ow_ub, step) + with ir.InsertionPoint(ow_for.body): + ow = ow_for.induction_variable + + off_i8 = tensor.ExtractOp(offsets_tensor, [n, c, oh, ow]).result + off_i64 = arith.ExtSIOp(i64, off_i8).result + + h_inc = arith.DivSIOp(off_i64, kw_const).result + w_inc = arith.RemSIOp(off_i64, kw_const).result + + oh_i64 = arith.IndexCastOp(i64, oh).result + ow_i64 = arith.IndexCastOp(i64, ow).result + + hbase = arith.SubIOp( + arith.MulIOp(oh_i64, sh_const).result, ph_const + ).result + wbase = arith.SubIOp( + arith.MulIOp(ow_i64, sw_const).result, pw_const + ).result + + ih = arith.AddIOp( + hbase, arith.MulIOp(h_inc, dh_const).result + ).result + iw = arith.AddIOp( + wbase, arith.MulIOp(w_inc, dw_const).result + ).result + + flat_idx = arith.AddIOp( + arith.MulIOp(ih, in_w_const).result, iw + ).result + memref.StoreOp(flat_idx, indices_memref.result, [n, c, oh, ow]) + scf.YieldOp([]) + scf.YieldOp([]) + scf.YieldOp([]) + scf.YieldOp([]) + + indices_type = ir.RankedTensorType.get(shape, i64) + return bufferization.ToTensorOp( + indices_type, indices_memref.result, restrict=True + ).result + + +def low_memory_max_pool_with_offsets_op( + node: LowMemoryMaxPoolWithOffsetsOp, + symbol_table: dict[tuple[str, int], ir.Operation], +): + input_tensor = symbol_table.get((str(node.args[0]), 0)) + if input_tensor is None: + return + + input_shape = list(ir.RankedTensorType(input_tensor.type).shape) + if any(dim < 0 for dim in input_shape): + raise NotImplementedError( + "_low_memory_max_pool_with_offsets requires static shapes" + ) + + dim = len(input_shape) - 2 + if dim not in (2, 3): + raise NotImplementedError( + "_low_memory_max_pool_with_offsets supports only 2d/3d" + ) + + kernel_size = _normalize_pool_param(node.args[1], dim, node.args[1]) + stride = _normalize_pool_param( + node.args[2] if len(node.args) > 2 else None, + dim, + kernel_size, + ) + padding = _normalize_pool_param( + node.args[3] if len(node.args) > 3 else None, + dim, + [0] * dim, + ) + dilation = _normalize_pool_param( + node.args[4] if len(node.args) > 4 else None, + dim, + [1] * dim, + ) + + if dim == 2: + pool_node = MaxPool2dWithIndicesOp() + pool_node._arguments = list(node.args) + values, indices = max_pool2d_with_indices_op(pool_node, symbol_table) + offsets = _max_pool_indices_to_offsets_2d( + indices, + _pool2d_input_hw(input_shape), + kernel_size, + stride, + padding, + dilation, + ) + return values, offsets + + pool_node = MaxPool3dOp() + pool_node._arguments = list(node.args) + values, indices = max_pool3d_op(pool_node, symbol_table) + offsets = _max_pool_indices_to_offsets_3d( + indices, + input_shape[-3:], + kernel_size, + stride, + padding, + dilation, + ) + return values, offsets + + +def max_pool2d_with_indices_op( + node: MaxPool2dWithIndicesOp, + symbol_table: dict[tuple[str, int], ir.Operation], +): + """ + Import the max_pool2d_with_indices operation. + From buddy MaxPool2dWithIndicesOp to MLIR operations using scf.for loops. + aten.max_pool2d_with_indices(input, kernel_size, stride, padding, + dilation, ceil_mode) -> (Tensor, Tensor) + + Returns both the max-pooled values and the indices of the max values. + Uses scf.for loops since TOSA max_pool2d doesn't support returning indices. + """ + input_tensor = symbol_table.get((str(node.args[0]), 0)) + kernel_size = node.args[1] + stride = node.args[2] if len(node.args) > 2 and node.args[2] else kernel_size + padding = node.args[3] if len(node.args) > 3 else [0, 0] + dilation = node.args[4] if len(node.args) > 4 else [1, 1] + ceil_mode = node.args[5] if len(node.args) > 5 else False + + ssa_shape = [int(x) for x in ir.RankedTensorType(input_tensor.type).shape] + src = node.args[0] if node.args else None + meta = None + if ( + src is not None + and hasattr(src, "tensor_meta") + and src.tensor_meta + and "shape" in src.tensor_meta + ): + meta = [int(x) for x in src.tensor_meta["shape"]] + if meta is not None: + if len(meta) != 4: + raise RuntimeError( + f"max_pool2d_with_indices requires 4D input meta: {meta}" + ) + N, C, H, W = meta + nhwc = ssa_shape == [N, H, W, C] + if not nhwc and ssa_shape != [N, C, H, W]: + raise RuntimeError( + f"max_pool2d_with_indices layout mismatch: ssa={ssa_shape} meta_nchw={meta}" + ) + else: + if len(ssa_shape) != 4: + raise RuntimeError( + f"max_pool2d_with_indices requires 4D input, got {ssa_shape}" + ) + n, a, b, c = ssa_shape + if a == b and c > a: + N, H, W, C = n, a, b, c + nhwc = True + elif b == c and a > b: + N, C, H, W = n, a, b, c + nhwc = False + else: + raise RuntimeError( + f"max_pool2d_with_indices cannot infer layout from ssa={ssa_shape}" + ) + + input_dtype = ir.RankedTensorType(input_tensor.type).element_type + + # Normalize kernel_size, stride, padding, dilation + if isinstance(kernel_size, int): + kh, kw = kernel_size, kernel_size + else: + kh = kernel_size[0] + kw = kernel_size[1] if len(kernel_size) > 1 else kernel_size[0] + + if isinstance(stride, int): + sh, sw = stride, stride + else: + sh = stride[0] + sw = stride[1] if len(stride) > 1 else stride[0] + + if isinstance(padding, int): + ph, pw = padding, padding + else: + ph = padding[0] + pw = padding[1] if len(padding) > 1 else padding[0] + + if isinstance(dilation, int): + dh, dw = dilation, dilation + else: + dh = dilation[0] + dw = dilation[1] if len(dilation) > 1 else dilation[0] + + # Calculate output dimensions + if ceil_mode: + out_h = (H + 2 * ph - dh * (kh - 1) - 1 + sh - 1) // sh + 1 + out_w = (W + 2 * pw - dw * (kw - 1) - 1 + sw - 1) // sw + 1 + else: + out_h = (H + 2 * ph - dh * (kh - 1) - 1) // sh + 1 + out_w = (W + 2 * pw - dw * (kw - 1) - 1) // sw + 1 + + output_shape = [N, out_h, out_w, C] if nhwc else [N, C, out_h, out_w] + output_type = ir.RankedTensorType.get(output_shape, input_dtype) + indices_dtype = ir.IntegerType.get_signless(64) + indices_type = ir.RankedTensorType.get(output_shape, indices_dtype) + + # Create memrefs for output and indices + output_memref_type = ir.MemRefType.get(output_shape, input_dtype) + indices_memref_type = ir.MemRefType.get(output_shape, indices_dtype) + + output_memref = memref.AllocOp(output_memref_type, [], []) + indices_memref = memref.AllocOp(indices_memref_type, [], []) + + # Initialize output with negative infinity + neg_inf = arith.ConstantOp( + input_dtype, ir.FloatAttr.get(input_dtype, float("-inf")) + ) + linalg.fill(neg_inf.result, outs=[output_memref.result]) + + # Initialize indices with zeros + zero_idx = arith.ConstantOp(indices_dtype, ir.IntegerAttr.get(indices_dtype, 0)) + linalg.fill(zero_idx.result, outs=[indices_memref.result]) + + # Convert input to memref + input_memref = bufferization.ToBufferOp( + ir.MemRefType.get(ssa_shape, input_dtype), input_tensor + ).result + + # Create constants + zero = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), 0) + ) + one = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), 1) + ) + n_bound = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), N) + ) + c_bound = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), C) + ) + out_h_bound = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), out_h) + ) + out_w_bound = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), out_w) + ) + kh_bound = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), kh) + ) + kw_bound = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), kw) + ) + + # Constants for calculations + sh_const = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), sh) + ) + sw_const = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), sw) + ) + ph_const = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), ph) + ) + pw_const = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), pw) + ) + dh_const = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), dh) + ) + dw_const = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), dw) + ) + h_const = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), H) + ) + w_const = arith.ConstantOp( + ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), W) + ) + w_i64 = arith.ConstantOp(indices_dtype, ir.IntegerAttr.get(indices_dtype, W)) + + # Six nested loops: n, c, oh, ow, kh, kw + n_loop = scf.ForOp(zero.result, n_bound.result, one.result) + with ir.InsertionPoint(n_loop.body): + n = n_loop.induction_variable + + c_loop = scf.ForOp(zero.result, c_bound.result, one.result) + with ir.InsertionPoint(c_loop.body): + c = c_loop.induction_variable + + oh_loop = scf.ForOp(zero.result, out_h_bound.result, one.result) + with ir.InsertionPoint(oh_loop.body): + oh = oh_loop.induction_variable + + ow_loop = scf.ForOp(zero.result, out_w_bound.result, one.result) + with ir.InsertionPoint(ow_loop.body): + ow = ow_loop.induction_variable + + # Base input position: oh * stride - padding + h_base = arith.MulIOp(oh, sh_const.result).result + h_base = arith.SubIOp(h_base, ph_const.result).result + w_base = arith.MulIOp(ow, sw_const.result).result + w_base = arith.SubIOp(w_base, pw_const.result).result + + # Inner loops over kernel + kih_loop = scf.ForOp(zero.result, kh_bound.result, one.result) + with ir.InsertionPoint(kih_loop.body): + kih = kih_loop.induction_variable + + kiw_loop = scf.ForOp(zero.result, kw_bound.result, one.result) + with ir.InsertionPoint(kiw_loop.body): + kiw = kiw_loop.induction_variable + + # Calculate input position with dilation + ih = arith.AddIOp( + h_base, + arith.MulIOp(kih, dh_const.result).result, + ).result + iw = arith.AddIOp( + w_base, + arith.MulIOp(kiw, dw_const.result).result, + ).result + + # Check bounds: 0 <= ih < H and 0 <= iw < W + ih_ge_0 = arith.CmpIOp( + arith.CmpIPredicate.sge, ih, zero.result + ).result + ih_lt_h = arith.CmpIOp( + arith.CmpIPredicate.slt, ih, h_const.result + ).result + iw_ge_0 = arith.CmpIOp( + arith.CmpIPredicate.sge, iw, zero.result + ).result + iw_lt_w = arith.CmpIOp( + arith.CmpIPredicate.slt, iw, w_const.result + ).result + + h_valid = arith.AndIOp(ih_ge_0, ih_lt_h).result + w_valid = arith.AndIOp(iw_ge_0, iw_lt_w).result + in_bounds = arith.AndIOp(h_valid, w_valid).result + + # If in bounds, load and compare + if_op = scf.IfOp(in_bounds, has_else=False) + with ir.InsertionPoint(if_op.then_block): + if nhwc: + in_idx = [n, ih, iw, c] + out_idx = [n, oh, ow, c] + else: + in_idx = [n, c, ih, iw] + out_idx = [n, c, oh, ow] + # Load input value + input_val = memref.LoadOp(input_memref, in_idx).result + + # Load current max + current_max = memref.LoadOp( + output_memref.result, out_idx + ).result + + # Check if input > current_max + is_greater = arith.CmpFOp( + arith.CmpFPredicate.OGT, + input_val, + current_max, + ).result + + inner_if = scf.IfOp(is_greater, has_else=False) + with ir.InsertionPoint(inner_if.then_block): + # Update max value + memref.StoreOp( + input_val, + output_memref.result, + out_idx, + ) + + # Calculate flattened index: ih * W + iw + ih_i64 = arith.IndexCastOp(indices_dtype, ih).result + iw_i64 = arith.IndexCastOp(indices_dtype, iw).result + flat_idx = arith.AddIOp( + arith.MulIOp(ih_i64, w_i64.result).result, + iw_i64, + ).result + + # Store index + memref.StoreOp( + flat_idx, + indices_memref.result, + out_idx, + ) + + scf.YieldOp([]) + + scf.YieldOp([]) + + scf.YieldOp([]) + + scf.YieldOp([]) + + scf.YieldOp([]) + + scf.YieldOp([]) + + scf.YieldOp([]) + + scf.YieldOp([]) + + # Convert memrefs back to tensors + output_result = bufferization.ToTensorOp( + output_type, output_memref.result, restrict=True + ) + indices_result = bufferization.ToTensorOp( + indices_type, indices_memref.result, restrict=True + ) + + return output_result, indices_result + + +def _quantize_input(activation, scale, nchw_to_nhwc=False): + input_shape = [int(x) for x in ir.RankedTensorType(activation.type).shape] + f32 = ir.F32Type.get() + i8 = ir.IntegerType.get_signless(8) + if ir.RankedTensorType(activation.type).element_type != f32: + raise ValueError("Quantized input quantization requires FP32") + scale = float(scale) + if not numpy.isfinite(scale) or scale <= 0.0: + raise ValueError( + "Quantized input quantization scale must be finite and positive" + ) + if nchw_to_nhwc: + if len(input_shape) != 4: + raise ValueError("NCHW-to-NHWC quantization requires rank 4") + n, c, h, w = input_shape + output_shape = [n, h, w, c] + dims = [ir.AffineExpr.get_dim(i) for i in range(4)] + input_map = ir.AffineMap.get(4, 0, [dims[0], dims[3], dims[1], dims[2]]) + output_map = ir.AffineMap.get_identity(4) + else: + output_shape = input_shape + input_map = ir.AffineMap.get_identity(len(input_shape)) + output_map = input_map + out_type = ir.RankedTensorType.get(output_shape, i8) + output = tensor.EmptyOp(output_shape, i8) + op = linalg.GenericOp( + [out_type], + [activation], + [output], + ir.ArrayAttr.get( + [ir.AffineMapAttr.get(input_map), ir.AffineMapAttr.get(output_map)] + ), + ir.ArrayAttr.get( + [ir.Attribute.parse("#linalg.iterator_type")] * len(output_shape) + ), + ) + op.operation.attributes["buckyball.quant_f32_to_i8"] = ir.BoolAttr.get(True) + op.operation.attributes["nchw_to_nhwc"] = ir.BoolAttr.get(nchw_to_nhwc) + op.operation.attributes["scale"] = ir.FloatAttr.get(f32, 1.0 / scale) + block = ir.Block.create_at_start(op.region, [f32, i8]) + block.append(linalg.YieldOp([block.arguments[1]])) + return op.result + + +def quantized_conv2d_op(node, symbol_table): + activation = symbol_table.get((str(node.args[0]), 0)) + weight = symbol_table.get((str(node.args[1]), 0)) + if activation is None or weight is None: + raise ValueError(f"missing Quantized Conv2D operand for {node.name}") + + f32 = ir.F32Type.get() + i8 = ir.IntegerType.get_signless(8) + i32 = ir.IntegerType.get_signless(32) + activation_type = ir.RankedTensorType(activation.type) + input_shape = [int(x) for x in activation_type.shape] + n, cin, h, w = node._input_shape + if activation_type.element_type == f32: + if input_shape == [n, cin, h, w]: + nchw_to_nhwc = True + elif input_shape == [n, h, w, cin]: + nchw_to_nhwc = False + else: + raise ValueError( + f"FP32 Quantized Conv2D input layout mismatch for {node.name}: " + f"expected {[n, cin, h, w]} or {[n, h, w, cin]}, got {input_shape}" + ) + activation = _quantize_input( + activation, node._input_scale, nchw_to_nhwc=nchw_to_nhwc + ) + elif activation_type.element_type == i8: + if input_shape != [n, h, w, cin]: + raise ValueError( + f"INT8 Quantized Conv2D input layout mismatch for {node.name}: " + f"expected NHWC {[n, h, w, cin]}, got {input_shape}" + ) + else: + raise ValueError("Quantized Conv2D activation must be FP32 or INT8") + + depthwise = isinstance(node, QuantizedDepthwiseConv2dOp) + weight_type = ir.RankedTensorType(weight.type) + cout, weight_cin, kh, kw = node._weight_shape + actual_weight_shape = [int(x) for x in weight_type.shape] + if actual_weight_shape != list(node._weight_shape): + raise ValueError(f"Quantized convolution weight shape mismatch for {node.name}") + if ( + weight_type.element_type != i8 + or (depthwise and (weight_cin != 1 or cin != cout)) + or (not depthwise and cin != weight_cin) + ): + raise ValueError("Quantized Conv2D requires INT8 OIHW weight with matching Cin") + + out_n, out_c, out_h, out_w = node._output_shape + if out_n != n or out_c != cout: + raise ValueError(f"Quantized Conv2D output shape mismatch for {node.name}") + scale_values = ( + node._dequant_scale + if node._final_output and node._activation != 2 + else node._requant_scale + ) + if len(node._bias_i32) != cout or len(scale_values) != cout: + raise ValueError(f"Quantized Conv2D quant parameter mismatch for {node.name}") + bias_type = ir.RankedTensorType.get([cout], i32) + bias_attr = ir.DenseElementsAttr.get( + numpy.asarray(node._bias_i32, dtype=numpy.int32), type=bias_type + ) + bias = arith.ConstantOp(bias_type, bias_attr).result + scale_type = ir.RankedTensorType.get([cout], f32) + scale_attr = ir.DenseElementsAttr.get( + numpy.asarray(scale_values, dtype=numpy.float32), type=scale_type + ) + scale = arith.ConstantOp(scale_type, scale_attr).result + lut_type = ir.RankedTensorType.get([len(node._lut_i8)], i8) + lut_attr = ir.DenseElementsAttr.get( + numpy.asarray(node._lut_i8, dtype=numpy.int8), type=lut_type + ) + lut = arith.ConstantOp(lut_type, lut_attr).result + output_shape = ( + [n, cout, out_h, out_w] + if node._final_output and not node._output_nhwc + else [n, out_h, out_w, cout] + ) + output_element = f32 if node._final_output else i8 + out_type = ir.RankedTensorType.get(output_shape, output_element) + output = tensor.EmptyOp(output_shape, output_element) + dims = [ir.AffineExpr.get_dim(i) for i in range(4)] + zero = ir.AffineExpr.get_constant(0) + output_map = ir.AffineMap.get_identity(4) + maps = [ + ir.AffineMap.get(4, 0, [zero, zero, zero, zero]), + ir.AffineMap.get(4, 0, [zero, zero, zero, zero]), + ir.AffineMap.get(4, 0, [zero]), + ir.AffineMap.get(4, 0, [zero]), + ir.AffineMap.get(4, 0, [zero]), + output_map, + ] + op = linalg.GenericOp( + [out_type], + [activation, weight, bias, scale, lut], + [output], + ir.ArrayAttr.get([ir.AffineMapAttr.get(x) for x in maps]), + ir.ArrayAttr.get([ir.Attribute.parse("#linalg.iterator_type")] * 4), + ) + i64 = ir.IntegerType.get_signless(64) + marker = ( + "buckyball.quantized_conv2d_depthwise" + if depthwise + else "buckyball.quantized_conv2d" + ) + op.operation.attributes[marker] = ir.BoolAttr.get(True) + op.operation.attributes["stride"] = ir.IntegerAttr.get(i64, node._stride) + op.operation.attributes["kernel"] = ir.IntegerAttr.get(i64, kh) + op.operation.attributes["pad_low"] = ir.IntegerAttr.get(i64, node._padding) + op.operation.attributes["pad_high"] = ir.IntegerAttr.get(i64, node._padding) + op.operation.attributes["activation"] = ir.IntegerAttr.get(i64, node._activation) + op.operation.attributes["output_scale"] = ir.FloatAttr.get(f32, node._output_scale) + if getattr(node, "_lane_output_scales", []): + op.operation.attributes["lane_output_scales"] = ir.DenseF32ArrayAttr.get( + node._lane_output_scales + ) + op.operation.attributes["final_output"] = ir.BoolAttr.get(node._final_output) + op.operation.attributes["output_nhwc"] = ir.BoolAttr.get(node._output_nhwc) + if node._output_nhwc or not node._final_output: + op.operation.attributes["buddy.layout"] = ir.StringAttr.get("nhwc") + block = ir.Block.create_at_start(op.region, [i8, i8, i32, f32, i8, output_element]) + block.append(linalg.YieldOp([block.arguments[5]])) + return op.result + + +def quantized_matmul_op(node, symbol_table): + activation = symbol_table.get((str(node.args[0]), 0)) + weight = symbol_table.get((str(node.args[1]), 0)) + if activation is None or weight is None: + raise ValueError(f"missing Quantized MatMul operand for {node.name}") + + f32 = ir.F32Type.get() + i8 = ir.IntegerType.get_signless(8) + i32 = ir.IntegerType.get_signless(32) + activation_type = ir.RankedTensorType(activation.type) + if activation_type.element_type == f32: + activation = _quantize_input(activation, node._input_scale, nchw_to_nhwc=False) + activation_type = ir.RankedTensorType(activation.type) + weight_type = ir.RankedTensorType(weight.type) + if ( + activation_type.element_type != i8 + or weight_type.element_type != i8 + or activation_type.rank != 2 + or weight_type.rank != 2 + ): + raise ValueError("Quantized MatMul requires rank-2 INT8 activation and weight") + m, k = [int(x) for x in activation_type.shape] + n, wk = [int(x) for x in weight_type.shape] + scale_values = ( + node._dequant_scale + if node._final_output and node._activation != 2 + else node._requant_scale + ) + if k != wk or len(node._bias_i32) != n or len(scale_values) != n: + raise ValueError(f"Quantized MatMul shape mismatch for {node.name}") + + bias_type = ir.RankedTensorType.get([n], i32) + bias_attr = ir.DenseElementsAttr.get( + numpy.asarray(node._bias_i32, dtype=numpy.int32), type=bias_type + ) + bias = arith.ConstantOp(bias_type, bias_attr).result + scale_type = ir.RankedTensorType.get([n], f32) + scale_attr = ir.DenseElementsAttr.get( + numpy.asarray(scale_values, dtype=numpy.float32), type=scale_type + ) + scale = arith.ConstantOp(scale_type, scale_attr).result + lut_type = ir.RankedTensorType.get([len(node._lut_i8)], i8) + lut_attr = ir.DenseElementsAttr.get( + numpy.asarray(node._lut_i8, dtype=numpy.int8), type=lut_type + ) + lut = arith.ConstantOp(lut_type, lut_attr).result + output_element = f32 if node._final_output else i8 + out_type = ir.RankedTensorType.get([m, n], output_element) + output = tensor.EmptyOp([m, n], output_element) + dims = [ir.AffineExpr.get_dim(i) for i in range(3)] + maps = [ + ir.AffineMap.get(3, 0, [dims[0], dims[2]]), + ir.AffineMap.get(3, 0, [dims[1], dims[2]]), + ir.AffineMap.get(3, 0, [dims[1]]), + ir.AffineMap.get(3, 0, [dims[1]]), + ir.AffineMap.get(3, 0, [ir.AffineExpr.get_constant(0)]), + ir.AffineMap.get(3, 0, [dims[0], dims[1]]), + ] + op = linalg.GenericOp( + [out_type], + [activation, weight, bias, scale, lut], + [output], + ir.ArrayAttr.get([ir.AffineMapAttr.get(x) for x in maps]), + ir.ArrayAttr.get( + [ + ir.Attribute.parse("#linalg.iterator_type"), + ir.Attribute.parse("#linalg.iterator_type"), + ir.Attribute.parse("#linalg.iterator_type"), + ] + ), + ) + op.operation.attributes["buckyball.quantized_matmul"] = ir.BoolAttr.get(True) + op.operation.attributes["activation"] = ir.IntegerAttr.get( + ir.IntegerType.get_signless(64), node._activation + ) + op.operation.attributes["output_scale"] = ir.FloatAttr.get(f32, node._output_scale) + op.operation.attributes["final_output"] = ir.BoolAttr.get(node._final_output) + block = ir.Block.create_at_start(op.region, [i8, i8, i32, f32, i8, output_element]) + block.append(linalg.YieldOp([block.arguments[5]])) + return op.result + + +def quantized_max_pool2d_op(node, symbol_table): + input_value = symbol_table.get((str(node.args[0]), 0)) + if input_value is None: + raise ValueError(f"missing Quantized MaxPool2D input for {node.name}") + i8 = ir.IntegerType.get_signless(8) + input_type = ir.RankedTensorType(input_value.type) + if input_type.element_type != i8 or input_type.rank != 4: + raise ValueError("Quantized MaxPool2D input must be rank-4 INT8 NHWC") + n, h, w, c = [int(x) for x in input_type.shape] + out_n, out_c, out_h, out_w = node._output_shape + if ( + [n, c] != [out_n, out_c] + or node._kernel <= 0 + or node._stride <= 0 + or node._padding < 0 + or (h + 2 * node._padding - node._kernel) // node._stride + 1 != out_h + or (w + 2 * node._padding - node._kernel) // node._stride + 1 != out_w + ): + raise ValueError(f"Quantized MaxPool2D geometry mismatch for {node.name}") + output_shape = ( + [out_n, out_c, out_h, out_w] + if node._final_output + else [out_n, out_h, out_w, out_c] + ) + output_type = ir.RankedTensorType.get(output_shape, i8) + output = tensor.EmptyOp(output_shape, i8) + # This is a marker consumed by Tile lowering, like Quantized Conv. Iterate + # over the output; input-sized loops overrun odd-sized pooling results. + zero = ir.AffineExpr.get_constant(0) + input_map = ir.AffineMap.get(4, 0, [zero, zero, zero, zero]) + output_map = ir.AffineMap.get_identity(4) + op = linalg.GenericOp( + [output_type], + [input_value], + [output], + ir.ArrayAttr.get( + [ir.AffineMapAttr.get(input_map), ir.AffineMapAttr.get(output_map)] + ), + ir.ArrayAttr.get([ir.Attribute.parse("#linalg.iterator_type")] * 4), + ) + i64 = ir.IntegerType.get_signless(64) + op.operation.attributes["buckyball.quantized_max_pool2d"] = ir.BoolAttr.get(True) + op.operation.attributes["kernel"] = ir.IntegerAttr.get(i64, node._kernel) + op.operation.attributes["stride"] = ir.IntegerAttr.get(i64, node._stride) + op.operation.attributes["padding"] = ir.IntegerAttr.get(i64, node._padding) + op.operation.attributes["final_output"] = ir.BoolAttr.get(node._final_output) + block = ir.Block.create_at_start(op.region, [i8, i8]) + block.append(linalg.YieldOp([block.arguments[1]])) + return op.result + + +def quantized_global_avg_pool_op(node, symbol_table): + input_value = symbol_table.get((str(node.args[0]), 0)) + if input_value is None: + raise ValueError(f"missing Quantized global-average input for {node.name}") + i8 = ir.IntegerType.get_signless(8) + input_type = ir.RankedTensorType(input_value.type) + if input_type.element_type != i8 or input_type.rank != 4: + raise ValueError("Quantized global-average input must be rank-4 INT8 NHWC") + n, h, w, c = [int(x) for x in input_type.shape] + if min(n, h, w, c) <= 0: + raise ValueError("Quantized global-average shape must be positive") + output_type = ir.RankedTensorType.get([n, 1, 1, c], i8) + output = tensor.EmptyOp([n, 1, 1, c], i8) + dims = [ir.AffineExpr.get_dim(i) for i in range(4)] + zero = ir.AffineExpr.get_constant(0) + op = linalg.GenericOp( + [output_type], + [input_value], + [output], + ir.ArrayAttr.get( + [ + ir.AffineMapAttr.get(ir.AffineMap.get_identity(4)), + ir.AffineMapAttr.get( + ir.AffineMap.get(4, 0, [dims[0], zero, zero, dims[3]]) + ), + ] + ), + ir.ArrayAttr.get( + [ + ir.Attribute.parse("#linalg.iterator_type"), + ir.Attribute.parse("#linalg.iterator_type"), + ir.Attribute.parse("#linalg.iterator_type"), + ir.Attribute.parse("#linalg.iterator_type"), + ] + ), + ) + op.operation.attributes["buckyball.quantized_global_avg_pool"] = ir.BoolAttr.get( + True + ) + f32 = ir.F32Type.get() + op.operation.attributes["input_scale"] = ir.FloatAttr.get(f32, node._input_scale) + op.operation.attributes["output_scale"] = ir.FloatAttr.get(f32, node._output_scale) + block = ir.Block.create_at_start(op.region, [i8, i8]) + block.append(linalg.YieldOp([block.arguments[1]])) + return op.result + + +def quantized_elementwise_op(node, symbol_table, kind): + lhs = symbol_table.get((str(node.args[0]), 0)) + rhs = symbol_table.get((str(node.args[1]), 0)) + if lhs is None or rhs is None: + raise ValueError(f"missing Quantized INT8 {kind} input for {node.name}") + i8 = ir.IntegerType.get_signless(8) + lhs_type = ir.RankedTensorType(lhs.type) + rhs_type = ir.RankedTensorType(rhs.type) + if lhs_type.element_type != i8 or rhs_type.element_type != i8: + raise ValueError( + f"Quantized INT8 {kind} operands must be INT8 for {node.name}: " + f"{node.args[0]}={lhs_type}, {node.args[1]}={rhs_type}" + ) + lhs_shape = [int(x) for x in lhs_type.shape] + rhs_shape = [int(x) for x in rhs_type.shape] + if len(lhs_shape) != 4 or len(rhs_shape) != 4: + raise ValueError(f"Quantized INT8 {kind} operands must be rank 4") + output_shape = [max(a, b) for a, b in zip(lhs_shape, rhs_shape)] + if any( + a not in (1, out) or b not in (1, out) + for a, b, out in zip(lhs_shape, rhs_shape, output_shape) + ): + raise ValueError(f"Quantized INT8 {kind} operands are not broadcast-compatible") + if kind == "add" and lhs_shape != rhs_shape: + raise ValueError("Quantized residual add requires equal shapes") + output_type = ir.RankedTensorType.get(output_shape, i8) + output = tensor.EmptyOp(output_shape, i8) + dims = [ir.AffineExpr.get_dim(i) for i in range(4)] + zero = ir.AffineExpr.get_constant(0) + lhs_map = ir.AffineMap.get( + 4, + 0, + [zero if size == 1 else dims[i] for i, size in enumerate(lhs_shape)], + ) + rhs_map = ir.AffineMap.get( + 4, + 0, + [zero if size == 1 else dims[i] for i, size in enumerate(rhs_shape)], + ) + op = linalg.GenericOp( + [output_type], + [lhs, rhs], + [output], + ir.ArrayAttr.get( + [ + ir.AffineMapAttr.get(lhs_map), + ir.AffineMapAttr.get(rhs_map), + ir.AffineMapAttr.get(ir.AffineMap.get_identity(4)), + ] + ), + ir.ArrayAttr.get([ir.Attribute.parse("#linalg.iterator_type")] * 4), + ) + op.operation.attributes[f"buckyball.quantized_int8_{kind}"] = ir.BoolAttr.get(True) + f32 = ir.F32Type.get() + op.operation.attributes["lhs_scale"] = ir.FloatAttr.get(f32, node._lhs_scale) + op.operation.attributes["rhs_scale"] = ir.FloatAttr.get(f32, node._rhs_scale) + op.operation.attributes["output_scale"] = ir.FloatAttr.get(f32, node._output_scale) + op.operation.attributes["activation"] = ir.IntegerAttr.get( + ir.IntegerType.get_signless(64), node._activation + ) + block = ir.Block.create_at_start(op.region, [i8, i8, i8]) + block.append(linalg.YieldOp([block.arguments[2]])) + return op.result + + +def quantized_channel_slice_op(node, symbol_table): + source = symbol_table.get((str(node.args[0]), 0)) + if source is None: + raise ValueError(f"missing Quantized channel-slice input for {node.name}") + i8 = ir.IntegerType.get_signless(8) + source_type = ir.RankedTensorType(source.type) + if source_type.element_type != i8 or len(source_type.shape) != 4: + raise ValueError(f"Quantized channel-slice requires rank-4 INT8 at {node.name}") + n, c, h, w = node._output_shape + output_shape = [n, h, w, c] + output_type = ir.RankedTensorType.get(output_shape, i8) + output = tensor.EmptyOp(output_shape, i8) + source_slice = tensor.ExtractSliceOp( + output_type, + source, + [], + [], + [], + ir._denseI64ArrayAttr([0, 0, 0, node._offset], None), + ir._denseI64ArrayAttr(output_shape, None), + ir._denseI64ArrayAttr([1, 1, 1, 1], None), + ) + op = linalg.GenericOp( + [output_type], + [source_slice], + [output], + ir.ArrayAttr.get( + [ + ir.AffineMapAttr.get(ir.AffineMap.get_identity(4)), + ir.AffineMapAttr.get(ir.AffineMap.get_identity(4)), + ] + ), + ir.ArrayAttr.get([ir.Attribute.parse("#linalg.iterator_type")] * 4), + ) + op.operation.attributes["buckyball.quantized_channel_slice"] = ir.BoolAttr.get(True) + op.operation.attributes["offset"] = ir.IntegerAttr.get( + ir.IntegerType.get_signless(64), 0 + ) + block = ir.Block.create_at_start(op.region, [i8, i8]) + block.append(linalg.YieldOp([block.arguments[0]])) + return op.result + + +def quantized_channel_concat_op(node, symbol_table): + inputs = [symbol_table.get((str(name), 0)) for name in node.args] + if not inputs or any(value is None for value in inputs): + raise ValueError(f"missing Quantized channel-concat input for {node.name}") + i8 = ir.IntegerType.get_signless(8) + input_shapes = [list(ir.RankedTensorType(value.type).shape) for value in inputs] + if any( + ir.RankedTensorType(value.type).element_type != i8 or len(shape) != 4 + for value, shape in zip(inputs, input_shapes) + ): + raise ValueError( + f"Quantized channel-concat requires rank-4 INT8 at {node.name}" + ) + n, c, h, w = node._output_shape + output_shape = [n, h, w, c] + if ( + any(shape[:3] != [n, h, w] for shape in input_shapes) + or sum(shape[3] for shape in input_shapes) != c + ): + raise ValueError(f"Quantized channel-concat shape mismatch at {node.name}") + output_type = ir.RankedTensorType.get(output_shape, i8) + output = tensor.EmptyOp(output_shape, i8) + dims = [ir.AffineExpr.get_dim(i) for i in range(4)] + zero = ir.AffineExpr.get_constant(0) + maps = [ + ir.AffineMapAttr.get(ir.AffineMap.get(4, 0, [dims[0], dims[1], dims[2], zero])) + for _ in inputs + ] + maps.append(ir.AffineMapAttr.get(ir.AffineMap.get_identity(4))) + op = linalg.GenericOp( + [output_type], + inputs, + [output], + ir.ArrayAttr.get(maps), + ir.ArrayAttr.get([ir.Attribute.parse("#linalg.iterator_type")] * 4), + ) + op.operation.attributes["buckyball.quantized_channel_concat"] = ir.BoolAttr.get( + True + ) + op.operation.attributes["segments"] = ir.DenseI64ArrayAttr.get( + [shape[3] for shape in input_shapes] + ) + block = ir.Block.create_at_start(op.region, [i8] * (len(inputs) + 1)) + block.append(linalg.YieldOp([block.arguments[-1]])) + return op.result + + +def quantized_resize_nearest_op(node, symbol_table): + source = symbol_table.get((str(node.args[0]), 0)) + if source is None: + raise ValueError(f"missing Quantized resize input for {node.name}") + i8 = ir.IntegerType.get_signless(8) + source_type = ir.RankedTensorType(source.type) + if source_type.element_type != i8 or len(source_type.shape) != 4: + raise ValueError(f"Quantized resize requires rank-4 INT8 at {node.name}") + n, c, h, w = node._output_shape + output_shape = [n, h, w, c] + output_type = ir.RankedTensorType.get(output_shape, i8) + output = tensor.EmptyOp(output_shape, i8) + dims = [ir.AffineExpr.get_dim(i) for i in range(4)] + zero = ir.AffineExpr.get_constant(0) + op = linalg.GenericOp( + [output_type], + [source], + [output], + ir.ArrayAttr.get( + [ + ir.AffineMapAttr.get( + ir.AffineMap.get(4, 0, [dims[0], zero, zero, dims[3]]) + ), + ir.AffineMapAttr.get(ir.AffineMap.get_identity(4)), + ] + ), + ir.ArrayAttr.get([ir.Attribute.parse("#linalg.iterator_type")] * 4), + ) + op.operation.attributes["buckyball.quantized_resize_nearest"] = ir.BoolAttr.get( + True + ) + op.operation.attributes["scale_h"] = ir.IntegerAttr.get( + ir.IntegerType.get_signless(64), node._scale_h + ) + op.operation.attributes["scale_w"] = ir.IntegerAttr.get( + ir.IntegerType.get_signless(64), node._scale_w + ) + block = ir.Block.create_at_start(op.region, [i8, i8]) + block.append(linalg.YieldOp([block.arguments[0]])) + return op.result + + +def quantized(lower): + def emit(node, symbols): + result = lower(node, symbols) + if node.quantization_trace_meta is not None: + from types import SimpleNamespace + + result.owner.attributes["trace_node"] = ir.DictAttr.get( + _trace_attrs(SimpleNamespace(trace_meta=node.quantization_trace_meta)) + ) + return result + + return emit + + +from buddy.compiler.ops.linalg import ops_registry as upstream_registry + +ops_registry = { + **upstream_registry, + "QuantizedMulOp": quantized( + lambda node, symbols: quantized_elementwise_op(node, symbols, "mul") + ), + "QuantizedAddOp": quantized( + lambda node, symbols: quantized_elementwise_op(node, symbols, "add") + ), + "QuantizedConv2dOp": quantized(quantized_conv2d_op), + "QuantizedDepthwiseConv2dOp": quantized(quantized_conv2d_op), + "QuantizedMatmulOp": quantized(quantized_matmul_op), + "QuantizedMaxPool2dOp": quantized(quantized_max_pool2d_op), + "QuantizedGlobalAvgPoolOp": quantized(quantized_global_avg_pool_op), + "QuantizedChannelSliceOp": quantized(quantized_channel_slice_op), + "QuantizedChannelConcatOp": quantized(quantized_channel_concat_op), + "QuantizedResizeNearestOp": quantized(quantized_resize_nearest_op), + "CatOp": cat_op, + "SplitOp": split_op, + "MaxPool2dWithIndicesOp": max_pool2d_with_indices_op, + "LowMemoryMaxPoolWithOffsetsOp": low_memory_max_pool_with_offsets_op, +} diff --git a/waste/stack/compiler/frontend/ops/native_mxfp8.py b/waste/stack/compiler/frontend/ops/native_mxfp8.py new file mode 100644 index 000000000..146a2eccb --- /dev/null +++ b/waste/stack/compiler/frontend/ops/native_mxfp8.py @@ -0,0 +1,187 @@ +"""Portable MXFP8 codecs: E4M3FN values and one E8M0 scale per 32 values.""" + +from buddy_mlir import ir +from buddy_mlir.dialects import arith, linalg, math, tensor + +from stack.compiler.frontend.ops.reductions import _generic + + +def _constant(dtype, value): + return arith.ConstantOp(dtype, value).result + + +def _power(scale): + i32, f32 = (ir.IntegerType.get_signless(32), ir.F32Type.get()) + exponent = arith.ExtUIOp(i32, scale).result + bits = arith.ShLIOp(exponent, _constant(i32, 23)).result + normal = arith.BitcastOp(f32, bits).result + return arith.SelectOp( + arith.CmpIOp(arith.CmpIPredicate.eq, exponent, _constant(i32, 0)).result, + _constant(f32, 2.0 ** (-127)), + normal, + ).result + + +def _decode(code, scale): + i32, f32 = (ir.IntegerType.get_signless(32), ir.F32Type.get()) + value = arith.ExtUIOp(i32, code).result + sign = arith.AndIOp(value, _constant(i32, 128)).result + fraction = arith.AndIOp(value, _constant(i32, 7)).result + exponent = arith.AndIOp( + arith.ShRUIOp(value, _constant(i32, 3)).result, _constant(i32, 15) + ).result + bits = arith.OrIOp( + arith.ShLIOp( + arith.AddIOp(exponent, _constant(i32, 120)).result, + _constant(i32, 23), + ).result, + arith.ShLIOp(fraction, _constant(i32, 20)).result, + ).result + normal = arith.BitcastOp(f32, bits).result + subnormal = arith.MulFOp( + arith.UIToFPOp(f32, fraction).result, _constant(f32, 1.0 / 512) + ).result + magnitude = arith.SelectOp( + arith.CmpIOp(arith.CmpIPredicate.eq, exponent, _constant(i32, 0)).result, + subnormal, + normal, + ).result + signed = arith.SelectOp( + arith.CmpIOp(arith.CmpIPredicate.ne, sign, _constant(i32, 0)).result, + arith.NegFOp(magnitude).result, + magnitude, + ).result + return arith.MulFOp(signed, _power(scale)).result + + +def _encode(value, scale): + i8, i32, f32 = ( + ir.IntegerType.get_signless(8), + ir.IntegerType.get_signless(32), + ir.F32Type.get(), + ) + scaled = arith.DivFOp(value, _power(scale)).result + magnitude = arith.MinimumFOp( + math.AbsFOp(scaled).result, _constant(f32, 448.0) + ).result + bits = arith.BitcastOp(i32, magnitude).result + exponent = arith.SubIOp( + arith.ShRUIOp(bits, _constant(i32, 23)).result, _constant(i32, 120) + ).result + fraction = arith.AndIOp(bits, _constant(i32, 8388607)).result + high = arith.ShRUIOp(fraction, _constant(i32, 20)).result + low = arith.AndIOp(fraction, _constant(i32, 1048575)).result + increment = arith.OrIOp( + arith.CmpIOp(arith.CmpIPredicate.ugt, low, _constant(i32, 524288)).result, + arith.AndIOp( + arith.CmpIOp(arith.CmpIPredicate.eq, low, _constant(i32, 524288)).result, + arith.CmpIOp( + arith.CmpIPredicate.ne, + arith.AndIOp(high, _constant(i32, 1)).result, + _constant(i32, 0), + ).result, + ).result, + ).result + rounded = arith.AddIOp(high, arith.ExtUIOp(i32, increment).result).result + normal = arith.AddIOp( + arith.MulIOp(exponent, _constant(i32, 8)).result, rounded + ).result + subnormal = arith.FPToSIOp( + i32, + math.RoundEvenOp(arith.MulFOp(magnitude, _constant(f32, 512.0)).result).result, + ).result + code = arith.SelectOp( + arith.CmpIOp(arith.CmpIPredicate.sle, exponent, _constant(i32, 0)).result, + subnormal, + normal, + ).result + sign = arith.AndIOp( + arith.ShRUIOp(arith.BitcastOp(i32, value).result, _constant(i32, 24)).result, + _constant(i32, 128), + ).result + return arith.TruncIOp(i8, arith.OrIOp(code, sign).result).result + + +def codec(node, symbols): + values, scales = [symbols[name, 0] for name in node.args] + shape = list(ir.RankedTensorType(values.type).shape) + rank = len(shape) + scale_shape = list(ir.RankedTensorType(scales.type).shape) + dims = [ir.AffineExpr.get_dim(i) for i in range(rank)] + scale_dims = [ + ir.AffineExpr.get_constant(0) if size == 1 else dim + for dim, size in zip(dims[:-1], scale_shape[:-1], strict=True) + ] + scale_dims.append( + ir.AffineExpr.get_floor_div(dims[-1], ir.AffineExpr.get_constant(32)) + ) + identity = ir.AffineMap.get_identity(rank) + scale_map = ir.AffineMap.get(rank, 0, scale_dims) + encoding = type(node).__name__ == "NativeMXFP8EncodeOp" + dtype = ir.IntegerType.get_signless(8) if encoding else ir.F32Type.get() + output = tensor.EmptyOp(shape, dtype).result + operation = _generic( + [values, scales], + output, + [identity, scale_map, identity], + ["parallel"] * rank, + ) + operation.attributes["buddy.mxfp8"] = ir.StringAttr.get( + "encode" if encoding else "decode" + ) + body = ir.Block.create_at_start( + operation.region, + [ + ir.RankedTensorType(values.type).element_type, + ir.IntegerType.get_signless(8), + dtype, + ], + ) + with ir.InsertionPoint(body): + result = (_encode if encoding else _decode)(*body.arguments[:2]) + linalg.YieldOp([result]) + return operation.result + + +def concat(node, symbols): + """LLM tensors retain logical axes, including singleton head dimensions.""" + from buddy_mlir.dialects import tosa + + values = [symbols[(name, 0)] for name in node.args[0]] + axis = node.args[1] if len(node.args) == 2 else 0 + rank = ir.RankedTensorType(values[0].type).rank + return tosa.ConcatOp(values, axis % rank).result + + +def matmul(node, symbols): + """Decode E4M3 operands inside the reduction without expanding weight storage.""" + from stack.compiler.frontend.ops.reductions import _cast, _filled + + activation, activation_scales, weights, weight_scales = [ + symbols[(name, 0)] for name in node.args + ] + m, n, k = [ir.AffineExpr.get_dim(i) for i in range(3)] + block = ir.AffineExpr.get_floor_div(k, ir.AffineExpr.get_constant(32)) + zero = ir.AffineExpr.get_constant(0) + maps = [ + ir.AffineMap.get(3, 0, dims) + for dims in ([m, k], [zero, block], [n, k], [n, block], [m, n]) + ] + f64, i8 = ir.F64Type.get(), ir.IntegerType.get_signless(8) + output = _filled(list(node.tensor_meta["shape"]), f64, 0.0) + operation = _generic( + [activation, activation_scales, weights, weight_scales], + output, + maps, + ["parallel", "parallel", "reduction"], + ) + operation.attributes["buddy.mxfp8"] = ir.StringAttr.get("matmul") + body = ir.Block.create_at_start(operation.region, [i8, i8, i8, i8, f64]) + with ir.InsertionPoint(body): + a, a_scale, w, w_scale, accumulator = body.arguments + product = arith.MulFOp( + arith.ExtFOp(f64, _decode(a, a_scale)).result, + arith.ExtFOp(f64, _decode(w, w_scale)).result, + ).result + linalg.YieldOp([arith.AddFOp(accumulator, product).result]) + return _cast(operation.result, ir.F32Type.get()) diff --git a/waste/stack/compiler/frontend/ops/reductions.py b/waste/stack/compiler/frontend/ops/reductions.py new file mode 100644 index 000000000..d7ad7460a --- /dev/null +++ b/waste/stack/compiler/frontend/ops/reductions.py @@ -0,0 +1,308 @@ +"""Stable floating reductions between quantized integer contractions.""" + +import math as scalar_math + +from buddy_mlir import ir +from buddy_mlir.dialects import arith, linalg, math, tensor + + +def _filled(shape, dtype, value): + empty = tensor.EmptyOp(shape, dtype) + scalar = arith.ConstantOp(dtype, value).result + result = linalg.FillOp([empty.result.type], [scalar], [empty.result]) + linalg.fill_builtin_region(result.operation) + return result.result + + +def _generic(inputs, output, maps, iterators): + return linalg.GenericOp( + [output.type], + inputs, + [output], + ir.ArrayAttr.get([ir.AffineMapAttr.get(m) for m in maps]), + ir.ArrayAttr.get( + [ + ir.Attribute.parse("#linalg.iterator_type<" + item + ">") + for item in iterators + ] + ), + ) + + +def _cast(value, dtype): + ty = ir.RankedTensorType(value.type) + identity = ir.AffineMap.get_identity(ty.rank) + empty = tensor.EmptyOp(ty.shape, dtype).result + op = _generic([value], empty, [identity, identity], ["parallel"] * ty.rank) + body = ir.Block.create_at_start(op.region, [ty.element_type, dtype]) + with ir.InsertionPoint(body): + converted = ( + arith.ExtFOp(dtype, body.arguments[0]).result + if ir.FloatType(dtype).width > ir.FloatType(ty.element_type).width + else arith.TruncFOp(dtype, body.arguments[0]).result + ) + linalg.YieldOp([converted]) + return op.result + + +def _sum_f64(value, axes, keepdim=True): + ty = ir.RankedTensorType(value.type) + shape = list(ty.shape) + dims = [ir.AffineExpr.get_dim(i) for i in range(ty.rank)] + output_shape = ( + [1 if i in axes else size for i, size in enumerate(shape)] + if keepdim + else [size for i, size in enumerate(shape) if i not in axes] + ) + results = ( + [ + ir.AffineExpr.get_constant(0) if i in axes else dim + for i, dim in enumerate(dims) + ] + if keepdim + else [dim for i, dim in enumerate(dims) if i not in axes] + ) + f64 = ir.F64Type.get() + op = _generic( + [value], + _filled(output_shape, f64, 0.0), + [ + ir.AffineMap.get_identity(ty.rank), + ir.AffineMap.get(ty.rank, 0, results), + ], + ["reduction" if i in axes else "parallel" for i in range(ty.rank)], + ) + body = ir.Block.create_at_start(op.region, [ty.element_type, f64]) + with ir.InsertionPoint(body): + number = ( + body.arguments[0] + if ty.element_type == f64 + else arith.ExtFOp(f64, body.arguments[0]).result + ) + linalg.YieldOp([arith.AddFOp(number, body.arguments[1]).result]) + return op.result + + +def _mean_f64(value, axes, keepdim=True): + summed = _sum_f64(value, axes, keepdim) + ty = ir.RankedTensorType(summed.type) + identity = ir.AffineMap.get_identity(ty.rank) + op = _generic( + [summed], + tensor.EmptyOp(ty.shape, ty.element_type).result, + [identity, identity], + ["parallel"] * ty.rank, + ) + body = ir.Block.create_at_start(op.region, [ty.element_type] * 2) + with ir.InsertionPoint(body): + count = scalar_math.prod(ir.RankedTensorType(value.type).shape[i] for i in axes) + divisor = arith.ConstantOp(ty.element_type, float(count)).result + linalg.YieldOp([arith.DivFOp(body.arguments[0], divisor).result]) + return op.result + + +def mean(node, symbols): + from stack.compiler.frontend.ops.tosa import _nhwc_to_nchw + from stack.compiler.frontend.layout.provenance import has_nhwc_layout + + value = symbols[(node.args[0], 0)] + if has_nhwc_layout(value): + value = _nhwc_to_nchw(value) + rank = ir.RankedTensorType(value.type).rank + return _cast( + _mean_f64(value, {int(axis) % rank for axis in node.args[1]}, node.args[2]), + ir.F32Type.get(), + ) + + +def bmm(node, symbols): + lhs, rhs = [symbols[(name, 0)] for name in node.args] + f32, f64 = ir.F32Type.get(), ir.F64Type.get() + b, m, n, k = [ir.AffineExpr.get_dim(i) for i in range(4)] + maps = [ + ir.AffineMap.get(4, 0, [b, m, k]), + ir.AffineMap.get(4, 0, [b, k, n]), + ir.AffineMap.get(4, 0, [b, m, n]), + ] + op = _generic( + [lhs, rhs], + _filled(node.tensor_meta["shape"], f64, 0.0), + maps, + ["parallel"] * 3 + ["reduction"], + ) + body = ir.Block.create_at_start(op.region, [f32, f32, f64]) + with ir.InsertionPoint(body): + a = arith.ExtFOp(f64, body.arguments[0]).result + b = arith.ExtFOp(f64, body.arguments[1]).result + linalg.YieldOp([math.FmaOp(a, b, body.arguments[2]).result]) + return _cast(op.result, f32) + + +def softmax(node, symbols): + value = symbols[(node.args[0], 0)] + ty = ir.RankedTensorType(value.type) + shape = list(ty.shape) + rank = ty.rank + axis = int(node.args[1]) % rank + f32, f64 = ir.F32Type.get(), ir.F64Type.get() + dims = [ir.AffineExpr.get_dim(i) for i in range(rank)] + reduced_shape = [1 if i == axis else size for i, size in enumerate(shape)] + small = ir.AffineMap.get( + rank, + 0, + [ + ir.AffineExpr.get_constant(0) if i == axis else dim + for i, dim in enumerate(dims) + ], + ) + identity = ir.AffineMap.get_identity(rank) + maximum = _generic( + [value], + _filled(reduced_shape, f32, float("-inf")), + [identity, small], + ["reduction" if i == axis else "parallel" for i in range(rank)], + ) + body = ir.Block.create_at_start(maximum.region, [f32, f32]) + with ir.InsertionPoint(body): + linalg.YieldOp([arith.MaximumFOp(*body.arguments).result]) + exponent = _generic( + [value, maximum.result], + tensor.EmptyOp(shape, f64).result, + [identity, small, identity], + ["parallel"] * rank, + ) + body = ir.Block.create_at_start(exponent.region, [f32, f32, f64]) + with ir.InsertionPoint(body): + x = arith.ExtFOp(f64, body.arguments[0]).result + maximum_value = arith.ExtFOp(f64, body.arguments[1]).result + linalg.YieldOp([math.ExpOp(arith.SubFOp(x, maximum_value).result).result]) + summed = _sum_f64(exponent.result, {axis}) + output = _generic( + [exponent.result, summed], + tensor.EmptyOp(shape, f32).result, + [identity, small, identity], + ["parallel"] * rank, + ) + body = ir.Block.create_at_start(output.region, [f64, f64, f32]) + with ir.InsertionPoint(body): + linalg.YieldOp( + [ + arith.TruncFOp( + f32, + arith.DivFOp(body.arguments[0], body.arguments[1]).result, + ).result + ] + ) + return output.result + + +def layer_norm(node, symbols): + value = symbols[(node.args[0], 0)] + weight = symbols[(node.args[2], 0)] + bias = symbols[(node.args[3], 0)] + shape = list(ir.RankedTensorType(value.type).shape) + rank = len(shape) + normalized = len(node.args[1]) + axes = set(range(rank - normalized, rank)) + f32, f64 = ir.F32Type.get(), ir.F64Type.get() + mean_value = _mean_f64(value, axes) + dims = [ir.AffineExpr.get_dim(i) for i in range(rank)] + identity = ir.AffineMap.get_identity(rank) + small = ir.AffineMap.get( + rank, + 0, + [ + ir.AffineExpr.get_constant(0) if i in axes else dim + for i, dim in enumerate(dims) + ], + ) + parameters = ir.AffineMap.get(rank, 0, dims[rank - normalized :]) + squared = _generic( + [value, mean_value], + tensor.EmptyOp(shape, f64).result, + [identity, small, identity], + ["parallel"] * rank, + ) + body = ir.Block.create_at_start(squared.region, [f32, f64, f64]) + with ir.InsertionPoint(body): + difference = arith.SubFOp( + arith.ExtFOp(f64, body.arguments[0]).result, body.arguments[1] + ).result + linalg.YieldOp([arith.MulFOp(difference, difference).result]) + variance = _mean_f64(squared.result, axes) + stat_shape = list(ir.RankedTensorType(variance.type).shape) + stat_identity = ir.AffineMap.get_identity(rank) + inverse = _generic( + [variance], + tensor.EmptyOp(stat_shape, f64).result, + [stat_identity, stat_identity], + ["parallel"] * rank, + ) + body = ir.Block.create_at_start(inverse.region, [f64, f64]) + with ir.InsertionPoint(body): + adjusted = arith.AddFOp( + body.arguments[0], arith.ConstantOp(f64, float(node.args[4])).result + ).result + root = math.SqrtOp(adjusted).result + linalg.YieldOp([arith.DivFOp(arith.ConstantOp(f64, 1.0).result, root).result]) + output = _generic( + [value, mean_value, inverse.result, weight, bias], + tensor.EmptyOp(shape, f32).result, + [identity, small, small, parameters, parameters, identity], + ["parallel"] * rank, + ) + body = ir.Block.create_at_start(output.region, [f32, f64, f64, f32, f32, f32]) + with ir.InsertionPoint(body): + difference = arith.SubFOp( + arith.ExtFOp(f64, body.arguments[0]).result, body.arguments[1] + ).result + normalized_value = arith.MulFOp(difference, body.arguments[2]).result + gamma = arith.ExtFOp(f64, body.arguments[3]).result + beta = arith.ExtFOp(f64, body.arguments[4]).result + result = math.FmaOp(normalized_value, gamma, beta).result + linalg.YieldOp([arith.TruncFOp(f32, result).result]) + return output.result, _cast(mean_value, f32), _cast(inverse.result, f32) + + +def gelu(node, symbols): + value = symbols[(node.args[0], 0)] + ty = ir.RankedTensorType(value.type) + f32, f64 = ir.F32Type.get(), ir.F64Type.get() + identity = ir.AffineMap.get_identity(ty.rank) + op = _generic( + [value], + tensor.EmptyOp(ty.shape, f32).result, + [identity, identity], + ["parallel"] * ty.rank, + ) + body = ir.Block.create_at_start(op.region, [f32, f32]) + with ir.InsertionPoint(body): + x = arith.ExtFOp(f64, body.arguments[0]).result + scaled = arith.MulFOp( + x, arith.ConstantOp(f64, scalar_math.sqrt(0.5)).result + ).result + erf = math.ErfOp(scaled).result + factor = arith.AddFOp(arith.ConstantOp(f64, 1.0).result, erf).result + half = arith.MulFOp(x, arith.ConstantOp(f64, 0.5).result).result + result = arith.MulFOp(half, factor).result + linalg.YieldOp([arith.TruncFOp(f32, result).result]) + return op.result + + +def tanh(node, symbols): + value = symbols[(node.args[0], 0)] + ty = ir.RankedTensorType(value.type) + f32, f64 = ir.F32Type.get(), ir.F64Type.get() + identity = ir.AffineMap.get_identity(ty.rank) + op = _generic( + [value], + tensor.EmptyOp(ty.shape, f32).result, + [identity, identity], + ["parallel"] * ty.rank, + ) + body = ir.Block.create_at_start(op.region, [f32, f32]) + with ir.InsertionPoint(body): + extended = arith.ExtFOp(f64, body.arguments[0]).result + result = math.TanhOp(extended).result + linalg.YieldOp([arith.TruncFOp(f32, result).result]) + return op.result diff --git a/waste/stack/compiler/frontend/ops/tosa.py b/waste/stack/compiler/frontend/ops/tosa.py new file mode 100644 index 000000000..0ee3b87ba --- /dev/null +++ b/waste/stack/compiler/frontend/ops/tosa.py @@ -0,0 +1,2271 @@ +# ===- tosa.py ----------------------------------------------------------------- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ===--------------------------------------------------------------------------- +# +# The registry of mappings from Buddy Graph to MLIR tosa dialect operations. +# +# ===--------------------------------------------------------------------------- + +import array +from collections.abc import Sequence +import buddy_mlir.ir as ir +import numpy +from buddy_mlir.dialects import ( + affine, + arith, + bufferization, + linalg, + math, + memref, + scf, + tensor, + tosa, + vector, +) +from buddy_mlir.dialects import ( + complex as complex_dialect, +) +from buddy_mlir.ir import F32Type, IndexType +from buddy.compiler.graph import ( + AbsOp, + AdaptiveAvgPool1dOp, + AdaptiveAvgPool2dBackwardOp, + AdaptiveAvgPool2dOp, + AdaptiveAvgPool3dOp, + AdaptiveMaxPool1dOp, + AdaptiveMaxPool2dOp, + AddbmmOp, + AddCMulOp, + AddMMOp, + AddOp, + AddScalarOp, + AliasOp, + AllDimsOp, + AllOp, + AmaxOp, + AminOp, + AnyDimsOp, + AnyOp, + ArangeStartStepOp, + ArgMaxOp, + ArgMinOp, + AvgPool1dOp, + AvgPool2dBackwardOp, + AvgPool2dOp, + BaddbmmOp, + BatchMatmulOp, + BitwiseAndScalarOp, + BitwiseAndTensorOp, + BitwiseLeftShiftOp, + BitwiseNotOp, + BitwiseOrOp, + BitwiseOrScalarOp, + BitwiseRightShiftOp, + BitwiseXorOp, + BitwiseXorScalarOp, + CauchyOp, + CdistForwardOp, + CeilOp, + ClampMaxOp, + ClampMaxTensorOp, + ClampMinOp, + ClampMinTensorOp, + ClampOp, + ClampTensorOp, + CloneOp, + ConstantPadNdOp, + Conv2dOp, + ConvertElementTypeOp, + ConvolutionBackwardOp, + CopysignOp, + CummaxOp, + CumminOp, + DiagonalOp, + DigammaOp, + DivOp, + DivScalarModeOp, + DivScalarOp, + DivTensorModeOp, + EmbeddingBagOp, + EmbeddingOp, + EmptyStridedOp, + EqTensorOp, + ErfcOp, + ErfinvOp, + ErfOp, + ExpandOp, + ExpOp, + FillScalarOp, + FlashAttentionForCpuPrefillOp, + FlipOp, + FloorDivideOp, + FloorOp, + FmodOp, + FrexpOp, + FullLikeOp, + GeTensorOp, + GluOp, + GQAAttentionFusedOp, + GtOp, + GtTensorOp, + HardswishOp, + HypotOp, + I0Op, + IgammacOp, + IgammaOp, + Int4UnpackOp, + IotaOp, + IsInfOp, + IsNanOp, + LeOp, + LerpOp, + LeScalarOp, + LeTensorOp, + LgammaOp, + LocalScalarDenseOp, + LogicalAndOp, + LogicalNotOp, + LogicalOrOp, + LogicalXorOp, + LogOp, + LtScalarOp, + LtTensorOp, + MaskedFillOp, + MaskedScatterOp, + MaxDimOp, + MaximumOp, + MaxPool1dOp, + MaxPool2dOp, + MeanDefaultOp, + MeanOp, + MinDimOp, + MinimumOp, + MulOp, + NativeDropoutOp, + NativeGroupNormBackwardOp, + NativeGroupNormOp, + NativeLayerNormBackwardOp, + NegOp, + NeScalarOp, + NeTensorOp, + NextafterOp, + NormScalarOp, + NormScalarOptDimOp, + OnesLikeOp, + PermuteOp, + PowScalarOp, + PowTensorTensorOp, + ProdOp, + RandIntLowOp, + RandpermOp, + ReciprocalOp, + ReflectionPad1dOp, + ReflectionPad2dOp, + ReflectionPad3dOp, + ReluOp, + RemainderOp, + RepeatOp, + ReplicationPad2dOp, + ReplicationPad3dOp, + ReshapeOp, + ResizeOp, + RevOp, + RreluWithNoiseOp, + RsqrtOp, + ScaledDotProductFlashAttentionForCpuOp, + SelectOp, + SigmoidOp, + SignbitOp, + SignOp, + SiluOp, + SliceOp, + SoftplusOp, + SpecialAiryAiOp, + SpecialBesselJ0Op, + SpecialBesselJ1Op, + SpecialBesselY0Op, + SpecialBesselY1Op, + SpecialChebyshevPolynomialTOp, + SpecialChebyshevPolynomialUOp, + SpecialChebyshevPolynomialVOp, + SpecialChebyshevPolynomialWOp, + SpecialEntrOp, + SpecialErfcxOp, + SpecialHermitePolynomialHeOp, + SpecialHermitePolynomialHOp, + SpecialI0eOp, + SpecialI1eOp, + SpecialI1Op, + SpecialLaguerrePolynomialLOp, + SpecialLegendrePolynomialPOp, + SpecialLogNdtrOp, + SpecialModifiedBesselK0Op, + SpecialModifiedBesselK1Op, + SpecialNdtriOp, + SpecialNdtrOp, + SpecialScaledModifiedBesselK0Op, + SpecialScaledModifiedBesselK1Op, + SpecialShiftedChebyshevPolynomialTOp, + SpecialShiftedChebyshevPolynomialUOp, + SpecialShiftedChebyshevPolynomialVOp, + SpecialShiftedChebyshevPolynomialWOp, + SpecialSphericalBesselJ0Op, + SpecialZetaOp, + SplitWithSizesOp, + SqueezeDimOp, + SqueezeDimsOp, + SqueezeOp, + StackOp, + StdCorrectionOp, + StdDefaultOp, + StdDimOp, + SubOp, + SubScalarOp, + SumDefaultOp, + SumDimOp, + TanhOp, + TensorDType, + TileOp, + TOp, + TransposeOp, + UnbindOp, + UnfoldOp, + UniformOp, + UnsqueezeOp, + VarCorrectionOp, + VarDefaultOp, + VarDimOp, + VarMeanOp, + ViewDtypeOp, + WhereOp, + ZerosLikeOp, + ZerosOp, +) +from stack.compiler.frontend.layout.provenance import has_nhwc_layout +from stack.compiler.frontend.ops.expf import exp_f32 +from stack.compiler.frontend.ops.int8 import conv2d as int8_conv2d_op +from stack.compiler.frontend.ops.int8 import matmul as int8_matmul_op +from stack.compiler.frontend.ops.int8 import quantize_activation as int8_quantize_op +from buddy.compiler.ops.utils import * +from buddy.compiler.ops.tosa import ( + _build_reassociation_attr, + _create_mul_shift_operand, + _create_permutation_attr, + _create_shape_operand, + _create_zero_point_tensor, + _create_zero_tensor, + _gen_arith_binary_op, + _normalize_reduce_dims, + _reshape_or_extract_for_complex, +) + + +def _to_nhwc(val, channels, elem_ty, nchw_meta=None): + """Transpose NCHW->NHWC only when SSA is still NCHW. Fail if layout unknown.""" + shp = [int(x) for x in ir.RankedTensorType(val.type).shape] + if len(shp) != 4: + raise RuntimeError(f"expected rank-4 activation, got {shp}") + # Equal channel/spatial extents cannot identify the axis order by shape. + if shp[1] == channels and shp[3] == channels: + if has_nhwc_layout(val): + n, h, w, c = shp + return val, n, h, w, c + n, c, h, w = shp + val = tosa.TransposeOp( + ir.RankedTensorType.get([n, h, w, c], elem_ty), + val, + _create_permutation_attr([0, 2, 3, 1]), + ).result + return val, n, h, w, c + if nchw_meta is not None: + n, c, h, w = [int(x) for x in nchw_meta] + if shp == [n, c, h, w]: + out_ty = ir.RankedTensorType.get([n, h, w, c], elem_ty) + val = tosa.TransposeOp( + out_ty, val, _create_permutation_attr([0, 2, 3, 1]) + ).result + return val, n, h, w, c + if shp == [n, h, w, c]: + return val, n, h, w, c + raise RuntimeError( + f"cannot infer NCHW/NHWC for ssa={shp} meta_nchw={[n, c, h, w]}" + ) + if shp[3] == channels: + n, h, w, c = shp + return val, n, h, w, c + if shp[1] == channels: + n, c, h, w = shp + out_ty = ir.RankedTensorType.get([n, h, w, c], elem_ty) + val = tosa.TransposeOp( + out_ty, val, _create_permutation_attr([0, 2, 3, 1]) + ).result + return val, n, h, w, c + raise RuntimeError( + f"cannot infer NCHW/NHWC for shape {shp} with channels={channels}" + ) + + +def _nhwc_to_nchw(value: ir.Value) -> ir.Value: + shape = [int(x) for x in ir.RankedTensorType(value.type).shape] + if len(shape) != 4: + raise RuntimeError(f"_nhwc_to_nchw expects rank-4, got {shape}") + n, h, w, c = shape + elem = ir.RankedTensorType(value.type).element_type + out_ty = ir.RankedTensorType.get([n, c, h, w], elem) + return tosa.TransposeOp( + out_ty, value, _create_permutation_attr([0, 3, 1, 2]) + ).result + + +def _nchw_to_nhwc(value: ir.Value) -> ir.Value: + shape = [int(x) for x in ir.RankedTensorType(value.type).shape] + if len(shape) != 4: + raise RuntimeError(f"_nchw_to_nhwc expects rank-4, got {shape}") + n, c, h, w = shape + elem = ir.RankedTensorType(value.type).element_type + out_ty = ir.RankedTensorType.get([n, h, w, c], elem) + return tosa.TransposeOp( + out_ty, value, _create_permutation_attr([0, 2, 3, 1]) + ).result + + +def _split_via_tosa_slice( + input_tensor: ir.Value, + input_shape: list[int], + dim: int, + split_sizes: list[int], +) -> list[ir.Value]: + rank = len(input_shape) + elem = ir.RankedTensorType(input_tensor.type).element_type + results = [] + offset = 0 + for size in split_sizes: + start = [0] * rank + start[dim] = offset + out_shape = list(input_shape) + out_shape[dim] = size + out_ty = ir.RankedTensorType.get(out_shape, elem) + results.append( + tosa.SliceOp( + out_ty, + input_tensor, + _create_shape_operand(start), + _create_shape_operand(out_shape), + ).result + ) + offset += size + return results + + +def _align_binary_layout(input1, input2): + """Keep NHWC activations and align logical NCHW broadcast operands.""" + if not (has_nhwc_layout(input1) or has_nhwc_layout(input2)): + return input1, input2 + operands = [input1, input2] + for index, value in enumerate(operands): + if not isinstance(value, ir.Value) or has_nhwc_layout(value): + continue + shape = list(ir.RankedTensorType(value.type).shape) + shape = [1] * (4 - len(shape)) + shape + value = tosa.ReshapeOp(value, _create_shape_operand(shape)).result + if sum(size != 1 for size in shape) <= 1: + value = tosa.ReshapeOp( + value, + _create_shape_operand([shape[0], shape[2], shape[3], shape[1]]), + ).result + else: + value = _nchw_to_nhwc(value) + value.owner.attributes["buddy.layout"] = ir.StringAttr.get("nhwc") + operands[index] = value + return operands + + +def add_op(node: AddOp, symbol_table): + """ + Import tensor addition operation. + From buddy graph ir's `AddOp` operator to MLIR TOSA `add` operation. + """ + input1 = symbol_table.get((str(node.args[0]), 0), node.args[0]) + input2 = symbol_table.get((str(node.args[1]), 0), node.args[1]) + dtype = node.tensor_meta["dtype"] + mlir_dtype = mlir_element_type_get(dtype) + if isinstance(node.args[0], str) and isinstance(node.args[1], str): + input1_dtype = ir.RankedTensorType(input1.type).element_type + input2_dtype = ir.RankedTensorType(input2.type).element_type + if input1_dtype != mlir_dtype: + input1 = tosa.CastOp( + ir.RankedTensorType.get( + ir.RankedTensorType(input1.type).shape, + mlir_dtype, + ), + input1, + ).result + if input2_dtype != mlir_dtype: + input2 = tosa.CastOp( + ir.RankedTensorType.get( + ir.RankedTensorType(input2.type).shape, + mlir_dtype, + ), + input2, + ).result + input1, input2 = _align_binary_layout(input1, input2) + return _gen_arith_binary_op(input1, input2, tosa.AddOp) + + +def sub_op(node: SubOp, symbol_table): + """ + Import tensor subtraction operation. + From buddy graph ir's `SubOp` operator to MLIR TOSA `sub` operation. + """ + input1 = symbol_table.get((str(node.args[0]), 0), node.args[0]) + input2 = symbol_table.get((str(node.args[1]), 0), node.args[1]) + dtype = node.tensor_meta["dtype"] + mlir_dtype = mlir_element_type_get(dtype) + if isinstance(node.args[0], str) and isinstance(node.args[1], str): + input1_dtype = ir.RankedTensorType(input1.type).element_type + input2_dtype = ir.RankedTensorType(input2.type).element_type + if input1_dtype != mlir_dtype: + input1 = tosa.CastOp( + ir.RankedTensorType.get( + ir.RankedTensorType(input1.type).shape, + mlir_dtype, + ), + input1, + ).result + if input2_dtype != mlir_dtype: + input2 = tosa.CastOp( + ir.RankedTensorType.get( + ir.RankedTensorType(input2.type).shape, + mlir_dtype, + ), + input2, + ).result + input1, input2 = _align_binary_layout(input1, input2) + return _gen_arith_binary_op(input1, input2, tosa.SubOp) + + +def mul_op(node: MulOp, symbol_table): + """ + Import tensor division operation. + From buddy graph ir's `DivOp` operator to MLIR TOSA `div` operation. + """ + + def _inner_op(result_type, input1, input2): + shift = _create_mul_shift_operand() + return tosa.MulOp(result_type, input1, input2, shift) + + output_shape = list(node.tensor_meta["shape"]) + dtype = node.tensor_meta["dtype"] + mlir_dtype = mlir_element_type_get(dtype) + + if isinstance(node.args[0], str): + input1 = symbol_table.get((str(node.args[0]), 0), node.args[0]) + else: + data = [node.args[0]] + input1_shape = numpy.array(data).shape + tensor_type = ir.RankedTensorType.get(input1_shape, mlir_dtype) + element = mlir_element_attr_get(dtype, node.args[0]) + attr = ir.DenseElementsAttr.get_splat(tensor_type, element) + input2 = arith.ConstantOp(tensor_type, attr).result + + if isinstance(node.args[1], str): + input2 = symbol_table.get((str(node.args[1]), 0), node.args[1]) + else: + data = [node.args[1]] + input2_shape = numpy.array(data).shape + tensor_type = ir.RankedTensorType.get(input2_shape, mlir_dtype) + element = mlir_element_attr_get(dtype, node.args[1]) + attr = ir.DenseElementsAttr.get_splat(tensor_type, element) + input2 = arith.ConstantOp(tensor_type, attr).result + + input1_dtype = ir.RankedTensorType(input1.type).element_type + input2_dtype = ir.RankedTensorType(input2.type).element_type + if input1_dtype != mlir_dtype: + input1 = tosa.CastOp( + ir.RankedTensorType.get( + ir.RankedTensorType(input1.type).shape, + mlir_dtype, + ), + input1, + ).result + if input2_dtype != mlir_dtype: + input2 = tosa.CastOp( + ir.RankedTensorType.get( + ir.RankedTensorType(input2.type).shape, + mlir_dtype, + ), + input2, + ).result + + input1, input2 = _align_binary_layout(input1, input2) + return _gen_arith_binary_op(input1, input2, _inner_op) + + +def div_op(node: DivOp, symbol_table): + """ + Import tensor division operation. + From buddy graph ir's `DivOp` operator to MLIR TOSA `div` operation. + """ + + def _inner_op(result_type, input1, input2): + # Preserve true division. Reciprocal followed by multiplication can + # cross round-to-even boundaries in activation quantization. + shape = list(result_type.shape) + rank = len(shape) + maps = [] + for value in (input1, input2): + dims = [ + ( + ir.AffineExpr.get_constant(0) + if extent == 1 + else ir.AffineExpr.get_dim(i) + ) + for i, extent in enumerate(ir.RankedTensorType(value.type).shape) + ] + maps.append(ir.AffineMapAttr.get(ir.AffineMap.get(rank, 0, dims))) + maps.append(ir.AffineMapAttr.get(ir.AffineMap.get_identity(rank))) + empty = tensor.EmptyOp(shape, result_type.element_type) + division = linalg.GenericOp( + [result_type], + [input1, input2], + [empty.result], + ir.ArrayAttr.get(maps), + ir.ArrayAttr.get( + [ir.Attribute.parse("#linalg.iterator_type")] * rank + ), + ) + block = ir.Block.create_at_start( + division.region, [result_type.element_type] * 3 + ) + with ir.InsertionPoint(block): + value = arith.DivFOp(block.arguments[0], block.arguments[1]).result + linalg.YieldOp([value]) + if has_nhwc_layout(input1) or has_nhwc_layout(input2): + division.operation.attributes["buddy.layout"] = ir.StringAttr.get("nhwc") + return division + + input1 = symbol_table.get((str(node.args[0]), 0), node.args[0]) + input2 = symbol_table.get((str(node.args[1]), 0), node.args[1]) + + # PyTorch integer division returns float32. Cast any integer tensor inputs to + # float32 so tosa.reciprocal is legalizable and the result type matches Torch + # semantics (true division). + def _is_int_tensor(value: object) -> bool: + if not isinstance(value, ir.Value): + return False + element_type = ir.RankedTensorType(value.type).element_type + return isinstance(element_type, ir.IntegerType) + + if _is_int_tensor(input1) or _is_int_tensor(input2): + f32 = ir.F32Type.get() + if isinstance(input1, ir.Value): + in1_ty = ir.RankedTensorType.get( + list(ir.RankedTensorType(input1.type).shape), f32 + ) + input1 = tosa.CastOp(in1_ty, input1).result + elif isinstance(input1, int): + input1 = float(input1) + if isinstance(input2, ir.Value): + in2_ty = ir.RankedTensorType.get( + list(ir.RankedTensorType(input2.type).shape), f32 + ) + input2 = tosa.CastOp(in2_ty, input2).result + elif isinstance(input2, int): + input2 = float(input2) + + input1, input2 = _align_binary_layout(input1, input2) + return _gen_arith_binary_op(input1, input2, _inner_op) + + +def amax_op(node: AmaxOp, symbol_table): + """ + Import the amax operation. + From buddy graph ir's `AmaxOp` operator to MLIR TOSA `reduce_max` + operation. + """ + input1 = symbol_table.get((str(node.args[0]), 0)) + dim_arg = node.args[1] if len(node.args) > 1 else None + keepdim = node.args[2] if len(node.args) > 2 else False + rank = len(ir.RankedTensorType(input1.type).shape) + dims = _normalize_reduce_dims(dim_arg, rank) + input_nhwc = has_nhwc_layout(input1) + if input_nhwc: + dims = [[0, 3, 1, 2][axis] for axis in dims] + + result = input1 + for dim_val in dims: + dim_attr = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), dim_val) + result = tosa.ReduceMaxOp(result, dim_attr).result + + if input_nhwc: + shape = list(ir.RankedTensorType(result.type).shape) + if sum(size != 1 for size in shape) <= 1: + logical_shape = [shape[0], shape[3], shape[1], shape[2]] + result = tosa.ReshapeOp(result, _create_shape_operand(logical_shape)).result + else: + result = _nhwc_to_nchw(result) + if not keepdim: + output_shape = list(node.tensor_meta["shape"]) + reshape_operand = _create_shape_operand(output_shape) + result = tosa.ReshapeOp(result, reshape_operand).result + + return result + + +def avg_pool2d_op(node: AvgPool2dOp, symbol_table): + """ + Import the avg_pool2d operation. + From Buddy AvgPool2dOp to MLIR TOSA `avg_pool2d` operation. + """ + input1 = symbol_table.get((str(node.args[0]), 0)) + kernel = node.args[1] + stride = node.args[2] + if len(node.args) > 3: + pad = node.args[3] + else: + pad = [0 for _ in kernel] + dtype = node.tensor_meta["dtype"] + result_element_type = mlir_element_type_get(dtype) + acc_type = ir.TypeAttr.get(result_element_type) + + meta = list(node.tensor_meta["shape"]) + channels = int(meta[1]) + src = node.args[0] + nchw_meta = ( + list(src.tensor_meta["shape"]) + if hasattr(src, "tensor_meta") and "shape" in src.tensor_meta + else None + ) + input1, N, H, W, C = _to_nhwc(input1, channels, result_element_type, nchw_meta) + input_shape_nhwc = [N, H, W, C] + + # Expand pad to 4 elements if needed + if len(pad) == 1: + pad = [pad[0]] * 4 + elif len(pad) == 2: + pad = [pad[0]] * 2 + [pad[1]] * 2 + + # Calculate output shape in NHWC format + # Formula: output = (input + pad_top + pad_bottom - kernel) / stride + 1 + pad_top, pad_bottom, pad_left, pad_right = pad + output_h = (H + pad_top + pad_bottom - kernel[0]) // stride[0] + 1 + output_w = (W + pad_left + pad_right - kernel[1]) // stride[1] + 1 + output_shape_nhwc = [N, output_h, output_w, C] + + kernel_attr = ir._denseI64ArrayAttr(kernel, None) + stride_attr = ir._denseI64ArrayAttr(stride, None) + pad_attr = ir._denseI64ArrayAttr(pad, None) + + output = ir.RankedTensorType.get(output_shape_nhwc, result_element_type) + input_zp = _create_zero_point_tensor(input1) + output_zp = _create_zero_point_tensor(input1) + op = tosa.AvgPool2dOp( + output, + input1, + input_zp, + output_zp, + kernel=kernel_attr, + stride=stride_attr, + pad=pad_attr, + acc_type=acc_type, + ) + return op + + +def adaptive_avg_pool2d_op(node: AdaptiveAvgPool2dOp, symbol_table): + """ + Import the adaptive_avg_pool2d operation. + From buddy graph ir's `AdaptiveAvgPool2dOp` operator to MLIR operations. + Uses TOSA avg_pool2d with calculated kernel/stride. + """ + input1 = symbol_table.get((str(node.args[0]), 0)) + input_is_nhwc = has_nhwc_layout(input1) + output_size = node.args[1] + if isinstance(output_size, int): + output_size = [output_size, output_size] + + dtype = node.tensor_meta["dtype"] + result_element_type = mlir_element_type_get(dtype) + acc_type = ir.TypeAttr.get(result_element_type) + + # Get input shape; sticky NHWC after conv keeps channels last. + channels = int(list(node.tensor_meta["shape"])[1]) + src = node.args[0] + nchw_meta = ( + list(src.tensor_meta["shape"]) + if hasattr(src, "tensor_meta") and "shape" in src.tensor_meta + else None + ) + input1, N, H, W, C = _to_nhwc(input1, channels, result_element_type, nchw_meta) + out_h, out_w = output_size + + # Calculate kernel and stride for adaptive pooling + kernel_h = (H + out_h - 1) // out_h + kernel_w = (W + out_w - 1) // out_w + stride_h = H // out_h + stride_w = W // out_w + + # Apply avg_pool2d + kernel_attr = ir._denseI64ArrayAttr([kernel_h, kernel_w], None) + stride_attr = ir._denseI64ArrayAttr([stride_h, stride_w], None) + pad_attr = ir._denseI64ArrayAttr([0, 0, 0, 0], None) + + pool_nhwc_shape = [N, out_h, out_w, C] + pool_type = ir.RankedTensorType.get(pool_nhwc_shape, result_element_type) + input_zp = _create_zero_point_tensor(input1) + output_zp = _create_zero_point_tensor(input1) + pooled = tosa.AvgPool2dOp( + pool_type, + input1, + input_zp, + output_zp, + kernel=kernel_attr, + stride=stride_attr, + pad=pad_attr, + acc_type=acc_type, + ) + if input_is_nhwc: + return pooled + return tosa.TransposeOp( + ir.RankedTensorType.get([N, C, out_h, out_w], result_element_type), + pooled.result, + _create_permutation_attr([0, 3, 1, 2]), + ) + + +def _reshape_input_nchw_meta(node, symbol_table): + """PyTorch NCHW shape of reshape/view input. args[0] is often a name str.""" + src = node.args[0] + if hasattr(src, "tensor_meta") and "shape" in src.tensor_meta: + return [int(x) for x in src.tensor_meta["shape"]] + if isinstance(src, str): + ops = symbol_table.get(("__buddy_ops_by_name__", 0)) + if ops is None: + return None + pred = ops.get(src) + if pred is None or not ( + hasattr(pred, "tensor_meta") and "shape" in pred.tensor_meta + ): + return None + return [int(x) for x in pred.tensor_meta["shape"]] + return None + + +def reshape_op(node: ReshapeOp, symbol_table): + """ + Import the reshape operation. + From buddy graph ir's `ReshapeOp` operator to MLIR TOSA `reshape` + operation. + + Note: If the new shape contains one and only one `-1`, the size of the new + shape will be inferred automatically. + """ + input1 = symbol_table.get((str(node.args[0]), 0)) + new_shape = [] + if node._newshape is None: + shape_arg = node.args[1] + + if isinstance(shape_arg, (list, tuple)): + new_shape = list(shape_arg) + else: + try: + new_shape = list(shape_arg) + except TypeError: + new_shape = [shape_arg] + else: + new_shape = list(node._newshape) + + now_shape = ir.RankedTensorType(input1.type).shape + total_size = 1 + for dim_siz in now_shape: + total_size *= dim_siz + + neg_one_cnt = 0 + rest_size = 1 + for dim_siz in new_shape: + if dim_siz == -1: + neg_one_cnt += 1 + continue + rest_size *= dim_siz + + if neg_one_cnt != 0: + if neg_one_cnt > 1 or total_size % rest_size != 0: + raise ValueError("Can not infer the new shape!") + infer_dim_size = total_size // rest_size + for i, _ in enumerate(new_shape): + if new_shape[i] == -1: + new_shape[i] = infer_dim_size + + producer = input1.owner + if isinstance(producer, ir.OpView): + producer = producer.operation + if len(now_shape) == 4 and isinstance(producer, ir.Operation): + attrs = producer.attributes + if ( + ( + "buckyball.mega_conv2d" in attrs + or "buckyball.mega_conv2d_depthwise" in attrs + ) + and "final_output" in attrs + and ir.BoolAttr(attrs["final_output"]).value + and ir.RankedTensorType(input1.type).element_type == ir.F32Type.get() + ): + return _reshape_or_extract_for_complex(input1, new_shape) + + if len(now_shape) == 4 and has_nhwc_layout(input1): + input_meta = _reshape_input_nchw_meta(node, symbol_table) + if input_meta is None or len(input_meta) != 4: + raise RuntimeError("reshape of NHWC activation requires Torch input shape") + n, c, h, w = input_meta + if list(now_shape) != [n, h, w, c]: + raise RuntimeError( + f"reshape NHWC layout mismatch: ssa={list(now_shape)} torch={input_meta}" + ) + element_type = ir.RankedTensorType(input1.type).element_type + input1 = tosa.TransposeOp( + ir.RankedTensorType.get(input_meta, element_type), + input1, + _create_permutation_attr([0, 3, 1, 2]), + ).result + + return _reshape_or_extract_for_complex(input1, new_shape) + + +def maxpool2d_op(node: MaxPool2dOp, symbol_table): + """ + Import the maxpool2d operation. + From Buddy MaxPool2dOp to MLIR TOSA `max_pool2d` operation. + """ + if len(node.args) == 5: + raise NotImplementedError + input1 = symbol_table.get((str(node.args[0]), 0)) + kernel = node.args[1] + stride = node.args[2] + if len(node.args) > 3: + pad = node.args[3] + else: + pad = [0 for _ in kernel] + dtype = node.tensor_meta["dtype"] + result_element_type = mlir_element_type_get(dtype) + + original_shape = list(ir.RankedTensorType(input1.type).shape) + channels = int(list(node.tensor_meta["shape"])[1]) + src = node.args[0] + nchw_meta = ( + list(src.tensor_meta["shape"]) + if hasattr(src, "tensor_meta") and "shape" in src.tensor_meta + else None + ) + input1, n, h, w, c = _to_nhwc(input1, channels, result_element_type, nchw_meta) + in_n, in_c, in_h, in_w = n, c, h, w + + out_shape = node.tensor_meta["shape"] + if len(pad) == 1: + pad = [pad[0]] * 4 + elif len(pad) == 2: + pad = [pad[0]] * 2 + [pad[1]] * 2 + + k_h, k_w = kernel + s_h, s_w = stride + pt, pb, pl, pr = pad + # Preserve the leading padding. TOSA requires an exact output extent; + # adjust only the trailing padding to match Torch floor/ceil semantics. + out_h, out_w = node.tensor_meta["shape"][2:] + pb = max(0, (out_h - 1) * s_h + k_h - in_h - pt) + pr = max(0, (out_w - 1) * s_w + k_w - in_w - pl) + + kernel_attr = ir._denseI64ArrayAttr(kernel, None) + stride_attr = ir._denseI64ArrayAttr(stride, None) + pad_attr = ir._denseI64ArrayAttr([pt, pb, pl, pr], None) + out_h = (in_h + pt + pb - k_h) // s_h + 1 + out_w = (in_w + pl + pr - k_w) // s_w + 1 + out_shape_nhwc = [in_n, out_h, out_w, in_c] + output = ir.RankedTensorType.get(out_shape_nhwc, result_element_type) + op = tosa.MaxPool2dOp(output, input1, kernel_attr, stride_attr, pad_attr) + return op + + +def convolution2d_op(node: Conv2dOp, symbol_table): + """ + Import the convolution operation. + From Buddy Conv2dOp to MLIR TOSA `conv2d` operation. + arg[0]: Tensor input + arg[1]: Tensor weight + arg[2]: Tensor? bias + arg[3]: SymInt[] stride + arg[4]: SymInt[] padding + arg[5]: SymInt[] dilation + arg[6]: bool transposed + arg[7]: SymInt[] output_padding + arg[8]: SymInt groups + """ + # Get arguments from convolution node. + assert len(node.args) == 9 + input = node.args[0] + weight = node.args[1] + bias = node.args[2] + stride = node.args[3] + input_padding = node.args[4] + dilation = node.args[5] + is_kernel_transposed = node.args[6] + out_padding = node.args[7] + groups = node.args[8] + + # Prepare input, weight, and output information. + input_val = symbol_table.get((str(input), 0)) + input_shape = list(ir.RankedTensorType(input_val.type).shape) + weight_val = symbol_table.get((str(weight), 0)) + weight_shape = ir.RankedTensorType(weight_val.type).shape + dtype = node.tensor_meta["dtype"] + result_element_type = mlir_element_type_get(dtype) + out_shape = node.tensor_meta["shape"] + acc_type = ir.TypeAttr.get(result_element_type) + + # Prepare Depthwise Conv2D information + is_grouped = (list(weight_shape)[1] == 1) and (groups != 1) + is_depthwise = (groups == list(weight_shape)[0]) and is_grouped + + # Prepare input channel and output channel. + if is_kernel_transposed: + in_channels = list(weight_shape)[0] + out_channels = list(weight_shape)[1] * groups + else: + in_channels = list(weight_shape)[1] * groups + out_channels = list(weight_shape)[0] + + # Prepare bias tensor. + if len(node._parents) == 2: + new_size_tensor_type = ir.RankedTensorType.get( + [out_channels], result_element_type + ) + element = mlir_element_attr_get(dtype, 0) + new_size_attr = ir.DenseElementsAttr.get_splat(new_size_tensor_type, element) + bias_tensor = tosa.ConstOp(new_size_attr).results[0] + else: + bias_tensor = symbol_table.get((str(bias), 0)) + + # Prepare attributes. + dilation_attr = ir._denseI64ArrayAttr(dilation, None) + stride_attr = ir._denseI64ArrayAttr(stride, None) + + # Convolution 2D + if len(weight_shape) == 4: + # Prepare input padding. + if len(input_padding) == 1: + input_padding = [input_padding[0]] * 4 + elif len(input_padding) == 2: + input_padding = [input_padding[0]] * 2 + [input_padding[1]] * 2 + + t, b, l, r = input_padding + sy, sx = int(stride[0]), int(stride[1]) + dy, dx = int(dilation[0]), int(dilation[1]) + kernel_h = int(list(weight_shape)[2]) + kernel_w = int(list(weight_shape)[3]) + + extra_out_h = 0 + extra_out_w = 0 + src = node.args[0] + nchw_meta = ( + list(src.tensor_meta["shape"]) + if hasattr(src, "tensor_meta") and "shape" in src.tensor_meta + else None + ) + input_val, _, input_h, input_w, _ = _to_nhwc( + input_val, in_channels, result_element_type, nchw_meta + ) + input_h = int(input_h) + input_w = int(input_w) + + def _adjust_padding(input_size, kernel, dil, stride_val, pad0, pad1): + """Ensure stride divisibility by adding pad to the second side.""" + base = input_size - 1 - (kernel - 1) * dil + total = base + pad0 + pad1 + remainder = total % stride_val + if remainder == 0: + return pad0, pad1, 0 + pad_needed = stride_val - remainder + pad1 += pad_needed + return pad0, pad1, 1 + + if not is_kernel_transposed: + t, b, extra_h = _adjust_padding(input_h, kernel_h, dy, sy, int(t), int(b)) + l, r, extra_w = _adjust_padding(input_w, kernel_w, dx, sx, int(l), int(r)) + extra_out_h += extra_h + extra_out_w += extra_w + input_padding = [t, b, l, r] + + # Prepare input_padding attributes. + input_padding_attr = ir._denseI64ArrayAttr(input_padding, None) + # TOSA conv2d is NHWC; keep output in NHWC (no convert-back). + out_shape = [ + out_shape[0], + out_shape[2], + out_shape[3], + out_shape[1], + ] + conv_out_shape = list(out_shape) + if extra_out_h: + conv_out_shape[1] += extra_out_h + if extra_out_w: + conv_out_shape[2] += extra_out_w + output_type = ir.RankedTensorType.get(conv_out_shape, result_element_type) + + if ( + str(ir.RankedTensorType(weight_val.type).element_type) == "i8" + and not is_depthwise + ): + if is_kernel_transposed: + raise ValueError("quantized transpose convolution is unsupported") + if stride[0] != stride[1]: + raise ValueError("quantized convolution requires equal strides") + pad_constant = _create_shape_operand( + [0, 0, int(t), int(b), int(l), int(r), 0, 0] + ) + pad_value = _create_zero_point_tensor(input_val) + padded_type = ir.RankedTensorType.get( + [ + conv_out_shape[0], + input_h + int(t) + int(b), + input_w + int(l) + int(r), + in_channels, + ], + result_element_type, + ) + padded_input = tosa.PadOp( + padded_type, input_val, pad_constant, pad_value + ).result + quant_weight = weight_val + if node._layout.find("FCHW") != -1: + perm_shape = [ + weight_shape[0], + weight_shape[2], + weight_shape[3], + weight_shape[1], + ] + quant_weight = tosa.TransposeOp( + ir.RankedTensorType.get(perm_shape, ir.IntegerType.get_signless(8)), + weight_val, + _create_permutation_attr([0, 2, 3, 1]), + ).result + bias_output = tensor.EmptyOp(conv_out_shape, result_element_type) + generic_map = ir.AffineMap.get_permutation([0, 1, 2, 3]) + bias_init = linalg.GenericOp( + [output_type], + [bias_tensor], + [bias_output], + ir.ArrayAttr.get( + [ + ir.AffineMapAttr.get(generic_map.get_submap([3])), + ir.AffineMapAttr.get(generic_map), + ] + ), + ir.ArrayAttr.get( + [ir.Attribute.parse("#linalg.iterator_type")] * 4 + ), + ) + bias_block = ir.Block.create_at_start( + bias_init.region, [result_element_type, result_element_type] + ) + bias_init.region.blocks[0].append(linalg.YieldOp([bias_block.arguments[0]])) + quant_conv = linalg.Conv2DNhwcFhwcOp( + [output_type], + [padded_input, quant_weight], + [bias_init.result], + strides=stride_attr, + dilations=dilation_attr, + ) + linalg.fill_builtin_region(quant_conv.operation) + i64 = ir.IntegerType.get_signless(64) + quant_conv.operation.attributes["dw_addr"] = ir.IntegerAttr.get( + i64, node._dw_addr + ) + quant_conv.operation.attributes["dw_bytes"] = ir.IntegerAttr.get( + i64, node._dw_bytes + ) + quant_conv.operation.attributes["per_channel"] = ir.BoolAttr.get( + node._per_channel + ) + quant_conv.operation.attributes["strides"] = ir.DenseElementsAttr.get( + numpy.array([int(stride[0]), int(stride[1])], dtype=numpy.int64), + type=ir.VectorType.get([2], i64), + ) + op = quant_conv + if extra_out_h or extra_out_w: + slice_start = _create_shape_operand([0, 0, 0, 0]) + slice_size = _create_shape_operand(out_shape) + slice_type = ir.RankedTensorType.get(out_shape, result_element_type) + op = tosa.SliceOp(slice_type, op.result, slice_start, slice_size) + return op + + # Depthwise Conv2D Operation. + if is_depthwise is True: + # If groups == in_channels,out_channels == in_channels + weight_depthwise = weight_val + if node._layout.find("FCHW") != -1: + perm_list = [2, 3, 0, 1] + perms_attr = _create_permutation_attr(perm_list) + perm_shape = [] + perm_shape.append(weight_shape[2]) + perm_shape.append(weight_shape[3]) + perm_shape.append(weight_shape[0]) + perm_shape.append(weight_shape[1]) + permute_result_type = ir.RankedTensorType.get( + perm_shape, + ir.RankedTensorType(weight_val.type).element_type, + ) + weight_depthwise = tosa.TransposeOp( + permute_result_type, weight_val, perms_attr + ).result + if str(ir.RankedTensorType(weight_depthwise.type).element_type) == "i8": + pad_constant = _create_shape_operand( + [0, 0, int(t), int(b), int(l), int(r), 0, 0] + ) + pad_value = _create_zero_point_tensor(input_val) + padded_shape = [ + conv_out_shape[0], + input_h + int(t) + int(b), + input_w + int(l) + int(r), + conv_out_shape[3], + ] + padded_type = ir.RankedTensorType.get(padded_shape, result_element_type) + input_val = tosa.PadOp( + padded_type, input_val, pad_constant, pad_value + ).result + depth_shape = [ + conv_out_shape[0], + conv_out_shape[1], + conv_out_shape[2], + conv_out_shape[3], + 1, + ] + depth_type = ir.RankedTensorType.get(depth_shape, result_element_type) + depth_init = tensor.EmptyOp(depth_shape, result_element_type) + identity = ir.AffineMap.get_permutation([0, 1, 2, 3, 4]) + bias_init = linalg.GenericOp( + [depth_type], + [bias_tensor], + [depth_init], + ir.ArrayAttr.get( + [ + ir.AffineMapAttr.get(identity.get_submap([3])), + ir.AffineMapAttr.get(identity), + ] + ), + ir.ArrayAttr.get( + [ir.Attribute.parse("#linalg.iterator_type")] * 5 + ), + ) + bias_block = ir.Block.create_at_start( + bias_init.region, [result_element_type, result_element_type] + ) + bias_init.region.blocks[0].append( + linalg.YieldOp([bias_block.arguments[0]]) + ) + depthwise_op = linalg.DepthwiseConv2DNhwcHwcmOp( + [depth_type], + [input_val, weight_depthwise], + [bias_init.result], + strides=stride_attr, + dilations=dilation_attr, + ) + linalg.fill_builtin_region(depthwise_op.operation) + depthwise_op.operation.attributes["dw_addr"] = ir.IntegerAttr.get( + ir.IntegerType.get_signless(64), node._dw_addr + ) + depthwise_op.operation.attributes["dw_bytes"] = ir.IntegerAttr.get( + ir.IntegerType.get_signless(64), node._dw_bytes + ) + depthwise_op.operation.attributes["per_channel"] = ir.BoolAttr.get( + node._per_channel + ) + collapse = tensor.CollapseShapeOp( + output_type, + depthwise_op.result, + _build_reassociation_attr([[0], [1], [2], [3, 4]]), + ).result + if extra_out_h or extra_out_w: + slice_start = _create_shape_operand([0, 0, 0, 0]) + slice_size = _create_shape_operand(out_shape) + slice_type = ir.RankedTensorType.get(out_shape, result_element_type) + return tosa.SliceOp(slice_type, collapse, slice_start, slice_size) + return collapse + input_zp = _create_zero_point_tensor(input_val) + weight_zp = _create_zero_point_tensor(weight_depthwise) + depthwise_op = tosa.DepthwiseConv2DOp( + output_type, + input_val, + weight_depthwise, + bias_tensor, + input_zp, + weight_zp, + input_padding_attr, + stride_attr, + dilation_attr, + acc_type, + ) + if hasattr(node, "_dw_addr"): + i64 = ir.IntegerType.get_signless(64) + depthwise_op.operation.attributes["dw_addr"] = ir.IntegerAttr.get( + i64, node._dw_addr + ) + depthwise_op.operation.attributes["dw_bytes"] = ir.IntegerAttr.get( + i64, node._dw_bytes + ) + depthwise_op.operation.attributes["per_channel"] = ir.BoolAttr.get( + node._per_channel + ) + op = depthwise_op + else: + # Transpose Conv2D Operation. + if is_kernel_transposed: + if sum(input_padding) > 0 or sum(dilation) > len(dilation): + raise NotImplementedError + for i in range(len(out_padding), 4): + out_padding = [0] + out_padding + out_padding_attr = ir._denseI64ArrayAttr(out_padding, None) + input_zp = _create_zero_point_tensor(input_val) + weight_zp = _create_zero_point_tensor(weight_val) + op = tosa.TransposeConv2DOp( + output_type, + input_val, + weight_val, + bias_tensor, + input_zp, + weight_zp, + out_padding_attr, + stride_attr, + acc_type, + ) + # Generic Conv2D Operation. + else: + if node._layout.find("FCHW") != -1: + perm_list = [0, 2, 3, 1] + perms_attr = _create_permutation_attr(perm_list) + perm_shape = [] + perm_shape.append(weight_shape[0]) + perm_shape.append(weight_shape[2]) + perm_shape.append(weight_shape[3]) + perm_shape.append(weight_shape[1]) + permute_result_type = ir.RankedTensorType.get( + perm_shape, + ir.RankedTensorType(weight_val.type).element_type, + ) + weight_val = tosa.TransposeOp( + permute_result_type, + weight_val, + perms_attr, + ).result + input_zp = _create_zero_point_tensor(input_val) + weight_zp = _create_zero_point_tensor(weight_val) + conv_op = tosa.Conv2DOp( + output_type, + input_val, + weight_val, + bias_tensor, + input_zp, + weight_zp, + input_padding_attr, + stride_attr, + dilation_attr, + acc_type, + ) + if hasattr(node, "_dw_addr"): + i64 = ir.IntegerType.get_signless(64) + conv_op.operation.attributes["dw_addr"] = ir.IntegerAttr.get( + i64, node._dw_addr + ) + conv_op.operation.attributes["dw_bytes"] = ir.IntegerAttr.get( + i64, node._dw_bytes + ) + conv_op.operation.attributes["per_channel"] = ir.BoolAttr.get( + node._per_channel + ) + op = conv_op + + if extra_out_h or extra_out_w: + # Slice off the padded rows/cols that exceed the expected output. + slice_start = _create_shape_operand([0, 0, 0, 0]) + slice_size = _create_shape_operand(out_shape) + slice_type = ir.RankedTensorType.get(out_shape, result_element_type) + slice_op = tosa.SliceOp(slice_type, op.result, slice_start, slice_size) + op = slice_op + # Convolution 1D + elif len(weight_shape) == 3: + # Prepare input with padding. + if input_padding[0] != 0: + input_shape = list(ir.RankedTensorType(input_val.type).shape) + padded_type = ir.RankedTensorType.get( + [ + input_shape[0], + input_shape[1], + input_shape[2] + 2 * input_padding[0], + ], + result_element_type, + ) + + pad_constant = _create_shape_operand( + [0, 0, 0, 0, input_padding[0], input_padding[0]] + ) + ty = ir.Type.parse("tensor<1xf32>") + pad_zp = tosa.ConstOp( + ir.DenseElementsAttr.get_splat(ty, ir.FloatAttr.get_f32(0.0)) + ).result + input_val = tosa.PadOp(padded_type, input_val, pad_constant, pad_zp) + output_type = ir.RankedTensorType.get(out_shape, result_element_type) + output_conv = _create_zero_tensor(output_type) + assert groups == 1, "only support one group" + # Con1D Operation Without Bias + conv_op = linalg.conv_1d_ncw_fcw( + input_val, + weight_val, + outs=[output_conv], + strides=stride_attr, + dilations=dilation_attr, + ) + output = tensor.EmptyOp(list(out_shape), result_element_type) + generic_map = ir.AffineMap.get_permutation( + [i for i in range(len(list(out_shape)))] + ) + loop_type = [ir.Attribute.parse("#linalg.iterator_type")] * len( + list(out_shape) + ) + loop_type[1] = ir.Attribute.parse("#linalg.iterator_type") + # Add Bias To Conv2d. + op = linalg.GenericOp( + [output_type], + [conv_op, bias_tensor], + [output], + ir.ArrayAttr.get( + [ + ir.AffineMapAttr.get( + generic_map.get_submap([i for i in range(len(list(out_shape)))]) + ), + ir.AffineMapAttr.get(generic_map.get_submap([1])), + ir.AffineMapAttr.get( + generic_map.get_submap([i for i in range(len(list(out_shape)))]) + ), + ] + ), + ir.ArrayAttr.get(loop_type), + ) + block = ir.Block.create_at_start( + op.region, + [ + result_element_type, + ir.RankedTensorType(bias_tensor.type).element_type, + result_element_type, + ], + ) + add_op = arith.AddFOp(block.arguments[1], block.arguments[0]) + block.append(add_op) + block.append(linalg.YieldOp([add_op.result])) + + return op + + +def silu_op(node: SiluOp, symbol_table): + """Preserve x / (1 + exp(-x)) rounding before activation quantization.""" + value = symbol_table[(str(node.args[0]), 0)] + if has_nhwc_layout(value): + value = _nhwc_to_nchw(value) + result_type = ir.RankedTensorType(value.type) + shape = list(result_type.shape) + dtype = result_type.element_type + identity = ir.AffineMapAttr.get(ir.AffineMap.get_identity(len(shape))) + empty = tensor.EmptyOp(shape, dtype) + op = linalg.GenericOp( + [result_type], + [value], + [empty.result], + ir.ArrayAttr.get([identity, identity]), + ir.ArrayAttr.get( + [ir.Attribute.parse("#linalg.iterator_type")] * len(shape) + ), + ) + block = ir.Block.create_at_start(op.region, [dtype, dtype]) + with ir.InsertionPoint(block): + argument = block.arguments[0] + opmath = ( + ir.F32Type.get() + if dtype in (ir.F16Type.get(), ir.BF16Type.get()) + else dtype + ) + if opmath != dtype: + argument = arith.ExtFOp(opmath, argument).result + negative = arith.NegFOp(argument).result + exponential = ( + exp_f32(negative) + if opmath == ir.F32Type.get() + else math.ExpOp(negative).result + ) + one = arith.ConstantOp(opmath, ir.FloatAttr.get(opmath, 1.0)).result + denominator = arith.AddFOp(one, exponential).result + result = arith.DivFOp(argument, denominator).result + if opmath != dtype: + result = arith.TruncFOp(dtype, result).result + linalg.YieldOp([result]) + return op + + +def mean_op(node: MeanOp, symbol_table): + """ + Import the buddy MeanOp. + From Buddy MeanOp to MLIR TOSA operation. + """ + + def _inner_op(result_type, input1, input2): + shift = _create_mul_shift_operand() + return tosa.MulOp(result_type, input1, input2, shift) + + input_tensor = symbol_table.get((str(node.args[0]), 0)) + keepdim = node.args[2] + dims = [int(x) for x in node.args[1]] + if isinstance(dims, int): + dims = [dims] + + act = [int(x) for x in ir.RankedTensorType(input_tensor.type).shape] + for i, d in enumerate(dims): + if d < 0: + dims[i] = d + len(act) + + # PyTorch dims are NCHW; remap when SSA stuck in NHWC. + out_meta = [int(x) for x in node.tensor_meta["shape"]] + if len(out_meta) == 4 or len(out_meta) == 2: + c_meta = out_meta[1] + elif len(out_meta) == 1: + c_meta = out_meta[0] + else: + c_meta = None + if c_meta is not None and len(act) == 4 and act[3] == c_meta and act[1] != c_meta: + dims = [{0: 0, 1: 3, 2: 1, 3: 2}[d] for d in dims] + elif c_meta is not None and len(act) == 4 and act[1] != c_meta: + raise RuntimeError(f"mean layout mismatch: ssa={act} out_meta={out_meta}") + + reduce_sum_result = input_tensor + for dim_item in dims: + reduce_dim_attr = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), dim_item) + reduce_sum_op = tosa.ReduceSumOp(reduce_sum_result, reduce_dim_attr) + reduce_sum_result = reduce_sum_op.results[0] + + tensor_shp = ir.RankedTensorType(input_tensor.type).shape + dim_size = 1 + + for dim_item in dims: + dim_size *= tensor_shp[dim_item] + + denominator_const_op = tosa.ConstOp( + ir.DenseElementsAttr.get(memoryview(array.array("f", [dim_size]))) + ) + reciprocal_op = tosa.ReciprocalOp( + denominator_const_op.results[0].type, denominator_const_op + ) + ret = _gen_arith_binary_op( + reciprocal_op.results[0], reduce_sum_op.results[0], _inner_op + ) + + if not keepdim: + result_shp = ir.RankedTensorType(ret.results[0].type).shape + result_shp = [siz for siz in result_shp if siz != 1] + reshape_operand = _create_shape_operand(result_shp) + ret = tosa.ReshapeOp(ret.results[0], reshape_operand) + + return ret + + +def alias_op(node: AliasOp, symbol_table): + """ + Import the alias operation. + From buddy graph ir's `AliasOp` operator to MLIR TOSA operations. + Creates an alias (view) of the input tensor. + aten.alias.default(input) -> Tensor + """ + input1 = symbol_table.get((str(node.args[0]), 0)) + + input_shape = list(ir.RankedTensorType(input1.type).shape) + input_dtype = ir.RankedTensorType(input1.type).element_type + + output_shape = list(node.tensor_meta["shape"]) + output_dtype = mlir_element_type_get(node.tensor_meta["dtype"]) + + if input_shape == output_shape and input_dtype == output_dtype: + return input1 + + # Sticky NHWC: alias must not invent NCHW type via identity. + if len(input_shape) == 4 and len(output_shape) == 4: + ns = [int(x) for x in input_shape] + nw = [int(x) for x in output_shape] + if ns == [nw[0], nw[2], nw[3], nw[1]]: + return input1 + if nw == [ns[0], ns[2], ns[3], ns[1]]: + out_ty = ir.RankedTensorType.get(nw, output_dtype) + return tosa.TransposeOp( + out_ty, input1, _create_permutation_attr([0, 2, 3, 1]) + ).result + + raise RuntimeError( + f"alias shape/layout mismatch: ssa={input_shape} meta={output_shape}" + ) + + +def _bn_channel_and_broadcast(input_tensor, channel_param): + shape = [int(x) for x in ir.RankedTensorType(input_tensor.type).shape] + c = int(list(ir.RankedTensorType(channel_param.type).shape)[0]) + if len(shape) == 4: + if has_nhwc_layout(input_tensor): + return c, [1, 1, 1, c] + if shape[1] == c: + return c, [1, c, 1, 1] + if shape[3] == c: + return c, [1, 1, 1, c] + raise RuntimeError(f"BN channel {c} mismatch with 4D shape {shape}") + if len(shape) >= 2: + if shape[1] == c: + return c, [1, c] + [1] * (len(shape) - 2) + if shape[-1] == c: + return c, [1] * (len(shape) - 1) + [c] + raise RuntimeError(f"BN channel {c} mismatch with shape {shape}") + + +def native_batch_norm_legit_op(node, symbol_table): + """ + Import the native batch norm legit operation. + From buddy graph ir's `NativeBatchNormLegitOp` operator to MLIR TOSA operations. + aten._native_batch_norm_legit(input, weight, bias, running_mean, running_var, training, momentum, eps) + -> (Tensor, Tensor, Tensor) + + Returns a tuple of (output, save_mean, save_invstd) where: + - output: normalized tensor with same shape as input + - save_mean: mean per channel, shape (C,) + - save_invstd: inverse std per channel, shape (C,) + + Batch normalization formula: + output = (input - mean) / sqrt(var + eps) * weight + bias + + During training: uses batch statistics + During inference: uses running_mean and running_var + """ + input_tensor = symbol_table.get((str(node.args[0]), 0)) + weight = ( + symbol_table.get((str(node.args[1]), 0)) if node.args[1] is not None else None + ) + bias = ( + symbol_table.get((str(node.args[2]), 0)) if node.args[2] is not None else None + ) + running_mean = ( + symbol_table.get((str(node.args[3]), 0)) if node.args[3] is not None else None + ) + running_var = ( + symbol_table.get((str(node.args[4]), 0)) if node.args[4] is not None else None + ) + training = node.args[5] + momentum = node.args[6] + eps = node.args[7] + + input_shape = list(ir.RankedTensorType(input_tensor.type).shape) + input_dtype = ir.RankedTensorType(input_tensor.type).element_type + + channel_ref = running_mean or weight or bias + if channel_ref is None: + raise RuntimeError("batch norm needs running_mean, weight, or bias") + C, broadcast_shape = _bn_channel_and_broadcast(input_tensor, channel_ref) + N = input_shape[0] + spatial_dims = input_shape[2:] + spatial_size = 1 + for d in spatial_dims: + spatial_size *= d + + if not training and running_mean is not None and running_var is not None: + # Inference mode: use running statistics + mean = running_mean + var = running_var + + scalar_broadcast_shape = [1] * len(input_shape) # for scalars like eps + + mean_broadcast = tosa.ReshapeOp( + mean, _create_shape_operand(broadcast_shape) + ).result + var_broadcast = tosa.ReshapeOp( + var, _create_shape_operand(broadcast_shape) + ).result + + # (input - mean) + centered = tosa.SubOp( + ir.RankedTensorType.get(input_shape, input_dtype), + input_tensor, + mean_broadcast, + ).result + + # var + eps - reshape eps to same rank for TOSA compatibility + eps_attr = ir.DenseElementsAttr.get( + memoryview(array.array("f", [eps])), + type=ir.RankedTensorType.get([1], input_dtype), + ) + eps_tensor = tosa.ConstOp(eps_attr).result + eps_broadcast = tosa.ReshapeOp( + eps_tensor, _create_shape_operand(scalar_broadcast_shape) + ).result + + var_plus_eps = tosa.AddOp( + ir.RankedTensorType.get(broadcast_shape, input_dtype), + var_broadcast, + eps_broadcast, + ).result + + # rsqrt(var + eps) + invstd = tosa.RsqrtOp( + ir.RankedTensorType.get(broadcast_shape, input_dtype), var_plus_eps + ).result + + # normalized = (input - mean) * invstd + normalized = tosa.MulOp( + ir.RankedTensorType.get(input_shape, input_dtype), centered, invstd + ).result + + output = normalized + + # Apply weight (gamma) + if weight is not None: + weight_broadcast = tosa.ReshapeOp( + weight, _create_shape_operand(broadcast_shape) + ).result + output = tosa.MulOp( + ir.RankedTensorType.get(input_shape, input_dtype), + output, + weight_broadcast, + ).result + + # Apply bias (beta) + if bias is not None: + bias_broadcast = tosa.ReshapeOp( + bias, _create_shape_operand(broadcast_shape) + ).result + output = tosa.AddOp( + ir.RankedTensorType.get(input_shape, input_dtype), + output, + bias_broadcast, + ).result + + # save_mean and save_invstd with shape (C,) + save_mean = tosa.ReshapeOp(mean_broadcast, _create_shape_operand([C])).result + save_invstd = tosa.ReshapeOp(invstd, _create_shape_operand([C])).result + + return output, save_mean, save_invstd + + # Training mode: compute batch statistics + train_input = input_tensor + train_shape = list(input_shape) + if len(train_shape) == 4 and train_shape[3] == C: + nchw_shape = [train_shape[0], C, train_shape[1], train_shape[2]] + train_input = tosa.TransposeOp( + ir.RankedTensorType.get(nchw_shape, input_dtype), + input_tensor, + _create_permutation_attr([0, 3, 1, 2]), + ).result + train_shape = nchw_shape + elif not (len(train_shape) >= 2 and train_shape[1] == C): + raise RuntimeError(f"BN training layout mismatch: shape={input_shape} C={C}") + N = train_shape[0] + spatial_size = 1 + for d in train_shape[2:]: + spatial_size *= d + reshaped_input = tosa.ReshapeOp( + train_input, _create_shape_operand([N, C, spatial_size]) + ).result + + # Compute mean: reduce over N and spatial dimensions + # First reduce over spatial (axis=2) + axis_attr_2 = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), 2) + spatial_sum = tosa.ReduceSumOp(reshaped_input, axis_attr_2).results[0] # (N, C, 1) + + # Then reduce over batch (axis=0) + axis_attr_0 = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), 0) + batch_spatial_sum = tosa.ReduceSumOp(spatial_sum, axis_attr_0).results[ + 0 + ] # (1, C, 1) + + # mean = sum / (N * spatial_size) - reshape divisor for TOSA compatibility + divisor = N * spatial_size + divisor_attr = ir.DenseElementsAttr.get( + memoryview(array.array("f", [1.0 / divisor])), + type=ir.RankedTensorType.get([1], input_dtype), + ) + divisor_tensor = tosa.ConstOp(divisor_attr).result + divisor_3d = tosa.ReshapeOp(divisor_tensor, _create_shape_operand([1, 1, 1])).result + + mean_3d = tosa.MulOp( + ir.RankedTensorType.get([1, C, 1], input_dtype), + batch_spatial_sum, + divisor_3d, + ).result + + # (x - mean) + diff = tosa.SubOp( + ir.RankedTensorType.get([N, C, spatial_size], input_dtype), + reshaped_input, + mean_3d, + ).result + + # (x - mean)^2 + squared = tosa.MulOp( + ir.RankedTensorType.get([N, C, spatial_size], input_dtype), diff, diff + ).result + + # sum of squared + sq_spatial_sum = tosa.ReduceSumOp(squared, axis_attr_2).results[0] # (N, C, 1) + sq_batch_sum = tosa.ReduceSumOp(sq_spatial_sum, axis_attr_0).results[0] # (1, C, 1) + + # variance = sum / (N * spatial_size) + var_3d = tosa.MulOp( + ir.RankedTensorType.get([1, C, 1], input_dtype), + sq_batch_sum, + divisor_3d, + ).result + + # var + eps - reshape eps for TOSA compatibility + eps_attr = ir.DenseElementsAttr.get( + memoryview(array.array("f", [eps])), + type=ir.RankedTensorType.get([1], input_dtype), + ) + eps_tensor = tosa.ConstOp(eps_attr).result + eps_3d = tosa.ReshapeOp(eps_tensor, divisor_3d_shape_operand).result + + var_plus_eps = tosa.AddOp( + ir.RankedTensorType.get([1, C, 1], input_dtype), var_3d, eps_3d + ).result + + # invstd = rsqrt(var + eps) + invstd_3d = tosa.RsqrtOp( + ir.RankedTensorType.get([1, C, 1], input_dtype), var_plus_eps + ).result + + # normalized = (x - mean) * invstd + normalized_3d = tosa.MulOp( + ir.RankedTensorType.get([N, C, spatial_size], input_dtype), + diff, + invstd_3d, + shift, + ).result + + # Reshape back to original shape + train_shape_operand = _create_shape_operand(train_shape) + output = tosa.ReshapeOp(normalized_3d, train_shape_operand).result + if list(input_shape) != train_shape: + output = tosa.TransposeOp( + ir.RankedTensorType.get(input_shape, input_dtype), + output, + _create_permutation_attr([0, 2, 3, 1]), + ).result + + # Apply weight (gamma) + if weight is not None: + broadcast_shape_operand = _create_shape_operand(broadcast_shape) + weight_broadcast = tosa.ReshapeOp(weight, broadcast_shape_operand).result + output = tosa.MulOp( + ir.RankedTensorType.get(input_shape, input_dtype), + output, + weight_broadcast, + shift, + ).result + + # Apply bias (beta) + if bias is not None: + broadcast_shape_operand = _create_shape_operand(broadcast_shape) + bias_broadcast = tosa.ReshapeOp(bias, broadcast_shape_operand).result + output = tosa.AddOp( + ir.RankedTensorType.get(input_shape, input_dtype), + output, + bias_broadcast, + ).result + + # Return (output, save_mean, save_invstd) + c_shape_operand = _create_shape_operand([C]) + save_mean = tosa.ReshapeOp(mean_3d, c_shape_operand).result + save_invstd = tosa.ReshapeOp(invstd_3d, c_shape_operand).result + + return output, save_mean, save_invstd + + +def native_batch_norm_legit_no_stats_op(node, symbol_table): + """ + Import the native batch norm legit no_stats operation. + From buddy graph ir's `NativeBatchNormLegitNoStatsOp` operator to MLIR TOSA operations. + aten._native_batch_norm_legit.no_stats(input, weight, bias, training, momentum, eps) + -> (Tensor, Tensor, Tensor) + + This variant doesn't use running statistics, always computes from batch. + Returns: (output, save_mean, save_invstd) + """ + input_tensor = symbol_table.get((str(node.args[0]), 0)) + weight = ( + symbol_table.get((str(node.args[1]), 0)) if node.args[1] is not None else None + ) + bias = ( + symbol_table.get((str(node.args[2]), 0)) if node.args[2] is not None else None + ) + training = node.args[3] + momentum = node.args[4] + eps = node.args[5] + + input_shape = list(ir.RankedTensorType(input_tensor.type).shape) + input_dtype = ir.RankedTensorType(input_tensor.type).element_type + + channel_ref = weight or bias + if channel_ref is None: + raise RuntimeError("batch norm no_stats needs weight or bias") + C, broadcast_shape = _bn_channel_and_broadcast(input_tensor, channel_ref) + + train_input = input_tensor + train_shape = list(input_shape) + if len(train_shape) == 4 and train_shape[3] == C: + nchw_shape = [train_shape[0], C, train_shape[1], train_shape[2]] + train_input = tosa.TransposeOp( + ir.RankedTensorType.get(nchw_shape, input_dtype), + input_tensor, + _create_permutation_attr([0, 3, 1, 2]), + ).result + train_shape = nchw_shape + elif not (len(train_shape) >= 2 and train_shape[1] == C): + raise RuntimeError(f"BN no_stats layout mismatch: shape={input_shape} C={C}") + N = train_shape[0] + spatial_size = 1 + for d in train_shape[2:]: + spatial_size *= d + + reshaped_shape_operand = _create_shape_operand([N, C, spatial_size]) + reshaped_input = tosa.ReshapeOp(train_input, reshaped_shape_operand).result + + # Compute mean: reduce over N and spatial dimensions + axis_attr_2 = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), 2) + spatial_sum = tosa.ReduceSumOp(reshaped_input, axis_attr_2).results[0] # (N, C, 1) + + axis_attr_0 = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), 0) + batch_spatial_sum = tosa.ReduceSumOp(spatial_sum, axis_attr_0).results[ + 0 + ] # (1, C, 1) + + # mean = sum / (N * spatial_size) + divisor = N * spatial_size + divisor_attr = ir.DenseElementsAttr.get( + memoryview(array.array("f", [1.0 / divisor])), + type=ir.RankedTensorType.get([1], input_dtype), + ) + divisor_tensor = tosa.ConstOp(divisor_attr).result + divisor_3d_shape_operand = _create_shape_operand([1, 1, 1]) + divisor_3d = tosa.ReshapeOp(divisor_tensor, divisor_3d_shape_operand).result + + shift = _create_mul_shift_operand() + mean_3d = tosa.MulOp( + ir.RankedTensorType.get([1, C, 1], input_dtype), + batch_spatial_sum, + divisor_3d, + shift, + ).result + + # (x - mean) + diff = tosa.SubOp( + ir.RankedTensorType.get([N, C, spatial_size], input_dtype), + reshaped_input, + mean_3d, + ).result + + # (x - mean)^2 + squared = tosa.MulOp( + ir.RankedTensorType.get([N, C, spatial_size], input_dtype), + diff, + diff, + shift, + ).result + + # sum of squared + sq_spatial_sum = tosa.ReduceSumOp(squared, axis_attr_2).results[0] # (N, C, 1) + sq_batch_sum = tosa.ReduceSumOp(sq_spatial_sum, axis_attr_0).results[0] # (1, C, 1) + + # variance = sum / (N * spatial_size) + var_3d = tosa.MulOp( + ir.RankedTensorType.get([1, C, 1], input_dtype), + sq_batch_sum, + divisor_3d, + shift, + ).result + + # var + eps - reshape eps to match [1, C, 1] shape for TOSA compatibility + eps_attr = ir.DenseElementsAttr.get( + memoryview(array.array("f", [eps])), + type=ir.RankedTensorType.get([1], input_dtype), + ) + eps_tensor = tosa.ConstOp(eps_attr).result + eps_3d = tosa.ReshapeOp(eps_tensor, divisor_3d_shape_operand).result + + var_plus_eps = tosa.AddOp( + ir.RankedTensorType.get([1, C, 1], input_dtype), var_3d, eps_3d + ).result + + # invstd = rsqrt(var + eps) + invstd_3d = tosa.RsqrtOp( + ir.RankedTensorType.get([1, C, 1], input_dtype), var_plus_eps + ).result + + # normalized = (x - mean) * invstd + normalized_3d = tosa.MulOp( + ir.RankedTensorType.get([N, C, spatial_size], input_dtype), + diff, + invstd_3d, + shift, + ).result + + # Reshape back to original shape + train_shape_operand = _create_shape_operand(train_shape) + output = tosa.ReshapeOp(normalized_3d, train_shape_operand).result + if list(input_shape) != train_shape: + output = tosa.TransposeOp( + ir.RankedTensorType.get(input_shape, input_dtype), + output, + _create_permutation_attr([0, 2, 3, 1]), + ).result + + # Apply weight (gamma) + if weight is not None: + broadcast_shape_operand = _create_shape_operand(broadcast_shape) + weight_broadcast = tosa.ReshapeOp(weight, broadcast_shape_operand).result + output = tosa.MulOp( + ir.RankedTensorType.get(input_shape, input_dtype), + output, + weight_broadcast, + shift, + ).result + + # Apply bias (beta) + if bias is not None: + broadcast_shape_operand = _create_shape_operand(broadcast_shape) + bias_broadcast = tosa.ReshapeOp(bias, broadcast_shape_operand).result + output = tosa.AddOp( + ir.RankedTensorType.get(input_shape, input_dtype), + output, + bias_broadcast, + ).result + + # Return (output, save_mean, save_invstd) + c_shape_operand = _create_shape_operand([C]) + save_mean = tosa.ReshapeOp(mean_3d, c_shape_operand).result + save_invstd = tosa.ReshapeOp(invstd_3d, c_shape_operand).result + + return output, save_mean, save_invstd + + +def native_batch_norm_legit_no_training_op(node, symbol_table): + """ + Import the native batch norm legit no_training operation. + From buddy graph ir's `NativeBatchNormLegitNoTrainingOp` operator to MLIR TOSA operations. + aten._native_batch_norm_legit_no_training(input, weight, bias, running_mean, running_var, momentum, eps) + -> (Tensor, Tensor, Tensor) + + This variant is inference only, always uses running statistics. + Returns: (output, save_mean, save_invstd) + """ + input_tensor = symbol_table.get((str(node.args[0]), 0)) + weight = ( + symbol_table.get((str(node.args[1]), 0)) if node.args[1] is not None else None + ) + bias = ( + symbol_table.get((str(node.args[2]), 0)) if node.args[2] is not None else None + ) + running_mean = symbol_table.get((str(node.args[3]), 0)) + running_var = symbol_table.get((str(node.args[4]), 0)) + momentum = node.args[5] + eps = node.args[6] + + input_shape = list(ir.RankedTensorType(input_tensor.type).shape) + input_dtype = ir.RankedTensorType(input_tensor.type).element_type + + channel_ref = running_mean or weight or bias + if channel_ref is None: + raise RuntimeError("batch norm needs running_mean, weight, or bias") + C, broadcast_shape = _bn_channel_and_broadcast(input_tensor, channel_ref) + + # Inference mode: use running statistics + scalar_broadcast_shape = [1] * len(input_shape) # for scalars like eps + + broadcast_shape_operand = _create_shape_operand(broadcast_shape) + mean_broadcast = tosa.ReshapeOp(running_mean, broadcast_shape_operand).result + var_broadcast = tosa.ReshapeOp(running_var, broadcast_shape_operand).result + + # var + eps - reshape eps to same rank for TOSA compatibility + eps_attr = ir.DenseElementsAttr.get( + memoryview(array.array("f", [eps])), + type=ir.RankedTensorType.get([1], input_dtype), + ) + eps_tensor = tosa.ConstOp(eps_attr).result + scalar_broadcast_shape_operand = _create_shape_operand(scalar_broadcast_shape) + eps_broadcast = tosa.ReshapeOp(eps_tensor, scalar_broadcast_shape_operand).result + + var_plus_eps = tosa.AddOp( + ir.RankedTensorType.get(broadcast_shape, input_dtype), + var_broadcast, + eps_broadcast, + ).result + + # PyTorch computes 1 / sqrt(var + eps). An approximate reciprocal + # square root can cross an activation quantization threshold on NVIDIA. + rank = len(input_shape) + identity = ir.AffineMap.get_identity(rank) + channel_type = ir.RankedTensorType.get(broadcast_shape, input_dtype) + inverse_empty = tensor.EmptyOp(broadcast_shape, input_dtype) + inverse = linalg.GenericOp( + [channel_type], + [var_plus_eps], + [inverse_empty.result], + ir.ArrayAttr.get([ir.AffineMapAttr.get(identity)] * 2), + ir.ArrayAttr.get( + [ir.Attribute.parse("#linalg.iterator_type")] * rank + ), + ) + inverse_block = ir.Block.create_at_start(inverse.region, [input_dtype] * 2) + with ir.InsertionPoint(inverse_block): + root = math.SqrtOp(inverse_block.arguments[0]).result + one = arith.ConstantOp(input_dtype, ir.FloatAttr.get(input_dtype, 1.0)).result + reciprocal = arith.DivFOp(one, root).result + linalg.YieldOp([reciprocal]) + invstd = inverse.result + + # Match PyTorch inference: alpha = invstd * gamma, + # beta = bias - mean * alpha, output = input * alpha + beta. + # Reassociating this into (input - mean) * invstd * gamma + bias + # changes FP32 rounding before downstream activation quantization. + alpha = invstd + if weight is not None: + gamma = tosa.ReshapeOp(weight, broadcast_shape_operand).result + alpha = _gen_arith_binary_op(invstd, gamma, tosa.MulOp).result + if bias is None: + mean_scaled = _gen_arith_binary_op(mean_broadcast, alpha, tosa.MulOp).result + beta = tosa.NegateOp( + mean_scaled.type, + mean_scaled, + _create_zero_point_tensor(mean_scaled), + _create_zero_point_tensor(mean_scaled), + ).result + else: + bias_broadcast = tosa.ReshapeOp(bias, broadcast_shape_operand).result + beta_empty = tensor.EmptyOp(broadcast_shape, input_dtype) + beta_op = linalg.GenericOp( + [channel_type], + [mean_broadcast, alpha, bias_broadcast], + [beta_empty.result], + ir.ArrayAttr.get([ir.AffineMapAttr.get(identity)] * 4), + ir.ArrayAttr.get( + [ir.Attribute.parse("#linalg.iterator_type")] * rank + ), + ) + beta_block = ir.Block.create_at_start(beta_op.region, [input_dtype] * 4) + with ir.InsertionPoint(beta_block): + negative_mean = arith.NegFOp(beta_block.arguments[0]).result + coefficient = math.FmaOp( + negative_mean, beta_block.arguments[1], beta_block.arguments[2] + ).result + linalg.YieldOp([coefficient]) + beta = beta_op.result + # Keep the final multiply-add fused: rounding the product separately + # changes values close to downstream INT8 rounding thresholds. + channel_map = ir.AffineMap.get( + rank, + 0, + [ + (ir.AffineExpr.get_dim(i) if extent != 1 else ir.AffineExpr.get_constant(0)) + for i, extent in enumerate(broadcast_shape) + ], + ) + result_type = ir.RankedTensorType.get(input_shape, input_dtype) + empty = tensor.EmptyOp(input_shape, input_dtype) + fused = linalg.GenericOp( + [result_type], + [input_tensor, alpha, beta], + [empty.result], + ir.ArrayAttr.get( + [ + ir.AffineMapAttr.get(m) + for m in (identity, channel_map, channel_map, identity) + ] + ), + ir.ArrayAttr.get( + [ir.Attribute.parse("#linalg.iterator_type")] * rank + ), + ) + block = ir.Block.create_at_start(fused.region, [input_dtype] * 4) + with ir.InsertionPoint(block): + value = math.FmaOp(*block.arguments[:3]).result + linalg.YieldOp([value]) + if has_nhwc_layout(input_tensor): + fused.operation.attributes["buddy.layout"] = ir.StringAttr.get("nhwc") + output = fused.result + + # save_mean and save_invstd with shape (C,) + c_shape_operand = _create_shape_operand([C]) + save_mean = tosa.ReshapeOp(mean_broadcast, c_shape_operand).result + save_invstd = tosa.ReshapeOp(invstd, c_shape_operand).result + + return output, save_mean, save_invstd + + +def upsample_bilinear2d_vec_op(node, symbol_table): + """ + Import the upsample_bilinear2d.vec operation. + From buddy graph ir's `UpsampleBilinear2dVecOp` operator to MLIR TOSA operations. + aten.upsample_bilinear2d.vec(input, output_size, align_corners, scale_factors) -> Tensor + + Uses TOSA resize operation with BILINEAR mode. + Input: NCHW format, needs to be converted to NHWC for TOSA. + """ + input_tensor = symbol_table.get((str(node.args[0]), 0)) + output_size = node.args[1] # [H_out, W_out] or None + align_corners = node.args[2] + scale_factors = ( + node.args[3] if len(node.args) > 3 else None + ) # [scale_h, scale_w] or None + + input_shape = list(ir.RankedTensorType(input_tensor.type).shape) + input_dtype = ir.RankedTensorType(input_tensor.type).element_type + + channels = int(list(node.tensor_meta["shape"])[1]) + src = node.args[0] + nchw_meta = ( + list(src.tensor_meta["shape"]) + if hasattr(src, "tensor_meta") and "shape" in src.tensor_meta + else None + ) + input_tensor, N, H_in, W_in, C = _to_nhwc( + input_tensor, channels, input_dtype, nchw_meta + ) + + # Determine output size + if output_size is not None: + H_out, W_out = output_size + elif scale_factors is not None: + H_out = int(H_in * scale_factors[0]) + W_out = int(W_in * scale_factors[1]) + else: + raise ValueError("Either output_size or scale_factors must be provided") + + # Calculate scale for TOSA resize + # TOSA scale format: [scale_y_n, scale_y_d, scale_x_n, scale_x_d] + # For exact output size: scale = output_size / input_size + scale_y_n = H_out + scale_y_d = H_in + scale_x_n = W_out + scale_x_d = W_in + + scale_attr = ir._denseI64ArrayAttr( + [scale_y_n, scale_y_d, scale_x_n, scale_x_d], None + ) + offset_attr = ir._denseI64ArrayAttr([0, 0], None) + border_attr = ir._denseI64ArrayAttr([0, 0], None) + mode_attr = ir.StringAttr.get("BILINEAR") + + # Apply resize in NHWC format; keep NHWC output. + nhwc_out_type = ir.RankedTensorType.get([N, H_out, W_out, C], input_dtype) + resized = tosa.ResizeOp( + nhwc_out_type, + input_tensor, + scale_attr, + offset_attr, + border_attr, + mode_attr, + ) + + return resized + + +def upsample_nearest2d_vec_op(node, symbol_table): + """ + Import the upsample_nearest2d.vec operation. + From buddy graph ir's `UpsampleNearest2dVecOp` operator to MLIR TOSA operations. + aten.upsample_nearest2d.vec(input, output_size, scale_factors) -> Tensor + + Uses TOSA resize operation with NEAREST mode. + """ + input_tensor = symbol_table.get((str(node.args[0]), 0)) + output_size = node.args[1] # [H_out, W_out] or None + scale_factors = ( + node.args[2] if len(node.args) > 2 else None + ) # [scale_h, scale_w] or None + + input_dtype = ir.RankedTensorType(input_tensor.type).element_type + + channels = int(list(node.tensor_meta["shape"])[1]) + src = node.args[0] + nchw_meta = ( + list(src.tensor_meta["shape"]) + if hasattr(src, "tensor_meta") and "shape" in src.tensor_meta + else None + ) + input_tensor, N, H_in, W_in, C = _to_nhwc( + input_tensor, channels, input_dtype, nchw_meta + ) + + # Determine output size + if output_size is not None: + H_out, W_out = output_size + elif scale_factors is not None: + H_out = int(H_in * scale_factors[0]) + W_out = int(W_in * scale_factors[1]) + else: + raise ValueError("Either output_size or scale_factors must be provided") + + scale_attr = ir._denseI64ArrayAttr([H_out, H_in, W_out, W_in], None) + offset_attr = ir._denseI64ArrayAttr([0, 0], None) + border_attr = ir._denseI64ArrayAttr([0, 0], None) + mode_attr = ir.StringAttr.get("NEAREST") + + nhwc_out_type = ir.RankedTensorType.get([N, H_out, W_out, C], input_dtype) + return tosa.ResizeOp( + nhwc_out_type, + input_tensor, + scale_attr, + offset_attr, + border_attr, + mode_attr, + ) + + +from buddy.compiler.ops.tosa import convert_element_type_op, where_op +from buddy.compiler.ops.tosa import ops_registry as upstream_registry +from .linalg import ops_registry as linalg_registry + +ops_registry = { + **linalg_registry, + **upstream_registry, + "WhereOp": where_op, + "ToCopyOp": convert_element_type_op, + "AddOp": add_op, + "MulOp": mul_op, + "SubOp": sub_op, + "AmaxOp": amax_op, + "DivOp": div_op, + "ReshapeOp": reshape_op, + "ViewOp": reshape_op, + "MaxPool2dOp": maxpool2d_op, + "AvgPool2dOp": avg_pool2d_op, + "AdaptiveAvgPool2dOp": adaptive_avg_pool2d_op, + "Conv2dOp": convolution2d_op, + "Int8Conv2dOp": int8_conv2d_op, + "Int8MatmulOp": int8_matmul_op, + "Int8QuantizeOp": int8_quantize_op, + "SiluOp": silu_op, + "MeanOp": mean_op, + "AliasOp": alias_op, + "NativeBatchNormLegitOp": native_batch_norm_legit_op, + "NativeBatchNormLegitNoStatsOp": native_batch_norm_legit_no_stats_op, + "NativeBatchNormLegitNoTrainingOp": native_batch_norm_legit_no_training_op, + "UpsampleBilinear2dVecOp": upsample_bilinear2d_vec_op, + "UpsampleNearest2dVecOp": upsample_nearest2d_vec_op, +} diff --git a/arch/src/main/scala/framework/root/root-tile/src/csrc/README.md b/waste/stack/compiler/frontend/quant/__init__.py similarity index 100% rename from arch/src/main/scala/framework/root/root-tile/src/csrc/README.md rename to waste/stack/compiler/frontend/quant/__init__.py diff --git a/waste/stack/compiler/frontend/quant/bwq.py b/waste/stack/compiler/frontend/quant/bwq.py new file mode 100644 index 000000000..4329abaeb --- /dev/null +++ b/waste/stack/compiler/frontend/quant/bwq.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import tomllib +from dataclasses import dataclass +from pathlib import Path + + +@dataclass +class TensorSpec: + name: str + shape: list[int] + storage: str + axes: list[int] + weight_off: int + weight_len: int + scale_off: int + scale_len: int + + +@dataclass +class BwqPackage: + version: int + tensors: list[TensorSpec] + weights: bytes + scales: bytes + + +_FORBIDDEN = ("groupSize", "group_size", "zeroPoint", "zero_point") +_STORAGE = {"i8"} + + +def _numel(shape: list[int], name: str) -> int: + n = 1 + for d in shape: + if d <= 0: + raise ValueError(f"bad shape for {name}") + n *= d + return n + + +def validate_bwq(pkg: BwqPackage) -> None: + if pkg.version != 1: + raise ValueError(f"unsupported version: {pkg.version}") + if not pkg.tensors: + raise ValueError("empty tensor list") + seen: set[str] = set() + for t in pkg.tensors: + if t.name in seen: + raise ValueError(f"duplicate tensor name: {t.name}") + seen.add(t.name) + if t.storage not in _STORAGE: + raise ValueError(f"unsupported storage {t.storage!r} for {t.name}") + if t.weight_off < 0 or t.weight_len < 0: + raise ValueError(f"bad weight range for {t.name}") + if t.weight_off + t.weight_len > len(pkg.weights): + raise ValueError(f"weight OOB for {t.name}") + if t.scale_off < 0 or t.scale_len < 0: + raise ValueError(f"bad scale range for {t.name}") + if t.scale_off + t.scale_len > len(pkg.scales): + raise ValueError(f"scale OOB for {t.name}") + n = _numel(t.shape, t.name) + for ax in t.axes: + if ax < 0 or ax >= len(t.shape): + raise ValueError(f"axes OOB for {t.name}") + if t.weight_len != n: + raise ValueError(f"weight_len {t.weight_len} != numel {n} for {t.name}") + if t.axes: + scale_n = 1 + for ax in t.axes: + scale_n *= t.shape[ax] + if t.scale_len != scale_n * 4: + raise ValueError(f"scale_len mismatch for {t.name}") + elif t.scale_len != 4: + raise ValueError(f"per-tensor scale_len must be 4 for {t.name}") + + +def write_bwq(pkg: BwqPackage, path: Path) -> None: + validate_bwq(pkg) + path = Path(path) + if path.exists() and not path.is_dir(): + raise ValueError(f"not a directory: {path}") + path.mkdir(parents=True, exist_ok=True) + lines = [f"version = {pkg.version}", ""] + for t in pkg.tensors: + lines += [ + "[[tensor]]", + f'name = "{t.name}"', + f"shape = [{', '.join(str(x) for x in t.shape)}]", + f'storage = "{t.storage}"', + f"axes = [{', '.join(str(x) for x in t.axes)}]", + f"weight_off = {t.weight_off}", + f"weight_len = {t.weight_len}", + f"scale_off = {t.scale_off}", + f"scale_len = {t.scale_len}", + "", + ] + (path / "manifest.toml").write_text("\n".join(lines), encoding="utf-8") + (path / "weights.bin").write_bytes(pkg.weights) + (path / "scales.bin").write_bytes(pkg.scales) + + +def read_bwq(path: Path) -> BwqPackage: + path = Path(path) + if not path.is_dir(): + raise ValueError(f"bwq path is not a directory: {path}") + manifest = path / "manifest.toml" + if not manifest.is_file(): + raise ValueError(f"missing manifest.toml in {path}") + try: + meta = tomllib.loads(manifest.read_bytes().decode("utf-8")) + except tomllib.TOMLDecodeError as e: + raise ValueError(f"invalid manifest.toml: {e}") from e + except UnicodeDecodeError as e: + raise ValueError(f"invalid manifest.toml encoding: {e}") from e + for key in _FORBIDDEN: + if key in meta: + raise ValueError(f"forbidden key at top level: {key}") + if "version" not in meta: + raise ValueError("missing field version") + tensors: list[TensorSpec] = [] + for entry in meta.get("tensor", []): + for key in _FORBIDDEN: + if key in entry and entry[key] is not None: + raise ValueError(f"forbidden key {key} on tensor {entry.get('name')}") + need = ( + "name", + "shape", + "storage", + "axes", + "weight_off", + "weight_len", + "scale_off", + "scale_len", + ) + for k in need: + if k not in entry: + raise ValueError(f"missing field {k}") + tensors.append( + TensorSpec( + name=entry["name"], + shape=list(entry["shape"]), + storage=entry["storage"], + axes=list(entry["axes"]), + weight_off=int(entry["weight_off"]), + weight_len=int(entry["weight_len"]), + scale_off=int(entry["scale_off"]), + scale_len=int(entry["scale_len"]), + ) + ) + weights_path = path / "weights.bin" + scales_path = path / "scales.bin" + if not weights_path.is_file(): + raise ValueError(f"missing weights.bin in {path}") + if not scales_path.is_file(): + raise ValueError(f"missing scales.bin in {path}") + pkg = BwqPackage( + version=int(meta["version"]), + tensors=tensors, + weights=weights_path.read_bytes(), + scales=scales_path.read_bytes(), + ) + validate_bwq(pkg) + return pkg diff --git a/waste/stack/compiler/frontend/quant/importer.py b/waste/stack/compiler/frontend/quant/importer.py new file mode 100644 index 000000000..92a1cd222 --- /dev/null +++ b/waste/stack/compiler/frontend/quant/importer.py @@ -0,0 +1,1187 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import torch +from buddy.compiler.graph.operation import ( + AdaptiveAvgPool2dOp, + CloneOp, + AddMMOp, + AddOp, + AliasOp, + CatOp, + ClampMaxOp, + ClampMinOp, + Conv2dOp, + DivOp, + ExpOp, + GetItemOp, + HardswishOp, + LowMemoryMaxPoolWithOffsetsOp, + MatmulOp, + MaxPool2dOp, + MeanOp, + MulOp, + NegOp, + PermuteOp, + ReluOp, + ReshapeOp, + SiluOp, + SplitOp, + SplitWithSizesOp, + TOp, + UnsafeIndexOp, + ViewOp, +) +from stack.compiler.frontend.operation import ( + QuantizedChannelConcatOp, + QuantizedChannelSliceOp, + QuantizedDepthwiseConv2dOp, + QuantizedConv2dOp, + QuantizedGlobalAvgPoolOp, + QuantizedAddOp, + QuantizedMulOp, + QuantizedMatmulOp, + QuantizedMaxPool2dOp, + QuantizedResizeNearestOp, +) +from buddy.compiler.graph.type import TensorDType + +from stack.compiler.frontend.quant.rax import ( + QuantTensor, + RaxQuantPackage, + write_payload, +) + + +def _quantize_operators(graph, parameter_names, arrays, weight_scales, calibration): + def uses(node, name): + pending = list(node.args) + while pending: + argument = pending.pop() + if isinstance(argument, (list, tuple)): + pending.extend(argument) + elif renamed.get(str(argument), str(argument)) == name: + return True + return False + + param_nodes = list(graph.params) + input_node_list = list(graph.inputs) + original_body = list(graph._body) + calibration_index = {} + plans = {} + renamed = {} + removed = set() + + for node in original_body: + if not isinstance(node, (AliasOp, CloneOp, AdaptiveAvgPool2dOp)) or ( + isinstance(node, AdaptiveAvgPool2dOp) + and graph.node_table[str(node.args[0])].tensor_meta["shape"] + != node.tensor_meta["shape"] + ): + continue + if not isinstance(node.args[0], str): + raise ValueError(f"unsupported Alias form for {node.name}") + source = str(node.args[0]) + source_node = graph.node_table.get(source) + if source_node is None or source_node.tensor_meta.get( + "shape" + ) != node.tensor_meta.get("shape"): + raise ValueError(f"Alias shape mismatch for {node.name}") + renamed[node.name] = renamed.get(source, source) + removed.add(node.name) + + for index, node in enumerate(original_body): + if not isinstance(node, (Conv2dOp, AddMMOp, MatmulOp)): + continue + if isinstance(node, Conv2dOp): + if len(node.args) != 9: + raise ValueError(f"unexpected Conv2d form for {node.name}") + if node.args[5] != [1, 1] or node.args[6]: + raise ValueError(f"unsupported Quantized Conv2D form for {node.name}") + if node.args[3][0] != node.args[3][1]: + raise ValueError(f"asymmetric Quantized Conv2D stride for {node.name}") + padding = node.args[4] + if len(padding) == 1: + padding = [padding[0], padding[0]] + if len(padding) != 2 or padding[0] != padding[1]: + raise ValueError(f"asymmetric Quantized Conv2D padding for {node.name}") + activation_name, weight_arg, bias_arg = node.args[:3] + activation_name = renamed.get(str(activation_name), str(activation_name)) + weight_node = graph.node_table[str(weight_arg)] + weight_name = parameter_names.get(weight_node.name) + if weight_name is None: + raise ValueError( + f"Quantized Conv2D weight is not a parameter for {node.name}" + ) + weight_shape = list(arrays[weight_name].shape) + input_channels = int( + graph.node_table[str(activation_name)].tensor_meta["shape"][1] + ) + groups = int(node.args[8]) + if groups == 1: + replacement = QuantizedConv2dOp() + reduction_size = weight_shape[1] * weight_shape[2] * weight_shape[3] + elif ( + groups == input_channels + and weight_shape[0] == input_channels + and weight_shape[1] == 1 + ): + replacement = QuantizedDepthwiseConv2dOp() + reduction_size = weight_shape[2] * weight_shape[3] + else: + raise ValueError(f"unsupported grouped Conv2D for {node.name}") + else: + if isinstance(node, AddMMOp): + if len(node.args) < 3: + raise ValueError(f"unexpected AddMM form for {node.name}") + bias_arg, activation_name, weight_arg = node.args[:3] + else: + if len(node.args) != 2: + raise ValueError(f"unexpected Matmul form for {node.name}") + activation_name, weight_arg = node.args + activation_name = renamed.get( + str(activation_name), str(activation_name) + ) + bias_arg = None + weight_value = graph.node_table[str(weight_arg)] + valid_transpose = isinstance(weight_value, TOp) or ( + isinstance(weight_value, PermuteOp) + and len(weight_value.args) == 2 + and list(weight_value.args[1]) == [1, 0] + ) + if not valid_transpose or len(weight_value._parents) != 1: + raise ValueError( + f"Quantized MatMul weight must be one transpose for {node.name}: " + f"{weight_value.name}/{type(weight_value).__name__}, " + f"parents={weight_value._parents}, args={weight_value.args}" + ) + weight_node = graph.node_table[weight_value._parents[0]] + weight_arg = weight_node.name + removed.add(weight_value.name) + replacement = QuantizedMatmulOp() + weight_name = parameter_names.get(weight_node.name) + if weight_name is None: + raise ValueError( + f"Quantized MatMul weight is not a parameter for {node.name}" + ) + weight_shape = list(arrays[weight_name].shape) + if len(weight_shape) != 2: + raise ValueError( + f"Quantized MatMul weight must be rank 2 for {node.name}" + ) + reduction_size = weight_shape[1] + + weight_name = parameter_names.get(weight_node.name) + if weight_name is None or weight_name not in weight_scales: + raise ValueError(f"missing quantized weight for {node.name}") + weight_node._quantized_weight = True + records = calibration.get(weight_name) + occurrence = calibration_index.get(weight_name, 0) + if records is None or occurrence >= len(records): + raise ValueError(f"missing calibration for {weight_name}") + calibration_index[weight_name] = occurrence + 1 + ( + calibrated_input, + raw_output_scale, + relu_output_scale, + raw_channel_scales, + hardswish_channel_scales, + ) = records[occurrence] + raw_channel_scales = np.asarray(raw_channel_scales, dtype=np.float32) + hardswish_channel_scales = np.asarray( + hardswish_channel_scales, dtype=np.float32 + ) + if ( + not np.isfinite(calibrated_input) + or calibrated_input <= 0.0 + or not np.isfinite(raw_output_scale) + or raw_output_scale <= 0.0 + or not np.isfinite(relu_output_scale) + or relu_output_scale <= 0.0 + or raw_channel_scales.shape != (weight_shape[0],) + or hardswish_channel_scales.shape != (weight_shape[0],) + or not np.all(np.isfinite(raw_channel_scales)) + or not np.all(np.isfinite(hardswish_channel_scales)) + or np.any(raw_channel_scales <= 0.0) + or np.any(hardswish_channel_scales <= 0.0) + ): + raise ValueError(f"invalid calibration scale for {node.name}") + activation_node = graph.node_table[str(activation_name)] + + dw = np.asarray(weight_scales[weight_name], dtype=np.float32).reshape(-1) + output_channels = weight_shape[0] + if dw.size != output_channels: + raise ValueError(f"weight scale count mismatch for {weight_name}") + if not np.all(np.isfinite(dw)) or np.any(dw <= 0.0): + raise ValueError(f"invalid weight scale for {weight_name}") + if bias_arg is None: + bias = np.zeros(output_channels, dtype=np.float32) + else: + bias_node = graph.node_table[str(bias_arg)] + bias_name = parameter_names.get(bias_node.name) + if bias_name is None: + raise ValueError( + f"Quantized bias is not an offline parameter for {node.name}" + ) + bias = np.asarray(arrays[bias_name], dtype=np.float32).reshape(-1) + if bias.size != output_channels: + raise ValueError(f"bias channel count mismatch for {node.name}") + if not np.all(np.isfinite(bias)): + raise ValueError(f"non-finite bias for {node.name}") + product_bound = 128 * 128 * reduction_size + bias_headroom = np.iinfo(np.int32).max - product_bound + if bias_headroom <= 1: + raise ValueError(f"INT32 accumulator cannot hold {node.name}") + minimum_weight_scale = np.abs(bias.astype(np.float64)) / ( + (bias_headroom - 1) * float(calibrated_input) + ) + minimum_weight_scale = np.nextafter( + minimum_weight_scale.astype(np.float32), np.float32(np.inf) + ) + dw = np.maximum(dw, minimum_weight_scale) + weight_scales[weight_name] = dw + weight_node._quantized_weight_scales = dw.copy() + required_input_scale = float(calibrated_input) + + direct_users = [ + candidate + for candidate in graph._body + if candidate is not node and uses(candidate, node.name) + ] + relus = [ + candidate + for candidate in graph._body + if isinstance(candidate, (ReluOp, HardswishOp, SiluOp)) + and len(candidate.args) == 1 + and str(candidate.args[0]) == node.name + ] + if len(relus) > 1: + raise ValueError(f"multiple activation users for {node.name}") + activation = relus[0] if relus else None + activation_nodes = [activation] if activation is not None else [] + activation_kind = ( + 2 + if isinstance(activation, (HardswishOp, SiluOp)) + else 1 if isinstance(activation, ReluOp) else 0 + ) + lut_kind = ( + "silu" + if isinstance(activation, SiluOp) + else "hardswish" if isinstance(activation, HardswishOp) else None + ) + + silu_negs = [ + candidate for candidate in direct_users if isinstance(candidate, NegOp) + ] + if silu_negs: + if activation is not None or len(silu_negs) != 1: + raise ValueError(f"ambiguous decomposed SiLU after {node.name}") + neg = silu_negs[0] + neg_users = [ + candidate for candidate in original_body if uses(candidate, neg.name) + ] + if len(neg_users) != 1 or not isinstance(neg_users[0], ExpOp): + raise ValueError(f"malformed decomposed SiLU exp after {node.name}") + exp = neg_users[0] + exp_users = [ + candidate for candidate in original_body if uses(candidate, exp.name) + ] + if ( + len(exp_users) != 1 + or not isinstance(exp_users[0], AddOp) + or list(exp_users[0].args) != [exp.name, 1] + ): + raise ValueError(f"malformed decomposed SiLU add after {node.name}") + add = exp_users[0] + add_users = [ + candidate for candidate in original_body if uses(candidate, add.name) + ] + if ( + len(add_users) != 1 + or not isinstance(add_users[0], DivOp) + or list(add_users[0].args) != [node.name, add.name] + or {candidate.name for candidate in direct_users} + != {neg.name, add_users[0].name} + ): + raise ValueError(f"malformed decomposed SiLU div after {node.name}") + activation = add_users[0] + activation_nodes = [neg, exp, add, activation] + activation_kind = 2 + lut_kind = "silu" + + hard_swish_adds = [ + candidate + for candidate in direct_users + if isinstance(candidate, AddOp) + and len(candidate.args) == 2 + and str(candidate.args[0]) == node.name + and candidate.args[1] == 3 + ] + if hard_swish_adds: + if activation is not None or len(hard_swish_adds) != 1: + raise ValueError(f"ambiguous HardSwish after {node.name}") + add = hard_swish_adds[0] + add_users = [ + candidate for candidate in graph._body if uses(candidate, add.name) + ] + if len(add_users) != 1 or not isinstance(add_users[0], ClampMinOp): + raise ValueError(f"malformed HardSwish clamp-min after {node.name}") + clamp_min = add_users[0] + if list(clamp_min.args) != [add.name, 0]: + raise ValueError( + f"malformed HardSwish clamp-min args after {node.name}" + ) + clamp_min_users = [ + candidate + for candidate in graph._body + if uses(candidate, clamp_min.name) + ] + if len(clamp_min_users) != 1 or not isinstance( + clamp_min_users[0], ClampMaxOp + ): + raise ValueError(f"malformed HardSwish clamp-max after {node.name}") + clamp_max = clamp_min_users[0] + if list(clamp_max.args) != [clamp_min.name, 6]: + raise ValueError( + f"malformed HardSwish clamp-max args after {node.name}" + ) + clamp_max_users = [ + candidate + for candidate in graph._body + if uses(candidate, clamp_max.name) + ] + if len(clamp_max_users) != 1: + raise ValueError(f"ambiguous hard activation after {node.name}") + after_clamp = clamp_max_users[0] + if isinstance(after_clamp, DivOp): + if list(after_clamp.args) != [clamp_max.name, 6]: + raise ValueError(f"malformed hard-sigmoid after {node.name}") + if {candidate.name for candidate in direct_users} != {add.name}: + raise ValueError( + f"hard-sigmoid input has extra users after {node.name}" + ) + activation = after_clamp + activation_nodes = [add, clamp_min, clamp_max, after_clamp] + activation_kind = 2 + lut_kind = "hardsigmoid" + elif isinstance(after_clamp, MulOp): + mul = after_clamp + if list(map(str, mul.args)) != [node.name, clamp_max.name]: + raise ValueError( + f"malformed HardSwish multiply args after {node.name}" + ) + if {candidate.name for candidate in direct_users} != { + add.name, + mul.name, + }: + raise ValueError( + f"HardSwish input has extra users after {node.name}" + ) + mul_users = [ + candidate for candidate in graph._body if uses(candidate, mul.name) + ] + if len(mul_users) != 1 or not isinstance(mul_users[0], DivOp): + raise ValueError(f"malformed HardSwish divide after {node.name}") + div = mul_users[0] + if list(div.args) != [mul.name, 6]: + raise ValueError( + f"malformed HardSwish divide args after {node.name}" + ) + activation = div + activation_nodes = [add, clamp_min, clamp_max, mul, div] + activation_kind = 2 + lut_kind = "hardswish" + else: + raise ValueError(f"unsupported hard activation after {node.name}") + + if (node.trace_meta is not None and node.trace_meta["tensor"]) or any( + item.trace_meta is not None for item in activation_nodes + ): + activation = None + activation_nodes = [] + activation_kind = 0 + lut_kind = None + + result_name = activation.name if activation is not None else node.name + result_users = [ + candidate + for candidate in graph._body + if candidate is not activation + and candidate is not node + and uses(candidate, result_name) + ] + plans[node.name] = { + "index": index, + "node": node, + "replacement": replacement, + "activation_name": str(activation_name), + "activation_node": activation_node, + "weight_arg": str(weight_arg), + "weight_shape": weight_shape, + "dw": dw, + "bias": bias, + "required_input_scale": required_input_scale, + "calibrated_output_scale": float( + relu_output_scale if activation_kind == 1 else raw_output_scale + ), + "raw_channel_scales": raw_channel_scales, + "hardswish_channel_scales": hardswish_channel_scales, + "activation": activation, + "activation_nodes": activation_nodes, + "activation_kind": activation_kind, + "lut_kind": lut_kind, + "direct_users": direct_users, + "result_name": result_name, + "users": result_users, + "padding": padding if isinstance(node, Conv2dOp) else None, + "product_bound": product_bound, + } + + first_conv = min( + (plan for plan in plans.values() if isinstance(plan["node"], Conv2dOp)), + key=lambda plan: plan["index"], + default=None, + ) + if first_conv is not None and first_conv["lut_kind"] == "hardswish": + consumers = [ + candidate + for candidate in plans.values() + if candidate["activation_name"] == first_conv["result_name"] + ] + if ( + len(first_conv["users"]) != 1 + or len(consumers) != 1 + or not isinstance(consumers[0]["replacement"], QuantizedDepthwiseConv2dOp) + or consumers[0]["weight_shape"][0] != first_conv["weight_shape"][0] + ): + raise ValueError( + "the first Hardswish Conv must feed one matching depthwise Conv" + ) + first_conv["lane_input_scales"] = first_conv["raw_channel_scales"] + first_conv["lane_output_scales"] = first_conv["hardswish_channel_scales"] + consumers[0]["input_channel_scales"] = first_conv["lane_output_scales"] + + for plan in plans.values(): + if plan["activation"] is not None: + renamed[plan["result_name"]] = plan["node"].name + removed.update(item.name for item in plan["activation_nodes"]) + + special = {} + quantized_values = { + name + for plan in plans.values() + for name in (plan["node"].name, plan["result_name"]) + } + for node in original_body: + if not isinstance(node, GetItemOp) or len(node.args) != 2: + continue + source = graph.node_table.get(str(node.args[0])) + if not isinstance(source, (SplitOp, SplitWithSizesOp)) or len(source.args) != 3: + continue + input_name = renamed.get(str(source.args[0]), str(source.args[0])) + if int(source.args[2]) == 1 and input_name in quantized_values: + quantized_values.add(node.name) + for node in original_body: + if node.name in removed: + continue + activation = None + result_name = node.name + arguments = None + attributes = {} + if isinstance(node, GetItemOp): + if len(node.args) != 2 or not isinstance(node.args[1], int): + continue + source = graph.node_table[str(node.args[0])] + index = int(node.args[1]) + if isinstance(source, (SplitOp, SplitWithSizesOp)): + if len(source.args) != 3: + continue + input_name = renamed.get(str(source.args[0]), str(source.args[0])) + if input_name not in quantized_values: + continue + if int(source.args[2]) != 1: + raise ValueError( + f"Quantized channel split must use NCHW C at {source.name}" + ) + if isinstance(source, SplitOp): + channels = int( + graph.node_table[str(source.args[0])].tensor_meta["shape"][1] + ) + size = int(source.args[1]) + sizes = [size] * (channels // size) + if channels % size: + sizes.append(channels % size) + else: + sizes = [int(value) for value in source.args[1]] + if index < 0 or index >= len(sizes): + raise ValueError( + f"Quantized channel split index is invalid at {node.name}" + ) + replacement = QuantizedChannelSliceOp() + arguments = [input_name] + attributes = { + "offset": sum(sizes[:index]), + "output_shape": list(node.tensor_meta["shape"]), + } + removed.add(source.name) + elif isinstance(source, LowMemoryMaxPoolWithOffsetsOp): + if index != 0 or len(source.args) != 6: + raise ValueError( + f"Quantized SPPF must select MaxPool values at {node.name}" + ) + kernel, stride, padding, dilation, ceil_mode = source.args[1:] + if ( + list(kernel) != [5, 5] + or list(stride) != [1, 1] + or list(padding) != [2, 2] + or list(dilation) != [1, 1] + or ceil_mode + ): + raise ValueError( + f"unsupported Quantized SPPF MaxPool at {source.name}" + ) + input_name = renamed.get(str(source.args[0]), str(source.args[0])) + if input_name not in quantized_values: + continue + replacement = QuantizedMaxPool2dOp() + arguments = [input_name] + attributes = { + "input_shape": list( + graph.node_table[str(source.args[0])].tensor_meta["shape"] + ), + "output_shape": list(node.tensor_meta["shape"]), + "kernel": 5, + "stride": 1, + "padding": 2, + } + removed.add(source.name) + else: + continue + elif isinstance(node, CatOp): + if len(node.args) != 2 or int(node.args[1]) != 1: + continue + arguments = [renamed.get(str(value), str(value)) for value in node.args[0]] + if len(arguments) < 2 or any( + value not in quantized_values for value in arguments + ): + continue + replacement = QuantizedChannelConcatOp() + attributes = {"output_shape": list(node.tensor_meta["shape"])} + elif isinstance(node, UnsafeIndexOp): + if len(node.args) != 2: + continue + input_name = renamed.get(str(node.args[0]), str(node.args[0])) + if input_name not in quantized_values: + continue + input_shape = list(graph.node_table[str(node.args[0])].tensor_meta["shape"]) + output_shape = list(node.tensor_meta["shape"]) + if ( + len(input_shape) != 4 + or len(output_shape) != 4 + or input_shape[0] != output_shape[0] + or input_shape[1] != output_shape[1] + or output_shape[2] % input_shape[2] + or output_shape[3] % input_shape[3] + ): + raise ValueError(f"unsupported Quantized nearest resize at {node.name}") + replacement = QuantizedResizeNearestOp() + arguments = [input_name] + attributes = { + "output_shape": output_shape, + "scale_h": output_shape[2] // input_shape[2], + "scale_w": output_shape[3] // input_shape[3], + } + elif isinstance(node, MaxPool2dOp): + # Pooling can sit beyond an FP32 reshape/transpose boundary. + # Only a quantized producer supplies an INT8 scale for fusion. + input_name = renamed.get(str(node.args[0]), str(node.args[0])) + if input_name not in quantized_values: + continue + if ( + len(node.args) not in (3, 4) + or len(node.args[1]) != 2 + or node.args[1][0] != node.args[1][1] + or len(node.args[2]) != 2 + or node.args[2][0] != node.args[2][1] + ): + raise ValueError( + f"unsupported quantized graph MaxPool2D for {node.name}" + ) + padding = node.args[3] if len(node.args) == 4 else [0, 0] + if len(padding) == 1: + padding = [padding[0], padding[0]] + if len(padding) != 2 or padding[0] != padding[1]: + raise ValueError( + f"asymmetric quantized graph MaxPool2D padding for {node.name}" + ) + replacement = QuantizedMaxPool2dOp() + elif isinstance(node, MeanOp): + if ( + len(node.args) != 3 + or list(node.args[1]) != [-1, -2] + or not node.args[2] + ): + continue + replacement = QuantizedGlobalAvgPoolOp() + elif ( + isinstance(node, MulOp) + and len(node.args) == 2 + and all(isinstance(arg, str) for arg in node.args) + ): + if any( + renamed.get(str(arg), str(arg)) not in quantized_values + for arg in node.args + ): + continue + replacement = QuantizedMulOp() + elif ( + isinstance(node, AddOp) + and len(node.args) == 2 + and all(isinstance(arg, str) for arg in node.args) + ): + if any( + renamed.get(str(arg), str(arg)) not in quantized_values + for arg in node.args + ): + continue + replacement = QuantizedAddOp() + else: + continue + if isinstance(replacement, (QuantizedAddOp, QuantizedMulOp)): + users = [ + candidate + for candidate in original_body + if candidate.name not in removed and uses(candidate, node.name) + ] + if (node.trace_meta is not None and node.trace_meta["tensor"]) or any( + isinstance(candidate, ReluOp) and candidate.trace_meta is not None + for candidate in users + ): + continue + if not any( + isinstance( + candidate, + ( + ReluOp, + Conv2dOp, + AddMMOp, + MatmulOp, + MeanOp, + MaxPool2dOp, + AddOp, + MulOp, + CatOp, + ), + ) + for candidate in users + ): + continue + relus = [candidate for candidate in users if isinstance(candidate, ReluOp)] + if relus: + if len(users) != 1 or len(relus) != 1: + raise ValueError( + f"Quantized INT8 elementwise ReLU must be its only user: {node.name}" + ) + activation = relus[0] + result_name = activation.name + renamed[result_name] = node.name + removed.add(result_name) + special[node.name] = { + "node": node, + "replacement": replacement, + "activation": activation, + "result_name": result_name, + "arguments": arguments, + **attributes, + } + quantized_values.add(node.name) + quantized_values.add(result_name) + + for plan in plans.values(): + consumers = [ + candidate + for candidate in plans.values() + if candidate["activation_name"] == plan["result_name"] + ] + plan["output_scale"] = ( + max(candidate["required_input_scale"] for candidate in consumers) + if consumers + else plan["calibrated_output_scale"] + ) + + for item in reversed(list(special.values())): + node = item["node"] + consumers = [ + candidate + for candidate in plans.values() + if renamed.get(candidate["activation_name"], candidate["activation_name"]) + == node.name + ] + downstream_scales = [ + candidate["output_scale"] + for candidate in special.values() + if "output_scale" in candidate + and any( + uses(candidate["node"], source) + for source in (node.name, item["result_name"]) + ) + ] + if not consumers and isinstance(node, (MeanOp, MaxPool2dOp)): + users = [ + candidate + for candidate in original_body + if candidate.name not in removed and uses(candidate, node.name) + ] + if len(users) != 1 or not isinstance(users[0], (ViewOp, ReshapeOp)): + raise ValueError( + f"quantized graph mean has no unique consumer: {node.name}" + ) + consumers = [ + candidate + for candidate in plans.values() + if renamed.get( + candidate["activation_name"], candidate["activation_name"] + ) + == users[0].name + ] + if not consumers and not downstream_scales: + users = [ + candidate.name + for candidate in original_body + if uses(candidate, node.name) + ] + raise ValueError( + f"quantized graph stage has no compute consumer: {node.name}, " + f"args={node.args}, users={users}" + ) + item["output_scale"] = max( + [consumer["required_input_scale"] for consumer in consumers] + + downstream_scales + ) + + for plan in plans.values(): + max_pool_consumers = [ + item + for item in special.values() + if isinstance(item["replacement"], QuantizedMaxPool2dOp) + and renamed.get(str(item["node"].args[0]), str(item["node"].args[0])) + == plan["node"].name + ] + if len(max_pool_consumers) > 1: + raise ValueError(f"multiple MaxPool2D consumers for {plan['node'].name}") + if max_pool_consumers: + plan["output_scale"] = max_pool_consumers[0]["output_scale"] + + changed = True + while changed: + changed = False + for name, item in special.items(): + if not isinstance( + item["replacement"], + ( + QuantizedChannelSliceOp, + QuantizedChannelConcatOp, + QuantizedResizeNearestOp, + QuantizedMaxPool2dOp, + ), + ): + continue + sources = item["arguments"] + if sources is None: + sources = [ + renamed.get(str(item["node"].args[0]), str(item["node"].args[0])) + ] + scales = [item["output_scale"]] + for source in sources: + if source in plans: + scales.append(plans[source]["output_scale"]) + elif source in special: + scales.append(special[source]["output_scale"]) + else: + raise ValueError( + f"Quantized view source has no scale at {name}: {source}" + ) + scale = max(scales) + if item["output_scale"] != scale: + item["output_scale"] = scale + changed = True + for source in sources: + producer = plans[source] if source in plans else special[source] + if producer["output_scale"] != scale: + producer["output_scale"] = scale + changed = True + + value_scale = {} + for plan in plans.values(): + value_scale[plan["node"].name] = plan["output_scale"] + value_scale[plan["result_name"]] = plan["output_scale"] + for name, item in special.items(): + value_scale[name] = item["output_scale"] + value_scale[item["result_name"]] = item["output_scale"] + + for plan in plans.values(): + activation_name = renamed.get(plan["activation_name"], plan["activation_name"]) + input_scale = value_scale.get(activation_name, plan["required_input_scale"]) + dw = plan["dw"] + input_channel_scales = plan.get("input_channel_scales") + if input_channel_scales is not None: + input_channel_scales = np.asarray(input_channel_scales, dtype=np.float32) + if not isinstance( + plan["replacement"], QuantizedDepthwiseConv2dOp + ) or input_channel_scales.shape != (dw.size,): + raise ValueError( + f"per-channel input scale requires matching depthwise Conv: " + f"{plan['node'].name}" + ) + minimum_weight_scale = np.abs(plan["bias"].astype(np.float64)) / ( + (np.iinfo(np.int32).max - plan["product_bound"] - 1) + * input_channel_scales.astype(np.float64) + ) + dw = np.maximum( + dw, + np.nextafter( + minimum_weight_scale.astype(np.float32), np.float32(np.inf) + ), + ) + plan["dw"] = dw + graph.node_table[plan["weight_arg"]]._quantized_weight_scales = dw.copy() + accumulator_input_scale = ( + input_channel_scales + if input_channel_scales is not None + else np.float32(input_scale) + ) + bias_i64 = np.rint( + plan["bias"].astype(np.float64) + / ( + np.asarray(accumulator_input_scale, dtype=np.float64) + * dw.astype(np.float64) + ) + ).astype(np.int64) + overflow = np.flatnonzero( + np.abs(bias_i64) + plan["product_bound"] > np.iinfo(np.int32).max + ) + if overflow.size: + channel = int(overflow[0]) + scale_index = channel if input_channel_scales is not None else 0 + raise ValueError( + f"INT32 accumulator overflow for {plan['node'].name} channel {channel}: " + f"input_scale={float(np.asarray(accumulator_input_scale).reshape(-1)[scale_index])}, " + f"weight_scale={float(dw[channel])}, " + f"bias_i32={int(bias_i64[channel])}, " + f"product_bound={plan['product_bound']}" + ) + + node = plan["node"] + replacement = plan["replacement"] + activation = plan["activation"] + replacement._name = node.name + replacement._arguments = [activation_name, plan["weight_arg"]] + replacement._parents = list(replacement._arguments) + replacement._children = [ + renamed.get(user.name, user.name) for user in plan["users"] + ] + replacement._tensor_meta = node._tensor_meta.copy() + direct_compute_consumers = [ + candidate + for candidate in plans.values() + if candidate["activation_name"] == plan["result_name"] + ] + replacement._final_output = ( + not isinstance(node, Conv2dOp) and not direct_compute_consumers + ) + replacement._tensor_meta["dtype"] = ( + TensorDType.Float32 if replacement._final_output else TensorDType.Int8 + ) + replacement._input_scale = input_scale + replacement._output_scale = plan["output_scale"] + replacement._bias_i32 = bias_i64.astype(np.int32).tolist() + requant_output_scale = np.asarray( + plan.get("lane_input_scales", plan["output_scale"]), + dtype=np.float32, + ) + replacement._requant_scale = ( + np.asarray(accumulator_input_scale, dtype=np.float32) + * dw + / requant_output_scale + ).tolist() + replacement._dequant_scale = ( + np.asarray(accumulator_input_scale, dtype=np.float32) * dw + ).tolist() + replacement._activation = plan["activation_kind"] + if replacement._activation == 2: + raw = np.arange(256, dtype=np.int16) + signed = np.where(raw < 128, raw, raw - 256).astype(np.float32) + if "lane_input_scales" in plan: + x = plan["lane_input_scales"][:, None] * signed[None, :] + else: + x = signed * np.float32(plan["output_scale"]) + if plan["lut_kind"] == "hardswish": + y = x * np.clip(x + np.float32(3.0), 0.0, 6.0) / np.float32(6.0) + elif plan["lut_kind"] == "hardsigmoid": + y = np.clip(x + np.float32(3.0), 0.0, 6.0) / np.float32(6.0) + elif plan["lut_kind"] == "silu": + y = x / ( + np.float32(1.0) + + np.exp(-np.clip(x, np.float32(-80.0), np.float32(80.0))) + ) + else: + raise ValueError(f"missing LUT function for {node.name}") + lut_output_scale = np.asarray( + plan.get("lane_output_scales", plan["output_scale"]), + dtype=np.float32, + ) + if lut_output_scale.ndim == 1: + lut_output_scale = lut_output_scale[:, None] + replacement._lut_i8 = ( + np.clip(np.rint(y / lut_output_scale), -128, 127) + .astype(np.int8) + .reshape(-1) + .tolist() + ) + replacement._lane_output_scales = plan.get( + "lane_output_scales", np.empty(0, dtype=np.float32) + ).tolist() + else: + replacement._lut_i8 = [0] + replacement._lane_output_scales = [] + replacement.trace_meta = node.trace_meta + if isinstance(replacement, (QuantizedConv2dOp, QuantizedDepthwiseConv2dOp)): + replacement._input_shape = list( + plan["activation_node"].tensor_meta["shape"] + ) + replacement._weight_shape = plan["weight_shape"] + replacement._output_shape = list(node.tensor_meta["shape"]) + replacement._stride = int(node.args[3][0]) + replacement._padding = int(plan["padding"][0]) + for name, item in special.items(): + node = item["node"] + replacement = item["replacement"] + args = item["arguments"] + if args is None: + args = [ + renamed.get(str(arg), str(arg)) + for arg in node.args + if isinstance(arg, str) + ] + if isinstance(replacement, QuantizedMaxPool2dOp): + if len(args) != 1 or args[0] not in value_scale: + raise ValueError(f"invalid Quantized MaxPool2D input for {name}") + replacement._input_scale = value_scale[args[0]] + replacement._output_scale = replacement._input_scale + if "input_shape" in item: + replacement._input_shape = item["input_shape"] + replacement._output_shape = item["output_shape"] + replacement._kernel = item["kernel"] + replacement._stride = item["stride"] + replacement._padding = item["padding"] + else: + replacement._input_shape = list( + graph.node_table[str(node.args[0])].tensor_meta["shape"] + ) + replacement._output_shape = list(node.tensor_meta["shape"]) + replacement._kernel = int(node.args[1][0]) + replacement._stride = int(node.args[2][0]) + padding = node.args[3] if len(node.args) == 4 else [0, 0] + replacement._padding = int(padding[0]) + replacement._final_output = False + elif isinstance(replacement, QuantizedChannelSliceOp): + if len(args) != 1 or args[0] not in value_scale: + raise ValueError(f"invalid Quantized channel-slice input for {name}") + replacement._offset = item["offset"] + replacement._output_shape = item["output_shape"] + elif isinstance(replacement, QuantizedChannelConcatOp): + if any(arg not in value_scale for arg in args): + raise ValueError(f"invalid Quantized channel-concat input for {name}") + replacement._output_shape = item["output_shape"] + elif isinstance(replacement, QuantizedResizeNearestOp): + if len(args) != 1 or args[0] not in value_scale: + raise ValueError(f"invalid Quantized resize input for {name}") + replacement._output_shape = item["output_shape"] + replacement._scale_h = item["scale_h"] + replacement._scale_w = item["scale_w"] + elif isinstance(replacement, QuantizedGlobalAvgPoolOp): + if len(args) != 1 or args[0] not in value_scale: + raise ValueError(f"invalid Quantized global-average input for {name}") + replacement._input_scale = value_scale[args[0]] + else: + if len(args) != 2 or any(arg not in value_scale for arg in args): + raise ValueError(f"invalid Quantized INT8 elementwise input for {name}") + replacement._lhs_scale = value_scale[args[0]] + replacement._rhs_scale = value_scale[args[1]] + replacement._activation = 1 if item["activation"] is not None else 0 + replacement._name = name + replacement._arguments = args + replacement._parents = list(args) + replacement._children = [ + renamed.get(child, child) + for child in ( + item["activation"]._children + if item["activation"] is not None + else node._children + ) + ] + replacement._tensor_meta = node._tensor_meta.copy() + replacement._tensor_meta["dtype"] = TensorDType.Int8 + replacement._output_scale = item["output_scale"] + replacement.trace_meta = node.trace_meta + + replacements = {name: plan["replacement"] for name, plan in plans.items()} | { + name: item["replacement"] for name, item in special.items() + } + + def canonical(argument): + if isinstance(argument, list): + return [canonical(value) for value in argument] + if isinstance(argument, tuple): + return tuple(canonical(value) for value in argument) + while isinstance(argument, str) and argument in renamed: + argument = renamed[argument] + return argument + + graph._body = [ + replacements.get(node.name, node) + for node in original_body + if node.name not in removed + ] + graph.node_table = {node.name: node for node in graph.body} + for node in graph.body: + node._arguments = canonical(node.args) + node._parents = [] + node._children = [] + for node in graph.body: + pending = list(node.args) + while pending: + argument = pending.pop(0) + if isinstance(argument, (list, tuple)): + pending[:0] = argument + elif isinstance(argument, str) and argument in graph.node_table: + if argument not in node._parents: + node._parents.append(argument) + graph.node_table[argument]._children.append(node.name) + for node in replacements.values(): + if isinstance( + node, (QuantizedConv2dOp, QuantizedDepthwiseConv2dOp, QuantizedMatmulOp) + ): + node._final_output = any(name not in replacements for name in node.children) + node._tensor_meta["dtype"] = ( + TensorDType.Float32 if node._final_output else TensorDType.Int8 + ) + elif isinstance(node, QuantizedMaxPool2dOp): + node._final_output = any(name not in replacements for name in node.children) + graph._fake_params = [graph.body.index(node) for node in param_nodes] + graph._inputs = [graph.body.index(node) for node in input_node_list] + from buddy.compiler.graph.transform.quantization.quantize import sort_graph + + sort_graph(graph) + + +def quantize_model_graph( + graph, + params, + names: list[str], + output_dir: Path, + model_name: str, + calibration: dict, + *, + weight_scales=None, + assets: dict[str, bytes] | None = None, +) -> None: + if len(graph.params) != len(params) or len(names) != len(params): + raise ValueError("parameter metadata does not match imported graph") + param_nodes = list(graph.params) + parameter_names, arrays = {}, {} + for node, param, name in zip(param_nodes, params, names): + array = param.detach().cpu().numpy().astype(np.float32) + parameter_names[node.name] = name + arrays[name] = array + + if weight_scales is None: + from stack.compiler.frontend.quant.transforms.symmetric import ( + quantize_symmetric, + ) + + weight_scales = { + name: quantize_symmetric(array, [0])[1] + for name, array in arrays.items() + if name.endswith(".weight") and array.ndim >= 2 + } + + _quantize_operators( + graph, + parameter_names, + arrays, + weight_scales, + calibration, + ) + + tensors = [] + weights, fp_params, scales = [], [], [] + weight_off = param_off = scale_off = 0 + for index, (node, name) in enumerate(zip(param_nodes, names)): + array = arrays[name] + if getattr(node, "_quantized_weight", False): + axes = [0] + dw = np.asarray(node._quantized_weight_scales, dtype=np.float32).reshape(-1) + if ( + dw.size != array.shape[0] + or not np.all(np.isfinite(dw)) + or np.any(dw <= 0) + ): + raise ValueError(f"invalid adjusted weight scales for {name}") + q = np.clip( + np.rint(array / dw.reshape((-1,) + (1,) * (array.ndim - 1))), + -128, + 127, + ).astype(np.int8) + node.tensor_meta["dtype"] = TensorDType.Int8 + node.tensor_meta["shape"] = list(q.shape) + params[index] = torch.from_numpy(q.copy()) + scale_bytes = dw.tobytes() + tensors.append( + QuantTensor( + name, + list(array.shape), + list(q.shape), + "i8", + axes, + weight_off, + q.nbytes, + scale_off, + len(scale_bytes), + ) + ) + weights.append(q.tobytes()) + scales.append(scale_bytes) + weight_off += q.nbytes + scale_off += len(scale_bytes) + else: + raw = array.tobytes() + tensors.append( + QuantTensor( + name, + list(array.shape), + list(array.shape), + "f32", + [], + param_off, + len(raw), + 0, + 0, + ) + ) + fp_params.append(raw) + param_off += len(raw) + package = RaxQuantPackage( + tensors, + b"".join(weights), + b"".join(fp_params), + b"".join(scales), + assets if assets is not None else {}, + ) + output_dir.mkdir(parents=True, exist_ok=True) + write_payload(package, output_dir / f"{model_name}.payload") + (output_dir / "weights.bin").write_bytes(package.weights) + (output_dir / "params.f32").write_bytes(package.params_f32) + (output_dir / "scales.bin").write_bytes(package.scales_f32) diff --git a/waste/stack/compiler/frontend/quant/mxfp8_cache.py b/waste/stack/compiler/frontend/quant/mxfp8_cache.py new file mode 100644 index 000000000..868920885 --- /dev/null +++ b/waste/stack/compiler/frontend/quant/mxfp8_cache.py @@ -0,0 +1,70 @@ +import torch + + +def _shape(values): + if values.ndim < 1 or values.shape[-1] == 0 or values.shape[-1] % 32: + raise ValueError("MXFP8 requires a positive last dimension divisible by 32") + return (*values.shape[:-1], values.shape[-1] // 32) + + +@torch.library.custom_op("buckyball::mxfp8_encode", mutates_args=()) +def encode(values: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + shape = _shape(values) + if values.dtype != torch.float32 or not torch.isfinite(values).all(): + raise ValueError("MXFP8 requires finite FP32 values") + blocks = values.reshape(*shape, 32) + maximum = blocks.abs().amax(-1) + exponent = torch.where(maximum == 0, 0, torch.frexp(maximum)[1] - 9) + exponent = exponent + (torch.ldexp(maximum, -exponent) > 448) + exponent = exponent.clamp(-127, 119) + scaled = torch.ldexp(blocks, -exponent[..., None]).contiguous() + bits = scaled.view(torch.int32).to(torch.int64) & 0xFFFFFFFF + sign = (bits >> 24) & 128 + element_exponent = ((bits >> 23) & 255) - 120 + fraction = bits & 0x7FFFFF + high, low = fraction >> 20, fraction & 0xFFFFF + rounded = high + ((low > 0x80000) | ((low == 0x80000) & ((high & 1) != 0))) + codes = torch.where( + element_exponent <= 0, + (scaled.abs() * 512).round().to(torch.int64), + element_exponent * 8 + rounded, + ) + codes = (codes.clamp_max(126) | sign).to(torch.int8).reshape(values.shape) + return codes, (exponent + 127).to(torch.int8) + + +@encode.register_fake +def _encode_fake(values): + shape = _shape(values) + return values.new_empty(values.shape, dtype=torch.int8), values.new_empty( + shape, dtype=torch.int8 + ) + + +@torch.library.custom_op("buckyball::mxfp8_decode", mutates_args=()) +def decode(codes: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: + shape = _shape(codes) + if codes.dtype != torch.int8 or scales.dtype != torch.int8 or scales.shape != shape: + raise ValueError("MXFP8 requires matching INT8 codes and scales") + unsigned = codes.to(torch.int16) & 255 + scale = scales.to(torch.int16) & 255 + if torch.any((unsigned & 127) == 127) or torch.any(scale == 255): + raise ValueError("MXFP8 NaN encoding") + exponent, fraction = (unsigned >> 3) & 15, (unsigned & 7).float() + elements = torch.where( + exponent == 0, fraction / 512, torch.ldexp(8 + fraction, exponent - 10) + ) + elements = torch.copysign(elements, torch.where((unsigned & 128) != 0, -1.0, 1.0)) + result = torch.ldexp(elements.reshape(*shape, 32), scale[..., None] - 127).reshape( + codes.shape + ) + if not torch.isfinite(result).all(): + raise ValueError("MXFP8 decoded value overflows FP32") + return result + + +@decode.register_fake +def _decode_fake(codes, scales): + if scales.shape != _shape(codes): + raise ValueError("MXFP8 code/scale shape mismatch") + return codes.new_empty(codes.shape, dtype=torch.float32) diff --git a/waste/stack/compiler/frontend/quant/native_mxfp8.py b/waste/stack/compiler/frontend/quant/native_mxfp8.py new file mode 100644 index 000000000..0de29635b --- /dev/null +++ b/waste/stack/compiler/frontend/quant/native_mxfp8.py @@ -0,0 +1,396 @@ +"""MXFP8 storage and offline activation calibration for native CPU/GPU models.""" + +import torch +from torch import nn + +from stack.compiler.frontend.quant import mxfp8_cache + + +@torch.library.custom_op("buddy_native::mxfp8_encode_fixed", mutates_args=()) +def encode_fixed(values: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: + exponent = (scales.to(torch.int16) & 255) - 127 + blocks = values.reshape(*values.shape[:-1], values.shape[-1] // 32, 32) + scaled = torch.ldexp(blocks, -exponent[..., None]).contiguous() + magnitude = scaled.abs().clamp_max(448) + bits = magnitude.view(torch.int32) + element_exponent = (bits >> 23) - 120 + fraction = bits & 0x7FFFFF + high, low = fraction >> 20, fraction & 0xFFFFF + rounded = high + ((low > 0x80000) | ((low == 0x80000) & ((high & 1) != 0))) + codes = torch.where( + element_exponent <= 0, + (magnitude * 512).round().to(torch.int32), + element_exponent * 8 + rounded, + ) + sign = (scaled.view(torch.int32) >> 24) & 128 + return (codes | sign).to(torch.int8).reshape(values.shape) + + +@encode_fixed.register_fake +def _encode_fake(values, scales): + return torch.empty_like(values, dtype=torch.int8) + + +@torch.library.custom_op("buddy_native::mxfp8_decode", mutates_args=()) +def decode(codes: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: + return mxfp8_cache.decode( + codes, + scales.expand(*codes.shape[:-1], codes.shape[-1] // 32).contiguous(), + ) + + +@decode.register_fake +def _decode_fake(codes, scales): + return torch.empty_like(codes, dtype=torch.float32) + + +@torch.library.custom_op("buddy_native::mxfp8_linear", mutates_args=()) +def matmul( + activation: torch.Tensor, + activation_scales: torch.Tensor, + weights: torch.Tensor, + weight_scales: torch.Tensor, +) -> torch.Tensor: + # Round the dot product once, before the next MXFP8 quantization boundary. + return nn.functional.linear( + decode(activation, activation_scales).double(), + decode(weights, weight_scales).double(), + ).float() + + +@matmul.register_fake +def _matmul_fake(activation, activation_scales, weights, weight_scales): + return activation.new_empty( + (activation.shape[0], weights.shape[0]), dtype=torch.float32 + ) + + +@torch.library.custom_op("buddy_native::attention_matmul", mutates_args=()) +def attention_matmul(lhs: torch.Tensor, rhs: torch.Tensor) -> torch.Tensor: + return torch.bmm(lhs.double(), rhs.double()).float() + + +@attention_matmul.register_fake +def _attention_matmul_fake(lhs, rhs): + return lhs.new_empty((lhs.shape[0], lhs.shape[1], rhs.shape[2])) + + +@torch.library.custom_op("buddy_native::attention_softmax", mutates_args=()) +def attention_softmax(value: torch.Tensor, axis: int) -> torch.Tensor: + return torch.softmax(value.double(), dim=axis).float() + + +@attention_softmax.register_fake +def _attention_softmax_fake(value, axis): + return torch.empty_like(value) + + +def attention( + module, + query, + key, + value, + attention_mask, + scaling, + dropout=0.0, + softcap=None, + **kwargs, +): + if dropout != 0.0: + raise ValueError("MXFP8 attention is an inference kernel") + batch, heads, length, width = query.shape + keys = key.shape[2] + groups = module.num_key_value_groups + key = ( + key[:, :, None] + .expand(batch, key.shape[1], groups, keys, width) + .reshape(batch * heads, keys, width) + ) + value = ( + value[:, :, None] + .expand(batch, value.shape[1], groups, keys, width) + .reshape(batch * heads, keys, width) + ) + query = query.reshape(batch * heads, length, width) + scores = ( + attention_matmul(query, key.transpose(1, 2)).reshape(batch, heads, length, keys) + * scaling + ) + if softcap is not None: + scores = torch.tanh(scores / softcap) * softcap + if attention_mask is not None: + scores = scores + attention_mask + probabilities = attention_softmax(scores, -1) + output = attention_matmul(probabilities.reshape(batch * heads, length, keys), value) + return ( + output.reshape(batch, heads, length, width).transpose(1, 2).contiguous(), + probabilities, + ) + + +def register(compiler): + from buddy.compiler.graph.operation import Op, OpType + from stack.compiler.frontend.ops import reductions + from stack.compiler.frontend.ops.native_mxfp8 import codec, concat + from stack.compiler.frontend.ops.native_mxfp8 import matmul as lower_matmul + from stack.compiler.frontend.ops.reductions import bmm, gelu, mean, softmax, tanh + + class NativeMXFP8EncodeOp(Op): + def __init__(self): + super().__init__() + self._op_type = OpType.BroadcastType + + class NativeMXFP8DecodeOp(Op): + def __init__(self): + super().__init__() + self._op_type = OpType.BroadcastType + + class NativeMXFP8MatmulOp(Op): + def __init__(self): + super().__init__() + self._op_type = OpType.ReduceType + + class NativeAttentionMatmulOp(Op): + pass + + class NativeAttentionSoftmaxOp(Op): + pass + + compiler._ops_map.update( + { + "buddy_native.mxfp8_encode_fixed.default": NativeMXFP8EncodeOp, + "buddy_native.mxfp8_decode.default": NativeMXFP8DecodeOp, + "buddy_native.mxfp8_linear.default": NativeMXFP8MatmulOp, + "buddy_native.attention_matmul.default": NativeAttentionMatmulOp, + "buddy_native.attention_softmax.default": NativeAttentionSoftmaxOp, + } + ) + compiler._ops_registry.update( + { + "NativeMXFP8EncodeOp": codec, + "NativeMXFP8DecodeOp": codec, + "CatOp": concat, + "NativeAttentionMatmulOp": reductions.bmm, + "NativeAttentionSoftmaxOp": reductions.softmax, + "MeanOp": mean, + "BatchMatmulOp": bmm, + "SoftmaxOp": softmax, + "GeluOp": gelu, + "TanhOp": tanh, + "NativeMXFP8MatmulOp": lower_matmul, + } + ) + + +def scale_for(maximum): + exponent = torch.where(maximum == 0, 0, torch.frexp(maximum)[1] - 9) + exponent += torch.ldexp(maximum, -exponent) > 448 + return (exponent.clamp(-127, 119) + 127).to(torch.int8) + + +class Linear(nn.Module): + def __init__(self, source): + super().__init__() + self.in_features, self.out_features = ( + source.in_features, + source.out_features, + ) + codes, scales = mxfp8_cache.encode(source.weight.detach().float()) + self.register_buffer("codes", codes) + self.register_buffer("scales", scales) + self.register_buffer( + "activation_scales", + torch.full((1, self.in_features // 32), 127, dtype=torch.int8), + ) + self.bias = source.bias + self.calibrating = True + self.register_buffer( + "maximum", torch.zeros(self.in_features // 32), persistent=False + ) + + def forward(self, values): + shape = values.shape + values = values.reshape(-1, self.in_features) + if self.calibrating: + maximum = ( + values.detach() + .reshape(-1, self.in_features // 32, 32) + .abs() + .amax((0, 2)) + ) + self.maximum = torch.maximum(self.maximum, maximum) + result = nn.functional.linear( + values, decode(self.codes, self.scales), self.bias + ) + else: + result = matmul( + encode_fixed(values, self.activation_scales), + self.activation_scales, + self.codes, + self.scales, + ) + if self.bias is not None: + result = result + self.bias + return result.reshape(*shape[:-1], self.out_features) + + +class Embedding(nn.Module): + def __init__(self, source, scale=1.0): + super().__init__() + self.scale = scale + codes, scales = mxfp8_cache.encode(source.weight.detach().float()) + self.register_buffer("codes", codes) + self.register_buffer("scales", scales) + self.num_embeddings, self.embedding_dim = ( + source.num_embeddings, + source.embedding_dim, + ) + + def forward(self, tokens): + return ( + decode( + nn.functional.embedding(tokens, self.codes), + nn.functional.embedding(tokens, self.scales), + ) + * self.scale + ) + + +def prepare(model, *, embedding_scales=None): + """Replace supported weight consumers before collecting activation ranges.""" + embedding_scales = {} if embedding_scales is None else embedding_scales + for name, module in list(model.named_modules()): + if not name: + continue + parent_name, _, child_name = name.rpartition(".") + parent = model.get_submodule(parent_name) + if type(module) is nn.Linear: + setattr(parent, child_name, Linear(module)) + elif isinstance(module, nn.Embedding): + setattr( + parent, + child_name, + Embedding(module, embedding_scales.get(name, 1.0)), + ) + from transformers.masking_utils import ALL_MASK_ATTENTION_FUNCTIONS + from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS + + ALL_ATTENTION_FUNCTIONS.register("buddy_mxfp8", attention) + ALL_MASK_ATTENTION_FUNCTIONS.register( + "buddy_mxfp8", ALL_MASK_ATTENTION_FUNCTIONS["eager"] + ) + model.set_attn_implementation("buddy_mxfp8") + return model + + +def finish_calibration(model): + scales = {} + for name, module in model.named_modules(): + if isinstance(module, Linear): + module.activation_scales.copy_(scale_for(module.maximum)[None, :]) + module.calibrating = False + scales[name] = (module.activation_scales.to(torch.int16) & 255).tolist() + del module.maximum + return scales + + +class CacheCalibration: + """Observe per-head KV blocks while the model executes calibration samples.""" + + def __init__(self): + self.maximum = {} + self.key_minimum = {} + self.key_maximum = {} + + def update(self, key, value, layer_idx): + maximum = torch.stack( + [ + t.detach() + .reshape(*t.shape[:-1], t.shape[-1] // 32, 32) + .abs() + .amax((0, 2, 4)) + for t in (key, value) + ] + ) + low, high = key.detach().amin((0, 2)), key.detach().amax((0, 2)) + if layer_idx in self.maximum: + low = torch.minimum(low, self.key_minimum[layer_idx]) + high = torch.maximum(high, self.key_maximum[layer_idx]) + maximum = torch.maximum(maximum, self.maximum[layer_idx]) + self.maximum[layer_idx] = maximum + self.key_minimum[layer_idx], self.key_maximum[layer_idx] = low, high + return key, value + + def parameters(self): + centers = {} + for layer, maximum in self.maximum.items(): + low, high = self.key_minimum[layer], self.key_maximum[layer] + centers[layer] = (low + high) * 0.5 + radius = (high - low) * 0.5 + maximum[0] = radius.reshape(*radius.shape[:-1], -1, 32).amax(-1) + return self.maximum, centers + + +class Cache: + """Explicit fixed-capacity cache; only MXFP8 bytes persist between calls.""" + + def __init__(self, codes, positions, layouts, scales, centers, capacity): + self.codes = codes + self.positions = positions + self.layouts = layouts + self.scales = scales + self.centers = centers + self.capacity = capacity + self.updated = [] + + def update(self, key, value, layer_idx): + offset, heads, width = self.layouts[layer_idx] + size = 2 * heads * self.capacity * width + old = self.codes[offset : offset + size].reshape( + 2, 1, heads, self.capacity, width + ) + scales = self.scales[layer_idx] + # A shared K translation cancels in softmax(Q K^T). Keep it centered. + key = key - self.centers[layer_idx].reshape(1, heads, 1, width) + keys = encode_fixed(key, scales[0].reshape(1, heads, 1, width // 32)) + values = encode_fixed(value, scales[1].reshape(1, heads, 1, width // 32)) + current = torch.stack((keys, values)) + update = torch.arange(self.capacity, device=key.device)[ + None, None, None, :, None + ] == self.positions.reshape(1, 1, 1, 1, 1) + packed = torch.where(update, current, old) + self.updated.append(packed.flatten()) + return decode(packed[0], scales[0].reshape(1, heads, 1, width // 32)), decode( + packed[1], scales[1].reshape(1, heads, 1, width // 32) + ) + + +def calibrate(model, dataset, *, samples, batch_size): + from torch.utils.data import DataLoader, Subset + + if not 0 < samples <= len(dataset): + raise ValueError("Calibration sample count must be within the dataset size") + indices = torch.randperm(len(dataset), generator=torch.Generator().manual_seed(0))[ + :samples + ] + loader = DataLoader(Subset(dataset, indices.tolist()), batch_size=batch_size) + cache = CacheCalibration() + completed = 0 + with torch.no_grad(): + for tokens in loader: + positions = torch.arange(tokens.shape[1])[None, :] + visible = positions[:, :, None] >= positions[:, None, :] + mask = torch.where(visible[:, None], 0.0, torch.finfo(torch.float32).min) + model( + input_ids=tokens, + position_ids=positions, + attention_mask={ + "full_attention": mask, + "sliding_attention": mask, + }, + past_key_values=cache, + use_cache=True, + ) + completed += tokens.shape[0] + print(f"Calibrated {completed}/{samples} sequences", flush=True) + return finish_calibration(model), cache diff --git a/arch/src/main/scala/framework/root/root-tile/src/main/resources/README.md b/waste/stack/compiler/frontend/quant/offline/__init__.py similarity index 100% rename from arch/src/main/scala/framework/root/root-tile/src/main/resources/README.md rename to waste/stack/compiler/frontend/quant/offline/__init__.py diff --git a/waste/stack/compiler/frontend/quant/offline/activations/__init__.py b/waste/stack/compiler/frontend/quant/offline/activations/__init__.py new file mode 100644 index 000000000..f28972127 --- /dev/null +++ b/waste/stack/compiler/frontend/quant/offline/activations/__init__.py @@ -0,0 +1,6 @@ +from stack.compiler.frontend.quant.offline.activations.calibrate import ( + calibrate_activations, + calibrate_layer_ranges, +) + +__all__ = ["calibrate_activations", "calibrate_layer_ranges"] diff --git a/waste/stack/compiler/frontend/quant/offline/activations/calibrate.py b/waste/stack/compiler/frontend/quant/offline/activations/calibrate.py new file mode 100644 index 000000000..3d9766a63 --- /dev/null +++ b/waste/stack/compiler/frontend/quant/offline/activations/calibrate.py @@ -0,0 +1,89 @@ +"""Calibrate symmetric INT8 scales from model inputs.""" + +from collections.abc import Mapping +from copy import deepcopy + +import torch +from torch.utils.data import DataLoader, Subset + +from stack.compiler.frontend.quant.offline.weights import dequantize_weights + + +def calibrate_activations( + model, + quantized, + dataset, + *, + samples, + layers, + batch_size=64, + granularities=None, +): + """Calibrate selected module inputs on a copy using dequantized weights. + + The model supplies its dataset and graph-node-to-module mapping. Sampling + and batching are fixed here so repeated calibration is deterministic. + """ + model, loader = _calibration_batches(model, quantized, dataset, samples, batch_size) + modules = dict(model.named_modules()) + maxima = dict.fromkeys(layers) + observed = set() + handles = [] + selected = [(name, modules[layer]) for name, layer in layers.items()] + for name, module in selected: + + def record(module, inputs, *, name=name): + values = inputs[0].detach() + if values.dtype != torch.float32 or not torch.isfinite(values).all(): + raise ValueError(f"{name}: expected finite FP32 activations") + if granularities is not None and granularities[name] == "channel": + maximum = values.abs().amax(dim=(0, 2, 3)).cpu() + else: + maximum = values.abs().amax().cpu() + maxima[name] = ( + maximum + if maxima[name] is None + else torch.maximum(maxima[name], maximum) + ) + observed.add(name) + + handles.append(module.register_forward_pre_hook(record)) + try: + with torch.no_grad(): + for inputs, _ in loader: + if isinstance(inputs, Mapping): + model(**inputs) + else: + model(inputs) + finally: + for handle in handles: + handle.remove() + if observed != set(layers) or not layers: + raise ValueError("Calibration did not observe every selected layer") + result = {} + for name, maximum in maxima.items(): + scale = torch.where(maximum == 0, 1.0, maximum / 127.0) + result[name] = float(scale) if scale.ndim == 0 else scale.tolist() + return result + + +def _calibration_batches(model, quantized, dataset, samples, batch_size=64): + model = deepcopy(model).eval() + if quantized is not None: + restored = dequantize_weights(quantized) + model.load_state_dict({name: restored[name] for name in model.state_dict()}) + if not 0 < samples <= len(dataset): + raise ValueError("Calibration sample count must be within the dataset size") + indices = torch.randperm(len(dataset), generator=torch.Generator().manual_seed(0))[ + :samples + ] + loader = DataLoader(Subset(dataset, indices.tolist()), batch_size=batch_size) + return model, loader + + +def calibrate_layer_ranges(model, dataset, *, samples, batch_size=64, quantized=None): + """Calibrate integer backend input/output ranges on deterministic dataset samples.""" + from stack.compiler.frontend.quant.transforms.activation import calibrate_layers + + model, loader = _calibration_batches(model, quantized, dataset, samples, batch_size) + return calibrate_layers(model, (inputs for inputs, _ in loader)) diff --git a/waste/stack/compiler/frontend/quant/offline/weights/__init__.py b/waste/stack/compiler/frontend/quant/offline/weights/__init__.py new file mode 100644 index 000000000..95a10bb97 --- /dev/null +++ b/waste/stack/compiler/frontend/quant/offline/weights/__init__.py @@ -0,0 +1,6 @@ +from stack.compiler.frontend.quant.offline.weights.quantize import ( + dequantize_weights, + quantize_weights, +) + +__all__ = ["dequantize_weights", "quantize_weights"] diff --git a/waste/stack/compiler/frontend/quant/offline/weights/quantize.py b/waste/stack/compiler/frontend/quant/offline/weights/quantize.py new file mode 100644 index 000000000..306316b1d --- /dev/null +++ b/waste/stack/compiler/frontend/quant/offline/weights/quantize.py @@ -0,0 +1,85 @@ +"""Named offline weight artifacts, independent of backend memory layouts.""" + +from collections.abc import Mapping + +import torch + +from stack.compiler.frontend.quant.offline.weights.strategy.fp32toint8 import ( + quantize_int8, +) + + +def quantize_weights( + parameters: Mapping[str, torch.Tensor], + weights: Mapping[str, Mapping], +) -> dict: + """Return quantized weights, scales, and channel axes without file I/O. + + Each weight specifies source_dtype, target_dtype, method, and axis. + Unselected parameters and buffers retain their values and dtypes. + Tensor shapes and element order are unchanged. + """ + if not weights: + raise ValueError("No weights selected for quantization") + unknown = weights.keys() - parameters.keys() + if unknown: + raise ValueError(f"Unknown weight names: {sorted(unknown)}") + + tensors, scales, channel_axes = {}, {}, {} + error_energy = 0.0 + elements = 0 + for name, parameter in parameters.items(): + if name in weights: + config = weights[name] + if set(config) != { + "source_dtype", + "target_dtype", + "method", + "axis", + }: + raise ValueError( + f"{name}: expected source_dtype, target_dtype, method, and axis" + ) + scheme = ( + config["source_dtype"], + config["target_dtype"], + config["method"], + ) + if scheme != ("float32", "int8", "symmetric"): + raise ValueError(f"{name}: unsupported weight quantization {scheme}") + axis = config["axis"] + tensors[name], scales[name] = quantize_int8(parameter, axis) + channel_axes[name] = None if axis is None else axis % parameter.ndim + scale_shape = [1] * parameter.ndim + if axis is not None: + scale_shape[channel_axes[name]] = parameter.shape[axis] + restored = tensors[name].float() * scales[name].reshape(scale_shape) + original = parameter.detach().cpu().double() + error = original - restored.double() + error_energy += error.square().sum().item() + elements += parameter.numel() + else: + tensors[name] = parameter.detach().cpu().contiguous() + + print(f"Weight quantization MSE: {error_energy / elements:.6e}", flush=True) + return { + "format": "buddy.offline.symmetric_int8", + "version": 1, + "tensors": tensors, + "scales": scales, + "axes": channel_axes, + } + + +def dequantize_weights(artifact: dict) -> dict[str, torch.Tensor]: + """Reconstruct FP32 weights for offline accuracy evaluation.""" + if artifact["format"] != "buddy.offline.symmetric_int8" or artifact["version"] != 1: + raise ValueError("Unsupported offline weight artifact") + tensors = dict(artifact["tensors"]) + for name, axis in artifact["axes"].items(): + values = tensors[name] + shape = [1] * values.ndim + if axis is not None: + shape[axis] = values.shape[axis] + tensors[name] = values.float() * artifact["scales"][name].reshape(shape) + return tensors diff --git a/arch/src/main/scala/framework/rvv/src/csrc/README.md b/waste/stack/compiler/frontend/quant/offline/weights/strategy/__init__.py similarity index 100% rename from arch/src/main/scala/framework/rvv/src/csrc/README.md rename to waste/stack/compiler/frontend/quant/offline/weights/strategy/__init__.py diff --git a/waste/stack/compiler/frontend/quant/offline/weights/strategy/fp32toint8.py b/waste/stack/compiler/frontend/quant/offline/weights/strategy/fp32toint8.py new file mode 100644 index 000000000..6c2096815 --- /dev/null +++ b/waste/stack/compiler/frontend/quant/offline/weights/strategy/fp32toint8.py @@ -0,0 +1,33 @@ +"""Offline symmetric INT8 quantization of FP32 weight tensors.""" + +import torch + + +def quantize_int8( + weight: torch.Tensor, axis: int | None +) -> tuple[torch.Tensor, torch.Tensor]: + """Return INT8 values and FP32 scales; axis=None selects one tensor scale.""" + if weight.dtype != torch.float32: + raise ValueError("INT8 weight quantization requires FP32 input") + if not weight.numel() or not torch.isfinite(weight).all(): + raise ValueError("Weights must be nonempty and finite") + if axis is not None and not -weight.ndim <= axis < weight.ndim: + raise ValueError("Quantization axis is outside the weight dimensions") + + weight = weight.detach().cpu() + if axis is None: + maximum = weight.abs().amax() + scale_shape = [1] * weight.ndim + else: + axis %= weight.ndim + maximum = weight.movedim(axis, 0).reshape(weight.shape[axis], -1).abs().amax(1) + scale_shape = [1] * weight.ndim + scale_shape[axis] = weight.shape[axis] + + # A zero channel is represented exactly by q=0 and scale=1. + scales = torch.where(maximum == 0, torch.ones_like(maximum), maximum / 127) + if (scales == 0).any(): + raise ValueError("Nonzero weight scale cannot be represented in FP32") + values = torch.round(weight / scales.reshape(scale_shape)) + values = values.clamp(-127, 127).to(torch.int8).contiguous() + return values, scales.contiguous() diff --git a/waste/stack/compiler/frontend/quant/rax.py b/waste/stack/compiler/frontend/quant/rax.py new file mode 100644 index 000000000..ded598f09 --- /dev/null +++ b/waste/stack/compiler/frontend/quant/rax.py @@ -0,0 +1,303 @@ +from __future__ import annotations + +import json +import math +import shutil +import struct +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from stack.compiler.frontend.quant.bwq import BwqPackage, validate_bwq + + +def scale_count(shape: list[int], axes: list[int]) -> int: + return math.prod(shape[axis] for axis in axes) + + +@dataclass +class QuantTensor: + name: str + shape: list[int] + payload_shape: list[int] + storage: str + axes: list[int] + payload_off: int + payload_len: int + scale_off: int + scale_len: int + layout: dict | None = None + + +@dataclass +class RaxQuantPackage: + tensors: list[QuantTensor] + weights: bytes + params_f32: bytes + scales_f32: bytes + assets: dict[str, bytes] + + +def rax_from_bwq(pkg: BwqPackage) -> RaxQuantPackage: + validate_bwq(pkg) + tensors: list[QuantTensor] = [] + weights: list[bytes] = [] + scales: list[bytes] = [] + weight_off = scale_off = 0 + for source in pkg.tensors: + if source.storage != "i8" or source.axes not in ([], [0]): + raise ValueError( + f"RAX W8A8 requires i8 tensor/channel weight: {source.name}" + ) + weight = pkg.weights[source.weight_off : source.weight_off + source.weight_len] + raw_scales = pkg.scales[source.scale_off : source.scale_off + source.scale_len] + raw_count = 1 if not source.axes else source.shape[0] + values = struct.unpack(f"<{raw_count}f", raw_scales) + padded_count = scale_count(source.shape, source.axes) + padded = values + (1.0,) * (padded_count - raw_count) + scale = struct.pack(f"<{padded_count}f", *padded) + tensors.append( + QuantTensor( + source.name, + source.shape, + source.shape, + source.storage, + source.axes, + weight_off, + len(weight), + scale_off, + len(scale), + ) + ) + weights.append(weight) + scales.append(scale) + weight_off += len(weight) + scale_off += len(scale) + return RaxQuantPackage(tensors, b"".join(weights), b"", b"".join(scales), {}) + + +def _numel(shape: list[int], name: str) -> int: + n = 1 + for d in shape: + if d <= 0: + raise ValueError(f"bad shape for {name}") + n *= d + return n + + +def validate_rax_quant(pkg: RaxQuantPackage) -> None: + if not pkg.tensors: + raise ValueError("empty tensor list") + names: set[str] = set() + weight_end = param_end = scale_end = 0 + for tensor in pkg.tensors: + if tensor.name in names: + raise ValueError(f"duplicate tensor name: {tensor.name}") + names.add(tensor.name) + if tensor.storage not in ("i8", "mxfp8", "f32"): + raise ValueError(f"unsupported storage for {tensor.name}: {tensor.storage}") + if tensor.payload_off < 0 or tensor.payload_len < 0: + raise ValueError(f"bad payload range for {tensor.name}") + if tensor.scale_off < 0 or tensor.scale_len < 0: + raise ValueError(f"bad scale range for {tensor.name}") + numel = _numel(tensor.shape, tensor.name) + payload_numel = _numel(tensor.payload_shape, tensor.name) + if tensor.storage == "i8": + if len(tensor.axes) > 1 or any( + axis < 0 or axis >= len(tensor.shape) for axis in tensor.axes + ): + raise ValueError( + f"unsupported scale axes for {tensor.name}: {tensor.axes}" + ) + if tensor.payload_off + tensor.payload_len > len(pkg.weights): + raise ValueError(f"weight OOB for {tensor.name}") + if tensor.payload_off != weight_end: + raise ValueError(f"non-contiguous weight payload for {tensor.name}") + if tensor.payload_len != payload_numel: + raise ValueError(f"weight_len mismatch for {tensor.name}") + if tensor.scale_off + tensor.scale_len > len(pkg.scales_f32): + raise ValueError(f"scale OOB for {tensor.name}") + scale_n = scale_count(tensor.shape, tensor.axes) + if tensor.scale_len != scale_n * 4: + raise ValueError(f"scale_len mismatch for {tensor.name}") + if tensor.scale_off % 4: + raise ValueError(f"unaligned scale offset for {tensor.name}") + if tensor.scale_off != scale_end: + raise ValueError(f"non-contiguous scale payload for {tensor.name}") + scales = struct.unpack_from( + f"<{scale_n}f", pkg.scales_f32, tensor.scale_off + ) + if any(not math.isfinite(scale) or scale <= 0.0 for scale in scales): + raise ValueError(f"invalid scale value for {tensor.name}") + channel_count = tensor.shape[tensor.axes[0]] if tensor.axes else 1 + if tensor.axes and any(scale != 1.0 for scale in scales[channel_count:]): + raise ValueError(f"bad channel-scale padding for {tensor.name}") + weight_end += tensor.payload_len + scale_end += tensor.scale_len + elif tensor.storage == "mxfp8": + if len(tensor.shape) != 2 or tensor.shape[1] % 32 or tensor.axes != [1]: + raise ValueError(f"invalid MXFP8 shape or block axis: {tensor.name}") + if len(tensor.payload_shape) == 2: + stride = tensor.shape[1] + tensor.shape[1] // 32 + expected = tensor.shape[0] * stride + if ( + tensor.payload_shape != [tensor.shape[0], stride] + or tensor.layout != {"row_stride": stride} + or tensor.payload_len != expected + ): + raise ValueError( + f"invalid MXFP8 embedding row layout: {tensor.name}" + ) + elif len(tensor.payload_shape) == 1: + rows, k = tensor.layout["tile_n"], tensor.layout["tile_k"] + stride = tensor.layout["panel_stride"] + expected = ( + ((tensor.shape[0] + rows - 1) // rows) + * ((tensor.shape[1] + k - 1) // k) + * stride + ) + if ( + rows % 16 + or k % 32 + or stride != tensor.layout["bank_bytes"] + or rows * k * 33 // 32 > stride + or tensor.payload_len != expected + or payload_numel != expected + ): + raise ValueError(f"invalid MXFP8 packed layout: {tensor.name}") + else: + raise ValueError(f"invalid MXFP8 payload rank: {tensor.name}") + if tensor.payload_off != weight_end or weight_end + expected > len( + pkg.weights + ): + raise ValueError(f"invalid MXFP8 weight range: {tensor.name}") + if tensor.scale_off or tensor.scale_len: + raise ValueError("MXFP8 scales are embedded E8M0 bytes") + weight_end += tensor.payload_len + else: + if tensor.axes or tensor.scale_off or tensor.scale_len: + raise ValueError(f"f32 parameter has quant metadata: {tensor.name}") + if tensor.payload_off + tensor.payload_len > len(pkg.params_f32): + raise ValueError(f"f32 parameter OOB for {tensor.name}") + if tensor.payload_off != param_end: + raise ValueError(f"non-contiguous f32 payload for {tensor.name}") + if tensor.payload_len != numel * 4: + raise ValueError(f"f32 parameter size mismatch for {tensor.name}") + param_end += tensor.payload_len + if weight_end != len(pkg.weights): + raise ValueError("unused weight payload") + if param_end != len(pkg.params_f32): + raise ValueError("unused FP32 parameter payload") + if scale_end != len(pkg.scales_f32): + raise ValueError("unused scale payload") + + +def _payload_dir(rax: Path) -> Path: + return rax.parent / f"{rax.stem}.payload" + + +def _quant_index(pkg: RaxQuantPackage) -> dict: + tensors = [] + for tensor in pkg.tensors: + tensors.append( + { + "name": tensor.name, + "shape": tensor.shape, + "payload_shape": tensor.payload_shape, + "storage": tensor.storage, + "payload": "params_f32" if tensor.storage == "f32" else "weights", + "payload_offset": tensor.payload_off, + "payload_bytes": tensor.payload_len, + "scale_offset": tensor.scale_off if tensor.storage == "i8" else 0, + "scale_bytes": tensor.scale_len if tensor.storage == "i8" else 0, + "scale_axes": tensor.axes, + **( + { + "layout": tensor.layout, + "block_size": 32, + "scale_encoding": "e8m0", + } + if tensor.storage == "mxfp8" + else {} + ), + } + ) + return {"version": 2, "tensors": tensors} + + +def _manifest( + model_name: str, + payload_name: str, + weights_bytes: int, + params_bytes: int, + scales_bytes: int, + index_bytes: int, + assets: dict[str, bytes], +) -> str: + asset_constants = "".join( + f' rhal.constant @{name.replace(".", "_")} {{id = {5 + i} : i32, storage = "external",\n' + f" type = tensor<{len(data)}xi8>,\n" + f' uri = "file:{payload_name}/{name}"}}\n' + for i, (name, data) in enumerate(assets.items()) + ) + return f"""rhal.module @quant_{model_name} attributes {{ + version = "0.1.0", + model_name = "{model_name}"}} {{ + rhal.constant @weights {{id = 1 : i32, storage = "external", + type = tensor<{weights_bytes}xi8>, + uri = "file:{payload_name}/weights.bin"}} + rhal.constant @params_f32 {{id = 2 : i32, storage = "external", + type = tensor<{params_bytes}xi8>, + uri = "file:{payload_name}/params.f32"}} + rhal.constant @scales_f32 {{id = 3 : i32, storage = "external", + type = tensor<{scales_bytes}xi8>, + uri = "file:{payload_name}/scales.bin"}} + rhal.constant @quant_index {{id = 4 : i32, storage = "external", + type = tensor<{index_bytes}xi8>, + uri = "file:{payload_name}/quant-index.json"}} +{asset_constants}}} +""" + + +def write_payload(pkg: RaxQuantPackage, payload_dir: Path | str) -> None: + validate_rax_quant(pkg) + payload_dir = Path(payload_dir) + if payload_dir.exists(): + shutil.rmtree(payload_dir) + payload_dir.mkdir(parents=True) + index = json.dumps(_quant_index(pkg), sort_keys=True, indent=2).encode("ascii") + (payload_dir / "weights.bin").write_bytes(pkg.weights) + (payload_dir / "params.f32").write_bytes(pkg.params_f32) + (payload_dir / "scales.bin").write_bytes(pkg.scales_f32) + (payload_dir / "quant-index.json").write_bytes(index) + for name, data in pkg.assets.items(): + (payload_dir / name).write_bytes(data) + + +def write_rax( + pkg: RaxQuantPackage, rax: Path | str, rax_pack: Path | str, model_name: str +) -> None: + rax = Path(rax) + rax_pack = Path(rax_pack) + if not rax_pack.is_file(): + raise ValueError(f"rax-pack not found: {rax_pack}") + payload_dir = _payload_dir(rax) + write_payload(pkg, payload_dir) + index = (payload_dir / "quant-index.json").read_bytes() + manifest = payload_dir / "quant.rhal.mlir" + manifest.write_text( + _manifest( + model_name, + payload_dir.name, + len(pkg.weights), + len(pkg.params_f32), + len(pkg.scales_f32), + len(index), + pkg.assets, + ), + encoding="ascii", + ) + subprocess.run( + [str(rax_pack), str(manifest), "-o", str(rax), "--embed-payload"], + check=True, + ) diff --git a/arch/src/main/scala/framework/rvv/src/main/resources/README.md b/waste/stack/compiler/frontend/quant/transforms/__init__.py similarity index 100% rename from arch/src/main/scala/framework/rvv/src/main/resources/README.md rename to waste/stack/compiler/frontend/quant/transforms/__init__.py diff --git a/waste/stack/compiler/frontend/quant/transforms/activation.py b/waste/stack/compiler/frontend/quant/transforms/activation.py new file mode 100644 index 000000000..f1f55fda8 --- /dev/null +++ b/waste/stack/compiler/frontend/quant/transforms/activation.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import numpy as np +import torch + + +def _activation_array(values: np.ndarray) -> np.ndarray: + values = np.asarray(values, dtype=np.float32) + if values.size == 0: + raise ValueError("activation tensor is empty") + if not np.all(np.isfinite(values)): + raise ValueError("activation tensor contains NaN or infinity") + return values + + +def activation_max_abs(values: np.ndarray) -> np.float32: + values = _activation_array(values) + return np.float32(np.max(np.abs(values))) + + +def activation_scale(values: np.ndarray) -> np.float32: + max_abs = activation_max_abs(values) + if max_abs == 0.0: + return np.float32(1.0) + return np.float32(max_abs / np.float32(127.0)) + + +def calibrate_layers(model: torch.nn.Module, batches) -> dict: + """Aggregate layer ranges across all batches before converting to scales.""" + ranges, calls = {}, {} + handles = [] + for name, module in model.named_modules(): + if not isinstance(module, (torch.nn.Conv2d, torch.nn.Linear)): + continue + + def record(layer, inputs, output, weight_name=f"{name}.weight"): + if ( + len(inputs) != 1 + or not torch.is_tensor(inputs[0]) + or not torch.is_tensor(output) + ): + raise ValueError(f"unsupported calibration signature for {weight_name}") + values = _activation_array(output.detach().cpu().numpy()) + axis = values.ndim - 1 if isinstance(layer, torch.nn.Linear) else 1 + axes = tuple(i for i in range(values.ndim) if i != axis) + hardswish = ( + values * np.clip(values + np.float32(3.0), 0.0, 6.0) / np.float32(6.0) + ) + observed = ( + activation_max_abs(inputs[0].detach().cpu().numpy()), + activation_max_abs(values), + activation_max_abs(np.maximum(values, np.float32(0.0))), + np.max(np.abs(values), axis=axes), + np.max(np.abs(hardswish), axis=axes), + ) + index = calls.get(weight_name, 0) + calls[weight_name] = index + 1 + records = ranges.setdefault(weight_name, []) + if index == len(records): + records.append(observed) + else: + records[index] = tuple( + np.maximum(a, b) + for a, b in zip(records[index], observed, strict=True) + ) + + handles.append(module.register_forward_hook(record)) + expected = None + try: + with torch.no_grad(): + for inputs in batches: + calls.clear() + if isinstance(inputs, dict): + model(**inputs) + else: + model(inputs) + if expected is None: + expected = dict(calls) + elif calls != expected: + raise ValueError( + "Calibration batches executed different model layers" + ) + finally: + for handle in handles: + handle.remove() + if not ranges: + raise ValueError("calibration found no Conv2d or Linear activations") + return { + name: [ + tuple( + np.where(value == 0, np.float32(1.0), value / np.float32(127.0)).astype( + np.float32 + ) + for value in record + ) + for record in records + ] + for name, records in ranges.items() + } + + +def quantize_activation_with_scale( + values: np.ndarray, da: float | np.float32 +) -> np.ndarray: + values = _activation_array(values) + da = np.float32(da) + if not np.isfinite(da) or da <= 0.0: + raise ValueError("Da must be finite and positive") + return np.clip(np.rint(values / da), -128, 127).astype(np.int8) + + +def quantize_activation(values: np.ndarray) -> tuple[np.ndarray, np.float32]: + da = activation_scale(values) + return quantize_activation_with_scale(values, da), da + + +def quantize_activation_banks( + values: np.ndarray, bank_rows: int +) -> tuple[np.ndarray, np.ndarray]: + values = _activation_array(values) + if values.ndim != 2: + raise ValueError("bank activation quantization requires a rank-2 matrix") + if type(bank_rows) is not int or bank_rows <= 0: + raise ValueError("bank_rows must be a positive integer") + quantized = np.empty(values.shape, dtype=np.int8) + bank_count = (values.shape[0] + bank_rows - 1) // bank_rows + scales = np.empty(bank_count, dtype=np.float32) + for bank, row in enumerate(range(0, values.shape[0], bank_rows)): + bank_quantized, da = quantize_activation(values[row : row + bank_rows]) + quantized[row : row + bank_rows] = bank_quantized + scales[bank] = da + return quantized, scales + + +def dequantize_accumulator( + accumulator: np.ndarray, da: float | np.float32, dw: float | np.ndarray +) -> np.ndarray: + accumulator = np.asarray(accumulator, dtype=np.int32) + da = np.float32(da) + dw = np.asarray(dw, dtype=np.float32) + if not np.isfinite(da) or da <= 0.0: + raise ValueError("Da must be finite and positive") + if not np.all(np.isfinite(dw)) or np.any(dw <= 0.0): + raise ValueError("Dw must be finite and positive") + if dw.ndim > 1 or (dw.ndim == 1 and accumulator.ndim == 0): + raise ValueError("Dw must be scalar or a last-dimension scale vector") + if dw.ndim == 1 and accumulator.shape[-1] != dw.shape[0]: + raise ValueError("Dw channel count does not match accumulator") + return accumulator.astype(np.float32) * da * dw + + +def w8a8_matmul( + activations: np.ndarray, + weights_i8: np.ndarray, + dw: float | np.ndarray, + bank_rows: int, +) -> tuple[np.ndarray, np.ndarray]: + activations = _activation_array(activations) + weights_i8 = np.asarray(weights_i8) + if activations.ndim != 2 or weights_i8.ndim != 2: + raise ValueError("W8A8 matmul requires rank-2 activation and weight tensors") + if weights_i8.dtype != np.int8: + raise ValueError("W8A8 weights must be INT8") + if activations.shape[1] != weights_i8.shape[0]: + raise ValueError("W8A8 matmul K dimensions do not match") + if type(bank_rows) is not int or bank_rows <= 0: + raise ValueError("bank_rows must be a positive integer") + quantized, da_values = quantize_activation_banks(activations, bank_rows) + result = np.empty((activations.shape[0], weights_i8.shape[1]), dtype=np.float32) + for bank, row in enumerate(range(0, activations.shape[0], bank_rows)): + accumulator = quantized[row : row + bank_rows].astype( + np.int32 + ) @ weights_i8.astype(np.int32) + result[row : row + bank_rows] = dequantize_accumulator( + accumulator, da_values[bank], dw + ) + return result, da_values diff --git a/waste/stack/compiler/frontend/quant/transforms/normalize.py b/waste/stack/compiler/frontend/quant/transforms/normalize.py new file mode 100644 index 000000000..0d600eff6 --- /dev/null +++ b/waste/stack/compiler/frontend/quant/transforms/normalize.py @@ -0,0 +1,20 @@ +import torch + + +def fold_batch_norms(model: torch.nn.Module) -> None: + if model.training: + raise ValueError("BatchNorm folding requires an eval-mode model") + for parent in model.modules(): + names = list(parent._modules) + for index in range(len(names) - 1): + conv_name, bn_name = names[index : index + 2] + conv = parent._modules[conv_name] + bn = parent._modules[bn_name] + if not isinstance(conv, torch.nn.Conv2d) or not isinstance( + bn, torch.nn.BatchNorm2d + ): + continue + parent._modules[conv_name] = torch.nn.utils.fusion.fuse_conv_bn_eval( + conv, bn + ) + parent._modules[bn_name] = torch.nn.Identity() diff --git a/waste/stack/compiler/frontend/quant/transforms/symmetric.py b/waste/stack/compiler/frontend/quant/transforms/symmetric.py new file mode 100644 index 000000000..d7b203858 --- /dev/null +++ b/waste/stack/compiler/frontend/quant/transforms/symmetric.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import numpy as np + + +def quantize_symmetric(w: np.ndarray, axes: list[int]) -> tuple[np.ndarray, np.ndarray]: + if w.dtype != np.float32: + raise ValueError(f"expected float32, got {w.dtype}") + if len(axes) != len(set(axes)): + raise ValueError("duplicate axes") + if any(a < 0 or a >= w.ndim for a in axes): + raise ValueError("axes out of range") + abs_w = np.abs(w) + if not axes: + max_abs = float(abs_w.max()) + if max_abs == 0.0: + raise ValueError("maxAbs is 0") + dw = np.float32(max_abs / 127.0) + q = np.clip(np.rint(w / dw), -128, 127).astype(np.int8) + return q, np.array(dw) + reduce_axes = tuple(i for i in range(w.ndim) if i not in axes) + max_abs = abs_w.max(axis=reduce_axes, keepdims=True) + if np.any(max_abs == 0): + raise ValueError("maxAbs is 0") + dw_keep = (max_abs / 127.0).astype(np.float32) + q = np.clip(np.rint(w / dw_keep), -128, 127).astype(np.int8) + dw = np.squeeze(dw_keep, axis=reduce_axes) + return q, dw.astype(np.float32) diff --git a/waste/stack/compiler/frontend/trace.py b/waste/stack/compiler/frontend/trace.py new file mode 100644 index 000000000..d756a2ccd --- /dev/null +++ b/waste/stack/compiler/frontend/trace.py @@ -0,0 +1,12 @@ +from pathlib import Path +from buddy.compiler.trace import TraceConfig, load_trace_config +from buddy.compiler.graph.transform.trace import TraceInsertionPass +from .operation import QuantizedOp + + +def trace(graph, path): + TraceInsertionPass(TraceConfig(load_trace_config(Path(path))))(graph) + for node in graph.body: + if isinstance(node, QuantizedOp): + node.quantization_trace_meta = node.trace_meta + node.trace_meta = None diff --git a/waste/stack/compiler/frontend/transforms.py b/waste/stack/compiler/frontend/transforms.py new file mode 100644 index 000000000..f8655e852 --- /dev/null +++ b/waste/stack/compiler/frontend/transforms.py @@ -0,0 +1,194 @@ +# ===- graph.py ---------------------------------------------------------------- +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ===--------------------------------------------------------------------------- +# +# This is the graph level of the Buddy Compiler frontend. +# +# ===--------------------------------------------------------------------------- + +import torch +from pathlib import Path + +from buddy.compiler.graph.operation import * +from buddy.compiler.graph.graph import NodeType, _replace_node_name +from buddy.compiler.graph.type import TensorDType +from stack.compiler.frontend.operation import * + + +def quantize( + graph, + config, + *, + parameter_names=None, + weights=None, + activation_scales=None, + calibration_model=None, + calibration_dataset=None, +): + """Apply the weight and activation policies from a quantization TOML. + + Offline artifacts supply data; the configuration selects transformations + and whether each activation scale is calibrated or computed online. + """ + from stack.compiler.frontend.quant.transforms.activation_quantize import ( + quantize_activations, + ) + + if "weights" in config: + if parameter_names is None: + raise ValueError("Weight quantization requires parameter names") + if weights is None: + from stack.compiler.frontend.quant.offline.weights import quantize_weights + + parameters = ( + dict(calibration_model.state_dict()) + if calibration_model is not None + else {} + ) + for node, value in zip(graph.params, graph._params_ref, strict=True): + if "weight_permutation" in node.tensor_meta: + permutation = node.tensor_meta["weight_permutation"] + inverse = [permutation.index(axis) for axis in range(value.ndim)] + value = value.permute(inverse).contiguous() + parameters[parameter_names[node.name]] = value + weights = quantize_weights(parameters, config["weights"]) + if set(config["weights"]) != set(weights["axes"]): + raise ValueError("Offline weights must match the configured weight names") + if "calibration" in config and activation_scales is None: + from stack.compiler.frontend.quant.offline.activations import ( + calibrate_activations, + ) + + calibration = dict(config["calibration"]) + if config.get("execution") == "int8": + calibration["layers"] = { + name: policy["module"] + for name, policy in config["activations"].items() + } + calibration["granularities"] = { + name: policy["granularity"] + for name, policy in config["activations"].items() + } + activation_scales = calibrate_activations( + calibration_model, + weights, + calibration_dataset, + **calibration, + ) + bind_quantized_weights(graph, weights, parameter_names) + if "activations" in config: + quantize_activations( + graph, + config["activations"], + activation_scales, + config.get("activation_target", "native"), + ) + if "execution" in config: + if config["execution"] != "int8": + raise ValueError("Unsupported quantized execution policy") + from stack.compiler.frontend.ops.reductions import ( + bmm, + gelu, + layer_norm, + mean, + softmax, + tanh, + ) + from stack.compiler.frontend.quant.transforms.integerize import integerize + + integerize(graph, config["activations"]) + graph._ops_registry = { + **graph._ops_registry, + "MeanOp": mean, + "BatchMatmulOp": bmm, + "SoftmaxOp": softmax, + "NativeLayerNormOp": layer_norm, + "GeluOp": gelu, + "TanhOp": tanh, + } + return graph._params_ref + + +def bind_quantized_weights(graph, quantized, parameter_names): + """Bind offline INT8 weights and insert runtime weight dequantization. + + parameter_names maps graph parameter names to checkpoint names. + Update the imported parameter list in place to match graph.params. + """ + from buddy.compiler.graph.transform.quantization.quantize import sort_graph + + parameters = list(graph.params) + values = {} + for node in parameters: + name = parameter_names[node.name] + value = quantized["tensors"][name] + if "weight_permutation" in node.tensor_meta: + value = value.permute(node.tensor_meta["weight_permutation"]).contiguous() + values[node.name] = value + if tuple(value.shape) != tuple(node.tensor_meta["shape"]): + raise ValueError(f"{name}: offline weight shape differs from graph") + if name not in quantized["axes"]: + continue + if value.dtype != torch.int8: + raise ValueError(f"{name}: expected offline INT8 weights") + node.tensor_meta["dtype"] = TensorDType.Int8 + shape = [1] * value.ndim + axis = quantized["axes"][name] + if axis is not None: + if "weight_permutation" in node.tensor_meta: + axis = node.tensor_meta["weight_permutation"].index(axis) + shape[axis] = value.shape[axis] + + scale = PlaceholderOp() + scale.name = f"scaler_{node.name}" + scale.tensor_meta = {"shape": shape, "dtype": TensorDType.Float32} + graph.add_node(scale, NodeType.FakeNode) + values[scale.name] = quantized["scales"][name].reshape(shape) + + cast = ConvertElementTypeOp() + cast.name = f"cast_{node.name}" + cast.add_argument(node.name) + cast.add_parent(node.name) + cast.tensor_meta = { + "shape": value.shape, + "dtype": TensorDType.Float32, + } + graph.add_node(cast) + + dequant = MulOp() + dequant.name = f"dequantized_{node.name}" + dequant._arguments = [cast.name, scale.name] + dequant._parents = [cast.name, scale.name] + dequant.tensor_meta = dict(cast.tensor_meta) + dequant._children = list(node._children) + graph.add_node(dequant) + for child_name in dequant._children: + child = graph.node_table[child_name] + child._arguments = _replace_node_name( + child.args, node.name, dequant.name, graph.node_table + ) + child._keyword_arguments = _replace_node_name( + child.kwargs, node.name, dequant.name, graph.node_table + ) + child._parents = [ + dequant.name if p == node.name else p for p in child.parents + ] + node._children = [cast.name] + cast._children = [dequant.name] + scale._children = [dequant.name] + + sort_graph(graph) + graph._params_ref[:] = [values[node.name] for node in graph.params] + return graph._params_ref diff --git a/waste/stack/compiler/src/Conversion/BallPasses.h b/waste/stack/compiler/src/Conversion/BallPasses.h new file mode 100644 index 000000000..8db10bbd7 --- /dev/null +++ b/waste/stack/compiler/src/Conversion/BallPasses.h @@ -0,0 +1,131 @@ +//===- BallPasses.h - Passes a Ball maintains ----------------------------===// +// +// A Ball owns two passes: one recognizes the Linalg operations it executes, +// the other lowers its operations to LLVM intrinsics. Bank lowering belongs to +// the framework, which a Ball joins through the interfaces in +// BuckyballInterfaces.td. +// +//===----------------------------------------------------------------------===// + +#ifndef BUDDY_CONVERSION_BALL_PASSES_H +#define BUDDY_CONVERSION_BALL_PASSES_H + +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/TypeConverter.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/IR/BuiltinOps.h" +#include "mlir/Pass/Pass.h" +#include "mlir/Transforms/DialectConversion.h" +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" + +#include "Buckyball/BuckyballDialect.h" + +#include +#include + +namespace mlir::buddy { + +// Claims the Linalg operations a Ball executes. Derived provides +// `void populate(RewritePatternSet &, int64_t bankWidthBytes, +// int64_t bankDepth, int64_t bankNum) const`. Operations it leaves stay in +// Linalg for the CPU; a pattern that rejects an operation it claims emits an +// error, which fails the pass. +template +class LinalgToBallPass : public PassWrapper> { +public: + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert<::buddy::buckyball::BuckyballDialect, func::FuncDialect, + memref::MemRefDialect, arith::ArithDialect, scf::SCFDialect, + linalg::LinalgDialect>(); + } + + void runOnOperation() override { + const auto &config = buckyball_target::getBuckyballTarget(); + MLIRContext *context = &this->getContext(); + RewritePatternSet patterns(context); + static_cast(this)->populate( + patterns, config.bankWidthBits / 8, config.bankDepth, config.bankNum); + // Rewrite only what the Ball claims; leave the rest of the IR as it is. + GreedyRewriteConfig rewrite; + rewrite.enableFolding(false); + rewrite.setRegionSimplificationLevel(GreedySimplifyRegionLevel::Disabled); + bool emittedError = false; + ScopedDiagnosticHandler diagnostics(context, [&](Diagnostic &message) { + emittedError |= message.getSeverity() == DiagnosticSeverity::Error; + return failure(); + }); + if (failed(applyPatternsGreedily(this->getOperation(), std::move(patterns), + rewrite)) || + emittedError) + this->signalPassFailure(); + } +}; + +// Lowers a Ball's operations to LLVM intrinsics. Derived provides +// `void populate(LLVMTypeConverter &, RewritePatternSet &, +// LLVMConversionTarget &, bool stable, int64_t bankDepth) const` and marks its +// operations illegal. +template +class LowerBallPass : public PassWrapper> { +public: + LowerBallPass() = default; + LowerBallPass(const LowerBallPass &other) + : PassWrapper>(other) {} + + Pass::Option stable{*this, "stable", + llvm::cl::desc("Use stable LLVM Buckyball " + "intrinsics."), + llvm::cl::init(false)}; + + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert(); + } + + void runOnOperation() override { + MLIRContext *context = &this->getContext(); + LLVMTypeConverter converter(context); + RewritePatternSet patterns(context); + LLVMConversionTarget target(*context); + // Intrinsics and the operations of other Balls stay; Derived marks its + // own operations illegal. + target.addLegalDialect(); + static_cast(this)->populate( + converter, patterns, target, stable, + buckyball_target::getBuckyballTarget().bankDepth); + ConversionConfig config; + config.allowPatternRollback = false; + if (failed(applyPartialConversion(this->getOperation(), target, + std::move(patterns), config))) + this->signalPassFailure(); + } +}; + +// `lower-buckyball` runs the framework lowering and then every Ball lowering +// pass registered here. +void addBallLowering(std::function(bool stable)> create); + +template void registerBallLoweringPass() { + PassRegistration(); + addBallLowering([](bool stable) { + auto pass = std::make_unique(); + pass->stable = stable; + return pass; + }); +} + +} // namespace mlir::buddy + +#endif // BUDDY_CONVERSION_BALL_PASSES_H diff --git a/waste/stack/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp b/waste/stack/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp new file mode 100644 index 000000000..592eaaa1a --- /dev/null +++ b/waste/stack/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp @@ -0,0 +1,97 @@ +#include "Conversion/LowerBuckyball/LowerBuckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/IR/Visitors.h" +#include "mlir/Pass/Pass.h" +#include "mlir/Rewrite/FrozenRewritePatternSet.h" +#include "mlir/Rewrite/PatternApplicator.h" + +#include + +#include "Buckyball/BuckyballDialect.h" +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; + +namespace { +class PhysicalBankAssignmentPattern + : public OpInterfaceRewritePattern< + ::buddy::buckyball::PhysicalBankAssignment> { +public: + PhysicalBankAssignmentPattern(MLIRContext *context, + mlir::buddy::PhysicalBankState &state) + : OpInterfaceRewritePattern(context), state(state) {} + + LogicalResult matchAndRewrite(::buddy::buckyball::PhysicalBankAssignment op, + PatternRewriter &rewriter) const override { + return op.assignPhysicalBanks(rewriter, state); + } + +private: + mlir::buddy::PhysicalBankState &state; +}; + +class AssignPhysicalBanksPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(AssignPhysicalBanksPass) + + StringRef getArgument() const final { return "assign-physical-banks"; } + StringRef getDescription() const final { + return "Assign physical banks for the selected Buckyball target."; + } + + void getDependentDialects(DialectRegistry ®istry) const override { + registry + .insert(); + } + + void runOnOperation() override { + const buckyball_target::BuckyballTargetConfig &target = + buckyball_target::getBuckyballTarget(); + for (func::FuncOp func : getOperation().getOps()) { + mlir::buddy::PhysicalBankState state(target.bankNum, + target.privateBankMax); + RewritePatternSet patterns(&getContext()); + mlir::buddy::addBaseAssignPhysicalBankPatterns(patterns, state); + patterns.add(&getContext(), state); + FrozenRewritePatternSet frozen(std::move(patterns)); + PatternApplicator applicator(frozen); + applicator.applyDefaultCostModel(); + PatternRewriter rewriter(&getContext()); + bool assignmentFailed = false; + std::function visitRegion = [&](Region ®ion) { + for (Block &block : region) { + for (auto it = block.begin(); it != block.end();) { + if (assignmentFailed) + return; + Operation *op = &*it++; + if (succeeded(applicator.matchAndRewrite(op, rewriter))) + continue; + if (op->getName().getStringRef().starts_with("buckyball.bank_")) { + assignmentFailed = true; + return; + } + for (Region &nested : op->getRegions()) + visitRegion(nested); + } + } + }; + for (Region ®ion : func->getRegions()) + visitRegion(region); + if (assignmentFailed || + mlir::failed(mlir::buddy::verifyNoBankSSAOps(func)) || + !state.empty()) { + signalPassFailure(); + return; + } + } + } +}; +} // namespace + +void mlir::buddy::registerAssignPhysicalBanksPass() { + PassRegistration(); +} diff --git a/waste/stack/compiler/src/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp b/waste/stack/compiler/src/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp new file mode 100644 index 000000000..87b4fdc96 --- /dev/null +++ b/waste/stack/compiler/src/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp @@ -0,0 +1,212 @@ +//===- BaseAssignPhysicalBankPatterns.cpp - Base bank assignment ----------===// + +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "Buckyball/BuckyballOps.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/IR/PatternMatch.h" + +using namespace mlir; +using namespace mlir::buddy; +using namespace ::buddy::buckyball; + +namespace { + +class BankAllocPattern : public OpRewritePattern { +public: + BankAllocPattern(MLIRContext *context, PhysicalBankState &state) + : OpRewritePattern(context), state(state) {} + + LogicalResult matchAndRewrite(BankAllocOp op, + PatternRewriter &rewriter) const override { + int64_t row = op.getRow(); + int64_t col = op.getCol(); + if (row <= 0 || col <= 0) + return op.emitError("assign-physical-banks: invalid bank shape"); + + auto bank = state.tryAlloc(row, col); + if (!bank) { + InFlightDiagnostic diagnostic = + op.emitError("assign-physical-banks: unavailable bank resources"); + diagnostic << " (request=" << row << "x" << col + << ", used=" << state.getUsedCount() << "/" + << state.getBankNum() + << ", private ID max=" << state.getPrivateBankMax() << ")"; + return failure(); + } + + state.createMset(rewriter, op.getLoc(), static_cast(*bank), true, + row, col); + rewriter.replaceOp(op, state.cstI64(rewriter, op.getLoc(), *bank)); + return success(); + } + +private: + PhysicalBankState &state; +}; + +class BankReleasePattern : public OpRewritePattern { +public: + BankReleasePattern(MLIRContext *context, PhysicalBankState &state) + : OpRewritePattern(context), state(state) {} + + LogicalResult matchAndRewrite(BankReleaseOp op, + PatternRewriter &rewriter) const override { + auto bank = state.getConstI64(op.getBank()); + if (!bank) { + return op.emitError( + "assign-physical-banks: release bank id is not constant"); + } + if (failed(state.release(op, *bank))) + return failure(); + + state.createMset(rewriter, op.getLoc(), static_cast(*bank), false, + 0, 0); + rewriter.eraseOp(op); + return success(); + } + +private: + PhysicalBankState &state; +}; + +class BankTransferPattern : public OpRewritePattern { +public: + BankTransferPattern(MLIRContext *context, PhysicalBankState &state) + : OpRewritePattern(context), state(state) {} + + LogicalResult matchAndRewrite(BankTransferOp op, + PatternRewriter &rewriter) const override { + auto source = state.getConstI64(op.getSource()); + auto target = state.getConstI64(op.getTarget()); + if (!source || !target) + return op.emitError("bank transfer requires constant virtual IDs"); + auto assigned = op.getSource().getDefiningOp(); + if (!assigned || assigned->getBlock() != op->getBlock()) + return op.emitError( + "bank transfer cannot consume a control-flow bank alias"); + for (Operation *user : op.getSource().getUsers()) { + if (user == op.getOperation()) + continue; + while (user && user->getBlock() != op->getBlock()) + user = user->getParentOp(); + if (!user || !user->isBeforeInBlock(op)) + return op.emitError( + "source bank handle is used after transfer or outside its block"); + } + if (failed(state.transfer(op, *source, *target))) + return failure(); + rewriter.create(op.getLoc(), op.getSource(), + op.getTarget()); + rewriter.replaceOp(op, state.cstI64(rewriter, op.getLoc(), *target)); + return success(); + } + +private: + PhysicalBankState &state; +}; + +class BankKernelPattern : public OpRewritePattern { +public: + BankKernelPattern(MLIRContext *context, PhysicalBankState &state) + : OpRewritePattern(context), state(state) {} + LogicalResult matchAndRewrite(BankKernelOp op, + PatternRewriter &rewriter) const override { + if (failed( + state.verifyKernelHandles(op, op.getReadBank(), op.getWriteBank()))) + return failure(); + auto module = op->getParentOfType(); + StringRef name = op.getCallee(); + SmallVector types(op.getOperandTypes()); + auto signature = rewriter.getFunctionType(types, TypeRange{}); + auto callee = module.lookupSymbol(name); + if (callee && callee.getFunctionType() != signature) + return op.emitError( + "bank kernel callee type does not match its operands"); + if (!callee) { + OpBuilder::InsertionGuard guard(rewriter); + rewriter.setInsertionPointToStart(module.getBody()); + auto function = + rewriter.create(op.getLoc(), name, signature); + function.setPrivate(); + function->setAttr("llvm.emit_c_interface", rewriter.getUnitAttr()); + } + rewriter.create(op.getLoc(), name, TypeRange{}, + op.getOperands()); + rewriter.replaceOp(op, ValueRange{op.getReadBank(), op.getWriteBank()}); + return success(); + } + +private: + PhysicalBankState &state; +}; + +class BankMvinPattern : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(BankMvinOp op, + PatternRewriter &rewriter) const override { + rewriter.create(op.getLoc(), op.getInput(), op.getBank(), + op.getDepth(), op.getStride(), op.getGroupAttr()); + rewriter.replaceOp(op, op.getBank()); + return success(); + } +}; + +class BankMvin2dPattern : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(BankMvin2dOp op, + PatternRewriter &rewriter) const override { + rewriter.create(op.getLoc(), op.getInput(), op.getBank(), + op.getHeight(), op.getPixelBytes(), + op.getSourceWidth(), op.getDstBase(), + op.getWidth(), op.getValidBytes()); + rewriter.replaceOp(op, op.getBank()); + return success(); + } +}; + +class BankMvoutPattern : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(BankMvoutOp op, + PatternRewriter &rewriter) const override { + rewriter.create(op.getLoc(), op.getOutput(), op.getBank(), + op.getDepth(), op.getStride(), op.getGroupAttr()); + rewriter.replaceOp(op, op.getBank()); + return success(); + } +}; + +} // namespace + +namespace mlir::buddy { + +LogicalResult verifyNoBankSSAOps(Operation *root) { + Operation *badOp = nullptr; + root->walk([&](Operation *op) { + if (op->getName().getStringRef().starts_with("buckyball.bank_")) { + badOp = op; + return WalkResult::interrupt(); + } + return WalkResult::advance(); + }); + if (!badOp) + return success(); + return badOp->emitError("assign-physical-banks: unsupported bank op"); +} + +void addBaseAssignPhysicalBankPatterns(RewritePatternSet &patterns, + PhysicalBankState &state) { + patterns.add(patterns.getContext(), state); + patterns.add( + patterns.getContext()); +} + +} // namespace mlir::buddy diff --git a/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyball.h b/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyball.h new file mode 100644 index 000000000..5a89310d2 --- /dev/null +++ b/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyball.h @@ -0,0 +1,70 @@ +//===- LowerBuckyball.h - Buckyball lowering hooks -------------*- C++ -*-===// + +#ifndef BUDDY_CONVERSION_LOWER_BUCKYBALL_LOWER_BUCKYBALL_H +#define BUDDY_CONVERSION_LOWER_BUCKYBALL_LOWER_BUCKYBALL_H + +#include "mlir/IR/Builders.h" +#include "mlir/IR/PatternMatch.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" + +#include +#include + +namespace mlir { +class Operation; + +namespace buddy { + +struct BankSlot { + int64_t base = -1; + int64_t row = 1; + int64_t col = 1; + llvm::SmallVector physicalGroups; +}; + +class PhysicalBankState { +public: + PhysicalBankState(int64_t bankNum, int64_t privateBankMax); + + int64_t getBankNum() const { return bankNum; } + int64_t getPrivateBankMax() const { return privateBankMax; } + int64_t getUsedCount() const; + bool empty() const { return vm.empty(); } + + std::optional getConstI64(Value value) const; + std::optional getSlot(Value value) const; + std::optional tryAlloc(int64_t row, int64_t col); + LogicalResult verifyKernelHandles(Operation *op, Value read, + Value write) const; + LogicalResult transfer(Operation *op, int64_t source, int64_t target); + LogicalResult release(Operation *op, int64_t bank); + + Value cstI64(OpBuilder &builder, Location loc, uint64_t value) const; + void createMset(OpBuilder &builder, Location loc, uint64_t bankId, bool alloc, + uint64_t row, uint64_t col) const; + +private: + void freeAlloc(const BankSlot &slot); + + int64_t bankNum = 0; + int64_t privateBankMax = 0; + llvm::DenseMap vm; + llvm::SmallVector used; +}; + +LogicalResult verifyNoBankSSAOps(Operation *root); +void addBaseAssignPhysicalBankPatterns(RewritePatternSet &patterns, + PhysicalBankState &state); +// Lowers the operations implementing BankSSALowering; for design passes that +// lower some operations themselves before the rest. +void populateBankSSALoweringPatterns(RewritePatternSet &patterns); +void registerAssignPhysicalBanksPass(); +// The single entry the compiler driver calls to register every pass. +void registerBuckyballPasses(); +void registerLowerBuckyballToBankSSAPass(); + +} // namespace buddy +} // namespace mlir + +#endif // BUDDY_CONVERSION_LOWER_BUCKYBALL_LOWER_BUCKYBALL_H diff --git a/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp b/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp new file mode 100644 index 000000000..02f36983c --- /dev/null +++ b/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp @@ -0,0 +1,109 @@ +#include "Conversion/BallPasses.h" +#include "Conversion/LowerBuckyball/LowerBuckyball.h" +#include "Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h" +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" + +#include "mlir/Pass/PassManager.h" +#include "mlir/Pass/PassRegistry.h" + +using namespace mlir; + +namespace { +// Lowers the framework operations: bank configuration, data movement and raw +// instructions. Ball operations are lowered by their own passes. +class LowerFrameworkPass + : public mlir::buddy::LowerBallPass { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerFrameworkPass) + + StringRef getArgument() const final { return "lower-buckyball-framework"; } + StringRef getDescription() const final { + return "Lower Buckyball framework operations to LLVM intrinsics."; + } + + void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, + LLVMConversionTarget &target, bool, int64_t) const { + ::buddy::buckyball::legalize::populateBaseLegalizeForLLVMExportPatterns( + converter, patterns, /*includeFuncOperandForwarding=*/false); + ::buddy::buckyball::legalize::configureBaseLegalizeForExportTarget(target); + } +}; + +// Turns the Buckyball intrinsic operations into calls of the RISC-V LLVM +// intrinsics, so any LLVM IR translation handles the result. +class LowerIntrinsicsPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerIntrinsicsPass) + + StringRef getArgument() const final { return "lower-buckyball-intrinsics"; } + StringRef getDescription() const final { + return "Lower buckyball.intr.* to llvm.call_intrinsic."; + } + + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert(); + } + + void runOnOperation() override { + SmallVector intrinsics; + getOperation().walk([&](Operation *op) { + if (op->getName().getStringRef().starts_with("buckyball.intr.")) + intrinsics.push_back(op); + }); + OpBuilder builder(&getContext()); + for (Operation *op : intrinsics) { + StringRef name = op->getName().getStringRef().drop_front(15); + SmallVector operands(op->getOperands()); + builder.setInsertionPoint(op); + std::string intrinsic = ("llvm.riscv.bb." + name).str(); + // The custom instruction carries its funct7 as an immediate argument. + if (name == "custom") { + intrinsic = "llvm.riscv.buckyball.custom"; + operands.push_back(builder.create( + op->getLoc(), builder.getI32Type(), + cast(op->getAttr("funct7")))); + } + builder.create( + op->getLoc(), builder.getStringAttr(intrinsic), operands); + op->erase(); + } + } +}; + +struct LowerBuckyballOptions + : public PassPipelineOptions { + Option stable{*this, "stable", + llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), + llvm::cl::init(false)}; +}; + +std::vector(bool)>> &ballLowerings() { + static std::vector(bool)>> lowerings; + return lowerings; +} +} // namespace + +void mlir::buddy::addBallLowering( + std::function(bool stable)> create) { + ballLowerings().push_back(std::move(create)); +} + +void mlir::buddy::registerBuckyballPasses() { + registerLowerLinalgToBuckyballPass(); + registerLowerBuckyballToBankSSAPass(); + registerAssignPhysicalBanksPass(); + PassRegistration(); + PassRegistration(); + PassPipelineRegistration( + "lower-buckyball", + "Lower Buckyball framework and Ball operations to LLVM intrinsics.", + [](OpPassManager &pm, const LowerBuckyballOptions &options) { + auto framework = std::make_unique(); + framework->stable = options.stable; + pm.addPass(std::move(framework)); + for (auto &create : ballLowerings()) + pm.addPass(create(options.stable)); + pm.addPass(std::make_unique()); + }); +} diff --git a/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp b/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp new file mode 100644 index 000000000..f9f8e5ead --- /dev/null +++ b/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp @@ -0,0 +1,80 @@ +//===- LowerBuckyballToBankSSAPass.cpp - Buckyball bank-SSA lowering -----===// +// +// Lowers every operation implementing BankSSALowering to bank-SSA operations. +// +//===----------------------------------------------------------------------===// + +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/Math/IR/Math.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/IR/Diagnostics.h" +#include "mlir/IR/PatternMatch.h" +#include "mlir/Pass/Pass.h" +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" + +#include "Buckyball/BuckyballDialect.h" +#include "Buckyball/BuckyballOps.h" +#include "Trace/TraceDialect.h" + +using namespace mlir; + +namespace { +class BankSSALoweringPattern + : public OpInterfaceRewritePattern<::buddy::buckyball::BankSSALowering> { +public: + using OpInterfaceRewritePattern::OpInterfaceRewritePattern; + + LogicalResult matchAndRewrite(::buddy::buckyball::BankSSALowering op, + PatternRewriter &rewriter) const override { + return op.lowerToBankSSA(rewriter); + } +}; + +class LowerBuckyballToBankSSAPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToBankSSAPass) + + StringRef getArgument() const final { return "lower-buckyball-to-bank-ssa"; } + StringRef getDescription() const final { + return "Lower Buckyball Buckyball ops to explicit bank-SSA ops."; + } + + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert<::buddy::trace::BuddyTraceDialect, arith::ArithDialect, + linalg::LinalgDialect, math::MathDialect, + memref::MemRefDialect, LLVM::LLVMDialect, scf::SCFDialect, + ::buddy::buckyball::BuckyballDialect>(); + } + + void runOnOperation() override { + RewritePatternSet patterns(&getContext()); + mlir::buddy::populateBankSSALoweringPatterns(patterns); + bool emittedError = false; + ScopedDiagnosticHandler diagnostics( + &getContext(), [&](Diagnostic &message) { + emittedError |= message.getSeverity() == DiagnosticSeverity::Error; + return failure(); + }); + if (failed(applyPatternsGreedily(getOperation(), std::move(patterns))) || + emittedError) + signalPassFailure(); + } +}; + +} // namespace + +void mlir::buddy::populateBankSSALoweringPatterns(RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} + +void mlir::buddy::registerLowerBuckyballToBankSSAPass() { + PassRegistration(); +} diff --git a/waste/stack/compiler/src/Conversion/LowerBuckyball/PhysicalBankState.cpp b/waste/stack/compiler/src/Conversion/LowerBuckyball/PhysicalBankState.cpp new file mode 100644 index 000000000..f73d3326d --- /dev/null +++ b/waste/stack/compiler/src/Conversion/LowerBuckyball/PhysicalBankState.cpp @@ -0,0 +1,238 @@ +//===- PhysicalBankState.cpp - Physical bank allocation state -------------===// + +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "Buckyball/BuckyballOps.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/IR/Diagnostics.h" +#include "llvm/ADT/STLExtras.h" + +using namespace mlir; +using namespace mlir::buddy; +using namespace ::buddy::buckyball; + +PhysicalBankState::PhysicalBankState(int64_t bankNum, int64_t privateBankMax) + : bankNum(bankNum), privateBankMax(privateBankMax), used(bankNum, 0) {} + +int64_t PhysicalBankState::getUsedCount() const { + return llvm::count(used, static_cast(1)); +} + +std::optional PhysicalBankState::getConstI64(Value value) const { + while (true) { + if (auto cst = value.getDefiningOp()) { + auto attr = dyn_cast(cst.getValue()); + if (!attr) + return std::nullopt; + return attr.getInt(); + } + if (auto op = value.getDefiningOp()) { + value = op.getBank(); + continue; + } + if (auto op = value.getDefiningOp()) { + value = op.getBank(); + continue; + } + if (auto op = value.getDefiningOp()) { + value = op.getBank(); + continue; + } + if (auto op = value.getDefiningOp()) { + value = op.getTarget(); + continue; + } + if (auto op = value.getDefiningOp()) { + value = cast(value).getResultNumber() == 0 ? op.getReadBank() + : op.getWriteBank(); + continue; + } + // Optional Balls: use op names so Cores can omit generated C++ types. + if (Operation *op = value.getDefiningOp()) { + StringRef name = op->getName().getStringRef(); + if (name == "buckyball.bank_transpose" || + name == "buckyball.bank_quant_f32_to_i8") { + value = op->getOperand(1); + continue; + } + if (name == "buckyball.bank_quant_i32_to_i8") { + value = op->getOperand(2); + continue; + } + if (name == "buckyball.bank_int32_to_fp32") { + value = op->getOperand(2); + continue; + } + if (name == "buckyball.bank_smatmul_bias") { + value = op->getOperand(0); + continue; + } + if (name == "buckyball.bank_im2col") { + value = op->getOperand(1); + continue; + } + if (name == "buckyball.bank_lut") { + value = op->getOperand(2); + continue; + } + if (name == "buckyball.bank_maxpool") { + value = op->getOperand(1); + continue; + } + if (name == "buckyball.bank_int8add") { + value = op->getOperand(2); + continue; + } + if (name == "buckyball.bank_int8mul") { + value = op->getOperand(2); + continue; + } + if (name == "buckyball.bank_smatmul" || name == "buckyball.bank_mxfp8" || + name == "buckyball.bank_vecmat16") { + value = op->getOperand(2); + continue; + } + if (name == "buckyball.bank_gemmini_preload") { + value = op->getOperand(1); + continue; + } + if (name == "buckyball.bank_gemmini_compute_preloaded" || + name == "buckyball.bank_gemmini_compute_accumulated") { + value = op->getOperand(2); + continue; + } + } + if (auto forOp = value.getDefiningOp()) { + unsigned resultNumber = cast(value).getResultNumber(); + value = cast(forOp.getBody()->getTerminator()) + .getResults()[resultNumber]; + continue; + } + if (auto ifOp = value.getDefiningOp()) { + unsigned resultNumber = cast(value).getResultNumber(); + auto thenYield = cast(ifOp.getThenRegion().front().back()); + auto elseYield = cast(ifOp.getElseRegion().front().back()); + auto thenBank = getConstI64(thenYield.getResults()[resultNumber]); + auto elseBank = getConstI64(elseYield.getResults()[resultNumber]); + if (!thenBank || !elseBank || *thenBank != *elseBank) + return std::nullopt; + return thenBank; + } + if (auto argument = dyn_cast(value)) { + auto forOp = dyn_cast(argument.getOwner()->getParentOp()); + if (!forOp || argument.getArgNumber() == 0) + return std::nullopt; + value = forOp.getInitArgs()[argument.getArgNumber() - 1]; + continue; + } + return std::nullopt; + } +} + +std::optional PhysicalBankState::getSlot(Value value) const { + auto bank = getConstI64(value); + if (!bank) + return std::nullopt; + auto slot = vm.find(*bank); + if (slot == vm.end()) + return std::nullopt; + for (auto [index, group] : llvm::enumerate(slot->second.physicalGroups)) + if (group != slot->second.base + static_cast(index)) { + emitError(value.getLoc(), + "bank operation requires contiguous physical groups"); + return std::nullopt; + } + return slot->second; +} + +std::optional PhysicalBankState::tryAlloc(int64_t row, int64_t col) { + if (row > bankNum / col || row * col > bankNum - getUsedCount()) + return std::nullopt; + int64_t bank = 0; + while (bank <= privateBankMax && vm.contains(bank)) + ++bank; + if (bank > privateBankMax) + return std::nullopt; + BankSlot slot; + slot.row = row; + slot.col = col; + for (int64_t group = 0; + group < bankNum && + static_cast(slot.physicalGroups.size()) < row * col; + ++group) + if (!used[group]) { + slot.physicalGroups.push_back(group); + used[group] = 1; + } + slot.base = slot.physicalGroups.front(); + vm[bank] = std::move(slot); + return bank; +} + +LogicalResult PhysicalBankState::verifyKernelHandles(Operation *op, Value read, + Value write) const { + auto r = getConstI64(read), w = getConstI64(write); + if (!r || !w || *r == *w || !vm.contains(*r) || !vm.contains(*w)) + return op->emitError( + "bank kernel requires distinct live read/write handles"); + return success(); +} + +LogicalResult PhysicalBankState::transfer(Operation *op, int64_t source, + int64_t target) { + if (source == target) + return op->emitError("bank transfer source and target must differ"); + if (target < 0 || target > privateBankMax) + return op->emitError( + "bank transfer target exceeds the private virtual ID range"); + auto sourceSlot = vm.find(source); + if (sourceSlot == vm.end()) + return op->emitError("transfer consumes an unknown virtual bank handle"); + BankSlot moved = sourceSlot->second; + vm.erase(sourceSlot); + auto targetSlot = vm.find(target); + if (targetSlot == vm.end()) { + vm[target] = std::move(moved); + } else { + auto &slot = targetSlot->second; + slot.physicalGroups.append(moved.physicalGroups); + slot.row = 1; + slot.col = slot.physicalGroups.size(); + } + return success(); +} + +LogicalResult PhysicalBankState::release(Operation *op, int64_t bank) { + auto it = vm.find(bank); + if (it == vm.end()) { + InFlightDiagnostic diagnostic = + op->emitError("release of unknown virtual bank handle"); + diagnostic << " (bank=" << bank << ", live=" << vm.size() + << ", used=" << getUsedCount() << "/" << bankNum << ")"; + return failure(); + } + freeAlloc(it->second); + vm.erase(it); + return success(); +} + +Value PhysicalBankState::cstI64(OpBuilder &builder, Location loc, + uint64_t value) const { + return builder.create(loc, builder.getI64Type(), + builder.getI64IntegerAttr(value)); +} + +void PhysicalBankState::createMset(OpBuilder &builder, Location loc, + uint64_t bankId, bool alloc, uint64_t row, + uint64_t col) const { + auto op = builder.create(loc, cstI64(builder, loc, bankId)); + op->setAttr("alloc", builder.getBoolAttr(alloc)); + op->setAttr("row", builder.getI64IntegerAttr(row)); + op->setAttr("col", builder.getI64IntegerAttr(col)); +} + +void PhysicalBankState::freeAlloc(const BankSlot &slot) { + for (int64_t group : slot.physicalGroups) + used[group] = 0; +} diff --git a/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/FormMegaKernelRegions.cpp b/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/FormMegaKernelRegions.cpp new file mode 100644 index 000000000..98086296f --- /dev/null +++ b/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/FormMegaKernelRegions.cpp @@ -0,0 +1,161 @@ +#include "Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "llvm/ADT/DenseMap.h" + +using namespace mlir; + +void mlir::buddy::formMegaKernelRegions(ArrayRef stageList) { + SmallVector stages(stageList); + DenseMap producers; + DenseMap positions; + for (auto [index, op] : llvm::enumerate(stages)) { + producers[op.getOutputs()[0]] = index; + positions[op.getOperation()] = index; + } + SmallVector next(stages.size(), -1), previous(stages.size(), -1); + for (auto [index, op] : llvm::enumerate(stages)) { + Value output = op.getOutputs()[0]; + if (!cast(output.getType()).getElementType().isInteger(8)) + continue; + SmallVector consumers; + bool external = false; + for (Operation *user : output.getUsers()) { + if (user == op || isa(user)) + continue; + auto found = positions.find(user); + if (found == positions.end()) { + external = true; + break; + } + if (!llvm::is_contained(consumers, found->second)) + consumers.push_back(found->second); + } + if (external || consumers.size() != 1) + continue; + unsigned target = consumers.front(); + unsigned parents = 0; + for (Value input : stages[target].getInputs()) + parents += producers.contains(input); + if (parents != 1 || op->getBlock() != stages[target]->getBlock() || + op->hasAttr("buckyball.quantized_matmul") != + stages[target]->hasAttr("buckyball.quantized_matmul")) + continue; + next[index] = target; + previous[target] = index; + } + unsigned group = 0; + for (unsigned root = 0; root < stages.size(); ++root) { + if (previous[root] >= 0) + continue; + SmallVector chain; + for (int64_t index = root; index >= 0; index = next[index]) + chain.push_back(index); + auto context = stages[root].getContext(); + Builder builder(context); + auto id = builder.getStringAttr("core." + std::to_string(group++)); + for (auto [index, position] : llvm::enumerate(chain)) { + auto op = stages[position]; + if (op->hasAttr("core.packed_weight")) { + Value packed = op.getInputs()[1]; + for (Operation *user : llvm::make_early_inc_range(packed.getUsers())) + if (isa(user)) + user->moveAfter(stages[chain.back()]); + op->removeAttr("core.packed_weight"); + } + + SmallVector remove; + SmallVector add; + for (NamedAttribute attr : op->getAttrs()) { + StringRef name = attr.getName().strref(); + if (name.consume_front("buckyball.quantized_")) { + remove.push_back(attr.getName()); + add.emplace_back(builder.getStringAttr("buckyball.mega_" + name), + attr.getValue()); + } + } + for (StringAttr name : remove) + op->removeAttr(name); + for (NamedAttribute attr : add) + op->setAttr(attr.getName(), attr.getValue()); + op->setAttr("buckyball.mega_kernel", builder.getBoolAttr(true)); + op->setAttr("mega_kernel_id", id); + op->setAttr("mega_kernel_stage", builder.getI64IntegerAttr(index)); + op->setAttr("mega_kernel_size", builder.getI64IntegerAttr(chain.size())); + } + } +} + +void mlir::buddy::prepareQuantizedWeights( + ArrayRef stageList) { + SmallVector stages(stageList); + for (auto op : stages) { + bool matrix = op->hasAttr("buckyball.quantized_matmul"); + bool conv = op->hasAttr("buckyball.quantized_conv2d"); + bool depthwise = op->hasAttr("buckyball.quantized_conv2d_depthwise"); + if (!matrix && !conv && !depthwise) + continue; + Value weight = op.getInputs()[1]; + auto type = cast(weight.getType()); + auto shape = type.getShape(); + // Pebble's large-convolution lowering reads the logical weights itself. + if (conv && op->getAttrOfType("kernel").getInt() > 7) + continue; + SmallVector packedShape; + if (matrix) + packedShape = {shape[1], shape[0]}; + else if (depthwise) + packedShape = {shape[2], shape[3], shape[0], 1}; + else + packedShape = {(shape[0] + 15) / 16, shape[1], + ((shape[2] * shape[3] + 15) / 16) * 16, 16}; + OpBuilder builder(op); + Location loc = op.getLoc(); + Value packed = memref::AllocOp::create( + builder, loc, MemRefType::get(packedShape, type.getElementType())); + Value zeroByte = arith::ConstantIntOp::create(builder, loc, 0, 8); + linalg::FillOp::create(builder, loc, zeroByte, packed); + SmallVector indices; + for (int64_t extent : shape) { + Value zero = arith::ConstantIndexOp::create(builder, loc, 0); + Value end = arith::ConstantIndexOp::create(builder, loc, extent); + Value one = arith::ConstantIndexOp::create(builder, loc, 1); + auto loop = scf::ForOp::create(builder, loc, zero, end, one); + builder.setInsertionPointToStart(loop.getBody()); + indices.push_back(loop.getInductionVar()); + } + Value value = memref::LoadOp::create(builder, loc, weight, indices); + SmallVector destination; + if (matrix) + destination = {indices[1], indices[0]}; + else if (depthwise) + destination = {indices[2], indices[3], indices[0], indices[1]}; + else { + Value lanes = arith::ConstantIndexOp::create(builder, loc, 16); + Value width = arith::ConstantIndexOp::create(builder, loc, shape[3]); + destination = {arith::DivUIOp::create(builder, loc, indices[0], lanes), + indices[1], + arith::AddIOp::create( + builder, loc, + arith::MulIOp::create(builder, loc, indices[2], width), + indices[3]), + arith::RemUIOp::create(builder, loc, indices[0], lanes)}; + } + memref::StoreOp::create(builder, loc, value, packed, destination); + op.getInputsMutable()[1].set(packed); + if (matrix) { + auto maps = op.getIndexingMapsArray(); + maps[1] = AffineMap::get(3, 0, + {getAffineDimExpr(2, op.getContext()), + getAffineDimExpr(1, op.getContext())}, + op.getContext()); + op.setIndexingMapsAttr(builder.getAffineMapArrayAttr(maps)); + } + builder.setInsertionPointAfter(op); + memref::DeallocOp::create(builder, loc, packed); + op->setAttr("core.packed_weight", UnitAttr::get(op.getContext())); + } +} diff --git a/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h b/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h new file mode 100644 index 000000000..b1aedeb00 --- /dev/null +++ b/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h @@ -0,0 +1,45 @@ +//===- LowerLinalgToBuckyball.h - Linalg to Buckyball conversion ----------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +// +// The framework collects the marked MegaKernel regions. Each Ball +// claims its own Linalg operations in its convert-linalg-to--ball pass, +// and a pipeline chooses which Balls claim; the rest stays in Linalg for the +// CPU. +// +//===----------------------------------------------------------------------===// + +#ifndef BUDDY_CONVERSION_LOWER_LINALG_TO_BUCKYBALL_H +#define BUDDY_CONVERSION_LOWER_LINALG_TO_BUCKYBALL_H + +#include "mlir/IR/PatternMatch.h" + +namespace mlir { +namespace linalg { +class GenericOp; +} +namespace buddy { + +void prepareQuantizedWeights(ArrayRef stages); +void formMegaKernelRegions(ArrayRef stages); + +// Collect the regions selected by the design midend. +void populateMegaKernelLinalgLoweringPatterns(RewritePatternSet &patterns); +void registerLowerLinalgToBuckyballPass(); + +} // namespace buddy +} // namespace mlir + +#endif // BUDDY_CONVERSION_LOWER_LINALG_TO_BUCKYBALL_H diff --git a/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyballPass.cpp b/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyballPass.cpp new file mode 100644 index 000000000..e0e1f6bd4 --- /dev/null +++ b/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyballPass.cpp @@ -0,0 +1,124 @@ +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/IR/BuiltinOps.h" +#include "mlir/Pass/Pass.h" +#include "mlir/Transforms/DialectConversion.h" +#include "mlir/Transforms/GreedyPatternRewriteDriver.h" + +#include "Buckyball/BuckyballDialect.h" +#include "Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h" + +using namespace mlir; + +namespace { +// A transpose that repeats an earlier one on the same input reuses its result. +class CancelDuplicateTransposePattern + : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(linalg::TransposeOp op, + PatternRewriter &rewriter) const override { + Value in = op.getInput(); + for (Operation *user : in.getUsers()) { + auto prev = dyn_cast(user); + if (!prev || prev == op || prev.getPermutation() != op.getPermutation()) + continue; + auto shape = cast(op.getInit().getType()).getShape(); + if (cast(in.getType()).getShape() != shape || + cast(prev.getInit().getType()).getShape() != shape) + continue; + if (prev.getInit() != op.getInit()) + rewriter.create(op.getLoc(), prev.getInit(), + op.getInit()); + rewriter.eraseOp(op); + return success(); + } + return failure(); + } +}; + +// Two transposes whose permutations compose to identity become one copy. +class CancelTransposePairPattern + : public OpRewritePattern { +public: + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(linalg::TransposeOp second, + PatternRewriter &rewriter) const override { + auto first = second.getInput().getDefiningOp(); + if (!first || first.getInit() != second.getInput() || + !second.getInput().hasOneUse()) + return failure(); + ArrayRef p1 = first.getPermutation(); + ArrayRef p2 = second.getPermutation(); + if (p1.size() != p2.size()) + return failure(); + for (int64_t i = 0; i < (int64_t)p2.size(); ++i) + if (p1[p2[i]] != i) + return failure(); + if (cast(first.getInput().getType()).getShape() != + cast(second.getInit().getType()).getShape()) + return failure(); + if (first.getInput() != second.getInit()) + rewriter.create(second.getLoc(), first.getInput(), + second.getInit()); + rewriter.eraseOp(second); + if (first.getInit().use_empty()) + rewriter.eraseOp(first); + return success(); + } +}; + +class LowerLinalgToBuckyballPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerLinalgToBuckyballPass) + StringRef getArgument() const final { return "convert-linalg-to-buckyball"; } + StringRef getDescription() const final { + return "Collect the marked MegaKernel regions. Each Ball claims its own " + "Linalg operations in convert-linalg-to--ball."; + } + + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert<::buddy::buckyball::BuckyballDialect, func::FuncDialect, + memref::MemRefDialect, arith::ArithDialect, scf::SCFDialect, + linalg::LinalgDialect>(); + } + + void runOnOperation() override { + MLIRContext *context = &getContext(); + { + RewritePatternSet cancel(context); + cancel.add( + context); + if (failed(applyPatternsGreedily(getOperation(), std::move(cancel)))) + return signalPassFailure(); + } + ConversionTarget target(*context); + target.addLegalDialect<::buddy::buckyball::BuckyballDialect, + memref::MemRefDialect, arith::ArithDialect, + scf::SCFDialect, func::FuncDialect>(); + // Marked stages must lower; other Linalg operations stay unknown. + target.addDynamicallyLegalOp( + [](linalg::GenericOp op) -> std::optional { + if (op->hasAttr("buckyball.mega_kernel")) + return false; + return std::nullopt; + }); + + RewritePatternSet patterns(context); + mlir::buddy::populateMegaKernelLinalgLoweringPatterns(patterns); + if (failed(applyPartialConversion(getOperation(), target, + std::move(patterns)))) + signalPassFailure(); + } +}; +} // namespace + +void mlir::buddy::registerLowerLinalgToBuckyballPass() { + PassRegistration(); +} diff --git a/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/MegaKernelPatterns.cpp b/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/MegaKernelPatterns.cpp new file mode 100644 index 000000000..e7bc1558f --- /dev/null +++ b/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/MegaKernelPatterns.cpp @@ -0,0 +1,346 @@ +//===- MegaKernelPatterns.cpp - marked MegaKernel regions ----------------===// +// +// Collects the generics the frontend marked as one cross-layer resident +// quantized MegaKernel into buckyball.mega_kernel. +// +//===----------------------------------------------------------------------===// + +#include + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/IR/PatternMatch.h" + +#include "Buckyball/BuckyballOps.h" +#include "Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +bool positive(FloatAttr attr) { + return attr && std::isfinite(attr.getValueAsDouble()) && + attr.getValueAsDouble() > 0.0; +} + +bool staticInt8(Value value, int64_t rank) { + auto type = dyn_cast(value.getType()); + return type && type.hasStaticShape() && type.getRank() == rank && + type.getElementType().isInteger(8); +} + +// The stages of one MegaKernel are consecutive generics sharing +// `mega_kernel_id`; each stage names its kind with a `buckyball.mega_*` marker. +class MegaKernelLowering : public OpRewritePattern { +public: + MegaKernelLowering(MLIRContext *context) + : OpRewritePattern(context, 2) {} + + LogicalResult matchAndRewrite(linalg::GenericOp op, + PatternRewriter &rewriter) const override { + auto first = op->getAttrOfType("mega_kernel_stage"); + if (!op->hasAttr("buckyball.mega_kernel") || !first || first.getInt() != 0) + return failure(); + auto size = op->getAttrOfType("mega_kernel_size"); + auto kernelId = op->getAttrOfType("mega_kernel_id"); + if (!size || size.getInt() <= 0 || !kernelId || kernelId.getValue().empty()) + return op.emitError("MegaKernel requires a positive size and an ID"); + + SmallVector stages(size.getInt()); + for (Operation &candidate : *op->getBlock()) { + auto generic = dyn_cast(candidate); + if (!generic || + generic->getAttrOfType("mega_kernel_id") != kernelId) + continue; + auto index = generic->getAttrOfType("mega_kernel_stage"); + auto stageSize = generic->getAttrOfType("mega_kernel_size"); + if (!generic->hasAttr("buckyball.mega_kernel") || !index || !stageSize || + stageSize.getInt() != size.getInt() || index.getInt() < 0 || + index.getInt() >= size.getInt()) + return generic.emitError("MegaKernel stage metadata is malformed"); + if (stages[index.getInt()]) + return generic.emitError("MegaKernel stage index is duplicated"); + stages[index.getInt()] = generic; + } + if (llvm::any_of(stages, [](linalg::GenericOp stage) { return !stage; })) + return op.emitError("MegaKernel stage sequence is incomplete"); + + Value kernelOutput = stages.back().getOutputs()[0]; + rewriter.setInsertionPointAfter(stages.back()); + auto kernel = rewriter.create( + op.getLoc(), stages.front().getInputs()[0], kernelOutput); + kernel.getBody().emplaceBlock(); + + OpBuilder::InsertionGuard guard(rewriter); + rewriter.setInsertionPointToStart(&kernel.getBody().front()); + for (auto [index, current] : llvm::enumerate(stages)) { + bool last = index + 1 == stages.size(); + bool normal = current->hasAttr("buckyball.mega_conv2d"); + bool depthwise = current->hasAttr("buckyball.mega_conv2d_depthwise"); + bool matmul = current->hasAttr("buckyball.mega_matmul"); + bool maxPool = current->hasAttr("buckyball.mega_max_pool2d"); + bool globalAvg = current->hasAttr("buckyball.mega_global_avg_pool"); + bool channelSlice = current->hasAttr("buckyball.mega_channel_slice"); + bool channelConcat = current->hasAttr("buckyball.mega_channel_concat"); + bool resizeNearest = current->hasAttr("buckyball.mega_resize_nearest"); + bool int8Mul = current->hasAttr("buckyball.mega_int8_mul"); + bool int8Add = current->hasAttr("buckyball.mega_int8_add"); + if (int(normal) + int(depthwise) + int(matmul) + int(globalAvg) + + int(maxPool) + int(channelSlice) + int(channelConcat) + + int(resizeNearest) + int(int8Mul) + int(int8Add) != + 1) + return current.emitError("MegaKernel stage kind is not unique"); + Location loc = current.getLoc(); + ValueRange inputs = current.getInputs(); + if (current.getOutputs().size() != 1) + return current.emitError("MegaKernel stage needs one output"); + Value output = current.getOutputs()[0]; + auto outputType = dyn_cast(output.getType()); + if (!outputType || !outputType.hasStaticShape() || + (outputType.getRank() != 4 && !matmul)) + return current.emitError("MegaKernel stage needs a static output"); + ArrayRef out = outputType.getShape(); + // Only the last stage writes the kernel output. + bool boundary = last ? output == kernelOutput : output != kernelOutput; + + if (channelSlice) { + auto offset = current->getAttrOfType("offset"); + if (inputs.size() != 1 || !staticInt8(inputs[0], 4) || + !staticInt8(output, 4) || !offset || offset.getInt() < 0 || + offset.getInt() % 16 != 0 || out[3] % 16 != 0) + return current.emitError("Mega channel slice contract is invalid"); + auto in = cast(inputs[0].getType()).getShape(); + if (in[0] != out[0] || in[1] != out[1] || in[2] != out[2] || + offset.getInt() + out[3] > in[3]) + return current.emitError("Mega channel slice contract is invalid"); + rewriter.create(loc, inputs[0], output, offset); + continue; + } + + if (channelConcat) { + auto segments = current->getAttrOfType("segments"); + if (inputs.empty() || !staticInt8(output, 4) || !segments || + segments.size() != inputs.size()) + return current.emitError("Mega channel concat has the wrong arity"); + int64_t channels = 0; + for (auto [input, segment] : llvm::zip(inputs, segments.asArrayRef())) { + if (!staticInt8(input, 4) || segment <= 0 || segment % 16 != 0) + return current.emitError("Mega channel concat contract is invalid"); + auto in = cast(input.getType()).getShape(); + if (in[0] != out[0] || in[1] != out[1] || in[2] != out[2] || + in[3] != segment) + return current.emitError("Mega channel concat contract is invalid"); + channels += segment; + } + if (out[3] != channels) + return current.emitError( + "Mega channel concat output channels are invalid"); + rewriter.create(loc, inputs, output, segments); + continue; + } + + if (resizeNearest) { + auto scaleH = current->getAttrOfType("scale_h"); + auto scaleW = current->getAttrOfType("scale_w"); + if (inputs.size() != 1 || !staticInt8(inputs[0], 4) || + !staticInt8(output, 4) || !scaleH || !scaleW || + scaleH.getInt() <= 0 || scaleW.getInt() <= 0) + return current.emitError("Mega resize-nearest contract is invalid"); + auto in = cast(inputs[0].getType()).getShape(); + if (in[0] != out[0] || in[3] != out[3] || + out[1] != in[1] * scaleH.getInt() || + out[2] != in[2] * scaleW.getInt()) + return current.emitError("Mega resize-nearest contract is invalid"); + rewriter.create(loc, inputs[0], output, scaleH, + scaleW); + continue; + } + + if (maxPool) { + auto kernelSize = current->getAttrOfType("kernel"); + auto stride = current->getAttrOfType("stride"); + auto padding = current->getAttrOfType("padding"); + auto finalOutput = current->getAttrOfType("final_output"); + if (inputs.size() != 1 || !staticInt8(inputs[0], 4) || + !staticInt8(output, 4) || !kernelSize || !stride || !padding || + !finalOutput || kernelSize.getInt() <= 0 || stride.getInt() <= 0 || + padding.getInt() < 0 || (finalOutput.getValue() && !last)) + return current.emitError("Mega MaxPool2D contract is invalid"); + auto in = cast(inputs[0].getType()).getShape(); + bool nchw = finalOutput.getValue(); + int64_t outH = nchw ? out[2] : out[1]; + int64_t outW = nchw ? out[3] : out[2]; + int64_t outC = nchw ? out[1] : out[3]; + int64_t window = 2 * padding.getInt() - kernelSize.getInt(); + if (out[0] != in[0] || outC != in[3] || + outH != (in[1] + window) / stride.getInt() + 1 || + outW != (in[2] + window) / stride.getInt() + 1) + return current.emitError("Mega MaxPool2D shape is invalid"); + rewriter.create(loc, inputs[0], output, kernelSize, + stride, padding, finalOutput); + continue; + } + + if (globalAvg) { + auto inputScale = current->getAttrOfType("input_scale"); + auto outputScale = current->getAttrOfType("output_scale"); + if (!boundary || inputs.size() != 1 || !staticInt8(inputs[0], 4) || + !staticInt8(output, 4) || !positive(inputScale) || + !positive(outputScale)) + return current.emitError("Mega global-average contract is invalid"); + auto in = cast(inputs[0].getType()).getShape(); + if (in[0] != out[0] || in[3] != out[3] || out[1] != 1 || out[2] != 1) + return current.emitError("Mega global-average contract is invalid"); + rewriter.create(loc, inputs[0], output, inputScale, + outputScale); + continue; + } + + if (int8Mul || int8Add) { + auto lhsScale = current->getAttrOfType("lhs_scale"); + auto rhsScale = current->getAttrOfType("rhs_scale"); + auto outputScale = current->getAttrOfType("output_scale"); + auto activation = current->getAttrOfType("activation"); + if (!boundary || inputs.size() != 2 || !staticInt8(inputs[0], 4) || + !staticInt8(inputs[1], 4) || !staticInt8(output, 4) || + !positive(lhsScale) || !positive(rhsScale) || + !positive(outputScale) || !activation || activation.getInt() < 0 || + activation.getInt() > 1) + return current.emitError("Mega INT8 elementwise contract is invalid"); + auto lhs = cast(inputs[0].getType()); + auto rhs = cast(inputs[1].getType()); + for (int64_t dimension = 0; dimension < 4; ++dimension) + if ((lhs.getShape()[dimension] != 1 && + lhs.getShape()[dimension] != out[dimension]) || + (rhs.getShape()[dimension] != 1 && + rhs.getShape()[dimension] != out[dimension])) + return current.emitError( + "Mega INT8 elementwise shapes do not broadcast"); + if (int8Add && lhs != rhs) + return current.emitError( + "Mega residual add requires equal input types"); + if (int8Mul) + rewriter.create(loc, inputs[0], inputs[1], output, + lhsScale, rhsScale, outputScale, + activation); + else + rewriter.create(loc, inputs[0], inputs[1], output, + lhsScale, rhsScale, outputScale, + activation); + continue; + } + + // Compute stages: input, weight, bias, scale and activation LUT. + auto activation = current->getAttrOfType("activation"); + auto finalOutput = current->getAttrOfType("final_output"); + auto outputScale = current->getAttrOfType("output_scale"); + if (!boundary || inputs.size() != 5 || !activation || + activation.getInt() < 0 || activation.getInt() > 2 || !finalOutput || + !positive(outputScale) || (finalOutput.getValue() && !last)) + return current.emitError("Mega compute stage contract is invalid"); + Value input = inputs[0], weight = inputs[1], bias = inputs[2], + scale = inputs[3], lut = inputs[4]; + auto inputType = cast(input.getType()); + auto weightType = cast(weight.getType()); + auto biasType = cast(bias.getType()); + auto scaleType = cast(scale.getType()); + auto lutType = cast(lut.getType()); + if (!inputType.hasStaticShape() || !weightType.hasStaticShape() || + !biasType.hasStaticShape() || !scaleType.hasStaticShape() || + !staticInt8(lut, 1)) + return current.emitError("Mega compute stage requires static shapes"); + // A 4096-entry LUT holds one table per lane of a 16-channel Conv whose + // only consumer is the depthwise stage after it. + int64_t lutSize = lutType.getShape()[0]; + if (lutSize == 4096) { + int64_t consumers = 0; + for (linalg::GenericOp candidate : stages) + consumers += llvm::is_contained(candidate.getInputs(), output); + if (!normal || index != 0 || activation.getInt() != 2 || out[3] != 16 || + stages.size() < 2 || + !stages[1]->hasAttr("buckyball.mega_conv2d_depthwise") || + stages[1].getInputs()[0] != output || consumers != 1) + return current.emitError( + "4096-entry lane LUT is only legal on stage 0 before one " + "16-channel depthwise Conv"); + } else if (lutSize != (activation.getInt() == 2 ? 256 : 1)) { + return current.emitError( + "MegaKernel activation LUT has the wrong shape"); + } + // Only FP32 marks the model-visible result; INT8 stays inside. + Type element = outputType.getElementType(); + if (last ? !element.isF32() && !element.isInteger(8) + : !element.isInteger(8)) + return current.emitError("MegaKernel stage output type is invalid"); + + if (matmul) { + int64_t m = inputType.getShape()[0], k = inputType.getShape()[1]; + int64_t n = weightType.getShape()[1]; + if (m <= 0 || k <= 0 || n <= 0 || weightType.getShape()[0] != k || + out[0] != m || out[1] != n || biasType.getShape()[0] != n || + scaleType.getShape()[0] != n || (last && !element.isF32())) + return current.emitError("MegaKernel MatMul shape mismatch"); + rewriter.create(loc, input, weight, bias, scale, lut, + output, activation, outputScale); + continue; + } + + auto kernelSize = current->getAttrOfType("kernel"); + auto stride = current->getAttrOfType("stride"); + auto padLow = current->getAttrOfType("pad_low"); + auto padHigh = current->getAttrOfType("pad_high"); + if (!kernelSize || !stride || !padLow || !padHigh) + return current.emitError( + "MegaKernel convolution attributes are missing"); + int64_t k = kernelSize.getInt(), s = stride.getInt(); + int64_t pads = padLow.getInt() + padHigh.getInt(); + auto in = inputType.getShape(); + auto filter = weightType.getShape(); + auto outputNhwc = current->getAttrOfType("output_nhwc"); + bool nchw = + last && element.isF32() && !(outputNhwc && outputNhwc.getValue()); + int64_t outC = nchw ? out[1] : out[3]; + int64_t outH = nchw ? out[2] : out[1]; + int64_t outW = nchw ? out[3] : out[2]; + int64_t channels = depthwise ? in[3] : outC; + bool filterValid = + depthwise ? filter == ArrayRef({k, k, in[3], 1}) + : filter == ArrayRef({(outC + 15) / 16, in[3], + (k * k + 15) / 16 * 16, 16}); + if (in[0] <= 0 || in[1] <= 0 || in[2] <= 0 || in[3] <= 0 || k <= 0 || + k > 7 || s <= 0 || padLow.getInt() < 0 || padHigh.getInt() < 0 || + !filterValid || out[0] != in[0] || outC != channels || + biasType.getShape()[0] != channels || + scaleType.getShape()[0] != channels || in[1] + pads < k || + in[2] + pads < k || outH != (in[1] + pads - k) / s + 1 || + outW != (in[2] + pads - k) / s + 1) + return current.emitError("MegaKernel convolution shape mismatch"); + Operation *lowered; + if (depthwise) + lowered = rewriter.create( + loc, input, weight, bias, scale, lut, output, kernelSize, stride, + padLow, padHigh, activation, outputScale); + else + lowered = rewriter.create( + loc, input, weight, bias, scale, lut, output, kernelSize, stride, + padLow, padHigh, activation, outputScale); + if (outputNhwc) + lowered->setAttr("output_nhwc", outputNhwc); + } + for (auto [source, target] : + llvm::zip_equal(stages, kernel.getBody().front())) + if (auto trace = source->getAttr("trace_node")) + target.setAttr("trace_node", trace); + rewriter.create(op.getLoc()); + for (int64_t index = stages.size(); index > 0; --index) + rewriter.eraseOp(stages[index - 1]); + return success(); + } +}; +} // namespace + +void mlir::buddy::populateMegaKernelLinalgLoweringPatterns( + RewritePatternSet &patterns) { + patterns.add(patterns.getContext()); +} diff --git a/waste/stack/compiler/src/Dialect/Buckyball/Buckyball.td b/waste/stack/compiler/src/Dialect/Buckyball/Buckyball.td new file mode 100644 index 000000000..3aa7b2a81 --- /dev/null +++ b/waste/stack/compiler/src/Dialect/Buckyball/Buckyball.td @@ -0,0 +1,446 @@ +//===-- Buckyball.td - Buckyball dialect operations -----*- tablegen -*-===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// + +#ifndef BUCKYBALL_OPS +#define BUCKYBALL_OPS + +include "mlir/Dialect/LLVMIR/LLVMOpBase.td" +include "mlir/IR/AttrTypeBase.td" +include "mlir/IR/PatternBase.td" +include "BuckyballInterfaces.td" + +//===----------------------------------------------------------------------===// +// Dialect and common operations +//===----------------------------------------------------------------------===// + +def Buckyball_Dialect : Dialect { + let name = "buckyball"; + let cppNamespace = "::buddy::buckyball"; + let summary = "Dialect for RISC-V Buckyball NPU extension"; + let description = [{ + Buckyball is an NPU Design Framework with composable Ball units. + Each Ball is a specialized compute unit that can be independently + developed and combined with others. + }]; +} + +class Buckyball_Op traits = []> : + Op {} + +def MvoverOp : Buckyball_Op<"mvover"> { + let summary = "Move contiguous rows between logical Thread-local Banks"; + let arguments = (ins I64:$sourceThread, I64:$sourceBank, I64:$sourceRow, + I64:$targetThread, I64:$targetBank, I64:$targetRow, + I64:$rows); + let assemblyFormat = "$sourceThread $sourceBank $sourceRow $targetThread " + "$targetBank $targetRow $rows prop-dict attr-dict"; +} + +def MvinOp : Buckyball_Op<"mvin"> { + let summary = "Load data from memory to scratchpad bank"; + let description = [{ + Loads data from DRAM to scratchpad bank. + Corresponds to bb_mvin instruction (funct7=33). + }]; + let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, + I64:$addr, I64:$depth, I64:$stride, + OptionalAttr:$group); + let assemblyFormat = "$input $addr $depth $stride prop-dict attr-dict `:` " + "type($input) type($addr) type($depth) type($stride)"; + let hasVerifier = 1; +} + +def Mvin2dOp : Buckyball_Op<"mvin_2d"> { + let summary = "Load an INT8 channel panel rectangle into a scratchpad bank"; + let arguments = (ins MemRefRankOf<[I8], [4]>:$input, I64:$addr, + I64:$height, I64:$pixelBytes, I64:$sourceWidth, + I64:$dstBase, I64:$width, I64:$validBytes); + let assemblyFormat = "$input $addr $height $pixelBytes $sourceWidth " + "$dstBase $width $validBytes prop-dict attr-dict `:` " + "type($input) type($addr) type($height) " + "type($pixelBytes) type($sourceWidth) " + "type($dstBase) type($width) type($validBytes)"; +} + +def MvinMmioOp : Buckyball_Op<"mvin_mmio"> { + let summary = "Load metadata from memory into MMIO SRAM"; + let arguments = (ins MemRefRankOf<[I8], [1, 2]>:$input, + I64:$mmioAddr, I64:$rows, I64:$columns); + let assemblyFormat = "$input $mmioAddr $rows $columns prop-dict attr-dict `:` " + "type($input) type($mmioAddr) type($rows) type($columns)"; +} + +def MvoutOp : Buckyball_Op<"mvout"> { + let summary = "Store data from scratchpad bank to memory"; + let description = [{ + Stores data from scratchpad bank to DRAM. + Corresponds to bb_mvout instruction (funct7=16). + }]; + let arguments = (ins MemRefRankOf<[AnyType], [2]>:$output, + I64:$addr, I64:$depth, I64:$stride, + OptionalAttr:$group); + let assemblyFormat = "$output $addr $depth $stride prop-dict attr-dict `:` " + "type($output) type($addr) type($depth) type($stride)"; + let hasVerifier = 1; +} + +def MsetOp : Buckyball_Op<"mset"> { + let summary = "Allocate or release scratchpad bank memory"; + let description = [{ + Allocates or releases scratchpad bank memory. + Corresponds to bb_mset instruction (funct7=32). + }]; + let arguments = (ins I64:$bankId, + DefaultValuedAttr:$alloc, + DefaultValuedAttr:$row, + DefaultValuedAttr:$col); + let assemblyFormat = "$bankId prop-dict attr-dict `:` type($bankId)"; +} + +def MsetTransferOp : Buckyball_Op<"mset_transfer"> { + let summary = "Transfer bank mappings to a target virtual bank"; + let arguments = (ins I64:$source, I64:$target); + let assemblyFormat = "$source $target prop-dict attr-dict `:` type($source) type($target)"; + let hasVerifier = 1; +} + +//===----------------------------------------------------------------------===// +// Bank SSA operations +//===----------------------------------------------------------------------===// + +def BankAllocOp : Buckyball_Op<"bank_alloc", [MemoryEffects<[MemRead, MemWrite]>]> { + let summary = "Allocate a virtual on-chip bank resource"; + let description = [{ + Create a virtual bank handle in SSA form. + - row/col encode bank-array shape used by allocator. + - The result i64 is a virtual bank handle, not physical id. + }]; + let arguments = (ins + DefaultValuedAttr:$row, + DefaultValuedAttr:$col); + let results = (outs I64:$bank); + let assemblyFormat = "prop-dict attr-dict"; +} + +def BankReleaseOp : Buckyball_Op<"bank_release", [MemoryEffects<[MemRead, MemWrite]>]> { + let summary = "Release a virtual bank handle"; + let arguments = (ins I64:$bank); + let assemblyFormat = "$bank prop-dict attr-dict `:` type($bank)"; +} + +def BankTransferOp : Buckyball_Op<"bank_transfer", [MemoryEffects<[MemRead, MemWrite]>]> { + let summary = "Consume a bank mapping and append its groups to a target handle"; + let arguments = (ins I64:$source, I64:$target); + let results = (outs I64:$bank); + let assemblyFormat = "$source $target prop-dict attr-dict `:` type($source) type($target)"; +} + +def BankKernelOp : Buckyball_Op<"bank_kernel", [MemoryEffects<[MemRead, MemWrite]>]> { + let summary = "Call a synchronous kernel with resident read/write bank handles"; + let arguments = (ins I64:$readBank, I64:$writeBank, + FlatSymbolRefAttr:$callee, Variadic:$kernelArgs); + let results = (outs I64:$readOut, I64:$writeOut); + let assemblyFormat = "$callee $readBank $writeBank `(` $kernelArgs `)` " + "prop-dict attr-dict `:` type($kernelArgs)"; +} + +def BankMvinOp : Buckyball_Op<"bank_mvin", [MemoryEffects<[MemRead, MemWrite]>]> { + let summary = "Load into a virtual bank handle"; + let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, I64:$bank, I64:$depth, + I64:$stride, OptionalAttr:$group); + let results = (outs I64:$bankOut); + let assemblyFormat = + "$input $bank $depth $stride prop-dict attr-dict `:` type($input) type($bank) type($depth) type($stride)"; +} + +def BankMvin2dOp : Buckyball_Op<"bank_mvin_2d"> { + let summary = "Load a rectangular INT8 channel panel into a virtual bank"; + let arguments = (ins MemRefRankOf<[I8], [4]>:$input, I64:$bank, + I64:$height, I64:$pixelBytes, I64:$sourceWidth, + I64:$dstBase, I64:$width, I64:$validBytes); + let results = (outs I64:$bankOut); + let assemblyFormat = "$input $bank $height $pixelBytes $sourceWidth " + "$dstBase $width $validBytes prop-dict attr-dict `:` " + "type($input) type($bank) type($height) " + "type($pixelBytes) type($sourceWidth) " + "type($dstBase) type($width) type($validBytes)"; +} + +def BankMvoutOp : Buckyball_Op<"bank_mvout", [MemoryEffects<[MemRead, MemWrite]>]> { + let summary = "Store from a virtual bank handle"; + let arguments = (ins MemRefRankOf<[AnyType], [2]>:$output, I64:$bank, I64:$depth, + I64:$stride, OptionalAttr:$group); + let results = (outs I64:$bankOut); + let assemblyFormat = + "$output $bank $depth $stride prop-dict attr-dict `:` type($output) type($bank) type($depth) type($stride)"; +} + +//===----------------------------------------------------------------------===// +// Mega kernel operations +//===----------------------------------------------------------------------===// + +def MegaYieldOp : Buckyball_Op<"mega_yield", [Terminator]> { + let summary = "Terminate a Buckyball MegaKernel region"; + let assemblyFormat = "prop-dict attr-dict"; +} + +def MegaKernelOp : Buckyball_Op<"mega_kernel", [ + SingleBlockImplicitTerminator<"MegaYieldOp">]> { + let summary = "One cross-layer resident quantized kernel"; + let arguments = (ins AnyMemRef:$input, + AnyMemRef:$output); + let regions = (region SizedRegion<1>:$body); + let assemblyFormat = [{$input $output prop-dict attr-dict `:` + type($input) type($output) $body}]; +} + +def MegaMatmulOp : Buckyball_Op<"mega_matmul"> { + let summary = "Quantized MatMul stage inside a MegaKernel"; + let description = [{ + One INT8 MatMul stage. Bias is already converted offline to INT32. A + non-final result is requantized to INT8 and stays inside the MegaKernel. + }]; + let arguments = (ins MemRefRankOf<[I8], [2]>:$input, + MemRefRankOf<[I8], [2]>:$weight, + MemRefRankOf<[I32], [1]>:$bias, + MemRefRankOf<[F32], [1]>:$scale, + MemRefRankOf<[I8], [1]>:$lut, + MemRefRankOf<[I8, F32], [2]>:$output, + I64Attr:$activation, + F32Attr:$outputScale); + let assemblyFormat = [{$input $weight $bias $scale $lut $output prop-dict attr-dict `:` + type($input) type($weight) type($bias) type($scale) type($lut) type($output)}]; +} + +def MegaConv2dOp : Buckyball_Op<"mega_conv2d"> { + let summary = "Quantized convolution stage inside a MegaKernel"; + let description = [{ + One resident NHWC INT8 convolution stage. Bias is already converted + offline to INT32. A non-final result is requantized to INT8 and stays + inside the MegaKernel. Activation LUTs contain either one shared + 256-entry table or sixteen channel-major tables. + }]; + let arguments = (ins MemRefRankOf<[I8], [4]>:$input, + MemRefRankOf<[I8], [4]>:$weight, + MemRefRankOf<[I32], [1]>:$bias, + MemRefRankOf<[F32], [1]>:$scale, + MemRefRankOf<[I8], [1]>:$lut, + MemRefRankOf<[I8, F32], [4]>:$output, + I64Attr:$kernel, + I64Attr:$stride, + I64Attr:$padLow, + I64Attr:$padHigh, + I64Attr:$activation, + F32Attr:$outputScale); + let assemblyFormat = [{$input $weight $bias $scale $lut $output prop-dict attr-dict `:` + type($input) type($weight) type($bias) type($scale) type($lut) type($output)}]; +} + +def MegaConv2dDepthwiseOp : Buckyball_Op<"mega_conv2d_depthwise"> { + let summary = "Quantized depthwise stage inside a MegaKernel"; + let description = [{ + One complete resident NHWC INT8 depthwise convolution layer with channel + multiplier one. Bias is converted offline to INT32 and output is + requantized to INT8 before leaving the mega kernel. + }]; + let arguments = (ins MemRefRankOf<[I8], [4]>:$input, + MemRefRankOf<[I8], [4]>:$weight, + MemRefRankOf<[I32], [1]>:$bias, + MemRefRankOf<[F32], [1]>:$scale, + MemRefRankOf<[I8], [1]>:$lut, + MemRefRankOf<[I8, F32], [4]>:$output, + I64Attr:$kernel, + I64Attr:$stride, + I64Attr:$padLow, + I64Attr:$padHigh, + I64Attr:$activation, + F32Attr:$outputScale); + let assemblyFormat = [{$input $weight $bias $scale $lut $output prop-dict attr-dict `:` + type($input) type($weight) type($bias) type($scale) type($lut) type($output)}]; +} + +def MegaMaxPool2dOp : Buckyball_Op<"mega_max_pool2d"> { + let summary = "INT8 MaxPool2D stage inside a MegaKernel"; + let description = [{ + Max pooling preserves the INT8 quantization scale. An intermediate output + is NHWC; a final output is NCHW so a following reshape preserves PyTorch + flatten order without a transpose operation. + }]; + let arguments = (ins MemRefRankOf<[I8], [4]>:$input, + MemRefRankOf<[I8], [4]>:$output, + I64Attr:$kernel, + I64Attr:$stride, + I64Attr:$padding, + BoolAttr:$finalOutput); + let assemblyFormat = [{$input $output prop-dict attr-dict `:` + type($input) type($output)}]; +} + +def MegaGlobalAvgPoolOp : Buckyball_Op<"mega_global_avg_pool"> { + let summary = "Global-average an INT8 NHWC tensor inside a MegaKernel"; + let description = [{ + This is the explicit X-memory boundary used by an SE branch. The result is + requantized to INT8 before the following pointwise stage. + }]; + let arguments = (ins MemRefRankOf<[I8], [4]>:$input, + MemRefRankOf<[I8], [4]>:$output, + F32Attr:$inputScale, + F32Attr:$outputScale); + let assemblyFormat = [{$input $output prop-dict attr-dict `:` + type($input) type($output)}]; +} + +def MegaInt8MulOp : Buckyball_Op<"mega_int8_mul"> { + let summary = "Scaled INT8 broadcast multiply inside a MegaKernel"; + let arguments = (ins MemRefRankOf<[I8], [4]>:$lhs, + MemRefRankOf<[I8], [4]>:$rhs, + MemRefRankOf<[I8], [4]>:$output, + F32Attr:$lhsScale, + F32Attr:$rhsScale, + F32Attr:$outputScale, + I64Attr:$activation); + let assemblyFormat = [{$lhs $rhs $output prop-dict attr-dict `:` + type($lhs) type($rhs) type($output)}]; +} + +def MegaInt8AddOp : Buckyball_Op<"mega_int8_add"> { + let summary = "Scaled INT8 residual add inside a MegaKernel"; + let arguments = (ins MemRefRankOf<[I8], [4]>:$lhs, + MemRefRankOf<[I8], [4]>:$rhs, + MemRefRankOf<[I8], [4]>:$output, + F32Attr:$lhsScale, + F32Attr:$rhsScale, + F32Attr:$outputScale, + I64Attr:$activation); + let assemblyFormat = [{$lhs $rhs $output prop-dict attr-dict `:` + type($lhs) type($rhs) type($output)}]; +} + +def MegaChannelSliceOp : Buckyball_Op<"mega_channel_slice"> { + let summary = "INT8 NHWC channel slice inside a MegaKernel"; + let arguments = (ins MemRefRankOf<[I8], [4]>:$input, + MemRefRankOf<[I8], [4]>:$output, + I64Attr:$offset); + let assemblyFormat = [{$input $output prop-dict attr-dict `:` type($input) type($output)}]; +} + +def MegaChannelConcatOp : Buckyball_Op<"mega_channel_concat"> { + let summary = "INT8 NHWC channel concatenation inside a MegaKernel"; + let arguments = (ins Variadic>:$inputs, + MemRefRankOf<[I8], [4]>:$output, + DenseI64ArrayAttr:$segments); + let assemblyFormat = [{$inputs $output prop-dict attr-dict `:` type($inputs) type($output)}]; +} + +def MegaResizeNearestOp : Buckyball_Op<"mega_resize_nearest"> { + let summary = "INT8 NHWC nearest-neighbor resize inside a MegaKernel"; + let arguments = (ins MemRefRankOf<[I8], [4]>:$input, + MemRefRankOf<[I8], [4]>:$output, + I64Attr:$scaleH, + I64Attr:$scaleW); + let assemblyFormat = [{$input $output prop-dict attr-dict `:` type($input) type($output)}]; +} + +//===----------------------------------------------------------------------===// +// LLVM intrinsic operations +//===----------------------------------------------------------------------===// + +class Buckyball_IntrOpBase traits = []> : + LLVM_IntrOpBase overloadedResults=*/[], + /*list overloadedOperands=*/[], + /*list traits=*/traits, + /*int numResults=*/0>; + +class Buckyball_CustomIntrOpBase immArgPositions, + list immArgAttrNames, + list traits = []> : + LLVM_IntrOpBase overloadedResults=*/[], + /*list overloadedOperands=*/[], + /*list traits=*/traits, + /*int numResults=*/0, + /*bit requiresAccessGroup=*/0, + /*bit requiresAliasAnalysis=*/0, + /*bit requiresFastmath=*/0, + /*bit requiresArgAndResultAttrs=*/0, + /*bit requiresOpBundles=*/0, + /*list immArgPositions=*/immArgPositions, + /*list immArgAttrNames=*/immArgAttrNames>; + +def MsetIntrOp : Buckyball_IntrOpBase<"mset">, + Arguments<(ins LLVM_Type, LLVM_Type)>; + +def MvinIntrOp : Buckyball_IntrOpBase<"mvin">, + Arguments<(ins LLVM_Type, LLVM_Type)>; + +def MvinMmioIntrOp : Buckyball_IntrOpBase<"mvin_mmio">, + Arguments<(ins LLVM_Type, LLVM_Type)>; + +def MvoutIntrOp : Buckyball_IntrOpBase<"mvout">, + Arguments<(ins LLVM_Type, LLVM_Type)>; + + + + + +def CustomIntrOp + : Buckyball_CustomIntrOpBase<"custom", "riscv_buckyball_custom", + [2], ["funct7"]>, + Arguments<(ins LLVM_Type:$rs1, LLVM_Type:$rs2, + ConfinedAttr, IntMaxValue<127>]>:$funct7)> { + let assemblyFormat = "$rs1 `,` $rs2 prop-dict attr-dict `:` type($rs1) `,` type($rs2)"; +} + +//===----------------------------------------------------------------------===// +// Generic instruction and debug operations +//===----------------------------------------------------------------------===// + +def InstOp : Buckyball_Op<"inst"> { + let summary = "Generic Buckyball custom instruction carrier"; + let description = [{ + Emits a Buckyball custom instruction with explicit rs1, rs2 and funct7. + }]; + let arguments = (ins I64:$rs1, I64:$rs2, + ConfinedAttr, IntMaxValue<127>]>:$funct7); + let assemblyFormat = "$rs1 `,` $rs2 prop-dict attr-dict `:` type($rs1)"; +} + +def PrintMemRefOp : Buckyball_Op<"print_memref"> { + let summary = "Print memref contents for debugging"; + let description = [{ + Print the contents of a memref for debugging purposes. + }]; + let arguments = (ins AnyMemRef:$input); + let assemblyFormat = "$input prop-dict attr-dict `:` type($input)"; +} + +def PrintScalarOp : Buckyball_Op<"print_scalar"> { + let summary = "Print scalar value for debugging"; + let description = [{ + Print a scalar value for debugging purposes. + }]; + let arguments = (ins AnyType:$value); + let assemblyFormat = "$value prop-dict attr-dict `:` type($value)"; +} + +#endif // BUCKYBALL_OPS diff --git a/waste/stack/compiler/src/Dialect/Buckyball/BuckyballDialect.h b/waste/stack/compiler/src/Dialect/Buckyball/BuckyballDialect.h new file mode 100644 index 000000000..3175776fb --- /dev/null +++ b/waste/stack/compiler/src/Dialect/Buckyball/BuckyballDialect.h @@ -0,0 +1,25 @@ +//===- BuckyballDialect.h - MLIR Dialect for RISC-V Gemmmini extension +//------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// + +#ifndef BUCKYBALL_BUCKYBALLDIALECT_H +#define BUCKYBALL_BUCKYBALLDIALECT_H + +#include "mlir/IR/Dialect.h" + +#include "BuckyballDialect.h.inc" + +#endif // BUCKYBALL_BUCKYBALLDIALECT_H diff --git a/waste/stack/compiler/src/Dialect/Buckyball/BuckyballInterfaces.td b/waste/stack/compiler/src/Dialect/Buckyball/BuckyballInterfaces.td new file mode 100644 index 000000000..cfa3534bd --- /dev/null +++ b/waste/stack/compiler/src/Dialect/Buckyball/BuckyballInterfaces.td @@ -0,0 +1,41 @@ +//===-- BuckyballInterfaces.td - Bank lowering interfaces --*- tablegen -*-===// +// +// The framework owns bank lowering. A Ball takes part by implementing these +// interfaces on its operations. +// +//===----------------------------------------------------------------------===// + +#ifndef BUCKYBALL_INTERFACES +#define BUCKYBALL_INTERFACES + +include "mlir/IR/OpBase.td" + +def BankSSALoweringInterface : OpInterface<"BankSSALowering"> { + let cppNamespace = "::buddy::buckyball"; + let description = [{ + A memref-level Ball operation that rewrites itself into explicit bank-SSA + operations: bank allocation, mvin, the bank-level Ball operation and mvout. + Lowered by `lower-buckyball-to-bank-ssa`. + }]; + let methods = [ + InterfaceMethod<"Replaces this operation with bank-SSA operations.", + "::mlir::LogicalResult", "lowerToBankSSA", + (ins "::mlir::PatternRewriter &":$rewriter)> + ]; +} + +def PhysicalBankAssignmentInterface : OpInterface<"PhysicalBankAssignment"> { + let cppNamespace = "::buddy::buckyball"; + let description = [{ + A bank-SSA Ball operation that rewrites itself into its physical-bank + operation once `assign-physical-banks` has mapped its bank handles. + }]; + let methods = [ + InterfaceMethod<"Replaces this operation with its physical operation.", + "::mlir::LogicalResult", "assignPhysicalBanks", + (ins "::mlir::PatternRewriter &":$rewriter, + "::mlir::buddy::PhysicalBankState &":$state)> + ]; +} + +#endif // BUCKYBALL_INTERFACES diff --git a/waste/stack/compiler/src/Dialect/Buckyball/BuckyballOps.h b/waste/stack/compiler/src/Dialect/Buckyball/BuckyballOps.h new file mode 100644 index 000000000..80e5ee63f --- /dev/null +++ b/waste/stack/compiler/src/Dialect/Buckyball/BuckyballOps.h @@ -0,0 +1,38 @@ +//===- BuckyballOps.h - MLIR Dialect for RISC-V Buckyball extension +//---------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// + +#ifndef BUCKYBALL_BUCKYBALLOPS_H +#define BUCKYBALL_BUCKYBALLOPS_H + +#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "mlir/IR/PatternMatch.h" + +namespace mlir::buddy { +class PhysicalBankState; +} // namespace mlir::buddy + +#include "BuckyballInterfaces.h.inc" + +#define GET_OP_CLASSES +#include "Buckyball.h.inc" +#endif // BUCKYBALL_BUCKYBALLOPS_H diff --git a/waste/stack/compiler/src/Dialect/Buckyball/IR/BuckyballDialect.cpp b/waste/stack/compiler/src/Dialect/Buckyball/IR/BuckyballDialect.cpp new file mode 100644 index 000000000..b1fb4fefe --- /dev/null +++ b/waste/stack/compiler/src/Dialect/Buckyball/IR/BuckyballDialect.cpp @@ -0,0 +1,79 @@ +//===- BuckyballDialect.cpp - MLIR Buckyball dialect implementation +//----------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +// +// This file implements the Buckyball dialect and its operations. +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/LLVMIR/LLVMTypes.h" +#include "mlir/Dialect/Vector/IR/VectorOps.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/DialectImplementation.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/IR/TypeUtilities.h" +#include "llvm/ADT/TypeSwitch.h" + +#include "Buckyball/BuckyballDialect.h" +#include "Buckyball/BuckyballOps.h" + +// Generated with unqualified names; included before the using-directives. +#include "BuckyballInterfaces.cpp.inc" + +using namespace mlir; +using namespace ::buddy::buckyball; + +#include "BuckyballDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "Buckyball.cpp.inc" + +void BuckyballDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "Buckyball.cpp.inc" + >(); +} + +LogicalResult MsetTransferOp::verify() { + for (Value operand : getOperands()) { + if (auto constant = operand.getDefiningOp()) + if (constant.value() < 0 || constant.value() > 1023) + return emitOpError("bank ID must be in [0, 1023]"); + } + auto source = getSource().getDefiningOp(); + auto target = getTarget().getDefiningOp(); + if (source && target) { + if (source.value() == target.value()) + return emitOpError("source and target must differ"); + } + return success(); +} + +LogicalResult MvinOp::verify() { + if (auto group = getGroup()) + if (*group < 0 || *group > 31) + return emitOpError("group must be in [0, 31]"); + return success(); +} + +LogicalResult MvoutOp::verify() { + if (auto group = getGroup()) + if (*group < 0 || *group > 31) + return emitOpError("group must be in [0, 31]"); + return success(); +} diff --git a/waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp b/waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp new file mode 100644 index 000000000..eb84d49d9 --- /dev/null +++ b/waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp @@ -0,0 +1,551 @@ +//===- LegalizeForLLVMExportBase.cpp - Buckyball base LLVM lowering -------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// + +#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" + +#include "mlir/Conversion/LLVMCommon/MemRefBuilder.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/IR/BuiltinOps.h" +#include "mlir/IR/BuiltinTypes.h" +#include "llvm/Support/ErrorHandling.h" + +#include "Buckyball/BuckyballDialect.h" +#include "Buckyball/BuckyballOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace buddy { +namespace buckyball { +namespace legalize { + +uint64_t fieldBits(uint64_t val, int startBit, int endBit) { + uint64_t width = endBit - startBit + 1; + uint64_t mask = (1ULL << width) - 1; + return (val & mask) << startBit; +} + +int64_t addrBitsForDepth(int64_t bankDepth) { + if (bankDepth <= 1) + return -1; + int64_t bits = 0; + int64_t x = bankDepth - 1; + while (x) { + x >>= 1; + ++bits; + } + return bits; +} + +Value cstI64(OpBuilder &b, Location loc, uint64_t v) { + return b.create(loc, b.getI64Type(), + b.getI64IntegerAttr(v)); +} + +static int64_t elemByteSize(Type el) { + if (auto it = dyn_cast(el)) + return it.getWidth() / 8; + if (auto ft = dyn_cast(el)) + return ft.getWidth() / 8; + return -1; +} + +struct MemrefAddress { + Value address; + Value hostPtr; +}; + +static MemrefAddress extractMemrefAddress(OpBuilder &b, Location loc, + Value memref) { + auto ty = cast(memref.getType()); + int64_t eb = elemByteSize(ty.getElementType()); + if (eb <= 0) + llvm_unreachable( + "bb memref intrinsic: unsupported element type for ptr offset"); + auto meta = b.create(loc, memref); + Value base = meta.getBaseBuffer(); + Value off = meta.getOffset(); + Value baseIdx = b.create( + loc, b.getIndexType(), base); + Value baseI64 = b.create(loc, b.getI64Type(), baseIdx); + Value offI64 = b.create(loc, b.getI64Type(), off); + Value offBytes = offI64; + if (eb != 1) + offBytes = b.create(loc, offI64, cstI64(b, loc, eb)); + Value address = b.create(loc, baseI64, offBytes); + Type ptrType = LLVM::LLVMPointerType::get(b.getContext()); + Value hostPtr = LLVM::IntToPtrOp::create(b, loc, ptrType, address); + return {address, hostPtr}; +} + +Value extractPtr(OpBuilder &b, Location loc, Value memref) { + return extractMemrefAddress(b, loc, memref).address; +} + +Value packRs1BanksIter(OpBuilder &b, Location loc, Value rBank0, Value rBank1, + Value wBank, Value iter) { + Value rBank0Field = + b.create(loc, rBank0, cstI64(b, loc, 0x3FF)); + Value rBank1Field = b.create( + loc, b.create(loc, rBank1, cstI64(b, loc, 0x3FF)), + cstI64(b, loc, 10)); + Value wBankField = b.create( + loc, b.create(loc, wBank, cstI64(b, loc, 0x3FF)), + cstI64(b, loc, 20)); + Value iterField = b.create( + loc, b.create(loc, iter, cstI64(b, loc, (1ULL << 34) - 1)), + cstI64(b, loc, 30)); + Value rs1Part01 = b.create(loc, rBank0Field, rBank1Field); + Value rs1Part012 = b.create(loc, rs1Part01, wBankField); + return b.create(loc, rs1Part012, iterField); +} + +Value packRs1BankIter(OpBuilder &b, Location loc, Value bankId, Value depth) { + Value z = cstI64(b, loc, 0); + return packRs1BanksIter(b, loc, bankId, z, z, depth); +} + +Value packRs1WriteBankIter(OpBuilder &b, Location loc, Value bankId, + Value depth) { + Value z = cstI64(b, loc, 0); + return packRs1BanksIter(b, loc, z, z, bankId, depth); +} + +Value packRs2MemStride(OpBuilder &b, Location loc, Value memAddr, + Value stride) { + Value mem = + b.create(loc, memAddr, cstI64(b, loc, (1ULL << 39) - 1)); + Value s = + b.create(loc, stride, cstI64(b, loc, (1ULL << 19) - 1)); + Value sHi = b.create(loc, s, cstI64(b, loc, 39)); + return b.create(loc, mem, sHi); +} + +void emitMset(OpBuilder &b, Location loc, uint64_t bankId, uint64_t row, + uint64_t col, uint64_t alloc) { + uint64_t rs1 = fieldBits(bankId, 0, 9); + uint64_t rs2 = + fieldBits(row, 0, 4) | fieldBits(col, 5, 9) | fieldBits(alloc, 10, 10); + b.create(loc, cstI64(b, loc, rs1), cstI64(b, loc, rs2)); +} + +static constexpr char kDmaTouchMvoutFn[] = "dma_touch_mvout"; + +static FlatSymbolRefAttr getOrInsertExtFunc(OpBuilder &b, ModuleOp module, + StringRef name, + LLVM::LLVMFunctionType type) { + if (module.lookupSymbol(name)) + return FlatSymbolRefAttr::get(b.getContext(), name); + + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToEnd(module.getBody()); + LLVM::LLVMFuncOp::create(b, module.getLoc(), name, type, + LLVM::Linkage::External, false, LLVM::CConv::C); + return FlatSymbolRefAttr::get(b.getContext(), name); +} + +static ModuleOp parentModule(OpBuilder &b) { + Operation *op = b.getInsertionBlock()->getParentOp(); + ModuleOp module = op->getParentOfType(); + if (!module) + llvm_unreachable("bb_dma: missing ModuleOp parent"); + return module; +} + +static void emitBbDmaBankColumns(OpBuilder &b, Location loc, StringRef name, + Value bankId, Value cols) { + ModuleOp module = parentModule(b); + auto i32Ty = IntegerType::get(b.getContext(), 32); + auto voidTy = LLVM::LLVMVoidType::get(b.getContext()); + auto fnTy = LLVM::LLVMFunctionType::get(voidTy, {i32Ty, i32Ty}); + FlatSymbolRefAttr callee = getOrInsertExtFunc(b, module, name, fnTy); + Value bankI32 = b.create(loc, i32Ty, bankId); + Value colsI32 = b.create(loc, i32Ty, cols); + LLVM::CallOp::create(b, loc, TypeRange{}, callee, + ValueRange{bankI32, colsI32}); +} + +static void emitBbDmaTouchMvout(OpBuilder &b, Location loc, Value hostPtr, + Value depth, Value stride, Value bankId) { + ModuleOp module = parentModule(b); + auto ptrTy = LLVM::LLVMPointerType::get(b.getContext()); + auto i64Ty = IntegerType::get(b.getContext(), 64); + auto i32Ty = IntegerType::get(b.getContext(), 32); + auto voidTy = LLVM::LLVMVoidType::get(b.getContext()); + auto fnTy = LLVM::LLVMFunctionType::get(voidTy, {ptrTy, i64Ty, i64Ty, i32Ty}); + FlatSymbolRefAttr callee = + getOrInsertExtFunc(b, module, kDmaTouchMvoutFn, fnTy); + Value bankI32 = b.create(loc, i32Ty, bankId); + LLVM::CallOp::create(b, loc, TypeRange{}, callee, + ValueRange{hostPtr, depth, stride, bankI32}); +} + +namespace { + +template +class ForwardOperands : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + LogicalResult + matchAndRewrite(OpTy op, typename OpTy::Adaptor adaptor, + ConversionPatternRewriter &rewriter) const final { + if (adaptor.getOperands().getTypes() == op->getOperands().getTypes()) + return rewriter.notifyMatchFailure(op, "operand types already match"); + rewriter.modifyOpInPlace(op, + [&]() { op->setOperands(adaptor.getOperands()); }); + return success(); + } +}; + +struct BuckyballMsetLowering : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + LogicalResult + matchAndRewrite(MsetOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + Value bankId = adaptor.getBankId(); + Value rs1 = rewriter.create(loc, bankId, + cstI64(rewriter, loc, 0x3FF)); + uint64_t allocBit = op.getAlloc() ? 1u : 0u; + uint64_t rowVal = op.getAlloc() ? static_cast(op.getRow()) : 0u; + uint64_t colVal = op.getAlloc() ? static_cast(op.getCol()) : 0u; + uint64_t rs2Val = fieldBits(rowVal, 0, 4) | fieldBits(colVal, 5, 9) | + fieldBits(allocBit, 10, 10); + Value colsForTouch = cstI64(rewriter, loc, op.getAlloc() ? colVal : 0u); + emitBbDmaBankColumns(rewriter, loc, + op.getAlloc() ? "dma_bank_allocate" + : "dma_bank_set_cols", + bankId, colsForTouch); + rewriter.replaceOpWithNewOp(op, rs1, + cstI64(rewriter, loc, rs2Val)); + return success(); + } +}; + +struct BuckyballMsetTransferLowering + : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + LogicalResult + matchAndRewrite(MsetTransferOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + Value source = adaptor.getSource(), target = adaptor.getTarget(); + auto i64 = rewriter.getI64Type(); + auto fnType = LLVM::LLVMFunctionType::get( + LLVM::LLVMVoidType::get(rewriter.getContext()), {i64, i64}); + auto callee = getOrInsertExtFunc(rewriter, parentModule(rewriter), + "dma_bank_transfer", fnType); + LLVM::CallOp::create(rewriter, loc, TypeRange{}, callee, + ValueRange{source, target}); + Value targetBits = + rewriter.create(loc, target, cstI64(rewriter, loc, 20)); + Value rs1 = rewriter.create(loc, source, targetBits); + rewriter.replaceOpWithNewOp(op, rs1, + cstI64(rewriter, loc, 4096)); + return success(); + } +}; + +struct BuckyballMvinLowering : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + LogicalResult + matchAndRewrite(MvinOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + MemrefAddress memref = extractMemrefAddress(rewriter, loc, op.getInput()); + Value rs1 = packRs1WriteBankIter(rewriter, loc, adaptor.getAddr(), + adaptor.getDepth()); + Value rs2 = + packRs2MemStride(rewriter, loc, memref.address, adaptor.getStride()); + if (auto group = op.getGroup()) + rs2 = rewriter.create( + loc, rs2, + cstI64(rewriter, loc, (1ULL << 63) | (uint64_t(*group) << 58))); + + rewriter.replaceOpWithNewOp(op, rs1, rs2); + return success(); + } +}; + +struct BuckyballMvin2dLowering : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + LogicalResult + matchAndRewrite(Mvin2dOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Location loc = op.getLoc(); + MemrefAddress memref = extractMemrefAddress(rewriter, loc, op.getInput()); + Value lowBits = rewriter.create(loc, memref.address, + cstI64(rewriter, loc, 7)); + Value aligned = rewriter.create( + loc, arith::CmpIPredicate::eq, lowBits, cstI64(rewriter, loc, 0)); + Value inRange = rewriter.create( + loc, arith::CmpIPredicate::ult, memref.address, + cstI64(rewriter, loc, 1ULL << 39)); + Value valid8 = rewriter.create(loc, arith::CmpIPredicate::eq, + adaptor.getValidBytes(), + cstI64(rewriter, loc, 8)); + Value valid16 = rewriter.create( + loc, arith::CmpIPredicate::eq, adaptor.getValidBytes(), + cstI64(rewriter, loc, 16)); + Value validLength = rewriter.create(loc, valid8, valid16); + Value validAddress = rewriter.create(loc, aligned, inRange); + Value validInput = + rewriter.create(loc, validAddress, validLength); + Value validInput64 = + rewriter.create(loc, rewriter.getI64Type(), validInput); + // This lowering also runs inside single-block SCF regions. + auto tail = LLVM::TailCallKindAttr::get( + rewriter.getContext(), LLVM::tailcallkind::TailCallKind::None); + LLVM::InlineAsmOp::create( + rewriter, loc, Type(), ValueRange{validInput64}, + rewriter.getStringAttr("bnez $0, 1f\n\tunimp\n1:"), + rewriter.getStringAttr("r,~{memory}"), rewriter.getUnitAttr(), + UnitAttr(), tail, UnitAttr(), nullptr, nullptr); + Value rs1 = packRs1WriteBankIter(rewriter, loc, adaptor.getAddr(), + adaptor.getHeight()); + Value address = rewriter.create(loc, memref.address, + cstI64(rewriter, loc, 3)); + Value pixel = rewriter.create(loc, adaptor.getPixelBytes(), + cstI64(rewriter, loc, 0x3ff)); + Value pixelUnits = + rewriter.create(loc, pixel, cstI64(rewriter, loc, 8)); + Value width = rewriter.create(loc, adaptor.getWidth(), + cstI64(rewriter, loc, 1)); + Value valid = + rewriter.create(loc, rewriter.getI64Type(), valid8); + Value rs2 = rewriter.create( + loc, address, + rewriter.create(loc, pixelUnits, + cstI64(rewriter, loc, 36))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getSourceWidth(), + cstI64(rewriter, loc, 43))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, adaptor.getDstBase(), + cstI64(rewriter, loc, 53))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, width, cstI64(rewriter, loc, 59))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, valid, cstI64(rewriter, loc, 62))); + rewriter.replaceOpWithNewOp(op, rs1, rs2, + rewriter.getI32IntegerAttr(34)); + return success(); + } +}; + +struct BuckyballMvinMmioLowering : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + LogicalResult + matchAndRewrite(MvinMmioOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + MemrefAddress memref = extractMemrefAddress(rewriter, loc, op.getInput()); + Value rs1 = rewriter.create(loc, adaptor.getRows(), + cstI64(rewriter, loc, 30)); + Value address = rewriter.create( + loc, memref.address, cstI64(rewriter, loc, (1ULL << 39) - 1)); + Value mmio = rewriter.create( + loc, adaptor.getMmioAddr(), cstI64(rewriter, loc, (1ULL << 17) - 1)); + Value columns = rewriter.create(loc, adaptor.getColumns(), + cstI64(rewriter, loc, 0xff)); + Value rs2 = rewriter.create( + loc, address, + rewriter.create(loc, mmio, cstI64(rewriter, loc, 39))); + rs2 = rewriter.create( + loc, rs2, + rewriter.create(loc, columns, + cstI64(rewriter, loc, 56))); + rewriter.replaceOpWithNewOp(op, rs1, rs2); + return success(); + } +}; + +struct BuckyballMvoutLowering : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + LogicalResult + matchAndRewrite(MvoutOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + MemrefAddress memref = extractMemrefAddress(rewriter, loc, op.getOutput()); + if (op.getGroup()) { + auto fnTy = LLVM::LLVMFunctionType::get( + LLVM::LLVMVoidType::get(rewriter.getContext()), + {LLVM::LLVMPointerType::get(rewriter.getContext()), + rewriter.getI64Type(), rewriter.getI64Type()}); + auto callee = getOrInsertExtFunc(rewriter, parentModule(rewriter), + "dma_touch_mvout_group", fnTy); + LLVM::CallOp::create( + rewriter, loc, TypeRange{}, callee, + ValueRange{memref.hostPtr, adaptor.getDepth(), adaptor.getStride()}); + } else { + emitBbDmaTouchMvout(rewriter, loc, memref.hostPtr, adaptor.getDepth(), + adaptor.getStride(), adaptor.getAddr()); + } + Value rs1 = + packRs1BankIter(rewriter, loc, adaptor.getAddr(), adaptor.getDepth()); + Value rs2 = + packRs2MemStride(rewriter, loc, memref.address, adaptor.getStride()); + if (auto group = op.getGroup()) + rs2 = rewriter.create( + loc, rs2, + cstI64(rewriter, loc, (1ULL << 63) | (uint64_t(*group) << 58))); + + rewriter.replaceOpWithNewOp(op, rs1, rs2); + return success(); + } +}; + +struct BuckyballMvoverLowering : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + LogicalResult + matchAndRewrite(MvoverOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + Location loc = op.getLoc(); + auto bounded = [&](Value value, uint64_t maximum) { + Value valid = + rewriter.create(loc, arith::CmpIPredicate::ule, value, + cstI64(rewriter, loc, maximum)); + rewriter.create(loc, valid, + "mvover operand exceeds ISA range"); + }; + bounded(adaptor.getSourceThread(), 255); + bounded(adaptor.getTargetThread(), 255); + bounded(adaptor.getSourceBank(), 1023); + bounded(adaptor.getTargetBank(), 1023); + bounded(adaptor.getSourceRow(), 65535); + bounded(adaptor.getTargetRow(), 65535); + Value count = rewriter.create(loc, adaptor.getRows(), + cstI64(rewriter, loc, 1)); + bounded(count, 65535); + auto shifted = [&](Value value, uint64_t shift) -> Value { + return rewriter.create(loc, value, + cstI64(rewriter, loc, shift)); + }; + Value rs1 = rewriter.create( + loc, adaptor.getSourceThread(), shifted(adaptor.getTargetThread(), 8)); + rs1 = rewriter.create(loc, rs1, + shifted(adaptor.getSourceBank(), 16)); + rs1 = rewriter.create(loc, rs1, + shifted(adaptor.getTargetBank(), 26)); + Value rs2 = rewriter.create( + loc, adaptor.getSourceRow(), shifted(adaptor.getTargetRow(), 16)); + rs2 = rewriter.create(loc, rs2, shifted(count, 32)); + rewriter.replaceOpWithNewOp(op, rs1, rs2, + rewriter.getI32IntegerAttr(13)); + return success(); + } +}; + +static bool contiguousRankOne(Type type) { + auto memref = dyn_cast(type); + if (!memref || memref.getRank() != 1) + return false; + SmallVector strides; + int64_t offset; + return succeeded(memref.getStridesAndOffset(strides, offset)) && + strides[0] == 1; +} + +struct ContiguousCopyLowering : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + LogicalResult + matchAndRewrite(memref::CopyOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + if (!contiguousRankOne(op.getSource().getType()) || + !contiguousRankOne(op.getTarget().getType())) + return failure(); + Location loc = op.getLoc(); + auto sourceType = cast(op.getSource().getType()); + auto targetType = cast(op.getTarget().getType()); + Value zero = + LLVM::ConstantOp::create(rewriter, loc, getIndexType(), + rewriter.getIntegerAttr(getIndexType(), 0)); + Value source = getStridedElementPtr(rewriter, loc, sourceType, + adaptor.getSource(), ValueRange{zero}); + Value target = getStridedElementPtr(rewriter, loc, targetType, + adaptor.getTarget(), ValueRange{zero}); + Value size = MemRefDescriptor(adaptor.getSource()).size(rewriter, loc, 0); + Value bytes = LLVM::MulOp::create( + rewriter, loc, size, + getSizeInBytes(loc, sourceType.getElementType(), rewriter)); + LLVM::MemcpyOp::create(rewriter, loc, target, source, bytes, false); + rewriter.eraseOp(op); + return success(); + } +}; + +struct BuckyballInstLowering : public ConvertOpToLLVMPattern { + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + LogicalResult + matchAndRewrite(InstOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + rewriter.replaceOpWithNewOp( + op, adaptor.getRs1(), adaptor.getRs2(), + rewriter.getI32IntegerAttr(op.getFunct7())); + return success(); + } +}; + +} // namespace + +void populateBaseLegalizeForLLVMExportPatterns( + LLVMTypeConverter &converter, RewritePatternSet &patterns, + bool includeFuncOperandForwarding) { + if (includeFuncOperandForwarding) { + patterns.add, + ForwardOperands, + ForwardOperands>(converter, + &converter.getContext()); + } + patterns.add(converter); + patterns.add(converter); + patterns.add(converter); + patterns.add(converter); + patterns.add(converter); + patterns.add(converter); + patterns.add(converter); + patterns.add(converter); +} + +void configureBaseLegalizeForExportTarget(LLVMConversionTarget &target) { + target.addLegalOp(); + target.addIllegalOp(); + target.addLegalDialect(); + target.addDynamicallyLegalOp([](memref::CopyOp op) { + return !contiguousRankOne(op.getSource().getType()) || + !contiguousRankOne(op.getTarget().getType()); + }); + target.addLegalDialect(); + target.addLegalDialect(); +} + +} // namespace legalize +} // namespace buckyball +} // namespace buddy diff --git a/waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h b/waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h new file mode 100644 index 000000000..d38bbea26 --- /dev/null +++ b/waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h @@ -0,0 +1,60 @@ +//===- LegalizeForLLVMExportBase.h - Buckyball base LLVM lowering -*- C++ +//-*-===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// + +#ifndef BUDDY_BUCKYBALL_LEGALIZE_FOR_LLVM_EXPORT_BASE_H +#define BUDDY_BUCKYBALL_LEGALIZE_FOR_LLVM_EXPORT_BASE_H + +#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" +#include "mlir/Conversion/LLVMCommon/TypeConverter.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/PatternMatch.h" + +#include + +namespace buddy { +namespace buckyball { +namespace legalize { + +uint64_t fieldBits(uint64_t val, int startBit, int endBit); +int64_t addrBitsForDepth(int64_t bankDepth); + +mlir::Value cstI64(mlir::OpBuilder &b, mlir::Location loc, uint64_t v); +mlir::Value extractPtr(mlir::OpBuilder &b, mlir::Location loc, + mlir::Value memref); +mlir::Value packRs1BanksIter(mlir::OpBuilder &b, mlir::Location loc, + mlir::Value rBank0, mlir::Value rBank1, + mlir::Value wBank, mlir::Value iter); +mlir::Value packRs1BankIter(mlir::OpBuilder &b, mlir::Location loc, + mlir::Value bankId, mlir::Value depth); +mlir::Value packRs1WriteBankIter(mlir::OpBuilder &b, mlir::Location loc, + mlir::Value bankId, mlir::Value depth); +mlir::Value packRs2MemStride(mlir::OpBuilder &b, mlir::Location loc, + mlir::Value memAddr, mlir::Value stride); +void emitMset(mlir::OpBuilder &b, mlir::Location loc, uint64_t bankId, + uint64_t row, uint64_t col, uint64_t alloc); + +void populateBaseLegalizeForLLVMExportPatterns( + mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns, + bool includeFuncOperandForwarding); + +void configureBaseLegalizeForExportTarget(mlir::LLVMConversionTarget &target); + +} // namespace legalize +} // namespace buckyball +} // namespace buddy + +#endif // BUDDY_BUCKYBALL_LEGALIZE_FOR_LLVM_EXPORT_BASE_H diff --git a/waste/stack/compiler/src/Plugin.cpp b/waste/stack/compiler/src/Plugin.cpp new file mode 100644 index 000000000..07559dd7b --- /dev/null +++ b/waste/stack/compiler/src/Plugin.cpp @@ -0,0 +1,30 @@ +//===- Plugin.cpp - The Buckyball compiler as a buddy-opt plugin ----------===// +// +// buddy-opt --load-dialect-plugin= --load-pass-plugin= loads the +// Buckyball dialect and the framework passes. Ball passes register themselves +// when the plugin library is loaded. +// +//===----------------------------------------------------------------------===// + +#include "Conversion/LowerBuckyball/LowerBuckyball.h" + +#include "mlir/Tools/Plugins/DialectPlugin.h" +#include "mlir/Tools/Plugins/PassPlugin.h" +#include "llvm/Config/llvm-config.h" + +#include "Buckyball/BuckyballDialect.h" + +using namespace mlir; + +extern "C" LLVM_ATTRIBUTE_WEAK DialectPluginLibraryInfo +mlirGetDialectPluginInfo() { + return {MLIR_PLUGIN_API_VERSION, "Buckyball", LLVM_VERSION_STRING, + [](DialectRegistry *registry) { + registry->insert<::buddy::buckyball::BuckyballDialect>(); + }}; +} + +extern "C" LLVM_ATTRIBUTE_WEAK PassPluginLibraryInfo mlirGetPassPluginInfo() { + return {MLIR_PLUGIN_API_VERSION, "Buckyball", LLVM_VERSION_STRING, + [] { mlir::buddy::registerBuckyballPasses(); }}; +} diff --git a/waste/stack/compiler/src/Target/BuckyballTargetRegistry.cpp b/waste/stack/compiler/src/Target/BuckyballTargetRegistry.cpp new file mode 100644 index 000000000..b15489471 --- /dev/null +++ b/waste/stack/compiler/src/Target/BuckyballTargetRegistry.cpp @@ -0,0 +1,74 @@ +#include "Target/BuckyballTargetRegistry.h" + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" + +using namespace llvm; + +#include "BuckyballTargetRegistry.inc" + +const buckyball_target::BuckyballTargetConfig & +buckyball_target::getBuckyballTarget() { + return kBuckyballTarget; +} + +static cl::list selectedBalls( + "buckyball-ball", cl::CommaSeparated, + cl::desc( + "Select Ball instance names when an instruction has multiple owners")); + +static const buckyball_target::BuckyballIsaEntry & +resolveInstruction(StringRef mnemonic) { + for (const std::string &name : selectedBalls) { + if (!llvm::any_of(kBuckyballTarget.ballMappings, + [&](const auto &ball) { return ball.name == name; })) + report_fatal_error(Twine("Unknown Ball instance: ") + name); + } + const buckyball_target::BuckyballIsaEntry *unique = nullptr; + const buckyball_target::BuckyballIsaEntry *selected = nullptr; + unsigned count = 0; + for (const auto &entry : kBuckyballTarget.isa) { + if (entry.mnemonic != mnemonic) + continue; + unique = &entry; + ++count; + auto name = kBuckyballTarget.ballMappings[entry.ballIndex].name; + if (llvm::is_contained(selectedBalls, name)) { + if (selected) + report_fatal_error( + Twine("Multiple selected Ball instances implement ") + mnemonic); + selected = &entry; + } + } + if (selected) + return *selected; + if (count == 1) + return *unique; + if (count > 1) + report_fatal_error(Twine("Ambiguous instruction ") + mnemonic + + "; select its instance with --buckyball-ball="); + report_fatal_error(Twine("Buckyball core ") + kBuckyballTarget.core + + " does not define ISA mnemonic " + mnemonic); +} + +int32_t buckyball_target::getBuckyballFunct7(llvm::StringRef mnemonic) { + return resolveInstruction(mnemonic).funct7; +} + +const buckyball_target::BuckyballBallMapping & +buckyball_target::getBuckyballBallMapping(llvm::StringRef mnemonic) { + return kBuckyballTarget.ballMappings[resolveInstruction(mnemonic).ballIndex]; +} + +int64_t buckyball_target::getBuckyballBallParam(llvm::StringRef mnemonic, + llvm::StringRef param) { + for (const BuckyballBallParam &entry : + getBuckyballBallMapping(mnemonic).params) { + if (entry.name == param) + return entry.value; + } + report_fatal_error(Twine("Buckyball instruction ") + mnemonic + + " has no parameter " + param); +} diff --git a/waste/stack/compiler/src/Target/BuckyballTargetRegistry.h b/waste/stack/compiler/src/Target/BuckyballTargetRegistry.h new file mode 100644 index 000000000..946df84c5 --- /dev/null +++ b/waste/stack/compiler/src/Target/BuckyballTargetRegistry.h @@ -0,0 +1,47 @@ +#ifndef BUCKYBALL_TARGET_REGISTRY_H +#define BUCKYBALL_TARGET_REGISTRY_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" + +#include + +namespace buckyball_target { + +struct BuckyballIsaEntry { + llvm::StringRef mnemonic; + int32_t funct7; + unsigned ballIndex; +}; + +struct BuckyballBallParam { + llvm::StringRef name; + int64_t value; +}; + +struct BuckyballBallMapping { + llvm::StringRef name; + int64_t inBW; + int64_t outBW; + llvm::ArrayRef params; +}; + +struct BuckyballTargetConfig { + llvm::StringRef core; + int64_t bankNum; + int64_t bankWidthBits; + int64_t bankDepth; + int64_t privateBankMax; + bool rvvEnabled; + llvm::ArrayRef ballMappings; + llvm::ArrayRef isa; +}; + +const BuckyballTargetConfig &getBuckyballTarget(); +int32_t getBuckyballFunct7(llvm::StringRef mnemonic); +const BuckyballBallMapping &getBuckyballBallMapping(llvm::StringRef mnemonic); +int64_t getBuckyballBallParam(llvm::StringRef mnemonic, llvm::StringRef param); + +} // namespace buckyball_target + +#endif // BUCKYBALL_TARGET_REGISTRY_H diff --git a/waste/stack/compiler/src/Utils/BankUtils.h b/waste/stack/compiler/src/Utils/BankUtils.h new file mode 100644 index 000000000..54000fa61 --- /dev/null +++ b/waste/stack/compiler/src/Utils/BankUtils.h @@ -0,0 +1,131 @@ +//===- BankUtils.h - Utilities for Bank operations -----------------------===// +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//===----------------------------------------------------------------------===// +// +// Utility functions for generating Bank-SSA operations. +// +//===----------------------------------------------------------------------===// + +#ifndef BUCKYBALL_CONVERSION_BANKUTILS_H +#define BUCKYBALL_CONVERSION_BANKUTILS_H + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/Value.h" + +#include "Buckyball/BuckyballOps.h" + +#include +#include +#include + +namespace buddy { +namespace buckyball { + +/// Create i64 constant. +static inline mlir::Value createI64Const(mlir::OpBuilder &b, mlir::Location loc, + int64_t val) { + return b.create(loc, b.getI64Type(), + b.getI64IntegerAttr(val)); +} + +/// Create i64 constant from uint64_t. +static inline mlir::Value createI64ConstU(mlir::OpBuilder &b, + mlir::Location loc, uint64_t val) { + return b.create(loc, b.getI64Type(), + b.getI64IntegerAttr(val)); +} + +static inline mlir::Value createI1Const(mlir::OpBuilder &b, mlir::Location loc, + bool val) { + return b.create(loc, b.getI1Type(), + b.getBoolAttr(val)); +} + +/// Pack bit field: extract bits [startBit, endBit] from val and shift to +/// position. +static inline uint64_t packBits(uint64_t val, int startBit, int endBit) { + uint64_t width = endBit - startBit + 1; + uint64_t mask = (1ULL << width) - 1; + if (val > mask) { + std::fprintf(stderr, "packBits: value %llu does not fit in [%d:%d]\n", + (unsigned long long)val, startBit, endBit); + std::abort(); + } + return val << startBit; +} + +static inline uint64_t matrixRs2(uint64_t rows, uint64_t cols, uint64_t k) { + if (rows == 0 || cols == 0 || cols > 0xfff || k == 0 || rows > 0xfff || + k > 0xfff || (rows != 1 && rows % 16) || cols % 16 || k % 16) { + std::fprintf( + stderr, + "matrix rs2: rows/cols/k must fit in 12 bits (got %llu %llu %llu)\n", + (unsigned long long)rows, (unsigned long long)cols, + (unsigned long long)k); + std::abort(); + } + return packBits(rows, 0, 11) | packBits(cols, 12, 23) | packBits(k, 24, 35); +} + +/// Allocate a bank with given row/col dimensions. +static inline mlir::Value allocBank(mlir::OpBuilder &b, mlir::Location loc, + int64_t row, int64_t col) { + auto i64Type = b.getI64Type(); + return b.create(loc, i64Type, b.getI64IntegerAttr(row), + b.getI64IntegerAttr(col)); +} + +/// Release a bank. +static inline void releaseBank(mlir::OpBuilder &b, mlir::Location loc, + mlir::Value bank) { + b.create(loc, bank); +} + +/// Move data from memref into bank. +static inline mlir::Value mvinBank(mlir::OpBuilder &b, mlir::Location loc, + mlir::Value memref, mlir::Value bank, + int64_t depth, int64_t stride = 1) { + mlir::Value depthVal = createI64Const(b, loc, depth); + mlir::Value strideVal = createI64Const(b, loc, stride); + return b.create(loc, bank.getType(), memref, bank, depthVal, + strideVal, mlir::IntegerAttr{}); +} + +/// Move data from bank to memref. +static inline mlir::Value mvoutBank(mlir::OpBuilder &b, mlir::Location loc, + mlir::Value memref, mlir::Value bank, + int64_t depth, int64_t stride = 1) { + mlir::Value depthVal = createI64Const(b, loc, depth); + mlir::Value strideVal = createI64Const(b, loc, stride); + return b.create(loc, bank.getType(), memref, bank, depthVal, + strideVal, mlir::IntegerAttr{}); +} + +/// Create mset operation for bank allocation/release. +static inline MsetOp createMset(mlir::OpBuilder &b, mlir::Location loc, + uint64_t bankId, bool alloc, uint64_t row, + uint64_t col) { + auto op = b.create(loc, createI64ConstU(b, loc, bankId)); + op->setAttr("alloc", b.getBoolAttr(alloc)); + op->setAttr("row", b.getI64IntegerAttr(row)); + op->setAttr("col", b.getI64IntegerAttr(col)); + return op; +} + +} // namespace buckyball +} // namespace buddy + +#endif // BUCKYBALL_CONVERSION_BANKUTILS_H diff --git a/waste/stack/compiler/src/Utils/StageTrace.cpp b/waste/stack/compiler/src/Utils/StageTrace.cpp new file mode 100644 index 000000000..b51cf86a6 --- /dev/null +++ b/waste/stack/compiler/src/Utils/StageTrace.cpp @@ -0,0 +1,66 @@ +#include "Utils/StageTrace.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/IR/BuiltinOps.h" +#include "mlir/IR/Diagnostics.h" + +using namespace mlir; + +LogicalResult mlir::buddy::emitStageTensorTrace(OpBuilder &builder, + Location loc, Value value, + DictionaryAttr metadata) { + auto path = cast(metadata.get("id_path")); + if (path.size() > 4) + return emitError(loc, "stage trace id_path exceeds runtime depth limit"); + auto type = cast(value.getType()); + auto element = type.getElementType(); + auto argumentType = MemRefType::get({ShapedType::kDynamic}, element); + constexpr StringLiteral name = "buckyballTraceStageI8Path"; + Operation *parent = builder.getInsertionBlock()->getParentOp(); + auto module = parent->getParentOfType(); + if (!module.lookupSymbol(name)) { + OpBuilder::InsertionGuard guard(builder); + builder.setInsertionPointToStart(module.getBody()); + Type i64 = builder.getI64Type(); + auto function = func::FuncOp::create( + builder, loc, name, + builder.getFunctionType({i64, i64, i64, i64, i64, i64, argumentType}, + {})); + function.setPrivate(); + function->setAttr("llvm.emit_c_interface", builder.getUnitAttr()); + } + Value packed; + if (!type.getLayout().isIdentity()) { + packed = memref::AllocOp::create(builder, loc, + MemRefType::get(type.getShape(), element)); + memref::CopyOp::create(builder, loc, value, packed); + value = packed; + } + int64_t size = type.getNumElements(); + if (type.getRank() != 1) { + SmallVector groups(1); + for (int64_t axis = 0; axis < type.getRank(); ++axis) + groups[0].push_back(axis); + value = memref::CollapseShapeOp::create( + builder, loc, MemRefType::get({size}, element), value, groups); + } + value = memref::CastOp::create(builder, loc, argumentType, value); + SmallVector arguments; + arguments.push_back(arith::ConstantIntOp::create( + builder, loc, cast(metadata.get("id")).getInt(), 64)); + arguments.push_back( + arith::ConstantIntOp::create(builder, loc, path.size(), 64)); + for (size_t index = 0; index < 4; ++index) { + int64_t component = + index < path.size() ? cast(path[index]).getInt() : -1; + arguments.push_back( + arith::ConstantIntOp::create(builder, loc, component, 64)); + } + arguments.push_back(value); + func::CallOp::create(builder, loc, name, TypeRange{}, arguments); + if (packed) + memref::DeallocOp::create(builder, loc, packed); + return success(); +} diff --git a/waste/stack/compiler/src/Utils/StageTrace.h b/waste/stack/compiler/src/Utils/StageTrace.h new file mode 100644 index 000000000..3b4b3a8c5 --- /dev/null +++ b/waste/stack/compiler/src/Utils/StageTrace.h @@ -0,0 +1,11 @@ +#ifndef BUCKYBALL_STAGE_TRACE_H +#define BUCKYBALL_STAGE_TRACE_H + +#include "mlir/IR/Builders.h" + +namespace mlir::buddy { +LogicalResult emitStageTensorTrace(OpBuilder &builder, Location loc, + Value value, DictionaryAttr metadata); +} + +#endif From 08ef1c6f831891fa69084443dc261dbdf53b8a02 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 26 Oct 2026 16:55:50 +0800 Subject: [PATCH 189/196] feat: sync to buckyball 0.2.1-a --- bbdev/api/steps/workload/00_clean_api.step.py | 2 +- .../api/steps/workload/00_clean_event.step.py | 4 +- .../device/core/acc/mlir_tests/CMakeLists.txt | 25 -- .../core/acc/mlir_tests/core/CMakeLists.txt | 394 +++++++++++++++++- .../core/acc/mlir_tests/linalg/CMakeLists.txt | 200 ++++++++- .../regression/bemu/workloads-bare.toml | 8 +- .../device/regression/p2e/workloads-bare.toml | 3 - .../regression/verilator/workloads-bare.toml | 3 - .../regression/verilator/workloads-linux.toml | 3 - .../designs/toy/stack/midend/CMakeLists.txt | 2 + .../midend/src/ConvertLinalgToBuckyball.cpp | 212 ++++++++++ .../core/acc/mlir_tests/linalg/CMakeLists.txt | 70 +++- stack | 2 +- 13 files changed, 833 insertions(+), 95 deletions(-) create mode 100644 examples/designs/toy/stack/midend/src/ConvertLinalgToBuckyball.cpp diff --git a/bbdev/api/steps/workload/00_clean_api.step.py b/bbdev/api/steps/workload/00_clean_api.step.py index 47617cb97..15c65f66a 100644 --- a/bbdev/api/steps/workload/00_clean_api.step.py +++ b/bbdev/api/steps/workload/00_clean_api.step.py @@ -23,7 +23,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: except ValueError as error: return ApiResponse(status=400, body={"error": str(error)}) - unknown = sorted(k for k in body if k not in {"design"}) + unknown = sorted(k for k in body if k not in {"design", "device", "core"}) if unknown: return ApiResponse( status=400, diff --git a/bbdev/api/steps/workload/00_clean_event.step.py b/bbdev/api/steps/workload/00_clean_event.step.py index 0c26105f1..bffa594e0 100644 --- a/bbdev/api/steps/workload/00_clean_event.step.py +++ b/bbdev/api/steps/workload/00_clean_event.step.py @@ -50,7 +50,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return bbdir = get_buckyball_path() - output_path = design_output_root(bbdir, design, selection=input_data["target"]) + output_path = os.path.join( + design_output_root(bbdir, design, selection=input_data["target"]), "workloads" + ) build_path = workload_build_dir(bbdir, design, selection=input_data["target"]) for path in (output_path, build_path): diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/CMakeLists.txt index a0cb74653..d00d54687 100644 --- a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/CMakeLists.txt +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/CMakeLists.txt @@ -1,30 +1,5 @@ add_custom_target(pebble-mlir-tests-build) -set(PEBBLE_CUSTOM_PASSES - -allow-unregistered-dialect - -form-custom-ops - -convert-linalg-to-buckyball - -lower-frontend - -eliminate-empty-tensors - -one-shot-bufferize=bufferize-function-boundaries - -ownership-based-buffer-deallocation - -buffer-deallocation-simplification - -bufferization-lower-deallocations - -convert-trace-to-llvm - -assign-physical-banks - -lower-buckyball - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - -convert-math-to-llvm - -convert-math-to-libm - -convert-arith-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) - add_subdirectory(bank) add_subdirectory(core) add_subdirectory(linalg) diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/CMakeLists.txt index 6ca7097fc..86c740952 100644 --- a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/CMakeLists.txt +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/CMakeLists.txt @@ -1,25 +1,218 @@ set(BUCKYBALL_MLIR_GROUP_TARGET pebble-mlir-tests-build) set(BUCKYBALL_MLIR_TEST_PREFIX core) -add_buckyball_mlir_test(lookup_channel_i8 TARGET pebble PASSES ${PEBBLE_CUSTOM_PASSES}) -add_buckyball_mlir_test(linear_chain_i8 TARGET pebble PASSES ${PEBBLE_CUSTOM_PASSES}) -add_buckyball_mlir_test(linear_int8 TARGET pebble PASSES ${PEBBLE_CUSTOM_PASSES}) -add_buckyball_mlir_test(generic_matmul_i8 TARGET pebble PASSES ${PEBBLE_CUSTOM_PASSES}) +add_buckyball_mlir_test(lookup_channel_i8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) +add_buckyball_mlir_test(linear_chain_i8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) +add_buckyball_mlir_test(linear_int8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) +add_buckyball_mlir_test(generic_matmul_i8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) add_buckyball_mlir_test(mega_matmul_m17_k65_n17 - TARGET pebble PASSES ${PEBBLE_CUSTOM_PASSES}) + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) add_buckyball_mlir_test(mega_conv2d TARGET pebble - PASSES ${PEBBLE_CUSTOM_PASSES}) + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) add_buckyball_mlir_test(mega_conv2d_k5_tail TARGET pebble - PASSES ${PEBBLE_CUSTOM_PASSES}) + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) add_buckyball_mlir_test(mega_conv2d_s2_depthwise_s2 TARGET pebble - PASSES ${PEBBLE_CUSTOM_PASSES}) + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv_exact_input.i8 PEBBLE_GAP_CONV_INPUT_HEX HEX) @@ -33,13 +226,82 @@ file(READ ${CMAKE_CURRENT_SOURCE_DIR}/data/mega_global_avg_conv2_exact_weight3.i PEBBLE_GAP_CONV_WEIGHT3_HEX HEX) add_buckyball_mlir_test(mega_global_avg_conv_exact TARGET pebble - PASSES ${PEBBLE_CUSTOM_PASSES}) + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) add_buckyball_mlir_test(mega_global_avg_7x7_c288_exact TARGET pebble - PASSES ${PEBBLE_CUSTOM_PASSES}) + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) add_buckyball_mlir_test(mega_global_avg_conv2_exact TARGET pebble - PASSES ${PEBBLE_CUSTOM_PASSES}) + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) unset(PEBBLE_GAP_CONV_INPUT_HEX) unset(PEBBLE_GAP_C288_INPUT_HEX) unset(PEBBLE_GAP_CONV_WEIGHT_HEX) @@ -48,16 +310,108 @@ unset(PEBBLE_GAP_CONV_WEIGHT3_HEX) add_buckyball_mlir_test(mega_se_gate_conv TARGET pebble - PASSES ${PEBBLE_CUSTOM_PASSES}) + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) add_buckyball_mlir_test(mega_matmul TARGET pebble - PASSES ${PEBBLE_CUSTOM_PASSES}) + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(core_matmul_16x16_i8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) -foreach(test - core_matmul_16x16_i8 - core_transpose_16x16_i8) - add_buckyball_mlir_test(${test} - TARGET pebble - PASSES ${PEBBLE_CUSTOM_PASSES}) -endforeach() +add_buckyball_mlir_test(core_transpose_16x16_i8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt index fa20eb6a1..fde4acf2e 100644 --- a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt @@ -1,15 +1,191 @@ set(BUCKYBALL_MLIR_GROUP_TARGET pebble-mlir-tests-build) set(BUCKYBALL_MLIR_TEST_PREFIX linalg) -foreach(test - linalg_relu_513x17_i32 - linalg_matmul_16x16_i8 - linalg_transpose_16x16_i8 - linalg_transpose_detect_1x84x100_f32 - linalg_transpose_nchw_nhwc_1x3x8x8_f32 - linalg_transpose_oihw_ohwi_4x8x3x3_f32 - linalg_mega_conv2d_k3_c16) - add_buckyball_mlir_test(${test} - TARGET pebble - PASSES ${PEBBLE_CUSTOM_PASSES}) -endforeach() +add_buckyball_mlir_test(linalg_relu_513x17_i32 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(linalg_matmul_16x16_i8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(linalg_transpose_16x16_i8 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(linalg_transpose_detect_1x84x100_f32 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(linalg_transpose_nchw_nhwc_1x3x8x8_f32 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(linalg_transpose_oihw_ohwi_4x8x3x3_f32 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(linalg_mega_conv2d_k3_c16 + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) diff --git a/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml index 285d0e7a2..1359cb046 100644 --- a/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml +++ b/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml @@ -7,8 +7,6 @@ tests = [ "toy-toy-mlirtest-ball_redmule_f16_chain-baremetal", "toy-toy-mlirtest-bank_redmule_f16_rows-baremetal", - "toy-toy-mlirtest-base_dma_group-baremetal", - "toy-toy-mlirtest-base_mset_transfer-baremetal", "toy-toy-ctest-mvin_mvout_group_test-baremetal", "toy-toy-ctest-mset_transfer_test-baremetal", "toy-toy-ctest-rvv_ball-baremetal", @@ -16,9 +14,6 @@ tests = [ "toy-toy-ctest-mvin_2d_pixel_stride_test-baremetal", "toy-toy-ctest-gemmini_accumulated_16x16_test-baremetal", "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-baremetal", - "toy-toy-mlirtest-bank_gemmini_conv_tail-baremetal", - "toy-toy-mlirtest-bank_gemmini_conv_direct_weight-baremetal", - "toy-toy-mlirtest-bank_gemmini_conv_round-baremetal", "toy-toy-ctest-mvin_2d_test-baremetal", "toy-toy-ctest-mvin_mvout_acc_test-baremetal", "toy-toy-ctest-mvin_mvout_test-baremetal", @@ -53,4 +48,7 @@ tests = [ "toy-toy-ctest-tlb_sv39_nonidentity_test-baremetal", "toy-toy-ctest-tlb_sv39_4k_alias_test-baremetal", "toy-toy-ctest-tlb_sfence_remap_test-baremetal", + "toy-toy-mlirtest-linalg_conv2d_nhwc_hwcf-baremetal", + "toy-toy-mlirtest-linalg_conv2d_nhwc_fhwc-baremetal", + "toy-toy-mlirtest-linalg_conv2d_nchw_fchw-baremetal", ] diff --git a/examples/designs/toy/sims/device/regression/p2e/workloads-bare.toml b/examples/designs/toy/sims/device/regression/p2e/workloads-bare.toml index 7f6c30521..6265d1cda 100644 --- a/examples/designs/toy/sims/device/regression/p2e/workloads-bare.toml +++ b/examples/designs/toy/sims/device/regression/p2e/workloads-bare.toml @@ -2,9 +2,6 @@ tests = [ "toy-toy-ctest-gemmini_accumulated_16x16_test-baremetal.hex", "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-baremetal.hex", - "toy-toy-mlirtest-bank_gemmini_conv_tail-baremetal.hex", - "toy-toy-mlirtest-bank_gemmini_conv_direct_weight-baremetal.hex", - "toy-toy-mlirtest-bank_gemmini_conv_round-baremetal.hex", "toy-toy-ctest-mvin_mvout_test-baremetal.hex", "toy-toy-ctest-mvin_mvout_stride_test-baremetal.hex", "toy-toy-ctest-bdb_counter_start_stop_test-baremetal.hex", diff --git a/examples/designs/toy/sims/device/regression/verilator/workloads-bare.toml b/examples/designs/toy/sims/device/regression/verilator/workloads-bare.toml index 0073d1782..6f18573a9 100644 --- a/examples/designs/toy/sims/device/regression/verilator/workloads-bare.toml +++ b/examples/designs/toy/sims/device/regression/verilator/workloads-bare.toml @@ -10,9 +10,6 @@ tests = [ "toy-toy-ctest-mvin_2d_pixel_stride_test-baremetal", "toy-toy-ctest-gemmini_accumulated_16x16_test-baremetal", "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-baremetal", - "toy-toy-mlirtest-bank_gemmini_conv_tail-baremetal", - "toy-toy-mlirtest-bank_gemmini_conv_direct_weight-baremetal", - "toy-toy-mlirtest-bank_gemmini_conv_round-baremetal", "toy-toy-ctest-mvin_2d_test-baremetal", "toy-toy-ctest-mvin_mvout_acc_test-baremetal", "toy-toy-ctest-mvin_mvout_test-baremetal", diff --git a/examples/designs/toy/sims/device/regression/verilator/workloads-linux.toml b/examples/designs/toy/sims/device/regression/verilator/workloads-linux.toml index 88c2741bc..aa36e4333 100644 --- a/examples/designs/toy/sims/device/regression/verilator/workloads-linux.toml +++ b/examples/designs/toy/sims/device/regression/verilator/workloads-linux.toml @@ -2,9 +2,6 @@ tests = [ "toy-toy-ctest-gemmini_accumulated_16x16_test-linux", "toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-linux", - "toy-toy-mlirtest-bank_gemmini_conv_tail-linux", - "toy-toy-mlirtest-bank_gemmini_conv_direct_weight-linux", - "toy-toy-mlirtest-bank_gemmini_conv_round-linux", "toy-toy-ctest-mvin_2d_test-linux", "toy-toy-ctest-mvin_mvout_acc_test-linux", "toy-toy-ctest-mvin_mvout_test-linux", diff --git a/examples/designs/toy/stack/midend/CMakeLists.txt b/examples/designs/toy/stack/midend/CMakeLists.txt index 7d445ba9a..d0775a0cb 100644 --- a/examples/designs/toy/stack/midend/CMakeLists.txt +++ b/examples/designs/toy/stack/midend/CMakeLists.txt @@ -4,6 +4,8 @@ set(TARGET ${DESIGN_CORE_TARGET_${CORE}}) set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${TARGET}}) set(BUCKYBALL_TARGET_TABLE "${DESIGN_ISA_DIR}/${TARGET}/target.inc") +# The core owns computation recognition and its conversion to bank dataflow. +file(GLOB_RECURSE BUCKYBALL_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/src/*.cpp") add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) add_library(${TARGET}-compiler MODULE $) diff --git a/examples/designs/toy/stack/midend/src/ConvertLinalgToBuckyball.cpp b/examples/designs/toy/stack/midend/src/ConvertLinalgToBuckyball.cpp new file mode 100644 index 000000000..cd9807ab6 --- /dev/null +++ b/examples/designs/toy/stack/midend/src/ConvertLinalgToBuckyball.cpp @@ -0,0 +1,212 @@ +//===- ConvertLinalgToBuckyball.cpp - Toy core lowering -------------------===// +// +// A 2-D convolution becomes a matrix product C[M x N] = A[M x K] @ B[K x N] +// with M the batch and output pixels, N the output channels and K the filter +// window and input channels. The CPU packs A (im2col) and B into 16x16 INT8 +// tiles, rounding the FP32 values to nearest even and saturating; Gemmini +// computes each C tile with matmul_i8_tile, accumulating over K; the CPU adds +// the INT32 tiles to the FP32 output. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballDialect.h" +#include "Buckyball/BuckyballOps.h" + +#include "mlir/Dialect/Affine/IR/AffineOps.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/Math/IR/Math.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Pass/Pass.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +namespace { +// Runs body(i, j) for i in [0, rows) and j in [0, cols). +void loops(OpBuilder &b, Location loc, int64_t rows, int64_t cols, + function_ref body) { + OpBuilder::InsertionGuard guard(b); + Value zero = arith::ConstantIndexOp::create(b, loc, 0); + Value one = arith::ConstantIndexOp::create(b, loc, 1); + auto outer = scf::ForOp::create( + b, loc, zero, arith::ConstantIndexOp::create(b, loc, rows), one); + b.setInsertionPointToStart(outer.getBody()); + auto inner = scf::ForOp::create( + b, loc, zero, arith::ConstantIndexOp::create(b, loc, cols), one); + b.setInsertionPointToStart(inner.getBody()); + body(outer.getInductionVar(), inner.getInductionVar()); +} + +LogicalResult lowerConvolution(linalg::LinalgOp op) { + FailureOr conv = + linalg::inferConvolutionDims(op); + Value input = op.getDpsInputs()[0], filter = op.getDpsInputs()[1]; + Value output = op.getDpsInits()[0]; + if (failed(conv) || !conv->depth.empty() || !op.hasPureBufferSemantics() || + !getElementTypeOrSelf(input.getType()).isF32() || + !getElementTypeOrSelf(filter.getType()).isF32() || + !getElementTypeOrSelf(output.getType()).isF32()) + return op->emitOpError("toy lowers FP32 2-D convolutions on memrefs"); + SmallVector ranges = op.getStaticLoopRanges(); + SmallVector mDims(conv->batch.begin(), conv->batch.end()); + SmallVector kDims(conv->filterLoop.begin(), conv->filterLoop.end()); + llvm::append_range(mDims, conv->outputImage); + llvm::append_range(kDims, conv->inputChannel); + ArrayRef nDims = conv->outputChannel; + auto extent = [&](ArrayRef dims) { + int64_t size = 1; + for (unsigned dim : dims) + size *= ranges[dim]; + return size; + }; + int64_t m = extent(mDims), n = extent(nDims), k = extent(kDims); + int64_t mt = (m + 15) / 16, nt = (n + 15) / 16, kt = (k + 15) / 16; + + OpBuilder b(op); + Location loc = op.getLoc(); + Type i8 = b.getI8Type(), i32 = b.getI32Type(), i64 = b.getI64Type(); + auto idx = [&](int64_t v) -> Value { + return arith::ConstantIndexOp::create(b, loc, v); + }; + auto f32 = [&](float v) -> Value { + return arith::ConstantOp::create(b, loc, b.getF32FloatAttr(v)); + }; + // The loop dimension values of flat indices along each dimension group. + auto iterators = [&](ArrayRef>> flats) { + SmallVector iv(ranges.size(), idx(0)); + for (auto [flat, dims] : flats) + for (unsigned dim : llvm::reverse(dims)) { + iv[dim] = arith::RemUIOp::create(b, loc, flat, idx(ranges[dim])); + flat = arith::DivUIOp::create(b, loc, flat, idx(ranges[dim])); + } + return iv; + }; + auto element = [&](OpOperand &operand, ArrayRef iv) { + AffineMap map = op.getMatchingIndexingMap(&operand); + SmallVector indices; + for (AffineExpr result : map.getResults()) + indices.push_back(affine::AffineApplyOp::create( + b, loc, AffineMap::get(map.getNumDims(), 0, result), iv)); + return indices; + }; + auto quantize = [&](Value v) -> Value { + v = arith::MaximumFOp::create(b, loc, v, f32(-128)); + v = arith::MinimumFOp::create(b, loc, v, f32(127)); + v = math::RoundEvenOp::create(b, loc, v); + return arith::FPToSIOp::create(b, loc, i8, v); + }; + // Tile (row / 16) * stride + col / 16, element (row % 16, col % 16). + auto tileIndex = [&](Value row, Value col, int64_t stride) { + Value sixteen = idx(16); + Value tile = arith::AddIOp::create( + b, loc, + arith::MulIOp::create( + b, loc, arith::DivUIOp::create(b, loc, row, sixteen), idx(stride)), + arith::DivUIOp::create(b, loc, col, sixteen)); + return SmallVector{tile, + arith::RemUIOp::create(b, loc, row, sixteen), + arith::RemUIOp::create(b, loc, col, sixteen)}; + }; + + Value aTiles = memref::AllocOp::create( + b, loc, MemRefType::get({mt * kt, 16, 16}, i8), b.getI64IntegerAttr(64)); + Value bTiles = memref::AllocOp::create( + b, loc, MemRefType::get({kt * nt, 16, 16}, i8), b.getI64IntegerAttr(64)); + Value cTiles = memref::AllocOp::create( + b, loc, MemRefType::get({mt * nt, 16, 16}, i32), b.getI64IntegerAttr(64)); + Value zero8 = arith::ConstantIntOp::create(b, loc, 0, 8); + linalg::FillOp::create(b, loc, zero8, aTiles); + linalg::FillOp::create(b, loc, zero8, bTiles); + loops(b, loc, m, k, [&](Value row, Value col) { + SmallVector iv = iterators({{row, mDims}, {col, kDims}}); + Value v = + memref::LoadOp::create(b, loc, input, element(op->getOpOperand(0), iv)); + memref::StoreOp::create(b, loc, quantize(v), aTiles, + tileIndex(row, col, kt)); + }); + loops(b, loc, k, n, [&](Value row, Value col) { + SmallVector iv = iterators({{row, kDims}, {col, nDims}}); + Value v = memref::LoadOp::create(b, loc, filter, + element(op->getOpOperand(1), iv)); + memref::StoreOp::create(b, loc, quantize(v), bTiles, + tileIndex(row, col, nt)); + }); + + auto tile = [&](Value tiles, int64_t index) -> Value { + return memref::SubViewOp::create( + b, loc, tiles, ArrayRef{index, 0, 0}, + ArrayRef{1, 16, 16}, ArrayRef{1, 1, 1}); + }; + Value base = arith::ConstantIntOp::create(b, loc, 0, 64); + Value aBank = BankAllocOp::create(b, loc, i64, 1, 1); + Value bBank = BankAllocOp::create(b, loc, i64, 1, 1); + for (int64_t i = 0; i < mt; ++i) + for (int64_t j = 0; j < nt; ++j) { + Value cBank = BankAllocOp::create(b, loc, i64, 1, 4); + for (int64_t r = 0; r < kt; ++r) { + aBank = BankMvinOp::create(b, loc, i64, tile(aTiles, i * kt + r), aBank, + 16, 1); + bBank = BankMvinOp::create(b, loc, i64, tile(bTiles, r * nt + j), bBank, + 16, 1); + auto matmul = BankCustomOp::create( + b, loc, i64, ValueRange{aBank, bBank}, cBank, + ValueRange{base, base, base}, "matmul_i8_tile", r > 0); + if (r > 0) + matmul->setAttr("shift", b.getI64IntegerAttr(0)); + cBank = matmul.getDestinationOut(); + } + cBank = BankMvoutOp::create(b, loc, i64, tile(cTiles, i * nt + j), cBank, + 16, 1); + BankReleaseOp::create(b, loc, cBank); + } + BankReleaseOp::create(b, loc, aBank); + BankReleaseOp::create(b, loc, bBank); + + loops(b, loc, m, n, [&](Value row, Value col) { + SmallVector iv = iterators({{row, mDims}, {col, nDims}}); + SmallVector at = element(op->getOpOperand(2), iv); + Value sum = memref::LoadOp::create(b, loc, cTiles, tileIndex(row, col, nt)); + Value result = arith::AddFOp::create( + b, loc, memref::LoadOp::create(b, loc, output, at), + arith::SIToFPOp::create(b, loc, b.getF32Type(), sum)); + memref::StoreOp::create(b, loc, result, output, at); + }); + for (Value buffer : {aTiles, bTiles, cTiles}) + memref::DeallocOp::create(b, loc, buffer); + op->erase(); + return success(); +} + +class ConvertLinalgToBuckyballPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ConvertLinalgToBuckyballPass) + StringRef getArgument() const final { return "convert-linalg-to-buckyball"; } + StringRef getDescription() const final { + return "Lower 2-D convolutions to the toy core bank dataflow"; + } + void getDependentDialects(DialectRegistry ®istry) const override { + registry + .insert(); + } + void runOnOperation() override { + SmallVector convolutions; + getOperation().walk([&](linalg::LinalgOp op) { + if (isa(op)) + convolutions.push_back(op); + }); + for (linalg::LinalgOp op : convolutions) + if (failed(lowerConvolution(op))) + return signalPassFailure(); + } +}; + +PassRegistration registration; +} // namespace diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt index f193aaaf9..5ee75df0d 100644 --- a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt @@ -3,25 +3,53 @@ set(BUCKYBALL_MLIR_TEST_PREFIX linalg) add_custom_target(linalg-mlir-tests-build) -set(TOY_LINALG_FUNCTIONAL_PASSES - -convert-linalg-to-buckyball - -lower-buckyball-to-bank-ssa - -assign-physical-banks - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-linalg-to-loops - -expand-strided-metadata - -lower-affine - -convert-scf-to-cf - -convert-cf-to-llvm - ${BUCKYBALL_LOWER_BUCKYBALL} - -convert-arith-to-llvm - -convert-math-to-llvm - -finalize-memref-to-llvm - -convert-func-to-llvm - -reconcile-unrealized-casts) +add_buckyball_mlir_test(conv2d_nhwc_hwcf + TARGET toy + PASSES + -convert-linalg-to-buckyball + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-arith-to-llvm + -convert-math-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) -foreach(test conv2d_nhwc_hwcf conv2d_nhwc_fhwc conv2d_nchw_fchw) - add_buckyball_mlir_test(${test} - TARGET toy - PASSES ${TOY_LINALG_FUNCTIONAL_PASSES}) -endforeach() +add_buckyball_mlir_test(conv2d_nhwc_fhwc + TARGET toy + PASSES + -convert-linalg-to-buckyball + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-arith-to-llvm + -convert-math-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(conv2d_nchw_fchw + TARGET toy + PASSES + -convert-linalg-to-buckyball + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-arith-to-llvm + -convert-math-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) diff --git a/stack b/stack index 256e889e1..40446484a 160000 --- a/stack +++ b/stack @@ -1 +1 @@ -Subproject commit 256e889e1b75d23201c41f7505c656c1a957ab8b +Subproject commit 40446484abe13b033387575e46ac7ba418b89388 From 49421e3d68bf0576dc11f6f6793e7e6e84daaa00 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 26 Oct 2026 17:32:43 +0800 Subject: [PATCH 190/196] feat: clean up to buckyball 0.2.1-a --- .gitignore | 1 + bbdev/api/steps/kernel/01_build_event.step.py | 4 +- .../regression/bemu/workloads-linux.toml | 2 +- .../regression/p2e/workloads-linux.toml | 2 +- .../device/workloads/kernel/CMakeLists.txt | 5 - stack | 2 +- .../src/MLIRTest/base/base_dma_group.mlir | 52 - .../src/MLIRTest/base/base_dma_group_main.cpp | 46 - .../src/MLIRTest/base/base_mset_transfer.mlir | 49 - .../MLIRTest/base/base_mset_transfer_main.cpp | 37 - .../MLIRTest/compiler/bank_kernel_calls.mlir | 17 - .../compiler/bank_kernel_invalid.mlir | 46 - .../compiler/bank_transfer_cfg_invalid.mlir | 40 - .../compiler/bank_transfer_invalid.mlir | 64 - .../compiler/bank_transfer_ownership.mlir | 65 - .../compiler/contiguous_scatter_rows.mlir | 98 - .../MLIRTest/compiler/dma_group_invalid.mlir | 14 - .../MLIRTest/compiler/dma_group_lowering.mlir | 28 - .../MLIRTest/compiler/fp32_packmm_mixed.mlir | 28 - .../compiler/mset_transfer_invalid.mlir | 23 - .../compiler/mset_transfer_lowering.mlir | 23 - .../compiler/norm_quant_window_mixed.mlir | 51 - .../src/MLIRTest/compiler/rvv_rms_suffix.mlir | 94 - .../src/MLIRTest/compiler/rvv_tanh.mlir | 61 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 16 - .../stack/src/Dialect/Buckyball/F32AddBall.td | 30 - .../stack/src/Dialect/Buckyball/IR/F32Add.cpp | 30 - .../Transforms/LegalizeForLLVMExport.cpp | 46 - .../balls/f32add/stack/src/Passes.cpp | 39 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 76 - .../LowerBuckyball/LowerToBankSSA.cpp | 307 -- .../Int8Contractions.cpp | 132 - .../src/Dialect/Buckyball/GemminiBall.td | 73 - .../src/Dialect/Buckyball/IR/GemminiConv.cpp | 37 - .../Transforms/LegalizeForLLVMExport.cpp | 46 - .../balls/gemmini/stack/src/Passes.cpp | 56 - .../include/IntrinsicsRISCVIm2colBall.td | 4 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 23 - .../stack/src/Dialect/Buckyball/Im2colBall.td | 133 - .../Transforms/LegalizeForLLVMExport.cpp | 234 -- .../balls/im2col/stack/src/Passes.cpp | 39 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 21 - .../stack/src/Dialect/Buckyball/Int2FpBall.td | 40 - .../Transforms/LegalizeForLLVMExport.cpp | 46 - .../balls/int2fp/stack/src/Passes.cpp | 39 - .../include/IntrinsicsRISCVInt8AddBall.td | 1 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 19 - .../src/Dialect/Buckyball/Int8AddBall.td | 26 - .../Transforms/LegalizeForLLVMExport.cpp | 56 - .../balls/int8add/stack/src/Passes.cpp | 39 - .../include/IntrinsicsRISCVInt8MulBall.td | 1 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 19 - .../src/Dialect/Buckyball/Int8MulBall.td | 26 - .../Transforms/LegalizeForLLVMExport.cpp | 52 - .../balls/int8mul/stack/src/Passes.cpp | 39 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 35 - .../stack/src/Dialect/Buckyball/LutBall.td | 22 - .../Transforms/LegalizeForLLVMExport.cpp | 60 - .../components/balls/lut/stack/src/Passes.cpp | 39 - .../include/IntrinsicsRISCVMatAddBall.td | 1 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 18 - .../stack/src/Dialect/Buckyball/MatAddBall.td | 21 - .../Transforms/LegalizeForLLVMExport.cpp | 44 - .../balls/matadd/stack/src/Passes.cpp | 39 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 22 - .../src/Dialect/Buckyball/MaxPoolBall.td | 45 - .../Transforms/LegalizeForLLVMExport.cpp | 113 - .../balls/maxpool/stack/src/Passes.cpp | 39 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 18 - .../src/Dialect/Buckyball/Mxfp2IntBall.td | 26 - .../Transforms/LegalizeForLLVMExport.cpp | 50 - .../balls/mxfp2int/stack/src/Passes.cpp | 39 - .../balls/mxmm/stack/python/fp32.py | 66 - .../mxmm/stack/python/grouped_attention.py | 107 - .../balls/mxmm/stack/python/layout.py | 77 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 39 - .../LowerBuckyball/LowerToBankSSA.cpp | 672 ---- .../stack/src/Dialect/Buckyball/IR/FP32.cpp | 46 - .../stack/src/Dialect/Buckyball/IR/MXFP8.cpp | 157 - .../stack/src/Dialect/Buckyball/MxmmBall.td | 145 - .../Transforms/LegalizeForLLVMExport.cpp | 157 - .../Buckyball/Transforms/WindowPatterns.inc | 75 - .../balls/mxmm/stack/src/Passes.cpp | 41 - .../stack/src/Transforms/NormQuantWindow.cpp | 346 -- .../LowerBuckyball/AssignPhysicalBanks.cpp | 16 - .../src/Dialect/Buckyball/MxquantBall.td | 21 - .../Transforms/LegalizeForLLVMExport.cpp | 56 - .../balls/mxquant/stack/src/Passes.cpp | 39 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 17 - .../src/Dialect/Buckyball/IR/Redmule.cpp | 40 - .../src/Dialect/Buckyball/RedmuleBall.td | 23 - .../Transforms/LegalizeForLLVMExport.cpp | 44 - .../balls/redmule/stack/src/Passes.cpp | 39 - .../include/IntrinsicsRISCVReluBall.td | 4 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 18 - .../LowerBuckyball/LowerToBankSSA.cpp | 102 - .../LowerLinalgToBuckyball/ReluPatterns.cpp | 118 - .../stack/src/Dialect/Buckyball/ReluBall.td | 31 - .../Transforms/LegalizeForLLVMExport.cpp | 50 - .../balls/relu/stack/src/Passes.cpp | 55 - .../include/IntrinsicsRISCVSMatMulBall.td | 7 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 31 - .../LowerBuckyballToBankSSAPatterns.cpp | 356 -- .../MegaKernelToBankSSAPatterns.cpp | 480 --- .../LowerLinalgToBuckyball/MatMulPatterns.cpp | 416 --- .../src/Dialect/Buckyball/SMatMulBall.td | 71 - .../Transforms/LegalizeForLLVMExport.cpp | 263 -- .../balls/smatmul/stack/src/Passes.cpp | 86 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 33 - .../LowerBuckyball/LowerToBankSSA.cpp | 156 - .../QuantizePatterns.cpp | 64 - .../stack/src/Dialect/Buckyball/ToInt8Ball.td | 69 - .../Transforms/LegalizeForLLVMExport.cpp | 122 - .../balls/toint8/stack/src/Passes.cpp | 57 - .../stack/src/Dialect/Buckyball/TraceBall.td | 18 - .../Transforms/LegalizeForLLVMExport.cpp | 55 - .../balls/trace/stack/src/Passes.cpp | 39 - .../include/IntrinsicsRISCVTransposeBall.td | 4 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 20 - .../LowerBuckyball/LowerToBankSSA.cpp | 34 - .../TransposePatterns.cpp | 263 -- .../Transforms/LegalizeForLLVMExport.cpp | 115 - .../src/Dialect/Buckyball/TransposeBall.td | 52 - .../balls/transpose/stack/src/Passes.cpp | 59 - .../Backend/include/IntrinsicsRISCVVecBall.td | 4 - .../LowerBuckyball/AssignPhysicalBanks.cpp | 20 - .../Transforms/LegalizeForLLVMExport.cpp | 47 - .../stack/src/Dialect/Buckyball/VecBall.td | 39 - .../balls/vector/stack/src/Passes.cpp | 39 - .../stack/midend/src/FormMegaKernels.cpp | 33 - .../designs/pebble/stack/midend/ops.py | 110 - .../MegaConv2dToBankSSAPatterns.cpp | 468 --- .../MemTransposeToBankSSAPatterns.cpp | 282 -- .../ResidentConvRegionToBankSSAPatterns.cpp | 3250 ----------------- .../stack/midend/src/LargeConvolution.cpp | 158 - .../pebble/stack/midend/src/Passes.cpp | 140 - waste/stack/compiler/frontend/__init__.py | 0 .../compiler/frontend/layout/__init__.py | 0 .../compiler/frontend/layout/provenance.py | 62 - waste/stack/compiler/frontend/operation.py | 107 - waste/stack/compiler/frontend/ops/__init__.py | 0 waste/stack/compiler/frontend/ops/expf.py | 83 - waste/stack/compiler/frontend/ops/int8.py | 250 -- waste/stack/compiler/frontend/ops/linalg.py | 1599 -------- .../compiler/frontend/ops/native_mxfp8.py | 187 - .../stack/compiler/frontend/ops/reductions.py | 308 -- waste/stack/compiler/frontend/ops/tosa.py | 2271 ------------ .../stack/compiler/frontend/quant/__init__.py | 0 waste/stack/compiler/frontend/quant/bwq.py | 163 - .../stack/compiler/frontend/quant/importer.py | 1187 ------ .../compiler/frontend/quant/mxfp8_cache.py | 70 - .../compiler/frontend/quant/native_mxfp8.py | 396 -- .../frontend/quant/offline/__init__.py | 0 .../quant/offline/activations/__init__.py | 6 - .../quant/offline/activations/calibrate.py | 89 - .../quant/offline/weights/__init__.py | 6 - .../quant/offline/weights/quantize.py | 85 - .../offline/weights/strategy/__init__.py | 0 .../offline/weights/strategy/fp32toint8.py | 33 - waste/stack/compiler/frontend/quant/rax.py | 303 -- .../frontend/quant/transforms/__init__.py | 0 .../frontend/quant/transforms/activation.py | 177 - .../frontend/quant/transforms/normalize.py | 20 - .../frontend/quant/transforms/symmetric.py | 28 - waste/stack/compiler/frontend/trace.py | 12 - waste/stack/compiler/frontend/transforms.py | 194 - .../compiler/src/Conversion/BallPasses.h | 131 - .../AssignPhysicalBanksPass.cpp | 97 - .../BaseAssignPhysicalBankPatterns.cpp | 212 -- .../LowerBuckyball/LowerBuckyball.h | 70 - .../LowerBuckyball/LowerBuckyballPass.cpp | 109 - .../LowerBuckyballToBankSSAPass.cpp | 80 - .../LowerBuckyball/PhysicalBankState.cpp | 238 -- .../FormMegaKernelRegions.cpp | 161 - .../LowerLinalgToBuckyball.h | 45 - .../LowerLinalgToBuckyballPass.cpp | 124 - .../MegaKernelPatterns.cpp | 346 -- .../src/Dialect/Buckyball/Buckyball.td | 446 --- .../src/Dialect/Buckyball/BuckyballDialect.h | 25 - .../Dialect/Buckyball/BuckyballInterfaces.td | 41 - .../src/Dialect/Buckyball/BuckyballOps.h | 38 - .../Dialect/Buckyball/IR/BuckyballDialect.cpp | 79 - .../Transforms/LegalizeForLLVMExportBase.cpp | 551 --- .../Transforms/LegalizeForLLVMExportBase.h | 60 - waste/stack/compiler/src/Plugin.cpp | 30 - .../src/Target/BuckyballTargetRegistry.cpp | 74 - .../src/Target/BuckyballTargetRegistry.h | 47 - waste/stack/compiler/src/Utils/BankUtils.h | 131 - waste/stack/compiler/src/Utils/StageTrace.cpp | 66 - waste/stack/compiler/src/Utils/StageTrace.h | 11 - 190 files changed, 7 insertions(+), 24130 deletions(-) delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/base/base_dma_group.mlir delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/base/base_dma_group_main.cpp delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer.mlir delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer_main.cpp delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_calls.mlir delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_invalid.mlir delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_cfg_invalid.mlir delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_invalid.mlir delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_ownership.mlir delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/compiler/contiguous_scatter_rows.mlir delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/compiler/dma_group_invalid.mlir delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/compiler/dma_group_lowering.mlir delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/compiler/fp32_packmm_mixed.mlir delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_invalid.mlir delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_lowering.mlir delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/compiler/norm_quant_window_mixed.mlir delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/compiler/rvv_rms_suffix.mlir delete mode 100644 waste/bb-tests/workloads/src/MLIRTest/compiler/rvv_tanh.mlir delete mode 100644 waste/examples/components/balls/f32add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/F32AddBall.td delete mode 100644 waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/IR/F32Add.cpp delete mode 100644 waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/f32add/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp delete mode 100644 waste/examples/components/balls/gemmini/stack/src/Conversion/LowerLinalgToBuckyball/Int8Contractions.cpp delete mode 100644 waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/GemminiBall.td delete mode 100644 waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/IR/GemminiConv.cpp delete mode 100644 waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/gemmini/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/im2col/stack/src/Backend/include/IntrinsicsRISCVIm2colBall.td delete mode 100644 waste/examples/components/balls/im2col/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/im2col/stack/src/Dialect/Buckyball/Im2colBall.td delete mode 100644 waste/examples/components/balls/im2col/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/im2col/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/int2fp/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/int2fp/stack/src/Dialect/Buckyball/Int2FpBall.td delete mode 100644 waste/examples/components/balls/int2fp/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/int2fp/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/int8add/stack/src/Backend/include/IntrinsicsRISCVInt8AddBall.td delete mode 100644 waste/examples/components/balls/int8add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/int8add/stack/src/Dialect/Buckyball/Int8AddBall.td delete mode 100644 waste/examples/components/balls/int8add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/int8add/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/int8mul/stack/src/Backend/include/IntrinsicsRISCVInt8MulBall.td delete mode 100644 waste/examples/components/balls/int8mul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/int8mul/stack/src/Dialect/Buckyball/Int8MulBall.td delete mode 100644 waste/examples/components/balls/int8mul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/int8mul/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/lut/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/lut/stack/src/Dialect/Buckyball/LutBall.td delete mode 100644 waste/examples/components/balls/lut/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/lut/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/matadd/stack/src/Backend/include/IntrinsicsRISCVMatAddBall.td delete mode 100644 waste/examples/components/balls/matadd/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/matadd/stack/src/Dialect/Buckyball/MatAddBall.td delete mode 100644 waste/examples/components/balls/matadd/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/matadd/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/maxpool/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/maxpool/stack/src/Dialect/Buckyball/MaxPoolBall.td delete mode 100644 waste/examples/components/balls/maxpool/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/maxpool/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/mxfp2int/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/mxfp2int/stack/src/Dialect/Buckyball/Mxfp2IntBall.td delete mode 100644 waste/examples/components/balls/mxfp2int/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/mxfp2int/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/mxmm/stack/python/fp32.py delete mode 100644 waste/examples/components/balls/mxmm/stack/python/grouped_attention.py delete mode 100644 waste/examples/components/balls/mxmm/stack/python/layout.py delete mode 100644 waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp delete mode 100644 waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/IR/FP32.cpp delete mode 100644 waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/IR/MXFP8.cpp delete mode 100644 waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/MxmmBall.td delete mode 100644 waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/Transforms/WindowPatterns.inc delete mode 100644 waste/examples/components/balls/mxmm/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/mxmm/stack/src/Transforms/NormQuantWindow.cpp delete mode 100644 waste/examples/components/balls/mxquant/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/mxquant/stack/src/Dialect/Buckyball/MxquantBall.td delete mode 100644 waste/examples/components/balls/mxquant/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/mxquant/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/redmule/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/IR/Redmule.cpp delete mode 100644 waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/RedmuleBall.td delete mode 100644 waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/redmule/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/relu/stack/src/Backend/include/IntrinsicsRISCVReluBall.td delete mode 100644 waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp delete mode 100644 waste/examples/components/balls/relu/stack/src/Conversion/LowerLinalgToBuckyball/ReluPatterns.cpp delete mode 100644 waste/examples/components/balls/relu/stack/src/Dialect/Buckyball/ReluBall.td delete mode 100644 waste/examples/components/balls/relu/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/relu/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/smatmul/stack/src/Backend/include/IntrinsicsRISCVSMatMulBall.td delete mode 100644 waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp delete mode 100644 waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp delete mode 100644 waste/examples/components/balls/smatmul/stack/src/Conversion/LowerLinalgToBuckyball/MatMulPatterns.cpp delete mode 100644 waste/examples/components/balls/smatmul/stack/src/Dialect/Buckyball/SMatMulBall.td delete mode 100644 waste/examples/components/balls/smatmul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/smatmul/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp delete mode 100644 waste/examples/components/balls/toint8/stack/src/Conversion/LowerLinalgToBuckyball/QuantizePatterns.cpp delete mode 100644 waste/examples/components/balls/toint8/stack/src/Dialect/Buckyball/ToInt8Ball.td delete mode 100644 waste/examples/components/balls/toint8/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/toint8/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/trace/stack/src/Dialect/Buckyball/TraceBall.td delete mode 100644 waste/examples/components/balls/trace/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/trace/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/transpose/stack/src/Backend/include/IntrinsicsRISCVTransposeBall.td delete mode 100644 waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp delete mode 100644 waste/examples/components/balls/transpose/stack/src/Conversion/LowerLinalgToBuckyball/TransposePatterns.cpp delete mode 100644 waste/examples/components/balls/transpose/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/transpose/stack/src/Dialect/Buckyball/TransposeBall.td delete mode 100644 waste/examples/components/balls/transpose/stack/src/Passes.cpp delete mode 100644 waste/examples/components/balls/vector/stack/src/Backend/include/IntrinsicsRISCVVecBall.td delete mode 100644 waste/examples/components/balls/vector/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp delete mode 100644 waste/examples/components/balls/vector/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp delete mode 100644 waste/examples/components/balls/vector/stack/src/Dialect/Buckyball/VecBall.td delete mode 100644 waste/examples/components/balls/vector/stack/src/Passes.cpp delete mode 100644 waste/examples/designs/goban/stack/midend/src/FormMegaKernels.cpp delete mode 100644 waste/examples/designs/pebble/stack/midend/ops.py delete mode 100644 waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/MegaConv2dToBankSSAPatterns.cpp delete mode 100644 waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/MemTransposeToBankSSAPatterns.cpp delete mode 100644 waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/ResidentConvRegionToBankSSAPatterns.cpp delete mode 100644 waste/examples/designs/pebble/stack/midend/src/LargeConvolution.cpp delete mode 100644 waste/examples/designs/pebble/stack/midend/src/Passes.cpp delete mode 100644 waste/stack/compiler/frontend/__init__.py delete mode 100644 waste/stack/compiler/frontend/layout/__init__.py delete mode 100644 waste/stack/compiler/frontend/layout/provenance.py delete mode 100644 waste/stack/compiler/frontend/operation.py delete mode 100644 waste/stack/compiler/frontend/ops/__init__.py delete mode 100644 waste/stack/compiler/frontend/ops/expf.py delete mode 100644 waste/stack/compiler/frontend/ops/int8.py delete mode 100644 waste/stack/compiler/frontend/ops/linalg.py delete mode 100644 waste/stack/compiler/frontend/ops/native_mxfp8.py delete mode 100644 waste/stack/compiler/frontend/ops/reductions.py delete mode 100644 waste/stack/compiler/frontend/ops/tosa.py delete mode 100644 waste/stack/compiler/frontend/quant/__init__.py delete mode 100644 waste/stack/compiler/frontend/quant/bwq.py delete mode 100644 waste/stack/compiler/frontend/quant/importer.py delete mode 100644 waste/stack/compiler/frontend/quant/mxfp8_cache.py delete mode 100644 waste/stack/compiler/frontend/quant/native_mxfp8.py delete mode 100644 waste/stack/compiler/frontend/quant/offline/__init__.py delete mode 100644 waste/stack/compiler/frontend/quant/offline/activations/__init__.py delete mode 100644 waste/stack/compiler/frontend/quant/offline/activations/calibrate.py delete mode 100644 waste/stack/compiler/frontend/quant/offline/weights/__init__.py delete mode 100644 waste/stack/compiler/frontend/quant/offline/weights/quantize.py delete mode 100644 waste/stack/compiler/frontend/quant/offline/weights/strategy/__init__.py delete mode 100644 waste/stack/compiler/frontend/quant/offline/weights/strategy/fp32toint8.py delete mode 100644 waste/stack/compiler/frontend/quant/rax.py delete mode 100644 waste/stack/compiler/frontend/quant/transforms/__init__.py delete mode 100644 waste/stack/compiler/frontend/quant/transforms/activation.py delete mode 100644 waste/stack/compiler/frontend/quant/transforms/normalize.py delete mode 100644 waste/stack/compiler/frontend/quant/transforms/symmetric.py delete mode 100644 waste/stack/compiler/frontend/trace.py delete mode 100644 waste/stack/compiler/frontend/transforms.py delete mode 100644 waste/stack/compiler/src/Conversion/BallPasses.h delete mode 100644 waste/stack/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp delete mode 100644 waste/stack/compiler/src/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp delete mode 100644 waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyball.h delete mode 100644 waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp delete mode 100644 waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp delete mode 100644 waste/stack/compiler/src/Conversion/LowerBuckyball/PhysicalBankState.cpp delete mode 100644 waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/FormMegaKernelRegions.cpp delete mode 100644 waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h delete mode 100644 waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyballPass.cpp delete mode 100644 waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/MegaKernelPatterns.cpp delete mode 100644 waste/stack/compiler/src/Dialect/Buckyball/Buckyball.td delete mode 100644 waste/stack/compiler/src/Dialect/Buckyball/BuckyballDialect.h delete mode 100644 waste/stack/compiler/src/Dialect/Buckyball/BuckyballInterfaces.td delete mode 100644 waste/stack/compiler/src/Dialect/Buckyball/BuckyballOps.h delete mode 100644 waste/stack/compiler/src/Dialect/Buckyball/IR/BuckyballDialect.cpp delete mode 100644 waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp delete mode 100644 waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h delete mode 100644 waste/stack/compiler/src/Plugin.cpp delete mode 100644 waste/stack/compiler/src/Target/BuckyballTargetRegistry.cpp delete mode 100644 waste/stack/compiler/src/Target/BuckyballTargetRegistry.h delete mode 100644 waste/stack/compiler/src/Utils/BankUtils.h delete mode 100644 waste/stack/compiler/src/Utils/StageTrace.cpp delete mode 100644 waste/stack/compiler/src/Utils/StageTrace.h diff --git a/.gitignore b/.gitignore index e615cab8e..d1e4052de 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,4 @@ skills-lock.json ucli.key cm.log tr_db.log +waste-bake/ diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index 5a0545533..3e6fe1bc1 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -52,6 +52,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: hart_count = len(design_config.device.cores) ddr = design_config.device.ddr guest_memory_mib = (ddr.bytes - ddr.model) // (1024 * 1024) + # A DDR without a model region leaves the base empty: init runs no model. + model_base = f"{ddr.base + ddr.bytes - ddr.model:#x}" if ddr.model else "" interactive = input_data.get("interactive", False) output_dir = os.path.join( bbdir, "examples/designs", design, "output/sims/device/workloads/kernel" @@ -72,7 +74,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"-DBUCKYBALL_GUEST_MEMORY_MIB={guest_memory_mib} " f"-DBUCKYBALL_KERNEL_DESIGN={design} " f"-DBUCKYBALL_KERNEL_INTERACTIVE={interactive_arg} " - f"-DBUCKYBALL_MODEL_BASE={ddr.base + ddr.bytes - ddr.model:#x} " + f"-DBUCKYBALL_MODEL_BASE={model_base} " f"-DBUCKYBALL_KERNEL_WORKLOADS={programs}" ) diff --git a/examples/designs/poly/sims/device/regression/bemu/workloads-linux.toml b/examples/designs/poly/sims/device/regression/bemu/workloads-linux.toml index c6e1afe47..e96de12dc 100644 --- a/examples/designs/poly/sims/device/regression/bemu/workloads-linux.toml +++ b/examples/designs/poly/sims/device/regression/bemu/workloads-linux.toml @@ -1,4 +1,4 @@ [workloads] tests = [ - "fw_payload-h9-poly-linux.elf", + "fw_payload-h9-poly-linux-mem4096M.elf", ] diff --git a/examples/designs/poly/sims/device/regression/p2e/workloads-linux.toml b/examples/designs/poly/sims/device/regression/p2e/workloads-linux.toml index 959714e33..31b629e95 100644 --- a/examples/designs/poly/sims/device/regression/p2e/workloads-linux.toml +++ b/examples/designs/poly/sims/device/regression/p2e/workloads-linux.toml @@ -1,4 +1,4 @@ [workloads] tests = [ - "fw_payload-h3-poly-linux.hex", + "fw_payload-h9-poly-linux-mem4096M-ddr.hex", ] diff --git a/examples/designs/toy/sims/device/workloads/kernel/CMakeLists.txt b/examples/designs/toy/sims/device/workloads/kernel/CMakeLists.txt index 5e7eb8f61..a827f3a1f 100644 --- a/examples/designs/toy/sims/device/workloads/kernel/CMakeLists.txt +++ b/examples/designs/toy/sims/device/workloads/kernel/CMakeLists.txt @@ -6,17 +6,12 @@ set(BUCKYBALL_KERNEL_PROGRAMS 0 toy-toy-ctest-redmule_f16_fused_test-linux 0 toy-toy-mlirtest-ball_redmule_f16_chain-linux 0 toy-toy-mlirtest-bank_redmule_f16_rows-linux - 0 toy-toy-mlirtest-base_dma_group-linux - 0 toy-toy-mlirtest-base_mset_transfer-linux 0 toy-toy-ctest-mvin_mvout_group_test-linux 0 toy-toy-ctest-mset_transfer_test-linux 0 toy-toy-ctest-rvv_ball-linux 0 toy-toy-ctest-rvv_kernel-linux 0 toy-toy-ctest-gemmini_accumulated_16x16_test-linux 0 toy-toy-mlirtest-ball_gemmini_accumulated_command_16x16-linux - 0 toy-toy-mlirtest-bank_gemmini_conv_tail-linux - 0 toy-toy-mlirtest-bank_gemmini_conv_direct_weight-linux - 0 toy-toy-mlirtest-bank_gemmini_conv_round-linux 0 toy-toy-ctest-mvin_2d_test-linux 0 toy-toy-ctest-mvin_mvout_acc_test-linux 0 toy-toy-ctest-mvin_mvout_test-linux diff --git a/stack b/stack index 40446484a..034548ef2 160000 --- a/stack +++ b/stack @@ -1 +1 @@ -Subproject commit 40446484abe13b033387575e46ac7ba418b89388 +Subproject commit 034548ef296e78e94de35b9f246cc4c1fea79f27 diff --git a/waste/bb-tests/workloads/src/MLIRTest/base/base_dma_group.mlir b/waste/bb-tests/workloads/src/MLIRTest/base/base_dma_group.mlir deleted file mode 100644 index adad6caa1..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/base/base_dma_group.mlir +++ /dev/null @@ -1,52 +0,0 @@ -func.func private @check_result(memref<16x48xi8>, memref<16x32xi8>) -func.func @main() -> i8 { - %zero = arith.constant 0 : i8 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c16 = arith.constant 16 : index - %c32 = arith.constant 32 : index - %c48 = arith.constant 48 : index - %n48 = arith.constant 48 : i32 - %n32 = arith.constant 32 : i32 - %n91 = arith.constant 91 : i8 - %n7 = arith.constant 7 : i32 - %bank = arith.constant 9 : i64 - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %stride2 = arith.constant 2 : i64 - %input = memref.alloc() alignment = 64 : memref<16x48xi8> - %patch = memref.alloc() alignment = 64 : memref<16x32xi8> - %output = memref.alloc() alignment = 64 : memref<16x48xi8> - %selected = memref.alloc() alignment = 64 : memref<16x32xi8> - scf.for %i = %c0 to %c16 step %c1 { - %ii = arith.index_cast %i : index to i32 - scf.for %j = %c0 to %c48 step %c1 { - %jj = arith.index_cast %j : index to i32 - %base = arith.muli %ii, %n48 : i32 - %value = arith.addi %base, %jj : i32 - %byte = arith.trunci %value : i32 to i8 - memref.store %byte, %input[%i, %j] : memref<16x48xi8> - } - scf.for %j = %c0 to %c32 step %c1 { - %jj = arith.index_cast %j : index to i32 - %base = arith.muli %ii, %n32 : i32 - %value = arith.addi %base, %jj : i32 - %shifted = arith.addi %value, %n7 : i32 - %byte = arith.trunci %shifted : i32 to i8 - memref.store %byte, %patch[%i, %j] : memref<16x32xi8> - memref.store %n91, %selected[%i, %j] : memref<16x32xi8> - } - } - buckyball.mset %bank : i64 - buckyball.mvin %input %bank %depth %stride : memref<16x48xi8> i64 i64 i64 - buckyball.mvin %patch %bank %depth %stride2 : memref<16x32xi8> i64 i64 i64 - buckyball.mvout %selected %bank %depth %stride2 : memref<16x32xi8> i64 i64 i64 - buckyball.mvout %output %bank %depth %stride : memref<16x48xi8> i64 i64 i64 - buckyball.mset %bank : i64 - func.call @check_result(%output, %selected) : (memref<16x48xi8>, memref<16x32xi8>) -> () - memref.dealloc %input : memref<16x48xi8> - memref.dealloc %patch : memref<16x32xi8> - memref.dealloc %output : memref<16x48xi8> - memref.dealloc %selected : memref<16x32xi8> - return %zero : i8 -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/base/base_dma_group_main.cpp b/waste/bb-tests/workloads/src/MLIRTest/base/base_dma_group_main.cpp deleted file mode 100644 index 19ae7a0e8..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/base/base_dma_group_main.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include -#include -#include - -static void fail(void) { -#ifdef BAREMETAL - volatile uint32_t *sim_exit = (volatile uint32_t *)0x60000000; - *sim_exit = 1; - while (1) { - } -#else - exit(1); -#endif -} - -extern "C" void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, - int64_t size0, int64_t size1, int64_t stride0, - int64_t stride1, int8_t *selectedAllocated, - int8_t *selectedAligned, int64_t selectedOffset, - int64_t selectedSize0, int64_t selectedSize1, - int64_t selectedStride0, int64_t selectedStride1) { - (void)allocated; - (void)selectedAllocated; - if (size0 != 16 || size1 != 48 || stride0 != 48 || stride1 != 1 || - selectedSize0 != 16 || selectedSize1 != 32 || selectedStride0 != 32 || - selectedStride1 != 1) { - fail(); - } - int8_t *out = aligned + offset; - int8_t *selected = selectedAligned + selectedOffset; - for (int i = 0; i < 16; ++i) { - for (int j = 0; j < 48; ++j) { - int expected = j >= 16 && j < 32 ? i * 32 + j - 16 + 7 : i * 48 + j; - if (out[i * stride0 + j] != (int8_t)expected) { - fail(); - } - } - for (int j = 0; j < 32; ++j) { - int expected = j < 16 ? i * 32 + j + 7 : 91; - if (selected[i * selectedStride0 + j] != (int8_t)expected) { - fail(); - } - } - } - printf("PASSED: selected DMA preserves other groups and stride gaps\n"); -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer.mlir b/waste/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer.mlir deleted file mode 100644 index 2ca759221..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer.mlir +++ /dev/null @@ -1,49 +0,0 @@ -// RUN: buddy-opt %s -report-bank-usage="bank_num=4 verbose=true" 2>&1 | FileCheck %s --check-prefix=REPORT -// REPORT: transfer b1 -> b7 groups=1 cur=2/4 -// REPORT: transfer b2 -> b7 groups=1 cur=2/4 -// REPORT: peak=2/4 alloc=2 release=1 leaked=0 - -func.func private @check_result(memref<16x32xi8>) -func.func @main() -> i8 { - %zero = arith.constant 0 : i8 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c16 = arith.constant 16 : index - %c32 = arith.constant 32 : index - %i32_16 = arith.constant 16 : i32 - %i32_127 = arith.constant 127 : i32 - %source0 = arith.constant 1 : i64 - %source1 = arith.constant 2 : i64 - %target = arith.constant 7 : i64 - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %input0 = memref.alloc() alignment = 64 : memref<16x16xi8> - %input1 = memref.alloc() alignment = 64 : memref<16x16xi8> - %output = memref.alloc() alignment = 64 : memref<16x32xi8> - scf.for %i = %c0 to %c16 step %c1 { - scf.for %j = %c0 to %c16 step %c1 { - %ii = arith.index_cast %i : index to i32 - %jj = arith.index_cast %j : index to i32 - %base = arith.muli %ii, %i32_16 : i32 - %value = arith.addi %base, %jj : i32 - %other = arith.subi %i32_127, %value : i32 - %byte = arith.trunci %value : i32 to i8 - %other_byte = arith.trunci %other : i32 to i8 - memref.store %byte, %input0[%i, %j] : memref<16x16xi8> - memref.store %other_byte, %input1[%i, %j] : memref<16x16xi8> - } - } - buckyball.mset %source0 : i64 - buckyball.mset %source1 : i64 - buckyball.mvin %input0 %source0 %depth %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mvin %input1 %source1 %depth %stride : memref<16x16xi8> i64 i64 i64 - buckyball.mset_transfer %source0 %target : i64 i64 - buckyball.mset_transfer %source1 %target : i64 i64 - buckyball.mvout %output %target %depth %stride : memref<16x32xi8> i64 i64 i64 - buckyball.mset %target : i64 - func.call @check_result(%output) : (memref<16x32xi8>) -> () - memref.dealloc %input0 : memref<16x16xi8> - memref.dealloc %input1 : memref<16x16xi8> - memref.dealloc %output : memref<16x32xi8> - return %zero : i8 -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer_main.cpp b/waste/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer_main.cpp deleted file mode 100644 index f277eeca9..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/base/base_mset_transfer_main.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include -#include -#include - -static void fail(void) { -#ifdef BAREMETAL - volatile uint32_t *sim_exit = (volatile uint32_t *)0x60000000; - *sim_exit = 1; - while (1) { - } -#else - exit(1); -#endif -} - -#ifdef __cplusplus -extern "C" -#endif - void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, - int64_t size0, int64_t size1, int64_t stride0, - int64_t stride1) { - (void)allocated; - if (size0 != 16 || size1 != 32 || stride0 != 32 || stride1 != 1) { - fail(); - } - int8_t *out = aligned + offset; - for (int i = 0; i < 16; ++i) { - for (int j = 0; j < 16; ++j) { - int value = i * 16 + j; - if (out[i * stride0 + j] != (int8_t)value || - out[i * stride0 + 16 + j] != (int8_t)(127 - value)) { - fail(); - } - } - } - printf("PASSED: base mset transfer preserves group order\n"); -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_calls.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_calls.mlir deleted file mode 100644 index 55a69ffd3..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_calls.mlir +++ /dev/null @@ -1,17 +0,0 @@ -// RUN: buddy-opt %s -assign-physical-banks | FileCheck %s -// CHECK-LABEL: func.func @group_output -// CHECK: call @mock -// CHECK: buckyball.mvout {{.*}} -// CHECK-COUNT-2: buckyball.mset {{.*}} ) { - %read = buckyball.bank_alloc - %write = buckyball.bank_alloc - %one = arith.constant 1 : i64 - %r, %w = buckyball.bank_kernel @mock %read %write (%one) : i64 - %stored = buckyball.bank_mvout %output %w %one %one : memref<1x4xf32> i64 i64 i64 - buckyball.bank_release %r : i64 - buckyball.bank_release %stored : i64 - return -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_invalid.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_invalid.mlir deleted file mode 100644 index 952596fdc..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_kernel_invalid.mlir +++ /dev/null @@ -1,46 +0,0 @@ -// RUN: buddy-opt %s -assign-physical-banks -split-input-file -verify-diagnostics -module { - func.func private @wrong_type(i64, i64, i32) - func.func @mismatched() { - %read = buckyball.bank_alloc - %write = buckyball.bank_alloc - %one = arith.constant 1 : i64 - // expected-error @+1 {{bank kernel callee type does not match its operands}} - %r, %w = buckyball.bank_kernel @wrong_type %read %write (%one) : i64 - buckyball.bank_release %r : i64 - buckyball.bank_release %w : i64 - return - } -} -// ----- -func.func @consumed_source() { - %source = buckyball.bank_alloc - %write = buckyball.bank_alloc - %target = arith.constant 7 : i64 - %one = arith.constant 1 : i64 - // expected-error @+1 {{source bank handle is used after transfer or outside its block}} - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - %r, %w = buckyball.bank_kernel @mock %source %write (%one) : i64 - buckyball.bank_release %aggregate : i64 - buckyball.bank_release %w : i64 - return -} -// ----- -func.func @unknown_read() { - %read = arith.constant 7 : i64 - %write = buckyball.bank_alloc - %one = arith.constant 1 : i64 - // expected-error @+1 {{bank kernel requires distinct live read/write handles}} - %r, %w = buckyball.bank_kernel @mock %read %write (%one) : i64 - buckyball.bank_release %w : i64 - return -} -// ----- -func.func @aliased_read_write() { - %read = buckyball.bank_alloc - %one = arith.constant 1 : i64 - // expected-error @+1 {{bank kernel requires distinct live read/write handles}} - %r, %w = buckyball.bank_kernel @mock %read %read (%one) : i64 - buckyball.bank_release %w : i64 - return -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_cfg_invalid.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_cfg_invalid.mlir deleted file mode 100644 index d4c223a75..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_cfg_invalid.mlir +++ /dev/null @@ -1,40 +0,0 @@ -// RUN: buddy-opt %s -assign-physical-banks -split-input-file -verify-diagnostics - -func.func @captured_allocation(%condition: i1) { - %source = buckyball.bank_alloc - %target = arith.constant 2 : i64 - scf.if %condition { - // expected-error @+1 {{bank transfer cannot consume a control-flow bank alias}} - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - buckyball.bank_release %aggregate : i64 - } - return -} -// ----- -func.func @if_result_alias(%condition: i1) { - %source = buckyball.bank_alloc - %target = arith.constant 2 : i64 - %alias = scf.if %condition -> i64 { - scf.yield %source : i64 - } else { - scf.yield %source : i64 - } - // expected-error @+1 {{bank transfer cannot consume a control-flow bank alias}} - %aggregate = buckyball.bank_transfer %alias %target : i64 i64 - buckyball.bank_release %aggregate : i64 - return -} -// ----- -func.func @loop_result_alias() { - %source = buckyball.bank_alloc - %target = arith.constant 2 : i64 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %alias = scf.for %i = %zero to %one step %one iter_args(%bank = %source) -> i64 { - scf.yield %bank : i64 - } - // expected-error @+1 {{bank transfer cannot consume a control-flow bank alias}} - %aggregate = buckyball.bank_transfer %alias %target : i64 i64 - buckyball.bank_release %aggregate : i64 - return -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_invalid.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_invalid.mlir deleted file mode 100644 index a1aacf677..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_invalid.mlir +++ /dev/null @@ -1,64 +0,0 @@ -// RUN: buddy-opt %s -assign-physical-banks -split-input-file -verify-diagnostics - -func.func @source_use_after_transfer() { - %source = buckyball.bank_alloc - %target = arith.constant 2 : i64 - // expected-error @+1 {{source bank handle is used after transfer or outside its block}} - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - buckyball.bank_release %source : i64 - buckyball.bank_release %aggregate : i64 - return -} -// ----- -func.func @self() { - %source = buckyball.bank_alloc - // expected-error @+1 {{bank transfer source and target must differ}} - %aggregate = buckyball.bank_transfer %source %source : i64 i64 - buckyball.bank_release %aggregate : i64 - return -} -// ----- -func.func @unknown() { - %source = arith.constant 7 : i64 - %target = arith.constant 2 : i64 - // expected-error @+1 {{transfer consumes an unknown virtual bank handle}} - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - buckyball.bank_release %aggregate : i64 - return -} -// ----- -func.func @keep_other_owner() { - %source = buckyball.bank_alloc - %keep = buckyball.bank_alloc - %target = buckyball.bank_alloc - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - buckyball.bank_release %aggregate : i64 - %all_free = buckyball.bank_alloc - // expected-error @+1 {{unavailable bank resources (request=1x1, used=16/16, private ID max=15)}} - %extra = buckyball.bank_alloc - buckyball.bank_release %extra : i64 - buckyball.bank_release %all_free : i64 - buckyball.bank_release %keep : i64 - return -} -// ----- -func.func @private_overflow() { - %source = buckyball.bank_alloc - %target = arith.constant 16 : i64 - // expected-error @+1 {{bank transfer target exceeds the private virtual ID range}} - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - buckyball.bank_release %aggregate : i64 - return -} -// ----- -func.func @old_source_alias_after_id_reuse() { - %source = buckyball.bank_alloc - %target = arith.constant 2 : i64 - // expected-error @+1 {{source bank handle is used after transfer or outside its block}} - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - %new_source_id = buckyball.bank_alloc - buckyball.bank_release %source : i64 - buckyball.bank_release %new_source_id : i64 - buckyball.bank_release %aggregate : i64 - return -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_ownership.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_ownership.mlir deleted file mode 100644 index 7b9c1ec06..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/compiler/bank_transfer_ownership.mlir +++ /dev/null @@ -1,65 +0,0 @@ -// RUN: buddy-opt %s -assign-physical-banks | FileCheck %s - -// CHECK-LABEL: func.func @release_fragmented -// CHECK: buckyball.mset {{.*}} {{.*}}col = 14 -// CHECK: %[[TARGET:.*]] = arith.constant 2 : i64 -// CHECK: buckyball.mset_transfer {{.*}} %[[TARGET]] -// CHECK: buckyball.mset {{.*}} - %aggregate = buckyball.bank_transfer %source %target : i64 i64 - buckyball.bank_release %aggregate : i64 - %all_free = buckyball.bank_alloc - buckyball.bank_release %all_free : i64 - buckyball.bank_release %keep : i64 - return -} - -// Virtual target 2 initially owns physical group 0. Later allocations own -// physical groups 1 and 2, with virtual IDs 0 and 1, preserving target 2. -// CHECK-LABEL: func.func @virtual_physical_collision -// CHECK: buckyball.mset -// CHECK: %[[START:.*]] = arith.constant 0 : i64 -// CHECK: %[[TARGET2:.*]] = arith.constant 2 : i64 -// CHECK: buckyball.mset_transfer %[[START]] %[[TARGET2]] -// CHECK: %[[FIRST:.*]] = arith.constant 0 : i64 -// CHECK: buckyball.mset %[[FIRST]] -// CHECK: %[[SECOND:.*]] = arith.constant 1 : i64 -// CHECK: buckyball.mset %[[SECOND]] -// CHECK: buckyball.mset {{.*}} -// CHECK: %[[OUTPUT:.*]] = memref.alloc() : memref<1x4x16x128xf32> -// CHECK: scf.for %[[HEAD:.*]] = -// CHECK: scf.for %[[TOKEN:.*]] = -// CHECK: %[[POSITION:.*]] = memref.load -// CHECK-NOT: vector.transfer -// CHECK: %[[SOURCE:.*]] = memref.subview %[[INPUT]][{{.*}}, %[[HEAD]], %[[TOKEN]], {{.*}}] [1, 1, 1, 128] [1, 1, 1, 1] -// CHECK: %[[DESTINATION:.*]] = memref.subview %[[OUTPUT]][{{.*}}, %[[HEAD]], %[[POSITION]], {{.*}}] [1, 1, 1, 128] [1, 1, 1, 1] -// CHECK: memref.copy %[[SOURCE]], %[[DESTINATION]] -// CHECK-NOT: vector.transfer -// CHECK: return %[[OUTPUT]] -func.func @scatter_rows(%input: memref<1x4x8x128xf32>, %positions: memref<8xindex>) -> memref<1x4x16x128xf32> { - %output = memref.alloc() : memref<1x4x16x128xf32> - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %four = arith.constant 4 : index - %eight = arith.constant 8 : index - %padding = arith.constant 0.0 : f32 - scf.for %head = %zero to %four step %one { - scf.for %token = %zero to %eight step %one { - %position = memref.load %positions[%token] : memref<8xindex> - %row = vector.transfer_read %input[%zero, %head, %token, %zero], %padding {in_bounds = [true]} : memref<1x4x8x128xf32>, vector<128xf32> - vector.transfer_write %row, %output[%zero, %head, %position, %zero] {in_bounds = [true]} : vector<128xf32>, memref<1x4x16x128xf32> - } - } - return %output : memref<1x4x16x128xf32> -} - -// CHECK-LABEL: func.func @alias_view -// CHECK-NOT: memref.copy -// CHECK: vector.transfer_read -// CHECK: vector.transfer_write -func.func @alias_view(%row: index) -> memref<4x128xf32> { - %buffer = memref.alloc() : memref<4x128xf32> - %view = memref.cast %buffer : memref<4x128xf32> to memref - %zero = arith.constant 0 : index - %padding = arith.constant 0.0 : f32 - %value = vector.transfer_read %view[%zero, %zero], %padding {in_bounds = [true]} : memref, vector<128xf32> - vector.transfer_write %value, %buffer[%row, %zero] {in_bounds = [true]} : vector<128xf32>, memref<4x128xf32> - return %buffer : memref<4x128xf32> -} - -// CHECK-LABEL: func.func @masked -// CHECK-NOT: memref.copy -// CHECK: vector.transfer_read -// CHECK: vector.transfer_write -func.func @masked(%input: memref<4x128xf32>, %mask: vector<128xi1>) -> memref<4x128xf32> { - %output = memref.alloc() : memref<4x128xf32> - %zero = arith.constant 0 : index - %padding = arith.constant 0.0 : f32 - %value = vector.transfer_read %input[%zero, %zero], %padding, %mask {in_bounds = [true]} : memref<4x128xf32>, vector<128xf32> - vector.transfer_write %value, %output[%zero, %zero], %mask {in_bounds = [true]} : vector<128xf32>, memref<4x128xf32> - return %output : memref<4x128xf32> -} - -// CHECK-LABEL: func.func @strided -// CHECK-NOT: memref.copy -// CHECK: vector.transfer_read -// CHECK: vector.transfer_write -func.func @strided(%input: memref<4x128xf32, strided<[256, 2], offset: ?>>) -> memref<4x128xf32> { - %output = memref.alloc() : memref<4x128xf32> - %zero = arith.constant 0 : index - %padding = arith.constant 0.0 : f32 - %value = vector.transfer_read %input[%zero, %zero], %padding {in_bounds = [true]} : memref<4x128xf32, strided<[256, 2], offset: ?>>, vector<128xf32> - vector.transfer_write %value, %output[%zero, %zero] {in_bounds = [true]} : vector<128xf32>, memref<4x128xf32> - return %output : memref<4x128xf32> -} - -// CHECK-LABEL: func.func @padding -// CHECK-NOT: memref.copy -// CHECK: vector.transfer_read -// CHECK: vector.transfer_write -func.func @padding(%input: memref<4x128xf32>, %offset: index) -> memref<4x128xf32> { - %output = memref.alloc() : memref<4x128xf32> - %zero = arith.constant 0 : index - %padding = arith.constant 0.0 : f32 - %value = vector.transfer_read %input[%zero, %offset], %padding {in_bounds = [false]} : memref<4x128xf32>, vector<128xf32> - vector.transfer_write %value, %output[%zero, %zero] {in_bounds = [true]} : vector<128xf32>, memref<4x128xf32> - return %output : memref<4x128xf32> -} - -// CHECK-LABEL: func.func @intervening_store -// CHECK-NOT: memref.copy -// CHECK: vector.transfer_read -// CHECK: memref.store -// CHECK: vector.transfer_write -func.func @intervening_store(%input: memref<4x128xf32>) -> memref<4x128xf32> { - %output = memref.alloc() : memref<4x128xf32> - %zero = arith.constant 0 : index - %padding = arith.constant 0.0 : f32 - %value = vector.transfer_read %input[%zero, %zero], %padding {in_bounds = [true]} : memref<4x128xf32>, vector<128xf32> - memref.store %padding, %input[%zero, %zero] : memref<4x128xf32> - vector.transfer_write %value, %output[%zero, %zero] {in_bounds = [true]} : vector<128xf32>, memref<4x128xf32> - return %output : memref<4x128xf32> -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/compiler/dma_group_invalid.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/dma_group_invalid.mlir deleted file mode 100644 index 928e4843a..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/compiler/dma_group_invalid.mlir +++ /dev/null @@ -1,14 +0,0 @@ -// RUN: buddy-opt %s -split-input-file -verify-diagnostics -func.func @negative(%input: memref<16x16xi8>, %bank: i64, %depth: i64, %stride: i64) { - // expected-error @+1 {{group must be in [0, 31]}} - buckyball.mvin %input %bank %depth %stride : memref<16x16xi8> i64 i64 i64 - return -} - -// ----- - -func.func @overflow(%output: memref<16x16xi8>, %bank: i64, %depth: i64, %stride: i64) { - // expected-error @+1 {{group must be in [0, 31]}} - buckyball.mvout %output %bank %depth %stride : memref<16x16xi8> i64 i64 i64 - return -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/compiler/dma_group_lowering.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/dma_group_lowering.mlir deleted file mode 100644 index 5c65def9c..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/compiler/dma_group_lowering.mlir +++ /dev/null @@ -1,28 +0,0 @@ -// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @selected -// CHECK-DAG: %[[MODE:.*]] = arith.constant -8935140561191436288 : i64 -// CHECK: arith.ori {{.*}}, %[[MODE]] -// CHECK: llvm.call_intrinsic "llvm.riscv.bb.mvin" -// CHECK: llvm.call @dma_touch_mvout_group -// CHECK: arith.ori {{.*}}, %[[MODE]] -// CHECK: llvm.call_intrinsic "llvm.riscv.bb.mvout" -func.func @selected(%input: memref<16x32xi8>, %output: memref<16x32xi8>, %bank: i64) { - %depth = arith.constant 16 : i64 - %stride = arith.constant 2 : i64 - buckyball.mvin %input %bank %depth %stride : memref<16x32xi8> i64 i64 i64 - buckyball.mvout %output %bank %depth %stride : memref<16x32xi8> i64 i64 i64 - return -} - -// CHECK-LABEL: func.func @whole -// CHECK-NOT: dma_touch_mvout_group -// CHECK: llvm.call_intrinsic "llvm.riscv.bb.mvin" -// CHECK: llvm.call @dma_touch_mvout -// CHECK: llvm.call_intrinsic "llvm.riscv.bb.mvout" -func.func @whole(%input: memref<16x48xi8>, %output: memref<16x48xi8>, %bank: i64) { - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - buckyball.mvin %input %bank %depth %stride : memref<16x48xi8> i64 i64 i64 - buckyball.mvout %output %bank %depth %stride : memref<16x48xi8> i64 i64 i64 - return -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/compiler/fp32_packmm_mixed.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/fp32_packmm_mixed.mlir deleted file mode 100644 index 2f4d4557a..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/compiler/fp32_packmm_mixed.mlir +++ /dev/null @@ -1,28 +0,0 @@ -// RUN: buddy-opt %s -lower-buckyball-to-bank-ssa | FileCheck %s --check-prefix=SSA -// RUN: buddy-opt %s -lower-buckyball-to-bank-ssa -assign-physical-banks | FileCheck %s --check-prefix=BANK -// SSA-LABEL: func.func @packed -// SSA-COUNT-3: buckyball.bank_transfer -// SSA: buckyball.bank_kernel @rvv_packmm -// SSA: scf.for -// SSA: buckyball.bank_mvin {{.*}} -// SSA: buckyball.bank_mvin {{.*}} -// SSA: buckyball.bank_kernel @rvv_packmm -// SSA: buckyball.bank_mvout {{.*}} -// SSA: buckyball.bank_kernel @rvv_packmm -// SSA-COUNT-2: buckyball.bank_release -// SSA: return -// BANK-LABEL: func.func @packed -// BANK-COUNT-3: buckyball.mset_transfer -// BANK: call @rvv_packmm -// BANK: buckyball.mvin {{.*}} -// BANK: buckyball.mvin {{.*}} -// BANK: call @rvv_packmm -// BANK: buckyball.mvout {{.*}} -// BANK: call @rvv_packmm -// BANK-NOT: buckyball.bank_ -// BANK: return -func.func @packed(%lhs: memref<16x8xf32>, %rhs: memref<8x13xf32>, - %output: memref<16x13xf32>) { - buckyball.fp32_mem_matmul %lhs, %rhs, %output : memref<16x8xf32>, memref<8x13xf32>, memref<16x13xf32> - return -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_invalid.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_invalid.mlir deleted file mode 100644 index e880a259a..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_invalid.mlir +++ /dev/null @@ -1,23 +0,0 @@ -// RUN: buddy-opt %s -split-input-file -verify-diagnostics -func.func @negative() { - %source = arith.constant -1 : i64 - %target = arith.constant 17 : i64 - // expected-error @+1 {{bank ID must be in [0, 1023]}} - buckyball.mset_transfer %source %target : i64 i64 - return -} -// ----- -func.func @overflow() { - %source = arith.constant 1024 : i64 - %target = arith.constant 17 : i64 - // expected-error @+1 {{bank ID must be in [0, 1023]}} - buckyball.mset_transfer %source %target : i64 i64 - return -} -// ----- -func.func @self() { - %source = arith.constant 17 : i64 - // expected-error @+1 {{source and target must differ}} - buckyball.mset_transfer %source %source : i64 i64 - return -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_lowering.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_lowering.mlir deleted file mode 100644 index a7340b266..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/compiler/mset_transfer_lowering.mlir +++ /dev/null @@ -1,23 +0,0 @@ -// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @dynamic -// CHECK-SAME: %[[SRC:.*]]: i64, %[[DST:.*]]: i64 -// CHECK: llvm.call @dma_bank_transfer(%[[SRC]], %[[DST]]) : (i64, i64) -> () -// CHECK: arith.shli %[[DST]] -// CHECK: arith.ori %[[SRC]] -// CHECK: llvm.call_intrinsic "llvm.riscv.bb.mset" -func.func @dynamic(%source: i64, %target: i64) { - buckyball.mset_transfer %source %target : i64 i64 - return -} - -// CHECK-LABEL: func.func @constant -// CHECK-DAG: %[[MODE:.*]] = arith.constant 4096 : i64 -// CHECK-DAG: %[[PACKED:.*]] = arith.constant 17825793 : i64 -// CHECK: llvm.call @dma_bank_transfer -// CHECK: llvm.call_intrinsic "llvm.riscv.bb.mset"(%[[PACKED]], %[[MODE]]) -func.func @constant() { - %source = arith.constant 1 : i64 - %target = arith.constant 17 : i64 - buckyball.mset_transfer %source %target : i64 i64 - return -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/compiler/norm_quant_window_mixed.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/norm_quant_window_mixed.mlir deleted file mode 100644 index d4e483283..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/compiler/norm_quant_window_mixed.mlir +++ /dev/null @@ -1,51 +0,0 @@ -// RUN: buddy-opt %s -fuse-norm-quant-window | FileCheck %s --check-prefix=SSA -// RUN: buddy-opt %s -fuse-norm-quant-window -assign-physical-banks | FileCheck %s --check-prefix=BANK - -// SSA-LABEL: func.func @mixed -// SSA-COUNT-3: buckyball.bank_transfer -// SSA: buckyball.bank_kernel @rvv_norm_window -// SSA: scf.for -// SSA: buckyball.bank_mvin {{.*}} -// SSA: buckyball.bank_kernel @rvv_norm_window -// SSA: buckyball.bank_mvin {{.*}} -// SSA: buckyball.bank_kernel @rvv_norm_window -// SSA: buckyball.bank_mvout {{.*}} -// SSA: buckyball.bank_kernel @rvv_norm_window -// SSA-COUNT-2: buckyball.bank_release -// SSA-NOT: memref.copy -// SSA: return -// BANK-LABEL: func.func @mixed -// BANK-COUNT-3: buckyball.mset_transfer -// BANK: call @rvv_norm_window -// BANK: scf.for -// BANK: buckyball.mvin {{.*}} -// BANK: call @rvv_norm_window -// BANK: buckyball.mvin {{.*}} -// BANK: call @rvv_norm_window -// BANK: buckyball.mvout {{.*}} -// BANK: call @rvv_norm_window -// BANK-NOT: buckyball.bank_ -// BANK: return -module { - func.func private @rvv_norm(memref<*xf32>, memref<*xf32>, memref<*xf32>, i32, i32) - func.func private @mxfp8_quant(memref<*xf32>, memref<*xi8>, i64, i64, i64) - func.func @mixed(%input: memref<1x64xf32>, %gamma: memref<64xf32>, - %weights: memref<131072xi8, strided<[1], offset: ?>>) -> memref<1x20xf32> { - %norm = memref.alloc() : memref<1x64xf32> - %packed = memref.alloc() : memref<65536xi8> - %output = memref.alloc() : memref<1x20xf32> - %norm_view = memref.cast %norm : memref<1x64xf32> to memref<*xf32> - %input_view = memref.cast %input : memref<1x64xf32> to memref<*xf32> - %gamma_view = memref.cast %gamma : memref<64xf32> to memref<*xf32> - %packed_view = memref.cast %packed : memref<65536xi8> to memref<*xi8> - %mean = arith.constant 1015021568 : i32 - %epsilon = arith.constant 897988541 : i32 - %one = arith.constant 1 : i64 - %chunk = arith.constant 32 : i64 - %bytes = arith.constant 32768 : i64 - call @rvv_norm(%norm_view, %input_view, %gamma_view, %mean, %epsilon) : (memref<*xf32>, memref<*xf32>, memref<*xf32>, i32, i32) -> () - call @mxfp8_quant(%norm_view, %packed_view, %one, %chunk, %bytes) : (memref<*xf32>, memref<*xi8>, i64, i64, i64) -> () - buckyball.mxfp8_mem_matmul %packed, %weights, %output : memref<65536xi8>, memref<131072xi8, strided<[1], offset: ?>>, memref<1x20xf32> - return %output : memref<1x20xf32> - } -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/compiler/rvv_rms_suffix.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/rvv_rms_suffix.mlir deleted file mode 100644 index 07295d225..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/compiler/rvv_rms_suffix.mlir +++ /dev/null @@ -1,94 +0,0 @@ -// RUN: buddy-opt %s -outline-rvv-kernels -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @weighted_residual -// CHECK: call @rvv_norm( -// CHECK-NOT: math.powf -// CHECK-NOT: linalg.reduce -// CHECK: linalg.generic -// CHECK: arith.addf -// CHECK-NEXT: {{.*}} = arith.mulf -// CHECK-NEXT: linalg.yield -// CHECK-LABEL: func.func @unweighted -// CHECK: call @rvv_norm_no_weight( -// CHECK-NOT: math.powf -// CHECK-NOT: linalg.reduce -// CHECK: return -#identity = affine_map<(d0, d1) -> (d0, d1)> -#row = affine_map<(d0, d1) -> (d0, 0)> -#weight = affine_map<(d0, d1) -> (0, d1)> -module { - func.func @weighted_residual(%x: tensor<2x4xf32>, %w: tensor<4xf32>, %residual: tensor<2x4xf32>) -> tensor<2x4xf32> { - %zero = arith.constant 0.0 : f32 - %mean = arith.constant 0.25 : f32 - %eps = arith.constant 0.000001 : f32 - %power = arith.constant -0.5 : f32 - %two = arith.constant 2 : i32 - %scale = arith.constant 0.5 : f32 - %empty = tensor.empty() : tensor<2x4xf32> - %square = linalg.generic {indexing_maps=[#identity,#identity],iterator_types=["parallel","parallel"]} ins(%x:tensor<2x4xf32>) outs(%empty:tensor<2x4xf32>) { - ^bb0(%v:f32,%o:f32): - %s = math.fpowi %v, %two : f32, i32 - linalg.yield %s : f32 - } -> tensor<2x4xf32> - %rows = tensor.empty() : tensor<2xf32> - %init = linalg.fill ins(%zero:f32) outs(%rows:tensor<2xf32>) -> tensor<2xf32> - %sum = linalg.reduce ins(%square:tensor<2x4xf32>) outs(%init:tensor<2xf32>) dimensions=[1] (%v:f32,%acc:f32) { - %s = arith.addf %v, %acc : f32 - linalg.yield %s : f32 - } - %expanded = tensor.expand_shape %sum [[0,1]] output_shape [2,1] : tensor<2xf32> into tensor<2x1xf32> - %inverseEmpty = tensor.empty() : tensor<2x1xf32> - %inverse = linalg.generic {indexing_maps=[#identity,#identity],iterator_types=["parallel","parallel"]} ins(%expanded:tensor<2x1xf32>) outs(%inverseEmpty:tensor<2x1xf32>) { - ^bb0(%v:f32,%o:f32): - %a = arith.mulf %mean, %v : f32 - %b = arith.addf %a, %eps : f32 - %c = math.powf %b, %power : f32 - linalg.yield %c : f32 - } -> tensor<2x1xf32> - %weights = tensor.expand_shape %w [[0,1]] output_shape [1,4] : tensor<4xf32> into tensor<1x4xf32> - %result = linalg.generic {indexing_maps=[#identity,#row,#weight,#identity,#identity],iterator_types=["parallel","parallel"]} ins(%x,%inverse,%weights,%residual:tensor<2x4xf32>,tensor<2x1xf32>,tensor<1x4xf32>,tensor<2x4xf32>) outs(%empty:tensor<2x4xf32>) { - ^bb0(%v:f32,%inv:f32,%gamma:f32,%res:f32,%o:f32): - %a = arith.mulf %v, %inv : f32 - %b = arith.mulf %a, %gamma : f32 - %c = arith.addf %b, %res : f32 - %d = arith.mulf %c, %scale : f32 - linalg.yield %d : f32 - } -> tensor<2x4xf32> - return %result : tensor<2x4xf32> - } - func.func @unweighted(%x: tensor<2x4xf32>, %w: tensor<4xf32>, %residual: tensor<2x4xf32>) -> tensor<2x4xf32> { - %zero = arith.constant 0.0 : f32 - %mean = arith.constant 0.25 : f32 - %eps = arith.constant 0.000001 : f32 - %power = arith.constant -0.5 : f32 - %two = arith.constant 2 : i32 - %scale = arith.constant 0.5 : f32 - %empty = tensor.empty() : tensor<2x4xf32> - %square = linalg.generic {indexing_maps=[#identity,#identity],iterator_types=["parallel","parallel"]} ins(%x:tensor<2x4xf32>) outs(%empty:tensor<2x4xf32>) { - ^bb0(%v:f32,%o:f32): - %s = math.fpowi %v, %two : f32, i32 - linalg.yield %s : f32 - } -> tensor<2x4xf32> - %rows = tensor.empty() : tensor<2xf32> - %init = linalg.fill ins(%zero:f32) outs(%rows:tensor<2xf32>) -> tensor<2xf32> - %sum = linalg.reduce ins(%square:tensor<2x4xf32>) outs(%init:tensor<2xf32>) dimensions=[1] (%v:f32,%acc:f32) { - %s = arith.addf %v, %acc : f32 - linalg.yield %s : f32 - } - %expanded = tensor.expand_shape %sum [[0,1]] output_shape [2,1] : tensor<2xf32> into tensor<2x1xf32> - %inverseEmpty = tensor.empty() : tensor<2x1xf32> - %inverse = linalg.generic {indexing_maps=[#identity,#identity],iterator_types=["parallel","parallel"]} ins(%expanded:tensor<2x1xf32>) outs(%inverseEmpty:tensor<2x1xf32>) { - ^bb0(%v:f32,%o:f32): - %a = arith.mulf %mean, %v : f32 - %b = arith.addf %a, %eps : f32 - %c = math.powf %b, %power : f32 - linalg.yield %c : f32 - } -> tensor<2x1xf32> - %weights = tensor.expand_shape %w [[0,1]] output_shape [1,4] : tensor<4xf32> into tensor<1x4xf32> - %result = linalg.generic {indexing_maps=[#identity,#row,#weight,#identity,#identity],iterator_types=["parallel","parallel"]} ins(%x,%inverse,%weights,%residual:tensor<2x4xf32>,tensor<2x1xf32>,tensor<1x4xf32>,tensor<2x4xf32>) outs(%empty:tensor<2x4xf32>) { - ^bb0(%v:f32,%inv:f32,%gamma:f32,%res:f32,%o:f32): - %a = arith.mulf %v, %inv : f32 - linalg.yield %a : f32 - } -> tensor<2x4xf32> - return %result : tensor<2x4xf32> - } -} diff --git a/waste/bb-tests/workloads/src/MLIRTest/compiler/rvv_tanh.mlir b/waste/bb-tests/workloads/src/MLIRTest/compiler/rvv_tanh.mlir deleted file mode 100644 index d515a5618..000000000 --- a/waste/bb-tests/workloads/src/MLIRTest/compiler/rvv_tanh.mlir +++ /dev/null @@ -1,61 +0,0 @@ -// RUN: buddy-opt %s -outline-rvv-kernels -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @tanh -// CHECK: call @rvv_tanh -// CHECK-NOT: math.tanh -// CHECK: return -// CHECK-LABEL: func.func @mixed_expression -// CHECK-NOT: call @rvv_tanh -// CHECK: math.tanh -// CHECK: arith.addf -// CHECK: return -// CHECK-LABEL: func.func @sin -// CHECK: call @rvv_sin -// CHECK-NOT: math.sin -// CHECK: return -// CHECK-LABEL: func.func @cos -// CHECK: call @rvv_cos -// CHECK-NOT: math.cos -// CHECK: return -#identity = affine_map<(d0) -> (d0)> -module { - func.func @tanh(%x: tensor<17xf32>) -> tensor<17xf32> { - %empty = tensor.empty() : tensor<17xf32> - %result = linalg.generic {indexing_maps = [#identity, #identity], iterator_types = ["parallel"]} ins(%x : tensor<17xf32>) outs(%empty : tensor<17xf32>) { - ^bb0(%xval: f32, %out: f32): - %t = math.tanh %xval : f32 - linalg.yield %t : f32 - } -> tensor<17xf32> - return %result : tensor<17xf32> - } - func.func @mixed_expression(%x: tensor<17xf32>) -> tensor<17xf32> { - %empty = tensor.empty() : tensor<17xf32> - %one = arith.constant 1.0 : f32 - %result = linalg.generic {indexing_maps = [#identity, #identity], iterator_types = ["parallel"]} ins(%x : tensor<17xf32>) outs(%empty : tensor<17xf32>) { - ^bb0(%xval: f32, %out: f32): - %t = math.tanh %xval : f32 - %a = arith.addf %t, %one : f32 - linalg.yield %a : f32 - } -> tensor<17xf32> - return %result : tensor<17xf32> - } - func.func @sin(%x: tensor<17xf32>) -> tensor<17xf32> { - %empty = tensor.empty() : tensor<17xf32> - %result = linalg.generic {indexing_maps = [#identity, #identity], iterator_types = ["parallel"]} ins(%x : tensor<17xf32>) outs(%empty : tensor<17xf32>) { - ^bb0(%xval: f32, %out: f32): - %t = math.sin %xval : f32 - linalg.yield %t : f32 - } -> tensor<17xf32> - return %result : tensor<17xf32> - } - - func.func @cos(%x: tensor<17xf32>) -> tensor<17xf32> { - %empty = tensor.empty() : tensor<17xf32> - %result = linalg.generic {indexing_maps = [#identity, #identity], iterator_types = ["parallel"]} ins(%x : tensor<17xf32>) outs(%empty : tensor<17xf32>) { - ^bb0(%xval: f32, %out: f32): - %t = math.cos %xval : f32 - linalg.yield %t : f32 - } -> tensor<17xf32> - return %result : tensor<17xf32> - } - -} diff --git a/waste/examples/components/balls/f32add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/f32add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index e2f15c56d..000000000 --- a/waste/examples/components/balls/f32add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankF32AddOp::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankF32AddOp op = *this; - rewriter.create(op.getLoc(), op.getA(), op.getB(), op.getC(), - op.getRowsAttr(), op.getGroupAttr(), - op.getFirstAttr()); - rewriter.replaceOp(op, op.getC()); - return success(); -} diff --git a/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/F32AddBall.td b/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/F32AddBall.td deleted file mode 100644 index eca07ca5d..000000000 --- a/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/F32AddBall.td +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef F32ADDBALL_OPS -#define F32ADDBALL_OPS - -include "Buckyball.td" - -def F32AddOp : Buckyball_Op<"f32add", [MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "Ordered RNE accumulator plus a selected FP32 source group"; - let description = [{ - Each 128-bit row contains four FP32 elements. For each element, compute - accumulator + source with RNE and canonical NaN 0x7fc00000. When first is - true, use positive zero without reading b. The three virtual banks must - differ. Only a uses the selected group; b and c each use group zero. - }]; - let arguments = (ins I64:$a, I64:$b, I64:$c, - I64Attr:$rows, I64Attr:$group, BoolAttr:$first); - let hasVerifier = 1; - let assemblyFormat = "$a `,` $b `,` $c prop-dict attr-dict `:` type($a)"; -} - -def BankF32AddOp : Buckyball_Op<"bank_f32add", [DeclareOpInterfaceMethods, MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "FP32 grouped addition with an SSA output bank handle"; - let arguments = (ins I64:$a, I64:$b, I64:$c, - I64Attr:$rows, I64Attr:$group, BoolAttr:$first); - let results = (outs I64:$cOut); - let hasVerifier = 1; - let assemblyFormat = "$a $b $c prop-dict attr-dict `:` " - "type($a) type($b) type($c)"; -} - -#endif diff --git a/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/IR/F32Add.cpp b/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/IR/F32Add.cpp deleted file mode 100644 index bcd4bc844..000000000 --- a/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/IR/F32Add.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "mlir/Dialect/Arith/IR/Arith.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -template LogicalResult verifyAddition(Op op) { - if (op.getRows() <= 0 || op.getRows() > ((int64_t(1) << 34) - 1)) - return op.emitOpError("rows must be in [1, 17179869183]"); - if (op.getGroup() < 0 || op.getGroup() > 31) - return op.emitOpError("group must be in [0, 31]"); - auto banks = op.getOperands(); - for (unsigned i = 0; i < banks.size(); ++i) { - auto lhs = banks[i].template getDefiningOp(); - if (lhs && (lhs.value() < 0 || lhs.value() > 1023)) - return op.emitOpError("bank ID must be in [0, 1023]"); - for (unsigned j = 0; j < i; ++j) { - auto rhs = banks[j].template getDefiningOp(); - if (banks[i] == banks[j] || (lhs && rhs && lhs.value() == rhs.value())) - return op.emitOpError( - "source, accumulator and output banks must differ"); - } - } - return success(); -} -} // namespace - -LogicalResult F32AddOp::verify() { return verifyAddition(*this); } -LogicalResult BankF32AddOp::verify() { return verifyAddition(*this); } diff --git a/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 354fc9d80..000000000 --- a/waste/examples/components/balls/f32add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -class F32AddLowering : public ConvertOpToLLVMPattern { -public: - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - LogicalResult - matchAndRewrite(F32AddOp op, OpAdaptor operands, - ConversionPatternRewriter &rewriter) const override { - - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || op.getRows() > target.bankDepth) - return op.emitError( - "F32ADD rows exceed the target 128-bit bank capacity"); - Location loc = op.getLoc(); - Value rs1 = - packRs1BanksIter(rewriter, loc, operands.getA(), operands.getB(), - operands.getC(), cstI64(rewriter, loc, op.getRows())); - Value rs2 = cstI64(rewriter, loc, op.getGroup() | (op.getFirst() ? 32 : 0)); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("F32ADD"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateF32AddBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { - patterns.add(converter); -} - -void configureF32AddBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/f32add/stack/src/Passes.cpp b/waste/examples/components/balls/f32add/stack/src/Passes.cpp deleted file mode 100644 index a2b0e71aa..000000000 --- a/waste/examples/components/balls/f32add/stack/src/Passes.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- Passes.cpp - F32AddBall passes -------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -namespace buckyball { -void populateF32AddBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureF32AddBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LowerF32AddPass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerF32AddPass) - StringRef getArgument() const final { return "lower-f32add-ball"; } - StringRef getDescription() const final { - return "Lower F32AddBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateF32AddBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureF32AddBallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - return true; -}(); diff --git a/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index c7fbdcdcf..000000000 --- a/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,76 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -Value shift(PatternRewriter &rewriter, Location loc, Value value, - int64_t bits) { - return rewriter.create( - loc, value, rewriter.create(loc, bits, 64)); -} - -Value rs1(PatternRewriter &rewriter, Location loc, Value a, Value b, Value c, - Value iter) { - Value ab = rewriter.create(loc, a, shift(rewriter, loc, b, 10)); - Value ci = rewriter.create(loc, shift(rewriter, loc, c, 20), - shift(rewriter, loc, iter, 30)); - return rewriter.create(loc, ab, ci); -} - -Value rs2(PatternRewriter &rewriter, Location loc, Value a, Value b, Value c, - int64_t subcommand) { - Value packed = rewriter.create(loc, subcommand, 64); - packed = - rewriter.create(loc, packed, shift(rewriter, loc, a, 6)); - packed = - rewriter.create(loc, packed, shift(rewriter, loc, b, 16)); - return rewriter.create(loc, packed, - shift(rewriter, loc, c, 26)); -} - -// Computes A x B into the output bank; accumulation sets rs1 bit 29. -template -LogicalResult compute(Op op, PatternRewriter &rewriter, StringRef mnemonic, - int64_t subcommand) { - Value operands = rs1(rewriter, op.getLoc(), op.getABank(), op.getBBank(), - op.getOutBank(), op.getIter()); - if (subcommand == 3) - operands = rewriter.create( - op.getLoc(), operands, - rewriter.create(op.getLoc(), 1LL << 29, 64)); - rewriter.create( - op.getLoc(), rewriter.getStringAttr(mnemonic), operands, - rs2(rewriter, op.getLoc(), op.getABase(), op.getBBase(), op.getOutBase(), - subcommand)); - rewriter.replaceOp(op, op.getOutBank()); - return success(); -} -} // namespace - -LogicalResult -BankGemminiPreloadOp::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - Value zero = rewriter.create(getLoc(), 0, 64); - rewriter.create( - getLoc(), rewriter.getStringAttr("GEMMINI_PRELOAD"), - rs1(rewriter, getLoc(), getInBank(), zero, getOutBank(), getIter()), - rs2(rewriter, getLoc(), getInBase(), zero, getOutBase(), 1)); - rewriter.replaceOp(*this, getOutBank()); - return success(); -} - -LogicalResult BankGemminiComputePreloadedOp::assignPhysicalBanks( - PatternRewriter &rewriter, mlir::buddy::PhysicalBankState &) { - return compute(*this, rewriter, "GEMMINI_COMPUTE_PRELOADED", 2); -} - -LogicalResult BankGemminiComputeAccumulatedOp::assignPhysicalBanks( - PatternRewriter &rewriter, mlir::buddy::PhysicalBankState &) { - return compute(*this, rewriter, "GEMMINI_COMPUTE_ACCUMULATED", 3); -} diff --git a/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp b/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp deleted file mode 100644 index c2aac15d6..000000000 --- a/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp +++ /dev/null @@ -1,307 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Math/IR/Math.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult GemminiMemConvOp::lowerToBankSSA(PatternRewriter &b) { - GemminiMemConvOp op = *this; - auto in = cast(op.getInput().getType()); - auto w = cast(op.getWeight().getType()); - auto bias = cast(op.getBias().getType()); - auto out = cast(op.getOutput().getType()); - bool integer = in.getElementType().isInteger(8) && - bias.getElementType().isInteger(32) && - out.getElementType().isInteger(32); - bool floating = in.getElementType().isF32() && - bias.getElementType().isF32() && out.getElementType().isF32(); - if (in.getRank() != 4 || w.getRank() != 2 || bias.getRank() != 1 || - out.getRank() != 4 || !in.hasStaticShape() || !w.hasStaticShape() || - !bias.hasStaticShape() || !out.hasStaticShape() || - !w.getElementType().isInteger(8) || (!integer && !floating)) - return op.emitError( - "requires static NCHW input/output and I8 KxN weights; " - "use I8 input with I32 output/bias, or F32 input/output/bias"); - - int64_t C = in.getShape()[1], H = in.getShape()[2], W = in.getShape()[3]; - int64_t OH = out.getShape()[2], OW = out.getShape()[3]; - int64_t kh = op.getKh(), kw = op.getKw(), stride = op.getStride(); - int64_t pad = op.getPadding(); - int64_t M = in.getShape()[0] * OH * OW, K = C * kh * kw; - int64_t N = w.getShape()[1], kTiles = (K + 15) / 16; - if (w.getShape()[0] != K || bias.getShape()[0] != N || - out.getShape()[0] != in.getShape()[0] || out.getShape()[1] != N || - OH != (H + 2 * pad - kh) / stride + 1 || - OW != (W + 2 * pad - kw) / stride + 1) - return op.emitError("conv shape mismatch"); - float inScale = op.getInputScale().convertToFloat(); - float wScale = op.getWeightScale().convertToFloat(); - if (!std::isfinite(inScale) || inScale <= 0 || !std::isfinite(wScale) || - wScale <= 0) - return op.emitError("scales must be finite and positive"); - - if (integer && (inScale != 1.0f || wScale != 1.0f)) - return op.emitError("native INT8 convolution requires unit scales"); - - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth < 16) - return op.emitError( - "GemminiBall requires a 128-bit bank with at least 16 lines"); - - Location loc = op.getLoc(); - auto idx = [&](int64_t v) -> Value { - return b.create(loc, v); - }; - using P = arith::CmpIPredicate; - Value zero = idx(0), one = idx(1), sixteen = idx(16); - Value zeroI8 = b.create(loc, 0, 8); - IntegerAttr align = b.getI64IntegerAttr(64); - - // Quantized im2col panels, one 16x16 tile per (M tile, K tile). - Value panels = b.create( - loc, MemRefType::get({(M + 15) / 16 * kTiles, 16, 16}, b.getI8Type()), - align); - b.create(loc, zeroI8, panels); - auto rows = b.create(loc, zero, idx(M), one); - b.setInsertionPointToStart(rows.getBody()); - Value r = rows.getInductionVar(); - Value y0 = b.create( - loc, - b.create( - loc, - b.create( - loc, b.create(loc, r, idx(OW)), idx(OH)), - idx(stride)), - idx(pad)); - Value x0 = b.create( - loc, - b.create(loc, b.create(loc, r, idx(OW)), - idx(stride)), - idx(pad)); - auto inners = b.create(loc, zero, idx(K), one); - b.setInsertionPointToStart(inners.getBody()); - Value i = inners.getInductionVar(); - Value y = b.create( - loc, y0, b.create(loc, i, idx(kw * C))); - Value x = b.create( - loc, x0, - b.create(loc, b.create(loc, i, idx(C)), - idx(kw))); - Value yIn = b.create( - loc, b.create(loc, P::sge, y, zero), - b.create(loc, P::slt, y, idx(H))); - Value xIn = b.create( - loc, b.create(loc, P::sge, x, zero), - b.create(loc, P::slt, x, idx(W))); - auto inside = - b.create(loc, b.create(loc, yIn, xIn)); - b.setInsertionPointToStart(&inside.getThenRegion().front()); - Value v = b.create( - loc, op.getInput(), - ValueRange{b.create(loc, r, idx(OH * OW)), - b.create(loc, i, idx(C)), y, x}); - if (!integer) { - v = b.create( - loc, v, b.create(loc, b.getF32FloatAttr(inScale))); - v = b.create( - loc, v, b.create(loc, b.getF32FloatAttr(-128))); - v = b.create( - loc, v, b.create(loc, b.getF32FloatAttr(127))); - v = b.create(loc, v); - v = b.create(loc, b.getI32Type(), v); - v = b.create(loc, b.getI8Type(), v); - } - Value panel = b.create( - loc, - b.create(loc, b.create(loc, r, sixteen), - idx(kTiles)), - b.create(loc, i, sixteen)); - b.create( - loc, v, panels, - ValueRange{panel, b.create(loc, r, sixteen), - b.create(loc, i, sixteen)}); - b.setInsertionPointAfter(rows); - - auto tileType = MemRefType::get({16, 16}, b.getI8Type()); - Value dTile = b.create(loc, tileType, align); - Value wTile = b.create(loc, tileType, align); - Value acc = b.create( - loc, MemRefType::get({16, 16}, b.getI32Type()), align); - b.create(loc, zeroI8, dTile); - auto meta = b.create(loc, op.getWeight()); - Value wStride = meta.getStrides()[0]; - Value wDirect = b.create( - loc, - b.create( - loc, P::eq, b.create(loc, wStride, sixteen), zero), - b.create(loc, P::eq, meta.getStrides()[1], one)); - Value wLines = b.create( - loc, b.getI64Type(), b.create(loc, wStride, sixteen)); - Value iter = createI64Const(b, loc, 16), base = createI64Const(b, loc, 0); - // Weight-stationary dataflow. - b.create(loc, b.getStringAttr("GEMMINI_CONFIG"), base, - createI64Const(b, loc, 16)); - Value ab = allocBank(b, loc, 1, 1), wb = allocBank(b, loc, 1, 1); - Value db = allocBank(b, loc, 1, 1), cb = allocBank(b, loc, 1, 4); - db = mvinBank(b, loc, dTile, db, 16); - - auto rowLoop = - b.create(loc, zero, idx(M), sixteen, ValueRange{ab, wb, cb}); - b.setInsertionPointToStart(rowLoop.getBody()); - Value row = rowLoop.getInductionVar(); - auto colLoop = b.create(loc, zero, idx(N), sixteen, - rowLoop.getRegionIterArgs()); - b.setInsertionPointToStart(colLoop.getBody()); - Value col = colLoop.getInductionVar(); - auto kLoop = b.create(loc, zero, idx(K), sixteen, - colLoop.getRegionIterArgs()); - b.setInsertionPointToStart(kLoop.getBody()); - Value k = kLoop.getInductionVar(); - ab = kLoop.getRegionIterArgs()[0]; - wb = kLoop.getRegionIterArgs()[1]; - cb = kLoop.getRegionIterArgs()[2]; - - panel = b.create( - loc, - b.create(loc, b.create(loc, row, sixteen), - idx(kTiles)), - b.create(loc, k, sixteen)); - SmallVector panelOffsets{panel, b.getIndexAttr(0), - b.getIndexAttr(0)}; - SmallVector panelSizes{b.getIndexAttr(1), b.getIndexAttr(16), - b.getIndexAttr(16)}; - SmallVector panelSteps(3, b.getIndexAttr(1)); - auto panelType = memref::SubViewOp::inferRankReducedResultType( - {16, 16}, cast(panels.getType()), panelOffsets, panelSizes, - panelSteps); - Value aTile = b.create( - loc, panelType, panels, panelOffsets, panelSizes, panelSteps); - ab = mvinBank(b, loc, aTile, ab, 16); - - Value full = b.create( - loc, wDirect, - b.create( - loc, - b.create( - loc, P::ule, b.create(loc, k, sixteen), idx(K)), - b.create(loc, P::ule, - b.create(loc, col, sixteen), - idx(N)))); - auto weight = b.create(loc, b.getI64Type(), full, true); - b.setInsertionPointToStart(&weight.getThenRegion().front()); - SmallVector wOffsets{k, col}; - SmallVector wSizes(2, b.getIndexAttr(16)); - SmallVector wSteps(2, b.getIndexAttr(1)); - Value wBlock = b.create(loc, op.getWeight(), wOffsets, - wSizes, wSteps); - b.create( - loc, ValueRange{b.create(loc, b.getI64Type(), wBlock, wb, - iter, wLines, IntegerAttr{})}); - b.setInsertionPointToStart(&weight.getElseRegion().front()); - auto wi = b.create(loc, zero, sixteen, one); - b.setInsertionPointToStart(wi.getBody()); - auto wj = b.create(loc, zero, sixteen, one); - b.setInsertionPointToStart(wj.getBody()); - Value wk = b.create(loc, k, wi.getInductionVar()); - Value wn = b.create(loc, col, wj.getInductionVar()); - auto pick = b.create( - loc, b.getI8Type(), - b.create(loc, - b.create(loc, P::ult, wk, idx(K)), - b.create(loc, P::ult, wn, idx(N))), - true); - b.setInsertionPointToStart(&pick.getThenRegion().front()); - b.create(loc, ValueRange{b.create( - loc, op.getWeight(), ValueRange{wk, wn})}); - b.setInsertionPointToStart(&pick.getElseRegion().front()); - b.create(loc, zeroI8); - b.setInsertionPointAfter(pick); - b.create( - loc, pick.getResult(0), wTile, - ValueRange{wi.getInductionVar(), wj.getInductionVar()}); - b.setInsertionPointAfter(wi); - b.create(loc, mvinBank(b, loc, wTile, wb, 16)); - b.setInsertionPointAfter(weight); - wb = weight.getResult(0); - - auto first = b.create(loc, P::eq, k, zero); - auto compute = b.create(loc, b.getI64Type(), first, true); - b.setInsertionPointToStart(&compute.getThenRegion().front()); - Value preloaded = b.create(loc, b.getI64Type(), wb, cb, - iter, base, base); - b.create( - loc, - ValueRange{b.create( - loc, b.getI64Type(), ab, db, preloaded, iter, base, base, base)}); - b.setInsertionPointToStart(&compute.getElseRegion().front()); - b.create( - loc, ValueRange{b.create( - loc, b.getI64Type(), ab, wb, cb, iter, base, base, base)}); - b.setInsertionPointAfter(compute); - b.create(loc, ValueRange{ab, wb, compute.getResult(0)}); - b.setInsertionPointAfter(kLoop); - - cb = mvoutBank(b, loc, acc, kLoop.getResult(2), 16); - auto ei = b.create( - loc, zero, - b.create(loc, sixteen, - b.create(loc, idx(M), row)), - one); - b.setInsertionPointToStart(ei.getBody()); - auto ej = b.create( - loc, zero, - b.create(loc, sixteen, - b.create(loc, idx(N), col)), - one); - b.setInsertionPointToStart(ej.getBody()); - Value ri = b.create(loc, row, ei.getInductionVar()); - Value cj = b.create(loc, col, ej.getInductionVar()); - Value sum = b.create( - loc, acc, ValueRange{ei.getInductionVar(), ej.getInductionVar()}); - Value result; - if (integer) { - result = b.create( - loc, sum, b.create(loc, op.getBias(), ValueRange{cj})); - } else { - result = b.create( - loc, - b.create( - loc, b.create(loc, b.getF32Type(), sum), - b.create(loc, - b.getF32FloatAttr(inScale * wScale))), - b.create(loc, op.getBias(), ValueRange{cj})); - } - b.create( - loc, result, op.getOutput(), - ValueRange{b.create(loc, ri, idx(OH * OW)), cj, - b.create( - loc, b.create(loc, ri, idx(OW)), idx(OH)), - b.create(loc, ri, idx(OW))}); - b.setInsertionPointAfter(ei); - b.create( - loc, ValueRange{kLoop.getResult(0), kLoop.getResult(1), cb}); - b.setInsertionPointAfter(colLoop); - b.create(loc, colLoop.getResults()); - b.setInsertionPointAfter(rowLoop); - - for (Value bank : rowLoop.getResults()) - releaseBank(b, loc, bank); - releaseBank(b, loc, db); - for (Value buffer : {panels, dTile, wTile, acc}) - b.create(loc, buffer); - b.eraseOp(op); - return success(); -} diff --git a/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerLinalgToBuckyball/Int8Contractions.cpp b/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerLinalgToBuckyball/Int8Contractions.cpp deleted file mode 100644 index 5b3d7a8e0..000000000 --- a/waste/examples/components/balls/gemmini/stack/src/Conversion/LowerLinalgToBuckyball/Int8Contractions.cpp +++ /dev/null @@ -1,132 +0,0 @@ -//===- Int8Contractions.cpp - INT8 contractions for GemminiBall -//------------===// - -#include "Buckyball/BuckyballOps.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/PatternMatch.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class Int8ContractionLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(linalg::GenericOp op, - PatternRewriter &rewriter) const override { - auto convolution = op->getAttrOfType("buddy.int8_conv"); - bool matrix = op->hasAttr("buddy.int8_matmul"); - if (!convolution && !matrix) - return failure(); - if (op.getInputs().size() != 2 || op.getOutputs().size() != 1) - return op.emitError( - "INT8 contraction requires two inputs and one output"); - Value input = op.getInputs()[0], weight = op.getInputs()[1]; - Value output = op.getOutputs()[0]; - auto in = dyn_cast(input.getType()); - auto w = dyn_cast(weight.getType()); - auto out = dyn_cast(output.getType()); - if (!in || !w || !out || !in.hasStaticShape() || !w.hasStaticShape() || - !out.hasStaticShape() || !in.getElementType().isInteger(8) || - !w.getElementType().isInteger(8) || !out.getElementType().isInteger(32)) - return op.emitError( - "GemminiBall requires static INT8 operands and INT32 output"); - Location loc = op.getLoc(); - Value packedWeight; - SmallVector permutation; - if (matrix) { - auto map = op.getIndexingMapsArray()[1]; - if (map.getResult(0) == getAffineDimExpr(1, rewriter.getContext())) - permutation = {1, 0}; - } else { - auto layout = - cast(convolution.get("weight_permutation")) - .asArrayRef(); - if (layout == ArrayRef{0, 1, 2, 3}) - permutation = {2, 3, 1, 0}; - } - if (!permutation.empty()) { - SmallVector shape; - for (int64_t axis : permutation) - shape.push_back(w.getShape()[axis]); - packedWeight = memref::AllocOp::create( - rewriter, loc, MemRefType::get(shape, rewriter.getI8Type())); - SmallVector coordinates; - for (int64_t extent : w.getShape()) { - auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0); - auto end = arith::ConstantIndexOp::create(rewriter, loc, extent); - auto one = arith::ConstantIndexOp::create(rewriter, loc, 1); - auto loop = scf::ForOp::create(rewriter, loc, zero, end, one); - rewriter.setInsertionPointToStart(loop.getBody()); - coordinates.push_back(loop.getInductionVar()); - } - Value value = memref::LoadOp::create(rewriter, loc, weight, coordinates); - SmallVector indices; - for (int64_t axis : permutation) - indices.push_back(coordinates[axis]); - memref::StoreOp::create(rewriter, loc, value, packedWeight, indices); - rewriter.setInsertionPoint(op); - weight = packedWeight; - w = cast(weight.getType()); - } - int64_t kh = 1, kw = 1, stride = 1, channels; - if (matrix) { - if (in.getRank() != 2 || w.getRank() != 2 || out.getRank() != 2) - return op.emitError("GemminiBall matrix operands must have rank two"); - channels = out.getShape()[1]; - input = memref::ExpandShapeOp::create( - rewriter, loc, - ArrayRef{in.getShape()[0], in.getShape()[1], 1, 1}, input, - ArrayRef{{0}, {1, 2, 3}}); - output = memref::ExpandShapeOp::create( - rewriter, loc, ArrayRef{out.getShape()[0], channels, 1, 1}, - output, ArrayRef{{0}, {1, 2, 3}}); - } else { - auto steps = - dyn_cast_or_null(convolution.get("stride")); - auto dilation = - dyn_cast_or_null(convolution.get("dilation")); - auto permutation = dyn_cast_or_null( - convolution.get("weight_permutation")); - auto groups = dyn_cast_or_null(convolution.get("groups")); - if (in.getRank() != 4 || w.getRank() != 4 || out.getRank() != 4 || - !steps || steps.size() != 2 || steps[0] != steps[1] || !dilation || - dilation.asArrayRef() != ArrayRef{1, 1} || !groups || - groups.getInt() != 1 || !permutation || - (permutation.asArrayRef() != ArrayRef{2, 3, 1, 0} && - permutation.asArrayRef() != ArrayRef{0, 1, 2, 3})) - return op.emitError("GemminiBall requires ordinary NCHW convolution " - "with HWIO weights and unit dilation"); - kh = w.getShape()[0]; - kw = w.getShape()[1]; - stride = steps[0]; - channels = out.getShape()[1]; - weight = memref::CollapseShapeOp::create( - rewriter, loc, weight, - ArrayRef{{0, 1, 2}, {3}}); - } - auto biasType = MemRefType::get({channels}, rewriter.getI32Type()); - Value bias = memref::AllocOp::create(rewriter, loc, biasType); - Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32); - linalg::FillOp::create(rewriter, loc, zero, bias); - GemminiMemConvOp::create( - rewriter, loc, input, weight, bias, output, - rewriter.getI64IntegerAttr(kh), rewriter.getI64IntegerAttr(kw), - rewriter.getI64IntegerAttr(stride), rewriter.getI64IntegerAttr(0), - rewriter.getF32FloatAttr(1.0), rewriter.getF32FloatAttr(1.0)); - memref::DeallocOp::create(rewriter, loc, bias); - if (packedWeight) - memref::DeallocOp::create(rewriter, loc, packedWeight); - rewriter.eraseOp(op); - return success(); - } -}; -} // namespace -namespace mlir::buddy { -void populateGemminiBallLinalgPatterns(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} -} // namespace mlir::buddy diff --git a/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/GemminiBall.td b/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/GemminiBall.td deleted file mode 100644 index 9d8453483..000000000 --- a/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/GemminiBall.td +++ /dev/null @@ -1,73 +0,0 @@ -//===-- GemminiBall.td - Gemmini Ball operations ------------------------===// - -#ifndef GEMMINI_BALL_OPS -#define GEMMINI_BALL_OPS - -include "Buckyball.td" -include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td" - -def GemminiInstructionOp : Buckyball_Op<"gemmini"> { - let summary = "Gemmini command encoded for the target ISA"; - let description = [{ - Emits one Gemmini command. mnemonic must name an ISA entry enabled by the - selected target, and rs1/rs2 are the exact command operands defined by - that entry. This covers GEMMINI_CONFIG, GEMMINI_PRELOAD, - GEMMINI_COMPUTE_PRELOADED, GEMMINI_COMPUTE_ACCUMULATED, GEMMINI_LOOP_WS, - and GEMMINI_LOOP_CONV_WS without hard-coding funct7 values in the IR. - }]; - let arguments = (ins StrAttr:$mnemonic, I64:$rs1, I64:$rs2); - let assemblyFormat = "$mnemonic $rs1 `,` $rs2 prop-dict attr-dict `:` type($rs1)"; -} - -def BankGemminiPreloadOp : Buckyball_Op<"bank_gemmini_preload", [DeclareOpInterfaceMethods]> { - let summary = "BankSSA Gemmini preload"; - let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, I64:$inBase, - I64:$outBase); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$inBank $outBank $iter $inBase $outBase prop-dict attr-dict `:` " - "type($inBank) type($outBank) type($iter) type($inBase) type($outBase)"; -} - -def BankGemminiComputePreloadedOp : Buckyball_Op<"bank_gemmini_compute_preloaded", [DeclareOpInterfaceMethods]> { - let summary = "BankSSA Gemmini compute using preloaded state"; - let arguments = (ins I64:$aBank, I64:$bBank, I64:$outBank, I64:$iter, - I64:$aBase, I64:$bBase, I64:$outBase); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$aBank $bBank $outBank $iter $aBase $bBase $outBase prop-dict attr-dict `:` " - "type($aBank) type($bBank) type($outBank) type($iter) type($aBase) type($bBase) type($outBase)"; -} - -def BankGemminiComputeAccumulatedOp : Buckyball_Op<"bank_gemmini_compute_accumulated", [DeclareOpInterfaceMethods]> { - let summary = "BankSSA Gemmini compute accumulating the output bank"; - let arguments = (ins I64:$aBank, I64:$bBank, I64:$outBank, I64:$iter, - I64:$aBase, I64:$bBase, I64:$outBase); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$aBank $bBank $outBank $iter $aBase $bBase $outBase prop-dict attr-dict `:` " - "type($aBank) type($bBank) type($outBank) type($iter) type($aBase) type($bBase) type($outBase)"; -} - -def GemminiConvOp : Buckyball_Op<"gemmini_conv", [ - DeclareOpInterfaceMethods]> { - let summary = "NCHW convolution with INT8 HWIO matrix weights"; - let description = [{ - FP32 inputs are quantized using the supplied scales and produce FP32 output. - INT8 inputs bypass quantization and produce INT32 output with unit scales. - }]; - let arguments = (ins AnyRankedTensor:$input, AnyRankedTensor:$weight, - AnyRankedTensor:$bias, I64Attr:$kh, I64Attr:$kw, I64Attr:$stride, - I64Attr:$padding, F32Attr:$inputScale, F32Attr:$weightScale); - let results = (outs AnyRankedTensor:$output); - let assemblyFormat = "$input `,` $weight `,` $bias prop-dict attr-dict `:` type($input) `,` type($weight) `,` type($bias) `->` type($output)"; -} -def GemminiMemConvOp : Buckyball_Op<"gemmini_mem_conv", [DeclareOpInterfaceMethods]> { - let arguments = (ins Arg]>:$input, - Arg]>:$weight, - Arg]>:$bias, - Arg]>:$output, - I64Attr:$kh, I64Attr:$kw, I64Attr:$stride, - I64Attr:$padding, F32Attr:$inputScale, F32Attr:$weightScale); - let assemblyFormat = "$input `,` $weight `,` $bias `,` $output prop-dict attr-dict `:` type($input) `,` type($weight) `,` type($bias) `,` type($output)"; -} - -#endif // GEMMINI_BALL_OPS diff --git a/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/IR/GemminiConv.cpp b/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/IR/GemminiConv.cpp deleted file mode 100644 index 82caba124..000000000 --- a/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/IR/GemminiConv.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -using namespace mlir; -using namespace mlir::bufferization; -using namespace ::buddy::buckyball; -bool GemminiConvOp::bufferizesToAllocation(Value) { return true; } -bool GemminiConvOp::bufferizesToMemoryRead(OpOperand &, const AnalysisState &) { - return true; -} -bool GemminiConvOp::bufferizesToMemoryWrite(OpOperand &, - const AnalysisState &) { - return false; -} -AliasingValueList GemminiConvOp::getAliasingValues(OpOperand &, - const AnalysisState &) { - return {}; -} -LogicalResult GemminiConvOp::bufferize(RewriterBase &b, - const BufferizationOptions &options, - BufferizationState &state) { - auto input = getBuffer(b, getInput(), options, state), - weight = getBuffer(b, getWeight(), options, state), - bias = getBuffer(b, getBias(), options, state); - if (failed(input) || failed(weight) || failed(bias)) - return failure(); - auto type = cast(getOutput().getType()); - if (!type.hasStaticShape()) - return emitError("Gemmini requires static model shapes"); - Value output = b.create( - getLoc(), MemRefType::get(type.getShape(), type.getElementType())); - b.create(getLoc(), *input, *weight, *bias, output, - getKhAttr(), getKwAttr(), getStrideAttr(), - getPaddingAttr(), getInputScaleAttr(), - getWeightScaleAttr()); - replaceOpWithBufferizedValues(b, getOperation(), output); - return success(); -} diff --git a/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 5c44ca176..000000000 --- a/waste/examples/components/balls/gemmini/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class GemminiInstructionLowering - : public ConvertOpToLLVMPattern { -public: - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(GemminiInstructionOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - rewriter.replaceOpWithNewOp( - op, adaptor.getRs1(), adaptor.getRs2(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7(op.getMnemonic()))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateGemminiBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - (void)stable; - (void)bankDepth; - - patterns.add(converter); -} - -void configureGemminiBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/gemmini/stack/src/Passes.cpp b/waste/examples/components/balls/gemmini/stack/src/Passes.cpp deleted file mode 100644 index 673db551c..000000000 --- a/waste/examples/components/balls/gemmini/stack/src/Passes.cpp +++ /dev/null @@ -1,56 +0,0 @@ -//===- Passes.cpp - GemminiBall passes ------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -void populateGemminiBallLinalgPatterns(RewritePatternSet &patterns); -namespace buckyball { -void populateGemminiBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureGemminiBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LinalgToGemminiPass - : public mlir::buddy::LinalgToBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LinalgToGemminiPass) - StringRef getArgument() const final { - return "convert-linalg-to-gemmini-ball"; - } - StringRef getDescription() const final { - return "Lower signed INT8 Linalg contractions to GemminiBall."; - } - void populate(RewritePatternSet &patterns, int64_t, int64_t, int64_t) const { - mlir::buddy::populateGemminiBallLinalgPatterns(patterns); - } -}; - -class LowerGemminiPass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerGemminiPass) - StringRef getArgument() const final { return "lower-gemmini-ball"; } - StringRef getDescription() const final { - return "Lower GemminiBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateGemminiBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureGemminiBallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - PassRegistration(); - return true; -}(); diff --git a/waste/examples/components/balls/im2col/stack/src/Backend/include/IntrinsicsRISCVIm2colBall.td b/waste/examples/components/balls/im2col/stack/src/Backend/include/IntrinsicsRISCVIm2colBall.td deleted file mode 100644 index 52037c087..000000000 --- a/waste/examples/components/balls/im2col/stack/src/Backend/include/IntrinsicsRISCVIm2colBall.td +++ /dev/null @@ -1,4 +0,0 @@ -//===- IntrinsicsRISCVIm2colBall.td - Im2colBall intrinsics --------------===// - -let TargetPrefix = "riscv" in -def int_riscv_bb_im2col : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; diff --git a/waste/examples/components/balls/im2col/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/im2col/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index f237f4067..000000000 --- a/waste/examples/components/balls/im2col/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,23 +0,0 @@ -//===- AssignPhysicalBanks.cpp - Im2col bank assignment -------------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankIm2colOp::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankIm2colOp op = *this; - rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), - op.getIter(), op.getKsize(), op.getStride(), - op.getPadding(), op.getInputBase(), op.getLane(), - op.getStartRowAttr(), op.getStartColAttr(), - op.getWindowStartAttr(), op.getWindowCountAttr()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); -} diff --git a/waste/examples/components/balls/im2col/stack/src/Dialect/Buckyball/Im2colBall.td b/waste/examples/components/balls/im2col/stack/src/Dialect/Buckyball/Im2colBall.td deleted file mode 100644 index 112001c9f..000000000 --- a/waste/examples/components/balls/im2col/stack/src/Dialect/Buckyball/Im2colBall.td +++ /dev/null @@ -1,133 +0,0 @@ -//===-- Im2colBall.td - Im2colBall operations --------------*- tablegen -*-===// -// -// Im2colBall operations for image-to-column transformation. -// -//===----------------------------------------------------------------------===// - -#ifndef IM2COLBALL_OPS -#define IM2COLBALL_OPS - -include "Buckyball.td" - -//===----------------------------------------------------------------------===// -// Im2colBall operations -//===----------------------------------------------------------------------===// - -// Memref-level im2col+gemm (like smatmul_matmul). Design bank-ssa decides -// f32 quant / Bank* composition; this op carries no quant policy. -def Im2colMatmulOp : Buckyball_Op<"im2col_matmul"> { - let summary = "Packed-plane im2col then matrix gemm"; - let description = [{ - One cin-plane im2col, then gemm over all N columns of `filter`. - `n` is OC (padded to a multiple of 16, 16..4096). `input` is one packed - HxW plane (bank rows x 16). `filter`/`output` are KxN and MxN. - Asym pad co-design: padding=max(lo,hi), startRow/Col=padding-lo. - }]; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, - MemRefRankOf<[AnyType], [2]>:$filter, - MemRefRankOf<[AnyType], [2]>:$output, - I64Attr:$inSize, - I64Attr:$ksize, - I64Attr:$n, - DefaultValuedAttr:$stride, - DefaultValuedAttr:$padding, - DefaultValuedAttr:$startRow, - DefaultValuedAttr:$startCol, - DefaultValuedAttr:$dwAddr, - DefaultValuedAttr:$dwBytes, - DefaultValuedAttr:$perChannel); - let assemblyFormat = [{$input $filter $output prop-dict attr-dict `:` - type($input) type($filter) type($output)}]; -} - -// Standard conv: nCin input planes -> one im2col+gemm with K = nCin*K^2. -def Im2colFatMatmulOp : Buckyball_Op<"im2col_fat_matmul"> { - let summary = "Multi-plane im2col then fat gemm over nCin input channels"; - let description = [{ - Stack $nCin input planes as [nCin*inRows, 16] and filters as - [nCin*bRows, n]. Im2col each plane, concatenate on K, then one - (wins x nCin*K^2) @ (nCin*K^2 x n) gemm. $n is OC padded to 16. - }]; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, - MemRefRankOf<[AnyType], [2]>:$filter, - MemRefRankOf<[AnyType], [2]>:$output, - I64Attr:$inSize, - I64Attr:$ksize, - I64Attr:$n, - I64Attr:$nCin, - DefaultValuedAttr:$stride, - DefaultValuedAttr:$padding, - DefaultValuedAttr:$startRow, - DefaultValuedAttr:$startCol, - DefaultValuedAttr:$dwAddr, - DefaultValuedAttr:$dwBytes, - DefaultValuedAttr:$perChannel); - let assemblyFormat = [{$input $filter $output prop-dict attr-dict `:` - type($input) type($filter) type($output)}]; -} - -// Depthwise: n channels (1..16) share one fat GEMM, not per-channel GEMV. -// Input planes stacked as [n*inRows, 16]; filter is block-diag [n*bRows, 16]. -def Im2colDepthwiseMatmulOp : Buckyball_Op<"im2col_depthwise_matmul"> { - let summary = "Multi-plane im2col then fat depthwise gemm (N=lane)"; - let description = [{ - For channel tile size $n (1..16): - im2col each plane, concatenate on K, then (wins x n*K^2) @ (n*K^2 x n) - with block-diagonal weights. Lane-full GEMM; not GEMV. - }]; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, - MemRefRankOf<[AnyType], [2]>:$filter, - MemRefRankOf<[AnyType], [2]>:$output, - I64Attr:$inSize, - I64Attr:$ksize, - I64Attr:$n, - DefaultValuedAttr:$stride, - DefaultValuedAttr:$padding, - DefaultValuedAttr:$startRow, - DefaultValuedAttr:$startCol, - DefaultValuedAttr:$dwAddr, - DefaultValuedAttr:$dwBytes, - DefaultValuedAttr:$perChannel); - let assemblyFormat = [{$input $filter $output prop-dict attr-dict `:` - type($input) type($filter) type($output)}]; -} - -def Im2colOp : Buckyball_Op<"im2col"> { - let summary = "Image-to-column transformation for convolution"; - let description = [{ - Transforms image data into column format for efficient convolution. - Each input bank row is one spatial position with 16 INT8 channels. - inputBase selects the first spatial row and lane selects one channel. - windowStart/windowCount select a contiguous output-window range and rebase - it to row zero in the output bank. - Legal range is defined in examples/balls/im2col/configs/default.toml. - }]; - let arguments = (ins I64:$inputBankId, I64:$outputBankId, - I64:$iter, I64:$ksize, I64:$stride, I64:$padding, - I64:$inputBase, I64:$lane, - I64Attr:$startRow, I64Attr:$startCol, - I64Attr:$windowStart, I64Attr:$windowCount); - let assemblyFormat = "$inputBankId `,` $outputBankId `,` " - "$iter `,` $ksize `,` $stride `,` $padding `,` " - "$inputBase `,` $lane prop-dict attr-dict `:` " - "type($inputBankId)"; -} - -def BankIm2colOp : Buckyball_Op<"bank_im2col", [DeclareOpInterfaceMethods]> { - let summary = "Im2col using virtual bank handles"; - let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, I64:$ksize, - I64:$stride, I64:$padding, - I64:$inputBase, I64:$lane, - I64Attr:$startRow, I64Attr:$startCol, - I64Attr:$windowStart, I64Attr:$windowCount); - let results = (outs I64:$outBankOut); - let assemblyFormat = - "$inBank $outBank $iter $ksize $stride $padding $inputBase $lane " - "prop-dict attr-dict `:` type($inBank) type($outBank) type($iter) type($ksize) " - "type($stride) type($padding) type($inputBase) type($lane)"; -} - -def Im2colIntrOp : Buckyball_IntrOpBase<"im2col">, - Arguments<(ins LLVM_Type, LLVM_Type)>; - -#endif // IM2COLBALL_OPS diff --git a/waste/examples/components/balls/im2col/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/im2col/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 603dbb6a7..000000000 --- a/waste/examples/components/balls/im2col/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,234 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -static LogicalResult -validateIm2colEncoding(Operation *op, Value iterValue, Value ksizeValue, - Value strideValue, Value paddingValue, - Value inputBaseValue, Value laneValue, int64_t startRow, - int64_t startCol, int64_t windowStart, - int64_t windowCount, int64_t bankDepth) { - auto getConst = [](Value value) -> std::optional { - if (auto constant = value.getDefiningOp()) - if (auto attr = dyn_cast(constant.getValue())) - return attr.getInt(); - return std::nullopt; - }; - auto iter = getConst(iterValue); - auto ksize = getConst(ksizeValue); - auto stride = getConst(strideValue); - auto padding = getConst(paddingValue); - auto inputBase = getConst(inputBaseValue); - auto lane = getConst(laneValue); - if (!iter || !ksize || !stride || !padding) - return op->emitError( - "Im2col instruction requires constant iter/ksize/stride/padding"); - if (*iter <= 0 || *ksize <= 0 || *ksize > 255 || *stride <= 0 || - *stride > 255 || *padding < 0 || *padding > 255 || startRow < 0 || - startRow > 255 || startCol < 0 || startCol > 255 || - (inputBase && (*inputBase < 0 || *inputBase >= 64)) || - (lane && (*lane < 0 || *lane >= 16))) - return op->emitError("Im2col instruction field out of range"); - int64_t inputRows = *iter * *iter; - if (inputBase && *inputBase + inputRows > bankDepth) - return op->emitError("Im2col input range exceeds the physical bank depth"); - if (startRow > *padding || startCol > *padding) - return op->emitError("Im2col startRow/startCol must not exceed padding"); - if (windowStart < 0 || windowStart >= 64 || windowCount <= 0 || - windowCount > 64) - return op->emitError( - "Im2col requires windowStart in [0, 63] and windowCount in [1, 64]"); - - int64_t padded = *iter + 2 * *padding; - if (padded < *ksize + startRow || padded < *ksize + startCol || - (padded - *ksize - startRow) % *stride || - (padded - *ksize - startCol) % *stride) - return op->emitError("Im2col shape does not produce integral windows"); - int64_t rows = (padded - *ksize - startRow) / *stride + 1; - int64_t cols = (padded - *ksize - startCol) / *stride + 1; - if (windowStart + windowCount > rows * cols) - return op->emitError("Im2col window range exceeds output shape"); - int64_t kernelTiles = (*ksize * *ksize + 15) / 16; - int64_t outputRows = (windowCount + 15) / 16 * kernelTiles * 16; - if (outputRows > bankDepth) - return op->emitError("Im2col output range exceeds the physical bank depth"); - return success(); -} - -struct Im2colLowering : public ConvertOpToLLVMPattern { - Im2colLowering(LLVMTypeConverter &converter, bool stable, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), stable(stable), - bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(Im2colOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - if (failed(validateIm2colEncoding( - op, op.getIter(), op.getKsize(), op.getStride(), op.getPadding(), - op.getInputBase(), op.getLane(), op.getStartRow(), op.getStartCol(), - op.getWindowStart(), op.getWindowCount(), bankDepth))) - return failure(); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - - Value rs2 = adaptor.getKsize(); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getStride(), - cstI64(rewriter, loc, 8))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getPadding(), - cstI64(rewriter, loc, 16))); - Value startCol = cstI64(rewriter, loc, op.getStartCol()); - Value startRow = cstI64(rewriter, loc, op.getStartRow()); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, startCol, - cstI64(rewriter, loc, 24))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, startRow, - cstI64(rewriter, loc, 32))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getInputBase(), - cstI64(rewriter, loc, 40))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getLane(), - cstI64(rewriter, loc, 46))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create( - loc, cstI64(rewriter, loc, op.getWindowStart()), - cstI64(rewriter, loc, 50))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create( - loc, cstI64(rewriter, loc, op.getWindowCount()), - cstI64(rewriter, loc, 56))); - - if (stable) { - rewriter.replaceOpWithNewOp(op, rs1, rs2); - return success(); - } - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("IM2COL"))); - return success(); - } - -private: - bool stable = false; - int64_t bankDepth; -}; -struct BankIm2colLowering : public ConvertOpToLLVMPattern { - BankIm2colLowering(LLVMTypeConverter &converter, bool stable, - int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), stable(stable), - bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(BankIm2colOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - if (failed(validateIm2colEncoding( - op, op.getIter(), op.getKsize(), op.getStride(), op.getPadding(), - op.getInputBase(), op.getLane(), op.getStartRow(), op.getStartCol(), - op.getWindowStart(), op.getWindowCount(), bankDepth))) - return failure(); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInBank(), - cstI64(rewriter, loc, 0), adaptor.getOutBank(), - adaptor.getIter()); - - Value rs2 = adaptor.getKsize(); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getStride(), - cstI64(rewriter, loc, 8))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getPadding(), - cstI64(rewriter, loc, 16))); - Value startCol = cstI64(rewriter, loc, op.getStartCol()); - Value startRow = cstI64(rewriter, loc, op.getStartRow()); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, startCol, - cstI64(rewriter, loc, 24))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, startRow, - cstI64(rewriter, loc, 32))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getInputBase(), - cstI64(rewriter, loc, 40))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getLane(), - cstI64(rewriter, loc, 46))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create( - loc, cstI64(rewriter, loc, op.getWindowStart()), - cstI64(rewriter, loc, 50))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create( - loc, cstI64(rewriter, loc, op.getWindowCount()), - cstI64(rewriter, loc, 56))); - - if (stable) { - rewriter.replaceOpWithNewOp(op, rs1, rs2); - return success(); - } - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("IM2COL"))); - return success(); - } - -private: - bool stable = false; - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateIm2colBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - patterns.add(converter, stable, bankDepth); - patterns.add(converter, stable, bankDepth); -} - -void configureIm2colBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - if (stable) - target.addLegalOp(); - else - target.addIllegalOp(); - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/im2col/stack/src/Passes.cpp b/waste/examples/components/balls/im2col/stack/src/Passes.cpp deleted file mode 100644 index 3234a5aac..000000000 --- a/waste/examples/components/balls/im2col/stack/src/Passes.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- Passes.cpp - Im2colBall passes -------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -namespace buckyball { -void populateIm2colBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureIm2colBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LowerIm2colPass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerIm2colPass) - StringRef getArgument() const final { return "lower-im2col-ball"; } - StringRef getDescription() const final { - return "Lower Im2colBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateIm2colBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureIm2colBallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - return true; -}(); diff --git a/waste/examples/components/balls/int2fp/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/int2fp/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index 30ab5dd49..000000000 --- a/waste/examples/components/balls/int2fp/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,21 +0,0 @@ -//===- AssignPhysicalBanks.cpp - Int2Fp bank assignment -------------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankInt32ToFp32Op::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankInt32ToFp32Op op = *this; - rewriter.create(op.getLoc(), op.getInBank(), op.getScaleBank(), - op.getOutBank(), op.getIter(), - op.getReluAttr()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); -} diff --git a/waste/examples/components/balls/int2fp/stack/src/Dialect/Buckyball/Int2FpBall.td b/waste/examples/components/balls/int2fp/stack/src/Dialect/Buckyball/Int2FpBall.td deleted file mode 100644 index 34297d84a..000000000 --- a/waste/examples/components/balls/int2fp/stack/src/Dialect/Buckyball/Int2FpBall.td +++ /dev/null @@ -1,40 +0,0 @@ -//===-- Int2FpBall.td - Int2FpBall operations --------------*- tablegen -*-===// -// -// Int2FpBall operations for Mega Kernel output conversion. -// -//===----------------------------------------------------------------------===// - -#ifndef INT2FPBALL_OPS -#define INT2FPBALL_OPS - -include "Buckyball.td" - -//===----------------------------------------------------------------------===// -// Int2FpBall operations -//===----------------------------------------------------------------------===// - -def Int32ToFp32Op : Buckyball_Op<"int32_to_fp32"> { - let summary = "Convert INT32 accumulators to FP32"; - let description = [{ - Converts four INT32 values per input row to FP32 and multiplies them by - the matching values from a 16-channel FP32 scale bank. Scale rows repeat - every four accumulator rows. - }]; - let arguments = (ins I64:$inputBankId, I64:$scaleBankId, - I64:$outputBankId, I64:$iter, BoolAttr:$relu); - let assemblyFormat = "$inputBankId `,` $scaleBankId `,` $outputBankId `,` " - "$iter prop-dict attr-dict `:` " - "type($inputBankId)"; -} - -def BankInt32ToFp32Op : Buckyball_Op<"bank_int32_to_fp32", [DeclareOpInterfaceMethods]> { - let summary = "INT32 to FP32 conversion with virtual bank handles"; - let arguments = (ins I64:$inBank, I64:$scaleBank, I64:$outBank, - I64:$iter, BoolAttr:$relu); - let results = (outs I64:$outBankOut); - let assemblyFormat = - "$inBank $scaleBank $outBank $iter prop-dict attr-dict `:` " - "type($inBank) type($scaleBank) type($outBank) type($iter)"; -} - -#endif // INT2FPBALL_OPS diff --git a/waste/examples/components/balls/int2fp/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/int2fp/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 93a4f6e14..000000000 --- a/waste/examples/components/balls/int2fp/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -struct Int32ToFp32Lowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(Int32ToFp32Op op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - adaptor.getScaleBankId(), - adaptor.getOutputBankId(), adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, cstI64(rewriter, loc, op.getRelu() ? 1 : 0), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("INT32_TO_FP32"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateInt2FpBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t) { - (void)stable; - patterns.add(converter); -} - -void configureInt2FpBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/int2fp/stack/src/Passes.cpp b/waste/examples/components/balls/int2fp/stack/src/Passes.cpp deleted file mode 100644 index c3d5c1093..000000000 --- a/waste/examples/components/balls/int2fp/stack/src/Passes.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- Passes.cpp - Int2FpBall passes -------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -namespace buckyball { -void populateInt2FpBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureInt2FpBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LowerInt2FpPass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerInt2FpPass) - StringRef getArgument() const final { return "lower-int2fp-ball"; } - StringRef getDescription() const final { - return "Lower Int2FpBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateInt2FpBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureInt2FpBallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - return true; -}(); diff --git a/waste/examples/components/balls/int8add/stack/src/Backend/include/IntrinsicsRISCVInt8AddBall.td b/waste/examples/components/balls/int8add/stack/src/Backend/include/IntrinsicsRISCVInt8AddBall.td deleted file mode 100644 index eddf2840a..000000000 --- a/waste/examples/components/balls/int8add/stack/src/Backend/include/IntrinsicsRISCVInt8AddBall.td +++ /dev/null @@ -1 +0,0 @@ -// Int8AddBall uses CustomIntrOp with funct7 from the target ballISA. diff --git a/waste/examples/components/balls/int8add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/int8add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index fe667e078..000000000 --- a/waste/examples/components/balls/int8add/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankInt8AddOp::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankInt8AddOp op = *this; - rewriter.create(op.getLoc(), op.getLhsBank(), op.getRhsBank(), - op.getOutputBank(), op.getIter(), op.getLhsRatio(), - op.getRhsRatio(), op.getReluAttr()); - rewriter.replaceOp(op, op.getOutputBank()); - return success(); -} diff --git a/waste/examples/components/balls/int8add/stack/src/Dialect/Buckyball/Int8AddBall.td b/waste/examples/components/balls/int8add/stack/src/Dialect/Buckyball/Int8AddBall.td deleted file mode 100644 index bcbca0cb1..000000000 --- a/waste/examples/components/balls/int8add/stack/src/Dialect/Buckyball/Int8AddBall.td +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef INT8ADDBALL_OPS -#define INT8ADDBALL_OPS - -include "Buckyball.td" - -def Int8AddOp : Buckyball_Op<"int8add"> { - let summary = "Scaled INT8 addition on physical banks"; - let arguments = (ins I64:$lhsBankId, I64:$rhsBankId, I64:$outputBankId, - I64:$iter, F32:$lhsRatio, F32:$rhsRatio, - BoolAttr:$relu); - let assemblyFormat = "$lhsBankId `,` $rhsBankId `,` $outputBankId `,` $iter `,` " - "$lhsRatio `,` $rhsRatio prop-dict attr-dict `:` type($lhsBankId)"; -} - -def BankInt8AddOp : Buckyball_Op<"bank_int8add", [DeclareOpInterfaceMethods]> { - let summary = "Scaled INT8 addition on virtual banks"; - let arguments = (ins I64:$lhsBank, I64:$rhsBank, I64:$outputBank, - I64:$iter, F32:$lhsRatio, F32:$rhsRatio, - BoolAttr:$relu); - let results = (outs I64:$outputBankOut); - let assemblyFormat = "$lhsBank $rhsBank $outputBank $iter $lhsRatio $rhsRatio " - "prop-dict attr-dict `:` type($lhsBank) type($rhsBank) " - "type($outputBank) type($iter) type($lhsRatio)"; -} - -#endif // INT8ADDBALL_OPS diff --git a/waste/examples/components/balls/int8add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/int8add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index aa924bb03..000000000 --- a/waste/examples/components/balls/int8add/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -struct Int8AddLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(Int8AddOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getLhsBankId(), - adaptor.getRhsBankId(), - adaptor.getOutputBankId(), adaptor.getIter()); - Value lhsBits = rewriter.create( - loc, rewriter.getI32Type(), adaptor.getLhsRatio()); - Value rhsBits = rewriter.create( - loc, rewriter.getI32Type(), adaptor.getRhsRatio()); - Value lhs64 = - rewriter.create(loc, rewriter.getI64Type(), lhsBits); - Value rhs64 = - rewriter.create(loc, rewriter.getI64Type(), rhsBits); - Value rs2 = rewriter.create( - loc, lhs64, - rewriter.create(loc, rhs64, cstI64(rewriter, loc, 32))); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr(buckyball_target::getBuckyballFunct7( - op.getRelu() ? "INT8ADD_RELU" : "INT8ADD"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateInt8AddBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { - patterns.add(converter); -} - -void configureInt8AddBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/int8add/stack/src/Passes.cpp b/waste/examples/components/balls/int8add/stack/src/Passes.cpp deleted file mode 100644 index 333431c85..000000000 --- a/waste/examples/components/balls/int8add/stack/src/Passes.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- Passes.cpp - Int8AddBall passes ------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -namespace buckyball { -void populateInt8AddBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureInt8AddBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LowerInt8AddPass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerInt8AddPass) - StringRef getArgument() const final { return "lower-int8add-ball"; } - StringRef getDescription() const final { - return "Lower Int8AddBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateInt8AddBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureInt8AddBallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - return true; -}(); diff --git a/waste/examples/components/balls/int8mul/stack/src/Backend/include/IntrinsicsRISCVInt8MulBall.td b/waste/examples/components/balls/int8mul/stack/src/Backend/include/IntrinsicsRISCVInt8MulBall.td deleted file mode 100644 index 0a743ab0f..000000000 --- a/waste/examples/components/balls/int8mul/stack/src/Backend/include/IntrinsicsRISCVInt8MulBall.td +++ /dev/null @@ -1 +0,0 @@ -// Int8MulBall uses CustomIntrOp with funct7 from the target ballISA. diff --git a/waste/examples/components/balls/int8mul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/int8mul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index 4b93ea121..000000000 --- a/waste/examples/components/balls/int8mul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankInt8MulOp::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankInt8MulOp op = *this; - rewriter.create(op.getLoc(), op.getGateBank(), op.getInputBank(), - op.getOutputBank(), op.getIter(), op.getRatio(), - op.getGateRow()); - rewriter.replaceOp(op, op.getOutputBank()); - return success(); -} diff --git a/waste/examples/components/balls/int8mul/stack/src/Dialect/Buckyball/Int8MulBall.td b/waste/examples/components/balls/int8mul/stack/src/Dialect/Buckyball/Int8MulBall.td deleted file mode 100644 index b4451aa67..000000000 --- a/waste/examples/components/balls/int8mul/stack/src/Dialect/Buckyball/Int8MulBall.td +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef INT8MULBALL_OPS -#define INT8MULBALL_OPS - -include "Buckyball.td" - -def Int8MulOp : Buckyball_Op<"int8mul"> { - let summary = "Scaled INT8 per-channel broadcast multiply on physical banks"; - let arguments = (ins I64:$gateBankId, I64:$inputBankId, I64:$outputBankId, - I64:$iter, F32:$ratio, I64:$gateRow); - let assemblyFormat = "$gateBankId `,` $inputBankId `,` $outputBankId `,` " - "$iter `,` $ratio `,` $gateRow attr-dict `:` " - "type($gateBankId)"; -} - -def BankInt8MulOp : Buckyball_Op<"bank_int8mul", [DeclareOpInterfaceMethods]> { - let summary = "Scaled INT8 per-channel broadcast multiply on virtual banks"; - let arguments = (ins I64:$gateBank, I64:$inputBank, I64:$outputBank, - I64:$iter, F32:$ratio, I64:$gateRow); - let results = (outs I64:$outputBankOut); - let assemblyFormat = "$gateBank $inputBank $outputBank $iter $ratio " - "$gateRow attr-dict `:` type($gateBank) " - "type($inputBank) type($outputBank) type($iter) " - "type($ratio) type($gateRow)"; -} - -#endif // INT8MULBALL_OPS diff --git a/waste/examples/components/balls/int8mul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/int8mul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index bcde87217..000000000 --- a/waste/examples/components/balls/int8mul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -struct Int8MulLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(Int8MulOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getGateBankId(), - adaptor.getInputBankId(), - adaptor.getOutputBankId(), adaptor.getIter()); - Value ratioBits = rewriter.create( - loc, rewriter.getI32Type(), adaptor.getRatio()); - Value ratio = - rewriter.create(loc, rewriter.getI64Type(), ratioBits); - Value gateRow = rewriter.create(loc, adaptor.getGateRow(), - cstI64(rewriter, loc, 32)); - Value rs2 = rewriter.create(loc, ratio, gateRow); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("INT8MUL"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateInt8MulBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { - patterns.add(converter); -} - -void configureInt8MulBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/int8mul/stack/src/Passes.cpp b/waste/examples/components/balls/int8mul/stack/src/Passes.cpp deleted file mode 100644 index 8008e2d09..000000000 --- a/waste/examples/components/balls/int8mul/stack/src/Passes.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- Passes.cpp - Int8MulBall passes ------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -namespace buckyball { -void populateInt8MulBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureInt8MulBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LowerInt8MulPass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerInt8MulPass) - StringRef getArgument() const final { return "lower-int8mul-ball"; } - StringRef getDescription() const final { - return "Lower Int8MulBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateInt8MulBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureInt8MulBallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - return true; -}(); diff --git a/waste/examples/components/balls/lut/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/lut/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index 759859bf7..000000000 --- a/waste/examples/components/balls/lut/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace mlir::buddy; -using namespace ::buddy::buckyball; - -LogicalResult -BankLutOp::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &state) { - BankLutOp op = *this; - auto input = state.getSlot(op.getInBank()); - auto lut = state.getSlot(op.getLutBank()); - auto output = state.getSlot(op.getOutBank()); - if (!input || !lut || !output) - return failure(); - if (input->row != 1 || input->col != 1 || output->row != 1 || - output->col != 1 || lut->row != 1 || (lut->col != 1 && lut->col != 4)) - return op.emitError( - "LUT requires col=1 input/output and col=1 or col=4 table"); - auto overlaps = [](const BankSlot &lhs, const BankSlot &rhs) { - return lhs.base < rhs.base + rhs.row * rhs.col && - rhs.base < lhs.base + lhs.row * lhs.col; - }; - if (overlaps(*input, *lut) || overlaps(*input, *output) || - overlaps(*lut, *output)) - return op.emitError("LUT bank groups must not overlap"); - rewriter.create(op.getLoc(), op.getInBank(), op.getLutBank(), - op.getOutBank(), op.getIter()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); -} diff --git a/waste/examples/components/balls/lut/stack/src/Dialect/Buckyball/LutBall.td b/waste/examples/components/balls/lut/stack/src/Dialect/Buckyball/LutBall.td deleted file mode 100644 index d119ceadf..000000000 --- a/waste/examples/components/balls/lut/stack/src/Dialect/Buckyball/LutBall.td +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef LUTBALL_OPS -#define LUTBALL_OPS - -include "Buckyball.td" - -def LutOp : Buckyball_Op<"lut"> { - let summary = "Apply a shared or 16-lane INT8 lookup table"; - let arguments = (ins I64:$inputBankId, I64:$lutBankId, - I64:$outputBankId, I64:$iter); - let assemblyFormat = "$inputBankId `,` $lutBankId `,` $outputBankId `,` " - "$iter attr-dict `:` type($inputBankId)"; -} - -def BankLutOp : Buckyball_Op<"bank_lut", [DeclareOpInterfaceMethods]> { - let summary = "Apply a shared or 16-lane INT8 lookup table with virtual banks"; - let arguments = (ins I64:$inBank, I64:$lutBank, I64:$outBank, I64:$iter); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$inBank $lutBank $outBank $iter attr-dict `:` " - "type($inBank) type($lutBank) type($outBank) type($iter)"; -} - -#endif // LUTBALL_OPS diff --git a/waste/examples/components/balls/lut/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/lut/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index e56f4f021..000000000 --- a/waste/examples/components/balls/lut/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,60 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -struct LutLowering : public ConvertOpToLLVMPattern { - LutLowering(LLVMTypeConverter &converter, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(LutOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - auto iterOp = op.getIter().getDefiningOp(); - auto iterAttr = - iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); - if (!iterAttr || iterAttr.getInt() <= 0 || iterAttr.getInt() > bankDepth) - return op.emitError( - "LUT iter must be a positive constant within one bank"); - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - adaptor.getLutBankId(), - adaptor.getOutputBankId(), adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, cstI64(rewriter, loc, 0), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("LUT"))); - return success(); - } - -private: - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateLutBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns, - bool stable, - int64_t bankDepth) { - (void)stable; - - patterns.add(converter, bankDepth); -} - -void configureLutBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/lut/stack/src/Passes.cpp b/waste/examples/components/balls/lut/stack/src/Passes.cpp deleted file mode 100644 index 30ac9667d..000000000 --- a/waste/examples/components/balls/lut/stack/src/Passes.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- Passes.cpp - LutBall passes ----------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -namespace buckyball { -void populateLutBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureLutBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LowerLutPass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerLutPass) - StringRef getArgument() const final { return "lower-lut-ball"; } - StringRef getDescription() const final { - return "Lower LutBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateLutBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureLutBallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - return true; -}(); diff --git a/waste/examples/components/balls/matadd/stack/src/Backend/include/IntrinsicsRISCVMatAddBall.td b/waste/examples/components/balls/matadd/stack/src/Backend/include/IntrinsicsRISCVMatAddBall.td deleted file mode 100644 index 125b1e629..000000000 --- a/waste/examples/components/balls/matadd/stack/src/Backend/include/IntrinsicsRISCVMatAddBall.td +++ /dev/null @@ -1 +0,0 @@ -// MatAddBall uses CustomIntrOp with funct7 from the target ballISA. diff --git a/waste/examples/components/balls/matadd/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/matadd/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index 7b4531ce9..000000000 --- a/waste/examples/components/balls/matadd/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankMatAddOp::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankMatAddOp op = *this; - rewriter.create(op.getLoc(), op.getABank(), op.getBBank(), - op.getCBank(), op.getIter()); - rewriter.replaceOp(op, op.getCBank()); - return success(); -} diff --git a/waste/examples/components/balls/matadd/stack/src/Dialect/Buckyball/MatAddBall.td b/waste/examples/components/balls/matadd/stack/src/Dialect/Buckyball/MatAddBall.td deleted file mode 100644 index a59ad403c..000000000 --- a/waste/examples/components/balls/matadd/stack/src/Dialect/Buckyball/MatAddBall.td +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef MATADDBALL_OPS -#define MATADDBALL_OPS - -include "Buckyball.td" - -def MatAddOp : Buckyball_Op<"matadd"> { - let summary = "Add matching INT32 physical-bank lines"; - let arguments = (ins I64:$aBankId, I64:$bBankId, I64:$cBankId, I64:$iter); - let assemblyFormat = "$aBankId `,` $bBankId `,` $cBankId `,` $iter attr-dict `:` " - "type($aBankId)"; -} - -def BankMatAddOp : Buckyball_Op<"bank_matadd", [DeclareOpInterfaceMethods]> { - let summary = "Add matching INT32 virtual-bank lines"; - let arguments = (ins I64:$aBank, I64:$bBank, I64:$cBank, I64:$iter); - let results = (outs I64:$cBankOut); - let assemblyFormat = "$aBank $bBank $cBank $iter attr-dict `:` " - "type($aBank) type($bBank) type($cBank) type($iter)"; -} - -#endif // MATADDBALL_OPS diff --git a/waste/examples/components/balls/matadd/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/matadd/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 1a4822e0d..000000000 --- a/waste/examples/components/balls/matadd/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -struct MatAddLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(MatAddOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getABankId(), - adaptor.getBBankId(), adaptor.getCBankId(), - adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, cstI64(rewriter, loc, 0), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MATADD"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateMatAddBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { - patterns.add(converter); -} - -void configureMatAddBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/matadd/stack/src/Passes.cpp b/waste/examples/components/balls/matadd/stack/src/Passes.cpp deleted file mode 100644 index f473d98b1..000000000 --- a/waste/examples/components/balls/matadd/stack/src/Passes.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- Passes.cpp - MatAddBall passes -------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -namespace buckyball { -void populateMatAddBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureMatAddBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LowerMatAddPass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerMatAddPass) - StringRef getArgument() const final { return "lower-matadd-ball"; } - StringRef getDescription() const final { - return "Lower MatAddBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateMatAddBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureMatAddBallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - return true; -}(); diff --git a/waste/examples/components/balls/maxpool/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/maxpool/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index a2b949c60..000000000 --- a/waste/examples/components/balls/maxpool/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankMaxPoolOp::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankMaxPoolOp op = *this; - rewriter.create( - op.getLoc(), op.getInBank(), op.getOutBank(), op.getIter(), - op.getInputSideAttr(), op.getOutputSideAttr(), op.getKernelAttr(), - op.getStrideAttr(), op.getPaddingAttr(), op.getInputBase(), - op.getOutputBase(), op.getOutputStride(), op.getStartRowAttr(), - op.getStartColAttr()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); -} diff --git a/waste/examples/components/balls/maxpool/stack/src/Dialect/Buckyball/MaxPoolBall.td b/waste/examples/components/balls/maxpool/stack/src/Dialect/Buckyball/MaxPoolBall.td deleted file mode 100644 index ed3cef778..000000000 --- a/waste/examples/components/balls/maxpool/stack/src/Dialect/Buckyball/MaxPoolBall.td +++ /dev/null @@ -1,45 +0,0 @@ -//===-- MaxPoolBall.td - MaxPoolBall operations ------------*- tablegen -*-===// - -#ifndef MAXPOOLBALL_OPS -#define MAXPOOLBALL_OPS - -include "Buckyball.td" - -def MaxPoolOp : Buckyball_Op<"maxpool"> { - let summary = "Signed INT8 spatial max pooling"; - let description = [{ - Each bank row contains the 16 channels of one spatial position. The input - and output tiles are square. inputBase selects the first input row. - outputBase and outputStride place the result inside a larger resident - feature-map tile, preserving destination rows outside the result. - iter is the number of output spatial positions. - }]; - let arguments = (ins I64:$inputBankId, I64:$outputBankId, I64:$iter, - I64Attr:$inputSide, I64Attr:$outputSide, - I64Attr:$kernel, I64Attr:$stride, I64Attr:$padding, - I64:$inputBase, I64:$outputBase, - I64:$outputStride, I64Attr:$startRow, - I64Attr:$startCol); - let assemblyFormat = "$inputBankId `,` $outputBankId `,` $iter `,` " - "$inputBase `,` $outputBase `,` $outputStride " - "prop-dict attr-dict `:` type($inputBankId) type($outputBankId) " - "type($iter) type($inputBase) type($outputBase) " - "type($outputStride)"; -} - -def BankMaxPoolOp : Buckyball_Op<"bank_maxpool", [DeclareOpInterfaceMethods]> { - let summary = "Signed INT8 spatial max pooling with virtual banks"; - let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, - I64Attr:$inputSide, I64Attr:$outputSide, - I64Attr:$kernel, I64Attr:$stride, I64Attr:$padding, - I64:$inputBase, I64:$outputBase, - I64:$outputStride, I64Attr:$startRow, - I64Attr:$startCol); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$inBank $outBank $iter $inputBase $outputBase " - "$outputStride prop-dict attr-dict `:` type($inBank) " - "type($outBank) type($iter) type($inputBase) " - "type($outputBase) type($outputStride)"; -} - -#endif // MAXPOOLBALL_OPS diff --git a/waste/examples/components/balls/maxpool/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/maxpool/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 56a78af40..000000000 --- a/waste/examples/components/balls/maxpool/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,113 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/IR/PatternMatch.h" -#include - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -struct MaxPoolLowering : public ConvertOpToLLVMPattern { - MaxPoolLowering(LLVMTypeConverter &converter, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(MaxPoolOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - int64_t inputSide = op.getInputSide(); - int64_t outputSide = op.getOutputSide(); - int64_t kernel = op.getKernel(); - int64_t stride = op.getStride(); - int64_t padding = op.getPadding(); - int64_t startRow = op.getStartRow(); - int64_t startCol = op.getStartCol(); - auto iterOp = op.getIter().getDefiningOp(); - auto iterAttr = - iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); - if (!iterAttr || inputSide <= 0 || inputSide > 15 || outputSide <= 0 || - outputSide > 15 || kernel <= 0 || kernel > 15 || stride <= 0 || - stride > 15 || padding < 0 || padding > 15 || startRow < 0 || - startRow > 15 || startCol < 0 || startCol > 15) - return op.emitError("MaxPool shape does not fit one physical bank tile"); - - int64_t lastSourceRow = startRow + (outputSide - 1) * stride + kernel - 1; - int64_t lastSourceCol = startCol + (outputSide - 1) * stride + kernel - 1; - int64_t inputEnd = padding + inputSide; - bool readsInput = startRow < inputEnd && startCol < inputEnd && - lastSourceRow >= padding && lastSourceCol >= padding; - int64_t inputFootprint = 0; - if (readsInput) { - int64_t maxInputRow = std::min(lastSourceRow, inputEnd - 1) - padding; - int64_t maxInputCol = std::min(lastSourceCol, inputEnd - 1) - padding; - inputFootprint = maxInputRow * inputSide + maxInputCol + 1; - } - if (inputFootprint > bankDepth || - outputSide * outputSide != iterAttr.getInt() || - inputSide + 2 * padding < kernel + startRow || - inputSide + 2 * padding < kernel + startCol || - startRow + (outputSide - 1) * stride + kernel > - inputSide + 2 * padding || - startCol + (outputSide - 1) * stride + kernel > inputSide + 2 * padding) - return op.emitError("MaxPool shape does not fit one physical bank tile"); - - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - rs1 = - rewriter.create(loc, rs1, cstI64(rewriter, loc, INT64_MIN)); - uint64_t geometry = static_cast(inputSide) | - (static_cast(outputSide) << 4) | - (static_cast(kernel) << 8) | - (static_cast(stride) << 12) | - (static_cast(padding) << 16) | - (static_cast(startRow) << 38) | - (static_cast(startCol) << 42); - Value rs2 = cstI64(rewriter, loc, geometry); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getInputBase(), - cstI64(rewriter, loc, 20))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getOutputBase(), - cstI64(rewriter, loc, 26))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getOutputStride(), - cstI64(rewriter, loc, 32))); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MAXPOOL"))); - return success(); - } - -private: - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateMaxPoolBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - (void)stable; - - patterns.add(converter, bankDepth); -} - -void configureMaxPoolBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/maxpool/stack/src/Passes.cpp b/waste/examples/components/balls/maxpool/stack/src/Passes.cpp deleted file mode 100644 index 70136ea2d..000000000 --- a/waste/examples/components/balls/maxpool/stack/src/Passes.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- Passes.cpp - MaxPoolBall passes ------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -namespace buckyball { -void populateMaxPoolBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureMaxPoolBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LowerMaxPoolPass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerMaxPoolPass) - StringRef getArgument() const final { return "lower-maxpool-ball"; } - StringRef getDescription() const final { - return "Lower MaxPoolBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateMaxPoolBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureMaxPoolBallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - return true; -}(); diff --git a/waste/examples/components/balls/mxfp2int/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/mxfp2int/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index 7c140d4dd..000000000 --- a/waste/examples/components/balls/mxfp2int/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankMxfp2IntOp::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankMxfp2IntOp op = *this; - rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), - op.getIter(), op.getSpecial()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); -} diff --git a/waste/examples/components/balls/mxfp2int/stack/src/Dialect/Buckyball/Mxfp2IntBall.td b/waste/examples/components/balls/mxfp2int/stack/src/Dialect/Buckyball/Mxfp2IntBall.td deleted file mode 100644 index 689f71f66..000000000 --- a/waste/examples/components/balls/mxfp2int/stack/src/Dialect/Buckyball/Mxfp2IntBall.td +++ /dev/null @@ -1,26 +0,0 @@ -//===-- Mxfp2IntBall.td - MXFP to INT8 dequantization operations --------===// - -#ifndef MXFP2INT_BALL_OPS -#define MXFP2INT_BALL_OPS - -include "Buckyball.td" - -def Mxfp2IntOp : Buckyball_Op<"mxfp2int"> { - let summary = "Dequantize MXFP blocks to INT8"; - let description = [{ - Reads MXFP blocks from inputBankId, writes two INT8 rows per block to - outputBankId, and uses special as rs2 for format and rounding controls. - }]; - let arguments = (ins I64:$inputBankId, I64:$outputBankId, I64:$iter, - I64:$special); - let assemblyFormat = "$inputBankId `,` $outputBankId `,` $iter `,` $special attr-dict `:` type($inputBankId)"; -} - -def BankMxfp2IntOp : Buckyball_Op<"bank_mxfp2int", [DeclareOpInterfaceMethods]> { - let summary = "MXFP to INT8 dequantization using bank-SSA handles"; - let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, I64:$special); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$inBank $outBank $iter $special attr-dict `:` type($inBank) type($outBank) type($iter) type($special)"; -} - -#endif // MXFP2INT_BALL_OPS diff --git a/waste/examples/components/balls/mxfp2int/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/mxfp2int/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 9b9b21ea6..000000000 --- a/waste/examples/components/balls/mxfp2int/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -class Mxfp2IntLowering : public ConvertOpToLLVMPattern { -public: - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(Mxfp2IntOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getSpecial(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MXFP2INT"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateMxfp2IntBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - (void)stable; - (void)bankDepth; - - patterns.add(converter); -} - -void configureMxfp2IntBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/mxfp2int/stack/src/Passes.cpp b/waste/examples/components/balls/mxfp2int/stack/src/Passes.cpp deleted file mode 100644 index c1a285ac1..000000000 --- a/waste/examples/components/balls/mxfp2int/stack/src/Passes.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- Passes.cpp - Mxfp2IntBall passes -----------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -namespace buckyball { -void populateMxfp2IntBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, - bool, int64_t); -void configureMxfp2IntBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LowerMxfp2IntPass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerMxfp2IntPass) - StringRef getArgument() const final { return "lower-mxfp2int-ball"; } - StringRef getDescription() const final { - return "Lower Mxfp2IntBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateMxfp2IntBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureMxfp2IntBallLegalizeForExportTarget( - target, stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - return true; -}(); diff --git a/waste/examples/components/balls/mxmm/stack/python/fp32.py b/waste/examples/components/balls/mxmm/stack/python/fp32.py deleted file mode 100644 index b6b200ca9..000000000 --- a/waste/examples/components/balls/mxmm/stack/python/fp32.py +++ /dev/null @@ -1,66 +0,0 @@ -import array - -from buddy_mlir import ir - - -def lower(node, symbols): - lhs, rhs = [symbols[(name, 0)] for name in node.args] - result = ir.RankedTensorType.get(node.tensor_meta["shape"], ir.F32Type.get()) - attributes = {"fused": ir.BoolAttr.get(node.fused)} - owner = rhs.owner - if isinstance(owner, ir.OpView): - owner = owner.operation - reshape = None - if isinstance(owner, ir.Operation) and owner.name == "tosa.reshape": - original = ir.RankedTensorType(owner.operands[0].type) - current = ir.RankedTensorType(rhs.type) - if ( - original.rank == 4 - and original.shape[0] == 1 - and list(current.shape) == list(original.shape[1:]) - ): - reshape = current - owner = owner.operands[0].owner - if isinstance(owner, ir.OpView): - owner = owner.operation - if isinstance(owner, ir.Operation) and owner.name == "tosa.transpose": - rank = ir.RankedTensorType(owner.operands[0].type).rank - permutation = list(ir.DenseI32ArrayAttr(owner.attributes["perms"])) - expected = list(range(rank)) - expected[-2:] = [rank - 1, rank - 2] - if permutation == expected: - rhs = owner.operands[0] - if reshape is not None: - shape = list(reshape.shape) - shape[-2:] = [shape[-1], shape[-2]] - tensor = ir.RankedTensorType.get(shape, ir.F32Type.get()) - shape_type = ir.Type.parse(f"!tosa.shape<{len(shape)}>") - dimensions = ir.Operation.create( - "tosa.const_shape", - results=[shape_type], - attributes={ - "values": ir.DenseElementsAttr.get( - array.array("q", shape), - type=ir.IndexType.get(), - shape=[len(shape)], - ) - }, - ).result - rhs = ir.Operation.create( - "tosa.reshape", operands=[rhs, dimensions], results=[tensor] - ).result - attributes["rhs_transposed"] = ir.UnitAttr.get() - return ir.Operation.create( - "buckyball.fp32_matmul", - operands=[lhs, rhs], - results=[result], - attributes=attributes, - ).result - - -def apply(graph, *, fused): - for node in graph._body: - if node.__class__.__name__ in ("MatmulOp", "BatchMatmulOp"): - node.fused = fused - graph._ops_registry["MatmulOp"] = lower - graph._ops_registry["BatchMatmulOp"] = lower diff --git a/waste/examples/components/balls/mxmm/stack/python/grouped_attention.py b/waste/examples/components/balls/mxmm/stack/python/grouped_attention.py deleted file mode 100644 index bd4a38a27..000000000 --- a/waste/examples/components/balls/mxmm/stack/python/grouped_attention.py +++ /dev/null @@ -1,107 +0,0 @@ -import array -import math - -from buddy_mlir import ir - -from . import fp32 - - -def reshape(value, shape): - shape_type = ir.Type.parse(f"!tosa.shape<{len(shape)}>") - dimensions = ir.Operation.create( - "tosa.const_shape", - results=[shape_type], - attributes={ - "values": ir.DenseElementsAttr.get( - array.array("q", shape), type=ir.IndexType.get(), shape=[len(shape)] - ) - }, - ).result - result_type = ir.RankedTensorType.get(shape, ir.F32Type.get()) - return ir.Operation.create( - "tosa.reshape", operands=[value, dimensions], results=[result_type] - ).result - - -def lower(node, symbols): - result = fp32.lower(node, symbols) - native = result.owner - if isinstance(native, ir.OpView): - native = native.operation - lhs, rhs = native.operands - repeated = rhs - while True: - owner = repeated.owner - if isinstance(owner, ir.OpView): - owner = owner.operation - if not isinstance(owner, ir.Operation) or owner.name != "tosa.reshape": - break - repeated = owner.operands[0] - # Other matmuls keep their native lowering. This pattern is a five-axis KV repeat. - repeated_type = ir.RankedTensorType(repeated.type) - if ( - not isinstance(owner, ir.Operation) - or owner.name != "tosa.add" - or repeated_type.rank != 5 - ): - return result - source, zeros = owner.operands - zero_op = zeros.owner - if isinstance(zero_op, ir.OpView): - zero_op = zero_op.operation - if not isinstance(zero_op, ir.Operation) or zero_op.name != "tosa.const": - raise ValueError("grouped attention requires an explicit +0 KV broadcast") - zero_values = ir.DenseElementsAttr(zero_op.attributes["values"]) - if not zero_values.is_splat: - raise ValueError("grouped attention KV broadcast must contain only +0") - zero = ir.FloatAttr(zero_values.get_splat_value()).value - if zero != 0.0 or math.copysign(1.0, zero) != 1.0: - raise ValueError("grouped attention KV broadcast must contain only +0") - if ir.BoolAttr(native.attributes["fused"]).value: - raise ValueError("grouped attention requires unfused FP32 accumulation") - if any(size <= 0 for size in repeated_type.shape): - raise ValueError("grouped attention requires positive static dimensions") - batch, kv_heads, groups, length, dimension = repeated_type.shape - source_shape = list(ir.RankedTensorType(source.type).shape) - left_shape = list(ir.RankedTensorType(lhs.type).shape) - right_shape = list(ir.RankedTensorType(rhs.type).shape) - transposed = "rhs_transposed" in native.attributes - reduction, columns = (dimension, length) if transposed else (length, dimension) - if ( - source_shape != [batch, kv_heads, 1, length, dimension] - or len(left_shape) != 3 - or left_shape[0] != batch * kv_heads * groups - or left_shape[2] != reduction - or right_shape != [batch * kv_heads * groups, length, dimension] - ): - raise ValueError( - "grouped attention requires contiguous head-major Q/P and KV shapes" - ) - rows = left_shape[1] - if rows <= 0: - raise ValueError("grouped attention requires a positive static row count") - grouped_left = reshape(lhs, [batch * kv_heads, groups * rows, reduction]) - grouped_right = reshape(source, [batch * kv_heads, length, dimension]) - attributes = {"fused": ir.BoolAttr.get(False)} - if transposed: - attributes["rhs_transposed"] = ir.UnitAttr.get() - grouped_type = ir.RankedTensorType.get( - [batch * kv_heads, groups * rows, columns], ir.F32Type.get() - ) - grouped = ir.Operation.create( - "buckyball.fp32_matmul", - operands=[grouped_left, grouped_right], - results=[grouped_type], - attributes=attributes, - ).result - shape = list(ir.RankedTensorType(result.type).shape) - native.erase() - # With finite operands, RNE and initial +0, removing the repeat's +0 changes - # only signed-zero products, which cannot change an unfused dot's result. - return reshape(grouped, shape) - - -def apply(graph): - fp32.apply(graph, fused=False) - graph._ops_registry["MatmulOp"] = lower - graph._ops_registry["BatchMatmulOp"] = lower diff --git a/waste/examples/components/balls/mxmm/stack/python/layout.py b/waste/examples/components/balls/mxmm/stack/python/layout.py deleted file mode 100644 index a1f3559a2..000000000 --- a/waste/examples/components/balls/mxmm/stack/python/layout.py +++ /dev/null @@ -1,77 +0,0 @@ -import math - -import torch - - -def plan(rows, columns, reduction, bank_bytes, *, mxfp8): - alignment = 32 if mxfp8 else 4 - if min(rows, columns, reduction, bank_bytes) <= 0 or reduction % alignment: - raise ValueError("matmul shape violates the format alignment") - row_limit = 1 if rows == 1 else (rows + 15) // 16 * 16 - column_limit = (columns + 15) // 16 * 16 - candidates = [] - for m in ([1] if rows == 1 else range(16, row_limit + 1, 16)): - for n in range(16, column_limit + 1, 16): - if m * n * 4 > bank_bytes: - continue - factor = 33 / 32 if mxfp8 else 4 - k = min(reduction, int(bank_bytes / (max(m, n) * factor))) - k = k // alignment * alignment - if not k: - continue - panels = ( - math.ceil(rows / m) * math.ceil(columns / n) * math.ceil(reduction / k) - ) - quant_panels = ( - math.ceil(rows / m) * math.ceil(reduction / k) if mxfp8 else 0 - ) - candidates.append(((panels + quant_panels, panels, -k, -m, -n), (m, n, k))) - if not candidates: - raise ValueError("matmul cannot fit an aligned block in its banks") - m, n, k = min(candidates)[1] - return { - "tile_m": m, - "tile_n": n, - "tile_k": k, - "bank_bytes": bank_bytes, - "panel_stride": bank_bytes, - } - - -def pack(codes, scales, layout): - if ( - not isinstance(codes, torch.Tensor) - or not isinstance(scales, torch.Tensor) - or codes.dtype != torch.int8 - or scales.dtype != torch.int8 - ): - raise ValueError("MXFP8 packing requires INT8 Tensor codes and scales") - if codes.ndim != 2 or scales.device != codes.device: - raise ValueError("MXFP8 packing requires a matrix and matching devices") - rows, width = codes.shape - n, k = layout["tile_n"], layout["tile_k"] - stride = layout["panel_stride"] - if width % 32 or scales.shape != (rows, width // 32): - raise ValueError("MXFP8 codes and scales disagree") - panels = [] - for begin_row in range(0, rows, n): - for begin_k in range(0, width, k): - count = min(k, width - begin_k) - plane = torch.zeros((n, count), dtype=torch.int8, device=codes.device) - scale = torch.full( - (n, count // 32), 127, dtype=torch.int8, device=codes.device - ) - valid = min(n, rows - begin_row) - plane[:valid] = codes[ - begin_row : begin_row + valid, begin_k : begin_k + count - ] - scale[:valid] = scales[ - begin_row : begin_row + valid, begin_k // 32 : (begin_k + count) // 32 - ] - payload = torch.cat((plane.flatten(), scale.flatten())) - if payload.numel() > stride: - raise ValueError("MXFP8 panel exceeds its bank window") - panels.append( - torch.nn.functional.pad(payload, (0, stride - payload.numel())) - ) - return torch.cat(panels) diff --git a/waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index 58fb2fb38..000000000 --- a/waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,39 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "mlir/IR/PatternMatch.h" -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankMXFP8Op::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankMXFP8Op op = *this; - rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), - op.getWrBank(), op.getConfig(), op.getFirst(), - op.getLast(), op.getOutputBase()); - rewriter.replaceOp(op, op.getWrBank()); - return success(); -} - -LogicalResult -BankMXFP8WindowOp::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankMXFP8WindowOp op = *this; - rewriter.create( - op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), op.getWrBank(), - op.getRows(), op.getCols(), op.getCount(), op.getFullK(), op.getStartK(), - op.getFirst(), op.getLast(), op.getOutputBase()); - rewriter.replaceOp(op, op.getWrBank()); - return success(); -} - -LogicalResult -BankFP32Op::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankFP32Op op = *this; - rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), - op.getWrBank(), op.getConfig(), op.getFirst(), - op.getLast(), op.getOutputBase(), op.getFusedAttr()); - rewriter.replaceOp(op, op.getWrBank()); - return success(); -} diff --git a/waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp b/waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp deleted file mode 100644 index a70ca1ad9..000000000 --- a/waste/examples/components/balls/mxmm/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp +++ /dev/null @@ -1,672 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include -#include -#include -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult FP32MemMatmulOp::lowerToBankSSA(PatternRewriter &b) { - FP32MemMatmulOp op = *this; - auto a = cast(op.getLhs().getType()), - w = cast(op.getRhs().getType()), - out = cast(op.getOutput().getType()); - int rank = a.getRank(); - if ((rank != 2 && rank != 3) || w.getRank() != rank || - out.getRank() != rank || !a.hasStaticShape() || !w.hasStaticShape() || - !out.hasStaticShape() || !a.getElementType().isF32() || - !w.getElementType().isF32() || !out.getElementType().isF32()) - return op.emitError( - "FP32 matmul requires static rank-2 or rank-3 FP32 operands"); - int64_t M = a.getShape()[rank - 2], K = a.getShape()[rank - 1], - N = w.getShape()[rank - 1], batches = rank == 3 ? a.getShape()[0] : 1; - if (M <= 0 || N <= 0 || K <= 0 || K % 4 || w.getShape()[rank - 2] != K || - out.getShape()[rank - 2] != M || out.getShape()[rank - 1] != N || - (rank == 3 && - (w.getShape()[0] != batches || out.getShape()[0] != batches))) - return op.emitError( - "FP32 matmul shape mismatch or K not divisible by four"); - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128) - return op.emitError("FP32 matmul requires 128-bit bank rows"); - int64_t bytes = target.bankDepth * 16, rows = 0, columns = 0, chunk = 0; - std::tuple best{INT64_MAX, 0, 0, 0}; - for (int64_t m = M == 1 ? 1 : 16; m <= (M == 1 ? 1 : (M + 15) / 16 * 16); - m += 16) - for (int64_t n = 16; n <= (N + 15) / 16 * 16; n += 16) { - if (m * n * 4 > bytes) - continue; - int64_t k = std::min(K, bytes / (4 * std::max(m, n))) / 4 * 4; - if (!k) - continue; - auto score = - std::tuple{((M + m - 1) / m) * ((N + n - 1) / n) * ((K + k - 1) / k), - -k, -m, -n}; - if (score < best) { - best = score; - rows = m; - columns = n; - chunk = k; - } - } - if (!rows) - return op.emitError("FP32 matmul cannot fit a bank block"); - SmallVector rhsStrides; - int64_t rhsOffset; - bool bankPack = target.bankNum >= 5 && target.rvvEnabled && - succeeded(w.getStridesAndOffset(rhsStrides, rhsOffset)) && - rhsStrides[rank - 2] == N && rhsStrides.back() == 1 && - K * N * 4 <= bytes && chunk == K && rows * columns * 4 >= 64; - Location loc = op.getLoc(); - auto idx = [&](int64_t v) -> Value { - return b.create(loc, v); - }; - Value zero = idx(0), one = idx(1), four = idx(4); - Value ab = allocBank(b, loc, 1, 1), wb = allocBank(b, loc, 1, 1), - cb = allocBank(b, loc, 1, 1); - auto packType = MemRefType::get({bytes / 16, 4}, b.getF32Type()); - Value ap = b.create(loc, packType), - bp = b.create(loc, packType), - cp = b.create(loc, packType); - for (Value buffer : {ap, bp, cp}) - buffer.getDefiningOp()->setAttr("alignment", b.getI64IntegerAttr(bytes)); - SmallVector banks{ab, wb, cb}; - if (bankPack) { - Value input = allocBank(b, loc, 1, 1); - Value descriptor = allocBank(b, loc, 1, 1); - Value read = b.create(loc, b.getI64Type(), ab, input); - Value write = b.create(loc, b.getI64Type(), wb, descriptor); - write = b.create(loc, b.getI64Type(), cb, write); - auto loaded = b.create( - loc, b.getI64Type(), b.getI64Type(), read, write, - FlatSymbolRefAttr::get(b.getContext(), "rvv_packmm"), - ValueRange{createI64Const(b, loc, 0), createI64Const(b, loc, K), - createI64Const(b, loc, N), createI64Const(b, loc, 0), - createI64Const(b, loc, 0), createI64Const(b, loc, K), - createI64Const(b, loc, columns), - createI64Const(b, loc, rows), createI64Const(b, loc, 1), - createI64Const(b, loc, 1), - createI64Const(b, loc, op.getFusedAttr().getValue())}); - banks = {loaded.getReadOut(), loaded.getWriteOut()}; - } - auto batchLoop = b.create(loc, zero, idx(batches), one, banks); - b.setInsertionPointToStart(batchLoop.getBody()); - Value batch = batchLoop.getInductionVar(); - SmallVector batchBanks(batchLoop.getRegionIterArgs()); - if (bankPack) { - SmallVector offsets(rank, b.getIndexAttr(0)); - SmallVector sizes; - for (int64_t dim : w.getShape()) - sizes.push_back(b.getIndexAttr(dim)); - SmallVector strides(rank, b.getIndexAttr(1)); - if (rank == 3) { - offsets[0] = batch; - sizes[0] = b.getIndexAttr(1); - } - auto rowType = memref::SubViewOp::inferRankReducedResultType( - {K, N}, w, offsets, sizes, strides); - Value source = b.create(loc, rowType, op.getRhs(), - offsets, sizes, strides); - Value flat = b.create( - loc, source, ArrayRef{{0, 1}}); - Value packed = b.create( - loc, ArrayRef{K * N / 4, 4}, flat, - ArrayRef{{0, 1}}); - batchBanks[0] = - b.create(loc, b.getI64Type(), packed, batchBanks[0], - createI64Const(b, loc, K * N / 4), - createI64Const(b, loc, 1), b.getI64IntegerAttr(0)); - } - auto rowLoop = b.create(loc, zero, idx(M), idx(rows), batchBanks); - b.setInsertionPointToStart(rowLoop.getBody()); - Value row = rowLoop.getInductionVar(); - auto colLoop = b.create(loc, zero, idx(N), idx(columns), - rowLoop.getRegionIterArgs()); - b.setInsertionPointToStart(colLoop.getBody()); - Value column = colLoop.getInductionVar(); - ab = colLoop.getRegionIterArgs()[0]; - wb = colLoop.getRegionIterArgs()[1]; - cb = bankPack ? wb : colLoop.getRegionIterArgs()[2]; - for (int64_t begin = 0; begin < K; begin += chunk) { - int64_t count = std::min(chunk, K - begin); - for (int operand = 0; operand < (bankPack ? 1 : 2); ++operand) { - Value source = operand ? op.getRhs() : op.getLhs(); - Value packed = operand ? bp : ap; - auto sourceType = cast(source.getType()); - int axis = operand ? rank - 2 : rank - 1; - SmallVector strides; - int64_t offset; - if (failed(sourceType.getStridesAndOffset(strides, offset))) - return op.emitError("FP32 matmul requires strided operands"); - int64_t blockRows = operand ? columns : rows; - int64_t logicalRows = operand ? N : M; - Value base = operand ? column : row; - Value validRows = b.create( - loc, idx(blockRows), - b.create(loc, idx(logicalRows), base)); - auto flatType = MemRefType::get({bytes / 4}, b.getF32Type()); - Value flatBuffer = b.create( - loc, flatType, packed, ArrayRef{{0, 1}}); - auto rLoop = b.create(loc, zero, validRows, one); - b.setInsertionPointToStart(rLoop.getBody()); - Value r = rLoop.getInductionVar(); - Value global = b.create(loc, base, r); - Value flat = b.create(loc, r, idx(count)); - if (strides[axis] == 1) { - SmallVector sourceOffsets(rank, b.getIndexAttr(0)); - SmallVector sourceSizes(rank, b.getIndexAttr(1)); - SmallVector sourceStrides(rank, b.getIndexAttr(1)); - if (rank == 3) - sourceOffsets[0] = batch; - sourceOffsets[rank - 2] = operand ? OpFoldResult(b.getIndexAttr(begin)) - : OpFoldResult(global); - sourceOffsets[rank - 1] = operand ? OpFoldResult(global) - : OpFoldResult(b.getIndexAttr(begin)); - sourceSizes[axis] = b.getIndexAttr(count); - auto sourceRowType = memref::SubViewOp::inferRankReducedResultType( - {count}, sourceType, sourceOffsets, sourceSizes, sourceStrides); - Value sourceRow = b.create( - loc, sourceRowType, source, sourceOffsets, sourceSizes, - sourceStrides); - SmallVector targetOffsets{flat}; - SmallVector targetSizes{b.getIndexAttr(count)}; - SmallVector targetStrides{b.getIndexAttr(1)}; - auto targetRowType = memref::SubViewOp::inferRankReducedResultType( - {count}, flatType, targetOffsets, targetSizes, targetStrides); - Value targetRow = b.create( - loc, targetRowType, flatBuffer, targetOffsets, targetSizes, - targetStrides); - b.create(loc, sourceRow, targetRow); - } else { - auto kLoop = b.create(loc, zero, idx(count), one); - b.setInsertionPointToStart(kLoop.getBody()); - Value k = kLoop.getInductionVar(); - Value kk = b.create(loc, k, idx(begin)); - SmallVector indices; - if (rank == 3) - indices.push_back(batch); - indices.push_back(operand ? kk : global); - indices.push_back(operand ? global : kk); - Value value = b.create(loc, source, indices); - Value destination = b.create(loc, flat, k); - b.create(loc, value, flatBuffer, - ValueRange{destination}); - } - b.setInsertionPointAfter(rLoop); - if (logicalRows % blockRows) { - Value start = b.create(loc, validRows, idx(count * 4)); - Value pointer = - b.create(loc, packed); - Value address = b.create(loc, pointer, start); - Value integer = - b.create(loc, b.getI64Type(), address); - Value destination = b.create( - loc, LLVM::LLVMPointerType::get(b.getContext()), integer); - Value length = - b.create(loc, idx(blockRows * count * 4), start); - Value byteLength = - b.create(loc, b.getI64Type(), length); - Value byteZero = b.create(loc, 0, 8); - b.create(loc, destination, byteZero, byteLength, false); - } - } - if (bankPack) { - ab = b.create( - loc, b.getI64Type(), ap, ab, createI64Const(b, loc, rows * count / 4), - createI64Const(b, loc, 1), b.getI64IntegerAttr(1)); - auto packed = b.create( - loc, b.getI64Type(), b.getI64Type(), ab, wb, - FlatSymbolRefAttr::get(b.getContext(), "rvv_packmm"), - ValueRange{ - createI64Const(b, loc, 1), createI64Const(b, loc, K), - createI64Const(b, loc, N), createI64Const(b, loc, begin), - b.create(loc, b.getI64Type(), column), - createI64Const(b, loc, count), createI64Const(b, loc, columns), - createI64Const(b, loc, rows), createI64Const(b, loc, begin == 0), - createI64Const(b, loc, begin + count == K), - createI64Const(b, loc, op.getFusedAttr().getValue())}); - ab = packed.getReadOut(); - wb = cb = packed.getWriteOut(); - } else { - ab = mvinBank(b, loc, ap, ab, rows * count / 4); - wb = mvinBank(b, loc, bp, wb, columns * count / 4); - cb = b.create(loc, b.getI64Type(), ab, wb, cb, - createI64Const(b, loc, - packBits(rows, 0, 11) | - packBits(columns, 12, 23) | - packBits(count, 24, 35)), - createI1Const(b, loc, begin == 0), - createI1Const(b, loc, begin + count == K), - createI64Const(b, loc, 0), op.getFusedAttr()) - .getWrBankOut(); - } - } - if (bankPack) - cb = b.create( - loc, b.getI64Type(), cp, cb, createI64Const(b, loc, rows * columns / 4), - createI64Const(b, loc, 1), b.getI64IntegerAttr(2)); - else - cb = mvoutBank(b, loc, cp, cb, rows * columns / 4); - SmallVector outputStrides; - int64_t outputOffset; - if (failed(out.getStridesAndOffset(outputStrides, outputOffset))) - return op.emitError("FP32 matmul requires a strided output"); - Value validColumns = b.create( - loc, idx(columns), b.create(loc, idx(N), column)); - auto rr = b.create( - loc, zero, - b.create(loc, idx(rows), - b.create(loc, idx(M), row)), - one); - b.setInsertionPointToStart(rr.getBody()); - Value r = rr.getInductionVar(); - if (outputStrides.back() == 1) { - auto flatType = MemRefType::get({bytes / 4}, b.getF32Type()); - Value flat = b.create( - loc, flatType, cp, ArrayRef{{0, 1}}); - SmallVector offsets{ - b.create(loc, r, idx(columns)).getResult()}; - SmallVector sizes{validColumns}, strides{b.getIndexAttr(1)}; - auto sourceType = memref::SubViewOp::inferRankReducedResultType( - {ShapedType::kDynamic}, flatType, offsets, sizes, strides); - Value source = b.create(loc, sourceType, flat, offsets, - sizes, strides); - SmallVector outOffsets(rank, b.getIndexAttr(0)); - SmallVector outSizes(rank, b.getIndexAttr(1)); - SmallVector outSteps(rank, b.getIndexAttr(1)); - if (rank == 3) - outOffsets[0] = batch; - outOffsets[rank - 2] = b.create(loc, row, r).getResult(); - outOffsets[rank - 1] = column; - outSizes.back() = validColumns; - auto rowType = memref::SubViewOp::inferRankReducedResultType( - {ShapedType::kDynamic}, out, outOffsets, outSizes, outSteps); - Value destination = b.create( - loc, rowType, op.getOutput(), outOffsets, outSizes, outSteps); - b.create(loc, source, destination); - } else { - auto cc = b.create(loc, zero, validColumns, one); - b.setInsertionPointToStart(cc.getBody()); - Value c = cc.getInductionVar(); - Value flat = b.create( - loc, b.create(loc, r, idx(columns)), c); - Value result = b.create( - loc, cp, - ValueRange{b.create(loc, flat, four), - b.create(loc, flat, four)}); - SmallVector outputIndices; - if (rank == 3) - outputIndices.push_back(batch); - outputIndices.push_back(b.create(loc, row, r)); - outputIndices.push_back(b.create(loc, column, c)); - b.create(loc, result, op.getOutput(), outputIndices); - } - b.setInsertionPointAfter(rr); - SmallVector nextBanks{ab, wb, cb}; - if (bankPack) - nextBanks = {ab, cb}; - b.create(loc, nextBanks); - b.setInsertionPointAfter(colLoop); - b.create(loc, colLoop.getResults()); - b.setInsertionPointAfter(rowLoop); - b.create(loc, rowLoop.getResults()); - b.setInsertionPointAfter(batchLoop); - if (bankPack) { - auto finished = b.create( - loc, b.getI64Type(), b.getI64Type(), batchLoop.getResult(0), - batchLoop.getResult(1), - FlatSymbolRefAttr::get(b.getContext(), "rvv_packmm"), - ValueRange{createI64Const(b, loc, 2), createI64Const(b, loc, K), - createI64Const(b, loc, N), createI64Const(b, loc, 0), - createI64Const(b, loc, 0), createI64Const(b, loc, K), - createI64Const(b, loc, columns), - createI64Const(b, loc, rows), createI64Const(b, loc, 1), - createI64Const(b, loc, 1), - createI64Const(b, loc, op.getFusedAttr().getValue())}); - releaseBank(b, loc, finished.getReadOut()); - releaseBank(b, loc, finished.getWriteOut()); - } else { - for (Value bank : batchLoop.getResults()) - releaseBank(b, loc, bank); - } - for (Value buffer : {ap, bp, cp}) - b.create(loc, buffer); - b.eraseOp(op); - return success(); -} - -LogicalResult MXFP8MemMatmulOp::lowerToBankSSA(PatternRewriter &b) { - MXFP8MemMatmulOp op = *this; - auto aTy = cast(op.getInput().getType()); - auto wTy = cast(op.getWeight().getType()); - auto oTy = cast(op.getOutput().getType()); - if (!aTy.hasStaticShape() || !wTy.hasStaticShape() || !oTy.hasStaticShape() || - aTy.getRank() != 1 || wTy.getRank() != 1 || oTy.getRank() != 2 || - !aTy.getElementType().isInteger(8) || - !wTy.getElementType().isInteger(8) || !oTy.getElementType().isF32()) - return op.emitError("MXFP8 requires static packed activation/weight bytes " - "and an FP32 output matrix"); - if (aTy.getStridesAndOffset().first[0] != 1 || - wTy.getStridesAndOffset().first[0] != 1) - return op.emitError("MXFP8 packed tensors must have unit byte stride"); - int64_t M = oTy.getShape()[0], K = op.getReductionK(), N = oTy.getShape()[1]; - int64_t rows = M == 1 ? 1 : op.getTileM(), columns = op.getTileN(); - int64_t chunk = op.getTileK(), activationChunk = op.getActivationTileK(); - bool windowInput = activationChunk != chunk; - if (rows <= 0 || (rows != 1 && rows % 16) || columns <= 0 || columns % 16 || - chunk <= 0 || chunk % 32 || chunk >= 4096) - return op.emitError( - "MXFP8 tile_k must be a positive multiple of 32 below 4096"); - int64_t chunks = (K + chunk - 1) / chunk; - int64_t stride = op.getBankBytes(); - if (windowInput && (activationChunk != K || K > 65535 || - (rows * K * 33 / 32 + 15) / 16 * 16 > stride)) - return op.emitError("MXFP8 activation window must hold the complete row"); - int64_t activationChunks = windowInput ? 1 : chunks; - if (M <= 0 || K <= 0 || N <= 0 || K % 32 || - aTy.getShape()[0] != - ((M + rows - 1) / rows) * activationChunks * stride || - wTy.getShape()[0] != ((N + columns - 1) / columns) * chunks * stride) - return op.emitError("MXFP8 matrix or packed weight shape mismatch"); - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth * 16 != stride || - (rows != 1 && (rows <= 0 || rows % 16)) || columns <= 0 || columns % 16 || - (rows * chunk * 33 / 32 + 15) / 16 * 16 > stride || - columns * chunk * 33 / 32 > stride || rows * columns * 4 > stride) - return op.emitError( - "MXFP8 packed K block exceeds the 128-bit bank capacity"); - Location loc = op.getLoc(); - Value zero = b.create(loc, 0), - one = b.create(loc, 1); - Value packed = op.getInput(); - auto dynamic = ShapedType::kDynamic; - auto bankView = [&](Value source, Value offset, int64_t size) -> Value { - Value slice = b.create( - loc, source, SmallVector{offset}, - SmallVector{b.getIndexAttr(stride)}, - SmallVector{b.getIndexAttr(1)}); - auto ty = MemRefType::get( - {stride / 16, 16}, b.getI8Type(), - StridedLayoutAttr::get(b.getContext(), dynamic, {16, 1})); - return b.create( - loc, ty, slice, ArrayRef{{0, 1}}); - }; - Value aBank = allocBank(b, loc, 1, 1), wBank = allocBank(b, loc, 1, 1), - cBank = allocBank(b, loc, 1, 1); - bool residentInput = activationChunks + 2 <= target.bankNum; - SmallVector banks{aBank}; - if (residentInput) - for (int64_t part = 1; part < activationChunks; ++part) - banks.push_back(allocBank(b, loc, 1, 1)); - banks.push_back(wBank); - banks.push_back(cBank); - auto outPackTy = MemRefType::get({stride / 16, 4}, b.getF32Type()); - Value outPack = b.create(loc, outPackTy); - outPack.getDefiningOp()->setAttr("alignment", b.getI64IntegerAttr(stride)); - Value limit = b.create(loc, N), - step = b.create(loc, columns); - Value rowLimit = b.create(loc, M), - rowStep = b.create(loc, rows); - auto rowBlocks = b.create(loc, zero, rowLimit, rowStep, banks); - b.setInsertionPointToStart(rowBlocks.getBody()); - Value m = rowBlocks.getInductionVar(); - SmallVector inputBanks(rowBlocks.getRegionIterArgs().drop_back(2)); - aBank = inputBanks.front(); - wBank = rowBlocks.getRegionIterArgs()[banks.size() - 2]; - cBank = rowBlocks.getRegionIterArgs()[banks.size() - 1]; - Value rowPanel = b.create(loc, m, rowStep); - if (residentInput) { - for (int64_t part = 0; part < activationChunks; ++part) { - Value offset = b.create( - loc, - b.create( - loc, - b.create( - loc, rowPanel, - b.create(loc, activationChunks)), - b.create(loc, part)), - b.create(loc, stride)); - int64_t count = windowInput ? K : std::min(chunk, K - part * chunk); - int64_t aBytes = (rows * count * 33 / 32 + 15) / 16 * 16; - inputBanks[part] = mvinBank(b, loc, bankView(packed, offset, stride), - inputBanks[part], aBytes / 16); - } - aBank = inputBanks.front(); - } - auto panels = b.create(loc, zero, limit, step, - ValueRange{aBank, wBank, cBank}); - b.setInsertionPointToStart(panels.getBody()); - Value n = panels.getInductionVar(); - aBank = panels.getRegionIterArgs()[0]; - wBank = panels.getRegionIterArgs()[1]; - cBank = panels.getRegionIterArgs()[2]; - Value state = cBank; - for (int64_t part = 0; part < chunks; ++part) { - int64_t count = std::min(chunk, K - part * chunk); - int64_t aBytes = (rows * count * 33 / 32 + 15) / 16 * 16, - bBytes = columns * count * 33 / 32; - Value aOffset = b.create( - loc, - b.create( - loc, - b.create( - loc, rowPanel, b.create(loc, chunks)), - b.create(loc, part)), - b.create(loc, stride)); - Value panel = b.create(loc, n, step); - Value wOffset = b.create( - loc, - b.create( - loc, - b.create( - loc, panel, b.create(loc, chunks)), - b.create(loc, part)), - b.create(loc, stride)); - if (!residentInput) - aBank = mvinBank(b, loc, bankView(packed, aOffset, aBytes), aBank, - aBytes / 16); - wBank = mvinBank(b, loc, bankView(op.getWeight(), wOffset, bBytes), wBank, - bBytes / 16); - if (windowInput) - state = b.create( - loc, b.getI64Type(), aBank, wBank, state, - createI64Const(b, loc, rows), createI64Const(b, loc, columns), - createI64Const(b, loc, count), createI64Const(b, loc, K), - createI64Const(b, loc, part * chunk), - createI1Const(b, loc, part == 0), - createI1Const(b, loc, part == chunks - 1), createI64Const(b, loc, 0)); - else - state = b.create( - loc, b.getI64Type(), residentInput ? inputBanks[part] : aBank, wBank, - state, createI64Const(b, loc, matrixRs2(rows, columns, count)), - createI1Const(b, loc, part == 0), - createI1Const(b, loc, part == chunks - 1), createI64Const(b, loc, 0)); - } - cBank = mvoutBank(b, loc, outPack, state, rows * columns / 4); - Value end = b.create(loc, step, - b.create(loc, limit, n)); - SmallVector outputStrides; - int64_t outputOffset; - if (failed(oTy.getStridesAndOffset(outputStrides, outputOffset))) - return op.emitError("MXFP8 matmul requires a strided output"); - auto rr = b.create( - loc, zero, - b.create(loc, rowStep, - b.create(loc, rowLimit, m)), - one); - b.setInsertionPointToStart(rr.getBody()); - Value r = rr.getInductionVar(); - if (outputStrides.back() == 1) { - auto flatType = MemRefType::get({stride / 4}, b.getF32Type()); - Value flat = b.create( - loc, flatType, outPack, ArrayRef{{0, 1}}); - SmallVector offsets{ - b.create(loc, r, - b.create(loc, columns)) - .getResult()}; - SmallVector sizes{end}, steps{b.getIndexAttr(1)}; - auto sourceType = memref::SubViewOp::inferRankReducedResultType( - {ShapedType::kDynamic}, flatType, offsets, sizes, steps); - Value source = b.create(loc, sourceType, flat, offsets, - sizes, steps); - SmallVector outOffsets{ - b.create(loc, m, r).getResult(), n}; - SmallVector outSizes{b.getIndexAttr(1), end}; - SmallVector outSteps(2, b.getIndexAttr(1)); - auto rowType = memref::SubViewOp::inferRankReducedResultType( - {ShapedType::kDynamic}, oTy, outOffsets, outSizes, outSteps); - Value destination = b.create( - loc, rowType, op.getOutput(), outOffsets, outSizes, outSteps); - b.create(loc, source, destination); - } else { - auto cc = b.create(loc, zero, end, one); - b.setInsertionPointToStart(cc.getBody()); - Value c = cc.getInductionVar(), - four = b.create(loc, 4); - Value pr = b.create( - loc, - b.create( - loc, r, b.create(loc, columns / 4)), - b.create(loc, c, four)); - Value pc = b.create(loc, c, four); - Value value = b.create(loc, outPack, ValueRange{pr, pc}); - b.create(loc, value, op.getOutput(), - ValueRange{b.create(loc, r, m), - b.create(loc, n, c)}); - } - b.setInsertionPointAfter(rr); - b.create(loc, ValueRange{aBank, wBank, cBank}); - b.setInsertionPointAfter(panels); - inputBanks[0] = panels.getResult(0); - inputBanks.push_back(panels.getResult(1)); - inputBanks.push_back(panels.getResult(2)); - b.create(loc, inputBanks); - b.setInsertionPointAfter(rowBlocks); - for (Value bank : rowBlocks.getResults()) - releaseBank(b, loc, bank); - b.create(loc, outPack); - b.eraseOp(op); - return success(); -} - -LogicalResult MXFP8SharedPanelOp::lowerToBankSSA(PatternRewriter &b) { - MXFP8SharedPanelOp op = *this; - auto a = cast(op.getInput().getType()); - auto w = cast(op.getWeight().getType()); - int64_t rows = op.getRows(), columns = op.getColumns(); - int64_t k = op.getReductionK(), chunk = op.getTileK(); - int64_t bytes = op.getBankBytes(); - int64_t sharedBytes = op.getSharedBytesAttr().getInt(); - const auto &target = buckyball_target::getBuckyballTarget(); - if ((rows != 1 && (rows <= 0 || rows % 16)) || columns <= 0 || columns % 16 || - rows > 4095 || columns > 4095 || k <= 0 || k % 32 || chunk <= 0 || - chunk % 32 || chunk >= 4096 || target.bankWidthBits != 128 || - target.bankDepth * 16 != bytes || sharedBytes <= 0 || sharedBytes % 16 || - rows * columns * 4 > sharedBytes || - (rows * chunk * 33 / 32 + 15) / 16 * 16 > bytes || - columns * chunk * 33 / 32 > bytes) - return op.emitError("shared MXFP8 panel exceeds the original bank layout"); - int64_t chunks = (k + chunk - 1) / chunk; - for (MemRefType type : {a, w}) - if (!type.hasStaticShape() || type.getRank() != 1 || - !type.getElementType().isInteger(8) || - type.getShape()[0] != chunks * bytes || - type.getStridesAndOffset().first[0] != 1) - return op.emitError("shared MXFP8 panel requires one packed row/N panel"); - Location loc = op.getLoc(); - Value ab = allocBank(b, loc, 1, 1), wb = allocBank(b, loc, 1, 1); - Value output = op.getBank(); - for (int64_t part = 0; part < chunks; ++part) { - int64_t count = std::min(chunk, k - part * chunk); - auto view = [&](Value packed) -> Value { - Value slice = b.create( - loc, packed, SmallVector{b.getIndexAttr(part * bytes)}, - SmallVector{b.getIndexAttr(bytes)}, - SmallVector{b.getIndexAttr(1)}); - auto type = *memref::ExpandShapeOp::computeExpandedType( - cast(slice.getType()), {bytes / 16, 16}, - ArrayRef{{0, 1}}); - return b.create( - loc, type, slice, ArrayRef{{0, 1}}); - }; - ab = mvinBank(b, loc, view(op.getInput()), ab, - (rows * count * 33 / 32 + 15) / 16); - wb = mvinBank(b, loc, view(op.getWeight()), wb, - columns * count * 33 / 32 / 16); - output = b.create( - loc, b.getI64Type(), ab, wb, output, - createI64Const(b, loc, matrixRs2(rows, columns, count)), - createI1Const(b, loc, part == 0), - createI1Const(b, loc, part == chunks - 1), createI64Const(b, loc, 0)); - } - releaseBank(b, loc, ab); - releaseBank(b, loc, wb); - b.replaceOp(op, output); - return success(); -} - -LogicalResult MXFP8SharedWindowOp::lowerToBankSSA(PatternRewriter &b) { - MXFP8SharedWindowOp op = *this; - auto a = cast(op.getInput().getType()); - auto w = cast(op.getWeight().getType()); - int64_t rows = op.getRows(), columns = op.getColumns(); - int64_t k = op.getReductionK(), chunk = op.getTileK(); - int64_t bytes = op.getBankBytes(); - int64_t sharedBytes = op.getSharedBytesAttr().getInt(); - const auto &target = buckyball_target::getBuckyballTarget(); - if ((rows != 1 && (rows <= 0 || rows % 16)) || rows > 4095 || columns <= 0 || - columns > 4095 || columns % 16 || k <= 0 || k > 65535 || k % 32 || - chunk <= 0 || chunk >= 4096 || chunk % 32 || - target.bankWidthBits != 128 || target.bankDepth * 16 != bytes || - sharedBytes <= 0 || sharedBytes % 16 || - rows * columns * 4 > sharedBytes || - (rows * k * 33 / 32 + 15) / 16 * 16 > bytes || - columns * chunk * 33 / 32 > bytes) - return op.emitError( - "shared MXFP8 window exceeds its activation or output bank"); - int64_t chunks = (k + chunk - 1) / chunk; - if (!a.hasStaticShape() || !w.hasStaticShape() || a.getShape()[0] != bytes || - w.getShape()[0] != chunks * bytes || - a.getStridesAndOffset().first[0] != 1 || - w.getStridesAndOffset().first[0] != 1) - return op.emitError( - "shared MXFP8 window requires a complete activation and one N panel"); - Location loc = op.getLoc(); - auto view = [&](Value source, int64_t offset) -> Value { - Value slice = b.create( - loc, source, SmallVector{b.getIndexAttr(offset)}, - SmallVector{b.getIndexAttr(bytes)}, - SmallVector{b.getIndexAttr(1)}); - auto type = *memref::ExpandShapeOp::computeExpandedType( - cast(slice.getType()), {bytes / 16, 16}, - ArrayRef{{0, 1}}); - return b.create( - loc, type, slice, ArrayRef{{0, 1}}); - }; - Value ab = allocBank(b, loc, 1, 1), wb = allocBank(b, loc, 1, 1); - ab = mvinBank(b, loc, view(op.getInput(), 0), ab, - (rows * k * 33 / 32 + 15) / 16); - Value output = op.getBank(); - for (int64_t part = 0; part < chunks; ++part) { - int64_t count = std::min(chunk, k - part * chunk); - wb = mvinBank(b, loc, view(op.getWeight(), part * bytes), wb, - columns * count * 33 / 32 / 16); - output = b.create( - loc, b.getI64Type(), ab, wb, output, createI64Const(b, loc, rows), - createI64Const(b, loc, columns), createI64Const(b, loc, count), - createI64Const(b, loc, k), createI64Const(b, loc, part * chunk), - createI1Const(b, loc, part == 0), - createI1Const(b, loc, part == chunks - 1), createI64Const(b, loc, 0)); - } - releaseBank(b, loc, ab); - releaseBank(b, loc, wb); - b.replaceOp(op, output); - return success(); -} diff --git a/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/IR/FP32.cpp b/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/IR/FP32.cpp deleted file mode 100644 index fc62bde67..000000000 --- a/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/IR/FP32.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -using namespace mlir; -using namespace mlir::bufferization; -using namespace ::buddy::buckyball; - -bool FP32MatmulOp::bufferizesToAllocation(Value) { return true; } -bool FP32MatmulOp::bufferizesToMemoryRead(OpOperand &, const AnalysisState &) { - return true; -} -bool FP32MatmulOp::bufferizesToMemoryWrite(OpOperand &, const AnalysisState &) { - return false; -} -AliasingValueList FP32MatmulOp::getAliasingValues(OpOperand &, - const AnalysisState &) { - return {}; -} -LogicalResult FP32MatmulOp::bufferize(RewriterBase &b, - const BufferizationOptions &options, - BufferizationState &state) { - auto a = getBuffer(b, getLhs(), options, state); - auto w = getBuffer(b, getRhs(), options, state); - if (failed(a) || failed(w)) - return failure(); - auto type = cast(getOutput().getType()); - if (!type.hasStaticShape()) - return emitError("requires static FP32 output shape"); - Value out = b.create( - getLoc(), MemRefType::get(type.getShape(), type.getElementType())); - Value rhs = *w; - if (getRhsTransposedAttr()) { - auto source = cast(rhs.getType()); - int rank = source.getRank(); - if (rank != 2 && rank != 3) - return emitError("transposed FP32 RHS requires rank 2 or 3"); - SmallVector permutation; - for (int index = 0; index < rank; ++index) - permutation.push_back(index); - std::swap(permutation[rank - 2], permutation[rank - 1]); - auto map = AffineMap::getPermutationMap(permutation, getContext()); - rhs = b.create(getLoc(), rhs, AffineMapAttr::get(map)); - } - b.create(getLoc(), *a, rhs, out, getFusedAttr()); - replaceOpWithBufferizedValues(b, getOperation(), out); - return success(); -} diff --git a/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/IR/MXFP8.cpp b/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/IR/MXFP8.cpp deleted file mode 100644 index 77fb3d642..000000000 --- a/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/IR/MXFP8.cpp +++ /dev/null @@ -1,157 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "llvm/Support/MathExtras.h" -using namespace mlir; -using namespace mlir::bufferization; -using namespace ::buddy::buckyball; - -namespace { -FailureOr packedSize(int64_t m, int64_t k, int64_t rows, int64_t chunk, - int64_t stride) { - if (m <= 0 || k <= 0 || k % 32 || rows <= 0 || (rows != 1 && rows % 16) || - chunk <= 0 || chunk % 32 || stride <= 0 || stride > 65536 || - !llvm::isPowerOf2_64(stride) || rows > stride || - chunk > stride * 32 / (rows * 33)) - return failure(); - int64_t panels, bytes; - if (llvm::MulOverflow((m - 1) / rows + 1, (k - 1) / chunk + 1, panels) || - llvm::MulOverflow(panels, stride, bytes)) - return failure(); - return bytes; -} -} // namespace - -LogicalResult MXFP8QuantOp::verify() { - auto input = cast(getInput().getType()); - auto output = cast(getOutput().getType()); - if (!input.hasStaticShape() || input.getRank() != 2 || - !input.getElementType().isF32() || !output.hasStaticShape() || - output.getRank() != 1 || !output.getElementType().isInteger(8)) - return emitOpError("requires a static FP32 matrix and packed byte tensor"); - auto bytes = packedSize(input.getShape()[0], input.getShape()[1], - getTileRows(), getTileK(), getBankBytes()); - if (failed(bytes) || output.getShape()[0] != *bytes) - return emitOpError("activation panel layout or packed byte shape mismatch"); - return success(); -} - -bool MXFP8QuantOp::bufferizesToAllocation(Value) { return true; } -bool MXFP8QuantOp::bufferizesToMemoryRead(OpOperand &, const AnalysisState &) { - return true; -} -bool MXFP8QuantOp::bufferizesToMemoryWrite(OpOperand &, const AnalysisState &) { - return false; -} -AliasingValueList MXFP8QuantOp::getAliasingValues(OpOperand &, - const AnalysisState &) { - return {}; -} -LogicalResult MXFP8QuantOp::bufferize(RewriterBase &b, - const BufferizationOptions &options, - BufferizationState &state) { - auto input = getBuffer(b, getInput(), options, state); - if (failed(input)) - return failure(); - auto type = cast(getOutput().getType()); - auto output = b.create( - getLoc(), MemRefType::get(type.getShape(), type.getElementType())); - output->setAttr("alignment", b.getI64IntegerAttr(getBankBytes())); - auto dynamic = ShapedType::kDynamic; - auto inputType = MemRefType::get( - {dynamic, dynamic}, b.getF32Type(), - StridedLayoutAttr::get(getContext(), dynamic, {dynamic, dynamic})); - auto bytesType = MemRefType::get({dynamic}, b.getI8Type()); - auto module = getOperation()->getParentOfType(); - if (!module.lookupSymbol("mxfp8_quant")) { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(module.getBody()); - auto function = b.create( - getLoc(), "mxfp8_quant", - b.getFunctionType({inputType, bytesType, b.getI64Type(), b.getI64Type(), - b.getI64Type()}, - {})); - function.setPrivate(); - function->setAttr("llvm.emit_c_interface", b.getUnitAttr()); - } - SmallVector arguments{ - b.create(getLoc(), inputType, *input), - b.create(getLoc(), bytesType, output)}; - for (int64_t value : {getTileRows(), getTileK(), getBankBytes()}) - arguments.push_back(b.create(getLoc(), value, 64)); - b.create(getLoc(), "mxfp8_quant", TypeRange{}, arguments); - replaceOpWithBufferizedValues(b, getOperation(), - ValueRange{output.getResult()}); - return success(); -} - -LogicalResult MXFP8MatmulOp::verify() { - auto input = cast(getInput().getType()); - auto weight = cast(getWeight().getType()); - auto output = cast(getOutput().getType()); - if (!input.hasStaticShape() || input.getRank() != 1 || - !input.getElementType().isInteger(8) || !weight.hasStaticShape() || - weight.getRank() != 1 || !weight.getElementType().isInteger(8) || - !output.hasStaticShape() || output.getRank() != 2 || - !output.getElementType().isF32()) - return emitOpError( - "requires packed activation/weight bytes and a static FP32 matrix"); - int64_t m = output.getShape()[0], n = output.getShape()[1], - k = getReductionK(); - if (getTileK() >= 4096) - return emitOpError("matrix panel K must be less than 4096"); - int64_t rows = m == 1 ? 1 : getTileM(), columns = getTileN(); - int64_t activationK = getActivationTileK(); - if (activationK != getTileK() && - (activationK != k || k > 65535 || rows * k * 33 / 32 > getBankBytes())) - return emitOpError("activation tile K must match the weight panel or fit a " - "complete row window"); - auto activationBytes = packedSize(m, k, rows, activationK, getBankBytes()); - auto weightBytes = packedSize(n, k, columns, getTileK(), getBankBytes()); - if (columns <= 0 || columns % 16 || failed(activationBytes) || - failed(weightBytes) || input.getShape()[0] != *activationBytes || - weight.getShape()[0] != *weightBytes) - return emitOpError("logical K, panel layout or packed byte shape mismatch"); - if (auto quant = getInput().getDefiningOp()) { - auto source = cast(quant.getInput().getType()); - if (source.getRank() != 2 || source.getShape()[0] != m || - source.getShape()[1] != k) - return emitOpError("activation quantization matrix shape mismatch"); - if (quant.getTileRows() != rows || quant.getTileK() != activationK || - quant.getBankBytes() != getBankBytes()) - return emitOpError("activation quantization panel layout mismatch"); - } - return success(); -} - -bool MXFP8MatmulOp::bufferizesToAllocation(Value) { return true; } -bool MXFP8MatmulOp::bufferizesToMemoryRead(OpOperand &, const AnalysisState &) { - return true; -} -bool MXFP8MatmulOp::bufferizesToMemoryWrite(OpOperand &, - const AnalysisState &) { - return false; -} -AliasingValueList MXFP8MatmulOp::getAliasingValues(OpOperand &, - const AnalysisState &) { - return {}; -} -LogicalResult MXFP8MatmulOp::bufferize(RewriterBase &b, - const BufferizationOptions &options, - BufferizationState &state) { - auto input = getBuffer(b, getInput(), options, state); - auto weight = getBuffer(b, getWeight(), options, state); - if (failed(input) || failed(weight)) - return failure(); - auto type = cast(getOutput().getType()); - auto output = b.create( - getLoc(), MemRefType::get(type.getShape(), type.getElementType())); - b.create(getLoc(), *input, *weight, output, - getReductionKAttr(), getTileKAttr(), - getTileMAttr(), getTileNAttr(), getBankBytesAttr(), - getActivationTileKAttr()); - replaceOpWithBufferizedValues(b, getOperation(), - ValueRange{output.getResult()}); - return success(); -} diff --git a/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/MxmmBall.td b/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/MxmmBall.td deleted file mode 100644 index a12c70dd8..000000000 --- a/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/MxmmBall.td +++ /dev/null @@ -1,145 +0,0 @@ -#ifndef MATMULBALL_OPS -#define MATMULBALL_OPS - -include "Buckyball.td" -include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td" - -def MXFP8QuantOp : Buckyball_Op<"mxfp8_quant", [NoMemoryEffect, - DeclareOpInterfaceMethods]> { - let summary = "Integer RNE activation quantization into aligned MXFP8 panels"; - let description = [{ - The input is an immutable finite FP32 matrix. Each panel contains all - row-major E4M3 codes followed by row-major block32 E8M0 scales. The final - K panel uses its actual remaining width. Unused bank-window padding is - undefined and must not be consumed as codes or scales. Quantization does - not read frm or modify fflags. This operation is not speculatable. - }]; - let arguments = (ins AnyRankedTensor:$input, I64Attr:$tile_rows, - I64Attr:$tile_k, I64Attr:$bank_bytes); - let results = (outs AnyRankedTensor:$output); - let hasVerifier = 1; - let assemblyFormat = "$input prop-dict attr-dict `:` type($input) `->` type($output)"; -} - -def MXFP8MatmulOp : Buckyball_Op<"mxfp8_matmul", [ - DeclareOpInterfaceMethods]> { - let arguments = (ins AnyRankedTensor:$input, AnyRankedTensor:$weight, I64Attr:$reduction_k, I64Attr:$tile_k, I64Attr:$tile_m, I64Attr:$tile_n, I64Attr:$bank_bytes, I64Attr:$activation_tile_k); - let results = (outs AnyRankedTensor:$output); - let hasVerifier = 1; - let assemblyFormat = "$input `,` $weight prop-dict attr-dict `:` type($input) `,` type($weight) `->` type($output)"; -} -def MXFP8MemMatmulOp : Buckyball_Op<"mxfp8_mem_matmul", [DeclareOpInterfaceMethods]> { - let arguments = (ins Arg]>:$input, - Arg]>:$weight, - Arg]>:$output, I64Attr:$reduction_k, I64Attr:$tile_k, I64Attr:$tile_m, I64Attr:$tile_n, I64Attr:$bank_bytes, I64Attr:$activation_tile_k); - let assemblyFormat = "$input `,` $weight `,` $output prop-dict attr-dict `:` type($input) `,` type($weight) `,` type($output)"; -} -def MXFP8SharedPanelOp : Buckyball_Op<"mxfp8_shared_panel", [DeclareOpInterfaceMethods, MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "Compute one original MXFP8 output panel into a live shared bank"; - let arguments = (ins MemRefRankOf<[I8], [1]>:$input, - MemRefRankOf<[I8], [1]>:$weight, I64:$bank, - I64Attr:$rows, I64Attr:$columns, I64Attr:$reduction_k, - I64Attr:$tile_k, I64Attr:$bank_bytes, I64Attr:$shared_bytes); - let results = (outs I64:$bank_out); - let assemblyFormat = "$input `,` $weight `,` $bank prop-dict attr-dict `:` " - "type($input) `,` type($weight) `,` type($bank)"; -} -def MXFP8SharedWindowOp : Buckyball_Op<"mxfp8_shared_window", [DeclareOpInterfaceMethods, MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "Compute a shared output panel from one complete MXFP8 activation bank"; - let arguments = (ins MemRefRankOf<[I8], [1]>:$input, - MemRefRankOf<[I8], [1]>:$weight, I64:$bank, - I64Attr:$rows, I64Attr:$columns, I64Attr:$reduction_k, - I64Attr:$tile_k, I64Attr:$bank_bytes, I64Attr:$shared_bytes); - let results = (outs I64:$bank_out); - let assemblyFormat = "$input `,` $weight `,` $bank prop-dict attr-dict `:` " - "type($input) `,` type($weight) `,` type($bank)"; -} -def MXFP8Op : Buckyball_Op<"mxfp8"> { - let summary = "MxmmBall compute"; - let description = [{ - ISA: MXMM_MXFP8. `config` contains rows, cols, and k. - }]; - let arguments = (ins I64:$op1BankId, I64:$op2BankId, - I64:$resultBankId, I64:$config, - I1:$first, I1:$last, I64:$outputBase); - let assemblyFormat = "$op1BankId `,` $op2BankId `,` $resultBankId `,` " - "$config `,` $first `,` $last `,` $outputBase " - "prop-dict attr-dict `:` " - "type($op1BankId)"; -} - -def BankMXFP8Op : Buckyball_Op<"bank_mxfp8", [DeclareOpInterfaceMethods]> { - let summary = "MxmmBall compute with virtual bank handles"; - let arguments = (ins I64:$op1Bank, I64:$op2Bank, I64:$wrBank, I64:$config, - I1:$first, I1:$last, I64:$outputBase); - let results = (outs I64:$wrBankOut); - let assemblyFormat = - "$op1Bank $op2Bank $wrBank $config $first $last $outputBase " - "prop-dict attr-dict `:` " - "type($op1Bank) type($op2Bank) type($wrBank) type($config)"; -} - -def MXFP8WindowOp : Buckyball_Op<"mxfp8_window"> { - let summary = "MXFP8 multiplication from a complete activation row window"; - let arguments = (ins I64:$op1BankId, I64:$op2BankId, I64:$resultBankId, - I64:$rows, I64:$cols, I64:$count, I64:$fullK, I64:$startK, - I1:$first, I1:$last, I64:$outputBase); - let assemblyFormat = "$op1BankId `,` $op2BankId `,` $resultBankId `,` " - "$rows `,` $cols `,` $count `,` $fullK `,` $startK `,` " - "$first `,` $last `,` $outputBase prop-dict attr-dict `:` type($op1BankId)"; -} - -def BankMXFP8WindowOp : Buckyball_Op<"bank_mxfp8_window", [DeclareOpInterfaceMethods, MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "MXFP8 activation row window with virtual bank handles"; - let arguments = (ins I64:$op1Bank, I64:$op2Bank, I64:$wrBank, - I64:$rows, I64:$cols, I64:$count, I64:$fullK, I64:$startK, - I1:$first, I1:$last, I64:$outputBase); - let results = (outs I64:$wrBankOut); - let assemblyFormat = "$op1Bank $op2Bank $wrBank $rows $cols $count " - "$fullK $startK $first $last $outputBase prop-dict attr-dict `:` type($op1Bank)"; -} - -def FP32MatmulOp : Buckyball_Op<"fp32_matmul", [ - DeclareOpInterfaceMethods]> { - let arguments = (ins AnyRankedTensor:$lhs, AnyRankedTensor:$rhs, - OptionalAttr:$rhs_transposed, BoolAttr:$fused); - let results = (outs AnyRankedTensor:$output); - let assemblyFormat = "$lhs `,` $rhs prop-dict attr-dict `:` type($lhs) `,` type($rhs) `->` type($output)"; -} -def FP32MemMatmulOp : Buckyball_Op<"fp32_mem_matmul", [DeclareOpInterfaceMethods]> { - let arguments = (ins Arg]>:$lhs, - Arg]>:$rhs, - Arg]>:$output, BoolAttr:$fused); - let assemblyFormat = "$lhs `,` $rhs `,` $output prop-dict attr-dict `:` type($lhs) `,` type($rhs) `,` type($output)"; -} -def FP32Op : Buckyball_Op<"matmul_f32"> { - let summary = "MxmmBall compute"; - let description = [{ - ISA: MXMM_FMA32 / MXMM_F32. `config` contains rows, cols, and k. - }]; - let arguments = (ins I64:$op1BankId, I64:$op2BankId, - I64:$resultBankId, I64:$config, - I1:$first, I1:$last, I64:$outputBase, BoolAttr:$fused); - let assemblyFormat = "$op1BankId `,` $op2BankId `,` $resultBankId `,` " - "$config `,` $first `,` $last `,` $outputBase " - "prop-dict attr-dict `:` " - "type($op1BankId)"; -} - -def BankFP32Op : Buckyball_Op<"bank_matmul_f32", [DeclareOpInterfaceMethods]> { - let summary = "MxmmBall compute with virtual bank handles"; - let arguments = (ins I64:$op1Bank, I64:$op2Bank, I64:$wrBank, I64:$config, - I1:$first, I1:$last, I64:$outputBase, BoolAttr:$fused); - let results = (outs I64:$wrBankOut); - let assemblyFormat = - "$op1Bank $op2Bank $wrBank $config $first $last $outputBase " - "prop-dict attr-dict `:` " - "type($op1Bank) type($op2Bank) type($wrBank) type($config)"; -} - - - -#endif diff --git a/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index ca982b05d..000000000 --- a/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,157 +0,0 @@ -#include -//===- LegalizeForLLVMExport.cpp - MxmmBall LLVM lowering --------------===// - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include "llvm/Support/ErrorHandling.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -uint64_t matrixCfg(uint64_t rows, uint64_t cols, bool first, bool last) { - return fieldBits(rows, 0, 11) | fieldBits(cols, 12, 23) | - (uint64_t(first) << 24) | (uint64_t(last) << 25); -} -struct MXFP8Lowering : public ConvertOpToLLVMPattern { - MXFP8Lowering(LLVMTypeConverter &converter, bool, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(MXFP8Op op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - auto config = op.getConfig().getDefiningOp(); - if (!config) - return op.emitError("mxmm config must be a constant"); - auto configAttr = dyn_cast(config.getValue()); - if (!configAttr) - return op.emitError("mxmm config must be an integer constant"); - uint64_t cfg = configAttr.getValue().getZExtValue(); - uint64_t rows = cfg & 0xfff; - uint64_t cols = (cfg >> 12) & 0xfff; - uint64_t k = (cfg >> 24) & 0xfff; - if (cfg >> 36 || (rows != 1 && (rows == 0 || rows % 16)) || cols == 0 || - cols % 16 || k == 0 || k % 32 || - (rows * k * 33 / 32 + 15) / 16 > static_cast(bankDepth) || - cols * k * 33 / 32 / 16 > static_cast(bankDepth) || - rows * cols / 4 > static_cast(bankDepth)) - return op.emitError("Mxmm matrix shape or C footprint is invalid"); - auto base = adaptor.getOutputBase().getDefiningOp(); - auto baseAttr = - base ? dyn_cast(base.getValue()) : IntegerAttr(); - if (!baseAttr || baseAttr.getInt() < 0 || baseAttr.getInt() > 63 || - baseAttr.getInt() + rows * cols / 4 > bankDepth) - return op.emitError("Mxmm outputBase must be constant and fit C in bank"); - Value rs1 = packRs1BanksIter( - rewriter, loc, adaptor.getOp1BankId(), adaptor.getOp2BankId(), - adaptor.getResultBankId(), cstI64(rewriter, loc, k)); - Value rs2 = cstI64(rewriter, loc, matrixCfg(rows, cols, false, false)); - Value first = rewriter.create(loc, rewriter.getI64Type(), - adaptor.getFirst()); - Value last = rewriter.create(loc, rewriter.getI64Type(), - adaptor.getLast()); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, first, cstI64(rewriter, loc, 24))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, last, cstI64(rewriter, loc, 25))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getOutputBase(), - cstI64(rewriter, loc, 26))); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MXMM_MXFP8"))); - return success(); - } - -private: - int64_t bankDepth; -}; - -struct FP32Lowering : public ConvertOpToLLVMPattern { - FP32Lowering(LLVMTypeConverter &converter, bool, int64_t depth) - : ConvertOpToLLVMPattern(converter), depth(depth) {} - - LogicalResult matchAndRewrite(FP32Op op, OpAdaptor adaptor, - ConversionPatternRewriter &b) const override { - - auto constant = op.getConfig().getDefiningOp(); - auto shape = - constant ? dyn_cast(constant.getValue()) : IntegerAttr(); - if (!shape) - return op.emitError("FP32 matmul requires a constant shape"); - uint64_t cfg = shape.getValue().getZExtValue(); - uint64_t rows = cfg & 0xfff, cols = (cfg >> 12) & 0xfff, - k = (cfg >> 24) & 0xfff; - if (cfg >> 36 || (rows != 1 && (rows == 0 || rows % 16)) || cols == 0 || - cols % 16 || !k || k % 4 || rows * k / 4 > uint64_t(depth) || - cols * k / 4 > uint64_t(depth) || - buckyball_target::getBuckyballTarget().bankWidthBits != 128) - return op.emitError( - "FP32 matmul shape does not fit the configured banks"); - auto baseOp = op.getOutputBase().getDefiningOp(); - auto base = - baseOp ? dyn_cast(baseOp.getValue()) : IntegerAttr(); - if (!base || base.getInt() < 0 || base.getInt() > 63 || - base.getInt() + rows * cols / 4 > uint64_t(depth)) - return op.emitError("FP32 output base does not fit the configured bank"); - Location loc = op.getLoc(); - Value rs1 = - packRs1BanksIter(b, loc, adaptor.getOp1BankId(), adaptor.getOp2BankId(), - adaptor.getResultBankId(), cstI64(b, loc, k)); - Value rs2 = cstI64(b, loc, matrixCfg(rows, cols, false, false)); - Value first = - b.create(loc, b.getI64Type(), adaptor.getFirst()); - Value last = - b.create(loc, b.getI64Type(), adaptor.getLast()); - rs2 = b.create( - loc, rs2, b.create(loc, first, cstI64(b, loc, 24))); - rs2 = b.create( - loc, rs2, b.create(loc, last, cstI64(b, loc, 25))); - rs2 = b.create( - loc, rs2, - b.create(loc, adaptor.getOutputBase(), - cstI64(b, loc, 26))); - b.replaceOpWithNewOp( - op, rs1, rs2, - b.getI32IntegerAttr(buckyball_target::getBuckyballFunct7( - op.getFused() ? "MXMM_FMA32" : "MXMM_F32"))); - return success(); - } - int64_t depth; -}; - -#include "WindowPatterns.inc" - -} // namespace -namespace mlir::buddy::buckyball { -void populateMxmmBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns, - bool stable, int64_t depth) { - patterns.add(converter, - stable, depth); -} -void configureMxmmBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/Transforms/WindowPatterns.inc b/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/Transforms/WindowPatterns.inc deleted file mode 100644 index 5ad880b4e..000000000 --- a/waste/examples/components/balls/mxmm/stack/src/Dialect/Buckyball/Transforms/WindowPatterns.inc +++ /dev/null @@ -1,75 +0,0 @@ -struct MXFP8WindowLowering : public ConvertOpToLLVMPattern { - MXFP8WindowLowering(LLVMTypeConverter &converter, bool, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), depth(bankDepth) {} - - LogicalResult matchAndRewrite(MXFP8WindowOp op, OpAdaptor a, - ConversionPatternRewriter &b) const override { - if (buckyball_target::getBuckyballTarget().bankWidthBits != 128) - return op.emitError("mxmm window: requires 16-byte bank rows"); - auto integer = [](Value value) -> std::optional { - if (auto constant = value.getDefiningOp()) - return cast(constant.getValue()).getInt(); - return std::nullopt; - }; - auto rows = integer(op.getRows()), cols = integer(op.getCols()); - auto count = integer(op.getCount()), full = integer(op.getFullK()); - auto start = integer(op.getStartK()), base = integer(op.getOutputBase()); - if (rows && (*rows > 4095 || (*rows != 1 && (*rows <= 0 || *rows % 16)))) - return op.emitError("mxmm window: rows must be 1 or a positive multiple " - "of 16 within 12 bits"); - if (cols && (*cols <= 0 || *cols > 4095 || *cols % 16)) - return op.emitError( - "mxmm window: cols must be a positive multiple of 16 within 12 bits"); - if (count && - (*count <= 0 || uint64_t(*count) >= (uint64_t(1) << 34) || *count % 32)) - return op.emitError("mxmm window: count must be a positive multiple of " - "32 within 34 bits"); - if (full && (*full <= 0 || *full > 65535 || *full % 32)) - return op.emitError("mxmm window: fullK must be a positive multiple of " - "32 within 16 bits"); - if (start && (*start < 0 || *start > 65535 || *start % 32)) - return op.emitError("mxmm window: startK must be a nonnegative multiple " - "of 32 within 16 bits"); - if (base && (*base < 0 || *base > 63)) - return op.emitError("mxmm window: outputBase must fit 6 bits"); - if (full && start && count && - uint64_t(*start) + uint64_t(*count) > uint64_t(*full)) - return op.emitError( - "mxmm window: reduction exceeds complete activation row"); - if (rows && full && - uint64_t(*rows) * uint64_t(*full) * 33 / 32 > uint64_t(depth) * 16) - return op.emitError( - "mxmm window: complete activation exceeds bank capacity"); - if (cols && count && - uint64_t(*cols) * uint64_t(*count) * 33 / 32 > uint64_t(depth) * 16) - return op.emitError("mxmm window: weight exceeds bank capacity"); - if (rows && cols && base && - uint64_t(*base) * 16 + uint64_t(*rows) * uint64_t(*cols) * 4 > - uint64_t(depth) * 16) - return op.emitError("mxmm window: output exceeds bank capacity"); - - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(b, loc, a.getOp1BankId(), a.getOp2BankId(), - a.getResultBankId(), a.getCount()); - Value rs2 = b.create(loc, a.getRows(), cstI64(b, loc, 4095)); - SmallVector fields{a.getCols(), a.getFirst(), a.getLast(), - a.getOutputBase(), a.getFullK(), a.getStartK()}; - const int64_t shifts[] = {12, 24, 25, 26, 32, 48}; - const int64_t masks[] = {4095, 1, 1, 63, 65535, 65535}; - for (size_t index = 0; index < fields.size(); ++index) { - Value value = fields[index]; - if (value.getType().isInteger(1)) - value = b.create(loc, b.getI64Type(), value); - value = b.create(loc, value, cstI64(b, loc, masks[index])); - rs2 = b.create( - loc, rs2, - b.create(loc, value, cstI64(b, loc, shifts[index]))); - } - b.replaceOpWithNewOp( - op, rs1, rs2, - b.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MXMM_MXFP8_WINDOW"))); - return success(); - } - int64_t depth; -}; diff --git a/waste/examples/components/balls/mxmm/stack/src/Passes.cpp b/waste/examples/components/balls/mxmm/stack/src/Passes.cpp deleted file mode 100644 index 8458a7ffc..000000000 --- a/waste/examples/components/balls/mxmm/stack/src/Passes.cpp +++ /dev/null @@ -1,41 +0,0 @@ -//===- Passes.cpp - MxmmBall passes ---------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -void registerNormQuantWindowPass(); -namespace buckyball { -void populateMxmmBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureMxmmBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LowerMxmmPass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerMxmmPass) - StringRef getArgument() const final { return "lower-mxmm-ball"; } - StringRef getDescription() const final { - return "Lower MxmmBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateMxmmBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureMxmmBallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - mlir::buddy::registerNormQuantWindowPass(); - return true; -}(); diff --git a/waste/examples/components/balls/mxmm/stack/src/Transforms/NormQuantWindow.cpp b/waste/examples/components/balls/mxmm/stack/src/Transforms/NormQuantWindow.cpp deleted file mode 100644 index 05452306b..000000000 --- a/waste/examples/components/balls/mxmm/stack/src/Transforms/NormQuantWindow.cpp +++ /dev/null @@ -1,346 +0,0 @@ -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Interfaces/SideEffectInterfaces.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" -#include "llvm/ADT/SmallPtrSet.h" -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -Value origin(Value value) { - while (Operation *op = value.getDefiningOp()) { - if (!isa( - op)) - break; - value = op->getOperand(0); - } - return value; -} - -bool privateUses(Value value, ArrayRef allowed, - SmallVectorImpl &views) { - llvm::SmallPtrSet seen; - std::function visit = [&](Value current) { - for (Operation *user : current.getUsers()) { - if (llvm::is_contained(allowed, user)) - continue; - if (!isa( - user)) - return false; - if (seen.insert(user).second) { - if (!visit(user->getResult(0))) - return false; - views.push_back(user); - } - } - return true; - }; - return visit(value); -} - -SmallVector matrix(PatternRewriter &b, MXFP8MemMatmulOp op, Value r, - Value w, Value mean, Value epsilon) { - Location loc = op.getLoc(); - int64_t width = op.getReductionK(), chunk = op.getTileK(); - int64_t columns = op.getTileN(), bytes = op.getBankBytes(); - auto outputType = cast(op.getOutput().getType()); - int64_t outputWidth = outputType.getDimSize(1); - int64_t chunks = (width + chunk - 1) / chunk; - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value four = b.create(loc, 4); - Value limit = b.create(loc, outputWidth); - Value step = b.create(loc, columns); - auto loop = b.create(loc, zero, limit, step, ValueRange{r, w}); - b.setInsertionPointToStart(loop.getBody()); - r = loop.getRegionIterArgs()[0]; - w = loop.getRegionIterArgs()[1]; - Value n = loop.getInductionVar(); - Value panel = b.create(loc, n, step); - for (int64_t part = 0; part < chunks; ++part) { - int64_t count = std::min(chunk, width - part * chunk); - Value offset = b.create( - loc, - b.create( - loc, - b.create( - loc, panel, b.create(loc, chunks)), - b.create(loc, part)), - b.create(loc, bytes)); - auto slice = b.create( - loc, op.getWeight(), SmallVector{offset}, - SmallVector{b.getIndexAttr(bytes)}, - SmallVector{b.getIndexAttr(1)}); - Value view = b.create( - loc, ArrayRef{bytes / 16, 16}, slice, - ArrayRef{{0, 1}}); - r = b.create( - loc, b.getI64Type(), view, r, - createI64Const(b, loc, columns * count * 33 / 32 / 16), - createI64Const(b, loc, 1), b.getI64IntegerAttr(0)); - auto window = b.create( - loc, b.getI64Type(), b.getI64Type(), r, w, - FlatSymbolRefAttr::get(b.getContext(), "rvv_norm_window"), - ValueRange{createI64Const(b, loc, 1), createI64Const(b, loc, width), - createI64Const(b, loc, columns), - createI64Const(b, loc, count), createI64Const(b, loc, width), - createI64Const(b, loc, part * chunk), - createI64Const(b, loc, part == 0), - createI64Const(b, loc, part == chunks - 1), - createI64Const(b, loc, bytes), mean, epsilon}); - r = window.getReadOut(); - w = window.getWriteOut(); - } - Value end = b.create(loc, step, - b.create(loc, limit, n)); - Value rows = b.create(loc, end, four); - SmallVector offsets{b.getIndexAttr(0), n}; - SmallVector sizes{b.getIndexAttr(1), end}; - SmallVector strides(2, b.getIndexAttr(1)); - auto sliceType = memref::SubViewOp::inferRankReducedResultType( - {ShapedType::kDynamic}, outputType, offsets, sizes, strides); - Value destination = b.create( - loc, sliceType, op.getOutput(), offsets, sizes, strides); - Value dmaView = b.create( - loc, ArrayRef{ShapedType::kDynamic, 4}, destination, - ArrayRef{{0, 1}}, - SmallVector{rows, b.getIndexAttr(4)}); - w = b.create( - loc, b.getI64Type(), dmaView, w, - b.create(loc, b.getI64Type(), rows), - createI64Const(b, loc, 1), b.getI64IntegerAttr(1)); - b.create(loc, ValueRange{r, w}); - b.setInsertionPointAfter(loop); - b.eraseOp(op); - return {loop.getResult(0), loop.getResult(1)}; -} - -class FuseNormQuantWindow : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(func::CallOp quant, - PatternRewriter &b) const override { - if (quant.getCallee() != "mxfp8_quant" || quant.getNumOperands() != 5) - return failure(); - Value normBuffer = origin(quant.getOperand(0)); - Value packedBuffer = origin(quant.getOperand(1)); - auto normType = dyn_cast(normBuffer.getType()); - auto packedType = dyn_cast(packedBuffer.getType()); - if (!normBuffer.getDefiningOp() || - !packedBuffer.getDefiningOp() || !normType || - !packedType || !normType.hasStaticShape() || normType.getRank() < 1 || - !packedType.hasStaticShape() || !normType.getElementType().isF32() || - !packedType.getElementType().isInteger(8)) - return failure(); - func::CallOp norm; - for (auto call : quant->getBlock()->getOps()) - if (call.getCallee() == "rvv_norm" && call.getNumOperands() == 5 && - origin(call.getOperand(0)) == normBuffer && - call->isBeforeInBlock(quant)) { - if (norm) - return failure(); - norm = call; - } - if (!norm) - return failure(); - SmallVector matrices; - for (auto op : quant->getBlock()->getOps()) - if (origin(op.getInput()) == packedBuffer) - matrices.push_back(op); - if (matrices.empty() || matrices.size() > 2) - return failure(); - if (matrices.size() == 2 && !matrices[0]->isBeforeInBlock(matrices[1])) - std::swap(matrices[0], matrices[1]); - const auto &target = buckyball_target::getBuckyballTarget(); - if (!target.rvvEnabled || target.bankWidthBits != 128 || - target.bankNum < 5 || - !llvm::any_of( - target.isa, - [](const auto &entry) { return entry.mnemonic == "MXQUANT"; }) || - !llvm::any_of( - target.isa, - [](const auto &entry) { - return entry.mnemonic == "MXMM_MXFP8_WINDOW"; - })) - return failure(); - auto module = quant->getParentOfType(); - auto normFunction = module.lookupSymbol("rvv_norm"); - auto quantFunction = module.lookupSymbol("mxfp8_quant"); - if (!normFunction || !normFunction.isExternal() || !quantFunction || - !quantFunction.isExternal()) - return failure(); - int64_t width = matrices[0].getReductionK(), chunk = matrices[0].getTileK(); - int64_t activationChunk = matrices[0].getActivationTileK(); - int64_t bytes = target.bankDepth * 16; - if (chunk <= 0 || chunk % 32 || chunk >= 4096 || width <= 0 || width % 32 || - (activationChunk != chunk && activationChunk != width)) - return failure(); - int64_t chunks = (width + chunk - 1) / chunk; - if (width <= 0 || width % 32 || width * 4 > bytes || - width + width / 32 > bytes || normType.getNumElements() != width || - normType.getDimSize(normType.getRank() - 1) != width || - packedType.getRank() != 1 || - packedType.getNumElements() != - ((width + activationChunk - 1) / activationChunk) * bytes) - return failure(); - const int64_t quantParameters[] = {1, activationChunk, bytes}; - for (size_t index = 0; index < 3; ++index) { - auto constant = - quant.getOperand(index + 2).getDefiningOp(); - if (!constant || cast(constant.getValue()).getInt() != - quantParameters[index]) - return failure(); - } - for (Operation *cursor = norm.getOperation(); - cursor != matrices.back().getOperation()->getNextNode(); - cursor = cursor->getNextNode()) { - if (cursor == norm || cursor == quant || cursor == matrices[0] || - cursor == matrices.back()) - continue; - if (isa(cursor)) - continue; - if (isa(cursor) || cursor->getNumRegions() || - !isMemoryEffectFree(cursor)) - return failure(); - } - for (auto op : matrices) { - auto output = cast(op.getOutput().getType()); - auto weight = cast(op.getWeight().getType()); - int64_t columns = op.getTileN(); - if (!output.hasStaticShape() || output.getRank() != 2 || - output.getDimSize(0) != 1 || output.getDimSize(1) % 4 || - output.getStridesAndOffset().first.back() != 1 || - !op.getOutput().getDefiningOp() || columns <= 0 || - columns % 16 || op.getReductionK() != width || - op.getTileK() != chunk || - op.getActivationTileK() != matrices[0].getActivationTileK() || - op.getBankBytes() != bytes || columns * chunk * 33 / 32 > bytes || - columns * 4 > bytes || weight.getRank() != 1 || - weight.getStridesAndOffset().first[0] != 1 || - weight.getDimSize(0) != - ((output.getDimSize(1) + columns - 1) / columns) * chunks * - bytes || - !quant->isBeforeInBlock(op)) - return failure(); - } - SmallVector views; - SmallVector packedUsers{quant.getOperation()}; - for (auto matrix : matrices) - packedUsers.push_back(matrix.getOperation()); - if (!privateUses(normBuffer, {norm.getOperation(), quant.getOperation()}, - views) || - !privateUses(packedBuffer, packedUsers, views)) - return failure(); - Value input = norm.getOperand(1), gamma = norm.getOperand(2); - for (Value *value : {&input, &gamma}) - while (auto cast = value->getDefiningOp()) - *value = cast.getSource(); - auto inputType = dyn_cast(input.getType()); - if (!inputType || inputType.getRank() < 1 || !inputType.hasStaticShape() || - inputType.getDimSize(inputType.getRank() - 1) != width || - origin(input) == normBuffer || origin(gamma) == normBuffer || - origin(input) == packedBuffer || origin(gamma) == packedBuffer) - return failure(); - for (Value value : {input, gamma}) { - auto type = dyn_cast(value.getType()); - if (!type || !type.hasStaticShape() || type.getNumElements() != width || - !type.getElementType().isF32()) - return failure(); - auto strides = type.getStridesAndOffset().first; - int64_t stride = 1; - for (int64_t axis = type.getRank() - 1; axis >= 0; --axis) { - if (type.getDimSize(axis) != 1 && strides[axis] != stride) - return failure(); - stride *= type.getDimSize(axis); - } - } - Location loc = norm.getLoc(); - b.setInsertionPoint(norm); - Value d = allocBank(b, loc, 1, 1), n = allocBank(b, loc, 1, 1), - x = allocBank(b, loc, 1, 1), g = allocBank(b, loc, 1, 1), - a = allocBank(b, loc, 1, 1); - auto rowView = [&](Value value) { - auto type = cast(value.getType()); - ReassociationIndices axes; - for (int64_t axis = 0; axis < type.getRank(); ++axis) - axes.push_back(axis); - if (type.getRank() != 1) - value = b.create( - loc, value, ArrayRef{axes}); - return Value(b.create( - loc, ArrayRef{width / 4, 4}, value, - ArrayRef{{0, 1}})); - }; - x = mvinBank(b, loc, rowView(input), x, width / 4); - g = mvinBank(b, loc, rowView(gamma), g, width / 4); - Value r = b.create(loc, b.getI64Type(), g, x); - Value w = b.create(loc, b.getI64Type(), n, d); - w = b.create(loc, b.getI64Type(), a, w); - auto initialized = b.create( - loc, b.getI64Type(), b.getI64Type(), r, w, - FlatSymbolRefAttr::get(b.getContext(), "rvv_norm_window"), - ValueRange{createI64Const(b, loc, 0), createI64Const(b, loc, width), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - createI64Const(b, loc, width), createI64Const(b, loc, 0), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - createI64Const(b, loc, bytes), norm.getOperand(3), - norm.getOperand(4)}); - Value mean = norm.getOperand(3), epsilon = norm.getOperand(4); - SmallVector state{initialized.getReadOut(), - initialized.getWriteOut()}; - b.eraseOp(norm); - b.eraseOp(quant); - for (auto op : matrices) { - b.setInsertionPoint(op); - state = matrix(b, op, state[0], state[1], mean, epsilon); - } - auto finished = b.create( - loc, b.getI64Type(), b.getI64Type(), state[0], state[1], - FlatSymbolRefAttr::get(b.getContext(), "rvv_norm_window"), - ValueRange{createI64Const(b, loc, 2), createI64Const(b, loc, width), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - createI64Const(b, loc, width), createI64Const(b, loc, 0), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - createI64Const(b, loc, bytes), mean, epsilon}); - releaseBank(b, loc, finished.getReadOut()); - releaseBank(b, loc, finished.getWriteOut()); - for (Operation *view : views) - b.eraseOp(view); - b.eraseOp(normBuffer.getDefiningOp()); - b.eraseOp(packedBuffer.getDefiningOp()); - return success(); - } -}; - -class NormQuantWindowPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(NormQuantWindowPass) - StringRef getArgument() const final { return "fuse-norm-quant-window"; } - StringRef getDescription() const final { - return "Keep full-row Norm and quantized activation in banks across paired " - "matrix consumers."; - } - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert(); - } - void runOnOperation() override { - RewritePatternSet patterns(&getContext()); - patterns.add(&getContext()); - if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) - signalPassFailure(); - } -}; -} // namespace -namespace mlir::buddy { -void registerNormQuantWindowPass() { PassRegistration(); } -} // namespace mlir::buddy diff --git a/waste/examples/components/balls/mxquant/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/mxquant/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index 8a341eae1..000000000 --- a/waste/examples/components/balls/mxquant/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "mlir/IR/PatternMatch.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankMxquantOp::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankMxquantOp op = *this; - rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), - op.getIter()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); -} diff --git a/waste/examples/components/balls/mxquant/stack/src/Dialect/Buckyball/MxquantBall.td b/waste/examples/components/balls/mxquant/stack/src/Dialect/Buckyball/MxquantBall.td deleted file mode 100644 index 5ba8fbe88..000000000 --- a/waste/examples/components/balls/mxquant/stack/src/Dialect/Buckyball/MxquantBall.td +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef MXQUANTBALL_OPS -#define MXQUANTBALL_OPS - -include "Buckyball.td" - -def MxquantOp : Buckyball_Op<"mxquant"> { - let summary = "Finite FP32 to block32 MXFP8 quantization"; - let arguments = (ins I64:$inputBankId, I64:$outputBankId, I64:$iter); - let assemblyFormat = "$inputBankId `,` $outputBankId `,` $iter " - "prop-dict attr-dict `:` type($inputBankId)"; -} - -def BankMxquantOp : Buckyball_Op<"bank_mxquant", [DeclareOpInterfaceMethods, MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "FP32 to block32 MXFP8 quantization with virtual banks"; - let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$inBank $outBank $iter prop-dict attr-dict `:` " - "type($inBank) type($outBank) type($iter)"; -} - -#endif diff --git a/waste/examples/components/balls/mxquant/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/mxquant/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 47654dab4..000000000 --- a/waste/examples/components/balls/mxquant/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -struct MxquantLowering : public ConvertOpToLLVMPattern { - MxquantLowering(LLVMTypeConverter &converter, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(MxquantOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - if (buckyball_target::getBuckyballTarget().bankWidthBits != 128) - return op.emitError("mxquant: requires 16-byte bank rows"); - if (auto constant = op.getIter().getDefiningOp()) { - int64_t count = cast(constant.getValue()).getInt(); - if (count <= 0 || count % 32) - return op.emitError("mxquant: count must be a positive multiple of 32"); - if (count > bankDepth * 4 || count + count / 32 > bankDepth * 16) - return op.emitError("mxquant: bank footprint exceeds capacity"); - } - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, cstI64(rewriter, loc, 0), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("MXQUANT"))); - return success(); - } - -private: - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateMxquantBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, - int64_t bankDepth) { - patterns.add(converter, bankDepth); -} -void configureMxquantBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/mxquant/stack/src/Passes.cpp b/waste/examples/components/balls/mxquant/stack/src/Passes.cpp deleted file mode 100644 index bcc56d4e0..000000000 --- a/waste/examples/components/balls/mxquant/stack/src/Passes.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- Passes.cpp - MxquantBall passes ------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -namespace buckyball { -void populateMxquantBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureMxquantBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LowerMxquantPass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerMxquantPass) - StringRef getArgument() const final { return "lower-mxquant-ball"; } - StringRef getDescription() const final { - return "Lower MxquantBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateMxquantBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureMxquantBallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - return true; -}(); diff --git a/waste/examples/components/balls/redmule/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/redmule/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index 8917a85d6..000000000 --- a/waste/examples/components/balls/redmule/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankRedmuleF16Op::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankRedmuleF16Op op = *this; - rewriter.create(op.getLoc(), op.getA(), op.getB(), op.getC(), - op.getRowsAttr(), op.getKAttr(), - op.getOp1BaseAttr(), op.getOp2BaseAttr(), - op.getWrBaseAttr(), op.getFirstAttr()); - rewriter.replaceOp(op, op.getC()); - return success(); -} diff --git a/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/IR/Redmule.cpp b/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/IR/Redmule.cpp deleted file mode 100644 index a7697234b..000000000 --- a/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/IR/Redmule.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "mlir/Dialect/Arith/IR/Arith.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -template LogicalResult verifyMatmul(Op op) { - if (op.getRows() <= 0 || op.getRows() > 4095) - return op.emitOpError("rows must be in [1, 4095]"); - if (op.getK() <= 0 || op.getK() % 8 || op.getK() >= (int64_t(1) << 34)) - return op.emitOpError( - "K must be a positive multiple of eight within 34 bits"); - for (auto base : {op.getOp1Base(), op.getOp2Base(), op.getWrBase()}) - if (base < 0 || base > 1023) - return op.emitOpError("bank row bases must be in [0, 1023]"); - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || - op.getOp1Base() + op.getRows() * (op.getK() / 8) > target.bankDepth || - op.getOp2Base() + op.getK() > target.bankDepth || - op.getWrBase() + op.getRows() > target.bankDepth) - return op.emitOpError("matrix window exceeds target 128-bit bank capacity"); - auto banks = op.getOperands(); - for (unsigned i = 0; i < banks.size(); ++i) { - auto lhs = banks[i].template getDefiningOp(); - if (lhs && (lhs.value() < 0 || lhs.value() > 1023)) - return op.emitOpError("bank ID must be in [0, 1023]"); - for (unsigned j = 0; j < i; ++j) { - auto rhs = banks[j].template getDefiningOp(); - if (banks[i] == banks[j] || (lhs && rhs && lhs.value() == rhs.value())) - return op.emitOpError("A, B and C banks must differ"); - } - } - return success(); -} -} // namespace - -LogicalResult RedmuleF16Op::verify() { return verifyMatmul(*this); } -LogicalResult BankRedmuleF16Op::verify() { return verifyMatmul(*this); } diff --git a/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/RedmuleBall.td b/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/RedmuleBall.td deleted file mode 100644 index 2c6dbca5d..000000000 --- a/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/RedmuleBall.td +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef REDMULEBALL_OPS -#define REDMULEBALL_OPS -include "Buckyball.td" - -def RedmuleF16Op : Buckyball_Op<"redmule_f16", [MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "Ordered FP16 FMA for A[M,K], B[K,8], C[M,8] bank windows"; - let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$k, - I64Attr:$op1_base, I64Attr:$op2_base, I64Attr:$wr_base, - BoolAttr:$first); - let hasVerifier = 1; - let assemblyFormat = "$a `,` $b `,` $c prop-dict attr-dict `:` type($a)"; -} - -def BankRedmuleF16Op : Buckyball_Op<"bank_redmule_f16", [DeclareOpInterfaceMethods, MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "FP16 bank matrix product with an SSA output bank handle"; - let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$k, - I64Attr:$op1_base, I64Attr:$op2_base, I64Attr:$wr_base, - BoolAttr:$first); - let results = (outs I64:$cOut); - let hasVerifier = 1; - let assemblyFormat = "$a $b $c prop-dict attr-dict `:` type($a) type($b) type($c)"; -} -#endif diff --git a/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 180b99a6c..000000000 --- a/waste/examples/components/balls/redmule/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -class RedmuleLowering : public ConvertOpToLLVMPattern { -public: - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - LogicalResult - matchAndRewrite(RedmuleF16Op op, OpAdaptor operands, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - Value rs1 = - packRs1BanksIter(rewriter, loc, operands.getA(), operands.getB(), - operands.getC(), cstI64(rewriter, loc, op.getK())); - uint64_t special = - uint64_t(op.getRows()) | uint64_t(op.getOp1Base()) << 12 | - uint64_t(op.getOp2Base()) << 22 | uint64_t(op.getWrBase()) << 32 | - uint64_t(op.getFirst()) << 42; - rewriter.replaceOpWithNewOp( - op, rs1, cstI64(rewriter, loc, special), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("REDMULE_F16"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateRedmuleBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool, int64_t) { - patterns.add(converter); -} -void configureRedmuleBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/redmule/stack/src/Passes.cpp b/waste/examples/components/balls/redmule/stack/src/Passes.cpp deleted file mode 100644 index 67b8443c8..000000000 --- a/waste/examples/components/balls/redmule/stack/src/Passes.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- Passes.cpp - RedmuleBall passes ------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -namespace buckyball { -void populateRedmuleBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureRedmuleBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LowerRedmulePass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerRedmulePass) - StringRef getArgument() const final { return "lower-redmule-ball"; } - StringRef getDescription() const final { - return "Lower RedmuleBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateRedmuleBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureRedmuleBallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - return true; -}(); diff --git a/waste/examples/components/balls/relu/stack/src/Backend/include/IntrinsicsRISCVReluBall.td b/waste/examples/components/balls/relu/stack/src/Backend/include/IntrinsicsRISCVReluBall.td deleted file mode 100644 index 803eccdf1..000000000 --- a/waste/examples/components/balls/relu/stack/src/Backend/include/IntrinsicsRISCVReluBall.td +++ /dev/null @@ -1,4 +0,0 @@ -//===- IntrinsicsRISCVReluBall.td - ReluBall intrinsics ------------------===// - -let TargetPrefix = "riscv" in -def int_riscv_bb_relu : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; diff --git a/waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index f7adc1f07..000000000 --- a/waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankReluOp::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankReluOp op = *this; - rewriter.create(op.getLoc(), op.getBank(), op.getGroup(), - op.getIter(), op.getStride()); - rewriter.replaceOp(op, op.getBank()); - return success(); -} diff --git a/waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp b/waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp deleted file mode 100644 index 2897421f8..000000000 --- a/waste/examples/components/balls/relu/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp +++ /dev/null @@ -1,102 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -constexpr int64_t kValuesPerLine = 4; - -} // namespace - -LogicalResult ReluMatrixOp::lowerToBankSSA(PatternRewriter &b) { - ReluMatrixOp op = *this; - auto inputType = dyn_cast(op.getInput().getType()); - auto outputType = dyn_cast(op.getOutput().getType()); - if (!inputType || !outputType || !inputType.hasStaticShape() || - inputType != outputType || !inputType.getElementType().isInteger(32)) - return op.emitError("requires matching static memref"); - - int64_t rows = inputType.getShape()[0]; - int64_t columns = inputType.getShape()[1]; - if (rows <= 0 || columns <= 0 || rows % 16 || columns % 16) - return op.emitError("requires positive 16-aligned dimensions"); - - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth <= 0 || - target.bankDepth % 16) - return op.emitError( - "ReluBall requires a 128-bit bank with 16-aligned depth"); - - Location loc = op.getLoc(); - int64_t lines = target.bankDepth; - int64_t values = lines * kValuesPerLine; - int64_t total = rows * columns; - Value bank = allocBank(b, loc, 1, 1); - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value four = b.create(loc, kValuesPerLine); - Value columnsValue = b.create(loc, columns); - auto packType = MemRefType::get({lines, kValuesPerLine}, b.getI32Type()); - - for (int64_t base = 0; base < total; base += values) { - int64_t count = std::min(values, total - base); - Value inputPack = b.create(loc, packType); - Value outputPack = b.create(loc, packType); - Value zeroValue = b.create(loc, 0, 32); - b.create(loc, zeroValue, inputPack); - - Value countValue = b.create(loc, count); - Value baseValue = b.create(loc, base); - auto copyIn = b.create(loc, zero, countValue, one); - b.setInsertionPointToStart(copyIn.getBody()); - Value index = copyIn.getInductionVar(); - Value source = b.create(loc, baseValue, index); - Value row = b.create(loc, source, columnsValue); - Value column = b.create(loc, source, columnsValue); - Value line = b.create(loc, index, four); - Value lane = b.create(loc, index, four); - Value value = - b.create(loc, op.getInput(), ValueRange{row, column}); - b.create(loc, value, inputPack, ValueRange{line, lane}); - b.setInsertionPointAfter(copyIn); - - Value loaded = mvinBank(b, loc, inputPack, bank, lines); - Value result = b.create( - loc, bank.getType(), loaded, createI64Const(b, loc, 0), - createI64Const(b, loc, lines), createI64Const(b, loc, lines)); - bank = mvoutBank(b, loc, outputPack, result, lines); - - auto copyOut = b.create(loc, zero, countValue, one); - b.setInsertionPointToStart(copyOut.getBody()); - index = copyOut.getInductionVar(); - source = b.create(loc, baseValue, index); - row = b.create(loc, source, columnsValue); - column = b.create(loc, source, columnsValue); - line = b.create(loc, index, four); - lane = b.create(loc, index, four); - value = b.create(loc, outputPack, ValueRange{line, lane}); - b.create(loc, value, op.getOutput(), - ValueRange{row, column}); - b.setInsertionPointAfter(copyOut); - b.create(loc, inputPack); - b.create(loc, outputPack); - } - - releaseBank(b, loc, bank); - b.eraseOp(op); - return success(); -} diff --git a/waste/examples/components/balls/relu/stack/src/Conversion/LowerLinalgToBuckyball/ReluPatterns.cpp b/waste/examples/components/balls/relu/stack/src/Conversion/LowerLinalgToBuckyball/ReluPatterns.cpp deleted file mode 100644 index f001c6cfd..000000000 --- a/waste/examples/components/balls/relu/stack/src/Conversion/LowerLinalgToBuckyball/ReluPatterns.cpp +++ /dev/null @@ -1,118 +0,0 @@ -//===- ReluPatterns.cpp - signed INT32 ReLU -> buckyball.relu_matrix -----===// -// -// Claims linalg.generic max(input, 0) over static memref. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/BuckyballOps.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" -#include "mlir/IR/SymbolTable.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -// The zero operand must be a constant zero splat global. -bool isZeroGlobal(Value value) { - auto global = value.getDefiningOp(); - if (!global) - return false; - auto constant = SymbolTable::lookupNearestSymbolFrom( - global, global.getNameAttr()); - auto values = - constant - ? dyn_cast_or_null(constant.getConstantInitValue()) - : nullptr; - return values && values.isSplat() && values.getSplatValue().isZero(); -} - -class ReluLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(linalg::GenericOp op, - PatternRewriter &b) const override { - Block &body = op.getRegion().front(); - if (body.getOperations().size() != 2 || op.getInputs().size() != 2 || - op.getOutputs().size() != 1) - return failure(); - auto maximum = dyn_cast(body.front()); - ValueRange args = body.getArguments(); - if (!maximum || body.back().getOperand(0) != maximum.getResult() || - !((maximum.getLhs() == args[0] && maximum.getRhs() == args[1]) || - (maximum.getLhs() == args[1] && maximum.getRhs() == args[0]))) - return failure(); - Value input = op.getInputs()[0]; - Value output = op.getOutputs()[0]; - auto inputType = dyn_cast(input.getType()); - if (!inputType || !inputType.hasStaticShape() || inputType.getRank() != 2 || - !inputType.getElementType().isInteger(32) || - op.getInputs()[1].getType() != inputType || - output.getType() != inputType || !isZeroGlobal(op.getInputs()[1]) || - !llvm::all_of(op.getIndexingMapsArray(), - [](AffineMap map) { return map.isIdentity(); })) - return failure(); - - int64_t rows = inputType.getShape()[0]; - int64_t columns = inputType.getShape()[1]; - int64_t paddedRows = llvm::alignTo(rows, 16); - int64_t paddedColumns = llvm::alignTo(columns, 16); - Location loc = op.getLoc(); - if (paddedRows == rows && paddedColumns == columns) { - b.create(loc, input, output); - b.eraseOp(op); - return success(); - } - - auto paddedType = - MemRefType::get({paddedRows, paddedColumns}, b.getI32Type()); - Value paddedInput = b.create(loc, paddedType); - Value paddedOutput = b.create(loc, paddedType); - Value zeroValue = b.create(loc, 0, 32); - b.create(loc, zeroValue, paddedInput); - - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value rowLimit = b.create(loc, rows); - Value columnLimit = b.create(loc, columns); - auto copyInputRows = b.create(loc, zero, rowLimit, one); - b.setInsertionPointToStart(copyInputRows.getBody()); - Value row = copyInputRows.getInductionVar(); - auto copyInputColumns = b.create(loc, zero, columnLimit, one); - b.setInsertionPointToStart(copyInputColumns.getBody()); - Value column = copyInputColumns.getInductionVar(); - Value value = b.create(loc, input, ValueRange{row, column}); - b.create(loc, value, paddedInput, ValueRange{row, column}); - b.setInsertionPointAfter(copyInputRows); - - b.create(loc, paddedInput, paddedOutput); - - auto copyOutputRows = b.create(loc, zero, rowLimit, one); - b.setInsertionPointToStart(copyOutputRows.getBody()); - row = copyOutputRows.getInductionVar(); - auto copyOutputColumns = b.create(loc, zero, columnLimit, one); - b.setInsertionPointToStart(copyOutputColumns.getBody()); - column = copyOutputColumns.getInductionVar(); - value = - b.create(loc, paddedOutput, ValueRange{row, column}); - b.create(loc, value, output, ValueRange{row, column}); - b.setInsertionPointAfter(copyOutputRows); - - b.create(loc, paddedInput); - b.create(loc, paddedOutput); - b.eraseOp(op); - return success(); - } -}; -} // namespace - -namespace mlir::buddy { -void populateReluBallLinalgPatterns(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} -} // namespace mlir::buddy diff --git a/waste/examples/components/balls/relu/stack/src/Dialect/Buckyball/ReluBall.td b/waste/examples/components/balls/relu/stack/src/Dialect/Buckyball/ReluBall.td deleted file mode 100644 index 71934ff29..000000000 --- a/waste/examples/components/balls/relu/stack/src/Dialect/Buckyball/ReluBall.td +++ /dev/null @@ -1,31 +0,0 @@ -//===-- ReluBall.td - ReluBall operations ------------------*- tablegen -*-===// - -#ifndef RELUBALL_OPS -#define RELUBALL_OPS - -include "Buckyball.td" - -def ReluMatrixOp : Buckyball_Op<"relu_matrix", [DeclareOpInterfaceMethods]> { - let summary = "Logical INT32 ReLU matrix tile"; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, - MemRefRankOf<[AnyType], [2]>:$output); - let assemblyFormat = [{$input $output attr-dict `:` - type($input) type($output)}]; -} - -def ReluOp : Buckyball_Op<"relu"> { - let summary = "In-place INT32 ReLU on one physical bank"; - let arguments = (ins I64:$bankId, I64:$group, I64:$iter, I64:$stride); - let assemblyFormat = "$bankId `,` $group `,` $iter `,` $stride attr-dict `:` " - "type($bankId)"; -} - -def BankReluOp : Buckyball_Op<"bank_relu", [DeclareOpInterfaceMethods]> { - let summary = "In-place INT32 ReLU with a virtual bank handle"; - let arguments = (ins I64:$bank, I64:$group, I64:$iter, I64:$stride); - let results = (outs I64:$bankOut); - let assemblyFormat = "$bank $group $iter $stride attr-dict `:` " - "type($bank) type($group) type($iter) type($stride)"; -} - -#endif // RELUBALL_OPS diff --git a/waste/examples/components/balls/relu/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/relu/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index d7dc345f5..000000000 --- a/waste/examples/components/balls/relu/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" - -#include "mlir/IR/PatternMatch.h" -#include - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -struct ReluLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(ReluOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - Value rs1 = - packRs1BanksIter(rewriter, loc, adaptor.getBankId(), adaptor.getGroup(), - adaptor.getBankId(), adaptor.getIter()); - rs1 = - rewriter.create(loc, rs1, cstI64(rewriter, loc, INT64_MIN)); - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getStride(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("RELU"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateReluBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns, - bool, int64_t) { - patterns.add(converter); -} - -void configureReluBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool) { - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/relu/stack/src/Passes.cpp b/waste/examples/components/balls/relu/stack/src/Passes.cpp deleted file mode 100644 index 972f95a5c..000000000 --- a/waste/examples/components/balls/relu/stack/src/Passes.cpp +++ /dev/null @@ -1,55 +0,0 @@ -//===- Passes.cpp - ReluBall passes ---------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -void populateReluBallLinalgPatterns(RewritePatternSet &patterns); -namespace buckyball { -void populateReluBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureReluBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LinalgToReluPass - : public mlir::buddy::LinalgToBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LinalgToReluPass) - StringRef getArgument() const final { return "convert-linalg-to-relu-ball"; } - StringRef getDescription() const final { - return "Lower the Linalg operations ReluBall executes."; - } - void populate(RewritePatternSet &patterns, int64_t, int64_t bankDepth, - int64_t) const { - mlir::buddy::populateReluBallLinalgPatterns(patterns); - } -}; - -class LowerReluPass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerReluPass) - StringRef getArgument() const final { return "lower-relu-ball"; } - StringRef getDescription() const final { - return "Lower ReluBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateReluBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureReluBallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - PassRegistration(); - return true; -}(); diff --git a/waste/examples/components/balls/smatmul/stack/src/Backend/include/IntrinsicsRISCVSMatMulBall.td b/waste/examples/components/balls/smatmul/stack/src/Backend/include/IntrinsicsRISCVSMatMulBall.td deleted file mode 100644 index 6b92a9730..000000000 --- a/waste/examples/components/balls/smatmul/stack/src/Backend/include/IntrinsicsRISCVSMatMulBall.td +++ /dev/null @@ -1,7 +0,0 @@ -//===- IntrinsicsRISCVSMatMulBall.td - SMatMulBall intrinsics --------------===// -// -// Stable LLVM intrinsic for SMatMulBall is not wired yet. Pebble uses -// CustomIntrOp(funct7 from Core ballISA SMATMUL). Keep this file as a placeholder for a future -// int_riscv_bb_smatmul addition to the RISC-V backend. -// -//===----------------------------------------------------------------------===// diff --git a/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index 8e824b317..000000000 --- a/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,31 +0,0 @@ -//===- AssignPhysicalBanks.cpp - Matrix bank assignment -------------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankSMatMulOp::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankSMatMulOp op = *this; - rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), - op.getWrBank(), op.getConfig(), op.getFirst(), - op.getLast(), op.getOutputBase()); - rewriter.replaceOp(op, op.getWrBank()); - return success(); -} - -LogicalResult -BankSMatMulBiasOp::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankSMatMulBiasOp op = *this; - rewriter.create(op.getLoc(), op.getBiasBank(), - op.getInputBase()); - rewriter.replaceOp(op, op.getBiasBank()); - return success(); -} diff --git a/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp b/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp deleted file mode 100644 index 0672e722d..000000000 --- a/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPatterns.cpp +++ /dev/null @@ -1,356 +0,0 @@ -//===- LowerBuckyballToBankSSAPatterns.cpp - Mega MatMul to banks --------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -LogicalResult lowerMatmulToBanks(MegaMatmulOp op, PatternRewriter &rewriter); -} // namespace mlir::buddy - -namespace { - -constexpr int64_t kTile = 16; -constexpr int64_t kInt32RowsPerTile = 64; - -class MegaMatmulToBankSSAPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(MegaMatmulOp op, - PatternRewriter &b) const override { - Location loc = op.getLoc(); - auto aTy = dyn_cast(op.getInput().getType()); - auto wTy = dyn_cast(op.getWeight().getType()); - auto biasTy = dyn_cast(op.getBias().getType()); - auto scaleTy = dyn_cast(op.getScale().getType()); - auto lutTy = dyn_cast(op.getLut().getType()); - auto outTy = dyn_cast(op.getOutput().getType()); - auto kernel = op->getParentOfType(); - if (!kernel) - return op.emitError( - "Mega MatMul must be nested in buckyball.mega_kernel"); - bool finalOutput = op.getOutput() == kernel.getOutput(); - if (!aTy || !wTy || !biasTy || !scaleTy || !lutTy || !outTy || - !aTy.hasStaticShape() || !wTy.hasStaticShape() || - !biasTy.hasStaticShape() || !scaleTy.hasStaticShape() || - !lutTy.hasStaticShape() || !outTy.hasStaticShape()) - return op.emitError("Mega MatMul requires static memrefs"); - if (!aTy.getElementType().isInteger(8) || - !wTy.getElementType().isInteger(8) || - !biasTy.getElementType().isInteger(32) || - !scaleTy.getElementType().isF32() || - !lutTy.getElementType().isInteger(8) || lutTy.getRank() != 1 || - lutTy.getShape()[0] != (op.getActivation() == 2 ? 256 : 1) || - op.getActivation() < 0 || op.getActivation() > 2 || - (finalOutput && op.getActivation() == 2) || - (finalOutput ? !outTy.getElementType().isF32() - : !outTy.getElementType().isInteger(8))) - return op.emitError( - "Mega MatMul input/weight must be INT8, bias INT32, scale FP32, " - "and only the last stage may output FP32"); - - int64_t M = aTy.getShape()[0]; - int64_t K = aTy.getShape()[1]; - int64_t N = wTy.getShape()[1]; - if (M <= 0 || K <= 0 || N <= 0 || wTy.getShape()[0] != K || - biasTy.getShape()[0] != N || scaleTy.getShape()[0] != N || - outTy.getShape()[0] != M || outTy.getShape()[1] != N) - return op.emitError("Mega MatMul shape mismatch"); - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth < kInt32RowsPerTile || - target.bankDepth % kInt32RowsPerTile) - return op.emitError( - "Mega MatMul requires 128-bit banks whose depth is a positive " - "multiple of 64 rows"); - if (buckyball_target::getBuckyballBallMapping("SMATMUL_OS").outBW != 1) - return op.emitError("Mega MatMul requires SMatMulBall outBW=1"); - - const int64_t hardwareRows = - buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); - if (hardwareRows <= 0 || kTile % hardwareRows) - return op.emitError( - "SMatMul tileRows must divide the 16-row software tile"); - const int64_t tileRows = M == 1 ? 1 : kTile; - const int64_t resultRows = tileRows * 4; - int64_t paddedM = (M + tileRows - 1) / tileRows * tileRows; - int64_t paddedK = (K + kTile - 1) / kTile * kTile; - int64_t kChunk = target.bankDepth / kTile * kTile; - if (kChunk <= 0) - return op.emitError("bank depth cannot hold one K tile"); - // outputBase is a six-bit row index; one 16x16 result uses 64 rows. - constexpr int64_t tilesPerResultBank = 1; - - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value tileSize = b.create(loc, kTile); - Value columns = b.create(loc, N); - Value zeroI8 = - b.create(loc, b.getI8Type(), b.getI8IntegerAttr(0)); - auto matrixRowLoop = b.create( - loc, zero, b.create(loc, paddedM), - b.create(loc, tileRows)); - b.setInsertionPointToStart(matrixRowLoop.getBody()); - Value m0 = matrixRowLoop.getInductionVar(); - Value validRows = b.create( - loc, b.create(loc, tileRows), - b.create(loc, b.create(loc, M), - m0)); - SmallVector inputPacks; - { - for (int64_t k0 = 0; k0 < paddedK; k0 += kChunk) { - int64_t thisK = std::min(kChunk, paddedK - k0); - int64_t validK = std::min(thisK, K - k0); - auto aPackTy = - MemRefType::get({thisK / kTile * tileRows, kTile}, b.getI8Type()); - Value aPack = b.create(loc, aPackTy); - inputPacks.push_back(aPack); - b.create(loc, zeroI8, aPack); - - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value rowEnd = validRows; - Value kTileEnd = - b.create(loc, (validK + kTile - 1) / kTile); - auto ktLoop = b.create(loc, zero, kTileEnd, one); - b.setInsertionPointToStart(ktLoop.getBody()); - Value kt = ktLoop.getInductionVar(); - auto rowLoop = b.create(loc, zero, rowEnd, one); - b.setInsertionPointToStart(rowLoop.getBody()); - Value row = rowLoop.getInductionVar(); - Value remaining = b.create( - loc, b.create(loc, validK), - b.create( - loc, kt, b.create(loc, kTile))); - Value columnEnd = b.create( - loc, remaining, b.create(loc, kTile)); - auto columnLoop = b.create(loc, zero, columnEnd, one); - b.setInsertionPointToStart(columnLoop.getBody()); - Value column = columnLoop.getInductionVar(); - Value sourceRow = b.create(loc, m0, row); - Value sourceColumn = b.create( - loc, b.create(loc, k0), - b.create( - loc, - b.create( - loc, kt, b.create(loc, kTile)), - column)); - Value hardwareRowsValue = b.create( - loc, std::min(hardwareRows, tileRows)); - Value rowTile = b.create(loc, row, hardwareRowsValue); - Value rowInTile = b.create(loc, row, hardwareRowsValue); - Value tileIndex = b.create( - loc, - b.create( - loc, rowTile, - b.create(loc, thisK / kTile)), - kt); - Value packedRow = b.create( - loc, b.create(loc, tileIndex, hardwareRowsValue), - rowInTile); - Value value = b.create( - loc, op.getInput(), ValueRange{sourceRow, sourceColumn}); - b.create(loc, value, aPack, - ValueRange{packedRow, column}); - b.setInsertionPointAfter(ktLoop); - } - } - auto panelLoop = b.create(loc, zero, columns, tileSize); - b.setInsertionPointToStart(panelLoop.getBody()); - { - Value n0 = panelLoop.getInductionVar(); - Value thisN = b.create( - loc, tileSize, b.create(loc, columns, n0)); - Value aBank = allocBank(b, loc, 1, 1); - Value wBank = allocBank(b, loc, 1, 1); - auto biasPackTy = MemRefType::get({4, 4}, b.getI32Type()); - auto scalePackTy = MemRefType::get({4, 4}, b.getF32Type()); - Value biasPack = b.create(loc, biasPackTy); - Value scalePack = b.create(loc, scalePackTy); - Value zeroI32 = b.create(loc, b.getI32Type(), - b.getI32IntegerAttr(0)); - Value oneF32 = b.create(loc, b.getF32Type(), - b.getF32FloatAttr(1.0)); - b.create(loc, zeroI32, biasPack); - b.create(loc, oneF32, scalePack); - auto channelLoop = b.create(loc, zero, thisN, one); - b.setInsertionPointToStart(channelLoop.getBody()); - { - Value i = channelLoop.getInductionVar(); - Value four = b.create(loc, 4); - Value source = b.create(loc, n0, i); - Value row = b.create(loc, i, four); - Value column = b.create(loc, i, four); - Value bias = b.create(loc, op.getBias(), source); - Value scale = b.create(loc, op.getScale(), source); - b.create(loc, bias, biasPack, ValueRange{row, column}); - b.create(loc, scale, scalePack, - ValueRange{row, column}); - } - b.setInsertionPointAfter(channelLoop); - - Value biasBank = allocBank(b, loc, 1, 1); - Value biasLoaded = mvinBank(b, loc, biasPack, biasBank, 4); - Value biasState = b.create( - loc, biasLoaded.getType(), biasLoaded, createI64Const(b, loc, 0)); - Value scaleBank = allocBank(b, loc, 1, 1); - Value scaleLoaded = mvinBank(b, loc, scalePack, scaleBank, 4); - - { - int64_t tileCount = tilesPerResultBank; - Value resultBank = allocBank(b, loc, 1, 1); - Value resultState = resultBank; - SmallVector hostPacks; - - for (int64_t tile = 0; tile < tileCount; ++tile) { - for (int64_t k0 = 0; k0 < paddedK; k0 += kChunk) { - int64_t thisK = std::min(kChunk, paddedK - k0); - int64_t validK = std::min(thisK, std::max(0, K - k0)); - Value aPack = inputPacks[k0 / kChunk]; - Value aLoaded = - mvinBank(b, loc, aPack, aBank, thisK / kTile * tileRows); - Value wLoaded; - auto [weightStrides, weightOffset] = wTy.getStridesAndOffset(); - if (validK == thisK && N % kTile == 0 && weightStrides[1] == 1 && - weightStrides[0] > 0 && weightStrides[0] % kTile == 0) { - Value panel = b.create( - loc, op.getWeight(), - SmallVector{b.getIndexAttr(k0), n0}, - SmallVector{b.getIndexAttr(thisK), - b.getIndexAttr(kTile)}, - SmallVector{b.getIndexAttr(1), - b.getIndexAttr(1)}); - wLoaded = mvinBank(b, loc, panel, wBank, thisK, - weightStrides[0] / kTile); - } else { - auto wPackTy = MemRefType::get({thisK, kTile}, b.getI8Type()); - Value wPack = b.create(loc, wPackTy); - hostPacks.push_back(wPack); - b.create(loc, zeroI8, wPack); - Value validKEnd = b.create(loc, validK); - Value validNEnd = thisN; - auto weightRowLoop = - b.create(loc, zero, validKEnd, one); - b.setInsertionPointToStart(weightRowLoop.getBody()); - Value weightRow = weightRowLoop.getInductionVar(); - auto weightColumnLoop = - b.create(loc, zero, validNEnd, one); - b.setInsertionPointToStart(weightColumnLoop.getBody()); - Value weightColumn = weightColumnLoop.getInductionVar(); - Value sourceWeightRow = b.create( - loc, b.create(loc, k0), weightRow); - Value sourceWeightColumn = - b.create(loc, n0, weightColumn); - Value weightValue = b.create( - loc, op.getWeight(), - ValueRange{sourceWeightRow, sourceWeightColumn}); - b.create(loc, weightValue, wPack, - ValueRange{weightRow, weightColumn}); - b.setInsertionPointAfter(weightRowLoop); - - wLoaded = mvinBank(b, loc, wPack, wBank, thisK); - } - uint64_t cfg = matrixRs2(tileRows, kTile, thisK); - auto smatmul = b.create( - loc, resultState.getType(), aLoaded, wLoaded, resultState, - createI64Const(b, loc, static_cast(cfg)), - createI1Const(b, loc, k0 == 0), - createI1Const(b, loc, k0 + thisK == paddedK), - createI64Const(b, loc, tile * resultRows)); - resultState = smatmul.getWrBankOut(); - aBank = aLoaded; - wBank = wLoaded; - } - } - - Value outputBank = allocBank(b, loc, 1, 1); - Value converted = - finalOutput - ? b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, - createI64Const(b, loc, tileCount * resultRows), - b.getBoolAttr(op.getActivation() == 1)) - .getOutBankOut() - : b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, - createI64Const(b, loc, tileCount * resultRows), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - b.getI64IntegerAttr(tileCount * tileRows), - b.getI64IntegerAttr(1), - b.getI64IntegerAttr(tileCount * tileRows), - b.getBoolAttr(op.getActivation() == 1)) - .getOutBankOut(); - releaseBank(b, loc, resultState); - - Type outputElementType = - finalOutput ? Type(b.getF32Type()) : Type(b.getI8Type()); - auto packedOutTy = - MemRefType::get({tileCount * tileRows, kTile}, outputElementType); - Value packedOut = b.create(loc, packedOutTy); - Value stored = mvoutBank(b, loc, packedOut, converted, - finalOutput ? tileCount * resultRows - : tileCount * tileRows); - for (Value pack : hostPacks) - b.create(loc, pack); - - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value rowEnd = validRows; - Value columnEnd = thisN; - auto rowLoop = b.create(loc, zero, rowEnd, one); - b.setInsertionPointToStart(rowLoop.getBody()); - Value row = rowLoop.getInductionVar(); - auto columnLoop = b.create(loc, zero, columnEnd, one); - b.setInsertionPointToStart(columnLoop.getBody()); - Value column = columnLoop.getInductionVar(); - Value value = - b.create(loc, packedOut, ValueRange{row, column}); - Value outputRow = b.create(loc, m0, row); - Value outputColumn = b.create(loc, n0, column); - b.create(loc, value, op.getOutput(), - ValueRange{outputRow, outputColumn}); - b.setInsertionPointAfter(rowLoop); - releaseBank(b, loc, stored); - b.create(loc, packedOut); - } - - releaseBank(b, loc, biasState); - releaseBank(b, loc, scaleLoaded); - b.create(loc, biasPack); - b.create(loc, scalePack); - releaseBank(b, loc, aBank); - releaseBank(b, loc, wBank); - } - b.setInsertionPointAfter(panelLoop); - for (Value pack : inputPacks) - b.create(loc, pack); - b.setInsertionPointAfter(matrixRowLoop); - b.eraseOp(op); - return success(); - } -}; - -} // namespace - -LogicalResult mlir::buddy::lowerMatmulToBanks(MegaMatmulOp op, - PatternRewriter &rewriter) { - return MegaMatmulToBankSSAPattern(op.getContext()) - .matchAndRewrite(op, rewriter); -} diff --git a/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp b/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp deleted file mode 100644 index 882b988bf..000000000 --- a/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerBuckyball/MegaKernelToBankSSAPatterns.cpp +++ /dev/null @@ -1,480 +0,0 @@ -//===- MegaKernelToBankSSAPatterns.cpp - MegaKernel to bank SSA ----------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Trace/TraceOps.h" -#include "Utils/BankUtils.h" -#include "Utils/StageTrace.h" -#include "llvm/Support/MathExtras.h" - -#include -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace mlir::buddy { -LogicalResult lowerMatmulToBanks(MegaMatmulOp op, PatternRewriter &rewriter); -} // namespace mlir::buddy - -namespace { - -constexpr int64_t kTile = 16; -constexpr int64_t kInt32Rows = 64; - -class MatmulMegaKernelToBankSSAPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(MegaKernelOp kernel, - PatternRewriter &b) const override { - if (kernel.getBody().empty()) - return kernel.emitError("MegaKernel region must contain one block"); - - Block &body = kernel.getBody().front(); - if (body.without_terminator().empty()) - return kernel.emitError( - "MegaKernel region must contain at least one stage"); - if (!isa(body.front())) - return failure(); - - SmallVector stages; - for (Operation &op : body.without_terminator()) { - auto matmul = dyn_cast(op); - if (!matmul) - return kernel.emitError( - "MatMul MegaKernel cannot contain non-MatMul stages"); - // The activation LUT runs on the LUT Ball. - if (matmul.getActivation() == 2 && - !llvm::any_of( - buckyball_target::getBuckyballTarget().isa, - [](const auto &entry) { return entry.mnemonic == "LUT"; })) - return matmul.emitError("MatMul MegaKernel LUT requires the LUT Ball"); - stages.push_back(matmul); - } - - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth < kInt32Rows || - !llvm::isPowerOf2_64(target.bankDepth)) - return kernel.emitError( - "MatMul MegaKernel requires 128-bit power-of-two-depth banks with " - "at least 64 rows"); - if (buckyball_target::getBuckyballBallMapping("SMATMUL_OS").outBW != 1) - return kernel.emitError("MatMul MegaKernel requires SMatMulBall outBW=1"); - - int64_t rows = - cast(stages.front().getInput().getType()).getShape()[0]; - if (stages.size() == 1) { - auto stage = stages.front(); - Value output = stage.getOutput(); - auto trace = stage->getAttrOfType("trace_node"); - b.setInsertionPoint(stage); - if (trace && cast(trace.get("cycle")).getValue()) { - auto start = b.create<::buddy::trace::StartOp>( - stage.getLoc(), cast(trace.get("id")), - b.getBoolAttr(true), b.getBoolAttr(false), - cast(trace.get("tag"))); - start->setAttr("id_path", trace.get("id_path")); - } - if (failed(mlir::buddy::lowerMatmulToBanks(stages.front(), b))) - return failure(); - b.setInsertionPoint(body.getTerminator()); - if (trace) { - auto end = b.create<::buddy::trace::EndOp>( - kernel.getLoc(), output.getType(), output, - cast(trace.get("id")), - cast(trace.get("cycle")), - cast(trace.get("tensor")), - cast(trace.get("tag"))); - end->setAttr("id_path", trace.get("id_path")); - } - b.eraseOp(body.getTerminator()); - b.inlineBlockBefore(&body, kernel); - b.eraseOp(kernel); - return success(); - } - Value expectedInput = kernel.getInput(); - for (auto [index, stage] : llvm::enumerate(stages)) { - bool last = index + 1 == stages.size(); - auto inputTy = cast(stage.getInput().getType()); - auto weightTy = cast(stage.getWeight().getType()); - auto biasTy = cast(stage.getBias().getType()); - auto scaleTy = cast(stage.getScale().getType()); - auto lutTy = cast(stage.getLut().getType()); - auto outputTy = cast(stage.getOutput().getType()); - if (stage.getInput() != expectedInput || - (last ? stage.getOutput() != kernel.getOutput() - : stage.getOutput() == kernel.getOutput())) - return stage.emitError("MatMul stage breaks the MegaKernel data chain"); - if (!inputTy.hasStaticShape() || !weightTy.hasStaticShape() || - !biasTy.hasStaticShape() || !scaleTy.hasStaticShape() || - !lutTy.hasStaticShape() || !outputTy.hasStaticShape()) - return stage.emitError("MatMul MegaKernel requires static shapes"); - int64_t m = inputTy.getShape()[0]; - int64_t k = inputTy.getShape()[1]; - int64_t n = weightTy.getShape()[1]; - if (m <= 0 || m != rows || k <= 0 || n <= 0 || - weightTy.getShape()[0] != k || outputTy.getShape()[0] != m || - outputTy.getShape()[1] != n || biasTy.getShape()[0] != n || - scaleTy.getShape()[0] != n || lutTy.getRank() != 1 || - !lutTy.getElementType().isInteger(8) || - lutTy.getShape()[0] != (stage.getActivation() == 2 ? 256 : 1) || - stage.getActivation() < 0 || stage.getActivation() > 2 || - (last && stage.getActivation() == 2) || - (last ? !outputTy.getElementType().isF32() - : !outputTy.getElementType().isInteger(8))) - return stage.emitError("MatMul MegaKernel requires matching " - "INT8/FP32 stage shapes"); - int64_t inputBanks = index == 0 ? 0 : (k + 4 * kTile - 1) / (4 * kTile); - int64_t outputBanks = (n + 4 * kTile - 1) / (4 * kTile); - int64_t requiredBanks = inputBanks + (last ? 5 : outputBanks + 6); - if (requiredBanks > target.bankNum) - return stage.emitError("MatMul MegaKernel exceeds bank capacity"); - expectedInput = stage.getOutput(); - } - - Location loc = kernel.getLoc(); - b.setInsertionPoint(kernel); - SmallVector hostPacks; - SmallVector activationBanks; - SmallVector activationK; - SmallVector initialPacks; - SmallVector initialK; - - auto firstTy = cast(stages.front().getInput().getType()); - int64_t firstK = firstTy.getShape()[1]; - Value zeroI8 = - b.create(loc, b.getI8Type(), b.getI8IntegerAttr(0)); - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value sixteen = b.create(loc, kTile); - auto rowLoop = b.create( - loc, zero, b.create(loc, rows), one); - b.setInsertionPointToStart(rowLoop.getBody()); - Value inputRow = rowLoop.getInductionVar(); - SmallVector> tracedOutputs; - int64_t paddedFirstK = (firstK + kTile - 1) / kTile * kTile; - for (int64_t k0 = 0; k0 < paddedFirstK; k0 += target.bankDepth) { - int64_t thisK = std::min(target.bankDepth, paddedFirstK - k0); - int64_t validK = std::min(thisK, firstK - k0); - Value inputPack = b.create( - loc, MemRefType::get({thisK / kTile, kTile}, b.getI8Type())); - hostPacks.push_back(inputPack); - b.create(loc, zeroI8, inputPack); - auto inputLoop = b.create( - loc, zero, b.create(loc, validK), one); - b.setInsertionPointToStart(inputLoop.getBody()); - Value localK = inputLoop.getInductionVar(); - Value sourceK = b.create( - loc, b.create(loc, k0), localK); - Value value = b.create(loc, stages.front().getInput(), - ValueRange{inputRow, sourceK}); - Value tile = b.create(loc, localK, sixteen); - Value packedColumn = b.create(loc, localK, sixteen); - b.create(loc, value, inputPack, - ValueRange{tile, packedColumn}); - b.setInsertionPointAfter(inputLoop); - initialPacks.push_back(inputPack); - initialK.push_back(thisK); - } - - for (auto [stageIndex, stage] : llvm::enumerate(stages)) { - bool last = stageIndex + 1 == stages.size(); - auto traceNode = stage->getAttrOfType("trace_node"); - bool cycle = - traceNode && cast(traceNode.get("cycle")).getValue(); - bool tensor = - traceNode && cast(traceNode.get("tensor")).getValue(); - if (cycle) { - auto start = b.create<::buddy::trace::StartOp>( - loc, cast(traceNode.get("id")), b.getBoolAttr(true), - b.getBoolAttr(false), cast(traceNode.get("tag"))); - start->setAttr("id_path", traceNode.get("id_path")); - } - int64_t logicalK = - cast(stage.getInput().getType()).getShape()[1]; - int64_t n = cast(stage.getWeight().getType()).getShape()[1]; - SmallVector outputs; - SmallVector outputK; - SmallVector finalPacks; - Value packedOutput; - int64_t packedRows = 0; - Value lutBank; - if (stage.getActivation() == 2) { - Value lutPack = b.create( - loc, MemRefType::get({kTile, kTile}, b.getI8Type())); - hostPacks.push_back(lutPack); - auto lutLoop = b.create( - loc, zero, b.create(loc, 256), one); - b.setInsertionPointToStart(lutLoop.getBody()); - Value index = lutLoop.getInductionVar(); - Value row = b.create(loc, index, sixteen); - Value column = b.create(loc, index, sixteen); - Value value = b.create(loc, stage.getLut(), index); - b.create(loc, value, lutPack, ValueRange{row, column}); - b.setInsertionPointAfter(lutLoop); - lutBank = allocBank(b, loc, 1, 1); - lutBank = mvinBank(b, loc, lutPack, lutBank, 16); - } - - for (int64_t n0 = 0; n0 < n; n0 += kTile) { - int64_t validN = std::min(kTile, n - n0); - if (!last && n0 % (4 * kTile) == 0) { - packedOutput = allocBank(b, loc, 1, 1); - packedRows = 0; - } - Value biasPack = b.create( - loc, MemRefType::get({4, 4}, b.getI32Type())); - Value scalePack = b.create( - loc, MemRefType::get({4, 4}, b.getF32Type())); - hostPacks.push_back(biasPack); - hostPacks.push_back(scalePack); - Value zeroI32 = b.create(loc, b.getI32Type(), - b.getI32IntegerAttr(0)); - Value oneF32 = b.create(loc, b.getF32Type(), - b.getF32FloatAttr(1.0)); - b.create(loc, zeroI32, biasPack); - b.create(loc, oneF32, scalePack); - auto channelLoop = b.create( - loc, zero, b.create(loc, validN), one); - b.setInsertionPointToStart(channelLoop.getBody()); - Value channel = channelLoop.getInductionVar(); - Value sourceChannel = b.create( - loc, b.create(loc, n0), channel); - Value group = b.create( - loc, channel, b.create(loc, 4)); - Value lane = b.create( - loc, channel, b.create(loc, 4)); - Value bias = - b.create(loc, stage.getBias(), sourceChannel); - Value scale = - b.create(loc, stage.getScale(), sourceChannel); - b.create(loc, bias, biasPack, ValueRange{group, lane}); - b.create(loc, scale, scalePack, - ValueRange{group, lane}); - b.setInsertionPointAfter(channelLoop); - - Value biasBank = allocBank(b, loc, 1, 1); - Value biasLoaded = mvinBank(b, loc, biasPack, biasBank, 4); - Value biasState = b.create( - loc, biasLoaded.getType(), biasLoaded, createI64Const(b, loc, 0)); - Value scaleBank = allocBank(b, loc, 1, 1); - Value scaleLoaded = mvinBank(b, loc, scalePack, scaleBank, 4); - Value weightBank = allocBank(b, loc, 1, 1); - Value resultState = allocBank(b, loc, 1, 1); - - int64_t kBase = 0; - size_t chunkCount = - stageIndex == 0 ? initialPacks.size() : activationBanks.size(); - for (size_t chunk = 0; chunk < chunkCount; ++chunk) { - int64_t thisK = - stageIndex == 0 ? initialK[chunk] : activationK[chunk]; - int64_t validK = std::min(thisK, logicalK - kBase); - Value activation; - if (stageIndex == 0) { - Value inputBank = allocBank(b, loc, 1, 1); - activation = - mvinBank(b, loc, initialPacks[chunk], inputBank, thisK / kTile); - } else { - activation = activationBanks[chunk]; - } - auto weightTy = cast(stage.getWeight().getType()); - auto [strides, offset] = weightTy.getStridesAndOffset(); - if (validK == thisK && validN == kTile && strides[1] == 1 && - strides[0] > 0 && strides[0] % kTile == 0) { - Value panel = b.create( - loc, stage.getWeight(), - SmallVector{b.getIndexAttr(kBase), - b.getIndexAttr(n0)}, - SmallVector{b.getIndexAttr(thisK), - b.getIndexAttr(kTile)}, - SmallVector{b.getIndexAttr(1), - b.getIndexAttr(1)}); - weightBank = - mvinBank(b, loc, panel, weightBank, thisK, strides[0] / kTile); - } else { - Value weightPack = b.create( - loc, MemRefType::get({thisK, kTile}, b.getI8Type())); - hostPacks.push_back(weightPack); - b.create(loc, zeroI8, weightPack); - auto kLoop = b.create( - loc, zero, b.create(loc, validK), one); - b.setInsertionPointToStart(kLoop.getBody()); - Value localK = kLoop.getInductionVar(); - auto nLoop = b.create( - loc, zero, b.create(loc, validN), one); - b.setInsertionPointToStart(nLoop.getBody()); - Value localN = nLoop.getInductionVar(); - Value sourceK = b.create( - loc, b.create(loc, kBase), localK); - Value sourceN = b.create( - loc, b.create(loc, n0), localN); - Value weight = b.create( - loc, stage.getWeight(), ValueRange{sourceK, sourceN}); - b.create(loc, weight, weightPack, - ValueRange{localK, localN}); - b.setInsertionPointAfter(kLoop); - - weightBank = mvinBank(b, loc, weightPack, weightBank, thisK); - } - auto smatmul = b.create( - loc, resultState.getType(), activation, weightBank, resultState, - createI64ConstU(b, loc, matrixRs2(1, kTile, thisK)), - createI1Const(b, loc, chunk == 0), - createI1Const(b, loc, chunk + 1 == chunkCount), - createI64Const(b, loc, 0)); - resultState = smatmul.getWrBankOut(); - if (stageIndex == 0) - releaseBank(b, loc, activation); - kBase += thisK; - } - - if (last) { - Value outputBank = allocBank(b, loc, 1, 1); - Value converted = b.create( - loc, outputBank.getType(), resultState, scaleLoaded, outputBank, - createI64Const(b, loc, 4), - b.getBoolAttr(stage.getActivation() == 1)); - releaseBank(b, loc, resultState); - Value packed = b.create( - loc, MemRefType::get({1, kTile}, b.getF32Type())); - hostPacks.push_back(packed); - Value stored = mvoutBank(b, loc, packed, converted, 4); - releaseBank(b, loc, stored); - finalPacks.push_back(packed); - } else { - Value converted = b.create( - loc, packedOutput.getType(), resultState, scaleLoaded, - packedOutput, createI64Const(b, loc, 4), - createI64Const(b, loc, packedRows), createI64Const(b, loc, 0), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), - b.getBoolAttr(stage.getActivation() == 1)); - releaseBank(b, loc, resultState); - packedOutput = converted; - ++packedRows; - if (packedRows == 4 || n0 + kTile >= n) { - if (lutBank) { - Value lutOutput = allocBank(b, loc, 1, 1); - OperationState lut(loc, "buckyball.bank_lut"); - lut.addOperands({packedOutput, lutBank, lutOutput, - createI64Const(b, loc, packedRows)}); - lut.addTypes(lutOutput.getType()); - Value transformed = b.create(lut)->getResult(0); - releaseBank(b, loc, packedOutput); - packedOutput = transformed; - } - outputs.push_back(packedOutput); - outputK.push_back(packedRows * kTile); - packedOutput = {}; - } - } - releaseBank(b, loc, biasState); - releaseBank(b, loc, scaleLoaded); - releaseBank(b, loc, weightBank); - } - if (lutBank) - releaseBank(b, loc, lutBank); - - bool traceStage = tensor && !last; - if (traceStage) { - int64_t channelBase = 0; - for (auto [bankIndex, output] : llvm::enumerate(outputs)) { - int64_t rows = outputK[bankIndex] / kTile; - Value pack = b.create( - loc, MemRefType::get({rows, kTile}, b.getI8Type())); - hostPacks.push_back(pack); - outputs[bankIndex] = mvoutBank(b, loc, pack, output, rows); - int64_t validChannels = std::min(outputK[bankIndex], n - channelBase); - auto channelLoop = b.create( - loc, zero, b.create(loc, validChannels), - one); - b.setInsertionPointToStart(channelLoop.getBody()); - Value channel = channelLoop.getInductionVar(); - Value row = b.create(loc, channel, sixteen); - Value column = b.create(loc, channel, sixteen); - Value value = - b.create(loc, pack, ValueRange{row, column}); - Value outputChannel = b.create( - loc, b.create(loc, channelBase), channel); - b.create(loc, value, stage.getOutput(), - ValueRange{inputRow, outputChannel}); - b.setInsertionPointAfter(channelLoop); - channelBase += outputK[bankIndex]; - } - tracedOutputs.push_back({stage.getOutput(), traceNode, false}); - } - - for (Value activation : activationBanks) - releaseBank(b, loc, activation); - activationBanks = outputs; - activationK = outputK; - - if (last) { - for (auto [panel, packed] : llvm::enumerate(finalPacks)) { - int64_t n0 = panel * kTile; - int64_t validN = std::min(kTile, n - n0); - auto outputLoop = b.create( - loc, zero, b.create(loc, validN), one); - b.setInsertionPointToStart(outputLoop.getBody()); - Value localN = outputLoop.getInductionVar(); - Value output = - b.create(loc, packed, ValueRange{zero, localN}); - Value outputN = b.create( - loc, b.create(loc, n0), localN); - b.create(loc, output, kernel.getOutput(), - ValueRange{inputRow, outputN}); - b.setInsertionPointAfter(outputLoop); - } - if (tensor) { - tracedOutputs.push_back({stage.getOutput(), traceNode, true}); - } - } - if (cycle) { - auto end = b.create<::buddy::trace::EndOp>( - loc, zero.getType(), zero, cast(traceNode.get("id")), - b.getBoolAttr(true), b.getBoolAttr(false), - cast(traceNode.get("tag"))); - end->setAttr("id_path", traceNode.get("id_path")); - } - } - - for (Value bank : activationBanks) - releaseBank(b, loc, bank); - for (Value pack : hostPacks) - b.create(loc, pack); - b.setInsertionPointAfter(rowLoop); - for (auto [output, meta, last] : tracedOutputs) { - if (!last && - cast(output.getType()).getElementType().isInteger(8)) { - if (failed(mlir::buddy::emitStageTensorTrace(b, loc, output, meta))) - return failure(); - } else { - auto trace = b.create<::buddy::trace::EndOp>( - loc, output.getType(), output, cast(meta.get("id")), - b.getBoolAttr(false), b.getBoolAttr(true), - cast(meta.get("tag"))); - trace->setAttr("id_path", meta.get("id_path")); - } - } - b.eraseOp(kernel); - return success(); - } -}; - -} // namespace - -namespace mlir::buddy { -void populateMatmulRegionToBankSSAPatterns(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} -} // namespace mlir::buddy diff --git a/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerLinalgToBuckyball/MatMulPatterns.cpp b/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerLinalgToBuckyball/MatMulPatterns.cpp deleted file mode 100644 index 5197d0f89..000000000 --- a/waste/examples/components/balls/smatmul/stack/src/Conversion/LowerLinalgToBuckyball/MatMulPatterns.cpp +++ /dev/null @@ -1,416 +0,0 @@ -//===- MatMulPatterns.cpp - linalg matmul -> buckyball.smatmul_matmul ----===// -// -// Claims linalg.matmul (plain or transposed B) and linalg.batch_matmul, which -// splits into one matmul per batch. Non-quantized matmul is split into -// bank-sized panels. Quantized matmul retains its full N dimension so bank -// lowering can quantize one activation chunk and hand it to every output panel. -// -//===----------------------------------------------------------------------===// - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -constexpr size_t kBankLane = 16; - -size_t ceilDiv(size_t a, size_t b) { return (a + b - 1) / b; } -size_t aMvinDepthLines(size_t m, size_t k) { return m * (k / kBankLane); } -size_t bMvinDepthLines(size_t k, size_t n) { return k * (n / kBankLane); } -size_t cMvoutDepthLines(size_t m, size_t n) { return m * (n / kBankLane); } - -// Copies B [N, K] into a new [K, N] buffer. -Value transposeB(PatternRewriter &rewriter, Location loc, Value b) { - auto type = cast(b.getType()); - int64_t n = type.getShape()[0], k = type.getShape()[1]; - Value transposed = rewriter.create( - loc, MemRefType::get({k, n}, type.getElementType())); - Value zero = rewriter.create(loc, 0); - Value one = rewriter.create(loc, 1); - auto nLoop = rewriter.create( - loc, zero, rewriter.create(loc, n), one); - OpBuilder::InsertionGuard guard(rewriter); - rewriter.setInsertionPointToStart(nLoop.getBody()); - auto kLoop = rewriter.create( - loc, zero, rewriter.create(loc, k), one); - rewriter.setInsertionPointToStart(kLoop.getBody()); - Value nIv = nLoop.getInductionVar(), kIv = kLoop.getInductionVar(); - Value value = rewriter.create(loc, b, ValueRange{nIv, kIv}); - rewriter.create(loc, value, transposed, - ValueRange{kIv, nIv}); - return transposed; -} - -// Drops a leading unit dimension of a rank-3 memref. -Value collapseUnitBatch(PatternRewriter &rewriter, Location loc, Value value) { - auto type = cast(value.getType()); - if (type.getRank() != 3 || type.getShape()[0] != 1) - return value; - return rewriter.create( - loc, value, ArrayRef{{0, 1}, {2}}); -} - -class MatMulLowering : public OpRewritePattern { -public: - MatMulLowering(MLIRContext *context, int64_t bankDepth) - : OpRewritePattern(context), bankDepth(bankDepth) {} - - LogicalResult matchAndRewrite(linalg::MatmulOp op, - PatternRewriter &rewriter) const override { - Location loc = op.getLoc(); - Attribute maps = op->getAttr("indexing_maps"); - bool isTransposeB = linalg::MatmulTransposeBOp::isDefaultIndexingMaps(maps); - if (!linalg::MatmulOp::isDefaultIndexingMaps(maps) && !isTransposeB) - return failure(); - for (Value operand : op->getOperands()) { - auto type = dyn_cast(operand.getType()); - if (!type || !type.hasStaticShape() || - (type.getRank() != 2 && - !(type.getRank() == 3 && type.getShape()[0] == 1))) - return failure(); - } - Type aElement = - cast(op.getInputs()[0].getType()).getElementType(); - Type bElement = - cast(op.getInputs()[1].getType()).getElementType(); - Type cElement = - cast(op.getOutputs()[0].getType()).getElementType(); - if (!(aElement.isF32() && bElement.isInteger(8) && cElement.isF32()) && - !((aElement.isF32() || aElement.isBF16()) && bElement == aElement && - cElement == aElement) && - !(aElement.isInteger(8) && bElement.isInteger(8) && - cElement.isInteger(32))) - return failure(); - - Value aMemArray = collapseUnitBatch(rewriter, loc, op.getInputs()[0]); - Value bMemArray = collapseUnitBatch(rewriter, loc, op.getInputs()[1]); - Value cMemArray = collapseUnitBatch(rewriter, loc, op.getOutputs()[0]); - if (isTransposeB) - bMemArray = transposeB(rewriter, loc, bMemArray); - - auto aType = cast(aMemArray.getType()); - auto bType = cast(bMemArray.getType()); - auto cType = cast(cMemArray.getType()); - - auto aShape = aType.getShape(); - auto bShape = bType.getShape(); - auto cShape = cType.getShape(); - size_t M = aShape[aShape.size() - 2]; - size_t K = aShape[aShape.size() - 1]; - size_t N = bShape[bShape.size() - 1]; - - if (bShape[bShape.size() - 2] != (int64_t)K || - cShape[cShape.size() - 2] != (int64_t)M || - cShape[cShape.size() - 1] != (int64_t)N) - return op.emitError("matmul input/output shapes mismatch"); - - const bool isQuantized = aType.getElementType().isF32() && - bType.getElementType().isInteger(8) && - cType.getElementType().isF32(); - const bool isWideFloat = - (aType.getElementType().isF32() || aType.getElementType().isBF16()) && - bType.getElementType() == aType.getElementType() && - cType.getElementType() == aType.getElementType(); - const bool isInt8 = aType.getElementType().isInteger(8) && - bType.getElementType().isInteger(8) && - cType.getElementType().isInteger(32); - if (!isQuantized && !isWideFloat && !isInt8) - return failure(); - auto dwAddrAttr = op->getAttrOfType("dw_addr"); - auto dwBytesAttr = op->getAttrOfType("dw_bytes"); - auto perChannelAttr = op->getAttrOfType("per_channel"); - if (isQuantized && (!dwAddrAttr || !dwBytesAttr || !perChannelAttr || - dwAddrAttr.getInt() < 16)) - return op.emitError("quantized matmul requires RAX Dw metadata"); - - constexpr size_t mTileSize = 8; - size_t M_pad = ceilDiv(M, mTileSize) * mTileSize; - size_t K_pad = ceilDiv(K, kBankLane) * kBankLane; - size_t N_pad = ceilDiv(N, kBankLane) * kBankLane; - bool needPadding = (M_pad != M) || (K_pad != K) || (N_pad != N); - - Value aMemArrayPadded = aMemArray; - Value bMemArrayPadded = bMemArray; - Value cMemArrayPadded = cMemArray; - - if (needPadding) { - auto aPadType = MemRefType::get({(int64_t)M_pad, (int64_t)K_pad}, - aType.getElementType()); - auto bPadType = MemRefType::get({(int64_t)K_pad, (int64_t)N_pad}, - bType.getElementType()); - auto cPadType = MemRefType::get({(int64_t)M_pad, (int64_t)N_pad}, - cType.getElementType()); - - aMemArrayPadded = rewriter.create(loc, aPadType); - bMemArrayPadded = rewriter.create(loc, bPadType); - cMemArrayPadded = rewriter.create(loc, cPadType); - - Value aZero = rewriter.create( - loc, aType.getElementType(), - rewriter.getZeroAttr(aType.getElementType())); - Value bZero = rewriter.create( - loc, bType.getElementType(), - rewriter.getZeroAttr(bType.getElementType())); - Value cZero = rewriter.create( - loc, cType.getElementType(), - rewriter.getZeroAttr(cType.getElementType())); - rewriter.create(loc, aZero, aMemArrayPadded); - rewriter.create(loc, bZero, bMemArrayPadded); - rewriter.create(loc, cZero, cMemArrayPadded); - - Value aView = rewriter.create( - loc, aMemArrayPadded, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(M), - rewriter.getIndexAttr(K)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - rewriter.create(loc, aMemArray, aView); - - Value bView = rewriter.create( - loc, bMemArrayPadded, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(K), - rewriter.getIndexAttr(N)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - rewriter.create(loc, bMemArray, bView); - } - - size_t M_tiling = needPadding ? M_pad : M; - size_t K_tiling = needPadding ? K_pad : K; - size_t N_tiling = needPadding ? N_pad : N; - - if (bankDepth <= 0 || (size_t)bankDepth % kBankLane != 0) - return op.emitError("bankDepth must be a positive multiple of bank lane"); - if (M_tiling % mTileSize || K_tiling % kBankLane || N_tiling % kBankLane) - return op.emitError( - "M must be a multiple of 8 and K/N must be multiples of bank lane"); - if (isQuantized) { - const int64_t dwRequired = perChannelAttr.getValue() ? N_tiling * 4 : 4; - if (dwBytesAttr.getInt() < dwRequired) - return op.emitError("Dw scale image does not cover padded N"); - } - - // Bank unit on M (depth) and N (lane). Quantized matmul carries one - // logical K into SMatMul so FP2INT computes one activation scale. - const size_t nTileSize = kBankLane; - const size_t kTileSize = - (isWideFloat || isQuantized) ? K_tiling : kBankLane; - if (M_tiling % mTileSize != 0) - return op.emitError("M does not split into bank-depth units"); - - if (!isWideFloat && !isQuantized) { - if (aMvinDepthLines(mTileSize, kTileSize) > (size_t)bankDepth || - bMvinDepthLines(kTileSize, nTileSize) > (size_t)bankDepth || - cMvoutDepthLines(mTileSize, nTileSize) > (size_t)bankDepth) - return op.emitError( - "bank-unit matmul exceeds 8-bit mvin or 32-bit mvout depth"); - } - - const size_t kTileNum = ceilDiv(K_tiling, kTileSize); - - OpBuilder::InsertionGuard guard(rewriter); - Value zeroIdx = rewriter.create(loc, 0); - Value mStepVal = rewriter.create(loc, mTileSize); - Value kStepVal = rewriter.create(loc, kTileSize); - Value mUpperVal = rewriter.create(loc, M_tiling); - Value kUpperVal = rewriter.create(loc, K_tiling); - - auto mLoop = rewriter.create(loc, zeroIdx, mUpperVal, mStepVal); - rewriter.setInsertionPointToStart(mLoop.getBody()); - Value mIv = mLoop.getInductionVar(); - - if (isQuantized) { - Value aTile = rewriter.create( - loc, aMemArrayPadded, - SmallVector{mIv, rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(mTileSize), - rewriter.getIndexAttr(K_tiling)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - Value bTile = rewriter.create( - loc, bMemArrayPadded, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(K_tiling), - rewriter.getIndexAttr(N_tiling)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - Value cTile = rewriter.create( - loc, cMemArrayPadded, - SmallVector{mIv, rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(mTileSize), - rewriter.getIndexAttr(N_tiling)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - Value zero = rewriter.create( - loc, cType.getElementType(), - rewriter.getZeroAttr(cType.getElementType())); - rewriter.create(loc, zero, cTile); - auto smatmul = rewriter.create(loc, aTile, bTile, cTile); - smatmul->setAttr("dwAddr", dwAddrAttr); - smatmul->setAttr("dwBytes", dwBytesAttr); - smatmul->setAttr("perChannel", perChannelAttr); - } else { - // N is static. Non-quantized matmul remains one physical N panel per op. - for (size_t n0 = 0; n0 < N_tiling; n0 += nTileSize) { - Value cTile = rewriter.create( - loc, cMemArrayPadded, - SmallVector{mIv, rewriter.getIndexAttr(n0)}, - SmallVector{rewriter.getIndexAttr(mTileSize), - rewriter.getIndexAttr(nTileSize)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - - auto cElemType = cType.getElementType(); - Value zero = rewriter.create( - loc, cElemType, rewriter.getZeroAttr(cElemType)); - rewriter.create(loc, zero, cTile); - - if (kTileNum == 1) { - Value aTile = rewriter.create( - loc, aMemArrayPadded, - SmallVector{mIv, rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(mTileSize), - rewriter.getIndexAttr(kTileSize)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - Value bTile = rewriter.create( - loc, bMemArrayPadded, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(n0)}, - SmallVector{rewriter.getIndexAttr(kTileSize), - rewriter.getIndexAttr(nTileSize)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - auto smatmul = - rewriter.create(loc, aTile, bTile, cTile); - } else { - auto kLoop = - rewriter.create(loc, zeroIdx, kUpperVal, kStepVal); - rewriter.setInsertionPointToStart(kLoop.getBody()); - Value kIv = kLoop.getInductionVar(); - Value aTile = rewriter.create( - loc, aMemArrayPadded, SmallVector{mIv, kIv}, - SmallVector{rewriter.getIndexAttr(mTileSize), - rewriter.getIndexAttr(kTileSize)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - Value bTile = rewriter.create( - loc, bMemArrayPadded, - SmallVector{kIv, rewriter.getIndexAttr(n0)}, - SmallVector{rewriter.getIndexAttr(kTileSize), - rewriter.getIndexAttr(nTileSize)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - if (isWideFloat) - return op.emitError( - "FP32 matmul must pass full K in one smatmul_matmul"); - // 8-bit legalize overwrites C; accumulate via host partial. - auto partialTy = MemRefType::get( - {(int64_t)mTileSize, (int64_t)nTileSize}, cElemType); - Value partial = rewriter.create(loc, partialTy); - rewriter.create(loc, zero, partial); - rewriter.create(loc, aTile, bTile, partial); - rewriter.create( - loc, ValueRange{cTile, partial}, ValueRange{cTile}, - linalg::ElementwiseKind::add); - rewriter.create(loc, partial); - rewriter.setInsertionPointAfter(kLoop); - } - } - } - - rewriter.setInsertionPointAfter(mLoop); - - if (needPadding) { - Value cView = rewriter.create( - loc, cMemArrayPadded, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(M), - rewriter.getIndexAttr(N)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - rewriter.create(loc, cView, cMemArray); - rewriter.create(loc, aMemArrayPadded); - rewriter.create(loc, bMemArrayPadded); - rewriter.create(loc, cMemArrayPadded); - } - - if (isTransposeB) - rewriter.create(loc, bMemArray); - rewriter.eraseOp(op); - return success(); - } - -private: - int64_t bankDepth; -}; - -// One matmul per batch; each is claimed by MatMulLowering. -class BatchMatMulLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(linalg::BatchMatmulOp op, - PatternRewriter &rewriter) const override { - Attribute maps = op->getAttr("indexing_maps"); - bool isTransposeB = - linalg::BatchMatmulTransposeBOp::isDefaultIndexingMaps(maps); - if (!linalg::BatchMatmulOp::isDefaultIndexingMaps(maps) && !isTransposeB) - return failure(); - Location loc = op.getLoc(); - Value a = op.getInputs()[0], b = op.getInputs()[1], c = op.getOutputs()[0]; - auto aType = dyn_cast(a.getType()); - auto bType = dyn_cast(b.getType()); - auto cType = dyn_cast(c.getType()); - if (!aType || !bType || !cType || !aType.hasStaticShape() || - !bType.hasStaticShape() || !cType.hasStaticShape()) - return failure(); - auto slice = [&](Value value, MemRefType type, int64_t batch) -> Value { - ArrayRef shape = type.getShape(); - Value sub = rewriter.create( - loc, value, ArrayRef{batch, 0, 0}, - ArrayRef{1, shape[1], shape[2]}, ArrayRef{1, 1, 1}); - return rewriter.create( - loc, sub, ArrayRef{{0, 1}, {2}}); - }; - for (int64_t batch = 0; batch < aType.getShape()[0]; ++batch) { - Value bBatch = slice(b, bType, batch); - if (isTransposeB) - bBatch = transposeB(rewriter, loc, bBatch); - rewriter.create( - loc, ValueRange{slice(a, aType, batch), bBatch}, - ValueRange{slice(c, cType, batch)}); - if (isTransposeB) - rewriter.create(loc, bBatch); - } - rewriter.eraseOp(op); - return success(); - } -}; - -} // namespace - -namespace mlir::buddy { -void populateSMatMulBallLinalgPatterns(RewritePatternSet &patterns, - int64_t bankDepth) { - patterns.add(patterns.getContext(), bankDepth); - patterns.add(patterns.getContext()); -} -} // namespace mlir::buddy diff --git a/waste/examples/components/balls/smatmul/stack/src/Dialect/Buckyball/SMatMulBall.td b/waste/examples/components/balls/smatmul/stack/src/Dialect/Buckyball/SMatMulBall.td deleted file mode 100644 index d76555db9..000000000 --- a/waste/examples/components/balls/smatmul/stack/src/Dialect/Buckyball/SMatMulBall.td +++ /dev/null @@ -1,71 +0,0 @@ -//===-- SMatMulBall.td - SMatMulBall ops -------*- tablegen -*-===// - -#ifndef SMATMULBALL_OPS -#define SMATMULBALL_OPS - -include "Buckyball.td" -include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td" - -def SMatMulMatmulOp : Buckyball_Op<"smatmul_matmul"> { - let summary = "Logical matmul lowered to SMatMulBall"; - let description = [{ - Performs C[M,N] = A[M,K] x B[K,N] with int8 inputs / int32 output. - Memrefs must already match the SMatMulBall bank tile layout. Logical N - may exceed 16; bank lowering emits one 16-lane C panel at a time. - }]; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$aMemArray, - MemRefRankOf<[AnyType], [2]>:$bMemArray, - MemRefRankOf<[AnyType], [2]>:$cMemArray, - DefaultValuedAttr:$dwAddr, - DefaultValuedAttr:$dwBytes, - DefaultValuedAttr:$perChannel); - let assemblyFormat = [{$aMemArray $bMemArray $cMemArray prop-dict attr-dict `:` - type($aMemArray) type($bMemArray) type($cMemArray)}]; -} - -def SMatMulOp : Buckyball_Op<"smatmul"> { - let summary = "SMatMulBall compute"; - let description = [{ - ISA: SMATMUL_OS. `config` contains rows, cols, and k. - }]; - let arguments = (ins I64:$op1BankId, I64:$op2BankId, - I64:$resultBankId, I64:$config, - I1:$first, I1:$last, I64:$outputBase); - let assemblyFormat = "$op1BankId `,` $op2BankId `,` $resultBankId `,` " - "$config `,` $first `,` $last `,` $outputBase " - "attr-dict `:` " - "type($op1BankId)"; -} - -def SMatMulBiasOp : Buckyball_Op<"smatmul_bias"> { - let summary = "Preload one 16-channel INT32 bias vector"; - let description = [{ - Reads four 128-bit SRAM rows containing 16 INT32 bias values. SMatMulBall - uses the retained vector to initialize every new 16x16 accumulator tile. - }]; - let arguments = (ins I64:$biasBankId, I64:$inputBase); - let assemblyFormat = "$biasBankId `,` $inputBase attr-dict `:` " - "type($biasBankId)"; -} - -def BankSMatMulOp : Buckyball_Op<"bank_smatmul", [DeclareOpInterfaceMethods]> { - let summary = "SMatMulBall compute with virtual bank handles"; - let arguments = (ins I64:$op1Bank, I64:$op2Bank, I64:$wrBank, I64:$config, - I1:$first, I1:$last, I64:$outputBase); - let results = (outs I64:$wrBankOut); - let assemblyFormat = - "$op1Bank $op2Bank $wrBank $config $first $last $outputBase " - "attr-dict `:` " - "type($op1Bank) type($op2Bank) type($wrBank) type($config)"; -} - -def BankSMatMulBiasOp : Buckyball_Op<"bank_smatmul_bias", [DeclareOpInterfaceMethods]> { - let summary = "SMatMul bias preload with a virtual bank handle"; - let arguments = (ins I64:$biasBank, I64:$inputBase); - let results = (outs I64:$biasBankOut); - let assemblyFormat = "$biasBank $inputBase attr-dict `:` " - "type($biasBank) type($inputBase)"; -} - - -#endif // SMATMULBALL_OPS diff --git a/waste/examples/components/balls/smatmul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/smatmul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 5ea5463c5..000000000 --- a/waste/examples/components/balls/smatmul/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,263 +0,0 @@ -//===- LegalizeForLLVMExport.cpp - SMatMulBall LLVM lowering --------------===// - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include "llvm/Support/ErrorHandling.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { - -uint64_t matrixCfg(uint64_t rows, uint64_t cols, bool first = true, - bool last = true) { - if (rows == 0 || rows > 0xfff || (rows != 1 && rows % 8) || cols != 16) - llvm::report_fatal_error( - "matrix cfg: rows must be 1 or a multiple of 8 and cols must be 16"); - return fieldBits(rows, 0, 11) | fieldBits(cols, 12, 23) | - (uint64_t(first) << 24) | (uint64_t(last) << 25); -} - -struct SMatMulMatmulLowering : public ConvertOpToLLVMPattern { - SMatMulMatmulLowering(LLVMTypeConverter &converter, bool /*stable*/) - : ConvertOpToLLVMPattern(converter) {} - - LogicalResult - matchAndRewrite(SMatMulMatmulOp op, OpAdaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - Value aMem = op.getAMemArray(); - Value bMem = op.getBMemArray(); - Value cMem = op.getCMemArray(); - - auto aTy = dyn_cast(aMem.getType()); - auto bTy = dyn_cast(bMem.getType()); - auto cTy = dyn_cast(cMem.getType()); - if (!aTy || !bTy || !cTy || !aTy.hasStaticShape() || - !bTy.hasStaticShape() || !cTy.hasStaticShape()) - return rewriter.notifyMatchFailure( - op, "buckyball.smatmul_matmul requires static memref shapes"); - - uint64_t m = aTy.getShape()[0]; - uint64_t k = aTy.getShape()[1]; - uint64_t kb = bTy.getShape()[0]; - uint64_t n = bTy.getShape()[1]; - if (k != kb || cTy.getShape()[0] != (int64_t)m || - cTy.getShape()[1] != (int64_t)n) - return rewriter.notifyMatchFailure(op, "matmul shapes mismatch"); - const int64_t bankDepth = buckyball_target::getBuckyballTarget().bankDepth; - if (bankDepth <= 0) - return op.emitError("smatmul lowering requires target bankDepth > 0"); - // A occupies m lines, B occupies k lines, and packed int32 C occupies 2m. - if (m == 0 || n != 16 || k != 16 || m % 8 != 0 || - m > static_cast(bankDepth / 2) || - k > static_cast(bankDepth)) - return rewriter.notifyMatchFailure( - op, "SMatMul requires K/N exactly 16 and A/B/C to fit their banks"); - if (!aTy.getElementType().isInteger(8) || - !bTy.getElementType().isInteger(8) || - !cTy.getElementType().isInteger(32)) - return rewriter.notifyMatchFailure( - op, "SMatMul Ball supports only 8-bit A/B with 32-bit C"); - - const uint64_t aBank = 0; - const uint64_t bBank = 1; - const uint64_t cBank = 2; - uint64_t depthA = m; // K <= 16 => one bank row per M row - uint64_t depthB = k; - uint64_t depthC = 2 * m; - - emitMset(rewriter, loc, aBank, 1, 1, 1); - emitMset(rewriter, loc, bBank, 1, 1, 1); - emitMset(rewriter, loc, cBank, 1, 2, 1); - - Value aPtr = extractPtr(rewriter, loc, aMem); - Value bPtr = extractPtr(rewriter, loc, bMem); - auto packedType = MemRefType::get({static_cast(depthC), 8}, - cTy.getElementType()); - Value packed = rewriter.create(loc, packedType); - Value packedPtr = extractPtr(rewriter, loc, packed); - - Value rs1A = packRs1BankIter(rewriter, loc, cstI64(rewriter, loc, aBank), - cstI64(rewriter, loc, depthA)); - Value rs2A = - packRs2MemStride(rewriter, loc, aPtr, cstI64(rewriter, loc, 1)); - - rewriter.create(loc, rs1A, rs2A); - - Value rs1B = packRs1BankIter(rewriter, loc, cstI64(rewriter, loc, bBank), - cstI64(rewriter, loc, depthB)); - Value rs2B = - packRs2MemStride(rewriter, loc, bPtr, cstI64(rewriter, loc, 1)); - - rewriter.create(loc, rs1B, rs2B); - - rewriter.create( - loc, - packRs1BanksIter(rewriter, loc, cstI64(rewriter, loc, aBank), - cstI64(rewriter, loc, bBank), - cstI64(rewriter, loc, cBank), - cstI64(rewriter, loc, k)), - cstI64(rewriter, loc, matrixCfg(m, n)), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("SMATMUL_OS"))); - - Value rs1C = packRs1BankIter(rewriter, loc, cstI64(rewriter, loc, cBank), - cstI64(rewriter, loc, depthC)); - Value rs2C = - packRs2MemStride(rewriter, loc, packedPtr, cstI64(rewriter, loc, 1)); - - rewriter.create(loc, rs1C, rs2C); - - Value indexZero = rewriter.create(loc, 0); - Value indexOne = rewriter.create(loc, 1); - Value indexTwo = rewriter.create(loc, 2); - Value indexEight = rewriter.create(loc, 8); - Value indexM = rewriter.create(loc, m); - Value indexN = rewriter.create(loc, n); - auto rowLoop = - rewriter.create(loc, indexZero, indexM, indexOne); - rewriter.setInsertionPointToStart(rowLoop.getBody()); - Value row = rowLoop.getInductionVar(); - auto columnLoop = - rewriter.create(loc, indexZero, indexN, indexOne); - rewriter.setInsertionPointToStart(columnLoop.getBody()); - Value column = columnLoop.getInductionVar(); - Value packedRow = rewriter.create( - loc, rewriter.create(loc, row, indexTwo), - rewriter.create(loc, column, indexEight)); - Value packedColumn = - rewriter.create(loc, column, indexEight); - Value value = rewriter.create( - loc, packed, ValueRange{packedRow, packedColumn}); - rewriter.create(loc, value, cMem, ValueRange{row, column}); - rewriter.setInsertionPointAfter(rowLoop); - rewriter.create(loc, packed); - - emitMset(rewriter, loc, aBank, 0, 0, 0); - emitMset(rewriter, loc, bBank, 0, 0, 0); - emitMset(rewriter, loc, cBank, 0, 0, 0); - - rewriter.eraseOp(op); - return success(); - } -}; - -struct SMatMulLowering : public ConvertOpToLLVMPattern { - SMatMulLowering(LLVMTypeConverter &converter, bool, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(SMatMulOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - auto config = op.getConfig().getDefiningOp(); - if (!config) - return op.emitError("smatmul config must be a constant"); - auto configAttr = dyn_cast(config.getValue()); - if (!configAttr) - return op.emitError("smatmul config must be an integer constant"); - uint64_t cfg = configAttr.getValue().getZExtValue(); - uint64_t rows = cfg & 0xfff; - uint64_t cols = (cfg >> 12) & 0xfff; - uint64_t k = (cfg >> 24) & 0xfff; - if (rows == 0 || (rows != 1 && rows % 8) || cols != 16 || k == 0 || - k % 16 || rows * 4 > static_cast(bankDepth)) - return op.emitError("SMatMul matrix shape or C footprint is invalid"); - auto base = adaptor.getOutputBase().getDefiningOp(); - auto baseAttr = - base ? dyn_cast(base.getValue()) : IntegerAttr(); - if (!baseAttr || baseAttr.getInt() < 0 || baseAttr.getInt() > 63 || - baseAttr.getInt() + rows * 4 > bankDepth) - return op.emitError( - "SMatMul outputBase must be constant and fit C in bank"); - Value rs1 = packRs1BanksIter( - rewriter, loc, adaptor.getOp1BankId(), adaptor.getOp2BankId(), - adaptor.getResultBankId(), cstI64(rewriter, loc, k)); - Value rs2 = cstI64(rewriter, loc, matrixCfg(rows, cols, false, false)); - Value first = rewriter.create(loc, rewriter.getI64Type(), - adaptor.getFirst()); - Value last = rewriter.create(loc, rewriter.getI64Type(), - adaptor.getLast()); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, first, cstI64(rewriter, loc, 24))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, last, cstI64(rewriter, loc, 25))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getOutputBase(), - cstI64(rewriter, loc, 26))); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("SMATMUL_OS"))); - return success(); - } - -private: - int64_t bankDepth; -}; - -struct SMatMulBiasLowering : public ConvertOpToLLVMPattern { - SMatMulBiasLowering(LLVMTypeConverter &converter, bool) - : ConvertOpToLLVMPattern(converter) {} - - LogicalResult - matchAndRewrite(SMatMulBiasOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - auto base = adaptor.getInputBase().getDefiningOp(); - auto baseAttr = - base ? dyn_cast(base.getValue()) : IntegerAttr(); - if (!baseAttr || baseAttr.getInt() < 0 || - baseAttr.getInt() + 4 > - static_cast( - buckyball_target::getBuckyballTarget().bankDepth)) - return op.emitError( - "SMatMul bias inputBase must be constant and fit in bank"); - Value rs1 = packRs1BankIter(rewriter, loc, adaptor.getBiasBankId(), - cstI64(rewriter, loc, 4)); - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getInputBase(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("SMATMUL_BIAS"))); - return success(); - } -}; - -} // namespace - -namespace mlir::buddy::buckyball { -void populateSMatMulBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - patterns.add(converter, stable); - patterns.add(converter, stable, bankDepth); - patterns.add(converter, stable); -} - -void configureSMatMulBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool /*stable*/) { - target.addIllegalOp(); - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/smatmul/stack/src/Passes.cpp b/waste/examples/components/balls/smatmul/stack/src/Passes.cpp deleted file mode 100644 index 2ae76231f..000000000 --- a/waste/examples/components/balls/smatmul/stack/src/Passes.cpp +++ /dev/null @@ -1,86 +0,0 @@ -//===- Passes.cpp - SMatMulBall passes ------------------------------------===// - -#include "Conversion/BallPasses.h" - -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -using namespace mlir; - -namespace mlir::buddy { -void populateSMatMulBallLinalgPatterns(RewritePatternSet &patterns, - int64_t bankDepth); -void populateMatmulRegionToBankSSAPatterns(RewritePatternSet &patterns); -namespace buckyball { -void populateSMatMulBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureSMatMulBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LinalgToSMatMulPass - : public mlir::buddy::LinalgToBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LinalgToSMatMulPass) - StringRef getArgument() const final { - return "convert-linalg-to-smatmul-ball"; - } - StringRef getDescription() const final { - return "Lower the Linalg operations SMatMulBall executes."; - } - void populate(RewritePatternSet &patterns, int64_t, int64_t bankDepth, - int64_t) const { - mlir::buddy::populateSMatMulBallLinalgPatterns(patterns, bankDepth); - } -}; - -class LowerSMatMulPass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerSMatMulPass) - StringRef getArgument() const final { return "lower-smatmul-ball"; } - StringRef getDescription() const final { - return "Lower SMatMulBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateSMatMulBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureSMatMulBallLegalizeForExportTarget(target, - stable); - } -}; - -// Lowers MegaKernels made of MatMul stages as one resident bank region. -class LowerMatmulRegionPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerMatmulRegionPass) - StringRef getArgument() const final { - return "lower-matmul-region-to-bank-ssa"; - } - StringRef getDescription() const final { - return "Lower MatMul MegaKernels to bank-SSA operations."; - } - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - void runOnOperation() override { - RewritePatternSet patterns(&getContext()); - mlir::buddy::populateMatmulRegionToBankSSAPatterns(patterns); - if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) - signalPassFailure(); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - PassRegistration(); - PassRegistration(); - return true; -}(); diff --git a/waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index b55db8d4e..000000000 --- a/waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,33 +0,0 @@ -//===- AssignPhysicalBanks.cpp - ToInt8 bank assignment -------------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankQuantF32ToI8Op::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankQuantF32ToI8Op op = *this; - rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), - op.getIter(), op.getScale()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); -} - -LogicalResult -BankQuantI32ToI8Op::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankQuantI32ToI8Op op = *this; - rewriter.create( - op.getLoc(), op.getInBank(), op.getScaleBank(), op.getOutBank(), - op.getIter(), op.getOutputBase(), op.getInputBase(), - op.getOutputWidthAttr(), op.getOutputHeightAttr(), - op.getOutputStrideAttr(), op.getReluAttr()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); -} diff --git a/waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp b/waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp deleted file mode 100644 index e1051330d..000000000 --- a/waste/examples/components/balls/toint8/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp +++ /dev/null @@ -1,156 +0,0 @@ -//===- LowerToBankSSA.cpp - Input quantization ----------------------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult QuantizeTensorF32ToI8Op::lowerToBankSSA(PatternRewriter &b) { - QuantizeTensorF32ToI8Op op = *this; - Location loc = op.getLoc(); - auto inputTy = dyn_cast(op.getInput().getType()); - auto outputTy = dyn_cast(op.getOutput().getType()); - if (!inputTy || !outputTy || !inputTy.hasStaticShape() || - !outputTy.hasStaticShape() || inputTy.getRank() != outputTy.getRank() || - (inputTy.getRank() != 2 && inputTy.getRank() != 4) || - !inputTy.getElementType().isF32() || - !outputTy.getElementType().isInteger(8)) - return op.emitError( - "input quantization requires static rank-2/rank-4 memrefs"); - ArrayRef inputShape = inputTy.getShape(); - ArrayRef outputShape = outputTy.getShape(); - bool nchwToNhwc = op.getNchwToNhwcAttr().getValue(); - if ((!nchwToNhwc && inputShape != outputShape) || - (nchwToNhwc && inputTy.getRank() != 4) || - (nchwToNhwc && - (inputShape[0] != outputShape[0] || inputShape[1] != outputShape[3] || - inputShape[2] != outputShape[1] || inputShape[3] != outputShape[2]))) - return op.emitError("input quantization layout shape mismatch"); - - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth < 4 || - target.bankDepth % 4) - return op.emitError("input quantization requires 128-bit banks with " - "depth divisible by 4"); - int64_t elements = 1; - for (int64_t dim : outputShape) { - if (dim <= 0) - return op.emitError("input quantization requires positive dimensions"); - elements *= dim; - } - - Value inputBank = allocBank(b, loc, 1, 1); - Value outputBank = allocBank(b, loc, 1, 1); - Value scale = - b.create(loc, b.getF32Type(), op.getScaleAttr()); - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value four = b.create(loc, 4); - Value sixteen = b.create(loc, 16); - Value zeroF32 = - b.create(loc, b.getF32Type(), b.getF32FloatAttr(0.0)); - int64_t maxElements = target.bankDepth * 4; - - Value elementCount = b.create(loc, elements); - Value chunkSize = b.create(loc, maxElements); - auto chunkLoop = b.create(loc, zero, elementCount, chunkSize, - ValueRange{inputBank, outputBank}); - b.setInsertionPointToStart(chunkLoop.getBody()); - Value offset = chunkLoop.getInductionVar(); - ValueRange bankStates = chunkLoop.getRegionIterArgs(); - auto inputPackTy = MemRefType::get({target.bankDepth, 4}, b.getF32Type()); - auto outputPackTy = - MemRefType::get({target.bankDepth / 4, 16}, b.getI8Type()); - Value inputPack = b.create(loc, inputPackTy); - Value outputPack = b.create(loc, outputPackTy); - b.create(loc, zeroF32, inputPack); - - auto copyIn = b.create(loc, zero, chunkSize, one); - b.setInsertionPointToStart(copyIn.getBody()); - Value i = copyIn.getInductionVar(); - Value linear = b.create(loc, offset, i); - auto inputValid = b.create( - loc, - b.create(loc, arith::CmpIPredicate::slt, linear, - elementCount), - false); - b.setInsertionPointToStart(&inputValid.getThenRegion().front()); - SmallVector outputIndices(outputTy.getRank()); - Value remaining = linear; - for (int64_t dim = outputTy.getRank() - 1; dim >= 0; --dim) { - Value extent = b.create(loc, outputShape[dim]); - outputIndices[dim] = b.create(loc, remaining, extent); - remaining = b.create(loc, remaining, extent); - } - SmallVector indices(outputTy.getRank()); - if (nchwToNhwc) { - indices[0] = outputIndices[0]; - indices[1] = outputIndices[3]; - indices[2] = outputIndices[1]; - indices[3] = outputIndices[2]; - } else { - indices = outputIndices; - } - Value value = b.create(loc, op.getInput(), indices); - Value packRow = b.create(loc, i, four); - Value packColumn = b.create(loc, i, four); - b.create(loc, value, inputPack, - ValueRange{packRow, packColumn}); - b.setInsertionPointAfter(inputValid); - b.setInsertionPointAfter(copyIn); - - Value inputNext = - mvinBank(b, loc, inputPack, bankStates[0], target.bankDepth); - Value outputNext = b.create( - loc, bankStates[1].getType(), inputNext, bankStates[1], - createI64Const(b, loc, target.bankDepth), scale); - outputNext = mvoutBank(b, loc, outputPack, outputNext, target.bankDepth / 4); - - auto copyOut = b.create(loc, zero, chunkSize, one); - b.setInsertionPointToStart(copyOut.getBody()); - i = copyOut.getInductionVar(); - linear = b.create(loc, offset, i); - auto outputValid = b.create( - loc, - b.create(loc, arith::CmpIPredicate::slt, linear, - elementCount), - false); - b.setInsertionPointToStart(&outputValid.getThenRegion().front()); - remaining = linear; - for (int64_t dim = outputTy.getRank() - 1; dim >= 0; --dim) { - Value extent = b.create(loc, outputShape[dim]); - indices[dim] = b.create(loc, remaining, extent); - remaining = b.create(loc, remaining, extent); - } - packRow = b.create(loc, i, sixteen); - packColumn = b.create(loc, i, sixteen); - value = b.create(loc, outputPack, - ValueRange{packRow, packColumn}); - b.create(loc, value, op.getOutput(), indices); - b.setInsertionPointAfter(outputValid); - b.setInsertionPointAfter(copyOut); - b.create(loc, inputPack); - b.create(loc, outputPack); - b.create(loc, ValueRange{inputNext, outputNext}); - b.setInsertionPointAfter(chunkLoop); - inputBank = chunkLoop.getResult(0); - outputBank = chunkLoop.getResult(1); - - releaseBank(b, loc, inputBank); - releaseBank(b, loc, outputBank); - b.eraseOp(op); - return success(); -} diff --git a/waste/examples/components/balls/toint8/stack/src/Conversion/LowerLinalgToBuckyball/QuantizePatterns.cpp b/waste/examples/components/balls/toint8/stack/src/Conversion/LowerLinalgToBuckyball/QuantizePatterns.cpp deleted file mode 100644 index 3e2c5f4f6..000000000 --- a/waste/examples/components/balls/toint8/stack/src/Conversion/LowerLinalgToBuckyball/QuantizePatterns.cpp +++ /dev/null @@ -1,64 +0,0 @@ -//===- QuantizePatterns.cpp - marked FP32-to-INT8 input quantization -----===// -// -// Claims the generics the frontend marked `buckyball.quant_f32_to_i8`. -// -//===----------------------------------------------------------------------===// - -#include - -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class QuantF32ToI8Lowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(linalg::GenericOp op, - PatternRewriter &rewriter) const override { - if (!op->hasAttr("buckyball.quant_f32_to_i8")) - return failure(); - if (op.getInputs().size() != 1 || op.getOutputs().size() != 1) - return op.emitError("FP32-to-INT8 quant requires one input and output"); - Value input = op.getInputs()[0]; - Value output = op.getOutputs()[0]; - auto inputType = dyn_cast(input.getType()); - auto outputType = dyn_cast(output.getType()); - auto nchwToNhwc = op->getAttrOfType("nchw_to_nhwc"); - auto scale = op->getAttrOfType("scale"); - if (!inputType || !outputType || !inputType.hasStaticShape() || - !outputType.hasStaticShape() || - inputType.getRank() != outputType.getRank() || - (inputType.getRank() != 2 && inputType.getRank() != 4) || - !inputType.getElementType().isF32() || - !outputType.getElementType().isInteger(8) || !nchwToNhwc || !scale || - !std::isfinite(scale.getValueAsDouble()) || - scale.getValueAsDouble() <= 0.0) - return op.emitError("FP32-to-INT8 quant requires matching static " - "rank-2/rank-4 memrefs and a positive scale"); - ArrayRef in = inputType.getShape(); - ArrayRef out = outputType.getShape(); - if (nchwToNhwc.getValue() - ? inputType.getRank() != 4 || in[0] != out[0] || in[1] != out[3] || - in[2] != out[1] || in[3] != out[2] - : in != out) - return op.emitError("FP32-to-INT8 quant layout shape mismatch"); - rewriter.create(op.getLoc(), input, output, scale, - nchwToNhwc); - rewriter.eraseOp(op); - return success(); - } -}; - -} // namespace - -namespace mlir::buddy { -void populateToInt8BallLinalgPatterns(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} -} // namespace mlir::buddy diff --git a/waste/examples/components/balls/toint8/stack/src/Dialect/Buckyball/ToInt8Ball.td b/waste/examples/components/balls/toint8/stack/src/Dialect/Buckyball/ToInt8Ball.td deleted file mode 100644 index 86c927de6..000000000 --- a/waste/examples/components/balls/toint8/stack/src/Dialect/Buckyball/ToInt8Ball.td +++ /dev/null @@ -1,69 +0,0 @@ -//===-- ToInt8Ball.td - ToInt8Ball operations ----------------*- tablegen -*-===// -// -// ToInt8Ball operations for quantization. -// -//===----------------------------------------------------------------------===// - -#ifndef TOINT8BALL_OPS -#define TOINT8BALL_OPS - -include "Buckyball.td" - -def QuantF32ToI8Op : Buckyball_Op<"quant_f32_to_i8"> { - let summary = "Per-tensor FP32 to INT8 quantization"; - let arguments = (ins I64:$inputBankId, I64:$outputBankId, - I64:$iter, F32:$scale); - let assemblyFormat = "$inputBankId `,` $outputBankId `,` $iter `,` $scale " - "prop-dict attr-dict `:` type($inputBankId)"; -} - -def BankQuantF32ToI8Op : Buckyball_Op<"bank_quant_f32_to_i8", [DeclareOpInterfaceMethods]> { - let summary = "Per-tensor FP32 to INT8 quantization with virtual banks"; - let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, F32:$scale); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$inBank $outBank $iter $scale prop-dict attr-dict `:` " - "type($inBank) type($outBank) type($iter) type($scale)"; -} - -def QuantI32ToI8Op : Buckyball_Op<"quant_i32_to_i8"> { - let summary = "Per-channel INT32 to INT8 requantization"; - let description = [{ - The output is a dense input stream placed in a strided 2-D bank tile. - iter/4 must equal outputWidth*outputHeight. - }]; - let arguments = (ins I64:$inputBankId, I64:$scaleBankId, - I64:$outputBankId, I64:$iter, I64:$outputBase, - I64:$inputBase, - I64Attr:$outputWidth, I64Attr:$outputHeight, - I64Attr:$outputStride, - BoolAttr:$relu); - let assemblyFormat = "$inputBankId `,` $scaleBankId `,` $outputBankId `,` " - "$iter `,` $outputBase `,` $inputBase prop-dict attr-dict `:` " - "type($inputBankId) type($outputBase) type($inputBase)"; -} - -def BankQuantI32ToI8Op : Buckyball_Op<"bank_quant_i32_to_i8", [DeclareOpInterfaceMethods]> { - let summary = "Per-channel INT32 to INT8 requantization with virtual banks"; - let arguments = (ins I64:$inBank, I64:$scaleBank, I64:$outBank, - I64:$iter, I64:$outputBase, - I64:$inputBase, - I64Attr:$outputWidth, I64Attr:$outputHeight, - I64Attr:$outputStride, BoolAttr:$relu); - let results = (outs I64:$outBankOut); - let assemblyFormat = "$inBank $scaleBank $outBank $iter $outputBase $inputBase " - "prop-dict attr-dict `:` type($inBank) type($scaleBank) " - "type($outBank) type($iter) type($outputBase) " - "type($inputBase)"; -} - -def QuantizeTensorF32ToI8Op : Buckyball_Op<"quantize_tensor_f32_to_i8", - [DeclareOpInterfaceMethods]> { - let summary = "Quantize one FP32 model input tensor to INT8"; - let arguments = (ins MemRefRankOf<[F32], [2, 4]>:$input, - MemRefRankOf<[I8], [2, 4]>:$output, - F32Attr:$scale, - BoolAttr:$nchwToNhwc); - let assemblyFormat = [{$input $output prop-dict attr-dict `:` type($input) type($output)}]; -} - -#endif // TOINT8BALL_OPS diff --git a/waste/examples/components/balls/toint8/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/toint8/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index ad0541f1d..000000000 --- a/waste/examples/components/balls/toint8/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,122 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -struct QuantF32ToI8Lowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(QuantF32ToI8Op op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - Value scaleBits = rewriter.create( - loc, rewriter.getI32Type(), adaptor.getScale()); - Value rs2 = - rewriter.create(loc, rewriter.getI64Type(), scaleBits); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("QUANT_F32_TO_I8"))); - return success(); - } -}; - -struct QuantI32ToI8Lowering : public ConvertOpToLLVMPattern { - QuantI32ToI8Lowering(LLVMTypeConverter &converter, int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), - bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(QuantI32ToI8Op op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - adaptor.getScaleBankId(), - adaptor.getOutputBankId(), adaptor.getIter()); - auto iterOp = op.getIter().getDefiningOp(); - auto iterAttr = - iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); - if (!iterAttr || iterAttr.getInt() <= 0) - return op.emitError("INT32 to INT8 iter must be a positive constant"); - if (iterAttr.getInt() % 4 != 0 || iterAttr.getInt() > bankDepth) - return op.emitError( - "INT32 to INT8 iter must be a multiple of four within bank depth"); - int64_t outputRows = iterAttr.getInt() / 4; - int64_t outputWidth = op.getOutputWidthAttr().getInt(); - int64_t outputHeight = op.getOutputHeightAttr().getInt(); - int64_t outputStride = op.getOutputStrideAttr().getInt(); - auto outputBaseOp = - adaptor.getOutputBase().getDefiningOp(); - auto inputBaseOp = - adaptor.getInputBase().getDefiningOp(); - auto outputBaseAttr = outputBaseOp - ? dyn_cast(outputBaseOp.getValue()) - : IntegerAttr(); - auto inputBaseAttr = inputBaseOp - ? dyn_cast(inputBaseOp.getValue()) - : IntegerAttr(); - if (outputWidth <= 0 || outputHeight <= 0 || outputStride < outputWidth || - outputWidth * outputHeight != outputRows || outputWidth >= 128 || - outputHeight >= 128 || outputStride >= 128 || !outputBaseAttr || - !inputBaseAttr || outputBaseAttr.getInt() < 0 || - inputBaseAttr.getInt() < 0 || - outputBaseAttr.getInt() + (outputHeight - 1) * outputStride + - outputWidth > - bankDepth || - inputBaseAttr.getInt() + iterAttr.getInt() > bankDepth) - return op.emitError( - "INT32 to INT8 output tile is invalid for the physical bank"); - Value rs2 = rewriter.create(loc, adaptor.getOutputBase(), - cstI64(rewriter, loc, 1)); - rs2 = rewriter.create( - loc, rs2, - cstI64(rewriter, loc, - (outputWidth << 8) | (outputHeight << 15) | - (outputStride << 22) | (op.getRelu() ? 1 : 0))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getInputBase(), - cstI64(rewriter, loc, 29))); - rewriter.replaceOpWithNewOp( - op, rs1, rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("QUANT_I32_TO_I8"))); - return success(); - } - -private: - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateToInt8BallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - (void)stable; - patterns.add(converter); - patterns.add(converter, bankDepth); -} - -void configureToInt8BallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/toint8/stack/src/Passes.cpp b/waste/examples/components/balls/toint8/stack/src/Passes.cpp deleted file mode 100644 index 9290eed04..000000000 --- a/waste/examples/components/balls/toint8/stack/src/Passes.cpp +++ /dev/null @@ -1,57 +0,0 @@ -//===- Passes.cpp - ToInt8Ball passes -------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -void populateToInt8BallLinalgPatterns(RewritePatternSet &patterns); -namespace buckyball { -void populateToInt8BallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureToInt8BallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LinalgToToInt8Pass - : public mlir::buddy::LinalgToBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LinalgToToInt8Pass) - StringRef getArgument() const final { - return "convert-linalg-to-toint8-ball"; - } - StringRef getDescription() const final { - return "Lower the Linalg operations ToInt8Ball executes."; - } - void populate(RewritePatternSet &patterns, int64_t, int64_t bankDepth, - int64_t) const { - mlir::buddy::populateToInt8BallLinalgPatterns(patterns); - } -}; - -class LowerToInt8Pass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerToInt8Pass) - StringRef getArgument() const final { return "lower-toint8-ball"; } - StringRef getDescription() const final { - return "Lower ToInt8Ball operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateToInt8BallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureToInt8BallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - PassRegistration(); - return true; -}(); diff --git a/waste/examples/components/balls/trace/stack/src/Dialect/Buckyball/TraceBall.td b/waste/examples/components/balls/trace/stack/src/Dialect/Buckyball/TraceBall.td deleted file mode 100644 index 398293258..000000000 --- a/waste/examples/components/balls/trace/stack/src/Dialect/Buckyball/TraceBall.td +++ /dev/null @@ -1,18 +0,0 @@ -//===-- TraceBall.td - Trace Ball operations ----------------------------===// - -#ifndef TRACE_BALL_OPS -#define TRACE_BALL_OPS - -include "Buckyball.td" - -def BdbCounterOp : Buckyball_Op<"bdb_counter"> { - let summary = "Start, stop, or read a Buckyball debug counter"; - let description = [{ - Encodes BDB_COUNTER rs2 as subcommand[3:0], counterId[7:4], and - payload[63:8]. rs1 is always zero. - }]; - let arguments = (ins I64:$subcommand, I64:$counterId, I64:$payload); - let assemblyFormat = "$subcommand `,` $counterId `,` $payload attr-dict `:` type($subcommand)"; -} - -#endif // TRACE_BALL_OPS diff --git a/waste/examples/components/balls/trace/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/trace/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index befba6731..000000000 --- a/waste/examples/components/balls/trace/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -class BdbCounterLowering : public ConvertOpToLLVMPattern { -public: - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(BdbCounterOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - Value counter = rewriter.create(loc, adaptor.getCounterId(), - cstI64(rewriter, loc, 4)); - Value payload = rewriter.create(loc, adaptor.getPayload(), - cstI64(rewriter, loc, 8)); - Value rs2 = - rewriter.create(loc, adaptor.getSubcommand(), counter); - rs2 = rewriter.create(loc, rs2, payload); - rewriter.replaceOpWithNewOp( - op, cstI64(rewriter, loc, 0), rs2, - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("BDB_COUNTER"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateTraceBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - (void)stable; - (void)bankDepth; - - patterns.add(converter); -} - -void configureTraceBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/trace/stack/src/Passes.cpp b/waste/examples/components/balls/trace/stack/src/Passes.cpp deleted file mode 100644 index b96f3b28e..000000000 --- a/waste/examples/components/balls/trace/stack/src/Passes.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- Passes.cpp - TraceBall passes --------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -namespace buckyball { -void populateTraceBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureTraceBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LowerTracePass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerTracePass) - StringRef getArgument() const final { return "lower-trace-ball"; } - StringRef getDescription() const final { - return "Lower TraceBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateTraceBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureTraceBallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - return true; -}(); diff --git a/waste/examples/components/balls/transpose/stack/src/Backend/include/IntrinsicsRISCVTransposeBall.td b/waste/examples/components/balls/transpose/stack/src/Backend/include/IntrinsicsRISCVTransposeBall.td deleted file mode 100644 index db632b990..000000000 --- a/waste/examples/components/balls/transpose/stack/src/Backend/include/IntrinsicsRISCVTransposeBall.td +++ /dev/null @@ -1,4 +0,0 @@ -//===- IntrinsicsRISCVTransposeBall.td - TransposeBall intrinsics --------===// - -let TargetPrefix = "riscv" in -def int_riscv_bb_transpose : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; diff --git a/waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index 7668e9666..000000000 --- a/waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,20 +0,0 @@ -//===- AssignPhysicalBanks.cpp - Transpose bank assignment ----------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankTransposeOp::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankTransposeOp op = *this; - rewriter.create(op.getLoc(), op.getInBank(), op.getOutBank(), - op.getIter(), op.getElemBits()); - rewriter.replaceOp(op, op.getOutBank()); - return success(); -} diff --git a/waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp b/waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp deleted file mode 100644 index f573e06b9..000000000 --- a/waste/examples/components/balls/transpose/stack/src/Conversion/LowerBuckyball/LowerToBankSSA.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Utils/StructuredOpsUtils.h" -#include "mlir/IR/AffineMap.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult MemTransposeOp::lowerToBankSSA(PatternRewriter &rewriter) { - MemTransposeOp op = *this; - auto input = dyn_cast(op.getInput().getType()); - auto output = dyn_cast(op.getOutput().getType()); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 2 || - output.getRank() != 2 || output.getShape()[0] != input.getShape()[1] || - output.getShape()[1] != input.getShape()[0]) - return op.emitError("requires matching static rank-2 transpose memrefs"); - MLIRContext *context = rewriter.getContext(); - AffineMap identity = AffineMap::getMultiDimIdentityMap(2, context); - AffineMap transpose = AffineMap::get( - 2, 0, {rewriter.getAffineDimExpr(1), rewriter.getAffineDimExpr(0)}, - context); - SmallVector iterators(2, utils::IteratorType::parallel); - rewriter.create( - op.getLoc(), TypeRange{}, ValueRange{op.getInput()}, - ValueRange{op.getOutput()}, ArrayRef{identity, transpose}, - iterators, [](OpBuilder &builder, Location location, ValueRange args) { - builder.create(location, args[0]); - }); - rewriter.eraseOp(op); - return success(); -} diff --git a/waste/examples/components/balls/transpose/stack/src/Conversion/LowerLinalgToBuckyball/TransposePatterns.cpp b/waste/examples/components/balls/transpose/stack/src/Conversion/LowerLinalgToBuckyball/TransposePatterns.cpp deleted file mode 100644 index ad3050a65..000000000 --- a/waste/examples/components/balls/transpose/stack/src/Conversion/LowerLinalgToBuckyball/TransposePatterns.cpp +++ /dev/null @@ -1,263 +0,0 @@ -//===- TransposePatterns.cpp - linalg.transpose -> buckyball.mem_transpose ===// -// -// Rank-2 transposes run on the Ball directly. Higher-rank permutations the -// Ball supports are decomposed into rank-2 transposes of contiguous slices; -// other permutations stay for the CPU. -// -//===----------------------------------------------------------------------===// - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class TransposeLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(linalg::TransposeOp op, - PatternRewriter &rewriter) const override { - Value input = op.getInput(); - Value output = op.getInit(); - Location loc = op.getLoc(); - auto inT = dyn_cast(input.getType()); - auto outT = dyn_cast(output.getType()); - if (!inT || !outT || !inT.hasStaticShape() || !outT.hasStaticShape() || - inT.getElementType() != outT.getElementType()) - return failure(); - - ArrayRef perm = op.getPermutation(); - ArrayRef is = inT.getShape(); - ArrayRef os = outT.getShape(); - Type elem = inT.getElementType(); - SmallVector temps; - - // The Ball and collapse_shape need identity-layout buffers. - auto asContig = [&](Value v) -> Value { - auto t = cast(v.getType()); - if (t.getLayout().isIdentity()) - return v; - Value b = rewriter.create( - loc, MemRefType::get(t.getShape(), t.getElementType())); - rewriter.create(loc, v, b); - temps.push_back(b); - return b; - }; - auto dstContig = [&]() -> Value { - if (outT.getLayout().isIdentity()) - return output; - Value b = - rewriter.create(loc, MemRefType::get(os, elem)); - temps.push_back(b); - return b; - }; - auto collapse = [&](Value src, ArrayRef shape2, - ArrayRef reassoc) -> Value { - return rewriter.create( - loc, MemRefType::get(shape2, elem), src, reassoc); - }; - auto finish = [&](Value dst) { - if (dst != output) - rewriter.create(loc, dst, output); - for (Value t : llvm::reverse(temps)) - rewriter.create(loc, t); - rewriter.eraseOp(op); - return success(); - }; - auto idx = [&](int64_t v) -> Value { - return rewriter.create(loc, v); - }; - auto ofr = [&](Value v) { return OpFoldResult(v); }; - // Copies one slice of a rank-4 source into a buffer and collapses it to - // rank 2. Returns {collapsed, buffer}; the caller deallocates the buffer. - auto slice2D = - [&](Value src4, ArrayRef off, ArrayRef sz4, - ArrayRef shape2, - ArrayRef reassoc) -> std::pair { - SmallVector sz = {ofr(idx(sz4[0])), ofr(idx(sz4[1])), - ofr(idx(sz4[2])), ofr(idx(sz4[3]))}; - SmallVector str(4, ofr(idx(1))); - Value sub = rewriter.create(loc, src4, off, sz, str); - Value buf = - rewriter.create(loc, MemRefType::get(sz4, elem)); - rewriter.create(loc, sub, buf); - return {collapse(buf, shape2, reassoc), buf}; - }; - // Transposes a rank-2 slice and stores it into a slice of the destination. - auto transposeStore = [&](Value in2, Value dst4, ArrayRef off, - ArrayRef outSz4, - ArrayRef shape2, - ArrayRef reassoc) { - Value outBuf = - rewriter.create(loc, MemRefType::get(outSz4, elem)); - rewriter.create(loc, in2, - collapse(outBuf, shape2, reassoc)); - SmallVector sz = {ofr(idx(outSz4[0])), ofr(idx(outSz4[1])), - ofr(idx(outSz4[2])), ofr(idx(outSz4[3]))}; - SmallVector str(4, ofr(idx(1))); - rewriter.create( - loc, outBuf, - rewriter.create(loc, dst4, off, sz, str)); - rewriter.create(loc, outBuf); - }; - auto loop = [&](int64_t upper) { - auto forOp = rewriter.create(loc, idx(0), idx(upper), idx(1)); - rewriter.setInsertionPointToStart(forOp.getBody()); - return forOp; - }; - // One Ball transpose of the whole tensor viewed as [rows, cols]. - auto whole = [&](ArrayRef in2, ArrayRef inR, - ArrayRef out2, - ArrayRef outR) { - Value src = asContig(input), dst = dstContig(); - rewriter.create(loc, collapse(src, in2, inR), - collapse(dst, out2, outR)); - return finish(dst); - }; - - if (inT.getRank() == 2 && perm == ArrayRef({1, 0})) { - Value dst = dstContig(); - rewriter.create(loc, asContig(input), dst); - return finish(dst); - } - - if (inT.getRank() == 3 && perm == ArrayRef({0, 2, 1}) && - is[0] == 1) - return whole({is[1], is[2]}, {{0, 1}, {2}}, {os[1], os[2]}, - {{0, 1}, {2}}); - - if (inT.getRank() != 4) - return failure(); - Value c0 = idx(0); - - // Moving a channel across singleton spatial dimensions keeps byte order. - if (perm == ArrayRef({0, 3, 1, 2}) && is[0] == 1 && is[1] == 1 && - is[2] == 1) { - Value src = asContig(input), dst = dstContig(); - rewriter.create( - loc, collapse(src, {1, is[3]}, {{0, 1, 2}, {3}}), - collapse(dst, {1, is[3]}, {{0}, {1, 2, 3}})); - return finish(dst); - } - - // 1x1 OIHW->OHWI is layout-identical after collapsing to [O,I]. - if (perm == ArrayRef({0, 2, 3, 1}) && is[2] == 1 && is[3] == 1) { - Value src = asContig(input), dst = dstContig(); - rewriter.create( - loc, collapse(src, {is[0], is[1]}, {{0}, {1, 2, 3}}), - collapse(dst, {os[0], os[3]}, {{0}, {1, 2, 3}})); - return finish(dst); - } - - // N=1 NCHW->NHWC: [1,C,H,W] -> [1,H,W,C]. - if (perm == ArrayRef({0, 2, 3, 1}) && is[0] == 1) - return whole({is[1], is[2] * is[3]}, {{0, 1}, {2, 3}}, - {is[2] * is[3], is[1]}, {{0, 1, 2}, {3}}); - - // Weight OIHW->OHWI: one [I, H*W] transpose per O. - if (perm == ArrayRef({0, 2, 3, 1})) { - int64_t i = is[1], h = is[2], w = is[3]; - Value src = asContig(input), dst = dstContig(); - auto oLoop = loop(is[0]); - SmallVector off = {ofr(oLoop.getInductionVar()), ofr(c0), - ofr(c0), ofr(c0)}; - auto [in2, inBuf] = - slice2D(src, off, {1, i, h, w}, {i, h * w}, {{0, 1}, {2, 3}}); - transposeStore(in2, dst, off, {1, h, w, i}, {h * w, i}, {{0, 1, 2}, {3}}); - rewriter.create(loc, inBuf); - rewriter.setInsertionPointAfter(oLoop); - return finish(dst); - } - - // N=1 NHWC->NCHW: [1,A,B,C] -> [1,C,A,B]. - if (perm == ArrayRef({0, 3, 1, 2}) && is[0] == 1) - return whole({is[1] * is[2], is[3]}, {{0, 1, 2}, {3}}, - {is[3], is[1] * is[2]}, {{0, 1}, {2, 3}}); - - // [N,A,B,C] -> [N,C,B,A]. - if (perm == ArrayRef({0, 3, 2, 1})) { - int64_t n = is[0], a = is[1], b = is[2], c = is[3]; - if (n == 1) - return whole({a * b, c}, {{0, 1, 2}, {3}}, {c, a * b}, - {{0, 1}, {2, 3}}); - Value src = asContig(input), dst = dstContig(); - auto nLoop = loop(n); - Value ni = nLoop.getInductionVar(); - if (b <= a) { - auto bLoop = loop(b); - SmallVector off = {ofr(ni), ofr(c0), - ofr(bLoop.getInductionVar()), ofr(c0)}; - auto [in2, inBuf] = - slice2D(src, off, {1, a, 1, c}, {a, c}, {{0, 1}, {2, 3}}); - transposeStore(in2, dst, off, {1, c, 1, a}, {c, a}, {{0, 1}, {2, 3}}); - rewriter.create(loc, inBuf); - } else { - auto aLoop = loop(a); - Value ai = aLoop.getInductionVar(); - SmallVector inOff = {ofr(ni), ofr(ai), ofr(c0), ofr(c0)}; - SmallVector outOff = {ofr(ni), ofr(c0), ofr(c0), ofr(ai)}; - auto [in2, inBuf] = - slice2D(src, inOff, {1, 1, b, c}, {b, c}, {{0, 1, 2}, {3}}); - transposeStore(in2, dst, outOff, {1, c, b, 1}, {c, b}, - {{0, 1}, {2, 3}}); - rewriter.create(loc, inBuf); - } - rewriter.setInsertionPointAfter(nLoop); - return finish(dst); - } - - // [N,A,B,C] -> [N,B,A,C]: one [A,B] transpose per (n, c). - if (perm == ArrayRef({0, 2, 1, 3})) { - int64_t a = is[1], b = is[2]; - Value src = asContig(input), dst = dstContig(); - auto nLoop = loop(is[0]); - auto cLoop = loop(is[3]); - SmallVector off = {ofr(nLoop.getInductionVar()), ofr(c0), - ofr(c0), ofr(cLoop.getInductionVar())}; - auto [in2, inBuf] = - slice2D(src, off, {1, a, b, 1}, {a, b}, {{0, 1}, {2, 3}}); - transposeStore(in2, dst, off, {1, b, a, 1}, {b, a}, {{0, 1}, {2, 3}}); - rewriter.create(loc, inBuf); - rewriter.setInsertionPointAfter(nLoop); - return finish(dst); - } - - // [N,H,A,B] -> [N,H,B,A]: one [A,B] transpose per (n, h). - if (perm == ArrayRef({0, 1, 3, 2})) { - int64_t a = is[2], b = is[3]; - Value src = asContig(input), dst = dstContig(); - auto nLoop = loop(is[0]); - auto hLoop = loop(is[1]); - SmallVector off = {ofr(nLoop.getInductionVar()), - ofr(hLoop.getInductionVar()), ofr(c0), - ofr(c0)}; - auto [in2, inBuf] = - slice2D(src, off, {1, 1, a, b}, {a, b}, {{0, 1, 2}, {3}}); - transposeStore(in2, dst, off, {1, 1, b, a}, {b, a}, {{0, 1, 2}, {3}}); - rewriter.create(loc, inBuf); - rewriter.setInsertionPointAfter(nLoop); - return finish(dst); - } - - // Depthwise OIHW->HWIO: [C,1,k,k] -> [k,k,C,1]. - if (perm == ArrayRef({2, 3, 0, 1}) && is[1] == 1) - return whole({is[0], is[2] * is[3]}, {{0, 1}, {2, 3}}, - {is[2] * is[3], is[0]}, {{0, 1}, {2, 3}}); - - return failure(); - } -}; -} // namespace - -namespace mlir::buddy { -void populateTransposeBallLinalgPatterns(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} -} // namespace mlir::buddy diff --git a/waste/examples/components/balls/transpose/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/transpose/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 0a3281237..000000000 --- a/waste/examples/components/balls/transpose/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,115 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -static LogicalResult validateTranspose(Operation *op, Value elemBits, - Value iter, int64_t bankDepth) { - auto constant = elemBits.getDefiningOp(); - auto value = - constant ? dyn_cast(constant.getValue()) : IntegerAttr(); - if (!value || (value.getInt() != 8 && value.getInt() != 32)) - return op->emitError("Transpose elemBits must be constant 8 or 32"); - auto iterOp = iter.getDefiningOp(); - auto iterAttr = - iterOp ? dyn_cast(iterOp.getValue()) : IntegerAttr(); - if (!iterAttr || iterAttr.getInt() <= 0 || iterAttr.getInt() > bankDepth) - return op->emitError( - "Transpose iter must be a positive constant within the bank depth"); - return success(); -} - -struct TransposeLowering : public ConvertOpToLLVMPattern { - TransposeLowering(LLVMTypeConverter &converter, bool stable, - int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), stable(stable), - bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(TransposeOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - if (failed( - validateTranspose(op, op.getElemBits(), op.getIter(), bankDepth))) - return failure(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInputBankId(), - cstI64(rewriter, loc, 0), - adaptor.getOutputBankId(), adaptor.getIter()); - if (stable) { - rewriter.replaceOpWithNewOp(op, rs1, - adaptor.getElemBits()); - return success(); - } - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getElemBits(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("TRANSPOSE"))); - return success(); - } - -private: - bool stable = false; - int64_t bankDepth; -}; -struct BankTransposeLowering : public ConvertOpToLLVMPattern { - BankTransposeLowering(LLVMTypeConverter &converter, bool stable, - int64_t bankDepth) - : ConvertOpToLLVMPattern(converter), stable(stable), - bankDepth(bankDepth) {} - - LogicalResult - matchAndRewrite(BankTransposeOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - if (failed( - validateTranspose(op, op.getElemBits(), op.getIter(), bankDepth))) - return failure(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getInBank(), - cstI64(rewriter, loc, 0), adaptor.getOutBank(), - adaptor.getIter()); - if (stable) { - rewriter.replaceOpWithNewOp(op, rs1, - adaptor.getElemBits()); - return success(); - } - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getElemBits(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("TRANSPOSE"))); - return success(); - } - -private: - bool stable = false; - int64_t bankDepth; -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateTransposeBallLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, bool stable, - int64_t bankDepth) { - patterns.add(converter, stable, bankDepth); - patterns.add(converter, stable, bankDepth); -} - -void configureTransposeBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - if (stable) - target.addLegalOp(); - else - target.addIllegalOp(); - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/transpose/stack/src/Dialect/Buckyball/TransposeBall.td b/waste/examples/components/balls/transpose/stack/src/Dialect/Buckyball/TransposeBall.td deleted file mode 100644 index f7cdf2480..000000000 --- a/waste/examples/components/balls/transpose/stack/src/Dialect/Buckyball/TransposeBall.td +++ /dev/null @@ -1,52 +0,0 @@ -//===-- TransposeBall.td - TransposeBall operations --------*- tablegen -*-===// -// -// TransposeBall operations for matrix transpose. -// -//===----------------------------------------------------------------------===// - -#ifndef TRANSPOSEBALL_OPS -#define TRANSPOSEBALL_OPS - -include "Buckyball.td" - -// Memref-level transpose (like smatmul_matmul). Design bank-ssa decides -// pad / tile / Bank* composition; this op carries no bank policy. -def MemTransposeOp : Buckyball_Op<"mem_transpose", [DeclareOpInterfaceMethods]> { - let summary = "Logical matrix transpose on memrefs"; - let description = [{ - Transposes input[R,C] into output[C,R]. Element type and layout - constraints are enforced by the design bank-ssa lowering. - }]; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, - MemRefRankOf<[AnyType], [2]>:$output); - let assemblyFormat = [{$input $output prop-dict attr-dict `:` - type($input) type($output)}]; -} - -def TransposeOp : Buckyball_Op<"transpose"> { - let summary = "Matrix transpose operation"; - let description = [{ - Transposes iter x W to W x iter in scratchpad banks. - ISA: bb_transpose(op1_bank, wr_bank, iter, elem_bits); funct7 - from Core ballISA. - W = (bankWidth * cols) / elem_bits. - }]; - let arguments = (ins I64:$inputBankId, I64:$outputBankId, - I64:$iter, I64:$elemBits); - let assemblyFormat = "$inputBankId `,` $outputBankId `,` " - "$iter `,` $elemBits prop-dict attr-dict `:` " - "type($inputBankId)"; -} - -def BankTransposeOp : Buckyball_Op<"bank_transpose", [DeclareOpInterfaceMethods]> { - let summary = "Transpose using virtual bank handles"; - let arguments = (ins I64:$inBank, I64:$outBank, I64:$iter, I64:$elemBits); - let results = (outs I64:$outBankOut); - let assemblyFormat = - "$inBank $outBank $iter $elemBits prop-dict attr-dict `:` type($inBank) type($outBank) type($iter) type($elemBits)"; -} - -def TransposeIntrOp : Buckyball_IntrOpBase<"transpose">, - Arguments<(ins LLVM_Type, LLVM_Type)>; - -#endif // TRANSPOSEBALL_OPS diff --git a/waste/examples/components/balls/transpose/stack/src/Passes.cpp b/waste/examples/components/balls/transpose/stack/src/Passes.cpp deleted file mode 100644 index 08e941ee2..000000000 --- a/waste/examples/components/balls/transpose/stack/src/Passes.cpp +++ /dev/null @@ -1,59 +0,0 @@ -//===- Passes.cpp - TransposeBall passes ----------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -void populateTransposeBallLinalgPatterns(RewritePatternSet &patterns); -namespace buckyball { -void populateTransposeBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, - bool, int64_t); -void configureTransposeBallLegalizeForExportTarget(LLVMConversionTarget &, - bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LinalgToTransposePass - : public mlir::buddy::LinalgToBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LinalgToTransposePass) - StringRef getArgument() const final { - return "convert-linalg-to-transpose-ball"; - } - StringRef getDescription() const final { - return "Lower the Linalg operations TransposeBall executes."; - } - void populate(RewritePatternSet &patterns, int64_t, int64_t bankDepth, - int64_t) const { - mlir::buddy::populateTransposeBallLinalgPatterns(patterns); - } -}; - -class LowerTransposePass - : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerTransposePass) - StringRef getArgument() const final { return "lower-transpose-ball"; } - StringRef getDescription() const final { - return "Lower TransposeBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateTransposeBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureTransposeBallLegalizeForExportTarget( - target, stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - PassRegistration(); - return true; -}(); diff --git a/waste/examples/components/balls/vector/stack/src/Backend/include/IntrinsicsRISCVVecBall.td b/waste/examples/components/balls/vector/stack/src/Backend/include/IntrinsicsRISCVVecBall.td deleted file mode 100644 index 5d4fe7537..000000000 --- a/waste/examples/components/balls/vector/stack/src/Backend/include/IntrinsicsRISCVVecBall.td +++ /dev/null @@ -1,4 +0,0 @@ -//===- IntrinsicsRISCVVecBall.td - VecBall intrinsics --------------------===// - -let TargetPrefix = "riscv" in -def int_riscv_bb_vecmat16 : Intrinsic<[], [llvm_i64_ty, llvm_i64_ty], []>; diff --git a/waste/examples/components/balls/vector/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp b/waste/examples/components/balls/vector/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp deleted file mode 100644 index bd46db46f..000000000 --- a/waste/examples/components/balls/vector/stack/src/Conversion/LowerBuckyball/AssignPhysicalBanks.cpp +++ /dev/null @@ -1,20 +0,0 @@ -//===- AssignPhysicalBanks.cpp - Vector bank assignment -------------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult -BankVecMat16Op::assignPhysicalBanks(PatternRewriter &rewriter, - mlir::buddy::PhysicalBankState &) { - BankVecMat16Op op = *this; - rewriter.create(op.getLoc(), op.getOp1Bank(), op.getOp2Bank(), - op.getWrBank(), op.getIter(), op.getMode()); - rewriter.replaceOp(op, op.getWrBank()); - return success(); -} diff --git a/waste/examples/components/balls/vector/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp b/waste/examples/components/balls/vector/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp deleted file mode 100644 index 27fb18385..000000000 --- a/waste/examples/components/balls/vector/stack/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExport.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using namespace ::buddy::buckyball::legalize; - -namespace { -struct VecMat16Lowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(VecMat16Op op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - Value rs1 = packRs1BanksIter(rewriter, loc, adaptor.getOp1BankId(), - adaptor.getOp2BankId(), adaptor.getWrBankId(), - adaptor.getIter()); - rewriter.replaceOpWithNewOp( - op, rs1, adaptor.getMode(), - rewriter.getI32IntegerAttr( - buckyball_target::getBuckyballFunct7("VECMAT16"))); - return success(); - } -}; -} // namespace - -namespace mlir::buddy::buckyball { -void populateVecBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &converter, - RewritePatternSet &patterns, - bool stable, int64_t) { - (void)stable; - patterns.add(converter); -} - -void configureVecBallLegalizeForExportTarget(LLVMConversionTarget &target, - bool stable) { - (void)stable; - target.addIllegalOp(); -} -} // namespace mlir::buddy::buckyball diff --git a/waste/examples/components/balls/vector/stack/src/Dialect/Buckyball/VecBall.td b/waste/examples/components/balls/vector/stack/src/Dialect/Buckyball/VecBall.td deleted file mode 100644 index 10aa2da48..000000000 --- a/waste/examples/components/balls/vector/stack/src/Dialect/Buckyball/VecBall.td +++ /dev/null @@ -1,39 +0,0 @@ -//===-- VecBall.td - VecBall operations --------------------*- tablegen -*-===// -// -// VecBall operations for matrix multiplication using warp-16 matrix array. -// -//===----------------------------------------------------------------------===// - -#ifndef VECBALL_OPS -#define VECBALL_OPS - -include "Buckyball.td" - -//===----------------------------------------------------------------------===// -// VecBall operations -//===----------------------------------------------------------------------===// - -def VecMat16Op : Buckyball_Op<"vecmat16"> { - let summary = "Warp-16 matrix multiplication"; - let description = [{ - Performs matrix multiplication using 16x16 matrix array. - Corresponds to bb_vecmat16 instruction (funct7 from Core ballISA). - Belongs to VecBall (bid=0). - }]; - let arguments = (ins I64:$op1BankId, I64:$op2BankId, - I64:$wrBankId, I64:$iter, I64:$mode); - let assemblyFormat = "$op1BankId `,` $op2BankId `,` $wrBankId `,` " - "$iter `,` $mode attr-dict `:` " - "type($op1BankId)"; -} - -def BankVecMat16Op : Buckyball_Op<"bank_vecmat16", [DeclareOpInterfaceMethods]> { - let summary = "Warp-16 matmul using virtual bank handles"; - let arguments = (ins I64:$op1Bank, I64:$op2Bank, I64:$wrBank, - I64:$iter, I64:$mode); - let results = (outs I64:$wrBankOut); - let assemblyFormat = - "$op1Bank $op2Bank $wrBank $iter $mode attr-dict `:` type($op1Bank) type($op2Bank) type($wrBank) type($iter) type($mode)"; -} - -#endif // VECBALL_OPS diff --git a/waste/examples/components/balls/vector/stack/src/Passes.cpp b/waste/examples/components/balls/vector/stack/src/Passes.cpp deleted file mode 100644 index 7d61a6486..000000000 --- a/waste/examples/components/balls/vector/stack/src/Passes.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- Passes.cpp - VecBall passes ----------------------------------------===// - -#include "Conversion/BallPasses.h" - -using namespace mlir; - -namespace mlir::buddy { -namespace buckyball { -void populateVecBallLegalizeForLLVMExportPatterns(LLVMTypeConverter &, - RewritePatternSet &, bool, - int64_t); -void configureVecBallLegalizeForExportTarget(LLVMConversionTarget &, bool); -} // namespace buckyball -} // namespace mlir::buddy - -namespace { -class LowerVecPass : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerVecPass) - StringRef getArgument() const final { return "lower-vector-ball"; } - StringRef getDescription() const final { - return "Lower VecBall operations to LLVM intrinsics."; - } - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool stable, - int64_t bankDepth) const { - mlir::buddy::buckyball::populateVecBallLegalizeForLLVMExportPatterns( - converter, patterns, stable, bankDepth); - mlir::buddy::buckyball::configureVecBallLegalizeForExportTarget(target, - stable); - } -}; -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - mlir::buddy::registerBallLoweringPass(); - return true; -}(); diff --git a/waste/examples/designs/goban/stack/midend/src/FormMegaKernels.cpp b/waste/examples/designs/goban/stack/midend/src/FormMegaKernels.cpp deleted file mode 100644 index 6a6575584..000000000 --- a/waste/examples/designs/goban/stack/midend/src/FormMegaKernels.cpp +++ /dev/null @@ -1,33 +0,0 @@ -#include "Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Pass/Pass.h" - -using namespace mlir; -namespace { -class FormGobanMegaKernelsPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(FormGobanMegaKernelsPass) - StringRef getArgument() const final { return "form-goban-mega-kernels"; } - StringRef getDescription() const final { - return "Fuse quantized matrix operators within one Goban core"; - } - void runOnOperation() override { - DenseMap> blocks; - getOperation().walk([&](linalg::GenericOp op) { - if (op->hasAttr("buckyball.quantized_matmul")) - blocks[op->getBlock()].push_back(op); - }); - for (auto &entry : blocks) { - mlir::buddy::prepareQuantizedWeights(entry.second); - mlir::buddy::formMegaKernelRegions(entry.second); - } - } -}; -[[maybe_unused]] const bool registered = [] { - PassRegistration(); - return true; -}(); -} // namespace diff --git a/waste/examples/designs/pebble/stack/midend/ops.py b/waste/examples/designs/pebble/stack/midend/ops.py deleted file mode 100644 index 23d343b3b..000000000 --- a/waste/examples/designs/pebble/stack/midend/ops.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Pebble's banked INT8 operator emission.""" - -from copy import deepcopy - -from buddy_mlir import ir -from buddy_mlir.dialects import tosa as dialect - -from stack.compiler.frontend.operation import ( - Int8MatmulOp, - QuantizedDepthwiseConv2dOp, - QuantizedConv2dOp, - QuantizedMatmulOp, -) -from stack.compiler.frontend.layout.provenance import has_nhwc_layout -from stack.compiler.frontend.ops import linalg, tosa -from stack.compiler.frontend.ops.int8 import _rescale - - -def contraction(node, symbols): - linear = isinstance(node, Int8MatmulOp) - if linear: - activation, weight, activation_scale, weight_scale, bias = node.args - operation = QuantizedMatmulOp() - else: - activation, weight = node.args[:2] - ( - bias, - stride, - padding, - dilation, - transposed, - _, - groups, - activation_scale, - weight_scale, - ) = node.args[2:] - source = symbols[("__buddy_ops_by_name__", 0)][activation] - input_shape = list(source.tensor_meta["shape"]) - depthwise = ( - groups > 1 and groups == input_shape[1] == node.tensor_meta["shape"][1] - ) - if ( - transposed - or list(dilation) != [1, 1] - or stride[0] != stride[1] - or padding[0] != padding[1] - or (groups != 1 and not depthwise) - ): - raise ValueError( - "Pebble requires ordinary or depthwise convolution with unit dilation" - ) - operation = QuantizedDepthwiseConv2dOp() if depthwise else QuantizedConv2dOp() - operation._input_shape = input_shape - operation._weight_shape = list( - ir.RankedTensorType(symbols[(weight, 0)].type).shape - ) - operation._output_shape = list(node.tensor_meta["shape"]) - operation._output_nhwc = node.tensor_meta.get("activation_layout") == "nhwc" - operation._stride, operation._padding = stride[0], padding[0] - value = symbols[(activation, 0)] - if not has_nhwc_layout(value): - n, c, h, w = input_shape - value = dialect.TransposeOp( - ir.RankedTensorType.get([n, h, w, c], ir.IntegerType.get_signless(8)), - value, - tosa._create_permutation_attr([0, 2, 3, 1]), - ).result - symbols = {**symbols, (activation, 0): value} - operation.name = node.name - operation._arguments = [activation, weight] - operation.tensor_meta = dict(node.tensor_meta) - operation.trace_meta = deepcopy(node.trace_meta) - channels = node.tensor_meta["shape"][1] - operation._bias_i32 = [0] * channels - operation._dequant_scale = [1.0] * channels - operation._requant_scale = [1.0] * channels - operation._final_output = True - operation._activation = 0 - operation._output_scale = 1.0 - operation._lut_i8 = [0] - result = (linalg.quantized_matmul_op if linear else linalg.quantized_conv2d_op)( - operation, symbols - ) - return _rescale( - result, - activation_scale, - weight_scale, - bias, - symbols, - channel_axis=3 if not linear and operation._output_nhwc else 1, - ) - - -def permute(node, symbols): - # Preserve NHWC storage across a projection's logical NCHW boundary. - if list(node.args[1]) == [0, 3, 1, 2]: - value = symbols[(str(node.args[0]), 0)] - n, c, h, w = node.tensor_meta["shape"] - if list(ir.RankedTensorType(value.type).shape) != [n, h, w, c]: - raise ValueError("Pebble projection boundary requires NHWC storage") - return value - return tosa.permute_op(node, symbols) - - -ops_registry = { - **tosa.ops_registry, - "Int8Conv2dOp": contraction, - "Int8MatmulOp": contraction, - "PermuteOp": permute, -} diff --git a/waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/MegaConv2dToBankSSAPatterns.cpp b/waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/MegaConv2dToBankSSAPatterns.cpp deleted file mode 100644 index 7cdc7ae1f..000000000 --- a/waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/MegaConv2dToBankSSAPatterns.cpp +++ /dev/null @@ -1,468 +0,0 @@ -//===- MegaConv2dToBankSSAPatterns.cpp - Mega Conv2D to banks -----------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -constexpr int64_t kTile = 16; - -template -class MegaConv2dToBankSSAPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(MegaOp op, PatternRewriter &b) const override { - Location loc = op.getLoc(); - auto inputTy = dyn_cast(op.getInput().getType()); - auto weightTy = dyn_cast(op.getWeight().getType()); - auto biasTy = dyn_cast(op.getBias().getType()); - auto scaleTy = dyn_cast(op.getScale().getType()); - auto lutTy = dyn_cast(op.getLut().getType()); - auto outputTy = dyn_cast(op.getOutput().getType()); - if (op->template getParentOfType()) - return failure(); - bool finalOutput = true; - auto layout = op->template getAttrOfType("output_nhwc"); - bool nchwOutput = !(layout && layout.getValue()); - if (!inputTy || !weightTy || !biasTy || !scaleTy || !lutTy || !outputTy || - !inputTy.hasStaticShape() || !weightTy.hasStaticShape() || - !biasTy.hasStaticShape() || !scaleTy.hasStaticShape() || - !lutTy.hasStaticShape() || !outputTy.hasStaticShape()) - return op.emitError("Mega Conv2D requires static memrefs"); - if (!inputTy.getElementType().isInteger(8) || - !weightTy.getElementType().isInteger(8) || - !biasTy.getElementType().isInteger(32) || - !scaleTy.getElementType().isF32() || - !lutTy.getElementType().isInteger(8) || lutTy.getRank() != 1 || - lutTy.getShape()[0] != 1 || op.getActivation() < 0 || - op.getActivation() > 1 || - (finalOutput ? !outputTy.getElementType().isF32() - : !outputTy.getElementType().isInteger(8))) - return op.emitError( - "Mega Conv2D input/weight must be INT8, bias INT32, scale FP32, " - "and only the last stage may output FP32"); - if (op.getPadLow() != op.getPadHigh()) - return op.emitError("Mega Conv2D requires symmetric padding"); - - auto inputShape = inputTy.getShape(); - auto weightShape = weightTy.getShape(); - auto outputShape = outputTy.getShape(); - int64_t batch = inputShape[0]; - int64_t inputSize = inputShape[1]; - int64_t cin = inputShape[3]; - int64_t kernelSize = weightShape[0]; - int64_t cout = kDepthwise ? cin : weightShape[3]; - int64_t outputSize = nchwOutput ? outputShape[2] : outputShape[1]; - int64_t stride = op.getStride(); - int64_t padding = op.getPadLow(); - bool weightShapeMatches = - weightShape[1] == kernelSize && weightShape[2] == cin && - (kDepthwise ? weightShape[3] == 1 : weightShape[3] == cout); - if (batch <= 0 || inputSize <= 0 || cin <= 0 || kernelSize <= 0 || - cout <= 0 || stride <= 0 || padding < 0 || inputShape[2] != inputSize || - !weightShapeMatches || outputShape[0] != batch || - (nchwOutput ? outputShape[3] != outputSize - : outputShape[2] != outputSize) || - (nchwOutput ? outputShape[1] != cout : outputShape[3] != cout) || - biasTy.getShape()[0] != cout || scaleTy.getShape()[0] != cout) - return op.emitError("Mega Conv2D NHWC/HWCF shape mismatch"); - if (kernelSize > 7) - return op.emitError("Mega Conv2D kernel size exceeds Im2colBall limit 7"); - int64_t numerator = inputSize + 2 * padding - kernelSize; - if (numerator < 0 || outputSize != numerator / stride + 1) - return op.emitError("Mega Conv2D output shape mismatch"); - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth < 4 || - target.bankDepth % 4) - return op.emitError( - "Mega Conv2D requires 128-bit banks with depth divisible by 4"); - if (buckyball_target::getBuckyballBallMapping("SMATMUL_OS").outBW != 1) - return op.emitError("Mega Conv2D requires SMatMulBall outBW=1"); - int64_t kernelRows = (kernelSize * kernelSize + kTile - 1) / kTile * kTile; - int64_t windowCapacity = - std::min(target.bankDepth / 4, target.bankDepth / kernelRows * kTile); - windowCapacity = windowCapacity / kTile * kTile; - if (windowCapacity <= 0) - return op.emitError("Mega Conv2D kernel tile exceeds one bank"); - int64_t blockSide = 1; - while ((blockSide + 1) * (blockSide + 1) <= windowCapacity) - ++blockSide; - blockSide = std::min(blockSide, outputSize); - while (blockSide > 0 && (outputSize % blockSide != 0 || - ((blockSide - 1) * stride + kernelSize) * - ((blockSide - 1) * stride + kernelSize) > - target.bankDepth)) - --blockSide; - if (blockSide == 0) - return op.emitError("Mega Conv2D input tile exceeds one bank"); - int64_t blockInputSize = (blockSide - 1) * stride + kernelSize; - int64_t inputRows = blockInputSize * blockInputSize; - - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value sixteen = b.create(loc, kTile); - Value outputSizeValue = b.create(loc, outputSize); - Value blockSideValue = b.create(loc, blockSide); - Value blockInputSizeValue = - b.create(loc, blockInputSize); - Value inputSizeValue = b.create(loc, inputSize); - Value coutValue = b.create(loc, cout); - Value paddedCoutValue = b.create( - loc, (cout + kTile - 1) / kTile * kTile); - Value strideValue = b.create(loc, stride); - Value paddingValue = b.create(loc, padding); - Value zeroI8 = - b.create(loc, b.getI8Type(), b.getI8IntegerAttr(0)); - - auto outputChannelTileLoop = - b.create(loc, zero, paddedCoutValue, sixteen); - b.setInsertionPointToStart(outputChannelTileLoop.getBody()); - Value n0 = outputChannelTileLoop.getInductionVar(); - auto biasPackTy = MemRefType::get({4, 4}, b.getI32Type()); - auto scalePackTy = MemRefType::get({4, 4}, b.getF32Type()); - Value biasPack = b.create(loc, biasPackTy); - Value scalePack = b.create(loc, scalePackTy); - Value zeroI32 = b.create(loc, b.getI32Type(), - b.getI32IntegerAttr(0)); - Value oneF32 = b.create(loc, b.getF32Type(), - b.getF32FloatAttr(1.0)); - b.create(loc, zeroI32, biasPack); - b.create(loc, oneF32, scalePack); - for (int64_t i = 0; i < kTile; ++i) { - Value source = b.create( - loc, n0, b.create(loc, i)); - Value row = b.create(loc, i / 4); - Value column = b.create(loc, i % 4); - Value validChannel = b.create( - loc, arith::CmpIPredicate::slt, source, coutValue); - auto storeChannel = b.create(loc, validChannel, false); - b.setInsertionPointToStart(&storeChannel.getThenRegion().front()); - Value bias = b.create(loc, op.getBias(), source); - Value scale = b.create(loc, op.getScale(), source); - b.create(loc, bias, biasPack, ValueRange{row, column}); - b.create(loc, scale, scalePack, ValueRange{row, column}); - b.setInsertionPointAfter(storeChannel); - } - - Value biasBank = allocBank(b, loc, 1, 1); - Value biasLoaded = mvinBank(b, loc, biasPack, biasBank, 4); - Value biasState = b.create( - loc, biasLoaded.getType(), biasLoaded, createI64Const(b, loc, 0)); - Value scaleBank = allocBank(b, loc, 1, 1); - Value scaleLoaded = mvinBank(b, loc, scalePack, scaleBank, 4); - - for (int64_t batchIndex = 0; batchIndex < batch; ++batchIndex) { - auto outputYLoop = - b.create(loc, zero, outputSizeValue, blockSideValue); - b.setInsertionPointToStart(outputYLoop.getBody()); - Value outputY0 = outputYLoop.getInductionVar(); - auto outputXLoop = - b.create(loc, zero, outputSizeValue, blockSideValue); - b.setInsertionPointToStart(outputXLoop.getBody()); - Value outputX0 = outputXLoop.getInductionVar(); - int64_t blockWindows = blockSide * blockSide; - int64_t paddedM = (blockWindows + kTile - 1) / kTile * kTile; - Value inputBank = allocBank(b, loc, 1, 1); - Value patchBank = allocBank(b, loc, 1, 1); - Value weightBank = allocBank(b, loc, 1, 1); - Value resultBank = allocBank(b, loc, 1, 1); - Value resultState = resultBank; - - int64_t accumulationBlocks = kDepthwise ? kTile : cin; - Value accumulationEnd = - b.create(loc, accumulationBlocks); - auto inputPacksTy = MemRefType::get( - {accumulationBlocks, inputRows, kTile}, b.getI8Type()); - auto weightPacksTy = MemRefType::get( - {accumulationBlocks, kernelRows, kTile}, b.getI8Type()); - Value inputPacks = b.create(loc, inputPacksTy); - Value weightPacks = b.create(loc, weightPacksTy); - b.create(loc, zeroI8, inputPacks); - b.create(loc, zeroI8, weightPacks); - - Value globalInputY0 = b.create( - loc, b.create(loc, outputY0, strideValue), - paddingValue); - Value globalInputX0 = b.create( - loc, b.create(loc, outputX0, strideValue), - paddingValue); - Value inputBatchValue = b.create(loc, batchIndex); - auto packChannelLoop = - b.create(loc, zero, accumulationEnd, one); - b.setInsertionPointToStart(packChannelLoop.getBody()); - Value block = packChannelLoop.getInductionVar(); - Value channelValue = - kDepthwise ? Value(b.create(loc, n0, block)) : block; - - auto inputRowLoop = - b.create(loc, zero, blockInputSizeValue, one); - b.setInsertionPointToStart(inputRowLoop.getBody()); - Value inputRow = inputRowLoop.getInductionVar(); - auto inputColumnLoop = - b.create(loc, zero, blockInputSizeValue, one); - b.setInsertionPointToStart(inputColumnLoop.getBody()); - Value inputColumn = inputColumnLoop.getInductionVar(); - Value globalInputY = - b.create(loc, globalInputY0, inputRow); - Value globalInputX = - b.create(loc, globalInputX0, inputColumn); - Value inBounds = b.create( - loc, - b.create( - loc, - b.create(loc, arith::CmpIPredicate::sge, - globalInputY, zero), - b.create(loc, arith::CmpIPredicate::slt, - globalInputY, inputSizeValue)), - b.create( - loc, - b.create(loc, arith::CmpIPredicate::sge, - globalInputX, zero), - b.create(loc, arith::CmpIPredicate::slt, - globalInputX, inputSizeValue))); - if constexpr (kDepthwise) - inBounds = b.create( - loc, inBounds, - b.create(loc, arith::CmpIPredicate::slt, - channelValue, coutValue)); - auto validInput = b.create(loc, inBounds, false); - b.setInsertionPointToStart(&validInput.getThenRegion().front()); - Value linear = b.create( - loc, b.create(loc, inputRow, blockInputSizeValue), - inputColumn); - Value inputValue = - b.create(loc, op.getInput(), - ValueRange{inputBatchValue, globalInputY, - globalInputX, channelValue}); - b.create(loc, inputValue, inputPacks, - ValueRange{block, linear, zero}); - b.setInsertionPointAfter(inputRowLoop); - - Value kernelEnd = b.create(loc, kernelSize); - auto kernelRowLoop = b.create(loc, zero, kernelEnd, one); - b.setInsertionPointToStart(kernelRowLoop.getBody()); - Value kernelRow = kernelRowLoop.getInductionVar(); - auto kernelColumnLoop = b.create(loc, zero, kernelEnd, one); - b.setInsertionPointToStart(kernelColumnLoop.getBody()); - Value kernelColumn = kernelColumnLoop.getInductionVar(); - Value weightRow = b.create( - loc, b.create(loc, kernelRow, kernelEnd), - kernelColumn); - if constexpr (kDepthwise) { - Value multiplier = b.create(loc, 0); - Value validChannel = b.create( - loc, arith::CmpIPredicate::slt, channelValue, coutValue); - auto storeWeight = b.create(loc, validChannel, false); - b.setInsertionPointToStart(&storeWeight.getThenRegion().front()); - Value weightValue = b.create( - loc, op.getWeight(), - ValueRange{kernelRow, kernelColumn, channelValue, multiplier}); - b.create(loc, weightValue, weightPacks, - ValueRange{block, weightRow, block}); - b.setInsertionPointAfter(storeWeight); - } else { - auto outputChannelLoop = b.create(loc, zero, sixteen, one); - b.setInsertionPointToStart(outputChannelLoop.getBody()); - Value outputChannel = outputChannelLoop.getInductionVar(); - Value sourceChannel = b.create(loc, n0, outputChannel); - Value validChannel = b.create( - loc, arith::CmpIPredicate::slt, sourceChannel, coutValue); - auto storeWeight = b.create(loc, validChannel, false); - b.setInsertionPointToStart(&storeWeight.getThenRegion().front()); - Value weightValue = b.create( - loc, op.getWeight(), - ValueRange{kernelRow, kernelColumn, channelValue, sourceChannel}); - b.create(loc, weightValue, weightPacks, - ValueRange{block, weightRow, outputChannel}); - b.setInsertionPointAfter(storeWeight); - } - b.setInsertionPointAfter(kernelRowLoop); - b.setInsertionPointAfter(packChannelLoop); - - auto emitAccumulation = [&](Value channel, Value inputState, - Value patchState, Value weightState, - Value accumulatorState, bool first, - bool last) -> SmallVector { - SmallVector inputOffsets = {channel, b.getIndexAttr(0), - b.getIndexAttr(0)}; - SmallVector inputSizes = {b.getIndexAttr(1), - b.getIndexAttr(inputRows), - b.getIndexAttr(kTile)}; - SmallVector strides(3, b.getIndexAttr(1)); - Value inputSlice = b.create( - loc, inputPacks, inputOffsets, inputSizes, strides); - Value inputPack = b.create( - loc, inputSlice, SmallVector{{0, 1}, {2}}); - SmallVector weightOffsets = {channel, b.getIndexAttr(0), - b.getIndexAttr(0)}; - SmallVector weightSizes = {b.getIndexAttr(1), - b.getIndexAttr(kernelRows), - b.getIndexAttr(kTile)}; - Value weightSlice = b.create( - loc, weightPacks, weightOffsets, weightSizes, strides); - Value weightPack = b.create( - loc, weightSlice, SmallVector{{0, 1}, {2}}); - - Value inputLoaded = mvinBank(b, loc, inputPack, inputState, inputRows); - Value patchLoaded = b.create( - loc, patchState.getType(), inputLoaded, patchState, - createI64Const(b, loc, blockInputSize), - createI64Const(b, loc, kernelSize), createI64Const(b, loc, stride), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - createI64Const(b, loc, 0), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(blockWindows)); - Value weightLoaded = - mvinBank(b, loc, weightPack, weightState, kernelRows); - Value accumulated = - b.create( - loc, accumulatorState.getType(), patchLoaded, weightLoaded, - accumulatorState, - createI64ConstU(b, loc, matrixRs2(paddedM, kTile, kernelRows)), - createI1Const(b, loc, first), createI1Const(b, loc, last), - createI64Const(b, loc, 0)) - .getWrBankOut(); - return {inputLoaded, patchLoaded, weightLoaded, accumulated}; - }; - - SmallVector states = - emitAccumulation(zero, inputBank, patchBank, weightBank, resultState, - true, accumulationBlocks == 1); - if (accumulationBlocks > 2) { - Value lastChannel = - b.create(loc, accumulationBlocks - 1); - auto middleLoop = b.create(loc, one, lastChannel, one, - ValueRange(states)); - b.setInsertionPointToStart(middleLoop.getBody()); - ValueRange iterStates = middleLoop.getRegionIterArgs(); - SmallVector next = emitAccumulation( - middleLoop.getInductionVar(), iterStates[0], iterStates[1], - iterStates[2], iterStates[3], false, false); - b.create(loc, next); - b.setInsertionPointAfter(middleLoop); - states.assign(middleLoop.getResults().begin(), - middleLoop.getResults().end()); - } - if (accumulationBlocks > 1) { - Value lastChannel = - b.create(loc, accumulationBlocks - 1); - states = emitAccumulation(lastChannel, states[0], states[1], states[2], - states[3], false, true); - } - inputBank = states[0]; - patchBank = states[1]; - weightBank = states[2]; - resultState = states[3]; - - Value outputBank = allocBank(b, loc, 1, 1); - Value converted = - finalOutput - ? b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, createI64Const(b, loc, paddedM * 4), - b.getBoolAttr(op.getActivation() == 1)) - .getOutBankOut() - : b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, createI64Const(b, loc, paddedM * 4), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - b.getI64IntegerAttr(paddedM), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(paddedM), - b.getBoolAttr(op.getActivation() == 1)) - .getOutBankOut(); - releaseBank(b, loc, resultState); - - Type outputElementType = - finalOutput ? Type(b.getF32Type()) : Type(b.getI8Type()); - auto packedOutputTy = - MemRefType::get({paddedM, kTile}, outputElementType); - Value packedOutput = b.create(loc, packedOutputTy); - Value stored = mvoutBank(b, loc, packedOutput, converted, - finalOutput ? paddedM * 4 : paddedM); - b.create(loc, inputPacks); - b.create(loc, weightPacks); - - Value outputChannelOffset = n0; - Value outputBatchValue = - b.create(loc, batchIndex); - auto outputRowLoop = b.create(loc, zero, blockSideValue, one); - b.setInsertionPointToStart(outputRowLoop.getBody()); - Value outputRow = outputRowLoop.getInductionVar(); - auto outputColumnLoop = - b.create(loc, zero, blockSideValue, one); - b.setInsertionPointToStart(outputColumnLoop.getBody()); - Value outputColumn = outputColumnLoop.getInductionVar(); - Value outputPackedRow = b.create( - loc, b.create(loc, outputRow, blockSideValue), - outputColumn); - Value outputChannelEnd = sixteen; - auto outputChannelLoop = - b.create(loc, zero, outputChannelEnd, one); - b.setInsertionPointToStart(outputChannelLoop.getBody()); - Value outputChannel = outputChannelLoop.getInductionVar(); - Value globalChannel = - b.create(loc, outputChannelOffset, outputChannel); - Value outputValue = b.create( - loc, packedOutput, ValueRange{outputPackedRow, outputChannel}); - Value globalY = b.create(loc, outputY0, outputRow); - Value globalX = b.create(loc, outputX0, outputColumn); - Value validChannel = b.create( - loc, arith::CmpIPredicate::slt, globalChannel, coutValue); - auto storeOutput = b.create(loc, validChannel, false); - b.setInsertionPointToStart(&storeOutput.getThenRegion().front()); - if (nchwOutput) - b.create( - loc, outputValue, op.getOutput(), - ValueRange{outputBatchValue, globalChannel, globalY, globalX}); - else - b.create( - loc, outputValue, op.getOutput(), - ValueRange{outputBatchValue, globalY, globalX, globalChannel}); - b.setInsertionPointAfter(storeOutput); - b.setInsertionPointAfter(outputRowLoop); - - releaseBank(b, loc, stored); - b.create(loc, packedOutput); - releaseBank(b, loc, inputBank); - releaseBank(b, loc, patchBank); - releaseBank(b, loc, weightBank); - b.setInsertionPointAfter(outputXLoop); - b.setInsertionPointAfter(outputYLoop); - } - - releaseBank(b, loc, biasState); - releaseBank(b, loc, scaleLoaded); - b.create(loc, biasPack); - b.create(loc, scalePack); - b.setInsertionPointAfter(outputChannelTileLoop); - - b.eraseOp(op); - return success(); - } -}; - -} // namespace - -namespace mlir::buddy { -void populatePebbleMegaConv2dToBankSSAPatterns(RewritePatternSet &patterns) { - patterns.add, - MegaConv2dToBankSSAPattern>( - patterns.getContext()); -} -} // namespace mlir::buddy diff --git a/waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/MemTransposeToBankSSAPatterns.cpp b/waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/MemTransposeToBankSSAPatterns.cpp deleted file mode 100644 index 474594ddc..000000000 --- a/waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/MemTransposeToBankSSAPatterns.cpp +++ /dev/null @@ -1,282 +0,0 @@ -//===- MemTransposeToBankSSAPatterns.cpp - mem_transpose -> Bank* ---------===// -// -// Bank-unit transpose (same model as transpose ctests): -// pad to bank-row width -> contiguous mvin(rows,stride=1) -// -> transpose(rows) -> mvout(rows,stride=1) -// Dest bank holds transposed data as stacked bank rows of width -// W = groups * (bankWidthBytes / elemBytes). -// That "16" for i8 is elems-per-bank-row from bank width, not a 16x16 tile. -// -// Packing: iter is always the mvin row count. For fat [R,C] with R<=maxW, -// pack long dim C as iter (up to bankDepth) so we do not emit iter=R tiny ops. -// -//===----------------------------------------------------------------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Utils/BankUtils.h" - -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -constexpr int64_t kBankWidthBytes = 16; -constexpr int64_t kMaxGroups = 4; - -static int64_t alignUp(int64_t x, int64_t a) { - if (a <= 0) - return x; - return (x + a - 1) / a * a; -} - -static size_t elemsPerBankRow(Type elemType) { - unsigned bitWidth = elemType.getIntOrFloatBitWidth(); - if (bitWidth == 0 || bitWidth % 8 != 0) - return 0; - return kBankWidthBytes / (bitWidth / 8); -} - -static Value zeroConst(OpBuilder &b, Location loc, Type elemTy) { - if (auto it = dyn_cast(elemTy)) - return b.create(loc, b.getIntegerAttr(it, 0)); - if (isa(elemTy)) - return b.create(loc, b.getF32FloatAttr(0.0f)); - return {}; -} - -static void emitBankTranspose(OpBuilder &b, Location loc, Value inContig, - Value outContig, int64_t rows, int64_t groups, - int64_t elemBits) { - // CPU packed inContig before mvin; CPU reads outContig after mvout. - Value src = allocBank(b, loc, 1, groups); - Value dst = allocBank(b, loc, 1, groups); - Value loaded = mvinBank(b, loc, inContig, src, rows, /*stride=*/1); - Value transposed = b.create( - loc, dst.getType(), loaded, dst, createI64Const(b, loc, rows), - createI64Const(b, loc, elemBits)); - mvoutBank(b, loc, outContig, transposed, rows, /*stride=*/1); - releaseBank(b, loc, loaded); - releaseBank(b, loc, transposed); -} - -// dst[i,j] = src[j, c0+i]; dst:[h,rows] src:[rows,cols] -static void gatherCols(OpBuilder &b, Location loc, Value src, Value dst, - int64_t rows, Value c0, int64_t h) { - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value hUb = b.create(loc, h); - Value rUb = b.create(loc, rows); - auto iLoop = b.create(loc, zero, hUb, one); - { - OpBuilder::InsertionGuard g(b); - b.setInsertionPointToStart(iLoop.getBody()); - Value i = iLoop.getInductionVar(); - Value col = b.create(loc, c0, i); - auto jLoop = b.create(loc, zero, rUb, one); - b.setInsertionPointToStart(jLoop.getBody()); - Value j = jLoop.getInductionVar(); - Value v = b.create(loc, src, ValueRange{j, col}); - b.create(loc, v, dst, ValueRange{i, j}); - } -} - -// dst[c0+i, j] = src[j, i]; src:[rows,h] dst:[cols,rows] -static void scatterCols(OpBuilder &b, Location loc, Value src, Value dst, - int64_t rows, Value c0, int64_t h) { - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - Value hUb = b.create(loc, h); - Value rUb = b.create(loc, rows); - auto iLoop = b.create(loc, zero, hUb, one); - { - OpBuilder::InsertionGuard g(b); - b.setInsertionPointToStart(iLoop.getBody()); - Value i = iLoop.getInductionVar(); - Value row = b.create(loc, c0, i); - auto jLoop = b.create(loc, zero, rUb, one); - b.setInsertionPointToStart(jLoop.getBody()); - Value j = jLoop.getInductionVar(); - Value v = b.create(loc, src, ValueRange{j, i}); - b.create(loc, v, dst, ValueRange{row, j}); - } -} - -class MemTransposeToBankSSAPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(MemTransposeOp op, - PatternRewriter &rewriter) const override { - Location loc = op.getLoc(); - const int64_t bankDepth = buckyball_target::getBuckyballTarget().bankDepth; - Value input = op.getInput(); - Value output = op.getOutput(); - auto inputType = dyn_cast(input.getType()); - auto outputType = dyn_cast(output.getType()); - if (!inputType || !outputType || !inputType.hasStaticShape() || - !outputType.hasStaticShape()) - return op.emitError("requires static input and output memrefs"); - - int64_t rows = inputType.getShape()[0]; - int64_t cols = inputType.getShape()[1]; - if (rows <= 0 || cols <= 0) - return op.emitError("transpose requires positive rows/cols"); - if (outputType.getShape()[0] != cols || outputType.getShape()[1] != rows) - return op.emitError("output shape must transpose the input shape"); - - Type elemTy = inputType.getElementType(); - int64_t elemBits = elemTy.getIntOrFloatBitWidth(); - if (elemBits != 8 && elemBits != 32) - return op.emitError("transpose only supports 8-bit or 32-bit elements"); - - size_t epr = elemsPerBankRow(elemTy); - if (epr == 0) - return op.emitError("unsupported transpose element type"); - - Value zero = zeroConst(rewriter, loc, elemTy); - if (!zero) - return op.emitError("unsupported transpose element type for pad"); - - int64_t eprI = (int64_t)epr; - int64_t rowsPad = alignUp(rows, eprI); - int64_t colsPad = alignUp(cols, eprI); - int64_t maxW = eprI * kMaxGroups; - - // A fixed full-depth tile lets the long dimension remain an scf loop. - // The extra tail is zero-padded and discarded by outView below. - bool fat = rowsPad <= maxW && colsPad > rowsPad; - bool chunkedFat = fat && colsPad > bankDepth; - int64_t colsStorage = chunkedFat ? alignUp(colsPad, bankDepth) : colsPad; - auto inPadTy = MemRefType::get({rowsPad, colsStorage}, elemTy); - auto outPadTy = MemRefType::get({colsStorage, rowsPad}, elemTy); - Value inPad = rewriter.create(loc, inPadTy); - Value outPad = rewriter.create(loc, outPadTy); - rewriter.create(loc, ValueRange{zero}, ValueRange{inPad}); - rewriter.create(loc, ValueRange{zero}, ValueRange{outPad}); - - Value inView = rewriter.create( - loc, inPad, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(rows), - rewriter.getIndexAttr(cols)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - rewriter.create(loc, input, inView); - - // Fat [R,C] with R fitting in bank width: pack C as iter (long side). - if (fat) { - if (rowsPad % eprI != 0) - return op.emitError("fat transpose rowsPad not bank-row aligned"); - int64_t groups = rowsPad / eprI; - if (chunkedFat) { - for (int64_t c0i = 0; c0i < colsStorage; c0i += bankDepth) { - int64_t h = std::min(bankDepth, colsStorage - c0i); - Value c0 = rewriter.create(loc, c0i); - auto inContigTy = MemRefType::get({h, rowsPad}, elemTy); - auto outContigTy = MemRefType::get({rowsPad, h}, elemTy); - Value inContig = rewriter.create(loc, inContigTy); - Value outContig = rewriter.create(loc, outContigTy); - gatherCols(rewriter, loc, inPad, inContig, rowsPad, c0, h); - emitBankTranspose(rewriter, loc, inContig, outContig, h, groups, - elemBits); - scatterCols(rewriter, loc, outContig, outPad, rowsPad, c0, h); - rewriter.create(loc, inContig); - rewriter.create(loc, outContig); - } - } else { - Value c0 = rewriter.create(loc, 0); - auto inContigTy = MemRefType::get({colsPad, rowsPad}, elemTy); - auto outContigTy = MemRefType::get({rowsPad, colsPad}, elemTy); - Value inContig = rewriter.create(loc, inContigTy); - Value outContig = rewriter.create(loc, outContigTy); - gatherCols(rewriter, loc, inPad, inContig, rowsPad, c0, colsPad); - emitBankTranspose(rewriter, loc, inContig, outContig, colsPad, groups, - elemBits); - scatterCols(rewriter, loc, outContig, outPad, rowsPad, c0, colsPad); - rewriter.create(loc, inContig); - rewriter.create(loc, outContig); - } - } else { - for (int64_t r0 = 0; r0 < rowsPad;) { - int64_t h = std::min(bankDepth, rowsPad - r0); - for (int64_t c0 = 0; c0 < colsPad;) { - int64_t w = std::min(maxW, colsPad - c0); - if (w % eprI != 0) - return op.emitError("transpose tile width not bank-row aligned"); - int64_t groups = w / eprI; - - Value inSub = rewriter.create( - loc, inPad, - SmallVector{rewriter.getIndexAttr(r0), - rewriter.getIndexAttr(c0)}, - SmallVector{rewriter.getIndexAttr(h), - rewriter.getIndexAttr(w)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - auto inContigTy = MemRefType::get({h, w}, elemTy); - Value inContig = rewriter.create(loc, inContigTy); - rewriter.create(loc, inSub, inContig); - - auto outContigTy = MemRefType::get({w, h}, elemTy); - Value outContig = rewriter.create(loc, outContigTy); - emitBankTranspose(rewriter, loc, inContig, outContig, h, groups, - elemBits); - - Value outSub = rewriter.create( - loc, outPad, - SmallVector{rewriter.getIndexAttr(c0), - rewriter.getIndexAttr(r0)}, - SmallVector{rewriter.getIndexAttr(w), - rewriter.getIndexAttr(h)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - rewriter.create(loc, outContig, outSub); - rewriter.create(loc, inContig); - rewriter.create(loc, outContig); - c0 += w; - } - r0 += h; - } - } - - Value outView = rewriter.create( - loc, outPad, - SmallVector{rewriter.getIndexAttr(0), - rewriter.getIndexAttr(0)}, - SmallVector{rewriter.getIndexAttr(cols), - rewriter.getIndexAttr(rows)}, - SmallVector{rewriter.getIndexAttr(1), - rewriter.getIndexAttr(1)}); - rewriter.create(loc, outView, output); - rewriter.create(loc, inPad); - rewriter.create(loc, outPad); - rewriter.eraseOp(op); - return success(); - } -}; - -} // namespace - -namespace mlir::buddy { - -void populatePebbleMemTransposeToBankSSAPatterns(RewritePatternSet &patterns) { - // Pebble keeps transposes on banks; it takes precedence over the transpose - // Ball's own lowering. - patterns.add(patterns.getContext(), - /*benefit=*/2); -} - -} // namespace mlir::buddy diff --git a/waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/ResidentConvRegionToBankSSAPatterns.cpp b/waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/ResidentConvRegionToBankSSAPatterns.cpp deleted file mode 100644 index 6c585b498..000000000 --- a/waste/examples/designs/pebble/stack/midend/src/Conversion/LowerBuckyball/ResidentConvRegionToBankSSAPatterns.cpp +++ /dev/null @@ -1,3250 +0,0 @@ -//===- ResidentConvRegionToBankSSAPatterns.cpp ---------------------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/Matchers.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Target/BuckyballTargetRegistry.h" -#include "Trace/TraceOps.h" -#include "Utils/BankUtils.h" -#include "Utils/StageTrace.h" - -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/ScopeExit.h" - -#include -#include -#include - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { - -constexpr int64_t kTile = 16; -constexpr int64_t kMvin2dAddressableRows = 64; - -class ResidentConvRegionPattern : public OpRewritePattern { -public: - ResidentConvRegionPattern(MLIRContext *context) - : OpRewritePattern(context, 4) {} - - LogicalResult matchAndRewrite(MegaKernelOp kernel, - PatternRewriter &b) const override { - if (kernel.getBody().empty()) - return kernel.emitError("MegaKernel region must contain one block"); - Block &body = kernel.getBody().front(); - if (body.without_terminator().empty() || isa(body.front())) - return failure(); - - struct Stage { - Operation *op; - Value input; - Value rhs; - ValueRange inputs; - Value output; - Value weight; - Value weightChannelStride; - Value bias; - Value scale; - Value lut; - int64_t inputHeight; - int64_t inputWidth; - int64_t inputChannels; - int64_t outputHeight; - int64_t outputWidth; - int64_t outputChannels; - int64_t kernel; - int64_t stride; - int64_t padding; - int64_t activation; - float lhsScale; - float rhsScale; - float outputScale; - bool pool; - bool add; - bool multiply; - bool average; - bool depthwise; - bool channelSlice; - bool channelConcat; - bool resizeNearest; - int64_t channelOffset; - ArrayRef channelSegments; - int64_t scaleH; - int64_t scaleW; - bool finalOutput; - bool outputNhwc = false; - int64_t lutEntries; - SmallVector lutOutputScales; - }; - - SmallVector stages; - DenseMap producer; - for (Operation &operation : body.without_terminator()) { - Stage stage{}; - stage.op = &operation; - if (auto conv = dyn_cast(operation)) { - auto input = dyn_cast(conv.getInput().getType()); - auto weight = dyn_cast(conv.getWeight().getType()); - auto bias = dyn_cast(conv.getBias().getType()); - auto scale = dyn_cast(conv.getScale().getType()); - auto lut = dyn_cast(conv.getLut().getType()); - auto output = dyn_cast(conv.getOutput().getType()); - // A one-stage MegaKernel can still produce an INT8 value for the - // next kernel. Region/value identity alone does not mean this is - // the FP32 boundary; the output element type is the contract. - bool finalOutput = output && output.getElementType().isF32(); - auto layout = conv->getAttrOfType("output_nhwc"); - bool outputNhwc = layout && layout.getValue(); - bool nchwOutput = finalOutput && !outputNhwc; - if (!input || !weight || !bias || !scale || !lut || !output || - !input.hasStaticShape() || !weight.hasStaticShape() || - !bias.hasStaticShape() || !scale.hasStaticShape() || - !lut.hasStaticShape() || !output.hasStaticShape() || - input.getRank() != 4 || weight.getRank() != 4 || - lut.getRank() != 1 || !lut.getElementType().isInteger(8) || - output.getRank() != 4 || input.getShape()[0] != 1 || - output.getShape()[0] != 1 || !input.getElementType().isInteger(8) || - !weight.getElementType().isInteger(8) || - !bias.getElementType().isInteger(32) || - !scale.getElementType().isF32() || - (finalOutput ? !output.getElementType().isF32() - : !output.getElementType().isInteger(8)) || - conv.getActivation() < 0 || conv.getActivation() > 2 || - conv.getStride() <= 0 || conv.getPadLow() < 0 || - conv.getPadLow() != conv.getPadHigh()) - return conv.emitError("unsupported Conv stage in resident region"); - auto in = input.getShape(); - auto weights = weight.getShape(); - auto out = output.getShape(); - int64_t kernelSize = conv.getKernel(); - int64_t outputChannels = bias.getShape()[0]; - int64_t outputHeight = nchwOutput ? out[2] : out[1]; - int64_t outputWidth = nchwOutput ? out[3] : out[2]; - int64_t paddedKernel = - ((kernelSize * kernelSize + kTile - 1) / kTile) * kTile; - if (kernelSize <= 0 || kernelSize > 7 || - weights != ArrayRef({(outputChannels + kTile - 1) / kTile, - in[3], paddedKernel, kTile}) || - bias.getShape() != ArrayRef({outputChannels}) || - scale.getShape() != ArrayRef({outputChannels}) || - (conv.getActivation() == 2 - ? (lut.getShape()[0] != 256 && lut.getShape()[0] != 4096) - : lut.getShape()[0] != 1) || - (lut.getShape()[0] == 4096 && - (conv.getActivation() != 2 || outputChannels != 16)) || - (in[1] + 2 * conv.getPadLow() - kernelSize) / conv.getStride() + - 1 != - outputHeight || - (in[2] + 2 * conv.getPadLow() - kernelSize) / conv.getStride() + - 1 != - outputWidth || - (nchwOutput ? (out[1] != outputChannels) - : (out[3] != outputChannels))) - return conv.emitError("Conv shape is inconsistent"); - stage.input = conv.getInput(); - stage.output = conv.getOutput(); - stage.weight = conv.getWeight(); - stage.bias = conv.getBias(); - stage.scale = conv.getScale(); - stage.lut = conv.getLut(); - stage.inputHeight = in[1]; - stage.inputWidth = in[2]; - stage.inputChannels = in[3]; - stage.outputHeight = outputHeight; - stage.outputWidth = outputWidth; - stage.outputChannels = outputChannels; - stage.kernel = kernelSize; - stage.stride = conv.getStride(); - stage.padding = conv.getPadLow(); - stage.activation = conv.getActivation(); - stage.outputScale = conv.getOutputScale().convertToFloat(); - stage.finalOutput = finalOutput; - stage.outputNhwc = outputNhwc; - stage.lutEntries = lut.getShape()[0]; - if (auto scales = operation.getAttrOfType( - "lane_output_scales")) - stage.lutOutputScales.assign(scales.asArrayRef().begin(), - scales.asArrayRef().end()); - if (!stage.lutOutputScales.empty() && - static_cast(stage.lutOutputScales.size()) != - outputChannels) - return conv.emitError( - "lane_output_scales count does not match output channels"); - } else if (auto conv = dyn_cast(operation)) { - auto input = dyn_cast(conv.getInput().getType()); - auto weight = dyn_cast(conv.getWeight().getType()); - auto bias = dyn_cast(conv.getBias().getType()); - auto scale = dyn_cast(conv.getScale().getType()); - auto lut = dyn_cast(conv.getLut().getType()); - auto output = dyn_cast(conv.getOutput().getType()); - bool finalOutput = output && output.getElementType().isF32(); - auto layout = conv->getAttrOfType("output_nhwc"); - bool outputNhwc = layout && layout.getValue(); - bool nchwOutput = finalOutput && !outputNhwc; - if (!input || !weight || !bias || !scale || !lut || !output || - !input.hasStaticShape() || !weight.hasStaticShape() || - !bias.hasStaticShape() || !scale.hasStaticShape() || - !lut.hasStaticShape() || !output.hasStaticShape() || - input.getRank() != 4 || weight.getRank() != 4 || - lut.getRank() != 1 || !lut.getElementType().isInteger(8) || - output.getRank() != 4 || input.getShape()[0] != 1 || - output.getShape()[0] != 1 || !input.getElementType().isInteger(8) || - !weight.getElementType().isInteger(8) || - !bias.getElementType().isInteger(32) || - !scale.getElementType().isF32() || - (finalOutput ? !output.getElementType().isF32() - : !output.getElementType().isInteger(8)) || - conv.getActivation() < 0 || conv.getActivation() > 2 || - conv.getStride() <= 0 || conv.getPadLow() < 0 || - conv.getPadLow() != conv.getPadHigh()) - return conv.emitError( - "unsupported Depthwise Conv stage in resident region"); - auto in = input.getShape(); - auto weights = weight.getShape(); - auto out = output.getShape(); - int64_t kernelSize = conv.getKernel(); - int64_t outputChannels = bias.getShape()[0]; - int64_t outputHeight = nchwOutput ? out[2] : out[1]; - int64_t outputWidth = nchwOutput ? out[3] : out[2]; - if (kernelSize <= 0 || kernelSize > 7 || - weights != ArrayRef({kernelSize, kernelSize, in[3], 1}) || - bias.getShape() != ArrayRef({outputChannels}) || - scale.getShape() != ArrayRef({outputChannels}) || - (conv.getActivation() == 2 - ? (lut.getShape()[0] != 256 && lut.getShape()[0] != 4096) - : lut.getShape()[0] != 1) || - lut.getShape()[0] == 4096 || in[3] != outputChannels || - (in[1] + 2 * conv.getPadLow() - kernelSize) / conv.getStride() + - 1 != - outputHeight || - (in[2] + 2 * conv.getPadLow() - kernelSize) / conv.getStride() + - 1 != - outputWidth || - (nchwOutput ? (out[1] != outputChannels) - : (out[3] != outputChannels))) - return conv.emitError("Depthwise Conv shape is inconsistent"); - stage.input = conv.getInput(); - stage.output = conv.getOutput(); - stage.weight = conv.getWeight(); - stage.bias = conv.getBias(); - stage.scale = conv.getScale(); - stage.lut = conv.getLut(); - stage.inputHeight = in[1]; - stage.inputWidth = in[2]; - stage.inputChannels = in[3]; - stage.outputHeight = outputHeight; - stage.outputWidth = outputWidth; - stage.outputChannels = outputChannels; - stage.kernel = kernelSize; - stage.stride = conv.getStride(); - stage.padding = conv.getPadLow(); - stage.activation = conv.getActivation(); - stage.outputScale = conv.getOutputScale().convertToFloat(); - stage.depthwise = true; - stage.finalOutput = finalOutput; - stage.outputNhwc = outputNhwc; - stage.lutEntries = lut.getShape()[0]; - if (auto scales = operation.getAttrOfType( - "lane_output_scales")) - stage.lutOutputScales.assign(scales.asArrayRef().begin(), - scales.asArrayRef().end()); - if (!stage.lutOutputScales.empty() && - static_cast(stage.lutOutputScales.size()) != - outputChannels) - return conv.emitError( - "lane_output_scales count does not match output channels"); - } else if (auto pool = dyn_cast(operation)) { - auto input = dyn_cast(pool.getInput().getType()); - auto output = dyn_cast(pool.getOutput().getType()); - bool finalOutput = pool.getFinalOutput(); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 4 || - output.getRank() != 4 || input.getShape()[0] != 1 || - output.getShape()[0] != 1 || !input.getElementType().isInteger(8) || - !output.getElementType().isInteger(8) || pool.getKernel() <= 0 || - pool.getKernel() > 8 || pool.getStride() <= 0 || - pool.getPadding() < 0) - return pool.emitError("unsupported MaxPool stage in resident region"); - auto in = input.getShape(); - auto out = output.getShape(); - int64_t outputHeight = finalOutput ? out[2] : out[1]; - int64_t outputWidth = finalOutput ? out[3] : out[2]; - int64_t outputChannels = finalOutput ? out[1] : out[3]; - if (in[3] != outputChannels || - (in[1] + 2 * pool.getPadding() - pool.getKernel()) / - pool.getStride() + - 1 != - outputHeight || - (in[2] + 2 * pool.getPadding() - pool.getKernel()) / - pool.getStride() + - 1 != - outputWidth) - return pool.emitError("MaxPool shape is inconsistent"); - stage.input = pool.getInput(); - stage.output = pool.getOutput(); - stage.inputHeight = in[1]; - stage.inputWidth = in[2]; - stage.inputChannels = in[3]; - stage.outputHeight = outputHeight; - stage.outputWidth = outputWidth; - stage.outputChannels = outputChannels; - stage.kernel = pool.getKernel(); - stage.stride = pool.getStride(); - stage.padding = pool.getPadding(); - stage.pool = true; - stage.finalOutput = finalOutput; - } else if (auto add = dyn_cast(operation)) { - auto lhs = dyn_cast(add.getLhs().getType()); - auto rhs = dyn_cast(add.getRhs().getType()); - auto output = dyn_cast(add.getOutput().getType()); - float lhsScale = add.getLhsScale().convertToFloat(); - float rhsScale = add.getRhsScale().convertToFloat(); - float outputScale = add.getOutputScale().convertToFloat(); - if (!lhs || !rhs || !output || !lhs.hasStaticShape() || - !rhs.hasStaticShape() || !output.hasStaticShape() || - lhs.getRank() != 4 || rhs != lhs || output != lhs || - lhs.getShape()[0] != 1 || !lhs.getElementType().isInteger(8) || - add.getActivation() < 0 || add.getActivation() > 1 || - !std::isfinite(lhsScale) || !std::isfinite(rhsScale) || - !std::isfinite(outputScale) || lhsScale <= 0.0f || - rhsScale <= 0.0f || outputScale <= 0.0f) - return add.emitError("unsupported INT8 Add stage in resident region"); - auto shape = lhs.getShape(); - stage.input = add.getLhs(); - stage.rhs = add.getRhs(); - stage.output = add.getOutput(); - stage.inputHeight = stage.outputHeight = shape[1]; - stage.inputWidth = stage.outputWidth = shape[2]; - stage.inputChannels = stage.outputChannels = shape[3]; - stage.kernel = stage.stride = 1; - stage.activation = add.getActivation(); - stage.lhsScale = lhsScale; - stage.rhsScale = rhsScale; - stage.outputScale = outputScale; - stage.add = true; - } else if (auto multiply = dyn_cast(operation)) { - auto gate = dyn_cast(multiply.getLhs().getType()); - auto input = dyn_cast(multiply.getRhs().getType()); - auto output = dyn_cast(multiply.getOutput().getType()); - float gateScale = multiply.getLhsScale().convertToFloat(); - float inputScale = multiply.getRhsScale().convertToFloat(); - float outputScale = multiply.getOutputScale().convertToFloat(); - if (!gate || !input || !output || !gate.hasStaticShape() || - !input.hasStaticShape() || !output.hasStaticShape() || - gate.getRank() != 4 || input.getRank() != 4 || output != input || - gate.getShape()[0] != 1 || gate.getShape()[1] != 1 || - gate.getShape()[2] != 1 || input.getShape()[0] != 1 || - gate.getShape()[3] != input.getShape()[3] || - input.getShape()[3] <= 0 || - input.getShape()[3] > - buckyball_target::getBuckyballTarget().bankDepth * kTile || - !gate.getElementType().isInteger(8) || - !input.getElementType().isInteger(8) || - multiply.getActivation() != 0 || !std::isfinite(gateScale) || - !std::isfinite(inputScale) || !std::isfinite(outputScale) || - gateScale <= 0.0f || inputScale <= 0.0f || outputScale <= 0.0f) - return multiply.emitError( - "INT8 Mul requires [1,1,1,C] gate and [1,H,W,C] input, " - "1 <= C <= bankDepth*16, and activation=0"); - auto shape = input.getShape(); - stage.input = multiply.getRhs(); - stage.rhs = multiply.getLhs(); - stage.output = multiply.getOutput(); - stage.inputHeight = stage.outputHeight = shape[1]; - stage.inputWidth = stage.outputWidth = shape[2]; - stage.inputChannels = stage.outputChannels = shape[3]; - stage.kernel = stage.stride = 1; - stage.lhsScale = gateScale; - stage.rhsScale = inputScale; - stage.outputScale = outputScale; - stage.multiply = true; - } else if (auto average = dyn_cast(operation)) { - auto input = dyn_cast(average.getInput().getType()); - auto output = dyn_cast(average.getOutput().getType()); - float inputScale = average.getInputScale().convertToFloat(); - float outputScale = average.getOutputScale().convertToFloat(); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 4 || - output.getRank() != 4 || input.getShape()[0] != 1 || - input.getShape()[3] <= 0 || - input.getShape()[3] > - buckyball_target::getBuckyballTarget().bankDepth * kTile || - output.getShape() != - ArrayRef({1, 1, 1, input.getShape()[3]}) || - !input.getElementType().isInteger(8) || - !output.getElementType().isInteger(8) || - !std::isfinite(inputScale) || !std::isfinite(outputScale) || - inputScale <= 0.0f || outputScale <= 0.0f) - return average.emitError( - "unsupported GlobalAvgPool stage in resident region"); - auto shape = input.getShape(); - stage.input = average.getInput(); - stage.output = average.getOutput(); - stage.inputHeight = shape[1]; - stage.inputWidth = shape[2]; - stage.inputChannels = shape[3]; - stage.outputHeight = stage.outputWidth = 1; - stage.outputChannels = shape[3]; - stage.kernel = stage.stride = 1; - stage.lhsScale = inputScale; - stage.outputScale = outputScale; - stage.average = true; - } else if (auto slice = dyn_cast(operation)) { - auto input = dyn_cast(slice.getInput().getType()); - auto output = dyn_cast(slice.getOutput().getType()); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 4 || - output.getRank() != 4 || input.getShape()[0] != 1 || - output.getShape()[0] != 1 || !input.getElementType().isInteger(8) || - !output.getElementType().isInteger(8) || slice.getOffset() < 0 || - slice.getOffset() % kTile != 0 || - output.getShape()[3] % kTile != 0 || - input.getShape()[1] != output.getShape()[1] || - input.getShape()[2] != output.getShape()[2] || - slice.getOffset() + output.getShape()[3] > input.getShape()[3]) - return slice.emitError( - "unsupported channel slice in resident region"); - stage.input = slice.getInput(); - stage.output = slice.getOutput(); - stage.inputHeight = stage.outputHeight = input.getShape()[1]; - stage.inputWidth = stage.outputWidth = input.getShape()[2]; - stage.inputChannels = input.getShape()[3]; - stage.outputChannels = output.getShape()[3]; - stage.kernel = stage.stride = 1; - stage.channelSlice = true; - stage.channelOffset = slice.getOffset(); - } else if (auto concat = dyn_cast(operation)) { - auto output = dyn_cast(concat.getOutput().getType()); - ArrayRef segments = concat.getSegments(); - if (!output || !output.hasStaticShape() || output.getRank() != 4 || - output.getShape()[0] != 1 || - !output.getElementType().isInteger(8) || - concat.getInputs().empty() || - segments.size() != concat.getInputs().size()) - return concat.emitError( - "unsupported channel concat in resident region"); - int64_t channels = 0; - for (auto [inputValue, segment] : - llvm::zip(concat.getInputs(), segments)) { - auto input = dyn_cast(inputValue.getType()); - if (!input || !input.hasStaticShape() || input.getRank() != 4 || - input.getShape()[0] != 1 || - !input.getElementType().isInteger(8) || segment <= 0 || - segment % kTile != 0 || - input.getShape()[1] != output.getShape()[1] || - input.getShape()[2] != output.getShape()[2] || - input.getShape()[3] != segment) - return concat.emitError("channel concat shape is inconsistent"); - channels += segment; - } - if (channels != output.getShape()[3]) - return concat.emitError("channel concat channels are inconsistent"); - stage.inputs = concat.getInputs(); - stage.output = concat.getOutput(); - stage.outputHeight = output.getShape()[1]; - stage.outputWidth = output.getShape()[2]; - stage.outputChannels = output.getShape()[3]; - stage.kernel = stage.stride = 1; - stage.channelConcat = true; - stage.channelSegments = segments; - } else if (auto resize = dyn_cast(operation)) { - auto input = dyn_cast(resize.getInput().getType()); - auto output = dyn_cast(resize.getOutput().getType()); - if (!input || !output || !input.hasStaticShape() || - !output.hasStaticShape() || input.getRank() != 4 || - output.getRank() != 4 || input.getShape()[0] != 1 || - output.getShape()[0] != 1 || !input.getElementType().isInteger(8) || - !output.getElementType().isInteger(8) || resize.getScaleH() <= 0 || - resize.getScaleW() <= 0 || - input.getShape()[3] != output.getShape()[3] || - output.getShape()[1] != input.getShape()[1] * resize.getScaleH() || - output.getShape()[2] != input.getShape()[2] * resize.getScaleW()) - return resize.emitError( - "unsupported resize-nearest in resident region"); - stage.input = resize.getInput(); - stage.output = resize.getOutput(); - stage.inputHeight = input.getShape()[1]; - stage.inputWidth = input.getShape()[2]; - stage.inputChannels = input.getShape()[3]; - stage.outputHeight = output.getShape()[1]; - stage.outputWidth = output.getShape()[2]; - stage.outputChannels = output.getShape()[3]; - stage.kernel = stage.stride = 1; - stage.resizeNearest = true; - stage.scaleH = resize.getScaleH(); - stage.scaleW = resize.getScaleW(); - } else { - return operation.emitError( - "resident Conv region supports Conv, Depthwise Conv, MaxPool, " - "INT8 Add, INT8 Mul, GlobalAvgPool, channel slice, channel concat, " - "and nearest resize stages"); - } - - // A MegaKernel may have multiple external activation inputs, e.g. the - // projection branch of a residual block. Such values are captured by - // the region directly; only values produced inside this region enter - // `producer`. - if (producer.contains(stage.output)) - return operation.emitError("region tensor has multiple producers"); - producer[stage.output] = stages.size(); - stages.push_back(stage); - } - if (stages.back().output != kernel.getOutput()) - return kernel.emitError("final stage must produce MegaKernel output"); - for (auto [index, stage] : llvm::enumerate(stages)) { - if (stage.lutEntries != 4096) - continue; - int64_t consumers = 0; - for (const Stage &candidate : stages) - consumers += - candidate.input == stage.output || candidate.rhs == stage.output; - if (index != 0 || stages.size() < 2 || !stages[1].depthwise || - stages[1].input != stage.output || consumers != 1) - return stage.op->emitError( - "4096-entry lane LUT is only legal on stage 0 before one " - "16-channel depthwise Conv"); - } - - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || target.bankDepth < 4 || - target.bankDepth % 4 != 0 || target.bankNum <= 0) - return kernel.emitError( - "resident Conv region requires 128-bit rows, bankDepth divisible " - "by 4, and a positive bank count"); - if (buckyball_target::getBuckyballBallMapping("SMATMUL_OS").outBW != 1) - return kernel.emitError("resident Conv region requires SMatMul outBW=1"); - const int64_t smatmulRows = - buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); - const int64_t smatmulCols = - buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileCols"); - if (smatmulRows <= 0 || kTile % smatmulRows != 0 || smatmulCols != kTile) - return kernel.emitError( - "resident Conv region requires SMatMul tileRows to divide 16 and " - "tileCols to equal 16"); - - Location loc = kernel.getLoc(); - b.setInsertionPoint(kernel); - Value zeroI8 = - b.create(loc, b.getI8Type(), b.getI8IntegerAttr(0)); - Value minI8 = b.create(loc, b.getI8Type(), - b.getI8IntegerAttr(-128)); - Value zeroI32 = b.create(loc, b.getI32Type(), - b.getI32IntegerAttr(0)); - Value oneF32 = b.create(loc, b.getF32Type(), - b.getF32FloatAttr(1.0)); - Value zero = b.create(loc, 0); - Value one = b.create(loc, 1); - SmallVector hostPacks; - Value zeroPack = b.create( - loc, MemRefType::get({target.bankDepth, kTile}, b.getI8Type())); - Value minPack = b.create( - loc, MemRefType::get({target.bankDepth, kTile}, b.getI8Type())); - hostPacks.push_back(zeroPack); - hostPacks.push_back(minPack); - b.create(loc, zeroI8, zeroPack); - b.create(loc, minI8, minPack); - - DenseMap packedBiases; - DenseMap packedScales; - DenseMap packedDepthwiseWeights; - DenseMap packedLuts; - DenseMap packedLutOutputScales; - for (Stage &stage : stages) { - if (stage.pool || stage.add || stage.multiply || stage.average || - stage.channelSlice || stage.channelConcat || stage.resizeNearest) - continue; - if (!stage.depthwise && stage.kernel == 1 && stage.stride == 1 && - stage.padding == 0 && stage.inputChannels % kTile == 0) { - auto metadata = - b.create(loc, stage.weight); - Value channelStride = metadata.getStrides()[1]; - Value contiguousLanes = b.create( - loc, arith::CmpIPredicate::eq, metadata.getStrides()[3], one); - Value strideLowerBound = b.create( - loc, arith::CmpIPredicate::sge, channelStride, - b.create(loc, kTile)); - Value strideUpperBound = b.create( - loc, arith::CmpIPredicate::sle, channelStride, - b.create(loc, 1016)); - Value strideAligned = b.create( - loc, arith::CmpIPredicate::eq, - b.create(loc, channelStride, - b.create(loc, 8)), - zero); - b.create( - loc, - b.create( - loc, - b.create(loc, contiguousLanes, strideLowerBound), - b.create(loc, strideUpperBound, strideAligned)), - "1x1 Conv weight requires contiguous lanes and a channel stride " - "between 16 and 1016 bytes divisible by 8"); - stage.weightChannelStride = - b.create(loc, b.getI64Type(), channelStride); - } - int64_t outputPanels = (stage.outputChannels + kTile - 1) / kTile; - Value biasPack = b.create( - loc, MemRefType::get({outputPanels, 4, 4}, b.getI32Type())); - Value scalePack = b.create( - loc, MemRefType::get({outputPanels, 4, 4}, b.getF32Type())); - b.create(loc, zeroI32, biasPack); - b.create(loc, oneF32, scalePack); - hostPacks.append({biasPack, scalePack}); - packedBiases[stage.op] = biasPack; - packedScales[stage.op] = scalePack; - if (stage.depthwise) { - int64_t paddedK = - (stage.kernel * stage.kernel + kTile - 1) / kTile * kTile; - Value weightPack = b.create( - loc, MemRefType::get({outputPanels, kTile, paddedK, kTile}, - b.getI8Type())); - b.create(loc, zeroI8, weightPack); - hostPacks.push_back(weightPack); - packedDepthwiseWeights[stage.op] = weightPack; - } - if (stage.finalOutput && stage.activation == 2) { - Value outputScalePack = b.create( - loc, MemRefType::get({outputPanels, 4, 4}, b.getF32Type())); - b.create(loc, oneF32, outputScalePack); - hostPacks.push_back(outputScalePack); - packedLutOutputScales[stage.op] = outputScalePack; - for (int64_t panel = 0; panel < outputPanels; ++panel) { - for (int64_t lane = 0; lane < kTile; ++lane) { - int64_t channel = panel * kTile + lane; - if (channel >= stage.outputChannels) - continue; - float outputScale = stage.lutOutputScales.empty() - ? stage.outputScale - : stage.lutOutputScales[channel]; - if (!std::isfinite(outputScale) || outputScale <= 0.0f) - return stage.op->emitError( - "LUT output scales must be finite and positive"); - b.create( - loc, - b.create(loc, b.getF32Type(), - b.getF32FloatAttr(outputScale)), - outputScalePack, - ValueRange{b.create(loc, panel), - b.create(loc, lane / 4), - b.create(loc, lane % 4)}); - } - } - } - auto outputPanelLoop = b.create( - loc, zero, b.create(loc, outputPanels), one); - b.setInsertionPointToStart(outputPanelLoop.getBody()); - Value outputPanel = outputPanelLoop.getInductionVar(); - Value channelBase = b.create( - loc, outputPanel, b.create(loc, kTile)); - auto outputLaneLoop = b.create( - loc, zero, b.create(loc, kTile), one); - b.setInsertionPointToStart(outputLaneLoop.getBody()); - Value outputLane = outputLaneLoop.getInductionVar(); - Value outputChannel = - b.create(loc, channelBase, outputLane); - auto validChannel = b.create( - loc, - b.create( - loc, arith::CmpIPredicate::slt, outputChannel, - b.create(loc, stage.outputChannels)), - false); - b.setInsertionPointToStart(&validChannel.getThenRegion().front()); - Value group = b.create( - loc, outputLane, b.create(loc, 4)); - Value groupLane = b.create( - loc, outputLane, b.create(loc, 4)); - b.create( - loc, b.create(loc, stage.bias, outputChannel), - biasPack, ValueRange{outputPanel, group, groupLane}); - b.create( - loc, b.create(loc, stage.scale, outputChannel), - scalePack, ValueRange{outputPanel, group, groupLane}); - if (stage.depthwise) { - for (int64_t ky = 0; ky < stage.kernel; ++ky) { - for (int64_t kx = 0; kx < stage.kernel; ++kx) { - Value weight = b.create( - loc, stage.weight, - ValueRange{b.create(loc, ky), - b.create(loc, kx), - outputChannel, zero}); - b.create( - loc, weight, packedDepthwiseWeights.lookup(stage.op), - ValueRange{outputPanel, outputLane, - b.create( - loc, ky * stage.kernel + kx), - outputLane}); - } - } - } - b.setInsertionPointAfter(validChannel); - b.setInsertionPointAfter(outputPanelLoop); - - if (stage.activation == 2) { - Value lutPack; - if (stage.lutEntries == 4096) { - constexpr int64_t laneLutRows = 4096 / (4 * kTile); - lutPack = b.create( - loc, MemRefType::get({laneLutRows, 4 * kTile}, b.getI8Type())); - } else { - lutPack = b.create( - loc, MemRefType::get({kTile, kTile}, b.getI8Type())); - } - hostPacks.push_back(lutPack); - packedLuts[stage.op] = lutPack; - auto lutLoop = b.create( - loc, zero, b.create(loc, stage.lutEntries), - one); - b.setInsertionPointToStart(lutLoop.getBody()); - Value index = lutLoop.getInductionVar(); - Value lutValue = b.create(loc, stage.lut, index); - if (stage.lutEntries == 4096) { - constexpr int64_t laneLutRows = 4096 / (4 * kTile); - Value group = b.create( - loc, index, - b.create(loc, laneLutRows * kTile)); - Value withinGroup = b.create( - loc, index, - b.create(loc, laneLutRows * kTile)); - b.create( - loc, lutValue, lutPack, - ValueRange{ - b.create( - loc, withinGroup, - b.create(loc, kTile)), - b.create( - loc, - b.create( - loc, group, - b.create(loc, kTile)), - b.create( - loc, withinGroup, - b.create(loc, kTile)))}); - } else { - b.create( - loc, lutValue, lutPack, - ValueRange{ - b.create( - loc, index, b.create(loc, kTile)), - b.create( - loc, index, - b.create(loc, kTile))}); - } - b.setInsertionPointAfter(lutLoop); - } - } - - // One resident zero tile is reused for all invalid-edge masking. - Value zeroBank = allocBank(b, loc, 1, 1); - zeroBank = mvinBank(b, loc, zeroPack, zeroBank, target.bankDepth); - - DenseSet materialized; - // Debug-only stages that were materialized for a stage hash. Downstream - // stages must consume the checked host tensor instead of recursively - // recomputing it, otherwise a passing hash does not isolate a boundary. - DenseSet traceMaterialized; - DenseSet tracingStages; - - struct TileBanks { - SmallVector banks; - int64_t panelRows; - int64_t panelsPerBank; - int64_t panelCount; - }; - - struct CachedTile { - int64_t stage = -1; - TileBanks tile{}; - int64_t firstPanel = 0; - Value y; - Value x; - int64_t width = 0; - }; - DenseMap gateCaches; - CachedTile residualCache; - - auto tilePanelsPerBank = [&](int64_t panelRows) { - if (panelRows <= 0 || panelRows > target.bankDepth) - return int64_t{0}; - if (panelRows > kMvin2dAddressableRows) - return int64_t{1}; - return std::min(target.bankDepth, kMvin2dAddressableRows) / - panelRows; - }; - - auto allocateTile = [&](int64_t panelCount, int64_t panelRows, Value fill) { - TileBanks tile{{}, panelRows, tilePanelsPerBank(panelRows), panelCount}; - if (tile.panelsPerBank == 0) { - kernel.emitError("resident tile does not fit one bank"); - return tile; - } - int64_t bankCount = - (panelCount + tile.panelsPerBank - 1) / tile.panelsPerBank; - for (int64_t index = 0; index < bankCount; ++index) { - int64_t panelsInBank = std::min( - tile.panelsPerBank, panelCount - index * tile.panelsPerBank); - Value bank = allocBank(b, loc, 1, 1); - tile.banks.push_back(mvinBank(b, loc, - fill == minI8 ? minPack : zeroPack, bank, - panelsInBank * panelRows)); - } - return tile; - }; - - auto releaseTile = [&](TileBanks &tile) { - for (Value bank : tile.banks) - releaseBank(b, loc, bank); - tile.banks.clear(); - }; - - auto loadInt8Tile = - [&](Value input, int64_t inputHeight, int64_t inputWidth, - int64_t inputChannels, Value y0, Value x0, Value firstPanel, - int64_t panelCount, int64_t height, int64_t width, - int64_t storagePanelRows = -1) -> FailureOr { - const int64_t panelRows = - storagePanelRows < 0 ? height * width : storagePanelRows; - TileBanks tile = allocateTile(panelCount, panelRows, zeroI8); - if (tile.banks.empty()) - return failure(); - Value yBegin = b.create( - loc, zero, b.create(loc, zero, y0)); - Value xBegin = b.create( - loc, zero, b.create(loc, zero, x0)); - Value yEnd = b.create( - loc, b.create(loc, height), - b.create( - loc, b.create(loc, inputHeight), y0)); - Value xEnd = b.create( - loc, b.create(loc, width), - b.create( - loc, b.create(loc, inputWidth), x0)); - auto inputType = cast(input.getType()); - SmallVector inputStrides; - int64_t inputOffset; - bool directMvin2d = - inputType.getElementType().isInteger(8) && - succeeded(inputType.getStridesAndOffset(inputStrides, inputOffset)) && - inputStrides[3] == 1 && inputStrides[2] == inputChannels && - inputStrides[1] == inputWidth * inputChannels && - inputChannels % 8 == 0 && inputChannels <= 1016 && - inputWidth <= 1023 && - std::min(panelCount, tile.panelsPerBank) * tile.panelRows <= - kMvin2dAddressableRows; - if (directMvin2d) { - Value eight = b.create(loc, 8); - Value pixelBytes = createI64Const(b, loc, inputChannels); - Value sourceWidth = createI64Const(b, loc, inputWidth); - for (int64_t localPanel = 0; localPanel < panelCount; ++localPanel) { - int64_t bankIndex = localPanel / tile.panelsPerBank; - int64_t slot = localPanel % tile.panelsPerBank; - Value channel = b.create( - loc, - b.create( - loc, firstPanel, - b.create(loc, localPanel)), - b.create(loc, kTile)); - Value validBytes = b.create( - loc, b.create(loc, kTile), - b.create( - loc, b.create(loc, inputChannels), - channel)); - Value validBytesI64 = - b.create(loc, b.getI64Type(), validBytes); - auto yLoop = b.create(loc, yBegin, yEnd, one, - ValueRange{tile.banks[bankIndex]}); - b.setInsertionPointToStart(yLoop.getBody()); - Value localY = yLoop.getInductionVar(); - auto xLoop = b.create(loc, xBegin, xEnd, eight, - yLoop.getRegionIterArgs()); - b.setInsertionPointToStart(xLoop.getBody()); - Value localX = xLoop.getInductionVar(); - Value globalY = b.create(loc, y0, localY); - Value globalX = b.create(loc, x0, localX); - Value copyWidth = b.create( - loc, eight, b.create(loc, xEnd, localX)); - Value slice = b.create( - loc, input, - SmallVector{b.getIndexAttr(0), globalY, globalX, - channel}, - SmallVector{b.getIndexAttr(1), b.getIndexAttr(1), - copyWidth, validBytes}, - SmallVector(4, b.getIndexAttr(1))); - Value row = b.create( - loc, b.create(loc, slot * tile.panelRows), - b.create( - loc, - b.create( - loc, localY, - b.create(loc, width)), - localX)); - Value loaded = b.create( - loc, b.getI64Type(), slice, xLoop.getRegionIterArgs().front(), - createI64Const(b, loc, 1), pixelBytes, sourceWidth, - b.create(loc, b.getI64Type(), row), - b.create(loc, b.getI64Type(), copyWidth), - validBytesI64); - b.create(loc, loaded); - b.setInsertionPointAfter(xLoop); - b.create(loc, xLoop.getResults()); - b.setInsertionPointAfter(yLoop); - tile.banks[bankIndex] = yLoop.getResult(0); - } - return tile; - } - for (size_t bankIndex = 0; bankIndex < tile.banks.size(); ++bankIndex) { - int64_t panelBegin = bankIndex * tile.panelsPerBank; - int64_t panelEnd = - std::min(panelCount, panelBegin + tile.panelsPerBank); - int64_t packedRows = (panelEnd - panelBegin) * tile.panelRows; - Value pack = b.create( - loc, MemRefType::get({packedRows, kTile}, b.getI8Type())); - b.create(loc, zeroI8, pack); - for (int64_t localPanel = panelBegin; localPanel < panelEnd; - ++localPanel) { - auto yLoop = b.create(loc, yBegin, yEnd, one); - b.setInsertionPointToStart(yLoop.getBody()); - Value localY = yLoop.getInductionVar(); - auto xLoop = b.create(loc, xBegin, xEnd, one); - b.setInsertionPointToStart(xLoop.getBody()); - Value localX = xLoop.getInductionVar(); - Value globalY = b.create(loc, y0, localY); - Value globalX = b.create(loc, x0, localX); - auto laneLoop = b.create( - loc, zero, b.create(loc, kTile), one); - b.setInsertionPointToStart(laneLoop.getBody()); - Value lane = laneLoop.getInductionVar(); - Value channel = b.create( - loc, - b.create( - loc, - b.create( - loc, firstPanel, - b.create(loc, localPanel)), - b.create(loc, kTile)), - lane); - int64_t slot = localPanel % tile.panelsPerBank; - Value row = b.create( - loc, b.create(loc, slot * tile.panelRows), - b.create( - loc, - b.create( - loc, localY, - b.create(loc, width)), - localX)); - auto channelValid = b.create( - loc, - b.create( - loc, arith::CmpIPredicate::slt, channel, - b.create(loc, inputChannels)), - false); - b.setInsertionPointToStart(&channelValid.getThenRegion().front()); - Value value = b.create( - loc, input, ValueRange{zero, globalY, globalX, channel}); - b.create(loc, value, pack, ValueRange{row, lane}); - b.setInsertionPointAfter(channelValid); - b.setInsertionPointAfter(laneLoop); - b.setInsertionPointAfter(xLoop); - b.setInsertionPointAfter(yLoop); - } - tile.banks[bankIndex] = - mvinBank(b, loc, pack, tile.banks[bankIndex], packedRows); - b.create(loc, pack); - } - return tile; - }; - - auto copyTile = [&](TileBanks &source, TileBanks &destination, - int64_t panelCount, int64_t height, int64_t width, - int64_t destinationBase, - int64_t destinationStride) -> LogicalResult { - if (source.panelCount != panelCount || source.panelRows != height * width) - return failure(); - SmallVector destinationStates(destination.banks.begin(), - destination.banks.end()); - for (int64_t localPanel = 0; localPanel < panelCount; ++localPanel) { - int64_t sourceBank = localPanel / source.panelsPerBank; - int64_t sourceSlot = localPanel % source.panelsPerBank; - int64_t destinationBank = localPanel / destination.panelsPerBank; - int64_t destinationSlot = localPanel % destination.panelsPerBank; - destinationStates[destinationBank] = - b.create( - loc, destinationStates[destinationBank].getType(), - source.banks[sourceBank], destinationStates[destinationBank], - createI64Const(b, loc, height * width), - b.getI64IntegerAttr(height), b.getI64IntegerAttr(width), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), - createI64Const(b, loc, sourceSlot * source.panelRows), - createI64Const(b, loc, - destinationSlot * destination.panelRows + - destinationBase), - createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - destination.banks.assign(destinationStates.begin(), - destinationStates.end()); - return success(); - }; - - auto repackSMatMulInput = [&](Value source, int64_t rows, - int64_t reduction) { - if (rows == 1 || smatmulRows == kTile || reduction == kTile) - return source; - Value packed = allocBank(b, loc, 1, 1); - int64_t reductionTiles = reduction / kTile; - for (int64_t row = 0; row < rows; row += smatmulRows) { - for (int64_t reductionTile = 0; reductionTile < reductionTiles; - ++reductionTile) { - int64_t sourceBase = reductionTile * kTile + row; - int64_t outputBase = - (row / smatmulRows * reductionTiles + reductionTile) * - smatmulRows; - // Copy contiguous groups rather than issuing one command per row. - // MAXPOOL with kernel=stride=1 is an identity copy of bank rows. - int64_t copySide = smatmulRows >= 4 ? 2 : 1; - int64_t copyRows = copySide * copySide; - for (int64_t offset = 0; offset < smatmulRows; offset += copyRows) { - packed = b.create( - loc, packed.getType(), source, packed, - createI64Const(b, loc, copyRows), - b.getI64IntegerAttr(copySide), - b.getI64IntegerAttr(copySide), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(0), - createI64Const(b, loc, sourceBase + offset), - createI64Const(b, loc, outputBase + offset), - createI64Const(b, loc, copySide), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - } - } - return packed; - }; - - auto copyFp32Tile = [&](Value source, Value destination, Value outputBase, - int64_t side, int64_t outputStride) -> Value { - // FP32 has four bank rows per spatial position. A contiguous tile can - // be copied as one square of raw rows, independent of its tensor shape. - if (outputStride == side * 4) { - int64_t rawSide = side * 2; - return b - .create( - loc, destination.getType(), source, destination, - createI64Const(b, loc, rawSide * rawSide), - b.getI64IntegerAttr(rawSide), b.getI64IntegerAttr(rawSide), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), createI64Const(b, loc, 0), outputBase, - createI64Const(b, loc, rawSide), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - for (int64_t y = 0; y < side; ++y) { - for (int64_t x = 0; x < side; ++x) { - Value base = b.create( - loc, outputBase, - createI64Const(b, loc, y * outputStride + x * 4)); - destination = b.create( - loc, destination.getType(), source, destination, - createI64Const(b, loc, 4), b.getI64IntegerAttr(2), - b.getI64IntegerAttr(2), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(0), - createI64Const(b, loc, (y * side + x) * 4), base, - createI64Const(b, loc, 2), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - } - return destination; - }; - - std::function - emitInto; - emitInto = [&](int64_t stageIndex, Value y0, Value x0, int64_t height, - int64_t width, Value firstPanel, int64_t panelCount, - TileBanks &destination, int64_t destinationBase, - int64_t destinationStride) -> LogicalResult { - // Keep nested stage emission before this anchor and resume after it. - Operation *insertionAnchor = - b.create(loc, 0).getOperation(); - b.setInsertionPoint(insertionAnchor); - Stage &stage = stages[stageIndex]; - auto trace = stage.op->getAttrOfType("trace_node"); - bool cycle = trace && cast(trace.get("cycle")).getValue() && - !tracingStages.contains(stageIndex); - if (cycle) { - tracingStages.insert(stageIndex); - auto start = b.create<::buddy::trace::StartOp>( - loc, cast(trace.get("id")), b.getBoolAttr(true), - b.getBoolAttr(false), cast(trace.get("tag"))); - start->setAttr("id_path", trace.get("id_path")); - } - llvm::scope_exit eraseInsertionAnchor([&]() { - b.setInsertionPointAfter(insertionAnchor); - if (cycle) { - auto end = b.create<::buddy::trace::EndOp>( - loc, zero.getType(), zero, cast(trace.get("id")), - b.getBoolAttr(true), b.getBoolAttr(false), - cast(trace.get("tag"))); - end->setAttr("id_path", trace.get("id_path")); - tracingStages.erase(stageIndex); - } - b.eraseOp(insertionAnchor); - }); - const bool fp32Output = stage.finalOutput && !stage.pool; - const bool externalInput = !producer.contains(stage.input); - int64_t totalPanels = (stage.outputChannels + kTile - 1) / kTile; - if (height <= 0 || width <= 0 || panelCount <= 0 || - panelCount > totalPanels || destination.panelCount != panelCount || - destinationBase < 0 || destinationStride < width || - destinationBase + (height - 1) * destinationStride + width > - destination.panelRows) - return stage.op->emitError("invalid resident tile request"); - - if (traceMaterialized.contains(stageIndex)) { - FailureOr loaded = - loadInt8Tile(stage.output, stage.outputHeight, stage.outputWidth, - stage.outputChannels, y0, x0, firstPanel, panelCount, - height, width); - if (failed(loaded)) - return stage.op->emitError("failed to reload traced resident stage"); - LogicalResult copied = - copyTile(*loaded, destination, panelCount, height, width, - destinationBase, destinationStride); - releaseTile(*loaded); - return copied; - } - - if (materialized.contains(stageIndex)) { - auto inputType = cast(stage.output.getType()); - SmallVector inputStrides; - int64_t inputOffset; - bool directMvin2d = - inputType.getElementType().isInteger(8) && - succeeded( - inputType.getStridesAndOffset(inputStrides, inputOffset)) && - inputStrides[3] == 1 && inputStrides[2] == stage.outputChannels && - inputStrides[1] == stage.outputWidth * stage.outputChannels && - stage.outputChannels % 8 == 0 && stage.outputChannels <= 1016 && - stage.outputWidth <= 1023 && - std::min(panelCount, destination.panelsPerBank) * - destination.panelRows <= - kMvin2dAddressableRows; - if (directMvin2d) { - Value yBegin = b.create(loc, y0, zero); - Value yEnd = b.create( - loc, - b.create( - loc, y0, b.create(loc, height)), - b.create(loc, stage.outputHeight)); - Value xBegin = b.create(loc, x0, zero); - Value xEnd = b.create( - loc, - b.create( - loc, x0, b.create(loc, width)), - b.create(loc, stage.outputWidth)); - Value eight = b.create(loc, 8); - Value pixelBytes = createI64Const(b, loc, stage.outputChannels); - Value sourceWidth = createI64Const(b, loc, stage.outputWidth); - for (int64_t localPanel = 0; localPanel < panelCount; ++localPanel) { - int64_t bankIndex = localPanel / destination.panelsPerBank; - int64_t bankSlot = localPanel % destination.panelsPerBank; - Value channel = b.create( - loc, - b.create( - loc, firstPanel, - b.create(loc, localPanel)), - b.create(loc, kTile)); - Value validBytes = b.create( - loc, b.create(loc, kTile), - b.create( - loc, - b.create(loc, stage.outputChannels), - channel)); - Value validBytesI64 = - b.create(loc, b.getI64Type(), validBytes); - auto yLoop = - b.create(loc, yBegin, yEnd, one, - ValueRange{destination.banks[bankIndex]}); - b.setInsertionPointToStart(yLoop.getBody()); - Value globalY = yLoop.getInductionVar(); - auto xLoop = b.create(loc, xBegin, xEnd, eight, - yLoop.getRegionIterArgs()); - b.setInsertionPointToStart(xLoop.getBody()); - Value globalX = xLoop.getInductionVar(); - Value copyWidth = b.create( - loc, eight, b.create(loc, xEnd, globalX)); - Value slice = b.create( - loc, stage.output, - SmallVector{b.getIndexAttr(0), globalY, globalX, - channel}, - SmallVector{b.getIndexAttr(1), b.getIndexAttr(1), - copyWidth, validBytes}, - SmallVector(4, b.getIndexAttr(1))); - Value destinationRow = b.create( - loc, - b.create( - loc, bankSlot * destination.panelRows + destinationBase), - b.create( - loc, - b.create( - loc, b.create(loc, globalY, y0), - b.create(loc, - destinationStride)), - b.create(loc, globalX, x0))); - Value loaded = b.create( - loc, b.getI64Type(), slice, xLoop.getRegionIterArgs().front(), - createI64Const(b, loc, 1), pixelBytes, sourceWidth, - b.create(loc, b.getI64Type(), - destinationRow), - b.create(loc, b.getI64Type(), copyWidth), - validBytesI64); - b.create(loc, loaded); - b.setInsertionPointAfter(xLoop); - b.create(loc, xLoop.getResults()); - b.setInsertionPointAfter(yLoop); - destination.banks[bankIndex] = yLoop.getResult(0); - } - return success(); - } - SmallVector packs; - for (Value bank : destination.banks) { - Value pack = b.create( - loc, MemRefType::get({target.bankDepth, kTile}, b.getI8Type())); - mvoutBank(b, loc, pack, bank, target.bankDepth); - packs.push_back(pack); - } - for (int64_t localPanel = 0; localPanel < panelCount; ++localPanel) { - int64_t bankIndex = localPanel / destination.panelsPerBank; - int64_t bankSlot = localPanel % destination.panelsPerBank; - auto yLoop = b.create( - loc, zero, b.create(loc, height), one); - b.setInsertionPointToStart(yLoop.getBody()); - Value localY = yLoop.getInductionVar(); - auto xLoop = b.create( - loc, zero, b.create(loc, width), one); - b.setInsertionPointToStart(xLoop.getBody()); - Value localX = xLoop.getInductionVar(); - Value globalY = b.create(loc, y0, localY); - Value globalX = b.create(loc, x0, localX); - Value yValid = b.create( - loc, - b.create(loc, arith::CmpIPredicate::sge, globalY, - zero), - b.create( - loc, arith::CmpIPredicate::slt, globalY, - b.create(loc, stage.outputHeight))); - Value xValid = b.create( - loc, - b.create(loc, arith::CmpIPredicate::sge, globalX, - zero), - b.create( - loc, arith::CmpIPredicate::slt, globalX, - b.create(loc, stage.outputWidth))); - auto valid = b.create( - loc, b.create(loc, yValid, xValid), false); - b.setInsertionPointToStart(&valid.getThenRegion().front()); - auto laneLoop = b.create( - loc, zero, b.create(loc, kTile), one); - b.setInsertionPointToStart(laneLoop.getBody()); - Value lane = laneLoop.getInductionVar(); - Value panel = b.create( - loc, firstPanel, - b.create(loc, localPanel)); - Value channel = b.create( - loc, - b.create( - loc, panel, b.create(loc, kTile)), - lane); - auto channelValid = b.create( - loc, - b.create( - loc, arith::CmpIPredicate::slt, channel, - b.create(loc, stage.outputChannels)), - false); - b.setInsertionPointToStart(&channelValid.getThenRegion().front()); - Value value = b.create( - loc, stage.output, ValueRange{zero, globalY, globalX, channel}); - Value row = b.create( - loc, - b.create( - loc, bankSlot * destination.panelRows + destinationBase), - b.create( - loc, - b.create( - loc, localY, - b.create(loc, destinationStride)), - localX)); - b.create(loc, value, packs[bankIndex], - ValueRange{row, lane}); - b.setInsertionPointAfter(channelValid); - b.setInsertionPointAfter(laneLoop); - b.setInsertionPointAfter(valid); - b.setInsertionPointAfter(yLoop); - } - for (size_t index = 0; index < destination.banks.size(); ++index) { - mvinBank(b, loc, packs[index], destination.banks[index], - target.bankDepth); - b.create(loc, packs[index]); - } - return success(); - } - - if (stage.channelSlice) { - IntegerAttr::ValueType requestedFirstPanel; - if (!matchPattern(firstPanel, m_ConstantInt(&requestedFirstPanel))) - return stage.op->emitError( - "resident channel slice panel offset must be constant"); - Value sourceFirstPanel = b.create( - loc, - requestedFirstPanel.getSExtValue() + stage.channelOffset / kTile); - TileBanks source = allocateTile(panelCount, height * width, zeroI8); - if (source.banks.empty()) - return failure(); - if (producer.contains(stage.input)) { - if (failed(emitInto(producer.lookup(stage.input), y0, x0, height, - width, sourceFirstPanel, panelCount, source, 0, - width))) - return failure(); - } else { - FailureOr loaded = - loadInt8Tile(stage.input, stage.inputHeight, stage.inputWidth, - stage.inputChannels, y0, x0, sourceFirstPanel, - panelCount, height, width); - if (failed(loaded)) - return stage.op->emitError( - "failed to load INT8 channel slice input"); - releaseTile(source); - source = std::move(*loaded); - } - LogicalResult copied = - copyTile(source, destination, panelCount, height, width, - destinationBase, destinationStride); - releaseTile(source); - return copied; - } - - if (stage.channelConcat) { - IntegerAttr::ValueType requestedFirstPanel; - if (!matchPattern(firstPanel, m_ConstantInt(&requestedFirstPanel))) - return stage.op->emitError( - "resident channel concat panel offset must be constant"); - int64_t requestedPanel = requestedFirstPanel.getSExtValue(); - for (int64_t localPanel = 0; localPanel < panelCount; ++localPanel) { - int64_t outputPanel = requestedPanel + localPanel; - int64_t sourcePanelBase = 0; - int64_t sourceIndex = -1; - for (auto [index, segment] : llvm::enumerate(stage.channelSegments)) { - int64_t panels = segment / kTile; - if (outputPanel >= sourcePanelBase && - outputPanel < sourcePanelBase + panels) { - sourceIndex = index; - break; - } - sourcePanelBase += panels; - } - if (sourceIndex < 0) - return stage.op->emitError( - "resident channel concat source is missing"); - Value sourceFirstPanel = b.create( - loc, outputPanel - sourcePanelBase); - TileBanks source = allocateTile(1, height * width, zeroI8); - if (source.banks.empty()) - return failure(); - Value sourceValue = stage.inputs[sourceIndex]; - if (producer.contains(sourceValue)) { - if (failed(emitInto(producer.lookup(sourceValue), y0, x0, height, - width, sourceFirstPanel, 1, source, 0, width))) - return failure(); - } else { - auto sourceType = cast(sourceValue.getType()); - FailureOr loaded = - loadInt8Tile(sourceValue, sourceType.getShape()[1], - sourceType.getShape()[2], sourceType.getShape()[3], - y0, x0, sourceFirstPanel, 1, height, width); - if (failed(loaded)) - return stage.op->emitError( - "failed to load INT8 channel concat input"); - releaseTile(source); - source = std::move(*loaded); - } - int64_t destinationBank = localPanel / destination.panelsPerBank; - int64_t destinationSlot = localPanel % destination.panelsPerBank; - destination.banks[destinationBank] = - b.create( - loc, destination.banks[destinationBank].getType(), - source.banks.front(), destination.banks[destinationBank], - createI64Const(b, loc, height * width), - b.getI64IntegerAttr(height), b.getI64IntegerAttr(width), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), createI64Const(b, loc, 0), - createI64Const(b, loc, - destinationSlot * destination.panelRows + - destinationBase), - createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - releaseTile(source); - } - return success(); - } - - if (stage.resizeNearest) { - for (int64_t localY = 0; localY < height; ++localY) { - Value sourceY = b.create( - loc, - b.create( - loc, y0, b.create(loc, localY)), - b.create(loc, stage.scaleH)); - for (int64_t localX = 0; localX < width; ++localX) { - Value sourceX = b.create( - loc, - b.create( - loc, x0, b.create(loc, localX)), - b.create(loc, stage.scaleW)); - if (producer.contains(stage.input)) { - if (failed(emitInto( - producer.lookup(stage.input), sourceY, sourceX, 1, 1, - firstPanel, panelCount, destination, - destinationBase + localY * destinationStride + localX, - destinationStride))) - return failure(); - } else { - TileBanks source = allocateTile(panelCount, 1, zeroI8); - if (source.banks.empty()) - return failure(); - FailureOr loaded = - loadInt8Tile(stage.input, stage.inputHeight, stage.inputWidth, - stage.inputChannels, sourceY, sourceX, - firstPanel, panelCount, 1, 1); - if (failed(loaded)) - return stage.op->emitError( - "failed to load INT8 resize-nearest input"); - releaseTile(source); - source = std::move(*loaded); - LogicalResult copied = copyTile( - source, destination, panelCount, 1, 1, - destinationBase + localY * destinationStride + localX, - destinationStride); - releaseTile(source); - if (failed(copied)) - return failure(); - } - } - } - return success(); - } - - CachedTile *cache = nullptr; - if (residualCache.stage == stageIndex) - cache = &residualCache; - if (cache) { - IntegerAttr::ValueType requestedFirstPanel; - if (!matchPattern(firstPanel, m_ConstantInt(&requestedFirstPanel))) - cache = nullptr; - else if (requestedFirstPanel.getSExtValue() < cache->firstPanel || - requestedFirstPanel.getSExtValue() + panelCount > - cache->firstPanel + cache->tile.panelCount) - cache = nullptr; - } - if (cache) { - IntegerAttr::ValueType requestedFirstPanel; - if (!matchPattern(firstPanel, m_ConstantInt(&requestedFirstPanel))) - return stage.op->emitError( - "cached tile panel offset must be constant"); - SmallVector destinationStates(destination.banks.begin(), - destination.banks.end()); - for (int64_t localPanel = 0; localPanel < panelCount; ++localPanel) { - int64_t sourcePanel = requestedFirstPanel.getSExtValue() - - cache->firstPanel + localPanel; - int64_t sourceBank = sourcePanel / cache->tile.panelsPerBank; - int64_t sourceSlot = sourcePanel % cache->tile.panelsPerBank; - int64_t destinationBank = localPanel / destination.panelsPerBank; - int64_t destinationSlot = localPanel % destination.panelsPerBank; - Value sourceBase = b.create( - loc, b.getI64Type(), - b.create( - loc, - b.create( - loc, sourceSlot * cache->tile.panelRows), - b.create( - loc, - b.create( - loc, b.create(loc, y0, cache->y), - b.create(loc, cache->width)), - b.create(loc, x0, cache->x)))); - destinationStates[destinationBank] = - b.create( - loc, destinationStates[destinationBank].getType(), - cache->tile.banks[sourceBank], - destinationStates[destinationBank], - createI64Const(b, loc, height * width), - b.getI64IntegerAttr(cache->width), - b.getI64IntegerAttr(width), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(0), sourceBase, - createI64Const(b, loc, - destinationSlot * destination.panelRows + - destinationBase), - createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - destination.banks.assign(destinationStates.begin(), - destinationStates.end()); - return success(); - } - - auto maskInvalidOutput = [&]() { - for (size_t bankIndex = 0; bankIndex < destination.banks.size(); - ++bankIndex) { - int64_t panelBegin = bankIndex * destination.panelsPerBank; - int64_t panelEnd = std::min( - panelCount, panelBegin + destination.panelsPerBank); - auto panelLoop = b.create( - loc, b.create(loc, panelBegin), - b.create(loc, panelEnd), one, - ValueRange{destination.banks[bankIndex]}); - b.setInsertionPointToStart(panelLoop.getBody()); - Value localPanel = panelLoop.getInductionVar(); - Value panelState = panelLoop.getRegionIterArgs().front(); - auto yLoop = b.create( - loc, zero, b.create(loc, height), one, - ValueRange{panelState}); - b.setInsertionPointToStart(yLoop.getBody()); - Value localY = yLoop.getInductionVar(); - Value yState = yLoop.getRegionIterArgs().front(); - auto xLoop = b.create( - loc, zero, b.create(loc, width), one, - ValueRange{yState}); - b.setInsertionPointToStart(xLoop.getBody()); - Value localX = xLoop.getInductionVar(); - Value xState = xLoop.getRegionIterArgs().front(); - Value globalY = b.create(loc, y0, localY); - Value globalX = b.create(loc, x0, localX); - Value yInvalid = b.create( - loc, - b.create(loc, arith::CmpIPredicate::slt, globalY, - zero), - b.create( - loc, arith::CmpIPredicate::sge, globalY, - b.create(loc, stage.outputHeight))); - Value xInvalid = b.create( - loc, - b.create(loc, arith::CmpIPredicate::slt, globalX, - zero), - b.create( - loc, arith::CmpIPredicate::sge, globalX, - b.create(loc, stage.outputWidth))); - auto invalid = b.create( - loc, b.create(loc, yInvalid, xInvalid), false); - b.setInsertionPointToStart(&invalid.getThenRegion().front()); - Value bankSlot = b.create( - loc, localPanel, - b.create(loc, panelBegin)); - Value outputBase = b.create( - loc, b.getI64Type(), - b.create( - loc, - b.create( - loc, - b.create(loc, bankSlot, - b.create( - loc, destination.panelRows)), - b.create(loc, destinationBase)), - b.create( - loc, - b.create(loc, localY, - b.create( - loc, destinationStride)), - b.create( - loc, localX, - b.create( - loc, - fp32Output && stage.activation != 2 ? 4 : 1))))); - b.create( - loc, xState.getType(), zeroBank, xState, - createI64Const(b, loc, - fp32Output && stage.activation != 2 ? 4 : 1), - b.getI64IntegerAttr(fp32Output && stage.activation != 2 ? 2 : 1), - b.getI64IntegerAttr(fp32Output && stage.activation != 2 ? 2 : 1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), createI64Const(b, loc, 0), outputBase, - createI64Const(b, loc, - fp32Output && stage.activation != 2 ? 2 : 1), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)); - b.setInsertionPointAfter(invalid); - b.create(loc, xState); - b.setInsertionPointAfter(xLoop); - b.create(loc, xLoop.getResult(0)); - b.setInsertionPointAfter(yLoop); - b.create(loc, yLoop.getResult(0)); - b.setInsertionPointAfter(panelLoop); - destination.banks[bankIndex] = panelLoop.getResult(0); - } - }; - - if (stage.add) { - Value lhsRatio = b.create( - loc, b.getF32Type(), - b.getF32FloatAttr(stage.lhsScale / stage.outputScale)); - Value rhsRatio = b.create( - loc, b.getF32Type(), - b.getF32FloatAttr(stage.rhsScale / stage.outputScale)); - - bool mainBranchExternal = !producer.contains(stage.input); - if (!mainBranchExternal) { - for (int64_t cursor = producer.lookup(stage.input); - stages[cursor].input != kernel.getInput();) { - if (!producer.contains(stages[cursor].input)) { - mainBranchExternal = true; - break; - } - int64_t next = producer.lookup(stages[cursor].input); - if (next >= cursor) - return stage.op->emitError("INT8 Add branch is not acyclic"); - cursor = next; - } - } - const bool usePanelAdd = - mainBranchExternal || !producer.contains(stage.rhs); - if (usePanelAdd) { - IntegerAttr::ValueType requestedFirstPanel; - if (!matchPattern(firstPanel, m_ConstantInt(&requestedFirstPanel))) - return stage.op->emitError( - "external INT8 Add panel offset must be constant"); - int64_t panelRows = height * width; - for (int64_t panelBegin = 0; panelBegin < panelCount;) { - int64_t destinationBank = panelBegin / destination.panelsPerBank; - int64_t firstDestinationSlot = - panelBegin % destination.panelsPerBank; - int64_t chunkCount = std::min( - {panelCount - panelBegin, tilePanelsPerBank(panelRows), - destination.panelsPerBank - firstDestinationSlot}); - Value panel = b.create( - loc, requestedFirstPanel.getSExtValue() + panelBegin); - TileBanks lhs = allocateTile(chunkCount, panelRows, zeroI8); - TileBanks rhs = allocateTile(chunkCount, panelRows, zeroI8); - if (lhs.banks.size() != 1 || rhs.banks.size() != 1) - return failure(); - if (producer.contains(stage.input)) { - if (failed(emitInto(producer.lookup(stage.input), y0, x0, height, - width, panel, chunkCount, lhs, 0, width))) - return failure(); - } else { - auto input = cast(stage.input.getType()); - FailureOr loaded = - loadInt8Tile(stage.input, input.getShape()[1], - input.getShape()[2], input.getShape()[3], y0, x0, - panel, chunkCount, height, width); - if (failed(loaded)) - return stage.op->emitError("failed to load INT8 Add lhs"); - releaseTile(lhs); - lhs = std::move(*loaded); - } - if (producer.contains(stage.rhs)) { - if (failed(emitInto(producer.lookup(stage.rhs), y0, x0, height, - width, panel, chunkCount, rhs, 0, width))) - return failure(); - } else { - auto input = cast(stage.rhs.getType()); - FailureOr loaded = - loadInt8Tile(stage.rhs, input.getShape()[1], - input.getShape()[2], input.getShape()[3], y0, x0, - panel, chunkCount, height, width); - if (failed(loaded)) - return stage.op->emitError("failed to load INT8 Add rhs"); - releaseTile(rhs); - rhs = std::move(*loaded); - } - TileBanks sum = allocateTile(chunkCount, panelRows, zeroI8); - if (sum.banks.size() != 1) - return failure(); - sum.banks.front() = b.create( - loc, sum.banks.front().getType(), lhs.banks.front(), - rhs.banks.front(), sum.banks.front(), - createI64Const(b, loc, target.bankDepth), lhsRatio, rhsRatio, - b.getBoolAttr(stage.activation == 1)); - for (int64_t localPanel = 0; localPanel < chunkCount; - ++localPanel) { - int64_t destinationSlot = firstDestinationSlot + localPanel; - destination.banks[destinationBank] = - b.create( - loc, destination.banks[destinationBank].getType(), - sum.banks.front(), destination.banks[destinationBank], - createI64Const(b, loc, height * width), - b.getI64IntegerAttr(height), b.getI64IntegerAttr(width), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), - createI64Const(b, loc, localPanel * panelRows), - createI64Const(b, loc, - destinationSlot * destination.panelRows + - destinationBase), - createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - releaseTile(lhs); - releaseTile(rhs); - releaseTile(sum); - panelBegin += chunkCount; - } - return success(); - } - - int64_t branchStage = producer.lookup(stage.input); - DenseSet branchAncestors; - for (int64_t cursor = branchStage;;) { - branchAncestors.insert(cursor); - Value input = stages[cursor].input; - if (input == kernel.getInput()) - break; - if (!producer.contains(input)) - return stage.op->emitError( - "INT8 Add main branch has no region producer"); - int64_t next = producer.lookup(input); - if (next >= cursor) - return stage.op->emitError("INT8 Add branch is not acyclic"); - cursor = next; - } - int64_t residualStage = producer.lookup(stage.rhs); - bool residualHasCommonProducer = true; - while (!branchAncestors.contains(residualStage)) { - Value input = stages[residualStage].input; - if (!producer.contains(input)) { - residualHasCommonProducer = false; - break; - } - int64_t next = producer.lookup(input); - if (next >= residualStage) - return stage.op->emitError("INT8 Add branch is not acyclic"); - residualStage = next; - } - Value residualY = y0; - Value residualX = x0; - int64_t residualHeight = height; - int64_t residualWidth = width; - for (int64_t cursor = branchStage; - residualHasCommonProducer && cursor != residualStage;) { - Stage &branch = stages[cursor]; - if (branch.add || branch.average || - branch.input == kernel.getInput() || - !producer.contains(branch.input)) - return stage.op->emitError( - "INT8 Add main branch must be a linear Conv chain from rhs"); - residualY = b.create( - loc, - b.create( - loc, residualY, - b.create(loc, branch.stride)), - b.create(loc, branch.padding)); - residualX = b.create( - loc, - b.create( - loc, residualX, - b.create(loc, branch.stride)), - b.create(loc, branch.padding)); - residualHeight = (residualHeight - 1) * branch.stride + branch.kernel; - residualWidth = (residualWidth - 1) * branch.stride + branch.kernel; - int64_t next = producer.lookup(branch.input); - if (next >= cursor) - return stage.op->emitError("INT8 Add branch is not acyclic"); - cursor = next; - } - int64_t residualPanelRows = residualHeight * residualWidth; - int64_t residualPanels = - (stages[residualStage].outputChannels + kTile - 1) / kTile; - int64_t residualPanelsPerBank = tilePanelsPerBank(residualPanelRows); - if (residualPanelsPerBank <= 0) - return stage.op->emitError( - "INT8 Add residual tile does not fit one bank"); - int64_t chunkPanels = std::min( - residualPanels, - std::min(destination.panelsPerBank, residualPanelsPerBank)); - if (chunkPanels <= 0) - return stage.op->emitError("INT8 Add panel chunk is empty"); - - for (int64_t panelBegin = 0; panelBegin < panelCount; - panelBegin += chunkPanels) { - int64_t chunkCount = - std::min(chunkPanels, panelCount - panelBegin); - Value chunkFirstPanel = firstPanel; - if (panelBegin != 0) { - IntegerAttr::ValueType firstPanelValue; - if (!matchPattern(firstPanel, m_ConstantInt(&firstPanelValue))) - return stage.op->emitError( - "INT8 Add panel offset must be constant"); - chunkFirstPanel = b.create( - loc, firstPanelValue.getSExtValue() + panelBegin); - } - - TileBanks lhs = - allocateTile(chunkCount, destination.panelRows, zeroI8); - TileBanks rhs = - allocateTile(chunkCount, destination.panelRows, zeroI8); - if (lhs.banks.size() != 1 || rhs.banks.size() != 1) - return stage.op->emitError( - "INT8 Add branch panel chunk must fit one bank"); - if (failed(emitInto(producer.lookup(stage.input), y0, x0, height, - width, chunkFirstPanel, chunkCount, lhs, 0, - destinationStride))) - return failure(); - - TileBanks residual = - allocateTile(chunkCount, residualPanelRows, zeroI8); - if (residual.banks.empty() || - failed(emitInto(residualStage, residualY, residualX, - residualHeight, residualWidth, chunkFirstPanel, - chunkCount, residual, 0, residualWidth))) - return failure(); - residualCache.stage = residualStage; - residualCache.tile = residual; - IntegerAttr::ValueType cachedFirstPanel; - if (!matchPattern(chunkFirstPanel, m_ConstantInt(&cachedFirstPanel))) - return stage.op->emitError( - "INT8 Add residual panel offset must be constant"); - residualCache.firstPanel = cachedFirstPanel.getSExtValue(); - residualCache.y = residualY; - residualCache.x = residualX; - residualCache.width = residualWidth; - if (failed(emitInto(producer.lookup(stage.rhs), y0, x0, height, width, - chunkFirstPanel, chunkCount, rhs, 0, - destinationStride))) - return failure(); - residualCache.stage = -1; - residualCache.tile.banks.clear(); - releaseTile(residual); - - TileBanks sum = - allocateTile(chunkCount, destination.panelRows, zeroI8); - if (sum.banks.size() != 1) - return stage.op->emitError( - "INT8 Add output panel chunk must fit one bank"); - b.create( - loc, sum.banks.front().getType(), lhs.banks.front(), - rhs.banks.front(), sum.banks.front(), - createI64Const(b, loc, target.bankDepth), lhsRatio, rhsRatio, - b.getBoolAttr(stage.activation == 1)); - int64_t destinationBank = panelBegin / destination.panelsPerBank; - Value destinationState = destination.banks[destinationBank]; - // BankInt8Add packs all panels in the chunk into one bank. Copy - // each panel's spatial rows into its destination slot explicitly. - for (int64_t localPanel = 0; localPanel < chunkCount; ++localPanel) { - int64_t destinationSlot = - (panelBegin + localPanel) % destination.panelsPerBank; - destinationState = - b.create( - loc, destinationState.getType(), sum.banks.front(), - destinationState, createI64Const(b, loc, height * width), - b.getI64IntegerAttr(height), b.getI64IntegerAttr(width), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), - createI64Const(b, loc, localPanel * destination.panelRows), - createI64Const(b, loc, - destinationSlot * destination.panelRows + - destinationBase), - createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - destination.banks[destinationBank] = destinationState; - releaseTile(lhs); - releaseTile(rhs); - releaseTile(sum); - } - maskInvalidOutput(); - return success(); - } - - if (stage.multiply) { - if (height * width > target.bankDepth) - return stage.op->emitError("INT8 Mul tile exceeds one bank"); - Value ratio = b.create( - loc, b.getF32Type(), - b.getF32FloatAttr(stage.lhsScale * stage.rhsScale / - stage.outputScale)); - int64_t gateStage = producer.lookup(stage.rhs); - auto gateIt = gateCaches.find(gateStage); - if (gateIt == gateCaches.end()) { - InFlightDiagnostic diagnostic = - stage.op->emitError("INT8 Mul gate was not precomputed"); - diagnostic << " (multiplyStage=" << stageIndex - << ", gateStage=" << gateStage << ", cachedStages="; - for (const auto &entry : gateCaches) - diagnostic << entry.first << " "; - diagnostic << ")"; - return failure(); - } - TileBanks &gate = gateIt->second; - if (gate.panelCount < panelCount || gate.banks.size() != 1) - return stage.op->emitError("INT8 Mul gate cache layout mismatch"); - - for (size_t destinationBank = 0; - destinationBank < destination.banks.size(); ++destinationBank) { - int64_t panelBegin = destinationBank * destination.panelsPerBank; - int64_t panelEnd = std::min( - panelCount, panelBegin + destination.panelsPerBank); - auto panelLoop = b.create( - loc, b.create(loc, panelBegin), - b.create(loc, panelEnd), one, - ValueRange{destination.banks[destinationBank]}); - b.setInsertionPointToStart(panelLoop.getBody()); - Value localPanel = panelLoop.getInductionVar(); - Value destinationState = panelLoop.getRegionIterArgs().front(); - Value panel = b.create(loc, firstPanel, localPanel); - TileBanks input; - if (producer.contains(stage.input)) { - input = allocateTile(1, height * width, zeroI8); - if (failed(emitInto(producer.lookup(stage.input), y0, x0, height, - width, panel, 1, input, 0, width))) - return failure(); - } else { - FailureOr loaded = loadInt8Tile( - stage.input, stage.inputHeight, stage.inputWidth, - stage.inputChannels, y0, x0, panel, 1, height, width); - if (failed(loaded)) - return stage.op->emitError("failed to load INT8 Mul input"); - input = std::move(*loaded); - } - Value multiplied = allocBank(b, loc, 1, 1); - multiplied = - b.create( - loc, multiplied.getType(), gate.banks.front(), - input.banks.front(), multiplied, - createI64Const(b, loc, height * width), ratio, - b.create(loc, b.getI64Type(), panel)) - .getOutputBankOut(); - - Value destinationSlot = b.create( - loc, localPanel, - b.create(loc, panelBegin)); - Value outputBase = b.create( - loc, b.getI64Type(), - b.create( - loc, - b.create(loc, destinationSlot, - b.create( - loc, destination.panelRows)), - b.create(loc, destinationBase))); - Value destinationNext = - b.create( - loc, destinationState.getType(), multiplied, - destinationState, createI64Const(b, loc, height * width), - b.getI64IntegerAttr(width), b.getI64IntegerAttr(width), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), createI64Const(b, loc, 0), - outputBase, createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - releaseTile(input); - releaseBank(b, loc, multiplied); - b.create(loc, destinationNext); - b.setInsertionPointAfter(panelLoop); - destination.banks[destinationBank] = panelLoop.getResult(0); - } - maskInvalidOutput(); - return success(); - } - - if (stage.average) { - int64_t inputRows = stage.inputHeight * stage.inputWidth; - int64_t paddedInputRows = ((inputRows + kTile - 1) / kTile) * kTile; - bool fitsOneBank = - paddedInputRows <= target.bankDepth && paddedInputRows <= 0xfff; - if (!fitsOneBank && !externalInput && - !materialized.contains(producer.lookup(stage.input))) - return stage.op->emitError( - "large GlobalAvgPool requires a materialized input"); - int64_t sumK = fitsOneBank ? paddedInputRows : kTile; - Value oneI8 = b.create(loc, b.getI8Type(), - b.getI8IntegerAttr(1)); - Value ratio = b.create( - loc, b.getF32Type(), - b.getF32FloatAttr(stage.lhsScale / - (inputRows * stage.outputScale))); - Value onesPack = b.create( - loc, MemRefType::get({sumK / kTile, kTile}, b.getI8Type())); - Value biasPack = b.create( - loc, MemRefType::get({4, 4}, b.getI32Type())); - Value scalePack = b.create( - loc, MemRefType::get({4, 4}, b.getF32Type())); - b.create(loc, oneI8, onesPack); - b.create(loc, zeroI32, biasPack); - b.create(loc, ratio, scalePack); - - auto panelLoop = b.create( - loc, zero, b.create(loc, panelCount), one, - ValueRange{destination.banks.front()}); - b.setInsertionPointToStart(panelLoop.getBody()); - Value localPanel = panelLoop.getInductionVar(); - Value destinationState = panelLoop.getRegionIterArgs().front(); - Value panel = b.create(loc, firstPanel, localPanel); - - TileBanks fullSource; - if (fitsOneBank) { - fullSource = allocateTile(1, sumK, zeroI8); - if (externalInput) { - FailureOr loaded = - loadInt8Tile(stage.input, stage.inputHeight, stage.inputWidth, - stage.inputChannels, zero, zero, panel, 1, - stage.inputHeight, stage.inputWidth, sumK); - if (failed(loaded)) - return stage.op->emitError( - "failed to load INT8 GlobalAvgPool input"); - releaseTile(fullSource); - fullSource = std::move(*loaded); - } else if (failed(emitInto(producer.lookup(stage.input), zero, zero, - stage.inputHeight, stage.inputWidth, panel, - 1, fullSource, 0, stage.inputWidth))) { - return failure(); - } - } - - Value onesBank = allocBank(b, loc, 1, 1); - Value onesLoaded = mvinBank(b, loc, onesPack, onesBank, sumK / kTile); - Value biasBank = allocBank(b, loc, 1, 1); - Value biasLoaded = mvinBank(b, loc, biasPack, biasBank, 4); - Value biasState = b.create( - loc, biasLoaded.getType(), biasLoaded, createI64Const(b, loc, 0)); - releaseBank(b, loc, biasState); - Value scaleBank = allocBank(b, loc, 1, 1); - Value scaleLoaded = mvinBank(b, loc, scalePack, scaleBank, 4); - Value result = allocBank(b, loc, 1, 1); - - Value resultState; - if (fitsOneBank) { - resultState = - b.create( - loc, result.getType(), onesLoaded, fullSource.banks.front(), - result, createI64ConstU(b, loc, matrixRs2(1, kTile, sumK)), - createI1Const(b, loc, true), createI1Const(b, loc, true), - createI64Const(b, loc, 0)) - .getWrBankOut(); - releaseTile(fullSource); - } else { - Value four = b.create(loc, 4); - auto yLoop = b.create( - loc, zero, - b.create(loc, stage.inputHeight), four, - ValueRange{result}); - b.setInsertionPointToStart(yLoop.getBody()); - Value y = yLoop.getInductionVar(); - Value yState = yLoop.getRegionIterArgs().front(); - auto xLoop = b.create( - loc, zero, - b.create(loc, stage.inputWidth), four, - ValueRange{yState}); - b.setInsertionPointToStart(xLoop.getBody()); - Value x = xLoop.getInductionVar(); - Value xState = xLoop.getRegionIterArgs().front(); - TileBanks source = allocateTile(1, kTile, zeroI8); - if (externalInput) { - FailureOr loaded = - loadInt8Tile(stage.input, stage.inputHeight, stage.inputWidth, - stage.inputChannels, y, x, panel, 1, 4, 4); - if (failed(loaded)) - return stage.op->emitError( - "failed to load INT8 GlobalAvgPool input"); - releaseTile(source); - source = std::move(*loaded); - } else if (failed(emitInto(producer.lookup(stage.input), y, x, 4, 4, - panel, 1, source, 0, 4))) { - return failure(); - } - Value first = b.create( - loc, - b.create(loc, arith::CmpIPredicate::eq, y, zero), - b.create(loc, arith::CmpIPredicate::eq, x, zero)); - Value last = b.create( - loc, - b.create( - loc, arith::CmpIPredicate::sge, - b.create(loc, y, four), - b.create(loc, stage.inputHeight)), - b.create( - loc, arith::CmpIPredicate::sge, - b.create(loc, x, four), - b.create(loc, stage.inputWidth))); - Value resultNext = - b.create( - loc, xState.getType(), onesLoaded, source.banks.front(), - xState, createI64ConstU(b, loc, matrixRs2(1, kTile, kTile)), - first, last, createI64Const(b, loc, 0)) - .getWrBankOut(); - releaseTile(source); - b.create(loc, resultNext); - b.setInsertionPointAfter(xLoop); - b.create(loc, xLoop.getResult(0)); - b.setInsertionPointAfter(yLoop); - resultState = yLoop.getResult(0); - } - releaseBank(b, loc, onesLoaded); - - Value quantized = allocBank(b, loc, 1, 1); - quantized = b.create( - loc, quantized.getType(), resultState, scaleLoaded, - quantized, createI64Const(b, loc, 4), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getBoolAttr(false)) - .getOutBankOut(); - releaseBank(b, loc, resultState); - releaseBank(b, loc, scaleLoaded); - Value outputBase = - b.create(loc, b.getI64Type(), localPanel); - Value destinationNext = - b.create( - loc, destinationState.getType(), quantized, destinationState, - createI64Const(b, loc, 1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(0), - createI64Const(b, loc, 0), outputBase, - createI64Const(b, loc, 1), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(0)) - .getOutBankOut(); - releaseBank(b, loc, quantized); - b.create(loc, destinationNext); - b.setInsertionPointAfter(panelLoop); - destination.banks.front() = panelLoop.getResult(0); - b.create(loc, onesPack); - b.create(loc, biasPack); - b.create(loc, scalePack); - return success(); - } - - bool streamMaterializedInput = - !stage.pool && !stage.depthwise && !externalInput && - materialized.contains(producer.lookup(stage.input)); - int64_t maxSide = std::min({4, height, width}); - int64_t lastInputBanks = -1; - int64_t lastPanelsPerBank = -1; - int64_t lastInputSide = -1; - while (maxSide > 0) { - int64_t inputSide = (maxSide - 1) * stage.stride + stage.kernel; - int64_t inputPanelRows = inputSide * inputSide; - // A recursively produced tile is assembled from subtiles using - // six-bit row bases. DMA-loaded tiles and long MatMul K blocks may - // use the full SRAM depth, but recursive scratch tiles must fit 64. - bool recursiveInput = - !externalInput && - !materialized.contains(producer.lookup(stage.input)) && - !traceMaterialized.contains(producer.lookup(stage.input)); - if (recursiveInput && inputPanelRows > kMvin2dAddressableRows) { - --maxSide; - continue; - } - int64_t inputPanels = stage.depthwise - ? panelCount - : (stage.inputChannels + kTile - 1) / kTile; - int64_t panelsPerBank = tilePanelsPerBank(inputPanelRows); - lastInputSide = inputSide; - lastPanelsPerBank = panelsPerBank; - int64_t inputBanks = - panelsPerBank == 0 ? target.bankNum + 1 - : externalInput ? (inputPanels + panelsPerBank - 1) / panelsPerBank - : streamMaterializedInput ? 1 - : stage.depthwise - ? 1 - : (inputPanels + panelsPerBank - 1) / panelsPerBank; - lastInputBanks = inputBanks; - int64_t reservedBanks = externalInput ? 6 : 11; - if (stage.activation == 2) - reservedBanks += 2; - if (inputBanks + static_cast(destination.banks.size()) + - reservedBanks <= - target.bankNum) - break; - --maxSide; - } - if (maxSide == 0) { - InFlightDiagnostic diagnostic = stage.op->emitError( - "resident stage tile exceeds physical bank capacity"); - diagnostic << " (request=" << height << "x" << width - << ", panels=" << panelCount - << ", destinationBanks=" << destination.banks.size() - << ", bankDepth=" << target.bankDepth - << ", bankNum=" << target.bankNum - << ", inputSide=" << lastInputSide - << ", inputBanks=" << lastInputBanks - << ", panelsPerBank=" << lastPanelsPerBank << ")"; - return failure(); - } - int64_t side = std::min({maxSide, height, width}); - if (height != side || width != side) { - if (failed(emitInto(stageIndex, y0, x0, side, side, firstPanel, - panelCount, destination, destinationBase, - destinationStride))) - return failure(); - Value nextX = b.create( - loc, x0, b.create(loc, side)); - if (width > side && - failed(emitInto( - stageIndex, y0, nextX, side, width - side, firstPanel, - panelCount, destination, - destinationBase + - side * (fp32Output && stage.activation != 2 ? 4 : 1), - destinationStride))) - return failure(); - Value nextY = b.create( - loc, y0, b.create(loc, side)); - if (height > side && - failed(emitInto(stageIndex, nextY, x0, height - side, width, - firstPanel, panelCount, destination, - destinationBase + side * destinationStride, - destinationStride))) - return failure(); - return success(); - } - - int64_t inputSide = (side - 1) * stage.stride + stage.kernel; - Value sourceY = b.create( - loc, - b.create( - loc, y0, b.create(loc, stage.stride)), - b.create(loc, stage.padding)); - Value sourceX = b.create( - loc, - b.create( - loc, x0, b.create(loc, stage.stride)), - b.create(loc, stage.padding)); - int64_t inputPanelCount = (stage.pool || stage.depthwise) - ? panelCount - : (stage.inputChannels + kTile - 1) / kTile; - TileBanks source; - if (externalInput && !stage.pool) { - source.panelRows = inputSide * inputSide; - // Keep the external input layout simple: one channel panel per bank. - // This also makes the bank lifetime explicit when Cin is large. - source.panelsPerBank = tilePanelsPerBank(source.panelRows); - source.panelCount = inputPanelCount; - if (source.panelRows <= 0 || source.panelRows > target.bankDepth) - return stage.op->emitError( - "resident region input panel does not fit one bank"); - } else if (stage.pool) { - // Pooling preserves channels. Stream input panels one bank at a time - // instead of pinning the whole channel tile (e.g. 16 banks for 256 C). - int64_t sourcePanelsPerBank = tilePanelsPerBank(inputSide * inputSide); - if (sourcePanelsPerBank <= 0) - return stage.op->emitError( - "resident pool input panel does not fit bank"); - for (int64_t sourcePanelBegin = 0; sourcePanelBegin < inputPanelCount; - sourcePanelBegin += sourcePanelsPerBank) { - int64_t sourcePanelEnd = std::min( - inputPanelCount, sourcePanelBegin + sourcePanelsPerBank); - TileBanks sourceChunk = allocateTile( - sourcePanelEnd - sourcePanelBegin, inputSide * inputSide, minI8); - if (sourceChunk.banks.size() != 1) - return stage.op->emitError( - "resident pool input chunk must fit one bank"); - if (externalInput) { - Value pack = b.create( - loc, MemRefType::get({target.bankDepth, kTile}, b.getI8Type())); - b.create(loc, minI8, pack); - auto yLoop = b.create( - loc, zero, b.create(loc, inputSide), - one); - b.setInsertionPointToStart(yLoop.getBody()); - Value localY = yLoop.getInductionVar(); - auto xLoop = b.create( - loc, zero, b.create(loc, inputSide), - one); - b.setInsertionPointToStart(xLoop.getBody()); - Value globalY = b.create(loc, sourceY, localY); - Value globalX = - b.create(loc, sourceX, xLoop.getInductionVar()); - Value yValid = b.create( - loc, - b.create(loc, arith::CmpIPredicate::sge, globalY, - zero), - b.create( - loc, arith::CmpIPredicate::slt, globalY, - b.create(loc, stage.inputHeight))); - Value xValid = b.create( - loc, - b.create(loc, arith::CmpIPredicate::sge, globalX, - zero), - b.create( - loc, arith::CmpIPredicate::slt, globalX, - b.create(loc, stage.inputWidth))); - auto valid = b.create( - loc, b.create(loc, yValid, xValid), false); - b.setInsertionPointToStart(&valid.getThenRegion().front()); - auto laneLoop = b.create( - loc, - b.create(loc, sourcePanelBegin * kTile), - b.create(loc, sourcePanelEnd * kTile), - one); - b.setInsertionPointToStart(laneLoop.getBody()); - Value channel = laneLoop.getInductionVar(); - Value inputChannel = b.create( - loc, channel, - b.create( - loc, firstPanel, - b.create(loc, kTile))); - auto channelValid = b.create( - loc, - b.create( - loc, arith::CmpIPredicate::slt, inputChannel, - b.create(loc, stage.inputChannels)), - false); - b.setInsertionPointToStart(&channelValid.getThenRegion().front()); - Value value = b.create( - loc, stage.input, - ValueRange{zero, globalY, globalX, inputChannel}); - Value row = b.create( - loc, - b.create( - loc, - b.create( - loc, - b.create( - loc, channel, - b.create( - loc, sourcePanelBegin * kTile)), - b.create(loc, kTile)), - b.create(loc, - inputSide * inputSide)), - b.create( - loc, - b.create( - loc, localY, - b.create(loc, inputSide)), - xLoop.getInductionVar())); - b.create( - loc, value, pack, - ValueRange{row, - b.create( - loc, channel, - b.create(loc, kTile))}); - b.setInsertionPointAfter(channelValid); - b.setInsertionPointAfter(laneLoop); - b.setInsertionPointAfter(valid); - b.setInsertionPointAfter(xLoop); - b.setInsertionPointAfter(yLoop); - sourceChunk.banks.front() = mvinBank( - b, loc, pack, sourceChunk.banks.front(), target.bankDepth); - b.create(loc, pack); - } else if (failed(emitInto(producer.lookup(stage.input), sourceY, - sourceX, inputSide, inputSide, - b.create( - loc, firstPanel, - b.create( - loc, sourcePanelBegin)), - sourcePanelEnd - sourcePanelBegin, - sourceChunk, 0, inputSide))) - return failure(); - for (int64_t localPanel = sourcePanelBegin; - localPanel < sourcePanelEnd; ++localPanel) { - int64_t sourceSlot = localPanel - sourcePanelBegin; - int64_t destinationBank = localPanel / destination.panelsPerBank; - int64_t destinationSlot = localPanel % destination.panelsPerBank; - destination.banks[destinationBank] = - b.create( - loc, destination.banks[destinationBank].getType(), - sourceChunk.banks.front(), - destination.banks[destinationBank], - createI64Const(b, loc, side * side), - b.getI64IntegerAttr(inputSide), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(stage.kernel), - b.getI64IntegerAttr(stage.stride), b.getI64IntegerAttr(0), - createI64Const(b, loc, sourceSlot * sourceChunk.panelRows), - createI64Const(b, loc, - destinationSlot * destination.panelRows + - destinationBase), - createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - releaseTile(sourceChunk); - } - maskInvalidOutput(); - return success(); - } else if (!externalInput && !stage.depthwise && - !streamMaterializedInput) { - source = allocateTile(inputPanelCount, inputSide * inputSide, zeroI8); - if (failed(emitInto(producer.lookup(stage.input), sourceY, sourceX, - inputSide, inputSide, zero, inputPanelCount, source, - 0, inputSide))) - return failure(); - } - - if (stage.depthwise) { - int64_t paddedK = - (stage.kernel * stage.kernel + kTile - 1) / kTile * kTile; - Value lutLoaded; - if (stage.activation == 2) { - Value lutBank = allocBank(b, loc, 1, 1); - lutLoaded = - mvinBank(b, loc, packedLuts.lookup(stage.op), lutBank, kTile); - } - SmallVector destinationStates(destination.banks.begin(), - destination.banks.end()); - for (int64_t localPanel = 0; localPanel < panelCount; ++localPanel) { - int64_t destinationBank = localPanel / destination.panelsPerBank; - int64_t destinationSlot = localPanel % destination.panelsPerBank; - Value outputPanel = b.create( - loc, firstPanel, - b.create(loc, localPanel)); - TileBanks panelSource; - if (externalInput) { - auto loaded = - loadInt8Tile(stage.input, stage.inputHeight, stage.inputWidth, - stage.inputChannels, sourceY, sourceX, outputPanel, - 1, inputSide, inputSide); - if (failed(loaded)) - return stage.op->emitError("failed to load INT8 Depthwise input"); - panelSource = std::move(*loaded); - } else { - panelSource = allocateTile(1, inputSide * inputSide, zeroI8); - if (panelSource.banks.size() != 1) - return stage.op->emitError( - "Depthwise Conv input panel must fit one bank"); - if (failed(emitInto(producer.lookup(stage.input), sourceY, sourceX, - inputSide, inputSide, outputPanel, 1, - panelSource, 0, inputSide))) - return failure(); - } - SmallVector parameterOffsets = { - outputPanel, b.getIndexAttr(0), b.getIndexAttr(0)}; - SmallVector parameterSizes = { - b.getIndexAttr(1), b.getIndexAttr(4), b.getIndexAttr(4)}; - SmallVector parameterStrides(3, b.getIndexAttr(1)); - Value biasSlice = b.create( - loc, packedBiases.lookup(stage.op), parameterOffsets, - parameterSizes, parameterStrides); - Value biasPack = b.create( - loc, biasSlice, SmallVector{{0, 1}, {2}}); - Value scaleSlice = b.create( - loc, packedScales.lookup(stage.op), parameterOffsets, - parameterSizes, parameterStrides); - Value scalePack = b.create( - loc, scaleSlice, SmallVector{{0, 1}, {2}}); - - Value biasBank = allocBank(b, loc, 1, 1); - Value biasLoaded = mvinBank(b, loc, biasPack, biasBank, 4); - Value biasState = b.create( - loc, biasLoaded.getType(), biasLoaded, createI64Const(b, loc, 0)); - releaseBank(b, loc, biasState); - Value scaleBank = allocBank(b, loc, 1, 1); - Value scaleLoaded = mvinBank(b, loc, scalePack, scaleBank, 4); - Value patchState = allocBank(b, loc, 1, 1); - Value weightState = allocBank(b, loc, 1, 1); - Value resultState = allocBank(b, loc, 1, 1); - auto laneLoop = b.create( - loc, zero, b.create(loc, kTile), one, - ValueRange{patchState, weightState, resultState}); - b.setInsertionPointToStart(laneLoop.getBody()); - Value lane = laneLoop.getInductionVar(); - ValueRange laneStates = laneLoop.getRegionIterArgs(); - Value laneI64 = - b.create(loc, b.getI64Type(), lane); - Value patchNext = - b.create( - loc, laneStates[0].getType(), panelSource.banks.front(), - laneStates[0], createI64Const(b, loc, inputSide), - createI64Const(b, loc, stage.kernel), - createI64Const(b, loc, stage.stride), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - laneI64, b.getI64IntegerAttr(0), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(side * side)) - .getOutBankOut(); - - Value weightSlice = b.create( - loc, packedDepthwiseWeights.lookup(stage.op), - SmallVector{outputPanel, lane, b.getIndexAttr(0), - b.getIndexAttr(0)}, - SmallVector{b.getIndexAttr(1), b.getIndexAttr(1), - b.getIndexAttr(paddedK), - b.getIndexAttr(kTile)}, - SmallVector(4, b.getIndexAttr(1))); - Value weightPack = b.create( - loc, weightSlice, - SmallVector{{0, 1, 2}, {3}}); - Value weightNext = - mvinBank(b, loc, weightPack, laneStates[1], paddedK); - Value first = b.create(loc, arith::CmpIPredicate::eq, - lane, zero); - Value last = b.create( - loc, arith::CmpIPredicate::eq, lane, - b.create(loc, kTile - 1)); - Value smatmulInput = repackSMatMulInput(patchNext, kTile, paddedK); - Value resultNext = - b.create( - loc, laneStates[2].getType(), smatmulInput, weightNext, - laneStates[2], - createI64ConstU(b, loc, matrixRs2(kTile, kTile, paddedK)), - first, last, createI64Const(b, loc, 0)) - .getWrBankOut(); - if (smatmulInput != patchNext) - releaseBank(b, loc, smatmulInput); - b.create(loc, - ValueRange{patchNext, weightNext, resultNext}); - b.setInsertionPointAfter(laneLoop); - patchState = laneLoop.getResult(0); - weightState = laneLoop.getResult(1); - resultState = laneLoop.getResult(2); - releaseBank(b, loc, patchState); - releaseBank(b, loc, weightState); - - Value outputBank = allocBank(b, loc, 1, 1); - Value outputState; - if (fp32Output && stage.activation == 2) { - Value quantizedState = - b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, createI64Const(b, loc, side * side * 4), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - b.getI64IntegerAttr(side), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(side), b.getBoolAttr(false)) - .getOutBankOut(); - releaseBank(b, loc, resultState); - releaseBank(b, loc, scaleLoaded); - Value lutOutput = allocBank(b, loc, 1, 1); - Value transformed = b.create( - loc, lutOutput.getType(), quantizedState, lutLoaded, lutOutput, - createI64Const(b, loc, side * side)); - releaseBank(b, loc, quantizedState); - Value destinationState = destinationStates[destinationBank]; - int64_t pixels = side * side; - for (int64_t pixel = 0; pixel < pixels; ++pixel) { - Value outputBase = - createI64Const(b, loc, - destinationSlot * destination.panelRows + - destinationBase + pixel); - destinationState = - b.create( - loc, destinationState.getType(), transformed, - destinationState, createI64Const(b, loc, 1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), createI64Const(b, loc, pixel), - outputBase, createI64Const(b, loc, 1), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - destinationStates[destinationBank] = destinationState; - releaseBank(b, loc, transformed); - } else if (fp32Output) { - outputState = - b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, createI64Const(b, loc, side * side * 4), - b.getBoolAttr(stage.activation == 1)) - .getOutBankOut(); - releaseBank(b, loc, resultState); - releaseBank(b, loc, scaleLoaded); - Value destinationState = destinationStates[destinationBank]; - Value outputBase = createI64Const( - b, loc, - destinationSlot * destination.panelRows + destinationBase); - destinationState = - copyFp32Tile(outputState, destinationState, outputBase, side, - destinationStride); - destinationStates[destinationBank] = destinationState; - releaseBank(b, loc, outputState); - } else { - Value quantizedState = - b.create( - loc, outputBank.getType(), resultState, scaleLoaded, - outputBank, createI64Const(b, loc, side * side * 4), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - b.getI64IntegerAttr(side), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(side), - b.getBoolAttr(stage.activation == 1)) - .getOutBankOut(); - releaseBank(b, loc, resultState); - releaseBank(b, loc, scaleLoaded); - Value transformed; - outputState = quantizedState; - if (stage.activation == 2) { - Value lutOutput = allocBank(b, loc, 1, 1); - transformed = b.create( - loc, lutOutput.getType(), quantizedState, lutLoaded, - lutOutput, createI64Const(b, loc, side * side)); - outputState = transformed; - } - Value outputBase = createI64Const( - b, loc, - destinationSlot * destination.panelRows + destinationBase); - destinationStates[destinationBank] = - b.create( - loc, destinationStates[destinationBank].getType(), - outputState, destinationStates[destinationBank], - createI64Const(b, loc, side * side), - b.getI64IntegerAttr(side), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), createI64Const(b, loc, 0), - outputBase, createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - releaseBank(b, loc, quantizedState); - if (transformed) - releaseBank(b, loc, transformed); - } - releaseTile(panelSource); - } - if (lutLoaded) - releaseBank(b, loc, lutLoaded); - maskInvalidOutput(); - return success(); - } - - int64_t kernelElements = stage.kernel * stage.kernel; - int64_t paddedK = (kernelElements + kTile - 1) / kTile * kTile; - Value lutLoaded; - if (stage.activation == 2) { - int64_t lutRows = - stage.lutEntries == 4096 ? stage.lutEntries / (4 * kTile) : kTile; - Value lutBank = allocBank(b, loc, 1, stage.lutEntries == 4096 ? 4 : 1); - lutLoaded = - mvinBank(b, loc, packedLuts.lookup(stage.op), lutBank, lutRows); - } - for (size_t destinationBank = 0; - destinationBank < destination.banks.size(); ++destinationBank) { - int64_t panelBegin = destinationBank * destination.panelsPerBank; - int64_t panelEnd = std::min( - panelCount, panelBegin + destination.panelsPerBank); - auto outputPanelLoop = b.create( - loc, b.create(loc, panelBegin), - b.create(loc, panelEnd), one, - ValueRange{destination.banks[destinationBank]}); - b.setInsertionPointToStart(outputPanelLoop.getBody()); - Value localPanel = outputPanelLoop.getInductionVar(); - Value destinationState = outputPanelLoop.getRegionIterArgs().front(); - Value outputPanel = - b.create(loc, firstPanel, localPanel); - SmallVector parameterOffsets = { - outputPanel, b.getIndexAttr(0), b.getIndexAttr(0)}; - SmallVector parameterSizes = { - b.getIndexAttr(1), b.getIndexAttr(4), b.getIndexAttr(4)}; - SmallVector parameterStrides(3, b.getIndexAttr(1)); - Value biasSlice = b.create( - loc, packedBiases.lookup(stage.op), parameterOffsets, - parameterSizes, parameterStrides); - Value biasPack = b.create( - loc, biasSlice, SmallVector{{0, 1}, {2}}); - Value scaleSlice = b.create( - loc, packedScales.lookup(stage.op), parameterOffsets, - parameterSizes, parameterStrides); - Value scalePack = b.create( - loc, scaleSlice, SmallVector{{0, 1}, {2}}); - - SmallVector states; - auto ensureStates = [&]() { - if (states.empty()) { - states = {allocBank(b, loc, 1, 1), allocBank(b, loc, 1, 1), - allocBank(b, loc, 1, 1)}; - if (stage.weightChannelStride) - states[0] = mvinBank(b, loc, zeroPack, states[0], kTile); - } - }; - auto accumulateSource = [&](Value sourceBank, Value sourcePanelBegin, - Value sourcePanelEnd, - int64_t sourcePanelRows) { - Value tileSize = b.create(loc, kTile); - Value channelBegin = - b.create(loc, sourcePanelBegin, tileSize); - Value channelEnd = b.create( - loc, b.create(loc, stage.inputChannels), - b.create(loc, sourcePanelEnd, tileSize)); - int64_t channelStep = stage.weightChannelStride ? kTile : 1; - auto channelLoop = - b.create(loc, channelBegin, channelEnd, - stage.weightChannelStride ? tileSize : one, - ValueRange{states[0], states[1], states[2]}); - b.setInsertionPointToStart(channelLoop.getBody()); - Value inputChannel = channelLoop.getInductionVar(); - ValueRange iterStates = channelLoop.getRegionIterArgs(); - Value sourceSlot = b.create( - loc, - b.create( - loc, inputChannel, - b.create(loc, kTile)), - sourcePanelBegin); - Value inputBase = b.create( - loc, b.getI64Type(), - b.create( - loc, sourceSlot, - b.create(loc, sourcePanelRows))); - Value patchNext; - Value weightNext; - if (stage.weightChannelStride) { - patchNext = b.create( - loc, iterStates[0].getType(), sourceBank, iterStates[0], - createI64Const(b, loc, side * side), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(side), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(0), inputBase, - createI64Const(b, loc, 0), createI64Const(b, loc, side), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)); - weightNext = iterStates[1]; - for (int64_t half = 0; half < 2; ++half) { - Value channel = b.create( - loc, inputChannel, - b.create(loc, half * 8)); - Value weightSlice = b.create( - loc, stage.weight, - SmallVector{outputPanel, channel, - b.getIndexAttr(0), - b.getIndexAttr(0)}, - SmallVector{ - b.getIndexAttr(1), b.getIndexAttr(8), b.getIndexAttr(1), - b.getIndexAttr(kTile)}, - SmallVector(4, b.getIndexAttr(1))); - weightNext = b.create( - loc, weightNext.getType(), weightSlice, weightNext, - createI64Const(b, loc, 1), stage.weightChannelStride, - createI64Const(b, loc, 8), createI64Const(b, loc, half * 8), - createI64Const(b, loc, 8), createI64Const(b, loc, kTile)); - } - } else { - Value inputLane = b.create( - loc, b.getI64Type(), - b.create( - loc, inputChannel, - b.create(loc, kTile))); - patchNext = - b.create( - loc, iterStates[0].getType(), sourceBank, iterStates[0], - createI64Const(b, loc, inputSide), - createI64Const(b, loc, stage.kernel), - createI64Const(b, loc, stage.stride), - createI64Const(b, loc, 0), inputBase, inputLane, - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(side * side)) - .getOutBankOut(); - SmallVector weightOffsets = { - outputPanel, inputChannel, b.getIndexAttr(0), - b.getIndexAttr(0)}; - SmallVector weightSizes = { - b.getIndexAttr(1), b.getIndexAttr(1), b.getIndexAttr(paddedK), - b.getIndexAttr(kTile)}; - SmallVector weightStrides(4, b.getIndexAttr(1)); - Value weightSlice = b.create( - loc, stage.weight, weightOffsets, weightSizes, weightStrides); - Value weightPack = b.create( - loc, weightSlice, - SmallVector{{0, 1, 2}, {3}}); - weightNext = mvinBank(b, loc, weightPack, iterStates[1], paddedK); - } - Value first = b.create(loc, arith::CmpIPredicate::eq, - inputChannel, zero); - Value last = b.create( - loc, arith::CmpIPredicate::eq, inputChannel, - b.create(loc, stage.inputChannels - - channelStep)); - int64_t smatmulM = side == 1 && paddedK == kTile ? 1 : kTile; - Value smatmulInput = repackSMatMulInput(patchNext, smatmulM, paddedK); - Value resultNext = - b.create( - loc, iterStates[2].getType(), smatmulInput, weightNext, - iterStates[2], - createI64ConstU(b, loc, matrixRs2(smatmulM, kTile, paddedK)), - first, last, createI64Const(b, loc, 0)) - .getWrBankOut(); - if (smatmulInput != patchNext) - releaseBank(b, loc, smatmulInput); - b.create(loc, - ValueRange{patchNext, weightNext, resultNext}); - b.setInsertionPointAfter(channelLoop); - states.assign(channelLoop.getResults().begin(), - channelLoop.getResults().end()); - }; - Value biasBank = allocBank(b, loc, 1, 1); - Value biasLoaded = mvinBank(b, loc, biasPack, biasBank, 4); - Value biasState = b.create( - loc, biasLoaded.getType(), biasLoaded, createI64Const(b, loc, 0)); - releaseBank(b, loc, biasState); - - if (externalInput) { - ensureStates(); - auto inputPanelLoop = b.create( - loc, zero, b.create(loc, inputPanelCount), - one, ValueRange{states[0], states[1], states[2]}); - b.setInsertionPointToStart(inputPanelLoop.getBody()); - Value panel = inputPanelLoop.getInductionVar(); - states.assign(inputPanelLoop.getRegionIterArgs().begin(), - inputPanelLoop.getRegionIterArgs().end()); - FailureOr loaded = - loadInt8Tile(stage.input, stage.inputHeight, stage.inputWidth, - stage.inputChannels, sourceY, sourceX, panel, 1, - inputSide, inputSide); - if (failed(loaded)) - return stage.op->emitError("failed to load external Conv input"); - accumulateSource(loaded->banks.front(), panel, - b.create(loc, panel, one), - loaded->panelRows); - releaseTile(*loaded); - b.create(loc, - ValueRange{states[0], states[1], states[2]}); - b.setInsertionPointAfter(inputPanelLoop); - states.assign(inputPanelLoop.getResults().begin(), - inputPanelLoop.getResults().end()); - } else if (streamMaterializedInput) { - ensureStates(); - int64_t streamedPanelsPerBank = - tilePanelsPerBank(inputSide * inputSide); - if (streamedPanelsPerBank <= 0) - return stage.op->emitError( - "streamed Conv input panel does not fit one bank"); - for (int64_t inputPanel = 0; inputPanel < inputPanelCount; - inputPanel += streamedPanelsPerBank) { - int64_t panelCountInBank = std::min( - streamedPanelsPerBank, inputPanelCount - inputPanel); - TileBanks streamedSource = - allocateTile(panelCountInBank, inputSide * inputSide, zeroI8); - if (streamedSource.banks.size() != 1) - return stage.op->emitError( - "streamed Conv input panel group must fit one bank"); - if (failed( - emitInto(producer.lookup(stage.input), sourceY, sourceX, - inputSide, inputSide, - b.create(loc, inputPanel), - panelCountInBank, streamedSource, 0, inputSide))) - return failure(); - accumulateSource(streamedSource.banks.front(), - b.create(loc, inputPanel), - b.create( - loc, inputPanel + panelCountInBank), - streamedSource.panelRows); - releaseTile(streamedSource); - } - } else { - ensureStates(); - for (size_t sourceBank = 0; sourceBank < source.banks.size(); - ++sourceBank) { - int64_t sourcePanelBegin = sourceBank * source.panelsPerBank; - int64_t sourcePanelEnd = std::min( - inputPanelCount, sourcePanelBegin + source.panelsPerBank); - accumulateSource( - source.banks[sourceBank], - b.create(loc, sourcePanelBegin), - b.create(loc, sourcePanelEnd), - source.panelRows); - } - } - releaseBank(b, loc, states[0]); - releaseBank(b, loc, states[1]); - Value scaleBank = allocBank(b, loc, 1, 1); - Value scaleLoaded = mvinBank(b, loc, scalePack, scaleBank, 4); - Value destinationSlot = b.create( - loc, localPanel, b.create(loc, panelBegin)); - Value destinationOffset = b.create( - loc, destinationSlot, - b.create(loc, destination.panelRows)); - Value outputBase = b.create( - loc, b.getI64Type(), - b.create( - loc, destinationOffset, - b.create(loc, destinationBase))); - Value outputBank = allocBank(b, loc, 1, 1); - Value outputState; - Value destinationNext = destinationState; - if (fp32Output && stage.activation == 2) { - Value quantizedState = - b.create( - loc, outputBank.getType(), states[2], scaleLoaded, - outputBank, createI64Const(b, loc, side * side * 4), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - b.getI64IntegerAttr(side), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(side), b.getBoolAttr(false)) - .getOutBankOut(); - releaseBank(b, loc, states[2]); - releaseBank(b, loc, scaleLoaded); - Value lutOutput = allocBank(b, loc, 1, 1); - Value transformed = b.create( - loc, lutOutput.getType(), quantizedState, lutLoaded, lutOutput, - createI64Const(b, loc, side * side)); - releaseBank(b, loc, quantizedState); - int64_t pixels = side * side; - for (int64_t pixel = 0; pixel < pixels; ++pixel) { - Value sourceBase = createI64Const(b, loc, pixel); - Value outputBase = b.create( - loc, - b.create( - loc, b.getI64Type(), - b.create( - loc, - b.create( - loc, destinationSlot, - b.create( - loc, destination.panelRows)), - b.create(loc, - destinationBase))), - createI64Const(b, loc, pixel)); - destinationNext = - b.create( - loc, destinationNext.getType(), transformed, - destinationNext, createI64Const(b, loc, 1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), sourceBase, outputBase, - createI64Const(b, loc, 1), b.getI64IntegerAttr(0), - b.getI64IntegerAttr(0)) - .getOutBankOut(); - } - releaseBank(b, loc, transformed); - } else if (fp32Output) { - outputState = - b.create( - loc, outputBank.getType(), states[2], scaleLoaded, - outputBank, createI64Const(b, loc, side * side * 4), - b.getBoolAttr(stage.activation == 1)) - .getOutBankOut(); - releaseBank(b, loc, states[2]); - releaseBank(b, loc, scaleLoaded); - Value outputBase = b.create( - loc, b.getI64Type(), - b.create( - loc, - b.create(loc, destinationSlot, - b.create( - loc, destination.panelRows)), - b.create(loc, destinationBase))); - destinationNext = copyFp32Tile(outputState, destinationNext, - outputBase, side, destinationStride); - releaseBank(b, loc, outputState); - } else { - Value quantizedState = - b.create( - loc, outputBank.getType(), states[2], scaleLoaded, - outputBank, createI64Const(b, loc, side * side * 4), - createI64Const(b, loc, 0), createI64Const(b, loc, 0), - b.getI64IntegerAttr(side), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(side), - b.getBoolAttr(stage.activation == 1)) - .getOutBankOut(); - releaseBank(b, loc, states[2]); - releaseBank(b, loc, scaleLoaded); - Value transformed; - outputState = quantizedState; - if (stage.activation == 2) { - Value lutOutput = allocBank(b, loc, 1, 1); - transformed = b.create( - loc, lutOutput.getType(), quantizedState, lutLoaded, lutOutput, - createI64Const(b, loc, side * side)); - outputState = transformed; - } - destinationNext = - b.create( - loc, destinationState.getType(), outputState, - destinationState, createI64Const(b, loc, side * side), - b.getI64IntegerAttr(side), b.getI64IntegerAttr(side), - b.getI64IntegerAttr(1), b.getI64IntegerAttr(1), - b.getI64IntegerAttr(0), createI64Const(b, loc, 0), - outputBase, createI64Const(b, loc, destinationStride), - b.getI64IntegerAttr(0), b.getI64IntegerAttr(0)) - .getOutBankOut(); - releaseBank(b, loc, quantizedState); - if (transformed) - releaseBank(b, loc, transformed); - } - b.create(loc, destinationNext); - b.setInsertionPointAfter(outputPanelLoop); - destination.banks[destinationBank] = outputPanelLoop.getResult(0); - } - if (lutLoaded) - releaseBank(b, loc, lutLoaded); - releaseTile(source); - maskInvalidOutput(); - return success(); - }; - - auto materializeStage = [&](int64_t stageIndex) -> LogicalResult { - Stage &stage = stages[stageIndex]; - const bool fp32Output = stage.finalOutput && !stage.pool; - // LUT results stay INT8 in banks; host stores apply the FP32 scale. - const int64_t outputStorageFactor = - fp32Output && stage.activation != 2 ? 4 : 1; - int64_t side = - std::min((stage.add || stage.average) ? 1 : 4, - std::min(stage.outputHeight, stage.outputWidth)); - while (side * side * outputStorageFactor > - std::min(target.bankDepth, kMvin2dAddressableRows)) - --side; - int64_t panelCount = (stage.outputChannels + kTile - 1) / kTile; - auto yLoop = b.create( - loc, zero, b.create(loc, stage.outputHeight), - b.create(loc, side)); - b.setInsertionPointToStart(yLoop.getBody()); - Value y = yLoop.getInductionVar(); - auto xLoop = b.create( - loc, zero, b.create(loc, stage.outputWidth), - b.create(loc, side)); - b.setInsertionPointToStart(xLoop.getBody()); - Value x = xLoop.getInductionVar(); - auto emitOutput = [&](Value firstPanel, - int64_t requestedPanels) -> LogicalResult { - const bool fp32LutOutput = fp32Output && stage.activation == 2; - if (fp32LutOutput && !packedLutOutputScales.lookup(stage.op)) - return stage.op->emitError( - "missing LUT output scales for FP32 final output"); - TileBanks output = allocateTile( - requestedPanels, side * side * outputStorageFactor, zeroI8); - if (output.banks.size() != 1) { - return failure(); - } - if (failed(emitInto(stageIndex, y, x, side, side, firstPanel, - requestedPanels, output, 0, - side * outputStorageFactor))) { - return failure(); - } - Type outputElementType = - fp32LutOutput - ? Type(b.getI8Type()) - : (fp32Output ? Type(b.getF32Type()) : Type(b.getI8Type())); - int64_t usedBankRows = - requestedPanels * side * side * outputStorageFactor; - int64_t packedRows = - fp32LutOutput ? usedBankRows : usedBankRows / outputStorageFactor; - Value pack = b.create( - loc, MemRefType::get({packedRows, kTile}, outputElementType)); - mvoutBank(b, loc, pack, output.banks.front(), usedBankRows); - - auto panelLoop = b.create( - loc, zero, b.create(loc, requestedPanels), - one); - b.setInsertionPointToStart(panelLoop.getBody()); - Value panelInRequest = panelLoop.getInductionVar(); - Value panel = b.create(loc, firstPanel, panelInRequest); - auto localYLoop = b.create( - loc, zero, b.create(loc, side), one); - b.setInsertionPointToStart(localYLoop.getBody()); - Value localY = localYLoop.getInductionVar(); - auto localXLoop = b.create( - loc, zero, b.create(loc, side), one); - b.setInsertionPointToStart(localXLoop.getBody()); - Value localX = localXLoop.getInductionVar(); - Value globalY = b.create(loc, y, localY); - Value globalX = b.create(loc, x, localX); - Value yValid = b.create( - loc, arith::CmpIPredicate::slt, globalY, - b.create(loc, stage.outputHeight)); - Value xValid = b.create( - loc, arith::CmpIPredicate::slt, globalX, - b.create(loc, stage.outputWidth)); - auto valid = b.create( - loc, b.create(loc, yValid, xValid), false); - b.setInsertionPointToStart(&valid.getThenRegion().front()); - auto laneLoop = b.create( - loc, zero, b.create(loc, kTile), one); - b.setInsertionPointToStart(laneLoop.getBody()); - Value lane = laneLoop.getInductionVar(); - Value channel = b.create( - loc, - b.create( - loc, panel, b.create(loc, kTile)), - lane); - auto channelValid = b.create( - loc, - b.create( - loc, arith::CmpIPredicate::slt, channel, - b.create(loc, stage.outputChannels)), - false); - b.setInsertionPointToStart(&channelValid.getThenRegion().front()); - Value row = b.create( - loc, - b.create( - loc, panelInRequest, - b.create(loc, side * side)), - b.create( - loc, - b.create( - loc, localY, b.create(loc, side)), - localX)); - Value packedLane = lane; - if (fp32Output && !fp32LutOutput) { - Value group = b.create( - loc, lane, b.create(loc, 4)); - Value laneInGroup = b.create( - loc, lane, b.create(loc, 4)); - packedLane = b.create( - loc, - b.create(loc, group, - b.create(loc, 4)), - laneInGroup); - } - Value value = - b.create(loc, pack, ValueRange{row, packedLane}); - if (fp32LutOutput) { - Value valueF32 = - b.create(loc, b.getF32Type(), value); - Value outputScale = b.create( - loc, packedLutOutputScales.lookup(stage.op), - ValueRange{ - panel, - b.create( - loc, lane, b.create(loc, 4)), - b.create( - loc, lane, b.create(loc, 4))}); - value = b.create(loc, valueF32, outputScale); - } - if (stage.finalOutput && !stage.outputNhwc) - b.create( - loc, value, stage.output, - ValueRange{zero, channel, globalY, globalX}); - else - b.create( - loc, value, stage.output, - ValueRange{zero, globalY, globalX, channel}); - b.setInsertionPointAfter(channelValid); - b.setInsertionPointAfter(laneLoop); - b.setInsertionPointAfter(valid); - b.setInsertionPointAfter(localYLoop); - b.setInsertionPointAfter(panelLoop); - b.create(loc, pack); - releaseTile(output); - return success(); - }; - - int64_t panelsPerBank = - tilePanelsPerBank(side * side * outputStorageFactor); - if (panelsPerBank <= 0) - return stage.op->emitError("resident output tile does not fit bank"); - if (stage.depthwise) { - auto depthwisePanelLoop = b.create( - loc, zero, b.create(loc, panelCount), one); - b.setInsertionPointToStart(depthwisePanelLoop.getBody()); - if (failed(emitOutput(depthwisePanelLoop.getInductionVar(), 1))) - return failure(); - b.setInsertionPointAfter(depthwisePanelLoop); - } else { - for (int64_t panelBegin = 0; panelBegin < panelCount; - panelBegin += panelsPerBank) { - int64_t chunkCount = - std::min(panelCount - panelBegin, panelsPerBank); - if (failed( - emitOutput(b.create(loc, panelBegin), - chunkCount))) - return failure(); - } - } - b.setInsertionPointAfter(xLoop); - b.setInsertionPointAfter(yLoop); - auto traceNode = stage.op->getAttrOfType("trace_node"); - if (traceNode && cast(traceNode.get("tensor")).getValue()) { - if (cast(stage.output.getType()) - .getElementType() - .isInteger(8)) { - if (failed(mlir::buddy::emitStageTensorTrace(b, loc, stage.output, - traceNode))) - return failure(); - } else { - auto trace = b.create<::buddy::trace::EndOp>( - loc, stage.output.getType(), stage.output, - cast(traceNode.get("id")), b.getBoolAttr(false), - b.getBoolAttr(true), cast(traceNode.get("tag"))); - trace->setAttr("id_path", traceNode.get("id_path")); - } - } - materialized.insert(stageIndex); - return success(); - }; - - DenseMap consumers; - for (const Stage &stage : stages) { - if (stage.channelConcat) { - for (Value input : stage.inputs) - ++consumers[input]; - } else { - ++consumers[stage.input]; - if (stage.add || stage.multiply) - ++consumers[stage.rhs]; - } - } - - for (auto [stageIndex, stage] : llvm::enumerate(stages)) { - if (stage.multiply) { - if (producer.contains(stage.input)) { - int64_t inputStage = producer.lookup(stage.input); - if (!materialized.contains(inputStage) && - failed(materializeStage(inputStage))) - return failure(); - } - if (!producer.contains(stage.rhs)) - return stage.op->emitError( - "INT8 Mul gate must have a region producer"); - int64_t gateStage = producer.lookup(stage.rhs); - if (!gateCaches.contains(gateStage)) { - int64_t gatePanels = - (stages[gateStage].outputChannels + kTile - 1) / kTile; - TileBanks gate = allocateTile(gatePanels, 1, zeroI8); - if (gate.banks.size() != 1 || - failed(emitInto(gateStage, zero, zero, 1, 1, zero, gatePanels, - gate, 0, 1))) - return stage.op->emitError( - "INT8 Mul gate must fit one complete bank"); - gateCaches.try_emplace(gateStage, std::move(gate)); - } - } - auto traceNode = stage.op->getAttrOfType("trace_node"); - if (traceNode && cast(traceNode.get("tensor")).getValue() && - !materialized.contains(stageIndex)) { - if (failed(materializeStage(stageIndex))) - return failure(); - traceMaterialized.insert(stageIndex); - } - // Spatial kernels reuse overlapping producer windows; resize repeats - // pixels. Shared DAG results also need one evaluation before consumers - // request tiles. - if ((stage.resizeNearest || stage.kernel > 1 || - consumers.lookup(stage.output) > 1) && - producer.contains(stage.input)) { - int64_t inputStage = producer.lookup(stage.input); - if (!materialized.contains(inputStage) && - failed(materializeStage(inputStage))) - return failure(); - } - if ((stage.resizeNearest || consumers.lookup(stage.output) > 1) && - !materialized.contains(stageIndex) && - failed(materializeStage(stageIndex))) - return failure(); - if (stage.add && producer.contains(stage.input)) { - int64_t inputStage = producer.lookup(stage.input); - if (!materialized.contains(inputStage) && - failed(materializeStage(inputStage))) - return failure(); - for (auto &entry : gateCaches) - releaseTile(entry.second); - gateCaches.clear(); - } - if ((stage.pool || stage.add) && !materialized.contains(stageIndex)) { - if (failed(materializeStage(stageIndex))) - return failure(); - for (auto &entry : gateCaches) - releaseTile(entry.second); - gateCaches.clear(); - } - } - - int64_t finalStage = stages.size() - 1; - if (!materialized.contains(finalStage) && - failed(materializeStage(finalStage))) - return failure(); - for (auto &entry : gateCaches) - releaseTile(entry.second); - releaseBank(b, loc, zeroBank); - for (Value pack : hostPacks) - b.create(loc, pack); - b.eraseOp(kernel); - return success(); - } -}; - -} // namespace - -namespace mlir::buddy { -void populatePebbleResidentConvRegionToBankSSAPatterns( - RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} -} // namespace mlir::buddy diff --git a/waste/examples/designs/pebble/stack/midend/src/LargeConvolution.cpp b/waste/examples/designs/pebble/stack/midend/src/LargeConvolution.cpp deleted file mode 100644 index b94b810ee..000000000 --- a/waste/examples/designs/pebble/stack/midend/src/LargeConvolution.cpp +++ /dev/null @@ -1,158 +0,0 @@ -#include "Buckyball/BuckyballOps.h" -#include "Trace/TraceOps.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Math/IR/Math.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult lowerPebbleLargeConvolutions(func::FuncOp function) { - SmallVector convolutions; - function.walk([&](linalg::GenericOp op) { - if (op->hasAttr("buckyball.quantized_conv2d") && - op->getAttrOfType("kernel").getInt() > 7) - convolutions.push_back(op); - }); - for (auto op : convolutions) { - OpBuilder b(op); - Location loc = op.getLoc(); - Value input = op.getInputs()[0], weight = op.getInputs()[1]; - Value output = op.getOutputs()[0]; - auto in = cast(input.getType()).getShape(); - auto wt = cast(weight.getType()).getShape(); - auto outType = cast(output.getType()); - auto out = outType.getShape(); - int64_t kernel = op->getAttrOfType("kernel").getInt(); - int64_t stride = op->getAttrOfType("stride").getInt(); - int64_t padding = op->getAttrOfType("pad_low").getInt(); - bool nchw = outType.getElementType().isF32() && - !op->getAttrOfType("output_nhwc").getValue(); - int64_t height = out[nchw ? 2 : 1], width = out[nchw ? 3 : 2]; - int64_t rows = height * width, columns = wt[0]; - int64_t reduction = in[3] * kernel * kernel; - int64_t packedReduction = ((reduction + 15) / 16) * 16; - if (in[0] != 1 || wt[1] != in[3] || - op->getAttrOfType("activation").getInt() > 1) - return op.emitError("large Pebble convolution requires batch one and " - "linear/ReLU activation"); - auto idx = [&](int64_t n) -> Value { - return arith::ConstantIndexOp::create(b, loc, n); - }; - Value zero = idx(0), one = idx(1); - Value zeroByte = arith::ConstantIntOp::create(b, loc, 0, 8); - Value patches = memref::AllocOp::create( - b, loc, MemRefType::get({rows, packedReduction}, b.getI8Type())); - Value weights = memref::AllocOp::create( - b, loc, MemRefType::get({packedReduction, columns}, b.getI8Type())); - Value result = memref::AllocOp::create( - b, loc, MemRefType::get({rows, columns}, b.getF32Type())); - linalg::FillOp::create(b, loc, zeroByte, patches); - linalg::FillOp::create(b, loc, zeroByte, weights); - auto trace = op->getAttrOfType("trace_node"); - if (trace && cast(trace.get("cycle")).getValue()) { - auto start = ::buddy::trace::StartOp::create( - b, loc, cast(trace.get("id")), b.getBoolAttr(true), - b.getBoolAttr(false), cast(trace.get("tag"))); - start->setAttr("id_path", trace.get("id_path")); - } - auto rowLoop = scf::ForOp::create(b, loc, zero, idx(rows), one); - b.setInsertionPointToStart(rowLoop.getBody()); - auto kLoop = scf::ForOp::create(b, loc, zero, idx(reduction), one); - b.setInsertionPointToStart(kLoop.getBody()); - Value row = rowLoop.getInductionVar(), k = kLoop.getInductionVar(); - Value channel = arith::DivUIOp::create(b, loc, k, idx(kernel * kernel)); - Value ky = arith::RemUIOp::create( - b, loc, arith::DivUIOp::create(b, loc, k, idx(kernel)), idx(kernel)); - Value kx = arith::RemUIOp::create(b, loc, k, idx(kernel)); - Value y = arith::SubIOp::create( - b, loc, - arith::AddIOp::create( - b, loc, - arith::MulIOp::create( - b, loc, arith::DivUIOp::create(b, loc, row, idx(width)), - idx(stride)), - ky), - idx(padding)); - Value x = arith::SubIOp::create( - b, loc, - arith::AddIOp::create( - b, loc, - arith::MulIOp::create( - b, loc, arith::RemUIOp::create(b, loc, row, idx(width)), - idx(stride)), - kx), - idx(padding)); - Value valid = arith::AndIOp::create( - b, loc, - arith::CmpIOp::create(b, loc, arith::CmpIPredicate::ult, y, idx(in[1])), - arith::CmpIOp::create(b, loc, arith::CmpIPredicate::ult, x, - idx(in[2]))); - auto present = scf::IfOp::create(b, loc, valid, false); - b.setInsertionPointToStart(present.thenBlock()); - Value value = - memref::LoadOp::create(b, loc, input, ValueRange{zero, y, x, channel}); - memref::StoreOp::create(b, loc, value, patches, ValueRange{row, k}); - b.setInsertionPointAfter(rowLoop); - auto outputChannel = scf::ForOp::create(b, loc, zero, idx(columns), one); - b.setInsertionPointToStart(outputChannel.getBody()); - auto weightK = scf::ForOp::create(b, loc, zero, idx(reduction), one); - b.setInsertionPointToStart(weightK.getBody()); - k = weightK.getInductionVar(); - channel = arith::DivUIOp::create(b, loc, k, idx(kernel * kernel)); - ky = arith::RemUIOp::create( - b, loc, arith::DivUIOp::create(b, loc, k, idx(kernel)), idx(kernel)); - kx = arith::RemUIOp::create(b, loc, k, idx(kernel)); - Value co = outputChannel.getInductionVar(); - value = - memref::LoadOp::create(b, loc, weight, ValueRange{co, channel, ky, kx}); - memref::StoreOp::create(b, loc, value, weights, ValueRange{k, co}); - b.setInsertionPointAfter(outputChannel); - auto region = MegaKernelOp::create(b, loc, patches, result); - region.getBody().emplaceBlock(); - b.setInsertionPointToStart(®ion.getBody().front()); - MegaMatmulOp::create(b, loc, patches, weights, op.getInputs()[2], - op.getInputs()[3], op.getInputs()[4], result, - op->getAttrOfType("activation"), - op->getAttrOfType("output_scale")); - MegaYieldOp::create(b, loc); - b.setInsertionPointAfter(region); - auto copyRow = scf::ForOp::create(b, loc, zero, idx(rows), one); - b.setInsertionPointToStart(copyRow.getBody()); - auto copyChannel = scf::ForOp::create(b, loc, zero, idx(columns), one); - b.setInsertionPointToStart(copyChannel.getBody()); - row = copyRow.getInductionVar(); - co = copyChannel.getInductionVar(); - value = memref::LoadOp::create(b, loc, result, ValueRange{row, co}); - if (outType.getElementType().isInteger(8)) { - value = math::RoundEvenOp::create(b, loc, value); - Value low = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(-128.0)); - Value high = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(127.0)); - value = arith::MaximumFOp::create(b, loc, value, low); - value = arith::MinimumFOp::create(b, loc, value, high); - value = arith::FPToSIOp::create(b, loc, b.getI8Type(), value); - } - y = arith::DivUIOp::create(b, loc, row, idx(width)); - x = arith::RemUIOp::create(b, loc, row, idx(width)); - memref::StoreOp::create(b, loc, value, output, - nchw ? ValueRange{zero, co, y, x} - : ValueRange{zero, y, x, co}); - b.setInsertionPointAfter(copyRow); - if (trace) { - auto end = ::buddy::trace::EndOp::create( - b, loc, output.getType(), output, cast(trace.get("id")), - cast(trace.get("cycle")), - cast(trace.get("tensor")), - cast(trace.get("tag"))); - end->setAttr("id_path", trace.get("id_path")); - } - memref::DeallocOp::create(b, loc, patches); - memref::DeallocOp::create(b, loc, weights); - memref::DeallocOp::create(b, loc, result); - op.erase(); - } - return success(); -} diff --git a/waste/examples/designs/pebble/stack/midend/src/Passes.cpp b/waste/examples/designs/pebble/stack/midend/src/Passes.cpp deleted file mode 100644 index 2066ffdbc..000000000 --- a/waste/examples/designs/pebble/stack/midend/src/Passes.cpp +++ /dev/null @@ -1,140 +0,0 @@ -//===- LowerPebbleToBankSSAPass.cpp - Pebble bank-SSA lowering ----------===// -// -// Pebble lowers its MegaKernels as resident regions and keeps transposes on -// banks, then lowers the remaining Ball operations through BankSSALowering. -// -//===----------------------------------------------------------------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Math/IR/Math.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Trace/TraceDialect.h" - -using namespace mlir; - -namespace mlir::buddy { -void populateMatmulRegionToBankSSAPatterns(RewritePatternSet &patterns); -void populatePebbleMegaConv2dToBankSSAPatterns(RewritePatternSet &patterns); -void populatePebbleMemTransposeToBankSSAPatterns(RewritePatternSet &patterns); -void populatePebbleResidentConvRegionToBankSSAPatterns( - RewritePatternSet &patterns); -} // namespace mlir::buddy - -LogicalResult lowerPebbleLargeConvolutions(func::FuncOp function); - -namespace { - -class FormPebbleMegaKernelsPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(FormPebbleMegaKernelsPass) - StringRef getArgument() const final { return "form-pebble-mega-kernels"; } - StringRef getDescription() const final { - return "Fuse quantized operators within one Pebble core"; - } - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - void runOnOperation() override { - if (failed(lowerPebbleLargeConvolutions(getOperation()))) { - signalPassFailure(); - return; - } - DenseMap> blocks; - getOperation().walk([&](linalg::GenericOp op) { - for (NamedAttribute attr : op->getAttrs()) - if (attr.getName().strref().starts_with("buckyball.quantized_")) { - blocks[op->getBlock()].push_back(op); - break; - } - }); - for (auto &entry : blocks) { - mlir::buddy::prepareQuantizedWeights(entry.second); - mlir::buddy::formMegaKernelRegions(entry.second); - } - } -}; - -class LowerPebbleToBankSSAPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerPebbleToBankSSAPass) - LowerPebbleToBankSSAPass() = default; - LowerPebbleToBankSSAPass(const LowerPebbleToBankSSAPass &) {} - - StringRef getArgument() const final { return "lower-pebble-to-bank-ssa"; } - StringRef getDescription() const final { - return "Lower Pebble Buckyball ops to explicit bank-SSA ops."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert(); - } - - void runOnOperation() override { - RewritePatternSet residentPatterns(&getContext()); - mlir::buddy::populatePebbleResidentConvRegionToBankSSAPatterns( - residentPatterns); - if (failed(applyPatternsGreedily(getOperation(), - std::move(residentPatterns)))) { - signalPassFailure(); - return; - } - RewritePatternSet patterns(&getContext()); - mlir::buddy::populateMatmulRegionToBankSSAPatterns(patterns); - mlir::buddy::populatePebbleMegaConv2dToBankSSAPatterns(patterns); - mlir::buddy::populatePebbleMemTransposeToBankSSAPatterns(patterns); - mlir::buddy::populateBankSSALoweringPatterns(patterns); - if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) { - signalPassFailure(); - return; - } - bool illegalMega = false; - getOperation().walk([&](Operation *op) { - if (isa<::buddy::buckyball::MegaKernelOp>(op)) { - op->emitError( - "MegaKernel region-wide bank SSA lowering is not implemented"); - illegalMega = true; - } else if (isa<::buddy::buckyball::MegaMatmulOp, - ::buddy::buckyball::MegaConv2dOp, - ::buddy::buckyball::MegaConv2dDepthwiseOp>(op) && - !op->getParentOfType<::buddy::buckyball::MegaKernelOp>()) { - op->emitError( - "MegaKernel stage is only legal inside buckyball.mega_kernel"); - illegalMega = true; - } - }); - if (illegalMega) - signalPassFailure(); - } -}; - -} // namespace - -// Registered when the compiler plugin is loaded. -[[maybe_unused]] static const bool registered = [] { - PassRegistration(); - PassRegistration(); - return true; -}(); diff --git a/waste/stack/compiler/frontend/__init__.py b/waste/stack/compiler/frontend/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/waste/stack/compiler/frontend/layout/__init__.py b/waste/stack/compiler/frontend/layout/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/waste/stack/compiler/frontend/layout/provenance.py b/waste/stack/compiler/frontend/layout/provenance.py deleted file mode 100644 index 3d6d0f232..000000000 --- a/waste/stack/compiler/frontend/layout/provenance.py +++ /dev/null @@ -1,62 +0,0 @@ -from buddy_mlir import ir - - -def has_nhwc_layout(value): - """Follow operations that preserve TOSA convolution/pooling axis order.""" - if not isinstance(value, ir.OpResult): - return False - producer = value.owner - if isinstance(producer, ir.OpView): - producer = producer.operation - if "buddy.layout" in producer.attributes: - return ir.StringAttr(producer.attributes["buddy.layout"]).value == "nhwc" - if producer.name in { - "tosa.conv2d", - "tosa.depthwise_conv2d", - "tosa.transpose_conv2d", - "tosa.avg_pool2d", - "tosa.max_pool2d", - }: - return True - if producer.name == "linalg.generic": - maps = ir.ArrayAttr(producer.attributes["indexing_maps"]) - identity = ir.AffineMap.get_identity(len(ir.RankedTensorType(value.type).shape)) - if ir.AffineMapAttr(maps[len(maps) - 1]).value != identity: - return False - return any( - ir.AffineMapAttr(mapping).value == identity - and isinstance(operand.type, ir.RankedTensorType) - and len(ir.RankedTensorType(operand.type).shape) == 4 - and has_nhwc_layout(operand) - for operand, mapping in zip(producer.operands, maps, strict=True) - ) - if producer.name in { - "tosa.add", - "tosa.sub", - "tosa.mul", - "tosa.maximum", - "tosa.minimum", - "tosa.clamp", - "tosa.sigmoid", - "tosa.tanh", - "tosa.exp", - "tosa.log", - "tosa.negate", - "tosa.abs", - "tosa.reciprocal", - "tosa.rsqrt", - "tosa.cast", - "tosa.select", - "tosa.slice", - "tosa.pad", - "tensor.cast", - "math.roundeven", - "buddy_trace.end", - }: - return any( - isinstance(operand.type, ir.RankedTensorType) - and len(ir.RankedTensorType(operand.type).shape) == 4 - and has_nhwc_layout(operand) - for operand in producer.operands - ) - return False diff --git a/waste/stack/compiler/frontend/operation.py b/waste/stack/compiler/frontend/operation.py deleted file mode 100644 index 153dcff5a..000000000 --- a/waste/stack/compiler/frontend/operation.py +++ /dev/null @@ -1,107 +0,0 @@ -# ===- operation.py ------------------------------------------------------------ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ===--------------------------------------------------------------------------- -# -# This is the operation structure of Buddy Compiler graph representation. -# -# ===--------------------------------------------------------------------------- - -from buddy.compiler.graph.operation import Conv2dOp, Op, OpType - - -class QuantizedOp(Op): - def __init__(self): - super().__init__() - self.quantization_trace_meta = None - - -class QuantizedMatmulOp(QuantizedOp): - def __init__(self) -> None: - super().__init__() - self._op_type = OpType.ReduceType - - -class QuantizedConv2dOp(QuantizedOp): - def __init__(self) -> None: - super().__init__() - self._op_type = OpType.ReduceType - self._output_nhwc = False - - -class QuantizedDepthwiseConv2dOp(QuantizedOp): - def __init__(self) -> None: - super().__init__() - self._op_type = OpType.ReduceType - self._output_nhwc = False - - -class QuantizedMaxPool2dOp(QuantizedOp): - def __init__(self) -> None: - super().__init__() - self._op_type = OpType.ReduceType - - -class QuantizedGlobalAvgPoolOp(QuantizedOp): - def __init__(self) -> None: - super().__init__() - self._op_type = OpType.ReduceType - - -class QuantizedChannelSliceOp(QuantizedOp): - def __init__(self) -> None: - super().__init__() - self._op_type = OpType.ReshapeType - - -class QuantizedChannelConcatOp(QuantizedOp): - def __init__(self) -> None: - super().__init__() - self._op_type = OpType.ConcatType - - -class QuantizedResizeNearestOp(QuantizedOp): - def __init__(self) -> None: - super().__init__() - self._op_type = OpType.ReshapeType - - -class QuantizedMulOp(QuantizedOp): - def __init__(self) -> None: - super().__init__() - self._op_type = OpType.ElementwiseType - - -class QuantizedAddOp(QuantizedOp): - def __init__(self) -> None: - super().__init__() - self._op_type = OpType.ElementwiseType - - -class Int8QuantizeOp(Op): - def __init__(self): - super().__init__() - self._op_type = OpType.ElementwiseType - - -class Int8Conv2dOp(Conv2dOp): - """Signed INT8 convolution with INT32 accumulation and FP32 rescaling.""" - - -class Int8MatmulOp(Op): - """Signed INT8 matrix product, scales and optional FP32 bias.""" - - def __init__(self) -> None: - super().__init__() - self._op_type = OpType.ReduceType diff --git a/waste/stack/compiler/frontend/ops/__init__.py b/waste/stack/compiler/frontend/ops/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/waste/stack/compiler/frontend/ops/expf.py b/waste/stack/compiler/frontend/ops/expf.py deleted file mode 100644 index c335162fa..000000000 --- a/waste/stack/compiler/frontend/ops/expf.py +++ /dev/null @@ -1,83 +0,0 @@ -# FP32 range reduction and coefficients follow SLEEF's expf. -# Copyright Naoki Shibata and contributors 2010 - 2025. -# Distributed under the Boost Software License, Version 1.0. -# https://github.com/shibatch/sleef/blob/3.9.0/src/libm/sleefsimdsp.c - -# Boost Software License - Version 1.0 - August 17th, 2003 -# -# Permission is hereby granted, free of charge, to any person or organization -# obtaining a copy of the software and accompanying documentation covered by -# this license (the "Software") to use, reproduce, display, distribute, -# execute, and transmit the Software, and to prepare derivative works of the -# Software, and to permit third-parties to whom the Software is furnished to -# do so, all subject to the following: -# -# The copyright notices in the Software and this entire statement, including -# the above license grant, this restriction and the following disclaimer, -# must be included in all copies of the Software, in whole or in part, and -# all derivative works of the Software, unless such copies or derivative -# works are solely in the form of machine-executable object code generated by -# a source language processor. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -# SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -# FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -from buddy_mlir import ir -from buddy_mlir.dialects import arith, math - - -def exp_f32(value): - """Emit a fixed FP32 exp approximation before activation rounding.""" - f32 = ir.F32Type.get() - i32 = ir.IntegerType.get_signless(32) - - def constant(number): - return arith.ConstantOp(f32, float(number)).result - - zero = constant(0.0) - lower = constant(-104.0) - upper = constant(100.0) - is_nan = arith.CmpFOp(arith.CmpFPredicate.UNO, value, value).result - underflow = arith.CmpFOp(arith.CmpFPredicate.OLT, value, lower).result - overflow = arith.CmpFOp(arith.CmpFPredicate.OGT, value, upper).result - # Bound the exponent conversion, including NaN and infinity inputs. - bounded = arith.SelectOp(underflow, lower, value).result - bounded = arith.SelectOp(overflow, upper, bounded).result - bounded = arith.SelectOp(is_nan, zero, bounded).result - rounded = math.RoundEvenOp( - arith.MulFOp(bounded, constant(1.4426950408889634)).result - ).result - exponent = arith.FPToSIOp(i32, rounded).result - residual = math.FmaOp(rounded, constant(-0.693145751953125), bounded).result - residual = math.FmaOp(rounded, constant(-1.428606765330187e-6), residual).result - polynomial = constant(0.00019852761761285365) - for coefficient in ( - 0.0013930435525253415, - 0.008333360776305199, - 0.041666485369205475, - 0.1666666716337204, - 0.5, - ): - polynomial = math.FmaOp(polynomial, residual, constant(coefficient)).result - square = arith.MulFOp(residual, residual).result - polynomial = arith.AddFOp( - constant(1.0), math.FmaOp(square, polynomial, residual).result - ).result - - one = arith.ConstantOp(i32, 1).result - bias = arith.ConstantOp(i32, 127).result - shift = arith.ConstantOp(i32, 23).result - half = arith.ShRSIOp(exponent, one).result - remainder = arith.SubIOp(exponent, half).result - for power in (half, remainder): - bits = arith.ShLIOp(arith.AddIOp(power, bias).result, shift).result - factor = arith.BitcastOp(f32, bits).result - polynomial = arith.MulFOp(polynomial, factor).result - result = arith.SelectOp(underflow, zero, polynomial).result - result = arith.SelectOp(overflow, constant(float("inf")), result).result - return arith.SelectOp(is_nan, value, result).result diff --git a/waste/stack/compiler/frontend/ops/int8.py b/waste/stack/compiler/frontend/ops/int8.py deleted file mode 100644 index bfbb53b4a..000000000 --- a/waste/stack/compiler/frontend/ops/int8.py +++ /dev/null @@ -1,250 +0,0 @@ -"""Native signed INT8 contractions with INT32 accumulators.""" - -import math as python_math - -from buddy_mlir import ir -from buddy_mlir.dialects import arith, linalg, math, tensor, tosa - - -def _maps(values): - return ir.ArrayAttr.get([ir.AffineMapAttr.get(value) for value in values]) - - -def _iterators(parallel, reduction): - return ir.ArrayAttr.get( - [ir.Attribute.parse("#linalg.iterator_type")] * parallel - + [ir.Attribute.parse("#linalg.iterator_type")] * reduction - ) - - -def _zero(shape): - dtype = ir.IntegerType.get_signless(32) - empty = tensor.EmptyOp(shape, dtype) - zero = arith.ConstantOp(dtype, 0).result - fill = linalg.FillOp([empty.result.type], [zero], [empty.result]) - linalg.fill_builtin_region(fill.operation) - return fill.result - - -def _multiply_accumulate(op): - i8 = ir.IntegerType.get_signless(8) - i32 = ir.IntegerType.get_signless(32) - body = ir.Block.create_at_start(op.region, [i8, i8, i32]) - with ir.InsertionPoint(body): - lhs = arith.ExtSIOp(i32, body.arguments[0]).result - rhs = arith.ExtSIOp(i32, body.arguments[1]).result - product = arith.MulIOp(lhs, rhs).result - linalg.YieldOp([arith.AddIOp(body.arguments[2], product).result]) - - -def _rescale( - accumulator, - activation_scale, - weight_scale, - bias, - symbols, - *, - channel_axis=1, -): - from buddy.compiler.ops.tosa import _create_shape_operand, _gen_arith_binary_op - - shape = list(ir.RankedTensorType(accumulator.type).shape) - rank = len(shape) - f32 = ir.F32Type.get() - if isinstance(activation_scale, str): - activation_scale = symbols[(activation_scale, 0)] - else: - ty = ir.RankedTensorType.get([1] * rank, f32) - activation_scale = tosa.ConstOp( - ir.DenseElementsAttr.get_splat(ty, ir.FloatAttr.get(f32, activation_scale)) - ).result - activation_shape = list(ir.RankedTensorType(activation_scale.type).shape) - if channel_axis == 3 and len(activation_shape) == 4 and activation_shape[1] != 1: - activation_scale = tosa.ReshapeOp( - activation_scale, - _create_shape_operand([1, 1, 1, activation_shape[1]]), - ).result - scale = symbols[(weight_scale, 0)] - count = python_math.prod(ir.RankedTensorType(scale.type).shape) - broadcast_shape = [1] * rank - broadcast_shape[channel_axis] = count - scale = tosa.ReshapeOp(scale, _create_shape_operand(broadcast_shape)).result - combined_scale = _gen_arith_binary_op(activation_scale, scale, tosa.MulOp).result - restored = ( - accumulator - if ir.RankedTensorType(accumulator.type).element_type == f32 - else tosa.CastOp(ir.RankedTensorType.get(shape, f32), accumulator).result - ) - result = _gen_arith_binary_op(restored, combined_scale, tosa.MulOp).result - if bias is not None: - value = symbols[(bias, 0)] - bias_shape = [1] * rank - bias_shape[channel_axis] = shape[channel_axis] - value = tosa.ReshapeOp(value, _create_shape_operand(bias_shape)).result - result = _gen_arith_binary_op(result, value, tosa.AddOp).result - return result - - -def conv2d(node, symbols): - from stack.compiler.frontend.layout.provenance import has_nhwc_layout - - value, weight = [symbols[(name, 0)] for name in node.args[:2]] - input_nhwc = has_nhwc_layout(value) - output_nhwc = node.tensor_meta.get("activation_layout") == "nhwc" - ( - bias, - stride, - padding, - dilation, - transposed, - _, - groups, - activation_scale, - weight_scale, - ) = node.args[2:] - if transposed: - raise NotImplementedError("INT8 transpose convolution is not supported") - input_shape = list(ir.RankedTensorType(value.type).shape) - stored_shape = list(ir.RankedTensorType(weight.type).shape) - permutation = node.tensor_meta.get("weight_permutation", [0, 1, 2, 3]) - weight_shape = node.tensor_meta.get( - "weight_shape", [stored_shape[permutation.index(i)] for i in range(4)] - ) - shape = list(node.tensor_meta["shape"]) - if output_nhwc: - shape = [shape[0], shape[2], shape[3], shape[1]] - ph, pw = padding - if ph or pw: - low = [0, ph, pw, 0] if input_nhwc else [0, 0, ph, pw] - padded_shape = [size + 2 * pad for size, pad in zip(input_shape, low)] - op = tensor.PadOp( - ir.RankedTensorType.get(padded_shape, ir.IntegerType.get_signless(8)), - value, - [], - [], - low, - low, - ) - block = ir.Block.create_at_start(op.region, [ir.IndexType.get()] * 4) - with ir.InsertionPoint(block): - tensor.YieldOp(arith.ConstantOp(ir.IntegerType.get_signless(8), 0).result) - value = op.result - n, f, h, w, c, kh, kw = [ir.AffineExpr.get_dim(i) for i in range(7)] - channel = ( - ir.AffineExpr.get_floor_div(f, weight_shape[0] // groups) * weight_shape[1] + c - ) - if node.tensor_meta.get("weight_layout") == "blocked_conv": - lanes = node.tensor_meta["output_lanes"] - d0, d1, d2, d3 = [ir.AffineExpr.get_dim(i) for i in range(4)] - indices = [ - ir.AffineExpr.get_floor_div(d0, lanes), - d1, - d2 * weight_shape[3] + d3, - ir.AffineExpr.get_mod(d0, lanes), - ] - unpacked = tensor.EmptyOp(weight_shape, ir.IntegerType.get_signless(8)) - unpack = linalg.GenericOp( - [unpacked.result.type], - [weight], - [unpacked.result], - _maps([ir.AffineMap.get(4, 0, indices), ir.AffineMap.get_identity(4)]), - _iterators(4, 0), - ) - block = ir.Block.create_at_start( - unpack.region, [ir.IntegerType.get_signless(8)] * 2 - ) - with ir.InsertionPoint(block): - linalg.YieldOp([block.arguments[0]]) - weight = unpack.result - permutation = [0, 1, 2, 3] - weight_indices = [[f, c, kh, kw][i] for i in permutation] - height = h * stride[0] + kh * dilation[0] - width = w * stride[1] + kw * dilation[1] - maps = [ - ir.AffineMap.get( - 7, - 0, - ( - [n, height, width, channel] - if input_nhwc - else [n, channel, height, width] - ), - ), - ir.AffineMap.get(7, 0, weight_indices), - ir.AffineMap.get(7, 0, [n, h, w, f] if output_nhwc else [n, f, h, w]), - ] - output = _zero(shape) - op = linalg.GenericOp( - [output.type], [value, weight], [output], _maps(maps), _iterators(4, 3) - ) - _multiply_accumulate(op) - op.operation.attributes["buddy.int8_conv"] = ir.DictAttr.get( - { - "stride": ir.DenseI64ArrayAttr.get(stride), - "dilation": ir.DenseI64ArrayAttr.get(dilation), - "groups": ir.IntegerAttr.get(ir.IntegerType.get_signless(64), groups), - "weight_permutation": ir.DenseI64ArrayAttr.get(permutation), - } - ) - result = _rescale( - op.result, - activation_scale, - weight_scale, - bias, - symbols, - channel_axis=3 if output_nhwc else 1, - ) - if output_nhwc: - result.owner.attributes["buddy.layout"] = ir.StringAttr.get("nhwc") - return result - - -def matmul(node, symbols): - activation, weight, activation_scale, weight_scale, bias = node.args - lhs, rhs = symbols[(activation, 0)], symbols[(weight, 0)] - shape = list(node.tensor_meta["shape"]) - m, n, k = [ir.AffineExpr.get_dim(i) for i in range(3)] - maps = [ - ir.AffineMap.get(3, 0, [m, k]), - ir.AffineMap.get( - 3, 0, [n, k] if node.tensor_meta.get("transpose_b") else [k, n] - ), - ir.AffineMap.get(3, 0, [m, n]), - ] - output = _zero(shape) - op = linalg.GenericOp( - [output.type], [lhs, rhs], [output], _maps(maps), _iterators(2, 1) - ) - _multiply_accumulate(op) - op.operation.attributes["buddy.int8_matmul"] = ir.UnitAttr.get() - return _rescale(op.result, activation_scale, weight_scale, bias, symbols) - - -def quantize_activation(node, symbols): - value = symbols[(node.args[0], 0)] - shape = list(ir.RankedTensorType(value.type).shape) - f32, i8 = ir.F32Type.get(), ir.IntegerType.get_signless(8) - output = tensor.EmptyOp(shape, i8) - identity = ir.AffineMap.get_identity(len(shape)) - op = linalg.GenericOp( - [output.result.type], - [value], - [output.result], - _maps([identity, identity]), - _iterators(len(shape), 0), - ) - multiplier = 1.0 / node.args[1] - op.operation.attributes["buckyball.quant_f32_to_i8"] = ir.BoolAttr.get(True) - op.operation.attributes["nchw_to_nhwc"] = ir.BoolAttr.get(False) - op.operation.attributes["scale"] = ir.FloatAttr.get(f32, multiplier) - block = ir.Block.create_at_start(op.region, [f32, i8]) - with ir.InsertionPoint(block): - scale = arith.ConstantOp(f32, ir.FloatAttr.get(f32, multiplier)).result - value = math.RoundEvenOp(arith.MulFOp(block.arguments[0], scale).result).result - minimum = arith.ConstantOp(f32, ir.FloatAttr.get(f32, -128.0)).result - maximum = arith.ConstantOp(f32, ir.FloatAttr.get(f32, 127.0)).result - value = arith.MinimumFOp( - arith.MaximumFOp(value, minimum).result, maximum - ).result - linalg.YieldOp([arith.FPToSIOp(i8, value).result]) - return op.result diff --git a/waste/stack/compiler/frontend/ops/linalg.py b/waste/stack/compiler/frontend/ops/linalg.py deleted file mode 100644 index e0339f10c..000000000 --- a/waste/stack/compiler/frontend/ops/linalg.py +++ /dev/null @@ -1,1599 +0,0 @@ -# ===- linalg.py --------------------------------------------------------------- -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ===--------------------------------------------------------------------------- -# -# The registry of mappings from Buddy Graph to MLIR linalg dialect operations. -# -# ===--------------------------------------------------------------------------- - -import array -import copy -import buddy_mlir.ir as ir -import numpy -from buddy_mlir.dialects import ( - arith, - bufferization, - linalg, - math, - memref, - scf, - tensor, - tosa, - vector, -) -from buddy_mlir.dialects import ( - complex as complex_dialect, -) -from buddy.compiler.graph import * -from buddy.compiler.graph.graph import TensorDType -from buddy.compiler.graph.transform.trace.lowering import _trace_attrs, has_trace -from buddy.compiler.ops.utils import * -from stack.compiler.frontend.operation import * -from buddy.compiler.ops.linalg import ( - _as_value, - _const_shape_operand, - _max_pool_indices_to_offsets_3d, - _normalize_pool_param, - max_pool3d_op, -) - - -def _perm_attr(perm: list[int]) -> ir.Attribute: - return ir.DenseI32ArrayAttr.get([int(x) for x in perm]) - - -def _nhwc_to_nchw(value: ir.Value) -> ir.Value: - shape = [int(x) for x in ir.RankedTensorType(value.type).shape] - if len(shape) != 4: - raise RuntimeError(f"_nhwc_to_nchw expects rank-4, got {shape}") - n, h, w, c = shape - elem = ir.RankedTensorType(value.type).element_type - out_ty = ir.RankedTensorType.get([n, c, h, w], elem) - return tosa.TransposeOp(out_ty, value, _perm_attr([0, 3, 1, 2])).result - - -def _nchw_to_nhwc(value: ir.Value) -> ir.Value: - shape = [int(x) for x in ir.RankedTensorType(value.type).shape] - if len(shape) != 4: - raise RuntimeError(f"_nchw_to_nhwc expects rank-4, got {shape}") - n, c, h, w = shape - elem = ir.RankedTensorType(value.type).element_type - out_ty = ir.RankedTensorType.get([n, h, w, c], elem) - return tosa.TransposeOp(out_ty, value, _perm_attr([0, 2, 3, 1])).result - - -def _split_via_tosa_slice( - input_tensor: ir.Value, - input_shape: list[int], - dim: int, - split_sizes: list[int], -) -> list[ir.Value]: - # tosa.slice / extract_slice on non-dim1 (esp. NHWC C) lowers wrong; - # always slice on dim1 via transpose. - rank = len(input_shape) - if dim != 1: - perm = list(range(rank)) - perm[1], perm[dim] = perm[dim], perm[1] - inv = [0] * rank - for i, j in enumerate(perm): - inv[j] = i - elem = ir.RankedTensorType(input_tensor.type).element_type - t_shape = [input_shape[i] for i in perm] - t = tosa.TransposeOp( - ir.RankedTensorType.get(t_shape, elem), - input_tensor, - _perm_attr(perm), - ).result - parts = _split_via_tosa_slice(t, t_shape, 1, split_sizes) - out = [] - for p in parts: - ps = [int(x) for x in ir.RankedTensorType(p.type).shape] - o_shape = [ps[i] for i in inv] - out.append( - tosa.TransposeOp( - ir.RankedTensorType.get(o_shape, elem), - p, - _perm_attr(inv), - ).result - ) - return out - - elem = ir.RankedTensorType(input_tensor.type).element_type - results = [] - offset = 0 - for size in split_sizes: - start = [0] * rank - start[1] = offset - out_shape = list(input_shape) - out_shape[1] = size - out_ty = ir.RankedTensorType.get(out_shape, elem) - results.append( - tosa.SliceOp( - out_ty, - input_tensor, - _const_shape_operand(start), - _const_shape_operand(out_shape), - ).result - ) - offset += size - return results - - -def cat_op( - node: CatOp, - symbol_table: dict[tuple[str, int], ir.Operation], -): - """ - Import the tensor concate operation. - From buddy CatOp to MLIR tensor `insert_slice` - operation. - - Note: This op, concate multiple input tensors. - Args: - node: Containing information from the input graph node. - symbol_table: A dictionary mapping symbols to their corresponding - operations. - - Returns: - op: The operation return the tensor.insert_slice op. - """ - # Support both 2 args and variable number of inputs - inputs = node.args[0] # List of input tensors - # dim is optional, default to 0 if not provided - dim = int(node.args[1]) if len(node.args) > 1 else 0 - - # Get all input tensors - input_tensors = [] - input_shapes = [] - - for inp in inputs: - t = symbol_table.get((str(inp), 0)) - if t is None: - raise ValueError(f"missing cat input for {node.name}: {inp}") - input_tensors.append(t) - input_shapes.append(list(ir.RankedTensorType(t.type).shape)) - - if len(input_tensors) == 0: - raise ValueError(f"cat has no inputs: {node.name}") - - rank = len(input_shapes[0]) - if dim < 0: - dim += rank - - # Sticky NHWC: PyTorch cat dim is NCHW; remap when SSA is NHWC. - out_meta = [int(x) for x in node.tensor_meta["shape"]] - if rank == 4 and dim == 1 and len(out_meta) == 4: - n, out_c, h, w = out_meta - nchw_inputs = [] - channels = 0 - for value, shape in zip(input_tensors, input_shapes): - if shape[0] == n and shape[1] == h and shape[2] == w: - nchw_inputs.append(_nhwc_to_nchw(value)) - channels += shape[3] - elif shape[0] == n and shape[2] == h and shape[3] == w: - nchw_inputs.append(value) - channels += shape[1] - else: - raise RuntimeError( - f"channel cat input layout mismatch: shape={shape} meta={out_meta}" - ) - if channels != out_c: - raise RuntimeError( - f"channel cat count mismatch: inputs={input_shapes} meta={out_meta}" - ) - return tosa.ConcatOp(nchw_inputs, 1).result - - if ( - rank == 4 - and len(out_meta) == 4 - and input_shapes[0][1] == out_meta[2] - and input_shapes[0][2] == out_meta[3] - and sum(s[3] for s in input_shapes) == out_meta[1] - ): - dim = {0: 0, 1: 3, 2: 1, 3: 2}[dim] - - for s in input_shapes[1:]: - if len(s) != rank: - raise RuntimeError(f"cat rank mismatch: {input_shapes[0]} vs {s}") - for d in range(rank): - if d != dim and s[d] != input_shapes[0][d]: - raise RuntimeError( - f"cat non-concat dim{d} mismatch: shapes={input_shapes} " - f"dim={dim} meta={out_meta}" - ) - - output_shape = list(input_shapes[0]) - output_shape[dim] = sum(s[dim] for s in input_shapes) - - dtype = node.tensor_meta["dtype"] - mlir_dtype = mlir_element_type_get(dtype) - - # insert_slice on NHWC C lowers wrong; concat on NCHW axis=1 instead. - nhwc_c = ( - rank == 4 - and dim == 3 - and input_shapes[0][1] == input_shapes[0][2] - and input_shapes[0][3] < output_shape[3] - ) - if nhwc_c: - nchw_inputs = [_nhwc_to_nchw(t) for t in input_tensors] - cat_nchw = tosa.ConcatOp(nchw_inputs, 1).result - return cat_nchw - - output = tensor.EmptyOp(output_shape, mlir_dtype) - - offset = [0] * rank - stride_attr = ir._denseI64ArrayAttr([1] * rank, None) - - current_result = output.result - - for input_tensor, input_shape in zip(input_tensors, input_shapes): - offset_attr = ir._denseI64ArrayAttr(offset, None) - size_attr = ir._denseI64ArrayAttr(input_shape, None) - insert_op = tensor.InsertSliceOp( - input_tensor, - current_result, - [], - [], - [], - offset_attr, - size_attr, - stride_attr, - ) - current_result = insert_op.result - offset[dim] += input_shape[dim] - - return insert_op - - -def split_op(node: SplitOp, symbol_table): - """ - Split the input tensor into smaller tensors along the specified dimension. - - Args: - node (SplitOp): The split operation node with metadata. - symbol_table: Mapping of variable names to tensor references. - - Returns: - List[Tensor]: List of split tensors. - """ - # Get the input tensor and parameters - input_tensor = symbol_table.get((str(node.args[0]), 0), node.args[0]) - if not hasattr(input_tensor, "type"): - raise ValueError(f"missing split input for {node.name}: {node.args[0]}") - split_size = node.args[1] # Size of each split tensor - input_shape = [int(x) for x in input_tensor.type.shape] - dim = int(node.args[2]) # Dimension to split along (PyTorch NCHW) - if dim < 0: - dim += len(input_shape) - - # Sticky NHWC: remap channel/spatial dims from NCHW numbering. - meta_shapes = node.tensor_meta["shape"] - if ( - isinstance(meta_shapes, (list, tuple)) - and len(meta_shapes) > 0 - and hasattr(meta_shapes[0], "__len__") - and len(input_shape) == 4 - and len(meta_shapes[0]) == 4 - ): - m0 = [int(x) for x in meta_shapes[0]] - # Full NCHW concat along dim would match input meta; NHWC has C last. - if ( - input_shape[3] == sum(int(s[1]) for s in meta_shapes) - and input_shape[1] == m0[2] - ): - dim = {0: 0, 1: 3, 2: 1, 3: 2}[dim] - - split_count = (input_shape[dim] + split_size - 1) // split_size # Round up - if ( - isinstance(meta_shapes, (list, tuple)) - and len(meta_shapes) > 0 - and hasattr(meta_shapes[0], "__len__") - and split_count != len(meta_shapes) - ): - raise RuntimeError( - f"split count mismatch: ssa_dim={dim} count={split_count} " - f"meta_outs={len(meta_shapes)} shape={input_shape}" - ) - - split_sizes = [ - min(split_size, input_shape[dim] - i * split_size) for i in range(split_count) - ] - return _split_via_tosa_slice(input_tensor, input_shape, dim, split_sizes) - - -def _infer_pool2d_nhwc(shape: list[int]) -> bool: - if len(shape) != 4: - raise RuntimeError(f"expected rank-4 pool tensor, got {shape}") - _, a, b, c = shape - if a == b and c > a: - return True - if b == c and a > b: - return False - raise RuntimeError(f"cannot infer pool layout from shape {shape}") - - -def _pool2d_input_hw(input_shape: list[int]) -> list[int]: - if len(input_shape) != 4: - raise RuntimeError(f"expected rank-4 pool input, got {input_shape}") - _, a, b, c = input_shape - if a == b and c > a: - return [a, b] - if b == c and a > b: - return [b, c] - raise RuntimeError(f"cannot infer pool input_hw from shape {input_shape}") - - -def _emit_max_pool_offset_store( - flat_idx, - oh, - ow, - out_idx, - w_const, - kw_const, - sh_const, - sw_const, - ph_const, - pw_const, - dh_const, - dw_const, - i64, - i8, - offsets_memref, -): - ih = arith.DivSIOp(flat_idx, w_const).result - iw = arith.RemSIOp(flat_idx, w_const).result - - oh_i64 = arith.IndexCastOp(i64, oh).result - ow_i64 = arith.IndexCastOp(i64, ow).result - - hbase = arith.SubIOp(arith.MulIOp(oh_i64, sh_const).result, ph_const).result - wbase = arith.SubIOp(arith.MulIOp(ow_i64, sw_const).result, pw_const).result - - h_inc = arith.DivSIOp(arith.SubIOp(ih, hbase).result, dh_const).result - w_inc = arith.DivSIOp(arith.SubIOp(iw, wbase).result, dw_const).result - offset_i64 = arith.AddIOp(arith.MulIOp(h_inc, kw_const).result, w_inc).result - offset_i8 = arith.TruncIOp(i8, offset_i64).result - memref.StoreOp(offset_i8, offsets_memref.result, out_idx) - - -def _max_pool_indices_to_offsets_2d( - indices_tensor: ir.Value, - input_hw: list[int], - kernel_size: list[int], - stride: list[int], - padding: list[int], - dilation: list[int], -): - indices_tensor = _as_value(indices_tensor) - shape = [int(x) for x in ir.RankedTensorType(indices_tensor.type).shape] - nhwc = _infer_pool2d_nhwc(shape) - i64 = ir.IntegerType.get_signless(64) - i8 = ir.IntegerType.get_signless(8) - index_ty = ir.IndexType.get() - - offsets_memref_type = ir.MemRefType.get(shape, i8) - offsets_memref = memref.AllocOp(offsets_memref_type, [], []) - - lb = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, 0)).result - step = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, 1)).result - - n_ub = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, shape[0])).result - if nhwc: - oh_ub = arith.ConstantOp( - index_ty, ir.IntegerAttr.get(index_ty, shape[1]) - ).result - ow_ub = arith.ConstantOp( - index_ty, ir.IntegerAttr.get(index_ty, shape[2]) - ).result - c_ub = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, shape[3])).result - else: - c_ub = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, shape[1])).result - oh_ub = arith.ConstantOp( - index_ty, ir.IntegerAttr.get(index_ty, shape[2]) - ).result - ow_ub = arith.ConstantOp( - index_ty, ir.IntegerAttr.get(index_ty, shape[3]) - ).result - - w_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(input_hw[1]))).result - kw_const = arith.ConstantOp( - i64, ir.IntegerAttr.get(i64, int(kernel_size[1])) - ).result - sh_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(stride[0]))).result - sw_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(stride[1]))).result - ph_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(padding[0]))).result - pw_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(padding[1]))).result - dh_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(dilation[0]))).result - dw_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(dilation[1]))).result - - n_for = scf.ForOp(lb, n_ub, step) - with ir.InsertionPoint(n_for.body): - n = n_for.induction_variable - if nhwc: - oh_for = scf.ForOp(lb, oh_ub, step) - with ir.InsertionPoint(oh_for.body): - oh = oh_for.induction_variable - ow_for = scf.ForOp(lb, ow_ub, step) - with ir.InsertionPoint(ow_for.body): - ow = ow_for.induction_variable - c_for = scf.ForOp(lb, c_ub, step) - with ir.InsertionPoint(c_for.body): - c = c_for.induction_variable - flat_idx = tensor.ExtractOp( - indices_tensor, [n, oh, ow, c] - ).result - out_idx = [n, oh, ow, c] - _emit_max_pool_offset_store( - flat_idx, - oh, - ow, - out_idx, - w_const, - kw_const, - sh_const, - sw_const, - ph_const, - pw_const, - dh_const, - dw_const, - i64, - i8, - offsets_memref, - ) - scf.YieldOp([]) - scf.YieldOp([]) - scf.YieldOp([]) - scf.YieldOp([]) - else: - c_for = scf.ForOp(lb, c_ub, step) - with ir.InsertionPoint(c_for.body): - c = c_for.induction_variable - oh_for = scf.ForOp(lb, oh_ub, step) - with ir.InsertionPoint(oh_for.body): - oh = oh_for.induction_variable - ow_for = scf.ForOp(lb, ow_ub, step) - with ir.InsertionPoint(ow_for.body): - ow = ow_for.induction_variable - flat_idx = tensor.ExtractOp( - indices_tensor, [n, c, oh, ow] - ).result - out_idx = [n, c, oh, ow] - _emit_max_pool_offset_store( - flat_idx, - oh, - ow, - out_idx, - w_const, - kw_const, - sh_const, - sw_const, - ph_const, - pw_const, - dh_const, - dw_const, - i64, - i8, - offsets_memref, - ) - scf.YieldOp([]) - scf.YieldOp([]) - scf.YieldOp([]) - scf.YieldOp([]) - - offsets_type = ir.RankedTensorType.get(shape, i8) - return bufferization.ToTensorOp( - offsets_type, offsets_memref.result, restrict=True - ).result - - -def _max_pool_offsets_to_indices_2d( - offsets_tensor: ir.Value, - input_hw: list[int], - kernel_size: list[int], - stride: list[int], - padding: list[int], - dilation: list[int], -): - offsets_tensor = _as_value(offsets_tensor) - shape = [int(x) for x in ir.RankedTensorType(offsets_tensor.type).shape] - nhwc = _infer_pool2d_nhwc(shape) - i64 = ir.IntegerType.get_signless(64) - i8 = ir.IntegerType.get_signless(8) - index_ty = ir.IndexType.get() - - indices_memref_type = ir.MemRefType.get(shape, i64) - indices_memref = memref.AllocOp(indices_memref_type, [], []) - - lb = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, 0)).result - step = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, 1)).result - - n_ub = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, shape[0])).result - if nhwc: - oh_ub = arith.ConstantOp( - index_ty, ir.IntegerAttr.get(index_ty, shape[1]) - ).result - ow_ub = arith.ConstantOp( - index_ty, ir.IntegerAttr.get(index_ty, shape[2]) - ).result - c_ub = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, shape[3])).result - else: - c_ub = arith.ConstantOp(index_ty, ir.IntegerAttr.get(index_ty, shape[1])).result - oh_ub = arith.ConstantOp( - index_ty, ir.IntegerAttr.get(index_ty, shape[2]) - ).result - ow_ub = arith.ConstantOp( - index_ty, ir.IntegerAttr.get(index_ty, shape[3]) - ).result - - in_w_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(input_hw[1]))).result - kw_const = arith.ConstantOp( - i64, ir.IntegerAttr.get(i64, int(kernel_size[1])) - ).result - sh_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(stride[0]))).result - sw_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(stride[1]))).result - ph_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(padding[0]))).result - pw_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(padding[1]))).result - dh_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(dilation[0]))).result - dw_const = arith.ConstantOp(i64, ir.IntegerAttr.get(i64, int(dilation[1]))).result - - n_for = scf.ForOp(lb, n_ub, step) - with ir.InsertionPoint(n_for.body): - n = n_for.induction_variable - if nhwc: - oh_for = scf.ForOp(lb, oh_ub, step) - with ir.InsertionPoint(oh_for.body): - oh = oh_for.induction_variable - ow_for = scf.ForOp(lb, ow_ub, step) - with ir.InsertionPoint(ow_for.body): - ow = ow_for.induction_variable - c_for = scf.ForOp(lb, c_ub, step) - with ir.InsertionPoint(c_for.body): - c = c_for.induction_variable - off_i8 = tensor.ExtractOp(offsets_tensor, [n, oh, ow, c]).result - off_i64 = arith.ExtSIOp(i64, off_i8).result - - h_inc = arith.DivSIOp(off_i64, kw_const).result - w_inc = arith.RemSIOp(off_i64, kw_const).result - - oh_i64 = arith.IndexCastOp(i64, oh).result - ow_i64 = arith.IndexCastOp(i64, ow).result - - hbase = arith.SubIOp( - arith.MulIOp(oh_i64, sh_const).result, ph_const - ).result - wbase = arith.SubIOp( - arith.MulIOp(ow_i64, sw_const).result, pw_const - ).result - - ih = arith.AddIOp( - hbase, arith.MulIOp(h_inc, dh_const).result - ).result - iw = arith.AddIOp( - wbase, arith.MulIOp(w_inc, dw_const).result - ).result - - flat_idx = arith.AddIOp( - arith.MulIOp(ih, in_w_const).result, iw - ).result - memref.StoreOp(flat_idx, indices_memref.result, [n, oh, ow, c]) - scf.YieldOp([]) - scf.YieldOp([]) - scf.YieldOp([]) - scf.YieldOp([]) - else: - c_for = scf.ForOp(lb, c_ub, step) - with ir.InsertionPoint(c_for.body): - c = c_for.induction_variable - oh_for = scf.ForOp(lb, oh_ub, step) - with ir.InsertionPoint(oh_for.body): - oh = oh_for.induction_variable - ow_for = scf.ForOp(lb, ow_ub, step) - with ir.InsertionPoint(ow_for.body): - ow = ow_for.induction_variable - - off_i8 = tensor.ExtractOp(offsets_tensor, [n, c, oh, ow]).result - off_i64 = arith.ExtSIOp(i64, off_i8).result - - h_inc = arith.DivSIOp(off_i64, kw_const).result - w_inc = arith.RemSIOp(off_i64, kw_const).result - - oh_i64 = arith.IndexCastOp(i64, oh).result - ow_i64 = arith.IndexCastOp(i64, ow).result - - hbase = arith.SubIOp( - arith.MulIOp(oh_i64, sh_const).result, ph_const - ).result - wbase = arith.SubIOp( - arith.MulIOp(ow_i64, sw_const).result, pw_const - ).result - - ih = arith.AddIOp( - hbase, arith.MulIOp(h_inc, dh_const).result - ).result - iw = arith.AddIOp( - wbase, arith.MulIOp(w_inc, dw_const).result - ).result - - flat_idx = arith.AddIOp( - arith.MulIOp(ih, in_w_const).result, iw - ).result - memref.StoreOp(flat_idx, indices_memref.result, [n, c, oh, ow]) - scf.YieldOp([]) - scf.YieldOp([]) - scf.YieldOp([]) - scf.YieldOp([]) - - indices_type = ir.RankedTensorType.get(shape, i64) - return bufferization.ToTensorOp( - indices_type, indices_memref.result, restrict=True - ).result - - -def low_memory_max_pool_with_offsets_op( - node: LowMemoryMaxPoolWithOffsetsOp, - symbol_table: dict[tuple[str, int], ir.Operation], -): - input_tensor = symbol_table.get((str(node.args[0]), 0)) - if input_tensor is None: - return - - input_shape = list(ir.RankedTensorType(input_tensor.type).shape) - if any(dim < 0 for dim in input_shape): - raise NotImplementedError( - "_low_memory_max_pool_with_offsets requires static shapes" - ) - - dim = len(input_shape) - 2 - if dim not in (2, 3): - raise NotImplementedError( - "_low_memory_max_pool_with_offsets supports only 2d/3d" - ) - - kernel_size = _normalize_pool_param(node.args[1], dim, node.args[1]) - stride = _normalize_pool_param( - node.args[2] if len(node.args) > 2 else None, - dim, - kernel_size, - ) - padding = _normalize_pool_param( - node.args[3] if len(node.args) > 3 else None, - dim, - [0] * dim, - ) - dilation = _normalize_pool_param( - node.args[4] if len(node.args) > 4 else None, - dim, - [1] * dim, - ) - - if dim == 2: - pool_node = MaxPool2dWithIndicesOp() - pool_node._arguments = list(node.args) - values, indices = max_pool2d_with_indices_op(pool_node, symbol_table) - offsets = _max_pool_indices_to_offsets_2d( - indices, - _pool2d_input_hw(input_shape), - kernel_size, - stride, - padding, - dilation, - ) - return values, offsets - - pool_node = MaxPool3dOp() - pool_node._arguments = list(node.args) - values, indices = max_pool3d_op(pool_node, symbol_table) - offsets = _max_pool_indices_to_offsets_3d( - indices, - input_shape[-3:], - kernel_size, - stride, - padding, - dilation, - ) - return values, offsets - - -def max_pool2d_with_indices_op( - node: MaxPool2dWithIndicesOp, - symbol_table: dict[tuple[str, int], ir.Operation], -): - """ - Import the max_pool2d_with_indices operation. - From buddy MaxPool2dWithIndicesOp to MLIR operations using scf.for loops. - aten.max_pool2d_with_indices(input, kernel_size, stride, padding, - dilation, ceil_mode) -> (Tensor, Tensor) - - Returns both the max-pooled values and the indices of the max values. - Uses scf.for loops since TOSA max_pool2d doesn't support returning indices. - """ - input_tensor = symbol_table.get((str(node.args[0]), 0)) - kernel_size = node.args[1] - stride = node.args[2] if len(node.args) > 2 and node.args[2] else kernel_size - padding = node.args[3] if len(node.args) > 3 else [0, 0] - dilation = node.args[4] if len(node.args) > 4 else [1, 1] - ceil_mode = node.args[5] if len(node.args) > 5 else False - - ssa_shape = [int(x) for x in ir.RankedTensorType(input_tensor.type).shape] - src = node.args[0] if node.args else None - meta = None - if ( - src is not None - and hasattr(src, "tensor_meta") - and src.tensor_meta - and "shape" in src.tensor_meta - ): - meta = [int(x) for x in src.tensor_meta["shape"]] - if meta is not None: - if len(meta) != 4: - raise RuntimeError( - f"max_pool2d_with_indices requires 4D input meta: {meta}" - ) - N, C, H, W = meta - nhwc = ssa_shape == [N, H, W, C] - if not nhwc and ssa_shape != [N, C, H, W]: - raise RuntimeError( - f"max_pool2d_with_indices layout mismatch: ssa={ssa_shape} meta_nchw={meta}" - ) - else: - if len(ssa_shape) != 4: - raise RuntimeError( - f"max_pool2d_with_indices requires 4D input, got {ssa_shape}" - ) - n, a, b, c = ssa_shape - if a == b and c > a: - N, H, W, C = n, a, b, c - nhwc = True - elif b == c and a > b: - N, C, H, W = n, a, b, c - nhwc = False - else: - raise RuntimeError( - f"max_pool2d_with_indices cannot infer layout from ssa={ssa_shape}" - ) - - input_dtype = ir.RankedTensorType(input_tensor.type).element_type - - # Normalize kernel_size, stride, padding, dilation - if isinstance(kernel_size, int): - kh, kw = kernel_size, kernel_size - else: - kh = kernel_size[0] - kw = kernel_size[1] if len(kernel_size) > 1 else kernel_size[0] - - if isinstance(stride, int): - sh, sw = stride, stride - else: - sh = stride[0] - sw = stride[1] if len(stride) > 1 else stride[0] - - if isinstance(padding, int): - ph, pw = padding, padding - else: - ph = padding[0] - pw = padding[1] if len(padding) > 1 else padding[0] - - if isinstance(dilation, int): - dh, dw = dilation, dilation - else: - dh = dilation[0] - dw = dilation[1] if len(dilation) > 1 else dilation[0] - - # Calculate output dimensions - if ceil_mode: - out_h = (H + 2 * ph - dh * (kh - 1) - 1 + sh - 1) // sh + 1 - out_w = (W + 2 * pw - dw * (kw - 1) - 1 + sw - 1) // sw + 1 - else: - out_h = (H + 2 * ph - dh * (kh - 1) - 1) // sh + 1 - out_w = (W + 2 * pw - dw * (kw - 1) - 1) // sw + 1 - - output_shape = [N, out_h, out_w, C] if nhwc else [N, C, out_h, out_w] - output_type = ir.RankedTensorType.get(output_shape, input_dtype) - indices_dtype = ir.IntegerType.get_signless(64) - indices_type = ir.RankedTensorType.get(output_shape, indices_dtype) - - # Create memrefs for output and indices - output_memref_type = ir.MemRefType.get(output_shape, input_dtype) - indices_memref_type = ir.MemRefType.get(output_shape, indices_dtype) - - output_memref = memref.AllocOp(output_memref_type, [], []) - indices_memref = memref.AllocOp(indices_memref_type, [], []) - - # Initialize output with negative infinity - neg_inf = arith.ConstantOp( - input_dtype, ir.FloatAttr.get(input_dtype, float("-inf")) - ) - linalg.fill(neg_inf.result, outs=[output_memref.result]) - - # Initialize indices with zeros - zero_idx = arith.ConstantOp(indices_dtype, ir.IntegerAttr.get(indices_dtype, 0)) - linalg.fill(zero_idx.result, outs=[indices_memref.result]) - - # Convert input to memref - input_memref = bufferization.ToBufferOp( - ir.MemRefType.get(ssa_shape, input_dtype), input_tensor - ).result - - # Create constants - zero = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), 0) - ) - one = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), 1) - ) - n_bound = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), N) - ) - c_bound = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), C) - ) - out_h_bound = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), out_h) - ) - out_w_bound = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), out_w) - ) - kh_bound = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), kh) - ) - kw_bound = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), kw) - ) - - # Constants for calculations - sh_const = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), sh) - ) - sw_const = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), sw) - ) - ph_const = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), ph) - ) - pw_const = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), pw) - ) - dh_const = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), dh) - ) - dw_const = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), dw) - ) - h_const = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), H) - ) - w_const = arith.ConstantOp( - ir.IndexType.get(), ir.IntegerAttr.get(ir.IndexType.get(), W) - ) - w_i64 = arith.ConstantOp(indices_dtype, ir.IntegerAttr.get(indices_dtype, W)) - - # Six nested loops: n, c, oh, ow, kh, kw - n_loop = scf.ForOp(zero.result, n_bound.result, one.result) - with ir.InsertionPoint(n_loop.body): - n = n_loop.induction_variable - - c_loop = scf.ForOp(zero.result, c_bound.result, one.result) - with ir.InsertionPoint(c_loop.body): - c = c_loop.induction_variable - - oh_loop = scf.ForOp(zero.result, out_h_bound.result, one.result) - with ir.InsertionPoint(oh_loop.body): - oh = oh_loop.induction_variable - - ow_loop = scf.ForOp(zero.result, out_w_bound.result, one.result) - with ir.InsertionPoint(ow_loop.body): - ow = ow_loop.induction_variable - - # Base input position: oh * stride - padding - h_base = arith.MulIOp(oh, sh_const.result).result - h_base = arith.SubIOp(h_base, ph_const.result).result - w_base = arith.MulIOp(ow, sw_const.result).result - w_base = arith.SubIOp(w_base, pw_const.result).result - - # Inner loops over kernel - kih_loop = scf.ForOp(zero.result, kh_bound.result, one.result) - with ir.InsertionPoint(kih_loop.body): - kih = kih_loop.induction_variable - - kiw_loop = scf.ForOp(zero.result, kw_bound.result, one.result) - with ir.InsertionPoint(kiw_loop.body): - kiw = kiw_loop.induction_variable - - # Calculate input position with dilation - ih = arith.AddIOp( - h_base, - arith.MulIOp(kih, dh_const.result).result, - ).result - iw = arith.AddIOp( - w_base, - arith.MulIOp(kiw, dw_const.result).result, - ).result - - # Check bounds: 0 <= ih < H and 0 <= iw < W - ih_ge_0 = arith.CmpIOp( - arith.CmpIPredicate.sge, ih, zero.result - ).result - ih_lt_h = arith.CmpIOp( - arith.CmpIPredicate.slt, ih, h_const.result - ).result - iw_ge_0 = arith.CmpIOp( - arith.CmpIPredicate.sge, iw, zero.result - ).result - iw_lt_w = arith.CmpIOp( - arith.CmpIPredicate.slt, iw, w_const.result - ).result - - h_valid = arith.AndIOp(ih_ge_0, ih_lt_h).result - w_valid = arith.AndIOp(iw_ge_0, iw_lt_w).result - in_bounds = arith.AndIOp(h_valid, w_valid).result - - # If in bounds, load and compare - if_op = scf.IfOp(in_bounds, has_else=False) - with ir.InsertionPoint(if_op.then_block): - if nhwc: - in_idx = [n, ih, iw, c] - out_idx = [n, oh, ow, c] - else: - in_idx = [n, c, ih, iw] - out_idx = [n, c, oh, ow] - # Load input value - input_val = memref.LoadOp(input_memref, in_idx).result - - # Load current max - current_max = memref.LoadOp( - output_memref.result, out_idx - ).result - - # Check if input > current_max - is_greater = arith.CmpFOp( - arith.CmpFPredicate.OGT, - input_val, - current_max, - ).result - - inner_if = scf.IfOp(is_greater, has_else=False) - with ir.InsertionPoint(inner_if.then_block): - # Update max value - memref.StoreOp( - input_val, - output_memref.result, - out_idx, - ) - - # Calculate flattened index: ih * W + iw - ih_i64 = arith.IndexCastOp(indices_dtype, ih).result - iw_i64 = arith.IndexCastOp(indices_dtype, iw).result - flat_idx = arith.AddIOp( - arith.MulIOp(ih_i64, w_i64.result).result, - iw_i64, - ).result - - # Store index - memref.StoreOp( - flat_idx, - indices_memref.result, - out_idx, - ) - - scf.YieldOp([]) - - scf.YieldOp([]) - - scf.YieldOp([]) - - scf.YieldOp([]) - - scf.YieldOp([]) - - scf.YieldOp([]) - - scf.YieldOp([]) - - scf.YieldOp([]) - - # Convert memrefs back to tensors - output_result = bufferization.ToTensorOp( - output_type, output_memref.result, restrict=True - ) - indices_result = bufferization.ToTensorOp( - indices_type, indices_memref.result, restrict=True - ) - - return output_result, indices_result - - -def _quantize_input(activation, scale, nchw_to_nhwc=False): - input_shape = [int(x) for x in ir.RankedTensorType(activation.type).shape] - f32 = ir.F32Type.get() - i8 = ir.IntegerType.get_signless(8) - if ir.RankedTensorType(activation.type).element_type != f32: - raise ValueError("Quantized input quantization requires FP32") - scale = float(scale) - if not numpy.isfinite(scale) or scale <= 0.0: - raise ValueError( - "Quantized input quantization scale must be finite and positive" - ) - if nchw_to_nhwc: - if len(input_shape) != 4: - raise ValueError("NCHW-to-NHWC quantization requires rank 4") - n, c, h, w = input_shape - output_shape = [n, h, w, c] - dims = [ir.AffineExpr.get_dim(i) for i in range(4)] - input_map = ir.AffineMap.get(4, 0, [dims[0], dims[3], dims[1], dims[2]]) - output_map = ir.AffineMap.get_identity(4) - else: - output_shape = input_shape - input_map = ir.AffineMap.get_identity(len(input_shape)) - output_map = input_map - out_type = ir.RankedTensorType.get(output_shape, i8) - output = tensor.EmptyOp(output_shape, i8) - op = linalg.GenericOp( - [out_type], - [activation], - [output], - ir.ArrayAttr.get( - [ir.AffineMapAttr.get(input_map), ir.AffineMapAttr.get(output_map)] - ), - ir.ArrayAttr.get( - [ir.Attribute.parse("#linalg.iterator_type")] * len(output_shape) - ), - ) - op.operation.attributes["buckyball.quant_f32_to_i8"] = ir.BoolAttr.get(True) - op.operation.attributes["nchw_to_nhwc"] = ir.BoolAttr.get(nchw_to_nhwc) - op.operation.attributes["scale"] = ir.FloatAttr.get(f32, 1.0 / scale) - block = ir.Block.create_at_start(op.region, [f32, i8]) - block.append(linalg.YieldOp([block.arguments[1]])) - return op.result - - -def quantized_conv2d_op(node, symbol_table): - activation = symbol_table.get((str(node.args[0]), 0)) - weight = symbol_table.get((str(node.args[1]), 0)) - if activation is None or weight is None: - raise ValueError(f"missing Quantized Conv2D operand for {node.name}") - - f32 = ir.F32Type.get() - i8 = ir.IntegerType.get_signless(8) - i32 = ir.IntegerType.get_signless(32) - activation_type = ir.RankedTensorType(activation.type) - input_shape = [int(x) for x in activation_type.shape] - n, cin, h, w = node._input_shape - if activation_type.element_type == f32: - if input_shape == [n, cin, h, w]: - nchw_to_nhwc = True - elif input_shape == [n, h, w, cin]: - nchw_to_nhwc = False - else: - raise ValueError( - f"FP32 Quantized Conv2D input layout mismatch for {node.name}: " - f"expected {[n, cin, h, w]} or {[n, h, w, cin]}, got {input_shape}" - ) - activation = _quantize_input( - activation, node._input_scale, nchw_to_nhwc=nchw_to_nhwc - ) - elif activation_type.element_type == i8: - if input_shape != [n, h, w, cin]: - raise ValueError( - f"INT8 Quantized Conv2D input layout mismatch for {node.name}: " - f"expected NHWC {[n, h, w, cin]}, got {input_shape}" - ) - else: - raise ValueError("Quantized Conv2D activation must be FP32 or INT8") - - depthwise = isinstance(node, QuantizedDepthwiseConv2dOp) - weight_type = ir.RankedTensorType(weight.type) - cout, weight_cin, kh, kw = node._weight_shape - actual_weight_shape = [int(x) for x in weight_type.shape] - if actual_weight_shape != list(node._weight_shape): - raise ValueError(f"Quantized convolution weight shape mismatch for {node.name}") - if ( - weight_type.element_type != i8 - or (depthwise and (weight_cin != 1 or cin != cout)) - or (not depthwise and cin != weight_cin) - ): - raise ValueError("Quantized Conv2D requires INT8 OIHW weight with matching Cin") - - out_n, out_c, out_h, out_w = node._output_shape - if out_n != n or out_c != cout: - raise ValueError(f"Quantized Conv2D output shape mismatch for {node.name}") - scale_values = ( - node._dequant_scale - if node._final_output and node._activation != 2 - else node._requant_scale - ) - if len(node._bias_i32) != cout or len(scale_values) != cout: - raise ValueError(f"Quantized Conv2D quant parameter mismatch for {node.name}") - bias_type = ir.RankedTensorType.get([cout], i32) - bias_attr = ir.DenseElementsAttr.get( - numpy.asarray(node._bias_i32, dtype=numpy.int32), type=bias_type - ) - bias = arith.ConstantOp(bias_type, bias_attr).result - scale_type = ir.RankedTensorType.get([cout], f32) - scale_attr = ir.DenseElementsAttr.get( - numpy.asarray(scale_values, dtype=numpy.float32), type=scale_type - ) - scale = arith.ConstantOp(scale_type, scale_attr).result - lut_type = ir.RankedTensorType.get([len(node._lut_i8)], i8) - lut_attr = ir.DenseElementsAttr.get( - numpy.asarray(node._lut_i8, dtype=numpy.int8), type=lut_type - ) - lut = arith.ConstantOp(lut_type, lut_attr).result - output_shape = ( - [n, cout, out_h, out_w] - if node._final_output and not node._output_nhwc - else [n, out_h, out_w, cout] - ) - output_element = f32 if node._final_output else i8 - out_type = ir.RankedTensorType.get(output_shape, output_element) - output = tensor.EmptyOp(output_shape, output_element) - dims = [ir.AffineExpr.get_dim(i) for i in range(4)] - zero = ir.AffineExpr.get_constant(0) - output_map = ir.AffineMap.get_identity(4) - maps = [ - ir.AffineMap.get(4, 0, [zero, zero, zero, zero]), - ir.AffineMap.get(4, 0, [zero, zero, zero, zero]), - ir.AffineMap.get(4, 0, [zero]), - ir.AffineMap.get(4, 0, [zero]), - ir.AffineMap.get(4, 0, [zero]), - output_map, - ] - op = linalg.GenericOp( - [out_type], - [activation, weight, bias, scale, lut], - [output], - ir.ArrayAttr.get([ir.AffineMapAttr.get(x) for x in maps]), - ir.ArrayAttr.get([ir.Attribute.parse("#linalg.iterator_type")] * 4), - ) - i64 = ir.IntegerType.get_signless(64) - marker = ( - "buckyball.quantized_conv2d_depthwise" - if depthwise - else "buckyball.quantized_conv2d" - ) - op.operation.attributes[marker] = ir.BoolAttr.get(True) - op.operation.attributes["stride"] = ir.IntegerAttr.get(i64, node._stride) - op.operation.attributes["kernel"] = ir.IntegerAttr.get(i64, kh) - op.operation.attributes["pad_low"] = ir.IntegerAttr.get(i64, node._padding) - op.operation.attributes["pad_high"] = ir.IntegerAttr.get(i64, node._padding) - op.operation.attributes["activation"] = ir.IntegerAttr.get(i64, node._activation) - op.operation.attributes["output_scale"] = ir.FloatAttr.get(f32, node._output_scale) - if getattr(node, "_lane_output_scales", []): - op.operation.attributes["lane_output_scales"] = ir.DenseF32ArrayAttr.get( - node._lane_output_scales - ) - op.operation.attributes["final_output"] = ir.BoolAttr.get(node._final_output) - op.operation.attributes["output_nhwc"] = ir.BoolAttr.get(node._output_nhwc) - if node._output_nhwc or not node._final_output: - op.operation.attributes["buddy.layout"] = ir.StringAttr.get("nhwc") - block = ir.Block.create_at_start(op.region, [i8, i8, i32, f32, i8, output_element]) - block.append(linalg.YieldOp([block.arguments[5]])) - return op.result - - -def quantized_matmul_op(node, symbol_table): - activation = symbol_table.get((str(node.args[0]), 0)) - weight = symbol_table.get((str(node.args[1]), 0)) - if activation is None or weight is None: - raise ValueError(f"missing Quantized MatMul operand for {node.name}") - - f32 = ir.F32Type.get() - i8 = ir.IntegerType.get_signless(8) - i32 = ir.IntegerType.get_signless(32) - activation_type = ir.RankedTensorType(activation.type) - if activation_type.element_type == f32: - activation = _quantize_input(activation, node._input_scale, nchw_to_nhwc=False) - activation_type = ir.RankedTensorType(activation.type) - weight_type = ir.RankedTensorType(weight.type) - if ( - activation_type.element_type != i8 - or weight_type.element_type != i8 - or activation_type.rank != 2 - or weight_type.rank != 2 - ): - raise ValueError("Quantized MatMul requires rank-2 INT8 activation and weight") - m, k = [int(x) for x in activation_type.shape] - n, wk = [int(x) for x in weight_type.shape] - scale_values = ( - node._dequant_scale - if node._final_output and node._activation != 2 - else node._requant_scale - ) - if k != wk or len(node._bias_i32) != n or len(scale_values) != n: - raise ValueError(f"Quantized MatMul shape mismatch for {node.name}") - - bias_type = ir.RankedTensorType.get([n], i32) - bias_attr = ir.DenseElementsAttr.get( - numpy.asarray(node._bias_i32, dtype=numpy.int32), type=bias_type - ) - bias = arith.ConstantOp(bias_type, bias_attr).result - scale_type = ir.RankedTensorType.get([n], f32) - scale_attr = ir.DenseElementsAttr.get( - numpy.asarray(scale_values, dtype=numpy.float32), type=scale_type - ) - scale = arith.ConstantOp(scale_type, scale_attr).result - lut_type = ir.RankedTensorType.get([len(node._lut_i8)], i8) - lut_attr = ir.DenseElementsAttr.get( - numpy.asarray(node._lut_i8, dtype=numpy.int8), type=lut_type - ) - lut = arith.ConstantOp(lut_type, lut_attr).result - output_element = f32 if node._final_output else i8 - out_type = ir.RankedTensorType.get([m, n], output_element) - output = tensor.EmptyOp([m, n], output_element) - dims = [ir.AffineExpr.get_dim(i) for i in range(3)] - maps = [ - ir.AffineMap.get(3, 0, [dims[0], dims[2]]), - ir.AffineMap.get(3, 0, [dims[1], dims[2]]), - ir.AffineMap.get(3, 0, [dims[1]]), - ir.AffineMap.get(3, 0, [dims[1]]), - ir.AffineMap.get(3, 0, [ir.AffineExpr.get_constant(0)]), - ir.AffineMap.get(3, 0, [dims[0], dims[1]]), - ] - op = linalg.GenericOp( - [out_type], - [activation, weight, bias, scale, lut], - [output], - ir.ArrayAttr.get([ir.AffineMapAttr.get(x) for x in maps]), - ir.ArrayAttr.get( - [ - ir.Attribute.parse("#linalg.iterator_type"), - ir.Attribute.parse("#linalg.iterator_type"), - ir.Attribute.parse("#linalg.iterator_type"), - ] - ), - ) - op.operation.attributes["buckyball.quantized_matmul"] = ir.BoolAttr.get(True) - op.operation.attributes["activation"] = ir.IntegerAttr.get( - ir.IntegerType.get_signless(64), node._activation - ) - op.operation.attributes["output_scale"] = ir.FloatAttr.get(f32, node._output_scale) - op.operation.attributes["final_output"] = ir.BoolAttr.get(node._final_output) - block = ir.Block.create_at_start(op.region, [i8, i8, i32, f32, i8, output_element]) - block.append(linalg.YieldOp([block.arguments[5]])) - return op.result - - -def quantized_max_pool2d_op(node, symbol_table): - input_value = symbol_table.get((str(node.args[0]), 0)) - if input_value is None: - raise ValueError(f"missing Quantized MaxPool2D input for {node.name}") - i8 = ir.IntegerType.get_signless(8) - input_type = ir.RankedTensorType(input_value.type) - if input_type.element_type != i8 or input_type.rank != 4: - raise ValueError("Quantized MaxPool2D input must be rank-4 INT8 NHWC") - n, h, w, c = [int(x) for x in input_type.shape] - out_n, out_c, out_h, out_w = node._output_shape - if ( - [n, c] != [out_n, out_c] - or node._kernel <= 0 - or node._stride <= 0 - or node._padding < 0 - or (h + 2 * node._padding - node._kernel) // node._stride + 1 != out_h - or (w + 2 * node._padding - node._kernel) // node._stride + 1 != out_w - ): - raise ValueError(f"Quantized MaxPool2D geometry mismatch for {node.name}") - output_shape = ( - [out_n, out_c, out_h, out_w] - if node._final_output - else [out_n, out_h, out_w, out_c] - ) - output_type = ir.RankedTensorType.get(output_shape, i8) - output = tensor.EmptyOp(output_shape, i8) - # This is a marker consumed by Tile lowering, like Quantized Conv. Iterate - # over the output; input-sized loops overrun odd-sized pooling results. - zero = ir.AffineExpr.get_constant(0) - input_map = ir.AffineMap.get(4, 0, [zero, zero, zero, zero]) - output_map = ir.AffineMap.get_identity(4) - op = linalg.GenericOp( - [output_type], - [input_value], - [output], - ir.ArrayAttr.get( - [ir.AffineMapAttr.get(input_map), ir.AffineMapAttr.get(output_map)] - ), - ir.ArrayAttr.get([ir.Attribute.parse("#linalg.iterator_type")] * 4), - ) - i64 = ir.IntegerType.get_signless(64) - op.operation.attributes["buckyball.quantized_max_pool2d"] = ir.BoolAttr.get(True) - op.operation.attributes["kernel"] = ir.IntegerAttr.get(i64, node._kernel) - op.operation.attributes["stride"] = ir.IntegerAttr.get(i64, node._stride) - op.operation.attributes["padding"] = ir.IntegerAttr.get(i64, node._padding) - op.operation.attributes["final_output"] = ir.BoolAttr.get(node._final_output) - block = ir.Block.create_at_start(op.region, [i8, i8]) - block.append(linalg.YieldOp([block.arguments[1]])) - return op.result - - -def quantized_global_avg_pool_op(node, symbol_table): - input_value = symbol_table.get((str(node.args[0]), 0)) - if input_value is None: - raise ValueError(f"missing Quantized global-average input for {node.name}") - i8 = ir.IntegerType.get_signless(8) - input_type = ir.RankedTensorType(input_value.type) - if input_type.element_type != i8 or input_type.rank != 4: - raise ValueError("Quantized global-average input must be rank-4 INT8 NHWC") - n, h, w, c = [int(x) for x in input_type.shape] - if min(n, h, w, c) <= 0: - raise ValueError("Quantized global-average shape must be positive") - output_type = ir.RankedTensorType.get([n, 1, 1, c], i8) - output = tensor.EmptyOp([n, 1, 1, c], i8) - dims = [ir.AffineExpr.get_dim(i) for i in range(4)] - zero = ir.AffineExpr.get_constant(0) - op = linalg.GenericOp( - [output_type], - [input_value], - [output], - ir.ArrayAttr.get( - [ - ir.AffineMapAttr.get(ir.AffineMap.get_identity(4)), - ir.AffineMapAttr.get( - ir.AffineMap.get(4, 0, [dims[0], zero, zero, dims[3]]) - ), - ] - ), - ir.ArrayAttr.get( - [ - ir.Attribute.parse("#linalg.iterator_type"), - ir.Attribute.parse("#linalg.iterator_type"), - ir.Attribute.parse("#linalg.iterator_type"), - ir.Attribute.parse("#linalg.iterator_type"), - ] - ), - ) - op.operation.attributes["buckyball.quantized_global_avg_pool"] = ir.BoolAttr.get( - True - ) - f32 = ir.F32Type.get() - op.operation.attributes["input_scale"] = ir.FloatAttr.get(f32, node._input_scale) - op.operation.attributes["output_scale"] = ir.FloatAttr.get(f32, node._output_scale) - block = ir.Block.create_at_start(op.region, [i8, i8]) - block.append(linalg.YieldOp([block.arguments[1]])) - return op.result - - -def quantized_elementwise_op(node, symbol_table, kind): - lhs = symbol_table.get((str(node.args[0]), 0)) - rhs = symbol_table.get((str(node.args[1]), 0)) - if lhs is None or rhs is None: - raise ValueError(f"missing Quantized INT8 {kind} input for {node.name}") - i8 = ir.IntegerType.get_signless(8) - lhs_type = ir.RankedTensorType(lhs.type) - rhs_type = ir.RankedTensorType(rhs.type) - if lhs_type.element_type != i8 or rhs_type.element_type != i8: - raise ValueError( - f"Quantized INT8 {kind} operands must be INT8 for {node.name}: " - f"{node.args[0]}={lhs_type}, {node.args[1]}={rhs_type}" - ) - lhs_shape = [int(x) for x in lhs_type.shape] - rhs_shape = [int(x) for x in rhs_type.shape] - if len(lhs_shape) != 4 or len(rhs_shape) != 4: - raise ValueError(f"Quantized INT8 {kind} operands must be rank 4") - output_shape = [max(a, b) for a, b in zip(lhs_shape, rhs_shape)] - if any( - a not in (1, out) or b not in (1, out) - for a, b, out in zip(lhs_shape, rhs_shape, output_shape) - ): - raise ValueError(f"Quantized INT8 {kind} operands are not broadcast-compatible") - if kind == "add" and lhs_shape != rhs_shape: - raise ValueError("Quantized residual add requires equal shapes") - output_type = ir.RankedTensorType.get(output_shape, i8) - output = tensor.EmptyOp(output_shape, i8) - dims = [ir.AffineExpr.get_dim(i) for i in range(4)] - zero = ir.AffineExpr.get_constant(0) - lhs_map = ir.AffineMap.get( - 4, - 0, - [zero if size == 1 else dims[i] for i, size in enumerate(lhs_shape)], - ) - rhs_map = ir.AffineMap.get( - 4, - 0, - [zero if size == 1 else dims[i] for i, size in enumerate(rhs_shape)], - ) - op = linalg.GenericOp( - [output_type], - [lhs, rhs], - [output], - ir.ArrayAttr.get( - [ - ir.AffineMapAttr.get(lhs_map), - ir.AffineMapAttr.get(rhs_map), - ir.AffineMapAttr.get(ir.AffineMap.get_identity(4)), - ] - ), - ir.ArrayAttr.get([ir.Attribute.parse("#linalg.iterator_type")] * 4), - ) - op.operation.attributes[f"buckyball.quantized_int8_{kind}"] = ir.BoolAttr.get(True) - f32 = ir.F32Type.get() - op.operation.attributes["lhs_scale"] = ir.FloatAttr.get(f32, node._lhs_scale) - op.operation.attributes["rhs_scale"] = ir.FloatAttr.get(f32, node._rhs_scale) - op.operation.attributes["output_scale"] = ir.FloatAttr.get(f32, node._output_scale) - op.operation.attributes["activation"] = ir.IntegerAttr.get( - ir.IntegerType.get_signless(64), node._activation - ) - block = ir.Block.create_at_start(op.region, [i8, i8, i8]) - block.append(linalg.YieldOp([block.arguments[2]])) - return op.result - - -def quantized_channel_slice_op(node, symbol_table): - source = symbol_table.get((str(node.args[0]), 0)) - if source is None: - raise ValueError(f"missing Quantized channel-slice input for {node.name}") - i8 = ir.IntegerType.get_signless(8) - source_type = ir.RankedTensorType(source.type) - if source_type.element_type != i8 or len(source_type.shape) != 4: - raise ValueError(f"Quantized channel-slice requires rank-4 INT8 at {node.name}") - n, c, h, w = node._output_shape - output_shape = [n, h, w, c] - output_type = ir.RankedTensorType.get(output_shape, i8) - output = tensor.EmptyOp(output_shape, i8) - source_slice = tensor.ExtractSliceOp( - output_type, - source, - [], - [], - [], - ir._denseI64ArrayAttr([0, 0, 0, node._offset], None), - ir._denseI64ArrayAttr(output_shape, None), - ir._denseI64ArrayAttr([1, 1, 1, 1], None), - ) - op = linalg.GenericOp( - [output_type], - [source_slice], - [output], - ir.ArrayAttr.get( - [ - ir.AffineMapAttr.get(ir.AffineMap.get_identity(4)), - ir.AffineMapAttr.get(ir.AffineMap.get_identity(4)), - ] - ), - ir.ArrayAttr.get([ir.Attribute.parse("#linalg.iterator_type")] * 4), - ) - op.operation.attributes["buckyball.quantized_channel_slice"] = ir.BoolAttr.get(True) - op.operation.attributes["offset"] = ir.IntegerAttr.get( - ir.IntegerType.get_signless(64), 0 - ) - block = ir.Block.create_at_start(op.region, [i8, i8]) - block.append(linalg.YieldOp([block.arguments[0]])) - return op.result - - -def quantized_channel_concat_op(node, symbol_table): - inputs = [symbol_table.get((str(name), 0)) for name in node.args] - if not inputs or any(value is None for value in inputs): - raise ValueError(f"missing Quantized channel-concat input for {node.name}") - i8 = ir.IntegerType.get_signless(8) - input_shapes = [list(ir.RankedTensorType(value.type).shape) for value in inputs] - if any( - ir.RankedTensorType(value.type).element_type != i8 or len(shape) != 4 - for value, shape in zip(inputs, input_shapes) - ): - raise ValueError( - f"Quantized channel-concat requires rank-4 INT8 at {node.name}" - ) - n, c, h, w = node._output_shape - output_shape = [n, h, w, c] - if ( - any(shape[:3] != [n, h, w] for shape in input_shapes) - or sum(shape[3] for shape in input_shapes) != c - ): - raise ValueError(f"Quantized channel-concat shape mismatch at {node.name}") - output_type = ir.RankedTensorType.get(output_shape, i8) - output = tensor.EmptyOp(output_shape, i8) - dims = [ir.AffineExpr.get_dim(i) for i in range(4)] - zero = ir.AffineExpr.get_constant(0) - maps = [ - ir.AffineMapAttr.get(ir.AffineMap.get(4, 0, [dims[0], dims[1], dims[2], zero])) - for _ in inputs - ] - maps.append(ir.AffineMapAttr.get(ir.AffineMap.get_identity(4))) - op = linalg.GenericOp( - [output_type], - inputs, - [output], - ir.ArrayAttr.get(maps), - ir.ArrayAttr.get([ir.Attribute.parse("#linalg.iterator_type")] * 4), - ) - op.operation.attributes["buckyball.quantized_channel_concat"] = ir.BoolAttr.get( - True - ) - op.operation.attributes["segments"] = ir.DenseI64ArrayAttr.get( - [shape[3] for shape in input_shapes] - ) - block = ir.Block.create_at_start(op.region, [i8] * (len(inputs) + 1)) - block.append(linalg.YieldOp([block.arguments[-1]])) - return op.result - - -def quantized_resize_nearest_op(node, symbol_table): - source = symbol_table.get((str(node.args[0]), 0)) - if source is None: - raise ValueError(f"missing Quantized resize input for {node.name}") - i8 = ir.IntegerType.get_signless(8) - source_type = ir.RankedTensorType(source.type) - if source_type.element_type != i8 or len(source_type.shape) != 4: - raise ValueError(f"Quantized resize requires rank-4 INT8 at {node.name}") - n, c, h, w = node._output_shape - output_shape = [n, h, w, c] - output_type = ir.RankedTensorType.get(output_shape, i8) - output = tensor.EmptyOp(output_shape, i8) - dims = [ir.AffineExpr.get_dim(i) for i in range(4)] - zero = ir.AffineExpr.get_constant(0) - op = linalg.GenericOp( - [output_type], - [source], - [output], - ir.ArrayAttr.get( - [ - ir.AffineMapAttr.get( - ir.AffineMap.get(4, 0, [dims[0], zero, zero, dims[3]]) - ), - ir.AffineMapAttr.get(ir.AffineMap.get_identity(4)), - ] - ), - ir.ArrayAttr.get([ir.Attribute.parse("#linalg.iterator_type")] * 4), - ) - op.operation.attributes["buckyball.quantized_resize_nearest"] = ir.BoolAttr.get( - True - ) - op.operation.attributes["scale_h"] = ir.IntegerAttr.get( - ir.IntegerType.get_signless(64), node._scale_h - ) - op.operation.attributes["scale_w"] = ir.IntegerAttr.get( - ir.IntegerType.get_signless(64), node._scale_w - ) - block = ir.Block.create_at_start(op.region, [i8, i8]) - block.append(linalg.YieldOp([block.arguments[0]])) - return op.result - - -def quantized(lower): - def emit(node, symbols): - result = lower(node, symbols) - if node.quantization_trace_meta is not None: - from types import SimpleNamespace - - result.owner.attributes["trace_node"] = ir.DictAttr.get( - _trace_attrs(SimpleNamespace(trace_meta=node.quantization_trace_meta)) - ) - return result - - return emit - - -from buddy.compiler.ops.linalg import ops_registry as upstream_registry - -ops_registry = { - **upstream_registry, - "QuantizedMulOp": quantized( - lambda node, symbols: quantized_elementwise_op(node, symbols, "mul") - ), - "QuantizedAddOp": quantized( - lambda node, symbols: quantized_elementwise_op(node, symbols, "add") - ), - "QuantizedConv2dOp": quantized(quantized_conv2d_op), - "QuantizedDepthwiseConv2dOp": quantized(quantized_conv2d_op), - "QuantizedMatmulOp": quantized(quantized_matmul_op), - "QuantizedMaxPool2dOp": quantized(quantized_max_pool2d_op), - "QuantizedGlobalAvgPoolOp": quantized(quantized_global_avg_pool_op), - "QuantizedChannelSliceOp": quantized(quantized_channel_slice_op), - "QuantizedChannelConcatOp": quantized(quantized_channel_concat_op), - "QuantizedResizeNearestOp": quantized(quantized_resize_nearest_op), - "CatOp": cat_op, - "SplitOp": split_op, - "MaxPool2dWithIndicesOp": max_pool2d_with_indices_op, - "LowMemoryMaxPoolWithOffsetsOp": low_memory_max_pool_with_offsets_op, -} diff --git a/waste/stack/compiler/frontend/ops/native_mxfp8.py b/waste/stack/compiler/frontend/ops/native_mxfp8.py deleted file mode 100644 index 146a2eccb..000000000 --- a/waste/stack/compiler/frontend/ops/native_mxfp8.py +++ /dev/null @@ -1,187 +0,0 @@ -"""Portable MXFP8 codecs: E4M3FN values and one E8M0 scale per 32 values.""" - -from buddy_mlir import ir -from buddy_mlir.dialects import arith, linalg, math, tensor - -from stack.compiler.frontend.ops.reductions import _generic - - -def _constant(dtype, value): - return arith.ConstantOp(dtype, value).result - - -def _power(scale): - i32, f32 = (ir.IntegerType.get_signless(32), ir.F32Type.get()) - exponent = arith.ExtUIOp(i32, scale).result - bits = arith.ShLIOp(exponent, _constant(i32, 23)).result - normal = arith.BitcastOp(f32, bits).result - return arith.SelectOp( - arith.CmpIOp(arith.CmpIPredicate.eq, exponent, _constant(i32, 0)).result, - _constant(f32, 2.0 ** (-127)), - normal, - ).result - - -def _decode(code, scale): - i32, f32 = (ir.IntegerType.get_signless(32), ir.F32Type.get()) - value = arith.ExtUIOp(i32, code).result - sign = arith.AndIOp(value, _constant(i32, 128)).result - fraction = arith.AndIOp(value, _constant(i32, 7)).result - exponent = arith.AndIOp( - arith.ShRUIOp(value, _constant(i32, 3)).result, _constant(i32, 15) - ).result - bits = arith.OrIOp( - arith.ShLIOp( - arith.AddIOp(exponent, _constant(i32, 120)).result, - _constant(i32, 23), - ).result, - arith.ShLIOp(fraction, _constant(i32, 20)).result, - ).result - normal = arith.BitcastOp(f32, bits).result - subnormal = arith.MulFOp( - arith.UIToFPOp(f32, fraction).result, _constant(f32, 1.0 / 512) - ).result - magnitude = arith.SelectOp( - arith.CmpIOp(arith.CmpIPredicate.eq, exponent, _constant(i32, 0)).result, - subnormal, - normal, - ).result - signed = arith.SelectOp( - arith.CmpIOp(arith.CmpIPredicate.ne, sign, _constant(i32, 0)).result, - arith.NegFOp(magnitude).result, - magnitude, - ).result - return arith.MulFOp(signed, _power(scale)).result - - -def _encode(value, scale): - i8, i32, f32 = ( - ir.IntegerType.get_signless(8), - ir.IntegerType.get_signless(32), - ir.F32Type.get(), - ) - scaled = arith.DivFOp(value, _power(scale)).result - magnitude = arith.MinimumFOp( - math.AbsFOp(scaled).result, _constant(f32, 448.0) - ).result - bits = arith.BitcastOp(i32, magnitude).result - exponent = arith.SubIOp( - arith.ShRUIOp(bits, _constant(i32, 23)).result, _constant(i32, 120) - ).result - fraction = arith.AndIOp(bits, _constant(i32, 8388607)).result - high = arith.ShRUIOp(fraction, _constant(i32, 20)).result - low = arith.AndIOp(fraction, _constant(i32, 1048575)).result - increment = arith.OrIOp( - arith.CmpIOp(arith.CmpIPredicate.ugt, low, _constant(i32, 524288)).result, - arith.AndIOp( - arith.CmpIOp(arith.CmpIPredicate.eq, low, _constant(i32, 524288)).result, - arith.CmpIOp( - arith.CmpIPredicate.ne, - arith.AndIOp(high, _constant(i32, 1)).result, - _constant(i32, 0), - ).result, - ).result, - ).result - rounded = arith.AddIOp(high, arith.ExtUIOp(i32, increment).result).result - normal = arith.AddIOp( - arith.MulIOp(exponent, _constant(i32, 8)).result, rounded - ).result - subnormal = arith.FPToSIOp( - i32, - math.RoundEvenOp(arith.MulFOp(magnitude, _constant(f32, 512.0)).result).result, - ).result - code = arith.SelectOp( - arith.CmpIOp(arith.CmpIPredicate.sle, exponent, _constant(i32, 0)).result, - subnormal, - normal, - ).result - sign = arith.AndIOp( - arith.ShRUIOp(arith.BitcastOp(i32, value).result, _constant(i32, 24)).result, - _constant(i32, 128), - ).result - return arith.TruncIOp(i8, arith.OrIOp(code, sign).result).result - - -def codec(node, symbols): - values, scales = [symbols[name, 0] for name in node.args] - shape = list(ir.RankedTensorType(values.type).shape) - rank = len(shape) - scale_shape = list(ir.RankedTensorType(scales.type).shape) - dims = [ir.AffineExpr.get_dim(i) for i in range(rank)] - scale_dims = [ - ir.AffineExpr.get_constant(0) if size == 1 else dim - for dim, size in zip(dims[:-1], scale_shape[:-1], strict=True) - ] - scale_dims.append( - ir.AffineExpr.get_floor_div(dims[-1], ir.AffineExpr.get_constant(32)) - ) - identity = ir.AffineMap.get_identity(rank) - scale_map = ir.AffineMap.get(rank, 0, scale_dims) - encoding = type(node).__name__ == "NativeMXFP8EncodeOp" - dtype = ir.IntegerType.get_signless(8) if encoding else ir.F32Type.get() - output = tensor.EmptyOp(shape, dtype).result - operation = _generic( - [values, scales], - output, - [identity, scale_map, identity], - ["parallel"] * rank, - ) - operation.attributes["buddy.mxfp8"] = ir.StringAttr.get( - "encode" if encoding else "decode" - ) - body = ir.Block.create_at_start( - operation.region, - [ - ir.RankedTensorType(values.type).element_type, - ir.IntegerType.get_signless(8), - dtype, - ], - ) - with ir.InsertionPoint(body): - result = (_encode if encoding else _decode)(*body.arguments[:2]) - linalg.YieldOp([result]) - return operation.result - - -def concat(node, symbols): - """LLM tensors retain logical axes, including singleton head dimensions.""" - from buddy_mlir.dialects import tosa - - values = [symbols[(name, 0)] for name in node.args[0]] - axis = node.args[1] if len(node.args) == 2 else 0 - rank = ir.RankedTensorType(values[0].type).rank - return tosa.ConcatOp(values, axis % rank).result - - -def matmul(node, symbols): - """Decode E4M3 operands inside the reduction without expanding weight storage.""" - from stack.compiler.frontend.ops.reductions import _cast, _filled - - activation, activation_scales, weights, weight_scales = [ - symbols[(name, 0)] for name in node.args - ] - m, n, k = [ir.AffineExpr.get_dim(i) for i in range(3)] - block = ir.AffineExpr.get_floor_div(k, ir.AffineExpr.get_constant(32)) - zero = ir.AffineExpr.get_constant(0) - maps = [ - ir.AffineMap.get(3, 0, dims) - for dims in ([m, k], [zero, block], [n, k], [n, block], [m, n]) - ] - f64, i8 = ir.F64Type.get(), ir.IntegerType.get_signless(8) - output = _filled(list(node.tensor_meta["shape"]), f64, 0.0) - operation = _generic( - [activation, activation_scales, weights, weight_scales], - output, - maps, - ["parallel", "parallel", "reduction"], - ) - operation.attributes["buddy.mxfp8"] = ir.StringAttr.get("matmul") - body = ir.Block.create_at_start(operation.region, [i8, i8, i8, i8, f64]) - with ir.InsertionPoint(body): - a, a_scale, w, w_scale, accumulator = body.arguments - product = arith.MulFOp( - arith.ExtFOp(f64, _decode(a, a_scale)).result, - arith.ExtFOp(f64, _decode(w, w_scale)).result, - ).result - linalg.YieldOp([arith.AddFOp(accumulator, product).result]) - return _cast(operation.result, ir.F32Type.get()) diff --git a/waste/stack/compiler/frontend/ops/reductions.py b/waste/stack/compiler/frontend/ops/reductions.py deleted file mode 100644 index d7ad7460a..000000000 --- a/waste/stack/compiler/frontend/ops/reductions.py +++ /dev/null @@ -1,308 +0,0 @@ -"""Stable floating reductions between quantized integer contractions.""" - -import math as scalar_math - -from buddy_mlir import ir -from buddy_mlir.dialects import arith, linalg, math, tensor - - -def _filled(shape, dtype, value): - empty = tensor.EmptyOp(shape, dtype) - scalar = arith.ConstantOp(dtype, value).result - result = linalg.FillOp([empty.result.type], [scalar], [empty.result]) - linalg.fill_builtin_region(result.operation) - return result.result - - -def _generic(inputs, output, maps, iterators): - return linalg.GenericOp( - [output.type], - inputs, - [output], - ir.ArrayAttr.get([ir.AffineMapAttr.get(m) for m in maps]), - ir.ArrayAttr.get( - [ - ir.Attribute.parse("#linalg.iterator_type<" + item + ">") - for item in iterators - ] - ), - ) - - -def _cast(value, dtype): - ty = ir.RankedTensorType(value.type) - identity = ir.AffineMap.get_identity(ty.rank) - empty = tensor.EmptyOp(ty.shape, dtype).result - op = _generic([value], empty, [identity, identity], ["parallel"] * ty.rank) - body = ir.Block.create_at_start(op.region, [ty.element_type, dtype]) - with ir.InsertionPoint(body): - converted = ( - arith.ExtFOp(dtype, body.arguments[0]).result - if ir.FloatType(dtype).width > ir.FloatType(ty.element_type).width - else arith.TruncFOp(dtype, body.arguments[0]).result - ) - linalg.YieldOp([converted]) - return op.result - - -def _sum_f64(value, axes, keepdim=True): - ty = ir.RankedTensorType(value.type) - shape = list(ty.shape) - dims = [ir.AffineExpr.get_dim(i) for i in range(ty.rank)] - output_shape = ( - [1 if i in axes else size for i, size in enumerate(shape)] - if keepdim - else [size for i, size in enumerate(shape) if i not in axes] - ) - results = ( - [ - ir.AffineExpr.get_constant(0) if i in axes else dim - for i, dim in enumerate(dims) - ] - if keepdim - else [dim for i, dim in enumerate(dims) if i not in axes] - ) - f64 = ir.F64Type.get() - op = _generic( - [value], - _filled(output_shape, f64, 0.0), - [ - ir.AffineMap.get_identity(ty.rank), - ir.AffineMap.get(ty.rank, 0, results), - ], - ["reduction" if i in axes else "parallel" for i in range(ty.rank)], - ) - body = ir.Block.create_at_start(op.region, [ty.element_type, f64]) - with ir.InsertionPoint(body): - number = ( - body.arguments[0] - if ty.element_type == f64 - else arith.ExtFOp(f64, body.arguments[0]).result - ) - linalg.YieldOp([arith.AddFOp(number, body.arguments[1]).result]) - return op.result - - -def _mean_f64(value, axes, keepdim=True): - summed = _sum_f64(value, axes, keepdim) - ty = ir.RankedTensorType(summed.type) - identity = ir.AffineMap.get_identity(ty.rank) - op = _generic( - [summed], - tensor.EmptyOp(ty.shape, ty.element_type).result, - [identity, identity], - ["parallel"] * ty.rank, - ) - body = ir.Block.create_at_start(op.region, [ty.element_type] * 2) - with ir.InsertionPoint(body): - count = scalar_math.prod(ir.RankedTensorType(value.type).shape[i] for i in axes) - divisor = arith.ConstantOp(ty.element_type, float(count)).result - linalg.YieldOp([arith.DivFOp(body.arguments[0], divisor).result]) - return op.result - - -def mean(node, symbols): - from stack.compiler.frontend.ops.tosa import _nhwc_to_nchw - from stack.compiler.frontend.layout.provenance import has_nhwc_layout - - value = symbols[(node.args[0], 0)] - if has_nhwc_layout(value): - value = _nhwc_to_nchw(value) - rank = ir.RankedTensorType(value.type).rank - return _cast( - _mean_f64(value, {int(axis) % rank for axis in node.args[1]}, node.args[2]), - ir.F32Type.get(), - ) - - -def bmm(node, symbols): - lhs, rhs = [symbols[(name, 0)] for name in node.args] - f32, f64 = ir.F32Type.get(), ir.F64Type.get() - b, m, n, k = [ir.AffineExpr.get_dim(i) for i in range(4)] - maps = [ - ir.AffineMap.get(4, 0, [b, m, k]), - ir.AffineMap.get(4, 0, [b, k, n]), - ir.AffineMap.get(4, 0, [b, m, n]), - ] - op = _generic( - [lhs, rhs], - _filled(node.tensor_meta["shape"], f64, 0.0), - maps, - ["parallel"] * 3 + ["reduction"], - ) - body = ir.Block.create_at_start(op.region, [f32, f32, f64]) - with ir.InsertionPoint(body): - a = arith.ExtFOp(f64, body.arguments[0]).result - b = arith.ExtFOp(f64, body.arguments[1]).result - linalg.YieldOp([math.FmaOp(a, b, body.arguments[2]).result]) - return _cast(op.result, f32) - - -def softmax(node, symbols): - value = symbols[(node.args[0], 0)] - ty = ir.RankedTensorType(value.type) - shape = list(ty.shape) - rank = ty.rank - axis = int(node.args[1]) % rank - f32, f64 = ir.F32Type.get(), ir.F64Type.get() - dims = [ir.AffineExpr.get_dim(i) for i in range(rank)] - reduced_shape = [1 if i == axis else size for i, size in enumerate(shape)] - small = ir.AffineMap.get( - rank, - 0, - [ - ir.AffineExpr.get_constant(0) if i == axis else dim - for i, dim in enumerate(dims) - ], - ) - identity = ir.AffineMap.get_identity(rank) - maximum = _generic( - [value], - _filled(reduced_shape, f32, float("-inf")), - [identity, small], - ["reduction" if i == axis else "parallel" for i in range(rank)], - ) - body = ir.Block.create_at_start(maximum.region, [f32, f32]) - with ir.InsertionPoint(body): - linalg.YieldOp([arith.MaximumFOp(*body.arguments).result]) - exponent = _generic( - [value, maximum.result], - tensor.EmptyOp(shape, f64).result, - [identity, small, identity], - ["parallel"] * rank, - ) - body = ir.Block.create_at_start(exponent.region, [f32, f32, f64]) - with ir.InsertionPoint(body): - x = arith.ExtFOp(f64, body.arguments[0]).result - maximum_value = arith.ExtFOp(f64, body.arguments[1]).result - linalg.YieldOp([math.ExpOp(arith.SubFOp(x, maximum_value).result).result]) - summed = _sum_f64(exponent.result, {axis}) - output = _generic( - [exponent.result, summed], - tensor.EmptyOp(shape, f32).result, - [identity, small, identity], - ["parallel"] * rank, - ) - body = ir.Block.create_at_start(output.region, [f64, f64, f32]) - with ir.InsertionPoint(body): - linalg.YieldOp( - [ - arith.TruncFOp( - f32, - arith.DivFOp(body.arguments[0], body.arguments[1]).result, - ).result - ] - ) - return output.result - - -def layer_norm(node, symbols): - value = symbols[(node.args[0], 0)] - weight = symbols[(node.args[2], 0)] - bias = symbols[(node.args[3], 0)] - shape = list(ir.RankedTensorType(value.type).shape) - rank = len(shape) - normalized = len(node.args[1]) - axes = set(range(rank - normalized, rank)) - f32, f64 = ir.F32Type.get(), ir.F64Type.get() - mean_value = _mean_f64(value, axes) - dims = [ir.AffineExpr.get_dim(i) for i in range(rank)] - identity = ir.AffineMap.get_identity(rank) - small = ir.AffineMap.get( - rank, - 0, - [ - ir.AffineExpr.get_constant(0) if i in axes else dim - for i, dim in enumerate(dims) - ], - ) - parameters = ir.AffineMap.get(rank, 0, dims[rank - normalized :]) - squared = _generic( - [value, mean_value], - tensor.EmptyOp(shape, f64).result, - [identity, small, identity], - ["parallel"] * rank, - ) - body = ir.Block.create_at_start(squared.region, [f32, f64, f64]) - with ir.InsertionPoint(body): - difference = arith.SubFOp( - arith.ExtFOp(f64, body.arguments[0]).result, body.arguments[1] - ).result - linalg.YieldOp([arith.MulFOp(difference, difference).result]) - variance = _mean_f64(squared.result, axes) - stat_shape = list(ir.RankedTensorType(variance.type).shape) - stat_identity = ir.AffineMap.get_identity(rank) - inverse = _generic( - [variance], - tensor.EmptyOp(stat_shape, f64).result, - [stat_identity, stat_identity], - ["parallel"] * rank, - ) - body = ir.Block.create_at_start(inverse.region, [f64, f64]) - with ir.InsertionPoint(body): - adjusted = arith.AddFOp( - body.arguments[0], arith.ConstantOp(f64, float(node.args[4])).result - ).result - root = math.SqrtOp(adjusted).result - linalg.YieldOp([arith.DivFOp(arith.ConstantOp(f64, 1.0).result, root).result]) - output = _generic( - [value, mean_value, inverse.result, weight, bias], - tensor.EmptyOp(shape, f32).result, - [identity, small, small, parameters, parameters, identity], - ["parallel"] * rank, - ) - body = ir.Block.create_at_start(output.region, [f32, f64, f64, f32, f32, f32]) - with ir.InsertionPoint(body): - difference = arith.SubFOp( - arith.ExtFOp(f64, body.arguments[0]).result, body.arguments[1] - ).result - normalized_value = arith.MulFOp(difference, body.arguments[2]).result - gamma = arith.ExtFOp(f64, body.arguments[3]).result - beta = arith.ExtFOp(f64, body.arguments[4]).result - result = math.FmaOp(normalized_value, gamma, beta).result - linalg.YieldOp([arith.TruncFOp(f32, result).result]) - return output.result, _cast(mean_value, f32), _cast(inverse.result, f32) - - -def gelu(node, symbols): - value = symbols[(node.args[0], 0)] - ty = ir.RankedTensorType(value.type) - f32, f64 = ir.F32Type.get(), ir.F64Type.get() - identity = ir.AffineMap.get_identity(ty.rank) - op = _generic( - [value], - tensor.EmptyOp(ty.shape, f32).result, - [identity, identity], - ["parallel"] * ty.rank, - ) - body = ir.Block.create_at_start(op.region, [f32, f32]) - with ir.InsertionPoint(body): - x = arith.ExtFOp(f64, body.arguments[0]).result - scaled = arith.MulFOp( - x, arith.ConstantOp(f64, scalar_math.sqrt(0.5)).result - ).result - erf = math.ErfOp(scaled).result - factor = arith.AddFOp(arith.ConstantOp(f64, 1.0).result, erf).result - half = arith.MulFOp(x, arith.ConstantOp(f64, 0.5).result).result - result = arith.MulFOp(half, factor).result - linalg.YieldOp([arith.TruncFOp(f32, result).result]) - return op.result - - -def tanh(node, symbols): - value = symbols[(node.args[0], 0)] - ty = ir.RankedTensorType(value.type) - f32, f64 = ir.F32Type.get(), ir.F64Type.get() - identity = ir.AffineMap.get_identity(ty.rank) - op = _generic( - [value], - tensor.EmptyOp(ty.shape, f32).result, - [identity, identity], - ["parallel"] * ty.rank, - ) - body = ir.Block.create_at_start(op.region, [f32, f32]) - with ir.InsertionPoint(body): - extended = arith.ExtFOp(f64, body.arguments[0]).result - result = math.TanhOp(extended).result - linalg.YieldOp([arith.TruncFOp(f32, result).result]) - return op.result diff --git a/waste/stack/compiler/frontend/ops/tosa.py b/waste/stack/compiler/frontend/ops/tosa.py deleted file mode 100644 index 0ee3b87ba..000000000 --- a/waste/stack/compiler/frontend/ops/tosa.py +++ /dev/null @@ -1,2271 +0,0 @@ -# ===- tosa.py ----------------------------------------------------------------- -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ===--------------------------------------------------------------------------- -# -# The registry of mappings from Buddy Graph to MLIR tosa dialect operations. -# -# ===--------------------------------------------------------------------------- - -import array -from collections.abc import Sequence -import buddy_mlir.ir as ir -import numpy -from buddy_mlir.dialects import ( - affine, - arith, - bufferization, - linalg, - math, - memref, - scf, - tensor, - tosa, - vector, -) -from buddy_mlir.dialects import ( - complex as complex_dialect, -) -from buddy_mlir.ir import F32Type, IndexType -from buddy.compiler.graph import ( - AbsOp, - AdaptiveAvgPool1dOp, - AdaptiveAvgPool2dBackwardOp, - AdaptiveAvgPool2dOp, - AdaptiveAvgPool3dOp, - AdaptiveMaxPool1dOp, - AdaptiveMaxPool2dOp, - AddbmmOp, - AddCMulOp, - AddMMOp, - AddOp, - AddScalarOp, - AliasOp, - AllDimsOp, - AllOp, - AmaxOp, - AminOp, - AnyDimsOp, - AnyOp, - ArangeStartStepOp, - ArgMaxOp, - ArgMinOp, - AvgPool1dOp, - AvgPool2dBackwardOp, - AvgPool2dOp, - BaddbmmOp, - BatchMatmulOp, - BitwiseAndScalarOp, - BitwiseAndTensorOp, - BitwiseLeftShiftOp, - BitwiseNotOp, - BitwiseOrOp, - BitwiseOrScalarOp, - BitwiseRightShiftOp, - BitwiseXorOp, - BitwiseXorScalarOp, - CauchyOp, - CdistForwardOp, - CeilOp, - ClampMaxOp, - ClampMaxTensorOp, - ClampMinOp, - ClampMinTensorOp, - ClampOp, - ClampTensorOp, - CloneOp, - ConstantPadNdOp, - Conv2dOp, - ConvertElementTypeOp, - ConvolutionBackwardOp, - CopysignOp, - CummaxOp, - CumminOp, - DiagonalOp, - DigammaOp, - DivOp, - DivScalarModeOp, - DivScalarOp, - DivTensorModeOp, - EmbeddingBagOp, - EmbeddingOp, - EmptyStridedOp, - EqTensorOp, - ErfcOp, - ErfinvOp, - ErfOp, - ExpandOp, - ExpOp, - FillScalarOp, - FlashAttentionForCpuPrefillOp, - FlipOp, - FloorDivideOp, - FloorOp, - FmodOp, - FrexpOp, - FullLikeOp, - GeTensorOp, - GluOp, - GQAAttentionFusedOp, - GtOp, - GtTensorOp, - HardswishOp, - HypotOp, - I0Op, - IgammacOp, - IgammaOp, - Int4UnpackOp, - IotaOp, - IsInfOp, - IsNanOp, - LeOp, - LerpOp, - LeScalarOp, - LeTensorOp, - LgammaOp, - LocalScalarDenseOp, - LogicalAndOp, - LogicalNotOp, - LogicalOrOp, - LogicalXorOp, - LogOp, - LtScalarOp, - LtTensorOp, - MaskedFillOp, - MaskedScatterOp, - MaxDimOp, - MaximumOp, - MaxPool1dOp, - MaxPool2dOp, - MeanDefaultOp, - MeanOp, - MinDimOp, - MinimumOp, - MulOp, - NativeDropoutOp, - NativeGroupNormBackwardOp, - NativeGroupNormOp, - NativeLayerNormBackwardOp, - NegOp, - NeScalarOp, - NeTensorOp, - NextafterOp, - NormScalarOp, - NormScalarOptDimOp, - OnesLikeOp, - PermuteOp, - PowScalarOp, - PowTensorTensorOp, - ProdOp, - RandIntLowOp, - RandpermOp, - ReciprocalOp, - ReflectionPad1dOp, - ReflectionPad2dOp, - ReflectionPad3dOp, - ReluOp, - RemainderOp, - RepeatOp, - ReplicationPad2dOp, - ReplicationPad3dOp, - ReshapeOp, - ResizeOp, - RevOp, - RreluWithNoiseOp, - RsqrtOp, - ScaledDotProductFlashAttentionForCpuOp, - SelectOp, - SigmoidOp, - SignbitOp, - SignOp, - SiluOp, - SliceOp, - SoftplusOp, - SpecialAiryAiOp, - SpecialBesselJ0Op, - SpecialBesselJ1Op, - SpecialBesselY0Op, - SpecialBesselY1Op, - SpecialChebyshevPolynomialTOp, - SpecialChebyshevPolynomialUOp, - SpecialChebyshevPolynomialVOp, - SpecialChebyshevPolynomialWOp, - SpecialEntrOp, - SpecialErfcxOp, - SpecialHermitePolynomialHeOp, - SpecialHermitePolynomialHOp, - SpecialI0eOp, - SpecialI1eOp, - SpecialI1Op, - SpecialLaguerrePolynomialLOp, - SpecialLegendrePolynomialPOp, - SpecialLogNdtrOp, - SpecialModifiedBesselK0Op, - SpecialModifiedBesselK1Op, - SpecialNdtriOp, - SpecialNdtrOp, - SpecialScaledModifiedBesselK0Op, - SpecialScaledModifiedBesselK1Op, - SpecialShiftedChebyshevPolynomialTOp, - SpecialShiftedChebyshevPolynomialUOp, - SpecialShiftedChebyshevPolynomialVOp, - SpecialShiftedChebyshevPolynomialWOp, - SpecialSphericalBesselJ0Op, - SpecialZetaOp, - SplitWithSizesOp, - SqueezeDimOp, - SqueezeDimsOp, - SqueezeOp, - StackOp, - StdCorrectionOp, - StdDefaultOp, - StdDimOp, - SubOp, - SubScalarOp, - SumDefaultOp, - SumDimOp, - TanhOp, - TensorDType, - TileOp, - TOp, - TransposeOp, - UnbindOp, - UnfoldOp, - UniformOp, - UnsqueezeOp, - VarCorrectionOp, - VarDefaultOp, - VarDimOp, - VarMeanOp, - ViewDtypeOp, - WhereOp, - ZerosLikeOp, - ZerosOp, -) -from stack.compiler.frontend.layout.provenance import has_nhwc_layout -from stack.compiler.frontend.ops.expf import exp_f32 -from stack.compiler.frontend.ops.int8 import conv2d as int8_conv2d_op -from stack.compiler.frontend.ops.int8 import matmul as int8_matmul_op -from stack.compiler.frontend.ops.int8 import quantize_activation as int8_quantize_op -from buddy.compiler.ops.utils import * -from buddy.compiler.ops.tosa import ( - _build_reassociation_attr, - _create_mul_shift_operand, - _create_permutation_attr, - _create_shape_operand, - _create_zero_point_tensor, - _create_zero_tensor, - _gen_arith_binary_op, - _normalize_reduce_dims, - _reshape_or_extract_for_complex, -) - - -def _to_nhwc(val, channels, elem_ty, nchw_meta=None): - """Transpose NCHW->NHWC only when SSA is still NCHW. Fail if layout unknown.""" - shp = [int(x) for x in ir.RankedTensorType(val.type).shape] - if len(shp) != 4: - raise RuntimeError(f"expected rank-4 activation, got {shp}") - # Equal channel/spatial extents cannot identify the axis order by shape. - if shp[1] == channels and shp[3] == channels: - if has_nhwc_layout(val): - n, h, w, c = shp - return val, n, h, w, c - n, c, h, w = shp - val = tosa.TransposeOp( - ir.RankedTensorType.get([n, h, w, c], elem_ty), - val, - _create_permutation_attr([0, 2, 3, 1]), - ).result - return val, n, h, w, c - if nchw_meta is not None: - n, c, h, w = [int(x) for x in nchw_meta] - if shp == [n, c, h, w]: - out_ty = ir.RankedTensorType.get([n, h, w, c], elem_ty) - val = tosa.TransposeOp( - out_ty, val, _create_permutation_attr([0, 2, 3, 1]) - ).result - return val, n, h, w, c - if shp == [n, h, w, c]: - return val, n, h, w, c - raise RuntimeError( - f"cannot infer NCHW/NHWC for ssa={shp} meta_nchw={[n, c, h, w]}" - ) - if shp[3] == channels: - n, h, w, c = shp - return val, n, h, w, c - if shp[1] == channels: - n, c, h, w = shp - out_ty = ir.RankedTensorType.get([n, h, w, c], elem_ty) - val = tosa.TransposeOp( - out_ty, val, _create_permutation_attr([0, 2, 3, 1]) - ).result - return val, n, h, w, c - raise RuntimeError( - f"cannot infer NCHW/NHWC for shape {shp} with channels={channels}" - ) - - -def _nhwc_to_nchw(value: ir.Value) -> ir.Value: - shape = [int(x) for x in ir.RankedTensorType(value.type).shape] - if len(shape) != 4: - raise RuntimeError(f"_nhwc_to_nchw expects rank-4, got {shape}") - n, h, w, c = shape - elem = ir.RankedTensorType(value.type).element_type - out_ty = ir.RankedTensorType.get([n, c, h, w], elem) - return tosa.TransposeOp( - out_ty, value, _create_permutation_attr([0, 3, 1, 2]) - ).result - - -def _nchw_to_nhwc(value: ir.Value) -> ir.Value: - shape = [int(x) for x in ir.RankedTensorType(value.type).shape] - if len(shape) != 4: - raise RuntimeError(f"_nchw_to_nhwc expects rank-4, got {shape}") - n, c, h, w = shape - elem = ir.RankedTensorType(value.type).element_type - out_ty = ir.RankedTensorType.get([n, h, w, c], elem) - return tosa.TransposeOp( - out_ty, value, _create_permutation_attr([0, 2, 3, 1]) - ).result - - -def _split_via_tosa_slice( - input_tensor: ir.Value, - input_shape: list[int], - dim: int, - split_sizes: list[int], -) -> list[ir.Value]: - rank = len(input_shape) - elem = ir.RankedTensorType(input_tensor.type).element_type - results = [] - offset = 0 - for size in split_sizes: - start = [0] * rank - start[dim] = offset - out_shape = list(input_shape) - out_shape[dim] = size - out_ty = ir.RankedTensorType.get(out_shape, elem) - results.append( - tosa.SliceOp( - out_ty, - input_tensor, - _create_shape_operand(start), - _create_shape_operand(out_shape), - ).result - ) - offset += size - return results - - -def _align_binary_layout(input1, input2): - """Keep NHWC activations and align logical NCHW broadcast operands.""" - if not (has_nhwc_layout(input1) or has_nhwc_layout(input2)): - return input1, input2 - operands = [input1, input2] - for index, value in enumerate(operands): - if not isinstance(value, ir.Value) or has_nhwc_layout(value): - continue - shape = list(ir.RankedTensorType(value.type).shape) - shape = [1] * (4 - len(shape)) + shape - value = tosa.ReshapeOp(value, _create_shape_operand(shape)).result - if sum(size != 1 for size in shape) <= 1: - value = tosa.ReshapeOp( - value, - _create_shape_operand([shape[0], shape[2], shape[3], shape[1]]), - ).result - else: - value = _nchw_to_nhwc(value) - value.owner.attributes["buddy.layout"] = ir.StringAttr.get("nhwc") - operands[index] = value - return operands - - -def add_op(node: AddOp, symbol_table): - """ - Import tensor addition operation. - From buddy graph ir's `AddOp` operator to MLIR TOSA `add` operation. - """ - input1 = symbol_table.get((str(node.args[0]), 0), node.args[0]) - input2 = symbol_table.get((str(node.args[1]), 0), node.args[1]) - dtype = node.tensor_meta["dtype"] - mlir_dtype = mlir_element_type_get(dtype) - if isinstance(node.args[0], str) and isinstance(node.args[1], str): - input1_dtype = ir.RankedTensorType(input1.type).element_type - input2_dtype = ir.RankedTensorType(input2.type).element_type - if input1_dtype != mlir_dtype: - input1 = tosa.CastOp( - ir.RankedTensorType.get( - ir.RankedTensorType(input1.type).shape, - mlir_dtype, - ), - input1, - ).result - if input2_dtype != mlir_dtype: - input2 = tosa.CastOp( - ir.RankedTensorType.get( - ir.RankedTensorType(input2.type).shape, - mlir_dtype, - ), - input2, - ).result - input1, input2 = _align_binary_layout(input1, input2) - return _gen_arith_binary_op(input1, input2, tosa.AddOp) - - -def sub_op(node: SubOp, symbol_table): - """ - Import tensor subtraction operation. - From buddy graph ir's `SubOp` operator to MLIR TOSA `sub` operation. - """ - input1 = symbol_table.get((str(node.args[0]), 0), node.args[0]) - input2 = symbol_table.get((str(node.args[1]), 0), node.args[1]) - dtype = node.tensor_meta["dtype"] - mlir_dtype = mlir_element_type_get(dtype) - if isinstance(node.args[0], str) and isinstance(node.args[1], str): - input1_dtype = ir.RankedTensorType(input1.type).element_type - input2_dtype = ir.RankedTensorType(input2.type).element_type - if input1_dtype != mlir_dtype: - input1 = tosa.CastOp( - ir.RankedTensorType.get( - ir.RankedTensorType(input1.type).shape, - mlir_dtype, - ), - input1, - ).result - if input2_dtype != mlir_dtype: - input2 = tosa.CastOp( - ir.RankedTensorType.get( - ir.RankedTensorType(input2.type).shape, - mlir_dtype, - ), - input2, - ).result - input1, input2 = _align_binary_layout(input1, input2) - return _gen_arith_binary_op(input1, input2, tosa.SubOp) - - -def mul_op(node: MulOp, symbol_table): - """ - Import tensor division operation. - From buddy graph ir's `DivOp` operator to MLIR TOSA `div` operation. - """ - - def _inner_op(result_type, input1, input2): - shift = _create_mul_shift_operand() - return tosa.MulOp(result_type, input1, input2, shift) - - output_shape = list(node.tensor_meta["shape"]) - dtype = node.tensor_meta["dtype"] - mlir_dtype = mlir_element_type_get(dtype) - - if isinstance(node.args[0], str): - input1 = symbol_table.get((str(node.args[0]), 0), node.args[0]) - else: - data = [node.args[0]] - input1_shape = numpy.array(data).shape - tensor_type = ir.RankedTensorType.get(input1_shape, mlir_dtype) - element = mlir_element_attr_get(dtype, node.args[0]) - attr = ir.DenseElementsAttr.get_splat(tensor_type, element) - input2 = arith.ConstantOp(tensor_type, attr).result - - if isinstance(node.args[1], str): - input2 = symbol_table.get((str(node.args[1]), 0), node.args[1]) - else: - data = [node.args[1]] - input2_shape = numpy.array(data).shape - tensor_type = ir.RankedTensorType.get(input2_shape, mlir_dtype) - element = mlir_element_attr_get(dtype, node.args[1]) - attr = ir.DenseElementsAttr.get_splat(tensor_type, element) - input2 = arith.ConstantOp(tensor_type, attr).result - - input1_dtype = ir.RankedTensorType(input1.type).element_type - input2_dtype = ir.RankedTensorType(input2.type).element_type - if input1_dtype != mlir_dtype: - input1 = tosa.CastOp( - ir.RankedTensorType.get( - ir.RankedTensorType(input1.type).shape, - mlir_dtype, - ), - input1, - ).result - if input2_dtype != mlir_dtype: - input2 = tosa.CastOp( - ir.RankedTensorType.get( - ir.RankedTensorType(input2.type).shape, - mlir_dtype, - ), - input2, - ).result - - input1, input2 = _align_binary_layout(input1, input2) - return _gen_arith_binary_op(input1, input2, _inner_op) - - -def div_op(node: DivOp, symbol_table): - """ - Import tensor division operation. - From buddy graph ir's `DivOp` operator to MLIR TOSA `div` operation. - """ - - def _inner_op(result_type, input1, input2): - # Preserve true division. Reciprocal followed by multiplication can - # cross round-to-even boundaries in activation quantization. - shape = list(result_type.shape) - rank = len(shape) - maps = [] - for value in (input1, input2): - dims = [ - ( - ir.AffineExpr.get_constant(0) - if extent == 1 - else ir.AffineExpr.get_dim(i) - ) - for i, extent in enumerate(ir.RankedTensorType(value.type).shape) - ] - maps.append(ir.AffineMapAttr.get(ir.AffineMap.get(rank, 0, dims))) - maps.append(ir.AffineMapAttr.get(ir.AffineMap.get_identity(rank))) - empty = tensor.EmptyOp(shape, result_type.element_type) - division = linalg.GenericOp( - [result_type], - [input1, input2], - [empty.result], - ir.ArrayAttr.get(maps), - ir.ArrayAttr.get( - [ir.Attribute.parse("#linalg.iterator_type")] * rank - ), - ) - block = ir.Block.create_at_start( - division.region, [result_type.element_type] * 3 - ) - with ir.InsertionPoint(block): - value = arith.DivFOp(block.arguments[0], block.arguments[1]).result - linalg.YieldOp([value]) - if has_nhwc_layout(input1) or has_nhwc_layout(input2): - division.operation.attributes["buddy.layout"] = ir.StringAttr.get("nhwc") - return division - - input1 = symbol_table.get((str(node.args[0]), 0), node.args[0]) - input2 = symbol_table.get((str(node.args[1]), 0), node.args[1]) - - # PyTorch integer division returns float32. Cast any integer tensor inputs to - # float32 so tosa.reciprocal is legalizable and the result type matches Torch - # semantics (true division). - def _is_int_tensor(value: object) -> bool: - if not isinstance(value, ir.Value): - return False - element_type = ir.RankedTensorType(value.type).element_type - return isinstance(element_type, ir.IntegerType) - - if _is_int_tensor(input1) or _is_int_tensor(input2): - f32 = ir.F32Type.get() - if isinstance(input1, ir.Value): - in1_ty = ir.RankedTensorType.get( - list(ir.RankedTensorType(input1.type).shape), f32 - ) - input1 = tosa.CastOp(in1_ty, input1).result - elif isinstance(input1, int): - input1 = float(input1) - if isinstance(input2, ir.Value): - in2_ty = ir.RankedTensorType.get( - list(ir.RankedTensorType(input2.type).shape), f32 - ) - input2 = tosa.CastOp(in2_ty, input2).result - elif isinstance(input2, int): - input2 = float(input2) - - input1, input2 = _align_binary_layout(input1, input2) - return _gen_arith_binary_op(input1, input2, _inner_op) - - -def amax_op(node: AmaxOp, symbol_table): - """ - Import the amax operation. - From buddy graph ir's `AmaxOp` operator to MLIR TOSA `reduce_max` - operation. - """ - input1 = symbol_table.get((str(node.args[0]), 0)) - dim_arg = node.args[1] if len(node.args) > 1 else None - keepdim = node.args[2] if len(node.args) > 2 else False - rank = len(ir.RankedTensorType(input1.type).shape) - dims = _normalize_reduce_dims(dim_arg, rank) - input_nhwc = has_nhwc_layout(input1) - if input_nhwc: - dims = [[0, 3, 1, 2][axis] for axis in dims] - - result = input1 - for dim_val in dims: - dim_attr = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), dim_val) - result = tosa.ReduceMaxOp(result, dim_attr).result - - if input_nhwc: - shape = list(ir.RankedTensorType(result.type).shape) - if sum(size != 1 for size in shape) <= 1: - logical_shape = [shape[0], shape[3], shape[1], shape[2]] - result = tosa.ReshapeOp(result, _create_shape_operand(logical_shape)).result - else: - result = _nhwc_to_nchw(result) - if not keepdim: - output_shape = list(node.tensor_meta["shape"]) - reshape_operand = _create_shape_operand(output_shape) - result = tosa.ReshapeOp(result, reshape_operand).result - - return result - - -def avg_pool2d_op(node: AvgPool2dOp, symbol_table): - """ - Import the avg_pool2d operation. - From Buddy AvgPool2dOp to MLIR TOSA `avg_pool2d` operation. - """ - input1 = symbol_table.get((str(node.args[0]), 0)) - kernel = node.args[1] - stride = node.args[2] - if len(node.args) > 3: - pad = node.args[3] - else: - pad = [0 for _ in kernel] - dtype = node.tensor_meta["dtype"] - result_element_type = mlir_element_type_get(dtype) - acc_type = ir.TypeAttr.get(result_element_type) - - meta = list(node.tensor_meta["shape"]) - channels = int(meta[1]) - src = node.args[0] - nchw_meta = ( - list(src.tensor_meta["shape"]) - if hasattr(src, "tensor_meta") and "shape" in src.tensor_meta - else None - ) - input1, N, H, W, C = _to_nhwc(input1, channels, result_element_type, nchw_meta) - input_shape_nhwc = [N, H, W, C] - - # Expand pad to 4 elements if needed - if len(pad) == 1: - pad = [pad[0]] * 4 - elif len(pad) == 2: - pad = [pad[0]] * 2 + [pad[1]] * 2 - - # Calculate output shape in NHWC format - # Formula: output = (input + pad_top + pad_bottom - kernel) / stride + 1 - pad_top, pad_bottom, pad_left, pad_right = pad - output_h = (H + pad_top + pad_bottom - kernel[0]) // stride[0] + 1 - output_w = (W + pad_left + pad_right - kernel[1]) // stride[1] + 1 - output_shape_nhwc = [N, output_h, output_w, C] - - kernel_attr = ir._denseI64ArrayAttr(kernel, None) - stride_attr = ir._denseI64ArrayAttr(stride, None) - pad_attr = ir._denseI64ArrayAttr(pad, None) - - output = ir.RankedTensorType.get(output_shape_nhwc, result_element_type) - input_zp = _create_zero_point_tensor(input1) - output_zp = _create_zero_point_tensor(input1) - op = tosa.AvgPool2dOp( - output, - input1, - input_zp, - output_zp, - kernel=kernel_attr, - stride=stride_attr, - pad=pad_attr, - acc_type=acc_type, - ) - return op - - -def adaptive_avg_pool2d_op(node: AdaptiveAvgPool2dOp, symbol_table): - """ - Import the adaptive_avg_pool2d operation. - From buddy graph ir's `AdaptiveAvgPool2dOp` operator to MLIR operations. - Uses TOSA avg_pool2d with calculated kernel/stride. - """ - input1 = symbol_table.get((str(node.args[0]), 0)) - input_is_nhwc = has_nhwc_layout(input1) - output_size = node.args[1] - if isinstance(output_size, int): - output_size = [output_size, output_size] - - dtype = node.tensor_meta["dtype"] - result_element_type = mlir_element_type_get(dtype) - acc_type = ir.TypeAttr.get(result_element_type) - - # Get input shape; sticky NHWC after conv keeps channels last. - channels = int(list(node.tensor_meta["shape"])[1]) - src = node.args[0] - nchw_meta = ( - list(src.tensor_meta["shape"]) - if hasattr(src, "tensor_meta") and "shape" in src.tensor_meta - else None - ) - input1, N, H, W, C = _to_nhwc(input1, channels, result_element_type, nchw_meta) - out_h, out_w = output_size - - # Calculate kernel and stride for adaptive pooling - kernel_h = (H + out_h - 1) // out_h - kernel_w = (W + out_w - 1) // out_w - stride_h = H // out_h - stride_w = W // out_w - - # Apply avg_pool2d - kernel_attr = ir._denseI64ArrayAttr([kernel_h, kernel_w], None) - stride_attr = ir._denseI64ArrayAttr([stride_h, stride_w], None) - pad_attr = ir._denseI64ArrayAttr([0, 0, 0, 0], None) - - pool_nhwc_shape = [N, out_h, out_w, C] - pool_type = ir.RankedTensorType.get(pool_nhwc_shape, result_element_type) - input_zp = _create_zero_point_tensor(input1) - output_zp = _create_zero_point_tensor(input1) - pooled = tosa.AvgPool2dOp( - pool_type, - input1, - input_zp, - output_zp, - kernel=kernel_attr, - stride=stride_attr, - pad=pad_attr, - acc_type=acc_type, - ) - if input_is_nhwc: - return pooled - return tosa.TransposeOp( - ir.RankedTensorType.get([N, C, out_h, out_w], result_element_type), - pooled.result, - _create_permutation_attr([0, 3, 1, 2]), - ) - - -def _reshape_input_nchw_meta(node, symbol_table): - """PyTorch NCHW shape of reshape/view input. args[0] is often a name str.""" - src = node.args[0] - if hasattr(src, "tensor_meta") and "shape" in src.tensor_meta: - return [int(x) for x in src.tensor_meta["shape"]] - if isinstance(src, str): - ops = symbol_table.get(("__buddy_ops_by_name__", 0)) - if ops is None: - return None - pred = ops.get(src) - if pred is None or not ( - hasattr(pred, "tensor_meta") and "shape" in pred.tensor_meta - ): - return None - return [int(x) for x in pred.tensor_meta["shape"]] - return None - - -def reshape_op(node: ReshapeOp, symbol_table): - """ - Import the reshape operation. - From buddy graph ir's `ReshapeOp` operator to MLIR TOSA `reshape` - operation. - - Note: If the new shape contains one and only one `-1`, the size of the new - shape will be inferred automatically. - """ - input1 = symbol_table.get((str(node.args[0]), 0)) - new_shape = [] - if node._newshape is None: - shape_arg = node.args[1] - - if isinstance(shape_arg, (list, tuple)): - new_shape = list(shape_arg) - else: - try: - new_shape = list(shape_arg) - except TypeError: - new_shape = [shape_arg] - else: - new_shape = list(node._newshape) - - now_shape = ir.RankedTensorType(input1.type).shape - total_size = 1 - for dim_siz in now_shape: - total_size *= dim_siz - - neg_one_cnt = 0 - rest_size = 1 - for dim_siz in new_shape: - if dim_siz == -1: - neg_one_cnt += 1 - continue - rest_size *= dim_siz - - if neg_one_cnt != 0: - if neg_one_cnt > 1 or total_size % rest_size != 0: - raise ValueError("Can not infer the new shape!") - infer_dim_size = total_size // rest_size - for i, _ in enumerate(new_shape): - if new_shape[i] == -1: - new_shape[i] = infer_dim_size - - producer = input1.owner - if isinstance(producer, ir.OpView): - producer = producer.operation - if len(now_shape) == 4 and isinstance(producer, ir.Operation): - attrs = producer.attributes - if ( - ( - "buckyball.mega_conv2d" in attrs - or "buckyball.mega_conv2d_depthwise" in attrs - ) - and "final_output" in attrs - and ir.BoolAttr(attrs["final_output"]).value - and ir.RankedTensorType(input1.type).element_type == ir.F32Type.get() - ): - return _reshape_or_extract_for_complex(input1, new_shape) - - if len(now_shape) == 4 and has_nhwc_layout(input1): - input_meta = _reshape_input_nchw_meta(node, symbol_table) - if input_meta is None or len(input_meta) != 4: - raise RuntimeError("reshape of NHWC activation requires Torch input shape") - n, c, h, w = input_meta - if list(now_shape) != [n, h, w, c]: - raise RuntimeError( - f"reshape NHWC layout mismatch: ssa={list(now_shape)} torch={input_meta}" - ) - element_type = ir.RankedTensorType(input1.type).element_type - input1 = tosa.TransposeOp( - ir.RankedTensorType.get(input_meta, element_type), - input1, - _create_permutation_attr([0, 3, 1, 2]), - ).result - - return _reshape_or_extract_for_complex(input1, new_shape) - - -def maxpool2d_op(node: MaxPool2dOp, symbol_table): - """ - Import the maxpool2d operation. - From Buddy MaxPool2dOp to MLIR TOSA `max_pool2d` operation. - """ - if len(node.args) == 5: - raise NotImplementedError - input1 = symbol_table.get((str(node.args[0]), 0)) - kernel = node.args[1] - stride = node.args[2] - if len(node.args) > 3: - pad = node.args[3] - else: - pad = [0 for _ in kernel] - dtype = node.tensor_meta["dtype"] - result_element_type = mlir_element_type_get(dtype) - - original_shape = list(ir.RankedTensorType(input1.type).shape) - channels = int(list(node.tensor_meta["shape"])[1]) - src = node.args[0] - nchw_meta = ( - list(src.tensor_meta["shape"]) - if hasattr(src, "tensor_meta") and "shape" in src.tensor_meta - else None - ) - input1, n, h, w, c = _to_nhwc(input1, channels, result_element_type, nchw_meta) - in_n, in_c, in_h, in_w = n, c, h, w - - out_shape = node.tensor_meta["shape"] - if len(pad) == 1: - pad = [pad[0]] * 4 - elif len(pad) == 2: - pad = [pad[0]] * 2 + [pad[1]] * 2 - - k_h, k_w = kernel - s_h, s_w = stride - pt, pb, pl, pr = pad - # Preserve the leading padding. TOSA requires an exact output extent; - # adjust only the trailing padding to match Torch floor/ceil semantics. - out_h, out_w = node.tensor_meta["shape"][2:] - pb = max(0, (out_h - 1) * s_h + k_h - in_h - pt) - pr = max(0, (out_w - 1) * s_w + k_w - in_w - pl) - - kernel_attr = ir._denseI64ArrayAttr(kernel, None) - stride_attr = ir._denseI64ArrayAttr(stride, None) - pad_attr = ir._denseI64ArrayAttr([pt, pb, pl, pr], None) - out_h = (in_h + pt + pb - k_h) // s_h + 1 - out_w = (in_w + pl + pr - k_w) // s_w + 1 - out_shape_nhwc = [in_n, out_h, out_w, in_c] - output = ir.RankedTensorType.get(out_shape_nhwc, result_element_type) - op = tosa.MaxPool2dOp(output, input1, kernel_attr, stride_attr, pad_attr) - return op - - -def convolution2d_op(node: Conv2dOp, symbol_table): - """ - Import the convolution operation. - From Buddy Conv2dOp to MLIR TOSA `conv2d` operation. - arg[0]: Tensor input - arg[1]: Tensor weight - arg[2]: Tensor? bias - arg[3]: SymInt[] stride - arg[4]: SymInt[] padding - arg[5]: SymInt[] dilation - arg[6]: bool transposed - arg[7]: SymInt[] output_padding - arg[8]: SymInt groups - """ - # Get arguments from convolution node. - assert len(node.args) == 9 - input = node.args[0] - weight = node.args[1] - bias = node.args[2] - stride = node.args[3] - input_padding = node.args[4] - dilation = node.args[5] - is_kernel_transposed = node.args[6] - out_padding = node.args[7] - groups = node.args[8] - - # Prepare input, weight, and output information. - input_val = symbol_table.get((str(input), 0)) - input_shape = list(ir.RankedTensorType(input_val.type).shape) - weight_val = symbol_table.get((str(weight), 0)) - weight_shape = ir.RankedTensorType(weight_val.type).shape - dtype = node.tensor_meta["dtype"] - result_element_type = mlir_element_type_get(dtype) - out_shape = node.tensor_meta["shape"] - acc_type = ir.TypeAttr.get(result_element_type) - - # Prepare Depthwise Conv2D information - is_grouped = (list(weight_shape)[1] == 1) and (groups != 1) - is_depthwise = (groups == list(weight_shape)[0]) and is_grouped - - # Prepare input channel and output channel. - if is_kernel_transposed: - in_channels = list(weight_shape)[0] - out_channels = list(weight_shape)[1] * groups - else: - in_channels = list(weight_shape)[1] * groups - out_channels = list(weight_shape)[0] - - # Prepare bias tensor. - if len(node._parents) == 2: - new_size_tensor_type = ir.RankedTensorType.get( - [out_channels], result_element_type - ) - element = mlir_element_attr_get(dtype, 0) - new_size_attr = ir.DenseElementsAttr.get_splat(new_size_tensor_type, element) - bias_tensor = tosa.ConstOp(new_size_attr).results[0] - else: - bias_tensor = symbol_table.get((str(bias), 0)) - - # Prepare attributes. - dilation_attr = ir._denseI64ArrayAttr(dilation, None) - stride_attr = ir._denseI64ArrayAttr(stride, None) - - # Convolution 2D - if len(weight_shape) == 4: - # Prepare input padding. - if len(input_padding) == 1: - input_padding = [input_padding[0]] * 4 - elif len(input_padding) == 2: - input_padding = [input_padding[0]] * 2 + [input_padding[1]] * 2 - - t, b, l, r = input_padding - sy, sx = int(stride[0]), int(stride[1]) - dy, dx = int(dilation[0]), int(dilation[1]) - kernel_h = int(list(weight_shape)[2]) - kernel_w = int(list(weight_shape)[3]) - - extra_out_h = 0 - extra_out_w = 0 - src = node.args[0] - nchw_meta = ( - list(src.tensor_meta["shape"]) - if hasattr(src, "tensor_meta") and "shape" in src.tensor_meta - else None - ) - input_val, _, input_h, input_w, _ = _to_nhwc( - input_val, in_channels, result_element_type, nchw_meta - ) - input_h = int(input_h) - input_w = int(input_w) - - def _adjust_padding(input_size, kernel, dil, stride_val, pad0, pad1): - """Ensure stride divisibility by adding pad to the second side.""" - base = input_size - 1 - (kernel - 1) * dil - total = base + pad0 + pad1 - remainder = total % stride_val - if remainder == 0: - return pad0, pad1, 0 - pad_needed = stride_val - remainder - pad1 += pad_needed - return pad0, pad1, 1 - - if not is_kernel_transposed: - t, b, extra_h = _adjust_padding(input_h, kernel_h, dy, sy, int(t), int(b)) - l, r, extra_w = _adjust_padding(input_w, kernel_w, dx, sx, int(l), int(r)) - extra_out_h += extra_h - extra_out_w += extra_w - input_padding = [t, b, l, r] - - # Prepare input_padding attributes. - input_padding_attr = ir._denseI64ArrayAttr(input_padding, None) - # TOSA conv2d is NHWC; keep output in NHWC (no convert-back). - out_shape = [ - out_shape[0], - out_shape[2], - out_shape[3], - out_shape[1], - ] - conv_out_shape = list(out_shape) - if extra_out_h: - conv_out_shape[1] += extra_out_h - if extra_out_w: - conv_out_shape[2] += extra_out_w - output_type = ir.RankedTensorType.get(conv_out_shape, result_element_type) - - if ( - str(ir.RankedTensorType(weight_val.type).element_type) == "i8" - and not is_depthwise - ): - if is_kernel_transposed: - raise ValueError("quantized transpose convolution is unsupported") - if stride[0] != stride[1]: - raise ValueError("quantized convolution requires equal strides") - pad_constant = _create_shape_operand( - [0, 0, int(t), int(b), int(l), int(r), 0, 0] - ) - pad_value = _create_zero_point_tensor(input_val) - padded_type = ir.RankedTensorType.get( - [ - conv_out_shape[0], - input_h + int(t) + int(b), - input_w + int(l) + int(r), - in_channels, - ], - result_element_type, - ) - padded_input = tosa.PadOp( - padded_type, input_val, pad_constant, pad_value - ).result - quant_weight = weight_val - if node._layout.find("FCHW") != -1: - perm_shape = [ - weight_shape[0], - weight_shape[2], - weight_shape[3], - weight_shape[1], - ] - quant_weight = tosa.TransposeOp( - ir.RankedTensorType.get(perm_shape, ir.IntegerType.get_signless(8)), - weight_val, - _create_permutation_attr([0, 2, 3, 1]), - ).result - bias_output = tensor.EmptyOp(conv_out_shape, result_element_type) - generic_map = ir.AffineMap.get_permutation([0, 1, 2, 3]) - bias_init = linalg.GenericOp( - [output_type], - [bias_tensor], - [bias_output], - ir.ArrayAttr.get( - [ - ir.AffineMapAttr.get(generic_map.get_submap([3])), - ir.AffineMapAttr.get(generic_map), - ] - ), - ir.ArrayAttr.get( - [ir.Attribute.parse("#linalg.iterator_type")] * 4 - ), - ) - bias_block = ir.Block.create_at_start( - bias_init.region, [result_element_type, result_element_type] - ) - bias_init.region.blocks[0].append(linalg.YieldOp([bias_block.arguments[0]])) - quant_conv = linalg.Conv2DNhwcFhwcOp( - [output_type], - [padded_input, quant_weight], - [bias_init.result], - strides=stride_attr, - dilations=dilation_attr, - ) - linalg.fill_builtin_region(quant_conv.operation) - i64 = ir.IntegerType.get_signless(64) - quant_conv.operation.attributes["dw_addr"] = ir.IntegerAttr.get( - i64, node._dw_addr - ) - quant_conv.operation.attributes["dw_bytes"] = ir.IntegerAttr.get( - i64, node._dw_bytes - ) - quant_conv.operation.attributes["per_channel"] = ir.BoolAttr.get( - node._per_channel - ) - quant_conv.operation.attributes["strides"] = ir.DenseElementsAttr.get( - numpy.array([int(stride[0]), int(stride[1])], dtype=numpy.int64), - type=ir.VectorType.get([2], i64), - ) - op = quant_conv - if extra_out_h or extra_out_w: - slice_start = _create_shape_operand([0, 0, 0, 0]) - slice_size = _create_shape_operand(out_shape) - slice_type = ir.RankedTensorType.get(out_shape, result_element_type) - op = tosa.SliceOp(slice_type, op.result, slice_start, slice_size) - return op - - # Depthwise Conv2D Operation. - if is_depthwise is True: - # If groups == in_channels,out_channels == in_channels - weight_depthwise = weight_val - if node._layout.find("FCHW") != -1: - perm_list = [2, 3, 0, 1] - perms_attr = _create_permutation_attr(perm_list) - perm_shape = [] - perm_shape.append(weight_shape[2]) - perm_shape.append(weight_shape[3]) - perm_shape.append(weight_shape[0]) - perm_shape.append(weight_shape[1]) - permute_result_type = ir.RankedTensorType.get( - perm_shape, - ir.RankedTensorType(weight_val.type).element_type, - ) - weight_depthwise = tosa.TransposeOp( - permute_result_type, weight_val, perms_attr - ).result - if str(ir.RankedTensorType(weight_depthwise.type).element_type) == "i8": - pad_constant = _create_shape_operand( - [0, 0, int(t), int(b), int(l), int(r), 0, 0] - ) - pad_value = _create_zero_point_tensor(input_val) - padded_shape = [ - conv_out_shape[0], - input_h + int(t) + int(b), - input_w + int(l) + int(r), - conv_out_shape[3], - ] - padded_type = ir.RankedTensorType.get(padded_shape, result_element_type) - input_val = tosa.PadOp( - padded_type, input_val, pad_constant, pad_value - ).result - depth_shape = [ - conv_out_shape[0], - conv_out_shape[1], - conv_out_shape[2], - conv_out_shape[3], - 1, - ] - depth_type = ir.RankedTensorType.get(depth_shape, result_element_type) - depth_init = tensor.EmptyOp(depth_shape, result_element_type) - identity = ir.AffineMap.get_permutation([0, 1, 2, 3, 4]) - bias_init = linalg.GenericOp( - [depth_type], - [bias_tensor], - [depth_init], - ir.ArrayAttr.get( - [ - ir.AffineMapAttr.get(identity.get_submap([3])), - ir.AffineMapAttr.get(identity), - ] - ), - ir.ArrayAttr.get( - [ir.Attribute.parse("#linalg.iterator_type")] * 5 - ), - ) - bias_block = ir.Block.create_at_start( - bias_init.region, [result_element_type, result_element_type] - ) - bias_init.region.blocks[0].append( - linalg.YieldOp([bias_block.arguments[0]]) - ) - depthwise_op = linalg.DepthwiseConv2DNhwcHwcmOp( - [depth_type], - [input_val, weight_depthwise], - [bias_init.result], - strides=stride_attr, - dilations=dilation_attr, - ) - linalg.fill_builtin_region(depthwise_op.operation) - depthwise_op.operation.attributes["dw_addr"] = ir.IntegerAttr.get( - ir.IntegerType.get_signless(64), node._dw_addr - ) - depthwise_op.operation.attributes["dw_bytes"] = ir.IntegerAttr.get( - ir.IntegerType.get_signless(64), node._dw_bytes - ) - depthwise_op.operation.attributes["per_channel"] = ir.BoolAttr.get( - node._per_channel - ) - collapse = tensor.CollapseShapeOp( - output_type, - depthwise_op.result, - _build_reassociation_attr([[0], [1], [2], [3, 4]]), - ).result - if extra_out_h or extra_out_w: - slice_start = _create_shape_operand([0, 0, 0, 0]) - slice_size = _create_shape_operand(out_shape) - slice_type = ir.RankedTensorType.get(out_shape, result_element_type) - return tosa.SliceOp(slice_type, collapse, slice_start, slice_size) - return collapse - input_zp = _create_zero_point_tensor(input_val) - weight_zp = _create_zero_point_tensor(weight_depthwise) - depthwise_op = tosa.DepthwiseConv2DOp( - output_type, - input_val, - weight_depthwise, - bias_tensor, - input_zp, - weight_zp, - input_padding_attr, - stride_attr, - dilation_attr, - acc_type, - ) - if hasattr(node, "_dw_addr"): - i64 = ir.IntegerType.get_signless(64) - depthwise_op.operation.attributes["dw_addr"] = ir.IntegerAttr.get( - i64, node._dw_addr - ) - depthwise_op.operation.attributes["dw_bytes"] = ir.IntegerAttr.get( - i64, node._dw_bytes - ) - depthwise_op.operation.attributes["per_channel"] = ir.BoolAttr.get( - node._per_channel - ) - op = depthwise_op - else: - # Transpose Conv2D Operation. - if is_kernel_transposed: - if sum(input_padding) > 0 or sum(dilation) > len(dilation): - raise NotImplementedError - for i in range(len(out_padding), 4): - out_padding = [0] + out_padding - out_padding_attr = ir._denseI64ArrayAttr(out_padding, None) - input_zp = _create_zero_point_tensor(input_val) - weight_zp = _create_zero_point_tensor(weight_val) - op = tosa.TransposeConv2DOp( - output_type, - input_val, - weight_val, - bias_tensor, - input_zp, - weight_zp, - out_padding_attr, - stride_attr, - acc_type, - ) - # Generic Conv2D Operation. - else: - if node._layout.find("FCHW") != -1: - perm_list = [0, 2, 3, 1] - perms_attr = _create_permutation_attr(perm_list) - perm_shape = [] - perm_shape.append(weight_shape[0]) - perm_shape.append(weight_shape[2]) - perm_shape.append(weight_shape[3]) - perm_shape.append(weight_shape[1]) - permute_result_type = ir.RankedTensorType.get( - perm_shape, - ir.RankedTensorType(weight_val.type).element_type, - ) - weight_val = tosa.TransposeOp( - permute_result_type, - weight_val, - perms_attr, - ).result - input_zp = _create_zero_point_tensor(input_val) - weight_zp = _create_zero_point_tensor(weight_val) - conv_op = tosa.Conv2DOp( - output_type, - input_val, - weight_val, - bias_tensor, - input_zp, - weight_zp, - input_padding_attr, - stride_attr, - dilation_attr, - acc_type, - ) - if hasattr(node, "_dw_addr"): - i64 = ir.IntegerType.get_signless(64) - conv_op.operation.attributes["dw_addr"] = ir.IntegerAttr.get( - i64, node._dw_addr - ) - conv_op.operation.attributes["dw_bytes"] = ir.IntegerAttr.get( - i64, node._dw_bytes - ) - conv_op.operation.attributes["per_channel"] = ir.BoolAttr.get( - node._per_channel - ) - op = conv_op - - if extra_out_h or extra_out_w: - # Slice off the padded rows/cols that exceed the expected output. - slice_start = _create_shape_operand([0, 0, 0, 0]) - slice_size = _create_shape_operand(out_shape) - slice_type = ir.RankedTensorType.get(out_shape, result_element_type) - slice_op = tosa.SliceOp(slice_type, op.result, slice_start, slice_size) - op = slice_op - # Convolution 1D - elif len(weight_shape) == 3: - # Prepare input with padding. - if input_padding[0] != 0: - input_shape = list(ir.RankedTensorType(input_val.type).shape) - padded_type = ir.RankedTensorType.get( - [ - input_shape[0], - input_shape[1], - input_shape[2] + 2 * input_padding[0], - ], - result_element_type, - ) - - pad_constant = _create_shape_operand( - [0, 0, 0, 0, input_padding[0], input_padding[0]] - ) - ty = ir.Type.parse("tensor<1xf32>") - pad_zp = tosa.ConstOp( - ir.DenseElementsAttr.get_splat(ty, ir.FloatAttr.get_f32(0.0)) - ).result - input_val = tosa.PadOp(padded_type, input_val, pad_constant, pad_zp) - output_type = ir.RankedTensorType.get(out_shape, result_element_type) - output_conv = _create_zero_tensor(output_type) - assert groups == 1, "only support one group" - # Con1D Operation Without Bias - conv_op = linalg.conv_1d_ncw_fcw( - input_val, - weight_val, - outs=[output_conv], - strides=stride_attr, - dilations=dilation_attr, - ) - output = tensor.EmptyOp(list(out_shape), result_element_type) - generic_map = ir.AffineMap.get_permutation( - [i for i in range(len(list(out_shape)))] - ) - loop_type = [ir.Attribute.parse("#linalg.iterator_type")] * len( - list(out_shape) - ) - loop_type[1] = ir.Attribute.parse("#linalg.iterator_type") - # Add Bias To Conv2d. - op = linalg.GenericOp( - [output_type], - [conv_op, bias_tensor], - [output], - ir.ArrayAttr.get( - [ - ir.AffineMapAttr.get( - generic_map.get_submap([i for i in range(len(list(out_shape)))]) - ), - ir.AffineMapAttr.get(generic_map.get_submap([1])), - ir.AffineMapAttr.get( - generic_map.get_submap([i for i in range(len(list(out_shape)))]) - ), - ] - ), - ir.ArrayAttr.get(loop_type), - ) - block = ir.Block.create_at_start( - op.region, - [ - result_element_type, - ir.RankedTensorType(bias_tensor.type).element_type, - result_element_type, - ], - ) - add_op = arith.AddFOp(block.arguments[1], block.arguments[0]) - block.append(add_op) - block.append(linalg.YieldOp([add_op.result])) - - return op - - -def silu_op(node: SiluOp, symbol_table): - """Preserve x / (1 + exp(-x)) rounding before activation quantization.""" - value = symbol_table[(str(node.args[0]), 0)] - if has_nhwc_layout(value): - value = _nhwc_to_nchw(value) - result_type = ir.RankedTensorType(value.type) - shape = list(result_type.shape) - dtype = result_type.element_type - identity = ir.AffineMapAttr.get(ir.AffineMap.get_identity(len(shape))) - empty = tensor.EmptyOp(shape, dtype) - op = linalg.GenericOp( - [result_type], - [value], - [empty.result], - ir.ArrayAttr.get([identity, identity]), - ir.ArrayAttr.get( - [ir.Attribute.parse("#linalg.iterator_type")] * len(shape) - ), - ) - block = ir.Block.create_at_start(op.region, [dtype, dtype]) - with ir.InsertionPoint(block): - argument = block.arguments[0] - opmath = ( - ir.F32Type.get() - if dtype in (ir.F16Type.get(), ir.BF16Type.get()) - else dtype - ) - if opmath != dtype: - argument = arith.ExtFOp(opmath, argument).result - negative = arith.NegFOp(argument).result - exponential = ( - exp_f32(negative) - if opmath == ir.F32Type.get() - else math.ExpOp(negative).result - ) - one = arith.ConstantOp(opmath, ir.FloatAttr.get(opmath, 1.0)).result - denominator = arith.AddFOp(one, exponential).result - result = arith.DivFOp(argument, denominator).result - if opmath != dtype: - result = arith.TruncFOp(dtype, result).result - linalg.YieldOp([result]) - return op - - -def mean_op(node: MeanOp, symbol_table): - """ - Import the buddy MeanOp. - From Buddy MeanOp to MLIR TOSA operation. - """ - - def _inner_op(result_type, input1, input2): - shift = _create_mul_shift_operand() - return tosa.MulOp(result_type, input1, input2, shift) - - input_tensor = symbol_table.get((str(node.args[0]), 0)) - keepdim = node.args[2] - dims = [int(x) for x in node.args[1]] - if isinstance(dims, int): - dims = [dims] - - act = [int(x) for x in ir.RankedTensorType(input_tensor.type).shape] - for i, d in enumerate(dims): - if d < 0: - dims[i] = d + len(act) - - # PyTorch dims are NCHW; remap when SSA stuck in NHWC. - out_meta = [int(x) for x in node.tensor_meta["shape"]] - if len(out_meta) == 4 or len(out_meta) == 2: - c_meta = out_meta[1] - elif len(out_meta) == 1: - c_meta = out_meta[0] - else: - c_meta = None - if c_meta is not None and len(act) == 4 and act[3] == c_meta and act[1] != c_meta: - dims = [{0: 0, 1: 3, 2: 1, 3: 2}[d] for d in dims] - elif c_meta is not None and len(act) == 4 and act[1] != c_meta: - raise RuntimeError(f"mean layout mismatch: ssa={act} out_meta={out_meta}") - - reduce_sum_result = input_tensor - for dim_item in dims: - reduce_dim_attr = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), dim_item) - reduce_sum_op = tosa.ReduceSumOp(reduce_sum_result, reduce_dim_attr) - reduce_sum_result = reduce_sum_op.results[0] - - tensor_shp = ir.RankedTensorType(input_tensor.type).shape - dim_size = 1 - - for dim_item in dims: - dim_size *= tensor_shp[dim_item] - - denominator_const_op = tosa.ConstOp( - ir.DenseElementsAttr.get(memoryview(array.array("f", [dim_size]))) - ) - reciprocal_op = tosa.ReciprocalOp( - denominator_const_op.results[0].type, denominator_const_op - ) - ret = _gen_arith_binary_op( - reciprocal_op.results[0], reduce_sum_op.results[0], _inner_op - ) - - if not keepdim: - result_shp = ir.RankedTensorType(ret.results[0].type).shape - result_shp = [siz for siz in result_shp if siz != 1] - reshape_operand = _create_shape_operand(result_shp) - ret = tosa.ReshapeOp(ret.results[0], reshape_operand) - - return ret - - -def alias_op(node: AliasOp, symbol_table): - """ - Import the alias operation. - From buddy graph ir's `AliasOp` operator to MLIR TOSA operations. - Creates an alias (view) of the input tensor. - aten.alias.default(input) -> Tensor - """ - input1 = symbol_table.get((str(node.args[0]), 0)) - - input_shape = list(ir.RankedTensorType(input1.type).shape) - input_dtype = ir.RankedTensorType(input1.type).element_type - - output_shape = list(node.tensor_meta["shape"]) - output_dtype = mlir_element_type_get(node.tensor_meta["dtype"]) - - if input_shape == output_shape and input_dtype == output_dtype: - return input1 - - # Sticky NHWC: alias must not invent NCHW type via identity. - if len(input_shape) == 4 and len(output_shape) == 4: - ns = [int(x) for x in input_shape] - nw = [int(x) for x in output_shape] - if ns == [nw[0], nw[2], nw[3], nw[1]]: - return input1 - if nw == [ns[0], ns[2], ns[3], ns[1]]: - out_ty = ir.RankedTensorType.get(nw, output_dtype) - return tosa.TransposeOp( - out_ty, input1, _create_permutation_attr([0, 2, 3, 1]) - ).result - - raise RuntimeError( - f"alias shape/layout mismatch: ssa={input_shape} meta={output_shape}" - ) - - -def _bn_channel_and_broadcast(input_tensor, channel_param): - shape = [int(x) for x in ir.RankedTensorType(input_tensor.type).shape] - c = int(list(ir.RankedTensorType(channel_param.type).shape)[0]) - if len(shape) == 4: - if has_nhwc_layout(input_tensor): - return c, [1, 1, 1, c] - if shape[1] == c: - return c, [1, c, 1, 1] - if shape[3] == c: - return c, [1, 1, 1, c] - raise RuntimeError(f"BN channel {c} mismatch with 4D shape {shape}") - if len(shape) >= 2: - if shape[1] == c: - return c, [1, c] + [1] * (len(shape) - 2) - if shape[-1] == c: - return c, [1] * (len(shape) - 1) + [c] - raise RuntimeError(f"BN channel {c} mismatch with shape {shape}") - - -def native_batch_norm_legit_op(node, symbol_table): - """ - Import the native batch norm legit operation. - From buddy graph ir's `NativeBatchNormLegitOp` operator to MLIR TOSA operations. - aten._native_batch_norm_legit(input, weight, bias, running_mean, running_var, training, momentum, eps) - -> (Tensor, Tensor, Tensor) - - Returns a tuple of (output, save_mean, save_invstd) where: - - output: normalized tensor with same shape as input - - save_mean: mean per channel, shape (C,) - - save_invstd: inverse std per channel, shape (C,) - - Batch normalization formula: - output = (input - mean) / sqrt(var + eps) * weight + bias - - During training: uses batch statistics - During inference: uses running_mean and running_var - """ - input_tensor = symbol_table.get((str(node.args[0]), 0)) - weight = ( - symbol_table.get((str(node.args[1]), 0)) if node.args[1] is not None else None - ) - bias = ( - symbol_table.get((str(node.args[2]), 0)) if node.args[2] is not None else None - ) - running_mean = ( - symbol_table.get((str(node.args[3]), 0)) if node.args[3] is not None else None - ) - running_var = ( - symbol_table.get((str(node.args[4]), 0)) if node.args[4] is not None else None - ) - training = node.args[5] - momentum = node.args[6] - eps = node.args[7] - - input_shape = list(ir.RankedTensorType(input_tensor.type).shape) - input_dtype = ir.RankedTensorType(input_tensor.type).element_type - - channel_ref = running_mean or weight or bias - if channel_ref is None: - raise RuntimeError("batch norm needs running_mean, weight, or bias") - C, broadcast_shape = _bn_channel_and_broadcast(input_tensor, channel_ref) - N = input_shape[0] - spatial_dims = input_shape[2:] - spatial_size = 1 - for d in spatial_dims: - spatial_size *= d - - if not training and running_mean is not None and running_var is not None: - # Inference mode: use running statistics - mean = running_mean - var = running_var - - scalar_broadcast_shape = [1] * len(input_shape) # for scalars like eps - - mean_broadcast = tosa.ReshapeOp( - mean, _create_shape_operand(broadcast_shape) - ).result - var_broadcast = tosa.ReshapeOp( - var, _create_shape_operand(broadcast_shape) - ).result - - # (input - mean) - centered = tosa.SubOp( - ir.RankedTensorType.get(input_shape, input_dtype), - input_tensor, - mean_broadcast, - ).result - - # var + eps - reshape eps to same rank for TOSA compatibility - eps_attr = ir.DenseElementsAttr.get( - memoryview(array.array("f", [eps])), - type=ir.RankedTensorType.get([1], input_dtype), - ) - eps_tensor = tosa.ConstOp(eps_attr).result - eps_broadcast = tosa.ReshapeOp( - eps_tensor, _create_shape_operand(scalar_broadcast_shape) - ).result - - var_plus_eps = tosa.AddOp( - ir.RankedTensorType.get(broadcast_shape, input_dtype), - var_broadcast, - eps_broadcast, - ).result - - # rsqrt(var + eps) - invstd = tosa.RsqrtOp( - ir.RankedTensorType.get(broadcast_shape, input_dtype), var_plus_eps - ).result - - # normalized = (input - mean) * invstd - normalized = tosa.MulOp( - ir.RankedTensorType.get(input_shape, input_dtype), centered, invstd - ).result - - output = normalized - - # Apply weight (gamma) - if weight is not None: - weight_broadcast = tosa.ReshapeOp( - weight, _create_shape_operand(broadcast_shape) - ).result - output = tosa.MulOp( - ir.RankedTensorType.get(input_shape, input_dtype), - output, - weight_broadcast, - ).result - - # Apply bias (beta) - if bias is not None: - bias_broadcast = tosa.ReshapeOp( - bias, _create_shape_operand(broadcast_shape) - ).result - output = tosa.AddOp( - ir.RankedTensorType.get(input_shape, input_dtype), - output, - bias_broadcast, - ).result - - # save_mean and save_invstd with shape (C,) - save_mean = tosa.ReshapeOp(mean_broadcast, _create_shape_operand([C])).result - save_invstd = tosa.ReshapeOp(invstd, _create_shape_operand([C])).result - - return output, save_mean, save_invstd - - # Training mode: compute batch statistics - train_input = input_tensor - train_shape = list(input_shape) - if len(train_shape) == 4 and train_shape[3] == C: - nchw_shape = [train_shape[0], C, train_shape[1], train_shape[2]] - train_input = tosa.TransposeOp( - ir.RankedTensorType.get(nchw_shape, input_dtype), - input_tensor, - _create_permutation_attr([0, 3, 1, 2]), - ).result - train_shape = nchw_shape - elif not (len(train_shape) >= 2 and train_shape[1] == C): - raise RuntimeError(f"BN training layout mismatch: shape={input_shape} C={C}") - N = train_shape[0] - spatial_size = 1 - for d in train_shape[2:]: - spatial_size *= d - reshaped_input = tosa.ReshapeOp( - train_input, _create_shape_operand([N, C, spatial_size]) - ).result - - # Compute mean: reduce over N and spatial dimensions - # First reduce over spatial (axis=2) - axis_attr_2 = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), 2) - spatial_sum = tosa.ReduceSumOp(reshaped_input, axis_attr_2).results[0] # (N, C, 1) - - # Then reduce over batch (axis=0) - axis_attr_0 = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), 0) - batch_spatial_sum = tosa.ReduceSumOp(spatial_sum, axis_attr_0).results[ - 0 - ] # (1, C, 1) - - # mean = sum / (N * spatial_size) - reshape divisor for TOSA compatibility - divisor = N * spatial_size - divisor_attr = ir.DenseElementsAttr.get( - memoryview(array.array("f", [1.0 / divisor])), - type=ir.RankedTensorType.get([1], input_dtype), - ) - divisor_tensor = tosa.ConstOp(divisor_attr).result - divisor_3d = tosa.ReshapeOp(divisor_tensor, _create_shape_operand([1, 1, 1])).result - - mean_3d = tosa.MulOp( - ir.RankedTensorType.get([1, C, 1], input_dtype), - batch_spatial_sum, - divisor_3d, - ).result - - # (x - mean) - diff = tosa.SubOp( - ir.RankedTensorType.get([N, C, spatial_size], input_dtype), - reshaped_input, - mean_3d, - ).result - - # (x - mean)^2 - squared = tosa.MulOp( - ir.RankedTensorType.get([N, C, spatial_size], input_dtype), diff, diff - ).result - - # sum of squared - sq_spatial_sum = tosa.ReduceSumOp(squared, axis_attr_2).results[0] # (N, C, 1) - sq_batch_sum = tosa.ReduceSumOp(sq_spatial_sum, axis_attr_0).results[0] # (1, C, 1) - - # variance = sum / (N * spatial_size) - var_3d = tosa.MulOp( - ir.RankedTensorType.get([1, C, 1], input_dtype), - sq_batch_sum, - divisor_3d, - ).result - - # var + eps - reshape eps for TOSA compatibility - eps_attr = ir.DenseElementsAttr.get( - memoryview(array.array("f", [eps])), - type=ir.RankedTensorType.get([1], input_dtype), - ) - eps_tensor = tosa.ConstOp(eps_attr).result - eps_3d = tosa.ReshapeOp(eps_tensor, divisor_3d_shape_operand).result - - var_plus_eps = tosa.AddOp( - ir.RankedTensorType.get([1, C, 1], input_dtype), var_3d, eps_3d - ).result - - # invstd = rsqrt(var + eps) - invstd_3d = tosa.RsqrtOp( - ir.RankedTensorType.get([1, C, 1], input_dtype), var_plus_eps - ).result - - # normalized = (x - mean) * invstd - normalized_3d = tosa.MulOp( - ir.RankedTensorType.get([N, C, spatial_size], input_dtype), - diff, - invstd_3d, - shift, - ).result - - # Reshape back to original shape - train_shape_operand = _create_shape_operand(train_shape) - output = tosa.ReshapeOp(normalized_3d, train_shape_operand).result - if list(input_shape) != train_shape: - output = tosa.TransposeOp( - ir.RankedTensorType.get(input_shape, input_dtype), - output, - _create_permutation_attr([0, 2, 3, 1]), - ).result - - # Apply weight (gamma) - if weight is not None: - broadcast_shape_operand = _create_shape_operand(broadcast_shape) - weight_broadcast = tosa.ReshapeOp(weight, broadcast_shape_operand).result - output = tosa.MulOp( - ir.RankedTensorType.get(input_shape, input_dtype), - output, - weight_broadcast, - shift, - ).result - - # Apply bias (beta) - if bias is not None: - broadcast_shape_operand = _create_shape_operand(broadcast_shape) - bias_broadcast = tosa.ReshapeOp(bias, broadcast_shape_operand).result - output = tosa.AddOp( - ir.RankedTensorType.get(input_shape, input_dtype), - output, - bias_broadcast, - ).result - - # Return (output, save_mean, save_invstd) - c_shape_operand = _create_shape_operand([C]) - save_mean = tosa.ReshapeOp(mean_3d, c_shape_operand).result - save_invstd = tosa.ReshapeOp(invstd_3d, c_shape_operand).result - - return output, save_mean, save_invstd - - -def native_batch_norm_legit_no_stats_op(node, symbol_table): - """ - Import the native batch norm legit no_stats operation. - From buddy graph ir's `NativeBatchNormLegitNoStatsOp` operator to MLIR TOSA operations. - aten._native_batch_norm_legit.no_stats(input, weight, bias, training, momentum, eps) - -> (Tensor, Tensor, Tensor) - - This variant doesn't use running statistics, always computes from batch. - Returns: (output, save_mean, save_invstd) - """ - input_tensor = symbol_table.get((str(node.args[0]), 0)) - weight = ( - symbol_table.get((str(node.args[1]), 0)) if node.args[1] is not None else None - ) - bias = ( - symbol_table.get((str(node.args[2]), 0)) if node.args[2] is not None else None - ) - training = node.args[3] - momentum = node.args[4] - eps = node.args[5] - - input_shape = list(ir.RankedTensorType(input_tensor.type).shape) - input_dtype = ir.RankedTensorType(input_tensor.type).element_type - - channel_ref = weight or bias - if channel_ref is None: - raise RuntimeError("batch norm no_stats needs weight or bias") - C, broadcast_shape = _bn_channel_and_broadcast(input_tensor, channel_ref) - - train_input = input_tensor - train_shape = list(input_shape) - if len(train_shape) == 4 and train_shape[3] == C: - nchw_shape = [train_shape[0], C, train_shape[1], train_shape[2]] - train_input = tosa.TransposeOp( - ir.RankedTensorType.get(nchw_shape, input_dtype), - input_tensor, - _create_permutation_attr([0, 3, 1, 2]), - ).result - train_shape = nchw_shape - elif not (len(train_shape) >= 2 and train_shape[1] == C): - raise RuntimeError(f"BN no_stats layout mismatch: shape={input_shape} C={C}") - N = train_shape[0] - spatial_size = 1 - for d in train_shape[2:]: - spatial_size *= d - - reshaped_shape_operand = _create_shape_operand([N, C, spatial_size]) - reshaped_input = tosa.ReshapeOp(train_input, reshaped_shape_operand).result - - # Compute mean: reduce over N and spatial dimensions - axis_attr_2 = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), 2) - spatial_sum = tosa.ReduceSumOp(reshaped_input, axis_attr_2).results[0] # (N, C, 1) - - axis_attr_0 = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), 0) - batch_spatial_sum = tosa.ReduceSumOp(spatial_sum, axis_attr_0).results[ - 0 - ] # (1, C, 1) - - # mean = sum / (N * spatial_size) - divisor = N * spatial_size - divisor_attr = ir.DenseElementsAttr.get( - memoryview(array.array("f", [1.0 / divisor])), - type=ir.RankedTensorType.get([1], input_dtype), - ) - divisor_tensor = tosa.ConstOp(divisor_attr).result - divisor_3d_shape_operand = _create_shape_operand([1, 1, 1]) - divisor_3d = tosa.ReshapeOp(divisor_tensor, divisor_3d_shape_operand).result - - shift = _create_mul_shift_operand() - mean_3d = tosa.MulOp( - ir.RankedTensorType.get([1, C, 1], input_dtype), - batch_spatial_sum, - divisor_3d, - shift, - ).result - - # (x - mean) - diff = tosa.SubOp( - ir.RankedTensorType.get([N, C, spatial_size], input_dtype), - reshaped_input, - mean_3d, - ).result - - # (x - mean)^2 - squared = tosa.MulOp( - ir.RankedTensorType.get([N, C, spatial_size], input_dtype), - diff, - diff, - shift, - ).result - - # sum of squared - sq_spatial_sum = tosa.ReduceSumOp(squared, axis_attr_2).results[0] # (N, C, 1) - sq_batch_sum = tosa.ReduceSumOp(sq_spatial_sum, axis_attr_0).results[0] # (1, C, 1) - - # variance = sum / (N * spatial_size) - var_3d = tosa.MulOp( - ir.RankedTensorType.get([1, C, 1], input_dtype), - sq_batch_sum, - divisor_3d, - shift, - ).result - - # var + eps - reshape eps to match [1, C, 1] shape for TOSA compatibility - eps_attr = ir.DenseElementsAttr.get( - memoryview(array.array("f", [eps])), - type=ir.RankedTensorType.get([1], input_dtype), - ) - eps_tensor = tosa.ConstOp(eps_attr).result - eps_3d = tosa.ReshapeOp(eps_tensor, divisor_3d_shape_operand).result - - var_plus_eps = tosa.AddOp( - ir.RankedTensorType.get([1, C, 1], input_dtype), var_3d, eps_3d - ).result - - # invstd = rsqrt(var + eps) - invstd_3d = tosa.RsqrtOp( - ir.RankedTensorType.get([1, C, 1], input_dtype), var_plus_eps - ).result - - # normalized = (x - mean) * invstd - normalized_3d = tosa.MulOp( - ir.RankedTensorType.get([N, C, spatial_size], input_dtype), - diff, - invstd_3d, - shift, - ).result - - # Reshape back to original shape - train_shape_operand = _create_shape_operand(train_shape) - output = tosa.ReshapeOp(normalized_3d, train_shape_operand).result - if list(input_shape) != train_shape: - output = tosa.TransposeOp( - ir.RankedTensorType.get(input_shape, input_dtype), - output, - _create_permutation_attr([0, 2, 3, 1]), - ).result - - # Apply weight (gamma) - if weight is not None: - broadcast_shape_operand = _create_shape_operand(broadcast_shape) - weight_broadcast = tosa.ReshapeOp(weight, broadcast_shape_operand).result - output = tosa.MulOp( - ir.RankedTensorType.get(input_shape, input_dtype), - output, - weight_broadcast, - shift, - ).result - - # Apply bias (beta) - if bias is not None: - broadcast_shape_operand = _create_shape_operand(broadcast_shape) - bias_broadcast = tosa.ReshapeOp(bias, broadcast_shape_operand).result - output = tosa.AddOp( - ir.RankedTensorType.get(input_shape, input_dtype), - output, - bias_broadcast, - ).result - - # Return (output, save_mean, save_invstd) - c_shape_operand = _create_shape_operand([C]) - save_mean = tosa.ReshapeOp(mean_3d, c_shape_operand).result - save_invstd = tosa.ReshapeOp(invstd_3d, c_shape_operand).result - - return output, save_mean, save_invstd - - -def native_batch_norm_legit_no_training_op(node, symbol_table): - """ - Import the native batch norm legit no_training operation. - From buddy graph ir's `NativeBatchNormLegitNoTrainingOp` operator to MLIR TOSA operations. - aten._native_batch_norm_legit_no_training(input, weight, bias, running_mean, running_var, momentum, eps) - -> (Tensor, Tensor, Tensor) - - This variant is inference only, always uses running statistics. - Returns: (output, save_mean, save_invstd) - """ - input_tensor = symbol_table.get((str(node.args[0]), 0)) - weight = ( - symbol_table.get((str(node.args[1]), 0)) if node.args[1] is not None else None - ) - bias = ( - symbol_table.get((str(node.args[2]), 0)) if node.args[2] is not None else None - ) - running_mean = symbol_table.get((str(node.args[3]), 0)) - running_var = symbol_table.get((str(node.args[4]), 0)) - momentum = node.args[5] - eps = node.args[6] - - input_shape = list(ir.RankedTensorType(input_tensor.type).shape) - input_dtype = ir.RankedTensorType(input_tensor.type).element_type - - channel_ref = running_mean or weight or bias - if channel_ref is None: - raise RuntimeError("batch norm needs running_mean, weight, or bias") - C, broadcast_shape = _bn_channel_and_broadcast(input_tensor, channel_ref) - - # Inference mode: use running statistics - scalar_broadcast_shape = [1] * len(input_shape) # for scalars like eps - - broadcast_shape_operand = _create_shape_operand(broadcast_shape) - mean_broadcast = tosa.ReshapeOp(running_mean, broadcast_shape_operand).result - var_broadcast = tosa.ReshapeOp(running_var, broadcast_shape_operand).result - - # var + eps - reshape eps to same rank for TOSA compatibility - eps_attr = ir.DenseElementsAttr.get( - memoryview(array.array("f", [eps])), - type=ir.RankedTensorType.get([1], input_dtype), - ) - eps_tensor = tosa.ConstOp(eps_attr).result - scalar_broadcast_shape_operand = _create_shape_operand(scalar_broadcast_shape) - eps_broadcast = tosa.ReshapeOp(eps_tensor, scalar_broadcast_shape_operand).result - - var_plus_eps = tosa.AddOp( - ir.RankedTensorType.get(broadcast_shape, input_dtype), - var_broadcast, - eps_broadcast, - ).result - - # PyTorch computes 1 / sqrt(var + eps). An approximate reciprocal - # square root can cross an activation quantization threshold on NVIDIA. - rank = len(input_shape) - identity = ir.AffineMap.get_identity(rank) - channel_type = ir.RankedTensorType.get(broadcast_shape, input_dtype) - inverse_empty = tensor.EmptyOp(broadcast_shape, input_dtype) - inverse = linalg.GenericOp( - [channel_type], - [var_plus_eps], - [inverse_empty.result], - ir.ArrayAttr.get([ir.AffineMapAttr.get(identity)] * 2), - ir.ArrayAttr.get( - [ir.Attribute.parse("#linalg.iterator_type")] * rank - ), - ) - inverse_block = ir.Block.create_at_start(inverse.region, [input_dtype] * 2) - with ir.InsertionPoint(inverse_block): - root = math.SqrtOp(inverse_block.arguments[0]).result - one = arith.ConstantOp(input_dtype, ir.FloatAttr.get(input_dtype, 1.0)).result - reciprocal = arith.DivFOp(one, root).result - linalg.YieldOp([reciprocal]) - invstd = inverse.result - - # Match PyTorch inference: alpha = invstd * gamma, - # beta = bias - mean * alpha, output = input * alpha + beta. - # Reassociating this into (input - mean) * invstd * gamma + bias - # changes FP32 rounding before downstream activation quantization. - alpha = invstd - if weight is not None: - gamma = tosa.ReshapeOp(weight, broadcast_shape_operand).result - alpha = _gen_arith_binary_op(invstd, gamma, tosa.MulOp).result - if bias is None: - mean_scaled = _gen_arith_binary_op(mean_broadcast, alpha, tosa.MulOp).result - beta = tosa.NegateOp( - mean_scaled.type, - mean_scaled, - _create_zero_point_tensor(mean_scaled), - _create_zero_point_tensor(mean_scaled), - ).result - else: - bias_broadcast = tosa.ReshapeOp(bias, broadcast_shape_operand).result - beta_empty = tensor.EmptyOp(broadcast_shape, input_dtype) - beta_op = linalg.GenericOp( - [channel_type], - [mean_broadcast, alpha, bias_broadcast], - [beta_empty.result], - ir.ArrayAttr.get([ir.AffineMapAttr.get(identity)] * 4), - ir.ArrayAttr.get( - [ir.Attribute.parse("#linalg.iterator_type")] * rank - ), - ) - beta_block = ir.Block.create_at_start(beta_op.region, [input_dtype] * 4) - with ir.InsertionPoint(beta_block): - negative_mean = arith.NegFOp(beta_block.arguments[0]).result - coefficient = math.FmaOp( - negative_mean, beta_block.arguments[1], beta_block.arguments[2] - ).result - linalg.YieldOp([coefficient]) - beta = beta_op.result - # Keep the final multiply-add fused: rounding the product separately - # changes values close to downstream INT8 rounding thresholds. - channel_map = ir.AffineMap.get( - rank, - 0, - [ - (ir.AffineExpr.get_dim(i) if extent != 1 else ir.AffineExpr.get_constant(0)) - for i, extent in enumerate(broadcast_shape) - ], - ) - result_type = ir.RankedTensorType.get(input_shape, input_dtype) - empty = tensor.EmptyOp(input_shape, input_dtype) - fused = linalg.GenericOp( - [result_type], - [input_tensor, alpha, beta], - [empty.result], - ir.ArrayAttr.get( - [ - ir.AffineMapAttr.get(m) - for m in (identity, channel_map, channel_map, identity) - ] - ), - ir.ArrayAttr.get( - [ir.Attribute.parse("#linalg.iterator_type")] * rank - ), - ) - block = ir.Block.create_at_start(fused.region, [input_dtype] * 4) - with ir.InsertionPoint(block): - value = math.FmaOp(*block.arguments[:3]).result - linalg.YieldOp([value]) - if has_nhwc_layout(input_tensor): - fused.operation.attributes["buddy.layout"] = ir.StringAttr.get("nhwc") - output = fused.result - - # save_mean and save_invstd with shape (C,) - c_shape_operand = _create_shape_operand([C]) - save_mean = tosa.ReshapeOp(mean_broadcast, c_shape_operand).result - save_invstd = tosa.ReshapeOp(invstd, c_shape_operand).result - - return output, save_mean, save_invstd - - -def upsample_bilinear2d_vec_op(node, symbol_table): - """ - Import the upsample_bilinear2d.vec operation. - From buddy graph ir's `UpsampleBilinear2dVecOp` operator to MLIR TOSA operations. - aten.upsample_bilinear2d.vec(input, output_size, align_corners, scale_factors) -> Tensor - - Uses TOSA resize operation with BILINEAR mode. - Input: NCHW format, needs to be converted to NHWC for TOSA. - """ - input_tensor = symbol_table.get((str(node.args[0]), 0)) - output_size = node.args[1] # [H_out, W_out] or None - align_corners = node.args[2] - scale_factors = ( - node.args[3] if len(node.args) > 3 else None - ) # [scale_h, scale_w] or None - - input_shape = list(ir.RankedTensorType(input_tensor.type).shape) - input_dtype = ir.RankedTensorType(input_tensor.type).element_type - - channels = int(list(node.tensor_meta["shape"])[1]) - src = node.args[0] - nchw_meta = ( - list(src.tensor_meta["shape"]) - if hasattr(src, "tensor_meta") and "shape" in src.tensor_meta - else None - ) - input_tensor, N, H_in, W_in, C = _to_nhwc( - input_tensor, channels, input_dtype, nchw_meta - ) - - # Determine output size - if output_size is not None: - H_out, W_out = output_size - elif scale_factors is not None: - H_out = int(H_in * scale_factors[0]) - W_out = int(W_in * scale_factors[1]) - else: - raise ValueError("Either output_size or scale_factors must be provided") - - # Calculate scale for TOSA resize - # TOSA scale format: [scale_y_n, scale_y_d, scale_x_n, scale_x_d] - # For exact output size: scale = output_size / input_size - scale_y_n = H_out - scale_y_d = H_in - scale_x_n = W_out - scale_x_d = W_in - - scale_attr = ir._denseI64ArrayAttr( - [scale_y_n, scale_y_d, scale_x_n, scale_x_d], None - ) - offset_attr = ir._denseI64ArrayAttr([0, 0], None) - border_attr = ir._denseI64ArrayAttr([0, 0], None) - mode_attr = ir.StringAttr.get("BILINEAR") - - # Apply resize in NHWC format; keep NHWC output. - nhwc_out_type = ir.RankedTensorType.get([N, H_out, W_out, C], input_dtype) - resized = tosa.ResizeOp( - nhwc_out_type, - input_tensor, - scale_attr, - offset_attr, - border_attr, - mode_attr, - ) - - return resized - - -def upsample_nearest2d_vec_op(node, symbol_table): - """ - Import the upsample_nearest2d.vec operation. - From buddy graph ir's `UpsampleNearest2dVecOp` operator to MLIR TOSA operations. - aten.upsample_nearest2d.vec(input, output_size, scale_factors) -> Tensor - - Uses TOSA resize operation with NEAREST mode. - """ - input_tensor = symbol_table.get((str(node.args[0]), 0)) - output_size = node.args[1] # [H_out, W_out] or None - scale_factors = ( - node.args[2] if len(node.args) > 2 else None - ) # [scale_h, scale_w] or None - - input_dtype = ir.RankedTensorType(input_tensor.type).element_type - - channels = int(list(node.tensor_meta["shape"])[1]) - src = node.args[0] - nchw_meta = ( - list(src.tensor_meta["shape"]) - if hasattr(src, "tensor_meta") and "shape" in src.tensor_meta - else None - ) - input_tensor, N, H_in, W_in, C = _to_nhwc( - input_tensor, channels, input_dtype, nchw_meta - ) - - # Determine output size - if output_size is not None: - H_out, W_out = output_size - elif scale_factors is not None: - H_out = int(H_in * scale_factors[0]) - W_out = int(W_in * scale_factors[1]) - else: - raise ValueError("Either output_size or scale_factors must be provided") - - scale_attr = ir._denseI64ArrayAttr([H_out, H_in, W_out, W_in], None) - offset_attr = ir._denseI64ArrayAttr([0, 0], None) - border_attr = ir._denseI64ArrayAttr([0, 0], None) - mode_attr = ir.StringAttr.get("NEAREST") - - nhwc_out_type = ir.RankedTensorType.get([N, H_out, W_out, C], input_dtype) - return tosa.ResizeOp( - nhwc_out_type, - input_tensor, - scale_attr, - offset_attr, - border_attr, - mode_attr, - ) - - -from buddy.compiler.ops.tosa import convert_element_type_op, where_op -from buddy.compiler.ops.tosa import ops_registry as upstream_registry -from .linalg import ops_registry as linalg_registry - -ops_registry = { - **linalg_registry, - **upstream_registry, - "WhereOp": where_op, - "ToCopyOp": convert_element_type_op, - "AddOp": add_op, - "MulOp": mul_op, - "SubOp": sub_op, - "AmaxOp": amax_op, - "DivOp": div_op, - "ReshapeOp": reshape_op, - "ViewOp": reshape_op, - "MaxPool2dOp": maxpool2d_op, - "AvgPool2dOp": avg_pool2d_op, - "AdaptiveAvgPool2dOp": adaptive_avg_pool2d_op, - "Conv2dOp": convolution2d_op, - "Int8Conv2dOp": int8_conv2d_op, - "Int8MatmulOp": int8_matmul_op, - "Int8QuantizeOp": int8_quantize_op, - "SiluOp": silu_op, - "MeanOp": mean_op, - "AliasOp": alias_op, - "NativeBatchNormLegitOp": native_batch_norm_legit_op, - "NativeBatchNormLegitNoStatsOp": native_batch_norm_legit_no_stats_op, - "NativeBatchNormLegitNoTrainingOp": native_batch_norm_legit_no_training_op, - "UpsampleBilinear2dVecOp": upsample_bilinear2d_vec_op, - "UpsampleNearest2dVecOp": upsample_nearest2d_vec_op, -} diff --git a/waste/stack/compiler/frontend/quant/__init__.py b/waste/stack/compiler/frontend/quant/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/waste/stack/compiler/frontend/quant/bwq.py b/waste/stack/compiler/frontend/quant/bwq.py deleted file mode 100644 index 4329abaeb..000000000 --- a/waste/stack/compiler/frontend/quant/bwq.py +++ /dev/null @@ -1,163 +0,0 @@ -from __future__ import annotations - -import tomllib -from dataclasses import dataclass -from pathlib import Path - - -@dataclass -class TensorSpec: - name: str - shape: list[int] - storage: str - axes: list[int] - weight_off: int - weight_len: int - scale_off: int - scale_len: int - - -@dataclass -class BwqPackage: - version: int - tensors: list[TensorSpec] - weights: bytes - scales: bytes - - -_FORBIDDEN = ("groupSize", "group_size", "zeroPoint", "zero_point") -_STORAGE = {"i8"} - - -def _numel(shape: list[int], name: str) -> int: - n = 1 - for d in shape: - if d <= 0: - raise ValueError(f"bad shape for {name}") - n *= d - return n - - -def validate_bwq(pkg: BwqPackage) -> None: - if pkg.version != 1: - raise ValueError(f"unsupported version: {pkg.version}") - if not pkg.tensors: - raise ValueError("empty tensor list") - seen: set[str] = set() - for t in pkg.tensors: - if t.name in seen: - raise ValueError(f"duplicate tensor name: {t.name}") - seen.add(t.name) - if t.storage not in _STORAGE: - raise ValueError(f"unsupported storage {t.storage!r} for {t.name}") - if t.weight_off < 0 or t.weight_len < 0: - raise ValueError(f"bad weight range for {t.name}") - if t.weight_off + t.weight_len > len(pkg.weights): - raise ValueError(f"weight OOB for {t.name}") - if t.scale_off < 0 or t.scale_len < 0: - raise ValueError(f"bad scale range for {t.name}") - if t.scale_off + t.scale_len > len(pkg.scales): - raise ValueError(f"scale OOB for {t.name}") - n = _numel(t.shape, t.name) - for ax in t.axes: - if ax < 0 or ax >= len(t.shape): - raise ValueError(f"axes OOB for {t.name}") - if t.weight_len != n: - raise ValueError(f"weight_len {t.weight_len} != numel {n} for {t.name}") - if t.axes: - scale_n = 1 - for ax in t.axes: - scale_n *= t.shape[ax] - if t.scale_len != scale_n * 4: - raise ValueError(f"scale_len mismatch for {t.name}") - elif t.scale_len != 4: - raise ValueError(f"per-tensor scale_len must be 4 for {t.name}") - - -def write_bwq(pkg: BwqPackage, path: Path) -> None: - validate_bwq(pkg) - path = Path(path) - if path.exists() and not path.is_dir(): - raise ValueError(f"not a directory: {path}") - path.mkdir(parents=True, exist_ok=True) - lines = [f"version = {pkg.version}", ""] - for t in pkg.tensors: - lines += [ - "[[tensor]]", - f'name = "{t.name}"', - f"shape = [{', '.join(str(x) for x in t.shape)}]", - f'storage = "{t.storage}"', - f"axes = [{', '.join(str(x) for x in t.axes)}]", - f"weight_off = {t.weight_off}", - f"weight_len = {t.weight_len}", - f"scale_off = {t.scale_off}", - f"scale_len = {t.scale_len}", - "", - ] - (path / "manifest.toml").write_text("\n".join(lines), encoding="utf-8") - (path / "weights.bin").write_bytes(pkg.weights) - (path / "scales.bin").write_bytes(pkg.scales) - - -def read_bwq(path: Path) -> BwqPackage: - path = Path(path) - if not path.is_dir(): - raise ValueError(f"bwq path is not a directory: {path}") - manifest = path / "manifest.toml" - if not manifest.is_file(): - raise ValueError(f"missing manifest.toml in {path}") - try: - meta = tomllib.loads(manifest.read_bytes().decode("utf-8")) - except tomllib.TOMLDecodeError as e: - raise ValueError(f"invalid manifest.toml: {e}") from e - except UnicodeDecodeError as e: - raise ValueError(f"invalid manifest.toml encoding: {e}") from e - for key in _FORBIDDEN: - if key in meta: - raise ValueError(f"forbidden key at top level: {key}") - if "version" not in meta: - raise ValueError("missing field version") - tensors: list[TensorSpec] = [] - for entry in meta.get("tensor", []): - for key in _FORBIDDEN: - if key in entry and entry[key] is not None: - raise ValueError(f"forbidden key {key} on tensor {entry.get('name')}") - need = ( - "name", - "shape", - "storage", - "axes", - "weight_off", - "weight_len", - "scale_off", - "scale_len", - ) - for k in need: - if k not in entry: - raise ValueError(f"missing field {k}") - tensors.append( - TensorSpec( - name=entry["name"], - shape=list(entry["shape"]), - storage=entry["storage"], - axes=list(entry["axes"]), - weight_off=int(entry["weight_off"]), - weight_len=int(entry["weight_len"]), - scale_off=int(entry["scale_off"]), - scale_len=int(entry["scale_len"]), - ) - ) - weights_path = path / "weights.bin" - scales_path = path / "scales.bin" - if not weights_path.is_file(): - raise ValueError(f"missing weights.bin in {path}") - if not scales_path.is_file(): - raise ValueError(f"missing scales.bin in {path}") - pkg = BwqPackage( - version=int(meta["version"]), - tensors=tensors, - weights=weights_path.read_bytes(), - scales=scales_path.read_bytes(), - ) - validate_bwq(pkg) - return pkg diff --git a/waste/stack/compiler/frontend/quant/importer.py b/waste/stack/compiler/frontend/quant/importer.py deleted file mode 100644 index 92a1cd222..000000000 --- a/waste/stack/compiler/frontend/quant/importer.py +++ /dev/null @@ -1,1187 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import numpy as np -import torch -from buddy.compiler.graph.operation import ( - AdaptiveAvgPool2dOp, - CloneOp, - AddMMOp, - AddOp, - AliasOp, - CatOp, - ClampMaxOp, - ClampMinOp, - Conv2dOp, - DivOp, - ExpOp, - GetItemOp, - HardswishOp, - LowMemoryMaxPoolWithOffsetsOp, - MatmulOp, - MaxPool2dOp, - MeanOp, - MulOp, - NegOp, - PermuteOp, - ReluOp, - ReshapeOp, - SiluOp, - SplitOp, - SplitWithSizesOp, - TOp, - UnsafeIndexOp, - ViewOp, -) -from stack.compiler.frontend.operation import ( - QuantizedChannelConcatOp, - QuantizedChannelSliceOp, - QuantizedDepthwiseConv2dOp, - QuantizedConv2dOp, - QuantizedGlobalAvgPoolOp, - QuantizedAddOp, - QuantizedMulOp, - QuantizedMatmulOp, - QuantizedMaxPool2dOp, - QuantizedResizeNearestOp, -) -from buddy.compiler.graph.type import TensorDType - -from stack.compiler.frontend.quant.rax import ( - QuantTensor, - RaxQuantPackage, - write_payload, -) - - -def _quantize_operators(graph, parameter_names, arrays, weight_scales, calibration): - def uses(node, name): - pending = list(node.args) - while pending: - argument = pending.pop() - if isinstance(argument, (list, tuple)): - pending.extend(argument) - elif renamed.get(str(argument), str(argument)) == name: - return True - return False - - param_nodes = list(graph.params) - input_node_list = list(graph.inputs) - original_body = list(graph._body) - calibration_index = {} - plans = {} - renamed = {} - removed = set() - - for node in original_body: - if not isinstance(node, (AliasOp, CloneOp, AdaptiveAvgPool2dOp)) or ( - isinstance(node, AdaptiveAvgPool2dOp) - and graph.node_table[str(node.args[0])].tensor_meta["shape"] - != node.tensor_meta["shape"] - ): - continue - if not isinstance(node.args[0], str): - raise ValueError(f"unsupported Alias form for {node.name}") - source = str(node.args[0]) - source_node = graph.node_table.get(source) - if source_node is None or source_node.tensor_meta.get( - "shape" - ) != node.tensor_meta.get("shape"): - raise ValueError(f"Alias shape mismatch for {node.name}") - renamed[node.name] = renamed.get(source, source) - removed.add(node.name) - - for index, node in enumerate(original_body): - if not isinstance(node, (Conv2dOp, AddMMOp, MatmulOp)): - continue - if isinstance(node, Conv2dOp): - if len(node.args) != 9: - raise ValueError(f"unexpected Conv2d form for {node.name}") - if node.args[5] != [1, 1] or node.args[6]: - raise ValueError(f"unsupported Quantized Conv2D form for {node.name}") - if node.args[3][0] != node.args[3][1]: - raise ValueError(f"asymmetric Quantized Conv2D stride for {node.name}") - padding = node.args[4] - if len(padding) == 1: - padding = [padding[0], padding[0]] - if len(padding) != 2 or padding[0] != padding[1]: - raise ValueError(f"asymmetric Quantized Conv2D padding for {node.name}") - activation_name, weight_arg, bias_arg = node.args[:3] - activation_name = renamed.get(str(activation_name), str(activation_name)) - weight_node = graph.node_table[str(weight_arg)] - weight_name = parameter_names.get(weight_node.name) - if weight_name is None: - raise ValueError( - f"Quantized Conv2D weight is not a parameter for {node.name}" - ) - weight_shape = list(arrays[weight_name].shape) - input_channels = int( - graph.node_table[str(activation_name)].tensor_meta["shape"][1] - ) - groups = int(node.args[8]) - if groups == 1: - replacement = QuantizedConv2dOp() - reduction_size = weight_shape[1] * weight_shape[2] * weight_shape[3] - elif ( - groups == input_channels - and weight_shape[0] == input_channels - and weight_shape[1] == 1 - ): - replacement = QuantizedDepthwiseConv2dOp() - reduction_size = weight_shape[2] * weight_shape[3] - else: - raise ValueError(f"unsupported grouped Conv2D for {node.name}") - else: - if isinstance(node, AddMMOp): - if len(node.args) < 3: - raise ValueError(f"unexpected AddMM form for {node.name}") - bias_arg, activation_name, weight_arg = node.args[:3] - else: - if len(node.args) != 2: - raise ValueError(f"unexpected Matmul form for {node.name}") - activation_name, weight_arg = node.args - activation_name = renamed.get( - str(activation_name), str(activation_name) - ) - bias_arg = None - weight_value = graph.node_table[str(weight_arg)] - valid_transpose = isinstance(weight_value, TOp) or ( - isinstance(weight_value, PermuteOp) - and len(weight_value.args) == 2 - and list(weight_value.args[1]) == [1, 0] - ) - if not valid_transpose or len(weight_value._parents) != 1: - raise ValueError( - f"Quantized MatMul weight must be one transpose for {node.name}: " - f"{weight_value.name}/{type(weight_value).__name__}, " - f"parents={weight_value._parents}, args={weight_value.args}" - ) - weight_node = graph.node_table[weight_value._parents[0]] - weight_arg = weight_node.name - removed.add(weight_value.name) - replacement = QuantizedMatmulOp() - weight_name = parameter_names.get(weight_node.name) - if weight_name is None: - raise ValueError( - f"Quantized MatMul weight is not a parameter for {node.name}" - ) - weight_shape = list(arrays[weight_name].shape) - if len(weight_shape) != 2: - raise ValueError( - f"Quantized MatMul weight must be rank 2 for {node.name}" - ) - reduction_size = weight_shape[1] - - weight_name = parameter_names.get(weight_node.name) - if weight_name is None or weight_name not in weight_scales: - raise ValueError(f"missing quantized weight for {node.name}") - weight_node._quantized_weight = True - records = calibration.get(weight_name) - occurrence = calibration_index.get(weight_name, 0) - if records is None or occurrence >= len(records): - raise ValueError(f"missing calibration for {weight_name}") - calibration_index[weight_name] = occurrence + 1 - ( - calibrated_input, - raw_output_scale, - relu_output_scale, - raw_channel_scales, - hardswish_channel_scales, - ) = records[occurrence] - raw_channel_scales = np.asarray(raw_channel_scales, dtype=np.float32) - hardswish_channel_scales = np.asarray( - hardswish_channel_scales, dtype=np.float32 - ) - if ( - not np.isfinite(calibrated_input) - or calibrated_input <= 0.0 - or not np.isfinite(raw_output_scale) - or raw_output_scale <= 0.0 - or not np.isfinite(relu_output_scale) - or relu_output_scale <= 0.0 - or raw_channel_scales.shape != (weight_shape[0],) - or hardswish_channel_scales.shape != (weight_shape[0],) - or not np.all(np.isfinite(raw_channel_scales)) - or not np.all(np.isfinite(hardswish_channel_scales)) - or np.any(raw_channel_scales <= 0.0) - or np.any(hardswish_channel_scales <= 0.0) - ): - raise ValueError(f"invalid calibration scale for {node.name}") - activation_node = graph.node_table[str(activation_name)] - - dw = np.asarray(weight_scales[weight_name], dtype=np.float32).reshape(-1) - output_channels = weight_shape[0] - if dw.size != output_channels: - raise ValueError(f"weight scale count mismatch for {weight_name}") - if not np.all(np.isfinite(dw)) or np.any(dw <= 0.0): - raise ValueError(f"invalid weight scale for {weight_name}") - if bias_arg is None: - bias = np.zeros(output_channels, dtype=np.float32) - else: - bias_node = graph.node_table[str(bias_arg)] - bias_name = parameter_names.get(bias_node.name) - if bias_name is None: - raise ValueError( - f"Quantized bias is not an offline parameter for {node.name}" - ) - bias = np.asarray(arrays[bias_name], dtype=np.float32).reshape(-1) - if bias.size != output_channels: - raise ValueError(f"bias channel count mismatch for {node.name}") - if not np.all(np.isfinite(bias)): - raise ValueError(f"non-finite bias for {node.name}") - product_bound = 128 * 128 * reduction_size - bias_headroom = np.iinfo(np.int32).max - product_bound - if bias_headroom <= 1: - raise ValueError(f"INT32 accumulator cannot hold {node.name}") - minimum_weight_scale = np.abs(bias.astype(np.float64)) / ( - (bias_headroom - 1) * float(calibrated_input) - ) - minimum_weight_scale = np.nextafter( - minimum_weight_scale.astype(np.float32), np.float32(np.inf) - ) - dw = np.maximum(dw, minimum_weight_scale) - weight_scales[weight_name] = dw - weight_node._quantized_weight_scales = dw.copy() - required_input_scale = float(calibrated_input) - - direct_users = [ - candidate - for candidate in graph._body - if candidate is not node and uses(candidate, node.name) - ] - relus = [ - candidate - for candidate in graph._body - if isinstance(candidate, (ReluOp, HardswishOp, SiluOp)) - and len(candidate.args) == 1 - and str(candidate.args[0]) == node.name - ] - if len(relus) > 1: - raise ValueError(f"multiple activation users for {node.name}") - activation = relus[0] if relus else None - activation_nodes = [activation] if activation is not None else [] - activation_kind = ( - 2 - if isinstance(activation, (HardswishOp, SiluOp)) - else 1 if isinstance(activation, ReluOp) else 0 - ) - lut_kind = ( - "silu" - if isinstance(activation, SiluOp) - else "hardswish" if isinstance(activation, HardswishOp) else None - ) - - silu_negs = [ - candidate for candidate in direct_users if isinstance(candidate, NegOp) - ] - if silu_negs: - if activation is not None or len(silu_negs) != 1: - raise ValueError(f"ambiguous decomposed SiLU after {node.name}") - neg = silu_negs[0] - neg_users = [ - candidate for candidate in original_body if uses(candidate, neg.name) - ] - if len(neg_users) != 1 or not isinstance(neg_users[0], ExpOp): - raise ValueError(f"malformed decomposed SiLU exp after {node.name}") - exp = neg_users[0] - exp_users = [ - candidate for candidate in original_body if uses(candidate, exp.name) - ] - if ( - len(exp_users) != 1 - or not isinstance(exp_users[0], AddOp) - or list(exp_users[0].args) != [exp.name, 1] - ): - raise ValueError(f"malformed decomposed SiLU add after {node.name}") - add = exp_users[0] - add_users = [ - candidate for candidate in original_body if uses(candidate, add.name) - ] - if ( - len(add_users) != 1 - or not isinstance(add_users[0], DivOp) - or list(add_users[0].args) != [node.name, add.name] - or {candidate.name for candidate in direct_users} - != {neg.name, add_users[0].name} - ): - raise ValueError(f"malformed decomposed SiLU div after {node.name}") - activation = add_users[0] - activation_nodes = [neg, exp, add, activation] - activation_kind = 2 - lut_kind = "silu" - - hard_swish_adds = [ - candidate - for candidate in direct_users - if isinstance(candidate, AddOp) - and len(candidate.args) == 2 - and str(candidate.args[0]) == node.name - and candidate.args[1] == 3 - ] - if hard_swish_adds: - if activation is not None or len(hard_swish_adds) != 1: - raise ValueError(f"ambiguous HardSwish after {node.name}") - add = hard_swish_adds[0] - add_users = [ - candidate for candidate in graph._body if uses(candidate, add.name) - ] - if len(add_users) != 1 or not isinstance(add_users[0], ClampMinOp): - raise ValueError(f"malformed HardSwish clamp-min after {node.name}") - clamp_min = add_users[0] - if list(clamp_min.args) != [add.name, 0]: - raise ValueError( - f"malformed HardSwish clamp-min args after {node.name}" - ) - clamp_min_users = [ - candidate - for candidate in graph._body - if uses(candidate, clamp_min.name) - ] - if len(clamp_min_users) != 1 or not isinstance( - clamp_min_users[0], ClampMaxOp - ): - raise ValueError(f"malformed HardSwish clamp-max after {node.name}") - clamp_max = clamp_min_users[0] - if list(clamp_max.args) != [clamp_min.name, 6]: - raise ValueError( - f"malformed HardSwish clamp-max args after {node.name}" - ) - clamp_max_users = [ - candidate - for candidate in graph._body - if uses(candidate, clamp_max.name) - ] - if len(clamp_max_users) != 1: - raise ValueError(f"ambiguous hard activation after {node.name}") - after_clamp = clamp_max_users[0] - if isinstance(after_clamp, DivOp): - if list(after_clamp.args) != [clamp_max.name, 6]: - raise ValueError(f"malformed hard-sigmoid after {node.name}") - if {candidate.name for candidate in direct_users} != {add.name}: - raise ValueError( - f"hard-sigmoid input has extra users after {node.name}" - ) - activation = after_clamp - activation_nodes = [add, clamp_min, clamp_max, after_clamp] - activation_kind = 2 - lut_kind = "hardsigmoid" - elif isinstance(after_clamp, MulOp): - mul = after_clamp - if list(map(str, mul.args)) != [node.name, clamp_max.name]: - raise ValueError( - f"malformed HardSwish multiply args after {node.name}" - ) - if {candidate.name for candidate in direct_users} != { - add.name, - mul.name, - }: - raise ValueError( - f"HardSwish input has extra users after {node.name}" - ) - mul_users = [ - candidate for candidate in graph._body if uses(candidate, mul.name) - ] - if len(mul_users) != 1 or not isinstance(mul_users[0], DivOp): - raise ValueError(f"malformed HardSwish divide after {node.name}") - div = mul_users[0] - if list(div.args) != [mul.name, 6]: - raise ValueError( - f"malformed HardSwish divide args after {node.name}" - ) - activation = div - activation_nodes = [add, clamp_min, clamp_max, mul, div] - activation_kind = 2 - lut_kind = "hardswish" - else: - raise ValueError(f"unsupported hard activation after {node.name}") - - if (node.trace_meta is not None and node.trace_meta["tensor"]) or any( - item.trace_meta is not None for item in activation_nodes - ): - activation = None - activation_nodes = [] - activation_kind = 0 - lut_kind = None - - result_name = activation.name if activation is not None else node.name - result_users = [ - candidate - for candidate in graph._body - if candidate is not activation - and candidate is not node - and uses(candidate, result_name) - ] - plans[node.name] = { - "index": index, - "node": node, - "replacement": replacement, - "activation_name": str(activation_name), - "activation_node": activation_node, - "weight_arg": str(weight_arg), - "weight_shape": weight_shape, - "dw": dw, - "bias": bias, - "required_input_scale": required_input_scale, - "calibrated_output_scale": float( - relu_output_scale if activation_kind == 1 else raw_output_scale - ), - "raw_channel_scales": raw_channel_scales, - "hardswish_channel_scales": hardswish_channel_scales, - "activation": activation, - "activation_nodes": activation_nodes, - "activation_kind": activation_kind, - "lut_kind": lut_kind, - "direct_users": direct_users, - "result_name": result_name, - "users": result_users, - "padding": padding if isinstance(node, Conv2dOp) else None, - "product_bound": product_bound, - } - - first_conv = min( - (plan for plan in plans.values() if isinstance(plan["node"], Conv2dOp)), - key=lambda plan: plan["index"], - default=None, - ) - if first_conv is not None and first_conv["lut_kind"] == "hardswish": - consumers = [ - candidate - for candidate in plans.values() - if candidate["activation_name"] == first_conv["result_name"] - ] - if ( - len(first_conv["users"]) != 1 - or len(consumers) != 1 - or not isinstance(consumers[0]["replacement"], QuantizedDepthwiseConv2dOp) - or consumers[0]["weight_shape"][0] != first_conv["weight_shape"][0] - ): - raise ValueError( - "the first Hardswish Conv must feed one matching depthwise Conv" - ) - first_conv["lane_input_scales"] = first_conv["raw_channel_scales"] - first_conv["lane_output_scales"] = first_conv["hardswish_channel_scales"] - consumers[0]["input_channel_scales"] = first_conv["lane_output_scales"] - - for plan in plans.values(): - if plan["activation"] is not None: - renamed[plan["result_name"]] = plan["node"].name - removed.update(item.name for item in plan["activation_nodes"]) - - special = {} - quantized_values = { - name - for plan in plans.values() - for name in (plan["node"].name, plan["result_name"]) - } - for node in original_body: - if not isinstance(node, GetItemOp) or len(node.args) != 2: - continue - source = graph.node_table.get(str(node.args[0])) - if not isinstance(source, (SplitOp, SplitWithSizesOp)) or len(source.args) != 3: - continue - input_name = renamed.get(str(source.args[0]), str(source.args[0])) - if int(source.args[2]) == 1 and input_name in quantized_values: - quantized_values.add(node.name) - for node in original_body: - if node.name in removed: - continue - activation = None - result_name = node.name - arguments = None - attributes = {} - if isinstance(node, GetItemOp): - if len(node.args) != 2 or not isinstance(node.args[1], int): - continue - source = graph.node_table[str(node.args[0])] - index = int(node.args[1]) - if isinstance(source, (SplitOp, SplitWithSizesOp)): - if len(source.args) != 3: - continue - input_name = renamed.get(str(source.args[0]), str(source.args[0])) - if input_name not in quantized_values: - continue - if int(source.args[2]) != 1: - raise ValueError( - f"Quantized channel split must use NCHW C at {source.name}" - ) - if isinstance(source, SplitOp): - channels = int( - graph.node_table[str(source.args[0])].tensor_meta["shape"][1] - ) - size = int(source.args[1]) - sizes = [size] * (channels // size) - if channels % size: - sizes.append(channels % size) - else: - sizes = [int(value) for value in source.args[1]] - if index < 0 or index >= len(sizes): - raise ValueError( - f"Quantized channel split index is invalid at {node.name}" - ) - replacement = QuantizedChannelSliceOp() - arguments = [input_name] - attributes = { - "offset": sum(sizes[:index]), - "output_shape": list(node.tensor_meta["shape"]), - } - removed.add(source.name) - elif isinstance(source, LowMemoryMaxPoolWithOffsetsOp): - if index != 0 or len(source.args) != 6: - raise ValueError( - f"Quantized SPPF must select MaxPool values at {node.name}" - ) - kernel, stride, padding, dilation, ceil_mode = source.args[1:] - if ( - list(kernel) != [5, 5] - or list(stride) != [1, 1] - or list(padding) != [2, 2] - or list(dilation) != [1, 1] - or ceil_mode - ): - raise ValueError( - f"unsupported Quantized SPPF MaxPool at {source.name}" - ) - input_name = renamed.get(str(source.args[0]), str(source.args[0])) - if input_name not in quantized_values: - continue - replacement = QuantizedMaxPool2dOp() - arguments = [input_name] - attributes = { - "input_shape": list( - graph.node_table[str(source.args[0])].tensor_meta["shape"] - ), - "output_shape": list(node.tensor_meta["shape"]), - "kernel": 5, - "stride": 1, - "padding": 2, - } - removed.add(source.name) - else: - continue - elif isinstance(node, CatOp): - if len(node.args) != 2 or int(node.args[1]) != 1: - continue - arguments = [renamed.get(str(value), str(value)) for value in node.args[0]] - if len(arguments) < 2 or any( - value not in quantized_values for value in arguments - ): - continue - replacement = QuantizedChannelConcatOp() - attributes = {"output_shape": list(node.tensor_meta["shape"])} - elif isinstance(node, UnsafeIndexOp): - if len(node.args) != 2: - continue - input_name = renamed.get(str(node.args[0]), str(node.args[0])) - if input_name not in quantized_values: - continue - input_shape = list(graph.node_table[str(node.args[0])].tensor_meta["shape"]) - output_shape = list(node.tensor_meta["shape"]) - if ( - len(input_shape) != 4 - or len(output_shape) != 4 - or input_shape[0] != output_shape[0] - or input_shape[1] != output_shape[1] - or output_shape[2] % input_shape[2] - or output_shape[3] % input_shape[3] - ): - raise ValueError(f"unsupported Quantized nearest resize at {node.name}") - replacement = QuantizedResizeNearestOp() - arguments = [input_name] - attributes = { - "output_shape": output_shape, - "scale_h": output_shape[2] // input_shape[2], - "scale_w": output_shape[3] // input_shape[3], - } - elif isinstance(node, MaxPool2dOp): - # Pooling can sit beyond an FP32 reshape/transpose boundary. - # Only a quantized producer supplies an INT8 scale for fusion. - input_name = renamed.get(str(node.args[0]), str(node.args[0])) - if input_name not in quantized_values: - continue - if ( - len(node.args) not in (3, 4) - or len(node.args[1]) != 2 - or node.args[1][0] != node.args[1][1] - or len(node.args[2]) != 2 - or node.args[2][0] != node.args[2][1] - ): - raise ValueError( - f"unsupported quantized graph MaxPool2D for {node.name}" - ) - padding = node.args[3] if len(node.args) == 4 else [0, 0] - if len(padding) == 1: - padding = [padding[0], padding[0]] - if len(padding) != 2 or padding[0] != padding[1]: - raise ValueError( - f"asymmetric quantized graph MaxPool2D padding for {node.name}" - ) - replacement = QuantizedMaxPool2dOp() - elif isinstance(node, MeanOp): - if ( - len(node.args) != 3 - or list(node.args[1]) != [-1, -2] - or not node.args[2] - ): - continue - replacement = QuantizedGlobalAvgPoolOp() - elif ( - isinstance(node, MulOp) - and len(node.args) == 2 - and all(isinstance(arg, str) for arg in node.args) - ): - if any( - renamed.get(str(arg), str(arg)) not in quantized_values - for arg in node.args - ): - continue - replacement = QuantizedMulOp() - elif ( - isinstance(node, AddOp) - and len(node.args) == 2 - and all(isinstance(arg, str) for arg in node.args) - ): - if any( - renamed.get(str(arg), str(arg)) not in quantized_values - for arg in node.args - ): - continue - replacement = QuantizedAddOp() - else: - continue - if isinstance(replacement, (QuantizedAddOp, QuantizedMulOp)): - users = [ - candidate - for candidate in original_body - if candidate.name not in removed and uses(candidate, node.name) - ] - if (node.trace_meta is not None and node.trace_meta["tensor"]) or any( - isinstance(candidate, ReluOp) and candidate.trace_meta is not None - for candidate in users - ): - continue - if not any( - isinstance( - candidate, - ( - ReluOp, - Conv2dOp, - AddMMOp, - MatmulOp, - MeanOp, - MaxPool2dOp, - AddOp, - MulOp, - CatOp, - ), - ) - for candidate in users - ): - continue - relus = [candidate for candidate in users if isinstance(candidate, ReluOp)] - if relus: - if len(users) != 1 or len(relus) != 1: - raise ValueError( - f"Quantized INT8 elementwise ReLU must be its only user: {node.name}" - ) - activation = relus[0] - result_name = activation.name - renamed[result_name] = node.name - removed.add(result_name) - special[node.name] = { - "node": node, - "replacement": replacement, - "activation": activation, - "result_name": result_name, - "arguments": arguments, - **attributes, - } - quantized_values.add(node.name) - quantized_values.add(result_name) - - for plan in plans.values(): - consumers = [ - candidate - for candidate in plans.values() - if candidate["activation_name"] == plan["result_name"] - ] - plan["output_scale"] = ( - max(candidate["required_input_scale"] for candidate in consumers) - if consumers - else plan["calibrated_output_scale"] - ) - - for item in reversed(list(special.values())): - node = item["node"] - consumers = [ - candidate - for candidate in plans.values() - if renamed.get(candidate["activation_name"], candidate["activation_name"]) - == node.name - ] - downstream_scales = [ - candidate["output_scale"] - for candidate in special.values() - if "output_scale" in candidate - and any( - uses(candidate["node"], source) - for source in (node.name, item["result_name"]) - ) - ] - if not consumers and isinstance(node, (MeanOp, MaxPool2dOp)): - users = [ - candidate - for candidate in original_body - if candidate.name not in removed and uses(candidate, node.name) - ] - if len(users) != 1 or not isinstance(users[0], (ViewOp, ReshapeOp)): - raise ValueError( - f"quantized graph mean has no unique consumer: {node.name}" - ) - consumers = [ - candidate - for candidate in plans.values() - if renamed.get( - candidate["activation_name"], candidate["activation_name"] - ) - == users[0].name - ] - if not consumers and not downstream_scales: - users = [ - candidate.name - for candidate in original_body - if uses(candidate, node.name) - ] - raise ValueError( - f"quantized graph stage has no compute consumer: {node.name}, " - f"args={node.args}, users={users}" - ) - item["output_scale"] = max( - [consumer["required_input_scale"] for consumer in consumers] - + downstream_scales - ) - - for plan in plans.values(): - max_pool_consumers = [ - item - for item in special.values() - if isinstance(item["replacement"], QuantizedMaxPool2dOp) - and renamed.get(str(item["node"].args[0]), str(item["node"].args[0])) - == plan["node"].name - ] - if len(max_pool_consumers) > 1: - raise ValueError(f"multiple MaxPool2D consumers for {plan['node'].name}") - if max_pool_consumers: - plan["output_scale"] = max_pool_consumers[0]["output_scale"] - - changed = True - while changed: - changed = False - for name, item in special.items(): - if not isinstance( - item["replacement"], - ( - QuantizedChannelSliceOp, - QuantizedChannelConcatOp, - QuantizedResizeNearestOp, - QuantizedMaxPool2dOp, - ), - ): - continue - sources = item["arguments"] - if sources is None: - sources = [ - renamed.get(str(item["node"].args[0]), str(item["node"].args[0])) - ] - scales = [item["output_scale"]] - for source in sources: - if source in plans: - scales.append(plans[source]["output_scale"]) - elif source in special: - scales.append(special[source]["output_scale"]) - else: - raise ValueError( - f"Quantized view source has no scale at {name}: {source}" - ) - scale = max(scales) - if item["output_scale"] != scale: - item["output_scale"] = scale - changed = True - for source in sources: - producer = plans[source] if source in plans else special[source] - if producer["output_scale"] != scale: - producer["output_scale"] = scale - changed = True - - value_scale = {} - for plan in plans.values(): - value_scale[plan["node"].name] = plan["output_scale"] - value_scale[plan["result_name"]] = plan["output_scale"] - for name, item in special.items(): - value_scale[name] = item["output_scale"] - value_scale[item["result_name"]] = item["output_scale"] - - for plan in plans.values(): - activation_name = renamed.get(plan["activation_name"], plan["activation_name"]) - input_scale = value_scale.get(activation_name, plan["required_input_scale"]) - dw = plan["dw"] - input_channel_scales = plan.get("input_channel_scales") - if input_channel_scales is not None: - input_channel_scales = np.asarray(input_channel_scales, dtype=np.float32) - if not isinstance( - plan["replacement"], QuantizedDepthwiseConv2dOp - ) or input_channel_scales.shape != (dw.size,): - raise ValueError( - f"per-channel input scale requires matching depthwise Conv: " - f"{plan['node'].name}" - ) - minimum_weight_scale = np.abs(plan["bias"].astype(np.float64)) / ( - (np.iinfo(np.int32).max - plan["product_bound"] - 1) - * input_channel_scales.astype(np.float64) - ) - dw = np.maximum( - dw, - np.nextafter( - minimum_weight_scale.astype(np.float32), np.float32(np.inf) - ), - ) - plan["dw"] = dw - graph.node_table[plan["weight_arg"]]._quantized_weight_scales = dw.copy() - accumulator_input_scale = ( - input_channel_scales - if input_channel_scales is not None - else np.float32(input_scale) - ) - bias_i64 = np.rint( - plan["bias"].astype(np.float64) - / ( - np.asarray(accumulator_input_scale, dtype=np.float64) - * dw.astype(np.float64) - ) - ).astype(np.int64) - overflow = np.flatnonzero( - np.abs(bias_i64) + plan["product_bound"] > np.iinfo(np.int32).max - ) - if overflow.size: - channel = int(overflow[0]) - scale_index = channel if input_channel_scales is not None else 0 - raise ValueError( - f"INT32 accumulator overflow for {plan['node'].name} channel {channel}: " - f"input_scale={float(np.asarray(accumulator_input_scale).reshape(-1)[scale_index])}, " - f"weight_scale={float(dw[channel])}, " - f"bias_i32={int(bias_i64[channel])}, " - f"product_bound={plan['product_bound']}" - ) - - node = plan["node"] - replacement = plan["replacement"] - activation = plan["activation"] - replacement._name = node.name - replacement._arguments = [activation_name, plan["weight_arg"]] - replacement._parents = list(replacement._arguments) - replacement._children = [ - renamed.get(user.name, user.name) for user in plan["users"] - ] - replacement._tensor_meta = node._tensor_meta.copy() - direct_compute_consumers = [ - candidate - for candidate in plans.values() - if candidate["activation_name"] == plan["result_name"] - ] - replacement._final_output = ( - not isinstance(node, Conv2dOp) and not direct_compute_consumers - ) - replacement._tensor_meta["dtype"] = ( - TensorDType.Float32 if replacement._final_output else TensorDType.Int8 - ) - replacement._input_scale = input_scale - replacement._output_scale = plan["output_scale"] - replacement._bias_i32 = bias_i64.astype(np.int32).tolist() - requant_output_scale = np.asarray( - plan.get("lane_input_scales", plan["output_scale"]), - dtype=np.float32, - ) - replacement._requant_scale = ( - np.asarray(accumulator_input_scale, dtype=np.float32) - * dw - / requant_output_scale - ).tolist() - replacement._dequant_scale = ( - np.asarray(accumulator_input_scale, dtype=np.float32) * dw - ).tolist() - replacement._activation = plan["activation_kind"] - if replacement._activation == 2: - raw = np.arange(256, dtype=np.int16) - signed = np.where(raw < 128, raw, raw - 256).astype(np.float32) - if "lane_input_scales" in plan: - x = plan["lane_input_scales"][:, None] * signed[None, :] - else: - x = signed * np.float32(plan["output_scale"]) - if plan["lut_kind"] == "hardswish": - y = x * np.clip(x + np.float32(3.0), 0.0, 6.0) / np.float32(6.0) - elif plan["lut_kind"] == "hardsigmoid": - y = np.clip(x + np.float32(3.0), 0.0, 6.0) / np.float32(6.0) - elif plan["lut_kind"] == "silu": - y = x / ( - np.float32(1.0) - + np.exp(-np.clip(x, np.float32(-80.0), np.float32(80.0))) - ) - else: - raise ValueError(f"missing LUT function for {node.name}") - lut_output_scale = np.asarray( - plan.get("lane_output_scales", plan["output_scale"]), - dtype=np.float32, - ) - if lut_output_scale.ndim == 1: - lut_output_scale = lut_output_scale[:, None] - replacement._lut_i8 = ( - np.clip(np.rint(y / lut_output_scale), -128, 127) - .astype(np.int8) - .reshape(-1) - .tolist() - ) - replacement._lane_output_scales = plan.get( - "lane_output_scales", np.empty(0, dtype=np.float32) - ).tolist() - else: - replacement._lut_i8 = [0] - replacement._lane_output_scales = [] - replacement.trace_meta = node.trace_meta - if isinstance(replacement, (QuantizedConv2dOp, QuantizedDepthwiseConv2dOp)): - replacement._input_shape = list( - plan["activation_node"].tensor_meta["shape"] - ) - replacement._weight_shape = plan["weight_shape"] - replacement._output_shape = list(node.tensor_meta["shape"]) - replacement._stride = int(node.args[3][0]) - replacement._padding = int(plan["padding"][0]) - for name, item in special.items(): - node = item["node"] - replacement = item["replacement"] - args = item["arguments"] - if args is None: - args = [ - renamed.get(str(arg), str(arg)) - for arg in node.args - if isinstance(arg, str) - ] - if isinstance(replacement, QuantizedMaxPool2dOp): - if len(args) != 1 or args[0] not in value_scale: - raise ValueError(f"invalid Quantized MaxPool2D input for {name}") - replacement._input_scale = value_scale[args[0]] - replacement._output_scale = replacement._input_scale - if "input_shape" in item: - replacement._input_shape = item["input_shape"] - replacement._output_shape = item["output_shape"] - replacement._kernel = item["kernel"] - replacement._stride = item["stride"] - replacement._padding = item["padding"] - else: - replacement._input_shape = list( - graph.node_table[str(node.args[0])].tensor_meta["shape"] - ) - replacement._output_shape = list(node.tensor_meta["shape"]) - replacement._kernel = int(node.args[1][0]) - replacement._stride = int(node.args[2][0]) - padding = node.args[3] if len(node.args) == 4 else [0, 0] - replacement._padding = int(padding[0]) - replacement._final_output = False - elif isinstance(replacement, QuantizedChannelSliceOp): - if len(args) != 1 or args[0] not in value_scale: - raise ValueError(f"invalid Quantized channel-slice input for {name}") - replacement._offset = item["offset"] - replacement._output_shape = item["output_shape"] - elif isinstance(replacement, QuantizedChannelConcatOp): - if any(arg not in value_scale for arg in args): - raise ValueError(f"invalid Quantized channel-concat input for {name}") - replacement._output_shape = item["output_shape"] - elif isinstance(replacement, QuantizedResizeNearestOp): - if len(args) != 1 or args[0] not in value_scale: - raise ValueError(f"invalid Quantized resize input for {name}") - replacement._output_shape = item["output_shape"] - replacement._scale_h = item["scale_h"] - replacement._scale_w = item["scale_w"] - elif isinstance(replacement, QuantizedGlobalAvgPoolOp): - if len(args) != 1 or args[0] not in value_scale: - raise ValueError(f"invalid Quantized global-average input for {name}") - replacement._input_scale = value_scale[args[0]] - else: - if len(args) != 2 or any(arg not in value_scale for arg in args): - raise ValueError(f"invalid Quantized INT8 elementwise input for {name}") - replacement._lhs_scale = value_scale[args[0]] - replacement._rhs_scale = value_scale[args[1]] - replacement._activation = 1 if item["activation"] is not None else 0 - replacement._name = name - replacement._arguments = args - replacement._parents = list(args) - replacement._children = [ - renamed.get(child, child) - for child in ( - item["activation"]._children - if item["activation"] is not None - else node._children - ) - ] - replacement._tensor_meta = node._tensor_meta.copy() - replacement._tensor_meta["dtype"] = TensorDType.Int8 - replacement._output_scale = item["output_scale"] - replacement.trace_meta = node.trace_meta - - replacements = {name: plan["replacement"] for name, plan in plans.items()} | { - name: item["replacement"] for name, item in special.items() - } - - def canonical(argument): - if isinstance(argument, list): - return [canonical(value) for value in argument] - if isinstance(argument, tuple): - return tuple(canonical(value) for value in argument) - while isinstance(argument, str) and argument in renamed: - argument = renamed[argument] - return argument - - graph._body = [ - replacements.get(node.name, node) - for node in original_body - if node.name not in removed - ] - graph.node_table = {node.name: node for node in graph.body} - for node in graph.body: - node._arguments = canonical(node.args) - node._parents = [] - node._children = [] - for node in graph.body: - pending = list(node.args) - while pending: - argument = pending.pop(0) - if isinstance(argument, (list, tuple)): - pending[:0] = argument - elif isinstance(argument, str) and argument in graph.node_table: - if argument not in node._parents: - node._parents.append(argument) - graph.node_table[argument]._children.append(node.name) - for node in replacements.values(): - if isinstance( - node, (QuantizedConv2dOp, QuantizedDepthwiseConv2dOp, QuantizedMatmulOp) - ): - node._final_output = any(name not in replacements for name in node.children) - node._tensor_meta["dtype"] = ( - TensorDType.Float32 if node._final_output else TensorDType.Int8 - ) - elif isinstance(node, QuantizedMaxPool2dOp): - node._final_output = any(name not in replacements for name in node.children) - graph._fake_params = [graph.body.index(node) for node in param_nodes] - graph._inputs = [graph.body.index(node) for node in input_node_list] - from buddy.compiler.graph.transform.quantization.quantize import sort_graph - - sort_graph(graph) - - -def quantize_model_graph( - graph, - params, - names: list[str], - output_dir: Path, - model_name: str, - calibration: dict, - *, - weight_scales=None, - assets: dict[str, bytes] | None = None, -) -> None: - if len(graph.params) != len(params) or len(names) != len(params): - raise ValueError("parameter metadata does not match imported graph") - param_nodes = list(graph.params) - parameter_names, arrays = {}, {} - for node, param, name in zip(param_nodes, params, names): - array = param.detach().cpu().numpy().astype(np.float32) - parameter_names[node.name] = name - arrays[name] = array - - if weight_scales is None: - from stack.compiler.frontend.quant.transforms.symmetric import ( - quantize_symmetric, - ) - - weight_scales = { - name: quantize_symmetric(array, [0])[1] - for name, array in arrays.items() - if name.endswith(".weight") and array.ndim >= 2 - } - - _quantize_operators( - graph, - parameter_names, - arrays, - weight_scales, - calibration, - ) - - tensors = [] - weights, fp_params, scales = [], [], [] - weight_off = param_off = scale_off = 0 - for index, (node, name) in enumerate(zip(param_nodes, names)): - array = arrays[name] - if getattr(node, "_quantized_weight", False): - axes = [0] - dw = np.asarray(node._quantized_weight_scales, dtype=np.float32).reshape(-1) - if ( - dw.size != array.shape[0] - or not np.all(np.isfinite(dw)) - or np.any(dw <= 0) - ): - raise ValueError(f"invalid adjusted weight scales for {name}") - q = np.clip( - np.rint(array / dw.reshape((-1,) + (1,) * (array.ndim - 1))), - -128, - 127, - ).astype(np.int8) - node.tensor_meta["dtype"] = TensorDType.Int8 - node.tensor_meta["shape"] = list(q.shape) - params[index] = torch.from_numpy(q.copy()) - scale_bytes = dw.tobytes() - tensors.append( - QuantTensor( - name, - list(array.shape), - list(q.shape), - "i8", - axes, - weight_off, - q.nbytes, - scale_off, - len(scale_bytes), - ) - ) - weights.append(q.tobytes()) - scales.append(scale_bytes) - weight_off += q.nbytes - scale_off += len(scale_bytes) - else: - raw = array.tobytes() - tensors.append( - QuantTensor( - name, - list(array.shape), - list(array.shape), - "f32", - [], - param_off, - len(raw), - 0, - 0, - ) - ) - fp_params.append(raw) - param_off += len(raw) - package = RaxQuantPackage( - tensors, - b"".join(weights), - b"".join(fp_params), - b"".join(scales), - assets if assets is not None else {}, - ) - output_dir.mkdir(parents=True, exist_ok=True) - write_payload(package, output_dir / f"{model_name}.payload") - (output_dir / "weights.bin").write_bytes(package.weights) - (output_dir / "params.f32").write_bytes(package.params_f32) - (output_dir / "scales.bin").write_bytes(package.scales_f32) diff --git a/waste/stack/compiler/frontend/quant/mxfp8_cache.py b/waste/stack/compiler/frontend/quant/mxfp8_cache.py deleted file mode 100644 index 868920885..000000000 --- a/waste/stack/compiler/frontend/quant/mxfp8_cache.py +++ /dev/null @@ -1,70 +0,0 @@ -import torch - - -def _shape(values): - if values.ndim < 1 or values.shape[-1] == 0 or values.shape[-1] % 32: - raise ValueError("MXFP8 requires a positive last dimension divisible by 32") - return (*values.shape[:-1], values.shape[-1] // 32) - - -@torch.library.custom_op("buckyball::mxfp8_encode", mutates_args=()) -def encode(values: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: - shape = _shape(values) - if values.dtype != torch.float32 or not torch.isfinite(values).all(): - raise ValueError("MXFP8 requires finite FP32 values") - blocks = values.reshape(*shape, 32) - maximum = blocks.abs().amax(-1) - exponent = torch.where(maximum == 0, 0, torch.frexp(maximum)[1] - 9) - exponent = exponent + (torch.ldexp(maximum, -exponent) > 448) - exponent = exponent.clamp(-127, 119) - scaled = torch.ldexp(blocks, -exponent[..., None]).contiguous() - bits = scaled.view(torch.int32).to(torch.int64) & 0xFFFFFFFF - sign = (bits >> 24) & 128 - element_exponent = ((bits >> 23) & 255) - 120 - fraction = bits & 0x7FFFFF - high, low = fraction >> 20, fraction & 0xFFFFF - rounded = high + ((low > 0x80000) | ((low == 0x80000) & ((high & 1) != 0))) - codes = torch.where( - element_exponent <= 0, - (scaled.abs() * 512).round().to(torch.int64), - element_exponent * 8 + rounded, - ) - codes = (codes.clamp_max(126) | sign).to(torch.int8).reshape(values.shape) - return codes, (exponent + 127).to(torch.int8) - - -@encode.register_fake -def _encode_fake(values): - shape = _shape(values) - return values.new_empty(values.shape, dtype=torch.int8), values.new_empty( - shape, dtype=torch.int8 - ) - - -@torch.library.custom_op("buckyball::mxfp8_decode", mutates_args=()) -def decode(codes: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: - shape = _shape(codes) - if codes.dtype != torch.int8 or scales.dtype != torch.int8 or scales.shape != shape: - raise ValueError("MXFP8 requires matching INT8 codes and scales") - unsigned = codes.to(torch.int16) & 255 - scale = scales.to(torch.int16) & 255 - if torch.any((unsigned & 127) == 127) or torch.any(scale == 255): - raise ValueError("MXFP8 NaN encoding") - exponent, fraction = (unsigned >> 3) & 15, (unsigned & 7).float() - elements = torch.where( - exponent == 0, fraction / 512, torch.ldexp(8 + fraction, exponent - 10) - ) - elements = torch.copysign(elements, torch.where((unsigned & 128) != 0, -1.0, 1.0)) - result = torch.ldexp(elements.reshape(*shape, 32), scale[..., None] - 127).reshape( - codes.shape - ) - if not torch.isfinite(result).all(): - raise ValueError("MXFP8 decoded value overflows FP32") - return result - - -@decode.register_fake -def _decode_fake(codes, scales): - if scales.shape != _shape(codes): - raise ValueError("MXFP8 code/scale shape mismatch") - return codes.new_empty(codes.shape, dtype=torch.float32) diff --git a/waste/stack/compiler/frontend/quant/native_mxfp8.py b/waste/stack/compiler/frontend/quant/native_mxfp8.py deleted file mode 100644 index 0de29635b..000000000 --- a/waste/stack/compiler/frontend/quant/native_mxfp8.py +++ /dev/null @@ -1,396 +0,0 @@ -"""MXFP8 storage and offline activation calibration for native CPU/GPU models.""" - -import torch -from torch import nn - -from stack.compiler.frontend.quant import mxfp8_cache - - -@torch.library.custom_op("buddy_native::mxfp8_encode_fixed", mutates_args=()) -def encode_fixed(values: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: - exponent = (scales.to(torch.int16) & 255) - 127 - blocks = values.reshape(*values.shape[:-1], values.shape[-1] // 32, 32) - scaled = torch.ldexp(blocks, -exponent[..., None]).contiguous() - magnitude = scaled.abs().clamp_max(448) - bits = magnitude.view(torch.int32) - element_exponent = (bits >> 23) - 120 - fraction = bits & 0x7FFFFF - high, low = fraction >> 20, fraction & 0xFFFFF - rounded = high + ((low > 0x80000) | ((low == 0x80000) & ((high & 1) != 0))) - codes = torch.where( - element_exponent <= 0, - (magnitude * 512).round().to(torch.int32), - element_exponent * 8 + rounded, - ) - sign = (scaled.view(torch.int32) >> 24) & 128 - return (codes | sign).to(torch.int8).reshape(values.shape) - - -@encode_fixed.register_fake -def _encode_fake(values, scales): - return torch.empty_like(values, dtype=torch.int8) - - -@torch.library.custom_op("buddy_native::mxfp8_decode", mutates_args=()) -def decode(codes: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: - return mxfp8_cache.decode( - codes, - scales.expand(*codes.shape[:-1], codes.shape[-1] // 32).contiguous(), - ) - - -@decode.register_fake -def _decode_fake(codes, scales): - return torch.empty_like(codes, dtype=torch.float32) - - -@torch.library.custom_op("buddy_native::mxfp8_linear", mutates_args=()) -def matmul( - activation: torch.Tensor, - activation_scales: torch.Tensor, - weights: torch.Tensor, - weight_scales: torch.Tensor, -) -> torch.Tensor: - # Round the dot product once, before the next MXFP8 quantization boundary. - return nn.functional.linear( - decode(activation, activation_scales).double(), - decode(weights, weight_scales).double(), - ).float() - - -@matmul.register_fake -def _matmul_fake(activation, activation_scales, weights, weight_scales): - return activation.new_empty( - (activation.shape[0], weights.shape[0]), dtype=torch.float32 - ) - - -@torch.library.custom_op("buddy_native::attention_matmul", mutates_args=()) -def attention_matmul(lhs: torch.Tensor, rhs: torch.Tensor) -> torch.Tensor: - return torch.bmm(lhs.double(), rhs.double()).float() - - -@attention_matmul.register_fake -def _attention_matmul_fake(lhs, rhs): - return lhs.new_empty((lhs.shape[0], lhs.shape[1], rhs.shape[2])) - - -@torch.library.custom_op("buddy_native::attention_softmax", mutates_args=()) -def attention_softmax(value: torch.Tensor, axis: int) -> torch.Tensor: - return torch.softmax(value.double(), dim=axis).float() - - -@attention_softmax.register_fake -def _attention_softmax_fake(value, axis): - return torch.empty_like(value) - - -def attention( - module, - query, - key, - value, - attention_mask, - scaling, - dropout=0.0, - softcap=None, - **kwargs, -): - if dropout != 0.0: - raise ValueError("MXFP8 attention is an inference kernel") - batch, heads, length, width = query.shape - keys = key.shape[2] - groups = module.num_key_value_groups - key = ( - key[:, :, None] - .expand(batch, key.shape[1], groups, keys, width) - .reshape(batch * heads, keys, width) - ) - value = ( - value[:, :, None] - .expand(batch, value.shape[1], groups, keys, width) - .reshape(batch * heads, keys, width) - ) - query = query.reshape(batch * heads, length, width) - scores = ( - attention_matmul(query, key.transpose(1, 2)).reshape(batch, heads, length, keys) - * scaling - ) - if softcap is not None: - scores = torch.tanh(scores / softcap) * softcap - if attention_mask is not None: - scores = scores + attention_mask - probabilities = attention_softmax(scores, -1) - output = attention_matmul(probabilities.reshape(batch * heads, length, keys), value) - return ( - output.reshape(batch, heads, length, width).transpose(1, 2).contiguous(), - probabilities, - ) - - -def register(compiler): - from buddy.compiler.graph.operation import Op, OpType - from stack.compiler.frontend.ops import reductions - from stack.compiler.frontend.ops.native_mxfp8 import codec, concat - from stack.compiler.frontend.ops.native_mxfp8 import matmul as lower_matmul - from stack.compiler.frontend.ops.reductions import bmm, gelu, mean, softmax, tanh - - class NativeMXFP8EncodeOp(Op): - def __init__(self): - super().__init__() - self._op_type = OpType.BroadcastType - - class NativeMXFP8DecodeOp(Op): - def __init__(self): - super().__init__() - self._op_type = OpType.BroadcastType - - class NativeMXFP8MatmulOp(Op): - def __init__(self): - super().__init__() - self._op_type = OpType.ReduceType - - class NativeAttentionMatmulOp(Op): - pass - - class NativeAttentionSoftmaxOp(Op): - pass - - compiler._ops_map.update( - { - "buddy_native.mxfp8_encode_fixed.default": NativeMXFP8EncodeOp, - "buddy_native.mxfp8_decode.default": NativeMXFP8DecodeOp, - "buddy_native.mxfp8_linear.default": NativeMXFP8MatmulOp, - "buddy_native.attention_matmul.default": NativeAttentionMatmulOp, - "buddy_native.attention_softmax.default": NativeAttentionSoftmaxOp, - } - ) - compiler._ops_registry.update( - { - "NativeMXFP8EncodeOp": codec, - "NativeMXFP8DecodeOp": codec, - "CatOp": concat, - "NativeAttentionMatmulOp": reductions.bmm, - "NativeAttentionSoftmaxOp": reductions.softmax, - "MeanOp": mean, - "BatchMatmulOp": bmm, - "SoftmaxOp": softmax, - "GeluOp": gelu, - "TanhOp": tanh, - "NativeMXFP8MatmulOp": lower_matmul, - } - ) - - -def scale_for(maximum): - exponent = torch.where(maximum == 0, 0, torch.frexp(maximum)[1] - 9) - exponent += torch.ldexp(maximum, -exponent) > 448 - return (exponent.clamp(-127, 119) + 127).to(torch.int8) - - -class Linear(nn.Module): - def __init__(self, source): - super().__init__() - self.in_features, self.out_features = ( - source.in_features, - source.out_features, - ) - codes, scales = mxfp8_cache.encode(source.weight.detach().float()) - self.register_buffer("codes", codes) - self.register_buffer("scales", scales) - self.register_buffer( - "activation_scales", - torch.full((1, self.in_features // 32), 127, dtype=torch.int8), - ) - self.bias = source.bias - self.calibrating = True - self.register_buffer( - "maximum", torch.zeros(self.in_features // 32), persistent=False - ) - - def forward(self, values): - shape = values.shape - values = values.reshape(-1, self.in_features) - if self.calibrating: - maximum = ( - values.detach() - .reshape(-1, self.in_features // 32, 32) - .abs() - .amax((0, 2)) - ) - self.maximum = torch.maximum(self.maximum, maximum) - result = nn.functional.linear( - values, decode(self.codes, self.scales), self.bias - ) - else: - result = matmul( - encode_fixed(values, self.activation_scales), - self.activation_scales, - self.codes, - self.scales, - ) - if self.bias is not None: - result = result + self.bias - return result.reshape(*shape[:-1], self.out_features) - - -class Embedding(nn.Module): - def __init__(self, source, scale=1.0): - super().__init__() - self.scale = scale - codes, scales = mxfp8_cache.encode(source.weight.detach().float()) - self.register_buffer("codes", codes) - self.register_buffer("scales", scales) - self.num_embeddings, self.embedding_dim = ( - source.num_embeddings, - source.embedding_dim, - ) - - def forward(self, tokens): - return ( - decode( - nn.functional.embedding(tokens, self.codes), - nn.functional.embedding(tokens, self.scales), - ) - * self.scale - ) - - -def prepare(model, *, embedding_scales=None): - """Replace supported weight consumers before collecting activation ranges.""" - embedding_scales = {} if embedding_scales is None else embedding_scales - for name, module in list(model.named_modules()): - if not name: - continue - parent_name, _, child_name = name.rpartition(".") - parent = model.get_submodule(parent_name) - if type(module) is nn.Linear: - setattr(parent, child_name, Linear(module)) - elif isinstance(module, nn.Embedding): - setattr( - parent, - child_name, - Embedding(module, embedding_scales.get(name, 1.0)), - ) - from transformers.masking_utils import ALL_MASK_ATTENTION_FUNCTIONS - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - - ALL_ATTENTION_FUNCTIONS.register("buddy_mxfp8", attention) - ALL_MASK_ATTENTION_FUNCTIONS.register( - "buddy_mxfp8", ALL_MASK_ATTENTION_FUNCTIONS["eager"] - ) - model.set_attn_implementation("buddy_mxfp8") - return model - - -def finish_calibration(model): - scales = {} - for name, module in model.named_modules(): - if isinstance(module, Linear): - module.activation_scales.copy_(scale_for(module.maximum)[None, :]) - module.calibrating = False - scales[name] = (module.activation_scales.to(torch.int16) & 255).tolist() - del module.maximum - return scales - - -class CacheCalibration: - """Observe per-head KV blocks while the model executes calibration samples.""" - - def __init__(self): - self.maximum = {} - self.key_minimum = {} - self.key_maximum = {} - - def update(self, key, value, layer_idx): - maximum = torch.stack( - [ - t.detach() - .reshape(*t.shape[:-1], t.shape[-1] // 32, 32) - .abs() - .amax((0, 2, 4)) - for t in (key, value) - ] - ) - low, high = key.detach().amin((0, 2)), key.detach().amax((0, 2)) - if layer_idx in self.maximum: - low = torch.minimum(low, self.key_minimum[layer_idx]) - high = torch.maximum(high, self.key_maximum[layer_idx]) - maximum = torch.maximum(maximum, self.maximum[layer_idx]) - self.maximum[layer_idx] = maximum - self.key_minimum[layer_idx], self.key_maximum[layer_idx] = low, high - return key, value - - def parameters(self): - centers = {} - for layer, maximum in self.maximum.items(): - low, high = self.key_minimum[layer], self.key_maximum[layer] - centers[layer] = (low + high) * 0.5 - radius = (high - low) * 0.5 - maximum[0] = radius.reshape(*radius.shape[:-1], -1, 32).amax(-1) - return self.maximum, centers - - -class Cache: - """Explicit fixed-capacity cache; only MXFP8 bytes persist between calls.""" - - def __init__(self, codes, positions, layouts, scales, centers, capacity): - self.codes = codes - self.positions = positions - self.layouts = layouts - self.scales = scales - self.centers = centers - self.capacity = capacity - self.updated = [] - - def update(self, key, value, layer_idx): - offset, heads, width = self.layouts[layer_idx] - size = 2 * heads * self.capacity * width - old = self.codes[offset : offset + size].reshape( - 2, 1, heads, self.capacity, width - ) - scales = self.scales[layer_idx] - # A shared K translation cancels in softmax(Q K^T). Keep it centered. - key = key - self.centers[layer_idx].reshape(1, heads, 1, width) - keys = encode_fixed(key, scales[0].reshape(1, heads, 1, width // 32)) - values = encode_fixed(value, scales[1].reshape(1, heads, 1, width // 32)) - current = torch.stack((keys, values)) - update = torch.arange(self.capacity, device=key.device)[ - None, None, None, :, None - ] == self.positions.reshape(1, 1, 1, 1, 1) - packed = torch.where(update, current, old) - self.updated.append(packed.flatten()) - return decode(packed[0], scales[0].reshape(1, heads, 1, width // 32)), decode( - packed[1], scales[1].reshape(1, heads, 1, width // 32) - ) - - -def calibrate(model, dataset, *, samples, batch_size): - from torch.utils.data import DataLoader, Subset - - if not 0 < samples <= len(dataset): - raise ValueError("Calibration sample count must be within the dataset size") - indices = torch.randperm(len(dataset), generator=torch.Generator().manual_seed(0))[ - :samples - ] - loader = DataLoader(Subset(dataset, indices.tolist()), batch_size=batch_size) - cache = CacheCalibration() - completed = 0 - with torch.no_grad(): - for tokens in loader: - positions = torch.arange(tokens.shape[1])[None, :] - visible = positions[:, :, None] >= positions[:, None, :] - mask = torch.where(visible[:, None], 0.0, torch.finfo(torch.float32).min) - model( - input_ids=tokens, - position_ids=positions, - attention_mask={ - "full_attention": mask, - "sliding_attention": mask, - }, - past_key_values=cache, - use_cache=True, - ) - completed += tokens.shape[0] - print(f"Calibrated {completed}/{samples} sequences", flush=True) - return finish_calibration(model), cache diff --git a/waste/stack/compiler/frontend/quant/offline/__init__.py b/waste/stack/compiler/frontend/quant/offline/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/waste/stack/compiler/frontend/quant/offline/activations/__init__.py b/waste/stack/compiler/frontend/quant/offline/activations/__init__.py deleted file mode 100644 index f28972127..000000000 --- a/waste/stack/compiler/frontend/quant/offline/activations/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from stack.compiler.frontend.quant.offline.activations.calibrate import ( - calibrate_activations, - calibrate_layer_ranges, -) - -__all__ = ["calibrate_activations", "calibrate_layer_ranges"] diff --git a/waste/stack/compiler/frontend/quant/offline/activations/calibrate.py b/waste/stack/compiler/frontend/quant/offline/activations/calibrate.py deleted file mode 100644 index 3d9766a63..000000000 --- a/waste/stack/compiler/frontend/quant/offline/activations/calibrate.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Calibrate symmetric INT8 scales from model inputs.""" - -from collections.abc import Mapping -from copy import deepcopy - -import torch -from torch.utils.data import DataLoader, Subset - -from stack.compiler.frontend.quant.offline.weights import dequantize_weights - - -def calibrate_activations( - model, - quantized, - dataset, - *, - samples, - layers, - batch_size=64, - granularities=None, -): - """Calibrate selected module inputs on a copy using dequantized weights. - - The model supplies its dataset and graph-node-to-module mapping. Sampling - and batching are fixed here so repeated calibration is deterministic. - """ - model, loader = _calibration_batches(model, quantized, dataset, samples, batch_size) - modules = dict(model.named_modules()) - maxima = dict.fromkeys(layers) - observed = set() - handles = [] - selected = [(name, modules[layer]) for name, layer in layers.items()] - for name, module in selected: - - def record(module, inputs, *, name=name): - values = inputs[0].detach() - if values.dtype != torch.float32 or not torch.isfinite(values).all(): - raise ValueError(f"{name}: expected finite FP32 activations") - if granularities is not None and granularities[name] == "channel": - maximum = values.abs().amax(dim=(0, 2, 3)).cpu() - else: - maximum = values.abs().amax().cpu() - maxima[name] = ( - maximum - if maxima[name] is None - else torch.maximum(maxima[name], maximum) - ) - observed.add(name) - - handles.append(module.register_forward_pre_hook(record)) - try: - with torch.no_grad(): - for inputs, _ in loader: - if isinstance(inputs, Mapping): - model(**inputs) - else: - model(inputs) - finally: - for handle in handles: - handle.remove() - if observed != set(layers) or not layers: - raise ValueError("Calibration did not observe every selected layer") - result = {} - for name, maximum in maxima.items(): - scale = torch.where(maximum == 0, 1.0, maximum / 127.0) - result[name] = float(scale) if scale.ndim == 0 else scale.tolist() - return result - - -def _calibration_batches(model, quantized, dataset, samples, batch_size=64): - model = deepcopy(model).eval() - if quantized is not None: - restored = dequantize_weights(quantized) - model.load_state_dict({name: restored[name] for name in model.state_dict()}) - if not 0 < samples <= len(dataset): - raise ValueError("Calibration sample count must be within the dataset size") - indices = torch.randperm(len(dataset), generator=torch.Generator().manual_seed(0))[ - :samples - ] - loader = DataLoader(Subset(dataset, indices.tolist()), batch_size=batch_size) - return model, loader - - -def calibrate_layer_ranges(model, dataset, *, samples, batch_size=64, quantized=None): - """Calibrate integer backend input/output ranges on deterministic dataset samples.""" - from stack.compiler.frontend.quant.transforms.activation import calibrate_layers - - model, loader = _calibration_batches(model, quantized, dataset, samples, batch_size) - return calibrate_layers(model, (inputs for inputs, _ in loader)) diff --git a/waste/stack/compiler/frontend/quant/offline/weights/__init__.py b/waste/stack/compiler/frontend/quant/offline/weights/__init__.py deleted file mode 100644 index 95a10bb97..000000000 --- a/waste/stack/compiler/frontend/quant/offline/weights/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from stack.compiler.frontend.quant.offline.weights.quantize import ( - dequantize_weights, - quantize_weights, -) - -__all__ = ["dequantize_weights", "quantize_weights"] diff --git a/waste/stack/compiler/frontend/quant/offline/weights/quantize.py b/waste/stack/compiler/frontend/quant/offline/weights/quantize.py deleted file mode 100644 index 306316b1d..000000000 --- a/waste/stack/compiler/frontend/quant/offline/weights/quantize.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Named offline weight artifacts, independent of backend memory layouts.""" - -from collections.abc import Mapping - -import torch - -from stack.compiler.frontend.quant.offline.weights.strategy.fp32toint8 import ( - quantize_int8, -) - - -def quantize_weights( - parameters: Mapping[str, torch.Tensor], - weights: Mapping[str, Mapping], -) -> dict: - """Return quantized weights, scales, and channel axes without file I/O. - - Each weight specifies source_dtype, target_dtype, method, and axis. - Unselected parameters and buffers retain their values and dtypes. - Tensor shapes and element order are unchanged. - """ - if not weights: - raise ValueError("No weights selected for quantization") - unknown = weights.keys() - parameters.keys() - if unknown: - raise ValueError(f"Unknown weight names: {sorted(unknown)}") - - tensors, scales, channel_axes = {}, {}, {} - error_energy = 0.0 - elements = 0 - for name, parameter in parameters.items(): - if name in weights: - config = weights[name] - if set(config) != { - "source_dtype", - "target_dtype", - "method", - "axis", - }: - raise ValueError( - f"{name}: expected source_dtype, target_dtype, method, and axis" - ) - scheme = ( - config["source_dtype"], - config["target_dtype"], - config["method"], - ) - if scheme != ("float32", "int8", "symmetric"): - raise ValueError(f"{name}: unsupported weight quantization {scheme}") - axis = config["axis"] - tensors[name], scales[name] = quantize_int8(parameter, axis) - channel_axes[name] = None if axis is None else axis % parameter.ndim - scale_shape = [1] * parameter.ndim - if axis is not None: - scale_shape[channel_axes[name]] = parameter.shape[axis] - restored = tensors[name].float() * scales[name].reshape(scale_shape) - original = parameter.detach().cpu().double() - error = original - restored.double() - error_energy += error.square().sum().item() - elements += parameter.numel() - else: - tensors[name] = parameter.detach().cpu().contiguous() - - print(f"Weight quantization MSE: {error_energy / elements:.6e}", flush=True) - return { - "format": "buddy.offline.symmetric_int8", - "version": 1, - "tensors": tensors, - "scales": scales, - "axes": channel_axes, - } - - -def dequantize_weights(artifact: dict) -> dict[str, torch.Tensor]: - """Reconstruct FP32 weights for offline accuracy evaluation.""" - if artifact["format"] != "buddy.offline.symmetric_int8" or artifact["version"] != 1: - raise ValueError("Unsupported offline weight artifact") - tensors = dict(artifact["tensors"]) - for name, axis in artifact["axes"].items(): - values = tensors[name] - shape = [1] * values.ndim - if axis is not None: - shape[axis] = values.shape[axis] - tensors[name] = values.float() * artifact["scales"][name].reshape(shape) - return tensors diff --git a/waste/stack/compiler/frontend/quant/offline/weights/strategy/__init__.py b/waste/stack/compiler/frontend/quant/offline/weights/strategy/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/waste/stack/compiler/frontend/quant/offline/weights/strategy/fp32toint8.py b/waste/stack/compiler/frontend/quant/offline/weights/strategy/fp32toint8.py deleted file mode 100644 index 6c2096815..000000000 --- a/waste/stack/compiler/frontend/quant/offline/weights/strategy/fp32toint8.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Offline symmetric INT8 quantization of FP32 weight tensors.""" - -import torch - - -def quantize_int8( - weight: torch.Tensor, axis: int | None -) -> tuple[torch.Tensor, torch.Tensor]: - """Return INT8 values and FP32 scales; axis=None selects one tensor scale.""" - if weight.dtype != torch.float32: - raise ValueError("INT8 weight quantization requires FP32 input") - if not weight.numel() or not torch.isfinite(weight).all(): - raise ValueError("Weights must be nonempty and finite") - if axis is not None and not -weight.ndim <= axis < weight.ndim: - raise ValueError("Quantization axis is outside the weight dimensions") - - weight = weight.detach().cpu() - if axis is None: - maximum = weight.abs().amax() - scale_shape = [1] * weight.ndim - else: - axis %= weight.ndim - maximum = weight.movedim(axis, 0).reshape(weight.shape[axis], -1).abs().amax(1) - scale_shape = [1] * weight.ndim - scale_shape[axis] = weight.shape[axis] - - # A zero channel is represented exactly by q=0 and scale=1. - scales = torch.where(maximum == 0, torch.ones_like(maximum), maximum / 127) - if (scales == 0).any(): - raise ValueError("Nonzero weight scale cannot be represented in FP32") - values = torch.round(weight / scales.reshape(scale_shape)) - values = values.clamp(-127, 127).to(torch.int8).contiguous() - return values, scales.contiguous() diff --git a/waste/stack/compiler/frontend/quant/rax.py b/waste/stack/compiler/frontend/quant/rax.py deleted file mode 100644 index ded598f09..000000000 --- a/waste/stack/compiler/frontend/quant/rax.py +++ /dev/null @@ -1,303 +0,0 @@ -from __future__ import annotations - -import json -import math -import shutil -import struct -import subprocess -from dataclasses import dataclass -from pathlib import Path - -from stack.compiler.frontend.quant.bwq import BwqPackage, validate_bwq - - -def scale_count(shape: list[int], axes: list[int]) -> int: - return math.prod(shape[axis] for axis in axes) - - -@dataclass -class QuantTensor: - name: str - shape: list[int] - payload_shape: list[int] - storage: str - axes: list[int] - payload_off: int - payload_len: int - scale_off: int - scale_len: int - layout: dict | None = None - - -@dataclass -class RaxQuantPackage: - tensors: list[QuantTensor] - weights: bytes - params_f32: bytes - scales_f32: bytes - assets: dict[str, bytes] - - -def rax_from_bwq(pkg: BwqPackage) -> RaxQuantPackage: - validate_bwq(pkg) - tensors: list[QuantTensor] = [] - weights: list[bytes] = [] - scales: list[bytes] = [] - weight_off = scale_off = 0 - for source in pkg.tensors: - if source.storage != "i8" or source.axes not in ([], [0]): - raise ValueError( - f"RAX W8A8 requires i8 tensor/channel weight: {source.name}" - ) - weight = pkg.weights[source.weight_off : source.weight_off + source.weight_len] - raw_scales = pkg.scales[source.scale_off : source.scale_off + source.scale_len] - raw_count = 1 if not source.axes else source.shape[0] - values = struct.unpack(f"<{raw_count}f", raw_scales) - padded_count = scale_count(source.shape, source.axes) - padded = values + (1.0,) * (padded_count - raw_count) - scale = struct.pack(f"<{padded_count}f", *padded) - tensors.append( - QuantTensor( - source.name, - source.shape, - source.shape, - source.storage, - source.axes, - weight_off, - len(weight), - scale_off, - len(scale), - ) - ) - weights.append(weight) - scales.append(scale) - weight_off += len(weight) - scale_off += len(scale) - return RaxQuantPackage(tensors, b"".join(weights), b"", b"".join(scales), {}) - - -def _numel(shape: list[int], name: str) -> int: - n = 1 - for d in shape: - if d <= 0: - raise ValueError(f"bad shape for {name}") - n *= d - return n - - -def validate_rax_quant(pkg: RaxQuantPackage) -> None: - if not pkg.tensors: - raise ValueError("empty tensor list") - names: set[str] = set() - weight_end = param_end = scale_end = 0 - for tensor in pkg.tensors: - if tensor.name in names: - raise ValueError(f"duplicate tensor name: {tensor.name}") - names.add(tensor.name) - if tensor.storage not in ("i8", "mxfp8", "f32"): - raise ValueError(f"unsupported storage for {tensor.name}: {tensor.storage}") - if tensor.payload_off < 0 or tensor.payload_len < 0: - raise ValueError(f"bad payload range for {tensor.name}") - if tensor.scale_off < 0 or tensor.scale_len < 0: - raise ValueError(f"bad scale range for {tensor.name}") - numel = _numel(tensor.shape, tensor.name) - payload_numel = _numel(tensor.payload_shape, tensor.name) - if tensor.storage == "i8": - if len(tensor.axes) > 1 or any( - axis < 0 or axis >= len(tensor.shape) for axis in tensor.axes - ): - raise ValueError( - f"unsupported scale axes for {tensor.name}: {tensor.axes}" - ) - if tensor.payload_off + tensor.payload_len > len(pkg.weights): - raise ValueError(f"weight OOB for {tensor.name}") - if tensor.payload_off != weight_end: - raise ValueError(f"non-contiguous weight payload for {tensor.name}") - if tensor.payload_len != payload_numel: - raise ValueError(f"weight_len mismatch for {tensor.name}") - if tensor.scale_off + tensor.scale_len > len(pkg.scales_f32): - raise ValueError(f"scale OOB for {tensor.name}") - scale_n = scale_count(tensor.shape, tensor.axes) - if tensor.scale_len != scale_n * 4: - raise ValueError(f"scale_len mismatch for {tensor.name}") - if tensor.scale_off % 4: - raise ValueError(f"unaligned scale offset for {tensor.name}") - if tensor.scale_off != scale_end: - raise ValueError(f"non-contiguous scale payload for {tensor.name}") - scales = struct.unpack_from( - f"<{scale_n}f", pkg.scales_f32, tensor.scale_off - ) - if any(not math.isfinite(scale) or scale <= 0.0 for scale in scales): - raise ValueError(f"invalid scale value for {tensor.name}") - channel_count = tensor.shape[tensor.axes[0]] if tensor.axes else 1 - if tensor.axes and any(scale != 1.0 for scale in scales[channel_count:]): - raise ValueError(f"bad channel-scale padding for {tensor.name}") - weight_end += tensor.payload_len - scale_end += tensor.scale_len - elif tensor.storage == "mxfp8": - if len(tensor.shape) != 2 or tensor.shape[1] % 32 or tensor.axes != [1]: - raise ValueError(f"invalid MXFP8 shape or block axis: {tensor.name}") - if len(tensor.payload_shape) == 2: - stride = tensor.shape[1] + tensor.shape[1] // 32 - expected = tensor.shape[0] * stride - if ( - tensor.payload_shape != [tensor.shape[0], stride] - or tensor.layout != {"row_stride": stride} - or tensor.payload_len != expected - ): - raise ValueError( - f"invalid MXFP8 embedding row layout: {tensor.name}" - ) - elif len(tensor.payload_shape) == 1: - rows, k = tensor.layout["tile_n"], tensor.layout["tile_k"] - stride = tensor.layout["panel_stride"] - expected = ( - ((tensor.shape[0] + rows - 1) // rows) - * ((tensor.shape[1] + k - 1) // k) - * stride - ) - if ( - rows % 16 - or k % 32 - or stride != tensor.layout["bank_bytes"] - or rows * k * 33 // 32 > stride - or tensor.payload_len != expected - or payload_numel != expected - ): - raise ValueError(f"invalid MXFP8 packed layout: {tensor.name}") - else: - raise ValueError(f"invalid MXFP8 payload rank: {tensor.name}") - if tensor.payload_off != weight_end or weight_end + expected > len( - pkg.weights - ): - raise ValueError(f"invalid MXFP8 weight range: {tensor.name}") - if tensor.scale_off or tensor.scale_len: - raise ValueError("MXFP8 scales are embedded E8M0 bytes") - weight_end += tensor.payload_len - else: - if tensor.axes or tensor.scale_off or tensor.scale_len: - raise ValueError(f"f32 parameter has quant metadata: {tensor.name}") - if tensor.payload_off + tensor.payload_len > len(pkg.params_f32): - raise ValueError(f"f32 parameter OOB for {tensor.name}") - if tensor.payload_off != param_end: - raise ValueError(f"non-contiguous f32 payload for {tensor.name}") - if tensor.payload_len != numel * 4: - raise ValueError(f"f32 parameter size mismatch for {tensor.name}") - param_end += tensor.payload_len - if weight_end != len(pkg.weights): - raise ValueError("unused weight payload") - if param_end != len(pkg.params_f32): - raise ValueError("unused FP32 parameter payload") - if scale_end != len(pkg.scales_f32): - raise ValueError("unused scale payload") - - -def _payload_dir(rax: Path) -> Path: - return rax.parent / f"{rax.stem}.payload" - - -def _quant_index(pkg: RaxQuantPackage) -> dict: - tensors = [] - for tensor in pkg.tensors: - tensors.append( - { - "name": tensor.name, - "shape": tensor.shape, - "payload_shape": tensor.payload_shape, - "storage": tensor.storage, - "payload": "params_f32" if tensor.storage == "f32" else "weights", - "payload_offset": tensor.payload_off, - "payload_bytes": tensor.payload_len, - "scale_offset": tensor.scale_off if tensor.storage == "i8" else 0, - "scale_bytes": tensor.scale_len if tensor.storage == "i8" else 0, - "scale_axes": tensor.axes, - **( - { - "layout": tensor.layout, - "block_size": 32, - "scale_encoding": "e8m0", - } - if tensor.storage == "mxfp8" - else {} - ), - } - ) - return {"version": 2, "tensors": tensors} - - -def _manifest( - model_name: str, - payload_name: str, - weights_bytes: int, - params_bytes: int, - scales_bytes: int, - index_bytes: int, - assets: dict[str, bytes], -) -> str: - asset_constants = "".join( - f' rhal.constant @{name.replace(".", "_")} {{id = {5 + i} : i32, storage = "external",\n' - f" type = tensor<{len(data)}xi8>,\n" - f' uri = "file:{payload_name}/{name}"}}\n' - for i, (name, data) in enumerate(assets.items()) - ) - return f"""rhal.module @quant_{model_name} attributes {{ - version = "0.1.0", - model_name = "{model_name}"}} {{ - rhal.constant @weights {{id = 1 : i32, storage = "external", - type = tensor<{weights_bytes}xi8>, - uri = "file:{payload_name}/weights.bin"}} - rhal.constant @params_f32 {{id = 2 : i32, storage = "external", - type = tensor<{params_bytes}xi8>, - uri = "file:{payload_name}/params.f32"}} - rhal.constant @scales_f32 {{id = 3 : i32, storage = "external", - type = tensor<{scales_bytes}xi8>, - uri = "file:{payload_name}/scales.bin"}} - rhal.constant @quant_index {{id = 4 : i32, storage = "external", - type = tensor<{index_bytes}xi8>, - uri = "file:{payload_name}/quant-index.json"}} -{asset_constants}}} -""" - - -def write_payload(pkg: RaxQuantPackage, payload_dir: Path | str) -> None: - validate_rax_quant(pkg) - payload_dir = Path(payload_dir) - if payload_dir.exists(): - shutil.rmtree(payload_dir) - payload_dir.mkdir(parents=True) - index = json.dumps(_quant_index(pkg), sort_keys=True, indent=2).encode("ascii") - (payload_dir / "weights.bin").write_bytes(pkg.weights) - (payload_dir / "params.f32").write_bytes(pkg.params_f32) - (payload_dir / "scales.bin").write_bytes(pkg.scales_f32) - (payload_dir / "quant-index.json").write_bytes(index) - for name, data in pkg.assets.items(): - (payload_dir / name).write_bytes(data) - - -def write_rax( - pkg: RaxQuantPackage, rax: Path | str, rax_pack: Path | str, model_name: str -) -> None: - rax = Path(rax) - rax_pack = Path(rax_pack) - if not rax_pack.is_file(): - raise ValueError(f"rax-pack not found: {rax_pack}") - payload_dir = _payload_dir(rax) - write_payload(pkg, payload_dir) - index = (payload_dir / "quant-index.json").read_bytes() - manifest = payload_dir / "quant.rhal.mlir" - manifest.write_text( - _manifest( - model_name, - payload_dir.name, - len(pkg.weights), - len(pkg.params_f32), - len(pkg.scales_f32), - len(index), - pkg.assets, - ), - encoding="ascii", - ) - subprocess.run( - [str(rax_pack), str(manifest), "-o", str(rax), "--embed-payload"], - check=True, - ) diff --git a/waste/stack/compiler/frontend/quant/transforms/__init__.py b/waste/stack/compiler/frontend/quant/transforms/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/waste/stack/compiler/frontend/quant/transforms/activation.py b/waste/stack/compiler/frontend/quant/transforms/activation.py deleted file mode 100644 index f1f55fda8..000000000 --- a/waste/stack/compiler/frontend/quant/transforms/activation.py +++ /dev/null @@ -1,177 +0,0 @@ -from __future__ import annotations - -import numpy as np -import torch - - -def _activation_array(values: np.ndarray) -> np.ndarray: - values = np.asarray(values, dtype=np.float32) - if values.size == 0: - raise ValueError("activation tensor is empty") - if not np.all(np.isfinite(values)): - raise ValueError("activation tensor contains NaN or infinity") - return values - - -def activation_max_abs(values: np.ndarray) -> np.float32: - values = _activation_array(values) - return np.float32(np.max(np.abs(values))) - - -def activation_scale(values: np.ndarray) -> np.float32: - max_abs = activation_max_abs(values) - if max_abs == 0.0: - return np.float32(1.0) - return np.float32(max_abs / np.float32(127.0)) - - -def calibrate_layers(model: torch.nn.Module, batches) -> dict: - """Aggregate layer ranges across all batches before converting to scales.""" - ranges, calls = {}, {} - handles = [] - for name, module in model.named_modules(): - if not isinstance(module, (torch.nn.Conv2d, torch.nn.Linear)): - continue - - def record(layer, inputs, output, weight_name=f"{name}.weight"): - if ( - len(inputs) != 1 - or not torch.is_tensor(inputs[0]) - or not torch.is_tensor(output) - ): - raise ValueError(f"unsupported calibration signature for {weight_name}") - values = _activation_array(output.detach().cpu().numpy()) - axis = values.ndim - 1 if isinstance(layer, torch.nn.Linear) else 1 - axes = tuple(i for i in range(values.ndim) if i != axis) - hardswish = ( - values * np.clip(values + np.float32(3.0), 0.0, 6.0) / np.float32(6.0) - ) - observed = ( - activation_max_abs(inputs[0].detach().cpu().numpy()), - activation_max_abs(values), - activation_max_abs(np.maximum(values, np.float32(0.0))), - np.max(np.abs(values), axis=axes), - np.max(np.abs(hardswish), axis=axes), - ) - index = calls.get(weight_name, 0) - calls[weight_name] = index + 1 - records = ranges.setdefault(weight_name, []) - if index == len(records): - records.append(observed) - else: - records[index] = tuple( - np.maximum(a, b) - for a, b in zip(records[index], observed, strict=True) - ) - - handles.append(module.register_forward_hook(record)) - expected = None - try: - with torch.no_grad(): - for inputs in batches: - calls.clear() - if isinstance(inputs, dict): - model(**inputs) - else: - model(inputs) - if expected is None: - expected = dict(calls) - elif calls != expected: - raise ValueError( - "Calibration batches executed different model layers" - ) - finally: - for handle in handles: - handle.remove() - if not ranges: - raise ValueError("calibration found no Conv2d or Linear activations") - return { - name: [ - tuple( - np.where(value == 0, np.float32(1.0), value / np.float32(127.0)).astype( - np.float32 - ) - for value in record - ) - for record in records - ] - for name, records in ranges.items() - } - - -def quantize_activation_with_scale( - values: np.ndarray, da: float | np.float32 -) -> np.ndarray: - values = _activation_array(values) - da = np.float32(da) - if not np.isfinite(da) or da <= 0.0: - raise ValueError("Da must be finite and positive") - return np.clip(np.rint(values / da), -128, 127).astype(np.int8) - - -def quantize_activation(values: np.ndarray) -> tuple[np.ndarray, np.float32]: - da = activation_scale(values) - return quantize_activation_with_scale(values, da), da - - -def quantize_activation_banks( - values: np.ndarray, bank_rows: int -) -> tuple[np.ndarray, np.ndarray]: - values = _activation_array(values) - if values.ndim != 2: - raise ValueError("bank activation quantization requires a rank-2 matrix") - if type(bank_rows) is not int or bank_rows <= 0: - raise ValueError("bank_rows must be a positive integer") - quantized = np.empty(values.shape, dtype=np.int8) - bank_count = (values.shape[0] + bank_rows - 1) // bank_rows - scales = np.empty(bank_count, dtype=np.float32) - for bank, row in enumerate(range(0, values.shape[0], bank_rows)): - bank_quantized, da = quantize_activation(values[row : row + bank_rows]) - quantized[row : row + bank_rows] = bank_quantized - scales[bank] = da - return quantized, scales - - -def dequantize_accumulator( - accumulator: np.ndarray, da: float | np.float32, dw: float | np.ndarray -) -> np.ndarray: - accumulator = np.asarray(accumulator, dtype=np.int32) - da = np.float32(da) - dw = np.asarray(dw, dtype=np.float32) - if not np.isfinite(da) or da <= 0.0: - raise ValueError("Da must be finite and positive") - if not np.all(np.isfinite(dw)) or np.any(dw <= 0.0): - raise ValueError("Dw must be finite and positive") - if dw.ndim > 1 or (dw.ndim == 1 and accumulator.ndim == 0): - raise ValueError("Dw must be scalar or a last-dimension scale vector") - if dw.ndim == 1 and accumulator.shape[-1] != dw.shape[0]: - raise ValueError("Dw channel count does not match accumulator") - return accumulator.astype(np.float32) * da * dw - - -def w8a8_matmul( - activations: np.ndarray, - weights_i8: np.ndarray, - dw: float | np.ndarray, - bank_rows: int, -) -> tuple[np.ndarray, np.ndarray]: - activations = _activation_array(activations) - weights_i8 = np.asarray(weights_i8) - if activations.ndim != 2 or weights_i8.ndim != 2: - raise ValueError("W8A8 matmul requires rank-2 activation and weight tensors") - if weights_i8.dtype != np.int8: - raise ValueError("W8A8 weights must be INT8") - if activations.shape[1] != weights_i8.shape[0]: - raise ValueError("W8A8 matmul K dimensions do not match") - if type(bank_rows) is not int or bank_rows <= 0: - raise ValueError("bank_rows must be a positive integer") - quantized, da_values = quantize_activation_banks(activations, bank_rows) - result = np.empty((activations.shape[0], weights_i8.shape[1]), dtype=np.float32) - for bank, row in enumerate(range(0, activations.shape[0], bank_rows)): - accumulator = quantized[row : row + bank_rows].astype( - np.int32 - ) @ weights_i8.astype(np.int32) - result[row : row + bank_rows] = dequantize_accumulator( - accumulator, da_values[bank], dw - ) - return result, da_values diff --git a/waste/stack/compiler/frontend/quant/transforms/normalize.py b/waste/stack/compiler/frontend/quant/transforms/normalize.py deleted file mode 100644 index 0d600eff6..000000000 --- a/waste/stack/compiler/frontend/quant/transforms/normalize.py +++ /dev/null @@ -1,20 +0,0 @@ -import torch - - -def fold_batch_norms(model: torch.nn.Module) -> None: - if model.training: - raise ValueError("BatchNorm folding requires an eval-mode model") - for parent in model.modules(): - names = list(parent._modules) - for index in range(len(names) - 1): - conv_name, bn_name = names[index : index + 2] - conv = parent._modules[conv_name] - bn = parent._modules[bn_name] - if not isinstance(conv, torch.nn.Conv2d) or not isinstance( - bn, torch.nn.BatchNorm2d - ): - continue - parent._modules[conv_name] = torch.nn.utils.fusion.fuse_conv_bn_eval( - conv, bn - ) - parent._modules[bn_name] = torch.nn.Identity() diff --git a/waste/stack/compiler/frontend/quant/transforms/symmetric.py b/waste/stack/compiler/frontend/quant/transforms/symmetric.py deleted file mode 100644 index d7b203858..000000000 --- a/waste/stack/compiler/frontend/quant/transforms/symmetric.py +++ /dev/null @@ -1,28 +0,0 @@ -from __future__ import annotations - -import numpy as np - - -def quantize_symmetric(w: np.ndarray, axes: list[int]) -> tuple[np.ndarray, np.ndarray]: - if w.dtype != np.float32: - raise ValueError(f"expected float32, got {w.dtype}") - if len(axes) != len(set(axes)): - raise ValueError("duplicate axes") - if any(a < 0 or a >= w.ndim for a in axes): - raise ValueError("axes out of range") - abs_w = np.abs(w) - if not axes: - max_abs = float(abs_w.max()) - if max_abs == 0.0: - raise ValueError("maxAbs is 0") - dw = np.float32(max_abs / 127.0) - q = np.clip(np.rint(w / dw), -128, 127).astype(np.int8) - return q, np.array(dw) - reduce_axes = tuple(i for i in range(w.ndim) if i not in axes) - max_abs = abs_w.max(axis=reduce_axes, keepdims=True) - if np.any(max_abs == 0): - raise ValueError("maxAbs is 0") - dw_keep = (max_abs / 127.0).astype(np.float32) - q = np.clip(np.rint(w / dw_keep), -128, 127).astype(np.int8) - dw = np.squeeze(dw_keep, axis=reduce_axes) - return q, dw.astype(np.float32) diff --git a/waste/stack/compiler/frontend/trace.py b/waste/stack/compiler/frontend/trace.py deleted file mode 100644 index d756a2ccd..000000000 --- a/waste/stack/compiler/frontend/trace.py +++ /dev/null @@ -1,12 +0,0 @@ -from pathlib import Path -from buddy.compiler.trace import TraceConfig, load_trace_config -from buddy.compiler.graph.transform.trace import TraceInsertionPass -from .operation import QuantizedOp - - -def trace(graph, path): - TraceInsertionPass(TraceConfig(load_trace_config(Path(path))))(graph) - for node in graph.body: - if isinstance(node, QuantizedOp): - node.quantization_trace_meta = node.trace_meta - node.trace_meta = None diff --git a/waste/stack/compiler/frontend/transforms.py b/waste/stack/compiler/frontend/transforms.py deleted file mode 100644 index f8655e852..000000000 --- a/waste/stack/compiler/frontend/transforms.py +++ /dev/null @@ -1,194 +0,0 @@ -# ===- graph.py ---------------------------------------------------------------- -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ===--------------------------------------------------------------------------- -# -# This is the graph level of the Buddy Compiler frontend. -# -# ===--------------------------------------------------------------------------- - -import torch -from pathlib import Path - -from buddy.compiler.graph.operation import * -from buddy.compiler.graph.graph import NodeType, _replace_node_name -from buddy.compiler.graph.type import TensorDType -from stack.compiler.frontend.operation import * - - -def quantize( - graph, - config, - *, - parameter_names=None, - weights=None, - activation_scales=None, - calibration_model=None, - calibration_dataset=None, -): - """Apply the weight and activation policies from a quantization TOML. - - Offline artifacts supply data; the configuration selects transformations - and whether each activation scale is calibrated or computed online. - """ - from stack.compiler.frontend.quant.transforms.activation_quantize import ( - quantize_activations, - ) - - if "weights" in config: - if parameter_names is None: - raise ValueError("Weight quantization requires parameter names") - if weights is None: - from stack.compiler.frontend.quant.offline.weights import quantize_weights - - parameters = ( - dict(calibration_model.state_dict()) - if calibration_model is not None - else {} - ) - for node, value in zip(graph.params, graph._params_ref, strict=True): - if "weight_permutation" in node.tensor_meta: - permutation = node.tensor_meta["weight_permutation"] - inverse = [permutation.index(axis) for axis in range(value.ndim)] - value = value.permute(inverse).contiguous() - parameters[parameter_names[node.name]] = value - weights = quantize_weights(parameters, config["weights"]) - if set(config["weights"]) != set(weights["axes"]): - raise ValueError("Offline weights must match the configured weight names") - if "calibration" in config and activation_scales is None: - from stack.compiler.frontend.quant.offline.activations import ( - calibrate_activations, - ) - - calibration = dict(config["calibration"]) - if config.get("execution") == "int8": - calibration["layers"] = { - name: policy["module"] - for name, policy in config["activations"].items() - } - calibration["granularities"] = { - name: policy["granularity"] - for name, policy in config["activations"].items() - } - activation_scales = calibrate_activations( - calibration_model, - weights, - calibration_dataset, - **calibration, - ) - bind_quantized_weights(graph, weights, parameter_names) - if "activations" in config: - quantize_activations( - graph, - config["activations"], - activation_scales, - config.get("activation_target", "native"), - ) - if "execution" in config: - if config["execution"] != "int8": - raise ValueError("Unsupported quantized execution policy") - from stack.compiler.frontend.ops.reductions import ( - bmm, - gelu, - layer_norm, - mean, - softmax, - tanh, - ) - from stack.compiler.frontend.quant.transforms.integerize import integerize - - integerize(graph, config["activations"]) - graph._ops_registry = { - **graph._ops_registry, - "MeanOp": mean, - "BatchMatmulOp": bmm, - "SoftmaxOp": softmax, - "NativeLayerNormOp": layer_norm, - "GeluOp": gelu, - "TanhOp": tanh, - } - return graph._params_ref - - -def bind_quantized_weights(graph, quantized, parameter_names): - """Bind offline INT8 weights and insert runtime weight dequantization. - - parameter_names maps graph parameter names to checkpoint names. - Update the imported parameter list in place to match graph.params. - """ - from buddy.compiler.graph.transform.quantization.quantize import sort_graph - - parameters = list(graph.params) - values = {} - for node in parameters: - name = parameter_names[node.name] - value = quantized["tensors"][name] - if "weight_permutation" in node.tensor_meta: - value = value.permute(node.tensor_meta["weight_permutation"]).contiguous() - values[node.name] = value - if tuple(value.shape) != tuple(node.tensor_meta["shape"]): - raise ValueError(f"{name}: offline weight shape differs from graph") - if name not in quantized["axes"]: - continue - if value.dtype != torch.int8: - raise ValueError(f"{name}: expected offline INT8 weights") - node.tensor_meta["dtype"] = TensorDType.Int8 - shape = [1] * value.ndim - axis = quantized["axes"][name] - if axis is not None: - if "weight_permutation" in node.tensor_meta: - axis = node.tensor_meta["weight_permutation"].index(axis) - shape[axis] = value.shape[axis] - - scale = PlaceholderOp() - scale.name = f"scaler_{node.name}" - scale.tensor_meta = {"shape": shape, "dtype": TensorDType.Float32} - graph.add_node(scale, NodeType.FakeNode) - values[scale.name] = quantized["scales"][name].reshape(shape) - - cast = ConvertElementTypeOp() - cast.name = f"cast_{node.name}" - cast.add_argument(node.name) - cast.add_parent(node.name) - cast.tensor_meta = { - "shape": value.shape, - "dtype": TensorDType.Float32, - } - graph.add_node(cast) - - dequant = MulOp() - dequant.name = f"dequantized_{node.name}" - dequant._arguments = [cast.name, scale.name] - dequant._parents = [cast.name, scale.name] - dequant.tensor_meta = dict(cast.tensor_meta) - dequant._children = list(node._children) - graph.add_node(dequant) - for child_name in dequant._children: - child = graph.node_table[child_name] - child._arguments = _replace_node_name( - child.args, node.name, dequant.name, graph.node_table - ) - child._keyword_arguments = _replace_node_name( - child.kwargs, node.name, dequant.name, graph.node_table - ) - child._parents = [ - dequant.name if p == node.name else p for p in child.parents - ] - node._children = [cast.name] - cast._children = [dequant.name] - scale._children = [dequant.name] - - sort_graph(graph) - graph._params_ref[:] = [values[node.name] for node in graph.params] - return graph._params_ref diff --git a/waste/stack/compiler/src/Conversion/BallPasses.h b/waste/stack/compiler/src/Conversion/BallPasses.h deleted file mode 100644 index 8db10bbd7..000000000 --- a/waste/stack/compiler/src/Conversion/BallPasses.h +++ /dev/null @@ -1,131 +0,0 @@ -//===- BallPasses.h - Passes a Ball maintains ----------------------------===// -// -// A Ball owns two passes: one recognizes the Linalg operations it executes, -// the other lowers its operations to LLVM intrinsics. Bank lowering belongs to -// the framework, which a Ball joins through the interfaces in -// BuckyballInterfaces.td. -// -//===----------------------------------------------------------------------===// - -#ifndef BUDDY_CONVERSION_BALL_PASSES_H -#define BUDDY_CONVERSION_BALL_PASSES_H - -#include "Target/BuckyballTargetRegistry.h" - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/TypeConverter.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/DialectConversion.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -#include "Buckyball/BuckyballDialect.h" - -#include -#include - -namespace mlir::buddy { - -// Claims the Linalg operations a Ball executes. Derived provides -// `void populate(RewritePatternSet &, int64_t bankWidthBytes, -// int64_t bankDepth, int64_t bankNum) const`. Operations it leaves stay in -// Linalg for the CPU; a pattern that rejects an operation it claims emits an -// error, which fails the pass. -template -class LinalgToBallPass : public PassWrapper> { -public: - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert<::buddy::buckyball::BuckyballDialect, func::FuncDialect, - memref::MemRefDialect, arith::ArithDialect, scf::SCFDialect, - linalg::LinalgDialect>(); - } - - void runOnOperation() override { - const auto &config = buckyball_target::getBuckyballTarget(); - MLIRContext *context = &this->getContext(); - RewritePatternSet patterns(context); - static_cast(this)->populate( - patterns, config.bankWidthBits / 8, config.bankDepth, config.bankNum); - // Rewrite only what the Ball claims; leave the rest of the IR as it is. - GreedyRewriteConfig rewrite; - rewrite.enableFolding(false); - rewrite.setRegionSimplificationLevel(GreedySimplifyRegionLevel::Disabled); - bool emittedError = false; - ScopedDiagnosticHandler diagnostics(context, [&](Diagnostic &message) { - emittedError |= message.getSeverity() == DiagnosticSeverity::Error; - return failure(); - }); - if (failed(applyPatternsGreedily(this->getOperation(), std::move(patterns), - rewrite)) || - emittedError) - this->signalPassFailure(); - } -}; - -// Lowers a Ball's operations to LLVM intrinsics. Derived provides -// `void populate(LLVMTypeConverter &, RewritePatternSet &, -// LLVMConversionTarget &, bool stable, int64_t bankDepth) const` and marks its -// operations illegal. -template -class LowerBallPass : public PassWrapper> { -public: - LowerBallPass() = default; - LowerBallPass(const LowerBallPass &other) - : PassWrapper>(other) {} - - Pass::Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball " - "intrinsics."), - llvm::cl::init(false)}; - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert(); - } - - void runOnOperation() override { - MLIRContext *context = &this->getContext(); - LLVMTypeConverter converter(context); - RewritePatternSet patterns(context); - LLVMConversionTarget target(*context); - // Intrinsics and the operations of other Balls stay; Derived marks its - // own operations illegal. - target.addLegalDialect(); - static_cast(this)->populate( - converter, patterns, target, stable, - buckyball_target::getBuckyballTarget().bankDepth); - ConversionConfig config; - config.allowPatternRollback = false; - if (failed(applyPartialConversion(this->getOperation(), target, - std::move(patterns), config))) - this->signalPassFailure(); - } -}; - -// `lower-buckyball` runs the framework lowering and then every Ball lowering -// pass registered here. -void addBallLowering(std::function(bool stable)> create); - -template void registerBallLoweringPass() { - PassRegistration(); - addBallLowering([](bool stable) { - auto pass = std::make_unique(); - pass->stable = stable; - return pass; - }); -} - -} // namespace mlir::buddy - -#endif // BUDDY_CONVERSION_BALL_PASSES_H diff --git a/waste/stack/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp b/waste/stack/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp deleted file mode 100644 index 592eaaa1a..000000000 --- a/waste/stack/compiler/src/Conversion/LowerBuckyball/AssignPhysicalBanksPass.cpp +++ /dev/null @@ -1,97 +0,0 @@ -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/IR/Visitors.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Rewrite/FrozenRewritePatternSet.h" -#include "mlir/Rewrite/PatternApplicator.h" - -#include - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; - -namespace { -class PhysicalBankAssignmentPattern - : public OpInterfaceRewritePattern< - ::buddy::buckyball::PhysicalBankAssignment> { -public: - PhysicalBankAssignmentPattern(MLIRContext *context, - mlir::buddy::PhysicalBankState &state) - : OpInterfaceRewritePattern(context), state(state) {} - - LogicalResult matchAndRewrite(::buddy::buckyball::PhysicalBankAssignment op, - PatternRewriter &rewriter) const override { - return op.assignPhysicalBanks(rewriter, state); - } - -private: - mlir::buddy::PhysicalBankState &state; -}; - -class AssignPhysicalBanksPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(AssignPhysicalBanksPass) - - StringRef getArgument() const final { return "assign-physical-banks"; } - StringRef getDescription() const final { - return "Assign physical banks for the selected Buckyball target."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); - } - - void runOnOperation() override { - const buckyball_target::BuckyballTargetConfig &target = - buckyball_target::getBuckyballTarget(); - for (func::FuncOp func : getOperation().getOps()) { - mlir::buddy::PhysicalBankState state(target.bankNum, - target.privateBankMax); - RewritePatternSet patterns(&getContext()); - mlir::buddy::addBaseAssignPhysicalBankPatterns(patterns, state); - patterns.add(&getContext(), state); - FrozenRewritePatternSet frozen(std::move(patterns)); - PatternApplicator applicator(frozen); - applicator.applyDefaultCostModel(); - PatternRewriter rewriter(&getContext()); - bool assignmentFailed = false; - std::function visitRegion = [&](Region ®ion) { - for (Block &block : region) { - for (auto it = block.begin(); it != block.end();) { - if (assignmentFailed) - return; - Operation *op = &*it++; - if (succeeded(applicator.matchAndRewrite(op, rewriter))) - continue; - if (op->getName().getStringRef().starts_with("buckyball.bank_")) { - assignmentFailed = true; - return; - } - for (Region &nested : op->getRegions()) - visitRegion(nested); - } - } - }; - for (Region ®ion : func->getRegions()) - visitRegion(region); - if (assignmentFailed || - mlir::failed(mlir::buddy::verifyNoBankSSAOps(func)) || - !state.empty()) { - signalPassFailure(); - return; - } - } - } -}; -} // namespace - -void mlir::buddy::registerAssignPhysicalBanksPass() { - PassRegistration(); -} diff --git a/waste/stack/compiler/src/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp b/waste/stack/compiler/src/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp deleted file mode 100644 index 87b4fdc96..000000000 --- a/waste/stack/compiler/src/Conversion/LowerBuckyball/BaseAssignPhysicalBankPatterns.cpp +++ /dev/null @@ -1,212 +0,0 @@ -//===- BaseAssignPhysicalBankPatterns.cpp - Base bank assignment ----------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "Buckyball/BuckyballOps.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/IR/PatternMatch.h" - -using namespace mlir; -using namespace mlir::buddy; -using namespace ::buddy::buckyball; - -namespace { - -class BankAllocPattern : public OpRewritePattern { -public: - BankAllocPattern(MLIRContext *context, PhysicalBankState &state) - : OpRewritePattern(context), state(state) {} - - LogicalResult matchAndRewrite(BankAllocOp op, - PatternRewriter &rewriter) const override { - int64_t row = op.getRow(); - int64_t col = op.getCol(); - if (row <= 0 || col <= 0) - return op.emitError("assign-physical-banks: invalid bank shape"); - - auto bank = state.tryAlloc(row, col); - if (!bank) { - InFlightDiagnostic diagnostic = - op.emitError("assign-physical-banks: unavailable bank resources"); - diagnostic << " (request=" << row << "x" << col - << ", used=" << state.getUsedCount() << "/" - << state.getBankNum() - << ", private ID max=" << state.getPrivateBankMax() << ")"; - return failure(); - } - - state.createMset(rewriter, op.getLoc(), static_cast(*bank), true, - row, col); - rewriter.replaceOp(op, state.cstI64(rewriter, op.getLoc(), *bank)); - return success(); - } - -private: - PhysicalBankState &state; -}; - -class BankReleasePattern : public OpRewritePattern { -public: - BankReleasePattern(MLIRContext *context, PhysicalBankState &state) - : OpRewritePattern(context), state(state) {} - - LogicalResult matchAndRewrite(BankReleaseOp op, - PatternRewriter &rewriter) const override { - auto bank = state.getConstI64(op.getBank()); - if (!bank) { - return op.emitError( - "assign-physical-banks: release bank id is not constant"); - } - if (failed(state.release(op, *bank))) - return failure(); - - state.createMset(rewriter, op.getLoc(), static_cast(*bank), false, - 0, 0); - rewriter.eraseOp(op); - return success(); - } - -private: - PhysicalBankState &state; -}; - -class BankTransferPattern : public OpRewritePattern { -public: - BankTransferPattern(MLIRContext *context, PhysicalBankState &state) - : OpRewritePattern(context), state(state) {} - - LogicalResult matchAndRewrite(BankTransferOp op, - PatternRewriter &rewriter) const override { - auto source = state.getConstI64(op.getSource()); - auto target = state.getConstI64(op.getTarget()); - if (!source || !target) - return op.emitError("bank transfer requires constant virtual IDs"); - auto assigned = op.getSource().getDefiningOp(); - if (!assigned || assigned->getBlock() != op->getBlock()) - return op.emitError( - "bank transfer cannot consume a control-flow bank alias"); - for (Operation *user : op.getSource().getUsers()) { - if (user == op.getOperation()) - continue; - while (user && user->getBlock() != op->getBlock()) - user = user->getParentOp(); - if (!user || !user->isBeforeInBlock(op)) - return op.emitError( - "source bank handle is used after transfer or outside its block"); - } - if (failed(state.transfer(op, *source, *target))) - return failure(); - rewriter.create(op.getLoc(), op.getSource(), - op.getTarget()); - rewriter.replaceOp(op, state.cstI64(rewriter, op.getLoc(), *target)); - return success(); - } - -private: - PhysicalBankState &state; -}; - -class BankKernelPattern : public OpRewritePattern { -public: - BankKernelPattern(MLIRContext *context, PhysicalBankState &state) - : OpRewritePattern(context), state(state) {} - LogicalResult matchAndRewrite(BankKernelOp op, - PatternRewriter &rewriter) const override { - if (failed( - state.verifyKernelHandles(op, op.getReadBank(), op.getWriteBank()))) - return failure(); - auto module = op->getParentOfType(); - StringRef name = op.getCallee(); - SmallVector types(op.getOperandTypes()); - auto signature = rewriter.getFunctionType(types, TypeRange{}); - auto callee = module.lookupSymbol(name); - if (callee && callee.getFunctionType() != signature) - return op.emitError( - "bank kernel callee type does not match its operands"); - if (!callee) { - OpBuilder::InsertionGuard guard(rewriter); - rewriter.setInsertionPointToStart(module.getBody()); - auto function = - rewriter.create(op.getLoc(), name, signature); - function.setPrivate(); - function->setAttr("llvm.emit_c_interface", rewriter.getUnitAttr()); - } - rewriter.create(op.getLoc(), name, TypeRange{}, - op.getOperands()); - rewriter.replaceOp(op, ValueRange{op.getReadBank(), op.getWriteBank()}); - return success(); - } - -private: - PhysicalBankState &state; -}; - -class BankMvinPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankMvinOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getInput(), op.getBank(), - op.getDepth(), op.getStride(), op.getGroupAttr()); - rewriter.replaceOp(op, op.getBank()); - return success(); - } -}; - -class BankMvin2dPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankMvin2dOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getInput(), op.getBank(), - op.getHeight(), op.getPixelBytes(), - op.getSourceWidth(), op.getDstBase(), - op.getWidth(), op.getValidBytes()); - rewriter.replaceOp(op, op.getBank()); - return success(); - } -}; - -class BankMvoutPattern : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(BankMvoutOp op, - PatternRewriter &rewriter) const override { - rewriter.create(op.getLoc(), op.getOutput(), op.getBank(), - op.getDepth(), op.getStride(), op.getGroupAttr()); - rewriter.replaceOp(op, op.getBank()); - return success(); - } -}; - -} // namespace - -namespace mlir::buddy { - -LogicalResult verifyNoBankSSAOps(Operation *root) { - Operation *badOp = nullptr; - root->walk([&](Operation *op) { - if (op->getName().getStringRef().starts_with("buckyball.bank_")) { - badOp = op; - return WalkResult::interrupt(); - } - return WalkResult::advance(); - }); - if (!badOp) - return success(); - return badOp->emitError("assign-physical-banks: unsupported bank op"); -} - -void addBaseAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state) { - patterns.add(patterns.getContext(), state); - patterns.add( - patterns.getContext()); -} - -} // namespace mlir::buddy diff --git a/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyball.h b/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyball.h deleted file mode 100644 index 5a89310d2..000000000 --- a/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyball.h +++ /dev/null @@ -1,70 +0,0 @@ -//===- LowerBuckyball.h - Buckyball lowering hooks -------------*- C++ -*-===// - -#ifndef BUDDY_CONVERSION_LOWER_BUCKYBALL_LOWER_BUCKYBALL_H -#define BUDDY_CONVERSION_LOWER_BUCKYBALL_LOWER_BUCKYBALL_H - -#include "mlir/IR/Builders.h" -#include "mlir/IR/PatternMatch.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVector.h" - -#include -#include - -namespace mlir { -class Operation; - -namespace buddy { - -struct BankSlot { - int64_t base = -1; - int64_t row = 1; - int64_t col = 1; - llvm::SmallVector physicalGroups; -}; - -class PhysicalBankState { -public: - PhysicalBankState(int64_t bankNum, int64_t privateBankMax); - - int64_t getBankNum() const { return bankNum; } - int64_t getPrivateBankMax() const { return privateBankMax; } - int64_t getUsedCount() const; - bool empty() const { return vm.empty(); } - - std::optional getConstI64(Value value) const; - std::optional getSlot(Value value) const; - std::optional tryAlloc(int64_t row, int64_t col); - LogicalResult verifyKernelHandles(Operation *op, Value read, - Value write) const; - LogicalResult transfer(Operation *op, int64_t source, int64_t target); - LogicalResult release(Operation *op, int64_t bank); - - Value cstI64(OpBuilder &builder, Location loc, uint64_t value) const; - void createMset(OpBuilder &builder, Location loc, uint64_t bankId, bool alloc, - uint64_t row, uint64_t col) const; - -private: - void freeAlloc(const BankSlot &slot); - - int64_t bankNum = 0; - int64_t privateBankMax = 0; - llvm::DenseMap vm; - llvm::SmallVector used; -}; - -LogicalResult verifyNoBankSSAOps(Operation *root); -void addBaseAssignPhysicalBankPatterns(RewritePatternSet &patterns, - PhysicalBankState &state); -// Lowers the operations implementing BankSSALowering; for design passes that -// lower some operations themselves before the rest. -void populateBankSSALoweringPatterns(RewritePatternSet &patterns); -void registerAssignPhysicalBanksPass(); -// The single entry the compiler driver calls to register every pass. -void registerBuckyballPasses(); -void registerLowerBuckyballToBankSSAPass(); - -} // namespace buddy -} // namespace mlir - -#endif // BUDDY_CONVERSION_LOWER_BUCKYBALL_LOWER_BUCKYBALL_H diff --git a/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp b/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp deleted file mode 100644 index 02f36983c..000000000 --- a/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballPass.cpp +++ /dev/null @@ -1,109 +0,0 @@ -#include "Conversion/BallPasses.h" -#include "Conversion/LowerBuckyball/LowerBuckyball.h" -#include "Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h" -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" - -#include "mlir/Pass/PassManager.h" -#include "mlir/Pass/PassRegistry.h" - -using namespace mlir; - -namespace { -// Lowers the framework operations: bank configuration, data movement and raw -// instructions. Ball operations are lowered by their own passes. -class LowerFrameworkPass - : public mlir::buddy::LowerBallPass { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerFrameworkPass) - - StringRef getArgument() const final { return "lower-buckyball-framework"; } - StringRef getDescription() const final { - return "Lower Buckyball framework operations to LLVM intrinsics."; - } - - void populate(LLVMTypeConverter &converter, RewritePatternSet &patterns, - LLVMConversionTarget &target, bool, int64_t) const { - ::buddy::buckyball::legalize::populateBaseLegalizeForLLVMExportPatterns( - converter, patterns, /*includeFuncOperandForwarding=*/false); - ::buddy::buckyball::legalize::configureBaseLegalizeForExportTarget(target); - } -}; - -// Turns the Buckyball intrinsic operations into calls of the RISC-V LLVM -// intrinsics, so any LLVM IR translation handles the result. -class LowerIntrinsicsPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerIntrinsicsPass) - - StringRef getArgument() const final { return "lower-buckyball-intrinsics"; } - StringRef getDescription() const final { - return "Lower buckyball.intr.* to llvm.call_intrinsic."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert(); - } - - void runOnOperation() override { - SmallVector intrinsics; - getOperation().walk([&](Operation *op) { - if (op->getName().getStringRef().starts_with("buckyball.intr.")) - intrinsics.push_back(op); - }); - OpBuilder builder(&getContext()); - for (Operation *op : intrinsics) { - StringRef name = op->getName().getStringRef().drop_front(15); - SmallVector operands(op->getOperands()); - builder.setInsertionPoint(op); - std::string intrinsic = ("llvm.riscv.bb." + name).str(); - // The custom instruction carries its funct7 as an immediate argument. - if (name == "custom") { - intrinsic = "llvm.riscv.buckyball.custom"; - operands.push_back(builder.create( - op->getLoc(), builder.getI32Type(), - cast(op->getAttr("funct7")))); - } - builder.create( - op->getLoc(), builder.getStringAttr(intrinsic), operands); - op->erase(); - } - } -}; - -struct LowerBuckyballOptions - : public PassPipelineOptions { - Option stable{*this, "stable", - llvm::cl::desc("Use stable LLVM Buckyball intrinsics."), - llvm::cl::init(false)}; -}; - -std::vector(bool)>> &ballLowerings() { - static std::vector(bool)>> lowerings; - return lowerings; -} -} // namespace - -void mlir::buddy::addBallLowering( - std::function(bool stable)> create) { - ballLowerings().push_back(std::move(create)); -} - -void mlir::buddy::registerBuckyballPasses() { - registerLowerLinalgToBuckyballPass(); - registerLowerBuckyballToBankSSAPass(); - registerAssignPhysicalBanksPass(); - PassRegistration(); - PassRegistration(); - PassPipelineRegistration( - "lower-buckyball", - "Lower Buckyball framework and Ball operations to LLVM intrinsics.", - [](OpPassManager &pm, const LowerBuckyballOptions &options) { - auto framework = std::make_unique(); - framework->stable = options.stable; - pm.addPass(std::move(framework)); - for (auto &create : ballLowerings()) - pm.addPass(create(options.stable)); - pm.addPass(std::make_unique()); - }); -} diff --git a/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp b/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp deleted file mode 100644 index f9f8e5ead..000000000 --- a/waste/stack/compiler/src/Conversion/LowerBuckyball/LowerBuckyballToBankSSAPass.cpp +++ /dev/null @@ -1,80 +0,0 @@ -//===- LowerBuckyballToBankSSAPass.cpp - Buckyball bank-SSA lowering -----===// -// -// Lowers every operation implementing BankSSALowering to bank-SSA operations. -// -//===----------------------------------------------------------------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Math/IR/Math.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/Diagnostics.h" -#include "mlir/IR/PatternMatch.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Trace/TraceDialect.h" - -using namespace mlir; - -namespace { -class BankSSALoweringPattern - : public OpInterfaceRewritePattern<::buddy::buckyball::BankSSALowering> { -public: - using OpInterfaceRewritePattern::OpInterfaceRewritePattern; - - LogicalResult matchAndRewrite(::buddy::buckyball::BankSSALowering op, - PatternRewriter &rewriter) const override { - return op.lowerToBankSSA(rewriter); - } -}; - -class LowerBuckyballToBankSSAPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerBuckyballToBankSSAPass) - - StringRef getArgument() const final { return "lower-buckyball-to-bank-ssa"; } - StringRef getDescription() const final { - return "Lower Buckyball Buckyball ops to explicit bank-SSA ops."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert<::buddy::trace::BuddyTraceDialect, arith::ArithDialect, - linalg::LinalgDialect, math::MathDialect, - memref::MemRefDialect, LLVM::LLVMDialect, scf::SCFDialect, - ::buddy::buckyball::BuckyballDialect>(); - } - - void runOnOperation() override { - RewritePatternSet patterns(&getContext()); - mlir::buddy::populateBankSSALoweringPatterns(patterns); - bool emittedError = false; - ScopedDiagnosticHandler diagnostics( - &getContext(), [&](Diagnostic &message) { - emittedError |= message.getSeverity() == DiagnosticSeverity::Error; - return failure(); - }); - if (failed(applyPatternsGreedily(getOperation(), std::move(patterns))) || - emittedError) - signalPassFailure(); - } -}; - -} // namespace - -void mlir::buddy::populateBankSSALoweringPatterns(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -void mlir::buddy::registerLowerBuckyballToBankSSAPass() { - PassRegistration(); -} diff --git a/waste/stack/compiler/src/Conversion/LowerBuckyball/PhysicalBankState.cpp b/waste/stack/compiler/src/Conversion/LowerBuckyball/PhysicalBankState.cpp deleted file mode 100644 index f73d3326d..000000000 --- a/waste/stack/compiler/src/Conversion/LowerBuckyball/PhysicalBankState.cpp +++ /dev/null @@ -1,238 +0,0 @@ -//===- PhysicalBankState.cpp - Physical bank allocation state -------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "Buckyball/BuckyballOps.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/Diagnostics.h" -#include "llvm/ADT/STLExtras.h" - -using namespace mlir; -using namespace mlir::buddy; -using namespace ::buddy::buckyball; - -PhysicalBankState::PhysicalBankState(int64_t bankNum, int64_t privateBankMax) - : bankNum(bankNum), privateBankMax(privateBankMax), used(bankNum, 0) {} - -int64_t PhysicalBankState::getUsedCount() const { - return llvm::count(used, static_cast(1)); -} - -std::optional PhysicalBankState::getConstI64(Value value) const { - while (true) { - if (auto cst = value.getDefiningOp()) { - auto attr = dyn_cast(cst.getValue()); - if (!attr) - return std::nullopt; - return attr.getInt(); - } - if (auto op = value.getDefiningOp()) { - value = op.getBank(); - continue; - } - if (auto op = value.getDefiningOp()) { - value = op.getBank(); - continue; - } - if (auto op = value.getDefiningOp()) { - value = op.getBank(); - continue; - } - if (auto op = value.getDefiningOp()) { - value = op.getTarget(); - continue; - } - if (auto op = value.getDefiningOp()) { - value = cast(value).getResultNumber() == 0 ? op.getReadBank() - : op.getWriteBank(); - continue; - } - // Optional Balls: use op names so Cores can omit generated C++ types. - if (Operation *op = value.getDefiningOp()) { - StringRef name = op->getName().getStringRef(); - if (name == "buckyball.bank_transpose" || - name == "buckyball.bank_quant_f32_to_i8") { - value = op->getOperand(1); - continue; - } - if (name == "buckyball.bank_quant_i32_to_i8") { - value = op->getOperand(2); - continue; - } - if (name == "buckyball.bank_int32_to_fp32") { - value = op->getOperand(2); - continue; - } - if (name == "buckyball.bank_smatmul_bias") { - value = op->getOperand(0); - continue; - } - if (name == "buckyball.bank_im2col") { - value = op->getOperand(1); - continue; - } - if (name == "buckyball.bank_lut") { - value = op->getOperand(2); - continue; - } - if (name == "buckyball.bank_maxpool") { - value = op->getOperand(1); - continue; - } - if (name == "buckyball.bank_int8add") { - value = op->getOperand(2); - continue; - } - if (name == "buckyball.bank_int8mul") { - value = op->getOperand(2); - continue; - } - if (name == "buckyball.bank_smatmul" || name == "buckyball.bank_mxfp8" || - name == "buckyball.bank_vecmat16") { - value = op->getOperand(2); - continue; - } - if (name == "buckyball.bank_gemmini_preload") { - value = op->getOperand(1); - continue; - } - if (name == "buckyball.bank_gemmini_compute_preloaded" || - name == "buckyball.bank_gemmini_compute_accumulated") { - value = op->getOperand(2); - continue; - } - } - if (auto forOp = value.getDefiningOp()) { - unsigned resultNumber = cast(value).getResultNumber(); - value = cast(forOp.getBody()->getTerminator()) - .getResults()[resultNumber]; - continue; - } - if (auto ifOp = value.getDefiningOp()) { - unsigned resultNumber = cast(value).getResultNumber(); - auto thenYield = cast(ifOp.getThenRegion().front().back()); - auto elseYield = cast(ifOp.getElseRegion().front().back()); - auto thenBank = getConstI64(thenYield.getResults()[resultNumber]); - auto elseBank = getConstI64(elseYield.getResults()[resultNumber]); - if (!thenBank || !elseBank || *thenBank != *elseBank) - return std::nullopt; - return thenBank; - } - if (auto argument = dyn_cast(value)) { - auto forOp = dyn_cast(argument.getOwner()->getParentOp()); - if (!forOp || argument.getArgNumber() == 0) - return std::nullopt; - value = forOp.getInitArgs()[argument.getArgNumber() - 1]; - continue; - } - return std::nullopt; - } -} - -std::optional PhysicalBankState::getSlot(Value value) const { - auto bank = getConstI64(value); - if (!bank) - return std::nullopt; - auto slot = vm.find(*bank); - if (slot == vm.end()) - return std::nullopt; - for (auto [index, group] : llvm::enumerate(slot->second.physicalGroups)) - if (group != slot->second.base + static_cast(index)) { - emitError(value.getLoc(), - "bank operation requires contiguous physical groups"); - return std::nullopt; - } - return slot->second; -} - -std::optional PhysicalBankState::tryAlloc(int64_t row, int64_t col) { - if (row > bankNum / col || row * col > bankNum - getUsedCount()) - return std::nullopt; - int64_t bank = 0; - while (bank <= privateBankMax && vm.contains(bank)) - ++bank; - if (bank > privateBankMax) - return std::nullopt; - BankSlot slot; - slot.row = row; - slot.col = col; - for (int64_t group = 0; - group < bankNum && - static_cast(slot.physicalGroups.size()) < row * col; - ++group) - if (!used[group]) { - slot.physicalGroups.push_back(group); - used[group] = 1; - } - slot.base = slot.physicalGroups.front(); - vm[bank] = std::move(slot); - return bank; -} - -LogicalResult PhysicalBankState::verifyKernelHandles(Operation *op, Value read, - Value write) const { - auto r = getConstI64(read), w = getConstI64(write); - if (!r || !w || *r == *w || !vm.contains(*r) || !vm.contains(*w)) - return op->emitError( - "bank kernel requires distinct live read/write handles"); - return success(); -} - -LogicalResult PhysicalBankState::transfer(Operation *op, int64_t source, - int64_t target) { - if (source == target) - return op->emitError("bank transfer source and target must differ"); - if (target < 0 || target > privateBankMax) - return op->emitError( - "bank transfer target exceeds the private virtual ID range"); - auto sourceSlot = vm.find(source); - if (sourceSlot == vm.end()) - return op->emitError("transfer consumes an unknown virtual bank handle"); - BankSlot moved = sourceSlot->second; - vm.erase(sourceSlot); - auto targetSlot = vm.find(target); - if (targetSlot == vm.end()) { - vm[target] = std::move(moved); - } else { - auto &slot = targetSlot->second; - slot.physicalGroups.append(moved.physicalGroups); - slot.row = 1; - slot.col = slot.physicalGroups.size(); - } - return success(); -} - -LogicalResult PhysicalBankState::release(Operation *op, int64_t bank) { - auto it = vm.find(bank); - if (it == vm.end()) { - InFlightDiagnostic diagnostic = - op->emitError("release of unknown virtual bank handle"); - diagnostic << " (bank=" << bank << ", live=" << vm.size() - << ", used=" << getUsedCount() << "/" << bankNum << ")"; - return failure(); - } - freeAlloc(it->second); - vm.erase(it); - return success(); -} - -Value PhysicalBankState::cstI64(OpBuilder &builder, Location loc, - uint64_t value) const { - return builder.create(loc, builder.getI64Type(), - builder.getI64IntegerAttr(value)); -} - -void PhysicalBankState::createMset(OpBuilder &builder, Location loc, - uint64_t bankId, bool alloc, uint64_t row, - uint64_t col) const { - auto op = builder.create(loc, cstI64(builder, loc, bankId)); - op->setAttr("alloc", builder.getBoolAttr(alloc)); - op->setAttr("row", builder.getI64IntegerAttr(row)); - op->setAttr("col", builder.getI64IntegerAttr(col)); -} - -void PhysicalBankState::freeAlloc(const BankSlot &slot) { - for (int64_t group : slot.physicalGroups) - used[group] = 0; -} diff --git a/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/FormMegaKernelRegions.cpp b/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/FormMegaKernelRegions.cpp deleted file mode 100644 index 98086296f..000000000 --- a/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/FormMegaKernelRegions.cpp +++ /dev/null @@ -1,161 +0,0 @@ -#include "Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "llvm/ADT/DenseMap.h" - -using namespace mlir; - -void mlir::buddy::formMegaKernelRegions(ArrayRef stageList) { - SmallVector stages(stageList); - DenseMap producers; - DenseMap positions; - for (auto [index, op] : llvm::enumerate(stages)) { - producers[op.getOutputs()[0]] = index; - positions[op.getOperation()] = index; - } - SmallVector next(stages.size(), -1), previous(stages.size(), -1); - for (auto [index, op] : llvm::enumerate(stages)) { - Value output = op.getOutputs()[0]; - if (!cast(output.getType()).getElementType().isInteger(8)) - continue; - SmallVector consumers; - bool external = false; - for (Operation *user : output.getUsers()) { - if (user == op || isa(user)) - continue; - auto found = positions.find(user); - if (found == positions.end()) { - external = true; - break; - } - if (!llvm::is_contained(consumers, found->second)) - consumers.push_back(found->second); - } - if (external || consumers.size() != 1) - continue; - unsigned target = consumers.front(); - unsigned parents = 0; - for (Value input : stages[target].getInputs()) - parents += producers.contains(input); - if (parents != 1 || op->getBlock() != stages[target]->getBlock() || - op->hasAttr("buckyball.quantized_matmul") != - stages[target]->hasAttr("buckyball.quantized_matmul")) - continue; - next[index] = target; - previous[target] = index; - } - unsigned group = 0; - for (unsigned root = 0; root < stages.size(); ++root) { - if (previous[root] >= 0) - continue; - SmallVector chain; - for (int64_t index = root; index >= 0; index = next[index]) - chain.push_back(index); - auto context = stages[root].getContext(); - Builder builder(context); - auto id = builder.getStringAttr("core." + std::to_string(group++)); - for (auto [index, position] : llvm::enumerate(chain)) { - auto op = stages[position]; - if (op->hasAttr("core.packed_weight")) { - Value packed = op.getInputs()[1]; - for (Operation *user : llvm::make_early_inc_range(packed.getUsers())) - if (isa(user)) - user->moveAfter(stages[chain.back()]); - op->removeAttr("core.packed_weight"); - } - - SmallVector remove; - SmallVector add; - for (NamedAttribute attr : op->getAttrs()) { - StringRef name = attr.getName().strref(); - if (name.consume_front("buckyball.quantized_")) { - remove.push_back(attr.getName()); - add.emplace_back(builder.getStringAttr("buckyball.mega_" + name), - attr.getValue()); - } - } - for (StringAttr name : remove) - op->removeAttr(name); - for (NamedAttribute attr : add) - op->setAttr(attr.getName(), attr.getValue()); - op->setAttr("buckyball.mega_kernel", builder.getBoolAttr(true)); - op->setAttr("mega_kernel_id", id); - op->setAttr("mega_kernel_stage", builder.getI64IntegerAttr(index)); - op->setAttr("mega_kernel_size", builder.getI64IntegerAttr(chain.size())); - } - } -} - -void mlir::buddy::prepareQuantizedWeights( - ArrayRef stageList) { - SmallVector stages(stageList); - for (auto op : stages) { - bool matrix = op->hasAttr("buckyball.quantized_matmul"); - bool conv = op->hasAttr("buckyball.quantized_conv2d"); - bool depthwise = op->hasAttr("buckyball.quantized_conv2d_depthwise"); - if (!matrix && !conv && !depthwise) - continue; - Value weight = op.getInputs()[1]; - auto type = cast(weight.getType()); - auto shape = type.getShape(); - // Pebble's large-convolution lowering reads the logical weights itself. - if (conv && op->getAttrOfType("kernel").getInt() > 7) - continue; - SmallVector packedShape; - if (matrix) - packedShape = {shape[1], shape[0]}; - else if (depthwise) - packedShape = {shape[2], shape[3], shape[0], 1}; - else - packedShape = {(shape[0] + 15) / 16, shape[1], - ((shape[2] * shape[3] + 15) / 16) * 16, 16}; - OpBuilder builder(op); - Location loc = op.getLoc(); - Value packed = memref::AllocOp::create( - builder, loc, MemRefType::get(packedShape, type.getElementType())); - Value zeroByte = arith::ConstantIntOp::create(builder, loc, 0, 8); - linalg::FillOp::create(builder, loc, zeroByte, packed); - SmallVector indices; - for (int64_t extent : shape) { - Value zero = arith::ConstantIndexOp::create(builder, loc, 0); - Value end = arith::ConstantIndexOp::create(builder, loc, extent); - Value one = arith::ConstantIndexOp::create(builder, loc, 1); - auto loop = scf::ForOp::create(builder, loc, zero, end, one); - builder.setInsertionPointToStart(loop.getBody()); - indices.push_back(loop.getInductionVar()); - } - Value value = memref::LoadOp::create(builder, loc, weight, indices); - SmallVector destination; - if (matrix) - destination = {indices[1], indices[0]}; - else if (depthwise) - destination = {indices[2], indices[3], indices[0], indices[1]}; - else { - Value lanes = arith::ConstantIndexOp::create(builder, loc, 16); - Value width = arith::ConstantIndexOp::create(builder, loc, shape[3]); - destination = {arith::DivUIOp::create(builder, loc, indices[0], lanes), - indices[1], - arith::AddIOp::create( - builder, loc, - arith::MulIOp::create(builder, loc, indices[2], width), - indices[3]), - arith::RemUIOp::create(builder, loc, indices[0], lanes)}; - } - memref::StoreOp::create(builder, loc, value, packed, destination); - op.getInputsMutable()[1].set(packed); - if (matrix) { - auto maps = op.getIndexingMapsArray(); - maps[1] = AffineMap::get(3, 0, - {getAffineDimExpr(2, op.getContext()), - getAffineDimExpr(1, op.getContext())}, - op.getContext()); - op.setIndexingMapsAttr(builder.getAffineMapArrayAttr(maps)); - } - builder.setInsertionPointAfter(op); - memref::DeallocOp::create(builder, loc, packed); - op->setAttr("core.packed_weight", UnitAttr::get(op.getContext())); - } -} diff --git a/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h b/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h deleted file mode 100644 index b1aedeb00..000000000 --- a/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h +++ /dev/null @@ -1,45 +0,0 @@ -//===- LowerLinalgToBuckyball.h - Linalg to Buckyball conversion ----------===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// -// -// The framework collects the marked MegaKernel regions. Each Ball -// claims its own Linalg operations in its convert-linalg-to--ball pass, -// and a pipeline chooses which Balls claim; the rest stays in Linalg for the -// CPU. -// -//===----------------------------------------------------------------------===// - -#ifndef BUDDY_CONVERSION_LOWER_LINALG_TO_BUCKYBALL_H -#define BUDDY_CONVERSION_LOWER_LINALG_TO_BUCKYBALL_H - -#include "mlir/IR/PatternMatch.h" - -namespace mlir { -namespace linalg { -class GenericOp; -} -namespace buddy { - -void prepareQuantizedWeights(ArrayRef stages); -void formMegaKernelRegions(ArrayRef stages); - -// Collect the regions selected by the design midend. -void populateMegaKernelLinalgLoweringPatterns(RewritePatternSet &patterns); -void registerLowerLinalgToBuckyballPass(); - -} // namespace buddy -} // namespace mlir - -#endif // BUDDY_CONVERSION_LOWER_LINALG_TO_BUCKYBALL_H diff --git a/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyballPass.cpp b/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyballPass.cpp deleted file mode 100644 index e0e1f6bd4..000000000 --- a/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyballPass.cpp +++ /dev/null @@ -1,124 +0,0 @@ -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/Pass/Pass.h" -#include "mlir/Transforms/DialectConversion.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h" - -using namespace mlir; - -namespace { -// A transpose that repeats an earlier one on the same input reuses its result. -class CancelDuplicateTransposePattern - : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(linalg::TransposeOp op, - PatternRewriter &rewriter) const override { - Value in = op.getInput(); - for (Operation *user : in.getUsers()) { - auto prev = dyn_cast(user); - if (!prev || prev == op || prev.getPermutation() != op.getPermutation()) - continue; - auto shape = cast(op.getInit().getType()).getShape(); - if (cast(in.getType()).getShape() != shape || - cast(prev.getInit().getType()).getShape() != shape) - continue; - if (prev.getInit() != op.getInit()) - rewriter.create(op.getLoc(), prev.getInit(), - op.getInit()); - rewriter.eraseOp(op); - return success(); - } - return failure(); - } -}; - -// Two transposes whose permutations compose to identity become one copy. -class CancelTransposePairPattern - : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(linalg::TransposeOp second, - PatternRewriter &rewriter) const override { - auto first = second.getInput().getDefiningOp(); - if (!first || first.getInit() != second.getInput() || - !second.getInput().hasOneUse()) - return failure(); - ArrayRef p1 = first.getPermutation(); - ArrayRef p2 = second.getPermutation(); - if (p1.size() != p2.size()) - return failure(); - for (int64_t i = 0; i < (int64_t)p2.size(); ++i) - if (p1[p2[i]] != i) - return failure(); - if (cast(first.getInput().getType()).getShape() != - cast(second.getInit().getType()).getShape()) - return failure(); - if (first.getInput() != second.getInit()) - rewriter.create(second.getLoc(), first.getInput(), - second.getInit()); - rewriter.eraseOp(second); - if (first.getInit().use_empty()) - rewriter.eraseOp(first); - return success(); - } -}; - -class LowerLinalgToBuckyballPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerLinalgToBuckyballPass) - StringRef getArgument() const final { return "convert-linalg-to-buckyball"; } - StringRef getDescription() const final { - return "Collect the marked MegaKernel regions. Each Ball claims its own " - "Linalg operations in convert-linalg-to--ball."; - } - - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert<::buddy::buckyball::BuckyballDialect, func::FuncDialect, - memref::MemRefDialect, arith::ArithDialect, scf::SCFDialect, - linalg::LinalgDialect>(); - } - - void runOnOperation() override { - MLIRContext *context = &getContext(); - { - RewritePatternSet cancel(context); - cancel.add( - context); - if (failed(applyPatternsGreedily(getOperation(), std::move(cancel)))) - return signalPassFailure(); - } - ConversionTarget target(*context); - target.addLegalDialect<::buddy::buckyball::BuckyballDialect, - memref::MemRefDialect, arith::ArithDialect, - scf::SCFDialect, func::FuncDialect>(); - // Marked stages must lower; other Linalg operations stay unknown. - target.addDynamicallyLegalOp( - [](linalg::GenericOp op) -> std::optional { - if (op->hasAttr("buckyball.mega_kernel")) - return false; - return std::nullopt; - }); - - RewritePatternSet patterns(context); - mlir::buddy::populateMegaKernelLinalgLoweringPatterns(patterns); - if (failed(applyPartialConversion(getOperation(), target, - std::move(patterns)))) - signalPassFailure(); - } -}; -} // namespace - -void mlir::buddy::registerLowerLinalgToBuckyballPass() { - PassRegistration(); -} diff --git a/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/MegaKernelPatterns.cpp b/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/MegaKernelPatterns.cpp deleted file mode 100644 index e7bc1558f..000000000 --- a/waste/stack/compiler/src/Conversion/LowerLinalgToBuckyball/MegaKernelPatterns.cpp +++ /dev/null @@ -1,346 +0,0 @@ -//===- MegaKernelPatterns.cpp - marked MegaKernel regions ----------------===// -// -// Collects the generics the frontend marked as one cross-layer resident -// quantized MegaKernel into buckyball.mega_kernel. -// -//===----------------------------------------------------------------------===// - -#include - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/IR/PatternMatch.h" - -#include "Buckyball/BuckyballOps.h" -#include "Conversion/LowerLinalgToBuckyball/LowerLinalgToBuckyball.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -bool positive(FloatAttr attr) { - return attr && std::isfinite(attr.getValueAsDouble()) && - attr.getValueAsDouble() > 0.0; -} - -bool staticInt8(Value value, int64_t rank) { - auto type = dyn_cast(value.getType()); - return type && type.hasStaticShape() && type.getRank() == rank && - type.getElementType().isInteger(8); -} - -// The stages of one MegaKernel are consecutive generics sharing -// `mega_kernel_id`; each stage names its kind with a `buckyball.mega_*` marker. -class MegaKernelLowering : public OpRewritePattern { -public: - MegaKernelLowering(MLIRContext *context) - : OpRewritePattern(context, 2) {} - - LogicalResult matchAndRewrite(linalg::GenericOp op, - PatternRewriter &rewriter) const override { - auto first = op->getAttrOfType("mega_kernel_stage"); - if (!op->hasAttr("buckyball.mega_kernel") || !first || first.getInt() != 0) - return failure(); - auto size = op->getAttrOfType("mega_kernel_size"); - auto kernelId = op->getAttrOfType("mega_kernel_id"); - if (!size || size.getInt() <= 0 || !kernelId || kernelId.getValue().empty()) - return op.emitError("MegaKernel requires a positive size and an ID"); - - SmallVector stages(size.getInt()); - for (Operation &candidate : *op->getBlock()) { - auto generic = dyn_cast(candidate); - if (!generic || - generic->getAttrOfType("mega_kernel_id") != kernelId) - continue; - auto index = generic->getAttrOfType("mega_kernel_stage"); - auto stageSize = generic->getAttrOfType("mega_kernel_size"); - if (!generic->hasAttr("buckyball.mega_kernel") || !index || !stageSize || - stageSize.getInt() != size.getInt() || index.getInt() < 0 || - index.getInt() >= size.getInt()) - return generic.emitError("MegaKernel stage metadata is malformed"); - if (stages[index.getInt()]) - return generic.emitError("MegaKernel stage index is duplicated"); - stages[index.getInt()] = generic; - } - if (llvm::any_of(stages, [](linalg::GenericOp stage) { return !stage; })) - return op.emitError("MegaKernel stage sequence is incomplete"); - - Value kernelOutput = stages.back().getOutputs()[0]; - rewriter.setInsertionPointAfter(stages.back()); - auto kernel = rewriter.create( - op.getLoc(), stages.front().getInputs()[0], kernelOutput); - kernel.getBody().emplaceBlock(); - - OpBuilder::InsertionGuard guard(rewriter); - rewriter.setInsertionPointToStart(&kernel.getBody().front()); - for (auto [index, current] : llvm::enumerate(stages)) { - bool last = index + 1 == stages.size(); - bool normal = current->hasAttr("buckyball.mega_conv2d"); - bool depthwise = current->hasAttr("buckyball.mega_conv2d_depthwise"); - bool matmul = current->hasAttr("buckyball.mega_matmul"); - bool maxPool = current->hasAttr("buckyball.mega_max_pool2d"); - bool globalAvg = current->hasAttr("buckyball.mega_global_avg_pool"); - bool channelSlice = current->hasAttr("buckyball.mega_channel_slice"); - bool channelConcat = current->hasAttr("buckyball.mega_channel_concat"); - bool resizeNearest = current->hasAttr("buckyball.mega_resize_nearest"); - bool int8Mul = current->hasAttr("buckyball.mega_int8_mul"); - bool int8Add = current->hasAttr("buckyball.mega_int8_add"); - if (int(normal) + int(depthwise) + int(matmul) + int(globalAvg) + - int(maxPool) + int(channelSlice) + int(channelConcat) + - int(resizeNearest) + int(int8Mul) + int(int8Add) != - 1) - return current.emitError("MegaKernel stage kind is not unique"); - Location loc = current.getLoc(); - ValueRange inputs = current.getInputs(); - if (current.getOutputs().size() != 1) - return current.emitError("MegaKernel stage needs one output"); - Value output = current.getOutputs()[0]; - auto outputType = dyn_cast(output.getType()); - if (!outputType || !outputType.hasStaticShape() || - (outputType.getRank() != 4 && !matmul)) - return current.emitError("MegaKernel stage needs a static output"); - ArrayRef out = outputType.getShape(); - // Only the last stage writes the kernel output. - bool boundary = last ? output == kernelOutput : output != kernelOutput; - - if (channelSlice) { - auto offset = current->getAttrOfType("offset"); - if (inputs.size() != 1 || !staticInt8(inputs[0], 4) || - !staticInt8(output, 4) || !offset || offset.getInt() < 0 || - offset.getInt() % 16 != 0 || out[3] % 16 != 0) - return current.emitError("Mega channel slice contract is invalid"); - auto in = cast(inputs[0].getType()).getShape(); - if (in[0] != out[0] || in[1] != out[1] || in[2] != out[2] || - offset.getInt() + out[3] > in[3]) - return current.emitError("Mega channel slice contract is invalid"); - rewriter.create(loc, inputs[0], output, offset); - continue; - } - - if (channelConcat) { - auto segments = current->getAttrOfType("segments"); - if (inputs.empty() || !staticInt8(output, 4) || !segments || - segments.size() != inputs.size()) - return current.emitError("Mega channel concat has the wrong arity"); - int64_t channels = 0; - for (auto [input, segment] : llvm::zip(inputs, segments.asArrayRef())) { - if (!staticInt8(input, 4) || segment <= 0 || segment % 16 != 0) - return current.emitError("Mega channel concat contract is invalid"); - auto in = cast(input.getType()).getShape(); - if (in[0] != out[0] || in[1] != out[1] || in[2] != out[2] || - in[3] != segment) - return current.emitError("Mega channel concat contract is invalid"); - channels += segment; - } - if (out[3] != channels) - return current.emitError( - "Mega channel concat output channels are invalid"); - rewriter.create(loc, inputs, output, segments); - continue; - } - - if (resizeNearest) { - auto scaleH = current->getAttrOfType("scale_h"); - auto scaleW = current->getAttrOfType("scale_w"); - if (inputs.size() != 1 || !staticInt8(inputs[0], 4) || - !staticInt8(output, 4) || !scaleH || !scaleW || - scaleH.getInt() <= 0 || scaleW.getInt() <= 0) - return current.emitError("Mega resize-nearest contract is invalid"); - auto in = cast(inputs[0].getType()).getShape(); - if (in[0] != out[0] || in[3] != out[3] || - out[1] != in[1] * scaleH.getInt() || - out[2] != in[2] * scaleW.getInt()) - return current.emitError("Mega resize-nearest contract is invalid"); - rewriter.create(loc, inputs[0], output, scaleH, - scaleW); - continue; - } - - if (maxPool) { - auto kernelSize = current->getAttrOfType("kernel"); - auto stride = current->getAttrOfType("stride"); - auto padding = current->getAttrOfType("padding"); - auto finalOutput = current->getAttrOfType("final_output"); - if (inputs.size() != 1 || !staticInt8(inputs[0], 4) || - !staticInt8(output, 4) || !kernelSize || !stride || !padding || - !finalOutput || kernelSize.getInt() <= 0 || stride.getInt() <= 0 || - padding.getInt() < 0 || (finalOutput.getValue() && !last)) - return current.emitError("Mega MaxPool2D contract is invalid"); - auto in = cast(inputs[0].getType()).getShape(); - bool nchw = finalOutput.getValue(); - int64_t outH = nchw ? out[2] : out[1]; - int64_t outW = nchw ? out[3] : out[2]; - int64_t outC = nchw ? out[1] : out[3]; - int64_t window = 2 * padding.getInt() - kernelSize.getInt(); - if (out[0] != in[0] || outC != in[3] || - outH != (in[1] + window) / stride.getInt() + 1 || - outW != (in[2] + window) / stride.getInt() + 1) - return current.emitError("Mega MaxPool2D shape is invalid"); - rewriter.create(loc, inputs[0], output, kernelSize, - stride, padding, finalOutput); - continue; - } - - if (globalAvg) { - auto inputScale = current->getAttrOfType("input_scale"); - auto outputScale = current->getAttrOfType("output_scale"); - if (!boundary || inputs.size() != 1 || !staticInt8(inputs[0], 4) || - !staticInt8(output, 4) || !positive(inputScale) || - !positive(outputScale)) - return current.emitError("Mega global-average contract is invalid"); - auto in = cast(inputs[0].getType()).getShape(); - if (in[0] != out[0] || in[3] != out[3] || out[1] != 1 || out[2] != 1) - return current.emitError("Mega global-average contract is invalid"); - rewriter.create(loc, inputs[0], output, inputScale, - outputScale); - continue; - } - - if (int8Mul || int8Add) { - auto lhsScale = current->getAttrOfType("lhs_scale"); - auto rhsScale = current->getAttrOfType("rhs_scale"); - auto outputScale = current->getAttrOfType("output_scale"); - auto activation = current->getAttrOfType("activation"); - if (!boundary || inputs.size() != 2 || !staticInt8(inputs[0], 4) || - !staticInt8(inputs[1], 4) || !staticInt8(output, 4) || - !positive(lhsScale) || !positive(rhsScale) || - !positive(outputScale) || !activation || activation.getInt() < 0 || - activation.getInt() > 1) - return current.emitError("Mega INT8 elementwise contract is invalid"); - auto lhs = cast(inputs[0].getType()); - auto rhs = cast(inputs[1].getType()); - for (int64_t dimension = 0; dimension < 4; ++dimension) - if ((lhs.getShape()[dimension] != 1 && - lhs.getShape()[dimension] != out[dimension]) || - (rhs.getShape()[dimension] != 1 && - rhs.getShape()[dimension] != out[dimension])) - return current.emitError( - "Mega INT8 elementwise shapes do not broadcast"); - if (int8Add && lhs != rhs) - return current.emitError( - "Mega residual add requires equal input types"); - if (int8Mul) - rewriter.create(loc, inputs[0], inputs[1], output, - lhsScale, rhsScale, outputScale, - activation); - else - rewriter.create(loc, inputs[0], inputs[1], output, - lhsScale, rhsScale, outputScale, - activation); - continue; - } - - // Compute stages: input, weight, bias, scale and activation LUT. - auto activation = current->getAttrOfType("activation"); - auto finalOutput = current->getAttrOfType("final_output"); - auto outputScale = current->getAttrOfType("output_scale"); - if (!boundary || inputs.size() != 5 || !activation || - activation.getInt() < 0 || activation.getInt() > 2 || !finalOutput || - !positive(outputScale) || (finalOutput.getValue() && !last)) - return current.emitError("Mega compute stage contract is invalid"); - Value input = inputs[0], weight = inputs[1], bias = inputs[2], - scale = inputs[3], lut = inputs[4]; - auto inputType = cast(input.getType()); - auto weightType = cast(weight.getType()); - auto biasType = cast(bias.getType()); - auto scaleType = cast(scale.getType()); - auto lutType = cast(lut.getType()); - if (!inputType.hasStaticShape() || !weightType.hasStaticShape() || - !biasType.hasStaticShape() || !scaleType.hasStaticShape() || - !staticInt8(lut, 1)) - return current.emitError("Mega compute stage requires static shapes"); - // A 4096-entry LUT holds one table per lane of a 16-channel Conv whose - // only consumer is the depthwise stage after it. - int64_t lutSize = lutType.getShape()[0]; - if (lutSize == 4096) { - int64_t consumers = 0; - for (linalg::GenericOp candidate : stages) - consumers += llvm::is_contained(candidate.getInputs(), output); - if (!normal || index != 0 || activation.getInt() != 2 || out[3] != 16 || - stages.size() < 2 || - !stages[1]->hasAttr("buckyball.mega_conv2d_depthwise") || - stages[1].getInputs()[0] != output || consumers != 1) - return current.emitError( - "4096-entry lane LUT is only legal on stage 0 before one " - "16-channel depthwise Conv"); - } else if (lutSize != (activation.getInt() == 2 ? 256 : 1)) { - return current.emitError( - "MegaKernel activation LUT has the wrong shape"); - } - // Only FP32 marks the model-visible result; INT8 stays inside. - Type element = outputType.getElementType(); - if (last ? !element.isF32() && !element.isInteger(8) - : !element.isInteger(8)) - return current.emitError("MegaKernel stage output type is invalid"); - - if (matmul) { - int64_t m = inputType.getShape()[0], k = inputType.getShape()[1]; - int64_t n = weightType.getShape()[1]; - if (m <= 0 || k <= 0 || n <= 0 || weightType.getShape()[0] != k || - out[0] != m || out[1] != n || biasType.getShape()[0] != n || - scaleType.getShape()[0] != n || (last && !element.isF32())) - return current.emitError("MegaKernel MatMul shape mismatch"); - rewriter.create(loc, input, weight, bias, scale, lut, - output, activation, outputScale); - continue; - } - - auto kernelSize = current->getAttrOfType("kernel"); - auto stride = current->getAttrOfType("stride"); - auto padLow = current->getAttrOfType("pad_low"); - auto padHigh = current->getAttrOfType("pad_high"); - if (!kernelSize || !stride || !padLow || !padHigh) - return current.emitError( - "MegaKernel convolution attributes are missing"); - int64_t k = kernelSize.getInt(), s = stride.getInt(); - int64_t pads = padLow.getInt() + padHigh.getInt(); - auto in = inputType.getShape(); - auto filter = weightType.getShape(); - auto outputNhwc = current->getAttrOfType("output_nhwc"); - bool nchw = - last && element.isF32() && !(outputNhwc && outputNhwc.getValue()); - int64_t outC = nchw ? out[1] : out[3]; - int64_t outH = nchw ? out[2] : out[1]; - int64_t outW = nchw ? out[3] : out[2]; - int64_t channels = depthwise ? in[3] : outC; - bool filterValid = - depthwise ? filter == ArrayRef({k, k, in[3], 1}) - : filter == ArrayRef({(outC + 15) / 16, in[3], - (k * k + 15) / 16 * 16, 16}); - if (in[0] <= 0 || in[1] <= 0 || in[2] <= 0 || in[3] <= 0 || k <= 0 || - k > 7 || s <= 0 || padLow.getInt() < 0 || padHigh.getInt() < 0 || - !filterValid || out[0] != in[0] || outC != channels || - biasType.getShape()[0] != channels || - scaleType.getShape()[0] != channels || in[1] + pads < k || - in[2] + pads < k || outH != (in[1] + pads - k) / s + 1 || - outW != (in[2] + pads - k) / s + 1) - return current.emitError("MegaKernel convolution shape mismatch"); - Operation *lowered; - if (depthwise) - lowered = rewriter.create( - loc, input, weight, bias, scale, lut, output, kernelSize, stride, - padLow, padHigh, activation, outputScale); - else - lowered = rewriter.create( - loc, input, weight, bias, scale, lut, output, kernelSize, stride, - padLow, padHigh, activation, outputScale); - if (outputNhwc) - lowered->setAttr("output_nhwc", outputNhwc); - } - for (auto [source, target] : - llvm::zip_equal(stages, kernel.getBody().front())) - if (auto trace = source->getAttr("trace_node")) - target.setAttr("trace_node", trace); - rewriter.create(op.getLoc()); - for (int64_t index = stages.size(); index > 0; --index) - rewriter.eraseOp(stages[index - 1]); - return success(); - } -}; -} // namespace - -void mlir::buddy::populateMegaKernelLinalgLoweringPatterns( - RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} diff --git a/waste/stack/compiler/src/Dialect/Buckyball/Buckyball.td b/waste/stack/compiler/src/Dialect/Buckyball/Buckyball.td deleted file mode 100644 index 3aa7b2a81..000000000 --- a/waste/stack/compiler/src/Dialect/Buckyball/Buckyball.td +++ /dev/null @@ -1,446 +0,0 @@ -//===-- Buckyball.td - Buckyball dialect operations -----*- tablegen -*-===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#ifndef BUCKYBALL_OPS -#define BUCKYBALL_OPS - -include "mlir/Dialect/LLVMIR/LLVMOpBase.td" -include "mlir/IR/AttrTypeBase.td" -include "mlir/IR/PatternBase.td" -include "BuckyballInterfaces.td" - -//===----------------------------------------------------------------------===// -// Dialect and common operations -//===----------------------------------------------------------------------===// - -def Buckyball_Dialect : Dialect { - let name = "buckyball"; - let cppNamespace = "::buddy::buckyball"; - let summary = "Dialect for RISC-V Buckyball NPU extension"; - let description = [{ - Buckyball is an NPU Design Framework with composable Ball units. - Each Ball is a specialized compute unit that can be independently - developed and combined with others. - }]; -} - -class Buckyball_Op traits = []> : - Op {} - -def MvoverOp : Buckyball_Op<"mvover"> { - let summary = "Move contiguous rows between logical Thread-local Banks"; - let arguments = (ins I64:$sourceThread, I64:$sourceBank, I64:$sourceRow, - I64:$targetThread, I64:$targetBank, I64:$targetRow, - I64:$rows); - let assemblyFormat = "$sourceThread $sourceBank $sourceRow $targetThread " - "$targetBank $targetRow $rows prop-dict attr-dict"; -} - -def MvinOp : Buckyball_Op<"mvin"> { - let summary = "Load data from memory to scratchpad bank"; - let description = [{ - Loads data from DRAM to scratchpad bank. - Corresponds to bb_mvin instruction (funct7=33). - }]; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, - I64:$addr, I64:$depth, I64:$stride, - OptionalAttr:$group); - let assemblyFormat = "$input $addr $depth $stride prop-dict attr-dict `:` " - "type($input) type($addr) type($depth) type($stride)"; - let hasVerifier = 1; -} - -def Mvin2dOp : Buckyball_Op<"mvin_2d"> { - let summary = "Load an INT8 channel panel rectangle into a scratchpad bank"; - let arguments = (ins MemRefRankOf<[I8], [4]>:$input, I64:$addr, - I64:$height, I64:$pixelBytes, I64:$sourceWidth, - I64:$dstBase, I64:$width, I64:$validBytes); - let assemblyFormat = "$input $addr $height $pixelBytes $sourceWidth " - "$dstBase $width $validBytes prop-dict attr-dict `:` " - "type($input) type($addr) type($height) " - "type($pixelBytes) type($sourceWidth) " - "type($dstBase) type($width) type($validBytes)"; -} - -def MvinMmioOp : Buckyball_Op<"mvin_mmio"> { - let summary = "Load metadata from memory into MMIO SRAM"; - let arguments = (ins MemRefRankOf<[I8], [1, 2]>:$input, - I64:$mmioAddr, I64:$rows, I64:$columns); - let assemblyFormat = "$input $mmioAddr $rows $columns prop-dict attr-dict `:` " - "type($input) type($mmioAddr) type($rows) type($columns)"; -} - -def MvoutOp : Buckyball_Op<"mvout"> { - let summary = "Store data from scratchpad bank to memory"; - let description = [{ - Stores data from scratchpad bank to DRAM. - Corresponds to bb_mvout instruction (funct7=16). - }]; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$output, - I64:$addr, I64:$depth, I64:$stride, - OptionalAttr:$group); - let assemblyFormat = "$output $addr $depth $stride prop-dict attr-dict `:` " - "type($output) type($addr) type($depth) type($stride)"; - let hasVerifier = 1; -} - -def MsetOp : Buckyball_Op<"mset"> { - let summary = "Allocate or release scratchpad bank memory"; - let description = [{ - Allocates or releases scratchpad bank memory. - Corresponds to bb_mset instruction (funct7=32). - }]; - let arguments = (ins I64:$bankId, - DefaultValuedAttr:$alloc, - DefaultValuedAttr:$row, - DefaultValuedAttr:$col); - let assemblyFormat = "$bankId prop-dict attr-dict `:` type($bankId)"; -} - -def MsetTransferOp : Buckyball_Op<"mset_transfer"> { - let summary = "Transfer bank mappings to a target virtual bank"; - let arguments = (ins I64:$source, I64:$target); - let assemblyFormat = "$source $target prop-dict attr-dict `:` type($source) type($target)"; - let hasVerifier = 1; -} - -//===----------------------------------------------------------------------===// -// Bank SSA operations -//===----------------------------------------------------------------------===// - -def BankAllocOp : Buckyball_Op<"bank_alloc", [MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "Allocate a virtual on-chip bank resource"; - let description = [{ - Create a virtual bank handle in SSA form. - - row/col encode bank-array shape used by allocator. - - The result i64 is a virtual bank handle, not physical id. - }]; - let arguments = (ins - DefaultValuedAttr:$row, - DefaultValuedAttr:$col); - let results = (outs I64:$bank); - let assemblyFormat = "prop-dict attr-dict"; -} - -def BankReleaseOp : Buckyball_Op<"bank_release", [MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "Release a virtual bank handle"; - let arguments = (ins I64:$bank); - let assemblyFormat = "$bank prop-dict attr-dict `:` type($bank)"; -} - -def BankTransferOp : Buckyball_Op<"bank_transfer", [MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "Consume a bank mapping and append its groups to a target handle"; - let arguments = (ins I64:$source, I64:$target); - let results = (outs I64:$bank); - let assemblyFormat = "$source $target prop-dict attr-dict `:` type($source) type($target)"; -} - -def BankKernelOp : Buckyball_Op<"bank_kernel", [MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "Call a synchronous kernel with resident read/write bank handles"; - let arguments = (ins I64:$readBank, I64:$writeBank, - FlatSymbolRefAttr:$callee, Variadic:$kernelArgs); - let results = (outs I64:$readOut, I64:$writeOut); - let assemblyFormat = "$callee $readBank $writeBank `(` $kernelArgs `)` " - "prop-dict attr-dict `:` type($kernelArgs)"; -} - -def BankMvinOp : Buckyball_Op<"bank_mvin", [MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "Load into a virtual bank handle"; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$input, I64:$bank, I64:$depth, - I64:$stride, OptionalAttr:$group); - let results = (outs I64:$bankOut); - let assemblyFormat = - "$input $bank $depth $stride prop-dict attr-dict `:` type($input) type($bank) type($depth) type($stride)"; -} - -def BankMvin2dOp : Buckyball_Op<"bank_mvin_2d"> { - let summary = "Load a rectangular INT8 channel panel into a virtual bank"; - let arguments = (ins MemRefRankOf<[I8], [4]>:$input, I64:$bank, - I64:$height, I64:$pixelBytes, I64:$sourceWidth, - I64:$dstBase, I64:$width, I64:$validBytes); - let results = (outs I64:$bankOut); - let assemblyFormat = "$input $bank $height $pixelBytes $sourceWidth " - "$dstBase $width $validBytes prop-dict attr-dict `:` " - "type($input) type($bank) type($height) " - "type($pixelBytes) type($sourceWidth) " - "type($dstBase) type($width) type($validBytes)"; -} - -def BankMvoutOp : Buckyball_Op<"bank_mvout", [MemoryEffects<[MemRead, MemWrite]>]> { - let summary = "Store from a virtual bank handle"; - let arguments = (ins MemRefRankOf<[AnyType], [2]>:$output, I64:$bank, I64:$depth, - I64:$stride, OptionalAttr:$group); - let results = (outs I64:$bankOut); - let assemblyFormat = - "$output $bank $depth $stride prop-dict attr-dict `:` type($output) type($bank) type($depth) type($stride)"; -} - -//===----------------------------------------------------------------------===// -// Mega kernel operations -//===----------------------------------------------------------------------===// - -def MegaYieldOp : Buckyball_Op<"mega_yield", [Terminator]> { - let summary = "Terminate a Buckyball MegaKernel region"; - let assemblyFormat = "prop-dict attr-dict"; -} - -def MegaKernelOp : Buckyball_Op<"mega_kernel", [ - SingleBlockImplicitTerminator<"MegaYieldOp">]> { - let summary = "One cross-layer resident quantized kernel"; - let arguments = (ins AnyMemRef:$input, - AnyMemRef:$output); - let regions = (region SizedRegion<1>:$body); - let assemblyFormat = [{$input $output prop-dict attr-dict `:` - type($input) type($output) $body}]; -} - -def MegaMatmulOp : Buckyball_Op<"mega_matmul"> { - let summary = "Quantized MatMul stage inside a MegaKernel"; - let description = [{ - One INT8 MatMul stage. Bias is already converted offline to INT32. A - non-final result is requantized to INT8 and stays inside the MegaKernel. - }]; - let arguments = (ins MemRefRankOf<[I8], [2]>:$input, - MemRefRankOf<[I8], [2]>:$weight, - MemRefRankOf<[I32], [1]>:$bias, - MemRefRankOf<[F32], [1]>:$scale, - MemRefRankOf<[I8], [1]>:$lut, - MemRefRankOf<[I8, F32], [2]>:$output, - I64Attr:$activation, - F32Attr:$outputScale); - let assemblyFormat = [{$input $weight $bias $scale $lut $output prop-dict attr-dict `:` - type($input) type($weight) type($bias) type($scale) type($lut) type($output)}]; -} - -def MegaConv2dOp : Buckyball_Op<"mega_conv2d"> { - let summary = "Quantized convolution stage inside a MegaKernel"; - let description = [{ - One resident NHWC INT8 convolution stage. Bias is already converted - offline to INT32. A non-final result is requantized to INT8 and stays - inside the MegaKernel. Activation LUTs contain either one shared - 256-entry table or sixteen channel-major tables. - }]; - let arguments = (ins MemRefRankOf<[I8], [4]>:$input, - MemRefRankOf<[I8], [4]>:$weight, - MemRefRankOf<[I32], [1]>:$bias, - MemRefRankOf<[F32], [1]>:$scale, - MemRefRankOf<[I8], [1]>:$lut, - MemRefRankOf<[I8, F32], [4]>:$output, - I64Attr:$kernel, - I64Attr:$stride, - I64Attr:$padLow, - I64Attr:$padHigh, - I64Attr:$activation, - F32Attr:$outputScale); - let assemblyFormat = [{$input $weight $bias $scale $lut $output prop-dict attr-dict `:` - type($input) type($weight) type($bias) type($scale) type($lut) type($output)}]; -} - -def MegaConv2dDepthwiseOp : Buckyball_Op<"mega_conv2d_depthwise"> { - let summary = "Quantized depthwise stage inside a MegaKernel"; - let description = [{ - One complete resident NHWC INT8 depthwise convolution layer with channel - multiplier one. Bias is converted offline to INT32 and output is - requantized to INT8 before leaving the mega kernel. - }]; - let arguments = (ins MemRefRankOf<[I8], [4]>:$input, - MemRefRankOf<[I8], [4]>:$weight, - MemRefRankOf<[I32], [1]>:$bias, - MemRefRankOf<[F32], [1]>:$scale, - MemRefRankOf<[I8], [1]>:$lut, - MemRefRankOf<[I8, F32], [4]>:$output, - I64Attr:$kernel, - I64Attr:$stride, - I64Attr:$padLow, - I64Attr:$padHigh, - I64Attr:$activation, - F32Attr:$outputScale); - let assemblyFormat = [{$input $weight $bias $scale $lut $output prop-dict attr-dict `:` - type($input) type($weight) type($bias) type($scale) type($lut) type($output)}]; -} - -def MegaMaxPool2dOp : Buckyball_Op<"mega_max_pool2d"> { - let summary = "INT8 MaxPool2D stage inside a MegaKernel"; - let description = [{ - Max pooling preserves the INT8 quantization scale. An intermediate output - is NHWC; a final output is NCHW so a following reshape preserves PyTorch - flatten order without a transpose operation. - }]; - let arguments = (ins MemRefRankOf<[I8], [4]>:$input, - MemRefRankOf<[I8], [4]>:$output, - I64Attr:$kernel, - I64Attr:$stride, - I64Attr:$padding, - BoolAttr:$finalOutput); - let assemblyFormat = [{$input $output prop-dict attr-dict `:` - type($input) type($output)}]; -} - -def MegaGlobalAvgPoolOp : Buckyball_Op<"mega_global_avg_pool"> { - let summary = "Global-average an INT8 NHWC tensor inside a MegaKernel"; - let description = [{ - This is the explicit X-memory boundary used by an SE branch. The result is - requantized to INT8 before the following pointwise stage. - }]; - let arguments = (ins MemRefRankOf<[I8], [4]>:$input, - MemRefRankOf<[I8], [4]>:$output, - F32Attr:$inputScale, - F32Attr:$outputScale); - let assemblyFormat = [{$input $output prop-dict attr-dict `:` - type($input) type($output)}]; -} - -def MegaInt8MulOp : Buckyball_Op<"mega_int8_mul"> { - let summary = "Scaled INT8 broadcast multiply inside a MegaKernel"; - let arguments = (ins MemRefRankOf<[I8], [4]>:$lhs, - MemRefRankOf<[I8], [4]>:$rhs, - MemRefRankOf<[I8], [4]>:$output, - F32Attr:$lhsScale, - F32Attr:$rhsScale, - F32Attr:$outputScale, - I64Attr:$activation); - let assemblyFormat = [{$lhs $rhs $output prop-dict attr-dict `:` - type($lhs) type($rhs) type($output)}]; -} - -def MegaInt8AddOp : Buckyball_Op<"mega_int8_add"> { - let summary = "Scaled INT8 residual add inside a MegaKernel"; - let arguments = (ins MemRefRankOf<[I8], [4]>:$lhs, - MemRefRankOf<[I8], [4]>:$rhs, - MemRefRankOf<[I8], [4]>:$output, - F32Attr:$lhsScale, - F32Attr:$rhsScale, - F32Attr:$outputScale, - I64Attr:$activation); - let assemblyFormat = [{$lhs $rhs $output prop-dict attr-dict `:` - type($lhs) type($rhs) type($output)}]; -} - -def MegaChannelSliceOp : Buckyball_Op<"mega_channel_slice"> { - let summary = "INT8 NHWC channel slice inside a MegaKernel"; - let arguments = (ins MemRefRankOf<[I8], [4]>:$input, - MemRefRankOf<[I8], [4]>:$output, - I64Attr:$offset); - let assemblyFormat = [{$input $output prop-dict attr-dict `:` type($input) type($output)}]; -} - -def MegaChannelConcatOp : Buckyball_Op<"mega_channel_concat"> { - let summary = "INT8 NHWC channel concatenation inside a MegaKernel"; - let arguments = (ins Variadic>:$inputs, - MemRefRankOf<[I8], [4]>:$output, - DenseI64ArrayAttr:$segments); - let assemblyFormat = [{$inputs $output prop-dict attr-dict `:` type($inputs) type($output)}]; -} - -def MegaResizeNearestOp : Buckyball_Op<"mega_resize_nearest"> { - let summary = "INT8 NHWC nearest-neighbor resize inside a MegaKernel"; - let arguments = (ins MemRefRankOf<[I8], [4]>:$input, - MemRefRankOf<[I8], [4]>:$output, - I64Attr:$scaleH, - I64Attr:$scaleW); - let assemblyFormat = [{$input $output prop-dict attr-dict `:` type($input) type($output)}]; -} - -//===----------------------------------------------------------------------===// -// LLVM intrinsic operations -//===----------------------------------------------------------------------===// - -class Buckyball_IntrOpBase traits = []> : - LLVM_IntrOpBase overloadedResults=*/[], - /*list overloadedOperands=*/[], - /*list traits=*/traits, - /*int numResults=*/0>; - -class Buckyball_CustomIntrOpBase immArgPositions, - list immArgAttrNames, - list traits = []> : - LLVM_IntrOpBase overloadedResults=*/[], - /*list overloadedOperands=*/[], - /*list traits=*/traits, - /*int numResults=*/0, - /*bit requiresAccessGroup=*/0, - /*bit requiresAliasAnalysis=*/0, - /*bit requiresFastmath=*/0, - /*bit requiresArgAndResultAttrs=*/0, - /*bit requiresOpBundles=*/0, - /*list immArgPositions=*/immArgPositions, - /*list immArgAttrNames=*/immArgAttrNames>; - -def MsetIntrOp : Buckyball_IntrOpBase<"mset">, - Arguments<(ins LLVM_Type, LLVM_Type)>; - -def MvinIntrOp : Buckyball_IntrOpBase<"mvin">, - Arguments<(ins LLVM_Type, LLVM_Type)>; - -def MvinMmioIntrOp : Buckyball_IntrOpBase<"mvin_mmio">, - Arguments<(ins LLVM_Type, LLVM_Type)>; - -def MvoutIntrOp : Buckyball_IntrOpBase<"mvout">, - Arguments<(ins LLVM_Type, LLVM_Type)>; - - - - - -def CustomIntrOp - : Buckyball_CustomIntrOpBase<"custom", "riscv_buckyball_custom", - [2], ["funct7"]>, - Arguments<(ins LLVM_Type:$rs1, LLVM_Type:$rs2, - ConfinedAttr, IntMaxValue<127>]>:$funct7)> { - let assemblyFormat = "$rs1 `,` $rs2 prop-dict attr-dict `:` type($rs1) `,` type($rs2)"; -} - -//===----------------------------------------------------------------------===// -// Generic instruction and debug operations -//===----------------------------------------------------------------------===// - -def InstOp : Buckyball_Op<"inst"> { - let summary = "Generic Buckyball custom instruction carrier"; - let description = [{ - Emits a Buckyball custom instruction with explicit rs1, rs2 and funct7. - }]; - let arguments = (ins I64:$rs1, I64:$rs2, - ConfinedAttr, IntMaxValue<127>]>:$funct7); - let assemblyFormat = "$rs1 `,` $rs2 prop-dict attr-dict `:` type($rs1)"; -} - -def PrintMemRefOp : Buckyball_Op<"print_memref"> { - let summary = "Print memref contents for debugging"; - let description = [{ - Print the contents of a memref for debugging purposes. - }]; - let arguments = (ins AnyMemRef:$input); - let assemblyFormat = "$input prop-dict attr-dict `:` type($input)"; -} - -def PrintScalarOp : Buckyball_Op<"print_scalar"> { - let summary = "Print scalar value for debugging"; - let description = [{ - Print a scalar value for debugging purposes. - }]; - let arguments = (ins AnyType:$value); - let assemblyFormat = "$value prop-dict attr-dict `:` type($value)"; -} - -#endif // BUCKYBALL_OPS diff --git a/waste/stack/compiler/src/Dialect/Buckyball/BuckyballDialect.h b/waste/stack/compiler/src/Dialect/Buckyball/BuckyballDialect.h deleted file mode 100644 index 3175776fb..000000000 --- a/waste/stack/compiler/src/Dialect/Buckyball/BuckyballDialect.h +++ /dev/null @@ -1,25 +0,0 @@ -//===- BuckyballDialect.h - MLIR Dialect for RISC-V Gemmmini extension -//------===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#ifndef BUCKYBALL_BUCKYBALLDIALECT_H -#define BUCKYBALL_BUCKYBALLDIALECT_H - -#include "mlir/IR/Dialect.h" - -#include "BuckyballDialect.h.inc" - -#endif // BUCKYBALL_BUCKYBALLDIALECT_H diff --git a/waste/stack/compiler/src/Dialect/Buckyball/BuckyballInterfaces.td b/waste/stack/compiler/src/Dialect/Buckyball/BuckyballInterfaces.td deleted file mode 100644 index cfa3534bd..000000000 --- a/waste/stack/compiler/src/Dialect/Buckyball/BuckyballInterfaces.td +++ /dev/null @@ -1,41 +0,0 @@ -//===-- BuckyballInterfaces.td - Bank lowering interfaces --*- tablegen -*-===// -// -// The framework owns bank lowering. A Ball takes part by implementing these -// interfaces on its operations. -// -//===----------------------------------------------------------------------===// - -#ifndef BUCKYBALL_INTERFACES -#define BUCKYBALL_INTERFACES - -include "mlir/IR/OpBase.td" - -def BankSSALoweringInterface : OpInterface<"BankSSALowering"> { - let cppNamespace = "::buddy::buckyball"; - let description = [{ - A memref-level Ball operation that rewrites itself into explicit bank-SSA - operations: bank allocation, mvin, the bank-level Ball operation and mvout. - Lowered by `lower-buckyball-to-bank-ssa`. - }]; - let methods = [ - InterfaceMethod<"Replaces this operation with bank-SSA operations.", - "::mlir::LogicalResult", "lowerToBankSSA", - (ins "::mlir::PatternRewriter &":$rewriter)> - ]; -} - -def PhysicalBankAssignmentInterface : OpInterface<"PhysicalBankAssignment"> { - let cppNamespace = "::buddy::buckyball"; - let description = [{ - A bank-SSA Ball operation that rewrites itself into its physical-bank - operation once `assign-physical-banks` has mapped its bank handles. - }]; - let methods = [ - InterfaceMethod<"Replaces this operation with its physical operation.", - "::mlir::LogicalResult", "assignPhysicalBanks", - (ins "::mlir::PatternRewriter &":$rewriter, - "::mlir::buddy::PhysicalBankState &":$state)> - ]; -} - -#endif // BUCKYBALL_INTERFACES diff --git a/waste/stack/compiler/src/Dialect/Buckyball/BuckyballOps.h b/waste/stack/compiler/src/Dialect/Buckyball/BuckyballOps.h deleted file mode 100644 index 80e5ee63f..000000000 --- a/waste/stack/compiler/src/Dialect/Buckyball/BuckyballOps.h +++ /dev/null @@ -1,38 +0,0 @@ -//===- BuckyballOps.h - MLIR Dialect for RISC-V Buckyball extension -//---------===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#ifndef BUCKYBALL_BUCKYBALLOPS_H -#define BUCKYBALL_BUCKYBALLOPS_H - -#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/Dialect.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/Interfaces/SideEffectInterfaces.h" - -#include "mlir/IR/PatternMatch.h" - -namespace mlir::buddy { -class PhysicalBankState; -} // namespace mlir::buddy - -#include "BuckyballInterfaces.h.inc" - -#define GET_OP_CLASSES -#include "Buckyball.h.inc" -#endif // BUCKYBALL_BUCKYBALLOPS_H diff --git a/waste/stack/compiler/src/Dialect/Buckyball/IR/BuckyballDialect.cpp b/waste/stack/compiler/src/Dialect/Buckyball/IR/BuckyballDialect.cpp deleted file mode 100644 index b1fb4fefe..000000000 --- a/waste/stack/compiler/src/Dialect/Buckyball/IR/BuckyballDialect.cpp +++ /dev/null @@ -1,79 +0,0 @@ -//===- BuckyballDialect.cpp - MLIR Buckyball dialect implementation -//----------===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// -// -// This file implements the Buckyball dialect and its operations. -// -//===----------------------------------------------------------------------===// - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/LLVMIR/LLVMTypes.h" -#include "mlir/Dialect/Vector/IR/VectorOps.h" -#include "mlir/IR/Builders.h" -#include "mlir/IR/DialectImplementation.h" -#include "mlir/IR/OpImplementation.h" -#include "mlir/IR/TypeUtilities.h" -#include "llvm/ADT/TypeSwitch.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" - -// Generated with unqualified names; included before the using-directives. -#include "BuckyballInterfaces.cpp.inc" - -using namespace mlir; -using namespace ::buddy::buckyball; - -#include "BuckyballDialect.cpp.inc" - -#define GET_OP_CLASSES -#include "Buckyball.cpp.inc" - -void BuckyballDialect::initialize() { - addOperations< -#define GET_OP_LIST -#include "Buckyball.cpp.inc" - >(); -} - -LogicalResult MsetTransferOp::verify() { - for (Value operand : getOperands()) { - if (auto constant = operand.getDefiningOp()) - if (constant.value() < 0 || constant.value() > 1023) - return emitOpError("bank ID must be in [0, 1023]"); - } - auto source = getSource().getDefiningOp(); - auto target = getTarget().getDefiningOp(); - if (source && target) { - if (source.value() == target.value()) - return emitOpError("source and target must differ"); - } - return success(); -} - -LogicalResult MvinOp::verify() { - if (auto group = getGroup()) - if (*group < 0 || *group > 31) - return emitOpError("group must be in [0, 31]"); - return success(); -} - -LogicalResult MvoutOp::verify() { - if (auto group = getGroup()) - if (*group < 0 || *group > 31) - return emitOpError("group must be in [0, 31]"); - return success(); -} diff --git a/waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp b/waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp deleted file mode 100644 index eb84d49d9..000000000 --- a/waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.cpp +++ /dev/null @@ -1,551 +0,0 @@ -//===- LegalizeForLLVMExportBase.cpp - Buckyball base LLVM lowering -------===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#include "Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h" - -#include "mlir/Conversion/LLVMCommon/MemRefBuilder.h" -#include "mlir/Conversion/LLVMCommon/Pattern.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/BuiltinTypes.h" -#include "llvm/Support/ErrorHandling.h" - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace buddy { -namespace buckyball { -namespace legalize { - -uint64_t fieldBits(uint64_t val, int startBit, int endBit) { - uint64_t width = endBit - startBit + 1; - uint64_t mask = (1ULL << width) - 1; - return (val & mask) << startBit; -} - -int64_t addrBitsForDepth(int64_t bankDepth) { - if (bankDepth <= 1) - return -1; - int64_t bits = 0; - int64_t x = bankDepth - 1; - while (x) { - x >>= 1; - ++bits; - } - return bits; -} - -Value cstI64(OpBuilder &b, Location loc, uint64_t v) { - return b.create(loc, b.getI64Type(), - b.getI64IntegerAttr(v)); -} - -static int64_t elemByteSize(Type el) { - if (auto it = dyn_cast(el)) - return it.getWidth() / 8; - if (auto ft = dyn_cast(el)) - return ft.getWidth() / 8; - return -1; -} - -struct MemrefAddress { - Value address; - Value hostPtr; -}; - -static MemrefAddress extractMemrefAddress(OpBuilder &b, Location loc, - Value memref) { - auto ty = cast(memref.getType()); - int64_t eb = elemByteSize(ty.getElementType()); - if (eb <= 0) - llvm_unreachable( - "bb memref intrinsic: unsupported element type for ptr offset"); - auto meta = b.create(loc, memref); - Value base = meta.getBaseBuffer(); - Value off = meta.getOffset(); - Value baseIdx = b.create( - loc, b.getIndexType(), base); - Value baseI64 = b.create(loc, b.getI64Type(), baseIdx); - Value offI64 = b.create(loc, b.getI64Type(), off); - Value offBytes = offI64; - if (eb != 1) - offBytes = b.create(loc, offI64, cstI64(b, loc, eb)); - Value address = b.create(loc, baseI64, offBytes); - Type ptrType = LLVM::LLVMPointerType::get(b.getContext()); - Value hostPtr = LLVM::IntToPtrOp::create(b, loc, ptrType, address); - return {address, hostPtr}; -} - -Value extractPtr(OpBuilder &b, Location loc, Value memref) { - return extractMemrefAddress(b, loc, memref).address; -} - -Value packRs1BanksIter(OpBuilder &b, Location loc, Value rBank0, Value rBank1, - Value wBank, Value iter) { - Value rBank0Field = - b.create(loc, rBank0, cstI64(b, loc, 0x3FF)); - Value rBank1Field = b.create( - loc, b.create(loc, rBank1, cstI64(b, loc, 0x3FF)), - cstI64(b, loc, 10)); - Value wBankField = b.create( - loc, b.create(loc, wBank, cstI64(b, loc, 0x3FF)), - cstI64(b, loc, 20)); - Value iterField = b.create( - loc, b.create(loc, iter, cstI64(b, loc, (1ULL << 34) - 1)), - cstI64(b, loc, 30)); - Value rs1Part01 = b.create(loc, rBank0Field, rBank1Field); - Value rs1Part012 = b.create(loc, rs1Part01, wBankField); - return b.create(loc, rs1Part012, iterField); -} - -Value packRs1BankIter(OpBuilder &b, Location loc, Value bankId, Value depth) { - Value z = cstI64(b, loc, 0); - return packRs1BanksIter(b, loc, bankId, z, z, depth); -} - -Value packRs1WriteBankIter(OpBuilder &b, Location loc, Value bankId, - Value depth) { - Value z = cstI64(b, loc, 0); - return packRs1BanksIter(b, loc, z, z, bankId, depth); -} - -Value packRs2MemStride(OpBuilder &b, Location loc, Value memAddr, - Value stride) { - Value mem = - b.create(loc, memAddr, cstI64(b, loc, (1ULL << 39) - 1)); - Value s = - b.create(loc, stride, cstI64(b, loc, (1ULL << 19) - 1)); - Value sHi = b.create(loc, s, cstI64(b, loc, 39)); - return b.create(loc, mem, sHi); -} - -void emitMset(OpBuilder &b, Location loc, uint64_t bankId, uint64_t row, - uint64_t col, uint64_t alloc) { - uint64_t rs1 = fieldBits(bankId, 0, 9); - uint64_t rs2 = - fieldBits(row, 0, 4) | fieldBits(col, 5, 9) | fieldBits(alloc, 10, 10); - b.create(loc, cstI64(b, loc, rs1), cstI64(b, loc, rs2)); -} - -static constexpr char kDmaTouchMvoutFn[] = "dma_touch_mvout"; - -static FlatSymbolRefAttr getOrInsertExtFunc(OpBuilder &b, ModuleOp module, - StringRef name, - LLVM::LLVMFunctionType type) { - if (module.lookupSymbol(name)) - return FlatSymbolRefAttr::get(b.getContext(), name); - - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToEnd(module.getBody()); - LLVM::LLVMFuncOp::create(b, module.getLoc(), name, type, - LLVM::Linkage::External, false, LLVM::CConv::C); - return FlatSymbolRefAttr::get(b.getContext(), name); -} - -static ModuleOp parentModule(OpBuilder &b) { - Operation *op = b.getInsertionBlock()->getParentOp(); - ModuleOp module = op->getParentOfType(); - if (!module) - llvm_unreachable("bb_dma: missing ModuleOp parent"); - return module; -} - -static void emitBbDmaBankColumns(OpBuilder &b, Location loc, StringRef name, - Value bankId, Value cols) { - ModuleOp module = parentModule(b); - auto i32Ty = IntegerType::get(b.getContext(), 32); - auto voidTy = LLVM::LLVMVoidType::get(b.getContext()); - auto fnTy = LLVM::LLVMFunctionType::get(voidTy, {i32Ty, i32Ty}); - FlatSymbolRefAttr callee = getOrInsertExtFunc(b, module, name, fnTy); - Value bankI32 = b.create(loc, i32Ty, bankId); - Value colsI32 = b.create(loc, i32Ty, cols); - LLVM::CallOp::create(b, loc, TypeRange{}, callee, - ValueRange{bankI32, colsI32}); -} - -static void emitBbDmaTouchMvout(OpBuilder &b, Location loc, Value hostPtr, - Value depth, Value stride, Value bankId) { - ModuleOp module = parentModule(b); - auto ptrTy = LLVM::LLVMPointerType::get(b.getContext()); - auto i64Ty = IntegerType::get(b.getContext(), 64); - auto i32Ty = IntegerType::get(b.getContext(), 32); - auto voidTy = LLVM::LLVMVoidType::get(b.getContext()); - auto fnTy = LLVM::LLVMFunctionType::get(voidTy, {ptrTy, i64Ty, i64Ty, i32Ty}); - FlatSymbolRefAttr callee = - getOrInsertExtFunc(b, module, kDmaTouchMvoutFn, fnTy); - Value bankI32 = b.create(loc, i32Ty, bankId); - LLVM::CallOp::create(b, loc, TypeRange{}, callee, - ValueRange{hostPtr, depth, stride, bankI32}); -} - -namespace { - -template -class ForwardOperands : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - LogicalResult - matchAndRewrite(OpTy op, typename OpTy::Adaptor adaptor, - ConversionPatternRewriter &rewriter) const final { - if (adaptor.getOperands().getTypes() == op->getOperands().getTypes()) - return rewriter.notifyMatchFailure(op, "operand types already match"); - rewriter.modifyOpInPlace(op, - [&]() { op->setOperands(adaptor.getOperands()); }); - return success(); - } -}; - -struct BuckyballMsetLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - LogicalResult - matchAndRewrite(MsetOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - Location loc = op.getLoc(); - Value bankId = adaptor.getBankId(); - Value rs1 = rewriter.create(loc, bankId, - cstI64(rewriter, loc, 0x3FF)); - uint64_t allocBit = op.getAlloc() ? 1u : 0u; - uint64_t rowVal = op.getAlloc() ? static_cast(op.getRow()) : 0u; - uint64_t colVal = op.getAlloc() ? static_cast(op.getCol()) : 0u; - uint64_t rs2Val = fieldBits(rowVal, 0, 4) | fieldBits(colVal, 5, 9) | - fieldBits(allocBit, 10, 10); - Value colsForTouch = cstI64(rewriter, loc, op.getAlloc() ? colVal : 0u); - emitBbDmaBankColumns(rewriter, loc, - op.getAlloc() ? "dma_bank_allocate" - : "dma_bank_set_cols", - bankId, colsForTouch); - rewriter.replaceOpWithNewOp(op, rs1, - cstI64(rewriter, loc, rs2Val)); - return success(); - } -}; - -struct BuckyballMsetTransferLowering - : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - LogicalResult - matchAndRewrite(MsetTransferOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - Location loc = op.getLoc(); - Value source = adaptor.getSource(), target = adaptor.getTarget(); - auto i64 = rewriter.getI64Type(); - auto fnType = LLVM::LLVMFunctionType::get( - LLVM::LLVMVoidType::get(rewriter.getContext()), {i64, i64}); - auto callee = getOrInsertExtFunc(rewriter, parentModule(rewriter), - "dma_bank_transfer", fnType); - LLVM::CallOp::create(rewriter, loc, TypeRange{}, callee, - ValueRange{source, target}); - Value targetBits = - rewriter.create(loc, target, cstI64(rewriter, loc, 20)); - Value rs1 = rewriter.create(loc, source, targetBits); - rewriter.replaceOpWithNewOp(op, rs1, - cstI64(rewriter, loc, 4096)); - return success(); - } -}; - -struct BuckyballMvinLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(MvinOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - Location loc = op.getLoc(); - MemrefAddress memref = extractMemrefAddress(rewriter, loc, op.getInput()); - Value rs1 = packRs1WriteBankIter(rewriter, loc, adaptor.getAddr(), - adaptor.getDepth()); - Value rs2 = - packRs2MemStride(rewriter, loc, memref.address, adaptor.getStride()); - if (auto group = op.getGroup()) - rs2 = rewriter.create( - loc, rs2, - cstI64(rewriter, loc, (1ULL << 63) | (uint64_t(*group) << 58))); - - rewriter.replaceOpWithNewOp(op, rs1, rs2); - return success(); - } -}; - -struct BuckyballMvin2dLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(Mvin2dOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - - Location loc = op.getLoc(); - MemrefAddress memref = extractMemrefAddress(rewriter, loc, op.getInput()); - Value lowBits = rewriter.create(loc, memref.address, - cstI64(rewriter, loc, 7)); - Value aligned = rewriter.create( - loc, arith::CmpIPredicate::eq, lowBits, cstI64(rewriter, loc, 0)); - Value inRange = rewriter.create( - loc, arith::CmpIPredicate::ult, memref.address, - cstI64(rewriter, loc, 1ULL << 39)); - Value valid8 = rewriter.create(loc, arith::CmpIPredicate::eq, - adaptor.getValidBytes(), - cstI64(rewriter, loc, 8)); - Value valid16 = rewriter.create( - loc, arith::CmpIPredicate::eq, adaptor.getValidBytes(), - cstI64(rewriter, loc, 16)); - Value validLength = rewriter.create(loc, valid8, valid16); - Value validAddress = rewriter.create(loc, aligned, inRange); - Value validInput = - rewriter.create(loc, validAddress, validLength); - Value validInput64 = - rewriter.create(loc, rewriter.getI64Type(), validInput); - // This lowering also runs inside single-block SCF regions. - auto tail = LLVM::TailCallKindAttr::get( - rewriter.getContext(), LLVM::tailcallkind::TailCallKind::None); - LLVM::InlineAsmOp::create( - rewriter, loc, Type(), ValueRange{validInput64}, - rewriter.getStringAttr("bnez $0, 1f\n\tunimp\n1:"), - rewriter.getStringAttr("r,~{memory}"), rewriter.getUnitAttr(), - UnitAttr(), tail, UnitAttr(), nullptr, nullptr); - Value rs1 = packRs1WriteBankIter(rewriter, loc, adaptor.getAddr(), - adaptor.getHeight()); - Value address = rewriter.create(loc, memref.address, - cstI64(rewriter, loc, 3)); - Value pixel = rewriter.create(loc, adaptor.getPixelBytes(), - cstI64(rewriter, loc, 0x3ff)); - Value pixelUnits = - rewriter.create(loc, pixel, cstI64(rewriter, loc, 8)); - Value width = rewriter.create(loc, adaptor.getWidth(), - cstI64(rewriter, loc, 1)); - Value valid = - rewriter.create(loc, rewriter.getI64Type(), valid8); - Value rs2 = rewriter.create( - loc, address, - rewriter.create(loc, pixelUnits, - cstI64(rewriter, loc, 36))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getSourceWidth(), - cstI64(rewriter, loc, 43))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, adaptor.getDstBase(), - cstI64(rewriter, loc, 53))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, width, cstI64(rewriter, loc, 59))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, valid, cstI64(rewriter, loc, 62))); - rewriter.replaceOpWithNewOp(op, rs1, rs2, - rewriter.getI32IntegerAttr(34)); - return success(); - } -}; - -struct BuckyballMvinMmioLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - LogicalResult - matchAndRewrite(MvinMmioOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - Location loc = op.getLoc(); - MemrefAddress memref = extractMemrefAddress(rewriter, loc, op.getInput()); - Value rs1 = rewriter.create(loc, adaptor.getRows(), - cstI64(rewriter, loc, 30)); - Value address = rewriter.create( - loc, memref.address, cstI64(rewriter, loc, (1ULL << 39) - 1)); - Value mmio = rewriter.create( - loc, adaptor.getMmioAddr(), cstI64(rewriter, loc, (1ULL << 17) - 1)); - Value columns = rewriter.create(loc, adaptor.getColumns(), - cstI64(rewriter, loc, 0xff)); - Value rs2 = rewriter.create( - loc, address, - rewriter.create(loc, mmio, cstI64(rewriter, loc, 39))); - rs2 = rewriter.create( - loc, rs2, - rewriter.create(loc, columns, - cstI64(rewriter, loc, 56))); - rewriter.replaceOpWithNewOp(op, rs1, rs2); - return success(); - } -}; - -struct BuckyballMvoutLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - - LogicalResult - matchAndRewrite(MvoutOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - Location loc = op.getLoc(); - MemrefAddress memref = extractMemrefAddress(rewriter, loc, op.getOutput()); - if (op.getGroup()) { - auto fnTy = LLVM::LLVMFunctionType::get( - LLVM::LLVMVoidType::get(rewriter.getContext()), - {LLVM::LLVMPointerType::get(rewriter.getContext()), - rewriter.getI64Type(), rewriter.getI64Type()}); - auto callee = getOrInsertExtFunc(rewriter, parentModule(rewriter), - "dma_touch_mvout_group", fnTy); - LLVM::CallOp::create( - rewriter, loc, TypeRange{}, callee, - ValueRange{memref.hostPtr, adaptor.getDepth(), adaptor.getStride()}); - } else { - emitBbDmaTouchMvout(rewriter, loc, memref.hostPtr, adaptor.getDepth(), - adaptor.getStride(), adaptor.getAddr()); - } - Value rs1 = - packRs1BankIter(rewriter, loc, adaptor.getAddr(), adaptor.getDepth()); - Value rs2 = - packRs2MemStride(rewriter, loc, memref.address, adaptor.getStride()); - if (auto group = op.getGroup()) - rs2 = rewriter.create( - loc, rs2, - cstI64(rewriter, loc, (1ULL << 63) | (uint64_t(*group) << 58))); - - rewriter.replaceOpWithNewOp(op, rs1, rs2); - return success(); - } -}; - -struct BuckyballMvoverLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - LogicalResult - matchAndRewrite(MvoverOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - Location loc = op.getLoc(); - auto bounded = [&](Value value, uint64_t maximum) { - Value valid = - rewriter.create(loc, arith::CmpIPredicate::ule, value, - cstI64(rewriter, loc, maximum)); - rewriter.create(loc, valid, - "mvover operand exceeds ISA range"); - }; - bounded(adaptor.getSourceThread(), 255); - bounded(adaptor.getTargetThread(), 255); - bounded(adaptor.getSourceBank(), 1023); - bounded(adaptor.getTargetBank(), 1023); - bounded(adaptor.getSourceRow(), 65535); - bounded(adaptor.getTargetRow(), 65535); - Value count = rewriter.create(loc, adaptor.getRows(), - cstI64(rewriter, loc, 1)); - bounded(count, 65535); - auto shifted = [&](Value value, uint64_t shift) -> Value { - return rewriter.create(loc, value, - cstI64(rewriter, loc, shift)); - }; - Value rs1 = rewriter.create( - loc, adaptor.getSourceThread(), shifted(adaptor.getTargetThread(), 8)); - rs1 = rewriter.create(loc, rs1, - shifted(adaptor.getSourceBank(), 16)); - rs1 = rewriter.create(loc, rs1, - shifted(adaptor.getTargetBank(), 26)); - Value rs2 = rewriter.create( - loc, adaptor.getSourceRow(), shifted(adaptor.getTargetRow(), 16)); - rs2 = rewriter.create(loc, rs2, shifted(count, 32)); - rewriter.replaceOpWithNewOp(op, rs1, rs2, - rewriter.getI32IntegerAttr(13)); - return success(); - } -}; - -static bool contiguousRankOne(Type type) { - auto memref = dyn_cast(type); - if (!memref || memref.getRank() != 1) - return false; - SmallVector strides; - int64_t offset; - return succeeded(memref.getStridesAndOffset(strides, offset)) && - strides[0] == 1; -} - -struct ContiguousCopyLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - LogicalResult - matchAndRewrite(memref::CopyOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - if (!contiguousRankOne(op.getSource().getType()) || - !contiguousRankOne(op.getTarget().getType())) - return failure(); - Location loc = op.getLoc(); - auto sourceType = cast(op.getSource().getType()); - auto targetType = cast(op.getTarget().getType()); - Value zero = - LLVM::ConstantOp::create(rewriter, loc, getIndexType(), - rewriter.getIntegerAttr(getIndexType(), 0)); - Value source = getStridedElementPtr(rewriter, loc, sourceType, - adaptor.getSource(), ValueRange{zero}); - Value target = getStridedElementPtr(rewriter, loc, targetType, - adaptor.getTarget(), ValueRange{zero}); - Value size = MemRefDescriptor(adaptor.getSource()).size(rewriter, loc, 0); - Value bytes = LLVM::MulOp::create( - rewriter, loc, size, - getSizeInBytes(loc, sourceType.getElementType(), rewriter)); - LLVM::MemcpyOp::create(rewriter, loc, target, source, bytes, false); - rewriter.eraseOp(op); - return success(); - } -}; - -struct BuckyballInstLowering : public ConvertOpToLLVMPattern { - using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; - LogicalResult - matchAndRewrite(InstOp op, OpAdaptor adaptor, - ConversionPatternRewriter &rewriter) const override { - rewriter.replaceOpWithNewOp( - op, adaptor.getRs1(), adaptor.getRs2(), - rewriter.getI32IntegerAttr(op.getFunct7())); - return success(); - } -}; - -} // namespace - -void populateBaseLegalizeForLLVMExportPatterns( - LLVMTypeConverter &converter, RewritePatternSet &patterns, - bool includeFuncOperandForwarding) { - if (includeFuncOperandForwarding) { - patterns.add, - ForwardOperands, - ForwardOperands>(converter, - &converter.getContext()); - } - patterns.add(converter); - patterns.add(converter); - patterns.add(converter); - patterns.add(converter); - patterns.add(converter); - patterns.add(converter); - patterns.add(converter); - patterns.add(converter); -} - -void configureBaseLegalizeForExportTarget(LLVMConversionTarget &target) { - target.addLegalOp(); - target.addIllegalOp(); - target.addLegalDialect(); - target.addDynamicallyLegalOp([](memref::CopyOp op) { - return !contiguousRankOne(op.getSource().getType()) || - !contiguousRankOne(op.getTarget().getType()); - }); - target.addLegalDialect(); - target.addLegalDialect(); -} - -} // namespace legalize -} // namespace buckyball -} // namespace buddy diff --git a/waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h b/waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h deleted file mode 100644 index d38bbea26..000000000 --- a/waste/stack/compiler/src/Dialect/Buckyball/Transforms/LegalizeForLLVMExportBase.h +++ /dev/null @@ -1,60 +0,0 @@ -//===- LegalizeForLLVMExportBase.h - Buckyball base LLVM lowering -*- C++ -//-*-===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// - -#ifndef BUDDY_BUCKYBALL_LEGALIZE_FOR_LLVM_EXPORT_BASE_H -#define BUDDY_BUCKYBALL_LEGALIZE_FOR_LLVM_EXPORT_BASE_H - -#include "mlir/Conversion/LLVMCommon/ConversionTarget.h" -#include "mlir/Conversion/LLVMCommon/TypeConverter.h" -#include "mlir/IR/Builders.h" -#include "mlir/IR/PatternMatch.h" - -#include - -namespace buddy { -namespace buckyball { -namespace legalize { - -uint64_t fieldBits(uint64_t val, int startBit, int endBit); -int64_t addrBitsForDepth(int64_t bankDepth); - -mlir::Value cstI64(mlir::OpBuilder &b, mlir::Location loc, uint64_t v); -mlir::Value extractPtr(mlir::OpBuilder &b, mlir::Location loc, - mlir::Value memref); -mlir::Value packRs1BanksIter(mlir::OpBuilder &b, mlir::Location loc, - mlir::Value rBank0, mlir::Value rBank1, - mlir::Value wBank, mlir::Value iter); -mlir::Value packRs1BankIter(mlir::OpBuilder &b, mlir::Location loc, - mlir::Value bankId, mlir::Value depth); -mlir::Value packRs1WriteBankIter(mlir::OpBuilder &b, mlir::Location loc, - mlir::Value bankId, mlir::Value depth); -mlir::Value packRs2MemStride(mlir::OpBuilder &b, mlir::Location loc, - mlir::Value memAddr, mlir::Value stride); -void emitMset(mlir::OpBuilder &b, mlir::Location loc, uint64_t bankId, - uint64_t row, uint64_t col, uint64_t alloc); - -void populateBaseLegalizeForLLVMExportPatterns( - mlir::LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns, - bool includeFuncOperandForwarding); - -void configureBaseLegalizeForExportTarget(mlir::LLVMConversionTarget &target); - -} // namespace legalize -} // namespace buckyball -} // namespace buddy - -#endif // BUDDY_BUCKYBALL_LEGALIZE_FOR_LLVM_EXPORT_BASE_H diff --git a/waste/stack/compiler/src/Plugin.cpp b/waste/stack/compiler/src/Plugin.cpp deleted file mode 100644 index 07559dd7b..000000000 --- a/waste/stack/compiler/src/Plugin.cpp +++ /dev/null @@ -1,30 +0,0 @@ -//===- Plugin.cpp - The Buckyball compiler as a buddy-opt plugin ----------===// -// -// buddy-opt --load-dialect-plugin= --load-pass-plugin= loads the -// Buckyball dialect and the framework passes. Ball passes register themselves -// when the plugin library is loaded. -// -//===----------------------------------------------------------------------===// - -#include "Conversion/LowerBuckyball/LowerBuckyball.h" - -#include "mlir/Tools/Plugins/DialectPlugin.h" -#include "mlir/Tools/Plugins/PassPlugin.h" -#include "llvm/Config/llvm-config.h" - -#include "Buckyball/BuckyballDialect.h" - -using namespace mlir; - -extern "C" LLVM_ATTRIBUTE_WEAK DialectPluginLibraryInfo -mlirGetDialectPluginInfo() { - return {MLIR_PLUGIN_API_VERSION, "Buckyball", LLVM_VERSION_STRING, - [](DialectRegistry *registry) { - registry->insert<::buddy::buckyball::BuckyballDialect>(); - }}; -} - -extern "C" LLVM_ATTRIBUTE_WEAK PassPluginLibraryInfo mlirGetPassPluginInfo() { - return {MLIR_PLUGIN_API_VERSION, "Buckyball", LLVM_VERSION_STRING, - [] { mlir::buddy::registerBuckyballPasses(); }}; -} diff --git a/waste/stack/compiler/src/Target/BuckyballTargetRegistry.cpp b/waste/stack/compiler/src/Target/BuckyballTargetRegistry.cpp deleted file mode 100644 index b15489471..000000000 --- a/waste/stack/compiler/src/Target/BuckyballTargetRegistry.cpp +++ /dev/null @@ -1,74 +0,0 @@ -#include "Target/BuckyballTargetRegistry.h" - -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/Twine.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" - -using namespace llvm; - -#include "BuckyballTargetRegistry.inc" - -const buckyball_target::BuckyballTargetConfig & -buckyball_target::getBuckyballTarget() { - return kBuckyballTarget; -} - -static cl::list selectedBalls( - "buckyball-ball", cl::CommaSeparated, - cl::desc( - "Select Ball instance names when an instruction has multiple owners")); - -static const buckyball_target::BuckyballIsaEntry & -resolveInstruction(StringRef mnemonic) { - for (const std::string &name : selectedBalls) { - if (!llvm::any_of(kBuckyballTarget.ballMappings, - [&](const auto &ball) { return ball.name == name; })) - report_fatal_error(Twine("Unknown Ball instance: ") + name); - } - const buckyball_target::BuckyballIsaEntry *unique = nullptr; - const buckyball_target::BuckyballIsaEntry *selected = nullptr; - unsigned count = 0; - for (const auto &entry : kBuckyballTarget.isa) { - if (entry.mnemonic != mnemonic) - continue; - unique = &entry; - ++count; - auto name = kBuckyballTarget.ballMappings[entry.ballIndex].name; - if (llvm::is_contained(selectedBalls, name)) { - if (selected) - report_fatal_error( - Twine("Multiple selected Ball instances implement ") + mnemonic); - selected = &entry; - } - } - if (selected) - return *selected; - if (count == 1) - return *unique; - if (count > 1) - report_fatal_error(Twine("Ambiguous instruction ") + mnemonic + - "; select its instance with --buckyball-ball="); - report_fatal_error(Twine("Buckyball core ") + kBuckyballTarget.core + - " does not define ISA mnemonic " + mnemonic); -} - -int32_t buckyball_target::getBuckyballFunct7(llvm::StringRef mnemonic) { - return resolveInstruction(mnemonic).funct7; -} - -const buckyball_target::BuckyballBallMapping & -buckyball_target::getBuckyballBallMapping(llvm::StringRef mnemonic) { - return kBuckyballTarget.ballMappings[resolveInstruction(mnemonic).ballIndex]; -} - -int64_t buckyball_target::getBuckyballBallParam(llvm::StringRef mnemonic, - llvm::StringRef param) { - for (const BuckyballBallParam &entry : - getBuckyballBallMapping(mnemonic).params) { - if (entry.name == param) - return entry.value; - } - report_fatal_error(Twine("Buckyball instruction ") + mnemonic + - " has no parameter " + param); -} diff --git a/waste/stack/compiler/src/Target/BuckyballTargetRegistry.h b/waste/stack/compiler/src/Target/BuckyballTargetRegistry.h deleted file mode 100644 index 946df84c5..000000000 --- a/waste/stack/compiler/src/Target/BuckyballTargetRegistry.h +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef BUCKYBALL_TARGET_REGISTRY_H -#define BUCKYBALL_TARGET_REGISTRY_H - -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/StringRef.h" - -#include - -namespace buckyball_target { - -struct BuckyballIsaEntry { - llvm::StringRef mnemonic; - int32_t funct7; - unsigned ballIndex; -}; - -struct BuckyballBallParam { - llvm::StringRef name; - int64_t value; -}; - -struct BuckyballBallMapping { - llvm::StringRef name; - int64_t inBW; - int64_t outBW; - llvm::ArrayRef params; -}; - -struct BuckyballTargetConfig { - llvm::StringRef core; - int64_t bankNum; - int64_t bankWidthBits; - int64_t bankDepth; - int64_t privateBankMax; - bool rvvEnabled; - llvm::ArrayRef ballMappings; - llvm::ArrayRef isa; -}; - -const BuckyballTargetConfig &getBuckyballTarget(); -int32_t getBuckyballFunct7(llvm::StringRef mnemonic); -const BuckyballBallMapping &getBuckyballBallMapping(llvm::StringRef mnemonic); -int64_t getBuckyballBallParam(llvm::StringRef mnemonic, llvm::StringRef param); - -} // namespace buckyball_target - -#endif // BUCKYBALL_TARGET_REGISTRY_H diff --git a/waste/stack/compiler/src/Utils/BankUtils.h b/waste/stack/compiler/src/Utils/BankUtils.h deleted file mode 100644 index 54000fa61..000000000 --- a/waste/stack/compiler/src/Utils/BankUtils.h +++ /dev/null @@ -1,131 +0,0 @@ -//===- BankUtils.h - Utilities for Bank operations -----------------------===// -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//===----------------------------------------------------------------------===// -// -// Utility functions for generating Bank-SSA operations. -// -//===----------------------------------------------------------------------===// - -#ifndef BUCKYBALL_CONVERSION_BANKUTILS_H -#define BUCKYBALL_CONVERSION_BANKUTILS_H - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/IR/Builders.h" -#include "mlir/IR/Value.h" - -#include "Buckyball/BuckyballOps.h" - -#include -#include -#include - -namespace buddy { -namespace buckyball { - -/// Create i64 constant. -static inline mlir::Value createI64Const(mlir::OpBuilder &b, mlir::Location loc, - int64_t val) { - return b.create(loc, b.getI64Type(), - b.getI64IntegerAttr(val)); -} - -/// Create i64 constant from uint64_t. -static inline mlir::Value createI64ConstU(mlir::OpBuilder &b, - mlir::Location loc, uint64_t val) { - return b.create(loc, b.getI64Type(), - b.getI64IntegerAttr(val)); -} - -static inline mlir::Value createI1Const(mlir::OpBuilder &b, mlir::Location loc, - bool val) { - return b.create(loc, b.getI1Type(), - b.getBoolAttr(val)); -} - -/// Pack bit field: extract bits [startBit, endBit] from val and shift to -/// position. -static inline uint64_t packBits(uint64_t val, int startBit, int endBit) { - uint64_t width = endBit - startBit + 1; - uint64_t mask = (1ULL << width) - 1; - if (val > mask) { - std::fprintf(stderr, "packBits: value %llu does not fit in [%d:%d]\n", - (unsigned long long)val, startBit, endBit); - std::abort(); - } - return val << startBit; -} - -static inline uint64_t matrixRs2(uint64_t rows, uint64_t cols, uint64_t k) { - if (rows == 0 || cols == 0 || cols > 0xfff || k == 0 || rows > 0xfff || - k > 0xfff || (rows != 1 && rows % 16) || cols % 16 || k % 16) { - std::fprintf( - stderr, - "matrix rs2: rows/cols/k must fit in 12 bits (got %llu %llu %llu)\n", - (unsigned long long)rows, (unsigned long long)cols, - (unsigned long long)k); - std::abort(); - } - return packBits(rows, 0, 11) | packBits(cols, 12, 23) | packBits(k, 24, 35); -} - -/// Allocate a bank with given row/col dimensions. -static inline mlir::Value allocBank(mlir::OpBuilder &b, mlir::Location loc, - int64_t row, int64_t col) { - auto i64Type = b.getI64Type(); - return b.create(loc, i64Type, b.getI64IntegerAttr(row), - b.getI64IntegerAttr(col)); -} - -/// Release a bank. -static inline void releaseBank(mlir::OpBuilder &b, mlir::Location loc, - mlir::Value bank) { - b.create(loc, bank); -} - -/// Move data from memref into bank. -static inline mlir::Value mvinBank(mlir::OpBuilder &b, mlir::Location loc, - mlir::Value memref, mlir::Value bank, - int64_t depth, int64_t stride = 1) { - mlir::Value depthVal = createI64Const(b, loc, depth); - mlir::Value strideVal = createI64Const(b, loc, stride); - return b.create(loc, bank.getType(), memref, bank, depthVal, - strideVal, mlir::IntegerAttr{}); -} - -/// Move data from bank to memref. -static inline mlir::Value mvoutBank(mlir::OpBuilder &b, mlir::Location loc, - mlir::Value memref, mlir::Value bank, - int64_t depth, int64_t stride = 1) { - mlir::Value depthVal = createI64Const(b, loc, depth); - mlir::Value strideVal = createI64Const(b, loc, stride); - return b.create(loc, bank.getType(), memref, bank, depthVal, - strideVal, mlir::IntegerAttr{}); -} - -/// Create mset operation for bank allocation/release. -static inline MsetOp createMset(mlir::OpBuilder &b, mlir::Location loc, - uint64_t bankId, bool alloc, uint64_t row, - uint64_t col) { - auto op = b.create(loc, createI64ConstU(b, loc, bankId)); - op->setAttr("alloc", b.getBoolAttr(alloc)); - op->setAttr("row", b.getI64IntegerAttr(row)); - op->setAttr("col", b.getI64IntegerAttr(col)); - return op; -} - -} // namespace buckyball -} // namespace buddy - -#endif // BUCKYBALL_CONVERSION_BANKUTILS_H diff --git a/waste/stack/compiler/src/Utils/StageTrace.cpp b/waste/stack/compiler/src/Utils/StageTrace.cpp deleted file mode 100644 index b51cf86a6..000000000 --- a/waste/stack/compiler/src/Utils/StageTrace.cpp +++ /dev/null @@ -1,66 +0,0 @@ -#include "Utils/StageTrace.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/Diagnostics.h" - -using namespace mlir; - -LogicalResult mlir::buddy::emitStageTensorTrace(OpBuilder &builder, - Location loc, Value value, - DictionaryAttr metadata) { - auto path = cast(metadata.get("id_path")); - if (path.size() > 4) - return emitError(loc, "stage trace id_path exceeds runtime depth limit"); - auto type = cast(value.getType()); - auto element = type.getElementType(); - auto argumentType = MemRefType::get({ShapedType::kDynamic}, element); - constexpr StringLiteral name = "buckyballTraceStageI8Path"; - Operation *parent = builder.getInsertionBlock()->getParentOp(); - auto module = parent->getParentOfType(); - if (!module.lookupSymbol(name)) { - OpBuilder::InsertionGuard guard(builder); - builder.setInsertionPointToStart(module.getBody()); - Type i64 = builder.getI64Type(); - auto function = func::FuncOp::create( - builder, loc, name, - builder.getFunctionType({i64, i64, i64, i64, i64, i64, argumentType}, - {})); - function.setPrivate(); - function->setAttr("llvm.emit_c_interface", builder.getUnitAttr()); - } - Value packed; - if (!type.getLayout().isIdentity()) { - packed = memref::AllocOp::create(builder, loc, - MemRefType::get(type.getShape(), element)); - memref::CopyOp::create(builder, loc, value, packed); - value = packed; - } - int64_t size = type.getNumElements(); - if (type.getRank() != 1) { - SmallVector groups(1); - for (int64_t axis = 0; axis < type.getRank(); ++axis) - groups[0].push_back(axis); - value = memref::CollapseShapeOp::create( - builder, loc, MemRefType::get({size}, element), value, groups); - } - value = memref::CastOp::create(builder, loc, argumentType, value); - SmallVector arguments; - arguments.push_back(arith::ConstantIntOp::create( - builder, loc, cast(metadata.get("id")).getInt(), 64)); - arguments.push_back( - arith::ConstantIntOp::create(builder, loc, path.size(), 64)); - for (size_t index = 0; index < 4; ++index) { - int64_t component = - index < path.size() ? cast(path[index]).getInt() : -1; - arguments.push_back( - arith::ConstantIntOp::create(builder, loc, component, 64)); - } - arguments.push_back(value); - func::CallOp::create(builder, loc, name, TypeRange{}, arguments); - if (packed) - memref::DeallocOp::create(builder, loc, packed); - return success(); -} diff --git a/waste/stack/compiler/src/Utils/StageTrace.h b/waste/stack/compiler/src/Utils/StageTrace.h deleted file mode 100644 index 3b4b3a8c5..000000000 --- a/waste/stack/compiler/src/Utils/StageTrace.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef BUCKYBALL_STAGE_TRACE_H -#define BUCKYBALL_STAGE_TRACE_H - -#include "mlir/IR/Builders.h" - -namespace mlir::buddy { -LogicalResult emitStageTensorTrace(OpBuilder &builder, Location loc, - Value value, DictionaryAttr metadata); -} - -#endif From 175356330223008de49fcda111b68458f6f4c60e Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 26 Oct 2026 20:10:30 +0800 Subject: [PATCH 191/196] feat: sync to buckyball 0.2.1-a --- .gitignore | 2 +- .../src/main/scala/arithmetic/FALU.scala | 18 +- .../execution/fpu/src/main/scala/FPU.scala | 10 +- .../main/scala/framework/lib/arith/Fma.scala | 6 +- .../framework/lib/arith/IntToFloat.scala | 2 +- .../src/MLIRTest/base/CMakeLists.txt | 2 + .../MLIRTest/base/base_bank_loop_carried.mlir | 41 ++ .../base/base_bank_loop_carried_main.cpp | 35 + .../src/MLIRTest/base/base_bank_transfer.mlir | 50 ++ .../MLIRTest/base/base_bank_transfer_main.cpp | 37 + .../base/base_mvin_mvout_16x16_i8.mlir | 8 +- .../bebop/p2e/02_buildbitstream_api.step.py | 12 - .../bebop/p2e/02_buildbitstream_event.step.py | 19 +- .../bebop/verilator/01_clean_event.step.py | 2 +- .../bebop/verilator/03_build_event.step.py | 2 +- .../bebop/verilator/04_sim_event.step.py | 2 +- .../bebop/verilator/06_batch_event.step.py | 2 +- bbdev/api/steps/dc/01_area_api.step.py | 2 +- bbdev/api/steps/dc/01_area_event.step.py | 3 +- bbdev/api/steps/dc/02_verilog_api.step.py | 2 +- bbdev/api/steps/dc/03_sim_event.step.py | 3 +- bbdev/api/steps/dc/04_power_api.step.py | 2 +- bbdev/api/steps/ip/01_generate_api.step.py | 4 +- bbdev/api/steps/ip/01_generate_event.step.py | 20 +- bbdev/api/steps/ip/02_replace_api.step.py | 2 +- bbdev/api/steps/ip/02_replace_event.step.py | 18 +- bbdev/api/steps/ip/scripts/generate.py | 160 +---- bbdev/api/steps/ip/scripts/macro_compiler.py | 42 -- bbdev/api/steps/ip/scripts/replace.py | 8 +- bbdev/api/steps/ip/scripts/sram_compiler.py | 20 +- .../mill/01_verilator_verilog_event.step.py | 5 +- .../steps/mill/02_vcs_verilog_event.step.py | 5 +- .../steps/mill/03_yosys_verilog_event.step.py | 9 +- .../steps/mill/04_dc_verilog_event.step.py | 12 +- .../06_bebop_verilator_verilog_event.step.py | 6 +- bbdev/api/steps/pt/01_ptpx_event.step.py | 2 +- bbdev/api/steps/uvm/scripts/ball.py | 5 +- bbdev/api/steps/yosys/01_run_api.step.py | 2 +- bbdev/api/steps/yosys/02_verilog_api.step.py | 2 +- bbdev/api/steps/yosys/03_synth_api.step.py | 2 +- bbdev/api/steps/yosys/03_synth_event.step.py | 8 +- .../api/steps/yosys/scripts/yosys-config.yaml | 2 +- bbdev/api/utils/path.py | 3 +- bbdev/bbdev | 3 +- bbdev/mcp/tools/bebop_p2e_buildbitstream.py | 2 - bebop | 2 +- .../balls/f32add/arch/F32AddRow.scala | 2 +- .../balls/f32add/stack/src/Lowering.cpp | 24 +- .../mlir_tests/ball/f32add_groups.mlir | 20 +- .../mlir_tests/ball/f32add_groups_main.cpp | 23 +- .../mlir_tests/ball/f32add_invalid.mlir | 26 +- .../mlir_tests/bank/f32add_groups.mlir | 37 +- .../mlir_tests/bank/f32add_groups_main.cpp | 24 +- .../mlir_tests/bank/f32add_invalid.mlir | 17 +- .../gemmini/arch/GemminiExCtrlCmdStates.scala | 10 +- .../gemmini/arch/GemminiExCtrlDefs.scala | 2 +- .../gemmini/emu/src/21_gemmini_preload.rs | 7 +- .../emu/src/66_gemmini_compute_preloaded.rs | 13 +- .../emu/src/67_gemmini_compute_accumulated.rs | 13 +- .../balls/gemmini/stack/src/Lowering.cpp | 125 ++-- .../gemmini/verify/filelists/gemmini_ball.f | 23 +- .../verify/src/common/gemmini_defs.svh | 3 + .../verify/src/gemmini_expansion_if.sv | 17 + .../balls/gemmini/verify/src/tb_top.sv | 225 ++++-- .../gemmini/verify/src/tests/gemmini_test.svh | 148 ++-- .../gemmini_accumulated_command_16x16.mlir | 11 +- .../ball/gemmini_preloaded_command_16x16.mlir | 11 +- .../workloads/mlir_tests/bank/CMakeLists.txt | 4 +- ...emmini_compute_accumulated_bank_tiles.mlir | 97 +++ ...ini_compute_accumulated_bank_tiles.mlir.in | 63 -- ...ni_compute_accumulated_bank_tiles_main.cpp | 20 +- .../gemmini_compute_preloaded_bank_tiles.mlir | 77 +++ ...mmini_compute_preloaded_bank_tiles.mlir.in | 62 -- ...mini_compute_preloaded_bank_tiles_main.cpp | 11 +- .../components/balls/im2col/arch/Im2col.scala | 524 +++++++------- .../balls/im2col/arch/Im2colConfigRegs.scala | 108 --- .../balls/im2col/arch/Im2colWindow.scala | 75 -- .../balls/im2col/arch/LineBufferManager.scala | 175 ----- .../balls/im2col/arch/StreamWriter.scala | 243 ------- .../components/balls/im2col/emu/src/im2col.rs | 173 +++-- .../im2col/verify/casegen/src/casegen.rs | 40 +- .../balls/im2col/verify/casegen/src/lib.rs | 4 +- .../im2col/verify/filelists/im2col_ball.f | 2 + .../im2col/verify/src/common/im2col_defs.svh | 14 +- .../im2col/verify/src/common/im2col_items.svh | 55 +- .../verify/src/env/im2col_scoreboard.svh | 37 +- .../verify/src/seq/im2col_sequences.svh | 3 +- .../balls/im2col/verify/src/tb_top.sv | 127 ++-- .../im2col/verify/src/tests/im2col_test.svh | 19 +- .../balls/im2col/workloads/isa/im2col.h | 23 +- .../mlir_tests/ball/im2col_6x6_k3.mlir | 10 +- .../components/balls/int2fp/arch/Int2Fp.scala | 23 +- .../components/balls/int2fp/emu/src/int2fp.rs | 20 +- .../balls/int2fp/stack/src/Lowering.cpp | 8 +- .../int2fp/verify/casegen/src/casegen.rs | 54 +- .../balls/int2fp/verify/casegen/src/lib.rs | 4 +- .../int2fp/verify/filelists/int2fp_ball.f | 2 + .../int2fp/verify/src/common/int2fp_defs.svh | 19 +- .../int2fp/verify/src/common/int2fp_items.svh | 57 +- .../int2fp/verify/src/cov/int2fp_cov.svh | 4 +- .../verify/src/env/int2fp_scoreboard.svh | 49 +- .../verify/src/seq/int2fp_sequences.svh | 3 +- .../balls/int2fp/verify/src/tb_top.sv | 146 ++-- .../int2fp/verify/src/tests/int2fp_test.svh | 6 +- .../balls/int2fp/workloads/isa/int2fp.h | 5 + .../workloads/mlir_tests/ball/int2fp_4x4.mlir | 12 +- .../mlir_tests/ball/int2fp_relu_4x4.mlir | 12 +- .../workloads/mlir_tests/bank/int2fp_4x4.mlir | 2 +- .../mlir_tests/bank/int2fp_relu_4x4.mlir | 2 +- .../mlir_tests/bank/int2fp_unit_scale.mlir.in | 2 +- .../balls/int8add/arch/Int8AddBall.scala | 4 +- .../int8add/verify/casegen/src/casegen.rs | 77 +-- .../balls/int8add/verify/casegen/src/lib.rs | 4 +- .../int8add/verify/filelists/int8add_ball.f | 10 +- .../verify/src/common/int8add_defs.svh | 21 +- .../verify/src/common/int8add_items.svh | 79 +-- .../int8add/verify/src/cov/int8add_cov.svh | 13 +- .../verify/src/env/int8add_scoreboard.svh | 58 +- .../verify/src/seq/int8add_sequences.svh | 3 +- .../balls/int8add/verify/src/tb_top.sv | 146 ++-- .../int8add/verify/src/tests/int8add_test.svh | 6 +- .../mlir_tests/ball/int8add_relu.mlir | 12 +- .../balls/int8mul/arch/Int8MulBall.scala | 4 +- .../balls/int8mul/stack/src/Lowering.cpp | 29 +- .../int8mul/verify/casegen/src/casegen.rs | 94 ++- .../balls/int8mul/verify/casegen/src/lib.rs | 4 +- .../int8mul/verify/filelists/int8mul_ball.f | 6 +- .../verify/src/common/int8mul_defs.svh | 21 +- .../verify/src/common/int8mul_items.svh | 70 +- .../int8mul/verify/src/cov/int8mul_cov.svh | 7 +- .../verify/src/env/int8mul_scoreboard.svh | 65 +- .../verify/src/seq/int8mul_sequences.svh | 3 +- .../balls/int8mul/verify/src/tb_top.sv | 146 ++-- .../int8mul/verify/src/tests/int8mul_test.svh | 6 +- .../mlir_tests/ball/int8mul_gate.mlir | 12 +- .../mlir_tests/bank/int8mul_unit_gate.mlir.in | 2 +- .../balls/lut/stack/src/Lowering.cpp | 7 + .../balls/lut/verify/casegen/src/casegen.rs | 59 +- .../balls/lut/verify/casegen/src/lib.rs | 4 +- .../balls/lut/verify/filelists/lut_ball.f | 2 + .../balls/lut/verify/src/common/lut_defs.svh | 21 +- .../balls/lut/verify/src/common/lut_items.svh | 57 +- .../balls/lut/verify/src/cov/lut_cov.svh | 4 +- .../lut/verify/src/env/lut_scoreboard.svh | 42 +- .../lut/verify/src/seq/lut_sequences.svh | 3 +- .../components/balls/lut/verify/src/tb_top.sv | 146 ++-- .../balls/lut/verify/src/tests/lut_test.svh | 6 +- .../mlir_tests/ball/lut_identity.mlir | 12 +- .../matadd/verify/filelists/matadd_ball.f | 2 + .../matadd/verify/src/common/matadd_defs.svh | 8 - .../matadd/verify/src/common/matadd_items.svh | 53 +- .../verify/src/env/matadd_scoreboard.svh | 37 +- .../verify/src/seq/matadd_sequences.svh | 3 +- .../balls/matadd/verify/src/tb_top.sv | 141 ++-- .../matadd/verify/src/tests/matadd_test.svh | 11 +- .../workloads/mlir_tests/ball/matadd_i32.mlir | 12 +- .../balls/maxpool/arch/MaxPoolBall.scala | 41 +- .../balls/maxpool/emu/src/maxpool.rs | 19 +- .../maxpool/verify/casegen/src/casegen.rs | 39 +- .../balls/maxpool/verify/casegen/src/lib.rs | 4 +- .../maxpool/verify/filelists/maxpool_ball.f | 2 + .../verify/src/common/maxpool_defs.svh | 20 +- .../verify/src/common/maxpool_items.svh | 59 +- .../maxpool/verify/src/cov/maxpool_cov.svh | 4 +- .../verify/src/env/maxpool_scoreboard.svh | 36 +- .../verify/src/seq/maxpool_sequences.svh | 3 +- .../balls/maxpool/verify/src/tb_top.sv | 122 ++-- .../maxpool/verify/src/tests/maxpool_test.svh | 6 +- .../balls/maxpool/workloads/isa/maxpool.h | 4 + .../mlir_tests/ball/maxpool_6x6_s2.mlir | 10 +- .../components/balls/mxmm/arch/AddPipe.scala | 4 +- .../balls/redmule/arch/RedmuleBall.scala | 3 +- .../balls/redmule/stack/src/Lowering.cpp | 32 +- .../redmule/verify/filelists/redmule_ball.f | 2 + .../redmule/verify/src/pkg/redmule_pkg.sv | 82 ++- .../balls/redmule/verify/src/tb_top.sv | 144 ++-- .../balls/redmule/workloads/isa/redmule.h | 6 +- .../mlir_tests/ball/redmule_f16_chain.mlir | 6 +- .../ball/redmule_f16_chain_main.cpp | 10 +- .../ball/redmule_f16_invalid_k.mlir | 5 +- .../bank/redmule_f16_invalid_alias.mlir | 5 +- .../mlir_tests/bank/redmule_f16_rows.mlir.in | 2 +- .../balls/relu/stack/src/Lowering.cpp | 27 +- .../mlir_tests/ball/relu_i32_16row.mlir | 10 +- .../bank/relu_i32_mixed_sign.mlir.in | 2 +- .../balls/smatmul/arch/SMatMulUnit.scala | 29 +- .../balls/smatmul/emu/src/smatmul.rs | 28 +- .../smatmul/verify/casegen/src/casegen.rs | 34 +- .../balls/smatmul/verify/casegen/src/lib.rs | 4 +- .../smatmul/verify/filelists/smatmul_ball.f | 2 + .../verify/src/common/smatmul_defs.svh | 13 +- .../verify/src/common/smatmul_items.svh | 48 +- .../smatmul/verify/src/cov/smatmul_cov.svh | 2 +- .../verify/src/env/smatmul_scoreboard.svh | 35 +- .../verify/src/seq/smatmul_sequences.svh | 3 +- .../balls/smatmul/verify/src/tb_top.sv | 146 ++-- .../smatmul/verify/src/tests/smatmul_test.svh | 6 +- .../balls/smatmul/workloads/isa/smatmul.h | 6 + .../ball/smatmul_bias_accumulate.mlir | 20 +- .../balls/toint8/arch/ToInt8Ball.scala | 33 +- .../components/balls/toint8/emu/src/quant.rs | 28 +- .../balls/toint8/stack/src/Lowering.cpp | 22 +- .../toint8/verify/casegen/src/casegen.rs | 93 ++- .../balls/toint8/verify/casegen/src/lib.rs | 4 +- .../toint8/verify/filelists/toint8_ball.f | 4 +- .../toint8/verify/src/common/toint8_defs.svh | 19 +- .../toint8/verify/src/common/toint8_items.svh | 71 +- .../toint8/verify/src/cov/toint8_cov.svh | 6 +- .../verify/src/env/toint8_scoreboard.svh | 49 +- .../verify/src/seq/toint8_sequences.svh | 3 +- .../balls/toint8/verify/src/tb_top.sv | 127 ++-- .../toint8/verify/src/tests/toint8_test.svh | 6 +- .../balls/toint8/workloads/isa/quant.h | 7 + .../mlir_tests/ball/quant_f32_to_i8.mlir | 10 +- .../mlir_tests/ball/quant_i32_to_i8.mlir | 27 +- .../mlir_tests/ball/quant_i32_to_i8_main.cpp | 9 +- .../mlir_tests/bank/quant_i32_pack16.mlir.in | 2 +- .../balls/transpose/verify/README.md | 33 +- .../transpose/verify/casegen/src/casegen.rs | 67 +- .../balls/transpose/verify/casegen/src/lib.rs | 4 +- .../verify/filelists/transpose_ball.f | 2 + .../verify/src/common/transpose_defs.svh | 18 +- .../verify/src/common/transpose_items.svh | 56 +- .../verify/src/cov/transpose_cov.svh | 5 +- .../verify/src/env/transpose_scoreboard.svh | 48 +- .../verify/src/seq/transpose_sequences.svh | 3 +- .../balls/transpose/verify/src/tb_top.sv | 125 ++-- .../verify/src/tests/transpose_test.svh | 22 +- .../mlir_tests/ball/transpose_16x16_i8.mlir | 10 +- .../device/workloads/kernel/CMakeLists.txt | 10 +- .../regression/bemu/workloads-bare.toml | 2 + .../pebble/sims/device/tapeout/config.toml | 2 +- .../pebble/stack/midend/CMakeLists.txt | 4 - .../midend/src/ConvertLinalgToBuckyball.cpp | 292 ++++---- .../pebble/stack/midend/src/FormCustomOps.cpp | 105 +-- .../stack/midend/src/FormSpatialOps.cpp | 75 +- .../stack/midend/src/LowerLinearTiles.cpp | 372 +++++----- .../stack/midend/src/LowerQuantizedOps.cpp | 349 +++++----- .../stack/midend/src/LowerSpatialOps.cpp | 646 ++++++++++-------- .../pebble/stack/midend/src/Lowering.h | 37 +- .../stack/midend/src/QuantizedChains.cpp | 453 ------------ .../stack/midend/tests/form-linear-chain.mlir | 83 --- .../midend/tests/form-quantized-chain.mlir | 19 - .../midend/tests/lower-bank-updates.mlir | 9 +- .../stack/midend/tests/lower-custom-ops.mlir | 10 +- .../regression/bemu/workloads-bare.toml | 3 + .../midend/src/ConvertLinalgToBuckyball.cpp | 214 +++--- .../toy/stack/midend/src/FormCustomOps.cpp | 108 +++ .../core/acc/mlir_tests/linalg/CMakeLists.txt | 57 +- .../mlir_tests/linalg/conv2d_nchw_fchw.mlir | 85 ++- .../linalg/conv2d_nchw_fchw_main.cpp | 54 +- .../mlir_tests/linalg/conv2d_nhwc_fhwc.mlir | 85 ++- .../linalg/conv2d_nhwc_fhwc_main.cpp | 54 +- .../mlir_tests/linalg/conv2d_nhwc_hwcf.mlir | 85 ++- .../linalg/conv2d_nhwc_hwcf_deep.mlir | 70 ++ .../linalg/conv2d_nhwc_hwcf_deep_main.cpp | 53 ++ .../linalg/conv2d_nhwc_hwcf_main.cpp | 54 +- stack | 2 +- thirdparty/soc-framework | 2 +- verify | 2 +- 260 files changed, 4958 insertions(+), 5751 deletions(-) create mode 100644 bb-tests/workloads/src/MLIRTest/base/base_bank_loop_carried.mlir create mode 100644 bb-tests/workloads/src/MLIRTest/base/base_bank_loop_carried_main.cpp create mode 100644 bb-tests/workloads/src/MLIRTest/base/base_bank_transfer.mlir create mode 100644 bb-tests/workloads/src/MLIRTest/base/base_bank_transfer_main.cpp delete mode 100644 bbdev/api/steps/ip/scripts/macro_compiler.py create mode 100644 examples/components/balls/gemmini/verify/src/gemmini_expansion_if.sv create mode 100644 examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir delete mode 100644 examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir.in create mode 100644 examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir delete mode 100644 examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir.in delete mode 100644 examples/components/balls/im2col/arch/Im2colConfigRegs.scala delete mode 100644 examples/components/balls/im2col/arch/Im2colWindow.scala delete mode 100644 examples/components/balls/im2col/arch/LineBufferManager.scala delete mode 100644 examples/components/balls/im2col/arch/StreamWriter.scala delete mode 100644 examples/designs/pebble/stack/midend/src/QuantizedChains.cpp delete mode 100644 examples/designs/pebble/stack/midend/tests/form-linear-chain.mlir delete mode 100644 examples/designs/pebble/stack/midend/tests/form-quantized-chain.mlir create mode 100644 examples/designs/toy/stack/midend/src/FormCustomOps.cpp create mode 100644 examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_deep.mlir create mode 100644 examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_deep_main.cpp diff --git a/.gitignore b/.gitignore index d1e4052de..751857d60 100644 --- a/.gitignore +++ b/.gitignore @@ -71,4 +71,4 @@ skills-lock.json ucli.key cm.log tr_db.log -waste-bake/ +waste/ diff --git a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/FALU.scala b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/FALU.scala index fd48ad896..2a9a8a3c0 100644 --- a/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/FALU.scala +++ b/arch/src/main/scala/framework/acc/gpdomain/src/main/scala/execution/src/main/scala/arithmetic/FALU.scala @@ -1,7 +1,5 @@ package framework.acc.gpdomain.execution.arithmetic -import chisel3.experimental.hierarchy.{Instance, Instantiate} - import framework.acc.gpdomain.configs.GpDomainParam import chisel3._ @@ -51,29 +49,29 @@ class FALU(gpParam: GpDomainParam) extends Module { val a = recFNFromFN(expWidth, sigWidth, av) val b = recFNFromFN(expWidth, sigWidth, bv) val c = recFNFromFN(expWidth, sigWidth, cv) - val add = Instantiate(new AddRecFN(expWidth, sigWidth)) + val add = Module(new AddRecFN(expWidth, sigWidth)) add.io.a := Mux(io.op === 39.U, b, a) add.io.b := Mux(io.op === 39.U, a, b) add.io.subOp := io.op === 2.U || io.op === 39.U add.io.roundingMode := io.roundingMode add.io.detectTininess := 1.U - val mul = Instantiate(new MulRecFN(expWidth, sigWidth)) + val mul = Module(new MulRecFN(expWidth, sigWidth)) mul.io.a := a mul.io.b := b mul.io.roundingMode := io.roundingMode mul.io.detectTininess := 1.U - val fma = Instantiate(new MulAddRecFN(expWidth, sigWidth)) + val fma = Module(new MulAddRecFN(expWidth, sigWidth)) fma.io.a := b fma.io.b := Mux(io.op(2), a, c) fma.io.c := Mux(io.op(2), c, a) fma.io.op := Cat(io.op(0), io.op(1) ^ io.op(0)) fma.io.roundingMode := io.roundingMode fma.io.detectTininess := 1.U - val cmp = Instantiate(new CompareRecFN(expWidth, sigWidth)) + val cmp = Module(new CompareRecFN(expWidth, sigWidth)) cmp.io.a := a cmp.io.b := b cmp.io.signaling := Seq(25, 27, 29, 31).map(n => io.op === n.U).reduce(_ || _) - val div = Instantiate(new DivSqrtRecFN_small(expWidth, sigWidth, 0)) + val div = Module(new DivSqrtRecFN_small(expWidth, sigWidth, 0)) div.io.a := Mux(io.op === 33.U, b, a) div.io.b := Mux(io.op === 33.U, a, b) div.io.sqrtOp := squareRoot @@ -230,11 +228,11 @@ class FALU(gpParam: GpDomainParam) extends Module { val (se, ss) = if (srcWidth == 32) (8, 24) else (11, 53) val (de, ds) = if (dstWidth == 32) (8, 24) else (11, 53) val matchesWidth = io.sew === (if (baseSelector != 0 || srcWidth == 32) 2 else 3).U - val floatToInt = Instantiate(new RecFNToIN(se, ss, dstWidth)) + val floatToInt = Module(new RecFNToIN(se, ss, dstWidth)) floatToInt.io.in := recFNFromFN(se, ss, io.a(srcWidth - 1, 0)) floatToInt.io.signedOut := io.subop(0) floatToInt.io.roundingMode := Mux(io.subop(2, 0) >= 6.U, 1.U, io.roundingMode) - val intToFloat = Instantiate(new INToRecFN(srcWidth, de, ds)) + val intToFloat = Module(new INToRecFN(srcWidth, de, ds)) intToFloat.io.in := io.a(srcWidth - 1, 0) intToFloat.io.signedIn := io.subop(0) intToFloat.io.roundingMode := io.roundingMode @@ -251,7 +249,7 @@ class FALU(gpParam: GpDomainParam) extends Module { io.flags := intToFloat.io.exceptionFlags } if (baseSelector != 0) { - val convert = Instantiate(new RecFNToRecFN(se, ss, de, ds)) + val convert = Module(new RecFNToRecFN(se, ss, de, ds)) convert.io.in := recFNFromFN(se, ss, io.a(srcWidth - 1, 0)) convert.io.roundingMode := Mux(io.subop === 21.U, 6.U, io.roundingMode) convert.io.detectTininess := 1.U diff --git a/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/FPU.scala b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/FPU.scala index 827e90c4d..8d05c6bb0 100644 --- a/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/FPU.scala +++ b/arch/src/main/scala/framework/core/execution/fpu/src/main/scala/FPU.scala @@ -132,7 +132,7 @@ class FPU(b: ModuleParam) extends Module { arithmeticFlags(index) := fma.io.exceptionFlags arithmeticValid(index) := fma.io.validout - val div = Instantiate(new hardfloat.DivSqrtRecFN_small(exp, sig, 0)) + val div = Module(new hardfloat.DivSqrtRecFN_small(exp, sig, 0)) div.io.inValid := start && isDivide div.io.sqrtOp := operation === 11.U div.io.a := a @@ -149,7 +149,7 @@ class FPU(b: ModuleParam) extends Module { divideValid(index) := div.io.outValid_div || div.io.outValid_sqrt divideReady(index) := div.io.inReady - val compare = Instantiate(new hardfloat.CompareRecFN(exp, sig)) + val compare = Module(new hardfloat.CompareRecFN(exp, sig)) compare.io.a := a compare.io.b := b compare.io.signaling := funct3 =/= 2.U @@ -168,7 +168,7 @@ class FPU(b: ModuleParam) extends Module { val injected = Cat(injectedSign, rawA(width - 2, 0)) val intInput = Mux(rs2(1), request.integer, Cat(Fill(32, !rs2(0) && request.integer(31)), request.integer(31, 0))) - val fromInt = Instantiate(new hardfloat.INToRecFN(64, exp, sig)) + val fromInt = Module(new hardfloat.INToRecFN(64, exp, sig)) fromInt.io.in := intInput fromInt.io.signedIn := !rs2(0) fromInt.io.roundingMode := rounding @@ -177,7 +177,7 @@ class FPU(b: ModuleParam) extends Module { val convertedIntegers = Wire(Vec(2, UInt(64.W))) val convertedFlags = Wire(Vec(2, UInt(5.W))) for ((intWidth, intIndex) <- Seq(32, 64).zipWithIndex) { - val toInt = Instantiate(new hardfloat.RecFNToIN(exp, sig, intWidth)) + val toInt = Module(new hardfloat.RecFNToIN(exp, sig, intWidth)) toInt.io.in := a toInt.io.signedOut := !rs2(0) toInt.io.roundingMode := rounding @@ -190,7 +190,7 @@ class FPU(b: ModuleParam) extends Module { if (single) request.a else Mux(request.a(63, 32).andR, request.a(31, 0), "h7fc00000".U(32.W)) - val convert = Instantiate(new hardfloat.RecFNToRecFN(sourceExp, sourceSig, exp, sig)) + val convert = Module(new hardfloat.RecFNToRecFN(sourceExp, sourceSig, exp, sig)) convert.io.in := hardfloat.recFNFromFN(sourceExp, sourceSig, source) convert.io.roundingMode := rounding convert.io.detectTininess := 1.U diff --git a/arch/src/main/scala/framework/lib/arith/Fma.scala b/arch/src/main/scala/framework/lib/arith/Fma.scala index 8c1947c0e..6f2a765d1 100644 --- a/arch/src/main/scala/framework/lib/arith/Fma.scala +++ b/arch/src/main/scala/framework/lib/arith/Fma.scala @@ -29,8 +29,8 @@ class Fma(latency: Int, expWidth: Int, sigWidth: Int) extends Module { //------------------------------------------------------------------------ //------------------------------------------------------------------------ - val mulAddRecFNToRaw_preMul = Instantiate(new hardfloat.MulAddRecFNToRaw_preMul(expWidth, sigWidth)) - val mulAddRecFNToRaw_postMul = Instantiate(new hardfloat.MulAddRecFNToRaw_postMul(expWidth, sigWidth)) + val mulAddRecFNToRaw_preMul = Module(new hardfloat.MulAddRecFNToRaw_preMul(expWidth, sigWidth)) + val mulAddRecFNToRaw_postMul = Module(new hardfloat.MulAddRecFNToRaw_postMul(expWidth, sigWidth)) mulAddRecFNToRaw_preMul.io.op := io.op mulAddRecFNToRaw_preMul.io.a := io.a @@ -73,7 +73,7 @@ class Fma(latency: Int, expWidth: Int, sigWidth: Int) extends Module { //------------------------------------------------------------------------ //------------------------------------------------------------------------ - val roundRawFNToRecFN = Instantiate(new hardfloat.RoundRawFNToRecFN(expWidth, sigWidth, 0)) + val roundRawFNToRecFN = Module(new hardfloat.RoundRawFNToRecFN(expWidth, sigWidth, 0)) val round_regs = if (latency >= 2) 1 else 0 roundRawFNToRecFN.io.invalidExc := Pipe(valid_stage0, mulAddRecFNToRaw_postMul.io.invalidExc, round_regs).bits diff --git a/arch/src/main/scala/framework/lib/arith/IntToFloat.scala b/arch/src/main/scala/framework/lib/arith/IntToFloat.scala index 8a9f08dbf..dc02170cd 100644 --- a/arch/src/main/scala/framework/lib/arith/IntToFloat.scala +++ b/arch/src/main/scala/framework/lib/arith/IntToFloat.scala @@ -89,7 +89,7 @@ class IntToFloat(intWidth: Int, expWidth: Int, sigWidth: Int) extends Module { val roundDetectTininess = RegEnable(midDetectTininess, midValid) val roundValid = RegNext(midValid, false.B) - val roundAnyRawFNToRecFN = Instantiate(new hardfloat.RoundAnyRawFNToRecFN( + val roundAnyRawFNToRecFN = Module(new hardfloat.RoundAnyRawFNToRecFN( rawExpWidth, intWidth, expWidth, diff --git a/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt b/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt index 4a96f2676..3ec3ceb77 100644 --- a/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt +++ b/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt @@ -10,4 +10,6 @@ foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) add_buckyball_mlir_test(base_memref_copy_dynamic TARGET ${BUCKYBALL_TARGET}) add_buckyball_mlir_test(base_mvin_mvout_16x16_i8 TARGET ${BUCKYBALL_TARGET}) add_buckyball_mlir_test(base_bank_mvin_mvout_16x16_i8 TARGET ${BUCKYBALL_TARGET}) + add_buckyball_mlir_test(base_bank_loop_carried TARGET ${BUCKYBALL_TARGET}) + add_buckyball_mlir_test(base_bank_transfer TARGET ${BUCKYBALL_TARGET}) endforeach() diff --git a/bb-tests/workloads/src/MLIRTest/base/base_bank_loop_carried.mlir b/bb-tests/workloads/src/MLIRTest/base/base_bank_loop_carried.mlir new file mode 100644 index 000000000..25c83be49 --- /dev/null +++ b/bb-tests/workloads/src/MLIRTest/base/base_bank_loop_carried.mlir @@ -0,0 +1,41 @@ +func.func private @check_result(memref<64x16xi8>) -> () + +// One bank carried through a loop: each iteration moves 16 rows through it. +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c16 = arith.constant 16 : index + %c64 = arith.constant 64 : index + %input = memref.alloc() alignment = 64 : memref<64x16xi8> + %output = memref.alloc() alignment = 64 : memref<64x16xi8> + + scf.for %i = %c0 to %c64 step %c1 { + scf.for %j = %c0 to %c16 step %c1 { + %ii = arith.index_cast %i : index to i32 + %jj = arith.index_cast %j : index to i32 + %sum = arith.addi %ii, %jj : i32 + %value = arith.trunci %sum : i32 to i8 + memref.store %value, %input[%i, %j] : memref<64x16xi8> + } + } + + %bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %last = scf.for %row = %c0 to %c64 step %c16 iter_args(%carried = %bank) -> (i64) { + %in = memref.subview %input[%row, 0] [16, 16] [1, 1] + : memref<64x16xi8> to memref<16x16xi8, strided<[16, 1], offset: ?>> + %out = memref.subview %output[%row, 0] [16, 16] [1, 1] + : memref<64x16xi8> to memref<16x16xi8, strided<[16, 1], offset: ?>> + %loaded = buckyball.bank_mvin %in, %carried <{depth = 16 : i64, stride = 1 : i64}> + : memref<16x16xi8, strided<[16, 1], offset: ?>> + %stored = buckyball.bank_mvout %out, %loaded <{depth = 16 : i64, stride = 1 : i64}> + : memref<16x16xi8, strided<[16, 1], offset: ?>> + scf.yield %stored : i64 + } + buckyball.bank_release %last + + func.call @check_result(%output) : (memref<64x16xi8>) -> () + memref.dealloc %input : memref<64x16xi8> + memref.dealloc %output : memref<64x16xi8> + return %zero_i8 : i8 +} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_bank_loop_carried_main.cpp b/bb-tests/workloads/src/MLIRTest/base/base_bank_loop_carried_main.cpp new file mode 100644 index 000000000..c4f4eb33d --- /dev/null +++ b/bb-tests/workloads/src/MLIRTest/base/base_bank_loop_carried_main.cpp @@ -0,0 +1,35 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + volatile uint32_t *sim_exit = (volatile uint32_t *)0x60000000; + *sim_exit = 1; + while (1) { + } +#else + exit(1); +#endif +} + +#ifdef __cplusplus +extern "C" +#endif + void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, + int64_t size0, int64_t size1, int64_t stride0, + int64_t stride1) { + (void)allocated; + if (size0 != 64 || size1 != 16 || stride0 != 16 || stride1 != 1) { + fail(); + } + int8_t *out = aligned + offset; + for (int i = 0; i < 64; ++i) { + for (int j = 0; j < 16; ++j) { + if (out[i * stride0 + j * stride1] != (int8_t)(i + j)) { + fail(); + } + } + } + printf("PASSED: base bank carried through a loop\n"); +} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_bank_transfer.mlir b/bb-tests/workloads/src/MLIRTest/base/base_bank_transfer.mlir new file mode 100644 index 000000000..4d8504dd7 --- /dev/null +++ b/bb-tests/workloads/src/MLIRTest/base/base_bank_transfer.mlir @@ -0,0 +1,50 @@ +func.func private @check_result(memref<2x48xi8>) -> () + +// The groups of a two-column bank appended to a one-column bank. +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c16 = arith.constant 16 : index + %c32 = arith.constant 32 : index + %first = memref.alloc() alignment = 64 : memref<2x16xi8> + %second = memref.alloc() alignment = 64 : memref<2x32xi8> + %result = memref.alloc() alignment = 64 : memref<2x48xi8> + + scf.for %row = %c0 to %c2 step %c1 { + scf.for %column = %c0 to %c32 step %c1 { + %r = arith.index_cast %row : index to i32 + %c = arith.index_cast %column : index to i32 + %c32_i32 = arith.constant 32 : i32 + %offset = arith.muli %r, %c32_i32 : i32 + %value = arith.addi %offset, %c : i32 + %b = arith.trunci %value : i32 to i8 + %high = arith.constant 64 : i8 + %second_value = arith.addi %b, %high : i8 + memref.store %second_value, %second[%row, %column] : memref<2x32xi8> + %in_first = arith.cmpi ult, %column, %c16 : index + scf.if %in_first { + %c16_i32 = arith.constant 16 : i32 + %first_offset = arith.muli %r, %c16_i32 : i32 + %first_value = arith.addi %first_offset, %c : i32 + %a = arith.trunci %first_value : i32 to i8 + memref.store %a, %first[%row, %column] : memref<2x16xi8> + } + } + } + + %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b = buckyball.bank_alloc <{row = 1 : i64, col = 2 : i64}> + %la = buckyball.bank_mvin %first, %a <{depth = 2 : i64, stride = 1 : i64}> : memref<2x16xi8> + %lb = buckyball.bank_mvin %second, %b <{depth = 2 : i64, stride = 1 : i64}> : memref<2x32xi8> + %merged = buckyball.bank_transfer %lb, %la + %stored = buckyball.bank_mvout %result, %merged <{depth = 2 : i64, stride = 1 : i64}> : memref<2x48xi8> + buckyball.bank_release %stored + + func.call @check_result(%result) : (memref<2x48xi8>) -> () + memref.dealloc %first : memref<2x16xi8> + memref.dealloc %second : memref<2x32xi8> + memref.dealloc %result : memref<2x48xi8> + return %zero_i8 : i8 +} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_bank_transfer_main.cpp b/bb-tests/workloads/src/MLIRTest/base/base_bank_transfer_main.cpp new file mode 100644 index 000000000..42f45ab36 --- /dev/null +++ b/bb-tests/workloads/src/MLIRTest/base/base_bank_transfer_main.cpp @@ -0,0 +1,37 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + volatile uint32_t *sim_exit = (volatile uint32_t *)0x60000000; + *sim_exit = 1; + while (1) { + } +#else + exit(1); +#endif +} + +#ifdef __cplusplus +extern "C" +#endif + void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, + int64_t size0, int64_t size1, int64_t stride0, + int64_t stride1) { + (void)allocated; + if (size0 != 2 || size1 != 48 || stride0 != 48 || stride1 != 1) { + fail(); + } + int8_t *out = aligned + offset; + for (int i = 0; i < 2; ++i) { + for (int j = 0; j < 48; ++j) { + int8_t expected = + j < 16 ? (int8_t)(i * 16 + j) : (int8_t)(i * 32 + j - 16 + 64); + if (out[i * stride0 + j * stride1] != expected) { + fail(); + } + } + } + printf("PASSED: base bank_transfer appends groups\n"); +} diff --git a/bb-tests/workloads/src/MLIRTest/base/base_mvin_mvout_16x16_i8.mlir b/bb-tests/workloads/src/MLIRTest/base/base_mvin_mvout_16x16_i8.mlir index 17ef2a197..eff30e0ba 100644 --- a/bb-tests/workloads/src/MLIRTest/base/base_mvin_mvout_16x16_i8.mlir +++ b/bb-tests/workloads/src/MLIRTest/base/base_mvin_mvout_16x16_i8.mlir @@ -20,10 +20,10 @@ func.func @main() -> i8 { } } - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mvin %input <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - buckyball.mvout %output <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvout %output <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> func.call @check_result(%output) : (memref<16x16xi8>) -> () memref.dealloc %input : memref<16x16xi8> diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py index 5690ce0b7..30160fbf4 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_api.step.py @@ -1,6 +1,5 @@ from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api -from pathlib import Path from utils.event_common import require_design from utils.path import get_buckyball_path, rtl_dir @@ -33,17 +32,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: selection=selection, ) - stop_after = body.get("stop_after", body.get("stop-after")) - if stop_after is not None and stop_after not in ("vsyn", "vcom"): - return ApiResponse( - status=400, body={"error": "stop-after must be vsyn or vcom"} - ) output_dir = body.get("output_dir") or body.get("output-dir") - if stop_after and output_dir and Path(output_dir).exists(): - return ApiResponse( - status=400, - body={"error": "Resource assessment requires a fresh output_dir"}, - ) data = { "design": design, "diff": bool(body.get("diff", False)), @@ -51,7 +40,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "mtrace": bool(body.get("mtrace", False)), "vsrc_dir": vsrc_dir, "output_dir": output_dir, - "stop_after": stop_after, } await ctx.enqueue( { diff --git a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py index f3ce81aba..81fc1527d 100644 --- a/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py +++ b/bbdev/api/steps/bebop/p2e/02_buildbitstream_event.step.py @@ -60,7 +60,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: normalize_p2e_timescale(vsrc_dir, ctx.logger) timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M") diff = bool(input_data.get("diff", False)) - stop_after = input_data.get("stop_after") build_dir = input_data.get("output_dir") or str( Path(bbdir) / "examples/designs" @@ -72,10 +71,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: build_root = (Path(bbdir) / "examples/designs" / design / "output/p2e").resolve() if build_root not in build_path.parents: raise ValueError(f"P2E output_dir must be under {build_root}: {build_path}") - if stop_after and build_path.exists(): - raise ValueError( - f"Resource assessment requires a fresh output_dir: {build_path}" - ) if build_path.exists(): shutil.rmtree(build_path) build_path.mkdir(parents=True) @@ -134,7 +129,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: *(["--diff"] if diff else []), *(["--itrace"] if input_data.get("itrace", False) else []), *(["--mtrace"] if input_data.get("mtrace", False) else []), - *(["--stop-after", stop_after] if stop_after else []), ] ) build_cmd = f"cd {shlex.quote(bbdir)} && {build_cmd}" @@ -165,18 +159,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "diff": diff, "timestamp": timestamp, } - if stop_after: - extra_fields = { - "task": "resource-assessment", - "stop_after": stop_after, - "runtime_ready": False, - "vsrc_dir": vsrc_dir, - "build_dir": build_dir, - "diff": diff, - "timestamp": timestamp, - } - else: - extra_fields["runtime_ready"] = build_result.returncode == 0 + extra_fields["runtime_ready"] = build_result.returncode == 0 await check_result( ctx, build_result.returncode, diff --git a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py index 62e35bfcc..deb179a21 100644 --- a/bbdev/api/steps/bebop/verilator/01_clean_event.step.py +++ b/bbdev/api/steps/bebop/verilator/01_clean_event.step.py @@ -37,7 +37,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: build_dir = rtl_dir( bbdir, design, - "verilog", + "bebop", selection=input_data["target"], ) command = f"rm -rf {build_dir}" diff --git a/bbdev/api/steps/bebop/verilator/03_build_event.step.py b/bbdev/api/steps/bebop/verilator/03_build_event.step.py index 5739b5a98..e961afe2f 100644 --- a/bbdev/api/steps/bebop/verilator/03_build_event.step.py +++ b/bbdev/api/steps/bebop/verilator/03_build_event.step.py @@ -42,7 +42,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: vsrc_dir = rtl_dir( bbdir, design, - "verilog", + "bebop", selection=input_data["target"], ) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") diff --git a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py index 57d13c69b..ecccbf291 100644 --- a/bbdev/api/steps/bebop/verilator/04_sim_event.step.py +++ b/bbdev/api/steps/bebop/verilator/04_sim_event.step.py @@ -64,7 +64,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: vsrc_dir = rtl_dir( bbdir, design, - "verilog", + "bebop", selection=input_data["target"], ) ctx.logger.info(f"Using verilog source directory: {vsrc_dir}") diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index 3ed3f88a4..b7cfd3d7e 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -89,7 +89,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: vsrc_dir = rtl_dir( bbdir, design, - "verilog", + "bebop", selection=input_data["target"], ) manifest = str(runner_manifest(bbdir, design, input_data["target"])) diff --git a/bbdev/api/steps/dc/01_area_api.step.py b/bbdev/api/steps/dc/01_area_api.step.py index 6fa48d211..935c99cf9 100644 --- a/bbdev/api/steps/dc/01_area_api.step.py +++ b/bbdev/api/steps/dc/01_area_api.step.py @@ -24,7 +24,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: "topic": "dc.verilog", "data": { "design": design, - "top": body.get("top") or "DigitalTop", + "top": body.get("top"), "from_area_workflow": True, "_trace_id": ctx.trace_id, "target": selection, diff --git a/bbdev/api/steps/dc/01_area_event.step.py b/bbdev/api/steps/dc/01_area_event.step.py index 3d7c7dcaf..a2a3516ec 100644 --- a/bbdev/api/steps/dc/01_area_event.step.py +++ b/bbdev/api/steps/dc/01_area_event.step.py @@ -36,7 +36,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) source_list_path = input_data.get("source_list") - top_module = input_data.get("top") or "DigitalTop" + top_module = input_data.get("top") analysis_dir = input_data.get("analysis_dir") if not isinstance(source_list_path, str) or not os.path.isfile(source_list_path): await check_result( @@ -79,6 +79,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: top_module, selection=input_data["target"], ) + top_module = contract.top_module except (OSError, ValueError) as exc: await check_result( ctx, diff --git a/bbdev/api/steps/dc/02_verilog_api.step.py b/bbdev/api/steps/dc/02_verilog_api.step.py index 0556ece91..0aa5a761c 100644 --- a/bbdev/api/steps/dc/02_verilog_api.step.py +++ b/bbdev/api/steps/dc/02_verilog_api.step.py @@ -24,7 +24,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: "topic": "dc.verilog", "data": { "design": design, - "top": body.get("top") or "DigitalTop", + "top": body.get("top"), "_trace_id": ctx.trace_id, "target": selection, }, diff --git a/bbdev/api/steps/dc/03_sim_event.step.py b/bbdev/api/steps/dc/03_sim_event.step.py index 7d55dd80d..19257914f 100644 --- a/bbdev/api/steps/dc/03_sim_event.step.py +++ b/bbdev/api/steps/dc/03_sim_event.step.py @@ -40,7 +40,7 @@ def _first(data: dict, *names: str): async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) design = input_data.get("design") - top_module = input_data.get("top") or "DigitalTop" + top_module = input_data.get("top") analysis_dir = input_data.get("analysis_dir") if not isinstance(analysis_dir, str) or not analysis_dir: await check_result( @@ -59,6 +59,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: top_module, selection=input_data["target"], ) + top_module = contract.top_module except (OSError, ValueError) as exc: await check_result( ctx, diff --git a/bbdev/api/steps/dc/04_power_api.step.py b/bbdev/api/steps/dc/04_power_api.step.py index f730dfd12..80be9c2e0 100644 --- a/bbdev/api/steps/dc/04_power_api.step.py +++ b/bbdev/api/steps/dc/04_power_api.step.py @@ -25,7 +25,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: "data": { **body, "design": design, - "top": body.get("top") or "DigitalTop", + "top": body.get("top"), "from_power_workflow": True, "_trace_id": ctx.trace_id, "target": selection, diff --git a/bbdev/api/steps/ip/01_generate_api.step.py b/bbdev/api/steps/ip/01_generate_api.step.py index 561971245..849cde514 100644 --- a/bbdev/api/steps/ip/01_generate_api.step.py +++ b/bbdev/api/steps/ip/01_generate_api.step.py @@ -10,7 +10,7 @@ def req_arg(body: dict, name: str): config = { "name": "ip-generate-api", - "description": "generate SRAM macros from elaborator mems.conf via MacroCompiler", + "description": "generate SRAM macro wrappers from elaborator mems.conf via the process sram function", "flows": ["ip", "dc", "yosys"], "triggers": [api("POST", "/ip/generate")], "enqueues": ["ip.generate"], @@ -31,7 +31,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "design": design, "consumer": req_arg(body, "consumer") or "dc", - "top": req_arg(body, "top") or "DigitalTop", + "top": req_arg(body, "top"), } if data["consumer"] not in ("dc", "yosys"): return ApiResponse(status=400, body={"error": "consumer must be dc or yosys"}) diff --git a/bbdev/api/steps/ip/01_generate_event.step.py b/bbdev/api/steps/ip/01_generate_event.step.py index c662f65c5..10487f1aa 100644 --- a/bbdev/api/steps/ip/01_generate_event.step.py +++ b/bbdev/api/steps/ip/01_generate_event.step.py @@ -2,6 +2,8 @@ import sys from pathlib import Path +import yaml + from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) @@ -14,11 +16,12 @@ from utils.event_common import check_result, get_origin_trace_id, require_design from utils.path import get_buckyball_path, rtl_dir +from steps.dc.scripts.tapeout import get_tapeout_contract from generate import generate_sram config = { "name": "ip-generate", - "description": "generate SRAM macros from elaborator mems.conf via MacroCompiler", + "description": "generate SRAM macro wrappers from elaborator mems.conf via the process sram function", "flows": ["ip", "dc", "yosys"], "triggers": [queue("ip.generate")], "enqueues": ["ip.replace"], @@ -32,7 +35,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: consumer = input_data.get("consumer") or "dc" if consumer not in ("dc", "yosys"): raise ValueError("consumer must be dc or yosys") - top = input_data.get("top") or "DigitalTop" + # The synthesis top: --top, else the design's tapeout config (dc) or the yosys config. + top = input_data.get("top") or ( + get_tapeout_contract( + get_buckyball_path(), design, None, selection=input_data["target"] + ).top_module + if consumer == "dc" + else yaml.safe_load( + ( + Path(__file__).parents[1] / "yosys/scripts/yosys-config.yaml" + ).read_text() + )["top"] + ) if not isinstance(top, str) or not top: raise ValueError("top must be a non-empty string") bbdir = Path(get_buckyball_path()).resolve() @@ -40,7 +54,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: rtl_dir( bbdir, design, - "tapeout" if consumer == "dc" else "verilog", + "dc" if consumer == "dc" else "synth", input_data.get("output_dir"), selection=input_data["target"], ) diff --git a/bbdev/api/steps/ip/02_replace_api.step.py b/bbdev/api/steps/ip/02_replace_api.step.py index ae59b95f0..b3196aadf 100644 --- a/bbdev/api/steps/ip/02_replace_api.step.py +++ b/bbdev/api/steps/ip/02_replace_api.step.py @@ -31,7 +31,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "design": design, "consumer": req_arg(body, "consumer") or "dc", - "top": req_arg(body, "top") or "DigitalTop", + "top": req_arg(body, "top"), } if data["consumer"] not in ("dc", "yosys"): return ApiResponse(status=400, body={"error": "consumer must be dc or yosys"}) diff --git a/bbdev/api/steps/ip/02_replace_event.step.py b/bbdev/api/steps/ip/02_replace_event.step.py index e5335a667..484b1d366 100644 --- a/bbdev/api/steps/ip/02_replace_event.step.py +++ b/bbdev/api/steps/ip/02_replace_event.step.py @@ -2,6 +2,8 @@ import sys from pathlib import Path +import yaml + from motia import FlowContext, queue utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) @@ -14,6 +16,7 @@ from utils.event_common import check_result, get_origin_trace_id, require_design from utils.path import get_buckyball_path, rtl_dir +from steps.dc.scripts.tapeout import get_tapeout_contract from replace import replace_sources config = { @@ -32,7 +35,18 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: consumer = input_data.get("consumer") or "dc" if consumer not in {"dc", "yosys"}: raise ValueError("consumer must be dc or yosys") - top_module = input_data.get("top") or "DigitalTop" + # The synthesis top: --top, else the design's tapeout config (dc) or the yosys config. + top_module = input_data.get("top") or ( + get_tapeout_contract( + get_buckyball_path(), design, None, selection=input_data["target"] + ).top_module + if consumer == "dc" + else yaml.safe_load( + ( + Path(__file__).parents[1] / "yosys/scripts/yosys-config.yaml" + ).read_text() + )["top"] + ) if not isinstance(top_module, str) or not top_module: raise ValueError("top must be a non-empty string") @@ -41,7 +55,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: rtl_dir( bbdir, design, - "tapeout" if consumer == "dc" else "verilog", + "dc" if consumer == "dc" else "synth", input_data.get("output_dir"), selection=input_data["target"], ) diff --git a/bbdev/api/steps/ip/scripts/generate.py b/bbdev/api/steps/ip/scripts/generate.py index 1b5d0a5db..d83939046 100644 --- a/bbdev/api/steps/ip/scripts/generate.py +++ b/bbdev/api/steps/ip/scripts/generate.py @@ -1,29 +1,15 @@ from __future__ import annotations import json -import re import sys -from collections import Counter from pathlib import Path if str(Path(__file__).resolve().parents[2] / "dc" / "scripts") not in sys.path: sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "dc" / "scripts")) from tapeout import SramGeom, get_tapeout_contract -from macro_compiler import run_macro_compiler from replace import source_scope -from sram_compiler import generate_sram_dbs, leaf_names_from_macros - - -def pad_mems_conf(text: str) -> str: - lines = [] - for raw in text.splitlines(): - if not raw.strip(): - continue - lines.append(raw.rstrip() + " ") - if not lines: - raise ValueError("empty mems.conf") - return "\n".join(lines) + "\n" +from sram_compiler import generate_sram_dbs, generate_sram_macros def load_mdf(mdf: Path) -> dict[str, tuple[int, int]]: @@ -50,106 +36,13 @@ def load_mdf(mdf: Path) -> dict[str, tuple[int, int]]: return index -def parse_mems(text: str) -> list[tuple[str, int, int]]: - out: list[tuple[str, int, int]] = [] - for i, raw in enumerate(text.splitlines(), 1): - line = raw.strip() - if not line: - continue - parts = line.split() - if ( - len(parts) < 6 - or parts[0] != "name" - or parts[2] != "depth" - or parts[4] != "width" - ): - raise ValueError( - f"mems.conf:{i}: expected 'name depth width ...', got {line!r}" - ) - try: - depth = int(parts[3]) - width = int(parts[5]) - except ValueError as exc: - raise ValueError(f"mems.conf:{i}: depth/width not int") from exc - if depth <= 0 or width <= 0: - raise ValueError(f"mems.conf:{i}: depth/width must be positive") - out.append((parts[1], depth, width)) - if not out: - raise ValueError("empty mems.conf") - return out - - -def module_insts(text: str) -> dict[str, Counter[str]]: - hits = list(re.finditer(r"(?m)^module\s+(\w+)\s*\(", text)) - out: dict[str, Counter[str]] = {} - for i, m in enumerate(hits): - name = m.group(1) - if name in out: - raise ValueError(f"duplicate module {name}") - start = m.end() - end = hits[i + 1].start() if i + 1 < len(hits) else len(text) - counts: Counter[str] = Counter() - for inst in re.finditer( - r"(?m)^\s*([A-Za-z_][A-Za-z0-9_$]*)\s+[A-Za-z_][A-Za-z0-9_$]*\s*\(", - text[start:end], - ): - leaf = inst.group(1) - if leaf in ("module", "assign", "wire", "input", "output"): - raise ValueError(f"module {name}: refused instance parse {leaf!r}") - counts[leaf] += 1 - out[name] = counts - return out - - -def check_mapping( - *, mems_text: str, macros_v: Path, mdf: dict[str, tuple[int, int]] -) -> None: - if not macros_v.is_file(): - raise FileNotFoundError(f"missing macros verilog: {macros_v}") - insts = module_insts(macros_v.read_text()) - lib = ", ".join(f"{n}={d}x{w}" for n, (d, w) in sorted(mdf.items())) - errs: list[str] = [] - for name, depth, width in parse_mems(mems_text): - need = depth * width - if name not in insts: - errs.append(f"{name}: missing module for {depth}x{width} ({need} bits)") - continue - counts = insts[name] - if not counts: - errs.append(f"{name}: empty mapping for {depth}x{width} ({need} bits)") - continue - got = 0 - parts = [] - unknown = False - for leaf, n in sorted(counts.items()): - if leaf not in mdf: - errs.append(f"{name}: unknown leaf {leaf}") - unknown = True - break - d, w = mdf[leaf] - got += n * d * w - parts.append(f"{n}x {leaf} ({d}x{w})") - if unknown: - continue - if got != need: - errs.append( - f"{name}: need {depth}x{width} = {need} bits, " - f"got {' + '.join(parts)} = {got} bits" - ) - if errs: - raise RuntimeError( - "SRAM mapping mismatch:\n " + "\n ".join(errs) + f"\nlibrary: {lib}" - ) - - -def geoms_from_macros( +def geoms_from_names( *, - macros_v: Path, + names: list[str], sram_table: dict[str, SramGeom], mdf: Path, ) -> list[SramGeom]: index = load_mdf(mdf) - names = leaf_names_from_macros(macros_v, set(sram_table) | set(index)) geoms: list[SramGeom] = [] for name in names: if name not in sram_table: @@ -171,7 +64,7 @@ def geoms_from_macros( def check_bitwrite_contract( - geoms: list[SramGeom], mdf: Path, macros_v: Path, sram_table: dict[str, SramGeom] + geoms: list[SramGeom], mdf: Path, sram_table: dict[str, SramGeom] ) -> None: raw = json.loads(mdf.read_text()) if not isinstance(raw, list): @@ -211,12 +104,13 @@ def check_bitwrite_contract( for geom in sram_table.values(): if geom.bitwrite and geom.name not in used: raise RuntimeError( - f"bitwrite sram {geom.name} not instantiated in {macros_v}" + f"bitwrite sram {geom.name} not instantiated by any memory" ) -def _leaf_paths(cache_dir: Path, name: str, corner: str) -> dict[str, str]: - leaf = cache_dir / name +def _leaf_paths(cache_dir: Path, g: SramGeom, corner: str) -> dict[str, str]: + name = g.name + leaf = cache_dir / f"{name}-mux{g.mux}" return { "v": str((leaf / f"{name}.v").resolve()), "lib": str((leaf / f"{name}_{corner}.lib").resolve()), @@ -240,14 +134,12 @@ def generate_sram( if not src.is_file(): raise FileNotFoundError(f"missing elaborator mems.conf: {src}") mems = src.read_text() - parse_mems(mems) reachable, _ = source_scope(Path(source_list).read_text().splitlines(), top_module) - selected = "\n".join( - line + selected = "".join( + line + "\n" for line in mems.splitlines() if line.strip() and line.split()[1] in reachable ) - padded = pad_mems_conf(selected) if selected else "" contract = get_tapeout_contract(bbdir, design, selection=selection) if out_dir is None: dest = build_dir / "ip-generate" @@ -258,31 +150,20 @@ def generate_sram( dest.mkdir(parents=True, exist_ok=True) mems_out = dest / "mems.conf" verilog = dest / "sram_macros.v" - firrtl = dest / "sram_macros.fir" - mems_out.write_text(padded) - arch_dir = bbdir / "arch" - if padded: - run_macro_compiler( - mems_conf=mems_out, - mdf=contract.sram_mdf, - verilog=verilog, - firrtl=firrtl, - arch_dir=arch_dir, - ) - check_mapping( - mems_text=padded, - macros_v=verilog, - mdf=load_mdf(contract.sram_mdf), + mems_out.write_text(selected) + if selected: + text, names = generate_sram_macros( + process=contract.sram_process, mems=selected, mdf=contract.sram_mdf ) - geoms = geoms_from_macros( - macros_v=verilog, + verilog.write_text(text) + geoms = geoms_from_names( + names=names, sram_table=contract.sram_table, mdf=contract.sram_mdf, ) - check_bitwrite_contract(geoms, contract.sram_mdf, verilog, contract.sram_table) + check_bitwrite_contract(geoms, contract.sram_mdf, contract.sram_table) else: verilog.write_text("") - firrtl.write_text("") geoms = [] ip_db = build_dir.parent / f"{build_dir.name}-ip-db" if geoms: @@ -301,14 +182,13 @@ def generate_sram( "sram_mdf": str(contract.sram_mdf), "mems_conf": str(mems_out), "sram_macros_v": str(verilog), - "sram_macros_fir": str(firrtl), "sram_corner": corner_tag, "ip_db": str(ip_db), "leaves": names, - "leaf_paths": {g.name: _leaf_paths(ip_db, g.name, corner_tag) for g in geoms}, + "leaf_paths": {g.name: _leaf_paths(ip_db, g, corner_tag) for g in geoms}, "link_dbs": [str(p) for p in db_paths], "leaf_count": len(names), - "mem_count": sum(1 for line in padded.splitlines() if line.strip()), + "mem_count": len(selected.splitlines()), } manifest_path = dest / "generate_manifest.json" manifest_path.write_text(json.dumps(man, indent=2) + "\n") diff --git a/bbdev/api/steps/ip/scripts/macro_compiler.py b/bbdev/api/steps/ip/scripts/macro_compiler.py deleted file mode 100644 index decd9a4ac..000000000 --- a/bbdev/api/steps/ip/scripts/macro_compiler.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import annotations - -import os -import sys -from pathlib import Path - -utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) -if utils_path not in sys.path: - sys.path.insert(0, utils_path) - -from utils.stream_run import stream_run -from utils.path import get_buckyball_path -from steps.mill.scripts.mill import mill_cmd - - -def run_macro_compiler( - *, mems_conf: Path, mdf: Path, verilog: Path, firrtl: Path, arch_dir: Path -) -> None: - verilog.parent.mkdir(parents=True, exist_ok=True) - cmd = mill_cmd( - get_buckyball_path(), - arch_dir, - "tapeout.runMain", - [ - "tapeout.macros.MacroCompiler", - "-n", - str(mems_conf), - "-v", - str(verilog), - "-f", - str(firrtl), - "--library", - str(mdf), - "--mode", - "strict", - ], - ) - r = stream_run(cmd, cwd=str(arch_dir)) - if r.returncode != 0: - raise RuntimeError(f"MacroCompiler failed:\n{r.stdout}\n{r.stderr}") - if not verilog.is_file() or verilog.stat().st_size == 0: - raise RuntimeError(f"MacroCompiler produced empty verilog: {verilog}") diff --git a/bbdev/api/steps/ip/scripts/replace.py b/bbdev/api/steps/ip/scripts/replace.py index e0008dc82..8eb667023 100644 --- a/bbdev/api/steps/ip/scripts/replace.py +++ b/bbdev/api/steps/ip/scripts/replace.py @@ -42,11 +42,13 @@ def source_scope( texts[cur], re.M | re.S, ): - if child in modules and child not in reachable: + # Undefined children (memory black boxes) stay in the set for mems.conf selection. + if child not in reachable: reachable.add(child) - pending.append(child) + if child in modules: + pending.append(child) - return reachable, {modules[name] for name in reachable} + return reachable, {modules[name] for name in reachable if name in modules} def replace_sources( diff --git a/bbdev/api/steps/ip/scripts/sram_compiler.py b/bbdev/api/steps/ip/scripts/sram_compiler.py index 1e2a27cdd..0b1ff1e7f 100644 --- a/bbdev/api/steps/ip/scripts/sram_compiler.py +++ b/bbdev/api/steps/ip/scripts/sram_compiler.py @@ -1,6 +1,5 @@ from __future__ import annotations -import re import sys from pathlib import Path @@ -12,20 +11,15 @@ sys.path.insert(0, str(path)) from smic180.compiler import generate_smic180_sram_dbs +from smic180.sram import smic180_sram -def leaf_names_from_macros(macros_v: Path, known: set[str]) -> list[str]: - if not macros_v.is_file(): - raise FileNotFoundError(f"missing macros verilog: {macros_v}") - text = macros_v.read_text() - ordered: list[str] = [] - for name in sorted(known): - pat = re.compile(rf"(?m)^\s*{re.escape(name)}\s+[A-Za-z_][A-Za-z0-9_$]*\s*\(") - if pat.search(text): - ordered.append(name) - if not ordered: - raise RuntimeError(f"no known sram leaf instances in {macros_v}") - return ordered +def generate_sram_macros( + *, process: str, mems: str, mdf: Path +) -> tuple[str, list[str]]: + if process == "smic180": + return smic180_sram(mems, mdf) + raise ValueError(f"unknown sram process {process!r}") def generate_sram_dbs( diff --git a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py index f99516b60..3273c059f 100644 --- a/bbdev/api/steps/mill/01_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/01_verilator_verilog_event.step.py @@ -1,4 +1,5 @@ import os +import shutil import sys from motia import FlowContext, queue @@ -37,7 +38,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: input_data.get("output_dir"), selection=input_data["target"], ) - os.makedirs(build_dir, exist_ok=True) + if os.path.exists(build_dir): + shutil.rmtree(build_dir) + os.makedirs(build_dir) ctx.logger.info(f"Using mill config: {mill_config}") ctx.logger.info(f"Using build directory: {build_dir}") command, directory = design_cmd( diff --git a/bbdev/api/steps/mill/02_vcs_verilog_event.step.py b/bbdev/api/steps/mill/02_vcs_verilog_event.step.py index 2afc11508..4980838a1 100644 --- a/bbdev/api/steps/mill/02_vcs_verilog_event.step.py +++ b/bbdev/api/steps/mill/02_vcs_verilog_event.step.py @@ -1,4 +1,5 @@ import os +import shutil import sys from motia import FlowContext, queue @@ -44,7 +45,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "tapeout", selection=input_data["target"], ) - os.makedirs(build_dir, exist_ok=True) + if os.path.exists(build_dir): + shutil.rmtree(build_dir) + os.makedirs(build_dir) returncode = ( await stream_run_logger_async( cmd=elaborate_cmd(design, input_data["target"], "verilog", build_dir), diff --git a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py index c7c94415d..e0b2ac305 100644 --- a/bbdev/api/steps/mill/03_yosys_verilog_event.step.py +++ b/bbdev/api/steps/mill/03_yosys_verilog_event.step.py @@ -1,4 +1,5 @@ import os +import shutil import sys import glob import re @@ -186,11 +187,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: mill_config, build_dir = rtl_out( bbdir, design, - "verilog", + "synth", input_data.get("output_dir"), selection=input_data["target"], ) - os.makedirs(build_dir, exist_ok=True) + if os.path.exists(build_dir): + shutil.rmtree(build_dir) + os.makedirs(build_dir) ctx.logger.info(f"Using mill config: {mill_config}") ctx.logger.info(f"Using build directory: {build_dir}") returncode = ( @@ -227,7 +230,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return failure_result yosys_cfg = load_yosys_config() - top_module = input_data.get("top") or yosys_cfg.get("top") or "DigitalTop" + top_module = input_data.get("top") or yosys_cfg["top"] yosys_log_dir = log_dir( bbdir, design, diff --git a/bbdev/api/steps/mill/04_dc_verilog_event.step.py b/bbdev/api/steps/mill/04_dc_verilog_event.step.py index 5f331a8a4..0e5d97bf2 100644 --- a/bbdev/api/steps/mill/04_dc_verilog_event.step.py +++ b/bbdev/api/steps/mill/04_dc_verilog_event.step.py @@ -1,4 +1,5 @@ import os +import shutil import sys import glob import re @@ -26,6 +27,7 @@ sys.path.insert(0, mill_scripts) from utils.path import rtl_out from steps.mill.scripts.mill import elaborate_cmd +from steps.dc.scripts.tapeout import get_tapeout_contract config = { "name": "dc-verilog", @@ -99,11 +101,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: mill_config, build_dir = rtl_out( bbdir, design, - "tapeout", + "dc", input_data.get("output_dir"), selection=input_data["target"], ) - os.makedirs(build_dir, exist_ok=True) + if os.path.exists(build_dir): + shutil.rmtree(build_dir) + os.makedirs(build_dir) ctx.logger.info(f"Using mill config: {mill_config}") ctx.logger.info(f"Using build directory: {build_dir}") returncode = ( @@ -139,7 +143,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return failure_result - top_module = input_data.get("top") or "DigitalTop" + top_module = get_tapeout_contract( + bbdir, design, input_data.get("top"), selection=input_data["target"] + ).top_module ctx.logger.info( f"Using DC RTL output directory: {build_dir}, synthesis top: {top_module}" ) diff --git a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py index fb0f2ff3d..c5dfa946a 100644 --- a/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py +++ b/bbdev/api/steps/mill/06_bebop_verilator_verilog_event.step.py @@ -5,7 +5,7 @@ """ import os -import time +import shutil import sys from motia import FlowContext, queue @@ -52,7 +52,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: mill_config, build_dir = rtl_out( bbdir, design, - "verilog", + "bebop", input_data.get("output_dir"), selection=input_data["target"], ) @@ -67,7 +67,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: selection=input_data["target"], ) if os.path.exists(build_dir): - os.rename(build_dir, f"{build_dir}.previous-{time.time_ns()}") + shutil.rmtree(build_dir) os.makedirs(build_dir) ctx.logger.info(f"Using mill target: {input_data["target"]}") ctx.logger.info(f"Using build directory: {build_dir}") diff --git a/bbdev/api/steps/pt/01_ptpx_event.step.py b/bbdev/api/steps/pt/01_ptpx_event.step.py index ad3c9e786..1aa1b3de2 100644 --- a/bbdev/api/steps/pt/01_ptpx_event.step.py +++ b/bbdev/api/steps/pt/01_ptpx_event.step.py @@ -34,7 +34,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) - top_module = input_data.get("top") or "DigitalTop" + top_module = input_data["top"] activity_path = input_data.get("activity") activity_format = input_data.get("format") analysis_dir = input_data.get("analysis_dir") diff --git a/bbdev/api/steps/uvm/scripts/ball.py b/bbdev/api/steps/uvm/scripts/ball.py index 3ee9fe85b..1fda45f1f 100644 --- a/bbdev/api/steps/uvm/scripts/ball.py +++ b/bbdev/api/steps/uvm/scripts/ball.py @@ -112,12 +112,11 @@ def vcs_defines(domain, mapping, thread, core, parameters): f"+define+BB_OUT_BW={mapping.out_bw}", f"+define+BB_MMIO_READ_BW={mapping.mmio_read_bw}", f"+define+BB_MMIO_WRITE_BW={mapping.mmio_write_bw}", - f"+define+BB_BANK_ID_W={(core.virtual_bank_count - 1).bit_length()}", + f"+define+BB_BANK_ID_W={thread.frontend.bank_id_len}", f"+define+BB_GROUP_COUNT_W={max_group_count.bit_length()}", f"+define+BB_GROUP_ID_W={(max_group_count - 1).bit_length()}", f"+define+BB_BANK_ADDR_W={(thread.mem.bank.entries - 1).bit_length()}", f"+define+BB_ROB_ID_W={(thread.frontend.rob_entries - 1).bit_length()}", - f"+define+BB_SUB_ROB_ID_W={(thread.frontend.sub_rob_depth * 4 - 1).bit_length()}", ] for e in mapping.instructions: defs.append(f"+define+{e.mnemonic}_FUNCT7={e.funct7}") @@ -185,7 +184,7 @@ def build_ball( Path(bbdir) / "examples/designs" / design_name - / "output/sims/device/arch" + / "output/sims/device/arch/verilog" / mill_cfg ) design = load_design(bbdir, design_name) diff --git a/bbdev/api/steps/yosys/01_run_api.step.py b/bbdev/api/steps/yosys/01_run_api.step.py index 2ebd1d401..0e67e3bfc 100644 --- a/bbdev/api/steps/yosys/01_run_api.step.py +++ b/bbdev/api/steps/yosys/01_run_api.step.py @@ -48,7 +48,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "design": design, - "top": req_arg(body, "top") or "DigitalTop", + "top": req_arg(body, "top"), "vcd": req_arg(body, "vcd"), "from_run_workflow": True, } diff --git a/bbdev/api/steps/yosys/02_verilog_api.step.py b/bbdev/api/steps/yosys/02_verilog_api.step.py index e27c8c229..0b0164cf9 100644 --- a/bbdev/api/steps/yosys/02_verilog_api.step.py +++ b/bbdev/api/steps/yosys/02_verilog_api.step.py @@ -45,7 +45,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - data = {"design": design, "top": req_arg(body, "top") or "DigitalTop"} + data = {"design": design, "top": req_arg(body, "top")} output_dir = req_arg(body, "output_dir") if output_dir: data["output_dir"] = output_dir diff --git a/bbdev/api/steps/yosys/03_synth_api.step.py b/bbdev/api/steps/yosys/03_synth_api.step.py index 481239790..066caf325 100644 --- a/bbdev/api/steps/yosys/03_synth_api.step.py +++ b/bbdev/api/steps/yosys/03_synth_api.step.py @@ -49,7 +49,7 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "design": design, "consumer": "yosys", - "top": req_arg(body, "top") or "DigitalTop", + "top": req_arg(body, "top"), "next_topic": "yosys.synth", } output_dir = req_arg(body, "output_dir") diff --git a/bbdev/api/steps/yosys/03_synth_event.step.py b/bbdev/api/steps/yosys/03_synth_event.step.py index 29875acda..3de7df02f 100644 --- a/bbdev/api/steps/yosys/03_synth_event.step.py +++ b/bbdev/api/steps/yosys/03_synth_event.step.py @@ -1,3 +1,4 @@ +import json import os import re import sys @@ -91,7 +92,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return yosys_cfg = load_yosys_config() - top_module = input_data.get("top") or yosys_cfg.get("top") or "DigitalTop" + top_module = input_data.get("top") or yosys_cfg["top"] liberty = yosys_cfg.get("liberty") if isinstance(liberty, str): liberty = os.path.expandvars(os.path.expanduser(liberty)) @@ -138,6 +139,11 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ctx.logger.info(f"Yosys log dir: {yosys_output_dir}") read_commands = "\n".join([f"read_verilog -sv {src}" for src in vsrcs]) + # SRAM macros instantiated by sram_macros.v are cells of their own liberty. + with open(input_data["generate_manifest"], "r") as f: + leaf_paths = json.load(f)["leaf_paths"] + for leaf in leaf_paths.values(): + read_commands += f"\nread_liberty -lib {leaf['lib']}" yosys_script = f"{yosys_output_dir}/synth_area.ys" with open(yosys_script, "w") as f: f.write(f"{read_commands}\n") diff --git a/bbdev/api/steps/yosys/scripts/yosys-config.yaml b/bbdev/api/steps/yosys/scripts/yosys-config.yaml index a50cb07c4..6cafe815c 100644 --- a/bbdev/api/steps/yosys/scripts/yosys-config.yaml +++ b/bbdev/api/steps/yosys/scripts/yosys-config.yaml @@ -6,7 +6,7 @@ liberty: # Default top module name (override with --top on CLI) -top: DigitalTop +top: DeviceTop # OpenSTA timing analysis settings # Clock period in nanoseconds (used for STA constraint) diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index 58e223844..006471f52 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -132,9 +132,10 @@ def sim_name(bbdir, design, product, *, selection): def rtl_dir(bbdir, design, product, output_dir=None, *, selection): + # Each product (verilog, tapeout, dc, p2e) elaborates its own RTL. name = sim_name(bbdir, design, product, selection=selection) return output_dir or os.path.join( - design_arch_root(bbdir, design, selection=selection), name + design_arch_root(bbdir, design, selection=selection), product, name ) diff --git a/bbdev/bbdev b/bbdev/bbdev index 736b25ead..1559d5e67 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -196,8 +196,7 @@ WORKFLOW_COMMANDS = { "buildbitstream": ( 'Build Bebop P2E runtime case. Args: "--design (--device | --core' " ) [--diff] [--itrace] [--mtrace] [--vsrc-dir ] [--output-dir" - ' ] [--stop-after vsyn|vcom]"; stop-after produces resource assessment' - " only, not a runnable case" + ' ]"' ), "runworkload": ( 'Cold-load Bebop P2E workload. Args: "--design (--device | --core' diff --git a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py index d21004947..bd8d5c6c0 100644 --- a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py +++ b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py @@ -16,7 +16,6 @@ def bbdev_bebop_p2e_buildbitstream( mtrace: bool = False, vsrc_dir: Optional[str] = None, output_dir: Optional[str] = None, - stop_after: Optional[str] = None, device: bool = False, core: str | None = None, ) -> str: @@ -27,7 +26,6 @@ def bbdev_bebop_p2e_buildbitstream( {"design": design, "diff": diff, "itrace": itrace, "mtrace": mtrace}, vsrc_dir=vsrc_dir, output_dir=output_dir, - stop_after=stop_after, ) if device: request["device"] = True diff --git a/bebop b/bebop index 4f1c04400..6db161d4d 160000 --- a/bebop +++ b/bebop @@ -1 +1 @@ -Subproject commit 4f1c04400865feb36bfefe074f0768cfc0cd7fae +Subproject commit 6db161d4df133f42b49c4d922b5905ba07a6a0b8 diff --git a/examples/components/balls/f32add/arch/F32AddRow.scala b/examples/components/balls/f32add/arch/F32AddRow.scala index b1a762835..6d33b6776 100644 --- a/examples/components/balls/f32add/arch/F32AddRow.scala +++ b/examples/components/balls/f32add/arch/F32AddRow.scala @@ -20,7 +20,7 @@ class F32AddRow extends Module { val result = Wire(Vec(4, UInt(32.W))) for (lane <- 0 until 4) { - val add = Instantiate(new AddRecFN(8, 24)) + val add = Module(new AddRecFN(8, 24)) add.io.subOp := false.B add.io.a := recFNFromFN(8, 24, io.accumulator(32 * lane + 31, 32 * lane)) add.io.b := recFNFromFN(8, 24, io.source(32 * lane + 31, 32 * lane)) diff --git a/examples/components/balls/f32add/stack/src/Lowering.cpp b/examples/components/balls/f32add/stack/src/Lowering.cpp index 332e986f9..476c7f3ea 100644 --- a/examples/components/balls/f32add/stack/src/Lowering.cpp +++ b/examples/components/balls/f32add/stack/src/Lowering.cpp @@ -1,8 +1,12 @@ //===- Lowering.cpp - F32AddBall in lower-buckyball -----------------------===// // -// add_f32: C = B + A[group] over `rows` rows of four f32, RNE with canonical -// NaN. A may have several groups; B and C have one. With `first` the sum -// starts from +0 instead of reading B, which still names an allocated bank. +// add_f32: C = A + B over `rows` rows of four f32, RNE with canonical NaN. +// A, B and C are distinct one-group banks. +// +// The Ball reads both addends from its source banks and cannot read the old +// destination, so it takes no read_destination: a sum of more terms is a +// chain of add_f32 into fresh banks. The Ball's input group is group 0 and it +// always reads B. // //===----------------------------------------------------------------------===// @@ -23,24 +27,20 @@ class AddF32Lowering : public OpRewritePattern { return failure(); Location loc = op.getLoc(); int64_t rows = cast(op->getAttr("rows")).getInt(); - int64_t group = cast(op->getAttr("group")).getInt(); - bool first = cast(op->getAttr("first")).getValue(); if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || - group < 0 || group > 31 || op.getSources().size() != 2 || + op.getReadDestination() || op.getSources().size() != 2 || op.getSources()[0] == op.getSources()[1] || llvm::is_contained(op.getSources(), op.getDestination())) - return op.emitOpError( - "add_f32 requires distinct banks, rows within a bank and a group " - "in [0, 31]"); + return op.emitOpError("add_f32 requires distinct A, B and C banks, rows " + "within a bank and no read_destination"); + // rs2: group 0 in [4:0]; bit 5 clear reads B. mlir::buddy::emitInstruction( rewriter, loc, buckyball_target::getBuckyballFunct7("F32ADD"), mlir::buddy::constantI64(rewriter, loc, mlir::buddy::bankFields(op.getSources(), op.getDestination(), false, rows)), - mlir::buddy::constantI64(rewriter, loc, - mlir::buddy::field(group, 0, 4) | - mlir::buddy::field(first, 5, 5))); + mlir::buddy::constantI64(rewriter, loc, 0)); rewriter.eraseOp(op); return success(); } diff --git a/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir b/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir index 16aab886f..dc95ca6e2 100644 --- a/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir +++ b/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir @@ -1,16 +1,16 @@ // RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s // CHECK-LABEL: func.func @reduce_groups // CHECK-DAG: %[[F76:.*]] = llvm.mlir.constant(76 : i32) : i32 -// CHECK-DAG: %[[AC:[^ ]+]] = arith.constant 2183169056 : i64 -// CHECK-DAG: %[[CA:[^ ]+]] = arith.constant 2182121504 : i64 -// CHECK-DAG: %[[FIRST:[^ ]+]] = arith.constant 32 : i64 -// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[AC]], %[[FIRST]], %[[F76]]) -// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[CA]], {{.*}}, %[[F76]]) -// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[AC]], {{.*}}, %[[F76]]) +// CHECK-DAG: %[[AB:[^ ]+]] = arith.constant 2183169056 : i64 +// CHECK-DAG: %[[DC:[^ ]+]] = arith.constant 2182121507 : i64 +// CHECK-DAG: %[[ZERO:[^ ]+]] = arith.constant 0 : i64 +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[AB]], %[[ZERO]], %[[F76]]) +// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[DC]], %[[ZERO]], %[[F76]]) +// The banks 32 (A), 33 (B) and 35 (D) are loaded by the caller: 34 = A + B, +// then 33 = D + 34. func.func @reduce_groups() -> i64 attributes {llvm.emit_c_interface} { - %output = arith.constant 34 : i64 - buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 2 : i64, group = 0 : i64, first = true} - buckyball.custom "add_f32" <{sources = array, destination = 33 : i64}> {rows = 2 : i64, group = 1 : i64, first = false} - buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 2 : i64, group = 2 : i64, first = false} + %output = arith.constant 33 : i64 + buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 2 : i64} + buckyball.custom "add_f32" <{sources = array, destination = 33 : i64}> {rows = 2 : i64} return %output : i64 } diff --git a/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp b/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp index dc797d410..bb04bdffb 100644 --- a/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp +++ b/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp @@ -14,25 +14,23 @@ static int checkF32Add() { 0xff7fffff, 1}, {0xbf800000, 0x3f000000, 0x40a00000, 0xc0a00000, 0, 0x80000000, 0x3f800000, 0x80000001}}; - alignas(64) static const uint32_t poison[8] = { - 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001, - 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001}; alignas(64) static uint32_t output[8]; - bb_mem_alloc(32, 1, 3); + bb_mem_alloc(32, 1, 1); bb_mem_alloc(33, 1, 1); bb_mem_alloc(34, 1, 1); - bb_mvin((uintptr_t)poison, 33, 2, 1); - for (unsigned group = 0; group < 3; ++group) - bb_mvin_group((uintptr_t)input[group], 32, group, 2, 1); - if (_mlir_ciface_reduce_groups() != 34) + bb_mem_alloc(35, 1, 1); + bb_mvin((uintptr_t)input[0], 32, 2, 1); + bb_mvin((uintptr_t)input[1], 33, 2, 1); + bb_mvin((uintptr_t)input[2], 35, 2, 1); + if (_mlir_ciface_reduce_groups() != 33) return 1; - bb_mvout((uintptr_t)output, 34, 2, 1); + bb_mvout((uintptr_t)output, 33, 2, 1); for (unsigned i = 0; i < 8; ++i) { union { uint32_t bits; float value; - } expected = {0}, source; - for (unsigned group = 0; group < 3; ++group) { + } expected = {input[0][i]}, source; + for (unsigned group = 1; group < 3; ++group) { source.bits = input[group][i]; asm volatile("fadd.s %0, %1, %2, rne" : "=f"(expected.value) @@ -44,12 +42,13 @@ static int checkF32Add() { bb_mem_release(32); bb_mem_release(33); bb_mem_release(34); + bb_mem_release(35); return 0; } int main() { int result = checkF32Add(); if (!result) - puts("F32ADD Ball MLIR grouped/RNE/NaN/first/ordered sum PASS"); + puts("F32ADD Ball MLIR RNE/NaN/ordered sum PASS"); return result; } diff --git a/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir b/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir index 98a8b0c4e..ac2b1f3cd 100644 --- a/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir +++ b/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir @@ -1,28 +1,24 @@ // RUN: buddy-opt %s -split-input-file -lower-buckyball -verify-diagnostics func.func @zero_rows() { - // expected-error @+2 {{add_f32 requires distinct banks, rows within a bank and a group in [0, 31]}} - // expected-error @+1 {{has no lowering on this core}} - buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 0 : i64, group = 0 : i64, first = true} + // expected-error @+1 {{add_f32 requires distinct A, B and C banks, rows within a bank and no read_destination}} + buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 0 : i64} return } // ----- -func.func @wide_group() { - // expected-error @+2 {{add_f32 requires distinct banks, rows within a bank and a group in [0, 31]}} - // expected-error @+1 {{has no lowering on this core}} - buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 1 : i64, group = 32 : i64, first = false} +func.func @wide_rows() { + // expected-error @+1 {{add_f32 requires distinct A, B and C banks, rows within a bank and no read_destination}} + buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 17179869184 : i64} return } // ----- -func.func @wide_rows() { - // expected-error @+2 {{add_f32 requires distinct banks, rows within a bank and a group in [0, 31]}} - // expected-error @+1 {{has no lowering on this core}} - buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 17179869184 : i64, group = 0 : i64, first = false} +func.func @equal_banks() { + // expected-error @+1 {{add_f32 requires distinct A, B and C banks, rows within a bank and no read_destination}} + buckyball.custom "add_f32" <{sources = array, destination = 32 : i64}> {rows = 1 : i64} return } // ----- -func.func @equal_banks() { - // expected-error @+2 {{add_f32 requires distinct banks, rows within a bank and a group in [0, 31]}} - // expected-error @+1 {{has no lowering on this core}} - buckyball.custom "add_f32" <{sources = array, destination = 32 : i64}> {rows = 1 : i64, group = 0 : i64, first = true} +func.func @read_destination() { + // expected-error @+1 {{add_f32 requires distinct A, B and C banks, rows within a bank and no read_destination}} + buckyball.custom "add_f32" <{sources = array, destination = 34 : i64, read_destination}> {rows = 1 : i64} return } diff --git a/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir b/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir index 4c6f162ca..5844f9e4b 100644 --- a/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir +++ b/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir @@ -1,23 +1,24 @@ -// The reduce core has four banks: groups 0 and 1 share a two-group source -// bank, group 2 then reuses its banks. The first add_f32 starts from +0 -// instead of the poison in the accumulator. -func.func @reduce_groups(%pair: memref<2x2x4xi32>, %last: memref<2x4xi32>, - %poison: memref<2x4xi32>, %output: memref<2x4xi32>) +// A three-term sum as a chain of add_f32 into fresh banks: the first sum +// lands in a bank holding poison, which it overwrites, and the second reuses +// a released source bank. +func.func @reduce_groups(%first: memref<2x4xi32>, %second: memref<2x4xi32>, + %last: memref<2x4xi32>, %poison: memref<2x4xi32>, + %output: memref<2x4xi32>) attributes {llvm.emit_c_interface} { - %source = buckyball.bank_alloc <{row = 1 : i64, col = 2 : i64}> - %accumulator = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %sum = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> - %groups = buckyball.bank_mvin %pair, %source <{depth = 2 : i64, stride = 1 : i64}> : memref<2x2x4xi32> - %poisoned = buckyball.bank_mvin %poison, %accumulator <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> - %first = buckyball.bank_custom "add_f32" (%groups, %poisoned) -> %sum {rows = 2 : i64, group = 0 : i64, first = true} - %second = buckyball.bank_custom "add_f32" (%groups, %first) -> %poisoned {rows = 2 : i64, group = 1 : i64, first = false} - buckyball.bank_release %groups - %single = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> - %group2 = buckyball.bank_mvin %last, %single <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> - %third = buckyball.bank_custom "add_f32" (%group2, %second) -> %first {rows = 2 : i64, group = 0 : i64, first = false} - %stored = buckyball.bank_mvout %output, %third <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> - buckyball.bank_release %group2 - buckyball.bank_release %second + %la = buckyball.bank_mvin %first, %a <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> + %lb = buckyball.bank_mvin %second, %b <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> + %poisoned = buckyball.bank_mvin %poison, %sum <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> + %partial = buckyball.bank_custom "add_f32" (%la, %lb) -> %poisoned {rows = 2 : i64} + buckyball.bank_release %la + %d = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %ld = buckyball.bank_mvin %last, %d <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> + %total = buckyball.bank_custom "add_f32" (%ld, %partial) -> %lb {rows = 2 : i64} + %stored = buckyball.bank_mvout %output, %total <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> + buckyball.bank_release %ld + buckyball.bank_release %partial buckyball.bank_release %stored return } diff --git a/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp b/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp index 719887296..a83a0bb2c 100644 --- a/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp +++ b/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp @@ -3,8 +3,8 @@ #include using Rows = StridedMemRefType; -extern "C" void _mlir_ciface_reduce_groups(StridedMemRefType *, - Rows *, Rows *, Rows *); +extern "C" void _mlir_ciface_reduce_groups(Rows *, Rows *, Rows *, Rows *, + Rows *); static int checkF32Add() { alignas(64) static const uint32_t input[3][8] = { @@ -18,24 +18,22 @@ static int checkF32Add() { 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001, 0x7fa00001}; alignas(64) static uint32_t output[8]; - // A whole-bank mvin reads row r of group g at (r * 2 + g) * 16 bytes. - alignas(64) static uint32_t pair[2][2][4]; - for (unsigned group = 0; group < 2; ++group) - for (unsigned i = 0; i < 8; ++i) - pair[i / 4][group][i % 4] = input[group][i]; - StridedMemRefType pairRef = { - &pair[0][0][0], &pair[0][0][0], 0, {2, 2, 4}, {8, 4, 1}}; + Rows firstRef = { + (uint32_t *)input[0], (uint32_t *)input[0], 0, {2, 4}, {4, 1}}; + Rows secondRef = { + (uint32_t *)input[1], (uint32_t *)input[1], 0, {2, 4}, {4, 1}}; Rows lastRef = { (uint32_t *)input[2], (uint32_t *)input[2], 0, {2, 4}, {4, 1}}; Rows poisonRef = {(uint32_t *)poison, (uint32_t *)poison, 0, {2, 4}, {4, 1}}; Rows outputRef = {output, output, 0, {2, 4}, {4, 1}}; - _mlir_ciface_reduce_groups(&pairRef, &lastRef, &poisonRef, &outputRef); + _mlir_ciface_reduce_groups(&firstRef, &secondRef, &lastRef, &poisonRef, + &outputRef); for (unsigned i = 0; i < 8; ++i) { union { uint32_t bits; float value; - } expected = {0}, source; - for (unsigned group = 0; group < 3; ++group) { + } expected = {input[0][i]}, source; + for (unsigned group = 1; group < 3; ++group) { source.bits = input[group][i]; asm volatile("fadd.s %0, %1, %2, rne" : "=f"(expected.value) @@ -50,6 +48,6 @@ static int checkF32Add() { int main() { int result = checkF32Add(); if (!result) - puts("F32ADD Bank MLIR grouped/RNE/NaN/first/SSA output PASS"); + puts("F32ADD Bank MLIR chained RNE/NaN/SSA output PASS"); return result; } diff --git a/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir b/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir index 6bf585533..acf1ba50d 100644 --- a/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir +++ b/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir @@ -3,9 +3,8 @@ func.func @same_bank(%input: memref<1x4xi32>) { %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %la = buckyball.bank_mvin %input, %a <{depth = 1 : i64, stride = 1 : i64}> : memref<1x4xi32> - // expected-error @+2 {{add_f32 requires distinct banks, rows within a bank and a group in [0, 31]}} - // expected-error @+1 {{has no lowering on this core}} - %out = buckyball.bank_custom "add_f32" (%la, %b) -> %b {rows = 1 : i64, group = 0 : i64, first = true} + // expected-error @+1 {{add_f32 requires distinct A, B and C banks, rows within a bank and no read_destination}} + %out = buckyball.bank_custom "add_f32" (%la, %b) -> %b {rows = 1 : i64} buckyball.bank_release %la buckyball.bank_release %out return @@ -16,23 +15,21 @@ func.func @zero_rows(%input: memref<1x4xi32>) { %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %c = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %la = buckyball.bank_mvin %input, %a <{depth = 1 : i64, stride = 1 : i64}> : memref<1x4xi32> - // expected-error @+2 {{add_f32 requires distinct banks, rows within a bank and a group in [0, 31]}} - // expected-error @+1 {{has no lowering on this core}} - %out = buckyball.bank_custom "add_f32" (%la, %b) -> %c {rows = 0 : i64, group = 0 : i64, first = true} + // expected-error @+1 {{add_f32 requires distinct A, B and C banks, rows within a bank and no read_destination}} + %out = buckyball.bank_custom "add_f32" (%la, %b) -> %c {rows = 0 : i64} buckyball.bank_release %la buckyball.bank_release %b buckyball.bank_release %out return } // ----- -func.func @negative_group(%input: memref<1x4xi32>) { +func.func @read_destination(%input: memref<1x4xi32>) { %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %c = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %la = buckyball.bank_mvin %input, %a <{depth = 1 : i64, stride = 1 : i64}> : memref<1x4xi32> - // expected-error @+2 {{add_f32 requires distinct banks, rows within a bank and a group in [0, 31]}} - // expected-error @+1 {{has no lowering on this core}} - %out = buckyball.bank_custom "add_f32" (%la, %b) -> %c {rows = 1 : i64, group = -1 : i64, first = false} + // expected-error @+1 {{add_f32 requires distinct A, B and C banks, rows within a bank and no read_destination}} + %out = buckyball.bank_custom "add_f32" (%la, %b) -> %c <{read_destination}> {rows = 1 : i64} buckyball.bank_release %la buckyball.bank_release %b buckyball.bank_release %out diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrlCmdStates.scala b/examples/components/balls/gemmini/arch/GemminiExCtrlCmdStates.scala index 89bd21e07..11729973a 100644 --- a/examples/components/balls/gemmini/arch/GemminiExCtrlCmdStates.scala +++ b/examples/components/balls/gemmini/arch/GemminiExCtrlCmdStates.scala @@ -27,8 +27,9 @@ trait GemminiExCtrlCmdStates { this: GemminiExCtrl => op2_bank := io.cmdReq.bits.op2_bank wr_bank := io.cmdReq.bits.wr_bank wr_read_bank := io.cmdReq.bits.wr_read_bank - op1_group := io.cmdReq.bits.op1_group - op2_group := io.cmdReq.bits.op2_group + // rs2[41:36] and rs2[47:42] select a group of op1 and op2 within their banks. + op1_group := io.cmdReq.bits.op1_group + io.cmdReq.bits.rs2(41, 36) + op2_group := io.cmdReq.bits.op2_group + io.cmdReq.bits.rs2(47, 42) wr_group := io.cmdReq.bits.wr_group op1_base := io.cmdReq.bits.rs2(15, 6) op2_base := io.cmdReq.bits.rs2(25, 16) @@ -55,6 +56,7 @@ trait GemminiExCtrlCmdStates { this: GemminiExCtrl => ) state := sCommit }.elsewhen(sub_cmd === GemminiSubCmd.PRELOAD) { + assert(io.cmdReq.bits.rs2(41, 36) < io.cmdReq.bits.op1_col, "Gemmini preload op1 group outside the bank") read_row_cnt := 0.U feed_row_cnt := 0.U req_sent := false.B @@ -64,6 +66,10 @@ trait GemminiExCtrlCmdStates { this: GemminiExCtrl => }.elsewhen( sub_cmd === GemminiSubCmd.COMPUTE_PRELOADED || sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED ) { + assert( + io.cmdReq.bits.rs2(41, 36) < io.cmdReq.bits.op1_col && io.cmdReq.bits.rs2(47, 42) < io.cmdReq.bits.op2_col, + "Gemmini compute operand group outside the bank" + ) read_row_cnt := 0.U feed_row_cnt := 0.U outBufRows := 0.U diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrlDefs.scala b/examples/components/balls/gemmini/arch/GemminiExCtrlDefs.scala index 809ebf573..954717b38 100644 --- a/examples/components/balls/gemmini/arch/GemminiExCtrlDefs.scala +++ b/examples/components/balls/gemmini/arch/GemminiExCtrlDefs.scala @@ -58,7 +58,7 @@ trait GemminiExCtrlDefs { this: GemminiExCtrl => val io = ctrlIo - val mesh = Instantiate(new MeshWithDelays( + val mesh = Module(new MeshWithDelays( inputType, inputType, meshOutputType, diff --git a/examples/components/balls/gemmini/emu/src/21_gemmini_preload.rs b/examples/components/balls/gemmini/emu/src/21_gemmini_preload.rs index ef1ca1a1f..47769bfd0 100644 --- a/examples/components/balls/gemmini/emu/src/21_gemmini_preload.rs +++ b/examples/components/balls/gemmini/emu/src/21_gemmini_preload.rs @@ -1,5 +1,5 @@ use super::bank_matrix::read_i8_nn_at; -use super::decode::{pbank, rs1_b0, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b2, rs1_iter}; use super::gemmini_state::gemini; use super::instruction::{BallModel, ExecContext}; @@ -18,7 +18,10 @@ impl BallModel for GemminiPreload { panic!("gemmini_preload: bad iter"); } - let p1 = pbank(ctx, op1); + // rs2[41:36] selects the group of op1. + let group1 = (xs2 >> 36) & 0x3f; + assert!(group1 < ctx.config(op1).cols, "gemmini_preload: op1 group outside the bank"); + let p1 = pbank_group(ctx, op1, group1); let state = gemini(ctx); let mut gm = state.lock().unwrap(); diff --git a/examples/components/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs b/examples/components/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs index afb8f7c4c..f35164cee 100644 --- a/examples/components/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs +++ b/examples/components/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs @@ -1,5 +1,5 @@ use super::bank_matrix::{read_i32_nn_at, read_i8_nn_at, write_i32_nn_groups_at}; -use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::gemmini_state::{gemini, in_shift as apply_in_shift}; use super::instruction::{BallModel, ExecContext}; @@ -19,8 +19,15 @@ impl BallModel for GemminiComputePreloaded { panic!("gemmini_compute_preloaded: bad iter"); } - let pa = pbank(ctx, op_a); - let pb = pbank(ctx, op_b); + // rs2[41:36] and rs2[47:42] select the groups of op1 and op2. + let group_a = (xs2 >> 36) & 0x3f; + let group_b = (xs2 >> 42) & 0x3f; + assert!( + group_a < ctx.config(op_a).cols && group_b < ctx.config(op_b).cols, + "gemmini_compute_preloaded: operand group outside the bank" + ); + let pa = pbank_group(ctx, op_a, group_a); + let pb = pbank_group(ctx, op_b, group_b); let pw: Vec<_> = (0..ctx.config(wr).cols) .map(|group| pbank_group(ctx, wr, group)) .collect(); diff --git a/examples/components/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs b/examples/components/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs index 3f31e93f4..2c124c6ef 100644 --- a/examples/components/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs +++ b/examples/components/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs @@ -1,5 +1,5 @@ use super::bank_matrix::{read_i32_nn_groups_at, read_i8_nn_at, write_i32_nn_groups_at}; -use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::gemmini_state::{gemini, in_shift as apply_in_shift}; use super::instruction::{BallModel, ExecContext}; @@ -19,8 +19,15 @@ impl BallModel for GemminiComputeAccumulated { panic!("gemmini_compute_accumulated: bad iter"); } - let pa = pbank(ctx, op_a); - let pb = pbank(ctx, op_b); + // rs2[41:36] and rs2[47:42] select the groups of op1 and op2. + let group_a = (xs2 >> 36) & 0x3f; + let group_b = (xs2 >> 42) & 0x3f; + assert!( + group_a < ctx.config(op_a).cols && group_b < ctx.config(op_b).cols, + "gemmini_compute_accumulated: operand group outside the bank" + ); + let pa = pbank_group(ctx, op_a, group_a); + let pb = pbank_group(ctx, op_b, group_b); let pw: Vec<_> = (0..ctx.config(wr).cols) .map(|group| pbank_group(ctx, wr, group)) .collect(); diff --git a/examples/components/balls/gemmini/stack/src/Lowering.cpp b/examples/components/balls/gemmini/stack/src/Lowering.cpp index 735b1b3ce..d10a070df 100644 --- a/examples/components/balls/gemmini/stack/src/Lowering.cpp +++ b/examples/components/balls/gemmini/stack/src/Lowering.cpp @@ -1,18 +1,31 @@ //===- Lowering.cpp - GemminiBall in lower-buckyball ----------------------===// // -// matmul_i8_tile: C = A @ B for one 16x16 tile. A holds sixteen rows of -// sixteen i8 (M x K) at row a_base, B sixteen rows of sixteen i8 (K x N) at -// row b_base, and C, a bank of four columns, receives sixteen rows of sixteen -// i32 at row c_base. a_base, b_base and c_base are the i64 arguments. +// matmul_i8_tile: one 16x16 tile C = A @ B over `k`, a positive multiple of +// sixteen. A and B fill their groups in order, each group a bank depth of +// rows before the next, so a K deeper than a bank spans several groups +// (allocated together or gathered with bank_transfer). +// A: k / 16 blocks of sixteen rows of sixteen i8; row i of block r is +// A[i, 16r .. 16r + 15]. +// B: k rows of sixteen i8, B[0 .. k - 1, 0 .. 15]. +// C: a bank of four groups, sixteen rows of sixteen i32. // With read_destination C = in_shift(C + A @ B, shift), rounding to nearest -// even; without, C = A @ B on the output-stationary array. +// even; without, C = A @ B. +// +// This is not smatmul's matmul_i8: C holds one output row per bank row across +// four groups instead of four bank rows of one group, there is no bias row, +// and read_destination accumulates the whole old C with a rounding shift. +// +// The Ball issues one instruction per 16-deep block, the operand groups and +// row bases derived from the block index. Without read_destination the first +// block runs on the output-stationary array (PRELOAD primes D = 0); every +// other block accumulates on the weight-stationary array, reading the old C. +// Only the last block applies the shift. // //===----------------------------------------------------------------------===// #include "Buckyball/BuckyballOps.h" #include "Conversion/Buckyball.h" #include "Target/BuckyballTargetRegistry.h" -#include "mlir/Dialect/Arith/IR/Arith.h" using namespace mlir; using namespace ::buddy::buckyball; @@ -26,67 +39,69 @@ class MatmulI8TileLowering : public OpRewritePattern { if (op.getKind() != "matmul_i8_tile") return failure(); Location loc = op.getLoc(); - bool accumulate = op.getReadDestination(); + bool readDestination = op.getReadDestination(); + int64_t k = cast(op->getAttr("k")).getInt(); int64_t shift = - accumulate ? cast(op->getAttr("shift")).getInt() : 0; + readDestination ? cast(op->getAttr("shift")).getInt() : 0; int64_t meshRows = buckyball_target::getBuckyballBallParam( "GEMMINI_COMPUTE_PRELOADED", "meshRows"); int64_t meshColumns = buckyball_target::getBuckyballBallParam( "GEMMINI_COMPUTE_PRELOADED", "meshColumns"); const auto &target = buckyball_target::getBuckyballTarget(); + ArrayRef sources = op.getSources(); + int64_t c = op.getDestination(); if (target.bankWidthBits != 128 || meshRows != 16 || meshColumns != 16 || - target.bankDepth < 16 || op.getSources().size() != 2 || - op.getArguments().size() != 3 || - op.getSources()[0] == op.getSources()[1] || - llvm::is_contained(op.getSources(), op.getDestination())) + target.bankDepth % 16 || target.bankDepth > 1024 || + sources.size() != 2 || sources[0] == sources[1] || k <= 0 || k % 16 || + k > 64 * target.bankDepth || llvm::is_contained(sources, c)) return op.emitOpError("matmul_i8_tile requires distinct A, B and C " - "banks, three row bases and a 16x16 mesh"); - int64_t a = op.getSources()[0], bBank = op.getSources()[1]; - int64_t c = op.getDestination(); - // rs2: the subcommand in [3:0], then the A, B and C row bases. - auto rs2 = [&](uint64_t subcommand, Value aBase, Value bBase) { - Value word = mlir::buddy::constantI64(b, loc, subcommand); - for (auto [base, bit] : {std::pair{aBase, 6}, std::pair{bBase, 16}, - std::pair{op.getArguments()[2], 26}}) - if (base) - word = arith::OrIOp::create( - b, loc, word, - arith::ShLIOp::create(b, loc, base, - mlir::buddy::constantI64(b, loc, bit))); - return word; - }; - Value aBase = op.getArguments()[0], bBase = op.getArguments()[1]; - // CONFIG: weight-stationary for the accumulation, as the ISA tests use it, - // output-stationary for the plain product. - mlir::buddy::emitInstruction( - b, loc, buckyball_target::getBuckyballFunct7("GEMMINI_CONFIG"), - mlir::buddy::constantI64(b, loc, 0), - mlir::buddy::constantI64(b, loc, - mlir::buddy::field(accumulate, 4, 4) | - mlir::buddy::field(shift, 9, 40))); - if (accumulate) { + "banks, k a multiple of 16 within 64 groups and a " + "16x16 mesh"); + int64_t a = sources[0], bBank = sources[1]; + int64_t steps = k / 16, groupBlocks = target.bankDepth / 16; + uint64_t config = ~0ULL; + for (int64_t step = 0; step < steps; ++step) { + uint64_t group = step / groupBlocks, base = step % groupBlocks * 16; + bool accumulate = readDestination || step > 0; + // CONFIG: rs2[4] picks the weight-stationary array, rs2[40:9] the shift. + uint64_t word = mlir::buddy::field(accumulate, 4, 4) | + mlir::buddy::field(step == steps - 1 ? shift : 0, 9, 40); + if (word != config) + mlir::buddy::emitInstruction( + b, loc, buckyball_target::getBuckyballFunct7("GEMMINI_CONFIG"), + mlir::buddy::constantI64(b, loc, 0), + mlir::buddy::constantI64(b, loc, word)); + config = word; + // rs2: the subcommand in [3:0], the first and second source row bases + // in [15:6] and [25:16], their groups in [41:36] and [47:42]; C is at + // row 0. + uint64_t bases = + mlir::buddy::field(base, 6, 15) | mlir::buddy::field(base, 16, 25) | + mlir::buddy::field(group, 36, 41) | mlir::buddy::field(group, 42, 47); + if (accumulate) { + mlir::buddy::emitInstruction( + b, loc, + buckyball_target::getBuckyballFunct7("GEMMINI_COMPUTE_ACCUMULATED"), + mlir::buddy::constantI64( + b, loc, mlir::buddy::bankFields({a, bBank}, c, true, 16)), + mlir::buddy::constantI64(b, loc, 3 | bases)); + continue; + } + // The output-stationary preload reads sixteen rows of B. + mlir::buddy::emitInstruction( + b, loc, buckyball_target::getBuckyballFunct7("GEMMINI_PRELOAD"), + mlir::buddy::constantI64( + b, loc, mlir::buddy::bankFields({bBank}, c, false, 16)), + mlir::buddy::constantI64(b, loc, + 1 | mlir::buddy::field(base, 6, 15) | + mlir::buddy::field(group, 36, 41))); mlir::buddy::emitInstruction( b, loc, - buckyball_target::getBuckyballFunct7("GEMMINI_COMPUTE_ACCUMULATED"), + buckyball_target::getBuckyballFunct7("GEMMINI_COMPUTE_PRELOADED"), mlir::buddy::constantI64( - b, loc, mlir::buddy::bankFields({a, bBank}, c, true, 16)), - rs2(3, aBase, bBase)); - b.eraseOp(op); - return success(); + b, loc, mlir::buddy::bankFields({a, bBank}, c, false, 16)), + mlir::buddy::constantI64(b, loc, 2 | bases)); } - // The output-stationary preload only primes the array with D = 0; it - // reads B, which is valid for sixteen rows. - mlir::buddy::emitInstruction( - b, loc, buckyball_target::getBuckyballFunct7("GEMMINI_PRELOAD"), - mlir::buddy::constantI64( - b, loc, mlir::buddy::bankFields({bBank}, c, false, 16)), - rs2(1, bBase, Value())); - mlir::buddy::emitInstruction( - b, loc, - buckyball_target::getBuckyballFunct7("GEMMINI_COMPUTE_PRELOADED"), - mlir::buddy::constantI64( - b, loc, mlir::buddy::bankFields({a, bBank}, c, false, 16)), - rs2(2, aBase, bBase)); b.eraseOp(op); return success(); } diff --git a/examples/components/balls/gemmini/verify/filelists/gemmini_ball.f b/examples/components/balls/gemmini/verify/filelists/gemmini_ball.f index 187506b5e..94a5fe6c5 100644 --- a/examples/components/balls/gemmini/verify/filelists/gemmini_ball.f +++ b/examples/components/balls/gemmini/verify/filelists/gemmini_ball.f @@ -1,29 +1,14 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+src +incdir+src/pkg +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv -@RTL@/PE.sv -@RTL@/AlwaysOutTransposer.sv -@RTL@/MacUnit.sv -@RTL@/PE_256.sv -@RTL@/Core.sv -@RTL@/Mesh.sv -@RTL@/TagQueue.sv -@RTL@/ram_6x16.sv -@RTL@/Queue6_TagWithIdAndTotalRows.sv -@RTL@/MeshWithDelays.sv -@RTL@/ram_data_16x128.sv -@RTL@/Queue16_SramReadResp.sv -@RTL@/GemminiExCtrl.sv -@RTL@/LoopMatmulUnroller.sv -@RTL@/LoopConvAddrGen.sv -@RTL@/LoopConvUnroller.sv -@RTL@/LoopCmdEncoder.sv -@RTL@/Arbiter2_LoopCmd.sv -@RTL@/GemminiBall.sv +src/gemmini_expansion_if.sv +-F @RTL@/filelist.f src/pkg/gemmini_pkg.sv src/tb_top.sv diff --git a/examples/components/balls/gemmini/verify/src/common/gemmini_defs.svh b/examples/components/balls/gemmini/verify/src/common/gemmini_defs.svh index bf2542620..994104200 100644 --- a/examples/components/balls/gemmini/verify/src/common/gemmini_defs.svh +++ b/examples/components/balls/gemmini/verify/src/common/gemmini_defs.svh @@ -1,3 +1,6 @@ +`ifndef GEMMINI_CONFIG_FUNCT7 +`error "GEMMINI_CONFIG_FUNCT7 must be provided by the selected Thread ballISA" +`endif `ifndef GEMMINI_LOOP_WS_CONFIG_BOUNDS_FUNCT7 `error "GEMMINI_LOOP_WS_CONFIG_BOUNDS_FUNCT7 must be provided by the selected Thread ballISA" `endif diff --git a/examples/components/balls/gemmini/verify/src/gemmini_expansion_if.sv b/examples/components/balls/gemmini/verify/src/gemmini_expansion_if.sv new file mode 100644 index 000000000..0bd797428 --- /dev/null +++ b/examples/components/balls/gemmini/verify/src/gemmini_expansion_if.sv @@ -0,0 +1,17 @@ +// SubROB expansion port of GemminiBall: loop configuration and loop triggers. +interface gemmini_expansion_if; + logic command_ready; + logic command_valid; + logic [`BB_ROB_ID_W-1:0] command_bits_rob_id; + logic [ 6:0] command_bits_funct; + logic [ 63:0] command_bits_rs2; + logic accepts; + logic starts; + logic complete_ready; + logic complete_valid; + logic [`BB_ROB_ID_W-1:0] complete_bits_rob_id; + logic write_ready; + logic write_valid; + logic [ 9:0] write_bits_addr; + logic [ 7:0] write_bits_data; +endinterface diff --git a/examples/components/balls/gemmini/verify/src/tb_top.sv b/examples/components/balls/gemmini/verify/src/tb_top.sv index a4b30301e..a7d7fe42a 100644 --- a/examples/components/balls/gemmini/verify/src/tb_top.sv +++ b/examples/components/balls/gemmini/verify/src/tb_top.sv @@ -3,72 +3,176 @@ module tb_top; import gemmini_pkg::*; bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + gemmini_expansion_if expansion (); GemminiBall dut ( .clock(intf.clock), .reset(intf.reset), - .io_bankAccessReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_bankWrite_1_io_req_ready(intf.bank_write_req_ready[1]), - .io_bankWrite_1_io_req_valid(intf.bank_write_req_valid[1]), - .io_bankWrite_1_io_resp_ready(intf.bank_write_resp_ready[1]), - .io_bankWrite_1_io_resp_valid(intf.bank_write_resp_valid[1]), - .io_bankWrite_1_io_resp_bits_ok(intf.bank_write_resp_ok[1]), - .io_bankWrite_2_io_req_ready(intf.bank_write_req_ready[2]), - .io_bankWrite_2_io_req_valid(intf.bank_write_req_valid[2]), - .io_bankWrite_2_io_resp_ready(intf.bank_write_resp_ready[2]), - .io_bankWrite_2_io_resp_valid(intf.bank_write_resp_valid[2]), - .io_bankWrite_2_io_resp_bits_ok(intf.bank_write_resp_ok[2]), - .io_bankWrite_3_io_req_ready(intf.bank_write_req_ready[3]), - .io_bankWrite_3_io_req_valid(intf.bank_write_req_valid[3]), - .io_bankWrite_3_io_resp_ready(intf.bank_write_resp_ready[3]), - .io_bankWrite_3_io_resp_valid(intf.bank_write_resp_valid[3]), - .io_bankWrite_3_io_resp_bits_ok(intf.bank_write_resp_ok[3]), - .io_subRobReq_ready(intf.sub_rob_req_ready) + .expansion_command_ready(expansion.command_ready), + .expansion_command_valid(expansion.command_valid), + .expansion_command_bits_rob_id(expansion.command_bits_rob_id), + .expansion_command_bits_funct(expansion.command_bits_funct), + .expansion_command_bits_rs2(expansion.command_bits_rs2), + .expansion_accepts(expansion.accepts), + .expansion_starts(expansion.starts), + .expansion_rows_ready(1'b1), + .expansion_complete_ready(expansion.complete_ready), + .expansion_complete_valid(expansion.complete_valid), + .expansion_complete_bits_rob_id(expansion.complete_bits_rob_id), + .expansion_read_req_ready(1'b0), + .expansion_read_resp_valid(1'b0), + .expansion_read_resp_bits_data(8'h00), + .expansion_write_ready(expansion.write_ready), + .expansion_write_valid(expansion.write_valid), + .expansion_write_bits_addr(expansion.write_bits_addr), + .expansion_write_bits_data(expansion.write_bits_data), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .blink_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .blink_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .blink_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .blink_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .blink_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .blink_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .blink_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .blink_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .blink_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .blink_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .blink_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .blink_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .blink_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .blink_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .blink_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .blink_bankWrite_1_bank_id(intf.bank_write_bank_id[1]), + .blink_bankWrite_1_rob_id(intf.bank_write_rob_id[1]), + .blink_bankWrite_1_group_id(intf.bank_write_group_id[1]), + .blink_bankWrite_1_io_req_ready(intf.bank_write_req_ready[1]), + .blink_bankWrite_1_io_req_valid(intf.bank_write_req_valid[1]), + .blink_bankWrite_1_io_req_bits_addr(intf.bank_write_req_addr[1]), + .blink_bankWrite_1_io_req_bits_mask_0(intf.bank_write_req_mask[1][0]), + .blink_bankWrite_1_io_req_bits_mask_1(intf.bank_write_req_mask[1][1]), + .blink_bankWrite_1_io_req_bits_mask_2(intf.bank_write_req_mask[1][2]), + .blink_bankWrite_1_io_req_bits_mask_3(intf.bank_write_req_mask[1][3]), + .blink_bankWrite_1_io_req_bits_mask_4(intf.bank_write_req_mask[1][4]), + .blink_bankWrite_1_io_req_bits_mask_5(intf.bank_write_req_mask[1][5]), + .blink_bankWrite_1_io_req_bits_mask_6(intf.bank_write_req_mask[1][6]), + .blink_bankWrite_1_io_req_bits_mask_7(intf.bank_write_req_mask[1][7]), + .blink_bankWrite_1_io_req_bits_mask_8(intf.bank_write_req_mask[1][8]), + .blink_bankWrite_1_io_req_bits_mask_9(intf.bank_write_req_mask[1][9]), + .blink_bankWrite_1_io_req_bits_mask_10(intf.bank_write_req_mask[1][10]), + .blink_bankWrite_1_io_req_bits_mask_11(intf.bank_write_req_mask[1][11]), + .blink_bankWrite_1_io_req_bits_mask_12(intf.bank_write_req_mask[1][12]), + .blink_bankWrite_1_io_req_bits_mask_13(intf.bank_write_req_mask[1][13]), + .blink_bankWrite_1_io_req_bits_mask_14(intf.bank_write_req_mask[1][14]), + .blink_bankWrite_1_io_req_bits_mask_15(intf.bank_write_req_mask[1][15]), + .blink_bankWrite_1_io_req_bits_data(intf.bank_write_req_data[1]), + .blink_bankWrite_1_io_resp_valid(intf.bank_write_resp_valid[1]), + .blink_bankWrite_1_io_resp_bits_ok(intf.bank_write_resp_ok[1]), + .blink_bankWrite_2_bank_id(intf.bank_write_bank_id[2]), + .blink_bankWrite_2_rob_id(intf.bank_write_rob_id[2]), + .blink_bankWrite_2_group_id(intf.bank_write_group_id[2]), + .blink_bankWrite_2_io_req_ready(intf.bank_write_req_ready[2]), + .blink_bankWrite_2_io_req_valid(intf.bank_write_req_valid[2]), + .blink_bankWrite_2_io_req_bits_addr(intf.bank_write_req_addr[2]), + .blink_bankWrite_2_io_req_bits_mask_0(intf.bank_write_req_mask[2][0]), + .blink_bankWrite_2_io_req_bits_mask_1(intf.bank_write_req_mask[2][1]), + .blink_bankWrite_2_io_req_bits_mask_2(intf.bank_write_req_mask[2][2]), + .blink_bankWrite_2_io_req_bits_mask_3(intf.bank_write_req_mask[2][3]), + .blink_bankWrite_2_io_req_bits_mask_4(intf.bank_write_req_mask[2][4]), + .blink_bankWrite_2_io_req_bits_mask_5(intf.bank_write_req_mask[2][5]), + .blink_bankWrite_2_io_req_bits_mask_6(intf.bank_write_req_mask[2][6]), + .blink_bankWrite_2_io_req_bits_mask_7(intf.bank_write_req_mask[2][7]), + .blink_bankWrite_2_io_req_bits_mask_8(intf.bank_write_req_mask[2][8]), + .blink_bankWrite_2_io_req_bits_mask_9(intf.bank_write_req_mask[2][9]), + .blink_bankWrite_2_io_req_bits_mask_10(intf.bank_write_req_mask[2][10]), + .blink_bankWrite_2_io_req_bits_mask_11(intf.bank_write_req_mask[2][11]), + .blink_bankWrite_2_io_req_bits_mask_12(intf.bank_write_req_mask[2][12]), + .blink_bankWrite_2_io_req_bits_mask_13(intf.bank_write_req_mask[2][13]), + .blink_bankWrite_2_io_req_bits_mask_14(intf.bank_write_req_mask[2][14]), + .blink_bankWrite_2_io_req_bits_mask_15(intf.bank_write_req_mask[2][15]), + .blink_bankWrite_2_io_req_bits_data(intf.bank_write_req_data[2]), + .blink_bankWrite_2_io_resp_valid(intf.bank_write_resp_valid[2]), + .blink_bankWrite_2_io_resp_bits_ok(intf.bank_write_resp_ok[2]), + .blink_bankWrite_3_bank_id(intf.bank_write_bank_id[3]), + .blink_bankWrite_3_rob_id(intf.bank_write_rob_id[3]), + .blink_bankWrite_3_group_id(intf.bank_write_group_id[3]), + .blink_bankWrite_3_io_req_ready(intf.bank_write_req_ready[3]), + .blink_bankWrite_3_io_req_valid(intf.bank_write_req_valid[3]), + .blink_bankWrite_3_io_req_bits_addr(intf.bank_write_req_addr[3]), + .blink_bankWrite_3_io_req_bits_mask_0(intf.bank_write_req_mask[3][0]), + .blink_bankWrite_3_io_req_bits_mask_1(intf.bank_write_req_mask[3][1]), + .blink_bankWrite_3_io_req_bits_mask_2(intf.bank_write_req_mask[3][2]), + .blink_bankWrite_3_io_req_bits_mask_3(intf.bank_write_req_mask[3][3]), + .blink_bankWrite_3_io_req_bits_mask_4(intf.bank_write_req_mask[3][4]), + .blink_bankWrite_3_io_req_bits_mask_5(intf.bank_write_req_mask[3][5]), + .blink_bankWrite_3_io_req_bits_mask_6(intf.bank_write_req_mask[3][6]), + .blink_bankWrite_3_io_req_bits_mask_7(intf.bank_write_req_mask[3][7]), + .blink_bankWrite_3_io_req_bits_mask_8(intf.bank_write_req_mask[3][8]), + .blink_bankWrite_3_io_req_bits_mask_9(intf.bank_write_req_mask[3][9]), + .blink_bankWrite_3_io_req_bits_mask_10(intf.bank_write_req_mask[3][10]), + .blink_bankWrite_3_io_req_bits_mask_11(intf.bank_write_req_mask[3][11]), + .blink_bankWrite_3_io_req_bits_mask_12(intf.bank_write_req_mask[3][12]), + .blink_bankWrite_3_io_req_bits_mask_13(intf.bank_write_req_mask[3][13]), + .blink_bankWrite_3_io_req_bits_mask_14(intf.bank_write_req_mask[3][14]), + .blink_bankWrite_3_io_req_bits_mask_15(intf.bank_write_req_mask[3][15]), + .blink_bankWrite_3_io_req_bits_data(intf.bank_write_req_data[3]), + .blink_bankWrite_3_io_resp_valid(intf.bank_write_resp_valid[3]), + .blink_bankWrite_3_io_resp_bits_ok(intf.bank_write_resp_ok[3]) ); + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_resp_ready[0] = dut.blink_bankWrite_0_io_resp_ready; + assign intf.bank_write_resp_ready[1] = dut.blink_bankWrite_1_io_resp_ready; + assign intf.bank_write_resp_ready[2] = dut.blink_bankWrite_2_io_resp_ready; + assign intf.bank_write_resp_ready[3] = dut.blink_bankWrite_3_io_resp_ready; + initial begin intf.clock = 1'b0; forever #5 intf.clock = ~intf.clock; @@ -76,6 +180,7 @@ module tb_top; initial begin uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + uvm_config_db#(virtual gemmini_expansion_if)::set(null, "*", "expansion", expansion); run_test("gemmini_ball_test"); end endmodule diff --git a/examples/components/balls/gemmini/verify/src/tests/gemmini_test.svh b/examples/components/balls/gemmini/verify/src/tests/gemmini_test.svh index f986e8e0d..f71d49588 100644 --- a/examples/components/balls/gemmini/verify/src/tests/gemmini_test.svh +++ b/examples/components/balls/gemmini/verify/src/tests/gemmini_test.svh @@ -3,6 +3,7 @@ class gemmini_ball_test extends uvm_test; typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; vif_t vif; + virtual gemmini_expansion_if xif; function new(string name, uvm_component parent); super.new(name, parent); @@ -12,40 +13,41 @@ class gemmini_ball_test extends uvm_test; super.build_phase(phase); if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) `uvm_fatal("NOVIF", "bb_blink_if not found") + if (!uvm_config_db#(virtual gemmini_expansion_if)::get(this, "", "expansion", xif)) + `uvm_fatal("NOVIF", "gemmini_expansion_if not found") endfunction task run_phase(uvm_phase phase); - int cycles; - int unsigned bid; phase.raise_objection(this); apply_reset(); + run_config(); + run_loop_bounds(); + phase.drop_objection(this); + endtask + // bb_gemmini_config(dataflow=1, activation=0, a_transpose=1, b_transpose=0, in_shift=3): + // a BLink command answered by the execution controller without bank traffic. + task run_config(); + int cycles; + bit [63:0] rs2 = (64'd1 << 4) | (64'd1 << 7) | (64'd3 << 9); @(negedge vif.clock); - if (!$value$plusargs("BID=%d", bid)) `uvm_fatal("BID", "+BID is required") - vif.cmd_req_bits_cmd_bid = bid[4:0]; - vif.cmd_req_bits_cmd_funct7 = `GEMMINI_LOOP_WS_CONFIG_BOUNDS_FUNCT7; - vif.cmd_req_bits_cmd_iter = '0; - vif.cmd_req_bits_cmd_op1_en = 1'b0; - vif.cmd_req_bits_cmd_op2_en = 1'b0; - vif.cmd_req_bits_cmd_wr_spad_en = 1'b0; - vif.cmd_req_bits_cmd_op1_from_spad = 1'b0; - vif.cmd_req_bits_cmd_op2_from_spad = 1'b0; - vif.cmd_req_bits_cmd_special = 64'h0003_0002_0001; - vif.cmd_req_bits_cmd_op1_bank = '0; - vif.cmd_req_bits_cmd_op2_bank = '0; - vif.cmd_req_bits_cmd_wr_bank = '0; - vif.cmd_req_bits_cmd_op1_col = '0; - vif.cmd_req_bits_cmd_op2_col = '0; - vif.cmd_req_bits_cmd_wr_col = '0; - vif.cmd_req_bits_cmd_meta_bank = '0; - vif.cmd_req_bits_cmd_rs1 = '0; - vif.cmd_req_bits_cmd_rs2 = '0; - vif.cmd_req_bits_rob_id = 4'h9; - vif.cmd_req_bits_is_sub = 1'b1; - vif.cmd_req_bits_sub_rob_id = 8'h5a; - vif.cmd_req_valid = 1'b1; - - cycles = 0; + vif.cmd_req_bits_rob_id = 'h5; + vif.cmd_req_bits_funct = `GEMMINI_CONFIG_FUNCT7; + vif.cmd_req_bits_rs2 = rs2; + vif.cmd_req_bits_iter = '0; + vif.cmd_req_bits_op1_bank = '0; + vif.cmd_req_bits_op2_bank = '0; + vif.cmd_req_bits_wr_bank = '0; + vif.cmd_req_bits_op1_group = '0; + vif.cmd_req_bits_op2_group = '0; + vif.cmd_req_bits_wr_group = '0; + vif.cmd_req_bits_op1_col = '0; + vif.cmd_req_bits_op2_col = '0; + vif.cmd_req_bits_wr_col = '0; + vif.cmd_req_bits_wr_read_bank = '0; + vif.cmd_req_bits_wr_arch_bank = '0; + vif.cmd_req_valid = 1'b1; + cycles = 0; do begin @(posedge vif.clock); cycles++; @@ -54,32 +56,81 @@ class gemmini_ball_test extends uvm_test; @(negedge vif.clock); vif.cmd_req_valid = 1'b0; - if (!vif.cmd_resp_valid) `uvm_fatal("RESP", "Gemmini config response was not observed") - if (vif.cmd_resp_bits_rob_id !== 4'h9 || vif.cmd_resp_bits_is_sub !== 1'b1 || - vif.cmd_resp_bits_sub_rob_id !== 8'h5a) - `uvm_fatal("RESP", $sformatf( - "metadata mismatch: rob=%0h is_sub=%0b sub_rob=%0h", - vif.cmd_resp_bits_rob_id, - vif.cmd_resp_bits_is_sub, - vif.cmd_resp_bits_sub_rob_id - )) - for (int i = 0; i < `BB_IN_BW; i++) begin - if (vif.bank_read_req_valid[i]) - `uvm_fatal("BANK", "Gemmini config command unexpectedly read a bank") - end - for (int i = 0; i < `BB_OUT_BW; i++) begin - if (vif.bank_write_req_valid[i]) - `uvm_fatal("BANK", "Gemmini config command unexpectedly wrote a bank") + cycles = 0; + while (!vif.cmd_resp_valid) begin + check_no_bank_traffic("config"); + @(negedge vif.clock); + cycles++; + if (cycles > 20) `uvm_fatal("RESP", "Gemmini config response was not observed") end + verification_pkg::evidence::checks++; + if (vif.cmd_resp_bits_rob_id !== 'h5) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0h", vif.cmd_resp_bits_rob_id)) + @(negedge vif.clock); + `uvm_info("GEMMINI", "config command passed", UVM_LOW) + endtask - `uvm_info("GEMMINI", "loop-ws config command passed", UVM_LOW) - phase.drop_objection(this); + // bb_gemmini_loop_ws_config_bounds(max_i=1, max_j=2, max_k=3) arrives on the SubROB + // expansion port and is stored little-endian into loop metadata bytes 0..7. + task run_loop_bounds(); + int cycles; + int unsigned writes; + bit [63:0] rs2 = 64'h0001_0002_0003; + @(negedge vif.clock); + xif.command_bits_rob_id = 'h9; + xif.command_bits_funct = `GEMMINI_LOOP_WS_CONFIG_BOUNDS_FUNCT7; + xif.command_bits_rs2 = rs2; + xif.command_valid = 1'b1; + #1; + verification_pkg::evidence::checks++; + if (xif.accepts !== 1'b1 || xif.starts !== 1'b0) + `uvm_fatal("EXPANSION", "loop bounds config must be accepted without starting a loop") + cycles = 0; + do begin + @(posedge vif.clock); + cycles++; + if (cycles > 20) `uvm_fatal("TIMEOUT", "loop bounds command was not accepted") + end while (!xif.command_ready); + @(negedge vif.clock); + xif.command_valid = 1'b0; + + writes = 0; + cycles = 0; + while (!xif.complete_valid) begin + check_no_bank_traffic("loop bounds"); + if (xif.write_valid) begin + verification_pkg::evidence::checks++; + if (xif.write_bits_addr !== writes || xif.write_bits_data !== rs2[writes*8+:8]) + `uvm_fatal("EXPANSION", $sformatf( + "metadata write %0d: addr=%0d data=%0h", + writes, + xif.write_bits_addr, + xif.write_bits_data + )) + writes++; + end + @(negedge vif.clock); + cycles++; + if (cycles > 40) `uvm_fatal("TIMEOUT", "loop bounds command did not complete") + end + verification_pkg::evidence::checks++; + if (writes != 8 || xif.complete_bits_rob_id !== 'h9) + `uvm_fatal("EXPANSION", $sformatf( + "completion mismatch: writes=%0d rob=%0h", writes, xif.complete_bits_rob_id)) + @(negedge vif.clock); + `uvm_info("GEMMINI", "loop-ws bounds config passed", UVM_LOW) endtask + function void check_no_bank_traffic(string what); + for (int i = 0; i < `BB_IN_BW; i++) + if (vif.bank_read_req_valid[i]) `uvm_fatal("BANK", {what, " unexpectedly read a bank"}) + for (int i = 0; i < `BB_OUT_BW; i++) + if (vif.bank_write_req_valid[i]) `uvm_fatal("BANK", {what, " unexpectedly wrote a bank"}) + endfunction + task apply_reset(); - vif.cmd_req_valid = 1'b0; + vif.cmd_req_valid = 1'b0; vif.cmd_resp_ready = 1'b1; - vif.sub_rob_req_ready = 1'b1; for (int i = 0; i < `BB_IN_BW; i++) begin vif.bank_read_req_ready[i] = 1'b1; vif.bank_read_resp_valid[i] = 1'b0; @@ -90,6 +141,9 @@ class gemmini_ball_test extends uvm_test; vif.bank_write_resp_valid[i] = 1'b0; vif.bank_write_resp_ok[i] = 1'b1; end + xif.command_valid = 1'b0; + xif.complete_ready = 1'b1; + xif.write_ready = 1'b1; vif.reset = 1'b1; repeat (5) @(posedge vif.clock); vif.reset = 1'b0; diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir b/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir index 0dde99a71..c7e2d6913 100644 --- a/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir +++ b/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir @@ -9,7 +9,6 @@ func.func @main() -> i8 { %five = arith.constant 5 : index %seven = arith.constant 7 : index %sixteen = arith.constant 16 : index - %base = arith.constant 0 : i64 %a = memref.alloc() : memref<16x16xi8> %b = memref.alloc() : memref<16x16xi8> %c = memref.alloc() : memref<16x16xi32> @@ -32,18 +31,18 @@ func.func @main() -> i8 { memref.store %bv, %b[%row, %col] : memref<16x16xi8> } } - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 4 : i64}> - buckyball.mvin %a <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - buckyball.mvin %b <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %a <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %b <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> buckyball.mvin %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> // C = in_shift(C + A @ B, 4). - buckyball.custom "matmul_i8_tile" args(%base, %base, %base : i64, i64, i64) <{sources = array, destination = 3 : i64, read_destination}> {shift = 4 : i64} + buckyball.custom "matmul_i8_tile" <{sources = array, destination = 3 : i64, read_destination}> {k = 16 : i64, shift = 4 : i64} buckyball.mvout %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> func.call @check_result(%c) : (memref<16x16xi32>) -> () - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> memref.dealloc %a : memref<16x16xi8> memref.dealloc %b : memref<16x16xi8> diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir b/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir index b825b9083..1e4b9d715 100644 --- a/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir +++ b/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir @@ -9,7 +9,6 @@ func.func @main() -> i8 { %five = arith.constant 5 : index %seven = arith.constant 7 : index %sixteen = arith.constant 16 : index - %base = arith.constant 0 : i64 %a = memref.alloc() : memref<16x16xi8> %b = memref.alloc() : memref<16x16xi8> %c = memref.alloc() : memref<16x16xi32> @@ -32,17 +31,17 @@ func.func @main() -> i8 { memref.store %bv, %b[%row, %col] : memref<16x16xi8> } } - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 4 : i64}> - buckyball.mvin %a <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - buckyball.mvin %b <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %a <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %b <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> // C = A @ B on the output-stationary array. - buckyball.custom "matmul_i8_tile" args(%base, %base, %base : i64, i64, i64) <{sources = array, destination = 3 : i64}> + buckyball.custom "matmul_i8_tile" <{sources = array, destination = 3 : i64}> {k = 16 : i64} buckyball.mvout %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> func.call @check_result(%c) : (memref<16x16xi32>) -> () - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> memref.dealloc %a : memref<16x16xi8> memref.dealloc %b : memref<16x16xi8> diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt index 7bc2d80e5..d38664730 100644 --- a/examples/components/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt +++ b/examples/components/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt @@ -5,9 +5,9 @@ set(BUCKYBALL_MLIR_TEST_PREFIX bank) set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") -if(_lines LESS 16) +if(NOT _lines EQUAL 1024) message(FATAL_ERROR - "Gemmini BankOp test requires BANK_LINES >= 16 (got ${_lines})") + "Gemmini BankOp tests require BANK_LINES = 1024 (got ${_lines})") endif() add_buckyball_mlir_test(gemmini_compute_preloaded_bank_tiles TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) add_buckyball_mlir_test(gemmini_compute_accumulated_bank_tiles TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir new file mode 100644 index 000000000..4bbbe0519 --- /dev/null +++ b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir @@ -0,0 +1,97 @@ +func.func private @check_result(memref<16x16xi32>) -> () + +// Each group holds a whole bank of K: A as blocks of sixteen rows, B as one +// row per K. A[i, kk] = (3i + 5kk) % 7 - 3 with kk = first + 16 * (q / 16) + col +// at bank row q, i = q % 16. +func.func @fill_a(%a: memref<1024x16xi8>, %first: index) { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c16 = arith.constant 16 : index + %lines = arith.constant 1024 : index + scf.for %q = %c0 to %lines step %c1 { + %i = arith.remui %q, %c16 : index + %block = arith.subi %q, %i : index + %start = arith.addi %first, %block : index + scf.for %col = %c0 to %c16 step %c1 { + %kk = arith.addi %start, %col : index + %i3 = arith.muli %i, %c3 : index + %k5 = arith.muli %kk, %c5 : index + %sum = arith.addi %i3, %k5 : index + %rem = arith.remui %sum, %c7 : index + %value = arith.subi %rem, %c3 : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %a[%q, %col] : memref<1024x16xi8> + } + } + return +} + +// B[kk, j] = (2kk + 3j) % 5 - 2 with kk = first + q at bank row q. +func.func @fill_b(%b: memref<1024x16xi8>, %first: index) { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c5 = arith.constant 5 : index + %c16 = arith.constant 16 : index + %lines = arith.constant 1024 : index + scf.for %q = %c0 to %lines step %c1 { + %kk = arith.addi %first, %q : index + scf.for %j = %c0 to %c16 step %c1 { + %k2 = arith.muli %kk, %c2 : index + %j3 = arith.muli %j, %c3 : index + %sum = arith.addi %k2, %j3 : index + %rem = arith.remui %sum, %c5 : index + %value = arith.subi %rem, %c2 : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %b[%q, %j] : memref<1024x16xi8> + } + } + return +} + +// C = in_shift(7 + A[16 x 2048] @ B[2048 x 16], 2), A and B each gathered +// from two full 1024-row banks. +func.func @main() -> i8 { + %z8 = arith.constant 0 : i8 + %initial = arith.constant 7 : i32 + %c0 = arith.constant 0 : index + %lines = arith.constant 1024 : index + %a0 = memref.alloc() : memref<1024x16xi8> + %a1 = memref.alloc() : memref<1024x16xi8> + %b0 = memref.alloc() : memref<1024x16xi8> + %b1 = memref.alloc() : memref<1024x16xi8> + %c = memref.alloc() : memref<16x16xi32> + linalg.fill ins(%initial : i32) outs(%c : memref<16x16xi32>) + func.call @fill_a(%a0, %c0) : (memref<1024x16xi8>, index) -> () + func.call @fill_a(%a1, %lines) : (memref<1024x16xi8>, index) -> () + func.call @fill_b(%b0, %c0) : (memref<1024x16xi8>, index) -> () + func.call @fill_b(%b1, %lines) : (memref<1024x16xi8>, index) -> () + %ab0 = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %bb0 = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %ab1 = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %bb1 = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %cb = buckyball.bank_alloc <{row = 1 : i64, col = 4 : i64}> + %al0 = buckyball.bank_mvin %a0, %ab0 <{depth = 1024 : i64, stride = 1 : i64}> : memref<1024x16xi8> + %bl0 = buckyball.bank_mvin %b0, %bb0 <{depth = 1024 : i64, stride = 1 : i64}> : memref<1024x16xi8> + %al1 = buckyball.bank_mvin %a1, %ab1 <{depth = 1024 : i64, stride = 1 : i64}> : memref<1024x16xi8> + %bl1 = buckyball.bank_mvin %b1, %bb1 <{depth = 1024 : i64, stride = 1 : i64}> : memref<1024x16xi8> + %cl = buckyball.bank_mvin %c, %cb <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + %al = buckyball.bank_transfer %al1, %al0 + %bl = buckyball.bank_transfer %bl1, %bl0 + %r = buckyball.bank_custom "matmul_i8_tile" (%al, %bl) -> %cl <{read_destination}> {k = 2048 : i64, shift = 2 : i64} + %stored = buckyball.bank_mvout %c, %r <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + func.call @check_result(%c) : (memref<16x16xi32>) -> () + buckyball.bank_release %al + buckyball.bank_release %bl + buckyball.bank_release %stored + memref.dealloc %a0 : memref<1024x16xi8> + memref.dealloc %a1 : memref<1024x16xi8> + memref.dealloc %b0 : memref<1024x16xi8> + memref.dealloc %b1 : memref<1024x16xi8> + memref.dealloc %c : memref<16x16xi32> + return %z8 : i8 +} diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir.in b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir.in deleted file mode 100644 index 75c15d6f6..000000000 --- a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir.in +++ /dev/null @@ -1,63 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x16xi32>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %z32 = arith.constant 0 : i32 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %dim = arith.constant 16 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %zero64 = arith.constant 0 : i64 - %a = memref.alloc() : memref<@BB_BANK_LINES@x16xi8> - %b = memref.alloc() : memref<16x16xi8> - %c = memref.alloc() : memref<@BB_BANK_LINES@x16xi32> - linalg.fill ins(%z32 : i32) outs(%c : memref<@BB_BANK_LINES@x16xi32>) - scf.for %tile = %z to %rows step %dim { - scf.for %row = %z to %dim step %one { - %arow = arith.addi %tile, %row : index - scf.for %k = %z to %dim step %one { - %r3 = arith.muli %row, %three : index - %k5 = arith.muli %k, %five : index - %sum = arith.addi %r3, %k5 : index - %rem = arith.remui %sum, %seven : index - %value = arith.subi %rem, %three : index - %v8 = arith.index_cast %value : index to i8 - memref.store %v8, %a[%arow, %k] : memref<@BB_BANK_LINES@x16xi8> - } - } - } - scf.for %row = %z to %dim step %one { - scf.for %col = %z to %dim step %one { - %r2 = arith.muli %row, %two : index - %c3 = arith.muli %col, %three : index - %sum = arith.addi %r2, %c3 : index - %rem = arith.remui %sum, %five : index - %value = arith.subi %rem, %two : index - %v8 = arith.index_cast %value : index to i8 - memref.store %v8, %b[%row, %col] : memref<16x16xi8> - } - } - %ab = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> - %bb = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> - %cb = buckyball.bank_alloc <{row = 1 : i64, col = 4 : i64}> - %al = buckyball.bank_mvin %a, %ab <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x16xi8> - %bl = buckyball.bank_mvin %b, %bb <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - // Tile t of A (rows 16t..16t+15) times B into rows 16t..16t+15 of C. - scf.for %base = %z to %rows step %dim { - %base64 = arith.index_cast %base : index to i64 - %r = buckyball.bank_custom "matmul_i8_tile" (%al, %bl) -> %cb args(%base64, %zero64, %base64 : i64, i64, i64) - %sum = buckyball.bank_custom "matmul_i8_tile" (%al, %bl) -> %r args(%base64, %zero64, %base64 : i64, i64, i64) <{read_destination}> {shift = 0 : i64} - } - %stored = buckyball.bank_mvout %c, %cb <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x16xi32> - func.call @check_result(%c) : (memref<@BB_BANK_LINES@x16xi32>) -> () - buckyball.bank_release %al - buckyball.bank_release %bl - buckyball.bank_release %stored - memref.dealloc %a : memref<@BB_BANK_LINES@x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %c : memref<@BB_BANK_LINES@x16xi32> - return %z8 : i8 -} diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp index 225063d92..85ade8e02 100644 --- a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp +++ b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp @@ -2,17 +2,23 @@ #include #include #include +static_assert(BANK_LINES == 1024); extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { - if (rows != BANK_LINES || lanes != 16 || stride != 16 || lane_stride != 1) + if (rows != 16 || lanes != 16 || stride != 16 || lane_stride != 1) exit(1); - for (int row = 0; row < BANK_LINES; ++row) + for (int row = 0; row < 16; ++row) for (int column = 0; column < 16; ++column) { - int32_t expected = 0; - for (int k = 0; k < 16; ++k) - expected += 2 * ((3 * (row % 16) + 5 * k) % 7 - 3) * - ((2 * k + 3 * column) % 5 - 2); + int32_t expected = 7; + for (int k = 0; k < 2048; ++k) + expected += + ((3 * row + 5 * k) % 7 - 3) * ((2 * k + 3 * column) % 5 - 2); + // in_shift by two, rounding to nearest even. + const uint32_t bits = (uint32_t)expected; + const int rounded = + ((bits >> 1) & 1) && ((bits & 1) || ((bits >> 2) & 1)); + expected = (expected >> 2) + rounded; if (data[offset + row * stride + column] != expected) { printf("gemmini accumulated mismatch row=%d column=%d expected=%d " "actual=%d\n", @@ -20,5 +26,5 @@ extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, exit(1); } } - printf("gemmini compute-accumulated bank cores PASS rows=%d\n", BANK_LINES); + printf("gemmini matmul_i8_tile read_destination over two groups PASS\n"); } diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir new file mode 100644 index 000000000..c656eaca6 --- /dev/null +++ b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir @@ -0,0 +1,77 @@ +func.func private @check_result(memref<16x16xi32>) -> () + +// A holds K = 96 as six blocks of sixteen rows: +// A[i, kk] = (3i + 5kk) % 7 - 3 with kk = 16 * (q / 16) + col at bank row q, +// i = q % 16. +func.func @fill_a(%a: memref<96x16xi8>) { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c16 = arith.constant 16 : index + %c96 = arith.constant 96 : index + scf.for %q = %c0 to %c96 step %c1 { + %i = arith.remui %q, %c16 : index + %block = arith.subi %q, %i : index + scf.for %col = %c0 to %c16 step %c1 { + %kk = arith.addi %block, %col : index + %i3 = arith.muli %i, %c3 : index + %k5 = arith.muli %kk, %c5 : index + %sum = arith.addi %i3, %k5 : index + %rem = arith.remui %sum, %c7 : index + %value = arith.subi %rem, %c3 : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %a[%q, %col] : memref<96x16xi8> + } + } + return +} + +// B[kk, j] = (2kk + 3j) % 5 - 2 with kk = q at bank row q. +func.func @fill_b(%b: memref<96x16xi8>) { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c5 = arith.constant 5 : index + %c16 = arith.constant 16 : index + %c96 = arith.constant 96 : index + scf.for %q = %c0 to %c96 step %c1 { + scf.for %j = %c0 to %c16 step %c1 { + %k2 = arith.muli %q, %c2 : index + %j3 = arith.muli %j, %c3 : index + %sum = arith.addi %k2, %j3 : index + %rem = arith.remui %sum, %c5 : index + %value = arith.subi %rem, %c2 : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %b[%q, %j] : memref<96x16xi8> + } + } + return +} + +// C = A[16 x 96] @ B[96 x 16] in one operation of k = 96. +func.func @main() -> i8 { + %z8 = arith.constant 0 : i8 + %a = memref.alloc() : memref<96x16xi8> + %b = memref.alloc() : memref<96x16xi8> + %c = memref.alloc() : memref<16x16xi32> + func.call @fill_a(%a) : (memref<96x16xi8>) -> () + func.call @fill_b(%b) : (memref<96x16xi8>) -> () + %ab = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %bb = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %cb = buckyball.bank_alloc <{row = 1 : i64, col = 4 : i64}> + %al = buckyball.bank_mvin %a, %ab <{depth = 96 : i64, stride = 1 : i64}> : memref<96x16xi8> + %bl = buckyball.bank_mvin %b, %bb <{depth = 96 : i64, stride = 1 : i64}> : memref<96x16xi8> + %r = buckyball.bank_custom "matmul_i8_tile" (%al, %bl) -> %cb {k = 96 : i64} + %stored = buckyball.bank_mvout %c, %r <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + func.call @check_result(%c) : (memref<16x16xi32>) -> () + buckyball.bank_release %al + buckyball.bank_release %bl + buckyball.bank_release %stored + memref.dealloc %a : memref<96x16xi8> + memref.dealloc %b : memref<96x16xi8> + memref.dealloc %c : memref<16x16xi32> + return %z8 : i8 +} diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir.in b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir.in deleted file mode 100644 index b191e7234..000000000 --- a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir.in +++ /dev/null @@ -1,62 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x16xi32>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %z32 = arith.constant 0 : i32 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %dim = arith.constant 16 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %zero64 = arith.constant 0 : i64 - %a = memref.alloc() : memref<@BB_BANK_LINES@x16xi8> - %b = memref.alloc() : memref<16x16xi8> - %c = memref.alloc() : memref<@BB_BANK_LINES@x16xi32> - linalg.fill ins(%z32 : i32) outs(%c : memref<@BB_BANK_LINES@x16xi32>) - scf.for %tile = %z to %rows step %dim { - scf.for %row = %z to %dim step %one { - %arow = arith.addi %tile, %row : index - scf.for %k = %z to %dim step %one { - %r3 = arith.muli %row, %three : index - %k5 = arith.muli %k, %five : index - %sum = arith.addi %r3, %k5 : index - %rem = arith.remui %sum, %seven : index - %value = arith.subi %rem, %three : index - %v8 = arith.index_cast %value : index to i8 - memref.store %v8, %a[%arow, %k] : memref<@BB_BANK_LINES@x16xi8> - } - } - } - scf.for %row = %z to %dim step %one { - scf.for %col = %z to %dim step %one { - %r2 = arith.muli %row, %two : index - %c3 = arith.muli %col, %three : index - %sum = arith.addi %r2, %c3 : index - %rem = arith.remui %sum, %five : index - %value = arith.subi %rem, %two : index - %v8 = arith.index_cast %value : index to i8 - memref.store %v8, %b[%row, %col] : memref<16x16xi8> - } - } - %ab = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> - %bb = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> - %cb = buckyball.bank_alloc <{row = 1 : i64, col = 4 : i64}> - %al = buckyball.bank_mvin %a, %ab <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x16xi8> - %bl = buckyball.bank_mvin %b, %bb <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - // Tile t of A (rows 16t..16t+15) times B into rows 16t..16t+15 of C. - scf.for %base = %z to %rows step %dim { - %base64 = arith.index_cast %base : index to i64 - %r = buckyball.bank_custom "matmul_i8_tile" (%al, %bl) -> %cb args(%base64, %zero64, %base64 : i64, i64, i64) - } - %stored = buckyball.bank_mvout %c, %cb <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x16xi32> - func.call @check_result(%c) : (memref<@BB_BANK_LINES@x16xi32>) -> () - buckyball.bank_release %al - buckyball.bank_release %bl - buckyball.bank_release %stored - memref.dealloc %a : memref<@BB_BANK_LINES@x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %c : memref<@BB_BANK_LINES@x16xi32> - return %z8 : i8 -} diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp index fc21b551d..e8160d0dd 100644 --- a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp +++ b/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp @@ -1,23 +1,22 @@ -#include #include #include #include extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { - if (rows != BANK_LINES || lanes != 16 || stride != 16 || lane_stride != 1) + if (rows != 16 || lanes != 16 || stride != 16 || lane_stride != 1) exit(1); - for (int row = 0; row < BANK_LINES; ++row) + for (int row = 0; row < 16; ++row) for (int column = 0; column < 16; ++column) { int32_t expected = 0; - for (int k = 0; k < 16; ++k) + for (int k = 0; k < 96; ++k) expected += - ((3 * (row % 16) + 5 * k) % 7 - 3) * ((2 * k + 3 * column) % 5 - 2); + ((3 * row + 5 * k) % 7 - 3) * ((2 * k + 3 * column) % 5 - 2); if (data[offset + row * stride + column] != expected) { printf("gemmini bank mismatch row=%d column=%d expected=%d actual=%d\n", row, column, expected, data[offset + row * stride + column]); exit(1); } } - printf("gemmini compute-preloaded bank cores PASS rows=%d\n", BANK_LINES); + printf("gemmini matmul_i8_tile k=96 PASS\n"); } diff --git a/examples/components/balls/im2col/arch/Im2col.scala b/examples/components/balls/im2col/arch/Im2col.scala index eed182542..59705f0db 100644 --- a/examples/components/balls/im2col/arch/Im2col.scala +++ b/examples/components/balls/im2col/arch/Im2col.scala @@ -9,11 +9,21 @@ import framework.acc.memdomain.configs.MemDomainParam import chisel3._ import framework.acc.balldomain.ball.blink.{BallCommand, BallResponse} import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} +import chisel3.experimental.hierarchy.{instantiable, public, Instantiate} import framework.acc.balldomain.ball.blink.BallStatus import framework.acc.memdomain.{BankRead, BankWrite} import examples.balls.im2col.configs.Im2colBallParam +/** + * Im2col over channel-block groups. + * + * Input group g is channel block g; line inputBase + y * size + x is pixel + * (y, x). K is ordered (kernel row, kernel column, channel), so K tile + * t = (kr * kernel + kc) * inputGroups + g is one whole input row (zero in + * the padding). Output group j holds K tiles j * groupTiles .. in the SMatMul + * A layout: line ((w / tileRows) * groupTiles + localTile) * tileRows + + * w % tileRows. Windows beyond windowCount in the last M tile are zero. + */ @instantiable class Im2col( val frontendParam: FrontendParam, @@ -22,246 +32,302 @@ class Im2col( val instance: BallRouteParam, val ballParam: Im2colBallParam) extends Module { - val commandDataBits = frontendParam.commandDataBits - val ballCfg = ballParam - val maxIter = ballCfg.maxIter - val maxKSize = ballCfg.maxKSize - val maxPad = ballCfg.maxPadding - val core = 16 - - val mapping = instance - - require(mapping.inBW == 1, "Im2colBall requires inBW=1") - require(mapping.outBW == 1, "Im2colBall requires outBW=1") + val maxIter = ballParam.maxIter + val maxKSize = ballParam.maxKSize + val maxPad = ballParam.maxPadding + val addressWidth = log2Ceil(memParam.bankEntries) + val groupWidth = log2Up(memParam.bankNum + 1) + + require(instance.inBW == 1, "Im2colBall requires inBW=1") + require(instance.outBW == 1, "Im2colBall requires outBW=1") require(memParam.bankWidth == 128, "Im2colBall requires 128-bit bank rows") - require( - isPow2(memParam.bankEntries), - "Im2colBall requires power-of-two bank depth" - ) + require(ballParam.inputWidth == 8, "Im2colBall moves 16 INT8 channels per row") + require(isPow2(memParam.bankEntries), "Im2colBall requires power-of-two bank depth") require(maxIter + 2 * maxPad <= 255, "Im2colBall dimensions must fit the divider") + require(maxKSize <= 15 && maxPad <= 15, "Im2colBall kernel and padding fields are four bits") @public val io = IO(new Bundle { - - val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) - - val cmdResp = Decoupled(new BallResponse(frontendParam)) - - val bankRead = Vec( - mapping.inBW, - Flipped(new BankRead( - frontendParam, - memParam - )) - ) - - val bankWrite = Vec( - mapping.outBW, - Flipped(new BankWrite( - frontendParam, - memParam - )) - ) - - val status = new BallStatus + val cmdReq = Flipped(Decoupled(new BallCommand(frontendParam, memParam))) + val cmdResp = Decoupled(new BallResponse(frontendParam)) + val bankRead = Vec(1, Flipped(new BankRead(frontendParam, memParam))) + val bankWrite = Vec(1, Flipped(new BankWrite(frontendParam, memParam))) + val status = new BallStatus }) - val cfg = - Instantiate(new Im2colConfigRegs(frontendParam, memParam, maxIter, maxKSize, maxPad, ballParam)) - - val win = - Instantiate(new Im2colWindow( - maxKSize, - ballParam - )) + val idle :: divRows :: divCols :: divStart :: divTiles :: check :: select :: readReq :: readResp :: writeReq :: writeResp :: complete :: Nil = + Enum(12) + val state = RegInit(idle) + + val robId = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + val inBank = RegInit(0.U(frontendParam.bank_id_len.W)) + val outBank = RegInit(0.U(frontendParam.bank_id_len.W)) + val inBase = RegInit(0.U(memParam.groupIdWidth.W)) + val outBase = RegInit(0.U(memParam.groupIdWidth.W)) + val inGroups = RegInit(0.U(groupWidth.W)) + val outGroups = RegInit(0.U(groupWidth.W)) + val size = RegInit(0.U(8.W)) + val kernel = RegInit(0.U(4.W)) + val stride = RegInit(0.U(4.W)) + val padding = RegInit(0.U(4.W)) + val startCol = RegInit(0.U(4.W)) + val startRow = RegInit(0.U(4.W)) + val inputBase = RegInit(0.U(6.W)) + val windowStart = RegInit(0.U(10.W)) + val windowCount = RegInit(0.U(10.W)) + val tileShift = RegInit(0.U(3.W)) + val outRows = RegInit(0.U(16.W)) + val outCols = RegInit(0.U(16.W)) + val firstRow = RegInit(0.U(16.W)) + val firstCol = RegInit(0.U(16.W)) + val groupTiles = RegInit(0.U(16.W)) + + // Current K tile and output window. + val kernelRow = RegInit(0.U(4.W)) + val kernelCol = RegInit(0.U(4.W)) + val inGroup = RegInit(0.U(groupWidth.W)) + val outGroup = RegInit(0.U(groupWidth.W)) + val localTile = RegInit(0.U(16.W)) + val window = RegInit(0.U(11.W)) + val windowRow = RegInit(0.U(16.W)) + val windowCol = RegInit(0.U(16.W)) + val rowData = Reg(UInt(128.W)) val div = Instantiate(new RestoringDiv(16, 8, ballParam)) - - val lineBuf: Instance[LineBufferManager] = - Instantiate(new LineBufferManager(frontendParam, memParam, routingParam, instance, ballParam)) - val writer: Instance[StreamWriter] = - Instantiate(new StreamWriter(frontendParam, memParam, routingParam, instance, ballParam)) - - val cIdle :: cDivRow :: cDivCol :: cCheck :: cDivStart :: cLaunch :: Nil = Enum(6) - val calc = RegInit(cIdle) - - val running = RegInit(false.B) - val inputReady = RegInit(false.B) - val respPending = RegInit(false.B) - val armed = RegInit(false.B) - val outRowsReg = RegInit(0.U(16.W)) - val outColsReg = RegInit(0.U(16.W)) - val startOutRow = RegInit(0.U(16.W)) - val startOutCol = RegInit(0.U(16.W)) - val localWindow = RegInit(0.U(12.W)) - val elemAge = RegInit(0.U(2.W)) - val launch = WireDefault(false.B) - - cfg.io.cmd := io.cmdReq.bits - cfg.io.load := io.cmdReq.fire - - io.cmdReq.ready := !running && !respPending && !armed && calc === cIdle - io.cmdResp.valid := respPending - io.cmdResp.bits.rob_id := cfg.io.robId - io.status.idle := !running && !respPending && !armed && calc === cIdle - io.status.running := running || armed || calc =/= cIdle - div.io.start := false.B div.io.a := 0.U div.io.b := 0.U - when(io.cmdReq.fire) { - val command = io.cmdReq.bits - - assert(command.rs2(63) === 0.U, "IM2COL reserves rs2[63]") - assert( - command.iter(frontendParam.iter_len - 1, 16) === 0.U, - "IM2COL input size exceeds 16 bits" - ) - assert( - command.op1_col === 1.U && command.op2_col === 0.U && command.wr_col === 1.U, - "IM2COL requires one physical input bank and one physical output bank" - ) - assert( - command.rs2(7, 0) > 0.U && command.rs2(7, 0) <= maxKSize.U, - "IM2COL kernel must be in the configured range" - ) - armed := true.B + val padded = size +& (padding << 1) + val tileRows = 1.U(5.W) << tileShift + val paddedWindows = ((windowCount +& tileRows - 1.U) >> tileShift) << tileShift + val sourceRow = startRow +& windowRow * stride +& kernelRow + val sourceCol = startCol +& windowCol * stride +& kernelCol + val inBounds = sourceRow >= padding && sourceRow < padding +& size && + sourceCol >= padding && sourceCol < padding +& size + val sourceLine = inputBase +& (sourceRow - padding) * size +& (sourceCol - padding) + val outputLine = (((window >> tileShift) * groupTiles +& localTile) << tileShift) | + (window & (tileRows - 1.U)) + val lastTile = kernelRow +& 1.U === kernel && kernelCol +& 1.U === kernel && inGroup +& 1.U === inGroups + + io.cmdReq.ready := state === idle + io.cmdResp.valid := state === complete + io.cmdResp.bits.rob_id := robId + io.status.idle := state === idle + io.status.running := state =/= idle && state =/= complete + + io.bankRead(0).rob_id := robId + io.bankRead(0).bank_id := inBank + io.bankRead(0).group_id := inBase + inGroup + io.bankRead(0).io.req.valid := state === readReq + io.bankRead(0).io.req.bits.addr := sourceLine(addressWidth - 1, 0) + io.bankRead(0).io.resp.ready := state === readResp + + io.bankWrite(0).rob_id := robId + io.bankWrite(0).bank_id := outBank + io.bankWrite(0).group_id := outBase + outGroup + io.bankWrite(0).io.req.valid := state === writeReq + io.bankWrite(0).io.req.bits.addr := outputLine(addressWidth - 1, 0) + io.bankWrite(0).io.req.bits.data := rowData + io.bankWrite(0).io.req.bits.mask := VecInit(Seq.fill(memParam.bankMaskLen)(true.B)) + io.bankWrite(0).io.resp.ready := state === writeResp + + switch(state) { + is(idle) { + when(io.cmdReq.fire) { + val cmd = io.cmdReq.bits + val tileValue = cmd.rs2(50, 46) + assert(cmd.rs2(63, 51) === 0.U, "IM2COL reserves rs2[63:51]") + assert( + cmd.iter =/= 0.U && cmd.iter <= maxIter.U, + "IM2COL input size must be in the configured range" + ) + assert( + cmd.rs2(3, 0) =/= 0.U && cmd.rs2(3, 0) <= maxKSize.U, + "IM2COL kernel must be in the configured range" + ) + assert(cmd.rs2(7, 4) =/= 0.U, "IM2COL stride must be positive") + assert(cmd.rs2(11, 8) <= maxPad.U, "IM2COL padding exceeds the configured range") + assert( + cmd.rs2(15, 12) <= cmd.rs2(11, 8) && cmd.rs2(19, 16) <= cmd.rs2(11, 8), + "IM2COL start must not exceed padding" + ) + assert(cmd.rs2(45, 36) =/= 0.U, "IM2COL window count must be positive") + assert(PopCount(tileValue) === 1.U && tileValue <= 16.U, "IM2COL tile rows must be 1, 2, 4, 8, or 16") + assert( + cmd.rs2(25, 20) +& cmd.iter * cmd.iter <= memParam.bankEntries.U, + "IM2COL input footprint exceeds bank depth" + ) + assert( + cmd.op1_col =/= 0.U && cmd.op2_col === 0.U && cmd.wr_col =/= 0.U && + cmd.op1_col <= memParam.bankNum.U && cmd.wr_col <= memParam.bankNum.U, + "IM2COL requires input and output groups" + ) + assert(cmd.op1_bank =/= cmd.wr_bank, "IM2COL input and output banks must differ") + assert( + cmd.iter +& (cmd.rs2(11, 8) << 1) >= cmd.rs2(3, 0) +& cmd.rs2(19, 16) && + cmd.iter +& (cmd.rs2(11, 8) << 1) >= cmd.rs2(3, 0) +& cmd.rs2(15, 12), + "IM2COL kernel and start exceed padded input" + ) + + robId := cmd.rob_id + inBank := cmd.op1_bank + outBank := cmd.wr_bank + inBase := cmd.op1_group + outBase := cmd.wr_group + inGroups := cmd.op1_col + outGroups := cmd.wr_col + size := cmd.iter + kernel := cmd.rs2(3, 0) + stride := cmd.rs2(7, 4) + padding := cmd.rs2(11, 8) + startCol := cmd.rs2(15, 12) + startRow := cmd.rs2(19, 16) + inputBase := cmd.rs2(25, 20) + windowStart := cmd.rs2(35, 26) + windowCount := cmd.rs2(45, 36) + tileShift := OHToUInt(tileValue) + state := divRows + div.io.start := true.B + div.io.a := cmd.iter +& (cmd.rs2(11, 8) << 1) - cmd.rs2(3, 0) - cmd.rs2(19, 16) + div.io.b := cmd.rs2(7, 4) + } + } + + is(divRows) { + when(div.io.done) { + assert(div.io.r === 0.U, "IM2COL output row shape must be integral") + outRows := div.io.q + 1.U + div.io.start := true.B + div.io.a := padded - kernel - startCol + div.io.b := stride + state := divCols + } + } + + is(divCols) { + when(div.io.done) { + assert(div.io.r === 0.U, "IM2COL output column shape must be integral") + outCols := div.io.q + 1.U + div.io.start := true.B + div.io.a := windowStart + div.io.b := div.io.q + 1.U + state := divStart + } + } + + is(divStart) { + when(div.io.done) { + firstRow := div.io.q + firstCol := div.io.r + div.io.start := true.B + div.io.a := kernel * kernel * inGroups + div.io.b := outGroups + state := divTiles + } + } + + is(divTiles) { + when(div.io.done) { + assert(div.io.r === 0.U, "IM2COL output groups must divide the K tiles") + groupTiles := div.io.q + state := check + } + } + + is(check) { + assert( + windowStart +& windowCount <= outRows * outCols, + "IM2COL selected window range exceeds the output shape" + ) + assert( + paddedWindows * groupTiles <= memParam.bankEntries.U, + "IM2COL output footprint exceeds bank depth" + ) + kernelRow := 0.U + kernelCol := 0.U + inGroup := 0.U + outGroup := 0.U + localTile := 0.U + window := 0.U + windowRow := firstRow + windowCol := firstCol + state := select + } + + // The current output row is an input row, or zero for padding and for + // windows past windowCount. + is(select) { + when(window < windowCount && inBounds) { + state := readReq + }.otherwise { + rowData := 0.U + state := writeReq + } + } + + is(readReq) { + when(io.bankRead(0).io.req.fire) { + state := readResp + } + } + + is(readResp) { + when(io.bankRead(0).io.resp.fire) { + rowData := io.bankRead(0).io.resp.bits.data + state := writeReq + } + } + + is(writeReq) { + when(io.bankWrite(0).io.req.fire) { + state := writeResp + } + } + + is(writeResp) { + when(io.bankWrite(0).io.resp.fire) { + when(window +& 1.U =/= paddedWindows) { + window := window + 1.U + when(windowCol +& 1.U === outCols) { + windowCol := 0.U + windowRow := windowRow + 1.U + }.otherwise { + windowCol := windowCol + 1.U + } + state := select + }.elsewhen(lastTile) { + state := complete + }.otherwise { + window := 0.U + windowRow := firstRow + windowCol := firstCol + when(inGroup +& 1.U === inGroups) { + inGroup := 0.U + when(kernelCol +& 1.U === kernel) { + kernelCol := 0.U + kernelRow := kernelRow + 1.U + }.otherwise { + kernelCol := kernelCol + 1.U + } + }.otherwise { + inGroup := inGroup + 1.U + } + when(localTile +& 1.U === groupTiles) { + localTile := 0.U + outGroup := outGroup + 1.U + }.otherwise { + localTile := localTile + 1.U + } + state := select + } + } + } + + is(complete) { + when(io.cmdResp.fire) { + state := idle + } + } } - - when(armed) { - armed := false.B - assert(!cfg.io.invalid, "IM2COL command has an illegal shape or bank assignment") - val padded = cfg.io.inputSize +& (cfg.io.padding << 1) - div.io.start := true.B - div.io.a := (padded - cfg.io.kernel - cfg.io.startRow)(15, 0) - div.io.b := cfg.io.stride - calc := cDivRow - } - - when(calc === cDivRow && div.io.done) { - assert(div.io.r === 0.U, "IM2COL output row shape must be integral") - outRowsReg := div.io.q + 1.U - val padded = cfg.io.inputSize +& (cfg.io.padding << 1) - div.io.start := true.B - div.io.a := (padded - cfg.io.kernel - cfg.io.startCol)(15, 0) - div.io.b := cfg.io.stride - calc := cDivCol - } - - when(calc === cDivCol && div.io.done) { - assert(div.io.r === 0.U, "IM2COL output column shape must be integral") - outColsReg := div.io.q + 1.U - calc := cCheck - } - - when(calc === cCheck) { - val inputValues = cfg.io.inputSize * cfg.io.inputSize - val inputRows = (inputValues +& (core - 1).U) >> log2Ceil(core) - val windows = outRowsReg * outColsReg - val windowEnd = cfg.io.windowStart +& cfg.io.windowCount - val kernelElems = cfg.io.kernel * cfg.io.kernel - val mTiles = (cfg.io.windowCount +& (core - 1).U) >> log2Ceil(core) - val kTiles = (kernelElems +& (core - 1).U) >> log2Ceil(core) - val outputRows = mTiles * kTiles * core.U - - assert(cfg.io.windowStart < windows && windowEnd <= windows, "IM2COL selected window range exceeds the output shape") - assert( - inputRows <= memParam.bankEntries.U, - "IM2COL input footprint exceeds bank depth" - ) - assert( - outputRows <= memParam.bankEntries.U, - "IM2COL output footprint exceeds bank depth" - ) - - div.io.start := true.B - div.io.a := cfg.io.windowStart - div.io.b := outColsReg(7, 0) - calc := cDivStart - } - - when(calc === cDivStart && div.io.done) { - assert(div.io.q < outRowsReg && div.io.r < outColsReg, "IM2COL window start decoding exceeded the output shape") - startOutRow := div.io.q - startOutCol := div.io.r - calc := cLaunch - } - - when(calc === cLaunch) { - launch := true.B - running := true.B - inputReady := false.B - localWindow := 0.U - elemAge := 0.U - calc := cIdle - } - - when(io.cmdResp.fire) { - respPending := false.B - } - - win.io.init := launch - win.io.next := false.B - win.io.kernel := cfg.io.kernel - win.io.outputCols := outColsReg - win.io.startRow := startOutRow - win.io.startCol := startOutCol - win.io.windowCount := cfg.io.windowCount - val canEmitElem = running && inputReady && elemAge === 3.U - win.io.elemFire := canEmitElem && writer.io.elemIn.ready - - lineBuf.io.bankRead(0) <> io.bankRead(0) - lineBuf.io.start := launch - lineBuf.io.inRows := cfg.io.inputSize - lineBuf.io.inCols := cfg.io.inputSize - lineBuf.io.rowStride := cfg.io.stride - lineBuf.io.colStride := cfg.io.stride - lineBuf.io.padding := cfg.io.padding - lineBuf.io.startRow := cfg.io.startRow - lineBuf.io.startCol := cfg.io.startCol - lineBuf.io.inputBase := cfg.io.inputBase - lineBuf.io.lane := cfg.io.lane - lineBuf.io.outRow := win.io.outRow - lineBuf.io.outCol := win.io.outCol - lineBuf.io.kRowIdx := win.io.kRowIdx - lineBuf.io.kColIdx := win.io.kColIdx - lineBuf.io.rBankId := cfg.io.rBank - lineBuf.io.rGroup := cfg.io.rGroup - lineBuf.io.robId := cfg.io.robId - - writer.io.bankWrite(0) <> io.bankWrite(0) - writer.io.init := launch - writer.io.wBankId := cfg.io.wBank - writer.io.wGroup := cfg.io.wGroup - writer.io.robId := cfg.io.robId - writer.io.elemIn.valid := canEmitElem - writer.io.elemIn.bits := lineBuf.io.elemData - writer.io.elemLast := win.io.elemLast - writer.io.finalWindow := win.io.last - writer.io.kRows := cfg.io.kernel - writer.io.kCols := cfg.io.kernel - writer.io.windowIdx := localWindow - - when(launch) { - inputReady := false.B - elemAge := 0.U - }.elsewhen(running && !inputReady && lineBuf.io.loadDone) { - inputReady := true.B - elemAge := 0.U - }.elsewhen(writer.io.elemIn.fire || writer.io.windowComplete) { - elemAge := 0.U - }.elsewhen(running && inputReady && elemAge =/= 3.U) { - elemAge := elemAge + 1.U - } - - when(writer.io.opComplete) { - inputReady := false.B - running := false.B - respPending := true.B - }.elsewhen(writer.io.windowComplete) { - win.io.next := true.B - localWindow := localWindow + 1.U - } - } diff --git a/examples/components/balls/im2col/arch/Im2colConfigRegs.scala b/examples/components/balls/im2col/arch/Im2colConfigRegs.scala deleted file mode 100644 index dd3baa96f..000000000 --- a/examples/components/balls/im2col/arch/Im2colConfigRegs.scala +++ /dev/null @@ -1,108 +0,0 @@ -package examples.balls.im2col - -import chisel3.experimental.hierarchy.{instantiable, public} - -import framework.acc.frontend.configs.FrontendParam -import framework.acc.memdomain.configs.MemDomainParam - -import examples.balls.im2col.configs.Im2colBallParam - -import chisel3._ -import framework.acc.balldomain.ball.blink.BallCommand -import chisel3.util._ - -@instantiable -class Im2colConfigRegs( - frontendParam: FrontendParam, - memParam: MemDomainParam, - val maxIter: Int, - val maxKSize: Int, - val maxPadding: Int, - val ballParam: Im2colBallParam) - extends Module { - val commandDataBits = frontendParam.commandDataBits - - val kW = log2Ceil(maxKSize + 1) - - @public - val io = IO(new Bundle { - - val cmd = Input(new BallCommand(frontendParam, memParam)) - - val load = Input(Bool()) - val invalid = Output(Bool()) - val robId = Output(UInt(log2Up(frontendParam.rob_entries).W)) - val rBank = Output(UInt(frontendParam.bank_id_len.W)) - val wBank = Output(UInt(frontendParam.bank_id_len.W)) - val rGroup = Output(UInt(memParam.groupIdWidth.W)) - val wGroup = Output(UInt(memParam.groupIdWidth.W)) - val inputSize = Output(UInt(16.W)) - val kernel = Output(UInt(kW.W)) - val stride = Output(UInt(8.W)) - val padding = Output(UInt(8.W)) - val startRow = Output(UInt(8.W)) - val startCol = Output(UInt(8.W)) - val inputBase = Output(UInt(6.W)) - val lane = Output(UInt(4.W)) - val windowStart = Output(UInt(6.W)) - val windowCount = Output(UInt(7.W)) - }) - - val robId = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) - val rBank = RegInit(0.U(frontendParam.bank_id_len.W)) - val wBank = RegInit(0.U(frontendParam.bank_id_len.W)) - val rGroup = RegInit(0.U(memParam.groupIdWidth.W)) - val wGroup = RegInit(0.U(memParam.groupIdWidth.W)) - val inputSize = RegInit(0.U(16.W)) - val kernel = RegInit(0.U(kW.W)) - val stride = RegInit(0.U(8.W)) - val padding = RegInit(0.U(8.W)) - val startRow = RegInit(0.U(8.W)) - val startCol = RegInit(0.U(8.W)) - val inputBase = RegInit(0.U(6.W)) - val lane = RegInit(0.U(4.W)) - val windowStart = RegInit(0.U(6.W)) - val windowCount = RegInit(0.U(7.W)) - - when(io.load) { - robId := io.cmd.rob_id - rBank := io.cmd.op1_bank - wBank := io.cmd.wr_bank - rGroup := io.cmd.op1_group - wGroup := io.cmd.wr_group - inputSize := io.cmd.iter - kernel := io.cmd.rs2(7, 0)(kW - 1, 0) - stride := io.cmd.rs2(15, 8) - padding := io.cmd.rs2(23, 16) - startCol := io.cmd.rs2(31, 24) - startRow := io.cmd.rs2(39, 32) - inputBase := io.cmd.rs2(45, 40) - lane := io.cmd.rs2(49, 46) - windowStart := io.cmd.rs2(55, 50) - windowCount := io.cmd.rs2(62, 56) - } - - val padded = inputSize +& (padding << 1) - io.invalid := inputSize === 0.U || inputSize > maxIter.U || - kernel === 0.U || kernel > maxKSize.U || stride === 0.U || - padding > maxPadding.U || startRow > padding || startCol > padding || - inputBase +& inputSize * inputSize > memParam.bankEntries.U || - padded < kernel + startRow || padded < kernel + startCol || - windowCount === 0.U || rBank === wBank - - io.robId := robId - io.rBank := rBank - io.wBank := wBank - io.rGroup := rGroup - io.wGroup := wGroup - io.inputSize := inputSize - io.kernel := kernel - io.stride := stride - io.padding := padding - io.startRow := startRow - io.startCol := startCol - io.inputBase := inputBase - io.lane := lane - io.windowStart := windowStart - io.windowCount := windowCount -} diff --git a/examples/components/balls/im2col/arch/Im2colWindow.scala b/examples/components/balls/im2col/arch/Im2colWindow.scala deleted file mode 100644 index 36de20bd1..000000000 --- a/examples/components/balls/im2col/arch/Im2colWindow.scala +++ /dev/null @@ -1,75 +0,0 @@ -package examples.balls.im2col - -import chisel3.experimental.hierarchy.{instantiable, public} - -import examples.balls.im2col.configs.Im2colBallParam - -import chisel3._ -import chisel3.util._ - -@instantiable -class Im2colWindow(val maxK: Int, val ballParam: Im2colBallParam) extends Module { - - val kW = log2Ceil(maxK + 1) - - @public - val io = IO(new Bundle { - val init = Input(Bool()) - val next = Input(Bool()) - val elemFire = Input(Bool()) - val kernel = Input(UInt(kW.W)) - val outputCols = Input(UInt(16.W)) - val startRow = Input(UInt(16.W)) - val startCol = Input(UInt(16.W)) - val windowCount = Input(UInt(12.W)) - val outRow = Output(UInt(16.W)) - val outCol = Output(UInt(16.W)) - val kRowIdx = Output(UInt(kW.W)) - val kColIdx = Output(UInt(kW.W)) - val elemLast = Output(Bool()) - val last = Output(Bool()) - }) - - val outRowReg = RegInit(0.U(16.W)) - val outColReg = RegInit(0.U(16.W)) - val kRowReg = RegInit(0.U(kW.W)) - val kColReg = RegInit(0.U(kW.W)) - val colsReg = RegInit(0.U(16.W)) - val remaining = RegInit(0.U(12.W)) - - val elemLast = kRowReg === io.kernel - 1.U && kColReg === io.kernel - 1.U - - when(io.init) { - outRowReg := io.startRow - outColReg := io.startCol - kRowReg := 0.U - kColReg := 0.U - colsReg := io.outputCols - remaining := io.windowCount - }.elsewhen(io.next) { - assert(remaining > 1.U, "Im2colWindow cannot advance past the selected range") - kRowReg := 0.U - kColReg := 0.U - remaining := remaining - 1.U - when(outColReg + 1.U === colsReg) { - outRowReg := outRowReg + 1.U - outColReg := 0.U - }.otherwise { - outColReg := outColReg + 1.U - } - }.elsewhen(io.elemFire && !elemLast) { - when(kColReg === io.kernel - 1.U) { - kColReg := 0.U - kRowReg := kRowReg + 1.U - }.otherwise { - kColReg := kColReg + 1.U - } - } - - io.outRow := outRowReg - io.outCol := outColReg - io.kRowIdx := kRowReg - io.kColIdx := kColReg - io.elemLast := elemLast - io.last := remaining === 1.U -} diff --git a/examples/components/balls/im2col/arch/LineBufferManager.scala b/examples/components/balls/im2col/arch/LineBufferManager.scala deleted file mode 100644 index cdc62d6fe..000000000 --- a/examples/components/balls/im2col/arch/LineBufferManager.scala +++ /dev/null @@ -1,175 +0,0 @@ -package examples.balls.im2col - -import framework.acc.balldomain.configs.BallRouteParam - -import framework.acc.balldomain.configs.BallDomainParam -import framework.acc.frontend.configs.FrontendParam -import framework.acc.memdomain.configs.MemDomainParam - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} -import framework.acc.memdomain.BankRead -import examples.balls.im2col.configs.Im2colBallParam - -@instantiable -class LineBufferManager( - val frontendParam: FrontendParam, - val memParam: MemDomainParam, - val routingParam: BallDomainParam, - val instance: BallRouteParam, - val ballParam: Im2colBallParam) - extends Module { - val ballCfg = ballParam - val maxIter = ballCfg.maxIter - val maxKSize = ballCfg.maxKSize - val elemWidth = ballCfg.inputWidth - val bankWidth = memParam.bankWidth - val lanesPerBeat = bankWidth / elemWidth - require(isPow2(lanesPerBeat), "LineBuffer lanesPerBeat must be power of 2") - val maxBeats = maxIter * maxIter - require(isPow2(maxBeats) && maxBeats >= 2, "LineBuffer depth must be an even power of two") - val halfBeats = maxBeats / 2 - // FIXME: This is a local line-buffer address, sized for maxIter^2; it must not use - // the external bank address width. Each command's external footprint is - // checked separately against bankEntries in Im2colConfigRegs. - val memoryAddrW = log2Ceil(maxBeats) - val kW = log2Ceil(maxKSize + 1) - val addrW = log2Ceil(memParam.bankEntries) - - val map = instance - - val inBW = map.inBW - - @public - val io = IO(new Bundle { - - val bankRead = Vec( - inBW, - Flipped(new BankRead( - frontendParam, - memParam - )) - ) - - val start = Input(Bool()) - val inRows = Input(UInt(16.W)) - val inCols = Input(UInt(16.W)) - val rowStride = Input(UInt(8.W)) - val colStride = Input(UInt(8.W)) - val padding = Input(UInt(8.W)) - val startRow = Input(UInt(8.W)) - val startCol = Input(UInt(8.W)) - val inputBase = Input(UInt(6.W)) - val lane = Input(UInt(log2Ceil(lanesPerBeat).W)) - val outRow = Input(UInt(16.W)) - val outCol = Input(UInt(16.W)) - val kRowIdx = Input(UInt(kW.W)) - val kColIdx = Input(UInt(kW.W)) - val rBankId = Input(UInt(frontendParam.bank_id_len.W)) - val rGroup = Input(UInt(memParam.groupIdWidth.W)) - val robId = Input(UInt(log2Up(frontendParam.rob_entries).W)) - val loadDone = Output(Bool()) - val elemData = Output(UInt(elemWidth.W)) - }) - - val bufLo = SyncReadMem(halfBeats, UInt(bankWidth.W)) - val bufHi = SyncReadMem(halfBeats, UInt(bankWidth.W)) - val active = RegInit(false.B) - val pending = RegInit(false.B) - val beat = RegInit(0.U(addrW.W)) - val dimW = log2Ceil(maxIter + 1) - val beatsReg = RegInit(0.U((addrW + 1).W)) - - for (i <- 0 until inBW) { - io.bankRead(i).io.req.valid := false.B - io.bankRead(i).io.req.bits.addr := 0.U - io.bankRead(i).io.resp.ready := false.B - io.bankRead(i).bank_id := io.rBankId - io.bankRead(i).rob_id := io.robId - io.bankRead(i).group_id := io.rGroup - } - - io.bankRead(0).io.req.valid := active && !pending - io.bankRead(0).io.req.bits.addr := (io.inputBase +& beat).asTypeOf(UInt(addrW.W)) - io.bankRead(0).io.resp.ready := pending - io.loadDone := !active - - when(io.start) { - val rows = io.inRows(dimW - 1, 0) - val cols = io.inCols(dimW - 1, 0) - beatsReg := rows * cols - active := true.B - pending := false.B - beat := 0.U - }.elsewhen(io.bankRead(0).io.req.fire) { - pending := true.B - } - - when(io.bankRead(0).io.resp.fire) { - pending := false.B - when(beat +& 1.U === beatsReg) { - active := false.B - }.otherwise { - beat := beat + 1.U - } - } - - val paddedRow = io.startRow +& (io.outRow(dimW - 1, 0) * io.rowStride) +& io.kRowIdx - val paddedCol = io.startCol +& (io.outCol(dimW - 1, 0) * io.colStride) +& io.kColIdx - val rowValid = - paddedRow >= io.padding && paddedRow < io.padding + io.inRows - val colValid = - paddedCol >= io.padding && paddedCol < io.padding + io.inCols - val inBound = rowValid && colValid - val srcRow = Mux(inBound, paddedRow - io.padding, 0.U) - val srcCol = Mux(inBound, paddedCol - io.padding, 0.U) - val elemIndex = srcRow(dimW - 1, 0) * io.inCols(dimW - 1, 0) + srcCol(dimW - 1, 0) - val beatIndex = elemIndex - val laneIndex = io.lane - val memoryWrite = io.bankRead(0).io.resp.fire - val memoryRead = !active - - val memoryAddress = Mux( - memoryWrite, - beat.pad(memoryAddrW), - beatIndex(memoryAddrW - 1, 0) - ) - - val memoryBank = memoryAddress(memoryAddrW - 1) - val memoryRow = memoryAddress(log2Ceil(halfBeats) - 1, 0) - val memoryEnable = memoryWrite || memoryRead - - val loData = bufLo.readWrite( - memoryRow, - io.bankRead(0).io.resp.bits.data.asUInt, - memoryEnable && !memoryBank, - memoryWrite - ) - - val hiData = bufHi.readWrite( - memoryRow, - io.bankRead(0).io.resp.bits.data.asUInt, - memoryEnable && memoryBank, - memoryWrite - ) - - val readBank = RegEnable(memoryBank, false.B, memoryRead) - val memoryData = Mux(readBank, hiData, loData) - val laneR = RegInit(0.U(log2Ceil(lanesPerBeat).W)) - val boundR = RegInit(false.B) - val beatData = RegInit(0.U(bankWidth.W)) - val laneD = RegInit(0.U(log2Ceil(lanesPerBeat).W)) - val boundD = RegInit(false.B) - val elemHold = RegInit(0.U(elemWidth.W)) - laneR := laneIndex - boundR := inBound - beatData := memoryData - laneD := laneR - boundD := boundR - val lanes = - beatData.asTypeOf(Vec(lanesPerBeat, UInt(elemWidth.W))) - elemHold := Mux(boundD, lanes(laneD), 0.U) - - io.elemData := elemHold -} diff --git a/examples/components/balls/im2col/arch/StreamWriter.scala b/examples/components/balls/im2col/arch/StreamWriter.scala deleted file mode 100644 index cada34032..000000000 --- a/examples/components/balls/im2col/arch/StreamWriter.scala +++ /dev/null @@ -1,243 +0,0 @@ -package examples.balls.im2col - -import framework.acc.balldomain.configs.BallRouteParam - -import framework.acc.balldomain.configs.BallDomainParam -import framework.acc.frontend.configs.FrontendParam -import framework.acc.memdomain.configs.MemDomainParam - -import chisel3._ -import chisel3.util._ -import chisel3.experimental.hierarchy.{instantiable, public} - -import framework.acc.memdomain.BankWrite -import examples.balls.im2col.configs.Im2colBallParam - -/** - * Packs im2col K cores into 128-bit bank rows (SMatMulBall A layout). - * - * ((w / 16) * ceil(kernelElems / 16) + kt) * 16 + (w % 16) - * - * Partial final K-core lanes and partial final M-core rows are zero-filled. - */ -@instantiable -class StreamWriter( - val frontendParam: FrontendParam, - val memParam: MemDomainParam, - val routingParam: BallDomainParam, - val instance: BallRouteParam, - val ballParam: Im2colBallParam) - extends Module { - val ballCfg = ballParam - val maxKSize = ballCfg.maxKSize - val elemWidth = ballCfg.inputWidth - val bankWidth = memParam.bankWidth - val bankEntries = memParam.bankEntries - val lanesPerBeat = bankWidth / elemWidth - val laneIdxWidth = log2Ceil(lanesPerBeat) - val kW = log2Ceil(maxKSize + 1) - val bankAddrW = log2Ceil(bankEntries) - val maxKTiles = - (maxKSize * maxKSize + lanesPerBeat - 1) / lanesPerBeat - val kTilesW = log2Ceil(maxKTiles + 1) - require( - isPow2(bankEntries), - "StreamWriter requires power-of-two bank depth" - ) - - val mapping = instance - - val outBW = mapping.outBW - - require(outBW == 1, "StreamWriter requires outBW=1") - require(bankWidth % elemWidth == 0) - require(lanesPerBeat == 16, "Im2col SMatMulBall layout requires 16 int8 lanes") - - @public - val io = IO(new Bundle { - - val bankWrite = Vec( - outBW, - Flipped(new BankWrite( - frontendParam, - memParam - )) - ) - - val elemIn = Flipped(Decoupled(UInt(elemWidth.W))) - val elemLast = Input(Bool()) - val finalWindow = Input(Bool()) - val init = Input(Bool()) - val windowIdx = Input(UInt(32.W)) - val kRows = Input(UInt(kW.W)) - val kCols = Input(UInt(kW.W)) - val wBankId = Input(UInt(frontendParam.bank_id_len.W)) - val wGroup = Input(UInt(memParam.groupIdWidth.W)) - val robId = Input(UInt(log2Up(frontendParam.rob_entries).W)) - - val busy = Output(Bool()) - val windowComplete = Output(Bool()) - val opComplete = Output(Bool()) - }) - - val zeros = VecInit(Seq.fill(lanesPerBeat)(0.U(elemWidth.W))) - - val packCntReg = RegInit(0.U(log2Ceil(lanesPerBeat + 1).W)) - - val packReg = RegInit( - VecInit(Seq.fill(lanesPerBeat)(0.U(elemWidth.W))) - ) - - val chunkIdxReg = RegInit(0.U(kTilesW.W)) - val wrPendingReg = RegInit(false.B) - val issued = RegInit(false.B) - val endWindowReg = RegInit(false.B) - val finalWinReg = RegInit(false.B) - val padActive = RegInit(false.B) - val padMTile = RegInit(0.U(32.W)) - val padKTiles = RegInit(0.U(kTilesW.W)) - val padK = RegInit(0.U(kTilesW.W)) - val padRow = RegInit(0.U(laneIdxWidth.W)) - val padRow0 = RegInit(0.U(laneIdxWidth.W)) - val wAddrReg = RegInit(0.U(bankAddrW.W)) - - val kernelElems = io.kRows * io.kCols - val kTilesReg = RegInit(0.U(kTilesW.W)) - - val mTile = io.windowIdx >> laneIdxWidth - val mRow = io.windowIdx(laneIdxWidth - 1, 0) - - def tileOff(core: UInt, kTiles: UInt): UInt = { - MuxLookup(kTiles, 0.U)( - Seq( - 1.U -> core, - 2.U -> (core << 1), - 3.U -> ((core << 1) + core), - 4.U -> (core << 2) - ) - ) - } - - val targetAddr = - ((tileOff(mTile, kTilesReg) + chunkIdxReg) << laneIdxWidth) + mRow - val padAddr = - ((tileOff(padMTile, padKTiles) + padK) << laneIdxWidth) + padRow - val writeFire = io.bankWrite(0).io.req.fire - val writeResp = io.bankWrite(0).io.resp.fire - - io.busy := wrPendingReg || padActive - io.windowComplete := wrPendingReg && endWindowReg && writeResp && !finalWinReg - io.opComplete := Mux( - padActive, - wrPendingReg && writeResp && - (padK + 1.U === padKTiles) && (padRow === (lanesPerBeat - 1).U), - wrPendingReg && endWindowReg && writeResp && finalWinReg && - (mRow === (lanesPerBeat - 1).U) - ) - - for (i <- 0 until outBW) { - io.bankWrite(i).io.req.valid := false.B - io.bankWrite(i).io.req.bits.addr := 0.U - io.bankWrite(i).io.req.bits.data := 0.U - io.bankWrite(i).io.req.bits.mask := - VecInit(Seq.fill(memParam.bankMaskLen)(false.B)) - io.bankWrite(i).io.resp.ready := false.B - io.bankWrite(i).bank_id := 0.U - io.bankWrite(i).rob_id := 0.U - io.bankWrite(i).group_id := io.wGroup - } - - io.bankWrite(0).io.req.valid := wrPendingReg && !issued - io.bankWrite(0).io.req.bits.addr := wAddrReg - io.bankWrite(0).io.req.bits.data := Cat(packReg.reverse) - io.bankWrite(0).io.req.bits.mask := - VecInit(Seq.fill(memParam.bankMaskLen)(true.B)) - io.bankWrite(0).io.resp.ready := true.B - io.bankWrite(0).bank_id := io.wBankId - io.bankWrite(0).rob_id := io.robId - io.bankWrite(0).group_id := io.wGroup - - io.elemIn.ready := !wrPendingReg && !padActive - - when(io.init) { - packCntReg := 0.U - packReg := zeros - chunkIdxReg := 0.U - wrPendingReg := false.B - issued := false.B - endWindowReg := false.B - finalWinReg := false.B - wAddrReg := 0.U - padActive := false.B - padMTile := 0.U - padKTiles := 0.U - padK := 0.U - padRow := 0.U - padRow0 := 0.U - kTilesReg := ((kernelElems +& (lanesPerBeat - 1).U) >> laneIdxWidth) - .asTypeOf(UInt(kTilesW.W)) - assert( - ((kernelElems +& (lanesPerBeat - 1).U) >> laneIdxWidth) >= 1.U && - ((kernelElems +& (lanesPerBeat - 1).U) >> laneIdxWidth) <= 4.U, - "StreamWriter kTiles must be 1..4" - ) - }.otherwise { - when(writeFire) { - issued := true.B - } - when(writeResp) { - packCntReg := 0.U - packReg := zeros - wrPendingReg := false.B - issued := false.B - endWindowReg := false.B - when(padActive) { - when(padRow === (lanesPerBeat - 1).U) { - when(padK + 1.U === padKTiles) { - padActive := false.B - padK := 0.U - padRow := 0.U - }.otherwise { - padK := padK + 1.U - padRow := padRow0 - } - }.otherwise { - padRow := padRow + 1.U - } - }.elsewhen(endWindowReg) { - chunkIdxReg := 0.U - when(finalWinReg && (mRow =/= (lanesPerBeat - 1).U)) { - padActive := true.B - padMTile := mTile - padKTiles := kTilesReg - padK := 0.U - padRow := mRow + 1.U - padRow0 := mRow + 1.U - } - finalWinReg := false.B - }.otherwise { - chunkIdxReg := chunkIdxReg + 1.U - } - } - - when(padActive && !wrPendingReg) { - wrPendingReg := true.B - endWindowReg := false.B - finalWinReg := false.B - packReg := zeros - wAddrReg := padAddr.asTypeOf(UInt(bankAddrW.W)) - } - - when(io.elemIn.fire) { - packReg(packCntReg(laneIdxWidth - 1, 0)) := io.elemIn.bits - val nextCnt = packCntReg + 1.U - packCntReg := nextCnt - when(nextCnt === lanesPerBeat.U || io.elemLast) { - wrPendingReg := true.B - endWindowReg := io.elemLast - finalWinReg := io.elemLast && io.finalWindow - wAddrReg := targetAddr.asTypeOf(UInt(bankAddrW.W)) - } - } - } -} diff --git a/examples/components/balls/im2col/emu/src/im2col.rs b/examples/components/balls/im2col/emu/src/im2col.rs index 8d54ae705..6b51716a1 100644 --- a/examples/components/balls/im2col/emu/src/im2col.rs +++ b/examples/components/balls/im2col/emu/src/im2col.rs @@ -1,9 +1,7 @@ use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::instruction::ExecContext; -const CORE: usize = 16; - fn max_iter(instance: u32) -> usize { bebop_bemu::configs::ball_domain::parameters::("im2col", instance).max_iter.expect("maxIter") as usize } @@ -22,30 +20,37 @@ struct Shape { kernel: usize, stride: usize, padding: usize, - start_row: usize, start_col: usize, + start_row: usize, input_base: usize, - lane: usize, window_start: usize, window_count: usize, + tile_rows: usize, } fn decode_shape(xs1: u64, xs2: u64) -> Shape { + if rs1_b1(xs1) != 0 { + panic!("im2col: input bank 1 must be zero"); + } + if xs2 >> 51 != 0 { + panic!("im2col: rs2[63:51] must be zero"); + } Shape { input_size: rs1_iter(xs1) as usize, - kernel: (xs2 & 0xff) as usize, - stride: ((xs2 >> 8) & 0xff) as usize, - padding: ((xs2 >> 16) & 0xff) as usize, - start_col: ((xs2 >> 24) & 0xff) as usize, - start_row: ((xs2 >> 32) & 0xff) as usize, - input_base: ((xs2 >> 40) & 0x3f) as usize, - lane: ((xs2 >> 46) & 0xf) as usize, - window_start: ((xs2 >> 50) & 0x3f) as usize, - window_count: ((xs2 >> 56) & 0x7f) as usize, + kernel: (xs2 & 0xf) as usize, + stride: ((xs2 >> 4) & 0xf) as usize, + padding: ((xs2 >> 8) & 0xf) as usize, + start_col: ((xs2 >> 12) & 0xf) as usize, + start_row: ((xs2 >> 16) & 0xf) as usize, + input_base: ((xs2 >> 20) & 0x3f) as usize, + window_start: ((xs2 >> 26) & 0x3ff) as usize, + window_count: ((xs2 >> 36) & 0x3ff) as usize, + tile_rows: ((xs2 >> 46) & 0x1f) as usize, } } -fn dimensions(shape: Shape, instance: u32) -> (usize, usize, usize) { +// Returns the output columns of the full convolution output. +fn output_cols(shape: Shape, instance: u32) -> usize { if shape.input_size == 0 || shape.input_size > max_iter(instance) || shape.kernel == 0 @@ -55,15 +60,13 @@ fn dimensions(shape: Shape, instance: u32) -> (usize, usize, usize) { || shape.start_row > shape.padding || shape.start_col > shape.padding || shape.window_count == 0 - || shape.window_start >= 64 - || shape.window_count > 64 + || !shape.tile_rows.is_power_of_two() + || shape.tile_rows > 16 + || shape.input_base + shape.input_size * shape.input_size > bank_lines() { panic!("im2col: illegal shape field"); } - let padded = shape - .input_size - .checked_add(shape.padding * 2) - .unwrap_or_else(|| panic!("im2col: padded input overflow")); + let padded = shape.input_size + shape.padding * 2; if padded < shape.kernel + shape.start_row || padded < shape.kernel + shape.start_col { panic!("im2col: kernel and start exceed padded input"); } @@ -74,100 +77,82 @@ fn dimensions(shape: Shape, instance: u32) -> (usize, usize, usize) { } let output_rows = row_numerator / shape.stride + 1; let output_cols = col_numerator / shape.stride + 1; - let windows = output_rows - .checked_mul(output_cols) - .unwrap_or_else(|| panic!("im2col: output window count overflow")); - if shape.window_start >= windows || shape.window_start + shape.window_count > windows { + if shape.window_start + shape.window_count > output_rows * output_cols { panic!("im2col: selected window range exceeds output shape"); } - (output_rows, output_cols, windows) -} - -fn footprints(shape: Shape) -> (usize, usize) { - let input_values = shape - .input_size - .checked_mul(shape.input_size) - .unwrap_or_else(|| panic!("im2col: input footprint overflow")); - let kernel_values = shape.kernel * shape.kernel; - let input_rows = input_values; - let output_rows = shape.window_count.div_ceil(CORE) * kernel_values.div_ceil(CORE) * CORE; - if shape.input_base + input_rows > bank_lines() || output_rows > bank_lines() { - panic!("im2col: input or output footprint exceeds bank depth"); - } - (input_rows, output_rows) + output_cols } pub(crate) fn exec_im2col(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let input_bank = rs1_b0(xs1); let output_bank = rs1_b2(xs1); - if rs1_b1(xs1) != 0 { - panic!("im2col: input bank 1 must be zero"); - } - if xs2 >> 63 != 0 { - panic!("im2col: rs2[63] must be zero"); - } if input_bank == output_bank { panic!("im2col: input and output banks must differ"); } if !ctx.config(input_bank).allocated || !ctx.config(output_bank).allocated { panic!("im2col: bank not allocated"); } - if ctx.config(input_bank).cols != 1 || ctx.config(output_bank).cols != 1 { - panic!("im2col: input and output banks must each have one column"); - } - let shape = decode_shape(xs1, xs2); - let (_, output_cols, _) = dimensions(shape, ctx.ball_id.expect("Ball instance")); - let (_, output_bank_rows) = footprints(shape); - let kernel_values = shape.kernel * shape.kernel; - let kernel_tiles = kernel_values.div_ceil(CORE); - let pi = pbank(ctx, input_bank); - let po = pbank(ctx, output_bank); - let (input, output) = ctx.banks.read_write(pi, po); - output[..output_bank_rows * bank_row_bytes()].fill(0); + let output_cols = output_cols(shape, ctx.ball_id.expect("Ball instance")); + + // Input group g is channel block g. K is ordered (kernel row, kernel + // column, channel), so K tile t = kernel_position * channel_groups + g is + // one whole input row. Output group j holds K tiles j*group_tiles.. in the + // SMatMul A layout with tile_rows-row M tiles. + let channel_groups = ctx.config(input_bank).cols as usize; + let output_groups = ctx.config(output_bank).cols as usize; + let k_tiles = shape.kernel * shape.kernel * channel_groups; + if channel_groups == 0 || output_groups == 0 || k_tiles % output_groups != 0 { + panic!("im2col: output groups must divide the K tiles"); + } + let group_tiles = k_tiles / output_groups; + let padded_windows = shape.window_count.div_ceil(shape.tile_rows) * shape.tile_rows; + if padded_windows * group_tiles > bank_lines() { + panic!("im2col: output footprint exceeds bank depth"); + } - for local_window in 0..shape.window_count { - let global_window = shape.window_start + local_window; - let output_row = global_window / output_cols; - let output_col = global_window % output_cols; - for kernel_row in 0..shape.kernel { - for kernel_col in 0..shape.kernel { - let source_row = (shape.start_row + output_row * shape.stride + kernel_row) - as isize - - shape.padding as isize; - let source_col = (shape.start_col + output_col * shape.stride + kernel_col) - as isize - - shape.padding as isize; - if source_row < 0 - || source_col < 0 - || source_row >= shape.input_size as isize - || source_col >= shape.input_size as isize - { - continue; - } - let kernel_element = kernel_row * shape.kernel + kernel_col; - let bank_row = ((local_window / CORE) * kernel_tiles + kernel_element / CORE) - * CORE - + local_window % CORE; - let lane = kernel_element % CORE; - let source = source_row as usize * shape.input_size + source_col as usize; - output[bank_row * CORE + lane] = - input[(shape.input_base + source) * CORE + shape.lane]; + let row_bytes = bank_row_bytes(); + for tile in 0..k_tiles { + let channel_group = tile % channel_groups; + let kernel_row = tile / channel_groups / shape.kernel; + let kernel_col = tile / channel_groups % shape.kernel; + let pi = pbank_group(ctx, input_bank, channel_group as u64); + let po = pbank_group(ctx, output_bank, (tile / group_tiles) as u64); + let (input, output) = ctx.banks.read_write(pi, po); + for window in 0..padded_windows { + let line = ((window / shape.tile_rows) * group_tiles + tile % group_tiles) + * shape.tile_rows + + window % shape.tile_rows; + let target = &mut output[line * row_bytes..(line + 1) * row_bytes]; + target.fill(0); + if window >= shape.window_count { + continue; + } + let global = shape.window_start + window; + let source_row = (shape.start_row + global / output_cols * shape.stride + kernel_row) + as isize + - shape.padding as isize; + let source_col = (shape.start_col + global % output_cols * shape.stride + kernel_col) + as isize + - shape.padding as isize; + if source_row < 0 + || source_col < 0 + || source_row >= shape.input_size as isize + || source_col >= shape.input_size as isize + { + continue; } + let source = + shape.input_base + source_row as usize * shape.input_size + source_col as usize; + target.copy_from_slice(&input[source * row_bytes..(source + 1) * row_bytes]); } } 0 } pub(crate) fn latency(xs1: u64, xs2: u64, instance: u32) -> u64 { - if rs1_b1(xs1) != 0 { - panic!("im2col: input bank 1 must be zero"); - } - if xs2 >> 63 != 0 { - panic!("im2col: rs2[63] must be zero"); - } let shape = decode_shape(xs1, xs2); - dimensions(shape, instance); - footprints(shape); - (shape.window_count * shape.kernel * shape.kernel).max(16) as u64 + output_cols(shape, instance); + (shape.window_count.div_ceil(shape.tile_rows) * shape.tile_rows * shape.kernel * shape.kernel * 2) + as u64 } diff --git a/examples/components/balls/im2col/verify/casegen/src/casegen.rs b/examples/components/balls/im2col/verify/casegen/src/casegen.rs index ba16c6d90..d31b01622 100644 --- a/examples/components/balls/im2col/verify/casegen/src/casegen.rs +++ b/examples/components/balls/im2col/verify/casegen/src/casegen.rs @@ -6,7 +6,6 @@ pub const MAX_WORDS: usize = 128; #[repr(C)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Im2colCmd { - pub bid: u32, pub iter: u32, pub ksize: u32, pub stride: u32, @@ -14,6 +13,8 @@ pub struct Im2colCmd { pub window_count: u32, pub op1_bank: u32, pub wr_bank: u32, + pub op1_group: u32, + pub wr_group: u32, pub op1_col: u32, pub wr_col: u32, pub rob_id: u32, @@ -63,22 +64,23 @@ fn word_hi(data: &[u8], word_index: usize) -> u64 { u64::from_le_bytes(buf) } -pub fn gen_case(seed: u32, index: u32, bid: u32) -> Im2colCase { +pub fn gen_case(seed: u32, index: u32) -> Im2colCase { match index { - 0 => directed_k3_pad0(bid), - 1 => directed_k3_pad1(bid), - _ => random_case(seed, index, bid), + 0 => directed_k3_pad0(), + 1 => directed_k3_pad1(), + _ => random_case(seed, index), } } fn build_case( - bid: u32, iter: usize, ksize: usize, stride: usize, padding: usize, op1_bank: u32, wr_bank: u32, + op1_group: u32, + wr_group: u32, rob_id: u32, flat: &[u8], ) -> Im2colCase { @@ -105,7 +107,6 @@ fn build_case( } Im2colCase { cmd: Im2colCmd { - bid, iter: iter as u32, ksize: ksize as u32, stride: stride as u32, @@ -113,6 +114,8 @@ fn build_case( window_count: windows as u32, op1_bank, wr_bank, + op1_group, + wr_group, op1_col: 1, wr_col: 1, rob_id, @@ -124,7 +127,7 @@ fn build_case( } } -fn directed_k3_pad0(bid: u32) -> Im2colCase { +fn directed_k3_pad0() -> Im2colCase { let iter = 6usize; let k = 3usize; let flat: [i8; 36] = [ @@ -132,10 +135,10 @@ fn directed_k3_pad0(bid: u32) -> Im2colCase { -25, 26, 27, 0, -29, 30, 31, 32, 33, -34, 0, 36, ]; let bytes: Vec = flat.iter().map(|&x| x as u8).collect(); - build_case(bid, iter, k, 1, 0, 0, 1, 3, &bytes) + build_case(iter, k, 1, 0, 0, 1, 0, 0, 3, &bytes) } -fn directed_k3_pad1(bid: u32) -> Im2colCase { +fn directed_k3_pad1() -> Im2colCase { let iter = 6usize; let k = 3usize; let flat: [i8; 36] = [ @@ -143,7 +146,7 @@ fn directed_k3_pad1(bid: u32) -> Im2colCase { 24, 25, 26, -27, 28, 29, -30, 0, 32, 33, 34, ]; let bytes: Vec = flat.iter().map(|&x| x as u8).collect(); - build_case(bid, iter, k, 1, 1, 0, 1, 5, &bytes) + build_case(iter, k, 1, 1, 0, 1, 1, 2, 5, &bytes) } fn shape_ok(iter: usize, ksize: usize, stride: usize, padding: usize) -> bool { @@ -158,7 +161,7 @@ fn shape_ok(iter: usize, ksize: usize, stride: usize, padding: usize) -> bool { rows <= model::bank_entries() } -fn random_case(seed: u32, index: u32, bid: u32) -> Im2colCase { +fn random_case(seed: u32, index: u32) -> Im2colCase { let mut rng = Rng::new(seed, index); let iter_pool = [3usize, 4, 5, 6, 7]; let ksize_pool = [1usize, 3, 5]; @@ -182,16 +185,11 @@ fn random_case(seed: u32, index: u32, bid: u32) -> Im2colCase { for b in flat.iter_mut() { *b = (rng.next() & 0xff) as u8; } + let rob_id = rng.range(0, 15); + let op1_group = rng.range(0, 3); + let wr_group = rng.range(0, 3); return build_case( - bid, - iter, - ksize, - stride, - padding, - op1_bank, - wr_bank, - rng.range(0, 15), - &flat, + iter, ksize, stride, padding, op1_bank, wr_bank, op1_group, wr_group, rob_id, &flat, ); } panic!("random_case: failed to find legal shape for index={index}"); diff --git a/examples/components/balls/im2col/verify/casegen/src/lib.rs b/examples/components/balls/im2col/verify/casegen/src/lib.rs index 67176b88c..be6a3be65 100644 --- a/examples/components/balls/im2col/verify/casegen/src/lib.rs +++ b/examples/components/balls/im2col/verify/casegen/src/lib.rs @@ -10,8 +10,8 @@ thread_local! { } #[no_mangle] -pub extern "C" fn im2col_case_load(seed: u32, index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(seed, index, bid); +pub extern "C" fn im2col_case_load(seed: u32, index: u32) -> i32 { + let case = casegen::gen_case(seed, index); let nsrc = case.cmd.num_src_words as usize; let ndst = case.cmd.num_dst_words as usize; if nsrc > MAX_WORDS || ndst > MAX_WORDS { diff --git a/examples/components/balls/im2col/verify/filelists/im2col_ball.f b/examples/components/balls/im2col/verify/filelists/im2col_ball.f index 5cbccdd31..ce6f93759 100644 --- a/examples/components/balls/im2col/verify/filelists/im2col_ball.f +++ b/examples/components/balls/im2col/verify/filelists/im2col_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,6 +14,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv diff --git a/examples/components/balls/im2col/verify/src/common/im2col_defs.svh b/examples/components/balls/im2col/verify/src/common/im2col_defs.svh index 4ce04ad4f..c5a9d46e2 100644 --- a/examples/components/balls/im2col/verify/src/common/im2col_defs.svh +++ b/examples/components/balls/im2col/verify/src/common/im2col_defs.svh @@ -1,5 +1,4 @@ typedef struct { - int unsigned bid; int unsigned iter; int unsigned ksize; int unsigned stride; @@ -7,6 +6,8 @@ typedef struct { int unsigned window_count; int unsigned op1_bank; int unsigned wr_bank; + int unsigned op1_group; + int unsigned wr_group; int unsigned op1_col; int unsigned wr_col; int unsigned rob_id; @@ -16,8 +17,7 @@ typedef struct { import "DPI-C" function int im2col_case_load( input int unsigned seed, - input int unsigned index, - input int unsigned bid + input int unsigned index ); import "DPI-C" function void im2col_case_cmd(output im2col_cmd_dpi_t cmd); import "DPI-C" function longint unsigned im2col_case_src_word_lo(input int unsigned word_index); @@ -32,11 +32,3 @@ localparam int IM2COL_THREAD_FUNCT7 = `IM2COL_FUNCT7; localparam int IM2COL_MAX_WORDS = 128; localparam int IM2COL_TIMEOUT_CYCLES = 100000; localparam int IM2COL_SEED = 32'hCAFE_BABE; - -function automatic int unsigned im2col_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/components/balls/im2col/verify/src/common/im2col_items.svh b/examples/components/balls/im2col/verify/src/common/im2col_items.svh index 1bd487bba..a14e137b0 100644 --- a/examples/components/balls/im2col/verify/src/common/im2col_items.svh +++ b/examples/components/balls/im2col/verify/src/common/im2col_items.svh @@ -10,69 +10,48 @@ class im2col_cmd_item extends bb_blink_cmd_item; bit [127:0] src_words[IM2COL_MAX_WORDS]; bit [127:0] dst_words[IM2COL_MAX_WORDS]; - constraint legal_c { - funct7 == IM2COL_THREAD_FUNCT7[6:0]; - op1_en == 1'b1; - op2_en == 1'b0; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b0; - op2_bank == 5'd0; - op2_col == 5'd0; - meta_bank == 5'd0; - rs1 == 64'd0; - rs2[63] == 1'b0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - } - function new(string name = "im2col_cmd_item"); super.new(name); endfunction - function void load_rust_case(int unsigned seed, int unsigned index, int unsigned bid); + function void load_rust_case(int unsigned seed, int unsigned index); im2col_cmd_dpi_t cmd; int unsigned rc; longint unsigned w_lo; longint unsigned w_hi; int i; - rc = im2col_case_load(seed, index, bid); + rc = im2col_case_load(seed, index); if (rc != 0) begin `uvm_fatal("CASE", $sformatf("im2col_case_load returned %0d for index %0d", rc, index)) end im2col_case_cmd(cmd); - bid = cmd.bid[4:0]; - funct7 = IM2COL_THREAD_FUNCT7[6:0]; - iter = cmd.iter; ksize = cmd.ksize; stride = cmd.stride; padding = cmd.padding; window_count = cmd.window_count; - op1_bank = cmd.op1_bank[4:0]; - wr_bank = cmd.wr_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - wr_col = cmd.wr_col[4:0]; - rob_id = cmd.rob_id[3:0]; num_src_words = cmd.num_src_words; num_dst_words = cmd.num_dst_words; - special = 64'd0; - op1_en = 1'b1; - op2_en = 1'b0; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b0; - op2_bank = 5'd0; - op2_col = 5'd0; - meta_bank = 5'd0; - rs1 = 64'd0; + // bb_im2col: banks and input_size (iter) come from rs1, the window geometry from rs2. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = IM2COL_THREAD_FUNCT7[6:0]; rs2 = { 1'b0, window_count[6:0], 6'd0, 4'd0, 6'd0, 8'd0, 8'd0, padding[7:0], stride[7:0], ksize[7:0] }; - is_sub = 1'b0; - sub_rob_id = 8'h00; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = '0; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = '0; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = '0; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; if (num_src_words > IM2COL_MAX_WORDS || num_dst_words > IM2COL_MAX_WORDS) begin `uvm_fatal("CASE", $sformatf("word count out of range src=%0d dst=%0d", num_src_words, diff --git a/examples/components/balls/im2col/verify/src/env/im2col_scoreboard.svh b/examples/components/balls/im2col/verify/src/env/im2col_scoreboard.svh index 5067cc112..00f39d8cc 100644 --- a/examples/components/balls/im2col/verify/src/env/im2col_scoreboard.svh +++ b/examples/components/balls/im2col/verify/src/env/im2col_scoreboard.svh @@ -48,7 +48,7 @@ class im2col_scoreboard extends uvm_scoreboard; end mem_model.clear_mem(); for (i = 0; i < clone.num_src_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), i, clone.src_words[i]); + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group), i, clone.src_words[i]); end mem_model.arm(); @@ -67,26 +67,12 @@ class im2col_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, im2col_cmd_item exp); + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); verification_pkg::evidence::checks++; - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) - if (got.rs2[7:0] !== exp.ksize[7:0]) - `uvm_fatal("CMD", $sformatf("ksize mismatch: got %0d exp %0d", got.rs2[7:0], exp.ksize)) - if (got.rs2[15:8] !== exp.stride[7:0]) - `uvm_fatal("CMD", $sformatf("stride mismatch: got %0d exp %0d", got.rs2[15:8], exp.stride)) - if (got.rs2[23:16] !== exp.padding[7:0]) - `uvm_fatal("CMD", $sformatf("padding mismatch: got %0d exp %0d", got.rs2[23:16], exp.padding)) - if (got.rs1 !== 64'd0) `uvm_fatal("CMD", "rs1 must be 0") + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) endfunction function void write_read(bb_blink_read_item item); @@ -96,8 +82,9 @@ class im2col_scoreboard extends uvm_scoreboard; `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.op1_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== '0) - `uvm_fatal("READ", $sformatf("group_id mismatch: got %0d exp 0", item.group_id)) + if (item.group_id !== stim.op1_group) + `uvm_fatal("READ", $sformatf( + "group_id mismatch: got %0d exp %0d", item.group_id, stim.op1_group)) if (item.addr >= stim.num_src_words[9:0]) `uvm_fatal("READ", $sformatf( "addr out of range: got %0d max %0d", item.addr, stim.num_src_words)) @@ -112,8 +99,9 @@ class im2col_scoreboard extends uvm_scoreboard; `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== '0) - `uvm_fatal("WRITE", $sformatf("group_id mismatch: got %0d exp 0", item.group_id)) + if (item.group_id !== stim.wr_group) + `uvm_fatal("WRITE", $sformatf( + "group_id mismatch: got %0d exp %0d", item.group_id, stim.wr_group)) addr = int'(item.addr); if (addr >= expected_writes) `uvm_fatal("WRITE", $sformatf("addr out of range: got %0d max %0d", addr, expected_writes)) @@ -134,7 +122,6 @@ class im2col_scoreboard extends uvm_scoreboard; stim = current_stim("RESP"); if (item.rob_id !== stim.rob_id) `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") resp_count++; endfunction diff --git a/examples/components/balls/im2col/verify/src/seq/im2col_sequences.svh b/examples/components/balls/im2col/verify/src/seq/im2col_sequences.svh index 3f3064388..fa9fbe655 100644 --- a/examples/components/balls/im2col/verify/src/seq/im2col_sequences.svh +++ b/examples/components/balls/im2col/verify/src/seq/im2col_sequences.svh @@ -3,7 +3,6 @@ class im2col_basic_seq extends uvm_sequence #(bb_blink_cmd_item); int unsigned case_index; int unsigned seed; - int unsigned bid; function new(string name = "im2col_basic_seq"); super.new(name); @@ -14,7 +13,7 @@ class im2col_basic_seq extends uvm_sequence #(bb_blink_cmd_item); im2col_cmd_item item; item = im2col_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(seed, case_index, bid); + item.load_rust_case(seed, case_index); finish_item(item); endtask endclass diff --git a/examples/components/balls/im2col/verify/src/tb_top.sv b/examples/components/balls/im2col/verify/src/tb_top.sv index ce1c8fc12..f129380fd 100644 --- a/examples/components/balls/im2col/verify/src/tb_top.sv +++ b/examples/components/balls/im2col/verify/src/tb_top.sv @@ -7,77 +7,66 @@ module tb_top; Im2colBall dut ( .clock(intf.clock), .reset(intf.reset), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank[3:0]), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank[3:0]), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank[3:0]), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank[3:0]), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0][3:0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_group_id(intf.bank_read_group_id[0][3:0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0][3:0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0][3:0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready), - .io_status_idle(), - .io_status_running(), - .io_bankAccessReady(1'b1), - .io_bankRead_0_ball_id(), - .io_bankWrite_0_ball_id() + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) ); + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + assign intf.bank_write_resp_ready[0] = dut.blink_bankWrite_0_io_resp_ready; + initial begin intf.clock = 1'b0; intf.reset = 1'b1; diff --git a/examples/components/balls/im2col/verify/src/tests/im2col_test.svh b/examples/components/balls/im2col/verify/src/tests/im2col_test.svh index e969a2059..e87502ea0 100644 --- a/examples/components/balls/im2col/verify/src/tests/im2col_test.svh +++ b/examples/components/balls/im2col/verify/src/tests/im2col_test.svh @@ -18,27 +18,25 @@ class im2col_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid; - bid = im2col_require_bid(); phase.raise_objection(this); - run_directed(bid); - run_random(bid); + run_directed(); + run_random(); phase.drop_objection(this); endtask - task run_directed(int unsigned bid); - run_case(0, "DIR_K3_PAD0", bid); - run_case(1, "DIR_K3_PAD1", bid); + task run_directed(); + run_case(0, "DIR_K3_PAD0"); + run_case(1, "DIR_K3_PAD1"); endtask - task run_random(int unsigned bid); + task run_random(); int i; for (i = 2; i <= 21; i++) begin - run_case(i, $sformatf("RND_%0d", i), bid); + run_case(i, $sformatf("RND_%0d", i)); end endtask - task run_case(int unsigned idx, string label, int unsigned bid); + task run_case(int unsigned idx, string label); im2col_basic_seq seq; int cycles; @@ -47,7 +45,6 @@ class im2col_ball_test extends uvm_test; seq = im2col_basic_seq::type_id::create("seq"); seq.case_index = idx; - seq.bid = bid; seq.start(env.cmd_agent.seqr); cycles = 0; diff --git a/examples/components/balls/im2col/workloads/isa/im2col.h b/examples/components/balls/im2col/workloads/isa/im2col.h index aa7834692..9a0783bc6 100644 --- a/examples/components/balls/im2col/workloads/isa/im2col.h +++ b/examples/components/balls/im2col/workloads/isa/im2col.h @@ -4,15 +4,26 @@ #include #include +// Input virtual bank: group g is channel block g (channels 16g..16g+15); +// line input_base + y * input_size + x holds pixel (y, x) of that block. +// K is ordered (kernel row, kernel column, channel), so K tile +// t = (kr * kernel + kc) * input_groups + g is one input row (zero when the +// tap falls into padding). With K_TILES = kernel * kernel * input_groups, the +// output groups must divide K_TILES; output group j holds K tiles +// j * G .. j * G + G - 1 (G = K_TILES / output_groups) in the SMatMul A +// layout: window w, local tile kt -> line ((w / tile_rows) * G + kt) * +// tile_rows + w % tile_rows. Windows window_count .. up to the next multiple +// of tile_rows are zero. Window w is output pixel window_start + w in +// row-major order. #define bb_im2col(input_bank, output_bank, input_size, kernel, stride, \ - padding, input_base, lane, start_row, start_col, \ - window_start, window_count) \ + padding, input_base, start_row, start_col, window_start, \ + window_count, tile_rows) \ BUCKYBALL_INSTRUCTION_R_R( \ BB_BANK0(input_bank) | BB_BANK2(output_bank) | BB_ITER(input_size), \ - FIELD(kernel, 0, 7) | FIELD(stride, 8, 15) | FIELD(padding, 16, 23) | \ - FIELD(start_col, 24, 31) | FIELD(start_row, 32, 39) | \ - FIELD(input_base, 40, 45) | FIELD(lane, 46, 49) | \ - FIELD(window_start, 50, 55) | FIELD(window_count, 56, 62), \ + FIELD(kernel, 0, 3) | FIELD(stride, 4, 7) | FIELD(padding, 8, 11) | \ + FIELD(start_col, 12, 15) | FIELD(start_row, 16, 19) | \ + FIELD(input_base, 20, 25) | FIELD(window_start, 26, 35) | \ + FIELD(window_count, 36, 45) | FIELD(tile_rows, 46, 50), \ BB_FUNC7(IM2COL)) #endif // _BB_IM2COL_H_ diff --git a/examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir b/examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir index 6b0e1fbf4..6331c142b 100644 --- a/examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir +++ b/examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir @@ -28,13 +28,13 @@ func.func @main() -> i8 { } } - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mvin %input <{bank = 0 : i64, depth = 64 : i64, stride = 1 : i64}> : memref<64x16xi8> - buckyball.custom "im2col_i8" <{sources = array, destination = 1 : i64}> {input_side = 6 : i64, kernel = 3 : i64, stride = 1 : i64, padding = 0 : i64, start_row = 0 : i64, start_col = 0 : i64, op1_base = 5 : i64, lane = 7 : i64, window_start = 0 : i64, window_count = 16 : i64} - buckyball.mvout %output <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 64 : i64, stride = 1 : i64}> : memref<64x16xi8> + buckyball.custom "im2col_i8" <{sources = array, destination = 2 : i64}> {input_side = 6 : i64, kernel = 3 : i64, stride = 1 : i64, padding = 0 : i64, start_row = 0 : i64, start_col = 0 : i64, op1_base = 5 : i64, lane = 7 : i64, window_start = 0 : i64, window_count = 16 : i64} + buckyball.mvout %output <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> func.call @check_result(%output) : (memref<16x16xi8>) -> () memref.dealloc %input : memref<64x16xi8> diff --git a/examples/components/balls/int2fp/arch/Int2Fp.scala b/examples/components/balls/int2fp/arch/Int2Fp.scala index 31da0f780..c91bc41ad 100644 --- a/examples/components/balls/int2fp/arch/Int2Fp.scala +++ b/examples/components/balls/int2fp/arch/Int2Fp.scala @@ -69,6 +69,9 @@ class Int2Fp( val scaleBank = RegInit(0.U(frontendParam.bank_id_len.W)) val outputBank = RegInit(0.U(frontendParam.bank_id_len.W)) val bases = Reg(Vec(3, UInt(memParam.groupIdWidth.W))) + // Every group of a virtual bank is one 16-channel block with the same rows. + val groups = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) + val group = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) val iterReg = RegInit(0.U(frontendParam.iter_len.W)) val rowReg = RegInit(0.U(log2Ceil(memParam.bankEntries).W)) val reluReg = RegInit(false.B) @@ -105,14 +108,14 @@ class Int2Fp( for (i <- 0 until mapping.inBW) { io.bankRead(i).rob_id := robIdReg io.bankRead(i).bank_id := Mux(i.U === 0.U, inputBank, scaleBank) - io.bankRead(i).group_id := bases(i) + io.bankRead(i).group_id := bases(i) + group io.bankRead(i).io.req.valid := false.B io.bankRead(i).io.req.bits.addr := Mux(i.U === 0.U, rowReg, rowReg(1, 0)) io.bankRead(i).io.resp.ready := false.B } io.bankWrite(0).rob_id := robIdReg io.bankWrite(0).bank_id := outputBank - io.bankWrite(0).group_id := bases(2) + io.bankWrite(0).group_id := bases(2) + group io.bankWrite(0).io.req.valid := false.B io.bankWrite(0).io.req.bits.addr := rowReg io.bankWrite(0).io.req.bits.data := writeData @@ -144,8 +147,9 @@ class Int2Fp( "INT32_TO_FP32 input exceeds bank depth" ) assert( - cmd.op1_col === 1.U && cmd.op2_col === 1.U && cmd.wr_col === 1.U, - "INT32_TO_FP32 operands must each occupy one bank" + cmd.op1_col =/= 0.U && cmd.op1_col === cmd.op2_col && cmd.op1_col === cmd.wr_col && + cmd.op1_col <= memParam.bankNum.U, + "INT32_TO_FP32 operands must have the same channel groups" ) assert( cmd.op1_bank =/= cmd.op2_bank && cmd.op1_bank =/= cmd.wr_bank && cmd.op2_bank =/= cmd.wr_bank, @@ -158,6 +162,8 @@ class Int2Fp( scaleBank := cmd.op2_bank outputBank := cmd.wr_bank bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + groups := cmd.op1_col + group := 0.U iterReg := cmd.iter rowReg := 0.U reluReg := cmd.rs2(0) @@ -220,10 +226,15 @@ class Int2Fp( is(writeResp) { io.bankWrite(0).io.resp.ready := true.B when(io.bankWrite(0).io.resp.fire) { - when(rowReg === iterReg - 1.U) { + when(rowReg === iterReg - 1.U && group + 1.U === groups) { state := complete }.otherwise { - rowReg := rowReg + 1.U + when(rowReg === iterReg - 1.U) { + group := group + 1.U + rowReg := 0.U + }.otherwise { + rowReg := rowReg + 1.U + } readSent.foreach(_ := false.B) readReceived.foreach(_ := false.B) state := readReq diff --git a/examples/components/balls/int2fp/emu/src/int2fp.rs b/examples/components/balls/int2fp/emu/src/int2fp.rs index 47ba26fad..2dc602703 100644 --- a/examples/components/balls/int2fp/emu/src/int2fp.rs +++ b/examples/components/balls/int2fp/emu/src/int2fp.rs @@ -1,5 +1,5 @@ use bebop_bemu::thread::banks::bank_lines; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::instruction::ExecContext; pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { @@ -16,11 +16,12 @@ pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { { panic!("int32_to_fp32: bank not allocated"); } - if ctx.config(input_bank).cols != 1 - || ctx.config(scale_bank).cols != 1 - || ctx.config(output_bank).cols != 1 + let groups = ctx.config(input_bank).cols; + if groups == 0 + || ctx.config(scale_bank).cols != groups + || ctx.config(output_bank).cols != groups { - panic!("int32_to_fp32: operands must each occupy one bank"); + panic!("int32_to_fp32: operands must have the same channel groups"); } if iter == 0 || iter % 4 != 0 || iter > bank_lines() { panic!("int32_to_fp32: iter must be a positive multiple of four within bank depth"); @@ -30,9 +31,11 @@ pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { } let relu = xs2 & 1 != 0; - let input = pbank(ctx, input_bank); - let scale = pbank(ctx, scale_bank); - let output = pbank(ctx, output_bank); + // Group g of every operand is channel block g; each block uses the same rows. + for group in 0..groups { + let input = pbank_group(ctx, input_bank, group); + let scale = pbank_group(ctx, scale_bank, group); + let output = pbank_group(ctx, output_bank, group); for row in 0..iter { for lane in 0..4 { let offset = row * 16 + lane * 4; @@ -52,6 +55,7 @@ pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { ctx.banks[output][offset..offset + 4].copy_from_slice(&result.to_le_bytes()); } } + } 0 } diff --git a/examples/components/balls/int2fp/stack/src/Lowering.cpp b/examples/components/balls/int2fp/stack/src/Lowering.cpp index 7269e4bde..a844ec824 100644 --- a/examples/components/balls/int2fp/stack/src/Lowering.cpp +++ b/examples/components/balls/int2fp/stack/src/Lowering.cpp @@ -2,8 +2,8 @@ // // int32_to_fp32: source 0 holds `rows` rows of sixteen i32 (four bank rows // each), source 1 the sixteen f32 scales of the columns (four bank rows); the -// destination receives the f32 value * scale of every element. -// int32_to_fp32_relu: the same, with negative results replaced by zero. +// destination receives the f32 value * scale of every element, with `relu` +// negative results replaced by zero. // //===----------------------------------------------------------------------===// @@ -20,13 +20,13 @@ class Int32ToFp32Lowering : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; LogicalResult matchAndRewrite(CustomOp op, PatternRewriter &rewriter) const override { - bool relu = op.getKind() == "int32_to_fp32_relu"; - if (op.getKind() != "int32_to_fp32" && !relu) + if (op.getKind() != "int32_to_fp32") return failure(); if (op.getSources().size() != 2) return op.emitOpError("int32_to_fp32 reads the values and the scales"); Location loc = op.getLoc(); int64_t rows = cast(op->getAttr("rows")).getInt(); + bool relu = cast(op->getAttr("relu")).getValue(); mlir::buddy::emitInstruction( rewriter, loc, buckyball_target::getBuckyballFunct7("INT32_TO_FP32"), mlir::buddy::constantI64(rewriter, loc, diff --git a/examples/components/balls/int2fp/verify/casegen/src/casegen.rs b/examples/components/balls/int2fp/verify/casegen/src/casegen.rs index 1573dc550..998f85029 100644 --- a/examples/components/balls/int2fp/verify/casegen/src/casegen.rs +++ b/examples/components/balls/int2fp/verify/casegen/src/casegen.rs @@ -12,18 +12,18 @@ pub const NUM_CASES: u32 = 6; #[repr(C)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Int2FpCmd { - pub bid: u32, pub iter: u32, pub relu: u32, pub op1_bank: u32, pub op2_bank: u32, pub wr_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, pub op1_col: u32, pub op2_col: u32, pub wr_col: u32, pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, pub rs2_lo: u32, pub rs2_hi: u32, pub num_src_words: u32, @@ -65,17 +65,17 @@ impl Int2FpCase { } } -pub fn gen_case(index: u32, bid: u32) -> Int2FpCase { +pub fn gen_case(index: u32) -> Int2FpCase { if index >= NUM_CASES { panic!("int2fp: unsupported directed case {index}"); } match index { - 0 => directed(bid, 2, 4, false, &MLIR_INPUT, &[0.125; 16]), - 1 => directed(bid, 3, 4, true, &RELU_INPUT, &[1.0; 16]), - 2 => directed(bid, 4, 8, false, &ITER8, &SCALE16), - 3 => directed(bid, 5, 8, true, &ITER8, &SCALE16), - 4 => directed(bid, 6, 16, false, &ITER16, &SCALE16), - 5 => directed(bid, 7, 16, true, &ITER16, &SCALE16), + 0 => directed(2, 4, false, &MLIR_INPUT, &[0.125; 16], [0, 0, 0]), + 1 => directed(3, 4, true, &RELU_INPUT, &[1.0; 16], [1, 2, 3]), + 2 => directed(4, 8, false, &ITER8, &SCALE16, [2, 0, 1]), + 3 => directed(5, 8, true, &ITER8, &SCALE16, [3, 1, 0]), + 4 => directed(6, 16, false, &ITER16, &SCALE16, [0, 3, 2]), + 5 => directed(7, 16, true, &ITER16, &SCALE16, [1, 1, 1]), _ => panic!("int2fp: unsupported directed case {index}"), } } @@ -119,10 +119,6 @@ fn copy_words(src: &[u128]) -> [u128; N] { out } -fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { - u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) -} - fn split_rs(value: u64) -> (u32, u32) { (value as u32, (value >> 32) as u32) } @@ -139,7 +135,6 @@ pub(crate) fn validate( op2_col: u32, wr_col: u32, iter: u32, - rs1: u64, rs2: u64, scales: &[u128], ) { @@ -158,15 +153,6 @@ pub(crate) fn validate( if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { panic!("INT32_TO_FP32 banks must be distinct"); } - if (rs1 & 0x3ff) != u64::from(op1_bank) - || ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) - || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) - { - panic!("int2fp: rs1 bank fields mismatch"); - } - if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { - panic!("int2fp: rs1 iter mismatch"); - } if rs2 >> 1 != 0 { panic!("INT32_TO_FP32 reserves rs2[63:1]"); } @@ -200,7 +186,15 @@ fn dst_words(src: &[u128], scales: &[u128], relu: bool) -> Vec { .collect() } -fn directed(bid: u32, rob_id: u32, iter: u32, relu: bool, values: &[i32], scales: &[f32]) -> Int2FpCase { +// `groups` are the op1/op2/wr base groups. +fn directed( + rob_id: u32, + iter: u32, + relu: bool, + values: &[i32], + scales: &[f32], + groups: [u32; 3], +) -> Int2FpCase { if values.len() != iter as usize * LANES { panic!( "int2fp: {} values do not match iter {iter} * {LANES}", @@ -218,10 +212,9 @@ fn directed(bid: u32, rob_id: u32, iter: u32, relu: bool, values: &[i32], scales let op1_col = 1; let op2_col = 1; let wr_col = 1; - let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); let rs2 = u64::from(relu); validate( - op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs1, rs2, &scale, + op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs2, &scale, ); if src.len() != iter as usize { panic!("int2fp: src rows {} != iter {iter}", src.len()); @@ -230,22 +223,21 @@ fn directed(bid: u32, rob_id: u32, iter: u32, relu: bool, values: &[i32], scales if dst.len() != iter as usize { panic!("int2fp: dst rows {} != iter {iter}", dst.len()); } - let (rs1_lo, rs1_hi) = split_rs(rs1); let (rs2_lo, rs2_hi) = split_rs(rs2); Int2FpCase { cmd: Int2FpCmd { - bid, iter, relu: u32::from(relu), op1_bank, op2_bank, wr_bank, + op1_group: groups[0], + op2_group: groups[1], + wr_group: groups[2], op1_col, op2_col, wr_col, rob_id, - rs1_lo, - rs1_hi, rs2_lo, rs2_hi, num_src_words: src.len() as u32, diff --git a/examples/components/balls/int2fp/verify/casegen/src/lib.rs b/examples/components/balls/int2fp/verify/casegen/src/lib.rs index bf1d33335..8ae4bbf68 100644 --- a/examples/components/balls/int2fp/verify/casegen/src/lib.rs +++ b/examples/components/balls/int2fp/verify/casegen/src/lib.rs @@ -11,8 +11,8 @@ thread_local! { } #[no_mangle] -pub extern "C" fn int2fp_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid); +pub extern "C" fn int2fp_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); CURRENT.with(|current| *current.borrow_mut() = Some(case)); 0 } diff --git a/examples/components/balls/int2fp/verify/filelists/int2fp_ball.f b/examples/components/balls/int2fp/verify/filelists/int2fp_ball.f index 4bef90c87..937d5b30f 100644 --- a/examples/components/balls/int2fp/verify/filelists/int2fp_ball.f +++ b/examples/components/balls/int2fp/verify/filelists/int2fp_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,6 +14,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv diff --git a/examples/components/balls/int2fp/verify/src/common/int2fp_defs.svh b/examples/components/balls/int2fp/verify/src/common/int2fp_defs.svh index 5ae0d67a8..72a9778ec 100644 --- a/examples/components/balls/int2fp/verify/src/common/int2fp_defs.svh +++ b/examples/components/balls/int2fp/verify/src/common/int2fp_defs.svh @@ -1,16 +1,16 @@ typedef struct { - int unsigned bid; int unsigned iter; int unsigned relu; int unsigned op1_bank; int unsigned op2_bank; int unsigned wr_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; int unsigned op1_col; int unsigned op2_col; int unsigned wr_col; int unsigned rob_id; - int unsigned rs1_lo; - int unsigned rs1_hi; int unsigned rs2_lo; int unsigned rs2_hi; int unsigned num_src_words; @@ -18,10 +18,7 @@ typedef struct { int unsigned num_dst_words; } int2fp_cmd_dpi_t; -import "DPI-C" function int int2fp_case_load( - input int unsigned index, - input int unsigned bid -); +import "DPI-C" function int int2fp_case_load(input int unsigned index); import "DPI-C" function void int2fp_case_cmd(output int2fp_cmd_dpi_t cmd); import "DPI-C" function longint unsigned int2fp_case_src_word_lo(input int unsigned word_index); import "DPI-C" function longint unsigned int2fp_case_src_word_hi(input int unsigned word_index); @@ -40,11 +37,3 @@ localparam int INT2FP_MAX_SCALE_WORDS = 4; localparam int INT2FP_MAX_DST_WORDS = 16; localparam int INT2FP_NUM_CASES = 6; localparam int INT2FP_TIMEOUT_CYCLES = 20000; - -function automatic int unsigned int2fp_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/components/balls/int2fp/verify/src/common/int2fp_items.svh b/examples/components/balls/int2fp/verify/src/common/int2fp_items.svh index 12cccf89a..b33c411b8 100644 --- a/examples/components/balls/int2fp/verify/src/common/int2fp_items.svh +++ b/examples/components/balls/int2fp/verify/src/common/int2fp_items.svh @@ -9,64 +9,43 @@ class int2fp_cmd_item extends bb_blink_cmd_item; bit [127:0] scale_words[INT2FP_MAX_SCALE_WORDS]; bit [127:0] dst_words[INT2FP_MAX_DST_WORDS]; - constraint legal_c { - funct7 == INT32_TO_FP32_THREAD_FUNCT7[6:0]; - op1_en == 1'b1; - op2_en == 1'b1; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b1; - op1_col == 5'd1; - op2_col == 5'd1; - wr_col == 5'd1; - meta_bank == 5'd0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - special == 64'd0; - } - function new(string name = "int2fp_cmd_item"); super.new(name); endfunction - function void load_rust_case(int unsigned index, int unsigned bid); + function void load_rust_case(int unsigned index); int2fp_cmd_dpi_t cmd; int unsigned rc; longint unsigned w_lo; longint unsigned w_hi; int i; - rc = int2fp_case_load(index, bid); + rc = int2fp_case_load(index); if (rc != 0) begin `uvm_fatal("CASE", $sformatf("int2fp_case_load returned %0d for index %0d", rc, index)) end int2fp_case_cmd(cmd); - this.bid = cmd.bid[4:0]; + // bb_int32_to_fp32: input/scale/output banks and iter come from rs1, relu from rs2[0]. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = INT32_TO_FP32_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; relu = cmd.relu; - op1_bank = cmd.op1_bank[4:0]; - op2_bank = cmd.op2_bank[4:0]; - wr_bank = cmd.wr_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - op2_col = cmd.op2_col[4:0]; - wr_col = cmd.wr_col[4:0]; - rob_id = cmd.rob_id[3:0]; - rs1 = {cmd.rs1_hi, cmd.rs1_lo}; - rs2 = {cmd.rs2_hi, cmd.rs2_lo}; num_src_words = cmd.num_src_words; num_scale_words = cmd.num_scale_words; num_dst_words = cmd.num_dst_words; - funct7 = INT32_TO_FP32_THREAD_FUNCT7[6:0]; - special = 64'd0; - op1_en = 1'b1; - op2_en = 1'b1; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b1; - meta_bank = 5'd0; - is_sub = 1'b0; - sub_rob_id = 8'h00; if (iter == 0 || iter[1:0] != 2'b00) `uvm_fatal("CASE", "INT32_TO_FP32 iter must be a positive multiple of four") @@ -88,8 +67,6 @@ class int2fp_cmd_item extends bb_blink_cmd_item; `uvm_fatal("CASE", $sformatf("num_src_words out of range: %0d", num_src_words)) if (num_dst_words == 0 || num_dst_words > INT2FP_MAX_DST_WORDS) `uvm_fatal("CASE", $sformatf("num_dst_words out of range: %0d", num_dst_words)) - if (funct7 != INT32_TO_FP32_THREAD_FUNCT7[6:0]) - `uvm_fatal("CASE", $sformatf("invalid funct7: %0d", funct7)) for (i = 0; i < num_src_words; i++) begin w_lo = int2fp_case_src_word_lo(i); diff --git a/examples/components/balls/int2fp/verify/src/cov/int2fp_cov.svh b/examples/components/balls/int2fp/verify/src/cov/int2fp_cov.svh index ffa99a772..2c0867867 100644 --- a/examples/components/balls/int2fp/verify/src/cov/int2fp_cov.svh +++ b/examples/components/balls/int2fp/verify/src/cov/int2fp_cov.svh @@ -20,8 +20,8 @@ class int2fp_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != INT32_TO_FP32_THREAD_FUNCT7[6:0]) - `uvm_fatal("COV", $sformatf("unexpected funct7 %0d", item.funct7)) + if (item.funct != INT32_TO_FP32_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) cur_relu = item.rs2[0]; cur_iter = item.iter; cmd_count++; diff --git a/examples/components/balls/int2fp/verify/src/env/int2fp_scoreboard.svh b/examples/components/balls/int2fp/verify/src/env/int2fp_scoreboard.svh index 15d1636ca..e017d383e 100644 --- a/examples/components/balls/int2fp/verify/src/env/int2fp_scoreboard.svh +++ b/examples/components/balls/int2fp/verify/src/env/int2fp_scoreboard.svh @@ -47,10 +47,10 @@ class int2fp_scoreboard extends uvm_scoreboard; end mem_model.clear_mem(); for (i = 0; i < clone.num_src_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), i, clone.src_words[i]); + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group), i, clone.src_words[i]); end for (i = 0; i < clone.num_scale_words; i++) begin - mem_model.load_word(int'(clone.op2_bank), i, clone.scale_words[i]); + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group), i, clone.scale_words[i]); end mem_model.arm(); @@ -67,46 +67,39 @@ class int2fp_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, int2fp_cmd_item exp); + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); verification_pkg::evidence::checks++; - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.op2_bank !== exp.op2_bank || - got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.op2_col !== exp.op2_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rs1 !== exp.rs1 || got.rs2 !== exp.rs2) `uvm_fatal("CMD", "rs field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) - if (got.rs2[63:1] !== '0) `uvm_fatal("CMD", "INT32_TO_FP32 reserves rs2[63:1]") + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) endfunction function void write_read(bb_blink_read_item item); int2fp_cmd_item stim; int unsigned expect_bank; + int unsigned expect_group; int unsigned expect_addr; stim = current_stim("READ"); if (item.port != 0 && item.port != 1) `uvm_fatal("READ", $sformatf("Int2FpBall has two read ports, got port %0d", item.port)) - if (item.group_id !== 5'd0) - `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp 0", item.group_id)) if (item.rob_id !== stim.rob_id) `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) if (item.port == 0) begin - expect_bank = stim.op1_bank; - expect_addr = read_count[0]; + expect_bank = stim.op1_bank; + expect_group = stim.op1_group; + expect_addr = read_count[0]; end else begin - expect_bank = stim.op2_bank; - expect_addr = read_count[1] & 32'h3; + expect_bank = stim.op2_bank; + expect_group = stim.op2_group; + expect_addr = read_count[1] & 32'h3; end - if (item.bank_id !== expect_bank[4:0]) + if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) + if (item.group_id !== expect_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, expect_group)) if (item.addr !== expect_addr[BB_BLINK_BANK_ADDR_W-1:0]) `uvm_fatal("READ", $sformatf("addr mismatch: got %0d exp %0d", item.addr, expect_addr)) read_count[item.port]++; @@ -122,8 +115,9 @@ class int2fp_scoreboard extends uvm_scoreboard; `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== 5'd0) - `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp 0", item.group_id)) + if (item.group_id !== stim.wr_group) + `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + )) if (item.mask !== 16'hFFFF) `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) if (write_count >= stim.num_dst_words) @@ -146,9 +140,6 @@ class int2fp_scoreboard extends uvm_scoreboard; stim = current_stim("RESP"); if (item.rob_id !== stim.rob_id) `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") - if (item.sub_rob_id !== 8'h00) - `uvm_fatal("RESP", $sformatf("sub_rob_id mismatch: got 0x%0h", item.sub_rob_id)) resp_count++; endfunction diff --git a/examples/components/balls/int2fp/verify/src/seq/int2fp_sequences.svh b/examples/components/balls/int2fp/verify/src/seq/int2fp_sequences.svh index 0bea8e3e2..53dc6c26e 100644 --- a/examples/components/balls/int2fp/verify/src/seq/int2fp_sequences.svh +++ b/examples/components/balls/int2fp/verify/src/seq/int2fp_sequences.svh @@ -2,7 +2,6 @@ class int2fp_basic_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(int2fp_basic_seq) int unsigned case_index; - int unsigned bid; function new(string name = "int2fp_basic_seq"); super.new(name); @@ -12,7 +11,7 @@ class int2fp_basic_seq extends uvm_sequence #(bb_blink_cmd_item); int2fp_cmd_item item; item = int2fp_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(case_index, bid); + item.load_rust_case(case_index); finish_item(item); endtask endclass diff --git a/examples/components/balls/int2fp/verify/src/tb_top.sv b/examples/components/balls/int2fp/verify/src/tb_top.sv index 219f61c87..6d77594a4 100644 --- a/examples/components/balls/int2fp/verify/src/tb_top.sv +++ b/examples/components/balls/int2fp/verify/src/tb_top.sv @@ -7,87 +7,75 @@ module tb_top; Int2FpBall dut ( .clock(intf.clock), .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_bankAccessReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) ); + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + initial begin intf.clock = 1'b0; intf.reset = 1'b1; diff --git a/examples/components/balls/int2fp/verify/src/tests/int2fp_test.svh b/examples/components/balls/int2fp/verify/src/tests/int2fp_test.svh index 1806dbfa8..31961b3fc 100644 --- a/examples/components/balls/int2fp/verify/src/tests/int2fp_test.svh +++ b/examples/components/balls/int2fp/verify/src/tests/int2fp_test.svh @@ -17,20 +17,18 @@ class int2fp_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid = int2fp_require_bid(); phase.raise_objection(this); - for (int unsigned index = 0; index < INT2FP_NUM_CASES; index++) run_case(index, bid); + for (int unsigned index = 0; index < INT2FP_NUM_CASES; index++) run_case(index); phase.drop_objection(this); endtask - task run_case(int unsigned index, int unsigned bid); + task run_case(int unsigned index); int2fp_basic_seq seq; int cycles = 0; apply_reset(); env.scb.reset_counters(); seq = int2fp_basic_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); while (!env.scb.done()) begin @(posedge vif.clock); diff --git a/examples/components/balls/int2fp/workloads/isa/int2fp.h b/examples/components/balls/int2fp/workloads/isa/int2fp.h index 1a832365c..3e2a87e8d 100644 --- a/examples/components/balls/int2fp/workloads/isa/int2fp.h +++ b/examples/components/balls/int2fp/workloads/isa/int2fp.h @@ -4,6 +4,11 @@ #include #include +// Group g of every operand bank is channel block g (channels 16g..16g+15). +// Input, scale, and output banks have the same number of groups; every group +// uses the same rows. In group g, input row r holds 4 INT32 lanes, scale row +// r % 4 holds their scales, and output row r holds the 4 FP32 results. + #define bb_int32_to_fp32(input_bank, scale_bank, output_bank, iter, relu) \ BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(input_bank) | BB_BANK1(scale_bank) | \ BB_BANK2(output_bank) | BB_ITER(iter), \ diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir b/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir index 79ba711b9..3c85da332 100644 --- a/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir +++ b/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir @@ -12,17 +12,17 @@ func.func @main() -> i8 { %input = memref.get_global @input : memref<4x4xi32> %scales = memref.get_global @scales : memref<4x4xf32> %output = memref.alloc() alignment = 64 : memref<4x4xf32> - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mvin %input <{bank = 0 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> - buckyball.mvin %scales <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - buckyball.custom "int32_to_fp32" <{sources = array, destination = 2 : i64}> {rows = 1 : i64} - buckyball.mvout %output <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> + buckyball.mvin %scales <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + buckyball.custom "int32_to_fp32" <{sources = array, destination = 3 : i64}> {rows = 1 : i64, relu = false} + buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> func.call @check_result(%output) : (memref<4x4xf32>) -> () - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> memref.dealloc %output : memref<4x4xf32> return %zero_i8 : i8 } diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir b/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir index 993c6e850..87601fee7 100644 --- a/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir +++ b/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir @@ -11,17 +11,17 @@ func.func @main() -> i8 { %input = memref.get_global @input : memref<4x4xi32> %scales = memref.get_global @scales : memref<4x4xf32> %output = memref.alloc() alignment = 64 : memref<4x4xf32> - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mvin %input <{bank = 0 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> - buckyball.mvin %scales <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - buckyball.custom "int32_to_fp32_relu" <{sources = array, destination = 2 : i64}> {rows = 1 : i64} - buckyball.mvout %output <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> + buckyball.mvin %scales <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + buckyball.custom "int32_to_fp32" <{sources = array, destination = 3 : i64}> {rows = 1 : i64, relu = true} + buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> func.call @check_result(%output) : (memref<4x4xf32>) -> () - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> memref.dealloc %output : memref<4x4xf32> return %zero_i8 : i8 } diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir b/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir index a4288842c..6da48fada 100644 --- a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir +++ b/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir @@ -17,7 +17,7 @@ func.func @main() -> i8 { %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> %loaded_scales = buckyball.bank_mvin %scales, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - %result = buckyball.bank_custom "int32_to_fp32" (%loaded_input, %loaded_scales) -> %output_bank {rows = 1 : i64} + %result = buckyball.bank_custom "int32_to_fp32" (%loaded_input, %loaded_scales) -> %output_bank {rows = 1 : i64, relu = false} %stored = buckyball.bank_mvout %output, %result <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> func.call @check_result(%output) : (memref<4x4xf32>) -> () buckyball.bank_release %loaded_input diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir b/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir index 58ac1465d..3372b743d 100644 --- a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir +++ b/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir @@ -16,7 +16,7 @@ func.func @main() -> i8 { %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> %loaded_scales = buckyball.bank_mvin %scales, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - %result = buckyball.bank_custom "int32_to_fp32_relu" (%loaded_input, %loaded_scales) -> %output_bank {rows = 1 : i64} + %result = buckyball.bank_custom "int32_to_fp32" (%loaded_input, %loaded_scales) -> %output_bank {rows = 1 : i64, relu = true} %stored = buckyball.bank_mvout %output, %result <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> func.call @check_result(%output) : (memref<4x4xf32>) -> () buckyball.bank_release %loaded_input diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in b/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in index 26ca8c1e4..b2882d24a 100644 --- a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in +++ b/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in @@ -23,7 +23,7 @@ func.func @main() -> i8 { %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> %loaded_scale = buckyball.bank_mvin %scale, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - %result = buckyball.bank_custom "int32_to_fp32" (%loaded_input, %loaded_scale) -> %output_bank {rows = @BB_INT2FP_ROWS@ : i64} + %result = buckyball.bank_custom "int32_to_fp32" (%loaded_input, %loaded_scale) -> %output_bank {rows = @BB_INT2FP_ROWS@ : i64, relu = false} %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32> func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32>) -> () buckyball.bank_release %loaded_input diff --git a/examples/components/balls/int8add/arch/Int8AddBall.scala b/examples/components/balls/int8add/arch/Int8AddBall.scala index 11fab6a9b..5fdbee567 100644 --- a/examples/components/balls/int8add/arch/Int8AddBall.scala +++ b/examples/components/balls/int8add/arch/Int8AddBall.scala @@ -80,8 +80,8 @@ class Int8AddBall( val add = Instantiate(new PipelinedAddRawFN(8, 24)) - val round = Instantiate(new RoundRawFNToRecFN(8, 24, 0)) - val toInt = Instantiate(new RecFNToIN(8, 24, 8)) + val round = Module(new RoundRawFNToRecFN(8, 24, 0)) + val toInt = Module(new RecFNToIN(8, 24, 8)) val lhsValue = (lhsWord >> (lane << 3))(7, 0) val rhsValue = (rhsWord >> (lane << 3))(7, 0) val calculateIssue = state === calculate diff --git a/examples/components/balls/int8add/verify/casegen/src/casegen.rs b/examples/components/balls/int8add/verify/casegen/src/casegen.rs index 95ccc3b69..94a045b1b 100644 --- a/examples/components/balls/int8add/verify/casegen/src/casegen.rs +++ b/examples/components/balls/int8add/verify/casegen/src/casegen.rs @@ -1,23 +1,23 @@ pub const ROW_BYTES: usize = 16; pub const BANK_ENTRIES: u32 = 64; pub const MAX_ROWS: usize = 64; -pub const NUM_CASES: u32 = 2; +pub const NUM_CASES: u32 = 3; #[repr(C)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Int8AddCmd { pub relu: u32, - pub bid: u32, pub iter: u32, pub op1_bank: u32, pub op2_bank: u32, pub wr_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, pub op1_col: u32, pub op2_col: u32, pub wr_col: u32, pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, pub rs2_lo: u32, pub rs2_hi: u32, pub num_lhs_words: u32, @@ -59,13 +59,15 @@ impl Int8AddCase { } } -pub fn gen_case(index: u32, bid: u32) -> Int8AddCase { +// Word `i` of an operand is bank row `i % iter` in group `i / iter`: the Ball walks groups in order. +pub fn gen_case(index: u32) -> Int8AddCase { if index >= NUM_CASES { panic!("int8add: unsupported directed case {index}"); } match index { - 0 => directed(bid, 2, false), - 1 => directed(bid, 3, true), + 0 => directed(2, false, 1, [0, 0, 0]), + 1 => directed(3, true, 1, [0, 0, 0]), + 2 => directed(4, false, 2, [1, 3, 2]), _ => panic!("int8add: unsupported directed case {index}"), } } @@ -125,10 +127,6 @@ fn copy_words(src: &[u128]) -> [u128; N] { out } -fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { - u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) -} - fn encode_rs2(lhs_ratio: f32, rhs_ratio: f32) -> u64 { u64::from(lhs_ratio.to_bits()) | (u64::from(rhs_ratio.to_bits()) << 32) } @@ -145,7 +143,6 @@ pub(crate) fn validate( op2_col: u32, wr_col: u32, iter: u32, - rs1: u64, rs2: u64, lhs_ratio: f32, rhs_ratio: f32, @@ -153,7 +150,7 @@ pub(crate) fn validate( if iter == 0 || iter > BANK_ENTRIES { panic!("int8add: iter must fit in one physical bank"); } - if op1_col != 1 || op2_col != 1 || wr_col != 1 { + if op1_col == 0 || op1_col != op2_col || op1_col != wr_col { panic!("int8add: bank groups must match"); } if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { @@ -162,15 +159,6 @@ pub(crate) fn validate( if !positive_finite(lhs_ratio) || !positive_finite(rhs_ratio) { panic!("int8add: ratios must be finite and positive"); } - if (rs1 & 0x3ff) != u64::from(op1_bank) - || ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) - || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) - { - panic!("int8add: rs1 bank fields mismatch"); - } - if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { - panic!("int8add: rs1 iter mismatch"); - } if rs2 != encode_rs2(lhs_ratio, rhs_ratio) { panic!("int8add: rs2 ratio fields mismatch"); } @@ -179,13 +167,13 @@ pub(crate) fn validate( fn dst_words( lhs: &[u128], rhs: &[u128], - iter: u32, + words: usize, lhs_ratio: f32, rhs_ratio: f32, relu: bool, ) -> Vec { - let mut out = Vec::with_capacity(iter as usize); - for row in 0..iter as usize { + let mut out = Vec::with_capacity(words); + for row in 0..words { let mut packed = 0u128; for lane in 0..ROW_BYTES { let l = ((lhs[row] >> (lane * 8)) & 0xff) as u8 as i8; @@ -198,52 +186,35 @@ fn dst_words( out } -fn directed(bid: u32, rob_id: u32, relu: bool) -> Int8AddCase { +fn directed(rob_id: u32, relu: bool, cols: u32, groups: [u32; 3]) -> Int8AddCase { let iter = 7u32; + let words = (iter * cols) as usize; let lhs_ratio = 0.5f32; let rhs_ratio = 0.25f32; - let (lhs, rhs) = ctest_values(iter as usize); + let (lhs, rhs) = ctest_values(words); let op1_bank = 0; let op2_bank = 1; let wr_bank = 2; - let op1_col = 1; - let op2_col = 1; - let wr_col = 1; - let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); let rs2 = encode_rs2(lhs_ratio, rhs_ratio); validate( - op1_bank, - op2_bank, - wr_bank, - op1_col, - op2_col, - wr_col, - iter, - rs1, - rs2, - lhs_ratio, - rhs_ratio, + op1_bank, op2_bank, wr_bank, cols, cols, cols, iter, rs2, lhs_ratio, rhs_ratio, ); - if lhs.len() != iter as usize || rhs.len() != iter as usize { - panic!("int8add: operand rows != iter {iter}"); - } - let dst = dst_words(&lhs, &rhs, iter, lhs_ratio, rhs_ratio, relu); - let (rs1_lo, rs1_hi) = split_rs(rs1); + let dst = dst_words(&lhs, &rhs, words, lhs_ratio, rhs_ratio, relu); let (rs2_lo, rs2_hi) = split_rs(rs2); Int8AddCase { cmd: Int8AddCmd { relu: u32::from(relu), - bid, iter, op1_bank, op2_bank, wr_bank, - op1_col, - op2_col, - wr_col, + op1_group: groups[0], + op2_group: groups[1], + wr_group: groups[2], + op1_col: cols, + op2_col: cols, + wr_col: cols, rob_id, - rs1_lo, - rs1_hi, rs2_lo, rs2_hi, num_lhs_words: lhs.len() as u32, diff --git a/examples/components/balls/int8add/verify/casegen/src/lib.rs b/examples/components/balls/int8add/verify/casegen/src/lib.rs index e1c679c8a..a57b3c268 100644 --- a/examples/components/balls/int8add/verify/casegen/src/lib.rs +++ b/examples/components/balls/int8add/verify/casegen/src/lib.rs @@ -9,8 +9,8 @@ thread_local! { } #[no_mangle] -pub extern "C" fn int8add_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid); +pub extern "C" fn int8add_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); CURRENT.with(|current| *current.borrow_mut() = Some(case)); 0 } diff --git a/examples/components/balls/int8add/verify/filelists/int8add_ball.f b/examples/components/balls/int8add/verify/filelists/int8add_ball.f index 1a0f1aefc..33eecc654 100644 --- a/examples/components/balls/int8add/verify/filelists/int8add_ball.f +++ b/examples/components/balls/int8add/verify/filelists/int8add_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,17 +14,18 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv @RTL@/RoundAnyRawFNToRecFN_ie4_is8_oe8_os24.sv -@RTL@/RoundAnyRawFNToRecFN_ie8_is26_oe8_os24.sv +@RTL@/IntToFloat.sv @RTL@/MulAddRecFNToRaw_preMul_e8_s24.sv @RTL@/MulAddRecFNToRaw_postMul_e8_s24.sv -@RTL@/MulAddRecFNPipe_l3_e8_s24.sv -@RTL@/PipelinedINToRecFN_3.sv -@RTL@/PipelinedAddRawFN.sv +@RTL@/RoundAnyRawFNToRecFN_ie8_is26_oe8_os24.sv @RTL@/RoundRawFNToRecFN_e8_s24.sv +@RTL@/Fma_l3_e8_s24.sv +@RTL@/PipelinedAddRawFN.sv @RTL@/RecFNToIN_e8_s24_i8.sv @RTL@/Int8AddBall.sv diff --git a/examples/components/balls/int8add/verify/src/common/int8add_defs.svh b/examples/components/balls/int8add/verify/src/common/int8add_defs.svh index b39b98dc6..428c45e36 100644 --- a/examples/components/balls/int8add/verify/src/common/int8add_defs.svh +++ b/examples/components/balls/int8add/verify/src/common/int8add_defs.svh @@ -1,16 +1,16 @@ typedef struct { int unsigned relu; - int unsigned bid; int unsigned iter; int unsigned op1_bank; int unsigned op2_bank; int unsigned wr_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; int unsigned op1_col; int unsigned op2_col; int unsigned wr_col; int unsigned rob_id; - int unsigned rs1_lo; - int unsigned rs1_hi; int unsigned rs2_lo; int unsigned rs2_hi; int unsigned num_lhs_words; @@ -18,10 +18,7 @@ typedef struct { int unsigned num_dst_words; } int8add_cmd_dpi_t; -import "DPI-C" function int int8add_case_load( - input int unsigned index, - input int unsigned bid -); +import "DPI-C" function int int8add_case_load(input int unsigned index); import "DPI-C" function void int8add_case_cmd(output int8add_cmd_dpi_t cmd); import "DPI-C" function longint unsigned int8add_case_lhs_word_lo(input int unsigned word_index); import "DPI-C" function longint unsigned int8add_case_lhs_word_hi(input int unsigned word_index); @@ -40,13 +37,5 @@ import "DPI-C" function longint unsigned int8add_case_dst_word_hi(input int unsi localparam int INT8ADD_THREAD_FUNCT7 = `INT8ADD_FUNCT7; localparam int INT8ADD_RELU_THREAD_FUNCT7 = `INT8ADD_RELU_FUNCT7; localparam int INT8ADD_MAX_ROWS = 64; -localparam int INT8ADD_NUM_CASES = 2; +localparam int INT8ADD_NUM_CASES = 3; localparam int INT8ADD_TIMEOUT_CYCLES = 20000; - -function automatic int unsigned int8add_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/components/balls/int8add/verify/src/common/int8add_items.svh b/examples/components/balls/int8add/verify/src/common/int8add_items.svh index 6fd58d963..eed15ba3f 100644 --- a/examples/components/balls/int8add/verify/src/common/int8add_items.svh +++ b/examples/components/balls/int8add/verify/src/common/int8add_items.svh @@ -9,34 +9,18 @@ class int8add_cmd_item extends bb_blink_cmd_item; bit [127:0] rhs_words[INT8ADD_MAX_ROWS]; bit [127:0] dst_words[INT8ADD_MAX_ROWS]; - constraint legal_c { - funct7 inside {INT8ADD_THREAD_FUNCT7[6:0], INT8ADD_RELU_THREAD_FUNCT7[6:0]}; - op1_en == 1'b1; - op2_en == 1'b1; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b1; - op1_col == 5'd1; - op2_col == 5'd1; - wr_col == 5'd1; - meta_bank == 5'd0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - special == 64'd0; - } - function new(string name = "int8add_cmd_item"); super.new(name); endfunction - function void load_rust_case(int unsigned index, int unsigned bid); + function void load_rust_case(int unsigned index); int8add_cmd_dpi_t cmd; int unsigned rc; longint unsigned w_lo; longint unsigned w_hi; int i; - rc = int8add_case_load(index, bid); + rc = int8add_case_load(index); if (rc != 0) begin `uvm_fatal("CASE", $sformatf("int8add_case_load returned %0d for index %0d", rc, index)) end @@ -46,47 +30,44 @@ class int8add_cmd_item extends bb_blink_cmd_item; `uvm_fatal("CASE", $sformatf("invalid relu flag %0d", cmd.relu)) end + // bb_int8add: banks and iter come from rs1, ratios from rs2. relu = cmd.relu; - this.bid = cmd.bid[4:0]; - iter = cmd.iter; - op1_bank = cmd.op1_bank[4:0]; - op2_bank = cmd.op2_bank[4:0]; - wr_bank = cmd.wr_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - op2_col = cmd.op2_col[4:0]; - wr_col = cmd.wr_col[4:0]; - rob_id = cmd.rob_id[3:0]; - rs1 = {cmd.rs1_hi, cmd.rs1_lo}; + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = (relu == 0) ? INT8ADD_THREAD_FUNCT7[6:0] : INT8ADD_RELU_THREAD_FUNCT7[6:0]; rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; num_lhs_words = cmd.num_lhs_words; num_rhs_words = cmd.num_rhs_words; num_dst_words = cmd.num_dst_words; - special = 64'd0; - op1_en = 1'b1; - op2_en = 1'b1; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b1; - meta_bank = 5'd0; - is_sub = 1'b0; - sub_rob_id = 8'h00; - - if (relu == 0) funct7 = INT8ADD_THREAD_FUNCT7[6:0]; - else funct7 = INT8ADD_RELU_THREAD_FUNCT7[6:0]; if (iter == 0) `uvm_fatal("CASE", "Int8AddBall iter must be positive") - if (op1_col != 1 || op2_col != 1 || wr_col != 1) + if (op1_col == 0 || op1_col != op2_col || op1_col != wr_col) `uvm_fatal("CASE", "Int8AddBall bank groups must match") if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) `uvm_fatal("CASE", "Int8AddBall banks must be distinct") - if (num_lhs_words != iter) - `uvm_fatal("CASE", $sformatf("num_lhs_words %0d != iter %0d", num_lhs_words, iter)) - if (num_rhs_words != iter) - `uvm_fatal("CASE", $sformatf("num_rhs_words %0d != iter %0d", num_rhs_words, iter)) - if (num_dst_words != iter) - `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter %0d", num_dst_words, iter)) - if (funct7 != INT8ADD_THREAD_FUNCT7[6:0] && funct7 != INT8ADD_RELU_THREAD_FUNCT7[6:0]) - `uvm_fatal("CASE", $sformatf("invalid funct7: %0d", funct7)) + if (num_lhs_words != iter * op1_col || num_rhs_words != iter * op1_col || + num_dst_words != iter * op1_col) + `uvm_fatal("CASE", $sformatf( + "word counts %0d/%0d/%0d != iter %0d * col %0d", + num_lhs_words, + num_rhs_words, + num_dst_words, + iter, + op1_col + )) + if (num_lhs_words > INT8ADD_MAX_ROWS) + `uvm_fatal("CASE", $sformatf("word count %0d exceeds %0d", num_lhs_words, INT8ADD_MAX_ROWS)) for (i = 0; i < num_lhs_words; i++) begin w_lo = int8add_case_lhs_word_lo(i); diff --git a/examples/components/balls/int8add/verify/src/cov/int8add_cov.svh b/examples/components/balls/int8add/verify/src/cov/int8add_cov.svh index 53938ff62..f629564b5 100644 --- a/examples/components/balls/int8add/verify/src/cov/int8add_cov.svh +++ b/examples/components/balls/int8add/verify/src/cov/int8add_cov.svh @@ -3,10 +3,12 @@ class int8add_cov extends uvm_component; uvm_analysis_imp_cmd #(bb_blink_cmd_item, int8add_cov) cmd_imp; - bit [6:0] cur_funct7; + bit [6:0] cur_funct; + int unsigned cur_col; covergroup cmd_cg; - coverpoint cur_funct7 { + coverpoint cur_col {bins c1 = {1}; bins c2 = {2};} + coverpoint cur_funct { bins add = {INT8ADD_THREAD_FUNCT7[6:0]}; bins relu = {INT8ADD_RELU_THREAD_FUNCT7[6:0]}; } endgroup @@ -20,9 +22,10 @@ class int8add_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != INT8ADD_THREAD_FUNCT7[6:0] && item.funct7 != INT8ADD_RELU_THREAD_FUNCT7[6:0]) - `uvm_fatal("COV", $sformatf("unexpected funct7 %0d", item.funct7)) - cur_funct7 = item.funct7; + if (item.funct != INT8ADD_THREAD_FUNCT7[6:0] && item.funct != INT8ADD_RELU_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) + cur_funct = item.funct; + cur_col = item.op1_col; cmd_count++; cmd_cg.sample(); endfunction diff --git a/examples/components/balls/int8add/verify/src/env/int8add_scoreboard.svh b/examples/components/balls/int8add/verify/src/env/int8add_scoreboard.svh index e5f118865..f52fe175c 100644 --- a/examples/components/balls/int8add/verify/src/env/int8add_scoreboard.svh +++ b/examples/components/balls/int8add/verify/src/env/int8add_scoreboard.svh @@ -46,17 +46,20 @@ class int8add_scoreboard extends uvm_scoreboard; `uvm_fatal("SCB", "mem_model handle not set") end mem_model.clear_mem(); + // Word i lives at row i % iter, group base + i / iter. for (i = 0; i < clone.num_lhs_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), i, clone.lhs_words[i]); + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group) + i / clone.iter, + i % clone.iter, clone.lhs_words[i]); end for (i = 0; i < clone.num_rhs_words; i++) begin - mem_model.load_word(int'(clone.op2_bank), i, clone.rhs_words[i]); + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group) + i / clone.iter, + i % clone.iter, clone.rhs_words[i]); end mem_model.arm(); expected_writes = clone.num_dst_words; - expected_reads[0] = clone.iter; - expected_reads[1] = clone.iter; + expected_reads[0] = clone.num_lhs_words; + expected_reads[1] = clone.num_rhs_words; endfunction function void write_cmd(bb_blink_cmd_item item); @@ -67,44 +70,38 @@ class int8add_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, int8add_cmd_item exp); + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); verification_pkg::evidence::checks++; - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.op2_bank !== exp.op2_bank || - got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.op2_col !== exp.op2_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rs1 !== exp.rs1 || got.rs2 !== exp.rs2) `uvm_fatal("CMD", "rs field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) endfunction function void write_read(bb_blink_read_item item); int8add_cmd_item stim; int unsigned expect_bank; + int unsigned expect_group; int unsigned expect_addr; stim = current_stim("READ"); if (item.port != 0 && item.port != 1) `uvm_fatal("READ", $sformatf("Int8AddBall has two read ports, got port %0d", item.port)) - if (item.group_id !== 5'd0) - `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp 0", item.group_id)) if (item.rob_id !== stim.rob_id) `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) if (item.port == 0) begin - expect_bank = stim.op1_bank; - expect_addr = read_count[0]; + expect_bank = stim.op1_bank; + expect_group = stim.op1_group + read_count[0] / stim.iter; + expect_addr = read_count[0] % stim.iter; end else begin - expect_bank = stim.op2_bank; - expect_addr = read_count[1]; + expect_bank = stim.op2_bank; + expect_group = stim.op2_group + read_count[1] / stim.iter; + expect_addr = read_count[1] % stim.iter; end - if (item.bank_id !== expect_bank[4:0]) + if (item.group_id !== expect_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, expect_group)) + if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) if (item.addr !== expect_addr[BB_BLINK_BANK_ADDR_W-1:0]) `uvm_fatal("READ", $sformatf("addr mismatch: got %0d exp %0d", item.addr, expect_addr)) @@ -121,14 +118,14 @@ class int8add_scoreboard extends uvm_scoreboard; `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== 5'd0) - `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp 0", item.group_id)) + if (item.group_id !== stim.wr_group + write_count / stim.iter) + `uvm_fatal("WRITE", $sformatf("group mismatch at %0d: got %0d", write_count, item.group_id)) if (item.mask !== 16'hFFFF) `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) if (write_count >= stim.num_dst_words) `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) - if (item.addr !== write_count[BB_BLINK_BANK_ADDR_W-1:0]) - `uvm_fatal("WRITE", $sformatf("addr mismatch: got %0d exp %0d", item.addr, write_count)) + if (item.addr !== write_count % stim.iter) + `uvm_fatal("WRITE", $sformatf("addr mismatch at %0d: got %0d", write_count, item.addr)) if (item.data !== stim.dst_words[write_count]) `uvm_fatal("SCB", $sformatf( "data mismatch at addr %0d: got 0x%032h exp 0x%032h", @@ -145,9 +142,6 @@ class int8add_scoreboard extends uvm_scoreboard; stim = current_stim("RESP"); if (item.rob_id !== stim.rob_id) `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") - if (item.sub_rob_id !== 8'h00) - `uvm_fatal("RESP", $sformatf("sub_rob_id mismatch: got 0x%0h", item.sub_rob_id)) resp_count++; endfunction diff --git a/examples/components/balls/int8add/verify/src/seq/int8add_sequences.svh b/examples/components/balls/int8add/verify/src/seq/int8add_sequences.svh index 2582816d1..0536522b2 100644 --- a/examples/components/balls/int8add/verify/src/seq/int8add_sequences.svh +++ b/examples/components/balls/int8add/verify/src/seq/int8add_sequences.svh @@ -2,7 +2,6 @@ class int8add_basic_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(int8add_basic_seq) int unsigned case_index; - int unsigned bid; function new(string name = "int8add_basic_seq"); super.new(name); @@ -12,7 +11,7 @@ class int8add_basic_seq extends uvm_sequence #(bb_blink_cmd_item); int8add_cmd_item item; item = int8add_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(case_index, bid); + item.load_rust_case(case_index); finish_item(item); endtask endclass diff --git a/examples/components/balls/int8add/verify/src/tb_top.sv b/examples/components/balls/int8add/verify/src/tb_top.sv index cbd6b5183..995efdbb1 100644 --- a/examples/components/balls/int8add/verify/src/tb_top.sv +++ b/examples/components/balls/int8add/verify/src/tb_top.sv @@ -7,87 +7,75 @@ module tb_top; Int8AddBall dut ( .clock(intf.clock), .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_bankAccessReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) ); + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + initial begin intf.clock = 1'b0; intf.reset = 1'b1; diff --git a/examples/components/balls/int8add/verify/src/tests/int8add_test.svh b/examples/components/balls/int8add/verify/src/tests/int8add_test.svh index 7f109a88f..3e4c2c300 100644 --- a/examples/components/balls/int8add/verify/src/tests/int8add_test.svh +++ b/examples/components/balls/int8add/verify/src/tests/int8add_test.svh @@ -17,20 +17,18 @@ class int8add_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid = int8add_require_bid(); phase.raise_objection(this); - for (int unsigned index = 0; index < INT8ADD_NUM_CASES; index++) run_case(index, bid); + for (int unsigned index = 0; index < INT8ADD_NUM_CASES; index++) run_case(index); phase.drop_objection(this); endtask - task run_case(int unsigned index, int unsigned bid); + task run_case(int unsigned index); int8add_basic_seq seq; int cycles = 0; apply_reset(); env.scb.reset_counters(); seq = int8add_basic_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); while (!env.scb.done()) begin @(posedge vif.clock); diff --git a/examples/components/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir b/examples/components/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir index b2494a279..fb57444c5 100644 --- a/examples/components/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir +++ b/examples/components/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir @@ -22,16 +22,16 @@ func.func @main() -> i8 { memref.store %b8, %b[%row, %lane] : memref<4x16xi8> } } - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mvin %a <{bank = 0 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> - buckyball.mvin %b <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> - buckyball.custom "add_i8" args(%ratio, %ratio : f32, f32) <{sources = array, destination = 2 : i64}> {rows = 4 : i64, relu = true} - buckyball.mvout %out <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %a <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.mvin %b <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.custom "add_i8" args(%ratio, %ratio : f32, f32) <{sources = array, destination = 3 : i64}> {rows = 4 : i64, relu = true} + buckyball.mvout %out <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> func.call @check_result(%out) : (memref<4x16xi8>) -> () memref.dealloc %a : memref<4x16xi8> memref.dealloc %b : memref<4x16xi8> diff --git a/examples/components/balls/int8mul/arch/Int8MulBall.scala b/examples/components/balls/int8mul/arch/Int8MulBall.scala index 7c065936d..c612243ad 100644 --- a/examples/components/balls/int8mul/arch/Int8MulBall.scala +++ b/examples/components/balls/int8mul/arch/Int8MulBall.scala @@ -72,11 +72,11 @@ class Int8MulBall( val gateValue = (gateWord >> (lane << 3))(7, 0).asSInt val inputValue = (inputWord >> (lane << 3))(7, 0).asSInt val integerProduct = (gateValue * inputValue).asUInt - val productToFloat = Instantiate(new INToRecFN(16, 8, 24)) + val productToFloat = Module(new INToRecFN(16, 8, 24)) val scale = Instantiate(new Fma(3, 8, 24)) - val toInt8 = Instantiate(new RecFNToIN(8, 24, 8)) + val toInt8 = Module(new RecFNToIN(8, 24, 8)) productToFloat.io.signedIn := true.B productToFloat.io.in := productReg diff --git a/examples/components/balls/int8mul/stack/src/Lowering.cpp b/examples/components/balls/int8mul/stack/src/Lowering.cpp index 0957b9c32..f52d480db 100644 --- a/examples/components/balls/int8mul/stack/src/Lowering.cpp +++ b/examples/components/balls/int8mul/stack/src/Lowering.cpp @@ -1,11 +1,22 @@ -//===- Lowering.cpp - Int8MulBall in lower-buckyball -//-----------------------===// +//===- Lowering.cpp - Int8MulBall in lower-buckyball ----------------------===// +// +// gate_mul_i8: source 0 holds the gate, one row of sixteen i8; source 1 holds +// `rows` rows of sixteen i8. The destination receives +// clamp(round_half_even(gate * x * ratio), -128, 127) per column, in every +// group of the banks; the three banks have the same number of groups. +// Argument 0 is the f32 ratio, finite and positive. +// +//===----------------------------------------------------------------------===// + #include "Buckyball/BuckyballOps.h" #include "Conversion/Buckyball.h" #include "Target/BuckyballTargetRegistry.h" + #include "mlir/Dialect/Arith/IR/Arith.h" + using namespace mlir; using namespace ::buddy::buckyball; + namespace { class GateMulI8Lowering : public OpRewritePattern { public: @@ -15,20 +26,16 @@ class GateMulI8Lowering : public OpRewritePattern { if (op.getKind() != "gate_mul_i8") return failure(); int64_t rows = cast(op->getAttr("rows")).getInt(); - int64_t gateRow = cast(op->getAttr("gate_row")).getInt(); - int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; - if (rows <= 0 || rows > depth || gateRow < 0 || gateRow >= depth || + if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || op.getSources().size() != 2 || op.getSources()[0] == op.getSources()[1] || llvm::is_contained(op.getSources(), op.getDestination())) return op.emitOpError( - "gate_mul_i8 requires distinct banks and valid row addresses"); + "gate_mul_i8 requires distinct banks and rows within a bank"); + // rs2: ratio bits in [31:0]; the gate row in [63:32] is the first row. Value bits = arith::BitcastOp::create(b, op.getLoc(), b.getI32Type(), op.getArguments()[0]); - Value ratio = arith::ExtUIOp::create(b, op.getLoc(), b.getI64Type(), bits); - Value rs2 = arith::OrIOp::create( - b, op.getLoc(), ratio, - mlir::buddy::constantI64(b, op.getLoc(), uint64_t(gateRow) << 32)); + Value rs2 = arith::ExtUIOp::create(b, op.getLoc(), b.getI64Type(), bits); mlir::buddy::emitInstruction( b, op.getLoc(), buckyball_target::getBuckyballFunct7("INT8MUL"), mlir::buddy::constantI64(b, op.getLoc(), @@ -40,9 +47,11 @@ class GateMulI8Lowering : public OpRewritePattern { return success(); } }; + void populate(RewritePatternSet &patterns) { patterns.add(patterns.getContext()); } + [[maybe_unused]] const bool registered = (mlir::buddy::addBallLowering(populate), true); } // namespace diff --git a/examples/components/balls/int8mul/verify/casegen/src/casegen.rs b/examples/components/balls/int8mul/verify/casegen/src/casegen.rs index 118e07b5a..d041fb80d 100644 --- a/examples/components/balls/int8mul/verify/casegen/src/casegen.rs +++ b/examples/components/balls/int8mul/verify/casegen/src/casegen.rs @@ -1,25 +1,25 @@ pub const ROW_BYTES: usize = 16; pub const MAX_ROWS: usize = 64; -pub const NUM_CASES: u32 = 3; +pub const NUM_CASES: u32 = 4; pub const GATE_ROW: u32 = 1; pub const RATIO: f32 = 0.25; #[repr(C)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Int8MulCmd { - pub bid: u32, pub iter: u32, pub gate_bank: u32, pub input_bank: u32, pub output_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, pub op1_col: u32, pub op2_col: u32, pub wr_col: u32, pub gate_row: u32, pub bank_entries: u32, pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, pub rs2_lo: u32, pub rs2_hi: u32, pub num_gate_words: u32, @@ -61,7 +61,9 @@ impl Int8MulCase { } } -pub fn gen_case(index: u32, bid: u32, bank_entries: u32) -> Int8MulCase { +// Gate word `g` is row `gate_row` of group `g`; input/output word `i` is row `i % iter` of +// group `i / iter`: the Ball walks groups in order and reloads the gate per group. +pub fn gen_case(index: u32, bank_entries: u32) -> Int8MulCase { if bank_entries == 0 { panic!("int8mul: bank entries must be positive"); } @@ -69,9 +71,10 @@ pub fn gen_case(index: u32, bid: u32, bank_entries: u32) -> Int8MulCase { panic!("int8mul: unsupported directed case {index}"); } match index { - 0 => directed(bid, 2, 1, GATE_ROW, bank_entries), - 1 => directed(bid, 3, 4, GATE_ROW, bank_entries), - 2 => directed(bid, 4, 1, bank_entries - 1, bank_entries), + 0 => directed(2, 1, GATE_ROW, 1, [0, 0, 0], bank_entries), + 1 => directed(3, 4, GATE_ROW, 1, [0, 0, 0], bank_entries), + 2 => directed(4, 1, bank_entries - 1, 1, [0, 0, 0], bank_entries), + 3 => directed(5, 4, GATE_ROW, 2, [2, 1, 3], bank_entries), _ => panic!("int8mul: unsupported directed case {index}"), } } @@ -85,10 +88,10 @@ fn positive_finite(value: f32) -> bool { value.is_finite() && value > 0.0 } -fn ctest_gate() -> u128 { +fn ctest_gate(group: u32) -> u128 { let mut row = [0u8; ROW_BYTES]; for lane in 0..ROW_BYTES { - row[lane] = (lane as i8 - 8) as u8; + row[lane] = (lane as i8 - 8 + group as i8) as u8; } pack_bytes(&row)[0] } @@ -129,10 +132,6 @@ fn copy_words(src: &[u128]) -> [u128; N] { out } -fn encode_rs1(gate: u32, input: u32, output: u32, iter: u32) -> u64 { - u64::from(gate) | (u64::from(input) << 10) | (u64::from(output) << 20) | (u64::from(iter) << 30) -} - fn encode_rs2(ratio: f32, gate_row: u32) -> u64 { u64::from(ratio.to_bits()) | (u64::from(gate_row) << 32) } @@ -150,7 +149,6 @@ pub(crate) fn validate( wr_col: u32, iter: u32, gate_row: u32, - rs1: u64, rs2: u64, ratio: f32, bank_entries: u32, @@ -164,7 +162,7 @@ pub(crate) fn validate( if gate_row >= bank_entries { panic!("int8mul: gate row must fit one physical bank"); } - if op1_col != 1 || op2_col != 1 || wr_col != 1 { + if op1_col == 0 || op1_col != op2_col || op1_col != wr_col { panic!("int8mul: bank groups must match"); } if gate_bank == input_bank || gate_bank == output_bank || input_bank == output_bank { @@ -177,27 +175,19 @@ pub(crate) fn validate( if rs2 >> (32 + gate_row_bits) != 0 { panic!("int8mul: reserved rs2 bits must be zero"); } - if (rs1 & 0x3ff) != u64::from(gate_bank) - || ((rs1 >> 10) & 0x3ff) != u64::from(input_bank) - || ((rs1 >> 20) & 0x3ff) != u64::from(output_bank) - { - panic!("int8mul: rs1 bank fields mismatch"); - } - if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { - panic!("int8mul: rs1 iter mismatch"); - } if rs2 != encode_rs2(ratio, gate_row) { panic!("int8mul: rs2 ratio/gate_row fields mismatch"); } } -fn dst_words(gate_word: u128, input: &[u128], iter: u32, ratio: f32) -> Vec { - let mut out = Vec::with_capacity(iter as usize); - for row in 0..iter as usize { +fn dst_words(gate: &[u128], input: &[u128], iter: u32, ratio: f32) -> Vec { + let mut out = Vec::with_capacity(input.len()); + for (word, input_word) in input.iter().enumerate() { + let gate_word = gate[word / iter as usize]; let mut packed = 0u128; for lane in 0..ROW_BYTES { let g = ((gate_word >> (lane * 8)) & 0xff) as u8 as i8; - let inp = ((input[row] >> (lane * 8)) & 0xff) as u8 as i8; + let inp = ((input_word >> (lane * 8)) & 0xff) as u8 as i8; let q = compute(g, inp, ratio); packed |= u128::from(q as u8) << (lane * 8); } @@ -206,60 +196,58 @@ fn dst_words(gate_word: u128, input: &[u128], iter: u32, ratio: f32) -> Vec Int8MulCase { +fn directed( + rob_id: u32, + iter: u32, + gate_row: u32, + cols: u32, + groups: [u32; 3], + bank_entries: u32, +) -> Int8MulCase { let gate_bank = 0; let input_bank = 1; let output_bank = 2; - let op1_col = 1; - let op2_col = 1; - let wr_col = 1; let ratio = RATIO; - let gate = ctest_gate(); - let input = ctest_input(iter as usize); - let rs1 = encode_rs1(gate_bank, input_bank, output_bank, iter); + let gate: Vec = (0..cols).map(ctest_gate).collect(); + let input = ctest_input((iter * cols) as usize); let rs2 = encode_rs2(ratio, gate_row); validate( gate_bank, input_bank, output_bank, - op1_col, - op2_col, - wr_col, + cols, + cols, + cols, iter, gate_row, - rs1, rs2, ratio, bank_entries, ); - if input.len() != iter as usize { - panic!("int8mul: input rows {} != iter {iter}", input.len()); - } - let dst = dst_words(gate, &input, iter, ratio); - let (rs1_lo, rs1_hi) = split_rs(rs1); + let dst = dst_words(&gate, &input, iter, ratio); let (rs2_lo, rs2_hi) = split_rs(rs2); Int8MulCase { cmd: Int8MulCmd { - bid, iter, gate_bank, input_bank, output_bank, - op1_col, - op2_col, - wr_col, + op1_group: groups[0], + op2_group: groups[1], + wr_group: groups[2], + op1_col: cols, + op2_col: cols, + wr_col: cols, gate_row, bank_entries, rob_id, - rs1_lo, - rs1_hi, rs2_lo, rs2_hi, - num_gate_words: 1, + num_gate_words: gate.len() as u32, num_input_words: input.len() as u32, num_dst_words: dst.len() as u32, }, - gate_words: copy_words(&[gate]), + gate_words: copy_words(&gate), input_words: copy_words(&input), dst_words: copy_words(&dst), } diff --git a/examples/components/balls/int8mul/verify/casegen/src/lib.rs b/examples/components/balls/int8mul/verify/casegen/src/lib.rs index 41e8940f8..e995aab7b 100644 --- a/examples/components/balls/int8mul/verify/casegen/src/lib.rs +++ b/examples/components/balls/int8mul/verify/casegen/src/lib.rs @@ -10,8 +10,8 @@ thread_local! { } #[no_mangle] -pub extern "C" fn int8mul_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid, config::bank_entries()); +pub extern "C" fn int8mul_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index, config::bank_entries()); CURRENT.with(|current| *current.borrow_mut() = Some(case)); 0 } diff --git a/examples/components/balls/int8mul/verify/filelists/int8mul_ball.f b/examples/components/balls/int8mul/verify/filelists/int8mul_ball.f index db2d15f7d..89c6cfa4d 100644 --- a/examples/components/balls/int8mul/verify/filelists/int8mul_ball.f +++ b/examples/components/balls/int8mul/verify/filelists/int8mul_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,16 +14,17 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv @RTL@/RoundAnyRawFNToRecFN_ie5_is16_oe8_os24.sv -@RTL@/RoundAnyRawFNToRecFN_ie8_is26_oe8_os24.sv @RTL@/INToRecFN_i16_e8_s24.sv @RTL@/MulAddRecFNToRaw_preMul_e8_s24.sv @RTL@/MulAddRecFNToRaw_postMul_e8_s24.sv -@RTL@/MulAddRecFNPipe_l3_e8_s24.sv +@RTL@/RoundAnyRawFNToRecFN_ie8_is26_oe8_os24.sv @RTL@/RoundRawFNToRecFN_e8_s24.sv +@RTL@/Fma_l3_e8_s24.sv @RTL@/RecFNToIN_e8_s24_i8.sv @RTL@/Int8MulBall.sv diff --git a/examples/components/balls/int8mul/verify/src/common/int8mul_defs.svh b/examples/components/balls/int8mul/verify/src/common/int8mul_defs.svh index a9b509633..7999486e3 100644 --- a/examples/components/balls/int8mul/verify/src/common/int8mul_defs.svh +++ b/examples/components/balls/int8mul/verify/src/common/int8mul_defs.svh @@ -1,17 +1,17 @@ typedef struct { - int unsigned bid; int unsigned iter; int unsigned gate_bank; int unsigned input_bank; int unsigned output_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; int unsigned op1_col; int unsigned op2_col; int unsigned wr_col; int unsigned gate_row; int unsigned bank_entries; int unsigned rob_id; - int unsigned rs1_lo; - int unsigned rs1_hi; int unsigned rs2_lo; int unsigned rs2_hi; int unsigned num_gate_words; @@ -19,10 +19,7 @@ typedef struct { int unsigned num_dst_words; } int8mul_cmd_dpi_t; -import "DPI-C" function int int8mul_case_load( - input int unsigned index, - input int unsigned bid -); +import "DPI-C" function int int8mul_case_load(input int unsigned index); import "DPI-C" function void int8mul_case_cmd(output int8mul_cmd_dpi_t cmd); import "DPI-C" function longint unsigned int8mul_case_gate_word_lo(input int unsigned word_index); import "DPI-C" function longint unsigned int8mul_case_gate_word_hi(input int unsigned word_index); @@ -37,13 +34,5 @@ import "DPI-C" function longint unsigned int8mul_case_dst_word_hi(input int unsi localparam int INT8MUL_THREAD_FUNCT7 = `INT8MUL_FUNCT7; localparam int INT8MUL_MAX_ROWS = 64; -localparam int INT8MUL_NUM_CASES = 3; +localparam int INT8MUL_NUM_CASES = 4; localparam int INT8MUL_TIMEOUT_CYCLES = 20000; - -function automatic int unsigned int8mul_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/components/balls/int8mul/verify/src/common/int8mul_items.svh b/examples/components/balls/int8mul/verify/src/common/int8mul_items.svh index 017406d68..06a727bc4 100644 --- a/examples/components/balls/int8mul/verify/src/common/int8mul_items.svh +++ b/examples/components/balls/int8mul/verify/src/common/int8mul_items.svh @@ -10,69 +10,48 @@ class int8mul_cmd_item extends bb_blink_cmd_item; bit [127:0] input_words[INT8MUL_MAX_ROWS]; bit [127:0] dst_words[INT8MUL_MAX_ROWS]; - constraint legal_c { - funct7 == INT8MUL_THREAD_FUNCT7[6:0]; - op1_en == 1'b1; - op2_en == 1'b1; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b1; - op1_col == 5'd1; - op2_col == 5'd1; - wr_col == 5'd1; - meta_bank == 5'd0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - special == 64'd0; - } - function new(string name = "int8mul_cmd_item"); super.new(name); endfunction - function void load_rust_case(int unsigned index, int unsigned bid); + function void load_rust_case(int unsigned index); int8mul_cmd_dpi_t cmd; int unsigned rc; longint unsigned w_lo; longint unsigned w_hi; int i; - rc = int8mul_case_load(index, bid); + rc = int8mul_case_load(index); if (rc != 0) begin `uvm_fatal("CASE", $sformatf("int8mul_case_load returned %0d for index %0d", rc, index)) end int8mul_case_cmd(cmd); - this.bid = cmd.bid[4:0]; + // bb_int8mul: banks and iter come from rs1, ratio and gate_row from rs2. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = INT8MUL_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; iter = cmd.iter; - op1_bank = cmd.gate_bank[4:0]; - op2_bank = cmd.input_bank[4:0]; - wr_bank = cmd.output_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - op2_col = cmd.op2_col[4:0]; - wr_col = cmd.wr_col[4:0]; + op1_bank = cmd.gate_bank; + op2_bank = cmd.input_bank; + wr_bank = cmd.output_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.output_bank; gate_row = cmd.gate_row; bank_entries = cmd.bank_entries; - rob_id = cmd.rob_id[3:0]; - rs1 = {cmd.rs1_hi, cmd.rs1_lo}; - rs2 = {cmd.rs2_hi, cmd.rs2_lo}; num_gate_words = cmd.num_gate_words; num_input_words = cmd.num_input_words; num_dst_words = cmd.num_dst_words; - funct7 = INT8MUL_THREAD_FUNCT7[6:0]; - special = 64'd0; - op1_en = 1'b1; - op2_en = 1'b1; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b1; - meta_bank = 5'd0; - is_sub = 1'b0; - sub_rob_id = 8'h00; if (iter != 1 && iter != 4) `uvm_fatal("CASE", $sformatf("Int8MulBall iter %0d is not in {1,4}", iter)) - if (op1_col != 1 || op2_col != 1 || wr_col != 1) + if (op1_col == 0 || op1_col != op2_col || op1_col != wr_col) `uvm_fatal("CASE", "Int8MulBall bank groups must match") if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) `uvm_fatal("CASE", "Int8MulBall banks must be distinct") @@ -82,12 +61,13 @@ class int8mul_cmd_item extends bb_blink_cmd_item; "Int8MulBall gate_row %0d exceeds bank_entries %0d", gate_row, bank_entries)) if ((rs2 >> (32 + $clog2(bank_entries))) != 0) `uvm_fatal("CASE", $sformatf("Int8MulBall reserved rs2 bits set: 0x%016h", rs2)) - if (num_input_words != iter) - `uvm_fatal("CASE", $sformatf("num_input_words %0d != iter %0d", num_input_words, iter)) - if (num_dst_words != iter) - `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter %0d", num_dst_words, iter)) - if (funct7 != INT8MUL_THREAD_FUNCT7[6:0]) - `uvm_fatal("CASE", $sformatf("invalid funct7: %0d", funct7)) + if (num_gate_words != op1_col) + `uvm_fatal("CASE", $sformatf("num_gate_words %0d != col %0d", num_gate_words, op1_col)) + if (num_input_words != iter * op1_col || num_dst_words != iter * op1_col) + `uvm_fatal( + "CASE", $sformatf( + "word counts %0d/%0d != iter %0d * col %0d", num_input_words, num_dst_words, iter, op1_col + )) for (i = 0; i < num_gate_words; i++) begin w_lo = int8mul_case_gate_word_lo(i); diff --git a/examples/components/balls/int8mul/verify/src/cov/int8mul_cov.svh b/examples/components/balls/int8mul/verify/src/cov/int8mul_cov.svh index 027215c44..0b2dd0566 100644 --- a/examples/components/balls/int8mul/verify/src/cov/int8mul_cov.svh +++ b/examples/components/balls/int8mul/verify/src/cov/int8mul_cov.svh @@ -4,9 +4,11 @@ class int8mul_cov extends uvm_component; uvm_analysis_imp_cmd #(bb_blink_cmd_item, int8mul_cov) cmd_imp; int unsigned cur_iter; + int unsigned cur_col; covergroup cmd_cg; coverpoint cur_iter {bins i1 = {1}; bins i4 = {4};} + coverpoint cur_col {bins c1 = {1}; bins c2 = {2};} endgroup int cmd_count; @@ -18,9 +20,10 @@ class int8mul_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != INT8MUL_THREAD_FUNCT7[6:0]) - `uvm_fatal("COV", $sformatf("unexpected funct7 %0d", item.funct7)) + if (item.funct != INT8MUL_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) cur_iter = item.iter; + cur_col = item.op1_col; cmd_count++; cmd_cg.sample(); endfunction diff --git a/examples/components/balls/int8mul/verify/src/env/int8mul_scoreboard.svh b/examples/components/balls/int8mul/verify/src/env/int8mul_scoreboard.svh index 7e043059f..bfb0b5406 100644 --- a/examples/components/balls/int8mul/verify/src/env/int8mul_scoreboard.svh +++ b/examples/components/balls/int8mul/verify/src/env/int8mul_scoreboard.svh @@ -46,17 +46,21 @@ class int8mul_scoreboard extends uvm_scoreboard; `uvm_fatal("SCB", "mem_model handle not set") end mem_model.clear_mem(); + // Gate word g lives at row gate_row of group base + g; input word i at row i % iter of + // group base + i / iter. for (i = 0; i < clone.num_gate_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), int'(clone.gate_row) + i, clone.gate_words[i]); + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group) + i, int'(clone.gate_row), + clone.gate_words[i]); end for (i = 0; i < clone.num_input_words; i++) begin - mem_model.load_word(int'(clone.op2_bank), i, clone.input_words[i]); + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group) + i / clone.iter, + i % clone.iter, clone.input_words[i]); end mem_model.arm(); expected_writes = clone.num_dst_words; - expected_reads[0] = 1; - expected_reads[1] = clone.iter; + expected_reads[0] = clone.num_gate_words; + expected_reads[1] = clone.num_input_words; endfunction function void write_cmd(bb_blink_cmd_item item); @@ -67,47 +71,41 @@ class int8mul_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, int8mul_cmd_item exp); + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); verification_pkg::evidence::checks++; - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.op2_bank !== exp.op2_bank || - got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.op2_col !== exp.op2_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rs1 !== exp.rs1 || got.rs2 !== exp.rs2) `uvm_fatal("CMD", "rs field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) - if ((got.rs2 >> (32 + $clog2(exp.bank_entries))) != 0) - `uvm_fatal("CMD", $sformatf("reserved rs2 bits set: 0x%016h", got.rs2)) + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) endfunction function void write_read(bb_blink_read_item item); int8mul_cmd_item stim; int unsigned expect_bank; + int unsigned expect_group; int unsigned expect_addr; stim = current_stim("READ"); if (item.port != 0 && item.port != 1) `uvm_fatal("READ", $sformatf("Int8MulBall has two read ports, got port %0d", item.port)) - if (item.group_id !== 5'd0) - `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp 0", item.group_id)) if (item.rob_id !== stim.rob_id) `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + // The gate row is read once per group, before that group's input rows. if (item.port == 0) begin - expect_bank = stim.op1_bank; - expect_addr = stim.gate_row; - if (read_count[0] != 0) `uvm_fatal("READ", "Int8MulBall gate bank read more than once") + expect_bank = stim.op1_bank; + expect_group = stim.op1_group + read_count[0]; + expect_addr = stim.gate_row; + if (read_count[1] != read_count[0] * stim.iter) + `uvm_fatal("READ", "Int8MulBall gate read out of group order") end else begin - expect_bank = stim.op2_bank; - expect_addr = read_count[1]; + expect_bank = stim.op2_bank; + expect_group = stim.op2_group + read_count[1] / stim.iter; + expect_addr = read_count[1] % stim.iter; end - if (item.bank_id !== expect_bank[4:0]) + if (item.group_id !== expect_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, expect_group)) + if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) if (item.addr !== expect_addr[BB_BLINK_BANK_ADDR_W-1:0]) `uvm_fatal("READ", $sformatf("addr mismatch: got %0d exp %0d", item.addr, expect_addr)) @@ -124,14 +122,14 @@ class int8mul_scoreboard extends uvm_scoreboard; `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== 5'd0) - `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp 0", item.group_id)) + if (item.group_id !== stim.wr_group + write_count / stim.iter) + `uvm_fatal("WRITE", $sformatf("group mismatch at %0d: got %0d", write_count, item.group_id)) if (item.mask !== 16'hFFFF) `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) if (write_count >= stim.num_dst_words) `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) - if (item.addr !== write_count[BB_BLINK_BANK_ADDR_W-1:0]) - `uvm_fatal("WRITE", $sformatf("addr mismatch: got %0d exp %0d", item.addr, write_count)) + if (item.addr !== write_count % stim.iter) + `uvm_fatal("WRITE", $sformatf("addr mismatch at %0d: got %0d", write_count, item.addr)) if (item.data !== stim.dst_words[write_count]) `uvm_fatal("SCB", $sformatf( "data mismatch at addr %0d: got 0x%032h exp 0x%032h", @@ -148,9 +146,6 @@ class int8mul_scoreboard extends uvm_scoreboard; stim = current_stim("RESP"); if (item.rob_id !== stim.rob_id) `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") - if (item.sub_rob_id !== 8'h00) - `uvm_fatal("RESP", $sformatf("sub_rob_id mismatch: got 0x%0h", item.sub_rob_id)) resp_count++; endfunction diff --git a/examples/components/balls/int8mul/verify/src/seq/int8mul_sequences.svh b/examples/components/balls/int8mul/verify/src/seq/int8mul_sequences.svh index a9f4dd514..41abbd544 100644 --- a/examples/components/balls/int8mul/verify/src/seq/int8mul_sequences.svh +++ b/examples/components/balls/int8mul/verify/src/seq/int8mul_sequences.svh @@ -2,7 +2,6 @@ class int8mul_basic_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(int8mul_basic_seq) int unsigned case_index; - int unsigned bid; function new(string name = "int8mul_basic_seq"); super.new(name); @@ -12,7 +11,7 @@ class int8mul_basic_seq extends uvm_sequence #(bb_blink_cmd_item); int8mul_cmd_item item; item = int8mul_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(case_index, bid); + item.load_rust_case(case_index); finish_item(item); endtask endclass diff --git a/examples/components/balls/int8mul/verify/src/tb_top.sv b/examples/components/balls/int8mul/verify/src/tb_top.sv index cadcd55fe..322b44442 100644 --- a/examples/components/balls/int8mul/verify/src/tb_top.sv +++ b/examples/components/balls/int8mul/verify/src/tb_top.sv @@ -7,87 +7,75 @@ module tb_top; Int8MulBall dut ( .clock(intf.clock), .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_bankAccessReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) ); + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + initial begin intf.clock = 1'b0; intf.reset = 1'b1; diff --git a/examples/components/balls/int8mul/verify/src/tests/int8mul_test.svh b/examples/components/balls/int8mul/verify/src/tests/int8mul_test.svh index c1d6714ca..b8c62d485 100644 --- a/examples/components/balls/int8mul/verify/src/tests/int8mul_test.svh +++ b/examples/components/balls/int8mul/verify/src/tests/int8mul_test.svh @@ -17,20 +17,18 @@ class int8mul_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid = int8mul_require_bid(); phase.raise_objection(this); - for (int unsigned index = 0; index < INT8MUL_NUM_CASES; index++) run_case(index, bid); + for (int unsigned index = 0; index < INT8MUL_NUM_CASES; index++) run_case(index); phase.drop_objection(this); endtask - task run_case(int unsigned index, int unsigned bid); + task run_case(int unsigned index); int8mul_basic_seq seq; int cycles = 0; apply_reset(); env.scb.reset_counters(); seq = int8mul_basic_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); while (!env.scb.done()) begin @(posedge vif.clock); diff --git a/examples/components/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir b/examples/components/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir index 82b8fd88f..ddbd6f1f0 100644 --- a/examples/components/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir +++ b/examples/components/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir @@ -19,17 +19,17 @@ func.func @main() -> i8 { memref.store %value, %input[%row, %lane] : memref<4x16xi8> } } - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mvin %gate <{bank = 0 : i64, depth = 1 : i64, stride = 1 : i64}> : memref<1x16xi8> - buckyball.mvin %input <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> - buckyball.custom "gate_mul_i8" args(%ratio : f32) <{sources = array, destination = 2 : i64}> {rows = 4 : i64, gate_row = 0 : i64} - buckyball.mvout %output <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %gate <{bank = 1 : i64, depth = 1 : i64, stride = 1 : i64}> : memref<1x16xi8> + buckyball.mvin %input <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.custom "gate_mul_i8" args(%ratio : f32) <{sources = array, destination = 3 : i64}> {rows = 4 : i64} + buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> func.call @check_result(%output) : (memref<4x16xi8>) -> () - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> memref.dealloc %gate : memref<1x16xi8> memref.dealloc %input : memref<4x16xi8> memref.dealloc %output : memref<4x16xi8> diff --git a/examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in b/examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in index 2cb2f331d..3dd8ace9e 100644 --- a/examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in +++ b/examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in @@ -24,7 +24,7 @@ func.func @main() -> i8 { %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded_gate = buckyball.bank_mvin %gate, %gate_bank <{depth = 1 : i64, stride = 1 : i64}> : memref<1x@BB_BANK_I8_LANES@xi8> %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %result = buckyball.bank_custom "gate_mul_i8" (%loaded_gate, %loaded_input) -> %output_bank args(%ratio : f32) {rows = @BB_BANK_LINES@ : i64, gate_row = 0 : i64} + %result = buckyball.bank_custom "gate_mul_i8" (%loaded_gate, %loaded_input) -> %output_bank args(%ratio : f32) {rows = @BB_BANK_LINES@ : i64} %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () buckyball.bank_release %loaded_gate diff --git a/examples/components/balls/lut/stack/src/Lowering.cpp b/examples/components/balls/lut/stack/src/Lowering.cpp index 65ff0b717..43b4be4c6 100644 --- a/examples/components/balls/lut/stack/src/Lowering.cpp +++ b/examples/components/balls/lut/stack/src/Lowering.cpp @@ -1,4 +1,11 @@ //===- Lowering.cpp - LutBall in lower-buckyball --------------------------===// +// +// lookup_i8: source 0 holds `rows` rows of sixteen i8 codes, source 1 the +// table: one group of 256 entries shared by every channel, or four groups +// with 256 entries per channel (channel * 256 + code, 1024 bytes per group). +// The destination receives table[code] for every element. +// +//===----------------------------------------------------------------------===// #include "Buckyball/BuckyballOps.h" #include "Conversion/Buckyball.h" #include "Target/BuckyballTargetRegistry.h" diff --git a/examples/components/balls/lut/verify/casegen/src/casegen.rs b/examples/components/balls/lut/verify/casegen/src/casegen.rs index 631f250b1..c50d50e64 100644 --- a/examples/components/balls/lut/verify/casegen/src/casegen.rs +++ b/examples/components/balls/lut/verify/casegen/src/casegen.rs @@ -9,19 +9,17 @@ pub const NUM_CASES: u32 = 4; #[repr(C)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct LutCmd { - pub bid: u32, pub iter: u32, pub op1_bank: u32, pub op2_bank: u32, pub wr_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, pub op1_col: u32, pub op2_col: u32, pub wr_col: u32, pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, - pub rs2_lo: u32, - pub rs2_hi: u32, pub num_src_words: u32, pub num_lut_words: u32, pub num_dst_words: u32, @@ -61,15 +59,15 @@ impl LutCase { } } -pub fn gen_case(index: u32, bid: u32) -> LutCase { +pub fn gen_case(index: u32) -> LutCase { if index >= NUM_CASES { panic!("lut: unsupported directed case {index}"); } match index { - 0 => directed(bid, 2, 4, false), - 1 => directed(bid, 3, 1, false), - 2 => directed(bid, 4, 8, false), - 3 => directed(bid, 5, 4, true), + 0 => directed(2, 4, false, [0, 0, 0]), + 1 => directed(3, 1, false, [1, 2, 3]), + 2 => directed(4, 8, false, [2, 0, 1]), + 3 => directed(5, 4, true, [3, 1, 2]), _ => panic!("lut: unsupported directed case {index}"), } } @@ -142,14 +140,6 @@ fn copy_words(src: &[u128]) -> [u128; N] { out } -fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { - u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) -} - -fn split_rs(value: u64) -> (u32, u32) { - (value as u32, (value >> 32) as u32) -} - pub(crate) fn validate( op1_bank: u32, op2_bank: u32, @@ -158,8 +148,6 @@ pub(crate) fn validate( op2_col: u32, wr_col: u32, iter: u32, - rs1: u64, - rs2: u64, ) { if iter == 0 { panic!("lut: iter must be positive"); @@ -170,18 +158,6 @@ pub(crate) fn validate( if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { panic!("lut: banks must be distinct"); } - if (rs1 & 0x3ff) != u64::from(op1_bank) - || ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) - || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) - { - panic!("lut: rs1 bank fields mismatch"); - } - if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { - panic!("lut: rs1 iter mismatch"); - } - if rs2 != 0 { - panic!("lut: rs2 must be zero"); - } } fn dst_words(table: &[u8], input: &[u8], lane_table: bool) -> Vec { @@ -196,7 +172,8 @@ fn dst_words(table: &[u8], input: &[u8], lane_table: bool) -> Vec { pack_bytes(&mapped) } -fn directed(bid: u32, rob_id: u32, iter: u32, per_lane: bool) -> LutCase { +// `groups` are the op1/op2/wr base groups; a lane table spans four groups from its base. +fn directed(rob_id: u32, iter: u32, per_lane: bool, groups: [u32; 3]) -> LutCase { let nbytes = iter as usize * BYTES_PER_ROW; let table = if per_lane { lane_table() @@ -217,11 +194,7 @@ fn directed(bid: u32, rob_id: u32, iter: u32, per_lane: bool) -> LutCase { let op1_col = 1; let op2_col = if per_lane { 4 } else { 1 }; let wr_col = 1; - let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); - let rs2 = 0u64; - validate( - op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs1, rs2, - ); + validate(op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter); if src.len() != iter as usize { panic!("lut: src rows {} != iter {iter}", src.len()); } @@ -232,23 +205,19 @@ fn directed(bid: u32, rob_id: u32, iter: u32, per_lane: bool) -> LutCase { if dst.len() != iter as usize { panic!("lut: dst rows {} != iter {iter}", dst.len()); } - let (rs1_lo, rs1_hi) = split_rs(rs1); - let (rs2_lo, rs2_hi) = split_rs(rs2); LutCase { cmd: LutCmd { - bid, iter, op1_bank, op2_bank, wr_bank, + op1_group: groups[0], + op2_group: groups[1], + wr_group: groups[2], op1_col, op2_col, wr_col, rob_id, - rs1_lo, - rs1_hi, - rs2_lo, - rs2_hi, num_src_words: src.len() as u32, num_lut_words: lut.len() as u32, num_dst_words: dst.len() as u32, diff --git a/examples/components/balls/lut/verify/casegen/src/lib.rs b/examples/components/balls/lut/verify/casegen/src/lib.rs index b1143b748..bafa9652a 100644 --- a/examples/components/balls/lut/verify/casegen/src/lib.rs +++ b/examples/components/balls/lut/verify/casegen/src/lib.rs @@ -9,8 +9,8 @@ thread_local! { } #[no_mangle] -pub extern "C" fn lut_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid); +pub extern "C" fn lut_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); CURRENT.with(|current| *current.borrow_mut() = Some(case)); 0 } diff --git a/examples/components/balls/lut/verify/filelists/lut_ball.f b/examples/components/balls/lut/verify/filelists/lut_ball.f index f98bca01e..c5edcd905 100644 --- a/examples/components/balls/lut/verify/filelists/lut_ball.f +++ b/examples/components/balls/lut/verify/filelists/lut_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,6 +14,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv diff --git a/examples/components/balls/lut/verify/src/common/lut_defs.svh b/examples/components/balls/lut/verify/src/common/lut_defs.svh index 90be94d89..09021a2e2 100644 --- a/examples/components/balls/lut/verify/src/common/lut_defs.svh +++ b/examples/components/balls/lut/verify/src/common/lut_defs.svh @@ -1,26 +1,21 @@ typedef struct { - int unsigned bid; int unsigned iter; int unsigned op1_bank; int unsigned op2_bank; int unsigned wr_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; int unsigned op1_col; int unsigned op2_col; int unsigned wr_col; int unsigned rob_id; - int unsigned rs1_lo; - int unsigned rs1_hi; - int unsigned rs2_lo; - int unsigned rs2_hi; int unsigned num_src_words; int unsigned num_lut_words; int unsigned num_dst_words; } lut_cmd_dpi_t; -import "DPI-C" function int lut_case_load( - input int unsigned index, - input int unsigned bid -); +import "DPI-C" function int lut_case_load(input int unsigned index); import "DPI-C" function void lut_case_cmd(output lut_cmd_dpi_t cmd); import "DPI-C" function longint unsigned lut_case_src_word_lo(input int unsigned word_index); import "DPI-C" function longint unsigned lut_case_src_word_hi(input int unsigned word_index); @@ -40,11 +35,3 @@ localparam int LUT_LANE_TABLE_ROWS = 256; localparam int LUT_MAX_DST_WORDS = 8; localparam int LUT_NUM_CASES = 4; localparam int LUT_TIMEOUT_CYCLES = 20000; - -function automatic int unsigned lut_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/components/balls/lut/verify/src/common/lut_items.svh b/examples/components/balls/lut/verify/src/common/lut_items.svh index 39767cd5b..be05c7c3a 100644 --- a/examples/components/balls/lut/verify/src/common/lut_items.svh +++ b/examples/components/balls/lut/verify/src/common/lut_items.svh @@ -8,64 +8,42 @@ class lut_cmd_item extends bb_blink_cmd_item; bit [127:0] lut_words[LUT_LANE_TABLE_ROWS]; bit [127:0] dst_words[LUT_MAX_DST_WORDS]; - constraint legal_c { - funct7 == LUT_THREAD_FUNCT7[6:0]; - op1_en == 1'b1; - op2_en == 1'b1; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b1; - op1_col == 5'd1; - op2_col inside {5'd1, 5'd4}; - wr_col == 5'd1; - meta_bank == 5'd0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - special == 64'd0; - rs2 == 64'd0; - } - function new(string name = "lut_cmd_item"); super.new(name); endfunction - function void load_rust_case(int unsigned index, int unsigned bid); + function void load_rust_case(int unsigned index); lut_cmd_dpi_t cmd; int unsigned rc; longint unsigned w_lo; longint unsigned w_hi; int i; - rc = lut_case_load(index, bid); + rc = lut_case_load(index); if (rc != 0) begin `uvm_fatal("CASE", $sformatf("lut_case_load returned %0d for index %0d", rc, index)) end lut_case_cmd(cmd); - this.bid = cmd.bid[4:0]; + // bb_lut: input/table/output banks and iter come from rs1; rs2 is reserved zero. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = LUT_THREAD_FUNCT7[6:0]; + rs2 = 64'd0; iter = cmd.iter; - op1_bank = cmd.op1_bank[4:0]; - op2_bank = cmd.op2_bank[4:0]; - wr_bank = cmd.wr_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - op2_col = cmd.op2_col[4:0]; - wr_col = cmd.wr_col[4:0]; - rob_id = cmd.rob_id[3:0]; - rs1 = {cmd.rs1_hi, cmd.rs1_lo}; - rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; num_src_words = cmd.num_src_words; num_lut_words = cmd.num_lut_words; num_dst_words = cmd.num_dst_words; - funct7 = LUT_THREAD_FUNCT7[6:0]; - special = 64'd0; - op1_en = 1'b1; - op2_en = 1'b1; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b1; - meta_bank = 5'd0; - is_sub = 1'b0; - sub_rob_id = 8'h00; if (iter != 1 && iter != 4 && iter != 8) `uvm_fatal("CASE", $sformatf("LUT iter %0d is not in {1,4,8}", iter)) @@ -73,7 +51,6 @@ class lut_cmd_item extends bb_blink_cmd_item; `uvm_fatal("CASE", "LUT requires col=1 input/output and col=1 or col=4 table") if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) `uvm_fatal("CASE", "LUT banks must be distinct") - if (rs2 != 64'd0) `uvm_fatal("CASE", "LUT rs2 must be zero") if (num_src_words != iter) `uvm_fatal("CASE", $sformatf("num_src_words %0d != iter %0d", num_src_words, iter)) if (num_dst_words != iter) diff --git a/examples/components/balls/lut/verify/src/cov/lut_cov.svh b/examples/components/balls/lut/verify/src/cov/lut_cov.svh index 71c3ad503..38d7c4862 100644 --- a/examples/components/balls/lut/verify/src/cov/lut_cov.svh +++ b/examples/components/balls/lut/verify/src/cov/lut_cov.svh @@ -20,8 +20,8 @@ class lut_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != LUT_THREAD_FUNCT7[6:0]) - `uvm_fatal("COV", $sformatf("unexpected funct7 %0d", item.funct7)) + if (item.funct != LUT_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) cur_iter = item.iter; cur_lut_cols = item.op2_col; cmd_count++; diff --git a/examples/components/balls/lut/verify/src/env/lut_scoreboard.svh b/examples/components/balls/lut/verify/src/env/lut_scoreboard.svh index 1a5704a31..8809fa0e9 100644 --- a/examples/components/balls/lut/verify/src/env/lut_scoreboard.svh +++ b/examples/components/balls/lut/verify/src/env/lut_scoreboard.svh @@ -47,10 +47,12 @@ class lut_scoreboard extends uvm_scoreboard; end mem_model.clear_mem(); for (i = 0; i < clone.num_src_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), i, clone.src_words[i]); + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group), i, clone.src_words[i]); end + // A lane table spans four groups of 64 rows from the table base group. for (i = 0; i < clone.num_lut_words; i++) begin - mem_model.load_word_g(int'(clone.op2_bank), i / 64, i % 64, clone.lut_words[i]); + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group) + i / 64, i % 64, + clone.lut_words[i]); end mem_model.arm(); @@ -67,22 +69,12 @@ class lut_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, lut_cmd_item exp); + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); verification_pkg::evidence::checks++; - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.op2_bank !== exp.op2_bank || - got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.op2_col !== exp.op2_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rs1 !== exp.rs1 || got.rs2 !== exp.rs2) `uvm_fatal("CMD", "rs field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) - if (got.rs2 !== 64'd0) `uvm_fatal("CMD", "LUT rs2 must be zero") + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) endfunction function void write_read(bb_blink_read_item item); @@ -103,18 +95,18 @@ class lut_scoreboard extends uvm_scoreboard; if (item.port == 0) begin expect_bank = stim.op1_bank; expect_addr = read_count[0]; - expect_group = 0; + expect_group = stim.op1_group; end else begin expect_bank = stim.op2_bank; row = read_count[1] / 16; lane = read_count[1] % 16; input_byte = stim.src_words[row][lane*8+:8]; - expect_group = stim.op2_col == 4 ? lane / 4 : 0; + expect_group = stim.op2_group + (stim.op2_col == 4 ? lane / 4 : 0); expect_addr = (stim.op2_col == 4 ? (lane % 4) * 16 : 0) + input_byte[7:4]; end - if (item.bank_id !== expect_bank[4:0]) + if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) - if (item.group_id !== expect_group[4:0]) + if (item.group_id !== expect_group[BB_BLINK_GROUP_ID_W-1:0]) `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, expect_group)) if (item.addr !== expect_addr[BB_BLINK_BANK_ADDR_W-1:0]) `uvm_fatal("READ", $sformatf("addr mismatch: got %0d exp %0d", item.addr, expect_addr)) @@ -131,8 +123,9 @@ class lut_scoreboard extends uvm_scoreboard; `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== 5'd0) - `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp 0", item.group_id)) + if (item.group_id !== stim.wr_group) + `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + )) if (item.mask !== 16'hFFFF) `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) if (write_count >= stim.num_dst_words) @@ -155,9 +148,6 @@ class lut_scoreboard extends uvm_scoreboard; stim = current_stim("RESP"); if (item.rob_id !== stim.rob_id) `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") - if (item.sub_rob_id !== 8'h00) - `uvm_fatal("RESP", $sformatf("sub_rob_id mismatch: got 0x%0h", item.sub_rob_id)) resp_count++; endfunction diff --git a/examples/components/balls/lut/verify/src/seq/lut_sequences.svh b/examples/components/balls/lut/verify/src/seq/lut_sequences.svh index 7a8ffa6d4..8eb3d9978 100644 --- a/examples/components/balls/lut/verify/src/seq/lut_sequences.svh +++ b/examples/components/balls/lut/verify/src/seq/lut_sequences.svh @@ -2,7 +2,6 @@ class lut_basic_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(lut_basic_seq) int unsigned case_index; - int unsigned bid; function new(string name = "lut_basic_seq"); super.new(name); @@ -12,7 +11,7 @@ class lut_basic_seq extends uvm_sequence #(bb_blink_cmd_item); lut_cmd_item item; item = lut_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(case_index, bid); + item.load_rust_case(case_index); finish_item(item); endtask endclass diff --git a/examples/components/balls/lut/verify/src/tb_top.sv b/examples/components/balls/lut/verify/src/tb_top.sv index 468b257d5..6e0f6d8ff 100644 --- a/examples/components/balls/lut/verify/src/tb_top.sv +++ b/examples/components/balls/lut/verify/src/tb_top.sv @@ -7,87 +7,75 @@ module tb_top; LutBall dut ( .clock(intf.clock), .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_bankAccessReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) ); + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + initial begin intf.clock = 1'b0; intf.reset = 1'b1; diff --git a/examples/components/balls/lut/verify/src/tests/lut_test.svh b/examples/components/balls/lut/verify/src/tests/lut_test.svh index 18af79b1d..5d56a32e7 100644 --- a/examples/components/balls/lut/verify/src/tests/lut_test.svh +++ b/examples/components/balls/lut/verify/src/tests/lut_test.svh @@ -17,20 +17,18 @@ class lut_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid = lut_require_bid(); phase.raise_objection(this); - for (int unsigned index = 0; index < LUT_NUM_CASES; index++) run_case(index, bid); + for (int unsigned index = 0; index < LUT_NUM_CASES; index++) run_case(index); phase.drop_objection(this); endtask - task run_case(int unsigned index, int unsigned bid); + task run_case(int unsigned index); lut_basic_seq seq; int cycles = 0; apply_reset(); env.scb.reset_counters(); seq = lut_basic_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); while (!env.scb.done()) begin @(posedge vif.clock); diff --git a/examples/components/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir b/examples/components/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir index 2887beb3a..a374cd87a 100644 --- a/examples/components/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir +++ b/examples/components/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir @@ -30,16 +30,16 @@ func.func @main() -> i8 { memref.store %value, %input[%row, %lane] : memref<4x16xi8> } } - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mvin %input <{bank = 0 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> - buckyball.mvin %table <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - buckyball.custom "lookup_i8" <{sources = array, destination = 2 : i64}> {rows = 4 : i64} - buckyball.mvout %output <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.mvin %table <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.custom "lookup_i8" <{sources = array, destination = 3 : i64}> {rows = 4 : i64} + buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> func.call @check_result(%output) : (memref<4x16xi8>) -> () memref.dealloc %input : memref<4x16xi8> memref.dealloc %table : memref<16x16xi8> diff --git a/examples/components/balls/matadd/verify/filelists/matadd_ball.f b/examples/components/balls/matadd/verify/filelists/matadd_ball.f index f4f281933..87c3ff133 100644 --- a/examples/components/balls/matadd/verify/filelists/matadd_ball.f +++ b/examples/components/balls/matadd/verify/filelists/matadd_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -12,6 +13,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv diff --git a/examples/components/balls/matadd/verify/src/common/matadd_defs.svh b/examples/components/balls/matadd/verify/src/common/matadd_defs.svh index 2510d839c..5580a1ef8 100644 --- a/examples/components/balls/matadd/verify/src/common/matadd_defs.svh +++ b/examples/components/balls/matadd/verify/src/common/matadd_defs.svh @@ -5,11 +5,3 @@ localparam int MATADD_THREAD_FUNCT7 = `MATADD_FUNCT7; localparam int MATADD_MAX_LINES = 64; localparam int MATADD_TIMEOUT_CYCLES = 100000; - -function automatic int unsigned matadd_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/components/balls/matadd/verify/src/common/matadd_items.svh b/examples/components/balls/matadd/verify/src/common/matadd_items.svh index f6c589262..f63284e2b 100644 --- a/examples/components/balls/matadd/verify/src/common/matadd_items.svh +++ b/examples/components/balls/matadd/verify/src/common/matadd_items.svh @@ -5,51 +5,32 @@ class matadd_cmd_item extends bb_blink_cmd_item; int unsigned group_count; int unsigned case_index; - constraint legal_c { - funct7 == MATADD_THREAD_FUNCT7[6:0]; - op1_en == 1'b1; - op2_en == 1'b1; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b1; - meta_bank == 5'd0; - rs1 == 64'd0; - rs2 == 64'd0; - special == 64'd0; - is_sub == 1'b0; - sub_rob_id == 8'd0; - } - function new(string name = "matadd_cmd_item"); super.new(name); endfunction - function void load_case(int unsigned index, int unsigned ball_id); + // Case 0 covers one group at base 0; cases 1 and 2 cover two groups at nonzero bases. + function void load_case(int unsigned index); if (index > 2) `uvm_fatal("CASE", $sformatf("unsupported case %0d", index)) case_index = index; - this.bid = ball_id[4:0]; - funct7 = MATADD_THREAD_FUNCT7[6:0]; + // bb_matadd: banks and iter come from rs1, rs2 is zero. + rob_id = 4 + index; + funct = MATADD_THREAD_FUNCT7[6:0]; + rs2 = 64'd0; iter = (index == 2) ? MATADD_MAX_LINES : 16; lines = iter; group_count = (index == 0) ? 1 : 2; - op1_bank = 5'd0; - op2_bank = 5'd1; - wr_bank = 5'd2; - op1_col = group_count[4:0]; - op2_col = group_count[4:0]; - wr_col = group_count[4:0]; - rob_id = 4 + index; - op1_en = 1'b1; - op2_en = 1'b1; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b1; - meta_bank = 5'd0; - rs1 = 64'd0; - rs2 = 64'd0; - special = 64'd0; - is_sub = 1'b0; - sub_rob_id = 8'd0; + op1_bank = 0; + op2_bank = 1; + wr_bank = 2; + op1_group = (index == 0) ? 0 : 1; + op2_group = (index == 0) ? 0 : 3; + wr_group = (index == 0) ? 0 : index; + op1_col = group_count; + op2_col = group_count; + wr_col = group_count; + wr_read_bank = 0; + wr_arch_bank = wr_bank; endfunction function automatic bit [31:0] a_lane(int unsigned group, int unsigned line, int unsigned lane); diff --git a/examples/components/balls/matadd/verify/src/env/matadd_scoreboard.svh b/examples/components/balls/matadd/verify/src/env/matadd_scoreboard.svh index 14ddb898a..7f39dd5b2 100644 --- a/examples/components/balls/matadd/verify/src/env/matadd_scoreboard.svh +++ b/examples/components/balls/matadd/verify/src/env/matadd_scoreboard.svh @@ -34,8 +34,10 @@ class matadd_scoreboard extends uvm_scoreboard; mem_model.clear_mem(); for (int group = 0; group < stim.group_count; group++) begin for (int line = 0; line < stim.lines; line++) begin - mem_model.load_word_g(stim.op1_bank, group, line, stim.a_word(group, line)); - mem_model.load_word_g(stim.op2_bank, group, line, stim.b_word(group, line)); + mem_model.load_word_g(stim.op1_bank, stim.op1_group + group, line, stim.a_word(group, line + )); + mem_model.load_word_g(stim.op2_bank, stim.op2_group + group, line, stim.b_word(group, line + )); end end mem_model.arm(); @@ -51,24 +53,11 @@ class matadd_scoreboard extends uvm_scoreboard; endfunction function void write_cmd(bb_blink_cmd_item item); - matadd_cmd_item expected; - expected = current_stim("CMD"); - if (item.bid !== expected.bid) `uvm_fatal("CMD", "bid mismatch") - if (item.funct7 !== expected.funct7) `uvm_fatal("CMD", "funct7 mismatch") - if (item.iter !== expected.iter) `uvm_fatal("CMD", "iter mismatch") - if (item.op1_bank !== expected.op1_bank || item.op2_bank !== expected.op2_bank || - item.wr_bank !== expected.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (item.op1_col !== expected.group_count || item.op2_col !== expected.group_count || - item.wr_col !== expected.group_count) - `uvm_fatal("CMD", "group field mismatch") - if (item.rob_id !== expected.rob_id) `uvm_fatal("CMD", "rob_id mismatch") - if (item.op1_en !== 1'b1 || item.op2_en !== 1'b1 || item.wr_spad_en !== 1'b1 || - item.op1_from_spad !== 1'b1 || item.op2_from_spad !== 1'b1) - `uvm_fatal("CMD", "enable field mismatch") - if (item.meta_bank !== 0 || item.rs1 !== 0 || item.rs2 !== 0 || item.special !== 0 || - item.is_sub !== 0 || item.sub_rob_id !== 0) - `uvm_fatal("CMD", "reserved field mismatch") + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(current_stim("CMD")); + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) if (cmd_count != 0) `uvm_fatal("CMD", "extra command observed") cmd_count++; endfunction @@ -87,7 +76,8 @@ class matadd_scoreboard extends uvm_scoreboard; if (item.rob_id !== expected.rob_id) `uvm_fatal("READ", "rob_id mismatch") if (item.bank_id !== (item.port == 0 ? expected.op1_bank : expected.op2_bank)) `uvm_fatal("READ", "bank mismatch") - if (item.group_id !== group || item.addr !== line) + if (item.group_id !== (item.port == 0 ? expected.op1_group : expected.op2_group) + group || + item.addr !== line) `uvm_fatal("READ", $sformatf( "address mismatch port=%0d group=%0d line=%0d", item.port, item.group_id, item.addr )) @@ -104,7 +94,7 @@ class matadd_scoreboard extends uvm_scoreboard; line = write_count % expected.lines; if (item.port != 0 || item.bank_id !== expected.wr_bank || item.rob_id !== expected.rob_id) `uvm_fatal("WRITE", "port, bank, or rob_id mismatch") - if (item.group_id !== group || item.addr !== line) + if (item.group_id !== expected.wr_group + group || item.addr !== line) `uvm_fatal("WRITE", $sformatf("address mismatch group=%0d line=%0d", item.group_id, item.addr )) if (item.mask !== 16'hffff) `uvm_fatal("WRITE", "mask mismatch") @@ -117,8 +107,7 @@ class matadd_scoreboard extends uvm_scoreboard; matadd_cmd_item expected; expected = current_stim("RESP"); if (resp_count != 0) `uvm_fatal("RESP", "extra response observed") - if (item.rob_id !== expected.rob_id || item.is_sub !== 0 || item.sub_rob_id !== 0) - `uvm_fatal("RESP", "response mismatch") + if (item.rob_id !== expected.rob_id) `uvm_fatal("RESP", "response mismatch") resp_count++; endfunction diff --git a/examples/components/balls/matadd/verify/src/seq/matadd_sequences.svh b/examples/components/balls/matadd/verify/src/seq/matadd_sequences.svh index e621df916..345175084 100644 --- a/examples/components/balls/matadd/verify/src/seq/matadd_sequences.svh +++ b/examples/components/balls/matadd/verify/src/seq/matadd_sequences.svh @@ -2,7 +2,6 @@ class matadd_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(matadd_seq) int unsigned case_index; - int unsigned bid; function new(string name = "matadd_seq"); super.new(name); @@ -12,7 +11,7 @@ class matadd_seq extends uvm_sequence #(bb_blink_cmd_item); matadd_cmd_item item; item = matadd_cmd_item::type_id::create("item"); start_item(item); - item.load_case(case_index, bid); + item.load_case(case_index); finish_item(item); endtask endclass diff --git a/examples/components/balls/matadd/verify/src/tb_top.sv b/examples/components/balls/matadd/verify/src/tb_top.sv index 0edd42d94..fe7450698 100644 --- a/examples/components/balls/matadd/verify/src/tb_top.sv +++ b/examples/components/balls/matadd/verify/src/tb_top.sv @@ -7,82 +7,75 @@ module tb_top; MatAddBall dut ( .clock(intf.clock), .reset(intf.reset), - .io_bankAccessReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank[3:0]), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank[3:0]), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank[3:0]), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank[3:0]), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0][3:0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_group_id(intf.bank_read_group_id[0][3:0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1][3:0]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_group_id(intf.bank_read_group_id[1][3:0]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0][3:0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0][3:0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) ); + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + initial begin intf.clock = 1'b0; intf.reset = 1'b1; diff --git a/examples/components/balls/matadd/verify/src/tests/matadd_test.svh b/examples/components/balls/matadd/verify/src/tests/matadd_test.svh index 725553e42..7f4d68cd9 100644 --- a/examples/components/balls/matadd/verify/src/tests/matadd_test.svh +++ b/examples/components/balls/matadd/verify/src/tests/matadd_test.svh @@ -17,23 +17,20 @@ class matadd_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid; - bid = matadd_require_bid(); phase.raise_objection(this); - run_case(0, "ONE_GROUP_16_LINES", bid); - run_case(1, "TWO_GROUPS_16_LINES", bid); - run_case(2, "TWO_GROUPS_64_LINES", bid); + run_case(0, "ONE_GROUP_16_LINES"); + run_case(1, "TWO_GROUPS_16_LINES"); + run_case(2, "TWO_GROUPS_64_LINES"); phase.drop_objection(this); endtask - task run_case(int unsigned index, string name, int unsigned bid); + task run_case(int unsigned index, string name); matadd_seq seq; int cycles; apply_reset(); env.scb.reset_counters(); seq = matadd_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); cycles = 0; while (!env.scb.done()) begin diff --git a/examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir b/examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir index 72535d490..36444bc3c 100644 --- a/examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir +++ b/examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir @@ -19,16 +19,16 @@ func.func @main() -> i8 { memref.store %y, %b[%row, %lane] : memref<16x4xi32> } } - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mvin %a <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> - buckyball.mvin %b <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> - buckyball.custom "add_i32" <{sources = array, destination = 2 : i64}> {rows = 16 : i64} - buckyball.mvout %out <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %a <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + buckyball.mvin %b <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + buckyball.custom "add_i32" <{sources = array, destination = 3 : i64}> {rows = 16 : i64} + buckyball.mvout %out <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> func.call @check_result(%out) : (memref<16x4xi32>) -> () memref.dealloc %a : memref<16x4xi32> memref.dealloc %b : memref<16x4xi32> diff --git a/examples/components/balls/maxpool/arch/MaxPoolBall.scala b/examples/components/balls/maxpool/arch/MaxPoolBall.scala index 00cfdcaf4..1f3f7d26e 100644 --- a/examples/components/balls/maxpool/arch/MaxPoolBall.scala +++ b/examples/components/balls/maxpool/arch/MaxPoolBall.scala @@ -38,6 +38,9 @@ class MaxPoolBall( val outputBank = RegInit(0.U(fp.bank_id_len.W)) val bases = Reg(Vec(3, UInt(mp.groupIdWidth.W))) val iter = RegInit(0.U(fp.iter_len.W)) + // Every group of a virtual bank is one 16-channel block with the same rows. + val groups = RegInit(0.U(log2Up(mp.bankNum + 1).W)) + val group = RegInit(0.U(log2Up(mp.bankNum + 1).W)) val inputSide = RegInit(0.U(8.W)) val outputSide = RegInit(0.U(8.W)) val kernel = RegInit(0.U(8.W)) @@ -74,14 +77,14 @@ class MaxPoolBall( blink.bankRead(0).rob_id := robId blink.bankRead(0).bank_id := inputBank - blink.bankRead(0).group_id := bases(0) + blink.bankRead(0).group_id := bases(0) + group blink.bankRead(0).io.req.valid := false.B blink.bankRead(0).io.req.bits.addr := inputAddress blink.bankRead(0).io.resp.ready := false.B blink.bankWrite(0).rob_id := robId blink.bankWrite(0).bank_id := outputBank - blink.bankWrite(0).group_id := bases(2) + blink.bankWrite(0).group_id := bases(2) + group blink.bankWrite(0).io.req.valid := false.B blink.bankWrite(0).io.req.bits.addr := outputAddress blink.bankWrite(0).io.req.bits.data := writeData @@ -107,8 +110,9 @@ class MaxPoolBall( assert(cmd.rs2(63, 46) === 0.U, "MAXPOOL reserved rs2 bits must be zero") assert( - cmd.op1_col === 1.U && cmd.op2_col === 0.U && cmd.wr_col === 1.U, - "MAXPOOL requires one input bank and one output bank" + cmd.op1_col =/= 0.U && cmd.op2_col === 0.U && cmd.wr_col === cmd.op1_col && + cmd.op1_col <= mp.bankNum.U, + "MAXPOOL input and output must have the same channel groups" ) assert(cmd.op1_bank =/= cmd.wr_bank, "MAXPOOL input and output banks must differ") assert( @@ -145,6 +149,8 @@ class MaxPoolBall( outputBank := cmd.wr_bank bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) iter := cmd.iter + groups := cmd.op1_col + group := 0.U inputSide := inSide outputSide := outSide kernel := kernelSize @@ -217,20 +223,27 @@ class MaxPoolBall( is(writeResp) { blink.bankWrite(0).io.resp.ready := true.B when(blink.bankWrite(0).io.resp.fire) { - when(outputRow +& 1.U === iter) { + when(outputRow +& 1.U === iter && group +& 1.U === groups) { state := complete }.otherwise { - outputRow := outputRow + 1.U - when(outputX +& 1.U === outputSide) { - outputX := 0.U - outputY := outputY + 1.U + when(outputRow +& 1.U === iter) { + group := group + 1.U + outputRow := 0.U + outputY := 0.U + outputX := 0.U }.otherwise { - outputX := outputX + 1.U + outputRow := outputRow + 1.U + when(outputX +& 1.U === outputSide) { + outputX := 0.U + outputY := outputY + 1.U + }.otherwise { + outputX := outputX + 1.U + } } - kernelY := 0.U - kernelX := 0.U - maximum := VecInit(Seq.fill(16)("h80".U(8.W))) - state := scan + kernelY := 0.U + kernelX := 0.U + maximum := VecInit(Seq.fill(16)("h80".U(8.W))) + state := scan } } } diff --git a/examples/components/balls/maxpool/emu/src/maxpool.rs b/examples/components/balls/maxpool/emu/src/maxpool.rs index af47d4477..9fb1c7225 100644 --- a/examples/components/balls/maxpool/emu/src/maxpool.rs +++ b/examples/components/balls/maxpool/emu/src/maxpool.rs @@ -1,5 +1,5 @@ use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::instruction::{BallModel, ExecContext}; pub struct MaxPool; @@ -73,10 +73,12 @@ impl BallModel for MaxPool { if input_bank == output_bank { panic!("maxpool: input and output banks must differ"); } - for bank in [input_bank, output_bank] { - if !ctx.config(bank).allocated || ctx.config(bank).cols != 1 { - panic!("maxpool: each operand must occupy one allocated bank"); - } + if !ctx.config(input_bank).allocated || !ctx.config(output_bank).allocated { + panic!("maxpool: bank not allocated"); + } + let groups = ctx.config(input_bank).cols; + if groups == 0 || ctx.config(output_bank).cols != groups { + panic!("maxpool: input and output must have the same channel groups"); } if xs2 >> 46 != 0 || output_side * output_side != iter @@ -96,8 +98,10 @@ impl BallModel for MaxPool { panic!("maxpool: illegal square pooling geometry"); } - let pi = pbank(ctx, input_bank); - let po = pbank(ctx, output_bank); + // Group g of both operands is channel block g; each block uses the same rows. + for group in 0..groups { + let pi = pbank_group(ctx, input_bank, group); + let po = pbank_group(ctx, output_bank, group); let (input, output) = ctx.banks.read_write(pi, po); for output_y in 0..output_side { for output_x in 0..output_side { @@ -130,6 +134,7 @@ impl BallModel for MaxPool { } } } + } 0 } diff --git a/examples/components/balls/maxpool/verify/casegen/src/casegen.rs b/examples/components/balls/maxpool/verify/casegen/src/casegen.rs index aeccf796c..cf818f95c 100644 --- a/examples/components/balls/maxpool/verify/casegen/src/casegen.rs +++ b/examples/components/balls/maxpool/verify/casegen/src/casegen.rs @@ -7,15 +7,14 @@ pub const NUM_CASES: u32 = 9; #[repr(C)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct MaxPoolCmd { - pub bid: u32, pub iter: u32, pub op1_bank: u32, pub wr_bank: u32, + pub op1_group: u32, + pub wr_group: u32, pub op1_col: u32, pub wr_col: u32, pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, pub rs2_lo: u32, pub rs2_hi: u32, pub input_base: u32, @@ -79,10 +78,6 @@ fn field(val: u32, lo: u32, hi: u32) -> u64 { (u64::from(val) & mask) << lo } -fn encode_rs1(input_bank: u32, output_bank: u32, iter: u32) -> u64 { - field(input_bank, 0, 9) | field(0, 10, 19) | field(output_bank, 20, 29) | field(iter, 30, 63) -} - fn encode_rs2(geo: &model::Geo) -> u64 { field(geo.input_side as u32, 0, 3) | field(geo.output_side as u32, 4, 7) @@ -100,25 +95,13 @@ fn split_rs(value: u64) -> (u32, u32) { (value as u32, (value >> 32) as u32) } -fn validate_cmd(op1_bank: u32, wr_bank: u32, iter: u32, rs1: u64, rs2: u64) { +fn validate_cmd(op1_bank: u32, wr_bank: u32, iter: u32, rs2: u64) { if iter == 0 { panic!("maxpool: iter must be positive"); } if op1_bank == wr_bank { panic!("maxpool: input and output banks must differ"); } - if (rs1 & 0x3ff) != u64::from(op1_bank) { - panic!("maxpool: rs1 bank0 mismatch"); - } - if ((rs1 >> 10) & 0x3ff) != 0 { - panic!("maxpool: input bank 1 must be zero"); - } - if ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) { - panic!("maxpool: rs1 bank2 mismatch"); - } - if ((rs1 >> 30) & 0x3fff_ffff) != u64::from(iter) { - panic!("maxpool: rs1 iter mismatch"); - } if (rs2 >> 46) != 0 { panic!("maxpool: reserved rs2 bits must be zero"); } @@ -169,7 +152,7 @@ fn copy_addrs(src: &[u32]) -> [u32; N] { out } -fn build_case(spec: CaseSpec, bid: u32) -> MaxPoolCase { +fn build_case(spec: CaseSpec) -> MaxPoolCase { model::validate(&spec.geo); let iter = model::iter_for(&spec.geo); let input_bytes = if spec.ctest_input { @@ -222,24 +205,22 @@ fn build_case(spec: CaseSpec, bid: u32) -> MaxPoolCase { panic!("maxpool: gold rows {} != iter {iter}", dst_addrs.len()); } - let rs1 = encode_rs1(spec.op1_bank, spec.wr_bank, iter); let rs2 = encode_rs2(&spec.geo); - validate_cmd(spec.op1_bank, spec.wr_bank, iter, rs1, rs2); + validate_cmd(spec.op1_bank, spec.wr_bank, iter, rs2); - let (rs1_lo, rs1_hi) = split_rs(rs1); let (rs2_lo, rs2_hi) = split_rs(rs2); MaxPoolCase { cmd: MaxPoolCmd { - bid, iter, op1_bank: spec.op1_bank, wr_bank: spec.wr_bank, + // Vary the renamed group bases across cases; the Ball adds them to every access. + op1_group: spec.seed % 3, + wr_group: (spec.seed + 1) % 4, op1_col: 1, wr_col: 1, rob_id: spec.rob_id, - rs1_lo, - rs1_hi, rs2_lo, rs2_hi, input_base: spec.geo.input_base as u32, @@ -439,9 +420,9 @@ fn spec_for(index: u32) -> CaseSpec { } } -pub fn gen_case(index: u32, bid: u32) -> MaxPoolCase { +pub fn gen_case(index: u32) -> MaxPoolCase { if index >= NUM_CASES { panic!("maxpool: unsupported directed case {index}"); } - build_case(spec_for(index), bid) + build_case(spec_for(index)) } diff --git a/examples/components/balls/maxpool/verify/casegen/src/lib.rs b/examples/components/balls/maxpool/verify/casegen/src/lib.rs index 024a4fe8a..a556d5929 100644 --- a/examples/components/balls/maxpool/verify/casegen/src/lib.rs +++ b/examples/components/balls/maxpool/verify/casegen/src/lib.rs @@ -10,8 +10,8 @@ thread_local! { } #[no_mangle] -pub extern "C" fn maxpool_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid); +pub extern "C" fn maxpool_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); CURRENT.with(|current| *current.borrow_mut() = Some(case)); 0 } diff --git a/examples/components/balls/maxpool/verify/filelists/maxpool_ball.f b/examples/components/balls/maxpool/verify/filelists/maxpool_ball.f index 1d7b8e584..1ce8e6930 100644 --- a/examples/components/balls/maxpool/verify/filelists/maxpool_ball.f +++ b/examples/components/balls/maxpool/verify/filelists/maxpool_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,6 +14,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv diff --git a/examples/components/balls/maxpool/verify/src/common/maxpool_defs.svh b/examples/components/balls/maxpool/verify/src/common/maxpool_defs.svh index 4b00baf43..df00aad02 100644 --- a/examples/components/balls/maxpool/verify/src/common/maxpool_defs.svh +++ b/examples/components/balls/maxpool/verify/src/common/maxpool_defs.svh @@ -1,13 +1,12 @@ typedef struct { - int unsigned bid; int unsigned iter; int unsigned op1_bank; int unsigned wr_bank; + int unsigned op1_group; + int unsigned wr_group; int unsigned op1_col; int unsigned wr_col; int unsigned rob_id; - int unsigned rs1_lo; - int unsigned rs1_hi; int unsigned rs2_lo; int unsigned rs2_hi; int unsigned input_base; @@ -24,10 +23,7 @@ typedef struct { int unsigned num_dst_words; } maxpool_cmd_dpi_t; -import "DPI-C" function int maxpool_case_load( - input int unsigned index, - input int unsigned bid -); +import "DPI-C" function int maxpool_case_load(input int unsigned index); import "DPI-C" function void maxpool_case_cmd(output maxpool_cmd_dpi_t cmd); import "DPI-C" function longint unsigned maxpool_case_input_word_lo(input int unsigned word_index); import "DPI-C" function longint unsigned maxpool_case_input_word_hi(input int unsigned word_index); @@ -43,13 +39,5 @@ import "DPI-C" function int unsigned maxpool_case_dst_addr(input int unsigned wo localparam int MAXPOOL_THREAD_FUNCT7 = `MAXPOOL_FUNCT7; localparam int MAXPOOL_MAX_INPUT_WORDS = 64; localparam int MAXPOOL_MAX_DST_WORDS = 64; -localparam int MAXPOOL_NUM_CASES = 8; +localparam int MAXPOOL_NUM_CASES = 9; localparam int MAXPOOL_TIMEOUT_CYCLES = 20000; - -function automatic int unsigned maxpool_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/components/balls/maxpool/verify/src/common/maxpool_items.svh b/examples/components/balls/maxpool/verify/src/common/maxpool_items.svh index 51819a9c8..f69b45154 100644 --- a/examples/components/balls/maxpool/verify/src/common/maxpool_items.svh +++ b/examples/components/balls/maxpool/verify/src/common/maxpool_items.svh @@ -18,49 +18,39 @@ class maxpool_cmd_item extends bb_blink_cmd_item; bit [127:0] dst_words[MAXPOOL_MAX_DST_WORDS]; bit [31:0] dst_addr[MAXPOOL_MAX_DST_WORDS]; - constraint legal_c { - funct7 == MAXPOOL_THREAD_FUNCT7[6:0]; - op1_en == 1'b1; - op2_en == 1'b0; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b0; - op1_col == 5'd1; - op2_bank == 5'd0; - op2_col == 5'd0; - wr_col == 5'd1; - meta_bank == 5'd0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - special == 64'd0; - } - function new(string name = "maxpool_cmd_item"); super.new(name); endfunction - function void load_rust_case(int unsigned index, int unsigned bid); + function void load_rust_case(int unsigned index); maxpool_cmd_dpi_t cmd; int unsigned rc; longint unsigned w_lo; longint unsigned w_hi; int i; - rc = maxpool_case_load(index, bid); + rc = maxpool_case_load(index); if (rc != 0) begin `uvm_fatal("CASE", $sformatf("maxpool_case_load returned %0d for index %0d", rc, index)) end maxpool_case_cmd(cmd); - this.bid = cmd.bid[4:0]; - iter = cmd.iter; - op1_bank = cmd.op1_bank[4:0]; - wr_bank = cmd.wr_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - wr_col = cmd.wr_col[4:0]; - rob_id = cmd.rob_id[3:0]; - rs1 = {cmd.rs1_hi, cmd.rs1_lo}; + // bb_maxpool: banks and iter come from rs1, the pooling geometry from rs2. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = MAXPOOL_THREAD_FUNCT7[6:0]; rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = '0; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = '0; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = '0; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; input_base = cmd.input_base; output_base = cmd.output_base; output_stride = cmd.output_stride; @@ -73,22 +63,9 @@ class maxpool_cmd_item extends bb_blink_cmd_item; start_col = cmd.start_col; num_input_words = cmd.num_input_words; num_dst_words = cmd.num_dst_words; - funct7 = MAXPOOL_THREAD_FUNCT7[6:0]; - op1_en = 1'b1; - op2_en = 1'b0; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b0; - op2_bank = 5'd0; - op2_col = 5'd0; - meta_bank = 5'd0; - special = 64'd0; - is_sub = 1'b0; - sub_rob_id = 8'h00; - if (rs1[19:10] != 0) `uvm_fatal("CASE", "MAXPOOL reserves input bank 1") if (op1_col != 1 || wr_col != 1) - `uvm_fatal("CASE", "MAXPOOL requires one input bank and one output bank") + `uvm_fatal("CASE", "MAXPOOL requires one input group and one output group") if (op1_bank == wr_bank) `uvm_fatal("CASE", "MAXPOOL input and output banks must differ") if (rs2[63:46] != 0) `uvm_fatal("CASE", "MAXPOOL reserved rs2 bits must be zero") if (num_dst_words != iter) diff --git a/examples/components/balls/maxpool/verify/src/cov/maxpool_cov.svh b/examples/components/balls/maxpool/verify/src/cov/maxpool_cov.svh index c1f954c72..9211a2549 100644 --- a/examples/components/balls/maxpool/verify/src/cov/maxpool_cov.svh +++ b/examples/components/balls/maxpool/verify/src/cov/maxpool_cov.svh @@ -21,8 +21,8 @@ class maxpool_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != MAXPOOL_THREAD_FUNCT7[6:0]) - `uvm_fatal("COV", $sformatf("unexpected funct7 %0d", item.funct7)) + if (item.funct != MAXPOOL_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) cur_kernel = int'(item.rs2[11:8]); cur_stride = int'(item.rs2[15:12]); cur_padding = int'(item.rs2[19:16]); diff --git a/examples/components/balls/maxpool/verify/src/env/maxpool_scoreboard.svh b/examples/components/balls/maxpool/verify/src/env/maxpool_scoreboard.svh index 69bd0b791..18051983e 100644 --- a/examples/components/balls/maxpool/verify/src/env/maxpool_scoreboard.svh +++ b/examples/components/balls/maxpool/verify/src/env/maxpool_scoreboard.svh @@ -46,7 +46,8 @@ class maxpool_scoreboard extends uvm_scoreboard; end mem_model.clear_mem(); for (i = 0; i < clone.num_input_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), int'(clone.input_addr[i]), clone.input_words[i]); + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group), int'(clone.input_addr[i]), + clone.input_words[i]); end mem_model.arm(); @@ -61,22 +62,12 @@ class maxpool_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, maxpool_cmd_item exp); + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); verification_pkg::evidence::checks++; - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rs1 !== exp.rs1 || got.rs2 !== exp.rs2) `uvm_fatal("CMD", "rs field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) - if (got.op2_bank !== 0 || got.op2_col !== 0) - `uvm_fatal("CMD", "MAXPOOL input bank 1 must be zero") + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) endfunction function bit input_addr_legal(maxpool_cmd_item stim, int unsigned addr); @@ -100,8 +91,9 @@ class maxpool_scoreboard extends uvm_scoreboard; stim = current_stim("READ"); if (item.port != 0) `uvm_fatal("READ", $sformatf("MaxPoolBall has one read port, got port %0d", item.port)) - if (item.group_id !== 5'd0) - `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp 0", item.group_id)) + if (item.group_id !== stim.op1_group) + `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.op1_group + )) if (item.rob_id !== stim.rob_id) `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) if (item.bank_id !== stim.op1_bank) @@ -121,8 +113,9 @@ class maxpool_scoreboard extends uvm_scoreboard; `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== 5'd0) - `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp 0", item.group_id)) + if (item.group_id !== stim.wr_group) + `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + )) if (item.mask !== 16'hFFFF) `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) if (write_count >= stim.num_dst_words) @@ -146,9 +139,6 @@ class maxpool_scoreboard extends uvm_scoreboard; stim = current_stim("RESP"); if (item.rob_id !== stim.rob_id) `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") - if (item.sub_rob_id !== 8'h00) - `uvm_fatal("RESP", $sformatf("sub_rob_id mismatch: got 0x%0h", item.sub_rob_id)) resp_count++; endfunction diff --git a/examples/components/balls/maxpool/verify/src/seq/maxpool_sequences.svh b/examples/components/balls/maxpool/verify/src/seq/maxpool_sequences.svh index dab28f67c..d88ab6ebb 100644 --- a/examples/components/balls/maxpool/verify/src/seq/maxpool_sequences.svh +++ b/examples/components/balls/maxpool/verify/src/seq/maxpool_sequences.svh @@ -2,7 +2,6 @@ class maxpool_basic_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(maxpool_basic_seq) int unsigned case_index; - int unsigned bid; function new(string name = "maxpool_basic_seq"); super.new(name); @@ -12,7 +11,7 @@ class maxpool_basic_seq extends uvm_sequence #(bb_blink_cmd_item); maxpool_cmd_item item; item = maxpool_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(case_index, bid); + item.load_rust_case(case_index); finish_item(item); endtask endclass diff --git a/examples/components/balls/maxpool/verify/src/tb_top.sv b/examples/components/balls/maxpool/verify/src/tb_top.sv index 72ea12549..60371efe0 100644 --- a/examples/components/balls/maxpool/verify/src/tb_top.sv +++ b/examples/components/balls/maxpool/verify/src/tb_top.sv @@ -7,72 +7,66 @@ module tb_top; MaxPoolBall dut ( .clock(intf.clock), .reset(intf.reset), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) ); + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + initial begin intf.clock = 1'b0; intf.reset = 1'b1; diff --git a/examples/components/balls/maxpool/verify/src/tests/maxpool_test.svh b/examples/components/balls/maxpool/verify/src/tests/maxpool_test.svh index f24ddff2e..85119466e 100644 --- a/examples/components/balls/maxpool/verify/src/tests/maxpool_test.svh +++ b/examples/components/balls/maxpool/verify/src/tests/maxpool_test.svh @@ -17,20 +17,18 @@ class maxpool_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid = maxpool_require_bid(); phase.raise_objection(this); - for (int unsigned index = 0; index < MAXPOOL_NUM_CASES; index++) run_case(index, bid); + for (int unsigned index = 0; index < MAXPOOL_NUM_CASES; index++) run_case(index); phase.drop_objection(this); endtask - task run_case(int unsigned index, int unsigned bid); + task run_case(int unsigned index); maxpool_basic_seq seq; int cycles = 0; apply_reset(); env.scb.reset_counters(); seq = maxpool_basic_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); while (!env.scb.done()) begin @(posedge vif.clock); diff --git a/examples/components/balls/maxpool/workloads/isa/maxpool.h b/examples/components/balls/maxpool/workloads/isa/maxpool.h index d0004b701..5926ee7dc 100644 --- a/examples/components/balls/maxpool/workloads/isa/maxpool.h +++ b/examples/components/balls/maxpool/workloads/isa/maxpool.h @@ -4,6 +4,10 @@ #include #include +// Group g of the input and output banks is channel block g (16 INT8 channels +// per row). Both banks have the same number of groups; every group is pooled +// with the same geometry, so one instruction pools all channel blocks. + #define bb_maxpool(input_bank, output_bank, input_side, output_side, kernel, \ stride, padding, input_base, output_base, output_stride, \ start_row, start_col) \ diff --git a/examples/components/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir b/examples/components/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir index b3c877818..e3e6d65be 100644 --- a/examples/components/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir +++ b/examples/components/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir @@ -13,13 +13,13 @@ func.func @main() -> i8 { memref.store %value, %input[%row, %lane] : memref<36x16xi8> } } - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mvin %input <{bank = 0 : i64, depth = 36 : i64, stride = 1 : i64}> : memref<36x16xi8> - buckyball.custom "maxpool_i8" <{sources = array, destination = 1 : i64}> {input_side = 6 : i64, output_side = 3 : i64, kernel = 2 : i64, stride = 2 : i64, padding = 0 : i64, op1_base = 0 : i64, wr_base = 0 : i64, output_stride = 3 : i64, start_row = 0 : i64, start_col = 0 : i64} - buckyball.mvout %output <{bank = 1 : i64, depth = 9 : i64, stride = 1 : i64}> : memref<9x16xi8> - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 36 : i64, stride = 1 : i64}> : memref<36x16xi8> + buckyball.custom "maxpool_i8" <{sources = array, destination = 2 : i64}> {input_side = 6 : i64, output_side = 3 : i64, kernel = 2 : i64, stride = 2 : i64, padding = 0 : i64, op1_base = 0 : i64, wr_base = 0 : i64, output_stride = 3 : i64, start_row = 0 : i64, start_col = 0 : i64} + buckyball.mvout %output <{bank = 2 : i64, depth = 9 : i64, stride = 1 : i64}> : memref<9x16xi8> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> func.call @check_result(%output) : (memref<9x16xi8>) -> () memref.dealloc %input : memref<36x16xi8> memref.dealloc %output : memref<9x16xi8> diff --git a/examples/components/balls/mxmm/arch/AddPipe.scala b/examples/components/balls/mxmm/arch/AddPipe.scala index 83ac251f4..8688abfed 100644 --- a/examples/components/balls/mxmm/arch/AddPipe.scala +++ b/examples/components/balls/mxmm/arch/AddPipe.scala @@ -22,7 +22,7 @@ class AddPipe extends Module { val decodedA = RegEnable(rawFloatFromRecFN(8, 24, io.a), io.valid) val decodedB = RegEnable(rawFloatFromRecFN(8, 24, io.b), io.valid) val decodedValid = RegNext(io.valid, false.B) - val add = Instantiate(new AddRawFN(8, 24)) + val add = Module(new AddRawFN(8, 24)) add.io.subOp := false.B add.io.a := decodedA add.io.b := decodedB @@ -31,7 +31,7 @@ class AddPipe extends Module { val sum = RegEnable(add.io.rawOut, decodedValid) val invalid = RegEnable(add.io.invalidExc, decodedValid) val sumValid = RegNext(decodedValid, false.B) - val round = Instantiate(new RoundRawFNToRecFN(8, 24, 0)) + val round = Module(new RoundRawFNToRecFN(8, 24, 0)) round.io.in := sum round.io.invalidExc := invalid round.io.infiniteExc := false.B diff --git a/examples/components/balls/redmule/arch/RedmuleBall.scala b/examples/components/balls/redmule/arch/RedmuleBall.scala index 12de4f5c7..a009ad89e 100644 --- a/examples/components/balls/redmule/arch/RedmuleBall.scala +++ b/examples/components/balls/redmule/arch/RedmuleBall.scala @@ -82,10 +82,11 @@ class RedmuleBall( val inWindow = bankRow >= startRow && bankRow < (startRow +& span) val readPort = Mux(region === 2.U, 1.U, 0.U) + // A continuation reads the previous C from the bank the destination held before this write. val readBank = Mux( region === 1.U, issue.op1_bank, - Mux(region === 2.U, issue.op2_bank, issue.wr_bank) + Mux(region === 2.U, issue.op2_bank, issue.wr_read_bank) ) val readGroup = Mux(region === 1.U, issue.op1_group, Mux(region === 2.U, issue.op2_group, issue.wr_group)) diff --git a/examples/components/balls/redmule/stack/src/Lowering.cpp b/examples/components/balls/redmule/stack/src/Lowering.cpp index 306e5fd90..db4c60ed3 100644 --- a/examples/components/balls/redmule/stack/src/Lowering.cpp +++ b/examples/components/balls/redmule/stack/src/Lowering.cpp @@ -1,10 +1,14 @@ //===- Lowering.cpp - RedmuleBall in lower-buckyball ----------------------===// // // matmul_f16: C = A @ B in FP16 with ordered FMAs, or C = C + A @ B with -// read_destination. A (source 0) holds `rows` rows of `k` f16 from bank row -// op1_base, k / 8 bank rows each; B (source 1) holds `k` rows of eight f16 -// from bank row op2_base; C receives `rows` rows of eight f16 at bank row -// wr_base. `k` is a positive multiple of eight. +// read_destination. `k` is a positive multiple of eight. +// A (source 0): `rows` rows of `k` f16, k / 8 bank rows each. +// B (source 1): `k` bank rows of eight f16. +// C: `rows` bank rows of eight f16. +// +// The Ball places every operand at bank row 0. With read_destination rs1 +// marks the old C, which the Ball reads from the bank the destination held +// before this write, and the FMAs start from it instead of positive zero. // //===----------------------------------------------------------------------===// @@ -25,34 +29,28 @@ class MatmulF16Lowering : public OpRewritePattern { return failure(); int64_t rows = cast(op->getAttr("rows")).getInt(); int64_t k = cast(op->getAttr("k")).getInt(); - int64_t op1Base = cast(op->getAttr("op1_base")).getInt(); - int64_t op2Base = cast(op->getAttr("op2_base")).getInt(); - int64_t wrBase = cast(op->getAttr("wr_base")).getInt(); int64_t c = op.getDestination(); + bool readDestination = op.getReadDestination(); const auto &target = buckyball_target::getBuckyballTarget(); if (op.getSources().size() != 2 || op.getSources()[0] == op.getSources()[1] || llvm::is_contained(op.getSources(), c)) return op.emitOpError("FP16 matmul requires distinct A, B and C banks"); if (target.bankWidthBits != 128 || rows <= 0 || rows > 4095 || k <= 0 || - k % 8 || k >= (1LL << 33) || op1Base < 0 || op1Base > 1023 || - op2Base < 0 || op2Base > 1023 || wrBase < 0 || wrBase > 1023 || - op1Base + rows * (k / 8) > target.bankDepth || - op2Base + k > target.bankDepth || wrBase + rows > target.bankDepth) + k % 8 || k >= (1LL << 33) || rows * (k / 8) > target.bankDepth || + k > target.bankDepth || rows > target.bankDepth) return op.emitOpError("FP16 matmul shape exceeds the bank or ISA " "geometry"); - // The Ball reads the old C itself when not first; rs1 does not mark it. + // rs2: rows in [11:0], the A, B and C row bases (all 0) in [41:12], and + // in bit 42 whether the FMAs start from positive zero. uint64_t rs2 = mlir::buddy::field(rows, 0, 11) | - mlir::buddy::field(op1Base, 12, 21) | - mlir::buddy::field(op2Base, 22, 31) | - mlir::buddy::field(wrBase, 32, 41) | - mlir::buddy::field(!op.getReadDestination(), 42, 42); + mlir::buddy::field(!readDestination, 42, 42); mlir::buddy::emitInstruction( rewriter, op.getLoc(), buckyball_target::getBuckyballFunct7("REDMULE_F16"), mlir::buddy::constantI64( rewriter, op.getLoc(), - mlir::buddy::bankFields(op.getSources(), c, false, k)), + mlir::buddy::bankFields(op.getSources(), c, readDestination, k)), mlir::buddy::constantI64(rewriter, op.getLoc(), rs2)); rewriter.eraseOp(op); return success(); diff --git a/examples/components/balls/redmule/verify/filelists/redmule_ball.f b/examples/components/balls/redmule/verify/filelists/redmule_ball.f index 3ebb60a92..70b8c3fba 100644 --- a/examples/components/balls/redmule/verify/filelists/redmule_ball.f +++ b/examples/components/balls/redmule/verify/filelists/redmule_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -5,6 +6,7 @@ +incdir+@UVM@/src/ball/cov +incdir+@UVM@/src/ball/env +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv -F @RTL@/filelist.f diff --git a/examples/components/balls/redmule/verify/src/pkg/redmule_pkg.sv b/examples/components/balls/redmule/verify/src/pkg/redmule_pkg.sv index 7de087b0f..a2ccb1025 100644 --- a/examples/components/balls/redmule/verify/src/pkg/redmule_pkg.sv +++ b/examples/components/balls/redmule/verify/src/pkg/redmule_pkg.sv @@ -33,7 +33,6 @@ package redmule_uvm_pkg; vif.bank_write_resp_valid[0] <= 0; vif.bank_write_resp_ok[0] <= 1; vif.cmd_resp_ready <= 1; - vif.sub_rob_req_ready <= 1; end else begin for (int port = 0; port < 2; port++) begin if (vif.bank_read_resp_valid[port] && vif.bank_read_resp_ready[port]) @@ -41,11 +40,13 @@ package redmule_uvm_pkg; if (vif.bank_read_req_valid[port] && vif.bank_read_req_ready[port]) begin if (!mem.exists( vif.bank_read_bank_id[port] - ) || !mem[vif.bank_read_bank_id[port]][0].exists( + ) || !mem[vif.bank_read_bank_id[port]].exists( + vif.bank_read_group_id[port] + ) || !mem[vif.bank_read_bank_id[port]][vif.bank_read_group_id[port]].exists( vif.bank_read_req_addr[port] )) `uvm_fatal("MEM", "bank read was not preloaded") - vif.bank_read_resp_data[port] <= mem[vif.bank_read_bank_id[port]][0][vif.bank_read_req_addr[port]]; + vif.bank_read_resp_data[port] <= mem[vif.bank_read_bank_id[port]][vif.bank_read_group_id[port]][vif.bank_read_req_addr[port]]; vif.bank_read_resp_valid[port] <= 1; end vif.bank_read_req_ready[port] <= (!vif.bank_read_resp_valid[port] || vif.bank_read_resp_ready[port]) && @@ -79,33 +80,34 @@ package redmule_uvm_pkg; class redmule_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(redmule_seq) - int bid, rows, k, base, block_index, blocks; + int rows, k, base, block_index, blocks; + int groups[4]; function new(string name = "redmule_seq"); super.new(name); endfunction task body(); bb_blink_cmd_item item = bb_blink_cmd_item::type_id::create("item"); start_item(item); - item.bid = bid; - item.funct7 = `REDMULE_F16_FUNCT7; + // bb_redmule(a=0, b=1, c=2, m, k, base, base, base, first): banks and K from rs1. + // The frontend renames C on every write: block j writes bank 2 + j % 2 and a + // continuation reads the previous C from the other one, its wr_read_bank. + item.rob_id = block_index; + item.funct = `REDMULE_F16_FUNCT7; + item.rs2 = longint'(rows) | (longint'(base) << 12) | + (longint'(base) << 22) | (longint'(base) << 32) | + (longint'(block_index == 0) << 42); item.iter = k; - item.op1_en = 1; - item.op2_en = 1; - item.wr_spad_en = 1; - item.op1_from_spad = 1; - item.op2_from_spad = 1; item.op1_bank = 0; item.op2_bank = 1; - item.wr_bank = 2; + item.wr_bank = 2 + block_index % 2; + item.op1_group = groups[0]; + item.op2_group = groups[1]; + item.wr_group = groups[2]; item.op1_col = 1; item.op2_col = 1; item.wr_col = 1; - item.rs1 = (longint'(k) << 30) | (2 << 20) | (1 << 10); - item.rs2 = longint'(rows) | (longint'(base) << 12) | - (longint'(base) << 22) | (longint'(base) << 32) | - (longint'(block_index == 0) << 42); - item.special = item.rs2; - item.rob_id = block_index; + item.wr_read_bank = block_index == 0 ? 0 : 3 - block_index % 2; + item.wr_arch_bank = 2; finish_item(item); endtask endclass @@ -118,6 +120,7 @@ package redmule_uvm_pkg; uvm_analysis_imp_resp #(bb_blink_resp_item, redmule_scoreboard) resp_imp; bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; int rows, k, base, blocks, block_index; + int groups[4]; int reads[2], writes, commands, responses; bit first; bit a_seen[int], b_seen[int], c_seen[int]; @@ -141,8 +144,13 @@ package redmule_uvm_pkg; number_cg = new(); endfunction function void write_cmd(bb_blink_cmd_item item); - if (item.funct7 != `REDMULE_F16_FUNCT7 || item.iter != k || - item.rs2[11:0] != rows || item.rob_id != block_index) + if (item.funct != `REDMULE_F16_FUNCT7 || item.iter != k || + item.rs2[11:0] != rows || item.rob_id != block_index || + item.op1_bank != 0 || item.op2_bank != 1 || item.wr_bank != 2 + block_index % 2 || + item.op1_group != groups[0] || item.op2_group != groups[1] || + item.wr_group != groups[2] || + item.wr_read_bank != (block_index == 0 ? 0 : 3 - block_index % 2) || + item.wr_arch_bank != 2) `uvm_fatal("CMD", "unexpected RedMulE command") first = item.rs2[42]; m_bucket = rows; @@ -153,15 +161,15 @@ package redmule_uvm_pkg; endfunction function void write_read(bb_blink_read_item item); int limit; - if (item.group_id != 0 || item.rob_id != block_index) - `uvm_fatal("READ", "unexpected read metadata") + if (item.rob_id != block_index) `uvm_fatal("READ", "unexpected read metadata") + if (item.group_id != groups[item.bank_id]) `uvm_fatal("READ", "unexpected read group") if (item.port == 1 && item.bank_id == 1) begin limit = base + k; b_seen[item.addr] = 1; end else if (item.port == 0 && item.bank_id == 0) begin limit = base + rows * k / 8; a_seen[item.addr] = 1; - end else if (item.port == 0 && item.bank_id == 2 && !first) begin + end else if (item.port == 0 && item.bank_id == 3 - block_index % 2 && !first) begin limit = base + rows; c_seen[item.addr] = 1; end else `uvm_fatal("READ", "unexpected read bank") @@ -176,8 +184,8 @@ package redmule_uvm_pkg; redmule_case_word(2, block_index * rows + writes, 1), redmule_case_word(2, block_index * rows + writes, 0) }; - if (item.port != 0 || item.bank_id != 2 || item.addr != base + writes || - item.rob_id != block_index || item.group_id != 0 || item.mask != 16'hffff || + if (item.port != 0 || item.bank_id != 2 + block_index % 2 || item.addr != base + writes || + item.rob_id != block_index || item.group_id != groups[2] || item.mask != 16'hffff || item.data !== expected) `uvm_fatal("WRITE", $sformatf( "row %0d got %032h expected %032h", item.addr, item.data, expected)) @@ -190,11 +198,11 @@ package redmule_uvm_pkg; else number_class = 4; number_cg.sample(); end - mem_model.load_word(2, item.addr, item.data); + mem_model.load_word_g(item.bank_id, groups[2], item.addr, item.data); writes++; endfunction function void write_resp(bb_blink_resp_item item); - if (item.rob_id != block_index || item.is_sub != 0 || item.sub_rob_id != 0 || writes != rows) + if (item.rob_id != block_index || writes != rows) `uvm_fatal("RESP", "completion preceded required reads/writes") for (int row = base; row < base + rows * k / 8; row++) if (!a_seen.exists(row)) `uvm_fatal("READ", "missing A row") @@ -257,13 +265,13 @@ package redmule_uvm_pkg; env = redmule_env::type_id::create("env", this); endfunction task run_phase(uvm_phase phase); - int bid, rows, k, base, blocks, before_response, cycles; + int rows, k, base, blocks, before_response, cycles; + int groups[4]; longint unsigned shape; longint native_cycles, ball_started, ball_cycles, total_native = 0, total_ball = 0; redmule_seq seq; redmule_mem_model memory_agent; if (!$cast(memory_agent, env.mem_model)) `uvm_fatal("MEM", "wrong memory model type") - if (!$value$plusargs("BID=%d", bid)) `uvm_fatal("BID", "missing BID") phase.raise_objection(this); for (int index = 0; index < 67; index++) begin vif.reset = 1; @@ -281,19 +289,23 @@ package redmule_uvm_pkg; env.scb.k = k; env.scb.base = base; env.scb.blocks = blocks; + // Odd cases place A, B and C at nonzero group bases of their banks; both + // banks of C use the destination group. + groups = index % 2 ? '{1, 2, 3, 3} : '{0, 0, 0, 0}; + env.scb.groups = groups; env.mem_model.clear_mem(); for (int row = 0; row < base + rows * k / 8; row++) begin - env.mem_model.load_word(0, row, { - redmule_case_word(0, row, 1), redmule_case_word(0, row, 0)}); + env.mem_model.load_word_g(0, groups[0], row, { + redmule_case_word(0, row, 1), redmule_case_word(0, row, 0)}); reference.load_word(0, row, {redmule_case_word(0, row, 1), redmule_case_word(0, row, 0)}); end for (int row = 0; row < base + k; row++) begin - env.mem_model.load_word(1, row, { - redmule_case_word(1, row, 1), redmule_case_word(1, row, 0)}); + env.mem_model.load_word_g(1, groups[1], row, { + redmule_case_word(1, row, 1), redmule_case_word(1, row, 0)}); reference.load_word(1, row, {redmule_case_word(1, row, 1), redmule_case_word(1, row, 0)}); end for (int row = 0; row < base + rows; row++) begin - env.mem_model.load_word(2, row, 0); + env.mem_model.load_word_g(2, groups[2], row, 0); reference.load_word(2, row, 0); end env.mem_model.arm(); @@ -306,7 +318,7 @@ package redmule_uvm_pkg; env.scb.c_seen.delete(); before_response = env.scb.responses; seq = redmule_seq::type_id::create("seq"); - seq.bid = bid; + seq.groups = groups; seq.rows = rows; seq.k = k; seq.base = base; diff --git a/examples/components/balls/redmule/verify/src/tb_top.sv b/examples/components/balls/redmule/verify/src/tb_top.sv index 02293ac1c..4171189d3 100644 --- a/examples/components/balls/redmule/verify/src/tb_top.sv +++ b/examples/components/balls/redmule/verify/src/tb_top.sv @@ -16,85 +16,71 @@ module tb_top; RedmuleBall dut ( .clock(intf.clock), .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_bankAccessReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .blink_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .blink_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .blink_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .blink_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .blink_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .blink_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .blink_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .blink_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .blink_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .blink_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .blink_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .blink_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .blink_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .blink_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .blink_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) ); initial begin diff --git a/examples/components/balls/redmule/workloads/isa/redmule.h b/examples/components/balls/redmule/workloads/isa/redmule.h index e089fab71..963c37a89 100644 --- a/examples/components/balls/redmule/workloads/isa/redmule.h +++ b/examples/components/balls/redmule/workloads/isa/redmule.h @@ -4,7 +4,9 @@ #include // FP16 A[M,K], B[K,8], C[M,8]; ordered FP16 FMA, RNE, canonical NaN. -// Bases are bank rows. First starts with positive zero; every call writes C. +// Bases are bank rows. First starts with positive zero; otherwise rs1 marks +// the old C, which the Ball reads from the bank C held before this write. +// Every call writes C. static inline void bb_redmule(uint32_t a, uint32_t b, uint32_t c, uint32_t m, uint32_t k, uint32_t op1_base, uint32_t op2_base, uint32_t wr_base, uint32_t first) { @@ -16,7 +18,7 @@ static inline void bb_redmule(uint32_t a, uint32_t b, uint32_t c, uint32_t m, (uint64_t)op2_base << 22 | (uint64_t)wr_base << 32 | (uint64_t)first << 42; BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(a) | BB_BANK1(b) | BB_BANK2(c) | - BB_ITER(k), + BB_ITER(k) | (first ? 0 : BB_READ_WR_OLD), special, BB_FUNC7(REDMULE_F16)); } #endif diff --git a/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir b/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir index e75931747..f079b7c70 100644 --- a/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir +++ b/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir @@ -1,6 +1,6 @@ -// The banks 0 (A), 1 (B) and 2 (C) are allocated and loaded by the caller. +// The banks 1 (A), 2 (B) and 3 (C) are allocated and loaded by the caller. func.func @redmule_chain() attributes {llvm.emit_c_interface} { - buckyball.custom "matmul_f16" <{sources = array, destination = 2 : i64}> {rows = 2 : i64, k = 8 : i64, op1_base = 0 : i64, op2_base = 0 : i64, wr_base = 0 : i64} - buckyball.custom "matmul_f16" <{sources = array, destination = 2 : i64, read_destination}> {rows = 2 : i64, k = 8 : i64, op1_base = 0 : i64, op2_base = 0 : i64, wr_base = 0 : i64} + buckyball.custom "matmul_f16" <{sources = array, destination = 3 : i64}> {rows = 2 : i64, k = 8 : i64} + buckyball.custom "matmul_f16" <{sources = array, destination = 3 : i64, read_destination}> {rows = 2 : i64, k = 8 : i64} return } diff --git a/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp b/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp index a54f71af8..0c3e6ddb2 100644 --- a/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp +++ b/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp @@ -10,19 +10,19 @@ int main() { value = 0x3c00; for (auto &value : b) value = 0x3800; - bb_mem_alloc(0, 1, 1); bb_mem_alloc(1, 1, 1); bb_mem_alloc(2, 1, 1); - bb_mvin((uintptr_t)a, 0, 2, 1); - bb_mvin((uintptr_t)b, 1, 8, 1); + bb_mem_alloc(3, 1, 1); + bb_mvin((uintptr_t)a, 1, 2, 1); + bb_mvin((uintptr_t)b, 2, 8, 1); _mlir_ciface_redmule_chain(); - bb_mvout((uintptr_t)c, 2, 2, 1); + bb_mvout((uintptr_t)c, 3, 2, 1); for (auto value : c) if (value != 0x4800) __builtin_trap(); - bb_mem_release(0); bb_mem_release(1); bb_mem_release(2); + bb_mem_release(3); puts("RedMulE BallOp FP16 chain PASS"); return 0; } diff --git a/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir b/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir index e558d5513..0f1cf5852 100644 --- a/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir +++ b/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir @@ -1,7 +1,6 @@ // RUN: buddy-opt %s -lower-buckyball -verify-diagnostics func.func @invalid_k() { - // expected-error @+2 {{FP16 matmul shape exceeds the bank or ISA geometry}} - // expected-error @+1 {{has no lowering on this core}} - buckyball.custom "matmul_f16" <{sources = array, destination = 2 : i64}> {rows = 1 : i64, k = 7 : i64, op1_base = 0 : i64, op2_base = 0 : i64, wr_base = 0 : i64} + // expected-error @+1 {{FP16 matmul shape exceeds the bank or ISA geometry}} + buckyball.custom "matmul_f16" <{sources = array, destination = 3 : i64}> {rows = 1 : i64, k = 7 : i64} return } diff --git a/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir b/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir index e7c709e50..be089fca3 100644 --- a/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir +++ b/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir @@ -2,9 +2,8 @@ func.func @invalid_alias() { %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> - // expected-error @+2 {{FP16 matmul requires distinct A, B and C banks}} - // expected-error @+1 {{has no lowering on this core}} - %c = buckyball.bank_custom "matmul_f16" (%a, %b) -> %a {rows = 1 : i64, k = 8 : i64, op1_base = 0 : i64, op2_base = 0 : i64, wr_base = 0 : i64} + // expected-error @+1 {{FP16 matmul requires distinct A, B and C banks}} + %c = buckyball.bank_custom "matmul_f16" (%a, %b) -> %a {rows = 1 : i64, k = 8 : i64} buckyball.bank_release %c buckyball.bank_release %b return diff --git a/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in b/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in index e01fb0c17..121181a92 100644 --- a/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in +++ b/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in @@ -35,7 +35,7 @@ func.func @main() -> i8 { %loaded_a = buckyball.bank_mvin %a, %a_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x8xf16> %loaded_b = buckyball.bank_mvin %b, %b_bank <{depth = 8 : i64, stride = 1 : i64}> : memref<8x8xf16> %result = buckyball.bank_custom "matmul_f16" (%loaded_a, %loaded_b) -> %c_bank - {rows = @BB_BANK_LINES@ : i64, k = 8 : i64, op1_base = 0 : i64, op2_base = 0 : i64, wr_base = 0 : i64} + {rows = @BB_BANK_LINES@ : i64, k = 8 : i64} %stored = buckyball.bank_mvout %c, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x8xf16> buckyball.bank_release %loaded_a buckyball.bank_release %loaded_b diff --git a/examples/components/balls/relu/stack/src/Lowering.cpp b/examples/components/balls/relu/stack/src/Lowering.cpp index ff86e2073..3cf6f7178 100644 --- a/examples/components/balls/relu/stack/src/Lowering.cpp +++ b/examples/components/balls/relu/stack/src/Lowering.cpp @@ -1,9 +1,11 @@ //===- Lowering.cpp - ReluBall in lower-buckyball -------------------------===// // -// relu_i32: max(x, 0) on the i32 lanes of physical bank `group` of the bank, -// in place: the one source is the destination. In every `stride` rows of the -// physical bank the first `rows` rows are updated; `rows` is a multiple of -// sixteen no larger than `stride`, and `stride` divides the bank depth. +// relu_i32: max(x, 0) on the first `rows` rows of four i32 of a one-group +// bank, in place: the one source is the destination. +// +// The Ball updates whole groups of sixteen rows: it rounds `rows` up to a +// multiple of sixteen, which only touches rows past the valid data, and runs +// one segment as deep as the bank. // //===----------------------------------------------------------------------===// @@ -23,24 +25,21 @@ class ReluI32Lowering : public OpRewritePattern { if (op.getKind() != "relu_i32") return failure(); int64_t rows = cast(op->getAttr("rows")).getInt(); - int64_t stride = cast(op->getAttr("stride")).getInt(); - int64_t group = cast(op->getAttr("group")).getInt(); int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; if (op.getSources().size() != 1 || op.getSources()[0] != int64_t(op.getDestination())) return op.emitOpError("relu_i32 updates its one source bank in place"); - if (rows <= 0 || rows % 16 || rows > stride || depth % stride || - group < 0 || group >= 1024) - return op.emitOpError("relu_i32 rows, stride or group exceed the ISA"); - // The bank1 field selects the physical bank; bit 63 marks the in-place - // update. + if (rows <= 0 || rows > depth || depth % 16) + return op.emitOpError("relu_i32 rows exceed the bank"); + // Group 0 of the bank in the bank1 field; bit 63 marks the in-place + // update. rs2 is the segment stride. uint64_t rs1 = mlir::buddy::bankFields(op.getSources(), op.getDestination(), - false, rows) | - mlir::buddy::field(group, 10, 19) | (1ULL << 63); + false, (rows + 15) / 16 * 16) | + (1ULL << 63); mlir::buddy::emitInstruction( rewriter, op.getLoc(), buckyball_target::getBuckyballFunct7("RELU"), mlir::buddy::constantI64(rewriter, op.getLoc(), rs1), - mlir::buddy::constantI64(rewriter, op.getLoc(), stride)); + mlir::buddy::constantI64(rewriter, op.getLoc(), depth)); rewriter.eraseOp(op); return success(); } diff --git a/examples/components/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir b/examples/components/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir index bccae13c8..8626513d0 100644 --- a/examples/components/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir +++ b/examples/components/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir @@ -19,11 +19,11 @@ func.func @main() -> i8 { memref.store %signed, %input[%row, %lane] : memref<16x4xi32> } } - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mvin %input <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> - buckyball.custom "relu_i32" <{sources = array, destination = 0 : i64}> {rows = 16 : i64, stride = 16 : i64, group = 0 : i64} - buckyball.mvout %output <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + buckyball.custom "relu_i32" <{sources = array, destination = 1 : i64}> {rows = 16 : i64} + buckyball.mvout %output <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> func.call @check_result(%output) : (memref<16x4xi32>) -> () memref.dealloc %input : memref<16x4xi32> memref.dealloc %output : memref<16x4xi32> diff --git a/examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in b/examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in index e03f00def..a37f6f0fc 100644 --- a/examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in +++ b/examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in @@ -29,7 +29,7 @@ func.func @main() -> i8 { %loaded = buckyball.bank_mvin %input, %bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> %result = buckyball.bank_custom "relu_i32" (%loaded) -> %loaded - {rows = @BB_BANK_LINES@ : i64, stride = @BB_BANK_LINES@ : i64, group = 0 : i64} + {rows = @BB_BANK_LINES@ : i64} %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> buckyball.bank_release %stored diff --git a/examples/components/balls/smatmul/arch/SMatMulUnit.scala b/examples/components/balls/smatmul/arch/SMatMulUnit.scala index 89aa66478..5bc25cd34 100644 --- a/examples/components/balls/smatmul/arch/SMatMulUnit.scala +++ b/examples/components/balls/smatmul/arch/SMatMulUnit.scala @@ -53,6 +53,7 @@ class SMatMulUnit( "SMatMulBall bank depth must be divisible by four" ) require(maxOutputRows >= rows, "SMatMulBall accumulator must hold at least one core") + require(isPow2(memParam.bankEntries), "SMatMulBall requires power-of-two bank depth") require((osFunct >> 4) == 4, "SMATMUL_OS must encode two reads and one write") require((biasFunct >> 4) == 1, "SMATMUL_BIAS must encode one read") @@ -118,6 +119,9 @@ class SMatMulUnit( val bBank = RegInit(0.U(bankIdWidth.W)) val cBank = RegInit(0.U(bankIdWidth.W)) val bases = Reg(Vec(3, UInt(memParam.groupIdWidth.W))) + // A and B group g hold reduction block g; C rows continue across groups. + val kGroups = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) + val kGroup = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) val cBase = RegInit(0.U(addressWidth.W)) val chainRows = RegInit(0.U(12.W)) val chainCBank = RegInit(0.U(bankIdWidth.W)) @@ -183,7 +187,7 @@ class SMatMulUnit( for (port <- 0 until mapping.inBW) { io.bankRead(port).rob_id := robId io.bankRead(port).bank_id := Mux(port.U === 0.U, aBank, bBank) - io.bankRead(port).group_id := bases(port) + io.bankRead(port).group_id := bases(port) + kGroup io.bankRead(port).io.req.valid := false.B io.bankRead(port).io.req.bits.addr := 0.U io.bankRead(port).io.resp.ready := false.B @@ -207,7 +211,7 @@ class SMatMulUnit( val cWords = cRowData.asTypeOf(Vec(resultWords, UInt(128.W))) io.bankWrite(0).rob_id := robId io.bankWrite(0).bank_id := cBank - io.bankWrite(0).group_id := bases(2) + io.bankWrite(0).group_id := bases(2) + (cLine >> addressWidth) io.bankWrite(0).io.req.valid := state === writeResult io.bankWrite(0).io.req.bits.addr := cLine(addressWidth - 1, 0) io.bankWrite(0).io.req.bits.data := cWords(outputWord) @@ -240,6 +244,8 @@ class SMatMulUnit( robId := io.cmdReq.bits.rob_id biasCommand := isBias + kGroups := command.op1_col + kGroup := 0.U when(isBias) { assert(command.iter === 4.U, "SMATMUL_BIAS iter must be four rows") @@ -263,9 +269,11 @@ class SMatMulUnit( "SMATMUL_OS requires distinct A, B, and C banks" ) assert( - command.op1_col === 1.U && command.op2_col === 1.U && command.wr_col === 1.U, - "SMATMUL_OS requires one physical bank per operand" + command.op1_col =/= 0.U && command.op1_col === command.op2_col && + command.op1_col <= memParam.bankNum.U && command.wr_col =/= 0.U, + "SMATMUL_OS A and B must have the same reduction groups" ) + assert(rows <= accDepth.U, "SMATMUL_OS M exceeds the accumulator") assert( rows === 1.U || (rows =/= 0.U && rows(2, 0) === 0.U), "SMATMUL_OS M must be one or a positive multiple of 8" @@ -289,8 +297,8 @@ class SMatMulUnit( command.rs2( 31, 26 - ) +& rows * resultWords.U <= memParam.bankEntries.U, - "SMATMUL_OS C footprint exceeds bank depth" + ) +& rows * resultWords.U <= command.wr_col * memParam.bankEntries.U, + "SMATMUL_OS C footprint exceeds its groups" ) when(first) { assert(!chainLive, "SMATMUL_OS first block cannot start while a chain is live") @@ -404,8 +412,17 @@ class SMatMulUnit( bRowsRequested := 0.U bRowsStored := 0.U state := loadTile + }.elsewhen(kGroup + 1.U < kGroups) { + kGroup := kGroup + 1.U + reductionTile := 0.U + aRowsRequested := 0.U + aRowsStored := 0.U + bRowsRequested := 0.U + bRowsStored := 0.U + state := loadTile }.elsewhen(outputTile + 1.U < outputTileCount) { outputTile := outputTile + 1.U + kGroup := 0.U reductionTile := 0.U aRowsRequested := 0.U aRowsStored := 0.U diff --git a/examples/components/balls/smatmul/emu/src/smatmul.rs b/examples/components/balls/smatmul/emu/src/smatmul.rs index 5381f2257..dadb13ad0 100644 --- a/examples/components/balls/smatmul/emu/src/smatmul.rs +++ b/examples/components/balls/smatmul/emu/src/smatmul.rs @@ -1,5 +1,5 @@ use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::instruction::ExecContext; use std::collections::HashMap; @@ -87,15 +87,21 @@ pub(crate) fn exec_smatmul(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { { panic!("smatmul: bank not allocated"); } - if ctx.config(a_bank).cols != 1 || ctx.config(b_bank).cols != 1 || ctx.config(c_bank).cols != 1 - { - panic!("smatmul: A, B, and C banks must each have one column"); + // A and B group g hold reduction block g (K = iter per group); C rows + // continue across groups: line L lives in group L / bank_lines(). + let k_groups = ctx.config(a_bank).cols as usize; + let c_groups = ctx.config(c_bank).cols as usize; + if k_groups == 0 || ctx.config(b_bank).cols as usize != k_groups || c_groups == 0 { + panic!("smatmul: A and B must have the same reduction groups"); + } + if rows > (bank_lines() / 4).max(32) { + panic!("smatmul: M exceeds the accumulator"); } let a_rows = if rows == 1 { k / k_tile } else { rows * k / k_tile }; let b_rows = k; let c_rows = rows * 4; - if a_rows > bank_lines() || b_rows > bank_lines() || output_base + c_rows > bank_lines() { + if a_rows > bank_lines() || b_rows > bank_lines() || output_base + c_rows > c_groups * bank_lines() { panic!("smatmul: bank footprint exceeds bank depth"); } @@ -136,8 +142,6 @@ pub(crate) fn exec_smatmul(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { panic!("smatmul: continuation changed output shape or destination"); } - let pa = pbank(ctx, a_bank); - let pb = pbank(ctx, b_bank); let k_tiles = k / k_tile; let row_bytes = bank_row_bytes(); @@ -147,6 +151,9 @@ pub(crate) fn exec_smatmul(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { // reads one A byte and one complete B row, then updates all 16 lanes. // Wrapping arithmetic is intentionally retained to match the accumulator // semantics of the RTL and the previous emulator implementation. + for k_group in 0..k_groups { + let pa = pbank_group(ctx, a_bank, k_group as u64); + let pb = pbank_group(ctx, b_bank, k_group as u64); for inner in 0..k { let b_base = ((inner / k_tile) * k_tile + inner % k_tile) * row_bytes; let b_row = &ctx.banks[pb][b_base..b_base + cols]; @@ -168,12 +175,14 @@ pub(crate) fn exec_smatmul(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { } } } + } if last { - let pc = pbank(ctx, c_bank); for row in 0..rows { for col in 0..cols { - let offset = (output_base + row * 4 + col / 4) * bank_row_bytes() + col % 4 * 4; + let line = output_base + row * 4 + col / 4; + let pc = pbank_group(ctx, c_bank, (line / bank_lines()) as u64); + let offset = line % bank_lines() * bank_row_bytes() + col % 4 * 4; ctx.banks[pc][offset..offset + 4] .copy_from_slice(&chain.accumulators[row * cols + col].to_le_bytes()); } @@ -212,7 +221,6 @@ pub(crate) fn latency(xs1: u64, xs2: u64, instance: u32) -> u64 { || cols != n_tile as u64 || k == 0 || k % k_tile as u64 != 0 - || output_base + rows * 4 > bank_lines() as u64 { panic!("smatmul: illegal matrix encoding"); } diff --git a/examples/components/balls/smatmul/verify/casegen/src/casegen.rs b/examples/components/balls/smatmul/verify/casegen/src/casegen.rs index 9d8891b8c..f3e5bb4cf 100644 --- a/examples/components/balls/smatmul/verify/casegen/src/casegen.rs +++ b/examples/components/balls/smatmul/verify/casegen/src/casegen.rs @@ -7,7 +7,6 @@ pub const C_BANK: u32 = 5; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct MatrixCmd { pub kind: u32, - pub bid: u32, pub rob_id: u32, pub op1_words: u32, pub op2_words: u32, @@ -15,6 +14,9 @@ pub struct MatrixCmd { pub rs1_hi: u32, pub rs2_lo: u32, pub rs2_hi: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -26,20 +28,27 @@ pub struct MatrixCase { pub writes: Vec, } -pub fn gen_case(seed: u32, index: u32, bid: u32, tile_rows: usize) -> MatrixCase { +// Random cases also place every operand at a nonzero group base of its bank. +pub fn gen_case(seed: u32, index: u32, tile_rows: usize) -> MatrixCase { match index { - 0 => build_case(seed, bid, 16, 1, tile_rows), - 1 => build_case(seed, bid, 16, 2, tile_rows), - 2 => build_case(seed ^ index, bid, 16, 1, tile_rows), + 0 => build_case(seed, 16, 1, tile_rows, false), + 1 => build_case(seed, 16, 2, tile_rows, false), + 2 => build_case(seed ^ index, 16, 1, tile_rows, false), _ => { let mut rng = Rng::new(seed, index); let blocks = 1 + (rng.next() % 2) as usize; - build_case(rng.next(), bid, 16, blocks, tile_rows) + build_case(rng.next(), 16, blocks, tile_rows, true) } } } -fn build_case(seed: u32, bid: u32, rows: usize, blocks: usize, tile_rows: usize) -> MatrixCase { +fn build_case( + seed: u32, + rows: usize, + blocks: usize, + tile_rows: usize, + grouped: bool, +) -> MatrixCase { assert!(matches!(rows, 16 | 32 | 64)); assert!(matches!(blocks, 1 | 2)); assert!(rows * model::RESULT_WORDS <= model::BANK_DEPTH); @@ -55,11 +64,13 @@ fn build_case(seed: u32, bid: u32, rows: usize, blocks: usize, tile_rows: usize) let mut a = Vec::with_capacity(blocks); let mut b = Vec::with_capacity(blocks); let mut result = vec![bias_values.clone(); rows]; + let group = |rng: &mut Rng| if grouped { 1 + rng.next() % 3 } else { 0 }; let mut commands = Vec::with_capacity(blocks + 1); + let bias_group = group(&mut rng); commands.push(MatrixCmd { kind: 0, - bid, rob_id: 0, + op1_group: bias_group, op1_words: 4, rs1_lo: model::encode_rs1(BIAS_BANK, 0, 0, 4) as u32, rs1_hi: (model::encode_rs1(BIAS_BANK, 0, 0, 4) >> 32) as u32, @@ -89,10 +100,15 @@ fn build_case(seed: u32, bid: u32, rows: usize, blocks: usize, tile_rows: usize) } let rs1 = model::encode_rs1(a_bank, b_bank, C_BANK, model::CORE as u64); let rs2 = model::encode_rs2(rows, block == 0, block + 1 == blocks); + let op1_group = group(&mut rng); + let op2_group = group(&mut rng); + let wr_group = group(&mut rng); commands.push(MatrixCmd { kind: 1, - bid, rob_id: (block + 1) as u32, + op1_group, + op2_group, + wr_group, op1_words: rows as u32, op2_words: (model::CORE * rows.div_ceil(tile_rows)) as u32, rs1_lo: rs1 as u32, diff --git a/examples/components/balls/smatmul/verify/casegen/src/lib.rs b/examples/components/balls/smatmul/verify/casegen/src/lib.rs index db9ec2e44..3751bd564 100644 --- a/examples/components/balls/smatmul/verify/casegen/src/lib.rs +++ b/examples/components/balls/smatmul/verify/casegen/src/lib.rs @@ -10,9 +10,9 @@ thread_local! { } #[no_mangle] -pub extern "C" fn smatmul_case_load(seed: u32, index: u32, bid: u32, tile_rows: u32) { +pub extern "C" fn smatmul_case_load(seed: u32, index: u32, tile_rows: u32) { CURRENT.with(|current| { - *current.borrow_mut() = Some(casegen::gen_case(seed, index, bid, tile_rows as usize)) + *current.borrow_mut() = Some(casegen::gen_case(seed, index, tile_rows as usize)) }); } diff --git a/examples/components/balls/smatmul/verify/filelists/smatmul_ball.f b/examples/components/balls/smatmul/verify/filelists/smatmul_ball.f index e053cb0a9..455c8d17a 100644 --- a/examples/components/balls/smatmul/verify/filelists/smatmul_ball.f +++ b/examples/components/balls/smatmul/verify/filelists/smatmul_ball.f @@ -1,4 +1,5 @@ +lint=TFIPC-L ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -14,6 +15,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv diff --git a/examples/components/balls/smatmul/verify/src/common/smatmul_defs.svh b/examples/components/balls/smatmul/verify/src/common/smatmul_defs.svh index 3b7badf38..516798c5e 100644 --- a/examples/components/balls/smatmul/verify/src/common/smatmul_defs.svh +++ b/examples/components/balls/smatmul/verify/src/common/smatmul_defs.svh @@ -1,6 +1,5 @@ typedef struct { int unsigned kind; - int unsigned bid; int unsigned rob_id; int unsigned op1_words; int unsigned op2_words; @@ -8,6 +7,9 @@ typedef struct { int unsigned rs1_hi; int unsigned rs2_lo; int unsigned rs2_hi; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; } matrix_cmd_dpi_t; `ifndef SMATMUL_TILE_ROWS @@ -17,7 +19,6 @@ typedef struct { import "DPI-C" function void smatmul_case_load( input int unsigned seed, input int unsigned index, - input int unsigned bid, input int unsigned tile_rows ); import "DPI-C" function int unsigned smatmul_case_num_commands(); @@ -50,11 +51,3 @@ localparam int SMATMUL_OS_THREAD_FUNCT7 = `SMATMUL_OS_FUNCT7; localparam int SMATMUL_BIAS_THREAD_FUNCT7 = `SMATMUL_BIAS_FUNCT7; localparam int MATRIX_TIMEOUT_CYCLES = 200000; localparam int MATRIX_SEED = 32'hCAFE_BABE; - -function automatic int unsigned matrix_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/components/balls/smatmul/verify/src/common/smatmul_items.svh b/examples/components/balls/smatmul/verify/src/common/smatmul_items.svh index 6d46030e1..c7d21d1d1 100644 --- a/examples/components/balls/smatmul/verify/src/common/smatmul_items.svh +++ b/examples/components/balls/smatmul/verify/src/common/smatmul_items.svh @@ -6,6 +6,7 @@ class smatmul_cmd_item extends bb_blink_cmd_item; bit [31:0] command_count; bit [31:0] op1_words; bit [31:0] op2_words; + bit [63:0] rs1; function new(string name = "smatmul_cmd_item"); super.new(name); @@ -21,43 +22,35 @@ class smatmul_cmd_item extends bb_blink_cmd_item; kind = cmd.kind; command_index = index; command_count = smatmul_case_num_commands(); - bid = cmd.bid[4:0]; - rob_id = cmd.rob_id[3:0]; op1_words = cmd.op1_words; op2_words = cmd.op2_words; rs1 = {cmd.rs1_hi, cmd.rs1_lo}; + + // BB_BANK0/1/2 and BB_ITER fields of rs1; banks are not renamed at Ball level. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; rs2 = {cmd.rs2_hi, cmd.rs2_lo}; - op1_bank = rs1[4:0]; - op2_bank = rs1[14:10]; - wr_bank = rs1[24:20]; iter = rs1[63:30]; - special = 64'd0; - meta_bank = 5'd0; - is_sub = 1'b0; - sub_rob_id = 8'd0; + op1_bank = rs1[9:0]; + op2_bank = rs1[19:10]; + wr_bank = rs1[29:20]; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + wr_read_bank = '0; + wr_arch_bank = wr_bank; if (kind == 0) begin - funct7 = SMATMUL_BIAS_THREAD_FUNCT7[6:0]; - op1_en = 1'b1; - op2_en = 1'b0; - wr_spad_en = 1'b0; - op1_from_spad = 1'b1; - op2_from_spad = 1'b0; - op1_col = 5'd1; - op2_col = 5'd0; - wr_col = 5'd0; + funct = SMATMUL_BIAS_THREAD_FUNCT7[6:0]; + op1_col = 1; + op2_col = 0; + wr_col = 0; if (rs1[29:10] != 0 || iter != 4 || rs2 != 0 || op1_words != 4) `uvm_fatal("CASE", "invalid bias command") end else begin - funct7 = SMATMUL_OS_THREAD_FUNCT7[6:0]; - op1_en = 1'b1; - op2_en = 1'b1; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b1; - op1_col = 5'd1; - op2_col = 5'd1; - wr_col = 5'd1; + funct = SMATMUL_OS_THREAD_FUNCT7[6:0]; + op1_col = 1; + op2_col = 1; + wr_col = 1; if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) `uvm_fatal("CASE", "SMATMUL_OS banks must be distinct") if (rs2[63:26] != 0 || rs2[23:12] != 16 || iter != 16) @@ -74,5 +67,6 @@ class smatmul_cmd_item extends bb_blink_cmd_item; command_count = rhs_.command_count; op1_words = rhs_.op1_words; op2_words = rhs_.op2_words; + rs1 = rhs_.rs1; endfunction endclass diff --git a/examples/components/balls/smatmul/verify/src/cov/smatmul_cov.svh b/examples/components/balls/smatmul/verify/src/cov/smatmul_cov.svh index b2e6887df..764872cc7 100644 --- a/examples/components/balls/smatmul/verify/src/cov/smatmul_cov.svh +++ b/examples/components/balls/smatmul/verify/src/cov/smatmul_cov.svh @@ -18,7 +18,7 @@ class smatmul_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - is_bias = (item.funct7 == SMATMUL_BIAS_THREAD_FUNCT7[6:0]); + is_bias = (item.funct == SMATMUL_BIAS_THREAD_FUNCT7[6:0]); is_first = is_bias ? 0 : item.rs2[24]; cmd_count++; cmd_cg.sample(); diff --git a/examples/components/balls/smatmul/verify/src/env/smatmul_scoreboard.svh b/examples/components/balls/smatmul/verify/src/env/smatmul_scoreboard.svh index 063423230..4262d283a 100644 --- a/examples/components/balls/smatmul/verify/src/env/smatmul_scoreboard.svh +++ b/examples/components/balls/smatmul/verify/src/env/smatmul_scoreboard.svh @@ -48,19 +48,19 @@ class smatmul_scoreboard extends uvm_scoreboard; for (int i = 0; i < smatmul_case_bias_words(); i++) begin lo = smatmul_case_bias_word_lo(i); hi = smatmul_case_bias_word_hi(i); - mem_model.load_word(0, i, {hi, lo}); + mem_model.load_word_g(got.op1_bank, got.op1_group, i, {hi, lo}); end for (int block = 0; block < expected_commands - 1; block++) begin smatmul_case_cmd(block + 1, data_cmd); for (int i = 0; i < smatmul_case_a_words(block); i++) begin lo = smatmul_case_a_word_lo(block, i); hi = smatmul_case_a_word_hi(block, i); - mem_model.load_word(data_cmd.rs1_lo[9:0], i, {hi, lo}); + mem_model.load_word_g(data_cmd.rs1_lo[9:0], data_cmd.op1_group, i, {hi, lo}); end for (int i = 0; i < smatmul_case_b_words(block); i++) begin lo = smatmul_case_b_word_lo(block, i); hi = smatmul_case_b_word_hi(block, i); - mem_model.load_word(data_cmd.rs1_lo[19:10], i, {hi, lo}); + mem_model.load_word_g(data_cmd.rs1_lo[19:10], data_cmd.op2_group, i, {hi, lo}); end end mem_model.arm(); @@ -84,16 +84,11 @@ class smatmul_scoreboard extends uvm_scoreboard; endfunction function void write_cmd(bb_blink_cmd_item item); - smatmul_cmd_item exp; - exp = current_stim("CMD"); - if (item.bid !== exp.bid || item.funct7 !== exp.funct7 || - item.op1_bank !== exp.op1_bank || item.op2_bank !== exp.op2_bank || - item.wr_bank !== exp.wr_bank || item.rs1 !== exp.rs1 || - item.rs2 !== exp.rs2 || item.iter !== exp.iter || - item.op1_col !== exp.op1_col || item.op2_col !== exp.op2_col || - item.wr_col !== exp.wr_col || item.op1_en !== exp.op1_en || - item.op2_en !== exp.op2_en || item.wr_spad_en !== exp.wr_spad_en) - `uvm_fatal("CMD", "command differs from generated contract") + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(current_stim("CMD")); + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) cmd_count++; endfunction @@ -101,13 +96,16 @@ class smatmul_scoreboard extends uvm_scoreboard; smatmul_cmd_item stim; stim = current_stim("READ"); if (stim.kind == 0) begin - if (item.port != 0 || item.bank_id != stim.op1_bank || item.addr >= 4) + if (item.port != 0 || item.bank_id != stim.op1_bank || item.group_id != stim.op1_group || + item.addr >= 4) `uvm_fatal("READ", "invalid bias read") end else if (item.port == 0) begin - if (item.bank_id != stim.op1_bank || item.addr >= stim.op1_words) + if (item.bank_id != stim.op1_bank || item.group_id != stim.op1_group || + item.addr >= stim.op1_words) `uvm_fatal("READ", "A read outside command footprint") end else if (item.port == 1) begin - if (item.bank_id != stim.op2_bank || item.addr >= stim.op2_words) + if (item.bank_id != stim.op2_bank || item.group_id != stim.op2_group || + item.addr >= stim.op2_words) `uvm_fatal("READ", "B read outside command footprint") end else begin `uvm_fatal("READ", "unexpected read port") @@ -121,7 +119,7 @@ class smatmul_scoreboard extends uvm_scoreboard; stim = current_stim("WRITE"); if (stim.kind != 1 || stim.rs2[25] != 1) `uvm_fatal("WRITE", "write before the last accumulation block") - if (item.port != 0 || item.group_id != 0 || item.bank_id != stim.wr_bank || + if (item.port != 0 || item.group_id != stim.wr_group || item.bank_id != stim.wr_bank || item.mask != 16'hffff || item.rob_id != stim.rob_id) `uvm_fatal("WRITE", "write metadata violates SMatMul contract") for (int i = 0; i < expected_writes; i++) begin @@ -138,8 +136,7 @@ class smatmul_scoreboard extends uvm_scoreboard; function void write_resp(bb_blink_resp_item item); smatmul_cmd_item stim; stim = current_stim("RESP"); - if (item.rob_id !== stim.rob_id || item.is_sub !== 1'b0) - `uvm_fatal("RESP", "response metadata mismatch") + if (item.rob_id !== stim.rob_id) `uvm_fatal("RESP", "response metadata mismatch") void'(stim_q.pop_front()); resp_count++; endfunction diff --git a/examples/components/balls/smatmul/verify/src/seq/smatmul_sequences.svh b/examples/components/balls/smatmul/verify/src/seq/smatmul_sequences.svh index 7d4c067e2..107db086a 100644 --- a/examples/components/balls/smatmul/verify/src/seq/smatmul_sequences.svh +++ b/examples/components/balls/smatmul/verify/src/seq/smatmul_sequences.svh @@ -3,7 +3,6 @@ class smatmul_chain_seq extends uvm_sequence #(bb_blink_cmd_item); int unsigned case_index; int unsigned seed = MATRIX_SEED; - int unsigned bid; function new(string name = "smatmul_chain_seq"); super.new(name); @@ -11,7 +10,7 @@ class smatmul_chain_seq extends uvm_sequence #(bb_blink_cmd_item); task body(); int unsigned count; - smatmul_case_load(seed, case_index, bid, `SMATMUL_TILE_ROWS); + smatmul_case_load(seed, case_index, `SMATMUL_TILE_ROWS); count = smatmul_case_num_commands(); if (count < 2 || count > 3) `uvm_fatal("CASE", "command count must be 2 or 3") for (int unsigned index = 0; index < count; index++) begin diff --git a/examples/components/balls/smatmul/verify/src/tb_top.sv b/examples/components/balls/smatmul/verify/src/tb_top.sv index d29dae93e..e478bf3b5 100644 --- a/examples/components/balls/smatmul/verify/src/tb_top.sv +++ b/examples/components/balls/smatmul/verify/src/tb_top.sv @@ -7,87 +7,75 @@ module tb_top; SMatMulBall dut ( .clock(intf.clock), .reset(intf.reset), - .io_status_idle(), - .io_status_running(), - .io_bankAccessReady(1'b1), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_ball_id(), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankRead_1_bank_id(intf.bank_read_bank_id[1]), - .io_bankRead_1_rob_id(intf.bank_read_rob_id[1]), - .io_bankRead_1_ball_id(), - .io_bankRead_1_group_id(intf.bank_read_group_id[1]), - .io_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), - .io_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), - .io_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), - .io_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), - .io_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), - .io_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_ball_id(), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready) + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) ); + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + initial begin intf.clock = 1'b0; intf.reset = 1'b1; diff --git a/examples/components/balls/smatmul/verify/src/tests/smatmul_test.svh b/examples/components/balls/smatmul/verify/src/tests/smatmul_test.svh index 862de4e18..c293dbdf1 100644 --- a/examples/components/balls/smatmul/verify/src/tests/smatmul_test.svh +++ b/examples/components/balls/smatmul/verify/src/tests/smatmul_test.svh @@ -17,20 +17,18 @@ class smatmul_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid = matrix_require_bid(); phase.raise_objection(this); - for (int unsigned index = 0; index < 11; index++) run_case(index, bid); + for (int unsigned index = 0; index < 11; index++) run_case(index); phase.drop_objection(this); endtask - task run_case(int unsigned index, int unsigned bid); + task run_case(int unsigned index); smatmul_chain_seq seq; int cycles = 0; apply_reset(); env.scb.reset_counters(); seq = smatmul_chain_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); while (!env.scb.done()) begin @(posedge vif.clock); diff --git a/examples/components/balls/smatmul/workloads/isa/smatmul.h b/examples/components/balls/smatmul/workloads/isa/smatmul.h index 67b2334d7..621a7af38 100644 --- a/examples/components/balls/smatmul/workloads/isa/smatmul.h +++ b/examples/components/balls/smatmul/workloads/isa/smatmul.h @@ -3,6 +3,12 @@ #include #include + +// SMATMUL_OS: A and B have the same number of groups; group g holds reduction +// block g, so the instruction accumulates over K = k * groups with k per group +// (A[:, g*k : (g+1)*k] and B[:, g*k : (g+1)*k] in the single-group layouts). +// C rows continue across groups: line L = output_base + row * 4 + word lives +// in group L / BANK_LINES, line L % BANK_LINES. The bias bank has one group. #define BB_SMATMUL_CFG(rows, cols, first, last, output_base) \ (FIELD((rows), 0, 11) | FIELD((cols), 12, 23) | FIELD((first), 24, 24) | \ FIELD((last), 25, 25) | FIELD((output_base), 26, 31)) diff --git a/examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir b/examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir index bbdf49461..6966ff1cf 100644 --- a/examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir +++ b/examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir @@ -40,26 +40,26 @@ func.func @main() -> i8 { memref.store %bias_i32, %bias[%bias_row, %bias_lane] : memref<4x4xi32> } - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 4 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mvin %input0 <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - buckyball.mvin %weight0 <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - buckyball.mvin %input1 <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - buckyball.mvin %weight1 <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mset <{bank = 5 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input0 <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %weight0 <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %input1 <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %weight1 <{bank = 4 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> // The bias occupies the first four rows of C. - buckyball.mvin %bias <{bank = 4 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> - buckyball.custom "matmul_i8_block" <{sources = array, destination = 4 : i64, read_destination}> {rows = 16 : i64, k = 16 : i64, first = true, last = false} - buckyball.custom "matmul_i8_block" <{sources = array, destination = 4 : i64, read_destination}> {rows = 16 : i64, k = 16 : i64, first = false, last = true} - buckyball.mvout %output <{bank = 4 : i64, depth = 64 : i64, stride = 1 : i64}> : memref<64x4xi32> + buckyball.mvin %bias <{bank = 5 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> + buckyball.custom "matmul_i8_block" <{sources = array, destination = 5 : i64, read_destination}> {rows = 16 : i64, k = 16 : i64, first = true, last = false} + buckyball.custom "matmul_i8_block" <{sources = array, destination = 5 : i64, read_destination}> {rows = 16 : i64, k = 16 : i64, first = false, last = true} + buckyball.mvout %output <{bank = 5 : i64, depth = 64 : i64, stride = 1 : i64}> : memref<64x4xi32> func.call @check_result(%output) : (memref<64x4xi32>) -> () - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 4 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 5 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> memref.dealloc %input0 : memref<16x16xi8> memref.dealloc %input1 : memref<16x16xi8> memref.dealloc %weight0 : memref<16x16xi8> diff --git a/examples/components/balls/toint8/arch/ToInt8Ball.scala b/examples/components/balls/toint8/arch/ToInt8Ball.scala index 0e570d6f5..880597eb5 100644 --- a/examples/components/balls/toint8/arch/ToInt8Ball.scala +++ b/examples/components/balls/toint8/arch/ToInt8Ball.scala @@ -56,6 +56,10 @@ class ToInt8Ball( val scaleBank = RegInit(0.U(fp.bank_id_len.W)) val outputBank = RegInit(0.U(fp.bank_id_len.W)) val bases = Reg(Vec(3, UInt(mp.groupIdWidth.W))) + // Every group of a virtual bank is one 16-channel block with the same rows. + val groups = RegInit(0.U(log2Up(mp.bankNum + 1).W)) + val group = RegInit(0.U(log2Up(mp.bankNum + 1).W)) + val outputBase = RegInit(0.U(log2Ceil(mp.bankEntries).W)) val iterReg = RegInit(0.U(fp.iter_len.W)) val inputBaseReg = RegInit(0.U(log2Ceil(mp.bankEntries).W)) // Relative row within the command. inputBaseReg is the absolute bank row. @@ -78,14 +82,14 @@ class ToInt8Ball( def positiveFinite(value: UInt): Bool = !value(31) && value(30, 23) =/= 255.U && value(30, 0) =/= 0.U - val intToFloat = Instantiate(new INToRecFN(32, 8, 24)) + val intToFloat = Module(new INToRecFN(32, 8, 24)) // The unpipelined recoded-float multiplier was the critical Buckyball path // (about 8.5 ns before clock uncertainty). FMA mode with c=0 preserves the // multiplication result while splitting the datapath into three stages. val multiply = Instantiate(new Fma(3, 8, 24)) - val floatToInt = Instantiate(new RecFNToIN(8, 24, 8)) + val floatToInt = Module(new RecFNToIN(8, 24, 8)) val input = (inputData >> (lane << 5))(31, 0) val scale = Mux(i32Mode, scales(Cat(relativeRow(1, 0), lane)), tensorScale) val packIssue = state === pack @@ -142,14 +146,14 @@ class ToInt8Ball( blink.bankRead(0).rob_id := robIdReg blink.bankRead(0).bank_id := Mux(state === scaleReq || state === scaleResp, scaleBank, inputBank) - blink.bankRead(0).group_id := Mux(state === scaleReq || state === scaleResp, bases(1), bases(0)) + blink.bankRead(0).group_id := Mux(state === scaleReq || state === scaleResp, bases(1), bases(0)) + group blink.bankRead(0).io.req.valid := false.B blink.bankRead(0).io.req.bits.addr := 0.U blink.bankRead(0).io.resp.ready := false.B blink.bankWrite(0).rob_id := robIdReg blink.bankWrite(0).bank_id := outputBank - blink.bankWrite(0).group_id := bases(2) + blink.bankWrite(0).group_id := bases(2) + group blink.bankWrite(0).io.req.valid := false.B blink.bankWrite(0).io.req.bits.addr := outputRow blink.bankWrite(0).io.req.bits.data := writeData @@ -171,7 +175,10 @@ class ToInt8Ball( cmd.iter <= mp.bankEntries.U, "ToInt8Ball input exceeds bank depth" ) - assert(cmd.op1_col === 1.U && cmd.wr_col === 1.U, "ToInt8Ball input and output must each occupy one bank") + assert( + cmd.op1_col =/= 0.U && cmd.op1_col === cmd.wr_col && cmd.op1_col <= mp.bankNum.U, + "ToInt8Ball input and output must have the same channel groups" + ) assert( cmd.op1_bank =/= cmd.wr_bank, "ToInt8Ball input and output banks must differ" @@ -180,7 +187,7 @@ class ToInt8Ball( assert(cmd.rs2(63, 32) === 0.U, "QUANT_F32_TO_I8 reserves rs2[63:32]") assert(positiveFinite(cmd.rs2(31, 0)), "QUANT_F32_TO_I8 scale must be finite and positive") }.otherwise { - assert(cmd.op2_col === 1.U, "QUANT_I32_TO_I8 scale must occupy one bank") + assert(cmd.op2_col === cmd.op1_col, "QUANT_I32_TO_I8 scale must have one group per channel group") assert( cmd.op1_bank =/= cmd.op2_bank, "QUANT_I32_TO_I8 input and scale banks must differ" @@ -214,6 +221,9 @@ class ToInt8Ball( scaleBank := cmd.op2_bank outputBank := cmd.wr_bank bases := VecInit(cmd.op1_group, cmd.op2_group, cmd.wr_group) + groups := cmd.op1_col + group := 0.U + outputBase := Mux(isI32, cmd.rs2(7, 1), 0.U) iterReg := cmd.iter inputBaseReg := Mux(isI32, cmd.rs2(34, 29), 0.U) relativeRow := 0.U @@ -297,7 +307,16 @@ class ToInt8Ball( blink.bankWrite(0).io.resp.ready := true.B when(blink.bankWrite(0).io.resp.fire) { when(relativeRow === iterReg - 1.U) { - state := complete + when(group + 1.U === groups) { + state := complete + }.otherwise { + group := group + 1.U + relativeRow := 0.U + outputRow := outputBase + outputColumn := 0.U + scaleRow := 0.U + state := Mux(i32Mode, scaleReq, inputReq) + } }.otherwise { relativeRow := relativeRow + 1.U when(outputColumn === outputWidth - 1.U) { diff --git a/examples/components/balls/toint8/emu/src/quant.rs b/examples/components/balls/toint8/emu/src/quant.rs index 787628299..0e099595b 100644 --- a/examples/components/balls/toint8/emu/src/quant.rs +++ b/examples/components/balls/toint8/emu/src/quant.rs @@ -1,5 +1,5 @@ use bebop_bemu::thread::banks::bank_lines; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::instruction::ExecContext; pub(crate) fn exec_f32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { @@ -16,8 +16,9 @@ pub(crate) fn exec_f32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { if !ctx.config(src).allocated || !ctx.config(dst).allocated { panic!("quant_f32_to_i8: bank not allocated"); } - if ctx.config(src).cols != 1 || ctx.config(dst).cols != 1 { - panic!("quant_f32_to_i8: banks must have one column"); + let groups = ctx.config(src).cols; + if groups == 0 || ctx.config(dst).cols != groups { + panic!("quant_f32_to_i8: input and output must have the same channel groups"); } if iter == 0 || iter % 4 != 0 || iter > bank_lines() { panic!("quant_f32_to_i8: iter must be a positive multiple of four within bank depth"); @@ -30,8 +31,10 @@ pub(crate) fn exec_f32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { panic!("quant_f32_to_i8: scale must be finite and positive"); } - let ps = pbank(ctx, src); - let pd = pbank(ctx, dst); + // Group g of every operand is channel block g; each block uses the same rows. + for channel_group in 0..groups { + let ps = pbank_group(ctx, src, channel_group); + let pd = pbank_group(ctx, dst, channel_group); let (input, output) = ctx.banks.read_write(ps, pd); for output_row in 0..iter / 4 { let mut packed = [0u8; 16]; @@ -48,6 +51,7 @@ pub(crate) fn exec_f32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let base = output_row * 16; output[base..base + 16].copy_from_slice(&packed); } + } 0 } @@ -63,8 +67,9 @@ pub(crate) fn exec_i32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { { panic!("quant_i32_to_i8: bank not allocated"); } - if ctx.config(src).cols != 1 || ctx.config(scale_bank).cols != 1 || ctx.config(dst).cols != 1 { - panic!("quant_i32_to_i8: banks must have one column"); + let groups = ctx.config(src).cols; + if groups == 0 || ctx.config(scale_bank).cols != groups || ctx.config(dst).cols != groups { + panic!("quant_i32_to_i8: input, scale, and output must have the same channel groups"); } if iter == 0 || iter % 4 != 0 || iter > bank_lines() { panic!("quant_i32_to_i8: iter must be a positive multiple of four within bank depth"); @@ -91,7 +96,9 @@ pub(crate) fn exec_i32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { } let relu = xs2 & 1 != 0; - let pscale = pbank(ctx, scale_bank); + // Group g of every operand is channel block g; each block uses the same rows. + for channel_group in 0..groups { + let pscale = pbank_group(ctx, scale_bank, channel_group); let mut scales = [0.0f32; 16]; for group in 0..4 { let base = group * 16; @@ -106,8 +113,8 @@ pub(crate) fn exec_i32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { } } - let ps = pbank(ctx, src); - let pd = pbank(ctx, dst); + let ps = pbank_group(ctx, src, channel_group); + let pd = pbank_group(ctx, dst, channel_group); let (input, output) = ctx.banks.read_write(ps, pd); for output_row in 0..iter / 4 { let mut packed = [0u8; 16]; @@ -126,6 +133,7 @@ pub(crate) fn exec_i32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let base = row * 16; output[base..base + 16].copy_from_slice(&packed); } + } 0 } diff --git a/examples/components/balls/toint8/stack/src/Lowering.cpp b/examples/components/balls/toint8/stack/src/Lowering.cpp index eb41ae2d7..cad9db0fc 100644 --- a/examples/components/balls/toint8/stack/src/Lowering.cpp +++ b/examples/components/balls/toint8/stack/src/Lowering.cpp @@ -4,6 +4,10 @@ // four; the destination receives rows / 4 rows of sixteen i8, // clamp(round_half_even(x * multiplier), -128, 127). Argument 0 is the f32 // multiplier. +// int32_to_int8: source 0 holds `rows` rows of sixteen i32 (four bank rows +// each), source 1 the sixteen f32 scales of the columns (four bank rows); the +// destination receives `rows` rows of sixteen i8, +// clamp(round_half_even(x * scale), -128, 127), with `relu` at least zero. // //===----------------------------------------------------------------------===// @@ -48,20 +52,16 @@ class Int32ToInt8Lowering : public OpRewritePattern { if (op.getKind() != "int32_to_int8") return failure(); auto rows = cast(op->getAttr("rows")).getInt(); - auto base = cast(op->getAttr("wr_base")).getInt(); bool relu = cast(op->getAttr("relu")).getValue(); int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; - if (rows <= 0 || rows > 127 || rows * 4 > depth || base < 0 || base > 127 || - base + rows > depth) - return op.emitOpError( - "INT32-to-INT8 rows or wr_base exceed the ISA range"); - // One row contains sixteen output bytes. A partial update retains the - // destination version; reading an old destination alone does not do that. + if (rows <= 0 || rows > 127 || rows * 4 > depth) + return op.emitOpError("INT32-to-INT8 rows exceed the ISA range"); + // The output rows start at the base of the destination: width one, height + // `rows`, stride one. uint64_t rs1 = mlir::buddy::bankFields(op.getSources(), op.getDestination(), - false, rows * 4) | - (uint64_t(op.getReadDestination()) << 63); - uint64_t rs2 = uint64_t(relu) | (uint64_t(base) << 1) | (1ULL << 8) | - (uint64_t(rows) << 15) | (1ULL << 22); + false, rows * 4); + uint64_t rs2 = + uint64_t(relu) | (1ULL << 8) | (uint64_t(rows) << 15) | (1ULL << 22); mlir::buddy::emitInstruction( rewriter, op.getLoc(), buckyball_target::getBuckyballFunct7("QUANT_I32_TO_I8"), diff --git a/examples/components/balls/toint8/verify/casegen/src/casegen.rs b/examples/components/balls/toint8/verify/casegen/src/casegen.rs index 1b0a05907..8bbbe91ee 100644 --- a/examples/components/balls/toint8/verify/casegen/src/casegen.rs +++ b/examples/components/balls/toint8/verify/casegen/src/casegen.rs @@ -14,17 +14,17 @@ pub const MAX_DST: usize = 4; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ToInt8Cmd { pub kind: u32, - pub bid: u32, pub iter: u32, pub op1_bank: u32, pub op2_bank: u32, pub wr_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, pub op1_col: u32, pub op2_col: u32, pub wr_col: u32, pub rob_id: u32, - pub rs1_lo: u32, - pub rs1_hi: u32, pub rs2_lo: u32, pub rs2_hi: u32, pub input_base: u32, @@ -68,16 +68,16 @@ impl ToInt8Case { } } -pub fn gen_case(index: u32, bid: u32) -> ToInt8Case { +pub fn gen_case(index: u32) -> ToInt8Case { match index { - 0 => f32_signed(bid), - 1 => f32_zero(bid), - 2 => f32_rounding(bid), - 3 => f32_rows(bid), - 4 => i32_scale_rows(bid), - 5 => i32_relu_tile(bid), - 6 => i32_signed(bid), - 7 => i32_offset(bid), + 0 => f32_signed(), + 1 => f32_zero(), + 2 => f32_rounding(), + 3 => f32_rows(), + 4 => i32_scale_rows(), + 5 => i32_relu_tile(), + 6 => i32_signed(), + 7 => i32_offset(), _ => panic!("toint8: unsupported directed case {index}"), } } @@ -115,10 +115,6 @@ fn copy_addrs(src: &[u32]) -> [u32; MAX_DST] { out } -fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u32) -> u64 { - u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (u64::from(iter) << 30) -} - fn encode_i32_rs2( relu: bool, out_base: u32, @@ -153,7 +149,6 @@ pub(crate) fn validate( op2_col: u32, wr_col: u32, iter: u32, - rs1: u64, rs2: u64, ) { if kind != KIND_F32 && kind != KIND_I32 { @@ -177,14 +172,8 @@ pub(crate) fn validate( if op1_bank == wr_bank { panic!("ToInt8Ball input and output banks must differ"); } - if (rs1 & 0x3ff) != u64::from(op1_bank) || ((rs1 >> 20) & 0x3ff) != u64::from(wr_bank) { - panic!("toint8: rs1 bank fields mismatch"); - } - if ((rs1 >> 30) & 0x3_ffff_ffff) != u64::from(iter) { - panic!("toint8: rs1 iter mismatch"); - } if kind == KIND_F32 { - if ((rs1 >> 10) & 0x3ff) != 0 || op2_bank != 0 { + if op2_bank != 0 { panic!("QUANT_F32_TO_I8 reserves input bank 1"); } if rs2 >> 32 != 0 { @@ -197,9 +186,6 @@ pub(crate) fn validate( if op2_bank >= BANK_NUM { panic!("QUANT_I32_TO_I8 scale bank is invalid"); } - if ((rs1 >> 10) & 0x3ff) != u64::from(op2_bank) { - panic!("toint8: rs1 scale bank mismatch"); - } if op2_col != 1 { panic!("QUANT_I32_TO_I8 scale must occupy one bank"); } @@ -299,9 +285,10 @@ fn i32_addrs(iter: u32, out_base: u32, width: u32, stride: u32) -> Vec { .collect() } +// `groups` are the op1/op2/wr base groups. fn finish( kind: u32, - bid: u32, + groups: [u32; 3], iter: u32, op1_bank: u32, op2_bank: u32, @@ -317,9 +304,8 @@ fn finish( ) -> ToInt8Case { let op1_col = 1; let wr_col = 1; - let rs1 = encode_rs1(op1_bank, op2_bank, wr_bank, iter); validate( - kind, op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs1, rs2, + kind, op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs2, ); if src.len() != iter as usize { panic!("toint8: src rows {} != iter {iter}", src.len()); @@ -334,22 +320,21 @@ fn finish( } else if scale.len() != SCALE_ROWS { panic!("QUANT_I32_TO_I8 must emit four scale rows"); } - let (rs1_lo, rs1_hi) = split_rs(rs1); let (rs2_lo, rs2_hi) = split_rs(rs2); ToInt8Case { cmd: ToInt8Cmd { kind, - bid, iter, op1_bank, op2_bank, wr_bank, + op1_group: groups[0], + op2_group: groups[1], + wr_group: groups[2], op1_col, op2_col, wr_col, rob_id, - rs1_lo, - rs1_hi, rs2_lo, rs2_hi, input_base, @@ -364,7 +349,7 @@ fn finish( } } -fn f32_case(bid: u32, rob_id: u32, scale: f32, values: &[f32]) -> ToInt8Case { +fn f32_case(groups: [u32; 3], rob_id: u32, scale: f32, values: &[f32]) -> ToInt8Case { let bits: Vec = values.iter().map(|v| v.to_bits()).collect(); let src = pack_u32s(&bits); let iter = src.len() as u32; @@ -372,7 +357,7 @@ fn f32_case(bid: u32, rob_id: u32, scale: f32, values: &[f32]) -> ToInt8Case { let dst_addr: Vec = (0..dst.len() as u32).collect(); finish( KIND_F32, - bid, + groups, iter, 0, 0, @@ -389,7 +374,7 @@ fn f32_case(bid: u32, rob_id: u32, scale: f32, values: &[f32]) -> ToInt8Case { } fn i32_case( - bid: u32, + groups: [u32; 3], rob_id: u32, op1_bank: u32, op2_bank: u32, @@ -411,7 +396,7 @@ fn i32_case( let dst_addr = i32_addrs(iter, out_base, width, stride); finish( KIND_I32, - bid, + groups, iter, op1_bank, op2_bank, @@ -427,9 +412,9 @@ fn i32_case( ) } -fn f32_signed(bid: u32) -> ToInt8Case { +fn f32_signed() -> ToInt8Case { f32_case( - bid, + [0, 0, 0], 2, 0.5, &[ @@ -439,13 +424,13 @@ fn f32_signed(bid: u32) -> ToInt8Case { ) } -fn f32_zero(bid: u32) -> ToInt8Case { - f32_case(bid, 3, 1.0, &[0.0; 16]) +fn f32_zero() -> ToInt8Case { + f32_case([1, 0, 2], 3, 1.0, &[0.0; 16]) } -fn f32_rounding(bid: u32) -> ToInt8Case { +fn f32_rounding() -> ToInt8Case { f32_case( - bid, + [2, 0, 1], 4, 1.0, &[ @@ -455,9 +440,9 @@ fn f32_rounding(bid: u32) -> ToInt8Case { ) } -fn f32_rows(bid: u32) -> ToInt8Case { +fn f32_rows() -> ToInt8Case { f32_case( - bid, + [3, 0, 3], 5, 2.0, &[ @@ -468,37 +453,37 @@ fn f32_rows(bid: u32) -> ToInt8Case { ) } -fn i32_scale_rows(bid: u32) -> ToInt8Case { +fn i32_scale_rows() -> ToInt8Case { let mut values = [0i32; 16]; let mut scales = [0.0f32; 16]; for i in 0..16 { values[i] = i as i32 - 8; scales[i] = 0.5 + (i as f32) * 0.25; } - i32_case(bid, 6, 0, 1, 2, false, 0, 1, 1, 1, 0, &values, &scales) + i32_case([0, 0, 0], 6, 0, 1, 2, false, 0, 1, 1, 1, 0, &values, &scales) } -fn i32_relu_tile(bid: u32) -> ToInt8Case { +fn i32_relu_tile() -> ToInt8Case { let mut values = [0i32; 64]; for position in 0..4 { for channel in 0..16 { values[position * 16 + channel] = position as i32 * 20 + channel as i32 - 8; } } - i32_case(bid, 7, 0, 1, 2, true, 3, 2, 2, 4, 0, &values, &[1.0; 16]) + i32_case([1, 2, 3], 7, 0, 1, 2, true, 3, 2, 2, 4, 0, &values, &[1.0; 16]) } -fn i32_signed(bid: u32) -> ToInt8Case { +fn i32_signed() -> ToInt8Case { let values: [i32; 16] = [ -300, -128, -5, -1, 0, 1, 5, 127, 128, 200, -64, 64, -7, 7, 3, -3, ]; - i32_case(bid, 8, 0, 1, 2, false, 0, 1, 1, 1, 0, &values, &[1.0; 16]) + i32_case([3, 1, 2], 8, 0, 1, 2, false, 0, 1, 1, 1, 0, &values, &[1.0; 16]) } -fn i32_offset(bid: u32) -> ToInt8Case { +fn i32_offset() -> ToInt8Case { let mut values = [0i32; 32]; for i in 0..32 { values[i] = i as i32 - 16; } - i32_case(bid, 9, 0, 1, 2, true, 1, 2, 1, 2, 4, &values, &[0.5; 16]) + i32_case([2, 3, 1], 9, 0, 1, 2, true, 1, 2, 1, 2, 4, &values, &[0.5; 16]) } diff --git a/examples/components/balls/toint8/verify/casegen/src/lib.rs b/examples/components/balls/toint8/verify/casegen/src/lib.rs index 51878c5fa..1bf3b389d 100644 --- a/examples/components/balls/toint8/verify/casegen/src/lib.rs +++ b/examples/components/balls/toint8/verify/casegen/src/lib.rs @@ -11,8 +11,8 @@ thread_local! { } #[no_mangle] -pub extern "C" fn toint8_case_load(index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(index, bid); +pub extern "C" fn toint8_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); CURRENT.with(|current| *current.borrow_mut() = Some(case)); 0 } diff --git a/examples/components/balls/toint8/verify/filelists/toint8_ball.f b/examples/components/balls/toint8/verify/filelists/toint8_ball.f index 8750aeaa3..5c0be6498 100644 --- a/examples/components/balls/toint8/verify/filelists/toint8_ball.f +++ b/examples/components/balls/toint8/verify/filelists/toint8_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,6 +14,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv @@ -21,7 +23,7 @@ @RTL@/INToRecFN_i32_e8_s24.sv @RTL@/MulAddRecFNToRaw_preMul_e8_s24.sv @RTL@/MulAddRecFNToRaw_postMul_e8_s24.sv -@RTL@/MulAddRecFNPipe_l3_e8_s24.sv +@RTL@/Fma_l3_e8_s24.sv @RTL@/RoundRawFNToRecFN_e8_s24.sv @RTL@/RecFNToIN_e8_s24_i8.sv @RTL@/ToInt8Ball.sv diff --git a/examples/components/balls/toint8/verify/src/common/toint8_defs.svh b/examples/components/balls/toint8/verify/src/common/toint8_defs.svh index 41d20e9d6..ee65bc7e1 100644 --- a/examples/components/balls/toint8/verify/src/common/toint8_defs.svh +++ b/examples/components/balls/toint8/verify/src/common/toint8_defs.svh @@ -1,16 +1,16 @@ typedef struct { int unsigned kind; - int unsigned bid; int unsigned iter; int unsigned op1_bank; int unsigned op2_bank; int unsigned wr_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; int unsigned op1_col; int unsigned op2_col; int unsigned wr_col; int unsigned rob_id; - int unsigned rs1_lo; - int unsigned rs1_hi; int unsigned rs2_lo; int unsigned rs2_hi; int unsigned input_base; @@ -19,10 +19,7 @@ typedef struct { int unsigned num_dst_words; } toint8_cmd_dpi_t; -import "DPI-C" function int toint8_case_load( - input int unsigned index, - input int unsigned bid -); +import "DPI-C" function int toint8_case_load(input int unsigned index); import "DPI-C" function void toint8_case_cmd(output toint8_cmd_dpi_t cmd); import "DPI-C" function longint unsigned toint8_case_src_word_lo(input int unsigned word_index); import "DPI-C" function longint unsigned toint8_case_src_word_hi(input int unsigned word_index); @@ -48,11 +45,3 @@ localparam int TOINT8_MAX_SCALE_WORDS = 4; localparam int TOINT8_MAX_DST_WORDS = 4; localparam int TOINT8_NUM_CASES = 8; localparam int TOINT8_TIMEOUT_CYCLES = 4000; - -function automatic int unsigned toint8_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/components/balls/toint8/verify/src/common/toint8_items.svh b/examples/components/balls/toint8/verify/src/common/toint8_items.svh index 3f2bfc7b8..88d4d7470 100644 --- a/examples/components/balls/toint8/verify/src/common/toint8_items.svh +++ b/examples/components/balls/toint8/verify/src/common/toint8_items.svh @@ -11,31 +11,18 @@ class toint8_cmd_item extends bb_blink_cmd_item; bit [127:0] dst_words[TOINT8_MAX_DST_WORDS]; bit [31:0] dst_addr[TOINT8_MAX_DST_WORDS]; - constraint legal_c { - funct7 inside {QUANT_F32_TO_I8_THREAD_FUNCT7[6:0], QUANT_I32_TO_I8_THREAD_FUNCT7[6:0]}; - op1_en == 1'b1; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op1_col == 5'd1; - wr_col == 5'd1; - meta_bank == 5'd0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - special == 64'd0; - } - function new(string name = "toint8_cmd_item"); super.new(name); endfunction - function void load_rust_case(int unsigned index, int unsigned bid); + function void load_rust_case(int unsigned index); toint8_cmd_dpi_t cmd; int unsigned rc; longint unsigned w_lo; longint unsigned w_hi; int i; - rc = toint8_case_load(index, bid); + rc = toint8_case_load(index); if (rc != 0) begin `uvm_fatal("CASE", $sformatf("toint8_case_load returned %0d for index %0d", rc, index)) end @@ -45,43 +32,35 @@ class toint8_cmd_item extends bb_blink_cmd_item; `uvm_fatal("CASE", $sformatf("unknown toint8 kind %0d", cmd.kind)) end - kind = cmd.kind; - this.bid = cmd.bid[4:0]; - iter = cmd.iter; - op1_bank = cmd.op1_bank[4:0]; - op2_bank = cmd.op2_bank[4:0]; - wr_bank = cmd.wr_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - op2_col = cmd.op2_col[4:0]; - wr_col = cmd.wr_col[4:0]; - rob_id = cmd.rob_id[3:0]; - rs1 = {cmd.rs1_hi, cmd.rs1_lo}; - rs2 = {cmd.rs2_hi, cmd.rs2_lo}; - input_base = cmd.input_base; - num_src_words = cmd.num_src_words; + // Banks and iter come from rs1; rs2 carries the f32 scale or the i32 output geometry. + kind = cmd.kind; + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = kind == TOINT8_KIND_F32 ? QUANT_F32_TO_I8_THREAD_FUNCT7[6:0] : + QUANT_I32_TO_I8_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; + input_base = cmd.input_base; + num_src_words = cmd.num_src_words; num_scale_words = cmd.num_scale_words; - num_dst_words = cmd.num_dst_words; - special = 64'd0; - op1_en = 1'b1; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - meta_bank = 5'd0; - is_sub = 1'b0; - sub_rob_id = 8'h00; + num_dst_words = cmd.num_dst_words; if (kind == TOINT8_KIND_F32) begin - funct7 = QUANT_F32_TO_I8_THREAD_FUNCT7[6:0]; - op2_en = 1'b0; - op2_from_spad = 1'b0; - if (op2_bank != 0 || op2_col != 0 || rs1[19:10] != 0) - `uvm_fatal("CASE", "QUANT_F32_TO_I8 reserves input bank 1") + if (op2_bank != 0 || op2_col != 0) `uvm_fatal("CASE", "QUANT_F32_TO_I8 reserves input bank 1") if (rs2[63:32] != 0) `uvm_fatal("CASE", "QUANT_F32_TO_I8 reserves rs2[63:32]") if (num_scale_words != 0) `uvm_fatal("CASE", "QUANT_F32_TO_I8 must not emit scale rows") if (input_base != 0) `uvm_fatal("CASE", "QUANT_F32_TO_I8 input_base must be 0") end else begin - funct7 = QUANT_I32_TO_I8_THREAD_FUNCT7[6:0]; - op2_en = 1'b1; - op2_from_spad = 1'b1; if (op2_col != 1) `uvm_fatal("CASE", "QUANT_I32_TO_I8 scale must occupy one bank") if (op1_bank == op2_bank) `uvm_fatal("CASE", "QUANT_I32_TO_I8 input and scale banks must differ") @@ -103,8 +82,6 @@ class toint8_cmd_item extends bb_blink_cmd_item; `uvm_fatal("CASE", $sformatf("num_src_words out of range: %0d", num_src_words)) if (num_dst_words == 0 || num_dst_words > TOINT8_MAX_DST_WORDS) `uvm_fatal("CASE", $sformatf("num_dst_words out of range: %0d", num_dst_words)) - if (funct7 != QUANT_F32_TO_I8_THREAD_FUNCT7[6:0] && funct7 != QUANT_I32_TO_I8_THREAD_FUNCT7[6:0]) - `uvm_fatal("CASE", $sformatf("invalid funct7: %0d", funct7)) for (i = 0; i < num_src_words; i++) begin w_lo = toint8_case_src_word_lo(i); diff --git a/examples/components/balls/toint8/verify/src/cov/toint8_cov.svh b/examples/components/balls/toint8/verify/src/cov/toint8_cov.svh index 7113d9669..94d89c90a 100644 --- a/examples/components/balls/toint8/verify/src/cov/toint8_cov.svh +++ b/examples/components/balls/toint8/verify/src/cov/toint8_cov.svh @@ -3,10 +3,10 @@ class toint8_cov extends uvm_component; uvm_analysis_imp_cmd #(bb_blink_cmd_item, toint8_cov) cmd_imp; - bit [6:0] cur_funct7; + bit [6:0] cur_funct; covergroup cmd_cg; - coverpoint cur_funct7 { + coverpoint cur_funct { bins f32 = {QUANT_F32_TO_I8_THREAD_FUNCT7[6:0]}; bins i32 = {QUANT_I32_TO_I8_THREAD_FUNCT7[6:0]}; } @@ -21,7 +21,7 @@ class toint8_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - cur_funct7 = item.funct7; + cur_funct = item.funct; cmd_count++; cmd_cg.sample(); endfunction diff --git a/examples/components/balls/toint8/verify/src/env/toint8_scoreboard.svh b/examples/components/balls/toint8/verify/src/env/toint8_scoreboard.svh index a6b932840..57ed1d693 100644 --- a/examples/components/balls/toint8/verify/src/env/toint8_scoreboard.svh +++ b/examples/components/balls/toint8/verify/src/env/toint8_scoreboard.svh @@ -47,10 +47,11 @@ class toint8_scoreboard extends uvm_scoreboard; end mem_model.clear_mem(); for (i = 0; i < clone.num_src_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), int'(clone.input_base) + i, clone.src_words[i]); + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group), int'(clone.input_base) + i, + clone.src_words[i]); end for (i = 0; i < clone.num_scale_words; i++) begin - mem_model.load_word(int'(clone.op2_bank), i, clone.scale_words[i]); + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group), i, clone.scale_words[i]); end mem_model.arm(); @@ -66,47 +67,41 @@ class toint8_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, toint8_cmd_item exp); + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); verification_pkg::evidence::checks++; - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.op2_bank !== exp.op2_bank || - got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.op2_col !== exp.op2_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rs1 !== exp.rs1 || got.rs2 !== exp.rs2) `uvm_fatal("CMD", "rs field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) endfunction function void write_read(bb_blink_read_item item); toint8_cmd_item stim; int unsigned scale_reads; int unsigned expect_bank; + int unsigned expect_group; int unsigned expect_addr; stim = current_stim("READ"); if (item.port != 0) `uvm_fatal("READ", $sformatf("ToInt8Ball has one read port, got port %0d", item.port)) - if (item.group_id !== 5'd0) - `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp 0", item.group_id)) if (item.rob_id !== stim.rob_id) `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) scale_reads = stim.num_scale_words; if (read_count < scale_reads) begin - expect_bank = stim.op2_bank; - expect_addr = read_count; + expect_bank = stim.op2_bank; + expect_group = stim.op2_group; + expect_addr = read_count; end else begin - expect_bank = stim.op1_bank; - expect_addr = stim.input_base + (read_count - scale_reads); + expect_bank = stim.op1_bank; + expect_group = stim.op1_group; + expect_addr = stim.input_base + (read_count - scale_reads); end - if (item.bank_id !== expect_bank[4:0]) + if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) + if (item.group_id !== expect_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, expect_group)) if (item.addr !== expect_addr[BB_BLINK_BANK_ADDR_W-1:0]) `uvm_fatal("READ", $sformatf("addr mismatch: got %0d exp %0d", item.addr, expect_addr)) read_count++; @@ -122,8 +117,9 @@ class toint8_scoreboard extends uvm_scoreboard; `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== 5'd0) - `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp 0", item.group_id)) + if (item.group_id !== stim.wr_group) + `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + )) if (item.mask !== 16'hFFFF) `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) if (write_count >= stim.num_dst_words) @@ -147,9 +143,6 @@ class toint8_scoreboard extends uvm_scoreboard; stim = current_stim("RESP"); if (item.rob_id !== stim.rob_id) `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") - if (item.sub_rob_id !== 8'h00) - `uvm_fatal("RESP", $sformatf("sub_rob_id mismatch: got 0x%0h", item.sub_rob_id)) resp_count++; endfunction diff --git a/examples/components/balls/toint8/verify/src/seq/toint8_sequences.svh b/examples/components/balls/toint8/verify/src/seq/toint8_sequences.svh index f51579a1e..f72b2d3f5 100644 --- a/examples/components/balls/toint8/verify/src/seq/toint8_sequences.svh +++ b/examples/components/balls/toint8/verify/src/seq/toint8_sequences.svh @@ -2,7 +2,6 @@ class toint8_basic_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(toint8_basic_seq) int unsigned case_index; - int unsigned bid; function new(string name = "toint8_basic_seq"); super.new(name); @@ -12,7 +11,7 @@ class toint8_basic_seq extends uvm_sequence #(bb_blink_cmd_item); toint8_cmd_item item; item = toint8_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(case_index, bid); + item.load_rust_case(case_index); finish_item(item); endtask endclass diff --git a/examples/components/balls/toint8/verify/src/tb_top.sv b/examples/components/balls/toint8/verify/src/tb_top.sv index f1947f8b7..c54dcb03f 100644 --- a/examples/components/balls/toint8/verify/src/tb_top.sv +++ b/examples/components/balls/toint8/verify/src/tb_top.sv @@ -7,77 +7,66 @@ module tb_top; ToInt8Ball dut ( .clock(intf.clock), .reset(intf.reset), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready), - .io_status_idle(), - .io_status_running(), - .io_bankAccessReady(1'b1), - .io_bankRead_0_ball_id(), - .io_bankWrite_0_ball_id() + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) ); + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + initial begin intf.clock = 1'b0; forever #5 intf.clock = ~intf.clock; diff --git a/examples/components/balls/toint8/verify/src/tests/toint8_test.svh b/examples/components/balls/toint8/verify/src/tests/toint8_test.svh index 66e5b66d4..92d766203 100644 --- a/examples/components/balls/toint8/verify/src/tests/toint8_test.svh +++ b/examples/components/balls/toint8/verify/src/tests/toint8_test.svh @@ -17,20 +17,18 @@ class toint8_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid = toint8_require_bid(); phase.raise_objection(this); - for (int unsigned index = 0; index < TOINT8_NUM_CASES; index++) run_case(index, bid); + for (int unsigned index = 0; index < TOINT8_NUM_CASES; index++) run_case(index); phase.drop_objection(this); endtask - task run_case(int unsigned index, int unsigned bid); + task run_case(int unsigned index); toint8_basic_seq seq; int cycles = 0; apply_reset(); env.scb.reset_counters(); seq = toint8_basic_seq::type_id::create("seq"); seq.case_index = index; - seq.bid = bid; seq.start(env.cmd_agent.seqr); while (!env.scb.done()) begin @(posedge vif.clock); diff --git a/examples/components/balls/toint8/workloads/isa/quant.h b/examples/components/balls/toint8/workloads/isa/quant.h index e2ba22677..3d6f4ae31 100644 --- a/examples/components/balls/toint8/workloads/isa/quant.h +++ b/examples/components/balls/toint8/workloads/isa/quant.h @@ -4,6 +4,13 @@ #include #include +// Group g of every operand bank is channel block g (channels 16g..16g+15). +// The input, scale (I32 only), and output banks have the same number of +// groups; every group uses the same rows, so one instruction converts all +// channel blocks. In group g, input rows hold 4 FP32/INT32 lanes (four rows +// per pixel), scale rows 0..3 hold the 16 per-channel scales of block g, and +// each output row holds the 16 INT8 channels of one pixel. + static inline void bb_quant_f32_to_i8(uint64_t input_bank, uint64_t output_bank, uint64_t iter, float scale) { union { diff --git a/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir b/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir index aec1588b8..8c3bdd7cb 100644 --- a/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir +++ b/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir @@ -12,14 +12,14 @@ func.func @main() -> i8 { %scale = arith.constant 5.000000e-01 : f32 %input = memref.get_global @input : memref<4x4xf32> %output = memref.alloc() alignment = 64 : memref<1x16xi8> - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mvin %input <{bank = 0 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - buckyball.custom "fp32_to_int8" args(%scale : f32) <{sources = array, destination = 1 : i64}> {rows = 4 : i64} - buckyball.mvout %output <{bank = 1 : i64, depth = 1 : i64, stride = 1 : i64}> : memref<1x16xi8> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + buckyball.custom "fp32_to_int8" args(%scale : f32) <{sources = array, destination = 2 : i64}> {rows = 4 : i64} + buckyball.mvout %output <{bank = 2 : i64, depth = 1 : i64, stride = 1 : i64}> : memref<1x16xi8> func.call @check_result(%output) : (memref<1x16xi8>) -> () - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> memref.dealloc %output : memref<1x16xi8> return %zero_i8 : i8 } diff --git a/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir b/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir index 8e36bd460..940886a22 100644 --- a/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir +++ b/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir @@ -1,4 +1,4 @@ -func.func private @check_result(memref<12x16xi8>) -> () +func.func private @check_result(memref<4x16xi8>) -> () func.func @main() -> i8 { %zero_i8 = arith.constant 0 : i8 @@ -11,11 +11,9 @@ func.func @main() -> i8 { %eight = arith.constant 8 : index %input = memref.alloc() alignment = 64 : memref<16x4xi32> %scales = memref.alloc() alignment = 64 : memref<4x4xf32> - %output_seed = memref.alloc() alignment = 64 : memref<12x16xi8> - %output = memref.alloc() alignment = 64 : memref<12x16xi8> + %output = memref.alloc() alignment = 64 : memref<4x16xi8> linalg.fill ins(%one_f32 : f32) outs(%scales : memref<4x4xf32>) - linalg.fill ins(%zero_i8 : i8) outs(%output_seed : memref<12x16xi8>) - linalg.fill ins(%zero_i8 : i8) outs(%output : memref<12x16xi8>) + linalg.fill ins(%zero_i8 : i8) outs(%output : memref<4x16xi8>) scf.for %position = %zero to %four step %one { scf.for %channel = %zero to %sixteen step %one { %position_base = arith.muli %position, %twenty : index @@ -29,22 +27,19 @@ func.func @main() -> i8 { memref.store %value_i32, %input[%row, %lane] : memref<16x4xi32> } } - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mvin %input <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> - buckyball.mvin %scales <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - buckyball.mvin %output_seed <{bank = 2 : i64, depth = 12 : i64, stride = 1 : i64}> : memref<12x16xi8> - // Four positions to rows 3..6; the other rows keep the seed. - buckyball.custom "int32_to_int8" <{sources = array, destination = 2 : i64, read_destination}> {rows = 4 : i64, wr_base = 3 : i64, relu = true} - buckyball.mvout %output <{bank = 2 : i64, depth = 12 : i64, stride = 1 : i64}> : memref<12x16xi8> - func.call @check_result(%output) : (memref<12x16xi8>) -> () - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + buckyball.mvin %scales <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> + buckyball.custom "int32_to_int8" <{sources = array, destination = 3 : i64}> {rows = 4 : i64, relu = true} + buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + func.call @check_result(%output) : (memref<4x16xi8>) -> () buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> memref.dealloc %input : memref<16x4xi32> memref.dealloc %scales : memref<4x4xf32> - memref.dealloc %output_seed : memref<12x16xi8> - memref.dealloc %output : memref<12x16xi8> + memref.dealloc %output : memref<4x16xi8> return %zero_i8 : i8 } diff --git a/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp b/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp index cc1c33451..0c59f734f 100644 --- a/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp +++ b/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp @@ -16,13 +16,12 @@ extern "C" void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, int64_t rows, int64_t columns, int64_t row_stride, int64_t column_stride) { (void)allocated; - if (rows != 12 || columns != 16 || row_stride != 16 || column_stride != 1) + if (rows != 4 || columns != 16 || row_stride != 16 || column_stride != 1) fail(); int8_t *output = aligned + offset; - for (int row = 0; row < 12; ++row) { + for (int row = 0; row < 4; ++row) { for (int channel = 0; channel < 16; ++channel) { - int position = row >= 3 && row < 7 ? row - 3 : -1; - int expected = position < 0 ? 0 : position * 20 + channel - 8; + int expected = row * 20 + channel - 8; if (expected < 0) expected = 0; int actual = output[row * row_stride + channel * column_stride]; @@ -34,5 +33,5 @@ extern "C" void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, } } } - printf("PASSED: quant_i32_to_i8 relu and output base\n"); + printf("PASSED: quant_i32_to_i8 relu\n"); } diff --git a/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in b/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in index f41cb4fed..00f7759df 100644 --- a/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in +++ b/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in @@ -28,7 +28,7 @@ func.func @main() -> i8 { %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_QUANT_I32_INPUT_ROWS@ : i64, stride = 1 : i64}> : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> %loaded_scale = buckyball.bank_mvin %scale, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - %result = buckyball.bank_custom "int32_to_int8" (%loaded_input, %loaded_scale) -> %output_bank {rows = @BB_QUANT_I32_OUTPUT_ROWS@ : i64, wr_base = 0 : i64, relu = false} + %result = buckyball.bank_custom "int32_to_int8" (%loaded_input, %loaded_scale) -> %output_bank {rows = @BB_QUANT_I32_OUTPUT_ROWS@ : i64, relu = false} %stored = buckyball.bank_mvout %output, %result <{depth = @BB_QUANT_I32_OUTPUT_ROWS@ : i64, stride = 1 : i64}> : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> func.call @check_result(%output) : (memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () buckyball.bank_release %loaded_input diff --git a/examples/components/balls/transpose/verify/README.md b/examples/components/balls/transpose/verify/README.md index 0556427c9..be2ea3661 100644 --- a/examples/components/balls/transpose/verify/README.md +++ b/examples/components/balls/transpose/verify/README.md @@ -22,27 +22,16 @@ Verifies the generated `TransposeBall` module directly using the shared Blink UV - case 0: directed i8, iter=16 - case 1: directed i32, iter=8 -- cases 2..21: random with fixed seed `0xCAFE_BABE`, elem_bits in {8,32}, iter in {1,2,4,8,16} +- case 2: directed i8 16x32, rows span two groups (`transpose_i8_16x32_test.c`) +- cases 3..22: random with fixed seed `0xCAFE_BABE`, elem_bits in {8,32}, iter in {1,2,4,8,16}, + col in {1,2}, op1/wr group base in 0..3 Each case: reset, preload src words into `mem_model`, drive one command, wait for -`scb.done()` or timeout at 2000 cycles. Scoreboard compares observed writes -(data/addr/mask/bank/rob) with DPI `transpose_case_dst_word_*`. `transpose_cov` -requires both `elem_bits` bins and all `iter` bins hit across the whole test, else +`scb.done()` or timeout at 8000 cycles. Scoreboard compares observed writes +(data/addr/group/mask/bank/rob) with DPI `transpose_case_dst_word_*`. `transpose_cov` +requires both `elem_bits` bins, both `col` bins and all `iter` bins hit across the whole test, else `check_phase` fatal. -## BID (required) - -`+BID=` is mandatory. Missing it fatals at test start. There is no default bid. - -| Config | Plusarg | -|--------|---------| -| `sims.verilator.BuckyballToyVerilatorConfig` | `+BID=2` | -| `sims.verilator.BuckyballPebbleVerilatorConfig` | `+BID=0` | - -```console -./simv +UVM_TESTNAME=transpose_ball_test +BID=2 -``` - ## Build Enter the shared UVM/VCS environment: @@ -69,14 +58,13 @@ vcs -full64 -sverilog -timescale=1ns/1ps \ Run: ```console -./simv +UVM_TESTNAME=transpose_ball_test +BID=2 +./simv +UVM_TESTNAME=transpose_ball_test ``` Or via bbdev: ```console -bbdev uvm --build '--ball=transpose' -bbdev uvm --run '--ball=transpose' -- '+BID=2' +bbdev uvm --run '--design pebble --ball transpose' ``` @@ -85,7 +73,6 @@ bbdev uvm --run '--ball=transpose' -- '+BID=2' - [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` - [x] casegen reuses transpose model; whole-case DPI API - [x] directed i8/i32 scoreboard pass -- [x] deterministic random idx 2..21 pass +- [x] deterministic random idx 3..22 pass - [x] protocol + semantic cover enforced (`check_phase` fatal) -- [x] toy: `+BID=2` config `sims.verilator.BuckyballToyVerilatorConfig` — 22/22 -- [x] pebble: `+BID=0` config `sims.verilator.BuckyballPebbleVerilatorConfig` — 22/22 +- [x] pebble: 23/23 on the BLink command interface (2026-10-26) diff --git a/examples/components/balls/transpose/verify/casegen/src/casegen.rs b/examples/components/balls/transpose/verify/casegen/src/casegen.rs index 5d5a00338..01ea31e0c 100644 --- a/examples/components/balls/transpose/verify/casegen/src/casegen.rs +++ b/examples/components/balls/transpose/verify/casegen/src/casegen.rs @@ -2,15 +2,17 @@ use crate::model; pub const BANK_ROW_BYTES: usize = 16; pub const MAX_ITER: usize = 16; -pub const MAX_WORDS: usize = MAX_ITER; +pub const MAX_COLS: usize = 2; +pub const MAX_WORDS: usize = MAX_ITER * MAX_COLS; #[repr(C)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct TransposeCmd { - pub bid: u32, pub iter: u32, pub op1_bank: u32, pub wr_bank: u32, + pub op1_group: u32, + pub wr_group: u32, pub op1_col: u32, pub wr_col: u32, pub rob_id: u32, @@ -55,15 +57,17 @@ fn word_hi(data: &[u8], word_index: usize) -> u64 { u64::from_le_bytes(buf) } -pub fn gen_case(seed: u32, index: u32, bid: u32) -> TransposeCase { +// Word `i` of a case is bank row `i / cols`, group `i % cols`: each row spans `cols` groups. +pub fn gen_case(seed: u32, index: u32) -> TransposeCase { match index { - 0 => directed_i8(bid), - 1 => directed_i32(bid), - _ => random_case(seed, index, bid), + 0 => directed_i8(), + 1 => directed_i32(), + 2 => directed_i8_two_groups(), + _ => random_case(seed, index), } } -fn directed_i8(bid: u32) -> TransposeCase { +fn directed_i8() -> TransposeCase { let iter = 16usize; let elem_bytes = 1usize; let cols = 1usize; @@ -74,10 +78,11 @@ fn directed_i8(bid: u32) -> TransposeCase { let nwords = model::num_words(total * elem_bytes) as u32; TransposeCase { cmd: TransposeCmd { - bid, iter: iter as u32, op1_bank: 0, wr_bank: 1, + op1_group: 0, + wr_group: 0, op1_col: cols as u32, wr_col: cols as u32, rob_id: 3, @@ -90,7 +95,7 @@ fn directed_i8(bid: u32) -> TransposeCase { } } -fn directed_i32(bid: u32) -> TransposeCase { +fn directed_i32() -> TransposeCase { let iter = 8usize; let elem_bytes = 4usize; let cols = 1usize; @@ -104,10 +109,11 @@ fn directed_i32(bid: u32) -> TransposeCase { let nwords = model::num_words(total * elem_bytes) as u32; TransposeCase { cmd: TransposeCmd { - bid, iter: iter as u32, op1_bank: 0, wr_bank: 1, + op1_group: 0, + wr_group: 0, op1_col: cols as u32, wr_col: cols as u32, rob_id: 5, @@ -120,13 +126,44 @@ fn directed_i32(bid: u32) -> TransposeCase { } } -fn random_case(seed: u32, index: u32, bid: u32) -> TransposeCase { +// Mirrors transpose_i8_16x32_test.c: a 16x32 i8 matrix whose rows span two groups. +fn directed_i8_two_groups() -> TransposeCase { + let iter = 16usize; + let elem_bytes = 1usize; + let cols = 2usize; + let w = cols * (BANK_ROW_BYTES / elem_bytes); + let total = iter * w; + let src: Vec = (0..iter) + .flat_map(|r| (0..w).map(move |c| (r + c + 1) as u8)) + .collect(); + let dst = model::transpose_bytes(&src, iter, w, elem_bytes); + let nwords = model::num_words(total * elem_bytes) as u32; + TransposeCase { + cmd: TransposeCmd { + iter: iter as u32, + op1_bank: 0, + wr_bank: 1, + op1_group: 2, + wr_group: 1, + op1_col: cols as u32, + wr_col: cols as u32, + rob_id: 6, + elem_bits: (elem_bytes * 8) as u32, + num_src_words: nwords, + num_dst_words: nwords, + }, + src, + dst, + } +} + +fn random_case(seed: u32, index: u32) -> TransposeCase { let mut rng = Rng::new(seed, index); let elem_bits = if rng.next() & 1 == 0 { 8u32 } else { 32u32 }; let elem_bytes = (elem_bits / 8) as usize; let iter_pool = [1u32, 2, 4, 8, 16]; let iter = iter_pool[(rng.next() as usize) % iter_pool.len()] as usize; - let cols = 1usize; + let cols = (rng.next() as usize) % MAX_COLS + 1; let w = cols * (BANK_ROW_BYTES / elem_bytes); // Keep banks in 0..7 so the same cases work on configs with bankNum=8 or 16. @@ -136,6 +173,9 @@ fn random_case(seed: u32, index: u32, bid: u32) -> TransposeCase { wr_bank = (wr_bank + 1) & 7; } + let op1_group = rng.range(0, 3); + let wr_group = rng.range(0, 3); + let total = iter * w; let mut src = vec![0u8; total * elem_bytes]; for b in src.iter_mut() { @@ -146,10 +186,11 @@ fn random_case(seed: u32, index: u32, bid: u32) -> TransposeCase { TransposeCase { cmd: TransposeCmd { - bid, iter: iter as u32, op1_bank, wr_bank, + op1_group, + wr_group, op1_col: cols as u32, wr_col: cols as u32, rob_id: rng.range(0, 15), diff --git a/examples/components/balls/transpose/verify/casegen/src/lib.rs b/examples/components/balls/transpose/verify/casegen/src/lib.rs index fee2abd4b..d80b5fa2e 100644 --- a/examples/components/balls/transpose/verify/casegen/src/lib.rs +++ b/examples/components/balls/transpose/verify/casegen/src/lib.rs @@ -10,8 +10,8 @@ thread_local! { } #[no_mangle] -pub extern "C" fn transpose_case_load(seed: u32, index: u32, bid: u32) -> i32 { - let case = casegen::gen_case(seed, index, bid); +pub extern "C" fn transpose_case_load(seed: u32, index: u32) -> i32 { + let case = casegen::gen_case(seed, index); let nsrc = case.cmd.num_src_words as usize; let ndst = case.cmd.num_dst_words as usize; if nsrc > MAX_WORDS || ndst > MAX_WORDS { diff --git a/examples/components/balls/transpose/verify/filelists/transpose_ball.f b/examples/components/balls/transpose/verify/filelists/transpose_ball.f index 5bcc41e5c..bf90309d5 100644 --- a/examples/components/balls/transpose/verify/filelists/transpose_ball.f +++ b/examples/components/balls/transpose/verify/filelists/transpose_ball.f @@ -1,3 +1,4 @@ ++incdir+@UVM@/src/core +incdir+@UVM@/src/ball +incdir+@UVM@/src/ball/agents/cmd +incdir+@UVM@/src/ball/agents/mem @@ -13,6 +14,7 @@ +incdir+src/tests +incdir+@RTL@ +@UVM@/src/core/package.sv @UVM@/src/ball/bb_blink_if.sv @UVM@/src/ball/bb_uvm_pkg.sv diff --git a/examples/components/balls/transpose/verify/src/common/transpose_defs.svh b/examples/components/balls/transpose/verify/src/common/transpose_defs.svh index efc2543f5..9e3b9a49f 100644 --- a/examples/components/balls/transpose/verify/src/common/transpose_defs.svh +++ b/examples/components/balls/transpose/verify/src/common/transpose_defs.svh @@ -1,8 +1,9 @@ typedef struct { - int unsigned bid; int unsigned iter; int unsigned op1_bank; int unsigned wr_bank; + int unsigned op1_group; + int unsigned wr_group; int unsigned op1_col; int unsigned wr_col; int unsigned rob_id; @@ -13,8 +14,7 @@ typedef struct { import "DPI-C" function int transpose_case_load( input int unsigned seed, - input int unsigned index, - input int unsigned bid + input int unsigned index ); import "DPI-C" function void transpose_case_cmd(output transpose_cmd_dpi_t cmd); import "DPI-C" function longint unsigned transpose_case_src_word_lo(input int unsigned word_index); @@ -26,14 +26,6 @@ import "DPI-C" function longint unsigned transpose_case_dst_word_hi(input int un `error "TRANSPOSE_FUNCT7 must be provided by the selected Thread ballISA" `endif localparam int TRANSPOSE_THREAD_FUNCT7 = `TRANSPOSE_FUNCT7; -localparam int TRANSPOSE_MAX_WORDS = 16; -localparam int TRANSPOSE_TIMEOUT_CYCLES = 2000; +localparam int TRANSPOSE_MAX_WORDS = 32; +localparam int TRANSPOSE_TIMEOUT_CYCLES = 8000; localparam int TRANSPOSE_SEED = 32'hCAFE_BABE; - -function automatic int unsigned transpose_require_bid(); - int unsigned bid; - if (!$value$plusargs("BID=%d", bid)) begin - `uvm_fatal("BID", "missing required plusarg +BID=") - end - return bid; -endfunction diff --git a/examples/components/balls/transpose/verify/src/common/transpose_items.svh b/examples/components/balls/transpose/verify/src/common/transpose_items.svh index cfde7017c..3cf709fe1 100644 --- a/examples/components/balls/transpose/verify/src/common/transpose_items.svh +++ b/examples/components/balls/transpose/verify/src/common/transpose_items.svh @@ -7,65 +7,43 @@ class transpose_cmd_item extends bb_blink_cmd_item; bit [127:0] src_words[TRANSPOSE_MAX_WORDS]; bit [127:0] dst_words[TRANSPOSE_MAX_WORDS]; - constraint legal_c { - funct7 == TRANSPOSE_THREAD_FUNCT7[6:0]; - op1_en == 1'b1; - op2_en == 1'b0; - wr_spad_en == 1'b1; - op1_from_spad == 1'b1; - op2_from_spad == 1'b0; - op2_bank == 5'd0; - op2_col == 5'd0; - meta_bank == 5'd0; - rs1 == 64'd0; - is_sub == 1'b0; - sub_rob_id == 8'h00; - special == 64'd0; - } - function new(string name = "transpose_cmd_item"); super.new(name); endfunction - function void load_rust_case(int unsigned seed, int unsigned index, int unsigned bid); + function void load_rust_case(int unsigned seed, int unsigned index); transpose_cmd_dpi_t cmd; int unsigned rc; longint unsigned w_lo; longint unsigned w_hi; int i; - rc = transpose_case_load(seed, index, bid); + rc = transpose_case_load(seed, index); if (rc != 0) begin `uvm_fatal("CASE", $sformatf("transpose_case_load returned %0d for index %0d", rc, index)) end transpose_case_cmd(cmd); - bid = cmd.bid[4:0]; - funct7 = TRANSPOSE_THREAD_FUNCT7[6:0]; + // bb_transpose: banks and iter come from rs1, elem_bits from rs2[7:0]. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = TRANSPOSE_THREAD_FUNCT7[6:0]; + rs2 = {56'd0, cmd.elem_bits[7:0]}; iter = cmd.iter; - op1_bank = cmd.op1_bank[4:0]; - wr_bank = cmd.wr_bank[4:0]; - op1_col = cmd.op1_col[4:0]; - wr_col = cmd.wr_col[4:0]; - rob_id = cmd.rob_id[3:0]; + op1_bank = cmd.op1_bank; + op2_bank = '0; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = '0; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = '0; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; elem_bits = cmd.elem_bits; num_src_words = cmd.num_src_words; num_dst_words = cmd.num_dst_words; - rs2 = {56'd0, elem_bits[7:0]}; - op1_en = 1'b1; - op2_en = 1'b0; - wr_spad_en = 1'b1; - op1_from_spad = 1'b1; - op2_from_spad = 1'b0; - op2_bank = 5'd0; - op2_col = 5'd0; - meta_bank = 5'd0; - rs1 = 64'd0; - special = 64'd0; - is_sub = 1'b0; - sub_rob_id = 8'h00; - if (num_src_words > TRANSPOSE_MAX_WORDS || num_dst_words > TRANSPOSE_MAX_WORDS) begin `uvm_fatal("CASE", $sformatf("word count out of range src=%0d dst=%0d", num_src_words, num_dst_words)) diff --git a/examples/components/balls/transpose/verify/src/cov/transpose_cov.svh b/examples/components/balls/transpose/verify/src/cov/transpose_cov.svh index 1d414dcb7..897f10e78 100644 --- a/examples/components/balls/transpose/verify/src/cov/transpose_cov.svh +++ b/examples/components/balls/transpose/verify/src/cov/transpose_cov.svh @@ -5,6 +5,7 @@ class transpose_cov extends uvm_component; covergroup cmd_cg; coverpoint cur_elem_bits {bins i8 = {8}; bins i32 = {32};} + coverpoint cur_col {bins c1 = {1}; bins c2 = {2};} coverpoint cur_iter { bins it1 = {1}; bins it2 = {2}; bins it4 = {4}; bins it8 = {8}; bins it16 = {16}; } @@ -12,6 +13,7 @@ class transpose_cov extends uvm_component; int unsigned cur_elem_bits; int unsigned cur_iter; + int unsigned cur_col; int cmd_count; function new(string name, uvm_component parent); @@ -23,6 +25,7 @@ class transpose_cov extends uvm_component; function void write_cmd(bb_blink_cmd_item item); cur_elem_bits = int'(item.rs2[7:0]); cur_iter = int'(item.iter); + cur_col = int'(item.op1_col); cmd_count++; cmd_cg.sample(); endfunction @@ -35,7 +38,7 @@ class transpose_cov extends uvm_component; `uvm_fatal( "COV", $sformatf( - "transpose_cov coverage incomplete: %0.2f%% (need elem_bits {8,32} and iter {1,2,4,8,16})", + "transpose_cov coverage incomplete: %0.2f%% (need elem_bits {8,32}, col {1,2} and iter {1,2,4,8,16})", cmd_cg.get_coverage())) end endfunction diff --git a/examples/components/balls/transpose/verify/src/env/transpose_scoreboard.svh b/examples/components/balls/transpose/verify/src/env/transpose_scoreboard.svh index 552e478e4..3032cafc9 100644 --- a/examples/components/balls/transpose/verify/src/env/transpose_scoreboard.svh +++ b/examples/components/balls/transpose/verify/src/env/transpose_scoreboard.svh @@ -47,8 +47,10 @@ class transpose_scoreboard extends uvm_scoreboard; `uvm_fatal("SCB", "mem_model handle not set") end mem_model.clear_mem(); + // Word i lives at row i / col, group op1_group + i % col. for (i = 0; i < clone.num_src_words; i++) begin - mem_model.load_word(int'(clone.op1_bank), i, clone.src_words[i]); + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group) + i % clone.op1_col, + i / clone.op1_col, clone.src_words[i]); end mem_model.arm(); @@ -67,22 +69,12 @@ class transpose_scoreboard extends uvm_scoreboard; endfunction function void check_cmd(bb_blink_cmd_item got, transpose_cmd_item exp); + // Compare as plain BLink commands; the comparer rejects differing types. + bb_blink_cmd_item want = bb_blink_cmd_item::type_id::create("want"); + want.copy(exp); verification_pkg::evidence::checks++; - if (got.bid !== exp.bid) - `uvm_fatal("CMD", $sformatf("bid mismatch: got %0d exp %0d", got.bid, exp.bid)) - if (got.funct7 !== exp.funct7) - `uvm_fatal("CMD", $sformatf("funct7 mismatch: got %0d exp %0d", got.funct7, exp.funct7)) - if (got.iter !== exp.iter) - `uvm_fatal("CMD", $sformatf("iter mismatch: got %0d exp %0d", got.iter, exp.iter)) - if (got.op1_bank !== exp.op1_bank || got.wr_bank !== exp.wr_bank) - `uvm_fatal("CMD", "bank field mismatch") - if (got.op1_col !== exp.op1_col || got.wr_col !== exp.wr_col) - `uvm_fatal("CMD", "column field mismatch") - if (got.rob_id !== exp.rob_id) - `uvm_fatal("CMD", $sformatf("rob_id mismatch: got %0d exp %0d", got.rob_id, exp.rob_id)) - if (got.rs2[7:0] !== exp.elem_bits[7:0]) - `uvm_fatal("CMD", $sformatf( - "elem_bits mismatch: got %0d exp %0d", got.rs2[7:0], exp.elem_bits[7:0])) + if (!got.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", got.sprint(), want.sprint())) endfunction function void write_read(bb_blink_read_item item); @@ -92,12 +84,16 @@ class transpose_scoreboard extends uvm_scoreboard; `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.op1_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== '0) - `uvm_fatal("READ", $sformatf("group_id mismatch: got %0d exp 0", item.group_id)) - // DUT may re-read the same bank word across output columns (cache refill). - if (item.addr >= stim.num_src_words[6:0]) + if (item.group_id < stim.op1_group || item.group_id >= stim.op1_group + stim.op1_col) `uvm_fatal("READ", $sformatf( - "addr out of range: got %0d max %0d", item.addr, stim.num_src_words)) + "group_id out of range: got %0d base %0d col %0d", + item.group_id, + stim.op1_group, + stim.op1_col + )) + // DUT may re-read the same bank word across output columns (cache refill). + if (item.addr >= stim.iter) + `uvm_fatal("READ", $sformatf("addr out of range: got %0d iter %0d", item.addr, stim.iter)) read_count++; endfunction @@ -108,10 +104,11 @@ class transpose_scoreboard extends uvm_scoreboard; `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== '0) - `uvm_fatal("WRITE", $sformatf("group_id mismatch: got %0d exp 0", item.group_id)) - if (item.addr !== write_count[6:0]) - `uvm_fatal("WRITE", $sformatf("addr mismatch: got %0d exp %0d", item.addr, write_count)) + if (item.group_id !== stim.wr_group + write_count % stim.wr_col) + `uvm_fatal("WRITE", $sformatf("group_id mismatch at %0d: got %0d", write_count, item.group_id + )) + if (item.addr !== write_count / stim.wr_col) + `uvm_fatal("WRITE", $sformatf("addr mismatch at %0d: got %0d", write_count, item.addr)) if (item.mask !== 16'hFFFF) `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) if (item.data !== expected_dst[write_count]) @@ -129,7 +126,6 @@ class transpose_scoreboard extends uvm_scoreboard; stim = current_stim("RESP"); if (item.rob_id !== stim.rob_id) `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.is_sub !== 1'b0) `uvm_fatal("RESP", "is_sub should be 0") resp_count++; endfunction diff --git a/examples/components/balls/transpose/verify/src/seq/transpose_sequences.svh b/examples/components/balls/transpose/verify/src/seq/transpose_sequences.svh index 1a426859b..bf290a300 100644 --- a/examples/components/balls/transpose/verify/src/seq/transpose_sequences.svh +++ b/examples/components/balls/transpose/verify/src/seq/transpose_sequences.svh @@ -3,7 +3,6 @@ class transpose_basic_seq extends uvm_sequence #(bb_blink_cmd_item); int unsigned case_index; int unsigned seed; - int unsigned bid; function new(string name = "transpose_basic_seq"); super.new(name); @@ -14,7 +13,7 @@ class transpose_basic_seq extends uvm_sequence #(bb_blink_cmd_item); transpose_cmd_item item; item = transpose_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(seed, case_index, bid); + item.load_rust_case(seed, case_index); finish_item(item); endtask endclass diff --git a/examples/components/balls/transpose/verify/src/tb_top.sv b/examples/components/balls/transpose/verify/src/tb_top.sv index c42fbafb0..600d0e47e 100644 --- a/examples/components/balls/transpose/verify/src/tb_top.sv +++ b/examples/components/balls/transpose/verify/src/tb_top.sv @@ -7,75 +7,62 @@ module tb_top; TransposeBall dut ( .clock(intf.clock), .reset(intf.reset), - .io_cmdReq_ready(intf.cmd_req_ready), - .io_cmdReq_valid(intf.cmd_req_valid), - .io_cmdReq_bits_cmd_bid(intf.cmd_req_bits_cmd_bid), - .io_cmdReq_bits_cmd_funct7(intf.cmd_req_bits_cmd_funct7), - .io_cmdReq_bits_cmd_iter(intf.cmd_req_bits_cmd_iter), - .io_cmdReq_bits_cmd_op1_en(intf.cmd_req_bits_cmd_op1_en), - .io_cmdReq_bits_cmd_op2_en(intf.cmd_req_bits_cmd_op2_en), - .io_cmdReq_bits_cmd_wr_spad_en(intf.cmd_req_bits_cmd_wr_spad_en), - .io_cmdReq_bits_cmd_op1_from_spad(intf.cmd_req_bits_cmd_op1_from_spad), - .io_cmdReq_bits_cmd_op2_from_spad(intf.cmd_req_bits_cmd_op2_from_spad), - .io_cmdReq_bits_cmd_special(intf.cmd_req_bits_cmd_special), - .io_cmdReq_bits_cmd_op1_bank(intf.cmd_req_bits_cmd_op1_bank), - .io_cmdReq_bits_cmd_op2_bank(intf.cmd_req_bits_cmd_op2_bank), - .io_cmdReq_bits_cmd_wr_bank(intf.cmd_req_bits_cmd_wr_bank), - .io_cmdReq_bits_cmd_op1_col(intf.cmd_req_bits_cmd_op1_col), - .io_cmdReq_bits_cmd_op2_col(intf.cmd_req_bits_cmd_op2_col), - .io_cmdReq_bits_cmd_wr_col(intf.cmd_req_bits_cmd_wr_col), - .io_cmdReq_bits_cmd_meta_bank(intf.cmd_req_bits_cmd_meta_bank), - .io_cmdReq_bits_cmd_rs1(intf.cmd_req_bits_cmd_rs1), - .io_cmdReq_bits_cmd_rs2(intf.cmd_req_bits_cmd_rs2), - .io_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), - .io_cmdReq_bits_is_sub(intf.cmd_req_bits_is_sub), - .io_cmdReq_bits_sub_rob_id(intf.cmd_req_bits_sub_rob_id), - .io_cmdResp_ready(intf.cmd_resp_ready), - .io_cmdResp_valid(intf.cmd_resp_valid), - .io_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), - .io_cmdResp_bits_is_sub(intf.cmd_resp_bits_is_sub), - .io_cmdResp_bits_sub_rob_id(intf.cmd_resp_bits_sub_rob_id), - .io_bankRead_0_bank_id(intf.bank_read_bank_id[0]), - .io_bankRead_0_rob_id(intf.bank_read_rob_id[0]), - .io_bankRead_0_group_id(intf.bank_read_group_id[0]), - .io_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), - .io_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), - .io_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), - .io_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), - .io_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), - .io_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), - .io_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), - .io_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), - .io_bankWrite_0_group_id(intf.bank_write_group_id[0]), - .io_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), - .io_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), - .io_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), - .io_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), - .io_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), - .io_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), - .io_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), - .io_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), - .io_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), - .io_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), - .io_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), - .io_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), - .io_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), - .io_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), - .io_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), - .io_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), - .io_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), - .io_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), - .io_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), - .io_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), - .io_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), - .io_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), - .io_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), - .io_subRobReq_ready(intf.sub_rob_req_ready), - .io_status_idle(), - .io_status_running(), - .io_bankAccessReady(1'b1), - .io_bankRead_0_ball_id(), - .io_bankWrite_0_ball_id() + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .blink_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .blink_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .blink_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .blink_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .blink_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .blink_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .blink_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .blink_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .blink_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .blink_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .blink_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .blink_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .blink_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .blink_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .blink_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) ); initial begin diff --git a/examples/components/balls/transpose/verify/src/tests/transpose_test.svh b/examples/components/balls/transpose/verify/src/tests/transpose_test.svh index b235d24e5..962dab433 100644 --- a/examples/components/balls/transpose/verify/src/tests/transpose_test.svh +++ b/examples/components/balls/transpose/verify/src/tests/transpose_test.svh @@ -18,27 +18,26 @@ class transpose_ball_test extends uvm_test; endfunction task run_phase(uvm_phase phase); - int unsigned bid; - bid = transpose_require_bid(); phase.raise_objection(this); - run_directed(bid); - run_random(bid); + run_directed(); + run_random(); phase.drop_objection(this); endtask - task run_directed(int unsigned bid); - run_case(0, "DIR_I8", bid); - run_case(1, "DIR_I32", bid); + task run_directed(); + run_case(0, "DIR_I8"); + run_case(1, "DIR_I32"); + run_case(2, "DIR_I8_2G"); endtask - task run_random(int unsigned bid); + task run_random(); int i; - for (i = 2; i <= 21; i++) begin - run_case(i, $sformatf("RND_%0d", i), bid); + for (i = 3; i <= 22; i++) begin + run_case(i, $sformatf("RND_%0d", i)); end endtask - task run_case(int unsigned idx, string label, int unsigned bid); + task run_case(int unsigned idx, string label); transpose_basic_seq seq; int cycles; @@ -47,7 +46,6 @@ class transpose_ball_test extends uvm_test; seq = transpose_basic_seq::type_id::create("seq"); seq.case_index = idx; - seq.bid = bid; seq.start(env.cmd_agent.seqr); cycles = 0; diff --git a/examples/components/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir b/examples/components/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir index 2040907f4..f8c7f99da 100644 --- a/examples/components/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir +++ b/examples/components/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir @@ -23,13 +23,13 @@ func.func @main() -> i8 { } } - buckyball.mset <{bank = 0 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mvin %input <{bank = 0 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - buckyball.custom "transpose_i8" <{sources = array, destination = 1 : i64}> {rows = 16 : i64, cols = 16 : i64} - buckyball.mvout %output <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - buckyball.mset <{bank = 0 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.custom "transpose_i8" <{sources = array, destination = 2 : i64}> {rows = 16 : i64, cols = 16 : i64} + buckyball.mvout %output <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> func.call @check_result(%output) : (memref<16x16xi8>) -> () memref.dealloc %input : memref<16x16xi8> diff --git a/examples/designs/goban/sims/device/workloads/kernel/CMakeLists.txt b/examples/designs/goban/sims/device/workloads/kernel/CMakeLists.txt index 368e8cd4b..aa658be56 100644 --- a/examples/designs/goban/sims/device/workloads/kernel/CMakeLists.txt +++ b/examples/designs/goban/sims/device/workloads/kernel/CMakeLists.txt @@ -1,11 +1,5 @@ # Linux programs installed into the goban rootfs, each with the hart it runs on. +# Goban's matmul threads are OS-invisible: their tests are engine tests, not +# Linux programs, so Linux here only boots. set(BUCKYBALL_KERNEL_PROGRAMS - 0 goban-matmul-ctest-rvv_kernel-linux - 0 goban-matmul-ctest-rvv_silu-linux - 0 goban-matmul-ctest-rvv_swiglu-linux - 0 goban-matmul-ctest-rvv_snake-linux - 0 goban-matmul-ctest-rvv_mxfp8_quant-linux - 0 goban-matmul-ctest-rvv_matmul-linux - 0 goban-matmul-ctest-mvin_mvout_test-linux - 0 goban-matmul-ctest-mvin_mvout_stride_test-linux PARENT_SCOPE) diff --git a/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml index c49bf1ff5..05506cbd5 100644 --- a/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml +++ b/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml @@ -109,4 +109,6 @@ tests = [ "pebble-pebble-ctest-pipeline-same_ball_independent-multicore-baremetal", "pebble-pebble-ctest-pipeline-rob_window_probe-multicore-baremetal", "pebble-pebble-ctest-pipeline-write_bank_versions-multicore-baremetal", + "pebble-pebble-mlirtest-base_bank_loop_carried-baremetal", + "pebble-pebble-mlirtest-base_bank_transfer-baremetal", ] diff --git a/examples/designs/pebble/sims/device/tapeout/config.toml b/examples/designs/pebble/sims/device/tapeout/config.toml index 758d298bb..5ec421892 100644 --- a/examples/designs/pebble/sims/device/tapeout/config.toml +++ b/examples/designs/pebble/sims/device/tapeout/config.toml @@ -1,5 +1,5 @@ [tapeout] -top = "DigitalTop" +top = "DeviceTop" sram_process = "smic180" sram_mdf = "ip/smic180_sram_library.mdf.json" constraints_sdc = "area/constraints.sdc" diff --git a/examples/designs/pebble/stack/midend/CMakeLists.txt b/examples/designs/pebble/stack/midend/CMakeLists.txt index f4006e66f..e3e207cb1 100644 --- a/examples/designs/pebble/stack/midend/CMakeLists.txt +++ b/examples/designs/pebble/stack/midend/CMakeLists.txt @@ -17,13 +17,9 @@ set(TESTS "${CMAKE_CURRENT_SOURCE_DIR}/tests") add_custom_target(${TARGET}-compiler-check COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-custom-ops.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${TESTS}/form-custom-ops.mlir --implicit-check-not=buckyball." COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-int32-bias.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${TESTS}/form-int32-bias.mlir --implicit-check-not=buckyball." - COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-linear-chain.mlir -form-custom-ops | ${CHECK} ${TESTS}/form-linear-chain.mlir --check-prefix=FORM --implicit-check-not=custom.chain" - COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-linear-chain.mlir -form-custom-ops -convert-linalg-to-buckyball | ${CHECK} ${TESTS}/form-linear-chain.mlir --check-prefix=BANK" COMMAND bash -o pipefail -c "${OPT} ${TESTS}/lower-frontend-types.mlir -allow-unregistered-dialect -lower-frontend -split-input-file -verify-diagnostics | ${CHECK} ${TESTS}/lower-frontend-types.mlir --check-prefix=VALID --implicit-check-not=frontend." COMMAND bash -o pipefail -c "${OPT} ${TESTS}/lower-custom-ops.mlir -convert-linalg-to-buckyball | ${CHECK} ${TESTS}/lower-custom-ops.mlir --implicit-check-not=custom.op" COMMAND bash -o pipefail -c "${OPT} ${TESTS}/unknown-custom-op.mlir -convert-linalg-to-buckyball -verify-diagnostics -o /dev/null" - COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-quantized-chain.mlir -form-custom-ops | ${CHECK} ${TESTS}/form-quantized-chain.mlir --check-prefix=FORM" - COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-quantized-chain.mlir -form-custom-ops -convert-linalg-to-buckyball | ${CHECK} ${TESTS}/form-quantized-chain.mlir --check-prefix=BANK --implicit-check-not=buckyball.bank_mvout" COMMAND bash -o pipefail -c "${OPT} ${CMAKE_CURRENT_SOURCE_DIR}/../../workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${CMAKE_CURRENT_SOURCE_DIR}/../../workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir --check-prefix=FORM" COMMAND bash -o pipefail -c "${OPT} ${CMAKE_CURRENT_SOURCE_DIR}/../../workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir -allow-unregistered-dialect -form-custom-ops -convert-linalg-to-buckyball | ${CHECK} ${CMAKE_CURRENT_SOURCE_DIR}/../../workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir --check-prefix=BANK --implicit-check-not=buckyball.bank_mvout" COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-depthwise.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${TESTS}/form-depthwise.mlir --implicit-check-not=linalg.depthwise" diff --git a/examples/designs/pebble/stack/midend/src/ConvertLinalgToBuckyball.cpp b/examples/designs/pebble/stack/midend/src/ConvertLinalgToBuckyball.cpp index 3458cd7da..63a9ba7a4 100644 --- a/examples/designs/pebble/stack/midend/src/ConvertLinalgToBuckyball.cpp +++ b/examples/designs/pebble/stack/midend/src/ConvertLinalgToBuckyball.cpp @@ -13,8 +13,8 @@ #include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/Dialect/Utils/StaticValueUtils.h" #include "mlir/Pass/Pass.h" -#include "llvm/ADT/MapVector.h" using namespace mlir; using namespace ::buddy::buckyball; @@ -35,9 +35,65 @@ Value pebble::tensorBuffer(OpBuilder &builder, Location loc, Value tensor) { return bufferization::ToBufferOp::create(builder, loc, memory, tensor, true); } +SmallVector pebble::repeat( + OpBuilder &builder, Location loc, int64_t begin, int64_t end, int64_t step, + bool unroll, ValueRange carried, + function_ref(OpBuilder &, Value, ValueRange)> body) { + SmallVector values(carried); + if (unroll || end - begin <= step) { + for (int64_t index = begin; index < end; index += step) + values = body( + builder, arith::ConstantIndexOp::create(builder, loc, index), values); + return values; + } + auto loop = scf::ForOp::create( + builder, loc, arith::ConstantIndexOp::create(builder, loc, begin), + arith::ConstantIndexOp::create(builder, loc, end), + arith::ConstantIndexOp::create(builder, loc, step), carried, + [&](OpBuilder &nested, Location loc, Value index, ValueRange args) { + scf::YieldOp::create(nested, loc, body(nested, index, args)); + }); + return loop.getResults(); +} + +Value pebble::accumulate( + OpBuilder &builder, Location loc, int64_t total, int64_t size, bool unroll, + Value accumulator, + function_ref body) { + int64_t pieces = (total + size - 1) / size; + int64_t last = (pieces - 1) * size; + accumulator = body(builder, arith::ConstantIndexOp::create(builder, loc, 0), + std::min(size, total), true, pieces == 1, accumulator); + accumulator = pebble::repeat( + builder, loc, size, last, size, unroll, accumulator, + [&](OpBuilder &b, Value start, ValueRange args) -> SmallVector { + return {body(b, start, size, false, false, args[0])}; + })[0]; + if (pieces > 1) + accumulator = + body(builder, arith::ConstantIndexOp::create(builder, loc, last), + total - last, false, true, accumulator); + return accumulator; +} + +pebble::SpatialTile pebble::spatialTile(int64_t height, int64_t width, + int64_t pool, int64_t stride, + int64_t kernel) { + int64_t tile = 4 / pool; + while (tile > 0 && (height % tile || width % tile || + (tile * pool - 1) * stride + kernel > 8)) + --tile; + if (!tile) + return {0, 0}; + int64_t side = std::min(8, std::min(height, width)); + while (side % tile) + --side; + return {tile, side}; +} + namespace { LogicalResult lowerMatmul(::buddy::custom::CustomOp op, Value residentInput, - Value nextMultiplier, bool relu, Value &result) { + Value &result) { bool linear = op.getKind() == "linear_i8"; if (op.getInputs().size() != (linear ? 6 : 3) || op.getNumResults() != 1) return op.emitOpError("has the wrong number of inputs or results for ") @@ -46,23 +102,14 @@ LogicalResult lowerMatmul(::buddy::custom::CustomOp op, Value residentInput, auto a = cast(inputs[0].getType()); auto b = cast(inputs[1].getType()); auto resultType = cast(op.getResult(0).getType()); - if (a.hasStaticShape() && b.hasStaticShape() && a.getRank() == 2 && - b.getRank() == 2 && - (a.getDimSize(0) != 1 || a.getDimSize(1) % 16 || b.getDimSize(1) % 16)) { - if (residentInput || nextMultiplier) - return op.emitOpError("resident Linear requires one row and aligned K/N"); - return pebble::lowerLinearTiles(op, result); - } - const auto &target = buckyball_target::getBuckyballTarget(); if (!a.hasStaticShape() || !b.hasStaticShape() || a.getRank() != 2 || - b.getRank() != 2 || a.getDimSize(0) != 1 || - a.getDimSize(1) != b.getDimSize(0) || a.getDimSize(1) <= 0 || - b.getDimSize(1) <= 0 || a.getDimSize(1) % 16 || b.getDimSize(1) % 16) - return op.emitOpError("requires 1xK and KxN tensors with positive K and N " - "multiples of sixteen"); + b.getRank() != 2 || a.getDimSize(1) != b.getDimSize(0)) + return op.emitOpError("requires static MxK and KxN tensors"); + const auto &target = buckyball_target::getBuckyballTarget(); int64_t k = a.getDimSize(1), n = b.getDimSize(1); - if (target.bankWidthBits != 128 || k > target.bankDepth) - return op.emitOpError("operands do not fit the core bank geometry"); + // One activation row with aligned K and N stays in one bank. + if (a.getDimSize(0) != 1 || k % 16 || n % 16 || k > target.bankDepth) + return pebble::lowerLinearTiles(op, result); if (!b.getElementType().isInteger(8) || !(linear ? a.getElementType().isF32() : a.getElementType().isInteger(8)) || @@ -86,31 +133,19 @@ LogicalResult lowerMatmul(::buddy::custom::CustomOp op, Value residentInput, return op.emitOpError( "requires an INT32 initial accumulator matching the result"); - if (nextMultiplier && (n / 16 > 128 || n / 16 > target.bankDepth)) - return op.emitOpError( - "resident output exceeds the quantizer row address range"); - OpBuilder builder(op); Location loc = op.getLoc(); Type i64 = builder.getI64Type(); Value zero = arith::ConstantIndexOp::create(builder, loc, 0); Value weight = pebble::tensorBuffer(builder, loc, inputs[1]); - Value output; - if (!nextMultiplier) - output = memref::AllocOp::create( - builder, loc, MemRefType::get({1, n}, resultType.getElementType())); - Value residentOutput; - if (nextMultiplier) - residentOutput = BankAllocOp::create(builder, loc, i64, 1, 1); + Value output = memref::AllocOp::create( + builder, loc, MemRefType::get({1, n}, resultType.getElementType())); Value initial = pebble::tensorBuffer(builder, loc, inputs[2]); - Value scales, inputStep, outputMultiplier; + Value scales, inputStep; Value codes = residentInput; if (linear) { inputStep = tensor::ExtractOp::create(builder, loc, inputs[4], ValueRange{zero, zero}); - if (nextMultiplier) - outputMultiplier = tensor::ExtractOp::create(builder, loc, nextMultiplier, - ValueRange{zero, zero}); scales = memref::AllocOp::create( builder, loc, MemRefType::get({16}, builder.getF32Type())); } @@ -134,79 +169,75 @@ LogicalResult lowerMatmul(::buddy::custom::CustomOp op, Value residentInput, } } - // Keep the activation in its bank while processing each sixteen-column slice. - for (int64_t column = 0; column < n; column += 16) { - for (int64_t lane = 0; linear && lane < 16; ++lane) { - Value index = arith::ConstantIndexOp::create(builder, loc, lane); - Value channel = - arith::ConstantIndexOp::create(builder, loc, column + lane); - Value step = tensor::ExtractOp::create(builder, loc, inputs[5], - ValueRange{zero, channel}); - Value product = arith::MulFOp::create(builder, loc, inputStep, step); - if (nextMultiplier) - product = - arith::MulFOp::create(builder, loc, product, outputMultiplier); - memref::StoreOp::create(builder, loc, product, scales, index); - } - Value weightSlice = memref::SubViewOp::create( - builder, loc, weight, ArrayRef{0, column}, - ArrayRef{k, 16}, ArrayRef{1, 1}); - Value weightBank = BankAllocOp::create(builder, loc, i64, 1, 1); - weightBank = BankMvinOp::create(builder, loc, i64, weightSlice, weightBank, - k, n / 16); - Value accumulator = BankAllocOp::create(builder, loc, i64, 1, 1); - Value initialSlice = memref::SubViewOp::create( - builder, loc, initial, ArrayRef{0, column}, - ArrayRef{1, 16}, ArrayRef{1, 1}); - accumulator = - BankMvinOp::create(builder, loc, i64, initialSlice, accumulator, 4, 1); - auto matmul = - BankCustomOp::create(builder, loc, i64, ValueRange{codes, weightBank}, - accumulator, ValueRange{}, "matmul_i8", true); - matmul->setAttr("rows", builder.getI64IntegerAttr(1)); - matmul->setAttr("k", builder.getI64IntegerAttr(k)); - accumulator = matmul.getDestinationOut(); - BankReleaseOp::create(builder, loc, weightBank); + // Keep the activation in its bank while processing each sixteen-column + // slice. + pebble::repeat( + builder, loc, 0, n, 16, false, {}, + [&](OpBuilder &b, Value column, ValueRange) -> SmallVector { + if (linear) { + auto lanes = scf::ForOp::create( + b, loc, zero, arith::ConstantIndexOp::create(b, loc, 16), + arith::ConstantIndexOp::create(b, loc, 1)); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value channel = arith::AddIOp::create(b, loc, column, lane); + Value step = tensor::ExtractOp::create(b, loc, inputs[5], + ValueRange{zero, channel}); + Value product = arith::MulFOp::create(b, loc, inputStep, step); + memref::StoreOp::create(b, loc, product, scales, lane); + } + SmallVector offsets{b.getIndexAttr(0), column}; + SmallVector unit{b.getIndexAttr(1), b.getIndexAttr(1)}; + Value weightSlice = memref::SubViewOp::create( + b, loc, weight, offsets, + ArrayRef{b.getIndexAttr(k), b.getIndexAttr(16)}, + unit); + Value weightBank = BankAllocOp::create(b, loc, i64, 1, 1); + weightBank = + BankMvinOp::create(b, loc, i64, weightSlice, weightBank, k, n / 16); + Value accumulator = BankAllocOp::create(b, loc, i64, 1, 1); + Value initialSlice = memref::SubViewOp::create( + b, loc, initial, offsets, + ArrayRef{b.getIndexAttr(1), b.getIndexAttr(16)}, + unit); + accumulator = + BankMvinOp::create(b, loc, i64, initialSlice, accumulator, 4, 1); + auto matmul = + BankCustomOp::create(b, loc, i64, ValueRange{codes, weightBank}, + accumulator, ValueRange{}, "matmul_i8", true); + matmul->setAttr("rows", b.getI64IntegerAttr(1)); + matmul->setAttr("k", b.getI64IntegerAttr(k)); + accumulator = matmul.getDestinationOut(); + BankReleaseOp::create(b, loc, weightBank); - Value resultBank = accumulator; - if (linear) { - Value scaleBank = BankAllocOp::create(builder, loc, i64, 1, 1); - scaleBank = - BankMvinOp::create(builder, loc, i64, scales, scaleBank, 4, 1); - if (nextMultiplier) { - auto quantized = BankCustomOp::create( - builder, loc, i64, ValueRange{accumulator, scaleBank}, - residentOutput, ValueRange{}, "int32_to_int8", true); - quantized->setAttr("rows", builder.getI64IntegerAttr(1)); - quantized->setAttr("wr_base", builder.getI64IntegerAttr(column / 16)); - quantized->setAttr("relu", builder.getBoolAttr(relu)); - residentOutput = quantized.getDestinationOut(); - } else { - resultBank = BankAllocOp::create(builder, loc, i64, 1, 1); - auto restored = BankCustomOp::create( - builder, loc, i64, ValueRange{accumulator, scaleBank}, resultBank, - ValueRange{}, "int32_to_fp32"); - restored->setAttr("rows", builder.getI64IntegerAttr(1)); - resultBank = restored.getDestinationOut(); - } - BankReleaseOp::create(builder, loc, scaleBank); - } - if (!nextMultiplier) { - Value outputSlice = memref::SubViewOp::create( - builder, loc, output, ArrayRef{0, column}, - ArrayRef{1, 16}, ArrayRef{1, 1}); - resultBank = - BankMvoutOp::create(builder, loc, i64, outputSlice, resultBank, 4, 1); - if (linear) - BankReleaseOp::create(builder, loc, resultBank); - } - BankReleaseOp::create(builder, loc, accumulator); - } + Value resultBank = accumulator; + if (linear) { + Value scaleBank = BankAllocOp::create(b, loc, i64, 1, 1); + scaleBank = BankMvinOp::create(b, loc, i64, scales, scaleBank, 4, 1); + resultBank = BankAllocOp::create(b, loc, i64, 1, 1); + auto restored = BankCustomOp::create( + b, loc, i64, ValueRange{accumulator, scaleBank}, resultBank, + ValueRange{}, "int32_to_fp32"); + restored->setAttr("rows", b.getI64IntegerAttr(1)); + restored->setAttr("relu", b.getBoolAttr(false)); + resultBank = restored.getDestinationOut(); + BankReleaseOp::create(b, loc, scaleBank); + } + Value outputSlice = memref::SubViewOp::create( + b, loc, output, offsets, + ArrayRef{b.getIndexAttr(1), b.getIndexAttr(16)}, + unit); + resultBank = + BankMvoutOp::create(b, loc, i64, outputSlice, resultBank, 4, 1); + if (linear) + BankReleaseOp::create(b, loc, resultBank); + BankReleaseOp::create(b, loc, accumulator); + return SmallVector{}; + }); BankReleaseOp::create(builder, loc, codes); - result = residentOutput; - if (!nextMultiplier) - result = bufferization::ToTensorOp::create( - builder, loc, op.getResult(0).getType(), output, true, true); + result = bufferization::ToTensorOp::create( + builder, loc, op.getResult(0).getType(), output, true, true); return success(); } @@ -226,57 +257,11 @@ class ConvertLinalgToBuckyballPass linalg::LinalgDialect>(); } void runOnOperation() override { - if (failed(pebble::lowerQuantizedChains(getOperation()))) - return signalPassFailure(); DenseMap residentVectors; SmallVector spatial; if (failed( pebble::lowerSpatialOps(getOperation(), residentVectors, spatial))) return signalPassFailure(); - llvm::MapVector> chains; - getOperation().walk([&](Operation *op) { - if (auto group = op->getAttrOfType("custom.chain")) - chains[group.getInt()].push_back(op); - }); - for (auto &[id, links] : chains) { - SmallVector<::buddy::custom::CustomOp> stages; - for (Operation *link : links) - if (auto op = dyn_cast<::buddy::custom::CustomOp>(link)) - if (op.getKind() == "linear_i8") - stages.push_back(op); - DenseMap resident; - Value input = residentVectors.lookup(stages.front().getInputs()[0]); - for (auto [index, stage] : llvm::enumerate(stages)) { - bool last = index + 1 == stages.size(); - Value multiplier = last ? Value() : stages[index + 1].getInputs()[3]; - bool relu = - !last && stage->getAttrOfType("custom.relu").getValue(); - Value lowered; - if (failed(lowerMatmul(stage, input, multiplier, relu, lowered))) - return signalPassFailure(); - if (last) - stage.getResult(0).replaceAllUsesWith(lowered); - else - resident[stage.getResult(0)] = input = lowered; - } - for (Operation *link : links) { - if (auto op = dyn_cast<::buddy::custom::CustomOp>(link)) { - if (op.getKind() == "relu") - resident[op.getResult(0)] = resident.at(op.getInputs()[0]); - } else { - // Cycle-only markers observe the resident bank without spilling it. - Value bank = resident.at(link->getOperand(0)); - resident[link->getResult(0)] = bank; - OpBuilder builder(link); - Operation *trace = builder.clone(*link); - trace->setOperand(0, bank); - trace->getResult(0).setType(bank.getType()); - trace->removeAttr("custom.chain"); - } - } - for (Operation *link : llvm::reverse(links)) - link->erase(); - } SmallVector<::buddy::custom::CustomOp> computations; getOperation().walk([&](::buddy::custom::CustomOp op) { if (!llvm::is_contained(spatial, op.getOperation())) @@ -286,10 +271,9 @@ class ConvertLinalgToBuckyballPass if (op.getKind() == "global_avg_i8" || op.getKind() == "lookup_i8" || op.getKind() == "gate_mul_i8") { Value result; - LogicalResult status = - op.getKind() == "global_avg_i8" - ? pebble::lowerGlobalAverage(op, result, false) - : pebble::lowerChannelOp(op, result); + LogicalResult status = op.getKind() == "global_avg_i8" + ? pebble::lowerGlobalAverage(op, result) + : pebble::lowerChannelOp(op, result); if (failed(status)) return signalPassFailure(); op.getResult(0).replaceAllUsesWith(result); @@ -314,7 +298,7 @@ class ConvertLinalgToBuckyballPass } Value result; if (failed(lowerMatmul(op, residentVectors.lookup(op.getInputs()[0]), - Value(), false, result))) + result))) return signalPassFailure(); op.getResult(0).replaceAllUsesWith(result); op.erase(); diff --git a/examples/designs/pebble/stack/midend/src/FormCustomOps.cpp b/examples/designs/pebble/stack/midend/src/FormCustomOps.cpp index 3728268d2..b15b98773 100644 --- a/examples/designs/pebble/stack/midend/src/FormCustomOps.cpp +++ b/examples/designs/pebble/stack/midend/src/FormCustomOps.cpp @@ -2,6 +2,7 @@ #include "Custom/CustomOps.h" #include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/Linalg/IR/Linalg.h" @@ -9,7 +10,6 @@ #include "mlir/IR/Dominance.h" #include "mlir/IR/Matchers.h" #include "mlir/IR/PatternMatch.h" -#include "mlir/Interfaces/SideEffectInterfaces.h" #include "mlir/Pass/Pass.h" using namespace mlir; @@ -184,10 +184,17 @@ class FormCustomOpsPass SmallVector transposes; getOperation().walk( [&](linalg::TransposeOp op) { transposes.push_back(op); }); + const auto &target = buckyball_target::getBuckyballTarget(); for (auto op : transposes) { + // TRANSPOSE holds the whole matrix in two groups of bank columns of at + // most 31 banks each. auto type = dyn_cast(op.getInput().getType()); - if (!type || type.getRank() != 2 || !type.getElementType().isInteger(8) || - op.getPermutation() != ArrayRef({1, 0})) + if (!type || type.getRank() != 2 || !type.hasStaticShape() || + !type.getElementType().isInteger(8) || + op.getPermutation() != ArrayRef({1, 0}) || + type.getDimSize(0) > target.bankDepth || type.getDimSize(1) % 16 || + type.getDimSize(1) / 16 > 31 || + 2 * type.getDimSize(1) / 16 > target.bankNum) continue; rewriter.setInsertionPoint(op); auto custom = @@ -196,98 +203,6 @@ class FormCustomOpsPass rewriter.replaceOp(op, custom.getOutputs()); } pebble::formSpatialOps(getOperation()); - pebble::formQuantizedChains(getOperation()); - - // Form single-consumer Linear chains. The links remain explicit custom - // operations; bank lowering consumes each group as one resident kernel. - SmallVector linears; - int64_t group = 0; - getOperation().walk([&](CustomOp op) { - if (op.getKind() == "linear_i8") - linears.push_back(op); - if (auto id = op->getAttrOfType("custom.chain")) - group = std::max(group, id.getInt() + 1); - }); - for (CustomOp root : linears) { - if (root->hasAttr("custom.chain")) - continue; - SmallVector stages{root}; - SmallVector links; - SmallVector activations; - while (true) { - Value value = stages.back().getResult(0); - SmallVector edge; - bool relu = false; - while (value.hasOneUse()) { - Operation *user = *value.getUsers().begin(); - if (user->getBlock() != root->getBlock()) - break; - if (user->getName().getStringRef() == "buddy_trace.end") { - auto tensor = user->getAttrOfType("tensor"); - if (!tensor || tensor.getValue()) - break; - } else { - auto maximum = dyn_cast(user); - if (relu || !pebble::elementwise(maximum, "arith.maximumf") || - !llvm::is_contained(maximum.getInputs(), value)) - break; - Value other = - maximum.getInputs()[maximum.getInputs()[0] == value ? 1 : 0]; - if (!matchPattern(other, m_AnyZeroFloat())) - break; - relu = true; - } - edge.push_back(user); - value = user->getResult(0); - } - if (!value.hasOneUse()) - break; - auto next = dyn_cast(*value.getUsers().begin()); - if (!next || next.getKind() != "linear_i8" || - next->getBlock() != root->getBlock() || - next.getInputs()[0] != value || next->hasAttr("custom.chain") || - !dominance.properlyDominates(next.getInputs()[3], stages.back())) - break; - bool barrier = false; - for (Operation *between = stages.back()->getNextNode(); between != next; - between = between->getNextNode()) { - if (!isMemoryEffectFree(between) && - between->getName().getDialectNamespace() != "buddy_trace") { - barrier = true; - break; - } - } - if (barrier) - break; - stages.push_back(next); - llvm::append_range(links, edge); - activations.push_back(relu); - } - if (stages.size() < 2) - continue; - auto id = rewriter.getI64IntegerAttr(group++); - for (auto [index, stage] : llvm::enumerate(stages)) { - stage->setAttr("custom.chain", id); - if (index < activations.size()) - stage->setAttr("custom.relu", - rewriter.getBoolAttr(activations[index])); - } - for (Operation *link : links) { - if (auto maximum = dyn_cast(link)) { - rewriter.setInsertionPoint(maximum); - Value input = matchPattern(maximum.getInputs()[0], m_AnyZeroFloat()) - ? maximum.getInputs()[1] - : maximum.getInputs()[0]; - auto activation = CustomOp::create(rewriter, maximum.getLoc(), - maximum->getResultTypes(), "relu", - ValueRange{input}); - activation->setAttr("custom.chain", id); - rewriter.replaceOp(maximum, activation.getOutputs()); - } else { - link->setAttr("custom.chain", id); - } - } - } } }; diff --git a/examples/designs/pebble/stack/midend/src/FormSpatialOps.cpp b/examples/designs/pebble/stack/midend/src/FormSpatialOps.cpp index 8620fad01..0112e43df 100644 --- a/examples/designs/pebble/stack/midend/src/FormSpatialOps.cpp +++ b/examples/designs/pebble/stack/midend/src/FormSpatialOps.cpp @@ -1,6 +1,7 @@ //===- FormSpatialOps.cpp - Recognize quantized convolution chains --------===// #include "Custom/CustomOps.h" #include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Linalg/IR/Linalg.h" #include "mlir/Dialect/Tensor/IR/Tensor.h" @@ -55,6 +56,27 @@ void pebble::formSpatialOps(func::FuncOp function) { if (!lhs || !rhs) continue; auto quant = frontend(lhs->getOperand(0), "frontend.fp32_to_int8"); + // IM2COL takes square kernels up to its size with unit dilation, and the + // array tiles a single image with one stride. + auto weightType = cast(rhs->getOperand(0).getType()); + auto inputType = cast(input.getType()); + auto outputType = cast(op->getResult(0).getType()); + auto strides = + llvm::to_vector(cast(op->getAttr("strides")) + .getValues()); + auto dilations = + llvm::to_vector(cast(op->getAttr("dilations")) + .getValues()); + if (!weightType.hasStaticShape() || !inputType.hasStaticShape() || + !outputType.hasStaticShape() || inputType.getDimSize(0) != 1 || + weightType.getDimSize(2) != weightType.getDimSize(3) || + weightType.getDimSize(2) > + buckyball_target::getBuckyballBallParam("IM2COL", "maxKSize") || + strides[0] != strides[1] || dilations != SmallVector{1, 1} || + !pebble::spatialTile(outputType.getDimSize(1), outputType.getDimSize(2), + 1, strides[0], weightType.getDimSize(2)) + .tile) + continue; Operation *end = op; Value biasValue; Operation *initial = op->getOperand(2).getDefiningOp(); @@ -116,14 +138,8 @@ void pebble::formSpatialOps(func::FuncOp function) { ValueRange{quant->getOperand(0), rhs->getOperand(0), bias->getOperand(0), quant->getOperand(1), lhs->getOperand(1), rhs->getOperand(1)}); - custom->setAttr("strides", - rewriter.getDenseI64ArrayAttr(llvm::to_vector( - cast(op->getAttr("strides")) - .getValues()))); - custom->setAttr("dilations", - rewriter.getDenseI64ArrayAttr(llvm::to_vector( - cast(op->getAttr("dilations")) - .getValues()))); + custom->setAttr("strides", rewriter.getDenseI64ArrayAttr(strides)); + custom->setAttr("dilations", rewriter.getDenseI64ArrayAttr(dilations)); custom->setAttr("padding", rewriter.getDenseI64ArrayAttr(padding)); rewriter.replaceOp(end, custom.getOutputs()); if (depthwise) { @@ -147,11 +163,16 @@ void pebble::formSpatialOps(func::FuncOp function) { roots.push_back(op); }); DominanceInfo dominance(function); + const auto &target = buckyball_target::getBuckyballTarget(); + // The banks of the resident images that chains leave for the next stage. + DenseMap imageBanks; int64_t group = 0; for (auto root : roots) { Value value = root.getResult(0); + auto outputType = cast(value.getType()); SmallVector links; bool relu = false, pool = false, flatten = false; + int64_t window = 1; while (value.hasOneUse()) { Operation *user = *value.getUsers().begin(); if (user->getBlock() != root->getBlock()) @@ -171,10 +192,19 @@ void pebble::formSpatialOps(func::FuncOp function) { break; relu = true; } else if (auto pooling = dyn_cast(user)) { + // MAXPOOL pools square windows that tile the convolution output. + auto filter = + cast(pooling.getInputs()[1].getType()).getShape(); if (pool || flatten || !llvm::all_of(pooling.getDilations().getValues(), - [](int64_t v) { return v == 1; })) + [](int64_t v) { return v == 1; }) || + filter[0] != filter[1] || + !llvm::all_of(pooling.getStrides().getValues(), + [&](int64_t v) { return v == filter[0]; }) || + outputType.getDimSize(1) % filter[0] || + outputType.getDimSize(2) % filter[0]) break; + window = filter[0]; Value initial = pooling.getOutputs()[0]; if (auto fill = initial.getDefiningOp()) initial = fill.getInputs()[0]; @@ -232,6 +262,33 @@ void pebble::formSpatialOps(func::FuncOp function) { } if (barrier) continue; + // The chain keeps every panel of its pooled output in a bank while it + // reads the resident image before it and works in seven more banks. + auto weightType = cast(root.getInputs()[1].getType()); + int64_t height = outputType.getDimSize(1) / window; + int64_t width = outputType.getDimSize(2) / window; + int64_t channels = weightType.getDimSize(0); + auto [tile, side] = pebble::spatialTile( + height, width, window, + cast(root->getAttr("strides"))[0], + weightType.getDimSize(2)); + if (!tile) + continue; + int64_t banks = (height + side - 1) / side * ((width + side - 1) / side) * + ((channels + 15) / 16); + if (imageBanks.lookup(root.getInputs()[0]) + banks + 7 > target.bankNum) + continue; + // A flattened image is one transposed 16x16 panel that feeds a resident + // one-row Linear. + if (flatten) { + auto linearWeight = cast(next.getInputs()[1].getType()); + if (height != 4 || width != 4 || channels != 16 || + linearWeight.getDimSize(0) > target.bankDepth || + linearWeight.getDimSize(1) % 16) + continue; + } else if (!terminalQuant) { + imageBanks[value] = banks; + } if (terminalQuant) links.push_back(consumer); auto id = rewriter.getI64IntegerAttr(group++); diff --git a/examples/designs/pebble/stack/midend/src/LowerLinearTiles.cpp b/examples/designs/pebble/stack/midend/src/LowerLinearTiles.cpp index 64fc2a16a..3414701be 100644 --- a/examples/designs/pebble/stack/midend/src/LowerLinearTiles.cpp +++ b/examples/designs/pebble/stack/midend/src/LowerLinearTiles.cpp @@ -49,192 +49,216 @@ LogicalResult pebble::lowerLinearTiles(::buddy::custom::CustomOp op, if (m <= 0 || k <= 0 || n <= 0 || target.bankWidthBits != 128 || maxK == 0 || rows * 4 > target.bankDepth) return op.emitOpError("Linear tile does not fit the configured banks"); - OpBuilder b(op); + OpBuilder builder(op); Location loc = op.getLoc(); - auto index = [&](int64_t n) -> Value { + Type i64 = builder.getI64Type(); + auto index = [&](OpBuilder &b, int64_t n) -> Value { return arith::ConstantIndexOp::create(b, loc, n); }; - Value zero = index(0), one = index(1); - Type i64 = b.getI64Type(); - Value input = tensorBuffer(b, loc, args[0]); - Value weight = tensorBuffer(b, loc, args[1]); - Value bias = tensorBuffer(b, loc, args[2]); + // Host loops over the valid part of a padded tile: min(size, limit - start). + auto extent = [&](OpBuilder &b, int64_t size, int64_t limit, + Value start) -> Value { + return arith::MinSIOp::create( + b, loc, index(b, size), + arith::SubIOp::create(b, loc, index(b, limit), start)); + }; + Value zero = index(builder, 0), one = index(builder, 1); + Value input = tensorBuffer(builder, loc, args[0]); + Value weight = tensorBuffer(builder, loc, args[1]); + Value bias = tensorBuffer(builder, loc, args[2]); Value step; if (!integer) - step = tensorBuffer(b, loc, args[requant ? 3 : 5]); + step = tensorBuffer(builder, loc, args[requant ? 3 : 5]); Value multiplier, inputStep; if (!requant && !integer) { - multiplier = - tensor::ExtractOp::create(b, loc, args[3], ValueRange{zero, zero}); - inputStep = - tensor::ExtractOp::create(b, loc, args[4], ValueRange{zero, zero}); + multiplier = tensor::ExtractOp::create(builder, loc, args[3], + ValueRange{zero, zero}); + inputStep = tensor::ExtractOp::create(builder, loc, args[4], + ValueRange{zero, zero}); } Value output = memref::AllocOp::create( - b, loc, MemRefType::get({m, n}, resultType.getElementType())); + builder, loc, MemRefType::get({m, n}, resultType.getElementType())); Value resultPack = memref::AllocOp::create( - b, loc, MemRefType::get({rows, 16}, resultType.getElementType())); - for (int64_t col = 0; col < n; col += 16) { - Value biasPack = - memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getI32Type())); - Value scalePack; - if (!integer) - scalePack = memref::AllocOp::create( - b, loc, MemRefType::get({16}, b.getF32Type())); - Value biasZero = arith::ConstantIntOp::create(b, loc, 0, 32); - linalg::FillOp::create(b, loc, biasZero, biasPack); - Value scales; - if (!integer) { - Value scaleOne = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(1)); - linalg::FillOp::create(b, loc, scaleOne, scalePack); - auto lanes = scf::ForOp::create( - b, loc, zero, index(std::min(16, n - col)), one); - { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(); - Value channel = arith::AddIOp::create(b, loc, index(col), lane); - Value value = - memref::LoadOp::create(b, loc, bias, ValueRange{zero, channel}); - memref::StoreOp::create(b, loc, value, biasPack, lane); - Value scale = - memref::LoadOp::create(b, loc, step, ValueRange{zero, channel}); - if (!requant) - scale = arith::MulFOp::create(b, loc, inputStep, scale); - memref::StoreOp::create(b, loc, scale, scalePack, lane); - } - scales = BankAllocOp::create(b, loc, i64, 1, 1); - scales = BankMvinOp::create(b, loc, i64, scalePack, scales, 4, 1); - } - for (int64_t row = 0; row < m; row += rows) { - if (integer) { - auto lanes = scf::ForOp::create( - b, loc, zero, index(std::min(16, n - col)), one); - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(); - Value channel = arith::AddIOp::create(b, loc, index(col), lane); - Value value = memref::LoadOp::create(b, loc, bias, - ValueRange{index(row), channel}); - memref::StoreOp::create(b, loc, value, biasPack, lane); - } - Value accumulator = BankAllocOp::create(b, loc, i64, 1, 1); - accumulator = - BankMvinOp::create(b, loc, i64, biasPack, accumulator, 4, 1); - for (int64_t start = 0; start < k; start += maxK) { - int64_t valid = std::min(maxK, k - start), - reduction = (valid + 15) / 16 * 16; - Value inputPack = memref::AllocOp::create( - b, loc, - MemRefType::get({reduction / 16 * rows, 16}, a.getElementType())); - Value weightPack = memref::AllocOp::create( - b, loc, MemRefType::get({reduction, 16}, b.getI8Type())); - Value fzero = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(0)); - Value izero = arith::ConstantIntOp::create(b, loc, 0, 8); - linalg::FillOp::create(b, loc, (requant || integer) ? izero : fzero, - inputPack); - linalg::FillOp::create(b, loc, izero, weightPack); - auto inner = scf::ForOp::create(b, loc, zero, index(valid), one); - { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(inner.getBody()); - Value reductionIndex = inner.getInductionVar(); - Value sourceK = - arith::AddIOp::create(b, loc, reductionIndex, index(start)); - Value block = arith::MulIOp::create( - b, loc, arith::DivUIOp::create(b, loc, reductionIndex, index(16)), - index(rows)); - Value lane = - arith::RemUIOp::create(b, loc, reductionIndex, index(16)); - auto inputRows = scf::ForOp::create( - b, loc, zero, index(std::min(rows, m - row)), one); + builder, loc, MemRefType::get({rows, 16}, resultType.getElementType())); + Value biasPack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getI32Type())); + Value scalePack; + if (!integer) + scalePack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getF32Type())); + + pebble::repeat( + builder, loc, 0, n, 16, false, {}, + [&](OpBuilder &b, Value col, ValueRange) { + Value columns = extent(b, 16, n, col); + linalg::FillOp::create( + b, loc, ValueRange{arith::ConstantIntOp::create(b, loc, 0, 32)}, + biasPack); + Value scales; + if (!integer) { + linalg::FillOp::create(b, loc, + ValueRange{arith::ConstantOp::create( + b, loc, b.getF32FloatAttr(1))}, + scalePack); + auto lanes = scf::ForOp::create(b, loc, zero, columns, one); { - OpBuilder::InsertionGuard rowGuard(b); - b.setInsertionPointToStart(inputRows.getBody()); - Value r = inputRows.getInductionVar(); - Value sourceM = arith::AddIOp::create(b, loc, r, index(row)); - Value destination = arith::AddIOp::create(b, loc, block, r); - Value value = memref::LoadOp::create(b, loc, input, - ValueRange{sourceM, sourceK}); - memref::StoreOp::create(b, loc, value, inputPack, - ValueRange{destination, lane}); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value channel = arith::AddIOp::create(b, loc, col, lane); + Value value = + memref::LoadOp::create(b, loc, bias, ValueRange{zero, channel}); + memref::StoreOp::create(b, loc, value, biasPack, lane); + Value scale = + memref::LoadOp::create(b, loc, step, ValueRange{zero, channel}); + if (!requant) + scale = arith::MulFOp::create(b, loc, inputStep, scale); + memref::StoreOp::create(b, loc, scale, scalePack, lane); } - auto columns = scf::ForOp::create( - b, loc, zero, index(std::min(16, n - col)), one); - b.setInsertionPointToStart(columns.getBody()); - Value c = columns.getInductionVar(); - Value sourceN = arith::AddIOp::create(b, loc, c, index(col)); - Value value = memref::LoadOp::create(b, loc, weight, - ValueRange{sourceK, sourceN}); - memref::StoreOp::create(b, loc, value, weightPack, - ValueRange{reductionIndex, c}); + scales = BankAllocOp::create(b, loc, i64, 1, 1); + scales = BankMvinOp::create(b, loc, i64, scalePack, scales, 4, 1); } - Value codes = BankAllocOp::create(b, loc, i64, 1, 1); - if (requant || integer) { - codes = BankMvinOp::create(b, loc, i64, inputPack, codes, - rows * reduction / 16, 1); - } else { - Value fp = BankAllocOp::create(b, loc, i64, 1, 1); - fp = BankMvinOp::create(b, loc, i64, inputPack, fp, - rows * reduction / 4, 1); - auto quant = - BankCustomOp::create(b, loc, i64, ValueRange{fp}, codes, - ValueRange{multiplier}, "fp32_to_int8"); - quant->setAttr("rows", b.getI64IntegerAttr(rows * reduction / 4)); - codes = quant.getDestinationOut(); - BankReleaseOp::create(b, loc, fp); - } - Value weights = BankAllocOp::create(b, loc, i64, 1, 1); - weights = - BankMvinOp::create(b, loc, i64, weightPack, weights, reduction, 1); - auto matmul = BankCustomOp::create( - b, loc, i64, ValueRange{codes, weights}, accumulator, ValueRange{}, - "matmul_i8_block", true); - matmul->setAttr("rows", b.getI64IntegerAttr(rows)); - matmul->setAttr("k", b.getI64IntegerAttr(reduction)); - matmul->setAttr("first", b.getBoolAttr(start == 0)); - matmul->setAttr("last", b.getBoolAttr(start + valid == k)); - accumulator = matmul.getDestinationOut(); - BankReleaseOp::create(b, loc, codes); - BankReleaseOp::create(b, loc, weights); - } - Value restored = accumulator; - if (!integer) { - restored = BankAllocOp::create(b, loc, i64, 1, 1); - auto dequant = BankCustomOp::create( - b, loc, i64, ValueRange{accumulator, scales}, restored, - ValueRange{}, requant ? "int32_to_int8" : "int32_to_fp32"); - dequant->setAttr("rows", b.getI64IntegerAttr(rows)); - if (requant) { - dequant->setAttr("wr_base", b.getI64IntegerAttr(0)); - dequant->setAttr("relu", op->getAttr("relu")); - } - restored = dequant.getDestinationOut(); - } - BankMvoutOp::create(b, loc, i64, resultPack, restored, - rows * (requant ? 1 : 4), 1); - BankReleaseOp::create(b, loc, accumulator); - if (!integer) - BankReleaseOp::create(b, loc, restored); - auto outputRows = - scf::ForOp::create(b, loc, zero, index(std::min(rows, m - row)), one); - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(outputRows.getBody()); - Value r = outputRows.getInductionVar(); - auto columns = scf::ForOp::create( - b, loc, zero, index(std::min(16, n - col)), one); - b.setInsertionPointToStart(columns.getBody()); - Value c = columns.getInductionVar(); - Value value = - memref::LoadOp::create(b, loc, resultPack, ValueRange{r, c}); - memref::StoreOp::create( - b, loc, value, output, - ValueRange{arith::AddIOp::create(b, loc, r, index(row)), - arith::AddIOp::create(b, loc, c, index(col))}); - } - if (!integer) - BankReleaseOp::create(b, loc, scales); - } - result = - bufferization::ToTensorOp::create(b, loc, resultType, output, true, true); + pebble::repeat( + b, loc, 0, m, rows, false, {}, + [&](OpBuilder &b, Value row, ValueRange) { + Value valid = extent(b, rows, m, row); + if (integer) { + auto lanes = scf::ForOp::create(b, loc, zero, columns, one); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value channel = arith::AddIOp::create(b, loc, col, lane); + Value value = memref::LoadOp::create(b, loc, bias, + ValueRange{row, channel}); + memref::StoreOp::create(b, loc, value, biasPack, lane); + } + Value accumulator = BankAllocOp::create(b, loc, i64, 1, 1); + accumulator = + BankMvinOp::create(b, loc, i64, biasPack, accumulator, 4, 1); + accumulator = pebble::accumulate( + b, loc, k, maxK, false, accumulator, + [&](OpBuilder &b, Value start, int64_t count, bool first, + bool last, Value accumulator) -> Value { + int64_t reduction = (count + 15) / 16 * 16; + Value inputPack = memref::AllocOp::create( + b, loc, + MemRefType::get({reduction / 16 * rows, 16}, + a.getElementType())); + Value weightPack = memref::AllocOp::create( + b, loc, + MemRefType::get({reduction, 16}, b.getI8Type())); + Value fzero = + arith::ConstantOp::create(b, loc, b.getF32FloatAttr(0)); + Value izero = arith::ConstantIntOp::create(b, loc, 0, 8); + linalg::FillOp::create(b, loc, + (requant || integer) ? izero : fzero, + inputPack); + linalg::FillOp::create(b, loc, izero, weightPack); + auto inner = + scf::ForOp::create(b, loc, zero, index(b, count), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(inner.getBody()); + Value reductionIndex = inner.getInductionVar(); + Value sourceK = + arith::AddIOp::create(b, loc, reductionIndex, start); + Value block = arith::MulIOp::create( + b, loc, + arith::DivUIOp::create(b, loc, reductionIndex, + index(b, 16)), + index(b, rows)); + Value lane = arith::RemUIOp::create( + b, loc, reductionIndex, index(b, 16)); + auto inputRows = + scf::ForOp::create(b, loc, zero, valid, one); + { + OpBuilder::InsertionGuard rowGuard(b); + b.setInsertionPointToStart(inputRows.getBody()); + Value r = inputRows.getInductionVar(); + Value sourceM = arith::AddIOp::create(b, loc, r, row); + Value destination = + arith::AddIOp::create(b, loc, block, r); + Value value = memref::LoadOp::create( + b, loc, input, ValueRange{sourceM, sourceK}); + memref::StoreOp::create(b, loc, value, inputPack, + ValueRange{destination, lane}); + } + auto lanes = + scf::ForOp::create(b, loc, zero, columns, one); + b.setInsertionPointToStart(lanes.getBody()); + Value c = lanes.getInductionVar(); + Value sourceN = arith::AddIOp::create(b, loc, c, col); + Value value = memref::LoadOp::create( + b, loc, weight, ValueRange{sourceK, sourceN}); + memref::StoreOp::create(b, loc, value, weightPack, + ValueRange{reductionIndex, c}); + } + Value codes = BankAllocOp::create(b, loc, i64, 1, 1); + if (requant || integer) { + codes = BankMvinOp::create(b, loc, i64, inputPack, codes, + rows * reduction / 16, 1); + } else { + Value fp = BankAllocOp::create(b, loc, i64, 1, 1); + fp = BankMvinOp::create(b, loc, i64, inputPack, fp, + rows * reduction / 4, 1); + auto quant = BankCustomOp::create( + b, loc, i64, ValueRange{fp}, codes, + ValueRange{multiplier}, "fp32_to_int8"); + quant->setAttr("rows", + b.getI64IntegerAttr(rows * reduction / 4)); + codes = quant.getDestinationOut(); + BankReleaseOp::create(b, loc, fp); + } + Value weights = BankAllocOp::create(b, loc, i64, 1, 1); + weights = BankMvinOp::create(b, loc, i64, weightPack, + weights, reduction, 1); + auto matmul = BankCustomOp::create( + b, loc, i64, ValueRange{codes, weights}, accumulator, + ValueRange{}, "matmul_i8_block", true); + matmul->setAttr("rows", b.getI64IntegerAttr(rows)); + matmul->setAttr("k", b.getI64IntegerAttr(reduction)); + matmul->setAttr("first", b.getBoolAttr(first)); + matmul->setAttr("last", b.getBoolAttr(last)); + BankReleaseOp::create(b, loc, codes); + BankReleaseOp::create(b, loc, weights); + return matmul.getDestinationOut(); + }); + Value restored = accumulator; + if (!integer) { + restored = BankAllocOp::create(b, loc, i64, 1, 1); + auto dequant = BankCustomOp::create( + b, loc, i64, ValueRange{accumulator, scales}, restored, + ValueRange{}, requant ? "int32_to_int8" : "int32_to_fp32"); + dequant->setAttr("rows", b.getI64IntegerAttr(rows)); + dequant->setAttr("relu", requant ? op->getAttr("relu") + : b.getBoolAttr(false)); + restored = dequant.getDestinationOut(); + } + BankMvoutOp::create(b, loc, i64, resultPack, restored, + rows * (requant ? 1 : 4), 1); + BankReleaseOp::create(b, loc, accumulator); + if (!integer) + BankReleaseOp::create(b, loc, restored); + auto outputRows = scf::ForOp::create(b, loc, zero, valid, one); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(outputRows.getBody()); + Value r = outputRows.getInductionVar(); + auto lanes = scf::ForOp::create(b, loc, zero, columns, one); + b.setInsertionPointToStart(lanes.getBody()); + Value c = lanes.getInductionVar(); + Value value = + memref::LoadOp::create(b, loc, resultPack, ValueRange{r, c}); + memref::StoreOp::create( + b, loc, value, output, + ValueRange{arith::AddIOp::create(b, loc, r, row), + arith::AddIOp::create(b, loc, c, col)}); + return SmallVector{}; + }); + if (!integer) + BankReleaseOp::create(b, loc, scales); + return SmallVector{}; + }); + result = bufferization::ToTensorOp::create(builder, loc, resultType, output, + true, true); return success(); } diff --git a/examples/designs/pebble/stack/midend/src/LowerQuantizedOps.cpp b/examples/designs/pebble/stack/midend/src/LowerQuantizedOps.cpp index 4135be143..4b7c562df 100644 --- a/examples/designs/pebble/stack/midend/src/LowerQuantizedOps.cpp +++ b/examples/designs/pebble/stack/midend/src/LowerQuantizedOps.cpp @@ -12,7 +12,7 @@ using namespace mlir; using namespace ::buddy::buckyball; LogicalResult pebble::lowerGlobalAverage(::buddy::custom::CustomOp op, - Value &result, bool resident) { + Value &result) { auto type = cast(op.getInputs()[0].getType()); auto resultType = cast(op.getResult(0).getType()); if (!type.hasStaticShape() || type.getRank() != 4 || @@ -25,14 +25,12 @@ LogicalResult pebble::lowerGlobalAverage(::buddy::custom::CustomOp op, channels = type.getDimSize(3); int64_t pixels = height * width; int64_t capacity = buckyball_target::getBuckyballTarget().bankDepth / 16 * 16; - if (pixels <= 0 || channels <= 0 || capacity == 0) - return op.emitOpError("global average does not fit the configured banks"); OpBuilder b(op); Location loc = op.getLoc(); - auto index = [&](int64_t n) -> Value { + auto index = [&](OpBuilder &b, int64_t n) -> Value { return arith::ConstantIndexOp::create(b, loc, n); }; - Value zero = index(0), one = index(1); + Value zero = index(b, 0), one = index(b, 1); Type i64 = b.getI64Type(); float inputScale = cast(op->getAttr("input_scale")).getValueAsDouble(); @@ -40,12 +38,8 @@ LogicalResult pebble::lowerGlobalAverage(::buddy::custom::CustomOp op, cast(op->getAttr("output_scale")).getValueAsDouble(); float ratio = inputScale / (pixels * outputScale); Value input = tensorBuffer(b, loc, op.getInputs()[0]); - Value output, residentOutput; - if (resident) - residentOutput = BankAllocOp::create(b, loc, i64, 1, 1); - else - output = memref::AllocOp::create( - b, loc, MemRefType::get(resultType.getShape(), b.getI8Type())); + Value output = memref::AllocOp::create( + b, loc, MemRefType::get(resultType.getShape(), b.getI8Type())); Value bias = memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getI32Type())); Value scales = @@ -60,82 +54,84 @@ LogicalResult pebble::lowerGlobalAverage(::buddy::custom::CustomOp op, linalg::FillOp::create(b, loc, scale, scales); Value scaleBank = BankAllocOp::create(b, loc, i64, 1, 1); scaleBank = BankMvinOp::create(b, loc, i64, scales, scaleBank, 4, 1); - for (int64_t c = 0; c < channels; c += 16) { - Value acc = BankAllocOp::create(b, loc, i64, 1, 1); - acc = BankMvinOp::create(b, loc, i64, bias, acc, 4, 1); - for (int64_t first = 0; first < pixels; first += capacity) { - int64_t valid = std::min(capacity, pixels - first), - k = (valid + 15) / 16 * 16; - Value pack = memref::AllocOp::create( - b, loc, MemRefType::get({k, 16}, b.getI8Type())); - Value ones = - memref::AllocOp::create(b, loc, MemRefType::get({k}, b.getI8Type())); - linalg::FillOp::create(b, loc, izero, pack); - linalg::FillOp::create(b, loc, iOne, ones); - auto spatial = scf::ForOp::create(b, loc, zero, index(valid), one); - { + pebble::repeat( + b, loc, 0, channels, 16, false, {}, + [&](OpBuilder &b, Value c, ValueRange) -> SmallVector { + Value lanesEnd = arith::MinSIOp::create( + b, loc, index(b, 16), + arith::SubIOp::create(b, loc, index(b, channels), c)); + Value acc = BankAllocOp::create(b, loc, i64, 1, 1); + acc = BankMvinOp::create(b, loc, i64, bias, acc, 4, 1); + acc = pebble::accumulate( + b, loc, pixels, capacity, false, acc, + [&](OpBuilder &b, Value first, int64_t valid, bool isFirst, + bool isLast, Value acc) -> Value { + int64_t k = (valid + 15) / 16 * 16; + Value pack = memref::AllocOp::create( + b, loc, MemRefType::get({k, 16}, b.getI8Type())); + Value ones = memref::AllocOp::create( + b, loc, MemRefType::get({k}, b.getI8Type())); + linalg::FillOp::create(b, loc, izero, pack); + linalg::FillOp::create(b, loc, iOne, ones); + auto spatial = + scf::ForOp::create(b, loc, zero, index(b, valid), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(spatial.getBody()); + Value p = spatial.getInductionVar(); + Value source = arith::AddIOp::create(b, loc, p, first); + Value y = + arith::DivUIOp::create(b, loc, source, index(b, width)); + Value x = + arith::RemUIOp::create(b, loc, source, index(b, width)); + auto lanes = scf::ForOp::create(b, loc, zero, lanesEnd, one); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value channel = arith::AddIOp::create(b, loc, lane, c); + Value value = memref::LoadOp::create( + b, loc, input, ValueRange{zero, y, x, channel}); + memref::StoreOp::create(b, loc, value, pack, + ValueRange{p, lane}); + } + Value a = BankAllocOp::create(b, loc, i64, 1, 1); + a = BankMvinOp::create(b, loc, i64, ones, a, k / 16, 1); + Value w = BankAllocOp::create(b, loc, i64, 1, 1); + w = BankMvinOp::create(b, loc, i64, pack, w, k, 1); + auto sum = + BankCustomOp::create(b, loc, i64, ValueRange{a, w}, acc, + ValueRange{}, "matmul_i8_block", true); + sum->setAttr("rows", b.getI64IntegerAttr(1)); + sum->setAttr("k", b.getI64IntegerAttr(k)); + sum->setAttr("first", b.getBoolAttr(isFirst)); + sum->setAttr("last", b.getBoolAttr(isLast)); + BankReleaseOp::create(b, loc, a); + BankReleaseOp::create(b, loc, w); + return sum.getDestinationOut(); + }); + Value codes = BankAllocOp::create(b, loc, i64, 1, 1); + auto quant = + BankCustomOp::create(b, loc, i64, ValueRange{acc, scaleBank}, codes, + ValueRange{}, "int32_to_int8"); + quant->setAttr("rows", b.getI64IntegerAttr(1)); + quant->setAttr("relu", b.getBoolAttr(false)); + codes = quant.getDestinationOut(); + BankReleaseOp::create(b, loc, acc); + BankMvoutOp::create(b, loc, i64, values, codes, 1, 1); + BankReleaseOp::create(b, loc, codes); + auto lanes = scf::ForOp::create(b, loc, zero, lanesEnd, one); OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(spatial.getBody()); - Value p = spatial.getInductionVar(); - Value source = arith::AddIOp::create(b, loc, p, index(first)); - Value y = arith::DivUIOp::create(b, loc, source, index(width)); - Value x = arith::RemUIOp::create(b, loc, source, index(width)); - auto lanes = scf::ForOp::create( - b, loc, zero, index(std::min(16, channels - c)), one); b.setInsertionPointToStart(lanes.getBody()); Value lane = lanes.getInductionVar(); - Value channel = arith::AddIOp::create(b, loc, lane, index(c)); - Value value = memref::LoadOp::create(b, loc, input, - ValueRange{zero, y, x, channel}); - memref::StoreOp::create(b, loc, value, pack, ValueRange{p, lane}); - } - Value a = BankAllocOp::create(b, loc, i64, 1, 1); - a = BankMvinOp::create(b, loc, i64, ones, a, k / 16, 1); - Value w = BankAllocOp::create(b, loc, i64, 1, 1); - w = BankMvinOp::create(b, loc, i64, pack, w, k, 1); - auto sum = BankCustomOp::create(b, loc, i64, ValueRange{a, w}, acc, - ValueRange{}, "matmul_i8_block", true); - sum->setAttr("rows", b.getI64IntegerAttr(1)); - sum->setAttr("k", b.getI64IntegerAttr(k)); - sum->setAttr("first", b.getBoolAttr(first == 0)); - sum->setAttr("last", b.getBoolAttr(first + valid == pixels)); - acc = sum.getDestinationOut(); - BankReleaseOp::create(b, loc, a); - BankReleaseOp::create(b, loc, w); - } - Value codes = residentOutput; - if (!resident) - codes = BankAllocOp::create(b, loc, i64, 1, 1); - auto quant = - BankCustomOp::create(b, loc, i64, ValueRange{acc, scaleBank}, codes, - ValueRange{}, "int32_to_int8", resident); - quant->setAttr("rows", b.getI64IntegerAttr(1)); - quant->setAttr("wr_base", b.getI64IntegerAttr(resident ? c / 16 : 0)); - quant->setAttr("relu", b.getBoolAttr(false)); - codes = quant.getDestinationOut(); - BankReleaseOp::create(b, loc, acc); - if (resident) { - residentOutput = codes; - continue; - } - BankMvoutOp::create(b, loc, i64, values, codes, 1, 1); - BankReleaseOp::create(b, loc, codes); - auto lanes = scf::ForOp::create( - b, loc, zero, index(std::min(16, channels - c)), one); - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(); - Value value = memref::LoadOp::create(b, loc, values, lane); - memref::StoreOp::create( - b, loc, value, output, - ValueRange{zero, zero, zero, - arith::AddIOp::create(b, loc, lane, index(c))}); - } + Value value = memref::LoadOp::create(b, loc, values, lane); + memref::StoreOp::create( + b, loc, value, output, + ValueRange{zero, zero, zero, + arith::AddIOp::create(b, loc, lane, c)}); + return {}; + }); BankReleaseOp::create(b, loc, scaleBank); - result = residentOutput; - if (!resident) - result = bufferization::ToTensorOp::create(b, loc, resultType, output, true, - true); + result = + bufferization::ToTensorOp::create(b, loc, resultType, output, true, true); return success(); } @@ -159,10 +155,10 @@ LogicalResult pebble::lowerChannelOp(::buddy::custom::CustomOp op, int64_t capacity = buckyball_target::getBuckyballTarget().bankDepth; OpBuilder b(op); Location loc = op.getLoc(); - auto index = [&](int64_t n) -> Value { + auto index = [&](OpBuilder &b, int64_t n) -> Value { return arith::ConstantIndexOp::create(b, loc, n); }; - Value zero = index(0), one = index(1); + Value zero = index(b, 0), one = index(b, 1); Type i64 = b.getI64Type(); Value input = tensorBuffer(b, loc, op.getInputs()[lookup ? 0 : 1]); Value other = tensorBuffer(b, loc, op.getInputs()[lookup ? 1 : 0]); @@ -175,28 +171,29 @@ LogicalResult pebble::lowerChannelOp(::buddy::custom::CustomOp op, if (columns == 4) { Value packed = memref::AllocOp::create( b, loc, MemRefType::get({64, 64}, b.getI8Type())); - auto rows = scf::ForOp::create(b, loc, zero, index(64), one); + auto rows = scf::ForOp::create(b, loc, zero, index(b, 64), one); { OpBuilder::InsertionGuard guard(b); b.setInsertionPointToStart(rows.getBody()); Value row = rows.getInductionVar(); - auto cols = scf::ForOp::create(b, loc, zero, index(64), one); + auto cols = scf::ForOp::create(b, loc, zero, index(b, 64), one); b.setInsertionPointToStart(cols.getBody()); Value col = cols.getInductionVar(); Value channel = arith::AddIOp::create( b, loc, arith::MulIOp::create( - b, loc, arith::DivUIOp::create(b, loc, col, index(16)), - index(4)), - arith::DivUIOp::create(b, loc, row, index(16))); + b, loc, arith::DivUIOp::create(b, loc, col, index(b, 16)), + index(b, 4)), + arith::DivUIOp::create(b, loc, row, index(b, 16))); Value entry = arith::AddIOp::create( b, loc, arith::MulIOp::create( - b, loc, arith::RemUIOp::create(b, loc, row, index(16)), - index(16)), - arith::RemUIOp::create(b, loc, col, index(16))); + b, loc, arith::RemUIOp::create(b, loc, row, index(b, 16)), + index(b, 16)), + arith::RemUIOp::create(b, loc, col, index(b, 16))); Value offset = arith::AddIOp::create( - b, loc, arith::MulIOp::create(b, loc, channel, index(256)), entry); + b, loc, arith::MulIOp::create(b, loc, channel, index(b, 256)), + entry); Value value = memref::LoadOp::create(b, loc, other, offset); memref::StoreOp::create(b, loc, value, packed, ValueRange{row, col}); } @@ -209,78 +206,91 @@ LogicalResult pebble::lowerChannelOp(::buddy::custom::CustomOp op, ratio = arith::ConstantOp::create(b, loc, cast(op->getAttr("ratio"))); } - for (int64_t c = 0; c < channels; c += 16) { - Value gateBank; - if (!lookup) { - Value gate = - memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getI8Type())); - linalg::FillOp::create(b, loc, izero, gate); - auto lanes = scf::ForOp::create( - b, loc, zero, index(std::min(16, channels - c)), one); - { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(); - Value value = memref::LoadOp::create( - b, loc, other, - ValueRange{zero, arith::AddIOp::create(b, loc, lane, index(c))}); - memref::StoreOp::create(b, loc, value, gate, lane); - } - gateBank = BankAllocOp::create(b, loc, i64, 1, 1); - gateBank = BankMvinOp::create(b, loc, i64, gate, gateBank, 1, 1); - } - for (int64_t start = 0; start < rows; start += capacity) { - int64_t count = std::min(capacity, rows - start); - Value pack = memref::AllocOp::create( - b, loc, MemRefType::get({count, 16}, b.getI8Type())); - linalg::FillOp::create(b, loc, izero, pack); - auto spatial = scf::ForOp::create(b, loc, zero, index(count), one); - { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(spatial.getBody()); - Value r = spatial.getInductionVar(); - auto lanes = scf::ForOp::create( - b, loc, zero, index(std::min(16, channels - c)), one); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(); - Value value = memref::LoadOp::create( - b, loc, input, - ValueRange{arith::AddIOp::create(b, loc, r, index(start)), - arith::AddIOp::create(b, loc, lane, index(c))}); - memref::StoreOp::create(b, loc, value, pack, ValueRange{r, lane}); - } - Value source = BankAllocOp::create(b, loc, i64, 1, 1); - source = BankMvinOp::create(b, loc, i64, pack, source, count, 1); - Value destination = BankAllocOp::create(b, loc, i64, 1, 1); - auto operation = BankCustomOp::create( - b, loc, i64, - lookup ? ValueRange{source, tableBank} : ValueRange{gateBank, source}, - destination, lookup ? ValueRange{} : ValueRange{ratio}, - lookup ? "lookup_i8" : "gate_mul_i8"); - operation->setAttr("rows", b.getI64IntegerAttr(count)); - if (!lookup) - operation->setAttr("gate_row", b.getI64IntegerAttr(0)); - destination = operation.getDestinationOut(); - BankMvoutOp::create(b, loc, i64, pack, destination, count, 1); - BankReleaseOp::create(b, loc, source); - BankReleaseOp::create(b, loc, destination); - auto outputRows = scf::ForOp::create(b, loc, zero, index(count), one); - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(outputRows.getBody()); - Value r = outputRows.getInductionVar(); - auto lanes = scf::ForOp::create( - b, loc, zero, index(std::min(16, channels - c)), one); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(); - Value value = memref::LoadOp::create(b, loc, pack, ValueRange{r, lane}); - memref::StoreOp::create( - b, loc, value, output, - ValueRange{arith::AddIOp::create(b, loc, r, index(start)), - arith::AddIOp::create(b, loc, lane, index(c))}); - } - if (!lookup) - BankReleaseOp::create(b, loc, gateBank); - } + pebble::repeat( + b, loc, 0, channels, 16, false, {}, + [&](OpBuilder &b, Value c, ValueRange) { + Value lanesEnd = arith::MinSIOp::create( + b, loc, index(b, 16), + arith::SubIOp::create(b, loc, index(b, channels), c)); + Value gateBank; + if (!lookup) { + Value gate = memref::AllocOp::create( + b, loc, MemRefType::get({16}, b.getI8Type())); + linalg::FillOp::create(b, loc, izero, gate); + auto lanes = scf::ForOp::create(b, loc, zero, lanesEnd, one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value value = memref::LoadOp::create( + b, loc, other, + ValueRange{zero, arith::AddIOp::create(b, loc, lane, c)}); + memref::StoreOp::create(b, loc, value, gate, lane); + } + gateBank = BankAllocOp::create(b, loc, i64, 1, 1); + gateBank = BankMvinOp::create(b, loc, i64, gate, gateBank, 1, 1); + } + // Full chunks of `capacity` rows share one loop; the remaining rows + // follow. + auto chunk = [&](OpBuilder &b, Value start, int64_t count) { + Value pack = memref::AllocOp::create( + b, loc, MemRefType::get({count, 16}, b.getI8Type())); + linalg::FillOp::create(b, loc, izero, pack); + auto spatial = scf::ForOp::create(b, loc, zero, index(b, count), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(spatial.getBody()); + Value r = spatial.getInductionVar(); + auto lanes = scf::ForOp::create(b, loc, zero, lanesEnd, one); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value value = memref::LoadOp::create( + b, loc, input, + ValueRange{arith::AddIOp::create(b, loc, r, start), + arith::AddIOp::create(b, loc, lane, c)}); + memref::StoreOp::create(b, loc, value, pack, ValueRange{r, lane}); + } + Value source = BankAllocOp::create(b, loc, i64, 1, 1); + source = BankMvinOp::create(b, loc, i64, pack, source, count, 1); + Value destination = BankAllocOp::create(b, loc, i64, 1, 1); + auto operation = BankCustomOp::create( + b, loc, i64, + lookup ? ValueRange{source, tableBank} + : ValueRange{gateBank, source}, + destination, lookup ? ValueRange{} : ValueRange{ratio}, + lookup ? "lookup_i8" : "gate_mul_i8"); + operation->setAttr("rows", b.getI64IntegerAttr(count)); + destination = operation.getDestinationOut(); + BankMvoutOp::create(b, loc, i64, pack, destination, count, 1); + BankReleaseOp::create(b, loc, source); + BankReleaseOp::create(b, loc, destination); + auto outputRows = + scf::ForOp::create(b, loc, zero, index(b, count), one); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(outputRows.getBody()); + Value r = outputRows.getInductionVar(); + auto lanes = scf::ForOp::create(b, loc, zero, lanesEnd, one); + b.setInsertionPointToStart(lanes.getBody()); + Value lane = lanes.getInductionVar(); + Value value = + memref::LoadOp::create(b, loc, pack, ValueRange{r, lane}); + memref::StoreOp::create( + b, loc, value, output, + ValueRange{arith::AddIOp::create(b, loc, r, start), + arith::AddIOp::create(b, loc, lane, c)}); + }; + int64_t full = rows / capacity * capacity; + pebble::repeat(b, loc, 0, full, capacity, false, {}, + [&](OpBuilder &b, Value start, ValueRange) { + chunk(b, start, capacity); + return SmallVector{}; + }); + if (rows > full) + chunk(b, index(b, full), rows - full); + if (!lookup) + BankReleaseOp::create(b, loc, gateBank); + return SmallVector{}; + }); if (lookup) BankReleaseOp::create(b, loc, tableBank); result = bufferization::ToTensorOp::create(b, loc, type, output, true, true); @@ -291,14 +301,11 @@ LogicalResult pebble::lowerTranspose(::buddy::custom::CustomOp op, Value &result) { auto type = cast(op.getInputs()[0].getType()); auto outputType = cast(op.getResult(0).getType()); - const auto &target = buckyball_target::getBuckyballTarget(); if (!type.hasStaticShape() || type.getRank() != 2 || !type.getElementType().isInteger(8) || outputType.getShape() != - ArrayRef({type.getDimSize(1), type.getDimSize(0)}) || - type.getDimSize(1) % 16 || type.getDimSize(0) > target.bankDepth) - return op.emitOpError( - "transpose_i8 requires a static matrix fitting complete bank columns"); + ArrayRef({type.getDimSize(1), type.getDimSize(0)})) + return op.emitOpError("transpose_i8 requires a static INT8 matrix"); OpBuilder b(op); Location loc = op.getLoc(); Value input = tensorBuffer(b, loc, op.getInputs()[0]); diff --git a/examples/designs/pebble/stack/midend/src/LowerSpatialOps.cpp b/examples/designs/pebble/stack/midend/src/LowerSpatialOps.cpp index 553fadc47..b645d79af 100644 --- a/examples/designs/pebble/stack/midend/src/LowerSpatialOps.cpp +++ b/examples/designs/pebble/stack/midend/src/LowerSpatialOps.cpp @@ -9,6 +9,7 @@ #include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/Dialect/Utils/StaticValueUtils.h" using namespace mlir; using namespace ::buddy::buckyball; @@ -42,108 +43,260 @@ LogicalResult convolution(Computation op, const ImageBanks *inputImage, auto weightType = cast(args[1].getType()); auto outputType = cast(op.getResult(0).getType()); auto strides = cast(op->getAttr("strides")).asArrayRef(); - auto dilation = - cast(op->getAttr("dilations")).asArrayRef(); auto padding = cast(op->getAttr("padding")).asArrayRef(); if (args.size() != 6 || !inputType.hasStaticShape() || inputType.getRank() != 4 || !weightType.hasStaticShape() || weightType.getRank() != 4 || inputType.getDimSize(0) != 1 || !inputType.getElementType().isF32() || !weightType.getElementType().isInteger(8) || - !cast(args[2].getType()) - .getElementType() - .isInteger(32) || - strides[0] != strides[1] || dilation != ArrayRef({1, 1})) + !cast(args[2].getType()).getElementType().isInteger(32)) return op.emitOpError("requires static NHWC/FCHW convolution with INT32 " - "bias and unit dilation"); + "bias"); + // Recognition admits only geometries that fit the Balls and the banks. int64_t cin = inputType.getDimSize(3), cout = weightType.getDimSize(0); int64_t kernel = weightType.getDimSize(2), stride = strides[0]; int64_t height = outputType.getDimSize(1) / pool; int64_t width = outputType.getDimSize(2) / pool; - if (kernel != weightType.getDimSize(3) || - (depthwise ? (weightType.getDimSize(1) != 1 || cout != cin) - : cin != weightType.getDimSize(1)) || - outputType.getDimSize(1) % pool || outputType.getDimSize(2) % pool || - kernel > buckyball_target::getBuckyballBallParam("IM2COL", "maxKSize")) - return op.emitOpError("has incompatible convolution or pooling geometry"); - int64_t tile = 4 / pool; - while (tile > 0 && (height % tile || width % tile || - ((tile * pool - 1) * stride + kernel) > 8)) - --tile; - if (!tile) - return op.emitOpError("convolution window exceeds addressable bank rows"); + auto [tile, side] = pebble::spatialTile(height, width, pool, stride, kernel); int64_t convSide = tile * pool; int64_t inputSide = (convSide - 1) * stride + kernel; int64_t arrayRows = buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); int64_t rows = (convSide * convSide + arrayRows - 1) / arrayRows * arrayRows; int64_t reduction = (kernel * kernel + 15) / 16 * 16; - int64_t side = std::min(8, std::min(height, width)); - while (side % tile) - --side; int64_t panelsY = (height + side - 1) / side; int64_t panelsX = (width + side - 1) / side; - const auto &target = buckyball_target::getBuckyballTarget(); int64_t outputBanks = nextMultiplier ? panelsY * panelsX * ((cout + 15) / 16) : 0; - int64_t inputBanks = inputImage ? inputImage->banks.size() : 0; - if (target.bankWidthBits != 128 || - inputSide * inputSide * 4 > target.bankDepth || - rows * 4 > target.bankDepth || reduction > target.bankDepth || - inputBanks + outputBanks + 7 > target.bankNum) - return op.emitOpError("spatial chain exceeds the configured bank capacity"); + // A resident image selects a bank and bank rows per pixel, so its loops + // unroll; recognition bounds a resident image by the bank count. + bool resident = inputImage || nextMultiplier; - OpBuilder b(op); + OpBuilder builder(op); Location loc = op.getLoc(); - Type i64 = b.getI64Type(); - auto index = [&](int64_t n) -> Value { + Type i64 = builder.getI64Type(); + auto index = [&](OpBuilder &b, int64_t n) -> Value { return arith::ConstantIndexOp::create(b, loc, n); }; - Value zero = index(0), one = index(1); - Value fzero = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(0)); - Value izero = arith::ConstantIntOp::create(b, loc, 0, 8); + auto constant = [](Value value) { return *getConstantIntValue(value); }; + Value zero = index(builder, 0), one = index(builder, 1); + Value fzero = + arith::ConstantOp::create(builder, loc, builder.getF32FloatAttr(0)); + Value izero = arith::ConstantIntOp::create(builder, loc, 0, 8); Value input; if (!inputImage) - input = pebble::tensorBuffer(b, loc, args[0]); - Value weight = pebble::tensorBuffer(b, loc, args[1]); - Value bias = pebble::tensorBuffer(b, loc, args[2]); - Value weightStep = pebble::tensorBuffer(b, loc, args[5]); - Value step = tensor::ExtractOp::create(b, loc, args[4], + input = pebble::tensorBuffer(builder, loc, args[0]); + Value weight = pebble::tensorBuffer(builder, loc, args[1]); + Value bias = pebble::tensorBuffer(builder, loc, args[2]); + Value weightStep = pebble::tensorBuffer(builder, loc, args[5]); + Value step = tensor::ExtractOp::create(builder, loc, args[4], ValueRange{zero, zero, zero, zero}); + Value inputMultiplier; + if (!inputImage) + inputMultiplier = tensor::ExtractOp::create( + builder, loc, args[3], ValueRange{zero, zero, zero, zero}); Value multiplier; if (nextMultiplier) { SmallVector indices( cast(nextMultiplier.getType()).getRank(), zero); - multiplier = tensor::ExtractOp::create(b, loc, nextMultiplier, indices); + multiplier = + tensor::ExtractOp::create(builder, loc, nextMultiplier, indices); } outputImage = {{}, height, width, cout, side}; for (int64_t i = 0; i < outputBanks; ++i) - outputImage.banks.push_back(BankAllocOp::create(b, loc, i64, 1, 1)); + outputImage.banks.push_back(BankAllocOp::create(builder, loc, i64, 1, 1)); Value output; if (!nextMultiplier) output = memref::AllocOp::create( - b, loc, MemRefType::get(outputType.getShape(), b.getF32Type())); + builder, loc, + MemRefType::get(outputType.getShape(), builder.getF32Type())); Value zeroPanel = memref::AllocOp::create( - b, loc, MemRefType::get({inputSide * inputSide, 16}, b.getI8Type())); - linalg::FillOp::create(b, loc, izero, zeroPanel); + builder, loc, + MemRefType::get({inputSide * inputSide, 16}, builder.getI8Type())); + linalg::FillOp::create(builder, loc, izero, zeroPanel); Value floatPanel; if (!inputImage) floatPanel = memref::AllocOp::create( - b, loc, MemRefType::get({inputSide * inputSide, 16}, b.getF32Type())); + builder, loc, + MemRefType::get({inputSide * inputSide, 16}, builder.getF32Type())); Value outputPack; if (!nextMultiplier) outputPack = memref::AllocOp::create( - b, loc, MemRefType::get({rows, 16}, b.getF32Type())); + builder, loc, MemRefType::get({rows, 16}, builder.getF32Type())); + // Per output-channel group: the weights of every term, the biases and the + // scales of its sixteen lanes. + Value weightPack = memref::AllocOp::create( + builder, loc, + MemRefType::get({depthwise ? 16 : cin, reduction, 16}, + builder.getI8Type())); + Value biasPack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getI32Type())); + Value scalePack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getF32Type())); + + // The terms of one input-channel group: sixteen lanes of one source panel. + // Im2col selects the lane as an attribute, so the lanes unroll. + auto group = [&](OpBuilder &b, Value oc, Value y, Value x, Value ic, + int64_t lanes, bool first, bool last, Value acc) -> Value { + Value source = BankAllocOp::create(b, loc, i64, 1, 1); + if (inputImage) { + source = BankMvinOp::create(b, loc, i64, zeroPanel, source, + inputSide * inputSide, 1); + int64_t inPanelsY = + (inputImage->height + inputImage->side - 1) / inputImage->side; + int64_t inPanelsX = + (inputImage->width + inputImage->side - 1) / inputImage->side; + for (int64_t iy = 0; iy < inputSide; ++iy) + for (int64_t ix = 0; ix < inputSide; ++ix) { + int64_t gy = constant(y) * pool * stride + iy - padding[0], + gx = constant(x) * pool * stride + ix - padding[2]; + if (gy < 0 || gx < 0 || gy >= inputImage->height || + gx >= inputImage->width) + continue; + int64_t panel = + (constant(ic) / 16 * inPanelsY + gy / inputImage->side) * + inPanelsX + + gx / inputImage->side; + int64_t base = + gy % inputImage->side * inputImage->side + gx % inputImage->side; + source = compute(b, loc, "maxpool_i8", + ValueRange{inputImage->banks[panel]}, source, true, + {{"input_side", 1}, + {"output_side", 1}, + {"kernel", 1}, + {"stride", 1}, + {"padding", 0}, + {"op1_base", base}, + {"wr_base", iy * inputSide + ix}, + {"output_stride", 1}, + {"start_row", 0}, + {"start_col", 0}}) + .getDestinationOut(); + } + } else { + linalg::FillOp::create(b, loc, fzero, floatPanel); + Value rowStart = arith::SubIOp::create( + b, loc, arith::MulIOp::create(b, loc, y, index(b, pool * stride)), + index(b, padding[0])); + Value colStart = arith::SubIOp::create( + b, loc, arith::MulIOp::create(b, loc, x, index(b, pool * stride)), + index(b, padding[2])); + Value channels = arith::MinSIOp::create( + b, loc, index(b, 16), + arith::SubIOp::create(b, loc, index(b, cin), ic)); + auto pixels = scf::ForOp::create(b, loc, zero, + index(b, inputSide * inputSide), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(pixels.getBody()); + Value pixel = pixels.getInductionVar(); + Value iy = arith::AddIOp::create( + b, loc, arith::DivUIOp::create(b, loc, pixel, index(b, inputSide)), + rowStart); + Value ix = arith::AddIOp::create( + b, loc, arith::RemUIOp::create(b, loc, pixel, index(b, inputSide)), + colStart); + auto lanesLoop = scf::ForOp::create(b, loc, zero, channels, one); + b.setInsertionPointToStart(lanesLoop.getBody()); + Value lane = lanesLoop.getInductionVar(); + Value channel = arith::AddIOp::create(b, loc, lane, ic); + Value inBounds = arith::AndIOp::create( + b, loc, + arith::AndIOp::create( + b, loc, + arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sge, iy, + zero), + arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt, iy, + index(b, inputType.getDimSize(1)))), + arith::AndIOp::create( + b, loc, + arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sge, ix, + zero), + arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt, ix, + index(b, inputType.getDimSize(2))))); + auto valid = scf::IfOp::create(b, loc, inBounds, false); + b.setInsertionPointToStart(&valid.getThenRegion().front()); + Value value = memref::LoadOp::create(b, loc, input, + ValueRange{zero, iy, ix, channel}); + memref::StoreOp::create(b, loc, value, floatPanel, + ValueRange{pixel, lane}); + } + Value fp = BankAllocOp::create(b, loc, i64, 1, 1); + fp = BankMvinOp::create(b, loc, i64, floatPanel, fp, + inputSide * inputSide * 4, 1); + auto q = + BankCustomOp::create(b, loc, i64, ValueRange{fp}, source, + ValueRange{inputMultiplier}, "fp32_to_int8"); + q->setAttr("rows", b.getI64IntegerAttr(inputSide * inputSide * 4)); + source = q.getDestinationOut(); + BankReleaseOp::create(b, loc, fp); + } + for (int64_t lane = 0; lane < lanes; ++lane) { + Value patch = BankAllocOp::create(b, loc, i64, 1, 1); + patch = compute(b, loc, "im2col_i8", ValueRange{source}, patch, false, + {{"input_side", inputSide}, + {"kernel", kernel}, + {"stride", stride}, + {"padding", 0}, + {"start_row", 0}, + {"start_col", 0}, + {"op1_base", 0}, + {"lane", lane}, + {"window_start", 0}, + {"window_count", convSide * convSide}}) + .getDestinationOut(); + Value packed = BankAllocOp::create(b, loc, i64, 1, 1); + // Im2col packs M16/K16, while this array consumes tileRows/K16. + for (int64_t m = 0; m < rows; m += arrayRows) + for (int64_t k = 0; k < reduction / 16; ++k) + for (int64_t r = 0; r < arrayRows; r += 4) + packed = + compute( + b, loc, "maxpool_i8", ValueRange{patch}, packed, true, + {{"input_side", 2}, + {"output_side", 2}, + {"kernel", 1}, + {"stride", 1}, + {"padding", 0}, + {"op1_base", k * 16 + m + r}, + {"wr_base", + (m / arrayRows * (reduction / 16) + k) * arrayRows + r}, + {"output_stride", 2}, + {"start_row", 0}, + {"start_col", 0}}) + .getDestinationOut(); + BankReleaseOp::create(b, loc, patch); + // The weights of this term: its lane for depthwise, its input channel + // otherwise. + Value term = + depthwise + ? index(b, lane) + : arith::AddIOp::create(b, loc, ic, index(b, lane)).getResult(); + Value w = BankAllocOp::create(b, loc, i64, 1, 1); + Value slice = memref::SubViewOp::create( + b, loc, weightPack, + ArrayRef{term, b.getIndexAttr(0), b.getIndexAttr(0)}, + ArrayRef{b.getIndexAttr(1), b.getIndexAttr(reduction), + b.getIndexAttr(16)}, + ArrayRef{b.getIndexAttr(1), b.getIndexAttr(1), + b.getIndexAttr(1)}); + w = BankMvinOp::create(b, loc, i64, slice, w, reduction, 1); + auto matmul = compute(b, loc, "matmul_i8_block", ValueRange{packed, w}, + acc, true, {{"rows", rows}, {"k", reduction}}); + matmul->setAttr("first", b.getBoolAttr(first && lane == 0)); + matmul->setAttr("last", b.getBoolAttr(last && lane + 1 == lanes)); + acc = matmul.getDestinationOut(); + BankReleaseOp::create(b, loc, packed); + BankReleaseOp::create(b, loc, w); + } + BankReleaseOp::create(b, loc, source); + return acc; + }; - for (int64_t oc = 0; oc < cout; oc += 16) { - int64_t terms = depthwise ? std::min(16, cout - oc) : cin; - Value weightPack = memref::AllocOp::create( - b, loc, MemRefType::get({terms, reduction, 16}, b.getI8Type())); + // One output-channel group of `lanes` channels over every output tile. + auto outputChannels = [&](OpBuilder &b, Value oc, int64_t lanes) { + int64_t terms = depthwise ? lanes : cin; linalg::FillOp::create(b, loc, izero, weightPack); - Value biasPack = - memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getI32Type())); - Value scalePack = - memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getF32Type())); linalg::FillOp::create( b, loc, ValueRange{arith::ConstantIntOp::create(b, loc, 0, 32)}, biasPack); @@ -151,241 +304,134 @@ LogicalResult convolution(Computation op, const ImageBanks *inputImage, b, loc, ValueRange{arith::ConstantOp::create(b, loc, b.getF32FloatAttr(1))}, scalePack); - for (int64_t lane = 0; lane < std::min(16, cout - oc); ++lane) { - Value channel = index(oc + lane), laneIndex = index(lane); + auto laneLoop = scf::ForOp::create(b, loc, zero, index(b, lanes), one); + { + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(laneLoop.getBody()); + Value lane = laneLoop.getInductionVar(); + Value channel = arith::AddIOp::create(b, loc, oc, lane); Value biasValue = memref::LoadOp::create(b, loc, bias, channel); - memref::StoreOp::create(b, loc, biasValue, biasPack, laneIndex); + memref::StoreOp::create(b, loc, biasValue, biasPack, lane); Value weightScale = memref::LoadOp::create( b, loc, weightStep, ValueRange{channel, zero, zero, zero}); Value scale = arith::MulFOp::create(b, loc, step, weightScale); if (nextMultiplier) scale = arith::MulFOp::create(b, loc, scale, multiplier); - memref::StoreOp::create(b, loc, scale, scalePack, laneIndex); - auto cLoop = scf::ForOp::create(b, loc, index(depthwise ? lane : 0), - index(depthwise ? lane + 1 : cin), one); - OpBuilder::InsertionGuard guard(b); + memref::StoreOp::create(b, loc, scale, scalePack, lane); + auto cLoop = scf::ForOp::create( + b, loc, depthwise ? lane : zero, + depthwise ? arith::AddIOp::create(b, loc, lane, one).getResult() + : index(b, cin), + one); b.setInsertionPointToStart(cLoop.getBody()); Value c = cLoop.getInductionVar(); auto kLoop = - scf::ForOp::create(b, loc, zero, index(kernel * kernel), one); + scf::ForOp::create(b, loc, zero, index(b, kernel * kernel), one); b.setInsertionPointToStart(kLoop.getBody()); Value k = kLoop.getInductionVar(); - Value ky = arith::DivUIOp::create(b, loc, k, index(kernel)); - Value kx = arith::RemUIOp::create(b, loc, k, index(kernel)); + Value ky = arith::DivUIOp::create(b, loc, k, index(b, kernel)); + Value kx = arith::RemUIOp::create(b, loc, k, index(b, kernel)); Value w = memref::LoadOp::create( b, loc, weight, ValueRange{channel, depthwise ? zero : c, ky, kx}); - memref::StoreOp::create(b, loc, w, weightPack, - ValueRange{c, k, laneIndex}); + memref::StoreOp::create(b, loc, w, weightPack, ValueRange{c, k, lane}); } Value scales = BankAllocOp::create(b, loc, i64, 1, 1); scales = BankMvinOp::create(b, loc, i64, scalePack, scales, 4, 1); - for (int64_t y = 0; y < height; y += tile) { - for (int64_t x = 0; x < width; x += tile) { - Value acc = BankAllocOp::create(b, loc, i64, 1, 1); - acc = BankMvinOp::create(b, loc, i64, biasPack, acc, 4, 1); - Value source; - for (int64_t term = 0; term < terms; ++term) { - int64_t ic = depthwise ? oc + term : term; - if (ic % 16 == 0) { - source = BankAllocOp::create(b, loc, i64, 1, 1); - if (inputImage) { - source = BankMvinOp::create(b, loc, i64, zeroPanel, source, - inputSide * inputSide, 1); - int64_t inPanelsY = (inputImage->height + inputImage->side - 1) / - inputImage->side; - int64_t inPanelsX = - (inputImage->width + inputImage->side - 1) / inputImage->side; - for (int64_t iy = 0; iy < inputSide; ++iy) - for (int64_t ix = 0; ix < inputSide; ++ix) { - int64_t gy = y * pool * stride + iy - padding[0], - gx = x * pool * stride + ix - padding[2]; - if (gy < 0 || gx < 0 || gy >= inputImage->height || - gx >= inputImage->width) - continue; + pebble::repeat( + b, loc, 0, height, tile, resident, {}, + [&](OpBuilder &b, Value y, ValueRange) { + pebble::repeat( + b, loc, 0, width, tile, resident, {}, + [&](OpBuilder &b, Value x, ValueRange) { + Value acc = BankAllocOp::create(b, loc, i64, 1, 1); + acc = BankMvinOp::create(b, loc, i64, biasPack, acc, 4, 1); + // Depthwise reads the input channels of its own output group. + acc = pebble::accumulate( + b, loc, terms, 16, resident, acc, + [&](OpBuilder &b, Value start, int64_t count, bool first, + bool last, Value acc) { + return group(b, oc, y, x, depthwise ? oc : start, count, + first, last, acc); + }); + Value restored = BankAllocOp::create(b, loc, i64, 1, 1); + if (nextMultiplier) { + auto q = + compute(b, loc, "int32_to_int8", ValueRange{acc, scales}, + restored, false, {{"rows", rows}}); + q->setAttr("relu", b.getBoolAttr(relu)); + restored = q.getDestinationOut(); int64_t panel = - (ic / 16 * inPanelsY + gy / inputImage->side) * - inPanelsX + - gx / inputImage->side; - int64_t base = gy % inputImage->side * inputImage->side + - gx % inputImage->side; - source = compute(b, loc, "maxpool_i8", - ValueRange{inputImage->banks[panel]}, source, - true, - {{"input_side", 1}, - {"output_side", 1}, - {"kernel", 1}, - {"stride", 1}, - {"padding", 0}, - {"op1_base", base}, - {"wr_base", iy * inputSide + ix}, - {"output_stride", 1}, - {"start_row", 0}, - {"start_col", 0}}) - .getDestinationOut(); + (constant(oc) / 16 * panelsY + constant(y) / side) * + panelsX + + constant(x) / side; + outputImage.banks[panel] = + compute(b, loc, "maxpool_i8", ValueRange{restored}, + outputImage.banks[panel], true, + {{"input_side", convSide}, + {"output_side", tile}, + {"kernel", pool}, + {"stride", pool}, + {"padding", 0}, + {"op1_base", 0}, + {"wr_base", + constant(y) % side * side + constant(x) % side}, + {"output_stride", side}, + {"start_row", 0}, + {"start_col", 0}}) + .getDestinationOut(); + } else { + auto dequant = + compute(b, loc, "int32_to_fp32", ValueRange{acc, scales}, + restored, false, {{"rows", rows}}); + dequant->setAttr("relu", b.getBoolAttr(false)); + restored = dequant.getDestinationOut(); + restored = BankMvoutOp::create(b, loc, i64, outputPack, + restored, rows * 4, 1); + auto pixels = scf::ForOp::create(b, loc, zero, + index(b, tile * tile), one); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(pixels.getBody()); + Value pixel = pixels.getInductionVar(); + Value oy = arith::AddIOp::create( + b, loc, y, + arith::DivUIOp::create(b, loc, pixel, index(b, tile))); + Value ox = arith::AddIOp::create( + b, loc, x, + arith::RemUIOp::create(b, loc, pixel, index(b, tile))); + auto channels = + scf::ForOp::create(b, loc, zero, index(b, lanes), one); + b.setInsertionPointToStart(channels.getBody()); + Value c = channels.getInductionVar(); + Value value = memref::LoadOp::create(b, loc, outputPack, + ValueRange{pixel, c}); + memref::StoreOp::create( + b, loc, value, output, + ValueRange{zero, oy, ox, + arith::AddIOp::create(b, loc, c, oc)}); } - } else { - linalg::FillOp::create(b, loc, fzero, floatPanel); - auto pixels = scf::ForOp::create( - b, loc, zero, index(inputSide * inputSide), one); - { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(pixels.getBody()); - Value pixel = pixels.getInductionVar(); - Value iy = arith::AddIOp::create( - b, loc, - arith::DivUIOp::create(b, loc, pixel, index(inputSide)), - index(y * pool * stride - padding[0])); - Value ix = arith::AddIOp::create( - b, loc, - arith::RemUIOp::create(b, loc, pixel, index(inputSide)), - index(x * pool * stride - padding[2])); - auto channels = scf::ForOp::create( - b, loc, zero, index(std::min(16, cin - ic)), one); - b.setInsertionPointToStart(channels.getBody()); - Value lane = channels.getInductionVar(); - Value channel = arith::AddIOp::create(b, loc, lane, index(ic)); - Value inBounds = arith::AndIOp::create( - b, loc, - arith::AndIOp::create( - b, loc, - arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sge, - iy, zero), - arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt, - iy, - index(inputType.getDimSize(1)))), - arith::AndIOp::create( - b, loc, - arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sge, - ix, zero), - arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt, - ix, - index(inputType.getDimSize(2))))); - auto valid = scf::IfOp::create(b, loc, inBounds, false); - b.setInsertionPointToStart(&valid.getThenRegion().front()); - Value value = memref::LoadOp::create( - b, loc, input, ValueRange{zero, iy, ix, channel}); - memref::StoreOp::create(b, loc, value, floatPanel, - ValueRange{pixel, lane}); - } - Value fp = BankAllocOp::create(b, loc, i64, 1, 1); - fp = BankMvinOp::create(b, loc, i64, floatPanel, fp, - inputSide * inputSide * 4, 1); - Value inputMultiplier = tensor::ExtractOp::create( - b, loc, args[3], ValueRange{zero, zero, zero, zero}); - auto q = BankCustomOp::create(b, loc, i64, ValueRange{fp}, source, - ValueRange{inputMultiplier}, - "fp32_to_int8"); - q->setAttr("rows", - b.getI64IntegerAttr(inputSide * inputSide * 4)); - source = q.getDestinationOut(); - BankReleaseOp::create(b, loc, fp); - } - } - Value patch = BankAllocOp::create(b, loc, i64, 1, 1); - patch = compute(b, loc, "im2col_i8", ValueRange{source}, patch, false, - {{"input_side", inputSide}, - {"kernel", kernel}, - {"stride", stride}, - {"padding", 0}, - {"start_row", 0}, - {"start_col", 0}, - {"op1_base", 0}, - {"lane", ic % 16}, - {"window_start", 0}, - {"window_count", convSide * convSide}}) - .getDestinationOut(); - Value packed = BankAllocOp::create(b, loc, i64, 1, 1); - // Im2col packs M16/K16, while this array consumes tileRows/K16. - for (int64_t m = 0; m < rows; m += arrayRows) - for (int64_t k = 0; k < reduction / 16; ++k) - for (int64_t r = 0; r < arrayRows; r += 4) - packed = - compute(b, loc, "maxpool_i8", ValueRange{patch}, packed, - true, - {{"input_side", 2}, - {"output_side", 2}, - {"kernel", 1}, - {"stride", 1}, - {"padding", 0}, - {"op1_base", k * 16 + m + r}, - {"wr_base", (m / arrayRows * (reduction / 16) + - k) * arrayRows + - r}, - {"output_stride", 2}, - {"start_row", 0}, - {"start_col", 0}}) - .getDestinationOut(); - BankReleaseOp::create(b, loc, patch); - Value w = BankAllocOp::create(b, loc, i64, 1, 1); - Value slice = memref::SubViewOp::create( - b, loc, weightPack, ArrayRef{term, 0, 0}, - ArrayRef{1, reduction, 16}, ArrayRef{1, 1, 1}); - w = BankMvinOp::create(b, loc, i64, slice, w, reduction, 1); - auto matmul = - compute(b, loc, "matmul_i8_block", ValueRange{packed, w}, acc, - true, {{"rows", rows}, {"k", reduction}}); - matmul->setAttr("first", b.getBoolAttr(term == 0)); - matmul->setAttr("last", b.getBoolAttr(term + 1 == terms)); - acc = matmul.getDestinationOut(); - BankReleaseOp::create(b, loc, packed); - BankReleaseOp::create(b, loc, w); - if (ic % 16 == 15 || term + 1 == terms) - BankReleaseOp::create(b, loc, source); - } - Value restored = BankAllocOp::create(b, loc, i64, 1, 1); - if (nextMultiplier) { - auto q = compute(b, loc, "int32_to_int8", ValueRange{acc, scales}, - restored, false, {{"rows", rows}, {"wr_base", 0}}); - q->setAttr("relu", b.getBoolAttr(relu)); - restored = q.getDestinationOut(); - int64_t panel = (oc / 16 * panelsY + y / side) * panelsX + x / side; - outputImage.banks[panel] = - compute(b, loc, "maxpool_i8", ValueRange{restored}, - outputImage.banks[panel], true, - {{"input_side", convSide}, - {"output_side", tile}, - {"kernel", pool}, - {"stride", pool}, - {"padding", 0}, - {"op1_base", 0}, - {"wr_base", y % side * side + x % side}, - {"output_stride", side}, - {"start_row", 0}, - {"start_col", 0}}) - .getDestinationOut(); - } else { - restored = compute(b, loc, "int32_to_fp32", ValueRange{acc, scales}, - restored, false, {{"rows", rows}}) - .getDestinationOut(); - restored = BankMvoutOp::create(b, loc, i64, outputPack, restored, - rows * 4, 1); - for (int64_t iy = 0; iy < tile; ++iy) - for (int64_t ix = 0; ix < tile; ++ix) { - auto channels = scf::ForOp::create( - b, loc, zero, index(std::min(16, cout - oc)), one); - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(channels.getBody()); - Value c = channels.getInductionVar(); - Value value = memref::LoadOp::create( - b, loc, outputPack, ValueRange{index(iy * tile + ix), c}); - memref::StoreOp::create( - b, loc, value, output, - ValueRange{zero, index(y + iy), index(x + ix), - arith::AddIOp::create(b, loc, c, index(oc))}); - } - } - BankReleaseOp::create(b, loc, restored); - BankReleaseOp::create(b, loc, acc); - } - } + BankReleaseOp::create(b, loc, restored); + BankReleaseOp::create(b, loc, acc); + return SmallVector{}; + }); + return SmallVector{}; + }); BankReleaseOp::create(b, loc, scales); - } + }; + int64_t full = cout / 16 * 16; + pebble::repeat(builder, loc, 0, full, 16, resident, {}, + [&](OpBuilder &b, Value oc, ValueRange) { + outputChannels(b, oc, 16); + return SmallVector{}; + }); + if (cout > full) + outputChannels(builder, index(builder, full), cout - full); + if (inputImage) for (Value bank : inputImage->banks) - BankReleaseOp::create(b, loc, bank); + BankReleaseOp::create(builder, loc, bank); if (!nextMultiplier) - tensorOutput = bufferization::ToTensorOp::create(b, loc, outputType, output, - true, true); + tensorOutput = bufferization::ToTensorOp::create(builder, loc, outputType, + output, true, true); return success(); } } // namespace @@ -423,15 +469,8 @@ LogicalResult pebble::lowerSpatialOps(func::FuncOp function, if (auto op = dyn_cast(link)) { relu |= op.getKind() == "relu"; flatten |= op.getKind() == "flatten_nchw"; - if (op.getKind() == "maxpool2d") { - auto kernel = - cast(op->getAttr("kernel")).asArrayRef(); - auto strides = - cast(op->getAttr("strides")).asArrayRef(); - if (kernel[0] != kernel[1] || strides != kernel) - return op.emitOpError("requires square non-overlapping pooling"); - pool = kernel[0]; - } + if (op.getKind() == "maxpool2d") + pool = cast(op->getAttr("kernel"))[0]; } } } @@ -462,6 +501,7 @@ LogicalResult pebble::lowerSpatialOps(func::FuncOp function, auto index = [&](int64_t n) -> Value { return arith::ConstantIndexOp::create(b, loc, n); }; + Value zero = index(0), one = index(1); for (int64_t c = 0; c < result.channels; c += 16) for (int64_t y = 0; y < result.height; y += result.side) for (int64_t x = 0; x < result.width; x += result.side) { @@ -469,25 +509,35 @@ LogicalResult pebble::lowerSpatialOps(func::FuncOp function, x / result.side]; BankMvoutOp::create(b, loc, b.getI64Type(), pack, source, result.side * result.side, 1); - for (int64_t iy = 0; iy < std::min(result.side, result.height - y); - ++iy) - for (int64_t ix = 0; ix < std::min(result.side, result.width - x); - ++ix) { - auto channels = scf::ForOp::create( - b, loc, index(0), - index(std::min(16, result.channels - c)), - index(1)); - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(channels.getBody()); - Value lane = channels.getInductionVar(); - Value value = memref::LoadOp::create( - b, loc, pack, - ValueRange{index(iy * result.side + ix), lane}); - memref::StoreOp::create( - b, loc, value, memory, - ValueRange{index(0), index(y + iy), index(x + ix), - arith::AddIOp::create(b, loc, lane, index(c))}); - } + auto rows = scf::ForOp::create( + b, loc, zero, index(std::min(result.side, result.height - y)), + one); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(rows.getBody()); + Value iy = rows.getInductionVar(); + auto columns = scf::ForOp::create( + b, loc, zero, index(std::min(result.side, result.width - x)), + one); + b.setInsertionPointToStart(columns.getBody()); + Value ix = columns.getInductionVar(); + auto channels = scf::ForOp::create( + b, loc, zero, index(std::min(16, result.channels - c)), + one); + b.setInsertionPointToStart(channels.getBody()); + Value lane = channels.getInductionVar(); + Value value = memref::LoadOp::create( + b, loc, pack, + ValueRange{ + arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, iy, index(result.side)), + ix), + lane}); + memref::StoreOp::create( + b, loc, value, memory, + ValueRange{zero, arith::AddIOp::create(b, loc, iy, index(y)), + arith::AddIOp::create(b, loc, ix, index(x)), + arith::AddIOp::create(b, loc, lane, index(c))}); } tensor = bufferization::ToTensorOp::create(b, loc, type, memory, true, true); @@ -495,10 +545,6 @@ LogicalResult pebble::lowerSpatialOps(func::FuncOp function, for (Value source : result.banks) BankReleaseOp::create(b, loc, source); } else if (flatten) { - if (result.banks.size() != 1 || result.height * result.width != 16 || - result.channels != 16 || result.side != result.width) - return conv.emitOpError( - "resident NCHW flatten requires a 16x16 INT8 panel"); Value transposed = BankAllocOp::create(b, conv.getLoc(), b.getI64Type(), 1, 1); transposed = compute(b, conv.getLoc(), "transpose_i8", ValueRange{bank}, diff --git a/examples/designs/pebble/stack/midend/src/Lowering.h b/examples/designs/pebble/stack/midend/src/Lowering.h index cbd2b7024..13a0708ca 100644 --- a/examples/designs/pebble/stack/midend/src/Lowering.h +++ b/examples/designs/pebble/stack/midend/src/Lowering.h @@ -6,6 +6,7 @@ #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/IR/Builders.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLFunctionalExtras.h" namespace mlir::linalg { class GenericOp; @@ -14,16 +15,46 @@ namespace pebble { bool elementwise(mlir::linalg::GenericOp op, llvm::StringRef name); mlir::Value tensorBuffer(mlir::OpBuilder &builder, mlir::Location loc, mlir::Value tensor); + +// Emits `body` for index = begin, begin + step, ... < end and threads the +// carried banks through it. The iterations share one scf.for unless `unroll` +// is set: then each iteration gets a constant index, for bodies whose index +// selects a bank or sets a Ball attribute. +llvm::SmallVector +repeat(mlir::OpBuilder &builder, mlir::Location loc, int64_t begin, int64_t end, + int64_t step, bool unroll, mlir::ValueRange carried, + llvm::function_ref( + mlir::OpBuilder &, mlir::Value, mlir::ValueRange)> + body); + +// Emits `body` over [0, total) in pieces of `size` that accumulate into one +// bank. The first piece starts the accumulation and the last completes it; +// the pieces between go through `repeat`. +mlir::Value +accumulate(mlir::OpBuilder &builder, mlir::Location loc, int64_t total, + int64_t size, bool unroll, mlir::Value accumulator, + llvm::function_ref + body); + +// A convolution computes tile x tile pooled output pixels per matmul; a +// resident image keeps side x side pixels of sixteen channels per bank. The +// tile divides the output and its input window fits the eight-pixel side that +// MAXPOOL addresses. The tile is zero when none fits. +struct SpatialTile { + int64_t tile, side; +}; +SpatialTile spatialTile(int64_t height, int64_t width, int64_t pool, + int64_t stride, int64_t kernel); + mlir::LogicalResult lowerLinearTiles(::buddy::custom::CustomOp op, mlir::Value &result); mlir::LogicalResult lowerGlobalAverage(::buddy::custom::CustomOp op, - mlir::Value &result, bool resident); + mlir::Value &result); mlir::LogicalResult lowerChannelOp(::buddy::custom::CustomOp op, mlir::Value &result); mlir::LogicalResult lowerTranspose(::buddy::custom::CustomOp op, mlir::Value &result); -void formQuantizedChains(mlir::func::FuncOp function); -mlir::LogicalResult lowerQuantizedChains(mlir::func::FuncOp function); void formSpatialOps(mlir::func::FuncOp function); mlir::LogicalResult lowerSpatialOps(mlir::func::FuncOp function, diff --git a/examples/designs/pebble/stack/midend/src/QuantizedChains.cpp b/examples/designs/pebble/stack/midend/src/QuantizedChains.cpp deleted file mode 100644 index 0a4a47779..000000000 --- a/examples/designs/pebble/stack/midend/src/QuantizedChains.cpp +++ /dev/null @@ -1,453 +0,0 @@ -//===- QuantizedChains.cpp - Resident INT8 vector and channel chains ------===// -#include "Buckyball/BuckyballOps.h" -#include "Lowering.h" -#include "Target/BuckyballTargetRegistry.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Bufferization/IR/Bufferization.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Tensor/IR/Tensor.h" -#include "mlir/Interfaces/SideEffectInterfaces.h" -#include "llvm/ADT/MapVector.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using Computation = ::buddy::custom::CustomOp; - -void pebble::formQuantizedChains(func::FuncOp function) { - SmallVector roots; - function.walk([&](Computation op) { - if (op.getKind() == "global_avg_i8" || - (op.getKind() == "linear_i8_requant" && - cast(op.getResult(0).getType()).getDimSize(0) == 1)) - roots.push_back(op); - }); - int64_t id = 0; - for (auto root : roots) { - if (root->hasAttr("custom.quantized_chain")) - continue; - SmallVector chain{root}; - Value value = root.getResult(0); - int computations = 1; - while (value.hasOneUse()) { - Operation *user = *value.getUsers().begin(); - if (user->getBlock() != root->getBlock()) - break; - bool barrier = false; - for (Operation *between = chain.back()->getNextNode(); between != user; - between = between->getNextNode()) - if (!isMemoryEffectFree(between)) { - barrier = true; - break; - } - if (barrier) - break; - if (auto collapse = dyn_cast(user)) { - auto type = collapse.getResultType(); - if (type.getRank() != 2 || type.getDimSize(0) != 1) - break; - } else if (auto op = dyn_cast(user)) { - if (op.getInputs()[0] != value || op->hasAttr("custom.quantized_chain")) - break; - if (op.getKind() == "gate_mul_i8") { - if (!op.getResult(0).hasOneUse()) - break; - auto consumer = - dyn_cast(*op.getResult(0).getUsers().begin()); - auto type = cast(op.getInputs()[1].getType()); - if (!consumer || consumer.getKind() != "linear_i8_requant" || - consumer.getInputs()[0] != op.getResult(0) || - type.getDimSize(1) % 16 || - consumer->getBlock() != root->getBlock()) - break; - for (Operation *between = op->getNextNode(); between != consumer; - between = between->getNextNode()) - if (!isMemoryEffectFree(between)) { - barrier = true; - break; - } - if (barrier) - break; - chain.append({op, consumer}); - computations += 2; - break; - } - if (op.getKind() == "lookup_i8" && - cast(op.getInputs()[1].getType()) - .getNumElements() != 256) - break; - if ((op.getKind() != "linear_i8_requant" && - op.getKind() != "lookup_i8") || - cast(op.getResult(0).getType()).getDimSize(0) != - 1) - break; - ++computations; - } else { - break; - } - chain.push_back(user); - value = user->getResult(0); - } - if (computations < 2) - continue; - for (Operation *op : chain) - op->setAttr( - "custom.quantized_chain", - IntegerAttr::get(IntegerType::get(function.getContext(), 64), id)); - ++id; - } -} - -namespace { -LogicalResult vectorLinear(Computation op, Value input, Value &result) { - auto args = op.getInputs(); - auto weightType = cast(args[1].getType()); - int64_t k = weightType.getDimSize(0), n = weightType.getDimSize(1); - int64_t paddedK = (k + 15) / 16 * 16; - if (paddedK > buckyball_target::getBuckyballTarget().bankDepth || n > 2048) - return op.emitOpError( - "resident Linear exceeds its bank or quantizer row range"); - OpBuilder b(op); - Location loc = op.getLoc(); - auto index = [&](int64_t n) -> Value { - return arith::ConstantIndexOp::create(b, loc, n); - }; - Value zero = index(0), one = index(1); - Type i64 = b.getI64Type(); - Value weights = pebble::tensorBuffer(b, loc, args[1]); - Value biases = pebble::tensorBuffer(b, loc, args[2]); - Value scales = pebble::tensorBuffer(b, loc, args[3]); - result = BankAllocOp::create(b, loc, i64, 1, 1); - for (int64_t c = 0; c < n; c += 16) { - Value weightPack = memref::AllocOp::create( - b, loc, MemRefType::get({paddedK, 16}, b.getI8Type())); - Value biasPack = - memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getI32Type())); - Value scalePack = - memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getF32Type())); - Value zeroI8 = arith::ConstantIntOp::create(b, loc, 0, 8); - Value zeroI32 = arith::ConstantIntOp::create(b, loc, 0, 32); - Value oneF32 = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(1)); - linalg::FillOp::create(b, loc, zeroI8, weightPack); - linalg::FillOp::create(b, loc, zeroI32, biasPack); - linalg::FillOp::create(b, loc, oneF32, scalePack); - auto lanes = scf::ForOp::create(b, loc, zero, - index(std::min(16, n - c)), one); - { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(); - Value channel = arith::AddIOp::create(b, loc, lane, index(c)); - Value bias = - memref::LoadOp::create(b, loc, biases, ValueRange{zero, channel}); - Value scale = - memref::LoadOp::create(b, loc, scales, ValueRange{zero, channel}); - memref::StoreOp::create(b, loc, bias, biasPack, lane); - memref::StoreOp::create(b, loc, scale, scalePack, lane); - auto inner = scf::ForOp::create(b, loc, zero, index(k), one); - b.setInsertionPointToStart(inner.getBody()); - Value r = inner.getInductionVar(); - Value weight = - memref::LoadOp::create(b, loc, weights, ValueRange{r, channel}); - memref::StoreOp::create(b, loc, weight, weightPack, ValueRange{r, lane}); - } - Value w = BankAllocOp::create(b, loc, i64, 1, 1); - w = BankMvinOp::create(b, loc, i64, weightPack, w, paddedK, 1); - Value acc = BankAllocOp::create(b, loc, i64, 1, 1); - acc = BankMvinOp::create(b, loc, i64, biasPack, acc, 4, 1); - auto product = BankCustomOp::create(b, loc, i64, ValueRange{input, w}, acc, - ValueRange{}, "matmul_i8", true); - product->setAttr("rows", b.getI64IntegerAttr(1)); - product->setAttr("k", b.getI64IntegerAttr(paddedK)); - acc = product.getDestinationOut(); - BankReleaseOp::create(b, loc, w); - Value scale = BankAllocOp::create(b, loc, i64, 1, 1); - scale = BankMvinOp::create(b, loc, i64, scalePack, scale, 4, 1); - auto quant = - BankCustomOp::create(b, loc, i64, ValueRange{acc, scale}, result, - ValueRange{}, "int32_to_int8", true); - quant->setAttr("rows", b.getI64IntegerAttr(1)); - quant->setAttr("wr_base", b.getI64IntegerAttr(c / 16)); - quant->setAttr("relu", op->getAttr("relu")); - result = quant.getDestinationOut(); - BankReleaseOp::create(b, loc, acc); - BankReleaseOp::create(b, loc, scale); - } - BankReleaseOp::create(b, loc, input); - return success(); -} - -LogicalResult gatedLinear(Computation product, Computation multiply, Value gate, - Value &result) { - auto args = product.getInputs(); - auto inputType = cast(multiply.getInputs()[1].getType()); - auto resultType = cast(product.getResult(0).getType()); - int64_t m = inputType.getDimSize(0), k = inputType.getDimSize(1), - n = resultType.getDimSize(1); - int64_t rows = - buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); - int64_t reductionLimit = - std::min(64 * 16 / rows, - buckyball_target::getBuckyballTarget().bankDepth) / - 16 * 16; - OpBuilder b(product); - Location loc = product.getLoc(); - auto index = [&](int64_t n) -> Value { - return arith::ConstantIndexOp::create(b, loc, n); - }; - Value zero = index(0), one = index(1); - Type i64 = b.getI64Type(); - Value input = pebble::tensorBuffer(b, loc, multiply.getInputs()[1]); - Value weights = pebble::tensorBuffer(b, loc, args[1]); - Value biases = pebble::tensorBuffer(b, loc, args[2]); - Value scales = pebble::tensorBuffer(b, loc, args[3]); - Value ratio = arith::ConstantOp::create( - b, loc, cast(multiply->getAttr("ratio"))); - Value output = memref::AllocOp::create( - b, loc, MemRefType::get(resultType.getShape(), b.getI8Type())); - Value scratch = memref::AllocOp::create( - b, loc, MemRefType::get({rows, 16}, b.getI8Type())); - Value izero = arith::ConstantIntOp::create(b, loc, 0, 8); - for (int64_t c = 0; c < n; c += 16) { - Value biasPack = - memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getI32Type())); - Value scalePack = - memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getF32Type())); - Value zeroI32 = arith::ConstantIntOp::create(b, loc, 0, 32); - Value oneF32 = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(1)); - linalg::FillOp::create(b, loc, zeroI32, biasPack); - linalg::FillOp::create(b, loc, oneF32, scalePack); - auto lanes = scf::ForOp::create(b, loc, zero, - index(std::min(16, n - c)), one); - { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(), - channel = arith::AddIOp::create(b, loc, lane, index(c)); - Value bias = - memref::LoadOp::create(b, loc, biases, ValueRange{zero, channel}); - Value scale = - memref::LoadOp::create(b, loc, scales, ValueRange{zero, channel}); - memref::StoreOp::create(b, loc, bias, biasPack, lane); - memref::StoreOp::create(b, loc, scale, scalePack, lane); - } - Value scale = BankAllocOp::create(b, loc, i64, 1, 1); - scale = BankMvinOp::create(b, loc, i64, scalePack, scale, 4, 1); - for (int64_t r = 0; r < m; r += rows) { - Value acc = BankAllocOp::create(b, loc, i64, 1, 1); - acc = BankMvinOp::create(b, loc, i64, biasPack, acc, 4, 1); - for (int64_t start = 0; start < k; start += reductionLimit) { - int64_t count = std::min(reductionLimit, k - start); - Value a = BankAllocOp::create(b, loc, i64, 1, 1); - for (int64_t channel = start; channel < start + count; channel += 16) { - linalg::FillOp::create(b, loc, izero, scratch); - auto pixels = scf::ForOp::create(b, loc, zero, - index(std::min(rows, m - r)), one); - { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(pixels.getBody()); - Value row = pixels.getInductionVar(); - auto lanes = scf::ForOp::create(b, loc, zero, index(16), one); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(); - Value value = memref::LoadOp::create( - b, loc, input, - ValueRange{ - arith::AddIOp::create(b, loc, row, index(r)), - arith::AddIOp::create(b, loc, lane, index(channel))}); - memref::StoreOp::create(b, loc, value, scratch, - ValueRange{row, lane}); - } - Value source = BankAllocOp::create(b, loc, i64, 1, 1); - source = BankMvinOp::create(b, loc, i64, scratch, source, rows, 1); - Value gated = BankAllocOp::create(b, loc, i64, 1, 1); - auto mul = - BankCustomOp::create(b, loc, i64, ValueRange{gate, source}, gated, - ValueRange{ratio}, "gate_mul_i8"); - mul->setAttr("rows", b.getI64IntegerAttr(rows)); - mul->setAttr("gate_row", b.getI64IntegerAttr(channel / 16)); - gated = mul.getDestinationOut(); - BankReleaseOp::create(b, loc, source); - for (int64_t offset = 0; offset < rows; offset += 4) { - auto copy = BankCustomOp::create(b, loc, i64, ValueRange{gated}, a, - ValueRange{}, "maxpool_i8", true); - for (auto [name, value] : - std::initializer_list>{ - {"input_side", 2}, - {"output_side", 2}, - {"kernel", 1}, - {"stride", 1}, - {"padding", 0}, - {"op1_base", offset}, - {"wr_base", (channel - start) / 16 * rows + offset}, - {"output_stride", 2}, - {"start_row", 0}, - {"start_col", 0}}) - copy->setAttr(name, b.getI64IntegerAttr(value)); - a = copy.getDestinationOut(); - } - BankReleaseOp::create(b, loc, gated); - } - Value weightPack = memref::AllocOp::create( - b, loc, MemRefType::get({count, 16}, b.getI8Type())); - linalg::FillOp::create(b, loc, izero, weightPack); - auto inner = scf::ForOp::create(b, loc, zero, index(count), one); - { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(inner.getBody()); - Value row = inner.getInductionVar(); - auto lanes = scf::ForOp::create( - b, loc, zero, index(std::min(16, n - c)), one); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(); - Value value = memref::LoadOp::create( - b, loc, weights, - ValueRange{arith::AddIOp::create(b, loc, row, index(start)), - arith::AddIOp::create(b, loc, lane, index(c))}); - memref::StoreOp::create(b, loc, value, weightPack, - ValueRange{row, lane}); - } - Value w = BankAllocOp::create(b, loc, i64, 1, 1); - w = BankMvinOp::create(b, loc, i64, weightPack, w, count, 1); - auto matmul = - BankCustomOp::create(b, loc, i64, ValueRange{a, w}, acc, - ValueRange{}, "matmul_i8_block", true); - matmul->setAttr("rows", b.getI64IntegerAttr(rows)); - matmul->setAttr("k", b.getI64IntegerAttr(count)); - matmul->setAttr("first", b.getBoolAttr(start == 0)); - matmul->setAttr("last", b.getBoolAttr(start + count == k)); - acc = matmul.getDestinationOut(); - BankReleaseOp::create(b, loc, a); - BankReleaseOp::create(b, loc, w); - } - Value codes = BankAllocOp::create(b, loc, i64, 1, 1); - auto quant = BankCustomOp::create(b, loc, i64, ValueRange{acc, scale}, - codes, ValueRange{}, "int32_to_int8"); - quant->setAttr("rows", b.getI64IntegerAttr(rows)); - quant->setAttr("wr_base", b.getI64IntegerAttr(0)); - quant->setAttr("relu", product->getAttr("relu")); - codes = quant.getDestinationOut(); - BankMvoutOp::create(b, loc, i64, scratch, codes, rows, 1); - BankReleaseOp::create(b, loc, acc); - BankReleaseOp::create(b, loc, codes); - auto pixels = - scf::ForOp::create(b, loc, zero, index(std::min(rows, m - r)), one); - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(pixels.getBody()); - Value row = pixels.getInductionVar(); - auto lanes = scf::ForOp::create(b, loc, zero, - index(std::min(16, n - c)), one); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(); - Value value = - memref::LoadOp::create(b, loc, scratch, ValueRange{row, lane}); - memref::StoreOp::create( - b, loc, value, output, - ValueRange{arith::AddIOp::create(b, loc, row, index(r)), - arith::AddIOp::create(b, loc, lane, index(c))}); - } - BankReleaseOp::create(b, loc, scale); - } - BankReleaseOp::create(b, loc, gate); - result = - bufferization::ToTensorOp::create(b, loc, resultType, output, true, true); - return success(); -} -} // namespace - -LogicalResult pebble::lowerQuantizedChains(func::FuncOp function) { - llvm::MapVector> groups; - function.walk([&](Operation *op) { - if (auto id = op->getAttrOfType("custom.quantized_chain")) - groups[id.getInt()].push_back(op); - }); - for (auto &[id, chain] : groups) { - Value bank, result; - Computation multiply; - for (Operation *link : chain) { - auto op = dyn_cast(link); - if (!op) - continue; - if (op.getKind() == "global_avg_i8") { - if (failed(lowerGlobalAverage(op, bank, true))) - return failure(); - } else if (op.getKind() == "linear_i8_requant") { - if (multiply) { - if (failed(gatedLinear(op, multiply, bank, result))) - return failure(); - bank = Value(); - } else { - if (!bank) { - OpBuilder b(op); - Location loc = op.getLoc(); - auto type = cast(op.getInputs()[0].getType()); - int64_t k = type.getDimSize(1), padded = (k + 15) / 16 * 16; - Value pack = memref::AllocOp::create( - b, loc, MemRefType::get({1, padded}, b.getI8Type())); - Value zero = arith::ConstantIntOp::create(b, loc, 0, 8); - linalg::FillOp::create(b, loc, zero, pack); - Value slice = memref::SubViewOp::create( - b, loc, pack, ArrayRef{0, 0}, ArrayRef{1, k}, - ArrayRef{1, 1}); - memref::CopyOp::create( - b, loc, tensorBuffer(b, loc, op.getInputs()[0]), slice); - bank = BankAllocOp::create(b, loc, b.getI64Type(), 1, 1); - bank = BankMvinOp::create(b, loc, b.getI64Type(), pack, bank, - padded / 16, 1); - } - Value next; - if (failed(vectorLinear(op, bank, next))) - return failure(); - bank = next; - } - } else if (op.getKind() == "lookup_i8") { - OpBuilder b(op); - Location loc = op.getLoc(); - auto tableType = cast(op.getInputs()[1].getType()); - if (tableType.getNumElements() != 256) - return op.emitOpError( - "resident lookup requires one shared 256-entry table"); - Value table = BankAllocOp::create(b, loc, b.getI64Type(), 1, 1); - table = BankMvinOp::create(b, loc, b.getI64Type(), - tensorBuffer(b, loc, op.getInputs()[1]), - table, 16, 1); - Value destination = BankAllocOp::create(b, loc, b.getI64Type(), 1, 1); - auto lookup = BankCustomOp::create(b, loc, b.getI64Type(), - ValueRange{bank, table}, destination, - ValueRange{}, "lookup_i8"); - int64_t channels = - cast(op.getResult(0).getType()).getDimSize(1); - lookup->setAttr("rows", b.getI64IntegerAttr((channels + 15) / 16)); - BankReleaseOp::create(b, loc, bank); - BankReleaseOp::create(b, loc, table); - bank = lookup.getDestinationOut(); - } else if (op.getKind() == "gate_mul_i8") { - multiply = op; - } - } - Operation *last = chain.back(); - if (bank) { - OpBuilder b(last); - Location loc = last->getLoc(); - auto type = cast(last->getResult(0).getType()); - int64_t channels = type.getDimSize(1), padded = (channels + 15) / 16 * 16; - Value pack = memref::AllocOp::create( - b, loc, MemRefType::get({1, padded}, b.getI8Type())); - BankMvoutOp::create(b, loc, b.getI64Type(), pack, bank, padded / 16, 1); - BankReleaseOp::create(b, loc, bank); - Value output = memref::AllocOp::create( - b, loc, MemRefType::get({1, channels}, b.getI8Type())); - Value slice = memref::SubViewOp::create( - b, loc, pack, ArrayRef{0, 0}, ArrayRef{1, channels}, - ArrayRef{1, 1}); - memref::CopyOp::create(b, loc, slice, output); - result = - bufferization::ToTensorOp::create(b, loc, type, output, true, true); - } - last->getResult(0).replaceAllUsesWith(result); - for (Operation *op : llvm::reverse(chain)) - op->erase(); - } - return success(); -} diff --git a/examples/designs/pebble/stack/midend/tests/form-linear-chain.mlir b/examples/designs/pebble/stack/midend/tests/form-linear-chain.mlir deleted file mode 100644 index 0f5bbab58..000000000 --- a/examples/designs/pebble/stack/midend/tests/form-linear-chain.mlir +++ /dev/null @@ -1,83 +0,0 @@ -// FORM-LABEL: func.func @cycle_chain -// FORM: %[[FIRST:.*]] = custom.op "linear_i8"{{.*}}custom.chain = [[CHAIN:[0-9]+]] : i64, custom.relu = true -// FORM: %[[TRACE:.*]] = buddy_trace.end %[[FIRST]] {{.*}}custom.chain = [[CHAIN]] : i64{{.*}}tensor = false -// FORM: %[[RELU:.*]] = custom.op "relu"(%[[TRACE]]) {{.*}}custom.chain = [[CHAIN]] : i64 -// FORM: custom.op "linear_i8"(%[[RELU]], {{.*}}custom.chain = [[CHAIN]] : i64 -// FORM: return -// BANK-LABEL: func.func @cycle_chain -// BANK-NOT: {{fp32_to_int8|int32_to_int8|int32_to_fp32|bank_mvout}} -// BANK: buckyball.bank_custom "fp32_to_int8" -// BANK-NOT: {{fp32_to_int8|int32_to_int8|int32_to_fp32|bank_mvout}} -// BANK: buckyball.bank_custom "int32_to_int8"{{.*}}relu = true{{.*}}wr_base = 0 : i64 -// BANK-NOT: {{fp32_to_int8|int32_to_int8|int32_to_fp32|bank_mvout}} -// BANK: %[[RESIDENT:.*]] = buckyball.bank_custom "int32_to_int8"{{.*}}relu = true{{.*}}wr_base = 1 : i64 -// BANK-NOT: {{fp32_to_int8|int32_to_int8|int32_to_fp32|bank_mvout}} -// BANK: buddy_trace.end %[[RESIDENT]] {{.*}}tensor = false} : i64 -// BANK-NOT: {{fp32_to_int8|int32_to_int8|int32_to_fp32|bank_mvout}} -// BANK: buckyball.bank_custom "int32_to_fp32" -// BANK-NOT: {{fp32_to_int8|int32_to_int8|int32_to_fp32|bank_mvout}} -// BANK: buckyball.bank_mvout -// BANK-NOT: {{fp32_to_int8|int32_to_int8|int32_to_fp32|bank_mvout}} -// BANK: return -func.func @cycle_chain(%x: tensor<1x32xf32>, %w1: tensor<32x32xi8>, %b1: tensor<1x32xi32>, %s1: tensor<1x32xf32>, %w2: tensor<32x16xi8>, %b2: tensor<1x16xi32>, %s2: tensor<1x16xf32>, %multiplier: tensor<1x1xf32>, %step: tensor<1x1xf32>) -> tensor<1x16xf32> { - %zero = arith.constant dense<0.0> : tensor<1x32xf32> - %empty = tensor.empty() : tensor<1x32xf32> - %first = custom.op "linear_i8"(%x, %w1, %b1, %multiplier, %step, %s1) : (tensor<1x32xf32>, tensor<32x32xi8>, tensor<1x32xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x32xf32>) -> tensor<1x32xf32> - %trace = buddy_trace.end %first {cycle = true, id = 0 : i64, id_path = [0], tag = "first", tensor = false} : tensor<1x32xf32> - %relu = linalg.generic { - indexing_maps = [affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], - iterator_types = ["parallel", "parallel"] - } ins(%trace, %zero: tensor<1x32xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { - ^bb0(%value: f32, %zero_value: f32, %unused: f32): - %positive = arith.maximumf %value, %zero_value : f32 - linalg.yield %positive : f32 - } -> tensor<1x32xf32> - %second = custom.op "linear_i8"(%relu, %w2, %b2, %multiplier, %step, %s2) : (tensor<1x32xf32>, tensor<32x16xi8>, tensor<1x16xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x16xf32>) -> tensor<1x16xf32> - return %second : tensor<1x16xf32> -} - -// FORM-LABEL: func.func @tensor_trace -// FORM: custom.op "linear_i8" -// FORM: buddy_trace.end {{.*}}tensor = true} : tensor<1x32xf32> -// FORM: linalg.generic -// FORM: arith.maximumf -// FORM: custom.op "linear_i8" -// FORM: return -func.func @tensor_trace(%x: tensor<1x32xf32>, %w1: tensor<32x32xi8>, %b1: tensor<1x32xi32>, %s1: tensor<1x32xf32>, %w2: tensor<32x16xi8>, %b2: tensor<1x16xi32>, %s2: tensor<1x16xf32>, %multiplier: tensor<1x1xf32>, %step: tensor<1x1xf32>) -> tensor<1x16xf32> { - %zero = arith.constant dense<0.0> : tensor<1x32xf32> - %empty = tensor.empty() : tensor<1x32xf32> - %first = custom.op "linear_i8"(%x, %w1, %b1, %multiplier, %step, %s1) : (tensor<1x32xf32>, tensor<32x32xi8>, tensor<1x32xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x32xf32>) -> tensor<1x32xf32> - %trace = buddy_trace.end %first {cycle = true, id = 1 : i64, id_path = [1], tag = "first", tensor = true} : tensor<1x32xf32> - %relu = linalg.generic { - indexing_maps = [affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], - iterator_types = ["parallel", "parallel"] - } ins(%trace, %zero: tensor<1x32xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { - ^bb0(%value: f32, %zero_value: f32, %unused: f32): - %positive = arith.maximumf %value, %zero_value : f32 - linalg.yield %positive : f32 - } -> tensor<1x32xf32> - %second = custom.op "linear_i8"(%relu, %w2, %b2, %multiplier, %step, %s2) : (tensor<1x32xf32>, tensor<32x16xi8>, tensor<1x16xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x16xf32>) -> tensor<1x16xf32> - return %second : tensor<1x16xf32> -} - -// FORM-LABEL: func.func @multiple_consumers -// FORM: %[[OBSERVED:.*]] = custom.op "linear_i8" -// FORM: linalg.generic -// FORM: arith.maximumf -// FORM: custom.op "linear_i8" -// FORM: return {{.*}}, %[[OBSERVED]] -func.func @multiple_consumers(%x: tensor<1x32xf32>, %w1: tensor<32x32xi8>, %b1: tensor<1x32xi32>, %s1: tensor<1x32xf32>, %w2: tensor<32x16xi8>, %b2: tensor<1x16xi32>, %s2: tensor<1x16xf32>, %multiplier: tensor<1x1xf32>, %step: tensor<1x1xf32>) -> (tensor<1x16xf32>, tensor<1x32xf32>) { - %zero = arith.constant dense<0.0> : tensor<1x32xf32> - %empty = tensor.empty() : tensor<1x32xf32> - %first = custom.op "linear_i8"(%x, %w1, %b1, %multiplier, %step, %s1) : (tensor<1x32xf32>, tensor<32x32xi8>, tensor<1x32xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x32xf32>) -> tensor<1x32xf32> - %relu = linalg.generic { - indexing_maps = [affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>, affine_map<(m,n)->(m,n)>], - iterator_types = ["parallel", "parallel"] - } ins(%first, %zero: tensor<1x32xf32>, tensor<1x32xf32>) outs(%empty: tensor<1x32xf32>) { - ^bb0(%value: f32, %zero_value: f32, %unused: f32): - %positive = arith.maximumf %value, %zero_value : f32 - linalg.yield %positive : f32 - } -> tensor<1x32xf32> - %second = custom.op "linear_i8"(%relu, %w2, %b2, %multiplier, %step, %s2) : (tensor<1x32xf32>, tensor<32x16xi8>, tensor<1x16xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x16xf32>) -> tensor<1x16xf32> - return %second, %first : tensor<1x16xf32>, tensor<1x32xf32> -} diff --git a/examples/designs/pebble/stack/midend/tests/form-quantized-chain.mlir b/examples/designs/pebble/stack/midend/tests/form-quantized-chain.mlir deleted file mode 100644 index a1a263edb..000000000 --- a/examples/designs/pebble/stack/midend/tests/form-quantized-chain.mlir +++ /dev/null @@ -1,19 +0,0 @@ -// FORM-LABEL: func.func @squeeze_excite -// FORM-COUNT-7: custom.quantized_chain = 0 : i64 -// BANK-LABEL: func.func @squeeze_excite -// BANK: buckyball.bank_custom "lookup_i8" -// BANK: buckyball.bank_custom "gate_mul_i8" -// BANK: buckyball.bank_custom "matmul_i8_block" -// BANK: buckyball.bank_mvout -// BANK: return -func.func @squeeze_excite(%x: tensor<1x2x4x32xi8>, %w1: tensor<32x16xi8>, %b1: tensor<1x16xi32>, %s1: tensor<1x16xf32>, %w2: tensor<16x32xi8>, %b2: tensor<1x32xi32>, %s2: tensor<1x32xf32>, %lut: tensor<256xi8>, %w3: tensor<32x16xi8>) -> tensor<8x16xi8> { - %source = tensor.collapse_shape %x [[0,1,2],[3]] : tensor<1x2x4x32xi8> into tensor<8x32xi8> - %avg = custom.op "global_avg_i8"(%x) {input_scale = 1.0 : f32, output_scale = 1.0 : f32} : (tensor<1x2x4x32xi8>) -> tensor<1x1x1x32xi8> - %vector = tensor.collapse_shape %avg [[0,1,2],[3]] : tensor<1x1x1x32xi8> into tensor<1x32xi8> - %first = custom.op "linear_i8_requant"(%vector,%w1,%b1,%s1) {relu = true} : (tensor<1x32xi8>,tensor<32x16xi8>,tensor<1x16xi32>,tensor<1x16xf32>) -> tensor<1x16xi8> - %second = custom.op "linear_i8_requant"(%first,%w2,%b2,%s2) {relu = false} : (tensor<1x16xi8>,tensor<16x32xi8>,tensor<1x32xi32>,tensor<1x32xf32>) -> tensor<1x32xi8> - %gate = custom.op "lookup_i8"(%second,%lut) : (tensor<1x32xi8>,tensor<256xi8>) -> tensor<1x32xi8> - %multiplied = custom.op "gate_mul_i8"(%gate,%source) {ratio = 0.5 : f32} : (tensor<1x32xi8>,tensor<8x32xi8>) -> tensor<8x32xi8> - %result = custom.op "linear_i8_requant"(%multiplied,%w3,%b1,%s1) {relu = false} : (tensor<8x32xi8>,tensor<32x16xi8>,tensor<1x16xi32>,tensor<1x16xf32>) -> tensor<8x16xi8> - return %result : tensor<8x16xi8> -} diff --git a/examples/designs/pebble/stack/midend/tests/lower-bank-updates.mlir b/examples/designs/pebble/stack/midend/tests/lower-bank-updates.mlir index 31c37f388..60daa2b36 100644 --- a/examples/designs/pebble/stack/midend/tests/lower-bank-updates.mlir +++ b/examples/designs/pebble/stack/midend/tests/lower-bank-updates.mlir @@ -5,17 +5,10 @@ func.func @copy_rows() { buckyball.custom "maxpool_i8" {input_side = 1 : i64, output_side = 1 : i64, kernel = 1 : i64, stride = 1 : i64, padding = 0 : i64, op1_base = 0 : i64, wr_base = 1 : i64, output_stride = 1 : i64, start_row = 0 : i64, start_col = 0 : i64} return } -// CHECK-LABEL: func.func @update_quantized_rows -// CHECK: arith.constant -9223372032556660735 : i64 -// CHECK: return -func.func @update_quantized_rows() { - buckyball.custom "int32_to_int8" {rows = 1 : i64, wr_base = 1 : i64, relu = false} - return -} // CHECK-LABEL: func.func @define_quantized_rows // CHECK: arith.constant 4298115073 : i64 // CHECK: return func.func @define_quantized_rows() { - buckyball.custom "int32_to_int8" {rows = 1 : i64, wr_base = 0 : i64, relu = false} + buckyball.custom "int32_to_int8" {rows = 1 : i64, relu = false} return } diff --git a/examples/designs/pebble/stack/midend/tests/lower-custom-ops.mlir b/examples/designs/pebble/stack/midend/tests/lower-custom-ops.mlir index 788473a76..5622df7b8 100644 --- a/examples/designs/pebble/stack/midend/tests/lower-custom-ops.mlir +++ b/examples/designs/pebble/stack/midend/tests/lower-custom-ops.mlir @@ -1,10 +1,11 @@ // CHECK-LABEL: func.func @named // CHECK: buckyball.bank_mvin // CHECK-NOT: "fp32_to_int8" +// CHECK: scf.for {{.*}} to %c32 step %c16 // CHECK: buckyball.bank_custom "matmul_i8" // CHECK: buckyball.bank_mvout -// CHECK: buckyball.bank_custom "matmul_i8" -// CHECK: buckyball.bank_mvout +// CHECK: } +// CHECK-NOT: buckyball.bank_custom // CHECK: return func.func @named(%a: tensor<1x32xi8>, %b: tensor<32x32xi8>, %initial: tensor<1x32xi32>) -> tensor<1x32xi32> { %product = custom.op "matmul_i8"(%a, %b, %initial) : (tensor<1x32xi8>, tensor<32x32xi8>, tensor<1x32xi32>) -> tensor<1x32xi32> @@ -13,12 +14,11 @@ func.func @named(%a: tensor<1x32xi8>, %b: tensor<32x32xi8>, %initial: tensor<1x3 // CHECK-LABEL: func.func @linear // CHECK: buckyball.bank_custom "fp32_to_int8" +// CHECK: scf.for {{.*}} to %c32 step %c16 // CHECK: buckyball.bank_custom "matmul_i8" // CHECK: buckyball.bank_custom "int32_to_fp32" // CHECK: buckyball.bank_mvout -// CHECK: buckyball.bank_custom "matmul_i8" -// CHECK: buckyball.bank_custom "int32_to_fp32" -// CHECK: buckyball.bank_mvout +// CHECK-NOT: buckyball.bank_custom // CHECK: return func.func @linear(%a: tensor<1x32xf32>, %w: tensor<32x32xi8>, %bias: tensor<1x32xi32>, %multiplier: tensor<1x1xf32>, %input_step: tensor<1x1xf32>, %weight_step: tensor<1x32xf32>) -> tensor<1x32xf32> { %product = custom.op "linear_i8"(%a, %w, %bias, %multiplier, %input_step, %weight_step) : (tensor<1x32xf32>, tensor<32x32xi8>, tensor<1x32xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x32xf32>) -> tensor<1x32xf32> diff --git a/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml index 1359cb046..7c51b074c 100644 --- a/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml +++ b/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml @@ -51,4 +51,7 @@ tests = [ "toy-toy-mlirtest-linalg_conv2d_nhwc_hwcf-baremetal", "toy-toy-mlirtest-linalg_conv2d_nhwc_fhwc-baremetal", "toy-toy-mlirtest-linalg_conv2d_nchw_fchw-baremetal", + "toy-toy-mlirtest-linalg_conv2d_nhwc_hwcf_deep-baremetal", + "toy-toy-mlirtest-base_bank_loop_carried-baremetal", + "toy-toy-mlirtest-base_bank_transfer-baremetal", ] diff --git a/examples/designs/toy/stack/midend/src/ConvertLinalgToBuckyball.cpp b/examples/designs/toy/stack/midend/src/ConvertLinalgToBuckyball.cpp index cd9807ab6..d67c11652 100644 --- a/examples/designs/toy/stack/midend/src/ConvertLinalgToBuckyball.cpp +++ b/examples/designs/toy/stack/midend/src/ConvertLinalgToBuckyball.cpp @@ -1,24 +1,29 @@ //===- ConvertLinalgToBuckyball.cpp - Toy core lowering -------------------===// // -// A 2-D convolution becomes a matrix product C[M x N] = A[M x K] @ B[K x N] -// with M the batch and output pixels, N the output channels and K the filter -// window and input channels. The CPU packs A (im2col) and B into 16x16 INT8 -// tiles, rounding the FP32 values to nearest even and saturating; Gemmini -// computes each C tile with matmul_i8_tile, accumulating over K; the CPU adds -// the INT32 tiles to the FP32 output. +// conv2d_i8 becomes a matrix product C[M x N] = A[M x K] @ B[K x N] over the +// dimension groups form-custom-ops recorded. The CPU packs A (im2col) and B +// into zero-padded 16x16 INT8 tiles. For each 16x16 C tile, the A bank takes +// the K blocks of the tile's rows and the B bank the K rows of its columns, +// one bank-deep group at a time gathered with bank_transfer, and Gemmini's +// matmul_i8_tile computes C. The CPU adds C * input_step * weight_step to the +// initial FP32 output. // //===----------------------------------------------------------------------===// #include "Buckyball/BuckyballDialect.h" #include "Buckyball/BuckyballOps.h" +#include "Custom/CustomOps.h" +#include "Target/BuckyballTargetRegistry.h" #include "mlir/Dialect/Affine/IR/AffineOps.h" #include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/Math/IR/Math.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/Dialect/Utils/IndexingUtils.h" #include "mlir/Pass/Pass.h" using namespace mlir; @@ -40,30 +45,36 @@ void loops(OpBuilder &b, Location loc, int64_t rows, int64_t cols, body(outer.getInductionVar(), inner.getInductionVar()); } -LogicalResult lowerConvolution(linalg::LinalgOp op) { - FailureOr conv = - linalg::inferConvolutionDims(op); - Value input = op.getDpsInputs()[0], filter = op.getDpsInputs()[1]; - Value output = op.getDpsInits()[0]; - if (failed(conv) || !conv->depth.empty() || !op.hasPureBufferSemantics() || - !getElementTypeOrSelf(input.getType()).isF32() || - !getElementTypeOrSelf(filter.getType()).isF32() || - !getElementTypeOrSelf(output.getType()).isF32()) - return op->emitOpError("toy lowers FP32 2-D convolutions on memrefs"); - SmallVector ranges = op.getStaticLoopRanges(); - SmallVector mDims(conv->batch.begin(), conv->batch.end()); - SmallVector kDims(conv->filterLoop.begin(), conv->filterLoop.end()); - llvm::append_range(mDims, conv->outputImage); - llvm::append_range(kDims, conv->inputChannel); - ArrayRef nDims = conv->outputChannel; - auto extent = [&](ArrayRef dims) { +// The buffer of a contiguous tensor. +Value buffer(OpBuilder &b, Location loc, Value tensor) { + auto type = cast(tensor.getType()); + auto memory = MemRefType::get( + type.getShape(), type.getElementType(), + StridedLayoutAttr::get(b.getContext(), ShapedType::kDynamic, + computeSuffixProduct(type.getShape()))); + return bufferization::ToBufferOp::create(b, loc, memory, tensor, true); +} + +void lowerConvolution(::buddy::custom::CustomOp op) { + SmallVector maps = llvm::map_to_vector( + cast(op->getAttr("indexing_maps")), + [](Attribute map) { return cast(map).getValue(); }); + auto array = [&](StringRef name) { + return cast(op->getAttr(name)).asArrayRef(); + }; + ArrayRef ranges = array("loop_ranges"); + ArrayRef mDims = array("m_dims"), nDims = array("n_dims"), + kDims = array("k_dims"); + auto extent = [&](ArrayRef dims) { int64_t size = 1; - for (unsigned dim : dims) + for (int64_t dim : dims) size *= ranges[dim]; return size; }; int64_t m = extent(mDims), n = extent(nDims), k = extent(kDims); int64_t mt = (m + 15) / 16, nt = (n + 15) / 16, kt = (k + 15) / 16; + // The K blocks a bank group holds. + int64_t groupBlocks = buckyball_target::getBuckyballTarget().bankDepth / 16; OpBuilder b(op); Location loc = op.getLoc(); @@ -71,50 +82,57 @@ LogicalResult lowerConvolution(linalg::LinalgOp op) { auto idx = [&](int64_t v) -> Value { return arith::ConstantIndexOp::create(b, loc, v); }; - auto f32 = [&](float v) -> Value { - return arith::ConstantOp::create(b, loc, b.getF32FloatAttr(v)); - }; // The loop dimension values of flat indices along each dimension group. - auto iterators = [&](ArrayRef>> flats) { + auto iterators = [&](ArrayRef>> flats) { SmallVector iv(ranges.size(), idx(0)); for (auto [flat, dims] : flats) - for (unsigned dim : llvm::reverse(dims)) { + for (int64_t dim : llvm::reverse(dims)) { iv[dim] = arith::RemUIOp::create(b, loc, flat, idx(ranges[dim])); flat = arith::DivUIOp::create(b, loc, flat, idx(ranges[dim])); } return iv; }; - auto element = [&](OpOperand &operand, ArrayRef iv) { - AffineMap map = op.getMatchingIndexingMap(&operand); + auto element = [&](AffineMap map, ArrayRef iv) { SmallVector indices; for (AffineExpr result : map.getResults()) indices.push_back(affine::AffineApplyOp::create( b, loc, AffineMap::get(map.getNumDims(), 0, result), iv)); return indices; }; - auto quantize = [&](Value v) -> Value { - v = arith::MaximumFOp::create(b, loc, v, f32(-128)); - v = arith::MinimumFOp::create(b, loc, v, f32(127)); - v = math::RoundEvenOp::create(b, loc, v); - return arith::FPToSIOp::create(b, loc, i8, v); - }; - // Tile (row / 16) * stride + col / 16, element (row % 16, col % 16). - auto tileIndex = [&](Value row, Value col, int64_t stride) { + // Tile (outer / 16) * stride + inner / 16, element (row % 16, col % 16). + auto tileIndex = [&](Value outer, Value inner, int64_t stride, Value row, + Value col) { Value sixteen = idx(16); Value tile = arith::AddIOp::create( b, loc, - arith::MulIOp::create( - b, loc, arith::DivUIOp::create(b, loc, row, sixteen), idx(stride)), - arith::DivUIOp::create(b, loc, col, sixteen)); + arith::MulIOp::create(b, loc, + arith::DivUIOp::create(b, loc, outer, sixteen), + idx(stride)), + arith::DivUIOp::create(b, loc, inner, sixteen)); return SmallVector{tile, arith::RemUIOp::create(b, loc, row, sixteen), arith::RemUIOp::create(b, loc, col, sixteen)}; }; + auto scalar = [&](Value scale) -> Value { + SmallVector zeros(cast(scale.getType()).getRank(), + idx(0)); + return tensor::ExtractOp::create(b, loc, scale, zeros); + }; + + ValueRange inputs = op.getInputs(); + Value input = buffer(b, loc, inputs[0]), filter = buffer(b, loc, inputs[2]); + Value step = + arith::MulFOp::create(b, loc, scalar(inputs[1]), scalar(inputs[3])); + auto resultType = cast(op.getResult(0).getType()); + Value output = memref::AllocOp::create( + b, loc, MemRefType::get(resultType.getShape(), b.getF32Type())); + memref::CopyOp::create(b, loc, buffer(b, loc, inputs[4]), output); Value aTiles = memref::AllocOp::create( b, loc, MemRefType::get({mt * kt, 16, 16}, i8), b.getI64IntegerAttr(64)); + // B tiles by column: tile j * kt + r holds K rows 16r .. 16r + 15. Value bTiles = memref::AllocOp::create( - b, loc, MemRefType::get({kt * nt, 16, 16}, i8), b.getI64IntegerAttr(64)); + b, loc, MemRefType::get({nt * kt, 16, 16}, i8), b.getI64IntegerAttr(64)); Value cTiles = memref::AllocOp::create( b, loc, MemRefType::get({mt * nt, 16, 16}, i32), b.getI64IntegerAttr(64)); Value zero8 = arith::ConstantIntOp::create(b, loc, 0, 8); @@ -122,62 +140,68 @@ LogicalResult lowerConvolution(linalg::LinalgOp op) { linalg::FillOp::create(b, loc, zero8, bTiles); loops(b, loc, m, k, [&](Value row, Value col) { SmallVector iv = iterators({{row, mDims}, {col, kDims}}); - Value v = - memref::LoadOp::create(b, loc, input, element(op->getOpOperand(0), iv)); - memref::StoreOp::create(b, loc, quantize(v), aTiles, - tileIndex(row, col, kt)); + Value v = memref::LoadOp::create(b, loc, input, element(maps[0], iv)); + memref::StoreOp::create(b, loc, v, aTiles, + tileIndex(row, col, kt, row, col)); }); loops(b, loc, k, n, [&](Value row, Value col) { SmallVector iv = iterators({{row, kDims}, {col, nDims}}); - Value v = memref::LoadOp::create(b, loc, filter, - element(op->getOpOperand(1), iv)); - memref::StoreOp::create(b, loc, quantize(v), bTiles, - tileIndex(row, col, nt)); + Value v = memref::LoadOp::create(b, loc, filter, element(maps[1], iv)); + memref::StoreOp::create(b, loc, v, bTiles, + tileIndex(col, row, kt, row, col)); }); - auto tile = [&](Value tiles, int64_t index) -> Value { + // `count` consecutive tiles from `index`. + auto tile = [&](Value tiles, int64_t index, int64_t count) -> Value { return memref::SubViewOp::create( b, loc, tiles, ArrayRef{index, 0, 0}, - ArrayRef{1, 16, 16}, ArrayRef{1, 1, 1}); + ArrayRef{count, 16, 16}, ArrayRef{1, 1, 1}); + }; + // The kt tiles from `first` in one bank, a group of groupBlocks tiles at a + // time. + auto gather = [&](Value tiles, int64_t first) { + Value bank; + for (int64_t block = 0; block < kt; block += groupBlocks) { + int64_t count = std::min(groupBlocks, kt - block); + Value group = BankMvinOp::create( + b, loc, i64, tile(tiles, first + block, count), + BankAllocOp::create(b, loc, i64, 1, 1), count * 16, 1); + bank = block + ? BankTransferOp::create(b, loc, i64, group, bank).getResult() + : group; + } + return bank; }; - Value base = arith::ConstantIntOp::create(b, loc, 0, 64); - Value aBank = BankAllocOp::create(b, loc, i64, 1, 1); - Value bBank = BankAllocOp::create(b, loc, i64, 1, 1); for (int64_t i = 0; i < mt; ++i) for (int64_t j = 0; j < nt; ++j) { + Value aBank = gather(aTiles, i * kt); + Value bBank = gather(bTiles, j * kt); Value cBank = BankAllocOp::create(b, loc, i64, 1, 4); - for (int64_t r = 0; r < kt; ++r) { - aBank = BankMvinOp::create(b, loc, i64, tile(aTiles, i * kt + r), aBank, - 16, 1); - bBank = BankMvinOp::create(b, loc, i64, tile(bTiles, r * nt + j), bBank, - 16, 1); - auto matmul = BankCustomOp::create( - b, loc, i64, ValueRange{aBank, bBank}, cBank, - ValueRange{base, base, base}, "matmul_i8_tile", r > 0); - if (r > 0) - matmul->setAttr("shift", b.getI64IntegerAttr(0)); - cBank = matmul.getDestinationOut(); - } - cBank = BankMvoutOp::create(b, loc, i64, tile(cTiles, i * nt + j), cBank, - 16, 1); + auto matmul = + BankCustomOp::create(b, loc, i64, ValueRange{aBank, bBank}, cBank, + ValueRange{}, "matmul_i8_tile", false); + matmul->setAttr("k", b.getI64IntegerAttr(kt * 16)); + cBank = BankMvoutOp::create(b, loc, i64, tile(cTiles, i * nt + j, 1), + matmul.getDestinationOut(), 16, 1); + BankReleaseOp::create(b, loc, aBank); + BankReleaseOp::create(b, loc, bBank); BankReleaseOp::create(b, loc, cBank); } - BankReleaseOp::create(b, loc, aBank); - BankReleaseOp::create(b, loc, bBank); loops(b, loc, m, n, [&](Value row, Value col) { SmallVector iv = iterators({{row, mDims}, {col, nDims}}); - SmallVector at = element(op->getOpOperand(2), iv); - Value sum = memref::LoadOp::create(b, loc, cTiles, tileIndex(row, col, nt)); + SmallVector at = element(maps[2], iv); + Value sum = memref::LoadOp::create(b, loc, cTiles, + tileIndex(row, col, nt, row, col)); + Value scaled = arith::MulFOp::create( + b, loc, arith::SIToFPOp::create(b, loc, b.getF32Type(), sum), step); Value result = arith::AddFOp::create( - b, loc, memref::LoadOp::create(b, loc, output, at), - arith::SIToFPOp::create(b, loc, b.getF32Type(), sum)); + b, loc, memref::LoadOp::create(b, loc, output, at), scaled); memref::StoreOp::create(b, loc, result, output, at); }); - for (Value buffer : {aTiles, bTiles, cTiles}) - memref::DeallocOp::create(b, loc, buffer); - op->erase(); - return success(); + op.getResult(0).replaceAllUsesWith(bufferization::ToTensorOp::create( + b, loc, resultType, output, true, true)); + op.erase(); } class ConvertLinalgToBuckyballPass @@ -187,24 +211,26 @@ class ConvertLinalgToBuckyballPass MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ConvertLinalgToBuckyballPass) StringRef getArgument() const final { return "convert-linalg-to-buckyball"; } StringRef getDescription() const final { - return "Lower 2-D convolutions to the toy core bank dataflow"; + return "Lower the toy custom computations to the core bank dataflow"; } void getDependentDialects(DialectRegistry ®istry) const override { - registry - .insert(); + registry.insert(); } void runOnOperation() override { - SmallVector convolutions; - getOperation().walk([&](linalg::LinalgOp op) { - if (isa(op)) - convolutions.push_back(op); - }); - for (linalg::LinalgOp op : convolutions) - if (failed(lowerConvolution(op))) + SmallVector<::buddy::custom::CustomOp> computations; + getOperation().walk( + [&](::buddy::custom::CustomOp op) { computations.push_back(op); }); + for (auto op : computations) { + if (op.getKind() != "conv2d_i8") { + op.emitOpError("has no bank lowering on this core for ") + << op.getKind(); return signalPassFailure(); + } + lowerConvolution(op); + } } }; diff --git a/examples/designs/toy/stack/midend/src/FormCustomOps.cpp b/examples/designs/toy/stack/midend/src/FormCustomOps.cpp new file mode 100644 index 000000000..9751b391f --- /dev/null +++ b/examples/designs/toy/stack/midend/src/FormCustomOps.cpp @@ -0,0 +1,108 @@ +//===- FormCustomOps.cpp - Recognize the toy core computations ------------===// +// +// conv2d_i8(x, input_step, w, weight_step, init) = init + conv(x, w) * +// input_step * weight_step, from a 2-D convolution of +// int8_to_fp32(fp32_to_int8(x', multiplier), input_step) and +// int8_to_fp32(w, weight_step), both steps per tensor. The convolution keeps +// its iteration space: `indexing_maps` (input, filter, output), `loop_ranges` +// and the loop dimensions of the matrix product, `m_dims` (batch and output +// pixels), `n_dims` (output channels) and `k_dims` (filter window and input +// channels). +// +// The core forms it only when its banks fit: the C tile takes four groups, +// bank 0 is the zero bank, and A and B each take a group per bank depth of K. +// +//===----------------------------------------------------------------------===// + +#include "Custom/CustomOps.h" +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Pass/Pass.h" + +using namespace mlir; +using ::buddy::custom::CustomOp; + +namespace { +Operation *frontend(Value value, StringRef name) { + Operation *op = value.getDefiningOp(); + return op && op->getName().getStringRef() == name ? op : nullptr; +} + +bool perTensor(Value scale) { + return llvm::all_of(cast(scale.getType()).getShape(), + [](int64_t size) { return size == 1; }); +} + +class FormCustomOpsPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(FormCustomOpsPass) + StringRef getArgument() const final { return "form-custom-ops"; } + StringRef getDescription() const final { + return "Recognize the quantized convolutions of the toy core"; + } + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert<::buddy::custom::CustomDialect>(); + } + void runOnOperation() override { + const auto &target = buckyball_target::getBuckyballTarget(); + int64_t maxK = (target.bankNum - 5) / 2 * target.bankDepth; + SmallVector convolutions; + getOperation().walk([&](linalg::LinalgOp op) { + if (isa(op) && + op.hasPureTensorSemantics()) + convolutions.push_back(op); + }); + for (linalg::LinalgOp op : convolutions) { + Operation *input = + frontend(op.getDpsInputs()[0], "frontend.int8_to_fp32"); + Operation *weight = + frontend(op.getDpsInputs()[1], "frontend.int8_to_fp32"); + if (!input || !weight) + continue; + Operation *quantize = + frontend(input->getOperand(0), "frontend.fp32_to_int8"); + FailureOr conv = + linalg::inferConvolutionDims(op); + if (!quantize || !perTensor(input->getOperand(1)) || + !perTensor(weight->getOperand(1)) || failed(conv) || + !conv->depth.empty()) + continue; + SmallVector ranges = op.getStaticLoopRanges(); + int64_t k = 1; + for (unsigned dim : conv->filterLoop) + k *= ranges[dim]; + for (unsigned dim : conv->inputChannel) + k *= ranges[dim]; + if (k > maxK) + continue; + OpBuilder builder(op); + auto dims = [&](ArrayRef first, ArrayRef second) { + SmallVector all(first.begin(), first.end()); + all.append(second.begin(), second.end()); + return builder.getDenseI64ArrayAttr(all); + }; + auto custom = CustomOp::create( + builder, op.getLoc(), op->getResultTypes(), "conv2d_i8", + ValueRange{quantize->getResult(0), input->getOperand(1), + weight->getOperand(0), weight->getOperand(1), + op.getDpsInits()[0]}); + custom->setAttr("indexing_maps", op.getIndexingMaps()); + custom->setAttr("loop_ranges", builder.getDenseI64ArrayAttr(ranges)); + custom->setAttr("m_dims", dims(conv->batch, conv->outputImage)); + custom->setAttr("n_dims", dims(conv->outputChannel, {})); + custom->setAttr("k_dims", dims(conv->filterLoop, conv->inputChannel)); + op->replaceAllUsesWith(custom.getOutputs()); + op->erase(); + for (Operation *dequantize : {input, weight}) + if (dequantize->use_empty()) + dequantize->erase(); + } + } +}; + +PassRegistration registration; +} // namespace diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt index 5ee75df0d..f1391245c 100644 --- a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/CMakeLists.txt @@ -6,7 +6,15 @@ add_custom_target(linalg-mlir-tests-build) add_buckyball_mlir_test(conv2d_nhwc_hwcf TARGET toy PASSES + -allow-unregistered-dialect + -form-custom-ops -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations -assign-physical-banks -lower-buckyball -convert-linalg-to-loops @@ -14,8 +22,35 @@ add_buckyball_mlir_test(conv2d_nhwc_hwcf -lower-affine -convert-scf-to-cf -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(conv2d_nhwc_hwcf_deep + TARGET toy + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts) @@ -23,7 +58,15 @@ add_buckyball_mlir_test(conv2d_nhwc_hwcf add_buckyball_mlir_test(conv2d_nhwc_fhwc TARGET toy PASSES + -allow-unregistered-dialect + -form-custom-ops -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations -assign-physical-banks -lower-buckyball -convert-linalg-to-loops @@ -31,8 +74,9 @@ add_buckyball_mlir_test(conv2d_nhwc_fhwc -lower-affine -convert-scf-to-cf -convert-cf-to-llvm - -convert-arith-to-llvm -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts) @@ -40,7 +84,15 @@ add_buckyball_mlir_test(conv2d_nhwc_fhwc add_buckyball_mlir_test(conv2d_nchw_fchw TARGET toy PASSES + -allow-unregistered-dialect + -form-custom-ops -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations -assign-physical-banks -lower-buckyball -convert-linalg-to-loops @@ -48,8 +100,9 @@ add_buckyball_mlir_test(conv2d_nchw_fchw -lower-affine -convert-scf-to-cf -convert-cf-to-llvm - -convert-arith-to-llvm -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts) diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw.mlir b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw.mlir index a4d3b200d..27514b327 100644 --- a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw.mlir +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw.mlir @@ -1,28 +1,69 @@ -func.func private @check_result(memref<1x6x4x1xf32>) -> () +// Quantized convolution in the frontend QDQ form: input steps 0.25 with +// multiplier 4, weight step 0.5, initial output 1. The output is +// 1 + 0.125 * sum(xq * wq), so a wrong step fails the check. +func.func private @check_result(memref<1x4x3x3xf32>) -> () func.func @main() -> i8 { %zero_i8 = arith.constant 0 : i8 - %zero_f32 = arith.constant 0.0 : f32 - %one_f32 = arith.constant 1.0 : f32 - - %input = memref.alloc() : memref<1x8x4x1xf32> - %filter = memref.alloc() : memref<6x8x1x1xf32> - %output = memref.alloc() : memref<1x6x4x1xf32> - - linalg.fill ins(%one_f32 : f32) outs(%input : memref<1x8x4x1xf32>) - linalg.fill ins(%one_f32 : f32) outs(%filter : memref<6x8x1x1xf32>) - linalg.fill ins(%zero_f32 : f32) outs(%output : memref<1x6x4x1xf32>) - - linalg.conv_2d_nchw_fchw + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %quarter = arith.constant 0.25 : f32 + %input = memref.alloc() : memref<1x3x5x5xf32> + %weight = memref.alloc() : memref<4x3x3x3xi8> + scf.for %h = %c0 to %c5 step %c1 { + scf.for %w = %c0 to %c5 step %c1 { + scf.for %c = %c0 to %c3 step %c1 { + %h3 = arith.muli %h, %c3 : index + %w5 = arith.muli %w, %c5 : index + %c7c = arith.muli %c, %c7 : index + %s0 = arith.addi %h3, %w5 : index + %s = arith.addi %s0, %c7c : index + %r = arith.remui %s, %c9 : index + %v = arith.subi %r, %c4 : index + %vi = arith.index_cast %v : index to i32 + %vf = arith.sitofp %vi : i32 to f32 + %x = arith.mulf %vf, %quarter : f32 + memref.store %x, %input[%c0, %c, %h, %w] : memref<1x3x5x5xf32> + } + } + } + scf.for %f = %c0 to %c4 step %c1 { + scf.for %kh = %c0 to %c3 step %c1 { + scf.for %kw = %c0 to %c3 step %c1 { + scf.for %c = %c0 to %c3 step %c1 { + %f2 = arith.muli %f, %c2 : index + %kh3 = arith.muli %kh, %c3 : index + %kw5 = arith.muli %kw, %c5 : index + %s0 = arith.addi %f2, %kh3 : index + %s1 = arith.addi %s0, %kw5 : index + %s = arith.addi %s1, %c : index + %r = arith.remui %s, %c7 : index + %v = arith.subi %r, %c3 : index + %q = arith.index_cast %v : index to i8 + memref.store %q, %weight[%f, %c, %kh, %kw] : memref<4x3x3x3xi8> + } + } + } + } + %multiplier = arith.constant dense<4.0> : tensor<1x1x1x1xf32> + %input_step = arith.constant dense<0.25> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<0.5> : tensor<1x1x1x1xf32> + %init = arith.constant dense<1.0> : tensor<1x4x3x3xf32> + %it = bufferization.to_tensor %input restrict : memref<1x3x5x5xf32> to tensor<1x3x5x5xf32> + %wt = bufferization.to_tensor %weight restrict : memref<4x3x3x3xi8> to tensor<4x3x3x3xi8> + %iq = "frontend.fp32_to_int8"(%it, %multiplier) : (tensor<1x3x5x5xf32>, tensor<1x1x1x1xf32>) -> tensor<1x3x5x5xi8> + %id = "frontend.int8_to_fp32"(%iq, %input_step) : (tensor<1x3x5x5xi8>, tensor<1x1x1x1xf32>) -> tensor<1x3x5x5xf32> + %wd = "frontend.int8_to_fp32"(%wt, %weight_step) : (tensor<4x3x3x3xi8>, tensor<1x1x1x1xf32>) -> tensor<4x3x3x3xf32> + %result = linalg.conv_2d_nchw_fchw {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} - ins(%input, %filter : memref<1x8x4x1xf32>, memref<6x8x1x1xf32>) - outs(%output : memref<1x6x4x1xf32>) - - func.call @check_result(%output) : (memref<1x6x4x1xf32>) -> () - - memref.dealloc %input : memref<1x8x4x1xf32> - memref.dealloc %filter : memref<6x8x1x1xf32> - memref.dealloc %output : memref<1x6x4x1xf32> - + ins(%id, %wd : tensor<1x3x5x5xf32>, tensor<4x3x3x3xf32>) outs(%init : tensor<1x4x3x3xf32>) -> tensor<1x4x3x3xf32> + %memory = bufferization.to_buffer %result : tensor<1x4x3x3xf32> to memref<1x4x3x3xf32> + func.call @check_result(%memory) : (memref<1x4x3x3xf32>) -> () return %zero_i8 : i8 } diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw_main.cpp b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw_main.cpp index 9490f9692..529725211 100644 --- a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw_main.cpp +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nchw_fchw_main.cpp @@ -15,39 +15,39 @@ static void fail(void) { #ifdef __cplusplus extern "C" #endif - void check_result(int32_t *allocated, int32_t *aligned, int64_t offset, + void check_result(float *allocated, float *aligned, int64_t offset, int64_t size0, int64_t size1, int64_t size2, int64_t size3, int64_t stride0, int64_t stride1, int64_t stride2, int64_t stride3) { (void)allocated; - - if (size0 != 1 || size1 != 6 || size2 != 4 || size3 != 1 || stride0 != 24 || - stride1 != 4 || stride2 != 1 || stride3 != 1) { - printf("FAILED: linalg conv2d NCHW unexpected memref shape " - "(size=%dx%dx%dx%d stride=%dx%dx%dx%d)\n", - (int)size0, (int)size1, (int)size2, (int)size3, (int)stride0, - (int)stride1, (int)stride2, (int)stride3); + (void)stride0; + const int64_t sizes[4] = {1, 4, 3, 3}; + if (size0 != sizes[0] || size1 != sizes[1] || size2 != sizes[2] || + size3 != sizes[3]) { + printf("FAILED: linalg conv2d_nchw_fchw unexpected memref shape\n"); fail(); } - - int32_t *output = aligned + offset; - const int32_t expected = 0x41000000; - for (int i = 0; i < 1; ++i) { - for (int c = 0; c < 6; ++c) { - for (int h = 0; h < 4; ++h) { - for (int w = 0; w < 1; ++w) { - int64_t idx = i * stride0 + c * stride1 + h * stride2 + w * stride3; - int32_t got = output[idx]; - if (got != expected) { - printf("FAILED: linalg conv2d NCHW output[%d][%d][%d][%d] " - "(expected 0x%08x, got 0x%08x)\n", - i, c, h, w, expected, got); - fail(); - } + const float *output = aligned + offset; + for (int oh = 0; oh < 3; ++oh) + for (int ow = 0; ow < 3; ++ow) + for (int f = 0; f < 4; ++f) { + int sum = 0; + for (int kh = 0; kh < 3; ++kh) + for (int kw = 0; kw < 3; ++kw) + for (int c = 0; c < 3; ++c) { + int x = (3 * (oh + kh) + 5 * (ow + kw) + 7 * c) % 9 - 4; + int w = (2 * f + 3 * kh + 5 * kw + c) % 7 - 3; + sum += x * w; + } + float expected = 1.0f + 0.125f * (float)sum; + float got = output[f * stride1 + oh * stride2 + ow * stride3]; + if (got != expected) { + printf( + "FAILED: linalg conv2d_nchw_fchw oh=%d ow=%d f=%d expected %d/8, " + "got %d/8\n", + oh, ow, f, (int)(expected * 8), (int)(got * 8)); + fail(); } } - } - } - - printf("PASSED: linalg conv2d NCHW FCHW\n"); + printf("PASSED: linalg conv2d_nchw_fchw\n"); } diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc.mlir b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc.mlir index 1c8c39948..e0a757ead 100644 --- a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc.mlir +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc.mlir @@ -1,28 +1,69 @@ -func.func private @check_result(memref<1x4x1x6xf32>) -> () +// Quantized convolution in the frontend QDQ form: input steps 0.25 with +// multiplier 4, weight step 0.5, initial output 1. The output is +// 1 + 0.125 * sum(xq * wq), so a wrong step fails the check. +func.func private @check_result(memref<1x3x3x4xf32>) -> () func.func @main() -> i8 { %zero_i8 = arith.constant 0 : i8 - %zero_f32 = arith.constant 0.0 : f32 - %one_f32 = arith.constant 1.0 : f32 - - %input = memref.alloc() : memref<1x4x1x8xf32> - %filter = memref.alloc() : memref<6x1x1x8xf32> - %output = memref.alloc() : memref<1x4x1x6xf32> - - linalg.fill ins(%one_f32 : f32) outs(%input : memref<1x4x1x8xf32>) - linalg.fill ins(%one_f32 : f32) outs(%filter : memref<6x1x1x8xf32>) - linalg.fill ins(%zero_f32 : f32) outs(%output : memref<1x4x1x6xf32>) - - linalg.conv_2d_nhwc_fhwc + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %quarter = arith.constant 0.25 : f32 + %input = memref.alloc() : memref<1x5x5x3xf32> + %weight = memref.alloc() : memref<4x3x3x3xi8> + scf.for %h = %c0 to %c5 step %c1 { + scf.for %w = %c0 to %c5 step %c1 { + scf.for %c = %c0 to %c3 step %c1 { + %h3 = arith.muli %h, %c3 : index + %w5 = arith.muli %w, %c5 : index + %c7c = arith.muli %c, %c7 : index + %s0 = arith.addi %h3, %w5 : index + %s = arith.addi %s0, %c7c : index + %r = arith.remui %s, %c9 : index + %v = arith.subi %r, %c4 : index + %vi = arith.index_cast %v : index to i32 + %vf = arith.sitofp %vi : i32 to f32 + %x = arith.mulf %vf, %quarter : f32 + memref.store %x, %input[%c0, %h, %w, %c] : memref<1x5x5x3xf32> + } + } + } + scf.for %f = %c0 to %c4 step %c1 { + scf.for %kh = %c0 to %c3 step %c1 { + scf.for %kw = %c0 to %c3 step %c1 { + scf.for %c = %c0 to %c3 step %c1 { + %f2 = arith.muli %f, %c2 : index + %kh3 = arith.muli %kh, %c3 : index + %kw5 = arith.muli %kw, %c5 : index + %s0 = arith.addi %f2, %kh3 : index + %s1 = arith.addi %s0, %kw5 : index + %s = arith.addi %s1, %c : index + %r = arith.remui %s, %c7 : index + %v = arith.subi %r, %c3 : index + %q = arith.index_cast %v : index to i8 + memref.store %q, %weight[%f, %kh, %kw, %c] : memref<4x3x3x3xi8> + } + } + } + } + %multiplier = arith.constant dense<4.0> : tensor<1x1x1x1xf32> + %input_step = arith.constant dense<0.25> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<0.5> : tensor<1x1x1x1xf32> + %init = arith.constant dense<1.0> : tensor<1x3x3x4xf32> + %it = bufferization.to_tensor %input restrict : memref<1x5x5x3xf32> to tensor<1x5x5x3xf32> + %wt = bufferization.to_tensor %weight restrict : memref<4x3x3x3xi8> to tensor<4x3x3x3xi8> + %iq = "frontend.fp32_to_int8"(%it, %multiplier) : (tensor<1x5x5x3xf32>, tensor<1x1x1x1xf32>) -> tensor<1x5x5x3xi8> + %id = "frontend.int8_to_fp32"(%iq, %input_step) : (tensor<1x5x5x3xi8>, tensor<1x1x1x1xf32>) -> tensor<1x5x5x3xf32> + %wd = "frontend.int8_to_fp32"(%wt, %weight_step) : (tensor<4x3x3x3xi8>, tensor<1x1x1x1xf32>) -> tensor<4x3x3x3xf32> + %result = linalg.conv_2d_nhwc_fhwc {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} - ins(%input, %filter : memref<1x4x1x8xf32>, memref<6x1x1x8xf32>) - outs(%output : memref<1x4x1x6xf32>) - - func.call @check_result(%output) : (memref<1x4x1x6xf32>) -> () - - memref.dealloc %input : memref<1x4x1x8xf32> - memref.dealloc %filter : memref<6x1x1x8xf32> - memref.dealloc %output : memref<1x4x1x6xf32> - + ins(%id, %wd : tensor<1x5x5x3xf32>, tensor<4x3x3x3xf32>) outs(%init : tensor<1x3x3x4xf32>) -> tensor<1x3x3x4xf32> + %memory = bufferization.to_buffer %result : tensor<1x3x3x4xf32> to memref<1x3x3x4xf32> + func.call @check_result(%memory) : (memref<1x3x3x4xf32>) -> () return %zero_i8 : i8 } diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc_main.cpp b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc_main.cpp index dedcd26e1..cceb5458e 100644 --- a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc_main.cpp +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_fhwc_main.cpp @@ -15,39 +15,39 @@ static void fail(void) { #ifdef __cplusplus extern "C" #endif - void check_result(int32_t *allocated, int32_t *aligned, int64_t offset, + void check_result(float *allocated, float *aligned, int64_t offset, int64_t size0, int64_t size1, int64_t size2, int64_t size3, int64_t stride0, int64_t stride1, int64_t stride2, int64_t stride3) { (void)allocated; - - if (size0 != 1 || size1 != 4 || size2 != 1 || size3 != 6 || stride0 != 24 || - stride1 != 6 || stride2 != 6 || stride3 != 1) { - printf("FAILED: linalg conv2d NHWC unexpected memref shape " - "(size=%dx%dx%dx%d stride=%dx%dx%dx%d)\n", - (int)size0, (int)size1, (int)size2, (int)size3, (int)stride0, - (int)stride1, (int)stride2, (int)stride3); + (void)stride0; + const int64_t sizes[4] = {1, 3, 3, 4}; + if (size0 != sizes[0] || size1 != sizes[1] || size2 != sizes[2] || + size3 != sizes[3]) { + printf("FAILED: linalg conv2d_nhwc_fhwc unexpected memref shape\n"); fail(); } - - int32_t *output = aligned + offset; - const int32_t expected = 0x41000000; - for (int i = 0; i < 1; ++i) { - for (int h = 0; h < 4; ++h) { - for (int w = 0; w < 1; ++w) { - for (int c = 0; c < 6; ++c) { - int64_t idx = i * stride0 + h * stride1 + w * stride2 + c * stride3; - int32_t got = output[idx]; - if (got != expected) { - printf("FAILED: linalg conv2d NHWC output[%d][%d][%d][%d] " - "(expected 0x%08x, got 0x%08x)\n", - i, h, w, c, expected, got); - fail(); - } + const float *output = aligned + offset; + for (int oh = 0; oh < 3; ++oh) + for (int ow = 0; ow < 3; ++ow) + for (int f = 0; f < 4; ++f) { + int sum = 0; + for (int kh = 0; kh < 3; ++kh) + for (int kw = 0; kw < 3; ++kw) + for (int c = 0; c < 3; ++c) { + int x = (3 * (oh + kh) + 5 * (ow + kw) + 7 * c) % 9 - 4; + int w = (2 * f + 3 * kh + 5 * kw + c) % 7 - 3; + sum += x * w; + } + float expected = 1.0f + 0.125f * (float)sum; + float got = output[oh * stride1 + ow * stride2 + f * stride3]; + if (got != expected) { + printf( + "FAILED: linalg conv2d_nhwc_fhwc oh=%d ow=%d f=%d expected %d/8, " + "got %d/8\n", + oh, ow, f, (int)(expected * 8), (int)(got * 8)); + fail(); } } - } - } - - printf("PASSED: linalg conv2d NHWC FHWC\n"); + printf("PASSED: linalg conv2d_nhwc_fhwc\n"); } diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf.mlir b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf.mlir index cca9db045..96b135696 100644 --- a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf.mlir +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf.mlir @@ -1,28 +1,69 @@ -func.func private @check_result(memref<1x4x1x6xf32>) -> () +// Quantized convolution in the frontend QDQ form: input steps 0.25 with +// multiplier 4, weight step 0.5, initial output 1. The output is +// 1 + 0.125 * sum(xq * wq), so a wrong step fails the check. +func.func private @check_result(memref<1x3x3x4xf32>) -> () func.func @main() -> i8 { %zero_i8 = arith.constant 0 : i8 - %zero_f32 = arith.constant 0.0 : f32 - %one_f32 = arith.constant 1.0 : f32 - - %input = memref.alloc() : memref<1x4x1x8xf32> - %filter = memref.alloc() : memref<1x1x8x6xf32> - %output = memref.alloc() : memref<1x4x1x6xf32> - - linalg.fill ins(%one_f32 : f32) outs(%input : memref<1x4x1x8xf32>) - linalg.fill ins(%one_f32 : f32) outs(%filter : memref<1x1x8x6xf32>) - linalg.fill ins(%zero_f32 : f32) outs(%output : memref<1x4x1x6xf32>) - - linalg.conv_2d_nhwc_hwcf + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %quarter = arith.constant 0.25 : f32 + %input = memref.alloc() : memref<1x5x5x3xf32> + %weight = memref.alloc() : memref<3x3x3x4xi8> + scf.for %h = %c0 to %c5 step %c1 { + scf.for %w = %c0 to %c5 step %c1 { + scf.for %c = %c0 to %c3 step %c1 { + %h3 = arith.muli %h, %c3 : index + %w5 = arith.muli %w, %c5 : index + %c7c = arith.muli %c, %c7 : index + %s0 = arith.addi %h3, %w5 : index + %s = arith.addi %s0, %c7c : index + %r = arith.remui %s, %c9 : index + %v = arith.subi %r, %c4 : index + %vi = arith.index_cast %v : index to i32 + %vf = arith.sitofp %vi : i32 to f32 + %x = arith.mulf %vf, %quarter : f32 + memref.store %x, %input[%c0, %h, %w, %c] : memref<1x5x5x3xf32> + } + } + } + scf.for %f = %c0 to %c4 step %c1 { + scf.for %kh = %c0 to %c3 step %c1 { + scf.for %kw = %c0 to %c3 step %c1 { + scf.for %c = %c0 to %c3 step %c1 { + %f2 = arith.muli %f, %c2 : index + %kh3 = arith.muli %kh, %c3 : index + %kw5 = arith.muli %kw, %c5 : index + %s0 = arith.addi %f2, %kh3 : index + %s1 = arith.addi %s0, %kw5 : index + %s = arith.addi %s1, %c : index + %r = arith.remui %s, %c7 : index + %v = arith.subi %r, %c3 : index + %q = arith.index_cast %v : index to i8 + memref.store %q, %weight[%kh, %kw, %c, %f] : memref<3x3x3x4xi8> + } + } + } + } + %multiplier = arith.constant dense<4.0> : tensor<1x1x1x1xf32> + %input_step = arith.constant dense<0.25> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<0.5> : tensor<1x1x1x1xf32> + %init = arith.constant dense<1.0> : tensor<1x3x3x4xf32> + %it = bufferization.to_tensor %input restrict : memref<1x5x5x3xf32> to tensor<1x5x5x3xf32> + %wt = bufferization.to_tensor %weight restrict : memref<3x3x3x4xi8> to tensor<3x3x3x4xi8> + %iq = "frontend.fp32_to_int8"(%it, %multiplier) : (tensor<1x5x5x3xf32>, tensor<1x1x1x1xf32>) -> tensor<1x5x5x3xi8> + %id = "frontend.int8_to_fp32"(%iq, %input_step) : (tensor<1x5x5x3xi8>, tensor<1x1x1x1xf32>) -> tensor<1x5x5x3xf32> + %wd = "frontend.int8_to_fp32"(%wt, %weight_step) : (tensor<3x3x3x4xi8>, tensor<1x1x1x1xf32>) -> tensor<3x3x3x4xf32> + %result = linalg.conv_2d_nhwc_hwcf {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} - ins(%input, %filter : memref<1x4x1x8xf32>, memref<1x1x8x6xf32>) - outs(%output : memref<1x4x1x6xf32>) - - func.call @check_result(%output) : (memref<1x4x1x6xf32>) -> () - - memref.dealloc %input : memref<1x4x1x8xf32> - memref.dealloc %filter : memref<1x1x8x6xf32> - memref.dealloc %output : memref<1x4x1x6xf32> - + ins(%id, %wd : tensor<1x5x5x3xf32>, tensor<3x3x3x4xf32>) outs(%init : tensor<1x3x3x4xf32>) -> tensor<1x3x3x4xf32> + %memory = bufferization.to_buffer %result : tensor<1x3x3x4xf32> to memref<1x3x3x4xf32> + func.call @check_result(%memory) : (memref<1x3x3x4xf32>) -> () return %zero_i8 : i8 } diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_deep.mlir b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_deep.mlir new file mode 100644 index 000000000..24c3bf8fb --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_deep.mlir @@ -0,0 +1,70 @@ +// Quantized convolution with K = 3 * 3 * 130 = 1170, deeper than one bank: +// A and B each span two bank groups. Input steps 0.25 with multiplier 4, +// weight step 0.5, initial output 1; the output is 1 + 0.125 * sum(xq * wq). +func.func private @check_result(memref<1x1x1x4xf32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %c130 = arith.constant 130 : index + %quarter = arith.constant 0.25 : f32 + %input = memref.alloc() : memref<1x3x3x130xf32> + %weight = memref.alloc() : memref<3x3x130x4xi8> + scf.for %h = %c0 to %c3 step %c1 { + scf.for %w = %c0 to %c3 step %c1 { + scf.for %c = %c0 to %c130 step %c1 { + %h3 = arith.muli %h, %c3 : index + %w5 = arith.muli %w, %c5 : index + %c7c = arith.muli %c, %c7 : index + %s0 = arith.addi %h3, %w5 : index + %s = arith.addi %s0, %c7c : index + %r = arith.remui %s, %c9 : index + %v = arith.subi %r, %c4 : index + %vi = arith.index_cast %v : index to i32 + %vf = arith.sitofp %vi : i32 to f32 + %x = arith.mulf %vf, %quarter : f32 + memref.store %x, %input[%c0, %h, %w, %c] : memref<1x3x3x130xf32> + } + } + } + scf.for %f = %c0 to %c4 step %c1 { + scf.for %kh = %c0 to %c3 step %c1 { + scf.for %kw = %c0 to %c3 step %c1 { + scf.for %c = %c0 to %c130 step %c1 { + %f2 = arith.muli %f, %c2 : index + %kh3 = arith.muli %kh, %c3 : index + %kw5 = arith.muli %kw, %c5 : index + %s0 = arith.addi %f2, %kh3 : index + %s1 = arith.addi %s0, %kw5 : index + %s = arith.addi %s1, %c : index + %r = arith.remui %s, %c7 : index + %v = arith.subi %r, %c3 : index + %q = arith.index_cast %v : index to i8 + memref.store %q, %weight[%kh, %kw, %c, %f] : memref<3x3x130x4xi8> + } + } + } + } + %multiplier = arith.constant dense<4.0> : tensor<1x1x1x1xf32> + %input_step = arith.constant dense<0.25> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<0.5> : tensor<1x1x1x1xf32> + %init = arith.constant dense<1.0> : tensor<1x1x1x4xf32> + %it = bufferization.to_tensor %input restrict : memref<1x3x3x130xf32> to tensor<1x3x3x130xf32> + %wt = bufferization.to_tensor %weight restrict : memref<3x3x130x4xi8> to tensor<3x3x130x4xi8> + %iq = "frontend.fp32_to_int8"(%it, %multiplier) : (tensor<1x3x3x130xf32>, tensor<1x1x1x1xf32>) -> tensor<1x3x3x130xi8> + %id = "frontend.int8_to_fp32"(%iq, %input_step) : (tensor<1x3x3x130xi8>, tensor<1x1x1x1xf32>) -> tensor<1x3x3x130xf32> + %wd = "frontend.int8_to_fp32"(%wt, %weight_step) : (tensor<3x3x130x4xi8>, tensor<1x1x1x1xf32>) -> tensor<3x3x130x4xf32> + %result = linalg.conv_2d_nhwc_hwcf + {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} + ins(%id, %wd : tensor<1x3x3x130xf32>, tensor<3x3x130x4xf32>) outs(%init : tensor<1x1x1x4xf32>) -> tensor<1x1x1x4xf32> + %memory = bufferization.to_buffer %result : tensor<1x1x1x4xf32> to memref<1x1x1x4xf32> + func.call @check_result(%memory) : (memref<1x1x1x4xf32>) -> () + return %zero_i8 : i8 +} diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_deep_main.cpp b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_deep_main.cpp new file mode 100644 index 000000000..821decbf5 --- /dev/null +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_deep_main.cpp @@ -0,0 +1,53 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +#ifdef __cplusplus +extern "C" +#endif + void check_result(float *allocated, float *aligned, int64_t offset, + int64_t size0, int64_t size1, int64_t size2, + int64_t size3, int64_t stride0, int64_t stride1, + int64_t stride2, int64_t stride3) { + (void)allocated; + (void)stride0; + const int64_t sizes[4] = {1, 1, 1, 4}; + if (size0 != sizes[0] || size1 != sizes[1] || size2 != sizes[2] || + size3 != sizes[3]) { + printf("FAILED: linalg conv2d_nhwc_hwcf_deep unexpected memref shape\n"); + fail(); + } + const float *output = aligned + offset; + for (int oh = 0; oh < 1; ++oh) + for (int ow = 0; ow < 1; ++ow) + for (int f = 0; f < 4; ++f) { + int sum = 0; + for (int kh = 0; kh < 3; ++kh) + for (int kw = 0; kw < 3; ++kw) + for (int c = 0; c < 130; ++c) { + int x = (3 * (oh + kh) + 5 * (ow + kw) + 7 * c) % 9 - 4; + int w = (2 * f + 3 * kh + 5 * kw + c) % 7 - 3; + sum += x * w; + } + float expected = 1.0f + 0.125f * (float)sum; + float got = output[oh * stride1 + ow * stride2 + f * stride3]; + if (got != expected) { + printf("FAILED: linalg conv2d_nhwc_hwcf_deep oh=%d ow=%d f=%d " + "expected %d/8, " + "got %d/8\n", + oh, ow, f, (int)(expected * 8), (int)(got * 8)); + fail(); + } + } + printf("PASSED: linalg conv2d_nhwc_hwcf_deep\n"); +} diff --git a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_main.cpp b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_main.cpp index 2cdd0f448..ec8ce23a7 100644 --- a/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_main.cpp +++ b/examples/designs/toy/workloads/device/core/acc/mlir_tests/linalg/conv2d_nhwc_hwcf_main.cpp @@ -15,39 +15,39 @@ static void fail(void) { #ifdef __cplusplus extern "C" #endif - void check_result(int32_t *allocated, int32_t *aligned, int64_t offset, + void check_result(float *allocated, float *aligned, int64_t offset, int64_t size0, int64_t size1, int64_t size2, int64_t size3, int64_t stride0, int64_t stride1, int64_t stride2, int64_t stride3) { (void)allocated; - - if (size0 != 1 || size1 != 4 || size2 != 1 || size3 != 6 || stride0 != 24 || - stride1 != 6 || stride2 != 6 || stride3 != 1) { - printf("FAILED: linalg conv2d NHWC unexpected memref shape " - "(size=%dx%dx%dx%d stride=%dx%dx%dx%d)\n", - (int)size0, (int)size1, (int)size2, (int)size3, (int)stride0, - (int)stride1, (int)stride2, (int)stride3); + (void)stride0; + const int64_t sizes[4] = {1, 3, 3, 4}; + if (size0 != sizes[0] || size1 != sizes[1] || size2 != sizes[2] || + size3 != sizes[3]) { + printf("FAILED: linalg conv2d_nhwc_hwcf unexpected memref shape\n"); fail(); } - - int32_t *output = aligned + offset; - const int32_t expected = 0x41000000; - for (int i = 0; i < 1; ++i) { - for (int h = 0; h < 4; ++h) { - for (int w = 0; w < 1; ++w) { - for (int c = 0; c < 6; ++c) { - int64_t idx = i * stride0 + h * stride1 + w * stride2 + c * stride3; - int32_t got = output[idx]; - if (got != expected) { - printf("FAILED: linalg conv2d NHWC output[%d][%d][%d][%d] " - "(expected 0x%08x, got 0x%08x)\n", - i, h, w, c, expected, got); - fail(); - } + const float *output = aligned + offset; + for (int oh = 0; oh < 3; ++oh) + for (int ow = 0; ow < 3; ++ow) + for (int f = 0; f < 4; ++f) { + int sum = 0; + for (int kh = 0; kh < 3; ++kh) + for (int kw = 0; kw < 3; ++kw) + for (int c = 0; c < 3; ++c) { + int x = (3 * (oh + kh) + 5 * (ow + kw) + 7 * c) % 9 - 4; + int w = (2 * f + 3 * kh + 5 * kw + c) % 7 - 3; + sum += x * w; + } + float expected = 1.0f + 0.125f * (float)sum; + float got = output[oh * stride1 + ow * stride2 + f * stride3]; + if (got != expected) { + printf( + "FAILED: linalg conv2d_nhwc_hwcf oh=%d ow=%d f=%d expected %d/8, " + "got %d/8\n", + oh, ow, f, (int)(expected * 8), (int)(got * 8)); + fail(); } } - } - } - - printf("PASSED: linalg conv2d NHWC HWCF\n"); + printf("PASSED: linalg conv2d_nhwc_hwcf\n"); } diff --git a/stack b/stack index 034548ef2..ef427ae93 160000 --- a/stack +++ b/stack @@ -1 +1 @@ -Subproject commit 034548ef296e78e94de35b9f246cc4c1fea79f27 +Subproject commit ef427ae930afc8a4c40a98e427e580a8675dcd8c diff --git a/thirdparty/soc-framework b/thirdparty/soc-framework index ca4438456..a434559bc 160000 --- a/thirdparty/soc-framework +++ b/thirdparty/soc-framework @@ -1 +1 @@ -Subproject commit ca443845609d1c3cbebf96577e33f8132787e4ed +Subproject commit a434559bc6f5ac4c352466908366b5b087236aaf diff --git a/verify b/verify index b997b9687..00ddfc942 160000 --- a/verify +++ b/verify @@ -1 +1 @@ -Subproject commit b997b9687825cd5c6fcd0d12fd461faf72b082a1 +Subproject commit 00ddfc942bb906d3a10fad71787349a67c831d4f From 98b040b353118776a4007c23dbedd29319e2c883 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 26 Oct 2026 20:40:04 +0800 Subject: [PATCH 192/196] feat: sync to buckyball 0.2.1-a --- .../verify/src/common/gemmini_defs.svh | 6 + .../gemmini/verify/src/tests/gemmini_test.svh | 131 ++++++++ .../components/balls/im2col/arch/README.md | 74 ++--- .../balls/im2col/configs/default.toml | 15 +- .../components/balls/im2col/emu/src/im2col.rs | 27 +- .../balls/im2col/stack/src/Lowering.cpp | 61 ++-- .../components/balls/im2col/verify/README.md | 54 ++-- .../im2col/verify/casegen/src/casegen.rs | 306 ++++++++---------- .../balls/im2col/verify/casegen/src/lib.rs | 99 +++--- .../balls/im2col/verify/casegen/src/model.rs | 160 +++++---- .../im2col/verify/src/common/im2col_defs.svh | 33 +- .../im2col/verify/src/common/im2col_items.svh | 117 +++---- .../im2col/verify/src/cov/im2col_cov.svh | 50 +-- .../verify/src/env/im2col_scoreboard.svh | 96 +++--- .../verify/src/seq/im2col_sequences.svh | 4 +- .../im2col/verify/src/tests/im2col_test.svh | 27 +- .../im2col/workloads/ctests/im2col_k1_test.c | 85 ++--- .../ctests/im2col_k3_asym_pad_test.c | 90 ++---- .../workloads/ctests/im2col_k3_pad_test.c | 89 ++--- .../workloads/ctests/im2col_k3_partial_test.c | 85 ++--- .../im2col/workloads/ctests/im2col_k3_test.c | 89 ++--- .../im2col/workloads/ctests/im2col_k5_test.c | 82 ++--- .../im2col/workloads/ctests/im2col_k7_test.c | 90 ++---- .../workloads/ctests/im2col_window_test.c | 53 ++- .../components/balls/int2fp/emu/src/int2fp.rs | 42 +-- .../int2fp/verify/casegen/src/casegen.rs | 93 ++++-- .../int2fp/verify/src/common/int2fp_defs.svh | 8 +- .../int2fp/verify/src/common/int2fp_items.svh | 17 +- .../int2fp/verify/src/cov/int2fp_cov.svh | 7 +- .../verify/src/env/int2fp_scoreboard.svh | 31 +- .../int2fp/workloads/ctests/CMakeLists.txt | 1 + .../workloads/ctests/int2fp_groups_test.c | 49 +++ .../balls/maxpool/emu/src/maxpool.rs | 62 ++-- .../balls/maxpool/stack/src/Lowering.cpp | 53 +-- .../maxpool/verify/casegen/src/casegen.rs | 119 +++++-- .../balls/maxpool/verify/casegen/src/lib.rs | 22 ++ .../verify/src/common/maxpool_defs.svh | 6 +- .../verify/src/common/maxpool_items.svh | 14 +- .../maxpool/verify/src/cov/maxpool_cov.svh | 5 +- .../verify/src/env/maxpool_scoreboard.svh | 23 +- .../maxpool/workloads/ctests/CMakeLists.txt | 1 + .../workloads/ctests/maxpool_groups_test.c | 47 +++ examples/components/balls/smatmul/README.md | 13 +- .../balls/smatmul/emu/src/smatmul.rs | 84 +++-- .../balls/smatmul/stack/src/Lowering.cpp | 50 ++- .../smatmul/verify/casegen/src/casegen.rs | 138 +++++++- .../balls/smatmul/verify/casegen/src/model.rs | 10 +- .../verify/src/common/smatmul_defs.svh | 2 + .../verify/src/common/smatmul_items.svh | 13 +- .../verify/src/env/smatmul_scoreboard.svh | 29 +- .../smatmul/verify/src/tests/smatmul_test.svh | 2 +- .../smatmul/workloads/ctests/CMakeLists.txt | 2 + .../workloads/ctests/smatmul_c_groups_test.c | 55 ++++ .../workloads/ctests/smatmul_k_groups_test.c | 52 +++ .../components/balls/toint8/emu/src/quant.rs | 96 +++--- .../balls/toint8/stack/src/Lowering.cpp | 19 +- .../components/balls/toint8/verify/README.md | 2 + .../toint8/verify/casegen/src/casegen.rs | 108 ++++++- .../toint8/verify/src/common/toint8_defs.svh | 9 +- .../toint8/verify/src/common/toint8_items.svh | 18 +- .../toint8/verify/src/cov/toint8_cov.svh | 5 +- .../verify/src/env/toint8_scoreboard.svh | 34 +- .../toint8/workloads/ctests/CMakeLists.txt | 2 + .../workloads/ctests/quant_f32_groups_test.c | 40 +++ .../workloads/ctests/quant_i32_groups_test.c | 55 ++++ .../regression/bemu/workloads-bare.toml | 6 + .../regression/verilator/workloads-bare.toml | 5 + .../acc/ctests/dual_conv_add_int2fp_test.c | 17 +- .../ctests/im2col_mvin_smatmul_hazard_test.c | 2 +- .../im2col_smatmul_quant_pipeline/pebble.c | 21 +- .../core/acc/ctests/maxpool_im2col_test.c | 2 +- .../acc/ctests/mega_conv_k5_pipeline_test.c | 28 +- .../core/acc/ctests/mega_conv_pipeline_test.c | 14 +- 73 files changed, 2000 insertions(+), 1456 deletions(-) create mode 100644 examples/components/balls/int2fp/workloads/ctests/int2fp_groups_test.c create mode 100644 examples/components/balls/maxpool/workloads/ctests/maxpool_groups_test.c create mode 100644 examples/components/balls/smatmul/workloads/ctests/smatmul_c_groups_test.c create mode 100644 examples/components/balls/smatmul/workloads/ctests/smatmul_k_groups_test.c create mode 100644 examples/components/balls/toint8/workloads/ctests/quant_f32_groups_test.c create mode 100644 examples/components/balls/toint8/workloads/ctests/quant_i32_groups_test.c diff --git a/examples/components/balls/gemmini/verify/src/common/gemmini_defs.svh b/examples/components/balls/gemmini/verify/src/common/gemmini_defs.svh index 994104200..91abb7a33 100644 --- a/examples/components/balls/gemmini/verify/src/common/gemmini_defs.svh +++ b/examples/components/balls/gemmini/verify/src/common/gemmini_defs.svh @@ -4,3 +4,9 @@ `ifndef GEMMINI_LOOP_WS_CONFIG_BOUNDS_FUNCT7 `error "GEMMINI_LOOP_WS_CONFIG_BOUNDS_FUNCT7 must be provided by the selected Thread ballISA" `endif +`ifndef GEMMINI_PRELOAD_FUNCT7 +`error "GEMMINI_PRELOAD_FUNCT7 must be provided by the selected Thread ballISA" +`endif +`ifndef GEMMINI_COMPUTE_PRELOADED_FUNCT7 +`error "GEMMINI_COMPUTE_PRELOADED_FUNCT7 must be provided by the selected Thread ballISA" +`endif diff --git a/examples/components/balls/gemmini/verify/src/tests/gemmini_test.svh b/examples/components/balls/gemmini/verify/src/tests/gemmini_test.svh index f71d49588..8b183bcf6 100644 --- a/examples/components/balls/gemmini/verify/src/tests/gemmini_test.svh +++ b/examples/components/balls/gemmini/verify/src/tests/gemmini_test.svh @@ -22,6 +22,7 @@ class gemmini_ball_test extends uvm_test; apply_reset(); run_config(); run_loop_bounds(); + run_group_select(); phase.drop_objection(this); endtask @@ -121,6 +122,136 @@ class gemmini_ball_test extends uvm_test; `uvm_info("GEMMINI", "loop-ws bounds config passed", UVM_LOW) endtask + // Operands span three groups of their banks; rs2[41:36] and rs2[47:42] pick + // A's group 2 and B's group 1. The other groups hold different data, so a + // wrong group fails both the read check and the product. + localparam int GroupA = 2; + localparam int GroupB = 1; + + // Row `row` of group `group` of bank 1 (A) or bank 2 (B). + function bit [127:0] bank_row(int bank, int group, int row); + bit [127:0] data; + for (int lane = 0; lane < 16; lane++) + if (bank == 1) data[lane*8+:8] = group == GroupA ? 8'(lane == row % 16) : 8'd5; + else data[lane*8+:8] = group == GroupB ? 8'((row + 2 * lane + 1) % 7 - 3) : 8'd9; + return data; + endfunction + + // Issues one command and serves its bank traffic until it completes: + // reads answer one cycle after they fire, writes are recorded in c. + task issue(int rob, int funct, bit [63:0] rs2, int op1_bank, int op2_bank, + ref bit [31:0] c[16][16], ref int written); + bit read_fire[`BB_IN_BW], write_fire[`BB_OUT_BW]; + bit read_done[`BB_IN_BW], write_done[`BB_OUT_BW]; + int read_bank[`BB_IN_BW], read_group[`BB_IN_BW], read_addr[`BB_IN_BW]; + bit cmd_fire = 0; + int cycles; + @(negedge vif.clock); + vif.cmd_req_bits_rob_id = rob; + vif.cmd_req_bits_funct = funct; + vif.cmd_req_bits_rs2 = rs2; + vif.cmd_req_bits_iter = 16; + vif.cmd_req_bits_op1_bank = op1_bank; + vif.cmd_req_bits_op2_bank = op2_bank; + vif.cmd_req_bits_wr_bank = 3; + vif.cmd_req_bits_op1_group = '0; + vif.cmd_req_bits_op2_group = '0; + vif.cmd_req_bits_wr_group = '0; + vif.cmd_req_bits_op1_col = 3; + vif.cmd_req_bits_op2_col = 3; + vif.cmd_req_bits_wr_col = 4; + vif.cmd_req_bits_wr_read_bank = '0; + vif.cmd_req_bits_wr_arch_bank = 3; + vif.cmd_req_valid = 1'b1; + cycles = 0; + forever begin + // Values set at this negedge are sampled with the DUT outputs at the + // next posedge, so handshakes seen here fire at that edge. + if (cmd_fire) vif.cmd_req_valid = 1'b0; + for (int port = 0; port < `BB_IN_BW; port++) begin + if (read_done[port]) vif.bank_read_resp_valid[port] = 1'b0; + if (read_fire[port]) begin + if (vif.bank_read_resp_valid[port]) `uvm_fatal("BANK", "read response overrun") + vif.bank_read_resp_valid[port] = 1'b1; + vif.bank_read_resp_data[port] = + bank_row(read_bank[port], read_group[port], read_addr[port]); + end + end + for (int port = 0; port < `BB_OUT_BW; port++) begin + if (write_done[port]) vif.bank_write_resp_valid[port] = 1'b0; + if (write_fire[port]) vif.bank_write_resp_valid[port] = 1'b1; + end + #1; + cmd_fire = vif.cmd_req_valid && vif.cmd_req_ready; + for (int port = 0; port < `BB_IN_BW; port++) begin + read_done[port] = vif.bank_read_resp_valid[port] && vif.bank_read_resp_ready[port]; + read_fire[port] = vif.bank_read_req_valid[port] && vif.bank_read_req_ready[port]; + if (read_fire[port]) begin + read_bank[port] = vif.bank_read_bank_id[port]; + read_group[port] = vif.bank_read_group_id[port]; + read_addr[port] = vif.bank_read_req_addr[port]; + verification_pkg::evidence::checks++; + if (read_bank[port] != op1_bank && read_bank[port] != op2_bank) + `uvm_fatal("BANK", $sformatf("read of bank %0d", read_bank[port])) + if (read_group[port] != (read_bank[port] == 1 ? GroupA : GroupB)) + `uvm_fatal("GROUP", $sformatf( + "bank %0d read group %0d", read_bank[port], read_group[port])) + end + end + for (int port = 0; port < `BB_OUT_BW; port++) begin + write_done[port] = vif.bank_write_resp_valid[port] && vif.bank_write_resp_ready[port]; + write_fire[port] = vif.bank_write_req_valid[port] && vif.bank_write_req_ready[port]; + if (write_fire[port]) begin + int group = vif.bank_write_group_id[port]; + int row = vif.bank_write_req_addr[port]; + if (vif.bank_write_bank_id[port] != 3 || group > 3 || row > 15 || + vif.bank_write_req_mask[port] !== 16'hffff) + `uvm_fatal("BANK", $sformatf( + "write bank %0d group %0d row %0d", vif.bank_write_bank_id[port], group, row + )) + for (int lane = 0; lane < 4; lane++) + c[row][group*4+lane] = vif.bank_write_req_data[port][lane*32+:32]; + written++; + end + end + if (vif.cmd_resp_valid && vif.cmd_resp_ready) break; + @(negedge vif.clock); + if (++cycles > 2000) `uvm_fatal("TIMEOUT", $sformatf("command %0h did not complete", rs2)) + end + verification_pkg::evidence::checks++; + if (vif.cmd_resp_bits_rob_id !== rob) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0h", vif.cmd_resp_bits_rob_id)) + @(negedge vif.clock); + for (int port = 0; port < `BB_IN_BW; port++) vif.bank_read_resp_valid[port] = 1'b0; + for (int port = 0; port < `BB_OUT_BW; port++) vif.bank_write_resp_valid[port] = 1'b0; + endtask + + // CONFIG (output-stationary, no shift), PRELOAD of B and COMPUTE_PRELOADED of + // A @ B with A the identity: C must be B's selected group. + task run_group_select(); + bit [31:0] c[16][16]; + int written = 0; + issue('h6, `GEMMINI_CONFIG_FUNCT7, 64'd0, 0, 0, c, written); + issue('h7, `GEMMINI_PRELOAD_FUNCT7, 64'd1 | (64'(GroupB) << 36), 2, 0, c, written); + issue('h8, `GEMMINI_COMPUTE_PRELOADED_FUNCT7, 64'd2 | (64'(GroupA) << 36) | (64'(GroupB) << 42), + 1, 2, c, written); + verification_pkg::evidence::checks++; + if (written != 64) `uvm_fatal("WRITE", $sformatf("%0d row writes, expected 64", written)) + for (int row = 0; row < 16; row++) + for (int col = 0; col < 16; col++) + if ($signed(c[row][col]) != (row + 2 * col + 1) % 7 - 3) + `uvm_fatal("RESULT", $sformatf( + "C[%0d][%0d] = %0d, expected %0d", + row, + col, + $signed( + c[row][col] + ), + (row + 2 * col + 1) % 7 - 3 + )) + `uvm_info("GEMMINI", "operand group select passed", UVM_LOW) + endtask + function void check_no_bank_traffic(string what); for (int i = 0; i < `BB_IN_BW; i++) if (vif.bank_read_req_valid[i]) `uvm_fatal("BANK", {what, " unexpectedly read a bank"}) diff --git a/examples/components/balls/im2col/arch/README.md b/examples/components/balls/im2col/arch/README.md index c246a8dad..71463a5c1 100644 --- a/examples/components/balls/im2col/arch/README.md +++ b/examples/components/balls/im2col/arch/README.md @@ -6,63 +6,53 @@ function 48. ## ISA contract ```c -bb_im2col(input_bank, output_bank, iter, ksize, stride, padding) +bb_im2col(input_bank, output_bank, input_size, kernel, stride, padding, + input_base, start_row, start_col, window_start, window_count, + tile_rows) ``` -- `iter` is both the input height and input width. -- `ksize` is both the kernel height and kernel width. -- `stride` applies to output rows and columns. -- `padding` is zero-padding on every input edge. -- Input and output elements are signed or unsigned 8-bit payload bytes; Im2col - only rearranges bytes and does not multiply or accumulate them. +- `rs1`: bank0 is the input, bank2 the output, `iter` the input height and + width. +- `rs2`: `[3:0]` kernel, `[7:4]` stride, `[11:8]` padding, `[15:12]` + start_col, `[19:16]` start_row, `[25:20]` input_base, `[35:26]` + window_start, `[45:36]` window_count, `[50:46]` tile_rows (1, 2, 4, 8, or + 16); `[63:51]` are zero. +- Im2col only moves INT8 bytes; it does not multiply or accumulate. -The output dimension is: +## Layout -```text -outputDim = floor((iter + 2 * padding - ksize) / stride) + 1 -``` +Every group of the input virtual bank is one channel block: group `g` line +`input_base + y * input_size + x` holds channels `16g .. 16g + 15` of pixel +`(y, x)`. With `Cg` input groups, `C = 16 * Cg`. -Windows form the rows of SMatMulBall operand A. The kernel elements form its K -dimension. Output is stored in SMatMulBall's `(M tile, K tile, tile row, lane)` -layout, with both tile dimensions fixed at 16: +The output is SMatMulBall operand A for `M = window_count` windows and +`K = kernel * kernel * C`, ordered `(kernel row, kernel column, channel)`: ```text -M = outputDim * outputDim -K = ksize * ksize -mTiles = ceil(M / 16) -kTiles = ceil(K / 16) - -bankRow(window, kernel) = - ((window / 16) * kTiles + (kernel / 16)) * 16 + (window % 16) -lane(window, kernel) = kernel % 16 +outputRows = (input_size + 2 * padding - kernel - start_row) / stride + 1 +outputCols = (input_size + 2 * padding - kernel - start_col) / stride + 1 +window w = output pixel window_start + w (row-major) +K tile t = (kr * kernel + kc) * Cg + g // one whole input row +K_TILES = kernel * kernel * Cg +G = K_TILES / output_groups // output groups divide K_TILES ``` -Every bank row contains 16 int8 lanes. Unused lanes in the final K tile are -zero. Consequently each M tile occupies `kTiles * 16` bank rows; a kernel wider -than 16 elements continues in the next K tile rather than immediately after -the preceding window. - -For output coordinate `(orow, ocol)` and kernel coordinate `(krow, kcol)`, the -unpadded input coordinate is: +Output group `j` holds K tiles `j * G .. j * G + G - 1`, in the SMatMulBall A +layout with `tile_rows`-row M tiles: ```text -inputRow = orow * stride + krow - padding -inputCol = ocol * stride + kcol - padding +line(w, t) = ((w / tile_rows) * G + t % G) * tile_rows + w % tile_rows ``` -An out-of-range coordinate emits zero. Otherwise the source byte offset is: - -```text -inputRow * iter + inputCol -``` +A tile row is the input row of pixel +`(oy * stride + kr + start_row - padding, ox * stride + kc + start_col - padding)`, +or zero when that pixel is in the padding. Windows from `window_count` to the +next multiple of `tile_rows` are zero. Each output group therefore is the +SMatMulBall reduction block `j` with `k = 16 * G`. ## Files -- `Im2col.scala`: top-level command and response control. -- `Im2colConfigRegs.scala`: ISA field decoding and validation. -- `Im2colWindow.scala`: output-window and kernel traversal. -- `LineBufferManager.scala`: input preload and padded element selection. -- `StreamWriter.scala`: SMatMulBall-compatible tiled output packing and bank - writes. +- `Im2col.scala`: command decode, shape division, and the row-copy loop. +- `RestoringDiv.scala`: shared sequential divider. - `Im2colBall.scala`: Blink-compatible ball wrapper. - `configs/Im2colBallParam.scala`: maximum dimension and element width. diff --git a/examples/components/balls/im2col/configs/default.toml b/examples/components/balls/im2col/configs/default.toml index 863fa7b3f..b743f7bad 100644 --- a/examples/components/balls/im2col/configs/default.toml +++ b/examples/components/balls/im2col/configs/default.toml @@ -9,20 +9,15 @@ # paddedSize = iter + 2*padding >= ksize + start # op1_bank != wr_bank # inputBase + iter*iter <= bankEntries -# lane in 0 .. 15 -# outputDim = floor((iter + 2*padding - ksize - start) / stride) + 1 -# M = outputDim * outputDim -# K = ksize * ksize -# outputRows = ceil(M/16) * ceil(K/16) * 16 -# outputRows <= bankEntries # bankEntries from design memdomain +# window_start + window_count <= outputDim * outputDim +# K tiles = ksize * ksize * input groups, divisible by output groups +# ceil(window_count / tile_rows) * tile_rows * K tiles / output groups <= bankEntries # -# Partial final K-core lanes and partial final M-core rows are zero. -# One bank row is one spatial position with 16 INT8 channels. +# Input groups are 16-channel blocks; one bank row is one spatial position. inputWidth = 8 # Resident Conv lowering can request a 4x4 output core from a stride-2, -# kernel-7 producer, which needs a 13x13 local input. Keep this a power of two -# because the RTL line buffer splits maxIter^2 beats into two equal banks. +# kernel-7 producer, which needs a 13x13 local input. maxIter = 16 maxKSize = 7 maxPadding = 7 diff --git a/examples/components/balls/im2col/emu/src/im2col.rs b/examples/components/balls/im2col/emu/src/im2col.rs index 6b51716a1..c8269e234 100644 --- a/examples/components/balls/im2col/emu/src/im2col.rs +++ b/examples/components/balls/im2col/emu/src/im2col.rs @@ -1,17 +1,29 @@ -use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::instruction::ExecContext; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; fn max_iter(instance: u32) -> usize { - bebop_bemu::configs::ball_domain::parameters::("im2col", instance).max_iter.expect("maxIter") as usize + bebop_bemu::configs::ball_domain::parameters::< + bebop_bemu::configs::ball_domain::parameters::im2col::Params, + >("im2col", instance) + .max_iter + .expect("maxIter") as usize } fn max_kernel(instance: u32) -> usize { - bebop_bemu::configs::ball_domain::parameters::("im2col", instance).max_k_size.expect("maxKSize") as usize + bebop_bemu::configs::ball_domain::parameters::< + bebop_bemu::configs::ball_domain::parameters::im2col::Params, + >("im2col", instance) + .max_k_size + .expect("maxKSize") as usize } fn max_padding(instance: u32) -> usize { - bebop_bemu::configs::ball_domain::parameters::("im2col", instance).max_padding.expect("maxPadding") as usize + bebop_bemu::configs::ball_domain::parameters::< + bebop_bemu::configs::ball_domain::parameters::im2col::Params, + >("im2col", instance) + .max_padding + .expect("maxPadding") as usize } #[derive(Clone, Copy)] @@ -153,6 +165,9 @@ pub(crate) fn exec_im2col(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { pub(crate) fn latency(xs1: u64, xs2: u64, instance: u32) -> u64 { let shape = decode_shape(xs1, xs2); output_cols(shape, instance); - (shape.window_count.div_ceil(shape.tile_rows) * shape.tile_rows * shape.kernel * shape.kernel * 2) - as u64 + (shape.window_count.div_ceil(shape.tile_rows) + * shape.tile_rows + * shape.kernel + * shape.kernel + * 2) as u64 } diff --git a/examples/components/balls/im2col/stack/src/Lowering.cpp b/examples/components/balls/im2col/stack/src/Lowering.cpp index c29680164..e04b1f6e8 100644 --- a/examples/components/balls/im2col/stack/src/Lowering.cpp +++ b/examples/components/balls/im2col/stack/src/Lowering.cpp @@ -1,4 +1,17 @@ //===- Lowering.cpp - Im2colBall in lower-buckyball -----------------------===// +// +// im2col_i8: the source holds a square feature map of `input_side` pixels per +// side; group g is channel block g, line y * input_side + x holds the sixteen +// channels of pixel (y, x). The destination receives the windows of the +// `kernel` x `kernel` convolution with `stride` and zero `padding`, every +// window in row-major order, as the matmul_i8 A operand: K is ordered (kernel +// row, kernel column, channel), sixteen columns (one source line) per K tile, +// K tile t = (kr * kernel + kc) * groups + g. The destination groups divide +// the K tiles; group j holds the next K / 16 / groups tiles in M tiles of +// `tile_rows` rows (the layout of the matmul that consumes it), windows +// beyond the last zero up to a multiple of tile_rows. +// +//===----------------------------------------------------------------------===// #include "Buckyball/BuckyballOps.h" #include "Conversion/Buckyball.h" @@ -23,46 +36,28 @@ class Im2colI8Lowering : public OpRewritePattern { int64_t kernel = cast(op->getAttr("kernel")).getInt(); int64_t stride = cast(op->getAttr("stride")).getInt(); int64_t padding = cast(op->getAttr("padding")).getInt(); - int64_t startRow = cast(op->getAttr("start_row")).getInt(); - int64_t startCol = cast(op->getAttr("start_col")).getInt(); - int64_t base = cast(op->getAttr("op1_base")).getInt(); - int64_t lane = cast(op->getAttr("lane")).getInt(); - int64_t windowStart = - cast(op->getAttr("window_start")).getInt(); - int64_t windowCount = - cast(op->getAttr("window_count")).getInt(); + int64_t tileRows = cast(op->getAttr("tile_rows")).getInt(); const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || side <= 0 || side >= (1LL << 33) || + int64_t span = side + 2 * padding - kernel; + if (target.bankWidthBits != 128 || side <= 0 || side > buckyball_target::getBuckyballBallParam("IM2COL", "maxIter") || - kernel <= 0 || kernel > 255 || + side * side > target.bankDepth || kernel <= 0 || kernel > buckyball_target::getBuckyballBallParam("IM2COL", "maxKSize") || - stride <= 0 || stride > 255 || padding < 0 || padding > 255 || + stride <= 0 || stride > 15 || padding < 0 || padding > buckyball_target::getBuckyballBallParam("IM2COL", "maxPadding") || - startRow < 0 || startRow > padding || startCol < 0 || - startCol > padding || base < 0 || base > 63 || lane < 0 || lane > 15 || - windowStart < 0 || windowStart > 63 || windowCount <= 0 || - windowCount > 64) + span < 0 || span % stride || tileRows <= 0 || tileRows > 16 || + (tileRows & (tileRows - 1))) return op.emitOpError("im2col_i8 geometry exceeds the ISA field ranges"); - int64_t rowNumerator = side + 2 * padding - kernel - startRow; - int64_t colNumerator = side + 2 * padding - kernel - startCol; - if (rowNumerator < 0 || colNumerator < 0 || rowNumerator % stride || - colNumerator % stride || - windowStart + windowCount > - (rowNumerator / stride + 1) * (colNumerator / stride + 1) || - base + side * side > target.bankDepth || - ((windowCount + 15) / 16) * ((kernel * kernel + 15) / 16) * 16 > - target.bankDepth) - return op.emitOpError("im2col_i8 window or bank footprint is invalid"); - uint64_t rs2 = - mlir::buddy::field(kernel, 0, 7) | mlir::buddy::field(stride, 8, 15) | - mlir::buddy::field(padding, 16, 23) | - mlir::buddy::field(startCol, 24, 31) | - mlir::buddy::field(startRow, 32, 39) | - mlir::buddy::field(base, 40, 45) | mlir::buddy::field(lane, 46, 49) | - mlir::buddy::field(windowStart, 50, 55) | - mlir::buddy::field(windowCount, 56, 62); + int64_t windows = (span / stride + 1) * (span / stride + 1); + if (windows > 1023) + return op.emitOpError("im2col_i8 has more windows than one instruction"); + uint64_t rs2 = mlir::buddy::field(kernel, 0, 3) | + mlir::buddy::field(stride, 4, 7) | + mlir::buddy::field(padding, 8, 11) | + mlir::buddy::field(windows, 36, 45) | + mlir::buddy::field(tileRows, 46, 50); mlir::buddy::emitInstruction( rewriter, op.getLoc(), buckyball_target::getBuckyballFunct7("IM2COL"), mlir::buddy::constantI64(rewriter, op.getLoc(), diff --git a/examples/components/balls/im2col/verify/README.md b/examples/components/balls/im2col/verify/README.md index f2edcbf01..dfef4cb7b 100644 --- a/examples/components/balls/im2col/verify/README.md +++ b/examples/components/balls/im2col/verify/README.md @@ -8,37 +8,35 @@ Verifies the generated `Im2colBall` module directly using the shared Blink UVM f - `src/common/im2col_defs.svh`: DPI imports and constants - `src/common/im2col_items.svh`: `im2col_cmd_item` with `load_rust_case` - `src/seq/im2col_sequences.svh`: `im2col_basic_seq` driving one case per run -- `src/cov/im2col_cov.svh`: ksize/stride/pad/iter coverage -- `src/env/im2col_scoreboard.svh`: preloads mem model, compares writes vs DPI dst words +- `src/cov/im2col_cov.svh`: kernel/stride/padding/start/base/window/tile_rows/group coverage +- `src/env/im2col_scoreboard.svh`: preloads every input group, checks reads and writes in issue order - `src/env/im2col_env.svh`: extends `bb_blink_env#(1,1)` with scoreboard and cov -- `src/tests/im2col_test.svh`: directed cases 0,1 then random 2..21 with fixed seed +- `src/tests/im2col_test.svh`: runs the directed cases in order - `src/pkg/im2col_pkg.sv`: package include entry - `src/tb_top.sv`: instantiates `bb_blink_if#(1,1)` and `Im2colBall` - `filelists/im2col_ball.f`: VCS/Verilator filelist ## Test plan -`+UVM_TESTNAME=im2col_ball_test` runs: +`casegen/src/model.rs` is the reference (same semantics as `emu/src/im2col.rs`): +input group g is channel block g, K tile `t = (kr * kernel + kc) * in_groups + g` +copies one input row, output group `t / G` (`G = K_TILES / wr_col`) line +`((w / tile_rows) * G + t % G) * tile_rows + w % tile_rows`. The scoreboard checks +every read (group, line) and every write (group, line, data) in the Ball's issue +order: K tile outer, window inner. -- case 0: directed k3 6x6 pad0 (ctest `im2col_k3_test.c` data) -- case 1: directed k3 6x6 pad1 (ctest `im2col_k3_pad_test.c` data) -- cases 2..21: random with fixed seed `0xCAFE_BABE`, iter in {3,4,5,6,7}, - ksize in {1,3,5}, stride=1, pad in {0,1}, distinct banks 0..7 +Directed cases (`casegen/src/casegen.rs`): -Each case: reset, preload src words into `mem_model`, drive one command, wait for -`scb.done()` or timeout at 100000 cycles. Scoreboard compares observed writes -(data/addr/mask/bank/rob) with DPI `im2col_case_dst_word_*`. `im2col_cov` -requires ksize/stride/pad/iter bins hit across the whole test, else -`check_phase` fatal. +- 0: 1x1 kernel, 2x2 input, two input and two output groups +- 1: 3x3 kernel, 4x4 input, nine output groups (one tap each), tile_rows 8 +- 2: 3x3 kernel, padding 1, 2x2 input, one output group, tile_rows 4 +- 3: bottom/right padding (start_row = start_col = 1), 2x2 kernel, four output groups +- 4: window range 5..7 at input_base 2, partial M tile zero-filled +- 5: 5x5 kernel, tile_rows 1, five output groups (one image row each) +- 6: 7x7 stride-2 kernel, 9x9 input, seven output groups +- 7: 2x2 kernel, two input groups, input_base 3, windows 1..2, eight output groups -Blink command fields (not ISA rs packing): - -- `iter` = cmd.iter -- `ksize` = cmd.special[7:0] -- `stride` = cmd.special[15:8] -- `padding` = cmd.special[23:16] -- banks = op1_bank / wr_bank; funct7 is injected from the selected Core ballISA -- TB: op1_col=1, wr_col=1, rs1=0, rs2=0 +Op1/wr base groups are nonzero in several cases. ## BID (required) @@ -86,17 +84,5 @@ Run: Or via bbdev: ```console -bbdev uvm --build '--ball=im2col --config sims.verilator.BuckyballPebbleVerilatorConfig' -bbdev uvm --run '--ball=im2col' -- '+BID=2' +bbdev uvm --run '--design pebble --ball im2col' ``` - -## Acceptance (2026-08-04) - -- [x] Layout matches Blink common + ball-local split; filelist uses `@UVM@` / `@RTL@` -- [x] casegen pure im2col model (no ExecContext); whole-case DPI API -- [x] directed k3 pad0/pad1 scoreboard pass -- [x] deterministic random idx 2..21 pass -- [x] protocol + semantic cover enforced (`check_phase` fatal) -- [x] pebble: `+BID=2` — 22/22 -- [x] toy: `+BID=3` — 22/22 -- [x] Note: omit `-hsopt=off` for Im2col (VCS segfault); bbdev skips it for this ball diff --git a/examples/components/balls/im2col/verify/casegen/src/casegen.rs b/examples/components/balls/im2col/verify/casegen/src/casegen.rs index d31b01622..997dfbe11 100644 --- a/examples/components/balls/im2col/verify/casegen/src/casegen.rs +++ b/examples/components/balls/im2col/verify/casegen/src/casegen.rs @@ -1,16 +1,11 @@ -use crate::model; - -pub const BANK_ROW_BYTES: usize = model::BANK_ROW_BYTES; -pub const MAX_WORDS: usize = 128; +use crate::model::{im2col, Row, Shape}; #[repr(C)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Im2colCmd { pub iter: u32, - pub ksize: u32, - pub stride: u32, - pub padding: u32, - pub window_count: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, pub op1_bank: u32, pub wr_bank: u32, pub op1_group: u32, @@ -18,199 +13,170 @@ pub struct Im2colCmd { pub op1_col: u32, pub wr_col: u32, pub rob_id: u32, - pub num_src_words: u32, - pub num_dst_words: u32, + pub num_src: u32, + pub num_reads: u32, + pub num_dst: u32, } -#[derive(Clone, Debug, PartialEq, Eq)] pub struct Im2colCase { pub cmd: Im2colCmd, - pub src: Vec, - pub dst: Vec, -} - -impl Im2colCase { - pub fn src_word_lo(&self, word_index: usize) -> u64 { - word_lo(&self.src, word_index) - } - pub fn src_word_hi(&self, word_index: usize) -> u64 { - word_hi(&self.src, word_index) - } - pub fn dst_word_lo(&self, word_index: usize) -> u64 { - word_lo(&self.dst, word_index) - } - pub fn dst_word_hi(&self, word_index: usize) -> u64 { - word_hi(&self.dst, word_index) - } -} - -fn word_lo(data: &[u8], word_index: usize) -> u64 { - let off = word_index * BANK_ROW_BYTES; - if off + 8 > data.len() { - panic!("word_lo: word_index {word_index} out of range"); - } - let mut buf = [0u8; 8]; - buf.copy_from_slice(&data[off..off + 8]); - u64::from_le_bytes(buf) + // (input group, line, data) preloaded into the input bank. + pub src: Vec<(usize, usize, Row)>, + pub reads: Vec<(usize, usize)>, + pub dst: Vec<(usize, usize, Row)>, } -fn word_hi(data: &[u8], word_index: usize) -> u64 { - let off = word_index * BANK_ROW_BYTES + 8; - if off + 8 > data.len() { - panic!("word_hi: word_index {word_index} out of range"); - } - let mut buf = [0u8; 8]; - buf.copy_from_slice(&data[off..off + 8]); - u64::from_le_bytes(buf) -} - -pub fn gen_case(seed: u32, index: u32) -> Im2colCase { +pub fn gen_case(index: u32) -> Im2colCase { match index { - 0 => directed_k3_pad0(), - 1 => directed_k3_pad1(), - _ => random_case(seed, index), + 0 => k1_two_groups(), + 1 => k3_nine_output_groups(), + 2 => k3_pad_tile4(), + 3 => asym_pad(), + 4 => partial_window_range(), + 5 => k5_tile_rows_1(), + 6 => k7_stride2(), + 7 => window_input_base_two_groups(), + _ => panic!("im2col: unsupported directed case {index}"), } } -fn build_case( - iter: usize, - ksize: usize, - stride: usize, - padding: usize, - op1_bank: u32, - wr_bank: u32, - op1_group: u32, - wr_group: u32, +fn finish( + shape: Shape, + input: Vec>, + banks: (u32, u32), + groups: (u32, u32), rob_id: u32, - flat: &[u8], ) -> Im2colCase { - if op1_bank == wr_bank { - panic!("build_case: op1_bank and wr_bank must differ"); - } - if op1_bank > 7 || wr_bank > 7 { - panic!("build_case: banks must be in 0..7"); - } - if flat.len() != iter * iter { - panic!("build_case: flat len mismatch"); - } - let rows = model::output_rows(iter, ksize, stride, padding); - let windows = ((iter + 2 * padding - ksize) / stride + 1).pow(2); - if rows > model::bank_entries() { - panic!("build_case: output_rows={rows} exceeds bank depth"); - } - let dst_flat = model::im2col(flat, iter, ksize, stride, padding); - let src = model::pack_bank_words(flat); - let nsrc = model::num_words(src.len()); - let ndst = model::num_words(dst_flat.len()); - if nsrc > model::bank_entries() || ndst > model::bank_entries() || nsrc > MAX_WORDS || ndst > MAX_WORDS { - panic!("build_case: word count out of range src={nsrc} dst={ndst}"); + let trace = im2col(&shape, &input); + let rs2 = shape.rs2(); + let mut src = Vec::new(); + for (g, rows) in input.iter().enumerate() { + for (line, row) in rows.iter().enumerate() { + src.push((g, line, *row)); + } } Im2colCase { cmd: Im2colCmd { - iter: iter as u32, - ksize: ksize as u32, - stride: stride as u32, - padding: padding as u32, - window_count: windows as u32, - op1_bank, - wr_bank, - op1_group, - wr_group, - op1_col: 1, - wr_col: 1, + iter: shape.size as u32, + rs2_lo: rs2 as u32, + rs2_hi: (rs2 >> 32) as u32, + op1_bank: banks.0, + wr_bank: banks.1, + op1_group: groups.0, + wr_group: groups.1, + op1_col: shape.in_groups as u32, + wr_col: shape.out_groups as u32, rob_id, - num_src_words: nsrc as u32, - num_dst_words: ndst as u32, + num_src: src.len() as u32, + num_reads: trace.reads.len() as u32, + num_dst: trace.writes.len() as u32, }, src, - dst: dst_flat, + reads: trace.reads, + dst: trace.writes, } } -fn directed_k3_pad0() -> Im2colCase { - let iter = 6usize; - let k = 3usize; - let flat: [i8; 36] = [ - -1, 2, 3, 4, -5, 6, 0, 8, 9, -10, 11, 12, 13, 0, -15, 16, 17, 18, 19, -20, 0, 22, 23, -24, - -25, 26, 27, 0, -29, 30, 31, 32, 33, -34, 0, 36, - ]; - let bytes: Vec = flat.iter().map(|&x| x as u8).collect(); - build_case(iter, k, 1, 0, 0, 1, 0, 0, 3, &bytes) +fn row(seed: usize) -> Row { + let mut r = [0u8; 16]; + for (lane, byte) in r.iter_mut().enumerate() { + *byte = (seed * 16 + lane) as u8; + } + r } -fn directed_k3_pad1() -> Im2colCase { - let iter = 6usize; - let k = 3usize; - let flat: [i8; 36] = [ - 7, -2, 3, 0, 5, -6, 1, 8, -9, 10, 11, 0, -4, 12, 13, 14, 0, 16, 17, 0, 19, -20, 21, 22, 0, - 24, 25, 26, -27, 28, 29, -30, 0, 32, 33, 34, +// 1x1 kernel on a 2x2 input, 32 channels: output group g copies input group g. +fn k1_two_groups() -> Im2colCase { + let input = vec![ + (0..4).map(|p| row(p * 2 + 1)).collect(), + (0..4).map(|p| row(p * 2 + 40)).collect(), ]; - let bytes: Vec = flat.iter().map(|&x| x as u8).collect(); - build_case(iter, k, 1, 1, 0, 1, 1, 2, 5, &bytes) + let shape = Shape { + size: 2, kernel: 1, stride: 1, padding: 0, start_row: 0, start_col: 0, + input_base: 0, window_start: 0, window_count: 4, tile_rows: 4, + in_groups: 2, out_groups: 2, + }; + finish(shape, input, (0, 1), (0, 0), 1) } -fn shape_ok(iter: usize, ksize: usize, stride: usize, padding: usize) -> bool { - if iter == 0 || ksize == 0 || stride == 0 { - return false; - } - let padded = iter + 2 * padding; - if padded < ksize { - return false; - } - let rows = model::output_rows(iter, ksize, stride, padding); - rows <= model::bank_entries() +// 3x3 kernel on a 4x4 input: 2x2 windows, nine output groups (one tap each), +// lines 4..7 of each 8-row M tile are zero. +fn k3_nine_output_groups() -> Im2colCase { + let input = vec![(0..16).map(|p| row(p * 3 + 2)).collect()]; + let shape = Shape { + size: 4, kernel: 3, stride: 1, padding: 0, start_row: 0, start_col: 0, + input_base: 0, window_start: 0, window_count: 4, tile_rows: 8, + in_groups: 1, out_groups: 9, + }; + finish(shape, input, (2, 5), (1, 3), 2) } -fn random_case(seed: u32, index: u32) -> Im2colCase { - let mut rng = Rng::new(seed, index); - let iter_pool = [3usize, 4, 5, 6, 7]; - let ksize_pool = [1usize, 3, 5]; - let pad_pool = [0usize, 1]; - let base = (index - 2) as usize; +// 3x3 kernel, padding 1, 2x2 input: nine taps in one group, 4-row M tile. +fn k3_pad_tile4() -> Im2colCase { + let input = vec![(0..4).map(|p| row(p + 7)).collect()]; + let shape = Shape { + size: 2, kernel: 3, stride: 1, padding: 1, start_row: 0, start_col: 0, + input_base: 0, window_start: 0, window_count: 4, tile_rows: 4, + in_groups: 1, out_groups: 1, + }; + finish(shape, input, (3, 1), (0, 2), 3) +} - for attempt in 0..10_000 { - let iter = iter_pool[(base + attempt) % iter_pool.len()]; - let ksize = ksize_pool[((base / iter_pool.len()) + attempt) % ksize_pool.len()]; - let stride = 1usize; - let padding = pad_pool[(base + attempt / 3) % pad_pool.len()]; - if !shape_ok(iter, ksize, stride, padding) { - continue; - } - let op1_bank = rng.range(0, 7); - let mut wr_bank = rng.range(0, 7); - if wr_bank == op1_bank { - wr_bank = (wr_bank + 1) & 7; - } - let mut flat = vec![0u8; iter * iter]; - for b in flat.iter_mut() { - *b = (rng.next() & 0xff) as u8; - } - let rob_id = rng.range(0, 15); - let op1_group = rng.range(0, 3); - let wr_group = rng.range(0, 3); - return build_case( - iter, ksize, stride, padding, op1_bank, wr_bank, op1_group, wr_group, rob_id, &flat, - ); - } - panic!("random_case: failed to find legal shape for index={index}"); +// Bottom/right padding only: 2x2 kernel, padding 1, start 1/1, 3x3 input. +fn asym_pad() -> Im2colCase { + let input = vec![(0..9).map(|p| row(p * 5 + 1)).collect()]; + let shape = Shape { + size: 3, kernel: 2, stride: 1, padding: 1, start_row: 1, start_col: 1, + input_base: 0, window_start: 0, window_count: 9, tile_rows: 16, + in_groups: 1, out_groups: 4, + }; + finish(shape, input, (0, 4), (0, 0), 4) } -struct Rng { - state: u64, +// Windows 5..7 of a 1x1 im2col on a 4x4 input at input_base 2; lines 3..7 zero. +fn partial_window_range() -> Im2colCase { + let input = vec![(0..18).map(|p| row(p + 100)).collect()]; + let shape = Shape { + size: 4, kernel: 1, stride: 1, padding: 0, start_row: 0, start_col: 0, + input_base: 2, window_start: 5, window_count: 3, tile_rows: 8, + in_groups: 1, out_groups: 1, + }; + finish(shape, input, (1, 0), (2, 1), 5) } -impl Rng { - fn new(seed: u32, index: u32) -> Self { - let state = (u64::from(seed) << 32) ^ u64::from(index) ^ 0x9E37_79B9_7F4A_7C15; - Self { state } - } - fn next(&mut self) -> u32 { - self.state ^= self.state >> 12; - self.state ^= self.state << 25; - self.state ^= self.state >> 27; - ((self.state.wrapping_mul(0x2545_F491_4F6C_DD1D)) >> 32) as u32 - } - fn range(&mut self, lo: u32, hi: u32) -> u32 { - lo + (self.next() % (hi - lo + 1)) - } +// 5x5 kernel on a 5x5 input with tile_rows 1: output group kr is image row kr. +fn k5_tile_rows_1() -> Im2colCase { + let input = vec![(0..25).map(|p| row(p * 7 + 3)).collect()]; + let shape = Shape { + size: 5, kernel: 5, stride: 1, padding: 0, start_row: 0, start_col: 0, + input_base: 0, window_start: 0, window_count: 1, tile_rows: 1, + in_groups: 1, out_groups: 5, + }; + finish(shape, input, (0, 1), (0, 0), 6) +} + +// 7x7 stride-2 kernel on a 9x9 input: 2x2 windows, one kernel row per group. +fn k7_stride2() -> Im2colCase { + let input = vec![(0..81).map(|p| row(p * 11 + 5)).collect()]; + let shape = Shape { + size: 9, kernel: 7, stride: 2, padding: 0, start_row: 0, start_col: 0, + input_base: 0, window_start: 0, window_count: 4, tile_rows: 4, + in_groups: 1, out_groups: 7, + }; + finish(shape, input, (6, 2), (1, 0), 7) +} + +// 2x2 kernel on a 3x3 input with 32 channels at input_base 3, windows 1..2: +// eight output groups, one (tap, channel group) tile each. +fn window_input_base_two_groups() -> Im2colCase { + let input = vec![ + (0..12).map(|p| row(p * 3 + 9)).collect(), + (0..12).map(|p| row(p * 3 + 60)).collect(), + ]; + let shape = Shape { + size: 3, kernel: 2, stride: 1, padding: 0, start_row: 0, start_col: 0, + input_base: 3, window_start: 1, window_count: 2, tile_rows: 2, + in_groups: 2, out_groups: 8, + }; + finish(shape, input, (4, 7), (0, 2), 8) } diff --git a/examples/components/balls/im2col/verify/casegen/src/lib.rs b/examples/components/balls/im2col/verify/casegen/src/lib.rs index be6a3be65..db4b3458d 100644 --- a/examples/components/balls/im2col/verify/casegen/src/lib.rs +++ b/examples/components/balls/im2col/verify/casegen/src/lib.rs @@ -3,22 +3,10 @@ mod model; use std::cell::RefCell; -use casegen::{Im2colCase, Im2colCmd, MAX_WORDS}; +use casegen::{Im2colCase, Im2colCmd}; thread_local! { - static CURRENT: RefCell> = RefCell::new(None); -} - -#[no_mangle] -pub extern "C" fn im2col_case_load(seed: u32, index: u32) -> i32 { - let case = casegen::gen_case(seed, index); - let nsrc = case.cmd.num_src_words as usize; - let ndst = case.cmd.num_dst_words as usize; - if nsrc > MAX_WORDS || ndst > MAX_WORDS { - panic!("im2col_case_load: word count out of range src={nsrc} dst={ndst}"); - } - CURRENT.with(|c| *c.borrow_mut() = Some(case)); - 0 + static CURRENT: RefCell> = const { RefCell::new(None) }; } fn current_case R, R>(f: F) -> R { @@ -28,44 +16,75 @@ fn current_case R, R>(f: F) -> R { }) } +fn lo(row: &[u8; 16]) -> u64 { + u64::from_le_bytes(row[..8].try_into().unwrap()) +} + +fn hi(row: &[u8; 16]) -> u64 { + u64::from_le_bytes(row[8..].try_into().unwrap()) +} + #[no_mangle] -pub extern "C" fn im2col_case_cmd(out_ptr: *mut Im2colCmd) { - current_case(|case| unsafe { - if out_ptr.is_null() { - panic!("im2col_case_cmd: null out_ptr"); - } - *out_ptr = case.cmd; - }); +pub extern "C" fn im2col_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); + CURRENT.with(|c| *c.borrow_mut() = Some(case)); + 0 } #[no_mangle] -pub extern "C" fn im2col_case_src_word_lo(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("im2col_case_src_word_lo: word_index out of range"); +pub extern "C" fn im2col_case_cmd(out_ptr: *mut Im2colCmd) { + if out_ptr.is_null() { + panic!("im2col_case_cmd: null out_ptr"); } - current_case(|case| case.src_word_lo(word_index as usize)) + current_case(|case| unsafe { *out_ptr = case.cmd }); } #[no_mangle] -pub extern "C" fn im2col_case_src_word_hi(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("im2col_case_src_word_hi: word_index out of range"); - } - current_case(|case| case.src_word_hi(word_index as usize)) +pub extern "C" fn im2col_case_src_group(i: u32) -> u32 { + current_case(|case| case.src[i as usize].0 as u32) } #[no_mangle] -pub extern "C" fn im2col_case_dst_word_lo(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("im2col_case_dst_word_lo: word_index out of range"); - } - current_case(|case| case.dst_word_lo(word_index as usize)) +pub extern "C" fn im2col_case_src_addr(i: u32) -> u32 { + current_case(|case| case.src[i as usize].1 as u32) } #[no_mangle] -pub extern "C" fn im2col_case_dst_word_hi(word_index: u32) -> u64 { - if word_index as usize >= MAX_WORDS { - panic!("im2col_case_dst_word_hi: word_index out of range"); - } - current_case(|case| case.dst_word_hi(word_index as usize)) +pub extern "C" fn im2col_case_src_lo(i: u32) -> u64 { + current_case(|case| lo(&case.src[i as usize].2)) +} + +#[no_mangle] +pub extern "C" fn im2col_case_src_hi(i: u32) -> u64 { + current_case(|case| hi(&case.src[i as usize].2)) +} + +#[no_mangle] +pub extern "C" fn im2col_case_read_group(i: u32) -> u32 { + current_case(|case| case.reads[i as usize].0 as u32) +} + +#[no_mangle] +pub extern "C" fn im2col_case_read_addr(i: u32) -> u32 { + current_case(|case| case.reads[i as usize].1 as u32) +} + +#[no_mangle] +pub extern "C" fn im2col_case_dst_group(i: u32) -> u32 { + current_case(|case| case.dst[i as usize].0 as u32) +} + +#[no_mangle] +pub extern "C" fn im2col_case_dst_addr(i: u32) -> u32 { + current_case(|case| case.dst[i as usize].1 as u32) +} + +#[no_mangle] +pub extern "C" fn im2col_case_dst_lo(i: u32) -> u64 { + current_case(|case| lo(&case.dst[i as usize].2)) +} + +#[no_mangle] +pub extern "C" fn im2col_case_dst_hi(i: u32) -> u64 { + current_case(|case| hi(&case.dst[i as usize].2)) } diff --git a/examples/components/balls/im2col/verify/casegen/src/model.rs b/examples/components/balls/im2col/verify/casegen/src/model.rs index d4d374d2e..227a2b38a 100644 --- a/examples/components/balls/im2col/verify/casegen/src/model.rs +++ b/examples/components/balls/im2col/verify/casegen/src/model.rs @@ -1,103 +1,93 @@ -pub const CORE: usize = 16; -pub const BANK_ROW_BYTES: usize = CORE; +// Reference IM2COL model in the RTL's traversal order (see emu/src/im2col.rs +// and arch/README.md): K tile t outer, window w inner, one write per output +// line, one read per in-bounds tap of a window below window_count. + +pub type Row = [u8; 16]; pub fn bank_entries() -> usize { - let config = match std::env::var("BB_VERIFY_CONFIG") { - Ok(path) => path, - Err(_) if cfg!(test) => return 64, - Err(_) => panic!("BB_VERIFY_CONFIG is required"), - }; + let config = std::env::var("BB_VERIFY_CONFIG").expect("BB_VERIFY_CONFIG is required"); let text = std::fs::read_to_string(config).expect("failed to read BB_VERIFY_CONFIG"); text.lines() .find_map(|line| line.strip_prefix("bank_entries=")) - .unwrap_or_else(|| panic!("bank_entries missing from BB_VERIFY_CONFIG")) + .expect("bank_entries missing from BB_VERIFY_CONFIG") .parse() .expect("invalid bank_entries in BB_VERIFY_CONFIG") } -pub fn output_rows(iter: usize, ksize: usize, stride: usize, padding: usize) -> usize { - if iter == 0 || ksize == 0 || stride == 0 { - panic!("im2col model: iter/ksize/stride must be >= 1"); - } - let padded = iter + 2 * padding; - if padded < ksize { - panic!("im2col model: kernel larger than padded input"); - } - let out_dim = (padded - ksize) / stride + 1; - let windows = out_dim * out_dim; - let kernel = ksize * ksize; - let m_tiles = windows.div_ceil(CORE); - let k_tiles = kernel.div_ceil(CORE); - m_tiles * k_tiles * CORE +pub struct Shape { + pub size: usize, + pub kernel: usize, + pub stride: usize, + pub padding: usize, + pub start_row: usize, + pub start_col: usize, + pub input_base: usize, + pub window_start: usize, + pub window_count: usize, + pub tile_rows: usize, + pub in_groups: usize, + pub out_groups: usize, } -pub fn im2col(src: &[u8], iter: usize, ksize: usize, stride: usize, padding: usize) -> Vec { - if src.len() != iter * iter { - panic!( - "im2col model: src len {} != iter*iter {}", - src.len(), - iter * iter - ); - } - let padded = iter + 2 * padding; - if padded < ksize { - panic!("im2col model: kernel larger than padded input"); +impl Shape { + pub fn rs2(&self) -> u64 { + self.kernel as u64 + | (self.stride as u64) << 4 + | (self.padding as u64) << 8 + | (self.start_col as u64) << 12 + | (self.start_row as u64) << 16 + | (self.input_base as u64) << 20 + | (self.window_start as u64) << 26 + | (self.window_count as u64) << 36 + | (self.tile_rows as u64) << 46 } - if stride == 0 { - panic!("im2col model: stride must be >= 1"); - } - let out_dim = (padded - ksize) / stride + 1; - let windows = out_dim * out_dim; - let kernel_elems = ksize * ksize; - let m_tiles = windows.div_ceil(CORE); - let k_tiles = kernel_elems.div_ceil(CORE); - let rows = m_tiles * k_tiles * CORE; - let mut output = vec![0u8; rows * CORE]; - - let mut window = 0usize; - for orow in 0..out_dim { - for ocol in 0..out_dim { - for kr in 0..ksize { - for kc in 0..ksize { - let kernel = kr * ksize + kc; - let m_tile = window / CORE; - let m_row = window % CORE; - let k_tile = kernel / CORE; - let lane = kernel % CORE; - let bank_row = (m_tile * k_tiles + k_tile) * CORE + m_row; - let out = bank_row * CORE + lane; - if out >= output.len() { - panic!("im2col model: output range out={out}"); - } - let input_row = (orow * stride + kr) as isize - padding as isize; - let input_col = (ocol * stride + kc) as isize - padding as isize; - if input_row >= 0 - && input_row < iter as isize - && input_col >= 0 - && input_col < iter as isize - { - let src_i = input_row as usize * iter + input_col as usize; - output[out] = src[src_i]; - } - } - } - window += 1; - } - } - output } -pub fn pack_bank_words(flat: &[u8]) -> Vec { - let mut out = vec![0u8; flat.len() * BANK_ROW_BYTES]; - for (index, value) in flat.iter().enumerate() { - out[index * BANK_ROW_BYTES] = *value; - } - out +pub struct Trace { + // (input group, line) per read, in issue order. + pub reads: Vec<(usize, usize)>, + // (output group, line, data) per write, in issue order. + pub writes: Vec<(usize, usize, Row)>, } -pub fn num_words(total_bytes: usize) -> usize { - if total_bytes % BANK_ROW_BYTES != 0 { - panic!("num_words: total_bytes {total_bytes} not a multiple of bank row"); +// `input[g][line]` is input group g; lines below input_base are present too. +pub fn im2col(s: &Shape, input: &[Vec]) -> Trace { + let padded = s.size + 2 * s.padding; + let out_rows = (padded - s.kernel - s.start_row) / s.stride + 1; + let out_cols = (padded - s.kernel - s.start_col) / s.stride + 1; + assert!((padded - s.kernel - s.start_row) % s.stride == 0); + assert!((padded - s.kernel - s.start_col) % s.stride == 0); + assert!(s.window_start + s.window_count <= out_rows * out_cols); + assert!(s.input_base + s.size * s.size <= bank_entries()); + let k_tiles = s.kernel * s.kernel * s.in_groups; + assert!(k_tiles % s.out_groups == 0); + let group_tiles = k_tiles / s.out_groups; + let padded_windows = s.window_count.div_ceil(s.tile_rows) * s.tile_rows; + assert!(padded_windows * group_tiles <= bank_entries()); + + let mut trace = Trace { reads: Vec::new(), writes: Vec::new() }; + for tile in 0..k_tiles { + let g = tile % s.in_groups; + let kr = tile / s.in_groups / s.kernel; + let kc = tile / s.in_groups % s.kernel; + for w in 0..padded_windows { + let line = ((w / s.tile_rows) * group_tiles + tile % group_tiles) * s.tile_rows + + w % s.tile_rows; + let mut data = [0u8; 16]; + if w < s.window_count { + let window = s.window_start + w; + let y = (s.start_row + window / out_cols * s.stride + kr) as isize + - s.padding as isize; + let x = (s.start_col + window % out_cols * s.stride + kc) as isize + - s.padding as isize; + if y >= 0 && x >= 0 && (y as usize) < s.size && (x as usize) < s.size { + let source = s.input_base + y as usize * s.size + x as usize; + trace.reads.push((g, source)); + data = input[g][source]; + } + } + trace.writes.push((tile / group_tiles, line, data)); + } } - total_bytes / BANK_ROW_BYTES + trace } diff --git a/examples/components/balls/im2col/verify/src/common/im2col_defs.svh b/examples/components/balls/im2col/verify/src/common/im2col_defs.svh index c5a9d46e2..09b7678ab 100644 --- a/examples/components/balls/im2col/verify/src/common/im2col_defs.svh +++ b/examples/components/balls/im2col/verify/src/common/im2col_defs.svh @@ -1,9 +1,7 @@ typedef struct { int unsigned iter; - int unsigned ksize; - int unsigned stride; - int unsigned padding; - int unsigned window_count; + int unsigned rs2_lo; + int unsigned rs2_hi; int unsigned op1_bank; int unsigned wr_bank; int unsigned op1_group; @@ -11,24 +9,27 @@ typedef struct { int unsigned op1_col; int unsigned wr_col; int unsigned rob_id; - int unsigned num_src_words; - int unsigned num_dst_words; + int unsigned num_src; + int unsigned num_reads; + int unsigned num_dst; } im2col_cmd_dpi_t; -import "DPI-C" function int im2col_case_load( - input int unsigned seed, - input int unsigned index -); +import "DPI-C" function int im2col_case_load(input int unsigned index); import "DPI-C" function void im2col_case_cmd(output im2col_cmd_dpi_t cmd); -import "DPI-C" function longint unsigned im2col_case_src_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned im2col_case_src_word_hi(input int unsigned word_index); -import "DPI-C" function longint unsigned im2col_case_dst_word_lo(input int unsigned word_index); -import "DPI-C" function longint unsigned im2col_case_dst_word_hi(input int unsigned word_index); +import "DPI-C" function int unsigned im2col_case_src_group(input int unsigned i); +import "DPI-C" function int unsigned im2col_case_src_addr(input int unsigned i); +import "DPI-C" function longint unsigned im2col_case_src_lo(input int unsigned i); +import "DPI-C" function longint unsigned im2col_case_src_hi(input int unsigned i); +import "DPI-C" function int unsigned im2col_case_read_group(input int unsigned i); +import "DPI-C" function int unsigned im2col_case_read_addr(input int unsigned i); +import "DPI-C" function int unsigned im2col_case_dst_group(input int unsigned i); +import "DPI-C" function int unsigned im2col_case_dst_addr(input int unsigned i); +import "DPI-C" function longint unsigned im2col_case_dst_lo(input int unsigned i); +import "DPI-C" function longint unsigned im2col_case_dst_hi(input int unsigned i); `ifndef IM2COL_FUNCT7 `error "IM2COL_FUNCT7 must be provided by the selected Thread ballISA" `endif localparam int IM2COL_THREAD_FUNCT7 = `IM2COL_FUNCT7; -localparam int IM2COL_MAX_WORDS = 128; +localparam int IM2COL_NUM_CASES = 8; localparam int IM2COL_TIMEOUT_CYCLES = 100000; -localparam int IM2COL_SEED = 32'hCAFE_BABE; diff --git a/examples/components/balls/im2col/verify/src/common/im2col_items.svh b/examples/components/balls/im2col/verify/src/common/im2col_items.svh index a14e137b0..50c410b70 100644 --- a/examples/components/balls/im2col/verify/src/common/im2col_items.svh +++ b/examples/components/balls/im2col/verify/src/common/im2col_items.svh @@ -1,96 +1,71 @@ class im2col_cmd_item extends bb_blink_cmd_item; `uvm_object_utils(im2col_cmd_item) - bit [31:0] ksize; - bit [31:0] stride; - bit [31:0] padding; - bit [31:0] window_count; - bit [31:0] num_src_words; - bit [31:0] num_dst_words; - bit [127:0] src_words[IM2COL_MAX_WORDS]; - bit [127:0] dst_words[IM2COL_MAX_WORDS]; + // Input rows to preload, the expected reads, and the expected writes, each + // as (group, line[, data]) in the order the Ball issues them. + int unsigned src_group[$]; + int unsigned src_addr[$]; + bit [127:0] src_data[$]; + int unsigned read_group[$]; + int unsigned read_addr[$]; + int unsigned dst_group[$]; + int unsigned dst_addr[$]; + bit [127:0] dst_data[$]; function new(string name = "im2col_cmd_item"); super.new(name); endfunction - function void load_rust_case(int unsigned seed, int unsigned index); + function void load_rust_case(int unsigned index); im2col_cmd_dpi_t cmd; - int unsigned rc; - longint unsigned w_lo; - longint unsigned w_hi; - int i; - rc = im2col_case_load(seed, index); - if (rc != 0) begin - `uvm_fatal("CASE", $sformatf("im2col_case_load returned %0d for index %0d", rc, index)) - end + if (im2col_case_load(index) != 0) `uvm_fatal("CASE", $sformatf("im2col case %0d failed", index)) im2col_case_cmd(cmd); - ksize = cmd.ksize; - stride = cmd.stride; - padding = cmd.padding; - window_count = cmd.window_count; - num_src_words = cmd.num_src_words; - num_dst_words = cmd.num_dst_words; - - // bb_im2col: banks and input_size (iter) come from rs1, the window geometry from rs2. - rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; - funct = IM2COL_THREAD_FUNCT7[6:0]; - rs2 = { - 1'b0, window_count[6:0], 6'd0, 4'd0, 6'd0, 8'd0, 8'd0, padding[7:0], stride[7:0], ksize[7:0] - }; - iter = cmd.iter; - op1_bank = cmd.op1_bank; - op2_bank = '0; - wr_bank = cmd.wr_bank; - op1_group = cmd.op1_group; - op2_group = '0; - wr_group = cmd.wr_group; - op1_col = cmd.op1_col; - op2_col = '0; - wr_col = cmd.wr_col; + // bb_im2col: banks and input size come from rs1, the window geometry from rs2. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = IM2COL_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = '0; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = '0; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = '0; + wr_col = cmd.wr_col; wr_read_bank = '0; wr_arch_bank = cmd.wr_bank; - if (num_src_words > IM2COL_MAX_WORDS || num_dst_words > IM2COL_MAX_WORDS) begin - `uvm_fatal("CASE", $sformatf("word count out of range src=%0d dst=%0d", num_src_words, - num_dst_words)) + for (int i = 0; i < cmd.num_src; i++) begin + src_group.push_back(im2col_case_src_group(i)); + src_addr.push_back(im2col_case_src_addr(i)); + src_data.push_back({im2col_case_src_hi(i), im2col_case_src_lo(i)}); end - if (op1_bank == wr_bank) begin - `uvm_fatal("CASE", "op1_bank and wr_bank overlap") + for (int i = 0; i < cmd.num_reads; i++) begin + read_group.push_back(im2col_case_read_group(i)); + read_addr.push_back(im2col_case_read_addr(i)); end - if (stride == 0) begin - `uvm_fatal("CASE", "stride must be >= 1") - end - - for (i = 0; i < num_src_words; i++) begin - w_lo = im2col_case_src_word_lo(i); - w_hi = im2col_case_src_word_hi(i); - src_words[i] = {w_hi, w_lo}; - end - for (i = 0; i < num_dst_words; i++) begin - w_lo = im2col_case_dst_word_lo(i); - w_hi = im2col_case_dst_word_hi(i); - dst_words[i] = {w_hi, w_lo}; + for (int i = 0; i < cmd.num_dst; i++) begin + dst_group.push_back(im2col_case_dst_group(i)); + dst_addr.push_back(im2col_case_dst_addr(i)); + dst_data.push_back({im2col_case_dst_hi(i), im2col_case_dst_lo(i)}); end endfunction function void do_copy(uvm_object rhs); im2col_cmd_item rhs_; super.do_copy(rhs); - if (!$cast(rhs_, rhs)) begin - `uvm_fatal("COPY", "rhs is not im2col_cmd_item") - end - ksize = rhs_.ksize; - stride = rhs_.stride; - padding = rhs_.padding; - window_count = rhs_.window_count; - num_src_words = rhs_.num_src_words; - num_dst_words = rhs_.num_dst_words; - for (int i = 0; i < IM2COL_MAX_WORDS; i++) begin - src_words[i] = rhs_.src_words[i]; - dst_words[i] = rhs_.dst_words[i]; - end + if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not im2col_cmd_item") + src_group = rhs_.src_group; + src_addr = rhs_.src_addr; + src_data = rhs_.src_data; + read_group = rhs_.read_group; + read_addr = rhs_.read_addr; + dst_group = rhs_.dst_group; + dst_addr = rhs_.dst_addr; + dst_data = rhs_.dst_data; endfunction endclass diff --git a/examples/components/balls/im2col/verify/src/cov/im2col_cov.svh b/examples/components/balls/im2col/verify/src/cov/im2col_cov.svh index 0ff8c5efa..f45b4c22f 100644 --- a/examples/components/balls/im2col/verify/src/cov/im2col_cov.svh +++ b/examples/components/balls/im2col/verify/src/cov/im2col_cov.svh @@ -4,18 +4,28 @@ class im2col_cov extends uvm_component; uvm_analysis_imp_cmd #(bb_blink_cmd_item, im2col_cov) cmd_imp; covergroup cmd_cg; - coverpoint cur_ksize {bins k1 = {1}; bins k3 = {3}; bins k5 = {5};} - coverpoint cur_stride {bins s1 = {1};} - coverpoint cur_pad {bins p0 = {0}; bins p1 = {1};} - coverpoint cur_iter { - bins it3 = {3}; bins it4 = {4}; bins it5 = {5}; bins it6 = {6}; bins it7 = {7}; + coverpoint kernel {bins k1 = {1}; bins k2 = {2}; bins k3 = {3}; bins k5 = {5}; bins k7 = {7};} + coverpoint stride {bins s1 = {1}; bins s2 = {2};} + coverpoint padding {bins p0 = {0}; bins p1 = {1};} + coverpoint start {bins zero = {0}; bins shifted = {[1 : $]};} + coverpoint input_base {bins zero = {0}; bins offset = {[1 : $]};} + coverpoint window_start {bins zero = {0}; bins offset = {[1 : $]};} + coverpoint tile_rows { + bins t1 = {1}; bins t2 = {2}; bins t4 = {4}; bins t8 = {8}; bins t16 = {16}; } + coverpoint in_groups {bins one = {1}; bins two = {2};} + coverpoint out_groups {bins one = {1}; bins many = {[2 : $]};} endgroup - int unsigned cur_ksize; - int unsigned cur_stride; - int unsigned cur_pad; - int unsigned cur_iter; + int unsigned kernel; + int unsigned stride; + int unsigned padding; + int unsigned start; + int unsigned input_base; + int unsigned window_start; + int unsigned tile_rows; + int unsigned in_groups; + int unsigned out_groups; int cmd_count; function new(string name, uvm_component parent); @@ -25,10 +35,15 @@ class im2col_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - cur_ksize = int'(item.rs2[7:0]); - cur_stride = int'(item.rs2[15:8]); - cur_pad = int'(item.rs2[23:16]); - cur_iter = int'(item.iter); + kernel = int'(item.rs2[3:0]); + stride = int'(item.rs2[7:4]); + padding = int'(item.rs2[11:8]); + start = int'(item.rs2[19:12]); + input_base = int'(item.rs2[25:20]); + window_start = int'(item.rs2[35:26]); + tile_rows = int'(item.rs2[50:46]); + in_groups = int'(item.op1_col); + out_groups = int'(item.wr_col); cmd_count++; cmd_cg.sample(); endfunction @@ -37,12 +52,7 @@ class im2col_cov extends uvm_component; verification_pkg::evidence::checks++; super.check_phase(phase); if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") - if (cmd_cg.get_coverage() != 100.0) begin - `uvm_fatal( - "COV", - $sformatf( - "im2col_cov coverage incomplete: %0.2f%% (need ksize {1,3,5}, stride {1}, pad {0,1}, iter {3,4,5,6,7})", - cmd_cg.get_coverage())) - end + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("im2col_cov coverage incomplete: %0.2f%%", cmd_cg.get_coverage())) endfunction endclass diff --git a/examples/components/balls/im2col/verify/src/env/im2col_scoreboard.svh b/examples/components/balls/im2col/verify/src/env/im2col_scoreboard.svh index 00f39d8cc..a12962560 100644 --- a/examples/components/balls/im2col/verify/src/env/im2col_scoreboard.svh +++ b/examples/components/balls/im2col/verify/src/env/im2col_scoreboard.svh @@ -10,9 +10,6 @@ class im2col_scoreboard extends uvm_scoreboard; bb_blink_mem_model #(1, 1) mem_model; im2col_cmd_item stim_q[$]; - bit [127:0] expected_dst[IM2COL_MAX_WORDS]; - bit seen_addr[IM2COL_MAX_WORDS]; - int unsigned expected_writes; int unsigned cmd_count; int unsigned read_count; int unsigned write_count; @@ -30,33 +27,18 @@ class im2col_scoreboard extends uvm_scoreboard; function void write_stim(bb_blink_cmd_item item); im2col_cmd_item titem; im2col_cmd_item clone; - int i; - if (stim_q.size() != 0) begin - `uvm_fatal("SCB", "single outstanding command supported") - end - if (!$cast(titem, item)) begin - `uvm_fatal("SCB", "stim item is not im2col_cmd_item") - end - if (!$cast(clone, titem.clone())) begin - `uvm_fatal("SCB", "failed to clone stimulus item") - end + if (stim_q.size() != 0) `uvm_fatal("SCB", "single outstanding command supported") + if (!$cast(titem, item)) `uvm_fatal("SCB", "stim item is not im2col_cmd_item") + if (!$cast(clone, titem.clone())) `uvm_fatal("SCB", "failed to clone stimulus item") stim_q.push_back(clone); - if (mem_model == null) begin - `uvm_fatal("SCB", "mem_model handle not set") - end + if (mem_model == null) `uvm_fatal("SCB", "mem_model handle not set") mem_model.clear_mem(); - for (i = 0; i < clone.num_src_words; i++) begin - mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group), i, clone.src_words[i]); - end + foreach (clone.src_data[i]) + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group) + clone.src_group[i], + clone.src_addr[i], clone.src_data[i]); mem_model.arm(); - - for (i = 0; i < clone.num_dst_words; i++) begin - expected_dst[i] = clone.dst_words[i]; - seen_addr[i] = 1'b0; - end - expected_writes = clone.num_dst_words; endfunction function void write_cmd(bb_blink_cmd_item item); @@ -77,43 +59,58 @@ class im2col_scoreboard extends uvm_scoreboard; function void write_read(bb_blink_read_item item); im2col_cmd_item stim; + int unsigned group; stim = current_stim("READ"); + if (read_count >= stim.read_addr.size()) + `uvm_fatal("READ", $sformatf("extra read %0d", read_count)) + group = stim.op1_group + stim.read_group[read_count]; if (item.bank_id !== stim.op1_bank) `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.op1_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== stim.op1_group) + if (item.group_id !== group[BB_BLINK_GROUP_ID_W-1:0]) `uvm_fatal("READ", $sformatf( - "group_id mismatch: got %0d exp %0d", item.group_id, stim.op1_group)) - if (item.addr >= stim.num_src_words[9:0]) + "read %0d group mismatch: got %0d exp %0d", read_count, item.group_id, group)) + if (item.addr !== stim.read_addr[read_count][BB_BLINK_BANK_ADDR_W-1:0]) `uvm_fatal("READ", $sformatf( - "addr out of range: got %0d max %0d", item.addr, stim.num_src_words)) + "read %0d addr mismatch: got %0d exp %0d", + read_count, + item.addr, + stim.read_addr[read_count] + )) read_count++; endfunction function void write_write(bb_blink_write_item item); im2col_cmd_item stim; - int unsigned addr; + int unsigned group; stim = current_stim("WRITE"); + if (write_count >= stim.dst_addr.size()) + `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) + group = stim.wr_group + stim.dst_group[write_count]; if (item.bank_id !== stim.wr_bank) `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== stim.wr_group) + if (item.group_id !== group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("WRITE", $sformatf( + "write %0d group mismatch: got %0d exp %0d", write_count, item.group_id, group)) + if (item.addr !== stim.dst_addr[write_count][BB_BLINK_BANK_ADDR_W-1:0]) `uvm_fatal("WRITE", $sformatf( - "group_id mismatch: got %0d exp %0d", item.group_id, stim.wr_group)) - addr = int'(item.addr); - if (addr >= expected_writes) - `uvm_fatal("WRITE", $sformatf("addr out of range: got %0d max %0d", addr, expected_writes)) - if (seen_addr[addr]) `uvm_fatal("WRITE", $sformatf("duplicate write addr %0d", addr)) + "write %0d addr mismatch: got %0d exp %0d", + write_count, + item.addr, + stim.dst_addr[write_count] + )) if (item.mask !== 16'hFFFF) `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) - if (item.data !== expected_dst[addr]) - `uvm_fatal( - "SCB", $sformatf( - "data mismatch at addr %0d: got 0x%032h exp 0x%032h", addr, item.data, expected_dst[addr] - )) - seen_addr[addr] = 1'b1; + if (item.data !== stim.dst_data[write_count]) + `uvm_fatal("SCB", $sformatf( + "write %0d data mismatch: got 0x%032h exp 0x%032h", + write_count, + item.data, + stim.dst_data[write_count] + )) write_count++; endfunction @@ -134,25 +131,24 @@ class im2col_scoreboard extends uvm_scoreboard; endfunction function bit done(); - return cmd_count == 1 && write_count == expected_writes && resp_count == 1; + return stim_q.size() == 1 && cmd_count == 1 && read_count == stim_q[0].read_addr.size() && + write_count == stim_q[0].dst_addr.size() && resp_count == 1; endfunction function void reset_counters(); stim_q.delete(); - for (int i = 0; i < IM2COL_MAX_WORDS; i++) seen_addr[i] = 1'b0; - cmd_count = 0; - read_count = 0; + cmd_count = 0; + read_count = 0; write_count = 0; - resp_count = 0; - expected_writes = 0; + resp_count = 0; endfunction function void check_phase(uvm_phase phase); verification_pkg::evidence::checks++; super.check_phase(phase); if (!done()) begin - `uvm_fatal("SCB", $sformatf("incomplete: cmds=%0d reads=%0d writes=%0d/%0d resp=%0d", - cmd_count, read_count, write_count, expected_writes, resp_count)) + `uvm_fatal("SCB", $sformatf("incomplete: cmds=%0d reads=%0d writes=%0d resp=%0d", cmd_count, + read_count, write_count, resp_count)) end stim_q.delete(); endfunction diff --git a/examples/components/balls/im2col/verify/src/seq/im2col_sequences.svh b/examples/components/balls/im2col/verify/src/seq/im2col_sequences.svh index fa9fbe655..43768b92b 100644 --- a/examples/components/balls/im2col/verify/src/seq/im2col_sequences.svh +++ b/examples/components/balls/im2col/verify/src/seq/im2col_sequences.svh @@ -2,18 +2,16 @@ class im2col_basic_seq extends uvm_sequence #(bb_blink_cmd_item); `uvm_object_utils(im2col_basic_seq) int unsigned case_index; - int unsigned seed; function new(string name = "im2col_basic_seq"); super.new(name); - seed = IM2COL_SEED; endfunction task body(); im2col_cmd_item item; item = im2col_cmd_item::type_id::create("item"); start_item(item); - item.load_rust_case(seed, case_index); + item.load_rust_case(case_index); finish_item(item); endtask endclass diff --git a/examples/components/balls/im2col/verify/src/tests/im2col_test.svh b/examples/components/balls/im2col/verify/src/tests/im2col_test.svh index e87502ea0..b138a33a2 100644 --- a/examples/components/balls/im2col/verify/src/tests/im2col_test.svh +++ b/examples/components/balls/im2col/verify/src/tests/im2col_test.svh @@ -19,24 +19,11 @@ class im2col_ball_test extends uvm_test; task run_phase(uvm_phase phase); phase.raise_objection(this); - run_directed(); - run_random(); + for (int unsigned index = 0; index < IM2COL_NUM_CASES; index++) run_case(index); phase.drop_objection(this); endtask - task run_directed(); - run_case(0, "DIR_K3_PAD0"); - run_case(1, "DIR_K3_PAD1"); - endtask - - task run_random(); - int i; - for (i = 2; i <= 21; i++) begin - run_case(i, $sformatf("RND_%0d", i)); - end - endtask - - task run_case(int unsigned idx, string label); + task run_case(int unsigned idx); im2col_basic_seq seq; int cycles; @@ -53,15 +40,13 @@ class im2col_ball_test extends uvm_test; cycles++; if (cycles > IM2COL_TIMEOUT_CYCLES) begin `uvm_fatal("TIMEOUT", - $sformatf( - "Im2colBall %s (idx=%0d) timeout: cmds=%0d reads=%0d wr=%0d/%0d resp=%0d", - label, idx, env.scb.cmd_count, env.scb.read_count, env.scb.write_count, - env.scb.expected_writes, env.scb.resp_count)) + $sformatf("Im2colBall case %0d timeout: cmds=%0d reads=%0d writes=%0d resp=%0d", + idx, env.scb.cmd_count, env.scb.read_count, env.scb.write_count, + env.scb.resp_count)) end end - `uvm_info("IM2COL", $sformatf("Im2colBall %s (idx=%0d) passed in %0d cycles", label, idx, - cycles), UVM_LOW) + `uvm_info("IM2COL", $sformatf("Im2colBall case %0d passed in %0d cycles", idx, cycles), UVM_LOW) endtask task apply_reset(); diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_k1_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_k1_test.c index 2c28978c4..95141abab 100644 --- a/examples/components/balls/im2col/workloads/ctests/im2col_k1_test.c +++ b/examples/components/balls/im2col/workloads/ctests/im2col_k1_test.c @@ -1,74 +1,31 @@ #include "buckyball.h" #include -#include #include +#include #include -enum { ITER = 7, K = 1, STRIDE = 1, PAD = 0, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static elem_t in[ITER * ITER] __attribute__((aligned(64))) = { - 1, 2, -3, 4, 0, -6, 7, 8, -9, 0, 11, -12, 13, - 14, 0, 16, 17, -18, 19, 0, -21, 22, 23, -24, 0, 26, - -27, 28, 29, 0, 31, 32, -33, 34, 0, -36, 37, 38, -39, - 0, 41, -42, 43, 44, 0, 46, 47, -48, 49, -}; -static elem_t packed[BANK_LINES * LANES] __attribute__((aligned(64))); -static elem_t out[OUT_ROWS * LANES] __attribute__((aligned(64))); -static elem_t exp[OUT_ROWS * LANES] __attribute__((aligned(64))); - -static void build_expected(void) { - int w = 0; - clear_i8_matrix(exp, OUT_ROWS, LANES); - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int row = orow * STRIDE + krow - PAD; - int col = ocol * STRIDE + kcol - PAD; - exp[bank_row * LANES + ki % LANES] = - (row < 0 || row >= ITER || col < 0 || col >= ITER) - ? 0 - : in[row * ITER + col]; - } - } - w++; - } - } -} +/* 1x1 kernel over a 2x2 NHWC input with 32 channels (two channel groups) and + * two output groups: output group g is K tile g, so the output, read back as + * a whole two-group bank, is the input itself. */ +static int8_t input[4 * 32] __attribute__((aligned(64))); +static int8_t actual[4 * 32] __attribute__((aligned(64))); int main(void) { -#ifdef MULTITHREAD - multithread(MULTITHREAD); -#endif - clear_i8_matrix(out, OUT_ROWS, LANES); - build_expected(); - clear_i8_matrix(packed, BANK_LINES, LANES); - for (int i = 0; i < ITER * ITER; ++i) - packed[i * LANES] = in[i]; + for (int i = 0; i < 4 * 32; ++i) + input[i] = (int8_t)(i * 3 - 100); - bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)packed, 0, BANK_LINES, 1); - bb_im2col(0, 1, ITER, K, STRIDE, PAD, 0, 0, 0, 0, 0, WINDOWS); - bb_mvout((uintptr_t)out, 1, OUT_ROWS, 1); - bb_mem_release(0); - bb_mem_release(1); + bb_mem_alloc(0, 1, 2); + bb_mem_alloc(1, 1, 2); + bb_mvin((uintptr_t)input, 0, 4, 1); + bb_im2col(0, 1, 2, 1, 1, 0, 0, 0, 0, 0, 4, 4); + bb_mvout((uintptr_t)actual, 1, 4, 1); - if (compare_i8_matrices(out, exp, OUT_ROWS, LANES)) { - printf("im2col k1 7x7 PASSED\n"); - return 0; - } - printf("im2col k1 7x7 FAILED\n"); - return 1; + for (int i = 0; i < 4 * 32; ++i) + if (actual[i] != input[i]) { + printf("im2col_k1 FAIL index=%d expected=%d actual=%d\n", i, input[i], + actual[i]); + return 1; + } + printf("im2col_k1 PASS\n"); + return 0; } diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c index 301e1d28c..693707eda 100644 --- a/examples/components/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c +++ b/examples/components/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c @@ -1,73 +1,41 @@ #include "buckyball.h" #include -#include #include +#include #include -/* asym pad: padding=max(lo,hi)=2, start=padding-lo=1 (lo=1,hi=2) */ -enum { ITER = 6, K = 3, STRIDE = 1, PAD = 2, START = 1, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K - START) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static elem_t in[ITER * ITER] __attribute__((aligned(64))) = { - 7, -2, 3, 0, 5, -6, 1, 8, -9, 10, 11, 0, -4, 12, 13, 14, 0, 16, - 17, 0, 19, -20, 21, 22, 0, 24, 25, 26, -27, 28, 29, -30, 0, 32, 33, 34, -}; -static elem_t packed[BANK_LINES * LANES] __attribute__((aligned(64))); -static elem_t out[OUT_ROWS * LANES] __attribute__((aligned(64))); -static elem_t exp[OUT_ROWS * LANES] __attribute__((aligned(64))); - -static void build_expected(void) { - int w = 0; - clear_i8_matrix(exp, OUT_ROWS, LANES); - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int row = START + orow * STRIDE + krow - PAD; - int col = START + ocol * STRIDE + kcol - PAD; - exp[bank_row * LANES + ki % LANES] = - (row < 0 || row >= ITER || col < 0 || col >= ITER) - ? 0 - : in[row * ITER + col]; - } - } - w++; - } - } -} +/* Bottom/right-only padding: 2x2 kernel, padding 1, start_row = start_col = 1 + * over a 3x3 input gives 3x3 windows reading pixel (oy + kr, ox + kc), zero + * past the bottom/right edge. Four output groups hold one tap each; lines + * 9..15 pad the 16-row M tile. */ +static int8_t input[9 * 16] __attribute__((aligned(64))); +static int8_t actual[16 * 16] __attribute__((aligned(64))); int main(void) { -#ifdef MULTITHREAD - multithread(MULTITHREAD); -#endif - clear_i8_matrix(out, OUT_ROWS, LANES); - build_expected(); - clear_i8_matrix(packed, BANK_LINES, LANES); - for (int i = 0; i < ITER * ITER; ++i) - packed[i * LANES] = in[i]; + for (int i = 0; i < 9 * 16; ++i) + input[i] = (int8_t)(i * 9 - 50); bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)packed, 0, BANK_LINES, 1); - bb_im2col(0, 1, ITER, K, STRIDE, PAD, 0, 0, START, START, 0, WINDOWS); - bb_mvout((uintptr_t)out, 1, OUT_ROWS, 1); - bb_mem_release(0); - bb_mem_release(1); + bb_mem_alloc(1, 1, 4); + bb_mvin((uintptr_t)input, 0, 9, 1); + bb_im2col(0, 1, 3, 2, 1, 1, 0, 1, 1, 0, 9, 16); - if (compare_i8_matrices(out, exp, OUT_ROWS, LANES)) { - printf("im2col k3 6x6 asym pad PASSED\n"); - return 0; + for (int tap = 0; tap < 4; ++tap) { + bb_mvout_group((uintptr_t)actual, 1, tap, 16, 1); + for (int w = 0; w < 16; ++w) { + int y = w / 3 + tap / 2, x = w % 3 + tap % 2; + int inside = w < 9 && y < 3 && x < 3; + for (int lane = 0; lane < 16; ++lane) { + int8_t expected = inside ? input[(y * 3 + x) * 16 + lane] : 0; + if (actual[w * 16 + lane] != expected) { + printf("im2col_k3_asym_pad FAIL tap=%d window=%d lane=%d " + "expected=%d actual=%d\n", + tap, w, lane, expected, actual[w * 16 + lane]); + return 1; + } + } + } } - printf("im2col k3 6x6 asym pad FAILED\n"); - return 1; + printf("im2col_k3_asym_pad PASS\n"); + return 0; } diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_k3_pad_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_k3_pad_test.c index 16ed237b7..c314d0692 100644 --- a/examples/components/balls/im2col/workloads/ctests/im2col_k3_pad_test.c +++ b/examples/components/balls/im2col/workloads/ctests/im2col_k3_pad_test.c @@ -1,73 +1,40 @@ #include "buckyball.h" #include -#include #include +#include #include -/* padding path: out-of-bound window samples are zero */ -enum { ITER = 6, K = 3, STRIDE = 1, PAD = 1, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static elem_t in[ITER * ITER] __attribute__((aligned(64))) = { - 7, -2, 3, 0, 5, -6, 1, 8, -9, 10, 11, 0, -4, 12, 13, 14, 0, 16, - 17, 0, 19, -20, 21, 22, 0, 24, 25, 26, -27, 28, 29, -30, 0, 32, 33, 34, -}; -static elem_t packed[BANK_LINES * LANES] __attribute__((aligned(64))); -static elem_t out[OUT_ROWS * LANES] __attribute__((aligned(64))); -static elem_t exp[OUT_ROWS * LANES] __attribute__((aligned(64))); - -static void build_expected(void) { - int w = 0; - clear_i8_matrix(exp, OUT_ROWS, LANES); - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int row = orow * STRIDE + krow - PAD; - int col = ocol * STRIDE + kcol - PAD; - exp[bank_row * LANES + ki % LANES] = - (row < 0 || row >= ITER || col < 0 || col >= ITER) - ? 0 - : in[row * ITER + col]; - } - } - w++; - } - } -} +/* 3x3 kernel with padding 1 over a 2x2 input gives 2x2 windows. One output + * group holds all nine taps in one 4-row M tile: tap t, window w is line + * t * 4 + w, zero when the tap falls into the padding. */ +static int8_t input[4 * 16] __attribute__((aligned(64))); +static int8_t actual[36 * 16] __attribute__((aligned(64))); int main(void) { -#ifdef MULTITHREAD - multithread(MULTITHREAD); -#endif - clear_i8_matrix(out, OUT_ROWS, LANES); - build_expected(); - clear_i8_matrix(packed, BANK_LINES, LANES); - for (int i = 0; i < ITER * ITER; ++i) - packed[i * LANES] = in[i]; + for (int i = 0; i < 4 * 16; ++i) + input[i] = (int8_t)(i * 7 + 3); bb_mem_alloc(0, 1, 1); bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)packed, 0, BANK_LINES, 1); - bb_im2col(0, 1, ITER, K, STRIDE, PAD, 0, 0, 0, 0, 0, WINDOWS); - bb_mvout((uintptr_t)out, 1, OUT_ROWS, 1); - bb_mem_release(0); - bb_mem_release(1); + bb_mvin((uintptr_t)input, 0, 4, 1); + bb_im2col(0, 1, 2, 3, 1, 1, 0, 0, 0, 0, 4, 4); + bb_mvout((uintptr_t)actual, 1, 36, 1); - if (compare_i8_matrices(out, exp, OUT_ROWS, LANES)) { - printf("im2col k3 6x6 p1 PASSED\n"); - return 0; - } - printf("im2col k3 6x6 p1 FAILED\n"); - return 1; + for (int tap = 0; tap < 9; ++tap) + for (int w = 0; w < 4; ++w) { + int y = w / 2 + tap / 3 - 1, x = w % 2 + tap % 3 - 1; + int inside = y >= 0 && y < 2 && x >= 0 && x < 2; + for (int lane = 0; lane < 16; ++lane) { + int8_t expected = inside ? input[(y * 2 + x) * 16 + lane] : 0; + int8_t got = actual[(tap * 4 + w) * 16 + lane]; + if (got != expected) { + printf("im2col_k3_pad FAIL tap=%d window=%d lane=%d expected=%d " + "actual=%d\n", + tap, w, lane, expected, got); + return 1; + } + } + } + printf("im2col_k3_pad PASS\n"); + return 0; } diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_k3_partial_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_k3_partial_test.c index 55dec3a55..4c27cb0cc 100644 --- a/examples/components/balls/im2col/workloads/ctests/im2col_k3_partial_test.c +++ b/examples/components/balls/im2col/workloads/ctests/im2col_k3_partial_test.c @@ -1,73 +1,36 @@ #include "buckyball.h" #include -#include #include +#include #include -/* partial M core: 5x5 k3 -> 9 windows, unused m rows must be 0 */ -enum { ITER = 5, K = 3, STRIDE = 1, PAD = 0, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static elem_t in[ITER * ITER] __attribute__((aligned(64))) = { - -1, 2, 3, 0, 5, 6, 7, 0, 9, 10, 11, 0, -13, - 14, 15, 0, 17, 18, 19, 0, 21, 22, 23, 0, -25, -}; -static elem_t packed[BANK_LINES * LANES] __attribute__((aligned(64))); -static elem_t out[OUT_ROWS * LANES] __attribute__((aligned(64))); -static elem_t exp[OUT_ROWS * LANES] __attribute__((aligned(64))); - -static void build_expected(void) { - int w = 0; - clear_i8_matrix(exp, OUT_ROWS, LANES); - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int row = orow * STRIDE + krow - PAD; - int col = ocol * STRIDE + kcol - PAD; - exp[bank_row * LANES + ki % LANES] = - (row < 0 || row >= ITER || col < 0 || col >= ITER) - ? 0 - : in[row * ITER + col]; - } - } - w++; - } - } -} +/* Window range 5..7 of a 1x1 im2col over a 4x4 input: lines 0..2 are pixels + * 5, 6, 7 and lines 3..7 zero-fill the partial 8-row M tile. */ +static int8_t input[16 * 16] __attribute__((aligned(64))); +static int8_t actual[8 * 16] __attribute__((aligned(64))); int main(void) { -#ifdef MULTITHREAD - multithread(MULTITHREAD); -#endif - clear_i8_matrix(out, OUT_ROWS, LANES); - build_expected(); - clear_i8_matrix(packed, BANK_LINES, LANES); - for (int i = 0; i < ITER * ITER; ++i) - packed[i * LANES] = in[i]; + for (int i = 0; i < 16 * 16; ++i) + input[i] = (int8_t)(i * 11 + 7); + for (int i = 0; i < 8 * 16; ++i) + actual[i] = 99; bb_mem_alloc(0, 1, 1); bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)packed, 0, BANK_LINES, 1); - bb_im2col(0, 1, ITER, K, STRIDE, PAD, 0, 0, 0, 0, 0, WINDOWS); - bb_mvout((uintptr_t)out, 1, OUT_ROWS, 1); - bb_mem_release(0); - bb_mem_release(1); + bb_mvin((uintptr_t)input, 0, 16, 1); + bb_im2col(0, 1, 4, 1, 1, 0, 0, 0, 0, 5, 3, 8); + bb_mvout((uintptr_t)actual, 1, 8, 1); - if (compare_i8_matrices(out, exp, OUT_ROWS, LANES)) { - printf("im2col k3 5x5 partial-M PASSED\n"); - return 0; - } - printf("im2col k3 5x5 partial-M FAILED\n"); - return 1; + for (int line = 0; line < 8; ++line) + for (int lane = 0; lane < 16; ++lane) { + int8_t expected = line < 3 ? input[(5 + line) * 16 + lane] : 0; + if (actual[line * 16 + lane] != expected) { + printf("im2col_k3_partial FAIL line=%d lane=%d expected=%d " + "actual=%d\n", + line, lane, expected, actual[line * 16 + lane]); + return 1; + } + } + printf("im2col_k3_partial PASS\n"); + return 0; } diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_k3_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_k3_test.c index 387f355a6..a98641cf0 100644 --- a/examples/components/balls/im2col/workloads/ctests/im2col_k3_test.c +++ b/examples/components/balls/im2col/workloads/ctests/im2col_k3_test.c @@ -1,74 +1,39 @@ #include "buckyball.h" #include -#include #include +#include #include -/* exact M core: 6x6 k3 -> 16 windows */ -enum { ITER = 6, K = 3, STRIDE = 1, PAD = 0, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static elem_t in[ITER * ITER] __attribute__((aligned(64))) = { - -1, 2, 3, 4, -5, 6, 0, 8, 9, -10, 11, 12, - 13, 0, -15, 16, 17, 18, 19, -20, 0, 22, 23, -24, - -25, 26, 27, 0, -29, 30, 31, 32, 33, -34, 0, 36, -}; -static elem_t packed[BANK_LINES * LANES] __attribute__((aligned(64))); -static elem_t out[OUT_ROWS * LANES] __attribute__((aligned(64))); -static elem_t exp[OUT_ROWS * LANES] __attribute__((aligned(64))); - -static void build_expected(void) { - int w = 0; - clear_i8_matrix(exp, OUT_ROWS, LANES); - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int row = orow * STRIDE + krow - PAD; - int col = ocol * STRIDE + kcol - PAD; - exp[bank_row * LANES + ki % LANES] = - (row < 0 || row >= ITER || col < 0 || col >= ITER) - ? 0 - : in[row * ITER + col]; - } - } - w++; - } - } -} +/* 3x3 kernel over a 4x4 input (16 channels) gives 2x2 windows. Nine output + * groups hold one tap each: group kr * 3 + kc, line w is the input row of + * pixel (oy + kr, ox + kc); lines 4..7 pad the 8-row M tile with zeros. */ +static int8_t input[16 * 16] __attribute__((aligned(64))); +static int8_t actual[8 * 16] __attribute__((aligned(64))); int main(void) { -#ifdef MULTITHREAD - multithread(MULTITHREAD); -#endif - clear_i8_matrix(out, OUT_ROWS, LANES); - build_expected(); - clear_i8_matrix(packed, BANK_LINES, LANES); - for (int i = 0; i < ITER * ITER; ++i) - packed[i * LANES] = in[i]; + for (int i = 0; i < 16 * 16; ++i) + input[i] = (int8_t)(i * 5 + 1); bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)packed, 0, BANK_LINES, 1); - bb_im2col(0, 1, ITER, K, STRIDE, PAD, 0, 0, 0, 0, 0, WINDOWS); - bb_mvout((uintptr_t)out, 1, OUT_ROWS, 1); - bb_mem_release(0); - bb_mem_release(1); + bb_mem_alloc(1, 1, 9); + bb_mvin((uintptr_t)input, 0, 16, 1); + bb_im2col(0, 1, 4, 3, 1, 0, 0, 0, 0, 0, 4, 8); - if (compare_i8_matrices(out, exp, OUT_ROWS, LANES)) { - printf("im2col k3 6x6 exact-M PASSED\n"); - return 0; + for (int tap = 0; tap < 9; ++tap) { + bb_mvout_group((uintptr_t)actual, 1, tap, 8, 1); + for (int w = 0; w < 8; ++w) { + int pixel = (w / 2 + tap / 3) * 4 + w % 2 + tap % 3; + for (int lane = 0; lane < 16; ++lane) { + int8_t expected = w < 4 ? input[pixel * 16 + lane] : 0; + if (actual[w * 16 + lane] != expected) { + printf("im2col_k3 FAIL tap=%d window=%d lane=%d expected=%d " + "actual=%d\n", + tap, w, lane, expected, actual[w * 16 + lane]); + return 1; + } + } + } } - printf("im2col k3 6x6 exact-M FAILED\n"); - return 1; + printf("im2col_k3 PASS\n"); + return 0; } diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_k5_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_k5_test.c index 7a14defa2..6f648f429 100644 --- a/examples/components/balls/im2col/workloads/ctests/im2col_k5_test.c +++ b/examples/components/balls/im2col/workloads/ctests/im2col_k5_test.c @@ -1,73 +1,33 @@ #include "buckyball.h" #include -#include #include +#include #include -enum { ITER = 7, K = 5, STRIDE = 1, PAD = 0, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static elem_t in[ITER * ITER] __attribute__((aligned(64))) = { - 2, -4, 6, 0, 8, -10, 12, -1, 3, 5, 7, 0, 11, -13, 14, 0, 16, - -18, 20, 22, 24, -15, 17, 19, 0, 21, -23, 25, 26, 28, 0, 30, -32, 34, - 36, -27, 29, 31, 33, 0, 35, -37, 38, 0, 40, -42, 44, 46, 48, -}; -static elem_t packed[BANK_LINES * LANES] __attribute__((aligned(64))); -static elem_t out[OUT_ROWS * LANES] __attribute__((aligned(64))); -static elem_t exp[OUT_ROWS * LANES] __attribute__((aligned(64))); - -static void build_expected(void) { - int w = 0; - clear_i8_matrix(exp, OUT_ROWS, LANES); - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int row = orow * STRIDE + krow - PAD; - int col = ocol * STRIDE + kcol - PAD; - exp[bank_row * LANES + ki % LANES] = - (row < 0 || row >= ITER || col < 0 || col >= ITER) - ? 0 - : in[row * ITER + col]; - } - } - w++; - } - } -} +/* 5x5 kernel over a 5x5 input: one window, 25 taps. With tile_rows 1 and five + * output groups, group kr line kc is the input row of pixel (kr, kc), so each + * output group is one input image row. */ +static int8_t input[25 * 16] __attribute__((aligned(64))); +static int8_t actual[5 * 16] __attribute__((aligned(64))); int main(void) { -#ifdef MULTITHREAD - multithread(MULTITHREAD); -#endif - clear_i8_matrix(out, OUT_ROWS, LANES); - build_expected(); - clear_i8_matrix(packed, BANK_LINES, LANES); - for (int i = 0; i < ITER * ITER; ++i) - packed[i * LANES] = in[i]; + for (int i = 0; i < 25 * 16; ++i) + input[i] = (int8_t)(i * 13 - 7); bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)packed, 0, BANK_LINES, 1); - bb_im2col(0, 1, ITER, K, STRIDE, PAD, 0, 0, 0, 0, 0, WINDOWS); - bb_mvout((uintptr_t)out, 1, OUT_ROWS, 1); - bb_mem_release(0); - bb_mem_release(1); + bb_mem_alloc(1, 1, 5); + bb_mvin((uintptr_t)input, 0, 25, 1); + bb_im2col(0, 1, 5, 5, 1, 0, 0, 0, 0, 0, 1, 1); - if (compare_i8_matrices(out, exp, OUT_ROWS, LANES)) { - printf("im2col k5 7x7 PASSED\n"); - return 0; + for (int kr = 0; kr < 5; ++kr) { + bb_mvout_group((uintptr_t)actual, 1, kr, 5, 1); + for (int i = 0; i < 5 * 16; ++i) + if (actual[i] != input[kr * 5 * 16 + i]) { + printf("im2col_k5 FAIL row=%d index=%d expected=%d actual=%d\n", kr, i, + input[kr * 5 * 16 + i], actual[i]); + return 1; + } } - printf("im2col k5 7x7 FAILED\n"); - return 1; + printf("im2col_k5 PASS\n"); + return 0; } diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_k7_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_k7_test.c index 0387975a5..d16558b8b 100644 --- a/examples/components/balls/im2col/workloads/ctests/im2col_k7_test.c +++ b/examples/components/balls/im2col/workloads/ctests/im2col_k7_test.c @@ -1,73 +1,41 @@ #include "buckyball.h" #include -#include #include +#include #include -enum { ITER = 7, K = 7, STRIDE = 1, PAD = 0, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static elem_t in[ITER * ITER] __attribute__((aligned(64))) = { - -5, 1, 2, 3, 0, 4, 5, 6, -7, 8, 9, 10, 0, 11, 12, 13, -14, - 15, 16, 17, 0, 0, 18, 19, -20, 21, 22, 23, 24, 0, 25, 26, -27, 28, - 29, 30, 31, 0, 32, 33, -34, 35, 36, 37, 38, 0, 39, 40, -41, -}; -static elem_t packed[BANK_LINES * LANES] __attribute__((aligned(64))); -static elem_t out[OUT_ROWS * LANES] __attribute__((aligned(64))); -static elem_t exp[OUT_ROWS * LANES] __attribute__((aligned(64))); - -static void build_expected(void) { - int w = 0; - clear_i8_matrix(exp, OUT_ROWS, LANES); - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int row = orow * STRIDE + krow - PAD; - int col = ocol * STRIDE + kcol - PAD; - exp[bank_row * LANES + ki % LANES] = - (row < 0 || row >= ITER || col < 0 || col >= ITER) - ? 0 - : in[row * ITER + col]; - } - } - w++; - } - } -} +/* 7x7 stride-2 kernel over a 9x9 input gives 2x2 windows. Seven output groups + * hold one kernel row each (seven taps): group kr, tap kc, window w is line + * kc * 4 + w, the input row of pixel (oy * 2 + kr, ox * 2 + kc). */ +static int8_t input[81 * 16] __attribute__((aligned(64))); +static int8_t actual[28 * 16] __attribute__((aligned(64))); int main(void) { -#ifdef MULTITHREAD - multithread(MULTITHREAD); -#endif - clear_i8_matrix(out, OUT_ROWS, LANES); - build_expected(); - clear_i8_matrix(packed, BANK_LINES, LANES); - for (int i = 0; i < ITER * ITER; ++i) - packed[i * LANES] = in[i]; + for (int i = 0; i < 81 * 16; ++i) + input[i] = (int8_t)(i * 3 + 11); bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)packed, 0, BANK_LINES, 1); - bb_im2col(0, 1, ITER, K, STRIDE, PAD, 0, 0, 0, 0, 0, WINDOWS); - bb_mvout((uintptr_t)out, 1, OUT_ROWS, 1); - bb_mem_release(0); - bb_mem_release(1); + bb_mem_alloc(1, 1, 7); + bb_mvin((uintptr_t)input, 0, 81, 1); + bb_im2col(0, 1, 9, 7, 2, 0, 0, 0, 0, 0, 4, 4); - if (compare_i8_matrices(out, exp, OUT_ROWS, LANES)) { - printf("im2col k7 7x7 PASSED\n"); - return 0; + for (int kr = 0; kr < 7; ++kr) { + bb_mvout_group((uintptr_t)actual, 1, kr, 28, 1); + for (int kc = 0; kc < 7; ++kc) + for (int w = 0; w < 4; ++w) { + int pixel = (w / 2 * 2 + kr) * 9 + w % 2 * 2 + kc; + for (int lane = 0; lane < 16; ++lane) { + int8_t expected = input[pixel * 16 + lane]; + int8_t got = actual[(kc * 4 + w) * 16 + lane]; + if (got != expected) { + printf("im2col_k7 FAIL kr=%d kc=%d window=%d lane=%d expected=%d " + "actual=%d\n", + kr, kc, w, lane, expected, got); + return 1; + } + } + } } - printf("im2col k7 7x7 FAILED\n"); - return 1; + printf("im2col_k7 PASS\n"); + return 0; } diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_window_test.c b/examples/components/balls/im2col/workloads/ctests/im2col_window_test.c index 559aeeccc..ea87664e5 100644 --- a/examples/components/balls/im2col/workloads/ctests/im2col_window_test.c +++ b/examples/components/balls/im2col/workloads/ctests/im2col_window_test.c @@ -1,40 +1,39 @@ #include "buckyball.h" #include -#include #include #include #include -enum { INPUT_BASE = 5, LANE = 7 }; - -static int8_t input[64 * 16] __attribute__((aligned(64))); -static int8_t actual[16 * 16] __attribute__((aligned(64))); +/* 2x2 kernel over a 3x3 NHWC input with 32 channels (two channel groups) at + * input_base 3, windows 1..2 of the 2x2 output. K tile t = tap * 2 + g; each + * of the eight output groups holds one tile: line w is channels 16g..16g+15 + * of pixel (oy + kr, ox + kc) for window 1 + w. */ +static int8_t input[(3 + 9) * 32] __attribute__((aligned(64))); +static int8_t actual[2 * 16] __attribute__((aligned(64))); int main(void) { - for (int i = 0; i < 64 * 16; ++i) - input[i] = -99; - for (int i = 0; i < 36; ++i) - input[(INPUT_BASE + i) * 16 + LANE] = i + 1; + for (int i = 0; i < 3 * 32; ++i) + input[i] = 0x5a; + for (int i = 3 * 32; i < 12 * 32; ++i) + input[i] = (int8_t)(i * 7 - 9); - bb_mem_alloc(0, 1, 1); - bb_mem_alloc(1, 1, 1); - bb_mvin((uintptr_t)input, 0, 64, 1); - bb_im2col(0, 1, 6, 3, 1, 0, INPUT_BASE, LANE, 0, 0, 4, 8); - bb_mvout((uintptr_t)actual, 1, 16, 1); + bb_mem_alloc(0, 1, 2); + bb_mem_alloc(1, 1, 8); + bb_mvin((uintptr_t)input, 0, 12, 1); + bb_im2col(0, 1, 3, 2, 1, 0, 3, 0, 0, 1, 2, 2); - for (int local = 0; local < 8; ++local) { - int global = local + 4; - int output_row = global / 4; - int output_col = global % 4; - for (int kr = 0; kr < 3; ++kr) { - for (int kc = 0; kc < 3; ++kc) { - int expected = - input[(INPUT_BASE + (output_row + kr) * 6 + output_col + kc) * 16 + - LANE]; - int index = local * 16 + kr * 3 + kc; - if (actual[index] != expected) { - printf("im2col_window FAIL local=%d k=%d expected=%d actual=%d\n", - local, kr * 3 + kc, expected, actual[index]); + for (int tile = 0; tile < 8; ++tile) { + int tap = tile / 2, group = tile % 2; + bb_mvout_group((uintptr_t)actual, 1, tile, 2, 1); + for (int w = 0; w < 2; ++w) { + int window = 1 + w; + int pixel = (window / 2 + tap / 2) * 3 + window % 2 + tap % 2; + for (int lane = 0; lane < 16; ++lane) { + int8_t expected = input[(3 + pixel) * 32 + group * 16 + lane]; + if (actual[w * 16 + lane] != expected) { + printf("im2col_window FAIL tile=%d window=%d lane=%d expected=%d " + "actual=%d\n", + tile, w, lane, expected, actual[w * 16 + lane]); return 1; } } diff --git a/examples/components/balls/int2fp/emu/src/int2fp.rs b/examples/components/balls/int2fp/emu/src/int2fp.rs index 2dc602703..f550c5012 100644 --- a/examples/components/balls/int2fp/emu/src/int2fp.rs +++ b/examples/components/balls/int2fp/emu/src/int2fp.rs @@ -1,6 +1,6 @@ -use bebop_bemu::thread::banks::bank_lines; use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::instruction::ExecContext; +use bebop_bemu::thread::banks::bank_lines; pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let input_bank = rs1_b0(xs1); @@ -33,29 +33,29 @@ pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { // Group g of every operand is channel block g; each block uses the same rows. for group in 0..groups { - let input = pbank_group(ctx, input_bank, group); - let scale = pbank_group(ctx, scale_bank, group); - let output = pbank_group(ctx, output_bank, group); - for row in 0..iter { - for lane in 0..4 { - let offset = row * 16 + lane * 4; - let scale_offset = (row % 4) * 16 + lane * 4; - let value = - i32::from_le_bytes(ctx.banks[input][offset..offset + 4].try_into().unwrap()); - let factor = f32::from_bits(u32::from_le_bytes( - ctx.banks[scale][scale_offset..scale_offset + 4] - .try_into() - .unwrap(), - )); - if !factor.is_finite() || factor <= 0.0 { - panic!("int32_to_fp32: scales must be finite and positive"); + let input = pbank_group(ctx, input_bank, group); + let scale = pbank_group(ctx, scale_bank, group); + let output = pbank_group(ctx, output_bank, group); + for row in 0..iter { + for lane in 0..4 { + let offset = row * 16 + lane * 4; + let scale_offset = (row % 4) * 16 + lane * 4; + let value = + i32::from_le_bytes(ctx.banks[input][offset..offset + 4].try_into().unwrap()); + let factor = f32::from_bits(u32::from_le_bytes( + ctx.banks[scale][scale_offset..scale_offset + 4] + .try_into() + .unwrap(), + )); + if !factor.is_finite() || factor <= 0.0 { + panic!("int32_to_fp32: scales must be finite and positive"); + } + let value = if relu { value.max(0) } else { value }; + let result = (value as f32) * factor; + ctx.banks[output][offset..offset + 4].copy_from_slice(&result.to_le_bytes()); } - let value = if relu { value.max(0) } else { value }; - let result = (value as f32) * factor; - ctx.banks[output][offset..offset + 4].copy_from_slice(&result.to_le_bytes()); } } - } 0 } diff --git a/examples/components/balls/int2fp/verify/casegen/src/casegen.rs b/examples/components/balls/int2fp/verify/casegen/src/casegen.rs index 998f85029..6f047909c 100644 --- a/examples/components/balls/int2fp/verify/casegen/src/casegen.rs +++ b/examples/components/balls/int2fp/verify/casegen/src/casegen.rs @@ -4,10 +4,10 @@ pub const BANK_NUM: u32 = 24; pub const BANK_ENTRIES: u32 = 64; pub const LANES: usize = 4; pub const SCALE_ROWS: usize = 4; -pub const MAX_SRC: usize = 16; -pub const MAX_SCALE: usize = 4; -pub const MAX_DST: usize = 16; -pub const NUM_CASES: u32 = 6; +pub const MAX_SRC: usize = 48; +pub const MAX_SCALE: usize = 12; +pub const MAX_DST: usize = 48; +pub const NUM_CASES: u32 = 8; #[repr(C)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -70,12 +70,34 @@ pub fn gen_case(index: u32) -> Int2FpCase { panic!("int2fp: unsupported directed case {index}"); } match index { - 0 => directed(2, 4, false, &MLIR_INPUT, &[0.125; 16], [0, 0, 0]), - 1 => directed(3, 4, true, &RELU_INPUT, &[1.0; 16], [1, 2, 3]), - 2 => directed(4, 8, false, &ITER8, &SCALE16, [2, 0, 1]), - 3 => directed(5, 8, true, &ITER8, &SCALE16, [3, 1, 0]), - 4 => directed(6, 16, false, &ITER16, &SCALE16, [0, 3, 2]), - 5 => directed(7, 16, true, &ITER16, &SCALE16, [1, 1, 1]), + 0 => directed(2, 4, 1, false, &MLIR_INPUT, &[0.125; 16], [0, 0, 0]), + 1 => directed(3, 4, 1, true, &RELU_INPUT, &[1.0; 16], [1, 2, 3]), + 2 => directed(4, 8, 1, false, &ITER8, &SCALE16, [2, 0, 1]), + 3 => directed(5, 8, 1, true, &ITER8, &SCALE16, [3, 1, 0]), + 4 => directed(6, 16, 1, false, &ITER16, &SCALE16, [0, 3, 2]), + 5 => directed(7, 16, 1, true, &ITER16, &SCALE16, [1, 1, 1]), + // Two channel groups: group 0 is MLIR_INPUT with 0.125 scales, group 1 + // is RELU_INPUT with SCALE16. + 6 => directed( + 0, + 4, + 2, + false, + &[MLIR_INPUT, RELU_INPUT].concat(), + &[[0.125; 16], SCALE16].concat(), + [0, 0, 0], + ), + // Three channel groups with ReLU and nonzero base groups: group g holds + // ITER8, then the two halves of ITER16; scales are SCALE16, 1.0, 0.5. + 7 => directed( + 1, + 8, + 3, + true, + &[&ITER8[..], &ITER16[..32], &ITER16[32..]].concat(), + &[SCALE16, [1.0; 16], [0.5; 16]].concat(), + [1, 2, 0], + ), _ => panic!("int2fp: unsupported directed case {index}"), } } @@ -87,9 +109,9 @@ const ITER8: [i32; 32] = [ -9, 12, -12, 6, -6, 11, -11, ]; const ITER16: [i32; 64] = [ - 1, 2, 3, -1, -2, 0, 4, 5, 10, -10, 7, 100, -100, 8, 16, -8, 8, -4, 16, 0, 1, -2, 3, -8, 12, -12, - 6, -6, 9, -9, 11, -11, 20, -20, 24, -24, 15, -15, 18, -18, 30, -30, 40, -40, 25, -25, 28, -28, - 50, -50, 60, -60, 7, -7, 13, -13, 14, -14, 17, -17, 19, -19, 21, -21, + 1, 2, 3, -1, -2, 0, 4, 5, 10, -10, 7, 100, -100, 8, 16, -8, 8, -4, 16, 0, 1, -2, 3, -8, 12, + -12, 6, -6, 9, -9, 11, -11, 20, -20, 24, -24, 15, -15, 18, -18, 30, -30, 40, -40, 25, -25, 28, + -28, 50, -50, 60, -60, 7, -7, 13, -13, 14, -14, 17, -17, 19, -19, 21, -21, ]; const SCALE16: [f32; 16] = [ 0.125, 0.25, 0.5, 1.0, 0.25, 0.5, 1.0, 2.0, 0.5, 1.0, 2.0, 0.125, 1.0, 0.125, 0.25, 0.5, @@ -97,7 +119,10 @@ const SCALE16: [f32; 16] = [ fn pack_u32s(vals: &[u32]) -> Vec { if vals.len() % LANES != 0 { - panic!("int2fp: value count {} is not a multiple of four", vals.len()); + panic!( + "int2fp: value count {} is not a multiple of four", + vals.len() + ); } vals.chunks(LANES) .map(|chunk| { @@ -147,8 +172,8 @@ pub(crate) fn validate( if op1_bank >= BANK_NUM || op2_bank >= BANK_NUM || wr_bank >= BANK_NUM { panic!("INT32_TO_FP32 bank id is invalid"); } - if op1_col != 1 || op2_col != 1 || wr_col != 1 { - panic!("INT32_TO_FP32 operands must each occupy one bank"); + if op1_col == 0 || op1_col != op2_col || op1_col != wr_col { + panic!("INT32_TO_FP32 operands must have the same channel groups"); } if op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank { panic!("INT32_TO_FP32 banks must be distinct"); @@ -156,8 +181,8 @@ pub(crate) fn validate( if rs2 >> 1 != 0 { panic!("INT32_TO_FP32 reserves rs2[63:1]"); } - if scales.len() != SCALE_ROWS { - panic!("INT32_TO_FP32 must emit four scale rows"); + if scales.len() != SCALE_ROWS * op2_col as usize { + panic!("INT32_TO_FP32 must emit four scale rows per group"); } for word in scales { for lane in 0..LANES { @@ -169,11 +194,12 @@ pub(crate) fn validate( } } -fn dst_words(src: &[u128], scales: &[u128], relu: bool) -> Vec { +// Words are group-major: src/dst word i is group i / iter, row i % iter. +fn dst_words(src: &[u128], scales: &[u128], iter: usize, relu: bool) -> Vec { src.iter() .enumerate() - .map(|(row, word)| { - let scale = scales[row % SCALE_ROWS]; + .map(|(i, word)| { + let scale = scales[i / iter * SCALE_ROWS + i % iter % SCALE_ROWS]; let mut packed = 0u128; for lane in 0..LANES { let raw = (*word >> (lane * 32)) as u32 as i32; @@ -190,37 +216,42 @@ fn dst_words(src: &[u128], scales: &[u128], relu: bool) -> Vec { fn directed( rob_id: u32, iter: u32, + cols: u32, relu: bool, values: &[i32], scales: &[f32], groups: [u32; 3], ) -> Int2FpCase { - if values.len() != iter as usize * LANES { + if values.len() != (cols * iter) as usize * LANES { panic!( "int2fp: {} values do not match iter {iter} * {LANES}", values.len() ); } - if scales.len() != SCALE_ROWS * LANES { - panic!("int2fp: {} scales, expected {}", scales.len(), SCALE_ROWS * LANES); + if scales.len() != cols as usize * SCALE_ROWS * LANES { + panic!( + "int2fp: {} scales, expected {}", + scales.len(), + SCALE_ROWS * LANES + ); } let src = pack_u32s(&values.iter().map(|v| *v as u32).collect::>()); let scale = pack_u32s(&scales.iter().map(|v| v.to_bits()).collect::>()); let op1_bank = 0; let op2_bank = 1; let wr_bank = 2; - let op1_col = 1; - let op2_col = 1; - let wr_col = 1; + let op1_col = cols; + let op2_col = cols; + let wr_col = cols; let rs2 = u64::from(relu); validate( op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs2, &scale, ); - if src.len() != iter as usize { - panic!("int2fp: src rows {} != iter {iter}", src.len()); + if src.len() != (cols * iter) as usize { + panic!("int2fp: src rows {} != iter {iter} * groups", src.len()); } - let dst = dst_words(&src, &scale, relu); - if dst.len() != iter as usize { + let dst = dst_words(&src, &scale, iter as usize, relu); + if dst.len() != (cols * iter) as usize { panic!("int2fp: dst rows {} != iter {iter}", dst.len()); } let (rs2_lo, rs2_hi) = split_rs(rs2); diff --git a/examples/components/balls/int2fp/verify/src/common/int2fp_defs.svh b/examples/components/balls/int2fp/verify/src/common/int2fp_defs.svh index 72a9778ec..812dff051 100644 --- a/examples/components/balls/int2fp/verify/src/common/int2fp_defs.svh +++ b/examples/components/balls/int2fp/verify/src/common/int2fp_defs.svh @@ -32,8 +32,8 @@ import "DPI-C" function longint unsigned int2fp_case_dst_word_hi(input int unsig `endif localparam int INT32_TO_FP32_THREAD_FUNCT7 = `INT32_TO_FP32_FUNCT7; -localparam int INT2FP_MAX_SRC_WORDS = 16; -localparam int INT2FP_MAX_SCALE_WORDS = 4; -localparam int INT2FP_MAX_DST_WORDS = 16; -localparam int INT2FP_NUM_CASES = 6; +localparam int INT2FP_MAX_SRC_WORDS = 48; +localparam int INT2FP_MAX_SCALE_WORDS = 12; +localparam int INT2FP_MAX_DST_WORDS = 48; +localparam int INT2FP_NUM_CASES = 8; localparam int INT2FP_TIMEOUT_CYCLES = 20000; diff --git a/examples/components/balls/int2fp/verify/src/common/int2fp_items.svh b/examples/components/balls/int2fp/verify/src/common/int2fp_items.svh index b33c411b8..8d0c96076 100644 --- a/examples/components/balls/int2fp/verify/src/common/int2fp_items.svh +++ b/examples/components/balls/int2fp/verify/src/common/int2fp_items.svh @@ -51,18 +51,19 @@ class int2fp_cmd_item extends bb_blink_cmd_item; `uvm_fatal("CASE", "INT32_TO_FP32 iter must be a positive multiple of four") if (iter != 4 && iter != 8 && iter != 16) `uvm_fatal("CASE", $sformatf("INT32_TO_FP32 iter %0d is not in {4,8,16}", iter)) - if (op1_col != 1 || op2_col != 1 || wr_col != 1) - `uvm_fatal("CASE", "INT32_TO_FP32 operands must each occupy one bank") + if (op1_col == 0 || op1_col != op2_col || op1_col != wr_col) + `uvm_fatal("CASE", "INT32_TO_FP32 operands must have the same channel groups") if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) `uvm_fatal("CASE", "INT32_TO_FP32 banks must be distinct") if (rs2[63:1] != 0) `uvm_fatal("CASE", "INT32_TO_FP32 reserves rs2[63:1]") if (rs2[0] != relu[0]) `uvm_fatal("CASE", "INT32_TO_FP32 rs2[0] must match relu") - if (num_src_words != iter) - `uvm_fatal("CASE", $sformatf("num_src_words %0d != iter %0d", num_src_words, iter)) - if (num_dst_words != iter) - `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter %0d", num_dst_words, iter)) - if (num_scale_words != 4) - `uvm_fatal("CASE", $sformatf("num_scale_words %0d, expected 4", num_scale_words)) + // Words are group-major: word i is group i / iter, row i % iter. + if (num_src_words != iter * op1_col) + `uvm_fatal("CASE", $sformatf("num_src_words %0d != iter * groups", num_src_words)) + if (num_dst_words != iter * op1_col) + `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter * groups", num_dst_words)) + if (num_scale_words != 4 * op1_col) + `uvm_fatal("CASE", $sformatf("num_scale_words %0d, expected 4 per group", num_scale_words)) if (num_src_words == 0 || num_src_words > INT2FP_MAX_SRC_WORDS) `uvm_fatal("CASE", $sformatf("num_src_words out of range: %0d", num_src_words)) if (num_dst_words == 0 || num_dst_words > INT2FP_MAX_DST_WORDS) diff --git a/examples/components/balls/int2fp/verify/src/cov/int2fp_cov.svh b/examples/components/balls/int2fp/verify/src/cov/int2fp_cov.svh index 2c0867867..c62f847c8 100644 --- a/examples/components/balls/int2fp/verify/src/cov/int2fp_cov.svh +++ b/examples/components/balls/int2fp/verify/src/cov/int2fp_cov.svh @@ -5,10 +5,12 @@ class int2fp_cov extends uvm_component; bit cur_relu; int unsigned cur_iter; + int unsigned cur_groups; covergroup cmd_cg; coverpoint cur_relu {bins off = {0}; bins on = {1};} coverpoint cur_iter {bins i4 = {4}; bins i8 = {8}; bins i16 = {16};} + coverpoint cur_groups {bins g1 = {1}; bins g2 = {2}; bins g3 = {3};} endgroup int cmd_count; @@ -22,8 +24,9 @@ class int2fp_cov extends uvm_component; function void write_cmd(bb_blink_cmd_item item); if (item.funct != INT32_TO_FP32_THREAD_FUNCT7[6:0]) `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) - cur_relu = item.rs2[0]; - cur_iter = item.iter; + cur_relu = item.rs2[0]; + cur_iter = item.iter; + cur_groups = item.op1_col; cmd_count++; cmd_cg.sample(); endfunction diff --git a/examples/components/balls/int2fp/verify/src/env/int2fp_scoreboard.svh b/examples/components/balls/int2fp/verify/src/env/int2fp_scoreboard.svh index e017d383e..c7b35037b 100644 --- a/examples/components/balls/int2fp/verify/src/env/int2fp_scoreboard.svh +++ b/examples/components/balls/int2fp/verify/src/env/int2fp_scoreboard.svh @@ -47,16 +47,18 @@ class int2fp_scoreboard extends uvm_scoreboard; end mem_model.clear_mem(); for (i = 0; i < clone.num_src_words; i++) begin - mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group), i, clone.src_words[i]); + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group + i / clone.iter), + i % clone.iter, clone.src_words[i]); end for (i = 0; i < clone.num_scale_words; i++) begin - mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group), i, clone.scale_words[i]); + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group + i / 4), i % 4, + clone.scale_words[i]); end mem_model.arm(); expected_writes = clone.num_dst_words; - expected_reads[0] = clone.iter; - expected_reads[1] = clone.iter; + expected_reads[0] = clone.iter * clone.op1_col; + expected_reads[1] = clone.iter * clone.op1_col; endfunction function void write_cmd(bb_blink_cmd_item item); @@ -89,12 +91,12 @@ class int2fp_scoreboard extends uvm_scoreboard; if (item.port == 0) begin expect_bank = stim.op1_bank; - expect_group = stim.op1_group; - expect_addr = read_count[0]; + expect_group = stim.op1_group + read_count[0] / stim.iter; + expect_addr = read_count[0] % stim.iter; end else begin expect_bank = stim.op2_bank; - expect_group = stim.op2_group; - expect_addr = read_count[1] & 32'h3; + expect_group = stim.op2_group + read_count[1] / stim.iter; + expect_addr = (read_count[1] % stim.iter) & 32'h3; end if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) @@ -115,15 +117,18 @@ class int2fp_scoreboard extends uvm_scoreboard; `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== stim.wr_group) - `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group - )) + if (item.group_id !== stim.wr_group + write_count / stim.iter) + `uvm_fatal( + "WRITE", $sformatf( + "group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + write_count / stim.iter + )) if (item.mask !== 16'hFFFF) `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) if (write_count >= stim.num_dst_words) `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) - if (item.addr !== write_count[BB_BLINK_BANK_ADDR_W-1:0]) - `uvm_fatal("WRITE", $sformatf("addr mismatch: got %0d exp %0d", item.addr, write_count)) + if (item.addr !== write_count % stim.iter) + `uvm_fatal("WRITE", $sformatf( + "addr mismatch: got %0d exp %0d", item.addr, write_count % stim.iter)) if (item.data !== stim.dst_words[write_count]) `uvm_fatal("SCB", $sformatf( "data mismatch at addr %0d: got 0x%032h exp 0x%032h", diff --git a/examples/components/balls/int2fp/workloads/ctests/CMakeLists.txt b/examples/components/balls/int2fp/workloads/ctests/CMakeLists.txt index 97f268d47..a4e28a25f 100644 --- a/examples/components/balls/int2fp/workloads/ctests/CMakeLists.txt +++ b/examples/components/balls/int2fp/workloads/ctests/CMakeLists.txt @@ -4,3 +4,4 @@ add_buckyball_ctest(int2fp_tensor_rows_test.c) add_buckyball_ctest(int2fp_channel_lanes_test.c) add_buckyball_ctest(int2fp_channel_base_test.c) add_buckyball_ctest(int2fp_channel_two_rows_test.c) +add_buckyball_ctest(int2fp_groups_test.c) diff --git a/examples/components/balls/int2fp/workloads/ctests/int2fp_groups_test.c b/examples/components/balls/int2fp/workloads/ctests/int2fp_groups_test.c new file mode 100644 index 000000000..09161fb18 --- /dev/null +++ b/examples/components/balls/int2fp/workloads/ctests/int2fp_groups_test.c @@ -0,0 +1,49 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* INT32_TO_FP32 with ReLU over two channel groups: group g of the input, + * scale, and output banks is channel block g; every group uses the same rows. + */ +enum { GROUPS = 2, ROWS = 8 }; + +static int32_t input[GROUPS][ROWS * 4] __attribute__((aligned(64))); +static float scales[GROUPS][16] __attribute__((aligned(64))); +static float output[GROUPS][ROWS * 4] __attribute__((aligned(64))); + +int main(void) { + for (int group = 0; group < GROUPS; ++group) { + for (int i = 0; i < 16; ++i) + scales[group][i] = (float)(1 << ((group + i) % 3)) * 0.5f; + for (int i = 0; i < ROWS * 4; ++i) + input[group][i] = (group * 13 + i * 5) % 40 - 15; + } + + bb_mem_alloc(0, 1, GROUPS); + bb_mem_alloc(1, 1, GROUPS); + bb_mem_alloc(2, 1, GROUPS); + for (int group = 0; group < GROUPS; ++group) { + bb_mvin_group((uintptr_t)input[group], 0, group, ROWS, 1); + bb_mvin_group((uintptr_t)scales[group], 1, group, 4, 1); + } + bb_int32_to_fp32(0, 1, 2, ROWS, 1); + for (int group = 0; group < GROUPS; ++group) + bb_mvout_group((uintptr_t)output[group], 2, group, ROWS, 1); + + for (int group = 0; group < GROUPS; ++group) + for (int i = 0; i < ROWS * 4; ++i) { + int value = input[group][i] < 0 ? 0 : input[group][i]; + float expected = (float)value * scales[group][i % 16]; + if (output[group][i] != expected) { + printf("int2fp_groups FAIL group=%d index=%d\n", group, i); + return 1; + } + } + bb_mem_release(0); + bb_mem_release(1); + bb_mem_release(2); + printf("int2fp_groups PASS\n"); + return 0; +} diff --git a/examples/components/balls/maxpool/emu/src/maxpool.rs b/examples/components/balls/maxpool/emu/src/maxpool.rs index 9fb1c7225..4b3bd12e5 100644 --- a/examples/components/balls/maxpool/emu/src/maxpool.rs +++ b/examples/components/balls/maxpool/emu/src/maxpool.rs @@ -1,6 +1,6 @@ -use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::instruction::{BallModel, ExecContext}; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; pub struct MaxPool; @@ -100,41 +100,41 @@ impl BallModel for MaxPool { // Group g of both operands is channel block g; each block uses the same rows. for group in 0..groups { - let pi = pbank_group(ctx, input_bank, group); - let po = pbank_group(ctx, output_bank, group); - let (input, output) = ctx.banks.read_write(pi, po); - for output_y in 0..output_side { - for output_x in 0..output_side { - let output_offset = - (output_base + output_y * output_stride + output_x) * bank_row_bytes(); - let mut maximum = [-128i8; 16]; - for kernel_y in 0..kernel { - for kernel_x in 0..kernel { - let input_y = - (output_y * stride + kernel_y + start_row) as isize - padding as isize; - let input_x = - (output_x * stride + kernel_x + start_col) as isize - padding as isize; - if input_y < 0 - || input_x < 0 - || input_y >= input_side as isize - || input_x >= input_side as isize - { - continue; - } - let input_offset = - (input_base + input_y as usize * input_side + input_x as usize) - * bank_row_bytes(); - for lane in 0..16 { - maximum[lane] = maximum[lane].max(input[input_offset + lane] as i8); + let pi = pbank_group(ctx, input_bank, group); + let po = pbank_group(ctx, output_bank, group); + let (input, output) = ctx.banks.read_write(pi, po); + for output_y in 0..output_side { + for output_x in 0..output_side { + let output_offset = + (output_base + output_y * output_stride + output_x) * bank_row_bytes(); + let mut maximum = [-128i8; 16]; + for kernel_y in 0..kernel { + for kernel_x in 0..kernel { + let input_y = (output_y * stride + kernel_y + start_row) as isize + - padding as isize; + let input_x = (output_x * stride + kernel_x + start_col) as isize + - padding as isize; + if input_y < 0 + || input_x < 0 + || input_y >= input_side as isize + || input_x >= input_side as isize + { + continue; + } + let input_offset = + (input_base + input_y as usize * input_side + input_x as usize) + * bank_row_bytes(); + for lane in 0..16 { + maximum[lane] = maximum[lane].max(input[input_offset + lane] as i8); + } } } - } - for lane in 0..16 { - output[output_offset + lane] = maximum[lane] as u8; + for lane in 0..16 { + output[output_offset + lane] = maximum[lane] as u8; + } } } } - } 0 } diff --git a/examples/components/balls/maxpool/stack/src/Lowering.cpp b/examples/components/balls/maxpool/stack/src/Lowering.cpp index 8afa4ca45..6bd1ed83d 100644 --- a/examples/components/balls/maxpool/stack/src/Lowering.cpp +++ b/examples/components/balls/maxpool/stack/src/Lowering.cpp @@ -1,11 +1,17 @@ //===- Lowering.cpp - MaxPoolBall in lower-buckyball ----------------------===// +// +// maxpool_i8: the source holds a square feature map of `input_side` pixels +// per side; group g is channel block g, line y * input_side + x holds the +// sixteen i8 channels of pixel (y, x). The destination, with the same groups, +// receives the `kernel` x `kernel` maxima with `stride` and `padding` in the +// same layout, output_side = (input_side + 2 * padding - kernel) / stride + 1. +// +//===----------------------------------------------------------------------===// #include "Buckyball/BuckyballOps.h" #include "Conversion/Buckyball.h" #include "Target/BuckyballTargetRegistry.h" -#include - using namespace mlir; using namespace ::buddy::buckyball; @@ -22,49 +28,24 @@ class MaxpoolI8Lowering : public OpRewritePattern { return op.emitOpError( "maxpool_i8 requires distinct input and output banks"); int64_t inputSide = cast(op->getAttr("input_side")).getInt(); - int64_t outputSide = cast(op->getAttr("output_side")).getInt(); int64_t kernel = cast(op->getAttr("kernel")).getInt(); int64_t stride = cast(op->getAttr("stride")).getInt(); int64_t padding = cast(op->getAttr("padding")).getInt(); - int64_t inputBase = cast(op->getAttr("op1_base")).getInt(); - int64_t outputBase = cast(op->getAttr("wr_base")).getInt(); - int64_t outputStride = - cast(op->getAttr("output_stride")).getInt(); - int64_t startRow = cast(op->getAttr("start_row")).getInt(); - int64_t startCol = cast(op->getAttr("start_col")).getInt(); - const auto &target = buckyball_target::getBuckyballTarget(); - if (target.bankWidthBits != 128 || inputSide <= 0 || inputSide > 15 || - outputSide <= 0 || outputSide > 15 || kernel <= 0 || kernel > 15 || - stride <= 0 || stride > 15 || padding < 0 || padding > 15 || - inputBase < 0 || inputBase > 63 || outputBase < 0 || outputBase > 63 || - outputStride < outputSide || outputStride > 63 || startRow < 0 || - startRow > 15 || startCol < 0 || startCol > 15) + int64_t span = inputSide + 2 * padding - kernel; + if (inputSide <= 0 || inputSide > 15 || kernel <= 0 || kernel > 15 || + stride <= 0 || stride > 15 || padding < 0 || padding > 15 || span < 0 || + span % stride || span / stride + 1 > 15 || + inputSide * inputSide > + buckyball_target::getBuckyballTarget().bankDepth) return op.emitOpError("maxpool_i8 geometry exceeds the ISA field ranges"); - int64_t lastRow = startRow + (outputSide - 1) * stride + kernel - 1; - int64_t lastCol = startCol + (outputSide - 1) * stride + kernel - 1; - int64_t inputEnd = padding + inputSide; - if (lastRow >= inputSide + 2 * padding || - lastCol >= inputSide + 2 * padding || - outputBase + (outputSide - 1) * outputStride + outputSide > - target.bankDepth) - return op.emitOpError( - "maxpool_i8 window or output bank footprint is invalid"); - if (startRow < inputEnd && startCol < inputEnd && lastRow >= padding && - lastCol >= padding && - inputBase + (std::min(lastRow, inputEnd - 1) - padding) * inputSide + - std::min(lastCol, inputEnd - 1) - padding >= - target.bankDepth) - return op.emitOpError("maxpool_i8 input bank footprint is invalid"); + int64_t outputSide = span / stride + 1; + // Both maps start at the base of their groups; output rows are packed. uint64_t rs2 = mlir::buddy::field(inputSide, 0, 3) | mlir::buddy::field(outputSide, 4, 7) | mlir::buddy::field(kernel, 8, 11) | mlir::buddy::field(stride, 12, 15) | mlir::buddy::field(padding, 16, 19) | - mlir::buddy::field(inputBase, 20, 25) | - mlir::buddy::field(outputBase, 26, 31) | - mlir::buddy::field(outputStride, 32, 37) | - mlir::buddy::field(startRow, 38, 41) | - mlir::buddy::field(startCol, 42, 45); + mlir::buddy::field(outputSide, 32, 37); mlir::buddy::emitInstruction( rewriter, op.getLoc(), buckyball_target::getBuckyballFunct7("MAXPOOL"), mlir::buddy::constantI64( diff --git a/examples/components/balls/maxpool/verify/casegen/src/casegen.rs b/examples/components/balls/maxpool/verify/casegen/src/casegen.rs index cf818f95c..58c13bdce 100644 --- a/examples/components/balls/maxpool/verify/casegen/src/casegen.rs +++ b/examples/components/balls/maxpool/verify/casegen/src/casegen.rs @@ -1,8 +1,8 @@ use crate::model; -pub const MAX_INPUT_WORDS: usize = 64; +pub const MAX_INPUT_WORDS: usize = 128; pub const MAX_DST: usize = 64; -pub const NUM_CASES: u32 = 9; +pub const NUM_CASES: u32 = 11; #[repr(C)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -36,8 +36,11 @@ pub struct MaxPoolCase { pub cmd: MaxPoolCmd, pub input_words: [u128; MAX_INPUT_WORDS], pub input_addr: [u32; MAX_INPUT_WORDS], + // Channel group of each word, relative to the command's base group. + pub input_group: [u32; MAX_INPUT_WORDS], pub dst_words: [u128; MAX_DST], pub dst_addr: [u32; MAX_DST], + pub dst_group: [u32; MAX_DST], } impl MaxPoolCase { @@ -66,6 +69,9 @@ struct CaseSpec { rob_id: u32, ctest_input: bool, seed: u32, + // Channel groups of the input and output banks; group g is pooled with + // the same geometry after group g - 1. + groups: u32, } fn field(val: u32, lo: u32, hi: u32) -> u64 { @@ -155,11 +161,6 @@ fn copy_addrs(src: &[u32]) -> [u32; N] { fn build_case(spec: CaseSpec) -> MaxPoolCase { model::validate(&spec.geo); let iter = model::iter_for(&spec.geo); - let input_bytes = if spec.ctest_input { - ctest_tile(&spec.geo) - } else { - seeded_tile(&spec.geo, spec.seed) - }; let mut referenced = vec![false; spec.geo.input_side * spec.geo.input_side]; for output_y in 0..spec.geo.output_side { @@ -183,26 +184,39 @@ fn build_case(spec: CaseSpec) -> MaxPoolCase { let mut input_addrs = Vec::new(); let mut input_words = Vec::new(); - for y in 0..spec.geo.input_side { - for x in 0..spec.geo.input_side { - if !referenced[y * spec.geo.input_side + x] { - continue; + let mut input_groups = Vec::new(); + let mut dst_addrs = Vec::new(); + let mut dst_words = Vec::new(); + let mut dst_groups = Vec::new(); + for group in 0..spec.groups { + let input_bytes = if spec.ctest_input { + ctest_tile(&spec.geo) + } else { + seeded_tile(&spec.geo, spec.seed + group * 101) + }; + for y in 0..spec.geo.input_side { + for x in 0..spec.geo.input_side { + if !referenced[y * spec.geo.input_side + x] { + continue; + } + let addr = (spec.geo.input_base + y * spec.geo.input_side + x) as u32; + let row_off = (y * spec.geo.input_side + x) * model::BANK_ROW_BYTES; + let mut row = [0u8; model::BANK_ROW_BYTES]; + row.copy_from_slice(&input_bytes[row_off..row_off + model::BANK_ROW_BYTES]); + input_addrs.push(addr); + input_words.push(model::pack_row(&row)); + input_groups.push(group); } - let addr = (spec.geo.input_base + y * spec.geo.input_side + x) as u32; - let row_off = (y * spec.geo.input_side + x) * model::BANK_ROW_BYTES; - let mut row = [0u8; model::BANK_ROW_BYTES]; - row.copy_from_slice(&input_bytes[row_off..row_off + model::BANK_ROW_BYTES]); - input_addrs.push(addr); - input_words.push(model::pack_row(&row)); + } + for (addr, row) in model::maxpool_rows(&input_bytes, &spec.geo) { + dst_addrs.push(addr); + dst_words.push(model::pack_row(&row)); + dst_groups.push(group); } } - let gold = model::maxpool_rows(&input_bytes, &spec.geo); - let dst_addrs: Vec = gold.iter().map(|(addr, _)| *addr).collect(); - let dst_words: Vec = gold.iter().map(|(_, row)| model::pack_row(row)).collect(); - - if dst_addrs.len() as u32 != iter { - panic!("maxpool: gold rows {} != iter {iter}", dst_addrs.len()); + if dst_addrs.len() as u32 != iter * spec.groups { + panic!("maxpool: gold rows {} != iter {iter} x groups", dst_addrs.len()); } let rs2 = encode_rs2(&spec.geo); @@ -218,8 +232,8 @@ fn build_case(spec: CaseSpec) -> MaxPoolCase { // Vary the renamed group bases across cases; the Ball adds them to every access. op1_group: spec.seed % 3, wr_group: (spec.seed + 1) % 4, - op1_col: 1, - wr_col: 1, + op1_col: spec.groups, + wr_col: spec.groups, rob_id: spec.rob_id, rs2_lo, rs2_hi, @@ -234,12 +248,14 @@ fn build_case(spec: CaseSpec) -> MaxPoolCase { start_row: spec.geo.start_row as u32, start_col: spec.geo.start_col as u32, num_input_words: input_words.len() as u32, - num_dst_words: iter, + num_dst_words: dst_words.len() as u32, }, input_words: copy_words(&input_words), input_addr: copy_addrs::(&input_addrs), + input_group: copy_addrs::(&input_groups), dst_words: copy_words(&dst_words), dst_addr: copy_addrs::(&dst_addrs), + dst_group: copy_addrs::(&dst_groups), } } @@ -263,6 +279,7 @@ fn spec_for(index: u32) -> CaseSpec { rob_id: 2, ctest_input: true, seed: 0, + groups: 1, }, 1 => CaseSpec { geo: model::Geo { @@ -282,6 +299,7 @@ fn spec_for(index: u32) -> CaseSpec { rob_id: 3, ctest_input: false, seed: 1, + groups: 1, }, 2 => CaseSpec { geo: model::Geo { @@ -301,6 +319,7 @@ fn spec_for(index: u32) -> CaseSpec { rob_id: 4, ctest_input: false, seed: 2, + groups: 1, }, 3 => CaseSpec { geo: model::Geo { @@ -320,6 +339,7 @@ fn spec_for(index: u32) -> CaseSpec { rob_id: 5, ctest_input: false, seed: 3, + groups: 1, }, 4 => CaseSpec { geo: model::Geo { @@ -339,6 +359,7 @@ fn spec_for(index: u32) -> CaseSpec { rob_id: 6, ctest_input: false, seed: 4, + groups: 1, }, 5 => CaseSpec { geo: model::Geo { @@ -358,6 +379,7 @@ fn spec_for(index: u32) -> CaseSpec { rob_id: 7, ctest_input: false, seed: 5, + groups: 1, }, 6 => CaseSpec { geo: model::Geo { @@ -377,6 +399,7 @@ fn spec_for(index: u32) -> CaseSpec { rob_id: 8, ctest_input: false, seed: 6, + groups: 1, }, 7 => CaseSpec { geo: model::Geo { @@ -396,6 +419,7 @@ fn spec_for(index: u32) -> CaseSpec { rob_id: 9, ctest_input: false, seed: 7, + groups: 1, }, 8 => CaseSpec { geo: model::Geo { @@ -415,6 +439,49 @@ fn spec_for(index: u32) -> CaseSpec { rob_id: 10, ctest_input: false, seed: 8, + groups: 1, + }, + // Two channel groups, 3x3 kernel with padding. + 9 => CaseSpec { + geo: model::Geo { + input_side: 5, + output_side: 3, + kernel: 3, + stride: 2, + padding: 1, + input_base: 2, + output_base: 5, + output_stride: 4, + start_row: 0, + start_col: 0, + }, + op1_bank: 2, + wr_bank: 5, + rob_id: 11, + ctest_input: false, + seed: 9, + groups: 2, + }, + // Three channel groups, 2x2 stride-2 pooling. + 10 => CaseSpec { + geo: model::Geo { + input_side: 4, + output_side: 2, + kernel: 2, + stride: 2, + padding: 0, + input_base: 0, + output_base: 0, + output_stride: 2, + start_row: 0, + start_col: 0, + }, + op1_bank: 1, + wr_bank: 0, + rob_id: 12, + ctest_input: false, + seed: 10, + groups: 3, }, _ => panic!("maxpool: unsupported directed case {index}"), } diff --git a/examples/components/balls/maxpool/verify/casegen/src/lib.rs b/examples/components/balls/maxpool/verify/casegen/src/lib.rs index a556d5929..2b7d1437f 100644 --- a/examples/components/balls/maxpool/verify/casegen/src/lib.rs +++ b/examples/components/balls/maxpool/verify/casegen/src/lib.rs @@ -105,3 +105,25 @@ pub extern "C" fn maxpool_case_dst_addr(word_index: u32) -> u32 { case.dst_addr[i] }) } + +#[no_mangle] +pub extern "C" fn maxpool_case_input_group(word_index: u32) -> u32 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_input_words, "input_group"); + if i >= MAX_INPUT_WORDS { + panic!("maxpool_case_input_group: word_index out of range"); + } + case.input_group[i] + }) +} + +#[no_mangle] +pub extern "C" fn maxpool_case_dst_group(word_index: u32) -> u32 { + current_case(|case| { + let i = idx(word_index, case.cmd.num_dst_words, "dst_group"); + if i >= MAX_DST { + panic!("maxpool_case_dst_group: word_index out of range"); + } + case.dst_group[i] + }) +} diff --git a/examples/components/balls/maxpool/verify/src/common/maxpool_defs.svh b/examples/components/balls/maxpool/verify/src/common/maxpool_defs.svh index df00aad02..6bd4983b4 100644 --- a/examples/components/balls/maxpool/verify/src/common/maxpool_defs.svh +++ b/examples/components/balls/maxpool/verify/src/common/maxpool_defs.svh @@ -31,13 +31,15 @@ import "DPI-C" function int unsigned maxpool_case_input_addr(input int unsigned import "DPI-C" function longint unsigned maxpool_case_dst_word_lo(input int unsigned word_index); import "DPI-C" function longint unsigned maxpool_case_dst_word_hi(input int unsigned word_index); import "DPI-C" function int unsigned maxpool_case_dst_addr(input int unsigned word_index); +import "DPI-C" function int unsigned maxpool_case_input_group(input int unsigned word_index); +import "DPI-C" function int unsigned maxpool_case_dst_group(input int unsigned word_index); `ifndef MAXPOOL_FUNCT7 `error "MAXPOOL_FUNCT7 must be provided by the selected Thread ballISA" `endif localparam int MAXPOOL_THREAD_FUNCT7 = `MAXPOOL_FUNCT7; -localparam int MAXPOOL_MAX_INPUT_WORDS = 64; +localparam int MAXPOOL_MAX_INPUT_WORDS = 128; localparam int MAXPOOL_MAX_DST_WORDS = 64; -localparam int MAXPOOL_NUM_CASES = 9; +localparam int MAXPOOL_NUM_CASES = 11; localparam int MAXPOOL_TIMEOUT_CYCLES = 20000; diff --git a/examples/components/balls/maxpool/verify/src/common/maxpool_items.svh b/examples/components/balls/maxpool/verify/src/common/maxpool_items.svh index f69b45154..35485fa12 100644 --- a/examples/components/balls/maxpool/verify/src/common/maxpool_items.svh +++ b/examples/components/balls/maxpool/verify/src/common/maxpool_items.svh @@ -15,8 +15,10 @@ class maxpool_cmd_item extends bb_blink_cmd_item; bit [31:0] num_dst_words; bit [127:0] input_words[MAXPOOL_MAX_INPUT_WORDS]; bit [31:0] input_addr[MAXPOOL_MAX_INPUT_WORDS]; + bit [31:0] input_group[MAXPOOL_MAX_INPUT_WORDS]; bit [127:0] dst_words[MAXPOOL_MAX_DST_WORDS]; bit [31:0] dst_addr[MAXPOOL_MAX_DST_WORDS]; + bit [31:0] dst_group[MAXPOOL_MAX_DST_WORDS]; function new(string name = "maxpool_cmd_item"); super.new(name); @@ -64,12 +66,12 @@ class maxpool_cmd_item extends bb_blink_cmd_item; num_input_words = cmd.num_input_words; num_dst_words = cmd.num_dst_words; - if (op1_col != 1 || wr_col != 1) - `uvm_fatal("CASE", "MAXPOOL requires one input group and one output group") + if (op1_col == 0 || wr_col != op1_col) + `uvm_fatal("CASE", "MAXPOOL input and output must have the same channel groups") if (op1_bank == wr_bank) `uvm_fatal("CASE", "MAXPOOL input and output banks must differ") if (rs2[63:46] != 0) `uvm_fatal("CASE", "MAXPOOL reserved rs2 bits must be zero") - if (num_dst_words != iter) - `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter %0d", num_dst_words, iter)) + if (num_dst_words != iter * op1_col) + `uvm_fatal("CASE", $sformatf("num_dst_words %0d != iter %0d x groups", num_dst_words, iter)) if (iter != output_side * output_side) `uvm_fatal("CASE", $sformatf("iter %0d != output_side^2 %0d", iter, output_side * output_side )) @@ -83,12 +85,14 @@ class maxpool_cmd_item extends bb_blink_cmd_item; w_hi = maxpool_case_input_word_hi(i); input_words[i] = {w_hi, w_lo}; input_addr[i] = maxpool_case_input_addr(i); + input_group[i] = maxpool_case_input_group(i); end for (i = 0; i < num_dst_words; i++) begin w_lo = maxpool_case_dst_word_lo(i); w_hi = maxpool_case_dst_word_hi(i); dst_words[i] = {w_hi, w_lo}; dst_addr[i] = maxpool_case_dst_addr(i); + dst_group[i] = maxpool_case_dst_group(i); end endfunction @@ -113,10 +117,12 @@ class maxpool_cmd_item extends bb_blink_cmd_item; for (int i = 0; i < MAXPOOL_MAX_INPUT_WORDS; i++) begin input_words[i] = rhs_.input_words[i]; input_addr[i] = rhs_.input_addr[i]; + input_group[i] = rhs_.input_group[i]; end for (int i = 0; i < MAXPOOL_MAX_DST_WORDS; i++) begin dst_words[i] = rhs_.dst_words[i]; dst_addr[i] = rhs_.dst_addr[i]; + dst_group[i] = rhs_.dst_group[i]; end endfunction endclass diff --git a/examples/components/balls/maxpool/verify/src/cov/maxpool_cov.svh b/examples/components/balls/maxpool/verify/src/cov/maxpool_cov.svh index 9211a2549..9d53c45ba 100644 --- a/examples/components/balls/maxpool/verify/src/cov/maxpool_cov.svh +++ b/examples/components/balls/maxpool/verify/src/cov/maxpool_cov.svh @@ -6,12 +6,14 @@ class maxpool_cov extends uvm_component; int unsigned cur_kernel; int unsigned cur_stride; int unsigned cur_padding; + int unsigned cur_groups; int cmd_count; covergroup cmd_cg; coverpoint cur_kernel {bins k2 = {2}; bins k3 = {3};} coverpoint cur_stride {bins s1 = {1}; bins s2 = {2};} coverpoint cur_padding {bins p0 = {0}; bins p1 = {1};} + coverpoint cur_groups {bins one = {1}; bins many = {[2 : 24]};} endgroup function new(string name, uvm_component parent); @@ -26,6 +28,7 @@ class maxpool_cov extends uvm_component; cur_kernel = int'(item.rs2[11:8]); cur_stride = int'(item.rs2[15:12]); cur_padding = int'(item.rs2[19:16]); + cur_groups = int'(item.op1_col); cmd_count++; cmd_cg.sample(); endfunction @@ -38,7 +41,7 @@ class maxpool_cov extends uvm_component; `uvm_fatal( "COV", $sformatf( - "maxpool_cov coverage incomplete: %0.2f%% (need kernel {2,3}, stride {1,2}, pad {0,1})", + "maxpool_cov coverage incomplete: %0.2f%% (need kernel {2,3}, stride {1,2}, pad {0,1}, groups {1,>1})", cmd_cg.get_coverage())) end endfunction diff --git a/examples/components/balls/maxpool/verify/src/env/maxpool_scoreboard.svh b/examples/components/balls/maxpool/verify/src/env/maxpool_scoreboard.svh index 18051983e..4abf6d2ac 100644 --- a/examples/components/balls/maxpool/verify/src/env/maxpool_scoreboard.svh +++ b/examples/components/balls/maxpool/verify/src/env/maxpool_scoreboard.svh @@ -46,8 +46,8 @@ class maxpool_scoreboard extends uvm_scoreboard; end mem_model.clear_mem(); for (i = 0; i < clone.num_input_words; i++) begin - mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group), int'(clone.input_addr[i]), - clone.input_words[i]); + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group + clone.input_group[i]), + int'(clone.input_addr[i]), clone.input_words[i]); end mem_model.arm(); @@ -91,8 +91,12 @@ class maxpool_scoreboard extends uvm_scoreboard; stim = current_stim("READ"); if (item.port != 0) `uvm_fatal("READ", $sformatf("MaxPoolBall has one read port, got port %0d", item.port)) - if (item.group_id !== stim.op1_group) - `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.op1_group + // Groups are pooled in order: every read belongs to the group being written. + if (item.group_id !== stim.op1_group + stim.dst_group[write_count]) + `uvm_fatal("READ", $sformatf( + "group mismatch: got %0d exp %0d", + item.group_id, + stim.op1_group + stim.dst_group[write_count] )) if (item.rob_id !== stim.rob_id) `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) @@ -113,13 +117,16 @@ class maxpool_scoreboard extends uvm_scoreboard; `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== stim.wr_group) - `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + if (write_count >= stim.num_dst_words) + `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) + if (item.group_id !== stim.wr_group + stim.dst_group[write_count]) + `uvm_fatal("WRITE", $sformatf( + "group mismatch: got %0d exp %0d", + item.group_id, + stim.wr_group + stim.dst_group[write_count] )) if (item.mask !== 16'hFFFF) `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) - if (write_count >= stim.num_dst_words) - `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) if (item.addr !== stim.dst_addr[write_count][BB_BLINK_BANK_ADDR_W-1:0]) `uvm_fatal("WRITE", $sformatf( "addr mismatch: got %0d exp %0d", item.addr, stim.dst_addr[write_count])) diff --git a/examples/components/balls/maxpool/workloads/ctests/CMakeLists.txt b/examples/components/balls/maxpool/workloads/ctests/CMakeLists.txt index c2cd34b14..67f5c70c6 100644 --- a/examples/components/balls/maxpool/workloads/ctests/CMakeLists.txt +++ b/examples/components/balls/maxpool/workloads/ctests/CMakeLists.txt @@ -1,2 +1,3 @@ add_buckyball_ctest(maxpool_window_test.c) add_buckyball_ctest(maxpool_offset_test.c) +add_buckyball_ctest(maxpool_groups_test.c) diff --git a/examples/components/balls/maxpool/workloads/ctests/maxpool_groups_test.c b/examples/components/balls/maxpool/workloads/ctests/maxpool_groups_test.c new file mode 100644 index 000000000..b83d773fd --- /dev/null +++ b/examples/components/balls/maxpool/workloads/ctests/maxpool_groups_test.c @@ -0,0 +1,47 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* 2x2 stride-2 MAXPOOL over an NHWC 4x4 input with 32 channels: input and + * output group g are channel block g, so one instruction pools both blocks + * and a whole-bank DMA moves NHWC rows directly. */ +enum { GROUPS = 2, CHANNELS = 16 * GROUPS, SIDE = 4, OUT = 2 }; + +static int8_t input[SIDE * SIDE * CHANNELS] __attribute__((aligned(64))); +static int8_t actual[OUT * OUT * CHANNELS] __attribute__((aligned(64))); + +int main(void) { + for (int i = 0; i < SIDE * SIDE * CHANNELS; ++i) + input[i] = (int8_t)((i * 37 + 11) % 255 - 127); + + bb_mem_alloc(0, 1, GROUPS); + bb_mem_alloc(1, 1, GROUPS); + bb_mvin((uintptr_t)input, 0, SIDE * SIDE, 1); + bb_maxpool(0, 1, SIDE, OUT, 2, 2, 0, 0, 0, OUT, 0, 0); + bb_mvout((uintptr_t)actual, 1, OUT * OUT, 1); + + for (int oy = 0; oy < OUT; ++oy) + for (int ox = 0; ox < OUT; ++ox) + for (int channel = 0; channel < CHANNELS; ++channel) { + int expected = -128; + for (int ky = 0; ky < 2; ++ky) + for (int kx = 0; kx < 2; ++kx) { + int value = input[((oy * 2 + ky) * SIDE + ox * 2 + kx) * CHANNELS + + channel]; + expected = value > expected ? value : expected; + } + int got = actual[(oy * OUT + ox) * CHANNELS + channel]; + if (got != expected) { + printf("maxpool_groups FAIL y=%d x=%d channel=%d expected=%d " + "actual=%d\n", + oy, ox, channel, expected, got); + return 1; + } + } + bb_mem_release(0); + bb_mem_release(1); + printf("maxpool_groups PASS\n"); + return 0; +} diff --git a/examples/components/balls/smatmul/README.md b/examples/components/balls/smatmul/README.md index 482c99155..89d2f23e3 100644 --- a/examples/components/balls/smatmul/README.md +++ b/examples/components/balls/smatmul/README.md @@ -14,8 +14,13 @@ bias vector. bank1, bank2, and `rs2` are zero, and `iter` is four. `SMATMUL_OS` uses bank0 for A, bank1 for B, and bank2 for C. `iter` is K. `rs2[11:0]` is M, `rs2[23:12]` is N, and bits 24 and 25 are the first/last -accumulation-block flags. The remaining `rs2` bits are zero. M and K are -positive multiples of 16 and N is exactly 16. +accumulation-block flags and bits 31:26 the C base row. The remaining `rs2` +bits are zero. M is one or a positive multiple of tileRows, N is exactly 16. + +A and B have the same number of groups G. Group g holds reduction block g in +the single-group layouts below, so `iter` is K per group (a positive multiple +of 16) and the instruction accumulates over `K = iter * G`, walking group 0 to +group G-1 for every output tile. ## Layout @@ -26,6 +31,10 @@ Every result row has four 128-bit words. With `outBW=1`, logical output row line r*4 + word ``` +C rows continue across groups: line `L = base + r*4 + word` is written to +group `L / bankEntries`, line `L % bankEntries`. M is limited by the +accumulator (`max(bankEntries / 4, 32)` rows). + The Ball keeps partial sums internally between commands with `last=0`; only the command with `last=1` writes C. A later accumulation block may select new A and B banks, but each still starts at bank row zero. diff --git a/examples/components/balls/smatmul/emu/src/smatmul.rs b/examples/components/balls/smatmul/emu/src/smatmul.rs index dadb13ad0..1decf2c74 100644 --- a/examples/components/balls/smatmul/emu/src/smatmul.rs +++ b/examples/components/balls/smatmul/emu/src/smatmul.rs @@ -1,10 +1,22 @@ -use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::instruction::ExecContext; +use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; use std::collections::HashMap; -fn tile_rows(instance: u32) -> usize { bebop_bemu::configs::ball_domain::parameters::("smatmul", instance).tile_rows.expect("tileRows") as usize } -fn tile_cols(instance: u32) -> usize { bebop_bemu::configs::ball_domain::parameters::("smatmul", instance).tile_cols.expect("tileCols") as usize } +fn tile_rows(instance: u32) -> usize { + bebop_bemu::configs::ball_domain::parameters::< + bebop_bemu::configs::ball_domain::parameters::smatmul::Params, + >("smatmul", instance) + .tile_rows + .expect("tileRows") as usize +} +fn tile_cols(instance: u32) -> usize { + bebop_bemu::configs::ball_domain::parameters::< + bebop_bemu::configs::ball_domain::parameters::smatmul::Params, + >("smatmul", instance) + .tile_cols + .expect("tileCols") as usize +} #[derive(Default)] struct State { @@ -20,8 +32,6 @@ struct Chain { accumulators: Vec, } - - fn read_i32(bank: &[u8], row: usize, lane: usize) -> i32 { let offset = row * bank_row_bytes() + lane * 4; i32::from_le_bytes(bank[offset..offset + 4].try_into().unwrap()) @@ -75,7 +85,11 @@ pub(crate) fn exec_smatmul(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { if xs2 >> 32 != 0 { panic!("smatmul: rs2[63:32] must be zero"); } - if (rows != 1 && (rows == 0 || rows % m_tile != 0)) || cols != n_tile || k == 0 || k % k_tile != 0 { + if (rows != 1 && (rows == 0 || rows % m_tile != 0)) + || cols != n_tile + || k == 0 + || k % k_tile != 0 + { panic!("smatmul: M must be one or a positive multiple of 8, K must be a positive multiple of 16, and N must be 16"); } if a_bank == b_bank || a_bank == c_bank || b_bank == c_bank { @@ -98,10 +112,17 @@ pub(crate) fn exec_smatmul(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { panic!("smatmul: M exceeds the accumulator"); } - let a_rows = if rows == 1 { k / k_tile } else { rows * k / k_tile }; + let a_rows = if rows == 1 { + k / k_tile + } else { + rows * k / k_tile + }; let b_rows = k; let c_rows = rows * 4; - if a_rows > bank_lines() || b_rows > bank_lines() || output_base + c_rows > c_groups * bank_lines() { + if a_rows > bank_lines() + || b_rows > bank_lines() + || output_base + c_rows > c_groups * bank_lines() + { panic!("smatmul: bank footprint exceeds bank depth"); } @@ -152,30 +173,30 @@ pub(crate) fn exec_smatmul(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { // Wrapping arithmetic is intentionally retained to match the accumulator // semantics of the RTL and the previous emulator implementation. for k_group in 0..k_groups { - let pa = pbank_group(ctx, a_bank, k_group as u64); - let pb = pbank_group(ctx, b_bank, k_group as u64); - for inner in 0..k { - let b_base = ((inner / k_tile) * k_tile + inner % k_tile) * row_bytes; - let b_row = &ctx.banks[pb][b_base..b_base + cols]; - let mut b_values = vec![0i32; n_tile]; - for col in 0..cols { - b_values[col] = b_row[col] as i8 as i32; - } - for row in 0..rows { - let a_row = if rows == 1 { - inner / k_tile - } else { - ((row / m_tile) * k_tiles + inner / k_tile) * m_tile + row % m_tile - }; - let a = ctx.banks[pa][a_row * row_bytes + inner % k_tile] as i8 as i32; - let acc_base = row * cols; - let acc_row = &mut chain.accumulators[acc_base..acc_base + cols]; + let pa = pbank_group(ctx, a_bank, k_group as u64); + let pb = pbank_group(ctx, b_bank, k_group as u64); + for inner in 0..k { + let b_base = ((inner / k_tile) * k_tile + inner % k_tile) * row_bytes; + let b_row = &ctx.banks[pb][b_base..b_base + cols]; + let mut b_values = vec![0i32; n_tile]; for col in 0..cols { - acc_row[col] = acc_row[col].wrapping_add(a.wrapping_mul(b_values[col])); + b_values[col] = b_row[col] as i8 as i32; + } + for row in 0..rows { + let a_row = if rows == 1 { + inner / k_tile + } else { + ((row / m_tile) * k_tiles + inner / k_tile) * m_tile + row % m_tile + }; + let a = ctx.banks[pa][a_row * row_bytes + inner % k_tile] as i8 as i32; + let acc_base = row * cols; + let acc_row = &mut chain.accumulators[acc_base..acc_base + cols]; + for col in 0..cols { + acc_row[col] = acc_row[col].wrapping_add(a.wrapping_mul(b_values[col])); + } } } } - } if last { for row in 0..rows { @@ -189,7 +210,12 @@ pub(crate) fn exec_smatmul(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { } } else { { - ball_state.lock().unwrap().entry(ctx.hart_id).or_default().chain = Some(chain); + ball_state + .lock() + .unwrap() + .entry(ctx.hart_id) + .or_default() + .chain = Some(chain); }; } 0 diff --git a/examples/components/balls/smatmul/stack/src/Lowering.cpp b/examples/components/balls/smatmul/stack/src/Lowering.cpp index 0ec2620ca..df19834d3 100644 --- a/examples/components/balls/smatmul/stack/src/Lowering.cpp +++ b/examples/components/balls/smatmul/stack/src/Lowering.cpp @@ -1,13 +1,15 @@ //===- Lowering.cpp - SMatMulBall in lower-buckyball ----------------------===// // -// matmul_i8 with read_destination: C = C + A @ B for `rows` rows (one or a -// multiple of eight) and sixteen columns, over `k` (a multiple of sixteen). -// C starts with one row of sixteen i32 (four bank rows), the bias of every -// row, and receives `rows` rows of sixteen i32 (four bank rows each). -// A is contiguous for rows=1, otherwise packed in tileRows x 16 blocks; -// B holds k rows of sixteen i8. -// matmul_i8_block extends the reduction across instructions: first loads -// the bias from C, and last writes the completed accumulation to C. +// matmul_i8 (A, B) -> C with read_destination: C = bias + A @ B for `rows` +// rows (one, or a multiple of tileRows up to a quarter of the bank depth) and +// sixteen columns. A and B have the same number of groups; group g holds +// reduction block g of `k` columns (a multiple of sixteen), so K = k * groups. +// In group g, B holds k rows of sixteen i8. A holds k / 16 rows of sixteen i8 +// for rows = 1; otherwise M tiles of tileRows rows, each tile the k / 16 +// slices of sixteen columns one after another: line +// ((r / tileRows) * (k / 16) + c / 16) * tileRows + r % tileRows. +// C starts with the bias, one row of sixteen i32 in its first four lines, and +// receives `rows` rows of sixteen i32 (four lines each). // //===----------------------------------------------------------------------===// @@ -24,41 +26,35 @@ class MatmulI8Lowering : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; LogicalResult matchAndRewrite(CustomOp op, PatternRewriter &rewriter) const override { - bool block = op.getKind() == "matmul_i8_block"; - if (op.getKind() != "matmul_i8" && !block) + if (op.getKind() != "matmul_i8") return failure(); if (!op.getReadDestination() || op.getSources().size() != 2) - return op.emitOpError("INT8 matmul reads A, B and the old C"); + return op.emitOpError("INT8 matmul reads A, B and the bias in C"); Location loc = op.getLoc(); int64_t rows = cast(op->getAttr("rows")).getInt(); int64_t k = cast(op->getAttr("k")).getInt(); int64_t c = op.getDestination(); - bool first = !block || cast(op->getAttr("first")).getValue(); - bool last = !block || cast(op->getAttr("last")).getValue(); const auto &target = buckyball_target::getBuckyballTarget(); int64_t tileRows = buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); - int64_t tileCols = - buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileCols"); - if (target.bankWidthBits != 128 || tileCols != 16 || tileRows <= 0 || - rows <= 0 || rows > 4095 || (rows != 1 && rows % tileRows) || k <= 0 || - k >= (1LL << 33) || k > target.bankDepth || k % 16 || - rows * (k / 16) > target.bankDepth || rows * 4 > target.bankDepth) + if (target.bankWidthBits != 128 || rows <= 0 || + rows * 4 > target.bankDepth || (rows != 1 && rows % tileRows) || + k <= 0 || k > target.bankDepth || k % 16 || + rows * (k / 16) > target.bankDepth) return op.emitOpError( "INT8 matmul shape exceeds the bank or ISA geometry"); if (op.getSources()[0] == op.getSources()[1] || op.getSources()[0] == c || op.getSources()[1] == c) return op.emitOpError("INT8 matmul requires distinct A, B and C banks"); - // The bias of the accumulation: the four rows at the base of C. - if (first) - mlir::buddy::emitInstruction( - rewriter, loc, buckyball_target::getBuckyballFunct7("SMATMUL_BIAS"), - mlir::buddy::constantI64(rewriter, loc, - mlir::buddy::bankFields({c}, 0, false, 4)), - mlir::buddy::constantI64(rewriter, loc, 0)); + mlir::buddy::emitInstruction( + rewriter, loc, buckyball_target::getBuckyballFunct7("SMATMUL_BIAS"), + mlir::buddy::constantI64(rewriter, loc, + mlir::buddy::bankFields({c}, 0, false, 4)), + mlir::buddy::constantI64(rewriter, loc, 0)); + // One accumulation over every group: first and last in one instruction. uint64_t rs2 = mlir::buddy::field(rows, 0, 11) | mlir::buddy::field(16, 12, 23) | - mlir::buddy::field(first, 24, 24) | mlir::buddy::field(last, 25, 25); + mlir::buddy::field(1, 24, 24) | mlir::buddy::field(1, 25, 25); mlir::buddy::emitInstruction( rewriter, loc, buckyball_target::getBuckyballFunct7("SMATMUL_OS"), mlir::buddy::constantI64( diff --git a/examples/components/balls/smatmul/verify/casegen/src/casegen.rs b/examples/components/balls/smatmul/verify/casegen/src/casegen.rs index f3e5bb4cf..02f97a771 100644 --- a/examples/components/balls/smatmul/verify/casegen/src/casegen.rs +++ b/examples/components/balls/smatmul/verify/casegen/src/casegen.rs @@ -17,12 +17,15 @@ pub struct MatrixCmd { pub op1_group: u32, pub op2_group: u32, pub wr_group: u32, + pub k_groups: u32, + pub wr_col: u32, } #[derive(Clone, Debug, PartialEq, Eq)] pub struct MatrixCase { pub commands: Vec, pub bias: Vec, + // a[block] / b[block] hold every K group of the block, group-major. pub a: Vec>, pub b: Vec>, pub writes: Vec, @@ -34,6 +37,8 @@ pub fn gen_case(seed: u32, index: u32, tile_rows: usize) -> MatrixCase { 0 => build_case(seed, 16, 1, tile_rows, false), 1 => build_case(seed, 16, 2, tile_rows, false), 2 => build_case(seed ^ index, 16, 1, tile_rows, false), + 11 => three_k_groups(seed, tile_rows), + 12 => c_crosses_groups(seed, tile_rows), _ => { let mut rng = Rng::new(seed, index); let blocks = 1 + (rng.next() % 2) as usize; @@ -72,6 +77,7 @@ fn build_case( rob_id: 0, op1_group: bias_group, op1_words: 4, + k_groups: 1, rs1_lo: model::encode_rs1(BIAS_BANK, 0, 0, 4) as u32, rs1_hi: (model::encode_rs1(BIAS_BANK, 0, 0, 4) >> 32) as u32, ..MatrixCmd::default() @@ -99,7 +105,7 @@ fn build_case( } } let rs1 = model::encode_rs1(a_bank, b_bank, C_BANK, model::CORE as u64); - let rs2 = model::encode_rs2(rows, block == 0, block + 1 == blocks); + let rs2 = model::encode_rs2(rows, block == 0, block + 1 == blocks, 0); let op1_group = group(&mut rng); let op2_group = group(&mut rng); let wr_group = group(&mut rng); @@ -111,6 +117,8 @@ fn build_case( wr_group, op1_words: rows as u32, op2_words: (model::CORE * rows.div_ceil(tile_rows)) as u32, + k_groups: 1, + wr_col: 1, rs1_lo: rs1 as u32, rs1_hi: (rs1 >> 32) as u32, rs2_lo: rs2 as u32, @@ -123,7 +131,133 @@ fn build_case( bias, a, b, - writes: model::emit_writes(&result), + writes: model::emit_writes(&result, 0), + } +} + +fn bias_command(op1_group: u32) -> MatrixCmd { + let rs1 = model::encode_rs1(BIAS_BANK, 0, 0, 4); + MatrixCmd { + kind: 0, + op1_group, + op1_words: 4, + k_groups: 1, + rs1_lo: rs1 as u32, + rs1_hi: (rs1 >> 32) as u32, + ..MatrixCmd::default() + } +} + +fn bias_rows(values: &[i32]) -> Vec { + values.iter().flat_map(|value| value.to_le_bytes()).collect() +} + +// M=16, K=48 in one command: A and B have three groups, group g holding +// K block g (16 per group), at nonzero group bases. +fn three_k_groups(seed: u32, tile_rows: usize) -> MatrixCase { + let mut rng = Rng::new(seed, 11); + let bias: Vec = (0..model::CORE).map(|_| rng.value() as i32).collect(); + let a_values: Vec = (0..16 * 48).map(|_| rng.value()).collect(); + let b_values: Vec = (0..48 * model::CORE).map(|_| rng.value()).collect(); + // Group g of A is A[:, 16g..16g+16] row-major; group g of B is rows 16g... + let mut a = Vec::new(); + for group in 0..3 { + for row in 0..16 { + for k in 0..16 { + a.push(a_values[row * 48 + group * 16 + k] as u8); + } + } + } + let b: Vec = b_values.iter().map(|value| *value as u8).collect(); + let mut result = vec![bias.clone(); 16]; + for row in 0..16 { + for column in 0..model::CORE { + for k in 0..48 { + result[row][column] = result[row][column].wrapping_add( + a_values[row * 48 + k] as i32 * b_values[k * model::CORE + column] as i32, + ); + } + } + } + let rs1 = model::encode_rs1(1, 2, C_BANK, 16); + let rs2 = model::encode_rs2(16, true, true, 0); + MatrixCase { + commands: vec![ + bias_command(0), + MatrixCmd { + kind: 1, + rob_id: 1, + op1_group: 1, + op2_group: 2, + wr_group: 3, + op1_words: 16, + op2_words: (model::CORE * 16_usize.div_ceil(tile_rows)) as u32, + k_groups: 3, + wr_col: 1, + rs1_lo: rs1 as u32, + rs1_hi: (rs1 >> 32) as u32, + rs2_lo: rs2 as u32, + rs2_hi: (rs2 >> 32) as u32, + }, + ], + bias: bias_rows(&bias), + a: vec![a], + b: vec![b], + writes: model::emit_writes(&result, 0), + } +} + +// M=128, K=32 (two K groups), output_base 32: C lines 32..543 continue from +// C group 0 into group 1 on 512-line banks. +fn c_crosses_groups(seed: u32, tile_rows: usize) -> MatrixCase { + let mut rng = Rng::new(seed, 12); + let bias: Vec = (0..model::CORE).map(|_| rng.value() as i32).collect(); + let a_values: Vec = (0..128 * 32).map(|_| rng.value()).collect(); + let b_values: Vec = (0..32 * model::CORE).map(|_| rng.value()).collect(); + let mut a = Vec::new(); + for group in 0..2 { + for row in 0..128 { + for k in 0..16 { + a.push(a_values[row * 32 + group * 16 + k] as u8); + } + } + } + let b: Vec = b_values.iter().map(|value| *value as u8).collect(); + let mut result = vec![bias.clone(); 128]; + for row in 0..128 { + for column in 0..model::CORE { + for k in 0..32 { + result[row][column] = result[row][column].wrapping_add( + a_values[row * 32 + k] as i32 * b_values[k * model::CORE + column] as i32, + ); + } + } + } + let rs1 = model::encode_rs1(1, 2, C_BANK, 16); + let rs2 = model::encode_rs2(128, true, true, 32); + MatrixCase { + commands: vec![ + bias_command(0), + MatrixCmd { + kind: 1, + rob_id: 1, + op1_group: 0, + op2_group: 0, + wr_group: 0, + op1_words: 128, + op2_words: (model::CORE * 128_usize.div_ceil(tile_rows)) as u32, + k_groups: 2, + wr_col: 2, + rs1_lo: rs1 as u32, + rs1_hi: (rs1 >> 32) as u32, + rs2_lo: rs2 as u32, + rs2_hi: (rs2 >> 32) as u32, + }, + ], + bias: bias_rows(&bias), + a: vec![a], + b: vec![b], + writes: model::emit_writes(&result, 32), } } diff --git a/examples/components/balls/smatmul/verify/casegen/src/model.rs b/examples/components/balls/smatmul/verify/casegen/src/model.rs index 1e5557110..9b0594780 100644 --- a/examples/components/balls/smatmul/verify/casegen/src/model.rs +++ b/examples/components/balls/smatmul/verify/casegen/src/model.rs @@ -19,7 +19,8 @@ pub fn pack_b(src: &[i8]) -> Vec { src.iter().map(|value| *value as u8).collect() } -pub fn emit_writes(c: &[Vec]) -> Vec { +// `addr` is the line of C's virtual bank; it continues across groups. +pub fn emit_writes(c: &[Vec], output_base: usize) -> Vec { let mut writes = Vec::with_capacity(c.len() * RESULT_WORDS); for (row, values) in c.iter().enumerate() { assert_eq!(values.len(), CORE); @@ -30,7 +31,7 @@ pub fn emit_writes(c: &[Vec]) -> Vec { data[lane * 4..lane * 4 + 4].copy_from_slice(&value.to_le_bytes()); } writes.push(WriteExp { - addr: (row * RESULT_WORDS + word) as u32, + addr: (output_base + row * RESULT_WORDS + word) as u32, data, }); } @@ -49,10 +50,11 @@ pub fn encode_rs1(op1: u32, op2: u32, wr: u32, iter: u64) -> u64 { u64::from(op1) | (u64::from(op2) << 10) | (u64::from(wr) << 20) | (iter << 30) } -pub fn encode_rs2(rows: usize, first: bool, last: bool) -> u64 { - assert!(matches!(rows, 16 | 32 | 64)); +pub fn encode_rs2(rows: usize, first: bool, last: bool, output_base: usize) -> u64 { + assert!(matches!(rows, 16 | 32 | 64 | 128) && output_base < 64); rows as u64 | ((CORE as u64) << 12) | ((first as u64) << 24) | ((last as u64) << 25) + | ((output_base as u64) << 26) } diff --git a/examples/components/balls/smatmul/verify/src/common/smatmul_defs.svh b/examples/components/balls/smatmul/verify/src/common/smatmul_defs.svh index 516798c5e..ce049ef94 100644 --- a/examples/components/balls/smatmul/verify/src/common/smatmul_defs.svh +++ b/examples/components/balls/smatmul/verify/src/common/smatmul_defs.svh @@ -10,6 +10,8 @@ typedef struct { int unsigned op1_group; int unsigned op2_group; int unsigned wr_group; + int unsigned k_groups; + int unsigned wr_col; } matrix_cmd_dpi_t; `ifndef SMATMUL_TILE_ROWS diff --git a/examples/components/balls/smatmul/verify/src/common/smatmul_items.svh b/examples/components/balls/smatmul/verify/src/common/smatmul_items.svh index c7d21d1d1..7b1b87bc9 100644 --- a/examples/components/balls/smatmul/verify/src/common/smatmul_items.svh +++ b/examples/components/balls/smatmul/verify/src/common/smatmul_items.svh @@ -6,6 +6,7 @@ class smatmul_cmd_item extends bb_blink_cmd_item; bit [31:0] command_count; bit [31:0] op1_words; bit [31:0] op2_words; + bit [31:0] k_groups; bit [63:0] rs1; function new(string name = "smatmul_cmd_item"); @@ -24,6 +25,7 @@ class smatmul_cmd_item extends bb_blink_cmd_item; command_count = smatmul_case_num_commands(); op1_words = cmd.op1_words; op2_words = cmd.op2_words; + k_groups = cmd.k_groups; rs1 = {cmd.rs1_hi, cmd.rs1_lo}; // BB_BANK0/1/2 and BB_ITER fields of rs1; banks are not renamed at Ball level. @@ -48,12 +50,14 @@ class smatmul_cmd_item extends bb_blink_cmd_item; `uvm_fatal("CASE", "invalid bias command") end else begin funct = SMATMUL_OS_THREAD_FUNCT7[6:0]; - op1_col = 1; - op2_col = 1; - wr_col = 1; + // A and B group g hold K block g; C rows continue across wr_col groups. + op1_col = cmd.k_groups; + op2_col = cmd.k_groups; + wr_col = cmd.wr_col; + if (k_groups == 0 || cmd.wr_col == 0) `uvm_fatal("CASE", "SMATMUL_OS needs groups") if (op1_bank == op2_bank || op1_bank == wr_bank || op2_bank == wr_bank) `uvm_fatal("CASE", "SMATMUL_OS banks must be distinct") - if (rs2[63:26] != 0 || rs2[23:12] != 16 || iter != 16) + if (rs2[63:32] != 0 || rs2[23:12] != 16 || iter != 16) `uvm_fatal("CASE", "invalid SMATMUL_OS shape or reserved bits") end endfunction @@ -67,6 +71,7 @@ class smatmul_cmd_item extends bb_blink_cmd_item; command_count = rhs_.command_count; op1_words = rhs_.op1_words; op2_words = rhs_.op2_words; + k_groups = rhs_.k_groups; rs1 = rhs_.rs1; endfunction endclass diff --git a/examples/components/balls/smatmul/verify/src/env/smatmul_scoreboard.svh b/examples/components/balls/smatmul/verify/src/env/smatmul_scoreboard.svh index 4262d283a..f74028015 100644 --- a/examples/components/balls/smatmul/verify/src/env/smatmul_scoreboard.svh +++ b/examples/components/balls/smatmul/verify/src/env/smatmul_scoreboard.svh @@ -36,6 +36,7 @@ class smatmul_scoreboard extends uvm_scoreboard; matrix_cmd_dpi_t data_cmd; longint unsigned lo; longint unsigned hi; + int unsigned per_group; if (!$cast(got, item) || !$cast(clone, got.clone())) `uvm_fatal("SCB", "failed to clone smatmul stimulus") if (got.command_index != stim_count) `uvm_fatal("SCB", "commands arrived out of order") @@ -52,15 +53,20 @@ class smatmul_scoreboard extends uvm_scoreboard; end for (int block = 0; block < expected_commands - 1; block++) begin smatmul_case_cmd(block + 1, data_cmd); + // A and B words are group-major: K group g starts at g * per_group. + per_group = smatmul_case_a_words(block) / data_cmd.k_groups; for (int i = 0; i < smatmul_case_a_words(block); i++) begin lo = smatmul_case_a_word_lo(block, i); hi = smatmul_case_a_word_hi(block, i); - mem_model.load_word_g(data_cmd.rs1_lo[9:0], data_cmd.op1_group, i, {hi, lo}); + mem_model.load_word_g(data_cmd.rs1_lo[9:0], data_cmd.op1_group + i / per_group, + i % per_group, {hi, lo}); end + per_group = smatmul_case_b_words(block) / data_cmd.k_groups; for (int i = 0; i < smatmul_case_b_words(block); i++) begin lo = smatmul_case_b_word_lo(block, i); hi = smatmul_case_b_word_hi(block, i); - mem_model.load_word_g(data_cmd.rs1_lo[19:10], data_cmd.op2_group, i, {hi, lo}); + mem_model.load_word_g(data_cmd.rs1_lo[19:10], data_cmd.op2_group + i / per_group, + i % per_group, {hi, lo}); end end mem_model.arm(); @@ -77,8 +83,8 @@ class smatmul_scoreboard extends uvm_scoreboard; `uvm_fatal("SCB", "command count changed within a case") end - expected_reads[0] += got.op1_words; - expected_reads[1] += got.kind == 0 ? 0 : got.op2_words; + expected_reads[0] += got.kind == 0 ? got.op1_words : got.op1_words * got.k_groups; + expected_reads[1] += got.kind == 0 ? 0 : got.op2_words * got.k_groups; stim_q.push_back(clone); stim_count++; endfunction @@ -100,12 +106,12 @@ class smatmul_scoreboard extends uvm_scoreboard; item.addr >= 4) `uvm_fatal("READ", "invalid bias read") end else if (item.port == 0) begin - if (item.bank_id != stim.op1_bank || item.group_id != stim.op1_group || - item.addr >= stim.op1_words) + if (item.bank_id != stim.op1_bank || item.group_id < stim.op1_group || + item.group_id >= stim.op1_group + stim.k_groups || item.addr >= stim.op1_words) `uvm_fatal("READ", "A read outside command footprint") end else if (item.port == 1) begin - if (item.bank_id != stim.op2_bank || item.group_id != stim.op2_group || - item.addr >= stim.op2_words) + if (item.bank_id != stim.op2_bank || item.group_id < stim.op2_group || + item.group_id >= stim.op2_group + stim.k_groups || item.addr >= stim.op2_words) `uvm_fatal("READ", "B read outside command footprint") end else begin `uvm_fatal("READ", "unexpected read port") @@ -119,11 +125,14 @@ class smatmul_scoreboard extends uvm_scoreboard; stim = current_stim("WRITE"); if (stim.kind != 1 || stim.rs2[25] != 1) `uvm_fatal("WRITE", "write before the last accumulation block") - if (item.port != 0 || item.group_id != stim.wr_group || item.bank_id != stim.wr_bank || + if (item.port != 0 || item.bank_id != stim.wr_bank || item.mask != 16'hffff || item.rob_id != stim.rob_id) `uvm_fatal("WRITE", "write metadata violates SMatMul contract") for (int i = 0; i < expected_writes; i++) begin - if (!exp_seen[i] && item.addr === exp_addr[i][9:0] && item.data === exp_data[i]) begin + // exp_addr is C's virtual line: group wr_group + line / depth, addr line % depth. + if (!exp_seen[i] && item.addr === exp_addr[i] % (1 << `BB_BANK_ADDR_W) && + item.group_id === stim.wr_group + (exp_addr[i] >> `BB_BANK_ADDR_W) && + item.data === exp_data[i]) begin match_index = i; break; end diff --git a/examples/components/balls/smatmul/verify/src/tests/smatmul_test.svh b/examples/components/balls/smatmul/verify/src/tests/smatmul_test.svh index c293dbdf1..24fcba13c 100644 --- a/examples/components/balls/smatmul/verify/src/tests/smatmul_test.svh +++ b/examples/components/balls/smatmul/verify/src/tests/smatmul_test.svh @@ -18,7 +18,7 @@ class smatmul_ball_test extends uvm_test; task run_phase(uvm_phase phase); phase.raise_objection(this); - for (int unsigned index = 0; index < 11; index++) run_case(index); + for (int unsigned index = 0; index < 13; index++) run_case(index); phase.drop_objection(this); endtask diff --git a/examples/components/balls/smatmul/workloads/ctests/CMakeLists.txt b/examples/components/balls/smatmul/workloads/ctests/CMakeLists.txt index 3040cbea3..356a7c2c9 100644 --- a/examples/components/balls/smatmul/workloads/ctests/CMakeLists.txt +++ b/examples/components/balls/smatmul/workloads/ctests/CMakeLists.txt @@ -4,6 +4,8 @@ add_buckyball_ctest(smatmul_8x16_test.c) add_buckyball_ctest(smatmul_16x16_test.c) add_buckyball_ctest(smatmul_16x16x32_test.c) add_buckyball_ctest(smatmul_nondefault_banks_test.c) +add_buckyball_ctest(smatmul_k_groups_test.c) +add_buckyball_ctest(smatmul_c_groups_test.c) file( STRINGS "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/ballISA.h" F32_ISA diff --git a/examples/components/balls/smatmul/workloads/ctests/smatmul_c_groups_test.c b/examples/components/balls/smatmul/workloads/ctests/smatmul_c_groups_test.c new file mode 100644 index 000000000..657e899f4 --- /dev/null +++ b/examples/components/balls/smatmul/workloads/ctests/smatmul_c_groups_test.c @@ -0,0 +1,55 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include + +/* M=128, N=16, K=32 with output_base 32: C occupies lines 32..543, so its rows + * continue from group 0 into group 1 (line L is group L / BANK_LINES). A and B + * have two reduction groups. */ +enum { M = 128, N = 16, K = 32, BASE = 32 }; +static int8_t a[M * K] __attribute__((aligned(64))); +static int8_t b[K * N] __attribute__((aligned(64))); +static int32_t bias[N] __attribute__((aligned(64))); +static int32_t c[2][BANK_LINES * 4] __attribute__((aligned(64))); + +int main(void) { + for (int row = 0; row < M; ++row) + for (int k = 0; k < K; ++k) + a[row * K + k] = (row * 5 + k * 3) % 13 - 6; + for (int k = 0; k < K; ++k) + for (int col = 0; col < N; ++col) + b[k * N + col] = (k * 7 + col) % 11 - 5; + for (int col = 0; col < N; ++col) + bias[col] = col - 8; + + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 2); + bb_mem_alloc(2, 1, 2); + bb_mem_alloc(3, 1, 2); + bb_mvin((uintptr_t)bias, 0, 4, 1); + bb_mvin((uintptr_t)a, 1, M, 1); + bb_mvin_group((uintptr_t)b, 2, 0, 16, 1); + bb_mvin_group((uintptr_t)(b + 16 * N), 2, 1, 16, 1); + bb_smatmul_bias(0, 0); + bb_smatmul_os(1, 2, 3, M, N, 16, 1, 1, BASE); + bb_mvout_group((uintptr_t)c[0], 3, 0, BANK_LINES, 1); + bb_mvout_group((uintptr_t)c[1], 3, 1, BANK_LINES, 1); + + for (int row = 0; row < M; ++row) + for (int col = 0; col < N; ++col) { + int expected = bias[col]; + for (int k = 0; k < K; ++k) + expected += a[row * K + k] * b[k * N + col]; + int line = BASE + row * 4 + col / 4; + int actual = c[line / BANK_LINES][line % BANK_LINES * 4 + col % 4]; + if (actual != expected) { + printf("smatmul_c_groups FAIL row=%d col=%d expected=%d actual=%d\n", + row, col, expected, actual); + return 1; + } + } + printf("smatmul_c_groups PASS\n"); + return 0; +} diff --git a/examples/components/balls/smatmul/workloads/ctests/smatmul_k_groups_test.c b/examples/components/balls/smatmul/workloads/ctests/smatmul_k_groups_test.c new file mode 100644 index 000000000..e904c8207 --- /dev/null +++ b/examples/components/balls/smatmul/workloads/ctests/smatmul_k_groups_test.c @@ -0,0 +1,52 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* M=8, N=16, K=48 in one SMATMUL_OS: A and B have three groups, group g holds + * reduction block g (k = 16 per group). Row-major A[8][48] lands in a + * three-group bank as group g line r = A[r][16g..16g+15]. */ +enum { M = 8, N = 16, K = 48 }; +static int8_t a[M * K] __attribute__((aligned(64))); +static int8_t b[K * N] __attribute__((aligned(64))); +static int32_t bias[N] __attribute__((aligned(64))); +static int32_t actual[M * N] __attribute__((aligned(64))); + +int main(void) { + for (int row = 0; row < M; ++row) + for (int k = 0; k < K; ++k) + a[row * K + k] = (row * 5 + k * 3) % 13 - 6; + for (int k = 0; k < K; ++k) + for (int col = 0; col < N; ++col) + b[k * N + col] = (k * 7 + col) % 11 - 5; + for (int col = 0; col < N; ++col) + bias[col] = col * 3 - 20; + + bb_mem_alloc(0, 1, 1); + bb_mem_alloc(1, 1, 3); + bb_mem_alloc(2, 1, 3); + bb_mem_alloc(3, 1, 1); + bb_mvin((uintptr_t)bias, 0, 4, 1); + bb_mvin((uintptr_t)a, 1, M, 1); + bb_mvin_group((uintptr_t)b, 2, 0, 16, 1); + bb_mvin_group((uintptr_t)(b + 16 * N), 2, 1, 16, 1); + bb_mvin_group((uintptr_t)(b + 32 * N), 2, 2, 16, 1); + bb_smatmul_bias(0, 0); + bb_smatmul_os(1, 2, 3, M, N, 16, 1, 1, 0); + bb_mvout((uintptr_t)actual, 3, M * 4, 1); + + for (int row = 0; row < M; ++row) + for (int col = 0; col < N; ++col) { + int expected = bias[col]; + for (int k = 0; k < K; ++k) + expected += a[row * K + k] * b[k * N + col]; + if (actual[row * N + col] != expected) { + printf("smatmul_k_groups FAIL row=%d col=%d expected=%d actual=%d\n", + row, col, expected, actual[row * N + col]); + return 1; + } + } + printf("smatmul_k_groups PASS\n"); + return 0; +} diff --git a/examples/components/balls/toint8/emu/src/quant.rs b/examples/components/balls/toint8/emu/src/quant.rs index 0e099595b..13da7e92e 100644 --- a/examples/components/balls/toint8/emu/src/quant.rs +++ b/examples/components/balls/toint8/emu/src/quant.rs @@ -1,6 +1,6 @@ -use bebop_bemu::thread::banks::bank_lines; use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; use super::instruction::ExecContext; +use bebop_bemu::thread::banks::bank_lines; pub(crate) fn exec_f32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let src = rs1_b0(xs1); @@ -33,24 +33,24 @@ pub(crate) fn exec_f32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { // Group g of every operand is channel block g; each block uses the same rows. for channel_group in 0..groups { - let ps = pbank_group(ctx, src, channel_group); - let pd = pbank_group(ctx, dst, channel_group); - let (input, output) = ctx.banks.read_write(ps, pd); - for output_row in 0..iter / 4 { - let mut packed = [0u8; 16]; - for group in 0..4 { - let input_row = output_row * 4 + group; - let base = input_row * 16; - for lane in 0..4 { - let offset = base + lane * 4; - let bits = u32::from_le_bytes(input[offset..offset + 4].try_into().unwrap()); - packed[group * 4 + lane] = - super::model::quantize_f32(f32::from_bits(bits), scale, false) as u8; + let ps = pbank_group(ctx, src, channel_group); + let pd = pbank_group(ctx, dst, channel_group); + let (input, output) = ctx.banks.read_write(ps, pd); + for output_row in 0..iter / 4 { + let mut packed = [0u8; 16]; + for group in 0..4 { + let input_row = output_row * 4 + group; + let base = input_row * 16; + for lane in 0..4 { + let offset = base + lane * 4; + let bits = u32::from_le_bytes(input[offset..offset + 4].try_into().unwrap()); + packed[group * 4 + lane] = + super::model::quantize_f32(f32::from_bits(bits), scale, false) as u8; + } } + let base = output_row * 16; + output[base..base + 16].copy_from_slice(&packed); } - let base = output_row * 16; - output[base..base + 16].copy_from_slice(&packed); - } } 0 } @@ -98,41 +98,43 @@ pub(crate) fn exec_i32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { // Group g of every operand is channel block g; each block uses the same rows. for channel_group in 0..groups { - let pscale = pbank_group(ctx, scale_bank, channel_group); - let mut scales = [0.0f32; 16]; - for group in 0..4 { - let base = group * 16; - for lane in 0..4 { - let offset = base + lane * 4; - scales[group * 4 + lane] = f32::from_bits(u32::from_le_bytes( - ctx.banks[pscale][offset..offset + 4].try_into().unwrap(), - )); - if !scales[group * 4 + lane].is_finite() || scales[group * 4 + lane] <= 0.0 { - panic!("quant_i32_to_i8: scales must be finite and positive"); - } - } - } - - let ps = pbank_group(ctx, src, channel_group); - let pd = pbank_group(ctx, dst, channel_group); - let (input, output) = ctx.banks.read_write(ps, pd); - for output_row in 0..iter / 4 { - let mut packed = [0u8; 16]; + let pscale = pbank_group(ctx, scale_bank, channel_group); + let mut scales = [0.0f32; 16]; for group in 0..4 { - let input_row = input_base + output_row * 4 + group; - let base = input_row * 16; + let base = group * 16; for lane in 0..4 { let offset = base + lane * 4; - let value = i32::from_le_bytes(input[offset..offset + 4].try_into().unwrap()); - let channel = group * 4 + lane; - packed[channel] = super::model::quantize_i32(value, scales[channel], relu) as u8; + scales[group * 4 + lane] = f32::from_bits(u32::from_le_bytes( + ctx.banks[pscale][offset..offset + 4].try_into().unwrap(), + )); + if !scales[group * 4 + lane].is_finite() || scales[group * 4 + lane] <= 0.0 { + panic!("quant_i32_to_i8: scales must be finite and positive"); + } } } - let row = - output_base + (output_row / output_width) * output_stride + output_row % output_width; - let base = row * 16; - output[base..base + 16].copy_from_slice(&packed); - } + + let ps = pbank_group(ctx, src, channel_group); + let pd = pbank_group(ctx, dst, channel_group); + let (input, output) = ctx.banks.read_write(ps, pd); + for output_row in 0..iter / 4 { + let mut packed = [0u8; 16]; + for group in 0..4 { + let input_row = input_base + output_row * 4 + group; + let base = input_row * 16; + for lane in 0..4 { + let offset = base + lane * 4; + let value = i32::from_le_bytes(input[offset..offset + 4].try_into().unwrap()); + let channel = group * 4 + lane; + packed[channel] = + super::model::quantize_i32(value, scales[channel], relu) as u8; + } + } + let row = output_base + + (output_row / output_width) * output_stride + + output_row % output_width; + let base = row * 16; + output[base..base + 16].copy_from_slice(&packed); + } } 0 } diff --git a/examples/components/balls/toint8/stack/src/Lowering.cpp b/examples/components/balls/toint8/stack/src/Lowering.cpp index cad9db0fc..185946fe6 100644 --- a/examples/components/balls/toint8/stack/src/Lowering.cpp +++ b/examples/components/balls/toint8/stack/src/Lowering.cpp @@ -4,10 +4,12 @@ // four; the destination receives rows / 4 rows of sixteen i8, // clamp(round_half_even(x * multiplier), -128, 127). Argument 0 is the f32 // multiplier. -// int32_to_int8: source 0 holds `rows` rows of sixteen i32 (four bank rows -// each), source 1 the sixteen f32 scales of the columns (four bank rows); the +// int32_to_int8: source 0 holds `rows` rows of sixteen i32 (four lines each), +// source 1 the sixteen f32 scales of the columns (four lines); the // destination receives `rows` rows of sixteen i8, // clamp(round_half_even(x * scale), -128, 127), with `relu` at least zero. +// Both kinds work on every group: group g is channel block g, and the banks +// have the same number of groups. // //===----------------------------------------------------------------------===// @@ -54,14 +56,15 @@ class Int32ToInt8Lowering : public OpRewritePattern { auto rows = cast(op->getAttr("rows")).getInt(); bool relu = cast(op->getAttr("relu")).getValue(); int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; - if (rows <= 0 || rows > 127 || rows * 4 > depth) - return op.emitOpError("INT32-to-INT8 rows exceed the ISA range"); - // The output rows start at the base of the destination: width one, height - // `rows`, stride one. + if (rows <= 0 || rows * 4 > depth) + return op.emitOpError("INT32-to-INT8 rows exceed the bank depth"); + // The output rows start at the base of the destination, packed; the + // seven-bit height takes 128 rows as two columns of 64. + int64_t width = rows > 127 ? 2 : 1; uint64_t rs1 = mlir::buddy::bankFields(op.getSources(), op.getDestination(), false, rows * 4); - uint64_t rs2 = - uint64_t(relu) | (1ULL << 8) | (uint64_t(rows) << 15) | (1ULL << 22); + uint64_t rs2 = uint64_t(relu) | (uint64_t(width) << 8) | + (uint64_t(rows / width) << 15) | (uint64_t(width) << 22); mlir::buddy::emitInstruction( rewriter, op.getLoc(), buckyball_target::getBuckyballFunct7("QUANT_I32_TO_I8"), diff --git a/examples/components/balls/toint8/verify/README.md b/examples/components/balls/toint8/verify/README.md index b68e5d7e1..b3852149b 100644 --- a/examples/components/balls/toint8/verify/README.md +++ b/examples/components/balls/toint8/verify/README.md @@ -25,6 +25,8 @@ The test names are: - `toint8_rounding_test`: ties-to-even quantization boundaries - `toint8_rows_test`: two-row scan and packed output traversal - `toint8_scale_rows_test`: maximum value is in the second row +- case 8: QUANT_F32_TO_I8 over two channel groups (group g of input/output = channel block g) +- case 9: QUANT_I32_TO_I8 over three channel groups with per-group scales, input_base 3, and a 2x2 output core Each case: reset, preload src words into `mem_model` (INT8 uses `group_id` 0..3), drive one command, wait for `scb.done()` or timeout at 400 cycles. Scoreboard diff --git a/examples/components/balls/toint8/verify/casegen/src/casegen.rs b/examples/components/balls/toint8/verify/casegen/src/casegen.rs index 8bbbe91ee..74fde3292 100644 --- a/examples/components/balls/toint8/verify/casegen/src/casegen.rs +++ b/examples/components/balls/toint8/verify/casegen/src/casegen.rs @@ -6,9 +6,10 @@ pub const BANK_NUM: u32 = 24; pub const BANK_ENTRIES: u32 = 64; pub const SCALE_ROWS: usize = 4; pub const LANES: usize = 4; -pub const MAX_SRC: usize = 16; -pub const MAX_SCALE: usize = 4; -pub const MAX_DST: usize = 4; +// Words are stored group-major: all rows of group 0, then group 1, ... +pub const MAX_SRC: usize = 48; +pub const MAX_SCALE: usize = 12; +pub const MAX_DST: usize = 12; #[repr(C)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -78,6 +79,8 @@ pub fn gen_case(index: u32) -> ToInt8Case { 5 => i32_relu_tile(), 6 => i32_signed(), 7 => i32_offset(), + 8 => f32_two_groups(), + 9 => i32_three_groups(), _ => panic!("toint8: unsupported directed case {index}"), } } @@ -166,8 +169,8 @@ pub(crate) fn validate( if wr_bank >= BANK_NUM { panic!("ToInt8Ball output bank is invalid"); } - if op1_col != 1 || wr_col != 1 { - panic!("ToInt8Ball input and output must each occupy one bank"); + if op1_col == 0 || op1_col > BANK_NUM || wr_col != op1_col { + panic!("ToInt8Ball input and output must have the same channel groups"); } if op1_bank == wr_bank { panic!("ToInt8Ball input and output banks must differ"); @@ -186,8 +189,8 @@ pub(crate) fn validate( if op2_bank >= BANK_NUM { panic!("QUANT_I32_TO_I8 scale bank is invalid"); } - if op2_col != 1 { - panic!("QUANT_I32_TO_I8 scale must occupy one bank"); + if op2_col != op1_col { + panic!("QUANT_I32_TO_I8 scale must have one group per channel group"); } if op1_bank == op2_bank { panic!("QUANT_I32_TO_I8 input and scale banks must differ"); @@ -289,6 +292,7 @@ fn i32_addrs(iter: u32, out_base: u32, width: u32, stride: u32) -> Vec { fn finish( kind: u32, groups: [u32; 3], + cols: u32, iter: u32, op1_bank: u32, op2_bank: u32, @@ -302,23 +306,23 @@ fn finish( dst: Vec, dst_addr: Vec, ) -> ToInt8Case { - let op1_col = 1; - let wr_col = 1; + let op1_col = cols; + let wr_col = cols; validate( kind, op1_bank, op2_bank, wr_bank, op1_col, op2_col, wr_col, iter, rs2, ); - if src.len() != iter as usize { - panic!("toint8: src rows {} != iter {iter}", src.len()); + if src.len() != (iter * cols) as usize { + panic!("toint8: src rows {} != iter {iter} * groups {cols}", src.len()); } - if dst.len() != (iter / 4) as usize || dst_addr.len() != dst.len() { - panic!("toint8: dst rows do not match iter/4"); + if dst.len() != (iter / 4 * cols) as usize || dst_addr.len() != dst.len() { + panic!("toint8: dst rows do not match iter/4 per group"); } if kind == KIND_F32 { if !scale.is_empty() { panic!("QUANT_F32_TO_I8 must not emit scale rows"); } - } else if scale.len() != SCALE_ROWS { - panic!("QUANT_I32_TO_I8 must emit four scale rows"); + } else if scale.len() != SCALE_ROWS * cols as usize { + panic!("QUANT_I32_TO_I8 must emit four scale rows per group"); } let (rs2_lo, rs2_hi) = split_rs(rs2); ToInt8Case { @@ -358,6 +362,7 @@ fn f32_case(groups: [u32; 3], rob_id: u32, scale: f32, values: &[f32]) -> ToInt8 finish( KIND_F32, groups, + 1, iter, 0, 0, @@ -397,6 +402,7 @@ fn i32_case( finish( KIND_I32, groups, + 1, iter, op1_bank, op2_bank, @@ -487,3 +493,75 @@ fn i32_offset() -> ToInt8Case { } i32_case([2, 3, 1], 9, 0, 1, 2, true, 1, 2, 1, 2, 4, &values, &[0.5; 16]) } + +// Two channel groups; each group has two pixels (eight FP32 rows) quantized +// with the same rows into group g of the output. +fn f32_two_groups() -> ToInt8Case { + let mut values = [0.0f32; 64]; + for (i, value) in values.iter_mut().enumerate() { + *value = ((i * 7) % 50) as f32 - 25.0; + } + let bits: Vec = values.iter().map(|v| v.to_bits()).collect(); + let src = pack_u32s(&bits); + let dst = f32_dst(&src, 2.0); + // Output rows 0, 1 of group 0, then rows 0, 1 of group 1. + finish( + KIND_F32, + [1, 0, 3], + 2, + 8, + 0, + 0, + 1, + 0, + 10, + u64::from(2.0f32.to_bits()), + 0, + src, + Vec::new(), + dst, + vec![0, 1, 0, 1], + ) +} + +// Three channel groups, each with its own scales: input rows start at 3, +// output core base 1, width 2, height 2, stride 3 -> rows 1, 2, 4, 5 in +// every group. +fn i32_three_groups() -> ToInt8Case { + let mut values = [0u32; 3 * 64]; + for (i, value) in values.iter_mut().enumerate() { + *value = (((i * 11) % 160) as i32 - 80) as u32; + } + let src = pack_u32s(&values); + let mut scale_bits = [0u32; 3 * 16]; + for (i, bits) in scale_bits.iter_mut().enumerate() { + *bits = (if (i / 16 + i) % 2 == 1 { 2.0f32 } else { 0.5f32 }).to_bits(); + } + let scale = pack_u32s(&scale_bits); + let mut dst = Vec::new(); + for group in 0..3 { + dst.extend(i32_dst( + &src[group * 16..group * 16 + 16], + &scale[group * 4..group * 4 + 4], + false, + )); + } + let rs2 = encode_i32_rs2(false, 1, 2, 2, 3, 3); + finish( + KIND_I32, + [2, 0, 1], + 3, + 16, + 0, + 1, + 2, + 3, + 11, + rs2, + 3, + src, + scale, + dst, + vec![1, 2, 4, 5, 1, 2, 4, 5, 1, 2, 4, 5], + ) +} diff --git a/examples/components/balls/toint8/verify/src/common/toint8_defs.svh b/examples/components/balls/toint8/verify/src/common/toint8_defs.svh index ee65bc7e1..2f9f9eb6a 100644 --- a/examples/components/balls/toint8/verify/src/common/toint8_defs.svh +++ b/examples/components/balls/toint8/verify/src/common/toint8_defs.svh @@ -40,8 +40,9 @@ localparam int QUANT_F32_TO_I8_THREAD_FUNCT7 = `QUANT_F32_TO_I8_FUNCT7; localparam int QUANT_I32_TO_I8_THREAD_FUNCT7 = `QUANT_I32_TO_I8_FUNCT7; localparam int TOINT8_KIND_F32 = 0; localparam int TOINT8_KIND_I32 = 1; -localparam int TOINT8_MAX_SRC_WORDS = 16; -localparam int TOINT8_MAX_SCALE_WORDS = 4; -localparam int TOINT8_MAX_DST_WORDS = 4; -localparam int TOINT8_NUM_CASES = 8; +// Words are group-major: rows of channel group 0, then group 1, ... +localparam int TOINT8_MAX_SRC_WORDS = 48; +localparam int TOINT8_MAX_SCALE_WORDS = 12; +localparam int TOINT8_MAX_DST_WORDS = 12; +localparam int TOINT8_NUM_CASES = 10; localparam int TOINT8_TIMEOUT_CYCLES = 4000; diff --git a/examples/components/balls/toint8/verify/src/common/toint8_items.svh b/examples/components/balls/toint8/verify/src/common/toint8_items.svh index 88d4d7470..f74c67749 100644 --- a/examples/components/balls/toint8/verify/src/common/toint8_items.svh +++ b/examples/components/balls/toint8/verify/src/common/toint8_items.svh @@ -61,23 +61,27 @@ class toint8_cmd_item extends bb_blink_cmd_item; if (num_scale_words != 0) `uvm_fatal("CASE", "QUANT_F32_TO_I8 must not emit scale rows") if (input_base != 0) `uvm_fatal("CASE", "QUANT_F32_TO_I8 input_base must be 0") end else begin - if (op2_col != 1) `uvm_fatal("CASE", "QUANT_I32_TO_I8 scale must occupy one bank") + if (op2_col != op1_col) + `uvm_fatal("CASE", "QUANT_I32_TO_I8 scale must have one group per channel group") if (op1_bank == op2_bank) `uvm_fatal("CASE", "QUANT_I32_TO_I8 input and scale banks must differ") if (op2_bank == wr_bank) `uvm_fatal("CASE", "QUANT_I32_TO_I8 scale and output banks must differ") if (rs2[63:35] != 0) `uvm_fatal("CASE", "QUANT_I32_TO_I8 reserves rs2[63:35]") - if (num_scale_words != 4) `uvm_fatal("CASE", "QUANT_I32_TO_I8 must emit four scale rows") + if (num_scale_words != 4 * op1_col) + `uvm_fatal("CASE", "QUANT_I32_TO_I8 must emit four scale rows per group") end if (iter == 0 || iter[1:0] != 2'b00) `uvm_fatal("CASE", "ToInt8Ball iter must be a positive multiple of four") - if (op1_col != 1 || wr_col != 1) - `uvm_fatal("CASE", "ToInt8Ball input and output must each occupy one bank") + if (op1_col == 0 || wr_col != op1_col) + `uvm_fatal("CASE", "ToInt8Ball input and output must have the same channel groups") if (op1_bank == wr_bank) `uvm_fatal("CASE", "ToInt8Ball input and output banks must differ") - if (num_src_words != iter) - `uvm_fatal("CASE", $sformatf("num_src_words %0d != iter %0d", num_src_words, iter)) - if (num_dst_words != (iter >> 2)) `uvm_fatal("CASE", "num_dst_words does not match iter/4") + if (num_src_words != iter * op1_col) + `uvm_fatal("CASE", $sformatf( + "num_src_words %0d != iter %0d * groups %0d", num_src_words, iter, op1_col)) + if (num_dst_words != (iter >> 2) * op1_col) + `uvm_fatal("CASE", "num_dst_words does not match iter/4 per group") if (num_src_words == 0 || num_src_words > TOINT8_MAX_SRC_WORDS) `uvm_fatal("CASE", $sformatf("num_src_words out of range: %0d", num_src_words)) if (num_dst_words == 0 || num_dst_words > TOINT8_MAX_DST_WORDS) diff --git a/examples/components/balls/toint8/verify/src/cov/toint8_cov.svh b/examples/components/balls/toint8/verify/src/cov/toint8_cov.svh index 94d89c90a..db6bb5f8c 100644 --- a/examples/components/balls/toint8/verify/src/cov/toint8_cov.svh +++ b/examples/components/balls/toint8/verify/src/cov/toint8_cov.svh @@ -4,12 +4,14 @@ class toint8_cov extends uvm_component; uvm_analysis_imp_cmd #(bb_blink_cmd_item, toint8_cov) cmd_imp; bit [6:0] cur_funct; + int unsigned cur_groups; covergroup cmd_cg; coverpoint cur_funct { bins f32 = {QUANT_F32_TO_I8_THREAD_FUNCT7[6:0]}; bins i32 = {QUANT_I32_TO_I8_THREAD_FUNCT7[6:0]}; } + coverpoint cur_groups {bins single = {1}; bins multi = {[2 : $]};} endgroup int cmd_count; @@ -21,7 +23,8 @@ class toint8_cov extends uvm_component; endfunction function void write_cmd(bb_blink_cmd_item item); - cur_funct = item.funct; + cur_funct = item.funct; + cur_groups = item.op1_col; cmd_count++; cmd_cg.sample(); endfunction diff --git a/examples/components/balls/toint8/verify/src/env/toint8_scoreboard.svh b/examples/components/balls/toint8/verify/src/env/toint8_scoreboard.svh index 57ed1d693..0578d5c05 100644 --- a/examples/components/balls/toint8/verify/src/env/toint8_scoreboard.svh +++ b/examples/components/balls/toint8/verify/src/env/toint8_scoreboard.svh @@ -46,12 +46,14 @@ class toint8_scoreboard extends uvm_scoreboard; `uvm_fatal("SCB", "mem_model handle not set") end mem_model.clear_mem(); + // Group g of every operand is channel block g with the same rows. for (i = 0; i < clone.num_src_words; i++) begin - mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group), int'(clone.input_base) + i, - clone.src_words[i]); + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group) + i / int'(clone.iter), + int'(clone.input_base) + i % int'(clone.iter), clone.src_words[i]); end for (i = 0; i < clone.num_scale_words; i++) begin - mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group), i, clone.scale_words[i]); + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group) + i / 4, i % 4, + clone.scale_words[i]); end mem_model.arm(); @@ -78,6 +80,9 @@ class toint8_scoreboard extends uvm_scoreboard; function void write_read(bb_blink_read_item item); toint8_cmd_item stim; int unsigned scale_reads; + int unsigned group_reads; + int unsigned group; + int unsigned index; int unsigned expect_bank; int unsigned expect_group; int unsigned expect_addr; @@ -88,15 +93,19 @@ class toint8_scoreboard extends uvm_scoreboard; if (item.rob_id !== stim.rob_id) `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - scale_reads = stim.num_scale_words; - if (read_count < scale_reads) begin + // Each group reads its scale rows (I32 only), then its input rows. + scale_reads = stim.num_scale_words / stim.op1_col; + group_reads = scale_reads + stim.iter; + group = read_count / group_reads; + index = read_count % group_reads; + if (index < scale_reads) begin expect_bank = stim.op2_bank; - expect_group = stim.op2_group; - expect_addr = read_count; + expect_group = stim.op2_group + group; + expect_addr = index; end else begin expect_bank = stim.op1_bank; - expect_group = stim.op1_group; - expect_addr = stim.input_base + (read_count - scale_reads); + expect_group = stim.op1_group + group; + expect_addr = stim.input_base + (index - scale_reads); end if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) @@ -117,8 +126,11 @@ class toint8_scoreboard extends uvm_scoreboard; `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) if (item.rob_id !== stim.rob_id) `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) - if (item.group_id !== stim.wr_group) - `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + if (item.group_id !== stim.wr_group + write_count / (stim.iter >> 2)) + `uvm_fatal("WRITE", $sformatf( + "group mismatch: got %0d exp %0d", + item.group_id, + stim.wr_group + write_count / (stim.iter >> 2) )) if (item.mask !== 16'hFFFF) `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) diff --git a/examples/components/balls/toint8/workloads/ctests/CMakeLists.txt b/examples/components/balls/toint8/workloads/ctests/CMakeLists.txt index ddb7a1f5c..f6b44e859 100644 --- a/examples/components/balls/toint8/workloads/ctests/CMakeLists.txt +++ b/examples/components/balls/toint8/workloads/ctests/CMakeLists.txt @@ -4,3 +4,5 @@ add_buckyball_ctest(toint8_i8_rounding_test.c) add_buckyball_ctest(toint8_i8_rows_test.c) add_buckyball_ctest(toint8_i8_scale_rows_test.c) add_buckyball_ctest(toint8_i8_signed_test.c) +add_buckyball_ctest(quant_i32_groups_test.c) +add_buckyball_ctest(quant_f32_groups_test.c) diff --git a/examples/components/balls/toint8/workloads/ctests/quant_f32_groups_test.c b/examples/components/balls/toint8/workloads/ctests/quant_f32_groups_test.c new file mode 100644 index 000000000..b0fd0b952 --- /dev/null +++ b/examples/components/balls/toint8/workloads/ctests/quant_f32_groups_test.c @@ -0,0 +1,40 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* QUANT_F32_TO_I8 over three channel groups: input and output group g are + * channel block g; every group uses the same rows. */ +enum { GROUPS = 3, PIXELS = 2, ROWS = PIXELS * 4 }; + +static float input[GROUPS][PIXELS * 16] __attribute__((aligned(64))); +static int8_t actual[GROUPS][PIXELS * 16] __attribute__((aligned(64))); + +int main(void) { + for (int group = 0; group < GROUPS; ++group) + for (int i = 0; i < PIXELS * 16; ++i) + input[group][i] = (float)((group * 29 + i * 7) % 100 - 50); + + bb_mem_alloc(0, 1, GROUPS); + bb_mem_alloc(1, 1, GROUPS); + for (int group = 0; group < GROUPS; ++group) + bb_mvin_group((uintptr_t)input[group], 0, group, ROWS, 1); + bb_quant_f32_to_i8(0, 1, ROWS, 2.0f); + for (int group = 0; group < GROUPS; ++group) + bb_mvout_group((uintptr_t)actual[group], 1, group, PIXELS, 1); + + for (int group = 0; group < GROUPS; ++group) + for (int i = 0; i < PIXELS * 16; ++i) { + int value = (int)input[group][i] * 2; + int expected = value > 127 ? 127 : value < -128 ? -128 : value; + if (actual[group][i] != expected) { + printf("quant_f32_groups FAIL group=%d index=%d expected=%d " + "actual=%d\n", + group, i, expected, actual[group][i]); + return 1; + } + } + printf("quant_f32_groups PASS\n"); + return 0; +} diff --git a/examples/components/balls/toint8/workloads/ctests/quant_i32_groups_test.c b/examples/components/balls/toint8/workloads/ctests/quant_i32_groups_test.c new file mode 100644 index 000000000..cd47f9ace --- /dev/null +++ b/examples/components/balls/toint8/workloads/ctests/quant_i32_groups_test.c @@ -0,0 +1,55 @@ +#include "buckyball.h" +#include +#include +#include +#include + +/* QUANT_I32_TO_I8 over two channel groups: group g of the input, scale, and + * output banks is channel block g; every group uses the same output core. */ +enum { GROUPS = 2, PIXELS = 4, ROWS = PIXELS * 4, OUT_ROWS = 6 }; + +static int32_t input[GROUPS][PIXELS * 16] __attribute__((aligned(64))); +static float scales[GROUPS][16] __attribute__((aligned(64))); +static int8_t actual[GROUPS][OUT_ROWS * 16] __attribute__((aligned(64))); + +int main(void) { + for (int group = 0; group < GROUPS; ++group) { + for (int channel = 0; channel < 16; ++channel) + scales[group][channel] = (group + channel) % 2 ? 2.0f : 1.0f; + for (int i = 0; i < PIXELS * 16; ++i) + input[group][i] = (group * 37 + i * 11) % 160 - 80; + } + + bb_mem_alloc(0, 1, GROUPS); + bb_mem_alloc(1, 1, GROUPS); + bb_mem_alloc(2, 1, GROUPS); + for (int group = 0; group < GROUPS; ++group) { + bb_mvin_group((uintptr_t)input[group], 0, group, ROWS, 1); + bb_mvin_group((uintptr_t)scales[group], 1, group, 4, 1); + } + // Output core: base 1, width 2, height 2, stride 3 -> rows 1, 2, 4, 5. + bb_quant_i32_to_i8(0, 1, 2, ROWS, 0, 1, 2, 2, 3, 0); + for (int group = 0; group < GROUPS; ++group) + bb_mvout_group((uintptr_t)actual[group], 2, group, OUT_ROWS, 1); + + for (int group = 0; group < GROUPS; ++group) + for (int row = 0; row < OUT_ROWS; ++row) + for (int channel = 0; channel < 16; ++channel) { + int expected = 0; + if (row % 3) { + int pixel = row / 3 * 2 + row % 3 - 1; + int value = + input[group][pixel * 16 + channel] * (int)scales[group][channel]; + expected = value > 127 ? 127 : value < -128 ? -128 : value; + } + if (actual[group][row * 16 + channel] != expected) { + printf("quant_i32_groups FAIL group=%d row=%d channel=%d " + "expected=%d actual=%d\n", + group, row, channel, expected, + actual[group][row * 16 + channel]); + return 1; + } + } + printf("quant_i32_groups PASS\n"); + return 0; +} diff --git a/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml index 05506cbd5..2f7e09296 100644 --- a/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml +++ b/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml @@ -61,6 +61,8 @@ tests = [ "pebble-pebble-ctest-smatmul_16x16_test-baremetal", "pebble-pebble-ctest-smatmul_16x16x32_test-baremetal", "pebble-pebble-ctest-smatmul_nondefault_banks_test-baremetal", + "pebble-pebble-ctest-smatmul_k_groups_test-baremetal", + "pebble-pebble-ctest-smatmul_c_groups_test-baremetal", "pebble-pebble-mlirtest-ball_smatmul_bias_accumulate-baremetal", "pebble-pebble-ctest-quant_f32_to_i8_test-baremetal", "pebble-pebble-ctest-quant_i32_to_i8_test-baremetal", @@ -68,6 +70,8 @@ tests = [ "pebble-pebble-ctest-toint8_i8_rows_test-baremetal", "pebble-pebble-ctest-toint8_i8_scale_rows_test-baremetal", "pebble-pebble-ctest-toint8_i8_signed_test-baremetal", + "pebble-pebble-ctest-quant_i32_groups_test-baremetal", + "pebble-pebble-ctest-quant_f32_groups_test-baremetal", "pebble-pebble-mlirtest-ball_quant_f32_to_i8-baremetal", "pebble-pebble-mlirtest-ball_quant_i32_to_i8-baremetal", "pebble-pebble-ctest-im2col_window_test-baremetal", @@ -85,6 +89,7 @@ tests = [ "pebble-pebble-ctest-int2fp_channel_lanes_test-baremetal", "pebble-pebble-ctest-int2fp_channel_base_test-baremetal", "pebble-pebble-ctest-int2fp_channel_two_rows_test-baremetal", + "pebble-pebble-ctest-int2fp_groups_test-baremetal", "pebble-pebble-mlirtest-ball_int2fp_4x4-baremetal", "pebble-pebble-mlirtest-ball_int2fp_relu_4x4-baremetal", "pebble-pebble-ctest-lut_uniform_lookup_test-baremetal", @@ -92,6 +97,7 @@ tests = [ "pebble-pebble-mlirtest-ball_lut_identity-baremetal", "pebble-pebble-ctest-maxpool_window_test-baremetal", "pebble-pebble-ctest-maxpool_offset_test-baremetal", + "pebble-pebble-ctest-maxpool_groups_test-baremetal", "pebble-pebble-mlirtest-ball_maxpool_6x6_s2-baremetal", "pebble-pebble-ctest-int8add_scaled_test-baremetal", "pebble-pebble-ctest-int8add_relu_test-baremetal", diff --git a/examples/designs/pebble/sims/device/regression/verilator/workloads-bare.toml b/examples/designs/pebble/sims/device/regression/verilator/workloads-bare.toml index c50847150..a33ba804f 100644 --- a/examples/designs/pebble/sims/device/regression/verilator/workloads-bare.toml +++ b/examples/designs/pebble/sims/device/regression/verilator/workloads-bare.toml @@ -37,6 +37,7 @@ tests = [ "pebble-pebble-ctest-smatmul_16x16_test-baremetal", "pebble-pebble-ctest-smatmul_16x16x32_test-baremetal", "pebble-pebble-ctest-smatmul_nondefault_banks_test-baremetal", + "pebble-pebble-ctest-smatmul_k_groups_test-baremetal", "pebble-pebble-ctest-smatmul_mvout_bank_reuse_test-baremetal", "pebble-pebble-mlirtest-ball_smatmul_bias_accumulate-baremetal", "pebble-pebble-ctest-quant_f32_to_i8_test-baremetal", @@ -45,6 +46,8 @@ tests = [ "pebble-pebble-ctest-toint8_i8_rows_test-baremetal", "pebble-pebble-ctest-toint8_i8_scale_rows_test-baremetal", "pebble-pebble-ctest-toint8_i8_signed_test-baremetal", + "pebble-pebble-ctest-quant_i32_groups_test-baremetal", + "pebble-pebble-ctest-quant_f32_groups_test-baremetal", "pebble-pebble-mlirtest-ball_quant_f32_to_i8-baremetal", "pebble-pebble-mlirtest-ball_quant_i32_to_i8-baremetal", "pebble-pebble-ctest-im2col_window_test-baremetal", @@ -62,6 +65,7 @@ tests = [ "pebble-pebble-ctest-int2fp_channel_lanes_test-baremetal", "pebble-pebble-ctest-int2fp_channel_base_test-baremetal", "pebble-pebble-ctest-int2fp_channel_two_rows_test-baremetal", + "pebble-pebble-ctest-int2fp_groups_test-baremetal", "pebble-pebble-mlirtest-ball_int2fp_4x4-baremetal", "pebble-pebble-mlirtest-ball_int2fp_relu_4x4-baremetal", "pebble-pebble-ctest-lut_uniform_lookup_test-baremetal", @@ -69,6 +73,7 @@ tests = [ "pebble-pebble-mlirtest-ball_lut_identity-baremetal", "pebble-pebble-ctest-maxpool_window_test-baremetal", "pebble-pebble-ctest-maxpool_offset_test-baremetal", + "pebble-pebble-ctest-maxpool_groups_test-baremetal", "pebble-pebble-mlirtest-ball_maxpool_6x6_s2-baremetal", "pebble-pebble-ctest-int8add_scaled_test-baremetal", "pebble-pebble-ctest-int8add_relu_test-baremetal", diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/dual_conv_add_int2fp_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/dual_conv_add_int2fp_test.c index 077463b26..1c70757cf 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/dual_conv_add_int2fp_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/dual_conv_add_int2fp_test.c @@ -15,7 +15,9 @@ enum { K = 3, OUT = 2, WINDOWS = 4, - PACKED_K = 16, + M = 8, + // K is (kernel row, kernel column, channel); only channel 0 is nonzero. + PACKED_K = K * K * CORE, RESULT_ROWS = CORE * 2 }; static int8_t images[2][IN * IN * CORE] __attribute__((aligned(64))); @@ -34,15 +36,16 @@ static int32_t expected(int row, int column) { for (int ky = 0; ky < K; ++ky) for (int kx = 0; kx < K; ++kx) sum += image_value(branch, row / OUT + ky, row % OUT + kx) * - weights[branch][(ky * K + kx) * CORE + column]; + weights[branch][(ky * K + kx) * CORE * CORE + column]; return sum; } static void run_branch(int branch, int out_bank) { bb_mvin((uintptr_t)images[branch], 0, IN * IN, 1); - bb_im2col(0, 1, IN, K, 1, 0, 0, 0, 0, 0, 0, WINDOWS); + // One output group holds all nine K tiles in SMatMul's 8-row M tiles. + bb_im2col(0, 1, IN, K, 1, 0, 0, 0, 0, 0, WINDOWS, M); bb_mvin((uintptr_t)weights[branch], 0, PACKED_K, 1); - bb_smatmul_os(1, 0, out_bank, CORE, CORE, PACKED_K, 1, 1, 0); + bb_smatmul_os(1, 0, out_bank, M, CORE, PACKED_K, 1, 1, 0); } int main(void) { @@ -51,8 +54,10 @@ int main(void) { for (int column = 0; column < IN; ++column) images[branch][(row * IN + column) * CORE] = image_value(branch, row, column); - for (int i = 0; i < PACKED_K * CORE; ++i) - weights[branch][i] = (int8_t)((i / CORE * 3 + i % CORE) % 41 - 20); + for (int tap = 0; tap < K * K; ++tap) + for (int column = 0; column < CORE; ++column) + weights[branch][tap * CORE * CORE + column] = + (int8_t)((tap * 3 + column) % 41 - 20); } for (int lane = 0; lane < CORE; ++lane) scale[lane] = 0.05f; diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_mvin_smatmul_hazard_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_mvin_smatmul_hazard_test.c index 24963d993..ba7577460 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_mvin_smatmul_hazard_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_mvin_smatmul_hazard_test.c @@ -51,7 +51,7 @@ int main(void) { bb_mvin_2d((uintptr_t)(pixel_input + 16), 8, 1, 24, 28, 15, 3, 8); for (int iteration = 0; iteration < iterations; ++iteration) for (int lane = 0; lane < 16; ++lane) { - bb_im2col(8, 5, 3, 1, 1, 0, 0, lane, 0, 0, 0, 9); + bb_im2col(8, 5, 3, 1, 1, 0, 0, 0, 0, 0, 9, 8); bb_mvin((uintptr_t)weights, 6, 16, 1); bb_smatmul_os(5, 6, 7, 16, 16, 16, lane == 0, lane == 15, 0); } diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/pebble.c index e19dcd49b..6498865d3 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/pebble.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/pebble.c @@ -11,10 +11,11 @@ #error "Pebble thread pipeline requires 128-bit banks with at least 64 rows" #endif -enum { LANES = BANK_WIDTH / 8, INPUT_ROWS = 36, WINDOWS = 16 }; +// K is (kernel row, kernel column, channel); only channel 0 is nonzero. +enum { LANES = BANK_WIDTH / 8, INPUT_ROWS = 36, WINDOWS = 16, K = 9 * LANES }; static int8_t input[BB_CORE_NUM][INPUT_ROWS * LANES] __attribute__((aligned(64))); -static int8_t weight[BB_CORE_NUM][LANES * LANES] __attribute__((aligned(64))); +static int8_t weight[BB_CORE_NUM][K * LANES] __attribute__((aligned(64))); static int32_t bias[BB_CORE_NUM][LANES] __attribute__((aligned(64))); static float scale[BB_CORE_NUM][LANES] __attribute__((aligned(64))); static int8_t output[BB_CORE_NUM][WINDOWS * LANES] __attribute__((aligned(64))); @@ -29,19 +30,19 @@ extern "C" for (int channel = 0; channel < LANES; ++channel) { bias[thread][channel] = channel - 8; scale[thread][channel] = 1.0f; - for (int k = 0; k < LANES; ++k) - weight[thread][k * LANES + channel] = - k < 9 ? (2 * k + 3 * channel) % 5 - 2 : 0; + for (int k = 0; k < 9; ++k) + weight[thread][k * LANES * LANES + channel] = + (2 * k + 3 * channel) % 5 - 2; } for (int bank = 0; bank < 7; ++bank) bb_mem_alloc(bank, 1, 1); bb_mvin((uintptr_t)input[thread], 0, INPUT_ROWS, 1); - bb_mvin((uintptr_t)weight[thread], 2, LANES, 1); + bb_mvin((uintptr_t)weight[thread], 2, K, 1); bb_mvin((uintptr_t)bias[thread], 3, LANES * 32 / BANK_WIDTH, 1); bb_mvin((uintptr_t)scale[thread], 5, LANES * 32 / BANK_WIDTH, 1); - bb_im2col(0, 1, 6, 3, 1, 0, 0, 0, 0, 0, 0, WINDOWS); + bb_im2col(0, 1, 6, 3, 1, 0, 0, 0, 0, 0, WINDOWS, 8); bb_smatmul_bias(3, 0); - bb_smatmul_os(1, 2, 4, WINDOWS, LANES, LANES, 1, 1, 0); + bb_smatmul_os(1, 2, 4, WINDOWS, LANES, K, 1, 1, 0); bb_quant_i32_to_i8(4, 5, 6, WINDOWS * 4, 0, 0, 4, 4, 4, 0); bb_mvout((uintptr_t)output[thread], 6, WINDOWS, 1); for (int window = 0; window < WINDOWS; ++window) @@ -51,8 +52,8 @@ extern "C" for (int ky = 0; ky < 3; ++ky) for (int kx = 0; kx < 3; ++kx) { int k = ky * 3 + kx, row = (y + ky) * 6 + x + kx; - expected += - input[thread][row * LANES] * weight[thread][k * LANES + channel]; + expected += input[thread][row * LANES] * + weight[thread][k * LANES * LANES + channel]; } if (output[thread][window * LANES + channel] != expected) return 1; diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_im2col_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_im2col_test.c index 8cc97b799..00a0d7ca8 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_im2col_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_im2col_test.c @@ -19,7 +19,7 @@ int main(void) { bb_mem_alloc(2, 1, 1); bb_mvin((uintptr_t)input, 0, INPUT_ROWS, 1); bb_maxpool(0, 1, 6, 3, 2, 2, 0, 0, 0, 3, 0, 0); - bb_im2col(1, 2, 3, 1, 1, 0, 0, 0, 0, 0, 0, OUTPUT_ROWS); + bb_im2col(1, 2, 3, 1, 1, 0, 0, 0, 0, 0, OUTPUT_ROWS, 8); bb_mvout((uintptr_t)output, 2, OUTPUT_ROWS, 1); for (int y = 0; y < 3; ++y) for (int x = 0; x < 3; ++x) { diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_k5_pipeline_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_k5_pipeline_test.c index aa3183c70..bb1525e79 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_k5_pipeline_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_k5_pipeline_test.c @@ -2,13 +2,19 @@ #include #include #include -#include #include #include #include #include -enum { INPUT_SIDE = 8, OUTPUT_SIDE = 4, KERNEL = 5, CORE = 16, PADDED_K = 32 }; +// K is (kernel row, kernel column, channel); only channel 0 is nonzero. +enum { + INPUT_SIDE = 8, + OUTPUT_SIDE = 4, + KERNEL = 5, + CORE = 16, + PADDED_K = KERNEL * KERNEL * CORE +}; static int8_t input[INPUT_SIDE * INPUT_SIDE * CORE] __attribute__((aligned(64))); static int8_t weight[PADDED_K * CORE] __attribute__((aligned(64))); @@ -22,7 +28,7 @@ int main(void) { input[pixel * CORE] = pixel % 5 - 2; for (int k = 0; k < KERNEL * KERNEL; ++k) for (int channel = 0; channel < CORE; ++channel) - weight[k * CORE + channel] = (k + channel) % 3 - 1; + weight[k * CORE * CORE + channel] = (k + channel) % 3 - 1; for (int channel = 0; channel < CORE; ++channel) scale[channel] = 1.0f; @@ -32,17 +38,11 @@ int main(void) { bb_mvin((uintptr_t)weight, 2, PADDED_K, 1); bb_mvin((uintptr_t)bias, 3, 4, 1); bb_mvin((uintptr_t)scale, 5, 4, 1); - bb_im2col(0, 1, INPUT_SIDE, KERNEL, 1, 0, 0, 0, 0, 0, 0, - OUTPUT_SIDE * OUTPUT_SIDE); - for (int row_core = 0; row_core < 2; ++row_core) - for (int k_core = 0; k_core < 2; ++k_core) - for (int row = 0; row < 8; ++row) { - int input_row = k_core * 16 + row_core * 8 + row; - int output_row = (row_core * 2 + k_core) * 8 + row; - bb_maxpool(1, 7, 1, 1, 1, 1, 0, input_row, output_row, 1, 0, 0); - } + // Im2col writes SMatMul's 8-row M tiles directly. + bb_im2col(0, 1, INPUT_SIDE, KERNEL, 1, 0, 0, 0, 0, 0, + OUTPUT_SIDE * OUTPUT_SIDE, 8); bb_smatmul_bias(3, 0); - bb_smatmul_os(7, 2, 4, OUTPUT_SIDE * OUTPUT_SIDE, CORE, PADDED_K, 1, 1, 0); + bb_smatmul_os(1, 2, 4, OUTPUT_SIDE * OUTPUT_SIDE, CORE, PADDED_K, 1, 1, 0); bb_quant_i32_to_i8(4, 5, 6, OUTPUT_SIDE * OUTPUT_SIDE * 4, 0, 0, OUTPUT_SIDE, OUTPUT_SIDE, OUTPUT_SIDE, 0); bb_mvout((uintptr_t)output, 6, OUTPUT_SIDE * OUTPUT_SIDE, 1); @@ -55,7 +55,7 @@ int main(void) { for (int kx = 0; kx < KERNEL; ++kx) { int pixel = (y + ky) * INPUT_SIDE + x + kx; int k = ky * KERNEL + kx; - expected += input[pixel * CORE] * weight[k * CORE + channel]; + expected += input[pixel * CORE] * weight[k * CORE * CORE + channel]; } int index = (y * OUTPUT_SIDE + x) * CORE + channel; if (output[index] != expected) { diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_pipeline_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_pipeline_test.c index b56958308..bbd9c900f 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_pipeline_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/mega_conv_pipeline_test.c @@ -8,7 +8,8 @@ #include static int8_t input[36 * 16] __attribute__((aligned(64))); -static int8_t weight[16 * 16] __attribute__((aligned(64))); +// K is (kernel row, kernel column, channel); only channel 0 is nonzero. +static int8_t weight[9 * 16 * 16] __attribute__((aligned(64))); static int32_t bias[16] __attribute__((aligned(64))); static float scale[16] __attribute__((aligned(64))); static int8_t output[16 * 16] __attribute__((aligned(64))); @@ -22,18 +23,19 @@ int main(void) { bias[row] = row - 8; scale[row] = 1.0f; for (int col = 0; col < 16; ++col) - weight[row * 16 + col] = row < 9 ? (2 * row + 3 * col) % 5 - 2 : 0; + if (row < 9) + weight[row * 16 * 16 + col] = (2 * row + 3 * col) % 5 - 2; } for (int bank = 0; bank < 7; ++bank) bb_mem_alloc(bank, 1, 1); bb_mvin((uintptr_t)input, 0, 36, 1); - bb_mvin((uintptr_t)weight, 2, 16, 1); + bb_mvin((uintptr_t)weight, 2, 9 * 16, 1); bb_mvin((uintptr_t)bias, 3, 4, 1); bb_mvin((uintptr_t)scale, 5, 4, 1); - bb_im2col(0, 1, 6, 3, 1, 0, 0, 0, 0, 0, 0, 16); + bb_im2col(0, 1, 6, 3, 1, 0, 0, 0, 0, 0, 16, 8); bb_smatmul_bias(3, 0); - bb_smatmul_os(1, 2, 4, 16, 16, 16, 1, 1, 0); + bb_smatmul_os(1, 2, 4, 16, 16, 9 * 16, 1, 1, 0); bb_quant_i32_to_i8(4, 5, 6, 64, 0, 0, 4, 4, 4, 0); bb_mvout((uintptr_t)output, 6, 16, 1); @@ -43,7 +45,7 @@ int main(void) { for (int ky = 0; ky < 3; ++ky) for (int kx = 0; kx < 3; ++kx) { int k = ky * 3 + kx, row = (y + ky) * 6 + x + kx; - expected += input[row * 16] * weight[k * 16 + channel]; + expected += input[row * 16] * weight[k * 16 * 16 + channel]; } int index = window * 16 + channel; if (output[index] != expected) { From 56a42b1975a08e6f09e9598fb52315addab18287 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 26 Oct 2026 20:44:49 +0800 Subject: [PATCH 193/196] fix: restore CI dependency registration and proxy setup --- .github/scripts/ci_repo_lock.sh | 4 +++- .github/scripts/setproxy.sh | 8 +++++++- arch/thirdparty/common_cells | 1 + arch/thirdparty/cvfpu | 1 + arch/thirdparty/hci | 1 + arch/thirdparty/hwpe-ctrl | 1 + arch/thirdparty/hwpe-stream | 1 + arch/thirdparty/redmule | 1 + arch/thirdparty/tech_cells_generic | 1 + bbdev/flake.nix | 3 ++- .../components/cores/boom/thirdparty/riscv-boom | 1 + .../components/cores/boom/thirdparty/rocket-chip | 1 + .../cores/rocket/thirdparty/rocket-chip | 1 + .../cores/xiangshan/thirdparty/xiangshan | 1 + flake.nix | 1 + scripts/nix/download.sh | 16 +++------------- stack | 2 +- 17 files changed, 28 insertions(+), 17 deletions(-) create mode 160000 arch/thirdparty/common_cells create mode 160000 arch/thirdparty/cvfpu create mode 160000 arch/thirdparty/hci create mode 160000 arch/thirdparty/hwpe-ctrl create mode 160000 arch/thirdparty/hwpe-stream create mode 160000 arch/thirdparty/redmule create mode 160000 arch/thirdparty/tech_cells_generic create mode 160000 examples/components/cores/boom/thirdparty/riscv-boom create mode 160000 examples/components/cores/boom/thirdparty/rocket-chip create mode 160000 examples/components/cores/rocket/thirdparty/rocket-chip create mode 160000 examples/components/cores/xiangshan/thirdparty/xiangshan diff --git a/.github/scripts/ci_repo_lock.sh b/.github/scripts/ci_repo_lock.sh index c3db1ba6a..59f8f68ee 100755 --- a/.github/scripts/ci_repo_lock.sh +++ b/.github/scripts/ci_repo_lock.sh @@ -42,7 +42,9 @@ repo_reset() { git -C examples/components/cores/rocket/thirdparty/rocket-chip submodule update --init --force -- dependencies/cde git -C stack submodule sync - git -C stack submodule update --init --force -- thirdparty/buddy-mlir + git -C stack submodule update --init --force -- engine os/linux thirdparty/buddy-mlir + git -C stack submodule update --init --depth 1 --single-branch --recommend-shallow -- \ + os/thirdparty/linux os/thirdparty/busybox os/thirdparty/opensbi git -C stack/thirdparty/buddy-mlir submodule sync git -C stack/thirdparty/buddy-mlir submodule update --init --force -- llvm diff --git a/.github/scripts/setproxy.sh b/.github/scripts/setproxy.sh index f9d76cbae..f6e6077db 100644 --- a/.github/scripts/setproxy.sh +++ b/.github/scripts/setproxy.sh @@ -1 +1,7 @@ -export ALL_PROXY=http://127.0.0.1:7890 +export http_proxy=http://127.0.0.1:7890 +export https_proxy="$http_proxy" +export HTTP_PROXY="$http_proxy" +export HTTPS_PROXY="$https_proxy" +export ALL_PROXY="$http_proxy" +export no_proxy=localhost,127.0.0.1,::1 +export NO_PROXY="$no_proxy" diff --git a/arch/thirdparty/common_cells b/arch/thirdparty/common_cells new file mode 160000 index 000000000..128154569 --- /dev/null +++ b/arch/thirdparty/common_cells @@ -0,0 +1 @@ +Subproject commit 1281545696eb3fcba50ec5b4275993476a3c710e diff --git a/arch/thirdparty/cvfpu b/arch/thirdparty/cvfpu new file mode 160000 index 000000000..a8e0cba6d --- /dev/null +++ b/arch/thirdparty/cvfpu @@ -0,0 +1 @@ +Subproject commit a8e0cba6dd50f357ece73c2c955d96efc3c6c315 diff --git a/arch/thirdparty/hci b/arch/thirdparty/hci new file mode 160000 index 000000000..a8d8132b7 --- /dev/null +++ b/arch/thirdparty/hci @@ -0,0 +1 @@ +Subproject commit a8d8132b7f7839fd55f92df65cdc3e361ea80a6b diff --git a/arch/thirdparty/hwpe-ctrl b/arch/thirdparty/hwpe-ctrl new file mode 160000 index 000000000..dfcf1cb74 --- /dev/null +++ b/arch/thirdparty/hwpe-ctrl @@ -0,0 +1 @@ +Subproject commit dfcf1cb746709de206377388bbcb4a3571e24f7b diff --git a/arch/thirdparty/hwpe-stream b/arch/thirdparty/hwpe-stream new file mode 160000 index 000000000..05cb83036 --- /dev/null +++ b/arch/thirdparty/hwpe-stream @@ -0,0 +1 @@ +Subproject commit 05cb8303693065812e9d6bfe10976220fb455666 diff --git a/arch/thirdparty/redmule b/arch/thirdparty/redmule new file mode 160000 index 000000000..7fa9fbe8a --- /dev/null +++ b/arch/thirdparty/redmule @@ -0,0 +1 @@ +Subproject commit 7fa9fbe8a29e8572810ae12b92c19749045ac860 diff --git a/arch/thirdparty/tech_cells_generic b/arch/thirdparty/tech_cells_generic new file mode 160000 index 000000000..3a3de7363 --- /dev/null +++ b/arch/thirdparty/tech_cells_generic @@ -0,0 +1 @@ +Subproject commit 3a3de73632a06826b1bd9c65a0a2e92b32016845 diff --git a/bbdev/flake.nix b/bbdev/flake.nix index 039d33b78..4f3ec5e95 100644 --- a/bbdev/flake.nix +++ b/bbdev/flake.nix @@ -19,6 +19,7 @@ bbdevPythonPkgs # System tools + cacert curl git ]; @@ -31,8 +32,8 @@ if [ ! -d "$VENV_DIR" ]; then echo "Creating Python venv..." python3 -m venv "$VENV_DIR" --system-site-packages - "$VENV_DIR/bin/pip" install -q motia[otel]==1.0.0rc17 iii-sdk==0.2.0 fi + "$VENV_DIR/bin/pip" install -q 'motia[otel]==1.0.0rc17' iii-sdk==0.2.0 || exit $? export PATH="$VENV_DIR/bin:$PATH" echo "bbdev dev environment ready" diff --git a/examples/components/cores/boom/thirdparty/riscv-boom b/examples/components/cores/boom/thirdparty/riscv-boom new file mode 160000 index 000000000..ba03175e0 --- /dev/null +++ b/examples/components/cores/boom/thirdparty/riscv-boom @@ -0,0 +1 @@ +Subproject commit ba03175e0a3cf5c0d5a7f14d385ac1c8998c4428 diff --git a/examples/components/cores/boom/thirdparty/rocket-chip b/examples/components/cores/boom/thirdparty/rocket-chip new file mode 160000 index 000000000..ece7b9ad5 --- /dev/null +++ b/examples/components/cores/boom/thirdparty/rocket-chip @@ -0,0 +1 @@ +Subproject commit ece7b9ad544b07df39cd13b6fd7236562a26355d diff --git a/examples/components/cores/rocket/thirdparty/rocket-chip b/examples/components/cores/rocket/thirdparty/rocket-chip new file mode 160000 index 000000000..a4b4643bb --- /dev/null +++ b/examples/components/cores/rocket/thirdparty/rocket-chip @@ -0,0 +1 @@ +Subproject commit a4b4643bbcbe6d6deb6cfa8b95954cbf518a7073 diff --git a/examples/components/cores/xiangshan/thirdparty/xiangshan b/examples/components/cores/xiangshan/thirdparty/xiangshan new file mode 160000 index 000000000..fe2288cb0 --- /dev/null +++ b/examples/components/cores/xiangshan/thirdparty/xiangshan @@ -0,0 +1 @@ +Subproject commit fe2288cb09361b65afa26be3f403520c4b496dbf diff --git a/flake.nix b/flake.nix index 6b11fed03..89cb763ed 100644 --- a/flake.nix +++ b/flake.nix @@ -26,6 +26,7 @@ hardeningDisable = [ "libcxxhardeningfast" ]; buildInputs = with pkgs; [ + cacert SDL2 clibs.zlib-dev clibs.zlib diff --git a/scripts/nix/download.sh b/scripts/nix/download.sh index acecd350b..602218f2e 100755 --- a/scripts/nix/download.sh +++ b/scripts/nix/download.sh @@ -22,22 +22,12 @@ begin_step() { begin_step "0-2" "submodules init" cd ${BBDIR} -git submodule update --init --progress \ - arch/thirdparty/gemmini \ - arch/thirdparty/berkeley-hardfloat \ - bbdev \ - bebop \ - stack \ - docs \ - verify \ - thirdparty/firesim \ - thirdparty/soc-framework \ - thirdparty/waveform-mcp \ - .agents/skills \ - .agents/mcps +git submodule sync +git submodule update --init --progress git -C ${BBDIR}/stack submodule update --init --progress engine os/linux git -C ${BBDIR}/stack submodule update --init --depth 1 --single-branch --recommend-shallow --progress \ os/thirdparty/linux \ + os/thirdparty/busybox \ os/thirdparty/opensbi begin_step "0-4" "buddy-mlir llvm init" git -C ${BBDIR}/stack submodule update --init --progress thirdparty/buddy-mlir diff --git a/stack b/stack index ef427ae93..28dafe976 160000 --- a/stack +++ b/stack @@ -1 +1 @@ -Subproject commit ef427ae930afc8a4c40a98e427e580a8675dcd8c +Subproject commit 28dafe976e5d7826eb1706bbbd5274073c555a95 From 4d47387c050202f4e76ecfcc6509977f8aa444e6 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 26 Oct 2026 20:55:19 +0800 Subject: [PATCH 194/196] style: apply Python formatting required by CI --- .../steps/bebop/p2e/scripts/runtime_case.py | 18 +- .../steps/common/01_result_query_api.step.py | 24 +- bbdev/api/steps/common/02_cancel_api.step.py | 11 +- .../api/steps/common/02_cancel_event.step.py | 1 - bbdev/api/steps/config/scripts/model_pb2.py | 35 +- bbdev/api/steps/firesim/.gitignore | 2 +- .../api/steps/firesim/scripts/firesim_env.py | 23 +- bbdev/api/steps/model/scripts/config.py | 25 +- bbdev/api/steps/uvm/scripts/__init__.py | 1 - bbdev/api/tests/_api_test_helper.py | 2 +- bbdev/api/utils/port.py | 1 + bbdev/api/utils/process.py | 336 +++++++++--------- bbdev/api/utils/process_registry.py | 5 +- bbdev/api/utils/stream_run.py | 10 +- bbdev/api/utils/workers.py | 9 +- 15 files changed, 269 insertions(+), 234 deletions(-) diff --git a/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py b/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py index 57f23f255..926184eef 100644 --- a/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py +++ b/bbdev/api/steps/bebop/p2e/scripts/runtime_case.py @@ -11,13 +11,19 @@ def trace_modes(bitstream: str) -> set[str]: return modes -def validate_runtime_reuse(bitstream: str, diff: bool, itrace: bool = False, mtrace: bool = False) -> Path: +def validate_runtime_reuse( + bitstream: str, diff: bool, itrace: bool = False, mtrace: bool = False +) -> Path: bit = Path(bitstream).resolve() case = bit.parent.parent executable = case / "bebop-p2e" - files = [bit, executable, case / "vvacDir/runtimeDir/rtcfg", - case / "vvacDir/runtimeDir/lib/lib_arm/libvCtb.so", - case / "p2e_trace_mode"] + files = [ + bit, + executable, + case / "vvacDir/runtimeDir/rtcfg", + case / "vvacDir/runtimeDir/lib/lib_arm/libvCtb.so", + case / "p2e_trace_mode", + ] missing = [str(path) for path in files if not path.is_file()] if missing: raise ValueError(f"Reusable P2E case is incomplete: {missing}") @@ -28,5 +34,7 @@ def validate_runtime_reuse(bitstream: str, diff: bool, itrace: bool = False, mtr raise ValueError("Reusable P2E case does not support diff") for trace, requested in (("itrace", itrace), ("mtrace", mtrace)): if requested and trace not in modes: - raise ValueError(f"Reusable P2E case does not support {trace}; rebuild with --{trace}") + raise ValueError( + f"Reusable P2E case does not support {trace}; rebuild with --{trace}" + ) return executable diff --git a/bbdev/api/steps/common/01_result_query_api.step.py b/bbdev/api/steps/common/01_result_query_api.step.py index 88257656f..054807298 100644 --- a/bbdev/api/steps/common/01_result_query_api.step.py +++ b/bbdev/api/steps/common/01_result_query_api.step.py @@ -19,10 +19,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: trace_id = request.path_params.get("trace_id", "") if not trace_id: - return ApiResponse( - status=400, - body={"error": "trace_id is required"} - ) + return ApiResponse(status=400, body={"error": "trace_id is required"}) cancelled_state = await ctx.state.get(trace_id, "cancelled") if cancelled_state: @@ -41,8 +38,8 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: status=200, body={ "status": "success", - "body": success_state.get("body", success_state) - } + "body": success_state.get("body", success_state), + }, ) # Check for failure state @@ -52,23 +49,16 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: status=200, body={ "status": "failure", - "body": failure_state.get("body", failure_state) - } + "body": failure_state.get("body", failure_state), + }, ) # Check for processing state processing_state = await ctx.state.get(trace_id, "processing") if processing_state: - return ApiResponse( - status=200, - body={ - "status": "processing", - "body": {} - } - ) + return ApiResponse(status=200, body={"status": "processing", "body": {}}) # No state found return ApiResponse( - status=404, - body={"error": f"No result found for trace_id: {trace_id}"} + status=404, body={"error": f"No result found for trace_id: {trace_id}"} ) diff --git a/bbdev/api/steps/common/02_cancel_api.step.py b/bbdev/api/steps/common/02_cancel_api.step.py index 7da7dfbca..dee1a2375 100644 --- a/bbdev/api/steps/common/02_cancel_api.step.py +++ b/bbdev/api/steps/common/02_cancel_api.step.py @@ -1,6 +1,5 @@ from motia import ApiRequest, ApiResponse, FlowContext, api - config = { "name": "task-cancel-api", "description": "Cancel a queued or running bbdev task by trace_id", @@ -22,8 +21,10 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body={"error": "task is already terminal", "trace_id": trace_id}, ) - await ctx.enqueue({ - "topic": "common.task.cancel", - "data": {"_trace_id": trace_id}, - }) + await ctx.enqueue( + { + "topic": "common.task.cancel", + "data": {"_trace_id": trace_id}, + } + ) return ApiResponse(status=202, body={"trace_id": trace_id, "status": "cancelling"}) diff --git a/bbdev/api/steps/common/02_cancel_event.step.py b/bbdev/api/steps/common/02_cancel_event.step.py index 64c63d428..392cd7929 100644 --- a/bbdev/api/steps/common/02_cancel_event.step.py +++ b/bbdev/api/steps/common/02_cancel_event.step.py @@ -5,7 +5,6 @@ from utils.event_common import get_origin_trace_id from utils.process_registry import cancel_process, force_cancel_process, process_pid - config = { "name": "task-cancel", "description": "Terminate a task process group", diff --git a/bbdev/api/steps/config/scripts/model_pb2.py b/bbdev/api/steps/config/scripts/model_pb2.py index 3d6be48f8..105156b3a 100644 --- a/bbdev/api/steps/config/scripts/model_pb2.py +++ b/bbdev/api/steps/config/scripts/model_pb2.py @@ -4,39 +4,36 @@ # source: model.proto # Protobuf Python Version: 7.35.1 """Generated protocol buffer code.""" + from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder + _runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, - 7, - 35, - 1, - '', - 'model.proto' + _runtime_version.Domain.PUBLIC, 7, 35, 1, "", "model.proto" ) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0bmodel.proto\x12\x10\x62uckyball.config\"_\n\x0eModelParameter\x12\x11\n\x07integer\x18\x01 \x01(\x12H\x00\x12\x0e\n\x04real\x18\x02 \x01(\x01H\x00\x12\x0e\n\x04text\x18\x03 \x01(\tH\x00\x12\x11\n\x07\x62oolean\x18\x04 \x01(\x08H\x00\x42\x07\n\x05value\"\xc4\x01\n\x0bModelConfig\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x41\n\nparameters\x18\x03 \x03(\x0b\x32-.buckyball.config.ModelConfig.ParametersEntry\x1aS\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0b\x32 .buckyball.config.ModelParameter:\x02\x38\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x0bmodel.proto\x12\x10\x62uckyball.config"_\n\x0eModelParameter\x12\x11\n\x07integer\x18\x01 \x01(\x12H\x00\x12\x0e\n\x04real\x18\x02 \x01(\x01H\x00\x12\x0e\n\x04text\x18\x03 \x01(\tH\x00\x12\x11\n\x07\x62oolean\x18\x04 \x01(\x08H\x00\x42\x07\n\x05value"\xc4\x01\n\x0bModelConfig\x12\x0f\n\x07version\x18\x01 \x01(\r\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x41\n\nparameters\x18\x03 \x03(\x0b\x32-.buckyball.config.ModelConfig.ParametersEntry\x1aS\n\x0fParametersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0b\x32 .buckyball.config.ModelParameter:\x02\x38\x01\x62\x06proto3' +) _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'model_pb2', _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "model_pb2", _globals) if not _descriptor._USE_C_DESCRIPTORS: - DESCRIPTOR._loaded_options = None - _globals['_MODELCONFIG_PARAMETERSENTRY']._loaded_options = None - _globals['_MODELCONFIG_PARAMETERSENTRY']._serialized_options = b'8\001' - _globals['_MODELPARAMETER']._serialized_start=33 - _globals['_MODELPARAMETER']._serialized_end=128 - _globals['_MODELCONFIG']._serialized_start=131 - _globals['_MODELCONFIG']._serialized_end=327 - _globals['_MODELCONFIG_PARAMETERSENTRY']._serialized_start=244 - _globals['_MODELCONFIG_PARAMETERSENTRY']._serialized_end=327 + DESCRIPTOR._loaded_options = None + _globals["_MODELCONFIG_PARAMETERSENTRY"]._loaded_options = None + _globals["_MODELCONFIG_PARAMETERSENTRY"]._serialized_options = b"8\001" + _globals["_MODELPARAMETER"]._serialized_start = 33 + _globals["_MODELPARAMETER"]._serialized_end = 128 + _globals["_MODELCONFIG"]._serialized_start = 131 + _globals["_MODELCONFIG"]._serialized_end = 327 + _globals["_MODELCONFIG_PARAMETERSENTRY"]._serialized_start = 244 + _globals["_MODELCONFIG_PARAMETERSENTRY"]._serialized_end = 327 # @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/firesim/.gitignore b/bbdev/api/steps/firesim/.gitignore index 2a6160516..1e82fc7de 100644 --- a/bbdev/api/steps/firesim/.gitignore +++ b/bbdev/api/steps/firesim/.gitignore @@ -1 +1 @@ -*.yaml \ No newline at end of file +*.yaml diff --git a/bbdev/api/steps/firesim/scripts/firesim_env.py b/bbdev/api/steps/firesim/scripts/firesim_env.py index 01af74d1c..03af44084 100644 --- a/bbdev/api/steps/firesim/scripts/firesim_env.py +++ b/bbdev/api/steps/firesim/scripts/firesim_env.py @@ -4,7 +4,6 @@ import subprocess - def setup_firesim_env() -> dict: """Setup FireSim env vars + SSH agent. Returns env dict for subprocess.""" env = os.environ.copy() @@ -13,11 +12,20 @@ def setup_firesim_env() -> dict: env["FIRESIM_RUNFARM_PREFIX"] = "" # Create minimal env.sh for firesim if it doesn't exist - firesim_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..", "thirdparty", "firesim")) + firesim_dir = os.path.abspath( + os.path.join( + os.path.dirname(__file__), "../../../../..", "thirdparty", "firesim" + ) + ) env["PATH"] = os.path.join(firesim_dir, "deploy") + os.pathsep + env.get("PATH", "") env_sh = os.path.join(firesim_dir, "env.sh") if not os.path.exists(env_sh): - riscv_path = env.get("RISCV", os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..", "result"))) + riscv_path = env.get( + "RISCV", + os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../../../..", "result") + ), + ) with open(env_sh, "w") as f: f.write(f"""#!/usr/bin/env bash # Minimal env.sh for FireSim (auto-generated by firesim_env.py) @@ -29,9 +37,11 @@ def setup_firesim_env() -> dict: # Setup SSH agent for localhost FPGA enumeration key = os.path.expanduser("~/.ssh/firesim.pem") - if os.path.exists(key) and "firesim.pem" not in subprocess.run( - ["ssh-add", "-l"], capture_output=True, text=True - ).stdout: + if ( + os.path.exists(key) + and "firesim.pem" + not in subprocess.run(["ssh-add", "-l"], capture_output=True, text=True).stdout + ): if not env.get("SSH_AUTH_SOCK"): for line in subprocess.run( ["ssh-agent", "-s"], capture_output=True, text=True @@ -42,4 +52,3 @@ def setup_firesim_env() -> dict: subprocess.run(["ssh-add", key], env=env, capture_output=True) return env - diff --git a/bbdev/api/steps/model/scripts/config.py b/bbdev/api/steps/model/scripts/config.py index 81ab5edb6..246addab1 100644 --- a/bbdev/api/steps/model/scripts/config.py +++ b/bbdev/api/steps/model/scripts/config.py @@ -1,4 +1,5 @@ """Compile model-owned TOML parameters to protobuf and ordinary build arguments.""" + import ast import json import math @@ -9,9 +10,13 @@ from steps.config.scripts.model_pb2 import ModelConfig - -_OPERATORS = {ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, - ast.FloorDiv: operator.floordiv, ast.Mod: operator.mod} +_OPERATORS = { + ast.Add: operator.add, + ast.Sub: operator.sub, + ast.Mult: operator.mul, + ast.FloorDiv: operator.floordiv, + ast.Mod: operator.mod, +} def compile_config(source: Path, destination: Path, name: str) -> None: @@ -48,7 +53,9 @@ def evaluate(node): if type(left) not in (int, float) or type(right) not in (int, float): raise ValueError("Derived arithmetic requires numeric parameters") return _OPERATORS[type(node.op)](left, right) - raise ValueError("Model expressions support names, numbers, +, -, *, // and % only") + raise ValueError( + "Model expressions support names, numbers, +, -, *, // and % only" + ) for key in derived: resolve(key) @@ -57,12 +64,18 @@ def evaluate(node): for key, value in values.items(): if not re.fullmatch(r"[a-z][a-z0-9_]*", key): raise ValueError(f"Invalid model parameter name: {key}") - if type(value) not in fields or isinstance(value, float) and not math.isfinite(value): + if ( + type(value) not in fields + or isinstance(value, float) + and not math.isfinite(value) + ): raise ValueError(f"Invalid model parameter value: {key}") setattr(message.parameters[key], fields[type(value)], value) destination.parent.mkdir(parents=True, exist_ok=True) destination.write_bytes(message.SerializeToString(deterministic=True)) - destination.with_suffix(".json").write_text(json.dumps(values, indent=2, sort_keys=True) + "\n") + destination.with_suffix(".json").write_text( + json.dumps(values, indent=2, sort_keys=True) + "\n" + ) def cmake_arguments(path: Path, name: str) -> list[str]: diff --git a/bbdev/api/steps/uvm/scripts/__init__.py b/bbdev/api/steps/uvm/scripts/__init__.py index 8b1378917..e69de29bb 100644 --- a/bbdev/api/steps/uvm/scripts/__init__.py +++ b/bbdev/api/steps/uvm/scripts/__init__.py @@ -1 +0,0 @@ - diff --git a/bbdev/api/tests/_api_test_helper.py b/bbdev/api/tests/_api_test_helper.py index 7da793d8c..662c52b15 100644 --- a/bbdev/api/tests/_api_test_helper.py +++ b/bbdev/api/tests/_api_test_helper.py @@ -3,4 +3,4 @@ def run_bbdev_case(command: str): - subprocess.run(shlex.split(command), check=True) + subprocess.run(shlex.split(command), check=True) diff --git a/bbdev/api/utils/port.py b/bbdev/api/utils/port.py index e450ba414..c4067a612 100644 --- a/bbdev/api/utils/port.py +++ b/bbdev/api/utils/port.py @@ -1,5 +1,6 @@ import socket + def find_available_port(start_port: int = 5000, end_port: int = 5500) -> int: """Find an available port in the specified range""" for port in range(start_port, end_port + 1): diff --git a/bbdev/api/utils/process.py b/bbdev/api/utils/process.py index 7d5d45fdc..73794ee4d 100644 --- a/bbdev/api/utils/process.py +++ b/bbdev/api/utils/process.py @@ -6,214 +6,222 @@ def _stat(pid: int): - try: - with open(f"/proc/{pid}/stat") as f: - return f.read().split() - except (FileNotFoundError, PermissionError): - return None + try: + with open(f"/proc/{pid}/stat") as f: + return f.read().split() + except (FileNotFoundError, PermissionError): + return None def _exists(pid: int) -> bool: - stat = _stat(pid) - return stat is not None and stat[2] != "Z" + stat = _stat(pid) + return stat is not None and stat[2] != "Z" def _cmdline(pid: int) -> list[str]: - try: - with open(f"/proc/{pid}/cmdline", "rb") as f: - raw = f.read() - except (FileNotFoundError, PermissionError): - return [] - return [part.decode(errors="replace") for part in raw.split(b"\0") if part] + try: + with open(f"/proc/{pid}/cmdline", "rb") as f: + raw = f.read() + except (FileNotFoundError, PermissionError): + return [] + return [part.decode(errors="replace") for part in raw.split(b"\0") if part] def _cwd(pid: int) -> str | None: - try: - return os.readlink(f"/proc/{pid}/cwd") - except (FileNotFoundError, PermissionError, OSError): - return None + try: + return os.readlink(f"/proc/{pid}/cwd") + except (FileNotFoundError, PermissionError, OSError): + return None def _env(pid: int) -> dict[str, str]: - try: - with open(f"/proc/{pid}/environ", "rb") as f: - raw = f.read() - except (FileNotFoundError, PermissionError): - return {} + try: + with open(f"/proc/{pid}/environ", "rb") as f: + raw = f.read() + except (FileNotFoundError, PermissionError): + return {} - env = {} - for item in raw.split(b"\0"): - if not item or b"=" not in item: - continue - key, value = item.split(b"=", 1) - env[key.decode(errors="replace")] = value.decode(errors="replace") - return env + env = {} + for item in raw.split(b"\0"): + if not item or b"=" not in item: + continue + key, value = item.split(b"=", 1) + env[key.decode(errors="replace")] = value.decode(errors="replace") + return env def _own(pid: int) -> bool: - try: - return os.stat(f"/proc/{pid}").st_uid == os.getuid() - except (FileNotFoundError, PermissionError): - return False + try: + return os.stat(f"/proc/{pid}").st_uid == os.getuid() + except (FileNotFoundError, PermissionError): + return False def _children(pid: int) -> list[int]: - children = [] - try: - entries = os.listdir("/proc") - except FileNotFoundError: - return children - - for entry in entries: - if not entry.isdigit(): - continue - child = int(entry) - stat = _stat(child) - if not stat: - continue + children = [] try: - if int(stat[3]) == pid: - children.append(child) - children.extend(_children(child)) - except (IndexError, ValueError): - continue - return children + entries = os.listdir("/proc") + except FileNotFoundError: + return children + + for entry in entries: + if not entry.isdigit(): + continue + child = int(entry) + stat = _stat(child) + if not stat: + continue + try: + if int(stat[3]) == pid: + children.append(child) + children.extend(_children(child)) + except (IndexError, ValueError): + continue + return children def _wait_gone(pids: set[int], timeout: float) -> set[int]: - deadline = time.time() + timeout - alive = set(pids) - while alive and time.time() < deadline: - alive = {pid for pid in alive if _exists(pid)} - if alive: - time.sleep(0.1) - return {pid for pid in alive if _exists(pid)} + deadline = time.time() + timeout + alive = set(pids) + while alive and time.time() < deadline: + alive = {pid for pid in alive if _exists(pid)} + if alive: + time.sleep(0.1) + return {pid for pid in alive if _exists(pid)} def _signal(pids: set[int], sig: signal.Signals): - for pid in sorted(pids, reverse=True): - if pid == os.getpid(): - continue - try: - os.kill(pid, sig) - except (ProcessLookupError, PermissionError, OSError): - pass + for pid in sorted(pids, reverse=True): + if pid == os.getpid(): + continue + try: + os.kill(pid, sig) + except (ProcessLookupError, PermissionError, OSError): + pass def kill_tree(pid: int, term_timeout: float = 5.0): - targets = set(_children(pid)) - targets.add(pid) - _signal(targets, signal.SIGTERM) - alive = _wait_gone(targets, term_timeout) - if alive: - _signal(alive, signal.SIGKILL) - _wait_gone(alive, 1.0) + targets = set(_children(pid)) + targets.add(pid) + _signal(targets, signal.SIGTERM) + alive = _wait_gone(targets, term_timeout) + if alive: + _signal(alive, signal.SIGKILL) + _wait_gone(alive, 1.0) def terminate_group(proc: subprocess.Popen, term_timeout: float = 5.0): - if proc.poll() is not None: - return - kill_tree(proc.pid, term_timeout=term_timeout) - proc.wait(timeout=1) - - -def _is_workspace_server(pid: int, workflow_dir: str, worker_url: str | None = None) -> bool: - repo_dir = os.path.dirname(workflow_dir) - root_dir = os.path.dirname(repo_dir) - if not _own(pid): - return False - - cwd = _cwd(pid) - if cwd not in (workflow_dir, repo_dir, root_dir): - return False - - cmd = _cmdline(pid) - if not cmd: - return False - - exe = os.path.basename(cmd[0]) - is_bbdev = len(cmd) >= 2 and os.path.abspath(cmd[1]) == os.path.join(repo_dir, "bbdev") - if is_bbdev and worker_url is None: - return True - - if cwd != workflow_dir: - return False - if worker_url and _env(pid).get("III_URL") != worker_url: - return False - - text = " ".join(cmd) - return exe == "iii" or ("motia" in text and "dev" in cmd and "--dir" in cmd and "steps" in cmd) + if proc.poll() is not None: + return + kill_tree(proc.pid, term_timeout=term_timeout) + proc.wait(timeout=1) + + +def _is_workspace_server( + pid: int, workflow_dir: str, worker_url: str | None = None +) -> bool: + repo_dir = os.path.dirname(workflow_dir) + root_dir = os.path.dirname(repo_dir) + if not _own(pid): + return False + + cwd = _cwd(pid) + if cwd not in (workflow_dir, repo_dir, root_dir): + return False + + cmd = _cmdline(pid) + if not cmd: + return False + + exe = os.path.basename(cmd[0]) + is_bbdev = len(cmd) >= 2 and os.path.abspath(cmd[1]) == os.path.join( + repo_dir, "bbdev" + ) + if is_bbdev and worker_url is None: + return True + + if cwd != workflow_dir: + return False + if worker_url and _env(pid).get("III_URL") != worker_url: + return False + + text = " ".join(cmd) + return exe == "iii" or ( + "motia" in text and "dev" in cmd and "--dir" in cmd and "steps" in cmd + ) def _workspace_pids(workflow_dir: str, worker_url: str | None = None) -> set[int]: - pids = set() - try: - entries = os.listdir("/proc") - except FileNotFoundError: + pids = set() + try: + entries = os.listdir("/proc") + except FileNotFoundError: + return pids + + for entry in entries: + if entry.isdigit() and _is_workspace_server( + int(entry), workflow_dir, worker_url + ): + pids.add(int(entry)) return pids - for entry in entries: - if entry.isdigit() and _is_workspace_server(int(entry), workflow_dir, worker_url): - pids.add(int(entry)) - return pids - def _port_pids( - workflow_dir: str, - port: int | None, - worker_url: str | None = None, + workflow_dir: str, + port: int | None, + worker_url: str | None = None, ) -> set[int]: - if port is None: - return set() - - result = subprocess.run( - ["lsof", "-ti", f":{port}"], - capture_output=True, - text=True, - check=False, - ) - pids = set() - for line in result.stdout.splitlines(): - try: - pid = int(line) - if _is_workspace_server(pid, workflow_dir, worker_url): - pids.add(pid) - except ValueError: - pass - return pids + if port is None: + return set() + + result = subprocess.run( + ["lsof", "-ti", f":{port}"], + capture_output=True, + text=True, + check=False, + ) + pids = set() + for line in result.stdout.splitlines(): + try: + pid = int(line) + if _is_workspace_server(pid, workflow_dir, worker_url): + pids.add(pid) + except ValueError: + pass + return pids def stop_workspace_servers( - workflow_dir: str, - port: int | None = None, - worker_url: str | None = None, + workflow_dir: str, + port: int | None = None, + worker_url: str | None = None, ) -> int: - if port is None: - pids = _workspace_pids(workflow_dir, worker_url=worker_url) - else: - pids = _port_pids(workflow_dir, port, worker_url=worker_url) - if not pids: - return 0 - - targets = set(pids) - for pid in list(pids): - targets.update(_children(pid)) - - _signal(targets, signal.SIGTERM) - alive = _wait_gone(targets, 5.0) - if alive: - _signal(alive, signal.SIGKILL) - _wait_gone(alive, 1.0) - return len(targets) + if port is None: + pids = _workspace_pids(workflow_dir, worker_url=worker_url) + else: + pids = _port_pids(workflow_dir, port, worker_url=worker_url) + if not pids: + return 0 + + targets = set(pids) + for pid in list(pids): + targets.update(_children(pid)) + + _signal(targets, signal.SIGTERM) + alive = _wait_gone(targets, 5.0) + if alive: + _signal(alive, signal.SIGKILL) + _wait_gone(alive, 1.0) + return len(targets) def clean_python_caches(workflow_dir: str): - for rel in ("steps", "utils"): - root = os.path.join(workflow_dir, rel) - if not os.path.isdir(root): - continue - for dirpath, dirnames, _ in os.walk(root): - if "__pycache__" not in dirnames: - continue - shutil.rmtree(os.path.join(dirpath, "__pycache__"), ignore_errors=True) - dirnames.remove("__pycache__") + for rel in ("steps", "utils"): + root = os.path.join(workflow_dir, rel) + if not os.path.isdir(root): + continue + for dirpath, dirnames, _ in os.walk(root): + if "__pycache__" not in dirnames: + continue + shutil.rmtree(os.path.join(dirpath, "__pycache__"), ignore_errors=True) + dirnames.remove("__pycache__") diff --git a/bbdev/api/utils/process_registry.py b/bbdev/api/utils/process_registry.py index f2acd257b..f15d81690 100644 --- a/bbdev/api/utils/process_registry.py +++ b/bbdev/api/utils/process_registry.py @@ -8,11 +8,12 @@ from utils.process import kill_tree - _lock = threading.Lock() _processes: Dict[str, int] = {} _cancelled: set[str] = set() -_current_task_scope: ContextVar[str | None] = ContextVar("bbdev_task_scope", default=None) +_current_task_scope: ContextVar[str | None] = ContextVar( + "bbdev_task_scope", default=None +) def set_current_task_scope(scope: str) -> None: diff --git a/bbdev/api/utils/stream_run.py b/bbdev/api/utils/stream_run.py index 0fb2a02ab..13a4180df 100644 --- a/bbdev/api/utils/stream_run.py +++ b/bbdev/api/utils/stream_run.py @@ -4,7 +4,11 @@ from typing import Optional, List, Callable from utils.process import kill_tree -from utils.process_registry import current_task_scope, register_process, unregister_process +from utils.process_registry import ( + current_task_scope, + register_process, + unregister_process, +) class StreamResult: @@ -102,8 +106,8 @@ def emit(): stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - encoding='utf-8', - errors='replace', # Replace invalid UTF-8 bytes with � instead of crashing + encoding="utf-8", + errors="replace", # Replace invalid UTF-8 bytes with � instead of crashing bufsize=1, executable=executable, env=env, diff --git a/bbdev/api/utils/workers.py b/bbdev/api/utils/workers.py index 226243097..0386687ca 100644 --- a/bbdev/api/utils/workers.py +++ b/bbdev/api/utils/workers.py @@ -3,11 +3,14 @@ import subprocess from pathlib import Path + def server_ports_path(workflow_dir: str) -> str: return os.path.join(workflow_dir, "data", "bbdev-server.json") -def write_server_ports(workflow_dir: str, http_port: int, worker_port: int, bb_root: str) -> None: +def write_server_ports( + workflow_dir: str, http_port: int, worker_port: int, bb_root: str +) -> None: path = server_ports_path(workflow_dir) os.makedirs(os.path.dirname(path), exist_ok=True) tmp = f"{path}.tmp" @@ -62,7 +65,9 @@ def list_engine_workers(worker_port: int, timeout_s: float = 10.0) -> list: try: payload = json.loads(result.stdout) except json.JSONDecodeError as e: - raise RuntimeError(f"engine::workers::list returned non-JSON: {result.stdout[:500]}") from e + raise RuntimeError( + f"engine::workers::list returned non-JSON: {result.stdout[:500]}" + ) from e workers = payload.get("workers") if not isinstance(workers, list): raise RuntimeError(f"engine::workers::list missing workers list: {payload}") From 99924dc5bb0d50cdd2f2f8ff2efe4eaf63547997 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Mon, 26 Oct 2026 21:00:57 +0800 Subject: [PATCH 195/196] fix: prepare CI dependencies before simulation builds --- .github/scripts/ci_repo_lock.sh | 2 +- .github/workflows/check.yaml | 4 ++-- .github/workflows/regression.yml | 12 ++++++------ bebop | 2 +- scripts/nix/build-all.sh | 7 +++---- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/scripts/ci_repo_lock.sh b/.github/scripts/ci_repo_lock.sh index 59f8f68ee..12ff69c7f 100755 --- a/.github/scripts/ci_repo_lock.sh +++ b/.github/scripts/ci_repo_lock.sh @@ -25,7 +25,7 @@ repo_reset() { cd "${repo}" # Initialize the compiler in stack and its LLVM dependency explicitly. - git fetch --force --prune origin "${want_sha}" + git fetch --no-recurse-submodules --force --prune origin "${want_sha}" git checkout --detach --force "${want_sha}" git reset --hard "${want_sha}" git clean -ffd diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index fb4de1314..ea16cd7bf 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -46,7 +46,7 @@ jobs: cd "${root}" # fetch first to ensure if we modify ci_repo_lock.sh, we still use the old version to checkout. - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ prepare "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" @@ -113,7 +113,7 @@ jobs: root="${BB_ROOT/#\~/$HOME}" source "${root}/.github/scripts/setproxy.sh" cd "${root}" - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ enter "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 9e4b67eee..273585e8b 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -45,7 +45,7 @@ jobs: root="${BB_ROOT/#\~/$HOME}" source "${root}/.github/scripts/setproxy.sh" cd "${root}" - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ prepare "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" nix build @@ -97,7 +97,7 @@ jobs: root="${BB_ROOT/#\~/$HOME}" source "${root}/.github/scripts/setproxy.sh" cd "${root}" - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ enter "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" @@ -164,7 +164,7 @@ jobs: root="${BB_ROOT/#\~/$HOME}" source "${root}/.github/scripts/setproxy.sh" cd "${root}" - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ enter "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" @@ -220,7 +220,7 @@ jobs: root="${BB_ROOT/#\~/$HOME}" source "${root}/.github/scripts/setproxy.sh" cd "${root}" - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ enter "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" @@ -287,7 +287,7 @@ jobs: root="${BB_ROOT/#\~/$HOME}" source "${root}/.github/scripts/setproxy.sh" cd "${root}" - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ enter "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" @@ -352,7 +352,7 @@ jobs: root="${BB_ROOT/#\~/$HOME}" source "${root}/.github/scripts/setproxy.sh" cd "${root}" - git fetch --force --prune origin "${CHECKOUT_SHA}" + git fetch --no-recurse-submodules --force --prune origin "${CHECKOUT_SHA}" git show "${CHECKOUT_SHA}:.github/scripts/ci_repo_lock.sh" | bash -s -- \ enter "${CHECKOUT_SHA}" "${root}" "${CHECKOUT_REF}" diff --git a/bebop b/bebop index 6db161d4d..a0ea302f2 160000 --- a/bebop +++ b/bebop @@ -1 +1 @@ -Subproject commit 6db161d4df133f42b49c4d922b5905ba07a6a0b8 +Subproject commit a0ea302f218d7a3e96866812f9cb82de384eabce diff --git a/scripts/nix/build-all.sh b/scripts/nix/build-all.sh index 093669b05..95c8fd43c 100755 --- a/scripts/nix/build-all.sh +++ b/scripts/nix/build-all.sh @@ -15,7 +15,7 @@ usage() { echo " 3. RTL pre-compile sources" echo " 4. bb-tests pre-compile sources" echo " 5. waveform-mcp build" - echo " 6. bebop build" + echo " 6. bebop environment" echo " 7. verify build" echo " 8. pre-commit hooks installation" echo " 9. register project MCP" @@ -149,10 +149,9 @@ if run_step "5"; then fi if run_step "6"; then - begin_step "6" "bebop build" + begin_step "6" "bebop environment" cd "$BBDIR/bebop" - nix build - nix develop -c echo "bebop built successfully" + nix develop -c echo "bebop environment ready" fi From e6d7208ff68778ce5a4719f9c909e2840d891fe8 Mon Sep 17 00:00:00 2001 From: shirohasuki Date: Tue, 27 Oct 2026 23:56:57 +0800 Subject: [PATCH 196/196] feat: sync to buckyball 0.2.1-a --- .agents/skills | 2 +- .github/scripts/ci_repo_lock.sh | 9 +- .github/workflows/check.yaml | 60 +- .github/workflows/regression.yml | 56 +- .gitmodules | 16 +- README.md | 7 +- arch/src/csrc/include/monitor/trace.h | 7 +- arch/src/csrc/src/monitor/trace/ctrace.cc | 116 +- .../main/scala/configs/BallDomainParam.scala | 6 +- .../framework/acc/configs/AccParam.scala | 6 +- .../scala/framework/acc/debug/Debug.scala | 2 +- .../acc/frontend/configs/default.toml | 4 +- .../frontend/src/main/scala/Frontend.scala | 4 +- .../src/main/scala/boot/configs/uvm.toml | 4 +- .../main/scala/configs/FrontendParam.scala | 8 +- .../main/scala/decoder/GlobalDecoder.scala | 4 +- .../scheduler/src/main/scala/Scheduler.scala | 4 +- .../main/scala/rob/global/configs/uvm.toml | 4 +- .../rob/global/src/main/scala/GlobalROB.scala | 17 +- .../acc/memdomain/configs/default.toml | 4 +- .../memdomain/src/main/scala/MemDomain.scala | 4 +- .../main/scala/bank_router/configs/uvm.toml | 4 +- .../src/main/scala/BankRouter.scala | 17 +- .../bank_router/src/main/scala/blt/BLT.scala | 103 +- .../main/scala/configs/MemDomainParam.scala | 4 +- .../src/main/scala/ctrl/decoder/Decoder.scala | 16 +- .../src/main/scala/tomem/configs/uvm.toml | 4 +- .../scala/toddr/src/main/scala/ReadDma.scala | 53 +- .../scala/toddr/src/main/scala/WriteDma.scala | 77 +- .../core/execution/pipeline/Sequencer.scala | 13 +- .../src/main/resources/env/model.svh | 13 +- .../src/main/scala/Privileged.scala | 56 +- .../main/resources/preparation/env/model.svh | 18 + .../resources/prepared_map/agent/service.svh | 6 +- .../main/resources/prepared_map/env/env.svh | 14 +- .../preflight/src/main/scala/Preflight.scala | 122 +- .../src/main/scala/PreparedMap.scala | 43 +- .../core/memory/configs/MemoryParam.scala | 2 +- .../src/main/scala/router/configs/uvm.toml | 4 +- .../src/main/scala/rnf/BankedChiCache.scala | 2 +- .../coherence/configs/CoherenceParams.scala | 2 +- .../scala/framework/lib/queue/Queue.scala | 59 +- arch/thirdparty/build.sc | 2 +- arch/uvm.toml | 8 +- bb-tests/workloads/CMakeLists.txt | 69 +- bb-tests/workloads/lib/bbhw/isa/bb_func7.h | 12 +- .../workloads/lib/bbhw/mmio/mmio_allocator.h | 8 +- bb-tests/workloads/scripts/build.py | 140 -- bb-tests/workloads/src/CMakeLists.txt | 28 +- bb-tests/workloads/src/CTest/CMakeLists.txt | 40 +- .../workloads/src/CTest/base/CMakeLists.txt | 19 +- .../src/CTest/base/mset_transfer_test.c | 8 +- .../CTest/base/mvin_2d_pixel_stride_test.c | 16 +- .../workloads/src/CTest/base/mvin_2d_test.c | 3 +- .../src/CTest/base/mvin_mvout_acc_test.c | 3 +- .../src/CTest/base/mvin_mvout_group_test.c | 3 +- .../src/CTest/base/mvin_mvout_stride_test.c | 8 +- .../src/CTest/base/mvin_mvout_test.c | 3 +- .../CTest/base/mvin_mvout_unaligned_test.c | 8 +- .../src/CTest/base/mvin_unaligned_test.c | 8 +- .../src/CTest/coremark/CMakeLists.txt | 5 +- .../src/CTest/embench/CMakeLists.txt | 10 +- bb-tests/workloads/src/CTest/rvv/rvv_kernel.c | 9 +- .../src/CTest/tutorial/CMakeLists.txt | 2 +- .../workloads/src/MLIRTest/CMakeLists.txt | 44 +- .../src/MLIRTest/base/CMakeLists.txt | 2 +- bb-tests/workloads/src/SocTest/CMakeLists.txt | 7 +- .../api/steps/bebop/bemu/02_sim_event.step.py | 4 +- .../api/steps/bebop/bemu/03_batch_api.step.py | 9 - .../steps/bebop/bemu/03_batch_event.step.py | 20 +- .../steps/bebop/bemu/04_analysis_api.step.py | 2 +- .../bebop/bemu/04_analysis_event.step.py | 4 +- .../steps/bebop/bemu/scripts/bank_replay.py | 2 +- .../bebop/bemu/scripts/bank_replay_native.py | 2 +- .../steps/bebop/bemu/scripts/bemu_analysis.py | 15 +- .../bebop/p2e/03_runworkload_event.step.py | 4 +- .../steps/bebop/p2e/04_batch_event.step.py | 13 +- .../steps/bebop/p2e/scripts/resolve_image.py | 27 +- .../api/steps/bebop/vcs/03_batch_api.step.py | 7 - .../steps/bebop/vcs/03_batch_event.step.py | 4 +- .../bebop/verilator/06_batch_api.step.py | 8 - .../bebop/verilator/06_batch_event.step.py | 20 +- bbdev/api/steps/config/01_install_api.step.py | 8 +- .../api/steps/config/01_install_event.step.py | 49 +- bbdev/api/steps/config/scripts/1_toml2json.py | 82 +- .../config/scripts/2_parameter_derivation.py | 27 +- .../api/steps/config/scripts/3_json2proto.py | 140 +- .../steps/config/scripts/ball_parameters.py | 10 +- .../api/steps/config/scripts/hardware_pb2.py | 84 +- .../steps/config/scripts/proto/design.proto | 64 +- .../steps/config/scripts/proto/hardware.proto | 6 +- bbdev/api/steps/dc/01_area_event.step.py | 4 +- bbdev/api/steps/kernel/01_build_event.step.py | 32 +- bbdev/api/steps/model/01_build_api.step.py | 4 +- bbdev/api/steps/model/01_build_event.step.py | 1 + bbdev/api/steps/model/scripts/build.py | 19 +- bbdev/api/steps/stack/01_build_api.step.py | 4 +- bbdev/api/steps/stack/scripts/build.py | 52 +- bbdev/api/steps/uvm/01_build_api.step.py | 16 +- bbdev/api/steps/uvm/01_build_event.step.py | 9 +- bbdev/api/steps/uvm/02_run_api.step.py | 16 +- bbdev/api/steps/uvm/02_run_event.step.py | 9 +- bbdev/api/steps/uvm/scripts/ball.py | 109 +- bbdev/api/steps/uvm/scripts/config.py | 11 +- bbdev/api/steps/uvm/scripts/report.py | 8 +- bbdev/api/steps/uvm/scripts/run.py | 4 +- bbdev/api/steps/uvm/scripts/uvm_common.py | 143 +- .../api/steps/workload/01_build_event.step.py | 52 +- bbdev/api/utils/event_common.py | 22 +- bbdev/api/utils/path.py | 25 +- bbdev/api/utils/reports.py | 8 +- bbdev/api/utils/workload_manifest.py | 4 - bbdev/bbdev | 118 +- bbdev/mcp/common.py | 18 +- bbdev/mcp/server.py | 9 +- bbdev/mcp/tools/bebop_p2e_batch.py | 10 +- bbdev/mcp/tools/bebop_p2e_buildbitstream.py | 6 +- bbdev/mcp/tools/bebop_p2e_clean.py | 6 +- bbdev/mcp/tools/bebop_p2e_runworkload.py | 6 +- bbdev/mcp/tools/bebop_p2e_verilog.py | 6 +- bbdev/mcp/tools/bebop_verilator_batch.py | 10 +- bbdev/mcp/tools/bebop_verilator_build.py | 6 +- bbdev/mcp/tools/bebop_verilator_clean.py | 6 +- bbdev/mcp/tools/bebop_verilator_run.py | 6 +- bbdev/mcp/tools/bebop_verilator_sim.py | 6 +- bbdev/mcp/tools/bebop_verilator_verilog.py | 9 +- bbdev/mcp/tools/bemu_analysis.py | 6 +- bbdev/mcp/tools/bemu_batch.py | 10 +- bbdev/mcp/tools/bemu_sim.py | 8 +- bbdev/mcp/tools/dc_area.py | 6 +- bbdev/mcp/tools/dc_power.py | 6 +- bbdev/mcp/tools/dc_verilog.py | 6 +- bbdev/mcp/tools/firesim_buildbitstream.py | 6 +- bbdev/mcp/tools/firesim_enumeratefpgas.py | 6 +- bbdev/mcp/tools/firesim_infrasetup.py | 6 +- bbdev/mcp/tools/firesim_runworkload.py | 6 +- bbdev/mcp/tools/ip_generate.py | 6 +- bbdev/mcp/tools/ip_replace.py | 6 +- bbdev/mcp/tools/kernel_build.py | 6 +- bbdev/mcp/tools/model_build.py | 9 +- bbdev/mcp/tools/stack_build.py | 6 +- bbdev/mcp/tools/uvm_build.py | 6 +- bbdev/mcp/tools/uvm_run.py | 6 +- bbdev/mcp/tools/validate.py | 8 +- bbdev/mcp/tools/verilator_build.py | 6 +- bbdev/mcp/tools/verilator_clean.py | 6 +- bbdev/mcp/tools/verilator_run.py | 6 +- bbdev/mcp/tools/verilator_sim.py | 6 +- bbdev/mcp/tools/verilator_verilog.py | 6 +- bbdev/mcp/tools/workload_build.py | 6 +- bbdev/mcp/tools/workload_clean.py | 6 +- bbdev/mcp/tools/workload_tohex.py | 6 +- bbdev/mcp/tools/yosys_run.py | 6 +- bbdev/mcp/tools/yosys_synth.py | 6 +- bbdev/mcp/tools/yosys_verilog.py | 6 +- bebop | 2 +- examples/build.sc | 8 +- .../balls/f32add/stack/src/Lowering.cpp | 55 - .../mlir_tests/ball/f32add_invalid.mlir | 24 - .../balls/gemmini/stack/src/Lowering.cpp | 116 -- .../balls/im2col/stack/src/Lowering.cpp | 79 - .../mlir_tests/ball/im2col_6x6_k3.mlir | 43 - .../mlir_tests/ball/im2col_6x6_k3_main.cpp | 71 - .../workloads/mlir_tests/bank/CMakeLists.txt | 17 - .../im2col_k3_padded_bank_windows.mlir.in | 32 - .../im2col_k3_padded_bank_windows_main.cpp | 34 - .../balls/int2fp/stack/src/Lowering.cpp | 49 - .../balls/int8add/stack/src/Lowering.cpp | 66 - .../balls/int8mul/stack/src/Lowering.cpp | 57 - .../balls/lut/stack/src/Lowering.cpp | 45 - .../balls/matadd/stack/src/Lowering.cpp | 49 - .../balls/maxpool/stack/src/Lowering.cpp | 68 - .../workloads/mlir_tests/bank/CMakeLists.txt | 15 - .../balls/mxfp2int/emu/src/55_mxfp2int.rs | 103 - .../mxfp2int/workloads/ctests/CMakeLists.txt | 1 - .../balls/mxfp2int/workloads/isa/mxfp2int.h | 23 - .../workloads/mlir_tests/ball/CMakeLists.txt | 5 - .../ball/mxfp4_unit_scale_2blocks.mlir | 35 - .../workloads/mlir_tests/bank/CMakeLists.txt | 16 - .../bank/mxfp4_unit_scale_bank.mlir.in | 37 - .../bank/mxfp4_unit_scale_bank_main.cpp | 16 - .../workloads/mlir_tests/ball/CMakeLists.txt | 32 - .../workloads/mlir_tests/ball/mxmm_f32.mlir | 90 - .../mlir_tests/ball/mxmm_f32_main.cpp | 3 - .../workloads/mlir_tests/ball/mxmm_fma32.mlir | 90 - .../mlir_tests/ball/mxmm_fma32_main.cpp | 3 - .../workloads/mlir_tests/ball/mxmm_mxfp8.mlir | 90 - .../mlir_tests/ball/mxmm_mxfp8_main.cpp | 3 - .../mlir_tests/ball/mxmm_window.mlir | 28 - .../mlir_tests/ball/mxmm_window_chain.mlir | 8 - .../ball/mxmm_window_chain_main.cpp | 70 - .../mlir_tests/ball/mxmm_window_main.cpp | 88 - .../mlir_tests/ball/mxmm_window_rows.mlir | 8 - .../mlir_tests/ball/mxmm_window_rows_main.cpp | 88 - .../workloads/mlir_tests/bank/CMakeLists.txt | 32 - .../workloads/mlir_tests/bank/mxmm_f32.mlir | 90 - .../mlir_tests/bank/mxmm_f32_main.cpp | 3 - .../workloads/mlir_tests/bank/mxmm_fma32.mlir | 90 - .../mlir_tests/bank/mxmm_fma32_main.cpp | 3 - .../workloads/mlir_tests/bank/mxmm_mxfp8.mlir | 90 - .../mlir_tests/bank/mxmm_mxfp8_main.cpp | 3 - .../mlir_tests/bank/mxmm_window.mlir | 30 - .../mlir_tests/bank/mxmm_window_chain.mlir | 10 - .../bank/mxmm_window_chain_main.cpp | 70 - .../mlir_tests/bank/mxmm_window_main.cpp | 88 - .../mlir_tests/bank/mxmm_window_rows.mlir | 10 - .../mlir_tests/bank/mxmm_window_rows_main.cpp | 88 - .../workloads/mlir_tests/mxmm_native_steps.h | 33 - .../workloads/mlir_tests/mxmm_native_test.h | 89 - .../workloads/mlir_tests/CMakeLists.txt | 5 - .../workloads/mlir_tests/ball/CMakeLists.txt | 12 - .../workloads/mlir_tests/ball/mxquant.mlir | 22 - .../mlir_tests/ball/mxquant_main.cpp | 86 - .../workloads/mlir_tests/bank/CMakeLists.txt | 12 - .../workloads/mlir_tests/bank/mxquant.mlir | 27 - .../mlir_tests/bank/mxquant_main.cpp | 86 - .../balls/redmule/stack/src/Lowering.cpp | 66 - .../redmule/workloads/ctests/CMakeLists.txt | 17 - .../mlir_tests/ball/redmule_f16_chain.mlir | 6 - .../balls/relu/stack/src/Lowering.cpp | 54 - .../balls/smatmul/stack/src/Lowering.cpp | 75 - .../balls/toint8/stack/src/Lowering.cpp | 84 - .../components/balls/trace/arch/README.md | 127 -- .../balls/trace/emu/src/04_bdb_counter.rs | 15 - .../balls/trace/stack/src/Lowering.cpp | 66 - .../ball/bdb_counter_start_read_stop.mlir | 8 - .../balls/transpose/stack/src/Lowering.cpp | 55 - .../balls/vector/workloads/isa/vecmat16.h | 12 - .../workloads/mlir_tests/CMakeLists.txt | 15 - .../workloads/mlir_tests/ball/CMakeLists.txt | 6 - .../ball/vecmat16_atranspose_16x16.mlir | 57 - .../vecmat16_atranspose_bank_depth.mlir.in | 62 - .../components/cores/xiangshan/arch/build.sc | 41 - examples/designs/goban/arch/build.sc | 2 +- .../goban/configs/device/core/default.toml | 10 +- .../device/core/memory/access/goban.toml | 2 +- .../device/core/memory/dcache/shared.toml | 2 +- .../device/core/memory/dma-cache/shared.toml | 2 +- .../device/core/memory/home/goban.toml | 2 +- .../device/core/memory/icache/shared.toml | 2 +- .../designs/goban/configs/sims/default.toml | 3 + .../designs/goban/configs/sims/verilator.toml | 1 + .../designs/goban/sims/device/emu/build.rs | 5 +- .../designs/goban/sims/device/emu/src/lib.rs | 2 +- .../regression/bemu/workloads-linux.toml | 2 +- .../regression/p2e/workloads-linux.toml | 2 +- .../goban/sims/device/stack/CMakeLists.txt | 2 +- .../device/stack/core/goban/CMakeLists.txt | 6 - .../sims/device/workloads/CMakeLists.txt | 11 +- .../runtime => stack/device}/CMakeLists.txt | 6 +- .../goban/stack/device/core/CMakeLists.txt | 32 + .../designs/goban/stack/engine/CMakeLists.txt | 14 - .../designs/goban/stack/midend/CMakeLists.txt | 8 - .../device/core/engine/CMakeLists.txt | 8 +- .../device/core/engine/matmul_threads.c | 2 +- .../device/core/engine/mvover_rows.c | 18 +- examples/designs/multi-rocket/arch/build.sc | 2 +- .../configs/device/core/default.toml | 2 +- .../device/core/memory/access/core.toml | 2 +- .../device/core/memory/dcache/shared.toml | 2 +- .../configs/device/core/memory/home/core.toml | 2 +- .../device/core/memory/icache/shared.toml | 2 +- .../multi-rocket/configs/sims/default.toml | 3 + .../multi-rocket/configs/sims/verilator.toml | 1 + .../multi-rocket/sims/device/emu/Cargo.toml | 2 +- .../multi-rocket/sims/device/emu/build.rs | 5 +- .../multi-rocket/sims/device/emu/src/lib.rs | 2 +- .../regression/bemu/workloads-linux.toml | 2 +- .../regression/p2e/workloads-linux.toml | 2 +- .../sims/device/stack/CMakeLists.txt | 2 +- .../sims/device/workloads/CMakeLists.txt | 11 +- .../runtime => stack/device}/CMakeLists.txt | 4 +- .../device/core/ctests/CMakeLists.txt | 5 +- .../workloads/device/soc/CMakeLists.txt | 7 +- .../workloads/device/soc/coherent_exchange.c | 11 +- .../core/acc/frontend/bat-off-rob24.toml | 4 +- .../device/core/acc/frontend/rob24.toml | 4 +- .../device/core/acc/memdomain/blt-off.toml | 5 +- .../pebble/configs/device/core/default.toml | 2 +- .../device/core/memory/access/pebble.toml | 2 +- .../device/core/memory/dcache/pebble.toml | 2 +- .../device/core/memory/home/pebble.toml | 2 +- .../device/core/memory/icache/pebble.toml | 2 +- .../designs/pebble/configs/sims/default.toml | 3 + .../pebble/configs/sims/verilator.toml | 1 + .../designs/pebble/sims/device/emu/build.rs | 5 +- .../designs/pebble/sims/device/emu/src/lib.rs | 2 +- .../regression/bemu/workloads-bare.toml | 4 +- .../regression/bemu/workloads-linux.toml | 2 +- .../device/regression/p2e/workloads-bare.toml | 18 +- .../pebble/sims/device/stack/CMakeLists.txt | 2 +- .../device/stack/core/pebble/CMakeLists.txt | 6 - .../sims/device/tapeout/area/constraints.sdc | 96 +- .../pebble/sims/device/tapeout/area/dc.tcl | 2 +- .../sims/device/workloads/CMakeLists.txt | 11 +- .../device/workloads/kernel/CMakeLists.txt | 4 +- .../runtime => stack/device}/CMakeLists.txt | 6 +- .../pebble/stack/device/core/CMakeLists.txt | 61 + .../core/src/ConvertLinalgToBuckyball.cpp | 197 ++ .../core}/src/FormCustomOps.cpp | 101 + .../core}/src/FormSpatialOps.cpp | 229 +- .../core/src/LowerChannelOps.cpp} | 152 +- .../stack/device/core/src/LowerLinear.cpp | 498 +++++ .../stack/device/core/src/LowerSpatialOps.cpp | 521 +++++ .../pebble/stack/device/core/src/Lowering.h | 81 + .../core}/tests/form-custom-ops.mlir | 0 .../core}/tests/form-depthwise.mlir | 0 .../core}/tests/form-int32-bias.mlir | 0 .../device/core/tests/lower-bank-updates.mlir | 7 + .../core}/tests/lower-custom-ops.mlir | 19 +- .../core}/tests/lower-frontend-types.mlir | 0 .../core}/tests/unknown-custom-op.mlir | 0 .../pebble/stack/engine/CMakeLists.txt | 13 - .../pebble/stack/midend/CMakeLists.txt | 28 - .../midend/src/ConvertLinalgToBuckyball.cpp | 312 --- .../stack/midend/src/LowerLinearTiles.cpp | 264 --- .../stack/midend/src/LowerSpatialOps.cpp | 571 ----- .../pebble/stack/midend/src/Lowering.h | 64 - .../midend/tests/lower-bank-updates.mlir | 14 - .../designs/pebble/workloads/common/pebble.h | 5 +- .../workloads/device/core/acc/CMakeLists.txt | 4 +- .../device/core/acc/ctests/CMakeLists.txt | 2 +- .../acc/ctests/bank_capacity_rename_test.c | 25 +- .../bank_dma_pingpong-main.cpp | 2 +- .../acc/ctests/bank_dma_pingpong/pebble.c | 16 +- .../bank_dma_reuse/bank_dma_reuse-main.cpp | 2 +- .../core/acc/ctests/bank_dma_reuse/pebble.c | 18 +- .../core/acc/ctests/bank_recycle_test.c | 1 - .../bank_rename_overlap-main.cpp | 2 +- .../acc/ctests/bank_rename_overlap/pebble.c | 16 +- .../acc/ctests/bank_transfer_recycle_test.c | 1 - .../ctests/im2col_mvin_smatmul_hazard_test.c | 4 +- .../im2col_smatmul_quant_pipeline-main.cpp | 2 +- .../im2col_smatmul_quant_pipeline/pebble.c | 24 +- .../core/acc/ctests/maxpool_im2col_test.c | 2 +- .../maxpool_partial_update-main.cpp | 2 +- .../ctests/maxpool_partial_update/pebble.c | 8 +- .../acc/ctests/quant_smatmul_int2fp_test.c | 4 +- .../core/acc/ctests/read_bank_shared/pebble.c | 14 +- .../read_bank_shared-main.cpp | 2 +- .../core/acc/ctests/read_bank_split/pebble.c | 14 +- .../read_bank_split/read_bank_split-main.cpp | 2 +- .../core/acc/ctests/residual_add_lut_test.c | 2 +- .../core/acc/ctests/rob_window_probe/pebble.c | 16 +- .../rob_window_probe-main.cpp | 2 +- .../device/core/acc/ctests/runtime_job_test.c | 3 +- .../acc/ctests/same_ball_independent/pebble.c | 18 +- .../same_ball_independent-main.cpp | 2 +- .../device/core/acc/ctests/se_gate_lut_test.c | 2 +- .../acc/ctests/shared_bank_versions/pebble.c | 16 +- .../shared_bank_versions-main.cpp | 2 +- .../ctests/smatmul_mvout_bank_reuse_test.c | 4 +- .../acc/ctests/write_bank_versions/pebble.c | 14 +- .../write_bank_versions-main.cpp | 2 +- .../bank/quant_im2col_k1_single_pixel.mlir | 14 +- .../quant_im2col_k1_single_pixel_main.cpp | 16 +- .../core/acc/mlir_tests/core/CMakeLists.txt | 54 + .../acc/mlir_tests/core/linear_chain_i8.mlir | 5 + .../core/acc/mlir_tests/core/mega_conv2d.mlir | 12 +- .../core/mega_conv2d_flatten_linear.mlir | 93 + .../core/mega_conv2d_flatten_linear_main.cpp | 64 + .../mlir_tests/core/mega_conv2d_resident.mlir | 100 + .../core/mega_conv2d_resident_main.cpp | 73 + .../mega_global_avg_7x7_c288_exact_main.cpp | 9 +- .../core/mega_global_avg_conv2_exact_main.cpp | 9 +- examples/designs/poly/arch/build.sc | 2 +- .../device/cores/core0/acc/frontend/poly.toml | 4 +- .../cores/core0/acc/memdomain/poly.toml | 5 +- .../configs/device/cores/core0/default.toml | 2 +- .../device/cores/core0/dmaCache/core.toml | 2 +- .../cores/core0/memory/access/core.toml | 2 +- .../cores/core0/memory/dcache/core.toml | 2 +- .../device/cores/core0/memory/home/core.toml | 2 +- .../cores/core0/memory/icache/core.toml | 2 +- .../configs/device/cores/core1/default.toml | 12 +- .../device/cores/core1/dmaCache/core.toml | 2 +- .../cores/core1/memory/access/core.toml | 2 +- .../cores/core1/memory/dcache/core.toml | 2 +- .../device/cores/core1/memory/home/core.toml | 2 +- .../cores/core1/memory/icache/core.toml | 2 +- .../configs/device/cores/core2/default.toml | 12 +- .../device/cores/core2/dmaCache/core.toml | 2 +- .../cores/core2/memory/access/core.toml | 2 +- .../cores/core2/memory/dcache/core.toml | 2 +- .../device/cores/core2/memory/home/core.toml | 2 +- .../cores/core2/memory/icache/core.toml | 2 +- .../configs/device/cores/core3/default.toml | 12 +- .../device/cores/core3/dmaCache/core.toml | 2 +- .../cores/core3/memory/access/core.toml | 2 +- .../cores/core3/memory/dcache/core.toml | 2 +- .../device/cores/core3/memory/home/core.toml | 2 +- .../cores/core3/memory/icache/core.toml | 2 +- .../configs/device/cores/core4/default.toml | 12 +- .../device/cores/core4/dmaCache/core.toml | 2 +- .../cores/core4/memory/access/core.toml | 2 +- .../cores/core4/memory/dcache/core.toml | 2 +- .../device/cores/core4/memory/home/core.toml | 2 +- .../cores/core4/memory/icache/core.toml | 2 +- .../configs/device/cores/core5/default.toml | 12 +- .../device/cores/core5/dmaCache/core.toml | 2 +- .../cores/core5/memory/access/core.toml | 2 +- .../cores/core5/memory/dcache/core.toml | 2 +- .../device/cores/core5/memory/home/core.toml | 2 +- .../cores/core5/memory/icache/core.toml | 2 +- .../configs/device/cores/core6/default.toml | 12 +- .../device/cores/core6/dmaCache/core.toml | 2 +- .../cores/core6/memory/access/core.toml | 2 +- .../cores/core6/memory/dcache/core.toml | 2 +- .../device/cores/core6/memory/home/core.toml | 2 +- .../cores/core6/memory/icache/core.toml | 2 +- .../configs/device/cores/core7/default.toml | 12 +- .../device/cores/core7/dmaCache/core.toml | 2 +- .../cores/core7/memory/access/core.toml | 2 +- .../cores/core7/memory/dcache/core.toml | 2 +- .../device/cores/core7/memory/home/core.toml | 2 +- .../cores/core7/memory/icache/core.toml | 2 +- .../configs/device/cores/core8/default.toml | 12 +- .../device/cores/core8/dmaCache/core.toml | 2 +- .../cores/core8/memory/access/core.toml | 2 +- .../cores/core8/memory/dcache/core.toml | 2 +- .../device/cores/core8/memory/home/core.toml | 2 +- .../cores/core8/memory/icache/core.toml | 2 +- .../designs/poly/configs/sims/default.toml | 6 + examples/designs/poly/configs/sims/p2e.toml | 1 + .../designs/poly/configs/sims/tapeout.toml | 1 + .../designs/poly/configs/sims/verilator.toml | 9 + .../designs/poly/sims/device/emu/Cargo.toml | 2 +- .../designs/poly/sims/device/emu/build.rs | 5 +- .../designs/poly/sims/device/emu/src/lib.rs | 2 +- .../regression/bemu/workloads-bare.toml | 14 +- .../regression/bemu/workloads-linux.toml | 2 +- .../regression/p2e/workloads-linux.toml | 2 +- .../regression/verilator/workloads-bare.toml | 4 - .../poly/sims/device/stack/CMakeLists.txt | 2 +- .../device/stack/core/poly/CMakeLists.txt | 6 - .../poly/sims/device/workloads/CMakeLists.txt | 17 +- .../runtime => stack/device}/CMakeLists.txt | 6 +- .../poly/stack/device/core/CMakeLists.txt | 32 + .../stack/{engine => device/core}/execute.c | 0 .../designs/poly/stack/engine/CMakeLists.txt | 14 - .../designs/poly/stack/midend/CMakeLists.txt | 8 - .../designs/poly/workloads/CMakeLists.txt | 9 + .../workloads/device/core/ball/CMakeLists.txt | 2 +- .../device/core/engine/CMakeLists.txt | 25 +- .../device/core/engine/ant_bank_exercise.c | 24 +- .../device/core/engine/ant_bank_main.c | 15 +- .../core/engine/ant_bank_rows_exercise.c | 16 +- .../device/core/engine/ant_bank_rows_main.c | 15 +- .../device/core/engine/ant_highpa_main.c | 8 +- .../device/core/engine/ant_mxmm_main.cpp | 24 +- .../core/engine/ant_parallel_exercise.c | 12 +- .../device/core/engine/ant_parallel_main.c | 14 +- .../device/core/engine/ant_shared_exercise.c | 4 +- .../device/core/engine/ant_shared_main.c | 14 +- .../device/core/engine/mlir/CMakeLists.txt | 437 ++++ .../core/engine/mlir/ball_mxmm_f32_main.cpp | 88 + .../core/engine/mlir/ball_mxmm_fma32_main.cpp | 91 + .../mlir/ball_mxmm_mxfp8_chain_main.cpp | 96 + .../mlir/ball_mxmm_mxfp8_window_main.cpp | 87 + .../mlir/ball_mxquant_two_blocks_main.cpp | 92 + .../core/engine/mlir/bank_mxmm_f32_main.cpp | 88 + .../core/engine/mlir/bank_mxmm_fma32_main.cpp | 91 + .../mlir/bank_mxmm_mxfp8_chain_main.cpp | 88 + .../mlir/bank_mxmm_mxfp8_window_main.cpp | 87 + .../mlir/bank_mxquant_two_blocks_main.cpp | 92 + .../device/core/thread/CMakeLists.txt | 2 + .../device/core/thread/ctests/CMakeLists.txt | 28 +- .../device/core/thread/ctests/ant_bank.c | 14 +- .../device/core/thread/ctests/ant_bank_rows.c | 10 +- .../device/core/thread/ctests/ant_corelink.c | 15 +- .../core/thread/ctests/ant_corelink_bank.c | 15 +- .../core/thread/ctests/ant_corelink_reduce.c | 9 +- .../device/core/thread/ctests/ant_mxmm.c | 7 +- .../ctests/mesh_private_bank_move_test.c | 1 - .../core/thread/ctests/mesh_shared_mem_test.c | 18 +- .../device/core/thread/mlir/CMakeLists.txt | 353 ++++ .../core/thread/mlir/ball_mxmm_f32_entry.c | 20 + .../core/thread/mlir/ball_mxmm_fma32_entry.c | 20 + .../thread/mlir/ball_mxmm_mxfp8_chain_entry.c | 24 + .../mlir/ball_mxmm_mxfp8_window_entry.c | 20 + .../mlir/ball_mxquant_two_blocks_entry.c | 17 + .../core/thread/mlir/bank_mxmm_f32_entry.c | 25 + .../core/thread/mlir/bank_mxmm_fma32_entry.c | 25 + .../thread/mlir/bank_mxmm_mxfp8_chain_entry.c | 25 + .../mlir/bank_mxmm_mxfp8_window_entry.c | 25 + .../mlir/bank_mxquant_two_blocks_entry.c | 23 + .../workloads/device/core/thread/mlir/mlir.h | 20 + .../workloads/device/runtime/CMakeLists.txt | 49 +- .../device/runtime/ant_coherence_main.c | 25 +- .../device/runtime/ant_corelink_bank_main.c | 23 +- .../device/runtime/ant_corelink_main.c | 23 +- .../runtime/ant_corelink_reduce_control.c | 1 - .../device/runtime/ant_corelink_reduce_main.c | 38 +- .../workloads/device/runtime/ant_mesh_main.c | 19 +- .../device/runtime/ant_mesh_move_main.c | 14 +- .../device/runtime/ctests/corelink_lease.c | 3 +- .../workloads/device/runtime/stream_main.c | 3 +- .../poly/workloads/device/soc/CMakeLists.txt | 10 +- .../poly/workloads/device/soc/controllers.c | 4 +- examples/designs/toy/arch/build.sc | 2 +- .../device/core/acc/balldomain/default.toml | 20 + .../device/core/acc/frontend/demo.toml | 4 +- .../configs/device/core/acc/frontend/toy.toml | 4 +- .../device/core/acc/memdomain/paper.toml | 5 +- .../device/core/acc/memdomain/toy.toml | 5 +- .../toy/configs/device/core/default.toml | 2 +- .../device/core/memory/access/core.toml | 2 +- .../device/core/memory/dcache/core.toml | 2 +- .../configs/device/core/memory/home/core.toml | 2 +- .../device/core/memory/icache/core.toml | 2 +- .../designs/toy/configs/sims/default.toml | 3 + .../designs/toy/configs/sims/verilator.toml | 1 + examples/designs/toy/sims/device/emu/build.rs | 5 +- .../designs/toy/sims/device/emu/src/lib.rs | 2 +- .../device/emu/tests/instance_isolation.rs | 18 +- .../regression/bemu/workloads-bare.toml | 9 + .../regression/bemu/workloads-linux.toml | 2 +- .../regression/p2e/workloads-linux.toml | 2 +- .../regression/verilator/workloads-bare.toml | 7 + .../toy/sims/device/stack/CMakeLists.txt | 2 +- .../sims/device/stack/core/toy/CMakeLists.txt | 6 - .../toy/sims/device/tapeout/area/dc.tcl | 2 +- .../toy/sims/device/workloads/CMakeLists.txt | 11 +- .../runtime => stack/device}/CMakeLists.txt | 6 +- .../toy/stack/device/core/CMakeLists.txt | 36 + .../core}/src/ConvertLinalgToBuckyball.cpp | 19 +- .../core}/src/FormCustomOps.cpp | 0 .../designs/toy/stack/engine/CMakeLists.txt | 13 - .../designs/toy/stack/midend/CMakeLists.txt | 13 - .../workloads/device/core/acc/CMakeLists.txt | 4 +- .../device/core/acc/ctests/CMakeLists.txt | 4 +- .../gemmini_relu_trace_pipeline-main.cpp | 2 +- .../ctests/gemmini_relu_trace_pipeline/toy.c | 3 +- .../device/core/acc/ctests/rvv_ball.c | 11 +- .../toy/workloads/device/soc/CMakeLists.txt | 2 +- .../{components => prefab}/balls/build.sc | 0 .../balls/f32add/arch/F32AddBall.scala | 2 +- .../balls/f32add/arch/F32AddRow.scala | 0 .../balls/f32add/configs/default.toml | 0 .../balls/f32add/configs/f32add_params.proto | 0 .../balls/f32add/emu/src/add.rs | 3 +- .../balls/f32add/emu/src/dispatch.rs | 0 .../balls/f32add/emu/src/lib.rs | 0 .../balls/f32add/stack/src/f32addOps.cpp | 68 + .../prefab/balls/f32add/stack/src/f32addOps.h | 20 + .../balls/f32add/stack/src/f32addOps.td | 44 + .../balls/f32add/verify/casegen/Cargo.lock | 7 + .../balls/f32add/verify/casegen/Cargo.toml | 10 + .../f32add/verify/casegen/src/casegen.rs | 135 ++ .../balls/f32add/verify/casegen/src/lib.rs | 71 + .../balls/f32add/verify/casegen/src/model.rs | 50 + .../f32add/verify/filelists/f32add_ball.f | 29 + .../f32add/verify/src/common/f32add_defs.svh | 30 + .../f32add/verify/src/common/f32add_items.svh | 63 + .../f32add/verify/src/cov/f32add_cov.svh | 42 + .../f32add/verify/src/env/f32add_env.svh | 27 + .../verify/src/env/f32add_scoreboard.svh | 171 ++ .../balls/f32add/verify/src/pkg/f32add_pkg.sv | 13 + .../verify/src/seq/f32add_sequences.svh | 17 + .../prefab/balls/f32add/verify/src/tb_top.sv | 89 + .../f32add/verify/src/tests/f32add_test.svh | 49 + .../f32add/workloads/ctests/CMakeLists.txt | 0 .../f32add/workloads/ctests/f32add_groups.c | 3 +- .../balls/f32add/workloads/isa/f32add.h | 6 +- .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/f32add_groups.mlir | 4 +- .../mlir_tests/ball/f32add_groups_main.cpp | 3 +- .../mlir_tests/ball/f32add_invalid.mlir | 18 + .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../mlir_tests/bank/f32add_groups.mlir | 4 +- .../mlir_tests/bank/f32add_groups_main.cpp | 0 .../mlir_tests/bank/f32add_invalid.mlir | 21 +- .../balls/gemmini/arch/GemminiBall.scala | 0 .../balls/gemmini/arch/GemminiExCtrl.scala | 0 .../arch/GemminiExCtrlAccumulateState.scala | 0 .../gemmini/arch/GemminiExCtrlCmdStates.scala | 4 +- .../arch/GemminiExCtrlComputeFeedState.scala | 0 .../arch/GemminiExCtrlComputeReadState.scala | 0 .../gemmini/arch/GemminiExCtrlDefaults.scala | 0 .../gemmini/arch/GemminiExCtrlDefs.scala | 0 .../balls/gemmini/arch/GemminiExCtrlFsm.scala | 0 .../arch/GemminiExCtrlPreloadStates.scala | 0 .../gemmini/arch/GemminiExCtrlStoreOps.scala | 0 .../balls/gemmini/arch/LICENSE.gemmini | 0 .../balls/gemmini/arch/Loop.scala | 0 .../balls/gemmini/arch/LoopCmd.scala | 0 .../balls/gemmini/arch/LoopConvAddrGen.scala | 0 .../balls/gemmini/arch/LoopConvUnroller.scala | 0 .../gemmini/arch/LoopMatmulUnroller.scala | 0 .../gemmini/configs/GemminiBallParam.scala | 0 .../balls/gemmini/configs/default.toml | 0 .../gemmini/configs/gemmini_params.proto | 0 .../gemmini/emu/src/02_gemmini_config.rs | 0 .../balls/gemmini/emu/src/03_gemmini_flush.rs | 0 .../gemmini/emu/src/21_gemmini_preload.rs | 3 +- .../emu/src/66_gemmini_compute_preloaded.rs | 3 +- .../emu/src/67_gemmini_compute_accumulated.rs | 7 +- .../gemmini/emu/src/80_gemmini_loop_ws.rs | 0 .../emu/src/96_gemmini_loop_conv_ws.rs | 0 .../balls/gemmini/emu/src/dispatch.rs | 0 .../balls/gemmini/emu/src/gemmini_state.rs | 0 .../balls/gemmini/emu/src/lib.rs | 0 .../balls/gemmini/emu/src/loop_micro_ops.rs | 7 +- .../balls/gemmini/stack/src/gemminiOps.cpp | 115 + .../balls/gemmini/stack/src/gemminiOps.h | 20 + .../balls/gemmini/stack/src/gemminiOps.td | 62 + .../balls/gemmini/verify/casegen/Cargo.lock | 0 .../balls/gemmini/verify/casegen/Cargo.toml | 0 .../balls/gemmini/verify/casegen/src/lib.rs | 0 .../gemmini/verify/filelists/gemmini_ball.f | 0 .../verify/src/common/gemmini_defs.svh | 0 .../verify/src/gemmini_expansion_if.sv | 0 .../gemmini/verify/src/pkg/gemmini_pkg.sv | 0 .../balls/gemmini/verify/src/tb_top.sv | 0 .../gemmini/verify/src/tests/gemmini_test.svh | 0 .../gemmini/workloads/ctests/CMakeLists.txt | 0 .../ctests/gemmini_accumulated_16x16_test.c | 0 .../workloads/ctests/gemmini_loop_mmio.c | 8 +- .../workloads/ctests/gemmini_loop_subrob.c | 3 +- .../ctests/gemmini_os_cisc_atranspose_test.c | 0 .../ctests/gemmini_os_cisc_basic_test.c | 0 .../ctests/gemmini_os_cisc_btranspose_test.c | 0 .../ctests/gemmini_os_cisc_shift_test.c | 0 .../ctests/gemmini_os_risc_abtranspose_test.c | 0 .../ctests/gemmini_os_risc_atranspose_test.c | 0 .../ctests/gemmini_os_risc_basic_test.c | 0 .../ctests/gemmini_os_risc_btranspose_test.c | 0 .../ctests/gemmini_os_risc_shift_test.c | 0 .../ctests/gemmini_ws_cisc_conv_test.c | 0 .../ctests/gemmini_ws_risc_basic_test.c | 0 .../ctests/gemmini_ws_risc_btranspose_test.c | 0 .../ctests/gemmini_ws_risc_shift_test.c | 0 .../balls/gemmini/workloads/isa/gemmini.h | 0 .../isa/gemmini_compute_accumulated.h | 2 +- .../workloads/isa/gemmini_compute_preloaded.h | 2 +- .../gemmini/workloads/isa/gemmini_config.h | 2 +- .../gemmini/workloads/isa/gemmini_flush.h | 3 +- .../workloads/isa/gemmini_loop_conv_ws.h | 20 +- .../gemmini/workloads/isa/gemmini_loop_ws.h | 16 +- .../gemmini/workloads/isa/gemmini_preload.h | 2 +- .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../gemmini_accumulated_command_16x16.mlir | 2 +- ...gemmini_accumulated_command_16x16_main.cpp | 0 .../ball/gemmini_preloaded_command_16x16.mlir | 2 +- .../gemmini_preloaded_command_16x16_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 6 +- ...emmini_compute_accumulated_bank_tiles.mlir | 2 +- ...ni_compute_accumulated_bank_tiles_main.cpp | 3 +- .../gemmini_compute_preloaded_bank_tiles.mlir | 2 +- ...mini_compute_preloaded_bank_tiles_main.cpp | 0 .../balls/im2col/arch/Im2col.scala | 0 .../balls/im2col/arch/Im2colBall.scala | 0 .../balls/im2col/arch/README.md | 0 .../balls/im2col/arch/RestoringDiv.scala | 0 .../im2col/configs/Im2colBallParam.scala | 0 .../balls/im2col/configs/default.toml | 0 .../balls/im2col/configs/im2col_params.proto | 0 .../balls/im2col/emu/src/dispatch.rs | 0 .../balls/im2col/emu/src/im2col.rs | 3 +- .../balls/im2col/emu/src/lib.rs | 0 .../balls/im2col/stack/src/im2colOps.cpp | 85 + .../prefab/balls/im2col/stack/src/im2colOps.h | 20 + .../balls/im2col/stack/src/im2colOps.td | 52 + .../balls/im2col/verify/README.md | 0 .../balls/im2col/verify/casegen/Cargo.lock | 0 .../balls/im2col/verify/casegen/Cargo.toml | 0 .../im2col/verify/casegen/src/casegen.rs | 0 .../balls/im2col/verify/casegen/src/lib.rs | 0 .../balls/im2col/verify/casegen/src/model.rs | 0 .../im2col/verify/filelists/im2col_ball.f | 0 .../im2col/verify/src/common/im2col_defs.svh | 0 .../im2col/verify/src/common/im2col_items.svh | 0 .../im2col/verify/src/cov/im2col_cov.svh | 0 .../im2col/verify/src/env/im2col_env.svh | 0 .../verify/src/env/im2col_scoreboard.svh | 0 .../balls/im2col/verify/src/pkg/im2col_pkg.sv | 0 .../verify/src/seq/im2col_sequences.svh | 0 .../balls/im2col/verify/src/tb_top.sv | 0 .../im2col/verify/src/tests/im2col_test.svh | 0 .../im2col/workloads/ctests/CMakeLists.txt | 0 .../im2col/workloads/ctests/im2col_k1_test.c | 0 .../ctests/im2col_k3_asym_pad_test.c | 0 .../workloads/ctests/im2col_k3_pad_test.c | 0 .../workloads/ctests/im2col_k3_partial_test.c | 0 .../im2col/workloads/ctests/im2col_k3_test.c | 0 .../im2col/workloads/ctests/im2col_k5_test.c | 0 .../im2col/workloads/ctests/im2col_k7_test.c | 0 .../workloads/ctests/im2col_window_test.c | 0 .../balls/im2col/workloads/isa/im2col.h | 2 +- .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/im2col_6x6_k3.mlir | 42 + .../mlir_tests/ball/im2col_6x6_k3_main.cpp | 41 + .../workloads/mlir_tests/bank/CMakeLists.txt | 4 +- .../mlir_tests/bank/im2col_k3_two_blocks.mlir | 38 + .../bank/im2col_k3_two_blocks_main.cpp | 29 + .../balls/int2fp/arch/Fp32Mul.scala | 0 .../balls/int2fp/arch/Int2Fp.scala | 0 .../balls/int2fp/arch/Int2FpBall.scala | 0 .../int2fp/configs/Int2FpBallParam.scala | 0 .../balls/int2fp/configs/default.toml | 0 .../balls/int2fp/configs/int2fp_params.proto | 0 .../balls/int2fp/emu/src/dispatch.rs | 0 .../balls/int2fp/emu/src/int2fp.rs | 3 +- .../balls/int2fp/emu/src/lib.rs | 0 .../balls/int2fp/emu/src/model.rs | 0 .../balls/int2fp/stack/src/int2fpOps.cpp | 66 + .../prefab/balls/int2fp/stack/src/int2fpOps.h | 20 + .../balls/int2fp/stack/src/int2fpOps.td | 44 + .../balls/int2fp/verify/README.md | 0 .../balls/int2fp/verify/casegen/Cargo.lock | 0 .../balls/int2fp/verify/casegen/Cargo.toml | 0 .../int2fp/verify/casegen/src/casegen.rs | 0 .../balls/int2fp/verify/casegen/src/lib.rs | 0 .../int2fp/verify/filelists/int2fp_ball.f | 0 .../int2fp/verify/src/common/int2fp_defs.svh | 0 .../int2fp/verify/src/common/int2fp_items.svh | 0 .../int2fp/verify/src/cov/int2fp_cov.svh | 0 .../int2fp/verify/src/env/int2fp_env.svh | 0 .../verify/src/env/int2fp_scoreboard.svh | 0 .../balls/int2fp/verify/src/pkg/int2fp_pkg.sv | 0 .../verify/src/seq/int2fp_sequences.svh | 0 .../balls/int2fp/verify/src/tb_top.sv | 0 .../int2fp/verify/src/tests/int2fp_test.svh | 0 .../int2fp/workloads/ctests/CMakeLists.txt | 0 .../ctests/int2fp_channel_base_test.c | 0 .../ctests/int2fp_channel_lanes_test.c | 0 .../ctests/int2fp_channel_two_rows_test.c | 0 .../workloads/ctests/int2fp_groups_test.c | 0 .../ctests/int2fp_tensor_groups_test.c | 0 .../ctests/int2fp_tensor_rows_test.c | 0 .../ctests/int2fp_tensor_values_test.c | 0 .../balls/int2fp/workloads/isa/int2fp.h | 2 +- .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/int2fp_4x4.mlir | 2 +- .../mlir_tests/ball/int2fp_4x4_main.cpp | 0 .../mlir_tests/ball/int2fp_relu_4x4.mlir | 2 +- .../mlir_tests/ball/int2fp_relu_4x4_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 4 +- .../workloads/mlir_tests/bank/int2fp_4x4.mlir | 2 +- .../mlir_tests/bank/int2fp_4x4_main.cpp | 0 .../mlir_tests/bank/int2fp_relu_4x4.mlir | 2 +- .../mlir_tests/bank/int2fp_relu_4x4_main.cpp | 0 .../mlir_tests/bank/int2fp_unit_scale.mlir.in | 2 +- .../bank/int2fp_unit_scale_main.cpp | 7 +- .../balls/int8add/arch/Int8AddBall.scala | 0 .../int8add/arch/PipelinedAddRawFN.scala | 0 .../balls/int8add/configs/default.toml | 0 .../int8add/configs/int8add_params.proto | 0 .../balls/int8add/emu/src/dispatch.rs | 0 .../balls/int8add/emu/src/int8add.rs | 3 +- .../balls/int8add/emu/src/lib.rs | 0 .../balls/int8add/stack/src/int8addOps.cpp | 78 + .../balls/int8add/stack/src/int8addOps.h | 20 + .../balls/int8add/stack/src/int8addOps.td | 46 + .../balls/int8add/verify/casegen/Cargo.lock | 0 .../balls/int8add/verify/casegen/Cargo.toml | 0 .../int8add/verify/casegen/src/casegen.rs | 0 .../balls/int8add/verify/casegen/src/lib.rs | 0 .../int8add/verify/filelists/int8add_ball.f | 0 .../verify/src/common/int8add_defs.svh | 0 .../verify/src/common/int8add_items.svh | 0 .../int8add/verify/src/cov/int8add_cov.svh | 0 .../int8add/verify/src/env/int8add_env.svh | 0 .../verify/src/env/int8add_scoreboard.svh | 0 .../int8add/verify/src/pkg/int8add_pkg.sv | 0 .../verify/src/seq/int8add_sequences.svh | 0 .../balls/int8add/verify/src/tb_top.sv | 0 .../int8add/verify/src/tests/int8add_test.svh | 0 .../int8add/workloads/ctests/CMakeLists.txt | 0 .../workloads/ctests/int8add_relu_test.c | 2 +- .../workloads/ctests/int8add_scaled_test.c | 2 +- .../balls/int8add/workloads/isa/int8add.h | 4 +- .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/int8add_relu.mlir | 2 +- .../mlir_tests/ball/int8add_relu_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../bank/int8add_identity_scale.mlir.in | 2 +- .../bank/int8add_identity_scale_main.cpp | 7 +- .../balls/int8mul/arch/Int8MulBall.scala | 0 .../balls/int8mul/configs/default.toml | 0 .../int8mul/configs/int8mul_params.proto | 0 .../balls/int8mul/emu/src/dispatch.rs | 0 .../balls/int8mul/emu/src/int8mul.rs | 3 +- .../balls/int8mul/emu/src/lib.rs | 0 .../balls/int8mul/stack/src/int8mulOps.cpp | 75 + .../balls/int8mul/stack/src/int8mulOps.h | 20 + .../balls/int8mul/stack/src/int8mulOps.td | 46 + .../balls/int8mul/verify/casegen/Cargo.lock | 0 .../balls/int8mul/verify/casegen/Cargo.toml | 0 .../int8mul/verify/casegen/src/casegen.rs | 0 .../int8mul/verify/casegen/src/config.rs | 0 .../balls/int8mul/verify/casegen/src/lib.rs | 0 .../int8mul/verify/filelists/int8mul_ball.f | 0 .../verify/src/common/int8mul_defs.svh | 0 .../verify/src/common/int8mul_items.svh | 0 .../int8mul/verify/src/cov/int8mul_cov.svh | 0 .../int8mul/verify/src/env/int8mul_env.svh | 0 .../verify/src/env/int8mul_scoreboard.svh | 0 .../int8mul/verify/src/pkg/int8mul_pkg.sv | 0 .../verify/src/seq/int8mul_sequences.svh | 0 .../balls/int8mul/verify/src/tb_top.sv | 0 .../int8mul/verify/src/tests/int8mul_test.svh | 0 .../int8mul/workloads/ctests/CMakeLists.txt | 0 .../ctests/int8mul_large_gate_row_test.c | 5 +- .../ctests/int8mul_per_channel_gate_test.c | 0 .../balls/int8mul/workloads/isa/int8mul.h | 3 +- .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/int8mul_gate.mlir | 2 +- .../mlir_tests/ball/int8mul_gate_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../mlir_tests/bank/int8mul_unit_gate.mlir.in | 2 +- .../bank/int8mul_unit_gate_main.cpp | 7 +- .../balls/lut/arch/LutBall.scala | 0 .../balls/lut/configs/default.toml | 0 .../balls/lut/configs/lut_params.proto | 0 .../balls/lut/emu/src/dispatch.rs | 0 .../balls/lut/emu/src/lib.rs | 0 .../balls/lut/emu/src/lut.rs | 3 +- .../prefab/balls/lut/stack/src/lutOps.cpp | 68 + examples/prefab/balls/lut/stack/src/lutOps.h | 20 + examples/prefab/balls/lut/stack/src/lutOps.td | 43 + .../balls/lut/verify/casegen/Cargo.lock | 0 .../balls/lut/verify/casegen/Cargo.toml | 0 .../balls/lut/verify/casegen/src/casegen.rs | 0 .../balls/lut/verify/casegen/src/lib.rs | 0 .../balls/lut/verify/filelists/lut_ball.f | 0 .../balls/lut/verify/src/common/lut_defs.svh | 0 .../balls/lut/verify/src/common/lut_items.svh | 0 .../balls/lut/verify/src/cov/lut_cov.svh | 0 .../balls/lut/verify/src/env/lut_env.svh | 0 .../lut/verify/src/env/lut_scoreboard.svh | 0 .../balls/lut/verify/src/pkg/lut_pkg.sv | 0 .../lut/verify/src/seq/lut_sequences.svh | 0 .../balls/lut/verify/src/tb_top.sv | 0 .../balls/lut/verify/src/tests/lut_test.svh | 0 .../balls/lut/workloads/ctests/CMakeLists.txt | 0 .../ctests/lut_channel_table_lookup_test.c | 2 +- .../ctests/lut_uniform_lookup_test.c | 2 +- .../balls/lut/workloads/isa/lut.h | 2 +- .../lut/workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/lut_identity.mlir | 2 +- .../mlir_tests/ball/lut_identity_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../bank/lut_identity_table.mlir.in | 2 +- .../bank/lut_identity_table_main.cpp | 7 +- .../balls/matadd/README.md | 0 .../balls/matadd/arch/MatAdd.scala | 0 .../balls/matadd/arch/MatAddBall.scala | 0 .../balls/matadd/configs/default.toml | 0 .../balls/matadd/configs/matadd_params.proto | 0 .../balls/matadd/emu/src/64_matadd.rs | 3 +- .../balls/matadd/emu/src/dispatch.rs | 0 .../balls/matadd/emu/src/lib.rs | 0 .../balls/matadd/stack/src/mataddOps.cpp | 67 + .../prefab/balls/matadd/stack/src/mataddOps.h | 20 + .../balls/matadd/stack/src/mataddOps.td | 40 + .../balls/matadd/verify/casegen/Cargo.lock | 0 .../balls/matadd/verify/casegen/Cargo.toml | 0 .../balls/matadd/verify/casegen/src/lib.rs | 0 .../matadd/verify/filelists/matadd_ball.f | 0 .../matadd/verify/src/common/matadd_defs.svh | 0 .../matadd/verify/src/common/matadd_items.svh | 0 .../matadd/verify/src/env/matadd_env.svh | 0 .../verify/src/env/matadd_scoreboard.svh | 0 .../balls/matadd/verify/src/pkg/matadd_pkg.sv | 0 .../verify/src/seq/matadd_sequences.svh | 0 .../balls/matadd/verify/src/tb_top.sv | 0 .../matadd/verify/src/tests/matadd_test.svh | 0 .../matadd/workloads/ctests/CMakeLists.txt | 0 .../workloads/ctests/matadd_i32_16row_test.c | 0 .../balls/matadd/workloads/isa/matadd.h | 2 +- .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/matadd_i32.mlir | 2 +- .../mlir_tests/ball/matadd_i32_main.cpp | 1 - .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../bank/matadd_i32_elementwise.mlir.in | 2 +- .../bank/matadd_i32_elementwise_main.cpp | 8 +- .../balls/maxpool/arch/MaxPoolBall.scala | 0 .../balls/maxpool/configs/default.toml | 0 .../maxpool/configs/maxpool_params.proto | 0 .../balls/maxpool/emu/src/dispatch.rs | 0 .../balls/maxpool/emu/src/lib.rs | 0 .../balls/maxpool/emu/src/maxpool.rs | 8 +- .../balls/maxpool/stack/src/maxpoolOps.cpp | 80 + .../balls/maxpool/stack/src/maxpoolOps.h | 20 + .../balls/maxpool/stack/src/maxpoolOps.td | 45 + .../balls/maxpool/verify/casegen/Cargo.lock | 0 .../balls/maxpool/verify/casegen/Cargo.toml | 0 .../maxpool/verify/casegen/src/casegen.rs | 0 .../balls/maxpool/verify/casegen/src/lib.rs | 0 .../balls/maxpool/verify/casegen/src/model.rs | 0 .../maxpool/verify/filelists/maxpool_ball.f | 0 .../verify/src/common/maxpool_defs.svh | 0 .../verify/src/common/maxpool_items.svh | 0 .../maxpool/verify/src/cov/maxpool_cov.svh | 0 .../maxpool/verify/src/env/maxpool_env.svh | 0 .../verify/src/env/maxpool_scoreboard.svh | 0 .../maxpool/verify/src/pkg/maxpool_pkg.sv | 0 .../verify/src/seq/maxpool_sequences.svh | 0 .../balls/maxpool/verify/src/tb_top.sv | 0 .../maxpool/verify/src/tests/maxpool_test.svh | 0 .../maxpool/workloads/ctests/CMakeLists.txt | 0 .../workloads/ctests/maxpool_groups_test.c | 0 .../workloads/ctests/maxpool_offset_test.c | 2 +- .../workloads/ctests/maxpool_window_test.c | 7 +- .../balls/maxpool/workloads/isa/maxpool.h | 2 +- .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/maxpool_6x6_s2.mlir | 2 +- .../mlir_tests/ball/maxpool_6x6_s2_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 11 + .../bank/maxpool_k3_padded_spatial.mlir.in | 2 +- .../bank/maxpool_k3_padded_spatial_main.cpp | 7 +- .../balls/mxfp2int/arch/Mxfp2Int.scala | 51 +- .../balls/mxfp2int/arch/Mxfp2IntBall.scala | 8 +- .../balls/mxfp2int/arch/Mxfp4Decoder.scala | 0 .../balls/mxfp2int/arch/Mxfp6Decoder.scala | 0 .../balls/mxfp2int/arch/Mxfp8Decoder.scala | 0 .../balls/mxfp2int/arch/MxfpDecoder.scala | 8 +- .../mxfp2int/configs/Mxfp2IntBallParam.scala | 0 .../balls/mxfp2int/configs/default.toml | 0 .../mxfp2int/configs/mxfp2int_params.proto | 0 .../balls/mxfp2int/emu/src/55_mxfp2int.rs | 69 + .../balls/mxfp2int/emu/src/dispatch.rs | 0 .../balls/mxfp2int/emu/src/lib.rs | 0 .../balls/mxfp2int/stack/src/mxfp2intOps.cpp | 72 + .../balls/mxfp2int/stack/src/mxfp2intOps.h | 20 + .../balls/mxfp2int/stack/src/mxfp2intOps.td | 50 + .../balls/mxfp2int/verify/casegen/Cargo.lock | 7 + .../balls/mxfp2int/verify/casegen/Cargo.toml | 10 + .../mxfp2int/verify/casegen/src/casegen.rs | 231 ++ .../mxfp2int/verify/casegen/src/config.rs | 9 + .../balls/mxfp2int/verify/casegen/src/lib.rs | 116 ++ .../mxfp2int/verify/filelists/mxfp2int_ball.f | 25 + .../verify/src/common/mxfp2int_defs.svh | 39 + .../verify/src/common/mxfp2int_items.svh | 85 + .../mxfp2int/verify/src/cov/mxfp2int_cov.svh | 64 + .../mxfp2int/verify/src/env/mxfp2int_env.svh | 27 + .../verify/src/env/mxfp2int_scoreboard.svh | 196 ++ .../mxfp2int/verify/src/pkg/mxfp2int_pkg.sv | 13 + .../verify/src/seq/mxfp2int_sequences.svh | 17 + .../balls/mxfp2int/verify/src/tb_top.sv | 87 + .../verify/src/tests/mxfp2int_test.svh | 56 + .../mxfp2int/workloads/ctests/CMakeLists.txt | 2 + .../ctests/mxfp4_e8m0_scale_range_test.c | 72 + .../ctests/mxfp4_e8m0_two_block_test.c | 29 +- .../balls/mxfp2int/workloads/isa/mxfp2int.h | 21 + .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 4 + .../mlir_tests/ball/mxfp4_i8_2blocks.mlir | 28 + .../ball/mxfp4_i8_2blocks_main.cpp} | 7 +- .../workloads/mlir_tests/bank/CMakeLists.txt | 8 + .../bank/mxfp4_i8_bank_blocks.mlir.in | 59 + .../bank/mxfp4_i8_bank_blocks_main.cpp | 28 + .../balls/mxmm/arch/AddPipe.scala | 0 .../balls/mxmm/arch/Array.scala | 0 .../balls/mxmm/arch/InputWindow.scala | 0 .../balls/mxmm/arch/Mxfp8Decode.scala | 0 .../balls/mxmm/arch/MxmmBall.scala | 0 .../balls/mxmm/arch/PE.scala | 0 .../balls/mxmm/arch/Panels.scala | 0 .../balls/mxmm/arch/Unit.scala | 0 .../balls/mxmm/configs/MxmmBallParam.scala | 0 .../balls/mxmm/configs/default.toml | 0 .../balls/mxmm/configs/mxmm_params.proto | 0 .../balls/mxmm/emu/src/accumulator.rs | 0 .../balls/mxmm/emu/src/dispatch.rs | 0 .../balls/mxmm/emu/src/fp32.rs | 3 +- .../balls/mxmm/emu/src/lib.rs | 0 .../balls/mxmm/emu/src/mxfp8.rs | 3 +- .../balls/mxmm/stack/kernels/CMakeLists.txt | 2 +- .../balls/mxmm/stack/kernels/dispatch.cpp | 3 +- .../balls/mxmm/stack/kernels/norm_window.cpp | 15 +- .../mxmm/stack/kernels/norm_window_image.h | 0 .../mxmm/stack/kernels/pack_dispatch.cpp | 3 +- .../balls/mxmm/stack/kernels/packmm.cpp | 21 +- .../balls/mxmm/stack/kernels/packmm.h | 0 .../balls/mxmm/stack/kernels/packmm_image.h | 0 .../balls/mxmm/stack/kernels/window.h | 0 .../prefab/balls/mxmm/stack/src/mxmmOps.cpp | 167 ++ .../prefab/balls/mxmm/stack/src/mxmmOps.h | 20 + .../prefab/balls/mxmm/stack/src/mxmmOps.td | 118 ++ .../balls/mxmm/verify/casegen/Cargo.lock | 7 + .../balls/mxmm/verify/casegen/Cargo.toml | 10 + .../balls/mxmm/verify/casegen/src/casegen.rs | 408 ++++ .../balls/mxmm/verify/casegen/src/config.rs | 9 + .../balls/mxmm/verify/casegen/src/lib.rs | 116 ++ .../balls/mxmm/verify/filelists/mxmm_ball.f | 40 + .../mxmm/verify/src/common/mxmm_defs.svh | 51 + .../mxmm/verify/src/common/mxmm_items.svh | 89 + .../balls/mxmm/verify/src/cov/mxmm_cov.svh | 59 + .../balls/mxmm/verify/src/env/mxmm_env.svh | 27 + .../mxmm/verify/src/env/mxmm_scoreboard.svh | 196 ++ .../balls/mxmm/verify/src/pkg/mxmm_pkg.sv | 13 + .../mxmm/verify/src/seq/mxmm_sequences.svh | 17 + .../prefab/balls/mxmm/verify/src/tb_top.sv | 89 + .../balls/mxmm/verify/src/tests/mxmm_test.svh | 56 + .../mxmm/workloads/ctests/CMakeLists.txt | 9 +- .../ctests/mxmm_bank_capacity_test.c | 13 +- .../mxmm/workloads/ctests/mxmm_fp32_test.c | 0 .../mxmm/workloads/ctests/mxmm_mxfp8_test.c | 0 .../workloads/ctests/mxmm_window_chain_test.c | 16 +- .../workloads/ctests/mxmm_window_rows_test.c | 16 +- .../mxmm/workloads/ctests/mxmm_window_test.c | 0 .../balls/mxmm/workloads/isa/mxmm.h | 8 +- .../workloads/mlir_tests/ball/mxmm_f32.mlir | 10 + .../workloads/mlir_tests/ball/mxmm_fma32.mlir | 10 + .../mlir_tests/ball/mxmm_mxfp8_chain.mlir | 18 + .../mlir_tests/ball/mxmm_mxfp8_window.mlir | 10 + .../workloads/mlir_tests/bank/mxmm_f32.mlir | 17 + .../workloads/mlir_tests/bank/mxmm_fma32.mlir | 17 + .../mlir_tests/bank/mxmm_mxfp8_chain.mlir | 19 + .../mlir_tests/bank/mxmm_mxfp8_window.mlir | 17 + .../balls/mxquant/arch/Encode.scala | 0 .../balls/mxquant/arch/MxquantBall.scala | 0 .../balls/mxquant/configs/default.toml | 0 .../mxquant/configs/mxquant_params.proto | 0 .../balls/mxquant/emu/src/dispatch.rs | 0 .../balls/mxquant/emu/src/lib.rs | 0 .../balls/mxquant/emu/src/quant.rs | 3 +- .../mxquant/stack/kernels/CMakeLists.txt | 2 +- .../balls/mxquant/stack/kernels/dispatch.cpp | 0 .../balls/mxquant/stack/kernels/pack.cpp | 0 .../balls/mxquant/stack/src/mxquantOps.cpp | 69 + .../balls/mxquant/stack/src/mxquantOps.h | 20 + .../balls/mxquant/stack/src/mxquantOps.td | 47 + .../balls/mxquant/verify/casegen/Cargo.lock | 7 + .../balls/mxquant/verify/casegen/Cargo.toml | 10 + .../mxquant/verify/casegen/src/casegen.rs | 265 +++ .../mxquant/verify/casegen/src/config.rs | 9 + .../balls/mxquant/verify/casegen/src/lib.rs | 113 + .../mxquant/verify/filelists/mxquant_ball.f | 25 + .../verify/src/common/mxquant_defs.svh | 39 + .../verify/src/common/mxquant_items.svh | 80 + .../mxquant/verify/src/cov/mxquant_cov.svh | 76 + .../mxquant/verify/src/env/mxquant_env.svh | 27 + .../verify/src/env/mxquant_scoreboard.svh | 173 ++ .../mxquant/verify/src/pkg/mxquant_pkg.sv | 13 + .../verify/src/seq/mxquant_sequences.svh | 17 + .../prefab/balls/mxquant/verify/src/tb_top.sv | 78 + .../mxquant/verify/src/tests/mxquant_test.svh | 56 + .../mxquant/workloads/ctests/CMakeLists.txt | 12 +- .../workloads/ctests/mxquant_capacity_test.c | 11 +- .../workloads/ctests/mxquant_values_test.c | 0 .../balls/mxquant/workloads/isa/mxquant.h | 2 +- .../mlir_tests/ball/mxquant_two_blocks.mlir | 9 + .../mlir_tests/bank/mxquant_two_blocks.mlir | 14 + .../balls/mxquant/workloads/reference.h | 0 .../balls/redmule/arch/RedmuleBall.scala | 2 +- .../balls/redmule/arch/RedmuleNative.scala | 20 +- .../balls/redmule/arch/RedmuleNative.sv | 0 .../balls/redmule/configs/default.toml | 0 .../redmule/configs/redmule_params.proto | 0 .../balls/redmule/emu/src/fp16.rs | 0 .../balls/redmule/emu/src/lib.rs | 0 .../balls/redmule/emu/src/matmul.rs | 8 +- .../balls/redmule/stack/src/redmuleOps.cpp | 76 + .../balls/redmule/stack/src/redmuleOps.h | 20 + .../balls/redmule/stack/src/redmuleOps.td | 49 + .../balls/redmule/verify/casegen/Cargo.lock | 0 .../balls/redmule/verify/casegen/Cargo.toml | 0 .../balls/redmule/verify/casegen/src/lib.rs | 0 .../redmule/verify/filelists/redmule_ball.f | 0 .../balls/redmule/verify/parameters.toml | 0 .../redmule/verify/src/pkg/redmule_pkg.sv | 0 .../verify/src/reference/redmule_reference.sv | 0 .../balls/redmule/verify/src/tb_top.sv | 0 .../redmule/workloads/ctests/CMakeLists.txt | 12 + .../workloads/ctests/redmule_f16_chain_test.c | 3 +- .../workloads/ctests/redmule_f16_edges_test.c | 1 - .../workloads/ctests/redmule_f16_fused_test.c | 1 - .../workloads/ctests/redmule_f16_test.c | 3 +- .../balls/redmule/workloads/isa/redmule.h | 2 +- .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/redmule_f16_chain.mlir | 6 + .../ball/redmule_f16_chain_main.cpp | 1 - .../ball/redmule_f16_invalid_k.mlir | 2 +- .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../bank/redmule_f16_invalid_alias.mlir | 2 +- .../mlir_tests/bank/redmule_f16_rows.mlir.in | 4 +- .../mlir_tests/bank/redmule_f16_rows_main.cpp | 4 +- .../balls/relu/README.md | 0 .../balls/relu/arch/Relu.scala | 0 .../balls/relu/arch/ReluBall.scala | 0 .../balls/relu/configs/default.toml | 0 .../balls/relu/configs/relu_params.proto | 0 .../balls/relu/emu/src/50_relu.rs | 8 +- .../balls/relu/emu/src/dispatch.rs | 0 .../balls/relu/emu/src/lib.rs | 0 .../prefab/balls/relu/stack/src/reluOps.cpp | 66 + .../prefab/balls/relu/stack/src/reluOps.h | 20 + .../prefab/balls/relu/stack/src/reluOps.td | 44 + .../balls/relu/verify/casegen/Cargo.lock | 7 + .../balls/relu/verify/casegen/Cargo.toml | 10 + .../balls/relu/verify/casegen/src/casegen.rs | 89 + .../balls/relu/verify/casegen/src/lib.rs | 71 + .../balls/relu/verify/casegen/src/model.rs | 48 + .../balls/relu/verify/filelists/relu_ball.f | 25 + .../relu/verify/src/common/relu_defs.svh | 27 + .../relu/verify/src/common/relu_items.svh | 64 + .../balls/relu/verify/src/cov/relu_cov.svh | 47 + .../balls/relu/verify/src/env/relu_env.svh | 27 + .../relu/verify/src/env/relu_scoreboard.svh | 157 ++ .../balls/relu/verify/src/pkg/relu_pkg.sv | 13 + .../relu/verify/src/seq/relu_sequences.svh | 17 + .../prefab/balls/relu/verify/src/tb_top.sv | 80 + .../balls/relu/verify/src/tests/relu_test.svh | 48 + .../relu/workloads/ctests/CMakeLists.txt | 0 .../workloads/ctests/relu_i32_16x16_test.c | 0 .../balls/relu/workloads/isa/relu.h | 2 +- .../relu}/workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/relu_i32_16row.mlir | 2 +- .../mlir_tests/ball/relu_i32_16row_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../bank/relu_i32_mixed_sign.mlir.in | 4 +- .../bank/relu_i32_mixed_sign_main.cpp | 13 +- .../balls/smatmul/README.md | 0 .../balls/smatmul/arch/Array.scala | 0 .../balls/smatmul/arch/PE.scala | 0 .../balls/smatmul/arch/SMatMulBall.scala | 0 .../balls/smatmul/arch/SMatMulUnit.scala | 0 .../smatmul/configs/SMatMulBallParam.scala | 0 .../balls/smatmul/configs/default.toml | 0 .../balls/smatmul/configs/lane16.toml | 0 .../smatmul/configs/smatmul_params.proto | 0 .../balls/smatmul/emu/src/dispatch.rs | 0 .../balls/smatmul/emu/src/fp32.rs | 3 +- .../balls/smatmul/emu/src/lib.rs | 0 .../balls/smatmul/emu/src/mxfp8.rs | 3 +- .../balls/smatmul/emu/src/smatmul.rs | 4 +- .../balls/smatmul/stack/src/smatmulOps.cpp | 80 + .../balls/smatmul/stack/src/smatmulOps.h | 20 + .../balls/smatmul/stack/src/smatmulOps.td | 49 + .../balls/smatmul/verify/README.md | 0 .../balls/smatmul/verify/casegen/Cargo.lock | 0 .../balls/smatmul/verify/casegen/Cargo.toml | 0 .../smatmul/verify/casegen/src/casegen.rs | 0 .../balls/smatmul/verify/casegen/src/lib.rs | 0 .../balls/smatmul/verify/casegen/src/model.rs | 0 .../smatmul/verify/filelists/smatmul_ball.f | 0 .../balls/smatmul/verify/parameters.toml | 0 .../verify/src/common/smatmul_defs.svh | 0 .../verify/src/common/smatmul_items.svh | 0 .../smatmul/verify/src/cov/smatmul_cov.svh | 0 .../smatmul/verify/src/env/smatmul_env.svh | 0 .../verify/src/env/smatmul_scoreboard.svh | 0 .../smatmul/verify/src/pkg/smatmul_pkg.sv | 0 .../verify/src/seq/smatmul_sequences.svh | 0 .../balls/smatmul/verify/src/tb_top.sv | 0 .../smatmul/verify/src/tests/smatmul_test.svh | 0 .../smatmul/workloads/ctests/CMakeLists.txt | 8 +- .../workloads/ctests/smatmul_16x16_test.c | 0 .../workloads/ctests/smatmul_16x16x32_test.c | 0 .../workloads/ctests/smatmul_1x16x64_test.c | 0 .../workloads/ctests/smatmul_8x16_test.c | 0 .../ctests/smatmul_bias_accumulate_test.c | 0 .../workloads/ctests/smatmul_c_groups_test.c | 14 +- .../workloads/ctests/smatmul_f32_test.c | 0 .../workloads/ctests/smatmul_k_groups_test.c | 0 .../ctests/smatmul_nondefault_banks_test.c | 9 +- .../balls/smatmul/workloads/isa/smatmul.h | 12 +- .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../ball/smatmul_bias_accumulate.mlir | 8 +- .../ball/smatmul_bias_accumulate_main.cpp | 2 +- .../workloads/mlir_tests/bank/CMakeLists.txt | 9 +- .../bank/smatmul_bias_two_terms.mlir.in | 12 +- .../bank/smatmul_bias_two_terms_main.cpp | 9 +- .../balls/toint8/arch/ToInt8Ball.scala | 0 .../balls/toint8/configs/default.toml | 0 .../balls/toint8/configs/toint8_params.proto | 0 .../balls/toint8/emu/src/dispatch.rs | 0 .../balls/toint8/emu/src/lib.rs | 0 .../balls/toint8/emu/src/model.rs | 0 .../balls/toint8/emu/src/quant.rs | 4 +- .../balls/toint8/stack/src/toint8Ops.cpp | 104 + .../prefab/balls/toint8/stack/src/toint8Ops.h | 20 + .../balls/toint8/stack/src/toint8Ops.td | 67 + .../balls/toint8/verify/README.md | 0 .../balls/toint8/verify/casegen/Cargo.lock | 0 .../balls/toint8/verify/casegen/Cargo.toml | 0 .../toint8/verify/casegen/src/casegen.rs | 0 .../balls/toint8/verify/casegen/src/lib.rs | 0 .../toint8/verify/filelists/toint8_ball.f | 0 .../toint8/verify/src/common/toint8_defs.svh | 0 .../toint8/verify/src/common/toint8_items.svh | 0 .../toint8/verify/src/cov/toint8_cov.svh | 0 .../toint8/verify/src/env/toint8_env.svh | 0 .../verify/src/env/toint8_scoreboard.svh | 0 .../balls/toint8/verify/src/pkg/toint8_pkg.sv | 0 .../verify/src/seq/toint8_sequences.svh | 0 .../balls/toint8/verify/src/tb_top.sv | 0 .../toint8/verify/src/tests/toint8_test.svh | 0 .../toint8/workloads/ctests/CMakeLists.txt | 0 .../workloads/ctests/quant_f32_groups_test.c | 0 .../workloads/ctests/quant_f32_to_i8_test.c | 0 .../workloads/ctests/quant_i32_groups_test.c | 0 .../workloads/ctests/quant_i32_to_i8_test.c | 0 .../ctests/toint8_i8_rounding_test.c | 0 .../workloads/ctests/toint8_i8_rows_test.c | 0 .../ctests/toint8_i8_scale_rows_test.c | 0 .../workloads/ctests/toint8_i8_signed_test.c | 0 .../balls/toint8/workloads/isa/quant.h | 4 +- .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/quant_f32_to_i8.mlir | 2 +- .../mlir_tests/ball/quant_f32_to_i8_main.cpp | 0 .../mlir_tests/ball/quant_i32_to_i8.mlir | 2 +- .../mlir_tests/ball/quant_i32_to_i8_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 6 +- .../mlir_tests/bank/quant_f32_pack16.mlir.in | 2 +- .../mlir_tests/bank/quant_f32_pack16_main.cpp | 8 +- .../mlir_tests/bank/quant_i32_pack16.mlir.in | 2 +- .../mlir_tests/bank/quant_i32_pack16_main.cpp | 10 +- examples/prefab/balls/trace/arch/README.md | 64 + .../balls/trace/arch/Trace.scala | 3 + .../balls/trace/arch/TraceBall.scala | 0 .../balls/trace/arch/TraceDPI.scala | 0 .../balls/trace/configs/default.toml | 0 .../balls/trace/configs/trace_params.proto | 0 .../balls/trace/emu/src/04_bdb_counter.rs | 51 + .../balls/trace/emu/src/dispatch.rs | 0 .../balls/trace/emu/src/lib.rs | 0 .../prefab/balls/trace/stack/src/traceOps.cpp | 77 + .../prefab/balls/trace/stack/src/traceOps.h | 20 + .../prefab/balls/trace/stack/src/traceOps.td | 49 + .../balls/trace/verify/casegen/Cargo.lock | 7 + .../balls/trace/verify/casegen/Cargo.toml | 10 + .../balls/trace/verify/casegen/src/casegen.rs | 60 + .../balls/trace/verify/casegen/src/lib.rs | 110 + .../balls/trace/verify/casegen/src/model.rs | 69 + .../balls/trace/verify/filelists/trace_ball.f | 26 + .../trace/verify/src/common/trace_defs.svh | 17 + .../trace/verify/src/common/trace_items.svh | 29 + .../balls/trace/verify/src/cov/trace_cov.svh | 64 + .../balls/trace/verify/src/env/trace_env.svh | 26 + .../trace/verify/src/env/trace_scoreboard.svh | 98 + .../balls/trace/verify/src/pkg/trace_pkg.sv | 13 + .../trace/verify/src/seq/trace_sequences.svh | 19 + .../prefab/balls/trace/verify/src/tb_top.sv | 80 + .../trace/verify/src/tests/trace_test.svh | 53 + .../trace/workloads/ctests/CMakeLists.txt | 0 .../ctests/bdb_counter_multi_id_test.c | 0 .../ctests/bdb_counter_nested_pair_test.c | 0 .../ctests/bdb_counter_read_active_test.c | 0 .../ctests/bdb_counter_start_stop_test.c | 0 .../balls/trace/workloads/isa/bdb_counter.h | 8 +- .../trace/workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../ball/bdb_counter_start_read_stop.mlir | 8 + .../ball/bdb_counter_start_read_stop_main.cpp | 0 .../balls/transpose/arch/README.md | 2 +- .../balls/transpose/arch/Transpose.scala | 0 .../balls/transpose/arch/TransposeBall.scala | 0 .../configs/TransposeBallParam.scala | 0 .../balls/transpose/configs/default.toml | 0 .../transpose/configs/transpose_params.proto | 0 .../balls/transpose/emu/src/dispatch.rs | 0 .../balls/transpose/emu/src/lib.rs | 0 .../balls/transpose/emu/src/transpose.rs | 3 +- .../transpose/stack/src/transposeOps.cpp | 73 + .../balls/transpose/stack/src/transposeOps.h | 20 + .../balls/transpose/stack/src/transposeOps.td | 42 + .../balls/transpose/verify/README.md | 0 .../balls/transpose/verify/casegen/Cargo.lock | 0 .../balls/transpose/verify/casegen/Cargo.toml | 0 .../transpose/verify/casegen/src/casegen.rs | 0 .../balls/transpose/verify/casegen/src/lib.rs | 0 .../transpose/verify/casegen/src/model.rs | 0 .../verify/filelists/transpose_ball.f | 0 .../verify/src/common/transpose_defs.svh | 0 .../verify/src/common/transpose_items.svh | 0 .../verify/src/cov/transpose_cov.svh | 0 .../verify/src/env/transpose_env.svh | 0 .../verify/src/env/transpose_scoreboard.svh | 0 .../transpose/verify/src/pkg/transpose_pkg.sv | 0 .../verify/src/seq/transpose_sequences.svh | 0 .../balls/transpose/verify/src/tb_top.sv | 0 .../verify/src/tests/transpose_test.svh | 0 .../transpose/workloads/ctests/CMakeLists.txt | 0 .../ctests/transpose_i32_8x16_test.c | 2 +- .../ctests/transpose_i8_16x16_test.c | 2 +- .../ctests/transpose_i8_16x32_test.c | 2 +- .../balls/transpose/workloads/isa/transpose.h | 2 +- .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 0 .../mlir_tests/ball/transpose_16x16_i8.mlir | 2 +- .../ball/transpose_16x16_i8_main.cpp | 0 .../workloads/mlir_tests/bank/CMakeLists.txt | 0 .../bank/transpose_i8_packed_matrix.mlir.in | 2 +- .../bank/transpose_i8_packed_matrix_main.cpp | 11 +- .../balls/vector/arch/README.md | 0 .../balls/vector/arch/VecBall.scala | 4 - .../balls/vector/arch/VecCtrlUnit.scala | 0 .../balls/vector/arch/VecEXUnit.scala | 0 .../balls/vector/arch/VecLoadUnit.scala | 0 .../balls/vector/arch/VecMat16.scala | 64 +- .../balls/vector/arch/VecMat16AccLoad.scala | 11 +- .../balls/vector/arch/VecMat16Store.scala | 15 +- .../balls/vector/arch/VecMat16Tie.scala | 3 +- .../balls/vector/arch/VecStoreUnit.scala | 0 .../balls/vector/arch/VecUnit.scala | 0 .../balls/vector/arch/bond/BondWrapper.scala | 0 .../balls/vector/arch/bond/README.md | 0 .../balls/vector/arch/bond/vvv.scala | 0 .../balls/vector/arch/op/README.md | 0 .../balls/vector/arch/op/cascade.scala | 0 .../balls/vector/arch/op/mul.scala | 0 .../balls/vector/arch/thread/BaseThread.scala | 0 .../balls/vector/arch/thread/CasThread.scala | 0 .../balls/vector/arch/thread/MulThread.scala | 0 .../balls/vector/arch/thread/README.md | 0 .../balls/vector/arch/warp/MeshWarp.scala | 0 .../balls/vector/arch/warp/README.md | 0 .../vector/configs/VectorBallParam.scala | 0 .../balls/vector/configs/default.toml | 0 .../balls/vector/configs/vector_params.proto | 0 .../balls/vector/emu/src/64_vecmat16.rs | 13 +- .../balls/vector/emu/src/dispatch.rs | 0 .../balls/vector/emu/src/lib.rs | 0 .../balls/vector/stack/src/vectorOps.cpp | 68 + .../prefab/balls/vector/stack/src/vectorOps.h | 20 + .../balls/vector/stack/src/vectorOps.td | 47 + .../balls/vector/verify/casegen/Cargo.lock | 7 + .../balls/vector/verify/casegen/Cargo.toml | 10 + .../vector/verify/casegen/src/casegen.rs | 126 ++ .../balls/vector/verify/casegen/src/lib.rs | 81 + .../balls/vector/verify/casegen/src/model.rs | 50 + .../vector/verify/filelists/vector_ball.f | 28 + .../vector/verify/src/common/vector_defs.svh | 33 + .../vector/verify/src/common/vector_items.svh | 63 + .../vector/verify/src/cov/vector_cov.svh | 44 + .../vector/verify/src/env/vector_env.svh | 27 + .../verify/src/env/vector_scoreboard.svh | 200 ++ .../balls/vector/verify/src/pkg/vector_pkg.sv | 13 + .../verify/src/seq/vector_sequences.svh | 17 + .../prefab/balls/vector/verify/src/tb_top.sv | 165 ++ .../vector/verify/src/tests/vector_test.svh | 49 + .../vector/workloads/ctests/CMakeLists.txt | 1 + .../ctests/vecmat16_accumulate_k4_test.c | 51 + .../ctests/vecmat16_atranspose_16x16_test.c | 16 +- .../balls/vector/workloads/isa/vecmat16.h | 21 + .../workloads/mlir_tests/CMakeLists.txt | 0 .../workloads/mlir_tests/ball/CMakeLists.txt | 4 + .../mlir_tests/ball/matmul_at_i8_16x16.mlir | 51 + .../ball/matmul_at_i8_16x16_main.cpp} | 6 +- .../workloads/mlir_tests/bank/CMakeLists.txt | 4 + .../bank/matmul_at_i8_bank_depth.mlir.in | 57 + .../bank/matmul_at_i8_bank_depth_main.cpp} | 10 +- .../{components => prefab}/cores/boom/LICENSE | 0 .../cores/boom/arch/Boom.scala | 0 .../cores/boom/arch/Native.scala | 0 .../cores/boom/arch/build.sc | 9 +- .../prefab/cores/boom/arch/native/Core.scala | 1516 ++++++++++++++ .../cores/boom/arch/native/Decode.scala | 216 ++ .../boom/arch/native/ExecutionUnit.scala | 202 ++ .../cores/boom/arch/native/Native.scala | 179 ++ .../NestedPrefixModulesAnnotation.scala | 10 + .../prefab/cores/boom/arch/native/Rocc.scala | 235 +++ .../prefab/cores/boom/arch/native/Tile.scala | 295 +++ .../cores/boom/configs/Params.scala | 0 .../cores/boom/configs/boom_params.proto | 0 .../cores/boom/stack/CMakeLists.txt | 2 +- .../cores/boom/thirdparty/riscv-boom | 0 .../cores/boom/thirdparty/rocket-chip | 0 .../boom/workloads/ctests/CMakeLists.txt | 0 .../boom/workloads/ctests/memory_matrix.c | 0 .../cores/reduce/arch/ReduceCore.scala | 0 .../cores/reduce/arch/acc.scala | 0 .../cores/reduce/arch/build.sc | 0 .../cores/reduce/arch/loom.scala | 0 .../cores/reduce/emu/Cargo.lock | 0 .../cores/reduce/emu/Cargo.toml | 0 .../cores/reduce/emu/src/lib.rs | 0 .../cores/reduce/stack/CMakeLists.txt | 8 +- .../cores/rocket/LICENSE.Berkeley | 0 .../cores/rocket/LICENSE.SiFive | 0 .../cores/rocket/arch/Execution.scala | 4 +- .../cores/rocket/arch/Lsu.scala | 0 .../cores/rocket/arch/Rocket.scala | 0 .../cores/rocket/arch/build.sc | 0 .../cores/rocket/arch/pipeline/Config.scala | 0 .../cores/rocket/arch/pipeline/Pipeline.scala | 20 +- .../cores/rocket/arch/pipeline/Utility.scala | 0 .../cores/rocket/configs/Params.scala | 0 .../cores/rocket/configs/rocket_params.proto | 0 .../cores/rocket/emu/Cargo.lock | 0 .../cores/rocket/emu/Cargo.toml | 0 .../cores/rocket/emu/src/lib.rs | 0 .../cores/rocket/stack/CMakeLists.txt | 2 +- .../cores/rocket/thirdparty/rocket-chip | 0 .../cores/xiangshan/LICENSE | 0 .../cores/xiangshan/arch/Chi.scala | 0 .../cores/xiangshan/arch/Native.scala | 0 .../cores/xiangshan/arch/XiangShan.scala | 0 examples/prefab/cores/xiangshan/arch/build.sc | 88 + .../cores/xiangshan/arch/native/Backend.scala | 1151 ++++++++++ .../cores/xiangshan/arch/native/CSR.scala | 486 +++++ .../cores/xiangshan/arch/native/Core.scala | 398 ++++ .../xiangshan/arch/native/CtrlBlock.scala | 926 ++++++++ .../xiangshan/arch/native/DecodeStage.scala | 320 +++ .../xiangshan/arch/native/DecodeUnit.scala | 758 +++++++ .../cores/xiangshan/arch/native/Native.scala | 204 ++ .../cores/xiangshan/arch/native/Tile.scala | 231 ++ .../native/xiangshan/backend/fu/CSR.scala | 1685 +++++++++++++++ .../xiangshan/backend/fu/FuConfig.scala | 894 ++++++++ .../xiangshan/backend/fu/FuncUnit.scala | 315 +++ .../xiangshan/backend/fu/NewCSR/NewCSR.scala | 1853 +++++++++++++++++ .../cores/xiangshan/configs/Params.scala | 0 .../xiangshan/configs/xiangshan_params.proto | 0 .../cores/xiangshan/stack/CMakeLists.txt | 2 +- .../cores/xiangshan/thirdparty/xiangshan | 0 .../xiangshan/workloads/ctests/CMakeLists.txt | 0 .../workloads/ctests/memory_matrix.c | 0 .../attention/arch/AttentionThread.scala | 0 .../threads/attention/arch/acc.scala | 0 .../configs/balldomains/default.toml | 0 .../attention/configs/cpu/default.toml | 0 .../threads/attention/configs/default.toml | 0 .../attention/configs/frontends/default.toml | 4 +- .../attention/configs/gpdomain/default.toml | 0 .../attention/configs/memdomains/default.toml | 5 +- .../threads/attention/stack/CMakeLists.txt | 20 +- .../{components => prefab}/threads/build.sc | 0 .../threads/ffn/arch/FfnThread.scala | 0 .../threads/ffn/arch/acc.scala | 0 .../ffn/configs/balldomains/default.toml | 0 .../threads/ffn/configs/cpu/default.toml | 0 .../threads/ffn/configs/default.toml | 0 .../ffn}/configs/frontends/default.toml | 4 +- .../threads/ffn/configs/gpdomain/default.toml | 0 .../ffn/configs/memdomains/default.toml | 5 +- .../threads/ffn/stack/CMakeLists.txt | 20 +- .../threads/matmul/arch/MatMulThread.scala | 0 .../threads/matmul/arch/acc.scala | 0 .../threads/matmul/configs/ant.toml | 0 .../matmul/configs/balldomains/default.toml | 0 .../threads/matmul/configs/cpu/default.toml | 0 .../threads/matmul/configs/default.toml | 0 .../matmul}/configs/frontends/default.toml | 4 +- .../matmul/configs/gpdomain/default.toml | 0 .../matmul/configs/memdomains/default.toml | 5 +- .../threads/matmul/stack/CMakeLists.txt | 22 +- .../threads/matmul/stack/matmul.h | 0 .../threads/matmul/stack/prepare.c | 1 - .../threads/matmul/stack/program.c | 0 .../matmul/workloads/ctests/CMakeLists.txt | 2 +- .../workloads/ctests/rvv/CMakeLists.txt | 19 +- .../matmul/workloads/ctests/rvv/matmul.cpp | 0 .../workloads/ctests/rvv/mxfp8_quant.cpp | 0 .../matmul/workloads/ctests/rvv/norm.cpp | 3 +- .../matmul/workloads/ctests/rvv/silu.cpp | 0 .../matmul/workloads/ctests/rvv/snake.cpp | 0 .../matmul/workloads/ctests/rvv/swiglu.cpp | 0 .../CMakeLists.txt | 0 .../transpose_smatmul_quant_pipeline/goban.c | 12 +- .../transpose_smatmul_quant_pipeline-main.cpp | 2 +- scripts/nix/download.sh | 2 +- stack | 2 +- 1469 files changed, 29021 insertions(+), 7865 deletions(-) delete mode 100644 bb-tests/workloads/scripts/build.py delete mode 100644 examples/components/balls/f32add/stack/src/Lowering.cpp delete mode 100644 examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir delete mode 100644 examples/components/balls/gemmini/stack/src/Lowering.cpp delete mode 100644 examples/components/balls/im2col/stack/src/Lowering.cpp delete mode 100644 examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir delete mode 100644 examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp delete mode 100644 examples/components/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt delete mode 100644 examples/components/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows.mlir.in delete mode 100644 examples/components/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows_main.cpp delete mode 100644 examples/components/balls/int2fp/stack/src/Lowering.cpp delete mode 100644 examples/components/balls/int8add/stack/src/Lowering.cpp delete mode 100644 examples/components/balls/int8mul/stack/src/Lowering.cpp delete mode 100644 examples/components/balls/lut/stack/src/Lowering.cpp delete mode 100644 examples/components/balls/matadd/stack/src/Lowering.cpp delete mode 100644 examples/components/balls/maxpool/stack/src/Lowering.cpp delete mode 100644 examples/components/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt delete mode 100644 examples/components/balls/mxfp2int/emu/src/55_mxfp2int.rs delete mode 100644 examples/components/balls/mxfp2int/workloads/ctests/CMakeLists.txt delete mode 100644 examples/components/balls/mxfp2int/workloads/isa/mxfp2int.h delete mode 100644 examples/components/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt delete mode 100644 examples/components/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks.mlir delete mode 100644 examples/components/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt delete mode 100644 examples/components/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank.mlir.in delete mode 100644 examples/components/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank_main.cpp delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/CMakeLists.txt delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32_main.cpp delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32_main.cpp delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8.mlir delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_main.cpp delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window.mlir delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain.mlir delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain_main.cpp delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_main.cpp delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows.mlir delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows_main.cpp delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/CMakeLists.txt delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32_main.cpp delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32_main.cpp delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8.mlir delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_main.cpp delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window.mlir delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain.mlir delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain_main.cpp delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_main.cpp delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows.mlir delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows_main.cpp delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/mxmm_native_steps.h delete mode 100644 examples/components/balls/mxmm/workloads/mlir_tests/mxmm_native_test.h delete mode 100644 examples/components/balls/mxquant/workloads/mlir_tests/CMakeLists.txt delete mode 100644 examples/components/balls/mxquant/workloads/mlir_tests/ball/CMakeLists.txt delete mode 100644 examples/components/balls/mxquant/workloads/mlir_tests/ball/mxquant.mlir delete mode 100644 examples/components/balls/mxquant/workloads/mlir_tests/ball/mxquant_main.cpp delete mode 100644 examples/components/balls/mxquant/workloads/mlir_tests/bank/CMakeLists.txt delete mode 100644 examples/components/balls/mxquant/workloads/mlir_tests/bank/mxquant.mlir delete mode 100644 examples/components/balls/mxquant/workloads/mlir_tests/bank/mxquant_main.cpp delete mode 100644 examples/components/balls/redmule/stack/src/Lowering.cpp delete mode 100644 examples/components/balls/redmule/workloads/ctests/CMakeLists.txt delete mode 100644 examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir delete mode 100644 examples/components/balls/relu/stack/src/Lowering.cpp delete mode 100644 examples/components/balls/smatmul/stack/src/Lowering.cpp delete mode 100644 examples/components/balls/toint8/stack/src/Lowering.cpp delete mode 100644 examples/components/balls/trace/arch/README.md delete mode 100644 examples/components/balls/trace/emu/src/04_bdb_counter.rs delete mode 100644 examples/components/balls/trace/stack/src/Lowering.cpp delete mode 100644 examples/components/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir delete mode 100644 examples/components/balls/transpose/stack/src/Lowering.cpp delete mode 100644 examples/components/balls/vector/workloads/isa/vecmat16.h delete mode 100644 examples/components/balls/vector/workloads/mlir_tests/CMakeLists.txt delete mode 100644 examples/components/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt delete mode 100644 examples/components/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16.mlir delete mode 100644 examples/components/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth.mlir.in delete mode 100644 examples/components/cores/xiangshan/arch/build.sc create mode 100644 examples/designs/goban/configs/sims/verilator.toml delete mode 100644 examples/designs/goban/sims/device/stack/core/goban/CMakeLists.txt rename examples/designs/goban/{sims/device/stack/runtime => stack/device}/CMakeLists.txt (91%) create mode 100644 examples/designs/goban/stack/device/core/CMakeLists.txt delete mode 100644 examples/designs/goban/stack/engine/CMakeLists.txt delete mode 100644 examples/designs/goban/stack/midend/CMakeLists.txt create mode 100644 examples/designs/multi-rocket/configs/sims/verilator.toml rename examples/designs/multi-rocket/{sims/device/stack/runtime => stack/device}/CMakeLists.txt (94%) create mode 100644 examples/designs/pebble/configs/sims/verilator.toml delete mode 100644 examples/designs/pebble/sims/device/stack/core/pebble/CMakeLists.txt rename examples/designs/pebble/{sims/device/stack/runtime => stack/device}/CMakeLists.txt (91%) create mode 100644 examples/designs/pebble/stack/device/core/CMakeLists.txt create mode 100644 examples/designs/pebble/stack/device/core/src/ConvertLinalgToBuckyball.cpp rename examples/designs/pebble/stack/{midend => device/core}/src/FormCustomOps.cpp (67%) rename examples/designs/pebble/stack/{midend => device/core}/src/FormSpatialOps.cpp (61%) rename examples/designs/pebble/stack/{midend/src/LowerQuantizedOps.cpp => device/core/src/LowerChannelOps.cpp} (55%) create mode 100644 examples/designs/pebble/stack/device/core/src/LowerLinear.cpp create mode 100644 examples/designs/pebble/stack/device/core/src/LowerSpatialOps.cpp create mode 100644 examples/designs/pebble/stack/device/core/src/Lowering.h rename examples/designs/pebble/stack/{midend => device/core}/tests/form-custom-ops.mlir (100%) rename examples/designs/pebble/stack/{midend => device/core}/tests/form-depthwise.mlir (100%) rename examples/designs/pebble/stack/{midend => device/core}/tests/form-int32-bias.mlir (100%) create mode 100644 examples/designs/pebble/stack/device/core/tests/lower-bank-updates.mlir rename examples/designs/pebble/stack/{midend => device/core}/tests/lower-custom-ops.mlir (70%) rename examples/designs/pebble/stack/{midend => device/core}/tests/lower-frontend-types.mlir (100%) rename examples/designs/pebble/stack/{midend => device/core}/tests/unknown-custom-op.mlir (100%) delete mode 100644 examples/designs/pebble/stack/engine/CMakeLists.txt delete mode 100644 examples/designs/pebble/stack/midend/CMakeLists.txt delete mode 100644 examples/designs/pebble/stack/midend/src/ConvertLinalgToBuckyball.cpp delete mode 100644 examples/designs/pebble/stack/midend/src/LowerLinearTiles.cpp delete mode 100644 examples/designs/pebble/stack/midend/src/LowerSpatialOps.cpp delete mode 100644 examples/designs/pebble/stack/midend/src/Lowering.h delete mode 100644 examples/designs/pebble/stack/midend/tests/lower-bank-updates.mlir create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_flatten_linear.mlir create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_flatten_linear_main.cpp create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_resident.mlir create mode 100644 examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_resident_main.cpp create mode 100644 examples/designs/poly/configs/sims/p2e.toml create mode 100644 examples/designs/poly/configs/sims/tapeout.toml create mode 100644 examples/designs/poly/configs/sims/verilator.toml delete mode 100644 examples/designs/poly/sims/device/stack/core/poly/CMakeLists.txt rename examples/designs/poly/{sims/device/stack/runtime => stack/device}/CMakeLists.txt (91%) create mode 100644 examples/designs/poly/stack/device/core/CMakeLists.txt rename examples/designs/poly/stack/{engine => device/core}/execute.c (100%) delete mode 100644 examples/designs/poly/stack/engine/CMakeLists.txt delete mode 100644 examples/designs/poly/stack/midend/CMakeLists.txt create mode 100644 examples/designs/poly/workloads/device/core/engine/mlir/CMakeLists.txt create mode 100644 examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_f32_main.cpp create mode 100644 examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_fma32_main.cpp create mode 100644 examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_mxfp8_chain_main.cpp create mode 100644 examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_mxfp8_window_main.cpp create mode 100644 examples/designs/poly/workloads/device/core/engine/mlir/ball_mxquant_two_blocks_main.cpp create mode 100644 examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_f32_main.cpp create mode 100644 examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_fma32_main.cpp create mode 100644 examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_mxfp8_chain_main.cpp create mode 100644 examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_mxfp8_window_main.cpp create mode 100644 examples/designs/poly/workloads/device/core/engine/mlir/bank_mxquant_two_blocks_main.cpp create mode 100644 examples/designs/poly/workloads/device/core/thread/mlir/CMakeLists.txt create mode 100644 examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_f32_entry.c create mode 100644 examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_fma32_entry.c create mode 100644 examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_mxfp8_chain_entry.c create mode 100644 examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_mxfp8_window_entry.c create mode 100644 examples/designs/poly/workloads/device/core/thread/mlir/ball_mxquant_two_blocks_entry.c create mode 100644 examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_f32_entry.c create mode 100644 examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_fma32_entry.c create mode 100644 examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_mxfp8_chain_entry.c create mode 100644 examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_mxfp8_window_entry.c create mode 100644 examples/designs/poly/workloads/device/core/thread/mlir/bank_mxquant_two_blocks_entry.c create mode 100644 examples/designs/poly/workloads/device/core/thread/mlir/mlir.h create mode 100644 examples/designs/toy/configs/sims/verilator.toml delete mode 100644 examples/designs/toy/sims/device/stack/core/toy/CMakeLists.txt rename examples/designs/toy/{sims/device/stack/runtime => stack/device}/CMakeLists.txt (91%) create mode 100644 examples/designs/toy/stack/device/core/CMakeLists.txt rename examples/designs/toy/stack/{midend => device/core}/src/ConvertLinalgToBuckyball.cpp (94%) rename examples/designs/toy/stack/{midend => device/core}/src/FormCustomOps.cpp (100%) delete mode 100644 examples/designs/toy/stack/engine/CMakeLists.txt delete mode 100644 examples/designs/toy/stack/midend/CMakeLists.txt rename examples/{components => prefab}/balls/build.sc (100%) rename examples/{components => prefab}/balls/f32add/arch/F32AddBall.scala (98%) rename examples/{components => prefab}/balls/f32add/arch/F32AddRow.scala (100%) rename examples/{components => prefab}/balls/f32add/configs/default.toml (100%) rename examples/{components => prefab}/balls/f32add/configs/f32add_params.proto (100%) rename examples/{components => prefab}/balls/f32add/emu/src/add.rs (92%) rename examples/{components => prefab}/balls/f32add/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/f32add/emu/src/lib.rs (100%) create mode 100644 examples/prefab/balls/f32add/stack/src/f32addOps.cpp create mode 100644 examples/prefab/balls/f32add/stack/src/f32addOps.h create mode 100644 examples/prefab/balls/f32add/stack/src/f32addOps.td create mode 100644 examples/prefab/balls/f32add/verify/casegen/Cargo.lock create mode 100644 examples/prefab/balls/f32add/verify/casegen/Cargo.toml create mode 100644 examples/prefab/balls/f32add/verify/casegen/src/casegen.rs create mode 100644 examples/prefab/balls/f32add/verify/casegen/src/lib.rs create mode 100644 examples/prefab/balls/f32add/verify/casegen/src/model.rs create mode 100644 examples/prefab/balls/f32add/verify/filelists/f32add_ball.f create mode 100644 examples/prefab/balls/f32add/verify/src/common/f32add_defs.svh create mode 100644 examples/prefab/balls/f32add/verify/src/common/f32add_items.svh create mode 100644 examples/prefab/balls/f32add/verify/src/cov/f32add_cov.svh create mode 100644 examples/prefab/balls/f32add/verify/src/env/f32add_env.svh create mode 100644 examples/prefab/balls/f32add/verify/src/env/f32add_scoreboard.svh create mode 100644 examples/prefab/balls/f32add/verify/src/pkg/f32add_pkg.sv create mode 100644 examples/prefab/balls/f32add/verify/src/seq/f32add_sequences.svh create mode 100644 examples/prefab/balls/f32add/verify/src/tb_top.sv create mode 100644 examples/prefab/balls/f32add/verify/src/tests/f32add_test.svh rename examples/{components => prefab}/balls/f32add/workloads/ctests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/f32add/workloads/ctests/f32add_groups.c (95%) rename examples/{components => prefab}/balls/f32add/workloads/isa/f32add.h (71%) rename examples/{components => prefab}/balls/f32add/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/f32add/workloads/mlir_tests/ball/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir (78%) rename examples/{components => prefab}/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp (96%) create mode 100644 examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir rename examples/{components => prefab}/balls/f32add/workloads/mlir_tests/bank/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir (88%) rename examples/{components => prefab}/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp (100%) rename examples/{components => prefab}/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir (52%) rename examples/{components => prefab}/balls/gemmini/arch/GemminiBall.scala (100%) rename examples/{components => prefab}/balls/gemmini/arch/GemminiExCtrl.scala (100%) rename examples/{components => prefab}/balls/gemmini/arch/GemminiExCtrlAccumulateState.scala (100%) rename examples/{components => prefab}/balls/gemmini/arch/GemminiExCtrlCmdStates.scala (95%) rename examples/{components => prefab}/balls/gemmini/arch/GemminiExCtrlComputeFeedState.scala (100%) rename examples/{components => prefab}/balls/gemmini/arch/GemminiExCtrlComputeReadState.scala (100%) rename examples/{components => prefab}/balls/gemmini/arch/GemminiExCtrlDefaults.scala (100%) rename examples/{components => prefab}/balls/gemmini/arch/GemminiExCtrlDefs.scala (100%) rename examples/{components => prefab}/balls/gemmini/arch/GemminiExCtrlFsm.scala (100%) rename examples/{components => prefab}/balls/gemmini/arch/GemminiExCtrlPreloadStates.scala (100%) rename examples/{components => prefab}/balls/gemmini/arch/GemminiExCtrlStoreOps.scala (100%) rename examples/{components => prefab}/balls/gemmini/arch/LICENSE.gemmini (100%) rename examples/{components => prefab}/balls/gemmini/arch/Loop.scala (100%) rename examples/{components => prefab}/balls/gemmini/arch/LoopCmd.scala (100%) rename examples/{components => prefab}/balls/gemmini/arch/LoopConvAddrGen.scala (100%) rename examples/{components => prefab}/balls/gemmini/arch/LoopConvUnroller.scala (100%) rename examples/{components => prefab}/balls/gemmini/arch/LoopMatmulUnroller.scala (100%) rename examples/{components => prefab}/balls/gemmini/configs/GemminiBallParam.scala (100%) rename examples/{components => prefab}/balls/gemmini/configs/default.toml (100%) rename examples/{components => prefab}/balls/gemmini/configs/gemmini_params.proto (100%) rename examples/{components => prefab}/balls/gemmini/emu/src/02_gemmini_config.rs (100%) rename examples/{components => prefab}/balls/gemmini/emu/src/03_gemmini_flush.rs (100%) rename examples/{components => prefab}/balls/gemmini/emu/src/21_gemmini_preload.rs (89%) rename examples/{components => prefab}/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs (96%) rename examples/{components => prefab}/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs (92%) rename examples/{components => prefab}/balls/gemmini/emu/src/80_gemmini_loop_ws.rs (100%) rename examples/{components => prefab}/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs (100%) rename examples/{components => prefab}/balls/gemmini/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/gemmini/emu/src/gemmini_state.rs (100%) rename examples/{components => prefab}/balls/gemmini/emu/src/lib.rs (100%) rename examples/{components => prefab}/balls/gemmini/emu/src/loop_micro_ops.rs (90%) create mode 100644 examples/prefab/balls/gemmini/stack/src/gemminiOps.cpp create mode 100644 examples/prefab/balls/gemmini/stack/src/gemminiOps.h create mode 100644 examples/prefab/balls/gemmini/stack/src/gemminiOps.td rename examples/{components => prefab}/balls/gemmini/verify/casegen/Cargo.lock (100%) rename examples/{components => prefab}/balls/gemmini/verify/casegen/Cargo.toml (100%) rename examples/{components => prefab}/balls/gemmini/verify/casegen/src/lib.rs (100%) rename examples/{components => prefab}/balls/gemmini/verify/filelists/gemmini_ball.f (100%) rename examples/{components => prefab}/balls/gemmini/verify/src/common/gemmini_defs.svh (100%) rename examples/{components => prefab}/balls/gemmini/verify/src/gemmini_expansion_if.sv (100%) rename examples/{components => prefab}/balls/gemmini/verify/src/pkg/gemmini_pkg.sv (100%) rename examples/{components => prefab}/balls/gemmini/verify/src/tb_top.sv (100%) rename examples/{components => prefab}/balls/gemmini/verify/src/tests/gemmini_test.svh (100%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_accumulated_16x16_test.c (100%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_loop_mmio.c (89%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_loop_subrob.c (97%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c (100%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c (100%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c (100%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c (100%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c (100%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c (100%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c (100%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c (100%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c (100%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c (100%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c (100%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c (100%) rename examples/{components => prefab}/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c (100%) rename examples/{components => prefab}/balls/gemmini/workloads/isa/gemmini.h (100%) rename examples/{components => prefab}/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h (95%) rename examples/{components => prefab}/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h (95%) rename examples/{components => prefab}/balls/gemmini/workloads/isa/gemmini_config.h (94%) rename examples/{components => prefab}/balls/gemmini/workloads/isa/gemmini_flush.h (67%) rename examples/{components => prefab}/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h (81%) rename examples/{components => prefab}/balls/gemmini/workloads/isa/gemmini_loop_ws.h (77%) rename examples/{components => prefab}/balls/gemmini/workloads/isa/gemmini_preload.h (95%) rename examples/{components => prefab}/balls/gemmini/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/gemmini/workloads/mlir_tests/ball/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir (94%) rename examples/{components => prefab}/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16_main.cpp (100%) rename examples/{components => prefab}/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir (95%) rename examples/{components => prefab}/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16_main.cpp (100%) rename examples/{components => prefab}/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt (61%) rename examples/{components => prefab}/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir (97%) rename examples/{components => prefab}/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp (95%) rename examples/{components => prefab}/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir (97%) rename examples/{components => prefab}/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp (100%) rename examples/{components => prefab}/balls/im2col/arch/Im2col.scala (100%) rename examples/{components => prefab}/balls/im2col/arch/Im2colBall.scala (100%) rename examples/{components => prefab}/balls/im2col/arch/README.md (100%) rename examples/{components => prefab}/balls/im2col/arch/RestoringDiv.scala (100%) rename examples/{components => prefab}/balls/im2col/configs/Im2colBallParam.scala (100%) rename examples/{components => prefab}/balls/im2col/configs/default.toml (100%) rename examples/{components => prefab}/balls/im2col/configs/im2col_params.proto (100%) rename examples/{components => prefab}/balls/im2col/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/im2col/emu/src/im2col.rs (98%) rename examples/{components => prefab}/balls/im2col/emu/src/lib.rs (100%) create mode 100644 examples/prefab/balls/im2col/stack/src/im2colOps.cpp create mode 100644 examples/prefab/balls/im2col/stack/src/im2colOps.h create mode 100644 examples/prefab/balls/im2col/stack/src/im2colOps.td rename examples/{components => prefab}/balls/im2col/verify/README.md (100%) rename examples/{components => prefab}/balls/im2col/verify/casegen/Cargo.lock (100%) rename examples/{components => prefab}/balls/im2col/verify/casegen/Cargo.toml (100%) rename examples/{components => prefab}/balls/im2col/verify/casegen/src/casegen.rs (100%) rename examples/{components => prefab}/balls/im2col/verify/casegen/src/lib.rs (100%) rename examples/{components => prefab}/balls/im2col/verify/casegen/src/model.rs (100%) rename examples/{components => prefab}/balls/im2col/verify/filelists/im2col_ball.f (100%) rename examples/{components => prefab}/balls/im2col/verify/src/common/im2col_defs.svh (100%) rename examples/{components => prefab}/balls/im2col/verify/src/common/im2col_items.svh (100%) rename examples/{components => prefab}/balls/im2col/verify/src/cov/im2col_cov.svh (100%) rename examples/{components => prefab}/balls/im2col/verify/src/env/im2col_env.svh (100%) rename examples/{components => prefab}/balls/im2col/verify/src/env/im2col_scoreboard.svh (100%) rename examples/{components => prefab}/balls/im2col/verify/src/pkg/im2col_pkg.sv (100%) rename examples/{components => prefab}/balls/im2col/verify/src/seq/im2col_sequences.svh (100%) rename examples/{components => prefab}/balls/im2col/verify/src/tb_top.sv (100%) rename examples/{components => prefab}/balls/im2col/verify/src/tests/im2col_test.svh (100%) rename examples/{components => prefab}/balls/im2col/workloads/ctests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/im2col/workloads/ctests/im2col_k1_test.c (100%) rename examples/{components => prefab}/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c (100%) rename examples/{components => prefab}/balls/im2col/workloads/ctests/im2col_k3_pad_test.c (100%) rename examples/{components => prefab}/balls/im2col/workloads/ctests/im2col_k3_partial_test.c (100%) rename examples/{components => prefab}/balls/im2col/workloads/ctests/im2col_k3_test.c (100%) rename examples/{components => prefab}/balls/im2col/workloads/ctests/im2col_k5_test.c (100%) rename examples/{components => prefab}/balls/im2col/workloads/ctests/im2col_k7_test.c (100%) rename examples/{components => prefab}/balls/im2col/workloads/ctests/im2col_window_test.c (100%) rename examples/{components => prefab}/balls/im2col/workloads/isa/im2col.h (98%) rename examples/{components => prefab}/balls/im2col/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/im2col/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/prefab/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir create mode 100644 examples/prefab/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp rename examples/{components/balls/vector => prefab/balls/im2col}/workloads/mlir_tests/bank/CMakeLists.txt (60%) create mode 100644 examples/prefab/balls/im2col/workloads/mlir_tests/bank/im2col_k3_two_blocks.mlir create mode 100644 examples/prefab/balls/im2col/workloads/mlir_tests/bank/im2col_k3_two_blocks_main.cpp rename examples/{components => prefab}/balls/int2fp/arch/Fp32Mul.scala (100%) rename examples/{components => prefab}/balls/int2fp/arch/Int2Fp.scala (100%) rename examples/{components => prefab}/balls/int2fp/arch/Int2FpBall.scala (100%) rename examples/{components => prefab}/balls/int2fp/configs/Int2FpBallParam.scala (100%) rename examples/{components => prefab}/balls/int2fp/configs/default.toml (100%) rename examples/{components => prefab}/balls/int2fp/configs/int2fp_params.proto (100%) rename examples/{components => prefab}/balls/int2fp/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/int2fp/emu/src/int2fp.rs (96%) rename examples/{components => prefab}/balls/int2fp/emu/src/lib.rs (100%) rename examples/{components => prefab}/balls/int2fp/emu/src/model.rs (100%) create mode 100644 examples/prefab/balls/int2fp/stack/src/int2fpOps.cpp create mode 100644 examples/prefab/balls/int2fp/stack/src/int2fpOps.h create mode 100644 examples/prefab/balls/int2fp/stack/src/int2fpOps.td rename examples/{components => prefab}/balls/int2fp/verify/README.md (100%) rename examples/{components => prefab}/balls/int2fp/verify/casegen/Cargo.lock (100%) rename examples/{components => prefab}/balls/int2fp/verify/casegen/Cargo.toml (100%) rename examples/{components => prefab}/balls/int2fp/verify/casegen/src/casegen.rs (100%) rename examples/{components => prefab}/balls/int2fp/verify/casegen/src/lib.rs (100%) rename examples/{components => prefab}/balls/int2fp/verify/filelists/int2fp_ball.f (100%) rename examples/{components => prefab}/balls/int2fp/verify/src/common/int2fp_defs.svh (100%) rename examples/{components => prefab}/balls/int2fp/verify/src/common/int2fp_items.svh (100%) rename examples/{components => prefab}/balls/int2fp/verify/src/cov/int2fp_cov.svh (100%) rename examples/{components => prefab}/balls/int2fp/verify/src/env/int2fp_env.svh (100%) rename examples/{components => prefab}/balls/int2fp/verify/src/env/int2fp_scoreboard.svh (100%) rename examples/{components => prefab}/balls/int2fp/verify/src/pkg/int2fp_pkg.sv (100%) rename examples/{components => prefab}/balls/int2fp/verify/src/seq/int2fp_sequences.svh (100%) rename examples/{components => prefab}/balls/int2fp/verify/src/tb_top.sv (100%) rename examples/{components => prefab}/balls/int2fp/verify/src/tests/int2fp_test.svh (100%) rename examples/{components => prefab}/balls/int2fp/workloads/ctests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c (100%) rename examples/{components => prefab}/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c (100%) rename examples/{components => prefab}/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c (100%) rename examples/{components => prefab}/balls/int2fp/workloads/ctests/int2fp_groups_test.c (100%) rename examples/{components => prefab}/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c (100%) rename examples/{components => prefab}/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c (100%) rename examples/{components => prefab}/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c (100%) rename examples/{components => prefab}/balls/int2fp/workloads/isa/int2fp.h (92%) rename examples/{components => prefab}/balls/int2fp/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/int2fp/workloads/mlir_tests/ball/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir (92%) rename examples/{components => prefab}/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4_main.cpp (100%) rename examples/{components => prefab}/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir (92%) rename examples/{components => prefab}/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4_main.cpp (100%) rename examples/{components => prefab}/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt (62%) rename examples/{components => prefab}/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir (90%) rename examples/{components => prefab}/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4_main.cpp (100%) rename examples/{components => prefab}/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir (90%) rename examples/{components => prefab}/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4_main.cpp (100%) rename examples/{components => prefab}/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in (93%) rename examples/{components => prefab}/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp (73%) rename examples/{components => prefab}/balls/int8add/arch/Int8AddBall.scala (100%) rename examples/{components => prefab}/balls/int8add/arch/PipelinedAddRawFN.scala (100%) rename examples/{components => prefab}/balls/int8add/configs/default.toml (100%) rename examples/{components => prefab}/balls/int8add/configs/int8add_params.proto (100%) rename examples/{components => prefab}/balls/int8add/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/int8add/emu/src/int8add.rs (96%) rename examples/{components => prefab}/balls/int8add/emu/src/lib.rs (100%) create mode 100644 examples/prefab/balls/int8add/stack/src/int8addOps.cpp create mode 100644 examples/prefab/balls/int8add/stack/src/int8addOps.h create mode 100644 examples/prefab/balls/int8add/stack/src/int8addOps.td rename examples/{components => prefab}/balls/int8add/verify/casegen/Cargo.lock (100%) rename examples/{components => prefab}/balls/int8add/verify/casegen/Cargo.toml (100%) rename examples/{components => prefab}/balls/int8add/verify/casegen/src/casegen.rs (100%) rename examples/{components => prefab}/balls/int8add/verify/casegen/src/lib.rs (100%) rename examples/{components => prefab}/balls/int8add/verify/filelists/int8add_ball.f (100%) rename examples/{components => prefab}/balls/int8add/verify/src/common/int8add_defs.svh (100%) rename examples/{components => prefab}/balls/int8add/verify/src/common/int8add_items.svh (100%) rename examples/{components => prefab}/balls/int8add/verify/src/cov/int8add_cov.svh (100%) rename examples/{components => prefab}/balls/int8add/verify/src/env/int8add_env.svh (100%) rename examples/{components => prefab}/balls/int8add/verify/src/env/int8add_scoreboard.svh (100%) rename examples/{components => prefab}/balls/int8add/verify/src/pkg/int8add_pkg.sv (100%) rename examples/{components => prefab}/balls/int8add/verify/src/seq/int8add_sequences.svh (100%) rename examples/{components => prefab}/balls/int8add/verify/src/tb_top.sv (100%) rename examples/{components => prefab}/balls/int8add/verify/src/tests/int8add_test.svh (100%) rename examples/{components => prefab}/balls/int8add/workloads/ctests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/int8add/workloads/ctests/int8add_relu_test.c (94%) rename examples/{components => prefab}/balls/int8add/workloads/ctests/int8add_scaled_test.c (94%) rename examples/{components => prefab}/balls/int8add/workloads/isa/int8add.h (91%) rename examples/{components => prefab}/balls/int8add/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/int8add/workloads/mlir_tests/ball/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir (92%) rename examples/{components => prefab}/balls/int8add/workloads/mlir_tests/ball/int8add_relu_main.cpp (100%) rename examples/{components => prefab}/balls/int8add/workloads/mlir_tests/bank/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in (93%) rename examples/{components => prefab}/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp (68%) rename examples/{components => prefab}/balls/int8mul/arch/Int8MulBall.scala (100%) rename examples/{components => prefab}/balls/int8mul/configs/default.toml (100%) rename examples/{components => prefab}/balls/int8mul/configs/int8mul_params.proto (100%) rename examples/{components => prefab}/balls/int8mul/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/int8mul/emu/src/int8mul.rs (96%) rename examples/{components => prefab}/balls/int8mul/emu/src/lib.rs (100%) create mode 100644 examples/prefab/balls/int8mul/stack/src/int8mulOps.cpp create mode 100644 examples/prefab/balls/int8mul/stack/src/int8mulOps.h create mode 100644 examples/prefab/balls/int8mul/stack/src/int8mulOps.td rename examples/{components => prefab}/balls/int8mul/verify/casegen/Cargo.lock (100%) rename examples/{components => prefab}/balls/int8mul/verify/casegen/Cargo.toml (100%) rename examples/{components => prefab}/balls/int8mul/verify/casegen/src/casegen.rs (100%) rename examples/{components => prefab}/balls/int8mul/verify/casegen/src/config.rs (100%) rename examples/{components => prefab}/balls/int8mul/verify/casegen/src/lib.rs (100%) rename examples/{components => prefab}/balls/int8mul/verify/filelists/int8mul_ball.f (100%) rename examples/{components => prefab}/balls/int8mul/verify/src/common/int8mul_defs.svh (100%) rename examples/{components => prefab}/balls/int8mul/verify/src/common/int8mul_items.svh (100%) rename examples/{components => prefab}/balls/int8mul/verify/src/cov/int8mul_cov.svh (100%) rename examples/{components => prefab}/balls/int8mul/verify/src/env/int8mul_env.svh (100%) rename examples/{components => prefab}/balls/int8mul/verify/src/env/int8mul_scoreboard.svh (100%) rename examples/{components => prefab}/balls/int8mul/verify/src/pkg/int8mul_pkg.sv (100%) rename examples/{components => prefab}/balls/int8mul/verify/src/seq/int8mul_sequences.svh (100%) rename examples/{components => prefab}/balls/int8mul/verify/src/tb_top.sv (100%) rename examples/{components => prefab}/balls/int8mul/verify/src/tests/int8mul_test.svh (100%) rename examples/{components => prefab}/balls/int8mul/workloads/ctests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c (90%) rename examples/{components => prefab}/balls/int8mul/workloads/ctests/int8mul_per_channel_gate_test.c (100%) rename examples/{components => prefab}/balls/int8mul/workloads/isa/int8mul.h (91%) rename examples/{components => prefab}/balls/int8mul/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/int8mul/workloads/mlir_tests/ball/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir (93%) rename examples/{components => prefab}/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate_main.cpp (100%) rename examples/{components => prefab}/balls/int8mul/workloads/mlir_tests/bank/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in (93%) rename examples/{components => prefab}/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp (72%) rename examples/{components => prefab}/balls/lut/arch/LutBall.scala (100%) rename examples/{components => prefab}/balls/lut/configs/default.toml (100%) rename examples/{components => prefab}/balls/lut/configs/lut_params.proto (100%) rename examples/{components => prefab}/balls/lut/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/lut/emu/src/lib.rs (100%) rename examples/{components => prefab}/balls/lut/emu/src/lut.rs (95%) create mode 100644 examples/prefab/balls/lut/stack/src/lutOps.cpp create mode 100644 examples/prefab/balls/lut/stack/src/lutOps.h create mode 100644 examples/prefab/balls/lut/stack/src/lutOps.td rename examples/{components => prefab}/balls/lut/verify/casegen/Cargo.lock (100%) rename examples/{components => prefab}/balls/lut/verify/casegen/Cargo.toml (100%) rename examples/{components => prefab}/balls/lut/verify/casegen/src/casegen.rs (100%) rename examples/{components => prefab}/balls/lut/verify/casegen/src/lib.rs (100%) rename examples/{components => prefab}/balls/lut/verify/filelists/lut_ball.f (100%) rename examples/{components => prefab}/balls/lut/verify/src/common/lut_defs.svh (100%) rename examples/{components => prefab}/balls/lut/verify/src/common/lut_items.svh (100%) rename examples/{components => prefab}/balls/lut/verify/src/cov/lut_cov.svh (100%) rename examples/{components => prefab}/balls/lut/verify/src/env/lut_env.svh (100%) rename examples/{components => prefab}/balls/lut/verify/src/env/lut_scoreboard.svh (100%) rename examples/{components => prefab}/balls/lut/verify/src/pkg/lut_pkg.sv (100%) rename examples/{components => prefab}/balls/lut/verify/src/seq/lut_sequences.svh (100%) rename examples/{components => prefab}/balls/lut/verify/src/tb_top.sv (100%) rename examples/{components => prefab}/balls/lut/verify/src/tests/lut_test.svh (100%) rename examples/{components => prefab}/balls/lut/workloads/ctests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c (94%) rename examples/{components => prefab}/balls/lut/workloads/ctests/lut_uniform_lookup_test.c (92%) rename examples/{components => prefab}/balls/lut/workloads/isa/lut.h (88%) rename examples/{components => prefab}/balls/lut/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/lut/workloads/mlir_tests/ball/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir (95%) rename examples/{components => prefab}/balls/lut/workloads/mlir_tests/ball/lut_identity_main.cpp (100%) rename examples/{components => prefab}/balls/lut/workloads/mlir_tests/bank/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in (95%) rename examples/{components => prefab}/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp (69%) rename examples/{components => prefab}/balls/matadd/README.md (100%) rename examples/{components => prefab}/balls/matadd/arch/MatAdd.scala (100%) rename examples/{components => prefab}/balls/matadd/arch/MatAddBall.scala (100%) rename examples/{components => prefab}/balls/matadd/configs/default.toml (100%) rename examples/{components => prefab}/balls/matadd/configs/matadd_params.proto (100%) rename examples/{components => prefab}/balls/matadd/emu/src/64_matadd.rs (95%) rename examples/{components => prefab}/balls/matadd/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/matadd/emu/src/lib.rs (100%) create mode 100644 examples/prefab/balls/matadd/stack/src/mataddOps.cpp create mode 100644 examples/prefab/balls/matadd/stack/src/mataddOps.h create mode 100644 examples/prefab/balls/matadd/stack/src/mataddOps.td rename examples/{components => prefab}/balls/matadd/verify/casegen/Cargo.lock (100%) rename examples/{components => prefab}/balls/matadd/verify/casegen/Cargo.toml (100%) rename examples/{components => prefab}/balls/matadd/verify/casegen/src/lib.rs (100%) rename examples/{components => prefab}/balls/matadd/verify/filelists/matadd_ball.f (100%) rename examples/{components => prefab}/balls/matadd/verify/src/common/matadd_defs.svh (100%) rename examples/{components => prefab}/balls/matadd/verify/src/common/matadd_items.svh (100%) rename examples/{components => prefab}/balls/matadd/verify/src/env/matadd_env.svh (100%) rename examples/{components => prefab}/balls/matadd/verify/src/env/matadd_scoreboard.svh (100%) rename examples/{components => prefab}/balls/matadd/verify/src/pkg/matadd_pkg.sv (100%) rename examples/{components => prefab}/balls/matadd/verify/src/seq/matadd_sequences.svh (100%) rename examples/{components => prefab}/balls/matadd/verify/src/tb_top.sv (100%) rename examples/{components => prefab}/balls/matadd/verify/src/tests/matadd_test.svh (100%) rename examples/{components => prefab}/balls/matadd/workloads/ctests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/matadd/workloads/ctests/matadd_i32_16row_test.c (100%) rename examples/{components => prefab}/balls/matadd/workloads/isa/matadd.h (87%) rename examples/{components => prefab}/balls/matadd/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/matadd/workloads/mlir_tests/ball/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir (94%) rename examples/{components => prefab}/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp (96%) rename examples/{components => prefab}/balls/matadd/workloads/mlir_tests/bank/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in (94%) rename examples/{components => prefab}/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp (68%) rename examples/{components => prefab}/balls/maxpool/arch/MaxPoolBall.scala (100%) rename examples/{components => prefab}/balls/maxpool/configs/default.toml (100%) rename examples/{components => prefab}/balls/maxpool/configs/maxpool_params.proto (100%) rename examples/{components => prefab}/balls/maxpool/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/maxpool/emu/src/lib.rs (100%) rename examples/{components => prefab}/balls/maxpool/emu/src/maxpool.rs (94%) create mode 100644 examples/prefab/balls/maxpool/stack/src/maxpoolOps.cpp create mode 100644 examples/prefab/balls/maxpool/stack/src/maxpoolOps.h create mode 100644 examples/prefab/balls/maxpool/stack/src/maxpoolOps.td rename examples/{components => prefab}/balls/maxpool/verify/casegen/Cargo.lock (100%) rename examples/{components => prefab}/balls/maxpool/verify/casegen/Cargo.toml (100%) rename examples/{components => prefab}/balls/maxpool/verify/casegen/src/casegen.rs (100%) rename examples/{components => prefab}/balls/maxpool/verify/casegen/src/lib.rs (100%) rename examples/{components => prefab}/balls/maxpool/verify/casegen/src/model.rs (100%) rename examples/{components => prefab}/balls/maxpool/verify/filelists/maxpool_ball.f (100%) rename examples/{components => prefab}/balls/maxpool/verify/src/common/maxpool_defs.svh (100%) rename examples/{components => prefab}/balls/maxpool/verify/src/common/maxpool_items.svh (100%) rename examples/{components => prefab}/balls/maxpool/verify/src/cov/maxpool_cov.svh (100%) rename examples/{components => prefab}/balls/maxpool/verify/src/env/maxpool_env.svh (100%) rename examples/{components => prefab}/balls/maxpool/verify/src/env/maxpool_scoreboard.svh (100%) rename examples/{components => prefab}/balls/maxpool/verify/src/pkg/maxpool_pkg.sv (100%) rename examples/{components => prefab}/balls/maxpool/verify/src/seq/maxpool_sequences.svh (100%) rename examples/{components => prefab}/balls/maxpool/verify/src/tb_top.sv (100%) rename examples/{components => prefab}/balls/maxpool/verify/src/tests/maxpool_test.svh (100%) rename examples/{components => prefab}/balls/maxpool/workloads/ctests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/maxpool/workloads/ctests/maxpool_groups_test.c (100%) rename examples/{components => prefab}/balls/maxpool/workloads/ctests/maxpool_offset_test.c (93%) rename examples/{components => prefab}/balls/maxpool/workloads/ctests/maxpool_window_test.c (94%) rename examples/{components => prefab}/balls/maxpool/workloads/isa/maxpool.h (97%) rename examples/{components => prefab}/balls/maxpool/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/maxpool/workloads/mlir_tests/ball/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir (81%) rename examples/{components => prefab}/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2_main.cpp (100%) create mode 100644 examples/prefab/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt rename examples/{components => prefab}/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in (89%) rename examples/{components => prefab}/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp (85%) rename examples/{components => prefab}/balls/mxfp2int/arch/Mxfp2Int.scala (85%) rename examples/{components => prefab}/balls/mxfp2int/arch/Mxfp2IntBall.scala (88%) rename examples/{components => prefab}/balls/mxfp2int/arch/Mxfp4Decoder.scala (100%) rename examples/{components => prefab}/balls/mxfp2int/arch/Mxfp6Decoder.scala (100%) rename examples/{components => prefab}/balls/mxfp2int/arch/Mxfp8Decoder.scala (100%) rename examples/{components => prefab}/balls/mxfp2int/arch/MxfpDecoder.scala (85%) rename examples/{components => prefab}/balls/mxfp2int/configs/Mxfp2IntBallParam.scala (100%) rename examples/{components => prefab}/balls/mxfp2int/configs/default.toml (100%) rename examples/{components => prefab}/balls/mxfp2int/configs/mxfp2int_params.proto (100%) create mode 100644 examples/prefab/balls/mxfp2int/emu/src/55_mxfp2int.rs rename examples/{components => prefab}/balls/mxfp2int/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/mxfp2int/emu/src/lib.rs (100%) create mode 100644 examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.cpp create mode 100644 examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.h create mode 100644 examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.td create mode 100644 examples/prefab/balls/mxfp2int/verify/casegen/Cargo.lock create mode 100644 examples/prefab/balls/mxfp2int/verify/casegen/Cargo.toml create mode 100644 examples/prefab/balls/mxfp2int/verify/casegen/src/casegen.rs create mode 100644 examples/prefab/balls/mxfp2int/verify/casegen/src/config.rs create mode 100644 examples/prefab/balls/mxfp2int/verify/casegen/src/lib.rs create mode 100644 examples/prefab/balls/mxfp2int/verify/filelists/mxfp2int_ball.f create mode 100644 examples/prefab/balls/mxfp2int/verify/src/common/mxfp2int_defs.svh create mode 100644 examples/prefab/balls/mxfp2int/verify/src/common/mxfp2int_items.svh create mode 100644 examples/prefab/balls/mxfp2int/verify/src/cov/mxfp2int_cov.svh create mode 100644 examples/prefab/balls/mxfp2int/verify/src/env/mxfp2int_env.svh create mode 100644 examples/prefab/balls/mxfp2int/verify/src/env/mxfp2int_scoreboard.svh create mode 100644 examples/prefab/balls/mxfp2int/verify/src/pkg/mxfp2int_pkg.sv create mode 100644 examples/prefab/balls/mxfp2int/verify/src/seq/mxfp2int_sequences.svh create mode 100644 examples/prefab/balls/mxfp2int/verify/src/tb_top.sv create mode 100644 examples/prefab/balls/mxfp2int/verify/src/tests/mxfp2int_test.svh create mode 100644 examples/prefab/balls/mxfp2int/workloads/ctests/CMakeLists.txt create mode 100644 examples/prefab/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_scale_range_test.c rename examples/{components => prefab}/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c (71%) create mode 100644 examples/prefab/balls/mxfp2int/workloads/isa/mxfp2int.h rename examples/{components => prefab}/balls/mxfp2int/workloads/mlir_tests/CMakeLists.txt (100%) create mode 100644 examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt create mode 100644 examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_i8_2blocks.mlir rename examples/{components/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks_main.cpp => prefab/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_i8_2blocks_main.cpp} (73%) create mode 100644 examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt create mode 100644 examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_i8_bank_blocks.mlir.in create mode 100644 examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_i8_bank_blocks_main.cpp rename examples/{components => prefab}/balls/mxmm/arch/AddPipe.scala (100%) rename examples/{components => prefab}/balls/mxmm/arch/Array.scala (100%) rename examples/{components => prefab}/balls/mxmm/arch/InputWindow.scala (100%) rename examples/{components => prefab}/balls/mxmm/arch/Mxfp8Decode.scala (100%) rename examples/{components => prefab}/balls/mxmm/arch/MxmmBall.scala (100%) rename examples/{components => prefab}/balls/mxmm/arch/PE.scala (100%) rename examples/{components => prefab}/balls/mxmm/arch/Panels.scala (100%) rename examples/{components => prefab}/balls/mxmm/arch/Unit.scala (100%) rename examples/{components => prefab}/balls/mxmm/configs/MxmmBallParam.scala (100%) rename examples/{components => prefab}/balls/mxmm/configs/default.toml (100%) rename examples/{components => prefab}/balls/mxmm/configs/mxmm_params.proto (100%) rename examples/{components => prefab}/balls/mxmm/emu/src/accumulator.rs (100%) rename examples/{components => prefab}/balls/mxmm/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/mxmm/emu/src/fp32.rs (96%) rename examples/{components => prefab}/balls/mxmm/emu/src/lib.rs (100%) rename examples/{components => prefab}/balls/mxmm/emu/src/mxfp8.rs (97%) rename examples/{components => prefab}/balls/mxmm/stack/kernels/CMakeLists.txt (98%) rename examples/{components => prefab}/balls/mxmm/stack/kernels/dispatch.cpp (96%) rename examples/{components => prefab}/balls/mxmm/stack/kernels/norm_window.cpp (76%) rename examples/{components => prefab}/balls/mxmm/stack/kernels/norm_window_image.h (100%) rename examples/{components => prefab}/balls/mxmm/stack/kernels/pack_dispatch.cpp (95%) rename examples/{components => prefab}/balls/mxmm/stack/kernels/packmm.cpp (71%) rename examples/{components => prefab}/balls/mxmm/stack/kernels/packmm.h (100%) rename examples/{components => prefab}/balls/mxmm/stack/kernels/packmm_image.h (100%) rename examples/{components => prefab}/balls/mxmm/stack/kernels/window.h (100%) create mode 100644 examples/prefab/balls/mxmm/stack/src/mxmmOps.cpp create mode 100644 examples/prefab/balls/mxmm/stack/src/mxmmOps.h create mode 100644 examples/prefab/balls/mxmm/stack/src/mxmmOps.td create mode 100644 examples/prefab/balls/mxmm/verify/casegen/Cargo.lock create mode 100644 examples/prefab/balls/mxmm/verify/casegen/Cargo.toml create mode 100644 examples/prefab/balls/mxmm/verify/casegen/src/casegen.rs create mode 100644 examples/prefab/balls/mxmm/verify/casegen/src/config.rs create mode 100644 examples/prefab/balls/mxmm/verify/casegen/src/lib.rs create mode 100644 examples/prefab/balls/mxmm/verify/filelists/mxmm_ball.f create mode 100644 examples/prefab/balls/mxmm/verify/src/common/mxmm_defs.svh create mode 100644 examples/prefab/balls/mxmm/verify/src/common/mxmm_items.svh create mode 100644 examples/prefab/balls/mxmm/verify/src/cov/mxmm_cov.svh create mode 100644 examples/prefab/balls/mxmm/verify/src/env/mxmm_env.svh create mode 100644 examples/prefab/balls/mxmm/verify/src/env/mxmm_scoreboard.svh create mode 100644 examples/prefab/balls/mxmm/verify/src/pkg/mxmm_pkg.sv create mode 100644 examples/prefab/balls/mxmm/verify/src/seq/mxmm_sequences.svh create mode 100644 examples/prefab/balls/mxmm/verify/src/tb_top.sv create mode 100644 examples/prefab/balls/mxmm/verify/src/tests/mxmm_test.svh rename examples/{components => prefab}/balls/mxmm/workloads/ctests/CMakeLists.txt (56%) rename examples/{components => prefab}/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c (77%) rename examples/{components => prefab}/balls/mxmm/workloads/ctests/mxmm_fp32_test.c (100%) rename examples/{components => prefab}/balls/mxmm/workloads/ctests/mxmm_mxfp8_test.c (100%) rename examples/{components => prefab}/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c (78%) rename examples/{components => prefab}/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c (85%) rename examples/{components => prefab}/balls/mxmm/workloads/ctests/mxmm_window_test.c (100%) rename examples/{components => prefab}/balls/mxmm/workloads/isa/mxmm.h (87%) create mode 100644 examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir create mode 100644 examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir create mode 100644 examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_chain.mlir create mode 100644 examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_window.mlir create mode 100644 examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir create mode 100644 examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir create mode 100644 examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_chain.mlir create mode 100644 examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_window.mlir rename examples/{components => prefab}/balls/mxquant/arch/Encode.scala (100%) rename examples/{components => prefab}/balls/mxquant/arch/MxquantBall.scala (100%) rename examples/{components => prefab}/balls/mxquant/configs/default.toml (100%) rename examples/{components => prefab}/balls/mxquant/configs/mxquant_params.proto (100%) rename examples/{components => prefab}/balls/mxquant/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/mxquant/emu/src/lib.rs (100%) rename examples/{components => prefab}/balls/mxquant/emu/src/quant.rs (94%) rename examples/{components => prefab}/balls/mxquant/stack/kernels/CMakeLists.txt (88%) rename examples/{components => prefab}/balls/mxquant/stack/kernels/dispatch.cpp (100%) rename examples/{components => prefab}/balls/mxquant/stack/kernels/pack.cpp (100%) create mode 100644 examples/prefab/balls/mxquant/stack/src/mxquantOps.cpp create mode 100644 examples/prefab/balls/mxquant/stack/src/mxquantOps.h create mode 100644 examples/prefab/balls/mxquant/stack/src/mxquantOps.td create mode 100644 examples/prefab/balls/mxquant/verify/casegen/Cargo.lock create mode 100644 examples/prefab/balls/mxquant/verify/casegen/Cargo.toml create mode 100644 examples/prefab/balls/mxquant/verify/casegen/src/casegen.rs create mode 100644 examples/prefab/balls/mxquant/verify/casegen/src/config.rs create mode 100644 examples/prefab/balls/mxquant/verify/casegen/src/lib.rs create mode 100644 examples/prefab/balls/mxquant/verify/filelists/mxquant_ball.f create mode 100644 examples/prefab/balls/mxquant/verify/src/common/mxquant_defs.svh create mode 100644 examples/prefab/balls/mxquant/verify/src/common/mxquant_items.svh create mode 100644 examples/prefab/balls/mxquant/verify/src/cov/mxquant_cov.svh create mode 100644 examples/prefab/balls/mxquant/verify/src/env/mxquant_env.svh create mode 100644 examples/prefab/balls/mxquant/verify/src/env/mxquant_scoreboard.svh create mode 100644 examples/prefab/balls/mxquant/verify/src/pkg/mxquant_pkg.sv create mode 100644 examples/prefab/balls/mxquant/verify/src/seq/mxquant_sequences.svh create mode 100644 examples/prefab/balls/mxquant/verify/src/tb_top.sv create mode 100644 examples/prefab/balls/mxquant/verify/src/tests/mxquant_test.svh rename examples/{components => prefab}/balls/mxquant/workloads/ctests/CMakeLists.txt (54%) rename examples/{components => prefab}/balls/mxquant/workloads/ctests/mxquant_capacity_test.c (77%) rename examples/{components => prefab}/balls/mxquant/workloads/ctests/mxquant_values_test.c (100%) rename examples/{components => prefab}/balls/mxquant/workloads/isa/mxquant.h (86%) create mode 100644 examples/prefab/balls/mxquant/workloads/mlir_tests/ball/mxquant_two_blocks.mlir create mode 100644 examples/prefab/balls/mxquant/workloads/mlir_tests/bank/mxquant_two_blocks.mlir rename examples/{components => prefab}/balls/mxquant/workloads/reference.h (100%) rename examples/{components => prefab}/balls/redmule/arch/RedmuleBall.scala (99%) rename examples/{components => prefab}/balls/redmule/arch/RedmuleNative.scala (79%) rename examples/{components => prefab}/balls/redmule/arch/RedmuleNative.sv (100%) rename examples/{components => prefab}/balls/redmule/configs/default.toml (100%) rename examples/{components => prefab}/balls/redmule/configs/redmule_params.proto (100%) rename examples/{components => prefab}/balls/redmule/emu/src/fp16.rs (100%) rename examples/{components => prefab}/balls/redmule/emu/src/lib.rs (100%) rename examples/{components => prefab}/balls/redmule/emu/src/matmul.rs (90%) create mode 100644 examples/prefab/balls/redmule/stack/src/redmuleOps.cpp create mode 100644 examples/prefab/balls/redmule/stack/src/redmuleOps.h create mode 100644 examples/prefab/balls/redmule/stack/src/redmuleOps.td rename examples/{components => prefab}/balls/redmule/verify/casegen/Cargo.lock (100%) rename examples/{components => prefab}/balls/redmule/verify/casegen/Cargo.toml (100%) rename examples/{components => prefab}/balls/redmule/verify/casegen/src/lib.rs (100%) rename examples/{components => prefab}/balls/redmule/verify/filelists/redmule_ball.f (100%) rename examples/{components => prefab}/balls/redmule/verify/parameters.toml (100%) rename examples/{components => prefab}/balls/redmule/verify/src/pkg/redmule_pkg.sv (100%) rename examples/{components => prefab}/balls/redmule/verify/src/reference/redmule_reference.sv (100%) rename examples/{components => prefab}/balls/redmule/verify/src/tb_top.sv (100%) create mode 100644 examples/prefab/balls/redmule/workloads/ctests/CMakeLists.txt rename examples/{components => prefab}/balls/redmule/workloads/ctests/redmule_f16_chain_test.c (93%) rename examples/{components => prefab}/balls/redmule/workloads/ctests/redmule_f16_edges_test.c (98%) rename examples/{components => prefab}/balls/redmule/workloads/ctests/redmule_f16_fused_test.c (97%) rename examples/{components => prefab}/balls/redmule/workloads/ctests/redmule_f16_test.c (94%) rename examples/{components => prefab}/balls/redmule/workloads/isa/redmule.h (94%) rename examples/{components => prefab}/balls/redmule/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/redmule/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir rename examples/{components => prefab}/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp (96%) rename examples/{components => prefab}/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir (59%) rename examples/{components => prefab}/balls/redmule/workloads/mlir_tests/bank/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir (81%) rename examples/{components => prefab}/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in (94%) rename examples/{components => prefab}/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows_main.cpp (86%) rename examples/{components => prefab}/balls/relu/README.md (100%) rename examples/{components => prefab}/balls/relu/arch/Relu.scala (100%) rename examples/{components => prefab}/balls/relu/arch/ReluBall.scala (100%) rename examples/{components => prefab}/balls/relu/configs/default.toml (100%) rename examples/{components => prefab}/balls/relu/configs/relu_params.proto (100%) rename examples/{components => prefab}/balls/relu/emu/src/50_relu.rs (85%) rename examples/{components => prefab}/balls/relu/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/relu/emu/src/lib.rs (100%) create mode 100644 examples/prefab/balls/relu/stack/src/reluOps.cpp create mode 100644 examples/prefab/balls/relu/stack/src/reluOps.h create mode 100644 examples/prefab/balls/relu/stack/src/reluOps.td create mode 100644 examples/prefab/balls/relu/verify/casegen/Cargo.lock create mode 100644 examples/prefab/balls/relu/verify/casegen/Cargo.toml create mode 100644 examples/prefab/balls/relu/verify/casegen/src/casegen.rs create mode 100644 examples/prefab/balls/relu/verify/casegen/src/lib.rs create mode 100644 examples/prefab/balls/relu/verify/casegen/src/model.rs create mode 100644 examples/prefab/balls/relu/verify/filelists/relu_ball.f create mode 100644 examples/prefab/balls/relu/verify/src/common/relu_defs.svh create mode 100644 examples/prefab/balls/relu/verify/src/common/relu_items.svh create mode 100644 examples/prefab/balls/relu/verify/src/cov/relu_cov.svh create mode 100644 examples/prefab/balls/relu/verify/src/env/relu_env.svh create mode 100644 examples/prefab/balls/relu/verify/src/env/relu_scoreboard.svh create mode 100644 examples/prefab/balls/relu/verify/src/pkg/relu_pkg.sv create mode 100644 examples/prefab/balls/relu/verify/src/seq/relu_sequences.svh create mode 100644 examples/prefab/balls/relu/verify/src/tb_top.sv create mode 100644 examples/prefab/balls/relu/verify/src/tests/relu_test.svh rename examples/{components => prefab}/balls/relu/workloads/ctests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/relu/workloads/ctests/relu_i32_16x16_test.c (100%) rename examples/{components => prefab}/balls/relu/workloads/isa/relu.h (88%) rename examples/{components/balls/mxmm => prefab/balls/relu}/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/relu/workloads/mlir_tests/ball/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir (93%) rename examples/{components => prefab}/balls/relu/workloads/mlir_tests/ball/relu_i32_16row_main.cpp (100%) rename examples/{components => prefab}/balls/relu/workloads/mlir_tests/bank/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in (94%) rename examples/{components => prefab}/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp (68%) rename examples/{components => prefab}/balls/smatmul/README.md (100%) rename examples/{components => prefab}/balls/smatmul/arch/Array.scala (100%) rename examples/{components => prefab}/balls/smatmul/arch/PE.scala (100%) rename examples/{components => prefab}/balls/smatmul/arch/SMatMulBall.scala (100%) rename examples/{components => prefab}/balls/smatmul/arch/SMatMulUnit.scala (100%) rename examples/{components => prefab}/balls/smatmul/configs/SMatMulBallParam.scala (100%) rename examples/{components => prefab}/balls/smatmul/configs/default.toml (100%) rename examples/{components => prefab}/balls/smatmul/configs/lane16.toml (100%) rename examples/{components => prefab}/balls/smatmul/configs/smatmul_params.proto (100%) rename examples/{components => prefab}/balls/smatmul/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/smatmul/emu/src/fp32.rs (95%) rename examples/{components => prefab}/balls/smatmul/emu/src/lib.rs (100%) rename examples/{components => prefab}/balls/smatmul/emu/src/mxfp8.rs (97%) rename examples/{components => prefab}/balls/smatmul/emu/src/smatmul.rs (97%) create mode 100644 examples/prefab/balls/smatmul/stack/src/smatmulOps.cpp create mode 100644 examples/prefab/balls/smatmul/stack/src/smatmulOps.h create mode 100644 examples/prefab/balls/smatmul/stack/src/smatmulOps.td rename examples/{components => prefab}/balls/smatmul/verify/README.md (100%) rename examples/{components => prefab}/balls/smatmul/verify/casegen/Cargo.lock (100%) rename examples/{components => prefab}/balls/smatmul/verify/casegen/Cargo.toml (100%) rename examples/{components => prefab}/balls/smatmul/verify/casegen/src/casegen.rs (100%) rename examples/{components => prefab}/balls/smatmul/verify/casegen/src/lib.rs (100%) rename examples/{components => prefab}/balls/smatmul/verify/casegen/src/model.rs (100%) rename examples/{components => prefab}/balls/smatmul/verify/filelists/smatmul_ball.f (100%) rename examples/{components => prefab}/balls/smatmul/verify/parameters.toml (100%) rename examples/{components => prefab}/balls/smatmul/verify/src/common/smatmul_defs.svh (100%) rename examples/{components => prefab}/balls/smatmul/verify/src/common/smatmul_items.svh (100%) rename examples/{components => prefab}/balls/smatmul/verify/src/cov/smatmul_cov.svh (100%) rename examples/{components => prefab}/balls/smatmul/verify/src/env/smatmul_env.svh (100%) rename examples/{components => prefab}/balls/smatmul/verify/src/env/smatmul_scoreboard.svh (100%) rename examples/{components => prefab}/balls/smatmul/verify/src/pkg/smatmul_pkg.sv (100%) rename examples/{components => prefab}/balls/smatmul/verify/src/seq/smatmul_sequences.svh (100%) rename examples/{components => prefab}/balls/smatmul/verify/src/tb_top.sv (100%) rename examples/{components => prefab}/balls/smatmul/verify/src/tests/smatmul_test.svh (100%) rename examples/{components => prefab}/balls/smatmul/workloads/ctests/CMakeLists.txt (74%) rename examples/{components => prefab}/balls/smatmul/workloads/ctests/smatmul_16x16_test.c (100%) rename examples/{components => prefab}/balls/smatmul/workloads/ctests/smatmul_16x16x32_test.c (100%) rename examples/{components => prefab}/balls/smatmul/workloads/ctests/smatmul_1x16x64_test.c (100%) rename examples/{components => prefab}/balls/smatmul/workloads/ctests/smatmul_8x16_test.c (100%) rename examples/{components => prefab}/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c (100%) rename examples/{components => prefab}/balls/smatmul/workloads/ctests/smatmul_c_groups_test.c (76%) rename examples/{components => prefab}/balls/smatmul/workloads/ctests/smatmul_f32_test.c (100%) rename examples/{components => prefab}/balls/smatmul/workloads/ctests/smatmul_k_groups_test.c (100%) rename examples/{components => prefab}/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c (90%) rename examples/{components => prefab}/balls/smatmul/workloads/isa/smatmul.h (86%) rename examples/{components => prefab}/balls/smatmul/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/smatmul/workloads/mlir_tests/ball/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir (87%) rename examples/{components => prefab}/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp (94%) rename examples/{components => prefab}/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt (54%) rename examples/{components => prefab}/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in (88%) rename examples/{components => prefab}/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp (74%) rename examples/{components => prefab}/balls/toint8/arch/ToInt8Ball.scala (100%) rename examples/{components => prefab}/balls/toint8/configs/default.toml (100%) rename examples/{components => prefab}/balls/toint8/configs/toint8_params.proto (100%) rename examples/{components => prefab}/balls/toint8/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/toint8/emu/src/lib.rs (100%) rename examples/{components => prefab}/balls/toint8/emu/src/model.rs (100%) rename examples/{components => prefab}/balls/toint8/emu/src/quant.rs (96%) create mode 100644 examples/prefab/balls/toint8/stack/src/toint8Ops.cpp create mode 100644 examples/prefab/balls/toint8/stack/src/toint8Ops.h create mode 100644 examples/prefab/balls/toint8/stack/src/toint8Ops.td rename examples/{components => prefab}/balls/toint8/verify/README.md (100%) rename examples/{components => prefab}/balls/toint8/verify/casegen/Cargo.lock (100%) rename examples/{components => prefab}/balls/toint8/verify/casegen/Cargo.toml (100%) rename examples/{components => prefab}/balls/toint8/verify/casegen/src/casegen.rs (100%) rename examples/{components => prefab}/balls/toint8/verify/casegen/src/lib.rs (100%) rename examples/{components => prefab}/balls/toint8/verify/filelists/toint8_ball.f (100%) rename examples/{components => prefab}/balls/toint8/verify/src/common/toint8_defs.svh (100%) rename examples/{components => prefab}/balls/toint8/verify/src/common/toint8_items.svh (100%) rename examples/{components => prefab}/balls/toint8/verify/src/cov/toint8_cov.svh (100%) rename examples/{components => prefab}/balls/toint8/verify/src/env/toint8_env.svh (100%) rename examples/{components => prefab}/balls/toint8/verify/src/env/toint8_scoreboard.svh (100%) rename examples/{components => prefab}/balls/toint8/verify/src/pkg/toint8_pkg.sv (100%) rename examples/{components => prefab}/balls/toint8/verify/src/seq/toint8_sequences.svh (100%) rename examples/{components => prefab}/balls/toint8/verify/src/tb_top.sv (100%) rename examples/{components => prefab}/balls/toint8/verify/src/tests/toint8_test.svh (100%) rename examples/{components => prefab}/balls/toint8/workloads/ctests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/toint8/workloads/ctests/quant_f32_groups_test.c (100%) rename examples/{components => prefab}/balls/toint8/workloads/ctests/quant_f32_to_i8_test.c (100%) rename examples/{components => prefab}/balls/toint8/workloads/ctests/quant_i32_groups_test.c (100%) rename examples/{components => prefab}/balls/toint8/workloads/ctests/quant_i32_to_i8_test.c (100%) rename examples/{components => prefab}/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c (100%) rename examples/{components => prefab}/balls/toint8/workloads/ctests/toint8_i8_rows_test.c (100%) rename examples/{components => prefab}/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c (100%) rename examples/{components => prefab}/balls/toint8/workloads/ctests/toint8_i8_signed_test.c (100%) rename examples/{components => prefab}/balls/toint8/workloads/isa/quant.h (92%) rename examples/{components => prefab}/balls/toint8/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/toint8/workloads/mlir_tests/ball/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir (91%) rename examples/{components => prefab}/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8_main.cpp (100%) rename examples/{components => prefab}/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir (95%) rename examples/{components => prefab}/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp (100%) rename examples/{components => prefab}/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt (72%) rename examples/{components => prefab}/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in (93%) rename examples/{components => prefab}/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp (75%) rename examples/{components => prefab}/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in (93%) rename examples/{components => prefab}/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp (68%) create mode 100644 examples/prefab/balls/trace/arch/README.md rename examples/{components => prefab}/balls/trace/arch/Trace.scala (97%) rename examples/{components => prefab}/balls/trace/arch/TraceBall.scala (100%) rename examples/{components => prefab}/balls/trace/arch/TraceDPI.scala (100%) rename examples/{components => prefab}/balls/trace/configs/default.toml (100%) rename examples/{components => prefab}/balls/trace/configs/trace_params.proto (100%) create mode 100644 examples/prefab/balls/trace/emu/src/04_bdb_counter.rs rename examples/{components => prefab}/balls/trace/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/trace/emu/src/lib.rs (100%) create mode 100644 examples/prefab/balls/trace/stack/src/traceOps.cpp create mode 100644 examples/prefab/balls/trace/stack/src/traceOps.h create mode 100644 examples/prefab/balls/trace/stack/src/traceOps.td create mode 100644 examples/prefab/balls/trace/verify/casegen/Cargo.lock create mode 100644 examples/prefab/balls/trace/verify/casegen/Cargo.toml create mode 100644 examples/prefab/balls/trace/verify/casegen/src/casegen.rs create mode 100644 examples/prefab/balls/trace/verify/casegen/src/lib.rs create mode 100644 examples/prefab/balls/trace/verify/casegen/src/model.rs create mode 100644 examples/prefab/balls/trace/verify/filelists/trace_ball.f create mode 100644 examples/prefab/balls/trace/verify/src/common/trace_defs.svh create mode 100644 examples/prefab/balls/trace/verify/src/common/trace_items.svh create mode 100644 examples/prefab/balls/trace/verify/src/cov/trace_cov.svh create mode 100644 examples/prefab/balls/trace/verify/src/env/trace_env.svh create mode 100644 examples/prefab/balls/trace/verify/src/env/trace_scoreboard.svh create mode 100644 examples/prefab/balls/trace/verify/src/pkg/trace_pkg.sv create mode 100644 examples/prefab/balls/trace/verify/src/seq/trace_sequences.svh create mode 100644 examples/prefab/balls/trace/verify/src/tb_top.sv create mode 100644 examples/prefab/balls/trace/verify/src/tests/trace_test.svh rename examples/{components => prefab}/balls/trace/workloads/ctests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/trace/workloads/ctests/bdb_counter_multi_id_test.c (100%) rename examples/{components => prefab}/balls/trace/workloads/ctests/bdb_counter_nested_pair_test.c (100%) rename examples/{components => prefab}/balls/trace/workloads/ctests/bdb_counter_read_active_test.c (100%) rename examples/{components => prefab}/balls/trace/workloads/ctests/bdb_counter_start_stop_test.c (100%) rename examples/{components => prefab}/balls/trace/workloads/isa/bdb_counter.h (81%) rename examples/{components => prefab}/balls/trace/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/trace/workloads/mlir_tests/ball/CMakeLists.txt (100%) create mode 100644 examples/prefab/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir rename examples/{components => prefab}/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop_main.cpp (100%) rename examples/{components => prefab}/balls/transpose/arch/README.md (95%) rename examples/{components => prefab}/balls/transpose/arch/Transpose.scala (100%) rename examples/{components => prefab}/balls/transpose/arch/TransposeBall.scala (100%) rename examples/{components => prefab}/balls/transpose/configs/TransposeBallParam.scala (100%) rename examples/{components => prefab}/balls/transpose/configs/default.toml (100%) rename examples/{components => prefab}/balls/transpose/configs/transpose_params.proto (100%) rename examples/{components => prefab}/balls/transpose/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/transpose/emu/src/lib.rs (100%) rename examples/{components => prefab}/balls/transpose/emu/src/transpose.rs (94%) create mode 100644 examples/prefab/balls/transpose/stack/src/transposeOps.cpp create mode 100644 examples/prefab/balls/transpose/stack/src/transposeOps.h create mode 100644 examples/prefab/balls/transpose/stack/src/transposeOps.td rename examples/{components => prefab}/balls/transpose/verify/README.md (100%) rename examples/{components => prefab}/balls/transpose/verify/casegen/Cargo.lock (100%) rename examples/{components => prefab}/balls/transpose/verify/casegen/Cargo.toml (100%) rename examples/{components => prefab}/balls/transpose/verify/casegen/src/casegen.rs (100%) rename examples/{components => prefab}/balls/transpose/verify/casegen/src/lib.rs (100%) rename examples/{components => prefab}/balls/transpose/verify/casegen/src/model.rs (100%) rename examples/{components => prefab}/balls/transpose/verify/filelists/transpose_ball.f (100%) rename examples/{components => prefab}/balls/transpose/verify/src/common/transpose_defs.svh (100%) rename examples/{components => prefab}/balls/transpose/verify/src/common/transpose_items.svh (100%) rename examples/{components => prefab}/balls/transpose/verify/src/cov/transpose_cov.svh (100%) rename examples/{components => prefab}/balls/transpose/verify/src/env/transpose_env.svh (100%) rename examples/{components => prefab}/balls/transpose/verify/src/env/transpose_scoreboard.svh (100%) rename examples/{components => prefab}/balls/transpose/verify/src/pkg/transpose_pkg.sv (100%) rename examples/{components => prefab}/balls/transpose/verify/src/seq/transpose_sequences.svh (100%) rename examples/{components => prefab}/balls/transpose/verify/src/tb_top.sv (100%) rename examples/{components => prefab}/balls/transpose/verify/src/tests/transpose_test.svh (100%) rename examples/{components => prefab}/balls/transpose/workloads/ctests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c (97%) rename examples/{components => prefab}/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c (97%) rename examples/{components => prefab}/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c (98%) rename examples/{components => prefab}/balls/transpose/workloads/isa/transpose.h (87%) rename examples/{components => prefab}/balls/transpose/workloads/mlir_tests/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/transpose/workloads/mlir_tests/ball/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir (92%) rename examples/{components => prefab}/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8_main.cpp (100%) rename examples/{components => prefab}/balls/transpose/workloads/mlir_tests/bank/CMakeLists.txt (100%) rename examples/{components => prefab}/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in (92%) rename examples/{components => prefab}/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp (67%) rename examples/{components => prefab}/balls/vector/arch/README.md (100%) rename examples/{components => prefab}/balls/vector/arch/VecBall.scala (90%) rename examples/{components => prefab}/balls/vector/arch/VecCtrlUnit.scala (100%) rename examples/{components => prefab}/balls/vector/arch/VecEXUnit.scala (100%) rename examples/{components => prefab}/balls/vector/arch/VecLoadUnit.scala (100%) rename examples/{components => prefab}/balls/vector/arch/VecMat16.scala (75%) rename examples/{components => prefab}/balls/vector/arch/VecMat16AccLoad.scala (91%) rename examples/{components => prefab}/balls/vector/arch/VecMat16Store.scala (88%) rename examples/{components => prefab}/balls/vector/arch/VecMat16Tie.scala (96%) rename examples/{components => prefab}/balls/vector/arch/VecStoreUnit.scala (100%) rename examples/{components => prefab}/balls/vector/arch/VecUnit.scala (100%) rename examples/{components => prefab}/balls/vector/arch/bond/BondWrapper.scala (100%) rename examples/{components => prefab}/balls/vector/arch/bond/README.md (100%) rename examples/{components => prefab}/balls/vector/arch/bond/vvv.scala (100%) rename examples/{components => prefab}/balls/vector/arch/op/README.md (100%) rename examples/{components => prefab}/balls/vector/arch/op/cascade.scala (100%) rename examples/{components => prefab}/balls/vector/arch/op/mul.scala (100%) rename examples/{components => prefab}/balls/vector/arch/thread/BaseThread.scala (100%) rename examples/{components => prefab}/balls/vector/arch/thread/CasThread.scala (100%) rename examples/{components => prefab}/balls/vector/arch/thread/MulThread.scala (100%) rename examples/{components => prefab}/balls/vector/arch/thread/README.md (100%) rename examples/{components => prefab}/balls/vector/arch/warp/MeshWarp.scala (100%) rename examples/{components => prefab}/balls/vector/arch/warp/README.md (100%) rename examples/{components => prefab}/balls/vector/configs/VectorBallParam.scala (100%) rename examples/{components => prefab}/balls/vector/configs/default.toml (100%) rename examples/{components => prefab}/balls/vector/configs/vector_params.proto (100%) rename examples/{components => prefab}/balls/vector/emu/src/64_vecmat16.rs (76%) rename examples/{components => prefab}/balls/vector/emu/src/dispatch.rs (100%) rename examples/{components => prefab}/balls/vector/emu/src/lib.rs (100%) create mode 100644 examples/prefab/balls/vector/stack/src/vectorOps.cpp create mode 100644 examples/prefab/balls/vector/stack/src/vectorOps.h create mode 100644 examples/prefab/balls/vector/stack/src/vectorOps.td create mode 100644 examples/prefab/balls/vector/verify/casegen/Cargo.lock create mode 100644 examples/prefab/balls/vector/verify/casegen/Cargo.toml create mode 100644 examples/prefab/balls/vector/verify/casegen/src/casegen.rs create mode 100644 examples/prefab/balls/vector/verify/casegen/src/lib.rs create mode 100644 examples/prefab/balls/vector/verify/casegen/src/model.rs create mode 100644 examples/prefab/balls/vector/verify/filelists/vector_ball.f create mode 100644 examples/prefab/balls/vector/verify/src/common/vector_defs.svh create mode 100644 examples/prefab/balls/vector/verify/src/common/vector_items.svh create mode 100644 examples/prefab/balls/vector/verify/src/cov/vector_cov.svh create mode 100644 examples/prefab/balls/vector/verify/src/env/vector_env.svh create mode 100644 examples/prefab/balls/vector/verify/src/env/vector_scoreboard.svh create mode 100644 examples/prefab/balls/vector/verify/src/pkg/vector_pkg.sv create mode 100644 examples/prefab/balls/vector/verify/src/seq/vector_sequences.svh create mode 100644 examples/prefab/balls/vector/verify/src/tb_top.sv create mode 100644 examples/prefab/balls/vector/verify/src/tests/vector_test.svh rename examples/{components => prefab}/balls/vector/workloads/ctests/CMakeLists.txt (51%) create mode 100644 examples/prefab/balls/vector/workloads/ctests/vecmat16_accumulate_k4_test.c rename examples/{components => prefab}/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c (76%) create mode 100644 examples/prefab/balls/vector/workloads/isa/vecmat16.h rename examples/{components/balls/relu => prefab/balls/vector}/workloads/mlir_tests/CMakeLists.txt (100%) create mode 100644 examples/prefab/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt create mode 100644 examples/prefab/balls/vector/workloads/mlir_tests/ball/matmul_at_i8_16x16.mlir rename examples/{components/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16_main.cpp => prefab/balls/vector/workloads/mlir_tests/ball/matmul_at_i8_16x16_main.cpp} (80%) create mode 100644 examples/prefab/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt create mode 100644 examples/prefab/balls/vector/workloads/mlir_tests/bank/matmul_at_i8_bank_depth.mlir.in rename examples/{components/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth_main.cpp => prefab/balls/vector/workloads/mlir_tests/bank/matmul_at_i8_bank_depth_main.cpp} (70%) rename examples/{components => prefab}/cores/boom/LICENSE (100%) rename examples/{components => prefab}/cores/boom/arch/Boom.scala (100%) rename examples/{components => prefab}/cores/boom/arch/Native.scala (100%) rename examples/{components => prefab}/cores/boom/arch/build.sc (87%) create mode 100644 examples/prefab/cores/boom/arch/native/Core.scala create mode 100644 examples/prefab/cores/boom/arch/native/Decode.scala create mode 100644 examples/prefab/cores/boom/arch/native/ExecutionUnit.scala create mode 100644 examples/prefab/cores/boom/arch/native/Native.scala create mode 100644 examples/prefab/cores/boom/arch/native/NestedPrefixModulesAnnotation.scala create mode 100644 examples/prefab/cores/boom/arch/native/Rocc.scala create mode 100644 examples/prefab/cores/boom/arch/native/Tile.scala rename examples/{components => prefab}/cores/boom/configs/Params.scala (100%) rename examples/{components => prefab}/cores/boom/configs/boom_params.proto (100%) rename examples/{components => prefab}/cores/boom/stack/CMakeLists.txt (89%) rename examples/{components => prefab}/cores/boom/thirdparty/riscv-boom (100%) rename examples/{components => prefab}/cores/boom/thirdparty/rocket-chip (100%) rename examples/{components => prefab}/cores/boom/workloads/ctests/CMakeLists.txt (100%) rename examples/{components => prefab}/cores/boom/workloads/ctests/memory_matrix.c (100%) rename examples/{components => prefab}/cores/reduce/arch/ReduceCore.scala (100%) rename examples/{components => prefab}/cores/reduce/arch/acc.scala (100%) rename examples/{components => prefab}/cores/reduce/arch/build.sc (100%) rename examples/{components => prefab}/cores/reduce/arch/loom.scala (100%) rename examples/{components => prefab}/cores/reduce/emu/Cargo.lock (100%) rename examples/{components => prefab}/cores/reduce/emu/Cargo.toml (100%) rename examples/{components => prefab}/cores/reduce/emu/src/lib.rs (100%) rename examples/{components => prefab}/cores/reduce/stack/CMakeLists.txt (86%) rename examples/{components => prefab}/cores/rocket/LICENSE.Berkeley (100%) rename examples/{components => prefab}/cores/rocket/LICENSE.SiFive (100%) rename examples/{components => prefab}/cores/rocket/arch/Execution.scala (99%) rename examples/{components => prefab}/cores/rocket/arch/Lsu.scala (100%) rename examples/{components => prefab}/cores/rocket/arch/Rocket.scala (100%) rename examples/{components => prefab}/cores/rocket/arch/build.sc (100%) rename examples/{components => prefab}/cores/rocket/arch/pipeline/Config.scala (100%) rename examples/{components => prefab}/cores/rocket/arch/pipeline/Pipeline.scala (99%) rename examples/{components => prefab}/cores/rocket/arch/pipeline/Utility.scala (100%) rename examples/{components => prefab}/cores/rocket/configs/Params.scala (100%) rename examples/{components => prefab}/cores/rocket/configs/rocket_params.proto (100%) rename examples/{components => prefab}/cores/rocket/emu/Cargo.lock (100%) rename examples/{components => prefab}/cores/rocket/emu/Cargo.toml (100%) rename examples/{components => prefab}/cores/rocket/emu/src/lib.rs (100%) rename examples/{components => prefab}/cores/rocket/stack/CMakeLists.txt (89%) rename examples/{components => prefab}/cores/rocket/thirdparty/rocket-chip (100%) rename examples/{components => prefab}/cores/xiangshan/LICENSE (100%) rename examples/{components => prefab}/cores/xiangshan/arch/Chi.scala (100%) rename examples/{components => prefab}/cores/xiangshan/arch/Native.scala (100%) rename examples/{components => prefab}/cores/xiangshan/arch/XiangShan.scala (100%) create mode 100644 examples/prefab/cores/xiangshan/arch/build.sc create mode 100644 examples/prefab/cores/xiangshan/arch/native/Backend.scala create mode 100644 examples/prefab/cores/xiangshan/arch/native/CSR.scala create mode 100644 examples/prefab/cores/xiangshan/arch/native/Core.scala create mode 100644 examples/prefab/cores/xiangshan/arch/native/CtrlBlock.scala create mode 100644 examples/prefab/cores/xiangshan/arch/native/DecodeStage.scala create mode 100644 examples/prefab/cores/xiangshan/arch/native/DecodeUnit.scala create mode 100644 examples/prefab/cores/xiangshan/arch/native/Native.scala create mode 100644 examples/prefab/cores/xiangshan/arch/native/Tile.scala create mode 100644 examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/CSR.scala create mode 100644 examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/FuConfig.scala create mode 100644 examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/FuncUnit.scala create mode 100644 examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/NewCSR/NewCSR.scala rename examples/{components => prefab}/cores/xiangshan/configs/Params.scala (100%) rename examples/{components => prefab}/cores/xiangshan/configs/xiangshan_params.proto (100%) rename examples/{components => prefab}/cores/xiangshan/stack/CMakeLists.txt (89%) rename examples/{components => prefab}/cores/xiangshan/thirdparty/xiangshan (100%) rename examples/{components => prefab}/cores/xiangshan/workloads/ctests/CMakeLists.txt (100%) rename examples/{components => prefab}/cores/xiangshan/workloads/ctests/memory_matrix.c (100%) rename examples/{components => prefab}/threads/attention/arch/AttentionThread.scala (100%) rename examples/{components => prefab}/threads/attention/arch/acc.scala (100%) rename examples/{components => prefab}/threads/attention/configs/balldomains/default.toml (100%) rename examples/{components => prefab}/threads/attention/configs/cpu/default.toml (100%) rename examples/{components => prefab}/threads/attention/configs/default.toml (100%) rename examples/{components => prefab}/threads/attention/configs/frontends/default.toml (74%) rename examples/{components => prefab}/threads/attention/configs/gpdomain/default.toml (100%) rename examples/{components => prefab}/threads/attention/configs/memdomains/default.toml (98%) rename examples/{components => prefab}/threads/attention/stack/CMakeLists.txt (92%) rename examples/{components => prefab}/threads/build.sc (100%) rename examples/{components => prefab}/threads/ffn/arch/FfnThread.scala (100%) rename examples/{components => prefab}/threads/ffn/arch/acc.scala (100%) rename examples/{components => prefab}/threads/ffn/configs/balldomains/default.toml (100%) rename examples/{components => prefab}/threads/ffn/configs/cpu/default.toml (100%) rename examples/{components => prefab}/threads/ffn/configs/default.toml (100%) rename examples/{components/threads/matmul => prefab/threads/ffn}/configs/frontends/default.toml (74%) rename examples/{components => prefab}/threads/ffn/configs/gpdomain/default.toml (100%) rename examples/{components => prefab}/threads/ffn/configs/memdomains/default.toml (98%) rename examples/{components => prefab}/threads/ffn/stack/CMakeLists.txt (92%) rename examples/{components => prefab}/threads/matmul/arch/MatMulThread.scala (100%) rename examples/{components => prefab}/threads/matmul/arch/acc.scala (100%) rename examples/{components => prefab}/threads/matmul/configs/ant.toml (100%) rename examples/{components => prefab}/threads/matmul/configs/balldomains/default.toml (100%) rename examples/{components => prefab}/threads/matmul/configs/cpu/default.toml (100%) rename examples/{components => prefab}/threads/matmul/configs/default.toml (100%) rename examples/{components/threads/ffn => prefab/threads/matmul}/configs/frontends/default.toml (74%) rename examples/{components => prefab}/threads/matmul/configs/gpdomain/default.toml (100%) rename examples/{components => prefab}/threads/matmul/configs/memdomains/default.toml (98%) rename examples/{components => prefab}/threads/matmul/stack/CMakeLists.txt (91%) rename examples/{components => prefab}/threads/matmul/stack/matmul.h (100%) rename examples/{components => prefab}/threads/matmul/stack/prepare.c (98%) rename examples/{components => prefab}/threads/matmul/stack/program.c (100%) rename examples/{components => prefab}/threads/matmul/workloads/ctests/CMakeLists.txt (83%) rename examples/{components => prefab}/threads/matmul/workloads/ctests/rvv/CMakeLists.txt (79%) rename examples/{components => prefab}/threads/matmul/workloads/ctests/rvv/matmul.cpp (100%) rename examples/{components => prefab}/threads/matmul/workloads/ctests/rvv/mxfp8_quant.cpp (100%) rename examples/{components => prefab}/threads/matmul/workloads/ctests/rvv/norm.cpp (97%) rename examples/{components => prefab}/threads/matmul/workloads/ctests/rvv/silu.cpp (100%) rename examples/{components => prefab}/threads/matmul/workloads/ctests/rvv/snake.cpp (100%) rename examples/{components => prefab}/threads/matmul/workloads/ctests/rvv/swiglu.cpp (100%) rename examples/{components => prefab}/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt (100%) rename examples/{components => prefab}/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c (77%) rename examples/{components => prefab}/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp (95%) diff --git a/.agents/skills b/.agents/skills index 9ca889df7..565442ef7 160000 --- a/.agents/skills +++ b/.agents/skills @@ -1 +1 @@ -Subproject commit 9ca889df73af6c01634fe39cb18217ee205d6d3a +Subproject commit 565442ef74e2a0e0ba2802e6f120f07b293c8520 diff --git a/.github/scripts/ci_repo_lock.sh b/.github/scripts/ci_repo_lock.sh index 12ff69c7f..9f265534d 100755 --- a/.github/scripts/ci_repo_lock.sh +++ b/.github/scripts/ci_repo_lock.sh @@ -31,6 +31,9 @@ repo_reset() { git clean -ffd rm -rf arch/out + # Generated outputs (CMake caches, RTL, workloads) are ignored by git clean + # and keep stale source paths; every run starts from a fresh output tree. + rm -rf examples/designs/*/output rm -rf stack/thirdparty/buddy-mlir/llvm/build rm -rf stack/thirdparty/buddy-mlir/build @@ -38,11 +41,11 @@ repo_reset() { git submodule update --init --force # submodules in submodules we used need update by hands - git -C examples/components/cores/rocket/thirdparty/rocket-chip submodule sync -- dependencies/cde - git -C examples/components/cores/rocket/thirdparty/rocket-chip submodule update --init --force -- dependencies/cde + git -C examples/prefab/cores/rocket/thirdparty/rocket-chip submodule sync -- dependencies/cde + git -C examples/prefab/cores/rocket/thirdparty/rocket-chip submodule update --init --force -- dependencies/cde git -C stack submodule sync - git -C stack submodule update --init --force -- engine os/linux thirdparty/buddy-mlir + git -C stack submodule update --init --force -- os/linux thirdparty/buddy-mlir git -C stack submodule update --init --depth 1 --single-branch --recommend-shallow -- \ os/thirdparty/linux os/thirdparty/busybox os/thirdparty/opensbi git -C stack/thirdparty/buddy-mlir submodule sync diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index ea16cd7bf..bf5b551fc 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -82,18 +82,23 @@ jobs: matrix: include: - design: toy + config: ToyVerilatorConfig run_bemu_tests: true run_verilator_tests: true - design: pebble + config: PebbleVerilatorConfig run_bemu_tests: true run_verilator_tests: true - design: goban + config: GobanVerilatorConfig run_bemu_tests: true run_verilator_tests: true - design: poly + config: PolyVerilatorConfig run_bemu_tests: true run_verilator_tests: true - design: multi-rocket + config: MultiRocketVerilatorConfig run_bemu_tests: true run_verilator_tests: true env: @@ -131,9 +136,9 @@ jobs: set -euo pipefail root="${BB_ROOT/#\~/$HOME}" cd "${root}" - nix develop -c bbdev stack --build '--design ${{ matrix.design }} --device' - nix develop -c bbdev workload --clean '--design ${{ matrix.design }} --device' - nix develop -c bbdev workload --build '--design ${{ matrix.design }} --device' + nix develop -c bbdev stack --build '--design ${{ matrix.design }} --device ${{ matrix.config }}' + nix develop -c bbdev workload --clean '--design ${{ matrix.design }} --device ${{ matrix.config }}' + nix develop -c bbdev workload --build '--design ${{ matrix.design }} --device ${{ matrix.config }}' - name: bemu tests if: matrix.run_bemu_tests @@ -142,9 +147,11 @@ jobs: set -euo pipefail root="${BB_ROOT/#\~/$HOME}" cd "${root}" - nix develop -c bbdev bebop-bemu --batch "--design ${{ matrix.design }} --device --test bare-tests --jobs 16 --clean-before --junit ${root}/bebop/target/${{ matrix.design }}/test-reports/ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/bemu-bare.xml" - nix develop -c bbdev kernel --build '--design ${{ matrix.design }} --device' - nix develop -c bbdev bebop-bemu --batch "--design ${{ matrix.design }} --device --test linux-tests --jobs 16 --junit ${root}/bebop/target/${{ matrix.design }}/test-reports/ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/bemu-linux.xml" + # Other design jobs share this checkout; drop only this config's old batch runs. + rm -rf "${root}"/log/*-${{ matrix.design }}-bebop-bemu-device-${{ matrix.config }}-*(N) + nix develop -c bbdev bebop-bemu --batch "--design ${{ matrix.design }} --device ${{ matrix.config }} --test bare-tests --jobs 16" + nix develop -c bbdev kernel --build '--design ${{ matrix.design }} --device ${{ matrix.config }}' + nix develop -c bbdev bebop-bemu --batch "--design ${{ matrix.design }} --device ${{ matrix.config }} --test linux-tests --jobs 16" - name: verilator tests if: matrix.run_verilator_tests @@ -153,46 +160,33 @@ jobs: set -euo pipefail root="${BB_ROOT/#\~/$HOME}" cd "${root}" - nix develop -c bbdev bebop-verilator --clean '--design ${{ matrix.design }} --device' - nix develop -c bbdev bebop-verilator --verilog '--design ${{ matrix.design }} --device --diff' - nix develop -c bbdev bebop-verilator --build '--jobs 100 --design ${{ matrix.design }} --device --diff' - nix develop -c bbdev bebop-verilator --batch "--design ${{ matrix.design }} --device --test bare-tests --diff --jobs 16 --clean-before --junit ${root}/bebop/target/${{ matrix.design }}/test-reports/ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/verilator-bare.xml" - - - name: Detect bebop JUnit files - if: always() && (matrix.run_bemu_tests || matrix.run_verilator_tests) - id: junit - shell: zsh {0} - run: | - root="${BB_ROOT/#\~/$HOME}" - reports="${root}/bebop/target/${{ matrix.design }}/test-reports/ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - echo "directory=${reports}" >> "$GITHUB_OUTPUT" - files=("${reports}/"*.xml(N)) - if (( ${#files} == 0 )); then - echo "found=false" >> "$GITHUB_OUTPUT" - else - echo "found=true" >> "$GITHUB_OUTPUT" - fi + rm -rf "${root}"/log/*-${{ matrix.design }}-bebop-verilator-device-${{ matrix.config }}-*(N) + nix develop -c bbdev bebop-verilator --clean '--design ${{ matrix.design }} --device ${{ matrix.config }}' + nix develop -c bbdev bebop-verilator --verilog '--design ${{ matrix.design }} --device ${{ matrix.config }} --diff' + nix develop -c bbdev bebop-verilator --build '--jobs 100 --design ${{ matrix.design }} --device ${{ matrix.config }} --diff' + nix develop -c bbdev bebop-verilator --batch "--design ${{ matrix.design }} --device ${{ matrix.config }} --test bare-tests --diff --jobs 16" - name: Publish bebop test report - if: always() && (matrix.run_bemu_tests || matrix.run_verilator_tests) && steps.junit.outputs.found == 'true' + if: always() && (matrix.run_bemu_tests || matrix.run_verilator_tests) uses: mikepenz/action-junit-report@v4 with: - report_paths: "${{ steps.junit.outputs.directory }}/*.xml" + report_paths: "${{ env.BB_ROOT }}/log/*-${{ matrix.design }}-bebop-*-device-${{ matrix.config }}-*/junit.xml" check_name: "bebop regression (${{ matrix.design }})" detailed_summary: true include_passed: true - name: Upload bebop JUnit artifacts - if: always() && (matrix.run_bemu_tests || matrix.run_verilator_tests) && steps.junit.outputs.found == 'true' + if: always() && (matrix.run_bemu_tests || matrix.run_verilator_tests) uses: actions/upload-artifact@v4 + # The runner host has no DNS of its own; uploads go through the proxy. env: - ALL_PROXY: "" - HTTP_PROXY: "" - HTTPS_PROXY: "" - NO_PROXY: github.com,api.github.com,uploads.github.com,objects.githubusercontent.com,.blob.core.windows.net + HTTP_PROXY: http://127.0.0.1:7890 + HTTPS_PROXY: http://127.0.0.1:7890 + NO_PROXY: localhost,127.0.0.1,::1 with: name: bebop-junit-${{ matrix.design }} - path: "${{ steps.junit.outputs.directory }}/*.xml" + path: "${{ env.BB_ROOT }}/log/*-${{ matrix.design }}-bebop-*-device-${{ matrix.config }}-*/junit.xml" + if-no-files-found: error - name: Exit repo lock if: always() diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 273585e8b..a4d1aea9e 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -71,18 +71,23 @@ jobs: matrix: include: - design: toy + config: ToyVerilatorConfig run_area: true run_power: false - design: pebble + config: PebbleVerilatorConfig run_area: true run_power: false - design: goban + config: GobanVerilatorConfig run_area: true run_power: false - design: poly + config: PolyTapeoutConfig run_area: true run_power: false - design: multi-rocket + config: MultiRocketVerilatorConfig run_area: true run_power: false env: @@ -111,7 +116,7 @@ jobs: set +u source ~/.zshrc set -u - nix develop -c bbdev dc --area '--design ${{ matrix.design }} --device' + nix develop -c bbdev dc --area '--design ${{ matrix.design }} --device ${{ matrix.config }}' - name: DC power if: matrix.run_power @@ -123,7 +128,7 @@ jobs: set +u source ~/.zshrc set -u - nix develop -c bbdev dc --power '--design ${{ matrix.design }} --device' + nix develop -c bbdev dc --power '--design ${{ matrix.design }} --device ${{ matrix.config }}' - name: Exit repo lock if: always() && (matrix.run_area || matrix.run_power) @@ -143,14 +148,19 @@ jobs: matrix: include: - design: toy + config: ToyVerilatorConfig run_uvm: true - design: pebble + config: PebbleVerilatorConfig run_uvm: true - design: goban + config: GobanVerilatorConfig run_uvm: true - design: poly + config: PolyVerilatorConfig run_uvm: true - design: multi-rocket + config: MultiRocketVerilatorConfig run_uvm: true env: BB_ROOT: ~/Code/buckyball @@ -178,8 +188,8 @@ jobs: set +u source ~/.zshrc set -u - nix develop -c bbdev bebop-verilator --verilog '--design ${{ matrix.design }} --device' - nix develop -c bbdev uvm --run '--design ${{ matrix.design }} --device' + nix develop -c bbdev bebop-verilator --verilog '--design ${{ matrix.design }} --device ${{ matrix.config }}' + nix develop -c bbdev uvm --run '--design ${{ matrix.design }} --device ${{ matrix.config }}' - name: Exit repo lock if: always() && matrix.run_uvm @@ -200,14 +210,19 @@ jobs: matrix: include: - design: toy + config: ToyVerilatorConfig build_bitstream: true - design: pebble + config: PebbleVerilatorConfig build_bitstream: true - design: goban + config: GobanVerilatorConfig build_bitstream: true - design: poly + config: PolyP2EConfig build_bitstream: true - design: multi-rocket + config: MultiRocketVerilatorConfig build_bitstream: true env: BB_ROOT: ~/Code/buckyball @@ -234,8 +249,8 @@ jobs: for build_dir in "${root}"/bebop/build/"${{ matrix.design }}"-*(N); do rm -rf "${build_dir}" done - nix develop -c bbdev bebop-p2e --verilog '--design ${{ matrix.design }} --device --diff' - nix develop -c bbdev bebop-p2e --buildbitstream '--design ${{ matrix.design }} --device --diff' + nix develop -c bbdev bebop-p2e --verilog '--design ${{ matrix.design }} --device ${{ matrix.config }} --diff' + nix develop -c bbdev bebop-p2e --buildbitstream '--design ${{ matrix.design }} --device ${{ matrix.config }} --diff' bits=("${root}"/bebop/build/"${{ matrix.design }}"-*/fpgaCompDir/bitstream.bit(N)) if (( ${#bits} != 1 )); then echo "ERROR: expected exactly 1 bitstream.bit for ${{ matrix.design }}, got ${#bits}" >&2 @@ -262,18 +277,23 @@ jobs: matrix: include: - design: toy + config: ToyVerilatorConfig run_fpga_bare: true run_fpga_linux: true - design: pebble + config: PebbleVerilatorConfig run_fpga_bare: true run_fpga_linux: true - design: goban + config: GobanVerilatorConfig run_fpga_bare: true run_fpga_linux: false - design: poly + config: PolyP2EConfig run_fpga_bare: true run_fpga_linux: false - design: multi-rocket + config: MultiRocketVerilatorConfig run_fpga_bare: true run_fpga_linux: false env: @@ -306,15 +326,15 @@ jobs: fi if [[ "${{ matrix.run_fpga_bare }}" == "true" ]]; then - nix develop -c bbdev stack --build "--design ${{ matrix.design }} --device" - nix develop -c bbdev workload --build "--design ${{ matrix.design }} --device" - nix develop -c bbdev workload --tohex "--design ${{ matrix.design }} --device" - nix develop -c bbdev bebop-p2e --batch "--design ${{ matrix.design }} --device --bitstream ${bits[1]} --test bare-tests --diff" + nix develop -c bbdev stack --build "--design ${{ matrix.design }} --device ${{ matrix.config }}" + nix develop -c bbdev workload --build "--design ${{ matrix.design }} --device ${{ matrix.config }}" + nix develop -c bbdev workload --tohex "--design ${{ matrix.design }} --device ${{ matrix.config }}" + nix develop -c bbdev bebop-p2e --batch "--design ${{ matrix.design }} --device ${{ matrix.config }} --bitstream ${bits[1]} --test bare-tests --diff" fi if [[ "${{ matrix.run_fpga_linux }}" == "true" ]]; then - nix develop -c bbdev kernel --build "--design ${{ matrix.design }} --device" - nix develop -c bbdev bebop-p2e --batch "--design ${{ matrix.design }} --device --bitstream ${bits[1]} --test linux-tests --diff" + nix develop -c bbdev kernel --build "--design ${{ matrix.design }} --device ${{ matrix.config }}" + nix develop -c bbdev bebop-p2e --batch "--design ${{ matrix.design }} --device ${{ matrix.config }} --bitstream ${bits[1]} --test linux-tests --diff" fi - name: Exit repo lock @@ -335,12 +355,16 @@ jobs: matrix: include: - design: pebble + config: PebbleVerilatorConfig model: lenet - design: pebble + config: PebbleVerilatorConfig model: mobilenet # - design: pebble + # config: PebbleVerilatorConfig # model: resnet # - design: pebble + # config: PebbleVerilatorConfig # model: yolo env: BB_ROOT: ~/Code/buckyball @@ -366,9 +390,9 @@ jobs: echo "ERROR: eval-performance requires a real model, got model=none" >&2 exit 1 fi - nix develop -c bbdev stack --build "--design ${{ matrix.design }} --device" - nix develop -c bbdev workload --build "--design ${{ matrix.design }} --device --model ${{ matrix.model }}" - nix develop -c bbdev kernel --build "--design ${{ matrix.design }} --device --model ${{ matrix.model }}" + nix develop -c bbdev stack --build "--design ${{ matrix.design }} --device ${{ matrix.config }}" + nix develop -c bbdev workload --build "--design ${{ matrix.design }} --device ${{ matrix.config }} --model ${{ matrix.model }}" + nix develop -c bbdev kernel --build "--design ${{ matrix.design }} --device ${{ matrix.config }} --model ${{ matrix.model }}" - name: Model performance shell: zsh {0} @@ -382,7 +406,7 @@ jobs: printf '%s\n' "${bits[@]}" >&2 exit 1 fi - nix develop -c bbdev bebop-p2e --runworkload "--design ${{ matrix.design }} --device --bitstream ${bits[1]} --model ${{ matrix.model }}" + nix develop -c bbdev bebop-p2e --runworkload "--design ${{ matrix.design }} --device ${{ matrix.config }} --bitstream ${bits[1]} --model ${{ matrix.model }}" - name: Exit repo lock if: always() diff --git a/.gitmodules b/.gitmodules index 1557a2a7f..ecc807c1c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -53,16 +53,16 @@ [submodule "arch/thirdparty/tech_cells_generic"] path = arch/thirdparty/tech_cells_generic url = https://github.com/pulp-platform/tech_cells_generic.git -[submodule "examples/components/core/rocket/thirdparty/rocket-chip"] - path = examples/components/cores/rocket/thirdparty/rocket-chip +[submodule "examples/prefab/core/rocket/thirdparty/rocket-chip"] + path = examples/prefab/cores/rocket/thirdparty/rocket-chip url = https://github.com/SEU-ACAL/rocket-chip.git -[submodule "examples/components/core/xiangshan/thirdparty/xiangshan"] - path = examples/components/cores/xiangshan/thirdparty/xiangshan +[submodule "examples/prefab/core/xiangshan/thirdparty/xiangshan"] + path = examples/prefab/cores/xiangshan/thirdparty/xiangshan url = https://github.com/OpenXiangShan/XiangShan.git -[submodule "examples/components/cores/boom/thirdparty/riscv-boom"] - path = examples/components/cores/boom/thirdparty/riscv-boom +[submodule "examples/prefab/cores/boom/thirdparty/riscv-boom"] + path = examples/prefab/cores/boom/thirdparty/riscv-boom url = https://github.com/riscv-boom/riscv-boom.git -[submodule "examples/components/cores/boom/thirdparty/rocket-chip"] - path = examples/components/cores/boom/thirdparty/rocket-chip +[submodule "examples/prefab/cores/boom/thirdparty/rocket-chip"] + path = examples/prefab/cores/boom/thirdparty/rocket-chip url = https://github.com/chipsalliance/rocket-chip.git diff --git a/README.md b/README.md index 34f4395e7..b5655f62b 100644 --- a/README.md +++ b/README.md @@ -49,16 +49,17 @@ nix develop **3. Verify Installation** -Run Verilator simulation test to verify installation: +Run Verilator simulation test to verify installation. `--device` names a device config +registered in the design's `configs/sims/default.toml` (`[targets.device.configs]`): ```bash -bbdev verilator --run '--jobs 16 --design toy --device --binary toy-toy-ctest-relu_i32_16x16_test-baremetal --batch' +bbdev verilator --run '--jobs 16 --design toy --device ToyVerilatorConfig --binary toy-toy-ctest-relu_i32_16x16_test-baremetal --batch' ``` **4. Try faster simulation using bebop** ```bash -bbdev bebop-verilator --run '--design toy --device --binary toy-toy-ctest-relu_i32_16x16_test-baremetal --itrace --mtrace --pmctrace --ctrace' +bbdev bebop-verilator --run '--design toy --device ToyVerilatorConfig --binary toy-toy-ctest-relu_i32_16x16_test-baremetal --itrace --mtrace --pmctrace --ctrace' ``` ## Tutorial diff --git a/arch/src/csrc/include/monitor/trace.h b/arch/src/csrc/include/monitor/trace.h index 44409f9fa..af6db349f 100644 --- a/arch/src/csrc/include/monitor/trace.h +++ b/arch/src/csrc/include/monitor/trace.h @@ -51,9 +51,10 @@ void dpi_mem_pmctrace(unsigned char is_store, // 1 = store, 0 = load // DPI-C function for cycle counter trace (ctrace) // Called from TraceBall when counter commands are executed -void dpi_ctrace(unsigned char subcmd, // 0=START, 1=STOP, 2=READ - unsigned int ctr_id, unsigned long long tag, - unsigned long long elapsed, unsigned long long cycle); +void dpi_ctrace(unsigned int subcmd, // 0=START, 1=STOP, 2=READ + unsigned int ctr_id, unsigned int tag_lo, unsigned int tag_hi, + unsigned int elapsed_lo, unsigned int elapsed_hi, + unsigned int cycle_lo, unsigned int cycle_hi); // DPI-C function for bank hash trace (banktrace) // Called when a Ball instruction writes a bank diff --git a/arch/src/csrc/src/monitor/trace/ctrace.cc b/arch/src/csrc/src/monitor/trace/ctrace.cc index f0deac7ae..f0b2be024 100644 --- a/arch/src/csrc/src/monitor/trace/ctrace.cc +++ b/arch/src/csrc/src/monitor/trace/ctrace.cc @@ -9,31 +9,6 @@ extern const char *log_path; static FILE *ctrace_fp = NULL; -// Backdoor write state -static uint32_t bd_write_row_cursor = 0; -static uint64_t bd_write_data_lo = 0; -static uint64_t bd_write_data_hi = 0; - -// Backdoor read state -static uint32_t bd_read_row_cursor = 0; - -// Backdoor test data generator -// Generates a simple incremental pattern for each row. -// 128-bit = 16 bytes, each byte = (row * 16 + byte_offset) & 0xFF -static void generate_test_data(uint32_t row, uint64_t *data_lo, - uint64_t *data_hi) { - uint8_t data[16]; - for (int i = 0; i < 16; i++) { - data[i] = (uint8_t)((row * 16 + i) & 0xFF); - } - *data_lo = 0; - *data_hi = 0; - for (int i = 0; i < 8; i++) { - *data_lo |= ((uint64_t)data[i]) << (i * 8); - *data_hi |= ((uint64_t)data[i + 8]) << (i * 8); - } -} - static void init_ctrace() { if (ctrace_fp == NULL && log_path != NULL) { ctrace_fp = fopen(log_path, "a"); @@ -43,25 +18,22 @@ static void init_ctrace() { } } -static void u128_hex(char *buf, size_t n, unsigned long long hi, - unsigned long long lo) { - int ret = snprintf(buf, n, "0x%016llX%016llX", hi, lo); - if (ret < 0 || (size_t)ret >= n) { - panic("snprintf failed in ctrace u128_hex"); - } -} - // ================================================================ // Cycle counter DPI-C // ================================================================ -extern "C" void dpi_ctrace(unsigned char subcmd, unsigned int ctr_id, - unsigned long long tag, unsigned long long elapsed, - unsigned long long cycle) { +extern "C" void dpi_ctrace(unsigned int subcmd, unsigned int ctr_id, + unsigned int tag_lo, unsigned int tag_hi, + unsigned int elapsed_lo, unsigned int elapsed_hi, + unsigned int cycle_lo, unsigned int cycle_hi) { if (!bdb_trace_on(BDB_TR_CTRACE)) { return; } init_ctrace(); + unsigned long long tag = ((unsigned long long)tag_hi << 32) | tag_lo; + unsigned long long elapsed = + ((unsigned long long)elapsed_hi << 32) | elapsed_lo; + unsigned long long cycle = ((unsigned long long)cycle_hi << 32) | cycle_lo; if (ctrace_fp) { switch (subcmd) { @@ -91,78 +63,6 @@ extern "C" void dpi_ctrace(unsigned char subcmd, unsigned int ctr_id, } } -// ================================================================ -// Backdoor DPI-C functions -// RTL calls these each iteration. C++ auto-increments row. -// bank_id comes from RTL (instruction encoding), not from C++. -// ================================================================ - -// RTL calls this to get the next row to read. -// Returns row in [31:0], upper bits = 0. -extern "C" unsigned long long dpi_backdoor_get_read_addr(void) { - uint32_t row = bd_read_row_cursor; - bd_read_row_cursor++; - return (uint64_t)row; -} - -// RTL calls this to get the next row to write. -// Also pre-generates test data for this row. -// Returns row in [31:0], upper bits = 0. -extern "C" unsigned long long dpi_backdoor_get_write_addr(void) { - uint32_t row = bd_write_row_cursor; - generate_test_data(row, &bd_write_data_lo, &bd_write_data_hi); - bd_write_row_cursor++; - return (uint64_t)row; -} - -// RTL calls this to get write data (128-bit as lo/hi). -// Data was pre-generated by the preceding get_write_addr call. -extern "C" void dpi_backdoor_get_write_data(unsigned long long *data_lo, - unsigned long long *data_hi) { - *data_lo = bd_write_data_lo; - *data_hi = bd_write_data_hi; -} - -// RTL calls this after reading SRAM — logs the data to NDJSON trace file -extern "C" void dpi_backdoor_put_read_data(unsigned int bank_id, - unsigned int row, - unsigned long long data_lo, - unsigned long long data_hi) { - if (!bdb_trace_on(BDB_TR_BANKTRACE)) { - return; - } - init_ctrace(); - if (ctrace_fp) { - char data_hex[35]; - u128_hex(data_hex, sizeof(data_hex), data_hi, data_lo); - fprintf(ctrace_fp, - "{\"type\":\"banktrace\",\"clk\":%llu,\"event\":\"backdoor_read\"," - "\"bank_id\":%u,\"row\":%u,\"data\":\"%s\"}\n", - (unsigned long long)bdb_rtl_clk, bank_id, row, data_hex); - fflush(ctrace_fp); - } -} - -// RTL calls this after writing SRAM — logs the write to NDJSON trace file -extern "C" void dpi_backdoor_put_write_done(unsigned int bank_id, - unsigned int row, - unsigned long long data_lo, - unsigned long long data_hi) { - if (!bdb_trace_on(BDB_TR_BANKTRACE)) { - return; - } - init_ctrace(); - if (ctrace_fp) { - char data_hex[35]; - u128_hex(data_hex, sizeof(data_hex), data_hi, data_lo); - fprintf(ctrace_fp, - "{\"type\":\"banktrace\",\"clk\":%llu,\"event\":\"backdoor_write\"," - "\"bank_id\":%u,\"row\":%u,\"data\":\"%s\"}\n", - (unsigned long long)bdb_rtl_clk, bank_id, row, data_hex); - fflush(ctrace_fp); - } -} - // RTL calls this when a Ball instruction writes a bank: logs the bank hash extern "C" void dpi_btrace(unsigned int inst_id_lo, unsigned int inst_id_hi, unsigned int thread_id_lo, unsigned int thread_id_hi, diff --git a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/configs/BallDomainParam.scala b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/configs/BallDomainParam.scala index 15ac4ee84..6f1717c9e 100644 --- a/arch/src/main/scala/framework/acc/balldomain/src/main/scala/configs/BallDomainParam.scala +++ b/arch/src/main/scala/framework/acc/balldomain/src/main/scala/configs/BallDomainParam.scala @@ -35,14 +35,14 @@ case class BallRouteParam( object BallRouteParam { def fromProto(source: BallInstanceProto): BallRouteParam = BallRouteParam( - source.getBallId, - source.getBallName, + source.getId, + source.getName, source.getInBw, source.getOutBw, source.getMmioReadBw, source.getMmioWriteBw, source.getInstructionsList.asScala.map(i => InstructionParam(i.getMnemonic, i.getFunct7)).toSeq, - source.getBallDir, + source.getComponent, source.getParameters, source.getRtlClass ) diff --git a/arch/src/main/scala/framework/acc/configs/AccParam.scala b/arch/src/main/scala/framework/acc/configs/AccParam.scala index 9b89d3a1c..f48b52430 100644 --- a/arch/src/main/scala/framework/acc/configs/AccParam.scala +++ b/arch/src/main/scala/framework/acc/configs/AccParam.scala @@ -19,16 +19,16 @@ case class AccParam( "virtualBankCount must fit the frontend bank ID width" ) require( - frontend.vbank_id_upper_bound < memDomain.virtualBankCount, + frontend.max_private_vbank < memDomain.virtualBankCount, "Private bank upper bound must be below virtualBankCount" ) if (memDomain.sharedEnable) { require( - frontend.shared_bank_id_base > frontend.vbank_id_upper_bound, + frontend.shared_vbank_base > frontend.max_private_vbank, "Shared bank base must be above the private bank range" ) require( - frontend.shared_bank_id_base <= memDomain.virtualBankCount, + frontend.shared_vbank_base <= memDomain.virtualBankCount, "Shared bank base must not exceed virtualBankCount" ) } diff --git a/arch/src/main/scala/framework/acc/debug/Debug.scala b/arch/src/main/scala/framework/acc/debug/Debug.scala index 0c7092f36..bf415f62d 100644 --- a/arch/src/main/scala/framework/acc/debug/Debug.scala +++ b/arch/src/main/scala/framework/acc/debug/Debug.scala @@ -134,7 +134,7 @@ class BankCheck( sink.io.instId := io.pipeline.commit.bits.instId sink.io.threadId := traceId.U sink.io.ownerThreadId := Mux( - memParam.sharedEnable.B && io.pipeline.commit.bits.bank >= frontendParam.shared_bank_id_base.U, + memParam.sharedEnable.B && io.pipeline.commit.bits.bank >= frontendParam.shared_vbank_base.U, sharedOwnerId.U, traceId.U ) diff --git a/arch/src/main/scala/framework/acc/frontend/configs/default.toml b/arch/src/main/scala/framework/acc/frontend/configs/default.toml index 114197ff6..5e290e70a 100644 --- a/arch/src/main/scala/framework/acc/frontend/configs/default.toml +++ b/arch/src/main/scala/framework/acc/frontend/configs/default.toml @@ -2,8 +2,8 @@ bankAliasEnable = true robEntries = 8 rsOutOfOrderResponse = true bankIdLen = 10 -vbankIdUpperBound = 23 -sharedBankIdBase = 32 +maxPrivateVbank = 23 +sharedVbankBase = 32 iterLen = 34 subRobEnable = false subRobDepth = 64 diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/Frontend.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/Frontend.scala index c6b2e18f0..beac08af1 100644 --- a/arch/src/main/scala/framework/acc/frontend/src/main/scala/Frontend.scala +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/Frontend.scala @@ -14,7 +14,7 @@ import chisel3.util._ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantiate} import framework.acc.frontend.boot.BootRom import framework.acc.frontend.decoder.GlobalDecoder -import framework.acc.memdomain.bank_router.BankMapping +import framework.acc.memdomain.bank_router.BankMappingIO import framework.acc.frontend.scheduler.{Complete, Issue, MemoryIssue, Scheduler} import framework.acc.memdomain.{Response => MemoryResponse} @@ -51,7 +51,7 @@ class Frontend( val ballCompletion = Flipped(Decoupled(new MemoryResponse(frontendParam))) // Bank-version update request to MemDomain; its ready input gates the update. - val mappingUpdate = Decoupled(new BankMapping(frontendParam)) + val mappingUpdate = new BankMappingIO(frontendParam) val memoryRequest = Decoupled(new MemoryIssue(frontendParam, memParam, permissionParam)) val memoryCompletion = Flipped(Decoupled(new MemoryResponse(frontendParam))) diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/configs/uvm.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/configs/uvm.toml index 76af217fb..e57661e5d 100644 --- a/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/configs/uvm.toml +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/boot/configs/uvm.toml @@ -12,8 +12,8 @@ rob_entries = 4 bank_alias_enable = true rs_out_of_order_response = true bank_id_len = 4 -vbank_id_upper_bound = 3 -shared_bank_id_base = 16 +max_private_vbank = 3 +shared_vbank_base = 16 iter_len = 16 sub_rob_enable = false sub_rob_depth = 4 diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/configs/FrontendParam.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/configs/FrontendParam.scala index a13a88996..8590840ea 100644 --- a/arch/src/main/scala/framework/acc/frontend/src/main/scala/configs/FrontendParam.scala +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/configs/FrontendParam.scala @@ -14,9 +14,9 @@ case class FrontendParam( bank_alias_enable: Boolean, rs_out_of_order_response: Boolean, bank_id_len: Int, - vbank_id_upper_bound: Int, + max_private_vbank: Int, /** ISA bank-id namespace base for shared banks: raw id >= this means shared. */ - shared_bank_id_base: Int, + shared_vbank_base: Int, iter_len: Int, sub_rob_enable: Boolean, sub_rob_depth: Int) @@ -32,8 +32,8 @@ object FrontendParam { p.getBankAliasEnable, p.getRsOutOfOrderResponse, p.getBankIdLen, - p.getVbankIdUpperBound, - p.getSharedBankIdBase, + p.getMaxPrivateVbank, + p.getSharedVbankBase, p.getIterLen, p.getSubRobEnable, p.getSubRobDepth diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/GlobalDecoder.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/GlobalDecoder.scala index 3aef67780..7caffcb92 100644 --- a/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/GlobalDecoder.scala +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/decoder/GlobalDecoder.scala @@ -148,9 +148,9 @@ class GlobalDecoder( bankAccess.wr_update := inplaceUpdate def legalBank(raw: UInt): Bool = - raw <= frontendParam.vbank_id_upper_bound.U || + raw <= frontendParam.max_private_vbank.U || (gpParam.enable.B && raw >= memParam.virtualBankCount.U && raw < (memParam.virtualBankCount + 2).U) || - (memParam.sharedEnable.B && raw >= frontendParam.shared_bank_id_base.U && + (memParam.sharedEnable.B && raw >= frontendParam.shared_vbank_base.U && raw < memParam.virtualBankCount.U) val usesArchitecturalBank = is_ball_inst || func7 === MSET_BITPAT || func7 === MVIN_BITPAT || diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Scheduler.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Scheduler.scala index 800ebb789..964a70066 100644 --- a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Scheduler.scala +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/Scheduler.scala @@ -20,7 +20,7 @@ import framework.acc.frontend.scheduler.bat.BankAliasTable import framework.acc.frontend.scheduler.scoreboard.{BankAccessInfo, BankScoreboard} import framework.acc.debug.trace.InstructionEvent import framework.acc.debug.DebugPort -import framework.acc.memdomain.bank_router.BankMapping +import framework.acc.memdomain.bank_router.BankMappingIO import framework.acc.memdomain.{Response => MemoryResponse, MemoryContext} import framework.acc.memdomain.toddr.DmaError @@ -74,7 +74,7 @@ class Scheduler( val gpRequest = Option.when(gpParam.enable)(Decoupled(new Issue(frontendParam, memParam))) val gpCompletion = Option.when(gpParam.enable)(Flipped(Decoupled(new Complete(frontendParam)))) - val mappingUpdate = Decoupled(new BankMapping(frontendParam)) + val mappingUpdate = new BankMappingIO(frontendParam) val barrier = new Bundle { val arrive = Output(Bool()) diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/configs/uvm.toml b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/configs/uvm.toml index 8a3594f8d..82c562bec 100644 --- a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/configs/uvm.toml +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/configs/uvm.toml @@ -8,8 +8,8 @@ rob_entries = 4 bank_alias_enable = true rs_out_of_order_response = true bank_id_len = 4 -vbank_id_upper_bound = 3 -shared_bank_id_base = 16 +max_private_vbank = 3 +shared_vbank_base = 16 iter_len = 16 sub_rob_enable = false sub_rob_depth = 4 diff --git a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/GlobalROB.scala b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/GlobalROB.scala index 88ade204d..beb10f5eb 100644 --- a/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/GlobalROB.scala +++ b/arch/src/main/scala/framework/acc/frontend/src/main/scala/scheduler/src/main/scala/rob/global/src/main/scala/GlobalROB.scala @@ -16,7 +16,7 @@ import framework.acc.frontend.scheduler.scoreboard.BankAccessInfo import framework.acc.memdomain.DISA.MSET_BITPAT import framework.acc.memdomain.DISA.MVIN_KERNEL_BITPAT import framework.acc.gpdomain.decoder.DISA.RUN_KERNEL_BITPAT -import framework.acc.memdomain.bank_router.BankMapping +import framework.acc.memdomain.bank_router.{BankMapping, BankMappingIO} import framework.acc.memdomain.{MemoryContext, PhysicalBankHash} import framework.acc.debug.trace.InstructionEvent import framework.acc.debug.DebugPort @@ -64,8 +64,8 @@ class GlobalROB( val physicalBankCount = memParam.bankNum + sharedHashCount require( - frontendParam.vbank_id_upper_bound < memParam.virtualBankCount, - s"vbank_id_upper_bound(${frontendParam.vbank_id_upper_bound}) must be < virtualBankCount(${memParam.virtualBankCount})" + frontendParam.max_private_vbank < memParam.virtualBankCount, + s"max_private_vbank(${frontendParam.max_private_vbank}) must be < virtualBankCount(${memParam.virtualBankCount})" ) @public @@ -75,7 +75,7 @@ class GlobalROB( Flipped(new DecoupledIO(new DecodedInstruction(frontendParam, memParam, permissionParam))) val issue = new DecoupledIO(new GlobalRobEntry(frontendParam, memParam, permissionParam)) val complete = Flipped(new DecoupledIO(UInt(idWidth.W))) - val mapping = Decoupled(new BankMapping(frontendParam)) + val mapping = new BankMappingIO(frontendParam) val bat = new Bundle { @@ -117,7 +117,9 @@ class GlobalROB( val mappings = Instantiate(new Arbiter(new BankMapping(frontendParam), 2)) mappings.io.in(0) <> mappingRetire mappings.io.in(1) <> mappingIssue - io.mapping <> mappings.io.out + io.mapping.valid := mappings.io.out.valid + io.mapping.bits := mappings.io.out.bits + mappings.io.out.ready := io.mapping.ready // --------------------------------------------------------------------------- // Passive instruction observations consumed by the thread trace collector. @@ -379,7 +381,8 @@ class GlobalROB( // remains one while the extra latency is hidden by in-flight commands. val issuePtrValid = RegInit(false.B) val issuePtr = Reg(UInt(idWidth.W)) - val issuePtrFire = issuePtrValid && issueStageReady + // The BLT is still installing the mapping this instruction issued; hold it. + val issuePtrFire = issuePtrValid && issueStageReady && !io.mapping.busy val issuePtrReady = !issuePtrValid || issuePtrFire mappingIssue.bits.bank := robEntries(actualIssuePtr).cmd.bankAccess.wr_bank_id mappingIssue.bits.vbank := robEntries(actualIssuePtr).renamedBankAccess.wr_bank_id @@ -418,7 +421,7 @@ class GlobalROB( } } when(issueStageReady) { - issueStageValid := issuePtrValid + issueStageValid := issuePtrFire when(issuePtrFire) { issueStageEntry := issuePayload } diff --git a/arch/src/main/scala/framework/acc/memdomain/configs/default.toml b/arch/src/main/scala/framework/acc/memdomain/configs/default.toml index a265701ad..f98b00337 100644 --- a/arch/src/main/scala/framework/acc/memdomain/configs/default.toml +++ b/arch/src/main/scala/framework/acc/memdomain/configs/default.toml @@ -1,3 +1,4 @@ +addrLen = 39 [bank] layoutEnable = true @@ -16,6 +17,3 @@ bankNum = 5 bankEntries = 1024 bankWidth = 8 readWidth = 8 - -[mem] -addrLen = 39 diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemDomain.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemDomain.scala index 4cc467e7b..16253eca3 100644 --- a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemDomain.scala +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/MemDomain.scala @@ -23,7 +23,7 @@ import framework.acc.memdomain.tomem.ToMem import framework.core.memory.MemoryPort import framework.core.memory.access.preflight.Preparation import framework.ip.bus.axi4 -import framework.acc.memdomain.bank_router.{BankMapping, BankRouter} +import framework.acc.memdomain.bank_router.{BankMappingIO, BankRouter} import framework.acc.memdomain.prm.PRM import framework.acc.memdomain.{MvoverISA, MvoverPort} import framework.ip.memory.mesh_shm.MeshLocalBankPort @@ -109,7 +109,7 @@ class MemDomain( // Command Channel val issue = Flipped(Decoupled(new MemoryIssue(frontendParam, memParam, permissionParam))) val complete = Decoupled(new Response(frontendParam)) - val mapping = Flipped(Decoupled(new BankMapping(frontendParam))) + val mapping = Flipped(new BankMappingIO(frontendParam)) // Inside Channel val banks = Vec(memParam.bankRequestPorts, Flipped(new BankPort(frontendParam, memParam))) val ballLayoutRequest = Flipped(Decoupled(new BankLayoutRequest(frontendParam))) diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/configs/uvm.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/configs/uvm.toml index f6d98beb5..df9b1fde4 100644 --- a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/configs/uvm.toml +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/configs/uvm.toml @@ -8,8 +8,8 @@ rob_entries = 4 bank_alias_enable = true rs_out_of_order_response = true bank_id_len = 4 -vbank_id_upper_bound = 3 -shared_bank_id_base = 16 +max_private_vbank = 3 +shared_vbank_base = 16 iter_len = 16 sub_rob_enable = false sub_rob_depth = 4 diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/BankRouter.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/BankRouter.scala index c3e0f40d4..97660fa2e 100644 --- a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/BankRouter.scala +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/BankRouter.scala @@ -28,6 +28,12 @@ class BankMapping(frontendParam: FrontendParam) extends Bundle { val write = Bool() } +// The BLT installs an accepted mapping over several cycles; busy stays high +// until it is installed, and the ROB holds the issuing instruction meanwhile. +class BankMappingIO(frontendParam: FrontendParam) extends DecoupledIO(new BankMapping(frontendParam)) { + val busy = Input(Bool()) +} + @instantiable class BankRouter( val frontendParam: FrontendParam, @@ -54,7 +60,7 @@ class BankRouter( val requests = Vec(memParam.bankRequestPorts, Flipped(new BankPort(frontendParam, memParam))) val directRequest = Vec(gpRequests + 2, Flipped(new BankPort(frontendParam, memParam))) val config = Flipped(Decoupled(new LayoutConfig(frontendParam, memParam))) - val mapping = Flipped(Decoupled(new BankMapping(frontendParam))) + val mapping = Flipped(new BankMappingIO(frontendParam)) val queryVbank = Input(UInt(frontendParam.bank_id_len.W)) val queryShared = Input(Bool()) val queryGroups = Output(UInt(memParam.groupCountWidth.W)) @@ -153,7 +159,7 @@ class BankRouter( val sharedCount = RegNext(sharedGroups(delayedBank), 0.U) val owner = owners(bank) val shared = memParam.sharedEnable.B && - owner >= frontendParam.shared_bank_id_base.U && + owner >= frontendParam.shared_vbank_base.U && owner < memParam.virtualBankCount.U val returnedShared = RegNext(RegNext(shared, false.B), false.B) val enabled = RegNext(RegNext(request.bits.enabled(operand), false.B), false.B) @@ -169,7 +175,7 @@ class BankRouter( val localBusy = WireDefault(false.B) val sharedMapping = - memParam.sharedEnable.B && io.mapping.bits.bank >= frontendParam.shared_bank_id_base.U && + memParam.sharedEnable.B && io.mapping.bits.bank >= frontendParam.shared_vbank_base.U && io.mapping.bits.bank < memParam.virtualBankCount.U && (io.mapping.bits.allocate || io.mapping.bits.retire || io.mapping.bits.write) @@ -179,6 +185,7 @@ class BankRouter( io.shared.mapping.valid := io.mapping.valid && sharedMapping io.shared.mapping.bits := io.mapping.bits io.mapping.ready := Mux(sharedMapping, io.shared.mapping.ready, blt.io.mapping.ready && retireAllowed) + io.mapping.busy := blt.io.mapping.busy when(io.mapping.fire && (io.mapping.bits.allocate || io.mapping.bits.write)) { owners(io.mapping.bits.vbank) := io.mapping.bits.bank } @@ -249,7 +256,7 @@ class BankRouter( when(normal.write.req.fire)(normalWritePending := true.B) when(normal.write.resp.fire)(normalWritePending := false.B) when(local.request.fire) { - assert(local.request.bits.bank <= frontendParam.vbank_id_upper_bound.U) + assert(local.request.bits.bank <= frontendParam.max_private_vbank.U) assert(local.request.bits.addr < memParam.bankEntries.U) assert(!local.request.bits.write || local.request.bits.mask.andR) localPending := true.B @@ -274,7 +281,7 @@ class BankRouter( val memory = privateRequests(i) val owner = owners(request.bank_id) val shared = - memParam.sharedEnable.B && owner >= frontendParam.shared_bank_id_base.U && + memParam.sharedEnable.B && owner >= frontendParam.shared_vbank_base.U && owner < memParam.virtualBankCount.U memory <> request memory.is_shared := false.B diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/BLT.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/BLT.scala index ca0f2d870..897e86e0f 100644 --- a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/BLT.scala +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/bank_router/src/main/scala/blt/BLT.scala @@ -12,7 +12,7 @@ import chisel3.experimental.hierarchy.{instantiable, public, Instance, Instantia import framework.acc.memdomain.{LayoutConfig, LayoutQuery} import framework.acc.memdomain.BankPort import framework.acc.debug.trace.MemoryEvent -import framework.acc.memdomain.bank_router.BankMapping +import framework.acc.memdomain.bank_router.{BankMapping, BankMappingIO} import framework.acc.memdomain.prm.PhysicalBankPort import framework.acc.memdomain.PhysicalBankHash @@ -45,7 +45,7 @@ class BankLayoutTable( val mem_req = Vec(memParam.bankRequestPorts, Flipped(new BankPort(frontendParam, memParam))) val kernel_req = Vec(kernelRequests + 2, Flipped(new BankPort(frontendParam, memParam))) val config = Flipped(Decoupled(new LayoutConfig(frontendParam, memParam))) - val mapping = Flipped(Decoupled(new BankMapping(frontendParam))) + val mapping = Flipped(new BankMappingIO(frontendParam)) // Query interface for frontend to get group count val query_vbank_id = Input(UInt(frontendParam.bank_id_len.W)) @@ -106,7 +106,7 @@ class BankLayoutTable( VecInit(Seq.fill(memParam.bankNum)(0.U.asTypeOf(new MappingTableEntry))) ) - // The frontend contract reserves [0, vbank_id_upper_bound] for private + // The frontend contract reserves [0, max_private_vbank] for private // banks. Keep the direct route table at that architectural size. val privateVbankCount = 2 * (memParam.virtualBankCount + (if (gpParam.enable) 2 @@ -237,6 +237,9 @@ class BankLayoutTable( } io.clearBusy := VecInit(banks.map(_.clearing)).asUInt.orR + // High while a mapping is being installed (see Bank Alloc/Release). + val mappingBusy = RegInit(false.B) + val realloc = io.config.bits.group_id === 0.U val freePbankMask = @@ -274,8 +277,8 @@ class BankLayoutTable( ) } } - io.config.ready := !io.config.bits.alloc || configBankAvailable - when(io.config.valid && io.config.bits.alloc && !configBankAvailable) { + io.config.ready := !mappingBusy && (!io.config.bits.alloc || configBankAvailable) + when(io.config.valid && !mappingBusy && io.config.bits.alloc && !configBankAvailable) { assert(false.B, "BLT allocation failed: no free physical bank\n") } @@ -283,20 +286,43 @@ class BankLayoutTable( // Bank Alloc/Release // ----------------------------------------------------------------------------- - val change = io.mapping.bits + // A mapping is accepted into registers and installed over the following + // cycles, one group per cycle. busy stays high until it is installed; the + // ROB holds the issuing instruction until then, so no consumer ever sees a + // partially installed mapping. + val change = Reg(new BankMapping(frontendParam)) + val nextGroup = RegInit(0.U(log2Up(memParam.bankNum + 1).W)) + io.mapping.busy := mappingBusy + + // An allocation is accepted only when enough banks exist: the unbound ones + // plus, for an in-place rename, the groups of the alias it replaces. + val incoming = io.mapping.bits + val reusable = Mux(incoming.reuse, groupCountByVbank(incoming.previous(privateVbankIdWidth - 1, 0)), 0.U) + val enough = PopCount(mappingTable.map(!_.valid)) +& reusable >= architecturalColumns(incoming.bank) + io.mapping.ready := !mappingBusy && !io.config.valid && (!incoming.allocate || + (if (memParam.bankLayoutEnable) enough else true.B)) + when(io.mapping.fire) { + change := incoming + nextGroup := 0.U + mappingBusy := true.B + } + val columns = architecturalColumns(change.bank) val freeBanks = VecInit(mappingTable.map(entry => !entry.valid || (change.reuse && entry.vbank_id === change.previous))).asUInt - io.mapping.ready := !io.config.valid && (!change.allocate || - (if (memParam.bankLayoutEnable) PopCount(freeBanks) >= columns else true.B)) - when(io.mapping.fire && change.allocate) { + val first = mappingBusy && nextGroup === 0.U + when(mappingBusy) { + nextGroup := nextGroup + 1.U + when(!change.allocate || nextGroup + 1.U >= columns)(mappingBusy := false.B) + } + when(first && change.allocate) { assert( columns =/= 0.U && columns <= memParam.bankNum.U, "Renamed bank requires an allocated layout" ) owners(change.vbank) := change.bank } - when(io.mapping.fire && change.retire) { + when(first && change.retire) { committed(change.bank) := change.vbank singleRouteValid(change.previous) := false.B groupCountByVbank(change.previous) := 0.U @@ -305,47 +331,34 @@ class BankLayoutTable( } } for (bank <- 0 until memParam.bankNum) { - when(io.mapping.fire && change.write && !change.allocate && + when(first && change.write && !change.allocate && mappingTable(bank).valid && mappingTable(bank).vbank_id === change.vbank) { io.physical(bank).bind := true.B } } - val remaining = Wire(Vec( - memParam.bankNum + 1, - UInt(memParam.bankNum.W) - )) - - remaining(0) := freeBanks - for (group <- 0 until memParam.bankNum) { + when(mappingBusy && change.allocate) { val physical = - if (memParam.bankLayoutEnable) PriorityEncoder(remaining(group)) - else (change.bank + group.U)(pbankIndexWidth - 1, 0) - remaining(group + 1) := remaining(group) & ~UIntToOH( - physical, - memParam.bankNum - ) - when(io.mapping.fire && change.allocate && group.U < columns) { - if (memParam.bankLayoutEnable) { - addEntry(change.threadId, change.vbank, physical, columns > 1.U, group.U) - mappingTable(physical).architectural_id := change.bank - } else { - assert( - mappingTable(physical).valid && mappingTable(physical).architectural_id === change.bank, - "Fixed bank alias requires its allocated physical slot" - ) - mappingTable(physical).vbank_id := change.vbank - } - - for (bank <- 0 until memParam.bankNum) { - when(physical === bank.U)(io.physical(bank).bind := true.B) - } - - when(group.U === 0.U) { - singleRoutePbank(change.vbank) := physical - singleRouteValid(change.vbank) := columns === 1.U - groupCountByVbank(change.vbank) := columns - } + if (memParam.bankLayoutEnable) PriorityEncoder(freeBanks) + else (change.bank + nextGroup)(pbankIndexWidth - 1, 0) + if (memParam.bankLayoutEnable) { + assert(freeBanks.orR, "BLT mapping found no free physical bank") + addEntry(change.threadId, change.vbank, physical, columns > 1.U, nextGroup) + mappingTable(physical).architectural_id := change.bank + } else { + assert( + mappingTable(physical).valid && mappingTable(physical).architectural_id === change.bank, + "Fixed bank alias requires its allocated physical slot" + ) + mappingTable(physical).vbank_id := change.vbank + } + for (bank <- 0 until memParam.bankNum) { + when(physical === bank.U)(io.physical(bank).bind := true.B) + } + when(nextGroup === 0.U) { + singleRoutePbank(change.vbank) := physical + singleRouteValid(change.vbank) := columns === 1.U + groupCountByVbank(change.vbank) := columns } } diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/configs/MemDomainParam.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/configs/MemDomainParam.scala index 511f834df..993f5883c 100644 --- a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/configs/MemDomainParam.scala +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/configs/MemDomainParam.scala @@ -52,7 +52,7 @@ object MemDomainParam { virtualBanks: Int, participants: Int ): MemDomainParam = { - require(p.hasBank && p.hasDma && p.hasMmio && p.hasMem, "Memory domain configuration is incomplete") + require(p.hasBank && p.hasDma && p.hasMmio, "Memory domain configuration is incomplete") if (shared.getEnable) require(participants > 0 && shared.getInputChannels % participants == 0) val bank = p.getBank require(bank.hasLayoutEnable, "Bank layout policy is required") @@ -77,7 +77,7 @@ object MemDomainParam { shared.getDefaultGroupCount, bank.getRequestPorts, p.getDma.getBusWidth, - p.getMem.getAddrLen, + p.getAddrLen, mmio.getEnable, mmio.getBankNum, mmio.getBankEntries, diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/Decoder.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/Decoder.scala index 94e6f5aa6..f0dce0afe 100644 --- a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/Decoder.scala +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/ctrl/decoder/Decoder.scala @@ -46,15 +46,15 @@ class Decoder(val frontendParam: FrontendParam, val memParam: MemDomainParam) ex val commandDataBits = frontendParam.commandDataBits import DecodeConstants._ - val sharedBankIdBase = frontendParam.shared_bank_id_base - require(sharedBankIdBase > 0, s"sharedBankIdBase($sharedBankIdBase) must be > 0") + val sharedVbankBase = frontendParam.shared_vbank_base + require(sharedVbankBase > 0, s"sharedVbankBase($sharedVbankBase) must be > 0") require( - sharedBankIdBase < (1 << frontendParam.bank_id_len), - s"sharedBankIdBase($sharedBankIdBase) must fit in bankIdLen(${frontendParam.bank_id_len})" + sharedVbankBase < (1 << frontendParam.bank_id_len), + s"sharedVbankBase($sharedVbankBase) must fit in bankIdLen(${frontendParam.bank_id_len})" ) require( - sharedBankIdBase > frontendParam.vbank_id_upper_bound, - s"sharedBankIdBase($sharedBankIdBase) must be > vbankIdUpperBound(${frontendParam.vbank_id_upper_bound})" + sharedVbankBase > frontendParam.max_private_vbank, + s"sharedVbankBase($sharedVbankBase) must be > maxPrivateVbank(${frontendParam.max_private_vbank})" ) @public @@ -117,7 +117,7 @@ class Decoder(val frontendParam: FrontendParam, val memParam: MemDomainParam) ex val config_bank = Mux(func7 === MSET_BITPAT && !transfer, io.request.bits.bankAccess.wr_bank_id, architectural_bank) val shared_bank_id = - memParam.sharedEnable.B && config_bank >= sharedBankIdBase.U + memParam.sharedEnable.B && config_bank >= sharedVbankBase.U // format: off // BB_BANK0 is encoded in rs1[9:0]. Keep the configured architectural ID; // private and shared banks occupy disjoint ranges in that namespace. @@ -139,7 +139,7 @@ class Decoder(val frontendParam: FrontendParam, val memParam: MemDomainParam) ex assert(rs1(63, 30) === 0.U && rs1(19, 10) === 0.U, "MSET transfer reserves rs1 outside BANK0/BANK2") assert(rs2 === (1.U << 12), "MSET transfer reserves other rs2 bits") assert(rs1(9, 0) =/= rs1(29, 20), "MSET transfer requires distinct source and target") - assert((rs1(9, 0) >= sharedBankIdBase.U) === (rs1(29, 20) >= sharedBankIdBase.U), "MSET transfer requires one bank namespace") + assert((rs1(9, 0) >= sharedVbankBase.U) === (rs1(29, 20) >= sharedVbankBase.U), "MSET transfer requires one bank namespace") }.otherwise { assert(rs1(63, 10) === 0.U, "MSET reserves rs1[63:10]") assert(rs2(63, 12) === 0.U, "MSET reserves rs2[63:12]") diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/uvm.toml b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/uvm.toml index fee3dc162..1fd3322af 100644 --- a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/uvm.toml +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/configs/uvm.toml @@ -7,8 +7,8 @@ rob_entries = 4 bank_alias_enable = true rs_out_of_order_response = true bank_id_len = 4 -vbank_id_upper_bound = 3 -shared_bank_id_base = 16 +max_private_vbank = 3 +shared_vbank_base = 16 iter_len = 16 sub_rob_enable = false sub_rob_depth = 4 diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ReadDma.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ReadDma.scala index e759006db..2ec930674 100644 --- a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ReadDma.scala +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/ReadDma.scala @@ -36,24 +36,25 @@ class ReadDma(val b: ModuleParam) extends Module { }) - val idle :: shape :: divide :: products :: offset :: check :: plan :: query :: offer :: receive :: respond :: drain :: failed :: Nil = - Enum(13) - val state = RegInit(idle) - val command = Reg(new ReadTransfer) - val parent = Reg(UInt(prepared.idBits.W)) - val total = Reg(UInt(17.W)) - val progress = RegInit(0.U(17.W)) - val group = RegInit(0.U(6.W)) - val column = RegInit(0.U(4.W)) - val address = Reg(UInt(64.W)) - val queryVA = Reg(UInt(64.W)) - val physical = Reg(UInt(axiParams.addressBits.W)) - val candidate = Reg(UInt(9.W)) - val remaining = Reg(UInt(9.W)) - val second = RegInit(false.B) - val firstPart = Reg(UInt(128.W)) - val result = Reg(UInt(128.W)) - val fault = RegInit(0.U.asTypeOf(new DmaStatus)) + val (idle :: shape :: divide :: products :: offset :: check :: plan :: query :: lookup :: answer :: validate :: + offer :: receive :: respond :: drain :: failed :: Nil) = Enum(16) + val state = RegInit(idle) + val command = Reg(new ReadTransfer) + val parent = Reg(UInt(prepared.idBits.W)) + val total = Reg(UInt(17.W)) + val progress = RegInit(0.U(17.W)) + val group = RegInit(0.U(6.W)) + val column = RegInit(0.U(4.W)) + val address = Reg(UInt(64.W)) + val queryVA = Reg(UInt(64.W)) + val physical = Reg(UInt(axiParams.addressBits.W)) + val candidate = Reg(UInt(9.W)) + val remaining = Reg(UInt(9.W)) + val second = RegInit(false.B) + val firstPart = Reg(UInt(128.W)) + val result = Reg(UInt(128.W)) + val fault = RegInit(0.U.asTypeOf(new DmaStatus)) + val mapped = Reg(new MapResult(b.mapping)) io.req.ready := state === idle && io.decisionValid io.busy := state =/= idle @@ -63,7 +64,8 @@ class ReadDma(val b: ModuleParam) extends Module { io.resp.bits.addrcounter := progress(15, 0) io.resp.bits.fault := Mux(state === failed, fault, 0.U.asTypeOf(new DmaStatus)) io.query := 0.U.asTypeOf(new MapQuery(b.mapping)) - io.query.valid := state === query + // The prepared map answers a query held for three cycles. + io.query.valid := state === query || state === lookup || state === answer io.query.id := parent io.query.va := queryVA io.query.bytes := candidate << 4 @@ -163,9 +165,12 @@ class ReadDma(val b: ModuleParam) extends Module { candidate := Mux(address(3, 0) =/= 0.U, 1.U, Mux(limited < pageBeats, limited, pageBeats)) state := query } - when(state === query) { - when(!io.mapping.hit || io.mapping.error =/= Error.Ok.U) { - val error = MuxLookup(io.mapping.error, DmaError.AccessFault.U)(Seq( + when(state === query)(state := lookup) + when(state === lookup)(state := answer) + when(state === answer) { mapped := io.mapping; state := validate } + when(state === validate) { + when(!mapped.hit || mapped.error =/= Error.Ok.U) { + val error = MuxLookup(mapped.error, DmaError.AccessFault.U)(Seq( Error.Shape.U -> DmaError.Shape.U, Error.Overflow.U -> DmaError.Shape.U, Error.PageFault.U -> DmaError.PageFault.U, @@ -173,9 +178,9 @@ class ReadDma(val b: ModuleParam) extends Module { Error.Context.U -> DmaError.Context.U )) fail(error, queryVA) - }.elsewhen(io.mapping.pa(3, 0) =/= 0.U || (io.mapping.pa(11, 0) +& (candidate << 4)) > 4096.U) { + }.elsewhen(mapped.pa(3, 0) =/= 0.U || (mapped.pa(11, 0) +& (candidate << 4)) > 4096.U) { fail(DmaError.AccessFault.U, queryVA) - }.otherwise { physical := io.mapping.pa; state := offer } + }.otherwise { physical := mapped.pa; state := offer } } when(io.axi.ar.fire) { remaining := candidate; state := receive } when(io.axi.r.fire) { diff --git a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/WriteDma.scala b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/WriteDma.scala index 9af1a150d..ca9cd3017 100644 --- a/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/WriteDma.scala +++ b/arch/src/main/scala/framework/acc/memdomain/src/main/scala/tomem/src/main/scala/toddr/src/main/scala/WriteDma.scala @@ -39,36 +39,37 @@ class WriteDma(val b: ModuleParam, permission: PmpParams) extends Module { }) - val idle :: plan :: query :: stream :: waitB :: drain :: respond :: Nil = Enum(7) - val state = RegInit(idle) - val command = Reg(new WriteTransfer(b, permission)) - val parent = Reg(UInt(prepared.idBits.W)) - val endVA = Reg(UInt(64.W)) - val cursorVA = Reg(UInt(64.W)) - val inputTotal = Reg(UInt(29.W)) - val consumed = RegInit(0.U(29.W)) - val sourceEnded = RegInit(false.B) - val remaining = Reg(UInt(29.W)) - val firstBeat = RegInit(true.B) - val beats = Reg(UInt(9.W)) - val beatIndex = RegInit(0.U(9.W)) - val awSent = RegInit(false.B) - val wDone = RegInit(false.B) - val queryVA = Reg(UInt(64.W)) - val queryBytes = Reg(UInt(13.W)) - val headOffset = Reg(UInt(4.W)) - val physical = Reg(UInt(axiParams.addressBits.W)) - val carry = RegInit(0.U(128.W)) - val outputValid = RegInit(false.B) - val outputData = Reg(UInt(128.W)) - val outputMask = Reg(UInt(16.W)) - val outputLast = Reg(Bool()) - val fault = RegInit(0.U.asTypeOf(new DmaStatus)) - val offset = command.vaddr(3, 0) - val expectedLast = consumed +& 1.U === inputTotal - val outputRoom = !outputValid || (io.axi.w.ready && !outputLast) - val outputIndex = beatIndex + io.axi.w.fire.asUInt - val outputHead = firstBeat && !io.axi.w.fire + val idle :: plan :: query :: lookup :: answer :: validate :: stream :: waitB :: drain :: respond :: Nil = Enum(10) + val state = RegInit(idle) + val command = Reg(new WriteTransfer(b, permission)) + val parent = Reg(UInt(prepared.idBits.W)) + val endVA = Reg(UInt(64.W)) + val cursorVA = Reg(UInt(64.W)) + val inputTotal = Reg(UInt(29.W)) + val consumed = RegInit(0.U(29.W)) + val sourceEnded = RegInit(false.B) + val remaining = Reg(UInt(29.W)) + val firstBeat = RegInit(true.B) + val beats = Reg(UInt(9.W)) + val beatIndex = RegInit(0.U(9.W)) + val awSent = RegInit(false.B) + val wDone = RegInit(false.B) + val queryVA = Reg(UInt(64.W)) + val queryBytes = Reg(UInt(13.W)) + val headOffset = Reg(UInt(4.W)) + val physical = Reg(UInt(axiParams.addressBits.W)) + val carry = RegInit(0.U(128.W)) + val outputValid = RegInit(false.B) + val outputData = Reg(UInt(128.W)) + val outputMask = Reg(UInt(16.W)) + val outputLast = Reg(Bool()) + val fault = RegInit(0.U.asTypeOf(new DmaStatus)) + val mapped = Reg(new MapResult(b.mapping)) + val offset = command.vaddr(3, 0) + val expectedLast = consumed +& 1.U === inputTotal + val outputRoom = !outputValid || (io.axi.w.ready && !outputLast) + val outputIndex = beatIndex + io.axi.w.fire.asUInt + val outputHead = firstBeat && !io.axi.w.fire io.req.ready := state === idle && io.decisionValid io.busy := state =/= idle @@ -78,7 +79,8 @@ class WriteDma(val b: ModuleParam, permission: PmpParams) extends Module { io.resp.bits.done := fault.error === DmaError.None.U io.resp.bits.fault := fault io.query := 0.U.asTypeOf(new MapQuery(b.mapping)) - io.query.valid := state === query + // The prepared map answers a query held for three cycles. + io.query.valid := state === query || state === lookup || state === answer io.query.id := parent io.query.va := queryVA io.query.bytes := queryBytes @@ -135,15 +137,18 @@ class WriteDma(val b: ModuleParam, permission: PmpParams) extends Module { queryBytes := Mux(commandBytes < covered, commandBytes, covered) state := query } - when(state === query) { - val base = io.mapping.pa.pad(axiParams.addressBits + 1) - headOffset + when(state === query)(state := lookup) + when(state === lookup)(state := answer) + when(state === answer) { mapped := io.mapping; state := validate } + when(state === validate) { + val base = mapped.pa.pad(axiParams.addressBits + 1) - headOffset val lastPA = base +& ((beats << 4) - 1.U) - val invalid = !io.mapping.hit || io.mapping.error =/= Error.Ok.U || + val invalid = !mapped.hit || mapped.error =/= Error.Ok.U || base(axiParams.addressBits) || base(3, 0) =/= 0.U || lastPA(lastPA.getWidth - 1, axiParams.addressBits).orR || (base >> 12) =/= (lastPA >> 12) when(invalid) { - val error = MuxLookup(io.mapping.error, DmaError.AccessFault.U)(Seq( + val error = MuxLookup(mapped.error, DmaError.AccessFault.U)(Seq( Error.Shape.U -> DmaError.Shape.U, Error.Overflow.U -> DmaError.Shape.U, Error.PageFault.U -> DmaError.PageFault.U, @@ -153,7 +158,7 @@ class WriteDma(val b: ModuleParam, permission: PmpParams) extends Module { terminate(error, queryVA) }.otherwise { physical := base(axiParams.addressBits - 1, 0); state := stream } } - when(io.axi.aw.fire)(awSent := true.B) + when(io.axi.aw.fire)(awSent := true.B) // An accepted old W can be replaced immediately; refills take priority over clearing valid. when(io.axi.w.fire)(outputValid := false.B) diff --git a/arch/src/main/scala/framework/core/execution/pipeline/Sequencer.scala b/arch/src/main/scala/framework/core/execution/pipeline/Sequencer.scala index 305f15853..9f8b41223 100644 --- a/arch/src/main/scala/framework/core/execution/pipeline/Sequencer.scala +++ b/arch/src/main/scala/framework/core/execution/pipeline/Sequencer.scala @@ -420,15 +420,18 @@ class Sequencer(b: ExecutionParam, localRocc: Boolean) extends Module { host.complete.ready := true.B when(host.complete.fire)(commandsLive(host.complete.bits.tag) := false.B) } - // PMP replies retain the accepted decision through backpressure. + // Privileged answers an authorization one cycle after it is presented; the reply then retains + // the decision through backpressure. for ((virtual, index) <- Seq(io.fetch, io.memory).zipWithIndex) { - val pending = RegInit(false.B) - val allowed = Reg(Bool()) + val deciding = RegInit(false.B) + val pending = RegInit(false.B) + val allowed = Reg(Bool()) io.privileged.authorization(index) := virtual.authorizationRequest.bits - virtual.authorizationRequest.ready := !pending + virtual.authorizationRequest.ready := !deciding && !pending virtual.authorizationResponse.valid := pending virtual.authorizationResponse.bits := allowed - when(virtual.authorizationRequest.fire) { pending := true.B; allowed := io.privileged.permitted(index) } + when(virtual.authorizationRequest.fire)(deciding := true.B) + when(deciding) { deciding := false.B; pending := true.B; allowed := io.privileged.permitted(index) } when(virtual.authorizationResponse.fire)(pending := false.B) } diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/model.svh b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/model.svh index 69874dbc9..e24bf5331 100644 --- a/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/model.svh +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/resources/env/model.svh @@ -3,6 +3,9 @@ class reference_model extends uvm_subscriber #(cycle_item); chandle reference; int pmp_offsets[16][8] = `PRIV_PMP_OFFSETS; uvm_analysis_port #(result_item) expected_ap, actual_ap; + // permitted answers the authorization of the previous cycle. + bit [1:0] permitted; + bit primed; covergroup behavior with function sample ( int address, int operation, bit write, int privilege, bit trap, bit returning, bit interrupt ); @@ -34,6 +37,7 @@ class reference_model extends uvm_subscriber #(cycle_item); longint unsigned interrupts, cfg; if (t.reset) begin privileged_reset(reference); + primed = 0; return; end expected = result_item::type_id::create("expected"); @@ -85,7 +89,11 @@ class reference_model extends uvm_subscriber #(cycle_item); expected.payload[`PRIV_OUT_TVM_OFFSET+:`PRIV_OUT_TVM_WIDTH] = privileged_value(reference, 15); expected.payload[`PRIV_OUT_TW_OFFSET+:`PRIV_OUT_TW_WIDTH] = privileged_value(reference, 16); expected.payload[`PRIV_OUT_TSR_OFFSET+:`PRIV_OUT_TSR_WIDTH] = privileged_value(reference, 17); - expected.payload[`PRIV_OUT_PERMITTED_0_OFFSET +: `PRIV_OUT_PERMITTED_0_WIDTH] = privileged_permitted( + expected.payload[`PRIV_OUT_PERMITTED_0_OFFSET+:`PRIV_OUT_PERMITTED_0_WIDTH] = + primed ? permitted[0] : actual.payload[`PRIV_OUT_PERMITTED_0_OFFSET+:`PRIV_OUT_PERMITTED_0_WIDTH]; + expected.payload[`PRIV_OUT_PERMITTED_1_OFFSET+:`PRIV_OUT_PERMITTED_1_WIDTH] = + primed ? permitted[1] : actual.payload[`PRIV_OUT_PERMITTED_1_OFFSET+:`PRIV_OUT_PERMITTED_1_WIDTH]; + permitted[0] = privileged_permitted( reference, `PIN(t.inputs, AUTHORIZATION_0_PADDR), `PIN(t.inputs, AUTHORIZATION_0_SIZE), @@ -94,7 +102,7 @@ class reference_model extends uvm_subscriber #(cycle_item); `PIN(t.inputs, AUTHORIZATION_0_EXECUTE), `PIN(t.inputs, AUTHORIZATION_0_PRIVILEGE) ); - expected.payload[`PRIV_OUT_PERMITTED_1_OFFSET +: `PRIV_OUT_PERMITTED_1_WIDTH] = privileged_permitted( + permitted[1] = privileged_permitted( reference, `PIN(t.inputs, AUTHORIZATION_1_PADDR), `PIN(t.inputs, AUTHORIZATION_1_SIZE), @@ -103,6 +111,7 @@ class reference_model extends uvm_subscriber #(cycle_item); `PIN(t.inputs, AUTHORIZATION_1_EXECUTE), `PIN(t.inputs, AUTHORIZATION_1_PRIVILEGE) ); + primed = 1; for (int index = 0; index < 16; index++) begin cfg = privileged_pmp(reference, index, 0); expected.payload[pmp_offsets[index][0]] = cfg[7]; diff --git a/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Privileged.scala b/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Privileged.scala index 6e27c0126..c3a185161 100644 --- a/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Privileged.scala +++ b/arch/src/main/scala/framework/core/execution/privileged/src/main/scala/Privileged.scala @@ -280,31 +280,49 @@ class Privileged(b: ModuleParam) extends Module { } } - for (i <- 0 until 16) { + for (i <- 0 until 16) { io.pmp(i).cfg := pmpConfig(i).asTypeOf(io.pmp(i).cfg) io.pmp(i).addr := pmpAddress(i) io.pmp(i).mask := ((pmpAddress(i) ^ (pmpAddress(i) + 1.U)) << 2) | 3.U } + // PMP bounds and the authorization are registered: permitted answers the authorization of the + // previous cycle against the PMP state of that same cycle. Bounds are inclusive so the + // decision path only compares. + val bounds = Reg(Vec(16, new PmpBound)) + val requests = RegNext(io.authorization) + // Last byte of each authorization: paddr + 2^size - 1. + val lasts = RegNext(VecInit(io.authorization.map(a => a.paddr +& ((1.U(8.W) << a.size) - 1.U)))) + for (i <- 0 until 16) { + val encoded = (pmpAddress(i) << 2).pad(64) + val mask = io.pmp(i).mask.pad(64) + val mode = pmpConfig(i)(4, 3) + val below = if (i == 0) 0.U(64.W) else (pmpAddress(i - 1) << 2).pad(64) + bounds(i).low := Mux(mode === 1.U, below, Mux(mode === 3.U, encoded & ~mask, encoded)) + bounds(i).last := MuxLookup(mode, encoded | 3.U)(Seq( + 1.U -> ((pmpAddress(i) - 1.U) << 2 | 3.U).pad(64), + 3.U -> (encoded | mask) + )) + bounds(i).enabled := mode =/= 0.U && (mode =/= 1.U || below < encoded) + bounds(i).config := pmpConfig(i) + } for (port <- 0 until 2) { - val request = io.authorization(port) - val end = request.paddr +& (1.U(65.W) << request.size) - val matches = (0 until 16).map { i => - val encoded = (pmpAddress(i) << 2).pad(64) - val mask = io.pmp(i).mask.pad(64) - val mode = pmpConfig(i)(4, 3) - val low = Mux( - mode === 1.U, - if (i == 0) 0.U(64.W) else (pmpAddress(i - 1) << 2).pad(64), - Mux(mode === 3.U, encoded & ~mask, encoded) - ) - val high = Mux(mode === 1.U, encoded.pad(65), Mux(mode === 3.U, low +& mask + 1.U, encoded +& 4.U)) - val hit = mode =/= 0.U && low < high && request.paddr < high && end > low - val allowed = (request.privilege === 3.U && !pmpConfig(i)(7)) || - ((!request.read || pmpConfig(i)(0)) && (!request.write || pmpConfig(i)(1)) && (!request.execute || pmpConfig(i)( - 2 - ))) - hit -> (allowed && request.paddr >= low && end <= high) + val request = requests(port) + val last = lasts(port) + val matches = bounds.map { bound => + val hit = bound.enabled && request.paddr <= bound.last && last >= bound.low + val allowed = (request.privilege === 3.U && !bound.config(7)) || + ((!request.read || bound.config(0)) && (!request.write || bound.config(1)) && + (!request.execute || bound.config(2))) + hit -> (allowed && request.paddr >= bound.low && last <= bound.last) } io.permitted(port) := PriorityMux(matches :+ (true.B -> (request.privilege === 3.U))) } } + +/** One PMP entry as an inclusive byte range [low, last]. */ +class PmpBound extends Bundle { + val low = UInt(64.W) + val last = UInt(64.W) + val enabled = Bool() + val config = UInt(8.W) +} diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/model.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/model.svh index 36d8309d5..563aab945 100644 --- a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/model.svh +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/preparation/env/model.svh @@ -9,6 +9,8 @@ class model extends uvm_subscriber #(cycle_item); int lengths[int][int]; bit direction[int], notified[int], live[int]; longint query_key; + // A lookup answers in the third cycle of a held query; recent queries and expectations per port. + logic [255:0] recent[2][$]; covergroup behavior with function sample ( int error, bit writing, int mode, int policy, bit hit, bit released ); @@ -115,6 +117,20 @@ class model extends uvm_subscriber #(cycle_item); expected_pa = physical[id][i] + va - addresses[id][i]; end expected_error = !valid || expected_hit ? 0 : (!shape ? 1 : 4); + recent[port].push_back({ + 32'(valid), + 32'(id), + va, + 32'(bytes), + 1'(writing), + 1'(expected_hit), + expected_pa, + 3'(expected_error) + }); + if (recent[port].size() > 3) void'(recent[port].pop_front()); + if (recent[port].size() < 3 || recent[port][0] !== recent[port][1] || + recent[port][1] !== recent[port][2]) + return; verify(hit == expected_hit && pa == expected_pa && error == expected_error, $sformatf( "Query port%0d id%0d VA%h expected hit%0d PA%h error%0d got hit%0d PA%h error%0d", port, @@ -133,6 +149,8 @@ class model extends uvm_subscriber #(cycle_item); if (t.reset) begin cancelled.write(null); preflight_ref_reset(reference); + recent[0].delete(); + recent[1].delete(); index.delete(); errors.delete(); counts.delete(); diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/agent/service.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/agent/service.svh index 2f6400362..30309a5b2 100644 --- a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/agent/service.svh +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/agent/service.svh @@ -3,11 +3,13 @@ task monitor(); @(ctl.sample); if (ctl.sample.reset) begin pmap_ref_reset(model); + recent[0].delete(); + recent[1].delete(); notified.delete(); status.delete(); end else begin - query_check(ctl.sample.query_0, ctl.sample.result_0); - query_check(ctl.sample.query_1, ctl.sample.result_1); + query_check(0, ctl.sample.query_0, ctl.sample.result_0); + query_check(1, ctl.sample.query_1, ctl.sample.result_1); if (ctl.sample.query_0[`PF(QUERY, VALID)] && ctl.sample.query_1[`PF(QUERY, VALID)]) dual++; if (mapped.sample.valid) begin int error, id; diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/env/env.svh b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/env/env.svh index 14ab56009..e5c713cc9 100644 --- a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/env/env.svh +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/resources/prepared_map/env/env.svh @@ -6,6 +6,8 @@ class env extends uvm_env; virtual stream_if #(`PF_MAP_READY_WIDTH) mapped; virtual stream_if #(`PF_RELEASE_WIDTH) retire; chandle model; + // A lookup answers in the third cycle of a held query; recent queries and expectations per port. + logic [`PF_QUERY_WIDTH+`PF_RESULT_WIDTH-1:0] recent[2][$]; bit notified[int]; int status[int]; int @@ -48,13 +50,23 @@ class env extends uvm_env; this, "", "retire", retire), "retire missing"); model = pmap_ref_create(`PF_ADDRESS_BITS); endfunction - function void query_check(logic [`PF_QUERY_WIDTH-1:0] q, logic [`PF_RESULT_WIDTH-1:0] actual); + function void query_check(int port, logic [`PF_QUERY_WIDTH-1:0] q, + logic [`PF_RESULT_WIDTH-1:0] actual); int hit, error; u64 pa; + logic [`PF_RESULT_WIDTH-1:0] expected = '0; pmap_ref_query(model, q[`PF(QUERY, VALID)], q[`PF(QUERY, ID)], q[`PF(QUERY, VA)], q[ `PF(QUERY, BYTES)], q[`PF(QUERY, WRITE)], retire.sample.valid && retire.sample.ready, retire.sample.bits[`PF(RELEASE, ID)], hit, pa, error); + expected[`PF(RESULT, HIT)] = 1'(hit); + expected[`PF(RESULT, PA)] = `PF_ADDRESS_BITS'(pa); + expected[`PF(RESULT, ERROR)] = 3'(error); + recent[port].push_back({q, expected}); + if (recent[port].size() > 3) void'(recent[port].pop_front()); + if (recent[port].size() < 3 || recent[port][0] !== recent[port][1] || + recent[port][1] !== recent[port][2]) + return; verify_contract(actual[`PF(RESULT, HIT)] === 1'(hit) && actual[`PF(RESULT, PA) ] === `PF_ADDRESS_BITS'(pa) && actual[`PF(RESULT, ERROR)] === 3'(error), $sformatf( diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preflight.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preflight.scala index d266db412..87b60d5fc 100644 --- a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preflight.scala +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/Preflight.scala @@ -55,24 +55,33 @@ class Preflight(b: ModuleParam) extends Module { ) } - val idle :: checkShape :: span :: sendSpan :: segment :: walk :: waitWalk :: authorize :: waitPermission :: save :: Nil = - Enum(10) - val state = RegInit(idle) - val worker = Reg(UInt(contextBits.W)) - val cursor = RegInit(0.U(contextBits.W)) - val row = Reg(UInt(32.W)); val column = Reg(UInt(32.W)) - val dense = Reg(Bool()); val wholeBytes = Reg(UInt(32.W)) - val spanAddress = Reg(UInt(64.W)); val spanBytes = Reg(UInt(32.W)) - val spanLast = Reg(Bool()) - val currentVA = Reg(UInt(64.W)); val currentPA = Reg(UInt(p.bus.addressBits.W)) - val currentBytes = Reg(UInt(13.W)); val segmentLast = Reg(Bool()) - val unionAuthorization = RegInit(false.B) - val previous = (counts(worker) - 1.U)(2, 0) - val previousVA = vas(worker)(previous) - val previousPA = pas(worker)(previous) - val previousEnd = previousVA +& sizes(worker)(previous) - val currentEnd = currentVA +& currentBytes - val cancelSegmenter = WireDefault(false.B) + val (idle :: checkShape :: multiply :: measure :: span :: sendSpan :: segment :: walk :: waitWalk :: authorize :: + waitPermission :: save :: retire :: Nil) = Enum(13) + val state = RegInit(idle) + val worker = Reg(UInt(contextBits.W)) + val cursor = RegInit(0.U(contextBits.W)) + val row = Reg(UInt(32.W)); val column = Reg(UInt(32.W)) + val dense = Reg(Bool()); val wholeBytes = Reg(UInt(32.W)) + val spanAddress = Reg(UInt(64.W)); val spanBytes = Reg(UInt(32.W)) + val spanLast = Reg(Bool()) + val currentVA = Reg(UInt(64.W)); val currentPA = Reg(UInt(p.bus.addressBits.W)) + val currentBytes = Reg(UInt(13.W)); val segmentLast = Reg(Bool()) + val unionAuthorization = RegInit(false.B) + // The worker's command, copied when it is selected so shape arithmetic starts at registers. + val cmd = Reg(new Command(b)) + // VA of the current row and of the current span: baseVA + row * rowStride (+ column * columnStride). + val rowStart = Reg(UInt(67.W)); val spanStart = Reg(UInt(67.W)) + // Shape products, built eight multiplier bits per cycle: columns * spanBytes, rows * rowStride, + // columns * columnStride and rows * (columns * spanBytes)(31, 0). + val products = Reg(Vec(4, UInt(64.W))) + val product = Reg(UInt(2.W)); val digit = Reg(UInt(2.W)) + val outcome = Reg(UInt(3.W)); val outcomeVA = Reg(UInt(64.W)) + val previous = (counts(worker) - 1.U)(2, 0) + val previousVA = vas(worker)(previous) + val previousPA = pas(worker)(previous) + val previousEnd = previousVA +& sizes(worker)(previous) + val currentEnd = currentVA +& currentBytes + val cancelSegmenter = WireDefault(false.B) val splitter: Instance[Segmenter] = Instantiate(new Segmenter) @@ -80,46 +89,71 @@ class Preflight(b: ModuleParam) extends Module { val walker: Instance[Walker] = Instantiate(new Walker(p.walker)) - val cmd = commands(worker) - + // The outcome is recorded one cycle after it is decided. def finish(code: UInt, address: UInt): Unit = { - errors(worker) := code; faultVA(worker) := address; phase(worker) := ready; state := idle - when(code =/= Error.Ok.U) { counts(worker) := 0.U; cancelSegmenter := true.B } + outcome := code; outcomeVA := address; state := retire + } + + when(state === retire) { + errors(worker) := outcome; faultVA(worker) := outcomeVA; phase(worker) := ready; state := idle + when(outcome =/= Error.Ok.U) { counts(worker) := 0.U; cancelSegmenter := true.B } } - val waiting = VecInit(phase.map(_ === queued)) + val waiting = VecInit(phase.map(_ === queued)) when(state === idle && waiting.asUInt.orR) { val slot = select(waiting, cursor) - worker := slot; cursor := slot + 1.U; phase(slot) := working; state := checkShape + worker := slot; cursor := slot + 1.U; phase(slot) := working; cmd := commands(slot); state := checkShape } when(state === checkShape) { - val rowExtent = cmd.columns * cmd.spanBytes - val total = cmd.rows * rowExtent - val isDense = (cmd.columns === 1.U || cmd.columnStride === cmd.spanBytes) && - (cmd.rows === 1.U || cmd.rowStride === rowExtent) - val end = cmd.baseVA.pad(67) + ((cmd.rows - 1.U) * cmd.rowStride).pad(67) + - ((cmd.columns - 1.U) * cmd.columnStride).pad(67) + cmd.spanBytes.pad(67) - 1.U when(cmd.rows === 0.U || cmd.columns === 0.U || cmd.spanBytes === 0.U || (cmd.rows > 1.U && cmd.rowStride === 0.U) || (cmd.columns > 1.U && cmd.columnStride === 0.U)) { finish(Error.Shape.U, cmd.baseVA) }.elsewhen(cmd.mode =/= 0.U && cmd.mode =/= 8.U || cmd.privilege === 2.U) { finish(Error.Context.U, cmd.baseVA) - }.elsewhen(end(66, 64).orR || (isDense && total > "hffffffff".U)) { + }.otherwise { + product := 0.U; digit := 0.U; state := multiply + } + } + // Least significant multiplier byte first. The upper half of products(k) carries the running + // sum, which stays below 2^32; each step shifts one finished product byte into the lower half. + val multiplicand = VecInit(cmd.columns, cmd.rows, cmd.columns, cmd.rows)(product) + val multiplier = VecInit(cmd.spanBytes, cmd.rowStride, cmd.columnStride, products(0)(31, 0))(product) + val multiplierByte = VecInit((0 until 4).map(i => multiplier(8 * i + 7, 8 * i)))(digit) + when(state === multiply) { + val step = Mux(digit === 0.U, 0.U, products(product)(63, 32)) +& multiplicand * multiplierByte + products(product) := Cat(step(39, 0), products(product)(31, 8)) + digit := digit + 1.U + when(digit === 3.U) { + product := product + 1.U + when(product === 3.U)(state := measure) + } + } + when(state === measure) { + val rowExtent = products(0) + val total = products(3) + val isDense = (cmd.columns === 1.U || cmd.columnStride === cmd.spanBytes) && + (cmd.rows === 1.U || cmd.rowStride === rowExtent) + // baseVA + (rows - 1) * rowStride + (columns - 1) * columnStride + spanBytes - 1 + val end = cmd.baseVA.pad(67) + products(1).pad(67) - cmd.rowStride.pad(67) + products(2).pad(67) - + cmd.columnStride.pad(67) + cmd.spanBytes.pad(67) - 1.U + // rows >= 1, so rows * rowExtent exceeds 32 bits whenever rowExtent does. + when(end(66, 64).orR || (isDense && (rowExtent(63, 32).orR || total(63, 32).orR))) { finish(Error.Overflow.U, cmd.baseVA) }.otherwise { - dense := isDense; wholeBytes := total(31, 0); row := 0.U; column := 0.U; state := span + dense := isDense; wholeBytes := total(31, 0); row := 0.U; column := 0.U + rowStart := cmd.baseVA; spanStart := cmd.baseVA; state := span } } when(state === span) { - val start = cmd.baseVA.pad(67) + (row * cmd.rowStride).pad(67) + (column * cmd.columnStride).pad(67) - val bytes = Mux(dense, wholeBytes, cmd.spanBytes) - val mask = (~(BigInt(p.beatBytes) - 1) & ((BigInt(1) << 67) - 1)).U(67.W) - val first = Mux(cmd.write, start, start & mask) - val end = Mux(cmd.write, start + bytes.pad(67), (start + bytes.pad(67) + (p.beatBytes - 1).U) & mask) - val length = end - first - when(length > "hffffffff".U)(finish(Error.Overflow.U, start(63, 0))) + val bytes = Mux(dense, wholeBytes, cmd.spanBytes) + val beatShift = log2Ceil(p.beatBytes) + // Reads widen to whole beats: (start % beat + bytes + beat - 1) rounded down to a beat. + val widened = (spanStart(beatShift - 1, 0).pad(34) + bytes.pad(34) + (p.beatBytes - 1).U) >> beatShift << beatShift + val length = Mux(cmd.write, bytes.pad(34), widened) + val first = Mux(cmd.write, spanStart(63, 0), spanStart(63, beatShift) << beatShift) + when(length(33, 32).orR)(finish(Error.Overflow.U, spanStart(63, 0))) .otherwise { - spanAddress := first(63, 0); spanBytes := length(31, 0) + spanAddress := first; spanBytes := length(31, 0) spanLast := dense || (row === cmd.rows - 1.U && column === cmd.columns - 1.U) state := sendSpan } @@ -219,9 +253,11 @@ class Preflight(b: ModuleParam) extends Module { } when(segmentLast && spanLast)(finish(Error.Ok.U, currentVA)) .elsewhen(segmentLast) { - when(column === cmd.columns - 1.U) { column := 0.U; row := row + 1.U } - .otherwise(column := column + 1.U) - state := span + when(column === cmd.columns - 1.U) { + column := 0.U; row := row + 1.U + rowStart := rowStart + cmd.rowStride; spanStart := rowStart + cmd.rowStride + }.otherwise { column := column + 1.U; spanStart := spanStart + cmd.columnStride } + state := span }.otherwise(state := segment) } val offer = RegInit(false.B); val outputSlot = Reg(UInt(contextBits.W)); val outputIndex = Reg(UInt(3.W)) diff --git a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PreparedMap.scala b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PreparedMap.scala index a47b2277a..6dad42c53 100644 --- a/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PreparedMap.scala +++ b/arch/src/main/scala/framework/core/memory/access/src/main/scala/preflight/src/main/scala/PreparedMap.scala @@ -137,32 +137,45 @@ class PreparedMap(b: ModuleParam) extends Module { errors(releaseSlot) := Error.Ok.U } + // A lookup takes three cycles and the query must be held throughout: the owning context is + // registered, then the first covering range, and the third cycle answers. The owning context + // must not change while it is queried. for (port <- 0 until 2) { - val query = io.queries(port) - val owner = matches(query.id) - val slot = PriorityEncoder(owner) + val query = io.queries(port) + val owner = matches(query.id) + val owned = RegNext(owner.asUInt.orR) + val slot = RegNext(PriorityEncoder(owner)) + val last = query.va +& (query.bytes - 1.U) + def covers(context: UInt, index: Int): Bool = + index.U < counts(context) && query.va >= vas(context)(index) && + last < (vas(context)(index) +& sizes(context)(index)) && direction(context) === query.write + val hits = VecInit((0 until p.maxRanges).map(covers(slot, _))) + val covered = RegNext(hits.asUInt.orR) + val index = RegNext(PriorityEncoder(hits)) val retiring = io.release.fire && io.release.bits.id === query.id - val last = query.va +& (query.bytes - 1.U) val shapeOK = query.bytes =/= 0.U && !last(64) && (query.va(11, 0) +& query.bytes) <= 4096.U - val authorized = owner.asUInt.orR && sealedList(slot) && notified(slot) && + val authorized = owned && sealedList(slot) && notified(slot) && errors(slot) === Error.Ok.U && !retiring && !reset.asBool - val hits = VecInit((0 until p.maxRanges).map { index => - index.U < counts(slot) && query.va >= vas(slot)(index) && - last < (vas(slot)(index) +& sizes(slot)(index)) && direction(slot) === query.write - }) - val index = PriorityEncoder(hits) val translated = pas(slot)(index) +& (query.va - vas(slot)(index)) - val hit = query.valid && shapeOK && authorized && hits.asUInt.orR + val hit = query.valid && shapeOK && authorized && covered io.results(port).hit := hit io.results(port).pa := Mux(hit, translated(p.bus.addressBits - 1, 0), 0.U) io.results(port).error := Mux(!query.valid || hit, Error.Ok.U, Mux(!shapeOK, Error.Shape.U, Error.AccessFault.U)) - when(hit) { - assert(!translated(translated.getWidth - 1, p.bus.addressBits).orR, "PreparedMap translated address overflows PA") + + // Map invariants, checked on the query of this cycle against its current owner. + val present = PriorityEncoder(owner) + val presentHits = VecInit((0 until p.maxRanges).map(covers(present, _))) + val presentAllowed = owner.asUInt.orR && sealedList(present) && notified(present) && + errors(present) === Error.Ok.U && !retiring && !reset.asBool + val presentPA = pas(present)(PriorityEncoder(presentHits)) +& + (query.va - vas(present)(PriorityEncoder(presentHits))) + when(query.valid && shapeOK && presentAllowed && presentHits.asUInt.orR) { + assert(!presentPA(presentPA.getWidth - 1, p.bus.addressBits).orR, "PreparedMap translated address overflows PA") for (other <- 0 until p.maxRanges) { - when(hits(other)) { + when(presentHits(other)) { assert( - (pas(slot)(other) +& (query.va - vas(slot)(other))) === translated, + (pas(present)(other) +& (query.va - vas(present)(other))) === presentPA, "PreparedMap overlapping records disagree on PA" ) } diff --git a/arch/src/main/scala/framework/core/memory/configs/MemoryParam.scala b/arch/src/main/scala/framework/core/memory/configs/MemoryParam.scala index 8c91fd5c4..90a1738b0 100644 --- a/arch/src/main/scala/framework/core/memory/configs/MemoryParam.scala +++ b/arch/src/main/scala/framework/core/memory/configs/MemoryParam.scala @@ -28,7 +28,7 @@ object MemoryParam { val requester = homep.memory.chi MemoryParam( AccessParam( - AccessParams(requester, a.getAccessTagBits), + AccessParams(requester, a.getTagBits), a.getRegionsList.asScala.map(PhysicalRegion.fromProto).toSeq, a.getTranslationEntries, a.getResultLineBits diff --git a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/configs/uvm.toml b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/configs/uvm.toml index cb0c04b6e..d0bfc6b0c 100644 --- a/arch/src/main/scala/framework/core/tbus/src/main/scala/router/configs/uvm.toml +++ b/arch/src/main/scala/framework/core/tbus/src/main/scala/router/configs/uvm.toml @@ -10,8 +10,8 @@ rob_entries = 4 bank_alias_enable = true rs_out_of_order_response = true bank_id_len = 4 -vbank_id_upper_bound = 3 -shared_bank_id_base = 16 +max_private_vbank = 3 +shared_vbank_base = 16 iter_len = 16 sub_rob_enable = false sub_rob_depth = 4 diff --git a/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/BankedChiCache.scala b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/BankedChiCache.scala index a99cbaefc..6c9bd0134 100644 --- a/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/BankedChiCache.scala +++ b/arch/src/main/scala/framework/ip/chi/src/main/scala/rnf/BankedChiCache.scala @@ -31,7 +31,7 @@ object RnfParams { def fromProto(p: RnfHardware, requester: Params): RnfParams = RnfParams( requester, p.getNodeId, - p.getCacheLines, + p.getLines, p.getHomeId, p.getHomeCount, p.getBanks, diff --git a/arch/src/main/scala/framework/ip/coherence/configs/CoherenceParams.scala b/arch/src/main/scala/framework/ip/coherence/configs/CoherenceParams.scala index 8ea86be8a..bf78a86cf 100644 --- a/arch/src/main/scala/framework/ip/coherence/configs/CoherenceParams.scala +++ b/arch/src/main/scala/framework/ip/coherence/configs/CoherenceParams.scala @@ -28,7 +28,7 @@ object CoherenceParams { CacheParams.fromProto(p.getCache), p.getAgents, p.getMshrEntries, - p.getHomeId + p.getNodeId ) } diff --git a/arch/src/main/scala/framework/lib/queue/Queue.scala b/arch/src/main/scala/framework/lib/queue/Queue.scala index edf19148c..dc670b597 100644 --- a/arch/src/main/scala/framework/lib/queue/Queue.scala +++ b/arch/src/main/scala/framework/lib/queue/Queue.scala @@ -15,11 +15,8 @@ class Queue[T <: Data]( @public val io = IO(new QueueIO(gen, entries)) - val memory = SyncReadMem(entries, gen) val count = RegInit(0.U(log2Ceil(entries + 1).W)) val memoryCount = RegInit(0.U(log2Ceil(entries + 1).W)) - val head = RegInit(0.U(math.max(1, log2Ceil(entries)).W)) - val tail = RegInit(0.U(math.max(1, log2Ceil(entries)).W)) val front = Reg(gen) val frontValid = RegInit(false.B) val skid = Reg(gen) @@ -31,14 +28,10 @@ class Queue[T <: Data]( io.deq.bits := (if (flow) Mux(frontValid, front, io.enq.bits) else front) val dequeue = frontValid && io.deq.ready io.enq.ready := count < entries.U || (if (pipe) dequeue else false.B) - val bypass = if (flow) count === 0.U && io.deq.fire else false.B - val enqueue = io.enq.fire && !bypass - val direct = enqueue && memoryCount === 0.U && !readPending && !skidValid && (!frontValid || dequeue) - val write = enqueue && !direct - val held = frontValid.asUInt +& skidValid.asUInt - val occupied = held +& readPending.asUInt - val read = memoryCount =/= 0.U && !write && occupied < (2.U + dequeue.asUInt) - val readData = memory.readWrite(Mux(write, tail, head), io.enq.bits, write || read, write) + val bypass = if (flow) count === 0.U && io.deq.fire else false.B + val enqueue = io.enq.fire && !bypass + val direct = enqueue && memoryCount === 0.U && !readPending && !skidValid && (!frontValid || dequeue) + val write = enqueue && !direct val nextFront = WireDefault(front) val nextFrontValid = WireDefault(frontValid && !dequeue) @@ -49,12 +42,32 @@ class Queue[T <: Data]( nextFrontValid := true.B nextSkidValid := false.B } - when(readPending) { - when((!frontValid || dequeue) && !skidValid) { - nextFront := readData - nextFrontValid := true.B - }.otherwise { - nextSkid := readData + if (entries > 2) { + val memory = SyncReadMem(entries, gen) + val head = RegInit(0.U(log2Ceil(entries).W)) + val tail = RegInit(0.U(log2Ceil(entries).W)) + val occupied = frontValid.asUInt +& skidValid.asUInt +& readPending.asUInt + val read = memoryCount =/= 0.U && !write && occupied < (2.U + dequeue.asUInt) + val readData = memory.readWrite(Mux(write, tail, head), io.enq.bits, write || read, write) + when(readPending) { + when((!frontValid || dequeue) && !skidValid) { + nextFront := readData + nextFrontValid := true.B + }.otherwise { + nextSkid := readData + nextSkidValid := true.B + } + } + readPending := read + when(write =/= read) { + memoryCount := Mux(write, memoryCount + 1.U, memoryCount - 1.U) + } + when(write)(tail := Mux(tail === (entries - 1).U, 0.U, tail + 1.U)) + when(read)(head := Mux(head === (entries - 1).U, 0.U, head + 1.U)) + } else { + // Front and skid registers hold up to two entries, so small queues need no memory + when(write) { + nextSkid := io.enq.bits nextSkidValid := true.B } } @@ -63,17 +76,11 @@ class Queue[T <: Data]( nextFrontValid := true.B } front := nextFront - frontValid := nextFrontValid - skid := nextSkid - skidValid := nextSkidValid - readPending := read + frontValid := nextFrontValid + skid := nextSkid + skidValid := nextSkidValid when(enqueue =/= dequeue) { count := Mux(enqueue, count + 1.U, count - 1.U) } - when(write =/= read) { - memoryCount := Mux(write, memoryCount + 1.U, memoryCount - 1.U) - } - when(write)(tail := Mux(tail === (entries - 1).U, 0.U, tail + 1.U)) - when(read)(head := Mux(head === (entries - 1).U, 0.U, head + 1.U)) } diff --git a/arch/thirdparty/build.sc b/arch/thirdparty/build.sc index d409c958a..c0f3f7ba6 100644 --- a/arch/thirdparty/build.sc +++ b/arch/thirdparty/build.sc @@ -11,7 +11,7 @@ object protoJava extends JavaModule { def protoSources = T.sources { Seq(PathRef(protoDir() / "design.proto"), PathRef(protoDir() / "hardware.proto")) ++ Seq("balls", "cores").flatMap(kind => - os.walk(archRoot / os.up / "examples" / "components" / kind).filter(p => p.ext == "proto" && !p.toString.contains("/thirdparty/") && !p.toString.contains("/out/")).map(PathRef(_))) + os.walk(archRoot / os.up / "examples" / "prefab" / kind).filter(p => p.ext == "proto" && !p.toString.contains("/thirdparty/") && !p.toString.contains("/out/")).map(PathRef(_))) } def generatedSources = T { diff --git a/arch/uvm.toml b/arch/uvm.toml index 25a059dcd..be18d381a 100644 --- a/arch/uvm.toml +++ b/arch/uvm.toml @@ -171,7 +171,7 @@ path = "arch/src/main/scala/framework/acc/frontend" [modules.frontend.build] directory = "arch/src/main/scala/framework" module = "acc.frontend" -arguments = ["--config", "{repo}/examples/designs/{design}/output/generated/design.pb"] +arguments = ["--config", "{repo}/examples/designs/{design}/output/sims/device/generated/design.pb"] [modules.memdomain] level = "domain" @@ -189,7 +189,7 @@ path = "arch/src/main/scala/framework/acc/gpdomain" [modules.gpdomain.build] directory = "arch/src/main/scala/framework" module = "acc.gpdomain" -arguments = ["--config", "{repo}/examples/designs/{design}/output/generated/design.pb"] +arguments = ["--config", "{repo}/examples/designs/{design}/output/sims/device/generated/design.pb"] [modules.balldomain] level = "domain" @@ -199,7 +199,7 @@ path = "arch/src/main/scala/framework/acc/balldomain" runtime = [{ directory = "examples", module = "library" }] directory = "arch/src/main/scala/framework" module = "acc.balldomain" -arguments = ["--config", "{repo}/examples/designs/{design}/output/generated/design.pb"] +arguments = ["--config", "{repo}/examples/designs/{design}/output/sims/device/generated/design.pb"] [modules.ant] level = "thread" @@ -208,7 +208,7 @@ path = "arch/src/main/scala/framework/thread/ant" [modules.ant.build] directory = "arch/src/main/scala/framework" module = "thread.ant" -arguments = ["--config", "{repo}/examples/designs/{design}/output/generated/design.pb", "--core", "1"] +arguments = ["--config", "{repo}/examples/designs/{design}/output/sims/device/generated/design.pb", "--core", "1"] [modules.mesh_shm] level = "subsystem" diff --git a/bb-tests/workloads/CMakeLists.txt b/bb-tests/workloads/CMakeLists.txt index 277b7c022..a6d9dbdbd 100644 --- a/bb-tests/workloads/CMakeLists.txt +++ b/bb-tests/workloads/CMakeLists.txt @@ -1,59 +1,42 @@ +# ===----------------------------------------------------------------------=== +# The workloads of one simulation of a design, configured with DESIGN_CMAKE, +# generated from its design.pb by `bbdev stack --build`: +# +# cmake -G Ninja -S bb-tests/workloads -B -DDESIGN_CMAKE= +# ninja -C sync-bin | sync-ctest-bin | sync-mlirtest-bin | sync-soctest-bin +# +# Products are copied to ${DESIGN_OUTPUT}/workloads. +# ===----------------------------------------------------------------------=== cmake_minimum_required(VERSION 3.26) +set(CMAKE_C_COMPILER riscv64-unknown-linux-gnu-gcc) +set(CMAKE_CXX_COMPILER riscv64-unknown-linux-gnu-g++) project(BuckyballWorkloads C CXX) +include("${DESIGN_CMAKE}") set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_BUILD_TYPE Release) +set(CMAKE_C_FLAGS_RELEASE "-O2 -UNDEBUG") +set(CMAKE_CXX_FLAGS_RELEASE "-O2 -UNDEBUG") set(WORKLOAD_DIR ${CMAKE_CURRENT_LIST_DIR}) set(WORKLOAD_SRC_DIR ${WORKLOAD_DIR}/src) -get_filename_component(BUCKYBALL_REPO_ROOT ${WORKLOAD_DIR}/../.. ABSOLUTE) +set(BUCKYBALL_REPO_ROOT ${DESIGN_REPO}) set(WORKLOAD_LIB_DIR ${WORKLOAD_DIR}/lib) set(BAREMETAL_DIR ${BUCKYBALL_REPO_ROOT}/stack/os/baremetal) +# bb_cores.h: the cores of DESIGN_DEFS. +set(RUNTIME_INCLUDE_DIR ${BUCKYBALL_REPO_ROOT}/stack/runtime/include) -find_package(Python3 REQUIRED COMPONENTS Interpreter) find_program(ELF_CC riscv64-unknown-elf-gcc REQUIRED) find_program(ELF_CXX riscv64-unknown-elf-g++ REQUIRED) find_program(ELF_OBJCOPY riscv64-unknown-elf-objcopy REQUIRED) set(LINUX_CC ${CMAKE_C_COMPILER}) set(LINUX_CXX ${CMAKE_CXX_COMPILER}) -set(BUCKYBALL_WORKLOAD_DESIGN "" CACHE STRING "Buckyball design for workloads") -if(BUCKYBALL_WORKLOAD_DESIGN STREQUAL "") - message( - FATAL_ERROR - "BUCKYBALL_WORKLOAD_DESIGN is required. Pass -DBUCKYBALL_WORKLOAD_DESIGN=." - ) -endif() -if(NOT IS_DIRECTORY "${BUCKYBALL_REPO_ROOT}/examples/designs/${BUCKYBALL_WORKLOAD_DESIGN}") - message(FATAL_ERROR "Buckyball design does not exist: ${BUCKYBALL_WORKLOAD_DESIGN}") -endif() - -if(NOT DEFINED BUCKYBALL_DESIGN_PB OR BUCKYBALL_DESIGN_PB STREQUAL "") - message(FATAL_ERROR "BUCKYBALL_DESIGN_PB is required") -endif() - -if(NOT EXISTS "${BUCKYBALL_DESIGN_PB}") - message(FATAL_ERROR "missing Design.pb: ${BUCKYBALL_DESIGN_PB}") -endif() -# Everything this build needs from design.pb: ISA headers and the DESIGN_* lists. -execute_process( - COMMAND - "${Python3_EXECUTABLE}" "${BUCKYBALL_REPO_ROOT}/stack/scripts/design.py" - --design-pb "${BUCKYBALL_DESIGN_PB}" --target "${BUCKYBALL_SIM_TARGET}" - --out "${CMAKE_BINARY_DIR}/generated" - COMMAND_ERROR_IS_FATAL ANY -) -set_property( - DIRECTORY - APPEND - PROPERTY - CMAKE_CONFIGURE_DEPENDS - "${BUCKYBALL_DESIGN_PB}" - "${BUCKYBALL_REPO_ROOT}/stack/scripts/design.py" -) -include("${CMAKE_BINARY_DIR}/generated/design.cmake") # runtime, runtime-baremetal, runtime-linux and engine of every program here: -# built by `bbdev stack --build --device`. -set(BUCKYBALL_ISA_DIR "${CMAKE_BINARY_DIR}/generated/isa") -set(BUCKYBALL_CONTROL_TARGET "core${DESIGN_CONTROL_HART}") +# built by `bbdev stack --build "--design --device "`. +set(BUCKYBALL_CONTROL_TARGET "core${DESIGN_WORKLOAD_CONTROL_HART}") +# The definitions of every program of the design, as compiler flags. +set(DESIGN_DEFINES ${DESIGN_DEFS}) +list(TRANSFORM DESIGN_DEFINES PREPEND "-D") set(BBSIM_LD ${BAREMETAL_DIR}/bbsim.ld) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=rv64gc") @@ -61,7 +44,7 @@ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -Wl,--no-dynamic-linker" ) -set(OUTPUT_BIN_DIR "${BUCKYBALL_WORKLOAD_OUTPUT}") +set(OUTPUT_BIN_DIR "${DESIGN_OUTPUT}/workloads") set(CTEST_MAX_LINES 100) set(BBHW_MEM_C ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/dma.c) @@ -81,12 +64,12 @@ set(BUCKYBALL_CTEST_C_FLAGS file( GLOB BUCKYBALL_BALL_WORKLOAD_DIRS CONFIGURE_DEPENDS - "${BUCKYBALL_REPO_ROOT}/examples/components/balls/*/workloads" + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/*/workloads" ) file( GLOB BUCKYBALL_BALL_ISA_DEPS CONFIGURE_DEPENDS - "${BUCKYBALL_REPO_ROOT}/examples/components/balls/*/workloads/isa/*" + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/*/workloads/isa/*" ) set(BUDDY_MLIR_DIR ${BUCKYBALL_REPO_ROOT}/stack/thirdparty/buddy-mlir) diff --git a/bb-tests/workloads/lib/bbhw/isa/bb_func7.h b/bb-tests/workloads/lib/bbhw/isa/bb_func7.h index 7db8ad0bd..855065681 100644 --- a/bb-tests/workloads/lib/bbhw/isa/bb_func7.h +++ b/bb-tests/workloads/lib/bbhw/isa/bb_func7.h @@ -1,9 +1,9 @@ -#ifndef BB_FUNC7_H -#define BB_FUNC7_H +#ifndef FUNCT7_H +#define FUNCT7_H -#include "ballISA.h" - -#define BB_FUNC7(mne) BB_FUNC7_##mne -#define BB_FUNC7_INSTANCE(id, mne) BB_FUNC7_BALL_##id##_##mne +// The funct7 of instruction `mne` of the Ball in directory `ball`, from the +// compile definitions of the target (DESIGN_TARGET_DEFS in design.cmake). +#define FUNCT7(ball, mne) \ + THREAD_BALLDOMAIN_BALLS_##ball##_INSTRUCTIONS_##mne##_FUNCT7 #endif diff --git a/bb-tests/workloads/lib/bbhw/mmio/mmio_allocator.h b/bb-tests/workloads/lib/bbhw/mmio/mmio_allocator.h index 786a13513..8c8614541 100644 --- a/bb-tests/workloads/lib/bbhw/mmio/mmio_allocator.h +++ b/bb-tests/workloads/lib/bbhw/mmio/mmio_allocator.h @@ -1,13 +1,13 @@ #ifndef BBHW_MMIO_ALLOCATOR_H #define BBHW_MMIO_ALLOCATOR_H -#include #include #include -#define MMIO_BANK_BYTES ((MMIO_BANK_ENTRIES) * ((MMIO_BANK_WIDTH_BITS) / 8)) -#define MMIO_TOTAL_BYTES ((MMIO_BANK_NUM) * (MMIO_BANK_BYTES)) -#define MMIO_LOGICAL_ROW_BYTES ((BANK_WIDTH) / 8) +#define MMIO_BANK_BYTES \ + ((THREAD_MEM_MMIO_BANK_ENTRIES) * ((THREAD_MEM_MMIO_BANK_WIDTH) / 8)) +#define MMIO_TOTAL_BYTES ((THREAD_MEM_MMIO_BANK_NUM) * (MMIO_BANK_BYTES)) +#define MMIO_LOGICAL_ROW_BYTES ((THREAD_MEM_BANK_WIDTH) / 8) #define MMIO_LOGICAL_ROWS ((MMIO_TOTAL_BYTES) / (MMIO_LOGICAL_ROW_BYTES)) _Static_assert((MMIO_TOTAL_BYTES) % (MMIO_LOGICAL_ROW_BYTES) == 0, "MMIO bytes must be a multiple of SRAM row bytes"); diff --git a/bb-tests/workloads/scripts/build.py b/bb-tests/workloads/scripts/build.py deleted file mode 100644 index a37499e03..000000000 --- a/bb-tests/workloads/scripts/build.py +++ /dev/null @@ -1,140 +0,0 @@ -from __future__ import annotations -import re -import shlex -import shutil -import subprocess -import sys -from pathlib import Path - - -def _run( - cmd: list[str], - *, - cwd: Path, - prefix: str, - logger: object | None = None, - task_scope: str | None = None, -) -> None: - if logger is None: - result = subprocess.run(cmd, cwd=cwd) - else: - from utils.stream_run import stream_run_logger - - result = stream_run_logger( - cmd=shlex.join(cmd), - logger=logger, - cwd=str(cwd), - stdout_prefix=prefix, - stderr_prefix=prefix, - task_scope=task_scope, - ) - if result.returncode != 0: - raise RuntimeError(f"command failed ({result.returncode}): {' '.join(cmd)}") - - -def build_workload( - repo: str | Path, - design: str, - *, - selection: str, - ctest: bool = False, - mlirtest: bool = False, - soctest: bool = False, - logger: object | None = None, - task_scope: str | None = None, -) -> None: - if not re.fullmatch("[A-Za-z0-9_-]+", design): - raise ValueError(f"invalid design: {design}") - if sum((ctest, mlirtest, soctest)) > 1: - raise ValueError("--ctest, --mlirtest and --soctest are mutually exclusive") - root = Path(repo).resolve() - design_pb = root / "examples/designs" / design / "output/generated/design.pb" - sys.path.insert(0, str(root)) - from stack.scripts.design import load - - target = load(design_pb).simulations[selection] - if not target.directory: - raise ValueError(f"{design}: unregistered simulation entry {selection}") - defs = { - "BUCKYBALL_WORKLOAD_DESIGN": design, - "BUCKYBALL_DESIGN_PB": str(design_pb), - } - defs["BUCKYBALL_WORKLOAD_SCOPE"] = ( - "ctest" - if ctest - else "mlirtest" if mlirtest else "soctest" if soctest else "all" - ) - defs["BUCKYBALL_SIM_TARGET"] = selection - defs["BUCKYBALL_DESIGN_WORKLOAD_DIR"] = str(root / target.workloads) - defs["BUCKYBALL_WORKLOAD_OUTPUT"] = str( - root / "examples/designs" / design / "output/sims" / selection / "workloads" - ) - tools = {} - for name in ( - "cmake", - "riscv64-unknown-elf-gcc", - "riscv64-unknown-elf-g++", - "riscv64-unknown-elf-objcopy", - "riscv64-unknown-linux-gnu-gcc", - "riscv64-unknown-linux-gnu-g++", - ): - executable = shutil.which(name) - if executable is None: - raise FileNotFoundError(f"{name} is required; enter nix develop") - tools[name] = str(Path(executable).absolute()) - source = root / "bb-tests" / "workloads" - build = ( - root - / "examples/designs" - / design - / "output/sims" - / selection - / "workloads-build" - ) - target = ( - "sync-ctest-bin" - if ctest - else ( - "sync-mlirtest-bin" - if mlirtest - else "sync-soctest-bin" if soctest else "sync-bin" - ) - ) - build.mkdir(parents=True, exist_ok=True) - cmake_args = [ - tools["cmake"], - "-G", - "Ninja", - "-DCMAKE_BUILD_TYPE=Release", - "-DCMAKE_C_FLAGS_RELEASE=-O2 -UNDEBUG", - "-DCMAKE_CXX_FLAGS_RELEASE=-O2 -UNDEBUG", - "-S", - str(source), - "-B", - str(build), - f"-DPython3_EXECUTABLE={sys.executable}", - f"-DCMAKE_C_COMPILER={tools['riscv64-unknown-linux-gnu-gcc']}", - f"-DCMAKE_CXX_COMPILER={tools['riscv64-unknown-linux-gnu-g++']}", - ] - for variable, executable in ( - ("ELF_CC", "riscv64-unknown-elf-gcc"), - ("ELF_CXX", "riscv64-unknown-elf-g++"), - ("ELF_OBJCOPY", "riscv64-unknown-elf-objcopy"), - ): - cmake_args.append(f"-D{variable}={tools[executable]}") - for key, value in defs.items(): - cmake_args.append(f"-D{key}={value}") - _run( - cmake_args, - cwd=root, - prefix="workload configure", - logger=logger, - task_scope=task_scope, - ) - _run( - [tools["cmake"], "--build", str(build), "--parallel", "1", "--target", target], - cwd=root, - prefix="workload build", - logger=logger, - task_scope=task_scope, - ) diff --git a/bb-tests/workloads/src/CMakeLists.txt b/bb-tests/workloads/src/CMakeLists.txt index 3dfae01a8..18c407fa4 100644 --- a/bb-tests/workloads/src/CMakeLists.txt +++ b/bb-tests/workloads/src/CMakeLists.txt @@ -3,30 +3,10 @@ add_subdirectory(MLIRTest) add_subdirectory(SocTest) # =============================================================== -# Design-specific workloads +# Design-specific workloads: the workload entry of the simulation. +# It defines target design-workloads-build. # =============================================================== -if(NOT DEFINED BUCKYBALL_DESIGN_WORKLOAD_DIR) - message( - FATAL_ERROR - "BUCKYBALL_DESIGN_WORKLOAD_DIR must select a registered simulation workload entry" - ) -endif() - -if(NOT EXISTS ${BUCKYBALL_DESIGN_WORKLOAD_DIR}/CMakeLists.txt) - message( - FATAL_ERROR - "Buckyball design workloads do not exist: ${BUCKYBALL_DESIGN_WORKLOAD_DIR}" - ) -endif() - add_subdirectory( - ${BUCKYBALL_DESIGN_WORKLOAD_DIR} - ${CMAKE_CURRENT_BINARY_DIR}/CTest/designs/${BUCKYBALL_WORKLOAD_DESIGN} + ${DESIGN_WORKLOADS} + ${CMAKE_CURRENT_BINARY_DIR}/CTest/designs/${DESIGN_NAME} ) - -if(NOT TARGET design-workloads-build) - message( - FATAL_ERROR - "Buckyball design workloads must define target design-workloads-build: ${BUCKYBALL_WORKLOAD_DESIGN}" - ) -endif() diff --git a/bb-tests/workloads/src/CTest/CMakeLists.txt b/bb-tests/workloads/src/CTest/CMakeLists.txt index 201540ae2..77ea75cd6 100644 --- a/bb-tests/workloads/src/CTest/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/CMakeLists.txt @@ -38,7 +38,7 @@ function(buckyball_ctest_deps OUT_DEPS SOURCE_DIR SOURCE_FILE) message(FATAL_ERROR "BUCKYBALL_CTEST_TARGET is required for C tests") endif() list( - FIND DESIGN_TARGETS + FIND DESIGN_PROFILES "${BUCKYBALL_CTEST_TARGET}" _buckyball_target_index ) @@ -54,9 +54,6 @@ function(buckyball_ctest_deps OUT_DEPS SOURCE_DIR SOURCE_FILE) ${SOURCE_DIR}/${SOURCE_FILE} ${WORKLOAD_LIB_DIR}/buckyball.c ${WORKLOAD_LIB_DIR}/buckyball.h - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/ballISA.h - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h - ${BUCKYBALL_ISA_DIR}/design/instances.h ${BBHW_MEM_C} ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/dma.h ${BUCKYBALL_DESIGN_COMMON_SOURCES} @@ -86,11 +83,14 @@ function(add_buckyball_linux_ctest TEST_NAME SOURCE_DIR SOURCE_FILE) ${WORKLOAD_LIB_DIR} ${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler ${BUCKYBALL_REPO_ROOT}/stack/runtime/services - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} ${BUCKYBALL_DESIGN_COMMON_INCLUDE_DIRS} ${BUCKYBALL_BALL_WORKLOAD_DIRS} ${SOURCE_DIR} ) + target_compile_definitions( + ${EXECUTABLE} + PRIVATE ${DESIGN_TARGET_DEFS_${BUCKYBALL_CTEST_TARGET}} + ) set_target_properties(${EXECUTABLE} PROPERTIES LINKER_LANGUAGE C) add_custom_target(${TEST_NAME}-linux-build DEPENDS ${EXECUTABLE}) @@ -107,6 +107,8 @@ function(add_buckyball_baremetal_ctest TEST_NAME SOURCE_DIR SOURCE_FILE) -I${DESIGN_COMMON_INCLUDE_DIR} ) endforeach() + set(TARGET_DEFINES ${DESIGN_TARGET_DEFS_${BUCKYBALL_CTEST_TARGET}}) + list(TRANSFORM TARGET_DEFINES PREPEND "-D") set(BALL_ISA_INCLUDE_FLAGS) foreach(BALL_WORKLOAD_DIR ${BUCKYBALL_BALL_WORKLOAD_DIRS}) list(APPEND BALL_ISA_INCLUDE_FLAGS -I${BALL_WORKLOAD_DIR}) @@ -121,13 +123,14 @@ function(add_buckyball_baremetal_ctest TEST_NAME SOURCE_DIR SOURCE_FILE) ${SOURCE_DIR}/${SOURCE_FILE} -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services - -I${WORKLOAD_LIB_DIR} -I${BUCKYBALL_ISA_DIR}/design - -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} + -I${WORKLOAD_LIB_DIR} ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} + ${TARGET_DEFINES} ${BUCKYBALL_DESIGN_COMMON_INCLUDE_FLAGS} ${BALL_ISA_INCLUDE_FLAGS} -I${SOURCE_DIR} DEPENDS ${TEST_DEPS} ${BAREMETAL_DIR}/crt0.S ${BBSIM_LD} COMMENT "Building baremetal executable: ${EXECUTABLE}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM ) add_custom_target( @@ -137,12 +140,6 @@ function(add_buckyball_baremetal_ctest TEST_NAME SOURCE_DIR SOURCE_FILE) endfunction() function(add_buckyball_ctest SOURCE_FILE) - if(BUCKYBALL_WORKLOAD_DESIGN STREQUAL "") - message( - FATAL_ERROR - "BUCKYBALL_WORKLOAD_DESIGN must be set before add_buckyball_ctest()" - ) - endif() get_filename_component(_stem ${SOURCE_FILE} NAME_WE) if(NOT DEFINED BUCKYBALL_CTEST_TARGET OR BUCKYBALL_CTEST_TARGET STREQUAL "") message( @@ -151,7 +148,7 @@ function(add_buckyball_ctest SOURCE_FILE) ) endif() set(TEST_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-${_stem}" + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-${_stem}" ) set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) @@ -188,9 +185,11 @@ function(add_buckyball_component_workload NAME) "Workload target and sources are required for ${NAME}" ) endif() - set(_profile "${DESIGN_TARGET_ID_${BUCKYBALL_CTEST_TARGET}}") + list(FIND DESIGN_PROFILES "${BUCKYBALL_CTEST_TARGET}" _profile) + set(_defines ${DESIGN_TARGET_DEFS_${BUCKYBALL_CTEST_TARGET}}) + list(TRANSFORM _defines PREPEND "-D") set(_test - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-pipeline-${NAME}" + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-pipeline-${NAME}" ) set(_elf "${_test}-multicore-baremetal") set(_sources) @@ -213,18 +212,17 @@ function(add_buckyball_component_workload NAME) -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services -I${WORKLOAD_LIB_DIR} -I${BAREMETAL_DIR}/multithread - -I${BUCKYBALL_ISA_DIR}/design - -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} ${_ball_includes} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} + ${_defines} ${_ball_includes} -o ${_elf} ${BAREMETAL_DIR}/start.S ${_sources} ${WORKLOAD_LIB_DIR}/buckyball.c ${BBHW_MEM_C} DEPENDS ${_sources} ${_headers} ${BAREMETAL_DIR}/start.S - ${BUCKYBALL_ISA_DIR}/design/instances.h - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/ballISA.h - ${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h + ${RUNTIME_INCLUDE_DIR}/bb_cores.h WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM ) add_custom_target( ${_test}-thread-build diff --git a/bb-tests/workloads/src/CTest/base/CMakeLists.txt b/bb-tests/workloads/src/CTest/base/CMakeLists.txt index 6d7d55b5f..fdcb2160c 100644 --- a/bb-tests/workloads/src/CTest/base/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/base/CMakeLists.txt @@ -2,29 +2,28 @@ add_custom_target(base-ctests-build) set(BUCKYBALL_CTEST_GROUP_TARGET base-ctests-build) foreach(BUCKYBALL_CTEST_TARGET IN LISTS DESIGN_CPU_TARGETS) - set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h") - foreach(_name BANK_NUM BANK_WIDTH BANK_LINES BB_PRIVATE_VBANK_MAX) - file(STRINGS "${_params}" _line REGEX "^#define ${_name} ") - string(REGEX REPLACE ".* " "" _${_name} "${_line}") - endforeach() - if(_BB_PRIVATE_VBANK_MAX EQUAL 0) + set(_BANK_NUM ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_NUM}) + set(_BANK_WIDTH ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_WIDTH}) + set(_BANK_ENTRIES ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_ENTRIES}) + set(_MAX_PRIVATE_VBANK ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_FRONTEND_MAX_PRIVATE_VBANK}) + if(_MAX_PRIVATE_VBANK EQUAL 0) continue() endif() math(EXPR _word_remainder "${_BANK_WIDTH} % 32") if( _BANK_NUM LESS 4 - OR _BB_PRIVATE_VBANK_MAX LESS 2 - OR _BANK_LINES LESS 24 + OR _MAX_PRIVATE_VBANK LESS 2 + OR _BANK_ENTRIES LESS 24 OR _BANK_WIDTH LESS 64 OR _BANK_WIDTH GREATER 2048 OR NOT _word_remainder EQUAL 0 ) message( FATAL_ERROR - "base DMA tests require 4 banks for ACC, vbank2, >=24 rows for 2D, and 64..2048-bit rows divisible by 32; ${BUCKYBALL_CTEST_TARGET}: banks=${_BANK_NUM}, privateMax=${_BB_PRIVATE_VBANK_MAX}, lines=${_BANK_LINES}, width=${_BANK_WIDTH}" + "base DMA tests require 4 banks for ACC, vbank2, >=24 rows for 2D, and 64..2048-bit rows divisible by 32; ${BUCKYBALL_CTEST_TARGET}: banks=${_BANK_NUM}, privateMax=${_MAX_PRIVATE_VBANK}, lines=${_BANK_ENTRIES}, width=${_BANK_WIDTH}" ) endif() - if(_BB_PRIVATE_VBANK_MAX GREATER_EQUAL 4) + if(_MAX_PRIVATE_VBANK GREATER_EQUAL 4) add_buckyball_ctest(mset_transfer_test.c) endif() add_buckyball_ctest(mvin_mvout_group_test.c) diff --git a/bb-tests/workloads/src/CTest/base/mset_transfer_test.c b/bb-tests/workloads/src/CTest/base/mset_transfer_test.c index 540d27834..9a23fe1d3 100644 --- a/bb-tests/workloads/src/CTest/base/mset_transfer_test.c +++ b/bb-tests/workloads/src/CTest/base/mset_transfer_test.c @@ -1,16 +1,16 @@ #include -#include #include #include #include -enum { ROW_BYTES = BANK_WIDTH / 8 }; +enum { ROW_BYTES = THREAD_MEM_BANK_WIDTH / 8 }; static uint8_t first[2 * ROW_BYTES] __attribute__((aligned(16))); static uint8_t second[4 * ROW_BYTES] __attribute__((aligned(16))); static uint8_t replacement[2 * ROW_BYTES] __attribute__((aligned(16))); static uint8_t result[6 * ROW_BYTES] __attribute__((aligned(16))); -static uint8_t zeros[BANK_NUM * ROW_BYTES] __attribute__((aligned(16))); +static uint8_t zeros[THREAD_MEM_BANK_NUM * ROW_BYTES] + __attribute__((aligned(16))); int main(int argc, char **argv) { for (unsigned i = 0; i < sizeof(first); ++i) { @@ -54,7 +54,7 @@ int main(int argc, char **argv) { } bb_mem_release(0); bb_mem_release(3); - bb_mset_clear(4, 1, BANK_NUM); + bb_mset_clear(4, 1, THREAD_MEM_BANK_NUM); bb_mvout((uintptr_t)zeros, 4, 1, 1); for (unsigned i = 0; i < sizeof(zeros); ++i) if (zeros[i]) diff --git a/bb-tests/workloads/src/CTest/base/mvin_2d_pixel_stride_test.c b/bb-tests/workloads/src/CTest/base/mvin_2d_pixel_stride_test.c index b9a45bb46..9bf01a0e1 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_2d_pixel_stride_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_2d_pixel_stride_test.c @@ -1,19 +1,20 @@ #include "buckyball.h" #include #include -#include #include #include -_Static_assert(BANK_WIDTH == 128 && BANK_LINES >= 64, +_Static_assert(THREAD_MEM_BANK_WIDTH == 128 && THREAD_MEM_BANK_ENTRIES >= 64, "MVIN2D boundary test requires 128-bit rows and at least 1 KiB"); static const uint8_t input[8 * 256] __attribute__((aligned(128))) = { [0 * 256] = 11, [1 * 256] = 22, [2 * 256] = 33, [3 * 256] = 44, [4 * 256] = 55, [5 * 256] = 66, [6 * 256] = 77, [7 * 256] = 88, }; -static const uint8_t zeros[64 * BANK_WIDTH / 8] __attribute__((aligned(128))); -static uint8_t output[64 * BANK_WIDTH / 8] __attribute__((aligned(128))); +static const uint8_t zeros[64 * THREAD_MEM_BANK_WIDTH / 8] + __attribute__((aligned(128))); +static uint8_t output[64 * THREAD_MEM_BANK_WIDTH / 8] + __attribute__((aligned(128))); // MobileNet's 72-byte pixels: the last read straddles the 4 KiB page. static const uint8_t page_input[8192] __attribute__((aligned(4096))) = { @@ -32,15 +33,16 @@ int main(void) { bb_mem_alloc(bank, 1, 1); if (test == 2) bb_mvin((uintptr_t)zeros, bank, 64, 1); - bb_mvin_2d(source, bank, 1, stride, 8, first_row, 8, BANK_WIDTH / 8); + bb_mvin_2d(source, bank, 1, stride, 8, first_row, 8, + THREAD_MEM_BANK_WIDTH / 8); bb_mvout((uintptr_t)output, bank, rows, 1); for (int row = 0; row < rows; ++row) { - for (int byte = 0; byte < BANK_WIDTH / 8; ++byte) { + for (int byte = 0; byte < THREAD_MEM_BANK_WIDTH / 8; ++byte) { uint8_t expected = row >= first_row && byte == 0 ? (uint8_t)((row - first_row + 1) * 11) : 0; - uint8_t actual = output[row * (BANK_WIDTH / 8) + byte]; + uint8_t actual = output[row * (THREAD_MEM_BANK_WIDTH / 8) + byte]; if (actual != expected) { printf("mvin_2d pixel stride mismatch case=%d row=%d byte=%d " "expected=%u " diff --git a/bb-tests/workloads/src/CTest/base/mvin_2d_test.c b/bb-tests/workloads/src/CTest/base/mvin_2d_test.c index 8b1ec2f9f..eeac6de55 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_2d_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_2d_test.c @@ -1,13 +1,12 @@ #include "buckyball.h" #include #include -#include #include enum { SOURCE_HEIGHT = 5, SOURCE_WIDTH = 7, - PANEL_BYTES = (BANK_WIDTH / 8), + PANEL_BYTES = (THREAD_MEM_BANK_WIDTH / 8), CHANNELS = (PANEL_BYTES + PANEL_BYTES / 2), CORE_HEIGHT = 4, CORE_WIDTH = 3, diff --git a/bb-tests/workloads/src/CTest/base/mvin_mvout_acc_test.c b/bb-tests/workloads/src/CTest/base/mvin_mvout_acc_test.c index 39aab8ea0..08124ee3a 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_mvout_acc_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_mvout_acc_test.c @@ -1,13 +1,12 @@ #include "buckyball.h" #include #include -#include #include #include #include // One ACC row spans four configured physical bank rows. -enum { ROWS = 16, COLS = (4 * BANK_WIDTH / 32) }; +enum { ROWS = 16, COLS = (4 * THREAD_MEM_BANK_WIDTH / 32) }; static result_t output_matrix[ROWS * COLS] __attribute__((aligned(64))); static result_t expected_matrix[ROWS * COLS] __attribute__((aligned(64))); diff --git a/bb-tests/workloads/src/CTest/base/mvin_mvout_group_test.c b/bb-tests/workloads/src/CTest/base/mvin_mvout_group_test.c index 3115e5861..c2b3116ec 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_mvout_group_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_mvout_group_test.c @@ -1,9 +1,8 @@ #include -#include #include #include #include -enum { ROW_BYTES = BANK_WIDTH / 8 }; +enum { ROW_BYTES = THREAD_MEM_BANK_WIDTH / 8 }; static uint8_t initial[6 * ROW_BYTES] __attribute__((aligned(16))); static uint8_t replacement[3 * ROW_BYTES] __attribute__((aligned(16))); static uint8_t full[6 * ROW_BYTES] __attribute__((aligned(16))); diff --git a/bb-tests/workloads/src/CTest/base/mvin_mvout_stride_test.c b/bb-tests/workloads/src/CTest/base/mvin_mvout_stride_test.c index 13d65c592..b5e899b91 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_mvout_stride_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_mvout_stride_test.c @@ -1,11 +1,15 @@ #include "buckyball.h" #include #include -#include #include #include -enum { ROW_BYTES = (BANK_WIDTH / 8), ROWS = 4, IN_STRIDE = 3, OUT_STRIDE = 5 }; +enum { + ROW_BYTES = (THREAD_MEM_BANK_WIDTH / 8), + ROWS = 4, + IN_STRIDE = 3, + OUT_STRIDE = 5 +}; static elem_t input_matrix[ROWS * IN_STRIDE * ROW_BYTES] __attribute__((aligned(128))); diff --git a/bb-tests/workloads/src/CTest/base/mvin_mvout_test.c b/bb-tests/workloads/src/CTest/base/mvin_mvout_test.c index dc0e68c55..7ddecaaf5 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_mvout_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_mvout_test.c @@ -1,12 +1,11 @@ #include "buckyball.h" #include #include -#include #include #include #include -enum { ROWS = 16, COLS = (BANK_WIDTH / 8) }; +enum { ROWS = 16, COLS = (THREAD_MEM_BANK_WIDTH / 8) }; // Test matrices static elem_t input_matrix[ROWS * COLS] __attribute__((aligned(128))); diff --git a/bb-tests/workloads/src/CTest/base/mvin_mvout_unaligned_test.c b/bb-tests/workloads/src/CTest/base/mvin_mvout_unaligned_test.c index 33cbc9d2a..c6d9c762a 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_mvout_unaligned_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_mvout_unaligned_test.c @@ -1,12 +1,16 @@ #include "buckyball.h" #include #include -#include #include #include #include -enum { ROW_BYTES = (BANK_WIDTH / 8), ROWS = 4, OFFSET = 4, GUARD = 0x5a }; +enum { + ROW_BYTES = (THREAD_MEM_BANK_WIDTH / 8), + ROWS = 4, + OFFSET = 4, + GUARD = 0x5a +}; static elem_t input_matrix[ROWS * ROW_BYTES] __attribute__((aligned(128))); static uint8_t output_bytes[OFFSET + ROWS * ROW_BYTES + ROW_BYTES] diff --git a/bb-tests/workloads/src/CTest/base/mvin_unaligned_test.c b/bb-tests/workloads/src/CTest/base/mvin_unaligned_test.c index 75dd88175..e4b966965 100644 --- a/bb-tests/workloads/src/CTest/base/mvin_unaligned_test.c +++ b/bb-tests/workloads/src/CTest/base/mvin_unaligned_test.c @@ -1,12 +1,16 @@ #include "buckyball.h" #include #include -#include #include #include #include -enum { ROW_BYTES = (BANK_WIDTH / 8), ROWS = 4, OFFSET = 4, GUARD = 0xa5 }; +enum { + ROW_BYTES = (THREAD_MEM_BANK_WIDTH / 8), + ROWS = 4, + OFFSET = 4, + GUARD = 0xa5 +}; static uint8_t input_bytes[OFFSET + ROWS * ROW_BYTES + ROW_BYTES] __attribute__((aligned(128))); diff --git a/bb-tests/workloads/src/CTest/coremark/CMakeLists.txt b/bb-tests/workloads/src/CTest/coremark/CMakeLists.txt index c57961177..663236780 100644 --- a/bb-tests/workloads/src/CTest/coremark/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/coremark/CMakeLists.txt @@ -33,7 +33,7 @@ set(COREMARK_C_FLAGS -Wl,-T,${BBSIM_LD} -I${COREMARK_SRC_DIR} -I${COREMARK_PORT_DIR} - -I${BUCKYBALL_ISA_DIR}/control + -DCORE_HART_ID=${DESIGN_WORKLOAD_CONTROL_HART} -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services -I${WORKLOAD_LIB_DIR} @@ -45,7 +45,7 @@ set(COREMARK_C_FLAGS ) set(COREMARK_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${COREMARK_THREAD}-ctest-benchmark-coremark" + "${DESIGN_NAME}-${COREMARK_THREAD}-ctest-benchmark-coremark" ) set(EXECUTABLE "${COREMARK_NAME}-baremetal") @@ -59,7 +59,6 @@ add_custom_command( ${COREMARK_HEADERS} ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c - ${BUCKYBALL_ISA_DIR}/control/params.h COMMENT "Building coremark: ${EXECUTABLE}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) diff --git a/bb-tests/workloads/src/CTest/embench/CMakeLists.txt b/bb-tests/workloads/src/CTest/embench/CMakeLists.txt index f4afb9029..0faa0d99b 100644 --- a/bb-tests/workloads/src/CTest/embench/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/embench/CMakeLists.txt @@ -34,7 +34,7 @@ set(EMBENCH_C_FLAGS -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services -I${WORKLOAD_LIB_DIR} - -I${BUCKYBALL_ISA_DIR}/control + -DCORE_HART_ID=${DESIGN_WORKLOAD_CONTROL_HART} -ffunction-sections -fdata-sections -Wl,--gc-sections @@ -51,7 +51,7 @@ set(EMBENCH_TOP_OBJ_FLAGS -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services -I${WORKLOAD_LIB_DIR} - -I${BUCKYBALL_ISA_DIR}/control + -DCORE_HART_ID=${DESIGN_WORKLOAD_CONTROL_HART} -ffunction-sections -fdata-sections -I${EMBENCH_SUPPORT_DIR} @@ -60,7 +60,7 @@ set(EMBENCH_TOP_OBJ_FLAGS function(add_embench_target BMARK_NAME) file(GLOB BMARK_SOURCES "${EMBENCH_SRC_DIR}/${BMARK_NAME}/*.c") set(EMBENCH_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${EMBENCH_THREAD}-ctest-benchmark-embench_${BMARK_NAME}" + "${DESIGN_NAME}-${EMBENCH_THREAD}-ctest-benchmark-embench_${BMARK_NAME}" ) set(EXECUTABLE "${EMBENCH_NAME}-baremetal") @@ -91,7 +91,7 @@ function(add_embench_target BMARK_NAME) OR BMARK_NAME STREQUAL "sglib-combined" ) set(CHECK_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${EMBENCH_THREAD}-ctest-check-embench_${BMARK_NAME}" + "${DESIGN_NAME}-${EMBENCH_THREAD}-ctest-check-embench_${BMARK_NAME}" ) set(CHECK_EXECUTABLE ${CHECK_NAME}-baremetal) set(CHECK_SUPPORT_SOURCES ${EMBENCH_SUPPORT_SOURCES}) @@ -176,7 +176,7 @@ function(add_embench_top_target) add_embench_top_benchmark(wikisort wikisort) set(EMBENCH_TOP_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${EMBENCH_THREAD}-ctest-benchmark-embench_top" + "${DESIGN_NAME}-${EMBENCH_THREAD}-ctest-benchmark-embench_top" ) set(EXECUTABLE "${EMBENCH_TOP_NAME}-baremetal") add_custom_command( diff --git a/bb-tests/workloads/src/CTest/rvv/rvv_kernel.c b/bb-tests/workloads/src/CTest/rvv/rvv_kernel.c index 05fc9bab2..9b50e31fa 100644 --- a/bb-tests/workloads/src/CTest/rvv/rvv_kernel.c +++ b/bb-tests/workloads/src/CTest/rvv/rvv_kernel.c @@ -1,5 +1,4 @@ #include -#include #include #include @@ -24,16 +23,16 @@ int main(void) { bb_mem_transfer(2, 4); bb_mem_transfer(0, 5); bb_mem_transfer(1, 5); - mvin_kernel(image + 1, sizeof(image) - 4, VIRTUAL_BANK_NUM); - run_kernel(4, VIRTUAL_BANK_NUM, 5, 0); + mvin_kernel(image + 1, sizeof(image) - 4, CORE_VIRTUAL_BANK_COUNT); + run_kernel(4, CORE_VIRTUAL_BANK_COUNT, 5, 0); call.args[0] = (uint64_t)4 << 32; call.args[1] = (uint64_t)1 << 32; bb_mvin_group((uintptr_t)&call, 5, 0, sizeof(call) / 16, 1); - run_kernel(4, VIRTUAL_BANK_NUM, 5, 0); + run_kernel(4, CORE_VIRTUAL_BANK_COUNT, 5, 0); uint32_t groups[2][64] __attribute__((aligned(16))); bb_mvout_group((uintptr_t)groups[0], 5, 1, 16, 1); bb_mvout_group((uintptr_t)groups[1], 5, 2, 16, 1); - release_kernel(VIRTUAL_BANK_NUM); + release_kernel(CORE_VIRTUAL_BANK_COUNT); bb_mem_release(4); bb_mem_release(5); for (unsigned row = 0; row < 16; ++row) diff --git a/bb-tests/workloads/src/CTest/tutorial/CMakeLists.txt b/bb-tests/workloads/src/CTest/tutorial/CMakeLists.txt index 3b59bb9c1..2b4259f3d 100644 --- a/bb-tests/workloads/src/CTest/tutorial/CMakeLists.txt +++ b/bb-tests/workloads/src/CTest/tutorial/CMakeLists.txt @@ -4,7 +4,7 @@ set(BUCKYBALL_CTEST_GROUP_TARGET tutorial-ctests-build) # Runs on the selected bare-metal hart. foreach(BUCKYBALL_CTEST_TARGET IN LISTS DESIGN_CPU_TARGETS) set(BUCKYBALL_TUTORIAL_TEST - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-tutorial" + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-tutorial" ) add_buckyball_baremetal_ctest( ${BUCKYBALL_TUTORIAL_TEST} diff --git a/bb-tests/workloads/src/MLIRTest/CMakeLists.txt b/bb-tests/workloads/src/MLIRTest/CMakeLists.txt index e7e7edeab..a48fffd69 100644 --- a/bb-tests/workloads/src/MLIRTest/CMakeLists.txt +++ b/bb-tests/workloads/src/MLIRTest/CMakeLists.txt @@ -31,52 +31,32 @@ function(add_buckyball_mlir_test NAME) if(NOT DEFINED BUCKYBALL_MLIR_TEST_PREFIX) message(FATAL_ERROR "BUCKYBALL_MLIR_TEST_PREFIX is not set") endif() - if(BUCKYBALL_WORKLOAD_DESIGN STREQUAL "") - message( - FATAL_ERROR - "BUCKYBALL_WORKLOAD_DESIGN must be set before mlir tests" - ) - endif() if(NOT ARG_TARGET) message(FATAL_ERROR "${NAME}: TARGET is required") endif() - list(FIND DESIGN_TARGETS "${ARG_TARGET}" TARGET_INDEX) + list(FIND DESIGN_PROFILES "${ARG_TARGET}" TARGET_INDEX) if(TARGET_INDEX EQUAL -1) message(FATAL_ERROR "${NAME}: unknown Buckyball target ${ARG_TARGET}") endif() - if(NOT ARG_TARGET IN_LIST DESIGN_BANK_TARGETS) + if(NOT DESIGN_PROFILES_${ARG_TARGET}_THREAD_MEM_BANK_NUM GREATER 0) message(FATAL_ERROR "${ARG_TARGET}: no physical banks") endif() set(ASSIGN_PHYSICAL_BANKS -assign-physical-banks) set(BIN "${DESIGN_COMPILER}/bin") # buddy-opt with this target's compiler plugins. string(JOIN " " OPT ${DESIGN_TARGET_OPT_${ARG_TARGET}}) - set(BB_BALL_ISA_INCLUDE_DIR "${BUCKYBALL_ISA_DIR}/${ARG_TARGET}") - set(BB_BALL_ISA_H "${BB_BALL_ISA_INCLUDE_DIR}/ballISA.h") - set(BB_PARAMS_H "${BB_BALL_ISA_INCLUDE_DIR}/params.h") - if(NOT EXISTS "${BB_BALL_ISA_H}") - message( - FATAL_ERROR - "${NAME}: missing target ISA header ${BB_BALL_ISA_H}" - ) - endif() - if(NOT EXISTS "${BB_PARAMS_H}") - message(FATAL_ERROR "${NAME}: missing target params.h ${BB_PARAMS_H}") - endif() + # The banks and Ball ISA of the target, as compiler flags. + set(TARGET_DEFINES ${DESIGN_TARGET_DEFS_${ARG_TARGET}}) + list(TRANSFORM TARGET_DEFINES PREPEND "-D") set(MLIR_SRC ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}.mlir) set(MLIR_TEMPLATE ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}.mlir.in) if(EXISTS ${MLIR_TEMPLATE}) - file(STRINGS ${BB_PARAMS_H} _bank_lines REGEX "^#define BANK_LINES ") - file(STRINGS ${BB_PARAMS_H} _bank_width REGEX "^#define BANK_WIDTH ") - file(STRINGS ${BB_PARAMS_H} _bank_isqrt REGEX "^#define BANK_ISQRT ") - string(REGEX REPLACE ".* " "" BB_BANK_LINES "${_bank_lines}") - string(REGEX REPLACE ".* " "" BB_BANK_WIDTH_BITS "${_bank_width}") - string(REGEX REPLACE ".* " "" BB_BANK_ISQRT "${_bank_isqrt}") + set(BB_BANK_LINES ${DESIGN_PROFILES_${ARG_TARGET}_THREAD_MEM_BANK_ENTRIES}) + set(BB_BANK_WIDTH_BITS ${DESIGN_PROFILES_${ARG_TARGET}_THREAD_MEM_BANK_WIDTH}) math(EXPR BB_BANK_I8_LANES "${BB_BANK_WIDTH_BITS} / 8") math(EXPR BB_BANK_I32_LANES "${BB_BANK_WIDTH_BITS} / 32") if( BB_BANK_LINES LESS 1 - OR BB_BANK_ISQRT LESS 1 OR BB_BANK_I8_LANES LESS 1 OR BB_BANK_I32_LANES LESS 1 ) @@ -95,7 +75,7 @@ function(add_buckyball_mlir_test NAME) set(TEST_CASE ${CMAKE_MATCH_1}) endif() set(TEST_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${ARG_TARGET}-mlirtest-${PREFIX}_${TEST_CASE}" + "${DESIGN_NAME}-${ARG_TARGET}-mlirtest-${PREFIX}_${TEST_CASE}" ) set(BAREMETAL_BIN ${TEST_NAME}-baremetal) set(LINUX_BIN ${TEST_NAME}-linux) @@ -184,7 +164,7 @@ function(add_buckyball_mlir_test NAME) ${ELF_CC} ${C_FLAGS} -I${LLVM_MLIR_EXECUTION_ENGINE_DIR} -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services - -I${WORKLOAD_LIB_DIR} -I${BB_BALL_ISA_INCLUDE_DIR} -o + -I${WORKLOAD_LIB_DIR} ${TARGET_DEFINES} -o ${CMAKE_CURRENT_BINARY_DIR}/${BAREMETAL_BIN} ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c ${CRUNNER_UTILS_SRC} ${BBHW_MEM_C} ${MAIN_SRC} ${OBJ} ${MLIR_BAREMETAL_LIBRARIES} @@ -195,8 +175,6 @@ function(add_buckyball_mlir_test NAME) ${MAIN_SRC} ${CRUNNER_UTILS_SRC} ${BBHW_MEM_C} - ${BB_BALL_ISA_H} - ${BB_PARAMS_H} ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c ${BBSIM_LD} @@ -211,7 +189,7 @@ function(add_buckyball_mlir_test NAME) ${LINUX_CXX} ${LINUX_FLAGS} -I${LLVM_MLIR_EXECUTION_ENGINE_DIR} -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services - -I${WORKLOAD_LIB_DIR} -I${BB_BALL_ISA_INCLUDE_DIR} -o + -I${WORKLOAD_LIB_DIR} ${TARGET_DEFINES} -o ${CMAKE_CURRENT_BINARY_DIR}/${LINUX_BIN} ${CRUNNER_UTILS_SRC} -x c ${BBHW_MEM_C} -x none ${MAIN_SRC} ${OBJ} ${MLIR_LINK_LIBRARIES} DEPENDS @@ -220,8 +198,6 @@ function(add_buckyball_mlir_test NAME) ${MAIN_SRC} ${CRUNNER_UTILS_SRC} ${BBHW_MEM_C} - ${BB_BALL_ISA_H} - ${BB_PARAMS_H} COMMENT "Linking ${LINUX_BIN}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} VERBATIM diff --git a/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt b/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt index 3ec3ceb77..f47c8b940 100644 --- a/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt +++ b/bb-tests/workloads/src/MLIRTest/base/CMakeLists.txt @@ -4,7 +4,7 @@ set(BUCKYBALL_MLIR_GROUP_TARGET base-mlirtest-build) set(BUCKYBALL_MLIR_TEST_PREFIX base) foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) - if(NOT BUCKYBALL_TARGET IN_LIST DESIGN_BANK_TARGETS) + if(NOT DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_MEM_BANK_NUM GREATER 0) continue() endif() add_buckyball_mlir_test(base_memref_copy_dynamic TARGET ${BUCKYBALL_TARGET}) diff --git a/bb-tests/workloads/src/SocTest/CMakeLists.txt b/bb-tests/workloads/src/SocTest/CMakeLists.txt index 95b1b97a1..80dd35e77 100644 --- a/bb-tests/workloads/src/SocTest/CMakeLists.txt +++ b/bb-tests/workloads/src/SocTest/CMakeLists.txt @@ -24,7 +24,7 @@ add_custom_target(soctests-build) function(add_buckyball_soctest SOURCE_FILE) get_filename_component(_stem ${SOURCE_FILE} NAME_WE) set(TEST_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_SOCTEST_TARGET}-soctest-${_stem}" + "${DESIGN_NAME}-${BUCKYBALL_SOCTEST_TARGET}-soctest-${_stem}" ) set(SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_FILE}) file(STRINGS ${SOURCE} _lines) @@ -37,23 +37,22 @@ function(add_buckyball_soctest SOURCE_FILE) endif() set(EXECUTABLE "${TEST_NAME}-baremetal") - set(PARAMS ${BUCKYBALL_ISA_DIR}/control/params.h) add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${EXECUTABLE} COMMAND ${ELF_CC} ${BUCKYBALL_SOCTEST_C_FLAGS} -o ${EXECUTABLE} ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c ${SOURCE} - -I${BUCKYBALL_ISA_DIR}/control -I${BUCKYBALL_ISA_DIR}/design + -DCORE_HART_ID=${DESIGN_WORKLOAD_CONTROL_HART} ${DESIGN_DEFINES} -I${CMAKE_CURRENT_SOURCE_DIR} DEPENDS ${SOURCE} - ${PARAMS} ${BAREMETAL_DIR}/crt0.S ${BBSIM_LD} ${BAREMETAL_DIR}/syscalls.c ${CMAKE_CURRENT_SOURCE_DIR}/soc.h COMMENT "Building SoC test: ${EXECUTABLE}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM ) add_custom_target( ${TEST_NAME}-baremetal-build diff --git a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py index 8233dc2ef..0190903bd 100644 --- a/bbdev/api/steps/bebop/bemu/02_sim_event.step.py +++ b/bbdev/api/steps/bebop/bemu/02_sim_event.step.py @@ -143,14 +143,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: run_args.append("--disasm") if input_data.get("tool-profile"): run_args.append("--tool-profile") - for trace_name in ("itrace", "mtrace"): + for trace_name in ("itrace", "mtrace", "ctrace"): if input_data.get(trace_name, False): run_args.append(f"--{trace_name}") inner_cmd = f"cd {shlex.quote(binary_dir)} && {shlex.join(run_args)}" run_cmd = f"nix develop -c sh -c {shlex.quote(inner_cmd)}" run_cmd = f"set -o pipefail; {run_cmd} 2>&1 | tee {shlex.quote(str(Path(run_log) / 'run.log'))}" ctx.logger.info(f"Running bebop bemu: {run_cmd}") - report_context = source_context(bbdir, design) + report_context = source_context(bbdir, design, input_data["target"]) run_result = await stream_run_logger_async( cmd=run_cmd, logger=ctx.logger, diff --git a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py index fe889b77a..16c74f537 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_api.step.py @@ -31,11 +31,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: require_design({"design": design}) except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) - junit = body.get("junit") - if junit is not None and (not isinstance(junit, str) or not os.path.isabs(junit)): - return ApiResponse( - status=400, body={"error": "junit must be an absolute file path"} - ) test_type = body.get("test") if not test_type: return ApiResponse( @@ -57,11 +52,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "data": { "design": design, "test": test_type, - "junit": junit, "jobs": body.get("jobs", 1), - "clean-before": body.get( - "clean-before", body.get("clean_before", False) - ), "_trace_id": ctx.trace_id, "target": selection, }, diff --git a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py index 0e6854ac1..459323f8c 100644 --- a/bbdev/api/steps/bebop/bemu/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/bemu/03_batch_event.step.py @@ -6,8 +6,9 @@ 2. Run the BEMU regression harness """ +from datetime import datetime from pathlib import Path -from utils.path import design_output_root +from utils.path import design_output_root, log_dir import os @@ -121,19 +122,26 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: trace_id=origin_tid, ) return + run_log = log_dir( + bbdir, + design, + "verilog", + datetime.now().strftime("%Y-%m-%d-%H-%M"), + "bebop-bemu", + test_type, + selection=input_data["target"], + ) harness_args = [ "--", "--workload-toml", workload_toml, "--bb-tests-root", elf_root, + "--log-dir", + run_log, "--jobs", str(input_data.get("jobs", 1)), ] - if input_data.get("clean-before", input_data.get("clean_before", False)): - harness_args.append("--clean-before") - if input_data.get("junit"): - harness_args.extend(["--junit", input_data["junit"]]) harness = shlex.join(harness_args) test_cmd = ( "nix develop -c cargo test --manifest-path" @@ -159,6 +167,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "design": design, "test_type": test_type, "workload_toml": workload_toml, + "log_dir": run_log, + "junit": os.path.join(run_log, "junit.xml"), }, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py b/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py index 2b59f727c..15e36d83c 100644 --- a/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py +++ b/bbdev/api/steps/bebop/bemu/04_analysis_api.step.py @@ -50,7 +50,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: if not design: return _fail("design parameter is required") try: - design_maps(get_buckyball_path(), design) + design_maps(get_buckyball_path(), design, selection) except ValueError as e: return _fail(str(e)) diff --git a/bbdev/api/steps/bebop/bemu/04_analysis_event.step.py b/bbdev/api/steps/bebop/bemu/04_analysis_event.step.py index 70aa8377c..82035624a 100644 --- a/bbdev/api/steps/bebop/bemu/04_analysis_event.step.py +++ b/bbdev/api/steps/bebop/bemu/04_analysis_event.step.py @@ -53,7 +53,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: itrace = bool(input_data.get("itrace", False)) mtrace = bool(input_data.get("mtrace", False)) try: - names, matrix, depth = design_maps(get_buckyball_path(), design) + names, matrix, depth = design_maps( + get_buckyball_path(), design, input_data["target"] + ) text = analysis_dir( log_path, names=names, diff --git a/bbdev/api/steps/bebop/bemu/scripts/bank_replay.py b/bbdev/api/steps/bebop/bemu/scripts/bank_replay.py index c1d479c3f..6ba6add41 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bank_replay.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bank_replay.py @@ -29,7 +29,7 @@ def __init__(self, thread, virtual_banks, calibration): self.depth = thread.frontend.rob_entries self.alias_base = virtual_banks + (2 if thread.gpdomain.enable else 0) self.bids = { - i.funct7: ball.ball_id + i.funct7: ball.id for ball in thread.balldomain.balls for i in ball.instructions } diff --git a/bbdev/api/steps/bebop/bemu/scripts/bank_replay_native.py b/bbdev/api/steps/bebop/bemu/scripts/bank_replay_native.py index a30c79baa..ca667cd57 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bank_replay_native.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bank_replay_native.py @@ -370,7 +370,7 @@ def simulate( coefficients = np.zeros((128, 4)) for ball in thread.balldomain.balls: for isa in ball.instructions: - bids[isa.funct7] = ball.ball_id + bids[isa.funct7] = ball.id for f, row in calibration.items(): if "model" in row: assert ( diff --git a/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py index 96244d7ce..d77c783c8 100644 --- a/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py +++ b/bbdev/api/steps/bebop/bemu/scripts/bemu_analysis.py @@ -19,7 +19,9 @@ def _hex_u(value: object, field: str, line_no: int) -> int: return int(value, 16) -def design_maps(bbdir: str, design: str) -> tuple[dict[int, str], set[int], int]: +def design_maps( + bbdir: str, design: str, selection: str +) -> tuple[dict[int, str], set[int], int]: system_isa = { 0: "fence", 1: "barrier", @@ -31,12 +33,12 @@ def design_maps(bbdir: str, design: str) -> tuple[dict[int, str], set[int], int] } from utils.path import load_design_config - data = load_design_config(bbdir, design) + data = load_design_config(bbdir, design, selection=selection) configurations = { ( - p.config.balldomain.SerializeToString(deterministic=True), - p.config.mem.bank.entries, - ): p.config + p.thread.balldomain.SerializeToString(deterministic=True), + p.thread.mem.bank.entries, + ): p.thread for p in data.profiles } if len(configurations) != 1: @@ -176,6 +178,7 @@ def write_report(log_dir: Path, text: str) -> Path: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--design", required=True) + parser.add_argument("--target", required=True) parser.add_argument("--run", required=True) parser.add_argument("--bbdir", type=Path) parser.add_argument("--itrace", action="store_true") @@ -190,7 +193,7 @@ def main(argv: list[str] | None = None) -> int: bbdir = args.bbdir if args.bbdir is not None else Path(get_buckyball_path()) log_dir = run_log_dir(bbdir, args.run) - names, matrix, depth = design_maps(str(bbdir), args.design) + names, matrix, depth = design_maps(str(bbdir), args.design, args.target) text = analysis_dir( log_dir, names=names, diff --git a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py index 276e191f3..0bb0d3a85 100644 --- a/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py +++ b/bbdev/api/steps/bebop/p2e/03_runworkload_event.step.py @@ -138,7 +138,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return wave = True try: - image_path = resolve_image(bbdir, image_name, design) + image_path = resolve_image(bbdir, image_name, design, input_data["target"]) image_base = os.path.splitext(image_path)[0] elf_path = ( f"{image_base}.elf" if os.path.isfile(f"{image_base}.elf") else image_base @@ -309,7 +309,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ] run_args += ["--image", image_path] run_cmd = f"cd {shlex.quote(bbdir)} && {shlex.join(run_args)}" - report_context = source_context(bbdir, design) + report_context = source_context(bbdir, design, input_data["target"]) with open(bitstream, "rb") as source: report_context["provenance"]["bitstream_sha256"] = hashlib.file_digest( source, "sha256" diff --git a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py index d10027b7a..6a8dd470b 100644 --- a/bbdev/api/steps/bebop/p2e/04_batch_event.step.py +++ b/bbdev/api/steps/bebop/p2e/04_batch_event.step.py @@ -8,8 +8,9 @@ """ from utils.path import runner_manifest +from datetime import datetime from pathlib import Path -from utils.path import design_output_root +from utils.path import design_output_root, log_dir import os @@ -176,6 +177,16 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: bitstream, "--p2e-fpga-location", input_data.get("fpga-location", "0.A"), + "--log-dir", + log_dir( + bbdir, + design, + "p2e", + datetime.now().strftime("%Y-%m-%d-%H-%M"), + "bebop-p2e", + test_type, + selection=input_data["target"], + ), ] if diff: harness_args.append("--diff") diff --git a/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py b/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py index ce47d7287..161e4f3b3 100644 --- a/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py +++ b/bbdev/api/steps/bebop/p2e/scripts/resolve_image.py @@ -1,34 +1,25 @@ import glob import os +from utils.path import design_output_root -def resolve_image(bbdir: str, image_name: str, design: str) -> str: + +def resolve_image(bbdir: str, image_name: str, design: str, selection: str) -> str: + """The hex image of a workload or kernel built for the simulation entry.""" image_name = image_name.replace(r"\_", "_") if os.path.isabs(image_name): raise ValueError( f"image must be a logical name, not an absolute path: {image_name!r}" ) - matches = glob.glob( - os.path.join( - bbdir, - "bb-tests", - "output", - design, - "workloads", - "**", - f"{image_name}.hex", - ), - recursive=True, + workloads = os.path.join( + design_output_root(bbdir, design, selection=selection), "workloads" ) - kernel_image = os.path.join( - bbdir, "stack", "output", "kernel", design, f"{image_name}.hex" + matches = glob.glob( + os.path.join(workloads, "**", f"{image_name}.hex"), recursive=True ) - if os.path.isfile(kernel_image): - matches.append(kernel_image) if len(matches) != 1: raise ValueError( - f"expected one image for {image_name!r} under " - f"bb-tests/output/{design}/workloads/ or stack/output/kernel/{design}/, " + f"expected one image for {image_name!r} under {workloads}, " f"found {len(matches)}" ) return matches[0] diff --git a/bbdev/api/steps/bebop/vcs/03_batch_api.step.py b/bbdev/api/steps/bebop/vcs/03_batch_api.step.py index aac5c160e..01fc0a6ea 100644 --- a/bbdev/api/steps/bebop/vcs/03_batch_api.step.py +++ b/bbdev/api/steps/bebop/vcs/03_batch_api.step.py @@ -1,5 +1,4 @@ from utils.event_common import require_target -import os from motia import ApiRequest, ApiResponse, FlowContext, api @@ -19,18 +18,12 @@ async def handler(req: ApiRequest, ctx: FlowContext) -> ApiResponse: selection = require_target(body, ("device",)) except ValueError as error: return ApiResponse(status=400, body={"error": str(error)}) - junit = body.get("junit") - if junit is not None and (not isinstance(junit, str) or not os.path.isabs(junit)): - return ApiResponse( - status=400, body={"error": "junit must be an absolute file path"} - ) await ctx.enqueue( { "topic": "bebop.vcs.batch", "data": { "design": require_design(body), "test": body.get("test", "bare-tests"), - "junit": junit, "jobs": body.get("jobs", 1), "_trace_id": ctx.trace_id, "target": selection, diff --git a/bbdev/api/steps/bebop/vcs/03_batch_event.step.py b/bbdev/api/steps/bebop/vcs/03_batch_event.step.py index 786663635..2199e6697 100644 --- a/bbdev/api/steps/bebop/vcs/03_batch_event.step.py +++ b/bbdev/api/steps/bebop/vcs/03_batch_event.step.py @@ -63,8 +63,6 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "--jobs", str(input_data.get("jobs", 1)), ] - if input_data.get("junit"): - command.extend(["--junit", input_data["junit"]]) result = await stream_run_logger_async( cmd=shlex.join(command), logger=ctx.logger, @@ -79,7 +77,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "task": "batch", "backend": "vcs", "log_dir": str(report), - "junit": input_data.get("junit"), + "junit": str(report / "junit.xml"), }, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py index 2344e2fb2..af39185af 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_api.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_api.step.py @@ -1,5 +1,4 @@ from utils.event_common import require_target -import os from motia import ApiRequest, ApiResponse, FlowContext, api from utils.event_common import require_design @@ -23,11 +22,6 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: except ValueError as e: return ApiResponse(status=400, body={"error": str(e)}) diff = bool(body.get("diff", False)) - junit = body.get("junit") - if junit is not None and (not isinstance(junit, str) or not os.path.isabs(junit)): - return ApiResponse( - status=400, body={"error": "junit must be an absolute file path"} - ) test_type = body.get("test") if not test_type: return ApiResponse( @@ -46,9 +40,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: data = { "design": design, "test": test_type, - "junit": junit, "jobs": body.get("jobs", 1), - "clean-before": body.get("clean-before", body.get("clean_before", False)), "diff": diff, } await ctx.enqueue( diff --git a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py index b7cfd3d7e..1e2524e70 100644 --- a/bbdev/api/steps/bebop/verilator/06_batch_event.step.py +++ b/bbdev/api/steps/bebop/verilator/06_batch_event.step.py @@ -4,8 +4,9 @@ Runs bebop verilator batch regression (requires prior --build). """ +from datetime import datetime from pathlib import Path -from utils.path import design_output_root +from utils.path import design_output_root, log_dir from utils.path import runner_manifest @@ -94,6 +95,15 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) manifest = str(runner_manifest(bbdir, design, input_data["target"])) features = "verilator,bemu" if diff else "verilator" + run_log = log_dir( + bbdir, + design, + "verilog", + datetime.now().strftime("%Y-%m-%d-%H-%M"), + "bebop-verilator", + test_type, + selection=input_data["target"], + ) harness_args = [ "--", "--workload-toml", @@ -102,15 +112,13 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: elf_root, "--arch-config", design, + "--log-dir", + run_log, "--jobs", str(input_data.get("jobs", 1)), ] - if input_data.get("clean-before", input_data.get("clean_before", False)): - harness_args.append("--clean-before") if diff: harness_args.append("--diff") - if input_data.get("junit"): - harness_args.extend(["--junit", input_data["junit"]]) harness = shlex.join(harness_args) test_cmd = ( "nix develop -c cargo test --release --target-dir" @@ -138,6 +146,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: "test_type": test_type, "diff": diff, "workload_toml": workload_toml, + "log_dir": run_log, + "junit": os.path.join(run_log, "junit.xml"), }, trace_id=origin_tid, ) diff --git a/bbdev/api/steps/config/01_install_api.step.py b/bbdev/api/steps/config/01_install_api.step.py index 6f5f8fb03..98b7ab7ff 100644 --- a/bbdev/api/steps/config/01_install_api.step.py +++ b/bbdev/api/steps/config/01_install_api.step.py @@ -1,4 +1,3 @@ -from utils.event_common import require_target from motia import ApiRequest, ApiResponse, FlowContext, api config = { @@ -12,12 +11,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - if "design" in body: - try: - require_target(body) - except ValueError as error: - return ApiResponse(status=400, body={"error": str(error)}) - unknown = set(body) - {"design", "device", "core"} + unknown = set(body) - {"design"} if unknown: return ApiResponse( status=400, body={"error": f"Unexpected parameters: {sorted(unknown)}"} diff --git a/bbdev/api/steps/config/01_install_event.step.py b/bbdev/api/steps/config/01_install_event.step.py index 0fbe3905d..277217822 100644 --- a/bbdev/api/steps/config/01_install_event.step.py +++ b/bbdev/api/steps/config/01_install_event.step.py @@ -56,22 +56,35 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: toml2json = _load_script("1_toml2json.py") derive = _load_script("2_parameter_derivation.py") - # step 1: generate design config JSONs from toml files + # step 1: expand each design, then apply the overrides of each of its simulations json_outs: list[Path] = [] + designs_of: list[str] = [] for top in tops: data = toml2json.design2json(top) - out = top.parent.parent / "output" / "generated" / "config" / "config.json" - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text( - json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) - json_outs.append(out) - ctx.logger.info(f"Wrote design config JSON: {out}") - - # step 2: derive computed parameters from config JSONs + for entry, name, selection in toml2json.simulations(data): + out = ( + top.parent.parent + / "output/sims" + / selection + / "generated/config/config.json" + ) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text( + json.dumps( + toml2json.simulation2json(data, entry, name, selection), + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + json_outs.append(out) + designs_of.append(top.parent.parent.name) + ctx.logger.info(f"Wrote simulation config JSON: {out}") + + # step 2: derive computed parameters from each simulation config derived_outs: list[Path] = [] - for top, json_path in zip(tops, json_outs): - design = top.parent.parent.name + for design, json_path in zip(designs_of, json_outs): data = json.loads(json_path.read_text(encoding="utf-8")) derived_path = json_path.parent / "derived.json" derive.write_derived(data, bbdir, design, derived_path) @@ -84,12 +97,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: schemas = [ scripts / "proto" / "design.proto", scripts / "proto" / "hardware.proto", - *sorted( - Path(bbdir).glob("examples/components/balls/*/configs/*_params.proto") - ), - *sorted( - Path(bbdir).glob("examples/components/cores/*/configs/*_params.proto") - ), + *sorted(Path(bbdir).glob("examples/prefab/balls/*/configs/*_params.proto")), + *sorted(Path(bbdir).glob("examples/prefab/cores/*/configs/*_params.proto")), ] includes = sorted({schema.parent for schema in schemas}) cmd = " ".join( @@ -117,14 +126,14 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: destination.write_bytes(content) ctx.logger.info(f"Generated design python bindings: {scripts / 'design_pb2.py'}") - # step 3.2: design.pb, the one input of every build system + # step 3.2: one design.pb per simulation, the one input of every build system json2proto = _load_script("3_json2proto.py") pb_outs: list[Path] = [] for json_path, derived_path in zip(json_outs, derived_outs): out = json_path.parent.parent / "design.pb" json2proto.write_design_pb(json_path, derived_path, out, bbdir) pb_outs.append(out) - ctx.logger.info(f"Wrote design protobuf: {out}") + ctx.logger.info(f"Wrote simulation protobuf: {out}") await check_result( ctx, diff --git a/bbdev/api/steps/config/scripts/1_toml2json.py b/bbdev/api/steps/config/scripts/1_toml2json.py index 5ee24f0dc..ec014c380 100644 --- a/bbdev/api/steps/config/scripts/1_toml2json.py +++ b/bbdev/api/steps/config/scripts/1_toml2json.py @@ -4,6 +4,7 @@ from __future__ import annotations import json +import re import sys import tomllib from pathlib import Path @@ -80,6 +81,7 @@ def _walk_file( path: Path, includes: list[str], stack: list[str], + override: bool = False, ) -> dict[str, Any]: path = path.resolve() key = str(path) @@ -89,7 +91,7 @@ def _walk_file( includes.append(key) data = _load(path) stack.append(key) - out = _walk(data, path.parent, includes, stack) + out = _walk(data, path.parent, includes, stack, override=override) stack.pop() if not isinstance(out, dict): _die(f"{path}: root must be a table") @@ -105,6 +107,7 @@ def _walk( includes: list[str], stack: list[str], slot: str | None = None, + override: bool = False, ) -> Any: if isinstance(obj, dict): inc = obj.get("include") @@ -113,9 +116,13 @@ def _walk( _die(f"{base}: include must be a non-empty string") loaded = _walk_file(_reference(base, inc, slot), includes, stack) rest = {k: v for k, v in obj.items() if k != "include"} - walked = _walk(rest, base, includes, stack) + walked = _walk(rest, base, includes, stack, override=override) if not isinstance(walked, dict): _die(f"{base}: include siblings must be a table") + if override: + # Inside a device config, include siblings replace included keys. + _override(loaded, walked, str(base / inc)) + return loaded return _merge(loaded, walked, base / inc) result = {} for key, value in obj.items(): @@ -130,23 +137,26 @@ def _walk( _die(f"{base}: invalid Ball component {component!r}") path = ( Path(__file__).resolve().parents[5] - / "examples/components/balls" + / "examples/prefab/balls" / component / "configs/default.toml" ) result[key] = _walk_file(path, includes, stack) else: - result[key] = _walk(value, base, includes, stack, key) + # A device config holds overrides of the design config. + result[key] = _walk( + value, base, includes, stack, key, override or key == "configs" + ) return result if isinstance(obj, list): - return [_walk(x, base, includes, stack, slot) for x in obj] + return [_walk(x, base, includes, stack, slot, override) for x in obj] if isinstance(obj, str) and obj == "default" and slot in CONFIG_REFERENCES: return _walk_file(_reference(base, obj, slot), includes, stack) if isinstance(obj, str) and obj.endswith(".toml"): path = (base / obj).resolve() if not path.is_file(): _die(f"toml path does not exist: {path} (from {base} + {obj!r})") - return _walk_file(path, includes, stack) + return _walk_file(path, includes, stack, override) return obj @@ -190,10 +200,64 @@ def design2json(top: Path) -> dict[str, Any]: return data +def _override(config: dict, override: dict, path: str) -> None: + for key, value in override.items(): + if key == "_file": + continue + if key not in config: + _die(f"{path}: override key {key!r} is not in the design config") + if isinstance(config[key], dict) and isinstance(value, dict): + _override(config[key], value, f"{path}.{key}") + else: + config[key] = value + + +def simulations(data: dict[str, Any]) -> list[tuple[str, str | None, str]]: + """(entry, config, selection) of every simulation of the design: one per + config registered on the device entry, one per core entry.""" + design = Path(data["_file"]).parent.parent.name + prefix = "".join(part.capitalize() for part in re.split("[-_]", design)) + result = [] + for entry, registration in data["sims"]["targets"].items(): + if entry == "_file": + continue + if entry != "device": + if "configs" in registration: + _die(f"targets.{entry}: only the device entry registers configs") + result.append((entry, None, entry)) + continue + if not registration["configs"]: + _die("targets.device.configs registers no config") + for name in registration["configs"]: + if not re.fullmatch(prefix + "[A-Za-z0-9]*Config", name): + _die(f"targets.device.configs: {name!r} must be {prefix}Config") + result.append((entry, name, f"device/{name}")) + return result + + +def simulation2json( + data: dict[str, Any], entry: str, name: str | None, selection: str +) -> dict[str, Any]: + """The config of one simulation: the design config with the tables of device + config `name` replacing the same keys (arrays whole).""" + config = json.loads(json.dumps(data)) + registration = config["sims"]["targets"][entry] + if name is not None: + overrides = registration.pop("configs")[name] + _override(config, overrides, overrides["_file"]) + config["sims"]["targets"] = {entry: registration} + config["sims"]["selection"] = selection + return config + + def main() -> None: - if len(sys.argv) != 2: - _die(f"usage: {sys.argv[0]} DESIGN.toml") - out = design2json(Path(sys.argv[1])) + if len(sys.argv) != 3: + _die(f"usage: {sys.argv[0]} DESIGN.toml SELECTION") + data = design2json(Path(sys.argv[1])) + found = [s for s in simulations(data) if s[2] == sys.argv[2]] + if not found: + _die(f"{sys.argv[1]}: no simulation {sys.argv[2]!r}") + out = simulation2json(data, *found[0]) sys.stdout.write(json.dumps(out, indent=2, sort_keys=True) + "\n") diff --git a/bbdev/api/steps/config/scripts/2_parameter_derivation.py b/bbdev/api/steps/config/scripts/2_parameter_derivation.py index b39c6c176..c6dbda817 100644 --- a/bbdev/api/steps/config/scripts/2_parameter_derivation.py +++ b/bbdev/api/steps/config/scripts/2_parameter_derivation.py @@ -134,7 +134,7 @@ def _derive_threads(repo: Path, topo: dict[str, Any]) -> list[dict[str, Any]]: if not isinstance(bd, dict): _die(f"{pkg}: missing balldomain") entry["balls"] = [ - {"ball_id": ball["id"], "ball_dir": ball["component"]} + {"id": ball["id"], "component": ball["component"]} for ball in bd["balls"] ] out.append(entry) @@ -248,7 +248,7 @@ def _bemu_balls(repo: Path, profiles: list[dict[str, Any]]) -> list[dict[str, st emu_lib = ( repo / "examples" - / "components" + / "prefab" / "balls" / ball_dir / "emu" @@ -259,8 +259,8 @@ def _bemu_balls(repo: Path, profiles: list[dict[str, Any]]) -> list[dict[str, st _die(f"missing BEMU ball source for {ball_class}: {emu_lib}") balls.append( { - "ball_class": ball_class, - "ball_dir": ball_dir, + "emu_class": ball_class, + "component": ball_dir, "emu_lib": _repo_rel(repo, emu_lib), } ) @@ -276,7 +276,7 @@ def _ball_ctest_dirs(repo: Path, thread: dict[str, Any]) -> list[str]: path = ( repo / "examples" - / "components" + / "prefab" / "balls" / ball["component"] / "workloads" @@ -319,11 +319,8 @@ def _derive_targets( targets = {} for profile in profiles: name, pkg, config = profile["name"], profile["component"], profile["config"] - num, width, entries = bank_params(config, pkg) + bank_params(config, pkg) targets[name] = { - "pkg": pkg, - "compiler": "compiler", - "bank": {"num": num, "width": width, "entries": entries}, "ball_ctest_dirs": _ball_ctest_dirs(repo, config), } return targets @@ -335,15 +332,6 @@ def _derive_harts( return [{"hart_id": core["hart_id"]} for core in cores] -def _workload( - design: str, targets: dict[str, dict[str, Any]], repo: Path -) -> dict[str, Any]: - defs: dict[str, str] = { - "BUCKYBALL_WORKLOAD_DESIGN": design, - } - return {"cmake_param": defs} - - def derive(data: dict[str, Any], repo: Path, design: str) -> dict[str, Any]: repo = repo.resolve() topo = data.get("device") @@ -373,7 +361,7 @@ def derive(data: dict[str, Any], repo: Path, design: str) -> dict[str, Any]: return { "design": design, "name": Path(root_file).parent.parent.name, - "design_path": _repo_rel(repo, root_file), + "path": _repo_rel(repo, root_file), "core_config_path": _repo_rel(repo, design_file), "includes": [_repo_rel(repo, p) for p in includes_raw], "n_cores": n_cores, @@ -384,7 +372,6 @@ def derive(data: dict[str, Any], repo: Path, design: str) -> dict[str, Any]: "bemu": { "balls": _bemu_balls(repo, profiles), }, - "workload": _workload(design, targets, repo), } diff --git a/bbdev/api/steps/config/scripts/3_json2proto.py b/bbdev/api/steps/config/scripts/3_json2proto.py index e9f8372a0..bfce53628 100644 --- a/bbdev/api/steps/config/scripts/3_json2proto.py +++ b/bbdev/api/steps/config/scripts/3_json2proto.py @@ -65,17 +65,17 @@ def _fill_mem(msg: pb.MemDomainConfig, d: dict[str, Any], bbdir: Path) -> None: msg.mmio.bank_entries = mmio["bankEntries"] msg.mmio.bank_width = mmio["bankWidth"] msg.mmio.read_width = mmio["readWidth"] - msg.mem.addr_len = d["mem"]["addrLen"] + msg.addr_len = d["addrLen"] def _fill_ball(msg: pb.BallDomain, d: dict[str, Any], bbdir: Path) -> None: for ball in d["balls"]: instance = msg.balls.add() - instance.ball_id = ball["id"] - instance.ball_name = ball["name"] - instance.ball_dir = ball["component"] + instance.id = ball["id"] + instance.name = ball["name"] + instance.component = ball["component"] classes = [] - root = bbdir / "examples/components/balls" / ball["component"] / "arch" + root = bbdir / "examples/prefab/balls" / ball["component"] / "arch" for source in sorted(root.rglob("*Ball.scala")): text = source.read_text() names = re.findall( @@ -118,8 +118,8 @@ def _fill_frontend(msg: pb.FrontendConfig, d: dict[str, Any], bbdir: Path) -> No msg.rob_entries = d["robEntries"] msg.rs_out_of_order_response = d["rsOutOfOrderResponse"] msg.bank_id_len = d["bankIdLen"] - msg.vbank_id_upper_bound = d["vbankIdUpperBound"] - msg.shared_bank_id_base = d["sharedBankIdBase"] + msg.max_private_vbank = d["maxPrivateVbank"] + msg.shared_vbank_base = d["sharedVbankBase"] msg.iter_len = d["iterLen"] msg.sub_rob_enable = d["subRobEnable"] msg.sub_rob_depth = d["subRobDepth"] @@ -258,7 +258,7 @@ def fill_design( """Serialize explicitly declared instances and their typed parameters into design.pb.""" b = pb.Design() b.name = derived["name"] - b.design_path = derived["design_path"] + b.path = derived["path"] b.n_cores = derived["n_cores"] b.includes.extend(derived["includes"]) @@ -289,36 +289,36 @@ def fill_design( if not b.device.IsInitialized() or not b.sims.IsInitialized(): raise ValueError("Device and simulation parameters must be complete") - root = Path(derived["design_path"]).parent.parent - for name, registration in config["sims"]["targets"].items(): - if name == "_file": - continue - if name != "device" and not re.fullmatch(r"core/[A-Za-z0-9_-]+", name): - raise ValueError(f"invalid simulation target: {name}") - if name.startswith("core/") and "regression" in registration: - raise ValueError(f"{name}: regression belongs to the device entry") - target = b.simulations[name] - target.directory = str(root / "sims" / name) - for field in ("emu", "stack", "workloads"): - setattr(target, field, str(root / "sims" / name / field)) - target.arch.directory = str(root / "sims" / name / "arch") - target.arch.module = "simulation" - target.arch.config = str(root / "output/generated/design.pb") - target.entry_class = registration["config"] - target.top = "BBSimHarness" - target.stack_name = registration["stack_name"] - manifest = bbdir / target.emu / "Cargo.toml" - target.emu_package = tomllib.loads(manifest.read_text())["package"]["name"] - for resource in ("regression", "kernel", "tapeout"): - if resource in registration: - target.resources[resource] = str(root / registration[resource]) - for entry in ( - Path(target.arch.directory) / "build.sc", - Path(target.stack) / "CMakeLists.txt", - Path(target.workloads) / "CMakeLists.txt", - ): - if not (bbdir / entry).is_file(): - raise FileNotFoundError(bbdir / entry) + root = Path(derived["path"]).parent.parent + ((name, registration),) = config["sims"]["targets"].items() + if not re.fullmatch(r"(core/)?[A-Za-z0-9_-]+", name) or name == "core": + raise ValueError(f"invalid simulation target: {name}") + if name.startswith("core/") and "regression" in registration: + raise ValueError(f"{name}: regression belongs to the device entry") + target = b.simulation + target.selection = config["sims"]["selection"] + target.output = str(root / "output/sims" / target.selection) + target.directory = str(root / "sims" / name) + for field in ("emu", "stack", "workloads"): + setattr(target, field, str(root / "sims" / name / field)) + target.arch.directory = str(root / "sims" / name / "arch") + target.arch.module = "simulation" + target.arch.config = str(Path(target.output) / "generated/design.pb") + target.entry_class = registration["config"] + target.top = "BBSimHarness" + target.stack_name = registration["stack_name"] + manifest = bbdir / target.emu / "Cargo.toml" + target.emu_package = tomllib.loads(manifest.read_text())["package"]["name"] + for resource in ("regression", "kernel", "tapeout"): + if resource in registration: + target.resources[resource] = str(root / registration[resource]) + for entry in ( + Path(target.arch.directory) / "build.sc", + Path(target.stack) / "CMakeLists.txt", + Path(target.workloads) / "CMakeLists.txt", + ): + if not (bbdir / entry).is_file(): + raise FileNotFoundError(bbdir / entry) from verification import ip_registrations profiles = ip_registrations( @@ -355,7 +355,7 @@ def fill_design( ] domains.extend(thread.balldomain for thread in b.threads) b.verification.balls.extend( - dict.fromkeys(ball.ball_dir for domain in domains for ball in domain.balls) + dict.fromkeys(ball.component for domain in domains for ball in domain.balls) ) targets = derived["targets"] @@ -376,7 +376,7 @@ def fill_design( instance = pb.ThreadInstance() _fill_thread(instance, raw, meta, bbdir, hardware=False) if name in compiled: - if compiled[name].config.signature != instance.signature: + if compiled[name].thread.signature != instance.signature: raise ValueError( f"compiler {name}: instances have different parameters; declare distinct compiler names" ) @@ -384,37 +384,24 @@ def fill_design( p = b.profiles.add() compiled[name] = p p.name = name - p.pkg = profile["component"] - p.compiler = "compiler" - bank = targets[name]["bank"] - p.bank_num, p.bank_width, p.bank_entries = ( - bank["num"], - bank["width"], - bank["entries"], - ) p.ball_ctest_dirs.extend(targets[name]["ball_ctest_dirs"]) - p.config.CopyFrom(instance) - - for name, selected in config["workloads"].get("profiles", {}).items(): - if name == "_file": - continue - if ( - type(selected) is not int - or selected < 0 - or not any( - c.hart_id == selected and c.profile == name for c in b.device.cores + p.thread.CopyFrom(instance) + # Programs of the profile run on the first core that runs it, on its + # accelerator or on one of its threads. + p.core.CopyFrom( + next( + core + for core in b.device.cores + if core.profile == name + or any(b.threads[i].profile == name for i in core.thread_indices) ) - ): - raise ValueError( - f"compiler {name}: test hart is not a direct accelerator binding" - ) - compiled[name].test_core = selected + ) bemu = derived["bemu"] for ball in bemu["balls"]: e = b.bemu.balls.add() - e.ball_class = ball["ball_class"] - e.ball_dir = ball["ball_dir"] + e.emu_class = ball["emu_class"] + e.component = ball["component"] e.emu_lib = ball["emu_lib"] harts = {core.hart_id for core in b.device.cores} @@ -424,9 +411,6 @@ def fill_design( if type(hart) is not int or hart not in harts: raise ValueError(f"workload {key}: unknown hart {hart!r}") setattr(b.workload, field, hart) - - for key, val in derived["workload"]["cmake_param"].items(): - b.workload.cmake_param[key] = val return b @@ -493,7 +477,7 @@ def _append_core(design: pb.Design, raw: dict[str, Any], bbdir: Path) -> None: core.shared_mem.input_channels = shared["inputChannels"] core.shared_mem.default_group_count = shared["defaultGroupCount"] core.shared_mem.virtual_bank_count = shared["virtualBankCount"] - core.shared_mem.mem_cycles = shared["memCycles"] + core.shared_mem.latency = shared["latency"] if "accelerator" in raw: acc = raw["accelerator"] _fill_ball(core.accelerator.balldomain, acc["balldomain"], bbdir) @@ -577,9 +561,7 @@ def write_design_pb( parser = argparse.ArgumentParser( description="Build a configured core protobuf using the standard TOML expansion" ) - source_kind = parser.add_mutually_exclusive_group(required=True) - source_kind.add_argument("--core", type=Path) - source_kind.add_argument("--design", type=Path) + parser.add_argument("--core", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--repo", type=Path, required=True) options = parser.parse_args() @@ -587,17 +569,5 @@ def write_design_pb( spec = importlib.util.spec_from_file_location("toml_expansion", source) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - if options.core is not None: - output = write_core_pb( - module.toml2json(options.core), options.output, options.repo - ) - else: - data = module.design2json(options.design) - derived = _derive.derive( - data, options.repo, options.design.resolve().parent.parent.name - ) - design = fill_design(data, derived, options.repo) - options.output.parent.mkdir(parents=True, exist_ok=True) - options.output.write_bytes(design.SerializeToString(deterministic=True)) - output = options.output + output = write_core_pb(module.toml2json(options.core), options.output, options.repo) print(output) diff --git a/bbdev/api/steps/config/scripts/ball_parameters.py b/bbdev/api/steps/config/scripts/ball_parameters.py index 1bf89dd9a..a2d7d3f91 100644 --- a/bbdev/api/steps/config/scripts/ball_parameters.py +++ b/bbdev/api/steps/config/scripts/ball_parameters.py @@ -5,7 +5,7 @@ def pack_parameters(instance, values): - message = import_module(f"{instance.ball_dir}_params_pb2").Params() + message = import_module(f"{instance.component}_params_pb2").Params() ParseDict(values, message) missing = [ field.name @@ -13,19 +13,19 @@ def pack_parameters(instance, values): if not message.HasField(field.name) ] if missing: - raise ValueError(f"{instance.ball_name}: missing parameters {missing}") + raise ValueError(f"{instance.name}: missing parameters {missing}") instance.parameters.Pack(message, deterministic=True) def unpack_parameters(instance): - message = import_module(f"{instance.ball_dir}_params_pb2").Params() + message = import_module(f"{instance.component}_params_pb2").Params() if not instance.parameters.Unpack(message): - raise ValueError(f"{instance.ball_name}: wrong parameter message type") + raise ValueError(f"{instance.name}: wrong parameter message type") missing = [ field.name for field in message.DESCRIPTOR.fields if not message.HasField(field.name) ] if missing: - raise ValueError(f"{instance.ball_name}: missing parameters {missing}") + raise ValueError(f"{instance.name}: missing parameters {missing}") return message diff --git a/bbdev/api/steps/config/scripts/hardware_pb2.py b/bbdev/api/steps/config/scripts/hardware_pb2.py index 389cbeec6..169879f0d 100644 --- a/bbdev/api/steps/config/scripts/hardware_pb2.py +++ b/bbdev/api/steps/config/scripts/hardware_pb2.py @@ -20,7 +20,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x0ehardware.proto\x12\x10\x62uckyball.config"L\n\x0b\x43hiHardware\x12\x14\n\x0cnode_id_bits\x18\x01 \x02(\r\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x02 \x02(\r\x12\x11\n\tdata_bits\x18\x03 \x02(\r"G\n\x0b\x41xiHardware\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x01 \x02(\r\x12\x11\n\tdata_bits\x18\x02 \x02(\r\x12\x0f\n\x07id_bits\x18\x03 \x02(\r"r\n\x10TrackingHardware\x12\x0f\n\x07\x65ntries\x18\x01 \x02(\r\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x02 \x02(\r\x12\x0f\n\x07id_bits\x18\x03 \x02(\r\x12\x12\n\nline_bytes\x18\x04 \x02(\r\x12\x12\n\nmax_ranges\x18\x05 \x02(\r"\x98\x01\n\x0eRegionHardware\x12\x0c\n\x04\x62\x61se\x18\x01 \x02(\x04\x12\r\n\x05\x62ytes\x18\x02 \x02(\x04\x12\x11\n\tcacheable\x18\x03 \x02(\x08\x12\x12\n\nexecutable\x18\x04 \x02(\x08\x12\x10\n\x08readable\x18\x05 \x02(\x08\x12\x10\n\x08writable\x18\x06 \x02(\x08\x12\x0e\n\x06\x61tomic\x18\x07 \x02(\x08\x12\x0e\n\x06normal\x18\x08 \x02(\x08"\x95\x01\n\rCacheHardware\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x01 \x02(\r\x12\x12\n\nline_bytes\x18\x02 \x02(\r\x12\x0c\n\x04sets\x18\x03 \x02(\r\x12\x0c\n\x04ways\x18\x04 \x02(\r\x12\x0f\n\x07id_bits\x18\x05 \x02(\r\x12\x15\n\rmetadata_bits\x18\x06 \x02(\r\x12\x16\n\x0eresponse_depth\x18\x07 \x02(\r"\x91\x01\n\x0bRnfHardware\x12\x0f\n\x07node_id\x18\x01 \x02(\r\x12\x13\n\x0b\x63\x61\x63he_lines\x18\x02 \x02(\r\x12\x0f\n\x07home_id\x18\x03 \x02(\r\x12\x12\n\nhome_count\x18\x04 \x02(\r\x12\r\n\x05\x62\x61nks\x18\x05 \x02(\r\x12\x13\n\x0bline_result\x18\x06 \x02(\x08\x12\r\n\x05probe\x18\x07 \x02(\x08J\x04\x08\x08\x10\t"\xa8\x01\n\x0cHomeHardware\x12.\n\x05\x63\x61\x63he\x18\x01 \x02(\x0b\x32\x1f.buckyball.config.CacheHardware\x12\x0e\n\x06\x61gents\x18\x02 \x02(\r\x12\x14\n\x0cmshr_entries\x18\x03 \x02(\r\x12\x0f\n\x07home_id\x18\x04 \x02(\r\x12\x31\n\nrequesters\x18\x05 \x02(\x0b\x32\x1d.buckyball.config.ChiHardware"\x8c\x01\n\x13PreparationHardware\x12*\n\x03\x62us\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.ChiHardware\x12\x10\n\x08\x63ontexts\x18\x02 \x02(\r\x12\x12\n\nmax_ranges\x18\x03 \x02(\r\x12\x0f\n\x07id_bits\x18\x04 \x02(\r\x12\x12\n\nbeat_bytes\x18\x05 \x02(\r"\xe1\x01\n\x11TransportHardware\x12\x34\n\x08tracking\x18\x01 \x02(\x0b\x32".buckyball.config.TrackingHardware\x12\x37\n\x08prepared\x18\x02 \x02(\x0b\x32%.buckyball.config.PreparationHardware\x12*\n\x03\x61xi\x18\x03 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware\x12\x31\n\x07regions\x18\x04 \x03(\x0b\x32 .buckyball.config.RegionHardware"\xc6\x01\n\x0b\x44\x64rHardware\x12+\n\x04line\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.ChiHardware\x12\x0f\n\x07\x63lients\x18\x02 \x02(\r\x12\x18\n\x10slots_per_client\x18\x03 \x02(\r\x12\x11\n\tdata_bits\x18\x04 \x02(\r\x12\x0f\n\x07id_bits\x18\x05 \x02(\r\x12\x0f\n\x07id_base\x18\x06 \x02(\r\x12\x0c\n\x04\x62\x61se\x18\x07 \x02(\x04\x12\r\n\x05\x62ytes\x18\x08 \x02(\x04\x12\r\n\x05model\x18\t \x02(\x04"=\n\x0f\x42ootRomHardware\x12\x0c\n\x04\x62\x61se\x18\x01 \x02(\x04\x12\r\n\x05\x62ytes\x18\x02 \x02(\x04\x12\r\n\x05image\x18\x03 \x03(\x11"S\n\rClintHardware\x12\x10\n\x08hart_ids\x18\x01 \x03(\r\x12\x0c\n\x04\x62\x61se\x18\x02 \x02(\x04\x12\r\n\x05\x62ytes\x18\x03 \x02(\x04\x12\x13\n\x0btick_cycles\x18\x04 \x02(\r"e\n\x0cPlicHardware\x12\x10\n\x08hart_ids\x18\x01 \x03(\r\x12\x0c\n\x04\x62\x61se\x18\x02 \x02(\x04\x12\r\n\x05\x62ytes\x18\x03 \x02(\x04\x12\x0f\n\x07sources\x18\x04 \x02(\r\x12\x15\n\rpriority_bits\x18\x05 \x02(\r"f\n\x0bScuHardware\x12\x14\n\x0c\x62\x61se_address\x18\x01 \x02(\x04\x12\x14\n\x0cstride_bytes\x18\x02 \x02(\x04\x12\x18\n\x10total_size_bytes\x18\x03 \x02(\x04\x12\x11\n\tmax_harts\x18\x04 \x02(\r"E\n\x11SimMemoryHardware\x12\x12\n\nline_bytes\x18\x03 \x02(\r\x12\x10\n\x08\x63lock_hz\x18\x04 \x02(\x04J\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03"[\n\x0bSimHardware\x12\x11\n\tdiff_test\x18\x01 \x02(\x08\x12\x33\n\x06memory\x18\x02 \x02(\x0b\x32#.buckyball.config.SimMemoryHardwareJ\x04\x08\x03\x10\x04"\xa3\x01\n\x12MemoryAccessConfig\x12\x17\n\x0f\x61\x63\x63\x65ss_tag_bits\x18\x02 \x02(\r\x12\x1b\n\x13translation_entries\x18\x03 \x02(\r\x12\x18\n\x10result_line_bits\x18\x04 \x02(\r\x12\x31\n\x07regions\x18\x06 \x03(\x0b\x32 .buckyball.config.RegionHardwareJ\x04\x08\x01\x10\x02J\x04\x08\x05\x10\x06"r\n\x0c\x44\x43\x61\x63heConfig\x12,\n\x05\x63\x61\x63he\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.RnfHardware\x12\x34\n\x08tracking\x18\x02 \x02(\x0b\x32".buckyball.config.TrackingHardware"K\n\x0eUncachedConfig\x12\r\n\x05ports\x18\x01 \x02(\r\x12*\n\x03\x61xi\x18\x02 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware"\x8b\x02\n\x0cMemoryConfig\x12\x32\n\x08uncached\x18\x06 \x02(\x0b\x32 .buckyball.config.UncachedConfig\x12\x34\n\x06\x61\x63\x63\x65ss\x18\x02 \x02(\x0b\x32$.buckyball.config.MemoryAccessConfig\x12-\n\x06icache\x18\x03 \x02(\x0b\x32\x1d.buckyball.config.RnfHardware\x12.\n\x06\x64\x63\x61\x63he\x18\x04 \x02(\x0b\x32\x1e.buckyball.config.DCacheConfig\x12,\n\x04home\x18\x05 \x02(\x0b\x32\x1e.buckyball.config.HomeHardwareJ\x04\x08\x01\x10\x02"(\n\x0b\x46\x65tchConfig\x12\x19\n\x11instruction_bytes\x18\x01 \x02(\r"\x1a\n\tFpuConfig\x12\r\n\x05\x66_len\x18\x01 \x02(\r"O\n\x12InterconnectConfig\x12*\n\x03\x61xi\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware\x12\r\n\x05slots\x18\x02 \x02(\r"\\\n\tC2CConfig\x12\x0f\n\x07\x65nabled\x18\x01 \x02(\x08\x12\x12\n\npeer_cores\x18\x02 \x02(\r\x12*\n\x03\x61xi\x18\x03 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware" \n\nTBusConfig\x12\x12\n\nbank_nodes\x18\x01 \x03(\r"!\n\x10\x43ontrollerConfig\x12\r\n\x05moves\x18\x01 \x02(\x08\x42\x14\n\x10\x62uckyball.configP\x01' + b'\n\x0ehardware.proto\x12\x10\x62uckyball.config"L\n\x0b\x43hiHardware\x12\x14\n\x0cnode_id_bits\x18\x01 \x02(\r\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x02 \x02(\r\x12\x11\n\tdata_bits\x18\x03 \x02(\r"G\n\x0b\x41xiHardware\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x01 \x02(\r\x12\x11\n\tdata_bits\x18\x02 \x02(\r\x12\x0f\n\x07id_bits\x18\x03 \x02(\r"r\n\x10TrackingHardware\x12\x0f\n\x07\x65ntries\x18\x01 \x02(\r\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x02 \x02(\r\x12\x0f\n\x07id_bits\x18\x03 \x02(\r\x12\x12\n\nline_bytes\x18\x04 \x02(\r\x12\x12\n\nmax_ranges\x18\x05 \x02(\r"\x98\x01\n\x0eRegionHardware\x12\x0c\n\x04\x62\x61se\x18\x01 \x02(\x04\x12\r\n\x05\x62ytes\x18\x02 \x02(\x04\x12\x11\n\tcacheable\x18\x03 \x02(\x08\x12\x12\n\nexecutable\x18\x04 \x02(\x08\x12\x10\n\x08readable\x18\x05 \x02(\x08\x12\x10\n\x08writable\x18\x06 \x02(\x08\x12\x0e\n\x06\x61tomic\x18\x07 \x02(\x08\x12\x0e\n\x06normal\x18\x08 \x02(\x08"\x95\x01\n\rCacheHardware\x12\x14\n\x0c\x61\x64\x64ress_bits\x18\x01 \x02(\r\x12\x12\n\nline_bytes\x18\x02 \x02(\r\x12\x0c\n\x04sets\x18\x03 \x02(\r\x12\x0c\n\x04ways\x18\x04 \x02(\r\x12\x0f\n\x07id_bits\x18\x05 \x02(\r\x12\x15\n\rmetadata_bits\x18\x06 \x02(\r\x12\x16\n\x0eresponse_depth\x18\x07 \x02(\r"\x8b\x01\n\x0bRnfHardware\x12\x0f\n\x07node_id\x18\x01 \x02(\r\x12\r\n\x05lines\x18\x02 \x02(\r\x12\x0f\n\x07home_id\x18\x03 \x02(\r\x12\x12\n\nhome_count\x18\x04 \x02(\r\x12\r\n\x05\x62\x61nks\x18\x05 \x02(\r\x12\x13\n\x0bline_result\x18\x06 \x02(\x08\x12\r\n\x05probe\x18\x07 \x02(\x08J\x04\x08\x08\x10\t"\xa8\x01\n\x0cHomeHardware\x12.\n\x05\x63\x61\x63he\x18\x01 \x02(\x0b\x32\x1f.buckyball.config.CacheHardware\x12\x0e\n\x06\x61gents\x18\x02 \x02(\r\x12\x14\n\x0cmshr_entries\x18\x03 \x02(\r\x12\x0f\n\x07node_id\x18\x04 \x02(\r\x12\x31\n\nrequesters\x18\x05 \x02(\x0b\x32\x1d.buckyball.config.ChiHardware"\x8c\x01\n\x13PreparationHardware\x12*\n\x03\x62us\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.ChiHardware\x12\x10\n\x08\x63ontexts\x18\x02 \x02(\r\x12\x12\n\nmax_ranges\x18\x03 \x02(\r\x12\x0f\n\x07id_bits\x18\x04 \x02(\r\x12\x12\n\nbeat_bytes\x18\x05 \x02(\r"\xe1\x01\n\x11TransportHardware\x12\x34\n\x08tracking\x18\x01 \x02(\x0b\x32".buckyball.config.TrackingHardware\x12\x37\n\x08prepared\x18\x02 \x02(\x0b\x32%.buckyball.config.PreparationHardware\x12*\n\x03\x61xi\x18\x03 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware\x12\x31\n\x07regions\x18\x04 \x03(\x0b\x32 .buckyball.config.RegionHardware"\xc6\x01\n\x0b\x44\x64rHardware\x12+\n\x04line\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.ChiHardware\x12\x0f\n\x07\x63lients\x18\x02 \x02(\r\x12\x18\n\x10slots_per_client\x18\x03 \x02(\r\x12\x11\n\tdata_bits\x18\x04 \x02(\r\x12\x0f\n\x07id_bits\x18\x05 \x02(\r\x12\x0f\n\x07id_base\x18\x06 \x02(\r\x12\x0c\n\x04\x62\x61se\x18\x07 \x02(\x04\x12\r\n\x05\x62ytes\x18\x08 \x02(\x04\x12\r\n\x05model\x18\t \x02(\x04"=\n\x0f\x42ootRomHardware\x12\x0c\n\x04\x62\x61se\x18\x01 \x02(\x04\x12\r\n\x05\x62ytes\x18\x02 \x02(\x04\x12\r\n\x05image\x18\x03 \x03(\x11"S\n\rClintHardware\x12\x10\n\x08hart_ids\x18\x01 \x03(\r\x12\x0c\n\x04\x62\x61se\x18\x02 \x02(\x04\x12\r\n\x05\x62ytes\x18\x03 \x02(\x04\x12\x13\n\x0btick_cycles\x18\x04 \x02(\r"e\n\x0cPlicHardware\x12\x10\n\x08hart_ids\x18\x01 \x03(\r\x12\x0c\n\x04\x62\x61se\x18\x02 \x02(\x04\x12\r\n\x05\x62ytes\x18\x03 \x02(\x04\x12\x0f\n\x07sources\x18\x04 \x02(\r\x12\x15\n\rpriority_bits\x18\x05 \x02(\r"f\n\x0bScuHardware\x12\x14\n\x0c\x62\x61se_address\x18\x01 \x02(\x04\x12\x14\n\x0cstride_bytes\x18\x02 \x02(\x04\x12\x18\n\x10total_size_bytes\x18\x03 \x02(\x04\x12\x11\n\tmax_harts\x18\x04 \x02(\r"E\n\x11SimMemoryHardware\x12\x12\n\nline_bytes\x18\x03 \x02(\r\x12\x10\n\x08\x63lock_hz\x18\x04 \x02(\x04J\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03"[\n\x0bSimHardware\x12\x11\n\tdiff_test\x18\x01 \x02(\x08\x12\x33\n\x06memory\x18\x02 \x02(\x0b\x32#.buckyball.config.SimMemoryHardwareJ\x04\x08\x03\x10\x04"\x9c\x01\n\x12MemoryAccessConfig\x12\x10\n\x08tag_bits\x18\x02 \x02(\r\x12\x1b\n\x13translation_entries\x18\x03 \x02(\r\x12\x18\n\x10result_line_bits\x18\x04 \x02(\r\x12\x31\n\x07regions\x18\x06 \x03(\x0b\x32 .buckyball.config.RegionHardwareJ\x04\x08\x01\x10\x02J\x04\x08\x05\x10\x06"r\n\x0c\x44\x43\x61\x63heConfig\x12,\n\x05\x63\x61\x63he\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.RnfHardware\x12\x34\n\x08tracking\x18\x02 \x02(\x0b\x32".buckyball.config.TrackingHardware"K\n\x0eUncachedConfig\x12\r\n\x05ports\x18\x01 \x02(\r\x12*\n\x03\x61xi\x18\x02 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware"\x8b\x02\n\x0cMemoryConfig\x12\x32\n\x08uncached\x18\x06 \x02(\x0b\x32 .buckyball.config.UncachedConfig\x12\x34\n\x06\x61\x63\x63\x65ss\x18\x02 \x02(\x0b\x32$.buckyball.config.MemoryAccessConfig\x12-\n\x06icache\x18\x03 \x02(\x0b\x32\x1d.buckyball.config.RnfHardware\x12.\n\x06\x64\x63\x61\x63he\x18\x04 \x02(\x0b\x32\x1e.buckyball.config.DCacheConfig\x12,\n\x04home\x18\x05 \x02(\x0b\x32\x1e.buckyball.config.HomeHardwareJ\x04\x08\x01\x10\x02"(\n\x0b\x46\x65tchConfig\x12\x19\n\x11instruction_bytes\x18\x01 \x02(\r"\x1a\n\tFpuConfig\x12\r\n\x05\x66_len\x18\x01 \x02(\r"O\n\x12InterconnectConfig\x12*\n\x03\x61xi\x18\x01 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware\x12\r\n\x05slots\x18\x02 \x02(\r"\\\n\tC2CConfig\x12\x0f\n\x07\x65nabled\x18\x01 \x02(\x08\x12\x12\n\npeer_cores\x18\x02 \x02(\r\x12*\n\x03\x61xi\x18\x03 \x02(\x0b\x32\x1d.buckyball.config.AxiHardware" \n\nTBusConfig\x12\x12\n\nbank_nodes\x18\x01 \x03(\r"!\n\x10\x43ontrollerConfig\x12\r\n\x05moves\x18\x01 \x02(\x08\x42\x14\n\x10\x62uckyball.configP\x01' ) _globals = globals() @@ -40,45 +40,45 @@ _globals["_CACHEHARDWARE"]._serialized_start = 459 _globals["_CACHEHARDWARE"]._serialized_end = 608 _globals["_RNFHARDWARE"]._serialized_start = 611 - _globals["_RNFHARDWARE"]._serialized_end = 756 - _globals["_HOMEHARDWARE"]._serialized_start = 759 - _globals["_HOMEHARDWARE"]._serialized_end = 927 - _globals["_PREPARATIONHARDWARE"]._serialized_start = 930 - _globals["_PREPARATIONHARDWARE"]._serialized_end = 1070 - _globals["_TRANSPORTHARDWARE"]._serialized_start = 1073 - _globals["_TRANSPORTHARDWARE"]._serialized_end = 1298 - _globals["_DDRHARDWARE"]._serialized_start = 1301 - _globals["_DDRHARDWARE"]._serialized_end = 1499 - _globals["_BOOTROMHARDWARE"]._serialized_start = 1501 - _globals["_BOOTROMHARDWARE"]._serialized_end = 1562 - _globals["_CLINTHARDWARE"]._serialized_start = 1564 - _globals["_CLINTHARDWARE"]._serialized_end = 1647 - _globals["_PLICHARDWARE"]._serialized_start = 1649 - _globals["_PLICHARDWARE"]._serialized_end = 1750 - _globals["_SCUHARDWARE"]._serialized_start = 1752 - _globals["_SCUHARDWARE"]._serialized_end = 1854 - _globals["_SIMMEMORYHARDWARE"]._serialized_start = 1856 - _globals["_SIMMEMORYHARDWARE"]._serialized_end = 1925 - _globals["_SIMHARDWARE"]._serialized_start = 1927 - _globals["_SIMHARDWARE"]._serialized_end = 2018 - _globals["_MEMORYACCESSCONFIG"]._serialized_start = 2021 - _globals["_MEMORYACCESSCONFIG"]._serialized_end = 2184 - _globals["_DCACHECONFIG"]._serialized_start = 2186 - _globals["_DCACHECONFIG"]._serialized_end = 2300 - _globals["_UNCACHEDCONFIG"]._serialized_start = 2302 - _globals["_UNCACHEDCONFIG"]._serialized_end = 2377 - _globals["_MEMORYCONFIG"]._serialized_start = 2380 - _globals["_MEMORYCONFIG"]._serialized_end = 2647 - _globals["_FETCHCONFIG"]._serialized_start = 2649 - _globals["_FETCHCONFIG"]._serialized_end = 2689 - _globals["_FPUCONFIG"]._serialized_start = 2691 - _globals["_FPUCONFIG"]._serialized_end = 2717 - _globals["_INTERCONNECTCONFIG"]._serialized_start = 2719 - _globals["_INTERCONNECTCONFIG"]._serialized_end = 2798 - _globals["_C2CCONFIG"]._serialized_start = 2800 - _globals["_C2CCONFIG"]._serialized_end = 2892 - _globals["_TBUSCONFIG"]._serialized_start = 2894 - _globals["_TBUSCONFIG"]._serialized_end = 2926 - _globals["_CONTROLLERCONFIG"]._serialized_start = 2928 - _globals["_CONTROLLERCONFIG"]._serialized_end = 2961 + _globals["_RNFHARDWARE"]._serialized_end = 750 + _globals["_HOMEHARDWARE"]._serialized_start = 753 + _globals["_HOMEHARDWARE"]._serialized_end = 921 + _globals["_PREPARATIONHARDWARE"]._serialized_start = 924 + _globals["_PREPARATIONHARDWARE"]._serialized_end = 1064 + _globals["_TRANSPORTHARDWARE"]._serialized_start = 1067 + _globals["_TRANSPORTHARDWARE"]._serialized_end = 1292 + _globals["_DDRHARDWARE"]._serialized_start = 1295 + _globals["_DDRHARDWARE"]._serialized_end = 1493 + _globals["_BOOTROMHARDWARE"]._serialized_start = 1495 + _globals["_BOOTROMHARDWARE"]._serialized_end = 1556 + _globals["_CLINTHARDWARE"]._serialized_start = 1558 + _globals["_CLINTHARDWARE"]._serialized_end = 1641 + _globals["_PLICHARDWARE"]._serialized_start = 1643 + _globals["_PLICHARDWARE"]._serialized_end = 1744 + _globals["_SCUHARDWARE"]._serialized_start = 1746 + _globals["_SCUHARDWARE"]._serialized_end = 1848 + _globals["_SIMMEMORYHARDWARE"]._serialized_start = 1850 + _globals["_SIMMEMORYHARDWARE"]._serialized_end = 1919 + _globals["_SIMHARDWARE"]._serialized_start = 1921 + _globals["_SIMHARDWARE"]._serialized_end = 2012 + _globals["_MEMORYACCESSCONFIG"]._serialized_start = 2015 + _globals["_MEMORYACCESSCONFIG"]._serialized_end = 2171 + _globals["_DCACHECONFIG"]._serialized_start = 2173 + _globals["_DCACHECONFIG"]._serialized_end = 2287 + _globals["_UNCACHEDCONFIG"]._serialized_start = 2289 + _globals["_UNCACHEDCONFIG"]._serialized_end = 2364 + _globals["_MEMORYCONFIG"]._serialized_start = 2367 + _globals["_MEMORYCONFIG"]._serialized_end = 2634 + _globals["_FETCHCONFIG"]._serialized_start = 2636 + _globals["_FETCHCONFIG"]._serialized_end = 2676 + _globals["_FPUCONFIG"]._serialized_start = 2678 + _globals["_FPUCONFIG"]._serialized_end = 2704 + _globals["_INTERCONNECTCONFIG"]._serialized_start = 2706 + _globals["_INTERCONNECTCONFIG"]._serialized_end = 2785 + _globals["_C2CCONFIG"]._serialized_start = 2787 + _globals["_C2CCONFIG"]._serialized_end = 2879 + _globals["_TBUSCONFIG"]._serialized_start = 2881 + _globals["_TBUSCONFIG"]._serialized_end = 2913 + _globals["_CONTROLLERCONFIG"]._serialized_start = 2915 + _globals["_CONTROLLERCONFIG"]._serialized_end = 2948 # @@protoc_insertion_point(module_scope) diff --git a/bbdev/api/steps/config/scripts/proto/design.proto b/bbdev/api/steps/config/scripts/proto/design.proto index 019983822..f9bb17c83 100644 --- a/bbdev/api/steps/config/scripts/proto/design.proto +++ b/bbdev/api/steps/config/scripts/proto/design.proto @@ -5,11 +5,18 @@ syntax = "proto3"; package buckyball.config; import "google/protobuf/any.proto"; +import "google/protobuf/descriptor.proto"; import "hardware.proto"; option java_package = "buckyball.config"; option java_multiple_files = true; +// Marks the field that names an element of a repeated message; design.cmake +// keys the element by it (stack/scripts/design.py). Unmarked: its position. +extend google.protobuf.FieldOptions { + bool key = 50000; +} + message MillConfig { reserved 1, 2; map targets = 3; @@ -38,22 +45,19 @@ message DmaConfig { uint32 bus_width = 3; } -message MemAddrConfig { - uint32 addr_len = 1; -} - message MemDomainConfig { BankConfig bank = 1; DmaConfig dma = 2; MmioConfig mmio = 5; - MemAddrConfig mem = 6; + reserved 6; string source_path = 7; + uint32 addr_len = 8; } message BallInstance { - uint32 ball_id = 1; - string ball_name = 2; - string ball_dir = 3; + uint32 id = 1; + string name = 2; + string component = 3 [(key) = true]; uint32 in_bw = 4; uint32 out_bw = 5; uint32 mmio_read_bw = 6; @@ -65,7 +69,7 @@ message BallInstance { } message BallInstruction { - string mnemonic = 1; + string mnemonic = 1 [(key) = true]; uint32 funct7 = 2; } @@ -77,8 +81,10 @@ message FrontendConfig { uint32 rob_entries = 1; bool rs_out_of_order_response = 2; uint32 bank_id_len = 3; - uint32 vbank_id_upper_bound = 4; - uint32 shared_bank_id_base = 5; + // Virtual bank ids [0, max_private_vbank] are private. + uint32 max_private_vbank = 4; + // Virtual bank ids from shared_vbank_base up are shared. + uint32 shared_vbank_base = 5; uint32 iter_len = 6; bool sub_rob_enable = 7; uint32 sub_rob_depth = 8; @@ -148,8 +154,8 @@ message ThreadInstance { } message BemuBall { - string ball_class = 1; - string ball_dir = 2; + string emu_class = 1; + string component = 2 [(key) = true]; string emu_lib = 3; } @@ -164,7 +170,7 @@ message SharedMemConfig { uint32 input_channels = 3; uint32 default_group_count = 4; uint32 virtual_bank_count = 5; - uint32 mem_cycles = 6; + uint32 latency = 6; uint32 bank_entries = 7; uint32 bank_width = 8; } @@ -193,7 +199,7 @@ message CorePlacement { LocalAccelerator accelerator = 14; string profile = 16; reserved 17; - uint32 hart_id = 18; + uint32 hart_id = 18 [(key) = true]; uint64 reset_vector = 19; uint64 mtime_address = 20; MemoryConfig memory = 21; @@ -211,22 +217,20 @@ message CorePlacement { } message WorkloadConfig { - reserved 1; - map cmake_param = 2; + reserved 1, 2; optional uint32 control_hart = 3; optional uint32 report_hart = 4; } +// A compiler target. Its programs see `thread` and `core` by their paths +// from the profile: THREAD_MEM_BANK_WIDTH, CORE_HART_ID. message CompilerProfile { - string name = 1; - string pkg = 2; - string compiler = 3; - uint32 bank_num = 4; - uint32 bank_width = 5; - uint32 bank_entries = 6; + string name = 1 [(key) = true]; + reserved 2, 3, 4, 5, 6, 9; repeated string ball_ctest_dirs = 7; - ThreadInstance config = 8; - optional uint32 test_core = 9; + ThreadInstance thread = 8; + // The core its programs run on: the first core running the profile. + CorePlacement core = 10; } message Design { @@ -240,13 +244,15 @@ message Design { reserved 9; BemuConfig bemu = 10; WorkloadConfig workload = 11; - string design_path = 12; + string path = 12; repeated CompilerProfile profiles = 13; reserved 14; DeviceConfig device = 15; SimHardware sims = 16; VerificationConfig verification = 17; - map simulations = 18; + reserved 18; + // The simulation this design.pb was generated for. + SimulationTarget simulation = 19; } message SharedBankFabricConfig { @@ -332,4 +338,8 @@ message SimulationTarget { string stack_name = 8; string workloads = 9; map resources = 10; + // device/ or core/: names the run and its output root. + string selection = 11; + // output/sims/, relative to the repository. + string output = 12; } diff --git a/bbdev/api/steps/config/scripts/proto/hardware.proto b/bbdev/api/steps/config/scripts/proto/hardware.proto index 967aba249..c3f12f596 100644 --- a/bbdev/api/steps/config/scripts/proto/hardware.proto +++ b/bbdev/api/steps/config/scripts/proto/hardware.proto @@ -41,7 +41,7 @@ message CacheHardware { } message RnfHardware { required uint32 node_id = 1; - required uint32 cache_lines = 2; + required uint32 lines = 2; required uint32 home_id = 3; required uint32 home_count = 4; required uint32 banks = 5; @@ -53,7 +53,7 @@ message HomeHardware { required CacheHardware cache = 1; required uint32 agents = 2; required uint32 mshr_entries = 3; - required uint32 home_id = 4; + required uint32 node_id = 4; required ChiHardware requesters = 5; } message PreparationHardware { @@ -121,7 +121,7 @@ message SimHardware { message MemoryAccessConfig { reserved 1; - required uint32 access_tag_bits = 2; + required uint32 tag_bits = 2; required uint32 translation_entries = 3; required uint32 result_line_bits = 4; reserved 5; diff --git a/bbdev/api/steps/dc/01_area_event.step.py b/bbdev/api/steps/dc/01_area_event.step.py index a2a3516ec..e33a8405b 100644 --- a/bbdev/api/steps/dc/01_area_event.step.py +++ b/bbdev/api/steps/dc/01_area_event.step.py @@ -137,7 +137,9 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return script_path = contract.dc_script - report_context = source_context(get_buckyball_path(), contract.design) + report_context = source_context( + get_buckyball_path(), contract.design, input_data["target"] + ) dc_log = os.path.join(analysis_dir, "dc_shell.log") ctx.logger.info( f"Running design-owned DC synthesis for {contract.design}, top {top_module}: {script_path}" diff --git a/bbdev/api/steps/kernel/01_build_event.step.py b/bbdev/api/steps/kernel/01_build_event.step.py index 3e6fe1bc1..a23b2b6e4 100644 --- a/bbdev/api/steps/kernel/01_build_event.step.py +++ b/bbdev/api/steps/kernel/01_build_event.step.py @@ -10,7 +10,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path, load_design_config +from utils.path import design_output_root, get_buckyball_path, load_design_config from utils.stream_run import stream_run_logger_async from utils.event_common import check_result, get_origin_trace_id @@ -24,43 +24,39 @@ def kernel_build_dir( - bbdir: str, hart_count: int, design: str, interactive: bool, guest_memory_mib: int + bbdir: str, + hart_count: int, + design: str, + selection: str, + interactive: bool, + guest_memory_mib: int, ) -> str: suffix = "" if guest_memory_mib == 512 else f"-mem{guest_memory_mib}M" - suffix += f"-design-{design}" + suffix += f"-design-{design}-{selection.replace('/', '-')}" if interactive: suffix += "-interactive" name = "kernel" if hart_count == 64 else f"kernel-h{hart_count}" return os.path.join(bbdir, "bb-tests", "build", f"{name}{suffix}") -def fw_payload_name(hart_count: int, design: str, guest_memory_mib: int) -> str: - name = "fw_payload" if hart_count == 64 else f"fw_payload-h{hart_count}" - name += f"-{design}-linux" - if guest_memory_mib != 512: - name += f"-mem{guest_memory_mib}M" - return name - - async def handler(input_data: dict, ctx: FlowContext) -> None: origin_tid = get_origin_trace_id(input_data, ctx) bbdir = get_buckyball_path() kernel_src = os.path.join(bbdir, "stack", "os", "linux") design = input_data["design"] - design_config = load_design_config(bbdir, design) + design_config = load_design_config(bbdir, design, selection=input_data["target"]) hart_count = len(design_config.device.cores) ddr = design_config.device.ddr guest_memory_mib = (ddr.bytes - ddr.model) // (1024 * 1024) # A DDR without a model region leaves the base empty: init runs no model. model_base = f"{ddr.base + ddr.bytes - ddr.model:#x}" if ddr.model else "" interactive = input_data.get("interactive", False) - output_dir = os.path.join( - bbdir, "examples/designs", design, "output/sims/device/workloads/kernel" - ) + output_root = design_output_root(bbdir, design, selection=input_data["target"]) + output_dir = os.path.join(output_root, "workloads/kernel") os.makedirs(output_dir, exist_ok=True) kernel_build = kernel_build_dir( - bbdir, hart_count, design, interactive, guest_memory_mib + bbdir, hart_count, design, input_data["target"], interactive, guest_memory_mib ) interactive_arg = "ON" if interactive else "OFF" programs = design_resource_dir( @@ -73,6 +69,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: f"-DBUCKYBALL_HART_COUNT={hart_count} " f"-DBUCKYBALL_GUEST_MEMORY_MIB={guest_memory_mib} " f"-DBUCKYBALL_KERNEL_DESIGN={design} " + f"-DBUCKYBALL_KERNEL_OUTPUT={output_root} " f"-DBUCKYBALL_KERNEL_INTERACTIVE={interactive_arg} " f"-DBUCKYBALL_MODEL_BASE={model_base} " f"-DBUCKYBALL_KERNEL_WORKLOADS={programs}" @@ -105,7 +102,8 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: ) return - payload_name = fw_payload_name(hart_count, design, guest_memory_mib) + # The output directory belongs to one device config, so the name is fixed. + payload_name = "fw_payload-linux" fw_payload_bin = os.path.join(output_dir, f"{payload_name}.bin") fw_payload_elf = os.path.join(output_dir, f"{payload_name}.elf") built_payload_elf = os.path.join( diff --git a/bbdev/api/steps/model/01_build_api.step.py b/bbdev/api/steps/model/01_build_api.step.py index b9433bee1..4040fd5b3 100644 --- a/bbdev/api/steps/model/01_build_api.step.py +++ b/bbdev/api/steps/model/01_build_api.step.py @@ -1,6 +1,6 @@ from motia import ApiRequest, ApiResponse, FlowContext, api -from utils.event_common import require_design +from utils.event_common import require_design, require_target config = { "name": "model-build-api", @@ -15,6 +15,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} try: design = require_design(body) + selection = require_target(body, ("device",)) except ValueError as error: return ApiResponse(status=400, body={"error": str(error)}) await ctx.enqueue( @@ -23,6 +24,7 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: "data": { "design": design, "model": body.get("model"), + "target": selection, "_trace_id": ctx.trace_id, }, } diff --git a/bbdev/api/steps/model/01_build_event.step.py b/bbdev/api/steps/model/01_build_event.step.py index 237f75779..eb91fd441 100644 --- a/bbdev/api/steps/model/01_build_event.step.py +++ b/bbdev/api/steps/model/01_build_event.step.py @@ -27,6 +27,7 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: get_buckyball_path(), params["design"], params["model"], + selection=params["target"], logger=ctx.logger, task_scope=trace_id, ) diff --git a/bbdev/api/steps/model/scripts/build.py b/bbdev/api/steps/model/scripts/build.py index 5d2071593..cfc82c61c 100644 --- a/bbdev/api/steps/model/scripts/build.py +++ b/bbdev/api/steps/model/scripts/build.py @@ -5,10 +5,16 @@ def build_model( - repo: str, design: str, model: str | None = None, *, logger, task_scope: str + repo: str, + design: str, + model: str | None = None, + *, + selection: str, + logger, + task_scope: str, ) -> None: """Build one model recipe of `design`, or every recipe of it when `model` is None. - Models run on the whole device, so they build against its device entry.""" + Models run on the whole device, so they build against a device config.""" recipes = Path(repo) / "examples/models" / design models = ( [model] @@ -19,11 +25,11 @@ def build_model( from steps.stack.scripts.build import build_design # Models compile with the design's compiler plugins and link its device runtime. - build_design( - repo, design=design, selection="device", logger=logger, task_scope=task_scope + design_cmake = build_design( + repo, design=design, selection=selection, logger=logger, task_scope=task_scope ) for model in models: - build = source / "build" / design / model + build = source / "build" / design / selection / model build_command = ["cmake", "--build", str(build), "--parallel", "1"] build_command.extend(["--target", f"{model}-model-build"]) # A recipe reads the prepared model while it configures, so the model @@ -52,8 +58,7 @@ def build_model( "-G", "Ninja", "-DARCH=buckyball", - f"-DDESIGN={design}", - "-DBUCKYBALL_SIM_TARGET=device", + f"-DDESIGN_CMAKE={design_cmake}", f"-DMODEL={model}", ], build_command, diff --git a/bbdev/api/steps/stack/01_build_api.step.py b/bbdev/api/steps/stack/01_build_api.step.py index 94e0458b4..fd02f3910 100644 --- a/bbdev/api/steps/stack/01_build_api.step.py +++ b/bbdev/api/steps/stack/01_build_api.step.py @@ -25,7 +25,9 @@ async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: except ValueError as error: return ApiResponse(status=400, body={"error": str(error)}) if set(body) - {"design", "device", "core"}: - raise ValueError("stack build takes --design and --device or --core") + raise ValueError( + "stack build takes --design and --device or --core " + ) if not isinstance(body.get("design"), str): raise ValueError("--design requires a design name") await ctx.enqueue( diff --git a/bbdev/api/steps/stack/scripts/build.py b/bbdev/api/steps/stack/scripts/build.py index 223633eeb..a8e00560c 100644 --- a/bbdev/api/steps/stack/scripts/build.py +++ b/bbdev/api/steps/stack/scripts/build.py @@ -10,7 +10,7 @@ from pathlib import Path -def _run( +def run( cmd: list[str], *, repo: Path, @@ -101,14 +101,14 @@ def build_llvm( with open(llvm_build / ".bbdev.lock", "a+b") as lockf: fcntl.flock(lockf.fileno(), fcntl.LOCK_EX) if not (llvm_build / "build.ninja").is_file(): - _run( + run( cmake, repo=root, logger=logger, task_scope=task_scope, output_prefix="compiler llvm configure", ) - _run( + run( ["ninja", "-C", str(llvm_build), "-j", str(os.cpu_count() or 1)], repo=root, logger=logger, @@ -148,14 +148,14 @@ def build_buddy( ] with open(buddy / ".bbdev.lock", "a+b") as lockf: fcntl.flock(lockf.fileno(), fcntl.LOCK_EX) - _run( + run( cmake, repo=root, logger=logger, task_scope=task_scope, output_prefix="buddy-mlir configure", ) - _run( + run( [ "ninja", "-C", @@ -177,15 +177,6 @@ def build_buddy( return build -def _design(repo: Path, design: str): - sys.path.insert(0, str(repo)) - from stack.scripts import design as design_config - - return design_config, design_config.load( - repo / "examples/designs" / design / "output/generated/design.pb" - ) - - def build_design( repo: str | Path, *, @@ -193,26 +184,36 @@ def build_design( selection: str, logger: object | None = None, task_scope: str | None = None, -) -> None: - """Build the software stack of one simulation entry of `design`. +) -> Path: + """Build the software stack of one simulation of `design`, and return its + design.cmake, the input of every software build of that simulation. - sims//stack/CMakeLists.txt is the top of that build: it lists the + The stack/CMakeLists.txt of its entry is the top of that build: it lists the thread, core and device stacks the entry needs and builds each of them. """ root = Path(repo).resolve() - design_config, config = _design(root, design) - if selection not in config.simulations: - raise ValueError(f"{design}: unregistered simulation entry {selection}") - build = design_config.stack_build(config, selection) - design_config.generate(config, build / "generated", selection) + sys.path.insert(0, str(root)) + from stack.scripts import design as design_config + + design_pb = ( + root + / "examples/designs" + / design + / "output/sims" + / selection + / "generated/design.pb" + ) + config = design_config.load(design_pb) + build = design_config.stack_build(config) + design_config.generate(design_pb, build / "generated") build_buddy(root, logger=logger, task_scope=task_scope) - _run( + run( [ "cmake", "-G", "Ninja", "-S", - config.simulations[selection].stack, + config.simulation.stack, "-B", str(build), f"-DDESIGN_CMAKE={build / 'generated/design.cmake'}", @@ -222,13 +223,14 @@ def build_design( task_scope=task_scope, output_prefix=f"stack {selection} configure", ) - _run( + run( ["ninja", "-C", str(build), "-j", str(os.cpu_count() or 1)], repo=root, logger=logger, task_scope=task_scope, output_prefix=f"stack {selection} build", ) + return build / "generated/design.cmake" def main() -> None: diff --git a/bbdev/api/steps/uvm/01_build_api.step.py b/bbdev/api/steps/uvm/01_build_api.step.py index bf6331295..b52dda5a9 100644 --- a/bbdev/api/steps/uvm/01_build_api.step.py +++ b/bbdev/api/steps/uvm/01_build_api.step.py @@ -23,6 +23,8 @@ def check_args(body: dict) -> str | None: return "Parameter --ball requires a value" if body.get("ip") is True: return "Parameter --ip requires a value" + if body.get("ball") and not body.get("design"): + return "Ball UVM requires --design and --device " if body.get("ball") and body.get("ip"): return "Parameters --ball and --ip are mutually exclusive" return None @@ -30,13 +32,19 @@ def check_args(body: dict) -> str | None: async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - try: - selection = require_target(body, ("device", "ip", "ball")) - except ValueError as error: - return ApiResponse(status=400, body={"error": str(error)}) error = check_args(body) if error: return ApiResponse(status=400, body={"error": error}) + # A design is verified through one of its device configs. + selection = None + if "design" in body: + try: + selection = require_target( + {key: body[key] for key in ("design", "device") if key in body}, + ("device",), + ) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) await ctx.enqueue( { diff --git a/bbdev/api/steps/uvm/01_build_event.step.py b/bbdev/api/steps/uvm/01_build_event.step.py index f3926c399..e6dbf5538 100644 --- a/bbdev/api/steps/uvm/01_build_event.step.py +++ b/bbdev/api/steps/uvm/01_build_event.step.py @@ -32,13 +32,20 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: try: if input_data.get("ball") is not None: info = await asyncio.to_thread( - run_design, bbdir, input_data["design"], input_data["ball"], ctx, False + run_design, + bbdir, + input_data["design"], + input_data["target"], + input_data["ball"], + ctx, + False, ) else: info = await asyncio.to_thread( run_verification, bbdir, input_data.get("design"), + input_data["target"], input_data.get("ip"), ctx, False, diff --git a/bbdev/api/steps/uvm/02_run_api.step.py b/bbdev/api/steps/uvm/02_run_api.step.py index 922f65939..0ec71ab99 100644 --- a/bbdev/api/steps/uvm/02_run_api.step.py +++ b/bbdev/api/steps/uvm/02_run_api.step.py @@ -23,6 +23,8 @@ def check_args(body: dict) -> str | None: return "Parameter --ball requires a value" if body.get("ip") is True: return "Parameter --ip requires a value" + if body.get("ball") and not body.get("design"): + return "Ball UVM requires --design and --device " if body.get("ball") and body.get("ip"): return "Parameters --ball and --ip are mutually exclusive" return None @@ -30,13 +32,19 @@ def check_args(body: dict) -> str | None: async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse: body = request.body or {} - try: - selection = require_target(body, ("device", "ip", "ball")) - except ValueError as error: - return ApiResponse(status=400, body={"error": str(error)}) error = check_args(body) if error: return ApiResponse(status=400, body={"error": error}) + # A design is verified through one of its device configs. + selection = None + if "design" in body: + try: + selection = require_target( + {key: body[key] for key in ("design", "device") if key in body}, + ("device",), + ) + except ValueError as error: + return ApiResponse(status=400, body={"error": str(error)}) await ctx.enqueue( { diff --git a/bbdev/api/steps/uvm/02_run_event.step.py b/bbdev/api/steps/uvm/02_run_event.step.py index beb9a08ec..f08116123 100644 --- a/bbdev/api/steps/uvm/02_run_event.step.py +++ b/bbdev/api/steps/uvm/02_run_event.step.py @@ -32,13 +32,20 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: try: if input_data.get("ball") is not None: info = await asyncio.to_thread( - run_design, bbdir, input_data["design"], input_data["ball"], ctx, True + run_design, + bbdir, + input_data["design"], + input_data["target"], + input_data["ball"], + ctx, + True, ) else: info = await asyncio.to_thread( run_verification, bbdir, input_data.get("design"), + input_data["target"], input_data.get("ip"), ctx, True, diff --git a/bbdev/api/steps/uvm/scripts/ball.py b/bbdev/api/steps/uvm/scripts/ball.py index 1fda45f1f..6812d500d 100644 --- a/bbdev/api/steps/uvm/scripts/ball.py +++ b/bbdev/api/steps/uvm/scripts/ball.py @@ -5,31 +5,19 @@ import tomllib from datetime import datetime from pathlib import Path -from utils.path import log_dir +from utils.path import design_arch_root, log_dir from steps.mill.scripts.mill import mill_cmd from utils.stream_run import stream_run_logger sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "config/scripts")) -def load_design(bbdir: str, design: str): - import design_pb2 +def load_design(bbdir: str, design: str, selection: str): + from utils.path import load_design_config - path = ( - Path(bbdir) - / "examples" - / "designs" - / design - / "output" - / "generated" - / "design.pb" - ) - if not path.is_file(): - raise FileNotFoundError(f"missing {path}; run bbdev config --install") - msg = design_pb2.Design() - msg.ParseFromString(path.read_bytes()) + msg = load_design_config(bbdir, design, selection=selection) if not msg.name or not msg.device.cores: - raise ValueError(f"empty design.pb: {path}") + raise ValueError(f"empty design.pb of {design} {selection}") return msg @@ -54,7 +42,7 @@ def ball_domain(design, ball: str | None = None): (domain, mapping) for domain in domains for mapping in domain.balls - if mapping.ball_dir == ball + if mapping.component == ball ] if not matches: raise ValueError(f"ball {ball!r} not in design.pb") @@ -68,7 +56,7 @@ def selected_mappings(domain, ball: str | None): if ball is None: return list(domain.balls) for m in domain.balls: - if m.ball_dir == ball: + if m.component == ball: return [m] raise ValueError(f"ball {ball!r} not in design.pb") @@ -81,14 +69,19 @@ def ball_owner(design, ball): owners.extend( (acc, core) for acc in accelerators - if any(item.ball_dir == ball for item in acc.balldomain.balls) + if any(item.component == ball for item in acc.balldomain.balls) ) if not owners: raise ValueError(f"ball {ball!r} not in design.pb") + # The first owner elaborates the unsuffixed Ball module that tb_top + # instantiates. Owners may differ only in bank count, which changes the + # group and column port widths of their own module copies. accelerator, core = owners[0] for other, owner in owners[1:]: if ( - other.mem.bank != accelerator.mem.bank + other.mem.bank.entries != accelerator.mem.bank.entries + or other.mem.bank.width != accelerator.mem.bank.width + or other.mem.bank.mask_len != accelerator.mem.bank.mask_len or other.frontend.rob_entries != accelerator.frontend.rob_entries or other.frontend.sub_rob_depth != accelerator.frontend.sub_rob_depth or owner.shared_mem != core.shared_mem @@ -102,7 +95,7 @@ def ball_owner(design, ball): def vcs_defines(domain, mapping, thread, core, parameters): shared_bank_num = ( - core.shared_mem.entries // thread.mem.bank.entries + core.shared_mem.entries // core.shared_mem.bank_entries if core.shared_mem.enable else 0 ) @@ -175,21 +168,17 @@ def check_uvm_result( def build_ball( - bbdir: str, design_name: str, mill_cfg: str, domain, mapping, ctx + bbdir: str, design_name: str, selection: str, mill_cfg: str, domain, mapping, ctx ) -> None: - ball = mapping.ball_dir - verify_dir = Path(bbdir) / "examples" / "components" / "balls" / ball / "verify" + ball = mapping.component + verify_dir = Path(bbdir) / "examples" / "prefab" / "balls" / ball / "verify" casegen = verify_dir / "casegen" / "Cargo.toml" - rtl_dir = ( - Path(bbdir) - / "examples/designs" - / design_name - / "output/sims/device/arch/verilog" - / mill_cfg + rtl_dir = Path(design_arch_root(bbdir, design_name, selection=selection)) / ( + f"verilog/{mill_cfg}" ) - design = load_design(bbdir, design_name) + design = load_design(bbdir, design_name, selection) thread, core = ball_owner(design, ball) - sim_dir = verify_dir.parent / "output" / "uvm" / design_name + sim_dir = verify_dir.parent / "output" / "uvm" / design_name / selection uvm_rel = os.path.relpath(Path(bbdir) / "verify" / "uvm", verify_dir) flist = _filelist(verify_dir, ball, uvm_rel, rtl_dir, sim_dir) parameters_path = verify_dir / "parameters.toml" @@ -247,12 +236,19 @@ def build_ball( def run_ball( - bbdir: str, design_name: str, mill_cfg: str, domain, mapping, ctx, cov_dir: str + bbdir: str, + design_name: str, + selection: str, + mill_cfg: str, + domain, + mapping, + ctx, + cov_dir: str, ) -> None: - ball = mapping.ball_dir - verify_dir = Path(bbdir) / "examples" / "components" / "balls" / ball / "verify" - simv = verify_dir.parent / "output" / "uvm" / design_name / "simv" - build_ball(bbdir, design_name, mill_cfg, domain, mapping, ctx) + ball = mapping.component + verify_dir = Path(bbdir) / "examples" / "prefab" / "balls" / ball / "verify" + simv = verify_dir.parent / "output" / "uvm" / design_name / selection / "simv" + build_ball(bbdir, design_name, selection, mill_cfg, domain, mapping, ctx) crate = tomllib.loads((verify_dir / "casegen" / "Cargo.toml").read_text())[ "package" ]["name"] @@ -265,12 +261,17 @@ def run_ball( ) test = f"{ball}_ball_test" verify_config = ( - verify_dir.parent / "output" / "uvm" / design_name / "verify_config.env" + verify_dir.parent + / "output" + / "uvm" + / design_name + / selection + / "verify_config.env" ) - design = load_design(bbdir, design_name) + design = load_design(bbdir, design_name, selection) thread, _ = ball_owner(design, ball) verify_config.write_text( - f"bank_entries={thread.mem.bank.entries}\nball_id={mapping.ball_id}\n" + f"bank_entries={thread.mem.bank.entries}\nball_id={mapping.id}\n" ) simv_q = shlex.quote(str(simv)) script = ( @@ -280,7 +281,7 @@ def run_ball( f"BB_VERIFY_CONFIG={shlex.quote(str(verify_config))} " f'"$loader" --library-path "$library_path" {simv_q} -no_save ' f"-sv_lib {shlex.quote(str(dpi))} " - f"+UVM_TESTNAME={shlex.quote(test)} +BID={mapping.ball_id} " + f"+UVM_TESTNAME={shlex.quote(test)} +BID={mapping.id} " f"-cm line+cond+tgl+assert -cm_name {shlex.quote(test)}" ) cmd = f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command zsh -ic {shlex.quote(script)}" @@ -328,10 +329,12 @@ def dashboard_summary(cov_dir: str) -> dict[str, str]: return dict(zip(keys, vals)) -def run_design(bbdir: str, design: str, ball: str | None, ctx, do_run: bool) -> dict: - msg = load_design(bbdir, design) +def run_design( + bbdir: str, design: str, selection: str, ball: str | None, ctx, do_run: bool +) -> dict: + msg = load_design(bbdir, design, selection) domain = ball_domain(msg, ball) - mill_cfg = msg.simulations["device"].top + mill_cfg = msg.simulation.top if not mill_cfg: raise ValueError(f"design {design!r} has no sims.verilator for UVM") maps = selected_mappings(domain, ball) @@ -340,22 +343,24 @@ def run_design(bbdir: str, design: str, ball: str | None, ctx, do_run: bool) -> run_root = None if do_run: stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - run_name = "all" if ball is None else maps[0].ball_dir - run_root = log_dir(bbdir, design, "verilog", stamp, "uvm", run_name) + run_name = "all" if ball is None else maps[0].component + run_root = log_dir( + bbdir, design, "verilog", stamp, "uvm", run_name, selection=selection + ) for m in maps: if do_run: cov = ( - os.path.join(run_root, m.ball_dir, "coverage") + os.path.join(run_root, m.component, "coverage") if ball is None else os.path.join(run_root, "coverage") ) - run_ball(bbdir, design, mill_cfg, domain, m, ctx, cov) + run_ball(bbdir, design, selection, mill_cfg, domain, m, ctx, cov) if not (Path(cov) / "dashboard.txt").is_file(): raise RuntimeError(f"urg wrote no dashboard.txt under {cov}") - covs.append((m.ball_dir, cov)) + covs.append((m.component, cov)) else: - build_ball(bbdir, design, mill_cfg, domain, m, ctx) - ran.append(m.ball_dir) + build_ball(bbdir, design, selection, mill_cfg, domain, m, ctx) + ran.append(m.component) info = {"design": design, "mill": mill_cfg, "balls": ran} if do_run and ball is None: index_dir = Path(run_root) / "coverage" diff --git a/bbdev/api/steps/uvm/scripts/config.py b/bbdev/api/steps/uvm/scripts/config.py index 248b574cb..20946d672 100644 --- a/bbdev/api/steps/uvm/scripts/config.py +++ b/bbdev/api/steps/uvm/scripts/config.py @@ -4,7 +4,7 @@ from google.protobuf.json_format import MessageToDict -def profiles(repo, design, ip): +def profiles(repo, design, selection, ip): repo = Path(repo) sys.path.insert(0, str(repo / "bbdev/api/steps/config/scripts")) import design_pb2 @@ -14,7 +14,14 @@ def profiles(repo, design, ip): path = default_ip_config(repo, ip) else: - path = repo / "examples/designs" / design / "output/generated/design.pb" + path = ( + repo + / "examples/designs" + / design + / "output/sims" + / selection + / "generated/design.pb" + ) config = design_pb2.Design.FromString(path.read_bytes()) entries = decode_profiles(config) selected = [entry for entry in entries if ip is None or entry["ip"] == ip] diff --git a/bbdev/api/steps/uvm/scripts/report.py b/bbdev/api/steps/uvm/scripts/report.py index cf09b7d50..79183b6fd 100644 --- a/bbdev/api/steps/uvm/scripts/report.py +++ b/bbdev/api/steps/uvm/scripts/report.py @@ -5,21 +5,21 @@ from .uvm_common import dashboard_summary, load_ip -def verification_targets(repo, design, ball=None, ip=None): +def verification_targets(repo, design, selection, ball=None, ip=None): if ip is not None: return [ f"ip/{ip}/{item['name']}" - for profile in load_ip(str(repo), ip, design) + for profile in load_ip(str(repo), ip, design, selection) for item in profile["targets"] ] if ball is not None: return [f"ball/{ball}"] - config = load_design_config(repo, design).verification + config = load_design_config(repo, design, selection=selection).verification targets = [f"ball/{ball}" for ball in config.balls] for ip in config.ips: targets.extend( f"ip/{ip}/{target['name']}" - for profile in load_ip(str(repo), ip, design=design) + for profile in load_ip(str(repo), ip, design, selection) for target in profile["targets"] ) return targets diff --git a/bbdev/api/steps/uvm/scripts/run.py b/bbdev/api/steps/uvm/scripts/run.py index 6fb3dcfc2..f505dbcc4 100644 --- a/bbdev/api/steps/uvm/scripts/run.py +++ b/bbdev/api/steps/uvm/scripts/run.py @@ -159,8 +159,8 @@ def run_profile(repo, design, profile, directory, do_run, ctx): } -def run_verification(repo, design, ip, ctx, do_run): - selected = profiles(repo, design, ip) +def run_verification(repo, design, selection, ip, ctx, do_run): + selected = profiles(repo, design, selection, ip) return run_suite(repo, design, selected, ip, ctx, do_run) diff --git a/bbdev/api/steps/uvm/scripts/uvm_common.py b/bbdev/api/steps/uvm/scripts/uvm_common.py index fd504c971..543bfed02 100644 --- a/bbdev/api/steps/uvm/scripts/uvm_common.py +++ b/bbdev/api/steps/uvm/scripts/uvm_common.py @@ -13,7 +13,7 @@ if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path, log_dir +from utils.path import design_arch_root, get_buckyball_path, log_dir from steps.mill.scripts.mill import mill_cmd from utils.stream_run import stream_run_logger @@ -23,31 +23,17 @@ sys.path.insert(0, config_scripts) -def load_design(bbdir: str, design: str): - import design_pb2 +def load_design(bbdir: str, design: str, selection: str): + from utils.path import load_design_config - path = ( - Path(bbdir) - / "examples" - / "designs" - / design - / "output" - / "generated" - / "design.pb" - ) - if not path.is_file(): - raise FileNotFoundError(f"missing {path}; run bbdev config --install") - msg = design_pb2.Design() - msg.ParseFromString(path.read_bytes()) + msg = load_design_config(bbdir, design, selection=selection) if not msg.name or not msg.device.cores: - raise ValueError(f"empty design.pb: {path}") + raise ValueError(f"empty design.pb of {design} {selection}") return msg -def load_design_uvm(bbdir: str, design: str) -> dict[str, list[str]]: - from utils.path import load_design_config - - msg = load_design_config(bbdir, design) +def load_design_uvm(bbdir: str, design: str, selection: str) -> dict[str, list[str]]: + msg = load_design(bbdir, design, selection) config = {"balls": list(msg.verification.balls), "ips": list(msg.verification.ips)} if not config["balls"] and not config["ips"]: raise ValueError(f"design {design}: protobuf has no UVM targets") @@ -75,7 +61,7 @@ def ball_domain(design, ball: str | None = None): (domain, mapping) for domain in domains for mapping in domain.balls - if mapping.ball_dir == ball + if mapping.component == ball ] if not matches: raise ValueError(f"ball {ball!r} not in design.pb") @@ -89,7 +75,7 @@ def selected_mappings(domain, ball: str | None): if ball is None: return list(domain.balls) for m in domain.balls: - if m.ball_dir == ball: + if m.component == ball: return [m] raise ValueError(f"ball {ball!r} not in design.pb") @@ -102,7 +88,7 @@ def ball_owner(design, ball): owners.extend( (acc, core) for acc in accelerators - if any(item.ball_dir == ball for item in acc.balldomain.balls) + if any(item.component == ball for item in acc.balldomain.balls) ) if not owners: raise ValueError(f"ball {ball!r} not in design.pb") @@ -123,7 +109,7 @@ def ball_owner(design, ball): def vcs_defines(domain, mapping, thread, core, parameters): shared_bank_num = ( - core.shared_mem.entries // thread.mem.bank.entries + core.shared_mem.entries // core.shared_mem.bank_entries if core.shared_mem.enable else 0 ) @@ -167,11 +153,13 @@ def _filelist( return str(dst) -def load_ip(bbdir: str, name: str, design: str | None = None) -> list[dict]: +def load_ip( + bbdir: str, name: str, design: str | None, selection: str | None +) -> list[dict]: from verification import default_ip_config if design is not None: - msg = load_design(bbdir, design) + msg = load_design(bbdir, design, selection) else: import design_pb2 @@ -368,9 +356,11 @@ def check_uvm_result( ) -def build_ip(bbdir: str, design: str | None, name: str, ctx) -> dict: +def build_ip( + bbdir: str, design: str | None, selection: str | None, name: str, ctx +) -> dict: targets = [] - for config in load_ip(bbdir, name, design): + for config in load_ip(bbdir, name, design, selection): build_ip_profile(bbdir, design, name, ctx, config) targets.extend(config["targets"]) return {"targets": targets} @@ -628,10 +618,12 @@ def report_ip_coverage( ) -def run_ip(bbdir: str, design: str | None, name: str, ctx, cov_root: str) -> dict: +def run_ip( + bbdir: str, design: str | None, selection: str | None, name: str, ctx, cov_root: str +) -> dict: results = [ run_ip_profile(bbdir, design, name, ctx, cov_root, config) - for config in load_ip(bbdir, name, design) + for config in load_ip(bbdir, name, design, selection) ] return {"design": design, "results": results, "failures": []} @@ -727,21 +719,17 @@ def run_ip_profile( def build_ball( - bbdir: str, design_name: str, mill_cfg: str, domain, mapping, ctx + bbdir: str, design_name: str, selection: str, mill_cfg: str, domain, mapping, ctx ) -> None: - ball = mapping.ball_dir - verify_dir = Path(bbdir) / "examples" / "components" / "balls" / ball / "verify" + ball = mapping.component + verify_dir = Path(bbdir) / "examples" / "prefab" / "balls" / ball / "verify" casegen = verify_dir / "casegen" / "Cargo.toml" - rtl_dir = ( - Path(bbdir) - / "examples/designs" - / design_name - / "output/sims/device/arch" - / mill_cfg + rtl_dir = Path(design_arch_root(bbdir, design_name, selection=selection)) / ( + f"verilog/{mill_cfg}" ) - design = load_design(bbdir, design_name) + design = load_design(bbdir, design_name, selection) thread, core = ball_owner(design, ball) - sim_dir = verify_dir.parent / "output" / "uvm" / design_name + sim_dir = verify_dir.parent / "output" / "uvm" / design_name / selection uvm_rel = os.path.relpath(Path(bbdir) / "verify" / "uvm", verify_dir) flist = _filelist(verify_dir, ball, uvm_rel, rtl_dir, sim_dir) parameters_path = verify_dir / "parameters.toml" @@ -799,12 +787,19 @@ def build_ball( def run_ball( - bbdir: str, design_name: str, mill_cfg: str, domain, mapping, ctx, cov_dir: str + bbdir: str, + design_name: str, + selection: str, + mill_cfg: str, + domain, + mapping, + ctx, + cov_dir: str, ) -> None: - ball = mapping.ball_dir - verify_dir = Path(bbdir) / "examples" / "components" / "balls" / ball / "verify" - simv = verify_dir.parent / "output" / "uvm" / design_name / "simv" - build_ball(bbdir, design_name, mill_cfg, domain, mapping, ctx) + ball = mapping.component + verify_dir = Path(bbdir) / "examples" / "prefab" / "balls" / ball / "verify" + simv = verify_dir.parent / "output" / "uvm" / design_name / selection / "simv" + build_ball(bbdir, design_name, selection, mill_cfg, domain, mapping, ctx) crate = tomllib.loads((verify_dir / "casegen" / "Cargo.toml").read_text())[ "package" ]["name"] @@ -817,12 +812,17 @@ def run_ball( ) test = f"{ball}_ball_test" verify_config = ( - verify_dir.parent / "output" / "uvm" / design_name / "verify_config.env" + verify_dir.parent + / "output" + / "uvm" + / design_name + / selection + / "verify_config.env" ) - design = load_design(bbdir, design_name) + design = load_design(bbdir, design_name, selection) thread, _ = ball_owner(design, ball) verify_config.write_text( - f"bank_entries={thread.mem.bank.entries}\nball_id={mapping.ball_id}\n" + f"bank_entries={thread.mem.bank.entries}\nball_id={mapping.id}\n" ) simv_q = shlex.quote(str(simv)) script = ( @@ -832,7 +832,7 @@ def run_ball( f"BB_VERIFY_CONFIG={shlex.quote(str(verify_config))} " f'"$loader" --library-path "$library_path" {simv_q} -no_save ' f"-sv_lib {shlex.quote(str(dpi))} " - f"+UVM_TESTNAME={shlex.quote(test)} +BID={mapping.ball_id} " + f"+UVM_TESTNAME={shlex.quote(test)} +BID={mapping.id} " f"-cm line+cond+tgl+assert -cm_name {shlex.quote(test)}" ) cmd = f"nix develop {shlex.quote(str(Path(bbdir) / 'verify'))} --command zsh -ic {shlex.quote(script)}" @@ -880,10 +880,12 @@ def dashboard_summary(cov_dir: str) -> dict[str, str]: return dict(zip(keys, vals)) -def run_design(bbdir: str, design: str, ball: str | None, ctx, do_run: bool) -> dict: - msg = load_design(bbdir, design) +def run_design( + bbdir: str, design: str, selection: str, ball: str | None, ctx, do_run: bool +) -> dict: + msg = load_design(bbdir, design, selection) domain = ball_domain(msg, ball) - mill_cfg = msg.simulations["device"].top + mill_cfg = msg.simulation.top if not mill_cfg: raise ValueError(f"design {design!r} has no sims.verilator for UVM") maps = selected_mappings(domain, ball) @@ -892,22 +894,24 @@ def run_design(bbdir: str, design: str, ball: str | None, ctx, do_run: bool) -> run_root = None if do_run: stamp = datetime.now().strftime("%Y-%m-%d-%H-%M") - run_name = "all" if ball is None else maps[0].ball_dir - run_root = log_dir(bbdir, design, "verilog", stamp, "uvm", run_name) + run_name = "all" if ball is None else maps[0].component + run_root = log_dir( + bbdir, design, "verilog", stamp, "uvm", run_name, selection=selection + ) for m in maps: if do_run: cov = ( - os.path.join(run_root, m.ball_dir, "coverage") + os.path.join(run_root, m.component, "coverage") if ball is None else os.path.join(run_root, "coverage") ) - run_ball(bbdir, design, mill_cfg, domain, m, ctx, cov) + run_ball(bbdir, design, selection, mill_cfg, domain, m, ctx, cov) if not (Path(cov) / "dashboard.txt").is_file(): raise RuntimeError(f"urg wrote no dashboard.txt under {cov}") - covs.append((m.ball_dir, cov)) + covs.append((m.component, cov)) else: - build_ball(bbdir, design, mill_cfg, domain, m, ctx) - ran.append(m.ball_dir) + build_ball(bbdir, design, selection, mill_cfg, domain, m, ctx) + ran.append(m.component) info = {"design": design, "mill": mill_cfg, "balls": ran} if do_run and ball is None: index_dir = Path(run_root) / "coverage" @@ -931,13 +935,14 @@ def run_design(bbdir: str, design: str, ball: str | None, ctx, do_run: bool) -> def run_uvm( bbdir: str, design: str, + selection: str, ball: str | None, ip: str | None, ctx, do_run: bool, ) -> dict: if ball is not None: - return run_design(bbdir, design, ball, ctx, do_run) + return run_design(bbdir, design, selection, ball, ctx, do_run) if ip is not None: if do_run: stamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") @@ -949,19 +954,19 @@ def run_uvm( "uvm", ip, ) - return run_ip(bbdir, design, ip, ctx, run_root) - config = build_ip(bbdir, design, ip, ctx) + return run_ip(bbdir, design, selection, ip, ctx, run_root) + config = build_ip(bbdir, design, selection, ip, ctx) return { "design": design, "ip": ip, "targets": [target["name"] for target in config["targets"]], } - targets = load_design_uvm(bbdir, design) + targets = load_design_uvm(bbdir, design, selection) results, failures = [], [] for ball in targets["balls"]: try: - results.append(run_design(bbdir, design, ball, ctx, do_run)) + results.append(run_design(bbdir, design, selection, ball, ctx, do_run)) except Exception as error: if not do_run: raise @@ -979,9 +984,11 @@ def run_uvm( "uvm", ip, ) - results.extend(run_ip(bbdir, design, ip, ctx, run_root)["results"]) + results.extend( + run_ip(bbdir, design, selection, ip, ctx, run_root)["results"] + ) else: - config = build_ip(bbdir, design, ip, ctx) + config = build_ip(bbdir, design, selection, ip, ctx) results.append( { "design": design, diff --git a/bbdev/api/steps/workload/01_build_event.step.py b/bbdev/api/steps/workload/01_build_event.step.py index aefb3ef27..cc8a8dac1 100644 --- a/bbdev/api/steps/workload/01_build_event.step.py +++ b/bbdev/api/steps/workload/01_build_event.step.py @@ -8,20 +8,9 @@ utils_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) if utils_path not in sys.path: sys.path.insert(0, utils_path) -from utils.path import get_buckyball_path +from utils.path import get_buckyball_path, workload_build_dir from utils.event_common import check_result, get_origin_trace_id -_workload_build_path = os.path.join( - get_buckyball_path(), "bb-tests", "workloads", "scripts", "build.py" -) -_spec = importlib.util.spec_from_file_location( - "workload_build_module", _workload_build_path -) -if _spec is None or _spec.loader is None: - raise RuntimeError(f"cannot load {_workload_build_path}") -workload_build = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(workload_build) - _stack_build_path = os.path.join( os.path.dirname(__file__), "..", "stack", "scripts", "build.py" ) @@ -118,22 +107,45 @@ async def handler(input_data: dict, ctx: FlowContext) -> None: return try: # Workloads link the device runtime; MLIR tests compile with its plugins. - stack_build.build_design( + design_cmake = stack_build.build_design( bbdir, design=design, selection=input_data["target"], logger=ctx.logger, task_scope=origin_tid, ) - workload_build.build_workload( - bbdir, - design, - selection=input_data["target"], - ctest=ctest, - mlirtest=mlirtest, - soctest=soctest, + build = workload_build_dir(bbdir, design, selection=input_data["target"]) + stack_build.run( + [ + "cmake", + "-G", + "Ninja", + "-S", + os.path.join(bbdir, "bb-tests", "workloads"), + "-B", + build, + f"-DDESIGN_CMAKE={design_cmake}", + ], + repo=Path(bbdir), + logger=ctx.logger, + task_scope=origin_tid, + output_prefix="workload configure", + ) + target = ( + "sync-ctest-bin" + if ctest + else ( + "sync-mlirtest-bin" + if mlirtest + else "sync-soctest-bin" if soctest else "sync-bin" + ) + ) + stack_build.run( + ["ninja", "-C", build, "-j", "1", target], + repo=Path(bbdir), logger=ctx.logger, task_scope=origin_tid, + output_prefix="workload build", ) except Exception as error: ctx.logger.error(str(error)) diff --git a/bbdev/api/utils/event_common.py b/bbdev/api/utils/event_common.py index d307a1e3d..1f7cd1113 100644 --- a/bbdev/api/utils/event_common.py +++ b/bbdev/api/utils/event_common.py @@ -4,6 +4,7 @@ import re +from utils.path import design_pb_path, get_buckyball_path from utils.process_registry import set_current_task_scope @@ -22,22 +23,29 @@ def require_design(data: dict) -> str: def require_target(data: dict, kinds=("device", "core")) -> str: + """The selection of a request: device/ for `--device `, a + config registered by the design's sims config, or core/.""" selected = [kind for kind in ("device", "core", "ip", "ball") if kind in data] if len(selected) != 1: raise ValueError( - "Specify exactly one of --device, --core , --ip , --ball after --design" + "Specify exactly one of --device , --core , --ip , --ball after --design" ) kind = selected[0] if kind not in kinds: raise ValueError(f"--{kind} is not supported by this operation") value = data[kind] - if kind == "device": - if value is not True: - raise ValueError("--device is a flag and takes no value") - return "device" if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", value): - raise ValueError(f"--{kind} requires an entry name") - return f"{kind}/{value}" + raise ValueError(f"--{kind} requires a name") + if kind != "device": + return f"{kind}/{value}" + design = require_design(data) + pb = design_pb_path(get_buckyball_path(), design, selection=f"device/{value}") + if not pb.is_file(): + raise ValueError( + f"design {design!r} registers no device config {value!r}" + " (run bbdev config --install after editing its sims config)" + ) + return f"device/{value}" def get_origin_trace_id(input_data, ctx): diff --git a/bbdev/api/utils/path.py b/bbdev/api/utils/path.py index 006471f52..118fc3d6c 100644 --- a/bbdev/api/utils/path.py +++ b/bbdev/api/utils/path.py @@ -92,30 +92,27 @@ def design_arch_root(bbdir, design, *, selection): ) -def load_design_config(bbdir, design): +def design_pb_path(bbdir, design, *, selection): + """The design.pb of one simulation: the design with its config's overrides.""" + return Path(design_output_root(bbdir, design, selection=selection)) / ( + "generated/design.pb" + ) + + +def load_design_config(bbdir, design, *, selection): sys.path.insert(0, str(Path(bbdir) / "bbdev/api/steps/config/scripts")) import design_pb2 - path = ( - Path(bbdir) - / "examples/designs" - / _design_name(design) - / "output/generated/design.pb" - ) + path = design_pb_path(bbdir, design, selection=selection) return design_pb2.Design.FromString(path.read_bytes()) def simulation_target(bbdir, design, selection): - config = load_design_config(bbdir, design) - if selection not in config.simulations: - raise ValueError( - f"design {design!r} has no registered simulation entry {selection!r}" - ) - return config.simulations[selection] + return load_design_config(bbdir, design, selection=selection).simulation def runner_manifest(bbdir, design, selection): - """The simulation entry's emu crate: its model and the runners built from it.""" + """The emu crate of the simulation's entry: its model and the runners built from it.""" return Path(bbdir) / simulation_target(bbdir, design, selection).emu / "Cargo.toml" diff --git a/bbdev/api/utils/reports.py b/bbdev/api/utils/reports.py index 5d03336c9..221584b17 100644 --- a/bbdev/api/utils/reports.py +++ b/bbdev/api/utils/reports.py @@ -4,7 +4,7 @@ import subprocess from datetime import datetime, timezone from pathlib import Path -from utils.path import _design_name +from utils.path import _design_name, design_pb_path from utils.report_schema import ( report_directory, @@ -13,12 +13,10 @@ ) -def source_context(repo, design): +def source_context(repo, design, selection): repo = Path(repo) design = _design_name(design) - config = ( - repo / "examples/designs" / design / "output/generated/design.pb" - ).read_bytes() + config = design_pb_path(repo, design, selection=selection).read_bytes() def git(*args): return subprocess.check_output(["git", *args], cwd=repo, text=True).strip() diff --git a/bbdev/api/utils/workload_manifest.py b/bbdev/api/utils/workload_manifest.py index d1902f1e1..59ca1efcf 100644 --- a/bbdev/api/utils/workload_manifest.py +++ b/bbdev/api/utils/workload_manifest.py @@ -16,10 +16,6 @@ def validate_design(design: str | None) -> str: def design_regression_dir( design: str, backend: str, bbdir: str | None = None, *, selection: str ) -> Path: - if selection != "device": - raise ValueError( - "batch regression requires --device; use a single workload for --core" - ) design = validate_design(design) root = Path(bbdir or get_buckyball_path()) return ( diff --git a/bbdev/bbdev b/bbdev/bbdev index 1559d5e67..6f5031086 100755 --- a/bbdev/bbdev +++ b/bbdev/bbdev @@ -30,182 +30,181 @@ workspace_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) WORKFLOW_COMMANDS = { "verilator": { "clean": ( - 'Clean build directory. Args: "--design (--device | --core' + 'Clean build directory. Args: "--design (--device | --core' ' )"' ), "verilog": ( - 'Generate verilog files. Args: "--design (--device | --core' + 'Generate verilog files. Args: "--design (--device | --core' ' )"' ), "build": ( - 'Build simulation executable. Args: "--design (--device | --core' + 'Build simulation executable. Args: "--design (--device | --core' ' ) [--jobs ]"' ), "sim": ( "Run a built simulation on a baremetal workload of the entry. Args:" - ' "--design (--device | --core ) --binary ' + ' "--design (--device | --core ) --binary ' " [--batch] [--no-wave] [--coverage] [--itrace] [--mtrace] [--pmctrace]" ' [--ctrace] [--banktrace]"' ), "run": ( - 'Generate RTL, build and run. Args: "--design (--device | --core' + 'Generate RTL, build and run. Args: "--design (--device | --core' " ) --binary [--jobs ] [--batch] [--no-wave]" ' [--coverage] [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--banktrace]"' ), }, "vcs": { "clean": ( - 'Clean the entry VCS build. Args: "--design (--device | --core' + 'Clean the entry VCS build. Args: "--design (--device | --core' ' )"' ), "verilog": ( - 'Generate the entry tapeout RTL for VCS. Args: "--design (--device' + 'Generate the entry tapeout RTL for VCS. Args: "--design (--device ' ' | --core )"' ), "build": ( - 'Build the entry VCS simv. Args: "--design (--device | --core' + 'Build the entry VCS simv. Args: "--design (--device | --core' ' ) [--jobs ]"' ), "sim": ( 'Run a built simv on a baremetal workload of the entry. Args: "--design' - " (--device | --core ) --binary [--batch]" + " (--device | --core ) --binary [--batch]" ' [--wave]"' ), "run": ( - 'Generate RTL, build and run VCS. Args: "--design (--device |' + 'Generate RTL, build and run VCS. Args: "--design (--device |' ' --core ) --binary [--jobs ] [--batch]"' ), }, "workload": { "clean": ( - 'Clean workload output directory. Args: "--design (--device |' + 'Clean workload output directory. Args: "--design (--device |' ' --core )"' ), "build": ( - 'Build test workloads. Args: "--design (--device | --core )' + 'Build test workloads. Args: "--design (--device | --core )' ' [--ctest|--mlirtest|--soctest]"' ), "tohex": ( - 'Convert design workload ELF to hex. Args: "--design (--device |' + 'Convert design workload ELF to hex. Args: "--design (--device |' ' --core )"' ), }, "model": { "build": ( "Build model recipes on the device of a design (all its models when --model" - ' is omitted). Args: "--design [--model ]"' + ' is omitted). Args: "--design --device [--model ]"' ), }, "kernel": { "build": ( "Build RISC-V kernel + rootfs carrying the Linux programs of a design." - ' Args: "--design --device [--interactive]"' + ' Args: "--design --device [--interactive]"' ), }, "firesim": { "enumeratefpgas": ( - 'Enumerate FPGAs. Args: "--design (--device | --core )"' + 'Enumerate FPGAs. Args: "--design (--device | --core )"' ), "buildbitstream": ( - 'Build bitstream. Args: "--design (--device | --core )"' + 'Build bitstream. Args: "--design (--device | --core )"' ), "infrasetup": ( - 'Infrasetup. Args: "--design (--device | --core )"' + 'Infrasetup. Args: "--design (--device | --core )"' ), "runworkload": ( - 'Run workload. Args: "--design (--device | --core )"' + 'Run workload. Args: "--design (--device | --core )"' ), }, "stack": { "build": ( - 'Build a registered simulation stack. Args: "--design (--device |' + 'Build a registered simulation stack. Args: "--design (--device |' ' --core )"' ), }, "bebop-bemu": { "sim": ( - 'Run Bebop emulator simulation. Args: "--design (--device | --core' + 'Run Bebop emulator simulation. Args: "--design (--device | --core' " ) --binary [--disasm] [--tool-profile] [--itrace]" - ' [--mtrace]"; resolves files in the selected design workload/kernel' + ' [--mtrace] [--ctrace]"; resolves files in the selected design workload/kernel' " outputs" ), "analysis": ( 'Analyze itrace/mtrace in an existing log dir. Args: "--design ' - ' (--device | --core ) --run --itrace|--mtrace"' + ' (--device | --core ) --run --itrace|--mtrace"' ), "batch": ( - 'Run Bebop BEMU batch regression. Args: "--design --device --test' - ' [--jobs ] [--clean-before]"' + 'Run Bebop BEMU batch regression. Args: "--design --device --test' + ' [--jobs ]"' ), }, "bebop-verilator": { "clean": ( - 'Clean the entry RTL. Args: "--design (--device | --core )"' + 'Clean the entry RTL. Args: "--design (--device | --core )"' ), "verilog": ( - 'Generate the entry RTL. Args: "--design (--device | --core' + 'Generate the entry RTL. Args: "--design (--device | --core' ' ) [--diff]"' ), "build": ( - 'Build the entry Verilator runner. Args: "--design (--device |' + 'Build the entry Verilator runner. Args: "--design (--device |' ' --core ) [--diff] [--jobs ]"' ), "sim": ( 'Run a built runner on a workload of the entry. Args: "--design ' - " (--device | --core ) --binary [--diff] [--no-wave]" + " (--device | --core ) --binary [--diff] [--no-wave]" ' [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--banktrace]"' ), "run": ( - 'Generate RTL, build and run. Args: "--design (--device | --core' + 'Generate RTL, build and run. Args: "--design (--device | --core' " ) --binary [--diff] [--jobs ] [--no-wave]" ' [--itrace] [--mtrace] [--pmctrace] [--ctrace] [--banktrace]"' ), "batch": ( - 'Run the entry regression. Args: "--design --device --test' - " [--diff] [--jobs ] [--clean-before]" - ' [--junit ]"' + 'Run the entry regression. Args: "--design --device --test' + ' [--diff] [--jobs ]"' ), }, "bebop-vcs": { "sim": ( 'Run one SoC workload through the Bebop VCS node. Args: "--design ' - ' (--device | --core ) --binary [--wave]"' + ' (--device | --core ) --binary [--wave]"' ), "verilog": ( - 'Generate Bebop VCS RTL once. Args: "--design (--device | --core' + 'Generate Bebop VCS RTL once. Args: "--design (--device | --core' ' )"' ), "build": ( - 'Build the reusable Bebop VCS simv. Args: "--design (--device |' + 'Build the reusable Bebop VCS simv. Args: "--design (--device |' ' --core ) [--jobs ]"' ), "batch": ( - 'Run design-owned VCS regression. Args: "--design --device --test' - ' [--jobs ] [--junit ]"' + 'Run design-owned VCS regression. Args: "--design --device --test' + ' [--jobs ]"' ), }, "bebop-p2e": { "clean": ( - 'Clean Bebop P2E build directory. Args: "--design (--device |' + 'Clean Bebop P2E build directory. Args: "--design (--device |' ' --core ) [--output-dir ]"' ), "verilog": ( 'Generate Bebop P2E SystemVerilog for DDR4 backdoor. Args: "--design' - ' (--device | --core ) [--diff] [--output-dir ]"' + ' (--device | --core ) [--diff] [--output-dir ]"' ), "buildbitstream": ( - 'Build Bebop P2E runtime case. Args: "--design (--device | --core' + 'Build Bebop P2E runtime case. Args: "--design (--device | --core' " ) [--diff] [--itrace] [--mtrace] [--vsrc-dir ] [--output-dir" ' ]"' ), "runworkload": ( - 'Cold-load Bebop P2E workload. Args: "--design (--device | --core' + 'Cold-load Bebop P2E workload. Args: "--design (--device | --core' " ) --image --bitstream [--fpga-location" " ] [--multi-fpga] [--wave] [--wave-start ] [--diff] [--itrace]" ' [--mtrace] [--pmctrace] [--ctrace] [--reuse-runtime]"' ), "batch": ( - 'Run Bebop P2E batch regression. Args: "--design --device' + 'Run Bebop P2E batch regression. Args: "--design --device ' " --bitstream --test [--diff]" ' [--fpga-location ]"' ), @@ -213,17 +212,17 @@ WORKFLOW_COMMANDS = { "dc": { "verilog": ( "Generate top-scoped DC RTL under arch/build//. Args:" - ' "--design (--device | --core ) [--top ]"' + ' "--design (--device | --core ) [--top ]"' ), "area": ( "Generate RTL, run DC synthesis, and report area under" ' log////area. Args: "--design ' - ' (--device | --core ) [--top ]".' + ' (--device | --core ) [--top ]".' ), "power": ( "Generate RTL, run design-owned power simulation, then run PrimeTime PX" ' under log////power. Args: "--design ' - " (--device | --core ) [--top ] [--workload | --core ) [--top ] [--workload ] [--start-ns ] [--end-ns ] [--activity --format" ' ]".' ), @@ -231,24 +230,24 @@ WORKFLOW_COMMANDS = { "ip": { "generate": ( "Generate SRAM macros from elaborator mems.conf under" - ' arch/build//. Args: "--design (--device | --core' + ' arch/build//. Args: "--design (--device | --core' " ) [--consumer ] [--top ] [--output-dir ]" ' [--next-topic ]"' ), "replace": ( "Assemble top-scoped synthesis RTL plus SRAM macros under" - ' arch/build//. Args: "--design (--device | --core' + ' arch/build//. Args: "--design (--device | --core' " ) [--consumer ] [--top ] [--output-dir ]" ' [--next-topic ]"' ), }, "yosys": { "run": ( - 'Run yosys flow. Args: "[--top ] [--design (--device |' + 'Run yosys flow. Args: "[--top ] [--design (--device |' ' --core )] [--output-dir ] [--vcd ]"' ), "verilog": ( - 'Generate verilog for yosys flow. Args: "[--design (--device |' + 'Generate verilog for yosys flow. Args: "[--design (--device |' ' --core )] [--output-dir ]"' ), "synth": ( @@ -262,18 +261,18 @@ WORKFLOW_COMMANDS = { ' "' ), "build": ( - 'Build a Ball or IP UVM simulation. Args: "[--design ] (--device |' - ' --ball | --ip )"' + 'Build a Ball or IP UVM simulation. Args: "[--design --device' + ' ] [--ball | --ip ]"' ), "run": ( - 'Build and run a Ball or IP UVM simulation. Args: "[--design ]' - ' (--device | --ball | --ip )"' + 'Build and run a Ball or IP UVM simulation. Args: "[--design ' + ' --device ] [--ball | --ip ]"' ), }, "config": { "install": ( - 'Install design configurations. Args: "[--design (--device |' - ' --core )]"' + "Install design configurations: one design.pb per device config and core" + ' entry. Args: "[--design ]"' ), }, } @@ -369,7 +368,12 @@ def parse_args(argv: list[str]) -> argparse.Namespace: parser = argparse.ArgumentParser( prog="bbdev", - description="Development tool for buckyball project", + description=( + "Development tool for buckyball project.\n" + "--device names a device config registered in the design's sims" + " config\n(configs/sims/default.toml [targets.device.configs]), e.g." + " --device PolyVerilatorConfig." + ), formatter_class=argparse.RawTextHelpFormatter, ) diff --git a/bbdev/mcp/common.py b/bbdev/mcp/common.py index 4371a17ce..782017bfa 100644 --- a/bbdev/mcp/common.py +++ b/bbdev/mcp/common.py @@ -372,7 +372,7 @@ def _load_toml(path: Path) -> Dict[str, Any]: return data -def _balldomain_path(design: str, balldomain: Optional[str]) -> Path: +def _balldomain_path(design: str, device: str, balldomain: Optional[str]) -> Path: if balldomain: raw = Path(balldomain) if raw.is_absolute() or len(raw.parts) > 1: @@ -382,7 +382,9 @@ def _balldomain_path(design: str, balldomain: Optional[str]) -> Path: / "examples" / "designs" / design - / "output/generated/config/derived.json" + / "output/sims/device" + / device + / "generated/config/derived.json" ) cfg = json.loads(derived.read_text(encoding="utf-8")) domains = {thread["balldomain_base_dir"] for thread in cfg["threads"]} @@ -420,7 +422,9 @@ def _validate_domain(path: Path, cfg: Dict[str, Any]) -> Dict[str, Any]: } -def _validate_design(design: str, balldomain: Optional[str]) -> Dict[str, Any]: +def _validate_design( + design: str, device: str, balldomain: Optional[str] +) -> Dict[str, Any]: scripts = repo_path() / "bbdev/api/steps/config/scripts" sys.path.insert(0, str(scripts)) from ball_normalize import normalize_ball_domain @@ -434,10 +438,14 @@ def _validate_design(design: str, balldomain: Optional[str]) -> Dict[str, Any]: repo_path() / "examples/designs" / design - / "output/generated/config/config.json" + / "output/sims/device" + / device + / "generated/config/config.json" ) config = json.loads(expanded.read_text(encoding="utf-8")) - selected = _balldomain_path(design, balldomain) if balldomain is not None else None + selected = ( + _balldomain_path(design, device, balldomain) if balldomain is not None else None + ) results = [] for index, thread in enumerate(topology.iter_threads(config["device"])): raw = thread.get("balldomain") diff --git a/bbdev/mcp/server.py b/bbdev/mcp/server.py index c0918605c..96fa4caa7 100644 --- a/bbdev/mcp/server.py +++ b/bbdev/mcp/server.py @@ -6,7 +6,14 @@ from tools import register_all -mcp = FastMCP("buckyball-dev") +mcp = FastMCP( + "buckyball-dev", + instructions=( + "`device` names a device config registered in the design's" + " configs/sims/default.toml [targets.device.configs], e.g." + " PolyVerilatorConfig; `core` names a core simulation entry." + ), +) register_all(mcp) diff --git a/bbdev/mcp/tools/bebop_p2e_batch.py b/bbdev/mcp/tools/bebop_p2e_batch.py index f8f352d97..4684a6d2a 100644 --- a/bbdev/mcp/tools/bebop_p2e_batch.py +++ b/bbdev/mcp/tools/bebop_p2e_batch.py @@ -9,7 +9,11 @@ def register(mcp): @mcp.tool() def bbdev_bebop_p2e_batch( - design: str, bitstream: str, test: str, diff: bool = False, device: bool = False + design: str, + bitstream: str, + test: str, + diff: bool = False, + device: str | None = None, ) -> str: """Batch bebop-p2e regression. test: bare-tests|linux-tests. POST /bebop/p2e/batch.""" for n, v in (("design", design), ("bitstream", bitstream)): @@ -23,8 +27,8 @@ def bbdev_bebop_p2e_batch( "test": test, "diff": diff, } - if device: - request["device"] = True + if device is not None: + request["device"] = device return fmt( submit( "/bebop/p2e/batch", diff --git a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py index bd8d5c6c0..d02796f23 100644 --- a/bbdev/mcp/tools/bebop_p2e_buildbitstream.py +++ b/bbdev/mcp/tools/bebop_p2e_buildbitstream.py @@ -16,7 +16,7 @@ def bbdev_bebop_p2e_buildbitstream( mtrace: bool = False, vsrc_dir: Optional[str] = None, output_dir: Optional[str] = None, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Build bebop-p2e bitstream. POST /bebop/p2e/buildbitstream.""" @@ -27,8 +27,8 @@ def bbdev_bebop_p2e_buildbitstream( vsrc_dir=vsrc_dir, output_dir=output_dir, ) - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt( diff --git a/bbdev/mcp/tools/bebop_p2e_clean.py b/bbdev/mcp/tools/bebop_p2e_clean.py index 58c07f1f0..657a11bd3 100644 --- a/bbdev/mcp/tools/bebop_p2e_clean.py +++ b/bbdev/mcp/tools/bebop_p2e_clean.py @@ -9,14 +9,14 @@ def register(mcp): @mcp.tool() def bbdev_bebop_p2e_clean( - design: str, device: bool = False, core: str | None = None + design: str, device: str | None = None, core: str | None = None ) -> str: """Clean bebop-p2e build. POST /bebop/p2e/clean.""" if e := need("design", design): return err(e) request = {"design": design} - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt(submit("/bebop/p2e/clean", request)) diff --git a/bbdev/mcp/tools/bebop_p2e_runworkload.py b/bbdev/mcp/tools/bebop_p2e_runworkload.py index 9df2cddba..195f6d4c7 100644 --- a/bbdev/mcp/tools/bebop_p2e_runworkload.py +++ b/bbdev/mcp/tools/bebop_p2e_runworkload.py @@ -19,7 +19,7 @@ def bbdev_bebop_p2e_runworkload( ctrace: bool = False, reuse_runtime: bool = False, diff: bool = False, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Cold-load one hex image on P2E.""" @@ -43,8 +43,8 @@ def bbdev_bebop_p2e_runworkload( params["ctrace"] = True if diff: params["diff"] = True - if device: - params["device"] = True + if device is not None: + params["device"] = device if core is not None: params["core"] = core return fmt(submit("/bebop/p2e/runworkload", params)) diff --git a/bbdev/mcp/tools/bebop_p2e_verilog.py b/bbdev/mcp/tools/bebop_p2e_verilog.py index b1ec46f6b..57ef7f892 100644 --- a/bbdev/mcp/tools/bebop_p2e_verilog.py +++ b/bbdev/mcp/tools/bebop_p2e_verilog.py @@ -13,15 +13,15 @@ def bbdev_bebop_p2e_verilog( design: str, output_dir: Optional[str] = None, diff: bool = False, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Generate Verilog for bebop-p2e. POST /bebop/p2e/verilog.""" if e := need("design", design): return err(e) request = opt({"design": design, "diff": diff}, output_dir=output_dir) - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt( diff --git a/bbdev/mcp/tools/bebop_verilator_batch.py b/bbdev/mcp/tools/bebop_verilator_batch.py index cf17ee0dc..3f8faf4da 100644 --- a/bbdev/mcp/tools/bebop_verilator_batch.py +++ b/bbdev/mcp/tools/bebop_verilator_batch.py @@ -10,10 +10,8 @@ def register(mcp): def bbdev_bebop_verilator_batch( design: str, test: str, - clean_before: bool = False, diff: bool = False, - junit: str | None = None, - device: bool = False, + device: str | None = None, ) -> str: """Batch bebop-verilator regression. POST /bebop/verilator/batch.""" if e := need("design", design): @@ -23,12 +21,10 @@ def bbdev_bebop_verilator_batch( request = { "design": design, "test": test, - "clean-before": clean_before, "diff": diff, - "junit": junit, } - if device: - request["device"] = True + if device is not None: + request["device"] = device return fmt( submit( "/bebop/verilator/batch", diff --git a/bbdev/mcp/tools/bebop_verilator_build.py b/bbdev/mcp/tools/bebop_verilator_build.py index dcd093b90..209556500 100644 --- a/bbdev/mcp/tools/bebop_verilator_build.py +++ b/bbdev/mcp/tools/bebop_verilator_build.py @@ -12,15 +12,15 @@ def bbdev_bebop_verilator_build( design: str, jobs: int = 16, diff: bool = False, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Build bebop verilator binary. POST /bebop/verilator/build.""" if e := need("design", design): return err(e) request = {"design": design, "jobs": jobs, "diff": diff} - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt( diff --git a/bbdev/mcp/tools/bebop_verilator_clean.py b/bbdev/mcp/tools/bebop_verilator_clean.py index 72014b031..cd215bf34 100644 --- a/bbdev/mcp/tools/bebop_verilator_clean.py +++ b/bbdev/mcp/tools/bebop_verilator_clean.py @@ -9,14 +9,14 @@ def register(mcp): @mcp.tool() def bbdev_bebop_verilator_clean( - design: str, device: bool = False, core: str | None = None + design: str, device: str | None = None, core: str | None = None ) -> str: """Clean bebop-verilator build. POST /bebop/verilator/clean.""" if e := need("design", design): return err(e) request = {"design": design} - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt(submit("/bebop/verilator/clean", request)) diff --git a/bbdev/mcp/tools/bebop_verilator_run.py b/bbdev/mcp/tools/bebop_verilator_run.py index f657bcc5e..be0f6ca51 100644 --- a/bbdev/mcp/tools/bebop_verilator_run.py +++ b/bbdev/mcp/tools/bebop_verilator_run.py @@ -18,7 +18,7 @@ def bbdev_bebop_verilator_run( banktrace: bool = False, no_wave: bool = False, diff: bool = False, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Full bebop-verilator flow. POST /bebop/verilator/run.""" @@ -37,8 +37,8 @@ def bbdev_bebop_verilator_run( "no-wave": no_wave, "diff": diff, } - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt( diff --git a/bbdev/mcp/tools/bebop_verilator_sim.py b/bbdev/mcp/tools/bebop_verilator_sim.py index fb3810dc9..a0307681c 100644 --- a/bbdev/mcp/tools/bebop_verilator_sim.py +++ b/bbdev/mcp/tools/bebop_verilator_sim.py @@ -18,7 +18,7 @@ def bbdev_bebop_verilator_sim( banktrace: bool = False, no_wave: bool = False, diff: bool = False, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Run one workload on bebop-verilator. POST /bebop/verilator/sim.""" @@ -36,8 +36,8 @@ def bbdev_bebop_verilator_sim( "no-wave": no_wave, "diff": diff, } - if device: - params["device"] = True + if device is not None: + params["device"] = device if core is not None: params["core"] = core return fmt(submit("/bebop/verilator/sim", params)) diff --git a/bbdev/mcp/tools/bebop_verilator_verilog.py b/bbdev/mcp/tools/bebop_verilator_verilog.py index c0ecb2780..abe6e52c2 100644 --- a/bbdev/mcp/tools/bebop_verilator_verilog.py +++ b/bbdev/mcp/tools/bebop_verilator_verilog.py @@ -9,14 +9,17 @@ def register(mcp): @mcp.tool() def bbdev_bebop_verilator_verilog( - design: str, diff: bool = False, device: bool = False, core: str | None = None + design: str, + diff: bool = False, + device: str | None = None, + core: str | None = None, ) -> str: """Generate Verilog for bebop-verilator. POST /bebop/verilator/verilog.""" if e := need("design", design): return err(e) request = {"design": design, "diff": diff} - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt(submit("/bebop/verilator/verilog", request)) diff --git a/bbdev/mcp/tools/bemu_analysis.py b/bbdev/mcp/tools/bemu_analysis.py index f1529d824..2443319be 100644 --- a/bbdev/mcp/tools/bemu_analysis.py +++ b/bbdev/mcp/tools/bemu_analysis.py @@ -12,7 +12,7 @@ def bbdev_bemu_analysis( run: str, itrace: bool = False, mtrace: bool = False, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Analyze a run directory under the repository log/ directory. POST /bebop/bemu/analysis.""" @@ -28,8 +28,8 @@ def bbdev_bemu_analysis( "itrace": itrace, "mtrace": mtrace, } - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt( diff --git a/bbdev/mcp/tools/bemu_batch.py b/bbdev/mcp/tools/bemu_batch.py index 63c2a038d..286cf6c36 100644 --- a/bbdev/mcp/tools/bemu_batch.py +++ b/bbdev/mcp/tools/bemu_batch.py @@ -10,10 +10,8 @@ def register(mcp): def bbdev_bemu_batch( design: str, test: str, - clean_before: bool = False, jobs: int = 1, - junit: str | None = None, - device: bool = False, + device: str | None = None, ) -> str: """Batch bemu regression. test: bare-tests|linux-tests. POST /bebop/bemu/batch.""" if e := need("design", design): @@ -23,12 +21,10 @@ def bbdev_bemu_batch( request = { "design": design, "test": test, - "clean-before": clean_before, "jobs": jobs, - "junit": junit, } - if device: - request["device"] = True + if device is not None: + request["device"] = device return fmt( submit( "/bebop/bemu/batch", diff --git a/bbdev/mcp/tools/bemu_sim.py b/bbdev/mcp/tools/bemu_sim.py index 2b7f93800..e8802ee4a 100644 --- a/bbdev/mcp/tools/bemu_sim.py +++ b/bbdev/mcp/tools/bemu_sim.py @@ -18,7 +18,8 @@ def bbdev_bemu_sim( tool_profile: bool = False, itrace: bool = False, mtrace: bool = False, - device: bool = False, + ctrace: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Run a workload or kernel ELF on BEMU. POST /bebop/bemu/sim.""" @@ -32,6 +33,7 @@ def bbdev_bemu_sim( "tool-profile": tool_profile, "itrace": itrace, "mtrace": mtrace, + "ctrace": ctrace, } for key, value in ( ("dtb", dtb), @@ -40,8 +42,8 @@ def bbdev_bemu_sim( ): if value is not None: params[key] = value - if device: - params["device"] = True + if device is not None: + params["device"] = device if core is not None: params["core"] = core return fmt(submit("/bebop/bemu/sim", params)) diff --git a/bbdev/mcp/tools/dc_area.py b/bbdev/mcp/tools/dc_area.py index f86d8f7e4..20ed90b5c 100644 --- a/bbdev/mcp/tools/dc_area.py +++ b/bbdev/mcp/tools/dc_area.py @@ -12,15 +12,15 @@ def register(mcp): def bbdev_dc_area( design: Optional[str] = None, top: Optional[str] = None, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Run DC synthesis and area reporting. Defaults to DigitalTop.""" if not design: return err("design is required") request = opt({}, design=design, top=top) - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt(submit("/dc/area", request)) diff --git a/bbdev/mcp/tools/dc_power.py b/bbdev/mcp/tools/dc_power.py index cfc3fb4c7..936a15194 100644 --- a/bbdev/mcp/tools/dc_power.py +++ b/bbdev/mcp/tools/dc_power.py @@ -18,7 +18,7 @@ def bbdev_dc_power( workload: Optional[str] = None, start_ns: Optional[str] = None, end_ns: Optional[str] = None, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Run DC synthesis, design-owned activity simulation, and PrimeTime PX. @@ -41,8 +41,8 @@ def bbdev_dc_power( start_ns=start_ns, end_ns=end_ns, ) - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt( diff --git a/bbdev/mcp/tools/dc_verilog.py b/bbdev/mcp/tools/dc_verilog.py index 3b82546a6..7b0f07bd2 100644 --- a/bbdev/mcp/tools/dc_verilog.py +++ b/bbdev/mcp/tools/dc_verilog.py @@ -12,15 +12,15 @@ def register(mcp): def bbdev_dc_verilog( design: Optional[str] = None, top: Optional[str] = None, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Generate top-scoped DC RTL collateral. Defaults to DigitalTop.""" if not design: return err("design is required") request = opt({}, design=design, top=top) - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt( diff --git a/bbdev/mcp/tools/firesim_buildbitstream.py b/bbdev/mcp/tools/firesim_buildbitstream.py index 6e7198d63..1d5d83ac1 100644 --- a/bbdev/mcp/tools/firesim_buildbitstream.py +++ b/bbdev/mcp/tools/firesim_buildbitstream.py @@ -8,14 +8,14 @@ def register(mcp): @mcp.tool() def bbdev_firesim_buildbitstream( - design: str, device: bool = False, core: str | None = None + design: str, device: str | None = None, core: str | None = None ) -> str: """Build FireSim bitstream. POST /firesim/buildbitstream.""" if e := need("design", design): return err(e) request = {"design": design} - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt(submit("/firesim/buildbitstream", request)) diff --git a/bbdev/mcp/tools/firesim_enumeratefpgas.py b/bbdev/mcp/tools/firesim_enumeratefpgas.py index 4f7253723..d809abf4d 100644 --- a/bbdev/mcp/tools/firesim_enumeratefpgas.py +++ b/bbdev/mcp/tools/firesim_enumeratefpgas.py @@ -8,14 +8,14 @@ def register(mcp): @mcp.tool() def bbdev_firesim_enumeratefpgas( - design: str, device: bool = False, core: str | None = None + design: str, device: str | None = None, core: str | None = None ) -> str: """Enumerate FireSim FPGAs. POST /firesim/enumeratefpgas.""" if e := need("design", design): return err(e) request = {"design": design} - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt(submit("/firesim/enumeratefpgas", request)) diff --git a/bbdev/mcp/tools/firesim_infrasetup.py b/bbdev/mcp/tools/firesim_infrasetup.py index f9feff958..e57356e32 100644 --- a/bbdev/mcp/tools/firesim_infrasetup.py +++ b/bbdev/mcp/tools/firesim_infrasetup.py @@ -8,14 +8,14 @@ def register(mcp): @mcp.tool() def bbdev_firesim_infrasetup( - design: str, device: bool = False, core: str | None = None + design: str, device: str | None = None, core: str | None = None ) -> str: """FireSim infrasetup. POST /firesim/infrasetup.""" if e := need("design", design): return err(e) request = {"design": design} - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt(submit("/firesim/infrasetup", request)) diff --git a/bbdev/mcp/tools/firesim_runworkload.py b/bbdev/mcp/tools/firesim_runworkload.py index dede134d5..81a5502c9 100644 --- a/bbdev/mcp/tools/firesim_runworkload.py +++ b/bbdev/mcp/tools/firesim_runworkload.py @@ -8,14 +8,14 @@ def register(mcp): @mcp.tool() def bbdev_firesim_runworkload( - design: str, device: bool = False, core: str | None = None + design: str, device: str | None = None, core: str | None = None ) -> str: """Run FireSim workload. POST /firesim/runworkload.""" if e := need("design", design): return err(e) request = {"design": design} - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt(submit("/firesim/runworkload", request)) diff --git a/bbdev/mcp/tools/ip_generate.py b/bbdev/mcp/tools/ip_generate.py index 31247a50c..fcadd26ab 100644 --- a/bbdev/mcp/tools/ip_generate.py +++ b/bbdev/mcp/tools/ip_generate.py @@ -15,7 +15,7 @@ def bbdev_ip_generate( top: Optional[str] = None, output_dir: Optional[str] = None, next_topic: Optional[str] = None, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Generate SRAM macros from mems.conf. POST /ip/generate.""" @@ -28,8 +28,8 @@ def bbdev_ip_generate( output_dir=output_dir, next_topic=next_topic, ) - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt( diff --git a/bbdev/mcp/tools/ip_replace.py b/bbdev/mcp/tools/ip_replace.py index 7f270e7d9..a6f186e2e 100644 --- a/bbdev/mcp/tools/ip_replace.py +++ b/bbdev/mcp/tools/ip_replace.py @@ -15,7 +15,7 @@ def bbdev_ip_replace( top: Optional[str] = None, output_dir: Optional[str] = None, next_topic: Optional[str] = None, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Assemble synthesis source list plus SRAM macros. POST /ip/replace.""" @@ -28,8 +28,8 @@ def bbdev_ip_replace( output_dir=output_dir, next_topic=next_topic, ) - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt( diff --git a/bbdev/mcp/tools/kernel_build.py b/bbdev/mcp/tools/kernel_build.py index 453f554fb..453f2c10f 100644 --- a/bbdev/mcp/tools/kernel_build.py +++ b/bbdev/mcp/tools/kernel_build.py @@ -12,7 +12,7 @@ def register(mcp): def bbdev_kernel_build( design: str, interactive: bool = False, - device: bool = False, + device: str | None = None, model: str | None = None, ) -> str: """Build RISC-V kernel + rootfs. POST /kernel/build. @@ -30,8 +30,8 @@ def bbdev_kernel_build( } if interactive: params["interactive"] = True - if device: - params["device"] = True + if device is not None: + params["device"] = device if model is not None: params["model"] = model return fmt(submit("/kernel/build", params)) diff --git a/bbdev/mcp/tools/model_build.py b/bbdev/mcp/tools/model_build.py index 5dae3f44e..f1ae18a8d 100644 --- a/bbdev/mcp/tools/model_build.py +++ b/bbdev/mcp/tools/model_build.py @@ -5,11 +5,12 @@ def register(mcp): @mcp.tool() - def bbdev_model_build(design: str, model: str | None = None) -> str: - """Build one model recipe, or all recipes for the design when model is omitted. POST /model/build.""" - if error := need("design", design): + def bbdev_model_build(design: str, device: str, model: str | None = None) -> str: + """Build one model recipe on a device config, or all recipes of the design + when model is omitted. POST /model/build.""" + if error := need("design", design) or need("device", device): return err(error) - params = {"design": design} + params = {"design": design, "device": device} if model is not None: params["model"] = model return fmt(submit("/model/build", params)) diff --git a/bbdev/mcp/tools/stack_build.py b/bbdev/mcp/tools/stack_build.py index 524e66a11..9d7985f13 100644 --- a/bbdev/mcp/tools/stack_build.py +++ b/bbdev/mcp/tools/stack_build.py @@ -9,12 +9,12 @@ def register(mcp): @mcp.tool() def bbdev_stack_build( - design: str, core: str | None = None, device: bool = False + design: str, core: str | None = None, device: str | None = None ) -> str: """Build a registered device or core simulation stack. POST /stack/build.""" body = {"design": design} if core is not None: body["core"] = core - if device: - body["device"] = True + if device is not None: + body["device"] = device return fmt(submit("/stack/build", body)) diff --git a/bbdev/mcp/tools/uvm_build.py b/bbdev/mcp/tools/uvm_build.py index 097aa3bce..d7ebf8787 100644 --- a/bbdev/mcp/tools/uvm_build.py +++ b/bbdev/mcp/tools/uvm_build.py @@ -13,7 +13,7 @@ def bbdev_uvm_build( design: Optional[str] = None, ball: Optional[str] = None, ip: Optional[str] = None, - device: bool = False, + device: str | None = None, ) -> str: """Build a Ball or IP UVM simulation. POST /uvm/build.""" if design is None and not ip: @@ -25,6 +25,6 @@ def bbdev_uvm_build( params["ball"] = ball if ip: params["ip"] = ip - if device: - params["device"] = True + if device is not None: + params["device"] = device return fmt(submit("/uvm/build", params)) diff --git a/bbdev/mcp/tools/uvm_run.py b/bbdev/mcp/tools/uvm_run.py index b75331f59..de62f62ba 100644 --- a/bbdev/mcp/tools/uvm_run.py +++ b/bbdev/mcp/tools/uvm_run.py @@ -13,7 +13,7 @@ def bbdev_uvm_run( design: Optional[str] = None, ball: Optional[str] = None, ip: Optional[str] = None, - device: bool = False, + device: str | None = None, ) -> str: """Build and run a Ball or IP UVM simulation. POST /uvm/run.""" if design is None and not ip: @@ -25,6 +25,6 @@ def bbdev_uvm_run( params["ball"] = ball if ip: params["ip"] = ip - if device: - params["device"] = True + if device is not None: + params["device"] = device return fmt(submit("/uvm/run", params)) diff --git a/bbdev/mcp/tools/validate.py b/bbdev/mcp/tools/validate.py index 902d09a78..746cef31a 100644 --- a/bbdev/mcp/tools/validate.py +++ b/bbdev/mcp/tools/validate.py @@ -9,12 +9,12 @@ def register(mcp): @mcp.tool() - def validate(design: str = "toy", balldomain: Optional[str] = None) -> str: - """Validate the compiler Thread's BallDomain registration for a Design.""" - if e := need("design", design): + def validate(design: str, device: str, balldomain: Optional[str] = None) -> str: + """Validate the compiler Thread's BallDomain registration for a device config of a Design.""" + if e := need("design", design) or need("device", device): return err(e) try: - return fmt(validate_balldomains(design, balldomain)) + return fmt(validate_balldomains(design, device, balldomain)) except Exception as ex: return fmt( {"passed": False, "success": False, "failure": True, "error": str(ex)} diff --git a/bbdev/mcp/tools/verilator_build.py b/bbdev/mcp/tools/verilator_build.py index 77a1999d7..d9f599325 100644 --- a/bbdev/mcp/tools/verilator_build.py +++ b/bbdev/mcp/tools/verilator_build.py @@ -11,7 +11,7 @@ def bbdev_verilator_build( design: str, jobs: int = 16, coverage: bool = False, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Build non-bebop Verilator sim binary. POST /verilator/build.""" @@ -20,8 +20,8 @@ def bbdev_verilator_build( params = {"design": design, "jobs": jobs} if coverage: params["coverage"] = True - if device: - params["device"] = True + if device is not None: + params["device"] = device if core is not None: params["core"] = core return fmt(submit("/verilator/build", params)) diff --git a/bbdev/mcp/tools/verilator_clean.py b/bbdev/mcp/tools/verilator_clean.py index 52ab1dc93..a74844a5f 100644 --- a/bbdev/mcp/tools/verilator_clean.py +++ b/bbdev/mcp/tools/verilator_clean.py @@ -8,14 +8,14 @@ def register(mcp): @mcp.tool() def bbdev_verilator_clean( - design: str, device: bool = False, core: str | None = None + design: str, device: str | None = None, core: str | None = None ) -> str: """Clean non-bebop Verilator build. POST /verilator/clean.""" if e := need("design", design): return err(e) request = {"design": design} - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt(submit("/verilator/clean", request)) diff --git a/bbdev/mcp/tools/verilator_run.py b/bbdev/mcp/tools/verilator_run.py index 76016948b..6004a6423 100644 --- a/bbdev/mcp/tools/verilator_run.py +++ b/bbdev/mcp/tools/verilator_run.py @@ -21,7 +21,7 @@ def bbdev_verilator_run( pmctrace: bool = False, ctrace: bool = False, banktrace: bool = False, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Full non-bebop Verilator flow. POST /verilator/run.""" @@ -43,8 +43,8 @@ def bbdev_verilator_run( } if jobs is not None: params["jobs"] = jobs - if device: - params["device"] = True + if device is not None: + params["device"] = device if core is not None: params["core"] = core return fmt(submit("/verilator/run", params)) diff --git a/bbdev/mcp/tools/verilator_sim.py b/bbdev/mcp/tools/verilator_sim.py index d168c063e..0a74da65c 100644 --- a/bbdev/mcp/tools/verilator_sim.py +++ b/bbdev/mcp/tools/verilator_sim.py @@ -20,7 +20,7 @@ def bbdev_verilator_sim( pmctrace: bool = False, ctrace: bool = False, banktrace: bool = False, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Run one workload on non-bebop Verilator. POST /verilator/sim.""" @@ -40,8 +40,8 @@ def bbdev_verilator_sim( } if coverage: params["coverage"] = True - if device: - params["device"] = True + if device is not None: + params["device"] = device if core is not None: params["core"] = core return fmt(submit("/verilator/sim", params)) diff --git a/bbdev/mcp/tools/verilator_verilog.py b/bbdev/mcp/tools/verilator_verilog.py index 7468dc342..f4a91da8d 100644 --- a/bbdev/mcp/tools/verilator_verilog.py +++ b/bbdev/mcp/tools/verilator_verilog.py @@ -8,14 +8,14 @@ def register(mcp): @mcp.tool() def bbdev_verilator_verilog( - design: str, device: bool = False, core: str | None = None + design: str, device: str | None = None, core: str | None = None ) -> str: """Generate Verilog for non-bebop Verilator. POST /verilator/verilog.""" if e := need("design", design): return err(e) request = {"design": design} - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt(submit("/verilator/verilog", request)) diff --git a/bbdev/mcp/tools/workload_build.py b/bbdev/mcp/tools/workload_build.py index c4979f417..e5580c86e 100644 --- a/bbdev/mcp/tools/workload_build.py +++ b/bbdev/mcp/tools/workload_build.py @@ -12,7 +12,7 @@ def bbdev_workload_build( design: str, ctest: bool = False, mlirtest: bool = False, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Build workloads for a design. POST /workload/build.""" @@ -25,8 +25,8 @@ def bbdev_workload_build( params["ctest"] = True if mlirtest: params["mlirtest"] = True - if device: - params["device"] = True + if device is not None: + params["device"] = device if core is not None: params["core"] = core return fmt(submit("/workload/build", params)) diff --git a/bbdev/mcp/tools/workload_clean.py b/bbdev/mcp/tools/workload_clean.py index 7a322948f..732825afc 100644 --- a/bbdev/mcp/tools/workload_clean.py +++ b/bbdev/mcp/tools/workload_clean.py @@ -8,12 +8,12 @@ def register(mcp): @mcp.tool() def bbdev_workload_clean( - design: str, device: bool = False, core: str | None = None + design: str, device: str | None = None, core: str | None = None ) -> str: """Clean workload artifacts for one design. POST /workload/clean.""" request = {"design": design} - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt(submit("/workload/clean", request)) diff --git a/bbdev/mcp/tools/workload_tohex.py b/bbdev/mcp/tools/workload_tohex.py index 779355899..3b663f6e3 100644 --- a/bbdev/mcp/tools/workload_tohex.py +++ b/bbdev/mcp/tools/workload_tohex.py @@ -8,12 +8,12 @@ def register(mcp): @mcp.tool() def bbdev_workload_tohex( - design: str, device: bool = False, core: str | None = None + design: str, device: str | None = None, core: str | None = None ) -> str: """Convert design workload ELF to hex. POST /workload/tohex.""" request = {"design": design} - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt(submit("/workload/tohex", request)) diff --git a/bbdev/mcp/tools/yosys_run.py b/bbdev/mcp/tools/yosys_run.py index b38d15bed..4aaf303df 100644 --- a/bbdev/mcp/tools/yosys_run.py +++ b/bbdev/mcp/tools/yosys_run.py @@ -14,7 +14,7 @@ def bbdev_yosys_run( top: Optional[str] = None, output_dir: Optional[str] = None, vcd: Optional[str] = None, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Full yosys flow. POST /yosys/run.""" @@ -26,8 +26,8 @@ def bbdev_yosys_run( output_dir=output_dir, vcd=vcd, ) - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt( diff --git a/bbdev/mcp/tools/yosys_synth.py b/bbdev/mcp/tools/yosys_synth.py index c02cd1886..634f13c56 100644 --- a/bbdev/mcp/tools/yosys_synth.py +++ b/bbdev/mcp/tools/yosys_synth.py @@ -14,7 +14,7 @@ def bbdev_yosys_synth( top: Optional[str] = None, output_dir: Optional[str] = None, vcd: Optional[str] = None, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Yosys synthesis + OpenSTA. POST /yosys/synth.""" @@ -26,8 +26,8 @@ def bbdev_yosys_synth( output_dir=output_dir, vcd=vcd, ) - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt( diff --git a/bbdev/mcp/tools/yosys_verilog.py b/bbdev/mcp/tools/yosys_verilog.py index 2387c6caa..6c2186c5d 100644 --- a/bbdev/mcp/tools/yosys_verilog.py +++ b/bbdev/mcp/tools/yosys_verilog.py @@ -13,7 +13,7 @@ def bbdev_yosys_verilog( design: str, top: Optional[str] = None, output_dir: Optional[str] = None, - device: bool = False, + device: str | None = None, core: str | None = None, ) -> str: """Generate Verilog for yosys flow. POST /yosys/verilog.""" @@ -24,8 +24,8 @@ def bbdev_yosys_verilog( top=top, output_dir=output_dir, ) - if device: - request["device"] = True + if device is not None: + request["device"] = device if core is not None: request["core"] = core return fmt( diff --git a/bebop b/bebop index a0ea302f2..4b022aec6 160000 --- a/bebop +++ b/bebop @@ -1 +1 @@ -Subproject commit a0ea302f218d7a3e96866812f9cb82de384eabce +Subproject commit 4b022aec6daac142f3ea8b076ef52e004994fa11 diff --git a/examples/build.sc b/examples/build.sc index d7baddb10..519b74f3e 100644 --- a/examples/build.sc +++ b/examples/build.sc @@ -2,8 +2,8 @@ import mill._ import mill.scalalib._ import $file.^.arch.src.main.scala.framework.{build => framework_build} import $file.^.arch.thirdparty.{build => thirdparty_build} -import $file.components.balls.{build => ball_components} -import $file.components.threads.{build => thread_components} +import $file.prefab.balls.{build => ball_components} +import $file.prefab.threads.{build => thread_components} import $file.^.arch.src.main.scala.framework.acc.balldomain.src.main.scala.bbus.{build => bbus_build} import $file.^.arch.src.main.scala.framework.core.tbus.{build => tbus_build} @@ -32,8 +32,8 @@ object library extends SbtModule { override def scalacOptions = Seq("-deprecation", "-feature", "-language:reflectiveCalls", "-Ymacro-annotations") override def generatedSources = T { - bbus_build.registry(ball_components.sources(), examplesRoot / "components" / "balls", T.dest) ++ - tbus_build.registry(thread_components.sources(), examplesRoot / "components" / "threads", T.dest) + bbus_build.registry(ball_components.sources(), examplesRoot / "prefab" / "balls", T.dest) ++ + tbus_build.registry(thread_components.sources(), examplesRoot / "prefab" / "threads", T.dest) } override def sources = diff --git a/examples/components/balls/f32add/stack/src/Lowering.cpp b/examples/components/balls/f32add/stack/src/Lowering.cpp deleted file mode 100644 index 476c7f3ea..000000000 --- a/examples/components/balls/f32add/stack/src/Lowering.cpp +++ /dev/null @@ -1,55 +0,0 @@ -//===- Lowering.cpp - F32AddBall in lower-buckyball -----------------------===// -// -// add_f32: C = A + B over `rows` rows of four f32, RNE with canonical NaN. -// A, B and C are distinct one-group banks. -// -// The Ball reads both addends from its source banks and cannot read the old -// destination, so it takes no read_destination: a sum of more terms is a -// chain of add_f32 into fresh banks. The Ball's input group is group 0 and it -// always reads B. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/BuckyballOps.h" -#include "Conversion/Buckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class AddF32Lowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &rewriter) const override { - if (op.getKind() != "add_f32") - return failure(); - Location loc = op.getLoc(); - int64_t rows = cast(op->getAttr("rows")).getInt(); - if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || - op.getReadDestination() || op.getSources().size() != 2 || - op.getSources()[0] == op.getSources()[1] || - llvm::is_contained(op.getSources(), op.getDestination())) - return op.emitOpError("add_f32 requires distinct A, B and C banks, rows " - "within a bank and no read_destination"); - // rs2: group 0 in [4:0]; bit 5 clear reads B. - mlir::buddy::emitInstruction( - rewriter, loc, buckyball_target::getBuckyballFunct7("F32ADD"), - mlir::buddy::constantI64(rewriter, loc, - mlir::buddy::bankFields(op.getSources(), - op.getDestination(), - false, rows)), - mlir::buddy::constantI64(rewriter, loc, 0)); - rewriter.eraseOp(op); - return success(); - } -}; - -void populate(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -[[maybe_unused]] const bool registered = - (mlir::buddy::addBallLowering(populate), true); -} // namespace diff --git a/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir b/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir deleted file mode 100644 index ac2b1f3cd..000000000 --- a/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir +++ /dev/null @@ -1,24 +0,0 @@ -// RUN: buddy-opt %s -split-input-file -lower-buckyball -verify-diagnostics -func.func @zero_rows() { - // expected-error @+1 {{add_f32 requires distinct A, B and C banks, rows within a bank and no read_destination}} - buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 0 : i64} - return -} -// ----- -func.func @wide_rows() { - // expected-error @+1 {{add_f32 requires distinct A, B and C banks, rows within a bank and no read_destination}} - buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 17179869184 : i64} - return -} -// ----- -func.func @equal_banks() { - // expected-error @+1 {{add_f32 requires distinct A, B and C banks, rows within a bank and no read_destination}} - buckyball.custom "add_f32" <{sources = array, destination = 32 : i64}> {rows = 1 : i64} - return -} -// ----- -func.func @read_destination() { - // expected-error @+1 {{add_f32 requires distinct A, B and C banks, rows within a bank and no read_destination}} - buckyball.custom "add_f32" <{sources = array, destination = 34 : i64, read_destination}> {rows = 1 : i64} - return -} diff --git a/examples/components/balls/gemmini/stack/src/Lowering.cpp b/examples/components/balls/gemmini/stack/src/Lowering.cpp deleted file mode 100644 index d10a070df..000000000 --- a/examples/components/balls/gemmini/stack/src/Lowering.cpp +++ /dev/null @@ -1,116 +0,0 @@ -//===- Lowering.cpp - GemminiBall in lower-buckyball ----------------------===// -// -// matmul_i8_tile: one 16x16 tile C = A @ B over `k`, a positive multiple of -// sixteen. A and B fill their groups in order, each group a bank depth of -// rows before the next, so a K deeper than a bank spans several groups -// (allocated together or gathered with bank_transfer). -// A: k / 16 blocks of sixteen rows of sixteen i8; row i of block r is -// A[i, 16r .. 16r + 15]. -// B: k rows of sixteen i8, B[0 .. k - 1, 0 .. 15]. -// C: a bank of four groups, sixteen rows of sixteen i32. -// With read_destination C = in_shift(C + A @ B, shift), rounding to nearest -// even; without, C = A @ B. -// -// This is not smatmul's matmul_i8: C holds one output row per bank row across -// four groups instead of four bank rows of one group, there is no bias row, -// and read_destination accumulates the whole old C with a rounding shift. -// -// The Ball issues one instruction per 16-deep block, the operand groups and -// row bases derived from the block index. Without read_destination the first -// block runs on the output-stationary array (PRELOAD primes D = 0); every -// other block accumulates on the weight-stationary array, reading the old C. -// Only the last block applies the shift. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/BuckyballOps.h" -#include "Conversion/Buckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class MatmulI8TileLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &b) const override { - if (op.getKind() != "matmul_i8_tile") - return failure(); - Location loc = op.getLoc(); - bool readDestination = op.getReadDestination(); - int64_t k = cast(op->getAttr("k")).getInt(); - int64_t shift = - readDestination ? cast(op->getAttr("shift")).getInt() : 0; - int64_t meshRows = buckyball_target::getBuckyballBallParam( - "GEMMINI_COMPUTE_PRELOADED", "meshRows"); - int64_t meshColumns = buckyball_target::getBuckyballBallParam( - "GEMMINI_COMPUTE_PRELOADED", "meshColumns"); - const auto &target = buckyball_target::getBuckyballTarget(); - ArrayRef sources = op.getSources(); - int64_t c = op.getDestination(); - if (target.bankWidthBits != 128 || meshRows != 16 || meshColumns != 16 || - target.bankDepth % 16 || target.bankDepth > 1024 || - sources.size() != 2 || sources[0] == sources[1] || k <= 0 || k % 16 || - k > 64 * target.bankDepth || llvm::is_contained(sources, c)) - return op.emitOpError("matmul_i8_tile requires distinct A, B and C " - "banks, k a multiple of 16 within 64 groups and a " - "16x16 mesh"); - int64_t a = sources[0], bBank = sources[1]; - int64_t steps = k / 16, groupBlocks = target.bankDepth / 16; - uint64_t config = ~0ULL; - for (int64_t step = 0; step < steps; ++step) { - uint64_t group = step / groupBlocks, base = step % groupBlocks * 16; - bool accumulate = readDestination || step > 0; - // CONFIG: rs2[4] picks the weight-stationary array, rs2[40:9] the shift. - uint64_t word = mlir::buddy::field(accumulate, 4, 4) | - mlir::buddy::field(step == steps - 1 ? shift : 0, 9, 40); - if (word != config) - mlir::buddy::emitInstruction( - b, loc, buckyball_target::getBuckyballFunct7("GEMMINI_CONFIG"), - mlir::buddy::constantI64(b, loc, 0), - mlir::buddy::constantI64(b, loc, word)); - config = word; - // rs2: the subcommand in [3:0], the first and second source row bases - // in [15:6] and [25:16], their groups in [41:36] and [47:42]; C is at - // row 0. - uint64_t bases = - mlir::buddy::field(base, 6, 15) | mlir::buddy::field(base, 16, 25) | - mlir::buddy::field(group, 36, 41) | mlir::buddy::field(group, 42, 47); - if (accumulate) { - mlir::buddy::emitInstruction( - b, loc, - buckyball_target::getBuckyballFunct7("GEMMINI_COMPUTE_ACCUMULATED"), - mlir::buddy::constantI64( - b, loc, mlir::buddy::bankFields({a, bBank}, c, true, 16)), - mlir::buddy::constantI64(b, loc, 3 | bases)); - continue; - } - // The output-stationary preload reads sixteen rows of B. - mlir::buddy::emitInstruction( - b, loc, buckyball_target::getBuckyballFunct7("GEMMINI_PRELOAD"), - mlir::buddy::constantI64( - b, loc, mlir::buddy::bankFields({bBank}, c, false, 16)), - mlir::buddy::constantI64(b, loc, - 1 | mlir::buddy::field(base, 6, 15) | - mlir::buddy::field(group, 36, 41))); - mlir::buddy::emitInstruction( - b, loc, - buckyball_target::getBuckyballFunct7("GEMMINI_COMPUTE_PRELOADED"), - mlir::buddy::constantI64( - b, loc, mlir::buddy::bankFields({a, bBank}, c, false, 16)), - mlir::buddy::constantI64(b, loc, 2 | bases)); - } - b.eraseOp(op); - return success(); - } -}; - -void populate(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -[[maybe_unused]] const bool registered = - (mlir::buddy::addBallLowering(populate), true); -} // namespace diff --git a/examples/components/balls/im2col/stack/src/Lowering.cpp b/examples/components/balls/im2col/stack/src/Lowering.cpp deleted file mode 100644 index e04b1f6e8..000000000 --- a/examples/components/balls/im2col/stack/src/Lowering.cpp +++ /dev/null @@ -1,79 +0,0 @@ -//===- Lowering.cpp - Im2colBall in lower-buckyball -----------------------===// -// -// im2col_i8: the source holds a square feature map of `input_side` pixels per -// side; group g is channel block g, line y * input_side + x holds the sixteen -// channels of pixel (y, x). The destination receives the windows of the -// `kernel` x `kernel` convolution with `stride` and zero `padding`, every -// window in row-major order, as the matmul_i8 A operand: K is ordered (kernel -// row, kernel column, channel), sixteen columns (one source line) per K tile, -// K tile t = (kr * kernel + kc) * groups + g. The destination groups divide -// the K tiles; group j holds the next K / 16 / groups tiles in M tiles of -// `tile_rows` rows (the layout of the matmul that consumes it), windows -// beyond the last zero up to a multiple of tile_rows. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/BuckyballOps.h" -#include "Conversion/Buckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class Im2colI8Lowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &rewriter) const override { - if (op.getKind() != "im2col_i8") - return failure(); - if (op.getSources().size() != 1 || - op.getSources()[0] == op.getDestination()) - return op.emitOpError( - "im2col_i8 requires distinct input and output banks"); - int64_t side = cast(op->getAttr("input_side")).getInt(); - int64_t kernel = cast(op->getAttr("kernel")).getInt(); - int64_t stride = cast(op->getAttr("stride")).getInt(); - int64_t padding = cast(op->getAttr("padding")).getInt(); - int64_t tileRows = cast(op->getAttr("tile_rows")).getInt(); - const auto &target = buckyball_target::getBuckyballTarget(); - int64_t span = side + 2 * padding - kernel; - if (target.bankWidthBits != 128 || side <= 0 || - side > buckyball_target::getBuckyballBallParam("IM2COL", "maxIter") || - side * side > target.bankDepth || kernel <= 0 || - kernel > - buckyball_target::getBuckyballBallParam("IM2COL", "maxKSize") || - stride <= 0 || stride > 15 || padding < 0 || - padding > - buckyball_target::getBuckyballBallParam("IM2COL", "maxPadding") || - span < 0 || span % stride || tileRows <= 0 || tileRows > 16 || - (tileRows & (tileRows - 1))) - return op.emitOpError("im2col_i8 geometry exceeds the ISA field ranges"); - int64_t windows = (span / stride + 1) * (span / stride + 1); - if (windows > 1023) - return op.emitOpError("im2col_i8 has more windows than one instruction"); - uint64_t rs2 = mlir::buddy::field(kernel, 0, 3) | - mlir::buddy::field(stride, 4, 7) | - mlir::buddy::field(padding, 8, 11) | - mlir::buddy::field(windows, 36, 45) | - mlir::buddy::field(tileRows, 46, 50); - mlir::buddy::emitInstruction( - rewriter, op.getLoc(), buckyball_target::getBuckyballFunct7("IM2COL"), - mlir::buddy::constantI64(rewriter, op.getLoc(), - mlir::buddy::bankFields(op.getSources(), - op.getDestination(), - false, side)), - mlir::buddy::constantI64(rewriter, op.getLoc(), rs2)); - rewriter.eraseOp(op); - return success(); - } -}; - -void populate(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -[[maybe_unused]] const bool registered = - (mlir::buddy::addBallLowering(populate), true); -} // namespace diff --git a/examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir b/examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir deleted file mode 100644 index 6331c142b..000000000 --- a/examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir +++ /dev/null @@ -1,43 +0,0 @@ -// Physical-bank im2col_i8: 6x6 input at rows 5.. lane 7, k3, 16 windows. - -func.func private @check_result(memref<16x16xi8>) -> () - -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %c0 = arith.constant 0 : index - %c1 = arith.constant 1 : index - %c5 = arith.constant 5 : index - %c6 = arith.constant 6 : index - %c7 = arith.constant 7 : index - - %input = memref.alloc() alignment = 64 : memref<64x16xi8> - %output = memref.alloc() alignment = 64 : memref<16x16xi8> - linalg.fill ins(%zero_i8 : i8) outs(%input : memref<64x16xi8>) - linalg.fill ins(%zero_i8 : i8) outs(%output : memref<16x16xi8>) - - scf.for %r = %c0 to %c6 step %c1 { - scf.for %col = %c0 to %c6 step %c1 { - %tmp = arith.muli %r, %c6 : index - %idx = arith.addi %tmp, %col : index - %bank_row = arith.addi %c5, %idx : index - %ri = arith.index_cast %r : index to i32 - %ci = arith.index_cast %col : index to i32 - %v32 = arith.addi %ri, %ci : i32 - %v8 = arith.trunci %v32 : i32 to i8 - memref.store %v8, %input[%bank_row, %c7] : memref<64x16xi8> - } - } - - buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> - buckyball.mvin %input <{bank = 1 : i64, depth = 64 : i64, stride = 1 : i64}> : memref<64x16xi8> - buckyball.custom "im2col_i8" <{sources = array, destination = 2 : i64}> {input_side = 6 : i64, kernel = 3 : i64, stride = 1 : i64, padding = 0 : i64, start_row = 0 : i64, start_col = 0 : i64, op1_base = 5 : i64, lane = 7 : i64, window_start = 0 : i64, window_count = 16 : i64} - buckyball.mvout %output <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> - buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> - - func.call @check_result(%output) : (memref<16x16xi8>) -> () - memref.dealloc %input : memref<64x16xi8> - memref.dealloc %output : memref<16x16xi8> - return %zero_i8 : i8 -} diff --git a/examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp b/examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp deleted file mode 100644 index 3e388bacd..000000000 --- a/examples/components/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp +++ /dev/null @@ -1,71 +0,0 @@ -#include -#include -#include - -enum { ITER = 6, K = 3, STRIDE = 1, PAD = 0, LANES = 16 }; -enum { - OUT_DIM = (ITER + 2 * PAD - K) / STRIDE + 1, - WINDOWS = OUT_DIM * OUT_DIM, - KERNEL = K * K, - M_TILES = (WINDOWS + LANES - 1) / LANES, - K_TILES = (KERNEL + LANES - 1) / LANES, - OUT_ROWS = M_TILES * K_TILES * LANES, -}; - -static void fail(void) { -#ifdef BAREMETAL - volatile uint32_t *sim_exit = (volatile uint32_t *)0x60000000; - *sim_exit = 1; - while (1) { - } -#else - exit(1); -#endif -} - -static int8_t src(int row, int col) { - if (row < 0 || row >= ITER || col < 0 || col >= ITER) - return 0; - return (int8_t)(row + col); -} - -#ifdef __cplusplus -extern "C" -#endif - void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, - int64_t size0, int64_t size1, int64_t stride0, - int64_t stride1) { - (void)allocated; - if (size0 != OUT_ROWS || size1 != LANES || stride0 != LANES || stride1 != 1) { - printf("FAILED: im2col ball_im2col unexpected shape " - "(size=%ldx%ld stride=%ldx%ld)\n", - (long)size0, (long)size1, (long)stride0, (long)stride1); - fail(); - } - - int8_t *out = aligned + offset; - int w = 0; - for (int orow = 0; orow < OUT_DIM; orow++) { - for (int ocol = 0; ocol < OUT_DIM; ocol++) { - for (int krow = 0; krow < K; krow++) { - for (int kcol = 0; kcol < K; kcol++) { - int ki = krow * K + kcol; - int bank_row = - ((w / LANES) * K_TILES + ki / LANES) * LANES + (w % LANES); - int8_t exp = - src(orow * STRIDE + krow - PAD, ocol * STRIDE + kcol - PAD); - int8_t got = out[bank_row * stride0 + (ki % LANES) * stride1]; - if (got != exp) { - printf("FAILED: im2col ball_im2col w=%d ki=%d " - "(expected %d, got %d)\n", - w, ki, (int)exp, (int)got); - fail(); - } - } - } - w++; - } - } - - printf("PASSED: im2col ball_im2col 6x6 k3\n"); -} diff --git a/examples/components/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index 6fca83f3c..000000000 --- a/examples/components/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) - -set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") -file(STRINGS "${_params}" _lines_define REGEX "^#define BANK_LINES ") -file(STRINGS "${_params}" _width_define REGEX "^#define BANK_WIDTH ") -file(STRINGS "${_params}" _sqrt_define REGEX "^#define BANK_ISQRT ") -string(REGEX REPLACE ".* " "" _lines "${_lines_define}") -string(REGEX REPLACE ".* " "" _width "${_width_define}") -string(REGEX REPLACE ".* " "" _sqrt "${_sqrt_define}") -math(EXPR _remainder "${_lines} % 64") -if(_lines LESS 64 OR NOT _remainder EQUAL 0 OR _width LESS 72) - message(FATAL_ERROR "Im2col K3 bank batches require BANK_LINES divisible by 64 and at least 9 INT8 lanes (got lines=${_lines}, width=${_width})") -endif() -add_buckyball_mlir_test(im2col_k3_padded_bank_windows TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/components/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows.mlir.in b/examples/components/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows.mlir.in deleted file mode 100644 index 62ae91620..000000000 --- a/examples/components/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows.mlir.in +++ /dev/null @@ -1,32 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () -func.func @main() -> i8 { - %zero_i8 = arith.constant 0 : i8 - %zero = arith.constant 0 : index - %one = arith.constant 1 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %windows = arith.constant 64 : index - %lane = arith.constant 7 : index - %input = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %output = memref.alloc() : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - linalg.fill ins(%zero_i8 : i8) outs(%input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) - scf.for %index = %zero to %rows step %one { - %value32 = arith.index_cast %index : index to i32 - %value = arith.trunci %value32 : i32 to i8 - memref.store %value, %input[%index, %lane] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - } - scf.for %base = %zero to %rows step %windows { - %in = memref.subview %input[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> - %out = memref.subview %output[%base, 0] [64, @BB_BANK_I8_LANES@] [1, 1] : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> to memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> - %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> - %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> - %loaded = buckyball.bank_mvin %in, %input_bank <{depth = 64 : i64, stride = 1 : i64}> : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> - %result = buckyball.bank_custom "im2col_i8" (%loaded) -> %output_bank {input_side = 8 : i64, kernel = 3 : i64, stride = 1 : i64, padding = 1 : i64, start_row = 0 : i64, start_col = 0 : i64, op1_base = 0 : i64, lane = 7 : i64, window_start = 0 : i64, window_count = 64 : i64} - %stored = buckyball.bank_mvout %out, %result <{depth = 64 : i64, stride = 1 : i64}> : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> - buckyball.bank_release %loaded - buckyball.bank_release %stored - } - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () - memref.dealloc %input : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - memref.dealloc %output : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - return %zero_i8 : i8 -} diff --git a/examples/components/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows_main.cpp b/examples/components/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows_main.cpp deleted file mode 100644 index f0be7350b..000000000 --- a/examples/components/balls/im2col/workloads/mlir_tests/bank/im2col_k3_padded_bank_windows_main.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include -#include -#include -#include - -extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != BANK_LINES || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) - exit(1); - constexpr int windows = BANK_LINES; - constexpr int side = 8; - for (int window = 0; window < windows; ++window) { - int batch = window / 64; - int y = (window % 64) / side; - int x = window % side; - for (int ky = 0; ky < 3; ++ky) - for (int kx = 0; kx < 3; ++kx) { - int source_y = y + ky - 1; - int source_x = x + kx - 1; - int expected = 0; - if (source_y >= 0 && source_y < side && source_x >= 0 && - source_x < side) - expected = batch * 64 + source_y * side + source_x; - int lane = ky * 3 + kx; - if (data[offset + window * stride + lane] != (int8_t)expected) { - printf("im2col K3 bank mismatch window=%d lane=%d\n", window, lane); - exit(1); - } - } - } - printf("im2col K3 padded bank PASS rows=%d\n", windows); -} diff --git a/examples/components/balls/int2fp/stack/src/Lowering.cpp b/examples/components/balls/int2fp/stack/src/Lowering.cpp deleted file mode 100644 index a844ec824..000000000 --- a/examples/components/balls/int2fp/stack/src/Lowering.cpp +++ /dev/null @@ -1,49 +0,0 @@ -//===- Lowering.cpp - Int2FpBall in lower-buckyball -----------------------===// -// -// int32_to_fp32: source 0 holds `rows` rows of sixteen i32 (four bank rows -// each), source 1 the sixteen f32 scales of the columns (four bank rows); the -// destination receives the f32 value * scale of every element, with `relu` -// negative results replaced by zero. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/BuckyballOps.h" -#include "Conversion/Buckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class Int32ToFp32Lowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &rewriter) const override { - if (op.getKind() != "int32_to_fp32") - return failure(); - if (op.getSources().size() != 2) - return op.emitOpError("int32_to_fp32 reads the values and the scales"); - Location loc = op.getLoc(); - int64_t rows = cast(op->getAttr("rows")).getInt(); - bool relu = cast(op->getAttr("relu")).getValue(); - mlir::buddy::emitInstruction( - rewriter, loc, buckyball_target::getBuckyballFunct7("INT32_TO_FP32"), - mlir::buddy::constantI64(rewriter, loc, - mlir::buddy::bankFields(op.getSources(), - op.getDestination(), - false, rows * 4)), - mlir::buddy::constantI64(rewriter, loc, - mlir::buddy::field(relu, 0, 0))); - rewriter.eraseOp(op); - return success(); - } -}; - -void populate(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -[[maybe_unused]] const bool registered = - (mlir::buddy::addBallLowering(populate), true); -} // namespace diff --git a/examples/components/balls/int8add/stack/src/Lowering.cpp b/examples/components/balls/int8add/stack/src/Lowering.cpp deleted file mode 100644 index c170e1449..000000000 --- a/examples/components/balls/int8add/stack/src/Lowering.cpp +++ /dev/null @@ -1,66 +0,0 @@ -//===- Lowering.cpp - Int8AddBall in lower-buckyball ----------------------===// -// -// add_i8: the destination receives `rows` rows of sixteen i8, -// round_half_even(A * lhs_ratio + B * rhs_ratio) saturated to i8, and with -// `relu` clamped at zero. Arguments 0 and 1 are the f32 lhs_ratio and -// rhs_ratio, both finite and positive. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/BuckyballOps.h" -#include "Conversion/Buckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class AddI8Lowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &rewriter) const override { - if (op.getKind() != "add_i8") - return failure(); - Location loc = op.getLoc(); - int64_t rows = cast(op->getAttr("rows")).getInt(); - bool relu = cast(op->getAttr("relu")).getValue(); - if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || - op.getSources().size() != 2 || - op.getSources()[0] == op.getSources()[1] || - llvm::is_contained(op.getSources(), op.getDestination())) - return op.emitOpError( - "add_i8 requires distinct banks and rows within a bank"); - // rs2: lhs_ratio bits in [31:0], rhs_ratio bits in [63:32]. - auto ratioBits = [&](Value ratio) -> Value { - Value bits = - arith::BitcastOp::create(rewriter, loc, rewriter.getI32Type(), ratio); - return arith::ExtUIOp::create(rewriter, loc, rewriter.getI64Type(), bits); - }; - Value rhs = - arith::ShLIOp::create(rewriter, loc, ratioBits(op.getArguments()[1]), - mlir::buddy::constantI64(rewriter, loc, 32)); - Value rs2 = arith::OrIOp::create(rewriter, loc, - ratioBits(op.getArguments()[0]), rhs); - mlir::buddy::emitInstruction( - rewriter, loc, - buckyball_target::getBuckyballFunct7(relu ? "INT8ADD_RELU" : "INT8ADD"), - mlir::buddy::constantI64(rewriter, loc, - mlir::buddy::bankFields(op.getSources(), - op.getDestination(), - false, rows)), - rs2); - rewriter.eraseOp(op); - return success(); - } -}; - -void populate(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -[[maybe_unused]] const bool registered = - (mlir::buddy::addBallLowering(populate), true); -} // namespace diff --git a/examples/components/balls/int8mul/stack/src/Lowering.cpp b/examples/components/balls/int8mul/stack/src/Lowering.cpp deleted file mode 100644 index f52d480db..000000000 --- a/examples/components/balls/int8mul/stack/src/Lowering.cpp +++ /dev/null @@ -1,57 +0,0 @@ -//===- Lowering.cpp - Int8MulBall in lower-buckyball ----------------------===// -// -// gate_mul_i8: source 0 holds the gate, one row of sixteen i8; source 1 holds -// `rows` rows of sixteen i8. The destination receives -// clamp(round_half_even(gate * x * ratio), -128, 127) per column, in every -// group of the banks; the three banks have the same number of groups. -// Argument 0 is the f32 ratio, finite and positive. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/BuckyballOps.h" -#include "Conversion/Buckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class GateMulI8Lowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &b) const override { - if (op.getKind() != "gate_mul_i8") - return failure(); - int64_t rows = cast(op->getAttr("rows")).getInt(); - if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || - op.getSources().size() != 2 || - op.getSources()[0] == op.getSources()[1] || - llvm::is_contained(op.getSources(), op.getDestination())) - return op.emitOpError( - "gate_mul_i8 requires distinct banks and rows within a bank"); - // rs2: ratio bits in [31:0]; the gate row in [63:32] is the first row. - Value bits = arith::BitcastOp::create(b, op.getLoc(), b.getI32Type(), - op.getArguments()[0]); - Value rs2 = arith::ExtUIOp::create(b, op.getLoc(), b.getI64Type(), bits); - mlir::buddy::emitInstruction( - b, op.getLoc(), buckyball_target::getBuckyballFunct7("INT8MUL"), - mlir::buddy::constantI64(b, op.getLoc(), - mlir::buddy::bankFields(op.getSources(), - op.getDestination(), - false, rows)), - rs2); - b.eraseOp(op); - return success(); - } -}; - -void populate(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -[[maybe_unused]] const bool registered = - (mlir::buddy::addBallLowering(populate), true); -} // namespace diff --git a/examples/components/balls/lut/stack/src/Lowering.cpp b/examples/components/balls/lut/stack/src/Lowering.cpp deleted file mode 100644 index 43b4be4c6..000000000 --- a/examples/components/balls/lut/stack/src/Lowering.cpp +++ /dev/null @@ -1,45 +0,0 @@ -//===- Lowering.cpp - LutBall in lower-buckyball --------------------------===// -// -// lookup_i8: source 0 holds `rows` rows of sixteen i8 codes, source 1 the -// table: one group of 256 entries shared by every channel, or four groups -// with 256 entries per channel (channel * 256 + code, 1024 bytes per group). -// The destination receives table[code] for every element. -// -//===----------------------------------------------------------------------===// -#include "Buckyball/BuckyballOps.h" -#include "Conversion/Buckyball.h" -#include "Target/BuckyballTargetRegistry.h" -using namespace mlir; -using namespace ::buddy::buckyball; -namespace { -class LookupI8Lowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &b) const override { - if (op.getKind() != "lookup_i8") - return failure(); - int64_t rows = cast(op->getAttr("rows")).getInt(); - if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || - op.getSources().size() != 2 || - op.getSources()[0] == op.getSources()[1] || - llvm::is_contained(op.getSources(), op.getDestination())) - return op.emitOpError( - "lookup_i8 requires distinct banks and rows within bank depth"); - mlir::buddy::emitInstruction( - b, op.getLoc(), buckyball_target::getBuckyballFunct7("LUT"), - mlir::buddy::constantI64(b, op.getLoc(), - mlir::buddy::bankFields(op.getSources(), - op.getDestination(), - false, rows)), - mlir::buddy::constantI64(b, op.getLoc(), 0)); - b.eraseOp(op); - return success(); - } -}; -void populate(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} -[[maybe_unused]] const bool registered = - (mlir::buddy::addBallLowering(populate), true); -} // namespace diff --git a/examples/components/balls/matadd/stack/src/Lowering.cpp b/examples/components/balls/matadd/stack/src/Lowering.cpp deleted file mode 100644 index 6d79ce1a2..000000000 --- a/examples/components/balls/matadd/stack/src/Lowering.cpp +++ /dev/null @@ -1,49 +0,0 @@ -//===- Lowering.cpp - MatAddBall in lower-buckyball -----------------------===// -// -// add_i32: C = A + B modulo 2^32 over `rows` rows of four i32 in every group -// of the banks; A, B and C have the same number of groups. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/BuckyballOps.h" -#include "Conversion/Buckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class AddI32Lowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &rewriter) const override { - if (op.getKind() != "add_i32") - return failure(); - Location loc = op.getLoc(); - int64_t rows = cast(op->getAttr("rows")).getInt(); - if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || - op.getSources().size() != 2 || - op.getSources()[0] == op.getSources()[1] || - llvm::is_contained(op.getSources(), op.getDestination())) - return op.emitOpError( - "add_i32 requires distinct banks and rows within a bank"); - mlir::buddy::emitInstruction( - rewriter, loc, buckyball_target::getBuckyballFunct7("MATADD"), - mlir::buddy::constantI64(rewriter, loc, - mlir::buddy::bankFields(op.getSources(), - op.getDestination(), - false, rows)), - mlir::buddy::constantI64(rewriter, loc, 0)); - rewriter.eraseOp(op); - return success(); - } -}; - -void populate(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -[[maybe_unused]] const bool registered = - (mlir::buddy::addBallLowering(populate), true); -} // namespace diff --git a/examples/components/balls/maxpool/stack/src/Lowering.cpp b/examples/components/balls/maxpool/stack/src/Lowering.cpp deleted file mode 100644 index 6bd1ed83d..000000000 --- a/examples/components/balls/maxpool/stack/src/Lowering.cpp +++ /dev/null @@ -1,68 +0,0 @@ -//===- Lowering.cpp - MaxPoolBall in lower-buckyball ----------------------===// -// -// maxpool_i8: the source holds a square feature map of `input_side` pixels -// per side; group g is channel block g, line y * input_side + x holds the -// sixteen i8 channels of pixel (y, x). The destination, with the same groups, -// receives the `kernel` x `kernel` maxima with `stride` and `padding` in the -// same layout, output_side = (input_side + 2 * padding - kernel) / stride + 1. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/BuckyballOps.h" -#include "Conversion/Buckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class MaxpoolI8Lowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &rewriter) const override { - if (op.getKind() != "maxpool_i8") - return failure(); - if (op.getSources().size() != 1 || - op.getSources()[0] == op.getDestination()) - return op.emitOpError( - "maxpool_i8 requires distinct input and output banks"); - int64_t inputSide = cast(op->getAttr("input_side")).getInt(); - int64_t kernel = cast(op->getAttr("kernel")).getInt(); - int64_t stride = cast(op->getAttr("stride")).getInt(); - int64_t padding = cast(op->getAttr("padding")).getInt(); - int64_t span = inputSide + 2 * padding - kernel; - if (inputSide <= 0 || inputSide > 15 || kernel <= 0 || kernel > 15 || - stride <= 0 || stride > 15 || padding < 0 || padding > 15 || span < 0 || - span % stride || span / stride + 1 > 15 || - inputSide * inputSide > - buckyball_target::getBuckyballTarget().bankDepth) - return op.emitOpError("maxpool_i8 geometry exceeds the ISA field ranges"); - int64_t outputSide = span / stride + 1; - // Both maps start at the base of their groups; output rows are packed. - uint64_t rs2 = mlir::buddy::field(inputSide, 0, 3) | - mlir::buddy::field(outputSide, 4, 7) | - mlir::buddy::field(kernel, 8, 11) | - mlir::buddy::field(stride, 12, 15) | - mlir::buddy::field(padding, 16, 19) | - mlir::buddy::field(outputSide, 32, 37); - mlir::buddy::emitInstruction( - rewriter, op.getLoc(), buckyball_target::getBuckyballFunct7("MAXPOOL"), - mlir::buddy::constantI64( - rewriter, op.getLoc(), - mlir::buddy::bankFields(op.getSources(), op.getDestination(), false, - outputSide * outputSide) | - (1ULL << 63)), - mlir::buddy::constantI64(rewriter, op.getLoc(), rs2)); - rewriter.eraseOp(op); - return success(); - } -}; - -void populate(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -[[maybe_unused]] const bool registered = - (mlir::buddy::addBallLowering(populate), true); -} // namespace diff --git a/examples/components/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index 77ba0e692..000000000 --- a/examples/components/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) -set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") -file(STRINGS "${_params}" _lines_define REGEX "^#define BANK_LINES ") -file(STRINGS "${_params}" _sqrt_define REGEX "^#define BANK_ISQRT ") -string(REGEX REPLACE ".* " "" _lines "${_lines_define}") -string(REGEX REPLACE ".* " "" _sqrt "${_sqrt_define}") -math(EXPR _remainder "${_lines} % 64") -if(_lines LESS 64 OR NOT _remainder EQUAL 0) - message(FATAL_ERROR - "MaxPool K3 bank batches require BANK_LINES divisible by 64 (got ${_lines})") -endif() -add_buckyball_mlir_test(maxpool_k3_padded_spatial TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/components/balls/mxfp2int/emu/src/55_mxfp2int.rs b/examples/components/balls/mxfp2int/emu/src/55_mxfp2int.rs deleted file mode 100644 index 30744d9b8..000000000 --- a/examples/components/balls/mxfp2int/emu/src/55_mxfp2int.rs +++ /dev/null @@ -1,103 +0,0 @@ -//===- 55_mxfp2int.rs - MXFP2INT instruction (MXFP to INT8) ----------------===// -// -// Dequantizes MXFP4/6/8 data to INT8 using per-block E8M0 scales from MMIO. -// -// rs1[9:0]: op1_bank (BANK0, input MXFP data) -// rs1[29:20]: wr_bank (BANK2, output INT8 data) -// rs1[63:30]: iter (number of MXFP blocks to process) -// -// For MXFP4: each block = 32 x 4-bit FP = 16 bytes input → 32 bytes INT8 output -// Scale: per-block E8M0 (8-bit) from MMIO, indexed by block_idx. -// -//===-----------------------------------------------------------------===// - -use bebop_bemu::thread::banks::{bank_width, mmio_read_byte}; -use super::decode::{rs1_b0, rs1_b2, rs1_iter}; -use super::instruction::{BallModel, ExecContext}; - -pub struct Mxfp2Int; - -impl BallModel for Mxfp2Int { - fn exec(xs1: u64, _xs2: u64, ctx: &mut ExecContext) -> u64 { - let in_bank = rs1_b0(xs1) as u32; - let out_bank = rs1_b2(xs1) as u32; - let iter = rs1_iter(xs1) as usize; // number of MXFP blocks - - // Resolve physical banks - let in_pbank = ctx - .bank_map - .resolve(in_bank) - .unwrap_or_else(|| panic!("mxfp2int: input bank {} not allocated", in_bank)); - let out_pbank = ctx - .bank_map - .resolve(out_bank) - .unwrap_or_else(|| panic!("mxfp2int: output bank {} not allocated", out_bank)); - - const FP4_PER_BLOCK: usize = 32; - const BYTES_PER_OUTPUT_BLOCK: usize = 32; // 32 x INT8 - - for block_idx in 0..iter { - // Read E8M0 scale from the globally encoded MMIO address space. - let scale_e8m0 = mmio_read_byte(ctx.mmio_banks, block_idx); - - // Read input MXFP4 block (16 bytes = 1 bank row) - let in_row_addr = block_idx; - let row_bytes = bank_width() / 8; - let in_row_bytes = - &ctx.banks[in_pbank][in_row_addr * row_bytes..(in_row_addr + 1) * row_bytes]; - - // Dequantize 32 FP4 elements to INT8 - let mut out_bytes = [0i8; BYTES_PER_OUTPUT_BLOCK]; - for (elem_idx, out_byte) in out_bytes.iter_mut().enumerate().take(FP4_PER_BLOCK) { - let byte_idx = elem_idx / 2; - let nibble_shift = (elem_idx % 2) * 4; - let fp4_raw = (in_row_bytes[byte_idx] >> nibble_shift) & 0x0F; - - // FP4 decode: signed_val = raw - 8 (range -8..+7) - let signed_val = (fp4_raw as i32) - 8; - - // Scale: shift = scale_e8m0 - 127 - let shift = (scale_e8m0 as i32) - 127; - - // Apply scale and saturate to INT8 - let scaled = if shift >= 0 { - signed_val - .checked_shl(shift as u32) - .unwrap_or(if signed_val >= 0 { i32::MAX } else { i32::MIN }) - } else { - let abs_shift = (-shift) as u32; - if abs_shift >= 32 { - 0 // Shift too large, result is 0 - } else { - signed_val >> abs_shift - } - }; - let saturated = scaled.clamp(-128, 127) as i8; - - *out_byte = saturated; - } - - // Write output INT8 data (32 bytes = 2 bank rows) - let out_row_base = block_idx * 2; - for row_offset in 0..2 { - let out_row_addr = out_row_base + row_offset; - let out_row_start = out_row_addr * row_bytes; - let src_start = row_offset * row_bytes; - let src_end = src_start + row_bytes; - - // Copy 16 bytes (1 row) from out_bytes to output bank - for (i, &byte) in out_bytes[src_start..src_end].iter().enumerate() { - ctx.banks[out_pbank][out_row_start + i] = byte as u8; - } - } - } - - 0 - } - - fn latency(_xs1: u64, _xs2: u64) -> u64 { - let iter = rs1_iter(_xs1); - // Approximate: each block needs 1 MMIO read + 1 input read + 2 output writes - iter * 4 - } -} diff --git a/examples/components/balls/mxfp2int/workloads/ctests/CMakeLists.txt b/examples/components/balls/mxfp2int/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 211320815..000000000 --- a/examples/components/balls/mxfp2int/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_buckyball_ctest(mxfp4_e8m0_two_block_test.c) diff --git a/examples/components/balls/mxfp2int/workloads/isa/mxfp2int.h b/examples/components/balls/mxfp2int/workloads/isa/mxfp2int.h deleted file mode 100644 index 51d40cf2a..000000000 --- a/examples/components/balls/mxfp2int/workloads/isa/mxfp2int.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef _BB_MXFP2INT_H_ -#define _BB_MXFP2INT_H_ - -#include -#include - -// Basic version: -// rs1 = bank0(read) | bank2(write) | iter -// rs2 = 0 -#define bb_mxfp2int(bank_id, wr_bank_id, iter) \ - BUCKYBALL_INSTRUCTION_R_R( \ - (BB_BANK0(bank_id) | BB_BANK2(wr_bank_id) | BB_ITER(iter)), 0, \ - BB_FUNC7(MXFP2INT)) - -// Extended version: -// rs2 carries user-defined special field. -// Useful later for format select / rounding mode / debug flags. -#define bb_mxfp2int_ex(bank_id, wr_bank_id, iter, special) \ - BUCKYBALL_INSTRUCTION_R_R( \ - (BB_BANK0(bank_id) | BB_BANK2(wr_bank_id) | BB_ITER(iter)), (special), \ - BB_FUNC7(MXFP2INT)) - -#endif // _BB_MXFP2INT_H_ diff --git a/examples/components/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt deleted file mode 100644 index 313d9257a..000000000 --- a/examples/components/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX ball) -add_buckyball_mlir_test(mxfp4_unit_scale_2blocks TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/components/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks.mlir b/examples/components/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks.mlir deleted file mode 100644 index 5a6e31194..000000000 --- a/examples/components/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks.mlir +++ /dev/null @@ -1,35 +0,0 @@ -func.func private @check_result(memref<4x16xi8>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %raw = arith.constant -103 : i8 - %scale = arith.constant 127 : i8 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : i64 - %one64 = arith.constant 1 : i64 - %four = arith.constant 4 : i64 - %sixteen = arith.constant 16 : i64 - %zero64 = arith.constant 0 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() : memref<2x16xi8> - %scales = memref.alloc() : memref<1x16xi8> - %output = memref.alloc() : memref<4x16xi8> - linalg.fill ins(%raw : i8) outs(%input : memref<2x16xi8>) - linalg.fill ins(%scale : i8) outs(%scales : memref<1x16xi8>) - linalg.fill ins(%z8 : i8) outs(%output : memref<4x16xi8>) - %in = arith.constant 0 : i64 - %out = arith.constant 1 : i64 - buckyball.mset %in : i64 - buckyball.mset %out : i64 - buckyball.mvin %input %in %two %stride : memref<2x16xi8> i64 i64 i64 - buckyball.mvin_mmio %scales %zero64 %one64 %sixteen : memref<1x16xi8> i64 i64 i64 - buckyball.mxfp2int %in, %out, %two, %zero64 : i64 - buckyball.mvout %output %out %four %stride : memref<4x16xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<4x16xi8>) -> () - buckyball.mset %in : i64 - buckyball.mset %out : i64 - memref.dealloc %input : memref<2x16xi8> - memref.dealloc %scales : memref<1x16xi8> - memref.dealloc %output : memref<4x16xi8> - return %z8 : i8 -} diff --git a/examples/components/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index 0abb14ed4..000000000 --- a/examples/components/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) -set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") -file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") -string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") -math(EXPR _remainder "${_lines} % 32") -if(_lines LESS 32 OR NOT (_remainder EQUAL 0)) - message(FATAL_ERROR - "Mxfp2Int BankOp test requires BANK_LINES divisible by 32 " - "(got lines=${_lines})") -endif() -math(EXPR BB_MXFP_BLOCKS "${_lines} / 2") -math(EXPR BB_MXFP_MMIO_ROWS "${BB_MXFP_BLOCKS} / 16") -add_buckyball_mlir_test(mxfp4_unit_scale_bank TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/components/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank.mlir.in b/examples/components/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank.mlir.in deleted file mode 100644 index bc92a2351..000000000 --- a/examples/components/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank.mlir.in +++ /dev/null @@ -1,37 +0,0 @@ -func.func private @check_result(memref<@BB_BANK_LINES@x16xi8>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %raw = arith.constant -103 : i8 - %scale = arith.constant 127 : i8 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %inputDepth = arith.constant @BB_MXFP_BLOCKS@ : i64 - %outputDepth = arith.constant @BB_BANK_LINES@ : i64 - %scaleDepth = arith.constant @BB_MXFP_MMIO_ROWS@ : i64 - %columns = arith.constant 16 : i64 - %zero64 = arith.constant 0 : i64 - %stride = arith.constant 1 : i64 - %input = memref.alloc() : memref<@BB_MXFP_BLOCKS@x16xi8> - %scales = memref.alloc() : memref<@BB_MXFP_MMIO_ROWS@x16xi8> - %output = memref.alloc() : memref<@BB_BANK_LINES@x16xi8> - linalg.fill ins(%raw : i8) outs(%input : memref<@BB_MXFP_BLOCKS@x16xi8>) - linalg.fill ins(%scale : i8) outs(%scales : memref<@BB_MXFP_MMIO_ROWS@x16xi8>) - linalg.fill ins(%z8 : i8) outs(%output : memref<@BB_BANK_LINES@x16xi8>) - %inBank = buckyball.bank_alloc - %outBank = buckyball.bank_alloc - buckyball.mvin_mmio %scales %zero64 %scaleDepth %columns - : memref<@BB_MXFP_MMIO_ROWS@x16xi8> i64 i64 i64 - %loaded = buckyball.bank_mvin %input %inBank %inputDepth %stride - : memref<@BB_MXFP_BLOCKS@x16xi8> i64 i64 i64 - %result = buckyball.bank_mxfp2int %loaded %outBank %inputDepth %zero64 - : i64 i64 i64 i64 - %stored = buckyball.bank_mvout %output %result %outputDepth %stride - : memref<@BB_BANK_LINES@x16xi8> i64 i64 i64 - func.call @check_result(%output) : (memref<@BB_BANK_LINES@x16xi8>) -> () - buckyball.bank_release %loaded : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %input : memref<@BB_MXFP_BLOCKS@x16xi8> - memref.dealloc %scales : memref<@BB_MXFP_MMIO_ROWS@x16xi8> - memref.dealloc %output : memref<@BB_BANK_LINES@x16xi8> - return %z8 : i8 -} diff --git a/examples/components/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank_main.cpp b/examples/components/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank_main.cpp deleted file mode 100644 index 014f1d9fc..000000000 --- a/examples/components/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_unit_scale_bank_main.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include -#include -#include -#include -extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, - int64_t rows, int64_t lanes, int64_t stride, - int64_t lane_stride) { - if (rows != BANK_LINES || lanes != 16 || stride != 16 || lane_stride != 1) - exit(1); - for (int i = 0; i < rows * lanes; ++i) - if (data[offset + i] != 1) { - printf("mxfp4 unit-scale mismatch index=%d\n", i); - exit(1); - } - printf("mxfp4 unit-scale bank PASS rows=%d\n", BANK_LINES); -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/mxmm/workloads/mlir_tests/ball/CMakeLists.txt deleted file mode 100644 index 5a66bfbab..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/ball/CMakeLists.txt +++ /dev/null @@ -1,32 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX ball) -foreach(MODE mxfp8 fma32 f32) - add_buckyball_mlir_test(mxmm_${MODE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) - set(TEST_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_MLIR_ACTIVE_TARGET}-mlirtest-ball_mxmm_${MODE}" - ) - foreach(KIND baremetal linux) - add_custom_command( - OUTPUT ${TEST_NAME}-${KIND} - APPEND - DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_test.h - ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_steps.h - ) - endforeach() -endforeach() -foreach(CASE mxmm_window mxmm_window_rows mxmm_window_chain) - add_buckyball_mlir_test(${CASE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) - set(TEST_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_MLIR_ACTIVE_TARGET}-mlirtest-ball_${CASE}" - ) - foreach(KIND baremetal linux) - add_custom_command( - OUTPUT ${TEST_NAME}-${KIND} - APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../isa/mxmm.h - ) - endforeach() -endforeach() diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir deleted file mode 100644 index c7a5324b0..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir +++ /dev/null @@ -1,90 +0,0 @@ -func.func @step_m1_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32_main.cpp deleted file mode 100644 index e9dbeb578..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32_main.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#include "../mxmm_native_test.h" - -int main() { return run_mxmm_native_test(); } diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir deleted file mode 100644 index af6b58299..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir +++ /dev/null @@ -1,90 +0,0 @@ -func.func @step_m1_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.matmul_f32 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32_main.cpp deleted file mode 100644 index 539a04035..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32_main.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#include "../mxmm_native_test.h" - -int main() { return run_mxmm_native_test(); } diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8.mlir deleted file mode 100644 index 9632d76de..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8.mlir +++ /dev/null @@ -1,90 +0,0 @@ -func.func @step_m1_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032833 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032833 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032833 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163905 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163905 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m1_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163905 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032848 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032848 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032848 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163920 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163920 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m16_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163920 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032864 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032864 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032864 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163936 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163936 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} -func.func @step_m32_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163936 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - buckyball.mxfp8 %a, %b, %c, %shape, %first, %last, %base : i64 - return -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_main.cpp deleted file mode 100644 index deba14a6b..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_main.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#include "../mxmm_native_test.h" - -int main() { return run_mxmm_native_test(); } diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window.mlir deleted file mode 100644 index 07f60fb63..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window.mlir +++ /dev/null @@ -1,28 +0,0 @@ -// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 -// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F75]]) -// CHECK-LABEL: func.func @window_small -// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 -// CHECK-DAG: %[[RS1:[^ ]+]] = arith.constant 34367084547 : i64 -// CHECK-DAG: %[[RS2:[^ ]+]] = arith.constant 9007474250153985 : i64 -// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[RS1]], %[[RS2]], %[[F75]]) -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 - return -} -func.func @window_small() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 - %b = arith.constant 6 : i64 - %c = arith.constant 7 : i64 - %rows = arith.constant 1 : i64 - %cols = arith.constant 16 : i64 - %count = arith.constant 32 : i64 - %full = arith.constant 64 : i64 - %start = arith.constant 32 : i64 - %first = arith.constant true - %last = arith.constant true - %base = arith.constant 1 : i64 - buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 - return -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain.mlir deleted file mode 100644 index b8d641136..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain.mlir +++ /dev/null @@ -1,8 +0,0 @@ -// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 -// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F75]]) -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 - return -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain_main.cpp deleted file mode 100644 index bb3fc7a68..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_chain_main.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include "../../isa/mxmm.h" -#include "buckyball.h" -#include -#include -#include -#include -extern "C" void _mlir_ciface_window(uint64_t a, uint64_t b, uint64_t c, - uint64_t m, uint64_t n, uint64_t count, - uint64_t full, uint64_t start, bool first, - bool last, uint64_t base); - -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; -static uint8_t a[2][BYTES] __attribute__((aligned(64))); -static uint8_t packed[BYTES] __attribute__((aligned(64))); -static uint8_t w[BYTES] __attribute__((aligned(64))); -static uint8_t actual[BYTES] __attribute__((aligned(64))); -static uint8_t expected[BYTES] __attribute__((aligned(64))); -int main(void) { - memset(a, 255, sizeof(a)); - memset(a[0], 0x38, 32); - a[0][64] = 127; - memset(a[1] + 32, 0x40, 32); - a[1][97] = 127; - for (int col = 0; col < 16; ++col) { - memset(w + col * 32, col & 1 ? 0xb8 : 0x38, 32); - w[512 + col] = 127; - } - memset(actual, 0x5a, BYTES); - memset(expected, 0x5a, BYTES); - for (int bank = 3; bank <= 8; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a[0], 3, BANK_LINES, 1); - bb_mvin((uintptr_t)a[1], 4, BANK_LINES, 1); - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 7, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 8, BANK_LINES, 1); - for (int mode = 0; mode < 2; ++mode) - for (int part = 0; part < 2; ++part) { - if (mode == 0) { - memcpy(packed, a[part] + part * 32, 32); - packed[32] = 127; - bb_mvin((uintptr_t)packed, 5, 3, 1); - bb_mxmm_mxfp8(5, 6, 7, 1, 16, 32, part == 0, part == 1, 1); - bb_mvout((uintptr_t)expected, 7, BANK_LINES, 1); - } else { - _mlir_ciface_window(3 + part, 6, 8, 1, 16, 32, part ? 96 : 64, - part * 32, part == 0, part == 1, 1); - bb_mvout((uintptr_t)actual, 8, BANK_LINES, 1); - } - if (part == 0) - for (int i = 0; i < BYTES; ++i) - if ((mode ? actual : expected)[i] != 0x5a) - return 1; - } - for (int col = 0; col < 16; ++col) { - uint32_t bits; - memcpy(&bits, expected + 16 + col * 4, 4); - if (bits != (col & 1 ? 0xc2c00000u : 0x42c00000u)) - return 4; - } - if (memcmp(actual, expected, BYTES)) - return 2; - for (int i = 0; i < BYTES; ++i) - if ((i < 16 || i >= 80) && actual[i] != 0x5a) - return 3; - for (int bank = 3; bank <= 8; ++bank) - bb_mem_release(bank); - puts("mxmm A-window continuation bank/layout/poison/guard MLIR ball PASS"); - return 0; -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_main.cpp deleted file mode 100644 index f93584db2..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_main.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include "../../isa/mxmm.h" -#include "buckyball.h" -#include -#include -#include -#include -extern "C" void _mlir_ciface_window(uint64_t a, uint64_t b, uint64_t c, - uint64_t m, uint64_t n, uint64_t count, - uint64_t full, uint64_t start, bool first, - bool last, uint64_t base); -extern "C" void _mlir_ciface_window_small(); - -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; -static uint8_t a[BYTES] __attribute__((aligned(64))); -static uint8_t packed[BYTES] __attribute__((aligned(64))); -static uint8_t w[BYTES] __attribute__((aligned(64))); -static uint8_t actual[BYTES] __attribute__((aligned(64))); -static uint8_t expected[BYTES] __attribute__((aligned(64))); - -int main(void) { - const int shapes[][4] = {{1, 16, 64, 32}}; - const uint8_t codes[] = {0, 0x80, 1, 0x81, 0x38, - 0xb8, 0x37, 0xb7, 0x7e, 0xfe}; - const uint8_t scales[] = {0, 119, 127, 128}; - for (int test = 0; test < 1; ++test) { - int m = shapes[test][0], n = shapes[test][1]; - int full = shapes[test][2], start = shapes[test][3]; - memset(a, 0x5a, sizeof(a)); - for (int row = 0; row < m; ++row) { - for (int k = 0; k < full; ++k) - a[row * full + k] = k < start ? 0x7f : codes[(k + row) % 10]; - for (int block = 0; block < full / 32; ++block) - a[m * full + row * (full / 32) + block] = - block < start / 32 ? 255 : scales[(block + row) % 4]; - } - memset(actual, 0x5a, sizeof(actual)); - memset(expected, 0x5a, sizeof(expected)); - for (int bank = 3; bank <= 7; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 5, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 7, BANK_LINES, 1); - /* Complete old-71 chain before new-75 chain: one chain slot per hart. */ - for (int mode = 0; mode < 2; ++mode) { - for (int k = start; k < full; k += 480) { - int count = full - k < 480 ? full - k : 480; - memset(w, 0x5a, sizeof(w)); - for (int col = 0; col < n; ++col) { - for (int i = 0; i < count; ++i) - w[col * count + i] = codes[(k + i + col + 4) % 10]; - for (int block = 0; block < count / 32; ++block) - w[n * count + col * (count / 32) + block] = - scales[(k / 32 + block + col) % 4]; - } - if (mode == 0) { - memset(packed, 0x5a, sizeof(packed)); - for (int row = 0; row < m; ++row) { - memcpy(packed + row * count, a + row * full + k, count); - memcpy(packed + m * count + row * (count / 32), - a + m * full + row * (full / 32) + k / 32, count / 32); - } - } - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); - if (mode == 0) { - bb_mvin((uintptr_t)packed, 4, BANK_LINES, 1); - bb_mxmm_mxfp8(4, 6, 5, m, n, count, k == start, k + count == full, 1); - bb_mvout((uintptr_t)expected, 5, BANK_LINES, 1); - } else { - _mlir_ciface_window_small(); - bb_mvout((uintptr_t)actual, 7, BANK_LINES, 1); - } - if (k + count != full) - for (int i = 0; i < BYTES; ++i) - if ((mode ? actual : expected)[i] != 0x5a) - return 1; - } - } - if (memcmp(actual, expected, BYTES)) - return 2; - for (int i = 0; i < BYTES; ++i) - if ((i < 16 || i >= 16 + m * n * 4) && actual[i] != 0x5a) - return 3; - for (int bank = 3; bank <= 7; ++bank) - bb_mem_release(bank); - } - puts("mxmm A-window small old71/75/guard MLIR ball PASS"); - return 0; -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows.mlir deleted file mode 100644 index b8d641136..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows.mlir +++ /dev/null @@ -1,8 +0,0 @@ -// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 -// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F75]]) -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - buckyball.mxfp8_window %a, %b, %c, %rows, %cols, %count, %full, %start, %first, %last, %base : i64 - return -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows_main.cpp deleted file mode 100644 index e8197c986..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/ball/mxmm_window_rows_main.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include "../../isa/mxmm.h" -#include "buckyball.h" -#include -#include -#include -#include -extern "C" void _mlir_ciface_window(uint64_t a, uint64_t b, uint64_t c, - uint64_t m, uint64_t n, uint64_t count, - uint64_t full, uint64_t start, bool first, - bool last, uint64_t base); - -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; -static uint8_t a[BYTES] __attribute__((aligned(64))); -static uint8_t packed[BYTES] __attribute__((aligned(64))); -static uint8_t w[BYTES] __attribute__((aligned(64))); -static uint8_t actual[BYTES] __attribute__((aligned(64))); -static uint8_t expected[BYTES] __attribute__((aligned(64))); - -int main(void) { - const int shapes[][4] = {{16, 32, 960, 480}, {1, 32, 4096, 0}}; - const uint8_t codes[] = {0, 0x80, 1, 0x81, 0x38, - 0xb8, 0x37, 0xb7, 0x7e, 0xfe}; - const uint8_t scales[] = {0, 119, 127, 128}; - for (int test = 0; test < 2; ++test) { - int m = shapes[test][0], n = shapes[test][1]; - int full = shapes[test][2], start = shapes[test][3]; - memset(a, 0x5a, sizeof(a)); - for (int row = 0; row < m; ++row) { - for (int k = 0; k < full; ++k) - a[row * full + k] = k < start ? 0x7f : codes[(k + row) % 10]; - for (int block = 0; block < full / 32; ++block) - a[m * full + row * (full / 32) + block] = - block < start / 32 ? 255 : scales[(block + row) % 4]; - } - memset(actual, 0x5a, sizeof(actual)); - memset(expected, 0x5a, sizeof(expected)); - for (int bank = 3; bank <= 7; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 5, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 7, BANK_LINES, 1); - /* Complete old-71 chain before new-75 chain: one chain slot per hart. */ - for (int mode = 0; mode < 2; ++mode) { - for (int k = start; k < full; k += 480) { - int count = full - k < 480 ? full - k : 480; - memset(w, 0x5a, sizeof(w)); - for (int col = 0; col < n; ++col) { - for (int i = 0; i < count; ++i) - w[col * count + i] = codes[(k + i + col + 4) % 10]; - for (int block = 0; block < count / 32; ++block) - w[n * count + col * (count / 32) + block] = - scales[(k / 32 + block + col) % 4]; - } - if (mode == 0) { - memset(packed, 0x5a, sizeof(packed)); - for (int row = 0; row < m; ++row) { - memcpy(packed + row * count, a + row * full + k, count); - memcpy(packed + m * count + row * (count / 32), - a + m * full + row * (full / 32) + k / 32, count / 32); - } - } - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); - if (mode == 0) { - bb_mvin((uintptr_t)packed, 4, BANK_LINES, 1); - bb_mxmm_mxfp8(4, 6, 5, m, n, count, k == start, k + count == full, 1); - bb_mvout((uintptr_t)expected, 5, BANK_LINES, 1); - } else { - _mlir_ciface_window(3, 6, 7, m, n, count, full, k, k == start, - k + count == full, 1); - bb_mvout((uintptr_t)actual, 7, BANK_LINES, 1); - } - if (k + count != full) - for (int i = 0; i < BYTES; ++i) - if ((mode ? actual : expected)[i] != 0x5a) - return 1; - } - } - if (memcmp(actual, expected, BYTES)) - return 2; - for (int i = 0; i < BYTES; ++i) - if ((i < 16 || i >= 16 + m * n * 4) && actual[i] != 0x5a) - return 3; - for (int bank = 3; bank <= 7; ++bank) - bb_mem_release(bank); - } - puts("mxmm A-window row-gap/4096-chain/scale-byte/guard MLIR ball PASS"); - return 0; -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/mxmm/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index d055d2169..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,32 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX bank) -foreach(MODE mxfp8 fma32 f32) - add_buckyball_mlir_test(mxmm_${MODE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) - set(TEST_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_MLIR_ACTIVE_TARGET}-mlirtest-bank_mxmm_${MODE}" - ) - foreach(KIND baremetal linux) - add_custom_command( - OUTPUT ${TEST_NAME}-${KIND} - APPEND - DEPENDS - ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_test.h - ${CMAKE_CURRENT_SOURCE_DIR}/../mxmm_native_steps.h - ) - endforeach() -endforeach() -foreach(CASE mxmm_window mxmm_window_rows mxmm_window_chain) - add_buckyball_mlir_test(${CASE} TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) - set(TEST_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_MLIR_ACTIVE_TARGET}-mlirtest-bank_${CASE}" - ) - foreach(KIND baremetal linux) - add_custom_command( - OUTPUT ${TEST_NAME}-${KIND} - APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../isa/mxmm.h - ) - endforeach() -endforeach() diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir deleted file mode 100644 index b328dd843..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir +++ /dev/null @@ -1,90 +0,0 @@ -func.func @step_m1_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32_main.cpp deleted file mode 100644 index e9dbeb578..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32_main.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#include "../mxmm_native_test.h" - -int main() { return run_mxmm_native_test(); } diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir deleted file mode 100644 index 004ec5046..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir +++ /dev/null @@ -1,90 +0,0 @@ -func.func @step_m1_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807361 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938433 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807376 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938448 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073807392 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 1073938464 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_matmul_f32 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32_main.cpp deleted file mode 100644 index 539a04035..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32_main.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#include "../mxmm_native_test.h" - -int main() { return run_mxmm_native_test(); } diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8.mlir deleted file mode 100644 index 36c3a3500..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8.mlir +++ /dev/null @@ -1,90 +0,0 @@ -func.func @step_m1_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032833 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032833 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032833 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163905 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163905 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m1_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163905 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032848 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032848 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032848 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163920 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163920 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m16_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163920 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032864 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032864 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n16_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295032864 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_first() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163936 : i64 %first = arith.constant true %last = arith.constant false %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_last() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163936 : i64 %first = arith.constant false %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} -func.func @step_m32_n48_single() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 %b = arith.constant 4 : i64 %c = arith.constant 5 : i64 %shape = arith.constant 4295163936 : i64 %first = arith.constant true %last = arith.constant true %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8 %a %b %c %shape %first %last %base : i64 i64 i64 i64 - return -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_main.cpp deleted file mode 100644 index deba14a6b..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_main.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#include "../mxmm_native_test.h" - -int main() { return run_mxmm_native_test(); } diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window.mlir deleted file mode 100644 index 47a6b18b8..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window.mlir +++ /dev/null @@ -1,30 +0,0 @@ -// RUN: buddy-opt %s -assign-physical-banks | FileCheck %s --check-prefix=BANK -// BANK: buckyball.mxfp8_window -// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 -// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F75]]) -// CHECK-LABEL: func.func @window_small -// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 -// CHECK-DAG: %[[RS1:[^ ]+]] = arith.constant 34367084547 : i64 -// CHECK-DAG: %[[RS2:[^ ]+]] = arith.constant 9007474250153985 : i64 -// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[RS1]], %[[RS2]], %[[F75]]) -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 - return -} -func.func @window_small() attributes {llvm.emit_c_interface} { - %a = arith.constant 3 : i64 - %b = arith.constant 6 : i64 - %c = arith.constant 7 : i64 - %rows = arith.constant 1 : i64 - %cols = arith.constant 16 : i64 - %count = arith.constant 32 : i64 - %full = arith.constant 64 : i64 - %start = arith.constant 32 : i64 - %first = arith.constant true - %last = arith.constant true - %base = arith.constant 1 : i64 - %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 - return -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain.mlir deleted file mode 100644 index 52000bec5..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain.mlir +++ /dev/null @@ -1,10 +0,0 @@ -// RUN: buddy-opt %s -assign-physical-banks | FileCheck %s --check-prefix=BANK -// BANK: buckyball.mxfp8_window -// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 -// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F75]]) -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 - return -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain_main.cpp deleted file mode 100644 index c0d4394a4..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_chain_main.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include "../../isa/mxmm.h" -#include "buckyball.h" -#include -#include -#include -#include -extern "C" void _mlir_ciface_window(uint64_t a, uint64_t b, uint64_t c, - uint64_t m, uint64_t n, uint64_t count, - uint64_t full, uint64_t start, bool first, - bool last, uint64_t base); - -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; -static uint8_t a[2][BYTES] __attribute__((aligned(64))); -static uint8_t packed[BYTES] __attribute__((aligned(64))); -static uint8_t w[BYTES] __attribute__((aligned(64))); -static uint8_t actual[BYTES] __attribute__((aligned(64))); -static uint8_t expected[BYTES] __attribute__((aligned(64))); -int main(void) { - memset(a, 255, sizeof(a)); - memset(a[0], 0x38, 32); - a[0][64] = 127; - memset(a[1] + 32, 0x40, 32); - a[1][97] = 127; - for (int col = 0; col < 16; ++col) { - memset(w + col * 32, col & 1 ? 0xb8 : 0x38, 32); - w[512 + col] = 127; - } - memset(actual, 0x5a, BYTES); - memset(expected, 0x5a, BYTES); - for (int bank = 3; bank <= 8; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a[0], 3, BANK_LINES, 1); - bb_mvin((uintptr_t)a[1], 4, BANK_LINES, 1); - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 7, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 8, BANK_LINES, 1); - for (int mode = 0; mode < 2; ++mode) - for (int part = 0; part < 2; ++part) { - if (mode == 0) { - memcpy(packed, a[part] + part * 32, 32); - packed[32] = 127; - bb_mvin((uintptr_t)packed, 5, 3, 1); - bb_mxmm_mxfp8(5, 6, 7, 1, 16, 32, part == 0, part == 1, 1); - bb_mvout((uintptr_t)expected, 7, BANK_LINES, 1); - } else { - _mlir_ciface_window(3 + part, 6, 8, 1, 16, 32, part ? 96 : 64, - part * 32, part == 0, part == 1, 1); - bb_mvout((uintptr_t)actual, 8, BANK_LINES, 1); - } - if (part == 0) - for (int i = 0; i < BYTES; ++i) - if ((mode ? actual : expected)[i] != 0x5a) - return 1; - } - for (int col = 0; col < 16; ++col) { - uint32_t bits; - memcpy(&bits, expected + 16 + col * 4, 4); - if (bits != (col & 1 ? 0xc2c00000u : 0x42c00000u)) - return 4; - } - if (memcmp(actual, expected, BYTES)) - return 2; - for (int i = 0; i < BYTES; ++i) - if ((i < 16 || i >= 80) && actual[i] != 0x5a) - return 3; - for (int bank = 3; bank <= 8; ++bank) - bb_mem_release(bank); - puts("mxmm A-window continuation bank/layout/poison/guard MLIR bank PASS"); - return 0; -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_main.cpp deleted file mode 100644 index 1ac1d78df..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_main.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include "../../isa/mxmm.h" -#include "buckyball.h" -#include -#include -#include -#include -extern "C" void _mlir_ciface_window(uint64_t a, uint64_t b, uint64_t c, - uint64_t m, uint64_t n, uint64_t count, - uint64_t full, uint64_t start, bool first, - bool last, uint64_t base); -extern "C" void _mlir_ciface_window_small(); - -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; -static uint8_t a[BYTES] __attribute__((aligned(64))); -static uint8_t packed[BYTES] __attribute__((aligned(64))); -static uint8_t w[BYTES] __attribute__((aligned(64))); -static uint8_t actual[BYTES] __attribute__((aligned(64))); -static uint8_t expected[BYTES] __attribute__((aligned(64))); - -int main(void) { - const int shapes[][4] = {{1, 16, 64, 32}}; - const uint8_t codes[] = {0, 0x80, 1, 0x81, 0x38, - 0xb8, 0x37, 0xb7, 0x7e, 0xfe}; - const uint8_t scales[] = {0, 119, 127, 128}; - for (int test = 0; test < 1; ++test) { - int m = shapes[test][0], n = shapes[test][1]; - int full = shapes[test][2], start = shapes[test][3]; - memset(a, 0x5a, sizeof(a)); - for (int row = 0; row < m; ++row) { - for (int k = 0; k < full; ++k) - a[row * full + k] = k < start ? 0x7f : codes[(k + row) % 10]; - for (int block = 0; block < full / 32; ++block) - a[m * full + row * (full / 32) + block] = - block < start / 32 ? 255 : scales[(block + row) % 4]; - } - memset(actual, 0x5a, sizeof(actual)); - memset(expected, 0x5a, sizeof(expected)); - for (int bank = 3; bank <= 7; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 5, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 7, BANK_LINES, 1); - /* Complete old-71 chain before new-75 chain: one chain slot per hart. */ - for (int mode = 0; mode < 2; ++mode) { - for (int k = start; k < full; k += 480) { - int count = full - k < 480 ? full - k : 480; - memset(w, 0x5a, sizeof(w)); - for (int col = 0; col < n; ++col) { - for (int i = 0; i < count; ++i) - w[col * count + i] = codes[(k + i + col + 4) % 10]; - for (int block = 0; block < count / 32; ++block) - w[n * count + col * (count / 32) + block] = - scales[(k / 32 + block + col) % 4]; - } - if (mode == 0) { - memset(packed, 0x5a, sizeof(packed)); - for (int row = 0; row < m; ++row) { - memcpy(packed + row * count, a + row * full + k, count); - memcpy(packed + m * count + row * (count / 32), - a + m * full + row * (full / 32) + k / 32, count / 32); - } - } - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); - if (mode == 0) { - bb_mvin((uintptr_t)packed, 4, BANK_LINES, 1); - bb_mxmm_mxfp8(4, 6, 5, m, n, count, k == start, k + count == full, 1); - bb_mvout((uintptr_t)expected, 5, BANK_LINES, 1); - } else { - _mlir_ciface_window_small(); - bb_mvout((uintptr_t)actual, 7, BANK_LINES, 1); - } - if (k + count != full) - for (int i = 0; i < BYTES; ++i) - if ((mode ? actual : expected)[i] != 0x5a) - return 1; - } - } - if (memcmp(actual, expected, BYTES)) - return 2; - for (int i = 0; i < BYTES; ++i) - if ((i < 16 || i >= 16 + m * n * 4) && actual[i] != 0x5a) - return 3; - for (int bank = 3; bank <= 7; ++bank) - bb_mem_release(bank); - } - puts("mxmm A-window small old71/75/guard MLIR bank PASS"); - return 0; -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows.mlir b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows.mlir deleted file mode 100644 index 52000bec5..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows.mlir +++ /dev/null @@ -1,10 +0,0 @@ -// RUN: buddy-opt %s -assign-physical-banks | FileCheck %s --check-prefix=BANK -// BANK: buckyball.mxfp8_window -// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @window -// CHECK-DAG: %[[F75:.*]] = llvm.mlir.constant(75 : i32) : i32 -// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F75]]) -func.func @window(%a: i64, %b: i64, %c: i64, %rows: i64, %cols: i64, %count: i64, %full: i64, %start: i64, %first: i1, %last: i1, %base: i64) attributes {llvm.emit_c_interface} { - %state = buckyball.bank_mxfp8_window %a %b %c %rows %cols %count %full %start %first %last %base : i64 - return -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows_main.cpp b/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows_main.cpp deleted file mode 100644 index b35b1028f..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/bank/mxmm_window_rows_main.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include "../../isa/mxmm.h" -#include "buckyball.h" -#include -#include -#include -#include -extern "C" void _mlir_ciface_window(uint64_t a, uint64_t b, uint64_t c, - uint64_t m, uint64_t n, uint64_t count, - uint64_t full, uint64_t start, bool first, - bool last, uint64_t base); - -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; -static uint8_t a[BYTES] __attribute__((aligned(64))); -static uint8_t packed[BYTES] __attribute__((aligned(64))); -static uint8_t w[BYTES] __attribute__((aligned(64))); -static uint8_t actual[BYTES] __attribute__((aligned(64))); -static uint8_t expected[BYTES] __attribute__((aligned(64))); - -int main(void) { - const int shapes[][4] = {{16, 32, 960, 480}, {1, 32, 4096, 0}}; - const uint8_t codes[] = {0, 0x80, 1, 0x81, 0x38, - 0xb8, 0x37, 0xb7, 0x7e, 0xfe}; - const uint8_t scales[] = {0, 119, 127, 128}; - for (int test = 0; test < 2; ++test) { - int m = shapes[test][0], n = shapes[test][1]; - int full = shapes[test][2], start = shapes[test][3]; - memset(a, 0x5a, sizeof(a)); - for (int row = 0; row < m; ++row) { - for (int k = 0; k < full; ++k) - a[row * full + k] = k < start ? 0x7f : codes[(k + row) % 10]; - for (int block = 0; block < full / 32; ++block) - a[m * full + row * (full / 32) + block] = - block < start / 32 ? 255 : scales[(block + row) % 4]; - } - memset(actual, 0x5a, sizeof(actual)); - memset(expected, 0x5a, sizeof(expected)); - for (int bank = 3; bank <= 7; ++bank) - bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 5, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 7, BANK_LINES, 1); - /* Complete old-71 chain before new-75 chain: one chain slot per hart. */ - for (int mode = 0; mode < 2; ++mode) { - for (int k = start; k < full; k += 480) { - int count = full - k < 480 ? full - k : 480; - memset(w, 0x5a, sizeof(w)); - for (int col = 0; col < n; ++col) { - for (int i = 0; i < count; ++i) - w[col * count + i] = codes[(k + i + col + 4) % 10]; - for (int block = 0; block < count / 32; ++block) - w[n * count + col * (count / 32) + block] = - scales[(k / 32 + block + col) % 4]; - } - if (mode == 0) { - memset(packed, 0x5a, sizeof(packed)); - for (int row = 0; row < m; ++row) { - memcpy(packed + row * count, a + row * full + k, count); - memcpy(packed + m * count + row * (count / 32), - a + m * full + row * (full / 32) + k / 32, count / 32); - } - } - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); - if (mode == 0) { - bb_mvin((uintptr_t)packed, 4, BANK_LINES, 1); - bb_mxmm_mxfp8(4, 6, 5, m, n, count, k == start, k + count == full, 1); - bb_mvout((uintptr_t)expected, 5, BANK_LINES, 1); - } else { - _mlir_ciface_window(3, 6, 7, m, n, count, full, k, k == start, - k + count == full, 1); - bb_mvout((uintptr_t)actual, 7, BANK_LINES, 1); - } - if (k + count != full) - for (int i = 0; i < BYTES; ++i) - if ((mode ? actual : expected)[i] != 0x5a) - return 1; - } - } - if (memcmp(actual, expected, BYTES)) - return 2; - for (int i = 0; i < BYTES; ++i) - if ((i < 16 || i >= 16 + m * n * 4) && actual[i] != 0x5a) - return 3; - for (int bank = 3; bank <= 7; ++bank) - bb_mem_release(bank); - } - puts("mxmm A-window row-gap/4096-chain/scale-byte/guard MLIR bank PASS"); - return 0; -} diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/mxmm_native_steps.h b/examples/components/balls/mxmm/workloads/mlir_tests/mxmm_native_steps.h deleted file mode 100644 index 0d48c8f97..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/mxmm_native_steps.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -extern "C" void _mlir_ciface_step_m1_n16_first(); -extern "C" void _mlir_ciface_step_m1_n16_last(); -extern "C" void _mlir_ciface_step_m1_n16_single(); -extern "C" void _mlir_ciface_step_m1_n48_first(); -extern "C" void _mlir_ciface_step_m1_n48_last(); -extern "C" void _mlir_ciface_step_m1_n48_single(); -extern "C" void _mlir_ciface_step_m16_n16_first(); -extern "C" void _mlir_ciface_step_m16_n16_last(); -extern "C" void _mlir_ciface_step_m16_n16_single(); -extern "C" void _mlir_ciface_step_m16_n48_first(); -extern "C" void _mlir_ciface_step_m16_n48_last(); -extern "C" void _mlir_ciface_step_m16_n48_single(); -extern "C" void _mlir_ciface_step_m32_n16_first(); -extern "C" void _mlir_ciface_step_m32_n16_last(); -extern "C" void _mlir_ciface_step_m32_n16_single(); -extern "C" void _mlir_ciface_step_m32_n48_first(); -extern "C" void _mlir_ciface_step_m32_n48_last(); -extern "C" void _mlir_ciface_step_m32_n48_single(); -static void (*steps[3][2][3])() = { - {{_mlir_ciface_step_m1_n16_first, _mlir_ciface_step_m1_n16_last, - _mlir_ciface_step_m1_n16_single}, - {_mlir_ciface_step_m1_n48_first, _mlir_ciface_step_m1_n48_last, - _mlir_ciface_step_m1_n48_single}}, - {{_mlir_ciface_step_m16_n16_first, _mlir_ciface_step_m16_n16_last, - _mlir_ciface_step_m16_n16_single}, - {_mlir_ciface_step_m16_n48_first, _mlir_ciface_step_m16_n48_last, - _mlir_ciface_step_m16_n48_single}}, - {{_mlir_ciface_step_m32_n16_first, _mlir_ciface_step_m32_n16_last, - _mlir_ciface_step_m32_n16_single}, - {_mlir_ciface_step_m32_n48_first, _mlir_ciface_step_m32_n48_last, - _mlir_ciface_step_m32_n48_single}}}; diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/mxmm_native_test.h b/examples/components/balls/mxmm/workloads/mlir_tests/mxmm_native_test.h deleted file mode 100644 index 7d047db9d..000000000 --- a/examples/components/balls/mxmm/workloads/mlir_tests/mxmm_native_test.h +++ /dev/null @@ -1,89 +0,0 @@ -#include "mxmm_native_steps.h" -#include -#include -#include -#include -#include -#include - -enum class MxmmFormat { Mxfp8, Fma32, F32 }; - -template static int run_mxmm_native_test() { - constexpr int K = Format == MxmmFormat::Mxfp8 ? 256 : 64; - constexpr int bytes = BANK_LINES * BANK_WIDTH / 8; - alignas(bytes) uint8_t a[bytes]{}, b[bytes]{}; - alignas(bytes) uint32_t result[bytes / 4]; - for (int mindex = 0; mindex < 3; mindex++) - for (int nindex = 0; nindex < 2; nindex++) { - int m = mindex == 0 ? 1 : mindex * 16, n = nindex == 0 ? 16 : 48; - memset(a, 0, sizeof(a)); - memset(b, 0, sizeof(b)); - if (Format == MxmmFormat::Mxfp8) { - memset(a, 0x38, m * K); - memset(a + m * K, 127, m * K / 32); - memset(b, 0x40, n * K); - memset(b + n * K, 127, n * K / 32); - } else { - auto *x = (uint32_t *)a; - auto *y = (uint32_t *)b; - for (int r = 0; r < m; r++) { - x[r * K] = 0xbf800000; - x[r * K + 1] = 0x3f800001; - } - for (int c = 0; c < n; c++) { - y[c * K] = 0x3f800000; - y[c * K + 1] = 0x3f7ffffe; - } - } - for (auto &v : result) - v = 0x5a5a5a5a; - for (int bank = 3; bank <= 5; bank++) - bb_mem_alloc(bank, 1, 1); - int ar = Format == MxmmFormat::Mxfp8 ? (m * K * 33 / 32 + 15) / 16 - : m * K / 4, - br = Format == MxmmFormat::Mxfp8 ? n * K * 33 / 32 / 16 : n * K / 4; - bb_mvin((uintptr_t)a, 3, ar, 1); - bb_mvin((uintptr_t)b, 4, br, 1); - bb_mvin((uintptr_t)result, 5, 1 + m * n / 4, 1); - steps[mindex][nindex][0](); - bb_mvout((uintptr_t)result, 5, 1 + m * n / 4, 1); - for (int i = 0; i < 4 + m * n; i++) - if (result[i] != 0x5a5a5a5a) - return 1; - if (Format == MxmmFormat::Mxfp8) { - memset(a, 0xb8, m * K); - memset(b, 0x38, n * K); - } else - memset(a, 0, m * K * 4); - bb_mvin((uintptr_t)a, 3, ar, 1); - bb_mvin((uintptr_t)b, 4, br, 1); - steps[mindex][nindex][1](); - bb_mvout((uintptr_t)result, 5, 1 + m * n / 4, 1); - uint32_t expected = Format == MxmmFormat::Mxfp8 ? 0x43800000 - : Format == MxmmFormat::Fma32 ? 0xa8800000 - : 0; - for (int i = 0; i < 4; i++) - if (result[i] != 0x5a5a5a5a) - return 2; - for (int i = 4; i < 4 + m * n; i++) - if (result[i] != expected) - return 3; - if (Format == MxmmFormat::Mxfp8) { - memset(a, 0, sizeof(a)); - memset(b, 0, sizeof(b)); - memset(a, 0x81, m * K); - memset(b, 1, n * K); - bb_mvin((uintptr_t)a, 3, ar, 1); - bb_mvin((uintptr_t)b, 4, br, 1); - steps[mindex][nindex][2](); - bb_mvout((uintptr_t)result, 5, 1 + m * n / 4, 1); - for (int i = 4; i < 4 + m * n; i++) - if (result[i] != 0x80000000) - return 4; - } - for (int bank = 3; bank <= 5; bank++) - bb_mem_release(bank); - } - puts("Mxmm MLIR mode/shape/chain PASS"); - return 0; -} diff --git a/examples/components/balls/mxquant/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/mxquant/workloads/mlir_tests/CMakeLists.txt deleted file mode 100644 index 9d0ed5a15..000000000 --- a/examples/components/balls/mxquant/workloads/mlir_tests/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -add_subdirectory(ball) -add_subdirectory(bank) diff --git a/examples/components/balls/mxquant/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/mxquant/workloads/mlir_tests/ball/CMakeLists.txt deleted file mode 100644 index e0f52cc49..000000000 --- a/examples/components/balls/mxquant/workloads/mlir_tests/ball/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -set(BUCKYBALL_MLIR_TEST_PREFIX ball) -add_buckyball_mlir_test(mxquant TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) -set(TEST_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_MLIR_ACTIVE_TARGET}-mlirtest-ball_mxquant" -) -foreach(KIND baremetal linux) - add_custom_command( - OUTPUT ${TEST_NAME}-${KIND} - APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../reference.h - ) -endforeach() diff --git a/examples/components/balls/mxquant/workloads/mlir_tests/ball/mxquant.mlir b/examples/components/balls/mxquant/workloads/mlir_tests/ball/mxquant.mlir deleted file mode 100644 index 67f4775d0..000000000 --- a/examples/components/balls/mxquant/workloads/mlir_tests/ball/mxquant.mlir +++ /dev/null @@ -1,22 +0,0 @@ -// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @quant_block_32 -// CHECK-DAG: %[[F54:.*]] = llvm.mlir.constant(54 : i32) : i32 -// CHECK: %[[ZERO:[^ ]+]] = arith.constant 0 : i64 -// CHECK: %[[PACKED:[^ ]+]] = arith.constant 34363932675 : i64 -// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[PACKED]], %[[ZERO]], %[[F54]]) -// CHECK-LABEL: func.func @quant_dynamic -// CHECK-DAG: %[[F54:.*]] = llvm.mlir.constant(54 : i32) : i32 -// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F54]]) -func.func @quant_block_32() attributes {llvm.emit_c_interface} { - %input = arith.constant 3 : i64 - %output = arith.constant 4 : i64 - %count = arith.constant 32 : i64 - buckyball.mxquant %input, %output, %count : i64 - return -} -func.func @quant_dynamic(%count: i64) attributes {llvm.emit_c_interface} { - %input = arith.constant 3 : i64 - %output = arith.constant 4 : i64 - buckyball.mxquant %input, %output, %count : i64 - return -} diff --git a/examples/components/balls/mxquant/workloads/mlir_tests/ball/mxquant_main.cpp b/examples/components/balls/mxquant/workloads/mlir_tests/ball/mxquant_main.cpp deleted file mode 100644 index d11774f36..000000000 --- a/examples/components/balls/mxquant/workloads/mlir_tests/ball/mxquant_main.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include -#include -extern "C" void _mlir_ciface_quant_block_32(); -extern "C" void _mlir_ciface_quant_dynamic(int64_t count); -#include "../../reference.h" -static uint32_t input[32] __attribute__((aligned(64))); -static uint8_t actual[64] __attribute__((aligned(64))); -static uint8_t expected[64]; -int main(void) { - const uint32_t edges[] = {0, 0x80000000, 1, 0x80000001, - 0x007fffff, 0x00800000, 0x3f880000, 0x3f980000, - 0x43e00000, 0x7f7fffff, 0xff7fffff}; - const uint32_t subnormals[] = {0, 1, 0x1000, 0x2000, - 0x3000, 0x7fffff, 0x80000001, 0x807fffff}; - uint32_t random = 0x6d2b79f5; - for (int test = 0; test < 68; ++test) { - for (int i = 0; i < 32; ++i) { - random ^= random << 13; - random ^= random >> 17; - random ^= random << 5; - input[i] = - test == 0 ? (i & 1 ? 0x80000000u : 0) - : test == 1 ? edges[i % 11] - : test == 2 ? (i ? (i & 1 ? 0x3f880000u : 0x3f980000u) : 0x43e00000u) - : test == 3 ? subnormals[i % 8] - : ((random & 0x807fffffu) | - ((((test - 4) * 4 + (random >> 23 & 3)) % 255) << 23)); - } - memset(actual, 0x5a, sizeof(actual)); - memset(expected, 0x5a, sizeof(expected)); - mxquant_reference32(input, expected); - if (test == 0 && - (expected[0] != 0 || expected[1] != 128 || expected[32] != 127)) - return 1; - if (test == 1 && - (expected[9] != 126 || expected[10] != 254 || expected[32] != 246)) - return 2; - if (test == 2 && - (expected[0] != 126 || expected[1] != 56 || expected[2] != 58)) - return 2; - uint32_t state = 0x1fu | (test % 5) << 5, after; - asm volatile("csrw fcsr, %0" : : "r"(state)); - bb_mem_alloc(3, 1, 1); - bb_mem_alloc(4, 1, 1); - bb_mvin((uintptr_t)input, 3, 8, 1); - bb_mvin((uintptr_t)actual, 4, 4, 1); - _mlir_ciface_quant_block_32(); - bb_mvout((uintptr_t)actual, 4, 4, 1); - asm volatile("csrr %0, fcsr" : "=r"(after)); - if (after != state || memcmp(actual, expected, sizeof(actual))) - return 3; - bb_mem_release(3); - bb_mem_release(4); - } - enum { BANK_BYTES = BANK_LINES * BANK_WIDTH / 8, MAX_COUNT = BANK_BYTES / 4 }; - static uint32_t capacity_input[MAX_COUNT] __attribute__((aligned(64))); - static uint8_t capacity_actual[BANK_BYTES] __attribute__((aligned(64))); - const int counts[] = {32, 480, 960, MAX_COUNT}; - for (int test = 0; test < 4; ++test) { - int count = counts[test]; - for (int i = 0; i < MAX_COUNT; ++i) - capacity_input[i] = (i & 1) ? 0xbf800000u : 0x3f800000u; - memset(capacity_actual, 0x5a, sizeof(capacity_actual)); - bb_mem_alloc(3, 1, 1); - bb_mem_alloc(4, 1, 1); - bb_mvin((uintptr_t)capacity_input, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)capacity_actual, 4, BANK_LINES, 1); - _mlir_ciface_quant_dynamic(count); - bb_mvout((uintptr_t)capacity_actual, 4, BANK_LINES, 1); - for (int i = 0; i < BANK_BYTES; ++i) { - uint8_t expected = i < count ? ((i & 1) ? 0xf8 : 0x78) - : i < count + count / 32 ? 119 - : 0x5a; - if (capacity_actual[i] != expected) - return 1; - } - bb_mem_release(3); - bb_mem_release(4); - } - puts("mxquant ball MLIR finite/RNE/signed-zero/capacity/guard PASS"); - return 0; -} diff --git a/examples/components/balls/mxquant/workloads/mlir_tests/bank/CMakeLists.txt b/examples/components/balls/mxquant/workloads/mlir_tests/bank/CMakeLists.txt deleted file mode 100644 index a4a7d11f5..000000000 --- a/examples/components/balls/mxquant/workloads/mlir_tests/bank/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -set(BUCKYBALL_MLIR_TEST_PREFIX bank) -add_buckyball_mlir_test(mxquant TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) -set(TEST_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_MLIR_ACTIVE_TARGET}-mlirtest-bank_mxquant" -) -foreach(KIND baremetal linux) - add_custom_command( - OUTPUT ${TEST_NAME}-${KIND} - APPEND - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../reference.h - ) -endforeach() diff --git a/examples/components/balls/mxquant/workloads/mlir_tests/bank/mxquant.mlir b/examples/components/balls/mxquant/workloads/mlir_tests/bank/mxquant.mlir deleted file mode 100644 index 55cbb2106..000000000 --- a/examples/components/balls/mxquant/workloads/mlir_tests/bank/mxquant.mlir +++ /dev/null @@ -1,27 +0,0 @@ -// RUN: buddy-opt %s -assign-physical-banks | FileCheck %s --check-prefix=BANK -// BANK-LABEL: func.func @quant_block_32 -// BANK: buckyball.mxquant -// BANK-LABEL: func.func @quant_dynamic -// BANK: buckyball.mxquant -// RUN: buddy-opt %s -assign-physical-banks -lower-buckyball -canonicalize | FileCheck %s -// CHECK-LABEL: func.func @quant_block_32 -// CHECK-DAG: %[[F54:.*]] = llvm.mlir.constant(54 : i32) : i32 -// CHECK: %[[ZERO:[^ ]+]] = arith.constant 0 : i64 -// CHECK: %[[PACKED:[^ ]+]] = arith.constant 34363932675 : i64 -// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"(%[[PACKED]], %[[ZERO]], %[[F54]]) -// CHECK-LABEL: func.func @quant_dynamic -// CHECK-DAG: %[[F54:.*]] = llvm.mlir.constant(54 : i32) : i32 -// CHECK: llvm.call_intrinsic "llvm.riscv.buckyball.custom"({{.*}}, %[[F54]]) -func.func @quant_block_32() attributes {llvm.emit_c_interface} { - %input = arith.constant 3 : i64 - %output = arith.constant 4 : i64 - %count = arith.constant 32 : i64 - %result = buckyball.bank_mxquant %input %output %count : i64 i64 i64 - return -} -func.func @quant_dynamic(%count: i64) attributes {llvm.emit_c_interface} { - %input = arith.constant 3 : i64 - %output = arith.constant 4 : i64 - %result = buckyball.bank_mxquant %input %output %count : i64 i64 i64 - return -} diff --git a/examples/components/balls/mxquant/workloads/mlir_tests/bank/mxquant_main.cpp b/examples/components/balls/mxquant/workloads/mlir_tests/bank/mxquant_main.cpp deleted file mode 100644 index 8f98f53be..000000000 --- a/examples/components/balls/mxquant/workloads/mlir_tests/bank/mxquant_main.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include "buckyball.h" -#include -#include -#include -#include -#include -extern "C" void _mlir_ciface_quant_block_32(); -extern "C" void _mlir_ciface_quant_dynamic(int64_t count); -#include "../../reference.h" -static uint32_t input[32] __attribute__((aligned(64))); -static uint8_t actual[64] __attribute__((aligned(64))); -static uint8_t expected[64]; -int main(void) { - const uint32_t edges[] = {0, 0x80000000, 1, 0x80000001, - 0x007fffff, 0x00800000, 0x3f880000, 0x3f980000, - 0x43e00000, 0x7f7fffff, 0xff7fffff}; - const uint32_t subnormals[] = {0, 1, 0x1000, 0x2000, - 0x3000, 0x7fffff, 0x80000001, 0x807fffff}; - uint32_t random = 0x6d2b79f5; - for (int test = 0; test < 68; ++test) { - for (int i = 0; i < 32; ++i) { - random ^= random << 13; - random ^= random >> 17; - random ^= random << 5; - input[i] = - test == 0 ? (i & 1 ? 0x80000000u : 0) - : test == 1 ? edges[i % 11] - : test == 2 ? (i ? (i & 1 ? 0x3f880000u : 0x3f980000u) : 0x43e00000u) - : test == 3 ? subnormals[i % 8] - : ((random & 0x807fffffu) | - ((((test - 4) * 4 + (random >> 23 & 3)) % 255) << 23)); - } - memset(actual, 0x5a, sizeof(actual)); - memset(expected, 0x5a, sizeof(expected)); - mxquant_reference32(input, expected); - if (test == 0 && - (expected[0] != 0 || expected[1] != 128 || expected[32] != 127)) - return 1; - if (test == 1 && - (expected[9] != 126 || expected[10] != 254 || expected[32] != 246)) - return 2; - if (test == 2 && - (expected[0] != 126 || expected[1] != 56 || expected[2] != 58)) - return 2; - uint32_t state = 0x1fu | (test % 5) << 5, after; - asm volatile("csrw fcsr, %0" : : "r"(state)); - bb_mem_alloc(3, 1, 1); - bb_mem_alloc(4, 1, 1); - bb_mvin((uintptr_t)input, 3, 8, 1); - bb_mvin((uintptr_t)actual, 4, 4, 1); - _mlir_ciface_quant_block_32(); - bb_mvout((uintptr_t)actual, 4, 4, 1); - asm volatile("csrr %0, fcsr" : "=r"(after)); - if (after != state || memcmp(actual, expected, sizeof(actual))) - return 3; - bb_mem_release(3); - bb_mem_release(4); - } - enum { BANK_BYTES = BANK_LINES * BANK_WIDTH / 8, MAX_COUNT = BANK_BYTES / 4 }; - static uint32_t capacity_input[MAX_COUNT] __attribute__((aligned(64))); - static uint8_t capacity_actual[BANK_BYTES] __attribute__((aligned(64))); - const int counts[] = {32, 480, 960, MAX_COUNT}; - for (int test = 0; test < 4; ++test) { - int count = counts[test]; - for (int i = 0; i < MAX_COUNT; ++i) - capacity_input[i] = (i & 1) ? 0xbf800000u : 0x3f800000u; - memset(capacity_actual, 0x5a, sizeof(capacity_actual)); - bb_mem_alloc(3, 1, 1); - bb_mem_alloc(4, 1, 1); - bb_mvin((uintptr_t)capacity_input, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)capacity_actual, 4, BANK_LINES, 1); - _mlir_ciface_quant_dynamic(count); - bb_mvout((uintptr_t)capacity_actual, 4, BANK_LINES, 1); - for (int i = 0; i < BANK_BYTES; ++i) { - uint8_t expected = i < count ? ((i & 1) ? 0xf8 : 0x78) - : i < count + count / 32 ? 119 - : 0x5a; - if (capacity_actual[i] != expected) - return 1; - } - bb_mem_release(3); - bb_mem_release(4); - } - puts("mxquant bank MLIR finite/RNE/signed-zero/capacity/guard PASS"); - return 0; -} diff --git a/examples/components/balls/redmule/stack/src/Lowering.cpp b/examples/components/balls/redmule/stack/src/Lowering.cpp deleted file mode 100644 index db4c60ed3..000000000 --- a/examples/components/balls/redmule/stack/src/Lowering.cpp +++ /dev/null @@ -1,66 +0,0 @@ -//===- Lowering.cpp - RedmuleBall in lower-buckyball ----------------------===// -// -// matmul_f16: C = A @ B in FP16 with ordered FMAs, or C = C + A @ B with -// read_destination. `k` is a positive multiple of eight. -// A (source 0): `rows` rows of `k` f16, k / 8 bank rows each. -// B (source 1): `k` bank rows of eight f16. -// C: `rows` bank rows of eight f16. -// -// The Ball places every operand at bank row 0. With read_destination rs1 -// marks the old C, which the Ball reads from the bank the destination held -// before this write, and the FMAs start from it instead of positive zero. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/BuckyballOps.h" -#include "Conversion/Buckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class MatmulF16Lowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &rewriter) const override { - if (op.getKind() != "matmul_f16") - return failure(); - int64_t rows = cast(op->getAttr("rows")).getInt(); - int64_t k = cast(op->getAttr("k")).getInt(); - int64_t c = op.getDestination(); - bool readDestination = op.getReadDestination(); - const auto &target = buckyball_target::getBuckyballTarget(); - if (op.getSources().size() != 2 || - op.getSources()[0] == op.getSources()[1] || - llvm::is_contained(op.getSources(), c)) - return op.emitOpError("FP16 matmul requires distinct A, B and C banks"); - if (target.bankWidthBits != 128 || rows <= 0 || rows > 4095 || k <= 0 || - k % 8 || k >= (1LL << 33) || rows * (k / 8) > target.bankDepth || - k > target.bankDepth || rows > target.bankDepth) - return op.emitOpError("FP16 matmul shape exceeds the bank or ISA " - "geometry"); - // rs2: rows in [11:0], the A, B and C row bases (all 0) in [41:12], and - // in bit 42 whether the FMAs start from positive zero. - uint64_t rs2 = mlir::buddy::field(rows, 0, 11) | - mlir::buddy::field(!readDestination, 42, 42); - mlir::buddy::emitInstruction( - rewriter, op.getLoc(), - buckyball_target::getBuckyballFunct7("REDMULE_F16"), - mlir::buddy::constantI64( - rewriter, op.getLoc(), - mlir::buddy::bankFields(op.getSources(), c, readDestination, k)), - mlir::buddy::constantI64(rewriter, op.getLoc(), rs2)); - rewriter.eraseOp(op); - return success(); - } -}; - -void populate(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -[[maybe_unused]] const bool registered = - (mlir::buddy::addBallLowering(populate), true); -} // namespace diff --git a/examples/components/balls/redmule/workloads/ctests/CMakeLists.txt b/examples/components/balls/redmule/workloads/ctests/CMakeLists.txt deleted file mode 100644 index 448016697..000000000 --- a/examples/components/balls/redmule/workloads/ctests/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -file( - READ "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h" - REDMULE_PARAMS -) -string(REGEX MATCH "#define BANK_WIDTH ([0-9]+)" _ "${REDMULE_PARAMS}") -set(REDMULE_BANK_WIDTH ${CMAKE_MATCH_1}) -string(REGEX MATCH "#define BANK_LINES ([0-9]+)" _ "${REDMULE_PARAMS}") -if(NOT REDMULE_BANK_WIDTH EQUAL 128 OR CMAKE_MATCH_1 LESS 18) - message( - FATAL_ERROR - "RedmuleBall tests require 128-bit rows and at least 18 bank rows, got ${REDMULE_BANK_WIDTH} bits and ${CMAKE_MATCH_1} rows" - ) -endif() -add_buckyball_ctest(redmule_f16_test.c) -add_buckyball_ctest(redmule_f16_chain_test.c) -add_buckyball_ctest(redmule_f16_edges_test.c) -add_buckyball_ctest(redmule_f16_fused_test.c) diff --git a/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir b/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir deleted file mode 100644 index f079b7c70..000000000 --- a/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir +++ /dev/null @@ -1,6 +0,0 @@ -// The banks 1 (A), 2 (B) and 3 (C) are allocated and loaded by the caller. -func.func @redmule_chain() attributes {llvm.emit_c_interface} { - buckyball.custom "matmul_f16" <{sources = array, destination = 3 : i64}> {rows = 2 : i64, k = 8 : i64} - buckyball.custom "matmul_f16" <{sources = array, destination = 3 : i64, read_destination}> {rows = 2 : i64, k = 8 : i64} - return -} diff --git a/examples/components/balls/relu/stack/src/Lowering.cpp b/examples/components/balls/relu/stack/src/Lowering.cpp deleted file mode 100644 index 3cf6f7178..000000000 --- a/examples/components/balls/relu/stack/src/Lowering.cpp +++ /dev/null @@ -1,54 +0,0 @@ -//===- Lowering.cpp - ReluBall in lower-buckyball -------------------------===// -// -// relu_i32: max(x, 0) on the first `rows` rows of four i32 of a one-group -// bank, in place: the one source is the destination. -// -// The Ball updates whole groups of sixteen rows: it rounds `rows` up to a -// multiple of sixteen, which only touches rows past the valid data, and runs -// one segment as deep as the bank. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/BuckyballOps.h" -#include "Conversion/Buckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class ReluI32Lowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &rewriter) const override { - if (op.getKind() != "relu_i32") - return failure(); - int64_t rows = cast(op->getAttr("rows")).getInt(); - int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; - if (op.getSources().size() != 1 || - op.getSources()[0] != int64_t(op.getDestination())) - return op.emitOpError("relu_i32 updates its one source bank in place"); - if (rows <= 0 || rows > depth || depth % 16) - return op.emitOpError("relu_i32 rows exceed the bank"); - // Group 0 of the bank in the bank1 field; bit 63 marks the in-place - // update. rs2 is the segment stride. - uint64_t rs1 = mlir::buddy::bankFields(op.getSources(), op.getDestination(), - false, (rows + 15) / 16 * 16) | - (1ULL << 63); - mlir::buddy::emitInstruction( - rewriter, op.getLoc(), buckyball_target::getBuckyballFunct7("RELU"), - mlir::buddy::constantI64(rewriter, op.getLoc(), rs1), - mlir::buddy::constantI64(rewriter, op.getLoc(), depth)); - rewriter.eraseOp(op); - return success(); - } -}; - -void populate(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -[[maybe_unused]] const bool registered = - (mlir::buddy::addBallLowering(populate), true); -} // namespace diff --git a/examples/components/balls/smatmul/stack/src/Lowering.cpp b/examples/components/balls/smatmul/stack/src/Lowering.cpp deleted file mode 100644 index df19834d3..000000000 --- a/examples/components/balls/smatmul/stack/src/Lowering.cpp +++ /dev/null @@ -1,75 +0,0 @@ -//===- Lowering.cpp - SMatMulBall in lower-buckyball ----------------------===// -// -// matmul_i8 (A, B) -> C with read_destination: C = bias + A @ B for `rows` -// rows (one, or a multiple of tileRows up to a quarter of the bank depth) and -// sixteen columns. A and B have the same number of groups; group g holds -// reduction block g of `k` columns (a multiple of sixteen), so K = k * groups. -// In group g, B holds k rows of sixteen i8. A holds k / 16 rows of sixteen i8 -// for rows = 1; otherwise M tiles of tileRows rows, each tile the k / 16 -// slices of sixteen columns one after another: line -// ((r / tileRows) * (k / 16) + c / 16) * tileRows + r % tileRows. -// C starts with the bias, one row of sixteen i32 in its first four lines, and -// receives `rows` rows of sixteen i32 (four lines each). -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/BuckyballOps.h" -#include "Conversion/Buckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class MatmulI8Lowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &rewriter) const override { - if (op.getKind() != "matmul_i8") - return failure(); - if (!op.getReadDestination() || op.getSources().size() != 2) - return op.emitOpError("INT8 matmul reads A, B and the bias in C"); - Location loc = op.getLoc(); - int64_t rows = cast(op->getAttr("rows")).getInt(); - int64_t k = cast(op->getAttr("k")).getInt(); - int64_t c = op.getDestination(); - const auto &target = buckyball_target::getBuckyballTarget(); - int64_t tileRows = - buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); - if (target.bankWidthBits != 128 || rows <= 0 || - rows * 4 > target.bankDepth || (rows != 1 && rows % tileRows) || - k <= 0 || k > target.bankDepth || k % 16 || - rows * (k / 16) > target.bankDepth) - return op.emitOpError( - "INT8 matmul shape exceeds the bank or ISA geometry"); - if (op.getSources()[0] == op.getSources()[1] || op.getSources()[0] == c || - op.getSources()[1] == c) - return op.emitOpError("INT8 matmul requires distinct A, B and C banks"); - mlir::buddy::emitInstruction( - rewriter, loc, buckyball_target::getBuckyballFunct7("SMATMUL_BIAS"), - mlir::buddy::constantI64(rewriter, loc, - mlir::buddy::bankFields({c}, 0, false, 4)), - mlir::buddy::constantI64(rewriter, loc, 0)); - // One accumulation over every group: first and last in one instruction. - uint64_t rs2 = - mlir::buddy::field(rows, 0, 11) | mlir::buddy::field(16, 12, 23) | - mlir::buddy::field(1, 24, 24) | mlir::buddy::field(1, 25, 25); - mlir::buddy::emitInstruction( - rewriter, loc, buckyball_target::getBuckyballFunct7("SMATMUL_OS"), - mlir::buddy::constantI64( - rewriter, loc, - mlir::buddy::bankFields(op.getSources(), c, false, k)), - mlir::buddy::constantI64(rewriter, loc, rs2)); - rewriter.eraseOp(op); - return success(); - } -}; - -void populate(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -[[maybe_unused]] const bool registered = - (mlir::buddy::addBallLowering(populate), true); -} // namespace diff --git a/examples/components/balls/toint8/stack/src/Lowering.cpp b/examples/components/balls/toint8/stack/src/Lowering.cpp deleted file mode 100644 index 185946fe6..000000000 --- a/examples/components/balls/toint8/stack/src/Lowering.cpp +++ /dev/null @@ -1,84 +0,0 @@ -//===- Lowering.cpp - ToInt8Ball in lower-buckyball -----------------------===// -// -// fp32_to_int8: the source bank holds `rows` rows of four f32, a multiple of -// four; the destination receives rows / 4 rows of sixteen i8, -// clamp(round_half_even(x * multiplier), -128, 127). Argument 0 is the f32 -// multiplier. -// int32_to_int8: source 0 holds `rows` rows of sixteen i32 (four lines each), -// source 1 the sixteen f32 scales of the columns (four lines); the -// destination receives `rows` rows of sixteen i8, -// clamp(round_half_even(x * scale), -128, 127), with `relu` at least zero. -// Both kinds work on every group: group g is channel block g, and the banks -// have the same number of groups. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/BuckyballOps.h" -#include "Conversion/Buckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class Fp32ToInt8Lowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &rewriter) const override { - if (op.getKind() != "fp32_to_int8") - return failure(); - Location loc = op.getLoc(); - auto rows = cast(op->getAttr("rows")).getInt(); - uint64_t rs1 = mlir::buddy::bankFields(op.getSources(), op.getDestination(), - false, rows); - Value bits = arith::BitcastOp::create(rewriter, loc, rewriter.getI32Type(), - op.getArguments()[0]); - Value rs2 = - arith::ExtUIOp::create(rewriter, loc, rewriter.getI64Type(), bits); - mlir::buddy::emitInstruction( - rewriter, loc, buckyball_target::getBuckyballFunct7("QUANT_F32_TO_I8"), - mlir::buddy::constantI64(rewriter, loc, rs1), rs2); - rewriter.eraseOp(op); - return success(); - } -}; - -class Int32ToInt8Lowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &rewriter) const override { - if (op.getKind() != "int32_to_int8") - return failure(); - auto rows = cast(op->getAttr("rows")).getInt(); - bool relu = cast(op->getAttr("relu")).getValue(); - int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; - if (rows <= 0 || rows * 4 > depth) - return op.emitOpError("INT32-to-INT8 rows exceed the bank depth"); - // The output rows start at the base of the destination, packed; the - // seven-bit height takes 128 rows as two columns of 64. - int64_t width = rows > 127 ? 2 : 1; - uint64_t rs1 = mlir::buddy::bankFields(op.getSources(), op.getDestination(), - false, rows * 4); - uint64_t rs2 = uint64_t(relu) | (uint64_t(width) << 8) | - (uint64_t(rows / width) << 15) | (uint64_t(width) << 22); - mlir::buddy::emitInstruction( - rewriter, op.getLoc(), - buckyball_target::getBuckyballFunct7("QUANT_I32_TO_I8"), - mlir::buddy::constantI64(rewriter, op.getLoc(), rs1), - mlir::buddy::constantI64(rewriter, op.getLoc(), rs2)); - rewriter.eraseOp(op); - return success(); - } -}; - -void populate(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -[[maybe_unused]] const bool registered = - (mlir::buddy::addBallLowering(populate), true); -} // namespace diff --git a/examples/components/balls/trace/arch/README.md b/examples/components/balls/trace/arch/README.md deleted file mode 100644 index af1974e8e..000000000 --- a/examples/components/balls/trace/arch/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# TraceBall — debug/trace Ball - -## Overview - -TraceBall is a non-computing Ball that exposes runtime debug features through the Buckyball instruction channel. It has two main roles: - -1. **Cycle counter** — set/release multiple independent cycle counters via instructions to measure elapsed cycles over arbitrary code regions -2. **Bank backdoor** — inject data into SRAM banks or read bank data out via DPI-C - -All DPI-C hooks live inside TraceBall only and do not affect other modules. - ---- - -## Instruction encoding - -TraceBall uses two Core-owned `ballISA` mnemonics. The selected Core assigns -their `funct7` values at compile time. - -### Instruction 1: `bdb_counter` - -Cycle counter control. **Does not touch SRAM, needs no bank ports, completes in 1 cycle.** - -rs1 layout: -- rs1 = don’t care (unused; no need to set BB_RD0/BB_WR) - -rs2 layout (64-bit): -``` -rs2[3:0] = subcmd subcommand (0=START, 1=STOP, 2=READ) -rs2[7:4] = ctr_id counter id (0–15, up to 16 independent counters) -rs2[63:8] = payload -``` - -Subcommands: - -| subcmd | name | behavior | payload meaning | -|--------|------|----------|-----------------| -| 0 | `CTR_START` | Start counter `ctr_id`, record current cycle as start | payload = tag (user tag, echoed in trace) | -| 1 | `CTR_STOP` | Stop `ctr_id`, emit elapsed cycles to DPI-C trace, then free the counter | payload = ignored | -| 2 | `CTR_READ` | Read current value of `ctr_id` (does not stop), emit to DPI-C trace | payload = ignored | - -DPI-C trace format (written to bdb.log): -``` -[CTRACE] CTR_START ctr=0 tag=0xDEAD cycle=10042 -[CTRACE] CTR_STOP ctr=0 tag=0xDEAD elapsed=387 cycle=10429 -[CTRACE] CTR_READ ctr=0 current=200 cycle=10242 -``` - -### Instruction 2: `bdb_backdoor` - -SRAM backdoor read/write; **all parameters (bank_id, row, data) come from DPI-C**. **Requires bank ports (inBW=1, outBW=1).** - -rs1 layout: -``` -rs1[45] = BB_RD0 read: get address from DPI-C, read SRAM, send data to DPI-C -rs1[47] = BB_WR write: get address+data from DPI-C, write SRAM -rs1[63:48] = iter repeat count (0 = once, >0 = loop `iter` times) -``` - -rs2 layout: -- rs2 = don’t care (unused) - -Modes: - -| rs1 flag | behavior | DPI-C interaction | -|----------|----------|-------------------| -| BB_RD0 | read | RTL calls `dpi_backdoor_get_read_addr()` for (bank_id, row), reads SRAM, `dpi_backdoor_put_read_data()` for data | -| BB_WR | write | RTL calls `dpi_backdoor_get_write_req()` for (bank_id, row, data), writes SRAM | - -DPI-C trace format: -``` -[BANK-TRACE] BACKDOOR_READ bank=2 row=5 data=0x00010002000300040005000600070008 -[BANK-TRACE] BACKDOOR_WRITE bank=3 row=10 data=0xDEADBEEFDEADBEEFDEADBEEFDEADBEEF -``` - - -## Examples - -### Measure kernel cycles - -```c -// One matmul region -bdb_counter_start(0, 0xA001); // counter 0, tag=matmul -bb_vecmat16(A, B, C, 16); -bdb_counter_stop(0); // prints elapsed - -// Nested regions -bdb_counter_start(0, 0xB001); // outer: whole conv - bdb_counter_start(1, 0xB002); // inner: im2col - bb_im2col(...); - bdb_counter_stop(1); - - bdb_counter_start(2, 0xB003); // inner: matmul - bb_vecmat16(...); - bdb_counter_stop(2); -bdb_counter_stop(0); // outer end -``` - -bdb.log sample: -``` -[CTRACE] CTR_START ctr=0 tag=0xB001 cycle=0 -[CTRACE] CTR_START ctr=1 tag=0xB002 cycle=0 -[CTRACE] CTR_STOP ctr=1 tag=0xB002 elapsed=150 cycle=150 -[CTRACE] CTR_START ctr=2 tag=0xB003 cycle=0 -[CTRACE] CTR_STOP ctr=2 tag=0xB003 elapsed=300 cycle=300 -[CTRACE] CTR_STOP ctr=0 tag=0xB001 elapsed=456 cycle=456 -``` - -### Backdoor test data into SRAM - -TraceBall has a private bank that is not visible to normal configuration. - -```c -// No DMA: inject test data into bank 0 via DPI-C -// (C++ first injects into TraceBall’s internal bank via DPI-C) -bb_alloc(0, 1, 1); -bdb_backdoor_mvin(16); // inject 16 rows into private bank -bdb_backdoor_write(0, 16); // move 16 rows from private bank to bank 0 - -bb_transpose(0, 1, 16); -bdb_backdoor_peek(0, 15); // inspect last row - -bdb_backdoor_read(1, 16); // dump 16 rows from bank 1 to trace -``` - - - -All RTL changes stay inside TraceBall; do not modify external banks or shared infrastructure there. diff --git a/examples/components/balls/trace/emu/src/04_bdb_counter.rs b/examples/components/balls/trace/emu/src/04_bdb_counter.rs deleted file mode 100644 index 5ab37932e..000000000 --- a/examples/components/balls/trace/emu/src/04_bdb_counter.rs +++ /dev/null @@ -1,15 +0,0 @@ -//===- 04_bdb_counter.rs - BDB_COUNTER instruction -------------------------===// - -use super::instruction::{BallModel, ExecContext}; - -pub struct BdbCounter; - -impl BallModel for BdbCounter { - fn exec(_xs1: u64, _xs2: u64, _ctx: &mut ExecContext) -> u64 { - 0 - } - - fn latency(_xs1: u64, _xs2: u64) -> u64 { - 1 - } -} diff --git a/examples/components/balls/trace/stack/src/Lowering.cpp b/examples/components/balls/trace/stack/src/Lowering.cpp deleted file mode 100644 index 900fc5929..000000000 --- a/examples/components/balls/trace/stack/src/Lowering.cpp +++ /dev/null @@ -1,66 +0,0 @@ -//===- Lowering.cpp - TraceBall in lower-buckyball ------------------------===// -// -// The BDB counter touches no bank: its operations have no sources and -// destination 0, and rs1 is zero. `counter` (0 to 15) selects the counter. -// counter_start starts it with the i64 tag of argument 0; counter_read traces -// its current value; counter_stop traces the elapsed cycles and stops it. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/BuckyballOps.h" -#include "Conversion/Buckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class CounterLowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &rewriter) const override { - uint64_t subcommand; - if (op.getKind() == "counter_start") - subcommand = 0; - else if (op.getKind() == "counter_stop") - subcommand = 1; - else if (op.getKind() == "counter_read") - subcommand = 2; - else - return failure(); - Location loc = op.getLoc(); - int64_t counter = cast(op->getAttr("counter")).getInt(); - if (!op.getSources().empty() || op.getDestination() != 0 || - op.getReadDestination()) - return op.emitOpError("BDB counter touches no bank"); - if (counter < 0 || counter > 15 || - op.getArguments().size() != (subcommand == 0 ? 1 : 0)) - return op.emitOpError("BDB counter needs a counter in [0, 15] and a " - "tag only to start"); - Value rs2 = mlir::buddy::constantI64(rewriter, loc, - mlir::buddy::field(subcommand, 0, 3) | - mlir::buddy::field(counter, 4, 7)); - if (subcommand == 0) { - Value tag = - arith::ShLIOp::create(rewriter, loc, op.getArguments()[0], - mlir::buddy::constantI64(rewriter, loc, 8)); - rs2 = arith::OrIOp::create(rewriter, loc, rs2, tag); - } - mlir::buddy::emitInstruction( - rewriter, loc, buckyball_target::getBuckyballFunct7("BDB_COUNTER"), - mlir::buddy::constantI64(rewriter, loc, 0), rs2); - rewriter.eraseOp(op); - return success(); - } -}; - -void populate(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -[[maybe_unused]] const bool registered = - (mlir::buddy::addBallLowering(populate), true); -} // namespace diff --git a/examples/components/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir b/examples/components/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir deleted file mode 100644 index 479ff040c..000000000 --- a/examples/components/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir +++ /dev/null @@ -1,8 +0,0 @@ -func.func @main() -> i8 { - %zero8 = arith.constant 0 : i8 - %tag = arith.constant 48879 : i64 - buckyball.custom "counter_start" args(%tag : i64) <{sources = array, destination = 0 : i64}> {counter = 3 : i64} - buckyball.custom "counter_read" <{sources = array, destination = 0 : i64}> {counter = 3 : i64} - buckyball.custom "counter_stop" <{sources = array, destination = 0 : i64}> {counter = 3 : i64} - return %zero8 : i8 -} diff --git a/examples/components/balls/transpose/stack/src/Lowering.cpp b/examples/components/balls/transpose/stack/src/Lowering.cpp deleted file mode 100644 index 8fe1ee973..000000000 --- a/examples/components/balls/transpose/stack/src/Lowering.cpp +++ /dev/null @@ -1,55 +0,0 @@ -//===- Lowering.cpp - TransposeBall in lower-buckyball --------------------===// -// -// Both banks have cols / (bankWidthBits / 8) columns. The instruction packs -// the transposed bytes into the same bank geometry, with `rows` bank rows. -// -//===----------------------------------------------------------------------===// - -#include "Buckyball/BuckyballOps.h" -#include "Conversion/Buckyball.h" -#include "Target/BuckyballTargetRegistry.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -namespace { -class TransposeI8Lowering : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(CustomOp op, - PatternRewriter &rewriter) const override { - if (op.getKind() != "transpose_i8") - return failure(); - if (op.getSources().size() != 1 || - op.getSources()[0] == op.getDestination()) - return op.emitOpError( - "transpose_i8 requires distinct input and output banks"); - int64_t rows = cast(op->getAttr("rows")).getInt(); - int64_t cols = cast(op->getAttr("cols")).getInt(); - const auto &target = buckyball_target::getBuckyballTarget(); - int64_t lanes = target.bankWidthBits / 8; - if (lanes <= 0 || target.bankWidthBits % 8 || rows <= 0 || - rows >= (1LL << 33) || rows > target.bankDepth || cols <= 0 || - cols % lanes || cols / lanes > 31) - return op.emitOpError( - "transpose_i8 shape exceeds the bank or ISA geometry"); - mlir::buddy::emitInstruction( - rewriter, op.getLoc(), - buckyball_target::getBuckyballFunct7("TRANSPOSE"), - mlir::buddy::constantI64(rewriter, op.getLoc(), - mlir::buddy::bankFields(op.getSources(), - op.getDestination(), - false, rows)), - mlir::buddy::constantI64(rewriter, op.getLoc(), 8)); - rewriter.eraseOp(op); - return success(); - } -}; - -void populate(RewritePatternSet &patterns) { - patterns.add(patterns.getContext()); -} - -[[maybe_unused]] const bool registered = - (mlir::buddy::addBallLowering(populate), true); -} // namespace diff --git a/examples/components/balls/vector/workloads/isa/vecmat16.h b/examples/components/balls/vector/workloads/isa/vecmat16.h deleted file mode 100644 index 8c802d967..000000000 --- a/examples/components/balls/vector/workloads/isa/vecmat16.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BB_VECMAT16_H_ -#define _BB_VECMAT16_H_ - -#include -#include - -#define bb_vecmat16(op1_bank_id, op2_bank_id, wr_bank_id, iter, mode) \ - BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(op1_bank_id) | BB_BANK1(op2_bank_id) | \ - BB_BANK2(wr_bank_id) | BB_ITER(iter)), \ - (FIELD(mode, 0, 63)), BB_FUNC7(VECMAT16)) - -#endif // _BB_VECMAT16_H_ diff --git a/examples/components/balls/vector/workloads/mlir_tests/CMakeLists.txt b/examples/components/balls/vector/workloads/mlir_tests/CMakeLists.txt deleted file mode 100644 index 548af8da4..000000000 --- a/examples/components/balls/vector/workloads/mlir_tests/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") -file(STRINGS "${_params}" _num_define REGEX "^#define BANK_NUM ") -file(STRINGS "${_params}" _lines_define REGEX "^#define BANK_LINES ") -file(STRINGS "${_params}" _width_define REGEX "^#define BANK_WIDTH ") -string(REGEX REPLACE ".* " "" _num "${_num_define}") -string(REGEX REPLACE ".* " "" _lines "${_lines_define}") -string(REGEX REPLACE ".* " "" _width "${_width_define}") -if(NOT _width EQUAL 128 OR _lines LESS 16 OR _num LESS 6) - message(FATAL_ERROR - "VecBall MLIR tests require BANK_WIDTH=128, BANK_LINES>=16, and BANK_NUM>=6 " - "(got width=${_width}, lines=${_lines}, num=${_num})") -endif() - -add_subdirectory(ball) -add_subdirectory(bank) diff --git a/examples/components/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt b/examples/components/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt deleted file mode 100644 index c7d6655fa..000000000 --- a/examples/components/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) - set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) -endif() -set(BUCKYBALL_MLIR_TEST_PREFIX ball) -add_buckyball_mlir_test(vecmat16_atranspose_16x16 - TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/components/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16.mlir b/examples/components/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16.mlir deleted file mode 100644 index c966caf81..000000000 --- a/examples/components/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16.mlir +++ /dev/null @@ -1,57 +0,0 @@ -func.func private @check_result(memref<16x16xi32>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %z32 = arith.constant 0 : i32 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %dim = arith.constant 16 : index - %depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %mode = arith.constant 0 : i64 - %at = memref.alloc() alignment = 64 : memref<16x16xi8> - %b = memref.alloc() alignment = 64 : memref<16x16xi8> - %c = memref.alloc() alignment = 64 : memref<16x16xi32> - linalg.fill ins(%z32 : i32) outs(%c : memref<16x16xi32>) - scf.for %row = %z to %dim step %one { - scf.for %col = %z to %dim step %one { - %r3 = arith.muli %row, %three : index - %c5 = arith.muli %col, %five : index - %asum = arith.addi %r3, %c5 : index - %arem = arith.remui %asum, %seven : index - %aval = arith.subi %arem, %three : index - %atv = arith.index_cast %aval : index to i8 - %r2 = arith.muli %row, %two : index - %c3 = arith.muli %col, %three : index - %bsum = arith.addi %r2, %c3 : index - %brem = arith.remui %bsum, %five : index - %bval = arith.subi %brem, %two : index - %bv = arith.index_cast %bval : index to i8 - memref.store %atv, %at[%row, %col] : memref<16x16xi8> - memref.store %bv, %b[%row, %col] : memref<16x16xi8> - } - } - %ab = buckyball.bank_alloc - %bb = buckyball.bank_alloc - %cb = buckyball.bank_alloc - %al = buckyball.bank_mvin %at %ab %depth %stride - : memref<16x16xi8> i64 i64 i64 - %bl = buckyball.bank_mvin %b %bb %depth %stride - : memref<16x16xi8> i64 i64 i64 - %cl = buckyball.bank_mvin %c %cb %depth %stride - : memref<16x16xi32> i64 i64 i64 - buckyball.vecmat16 %al, %bl, %cl, %depth, %mode : i64 - %stored = buckyball.bank_mvout %c %cl %depth %stride - : memref<16x16xi32> i64 i64 i64 - func.call @check_result(%c) : (memref<16x16xi32>) -> () - buckyball.bank_release %al : i64 - buckyball.bank_release %bl : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %at : memref<16x16xi8> - memref.dealloc %b : memref<16x16xi8> - memref.dealloc %c : memref<16x16xi32> - return %z8 : i8 -} diff --git a/examples/components/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth.mlir.in b/examples/components/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth.mlir.in deleted file mode 100644 index d9f0c4e50..000000000 --- a/examples/components/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth.mlir.in +++ /dev/null @@ -1,62 +0,0 @@ -func.func private @check_result(memref<16x16xi32>) -> () -func.func @main() -> i8 { - %z8 = arith.constant 0 : i8 - %z32 = arith.constant 0 : i32 - %z = arith.constant 0 : index - %one = arith.constant 1 : index - %two = arith.constant 2 : index - %three = arith.constant 3 : index - %five = arith.constant 5 : index - %seven = arith.constant 7 : index - %dim = arith.constant 16 : index - %rows = arith.constant @BB_BANK_LINES@ : index - %depth = arith.constant @BB_BANK_LINES@ : i64 - %out_depth = arith.constant 16 : i64 - %stride = arith.constant 1 : i64 - %mode = arith.constant 0 : i64 - %at = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x16xi8> - %b = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x16xi8> - %c = memref.alloc() alignment = 64 : memref<16x16xi32> - linalg.fill ins(%z32 : i32) outs(%c : memref<16x16xi32>) - scf.for %k = %z to %rows step %one { - scf.for %lane = %z to %dim step %one { - %k3 = arith.muli %k, %three : index - %l5 = arith.muli %lane, %five : index - %asum = arith.addi %k3, %l5 : index - %arem = arith.remui %asum, %seven : index - %aval = arith.subi %arem, %three : index - %atv = arith.index_cast %aval : index to i8 - %k2 = arith.muli %k, %two : index - %l3 = arith.muli %lane, %three : index - %bsum = arith.addi %k2, %l3 : index - %brem = arith.remui %bsum, %five : index - %bval = arith.subi %brem, %two : index - %bv = arith.index_cast %bval : index to i8 - memref.store %atv, %at[%k, %lane] - : memref<@BB_BANK_LINES@x16xi8> - memref.store %bv, %b[%k, %lane] - : memref<@BB_BANK_LINES@x16xi8> - } - } - %ab = buckyball.bank_alloc - %bb = buckyball.bank_alloc - %cb = buckyball.bank_alloc - %al = buckyball.bank_mvin %at %ab %depth %stride - : memref<@BB_BANK_LINES@x16xi8> i64 i64 i64 - %bl = buckyball.bank_mvin %b %bb %depth %stride - : memref<@BB_BANK_LINES@x16xi8> i64 i64 i64 - %cl = buckyball.bank_mvin %c %cb %out_depth %stride - : memref<16x16xi32> i64 i64 i64 - %result = buckyball.bank_vecmat16 %al %bl %cl %depth %mode - : i64 i64 i64 i64 i64 - %stored = buckyball.bank_mvout %c %result %out_depth %stride - : memref<16x16xi32> i64 i64 i64 - func.call @check_result(%c) : (memref<16x16xi32>) -> () - buckyball.bank_release %al : i64 - buckyball.bank_release %bl : i64 - buckyball.bank_release %stored : i64 - memref.dealloc %at : memref<@BB_BANK_LINES@x16xi8> - memref.dealloc %b : memref<@BB_BANK_LINES@x16xi8> - memref.dealloc %c : memref<16x16xi32> - return %z8 : i8 -} diff --git a/examples/components/cores/xiangshan/arch/build.sc b/examples/components/cores/xiangshan/arch/build.sc deleted file mode 100644 index 2de8ecf46..000000000 --- a/examples/components/cores/xiangshan/arch/build.sc +++ /dev/null @@ -1,41 +0,0 @@ -import mill._ -import mill.scalalib._ -import $file.^.^.^.^.^.arch.src.main.scala.framework.{build => framework} - -val componentRoot: os.Path = millSourcePath / os.up -val vendorRoot: os.Path = componentRoot / "thirdparty" / "xiangshan" -val vendorHead: os.Path = os.Path(os.proc("git", "rev-parse", "--git-path", "HEAD").call(cwd = vendorRoot).out.text().trim, vendorRoot) - -def nativeInputs = T.sources { - val paths = Seq( - "src", "rocket-chip/src", "rocket-chip/macros/src", "rocket-chip/cde/cde/src", - "rocket-chip/hardfloat/hardfloat/src", "difftest/src", "utility/src", "huancun/src", - "fudian/src", "yunsuan/src", "coupledL2/src", "openLLC/src", "openLLC/openNCB/src", - "ChiselAIA/src", "ChiselIOPMP/src", "build.sc", ".mill-version", - "rocket-chip/common.sc", "rocket-chip/cde/common.sc", "rocket-chip/hardfloat/common.sc", - "huancun/common.sc", "coupledL2/common.sc", "openLLC/common.sc" - ) - paths.map(path => PathRef(vendorRoot / os.RelPath(path))) :+ PathRef(vendorHead) -} - -def nativeRtl = T { - nativeInputs() - val output = T.dest / "xiangshan" - os.proc("mill", "--no-server", "native.runMain", "buckyball.Emit", - "--target-dir", output, "--target", "systemverilog").call(cwd = vendorRoot) - os.move(output / "extern_modules.sv", output / "XiangShanExtern.sv") - os.write.over(output / "filelist.f", os.read(output / "filelist.f").replace("extern_modules.sv", "XiangShanExtern.sv")) - PathRef(T.dest) -} - -trait ScalaBuild extends SbtModule { - override def scalaVersion = "2.13.16" - override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") - override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") - override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") -} -object library extends ScalaBuild { - override def moduleDeps = Seq(framework.library) - override def sources = T.sources(componentRoot / "arch", componentRoot / "configs") - override def resources = T.sources(Seq(nativeRtl())) -} diff --git a/examples/designs/goban/arch/build.sc b/examples/designs/goban/arch/build.sc index ca8f7e2b1..170086f40 100644 --- a/examples/designs/goban/arch/build.sc +++ b/examples/designs/goban/arch/build.sc @@ -2,7 +2,7 @@ import mill._ import mill.scalalib._ import $file.^.^.^.^.arch.src.main.scala.framework.device.cbus.{build => cbus_build} import $file.^.^.^.{build => examples} -import $file.^.^.^.components.cores.rocket.arch.{build => rocket} +import $file.^.^.^.prefab.cores.rocket.arch.{build => rocket} val designRoot: os.Path = millSourcePath / os.up diff --git a/examples/designs/goban/configs/device/core/default.toml b/examples/designs/goban/configs/device/core/default.toml index 46ea1c98b..3494faaff 100644 --- a/examples/designs/goban/configs/device/core/default.toml +++ b/examples/designs/goban/configs/device/core/default.toml @@ -26,7 +26,7 @@ entries = 16384 inputChannels = 52 defaultGroupCount = 1 virtualBankCount = 64 -memCycles = 10 +latency = 10 bankEntries = 1024 bankWidth = 128 @@ -39,22 +39,22 @@ dataBits = 128 id = 0 name = "matmul0" component = "matmul" -include = "../../../../../components/threads/matmul/configs/default.toml" +include = "../../../../../prefab/threads/matmul/configs/default.toml" [[threads]] id = 1 name = "matmul1" component = "matmul" -include = "../../../../../components/threads/matmul/configs/default.toml" +include = "../../../../../prefab/threads/matmul/configs/default.toml" [[threads]] id = 2 name = "matmul2" component = "matmul" -include = "../../../../../components/threads/matmul/configs/default.toml" +include = "../../../../../prefab/threads/matmul/configs/default.toml" [[threads]] id = 3 name = "matmul3" component = "matmul" -include = "../../../../../components/threads/matmul/configs/default.toml" +include = "../../../../../prefab/threads/matmul/configs/default.toml" diff --git a/examples/designs/goban/configs/device/core/memory/access/goban.toml b/examples/designs/goban/configs/device/core/memory/access/goban.toml index bb87f76a4..2e90444bc 100644 --- a/examples/designs/goban/configs/device/core/memory/access/goban.toml +++ b/examples/designs/goban/configs/device/core/memory/access/goban.toml @@ -1,5 +1,5 @@ translationEntries = 16 -accessTagBits = 6 +tagBits = 6 resultLineBits = 0 [[regions]] base = 0x80000000 diff --git a/examples/designs/goban/configs/device/core/memory/dcache/shared.toml b/examples/designs/goban/configs/device/core/memory/dcache/shared.toml index 0f17666dd..ba8310878 100644 --- a/examples/designs/goban/configs/device/core/memory/dcache/shared.toml +++ b/examples/designs/goban/configs/device/core/memory/dcache/shared.toml @@ -1,5 +1,5 @@ nodeId = 2 -cacheLines = 64 +lines = 64 homeId = 32 homeCount = 1 banks = 2 diff --git a/examples/designs/goban/configs/device/core/memory/dma-cache/shared.toml b/examples/designs/goban/configs/device/core/memory/dma-cache/shared.toml index 9dd394051..9a9a702c6 100644 --- a/examples/designs/goban/configs/device/core/memory/dma-cache/shared.toml +++ b/examples/designs/goban/configs/device/core/memory/dma-cache/shared.toml @@ -1,5 +1,5 @@ nodeId = 3 -cacheLines = 64 +lines = 64 homeId = 32 homeCount = 1 banks = 2 diff --git a/examples/designs/goban/configs/device/core/memory/home/goban.toml b/examples/designs/goban/configs/device/core/memory/home/goban.toml index 0e966ccbe..329d50021 100644 --- a/examples/designs/goban/configs/device/core/memory/home/goban.toml +++ b/examples/designs/goban/configs/device/core/memory/home/goban.toml @@ -1,6 +1,6 @@ agents = 3 mshrEntries = 8 -homeId = 32 +nodeId = 32 requesters = "default" [cache] diff --git a/examples/designs/goban/configs/device/core/memory/icache/shared.toml b/examples/designs/goban/configs/device/core/memory/icache/shared.toml index 873c62547..0796abb45 100644 --- a/examples/designs/goban/configs/device/core/memory/icache/shared.toml +++ b/examples/designs/goban/configs/device/core/memory/icache/shared.toml @@ -1,5 +1,5 @@ nodeId = 1 -cacheLines = 64 +lines = 64 homeId = 32 homeCount = 1 banks = 2 diff --git a/examples/designs/goban/configs/sims/default.toml b/examples/designs/goban/configs/sims/default.toml index 37e108378..38d332fa4 100644 --- a/examples/designs/goban/configs/sims/default.toml +++ b/examples/designs/goban/configs/sims/default.toml @@ -10,5 +10,8 @@ stack_name = "goban" regression = "sims/device/regression" kernel = "sims/device/workloads/kernel" +[targets.device.configs] +GobanVerilatorConfig = "verilator.toml" + [verification] registry = "../../sims/ip/uvm/registry.toml" diff --git a/examples/designs/goban/configs/sims/verilator.toml b/examples/designs/goban/configs/sims/verilator.toml new file mode 100644 index 000000000..0a960c260 --- /dev/null +++ b/examples/designs/goban/configs/sims/verilator.toml @@ -0,0 +1 @@ +# GobanVerilatorConfig: the goban design as configured, with no overrides. diff --git a/examples/designs/goban/sims/device/emu/build.rs b/examples/designs/goban/sims/device/emu/build.rs index d634d5dba..beb62e98c 100644 --- a/examples/designs/goban/sims/device/emu/build.rs +++ b/examples/designs/goban/sims/device/emu/build.rs @@ -1,7 +1,4 @@ -use std::{env, path::PathBuf}; - fn main() { - let entry = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - bebop_sim_build::balls(&entry.join("../../../output/generated/design.pb")); + bebop_sim_build::balls(); bebop_sim_build::runner_links(); } diff --git a/examples/designs/goban/sims/device/emu/src/lib.rs b/examples/designs/goban/sims/device/emu/src/lib.rs index ec91dc757..a24b7a8c9 100644 --- a/examples/designs/goban/sims/device/emu/src/lib.rs +++ b/examples/designs/goban/sims/device/emu/src/lib.rs @@ -6,6 +6,6 @@ extern crate bebop_design_goban; include!(concat!(env!("OUT_DIR"), "/balls.rs")); pub const SIMULATION: bebop_bemu::Simulation = bebop_bemu::Simulation { - design: include_bytes!("../../../../output/generated/design.pb"), + design: include_bytes!(concat!(env!("OUT_DIR"), "/design.pb")), instantiate: bebop_bemu::device::cbus::instantiate, }; diff --git a/examples/designs/goban/sims/device/regression/bemu/workloads-linux.toml b/examples/designs/goban/sims/device/regression/bemu/workloads-linux.toml index 07563aaaf..41a9cd13b 100644 --- a/examples/designs/goban/sims/device/regression/bemu/workloads-linux.toml +++ b/examples/designs/goban/sims/device/regression/bemu/workloads-linux.toml @@ -1,4 +1,4 @@ [workloads] tests = [ - "fw_payload-h1-goban-linux.elf", + "fw_payload-linux.elf", ] diff --git a/examples/designs/goban/sims/device/regression/p2e/workloads-linux.toml b/examples/designs/goban/sims/device/regression/p2e/workloads-linux.toml index 9a02148b5..01747a69b 100644 --- a/examples/designs/goban/sims/device/regression/p2e/workloads-linux.toml +++ b/examples/designs/goban/sims/device/regression/p2e/workloads-linux.toml @@ -1,4 +1,4 @@ [workloads] tests = [ - "fw_payload-h1-goban-linux.hex", + "fw_payload-linux.hex", ] diff --git a/examples/designs/goban/sims/device/stack/CMakeLists.txt b/examples/designs/goban/sims/device/stack/CMakeLists.txt index 18f9e5345..b91640c91 100644 --- a/examples/designs/goban/sims/device/stack/CMakeLists.txt +++ b/examples/designs/goban/sims/device/stack/CMakeLists.txt @@ -2,7 +2,7 @@ # The software stack of the goban device simulation: every level the device # needs, built bottom up. # -# bbdev stack --build "--design goban" +# bbdev stack --build "--design goban --device " # # configures this with DESIGN_CMAKE, generated from design.pb. Each level is a # CMake build of its own: cores and threads build buddy-opt plugins for the diff --git a/examples/designs/goban/sims/device/stack/core/goban/CMakeLists.txt b/examples/designs/goban/sims/device/stack/core/goban/CMakeLists.txt deleted file mode 100644 index 1eab481d1..000000000 --- a/examples/designs/goban/sims/device/stack/core/goban/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -cmake_minimum_required(VERSION 3.20) -project(goban-core-stack LANGUAGES C CXX) -include("${DESIGN_CMAKE}") - -add_subdirectory("${DESIGN_REPO}/examples/designs/goban/stack/engine" engine) -add_subdirectory("${DESIGN_REPO}/examples/designs/goban/stack/midend" midend) diff --git a/examples/designs/goban/sims/device/workloads/CMakeLists.txt b/examples/designs/goban/sims/device/workloads/CMakeLists.txt index 3d1f085d9..1deeb66ff 100644 --- a/examples/designs/goban/sims/device/workloads/CMakeLists.txt +++ b/examples/designs/goban/sims/device/workloads/CMakeLists.txt @@ -27,19 +27,16 @@ foreach(BUCKYBALL_COMPONENT_WORKLOAD_TARGET IN LISTS DESIGN_CPU_TARGETS) if( COMPONENT_WORKLOADS STREQUAL - "${BUCKYBALL_REPO_ROOT}/examples/designs/${BUCKYBALL_WORKLOAD_DESIGN}/workloads" + "${BUCKYBALL_REPO_ROOT}/examples/designs/${DESIGN_NAME}/workloads" ) continue() endif() - set(BUCKYBALL_COMPONENT_WORKLOAD_PROFILE - "${DESIGN_TARGET_ID_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" - ) set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}") set(BUCKYBALL_CTEST_GROUP_TARGET component-ctests-build) if(IS_DIRECTORY "${COMPONENT_WORKLOADS}/ctests") add_subdirectory( "${COMPONENT_WORKLOADS}/ctests" - "${CMAKE_CURRENT_BINARY_DIR}/components/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/prefab/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" ) endif() endforeach() @@ -58,9 +55,9 @@ add_subdirectory( ) foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) set(BUCKYBALL_MLIR_ACTIVE_TARGET "${BUCKYBALL_TARGET}") - foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_TARGET_BALLS_${BUCKYBALL_TARGET}) + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_BALLDOMAIN_BALLS) set(BALL_MLIR_TESTS - "${BUCKYBALL_REPO_ROOT}/examples/components/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" ) if(IS_DIRECTORY "${BALL_MLIR_TESTS}") add_subdirectory( diff --git a/examples/designs/goban/sims/device/stack/runtime/CMakeLists.txt b/examples/designs/goban/stack/device/CMakeLists.txt similarity index 91% rename from examples/designs/goban/sims/device/stack/runtime/CMakeLists.txt rename to examples/designs/goban/stack/device/CMakeLists.txt index 4408bd386..cf3a4abb5 100644 --- a/examples/designs/goban/sims/device/stack/runtime/CMakeLists.txt +++ b/examples/designs/goban/stack/device/CMakeLists.txt @@ -2,7 +2,7 @@ # The goban device: its cores, their hardware threads and the runtime that # schedules jobs onto the cores. # -# bbdev stack --build "--design goban" +# bbdev stack --build "--design goban --device " # # builds every thread and core stack of the device first, then configures this # with DESIGN_CMAKE, generated from design.pb, and the RISC-V Linux toolchain. @@ -24,12 +24,12 @@ add_library(bindings INTERFACE) foreach(CORE IN LISTS DESIGN_CORES) target_link_options(bindings INTERFACE "${DESIGN_CORE_BUILD_${CORE}}/execute.o") endforeach() -foreach(THREAD IN LISTS DESIGN_THREADS) +foreach(THREAD IN LISTS DESIGN_THREAD_STACKS) target_link_options(bindings INTERFACE "${DESIGN_THREAD_BUILD_${THREAD}}/loader.o") endforeach() # DMA helpers are sized by the target with the fewest bank lines. list(GET DESIGN_RUNTIME_TARGETS 0 PARAMS_TARGET) -set(RUNTIME_PARAMS "${DESIGN_ISA_DIR}/${PARAMS_TARGET}") +set(RUNTIME_DEFS ${DESIGN_TARGET_DEFS_${PARAMS_TARGET}}) add_subdirectory("${DESIGN_REPO}/stack/engine" engine) add_subdirectory("${DESIGN_REPO}/stack/runtime" runtime) diff --git a/examples/designs/goban/stack/device/core/CMakeLists.txt b/examples/designs/goban/stack/device/core/CMakeLists.txt new file mode 100644 index 000000000..0a1dc949c --- /dev/null +++ b/examples/designs/goban/stack/device/core/CMakeLists.txt @@ -0,0 +1,32 @@ +# ===----------------------------------------------------------------------=== +# Goban's own core: a CPU that runs the matmul hardware thread. +# +# Built by the stack top of each simulation entry after the core's threads, +# configured with DESIGN_CMAKE and CORE. Products: +# execute.o how the core runs jobs (through the engine) +# libengine-compiler.so buddy-opt pass plugin of the framework engine +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(goban-core-stack LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +set(ENGINE "${DESIGN_REPO}/stack/engine") +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# How the core runs jobs; instruction streams use the framework engine. +add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/execute.o" + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" + "-I${ENGINE}/include" -c "${RUNTIME}/execute/engine.c" -o "${CMAKE_BINARY_DIR}/execute.o" + DEPENDS "${RUNTIME}/execute/engine.c" "${RUNTIME}/scheduler/runtime.h" + "${ENGINE}/include/engine.h" + VERBATIM) +add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") + +# The engine's compiler plugin, for the threads this core runs. +add_subdirectory("${ENGINE}/compiler" engine-compiler) +add_library(engine-compiler-plugin MODULE $) +set_target_properties(engine-compiler-plugin PROPERTIES OUTPUT_NAME engine-compiler + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") diff --git a/examples/designs/goban/stack/engine/CMakeLists.txt b/examples/designs/goban/stack/engine/CMakeLists.txt deleted file mode 100644 index 21a7dac9a..000000000 --- a/examples/designs/goban/stack/engine/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Core execution binding; instruction streams use the framework engine. -set(RUNTIME "${DESIGN_REPO}/stack/runtime") -set(ENGINE "${DESIGN_REPO}/stack/engine") -find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) - -add_custom_command( - OUTPUT "${CMAKE_BINARY_DIR}/execute.o" - COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 - "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" - "-I${ENGINE}/include" -c "${RUNTIME}/execute/engine.c" -o "${CMAKE_BINARY_DIR}/execute.o" - DEPENDS "${RUNTIME}/execute/engine.c" "${RUNTIME}/scheduler/runtime.h" - "${ENGINE}/include/engine.h" - VERBATIM) -add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") diff --git a/examples/designs/goban/stack/midend/CMakeLists.txt b/examples/designs/goban/stack/midend/CMakeLists.txt deleted file mode 100644 index f8202e654..000000000 --- a/examples/designs/goban/stack/midend/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Host compiler plugin for this design's core. -set(ENGINE "${DESIGN_REPO}/stack/engine") -set(TARGET ${DESIGN_CORE_TARGET_${CORE}}) - -add_subdirectory("${ENGINE}/compiler" engine-compiler) -add_library(engine-compiler-plugin MODULE $) -set_target_properties(engine-compiler-plugin PROPERTIES OUTPUT_NAME engine-compiler - LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") diff --git a/examples/designs/goban/workloads/device/core/engine/CMakeLists.txt b/examples/designs/goban/workloads/device/core/engine/CMakeLists.txt index 6d362d881..8613c8300 100644 --- a/examples/designs/goban/workloads/device/core/engine/CMakeLists.txt +++ b/examples/designs/goban/workloads/device/core/engine/CMakeLists.txt @@ -1,9 +1,9 @@ set(ENGINE_DIR ${BUCKYBALL_REPO_ROOT}/stack/engine) set(MATMUL_TASK_DIR - ${BUCKYBALL_REPO_ROOT}/examples/components/threads/matmul/stack + ${BUCKYBALL_REPO_ROOT}/examples/prefab/threads/matmul/stack ) set(MATMUL_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-matmul-ctest-design-matmul_threads" + "${DESIGN_NAME}-matmul-ctest-design-matmul_threads" ) set(MATMUL_ELF ${MATMUL_NAME}-baremetal) add_custom_command( @@ -11,7 +11,7 @@ add_custom_command( COMMAND ${ELF_CC} -O2 -march=rv64gc -mcmodel=medany -msmall-data-limit=0 -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${BBSIM_LD} - -I${BUCKYBALL_ISA_DIR}/control -I${BUCKYBALL_ISA_DIR}/design + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${ENGINE_DIR}/include -I${MATMUL_TASK_DIR} -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services -I${BAREMETAL_DIR} @@ -28,10 +28,10 @@ add_custom_command( engine ${ENGINE_DIR}/include/thread.h ${ENGINE_DIR}/include/engine.h - ${BUCKYBALL_ISA_DIR}/control/params.h ${BBSIM_LD} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMAND_EXPAND_LISTS + VERBATIM ) add_custom_target( ${MATMUL_NAME}-ctest-build diff --git a/examples/designs/goban/workloads/device/core/engine/matmul_threads.c b/examples/designs/goban/workloads/device/core/engine/matmul_threads.c index 715d4c067..148586ee8 100644 --- a/examples/designs/goban/workloads/device/core/engine/matmul_threads.c +++ b/examples/designs/goban/workloads/device/core/engine/matmul_threads.c @@ -1,5 +1,5 @@ +#include #include -#include #include #include #include diff --git a/examples/designs/goban/workloads/device/core/engine/mvover_rows.c b/examples/designs/goban/workloads/device/core/engine/mvover_rows.c index 0953ab0f6..8be6c3185 100644 --- a/examples/designs/goban/workloads/device/core/engine/mvover_rows.c +++ b/examples/designs/goban/workloads/device/core/engine/mvover_rows.c @@ -1,17 +1,16 @@ #include -#include #include -#include #include #define ROWS 8 -#define ROW_BYTES (BANK_WIDTH / 8) -#if BANK_LINES < ROWS +#define ROW_BYTES (THREAD_MEM_BANK_WIDTH / 8) +#if THREAD_MEM_BANK_ENTRIES < ROWS #error "mvover_rows requires eight rows per bank" #endif static uint8_t input[ROWS * ROW_BYTES] __attribute__((aligned(64))); static uint8_t output[ROWS * ROW_BYTES] __attribute__((aligned(64))); -#if SHARED_BANK_NUM < 1 || BB_SHARED_BANK_BASE >= VIRTUAL_BANK_NUM +#if !CORE_SHARED_MEM_ENABLE || \ + THREAD_FRONTEND_SHARED_VBANK_BASE >= CORE_VIRTUAL_BANK_COUNT #error "mvover_rows requires a shared bank for issuer/owner trace coverage" #endif static uint8_t shared_output[ROWS * ROW_BYTES] __attribute__((aligned(64))); @@ -22,15 +21,16 @@ int main(void) { for (unsigned i = 0; i < sizeof(input); ++i) input[i] = (uint8_t)(i * 17 + 3); bb_mem_alloc(1, 1, 1); - bb_mem_alloc(BB_SHARED_BANK_BASE, 1, 1); + bb_mem_alloc(THREAD_FRONTEND_SHARED_VBANK_BASE, 1, 1); bb_mvin((uintptr_t)input, 1, ROWS, 1); // Hart 1 issues the write; the shared bank pool is owned by hart 0. - bb_mvin((uintptr_t)input, BB_SHARED_BANK_BASE, ROWS, 1); - bb_mvout((uintptr_t)shared_output, BB_SHARED_BANK_BASE, ROWS, 1); + bb_mvin((uintptr_t)input, THREAD_FRONTEND_SHARED_VBANK_BASE, ROWS, 1); + bb_mvout((uintptr_t)shared_output, THREAD_FRONTEND_SHARED_VBANK_BASE, ROWS, + 1); for (unsigned i = 0; i < sizeof(input); ++i) if (shared_output[i] != input[i]) return 1; - bb_mem_release(BB_SHARED_BANK_BASE); + bb_mem_release(THREAD_FRONTEND_SHARED_VBANK_BASE); __atomic_store_n(&source_ready, 1, __ATOMIC_RELEASE); } else if (core == 2) { bb_mem_alloc(8, 1, 1); diff --git a/examples/designs/multi-rocket/arch/build.sc b/examples/designs/multi-rocket/arch/build.sc index ca8f7e2b1..170086f40 100644 --- a/examples/designs/multi-rocket/arch/build.sc +++ b/examples/designs/multi-rocket/arch/build.sc @@ -2,7 +2,7 @@ import mill._ import mill.scalalib._ import $file.^.^.^.^.arch.src.main.scala.framework.device.cbus.{build => cbus_build} import $file.^.^.^.{build => examples} -import $file.^.^.^.components.cores.rocket.arch.{build => rocket} +import $file.^.^.^.prefab.cores.rocket.arch.{build => rocket} val designRoot: os.Path = millSourcePath / os.up diff --git a/examples/designs/multi-rocket/configs/device/core/default.toml b/examples/designs/multi-rocket/configs/device/core/default.toml index 8f3cda39a..af52db82d 100644 --- a/examples/designs/multi-rocket/configs/device/core/default.toml +++ b/examples/designs/multi-rocket/configs/device/core/default.toml @@ -25,6 +25,6 @@ entries = 16384 inputChannels = 1 defaultGroupCount = 1 virtualBankCount = 16 -memCycles = 10 +latency = 10 bankEntries = 1024 bankWidth = 128 diff --git a/examples/designs/multi-rocket/configs/device/core/memory/access/core.toml b/examples/designs/multi-rocket/configs/device/core/memory/access/core.toml index bb87f76a4..2e90444bc 100644 --- a/examples/designs/multi-rocket/configs/device/core/memory/access/core.toml +++ b/examples/designs/multi-rocket/configs/device/core/memory/access/core.toml @@ -1,5 +1,5 @@ translationEntries = 16 -accessTagBits = 6 +tagBits = 6 resultLineBits = 0 [[regions]] base = 0x80000000 diff --git a/examples/designs/multi-rocket/configs/device/core/memory/dcache/shared.toml b/examples/designs/multi-rocket/configs/device/core/memory/dcache/shared.toml index 0396137af..91cce2568 100644 --- a/examples/designs/multi-rocket/configs/device/core/memory/dcache/shared.toml +++ b/examples/designs/multi-rocket/configs/device/core/memory/dcache/shared.toml @@ -1,4 +1,4 @@ -cacheLines = 64 +lines = 64 homeId = 96 homeCount = 1 banks = 2 diff --git a/examples/designs/multi-rocket/configs/device/core/memory/home/core.toml b/examples/designs/multi-rocket/configs/device/core/memory/home/core.toml index bfc26d87b..8a021b8fb 100644 --- a/examples/designs/multi-rocket/configs/device/core/memory/home/core.toml +++ b/examples/designs/multi-rocket/configs/device/core/memory/home/core.toml @@ -1,6 +1,6 @@ agents = 64 mshrEntries = 8 -homeId = 96 +nodeId = 96 requesters = "default" [cache] diff --git a/examples/designs/multi-rocket/configs/device/core/memory/icache/shared.toml b/examples/designs/multi-rocket/configs/device/core/memory/icache/shared.toml index 0396137af..91cce2568 100644 --- a/examples/designs/multi-rocket/configs/device/core/memory/icache/shared.toml +++ b/examples/designs/multi-rocket/configs/device/core/memory/icache/shared.toml @@ -1,4 +1,4 @@ -cacheLines = 64 +lines = 64 homeId = 96 homeCount = 1 banks = 2 diff --git a/examples/designs/multi-rocket/configs/sims/default.toml b/examples/designs/multi-rocket/configs/sims/default.toml index 0fc0560fc..0cf750aff 100644 --- a/examples/designs/multi-rocket/configs/sims/default.toml +++ b/examples/designs/multi-rocket/configs/sims/default.toml @@ -10,5 +10,8 @@ stack_name = "multi-rocket" regression = "sims/device/regression" kernel = "sims/device/workloads/kernel" +[targets.device.configs] +MultiRocketVerilatorConfig = "verilator.toml" + [verification] registry = "../../sims/ip/uvm/registry.toml" diff --git a/examples/designs/multi-rocket/configs/sims/verilator.toml b/examples/designs/multi-rocket/configs/sims/verilator.toml new file mode 100644 index 000000000..b1c32c64e --- /dev/null +++ b/examples/designs/multi-rocket/configs/sims/verilator.toml @@ -0,0 +1 @@ +# MultiRocketVerilatorConfig: the multi-rocket design as configured, with no overrides. diff --git a/examples/designs/multi-rocket/sims/device/emu/Cargo.toml b/examples/designs/multi-rocket/sims/device/emu/Cargo.toml index f6c743f8f..38491051e 100644 --- a/examples/designs/multi-rocket/sims/device/emu/Cargo.toml +++ b/examples/designs/multi-rocket/sims/device/emu/Cargo.toml @@ -66,7 +66,7 @@ p2e = ["bebop/p2e"] bebop = { path = "../../../../../../bebop" } bebop-verilator = { path = "../../../../../../bebop/src/nodes/verilator", optional = true } bebop-bemu = { path = "../../../../../../bebop/src/nodes/bemu" } -bebop-core-rocket = { path = "../../../../../components/cores/rocket/emu" } +bebop-core-rocket = { path = "../../../../../prefab/cores/rocket/emu" } clap = { version = "4", features = ["derive"] } [build-dependencies] diff --git a/examples/designs/multi-rocket/sims/device/emu/build.rs b/examples/designs/multi-rocket/sims/device/emu/build.rs index d634d5dba..beb62e98c 100644 --- a/examples/designs/multi-rocket/sims/device/emu/build.rs +++ b/examples/designs/multi-rocket/sims/device/emu/build.rs @@ -1,7 +1,4 @@ -use std::{env, path::PathBuf}; - fn main() { - let entry = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - bebop_sim_build::balls(&entry.join("../../../output/generated/design.pb")); + bebop_sim_build::balls(); bebop_sim_build::runner_links(); } diff --git a/examples/designs/multi-rocket/sims/device/emu/src/lib.rs b/examples/designs/multi-rocket/sims/device/emu/src/lib.rs index c4c11131f..b43b00662 100644 --- a/examples/designs/multi-rocket/sims/device/emu/src/lib.rs +++ b/examples/designs/multi-rocket/sims/device/emu/src/lib.rs @@ -6,6 +6,6 @@ extern crate bebop_core_rocket; include!(concat!(env!("OUT_DIR"), "/balls.rs")); pub const SIMULATION: bebop_bemu::Simulation = bebop_bemu::Simulation { - design: include_bytes!("../../../../output/generated/design.pb"), + design: include_bytes!(concat!(env!("OUT_DIR"), "/design.pb")), instantiate: bebop_bemu::device::cbus::instantiate, }; diff --git a/examples/designs/multi-rocket/sims/device/regression/bemu/workloads-linux.toml b/examples/designs/multi-rocket/sims/device/regression/bemu/workloads-linux.toml index 42419fdd0..41a9cd13b 100644 --- a/examples/designs/multi-rocket/sims/device/regression/bemu/workloads-linux.toml +++ b/examples/designs/multi-rocket/sims/device/regression/bemu/workloads-linux.toml @@ -1,4 +1,4 @@ [workloads] tests = [ - "fw_payload-h32-multi-rocket-linux.elf", + "fw_payload-linux.elf", ] diff --git a/examples/designs/multi-rocket/sims/device/regression/p2e/workloads-linux.toml b/examples/designs/multi-rocket/sims/device/regression/p2e/workloads-linux.toml index 4da6227ba..01747a69b 100644 --- a/examples/designs/multi-rocket/sims/device/regression/p2e/workloads-linux.toml +++ b/examples/designs/multi-rocket/sims/device/regression/p2e/workloads-linux.toml @@ -1,4 +1,4 @@ [workloads] tests = [ - "fw_payload-h32-multi-rocket-linux.hex", + "fw_payload-linux.hex", ] diff --git a/examples/designs/multi-rocket/sims/device/stack/CMakeLists.txt b/examples/designs/multi-rocket/sims/device/stack/CMakeLists.txt index c73416325..da9385d20 100644 --- a/examples/designs/multi-rocket/sims/device/stack/CMakeLists.txt +++ b/examples/designs/multi-rocket/sims/device/stack/CMakeLists.txt @@ -2,7 +2,7 @@ # The software stack of the multi-rocket device simulation: every level the device # needs, built bottom up. # -# bbdev stack --build "--design multi-rocket" +# bbdev stack --build "--design multi-rocket --device " # # configures this with DESIGN_CMAKE, generated from design.pb. Each level is a # CMake build of its own: cores and threads build buddy-opt plugins for the diff --git a/examples/designs/multi-rocket/sims/device/workloads/CMakeLists.txt b/examples/designs/multi-rocket/sims/device/workloads/CMakeLists.txt index 65e317533..d606b9597 100644 --- a/examples/designs/multi-rocket/sims/device/workloads/CMakeLists.txt +++ b/examples/designs/multi-rocket/sims/device/workloads/CMakeLists.txt @@ -27,19 +27,16 @@ foreach(BUCKYBALL_COMPONENT_WORKLOAD_TARGET IN LISTS DESIGN_CPU_TARGETS) if( COMPONENT_WORKLOADS STREQUAL - "${BUCKYBALL_REPO_ROOT}/examples/designs/${BUCKYBALL_WORKLOAD_DESIGN}/workloads" + "${BUCKYBALL_REPO_ROOT}/examples/designs/${DESIGN_NAME}/workloads" ) continue() endif() - set(BUCKYBALL_COMPONENT_WORKLOAD_PROFILE - "${DESIGN_TARGET_ID_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" - ) set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}") set(BUCKYBALL_CTEST_GROUP_TARGET component-ctests-build) if(IS_DIRECTORY "${COMPONENT_WORKLOADS}/ctests") add_subdirectory( "${COMPONENT_WORKLOADS}/ctests" - "${CMAKE_CURRENT_BINARY_DIR}/components/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/prefab/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" ) endif() endforeach() @@ -58,9 +55,9 @@ add_subdirectory( ) foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) set(BUCKYBALL_MLIR_ACTIVE_TARGET "${BUCKYBALL_TARGET}") - foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_TARGET_BALLS_${BUCKYBALL_TARGET}) + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_BALLDOMAIN_BALLS) set(BALL_MLIR_TESTS - "${BUCKYBALL_REPO_ROOT}/examples/components/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" ) if(IS_DIRECTORY "${BALL_MLIR_TESTS}") add_subdirectory( diff --git a/examples/designs/multi-rocket/sims/device/stack/runtime/CMakeLists.txt b/examples/designs/multi-rocket/stack/device/CMakeLists.txt similarity index 94% rename from examples/designs/multi-rocket/sims/device/stack/runtime/CMakeLists.txt rename to examples/designs/multi-rocket/stack/device/CMakeLists.txt index be49a4d40..bf1118f54 100644 --- a/examples/designs/multi-rocket/sims/device/stack/runtime/CMakeLists.txt +++ b/examples/designs/multi-rocket/stack/device/CMakeLists.txt @@ -2,7 +2,7 @@ # The multi-rocket device: Rocket cores without Balls or hardware threads, and # the runtime that schedules jobs onto them. # -# bbdev stack --build "--design multi-rocket" +# bbdev stack --build "--design multi-rocket --device " # # builds every core stack of the device first, then configures this with # DESIGN_CMAKE, generated from design.pb, and the RISC-V Linux toolchain. @@ -24,7 +24,7 @@ foreach(CORE IN LISTS DESIGN_CORES) target_link_options(bindings INTERFACE "${DESIGN_CORE_BUILD_${CORE}}/execute.o") endforeach() # The device has no banks, so no DMA helpers. -set(RUNTIME_PARAMS) +set(RUNTIME_DEFS) add_subdirectory("${DESIGN_REPO}/stack/engine" engine) add_subdirectory("${DESIGN_REPO}/stack/runtime" runtime) diff --git a/examples/designs/multi-rocket/workloads/device/core/ctests/CMakeLists.txt b/examples/designs/multi-rocket/workloads/device/core/ctests/CMakeLists.txt index a714819f0..34e4852d3 100644 --- a/examples/designs/multi-rocket/workloads/device/core/ctests/CMakeLists.txt +++ b/examples/designs/multi-rocket/workloads/device/core/ctests/CMakeLists.txt @@ -1,11 +1,11 @@ set(CPU_ARITHMETIC - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CONTROL_TARGET}-ctest-cpu_arithmetic" + "${DESIGN_NAME}-${BUCKYBALL_CONTROL_TARGET}-ctest-cpu_arithmetic" ) set(CPU_ELF ${CMAKE_CURRENT_BINARY_DIR}/${CPU_ARITHMETIC}-baremetal) add_custom_command( OUTPUT ${CPU_ELF} COMMAND - ${ELF_CC} ${BUCKYBALL_CTEST_C_FLAGS} -I${BUCKYBALL_ISA_DIR}/control + ${ELF_CC} ${BUCKYBALL_CTEST_C_FLAGS} -DCORE_HART_ID=${DESIGN_WORKLOAD_CONTROL_HART} ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c ${CMAKE_CURRENT_SOURCE_DIR}/cpu_arithmetic.c -o ${CPU_ELF} DEPENDS @@ -13,7 +13,6 @@ add_custom_command( ${BAREMETAL_DIR}/crt0.S ${BAREMETAL_DIR}/syscalls.c ${BBSIM_LD} - ${BUCKYBALL_ISA_DIR}/control/params.h ) add_custom_target(${CPU_ARITHMETIC}-baremetal-build DEPENDS ${CPU_ELF}) add_dependencies(design-ctests-build ${CPU_ARITHMETIC}-baremetal-build) diff --git a/examples/designs/multi-rocket/workloads/device/soc/CMakeLists.txt b/examples/designs/multi-rocket/workloads/device/soc/CMakeLists.txt index 87ac0f726..9d50fc377 100644 --- a/examples/designs/multi-rocket/workloads/device/soc/CMakeLists.txt +++ b/examples/designs/multi-rocket/workloads/device/soc/CMakeLists.txt @@ -1,12 +1,13 @@ -set(COHERENT_ELF ${CMAKE_CURRENT_BINARY_DIR}/${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CONTROL_TARGET}-ctest-design-coherent_exchange-multicore-baremetal) +set(COHERENT_ELF ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-${BUCKYBALL_CONTROL_TARGET}-ctest-design-coherent_exchange-multicore-baremetal) add_custom_command(OUTPUT ${COHERENT_ELF} COMMAND ${ELF_CC} -O2 -march=rv64gc -mcmodel=medany -msmall-data-limit=0 -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${BBSIM_LD} - -I${BUCKYBALL_ISA_DIR}/design + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} ${BAREMETAL_DIR}/start.S ${CMAKE_CURRENT_SOURCE_DIR}/coherent_exchange.c -lgcc -o ${COHERENT_ELF} DEPENDS coherent_exchange.c ${BAREMETAL_DIR}/start.S ${BBSIM_LD} - ${BUCKYBALL_ISA_DIR}/design/instances.h) + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + VERBATIM) add_custom_target(coherent-exchange-build DEPENDS ${COHERENT_ELF}) add_dependencies(design-ctests-build coherent-exchange-build) add_dependencies(soctests-build coherent-exchange-build) diff --git a/examples/designs/multi-rocket/workloads/device/soc/coherent_exchange.c b/examples/designs/multi-rocket/workloads/device/soc/coherent_exchange.c index 3a3813344..a1a106c6e 100644 --- a/examples/designs/multi-rocket/workloads/device/soc/coherent_exchange.c +++ b/examples/designs/multi-rocket/workloads/device/soc/coherent_exchange.c @@ -1,7 +1,7 @@ -#include +#include #include -static uint64_t payload[BB_CORE_NUM][8] __attribute__((aligned(64))); +static uint64_t payload[BB_NUM_CORES][8] __attribute__((aligned(64))); static unsigned arrived[2] __attribute__((aligned(64))); static unsigned consumed[2] __attribute__((aligned(64))); static unsigned failed; @@ -15,15 +15,16 @@ int main(void) { payload[slot][0] = value; payload[slot][7] = ~value; __atomic_fetch_add(&arrived[round], 1, __ATOMIC_ACQ_REL); - while (__atomic_load_n(&arrived[round], __ATOMIC_ACQUIRE) != BB_CORE_NUM) { + while (__atomic_load_n(&arrived[round], __ATOMIC_ACQUIRE) != BB_NUM_CORES) { } - for (unsigned peer = 0; peer < BB_CORE_NUM; ++peer) { + for (unsigned peer = 0; peer < BB_NUM_CORES; ++peer) { uint64_t expected = 0x5a000000ULL + round * 0x10000 + peer * 9; if (payload[peer][0] != expected || payload[peer][7] != ~expected) __atomic_fetch_or(&failed, 1, __ATOMIC_RELAXED); } __atomic_fetch_add(&consumed[round], 1, __ATOMIC_ACQ_REL); - while (__atomic_load_n(&consumed[round], __ATOMIC_ACQUIRE) != BB_CORE_NUM) { + while (__atomic_load_n(&consumed[round], __ATOMIC_ACQUIRE) != + BB_NUM_CORES) { } } return __atomic_load_n(&failed, __ATOMIC_ACQUIRE); diff --git a/examples/designs/pebble/configs/device/core/acc/frontend/bat-off-rob24.toml b/examples/designs/pebble/configs/device/core/acc/frontend/bat-off-rob24.toml index f196d4b84..e14dee6d0 100644 --- a/examples/designs/pebble/configs/device/core/acc/frontend/bat-off-rob24.toml +++ b/examples/designs/pebble/configs/device/core/acc/frontend/bat-off-rob24.toml @@ -2,8 +2,8 @@ bankAliasEnable = false robEntries = 24 rsOutOfOrderResponse = true bankIdLen = 10 -vbankIdUpperBound = 23 -sharedBankIdBase = 32 +maxPrivateVbank = 23 +sharedVbankBase = 32 iterLen = 34 subRobEnable = false subRobDepth = 64 diff --git a/examples/designs/pebble/configs/device/core/acc/frontend/rob24.toml b/examples/designs/pebble/configs/device/core/acc/frontend/rob24.toml index 3964cea2e..ca2defa54 100644 --- a/examples/designs/pebble/configs/device/core/acc/frontend/rob24.toml +++ b/examples/designs/pebble/configs/device/core/acc/frontend/rob24.toml @@ -2,8 +2,8 @@ bankAliasEnable = true robEntries = 24 rsOutOfOrderResponse = true bankIdLen = 10 -vbankIdUpperBound = 23 -sharedBankIdBase = 32 +maxPrivateVbank = 23 +sharedVbankBase = 32 iterLen = 34 subRobEnable = false subRobDepth = 64 diff --git a/examples/designs/pebble/configs/device/core/acc/memdomain/blt-off.toml b/examples/designs/pebble/configs/device/core/acc/memdomain/blt-off.toml index 82d606195..faade6197 100644 --- a/examples/designs/pebble/configs/device/core/acc/memdomain/blt-off.toml +++ b/examples/designs/pebble/configs/device/core/acc/memdomain/blt-off.toml @@ -1,5 +1,7 @@ # Pebble thread memory domain configuration +addrLen = 39 + [bank] layoutEnable = false num = 24 @@ -17,6 +19,3 @@ bankNum = 5 bankEntries = 1024 bankWidth = 8 readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/designs/pebble/configs/device/core/default.toml b/examples/designs/pebble/configs/device/core/default.toml index 787135494..c07169d55 100644 --- a/examples/designs/pebble/configs/device/core/default.toml +++ b/examples/designs/pebble/configs/device/core/default.toml @@ -28,7 +28,7 @@ virtualBankCount = 24 entries = 16384 inputChannels = 1 defaultGroupCount = 1 -memCycles = 10 +latency = 10 [compiler] name = "pebble" diff --git a/examples/designs/pebble/configs/device/core/memory/access/pebble.toml b/examples/designs/pebble/configs/device/core/memory/access/pebble.toml index bb87f76a4..2e90444bc 100644 --- a/examples/designs/pebble/configs/device/core/memory/access/pebble.toml +++ b/examples/designs/pebble/configs/device/core/memory/access/pebble.toml @@ -1,5 +1,5 @@ translationEntries = 16 -accessTagBits = 6 +tagBits = 6 resultLineBits = 0 [[regions]] base = 0x80000000 diff --git a/examples/designs/pebble/configs/device/core/memory/dcache/pebble.toml b/examples/designs/pebble/configs/device/core/memory/dcache/pebble.toml index 97b8e2bba..4fb4beb6f 100644 --- a/examples/designs/pebble/configs/device/core/memory/dcache/pebble.toml +++ b/examples/designs/pebble/configs/device/core/memory/dcache/pebble.toml @@ -7,7 +7,7 @@ maxRanges = 8 [cache] nodeId = 2 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/pebble/configs/device/core/memory/home/pebble.toml b/examples/designs/pebble/configs/device/core/memory/home/pebble.toml index 3e0779cbc..aa39cd52e 100644 --- a/examples/designs/pebble/configs/device/core/memory/home/pebble.toml +++ b/examples/designs/pebble/configs/device/core/memory/home/pebble.toml @@ -1,6 +1,6 @@ agents = 2 mshrEntries = 8 -homeId = 64 +nodeId = 64 requesters = "default" [cache] diff --git a/examples/designs/pebble/configs/device/core/memory/icache/pebble.toml b/examples/designs/pebble/configs/device/core/memory/icache/pebble.toml index 75e065643..3cc7df83a 100644 --- a/examples/designs/pebble/configs/device/core/memory/icache/pebble.toml +++ b/examples/designs/pebble/configs/device/core/memory/icache/pebble.toml @@ -1,5 +1,5 @@ nodeId = 1 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/pebble/configs/sims/default.toml b/examples/designs/pebble/configs/sims/default.toml index 70c1347aa..b4ae21f73 100644 --- a/examples/designs/pebble/configs/sims/default.toml +++ b/examples/designs/pebble/configs/sims/default.toml @@ -11,5 +11,8 @@ regression = "sims/device/regression" kernel = "sims/device/workloads/kernel" tapeout = "sims/device/tapeout" +[targets.device.configs] +PebbleVerilatorConfig = "verilator.toml" + [verification] registry = "../../sims/ip/uvm/registry.toml" diff --git a/examples/designs/pebble/configs/sims/verilator.toml b/examples/designs/pebble/configs/sims/verilator.toml new file mode 100644 index 000000000..bc1d59673 --- /dev/null +++ b/examples/designs/pebble/configs/sims/verilator.toml @@ -0,0 +1 @@ +# PebbleVerilatorConfig: the pebble design as configured, with no overrides. diff --git a/examples/designs/pebble/sims/device/emu/build.rs b/examples/designs/pebble/sims/device/emu/build.rs index d634d5dba..beb62e98c 100644 --- a/examples/designs/pebble/sims/device/emu/build.rs +++ b/examples/designs/pebble/sims/device/emu/build.rs @@ -1,7 +1,4 @@ -use std::{env, path::PathBuf}; - fn main() { - let entry = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - bebop_sim_build::balls(&entry.join("../../../output/generated/design.pb")); + bebop_sim_build::balls(); bebop_sim_build::runner_links(); } diff --git a/examples/designs/pebble/sims/device/emu/src/lib.rs b/examples/designs/pebble/sims/device/emu/src/lib.rs index 5d34a2311..b92afd6f1 100644 --- a/examples/designs/pebble/sims/device/emu/src/lib.rs +++ b/examples/designs/pebble/sims/device/emu/src/lib.rs @@ -6,6 +6,6 @@ extern crate bebop_design_pebble; include!(concat!(env!("OUT_DIR"), "/balls.rs")); pub const SIMULATION: bebop_bemu::Simulation = bebop_bemu::Simulation { - design: include_bytes!("../../../../output/generated/design.pb"), + design: include_bytes!(concat!(env!("OUT_DIR"), "/design.pb")), instantiate: bebop_bemu::device::cbus::instantiate, }; diff --git a/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml index 2f7e09296..5a46c4f3e 100644 --- a/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml +++ b/examples/designs/pebble/sims/device/regression/bemu/workloads-bare.toml @@ -4,6 +4,8 @@ tests = [ "pebble-pebble-mlirtest-core_mega_matmul-baremetal", "pebble-pebble-mlirtest-core_mega_conv2d_k5_tail-baremetal", "pebble-pebble-mlirtest-core_mega_conv2d-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d_resident-baremetal", + "pebble-pebble-mlirtest-core_mega_conv2d_flatten_linear-baremetal", "pebble-pebble-mlirtest-core_lookup_channel_i8-baremetal", "pebble-pebble-mlirtest-core_generic_matmul_i8-baremetal", "pebble-pebble-mlirtest-core_linear_int8-baremetal", @@ -43,7 +45,7 @@ tests = [ "pebble-pebble-mlirtest-bank_int2fp_unit_scale-baremetal", "pebble-pebble-mlirtest-bank_quant_i32_pack16-baremetal", "pebble-pebble-mlirtest-bank_quant_f32_pack16-baremetal", - "pebble-pebble-mlirtest-bank_im2col_k3_padded_bank_windows-baremetal", + "pebble-pebble-mlirtest-bank_im2col_k3_two_blocks-baremetal", "pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-baremetal", "pebble-pebble-ctest-tlb_bare_mode_test-baremetal", "pebble-pebble-ctest-tlb_repeated_sfence_test-baremetal", diff --git a/examples/designs/pebble/sims/device/regression/bemu/workloads-linux.toml b/examples/designs/pebble/sims/device/regression/bemu/workloads-linux.toml index 3b03d2bd2..41a9cd13b 100644 --- a/examples/designs/pebble/sims/device/regression/bemu/workloads-linux.toml +++ b/examples/designs/pebble/sims/device/regression/bemu/workloads-linux.toml @@ -1,4 +1,4 @@ [workloads] tests = [ - "fw_payload-h1-pebble-linux.elf", + "fw_payload-linux.elf", ] diff --git a/examples/designs/pebble/sims/device/regression/p2e/workloads-bare.toml b/examples/designs/pebble/sims/device/regression/p2e/workloads-bare.toml index f483058b0..80d331ecf 100644 --- a/examples/designs/pebble/sims/device/regression/p2e/workloads-bare.toml +++ b/examples/designs/pebble/sims/device/regression/p2e/workloads-bare.toml @@ -20,14 +20,14 @@ tests = [ "pebble-pebble-mlirtest-bank_int2fp_unit_scale-baremetal.hex", "pebble-pebble-mlirtest-bank_quant_i32_pack16-baremetal.hex", "pebble-pebble-mlirtest-bank_quant_f32_pack16-baremetal.hex", - "pebble-pebble-mlirtest-bank_im2col_k3_padded_bank_windows-baremetal.hex", + "pebble-pebble-mlirtest-bank_im2col_k3_two_blocks-baremetal.hex", "pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-baremetal.hex", - "pebble-pebble-ctest-pipeline-bank_dma_reuse-multicore-baremetal", - "pebble-pebble-ctest-pipeline-maxpool_partial_update-multicore-baremetal", - "pebble-pebble-ctest-pipeline-bank_dma_pingpong-multicore-baremetal", - "pebble-pebble-ctest-pipeline-read_bank_shared-multicore-baremetal", - "pebble-pebble-ctest-pipeline-read_bank_split-multicore-baremetal", - "pebble-pebble-ctest-pipeline-same_ball_independent-multicore-baremetal", - "pebble-pebble-ctest-pipeline-rob_window_probe-multicore-baremetal", - "pebble-pebble-ctest-pipeline-write_bank_versions-multicore-baremetal", + "pebble-pebble-ctest-pipeline-bank_dma_reuse-multicore-baremetal.hex", + "pebble-pebble-ctest-pipeline-maxpool_partial_update-multicore-baremetal.hex", + "pebble-pebble-ctest-pipeline-bank_dma_pingpong-multicore-baremetal.hex", + "pebble-pebble-ctest-pipeline-read_bank_shared-multicore-baremetal.hex", + "pebble-pebble-ctest-pipeline-read_bank_split-multicore-baremetal.hex", + "pebble-pebble-ctest-pipeline-same_ball_independent-multicore-baremetal.hex", + "pebble-pebble-ctest-pipeline-rob_window_probe-multicore-baremetal.hex", + "pebble-pebble-ctest-pipeline-write_bank_versions-multicore-baremetal.hex", ] diff --git a/examples/designs/pebble/sims/device/stack/CMakeLists.txt b/examples/designs/pebble/sims/device/stack/CMakeLists.txt index 22b169301..4323021ce 100644 --- a/examples/designs/pebble/sims/device/stack/CMakeLists.txt +++ b/examples/designs/pebble/sims/device/stack/CMakeLists.txt @@ -2,7 +2,7 @@ # The software stack of the pebble device simulation: every level the device # needs, built bottom up. # -# bbdev stack --build "--design pebble" +# bbdev stack --build "--design pebble --device " # # configures this with DESIGN_CMAKE, generated from design.pb. Each level is a # CMake build of its own: cores and threads build buddy-opt plugins for the diff --git a/examples/designs/pebble/sims/device/stack/core/pebble/CMakeLists.txt b/examples/designs/pebble/sims/device/stack/core/pebble/CMakeLists.txt deleted file mode 100644 index 908456289..000000000 --- a/examples/designs/pebble/sims/device/stack/core/pebble/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -cmake_minimum_required(VERSION 3.20) -project(pebble-core-stack LANGUAGES C CXX) -include("${DESIGN_CMAKE}") - -add_subdirectory("${DESIGN_REPO}/examples/designs/pebble/stack/engine" engine) -add_subdirectory("${DESIGN_REPO}/examples/designs/pebble/stack/midend" midend) diff --git a/examples/designs/pebble/sims/device/tapeout/area/constraints.sdc b/examples/designs/pebble/sims/device/tapeout/area/constraints.sdc index a1877a3a3..7af7d73af 100644 --- a/examples/designs/pebble/sims/device/tapeout/area/constraints.sdc +++ b/examples/designs/pebble/sims/device/tapeout/area/constraints.sdc @@ -1,93 +1,9 @@ -set core_clock [get_ports auto_chipyard_prcictrl_domain_reset_setter_clock_in_member_allClocks_uncore_clock] -set debug_clock [get_ports debug_clock] -set jtag_clock [get_ports debug_systemjtag_jtag_TCK] -set serial_tl_clock [get_ports serial_tl_0_clock_in] -set jtag_inputs [get_ports {debug_systemjtag_jtag_TMS debug_systemjtag_jtag_TDI}] -set jtag_outputs [get_ports debug_systemjtag_jtag_TDO_data] -set serial_tl_inputs [get_ports {serial_tl_0_in_valid serial_tl_0_in_bits_phit* serial_tl_0_out_ready}] -set serial_tl_outputs [get_ports {serial_tl_0_in_ready serial_tl_0_out_valid serial_tl_0_out_bits_phit*}] -set core_inputs [get_ports { - debug_dmactiveAck - custom_boot - i2c_0_scl_in - i2c_0_sda_in - uart_0_rxd - gpio_0_pins_0_i_ival - gpio_0_pins_1_i_ival - gpio_0_pins_2_i_ival - gpio_0_pins_3_i_ival - gpio_0_pins_4_i_ival - gpio_0_pins_5_i_ival - gpio_0_pins_6_i_ival - gpio_0_pins_7_i_ival - spi_0_dq_0_i - spi_0_dq_1_i - spi_0_dq_2_i - spi_0_dq_3_i -}] -set core_outputs [get_ports { - i2c_0_scl_oe - i2c_0_sda_oe - uart_0_txd - gpio_0_pins_0_o_oval - gpio_0_pins_0_o_oe - gpio_0_pins_0_o_ie - gpio_0_pins_1_o_oval - gpio_0_pins_1_o_oe - gpio_0_pins_1_o_ie - gpio_0_pins_2_o_oval - gpio_0_pins_2_o_oe - gpio_0_pins_2_o_ie - gpio_0_pins_3_o_oval - gpio_0_pins_3_o_oe - gpio_0_pins_3_o_ie - gpio_0_pins_4_o_oval - gpio_0_pins_4_o_oe - gpio_0_pins_4_o_ie - gpio_0_pins_5_o_oval - gpio_0_pins_5_o_oe - gpio_0_pins_5_o_ie - gpio_0_pins_6_o_oval - gpio_0_pins_6_o_oe - gpio_0_pins_6_o_ie - gpio_0_pins_7_o_oval - gpio_0_pins_7_o_oe - gpio_0_pins_7_o_ie - spi_0_sck - spi_0_dq_0_o - spi_0_dq_0_ie - spi_0_dq_0_oe - spi_0_dq_1_o - spi_0_dq_1_ie - spi_0_dq_1_oe - spi_0_dq_2_o - spi_0_dq_2_ie - spi_0_dq_2_oe - spi_0_dq_3_o - spi_0_dq_3_ie - spi_0_dq_3_oe - spi_0_cs_0 -}] - +# DeviceTop has one clock; every other port is synchronous to it. +set core_clock [get_ports clock] create_clock -name core -period 10.0 $core_clock -create_clock -name debug -period 10.0 $debug_clock -create_clock -name jtag -period 100.0 $jtag_clock -create_clock -name serial_tl -period 10.0 $serial_tl_clock -set_clock_groups -asynchronous \ - -group [get_clocks core] \ - -group [get_clocks debug] \ - -group [get_clocks jtag] \ - -group [get_clocks serial_tl] - -set_clock_uncertainty -setup 3.0 [get_clocks {core debug serial_tl}] -set_clock_uncertainty -setup 10.0 [get_clocks jtag] -set_clock_transition 1.0 [get_clocks {core debug serial_tl}] -set_clock_transition 10.0 [get_clocks jtag] -set_input_delay 3.0 -clock core $core_inputs -set_output_delay 3.0 -clock core $core_outputs +set_clock_uncertainty -setup 3.0 [get_clocks core] +set_clock_transition 1.0 [get_clocks core] -set_input_delay 20.0 -clock_fall -clock jtag $jtag_inputs -set_output_delay 20.0 -clock jtag $jtag_outputs -set_input_delay 3.0 -clock serial_tl $serial_tl_inputs -set_output_delay 3.0 -clock serial_tl $serial_tl_outputs +set_input_delay 3.0 -clock core [remove_from_collection [all_inputs] $core_clock] +set_output_delay 3.0 -clock core [all_outputs] diff --git a/examples/designs/pebble/sims/device/tapeout/area/dc.tcl b/examples/designs/pebble/sims/device/tapeout/area/dc.tcl index f5d3b97a2..3d4646417 100755 --- a/examples/designs/pebble/sims/device/tapeout/area/dc.tcl +++ b/examples/designs/pebble/sims/device/tapeout/area/dc.tcl @@ -25,7 +25,7 @@ file mkdir $output_dir $report_dir [file join $report_dir work] set target_library [list $RUN_TARGET_LIBRARY] set synthetic_library $RUN_SYNTHETIC_LIBRARY set link_library [concat [list *] $target_library $synthetic_library $RUN_LINK_LIBRARY] -set_host_options -max_cores $RUN_MAX_CORES +set_host_options -max_cores $RUN_MAX_THREADS define_design_lib WORK -path [file join $report_dir work] set search_path [list .] set_app_var verilogout_no_tri true diff --git a/examples/designs/pebble/sims/device/workloads/CMakeLists.txt b/examples/designs/pebble/sims/device/workloads/CMakeLists.txt index c3678bd81..19ba06a88 100644 --- a/examples/designs/pebble/sims/device/workloads/CMakeLists.txt +++ b/examples/designs/pebble/sims/device/workloads/CMakeLists.txt @@ -27,19 +27,16 @@ foreach(BUCKYBALL_COMPONENT_WORKLOAD_TARGET IN LISTS DESIGN_CPU_TARGETS) if( COMPONENT_WORKLOADS STREQUAL - "${BUCKYBALL_REPO_ROOT}/examples/designs/${BUCKYBALL_WORKLOAD_DESIGN}/workloads" + "${BUCKYBALL_REPO_ROOT}/examples/designs/${DESIGN_NAME}/workloads" ) continue() endif() - set(BUCKYBALL_COMPONENT_WORKLOAD_PROFILE - "${DESIGN_TARGET_ID_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" - ) set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}") set(BUCKYBALL_CTEST_GROUP_TARGET component-ctests-build) if(IS_DIRECTORY "${COMPONENT_WORKLOADS}/ctests") add_subdirectory( "${COMPONENT_WORKLOADS}/ctests" - "${CMAKE_CURRENT_BINARY_DIR}/components/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/prefab/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" ) endif() endforeach() @@ -58,9 +55,9 @@ add_subdirectory( ) foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) set(BUCKYBALL_MLIR_ACTIVE_TARGET "${BUCKYBALL_TARGET}") - foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_TARGET_BALLS_${BUCKYBALL_TARGET}) + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_BALLDOMAIN_BALLS) set(BALL_MLIR_TESTS - "${BUCKYBALL_REPO_ROOT}/examples/components/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" ) if(IS_DIRECTORY "${BALL_MLIR_TESTS}") add_subdirectory( diff --git a/examples/designs/pebble/sims/device/workloads/kernel/CMakeLists.txt b/examples/designs/pebble/sims/device/workloads/kernel/CMakeLists.txt index 46341dc68..13c2e977f 100644 --- a/examples/designs/pebble/sims/device/workloads/kernel/CMakeLists.txt +++ b/examples/designs/pebble/sims/device/workloads/kernel/CMakeLists.txt @@ -2,6 +2,8 @@ set(BUCKYBALL_KERNEL_PROGRAMS 0 pebble-pebble-mlirtest-core_mega_matmul-linux 0 pebble-pebble-mlirtest-core_mega_conv2d-linux + 0 pebble-pebble-mlirtest-core_mega_conv2d_resident-linux + 0 pebble-pebble-mlirtest-core_mega_conv2d_flatten_linear-linux 0 pebble-pebble-mlirtest-core_lookup_channel_i8-linux 0 pebble-pebble-mlirtest-core_generic_matmul_i8-linux 0 pebble-pebble-mlirtest-core_linear_int8-linux @@ -37,6 +39,6 @@ set(BUCKYBALL_KERNEL_PROGRAMS 0 pebble-pebble-mlirtest-bank_int2fp_unit_scale-linux 0 pebble-pebble-mlirtest-bank_quant_i32_pack16-linux 0 pebble-pebble-mlirtest-bank_quant_f32_pack16-linux - 0 pebble-pebble-mlirtest-bank_im2col_k3_padded_bank_windows-linux + 0 pebble-pebble-mlirtest-bank_im2col_k3_two_blocks-linux 0 pebble-pebble-mlirtest-bank_transpose_i8_packed_matrix-linux PARENT_SCOPE) diff --git a/examples/designs/pebble/sims/device/stack/runtime/CMakeLists.txt b/examples/designs/pebble/stack/device/CMakeLists.txt similarity index 91% rename from examples/designs/pebble/sims/device/stack/runtime/CMakeLists.txt rename to examples/designs/pebble/stack/device/CMakeLists.txt index 1c3999e2b..995c43d41 100644 --- a/examples/designs/pebble/sims/device/stack/runtime/CMakeLists.txt +++ b/examples/designs/pebble/stack/device/CMakeLists.txt @@ -2,7 +2,7 @@ # The pebble device: its cores, their hardware threads and the runtime that # schedules jobs onto the cores. # -# bbdev stack --build "--design pebble" +# bbdev stack --build "--design pebble --device " # # builds every thread and core stack of the device first, then configures this # with DESIGN_CMAKE, generated from design.pb, and the RISC-V Linux toolchain. @@ -24,12 +24,12 @@ add_library(bindings INTERFACE) foreach(CORE IN LISTS DESIGN_CORES) target_link_options(bindings INTERFACE "${DESIGN_CORE_BUILD_${CORE}}/execute.o") endforeach() -foreach(THREAD IN LISTS DESIGN_THREADS) +foreach(THREAD IN LISTS DESIGN_THREAD_STACKS) target_link_options(bindings INTERFACE "${DESIGN_THREAD_BUILD_${THREAD}}/loader.o") endforeach() # DMA helpers are sized by the target with the fewest bank lines. list(GET DESIGN_RUNTIME_TARGETS 0 PARAMS_TARGET) -set(RUNTIME_PARAMS "${DESIGN_ISA_DIR}/${PARAMS_TARGET}") +set(RUNTIME_DEFS ${DESIGN_TARGET_DEFS_${PARAMS_TARGET}}) add_subdirectory("${DESIGN_REPO}/stack/engine" engine) add_subdirectory("${DESIGN_REPO}/stack/runtime" runtime) diff --git a/examples/designs/pebble/stack/device/core/CMakeLists.txt b/examples/designs/pebble/stack/device/core/CMakeLists.txt new file mode 100644 index 000000000..48995af5d --- /dev/null +++ b/examples/designs/pebble/stack/device/core/CMakeLists.txt @@ -0,0 +1,61 @@ +# ===----------------------------------------------------------------------=== +# Pebble's own core: a CPU with its Ball accelerator. +# +# Built by the stack top of each simulation entry, configured with DESIGN_CMAKE +# and CORE. Products: +# execute.o how the core runs jobs (direct execution) +# libpebble-compiler.so buddy-opt plugin: the core's fusion +# (form-custom-ops) and lower-buckyball around the +# framework bank layer +# pebble-compiler-check FileCheck tests of that plugin +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(pebble-core-stack LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +set(TARGET ${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_PROFILE}) +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# How the core runs jobs. +add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/execute.o" + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" + -c "${RUNTIME}/execute/direct.c" -o "${CMAKE_BINARY_DIR}/execute.o" + DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" + VERBATIM) +add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") + +# The core's compiler plugin. The core owns computation recognition and its +# conversion to bank dataflow. +set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${TARGET}}) +file(GLOB_RECURSE BUCKYBALL_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp") +add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) +add_library(${TARGET}-compiler MODULE) +target_link_libraries(${TARGET}-compiler PRIVATE buckyball-compiler) +set_target_properties(${TARGET}-compiler PROPERTIES + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") + +set(CHECK "${DESIGN_MLIR}/../../../bin/FileCheck") +set(OPT "${DESIGN_COMPILER}/bin/buddy-opt --load-dialect-plugin=$ --load-pass-plugin=$ --buckyball-design-pb=${DESIGN_PB} --buckyball-target=${TARGET}") +set(TESTS "${CMAKE_CURRENT_SOURCE_DIR}/tests") +set(CORE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/../../../workloads/device/core/acc/mlir_tests/core") +add_custom_target(${TARGET}-compiler-check + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-custom-ops.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${TESTS}/form-custom-ops.mlir --implicit-check-not=buckyball." + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-int32-bias.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${TESTS}/form-int32-bias.mlir --implicit-check-not=buckyball." + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/lower-frontend-types.mlir -allow-unregistered-dialect -lower-frontend -split-input-file -verify-diagnostics | ${CHECK} ${TESTS}/lower-frontend-types.mlir --check-prefix=VALID --implicit-check-not=frontend." + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/lower-custom-ops.mlir -convert-linalg-to-buckyball | ${CHECK} ${TESTS}/lower-custom-ops.mlir --implicit-check-not=custom.op" + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/unknown-custom-op.mlir -convert-linalg-to-buckyball -verify-diagnostics -o /dev/null" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/mega_conv2d.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${CORE_TESTS}/mega_conv2d.mlir --check-prefix=FORM" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/mega_conv2d.mlir -allow-unregistered-dialect -form-custom-ops -convert-linalg-to-buckyball | ${CHECK} ${CORE_TESTS}/mega_conv2d.mlir --check-prefix=BANK" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/linear_chain_i8.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${CORE_TESTS}/linear_chain_i8.mlir --check-prefix=FORM" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/linear_chain_i8.mlir -allow-unregistered-dialect -form-custom-ops -convert-linalg-to-buckyball | ${CHECK} ${CORE_TESTS}/linear_chain_i8.mlir --check-prefix=BANK" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/mega_conv2d_resident.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${CORE_TESTS}/mega_conv2d_resident.mlir --check-prefix=FORM" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/mega_conv2d_resident.mlir -allow-unregistered-dialect -form-custom-ops -convert-linalg-to-buckyball | ${CHECK} ${CORE_TESTS}/mega_conv2d_resident.mlir --check-prefix=BANK" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/mega_conv2d_flatten_linear.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${CORE_TESTS}/mega_conv2d_flatten_linear.mlir --check-prefix=FORM" + COMMAND bash -o pipefail -c "${OPT} ${CORE_TESTS}/mega_conv2d_flatten_linear.mlir -allow-unregistered-dialect -form-custom-ops -convert-linalg-to-buckyball | ${CHECK} ${CORE_TESTS}/mega_conv2d_flatten_linear.mlir --check-prefix=BANK" + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-depthwise.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${TESTS}/form-depthwise.mlir --implicit-check-not=linalg.depthwise" + COMMAND bash -o pipefail -c "${OPT} ${TESTS}/lower-bank-updates.mlir -lower-buckyball | ${CHECK} ${TESTS}/lower-bank-updates.mlir" + DEPENDS ${TARGET}-compiler + VERBATIM) diff --git a/examples/designs/pebble/stack/device/core/src/ConvertLinalgToBuckyball.cpp b/examples/designs/pebble/stack/device/core/src/ConvertLinalgToBuckyball.cpp new file mode 100644 index 000000000..3cc742876 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/src/ConvertLinalgToBuckyball.cpp @@ -0,0 +1,197 @@ +//===- ConvertLinalgToBuckyball.cpp - Pebble core lowering ----------------===// +// +// Lowers the custom computations to bank dataflow: the core tiles them at +// bank granularity and connects the banks; every Ball computation is one +// virtual-bank operation of the Ball's dialect, its contract. +// +//===----------------------------------------------------------------------===// + +#include "Buckyball/BuckyballDialect.h" +#include "Buckyball/BuckyballOps.h" +#include "Custom/CustomOps.h" +#include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" +#include "im2colOps.h" +#include "int2fpOps.h" +#include "int8mulOps.h" +#include "lutOps.h" +#include "mataddOps.h" +#include "maxpoolOps.h" +#include "smatmulOps.h" +#include "toint8Ops.h" +#include "transposeOps.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/Pass/Pass.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +// The recipe supplies contiguous tensors; offsets may point into packed +// weights. +Value pebble::tensorBuffer(OpBuilder &builder, Location loc, Value tensor) { + auto type = cast(tensor.getType()); + SmallVector strides(type.getRank()); + int64_t stride = 1; + for (int64_t dim = type.getRank() - 1; dim >= 0; --dim) { + strides[dim] = stride; + stride *= type.getDimSize(dim); + } + auto layout = StridedLayoutAttr::get(builder.getContext(), + ShapedType::kDynamic, strides); + auto memory = MemRefType::get(type.getShape(), type.getElementType(), layout); + return bufferization::ToBufferOp::create(builder, loc, memory, tensor, true); +} + +SmallVector pebble::repeat( + OpBuilder &builder, Location loc, int64_t begin, int64_t end, int64_t step, + bool unroll, ValueRange carried, + function_ref(OpBuilder &, Value, ValueRange)> body) { + SmallVector values(carried); + if (unroll || end - begin <= step) { + for (int64_t index = begin; index < end; index += step) + values = body( + builder, arith::ConstantIndexOp::create(builder, loc, index), values); + return values; + } + auto loop = scf::ForOp::create( + builder, loc, arith::ConstantIndexOp::create(builder, loc, begin), + arith::ConstantIndexOp::create(builder, loc, end), + arith::ConstantIndexOp::create(builder, loc, step), carried, + [&](OpBuilder &nested, Location loc, Value index, ValueRange args) { + scf::YieldOp::create(nested, loc, body(nested, index, args)); + }); + return loop.getResults(); +} + +int64_t pebble::matmulRows() { + return buckyball_target::getBuckyballTarget().bankDepth / 4; +} + +int64_t pebble::tileRows() { + return buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); +} + +int64_t pebble::im2colGroups(int64_t kernel, int64_t blocks, int64_t windows) { + int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; + int64_t rows = (windows + tileRows() - 1) / tileRows() * tileRows(); + int64_t tiles = kernel * kernel * blocks; + // A group holds rows x 16 tile lines of A and 16 lines per tile of B. + int64_t perGroup = std::min(depth / rows, depth / 16); + for (int64_t groups = 1; groups <= 9; ++groups) + if (tiles % groups == 0 && tiles / groups <= perGroup) + return groups; + return 0; +} + +pebble::ConvTiling pebble::convTiling(int64_t height, int64_t width, + int64_t pool, int64_t stride, + int64_t kernel, int64_t blocks, + bool depthwise, bool floating) { + int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; + int64_t maxSide = + buckyball_target::getBuckyballBallParam("IM2COL", "maxIter"); + int64_t largest = std::max(height, width) * pool; + for (int64_t side = largest; side >= pool; side -= 1) { + if (side % pool || + (side * side + tileRows() - 1) / tileRows() * tileRows() > matmulRows()) + continue; + int64_t inputSide = (side - 1) * stride + kernel; + if (inputSide > maxSide || + inputSide * inputSide * (floating ? 4 : 1) > depth) + continue; + int64_t limit = depthwise ? 1 : std::min(blocks, 8); + for (int64_t chunk = limit; chunk >= 1; --chunk) + if (im2colGroups(kernel, chunk, side * side)) + return {side, inputSide, chunk}; + } + return {0, 0, 0}; +} + +void pebble::observeResident(Operation *op, Value bank) { + for (Operation *user : llvm::make_early_inc_range(op->getUsers())) { + if (user->getName().getStringRef() != "buddy_trace.end") + continue; + OpBuilder builder(user); + Operation *trace = builder.clone(*user); + trace->setOperand(0, bank); + trace->getResult(0).setType(bank.getType()); + user->erase(); + } +} + +namespace { +class ConvertLinalgToBuckyballPass + : public PassWrapper> { +public: + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ConvertLinalgToBuckyballPass) + StringRef getArgument() const final { return "convert-linalg-to-buckyball"; } + StringRef getDescription() const final { + return "Lower custom tensor computations to the core bank dataflow"; + } + void getDependentDialects(DialectRegistry ®istry) const override { + registry.insert(); + } + void runOnOperation() override { + DenseMap residentVectors; + SmallVector consumed; + if (failed( + pebble::lowerSpatialOps(getOperation(), residentVectors, consumed))) + return signalPassFailure(); + SmallVector<::buddy::custom::CustomOp> computations; + getOperation().walk([&](::buddy::custom::CustomOp op) { + if (!llvm::is_contained(consumed, op.getOperation())) + computations.push_back(op); + }); + for (auto op : computations) { + StringRef kind = op.getKind(); + Value result, resident; + LogicalResult status = failure(); + if (kind == "global_avg_i8") + status = pebble::lowerGlobalAverage(op, result); + else if (kind == "lookup_i8" || kind == "gate_mul_i8") + status = pebble::lowerChannelOp(op, result); + else if (kind == "transpose_i8") + status = pebble::lowerTranspose(op, result); + else if (kind == "linear_i8" || kind == "linear_i8_requant" || + kind == "matmul_i8") + status = pebble::lowerLinear( + op, residentVectors.lookup(op.getInputs()[0]), result, resident); + else + op.emitOpError("has no bank lowering on this core for ") << kind; + if (failed(status)) + return signalPassFailure(); + if (resident) { + residentVectors[op.getResult(0)] = resident; + pebble::observeResident(op, resident); + consumed.push_back(op); + continue; + } + op.getResult(0).replaceAllUsesWith(result); + op.erase(); + } + for (Operation *op : llvm::reverse(consumed)) + op->erase(); + } +}; + +PassRegistration registration; +} // namespace diff --git a/examples/designs/pebble/stack/midend/src/FormCustomOps.cpp b/examples/designs/pebble/stack/device/core/src/FormCustomOps.cpp similarity index 67% rename from examples/designs/pebble/stack/midend/src/FormCustomOps.cpp rename to examples/designs/pebble/stack/device/core/src/FormCustomOps.cpp index b15b98773..f5b7fa500 100644 --- a/examples/designs/pebble/stack/midend/src/FormCustomOps.cpp +++ b/examples/designs/pebble/stack/device/core/src/FormCustomOps.cpp @@ -10,6 +10,7 @@ #include "mlir/IR/Dominance.h" #include "mlir/IR/Matchers.h" #include "mlir/IR/PatternMatch.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" #include "mlir/Pass/Pass.h" using namespace mlir; @@ -203,6 +204,106 @@ class FormCustomOpsPass rewriter.replaceOp(op, custom.getOutputs()); } pebble::formSpatialOps(getOperation()); + + // A one-row Linear whose single consumer chain [relu] ends in another + // one-row Linear requantizes for it. Its INT8 row stays in banks, one + // group per sixteen columns, when both Linears fit the banks with it. + SmallVector linears; + getOperation().walk([&](CustomOp op) { + if (op.getKind() == "linear_i8") + linears.push_back(op); + }); + auto oneRow = [](CustomOp op) { + return cast(op.getResult(0).getType()).getDimSize(0) == + 1; + }; + for (CustomOp op : linears) { + if (!oneRow(op)) + continue; + Value value = op.getResult(0); + SmallVector links, traces; + bool relu = false; + while (value.hasOneUse()) { + Operation *user = *value.getUsers().begin(); + if (user->getBlock() != op->getBlock()) + break; + if (user->getName().getStringRef() == "buddy_trace.end") { + if (cast(user->getAttr("tensor")).getValue()) + break; + traces.push_back(user); + } else { + auto maximum = dyn_cast(user); + if (relu || !pebble::elementwise(maximum, "arith.maximumf") || + !llvm::is_contained(maximum.getInputs(), value) || + !matchPattern( + maximum.getInputs()[maximum.getInputs()[0] == value ? 1 : 0], + m_AnyZeroFloat())) + break; + relu = true; + } + links.push_back(user); + value = user->getResult(0); + } + if (!value.hasOneUse()) + continue; + auto next = dyn_cast(*value.getUsers().begin()); + if (!next || next.getKind() != "linear_i8" || !oneRow(next) || + next.getInputs()[0] != value || + !dominance.properlyDominates(next.getInputs()[3], op)) + continue; + bool barrier = false; + for (Operation *between = op->getNextNode(); between != next; + between = between->getNextNode()) + if (!isMemoryEffectFree(between) && + between->getName().getDialectNamespace() != "buddy_trace") { + barrier = true; + break; + } + if (barrier) + continue; + // Banks of the producer: its A (and FP32 source) and B, or the resident + // row and B, the row it builds and four more; the consumer reads the + // row with as many groups of B. + auto input = op.getInputs()[0]; + auto producer = input.getDefiningOp(); + bool residentInput = producer && (producer->hasAttr("custom.resident") || + producer.getKind() == "flatten_nchw"); + auto inputType = cast(input.getType()); + int64_t k = inputType.getDimSize(1); + int64_t n = cast(value.getType()).getDimSize(1); + int64_t rowGroups = (n + 15) / 16; + int64_t inputBanks; + if (residentInput) { + inputBanks = producer.getKind() == "flatten_nchw" ? 2 : 2 * (k / 16); + } else { + bool floating = inputType.getElementType().isF32(); + inputBanks = + (floating ? 3 : 2) * pebble::linearSplit(k, 1, floating).groups; + } + bool resident = n % 16 == 0 && + inputBanks + rowGroups + 4 <= target.bankNum - 1 && + 2 * rowGroups + 4 <= target.bankNum - 1; + rewriter.setInsertionPoint(op); + SmallVector inputs(op.getInputs()); + inputs.push_back(next.getInputs()[3]); + auto fused = CustomOp::create( + rewriter, op.getLoc(), + TypeRange{RankedTensorType::get({1, n}, rewriter.getI8Type())}, + "linear_i8", inputs); + fused->setAttr("relu", rewriter.getBoolAttr(relu)); + if (resident) + fused->setAttr("custom.resident", rewriter.getUnitAttr()); + rewriter.setInsertionPointAfter(fused); + for (Operation *trace : traces) { + Operation *moved = rewriter.clone(*trace); + moved->setOperand(0, fused.getResult(0)); + moved->getResult(0).setType(fused.getResult(0).getType()); + } + next->setOperand(0, fused.getResult(0)); + for (Operation *link : llvm::reverse(links)) + rewriter.eraseOp(link); + rewriter.eraseOp(op); + } } }; diff --git a/examples/designs/pebble/stack/midend/src/FormSpatialOps.cpp b/examples/designs/pebble/stack/device/core/src/FormSpatialOps.cpp similarity index 61% rename from examples/designs/pebble/stack/midend/src/FormSpatialOps.cpp rename to examples/designs/pebble/stack/device/core/src/FormSpatialOps.cpp index 0112e43df..f3d3befd2 100644 --- a/examples/designs/pebble/stack/midend/src/FormSpatialOps.cpp +++ b/examples/designs/pebble/stack/device/core/src/FormSpatialOps.cpp @@ -73,9 +73,11 @@ void pebble::formSpatialOps(func::FuncOp function) { weightType.getDimSize(2) > buckyball_target::getBuckyballBallParam("IM2COL", "maxKSize") || strides[0] != strides[1] || dilations != SmallVector{1, 1} || - !pebble::spatialTile(outputType.getDimSize(1), outputType.getDimSize(2), - 1, strides[0], weightType.getDimSize(2)) - .tile) + !pebble::convTiling(outputType.getDimSize(1), outputType.getDimSize(2), + 1, strides[0], weightType.getDimSize(2), + (inputType.getDimSize(3) + 15) / 16, depthwise, + true) + .side) continue; Operation *end = op; Value biasValue; @@ -155,8 +157,9 @@ void pebble::formSpatialOps(func::FuncOp function) { rewriter.eraseOp(dead); } - // A spatial stage retains its ordinary tensor semantics until bank lowering. - // Only single-consumer links ending in another quantized computation fuse. + // A convolution fuses its single-consumer chain [relu] [maxpool] [flatten] + // ending in a quantization: the convolution requantizes for the consumer + // and produces the INT8 tensor the consumer reads. SmallVector roots; function.walk([&](CustomOp op) { if (op.getKind() == "conv2d_i8" || op.getKind() == "depthwise_conv2d_i8") @@ -164,15 +167,13 @@ void pebble::formSpatialOps(func::FuncOp function) { }); DominanceInfo dominance(function); const auto &target = buckyball_target::getBuckyballTarget(); - // The banks of the resident images that chains leave for the next stage. - DenseMap imageBanks; - int64_t group = 0; for (auto root : roots) { Value value = root.getResult(0); - auto outputType = cast(value.getType()); - SmallVector links; - bool relu = false, pool = false, flatten = false; - int64_t window = 1; + auto convType = cast(value.getType()); + // The chain in program order; the cycle markers among it. + SmallVector links, traces; + bool relu = false, flatten = false; + int64_t pool = 1; while (value.hasOneUse()) { Operation *user = *value.getUsers().begin(); if (user->getBlock() != root->getBlock()) @@ -180,8 +181,9 @@ void pebble::formSpatialOps(func::FuncOp function) { if (user->getName().getStringRef() == "buddy_trace.end") { if (cast(user->getAttr("tensor")).getValue()) break; + traces.push_back(user); } else if (auto maximum = dyn_cast(user)) { - if (relu || pool || + if (relu || pool > 1 || flatten || !linalg::isaElemwiseSingleBinaryOpInterface(maximum) || !isa(maximum.getBody()->front()) || !llvm::all_of(maximum.getIndexingMapsArray(), @@ -195,16 +197,15 @@ void pebble::formSpatialOps(func::FuncOp function) { // MAXPOOL pools square windows that tile the convolution output. auto filter = cast(pooling.getInputs()[1].getType()).getShape(); - if (pool || flatten || + if (pool > 1 || flatten || !llvm::all_of(pooling.getDilations().getValues(), [](int64_t v) { return v == 1; }) || filter[0] != filter[1] || !llvm::all_of(pooling.getStrides().getValues(), [&](int64_t v) { return v == filter[0]; }) || - outputType.getDimSize(1) % filter[0] || - outputType.getDimSize(2) % filter[0]) + convType.getDimSize(1) % filter[0] || + convType.getDimSize(2) % filter[0]) break; - window = filter[0]; Value initial = pooling.getOutputs()[0]; if (auto fill = initial.getDefiningOp()) initial = fill.getInputs()[0]; @@ -215,7 +216,7 @@ void pebble::formSpatialOps(func::FuncOp function) { if (!negativeLimit && !(relu && matchPattern(initial, m_AnyZeroFloat()))) break; - pool = true; + pool = filter[0]; } else if (auto transpose = dyn_cast(user)) { if (flatten || transpose.getPermutation() != ArrayRef({0, 3, 1, 2}) || @@ -239,18 +240,16 @@ void pebble::formSpatialOps(func::FuncOp function) { continue; Operation *consumer = *value.getUsers().begin(); auto next = dyn_cast(consumer); - bool terminalQuant = + bool terminal = consumer->getName().getStringRef() == "frontend.fp32_to_int8"; - if (!terminalQuant && (!next || - (next.getKind() != "conv2d_i8" && - next.getKind() != "depthwise_conv2d_i8" && - next.getKind() != "linear_i8") || - next.getInputs()[0] != value)) + if (terminal ? flatten + : (!next || next.getInputs()[0] != value || + (flatten ? next.getKind() != "linear_i8" + : next.getKind() != "conv2d_i8" && + next.getKind() != "depthwise_conv2d_i8"))) continue; - Value multiplier = - terminalQuant ? consumer->getOperand(1) : next.getInputs()[3]; - if ((terminalQuant && flatten) || - !dominance.properlyDominates(multiplier, root)) + Value multiplier = terminal ? consumer->getOperand(1) : next.getInputs()[3]; + if (!dominance.properlyDominates(multiplier, root)) continue; bool barrier = false; for (Operation *between = root->getNextNode(); between != consumer; @@ -260,74 +259,118 @@ void pebble::formSpatialOps(func::FuncOp function) { barrier = true; break; } - if (barrier) - continue; - // The chain keeps every panel of its pooled output in a bank while it - // reads the resident image before it and works in seven more banks. + auto strides = cast(root->getAttr("strides")); auto weightType = cast(root.getInputs()[1].getType()); - int64_t height = outputType.getDimSize(1) / window; - int64_t width = outputType.getDimSize(2) / window; - int64_t channels = weightType.getDimSize(0); - auto [tile, side] = pebble::spatialTile( - height, width, window, - cast(root->getAttr("strides"))[0], - weightType.getDimSize(2)); - if (!tile) - continue; - int64_t banks = (height + side - 1) / side * ((width + side - 1) / side) * - ((channels + 15) / 16); - if (imageBanks.lookup(root.getInputs()[0]) + banks + 7 > target.bankNum) + bool depthwise = root.getKind() == "depthwise_conv2d_i8"; + int64_t kernel = weightType.getDimSize(2); + int64_t blocksIn = + (cast(root.getInputs()[0].getType()).getDimSize(3) + + 15) / + 16; + int64_t height = convType.getDimSize(1), width = convType.getDimSize(2), + channels = convType.getDimSize(3); + pebble::ConvTiling tiling = + pebble::convTiling(height / pool, width / pool, pool, strides[0], + kernel, blocksIn, depthwise, + cast(root.getInputs()[0].getType()) + .getElementType() + .isF32()); + auto producer = root.getInputs()[0].getDefiningOp(); + bool residentInput = producer && producer->hasAttr("custom.resident"); + if (barrier || (!residentInput && !tiling.side)) continue; - // A flattened image is one transposed 16x16 panel that feeds a resident - // one-row Linear. - if (flatten) { - auto linearWeight = cast(next.getInputs()[1].getType()); - if (height != 4 || width != 4 || channels != 16 || - linearWeight.getDimSize(0) > target.bankDepth || - linearWeight.getDimSize(1) % 16) - continue; - } else if (!terminalQuant) { - imageBanks[value] = banks; - } - if (terminalQuant) - links.push_back(consumer); - auto id = rewriter.getI64IntegerAttr(group++); - root->setAttr("custom.spatial_chain", id); - for (Operation *link : links) { - rewriter.setInsertionPoint(link); - if (auto maximum = dyn_cast(link)) { - Value input = matchPattern(maximum.getInputs()[0], m_AnyZeroFloat()) - ? maximum.getInputs()[1] - : maximum.getInputs()[0]; - auto custom = CustomOp::create(rewriter, maximum.getLoc(), - maximum->getResultTypes(), "relu", - ValueRange{input}); - custom->setAttr("custom.spatial_chain", id); - rewriter.replaceOp(maximum, custom.getOutputs()); - } else if (auto pooling = dyn_cast(link)) { - auto custom = CustomOp::create(rewriter, pooling.getLoc(), - pooling->getResultTypes(), "maxpool2d", - ValueRange{pooling.getInputs()[0]}); - custom->setAttr("kernel", rewriter.getDenseI64ArrayAttr( - cast( - pooling.getInputs()[1].getType()) - .getShape())); - custom->setAttr("strides", - rewriter.getDenseI64ArrayAttr(llvm::to_vector( - pooling.getStrides().getValues()))); - custom->setAttr("custom.spatial_chain", id); - rewriter.replaceOp(pooling, custom.getOutputs()); - } else if (auto reshape = dyn_cast(link)) { - auto transpose = reshape.getSrc().getDefiningOp(); - auto custom = CustomOp::create( - rewriter, reshape.getLoc(), reshape->getResultTypes(), - "flatten_nchw", ValueRange{transpose.getInput()}); - custom->setAttr("custom.spatial_chain", id); - rewriter.replaceOp(reshape, custom.getOutputs()); - rewriter.eraseOp(transpose); - } else if (!isa(link)) { - link->setAttr("custom.spatial_chain", id); + + // The INT8 result stays in banks for a consumer that reads it whole: one + // tile covers the convolution, and both stages fit the banks with one + // group per channel block. + int64_t blocksOut = (channels + 15) / 16; + int64_t side = height / pool; + bool resident = false; + if (!depthwise && height == width && + (residentInput || tiling.side == height)) { + int64_t groups = pebble::im2colGroups( + kernel, residentInput ? blocksIn : tiling.chunk, height * width); + bool fits = (residentInput ? blocksIn : 0) + blocksOut + 2 * groups + 4 <= + target.bankNum - 1; + if (flatten) { + auto linearWeight = + cast(next.getInputs()[1].getType()); + resident = fits && side * side == 16 && channels == 16 && + linearWeight.getDimSize(0) <= target.bankDepth; + } else if (!terminal && next.getKind() == "conv2d_i8") { + auto nextWeight = cast(next.getInputs()[1].getType()); + auto nextPadding = cast(next->getAttr("padding")); + int64_t nextStride = + cast(next->getAttr("strides"))[0]; + int64_t nextKernel = nextWeight.getDimSize(2); + int64_t span = side + 2 * nextPadding[0] - nextKernel; + int64_t outputs = span / nextStride + 1; + int64_t nextGroups = + pebble::im2colGroups(nextKernel, blocksOut, outputs * outputs); + int64_t nextBlocks = (nextWeight.getDimSize(0) + 15) / 16; + resident = + fits && llvm::all_equal(nextPadding.asArrayRef()) && + nextPadding[0] <= buckyball_target::getBuckyballBallParam( + "IM2COL", "maxPadding") && + side <= + buckyball_target::getBuckyballBallParam("IM2COL", "maxIter") && + span >= 0 && span % nextStride == 0 && + (outputs * outputs + pebble::tileRows() - 1) / pebble::tileRows() * + pebble::tileRows() <= + pebble::matmulRows() && + nextGroups && + blocksOut + nextBlocks + 2 * nextGroups + 4 <= target.bankNum - 1; } } + + rewriter.setInsertionPoint(root); + SmallVector inputs(root.getInputs()); + inputs.push_back(multiplier); + auto fused = CustomOp::create( + rewriter, root.getLoc(), + TypeRange{RankedTensorType::get({1, side, width / pool, channels}, + rewriter.getI8Type())}, + root.getKind(), inputs); + fused->setAttrs(root->getAttrs()); + fused->setAttr("relu", rewriter.getBoolAttr(relu)); + fused->setAttr("pool", rewriter.getI64IntegerAttr(pool)); + if (resident) + fused->setAttr("custom.resident", rewriter.getUnitAttr()); + Value codes = fused.getResult(0); + // Cycle markers now observe the fused result. + rewriter.setInsertionPointAfter(fused); + for (Operation *trace : traces) { + Operation *moved = rewriter.clone(*trace); + moved->setOperand(0, codes); + moved->getResult(0).setType(codes.getType()); + } + rewriter.setInsertionPoint(consumer); + if (terminal) { + rewriter.replaceOp(consumer, codes); + } else if (flatten && resident) { + next->setOperand( + 0, CustomOp::create(rewriter, root.getLoc(), + TypeRange{cast(value.getType()) + .clone(rewriter.getI8Type())}, + "flatten_nchw", ValueRange{codes}) + .getResult(0)); + } else if (flatten) { + auto nchw = RankedTensorType::get({1, channels, side, width / pool}, + rewriter.getI8Type()); + Value empty = tensor::EmptyOp::create( + rewriter, root.getLoc(), nchw.getShape(), nchw.getElementType()); + Value transposed = + linalg::TransposeOp::create(rewriter, root.getLoc(), codes, empty, + ArrayRef{0, 3, 1, 2}) + .getResult()[0]; + next->setOperand(0, tensor::CollapseShapeOp::create( + rewriter, root.getLoc(), transposed, + ArrayRef{{0}, {1, 2, 3}})); + } else { + next->setOperand(0, codes); + } + for (Operation *link : llvm::reverse(links)) + rewriter.eraseOp(link); + rewriter.eraseOp(root); } } diff --git a/examples/designs/pebble/stack/midend/src/LowerQuantizedOps.cpp b/examples/designs/pebble/stack/device/core/src/LowerChannelOps.cpp similarity index 55% rename from examples/designs/pebble/stack/midend/src/LowerQuantizedOps.cpp rename to examples/designs/pebble/stack/device/core/src/LowerChannelOps.cpp index 4b7c562df..d844ec5a7 100644 --- a/examples/designs/pebble/stack/midend/src/LowerQuantizedOps.cpp +++ b/examples/designs/pebble/stack/device/core/src/LowerChannelOps.cpp @@ -1,140 +1,19 @@ -//===- LowerQuantizedOps.cpp - INT8 reductions and channel operations -----===// +//===- LowerChannelOps.cpp - Per-channel INT8 operations and transpose ---===// #include "Buckyball/BuckyballOps.h" #include "Lowering.h" #include "Target/BuckyballTargetRegistry.h" +#include "int8mulOps.h" +#include "lutOps.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Bufferization/IR/Bufferization.h" #include "mlir/Dialect/Linalg/IR/Linalg.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/SCF/IR/SCF.h" +#include "transposeOps.h" using namespace mlir; using namespace ::buddy::buckyball; -LogicalResult pebble::lowerGlobalAverage(::buddy::custom::CustomOp op, - Value &result) { - auto type = cast(op.getInputs()[0].getType()); - auto resultType = cast(op.getResult(0).getType()); - if (!type.hasStaticShape() || type.getRank() != 4 || - type.getDimSize(0) != 1 || !type.getElementType().isInteger(8) || - !resultType.getElementType().isInteger(8) || - resultType.getShape() != ArrayRef({1, 1, 1, type.getDimSize(3)})) - return op.emitOpError( - "global_avg_i8 requires NHWC INT8 input and 1x1x1xC INT8 output"); - int64_t height = type.getDimSize(1), width = type.getDimSize(2), - channels = type.getDimSize(3); - int64_t pixels = height * width; - int64_t capacity = buckyball_target::getBuckyballTarget().bankDepth / 16 * 16; - OpBuilder b(op); - Location loc = op.getLoc(); - auto index = [&](OpBuilder &b, int64_t n) -> Value { - return arith::ConstantIndexOp::create(b, loc, n); - }; - Value zero = index(b, 0), one = index(b, 1); - Type i64 = b.getI64Type(); - float inputScale = - cast(op->getAttr("input_scale")).getValueAsDouble(); - float outputScale = - cast(op->getAttr("output_scale")).getValueAsDouble(); - float ratio = inputScale / (pixels * outputScale); - Value input = tensorBuffer(b, loc, op.getInputs()[0]); - Value output = memref::AllocOp::create( - b, loc, MemRefType::get(resultType.getShape(), b.getI8Type())); - Value bias = - memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getI32Type())); - Value scales = - memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getF32Type())); - Value values = - memref::AllocOp::create(b, loc, MemRefType::get({16}, b.getI8Type())); - Value izero = arith::ConstantIntOp::create(b, loc, 0, 8); - Value biasZero = arith::ConstantIntOp::create(b, loc, 0, 32); - Value iOne = arith::ConstantIntOp::create(b, loc, 1, 8); - Value scale = arith::ConstantOp::create(b, loc, b.getF32FloatAttr(ratio)); - linalg::FillOp::create(b, loc, biasZero, bias); - linalg::FillOp::create(b, loc, scale, scales); - Value scaleBank = BankAllocOp::create(b, loc, i64, 1, 1); - scaleBank = BankMvinOp::create(b, loc, i64, scales, scaleBank, 4, 1); - pebble::repeat( - b, loc, 0, channels, 16, false, {}, - [&](OpBuilder &b, Value c, ValueRange) -> SmallVector { - Value lanesEnd = arith::MinSIOp::create( - b, loc, index(b, 16), - arith::SubIOp::create(b, loc, index(b, channels), c)); - Value acc = BankAllocOp::create(b, loc, i64, 1, 1); - acc = BankMvinOp::create(b, loc, i64, bias, acc, 4, 1); - acc = pebble::accumulate( - b, loc, pixels, capacity, false, acc, - [&](OpBuilder &b, Value first, int64_t valid, bool isFirst, - bool isLast, Value acc) -> Value { - int64_t k = (valid + 15) / 16 * 16; - Value pack = memref::AllocOp::create( - b, loc, MemRefType::get({k, 16}, b.getI8Type())); - Value ones = memref::AllocOp::create( - b, loc, MemRefType::get({k}, b.getI8Type())); - linalg::FillOp::create(b, loc, izero, pack); - linalg::FillOp::create(b, loc, iOne, ones); - auto spatial = - scf::ForOp::create(b, loc, zero, index(b, valid), one); - { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(spatial.getBody()); - Value p = spatial.getInductionVar(); - Value source = arith::AddIOp::create(b, loc, p, first); - Value y = - arith::DivUIOp::create(b, loc, source, index(b, width)); - Value x = - arith::RemUIOp::create(b, loc, source, index(b, width)); - auto lanes = scf::ForOp::create(b, loc, zero, lanesEnd, one); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(); - Value channel = arith::AddIOp::create(b, loc, lane, c); - Value value = memref::LoadOp::create( - b, loc, input, ValueRange{zero, y, x, channel}); - memref::StoreOp::create(b, loc, value, pack, - ValueRange{p, lane}); - } - Value a = BankAllocOp::create(b, loc, i64, 1, 1); - a = BankMvinOp::create(b, loc, i64, ones, a, k / 16, 1); - Value w = BankAllocOp::create(b, loc, i64, 1, 1); - w = BankMvinOp::create(b, loc, i64, pack, w, k, 1); - auto sum = - BankCustomOp::create(b, loc, i64, ValueRange{a, w}, acc, - ValueRange{}, "matmul_i8_block", true); - sum->setAttr("rows", b.getI64IntegerAttr(1)); - sum->setAttr("k", b.getI64IntegerAttr(k)); - sum->setAttr("first", b.getBoolAttr(isFirst)); - sum->setAttr("last", b.getBoolAttr(isLast)); - BankReleaseOp::create(b, loc, a); - BankReleaseOp::create(b, loc, w); - return sum.getDestinationOut(); - }); - Value codes = BankAllocOp::create(b, loc, i64, 1, 1); - auto quant = - BankCustomOp::create(b, loc, i64, ValueRange{acc, scaleBank}, codes, - ValueRange{}, "int32_to_int8"); - quant->setAttr("rows", b.getI64IntegerAttr(1)); - quant->setAttr("relu", b.getBoolAttr(false)); - codes = quant.getDestinationOut(); - BankReleaseOp::create(b, loc, acc); - BankMvoutOp::create(b, loc, i64, values, codes, 1, 1); - BankReleaseOp::create(b, loc, codes); - auto lanes = scf::ForOp::create(b, loc, zero, lanesEnd, one); - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(); - Value value = memref::LoadOp::create(b, loc, values, lane); - memref::StoreOp::create( - b, loc, value, output, - ValueRange{zero, zero, zero, - arith::AddIOp::create(b, loc, lane, c)}); - return {}; - }); - BankReleaseOp::create(b, loc, scaleBank); - result = - bufferization::ToTensorOp::create(b, loc, resultType, output, true, true); - return success(); -} - LogicalResult pebble::lowerChannelOp(::buddy::custom::CustomOp op, Value &result) { bool lookup = op.getKind() == "lookup_i8"; @@ -253,14 +132,12 @@ LogicalResult pebble::lowerChannelOp(::buddy::custom::CustomOp op, Value source = BankAllocOp::create(b, loc, i64, 1, 1); source = BankMvinOp::create(b, loc, i64, pack, source, count, 1); Value destination = BankAllocOp::create(b, loc, i64, 1, 1); - auto operation = BankCustomOp::create( - b, loc, i64, - lookup ? ValueRange{source, tableBank} - : ValueRange{gateBank, source}, - destination, lookup ? ValueRange{} : ValueRange{ratio}, - lookup ? "lookup_i8" : "gate_mul_i8"); - operation->setAttr("rows", b.getI64IntegerAttr(count)); - destination = operation.getDestinationOut(); + destination = + lookup ? Value(::buddy::lut_ball::BankLookupI8Op::create( + b, loc, i64, source, tableBank, destination, count)) + : Value(::buddy::int8mul_ball::BankGateMulI8Op::create( + b, loc, i64, gateBank, source, destination, ratio, + count)); BankMvoutOp::create(b, loc, i64, pack, destination, count, 1); BankReleaseOp::create(b, loc, source); BankReleaseOp::create(b, loc, destination); @@ -317,12 +194,9 @@ LogicalResult pebble::lowerTranspose(::buddy::custom::CustomOp op, BankAllocOp::create(b, loc, b.getI64Type(), 1, type.getDimSize(1) / 16); source = BankMvinOp::create(b, loc, b.getI64Type(), input, source, type.getDimSize(0), 1); - auto transpose = - BankCustomOp::create(b, loc, b.getI64Type(), ValueRange{source}, - destination, ValueRange{}, "transpose_i8"); - transpose->setAttr("rows", b.getI64IntegerAttr(type.getDimSize(0))); - transpose->setAttr("cols", b.getI64IntegerAttr(type.getDimSize(1))); - destination = transpose.getDestinationOut(); + destination = ::buddy::transpose_ball::BankTransposeI8Op::create( + b, loc, b.getI64Type(), source, destination, type.getDimSize(0), + type.getDimSize(1)); BankMvoutOp::create(b, loc, b.getI64Type(), output, destination, type.getDimSize(0), 1); BankReleaseOp::create(b, loc, source); diff --git a/examples/designs/pebble/stack/device/core/src/LowerLinear.cpp b/examples/designs/pebble/stack/device/core/src/LowerLinear.cpp new file mode 100644 index 000000000..72080628a --- /dev/null +++ b/examples/designs/pebble/stack/device/core/src/LowerLinear.cpp @@ -0,0 +1,498 @@ +//===- LowerLinear.cpp - Matrix products in bank tiles --------------------===// +// +// C = bias + A @ W over sixteen output columns per matmul_i8. M goes in tiles +// of the matmul rows; K goes in calls of up to six groups of A and B, each +// group one reduction block. The host packs every operand in the bank layout +// of matmul_i8, so a whole-bank DMA fills all groups of a bank at once. +// +//===----------------------------------------------------------------------===// +#include "Buckyball/BuckyballOps.h" +#include "Custom/CustomOps.h" +#include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" +#include "int2fpOps.h" +#include "mataddOps.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "smatmulOps.h" +#include "toint8Ops.h" + +using namespace mlir; +using namespace ::buddy::buckyball; + +pebble::LinearSplit pebble::linearSplit(int64_t k, int64_t rows, + bool floating) { + // The most groups of A (and of its FP32 source) and B in one matmul call; + // the accumulator, a partial, their sum and the scales take four banks + // more. + constexpr int64_t callGroups = 6; + int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; + // A group of A holds rows x k / 16 lines (four times that in FP32), B k. + int64_t maxK = std::min(depth, depth * (floating ? 4 : 16) / rows) / 16 * 16; + int64_t paddedK = (k + 15) / 16 * 16; + int64_t blocks = (paddedK + maxK - 1) / maxK; + int64_t calls = (blocks + callGroups - 1) / callGroups; + int64_t groups = (blocks + calls - 1) / calls; + int64_t blockK = + ((paddedK + calls * groups - 1) / (calls * groups) + 15) / 16 * 16; + return {calls, groups, blockK}; +} + +namespace { + +// output (M x N) = bias + input (M x K) @ weight (K x N), converted with the +// per-column `scales` and `relu` to the element type of `output`: INT32 as +// is, FP32 by int32_to_fp32, INT8 by int32_to_int8. An FP32 input is +// quantized with `multiplier`. A resident input bank of one input row holds +// K in groups of `residentBlock` columns and replaces the input of a single +// call. `bias` has one row, or one row per row of the input. Without +// `output`, the INT8 row stays in the returned bank, one group per sixteen +// columns. +Value linearTiles(OpBuilder &builder, Location loc, Value input, Value resident, + int64_t residentBlock, Value multiplier, Value weight, + Value bias, Value scales, bool relu, Value output) { + auto weightType = cast(weight.getType()); + int64_t k = weightType.getDimSize(0), n = weightType.getDimSize(1); + int64_t m = output ? cast(output.getType()).getDimSize(0) : 1; + bool floating = multiplier != nullptr; + bool perRow = m > 1 && cast(bias.getType()).getDimSize(0) == m; + Type outElement = output ? cast(output.getType()).getElementType() + : builder.getI8Type(); + int64_t rows = m == 1 ? 1 : pebble::tileRows(); + int64_t paddedK = (k + 15) / 16 * 16; + pebble::LinearSplit split = pebble::linearSplit(k, rows, floating); + int64_t calls = split.calls, groups = split.groups, blockK = split.blockK; + if (resident) { + calls = 1; + blockK = residentBlock; + groups = paddedK / residentBlock; + } + int64_t aLines = blockK / 16 * rows * (floating ? 4 : 1); + int64_t aWidth = groups * (floating ? 4 : 16); + int64_t slices = (n + 15) / 16; + Type i64 = builder.getI64Type(); + auto index = [&](OpBuilder &b, int64_t value) -> Value { + return arith::ConstantIndexOp::create(b, loc, value); + }; + auto extent = [&](OpBuilder &b, int64_t size, int64_t limit, + Value start) -> Value { + return arith::MinSIOp::create( + b, loc, index(b, size), + arith::SubIOp::create(b, loc, index(b, limit), start)); + }; + // Nested host loops over [0, ends) with the given body. + auto loops = [&](OpBuilder &b, ArrayRef ends, + function_ref body) { + OpBuilder::InsertionGuard guard(b); + SmallVector indices; + for (Value end : ends) { + auto loop = scf::ForOp::create(b, loc, index(b, 0), end, index(b, 1)); + b.setInsertionPointToStart(loop.getBody()); + indices.push_back(loop.getInductionVar()); + } + body(b, indices); + }; + // The reduction block of column `kk`: call, group and column in the block. + auto block = [&](OpBuilder &b, Value kk) { + Value inBlock = arith::RemUIOp::create(b, loc, kk, index(b, blockK)); + Value blockIndex = arith::DivUIOp::create(b, loc, kk, index(b, blockK)); + Value call = arith::DivUIOp::create(b, loc, blockIndex, index(b, groups)); + Value group = arith::RemUIOp::create(b, loc, blockIndex, index(b, groups)); + return std::make_tuple(call, group, inBlock); + }; + + // B of every output slice and call, packed once: line l of group g holds + // row g * blockK + l of the block. + Value weightPack = memref::AllocOp::create( + builder, loc, + MemRefType::get({slices, calls, blockK, groups * 16}, + builder.getI8Type())); + linalg::FillOp::create( + builder, loc, + ValueRange{arith::ConstantIntOp::create(builder, loc, 0, 8)}, weightPack); + loops(builder, {index(builder, k), index(builder, n)}, + [&](OpBuilder &b, ValueRange at) { + auto [call, group, inBlock] = block(b, at[0]); + Value value = memref::LoadOp::create(b, loc, weight, at); + Value slice = arith::DivUIOp::create(b, loc, at[1], index(b, 16)); + Value lane = arith::AddIOp::create( + b, loc, arith::MulIOp::create(b, loc, group, index(b, 16)), + arith::RemUIOp::create(b, loc, at[1], index(b, 16))); + memref::StoreOp::create(b, loc, value, weightPack, + ValueRange{slice, call, inBlock, lane}); + }); + Value inputPack; + if (!resident) + inputPack = memref::AllocOp::create( + builder, loc, + MemRefType::get({calls, aLines, aWidth}, + floating ? Type(builder.getF32Type()) + : builder.getI8Type())); + Value biasPack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getI32Type())); + Value zeroPack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getI32Type())); + Value tilePack; + if (perRow) + tilePack = memref::AllocOp::create( + builder, loc, MemRefType::get({rows, 16}, builder.getI32Type())); + Value scalePack; + if (scales) + scalePack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getF32Type())); + Value outPack = memref::AllocOp::create( + builder, loc, MemRefType::get({rows, 16}, outElement)); + Value izero = arith::ConstantIntOp::create(builder, loc, 0, 32); + linalg::FillOp::create(builder, loc, izero, zeroPack); + + // A of reduction call `call`, quantized on the core from FP32. + auto loadA = [&](OpBuilder &b, Value call) -> Value { + Value slice = memref::SubViewOp::create( + b, loc, inputPack, + ArrayRef{call, b.getIndexAttr(0), b.getIndexAttr(0)}, + ArrayRef{b.getIndexAttr(1), b.getIndexAttr(aLines), + b.getIndexAttr(aWidth)}, + SmallVector(3, b.getIndexAttr(1))); + Value a = BankAllocOp::create(b, loc, i64, 1, groups); + if (!floating) + return BankMvinOp::create(b, loc, i64, slice, a, aLines, 1); + Value values = BankAllocOp::create(b, loc, i64, 1, groups); + values = BankMvinOp::create(b, loc, i64, slice, values, aLines, 1); + a = ::buddy::toint8_ball::BankFp32ToInt8Op::create(b, loc, i64, values, a, + multiplier, aLines); + BankReleaseOp::create(b, loc, values); + return a; + }; + // Reduction call `call` of output slice `slice` into `destination`, read + // as its bias. `a` is the A of the call, or null to load it. + auto product = [&](OpBuilder &b, Value slice, Value call, Value a, + Value destination) -> Value { + Value loaded = a ? a : loadA(b, call); + Value w = BankAllocOp::create(b, loc, i64, 1, groups); + w = BankMvinOp::create( + b, loc, i64, + memref::SubViewOp::create( + b, loc, weightPack, + ArrayRef{slice, call, b.getIndexAttr(0), + b.getIndexAttr(0)}, + ArrayRef{b.getIndexAttr(1), b.getIndexAttr(1), + b.getIndexAttr(blockK), + b.getIndexAttr(groups * 16)}, + SmallVector(4, b.getIndexAttr(1))), + w, blockK, 1); + Value result = ::buddy::smatmul_ball::BankMatmulI8Op::create( + b, loc, i64, loaded, w, destination, rows, blockK); + if (!a) + BankReleaseOp::create(b, loc, loaded); + BankReleaseOp::create(b, loc, w); + return result; + }; + // sum = acc + partial; add_i32 writes a third bank. + auto add = [&](OpBuilder &b, Value acc, Value partial) -> Value { + Value sum = BankAllocOp::create(b, loc, i64, 1, 1); + sum = ::buddy::matadd_ball::BankAddI32Op::create(b, loc, i64, acc, partial, + sum, rows * 4); + BankReleaseOp::create(b, loc, acc); + BankReleaseOp::create(b, loc, partial); + return sum; + }; + + Value vector; + pebble::repeat( + builder, loc, 0, m, rows, false, {}, + [&](OpBuilder &b, Value row, ValueRange) -> SmallVector { + Value valid = extent(b, rows, m, row); + if (!resident) { + linalg::FillOp::create( + b, loc, + ValueRange{ + floating + ? arith::ConstantOp::create(b, loc, b.getF32FloatAttr(0)) + .getResult() + : arith::ConstantIntOp::create(b, loc, 0, 8).getResult()}, + inputPack); + loops(b, {valid, index(b, k)}, [&](OpBuilder &b, ValueRange at) { + auto [call, group, inBlock] = block(b, at[1]); + Value value = memref::LoadOp::create( + b, loc, input, + ValueRange{arith::AddIOp::create(b, loc, at[0], row), at[1]}); + // Slice t = inBlock / 16 of M row r is line t * rows + r. + Value line = arith::AddIOp::create( + b, loc, + arith::MulIOp::create( + b, loc, + arith::DivUIOp::create(b, loc, inBlock, index(b, 16)), + index(b, rows)), + at[0]); + Value lane = arith::RemUIOp::create(b, loc, inBlock, index(b, 16)); + if (floating) { + line = arith::AddIOp::create( + b, loc, arith::MulIOp::create(b, loc, line, index(b, 4)), + arith::DivUIOp::create(b, loc, lane, index(b, 4))); + lane = arith::RemUIOp::create(b, loc, lane, index(b, 4)); + } + lane = arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, group, + index(b, floating ? 4 : 16)), + lane); + memref::StoreOp::create(b, loc, value, inputPack, + ValueRange{call, line, lane}); + }); + } + // One call keeps A in its bank for every output slice. + Value a = resident; + if (!resident && calls == 1) + a = loadA(b, index(b, 0)); + auto outputSlice = [&](OpBuilder &b, + Value column) -> SmallVector { + Value columns = extent(b, 16, n, column); + Value slice = arith::DivUIOp::create(b, loc, column, index(b, 16)); + linalg::FillOp::create(b, loc, izero, biasPack); + if (!perRow) + loops(b, {columns}, [&](OpBuilder &b, ValueRange at) { + Value value = memref::LoadOp::create( + b, loc, bias, + ValueRange{index(b, 0), + arith::AddIOp::create(b, loc, at[0], column)}); + memref::StoreOp::create(b, loc, value, biasPack, at[0]); + }); + Value scaleBank; + if (scales) { + linalg::FillOp::create(b, loc, + ValueRange{arith::ConstantOp::create( + b, loc, b.getF32FloatAttr(1))}, + scalePack); + loops(b, {columns}, [&](OpBuilder &b, ValueRange at) { + Value value = memref::LoadOp::create( + b, loc, scales, + ValueRange{index(b, 0), + arith::AddIOp::create(b, loc, at[0], column)}); + memref::StoreOp::create(b, loc, value, scalePack, at[0]); + }); + scaleBank = BankAllocOp::create(b, loc, i64, 1, 1); + scaleBank = + BankMvinOp::create(b, loc, i64, scalePack, scaleBank, 4, 1); + } + Value acc = BankAllocOp::create(b, loc, i64, 1, 1); + acc = BankMvinOp::create(b, loc, i64, biasPack, acc, 4, 1); + acc = product(b, slice, index(b, 0), a, acc); + if (rows == 1) { + // One row accumulates in place: the old C is the bias. + acc = + pebble::repeat(b, loc, 1, calls, 1, false, {acc}, + [&](OpBuilder &b, Value call, ValueRange args) { + return SmallVector{ + product(b, slice, call, Value(), args[0])}; + })[0]; + } else { + for (int64_t call = 1; call < calls; ++call) { + Value partial = BankAllocOp::create(b, loc, i64, 1, 1); + partial = + BankMvinOp::create(b, loc, i64, zeroPack, partial, 4, 1); + acc = add(b, acc, + product(b, slice, index(b, call), Value(), partial)); + } + } + if (perRow) { + linalg::FillOp::create(b, loc, izero, tilePack); + loops(b, {valid, columns}, [&](OpBuilder &b, ValueRange at) { + Value value = memref::LoadOp::create( + b, loc, bias, + ValueRange{arith::AddIOp::create(b, loc, at[0], row), + arith::AddIOp::create(b, loc, at[1], column)}); + memref::StoreOp::create(b, loc, value, tilePack, at); + }); + Value initial = BankAllocOp::create(b, loc, i64, 1, 1); + initial = + BankMvinOp::create(b, loc, i64, tilePack, initial, rows * 4, 1); + acc = add(b, acc, initial); + } + Value restored = acc; + if (scales) { + restored = BankAllocOp::create(b, loc, i64, 1, 1); + restored = + outElement.isF32() + ? Value(::buddy::int2fp_ball::BankInt32ToFp32Op::create( + b, loc, i64, acc, scaleBank, restored, rows, relu)) + : Value(::buddy::toint8_ball::BankInt32ToInt8Op::create( + b, loc, i64, acc, scaleBank, restored, rows, relu)); + BankReleaseOp::create(b, loc, acc); + BankReleaseOp::create(b, loc, scaleBank); + } + if (!output) + return {restored}; + restored = + BankMvoutOp::create(b, loc, i64, outPack, restored, + outElement.isInteger(8) ? rows : rows * 4, 1); + BankReleaseOp::create(b, loc, restored); + loops(b, {valid, columns}, [&](OpBuilder &b, ValueRange at) { + Value value = memref::LoadOp::create(b, loc, outPack, at); + memref::StoreOp::create( + b, loc, value, output, + ValueRange{arith::AddIOp::create(b, loc, at[0], row), + arith::AddIOp::create(b, loc, at[1], column)}); + }); + return {}; + }; + if (output) { + pebble::repeat(b, loc, 0, n, 16, false, {}, + [&](OpBuilder &b, Value column, ValueRange) { + return outputSlice(b, column); + }); + } else { + // Each output slice appends its group to the resident row. + for (int64_t column = 0; column < n; column += 16) { + Value slice = outputSlice(b, index(b, column))[0]; + vector = vector ? BankTransferOp::create(b, loc, i64, slice, vector) + .getResult() + : slice; + } + } + if (a) + BankReleaseOp::create(b, loc, a); + return {}; + }); + return vector; +} +} // namespace + +LogicalResult pebble::lowerLinear(::buddy::custom::CustomOp op, + Value residentInput, Value &result, + Value &resident) { + StringRef kind = op.getKind(); + bool integer = kind == "matmul_i8", requant = kind == "linear_i8_requant"; + // A fused linear_i8 requantizes with the multiplier of its consumer. + ValueRange args = op.getInputs(); + bool fused = kind == "linear_i8" && args.size() == 7; + if ((!fused && args.size() != (integer ? 3 + : requant ? 4 + : 6)) || + op.getNumResults() != 1) + return op.emitOpError("has the wrong number of inputs or results for ") + << kind; + auto a = cast(args[0].getType()); + auto w = cast(args[1].getType()); + auto biasType = cast(args[2].getType()); + auto resultType = cast(op.getResult(0).getType()); + int64_t m = resultType.getDimSize(0), n = w.getDimSize(1); + bool codes = integer || requant || fused; + if (!a.hasStaticShape() || !w.hasStaticShape() || a.getRank() != 2 || + w.getRank() != 2 || a.getDimSize(1) != w.getDimSize(0) || + a.getDimSize(0) != m || !w.getElementType().isInteger(8) || + !biasType.getElementType().isInteger(32) || + biasType.getShape() != ArrayRef({integer ? m : 1, n}) || + resultType.getShape() != ArrayRef({m, n}) || + !(integer ? resultType.getElementType().isInteger(32) + : codes ? resultType.getElementType().isInteger(8) + : resultType.getElementType().isF32()) || + ((integer || requant) && !a.getElementType().isInteger(8))) + return op.emitOpError("requires matching MxK/KxN tensors and INT32 bias " + "for ") + << kind; + OpBuilder b(op); + Location loc = op.getLoc(); + Value zero = arith::ConstantIndexOp::create(b, loc, 0); + Value input, multiplier, scales; + if (!residentInput) + input = tensorBuffer(b, loc, args[0]); + if (!integer && !requant && a.getElementType().isF32()) + multiplier = + tensor::ExtractOp::create(b, loc, args[3], ValueRange{zero, zero}); + if (requant) { + scales = tensorBuffer(b, loc, args[3]); + } else if (!integer) { + // input_step * weight_step per column restores FP32 values; a fused + // Linear scales them to the codes of its consumer. + Value step = + tensor::ExtractOp::create(b, loc, args[4], ValueRange{zero, zero}); + if (fused) + step = arith::MulFOp::create( + b, loc, step, + tensor::ExtractOp::create(b, loc, args[6], ValueRange{zero, zero})); + Value weightSteps = tensorBuffer(b, loc, args[5]); + scales = memref::AllocOp::create(b, loc, + MemRefType::get({1, n}, b.getF32Type())); + auto loop = scf::ForOp::create(b, loc, zero, + arith::ConstantIndexOp::create(b, loc, n), + arith::ConstantIndexOp::create(b, loc, 1)); + OpBuilder::InsertionGuard guard(b); + b.setInsertionPointToStart(loop.getBody()); + Value c = loop.getInductionVar(); + Value product = arith::MulFOp::create( + b, loc, step, + memref::LoadOp::create(b, loc, weightSteps, ValueRange{zero, c})); + memref::StoreOp::create(b, loc, product, scales, ValueRange{zero, c}); + } + // A flattened map is one block of K; a resident Linear row has a group per + // sixteen columns. + auto producer = args[0].getDefiningOp<::buddy::custom::CustomOp>(); + int64_t residentBlock = + producer && producer.getKind() == "flatten_nchw" ? a.getDimSize(1) : 16; + bool relu = + (requant || fused) && cast(op->getAttr("relu")).getValue(); + Value output; + if (!op->hasAttr("custom.resident")) + output = memref::AllocOp::create( + b, loc, MemRefType::get({m, n}, resultType.getElementType())); + resident = linearTiles(b, loc, input, residentInput, residentBlock, + multiplier, tensorBuffer(b, loc, args[1]), + tensorBuffer(b, loc, args[2]), scales, relu, output); + if (output) + result = bufferization::ToTensorOp::create(b, loc, resultType, output, true, + true); + return success(); +} + +// The mean of every channel: ones (1 x pixels) @ input (pixels x channels), +// requantized with input_scale / (pixels * output_scale). +LogicalResult pebble::lowerGlobalAverage(::buddy::custom::CustomOp op, + Value &result) { + auto type = cast(op.getInputs()[0].getType()); + auto resultType = cast(op.getResult(0).getType()); + if (!type.hasStaticShape() || type.getRank() != 4 || + type.getDimSize(0) != 1 || !type.getElementType().isInteger(8) || + !resultType.getElementType().isInteger(8) || + resultType.getShape() != ArrayRef({1, 1, 1, type.getDimSize(3)})) + return op.emitOpError( + "global_avg_i8 requires NHWC INT8 input and 1x1x1xC INT8 output"); + int64_t pixels = type.getDimSize(1) * type.getDimSize(2), + channels = type.getDimSize(3); + float ratio = + cast(op->getAttr("input_scale")).getValueAsDouble() / + (pixels * + cast(op->getAttr("output_scale")).getValueAsDouble()); + OpBuilder b(op); + Location loc = op.getLoc(); + Value ones = memref::AllocOp::create( + b, loc, MemRefType::get({1, pixels}, b.getI8Type())); + linalg::FillOp::create( + b, loc, ValueRange{arith::ConstantIntOp::create(b, loc, 1, 8)}, ones); + Value bias = memref::AllocOp::create( + b, loc, MemRefType::get({1, channels}, b.getI32Type())); + linalg::FillOp::create( + b, loc, ValueRange{arith::ConstantIntOp::create(b, loc, 0, 32)}, bias); + Value scales = memref::AllocOp::create( + b, loc, MemRefType::get({1, channels}, b.getF32Type())); + linalg::FillOp::create( + b, loc, + ValueRange{arith::ConstantOp::create(b, loc, b.getF32FloatAttr(ratio))}, + scales); + Value weight = memref::CollapseShapeOp::create( + b, loc, tensorBuffer(b, loc, op.getInputs()[0]), + ArrayRef{{0, 1, 2}, {3}}); + Value output = memref::AllocOp::create( + b, loc, MemRefType::get({1, channels}, b.getI8Type())); + linearTiles(b, loc, ones, Value(), 0, Value(), weight, bias, scales, false, + output); + Value flat = bufferization::ToTensorOp::create( + b, loc, RankedTensorType::get({1, channels}, b.getI8Type()), output, true, + true); + result = tensor::ExpandShapeOp::create( + b, loc, resultType, flat, ArrayRef{{0, 1, 2}, {3}}); + return success(); +} diff --git a/examples/designs/pebble/stack/device/core/src/LowerSpatialOps.cpp b/examples/designs/pebble/stack/device/core/src/LowerSpatialOps.cpp new file mode 100644 index 000000000..82e8fb6f7 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/src/LowerSpatialOps.cpp @@ -0,0 +1,521 @@ +//===- LowerSpatialOps.cpp - Convolutions in bank tiles -------------------===// +// +// A convolution computes sixteen output channels per matmul_i8 over a tile of +// output pixels. The input tile, sixteen channels per group, goes through +// im2col_i8 into the A operand; the host packs the weights in the same K +// order into B. Input channels beyond one call form chunks whose partial +// products add up with add_i32. A fused convolution requantizes for its +// consumer and pools; when the consumer reads the whole result, the result +// stays in banks, one group per channel block. +// +//===----------------------------------------------------------------------===// +#include "Buckyball/BuckyballOps.h" +#include "Custom/CustomOps.h" +#include "Lowering.h" +#include "Target/BuckyballTargetRegistry.h" +#include "im2colOps.h" +#include "int2fpOps.h" +#include "mataddOps.h" +#include "maxpoolOps.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "smatmulOps.h" +#include "toint8Ops.h" +#include "transposeOps.h" + +using namespace mlir; +using namespace ::buddy::buckyball; +using Computation = ::buddy::custom::CustomOp; + +namespace { +// Lowers `op`. A resident input is a bank with one group per input channel +// block holding the whole input map. With custom.resident the INT8 result is +// returned in `image`, one group per output channel block; otherwise in +// `tensorOutput`. +LogicalResult convolution(Computation op, Value residentInput, Value &image, + Value &tensorOutput) { + bool depthwise = op.getKind() == "depthwise_conv2d_i8"; + bool fused = op.getInputs().size() == 7; + bool resident = op->hasAttr("custom.resident"); + auto args = op.getInputs(); + auto inputType = cast(args[0].getType()); + auto weightType = cast(args[1].getType()); + auto outputType = cast(op.getResult(0).getType()); + auto strides = cast(op->getAttr("strides")).asArrayRef(); + auto padding = cast(op->getAttr("padding")).asArrayRef(); + if (args.size() < 6 || !inputType.hasStaticShape() || + inputType.getRank() != 4 || !weightType.hasStaticShape() || + weightType.getRank() != 4 || inputType.getDimSize(0) != 1 || + !weightType.getElementType().isInteger(8) || + !cast(args[2].getType()).getElementType().isInteger(32)) + return op.emitOpError("requires static NHWC/FCHW convolution with INT32 " + "bias"); + bool floating = inputType.getElementType().isF32(); + int64_t pool = fused ? cast(op->getAttr("pool")).getInt() : 1; + bool relu = fused && cast(op->getAttr("relu")).getValue(); + int64_t cin = inputType.getDimSize(3), cout = weightType.getDimSize(0); + int64_t kernel = weightType.getDimSize(2), stride = strides[0]; + int64_t height = outputType.getDimSize(1) * pool, + width = outputType.getDimSize(2) * pool; + int64_t blocksIn = (cin + 15) / 16; + // The tile, the input tile around it and the channel blocks per chunk. A + // resident input is one tile of the whole map that im2col pads itself. + int64_t side, inputSide, chunk, im2colPadding; + if (residentInput) { + side = height; + inputSide = inputType.getDimSize(1); + chunk = blocksIn; + im2colPadding = padding[0]; + } else { + pebble::ConvTiling tiling = + pebble::convTiling(height / pool, width / pool, pool, stride, kernel, + blocksIn, depthwise, floating); + side = tiling.side; + inputSide = tiling.inputSide; + chunk = tiling.chunk; + im2colPadding = 0; + } + int64_t rows = (side * side + pebble::tileRows() - 1) / pebble::tileRows() * + pebble::tileRows(); + int64_t pooled = side / pool; + // Chunks of input channel blocks: [start, start + blocks). Depthwise reads + // the block of its output channels. + SmallVector> chunks; + for (int64_t start = 0; start < (depthwise ? 1 : blocksIn); start += chunk) + chunks.push_back({start, std::min(chunk, blocksIn - start)}); + + OpBuilder builder(op); + Location loc = op.getLoc(); + Type i64 = builder.getI64Type(); + auto index = [&](OpBuilder &b, int64_t n) -> Value { + return arith::ConstantIndexOp::create(b, loc, n); + }; + auto extent = [&](OpBuilder &b, int64_t size, int64_t limit, + Value start) -> Value { + return arith::MinSIOp::create( + b, loc, index(b, size), + arith::SubIOp::create(b, loc, index(b, limit), start)); + }; + auto loops = [&](OpBuilder &b, ArrayRef ends, + function_ref body) { + OpBuilder::InsertionGuard guard(b); + SmallVector indices; + for (Value end : ends) { + auto loop = scf::ForOp::create(b, loc, index(b, 0), end, index(b, 1)); + b.setInsertionPointToStart(loop.getBody()); + indices.push_back(loop.getInductionVar()); + } + body(b, indices); + }; + Value zero = index(builder, 0); + Value input; + if (!residentInput) + input = pebble::tensorBuffer(builder, loc, args[0]); + Value weight = pebble::tensorBuffer(builder, loc, args[1]); + Value bias = pebble::tensorBuffer(builder, loc, args[2]); + Value weightSteps = pebble::tensorBuffer(builder, loc, args[5]); + Value step = tensor::ExtractOp::create(builder, loc, args[4], + ValueRange{zero, zero, zero, zero}); + Value inputMultiplier, outputMultiplier; + if (floating) + inputMultiplier = tensor::ExtractOp::create( + builder, loc, args[3], ValueRange{zero, zero, zero, zero}); + if (fused) { + SmallVector indices( + cast(args[6].getType()).getRank(), zero); + outputMultiplier = + tensor::ExtractOp::create(builder, loc, args[6], indices); + } + Value output; + if (!resident) + output = memref::AllocOp::create( + builder, loc, + MemRefType::get(outputType.getShape(), outputType.getElementType())); + // Host packs: per chunk the weights of one output block in the bank layout + // of B, the input tile with sixteen channels per group, the bias row, the + // scales and one tile of results. + SmallVector groups; + SmallVector weightPacks, inputPacks; + for (auto [start, blocks] : chunks) { + int64_t outputGroups = pebble::im2colGroups(kernel, blocks, side * side); + int64_t tiles = kernel * kernel * blocks / outputGroups; + groups.push_back(outputGroups); + weightPacks.push_back(memref::AllocOp::create( + builder, loc, + MemRefType::get({16 * tiles, outputGroups * 16}, builder.getI8Type()))); + if (!residentInput) + inputPacks.push_back(memref::AllocOp::create( + builder, loc, + floating ? MemRefType::get({inputSide * inputSide * 4, blocks * 4}, + builder.getF32Type()) + : MemRefType::get({inputSide * inputSide, blocks * 16}, + builder.getI8Type()))); + } + Value biasPack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getI32Type())); + Value zeroPack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getI32Type())); + Value scalePack = memref::AllocOp::create( + builder, loc, MemRefType::get({16}, builder.getF32Type())); + Value outPack; + if (!resident) + outPack = memref::AllocOp::create( + builder, loc, + MemRefType::get({rows, 16}, fused ? Type(builder.getI8Type()) + : builder.getF32Type())); + Value izero = arith::ConstantIntOp::create(builder, loc, 0, 32); + Value byteZero = arith::ConstantIntOp::create(builder, loc, 0, 8); + linalg::FillOp::create(builder, loc, izero, zeroPack); + + // One output block of sixteen channels at `oc`; returns the INT8 result + // bank of a resident convolution. + auto outputBlock = [&](OpBuilder &b, Value oc) -> Value { + Value lanes = extent(b, 16, cout, oc); + // B: line l of group j is K tile j * tiles + l / 16, input channel l % 16 + // of the tile; tile t is kernel position t / blocks, block t % blocks. + for (auto [index_, chunkRange] : llvm::enumerate(chunks)) { + int64_t start = chunkRange.first, blocks = chunkRange.second; + int64_t tiles = kernel * kernel * blocks / groups[index_]; + Value pack = weightPacks[index_]; + linalg::FillOp::create(b, loc, byteZero, pack); + loops( + b, {lanes, index(b, 16 * tiles * groups[index_])}, + [&](OpBuilder &b, ValueRange at) { + Value lane = at[0], row = at[1]; + Value tile = arith::DivUIOp::create(b, loc, row, index(b, 16)); + Value channel = arith::RemUIOp::create(b, loc, row, index(b, 16)); + Value position = + arith::DivUIOp::create(b, loc, tile, index(b, blocks)); + Value blockIndex = + arith::RemUIOp::create(b, loc, tile, index(b, blocks)); + Value ky = + arith::DivUIOp::create(b, loc, position, index(b, kernel)); + Value kx = + arith::RemUIOp::create(b, loc, position, index(b, kernel)); + Value o = arith::AddIOp::create(b, loc, oc, lane); + Value inputChannel; + if (depthwise) + inputChannel = arith::AddIOp::create(b, loc, oc, channel); + else + inputChannel = arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, + arith::AddIOp::create(b, loc, + blockIndex, + index(b, start)), + index(b, 16)), + channel); + Value present = + depthwise + ? arith::CmpIOp::create(b, loc, arith::CmpIPredicate::eq, + channel, lane) + .getResult() + : arith::CmpIOp::create(b, loc, arith::CmpIPredicate::ult, + inputChannel, index(b, cin)) + .getResult(); + auto valid = scf::IfOp::create(b, loc, present, false); + b.setInsertionPointToStart(&valid.getThenRegion().front()); + Value value = memref::LoadOp::create( + b, loc, weight, + ValueRange{o, depthwise ? index(b, 0) : inputChannel, ky, kx}); + Value line = + arith::RemUIOp::create(b, loc, row, index(b, 16 * tiles)); + Value group = + arith::DivUIOp::create(b, loc, row, index(b, 16 * tiles)); + memref::StoreOp::create( + b, loc, value, pack, + ValueRange{line, arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, group, + index(b, 16)), + lane)}); + }); + } + linalg::FillOp::create(b, loc, izero, biasPack); + linalg::FillOp::create( + b, loc, + ValueRange{arith::ConstantOp::create(b, loc, b.getF32FloatAttr(1))}, + scalePack); + loops(b, {lanes}, [&](OpBuilder &b, ValueRange at) { + Value o = arith::AddIOp::create(b, loc, oc, at[0]); + memref::StoreOp::create(b, loc, memref::LoadOp::create(b, loc, bias, o), + biasPack, at[0]); + Value scale = arith::MulFOp::create( + b, loc, step, + memref::LoadOp::create( + b, loc, weightSteps, + ValueRange{o, index(b, 0), index(b, 0), index(b, 0)})); + if (fused) + scale = arith::MulFOp::create(b, loc, scale, outputMultiplier); + memref::StoreOp::create(b, loc, scale, scalePack, at[0]); + }); + Value scales = BankAllocOp::create(b, loc, i64, 1, 1); + scales = BankMvinOp::create(b, loc, i64, scalePack, scales, 4, 1); + Value result; + pebble::repeat( + b, loc, 0, height, side, false, {}, + [&](OpBuilder &b, Value y, ValueRange) -> SmallVector { + pebble::repeat( + b, loc, 0, width, side, false, {}, + [&](OpBuilder &b, Value x, ValueRange) -> SmallVector { + Value acc; + for (auto [index_, chunkRange] : llvm::enumerate(chunks)) { + int64_t start = chunkRange.first, blocks = chunkRange.second; + int64_t outputGroups = groups[index_]; + int64_t tiles = kernel * kernel * blocks / outputGroups; + Value source = residentInput; + if (!residentInput) { + // The input tile of this chunk; the halo outside the + // input stays zero. + Value pack = inputPacks[index_]; + Value first = + depthwise + ? arith::DivUIOp::create(b, loc, oc, index(b, 16)) + .getResult() + : index(b, start); + linalg::FillOp::create( + b, loc, + ValueRange{floating ? arith::ConstantOp::create( + b, loc, b.getF32FloatAttr(0)) + .getResult() + : byteZero}, + pack); + loops( + b, + {index(b, inputSide), index(b, inputSide), + index(b, blocks * 16)}, + [&](OpBuilder &b, ValueRange at) { + Value iy = arith::SubIOp::create( + b, loc, + arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, y, + index(b, stride)), + at[0]), + index(b, padding[0])); + Value ix = arith::SubIOp::create( + b, loc, + arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, x, + index(b, stride)), + at[1]), + index(b, padding[2])); + Value channel = arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, first, + index(b, 16)), + at[2]); + auto inRange = [&](Value value, int64_t limit) { + return arith::AndIOp::create( + b, loc, + arith::CmpIOp::create(b, loc, + arith::CmpIPredicate::sge, + value, index(b, 0)), + arith::CmpIOp::create(b, loc, + arith::CmpIPredicate::slt, + value, index(b, limit))); + }; + Value inside = arith::AndIOp::create( + b, loc, + arith::AndIOp::create( + b, loc, inRange(iy, inputType.getDimSize(1)), + inRange(ix, inputType.getDimSize(2))), + inRange(channel, cin)); + auto valid = scf::IfOp::create(b, loc, inside, false); + b.setInsertionPointToStart( + &valid.getThenRegion().front()); + Value value = memref::LoadOp::create( + b, loc, input, + ValueRange{index(b, 0), iy, ix, channel}); + Value pixel = arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, at[0], + index(b, inputSide)), + at[1]); + // FP32: line 4 * pixel + c / 4 of group c / 16. + Value line = pixel, lane = at[2]; + if (floating) { + Value block = arith::DivUIOp::create(b, loc, at[2], + index(b, 16)); + Value inBlock = arith::RemUIOp::create( + b, loc, at[2], index(b, 16)); + line = arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, pixel, + index(b, 4)), + arith::DivUIOp::create(b, loc, inBlock, + index(b, 4))); + lane = arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, block, + index(b, 4)), + arith::RemUIOp::create(b, loc, inBlock, + index(b, 4))); + } + memref::StoreOp::create(b, loc, value, pack, + ValueRange{line, lane}); + }); + int64_t pixels = inputSide * inputSide; + source = BankAllocOp::create(b, loc, i64, 1, blocks); + if (floating) { + Value values = + BankAllocOp::create(b, loc, i64, 1, blocks); + values = BankMvinOp::create(b, loc, i64, pack, values, + pixels * 4, 1); + source = ::buddy::toint8_ball::BankFp32ToInt8Op::create( + b, loc, i64, values, source, inputMultiplier, + pixels * 4); + BankReleaseOp::create(b, loc, values); + } else { + source = BankMvinOp::create(b, loc, i64, pack, source, + pixels, 1); + } + } + Value a = BankAllocOp::create(b, loc, i64, 1, outputGroups); + a = ::buddy::im2col_ball::BankIm2colI8Op::create( + b, loc, i64, source, a, inputSide, kernel, stride, + im2colPadding, pebble::tileRows()); + if (!residentInput) + BankReleaseOp::create(b, loc, source); + Value w = BankAllocOp::create(b, loc, i64, 1, outputGroups); + w = BankMvinOp::create(b, loc, i64, weightPacks[index_], w, + 16 * tiles, 1); + // The first chunk starts from the bias, the others from + // zero and add to the accumulator. + Value partial = BankAllocOp::create(b, loc, i64, 1, 1); + partial = BankMvinOp::create( + b, loc, i64, acc ? zeroPack : biasPack, partial, 4, 1); + partial = ::buddy::smatmul_ball::BankMatmulI8Op::create( + b, loc, i64, a, w, partial, rows, 16 * tiles); + BankReleaseOp::create(b, loc, a); + BankReleaseOp::create(b, loc, w); + if (!acc) { + acc = partial; + continue; + } + Value sum = BankAllocOp::create(b, loc, i64, 1, 1); + sum = ::buddy::matadd_ball::BankAddI32Op::create( + b, loc, i64, acc, partial, sum, rows * 4); + BankReleaseOp::create(b, loc, acc); + BankReleaseOp::create(b, loc, partial); + acc = sum; + } + Value converted = BankAllocOp::create(b, loc, i64, 1, 1); + converted = + fused + ? Value(::buddy::toint8_ball::BankInt32ToInt8Op::create( + b, loc, i64, acc, scales, converted, rows, relu)) + : Value(::buddy::int2fp_ball::BankInt32ToFp32Op::create( + b, loc, i64, acc, scales, converted, rows, relu)); + BankReleaseOp::create(b, loc, acc); + if (pool > 1) { + Value pooledBank = BankAllocOp::create(b, loc, i64, 1, 1); + pooledBank = ::buddy::maxpool_ball::BankMaxpoolI8Op::create( + b, loc, i64, converted, pooledBank, side, pool, pool, 0); + BankReleaseOp::create(b, loc, converted); + converted = pooledBank; + } + if (resident) { + result = converted; + return {}; + } + converted = BankMvoutOp::create( + b, loc, i64, outPack, converted, + fused ? (pool > 1 ? pooled * pooled : rows) : rows * 4, 1); + BankReleaseOp::create(b, loc, converted); + // The pixels of the tile inside the output. + Value py = arith::DivUIOp::create(b, loc, y, index(b, pool)); + Value px = arith::DivUIOp::create(b, loc, x, index(b, pool)); + loops( + b, + {extent(b, pooled, height / pool, py), + extent(b, pooled, width / pool, px), lanes}, + [&](OpBuilder &b, ValueRange at) { + Value pixel = arith::AddIOp::create( + b, loc, + arith::MulIOp::create(b, loc, at[0], + index(b, pooled)), + at[1]); + Value value = memref::LoadOp::create( + b, loc, outPack, ValueRange{pixel, at[2]}); + memref::StoreOp::create( + b, loc, value, output, + ValueRange{index(b, 0), + arith::AddIOp::create(b, loc, at[0], py), + arith::AddIOp::create(b, loc, at[1], px), + arith::AddIOp::create(b, loc, at[2], oc)}); + }); + return {}; + }); + return {}; + }); + BankReleaseOp::create(b, loc, scales); + return result; + }; + + if (resident) { + // Output blocks append their groups to one bank. + for (int64_t oc = 0; oc < cout; oc += 16) { + Value block = outputBlock(builder, index(builder, oc)); + image = image ? BankTransferOp::create(builder, loc, i64, block, image) + .getResult() + : block; + } + } else { + pebble::repeat(builder, loc, 0, cout, 16, false, {}, + [&](OpBuilder &b, Value oc, ValueRange) { + outputBlock(b, oc); + return SmallVector{}; + }); + tensorOutput = bufferization::ToTensorOp::create(builder, loc, outputType, + output, true, true); + } + if (residentInput) + BankReleaseOp::create(builder, loc, residentInput); + return success(); +} +} // namespace + +LogicalResult pebble::lowerSpatialOps(func::FuncOp function, + DenseMap &residentVectors, + SmallVector &consumed) { + SmallVector ops; + function.walk([&](Computation op) { + if (op.getKind() == "conv2d_i8" || op.getKind() == "depthwise_conv2d_i8" || + op.getKind() == "flatten_nchw") + ops.push_back(op); + }); + DenseMap images; + for (auto op : ops) { + Value input = images.lookup(op.getInputs()[0]); + if (op.getKind() == "flatten_nchw") { + // A 4x4 map of sixteen channels in channel-major order: the transposed + // 16x16 block, one K block of sixteen rows for a one-row matmul. + OpBuilder b(op); + Value vector = BankAllocOp::create(b, op.getLoc(), b.getI64Type(), 1, 1); + vector = ::buddy::transpose_ball::BankTransposeI8Op::create( + b, op.getLoc(), b.getI64Type(), input, vector, 16, 16); + BankReleaseOp::create(b, op.getLoc(), input); + residentVectors[op.getResult(0)] = vector; + consumed.push_back(op); + continue; + } + Value image, tensor; + if (failed(convolution(op, input, image, tensor))) + return failure(); + if (image) { + images[op.getResult(0)] = image; + consumed.push_back(op); + pebble::observeResident(op, image); + } else { + op.getResult(0).replaceAllUsesWith(tensor); + op.erase(); + } + } + return success(); +} diff --git a/examples/designs/pebble/stack/device/core/src/Lowering.h b/examples/designs/pebble/stack/device/core/src/Lowering.h new file mode 100644 index 000000000..ebdb49a11 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/src/Lowering.h @@ -0,0 +1,81 @@ +//===- Lowering.h - Custom computations in the Pebble midend -------------===// +#ifndef PEBBLE_LOWERING_H +#define PEBBLE_LOWERING_H + +#include "Custom/CustomOps.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/IR/Builders.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLFunctionalExtras.h" + +namespace mlir::linalg { +class GenericOp; +} +namespace pebble { +bool elementwise(mlir::linalg::GenericOp op, llvm::StringRef name); +mlir::Value tensorBuffer(mlir::OpBuilder &builder, mlir::Location loc, + mlir::Value tensor); + +// Emits `body` for index = begin, begin + step, ... < end and threads the +// carried banks through it. The iterations share one scf.for unless `unroll` +// is set: then each iteration gets a constant index, for bodies whose index +// selects a bank. +llvm::SmallVector +repeat(mlir::OpBuilder &builder, mlir::Location loc, int64_t begin, int64_t end, + int64_t step, bool unroll, mlir::ValueRange carried, + llvm::function_ref( + mlir::OpBuilder &, mlir::Value, mlir::ValueRange)> + body); + +// The bank capabilities the core tiles for: a matmul_i8 takes at most +// matmulRows() rows, im2col_i8 output feeds it in M tiles of tileRows(). +int64_t matmulRows(); +int64_t tileRows(); + +// Moves the cycle markers of `op`, whose result stays in `bank`, to the bank. +void observeResident(mlir::Operation *op, mlir::Value bank); + +// The groups of the im2col_i8 output (and of the matching weights) for +// `windows` windows of a `kernel` x `kernel` convolution over `blocks` +// channel blocks: the fewest that divide the K tiles and fit a group each. +// Zero when they exceed the banks a convolution may use. +int64_t im2colGroups(int64_t kernel, int64_t blocks, int64_t windows); + +// A convolution computes `side` x `side` output pixels (a multiple of the +// pool window) per matmul from an input tile of `inputSide` pixels per side, +// and reads the input channels in chunks of `chunk` channel blocks. An FP32 +// input tile takes four lines per pixel. `side` is zero when no tile fits. +struct ConvTiling { + int64_t side, inputSide, chunk; +}; +ConvTiling convTiling(int64_t height, int64_t width, int64_t pool, + int64_t stride, int64_t kernel, int64_t blocks, + bool depthwise, bool floating); + +// A Linear reduces K in `calls` matmul calls of `groups` groups of A and B, +// `blockK` columns each, over `rows` rows per tile. +struct LinearSplit { + int64_t calls, groups, blockK; +}; +LinearSplit linearSplit(int64_t k, int64_t rows, bool floating); + +// The result of a Linear with custom.resident stays in `resident`, one +// group per sixteen columns; the others give `result`. +mlir::LogicalResult lowerLinear(::buddy::custom::CustomOp op, + mlir::Value residentInput, mlir::Value &result, + mlir::Value &resident); +mlir::LogicalResult lowerGlobalAverage(::buddy::custom::CustomOp op, + mlir::Value &result); +mlir::LogicalResult lowerChannelOp(::buddy::custom::CustomOp op, + mlir::Value &result); +mlir::LogicalResult lowerTranspose(::buddy::custom::CustomOp op, + mlir::Value &result); +void formSpatialOps(mlir::func::FuncOp function); +// Lowers the convolutions; the ones whose results stay in banks remain in +// `consumed` until their consumers are lowered. +mlir::LogicalResult +lowerSpatialOps(mlir::func::FuncOp function, + llvm::DenseMap &residentVectors, + llvm::SmallVector &consumed); +} // namespace pebble +#endif diff --git a/examples/designs/pebble/stack/midend/tests/form-custom-ops.mlir b/examples/designs/pebble/stack/device/core/tests/form-custom-ops.mlir similarity index 100% rename from examples/designs/pebble/stack/midend/tests/form-custom-ops.mlir rename to examples/designs/pebble/stack/device/core/tests/form-custom-ops.mlir diff --git a/examples/designs/pebble/stack/midend/tests/form-depthwise.mlir b/examples/designs/pebble/stack/device/core/tests/form-depthwise.mlir similarity index 100% rename from examples/designs/pebble/stack/midend/tests/form-depthwise.mlir rename to examples/designs/pebble/stack/device/core/tests/form-depthwise.mlir diff --git a/examples/designs/pebble/stack/midend/tests/form-int32-bias.mlir b/examples/designs/pebble/stack/device/core/tests/form-int32-bias.mlir similarity index 100% rename from examples/designs/pebble/stack/midend/tests/form-int32-bias.mlir rename to examples/designs/pebble/stack/device/core/tests/form-int32-bias.mlir diff --git a/examples/designs/pebble/stack/device/core/tests/lower-bank-updates.mlir b/examples/designs/pebble/stack/device/core/tests/lower-bank-updates.mlir new file mode 100644 index 000000000..f97c826a3 --- /dev/null +++ b/examples/designs/pebble/stack/device/core/tests/lower-bank-updates.mlir @@ -0,0 +1,7 @@ +// CHECK-LABEL: func.func @define_quantized_rows +// CHECK: arith.constant 4298115073 : i64 +// CHECK: return +func.func @define_quantized_rows() { + toint8_ball.int32_to_int8 <{values = 1 : i64, scales = 2 : i64, output = 3 : i64, rows = 1 : i64}> + return +} diff --git a/examples/designs/pebble/stack/midend/tests/lower-custom-ops.mlir b/examples/designs/pebble/stack/device/core/tests/lower-custom-ops.mlir similarity index 70% rename from examples/designs/pebble/stack/midend/tests/lower-custom-ops.mlir rename to examples/designs/pebble/stack/device/core/tests/lower-custom-ops.mlir index 5622df7b8..db9942734 100644 --- a/examples/designs/pebble/stack/midend/tests/lower-custom-ops.mlir +++ b/examples/designs/pebble/stack/device/core/tests/lower-custom-ops.mlir @@ -1,11 +1,10 @@ // CHECK-LABEL: func.func @named +// CHECK-NOT: fp32_to_int8 // CHECK: buckyball.bank_mvin -// CHECK-NOT: "fp32_to_int8" -// CHECK: scf.for {{.*}} to %c32 step %c16 -// CHECK: buckyball.bank_custom "matmul_i8" +// CHECK: scf.for {{.*}} to %c32{{.*}} step %c16 +// CHECK: smatmul_ball.bank_matmul_i8 {{.*}}rows = 1, k = 32> // CHECK: buckyball.bank_mvout -// CHECK: } -// CHECK-NOT: buckyball.bank_custom +// CHECK-NOT: _ball.bank_ // CHECK: return func.func @named(%a: tensor<1x32xi8>, %b: tensor<32x32xi8>, %initial: tensor<1x32xi32>) -> tensor<1x32xi32> { %product = custom.op "matmul_i8"(%a, %b, %initial) : (tensor<1x32xi8>, tensor<32x32xi8>, tensor<1x32xi32>) -> tensor<1x32xi32> @@ -13,12 +12,12 @@ func.func @named(%a: tensor<1x32xi8>, %b: tensor<32x32xi8>, %initial: tensor<1x3 } // CHECK-LABEL: func.func @linear -// CHECK: buckyball.bank_custom "fp32_to_int8" -// CHECK: scf.for {{.*}} to %c32 step %c16 -// CHECK: buckyball.bank_custom "matmul_i8" -// CHECK: buckyball.bank_custom "int32_to_fp32" +// CHECK: toint8_ball.bank_fp32_to_int8 +// CHECK: scf.for {{.*}} to %c32{{.*}} step %c16 +// CHECK: smatmul_ball.bank_matmul_i8 +// CHECK: int2fp_ball.bank_int32_to_fp32 // CHECK: buckyball.bank_mvout -// CHECK-NOT: buckyball.bank_custom +// CHECK-NOT: _ball.bank_ // CHECK: return func.func @linear(%a: tensor<1x32xf32>, %w: tensor<32x32xi8>, %bias: tensor<1x32xi32>, %multiplier: tensor<1x1xf32>, %input_step: tensor<1x1xf32>, %weight_step: tensor<1x32xf32>) -> tensor<1x32xf32> { %product = custom.op "linear_i8"(%a, %w, %bias, %multiplier, %input_step, %weight_step) : (tensor<1x32xf32>, tensor<32x32xi8>, tensor<1x32xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x32xf32>) -> tensor<1x32xf32> diff --git a/examples/designs/pebble/stack/midend/tests/lower-frontend-types.mlir b/examples/designs/pebble/stack/device/core/tests/lower-frontend-types.mlir similarity index 100% rename from examples/designs/pebble/stack/midend/tests/lower-frontend-types.mlir rename to examples/designs/pebble/stack/device/core/tests/lower-frontend-types.mlir diff --git a/examples/designs/pebble/stack/midend/tests/unknown-custom-op.mlir b/examples/designs/pebble/stack/device/core/tests/unknown-custom-op.mlir similarity index 100% rename from examples/designs/pebble/stack/midend/tests/unknown-custom-op.mlir rename to examples/designs/pebble/stack/device/core/tests/unknown-custom-op.mlir diff --git a/examples/designs/pebble/stack/engine/CMakeLists.txt b/examples/designs/pebble/stack/engine/CMakeLists.txt deleted file mode 100644 index 686343604..000000000 --- a/examples/designs/pebble/stack/engine/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Core execution binding; instruction streams use the framework engine. -set(RUNTIME "${DESIGN_REPO}/stack/runtime") -set(ENGINE "${DESIGN_REPO}/stack/engine") -find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) - -add_custom_command( - OUTPUT "${CMAKE_BINARY_DIR}/execute.o" - COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 - "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" - -c "${RUNTIME}/execute/direct.c" -o "${CMAKE_BINARY_DIR}/execute.o" - DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" - VERBATIM) -add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") diff --git a/examples/designs/pebble/stack/midend/CMakeLists.txt b/examples/designs/pebble/stack/midend/CMakeLists.txt deleted file mode 100644 index e3e207cb1..000000000 --- a/examples/designs/pebble/stack/midend/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -# Host compiler plugin for this design's core. -set(ENGINE "${DESIGN_REPO}/stack/engine") -set(TARGET ${DESIGN_CORE_TARGET_${CORE}}) - -set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${TARGET}}) -set(BUCKYBALL_TARGET_TABLE "${DESIGN_ISA_DIR}/${TARGET}/target.inc") -# The core owns computation recognition and its conversion to bank dataflow. -file(GLOB_RECURSE BUCKYBALL_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/src/*.cpp") -add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) -add_library(${TARGET}-compiler MODULE $) -set_target_properties(${TARGET}-compiler PROPERTIES - LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") - -set(CHECK "${DESIGN_MLIR}/../../../bin/FileCheck") -set(OPT "${DESIGN_COMPILER}/bin/buddy-opt --load-dialect-plugin=$ --load-pass-plugin=$") -set(TESTS "${CMAKE_CURRENT_SOURCE_DIR}/tests") -add_custom_target(${TARGET}-compiler-check - COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-custom-ops.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${TESTS}/form-custom-ops.mlir --implicit-check-not=buckyball." - COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-int32-bias.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${TESTS}/form-int32-bias.mlir --implicit-check-not=buckyball." - COMMAND bash -o pipefail -c "${OPT} ${TESTS}/lower-frontend-types.mlir -allow-unregistered-dialect -lower-frontend -split-input-file -verify-diagnostics | ${CHECK} ${TESTS}/lower-frontend-types.mlir --check-prefix=VALID --implicit-check-not=frontend." - COMMAND bash -o pipefail -c "${OPT} ${TESTS}/lower-custom-ops.mlir -convert-linalg-to-buckyball | ${CHECK} ${TESTS}/lower-custom-ops.mlir --implicit-check-not=custom.op" - COMMAND bash -o pipefail -c "${OPT} ${TESTS}/unknown-custom-op.mlir -convert-linalg-to-buckyball -verify-diagnostics -o /dev/null" - COMMAND bash -o pipefail -c "${OPT} ${CMAKE_CURRENT_SOURCE_DIR}/../../workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${CMAKE_CURRENT_SOURCE_DIR}/../../workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir --check-prefix=FORM" - COMMAND bash -o pipefail -c "${OPT} ${CMAKE_CURRENT_SOURCE_DIR}/../../workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir -allow-unregistered-dialect -form-custom-ops -convert-linalg-to-buckyball | ${CHECK} ${CMAKE_CURRENT_SOURCE_DIR}/../../workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir --check-prefix=BANK --implicit-check-not=buckyball.bank_mvout" - COMMAND bash -o pipefail -c "${OPT} ${TESTS}/form-depthwise.mlir -allow-unregistered-dialect -form-custom-ops | ${CHECK} ${TESTS}/form-depthwise.mlir --implicit-check-not=linalg.depthwise" - COMMAND bash -o pipefail -c "${OPT} ${TESTS}/lower-bank-updates.mlir -lower-buckyball | ${CHECK} ${TESTS}/lower-bank-updates.mlir" - DEPENDS ${TARGET}-compiler - VERBATIM) diff --git a/examples/designs/pebble/stack/midend/src/ConvertLinalgToBuckyball.cpp b/examples/designs/pebble/stack/midend/src/ConvertLinalgToBuckyball.cpp deleted file mode 100644 index 63a9ba7a4..000000000 --- a/examples/designs/pebble/stack/midend/src/ConvertLinalgToBuckyball.cpp +++ /dev/null @@ -1,312 +0,0 @@ -//===- ConvertLinalgToBuckyball.cpp - Pebble core lowering ----------------===// - -#include "Buckyball/BuckyballDialect.h" -#include "Buckyball/BuckyballOps.h" -#include "Custom/CustomOps.h" -#include "Lowering.h" -#include "Target/BuckyballTargetRegistry.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Bufferization/IR/Bufferization.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Tensor/IR/Tensor.h" -#include "mlir/Dialect/Utils/StaticValueUtils.h" -#include "mlir/Pass/Pass.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -// The recipe supplies contiguous tensors; offsets may point into packed -// weights. -Value pebble::tensorBuffer(OpBuilder &builder, Location loc, Value tensor) { - auto type = cast(tensor.getType()); - SmallVector strides(type.getRank()); - int64_t stride = 1; - for (int64_t dim = type.getRank() - 1; dim >= 0; --dim) { - strides[dim] = stride; - stride *= type.getDimSize(dim); - } - auto layout = StridedLayoutAttr::get(builder.getContext(), - ShapedType::kDynamic, strides); - auto memory = MemRefType::get(type.getShape(), type.getElementType(), layout); - return bufferization::ToBufferOp::create(builder, loc, memory, tensor, true); -} - -SmallVector pebble::repeat( - OpBuilder &builder, Location loc, int64_t begin, int64_t end, int64_t step, - bool unroll, ValueRange carried, - function_ref(OpBuilder &, Value, ValueRange)> body) { - SmallVector values(carried); - if (unroll || end - begin <= step) { - for (int64_t index = begin; index < end; index += step) - values = body( - builder, arith::ConstantIndexOp::create(builder, loc, index), values); - return values; - } - auto loop = scf::ForOp::create( - builder, loc, arith::ConstantIndexOp::create(builder, loc, begin), - arith::ConstantIndexOp::create(builder, loc, end), - arith::ConstantIndexOp::create(builder, loc, step), carried, - [&](OpBuilder &nested, Location loc, Value index, ValueRange args) { - scf::YieldOp::create(nested, loc, body(nested, index, args)); - }); - return loop.getResults(); -} - -Value pebble::accumulate( - OpBuilder &builder, Location loc, int64_t total, int64_t size, bool unroll, - Value accumulator, - function_ref body) { - int64_t pieces = (total + size - 1) / size; - int64_t last = (pieces - 1) * size; - accumulator = body(builder, arith::ConstantIndexOp::create(builder, loc, 0), - std::min(size, total), true, pieces == 1, accumulator); - accumulator = pebble::repeat( - builder, loc, size, last, size, unroll, accumulator, - [&](OpBuilder &b, Value start, ValueRange args) -> SmallVector { - return {body(b, start, size, false, false, args[0])}; - })[0]; - if (pieces > 1) - accumulator = - body(builder, arith::ConstantIndexOp::create(builder, loc, last), - total - last, false, true, accumulator); - return accumulator; -} - -pebble::SpatialTile pebble::spatialTile(int64_t height, int64_t width, - int64_t pool, int64_t stride, - int64_t kernel) { - int64_t tile = 4 / pool; - while (tile > 0 && (height % tile || width % tile || - (tile * pool - 1) * stride + kernel > 8)) - --tile; - if (!tile) - return {0, 0}; - int64_t side = std::min(8, std::min(height, width)); - while (side % tile) - --side; - return {tile, side}; -} - -namespace { -LogicalResult lowerMatmul(::buddy::custom::CustomOp op, Value residentInput, - Value &result) { - bool linear = op.getKind() == "linear_i8"; - if (op.getInputs().size() != (linear ? 6 : 3) || op.getNumResults() != 1) - return op.emitOpError("has the wrong number of inputs or results for ") - << op.getKind(); - ValueRange inputs = op.getInputs(); - auto a = cast(inputs[0].getType()); - auto b = cast(inputs[1].getType()); - auto resultType = cast(op.getResult(0).getType()); - if (!a.hasStaticShape() || !b.hasStaticShape() || a.getRank() != 2 || - b.getRank() != 2 || a.getDimSize(1) != b.getDimSize(0)) - return op.emitOpError("requires static MxK and KxN tensors"); - const auto &target = buckyball_target::getBuckyballTarget(); - int64_t k = a.getDimSize(1), n = b.getDimSize(1); - // One activation row with aligned K and N stays in one bank. - if (a.getDimSize(0) != 1 || k % 16 || n % 16 || k > target.bankDepth) - return pebble::lowerLinearTiles(op, result); - if (!b.getElementType().isInteger(8) || - !(linear ? a.getElementType().isF32() - : a.getElementType().isInteger(8)) || - !(linear ? resultType.getElementType().isF32() - : resultType.getElementType().isInteger(32)) || - resultType.getShape() != ArrayRef({1, n})) - return op.emitOpError("has invalid operand or result types for ") - << op.getKind(); - const int64_t scalarDimensions[] = {1, 1}; - ArrayRef scalarShape(scalarDimensions); - if (linear && - (cast(inputs[3].getType()).getShape() != scalarShape || - cast(inputs[4].getType()).getShape() != scalarShape || - cast(inputs[5].getType()).getShape() != - ArrayRef({1, n}))) - return op.emitOpError("requires one activation scale and one weight scale " - "per output channel"); - auto initialType = cast(inputs[2].getType()); - if (!initialType.getElementType().isInteger(32) || - initialType.getShape() != resultType.getShape()) - return op.emitOpError( - "requires an INT32 initial accumulator matching the result"); - - OpBuilder builder(op); - Location loc = op.getLoc(); - Type i64 = builder.getI64Type(); - Value zero = arith::ConstantIndexOp::create(builder, loc, 0); - Value weight = pebble::tensorBuffer(builder, loc, inputs[1]); - Value output = memref::AllocOp::create( - builder, loc, MemRefType::get({1, n}, resultType.getElementType())); - Value initial = pebble::tensorBuffer(builder, loc, inputs[2]); - Value scales, inputStep; - Value codes = residentInput; - if (linear) { - inputStep = tensor::ExtractOp::create(builder, loc, inputs[4], - ValueRange{zero, zero}); - scales = memref::AllocOp::create( - builder, loc, MemRefType::get({16}, builder.getF32Type())); - } - if (!codes) { - Value input = pebble::tensorBuffer(builder, loc, inputs[0]); - codes = BankAllocOp::create(builder, loc, i64, 1, 1); - if (linear) { - Value multiplier = tensor::ExtractOp::create(builder, loc, inputs[3], - ValueRange{zero, zero}); - Value inputBank = BankAllocOp::create(builder, loc, i64, 1, 1); - inputBank = - BankMvinOp::create(builder, loc, i64, input, inputBank, k / 4, 1); - auto quantized = - BankCustomOp::create(builder, loc, i64, ValueRange{inputBank}, codes, - ValueRange{multiplier}, "fp32_to_int8"); - quantized->setAttr("rows", builder.getI64IntegerAttr(k / 4)); - codes = quantized.getDestinationOut(); - BankReleaseOp::create(builder, loc, inputBank); - } else { - codes = BankMvinOp::create(builder, loc, i64, input, codes, k / 16, 1); - } - } - - // Keep the activation in its bank while processing each sixteen-column - // slice. - pebble::repeat( - builder, loc, 0, n, 16, false, {}, - [&](OpBuilder &b, Value column, ValueRange) -> SmallVector { - if (linear) { - auto lanes = scf::ForOp::create( - b, loc, zero, arith::ConstantIndexOp::create(b, loc, 16), - arith::ConstantIndexOp::create(b, loc, 1)); - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(); - Value channel = arith::AddIOp::create(b, loc, column, lane); - Value step = tensor::ExtractOp::create(b, loc, inputs[5], - ValueRange{zero, channel}); - Value product = arith::MulFOp::create(b, loc, inputStep, step); - memref::StoreOp::create(b, loc, product, scales, lane); - } - SmallVector offsets{b.getIndexAttr(0), column}; - SmallVector unit{b.getIndexAttr(1), b.getIndexAttr(1)}; - Value weightSlice = memref::SubViewOp::create( - b, loc, weight, offsets, - ArrayRef{b.getIndexAttr(k), b.getIndexAttr(16)}, - unit); - Value weightBank = BankAllocOp::create(b, loc, i64, 1, 1); - weightBank = - BankMvinOp::create(b, loc, i64, weightSlice, weightBank, k, n / 16); - Value accumulator = BankAllocOp::create(b, loc, i64, 1, 1); - Value initialSlice = memref::SubViewOp::create( - b, loc, initial, offsets, - ArrayRef{b.getIndexAttr(1), b.getIndexAttr(16)}, - unit); - accumulator = - BankMvinOp::create(b, loc, i64, initialSlice, accumulator, 4, 1); - auto matmul = - BankCustomOp::create(b, loc, i64, ValueRange{codes, weightBank}, - accumulator, ValueRange{}, "matmul_i8", true); - matmul->setAttr("rows", b.getI64IntegerAttr(1)); - matmul->setAttr("k", b.getI64IntegerAttr(k)); - accumulator = matmul.getDestinationOut(); - BankReleaseOp::create(b, loc, weightBank); - - Value resultBank = accumulator; - if (linear) { - Value scaleBank = BankAllocOp::create(b, loc, i64, 1, 1); - scaleBank = BankMvinOp::create(b, loc, i64, scales, scaleBank, 4, 1); - resultBank = BankAllocOp::create(b, loc, i64, 1, 1); - auto restored = BankCustomOp::create( - b, loc, i64, ValueRange{accumulator, scaleBank}, resultBank, - ValueRange{}, "int32_to_fp32"); - restored->setAttr("rows", b.getI64IntegerAttr(1)); - restored->setAttr("relu", b.getBoolAttr(false)); - resultBank = restored.getDestinationOut(); - BankReleaseOp::create(b, loc, scaleBank); - } - Value outputSlice = memref::SubViewOp::create( - b, loc, output, offsets, - ArrayRef{b.getIndexAttr(1), b.getIndexAttr(16)}, - unit); - resultBank = - BankMvoutOp::create(b, loc, i64, outputSlice, resultBank, 4, 1); - if (linear) - BankReleaseOp::create(b, loc, resultBank); - BankReleaseOp::create(b, loc, accumulator); - return SmallVector{}; - }); - BankReleaseOp::create(builder, loc, codes); - result = bufferization::ToTensorOp::create( - builder, loc, op.getResult(0).getType(), output, true, true); - return success(); -} - -class ConvertLinalgToBuckyballPass - : public PassWrapper> { -public: - MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ConvertLinalgToBuckyballPass) - StringRef getArgument() const final { return "convert-linalg-to-buckyball"; } - StringRef getDescription() const final { - return "Lower custom tensor computations to the core bank dataflow"; - } - void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert(); - } - void runOnOperation() override { - DenseMap residentVectors; - SmallVector spatial; - if (failed( - pebble::lowerSpatialOps(getOperation(), residentVectors, spatial))) - return signalPassFailure(); - SmallVector<::buddy::custom::CustomOp> computations; - getOperation().walk([&](::buddy::custom::CustomOp op) { - if (!llvm::is_contained(spatial, op.getOperation())) - computations.push_back(op); - }); - for (auto op : computations) { - if (op.getKind() == "global_avg_i8" || op.getKind() == "lookup_i8" || - op.getKind() == "gate_mul_i8") { - Value result; - LogicalResult status = op.getKind() == "global_avg_i8" - ? pebble::lowerGlobalAverage(op, result) - : pebble::lowerChannelOp(op, result); - if (failed(status)) - return signalPassFailure(); - op.getResult(0).replaceAllUsesWith(result); - op.erase(); - continue; - } - if (op.getKind() == "transpose_i8" || - op.getKind() == "linear_i8_requant") { - Value result; - if (failed(op.getKind() == "transpose_i8" - ? pebble::lowerTranspose(op, result) - : pebble::lowerLinearTiles(op, result))) - return signalPassFailure(); - op.getResult(0).replaceAllUsesWith(result); - op.erase(); - continue; - } - if (op.getKind() != "linear_i8" && op.getKind() != "matmul_i8") { - op.emitOpError("has no bank lowering on this core for ") - << op.getKind(); - return signalPassFailure(); - } - Value result; - if (failed(lowerMatmul(op, residentVectors.lookup(op.getInputs()[0]), - result))) - return signalPassFailure(); - op.getResult(0).replaceAllUsesWith(result); - op.erase(); - } - for (Operation *op : llvm::reverse(spatial)) - op->erase(); - } -}; - -PassRegistration registration; -} // namespace diff --git a/examples/designs/pebble/stack/midend/src/LowerLinearTiles.cpp b/examples/designs/pebble/stack/midend/src/LowerLinearTiles.cpp deleted file mode 100644 index 3414701be..000000000 --- a/examples/designs/pebble/stack/midend/src/LowerLinearTiles.cpp +++ /dev/null @@ -1,264 +0,0 @@ -//===- LowerLinearTiles.cpp - Linear rows and reduction/output tails ------===// -#include "Buckyball/BuckyballOps.h" -#include "Custom/CustomOps.h" -#include "Lowering.h" -#include "Target/BuckyballTargetRegistry.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Bufferization/IR/Bufferization.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Tensor/IR/Tensor.h" - -using namespace mlir; -using namespace ::buddy::buckyball; - -LogicalResult pebble::lowerLinearTiles(::buddy::custom::CustomOp op, - Value &result) { - bool integer = op.getKind() == "matmul_i8"; - bool requant = op.getKind() == "linear_i8_requant"; - auto args = op.getInputs(); - auto a = cast(args[0].getType()); - auto w = cast(args[1].getType()); - auto biasType = cast(args[2].getType()); - auto resultType = cast(op.getResult(0).getType()); - if (!a.hasStaticShape() || !w.hasStaticShape() || a.getRank() != 2 || - w.getRank() != 2 || a.getDimSize(1) != w.getDimSize(0) || - !((requant || integer) ? a.getElementType().isInteger(8) - : a.getElementType().isF32()) || - !w.getElementType().isInteger(8) || - !(integer ? resultType.getElementType().isInteger(32) - : requant ? resultType.getElementType().isInteger(8) - : resultType.getElementType().isF32()) || - !biasType.getElementType().isInteger(32) || - biasType.getShape() != - ArrayRef({integer ? a.getDimSize(0) : 1, w.getDimSize(1)}) || - resultType.getShape() != - ArrayRef({a.getDimSize(0), w.getDimSize(1)})) - return op.emitOpError( - "requires matching MxK/KxN tensors and INT32 1xN bias"); - int64_t m = a.getDimSize(0), k = a.getDimSize(1), n = w.getDimSize(1); - const auto &target = buckyball_target::getBuckyballTarget(); - int64_t rows = integer ? 1 - : buckyball_target::getBuckyballBallParam("SMATMUL_OS", - "tileRows"); - int64_t maxK = - std::min(target.bankDepth, - target.bankDepth * ((requant || integer) ? 16 : 4) / rows) / - 16 * 16; - if (m <= 0 || k <= 0 || n <= 0 || target.bankWidthBits != 128 || maxK == 0 || - rows * 4 > target.bankDepth) - return op.emitOpError("Linear tile does not fit the configured banks"); - OpBuilder builder(op); - Location loc = op.getLoc(); - Type i64 = builder.getI64Type(); - auto index = [&](OpBuilder &b, int64_t n) -> Value { - return arith::ConstantIndexOp::create(b, loc, n); - }; - // Host loops over the valid part of a padded tile: min(size, limit - start). - auto extent = [&](OpBuilder &b, int64_t size, int64_t limit, - Value start) -> Value { - return arith::MinSIOp::create( - b, loc, index(b, size), - arith::SubIOp::create(b, loc, index(b, limit), start)); - }; - Value zero = index(builder, 0), one = index(builder, 1); - Value input = tensorBuffer(builder, loc, args[0]); - Value weight = tensorBuffer(builder, loc, args[1]); - Value bias = tensorBuffer(builder, loc, args[2]); - Value step; - if (!integer) - step = tensorBuffer(builder, loc, args[requant ? 3 : 5]); - Value multiplier, inputStep; - if (!requant && !integer) { - multiplier = tensor::ExtractOp::create(builder, loc, args[3], - ValueRange{zero, zero}); - inputStep = tensor::ExtractOp::create(builder, loc, args[4], - ValueRange{zero, zero}); - } - Value output = memref::AllocOp::create( - builder, loc, MemRefType::get({m, n}, resultType.getElementType())); - Value resultPack = memref::AllocOp::create( - builder, loc, MemRefType::get({rows, 16}, resultType.getElementType())); - Value biasPack = memref::AllocOp::create( - builder, loc, MemRefType::get({16}, builder.getI32Type())); - Value scalePack; - if (!integer) - scalePack = memref::AllocOp::create( - builder, loc, MemRefType::get({16}, builder.getF32Type())); - - pebble::repeat( - builder, loc, 0, n, 16, false, {}, - [&](OpBuilder &b, Value col, ValueRange) { - Value columns = extent(b, 16, n, col); - linalg::FillOp::create( - b, loc, ValueRange{arith::ConstantIntOp::create(b, loc, 0, 32)}, - biasPack); - Value scales; - if (!integer) { - linalg::FillOp::create(b, loc, - ValueRange{arith::ConstantOp::create( - b, loc, b.getF32FloatAttr(1))}, - scalePack); - auto lanes = scf::ForOp::create(b, loc, zero, columns, one); - { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(); - Value channel = arith::AddIOp::create(b, loc, col, lane); - Value value = - memref::LoadOp::create(b, loc, bias, ValueRange{zero, channel}); - memref::StoreOp::create(b, loc, value, biasPack, lane); - Value scale = - memref::LoadOp::create(b, loc, step, ValueRange{zero, channel}); - if (!requant) - scale = arith::MulFOp::create(b, loc, inputStep, scale); - memref::StoreOp::create(b, loc, scale, scalePack, lane); - } - scales = BankAllocOp::create(b, loc, i64, 1, 1); - scales = BankMvinOp::create(b, loc, i64, scalePack, scales, 4, 1); - } - pebble::repeat( - b, loc, 0, m, rows, false, {}, - [&](OpBuilder &b, Value row, ValueRange) { - Value valid = extent(b, rows, m, row); - if (integer) { - auto lanes = scf::ForOp::create(b, loc, zero, columns, one); - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(lanes.getBody()); - Value lane = lanes.getInductionVar(); - Value channel = arith::AddIOp::create(b, loc, col, lane); - Value value = memref::LoadOp::create(b, loc, bias, - ValueRange{row, channel}); - memref::StoreOp::create(b, loc, value, biasPack, lane); - } - Value accumulator = BankAllocOp::create(b, loc, i64, 1, 1); - accumulator = - BankMvinOp::create(b, loc, i64, biasPack, accumulator, 4, 1); - accumulator = pebble::accumulate( - b, loc, k, maxK, false, accumulator, - [&](OpBuilder &b, Value start, int64_t count, bool first, - bool last, Value accumulator) -> Value { - int64_t reduction = (count + 15) / 16 * 16; - Value inputPack = memref::AllocOp::create( - b, loc, - MemRefType::get({reduction / 16 * rows, 16}, - a.getElementType())); - Value weightPack = memref::AllocOp::create( - b, loc, - MemRefType::get({reduction, 16}, b.getI8Type())); - Value fzero = - arith::ConstantOp::create(b, loc, b.getF32FloatAttr(0)); - Value izero = arith::ConstantIntOp::create(b, loc, 0, 8); - linalg::FillOp::create(b, loc, - (requant || integer) ? izero : fzero, - inputPack); - linalg::FillOp::create(b, loc, izero, weightPack); - auto inner = - scf::ForOp::create(b, loc, zero, index(b, count), one); - { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(inner.getBody()); - Value reductionIndex = inner.getInductionVar(); - Value sourceK = - arith::AddIOp::create(b, loc, reductionIndex, start); - Value block = arith::MulIOp::create( - b, loc, - arith::DivUIOp::create(b, loc, reductionIndex, - index(b, 16)), - index(b, rows)); - Value lane = arith::RemUIOp::create( - b, loc, reductionIndex, index(b, 16)); - auto inputRows = - scf::ForOp::create(b, loc, zero, valid, one); - { - OpBuilder::InsertionGuard rowGuard(b); - b.setInsertionPointToStart(inputRows.getBody()); - Value r = inputRows.getInductionVar(); - Value sourceM = arith::AddIOp::create(b, loc, r, row); - Value destination = - arith::AddIOp::create(b, loc, block, r); - Value value = memref::LoadOp::create( - b, loc, input, ValueRange{sourceM, sourceK}); - memref::StoreOp::create(b, loc, value, inputPack, - ValueRange{destination, lane}); - } - auto lanes = - scf::ForOp::create(b, loc, zero, columns, one); - b.setInsertionPointToStart(lanes.getBody()); - Value c = lanes.getInductionVar(); - Value sourceN = arith::AddIOp::create(b, loc, c, col); - Value value = memref::LoadOp::create( - b, loc, weight, ValueRange{sourceK, sourceN}); - memref::StoreOp::create(b, loc, value, weightPack, - ValueRange{reductionIndex, c}); - } - Value codes = BankAllocOp::create(b, loc, i64, 1, 1); - if (requant || integer) { - codes = BankMvinOp::create(b, loc, i64, inputPack, codes, - rows * reduction / 16, 1); - } else { - Value fp = BankAllocOp::create(b, loc, i64, 1, 1); - fp = BankMvinOp::create(b, loc, i64, inputPack, fp, - rows * reduction / 4, 1); - auto quant = BankCustomOp::create( - b, loc, i64, ValueRange{fp}, codes, - ValueRange{multiplier}, "fp32_to_int8"); - quant->setAttr("rows", - b.getI64IntegerAttr(rows * reduction / 4)); - codes = quant.getDestinationOut(); - BankReleaseOp::create(b, loc, fp); - } - Value weights = BankAllocOp::create(b, loc, i64, 1, 1); - weights = BankMvinOp::create(b, loc, i64, weightPack, - weights, reduction, 1); - auto matmul = BankCustomOp::create( - b, loc, i64, ValueRange{codes, weights}, accumulator, - ValueRange{}, "matmul_i8_block", true); - matmul->setAttr("rows", b.getI64IntegerAttr(rows)); - matmul->setAttr("k", b.getI64IntegerAttr(reduction)); - matmul->setAttr("first", b.getBoolAttr(first)); - matmul->setAttr("last", b.getBoolAttr(last)); - BankReleaseOp::create(b, loc, codes); - BankReleaseOp::create(b, loc, weights); - return matmul.getDestinationOut(); - }); - Value restored = accumulator; - if (!integer) { - restored = BankAllocOp::create(b, loc, i64, 1, 1); - auto dequant = BankCustomOp::create( - b, loc, i64, ValueRange{accumulator, scales}, restored, - ValueRange{}, requant ? "int32_to_int8" : "int32_to_fp32"); - dequant->setAttr("rows", b.getI64IntegerAttr(rows)); - dequant->setAttr("relu", requant ? op->getAttr("relu") - : b.getBoolAttr(false)); - restored = dequant.getDestinationOut(); - } - BankMvoutOp::create(b, loc, i64, resultPack, restored, - rows * (requant ? 1 : 4), 1); - BankReleaseOp::create(b, loc, accumulator); - if (!integer) - BankReleaseOp::create(b, loc, restored); - auto outputRows = scf::ForOp::create(b, loc, zero, valid, one); - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(outputRows.getBody()); - Value r = outputRows.getInductionVar(); - auto lanes = scf::ForOp::create(b, loc, zero, columns, one); - b.setInsertionPointToStart(lanes.getBody()); - Value c = lanes.getInductionVar(); - Value value = - memref::LoadOp::create(b, loc, resultPack, ValueRange{r, c}); - memref::StoreOp::create( - b, loc, value, output, - ValueRange{arith::AddIOp::create(b, loc, r, row), - arith::AddIOp::create(b, loc, c, col)}); - return SmallVector{}; - }); - if (!integer) - BankReleaseOp::create(b, loc, scales); - return SmallVector{}; - }); - result = bufferization::ToTensorOp::create(builder, loc, resultType, output, - true, true); - return success(); -} diff --git a/examples/designs/pebble/stack/midend/src/LowerSpatialOps.cpp b/examples/designs/pebble/stack/midend/src/LowerSpatialOps.cpp deleted file mode 100644 index b645d79af..000000000 --- a/examples/designs/pebble/stack/midend/src/LowerSpatialOps.cpp +++ /dev/null @@ -1,571 +0,0 @@ -//===- LowerSpatialOps.cpp - Convolution and pooling on resident banks ----===// -#include "Buckyball/BuckyballOps.h" -#include "Custom/CustomOps.h" -#include "Lowering.h" -#include "Target/BuckyballTargetRegistry.h" -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/Bufferization/IR/Bufferization.h" -#include "mlir/Dialect/Linalg/IR/Linalg.h" -#include "mlir/Dialect/MemRef/IR/MemRef.h" -#include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Tensor/IR/Tensor.h" -#include "mlir/Dialect/Utils/StaticValueUtils.h" - -using namespace mlir; -using namespace ::buddy::buckyball; -using Computation = ::buddy::custom::CustomOp; - -namespace { -// Each spatial panel holds sixteen channels per pixel, with at most 64 pixels. -// MAXPOOL can address every panel origin with its six-bit bank row fields. -struct ImageBanks { - SmallVector banks; - int64_t height, width, channels, side; -}; - -BankCustomOp -compute(OpBuilder &b, Location loc, StringRef kind, ValueRange sources, - Value destination, bool read, - std::initializer_list> attrs) { - auto op = BankCustomOp::create(b, loc, b.getI64Type(), sources, destination, - ValueRange{}, kind, read); - for (auto [name, value] : attrs) - op->setAttr(name, b.getI64IntegerAttr(value)); - return op; -} - -LogicalResult convolution(Computation op, const ImageBanks *inputImage, - Value nextMultiplier, int64_t pool, bool relu, - ImageBanks &outputImage, Value &tensorOutput) { - bool depthwise = op.getKind() == "depthwise_conv2d_i8"; - auto args = op.getInputs(); - auto inputType = cast(args[0].getType()); - auto weightType = cast(args[1].getType()); - auto outputType = cast(op.getResult(0).getType()); - auto strides = cast(op->getAttr("strides")).asArrayRef(); - auto padding = cast(op->getAttr("padding")).asArrayRef(); - if (args.size() != 6 || !inputType.hasStaticShape() || - inputType.getRank() != 4 || !weightType.hasStaticShape() || - weightType.getRank() != 4 || inputType.getDimSize(0) != 1 || - !inputType.getElementType().isF32() || - !weightType.getElementType().isInteger(8) || - !cast(args[2].getType()).getElementType().isInteger(32)) - return op.emitOpError("requires static NHWC/FCHW convolution with INT32 " - "bias"); - // Recognition admits only geometries that fit the Balls and the banks. - int64_t cin = inputType.getDimSize(3), cout = weightType.getDimSize(0); - int64_t kernel = weightType.getDimSize(2), stride = strides[0]; - int64_t height = outputType.getDimSize(1) / pool; - int64_t width = outputType.getDimSize(2) / pool; - auto [tile, side] = pebble::spatialTile(height, width, pool, stride, kernel); - int64_t convSide = tile * pool; - int64_t inputSide = (convSide - 1) * stride + kernel; - int64_t arrayRows = - buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); - int64_t rows = (convSide * convSide + arrayRows - 1) / arrayRows * arrayRows; - int64_t reduction = (kernel * kernel + 15) / 16 * 16; - int64_t panelsY = (height + side - 1) / side; - int64_t panelsX = (width + side - 1) / side; - int64_t outputBanks = - nextMultiplier ? panelsY * panelsX * ((cout + 15) / 16) : 0; - // A resident image selects a bank and bank rows per pixel, so its loops - // unroll; recognition bounds a resident image by the bank count. - bool resident = inputImage || nextMultiplier; - - OpBuilder builder(op); - Location loc = op.getLoc(); - Type i64 = builder.getI64Type(); - auto index = [&](OpBuilder &b, int64_t n) -> Value { - return arith::ConstantIndexOp::create(b, loc, n); - }; - auto constant = [](Value value) { return *getConstantIntValue(value); }; - Value zero = index(builder, 0), one = index(builder, 1); - Value fzero = - arith::ConstantOp::create(builder, loc, builder.getF32FloatAttr(0)); - Value izero = arith::ConstantIntOp::create(builder, loc, 0, 8); - Value input; - if (!inputImage) - input = pebble::tensorBuffer(builder, loc, args[0]); - Value weight = pebble::tensorBuffer(builder, loc, args[1]); - Value bias = pebble::tensorBuffer(builder, loc, args[2]); - Value weightStep = pebble::tensorBuffer(builder, loc, args[5]); - Value step = tensor::ExtractOp::create(builder, loc, args[4], - ValueRange{zero, zero, zero, zero}); - Value inputMultiplier; - if (!inputImage) - inputMultiplier = tensor::ExtractOp::create( - builder, loc, args[3], ValueRange{zero, zero, zero, zero}); - Value multiplier; - if (nextMultiplier) { - SmallVector indices( - cast(nextMultiplier.getType()).getRank(), zero); - multiplier = - tensor::ExtractOp::create(builder, loc, nextMultiplier, indices); - } - outputImage = {{}, height, width, cout, side}; - for (int64_t i = 0; i < outputBanks; ++i) - outputImage.banks.push_back(BankAllocOp::create(builder, loc, i64, 1, 1)); - Value output; - if (!nextMultiplier) - output = memref::AllocOp::create( - builder, loc, - MemRefType::get(outputType.getShape(), builder.getF32Type())); - Value zeroPanel = memref::AllocOp::create( - builder, loc, - MemRefType::get({inputSide * inputSide, 16}, builder.getI8Type())); - linalg::FillOp::create(builder, loc, izero, zeroPanel); - Value floatPanel; - if (!inputImage) - floatPanel = memref::AllocOp::create( - builder, loc, - MemRefType::get({inputSide * inputSide, 16}, builder.getF32Type())); - Value outputPack; - if (!nextMultiplier) - outputPack = memref::AllocOp::create( - builder, loc, MemRefType::get({rows, 16}, builder.getF32Type())); - // Per output-channel group: the weights of every term, the biases and the - // scales of its sixteen lanes. - Value weightPack = memref::AllocOp::create( - builder, loc, - MemRefType::get({depthwise ? 16 : cin, reduction, 16}, - builder.getI8Type())); - Value biasPack = memref::AllocOp::create( - builder, loc, MemRefType::get({16}, builder.getI32Type())); - Value scalePack = memref::AllocOp::create( - builder, loc, MemRefType::get({16}, builder.getF32Type())); - - // The terms of one input-channel group: sixteen lanes of one source panel. - // Im2col selects the lane as an attribute, so the lanes unroll. - auto group = [&](OpBuilder &b, Value oc, Value y, Value x, Value ic, - int64_t lanes, bool first, bool last, Value acc) -> Value { - Value source = BankAllocOp::create(b, loc, i64, 1, 1); - if (inputImage) { - source = BankMvinOp::create(b, loc, i64, zeroPanel, source, - inputSide * inputSide, 1); - int64_t inPanelsY = - (inputImage->height + inputImage->side - 1) / inputImage->side; - int64_t inPanelsX = - (inputImage->width + inputImage->side - 1) / inputImage->side; - for (int64_t iy = 0; iy < inputSide; ++iy) - for (int64_t ix = 0; ix < inputSide; ++ix) { - int64_t gy = constant(y) * pool * stride + iy - padding[0], - gx = constant(x) * pool * stride + ix - padding[2]; - if (gy < 0 || gx < 0 || gy >= inputImage->height || - gx >= inputImage->width) - continue; - int64_t panel = - (constant(ic) / 16 * inPanelsY + gy / inputImage->side) * - inPanelsX + - gx / inputImage->side; - int64_t base = - gy % inputImage->side * inputImage->side + gx % inputImage->side; - source = compute(b, loc, "maxpool_i8", - ValueRange{inputImage->banks[panel]}, source, true, - {{"input_side", 1}, - {"output_side", 1}, - {"kernel", 1}, - {"stride", 1}, - {"padding", 0}, - {"op1_base", base}, - {"wr_base", iy * inputSide + ix}, - {"output_stride", 1}, - {"start_row", 0}, - {"start_col", 0}}) - .getDestinationOut(); - } - } else { - linalg::FillOp::create(b, loc, fzero, floatPanel); - Value rowStart = arith::SubIOp::create( - b, loc, arith::MulIOp::create(b, loc, y, index(b, pool * stride)), - index(b, padding[0])); - Value colStart = arith::SubIOp::create( - b, loc, arith::MulIOp::create(b, loc, x, index(b, pool * stride)), - index(b, padding[2])); - Value channels = arith::MinSIOp::create( - b, loc, index(b, 16), - arith::SubIOp::create(b, loc, index(b, cin), ic)); - auto pixels = scf::ForOp::create(b, loc, zero, - index(b, inputSide * inputSide), one); - { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(pixels.getBody()); - Value pixel = pixels.getInductionVar(); - Value iy = arith::AddIOp::create( - b, loc, arith::DivUIOp::create(b, loc, pixel, index(b, inputSide)), - rowStart); - Value ix = arith::AddIOp::create( - b, loc, arith::RemUIOp::create(b, loc, pixel, index(b, inputSide)), - colStart); - auto lanesLoop = scf::ForOp::create(b, loc, zero, channels, one); - b.setInsertionPointToStart(lanesLoop.getBody()); - Value lane = lanesLoop.getInductionVar(); - Value channel = arith::AddIOp::create(b, loc, lane, ic); - Value inBounds = arith::AndIOp::create( - b, loc, - arith::AndIOp::create( - b, loc, - arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sge, iy, - zero), - arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt, iy, - index(b, inputType.getDimSize(1)))), - arith::AndIOp::create( - b, loc, - arith::CmpIOp::create(b, loc, arith::CmpIPredicate::sge, ix, - zero), - arith::CmpIOp::create(b, loc, arith::CmpIPredicate::slt, ix, - index(b, inputType.getDimSize(2))))); - auto valid = scf::IfOp::create(b, loc, inBounds, false); - b.setInsertionPointToStart(&valid.getThenRegion().front()); - Value value = memref::LoadOp::create(b, loc, input, - ValueRange{zero, iy, ix, channel}); - memref::StoreOp::create(b, loc, value, floatPanel, - ValueRange{pixel, lane}); - } - Value fp = BankAllocOp::create(b, loc, i64, 1, 1); - fp = BankMvinOp::create(b, loc, i64, floatPanel, fp, - inputSide * inputSide * 4, 1); - auto q = - BankCustomOp::create(b, loc, i64, ValueRange{fp}, source, - ValueRange{inputMultiplier}, "fp32_to_int8"); - q->setAttr("rows", b.getI64IntegerAttr(inputSide * inputSide * 4)); - source = q.getDestinationOut(); - BankReleaseOp::create(b, loc, fp); - } - for (int64_t lane = 0; lane < lanes; ++lane) { - Value patch = BankAllocOp::create(b, loc, i64, 1, 1); - patch = compute(b, loc, "im2col_i8", ValueRange{source}, patch, false, - {{"input_side", inputSide}, - {"kernel", kernel}, - {"stride", stride}, - {"padding", 0}, - {"start_row", 0}, - {"start_col", 0}, - {"op1_base", 0}, - {"lane", lane}, - {"window_start", 0}, - {"window_count", convSide * convSide}}) - .getDestinationOut(); - Value packed = BankAllocOp::create(b, loc, i64, 1, 1); - // Im2col packs M16/K16, while this array consumes tileRows/K16. - for (int64_t m = 0; m < rows; m += arrayRows) - for (int64_t k = 0; k < reduction / 16; ++k) - for (int64_t r = 0; r < arrayRows; r += 4) - packed = - compute( - b, loc, "maxpool_i8", ValueRange{patch}, packed, true, - {{"input_side", 2}, - {"output_side", 2}, - {"kernel", 1}, - {"stride", 1}, - {"padding", 0}, - {"op1_base", k * 16 + m + r}, - {"wr_base", - (m / arrayRows * (reduction / 16) + k) * arrayRows + r}, - {"output_stride", 2}, - {"start_row", 0}, - {"start_col", 0}}) - .getDestinationOut(); - BankReleaseOp::create(b, loc, patch); - // The weights of this term: its lane for depthwise, its input channel - // otherwise. - Value term = - depthwise - ? index(b, lane) - : arith::AddIOp::create(b, loc, ic, index(b, lane)).getResult(); - Value w = BankAllocOp::create(b, loc, i64, 1, 1); - Value slice = memref::SubViewOp::create( - b, loc, weightPack, - ArrayRef{term, b.getIndexAttr(0), b.getIndexAttr(0)}, - ArrayRef{b.getIndexAttr(1), b.getIndexAttr(reduction), - b.getIndexAttr(16)}, - ArrayRef{b.getIndexAttr(1), b.getIndexAttr(1), - b.getIndexAttr(1)}); - w = BankMvinOp::create(b, loc, i64, slice, w, reduction, 1); - auto matmul = compute(b, loc, "matmul_i8_block", ValueRange{packed, w}, - acc, true, {{"rows", rows}, {"k", reduction}}); - matmul->setAttr("first", b.getBoolAttr(first && lane == 0)); - matmul->setAttr("last", b.getBoolAttr(last && lane + 1 == lanes)); - acc = matmul.getDestinationOut(); - BankReleaseOp::create(b, loc, packed); - BankReleaseOp::create(b, loc, w); - } - BankReleaseOp::create(b, loc, source); - return acc; - }; - - // One output-channel group of `lanes` channels over every output tile. - auto outputChannels = [&](OpBuilder &b, Value oc, int64_t lanes) { - int64_t terms = depthwise ? lanes : cin; - linalg::FillOp::create(b, loc, izero, weightPack); - linalg::FillOp::create( - b, loc, ValueRange{arith::ConstantIntOp::create(b, loc, 0, 32)}, - biasPack); - linalg::FillOp::create( - b, loc, - ValueRange{arith::ConstantOp::create(b, loc, b.getF32FloatAttr(1))}, - scalePack); - auto laneLoop = scf::ForOp::create(b, loc, zero, index(b, lanes), one); - { - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(laneLoop.getBody()); - Value lane = laneLoop.getInductionVar(); - Value channel = arith::AddIOp::create(b, loc, oc, lane); - Value biasValue = memref::LoadOp::create(b, loc, bias, channel); - memref::StoreOp::create(b, loc, biasValue, biasPack, lane); - Value weightScale = memref::LoadOp::create( - b, loc, weightStep, ValueRange{channel, zero, zero, zero}); - Value scale = arith::MulFOp::create(b, loc, step, weightScale); - if (nextMultiplier) - scale = arith::MulFOp::create(b, loc, scale, multiplier); - memref::StoreOp::create(b, loc, scale, scalePack, lane); - auto cLoop = scf::ForOp::create( - b, loc, depthwise ? lane : zero, - depthwise ? arith::AddIOp::create(b, loc, lane, one).getResult() - : index(b, cin), - one); - b.setInsertionPointToStart(cLoop.getBody()); - Value c = cLoop.getInductionVar(); - auto kLoop = - scf::ForOp::create(b, loc, zero, index(b, kernel * kernel), one); - b.setInsertionPointToStart(kLoop.getBody()); - Value k = kLoop.getInductionVar(); - Value ky = arith::DivUIOp::create(b, loc, k, index(b, kernel)); - Value kx = arith::RemUIOp::create(b, loc, k, index(b, kernel)); - Value w = memref::LoadOp::create( - b, loc, weight, ValueRange{channel, depthwise ? zero : c, ky, kx}); - memref::StoreOp::create(b, loc, w, weightPack, ValueRange{c, k, lane}); - } - Value scales = BankAllocOp::create(b, loc, i64, 1, 1); - scales = BankMvinOp::create(b, loc, i64, scalePack, scales, 4, 1); - pebble::repeat( - b, loc, 0, height, tile, resident, {}, - [&](OpBuilder &b, Value y, ValueRange) { - pebble::repeat( - b, loc, 0, width, tile, resident, {}, - [&](OpBuilder &b, Value x, ValueRange) { - Value acc = BankAllocOp::create(b, loc, i64, 1, 1); - acc = BankMvinOp::create(b, loc, i64, biasPack, acc, 4, 1); - // Depthwise reads the input channels of its own output group. - acc = pebble::accumulate( - b, loc, terms, 16, resident, acc, - [&](OpBuilder &b, Value start, int64_t count, bool first, - bool last, Value acc) { - return group(b, oc, y, x, depthwise ? oc : start, count, - first, last, acc); - }); - Value restored = BankAllocOp::create(b, loc, i64, 1, 1); - if (nextMultiplier) { - auto q = - compute(b, loc, "int32_to_int8", ValueRange{acc, scales}, - restored, false, {{"rows", rows}}); - q->setAttr("relu", b.getBoolAttr(relu)); - restored = q.getDestinationOut(); - int64_t panel = - (constant(oc) / 16 * panelsY + constant(y) / side) * - panelsX + - constant(x) / side; - outputImage.banks[panel] = - compute(b, loc, "maxpool_i8", ValueRange{restored}, - outputImage.banks[panel], true, - {{"input_side", convSide}, - {"output_side", tile}, - {"kernel", pool}, - {"stride", pool}, - {"padding", 0}, - {"op1_base", 0}, - {"wr_base", - constant(y) % side * side + constant(x) % side}, - {"output_stride", side}, - {"start_row", 0}, - {"start_col", 0}}) - .getDestinationOut(); - } else { - auto dequant = - compute(b, loc, "int32_to_fp32", ValueRange{acc, scales}, - restored, false, {{"rows", rows}}); - dequant->setAttr("relu", b.getBoolAttr(false)); - restored = dequant.getDestinationOut(); - restored = BankMvoutOp::create(b, loc, i64, outputPack, - restored, rows * 4, 1); - auto pixels = scf::ForOp::create(b, loc, zero, - index(b, tile * tile), one); - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(pixels.getBody()); - Value pixel = pixels.getInductionVar(); - Value oy = arith::AddIOp::create( - b, loc, y, - arith::DivUIOp::create(b, loc, pixel, index(b, tile))); - Value ox = arith::AddIOp::create( - b, loc, x, - arith::RemUIOp::create(b, loc, pixel, index(b, tile))); - auto channels = - scf::ForOp::create(b, loc, zero, index(b, lanes), one); - b.setInsertionPointToStart(channels.getBody()); - Value c = channels.getInductionVar(); - Value value = memref::LoadOp::create(b, loc, outputPack, - ValueRange{pixel, c}); - memref::StoreOp::create( - b, loc, value, output, - ValueRange{zero, oy, ox, - arith::AddIOp::create(b, loc, c, oc)}); - } - BankReleaseOp::create(b, loc, restored); - BankReleaseOp::create(b, loc, acc); - return SmallVector{}; - }); - return SmallVector{}; - }); - BankReleaseOp::create(b, loc, scales); - }; - int64_t full = cout / 16 * 16; - pebble::repeat(builder, loc, 0, full, 16, resident, {}, - [&](OpBuilder &b, Value oc, ValueRange) { - outputChannels(b, oc, 16); - return SmallVector{}; - }); - if (cout > full) - outputChannels(builder, index(builder, full), cout - full); - - if (inputImage) - for (Value bank : inputImage->banks) - BankReleaseOp::create(builder, loc, bank); - if (!nextMultiplier) - tensorOutput = bufferization::ToTensorOp::create(builder, loc, outputType, - output, true, true); - return success(); -} -} // namespace - -LogicalResult pebble::lowerSpatialOps(func::FuncOp function, - DenseMap &residentVectors, - SmallVector &consumed) { - SmallVector convolutions; - function.walk([&](Computation op) { - if (op.getKind() == "conv2d_i8" || op.getKind() == "depthwise_conv2d_i8") - convolutions.push_back(op); - }); - DenseMap images; - for (auto conv : convolutions) { - SmallVector links{conv}; - auto group = conv->getAttrOfType("custom.spatial_chain"); - if (group) - for (Operation *next = conv->getNextNode(); next; - next = next->getNextNode()) - if (next->getAttr("custom.spatial_chain") == group) - links.push_back(next); - Value output = links.back()->getResult(0); - Value multiplier; - int64_t pool = 1; - bool relu = false, flatten = false; - bool terminalQuant = - links.back()->getName().getStringRef() == "frontend.fp32_to_int8"; - if (group) { - if (terminalQuant) - multiplier = links.back()->getOperand(1); - else - multiplier = - cast(*output.getUsers().begin()).getInputs()[3]; - for (Operation *link : links) { - if (auto op = dyn_cast(link)) { - relu |= op.getKind() == "relu"; - flatten |= op.getKind() == "flatten_nchw"; - if (op.getKind() == "maxpool2d") - pool = cast(op->getAttr("kernel"))[0]; - } - } - } - auto input = images.find(conv.getInputs()[0]); - ImageBanks result; - Value tensor; - if (failed(convolution(conv, - input == images.end() ? nullptr : &input->second, - multiplier, pool, relu, result, tensor))) - return failure(); - if (!group) { - conv.getResult(0).replaceAllUsesWith(tensor); - consumed.push_back(conv); - continue; - } - OpBuilder b(conv); - Value bank = result.banks.front(); - if (terminalQuant) { - Location loc = conv.getLoc(); - auto type = cast(output.getType()); - Value memory = memref::AllocOp::create( - b, loc, MemRefType::get(type.getShape(), b.getI8Type())); - Value pack = memref::AllocOp::create( - b, loc, - MemRefType::get({result.side * result.side, 16}, b.getI8Type())); - int64_t py = (result.height + result.side - 1) / result.side; - int64_t px = (result.width + result.side - 1) / result.side; - auto index = [&](int64_t n) -> Value { - return arith::ConstantIndexOp::create(b, loc, n); - }; - Value zero = index(0), one = index(1); - for (int64_t c = 0; c < result.channels; c += 16) - for (int64_t y = 0; y < result.height; y += result.side) - for (int64_t x = 0; x < result.width; x += result.side) { - Value source = result.banks[(c / 16 * py + y / result.side) * px + - x / result.side]; - BankMvoutOp::create(b, loc, b.getI64Type(), pack, source, - result.side * result.side, 1); - auto rows = scf::ForOp::create( - b, loc, zero, index(std::min(result.side, result.height - y)), - one); - OpBuilder::InsertionGuard guard(b); - b.setInsertionPointToStart(rows.getBody()); - Value iy = rows.getInductionVar(); - auto columns = scf::ForOp::create( - b, loc, zero, index(std::min(result.side, result.width - x)), - one); - b.setInsertionPointToStart(columns.getBody()); - Value ix = columns.getInductionVar(); - auto channels = scf::ForOp::create( - b, loc, zero, index(std::min(16, result.channels - c)), - one); - b.setInsertionPointToStart(channels.getBody()); - Value lane = channels.getInductionVar(); - Value value = memref::LoadOp::create( - b, loc, pack, - ValueRange{ - arith::AddIOp::create( - b, loc, - arith::MulIOp::create(b, loc, iy, index(result.side)), - ix), - lane}); - memref::StoreOp::create( - b, loc, value, memory, - ValueRange{zero, arith::AddIOp::create(b, loc, iy, index(y)), - arith::AddIOp::create(b, loc, ix, index(x)), - arith::AddIOp::create(b, loc, lane, index(c))}); - } - tensor = - bufferization::ToTensorOp::create(b, loc, type, memory, true, true); - output.replaceAllUsesWith(tensor); - for (Value source : result.banks) - BankReleaseOp::create(b, loc, source); - } else if (flatten) { - Value transposed = - BankAllocOp::create(b, conv.getLoc(), b.getI64Type(), 1, 1); - transposed = compute(b, conv.getLoc(), "transpose_i8", ValueRange{bank}, - transposed, false, {{"rows", 16}, {"cols", 16}}) - .getDestinationOut(); - BankReleaseOp::create(b, conv.getLoc(), bank); - bank = transposed; - residentVectors[output] = bank; - } else { - images[output] = result; - } - for (Operation *link : links) { - if (link->getName().getStringRef() == "buddy_trace.end") { - b.setInsertionPoint(link); - Operation *trace = b.clone(*link); - trace->setOperand(0, bank); - trace->getResult(0).setType(bank.getType()); - trace->removeAttr("custom.spatial_chain"); - } - consumed.push_back(link); - } - } - return success(); -} diff --git a/examples/designs/pebble/stack/midend/src/Lowering.h b/examples/designs/pebble/stack/midend/src/Lowering.h deleted file mode 100644 index 13a0708ca..000000000 --- a/examples/designs/pebble/stack/midend/src/Lowering.h +++ /dev/null @@ -1,64 +0,0 @@ -//===- Lowering.h - Custom computations in the Pebble midend -------------===// -#ifndef PEBBLE_LOWERING_H -#define PEBBLE_LOWERING_H - -#include "Custom/CustomOps.h" -#include "mlir/Dialect/Func/IR/FuncOps.h" -#include "mlir/IR/Builders.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/STLFunctionalExtras.h" - -namespace mlir::linalg { -class GenericOp; -} -namespace pebble { -bool elementwise(mlir::linalg::GenericOp op, llvm::StringRef name); -mlir::Value tensorBuffer(mlir::OpBuilder &builder, mlir::Location loc, - mlir::Value tensor); - -// Emits `body` for index = begin, begin + step, ... < end and threads the -// carried banks through it. The iterations share one scf.for unless `unroll` -// is set: then each iteration gets a constant index, for bodies whose index -// selects a bank or sets a Ball attribute. -llvm::SmallVector -repeat(mlir::OpBuilder &builder, mlir::Location loc, int64_t begin, int64_t end, - int64_t step, bool unroll, mlir::ValueRange carried, - llvm::function_ref( - mlir::OpBuilder &, mlir::Value, mlir::ValueRange)> - body); - -// Emits `body` over [0, total) in pieces of `size` that accumulate into one -// bank. The first piece starts the accumulation and the last completes it; -// the pieces between go through `repeat`. -mlir::Value -accumulate(mlir::OpBuilder &builder, mlir::Location loc, int64_t total, - int64_t size, bool unroll, mlir::Value accumulator, - llvm::function_ref - body); - -// A convolution computes tile x tile pooled output pixels per matmul; a -// resident image keeps side x side pixels of sixteen channels per bank. The -// tile divides the output and its input window fits the eight-pixel side that -// MAXPOOL addresses. The tile is zero when none fits. -struct SpatialTile { - int64_t tile, side; -}; -SpatialTile spatialTile(int64_t height, int64_t width, int64_t pool, - int64_t stride, int64_t kernel); - -mlir::LogicalResult lowerLinearTiles(::buddy::custom::CustomOp op, - mlir::Value &result); -mlir::LogicalResult lowerGlobalAverage(::buddy::custom::CustomOp op, - mlir::Value &result); -mlir::LogicalResult lowerChannelOp(::buddy::custom::CustomOp op, - mlir::Value &result); -mlir::LogicalResult lowerTranspose(::buddy::custom::CustomOp op, - mlir::Value &result); -void formSpatialOps(mlir::func::FuncOp function); -mlir::LogicalResult -lowerSpatialOps(mlir::func::FuncOp function, - llvm::DenseMap &residentVectors, - llvm::SmallVector &consumed); -} // namespace pebble -#endif diff --git a/examples/designs/pebble/stack/midend/tests/lower-bank-updates.mlir b/examples/designs/pebble/stack/midend/tests/lower-bank-updates.mlir deleted file mode 100644 index 60daa2b36..000000000 --- a/examples/designs/pebble/stack/midend/tests/lower-bank-updates.mlir +++ /dev/null @@ -1,14 +0,0 @@ -// CHECK-LABEL: func.func @copy_rows -// CHECK: arith.constant -9223372035778936831 : i64 -// CHECK: return -func.func @copy_rows() { - buckyball.custom "maxpool_i8" {input_side = 1 : i64, output_side = 1 : i64, kernel = 1 : i64, stride = 1 : i64, padding = 0 : i64, op1_base = 0 : i64, wr_base = 1 : i64, output_stride = 1 : i64, start_row = 0 : i64, start_col = 0 : i64} - return -} -// CHECK-LABEL: func.func @define_quantized_rows -// CHECK: arith.constant 4298115073 : i64 -// CHECK: return -func.func @define_quantized_rows() { - buckyball.custom "int32_to_int8" {rows = 1 : i64, relu = false} - return -} diff --git a/examples/designs/pebble/workloads/common/pebble.h b/examples/designs/pebble/workloads/common/pebble.h index fec3265d8..2f021f2b4 100644 --- a/examples/designs/pebble/workloads/common/pebble.h +++ b/examples/designs/pebble/workloads/common/pebble.h @@ -2,12 +2,11 @@ #define PEBBLE_H #include "buckyball.h" -#include #include enum { - PEBBLE_INT8_LANES = BANK_WIDTH / 8, - PEBBLE_ACC_LANES = BANK_WIDTH / 8, + PEBBLE_INT8_LANES = THREAD_MEM_BANK_WIDTH / 8, + PEBBLE_ACC_LANES = THREAD_MEM_BANK_WIDTH / 8, }; /* diff --git a/examples/designs/pebble/workloads/device/core/acc/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/CMakeLists.txt index 7b5637b41..d17debe0e 100644 --- a/examples/designs/pebble/workloads/device/core/acc/CMakeLists.txt +++ b/examples/designs/pebble/workloads/device/core/acc/CMakeLists.txt @@ -1,9 +1,9 @@ add_subdirectory(ctests) foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_TARGET}") - foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_TARGET_BALLS_${BUCKYBALL_TARGET}) + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_BALLDOMAIN_BALLS) add_subdirectory( - "${BUCKYBALL_REPO_ROOT}/examples/components/balls/${BUCKYBALL_BALL_DIR}/workloads/ctests" + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/${BUCKYBALL_BALL_DIR}/workloads/ctests" "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}/ctests") endforeach() endforeach() diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/ctests/CMakeLists.txt index 670217993..6fe03733c 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/CMakeLists.txt +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/CMakeLists.txt @@ -23,6 +23,6 @@ add_subdirectory(read_bank_split) add_subdirectory(same_ball_independent) add_subdirectory(rob_window_probe) add_subdirectory(write_bank_versions) -if(DESIGN_TARGET_SHARED_BANKS_${BUCKYBALL_CTEST_TARGET} GREATER 0) +if(DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_CORE_SHARED_MEM_ENABLE) add_subdirectory(shared_bank_versions) endif() diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_capacity_rename_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_capacity_rename_test.c index c94c66a8e..2119c1e1a 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_capacity_rename_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_capacity_rename_test.c @@ -1,35 +1,36 @@ #include -#include #include #include -_Static_assert(VIRTUAL_BANK_NUM >= BANK_NUM, +_Static_assert(CORE_VIRTUAL_BANK_COUNT >= THREAD_MEM_BANK_NUM, "Test requires one virtual ID per physical bank"); -static uint8_t input[BANK_NUM][BANK_WIDTH / 8] __attribute__((aligned(64))); -static uint8_t output[BANK_NUM][BANK_WIDTH / 8] __attribute__((aligned(64))); +static uint8_t input[THREAD_MEM_BANK_NUM][THREAD_MEM_BANK_WIDTH / 8] + __attribute__((aligned(64))); +static uint8_t output[THREAD_MEM_BANK_NUM][THREAD_MEM_BANK_WIDTH / 8] + __attribute__((aligned(64))); int main(void) { - for (unsigned bank = 0; bank < BANK_NUM; ++bank) + for (unsigned bank = 0; bank < THREAD_MEM_BANK_NUM; ++bank) bb_mem_alloc(bank, 1, 1); for (unsigned round = 0; round < 3; ++round) { - for (unsigned bank = 0; bank < BANK_NUM; ++bank) - for (unsigned byte = 0; byte < BANK_WIDTH / 8; ++byte) { + for (unsigned bank = 0; bank < THREAD_MEM_BANK_NUM; ++bank) + for (unsigned byte = 0; byte < THREAD_MEM_BANK_WIDTH / 8; ++byte) { input[bank][byte] = bank * 7 + byte + round * 31; output[bank][byte] = 0; } - for (unsigned bank = 0; bank < BANK_NUM; ++bank) + for (unsigned bank = 0; bank < THREAD_MEM_BANK_NUM; ++bank) bb_mvin((uintptr_t)input[bank], bank, 1, 1); - for (unsigned bank = 0; bank < BANK_NUM; ++bank) + for (unsigned bank = 0; bank < THREAD_MEM_BANK_NUM; ++bank) bb_mvout((uintptr_t)output[bank], bank, 1, 1); - for (unsigned bank = 0; bank < BANK_NUM; ++bank) - for (unsigned byte = 0; byte < BANK_WIDTH / 8; ++byte) + for (unsigned bank = 0; bank < THREAD_MEM_BANK_NUM; ++bank) + for (unsigned byte = 0; byte < THREAD_MEM_BANK_WIDTH / 8; ++byte) if (output[bank][byte] != input[bank][byte]) { printf("capacity rename mismatch round=%u bank=%u byte=%u\n", round, bank, byte); return 1; } } - for (unsigned bank = 0; bank < BANK_NUM; ++bank) + for (unsigned bank = 0; bank < THREAD_MEM_BANK_NUM; ++bank) bb_mem_release(bank); puts("Bank capacity rename PASS"); return 0; diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/bank_dma_pingpong-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/bank_dma_pingpong-main.cpp index b38e76558..c4736d492 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/bank_dma_pingpong-main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/bank_dma_pingpong-main.cpp @@ -12,7 +12,7 @@ int main() { __ATOMIC_RELEASE); __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); } - if (hart == BB_REPORT_HART) { + if (hart == WORKLOAD_REPORT_HART) { while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) asm volatile("nop"); return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/pebble.c index c70aff2d4..a8579daeb 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/pebble.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_pingpong/pebble.c @@ -7,14 +7,14 @@ #include #include -enum { M = 8, N = 16, K = 64, LANES = BANK_WIDTH / 8 }; -static int8_t input[BB_CORE_NUM][M * K] __attribute__((aligned(64))); -static int8_t first[BB_CORE_NUM][K * N] __attribute__((aligned(64))); -static int8_t second[BB_CORE_NUM][K * N] __attribute__((aligned(64))); -static int32_t bias[BB_CORE_NUM][N] __attribute__((aligned(64))); -static int32_t out0[BB_CORE_NUM][M * N] __attribute__((aligned(64))); -static int32_t out1[BB_CORE_NUM][M * N] __attribute__((aligned(64))); -static volatile int32_t warmup[BB_CORE_NUM][N] __attribute__((aligned(64))); +enum { M = 8, N = 16, K = 64, LANES = THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t input[BB_NUM_CORES][M * K] __attribute__((aligned(64))); +static int8_t first[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int8_t second[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][N] __attribute__((aligned(64))); +static int32_t out0[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static int32_t out1[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_NUM_CORES][N] __attribute__((aligned(64))); extern "C" int pebble_pipeline(uint32_t instance) { int c = instance; for (int i = 0; i < M * K; ++i) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/bank_dma_reuse-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/bank_dma_reuse-main.cpp index b38e76558..c4736d492 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/bank_dma_reuse-main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/bank_dma_reuse-main.cpp @@ -12,7 +12,7 @@ int main() { __ATOMIC_RELEASE); __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); } - if (hart == BB_REPORT_HART) { + if (hart == WORKLOAD_REPORT_HART) { while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) asm volatile("nop"); return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/pebble.c index bea6e5826..222236cd0 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/pebble.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_dma_reuse/pebble.c @@ -7,15 +7,15 @@ #include #include -enum { M = 8, N = 16, K = 64, LANES = BANK_WIDTH / 8 }; -static int8_t input[BB_CORE_NUM][M * K] __attribute__((aligned(64))); -static int8_t first[BB_CORE_NUM][K * N] __attribute__((aligned(64))); -static int8_t second[BB_CORE_NUM][K * N] __attribute__((aligned(64))); -static int32_t bias[BB_CORE_NUM][N] __attribute__((aligned(64))); -static int32_t out0[BB_CORE_NUM][M * N] __attribute__((aligned(64))); -static int32_t out1[BB_CORE_NUM][M * N] __attribute__((aligned(64))); -static volatile int32_t warmup[BB_CORE_NUM][N] __attribute__((aligned(64))); -static volatile int8_t moved[BB_CORE_NUM][K * N] __attribute__((aligned(64))); +enum { M = 8, N = 16, K = 64, LANES = THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t input[BB_NUM_CORES][M * K] __attribute__((aligned(64))); +static int8_t first[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int8_t second[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][N] __attribute__((aligned(64))); +static int32_t out0[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static int32_t out1[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_NUM_CORES][N] __attribute__((aligned(64))); +static volatile int8_t moved[BB_NUM_CORES][K * N] __attribute__((aligned(64))); extern "C" int pebble_pipeline(uint32_t instance) { int c = instance; for (int i = 0; i < M * K; ++i) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_recycle_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_recycle_test.c index 76a9aef7f..4b8ab0b40 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_recycle_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_recycle_test.c @@ -1,5 +1,4 @@ #include -#include #include #include diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/bank_rename_overlap-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/bank_rename_overlap-main.cpp index b38e76558..c4736d492 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/bank_rename_overlap-main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/bank_rename_overlap-main.cpp @@ -12,7 +12,7 @@ int main() { __ATOMIC_RELEASE); __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); } - if (hart == BB_REPORT_HART) { + if (hart == WORKLOAD_REPORT_HART) { while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) asm volatile("nop"); return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/pebble.c index c8d99e78c..a4660ee94 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/pebble.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_rename_overlap/pebble.c @@ -6,14 +6,14 @@ #include #include -enum { M = 8, N = 16, K = 64, LANES = BANK_WIDTH / 8 }; -static int8_t input[BB_CORE_NUM][M * K] __attribute__((aligned(64))); -static int8_t first[BB_CORE_NUM][K * N] __attribute__((aligned(64))); -static int8_t second[BB_CORE_NUM][K * N] __attribute__((aligned(64))); -static int32_t bias[BB_CORE_NUM][N] __attribute__((aligned(64))); -static int32_t out0[BB_CORE_NUM][M * N] __attribute__((aligned(64))); -static int32_t out1[BB_CORE_NUM][M * N] __attribute__((aligned(64))); -static volatile int32_t warmup[BB_CORE_NUM][N] __attribute__((aligned(64))); +enum { M = 8, N = 16, K = 64, LANES = THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t input[BB_NUM_CORES][M * K] __attribute__((aligned(64))); +static int8_t first[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int8_t second[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][N] __attribute__((aligned(64))); +static int32_t out0[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static int32_t out1[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_NUM_CORES][N] __attribute__((aligned(64))); extern "C" int pebble_pipeline(uint32_t instance) { int c = instance; for (int i = 0; i < M * K; ++i) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_transfer_recycle_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_transfer_recycle_test.c index e30241516..be5fb47ec 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/bank_transfer_recycle_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/bank_transfer_recycle_test.c @@ -1,5 +1,4 @@ #include -#include #include #include diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_mvin_smatmul_hazard_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_mvin_smatmul_hazard_test.c index ba7577460..0e8be177f 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_mvin_smatmul_hazard_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_mvin_smatmul_hazard_test.c @@ -6,7 +6,7 @@ #include #include -static int8_t input[4096 * 16] __attribute__((aligned(128))); +static int8_t input[THREAD_MEM_BANK_ENTRIES * 16] __attribute__((aligned(128))); static int8_t pixel_input[28 * 24] __attribute__((aligned(128))); static int8_t weights[16 * 16] __attribute__((aligned(128))); static const int32_t bias[16] __attribute__((aligned(128))); @@ -46,7 +46,7 @@ int main(void) { bb_mvin((uintptr_t)bias, 4, 4, 1); bb_smatmul_bias(4, 0); bb_mvin((uintptr_t)input, 8, 18, 1); - bb_mvin((uintptr_t)input, 8, 4096, 1); + bb_mvin((uintptr_t)input, 8, THREAD_MEM_BANK_ENTRIES, 1); bb_mvin_2d((uintptr_t)pixel_input, 8, 1, 24, 28, 6, 3, 16); bb_mvin_2d((uintptr_t)(pixel_input + 16), 8, 1, 24, 28, 15, 3, 8); for (int iteration = 0; iteration < iterations; ++iteration) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp index b38e76558..c4736d492 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/im2col_smatmul_quant_pipeline-main.cpp @@ -12,7 +12,7 @@ int main() { __ATOMIC_RELEASE); __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); } - if (hart == BB_REPORT_HART) { + if (hart == WORKLOAD_REPORT_HART) { while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) asm volatile("nop"); return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/pebble.c index 6498865d3..9b0dbbb98 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/pebble.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/im2col_smatmul_quant_pipeline/pebble.c @@ -7,18 +7,24 @@ #include #include -#if BANK_WIDTH != 128 || BANK_LINES < 64 +#if THREAD_MEM_BANK_WIDTH != 128 || THREAD_MEM_BANK_ENTRIES < 64 #error "Pebble thread pipeline requires 128-bit banks with at least 64 rows" #endif // K is (kernel row, kernel column, channel); only channel 0 is nonzero. -enum { LANES = BANK_WIDTH / 8, INPUT_ROWS = 36, WINDOWS = 16, K = 9 * LANES }; -static int8_t input[BB_CORE_NUM][INPUT_ROWS * LANES] +enum { + LANES = THREAD_MEM_BANK_WIDTH / 8, + INPUT_ROWS = 36, + WINDOWS = 16, + K = 9 * LANES +}; +static int8_t input[BB_NUM_CORES][INPUT_ROWS * LANES] + __attribute__((aligned(64))); +static int8_t weight[BB_NUM_CORES][K * LANES] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][LANES] __attribute__((aligned(64))); +static float scale[BB_NUM_CORES][LANES] __attribute__((aligned(64))); +static int8_t output[BB_NUM_CORES][WINDOWS * LANES] __attribute__((aligned(64))); -static int8_t weight[BB_CORE_NUM][K * LANES] __attribute__((aligned(64))); -static int32_t bias[BB_CORE_NUM][LANES] __attribute__((aligned(64))); -static float scale[BB_CORE_NUM][LANES] __attribute__((aligned(64))); -static int8_t output[BB_CORE_NUM][WINDOWS * LANES] __attribute__((aligned(64))); #ifdef __cplusplus extern "C" @@ -38,8 +44,8 @@ extern "C" bb_mem_alloc(bank, 1, 1); bb_mvin((uintptr_t)input[thread], 0, INPUT_ROWS, 1); bb_mvin((uintptr_t)weight[thread], 2, K, 1); - bb_mvin((uintptr_t)bias[thread], 3, LANES * 32 / BANK_WIDTH, 1); - bb_mvin((uintptr_t)scale[thread], 5, LANES * 32 / BANK_WIDTH, 1); + bb_mvin((uintptr_t)bias[thread], 3, LANES * 32 / THREAD_MEM_BANK_WIDTH, 1); + bb_mvin((uintptr_t)scale[thread], 5, LANES * 32 / THREAD_MEM_BANK_WIDTH, 1); bb_im2col(0, 1, 6, 3, 1, 0, 0, 0, 0, 0, WINDOWS, 8); bb_smatmul_bias(3, 0); bb_smatmul_os(1, 2, 4, WINDOWS, LANES, K, 1, 1, 0); diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_im2col_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_im2col_test.c index 00a0d7ca8..ec7c22817 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_im2col_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_im2col_test.c @@ -6,7 +6,7 @@ #include #include -enum { LANES = BANK_WIDTH / 8, INPUT_ROWS = 36, OUTPUT_ROWS = 9 }; +enum { LANES = THREAD_MEM_BANK_WIDTH / 8, INPUT_ROWS = 36, OUTPUT_ROWS = 9 }; static int8_t input[INPUT_ROWS * LANES] __attribute__((aligned(64))); static int8_t output[OUTPUT_ROWS * LANES] __attribute__((aligned(64))); diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/maxpool_partial_update-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/maxpool_partial_update-main.cpp index b38e76558..c4736d492 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/maxpool_partial_update-main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/maxpool_partial_update-main.cpp @@ -12,7 +12,7 @@ int main() { __ATOMIC_RELEASE); __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); } - if (hart == BB_REPORT_HART) { + if (hart == WORKLOAD_REPORT_HART) { while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) asm volatile("nop"); return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/pebble.c index 7a3caef34..7900bd9f1 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/pebble.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/maxpool_partial_update/pebble.c @@ -5,10 +5,10 @@ #include #include -enum { ROWS = 16, LANES = BANK_WIDTH / 8 }; -static int8_t input[BB_CORE_NUM][2 * LANES] __attribute__((aligned(64))); -static int8_t initial[BB_CORE_NUM][ROWS * LANES] __attribute__((aligned(64))); -static int8_t output[BB_CORE_NUM][ROWS * LANES] __attribute__((aligned(64))); +enum { ROWS = 16, LANES = THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t input[BB_NUM_CORES][2 * LANES] __attribute__((aligned(64))); +static int8_t initial[BB_NUM_CORES][ROWS * LANES] __attribute__((aligned(64))); +static int8_t output[BB_NUM_CORES][ROWS * LANES] __attribute__((aligned(64))); extern "C" int pebble_pipeline(uint32_t instance) { int c = instance; diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/quant_smatmul_int2fp_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/quant_smatmul_int2fp_test.c index 63f4d4247..0a57e0359 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/quant_smatmul_int2fp_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/quant_smatmul_int2fp_test.c @@ -11,8 +11,8 @@ enum { M = 16, N = 16, K = 16, - F32_ROWS = M * N * 32 / BANK_WIDTH, - CHANNEL_ROWS = N * 32 / BANK_WIDTH + F32_ROWS = M * N * 32 / THREAD_MEM_BANK_WIDTH, + CHANNEL_ROWS = N * 32 / THREAD_MEM_BANK_WIDTH }; static float input[M * K] __attribute__((aligned(64))); static int8_t weight[K * N] __attribute__((aligned(64))); diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/pebble.c index 461d5b99e..c473ff832 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/pebble.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/pebble.c @@ -7,13 +7,13 @@ #include #include -enum { M = 8, N = 16, K = 16, LANES = BANK_WIDTH / 8 }; -static int8_t input[BB_CORE_NUM][K * N] __attribute__((aligned(64))); -static int8_t weight[BB_CORE_NUM][K * N] __attribute__((aligned(64))); -static int32_t bias[BB_CORE_NUM][N] __attribute__((aligned(64))); -static int8_t transposed[BB_CORE_NUM][K * N] __attribute__((aligned(64))); -static int32_t product[BB_CORE_NUM][M * N] __attribute__((aligned(64))); -static volatile int32_t warmup[BB_CORE_NUM][N] __attribute__((aligned(64))); +enum { M = 8, N = 16, K = 16, LANES = THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t input[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int8_t weight[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][N] __attribute__((aligned(64))); +static int8_t transposed[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t product[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_NUM_CORES][N] __attribute__((aligned(64))); extern "C" int pebble_pipeline(uint32_t instance) { int c = instance; for (int i = 0; i < K * N; ++i) { diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/read_bank_shared-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/read_bank_shared-main.cpp index b38e76558..c4736d492 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/read_bank_shared-main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_shared/read_bank_shared-main.cpp @@ -12,7 +12,7 @@ int main() { __ATOMIC_RELEASE); __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); } - if (hart == BB_REPORT_HART) { + if (hart == WORKLOAD_REPORT_HART) { while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) asm volatile("nop"); return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/pebble.c index 2c154df41..1ee586914 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/pebble.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/pebble.c @@ -7,13 +7,13 @@ #include #include -enum { M = 8, N = 16, K = 16, LANES = BANK_WIDTH / 8 }; -static int8_t input[BB_CORE_NUM][K * N] __attribute__((aligned(64))); -static int8_t weight[BB_CORE_NUM][K * N] __attribute__((aligned(64))); -static int32_t bias[BB_CORE_NUM][N] __attribute__((aligned(64))); -static int8_t transposed[BB_CORE_NUM][K * N] __attribute__((aligned(64))); -static int32_t product[BB_CORE_NUM][M * N] __attribute__((aligned(64))); -static volatile int32_t warmup[BB_CORE_NUM][N] __attribute__((aligned(64))); +enum { M = 8, N = 16, K = 16, LANES = THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t input[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int8_t weight[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][N] __attribute__((aligned(64))); +static int8_t transposed[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t product[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_NUM_CORES][N] __attribute__((aligned(64))); extern "C" int pebble_pipeline(uint32_t instance) { int c = instance; for (int i = 0; i < K * N; ++i) { diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/read_bank_split-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/read_bank_split-main.cpp index b38e76558..c4736d492 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/read_bank_split-main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/read_bank_split/read_bank_split-main.cpp @@ -12,7 +12,7 @@ int main() { __ATOMIC_RELEASE); __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); } - if (hart == BB_REPORT_HART) { + if (hart == WORKLOAD_REPORT_HART) { while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) asm volatile("nop"); return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/residual_add_lut_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/residual_add_lut_test.c index 0f4ad64a3..ce7e822a7 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/residual_add_lut_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/residual_add_lut_test.c @@ -6,7 +6,7 @@ #include #include -enum { ROWS = 4, LANES = BANK_WIDTH / 8, VALUES = ROWS * LANES }; +enum { ROWS = 4, LANES = THREAD_MEM_BANK_WIDTH / 8, VALUES = ROWS * LANES }; static int8_t lhs[VALUES] __attribute__((aligned(64))); static int8_t rhs[VALUES] __attribute__((aligned(64))); static int8_t table[256] __attribute__((aligned(64))); diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/pebble.c index 0614a56e5..01d3b09f9 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/pebble.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/pebble.c @@ -7,14 +7,14 @@ #include #include -enum { M = 8, N = 16, K = 64, LANES = BANK_WIDTH / 8, STORES = 16 }; -static int8_t input[BB_CORE_NUM][M * K] __attribute__((aligned(64))); -static int8_t weight[BB_CORE_NUM][K * N] __attribute__((aligned(64))); -static int8_t core[BB_CORE_NUM][N * N] __attribute__((aligned(64))); -static int32_t bias[BB_CORE_NUM][N] __attribute__((aligned(64))); -static int32_t output[BB_CORE_NUM][STORES][M * N] __attribute__((aligned(64))); -static int8_t transposed[BB_CORE_NUM][N * N] __attribute__((aligned(64))); -static volatile int32_t warmup[BB_CORE_NUM][N] __attribute__((aligned(64))); +enum { M = 8, N = 16, K = 64, LANES = THREAD_MEM_BANK_WIDTH / 8, STORES = 16 }; +static int8_t input[BB_NUM_CORES][M * K] __attribute__((aligned(64))); +static int8_t weight[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int8_t core[BB_NUM_CORES][N * N] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][N] __attribute__((aligned(64))); +static int32_t output[BB_NUM_CORES][STORES][M * N] __attribute__((aligned(64))); +static int8_t transposed[BB_NUM_CORES][N * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_NUM_CORES][N] __attribute__((aligned(64))); extern "C" int pebble_pipeline(uint32_t instance) { int c = instance; for (int i = 0; i < M * K; ++i) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/rob_window_probe-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/rob_window_probe-main.cpp index b38e76558..c4736d492 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/rob_window_probe-main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/rob_window_probe/rob_window_probe-main.cpp @@ -12,7 +12,7 @@ int main() { __ATOMIC_RELEASE); __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); } - if (hart == BB_REPORT_HART) { + if (hart == WORKLOAD_REPORT_HART) { while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) asm volatile("nop"); return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/runtime_job_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/runtime_job_test.c index 87e08e410..11237d7a7 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/runtime_job_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/runtime_job_test.c @@ -1,8 +1,7 @@ // Two runtime jobs on the core under test: the first moves data through a // bank of its accelerator, the second depends on it and checks the data. +#include #include -#include -#include #include #include #include diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/pebble.c index 10face5b5..60c357fed 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/pebble.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/pebble.c @@ -7,15 +7,15 @@ #include #include -enum { M = 8, N = 16, K = 64, LANES = BANK_WIDTH / 8 }; -static int8_t a0[BB_CORE_NUM][M * K] __attribute__((aligned(64))); -static int8_t b0[BB_CORE_NUM][K * N] __attribute__((aligned(64))); -static int8_t a1[BB_CORE_NUM][M * K] __attribute__((aligned(64))); -static int8_t b1[BB_CORE_NUM][K * N] __attribute__((aligned(64))); -static int32_t bias[BB_CORE_NUM][N] __attribute__((aligned(64))); -static int32_t out0[BB_CORE_NUM][M * N] __attribute__((aligned(64))); -static int32_t out1[BB_CORE_NUM][M * N] __attribute__((aligned(64))); -static volatile int32_t warmup[BB_CORE_NUM][N] __attribute__((aligned(64))); +enum { M = 8, N = 16, K = 64, LANES = THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t a0[BB_NUM_CORES][M * K] __attribute__((aligned(64))); +static int8_t b0[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int8_t a1[BB_NUM_CORES][M * K] __attribute__((aligned(64))); +static int8_t b1[BB_NUM_CORES][K * N] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][N] __attribute__((aligned(64))); +static int32_t out0[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static int32_t out1[BB_NUM_CORES][M * N] __attribute__((aligned(64))); +static volatile int32_t warmup[BB_NUM_CORES][N] __attribute__((aligned(64))); extern "C" int pebble_pipeline(uint32_t instance) { int c = instance; for (int i = 0; i < M * K; ++i) { diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/same_ball_independent-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/same_ball_independent-main.cpp index b38e76558..c4736d492 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/same_ball_independent-main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/same_ball_independent/same_ball_independent-main.cpp @@ -12,7 +12,7 @@ int main() { __ATOMIC_RELEASE); __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); } - if (hart == BB_REPORT_HART) { + if (hart == WORKLOAD_REPORT_HART) { while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) asm volatile("nop"); return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/se_gate_lut_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/se_gate_lut_test.c index 71295e5c3..c0a351f3b 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/se_gate_lut_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/se_gate_lut_test.c @@ -6,7 +6,7 @@ #include #include -enum { ROWS = 4, LANES = BANK_WIDTH / 8, VALUES = ROWS * LANES }; +enum { ROWS = 4, LANES = THREAD_MEM_BANK_WIDTH / 8, VALUES = ROWS * LANES }; static int8_t gate[LANES] __attribute__((aligned(64))); static int8_t input[VALUES] __attribute__((aligned(64))); static int8_t table[256] __attribute__((aligned(64))); diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/pebble.c index 3f478dfbd..492779b2a 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/pebble.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/pebble.c @@ -10,14 +10,14 @@ enum { ROWS = 16, NEW_ROWS = 8, - BYTES = ROWS * BANK_WIDTH / 8, - BASE = BB_SHARED_BANK_BASE + BYTES = ROWS * THREAD_MEM_BANK_WIDTH / 8, + BASE = THREAD_FRONTEND_SHARED_VBANK_BASE }; -static int8_t input[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); -static int8_t left[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); -static int8_t right[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); -static volatile int8_t warmup[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); -static int8_t output[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); +static int8_t input[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static int8_t left[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static int8_t right[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static volatile int8_t warmup[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static int8_t output[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); extern "C" int pebble_pipeline(uint32_t instance) { int c = instance; @@ -37,7 +37,7 @@ extern "C" int pebble_pipeline(uint32_t instance) { bb_transpose(BASE, BASE + 2, ROWS, 8); bb_int8add(BASE + 3, BASE + 4, BASE + 2, NEW_ROWS, 1.0f, 1.0f); bb_mvout((uintptr_t)output[c], BASE + 2, NEW_ROWS, 1); - for (int i = 0; i < NEW_ROWS * BANK_WIDTH / 8; ++i) + for (int i = 0; i < NEW_ROWS * THREAD_MEM_BANK_WIDTH / 8; ++i) if (output[c][i] != 7) return 1; for (int bank = BASE; bank < BASE + 5; ++bank) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/shared_bank_versions-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/shared_bank_versions-main.cpp index b38e76558..c4736d492 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/shared_bank_versions-main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/shared_bank_versions/shared_bank_versions-main.cpp @@ -12,7 +12,7 @@ int main() { __ATOMIC_RELEASE); __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); } - if (hart == BB_REPORT_HART) { + if (hart == WORKLOAD_REPORT_HART) { while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) asm volatile("nop"); return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/smatmul_mvout_bank_reuse_test.c b/examples/designs/pebble/workloads/device/core/acc/ctests/smatmul_mvout_bank_reuse_test.c index 9af9e9ad4..377c23e3f 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/smatmul_mvout_bank_reuse_test.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/smatmul_mvout_bank_reuse_test.c @@ -7,8 +7,8 @@ enum { DIM = 16, - OUTPUT_ROWS = DIM * DIM * 32 / BANK_WIDTH, - BIAS_ROWS = DIM * 32 / BANK_WIDTH, + OUTPUT_ROWS = DIM * DIM * 32 / THREAD_MEM_BANK_WIDTH, + BIAS_ROWS = DIM * 32 / THREAD_MEM_BANK_WIDTH, ITERATIONS = 24 }; static int8_t input[DIM * DIM] __attribute__((aligned(64))); diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/pebble.c b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/pebble.c index cd8d4571f..11f3418db 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/pebble.c +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/pebble.c @@ -7,12 +7,12 @@ #include #include -enum { ROWS = 16, NEW_ROWS = 8, BYTES = ROWS * BANK_WIDTH / 8 }; -static int8_t input[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); -static int8_t left[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); -static int8_t right[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); -static volatile int8_t warmup[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); -static int8_t output[BB_CORE_NUM][BYTES] __attribute__((aligned(64))); +enum { ROWS = 16, NEW_ROWS = 8, BYTES = ROWS * THREAD_MEM_BANK_WIDTH / 8 }; +static int8_t input[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static int8_t left[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static int8_t right[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static volatile int8_t warmup[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); +static int8_t output[BB_NUM_CORES][BYTES] __attribute__((aligned(64))); extern "C" int pebble_pipeline(uint32_t instance) { int c = instance; @@ -32,7 +32,7 @@ extern "C" int pebble_pipeline(uint32_t instance) { bb_transpose(0, 2, ROWS, 8); bb_int8add(3, 4, 2, NEW_ROWS, 1.0f, 1.0f); bb_mvout((uintptr_t)output[c], 2, NEW_ROWS, 1); - for (int i = 0; i < NEW_ROWS * BANK_WIDTH / 8; ++i) + for (int i = 0; i < NEW_ROWS * THREAD_MEM_BANK_WIDTH / 8; ++i) if (output[c][i] != 7) return 1; for (int bank = 0; bank < 5; ++bank) diff --git a/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/write_bank_versions-main.cpp b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/write_bank_versions-main.cpp index b38e76558..c4736d492 100644 --- a/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/write_bank_versions-main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/ctests/write_bank_versions/write_bank_versions-main.cpp @@ -12,7 +12,7 @@ int main() { __ATOMIC_RELEASE); __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); } - if (hart == BB_REPORT_HART) { + if (hart == WORKLOAD_REPORT_HART) { while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) asm volatile("nop"); return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir index 36d776314..58d793943 100644 --- a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel.mlir @@ -1,6 +1,6 @@ // Design integration: f32-to-i8 quantization -> im2col k=1 -> mvout. -func.func private @check_result(memref<16x16xi8>) -> () +func.func private @check_result(memref<8x16xi8>) -> () memref.global "private" constant @fp_in : memref<4x4xf32> = dense<[[1.0, 2.0, 3.0, 4.0], [0.0, 0.0, 0.0, 0.0], @@ -11,20 +11,20 @@ func.func @main() -> i8 { %scale = arith.constant 1.0 : f32 %input = memref.get_global @fp_in : memref<4x4xf32> - %output = memref.alloc() alignment = 64 : memref<16x16xi8> + %output = memref.alloc() alignment = 64 : memref<8x16xi8> %bin = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %bq = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %bout = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded = buckyball.bank_mvin %input, %bin <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - %q = buckyball.bank_custom "fp32_to_int8" (%loaded) -> %bq args(%scale : f32) {rows = 4 : i64} - %im = buckyball.bank_custom "im2col_i8" (%q) -> %bout {input_side = 1 : i64, kernel = 1 : i64, stride = 1 : i64, padding = 0 : i64, start_row = 0 : i64, start_col = 0 : i64, op1_base = 0 : i64, lane = 0 : i64, window_start = 0 : i64, window_count = 1 : i64} - %stored = buckyball.bank_mvout %output, %im <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + %q = toint8_ball.bank_fp32_to_int8 %loaded -> %bq multiplier(%scale) <{rows = 4 : i64}> + %im = im2col_ball.bank_im2col_i8 %q -> %bout <{input_side = 1 : i64, kernel = 1 : i64, stride = 1 : i64, padding = 0 : i64, tile_rows = 8 : i64}> + %stored = buckyball.bank_mvout %output, %im <{depth = 8 : i64, stride = 1 : i64}> : memref<8x16xi8> buckyball.bank_release %loaded buckyball.bank_release %q buckyball.bank_release %stored - func.call @check_result(%output) : (memref<16x16xi8>) -> () - memref.dealloc %output : memref<16x16xi8> + func.call @check_result(%output) : (memref<8x16xi8>) -> () + memref.dealloc %output : memref<8x16xi8> return %zero_i8 : i8 } diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel_main.cpp index 6d3a50db5..9dc2953be 100644 --- a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel_main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/bank/quant_im2col_k1_single_pixel_main.cpp @@ -20,7 +20,7 @@ extern "C" int64_t size0, int64_t size1, int64_t stride0, int64_t stride1) { (void)allocated; - if (size0 != 16 || size1 != 16 || stride0 != 16 || stride1 != 1) { + if (size0 != 8 || size1 != 16 || stride0 != 16 || stride1 != 1) { printf( "FAILED: quant_im2col_k1_single_pixel shape %ldx%ld stride %ldx%ld\n", (long)size0, (long)size1, (long)stride0, (long)stride1); @@ -28,11 +28,15 @@ extern "C" } int8_t *out = aligned + offset; - int32_t got = out[0 * stride0 + 0 * stride1]; - if (got != 1) { - printf("FAILED: quant_im2col_k1_single_pixel out[0][0] exp=1 got=%d\n", - (int)got); - fail(); + // The one window is line 0: the sixteen channels of the pixel. + for (int channel = 0; channel < 16; ++channel) { + int expected = channel < 4 ? channel + 1 : 0; + int got = out[channel * stride1]; + if (got != expected) { + printf("FAILED: quant_im2col_k1_single_pixel channel=%d exp=%d got=%d\n", + channel, expected, got); + fail(); + } } printf("PASSED: quant_im2col_k1_single_pixel\n"); } diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/CMakeLists.txt b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/CMakeLists.txt index 86c740952..78a4ba6ef 100644 --- a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/CMakeLists.txt +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/CMakeLists.txt @@ -160,6 +160,60 @@ add_buckyball_mlir_test(mega_conv2d -convert-func-to-llvm -reconcile-unrealized-casts) +add_buckyball_mlir_test(mega_conv2d_resident + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + +add_buckyball_mlir_test(mega_conv2d_flatten_linear + TARGET pebble + PASSES + -allow-unregistered-dialect + -form-custom-ops + -convert-linalg-to-buckyball + -lower-frontend + -eliminate-empty-tensors + -one-shot-bufferize=bufferize-function-boundaries + -ownership-based-buffer-deallocation + -buffer-deallocation-simplification + -bufferization-lower-deallocations + -convert-trace-to-llvm + -assign-physical-banks + -lower-buckyball + -convert-linalg-to-loops + -expand-strided-metadata + -lower-affine + -convert-scf-to-cf + -convert-cf-to-llvm + -convert-math-to-llvm + -convert-math-to-libm + -convert-arith-to-llvm + -finalize-memref-to-llvm + -convert-func-to-llvm + -reconcile-unrealized-casts) + add_buckyball_mlir_test(mega_conv2d_k5_tail TARGET pebble PASSES diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8.mlir index cefe53df5..21033212f 100644 --- a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8.mlir +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/linear_chain_i8.mlir @@ -1,3 +1,8 @@ +// FORM: custom.op "linear_i8"{{.*}}custom.resident, relu = true{{.*}}-> tensor<1x32xi8> +// BANK: toint8_ball.bank_int32_to_int8 +// BANK: buckyball.bank_transfer +// BANK-NOT: buckyball.bank_mvout +// BANK: smatmul_ball.bank_matmul_i8 {{.*}}k = 16> func.func private @check_result(memref<1x16xf32>) func.func @main() -> i8 { diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir index d065e7c45..bd62e2fbb 100644 --- a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d.mlir @@ -1,9 +1,9 @@ -// FORM: custom.op "conv2d_i8"{{.*}}custom.spatial_chain -// FORM: custom.op "relu"{{.*}}custom.spatial_chain -// FORM: custom.op "depthwise_conv2d_i8"{{.*}}custom.spatial_chain -// FORM: custom.op "maxpool2d"{{.*}}custom.spatial_chain -// FORM: "frontend.fp32_to_int8"{{.*}}custom.spatial_chain -// BANK: buckyball.bank_mvout +// FORM: %[[FIRST:.*]] = custom.op "conv2d_i8"{{.*}}pool = 1 : i64, relu = true{{.*}}-> tensor<1x4x4x16xi8> +// FORM: custom.op "depthwise_conv2d_i8"(%[[FIRST]],{{.*}}pool = 2 : i64, relu = false{{.*}}-> tensor<1x2x2x16xi8> +// FORM-NOT: frontend.fp32_to_int8 +// BANK: toint8_ball.bank_int32_to_int8 {{.*}}relu> +// BANK: im2col_ball.bank_im2col_i8 +// BANK: maxpool_ball.bank_maxpool_i8 // BANK: return func.func private @check_result(memref<1x16x2x2xi8>) -> () diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_flatten_linear.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_flatten_linear.mlir new file mode 100644 index 000000000..87be183f6 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_flatten_linear.mlir @@ -0,0 +1,93 @@ +// The pooled map stays in its bank and feeds the Linear transposed. +// FORM: custom.op "conv2d_i8"{{.*}}custom.resident{{.*}}pool = 2 : i64, relu = true +// FORM: custom.op "flatten_nchw" +// BANK: maxpool_ball.bank_maxpool_i8 +// BANK-NOT: buckyball.bank_mvout +// BANK: transpose_ball.bank_transpose_i8 +// BANK: smatmul_ball.bank_matmul_i8 +func.func private @check_result(memref<1x16xf32>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c8 = arith.constant 8 : index + %c9 = arith.constant 9 : index + %c16 = arith.constant 16 : index + %c256 = arith.constant 256 : index + %input = memref.alloc() alignment = 64 : memref<1x8x8x2xi8> + %conv_weight = memref.alloc() alignment = 64 : memref<16x2x3x3xi8> + %conv_bias = memref.alloc() alignment = 64 : memref<16xi32> + %linear_weight = memref.alloc() alignment = 64 : memref<256x16xi8> + %linear_bias = memref.alloc() alignment = 64 : memref<1x16xi32> + scf.for %h = %c0 to %c8 step %c1 { + scf.for %w = %c0 to %c8 step %c1 { + scf.for %ic = %c0 to %c2 step %c1 { + %h3 = arith.muli %h, %c3 : index + %w2 = arith.muli %w, %c2 : index + %sum0 = arith.addi %h3, %w2 : index + %sum = arith.addi %sum0, %ic : index + %rem = arith.remui %sum, %c7 : index + %signed = arith.subi %rem, %c3 : index + %value = arith.index_cast %signed : index to i8 + memref.store %value, %input[%c0, %h, %w, %ic] : memref<1x8x8x2xi8> + } + } + } + scf.for %oc = %c0 to %c16 step %c1 { + scf.for %ic = %c0 to %c2 step %c1 { + scf.for %p = %c0 to %c9 step %c1 { + %oc3 = arith.muli %oc, %c3 : index + %sum0 = arith.addi %p, %ic : index + %sum = arith.addi %sum0, %oc3 : index + %rem = arith.remui %sum, %c5 : index + %signed = arith.subi %rem, %c2 : index + %value = arith.index_cast %signed : index to i8 + %ky = arith.divui %p, %c3 : index + %kx = arith.remui %p, %c3 : index + memref.store %value, %conv_weight[%oc, %ic, %ky, %kx] : memref<16x2x3x3xi8> + } + } + %bias = arith.index_cast %oc : index to i32 + memref.store %bias, %conv_bias[%oc] : memref<16xi32> + memref.store %bias, %linear_bias[%c0, %oc] : memref<1x16xi32> + scf.for %k = %c0 to %c256 step %c1 { + %sum = arith.addi %k, %oc : index + %rem = arith.remui %sum, %c5 : index + %signed = arith.subi %rem, %c2 : index + %value = arith.index_cast %signed : index to i8 + memref.store %value, %linear_weight[%k, %oc] : memref<256x16xi8> + } + } + %unit = arith.constant dense<1.0> : tensor<1x1x1x1xf32> + %step = arith.constant dense<1.0> : tensor<1x1xf32> + %conv_step = arith.constant dense<1.0> : tensor<16x1x1x1xf32> + %linear_step = arith.constant dense<1.0> : tensor<1x16xf32> + %it = bufferization.to_tensor %input restrict : memref<1x8x8x2xi8> to tensor<1x8x8x2xi8> + %x = "frontend.int8_to_fp32"(%it, %unit) : (tensor<1x8x8x2xi8>, tensor<1x1x1x1xf32>) -> tensor<1x8x8x2xf32> + %wt = bufferization.to_tensor %conv_weight restrict : memref<16x2x3x3xi8> to tensor<16x2x3x3xi8> + %bt = bufferization.to_tensor %conv_bias restrict : memref<16xi32> to tensor<16xi32> + %lw = bufferization.to_tensor %linear_weight restrict : memref<256x16xi8> to tensor<256x16xi8> + %lb = bufferization.to_tensor %linear_bias restrict : memref<1x16xi32> to tensor<1x16xi32> + %conv = custom.op "conv2d_i8"(%x, %wt, %bt, %unit, %unit, %conv_step) {strides = array, dilations = array, padding = array} : (tensor<1x8x8x2xf32>, tensor<16x2x3x3xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x8x8x16xf32> + %zero = arith.constant dense<0.0> : tensor<1x8x8x16xf32> + %relu = linalg.generic {indexing_maps = [affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%conv, %zero: tensor<1x8x8x16xf32>, tensor<1x8x8x16xf32>) outs(%zero: tensor<1x8x8x16xf32>) { + ^bb0(%a: f32, %b: f32, %out: f32): + %r = arith.maximumf %a, %b : f32 + linalg.yield %r : f32 + } -> tensor<1x8x8x16xf32> + %filter = tensor.empty() : tensor<2x2xf32> + %minimum = arith.constant dense<-3.40282347E+38> : tensor<1x4x4x16xf32> + %pooled = linalg.pooling_nhwc_max {strides = dense<2> : vector<2xi64>, dilations = dense<1> : vector<2xi64>} ins(%relu, %filter: tensor<1x8x8x16xf32>, tensor<2x2xf32>) outs(%minimum: tensor<1x4x4x16xf32>) -> tensor<1x4x4x16xf32> + %empty = tensor.empty() : tensor<1x16x4x4xf32> + %nchw = linalg.transpose ins(%pooled: tensor<1x4x4x16xf32>) outs(%empty: tensor<1x16x4x4xf32>) permutation = [0, 3, 1, 2] + %flat = tensor.collapse_shape %nchw [[0], [1, 2, 3]] : tensor<1x16x4x4xf32> into tensor<1x256xf32> + %result = custom.op "linear_i8"(%flat, %lw, %lb, %step, %step, %linear_step) : (tensor<1x256xf32>, tensor<256x16xi8>, tensor<1x16xi32>, tensor<1x1xf32>, tensor<1x1xf32>, tensor<1x16xf32>) -> tensor<1x16xf32> + %memory = bufferization.to_buffer %result : tensor<1x16xf32> to memref<1x16xf32> + func.call @check_result(%memory) : (memref<1x16xf32>) -> () + return %zero_i8 : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_flatten_linear_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_flatten_linear_main.cpp new file mode 100644 index 000000000..3770cfc76 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_flatten_linear_main.cpp @@ -0,0 +1,64 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +static int clamp8(int value) { + return value < -128 ? -128 : (value > 127 ? 127 : value); +} + +extern "C" void check_result(float *allocated, float *aligned, int64_t offset, + int64_t rows, int64_t columns, int64_t row_stride, + int64_t column_stride) { + (void)allocated; + if (rows != 1 || columns != 16 || row_stride != 16 || column_stride != 1) + fail(); + int conv[8][8][16]; + for (int y = 0; y < 8; ++y) + for (int x = 0; x < 8; ++x) + for (int c = 0; c < 16; ++c) { + int sum = c; + for (int ky = 0; ky < 3; ++ky) + for (int kx = 0; kx < 3; ++kx) + for (int ic = 0; ic < 2; ++ic) { + int iy = y + ky - 1, ix = x + kx - 1; + if (iy >= 0 && iy < 8 && ix >= 0 && ix < 8) + sum += ((3 * iy + 2 * ix + ic) % 7 - 3) * + ((3 * ky + kx + ic + 3 * c) % 5 - 2); + } + conv[y][x][c] = sum < 0 ? 0 : clamp8(sum); + } + // The pooled map in channel-major order. + int flat[256]; + for (int c = 0; c < 16; ++c) + for (int y = 0; y < 4; ++y) + for (int x = 0; x < 4; ++x) { + int maximum = -128; + for (int dy = 0; dy < 2; ++dy) + for (int dx = 0; dx < 2; ++dx) + if (conv[2 * y + dy][2 * x + dx][c] > maximum) + maximum = conv[2 * y + dy][2 * x + dx][c]; + flat[c * 16 + y * 4 + x] = maximum; + } + float *output = aligned + offset; + for (int n = 0; n < 16; ++n) { + int sum = n; + for (int k = 0; k < 256; ++k) + sum += flat[k] * ((k + n) % 5 - 2); + if (output[n * column_stride] != (float)sum) { + printf("FAILED: mega_conv2d_flatten_linear n=%d exp=%d got=%f\n", n, sum, + output[n * column_stride]); + fail(); + } + } + printf("PASSED: mega_conv2d_flatten_linear resident map to Linear\n"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_resident.mlir b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_resident.mlir new file mode 100644 index 000000000..9ed4ec0cd --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_resident.mlir @@ -0,0 +1,100 @@ +// FORM: %[[FIRST:.*]] = custom.op "conv2d_i8"{{.*}}custom.resident{{.*}}relu = true{{.*}}-> tensor<1x4x4x16xi8> +// BANK: %[[CODES:.*]] = toint8_ball.bank_int32_to_int8 {{.*}}relu> +// BANK-NOT: buckyball.bank_mvout +// BANK: im2col_ball.bank_im2col_i8 %[[CODES]] ->{{.*}}padding = 1, +// BANK: maxpool_ball.bank_maxpool_i8 +// BANK: return +func.func private @check_result(memref<1x16x2x2xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero_i32 = arith.constant 0 : i32 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c3 = arith.constant 3 : index + %c4 = arith.constant 4 : index + %c5 = arith.constant 5 : index + %c7 = arith.constant 7 : index + %c9 = arith.constant 9 : index + %c16 = arith.constant 16 : index + %input = memref.alloc() alignment = 64 : memref<1x4x4x2xi8> + %conv_weight = memref.alloc() alignment = 64 : memref<16x2x3x3xi8> + %conv_bias = memref.alloc() alignment = 64 : memref<16xi32> + %second_weight = memref.alloc() alignment = 64 : memref<16x16x3x3xi8> + %second_bias = memref.alloc() alignment = 64 : memref<16xi32> + scf.for %h = %c0 to %c4 step %c1 { + scf.for %w = %c0 to %c4 step %c1 { + scf.for %ic = %c0 to %c2 step %c1 { + %h3 = arith.muli %h, %c3 : index + %w2 = arith.muli %w, %c2 : index + %sum0 = arith.addi %h3, %w2 : index + %sum = arith.addi %sum0, %ic : index + %rem = arith.remui %sum, %c7 : index + %signed = arith.subi %rem, %c3 : index + %value = arith.index_cast %signed : index to i8 + memref.store %value, %input[%c0, %h, %w, %ic] : memref<1x4x4x2xi8> + } + } + } + scf.for %ic = %c0 to %c2 step %c1 { + scf.for %p = %c0 to %c9 step %c1 { + scf.for %oc = %c0 to %c16 step %c1 { + %oc3 = arith.muli %oc, %c3 : index + %sum0 = arith.addi %p, %ic : index + %sum = arith.addi %sum0, %oc3 : index + %rem = arith.remui %sum, %c5 : index + %signed = arith.subi %rem, %c2 : index + %value = arith.index_cast %signed : index to i8 + %ky_index = arith.divui %p, %c3 : index + %kx_index = arith.remui %p, %c3 : index + memref.store %value, %conv_weight[%oc, %ic, %ky_index, %kx_index] : memref<16x2x3x3xi8> + } + } + } + scf.for %oc = %c0 to %c16 step %c1 { + %signed = arith.subi %oc, %c4 : index + %value = arith.index_cast %signed : index to i32 + memref.store %value, %conv_bias[%oc] : memref<16xi32> + } + scf.for %p = %c0 to %c9 step %c1 { + scf.for %ic = %c0 to %c16 step %c1 { + scf.for %oc = %c0 to %c16 step %c1 { + %sum0 = arith.addi %p, %ic : index + %sum = arith.addi %sum0, %oc : index + %rem = arith.remui %sum, %c5 : index + %signed = arith.subi %rem, %c2 : index + %value = arith.index_cast %signed : index to i8 + %kh = arith.divui %p, %c3 : index + %kw = arith.remui %p, %c3 : index + memref.store %value, %second_weight[%oc, %ic, %kh, %kw] : memref<16x16x3x3xi8> + } + } + } + linalg.fill ins(%zero_i32 : i32) outs(%second_bias : memref<16xi32>) + %unit = arith.constant dense<1.0> : tensor<1x1x1x1xf32> + %weight_step = arith.constant dense<1.0> : tensor<16x1x1x1xf32> + %it = bufferization.to_tensor %input restrict : memref<1x4x4x2xi8> to tensor<1x4x4x2xi8> + %input_float = "frontend.int8_to_fp32"(%it, %unit) : (tensor<1x4x4x2xi8>, tensor<1x1x1x1xf32>) -> tensor<1x4x4x2xf32> + %wt = bufferization.to_tensor %conv_weight restrict : memref<16x2x3x3xi8> to tensor<16x2x3x3xi8> + %bt = bufferization.to_tensor %conv_bias restrict : memref<16xi32> to tensor<16xi32> + %dt = bufferization.to_tensor %second_weight restrict : memref<16x16x3x3xi8> to tensor<16x16x3x3xi8> + %dbt = bufferization.to_tensor %second_bias restrict : memref<16xi32> to tensor<16xi32> + %first = custom.op "conv2d_i8"(%input_float, %wt, %bt, %unit, %unit, %weight_step) {strides = array, dilations = array, padding = array} : (tensor<1x4x4x2xf32>, tensor<16x2x3x3xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x4x4x16xf32> + %zero = arith.constant dense<0.0> : tensor<1x4x4x16xf32> + %relu = linalg.generic {indexing_maps = [affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>, affine_map<(n,h,w,c)->(n,h,w,c)>], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%first, %zero: tensor<1x4x4x16xf32>, tensor<1x4x4x16xf32>) outs(%zero: tensor<1x4x4x16xf32>) { + ^bb0(%a: f32, %b: f32, %out: f32): + %r = arith.maximumf %a, %b : f32 + linalg.yield %r : f32 + } -> tensor<1x4x4x16xf32> + %second = custom.op "conv2d_i8"(%relu, %dt, %dbt, %unit, %unit, %weight_step) {strides = array, dilations = array, padding = array} : (tensor<1x4x4x16xf32>, tensor<16x16x3x3xi8>, tensor<16xi32>, tensor<1x1x1x1xf32>, tensor<1x1x1x1xf32>, tensor<16x1x1x1xf32>) -> tensor<1x4x4x16xf32> + %filter = tensor.empty() : tensor<2x2xf32> + %minimum = arith.constant dense<-3.40282347E+38> : tensor<1x2x2x16xf32> + %pooled = linalg.pooling_nhwc_max {strides = dense<2> : vector<2xi64>, dilations = dense<1> : vector<2xi64>} ins(%second, %filter: tensor<1x4x4x16xf32>, tensor<2x2xf32>) outs(%minimum: tensor<1x2x2x16xf32>) -> tensor<1x2x2x16xf32> + %q = "frontend.fp32_to_int8"(%pooled, %unit) : (tensor<1x2x2x16xf32>, tensor<1x1x1x1xf32>) -> tensor<1x2x2x16xi8> + %empty = tensor.empty() : tensor<1x16x2x2xi8> + %nchw = linalg.transpose ins(%q: tensor<1x2x2x16xi8>) outs(%empty: tensor<1x16x2x2xi8>) permutation = [0, 3, 1, 2] + %memory = bufferization.to_buffer %nchw : tensor<1x16x2x2xi8> to memref<1x16x2x2xi8> + func.call @check_result(%memory) : (memref<1x16x2x2xi8>) -> () + return %zero_i8 : i8 +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_resident_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_resident_main.cpp new file mode 100644 index 000000000..2010702f4 --- /dev/null +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_conv2d_resident_main.cpp @@ -0,0 +1,73 @@ +#include +#include +#include + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +extern "C" void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, + int64_t n, int64_t channels, int64_t height, + int64_t width, int64_t n_stride, + int64_t channel_stride, int64_t height_stride, + int64_t width_stride) { + (void)allocated; + if (n != 1 || channels != 16 || height != 2 || width != 2 || n_stride != 64 || + channel_stride != 4 || height_stride != 2 || width_stride != 1) + fail(); + int8_t *output = aligned + offset; + int first[4][4][16]; + int second[4][4][16]; + for (int y = 0; y < 4; ++y) + for (int x = 0; x < 4; ++x) + for (int c = 0; c < 16; ++c) { + int sum = c - 4; + for (int ky = 0; ky < 3; ++ky) + for (int kx = 0; kx < 3; ++kx) + for (int ic = 0; ic < 2; ++ic) { + int iy = y + ky - 1, ix = x + kx - 1; + if (iy >= 0 && iy < 4 && ix >= 0 && ix < 4) + sum += ((3 * iy + 2 * ix + ic) % 7 - 3) * + ((3 * ky + kx + ic + 3 * c) % 5 - 2); + } + first[y][x][c] = sum < 0 ? 0 : (sum > 127 ? 127 : sum); + } + for (int y = 0; y < 4; ++y) + for (int x = 0; x < 4; ++x) + for (int c = 0; c < 16; ++c) { + int sum = 0; + for (int ky = 0; ky < 3; ++ky) + for (int kx = 0; kx < 3; ++kx) + for (int ic = 0; ic < 16; ++ic) { + int iy = y + ky - 1, ix = x + kx - 1; + if (iy >= 0 && iy < 4 && ix >= 0 && ix < 4) + sum += first[iy][ix][ic] * ((3 * ky + kx + ic + c) % 5 - 2); + } + second[y][x][c] = sum < -128 ? -128 : (sum > 127 ? 127 : sum); + } + for (int channel = 0; channel < 16; ++channel) { + for (int y = 0; y < 2; ++y) { + for (int x = 0; x < 2; ++x) { + int expected = -128; + for (int dy = 0; dy < 2; ++dy) + for (int dx = 0; dx < 2; ++dx) + if (second[2 * y + dy][2 * x + dx][channel] > expected) + expected = second[2 * y + dy][2 * x + dx][channel]; + int actual = output[channel * channel_stride + y * height_stride + x]; + if (actual != expected) { + printf("FAILED: mega_conv2d_resident c=%d y=%d x=%d exp=%d got=%d\n", + channel, y, x, expected, actual); + fail(); + } + } + } + } + printf("PASSED: mega_conv2d_resident Conv-to-Conv-to-MaxPool with a resident " + "map\n"); +} diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact_main.cpp index 0c2d34139..7d64ed1be 100644 --- a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact_main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_7x7_c288_exact_main.cpp @@ -5,7 +5,6 @@ #include "buckyball.h" #include #include -#include static void fail(void) { #ifdef BAREMETAL @@ -18,12 +17,12 @@ static void fail(void) { } enum { - PHYSICAL_BANKS = BANK_NUM, - BANK_ROWS = BANK_LINES, - ROW_BYTES = BANK_WIDTH / 8 + PHYSICAL_BANKS = THREAD_MEM_BANK_NUM, + BANK_ROWS = THREAD_MEM_BANK_ENTRIES, + ROW_BYTES = THREAD_MEM_BANK_WIDTH / 8 }; static_assert( - VIRTUAL_BANK_NUM >= PHYSICAL_BANKS, + CORE_VIRTUAL_BANK_COUNT >= PHYSICAL_BANKS, "bank reuse regression requires every physical bank to be addressable"); static elem_t stale_bank[BANK_ROWS * ROW_BYTES] __attribute__((aligned(64))); diff --git a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact_main.cpp b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact_main.cpp index ee1e6ccdf..89d8ec202 100644 --- a/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact_main.cpp +++ b/examples/designs/pebble/workloads/device/core/acc/mlir_tests/core/mega_global_avg_conv2_exact_main.cpp @@ -5,7 +5,6 @@ #include "buckyball.h" #include #include -#include static void fail(void) { #ifdef BAREMETAL @@ -18,12 +17,12 @@ static void fail(void) { } enum { - PHYSICAL_BANKS = BANK_NUM, - BANK_ROWS = BANK_LINES, - ROW_BYTES = BANK_WIDTH / 8 + PHYSICAL_BANKS = THREAD_MEM_BANK_NUM, + BANK_ROWS = THREAD_MEM_BANK_ENTRIES, + ROW_BYTES = THREAD_MEM_BANK_WIDTH / 8 }; static_assert( - PHYSICAL_BANKS > 5 && VIRTUAL_BANK_NUM >= PHYSICAL_BANKS, + PHYSICAL_BANKS > 5 && CORE_VIRTUAL_BANK_COUNT >= PHYSICAL_BANKS, "bank reuse regression requires physical bank 5 to be addressable"); static elem_t stale_bank[BANK_ROWS * ROW_BYTES] __attribute__((aligned(64))); diff --git a/examples/designs/poly/arch/build.sc b/examples/designs/poly/arch/build.sc index f64ac6e98..d983891ce 100644 --- a/examples/designs/poly/arch/build.sc +++ b/examples/designs/poly/arch/build.sc @@ -2,7 +2,7 @@ import mill._ import mill.scalalib._ import $file.^.^.^.^.arch.src.main.scala.framework.device.cbus.{build => cbus_build} import $file.^.^.^.{build => examples} -import $file.^.^.^.components.cores.reduce.arch.{build => reduce} +import $file.^.^.^.prefab.cores.reduce.arch.{build => reduce} val designRoot: os.Path = millSourcePath / os.up diff --git a/examples/designs/poly/configs/device/cores/core0/acc/frontend/poly.toml b/examples/designs/poly/configs/device/cores/core0/acc/frontend/poly.toml index c3e3d5e71..5464f2d95 100644 --- a/examples/designs/poly/configs/device/cores/core0/acc/frontend/poly.toml +++ b/examples/designs/poly/configs/device/cores/core0/acc/frontend/poly.toml @@ -2,8 +2,8 @@ bankAliasEnable = true robEntries = 16 rsOutOfOrderResponse = true bankIdLen = 10 -vbankIdUpperBound = 15 -sharedBankIdBase = 32 +maxPrivateVbank = 15 +sharedVbankBase = 32 iterLen = 34 subRobEnable = false subRobDepth = 64 diff --git a/examples/designs/poly/configs/device/cores/core0/acc/memdomain/poly.toml b/examples/designs/poly/configs/device/cores/core0/acc/memdomain/poly.toml index b502d7a33..63a9fc482 100644 --- a/examples/designs/poly/configs/device/cores/core0/acc/memdomain/poly.toml +++ b/examples/designs/poly/configs/device/cores/core0/acc/memdomain/poly.toml @@ -1,3 +1,5 @@ +addrLen = 39 + [bank] layoutEnable = true num = 4 @@ -15,6 +17,3 @@ bankNum = 1 bankEntries = 1 bankWidth = 128 readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/designs/poly/configs/device/cores/core0/default.toml b/examples/designs/poly/configs/device/cores/core0/default.toml index d1fb23449..e6731ebbc 100644 --- a/examples/designs/poly/configs/device/cores/core0/default.toml +++ b/examples/designs/poly/configs/device/cores/core0/default.toml @@ -30,7 +30,7 @@ virtualBankCount = 40 entries = 16384 inputChannels = 5 defaultGroupCount = 1 -memCycles = 10 +latency = 10 [compiler] name = "main" diff --git a/examples/designs/poly/configs/device/cores/core0/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core0/dmaCache/core.toml index 309ac5a3d..859ae7561 100644 --- a/examples/designs/poly/configs/device/cores/core0/dmaCache/core.toml +++ b/examples/designs/poly/configs/device/cores/core0/dmaCache/core.toml @@ -1,5 +1,5 @@ nodeId = 3 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core0/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core0/memory/access/core.toml index d961d54fc..abd42a128 100644 --- a/examples/designs/poly/configs/device/cores/core0/memory/access/core.toml +++ b/examples/designs/poly/configs/device/cores/core0/memory/access/core.toml @@ -1,5 +1,5 @@ translationEntries = 16 -accessTagBits = 6 +tagBits = 6 resultLineBits = 0 [[regions]] base = 0x80000000 diff --git a/examples/designs/poly/configs/device/cores/core0/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core0/memory/dcache/core.toml index 97b8e2bba..4fb4beb6f 100644 --- a/examples/designs/poly/configs/device/cores/core0/memory/dcache/core.toml +++ b/examples/designs/poly/configs/device/cores/core0/memory/dcache/core.toml @@ -7,7 +7,7 @@ maxRanges = 8 [cache] nodeId = 2 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core0/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core0/memory/home/core.toml index f20d607fe..0b7f0ff39 100644 --- a/examples/designs/poly/configs/device/cores/core0/memory/home/core.toml +++ b/examples/designs/poly/configs/device/cores/core0/memory/home/core.toml @@ -1,6 +1,6 @@ agents = 27 mshrEntries = 8 -homeId = 64 +nodeId = 64 requesters = "default" [cache] diff --git a/examples/designs/poly/configs/device/cores/core0/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core0/memory/icache/core.toml index 75e065643..3cc7df83a 100644 --- a/examples/designs/poly/configs/device/cores/core0/memory/icache/core.toml +++ b/examples/designs/poly/configs/device/cores/core0/memory/icache/core.toml @@ -1,5 +1,5 @@ nodeId = 1 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core1/default.toml b/examples/designs/poly/configs/device/cores/core1/default.toml index fd8332ae3..011971006 100644 --- a/examples/designs/poly/configs/device/cores/core1/default.toml +++ b/examples/designs/poly/configs/device/cores/core1/default.toml @@ -33,39 +33,39 @@ virtualBankCount = 40 entries = 16384 inputChannels = 65 defaultGroupCount = 1 -memCycles = 10 +latency = 10 [[threads]] id = 0 name = "attention0" component = "attention" transport = "transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 1 name = "attention1" component = "attention" transport = "transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 2 name = "ffn0" component = "ffn" transport = "transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 3 name = "ffn1" component = "ffn" transport = "transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 4 name = "ffn2" component = "ffn" transport = "transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core1/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core1/dmaCache/core.toml index b23c6c6cd..9b6a1a19a 100644 --- a/examples/designs/poly/configs/device/cores/core1/dmaCache/core.toml +++ b/examples/designs/poly/configs/device/cores/core1/dmaCache/core.toml @@ -1,5 +1,5 @@ nodeId = 6 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core1/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core1/memory/access/core.toml index d961d54fc..abd42a128 100644 --- a/examples/designs/poly/configs/device/cores/core1/memory/access/core.toml +++ b/examples/designs/poly/configs/device/cores/core1/memory/access/core.toml @@ -1,5 +1,5 @@ translationEntries = 16 -accessTagBits = 6 +tagBits = 6 resultLineBits = 0 [[regions]] base = 0x80000000 diff --git a/examples/designs/poly/configs/device/cores/core1/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core1/memory/dcache/core.toml index 1fa9b4c07..2d34f225a 100644 --- a/examples/designs/poly/configs/device/cores/core1/memory/dcache/core.toml +++ b/examples/designs/poly/configs/device/cores/core1/memory/dcache/core.toml @@ -7,7 +7,7 @@ maxRanges = 8 [cache] nodeId = 5 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core1/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core1/memory/home/core.toml index f20d607fe..0b7f0ff39 100644 --- a/examples/designs/poly/configs/device/cores/core1/memory/home/core.toml +++ b/examples/designs/poly/configs/device/cores/core1/memory/home/core.toml @@ -1,6 +1,6 @@ agents = 27 mshrEntries = 8 -homeId = 64 +nodeId = 64 requesters = "default" [cache] diff --git a/examples/designs/poly/configs/device/cores/core1/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core1/memory/icache/core.toml index 5a95e88f0..d35c0e05b 100644 --- a/examples/designs/poly/configs/device/cores/core1/memory/icache/core.toml +++ b/examples/designs/poly/configs/device/cores/core1/memory/icache/core.toml @@ -1,5 +1,5 @@ nodeId = 4 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core2/default.toml b/examples/designs/poly/configs/device/cores/core2/default.toml index f2caf5ff0..3a3ced771 100644 --- a/examples/designs/poly/configs/device/cores/core2/default.toml +++ b/examples/designs/poly/configs/device/cores/core2/default.toml @@ -33,39 +33,39 @@ virtualBankCount = 40 entries = 16384 inputChannels = 65 defaultGroupCount = 1 -memCycles = 10 +latency = 10 [[threads]] id = 0 name = "attention0" component = "attention" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 1 name = "attention1" component = "attention" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 2 name = "ffn0" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 3 name = "ffn1" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 4 name = "ffn2" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core2/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core2/dmaCache/core.toml index 714218f58..61d2b253d 100644 --- a/examples/designs/poly/configs/device/cores/core2/dmaCache/core.toml +++ b/examples/designs/poly/configs/device/cores/core2/dmaCache/core.toml @@ -1,5 +1,5 @@ nodeId = 9 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core2/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core2/memory/access/core.toml index d961d54fc..abd42a128 100644 --- a/examples/designs/poly/configs/device/cores/core2/memory/access/core.toml +++ b/examples/designs/poly/configs/device/cores/core2/memory/access/core.toml @@ -1,5 +1,5 @@ translationEntries = 16 -accessTagBits = 6 +tagBits = 6 resultLineBits = 0 [[regions]] base = 0x80000000 diff --git a/examples/designs/poly/configs/device/cores/core2/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core2/memory/dcache/core.toml index acc89ffe1..e16221d82 100644 --- a/examples/designs/poly/configs/device/cores/core2/memory/dcache/core.toml +++ b/examples/designs/poly/configs/device/cores/core2/memory/dcache/core.toml @@ -7,7 +7,7 @@ maxRanges = 8 [cache] nodeId = 8 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core2/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core2/memory/home/core.toml index f20d607fe..0b7f0ff39 100644 --- a/examples/designs/poly/configs/device/cores/core2/memory/home/core.toml +++ b/examples/designs/poly/configs/device/cores/core2/memory/home/core.toml @@ -1,6 +1,6 @@ agents = 27 mshrEntries = 8 -homeId = 64 +nodeId = 64 requesters = "default" [cache] diff --git a/examples/designs/poly/configs/device/cores/core2/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core2/memory/icache/core.toml index 82e254beb..804e53c8c 100644 --- a/examples/designs/poly/configs/device/cores/core2/memory/icache/core.toml +++ b/examples/designs/poly/configs/device/cores/core2/memory/icache/core.toml @@ -1,5 +1,5 @@ nodeId = 7 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core3/default.toml b/examples/designs/poly/configs/device/cores/core3/default.toml index 8eb441d2d..d960c27a9 100644 --- a/examples/designs/poly/configs/device/cores/core3/default.toml +++ b/examples/designs/poly/configs/device/cores/core3/default.toml @@ -33,39 +33,39 @@ virtualBankCount = 40 entries = 16384 inputChannels = 65 defaultGroupCount = 1 -memCycles = 10 +latency = 10 [[threads]] id = 0 name = "attention0" component = "attention" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 1 name = "attention1" component = "attention" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 2 name = "ffn0" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 3 name = "ffn1" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 4 name = "ffn2" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core3/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core3/dmaCache/core.toml index e37f02576..d005f7cfe 100644 --- a/examples/designs/poly/configs/device/cores/core3/dmaCache/core.toml +++ b/examples/designs/poly/configs/device/cores/core3/dmaCache/core.toml @@ -1,5 +1,5 @@ nodeId = 12 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core3/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core3/memory/access/core.toml index d961d54fc..abd42a128 100644 --- a/examples/designs/poly/configs/device/cores/core3/memory/access/core.toml +++ b/examples/designs/poly/configs/device/cores/core3/memory/access/core.toml @@ -1,5 +1,5 @@ translationEntries = 16 -accessTagBits = 6 +tagBits = 6 resultLineBits = 0 [[regions]] base = 0x80000000 diff --git a/examples/designs/poly/configs/device/cores/core3/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core3/memory/dcache/core.toml index f71df002a..2c4cef9ed 100644 --- a/examples/designs/poly/configs/device/cores/core3/memory/dcache/core.toml +++ b/examples/designs/poly/configs/device/cores/core3/memory/dcache/core.toml @@ -7,7 +7,7 @@ maxRanges = 8 [cache] nodeId = 11 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core3/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core3/memory/home/core.toml index f20d607fe..0b7f0ff39 100644 --- a/examples/designs/poly/configs/device/cores/core3/memory/home/core.toml +++ b/examples/designs/poly/configs/device/cores/core3/memory/home/core.toml @@ -1,6 +1,6 @@ agents = 27 mshrEntries = 8 -homeId = 64 +nodeId = 64 requesters = "default" [cache] diff --git a/examples/designs/poly/configs/device/cores/core3/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core3/memory/icache/core.toml index 9b09d3745..f1dd6ae49 100644 --- a/examples/designs/poly/configs/device/cores/core3/memory/icache/core.toml +++ b/examples/designs/poly/configs/device/cores/core3/memory/icache/core.toml @@ -1,5 +1,5 @@ nodeId = 10 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core4/default.toml b/examples/designs/poly/configs/device/cores/core4/default.toml index 0da5398b1..5577c51ee 100644 --- a/examples/designs/poly/configs/device/cores/core4/default.toml +++ b/examples/designs/poly/configs/device/cores/core4/default.toml @@ -33,39 +33,39 @@ virtualBankCount = 40 entries = 16384 inputChannels = 65 defaultGroupCount = 1 -memCycles = 10 +latency = 10 [[threads]] id = 0 name = "attention0" component = "attention" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 1 name = "attention1" component = "attention" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 2 name = "ffn0" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 3 name = "ffn1" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 4 name = "ffn2" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core4/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core4/dmaCache/core.toml index 8c645ccb0..825eebb12 100644 --- a/examples/designs/poly/configs/device/cores/core4/dmaCache/core.toml +++ b/examples/designs/poly/configs/device/cores/core4/dmaCache/core.toml @@ -1,5 +1,5 @@ nodeId = 15 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core4/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core4/memory/access/core.toml index d961d54fc..abd42a128 100644 --- a/examples/designs/poly/configs/device/cores/core4/memory/access/core.toml +++ b/examples/designs/poly/configs/device/cores/core4/memory/access/core.toml @@ -1,5 +1,5 @@ translationEntries = 16 -accessTagBits = 6 +tagBits = 6 resultLineBits = 0 [[regions]] base = 0x80000000 diff --git a/examples/designs/poly/configs/device/cores/core4/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core4/memory/dcache/core.toml index 83545e2a1..72a49ee07 100644 --- a/examples/designs/poly/configs/device/cores/core4/memory/dcache/core.toml +++ b/examples/designs/poly/configs/device/cores/core4/memory/dcache/core.toml @@ -7,7 +7,7 @@ maxRanges = 8 [cache] nodeId = 14 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core4/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core4/memory/home/core.toml index f20d607fe..0b7f0ff39 100644 --- a/examples/designs/poly/configs/device/cores/core4/memory/home/core.toml +++ b/examples/designs/poly/configs/device/cores/core4/memory/home/core.toml @@ -1,6 +1,6 @@ agents = 27 mshrEntries = 8 -homeId = 64 +nodeId = 64 requesters = "default" [cache] diff --git a/examples/designs/poly/configs/device/cores/core4/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core4/memory/icache/core.toml index 382168405..623288def 100644 --- a/examples/designs/poly/configs/device/cores/core4/memory/icache/core.toml +++ b/examples/designs/poly/configs/device/cores/core4/memory/icache/core.toml @@ -1,5 +1,5 @@ nodeId = 13 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core5/default.toml b/examples/designs/poly/configs/device/cores/core5/default.toml index 87531a19d..1ae5616c9 100644 --- a/examples/designs/poly/configs/device/cores/core5/default.toml +++ b/examples/designs/poly/configs/device/cores/core5/default.toml @@ -33,39 +33,39 @@ virtualBankCount = 40 entries = 16384 inputChannels = 65 defaultGroupCount = 1 -memCycles = 10 +latency = 10 [[threads]] id = 0 name = "attention0" component = "attention" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 1 name = "attention1" component = "attention" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 2 name = "ffn0" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 3 name = "ffn1" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 4 name = "ffn2" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core5/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core5/dmaCache/core.toml index 755ae918e..8f809d9ec 100644 --- a/examples/designs/poly/configs/device/cores/core5/dmaCache/core.toml +++ b/examples/designs/poly/configs/device/cores/core5/dmaCache/core.toml @@ -1,5 +1,5 @@ nodeId = 18 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core5/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core5/memory/access/core.toml index d961d54fc..abd42a128 100644 --- a/examples/designs/poly/configs/device/cores/core5/memory/access/core.toml +++ b/examples/designs/poly/configs/device/cores/core5/memory/access/core.toml @@ -1,5 +1,5 @@ translationEntries = 16 -accessTagBits = 6 +tagBits = 6 resultLineBits = 0 [[regions]] base = 0x80000000 diff --git a/examples/designs/poly/configs/device/cores/core5/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core5/memory/dcache/core.toml index cdc0c63e3..96526fec5 100644 --- a/examples/designs/poly/configs/device/cores/core5/memory/dcache/core.toml +++ b/examples/designs/poly/configs/device/cores/core5/memory/dcache/core.toml @@ -7,7 +7,7 @@ maxRanges = 8 [cache] nodeId = 17 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core5/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core5/memory/home/core.toml index f20d607fe..0b7f0ff39 100644 --- a/examples/designs/poly/configs/device/cores/core5/memory/home/core.toml +++ b/examples/designs/poly/configs/device/cores/core5/memory/home/core.toml @@ -1,6 +1,6 @@ agents = 27 mshrEntries = 8 -homeId = 64 +nodeId = 64 requesters = "default" [cache] diff --git a/examples/designs/poly/configs/device/cores/core5/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core5/memory/icache/core.toml index 81af78919..c9a5edf2c 100644 --- a/examples/designs/poly/configs/device/cores/core5/memory/icache/core.toml +++ b/examples/designs/poly/configs/device/cores/core5/memory/icache/core.toml @@ -1,5 +1,5 @@ nodeId = 16 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core6/default.toml b/examples/designs/poly/configs/device/cores/core6/default.toml index 4fa2fad54..1cfd398e5 100644 --- a/examples/designs/poly/configs/device/cores/core6/default.toml +++ b/examples/designs/poly/configs/device/cores/core6/default.toml @@ -33,39 +33,39 @@ virtualBankCount = 40 entries = 16384 inputChannels = 65 defaultGroupCount = 1 -memCycles = 10 +latency = 10 [[threads]] id = 0 name = "attention0" component = "attention" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 1 name = "attention1" component = "attention" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 2 name = "ffn0" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 3 name = "ffn1" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 4 name = "ffn2" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core6/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core6/dmaCache/core.toml index 413178519..0307dda51 100644 --- a/examples/designs/poly/configs/device/cores/core6/dmaCache/core.toml +++ b/examples/designs/poly/configs/device/cores/core6/dmaCache/core.toml @@ -1,5 +1,5 @@ nodeId = 21 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core6/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core6/memory/access/core.toml index d961d54fc..abd42a128 100644 --- a/examples/designs/poly/configs/device/cores/core6/memory/access/core.toml +++ b/examples/designs/poly/configs/device/cores/core6/memory/access/core.toml @@ -1,5 +1,5 @@ translationEntries = 16 -accessTagBits = 6 +tagBits = 6 resultLineBits = 0 [[regions]] base = 0x80000000 diff --git a/examples/designs/poly/configs/device/cores/core6/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core6/memory/dcache/core.toml index 92e94a216..1903dc0ed 100644 --- a/examples/designs/poly/configs/device/cores/core6/memory/dcache/core.toml +++ b/examples/designs/poly/configs/device/cores/core6/memory/dcache/core.toml @@ -7,7 +7,7 @@ maxRanges = 8 [cache] nodeId = 20 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core6/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core6/memory/home/core.toml index f20d607fe..0b7f0ff39 100644 --- a/examples/designs/poly/configs/device/cores/core6/memory/home/core.toml +++ b/examples/designs/poly/configs/device/cores/core6/memory/home/core.toml @@ -1,6 +1,6 @@ agents = 27 mshrEntries = 8 -homeId = 64 +nodeId = 64 requesters = "default" [cache] diff --git a/examples/designs/poly/configs/device/cores/core6/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core6/memory/icache/core.toml index 9c2d9947f..2386241f2 100644 --- a/examples/designs/poly/configs/device/cores/core6/memory/icache/core.toml +++ b/examples/designs/poly/configs/device/cores/core6/memory/icache/core.toml @@ -1,5 +1,5 @@ nodeId = 19 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core7/default.toml b/examples/designs/poly/configs/device/cores/core7/default.toml index e53fe7834..751a41018 100644 --- a/examples/designs/poly/configs/device/cores/core7/default.toml +++ b/examples/designs/poly/configs/device/cores/core7/default.toml @@ -33,39 +33,39 @@ virtualBankCount = 40 entries = 16384 inputChannels = 65 defaultGroupCount = 1 -memCycles = 10 +latency = 10 [[threads]] id = 0 name = "attention0" component = "attention" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 1 name = "attention1" component = "attention" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 2 name = "ffn0" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 3 name = "ffn1" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 4 name = "ffn2" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core7/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core7/dmaCache/core.toml index d315ca578..20cc52e02 100644 --- a/examples/designs/poly/configs/device/cores/core7/dmaCache/core.toml +++ b/examples/designs/poly/configs/device/cores/core7/dmaCache/core.toml @@ -1,5 +1,5 @@ nodeId = 24 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core7/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core7/memory/access/core.toml index d961d54fc..abd42a128 100644 --- a/examples/designs/poly/configs/device/cores/core7/memory/access/core.toml +++ b/examples/designs/poly/configs/device/cores/core7/memory/access/core.toml @@ -1,5 +1,5 @@ translationEntries = 16 -accessTagBits = 6 +tagBits = 6 resultLineBits = 0 [[regions]] base = 0x80000000 diff --git a/examples/designs/poly/configs/device/cores/core7/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core7/memory/dcache/core.toml index 9b77b7ae8..78dcd63c7 100644 --- a/examples/designs/poly/configs/device/cores/core7/memory/dcache/core.toml +++ b/examples/designs/poly/configs/device/cores/core7/memory/dcache/core.toml @@ -7,7 +7,7 @@ maxRanges = 8 [cache] nodeId = 23 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core7/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core7/memory/home/core.toml index f20d607fe..0b7f0ff39 100644 --- a/examples/designs/poly/configs/device/cores/core7/memory/home/core.toml +++ b/examples/designs/poly/configs/device/cores/core7/memory/home/core.toml @@ -1,6 +1,6 @@ agents = 27 mshrEntries = 8 -homeId = 64 +nodeId = 64 requesters = "default" [cache] diff --git a/examples/designs/poly/configs/device/cores/core7/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core7/memory/icache/core.toml index b7600e54a..d41891133 100644 --- a/examples/designs/poly/configs/device/cores/core7/memory/icache/core.toml +++ b/examples/designs/poly/configs/device/cores/core7/memory/icache/core.toml @@ -1,5 +1,5 @@ nodeId = 22 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core8/default.toml b/examples/designs/poly/configs/device/cores/core8/default.toml index b687187a6..430061c63 100644 --- a/examples/designs/poly/configs/device/cores/core8/default.toml +++ b/examples/designs/poly/configs/device/cores/core8/default.toml @@ -33,39 +33,39 @@ virtualBankCount = 40 entries = 16384 inputChannels = 65 defaultGroupCount = 1 -memCycles = 10 +latency = 10 [[threads]] id = 0 name = "attention0" component = "attention" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 1 name = "attention1" component = "attention" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/attention/configs/default.toml" +include = "../../../../../../prefab/threads/attention/configs/default.toml" [[threads]] id = 2 name = "ffn0" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 3 name = "ffn1" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" [[threads]] id = 4 name = "ffn2" component = "ffn" transport = "../core1/transport.toml" -include = "../../../../../../components/threads/ffn/configs/default.toml" +include = "../../../../../../prefab/threads/ffn/configs/default.toml" diff --git a/examples/designs/poly/configs/device/cores/core8/dmaCache/core.toml b/examples/designs/poly/configs/device/cores/core8/dmaCache/core.toml index f9fcaf866..17e8317ed 100644 --- a/examples/designs/poly/configs/device/cores/core8/dmaCache/core.toml +++ b/examples/designs/poly/configs/device/cores/core8/dmaCache/core.toml @@ -1,5 +1,5 @@ nodeId = 27 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core8/memory/access/core.toml b/examples/designs/poly/configs/device/cores/core8/memory/access/core.toml index d961d54fc..abd42a128 100644 --- a/examples/designs/poly/configs/device/cores/core8/memory/access/core.toml +++ b/examples/designs/poly/configs/device/cores/core8/memory/access/core.toml @@ -1,5 +1,5 @@ translationEntries = 16 -accessTagBits = 6 +tagBits = 6 resultLineBits = 0 [[regions]] base = 0x80000000 diff --git a/examples/designs/poly/configs/device/cores/core8/memory/dcache/core.toml b/examples/designs/poly/configs/device/cores/core8/memory/dcache/core.toml index 7feda67d6..ced088acd 100644 --- a/examples/designs/poly/configs/device/cores/core8/memory/dcache/core.toml +++ b/examples/designs/poly/configs/device/cores/core8/memory/dcache/core.toml @@ -7,7 +7,7 @@ maxRanges = 8 [cache] nodeId = 26 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/device/cores/core8/memory/home/core.toml b/examples/designs/poly/configs/device/cores/core8/memory/home/core.toml index f20d607fe..0b7f0ff39 100644 --- a/examples/designs/poly/configs/device/cores/core8/memory/home/core.toml +++ b/examples/designs/poly/configs/device/cores/core8/memory/home/core.toml @@ -1,6 +1,6 @@ agents = 27 mshrEntries = 8 -homeId = 64 +nodeId = 64 requesters = "default" [cache] diff --git a/examples/designs/poly/configs/device/cores/core8/memory/icache/core.toml b/examples/designs/poly/configs/device/cores/core8/memory/icache/core.toml index 7bee95556..835cdf2ec 100644 --- a/examples/designs/poly/configs/device/cores/core8/memory/icache/core.toml +++ b/examples/designs/poly/configs/device/cores/core8/memory/icache/core.toml @@ -1,5 +1,5 @@ nodeId = 25 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/poly/configs/sims/default.toml b/examples/designs/poly/configs/sims/default.toml index 39c913e06..0db22541b 100644 --- a/examples/designs/poly/configs/sims/default.toml +++ b/examples/designs/poly/configs/sims/default.toml @@ -10,5 +10,11 @@ stack_name = "poly" regression = "sims/device/regression" kernel = "sims/device/workloads/kernel" +# Each device config overrides a few tables of the design config. +[targets.device.configs] +PolyVerilatorConfig = "verilator.toml" +PolyTapeoutConfig = "tapeout.toml" +PolyP2EConfig = "p2e.toml" + [verification] registry = "../../sims/ip/uvm/registry.toml" diff --git a/examples/designs/poly/configs/sims/p2e.toml b/examples/designs/poly/configs/sims/p2e.toml new file mode 100644 index 000000000..7a912105b --- /dev/null +++ b/examples/designs/poly/configs/sims/p2e.toml @@ -0,0 +1 @@ +# PolyP2EConfig: the full nine-core design on the P2E FPGA, with no overrides. diff --git a/examples/designs/poly/configs/sims/tapeout.toml b/examples/designs/poly/configs/sims/tapeout.toml new file mode 100644 index 000000000..cd0089f58 --- /dev/null +++ b/examples/designs/poly/configs/sims/tapeout.toml @@ -0,0 +1 @@ +# PolyTapeoutConfig: the full nine-core design for VCS, DC and tapeout, with no overrides. diff --git a/examples/designs/poly/configs/sims/verilator.toml b/examples/designs/poly/configs/sims/verilator.toml new file mode 100644 index 000000000..c5e2139ee --- /dev/null +++ b/examples/designs/poly/configs/sims/verilator.toml @@ -0,0 +1,9 @@ +# PolyVerilatorConfig: functional simulation (BEMU / Verilator) with the reduce +# core and one poly core. +[device] +cores = [ + { include = "../device/cores/core0/default.toml", memory = { home = { agents = 6 } }, c2c = { peerCores = 2 } }, + { include = "../device/cores/core1/default.toml", memory = { home = { agents = 6 } }, c2c = { peerCores = 2 } }, +] +clint = { hartIds = [0, 1] } +plic = { hartIds = [0, 1] } diff --git a/examples/designs/poly/sims/device/emu/Cargo.toml b/examples/designs/poly/sims/device/emu/Cargo.toml index d7ab45df4..39204a91b 100644 --- a/examples/designs/poly/sims/device/emu/Cargo.toml +++ b/examples/designs/poly/sims/device/emu/Cargo.toml @@ -67,7 +67,7 @@ bebop = { path = "../../../../../../bebop" } bebop-verilator = { path = "../../../../../../bebop/src/nodes/verilator", optional = true } bebop-bemu = { path = "../../../../../../bebop/src/nodes/bemu" } bebop-design-poly = { path = "../../../emu" } -bebop-core-reduce = { path = "../../../../../components/cores/reduce/emu" } +bebop-core-reduce = { path = "../../../../../prefab/cores/reduce/emu" } clap = { version = "4", features = ["derive"] } [build-dependencies] diff --git a/examples/designs/poly/sims/device/emu/build.rs b/examples/designs/poly/sims/device/emu/build.rs index d634d5dba..beb62e98c 100644 --- a/examples/designs/poly/sims/device/emu/build.rs +++ b/examples/designs/poly/sims/device/emu/build.rs @@ -1,7 +1,4 @@ -use std::{env, path::PathBuf}; - fn main() { - let entry = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - bebop_sim_build::balls(&entry.join("../../../output/generated/design.pb")); + bebop_sim_build::balls(); bebop_sim_build::runner_links(); } diff --git a/examples/designs/poly/sims/device/emu/src/lib.rs b/examples/designs/poly/sims/device/emu/src/lib.rs index dc8107ed5..087ba4418 100644 --- a/examples/designs/poly/sims/device/emu/src/lib.rs +++ b/examples/designs/poly/sims/device/emu/src/lib.rs @@ -7,6 +7,6 @@ extern crate bebop_core_reduce; include!(concat!(env!("OUT_DIR"), "/balls.rs")); pub const SIMULATION: bebop_bemu::Simulation = bebop_bemu::Simulation { - design: include_bytes!("../../../../output/generated/design.pb"), + design: include_bytes!(concat!(env!("OUT_DIR"), "/design.pb")), instantiate: bebop_bemu::device::cbus::instantiate, }; diff --git a/examples/designs/poly/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/poly/sims/device/regression/bemu/workloads-bare.toml index f4abb54a7..444a0a401 100644 --- a/examples/designs/poly/sims/device/regression/bemu/workloads-bare.toml +++ b/examples/designs/poly/sims/device/regression/bemu/workloads-bare.toml @@ -12,8 +12,14 @@ tests = [ "poly-main-ctest-design-ant_bank_rows-multicore-baremetal", "poly-main-ctest-design-ant_parallel-multicore-baremetal", "poly-main-ctest-design-ant_highpa-multicore-baremetal", - "poly-main-ctest-design-ant_corelink-multicore-baremetal", - "poly-main-ctest-design-ant_corelink_bank-multicore-baremetal", - "poly-main-ctest-design-ant_corelink_reduce-multicore-baremetal", - "poly-main-ctest-design-ant_coherence-multicore-baremetal", + "poly-main-ctest-design-mlir_ball_mxmm_mxfp8_chain-multicore-baremetal", + "poly-main-ctest-design-mlir_ball_mxmm_mxfp8_window-multicore-baremetal", + "poly-main-ctest-design-mlir_ball_mxmm_fma32-multicore-baremetal", + "poly-main-ctest-design-mlir_ball_mxmm_f32-multicore-baremetal", + "poly-main-ctest-design-mlir_ball_mxquant_two_blocks-multicore-baremetal", + "poly-main-ctest-design-mlir_bank_mxmm_mxfp8_chain-multicore-baremetal", + "poly-main-ctest-design-mlir_bank_mxmm_mxfp8_window-multicore-baremetal", + "poly-main-ctest-design-mlir_bank_mxmm_fma32-multicore-baremetal", + "poly-main-ctest-design-mlir_bank_mxmm_f32-multicore-baremetal", + "poly-main-ctest-design-mlir_bank_mxquant_two_blocks-multicore-baremetal", ] diff --git a/examples/designs/poly/sims/device/regression/bemu/workloads-linux.toml b/examples/designs/poly/sims/device/regression/bemu/workloads-linux.toml index e96de12dc..41a9cd13b 100644 --- a/examples/designs/poly/sims/device/regression/bemu/workloads-linux.toml +++ b/examples/designs/poly/sims/device/regression/bemu/workloads-linux.toml @@ -1,4 +1,4 @@ [workloads] tests = [ - "fw_payload-h9-poly-linux-mem4096M.elf", + "fw_payload-linux.elf", ] diff --git a/examples/designs/poly/sims/device/regression/p2e/workloads-linux.toml b/examples/designs/poly/sims/device/regression/p2e/workloads-linux.toml index 31b629e95..ca955b614 100644 --- a/examples/designs/poly/sims/device/regression/p2e/workloads-linux.toml +++ b/examples/designs/poly/sims/device/regression/p2e/workloads-linux.toml @@ -1,4 +1,4 @@ [workloads] tests = [ - "fw_payload-h9-poly-linux-mem4096M-ddr.hex", + "fw_payload-linux-ddr.hex", ] diff --git a/examples/designs/poly/sims/device/regression/verilator/workloads-bare.toml b/examples/designs/poly/sims/device/regression/verilator/workloads-bare.toml index 7bc345182..b89b1795d 100644 --- a/examples/designs/poly/sims/device/regression/verilator/workloads-bare.toml +++ b/examples/designs/poly/sims/device/regression/verilator/workloads-bare.toml @@ -12,8 +12,4 @@ tests = [ "poly-main-ctest-design-ant_parallel-multicore-baremetal", "poly-main-ctest-design-ant_bank-multicore-baremetal", "poly-main-ctest-design-ant_highpa-multicore-baremetal", - "poly-main-ctest-design-ant_corelink-multicore-baremetal", - "poly-main-ctest-design-ant_corelink_bank-multicore-baremetal", - "poly-main-ctest-design-ant_corelink_reduce-multicore-baremetal", - "poly-main-ctest-design-ant_coherence-multicore-baremetal", ] diff --git a/examples/designs/poly/sims/device/stack/CMakeLists.txt b/examples/designs/poly/sims/device/stack/CMakeLists.txt index 383594aa1..35a0cd209 100644 --- a/examples/designs/poly/sims/device/stack/CMakeLists.txt +++ b/examples/designs/poly/sims/device/stack/CMakeLists.txt @@ -2,7 +2,7 @@ # The software stack of the poly device simulation: every level the device # needs, built bottom up. # -# bbdev stack --build "--design poly" +# bbdev stack --build "--design poly --device " # # configures this with DESIGN_CMAKE, generated from design.pb. Each level is a # CMake build of its own: cores and threads build buddy-opt plugins for the diff --git a/examples/designs/poly/sims/device/stack/core/poly/CMakeLists.txt b/examples/designs/poly/sims/device/stack/core/poly/CMakeLists.txt deleted file mode 100644 index 5898bd39d..000000000 --- a/examples/designs/poly/sims/device/stack/core/poly/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -cmake_minimum_required(VERSION 3.20) -project(poly-core-stack LANGUAGES C CXX) -include("${DESIGN_CMAKE}") - -add_subdirectory("${DESIGN_REPO}/examples/designs/poly/stack/engine" engine) -add_subdirectory("${DESIGN_REPO}/examples/designs/poly/stack/midend" midend) diff --git a/examples/designs/poly/sims/device/workloads/CMakeLists.txt b/examples/designs/poly/sims/device/workloads/CMakeLists.txt index 096844c37..f0d468355 100644 --- a/examples/designs/poly/sims/device/workloads/CMakeLists.txt +++ b/examples/designs/poly/sims/device/workloads/CMakeLists.txt @@ -27,19 +27,16 @@ foreach(BUCKYBALL_COMPONENT_WORKLOAD_TARGET IN LISTS DESIGN_CPU_TARGETS) if( COMPONENT_WORKLOADS STREQUAL - "${BUCKYBALL_REPO_ROOT}/examples/designs/${BUCKYBALL_WORKLOAD_DESIGN}/workloads" + "${BUCKYBALL_REPO_ROOT}/examples/designs/${DESIGN_NAME}/workloads" ) continue() endif() - set(BUCKYBALL_COMPONENT_WORKLOAD_PROFILE - "${DESIGN_TARGET_ID_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" - ) set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}") set(BUCKYBALL_CTEST_GROUP_TARGET component-ctests-build) if(IS_DIRECTORY "${COMPONENT_WORKLOADS}/ctests") add_subdirectory( "${COMPONENT_WORKLOADS}/ctests" - "${CMAKE_CURRENT_BINARY_DIR}/components/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/prefab/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" ) endif() endforeach() @@ -58,9 +55,9 @@ add_subdirectory( ) foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) set(BUCKYBALL_MLIR_ACTIVE_TARGET "${BUCKYBALL_TARGET}") - foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_TARGET_BALLS_${BUCKYBALL_TARGET}) + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_BALLDOMAIN_BALLS) set(BALL_MLIR_TESTS - "${BUCKYBALL_REPO_ROOT}/examples/components/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" ) if(IS_DIRECTORY "${BALL_MLIR_TESTS}") add_subdirectory( @@ -72,6 +69,12 @@ foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) endforeach() add_dependencies(MLIRTest-all base-mlirtest-build balls-mlir-tests-build) +# The multicore Ant cases this entry's cores can run. +set(POLY_ANT_CASES + mesh_move + mesh_private + mesh_shared +) add_subdirectory( "${BUCKYBALL_REPO_ROOT}/examples/designs/poly/workloads" design diff --git a/examples/designs/poly/sims/device/stack/runtime/CMakeLists.txt b/examples/designs/poly/stack/device/CMakeLists.txt similarity index 91% rename from examples/designs/poly/sims/device/stack/runtime/CMakeLists.txt rename to examples/designs/poly/stack/device/CMakeLists.txt index 33cb18cf2..33cd52904 100644 --- a/examples/designs/poly/sims/device/stack/runtime/CMakeLists.txt +++ b/examples/designs/poly/stack/device/CMakeLists.txt @@ -2,7 +2,7 @@ # The poly device: its cores, their hardware threads and the runtime that # schedules jobs onto the cores. # -# bbdev stack --build "--design poly" +# bbdev stack --build "--design poly --device " # # builds every thread and core stack of the device first, then configures this # with DESIGN_CMAKE, generated from design.pb, and the RISC-V Linux toolchain. @@ -24,12 +24,12 @@ add_library(bindings INTERFACE) foreach(CORE IN LISTS DESIGN_CORES) target_link_options(bindings INTERFACE "${DESIGN_CORE_BUILD_${CORE}}/execute.o") endforeach() -foreach(THREAD IN LISTS DESIGN_THREADS) +foreach(THREAD IN LISTS DESIGN_THREAD_STACKS) target_link_options(bindings INTERFACE "${DESIGN_THREAD_BUILD_${THREAD}}/loader.o") endforeach() # DMA helpers are sized by the target with the fewest bank lines. list(GET DESIGN_RUNTIME_TARGETS 0 PARAMS_TARGET) -set(RUNTIME_PARAMS "${DESIGN_ISA_DIR}/${PARAMS_TARGET}") +set(RUNTIME_DEFS ${DESIGN_TARGET_DEFS_${PARAMS_TARGET}}) add_subdirectory("${DESIGN_REPO}/stack/engine" engine) add_subdirectory("${DESIGN_REPO}/stack/runtime" runtime) diff --git a/examples/designs/poly/stack/device/core/CMakeLists.txt b/examples/designs/poly/stack/device/core/CMakeLists.txt new file mode 100644 index 000000000..ac07de8b4 --- /dev/null +++ b/examples/designs/poly/stack/device/core/CMakeLists.txt @@ -0,0 +1,32 @@ +# ===----------------------------------------------------------------------=== +# Poly's own core: a CPU that runs the attention and ffn hardware threads. +# +# Built by the stack top of each simulation entry after the core's threads, +# configured with DESIGN_CMAKE and CORE. Products: +# execute.o how the core runs jobs (execute.c, through the engine) +# libengine-compiler.so buddy-opt pass plugin of the framework engine +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(poly-core-stack LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +set(ENGINE "${DESIGN_REPO}/stack/engine") +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# How the core runs jobs; instruction streams use the framework engine. +add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/execute.o" + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" + "-I${ENGINE}/include" -c "${CMAKE_CURRENT_SOURCE_DIR}/execute.c" -o "${CMAKE_BINARY_DIR}/execute.o" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/execute.c" "${RUNTIME}/scheduler/runtime.h" + "${ENGINE}/include/engine.h" + VERBATIM) +add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") + +# The engine's compiler plugin, for the threads this core runs. +add_subdirectory("${ENGINE}/compiler" engine-compiler) +add_library(engine-compiler-plugin MODULE $) +set_target_properties(engine-compiler-plugin PROPERTIES OUTPUT_NAME engine-compiler + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") diff --git a/examples/designs/poly/stack/engine/execute.c b/examples/designs/poly/stack/device/core/execute.c similarity index 100% rename from examples/designs/poly/stack/engine/execute.c rename to examples/designs/poly/stack/device/core/execute.c diff --git a/examples/designs/poly/stack/engine/CMakeLists.txt b/examples/designs/poly/stack/engine/CMakeLists.txt deleted file mode 100644 index 2a03cfe3f..000000000 --- a/examples/designs/poly/stack/engine/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Core execution binding; instruction streams use the framework engine. -set(RUNTIME "${DESIGN_REPO}/stack/runtime") -set(ENGINE "${DESIGN_REPO}/stack/engine") -find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) - -add_custom_command( - OUTPUT "${CMAKE_BINARY_DIR}/execute.o" - COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 - "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" - "-I${ENGINE}/include" -c "${CMAKE_CURRENT_SOURCE_DIR}/execute.c" -o "${CMAKE_BINARY_DIR}/execute.o" - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/execute.c" "${RUNTIME}/scheduler/runtime.h" - "${ENGINE}/include/engine.h" - VERBATIM) -add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") diff --git a/examples/designs/poly/stack/midend/CMakeLists.txt b/examples/designs/poly/stack/midend/CMakeLists.txt deleted file mode 100644 index f8202e654..000000000 --- a/examples/designs/poly/stack/midend/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Host compiler plugin for this design's core. -set(ENGINE "${DESIGN_REPO}/stack/engine") -set(TARGET ${DESIGN_CORE_TARGET_${CORE}}) - -add_subdirectory("${ENGINE}/compiler" engine-compiler) -add_library(engine-compiler-plugin MODULE $) -set_target_properties(engine-compiler-plugin PROPERTIES OUTPUT_NAME engine-compiler - LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") diff --git a/examples/designs/poly/workloads/CMakeLists.txt b/examples/designs/poly/workloads/CMakeLists.txt index d1c990b8a..2ce389b29 100644 --- a/examples/designs/poly/workloads/CMakeLists.txt +++ b/examples/designs/poly/workloads/CMakeLists.txt @@ -3,6 +3,15 @@ set(BUCKYBALL_CTEST_GROUP_TARGET poly-ctests-build) set(BUCKYBALL_CTEST_TARGET attention) set(POLY_BOOT_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/device/common/controllers_start.S) set(BUCKYBALL_DESIGN_COMMON_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/device/core/thread/ctests) +# The cores that run Ant threads: the programs here keep state per such core. +set(POLY_THREAD_CORES 0) +foreach(CORE IN LISTS DESIGN_DEVICE_CORES) + if(NOT "${DESIGN_DEVICE_CORES_${CORE}_THREAD_INDICES}" STREQUAL "") + math(EXPR POLY_THREAD_CORES "${POLY_THREAD_CORES} + 1") + endif() +endforeach() +list(APPEND DESIGN_DEFS POLY_THREAD_CORES=${POLY_THREAD_CORES}) +list(APPEND DESIGN_DEFINES -DPOLY_THREAD_CORES=${POLY_THREAD_CORES}) add_subdirectory(device) diff --git a/examples/designs/poly/workloads/device/core/ball/CMakeLists.txt b/examples/designs/poly/workloads/device/core/ball/CMakeLists.txt index 0abc064dd..e0c194ea5 100644 --- a/examples/designs/poly/workloads/device/core/ball/CMakeLists.txt +++ b/examples/designs/poly/workloads/device/core/ball/CMakeLists.txt @@ -1,5 +1,5 @@ set(BUCKYBALL_CTEST_TARGET main) set(BUCKYBALL_CTEST_GROUP_TARGET poly-ctests-build) add_subdirectory( - ${BUCKYBALL_REPO_ROOT}/examples/components/balls/f32add/workloads/ctests + ${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/f32add/workloads/ctests f32add) diff --git a/examples/designs/poly/workloads/device/core/engine/CMakeLists.txt b/examples/designs/poly/workloads/device/core/engine/CMakeLists.txt index 6ae5e3818..12923655f 100644 --- a/examples/designs/poly/workloads/device/core/engine/CMakeLists.txt +++ b/examples/designs/poly/workloads/device/core/engine/CMakeLists.txt @@ -14,7 +14,7 @@ foreach( set(POLY_ANT_THREAD ffn) endif() set(POLY_ANT_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${POLY_ANT_THREAD}-ctest-design-ant_${POLY_ANT_CASE}" + "${DESIGN_NAME}-${POLY_ANT_THREAD}-ctest-design-ant_${POLY_ANT_CASE}" ) set(POLY_ANT_IMAGE ${POLY_ANT_IMAGE_DIR}/${POLY_ANT_NAME}-ant) set(POLY_ANT_CONTROL_SOURCES) @@ -51,7 +51,7 @@ foreach( endif() buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_MAIN}") set(POLY_ANT_HOST_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-main-ctest-design-ant_${POLY_ANT_CASE}" + "${DESIGN_NAME}-main-ctest-design-ant_${POLY_ANT_CASE}" ) set(BUCKYBALL_CTEST_TARGET ${POLY_ANT_THREAD}) if(NOT POLY_ANT_CASE STREQUAL "highpa") @@ -62,7 +62,11 @@ foreach( ) target_include_directories( ${POLY_ANT_HOST_NAME}-linux - PRIVATE ${BUCKYBALL_ISA_DIR}/design + PRIVATE ${RUNTIME_INCLUDE_DIR} + ) + target_compile_definitions( + ${POLY_ANT_HOST_NAME}-linux + PRIVATE ${DESIGN_DEFS} ) target_link_libraries( ${POLY_ANT_HOST_NAME}-linux @@ -76,6 +80,8 @@ foreach( set(POLY_ANT_CONTROL_OBJECTS) set(POLY_ANT_HOST_ELF ${POLY_ANT_HOST_NAME}-multicore-baremetal) + set(POLY_ANT_HOST_DEFINES ${DESIGN_TARGET_DEFS_${POLY_ANT_HOST_THREAD}}) + list(TRANSFORM POLY_ANT_HOST_DEFINES PREPEND "-D") add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_HOST_ELF} COMMAND @@ -83,9 +89,9 @@ foreach( -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${BBSIM_LD} ${POLY_ANT_HOST_DEFINITIONS} -I${POLY_ANT_SOURCE_DIR} - -I${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_THREAD} - -I${BUCKYBALL_ISA_DIR}/design -I${WORKLOAD_LIB_DIR} - -I${BUCKYBALL_REPO_ROOT}/examples/components/balls/f32add/workloads + ${POLY_ANT_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/f32add/workloads -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} @@ -103,16 +109,14 @@ foreach( ${BBSIM_LD} ${POLY_ANT_CONTROL_SOURCES} ${POLY_ANT_CONTROL_OBJECTS} - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_THREAD}/params.h - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_THREAD}/ballISA.h ${POLY_ANT_IMAGE}.bin ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.h - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD}/params.h - ${BUCKYBALL_ISA_DIR}/design/instances.h + ${RUNTIME_INCLUDE_DIR}/bb_cores.h WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM ) add_custom_target( ${POLY_ANT_HOST_NAME}-ctest-build @@ -120,3 +124,4 @@ foreach( ) add_dependencies(poly-ctests-build ${POLY_ANT_HOST_NAME}-ctest-build) endforeach() +add_subdirectory(mlir) diff --git a/examples/designs/poly/workloads/device/core/engine/ant_bank_exercise.c b/examples/designs/poly/workloads/device/core/engine/ant_bank_exercise.c index 318efbdbf..4c7227943 100644 --- a/examples/designs/poly/workloads/device/core/engine/ant_bank_exercise.c +++ b/examples/designs/poly/workloads/device/core/engine/ant_bank_exercise.c @@ -1,22 +1,20 @@ #include "ant_bank.h" -#include -#include #include extern const unsigned char ant_image[], ant_image_end[]; enum { - WORDS = BANK_LINES * BANK_WIDTH / 64, - SHARED_WORDS = BB_SHARED_BANK_LINES * BANK_WIDTH / 64, + WORDS = THREAD_MEM_BANK_ENTRIES * THREAD_MEM_BANK_WIDTH / 64, + SHARED_WORDS = CORE_SHARED_MEM_BANK_ENTRIES * THREAD_MEM_BANK_WIDTH / 64, INPUT_WORDS = WORDS > SHARED_WORDS ? WORDS : SHARED_WORDS }; -static uint64_t input[BB_THREAD_CORE_COUNT][INPUT_WORDS] +static uint64_t input[POLY_THREAD_CORES][INPUT_WORDS] __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t private_out[BB_THREAD_CORE_COUNT][WORDS] +static uint64_t private_out[POLY_THREAD_CORES][WORDS] __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t shared_out[BB_THREAD_CORE_COUNT][SHARED_WORDS] +static uint64_t shared_out[POLY_THREAD_CORES][SHARED_WORDS] __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t tail_out[BB_THREAD_CORE_COUNT][8] +static uint64_t tail_out[POLY_THREAD_CORES][8] __attribute__((section(".data"), aligned(4096))) = {0}; -unsigned bank_done[BB_THREAD_CORE_COUNT]; +unsigned bank_done[POLY_THREAD_CORES]; void bank_exercise(unsigned core) { const unsigned context = thread_find(THREAD_SIGNATURE, 0); for (unsigned i = 0; i < INPUT_WORDS; ++i) @@ -31,10 +29,10 @@ void bank_exercise(unsigned core) { if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) __builtin_trap(); - struct ant_bank_args args = {(uintptr_t)input[core], - (uintptr_t)private_out[core], - (uintptr_t)shared_out[core], - (uintptr_t)tail_out[core], BB_SHARED_BANK_LINES}; + struct ant_bank_args args = { + (uintptr_t)input[core], (uintptr_t)private_out[core], + (uintptr_t)shared_out[core], (uintptr_t)tail_out[core], + CORE_SHARED_MEM_BANK_ENTRIES}; thread_tss_write(0, ant_image, bytes); thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); thread_tss_write(0, &args, sizeof(args)); diff --git a/examples/designs/poly/workloads/device/core/engine/ant_bank_main.c b/examples/designs/poly/workloads/device/core/engine/ant_bank_main.c index bba2a6502..09d9e6dbe 100644 --- a/examples/designs/poly/workloads/device/core/engine/ant_bank_main.c +++ b/examples/designs/poly/workloads/device/core/engine/ant_bank_main.c @@ -1,7 +1,5 @@ #define _GNU_SOURCE #include "ant_bank.h" -#include -#include #include #ifdef __linux__ #include @@ -26,20 +24,21 @@ int main(void) { #ifdef __linux__ if (mlockall(MCL_CURRENT | MCL_FUTURE)) abort(); - pthread_t threads[BB_THREAD_CORE_COUNT]; - for (uintptr_t core = 0; core < BB_THREAD_CORE_COUNT; ++core) + pthread_t threads[POLY_THREAD_CORES]; + for (uintptr_t core = 0; core < POLY_THREAD_CORES; ++core) if (pthread_create(&threads[core], NULL, worker, (void *)core)) abort(); - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) if (pthread_join(threads[core], NULL)) abort(); printf("Ant bank boundary PASS cores=%u private=%u shared=%u\n", - (unsigned)BB_THREAD_CORE_COUNT, BANK_LINES, BB_SHARED_BANK_LINES); + (unsigned)POLY_THREAD_CORES, THREAD_MEM_BANK_ENTRIES, + CORE_SHARED_MEM_BANK_ENTRIES); #else uintptr_t hart; __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart == BB_REPORT_HART) { - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) while (!__atomic_load_n(&bank_done[core], __ATOMIC_ACQUIRE)) { } } else { diff --git a/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_exercise.c b/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_exercise.c index 44e572ef2..8af47dfa6 100644 --- a/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_exercise.c +++ b/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_exercise.c @@ -1,21 +1,19 @@ #include "ant_bank_rows.h" -#include -#include #include extern const unsigned char ant_image[], ant_image_end[]; enum { - WORDS = ANT_BANK_ROWS * BANK_WIDTH / 64, - SHARED_WORDS = ANT_BANK_ROWS * BANK_WIDTH / 64 + WORDS = ANT_BANK_ROWS * THREAD_MEM_BANK_WIDTH / 64, + SHARED_WORDS = ANT_BANK_ROWS * THREAD_MEM_BANK_WIDTH / 64 }; -static uint64_t input[BB_THREAD_CORE_COUNT][WORDS] +static uint64_t input[POLY_THREAD_CORES][WORDS] __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t private_out[BB_THREAD_CORE_COUNT][WORDS] +static uint64_t private_out[POLY_THREAD_CORES][WORDS] __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t shared_out[BB_THREAD_CORE_COUNT][SHARED_WORDS] +static uint64_t shared_out[POLY_THREAD_CORES][SHARED_WORDS] __attribute__((section(".data"), aligned(4096))) = {0}; -static uint64_t tail_out[BB_THREAD_CORE_COUNT][8] +static uint64_t tail_out[POLY_THREAD_CORES][8] __attribute__((section(".data"), aligned(4096))) = {0}; -unsigned bank_rows_done[BB_THREAD_CORE_COUNT]; +unsigned bank_rows_done[POLY_THREAD_CORES]; void bank_rows_exercise(unsigned core) { const unsigned context = thread_find(THREAD_SIGNATURE, 0); for (unsigned i = 0; i < WORDS; ++i) diff --git a/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_main.c b/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_main.c index 4769f77e4..1a80cc922 100644 --- a/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_main.c +++ b/examples/designs/poly/workloads/device/core/engine/ant_bank_rows_main.c @@ -1,7 +1,5 @@ #define _GNU_SOURCE #include "ant_bank_rows.h" -#include -#include #include #ifdef __linux__ #include @@ -26,20 +24,21 @@ int main(void) { #ifdef __linux__ if (mlockall(MCL_CURRENT | MCL_FUTURE)) abort(); - pthread_t threads[BB_THREAD_CORE_COUNT]; - for (uintptr_t core = 0; core < BB_THREAD_CORE_COUNT; ++core) + pthread_t threads[POLY_THREAD_CORES]; + for (uintptr_t core = 0; core < POLY_THREAD_CORES; ++core) if (pthread_create(&threads[core], NULL, worker, (void *)core)) abort(); - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) if (pthread_join(threads[core], NULL)) abort(); printf("Ant bank boundary PASS cores=%u private=%u shared=%u\n", - (unsigned)BB_THREAD_CORE_COUNT, BANK_LINES, BB_SHARED_BANK_LINES); + (unsigned)POLY_THREAD_CORES, THREAD_MEM_BANK_ENTRIES, + CORE_SHARED_MEM_BANK_ENTRIES); #else uintptr_t hart; __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart == BB_REPORT_HART) { - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) while (!__atomic_load_n(&bank_rows_done[core], __ATOMIC_ACQUIRE)) { } } else { diff --git a/examples/designs/poly/workloads/device/core/engine/ant_highpa_main.c b/examples/designs/poly/workloads/device/core/engine/ant_highpa_main.c index bcedfacf4..4bf3781b9 100644 --- a/examples/designs/poly/workloads/device/core/engine/ant_highpa_main.c +++ b/examples/designs/poly/workloads/device/core/engine/ant_highpa_main.c @@ -1,9 +1,7 @@ #include "ant_highpa.h" -#include -#include #include extern const unsigned char ant_image[], ant_image_end[]; -static unsigned done[BB_THREAD_CORE_COUNT]; +static unsigned done[POLY_THREAD_CORES]; static void exercise(unsigned core) { const unsigned context = thread_find(THREAD_SIGNATURE, 0); uintptr_t high = 0x180200000ULL + core * 0x20000ULL; @@ -57,8 +55,8 @@ static void exercise(unsigned core) { int main(void) { uintptr_t hart; __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart == BB_REPORT_HART) { - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { } } else { diff --git a/examples/designs/poly/workloads/device/core/engine/ant_mxmm_main.cpp b/examples/designs/poly/workloads/device/core/engine/ant_mxmm_main.cpp index b49e632da..ecde76a90 100644 --- a/examples/designs/poly/workloads/device/core/engine/ant_mxmm_main.cpp +++ b/examples/designs/poly/workloads/device/core/engine/ant_mxmm_main.cpp @@ -1,7 +1,5 @@ #define _GNU_SOURCE #include "ant_mxmm.h" -#include -#include #include #ifdef __linux__ #include @@ -12,13 +10,13 @@ #endif extern const unsigned char ant_image[], ant_image_end[]; struct matrices { - float a[BB_THREAD_CORE_COUNT][ANT_M * ANT_K]; - float b[BB_THREAD_CORE_COUNT][ANT_N * ANT_K]; - float out[BB_THREAD_CORE_COUNT][ANT_M * ANT_N]; + float a[POLY_THREAD_CORES][ANT_M * ANT_K]; + float b[POLY_THREAD_CORES][ANT_N * ANT_K]; + float out[POLY_THREAD_CORES][ANT_M * ANT_N]; }; static constexpr matrices prepare() { matrices data{}; - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { for (unsigned row = 0; row < ANT_M; ++row) for (unsigned k = 0; k < ANT_K; ++k) data.a[core][row * ANT_K + k] = row + core + 1; @@ -31,7 +29,7 @@ static constexpr matrices prepare() { return data; } alignas(64) static matrices data = prepare(); -static unsigned done[BB_THREAD_CORE_COUNT]; +static unsigned done[POLY_THREAD_CORES]; static void exercise(unsigned core) { const uint32_t context = thread_find(THREAD_SIGNATURE, 0); size_t bytes = ant_image_end - ant_image; @@ -77,19 +75,19 @@ int main(void) { #ifdef __linux__ if (mlockall(MCL_CURRENT | MCL_FUTURE)) abort(); - pthread_t threads[BB_THREAD_CORE_COUNT]; - for (uintptr_t core = 0; core < BB_THREAD_CORE_COUNT; ++core) + pthread_t threads[POLY_THREAD_CORES]; + for (uintptr_t core = 0; core < POLY_THREAD_CORES; ++core) if (pthread_create(&threads[core], NULL, worker, (void *)core)) abort(); - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) if (pthread_join(threads[core], NULL)) abort(); - printf("Ant Mxmm PASS cores=%u\n", (unsigned)BB_THREAD_CORE_COUNT); + printf("Ant Mxmm PASS cores=%u\n", (unsigned)POLY_THREAD_CORES); #else uintptr_t hart; __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart == BB_REPORT_HART) { - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { } } else { diff --git a/examples/designs/poly/workloads/device/core/engine/ant_parallel_exercise.c b/examples/designs/poly/workloads/device/core/engine/ant_parallel_exercise.c index 8144fab77..eb39f8319 100644 --- a/examples/designs/poly/workloads/device/core/engine/ant_parallel_exercise.c +++ b/examples/designs/poly/workloads/device/core/engine/ant_parallel_exercise.c @@ -1,18 +1,16 @@ #include "ant_parallel.h" -#include -#include #include extern const unsigned char ant_image[], ant_image_end[]; -static uint64_t input[BB_THREAD_CORE_COUNT][2][512] +static uint64_t input[POLY_THREAD_CORES][2][512] __attribute__((aligned(4096), section(".noinit"))); -static uint64_t output[BB_THREAD_CORE_COUNT][2][512] +static uint64_t output[POLY_THREAD_CORES][2][512] __attribute__((aligned(4096), section(".noinit"))); -unsigned parallel_done[BB_THREAD_CORE_COUNT]; +unsigned parallel_done[POLY_THREAD_CORES]; static unsigned reached[3] __attribute__((aligned(64))); static void barrier(unsigned phase) { __atomic_fetch_add(&reached[phase], 1, __ATOMIC_ACQ_REL); while (__atomic_load_n(&reached[phase], __ATOMIC_ACQUIRE) != - BB_THREAD_CORE_COUNT) { + POLY_THREAD_CORES) { } } void parallel_exercise(unsigned core) { @@ -49,7 +47,7 @@ void parallel_exercise(unsigned core) { } thread_tss_write(0, &zero, sizeof(zero)); barrier(0); - unsigned peer = (core + 1) % BB_THREAD_CORE_COUNT; + unsigned peer = (core + 1) % POLY_THREAD_CORES; for (unsigned context = 0; context < 2; ++context) for (unsigned word = 0; word < 2; ++word) if (*(volatile uint64_t *)&output[peer][context][word] != ~0ULL) diff --git a/examples/designs/poly/workloads/device/core/engine/ant_parallel_main.c b/examples/designs/poly/workloads/device/core/engine/ant_parallel_main.c index 6929f72f6..e935768ca 100644 --- a/examples/designs/poly/workloads/device/core/engine/ant_parallel_main.c +++ b/examples/designs/poly/workloads/device/core/engine/ant_parallel_main.c @@ -1,7 +1,5 @@ #define _GNU_SOURCE #include "ant_parallel.h" -#include -#include #include #ifdef __linux__ #include @@ -26,20 +24,20 @@ int main(void) { #ifdef __linux__ if (mlockall(MCL_CURRENT | MCL_FUTURE)) abort(); - pthread_t threads[BB_THREAD_CORE_COUNT]; - for (uintptr_t core = 0; core < BB_THREAD_CORE_COUNT; ++core) + pthread_t threads[POLY_THREAD_CORES]; + for (uintptr_t core = 0; core < POLY_THREAD_CORES; ++core) if (pthread_create(&threads[core], NULL, worker, (void *)core)) abort(); - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) if (pthread_join(threads[core], NULL)) abort(); printf("Ant concurrent DMA/cache sharing PASS cores=%u contexts=2\n", - (unsigned)BB_THREAD_CORE_COUNT); + (unsigned)POLY_THREAD_CORES); #else uintptr_t hart; __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart == BB_REPORT_HART) { - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) while (!__atomic_load_n(¶llel_done[core], __ATOMIC_ACQUIRE)) { } } else { diff --git a/examples/designs/poly/workloads/device/core/engine/ant_shared_exercise.c b/examples/designs/poly/workloads/device/core/engine/ant_shared_exercise.c index f01e8d09c..86786ec17 100644 --- a/examples/designs/poly/workloads/device/core/engine/ant_shared_exercise.c +++ b/examples/designs/poly/workloads/device/core/engine/ant_shared_exercise.c @@ -1,9 +1,7 @@ #include "ant_shared.h" -#include -#include #include extern const unsigned char ant_image[], ant_image_end[]; -unsigned shared_done[BB_THREAD_CORE_COUNT]; +unsigned shared_done[POLY_THREAD_CORES]; void shared_exercise(unsigned core) { const uint32_t contexts[] = {thread_find(THREAD_SIGNATURE, 0), thread_find(THREAD_SIGNATURE, 1)}; diff --git a/examples/designs/poly/workloads/device/core/engine/ant_shared_main.c b/examples/designs/poly/workloads/device/core/engine/ant_shared_main.c index 56057ff11..0c2243e06 100644 --- a/examples/designs/poly/workloads/device/core/engine/ant_shared_main.c +++ b/examples/designs/poly/workloads/device/core/engine/ant_shared_main.c @@ -1,7 +1,5 @@ #define _GNU_SOURCE #include "ant_shared.h" -#include -#include #include #ifdef __linux__ #include @@ -23,21 +21,21 @@ static void *worker(void *argument) { #endif int main(void) { #ifdef __linux__ - pthread_t threads[BB_THREAD_CORE_COUNT]; - for (uintptr_t core = 0; core < BB_THREAD_CORE_COUNT; ++core) + pthread_t threads[POLY_THREAD_CORES]; + for (uintptr_t core = 0; core < POLY_THREAD_CORES; ++core) if (pthread_create(&threads[core], NULL, worker, (void *)core)) abort(); - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) if (pthread_join(threads[core], NULL)) abort(); - printf("Ant shared/cancel PASS cores=%u\n", (unsigned)BB_THREAD_CORE_COUNT); + printf("Ant shared/cancel PASS cores=%u\n", (unsigned)POLY_THREAD_CORES); #else uintptr_t hart; __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart != BB_REPORT_HART) + if (hart != WORKLOAD_REPORT_HART) shared_exercise(hart - 1); else - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) while (!__atomic_load_n(&shared_done[core], __ATOMIC_ACQUIRE)) { } #endif diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/CMakeLists.txt b/examples/designs/poly/workloads/device/core/engine/mlir/CMakeLists.txt new file mode 100644 index 000000000..d3d38fa8c --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/CMakeLists.txt @@ -0,0 +1,437 @@ +# Host programs of the MxmmBall and MxquantBall MLIR tests: each embeds the +# thread image thread/mlir built for the test, runs it on the attention thread +# of every core and checks what it wrote. +set(POLY_MLIR_HOST_DEFINES ${DESIGN_TARGET_DEFS_attention}) +list(TRANSFORM POLY_MLIR_HOST_DEFINES PREPEND "-D") +set(POLY_MLIR_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../thread/mlir") +enable_language(ASM) + +# ball_mxmm_mxfp8_chain +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_chain_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_chain-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_mxfp8_chain_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_chain_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_chain-multicore-baremetal + DEPENDS + ball_mxmm_mxfp8_chain_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_chain_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_chain-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_chain-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_chain-ctest-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_chain-ctest-build) + +# ball_mxmm_mxfp8_window +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_window_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_window-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_mxfp8_window_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_window_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_window-multicore-baremetal + DEPENDS + ball_mxmm_mxfp8_window_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_window_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_window-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_window-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_window-ctest-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_mxfp8_window-ctest-build) + +# ball_mxmm_fma32 +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_fma32_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_fma32-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_fma32_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_fma32_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_fma32-multicore-baremetal + DEPENDS + ball_mxmm_fma32_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_fma32_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_fma32-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_fma32-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_fma32-ctest-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_fma32-ctest-build) + +# ball_mxmm_f32 +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_f32_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_f32-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_f32_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_f32_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_f32-multicore-baremetal + DEPENDS + ball_mxmm_f32_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_f32_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_f32-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_f32-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_f32-ctest-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxmm_f32-ctest-build) + +# ball_mxquant_two_blocks +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ball_mxquant_two_blocks_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxquant_two_blocks-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxquant_two_blocks_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxquant_two_blocks_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxquant_two_blocks-multicore-baremetal + DEPENDS + ball_mxquant_two_blocks_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxquant_two_blocks_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxquant_two_blocks-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_ball_mxquant_two_blocks-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_ball_mxquant_two_blocks-ctest-build ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_ball_mxquant_two_blocks-ctest-build) + +# bank_mxmm_mxfp8_chain +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_chain_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_chain-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_mxfp8_chain_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_chain_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_chain-multicore-baremetal + DEPENDS + bank_mxmm_mxfp8_chain_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_chain_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_chain-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_chain-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_chain-ctest-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_chain-ctest-build) + +# bank_mxmm_mxfp8_window +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_window_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_window-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_mxfp8_window_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_window_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_window-multicore-baremetal + DEPENDS + bank_mxmm_mxfp8_window_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_window_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_window-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_window-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_window-ctest-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_mxfp8_window-ctest-build) + +# bank_mxmm_fma32 +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_fma32_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_fma32-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_fma32_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_fma32_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_fma32-multicore-baremetal + DEPENDS + bank_mxmm_fma32_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_fma32_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_fma32-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_fma32-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_fma32-ctest-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_fma32-ctest-build) + +# bank_mxmm_f32 +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_f32_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_f32-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_f32_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_f32_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_f32-multicore-baremetal + DEPENDS + bank_mxmm_f32_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_f32_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_f32-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_f32-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_f32-ctest-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxmm_f32-ctest-build) + +# bank_mxquant_two_blocks +file( + GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/bank_mxquant_two_blocks_image.S + CONTENT + ".section .rodata.thread,\"a\",@progbits\n.balign 8\n.global thread_image, thread_image_end\nthread_image:\n.incbin \"${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.bin\"\nthread_image_end:\n.section .note.GNU-stack,\"\",@progbits\n" +) +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxquant_two_blocks-multicore-baremetal + COMMAND + ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 + -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${BBSIM_LD} -I${POLY_MLIR_SOURCE_DIR} ${POLY_MLIR_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler + -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services + -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} + ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxquant_two_blocks_main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxquant_two_blocks_image.S + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c -lgcc -o + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxquant_two_blocks-multicore-baremetal + DEPENDS + bank_mxquant_two_blocks_main.cpp + ${POLY_MLIR_SOURCE_DIR}/mlir.h + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxquant_two_blocks_image.S + ${POLY_MLIR_IMAGE_DIR}/${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.bin + ${POLY_BOOT_SOURCE} + ${BBSIM_LD} + ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c + ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h + ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c + ${RUNTIME_INCLUDE_DIR}/bb_cores.h + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM +) +add_custom_target( + ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxquant_two_blocks-ctest-build + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${DESIGN_NAME}-main-ctest-design-mlir_bank_mxquant_two_blocks-multicore-baremetal +) +add_dependencies(${DESIGN_NAME}-main-ctest-design-mlir_bank_mxquant_two_blocks-ctest-build ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks-thread-build) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-main-ctest-design-mlir_bank_mxquant_two_blocks-ctest-build) diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_f32_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_f32_main.cpp new file mode 100644 index 000000000..eecce4928 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_f32_main.cpp @@ -0,0 +1,88 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: 16 rows of 4 FP32. B: 16 columns of 4 FP32. C: 16x16 FP32, then a guard +// row. Row r of A is {-1, 1 + 2^-23, r, 1}, column j of B {1, 1 - 2^-23, 1, j}. +struct buffers { + uint32_t a[16 * 4]; + uint32_t b[16 * 4]; + uint32_t c[65 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned i = 0; i < 16; ++i) { + buffer.a[i * 4 + 0] = __builtin_bit_cast(uint32_t, -1.0f); + buffer.a[i * 4 + 1] = 0x3f800001; + buffer.a[i * 4 + 2] = __builtin_bit_cast(uint32_t, (float)i); + buffer.a[i * 4 + 3] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 0] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 1] = 0x3f7ffffe; + buffer.b[i * 4 + 2] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 3] = __builtin_bit_cast(uint32_t, (float)i); + } + for (unsigned i = 0; i < 65 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c}; + run(core, &args, sizeof(args)); + // Unfused: (1 + 2^-23)(1 - 2^-23) rounds to 1 and cancels the -1. + for (unsigned row = 0; row < 16; ++row) + for (unsigned column = 0; column < 16; ++column) + if (buffer.c[row * 16 + column] != + __builtin_bit_cast(uint32_t, (float)(row + column))) + __builtin_trap(); + for (unsigned i = 256; i < 260; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_fma32_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_fma32_main.cpp new file mode 100644 index 000000000..6efafcb21 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_fma32_main.cpp @@ -0,0 +1,91 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: 16 rows of 4 FP32. B: 16 columns of 4 FP32. C: 16x16 FP32, then a guard +// row. Row r of A is {-1, 1 + 2^-23, r, 1}, column j of B {1, 1 - 2^-23, 1, j}. +struct buffers { + uint32_t a[16 * 4]; + uint32_t b[16 * 4]; + uint32_t c[65 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned i = 0; i < 16; ++i) { + buffer.a[i * 4 + 0] = __builtin_bit_cast(uint32_t, -1.0f); + buffer.a[i * 4 + 1] = 0x3f800001; + buffer.a[i * 4 + 2] = __builtin_bit_cast(uint32_t, (float)i); + buffer.a[i * 4 + 3] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 0] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 1] = 0x3f7ffffe; + buffer.b[i * 4 + 2] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 3] = __builtin_bit_cast(uint32_t, (float)i); + } + for (unsigned i = 0; i < 65 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c}; + run(core, &args, sizeof(args)); + // Fused: -1 + (1 + 2^-23)(1 - 2^-23) leaves -2^-46, which survives only + // where r + j is 0. + for (unsigned row = 0; row < 16; ++row) + for (unsigned column = 0; column < 16; ++column) + if (buffer.c[row * 16 + column] != + (row == 0 && column == 0 + ? 0xa8800000u + : __builtin_bit_cast(uint32_t, (float)(row + column)))) + __builtin_trap(); + for (unsigned i = 256; i < 260; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_mxfp8_chain_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_mxfp8_chain_main.cpp new file mode 100644 index 000000000..7b473e246 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_mxfp8_chain_main.cpp @@ -0,0 +1,96 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: 16 rows of 32 E4M3 codes, then 16 scales. B: 16 columns of 32 codes, +// then 16 scales. C: 16x16 FP32 from row 1, the rest a guard pattern. +// Row r of A is 0.5 * (1 + (r % 8) / 8) (code 0x38 + r % 8, scale 2^-1); +// column j of B is 4 * (1 + (j % 8) / 8) (code 0x40 + j % 8, scale 2^1). +struct buffers { + uint8_t a[33 * 16]; + uint8_t b[33 * 16]; + uint32_t c[65 * 4]; + uint32_t between[65 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned i = 0; i < 16 * 32; ++i) { + buffer.a[i] = 0x38 + (i / 32) % 8; + buffer.b[i] = 0x40 + (i / 32) % 8; + } + for (unsigned i = 16 * 32; i < 16 * 33; ++i) { + buffer.a[i] = 126; + buffer.b[i] = 128; + } + for (unsigned i = 0; i < 65 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + for (unsigned i = 0; i < 65 * 4; ++i) + buffer.between[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_chain_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c, + (uintptr_t)buffer.between}; + run(core, &args, sizeof(args)); + // The first step does not write C. + for (unsigned i = 0; i < 65 * 4; ++i) + if (buffer.between[i] != 0x5a5a5a5a) + __builtin_trap(); + for (unsigned i = 0; i < 4; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + // Each step adds (8 + r % 8) * (8 + j % 8); the chain has two. + for (unsigned row = 0; row < 16; ++row) + for (unsigned column = 0; column < 16; ++column) + if (__builtin_bit_cast(float, buffer.c[4 + row * 16 + column]) != + 2.0f * (8 + row % 8) * (8 + column % 8)) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_mxfp8_window_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_mxfp8_window_main.cpp new file mode 100644 index 000000000..55905949e --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxmm_mxfp8_window_main.cpp @@ -0,0 +1,87 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: one row of 64 E4M3 codes, then its two scales. B: 16 columns of 32 +// codes, then 16 scales. C: one row of 16 FP32, then a guard row. +// Outside the window A holds NaN codes and a NaN scale, which the Ball must +// not read. Inside, element t is 1 + (t % 8) / 8; column j of B is +// 2 * (1 + (j % 8) / 8). +struct buffers { + uint8_t a[5 * 16]; + uint8_t b[33 * 16]; + uint32_t c[5 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned t = 0; t < 64; ++t) + buffer.a[t] = t < 32 ? 0x7f : 0x38 + t % 8; + buffer.a[64] = 255; + buffer.a[65] = 127; + for (unsigned i = 0; i < 16 * 32; ++i) + buffer.b[i] = 0x40 + (i / 32) % 8; + for (unsigned i = 16 * 32; i < 16 * 33; ++i) + buffer.b[i] = 127; + for (unsigned i = 0; i < 5 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c}; + run(core, &args, sizeof(args)); + // The window sums to 46, so column j is 46 * 2 * (8 + j % 8) / 8. + for (unsigned column = 0; column < 16; ++column) + if (__builtin_bit_cast(float, buffer.c[column]) != 11.5f * (8 + column % 8)) + __builtin_trap(); + for (unsigned i = 16; i < 20; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxquant_two_blocks_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxquant_two_blocks_main.cpp new file mode 100644 index 000000000..113908a31 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/ball_mxquant_two_blocks_main.cpp @@ -0,0 +1,92 @@ +#include "mlir.h" +#include "reference.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// Input: two blocks of 32 FP32. Output: the 64 codes, the two scales from +// byte 64, then guard bytes up to the end of row 4. Block 0 holds signed +// zeros, subnormals, rounding ties and the largest finite; block 1 one large +// element that sets the scale and small ones that round. +struct buffers { + uint32_t input[16 * 4]; + uint8_t output[5 * 16]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + const uint32_t edges[] = {0, 0x80000000, 1, 0x80000001, + 0x007fffff, 0x00800000, 0x3f880000, 0x3f980000, + 0x43e00000, 0x7f7fffff, 0xff7fffff}; + for (unsigned i = 0; i < 32; ++i) { + buffer.input[i] = edges[i % 11]; + buffer.input[32 + i] = + i ? (i & 1 ? 0x3f880000u : 0xbf980000u) : 0x43e00000u; + } + for (unsigned i = 0; i < 5 * 16; ++i) + buffer.output[i] = 0x5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxquant_args args = {(uintptr_t)buffer.input, + (uintptr_t)buffer.output}; + run(core, &args, sizeof(args)); + uint8_t expected[2][33]; + mxquant_reference32(buffer.input, expected[0]); + mxquant_reference32(buffer.input + 32, expected[1]); + for (unsigned block = 0; block < 2; ++block) { + for (unsigned i = 0; i < 32; ++i) + if (buffer.output[block * 32 + i] != expected[block][i]) + __builtin_trap(); + if (buffer.output[64 + block] != expected[block][32]) + __builtin_trap(); + } + for (unsigned i = 66; i < 80; ++i) + if (buffer.output[i] != 0x5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_f32_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_f32_main.cpp new file mode 100644 index 000000000..eecce4928 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_f32_main.cpp @@ -0,0 +1,88 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: 16 rows of 4 FP32. B: 16 columns of 4 FP32. C: 16x16 FP32, then a guard +// row. Row r of A is {-1, 1 + 2^-23, r, 1}, column j of B {1, 1 - 2^-23, 1, j}. +struct buffers { + uint32_t a[16 * 4]; + uint32_t b[16 * 4]; + uint32_t c[65 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned i = 0; i < 16; ++i) { + buffer.a[i * 4 + 0] = __builtin_bit_cast(uint32_t, -1.0f); + buffer.a[i * 4 + 1] = 0x3f800001; + buffer.a[i * 4 + 2] = __builtin_bit_cast(uint32_t, (float)i); + buffer.a[i * 4 + 3] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 0] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 1] = 0x3f7ffffe; + buffer.b[i * 4 + 2] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 3] = __builtin_bit_cast(uint32_t, (float)i); + } + for (unsigned i = 0; i < 65 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c}; + run(core, &args, sizeof(args)); + // Unfused: (1 + 2^-23)(1 - 2^-23) rounds to 1 and cancels the -1. + for (unsigned row = 0; row < 16; ++row) + for (unsigned column = 0; column < 16; ++column) + if (buffer.c[row * 16 + column] != + __builtin_bit_cast(uint32_t, (float)(row + column))) + __builtin_trap(); + for (unsigned i = 256; i < 260; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_fma32_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_fma32_main.cpp new file mode 100644 index 000000000..6efafcb21 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_fma32_main.cpp @@ -0,0 +1,91 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: 16 rows of 4 FP32. B: 16 columns of 4 FP32. C: 16x16 FP32, then a guard +// row. Row r of A is {-1, 1 + 2^-23, r, 1}, column j of B {1, 1 - 2^-23, 1, j}. +struct buffers { + uint32_t a[16 * 4]; + uint32_t b[16 * 4]; + uint32_t c[65 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned i = 0; i < 16; ++i) { + buffer.a[i * 4 + 0] = __builtin_bit_cast(uint32_t, -1.0f); + buffer.a[i * 4 + 1] = 0x3f800001; + buffer.a[i * 4 + 2] = __builtin_bit_cast(uint32_t, (float)i); + buffer.a[i * 4 + 3] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 0] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 1] = 0x3f7ffffe; + buffer.b[i * 4 + 2] = __builtin_bit_cast(uint32_t, 1.0f); + buffer.b[i * 4 + 3] = __builtin_bit_cast(uint32_t, (float)i); + } + for (unsigned i = 0; i < 65 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c}; + run(core, &args, sizeof(args)); + // Fused: -1 + (1 + 2^-23)(1 - 2^-23) leaves -2^-46, which survives only + // where r + j is 0. + for (unsigned row = 0; row < 16; ++row) + for (unsigned column = 0; column < 16; ++column) + if (buffer.c[row * 16 + column] != + (row == 0 && column == 0 + ? 0xa8800000u + : __builtin_bit_cast(uint32_t, (float)(row + column)))) + __builtin_trap(); + for (unsigned i = 256; i < 260; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_mxfp8_chain_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_mxfp8_chain_main.cpp new file mode 100644 index 000000000..576ac6cd9 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_mxfp8_chain_main.cpp @@ -0,0 +1,88 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: 16 rows of 32 E4M3 codes, then 16 scales. B: 16 columns of 32 codes, +// then 16 scales. C: 16x16 FP32 from row 1, the rest a guard pattern. +// Row r of A is 0.5 * (1 + (r % 8) / 8) (code 0x38 + r % 8, scale 2^-1); +// column j of B is 4 * (1 + (j % 8) / 8) (code 0x40 + j % 8, scale 2^1). +struct buffers { + uint8_t a[33 * 16]; + uint8_t b[33 * 16]; + uint32_t c[65 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned i = 0; i < 16 * 32; ++i) { + buffer.a[i] = 0x38 + (i / 32) % 8; + buffer.b[i] = 0x40 + (i / 32) % 8; + } + for (unsigned i = 16 * 32; i < 16 * 33; ++i) { + buffer.a[i] = 126; + buffer.b[i] = 128; + } + for (unsigned i = 0; i < 65 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c}; + run(core, &args, sizeof(args)); + for (unsigned i = 0; i < 4; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + // Each step adds (8 + r % 8) * (8 + j % 8); the chain has two. + for (unsigned row = 0; row < 16; ++row) + for (unsigned column = 0; column < 16; ++column) + if (__builtin_bit_cast(float, buffer.c[4 + row * 16 + column]) != + 2.0f * (8 + row % 8) * (8 + column % 8)) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_mxfp8_window_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_mxfp8_window_main.cpp new file mode 100644 index 000000000..55905949e --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxmm_mxfp8_window_main.cpp @@ -0,0 +1,87 @@ +#include "mlir.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// A: one row of 64 E4M3 codes, then its two scales. B: 16 columns of 32 +// codes, then 16 scales. C: one row of 16 FP32, then a guard row. +// Outside the window A holds NaN codes and a NaN scale, which the Ball must +// not read. Inside, element t is 1 + (t % 8) / 8; column j of B is +// 2 * (1 + (j % 8) / 8). +struct buffers { + uint8_t a[5 * 16]; + uint8_t b[33 * 16]; + uint32_t c[5 * 4]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + for (unsigned t = 0; t < 64; ++t) + buffer.a[t] = t < 32 ? 0x7f : 0x38 + t % 8; + buffer.a[64] = 255; + buffer.a[65] = 127; + for (unsigned i = 0; i < 16 * 32; ++i) + buffer.b[i] = 0x40 + (i / 32) % 8; + for (unsigned i = 16 * 32; i < 16 * 33; ++i) + buffer.b[i] = 127; + for (unsigned i = 0; i < 5 * 4; ++i) + buffer.c[i] = 0x5a5a5a5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxmm_args args = {(uintptr_t)buffer.a, (uintptr_t)buffer.b, + (uintptr_t)buffer.c}; + run(core, &args, sizeof(args)); + // The window sums to 46, so column j is 46 * 2 * (8 + j % 8) / 8. + for (unsigned column = 0; column < 16; ++column) + if (__builtin_bit_cast(float, buffer.c[column]) != 11.5f * (8 + column % 8)) + __builtin_trap(); + for (unsigned i = 16; i < 20; ++i) + if (buffer.c[i] != 0x5a5a5a5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxquant_two_blocks_main.cpp b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxquant_two_blocks_main.cpp new file mode 100644 index 000000000..113908a31 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/engine/mlir/bank_mxquant_two_blocks_main.cpp @@ -0,0 +1,92 @@ +#include "mlir.h" +#include "reference.h" +#include + +extern const unsigned char thread_image[], thread_image_end[]; + +// Input: two blocks of 32 FP32. Output: the 64 codes, the two scales from +// byte 64, then guard bytes up to the end of row 4. Block 0 holds signed +// zeros, subnormals, rounding ties and the largest finite; block 1 one large +// element that sets the scale and small ones that round. +struct buffers { + uint32_t input[16 * 4]; + uint8_t output[5 * 16]; +}; +struct all { + buffers core[POLY_THREAD_CORES]; +}; +static constexpr all prepare() { + all data{}; + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { + buffers &buffer = data.core[core]; + const uint32_t edges[] = {0, 0x80000000, 1, 0x80000001, + 0x007fffff, 0x00800000, 0x3f880000, 0x3f980000, + 0x43e00000, 0x7f7fffff, 0xff7fffff}; + for (unsigned i = 0; i < 32; ++i) { + buffer.input[i] = edges[i % 11]; + buffer.input[32 + i] = + i ? (i & 1 ? 0x3f880000u : 0xbf980000u) : 0x43e00000u; + } + for (unsigned i = 0; i < 5 * 16; ++i) + buffer.output[i] = 0x5a; + } + return data; +} +alignas(64) static all data = prepare(); +static unsigned done[POLY_THREAD_CORES]; + +static void run(unsigned core, const void *args, size_t size) { + const uint32_t context = thread_find(THREAD_SIGNATURE, 0); + size_t bytes = thread_image_end - thread_image; + if (bytes > thread_query(context, THREAD_FIELD_CODE_BYTES) || + thread_query(context, THREAD_FIELD_SIGNATURE) != THREAD_SIGNATURE) + __builtin_trap(); + thread_tss_write(0, thread_image, bytes); + thread_load(context, THREAD_CODE, 0, 0, (bytes + 7) & ~7u); + thread_tss_write(0, args, size); + thread_load(context, THREAD_TLS, 0, 0, (size + 7) & ~7u); + uint64_t base = thread_query(context, THREAD_FIELD_TLS_BASE), + capacity = thread_query(context, THREAD_FIELD_TLS_BYTES); + struct thread_task task = {core + 1, 0, bytes, base, base + capacity, + THREAD_SIGNATURE}; + thread_acquire(); + thread_start(context, &task); + int cancelled; + if (thread_wait(context, task.id, &cancelled) || cancelled) + __builtin_trap(); + thread_release(); +} + +static void exercise(unsigned core) { + buffers &buffer = data.core[core]; + struct mlir_mxquant_args args = {(uintptr_t)buffer.input, + (uintptr_t)buffer.output}; + run(core, &args, sizeof(args)); + uint8_t expected[2][33]; + mxquant_reference32(buffer.input, expected[0]); + mxquant_reference32(buffer.input + 32, expected[1]); + for (unsigned block = 0; block < 2; ++block) { + for (unsigned i = 0; i < 32; ++i) + if (buffer.output[block * 32 + i] != expected[block][i]) + __builtin_trap(); + if (buffer.output[64 + block] != expected[block][32]) + __builtin_trap(); + } + for (unsigned i = 66; i < 80; ++i) + if (buffer.output[i] != 0x5a) + __builtin_trap(); + __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); +} + +int main(void) { + uintptr_t hart; + __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) + while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { + } + } else { + exercise(hart - 1); + } + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/CMakeLists.txt b/examples/designs/poly/workloads/device/core/thread/CMakeLists.txt index 833069e63..9d5e4e197 100644 --- a/examples/designs/poly/workloads/device/core/thread/CMakeLists.txt +++ b/examples/designs/poly/workloads/device/core/thread/CMakeLists.txt @@ -1,3 +1,5 @@ add_subdirectory(ctests) set(POLY_ANT_IMAGE_DIR "${POLY_ANT_IMAGE_DIR}" PARENT_SCOPE) set(POLY_ANT_SOURCE_DIR "${POLY_ANT_SOURCE_DIR}" PARENT_SCOPE) +add_subdirectory(mlir) +set(POLY_MLIR_IMAGE_DIR "${POLY_MLIR_IMAGE_DIR}" PARENT_SCOPE) diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/CMakeLists.txt b/examples/designs/poly/workloads/device/core/thread/ctests/CMakeLists.txt index 4ce0d5d56..f83a7dc7b 100644 --- a/examples/designs/poly/workloads/device/core/thread/ctests/CMakeLists.txt +++ b/examples/designs/poly/workloads/device/core/thread/ctests/CMakeLists.txt @@ -32,24 +32,21 @@ foreach( set(POLY_ANT_THREAD ffn) endif() set(POLY_ANT_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${POLY_ANT_THREAD}-ctest-design-ant_${POLY_ANT_CASE}" + "${DESIGN_NAME}-${POLY_ANT_THREAD}-ctest-design-ant_${POLY_ANT_CASE}" ) - foreach(_name BANK_WIDTH BANK_LINES BB_PRIVATE_VBANK_MAX) - file( - STRINGS "${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD}/params.h" - _line - REGEX "^#define ${_name} " - ) - string(REGEX REPLACE ".* " "" _${_name} "${_line}") - endforeach() + set(_BANK_WIDTH ${DESIGN_PROFILES_${POLY_ANT_THREAD}_THREAD_MEM_BANK_WIDTH}) + set(_BANK_ENTRIES ${DESIGN_PROFILES_${POLY_ANT_THREAD}_THREAD_MEM_BANK_ENTRIES}) + set(_MAX_PRIVATE_VBANK ${DESIGN_PROFILES_${POLY_ANT_THREAD}_THREAD_FRONTEND_MAX_PRIVATE_VBANK}) + set(POLY_ANT_DEFINES ${DESIGN_TARGET_DEFS_${POLY_ANT_THREAD}}) + list(TRANSFORM POLY_ANT_DEFINES PREPEND "-D") if( NOT _BANK_WIDTH EQUAL 128 - OR _BANK_LINES LESS 128 - OR _BB_PRIVATE_VBANK_MAX LESS 5 + OR _BANK_ENTRIES LESS 128 + OR _MAX_PRIVATE_VBANK LESS 5 ) message( FATAL_ERROR - "Ant Mxmm needs 128-bit rows, >=128 rows/bank and private banks through 5; actual width=${_BANK_WIDTH}, rows=${_BANK_LINES}, privateMax=${_BB_PRIVATE_VBANK_MAX}" + "Ant Mxmm needs 128-bit rows, >=128 rows/bank and private banks through 5; actual width=${_BANK_WIDTH}, rows=${_BANK_ENTRIES}, privateMax=${_MAX_PRIVATE_VBANK}" ) endif() set(POLY_ANT_IMAGE ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_NAME}-ant) @@ -63,8 +60,8 @@ foreach( -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${ANT_RUNTIME_DIR}/code.ld -I${WORKLOAD_LIB_DIR} - -I${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD} - -I${BUCKYBALL_REPO_ROOT}/examples/components/balls/mxmm/workloads + ${POLY_ANT_DEFINES} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxmm/workloads -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services -I${BUCKYBALL_REPO_ROOT}/stack/engine/include @@ -79,9 +76,8 @@ foreach( ${POLY_ANT_HEADER} ${ANT_RUNTIME_DIR}/start.S ${ANT_RUNTIME_DIR}/code.ld - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD}/params.h - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD}/ballISA.h ${BUCKYBALL_BALL_ISA_DEPS} + VERBATIM ) add_custom_target( ${POLY_ANT_NAME}-ant-build diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.c index 730c1f791..e1327f6eb 100644 --- a/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank.c @@ -1,16 +1,16 @@ #include "ant_bank.h" #include -#include uint64_t entry(const struct ant_bank_args *args) { bb_mset(3, 1, 1, 1); - bb_mvin(args->input, 3, BANK_LINES, 1); - bb_mvout(args->private_out, 3, BANK_LINES, 1); + bb_mvin(args->input, 3, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvout(args->private_out, 3, THREAD_MEM_BANK_ENTRIES, 1); bb_mset(3, 0, 0, 0); - bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); - bb_mvin(args->input, BB_SHARED_BANK_BASE, args->shared_rows, 1); - bb_mvout(args->shared_out, BB_SHARED_BANK_BASE, args->shared_rows, 1); - bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 1, 1, 1); + bb_mvin(args->input, THREAD_FRONTEND_SHARED_VBANK_BASE, args->shared_rows, 1); + bb_mvout(args->shared_out, THREAD_FRONTEND_SHARED_VBANK_BASE, + args->shared_rows, 1); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 0, 0, 0); bb_mset(3, 1, 1, 1); bb_mvin_2d(args->input, 3, 2, 16, 2, 0, 2, 8); bb_mvout(args->tail_out, 3, 4, 1); diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.c index 8d2753ae4..8989e877d 100644 --- a/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_bank_rows.c @@ -1,16 +1,16 @@ #include "ant_bank_rows.h" #include -#include uint64_t entry(const struct ant_bank_rows_args *args) { bb_mset(3, 1, 1, 1); bb_mvin(args->input, 3, ANT_BANK_ROWS, 1); bb_mvout(args->private_out, 3, ANT_BANK_ROWS, 1); bb_mset(3, 0, 0, 0); - bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); - bb_mvin(args->input, BB_SHARED_BANK_BASE, args->shared_rows, 1); - bb_mvout(args->shared_out, BB_SHARED_BANK_BASE, args->shared_rows, 1); - bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 1, 1, 1); + bb_mvin(args->input, THREAD_FRONTEND_SHARED_VBANK_BASE, args->shared_rows, 1); + bb_mvout(args->shared_out, THREAD_FRONTEND_SHARED_VBANK_BASE, + args->shared_rows, 1); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 0, 0, 0); bb_mset(3, 1, 1, 1); bb_mvin_2d(args->input, 3, 2, 16, 2, 0, 2, 8); bb_mvout(args->tail_out, 3, 4, 1); diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.c index 2e9f9e9b5..3625c89c5 100644 --- a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink.c @@ -1,16 +1,17 @@ #include "ant_corelink.h" #include -#include uint64_t entry(const struct ant_corelink_args *args) { if (args->phase == 0) { - bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); - bb_mset(BB_SHARED_BANK_BASE + 1, 1, 1, 1); - bb_mvin(args->input, BB_SHARED_BANK_BASE, CORELINK_TEST_ROWS, 1); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 1, 1, 1); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE + 1, 1, 1, 1); + bb_mvin(args->input, THREAD_FRONTEND_SHARED_VBANK_BASE, CORELINK_TEST_ROWS, + 1); } else if (args->phase == 1) { - bb_mvout(args->output, BB_SHARED_BANK_BASE + 1, CORELINK_TEST_ROWS, 1); + bb_mvout(args->output, THREAD_FRONTEND_SHARED_VBANK_BASE + 1, + CORELINK_TEST_ROWS, 1); } else if (args->phase == 2) { - bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); - bb_mset(BB_SHARED_BANK_BASE + 1, 0, 0, 0); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 0, 0, 0); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE + 1, 0, 0, 0); } else __builtin_trap(); return 0; diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.c index 76c1a2982..16cdb7017 100644 --- a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_bank.c @@ -1,16 +1,17 @@ #include "ant_corelink_bank.h" #include -#include uint64_t entry(const struct ant_corelink_bank_args *args) { if (args->phase == 0) { - bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); - bb_mset(BB_SHARED_BANK_BASE + 1, 1, 1, 1); - bb_mvin(args->input, BB_SHARED_BANK_BASE, CORELINK_CROSS_ROWS, 1); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 1, 1, 1); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE + 1, 1, 1, 1); + bb_mvin(args->input, THREAD_FRONTEND_SHARED_VBANK_BASE, CORELINK_CROSS_ROWS, + 1); } else if (args->phase == 1) { - bb_mvout(args->output, BB_SHARED_BANK_BASE, CORELINK_CROSS_ROWS, 1); + bb_mvout(args->output, THREAD_FRONTEND_SHARED_VBANK_BASE, + CORELINK_CROSS_ROWS, 1); } else if (args->phase == 2) { - bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); - bb_mset(BB_SHARED_BANK_BASE + 1, 0, 0, 0); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 0, 0, 0); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE + 1, 0, 0, 0); } else __builtin_trap(); return 0; diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.c index 5a59fedec..3c46337d5 100644 --- a/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_corelink_reduce.c @@ -1,21 +1,20 @@ #include "ant_corelink_reduce.h" #include #include -#include uint64_t entry(const struct ant_corelink_reduce_args *args) { if (args->phase == 0) { - bb_mset(BB_SHARED_BANK_BASE, 1, 1, 1); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 1, 1, 1); bb_mset(0, 1, 1, 1); bb_mset(1, 1, 1, 1); bb_mvin(args->input, 0, 4, 1); bb_mvin(args->input + 16 * sizeof(float), 1, 64, 1); - bb_mxmm_fma32(0, 1, BB_SHARED_BANK_BASE, 1, 16, 16, 1, 1, 0); + bb_mxmm_fma32(0, 1, THREAD_FRONTEND_SHARED_VBANK_BASE, 1, 16, 16, 1, 1, 0); bb_mset(0, 0, 0, 0); bb_mset(1, 0, 0, 0); } else if (args->phase == 1) { - bb_mvout(args->output, BB_SHARED_BANK_BASE, 2, 1); + bb_mvout(args->output, THREAD_FRONTEND_SHARED_VBANK_BASE, 2, 1); } else if (args->phase == 2) { - bb_mset(BB_SHARED_BANK_BASE, 0, 0, 0); + bb_mset(THREAD_FRONTEND_SHARED_VBANK_BASE, 0, 0, 0); } else __builtin_trap(); return 0; diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/ant_mxmm.c b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mxmm.c index 88aebd321..c3769ae77 100644 --- a/examples/designs/poly/workloads/device/core/thread/ctests/ant_mxmm.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/ant_mxmm.c @@ -1,15 +1,14 @@ #include "ant_mxmm.h" #include #include -#include uint64_t entry(const struct ant_mxmm_args *args) { for (unsigned bank = 3; bank <= 5; ++bank) bb_mset(bank, 1, 1, 1); - bb_mvin(args->a, 3, ANT_M * ANT_K * 4 / (BANK_WIDTH / 8), 1); - bb_mvin(args->b, 4, ANT_N * ANT_K * 4 / (BANK_WIDTH / 8), 1); + bb_mvin(args->a, 3, ANT_M * ANT_K * 4 / (THREAD_MEM_BANK_WIDTH / 8), 1); + bb_mvin(args->b, 4, ANT_N * ANT_K * 4 / (THREAD_MEM_BANK_WIDTH / 8), 1); bb_mxmm_f32(3, 4, 5, ANT_M, ANT_N, ANT_K, 1, 1, 0); - bb_mvout(args->output, 5, ANT_M * ANT_N * 4 / (BANK_WIDTH / 8), 1); + bb_mvout(args->output, 5, ANT_M * ANT_N * 4 / (THREAD_MEM_BANK_WIDTH / 8), 1); for (unsigned bank = 3; bank <= 5; ++bank) bb_mset(bank, 0, 0, 0); return 0; diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/mesh_private_bank_move_test.c b/examples/designs/poly/workloads/device/core/thread/ctests/mesh_private_bank_move_test.c index ab28881da..d112fcbb8 100644 --- a/examples/designs/poly/workloads/device/core/thread/ctests/mesh_private_bank_move_test.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/mesh_private_bank_move_test.c @@ -1,6 +1,5 @@ #include "ant_mesh.h" #include -#include uint64_t entry(const struct ant_mesh_args *args) { if (args->index == 0) { diff --git a/examples/designs/poly/workloads/device/core/thread/ctests/mesh_shared_mem_test.c b/examples/designs/poly/workloads/device/core/thread/ctests/mesh_shared_mem_test.c index aa8b7bf69..c60289103 100644 --- a/examples/designs/poly/workloads/device/core/thread/ctests/mesh_shared_mem_test.c +++ b/examples/designs/poly/workloads/device/core/thread/ctests/mesh_shared_mem_test.c @@ -1,22 +1,24 @@ #include "ant_mesh.h" #include -#include enum { - BANKS = VIRTUAL_BANK_NUM - BB_SHARED_BANK_BASE, - ROW_BYTES = BANK_WIDTH / 8 + BANKS = CORE_VIRTUAL_BANK_COUNT - THREAD_FRONTEND_SHARED_VBANK_BASE, + ROW_BYTES = THREAD_MEM_BANK_WIDTH / 8 }; _Static_assert( - BANK_WIDTH == 128 && BANKS > 0 && BANKS <= SHARED_BANK_NUM, + THREAD_MEM_BANK_WIDTH == 128 && BANKS > 0 && CORE_SHARED_MEM_ENABLE && + BANKS * CORE_SHARED_MEM_BANK_ENTRIES <= CORE_SHARED_MEM_ENTRIES, "Shared bank test requires one physical bank per shared virtual bank"); uint64_t entry(const struct ant_mesh_args *args) { for (unsigned bank = 0; bank < BANKS; ++bank) { - bb_mem_alloc(BB_SHARED_BANK_BASE + bank, 1, 1); - bb_mvin(args->input + bank * ROW_BYTES, BB_SHARED_BANK_BASE + bank, 1, 1); + bb_mem_alloc(THREAD_FRONTEND_SHARED_VBANK_BASE + bank, 1, 1); + bb_mvin(args->input + bank * ROW_BYTES, + THREAD_FRONTEND_SHARED_VBANK_BASE + bank, 1, 1); } for (unsigned bank = 0; bank < BANKS; ++bank) - bb_mvout(args->output + bank * ROW_BYTES, BB_SHARED_BANK_BASE + bank, 1, 1); + bb_mvout(args->output + bank * ROW_BYTES, + THREAD_FRONTEND_SHARED_VBANK_BASE + bank, 1, 1); for (unsigned bank = 0; bank < BANKS; ++bank) - bb_mem_release(BB_SHARED_BANK_BASE + bank); + bb_mem_release(THREAD_FRONTEND_SHARED_VBANK_BASE + bank); return 0; } diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/CMakeLists.txt b/examples/designs/poly/workloads/device/core/thread/mlir/CMakeLists.txt new file mode 100644 index 000000000..009998887 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/CMakeLists.txt @@ -0,0 +1,353 @@ +# The MLIR tests of MxmmBall and MxquantBall, the Balls of poly's threads, as +# thread programs: each test MLIR is compiled for the attention thread and +# linked with its entry into a thread image (start.S, code.ld of the engine). +# engine/mlir embeds every image into a host program that runs it on the +# thread of each core and checks the result. +set(POLY_MLIR_THREAD_DEFINES ${DESIGN_TARGET_DEFS_attention}) +list(TRANSFORM POLY_MLIR_THREAD_DEFINES PREPEND "-D") +string(JOIN " " POLY_MLIR_OPT ${DESIGN_TARGET_OPT_attention}) +set(POLY_MLIR_BIN "${DESIGN_COMPILER}/bin") +set(POLY_MLIR_ENGINE "${BUCKYBALL_REPO_ROOT}/stack/engine/program") +set(MXMM_TESTS "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxmm/workloads/mlir_tests") +set(MXQUANT_TESTS "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxquant/workloads/mlir_tests") + +# ball_mxmm_mxfp8_chain +add_custom_command( + OUTPUT ball_mxmm_mxfp8_chain.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/ball/mxmm_mxfp8_chain.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o ball_mxmm_mxfp8_chain.o" + DEPENDS ${MXMM_TESTS}/ball/mxmm_mxfp8_chain.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.elf ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_mxfp8_chain_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_chain.o -o ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.elf + ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.bin + DEPENDS + ball_mxmm_mxfp8_chain_entry.c + mlir.h + ball_mxmm_mxfp8_chain.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_chain-thread-build) + +# ball_mxmm_mxfp8_window +add_custom_command( + OUTPUT ball_mxmm_mxfp8_window.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/ball/mxmm_mxfp8_window.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o ball_mxmm_mxfp8_window.o" + DEPENDS ${MXMM_TESTS}/ball/mxmm_mxfp8_window.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.elf ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_mxfp8_window_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_mxfp8_window.o -o ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.elf + ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.bin + DEPENDS + ball_mxmm_mxfp8_window_entry.c + mlir.h + ball_mxmm_mxfp8_window.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_mxfp8_window-thread-build) + +# ball_mxmm_fma32 +add_custom_command( + OUTPUT ball_mxmm_fma32.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/ball/mxmm_fma32.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o ball_mxmm_fma32.o" + DEPENDS ${MXMM_TESTS}/ball/mxmm_fma32.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.elf ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_fma32_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_fma32.o -o ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.elf + ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.bin + DEPENDS + ball_mxmm_fma32_entry.c + mlir.h + ball_mxmm_fma32.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_fma32-thread-build) + +# ball_mxmm_f32 +add_custom_command( + OUTPUT ball_mxmm_f32.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/ball/mxmm_f32.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o ball_mxmm_f32.o" + DEPENDS ${MXMM_TESTS}/ball/mxmm_f32.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.elf ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxmm_f32_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxmm_f32.o -o ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.elf + ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.bin + DEPENDS + ball_mxmm_f32_entry.c + mlir.h + ball_mxmm_f32.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-ball_mxmm_f32-thread-build) + +# ball_mxquant_two_blocks +add_custom_command( + OUTPUT ball_mxquant_two_blocks.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXQUANT_TESTS}/ball/mxquant_two_blocks.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o ball_mxquant_two_blocks.o" + DEPENDS ${MXQUANT_TESTS}/ball/mxquant_two_blocks.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.elf ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/ball_mxquant_two_blocks_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/ball_mxquant_two_blocks.o -o ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.elf + ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.bin + DEPENDS + ball_mxquant_two_blocks_entry.c + mlir.h + ball_mxquant_two_blocks.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-ball_mxquant_two_blocks-thread-build) + +# bank_mxmm_mxfp8_chain +add_custom_command( + OUTPUT bank_mxmm_mxfp8_chain.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/bank/mxmm_mxfp8_chain.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o bank_mxmm_mxfp8_chain.o" + DEPENDS ${MXMM_TESTS}/bank/mxmm_mxfp8_chain.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.elf ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_mxfp8_chain_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_chain.o -o ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.elf + ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.bin + DEPENDS + bank_mxmm_mxfp8_chain_entry.c + mlir.h + bank_mxmm_mxfp8_chain.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_chain-thread-build) + +# bank_mxmm_mxfp8_window +add_custom_command( + OUTPUT bank_mxmm_mxfp8_window.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/bank/mxmm_mxfp8_window.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o bank_mxmm_mxfp8_window.o" + DEPENDS ${MXMM_TESTS}/bank/mxmm_mxfp8_window.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.elf ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_mxfp8_window_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_mxfp8_window.o -o ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.elf + ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.bin + DEPENDS + bank_mxmm_mxfp8_window_entry.c + mlir.h + bank_mxmm_mxfp8_window.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_mxfp8_window-thread-build) + +# bank_mxmm_fma32 +add_custom_command( + OUTPUT bank_mxmm_fma32.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/bank/mxmm_fma32.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o bank_mxmm_fma32.o" + DEPENDS ${MXMM_TESTS}/bank/mxmm_fma32.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.elf ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_fma32_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_fma32.o -o ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.elf + ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.bin + DEPENDS + bank_mxmm_fma32_entry.c + mlir.h + bank_mxmm_fma32.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_fma32-thread-build) + +# bank_mxmm_f32 +add_custom_command( + OUTPUT bank_mxmm_f32.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXMM_TESTS}/bank/mxmm_f32.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o bank_mxmm_f32.o" + DEPENDS ${MXMM_TESTS}/bank/mxmm_f32.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.elf ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxmm_f32_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxmm_f32.o -o ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.elf + ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.bin + DEPENDS + bank_mxmm_f32_entry.c + mlir.h + bank_mxmm_f32.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-bank_mxmm_f32-thread-build) + +# bank_mxquant_two_blocks +add_custom_command( + OUTPUT bank_mxquant_two_blocks.o + COMMAND + bash -o pipefail -c + "${POLY_MLIR_OPT} ${MXQUANT_TESTS}/bank/mxquant_two_blocks.mlir -assign-physical-banks -lower-buckyball -convert-linalg-to-loops -expand-strided-metadata -lower-affine -convert-scf-to-cf -convert-cf-to-llvm -convert-arith-to-llvm -convert-math-to-llvm -finalize-memref-to-llvm -convert-func-to-llvm -reconcile-unrealized-casts | ${POLY_MLIR_BIN}/buddy-translate --buddy-to-llvmir | ${POLY_MLIR_BIN}/buddy-llc -filetype=obj -mtriple=riscv64 -O2 -code-model=medium -mattr=+xbuckyball,+m -target-abi=lp64 --riscv-disable-using-constant-pool-for-large-ints -o bank_mxquant_two_blocks.o" + DEPENDS ${MXQUANT_TESTS}/bank/mxquant_two_blocks.mlir ${DESIGN_TARGET_PLUGIN_attention} + VERBATIM +) +add_custom_command( + OUTPUT ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.elf ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.bin + COMMAND + ${ELF_CC} -O2 -g -march=rv64im -mabi=lp64 -mcmodel=medany + -ffreestanding -fno-builtin -fno-pic -fno-stack-protector + -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax + -nostdlib -nostartfiles -Wl,--no-relax + -Wl,-T,${POLY_MLIR_ENGINE}/code.ld -I${WORKLOAD_LIB_DIR} + ${POLY_MLIR_THREAD_DEFINES} ${POLY_MLIR_ENGINE}/start.S + ${CMAKE_CURRENT_SOURCE_DIR}/bank_mxquant_two_blocks_entry.c + ${CMAKE_CURRENT_BINARY_DIR}/bank_mxquant_two_blocks.o -o ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.elf + COMMAND + ${ELF_OBJCOPY} -O binary --only-section=.text ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.elf + ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.bin + DEPENDS + bank_mxquant_two_blocks_entry.c + mlir.h + bank_mxquant_two_blocks.o + ${POLY_MLIR_ENGINE}/start.S + ${POLY_MLIR_ENGINE}/code.ld + VERBATIM +) +add_custom_target(${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks-thread-build DEPENDS ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks.bin) +add_dependencies(poly-ctests-build ${DESIGN_NAME}-attention-mlirtest-bank_mxquant_two_blocks-thread-build) +set(POLY_MLIR_IMAGE_DIR "${CMAKE_CURRENT_BINARY_DIR}" PARENT_SCOPE) diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_f32_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_f32_entry.c new file mode 100644 index 000000000..7bce3e36b --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_f32_entry.c @@ -0,0 +1,20 @@ +#include "mlir.h" +#include + +void _mlir_ciface_single(void); + +// A in bank 1, B in bank 2, C in bank 3. +uint64_t entry(const struct mlir_mxmm_args *args) { + bb_mset(1, 1, 1, 1); + bb_mset(2, 1, 1, 1); + bb_mset(3, 1, 1, 1); + bb_mvin(args->a, 1, 16, 1); + bb_mvin(args->b, 2, 16, 1); + bb_mvin(args->c, 3, 65, 1); + _mlir_ciface_single(); + bb_mvout(args->c, 3, 65, 1); + bb_mset(1, 0, 0, 0); + bb_mset(2, 0, 0, 0); + bb_mset(3, 0, 0, 0); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_fma32_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_fma32_entry.c new file mode 100644 index 000000000..7bce3e36b --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_fma32_entry.c @@ -0,0 +1,20 @@ +#include "mlir.h" +#include + +void _mlir_ciface_single(void); + +// A in bank 1, B in bank 2, C in bank 3. +uint64_t entry(const struct mlir_mxmm_args *args) { + bb_mset(1, 1, 1, 1); + bb_mset(2, 1, 1, 1); + bb_mset(3, 1, 1, 1); + bb_mvin(args->a, 1, 16, 1); + bb_mvin(args->b, 2, 16, 1); + bb_mvin(args->c, 3, 65, 1); + _mlir_ciface_single(); + bb_mvout(args->c, 3, 65, 1); + bb_mset(1, 0, 0, 0); + bb_mset(2, 0, 0, 0); + bb_mset(3, 0, 0, 0); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_mxfp8_chain_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_mxfp8_chain_entry.c new file mode 100644 index 000000000..953be49ca --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_mxfp8_chain_entry.c @@ -0,0 +1,24 @@ +#include "mlir.h" +#include + +void _mlir_ciface_first(void); +void _mlir_ciface_last(void); + +// A in bank 1, B in bank 2, C in bank 3; C after the first step goes to +// `between`, after the last to `c`. +uint64_t entry(const struct mlir_mxmm_chain_args *args) { + bb_mset(1, 1, 1, 1); + bb_mset(2, 1, 1, 1); + bb_mset(3, 1, 1, 1); + bb_mvin(args->a, 1, 33, 1); + bb_mvin(args->b, 2, 33, 1); + bb_mvin(args->c, 3, 65, 1); + _mlir_ciface_first(); + bb_mvout(args->between, 3, 65, 1); + _mlir_ciface_last(); + bb_mvout(args->c, 3, 65, 1); + bb_mset(1, 0, 0, 0); + bb_mset(2, 0, 0, 0); + bb_mset(3, 0, 0, 0); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_mxfp8_window_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_mxfp8_window_entry.c new file mode 100644 index 000000000..bc49e1e32 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxmm_mxfp8_window_entry.c @@ -0,0 +1,20 @@ +#include "mlir.h" +#include + +void _mlir_ciface_window(void); + +// A in bank 1, B in bank 2, C in bank 3. +uint64_t entry(const struct mlir_mxmm_args *args) { + bb_mset(1, 1, 1, 1); + bb_mset(2, 1, 1, 1); + bb_mset(3, 1, 1, 1); + bb_mvin(args->a, 1, 5, 1); + bb_mvin(args->b, 2, 33, 1); + bb_mvin(args->c, 3, 5, 1); + _mlir_ciface_window(); + bb_mvout(args->c, 3, 5, 1); + bb_mset(1, 0, 0, 0); + bb_mset(2, 0, 0, 0); + bb_mset(3, 0, 0, 0); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxquant_two_blocks_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxquant_two_blocks_entry.c new file mode 100644 index 000000000..40494c50c --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/ball_mxquant_two_blocks_entry.c @@ -0,0 +1,17 @@ +#include "mlir.h" +#include + +void _mlir_ciface_quant(void); + +// The input in bank 1, the output in bank 2. +uint64_t entry(const struct mlir_mxquant_args *args) { + bb_mset(1, 1, 1, 1); + bb_mset(2, 1, 1, 1); + bb_mvin(args->input, 1, 16, 1); + bb_mvin(args->output, 2, 5, 1); + _mlir_ciface_quant(); + bb_mvout(args->output, 2, 5, 1); + bb_mset(1, 0, 0, 0); + bb_mset(2, 0, 0, 0); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_f32_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_f32_entry.c new file mode 100644 index 000000000..dfedf4b90 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_f32_entry.c @@ -0,0 +1,25 @@ +#include "mlir.h" + +void _mlir_ciface_single(struct mlir_memref2 *, struct mlir_memref2 *, + struct mlir_memref2 *); + +static void describe(struct mlir_memref2 *memref, uint64_t address, + int64_t rows, int64_t columns) { + memref->allocated = address; + memref->aligned = address; + memref->offset = 0; + memref->sizes[0] = rows; + memref->sizes[1] = columns; + memref->strides[0] = columns; + memref->strides[1] = 1; +} + +// The MLIR function moves the buffers in and out of its banks itself. +uint64_t entry(const struct mlir_mxmm_args *args) { + struct mlir_memref2 a, b, c; + describe(&a, args->a, 16, 4); + describe(&b, args->b, 16, 4); + describe(&c, args->c, 65, 4); + _mlir_ciface_single(&a, &b, &c); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_fma32_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_fma32_entry.c new file mode 100644 index 000000000..dfedf4b90 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_fma32_entry.c @@ -0,0 +1,25 @@ +#include "mlir.h" + +void _mlir_ciface_single(struct mlir_memref2 *, struct mlir_memref2 *, + struct mlir_memref2 *); + +static void describe(struct mlir_memref2 *memref, uint64_t address, + int64_t rows, int64_t columns) { + memref->allocated = address; + memref->aligned = address; + memref->offset = 0; + memref->sizes[0] = rows; + memref->sizes[1] = columns; + memref->strides[0] = columns; + memref->strides[1] = 1; +} + +// The MLIR function moves the buffers in and out of its banks itself. +uint64_t entry(const struct mlir_mxmm_args *args) { + struct mlir_memref2 a, b, c; + describe(&a, args->a, 16, 4); + describe(&b, args->b, 16, 4); + describe(&c, args->c, 65, 4); + _mlir_ciface_single(&a, &b, &c); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_mxfp8_chain_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_mxfp8_chain_entry.c new file mode 100644 index 000000000..34af77e35 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_mxfp8_chain_entry.c @@ -0,0 +1,25 @@ +#include "mlir.h" + +void _mlir_ciface_chain(struct mlir_memref2 *, struct mlir_memref2 *, + struct mlir_memref2 *); + +static void describe(struct mlir_memref2 *memref, uint64_t address, + int64_t rows, int64_t columns) { + memref->allocated = address; + memref->aligned = address; + memref->offset = 0; + memref->sizes[0] = rows; + memref->sizes[1] = columns; + memref->strides[0] = columns; + memref->strides[1] = 1; +} + +// The MLIR function moves the buffers in and out of its banks itself. +uint64_t entry(const struct mlir_mxmm_args *args) { + struct mlir_memref2 a, b, c; + describe(&a, args->a, 33, 16); + describe(&b, args->b, 33, 16); + describe(&c, args->c, 65, 16); + _mlir_ciface_chain(&a, &b, &c); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_mxfp8_window_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_mxfp8_window_entry.c new file mode 100644 index 000000000..5a208bcbf --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxmm_mxfp8_window_entry.c @@ -0,0 +1,25 @@ +#include "mlir.h" + +void _mlir_ciface_window(struct mlir_memref2 *, struct mlir_memref2 *, + struct mlir_memref2 *); + +static void describe(struct mlir_memref2 *memref, uint64_t address, + int64_t rows, int64_t columns) { + memref->allocated = address; + memref->aligned = address; + memref->offset = 0; + memref->sizes[0] = rows; + memref->sizes[1] = columns; + memref->strides[0] = columns; + memref->strides[1] = 1; +} + +// The MLIR function moves the buffers in and out of its banks itself. +uint64_t entry(const struct mlir_mxmm_args *args) { + struct mlir_memref2 a, b, c; + describe(&a, args->a, 5, 16); + describe(&b, args->b, 33, 16); + describe(&c, args->c, 5, 16); + _mlir_ciface_window(&a, &b, &c); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxquant_two_blocks_entry.c b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxquant_two_blocks_entry.c new file mode 100644 index 000000000..32de94355 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/bank_mxquant_two_blocks_entry.c @@ -0,0 +1,23 @@ +#include "mlir.h" + +void _mlir_ciface_quant(struct mlir_memref2 *, struct mlir_memref2 *); + +static void describe(struct mlir_memref2 *memref, uint64_t address, + int64_t rows, int64_t columns) { + memref->allocated = address; + memref->aligned = address; + memref->offset = 0; + memref->sizes[0] = rows; + memref->sizes[1] = columns; + memref->strides[0] = columns; + memref->strides[1] = 1; +} + +// The MLIR function moves the buffers in and out of its banks itself. +uint64_t entry(const struct mlir_mxquant_args *args) { + struct mlir_memref2 input, output; + describe(&input, args->input, 16, 4); + describe(&output, args->output, 5, 16); + _mlir_ciface_quant(&input, &output); + return 0; +} diff --git a/examples/designs/poly/workloads/device/core/thread/mlir/mlir.h b/examples/designs/poly/workloads/device/core/thread/mlir/mlir.h new file mode 100644 index 000000000..1182f2b28 --- /dev/null +++ b/examples/designs/poly/workloads/device/core/thread/mlir/mlir.h @@ -0,0 +1,20 @@ +#pragma once +#include + +// Arguments of the thread programs that run the MxmmBall and MxquantBall MLIR +// tests: the DRAM addresses of their buffers. +struct mlir_mxmm_args { + uint64_t a, b, c; +}; +struct mlir_mxmm_chain_args { + uint64_t a, b, c, between; +}; +struct mlir_mxquant_args { + uint64_t input, output; +}; + +// A two-dimensional memref descriptor as _mlir_ciface functions take it. +struct mlir_memref2 { + uint64_t allocated, aligned; + int64_t offset, sizes[2], strides[2]; +}; diff --git a/examples/designs/poly/workloads/device/runtime/CMakeLists.txt b/examples/designs/poly/workloads/device/runtime/CMakeLists.txt index 6a043b962..8eb914376 100644 --- a/examples/designs/poly/workloads/device/runtime/CMakeLists.txt +++ b/examples/designs/poly/workloads/device/runtime/CMakeLists.txt @@ -1,13 +1,6 @@ -foreach( - POLY_ANT_CASE - mesh_move - corelink - corelink_bank - corelink_reduce - coherence - mesh_private - mesh_shared -) +# POLY_ANT_CASES comes from the simulation entry: CoreLink and coherence cases +# need a power-of-two count (>= 2) of thread-bearing cores. +foreach(POLY_ANT_CASE IN LISTS POLY_ANT_CASES) set(POLY_ANT_THREAD attention) set(POLY_ANT_HEADER ant_${POLY_ANT_CASE}.h) set(POLY_ANT_HOST_DEFINITIONS) @@ -23,7 +16,7 @@ foreach( endif() set(POLY_ANT_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${POLY_ANT_THREAD}-ctest-design-ant_${POLY_ANT_CASE}" + "${DESIGN_NAME}-${POLY_ANT_THREAD}-ctest-design-ant_${POLY_ANT_CASE}" ) set(POLY_ANT_IMAGE ${POLY_ANT_IMAGE_DIR}/${POLY_ANT_NAME}-ant) set(POLY_ANT_CONTROL_SOURCES) @@ -55,7 +48,7 @@ foreach( buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/${POLY_ANT_MAIN}") set(POLY_ANT_HOST_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-main-ctest-design-ant_${POLY_ANT_CASE}" + "${DESIGN_NAME}-main-ctest-design-ant_${POLY_ANT_CASE}" ) set(BUCKYBALL_CTEST_TARGET ${POLY_ANT_THREAD}) @@ -66,12 +59,14 @@ foreach( set(POLY_ANT_CONTROL_OBJECTS ${CMAKE_CURRENT_BINARY_DIR}/ant_corelink_reduce_control.o ) + set(POLY_ANT_DEFINES ${DESIGN_TARGET_DEFS_${POLY_ANT_THREAD}}) + list(TRANSFORM POLY_ANT_DEFINES PREPEND "-D") add_custom_command( OUTPUT ${POLY_ANT_CONTROL_OBJECTS} COMMAND ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 -fno-builtin -I${POLY_ANT_SOURCE_DIR} - -I${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD} + ${POLY_ANT_DEFINES} -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -c @@ -79,12 +74,14 @@ foreach( DEPENDS ${POLY_ANT_CONTROL_SOURCES} ${POLY_ANT_SOURCE_DIR}/ant_corelink_reduce.h - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD}/params.h + VERBATIM ) set(POLY_ANT_CONTROL_SOURCES) endif() set(POLY_ANT_HOST_ELF ${POLY_ANT_HOST_NAME}-multicore-baremetal) + set(POLY_ANT_HOST_DEFINES ${DESIGN_TARGET_DEFS_${POLY_ANT_HOST_THREAD}}) + list(TRANSFORM POLY_ANT_HOST_DEFINES PREPEND "-D") add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${POLY_ANT_HOST_ELF} COMMAND @@ -92,9 +89,9 @@ foreach( -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${BBSIM_LD} ${POLY_ANT_HOST_DEFINITIONS} -I${POLY_ANT_SOURCE_DIR} - -I${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_THREAD} - -I${BUCKYBALL_ISA_DIR}/design -I${WORKLOAD_LIB_DIR} - -I${BUCKYBALL_REPO_ROOT}/examples/components/balls/f32add/workloads + ${POLY_ANT_HOST_DEFINES} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${WORKLOAD_LIB_DIR} + -I${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/f32add/workloads -I${BUCKYBALL_REPO_ROOT}/stack/runtime/scheduler -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services -I${BUCKYBALL_REPO_ROOT}/stack/engine/include -I${BAREMETAL_DIR} @@ -112,16 +109,14 @@ foreach( ${BBSIM_LD} ${POLY_ANT_CONTROL_SOURCES} ${POLY_ANT_CONTROL_OBJECTS} - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_THREAD}/params.h - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_HOST_THREAD}/ballISA.h ${POLY_ANT_IMAGE}.bin ${BUCKYBALL_REPO_ROOT}/stack/engine/src/thread.c ${BUCKYBALL_REPO_ROOT}/stack/engine/include/thread.h ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.c ${BUCKYBALL_REPO_ROOT}/stack/runtime/services/corelink.h - ${BUCKYBALL_ISA_DIR}/${POLY_ANT_THREAD}/params.h - ${BUCKYBALL_ISA_DIR}/design/instances.h + ${RUNTIME_INCLUDE_DIR}/bb_cores.h WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM ) add_custom_target( ${POLY_ANT_HOST_NAME}-ctest-build @@ -130,16 +125,20 @@ foreach( add_dependencies(poly-ctests-build ${POLY_ANT_HOST_NAME}-ctest-build) endforeach() -set(POLY_STREAM_NAME "${BUCKYBALL_WORKLOAD_DESIGN}-main-ctest-design-stream") +set(POLY_STREAM_NAME "${DESIGN_NAME}-main-ctest-design-stream") buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/stream_main.c") add_executable(${POLY_STREAM_NAME}-linux stream_main.c) target_include_directories( ${POLY_STREAM_NAME}-linux PRIVATE - ${BUCKYBALL_ISA_DIR}/design - ${BUCKYBALL_ISA_DIR}/attention + ${RUNTIME_INCLUDE_DIR} ${WORKLOAD_LIB_DIR} - ${BUCKYBALL_REPO_ROOT}/examples/components/balls/mxmm/workloads + ${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/mxmm/workloads +) +# The Ball instructions of the job go into the attention thread's stream. +target_compile_definitions( + ${POLY_STREAM_NAME}-linux + PRIVATE ${DESIGN_DEFS} ${DESIGN_TARGET_DEFS_attention} BUCKYBALL_STREAM ) target_link_libraries(${POLY_STREAM_NAME}-linux PRIVATE engine runtime-linux) add_dependencies(poly-ctests-build ${POLY_STREAM_NAME}-linux) diff --git a/examples/designs/poly/workloads/device/runtime/ant_coherence_main.c b/examples/designs/poly/workloads/device/runtime/ant_coherence_main.c index dc1d30ab3..5189cff77 100644 --- a/examples/designs/poly/workloads/device/runtime/ant_coherence_main.c +++ b/examples/designs/poly/workloads/device/runtime/ant_coherence_main.c @@ -1,20 +1,19 @@ #include "ant_coherence.h" -#include -#include +#include #include _Static_assert( - BB_CORE_NUM == BB_THREAD_CORE_COUNT + 1 && BB_THREAD_CORE_COUNT >= 2 && - !(BB_THREAD_CORE_COUNT & (BB_THREAD_CORE_COUNT - 1)), + BB_NUM_CORES == POLY_THREAD_CORES + 1 && POLY_THREAD_CORES >= 2 && + !(POLY_THREAD_CORES & (POLY_THREAD_CORES - 1)), "coherence gate requires a power-of-two thread-bearing core count"); extern const unsigned char ant_image[], ant_image_end[]; // Separate 64-byte lines/sets for data and synchronization; avoid main's // top-of-page stack sets. enum { INPUT = 8, - OUTPUT = INPUT + BB_THREAD_CORE_COUNT, - READY = OUTPUT + BB_THREAD_CORE_COUNT, - DONE = READY + BB_THREAD_CORE_COUNT, - ROUND = DONE + BB_THREAD_CORE_COUNT + OUTPUT = INPUT + POLY_THREAD_CORES, + READY = OUTPUT + POLY_THREAD_CORES, + DONE = READY + POLY_THREAD_CORES, + ROUND = DONE + POLY_THREAD_CORES }; static volatile uint64_t lines[ROUND + 1][8] __attribute__((section(".data"), aligned(4096))) = {0}; @@ -67,26 +66,26 @@ int main(void) { if (hart) controller(hart - 1); else { - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) while (!__atomic_load_n(&lines[READY + core][0], __ATOMIC_ACQUIRE)) { } for (unsigned round = 1; round <= 2; ++round) { - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) for (unsigned word = 0; word < 2; ++word) { lines[INPUT + core][word] = pattern(round, core, word); lines[OUTPUT + core][word] = ~pattern(round, core, word); } __atomic_store_n(&lines[ROUND][0], round, __ATOMIC_RELEASE); - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) while (__atomic_load_n(&lines[DONE + core][0], __ATOMIC_ACQUIRE) < round) { } - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) for (unsigned word = 0; word < 2; ++word) if (lines[OUTPUT + core][word] != pattern(round, core, word)) __builtin_trap(); } - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) while (__atomic_load_n(&lines[DONE + core][0], __ATOMIC_ACQUIRE) != 3) { } } diff --git a/examples/designs/poly/workloads/device/runtime/ant_corelink_bank_main.c b/examples/designs/poly/workloads/device/runtime/ant_corelink_bank_main.c index aad49586d..6e3f6d258 100644 --- a/examples/designs/poly/workloads/device/runtime/ant_corelink_bank_main.c +++ b/examples/designs/poly/workloads/device/runtime/ant_corelink_bank_main.c @@ -1,23 +1,22 @@ #include "ant_corelink_bank.h" +#include #include -#include -#include #include _Static_assert( - BB_CORE_NUM == BB_THREAD_CORE_COUNT + 1 && BB_THREAD_CORE_COUNT >= 2 && - !(BB_THREAD_CORE_COUNT & (BB_THREAD_CORE_COUNT - 1)), + BB_NUM_CORES == POLY_THREAD_CORES + 1 && POLY_THREAD_CORES >= 2 && + !(POLY_THREAD_CORES & (POLY_THREAD_CORES - 1)), "CoreLink gate requires a power-of-two thread-bearing core count"); extern const unsigned char ant_image[], ant_image_end[]; -static uint64_t input[BB_THREAD_CORE_COUNT][CORELINK_CROSS_BYTES / 8] +static uint64_t input[POLY_THREAD_CORES][CORELINK_CROSS_BYTES / 8] __attribute__((aligned(64))); -static uint64_t output[BB_THREAD_CORE_COUNT][CORELINK_CROSS_BYTES / 8] +static uint64_t output[POLY_THREAD_CORES][CORELINK_CROSS_BYTES / 8] __attribute__((aligned(64))); -static unsigned ready[BB_THREAD_CORE_COUNT], done[BB_THREAD_CORE_COUNT]; +static unsigned ready[POLY_THREAD_CORES], done[POLY_THREAD_CORES]; static unsigned arrived[2]; static void barrier(unsigned phase) { __atomic_fetch_add(&arrived[phase], 1, __ATOMIC_ACQ_REL); while (__atomic_load_n(&arrived[phase], __ATOMIC_ACQUIRE) != - BB_THREAD_CORE_COUNT) { + POLY_THREAD_CORES) { } } static unsigned pattern(unsigned core, unsigned i) { @@ -48,7 +47,7 @@ static void phase(unsigned core, unsigned operation) { static void exercise(unsigned core) { uint64_t bank_bytes = corelink_query(CORELINK_BANK_BYTES); if (corelink_query(CORELINK_CORE_ID) != core + 1 || - corelink_query(CORELINK_CORE_COUNT) != BB_CORE_NUM || + corelink_query(CORELINK_CORE_COUNT) != BB_NUM_CORES || bank_bytes < CORELINK_CROSS_BYTES || corelink_query(CORELINK_SHARED_BYTES) < bank_bytes * 2) __builtin_trap(); @@ -67,8 +66,8 @@ static void exercise(unsigned core) { } phase(core, 0); barrier(0); - unsigned next = (core + 1) % BB_THREAD_CORE_COUNT; - unsigned previous = (core + BB_THREAD_CORE_COUNT - 1) % BB_THREAD_CORE_COUNT; + unsigned next = (core + 1) % POLY_THREAD_CORES; + unsigned previous = (core + POLY_THREAD_CORES - 1) % POLY_THREAD_CORES; corelink_transfer(0, next + 1, bank_bytes - 16, 37); barrier(1); corelink_transfer(bank_bytes - 16, previous + 1, 128, 37); @@ -90,7 +89,7 @@ int main(void) { if (hart) exercise(hart - 1); else - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { } return 0; diff --git a/examples/designs/poly/workloads/device/runtime/ant_corelink_main.c b/examples/designs/poly/workloads/device/runtime/ant_corelink_main.c index b7e2dc138..daf39333b 100644 --- a/examples/designs/poly/workloads/device/runtime/ant_corelink_main.c +++ b/examples/designs/poly/workloads/device/runtime/ant_corelink_main.c @@ -1,23 +1,22 @@ #include "ant_corelink.h" +#include #include -#include -#include #include _Static_assert( - BB_CORE_NUM == BB_THREAD_CORE_COUNT + 1 && BB_THREAD_CORE_COUNT >= 2 && - !(BB_THREAD_CORE_COUNT & (BB_THREAD_CORE_COUNT - 1)), + BB_NUM_CORES == POLY_THREAD_CORES + 1 && POLY_THREAD_CORES >= 2 && + !(POLY_THREAD_CORES & (POLY_THREAD_CORES - 1)), "CoreLink gate requires a power-of-two thread-bearing core count"); extern const unsigned char ant_image[], ant_image_end[]; -static uint64_t input[BB_THREAD_CORE_COUNT][CORELINK_TEST_BYTES / 8] +static uint64_t input[POLY_THREAD_CORES][CORELINK_TEST_BYTES / 8] __attribute__((aligned(64))); -static uint64_t output[BB_THREAD_CORE_COUNT][CORELINK_TEST_BYTES / 8] +static uint64_t output[POLY_THREAD_CORES][CORELINK_TEST_BYTES / 8] __attribute__((aligned(64))); -static unsigned done[BB_THREAD_CORE_COUNT]; +static unsigned done[POLY_THREAD_CORES]; static unsigned arrived[2]; static void barrier(unsigned phase) { __atomic_fetch_add(&arrived[phase], 1, __ATOMIC_ACQ_REL); while (__atomic_load_n(&arrived[phase], __ATOMIC_ACQUIRE) != - BB_THREAD_CORE_COUNT) { + POLY_THREAD_CORES) { } } static unsigned pattern(unsigned core, unsigned i) { @@ -48,7 +47,7 @@ static void phase(unsigned core, unsigned operation) { static void exercise(unsigned core) { uint64_t bank_bytes = corelink_query(CORELINK_BANK_BYTES); if (corelink_query(CORELINK_CORE_ID) != core + 1 || - corelink_query(CORELINK_CORE_COUNT) != BB_CORE_NUM || + corelink_query(CORELINK_CORE_COUNT) != BB_NUM_CORES || corelink_query(CORELINK_BANK_BYTES) < CORELINK_TEST_BYTES || corelink_query(CORELINK_SHARED_BYTES) < bank_bytes * 2) __builtin_trap(); @@ -67,8 +66,8 @@ static void exercise(unsigned core) { } phase(core, 0); barrier(0); - unsigned next = (core + 1) % BB_THREAD_CORE_COUNT; - unsigned previous = (core + BB_THREAD_CORE_COUNT - 1) % BB_THREAD_CORE_COUNT; + unsigned next = (core + 1) % POLY_THREAD_CORES; + unsigned previous = (core + POLY_THREAD_CORES - 1) % POLY_THREAD_CORES; corelink_transfer(0, next + 1, bank_bytes + 16, 4101); barrier(1); phase(core, 1); @@ -86,7 +85,7 @@ int main(void) { if (hart) exercise(hart - 1); else - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { } return 0; diff --git a/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_control.c b/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_control.c index e9d708b42..5ed342439 100644 --- a/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_control.c +++ b/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_control.c @@ -1,5 +1,4 @@ #include "ant_corelink_reduce.h" -#include #include extern const unsigned char ant_image[], ant_image_end[]; void ant_corelink_reduce_phase(unsigned core, diff --git a/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_main.c b/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_main.c index b59a2e2f4..56982a88f 100644 --- a/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_main.c +++ b/examples/designs/poly/workloads/device/runtime/ant_corelink_reduce_main.c @@ -1,20 +1,18 @@ #include "ant_corelink_reduce.h" +#include #include #include -#include #include -#include #include _Static_assert( - BB_CORE_NUM == BB_THREAD_CORE_COUNT + 1 && BB_THREAD_CORE_COUNT >= 2 && - (BB_THREAD_CORE_COUNT & (BB_THREAD_CORE_COUNT - 1)) == 0, + BB_NUM_CORES == POLY_THREAD_CORES + 1 && POLY_THREAD_CORES >= 2 && + (POLY_THREAD_CORES & (POLY_THREAD_CORES - 1)) == 0, "Reduction needs main and a power-of-two thread-bearing core count"); -static float input[BB_THREAD_CORE_COUNT][272] __attribute__((aligned(64))); -static float output[BB_THREAD_CORE_COUNT][16] __attribute__((aligned(64))); -static unsigned ready[BB_THREAD_CORE_COUNT], broadcast, - done[BB_THREAD_CORE_COUNT]; +static float input[POLY_THREAD_CORES][272] __attribute__((aligned(64))); +static float output[POLY_THREAD_CORES][16] __attribute__((aligned(64))); +static unsigned ready[POLY_THREAD_CORES], broadcast, done[POLY_THREAD_CORES]; enum { WINDOW = 4 }; -static uint64_t shared_base[BB_THREAD_CORE_COUNT], main_base[WINDOW]; +static uint64_t shared_base[POLY_THREAD_CORES], main_base[WINDOW]; static unsigned main_ready, batch; static float partial(unsigned core, unsigned i) { return (float)(i + 1) * @@ -25,7 +23,8 @@ static void controller(unsigned core) { struct ant_corelink_reduce_args args = {0, (uintptr_t)input[core], (uintptr_t)output[core]}; ant_corelink_reduce_phase(core, &args); - shared_base[core] = corelink_shared_export(0, BB_SHARED_BANK_BASE, 0); + shared_base[core] = + corelink_shared_export(0, THREAD_FRONTEND_SHARED_VBANK_BASE, 0); while (!__atomic_load_n(&main_ready, __ATOMIC_ACQUIRE) || __atomic_load_n(&batch, __ATOMIC_ACQUIRE) != core / WINDOW) { } @@ -37,19 +36,18 @@ static void controller(unsigned core) { ant_corelink_reduce_phase(core, &args); for (unsigned i = 0; i < 4; ++i) { float expected = 0.0f; - for (unsigned rank = 0; rank < BB_THREAD_CORE_COUNT; ++rank) + for (unsigned rank = 0; rank < POLY_THREAD_CORES; ++rank) expected += partial(rank, i); if (output[core][i] != partial(core, i) || output[core][i + 4] != expected) __builtin_trap(); } - corelink_shared_release(0, BB_SHARED_BANK_BASE, 0); + corelink_shared_release(0, THREAD_FRONTEND_SHARED_VBANK_BASE, 0); args.phase = 2; ant_corelink_reduce_phase(core, &args); __atomic_store_n(&done[core], 1, __ATOMIC_RELEASE); } static void reduce(void) { - unsigned groups = - BB_THREAD_CORE_COUNT < WINDOW ? BB_THREAD_CORE_COUNT : WINDOW; + unsigned groups = POLY_THREAD_CORES < WINDOW ? POLY_THREAD_CORES : WINDOW; unsigned source = 32, accumulator = 33, target = 34; bb_mem_alloc(source, 1, groups); bb_mem_alloc(accumulator, 1, 1); @@ -57,7 +55,7 @@ static void reduce(void) { for (unsigned group = 0; group < groups; ++group) main_base[group] = corelink_shared_export(0, source, group); __atomic_store_n(&main_ready, 1, __ATOMIC_RELEASE); - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) { while (!__atomic_load_n(&ready[core], __ATOMIC_ACQUIRE)) { } bb_f32add(source, accumulator, target, 1, core % WINDOW, core == 0); @@ -69,10 +67,10 @@ static void reduce(void) { } } uint64_t result = corelink_shared_export(0, accumulator, 0); - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) corelink_transfer(result, core + 1, shared_base[core] + 16, 16); __atomic_store_n(&broadcast, 1, __ATOMIC_RELEASE); - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { } corelink_shared_release(0, accumulator, 0); @@ -85,11 +83,11 @@ static void reduce(void) { int main(void) { uintptr_t hart; __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart > BB_THREAD_CORE_COUNT || corelink_query(CORELINK_CORE_ID) != hart || - corelink_query(CORELINK_CORE_COUNT) != BB_THREAD_CORE_COUNT + 1 || + if (hart > POLY_THREAD_CORES || corelink_query(CORELINK_CORE_ID) != hart || + corelink_query(CORELINK_CORE_COUNT) != POLY_THREAD_CORES + 1 || corelink_query(CORELINK_BANK_BYTES) < 48 || corelink_query(CORELINK_SHARED_BYTES) < - corelink_query(CORELINK_BANK_BYTES) + BB_THREAD_CORE_COUNT * 16) + corelink_query(CORELINK_BANK_BYTES) + POLY_THREAD_CORES * 16) __builtin_trap(); if (hart) controller(hart - 1); diff --git a/examples/designs/poly/workloads/device/runtime/ant_mesh_main.c b/examples/designs/poly/workloads/device/runtime/ant_mesh_main.c index 84a7727b3..4b757b0b3 100644 --- a/examples/designs/poly/workloads/device/runtime/ant_mesh_main.c +++ b/examples/designs/poly/workloads/device/runtime/ant_mesh_main.c @@ -1,20 +1,17 @@ #include "ant_mesh.h" -#include -#include #include enum { - BANKS = VIRTUAL_BANK_NUM - BB_SHARED_BANK_BASE, - ROW_BYTES = BANK_WIDTH / 8 + BANKS = CORE_VIRTUAL_BANK_COUNT - THREAD_FRONTEND_SHARED_VBANK_BASE, + ROW_BYTES = THREAD_MEM_BANK_WIDTH / 8 }; -_Static_assert(BANKS >= 2 && BANK_WIDTH == 128, +_Static_assert(BANKS >= 2 && THREAD_MEM_BANK_WIDTH == 128, "Mesh tests require two 128-bit input rows"); -static uint8_t input[BB_THREAD_CORE_COUNT][BANKS * ROW_BYTES] +static uint8_t input[POLY_THREAD_CORES][BANKS * ROW_BYTES] __attribute__((aligned(64))); -static uint8_t output[BB_THREAD_CORE_COUNT] - [5 * 2 * ROW_BYTES + BANKS * ROW_BYTES] +static uint8_t output[POLY_THREAD_CORES][5 * 2 * ROW_BYTES + BANKS * ROW_BYTES] __attribute__((aligned(64))); -static unsigned done[BB_THREAD_CORE_COUNT]; +static unsigned done[POLY_THREAD_CORES]; extern const unsigned char ant_image[], ant_image_end[]; static void exercise(unsigned core) { if (thread_query(0, THREAD_FIELD_COUNT) != 5) @@ -70,8 +67,8 @@ static void exercise(unsigned core) { int main(void) { uintptr_t hart; __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart == BB_REPORT_HART) { - for (unsigned core = 0; core < BB_THREAD_CORE_COUNT; ++core) + if (hart == WORKLOAD_REPORT_HART) { + for (unsigned core = 0; core < POLY_THREAD_CORES; ++core) while (!__atomic_load_n(&done[core], __ATOMIC_ACQUIRE)) { } } else { diff --git a/examples/designs/poly/workloads/device/runtime/ant_mesh_move_main.c b/examples/designs/poly/workloads/device/runtime/ant_mesh_move_main.c index 1949c7c6d..62efbdc7b 100644 --- a/examples/designs/poly/workloads/device/runtime/ant_mesh_move_main.c +++ b/examples/designs/poly/workloads/device/runtime/ant_mesh_move_main.c @@ -1,16 +1,14 @@ #define _GNU_SOURCE #include "ant_mesh_move.h" -#include -#include #include extern const unsigned char ant_image[], ant_image_end[]; -enum { ROW_WORDS = BANK_WIDTH / 64, ROWS = 2 }; -static uint64_t input[BB_THREAD_CORE_COUNT][ROWS * ROW_WORDS] +enum { ROW_WORDS = THREAD_MEM_BANK_WIDTH / 64, ROWS = 2 }; +static uint64_t input[POLY_THREAD_CORES][ROWS * ROW_WORDS] __attribute__((section(".data"), aligned(4096))); -static uint64_t output[BB_THREAD_CORE_COUNT][ROWS * ROW_WORDS] +static uint64_t output[POLY_THREAD_CORES][ROWS * ROW_WORDS] __attribute__((section(".data"), aligned(4096))); -static unsigned done[BB_THREAD_CORE_COUNT]; +static unsigned done[POLY_THREAD_CORES]; static void run(unsigned context, unsigned id, struct ant_mesh_move_args *args) { @@ -84,10 +82,10 @@ static void exercise(unsigned tile) { int main(void) { uintptr_t hart; __asm__ volatile("csrr %0, mhartid" : "=r"(hart)); - if (hart != BB_REPORT_HART) + if (hart != WORKLOAD_REPORT_HART) exercise(hart - 1); else - for (unsigned tile = 0; tile < BB_THREAD_CORE_COUNT; ++tile) + for (unsigned tile = 0; tile < POLY_THREAD_CORES; ++tile) while (!__atomic_load_n(&done[tile], __ATOMIC_ACQUIRE)) { } return 0; diff --git a/examples/designs/poly/workloads/device/runtime/ctests/corelink_lease.c b/examples/designs/poly/workloads/device/runtime/ctests/corelink_lease.c index d20dc0b77..c05fa5967 100644 --- a/examples/designs/poly/workloads/device/runtime/ctests/corelink_lease.c +++ b/examples/designs/poly/workloads/device/runtime/ctests/corelink_lease.c @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -9,7 +8,7 @@ static const uint32_t input[2][4] __attribute__((aligned(64))) = { static volatile uint32_t output[8] __attribute__((aligned(64))); int main(void) { - unsigned bank = BB_SHARED_BANK_BASE; + unsigned bank = THREAD_FRONTEND_SHARED_VBANK_BASE; bb_mem_alloc(bank, 1, 2); for (unsigned i = 0; i < 8; ++i) output[i] = 0xaaaaaaaa; diff --git a/examples/designs/poly/workloads/device/runtime/stream_main.c b/examples/designs/poly/workloads/device/runtime/stream_main.c index bc431f5ca..14f379986 100644 --- a/examples/designs/poly/workloads/device/runtime/stream_main.c +++ b/examples/designs/poly/workloads/device/runtime/stream_main.c @@ -1,8 +1,7 @@ +#include #include -#include #include #include -#include #include #include #include diff --git a/examples/designs/poly/workloads/device/soc/CMakeLists.txt b/examples/designs/poly/workloads/device/soc/CMakeLists.txt index 892630029..33a05874a 100644 --- a/examples/designs/poly/workloads/device/soc/CMakeLists.txt +++ b/examples/designs/poly/workloads/device/soc/CMakeLists.txt @@ -1,6 +1,6 @@ # Cross-core Linux control threads share one coherent LLC. set(POLY_CONTROL_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-main-ctest-design-controllers" + "${DESIGN_NAME}-main-ctest-design-controllers" ) set(POLY_CONTROL_ELF ${POLY_CONTROL_NAME}-multicore-baremetal) buckyball_enforce_ctest_line_limit("${CMAKE_CURRENT_SOURCE_DIR}/controllers.c") @@ -9,15 +9,16 @@ add_custom_command( COMMAND ${ELF_CC} -O2 -g -march=rv64gc -mcmodel=medany -msmall-data-limit=0 -fno-builtin -nostdlib -nostartfiles -Wl,--no-relax -Wl,-T,${BBSIM_LD} - -I${BUCKYBALL_ISA_DIR}/design -I${BAREMETAL_DIR} -o ${POLY_CONTROL_ELF} + ${DESIGN_DEFINES} -I${RUNTIME_INCLUDE_DIR} -I${BAREMETAL_DIR} -o ${POLY_CONTROL_ELF} ${POLY_BOOT_SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/controllers.c -lgcc DEPENDS ${POLY_BOOT_SOURCE} controllers.c ${BAREMETAL_DIR}/start.S ${BBSIM_LD} - ${BUCKYBALL_ISA_DIR}/design/instances.h + ${RUNTIME_INCLUDE_DIR}/bb_cores.h WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + VERBATIM ) add_custom_target( ${POLY_CONTROL_NAME}-ctest-build @@ -29,8 +30,9 @@ set(BUCKYBALL_CTEST_TARGET attention) add_buckyball_linux_ctest(${POLY_CONTROL_NAME} ${CMAKE_CURRENT_SOURCE_DIR} controllers.c) target_include_directories( ${POLY_CONTROL_NAME}-linux - PRIVATE ${BUCKYBALL_ISA_DIR}/design + PRIVATE ${RUNTIME_INCLUDE_DIR} ) +target_compile_definitions(${POLY_CONTROL_NAME}-linux PRIVATE ${DESIGN_DEFS}) target_link_libraries(${POLY_CONTROL_NAME}-linux PRIVATE pthread) add_dependencies(poly-ctests-build ${POLY_CONTROL_NAME}-linux-build) diff --git a/examples/designs/poly/workloads/device/soc/controllers.c b/examples/designs/poly/workloads/device/soc/controllers.c index 939c30c21..3340dd79b 100644 --- a/examples/designs/poly/workloads/device/soc/controllers.c +++ b/examples/designs/poly/workloads/device/soc/controllers.c @@ -1,5 +1,5 @@ #define _GNU_SOURCE -#include +#include #include #ifdef __linux__ #include @@ -9,7 +9,7 @@ #include #endif // Controllers publish cache-line payloads and contend on one atomic counter. -enum { CONTROLLERS = BB_CORE_NUM, ROUNDS = 2 }; +enum { CONTROLLERS = BB_NUM_CORES, ROUNDS = 2 }; static uint64_t payload[CONTROLLERS][8] __attribute__((aligned(64))); static uint64_t counter __attribute__((aligned(64))); static uint64_t turn __attribute__((aligned(64))); diff --git a/examples/designs/toy/arch/build.sc b/examples/designs/toy/arch/build.sc index 1ef8185c8..b4ffafece 100644 --- a/examples/designs/toy/arch/build.sc +++ b/examples/designs/toy/arch/build.sc @@ -2,7 +2,7 @@ import mill._ import mill.scalalib._ import $file.^.^.^.^.arch.src.main.scala.framework.device.cbus.{build => cbus_build} import $file.^.^.^.{build => examples} -import $file.^.^.^.components.cores.rocket.arch.{build => rocket} +import $file.^.^.^.prefab.cores.rocket.arch.{build => rocket} val designRoot: os.Path = millSourcePath / os.up object design extends SbtModule { diff --git a/examples/designs/toy/configs/device/core/acc/balldomain/default.toml b/examples/designs/toy/configs/device/core/acc/balldomain/default.toml index 4b5b830f8..3ae947026 100644 --- a/examples/designs/toy/configs/device/core/acc/balldomain/default.toml +++ b/examples/designs/toy/configs/device/core/acc/balldomain/default.toml @@ -59,3 +59,23 @@ port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } instructions = [ { mnemonic = "REDMULE_F16", funct7 = 68 }, ] + +[[balls]] +id = 5 +name = "VecBall" +component = "vector" +param = "default" +port = { inBW = 2, outBW = 4, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "VECMAT16", funct7 = 64 }, +] + +[[balls]] +id = 6 +name = "Mxfp2IntBall" +component = "mxfp2int" +param = "default" +port = { inBW = 2, outBW = 1, mmioReadBW = 0, mmioWriteBW = 0 } +instructions = [ + { mnemonic = "MXFP2INT", funct7 = 65 }, +] diff --git a/examples/designs/toy/configs/device/core/acc/frontend/demo.toml b/examples/designs/toy/configs/device/core/acc/frontend/demo.toml index c3e3d5e71..5464f2d95 100644 --- a/examples/designs/toy/configs/device/core/acc/frontend/demo.toml +++ b/examples/designs/toy/configs/device/core/acc/frontend/demo.toml @@ -2,8 +2,8 @@ bankAliasEnable = true robEntries = 16 rsOutOfOrderResponse = true bankIdLen = 10 -vbankIdUpperBound = 15 -sharedBankIdBase = 32 +maxPrivateVbank = 15 +sharedVbankBase = 32 iterLen = 34 subRobEnable = false subRobDepth = 64 diff --git a/examples/designs/toy/configs/device/core/acc/frontend/toy.toml b/examples/designs/toy/configs/device/core/acc/frontend/toy.toml index b5d20a710..0a50370e8 100644 --- a/examples/designs/toy/configs/device/core/acc/frontend/toy.toml +++ b/examples/designs/toy/configs/device/core/acc/frontend/toy.toml @@ -2,8 +2,8 @@ bankAliasEnable = false robEntries = 16 rsOutOfOrderResponse = true bankIdLen = 10 -vbankIdUpperBound = 15 -sharedBankIdBase = 32 +maxPrivateVbank = 15 +sharedVbankBase = 32 iterLen = 34 subRobEnable = true subRobDepth = 4 diff --git a/examples/designs/toy/configs/device/core/acc/memdomain/paper.toml b/examples/designs/toy/configs/device/core/acc/memdomain/paper.toml index 912791949..c36e3debc 100644 --- a/examples/designs/toy/configs/device/core/acc/memdomain/paper.toml +++ b/examples/designs/toy/configs/device/core/acc/memdomain/paper.toml @@ -1,6 +1,8 @@ # Toy ACC Memory Domain Configuration (default) # Private (per-thread) memory parameters. Shared memory lives in the core TOML. +addrLen = 39 + [bank] layoutEnable = true num = 16 @@ -18,6 +20,3 @@ bankNum = 16 bankEntries = 64 bankWidth = 8 readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/designs/toy/configs/device/core/acc/memdomain/toy.toml b/examples/designs/toy/configs/device/core/acc/memdomain/toy.toml index 69b64fc76..27f20096d 100644 --- a/examples/designs/toy/configs/device/core/acc/memdomain/toy.toml +++ b/examples/designs/toy/configs/device/core/acc/memdomain/toy.toml @@ -1,6 +1,8 @@ # Toy ACC Memory Domain Configuration (default) # Private (per-thread) memory parameters. Shared memory lives in the core TOML. +addrLen = 39 + [bank] layoutEnable = true num = 16 @@ -18,6 +20,3 @@ bankNum = 16 bankEntries = 64 bankWidth = 8 readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/designs/toy/configs/device/core/default.toml b/examples/designs/toy/configs/device/core/default.toml index 14ec3cb36..27bc0c078 100644 --- a/examples/designs/toy/configs/device/core/default.toml +++ b/examples/designs/toy/configs/device/core/default.toml @@ -27,7 +27,7 @@ entries = 16384 inputChannels = 1 defaultGroupCount = 1 virtualBankCount = 16 -memCycles = 10 +latency = 10 bankEntries = 1024 bankWidth = 128 diff --git a/examples/designs/toy/configs/device/core/memory/access/core.toml b/examples/designs/toy/configs/device/core/memory/access/core.toml index bb87f76a4..2e90444bc 100644 --- a/examples/designs/toy/configs/device/core/memory/access/core.toml +++ b/examples/designs/toy/configs/device/core/memory/access/core.toml @@ -1,5 +1,5 @@ translationEntries = 16 -accessTagBits = 6 +tagBits = 6 resultLineBits = 0 [[regions]] base = 0x80000000 diff --git a/examples/designs/toy/configs/device/core/memory/dcache/core.toml b/examples/designs/toy/configs/device/core/memory/dcache/core.toml index 97b8e2bba..4fb4beb6f 100644 --- a/examples/designs/toy/configs/device/core/memory/dcache/core.toml +++ b/examples/designs/toy/configs/device/core/memory/dcache/core.toml @@ -7,7 +7,7 @@ maxRanges = 8 [cache] nodeId = 2 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/toy/configs/device/core/memory/home/core.toml b/examples/designs/toy/configs/device/core/memory/home/core.toml index 3e0779cbc..aa39cd52e 100644 --- a/examples/designs/toy/configs/device/core/memory/home/core.toml +++ b/examples/designs/toy/configs/device/core/memory/home/core.toml @@ -1,6 +1,6 @@ agents = 2 mshrEntries = 8 -homeId = 64 +nodeId = 64 requesters = "default" [cache] diff --git a/examples/designs/toy/configs/device/core/memory/icache/core.toml b/examples/designs/toy/configs/device/core/memory/icache/core.toml index 75e065643..3cc7df83a 100644 --- a/examples/designs/toy/configs/device/core/memory/icache/core.toml +++ b/examples/designs/toy/configs/device/core/memory/icache/core.toml @@ -1,5 +1,5 @@ nodeId = 1 -cacheLines = 64 +lines = 64 homeId = 64 homeCount = 1 banks = 2 diff --git a/examples/designs/toy/configs/sims/default.toml b/examples/designs/toy/configs/sims/default.toml index 0463ed6ae..99109f9f2 100644 --- a/examples/designs/toy/configs/sims/default.toml +++ b/examples/designs/toy/configs/sims/default.toml @@ -11,5 +11,8 @@ regression = "sims/device/regression" kernel = "sims/device/workloads/kernel" tapeout = "sims/device/tapeout" +[targets.device.configs] +ToyVerilatorConfig = "verilator.toml" + [verification] registry = "../../sims/ip/uvm/registry.toml" diff --git a/examples/designs/toy/configs/sims/verilator.toml b/examples/designs/toy/configs/sims/verilator.toml new file mode 100644 index 000000000..20c5b7796 --- /dev/null +++ b/examples/designs/toy/configs/sims/verilator.toml @@ -0,0 +1 @@ +# ToyVerilatorConfig: the toy design as configured, with no overrides. diff --git a/examples/designs/toy/sims/device/emu/build.rs b/examples/designs/toy/sims/device/emu/build.rs index d634d5dba..beb62e98c 100644 --- a/examples/designs/toy/sims/device/emu/build.rs +++ b/examples/designs/toy/sims/device/emu/build.rs @@ -1,7 +1,4 @@ -use std::{env, path::PathBuf}; - fn main() { - let entry = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - bebop_sim_build::balls(&entry.join("../../../output/generated/design.pb")); + bebop_sim_build::balls(); bebop_sim_build::runner_links(); } diff --git a/examples/designs/toy/sims/device/emu/src/lib.rs b/examples/designs/toy/sims/device/emu/src/lib.rs index 9684e5e32..3511b5518 100644 --- a/examples/designs/toy/sims/device/emu/src/lib.rs +++ b/examples/designs/toy/sims/device/emu/src/lib.rs @@ -6,6 +6,6 @@ extern crate bebop_design_toy; include!(concat!(env!("OUT_DIR"), "/balls.rs")); pub const SIMULATION: bebop_bemu::Simulation = bebop_bemu::Simulation { - design: include_bytes!("../../../../output/generated/design.pb"), + design: include_bytes!(concat!(env!("OUT_DIR"), "/design.pb")), instantiate: bebop_bemu::device::cbus::instantiate, }; diff --git a/examples/designs/toy/sims/device/emu/tests/instance_isolation.rs b/examples/designs/toy/sims/device/emu/tests/instance_isolation.rs index 866381f72..3991b60f6 100644 --- a/examples/designs/toy/sims/device/emu/tests/instance_isolation.rs +++ b/examples/designs/toy/sims/device/emu/tests/instance_isolation.rs @@ -12,11 +12,17 @@ fn independent_devices_keep_gemmini_configuration_and_memory_private() { let repo = manifest.ancestors().nth(6).expect("simulation entry location"); let output = manifest.join("target/verification/instance-isolation"); std::fs::create_dir_all(&output).unwrap(); - let isa = output.join("isa"); run(Command::new("python3").arg(repo.join("stack/scripts/design.py")) - .arg("--design-pb").arg(repo.join("examples/designs/toy/output/generated/design.pb")) - .arg("--target").arg("device") + .arg("--design-pb").arg(concat!(env!("OUT_DIR"), "/design.pb")) .arg("--out").arg(&output)); + // The compile definitions of the toy target, as design.cmake lists them. + let cmake = std::fs::read_to_string(output.join("design.cmake")).unwrap(); + let defs: Vec = cmake.lines() + .find_map(|line| line.strip_prefix("set(DESIGN_TARGET_DEFS_toy ")) + .expect("toy target in design.cmake") + .trim_end_matches(')').split_whitespace() + .map(|def| format!("-D{}", def.trim_matches('"'))) + .collect(); let bare = repo.join("stack/os/baremetal"); bebop_bemu::configs::load(simulation::SIMULATION.design); let mut cpus = Vec::new(); @@ -26,11 +32,11 @@ fn independent_devices_keep_gemmini_configuration_and_memory_private() { .args(["-nostdlib", "-nostartfiles", "-O2", "-march=rv64gc", "-mabi=lp64d", "-mcmodel=medany", "-msmall-data-limit=0", "-fno-builtin", "-Wl,--no-relax"]) .arg("-T").arg(bare.join("bbsim.ld")) - .arg("-I").arg(isa.join("toy")) + .args(&defs) .arg("-I").arg(repo.join("bb-tests/workloads/lib")) - .arg("-I").arg(repo.join("examples/components/balls/gemmini/workloads")) + .arg("-I").arg(repo.join("examples/prefab/balls/gemmini/workloads")) .arg(bare.join("crt0.S")) - .arg(repo.join(format!("examples/components/balls/gemmini/workloads/ctests/{name}.c"))) + .arg(repo.join(format!("examples/prefab/balls/gemmini/workloads/ctests/{name}.c"))) .arg("-o").arg(&elf)); let log = output.join(name); let args = Args::try_parse_from([Path::new("test"), Path::new("--elf"), &elf, diff --git a/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml b/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml index 7c51b074c..345418dc0 100644 --- a/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml +++ b/examples/designs/toy/sims/device/regression/bemu/workloads-bare.toml @@ -54,4 +54,13 @@ tests = [ "toy-toy-mlirtest-linalg_conv2d_nhwc_hwcf_deep-baremetal", "toy-toy-mlirtest-base_bank_loop_carried-baremetal", "toy-toy-mlirtest-base_bank_transfer-baremetal", + + "toy-toy-ctest-vecmat16_atranspose_16x16_test-baremetal", + "toy-toy-ctest-vecmat16_accumulate_k4_test-baremetal", + "toy-toy-mlirtest-ball_matmul_at_i8_16x16-baremetal", + "toy-toy-mlirtest-bank_matmul_at_i8_bank_depth-baremetal", + "toy-toy-ctest-mxfp4_e8m0_two_block_test-baremetal", + "toy-toy-ctest-mxfp4_e8m0_scale_range_test-baremetal", + "toy-toy-mlirtest-ball_mxfp4_i8_2blocks-baremetal", + "toy-toy-mlirtest-bank_mxfp4_i8_bank_blocks-baremetal", ] diff --git a/examples/designs/toy/sims/device/regression/bemu/workloads-linux.toml b/examples/designs/toy/sims/device/regression/bemu/workloads-linux.toml index 6bb6a056a..41a9cd13b 100644 --- a/examples/designs/toy/sims/device/regression/bemu/workloads-linux.toml +++ b/examples/designs/toy/sims/device/regression/bemu/workloads-linux.toml @@ -1,4 +1,4 @@ [workloads] tests = [ - "fw_payload-h1-toy-linux.elf", + "fw_payload-linux.elf", ] diff --git a/examples/designs/toy/sims/device/regression/p2e/workloads-linux.toml b/examples/designs/toy/sims/device/regression/p2e/workloads-linux.toml index 1a0a1bbd3..01747a69b 100644 --- a/examples/designs/toy/sims/device/regression/p2e/workloads-linux.toml +++ b/examples/designs/toy/sims/device/regression/p2e/workloads-linux.toml @@ -1,4 +1,4 @@ [workloads] tests = [ - "fw_payload-h1-toy-linux.hex", + "fw_payload-linux.hex", ] diff --git a/examples/designs/toy/sims/device/regression/verilator/workloads-bare.toml b/examples/designs/toy/sims/device/regression/verilator/workloads-bare.toml index 6f18573a9..4f59b52c7 100644 --- a/examples/designs/toy/sims/device/regression/verilator/workloads-bare.toml +++ b/examples/designs/toy/sims/device/regression/verilator/workloads-bare.toml @@ -36,4 +36,11 @@ tests = [ "toy-toy-ctest-tlb_sv39_4k_alias_test-baremetal", "toy-toy-ctest-tlb_sfence_remap_test-baremetal", "toy-toy-ctest-mvin_mvout_stride_test-baremetal", + + "toy-toy-ctest-vecmat16_atranspose_16x16_test-baremetal", + "toy-toy-ctest-vecmat16_accumulate_k4_test-baremetal", + "toy-toy-mlirtest-ball_matmul_at_i8_16x16-baremetal", + "toy-toy-ctest-mxfp4_e8m0_two_block_test-baremetal", + "toy-toy-ctest-mxfp4_e8m0_scale_range_test-baremetal", + "toy-toy-mlirtest-ball_mxfp4_i8_2blocks-baremetal", ] diff --git a/examples/designs/toy/sims/device/stack/CMakeLists.txt b/examples/designs/toy/sims/device/stack/CMakeLists.txt index b077e0e8e..b7c6f93b8 100644 --- a/examples/designs/toy/sims/device/stack/CMakeLists.txt +++ b/examples/designs/toy/sims/device/stack/CMakeLists.txt @@ -2,7 +2,7 @@ # The software stack of the toy device simulation: every level the device # needs, built bottom up. # -# bbdev stack --build "--design toy" +# bbdev stack --build "--design toy --device " # # configures this with DESIGN_CMAKE, generated from design.pb. Each level is a # CMake build of its own: cores and threads build buddy-opt plugins for the diff --git a/examples/designs/toy/sims/device/stack/core/toy/CMakeLists.txt b/examples/designs/toy/sims/device/stack/core/toy/CMakeLists.txt deleted file mode 100644 index da36ffb78..000000000 --- a/examples/designs/toy/sims/device/stack/core/toy/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -cmake_minimum_required(VERSION 3.20) -project(toy-core-stack LANGUAGES C CXX) -include("${DESIGN_CMAKE}") - -add_subdirectory("${DESIGN_REPO}/examples/designs/toy/stack/engine" engine) -add_subdirectory("${DESIGN_REPO}/examples/designs/toy/stack/midend" midend) diff --git a/examples/designs/toy/sims/device/tapeout/area/dc.tcl b/examples/designs/toy/sims/device/tapeout/area/dc.tcl index 7833680ca..460208870 100644 --- a/examples/designs/toy/sims/device/tapeout/area/dc.tcl +++ b/examples/designs/toy/sims/device/tapeout/area/dc.tcl @@ -27,7 +27,7 @@ file mkdir $output_dir $report_dir [file join $report_dir work] set target_library [list $RUN_TARGET_LIBRARY] set synthetic_library $RUN_SYNTHETIC_LIBRARY set link_library [concat [list *] $target_library $synthetic_library $RUN_LINK_LIBRARY] -set_host_options -max_cores $RUN_MAX_CORES +set_host_options -max_cores $RUN_MAX_THREADS define_design_lib WORK -path [file join $report_dir work] set search_path [list . [file dirname $RUN_SOURCE_LIST] [file dirname [file dirname $RUN_SOURCE_LIST]] [file dirname [file dirname [file dirname $RUN_SOURCE_LIST]]]] set_app_var verilogout_no_tri true diff --git a/examples/designs/toy/sims/device/workloads/CMakeLists.txt b/examples/designs/toy/sims/device/workloads/CMakeLists.txt index ebf202d4c..6253eaee7 100644 --- a/examples/designs/toy/sims/device/workloads/CMakeLists.txt +++ b/examples/designs/toy/sims/device/workloads/CMakeLists.txt @@ -27,19 +27,16 @@ foreach(BUCKYBALL_COMPONENT_WORKLOAD_TARGET IN LISTS DESIGN_CPU_TARGETS) if( COMPONENT_WORKLOADS STREQUAL - "${BUCKYBALL_REPO_ROOT}/examples/designs/${BUCKYBALL_WORKLOAD_DESIGN}/workloads" + "${BUCKYBALL_REPO_ROOT}/examples/designs/${DESIGN_NAME}/workloads" ) continue() endif() - set(BUCKYBALL_COMPONENT_WORKLOAD_PROFILE - "${DESIGN_TARGET_ID_${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}}" - ) set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}") set(BUCKYBALL_CTEST_GROUP_TARGET component-ctests-build) if(IS_DIRECTORY "${COMPONENT_WORKLOADS}/ctests") add_subdirectory( "${COMPONENT_WORKLOADS}/ctests" - "${CMAKE_CURRENT_BINARY_DIR}/components/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" + "${CMAKE_CURRENT_BINARY_DIR}/prefab/${BUCKYBALL_COMPONENT_WORKLOAD_TARGET}/ctests" ) endif() endforeach() @@ -58,9 +55,9 @@ add_subdirectory( ) foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) set(BUCKYBALL_MLIR_ACTIVE_TARGET "${BUCKYBALL_TARGET}") - foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_TARGET_BALLS_${BUCKYBALL_TARGET}) + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_BALLDOMAIN_BALLS) set(BALL_MLIR_TESTS - "${BUCKYBALL_REPO_ROOT}/examples/components/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/${BUCKYBALL_BALL_DIR}/workloads/mlir_tests" ) if(IS_DIRECTORY "${BALL_MLIR_TESTS}") add_subdirectory( diff --git a/examples/designs/toy/sims/device/stack/runtime/CMakeLists.txt b/examples/designs/toy/stack/device/CMakeLists.txt similarity index 91% rename from examples/designs/toy/sims/device/stack/runtime/CMakeLists.txt rename to examples/designs/toy/stack/device/CMakeLists.txt index 995b2c8bb..fa1e71aef 100644 --- a/examples/designs/toy/sims/device/stack/runtime/CMakeLists.txt +++ b/examples/designs/toy/stack/device/CMakeLists.txt @@ -2,7 +2,7 @@ # The toy device: its cores, their hardware threads and the runtime that # schedules jobs onto the cores. # -# bbdev stack --build "--design toy" +# bbdev stack --build "--design toy --device " # # builds every thread and core stack of the device first, then configures this # with DESIGN_CMAKE, generated from design.pb, and the RISC-V Linux toolchain. @@ -24,12 +24,12 @@ add_library(bindings INTERFACE) foreach(CORE IN LISTS DESIGN_CORES) target_link_options(bindings INTERFACE "${DESIGN_CORE_BUILD_${CORE}}/execute.o") endforeach() -foreach(THREAD IN LISTS DESIGN_THREADS) +foreach(THREAD IN LISTS DESIGN_THREAD_STACKS) target_link_options(bindings INTERFACE "${DESIGN_THREAD_BUILD_${THREAD}}/loader.o") endforeach() # DMA helpers are sized by the target with the fewest bank lines. list(GET DESIGN_RUNTIME_TARGETS 0 PARAMS_TARGET) -set(RUNTIME_PARAMS "${DESIGN_ISA_DIR}/${PARAMS_TARGET}") +set(RUNTIME_DEFS ${DESIGN_TARGET_DEFS_${PARAMS_TARGET}}) add_subdirectory("${DESIGN_REPO}/stack/engine" engine) add_subdirectory("${DESIGN_REPO}/stack/runtime" runtime) diff --git a/examples/designs/toy/stack/device/core/CMakeLists.txt b/examples/designs/toy/stack/device/core/CMakeLists.txt new file mode 100644 index 000000000..04e089c64 --- /dev/null +++ b/examples/designs/toy/stack/device/core/CMakeLists.txt @@ -0,0 +1,36 @@ +# ===----------------------------------------------------------------------=== +# Toy's own core: a CPU with its Ball accelerator. +# +# Built by the stack top of each simulation entry, configured with DESIGN_CMAKE +# and CORE. Products: +# execute.o how the core runs jobs (direct execution) +# libtoy-compiler.so buddy-opt plugin: the core's fusion (form-custom-ops) +# and lower-buckyball around the framework bank layer +# ===----------------------------------------------------------------------=== +cmake_minimum_required(VERSION 3.20) +project(toy-core-stack LANGUAGES C CXX) +include("${DESIGN_CMAKE}") + +set(RUNTIME "${DESIGN_REPO}/stack/runtime") +set(TARGET ${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_PROFILE}) +find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) + +# How the core runs jobs. +add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/execute.o" + COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" + -c "${RUNTIME}/execute/direct.c" -o "${CMAKE_BINARY_DIR}/execute.o" + DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" + VERBATIM) +add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") + +# The core's compiler plugin. The core owns computation recognition and its +# conversion to bank dataflow. +set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${TARGET}}) +file(GLOB_RECURSE BUCKYBALL_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp") +add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) +add_library(${TARGET}-compiler MODULE) +target_link_libraries(${TARGET}-compiler PRIVATE buckyball-compiler) +set_target_properties(${TARGET}-compiler PROPERTIES + LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") diff --git a/examples/designs/toy/stack/midend/src/ConvertLinalgToBuckyball.cpp b/examples/designs/toy/stack/device/core/src/ConvertLinalgToBuckyball.cpp similarity index 94% rename from examples/designs/toy/stack/midend/src/ConvertLinalgToBuckyball.cpp rename to examples/designs/toy/stack/device/core/src/ConvertLinalgToBuckyball.cpp index d67c11652..8da3baddd 100644 --- a/examples/designs/toy/stack/midend/src/ConvertLinalgToBuckyball.cpp +++ b/examples/designs/toy/stack/device/core/src/ConvertLinalgToBuckyball.cpp @@ -14,6 +14,7 @@ #include "Buckyball/BuckyballOps.h" #include "Custom/CustomOps.h" #include "Target/BuckyballTargetRegistry.h" +#include "gemminiOps.h" #include "mlir/Dialect/Affine/IR/AffineOps.h" #include "mlir/Dialect/Arith/IR/Arith.h" @@ -177,12 +178,11 @@ void lowerConvolution(::buddy::custom::CustomOp op) { Value aBank = gather(aTiles, i * kt); Value bBank = gather(bTiles, j * kt); Value cBank = BankAllocOp::create(b, loc, i64, 1, 4); - auto matmul = - BankCustomOp::create(b, loc, i64, ValueRange{aBank, bBank}, cBank, - ValueRange{}, "matmul_i8_tile", false); - matmul->setAttr("k", b.getI64IntegerAttr(kt * 16)); + cBank = ::buddy::gemmini_ball::BankMatmulI8TileOp::create( + b, loc, i64, aBank, bBank, cBank, b.getI64IntegerAttr(kt * 16), + UnitAttr(), IntegerAttr()); cBank = BankMvoutOp::create(b, loc, i64, tile(cTiles, i * nt + j, 1), - matmul.getDestinationOut(), 16, 1); + cBank, 16, 1); BankReleaseOp::create(b, loc, aBank); BankReleaseOp::create(b, loc, bBank); BankReleaseOp::create(b, loc, cBank); @@ -214,10 +214,11 @@ class ConvertLinalgToBuckyballPass return "Lower the toy custom computations to the core bank dataflow"; } void getDependentDialects(DialectRegistry ®istry) const override { - registry.insert(); + registry.insert(); } void runOnOperation() override { SmallVector<::buddy::custom::CustomOp> computations; diff --git a/examples/designs/toy/stack/midend/src/FormCustomOps.cpp b/examples/designs/toy/stack/device/core/src/FormCustomOps.cpp similarity index 100% rename from examples/designs/toy/stack/midend/src/FormCustomOps.cpp rename to examples/designs/toy/stack/device/core/src/FormCustomOps.cpp diff --git a/examples/designs/toy/stack/engine/CMakeLists.txt b/examples/designs/toy/stack/engine/CMakeLists.txt deleted file mode 100644 index 686343604..000000000 --- a/examples/designs/toy/stack/engine/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Core execution binding; instruction streams use the framework engine. -set(RUNTIME "${DESIGN_REPO}/stack/runtime") -set(ENGINE "${DESIGN_REPO}/stack/engine") -find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) - -add_custom_command( - OUTPUT "${CMAKE_BINARY_DIR}/execute.o" - COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 - "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" - -c "${RUNTIME}/execute/direct.c" -o "${CMAKE_BINARY_DIR}/execute.o" - DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" - VERBATIM) -add_custom_target(execute ALL DEPENDS "${CMAKE_BINARY_DIR}/execute.o") diff --git a/examples/designs/toy/stack/midend/CMakeLists.txt b/examples/designs/toy/stack/midend/CMakeLists.txt deleted file mode 100644 index d0775a0cb..000000000 --- a/examples/designs/toy/stack/midend/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Host compiler plugin for this design's core. -set(ENGINE "${DESIGN_REPO}/stack/engine") -set(TARGET ${DESIGN_CORE_TARGET_${CORE}}) - -set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${TARGET}}) -set(BUCKYBALL_TARGET_TABLE "${DESIGN_ISA_DIR}/${TARGET}/target.inc") -# The core owns computation recognition and its conversion to bank dataflow. -file(GLOB_RECURSE BUCKYBALL_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/src/*.cpp") - -add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) -add_library(${TARGET}-compiler MODULE $) -set_target_properties(${TARGET}-compiler PROPERTIES - LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") diff --git a/examples/designs/toy/workloads/device/core/acc/CMakeLists.txt b/examples/designs/toy/workloads/device/core/acc/CMakeLists.txt index 7b5637b41..d17debe0e 100644 --- a/examples/designs/toy/workloads/device/core/acc/CMakeLists.txt +++ b/examples/designs/toy/workloads/device/core/acc/CMakeLists.txt @@ -1,9 +1,9 @@ add_subdirectory(ctests) foreach(BUCKYBALL_TARGET IN LISTS DESIGN_CPU_TARGETS) set(BUCKYBALL_CTEST_TARGET "${BUCKYBALL_TARGET}") - foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_TARGET_BALLS_${BUCKYBALL_TARGET}) + foreach(BUCKYBALL_BALL_DIR IN LISTS DESIGN_PROFILES_${BUCKYBALL_TARGET}_THREAD_BALLDOMAIN_BALLS) add_subdirectory( - "${BUCKYBALL_REPO_ROOT}/examples/components/balls/${BUCKYBALL_BALL_DIR}/workloads/ctests" + "${BUCKYBALL_REPO_ROOT}/examples/prefab/balls/${BUCKYBALL_BALL_DIR}/workloads/ctests" "${CMAKE_CURRENT_BINARY_DIR}/balls/${BUCKYBALL_TARGET}/${BUCKYBALL_BALL_DIR}/ctests") endforeach() endforeach() diff --git a/examples/designs/toy/workloads/device/core/acc/ctests/CMakeLists.txt b/examples/designs/toy/workloads/device/core/acc/ctests/CMakeLists.txt index b89676936..cde0eb72a 100644 --- a/examples/designs/toy/workloads/device/core/acc/ctests/CMakeLists.txt +++ b/examples/designs/toy/workloads/device/core/acc/ctests/CMakeLists.txt @@ -9,13 +9,13 @@ file( ) add_buckyball_ctest(${RVV_KERNEL_SOURCE}) set(RVV_KERNEL_TEST - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_kernel" + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_kernel" ) add_dependencies(toy-ctests-build ${RVV_KERNEL_TEST}-ctest-build) add_buckyball_ctest(rvv_ball.c) set(RVV_BALL_TEST - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_ball" + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_ball" ) add_dependencies(toy-ctests-build ${RVV_BALL_TEST}-ctest-build) diff --git a/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp index 1a553d265..f2fb24be2 100644 --- a/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp +++ b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/gemmini_relu_trace_pipeline-main.cpp @@ -12,7 +12,7 @@ int main() { __ATOMIC_RELEASE); __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); } - if (hart == BB_REPORT_HART) { + if (hart == WORKLOAD_REPORT_HART) { while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) asm volatile("nop"); return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; diff --git a/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/toy.c b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/toy.c index ed565c654..13e3a8686 100644 --- a/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/toy.c +++ b/examples/designs/toy/workloads/device/core/acc/ctests/gemmini_relu_trace_pipeline/toy.c @@ -7,7 +7,8 @@ #include #include -#if BANK_WIDTH != 128 || BANK_LINES < 64 || BANK_LINES % 64 != 0 +#if THREAD_MEM_BANK_WIDTH != 128 || THREAD_MEM_BANK_ENTRIES < 64 || \ + THREAD_MEM_BANK_ENTRIES % 64 != 0 #error "Toy ACC pipeline requires 128-bit banks and depth divisible by 64" #endif diff --git a/examples/designs/toy/workloads/device/core/acc/ctests/rvv_ball.c b/examples/designs/toy/workloads/device/core/acc/ctests/rvv_ball.c index c234c4378..1be427efd 100644 --- a/examples/designs/toy/workloads/device/core/acc/ctests/rvv_ball.c +++ b/examples/designs/toy/workloads/device/core/acc/ctests/rvv_ball.c @@ -1,5 +1,4 @@ #include -#include #include #include @@ -17,20 +16,20 @@ int main(void) { 0, 12, 0x40002000, - {BB_BANK0(2) | BB_BANK2(2) | BB_ITER(16), BANK_LINES, - BB_BANK0(3) | BB_BANK2(3) | BB_ITER(16), BANK_LINES}, + {BB_BANK0(2) | BB_BANK2(2) | BB_ITER(16), THREAD_MEM_BANK_ENTRIES, + BB_BANK0(3) | BB_BANK2(3) | BB_ITER(16), THREAD_MEM_BANK_ENTRIES}, 0}; bb_mvin((uintptr_t)input, 2, 16, 1); bb_mvin((uintptr_t)&call, 0, sizeof(call) / 16, 1); bb_mem_alloc(4, 1, 1); bb_mem_transfer(0, 3); bb_mem_transfer(2, 3); - mvin_kernel(image, sizeof(image), VIRTUAL_BANK_NUM); - run_kernel(4, VIRTUAL_BANK_NUM, 3, 0); + mvin_kernel(image, sizeof(image), CORE_VIRTUAL_BANK_COUNT); + run_kernel(4, CORE_VIRTUAL_BANK_COUNT, 3, 0); int32_t groups[2][64] __attribute__((aligned(16))); bb_mvout_group((uintptr_t)groups[0], 3, 1, 16, 1); bb_mvout_group((uintptr_t)groups[1], 3, 2, 16, 1); - release_kernel(VIRTUAL_BANK_NUM); + release_kernel(CORE_VIRTUAL_BANK_COUNT); for (unsigned row = 0; row < 16; ++row) for (unsigned group = 0; group < 2; ++group) for (unsigned lane = 0; lane < 4; ++lane) diff --git a/examples/designs/toy/workloads/device/soc/CMakeLists.txt b/examples/designs/toy/workloads/device/soc/CMakeLists.txt index f100f3903..7a5138b0c 100644 --- a/examples/designs/toy/workloads/device/soc/CMakeLists.txt +++ b/examples/designs/toy/workloads/device/soc/CMakeLists.txt @@ -1,5 +1,5 @@ add_buckyball_ctest(scu_uart_rx_echo_test.c) set(SCU_UART_TEST - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-scu_uart_rx_echo_test" + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-scu_uart_rx_echo_test" ) add_dependencies(soctests-build ${SCU_UART_TEST}-baremetal-build) diff --git a/examples/components/balls/build.sc b/examples/prefab/balls/build.sc similarity index 100% rename from examples/components/balls/build.sc rename to examples/prefab/balls/build.sc diff --git a/examples/components/balls/f32add/arch/F32AddBall.scala b/examples/prefab/balls/f32add/arch/F32AddBall.scala similarity index 98% rename from examples/components/balls/f32add/arch/F32AddBall.scala rename to examples/prefab/balls/f32add/arch/F32AddBall.scala index 17da110ea..beb91c17b 100644 --- a/examples/components/balls/f32add/arch/F32AddBall.scala +++ b/examples/prefab/balls/f32add/arch/F32AddBall.scala @@ -91,7 +91,7 @@ class F32AddBall( "F32ADD source group must exist and accumulators must have one group" ) for (bank <- Seq(cmd.op1_bank, cmd.op2_bank, cmd.wr_bank)) { - val shared = bank >= fp.shared_bank_id_base.U + val shared = bank >= fp.shared_vbank_base.U assert( !shared || mp.sharedEnable.B, "F32ADD shared memory is disabled" diff --git a/examples/components/balls/f32add/arch/F32AddRow.scala b/examples/prefab/balls/f32add/arch/F32AddRow.scala similarity index 100% rename from examples/components/balls/f32add/arch/F32AddRow.scala rename to examples/prefab/balls/f32add/arch/F32AddRow.scala diff --git a/examples/components/balls/f32add/configs/default.toml b/examples/prefab/balls/f32add/configs/default.toml similarity index 100% rename from examples/components/balls/f32add/configs/default.toml rename to examples/prefab/balls/f32add/configs/default.toml diff --git a/examples/components/balls/f32add/configs/f32add_params.proto b/examples/prefab/balls/f32add/configs/f32add_params.proto similarity index 100% rename from examples/components/balls/f32add/configs/f32add_params.proto rename to examples/prefab/balls/f32add/configs/f32add_params.proto diff --git a/examples/components/balls/f32add/emu/src/add.rs b/examples/prefab/balls/f32add/emu/src/add.rs similarity index 92% rename from examples/components/balls/f32add/emu/src/add.rs rename to examples/prefab/balls/f32add/emu/src/add.rs index 2da22b6bd..cb868b5eb 100644 --- a/examples/components/balls/f32add/emu/src/add.rs +++ b/examples/prefab/balls/f32add/emu/src/add.rs @@ -1,7 +1,8 @@ -use bebop_bemu::thread::instructions::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use bebop_bemu::thread::instructions::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use bebop_bemu::ball::ExecContext; pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "add_f32: reads no old destination; rs1[29] must be clear"); let a = rs1_b0(xs1); let b = rs1_b1(xs1); let c = rs1_b2(xs1); diff --git a/examples/components/balls/f32add/emu/src/dispatch.rs b/examples/prefab/balls/f32add/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/f32add/emu/src/dispatch.rs rename to examples/prefab/balls/f32add/emu/src/dispatch.rs diff --git a/examples/components/balls/f32add/emu/src/lib.rs b/examples/prefab/balls/f32add/emu/src/lib.rs similarity index 100% rename from examples/components/balls/f32add/emu/src/lib.rs rename to examples/prefab/balls/f32add/emu/src/lib.rs diff --git a/examples/prefab/balls/f32add/stack/src/f32addOps.cpp b/examples/prefab/balls/f32add/stack/src/f32addOps.cpp new file mode 100644 index 000000000..958a452a2 --- /dev/null +++ b/examples/prefab/balls/f32add/stack/src/f32addOps.cpp @@ -0,0 +1,68 @@ +//===- f32addOps.cpp - F32AddBall operations ------------------------------===// +// +// The dialect of F32AddBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "f32addOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::f32add_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "f32addDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "f32addOps.cpp.inc" + +void F32AddBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "f32addOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankAddF32Op::getSourceBanks() { + return {getA(), getB()}; +} +Value BankAddF32Op::getDestinationBank() { return getC(); } +bool BankAddF32Op::readsDestination() { return false; } +Operation *BankAddF32Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return AddF32Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows()); +} + +SmallVector AddF32Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t AddF32Op::getDestinationBank() { return getC(); } +LogicalResult AddF32Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), a = getA(), b = getB(), c = getC(); + if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || + a == b || a == c || b == c) + return emitOpError( + "add_f32 requires distinct A, B and C banks and rows within a bank"); + // rs2: group 0 in [4:0]; bit 5 clear reads B. + emitInstruction(builder, loc, buckyball_target::getBuckyballFunct7("F32ADD"), + constantI64(builder, loc, bankFields({a, b}, c, false, rows)), + constantI64(builder, loc, 0)); + return success(); +} diff --git a/examples/prefab/balls/f32add/stack/src/f32addOps.h b/examples/prefab/balls/f32add/stack/src/f32addOps.h new file mode 100644 index 000000000..8a8671774 --- /dev/null +++ b/examples/prefab/balls/f32add/stack/src/f32addOps.h @@ -0,0 +1,20 @@ +//===- f32addOps.h - F32AddBall operations --------------------------------===// + +#ifndef F32ADD_OPS_H +#define F32ADD_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "f32addDialect.h.inc" + +#define GET_OP_CLASSES +#include "f32addOps.h.inc" + +#endif // F32ADD_OPS_H diff --git a/examples/prefab/balls/f32add/stack/src/f32addOps.td b/examples/prefab/balls/f32add/stack/src/f32addOps.td new file mode 100644 index 000000000..e97915505 --- /dev/null +++ b/examples/prefab/balls/f32add/stack/src/f32addOps.td @@ -0,0 +1,44 @@ +//===- f32addOps.td - F32AddBall operations ----------------*- tablegen -*-===// +// +// add_f32: C = A + B over `rows` rows of four f32, RNE with canonical NaN. +// A, B and C are distinct one-group banks. +// +// The Ball reads both addends from its source banks and cannot read the old +// destination: a sum of more terms is a chain of add_f32 into fresh banks. The +// Ball's input group is group 0 and it always reads B. +// +//===----------------------------------------------------------------------===// + +#ifndef F32ADD_OPS +#define F32ADD_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def F32AddBall_Dialect : Dialect { + let name = "f32add_ball"; + let cppNamespace = "::buddy::f32add_ball"; + let summary = "The operations of F32AddBall"; +} + +class F32AddBall_Op traits = []> + : Op])>; + +def BankAddF32Op : F32AddBall_Op<"bank_add_f32", + [DeclareOpInterfaceMethods]> { + let summary = "add_f32 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def AddF32Op : F32AddBall_Op<"add_f32", + [DeclareOpInterfaceMethods]> { + let summary = "add_f32 on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // F32ADD_OPS diff --git a/examples/prefab/balls/f32add/verify/casegen/Cargo.lock b/examples/prefab/balls/f32add/verify/casegen/Cargo.lock new file mode 100644 index 000000000..d776305a8 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "f32add-casegen" +version = "0.1.0" diff --git a/examples/prefab/balls/f32add/verify/casegen/Cargo.toml b/examples/prefab/balls/f32add/verify/casegen/Cargo.toml new file mode 100644 index 000000000..dae3cb329 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "f32add-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["rlib", "cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/prefab/balls/f32add/verify/casegen/src/casegen.rs b/examples/prefab/balls/f32add/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..86ccc9898 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/casegen/src/casegen.rs @@ -0,0 +1,135 @@ +use crate::model; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct F32AddCmd { + pub rows: u32, + pub a_bank: u32, + pub b_bank: u32, + pub c_bank: u32, + pub a_group_base: u32, + pub b_group: u32, + pub c_group: u32, + pub a_cols: u32, + pub group: u32, + pub first: u32, + pub rob_id: u32, +} + +#[derive(Clone, Debug)] +pub struct F32AddCase { + pub cmd: F32AddCmd, + pub a: Vec, + pub b: Vec, + pub c: Vec, +} + +struct Lcg(u64); + +impl Lcg { + fn next(&mut self) -> u32 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (self.0 >> 32) as u32 + } +} + +// (source, accumulator) pairs at the IEEE edges: signed zeros, infinities, +// quiet and signalling NaNs, overflow, subnormal results and ties. +const EDGES: [(u32, u32); 20] = [ + (0x0000_0000, 0x8000_0000), // +0 + -0 = +0 + (0x8000_0000, 0x8000_0000), // -0 + -0 = -0 + (0x7f80_0000, 0xff80_0000), // inf - inf = NaN + (0x7f80_0000, 0x3f80_0000), // inf + 1 + (0xff80_0000, 0xff80_0000), // -inf + -inf + (0x7fc0_1234, 0x3f80_0000), // quiet NaN payload + (0x3f80_0000, 0x7f80_0001), // signalling NaN + (0xffff_ffff, 0x0000_0000), // negative NaN + (0x7f7f_ffff, 0x7f7f_ffff), // overflow to inf + (0xff7f_ffff, 0xff7f_fffe), // overflow to -inf + (0x0000_0001, 0x0000_0001), // subnormal sum + (0x0000_0001, 0x8000_0001), // exact cancellation to +0 + (0x007f_ffff, 0x0000_0001), // subnormal to the smallest normal + (0x0080_0000, 0x8000_0001), // normal to subnormal + (0x3f80_0000, 0x3380_0000), // 1 + 2^-24: tie, stays even + (0x3f80_0001, 0x3380_0000), // tie rounds up to even + (0x3f80_0000, 0x33c0_0000), // above the tie + (0x3f80_0000, 0xbf80_0000), // 1 - 1 = +0 + (0x4b80_0000, 0xbf80_0000), // 2^24 - 1, exact + (0x7f00_0000, 0xfeff_ffff), // large cancellation +]; + +fn lane_pair(rng: &mut Lcg, index: usize, edges: bool) -> (u32, u32) { + if edges { + return EDGES[index % EDGES.len()]; + } + match index % 4 { + // Nearby exponents exercise alignment and rounding. + 0 | 1 => { + let a = rng.next(); + let b = (a & 0xff80_0000) ^ (rng.next() & 0x807f_ffff) ^ (rng.next() & 0x0380_0000); + (a, b) + } + _ => (rng.next(), rng.next()), + } +} + +fn pack(lanes: &[u32]) -> Vec { + lanes + .chunks(4) + .map(|chunk| { + let mut word = 0u128; + for (lane, value) in chunk.iter().enumerate() { + word |= u128::from(*value) << (32 * lane); + } + word + }) + .collect() +} + +pub fn gen_case(index: u32) -> F32AddCase { + let entries = model::bank_entries(); + // (rob_id, banks, rows, a_cols, group, first, group bases, edge values) + let (rob_id, banks, rows, a_cols, group, first, bases, edges) = match index { + 0 => (1, [0, 1, 2], 4, 1, 0, 0, [0, 0, 0], false), + 1 => (2, [3, 1, 0], 7, 1, 0, 1, [0, 0, 0], false), + 2 => (3, [2, 0, 3], 16, 3, 2, 0, [1, 2, 3], true), + 3 => (4, [33, 600, 1023], entries, 1, 0, 0, [0, 1, 0], false), + 4 => (5, [5, 6, 7], 9, 2, 1, 1, [2, 0, 1], true), + _ => panic!("f32add: unsupported case {index}"), + }; + model::validate(entries, banks[0], banks[1], banks[2], rows, group, first, a_cols); + let mut rng = Lcg(0xf32a_dd00 + u64::from(index)); + let lanes = rows as usize * 4; + let mut a_lanes = Vec::with_capacity(lanes); + let mut b_lanes = Vec::with_capacity(lanes); + for i in 0..lanes { + let (s, acc) = lane_pair(&mut rng, i, edges); + a_lanes.push(s); + b_lanes.push(acc); + } + let a = pack(&a_lanes); + let b = pack(&b_lanes); + let c = a + .iter() + .zip(&b) + .map(|(&s, &acc)| model::add_word(s, acc, first == 1)) + .collect(); + F32AddCase { + cmd: F32AddCmd { + rows, + a_bank: banks[0], + b_bank: banks[1], + c_bank: banks[2], + a_group_base: bases[0], + b_group: bases[1], + c_group: bases[2], + a_cols, + group, + first, + rob_id, + }, + a, + b, + c, + } +} diff --git a/examples/prefab/balls/f32add/verify/casegen/src/lib.rs b/examples/prefab/balls/f32add/verify/casegen/src/lib.rs new file mode 100644 index 000000000..79c832f35 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/casegen/src/lib.rs @@ -0,0 +1,71 @@ +mod casegen; +mod model; + +use std::cell::RefCell; + +use casegen::{F32AddCase, F32AddCmd}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn f32add_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("f32add_case: no case loaded; call f32add_case_load first"), + }) +} + +fn idx(index: u32, case: &F32AddCase, what: &str) -> usize { + if index >= case.cmd.rows { + panic!("f32add_case_{what}: index {index} out of range {}", case.cmd.rows); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn f32add_case_cmd(out_ptr: *mut F32AddCmd) { + if out_ptr.is_null() { + panic!("f32add_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +#[no_mangle] +pub extern "C" fn f32add_case_a_word_lo(index: u32) -> u64 { + current_case(|case| case.a[idx(index, case, "a_word_lo")] as u64) +} + +#[no_mangle] +pub extern "C" fn f32add_case_a_word_hi(index: u32) -> u64 { + current_case(|case| (case.a[idx(index, case, "a_word_hi")] >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn f32add_case_b_word_lo(index: u32) -> u64 { + current_case(|case| case.b[idx(index, case, "b_word_lo")] as u64) +} + +#[no_mangle] +pub extern "C" fn f32add_case_b_word_hi(index: u32) -> u64 { + current_case(|case| (case.b[idx(index, case, "b_word_hi")] >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn f32add_case_c_word_lo(index: u32) -> u64 { + current_case(|case| case.c[idx(index, case, "c_word_lo")] as u64) +} + +#[no_mangle] +pub extern "C" fn f32add_case_c_word_hi(index: u32) -> u64 { + current_case(|case| (case.c[idx(index, case, "c_word_hi")] >> 64) as u64) +} diff --git a/examples/prefab/balls/f32add/verify/casegen/src/model.rs b/examples/prefab/balls/f32add/verify/casegen/src/model.rs new file mode 100644 index 000000000..3a77cde64 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/casegen/src/model.rs @@ -0,0 +1,50 @@ +// F32AddBall contract: C = A + B over `rows` rows of four IEEE FP32 lanes, +// round to nearest even, every NaN result is the canonical 0x7fc00000. +// rs2[4:0] selects the source group of A; rs2[5] (first) replaces B with +0 +// and B is not read. + +pub const CANONICAL_NAN: u32 = 0x7fc0_0000; + +pub fn bank_entries() -> u32 { + let path = std::env::var("BB_VERIFY_CONFIG").expect("BB_VERIFY_CONFIG is required"); + let text = std::fs::read_to_string(path).expect("failed to read BB_VERIFY_CONFIG"); + text.lines() + .find_map(|line| line.strip_prefix("bank_entries=")) + .expect("bank_entries missing from BB_VERIFY_CONFIG") + .parse() + .expect("invalid bank_entries in BB_VERIFY_CONFIG") +} + +pub fn validate(entries: u32, a: u32, b: u32, c: u32, rows: u32, group: u32, first: u32, cols: u32) { + if a == b || a == c || b == c { + panic!("f32add: banks must differ"); + } + if group > 31 || first > 1 { + panic!("f32add: rs2 fields out of range"); + } + if group >= cols { + panic!("f32add: source group must exist"); + } + if rows == 0 || rows > entries { + panic!("f32add: row span exceeds a bank"); + } +} + +pub fn add_lane(source: u32, accumulator: u32) -> u32 { + let sum = f32::from_bits(accumulator) + f32::from_bits(source); + if sum.is_nan() { + CANONICAL_NAN + } else { + sum.to_bits() + } +} + +pub fn add_word(source: u128, accumulator: u128, first: bool) -> u128 { + let mut out = 0u128; + for lane in 0..4 { + let s = (source >> (32 * lane)) as u32; + let acc = if first { 0 } else { (accumulator >> (32 * lane)) as u32 }; + out |= u128::from(add_lane(s, acc)) << (32 * lane); + } + out +} diff --git a/examples/prefab/balls/f32add/verify/filelists/f32add_ball.f b/examples/prefab/balls/f32add/verify/filelists/f32add_ball.f new file mode 100644 index 000000000..74feb5a66 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/filelists/f32add_ball.f @@ -0,0 +1,29 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+src ++incdir+src/pkg ++incdir+src/common ++incdir+src/seq ++incdir+src/cov ++incdir+src/env ++incdir+src/tests ++incdir+@RTL@ + +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv + +@RTL@/RoundAnyRawFNToRecFN_ie8_is26_oe8_os24.sv +@RTL@/RoundRawFNToRecFN_e8_s24.sv +@RTL@/AddRawFN.sv +@RTL@/AddRecFN.sv +@RTL@/F32AddRow.sv +@RTL@/F32AddBall.sv + +src/pkg/f32add_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/f32add/verify/src/common/f32add_defs.svh b/examples/prefab/balls/f32add/verify/src/common/f32add_defs.svh new file mode 100644 index 000000000..7630a9a52 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/common/f32add_defs.svh @@ -0,0 +1,30 @@ +typedef struct { + int unsigned rows; + int unsigned a_bank; + int unsigned b_bank; + int unsigned c_bank; + int unsigned a_group_base; + int unsigned b_group; + int unsigned c_group; + int unsigned a_cols; + int unsigned group; + int unsigned first; + int unsigned rob_id; +} f32add_cmd_dpi_t; + +import "DPI-C" function int f32add_case_load(input int unsigned index); +import "DPI-C" function void f32add_case_cmd(output f32add_cmd_dpi_t cmd); +import "DPI-C" function longint unsigned f32add_case_a_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned f32add_case_a_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned f32add_case_b_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned f32add_case_b_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned f32add_case_c_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned f32add_case_c_word_hi(input int unsigned word_index); + +`ifndef F32ADD_FUNCT7 +`error "F32ADD_FUNCT7 must be provided by the selected ballISA" +`endif + +localparam int F32ADD_THREAD_FUNCT7 = `F32ADD_FUNCT7; +localparam int F32ADD_NUM_CASES = 5; +localparam int F32ADD_TIMEOUT_CYCLES = 50000; diff --git a/examples/prefab/balls/f32add/verify/src/common/f32add_items.svh b/examples/prefab/balls/f32add/verify/src/common/f32add_items.svh new file mode 100644 index 000000000..fe706760b --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/common/f32add_items.svh @@ -0,0 +1,63 @@ +class f32add_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(f32add_cmd_item) + + bit first; + bit [4:0] group; + bit [127:0] a_words[]; + bit [127:0] b_words[]; + bit [127:0] c_words[]; + + function new(string name = "f32add_cmd_item"); + super.new(name); + endfunction + + function void load_rust_case(int unsigned index); + f32add_cmd_dpi_t cmd; + int unsigned rc; + + rc = f32add_case_load(index); + if (rc != 0) + `uvm_fatal("CASE", $sformatf("f32add_case_load returned %0d for index %0d", rc, index)) + f32add_case_cmd(cmd); + + // bb_f32add: banks a, b, c from rs1, rows in iter; rs2[4:0] is the source + // group of a and rs2[5] starts from +0 instead of reading b. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = F32ADD_THREAD_FUNCT7[6:0]; + rs2 = {cmd.first[0], cmd.group[4:0]}; + iter = cmd.rows; + op1_bank = cmd.a_bank; + op2_bank = cmd.b_bank; + wr_bank = cmd.c_bank; + op1_group = cmd.a_group_base; + op2_group = cmd.b_group; + wr_group = cmd.c_group; + op1_col = cmd.a_cols; + op2_col = 1; + wr_col = 1; + wr_read_bank = '0; + wr_arch_bank = cmd.c_bank; + first = cmd.first[0]; + group = cmd.group[4:0]; + + a_words = new[cmd.rows]; + b_words = new[cmd.rows]; + c_words = new[cmd.rows]; + for (int i = 0; i < cmd.rows; i++) begin + a_words[i] = {f32add_case_a_word_hi(i), f32add_case_a_word_lo(i)}; + b_words[i] = {f32add_case_b_word_hi(i), f32add_case_b_word_lo(i)}; + c_words[i] = {f32add_case_c_word_hi(i), f32add_case_c_word_lo(i)}; + end + endfunction + + function void do_copy(uvm_object rhs); + f32add_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not f32add_cmd_item") + first = rhs_.first; + group = rhs_.group; + a_words = rhs_.a_words; + b_words = rhs_.b_words; + c_words = rhs_.c_words; + endfunction +endclass diff --git a/examples/prefab/balls/f32add/verify/src/cov/f32add_cov.svh b/examples/prefab/balls/f32add/verify/src/cov/f32add_cov.svh new file mode 100644 index 000000000..d35e099f2 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/cov/f32add_cov.svh @@ -0,0 +1,42 @@ +class f32add_cov extends uvm_component; + `uvm_component_utils(f32add_cov) + + uvm_analysis_imp_cmd #(bb_blink_cmd_item, f32add_cov) cmd_imp; + + bit cur_first; + int unsigned cur_group; + int unsigned cur_rows; + + covergroup cmd_cg; + coverpoint cur_first {bins accumulate = {0}; bins first = {1};} + coverpoint cur_group {bins zero = {0}; bins later = {[1 : 31]};} + coverpoint cur_rows {bins one_group = {[1 : 16]}; bins deep = {[17 : $]};} + cross cur_first, cur_group; + endgroup + + int cmd_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + cmd_imp = new("cmd_imp", this); + cmd_cg = new(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + if (item.funct != F32ADD_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) + cur_first = item.rs2[5]; + cur_group = item.rs2[4:0]; + cur_rows = item.iter; + cmd_count++; + cmd_cg.sample(); + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("f32add_cov %0.2f%%", cmd_cg.get_coverage())) + endfunction +endclass diff --git a/examples/prefab/balls/f32add/verify/src/env/f32add_env.svh b/examples/prefab/balls/f32add/verify/src/env/f32add_env.svh new file mode 100644 index 000000000..cb86fb3c0 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/env/f32add_env.svh @@ -0,0 +1,27 @@ +class f32add_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(f32add_env) + + f32add_scoreboard scb; + f32add_cov icov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scb = f32add_scoreboard::type_id::create("scb", this); + icov = f32add_cov::type_id::create("icov", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + scb.mem_model = mem_model; + cmd_agent.stim_ap.connect(scb.stim_imp); + cmd_agent.cmd_ap.connect(scb.cmd_imp); + cmd_agent.cmd_ap.connect(icov.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction +endclass diff --git a/examples/prefab/balls/f32add/verify/src/env/f32add_scoreboard.svh b/examples/prefab/balls/f32add/verify/src/env/f32add_scoreboard.svh new file mode 100644 index 000000000..0c4c828cc --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/env/f32add_scoreboard.svh @@ -0,0 +1,171 @@ +class f32add_scoreboard extends uvm_scoreboard; + `uvm_component_utils(f32add_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, f32add_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, f32add_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, f32add_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, f32add_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, f32add_scoreboard) resp_imp; + + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + + f32add_cmd_item stim_q[$]; + int unsigned cmd_count; + int unsigned read_count[2]; + int unsigned write_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + f32add_cmd_item got; + f32add_cmd_item clone; + + if (stim_q.size() != 0) `uvm_fatal("SCB", "single outstanding command supported") + if (!$cast(got, item)) `uvm_fatal("SCB", "stim item is not f32add_cmd_item") + if (!$cast(clone, got.clone())) `uvm_fatal("SCB", "failed to clone stimulus item") + stim_q.push_back(clone); + + if (mem_model == null) `uvm_fatal("SCB", "mem_model handle not set") + mem_model.clear_mem(); + for (int i = 0; i < clone.iter; i++) begin + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group + clone.group), i, + clone.a_words[i]); + // With first set B is not an operand: leaving it unloaded makes any read fail. + if (!clone.first) + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group), i, clone.b_words[i]); + end + mem_model.arm(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + bb_blink_cmd_item want; + f32add_cmd_item stim; + stim = current_stim("CMD"); + // Compare as plain BLink commands; the comparer rejects differing types. + want = bb_blink_cmd_item::type_id::create("want"); + want.copy(stim); + verification_pkg::evidence::checks++; + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) + cmd_count++; + endfunction + + function void write_read(bb_blink_read_item item); + f32add_cmd_item stim; + int unsigned expect_bank; + int unsigned expect_group; + + stim = current_stim("READ"); + if (item.port != 0 && item.port != 1) + `uvm_fatal("READ", $sformatf("F32AddBall has two read ports, got port %0d", item.port)) + if (item.port == 1 && stim.first) `uvm_fatal("READ", "B read although first is set") + if (read_count[item.port] >= stim.iter) + `uvm_fatal("READ", $sformatf("extra read on port %0d", item.port)) + if (read_count[item.port] != write_count) + `uvm_fatal("READ", $sformatf( + "port %0d read %0d before write %0d", item.port, read_count[item.port], write_count + )) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.port == 0) begin + expect_bank = stim.op1_bank; + expect_group = stim.op1_group + stim.group; + end else begin + expect_bank = stim.op2_bank; + expect_group = stim.op2_group; + end + if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, expect_bank)) + if (item.group_id !== expect_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, expect_group)) + if (item.addr !== read_count[item.port]) + `uvm_fatal("READ", $sformatf( + "addr mismatch: got %0d exp %0d", item.addr, read_count[item.port])) + read_count[item.port]++; + endfunction + + function void write_write(bb_blink_write_item item); + f32add_cmd_item stim; + + stim = current_stim("WRITE"); + if (item.port != 0) + `uvm_fatal("WRITE", $sformatf("F32AddBall has one write port, got %0d", item.port)) + if (write_count >= stim.iter) `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) + if (read_count[0] != write_count + 1 || (!stim.first && read_count[1] != write_count + 1)) + `uvm_fatal("WRITE", $sformatf("write %0d before its operands", write_count)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.group_id !== stim.wr_group) + `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + )) + if (item.mask !== 16'hFFFF) + `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) + if (item.addr !== write_count) + `uvm_fatal("WRITE", $sformatf("addr mismatch: got %0d exp %0d", item.addr, write_count)) + verification_pkg::evidence::checks++; + if (item.data !== stim.c_words[write_count]) + `uvm_fatal("SCB", $sformatf( + "data mismatch at row %0d: got 0x%032h exp 0x%032h", + item.addr, + item.data, + stim.c_words[write_count] + )) + write_count++; + endfunction + + function void write_resp(bb_blink_resp_item item); + f32add_cmd_item stim; + stim = current_stim("RESP"); + if (write_count != stim.iter) + `uvm_fatal("RESP", $sformatf("response after %0d of %0d writes", write_count, stim.iter)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + resp_count++; + endfunction + + function f32add_cmd_item current_stim(string tag); + if (stim_q.size() == 0) `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + return stim_q[0]; + endfunction + + function bit done(); + if (stim_q.size() == 0) return 1'b0; + return cmd_count == 1 && read_count[0] == stim_q[0].iter && + read_count[1] == (stim_q[0].first ? 0 : stim_q[0].iter) && + write_count == stim_q[0].iter && resp_count == 1; + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + read_count[0] = 0; + read_count[1] = 0; + write_count = 0; + resp_count = 0; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) + `uvm_fatal("SCB", $sformatf( + "incomplete: cmds=%0d reads=%0d,%0d writes=%0d resp=%0d", + cmd_count, + read_count[0], + read_count[1], + write_count, + resp_count + )) + stim_q.delete(); + endfunction +endclass diff --git a/examples/prefab/balls/f32add/verify/src/pkg/f32add_pkg.sv b/examples/prefab/balls/f32add/verify/src/pkg/f32add_pkg.sv new file mode 100644 index 000000000..f77a1dd26 --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/pkg/f32add_pkg.sv @@ -0,0 +1,13 @@ +package f32add_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + + `include "common/f32add_defs.svh" + `include "common/f32add_items.svh" + `include "seq/f32add_sequences.svh" + `include "cov/f32add_cov.svh" + `include "env/f32add_scoreboard.svh" + `include "env/f32add_env.svh" + `include "tests/f32add_test.svh" +endpackage diff --git a/examples/prefab/balls/f32add/verify/src/seq/f32add_sequences.svh b/examples/prefab/balls/f32add/verify/src/seq/f32add_sequences.svh new file mode 100644 index 000000000..cc1e21e9a --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/seq/f32add_sequences.svh @@ -0,0 +1,17 @@ +class f32add_basic_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(f32add_basic_seq) + + int unsigned case_index; + + function new(string name = "f32add_basic_seq"); + super.new(name); + endfunction + + task body(); + f32add_cmd_item item; + item = f32add_cmd_item::type_id::create("item"); + start_item(item); + item.load_rust_case(case_index); + finish_item(item); + endtask +endclass diff --git a/examples/prefab/balls/f32add/verify/src/tb_top.sv b/examples/prefab/balls/f32add/verify/src/tb_top.sv new file mode 100644 index 000000000..bca5c121c --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/tb_top.sv @@ -0,0 +1,89 @@ +module tb_top; + import uvm_pkg::*; + import f32add_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + F32AddBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("f32add_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/f32add/verify/src/tests/f32add_test.svh b/examples/prefab/balls/f32add/verify/src/tests/f32add_test.svh new file mode 100644 index 000000000..58a00a98c --- /dev/null +++ b/examples/prefab/balls/f32add/verify/src/tests/f32add_test.svh @@ -0,0 +1,49 @@ +class f32add_ball_test extends uvm_test; + `uvm_component_utils(f32add_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + f32add_env env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + env = f32add_env::type_id::create("env", this); + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + for (int unsigned index = 0; index < F32ADD_NUM_CASES; index++) run_case(index); + phase.drop_objection(this); + endtask + + task run_case(int unsigned index); + f32add_basic_seq seq; + int cycles = 0; + apply_reset(); + env.scb.reset_counters(); + seq = f32add_basic_seq::type_id::create("seq"); + seq.case_index = index; + seq.start(env.cmd_agent.seqr); + while (!env.scb.done()) begin + @(posedge vif.clock); + cycles++; + if (cycles > F32ADD_TIMEOUT_CYCLES) + `uvm_fatal("TIMEOUT", $sformatf("F32AddBall case %0d timeout", index)) + end + `uvm_info("F32ADD", $sformatf("F32AddBall case %0d passed in %0d cycles", index, cycles), + UVM_LOW) + endtask + + task apply_reset(); + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/components/balls/f32add/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/f32add/workloads/ctests/CMakeLists.txt similarity index 100% rename from examples/components/balls/f32add/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/f32add/workloads/ctests/CMakeLists.txt diff --git a/examples/components/balls/f32add/workloads/ctests/f32add_groups.c b/examples/prefab/balls/f32add/workloads/ctests/f32add_groups.c similarity index 95% rename from examples/components/balls/f32add/workloads/ctests/f32add_groups.c rename to examples/prefab/balls/f32add/workloads/ctests/f32add_groups.c index 7c19fd1ce..baeee1be2 100644 --- a/examples/components/balls/f32add/workloads/ctests/f32add_groups.c +++ b/examples/prefab/balls/f32add/workloads/ctests/f32add_groups.c @@ -1,6 +1,5 @@ #include #include -#include #include #include static const uint32_t inputs[3][8] __attribute__((aligned(64))) = { @@ -25,7 +24,7 @@ static uint32_t add(uint32_t a, uint32_t b) { return result.bits; } int main(void) { - unsigned source = BB_SHARED_BANK_BASE, accumulator = source + 1, + unsigned source = THREAD_FRONTEND_SHARED_VBANK_BASE, accumulator = source + 1, target = source + 2; bb_mem_alloc(source, 1, 3); bb_mem_alloc(accumulator, 1, 1); diff --git a/examples/components/balls/f32add/workloads/isa/f32add.h b/examples/prefab/balls/f32add/workloads/isa/f32add.h similarity index 71% rename from examples/components/balls/f32add/workloads/isa/f32add.h rename to examples/prefab/balls/f32add/workloads/isa/f32add.h index ccf3418f8..988b0caac 100644 --- a/examples/components/balls/f32add/workloads/isa/f32add.h +++ b/examples/prefab/balls/f32add/workloads/isa/f32add.h @@ -9,8 +9,8 @@ static inline void bb_f32add(uint32_t a, uint32_t b, uint32_t c, uint32_t rows, uint32_t group, uint32_t first) { if (a > 1023 || b > 1023 || c > 1023 || !rows || group > 31 || first > 1) __builtin_trap(); - BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(a) | BB_BANK1(b) | BB_BANK2(c) | - BB_ITER(rows), - group | (uint64_t)first << 5, BB_FUNC7(F32ADD)); + BUCKYBALL_INSTRUCTION_R_R( + BB_BANK0(a) | BB_BANK1(b) | BB_BANK2(c) | BB_ITER(rows), + group | (uint64_t)first << 5, FUNCT7(f32add, F32ADD)); } #endif diff --git a/examples/components/balls/f32add/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/f32add/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/f32add/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/f32add/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/f32add/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/f32add/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/components/balls/f32add/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/f32add/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir b/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir similarity index 78% rename from examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir rename to examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir index dc95ca6e2..4ebe35884 100644 --- a/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir +++ b/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_groups.mlir @@ -10,7 +10,7 @@ // then 33 = D + 34. func.func @reduce_groups() -> i64 attributes {llvm.emit_c_interface} { %output = arith.constant 33 : i64 - buckyball.custom "add_f32" <{sources = array, destination = 34 : i64}> {rows = 2 : i64} - buckyball.custom "add_f32" <{sources = array, destination = 33 : i64}> {rows = 2 : i64} + f32add_ball.add_f32 <{a = 32 : i64, b = 33 : i64, c = 34 : i64, rows = 2 : i64}> + f32add_ball.add_f32 <{a = 35 : i64, b = 34 : i64, c = 33 : i64, rows = 2 : i64}> return %output : i64 } diff --git a/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp b/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp similarity index 96% rename from examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp rename to examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp index bb04bdffb..17161e216 100644 --- a/examples/components/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp +++ b/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_groups_main.cpp @@ -1,10 +1,9 @@ #include #include #include -#include extern "C" int64_t _mlir_ciface_reduce_groups(); -static_assert(BB_SHARED_BANK_BASE == 32); +static_assert(THREAD_FRONTEND_SHARED_VBANK_BASE == 32); static int checkF32Add() { alignas(64) static const uint32_t input[3][8] = { diff --git a/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir b/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir new file mode 100644 index 000000000..9d266db88 --- /dev/null +++ b/examples/prefab/balls/f32add/workloads/mlir_tests/ball/f32add_invalid.mlir @@ -0,0 +1,18 @@ +// RUN: buddy-opt %s -split-input-file -lower-buckyball -verify-diagnostics +func.func @zero_rows() { + // expected-error @+1 {{add_f32 requires distinct A, B and C banks and rows within a bank}} + f32add_ball.add_f32 <{a = 32 : i64, b = 33 : i64, c = 34 : i64, rows = 0 : i64}> + return +} +// ----- +func.func @wide_rows() { + // expected-error @+1 {{add_f32 requires distinct A, B and C banks and rows within a bank}} + f32add_ball.add_f32 <{a = 32 : i64, b = 33 : i64, c = 34 : i64, rows = 17179869184 : i64}> + return +} +// ----- +func.func @equal_banks() { + // expected-error @+1 {{add_f32 requires distinct A, B and C banks and rows within a bank}} + f32add_ball.add_f32 <{a = 32 : i64, b = 33 : i64, c = 32 : i64, rows = 1 : i64}> + return +} diff --git a/examples/components/balls/f32add/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/f32add/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/components/balls/f32add/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/f32add/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir b/examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir similarity index 88% rename from examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir rename to examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir index 5844f9e4b..2a2a662ed 100644 --- a/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir +++ b/examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_groups.mlir @@ -11,11 +11,11 @@ func.func @reduce_groups(%first: memref<2x4xi32>, %second: memref<2x4xi32>, %la = buckyball.bank_mvin %first, %a <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> %lb = buckyball.bank_mvin %second, %b <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> %poisoned = buckyball.bank_mvin %poison, %sum <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> - %partial = buckyball.bank_custom "add_f32" (%la, %lb) -> %poisoned {rows = 2 : i64} + %partial = f32add_ball.bank_add_f32 %la, %lb -> %poisoned <{rows = 2 : i64}> buckyball.bank_release %la %d = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %ld = buckyball.bank_mvin %last, %d <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> - %total = buckyball.bank_custom "add_f32" (%ld, %partial) -> %lb {rows = 2 : i64} + %total = f32add_ball.bank_add_f32 %ld, %partial -> %lb <{rows = 2 : i64}> %stored = buckyball.bank_mvout %output, %total <{depth = 2 : i64, stride = 1 : i64}> : memref<2x4xi32> buckyball.bank_release %ld buckyball.bank_release %partial diff --git a/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp b/examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp similarity index 100% rename from examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp rename to examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_groups_main.cpp diff --git a/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir b/examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir similarity index 52% rename from examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir rename to examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir index acf1ba50d..d301fbb93 100644 --- a/examples/components/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir +++ b/examples/prefab/balls/f32add/workloads/mlir_tests/bank/f32add_invalid.mlir @@ -3,8 +3,8 @@ func.func @same_bank(%input: memref<1x4xi32>) { %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %la = buckyball.bank_mvin %input, %a <{depth = 1 : i64, stride = 1 : i64}> : memref<1x4xi32> - // expected-error @+1 {{add_f32 requires distinct A, B and C banks, rows within a bank and no read_destination}} - %out = buckyball.bank_custom "add_f32" (%la, %b) -> %b {rows = 1 : i64} + // expected-error @+1 {{add_f32 requires distinct A, B and C banks and rows within a bank}} + %out = f32add_ball.bank_add_f32 %la, %b -> %b <{rows = 1 : i64}> buckyball.bank_release %la buckyball.bank_release %out return @@ -15,21 +15,8 @@ func.func @zero_rows(%input: memref<1x4xi32>) { %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %c = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %la = buckyball.bank_mvin %input, %a <{depth = 1 : i64, stride = 1 : i64}> : memref<1x4xi32> - // expected-error @+1 {{add_f32 requires distinct A, B and C banks, rows within a bank and no read_destination}} - %out = buckyball.bank_custom "add_f32" (%la, %b) -> %c {rows = 0 : i64} - buckyball.bank_release %la - buckyball.bank_release %b - buckyball.bank_release %out - return -} -// ----- -func.func @read_destination(%input: memref<1x4xi32>) { - %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> - %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> - %c = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> - %la = buckyball.bank_mvin %input, %a <{depth = 1 : i64, stride = 1 : i64}> : memref<1x4xi32> - // expected-error @+1 {{add_f32 requires distinct A, B and C banks, rows within a bank and no read_destination}} - %out = buckyball.bank_custom "add_f32" (%la, %b) -> %c <{read_destination}> {rows = 1 : i64} + // expected-error @+1 {{add_f32 requires distinct A, B and C banks and rows within a bank}} + %out = f32add_ball.bank_add_f32 %la, %b -> %c <{rows = 0 : i64}> buckyball.bank_release %la buckyball.bank_release %b buckyball.bank_release %out diff --git a/examples/components/balls/gemmini/arch/GemminiBall.scala b/examples/prefab/balls/gemmini/arch/GemminiBall.scala similarity index 100% rename from examples/components/balls/gemmini/arch/GemminiBall.scala rename to examples/prefab/balls/gemmini/arch/GemminiBall.scala diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrl.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrl.scala similarity index 100% rename from examples/components/balls/gemmini/arch/GemminiExCtrl.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrl.scala diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrlAccumulateState.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlAccumulateState.scala similarity index 100% rename from examples/components/balls/gemmini/arch/GemminiExCtrlAccumulateState.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrlAccumulateState.scala diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrlCmdStates.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlCmdStates.scala similarity index 95% rename from examples/components/balls/gemmini/arch/GemminiExCtrlCmdStates.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrlCmdStates.scala index 11729973a..75528da5a 100644 --- a/examples/components/balls/gemmini/arch/GemminiExCtrlCmdStates.scala +++ b/examples/prefab/balls/gemmini/arch/GemminiExCtrlCmdStates.scala @@ -66,8 +66,10 @@ trait GemminiExCtrlCmdStates { this: GemminiExCtrl => }.elsewhen( sub_cmd === GemminiSubCmd.COMPUTE_PRELOADED || sub_cmd === GemminiSubCmd.COMPUTE_ACCUMULATED ) { + // A zeroed op2 (rs2[4]) reads no bank, so its group is not checked. assert( - io.cmdReq.bits.rs2(41, 36) < io.cmdReq.bits.op1_col && io.cmdReq.bits.rs2(47, 42) < io.cmdReq.bits.op2_col, + io.cmdReq.bits.rs2(41, 36) < io.cmdReq.bits.op1_col && + (io.cmdReq.bits.rs2(4) || io.cmdReq.bits.rs2(47, 42) < io.cmdReq.bits.op2_col), "Gemmini compute operand group outside the bank" ) read_row_cnt := 0.U diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrlComputeFeedState.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlComputeFeedState.scala similarity index 100% rename from examples/components/balls/gemmini/arch/GemminiExCtrlComputeFeedState.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrlComputeFeedState.scala diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrlComputeReadState.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlComputeReadState.scala similarity index 100% rename from examples/components/balls/gemmini/arch/GemminiExCtrlComputeReadState.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrlComputeReadState.scala diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrlDefaults.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlDefaults.scala similarity index 100% rename from examples/components/balls/gemmini/arch/GemminiExCtrlDefaults.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrlDefaults.scala diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrlDefs.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlDefs.scala similarity index 100% rename from examples/components/balls/gemmini/arch/GemminiExCtrlDefs.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrlDefs.scala diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrlFsm.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlFsm.scala similarity index 100% rename from examples/components/balls/gemmini/arch/GemminiExCtrlFsm.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrlFsm.scala diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrlPreloadStates.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlPreloadStates.scala similarity index 100% rename from examples/components/balls/gemmini/arch/GemminiExCtrlPreloadStates.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrlPreloadStates.scala diff --git a/examples/components/balls/gemmini/arch/GemminiExCtrlStoreOps.scala b/examples/prefab/balls/gemmini/arch/GemminiExCtrlStoreOps.scala similarity index 100% rename from examples/components/balls/gemmini/arch/GemminiExCtrlStoreOps.scala rename to examples/prefab/balls/gemmini/arch/GemminiExCtrlStoreOps.scala diff --git a/examples/components/balls/gemmini/arch/LICENSE.gemmini b/examples/prefab/balls/gemmini/arch/LICENSE.gemmini similarity index 100% rename from examples/components/balls/gemmini/arch/LICENSE.gemmini rename to examples/prefab/balls/gemmini/arch/LICENSE.gemmini diff --git a/examples/components/balls/gemmini/arch/Loop.scala b/examples/prefab/balls/gemmini/arch/Loop.scala similarity index 100% rename from examples/components/balls/gemmini/arch/Loop.scala rename to examples/prefab/balls/gemmini/arch/Loop.scala diff --git a/examples/components/balls/gemmini/arch/LoopCmd.scala b/examples/prefab/balls/gemmini/arch/LoopCmd.scala similarity index 100% rename from examples/components/balls/gemmini/arch/LoopCmd.scala rename to examples/prefab/balls/gemmini/arch/LoopCmd.scala diff --git a/examples/components/balls/gemmini/arch/LoopConvAddrGen.scala b/examples/prefab/balls/gemmini/arch/LoopConvAddrGen.scala similarity index 100% rename from examples/components/balls/gemmini/arch/LoopConvAddrGen.scala rename to examples/prefab/balls/gemmini/arch/LoopConvAddrGen.scala diff --git a/examples/components/balls/gemmini/arch/LoopConvUnroller.scala b/examples/prefab/balls/gemmini/arch/LoopConvUnroller.scala similarity index 100% rename from examples/components/balls/gemmini/arch/LoopConvUnroller.scala rename to examples/prefab/balls/gemmini/arch/LoopConvUnroller.scala diff --git a/examples/components/balls/gemmini/arch/LoopMatmulUnroller.scala b/examples/prefab/balls/gemmini/arch/LoopMatmulUnroller.scala similarity index 100% rename from examples/components/balls/gemmini/arch/LoopMatmulUnroller.scala rename to examples/prefab/balls/gemmini/arch/LoopMatmulUnroller.scala diff --git a/examples/components/balls/gemmini/configs/GemminiBallParam.scala b/examples/prefab/balls/gemmini/configs/GemminiBallParam.scala similarity index 100% rename from examples/components/balls/gemmini/configs/GemminiBallParam.scala rename to examples/prefab/balls/gemmini/configs/GemminiBallParam.scala diff --git a/examples/components/balls/gemmini/configs/default.toml b/examples/prefab/balls/gemmini/configs/default.toml similarity index 100% rename from examples/components/balls/gemmini/configs/default.toml rename to examples/prefab/balls/gemmini/configs/default.toml diff --git a/examples/components/balls/gemmini/configs/gemmini_params.proto b/examples/prefab/balls/gemmini/configs/gemmini_params.proto similarity index 100% rename from examples/components/balls/gemmini/configs/gemmini_params.proto rename to examples/prefab/balls/gemmini/configs/gemmini_params.proto diff --git a/examples/components/balls/gemmini/emu/src/02_gemmini_config.rs b/examples/prefab/balls/gemmini/emu/src/02_gemmini_config.rs similarity index 100% rename from examples/components/balls/gemmini/emu/src/02_gemmini_config.rs rename to examples/prefab/balls/gemmini/emu/src/02_gemmini_config.rs diff --git a/examples/components/balls/gemmini/emu/src/03_gemmini_flush.rs b/examples/prefab/balls/gemmini/emu/src/03_gemmini_flush.rs similarity index 100% rename from examples/components/balls/gemmini/emu/src/03_gemmini_flush.rs rename to examples/prefab/balls/gemmini/emu/src/03_gemmini_flush.rs diff --git a/examples/components/balls/gemmini/emu/src/21_gemmini_preload.rs b/examples/prefab/balls/gemmini/emu/src/21_gemmini_preload.rs similarity index 89% rename from examples/components/balls/gemmini/emu/src/21_gemmini_preload.rs rename to examples/prefab/balls/gemmini/emu/src/21_gemmini_preload.rs index 47769bfd0..74bf8015e 100644 --- a/examples/components/balls/gemmini/emu/src/21_gemmini_preload.rs +++ b/examples/prefab/balls/gemmini/emu/src/21_gemmini_preload.rs @@ -1,5 +1,5 @@ use super::bank_matrix::read_i8_nn_at; -use super::decode::{pbank_group, rs1_b0, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b2, rs1_iter, rs1_read_old}; use super::gemmini_state::gemini; use super::instruction::{BallModel, ExecContext}; @@ -7,6 +7,7 @@ pub struct GemminiPreload; impl BallModel for GemminiPreload { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "gemmini_preload: reads no old destination; rs1[29] must be clear"); let op1 = rs1_b0(xs1); let wr = rs1_b2(xs1); let n = rs1_iter(xs1) as usize; diff --git a/examples/components/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs b/examples/prefab/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs similarity index 96% rename from examples/components/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs rename to examples/prefab/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs index f35164cee..18c90b292 100644 --- a/examples/components/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs +++ b/examples/prefab/balls/gemmini/emu/src/66_gemmini_compute_preloaded.rs @@ -1,5 +1,5 @@ use super::bank_matrix::{read_i32_nn_at, read_i8_nn_at, write_i32_nn_groups_at}; -use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::gemmini_state::{gemini, in_shift as apply_in_shift}; use super::instruction::{BallModel, ExecContext}; @@ -7,6 +7,7 @@ pub struct GemminiComputePreloaded; impl BallModel for GemminiComputePreloaded { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "gemmini_compute_preloaded: reads no old destination; rs1[29] must be clear"); let op_a = rs1_b0(xs1); let op_b = rs1_b1(xs1); let wr = rs1_b2(xs1); diff --git a/examples/components/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs b/examples/prefab/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs similarity index 92% rename from examples/components/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs rename to examples/prefab/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs index 2c124c6ef..cc49087c7 100644 --- a/examples/components/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs +++ b/examples/prefab/balls/gemmini/emu/src/67_gemmini_compute_accumulated.rs @@ -1,5 +1,5 @@ use super::bank_matrix::{read_i32_nn_groups_at, read_i8_nn_at, write_i32_nn_groups_at}; -use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::gemmini_state::{gemini, in_shift as apply_in_shift}; use super::instruction::{BallModel, ExecContext}; @@ -7,6 +7,11 @@ pub struct GemminiComputeAccumulated; impl BallModel for GemminiComputeAccumulated { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + // The RTL reads the old C from wr_read_bank, which rs1[29] provides. + assert!( + rs1_read_old(xs1), + "gemmini_compute_accumulated: reads the old destination; rs1[29] must be set" + ); let op_a = rs1_b0(xs1); let op_b = rs1_b1(xs1); let wr = rs1_b2(xs1); diff --git a/examples/components/balls/gemmini/emu/src/80_gemmini_loop_ws.rs b/examples/prefab/balls/gemmini/emu/src/80_gemmini_loop_ws.rs similarity index 100% rename from examples/components/balls/gemmini/emu/src/80_gemmini_loop_ws.rs rename to examples/prefab/balls/gemmini/emu/src/80_gemmini_loop_ws.rs diff --git a/examples/components/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs b/examples/prefab/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs similarity index 100% rename from examples/components/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs rename to examples/prefab/balls/gemmini/emu/src/96_gemmini_loop_conv_ws.rs diff --git a/examples/components/balls/gemmini/emu/src/dispatch.rs b/examples/prefab/balls/gemmini/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/gemmini/emu/src/dispatch.rs rename to examples/prefab/balls/gemmini/emu/src/dispatch.rs diff --git a/examples/components/balls/gemmini/emu/src/gemmini_state.rs b/examples/prefab/balls/gemmini/emu/src/gemmini_state.rs similarity index 100% rename from examples/components/balls/gemmini/emu/src/gemmini_state.rs rename to examples/prefab/balls/gemmini/emu/src/gemmini_state.rs diff --git a/examples/components/balls/gemmini/emu/src/lib.rs b/examples/prefab/balls/gemmini/emu/src/lib.rs similarity index 100% rename from examples/components/balls/gemmini/emu/src/lib.rs rename to examples/prefab/balls/gemmini/emu/src/lib.rs diff --git a/examples/components/balls/gemmini/emu/src/loop_micro_ops.rs b/examples/prefab/balls/gemmini/emu/src/loop_micro_ops.rs similarity index 90% rename from examples/components/balls/gemmini/emu/src/loop_micro_ops.rs rename to examples/prefab/balls/gemmini/emu/src/loop_micro_ops.rs index 90e5d78ce..762184d45 100644 --- a/examples/components/balls/gemmini/emu/src/loop_micro_ops.rs +++ b/examples/prefab/balls/gemmini/emu/src/loop_micro_ops.rs @@ -17,8 +17,8 @@ fn execute(funct: u32, xs1: u64, xs2: u64, ctx: &mut ExecContext) { fn banks(bank0: u64, bank1: u64, bank2: u64, iter: u64) -> u64 { assert!(bank0 < (1 << 10), "Gemmini loop bank0 exceeds 10 bits"); assert!(bank1 < (1 << 10), "Gemmini loop bank1 exceeds 10 bits"); - assert!(bank2 < (1 << 10), "Gemmini loop bank2 exceeds 10 bits"); - assert!(iter < (1 << 34), "Gemmini loop iter exceeds 34 bits"); + assert!(bank2 < (1 << 9), "Gemmini loop bank2 exceeds 9 bits"); + assert!(iter < (1 << 33), "Gemmini loop iter exceeds 33 bits"); bank0 | (bank1 << 10) | (bank2 << 20) | (iter << 30) } @@ -80,7 +80,8 @@ pub fn compute( ) { let mode = if accumulated { 3 } else { 2 }; let xs2 = mode | ((zero_op2 as u64) << 4) | ((zero_op1_tail as u64) << 5); - let xs1 = banks(op1, op2, output, iter); + // The accumulated compute reads the old output (rs1[29]), as Loop sets rd_wr. + let xs1 = banks(op1, op2, output, iter) | ((accumulated as u64) << 29); if accumulated { GemminiComputeAccumulated::exec(xs1, xs2, ctx); } else { diff --git a/examples/prefab/balls/gemmini/stack/src/gemminiOps.cpp b/examples/prefab/balls/gemmini/stack/src/gemminiOps.cpp new file mode 100644 index 000000000..f5b2ce973 --- /dev/null +++ b/examples/prefab/balls/gemmini/stack/src/gemminiOps.cpp @@ -0,0 +1,115 @@ +//===- gemminiOps.cpp - GemminiBall operations ----------------------------===// +// +// The dialect of GemminiBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "gemminiOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::gemmini_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "gemminiDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "gemminiOps.cpp.inc" + +void GemminiBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "gemminiOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankMatmulI8TileOp::getSourceBanks() { + return {getA(), getB()}; +} +Value BankMatmulI8TileOp::getDestinationBank() { return getC(); } +bool BankMatmulI8TileOp::readsDestination() { return getReadDestination(); } +Operation *BankMatmulI8TileOp::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulI8TileOp::create( + builder, getLoc(), builder.getI64IntegerAttr(sources[0]), + builder.getI64IntegerAttr(sources[1]), + builder.getI64IntegerAttr(destination), getKAttr(), + getReadDestinationAttr(), getShiftAttr()); +} + +SmallVector MatmulI8TileOp::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t MatmulI8TileOp::getDestinationBank() { return getC(); } +LogicalResult MatmulI8TileOp::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + bool readDestination = getReadDestination(); + if (readDestination != getShift().has_value()) + return emitOpError( + "matmul_i8_tile takes a shift exactly with read_destination"); + int64_t k = getK(), a = getA(), b = getB(), c = getC(); + int64_t shift = readDestination ? int64_t(*getShift()) : 0; + int64_t meshRows = buckyball_target::getBuckyballBallParam( + "GEMMINI_COMPUTE_PRELOADED", "meshRows"); + int64_t meshColumns = buckyball_target::getBuckyballBallParam( + "GEMMINI_COMPUTE_PRELOADED", "meshColumns"); + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || meshRows != 16 || meshColumns != 16 || + target.bankDepth % 16 || target.bankDepth > 1024 || a == b || k <= 0 || + k % 16 || k > 64 * target.bankDepth || a == c || b == c) + return emitOpError("matmul_i8_tile requires distinct A, B and C banks, k " + "a multiple of 16 within 64 groups and a 16x16 mesh"); + int64_t steps = k / 16, groupBlocks = target.bankDepth / 16; + uint64_t config = ~0ULL; + for (int64_t step = 0; step < steps; ++step) { + uint64_t group = step / groupBlocks, base = step % groupBlocks * 16; + bool accumulate = readDestination || step > 0; + // CONFIG: rs2[4] picks the weight-stationary array, rs2[40:9] the shift. + uint64_t word = + field(accumulate, 4, 4) | field(step == steps - 1 ? shift : 0, 9, 40); + if (word != config) + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("GEMMINI_CONFIG"), + constantI64(builder, loc, 0), constantI64(builder, loc, word)); + config = word; + // rs2: the subcommand in [3:0], the first and second source row bases in + // [15:6] and [25:16], their groups in [41:36] and [47:42]; C is at row 0. + uint64_t bases = field(base, 6, 15) | field(base, 16, 25) | + field(group, 36, 41) | field(group, 42, 47); + if (accumulate) { + emitInstruction( + builder, loc, + buckyball_target::getBuckyballFunct7("GEMMINI_COMPUTE_ACCUMULATED"), + constantI64(builder, loc, bankFields({a, b}, c, true, 16)), + constantI64(builder, loc, 3 | bases)); + continue; + } + // The output-stationary preload reads sixteen rows of B. + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7("GEMMINI_PRELOAD"), + constantI64(builder, loc, bankFields({b}, c, false, 16)), + constantI64(builder, loc, + 1 | field(base, 6, 15) | field(group, 36, 41))); + emitInstruction( + builder, loc, + buckyball_target::getBuckyballFunct7("GEMMINI_COMPUTE_PRELOADED"), + constantI64(builder, loc, bankFields({a, b}, c, false, 16)), + constantI64(builder, loc, 2 | bases)); + } + return success(); +} diff --git a/examples/prefab/balls/gemmini/stack/src/gemminiOps.h b/examples/prefab/balls/gemmini/stack/src/gemminiOps.h new file mode 100644 index 000000000..ca9a31760 --- /dev/null +++ b/examples/prefab/balls/gemmini/stack/src/gemminiOps.h @@ -0,0 +1,20 @@ +//===- gemminiOps.h - GemminiBall operations ------------------------------===// + +#ifndef GEMMINI_OPS_H +#define GEMMINI_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "gemminiDialect.h.inc" + +#define GET_OP_CLASSES +#include "gemminiOps.h.inc" + +#endif // GEMMINI_OPS_H diff --git a/examples/prefab/balls/gemmini/stack/src/gemminiOps.td b/examples/prefab/balls/gemmini/stack/src/gemminiOps.td new file mode 100644 index 000000000..d758b655d --- /dev/null +++ b/examples/prefab/balls/gemmini/stack/src/gemminiOps.td @@ -0,0 +1,62 @@ +//===- gemminiOps.td - GemminiBall operations --------------*- tablegen -*-===// +// +// matmul_i8_tile: one 16x16 tile C = A @ B over `k`, a positive multiple of +// sixteen. A and B fill their groups in order, each group a bank depth of rows +// before the next, so a K deeper than a bank spans several groups (allocated +// together or gathered with bank_transfer). +// A: k / 16 blocks of sixteen rows of sixteen i8; row i of block r is +// A[i, 16r .. 16r + 15]. +// B: k rows of sixteen i8, B[0 .. k - 1, 0 .. 15]. +// C: a bank of four groups, sixteen rows of sixteen i32. +// With read_destination C = in_shift(C + A @ B, shift), rounding to nearest +// even; without, C = A @ B and there is no shift. +// +// This is not smatmul's matmul_i8: C holds one output row per bank row across +// four groups instead of four bank rows of one group, there is no bias row, and +// read_destination accumulates the whole old C with a rounding shift. +// +// The Ball issues one instruction per 16-deep block, the operand groups and row +// bases derived from the block index. Without read_destination the first block +// runs on the output-stationary array (PRELOAD primes D = 0); every other block +// accumulates on the weight-stationary array, reading the old C. Only the last +// block applies the shift. +// +//===----------------------------------------------------------------------===// + +#ifndef GEMMINI_OPS +#define GEMMINI_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def GemminiBall_Dialect : Dialect { + let name = "gemmini_ball"; + let cppNamespace = "::buddy::gemmini_ball"; + let summary = "The operations of GemminiBall"; +} + +class GemminiBall_Op traits = []> + : Op])>; + +def BankMatmulI8TileOp : GemminiBall_Op<"bank_matmul_i8_tile", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_i8_tile on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$k, + UnitAttr:$read_destination, + OptionalAttr:$shift); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulI8TileOp : GemminiBall_Op<"matmul_i8_tile", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_i8_tile on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$k, + UnitAttr:$read_destination, + OptionalAttr:$shift); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // GEMMINI_OPS diff --git a/examples/components/balls/gemmini/verify/casegen/Cargo.lock b/examples/prefab/balls/gemmini/verify/casegen/Cargo.lock similarity index 100% rename from examples/components/balls/gemmini/verify/casegen/Cargo.lock rename to examples/prefab/balls/gemmini/verify/casegen/Cargo.lock diff --git a/examples/components/balls/gemmini/verify/casegen/Cargo.toml b/examples/prefab/balls/gemmini/verify/casegen/Cargo.toml similarity index 100% rename from examples/components/balls/gemmini/verify/casegen/Cargo.toml rename to examples/prefab/balls/gemmini/verify/casegen/Cargo.toml diff --git a/examples/components/balls/gemmini/verify/casegen/src/lib.rs b/examples/prefab/balls/gemmini/verify/casegen/src/lib.rs similarity index 100% rename from examples/components/balls/gemmini/verify/casegen/src/lib.rs rename to examples/prefab/balls/gemmini/verify/casegen/src/lib.rs diff --git a/examples/components/balls/gemmini/verify/filelists/gemmini_ball.f b/examples/prefab/balls/gemmini/verify/filelists/gemmini_ball.f similarity index 100% rename from examples/components/balls/gemmini/verify/filelists/gemmini_ball.f rename to examples/prefab/balls/gemmini/verify/filelists/gemmini_ball.f diff --git a/examples/components/balls/gemmini/verify/src/common/gemmini_defs.svh b/examples/prefab/balls/gemmini/verify/src/common/gemmini_defs.svh similarity index 100% rename from examples/components/balls/gemmini/verify/src/common/gemmini_defs.svh rename to examples/prefab/balls/gemmini/verify/src/common/gemmini_defs.svh diff --git a/examples/components/balls/gemmini/verify/src/gemmini_expansion_if.sv b/examples/prefab/balls/gemmini/verify/src/gemmini_expansion_if.sv similarity index 100% rename from examples/components/balls/gemmini/verify/src/gemmini_expansion_if.sv rename to examples/prefab/balls/gemmini/verify/src/gemmini_expansion_if.sv diff --git a/examples/components/balls/gemmini/verify/src/pkg/gemmini_pkg.sv b/examples/prefab/balls/gemmini/verify/src/pkg/gemmini_pkg.sv similarity index 100% rename from examples/components/balls/gemmini/verify/src/pkg/gemmini_pkg.sv rename to examples/prefab/balls/gemmini/verify/src/pkg/gemmini_pkg.sv diff --git a/examples/components/balls/gemmini/verify/src/tb_top.sv b/examples/prefab/balls/gemmini/verify/src/tb_top.sv similarity index 100% rename from examples/components/balls/gemmini/verify/src/tb_top.sv rename to examples/prefab/balls/gemmini/verify/src/tb_top.sv diff --git a/examples/components/balls/gemmini/verify/src/tests/gemmini_test.svh b/examples/prefab/balls/gemmini/verify/src/tests/gemmini_test.svh similarity index 100% rename from examples/components/balls/gemmini/verify/src/tests/gemmini_test.svh rename to examples/prefab/balls/gemmini/verify/src/tests/gemmini_test.svh diff --git a/examples/components/balls/gemmini/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/gemmini/workloads/ctests/CMakeLists.txt similarity index 100% rename from examples/components/balls/gemmini/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/gemmini/workloads/ctests/CMakeLists.txt diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_accumulated_16x16_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_accumulated_16x16_test.c similarity index 100% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_accumulated_16x16_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_accumulated_16x16_test.c diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_loop_mmio.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_loop_mmio.c similarity index 89% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_loop_mmio.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_loop_mmio.c index 2c5fb0910..96072f44c 100644 --- a/examples/components/balls/gemmini/workloads/ctests/gemmini_loop_mmio.c +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_loop_mmio.c @@ -1,9 +1,8 @@ #include #include -#include #include -#define DIM (BANK_WIDTH / 8) +#define DIM (THREAD_MEM_BANK_WIDTH / 8) #define M DIM #define N (2 * DIM) #define K (2 * DIM) @@ -34,8 +33,9 @@ int main(void) { descriptor[6] = (uint64_t)(N * 4) << 32; descriptor[7] = 0; bb_gemmini_config(0, 0, 0, 0, 0); - bb_mvin_mmio((uintptr_t)descriptor, 0, sizeof(descriptor) / (BANK_WIDTH / 8), - BANK_WIDTH / 8); + bb_mvin_mmio((uintptr_t)descriptor, 0, + sizeof(descriptor) / (THREAD_MEM_BANK_WIDTH / 8), + THREAD_MEM_BANK_WIDTH / 8); bb_gemmini_loop_ws(0, 1, 2, 1); bb_barrier(); asm volatile("fence" ::: "memory"); diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_loop_subrob.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_loop_subrob.c similarity index 97% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_loop_subrob.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_loop_subrob.c index 93aac9a74..db9bedaf7 100644 --- a/examples/components/balls/gemmini/workloads/ctests/gemmini_loop_subrob.c +++ b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_loop_subrob.c @@ -1,9 +1,8 @@ #include #include -#include #include -#define DIM (BANK_WIDTH / 8) +#define DIM (THREAD_MEM_BANK_WIDTH / 8) #define M (2 * DIM) #define N DIM #define K (2 * DIM) diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c similarity index 100% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_atranspose_test.c diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c similarity index 100% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_basic_test.c diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c similarity index 100% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_btranspose_test.c diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c similarity index 100% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_cisc_shift_test.c diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c similarity index 100% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_abtranspose_test.c diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c similarity index 100% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_atranspose_test.c diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c similarity index 100% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_basic_test.c diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c similarity index 100% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_btranspose_test.c diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c similarity index 100% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_os_risc_shift_test.c diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c similarity index 100% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_cisc_conv_test.c diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c similarity index 100% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_risc_basic_test.c diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c similarity index 100% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_risc_btranspose_test.c diff --git a/examples/components/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c b/examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c similarity index 100% rename from examples/components/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c rename to examples/prefab/balls/gemmini/workloads/ctests/gemmini_ws_risc_shift_test.c diff --git a/examples/components/balls/gemmini/workloads/isa/gemmini.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini.h similarity index 100% rename from examples/components/balls/gemmini/workloads/isa/gemmini.h rename to examples/prefab/balls/gemmini/workloads/isa/gemmini.h diff --git a/examples/components/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h similarity index 95% rename from examples/components/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h rename to examples/prefab/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h index 6d0c03a5e..e015fc4e1 100644 --- a/examples/components/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h +++ b/examples/prefab/balls/gemmini/workloads/isa/gemmini_compute_accumulated.h @@ -18,6 +18,6 @@ BB_ITER(iter) | BB_READ_WR_OLD), \ BB_GEMMINI_COMPUTE_ACCUMULATED_RS2 | ((uint64_t)(op1_base) << 6) | \ ((uint64_t)(op2_base) << 16) | ((uint64_t)(wr_base) << 26), \ - BB_FUNC7(GEMMINI_COMPUTE_ACCUMULATED)) + FUNCT7(gemmini, GEMMINI_COMPUTE_ACCUMULATED)) #endif // _BB_GEMMINI_COMPUTE_ACCUMULATED_H_ diff --git a/examples/components/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h similarity index 95% rename from examples/components/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h rename to examples/prefab/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h index c39d3b10f..7f1ec4177 100644 --- a/examples/components/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h +++ b/examples/prefab/balls/gemmini/workloads/isa/gemmini_compute_preloaded.h @@ -18,6 +18,6 @@ BB_ITER(iter)), \ BB_GEMMINI_COMPUTE_PRELOADED_RS2 | ((uint64_t)(op1_base) << 6) | \ ((uint64_t)(op2_base) << 16) | ((uint64_t)(wr_base) << 26), \ - BB_FUNC7(GEMMINI_COMPUTE_PRELOADED)) + FUNCT7(gemmini, GEMMINI_COMPUTE_PRELOADED)) #endif // _BB_GEMMINI_COMPUTE_PRELOADED_H_ diff --git a/examples/components/balls/gemmini/workloads/isa/gemmini_config.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini_config.h similarity index 94% rename from examples/components/balls/gemmini/workloads/isa/gemmini_config.h rename to examples/prefab/balls/gemmini/workloads/isa/gemmini_config.h index fc50c0748..43aac5668 100644 --- a/examples/components/balls/gemmini/workloads/isa/gemmini_config.h +++ b/examples/prefab/balls/gemmini/workloads/isa/gemmini_config.h @@ -18,6 +18,6 @@ FIELD(a_transpose, 7, 7) | \ FIELD(b_transpose, 8, 8) | \ FIELD(in_shift, 9, 40)), \ - BB_FUNC7(GEMMINI_CONFIG)) + FUNCT7(gemmini, GEMMINI_CONFIG)) #endif // _BB_GEMMINI_CONFIG_H_ diff --git a/examples/components/balls/gemmini/workloads/isa/gemmini_flush.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini_flush.h similarity index 67% rename from examples/components/balls/gemmini/workloads/isa/gemmini_flush.h rename to examples/prefab/balls/gemmini/workloads/isa/gemmini_flush.h index 2c55b62c0..0e4d9628d 100644 --- a/examples/components/balls/gemmini/workloads/isa/gemmini_flush.h +++ b/examples/prefab/balls/gemmini/workloads/isa/gemmini_flush.h @@ -8,6 +8,7 @@ // Flush the matrix array state #define bb_gemmini_flush() \ - BUCKYBALL_INSTRUCTION_R_R(0, BB_GEMMINI_FLUSH_RS2, BB_FUNC7(GEMMINI_FLUSH)) + BUCKYBALL_INSTRUCTION_R_R(0, BB_GEMMINI_FLUSH_RS2, \ + FUNCT7(gemmini, GEMMINI_FLUSH)) #endif // _BB_GEMMINI_FLUSH_H_ diff --git a/examples/components/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h similarity index 81% rename from examples/components/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h rename to examples/prefab/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h index 5f04089f7..f6f80d77c 100644 --- a/examples/components/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h +++ b/examples/prefab/balls/gemmini/workloads/isa/gemmini_loop_conv_ws.h @@ -9,7 +9,7 @@ (FIELD(batch_size, 0, 15) | \ FIELD(in_dim, 16, 31) | \ FIELD(in_channels, 32, 47)), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_1)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_1)) #define bb_gemmini_loop_conv_ws_config_2(out_channels, out_dim, stride, \ padding) \ @@ -17,7 +17,7 @@ (FIELD(out_channels, 0, 15) | \ FIELD(out_dim, 16, 31) | FIELD(stride, 32, 39) | \ FIELD(padding, 40, 47)), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_2)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_2)) #define bb_gemmini_loop_conv_ws_config_3(kernel_dim, pool_size, pool_stride, \ pool_padding) \ @@ -25,38 +25,38 @@ 0, \ (FIELD(kernel_dim, 0, 7) | FIELD(pool_size, 8, 15) | \ FIELD(pool_stride, 16, 23) | FIELD(pool_padding, 24, 31)), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_3)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_3)) #define bb_gemmini_loop_conv_ws_config_4(addr_bias) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr_bias, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_4)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_4)) #define bb_gemmini_loop_conv_ws_config_5(addr_input) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr_input, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_5)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_5)) #define bb_gemmini_loop_conv_ws_config_6(addr_weight) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr_weight, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_6)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_6)) #define bb_gemmini_loop_conv_ws_config_7(addr_output) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr_output, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_7)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_7)) #define bb_gemmini_loop_conv_ws_config_8(input_stride, weight_stride) \ BUCKYBALL_INSTRUCTION_R_R( \ 0, (FIELD(input_stride, 0, 31) | FIELD(weight_stride, 32, 63)), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_8)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_8)) #define bb_gemmini_loop_conv_ws_config_9(output_stride) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(output_stride, 0, 31), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS_CONFIG_9)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS_CONFIG_9)) #define bb_gemmini_loop_conv_ws(bank_input, bank_weight, bank_output, no_bias) \ BUCKYBALL_INSTRUCTION_R_R( \ 0, \ (FIELD(bank_input, 0, 9) | FIELD(bank_weight, 10, 19) | \ FIELD(bank_output, 20, 29) | FIELD(no_bias, 30, 30)), \ - BB_FUNC7(GEMMINI_LOOP_CONV_WS)) + FUNCT7(gemmini, GEMMINI_LOOP_CONV_WS)) #endif diff --git a/examples/components/balls/gemmini/workloads/isa/gemmini_loop_ws.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini_loop_ws.h similarity index 77% rename from examples/components/balls/gemmini/workloads/isa/gemmini_loop_ws.h rename to examples/prefab/balls/gemmini/workloads/isa/gemmini_loop_ws.h index 769c1f660..9d76463b8 100644 --- a/examples/components/balls/gemmini/workloads/isa/gemmini_loop_ws.h +++ b/examples/prefab/balls/gemmini/workloads/isa/gemmini_loop_ws.h @@ -7,38 +7,38 @@ #define bb_gemmini_loop_ws_config_bounds(max_i, max_j, max_k) \ BUCKYBALL_INSTRUCTION_R_R( \ 0, (FIELD(max_k, 0, 15) | FIELD(max_j, 16, 31) | FIELD(max_i, 32, 47)), \ - BB_FUNC7(GEMMINI_LOOP_WS_CONFIG_BOUNDS)) + FUNCT7(gemmini, GEMMINI_LOOP_WS_CONFIG_BOUNDS)) #define bb_gemmini_loop_ws_config_addr_a(addr) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_WS_CONFIG_ADDR_A)) + FUNCT7(gemmini, GEMMINI_LOOP_WS_CONFIG_ADDR_A)) #define bb_gemmini_loop_ws_config_addr_b(addr) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_WS_CONFIG_ADDR_B)) + FUNCT7(gemmini, GEMMINI_LOOP_WS_CONFIG_ADDR_B)) #define bb_gemmini_loop_ws_config_addr_d(addr) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_WS_CONFIG_ADDR_D)) + FUNCT7(gemmini, GEMMINI_LOOP_WS_CONFIG_ADDR_D)) #define bb_gemmini_loop_ws_config_addr_c(addr) \ BUCKYBALL_INSTRUCTION_R_R(0, FIELD(addr, 0, 38), \ - BB_FUNC7(GEMMINI_LOOP_WS_CONFIG_ADDR_C)) + FUNCT7(gemmini, GEMMINI_LOOP_WS_CONFIG_ADDR_C)) #define bb_gemmini_loop_ws_config_strides_ab(stride_a, stride_b) \ BUCKYBALL_INSTRUCTION_R_R( \ 0, (FIELD(stride_a, 0, 31) | FIELD(stride_b, 32, 63)), \ - BB_FUNC7(GEMMINI_LOOP_WS_CONFIG_STRIDES_AB)) + FUNCT7(gemmini, GEMMINI_LOOP_WS_CONFIG_STRIDES_AB)) #define bb_gemmini_loop_ws_config_strides_dc(stride_d, stride_c) \ BUCKYBALL_INSTRUCTION_R_R( \ 0, (FIELD(stride_d, 0, 31) | FIELD(stride_c, 32, 63)), \ - BB_FUNC7(GEMMINI_LOOP_WS_CONFIG_STRIDES_DC)) + FUNCT7(gemmini, GEMMINI_LOOP_WS_CONFIG_STRIDES_DC)) #define bb_gemmini_loop_ws(bank_a, bank_b, bank_c, low_d) \ BUCKYBALL_INSTRUCTION_R_R(0, \ (FIELD(bank_a, 0, 9) | FIELD(bank_b, 10, 19) | \ FIELD(bank_c, 20, 29) | FIELD(low_d, 30, 30)), \ - BB_FUNC7(GEMMINI_LOOP_WS)) + FUNCT7(gemmini, GEMMINI_LOOP_WS)) #endif diff --git a/examples/components/balls/gemmini/workloads/isa/gemmini_preload.h b/examples/prefab/balls/gemmini/workloads/isa/gemmini_preload.h similarity index 95% rename from examples/components/balls/gemmini/workloads/isa/gemmini_preload.h rename to examples/prefab/balls/gemmini/workloads/isa/gemmini_preload.h index f1f578bec..b6ac3f0fb 100644 --- a/examples/components/balls/gemmini/workloads/isa/gemmini_preload.h +++ b/examples/prefab/balls/gemmini/workloads/isa/gemmini_preload.h @@ -15,6 +15,6 @@ (BB_BANK0(op1_bank_id) | BB_BANK2(wr_bank_id) | BB_ITER(iter)), \ BB_GEMMINI_PRELOAD_RS2 | ((uint64_t)(op1_base) << 6) | \ ((uint64_t)(wr_base) << 26), \ - BB_FUNC7(GEMMINI_PRELOAD)) + FUNCT7(gemmini, GEMMINI_PRELOAD)) #endif // _BB_GEMMINI_PRELOAD_H_ diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/gemmini/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/gemmini/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/gemmini/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/components/balls/gemmini/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/gemmini/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir b/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir similarity index 94% rename from examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir rename to examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir index c7e2d6913..3de982139 100644 --- a/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir +++ b/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16.mlir @@ -38,7 +38,7 @@ func.func @main() -> i8 { buckyball.mvin %b <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> buckyball.mvin %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> // C = in_shift(C + A @ B, 4). - buckyball.custom "matmul_i8_tile" <{sources = array, destination = 3 : i64, read_destination}> {k = 16 : i64, shift = 4 : i64} + gemmini_ball.matmul_i8_tile <{a = 1 : i64, b = 2 : i64, c = 3 : i64, k = 16 : i64, shift = 4 : i64, read_destination}> buckyball.mvout %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> func.call @check_result(%c) : (memref<16x16xi32>) -> () buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16_main.cpp b/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16_main.cpp similarity index 100% rename from examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16_main.cpp rename to examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_accumulated_command_16x16_main.cpp diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir b/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir similarity index 95% rename from examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir rename to examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir index 1e4b9d715..84ff88d7a 100644 --- a/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir +++ b/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16.mlir @@ -37,7 +37,7 @@ func.func @main() -> i8 { buckyball.mvin %a <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> buckyball.mvin %b <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> // C = A @ B on the output-stationary array. - buckyball.custom "matmul_i8_tile" <{sources = array, destination = 3 : i64}> {k = 16 : i64} + gemmini_ball.matmul_i8_tile <{a = 1 : i64, b = 2 : i64, c = 3 : i64, k = 16 : i64}> buckyball.mvout %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> func.call @check_result(%c) : (memref<16x16xi32>) -> () buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16_main.cpp b/examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16_main.cpp similarity index 100% rename from examples/components/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16_main.cpp rename to examples/prefab/balls/gemmini/workloads/mlir_tests/ball/gemmini_preloaded_command_16x16_main.cpp diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt similarity index 61% rename from examples/components/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt index d38664730..2187c846b 100644 --- a/examples/components/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt +++ b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/CMakeLists.txt @@ -2,12 +2,10 @@ if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) endif() set(BUCKYBALL_MLIR_TEST_PREFIX bank) -set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") -file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") -string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") +set(_lines ${DESIGN_PROFILES_${BUCKYBALL_MLIR_ACTIVE_TARGET}_THREAD_MEM_BANK_ENTRIES}) if(NOT _lines EQUAL 1024) message(FATAL_ERROR - "Gemmini BankOp tests require BANK_LINES = 1024 (got ${_lines})") + "Gemmini BankOp tests require 1024 bank entries (got ${_lines})") endif() add_buckyball_mlir_test(gemmini_compute_preloaded_bank_tiles TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) add_buckyball_mlir_test(gemmini_compute_accumulated_bank_tiles TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir similarity index 97% rename from examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir rename to examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir index 4bbbe0519..04caf4037 100644 --- a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir +++ b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles.mlir @@ -82,7 +82,7 @@ func.func @main() -> i8 { %cl = buckyball.bank_mvin %c, %cb <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> %al = buckyball.bank_transfer %al1, %al0 %bl = buckyball.bank_transfer %bl1, %bl0 - %r = buckyball.bank_custom "matmul_i8_tile" (%al, %bl) -> %cl <{read_destination}> {k = 2048 : i64, shift = 2 : i64} + %r = gemmini_ball.bank_matmul_i8_tile %al, %bl -> %cl <{k = 2048 : i64, shift = 2 : i64, read_destination}> %stored = buckyball.bank_mvout %c, %r <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> func.call @check_result(%c) : (memref<16x16xi32>) -> () buckyball.bank_release %al diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp similarity index 95% rename from examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp rename to examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp index 85ade8e02..80f85fe95 100644 --- a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp +++ b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_accumulated_bank_tiles_main.cpp @@ -1,8 +1,7 @@ -#include #include #include #include -static_assert(BANK_LINES == 1024); +static_assert(THREAD_MEM_BANK_ENTRIES == 1024); extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir similarity index 97% rename from examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir rename to examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir index c656eaca6..835e603cd 100644 --- a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir +++ b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles.mlir @@ -64,7 +64,7 @@ func.func @main() -> i8 { %cb = buckyball.bank_alloc <{row = 1 : i64, col = 4 : i64}> %al = buckyball.bank_mvin %a, %ab <{depth = 96 : i64, stride = 1 : i64}> : memref<96x16xi8> %bl = buckyball.bank_mvin %b, %bb <{depth = 96 : i64, stride = 1 : i64}> : memref<96x16xi8> - %r = buckyball.bank_custom "matmul_i8_tile" (%al, %bl) -> %cb {k = 96 : i64} + %r = gemmini_ball.bank_matmul_i8_tile %al, %bl -> %cb <{k = 96 : i64}> %stored = buckyball.bank_mvout %c, %r <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> func.call @check_result(%c) : (memref<16x16xi32>) -> () buckyball.bank_release %al diff --git a/examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp b/examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp similarity index 100% rename from examples/components/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp rename to examples/prefab/balls/gemmini/workloads/mlir_tests/bank/gemmini_compute_preloaded_bank_tiles_main.cpp diff --git a/examples/components/balls/im2col/arch/Im2col.scala b/examples/prefab/balls/im2col/arch/Im2col.scala similarity index 100% rename from examples/components/balls/im2col/arch/Im2col.scala rename to examples/prefab/balls/im2col/arch/Im2col.scala diff --git a/examples/components/balls/im2col/arch/Im2colBall.scala b/examples/prefab/balls/im2col/arch/Im2colBall.scala similarity index 100% rename from examples/components/balls/im2col/arch/Im2colBall.scala rename to examples/prefab/balls/im2col/arch/Im2colBall.scala diff --git a/examples/components/balls/im2col/arch/README.md b/examples/prefab/balls/im2col/arch/README.md similarity index 100% rename from examples/components/balls/im2col/arch/README.md rename to examples/prefab/balls/im2col/arch/README.md diff --git a/examples/components/balls/im2col/arch/RestoringDiv.scala b/examples/prefab/balls/im2col/arch/RestoringDiv.scala similarity index 100% rename from examples/components/balls/im2col/arch/RestoringDiv.scala rename to examples/prefab/balls/im2col/arch/RestoringDiv.scala diff --git a/examples/components/balls/im2col/configs/Im2colBallParam.scala b/examples/prefab/balls/im2col/configs/Im2colBallParam.scala similarity index 100% rename from examples/components/balls/im2col/configs/Im2colBallParam.scala rename to examples/prefab/balls/im2col/configs/Im2colBallParam.scala diff --git a/examples/components/balls/im2col/configs/default.toml b/examples/prefab/balls/im2col/configs/default.toml similarity index 100% rename from examples/components/balls/im2col/configs/default.toml rename to examples/prefab/balls/im2col/configs/default.toml diff --git a/examples/components/balls/im2col/configs/im2col_params.proto b/examples/prefab/balls/im2col/configs/im2col_params.proto similarity index 100% rename from examples/components/balls/im2col/configs/im2col_params.proto rename to examples/prefab/balls/im2col/configs/im2col_params.proto diff --git a/examples/components/balls/im2col/emu/src/dispatch.rs b/examples/prefab/balls/im2col/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/im2col/emu/src/dispatch.rs rename to examples/prefab/balls/im2col/emu/src/dispatch.rs diff --git a/examples/components/balls/im2col/emu/src/im2col.rs b/examples/prefab/balls/im2col/emu/src/im2col.rs similarity index 98% rename from examples/components/balls/im2col/emu/src/im2col.rs rename to examples/prefab/balls/im2col/emu/src/im2col.rs index c8269e234..a0fe9d9a2 100644 --- a/examples/components/balls/im2col/emu/src/im2col.rs +++ b/examples/prefab/balls/im2col/emu/src/im2col.rs @@ -1,4 +1,4 @@ -use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::ExecContext; use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; @@ -96,6 +96,7 @@ fn output_cols(shape: Shape, instance: u32) -> usize { } pub(crate) fn exec_im2col(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "im2col: reads no old destination; rs1[29] must be clear"); let input_bank = rs1_b0(xs1); let output_bank = rs1_b2(xs1); if input_bank == output_bank { diff --git a/examples/components/balls/im2col/emu/src/lib.rs b/examples/prefab/balls/im2col/emu/src/lib.rs similarity index 100% rename from examples/components/balls/im2col/emu/src/lib.rs rename to examples/prefab/balls/im2col/emu/src/lib.rs diff --git a/examples/prefab/balls/im2col/stack/src/im2colOps.cpp b/examples/prefab/balls/im2col/stack/src/im2colOps.cpp new file mode 100644 index 000000000..98ae6eb64 --- /dev/null +++ b/examples/prefab/balls/im2col/stack/src/im2colOps.cpp @@ -0,0 +1,85 @@ +//===- im2colOps.cpp - Im2colBall operations ------------------------------===// +// +// The dialect of Im2colBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "im2colOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::im2col_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "im2colDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "im2colOps.cpp.inc" + +void Im2colBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "im2colOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankIm2colI8Op::getSourceBanks() { return {getInput()}; } +Value BankIm2colI8Op::getDestinationBank() { return getOutput(); } +bool BankIm2colI8Op::readsDestination() { return false; } +Operation *BankIm2colI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return Im2colI8Op::create(builder, getLoc(), sources[0], destination, + getInputSide(), getKernel(), getStride(), + getPadding(), getTileRows()); +} + +SmallVector Im2colI8Op::getSourceBanks() { + return {int64_t(getInput())}; +} +int64_t Im2colI8Op::getDestinationBank() { return getOutput(); } +LogicalResult Im2colI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t input = getInput(), output = getOutput(); + if (input == output) + return emitOpError("im2col_i8 requires distinct input and output banks"); + int64_t side = getInputSide(), kernel = getKernel(), stride = getStride(), + padding = getPadding(), tileRows = getTileRows(); + const auto &target = buckyball_target::getBuckyballTarget(); + int64_t span = side + 2 * padding - kernel; + if (target.bankWidthBits != 128 || side <= 0 || + side > buckyball_target::getBuckyballBallParam("IM2COL", "maxIter") || + side * side > target.bankDepth || kernel <= 0 || + kernel > buckyball_target::getBuckyballBallParam("IM2COL", "maxKSize") || + stride <= 0 || stride > 15 || padding < 0 || + padding > + buckyball_target::getBuckyballBallParam("IM2COL", "maxPadding") || + span < 0 || span % stride || tileRows <= 0 || tileRows > 16 || + (tileRows & (tileRows - 1))) + return emitOpError("im2col_i8 geometry exceeds the ISA field ranges"); + int64_t windows = (span / stride + 1) * (span / stride + 1); + if (windows > 1023) + return emitOpError("im2col_i8 has more windows than one instruction"); + uint64_t rs2 = field(kernel, 0, 3) | field(stride, 4, 7) | + field(padding, 8, 11) | field(windows, 36, 45) | + field(tileRows, 46, 50); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("IM2COL"), + constantI64(builder, loc, bankFields({input}, output, false, side)), + constantI64(builder, loc, rs2)); + return success(); +} diff --git a/examples/prefab/balls/im2col/stack/src/im2colOps.h b/examples/prefab/balls/im2col/stack/src/im2colOps.h new file mode 100644 index 000000000..7afdc31ca --- /dev/null +++ b/examples/prefab/balls/im2col/stack/src/im2colOps.h @@ -0,0 +1,20 @@ +//===- im2colOps.h - Im2colBall operations --------------------------------===// + +#ifndef IM2COL_OPS_H +#define IM2COL_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "im2colDialect.h.inc" + +#define GET_OP_CLASSES +#include "im2colOps.h.inc" + +#endif // IM2COL_OPS_H diff --git a/examples/prefab/balls/im2col/stack/src/im2colOps.td b/examples/prefab/balls/im2col/stack/src/im2colOps.td new file mode 100644 index 000000000..41d44a409 --- /dev/null +++ b/examples/prefab/balls/im2col/stack/src/im2colOps.td @@ -0,0 +1,52 @@ +//===- im2colOps.td - Im2colBall operations ----------------*- tablegen -*-===// +// +// im2col_i8: the input bank holds a square feature map of `input_side` pixels +// per side; group g is channel block g, line y * input_side + x holds the +// sixteen channels of pixel (y, x). The output receives the windows of the +// `kernel` x `kernel` convolution with `stride` and zero `padding`, every +// window in row-major order, as the matmul_i8 A operand: K is ordered (kernel +// row, kernel column, channel), sixteen columns (one input line) per K tile, +// K tile t = (kr * kernel + kc) * groups + g. The output groups divide the K +// tiles; group j holds the next K / 16 / groups tiles in M tiles of +// `tile_rows` rows (the layout of the matmul that consumes it), windows beyond +// the last zero up to a multiple of tile_rows. +// +//===----------------------------------------------------------------------===// + +#ifndef IM2COL_OPS +#define IM2COL_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def Im2colBall_Dialect : Dialect { + let name = "im2col_ball"; + let cppNamespace = "::buddy::im2col_ball"; + let summary = "The operations of Im2colBall"; +} + +class Im2colBall_Op traits = []> + : Op])>; + +def BankIm2colI8Op : Im2colBall_Op<"bank_im2col_i8", + [DeclareOpInterfaceMethods]> { + let summary = "im2col_i8 on virtual banks"; + let arguments = (ins I64:$input, I64:$output, I64Attr:$input_side, + I64Attr:$kernel, I64Attr:$stride, I64Attr:$padding, + I64Attr:$tile_rows); + let results = (outs I64:$outputOut); + let assemblyFormat = "$input `->` $output prop-dict attr-dict"; +} + +def Im2colI8Op : Im2colBall_Op<"im2col_i8", + [DeclareOpInterfaceMethods]> { + let summary = "im2col_i8 on physical banks"; + let arguments = (ins I64Attr:$input, I64Attr:$output, I64Attr:$input_side, + I64Attr:$kernel, I64Attr:$stride, I64Attr:$padding, + I64Attr:$tile_rows); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // IM2COL_OPS diff --git a/examples/components/balls/im2col/verify/README.md b/examples/prefab/balls/im2col/verify/README.md similarity index 100% rename from examples/components/balls/im2col/verify/README.md rename to examples/prefab/balls/im2col/verify/README.md diff --git a/examples/components/balls/im2col/verify/casegen/Cargo.lock b/examples/prefab/balls/im2col/verify/casegen/Cargo.lock similarity index 100% rename from examples/components/balls/im2col/verify/casegen/Cargo.lock rename to examples/prefab/balls/im2col/verify/casegen/Cargo.lock diff --git a/examples/components/balls/im2col/verify/casegen/Cargo.toml b/examples/prefab/balls/im2col/verify/casegen/Cargo.toml similarity index 100% rename from examples/components/balls/im2col/verify/casegen/Cargo.toml rename to examples/prefab/balls/im2col/verify/casegen/Cargo.toml diff --git a/examples/components/balls/im2col/verify/casegen/src/casegen.rs b/examples/prefab/balls/im2col/verify/casegen/src/casegen.rs similarity index 100% rename from examples/components/balls/im2col/verify/casegen/src/casegen.rs rename to examples/prefab/balls/im2col/verify/casegen/src/casegen.rs diff --git a/examples/components/balls/im2col/verify/casegen/src/lib.rs b/examples/prefab/balls/im2col/verify/casegen/src/lib.rs similarity index 100% rename from examples/components/balls/im2col/verify/casegen/src/lib.rs rename to examples/prefab/balls/im2col/verify/casegen/src/lib.rs diff --git a/examples/components/balls/im2col/verify/casegen/src/model.rs b/examples/prefab/balls/im2col/verify/casegen/src/model.rs similarity index 100% rename from examples/components/balls/im2col/verify/casegen/src/model.rs rename to examples/prefab/balls/im2col/verify/casegen/src/model.rs diff --git a/examples/components/balls/im2col/verify/filelists/im2col_ball.f b/examples/prefab/balls/im2col/verify/filelists/im2col_ball.f similarity index 100% rename from examples/components/balls/im2col/verify/filelists/im2col_ball.f rename to examples/prefab/balls/im2col/verify/filelists/im2col_ball.f diff --git a/examples/components/balls/im2col/verify/src/common/im2col_defs.svh b/examples/prefab/balls/im2col/verify/src/common/im2col_defs.svh similarity index 100% rename from examples/components/balls/im2col/verify/src/common/im2col_defs.svh rename to examples/prefab/balls/im2col/verify/src/common/im2col_defs.svh diff --git a/examples/components/balls/im2col/verify/src/common/im2col_items.svh b/examples/prefab/balls/im2col/verify/src/common/im2col_items.svh similarity index 100% rename from examples/components/balls/im2col/verify/src/common/im2col_items.svh rename to examples/prefab/balls/im2col/verify/src/common/im2col_items.svh diff --git a/examples/components/balls/im2col/verify/src/cov/im2col_cov.svh b/examples/prefab/balls/im2col/verify/src/cov/im2col_cov.svh similarity index 100% rename from examples/components/balls/im2col/verify/src/cov/im2col_cov.svh rename to examples/prefab/balls/im2col/verify/src/cov/im2col_cov.svh diff --git a/examples/components/balls/im2col/verify/src/env/im2col_env.svh b/examples/prefab/balls/im2col/verify/src/env/im2col_env.svh similarity index 100% rename from examples/components/balls/im2col/verify/src/env/im2col_env.svh rename to examples/prefab/balls/im2col/verify/src/env/im2col_env.svh diff --git a/examples/components/balls/im2col/verify/src/env/im2col_scoreboard.svh b/examples/prefab/balls/im2col/verify/src/env/im2col_scoreboard.svh similarity index 100% rename from examples/components/balls/im2col/verify/src/env/im2col_scoreboard.svh rename to examples/prefab/balls/im2col/verify/src/env/im2col_scoreboard.svh diff --git a/examples/components/balls/im2col/verify/src/pkg/im2col_pkg.sv b/examples/prefab/balls/im2col/verify/src/pkg/im2col_pkg.sv similarity index 100% rename from examples/components/balls/im2col/verify/src/pkg/im2col_pkg.sv rename to examples/prefab/balls/im2col/verify/src/pkg/im2col_pkg.sv diff --git a/examples/components/balls/im2col/verify/src/seq/im2col_sequences.svh b/examples/prefab/balls/im2col/verify/src/seq/im2col_sequences.svh similarity index 100% rename from examples/components/balls/im2col/verify/src/seq/im2col_sequences.svh rename to examples/prefab/balls/im2col/verify/src/seq/im2col_sequences.svh diff --git a/examples/components/balls/im2col/verify/src/tb_top.sv b/examples/prefab/balls/im2col/verify/src/tb_top.sv similarity index 100% rename from examples/components/balls/im2col/verify/src/tb_top.sv rename to examples/prefab/balls/im2col/verify/src/tb_top.sv diff --git a/examples/components/balls/im2col/verify/src/tests/im2col_test.svh b/examples/prefab/balls/im2col/verify/src/tests/im2col_test.svh similarity index 100% rename from examples/components/balls/im2col/verify/src/tests/im2col_test.svh rename to examples/prefab/balls/im2col/verify/src/tests/im2col_test.svh diff --git a/examples/components/balls/im2col/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/im2col/workloads/ctests/CMakeLists.txt similarity index 100% rename from examples/components/balls/im2col/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/im2col/workloads/ctests/CMakeLists.txt diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_k1_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_k1_test.c similarity index 100% rename from examples/components/balls/im2col/workloads/ctests/im2col_k1_test.c rename to examples/prefab/balls/im2col/workloads/ctests/im2col_k1_test.c diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c similarity index 100% rename from examples/components/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c rename to examples/prefab/balls/im2col/workloads/ctests/im2col_k3_asym_pad_test.c diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_k3_pad_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_pad_test.c similarity index 100% rename from examples/components/balls/im2col/workloads/ctests/im2col_k3_pad_test.c rename to examples/prefab/balls/im2col/workloads/ctests/im2col_k3_pad_test.c diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_k3_partial_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_partial_test.c similarity index 100% rename from examples/components/balls/im2col/workloads/ctests/im2col_k3_partial_test.c rename to examples/prefab/balls/im2col/workloads/ctests/im2col_k3_partial_test.c diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_k3_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_k3_test.c similarity index 100% rename from examples/components/balls/im2col/workloads/ctests/im2col_k3_test.c rename to examples/prefab/balls/im2col/workloads/ctests/im2col_k3_test.c diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_k5_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_k5_test.c similarity index 100% rename from examples/components/balls/im2col/workloads/ctests/im2col_k5_test.c rename to examples/prefab/balls/im2col/workloads/ctests/im2col_k5_test.c diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_k7_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_k7_test.c similarity index 100% rename from examples/components/balls/im2col/workloads/ctests/im2col_k7_test.c rename to examples/prefab/balls/im2col/workloads/ctests/im2col_k7_test.c diff --git a/examples/components/balls/im2col/workloads/ctests/im2col_window_test.c b/examples/prefab/balls/im2col/workloads/ctests/im2col_window_test.c similarity index 100% rename from examples/components/balls/im2col/workloads/ctests/im2col_window_test.c rename to examples/prefab/balls/im2col/workloads/ctests/im2col_window_test.c diff --git a/examples/components/balls/im2col/workloads/isa/im2col.h b/examples/prefab/balls/im2col/workloads/isa/im2col.h similarity index 98% rename from examples/components/balls/im2col/workloads/isa/im2col.h rename to examples/prefab/balls/im2col/workloads/isa/im2col.h index 9a0783bc6..a6d056295 100644 --- a/examples/components/balls/im2col/workloads/isa/im2col.h +++ b/examples/prefab/balls/im2col/workloads/isa/im2col.h @@ -24,6 +24,6 @@ FIELD(start_col, 12, 15) | FIELD(start_row, 16, 19) | \ FIELD(input_base, 20, 25) | FIELD(window_start, 26, 35) | \ FIELD(window_count, 36, 45) | FIELD(tile_rows, 46, 50), \ - BB_FUNC7(IM2COL)) + FUNCT7(im2col, IM2COL)) #endif // _BB_IM2COL_H_ diff --git a/examples/components/balls/im2col/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/im2col/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/im2col/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/im2col/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/im2col/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/im2col/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/components/balls/im2col/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/im2col/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir b/examples/prefab/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir new file mode 100644 index 000000000..30f812b04 --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3.mlir @@ -0,0 +1,42 @@ +// Physical-bank im2col_i8: one group of sixteen channels, 6x6, k3, 16 windows +// in M tiles of eight rows. + +func.func private @check_result(memref<144x16xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c6 = arith.constant 6 : index + %c16 = arith.constant 16 : index + + %input = memref.alloc() alignment = 64 : memref<36x16xi8> + %output = memref.alloc() alignment = 64 : memref<144x16xi8> + linalg.fill ins(%zero_i8 : i8) outs(%output : memref<144x16xi8>) + + scf.for %r = %c0 to %c6 step %c1 { + scf.for %col = %c0 to %c6 step %c1 { + scf.for %ch = %c0 to %c16 step %c1 { + %tmp = arith.muli %r, %c6 : index + %pixel = arith.addi %tmp, %col : index + %sum = arith.addi %r, %col : index + %value = arith.addi %sum, %ch : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %input[%pixel, %ch] : memref<36x16xi8> + } + } + } + + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 36 : i64, stride = 1 : i64}> : memref<36x16xi8> + im2col_ball.im2col_i8 <{input = 1 : i64, output = 2 : i64, input_side = 6 : i64, kernel = 3 : i64, stride = 1 : i64, padding = 0 : i64, tile_rows = 8 : i64}> + buckyball.mvout %output <{bank = 2 : i64, depth = 144 : i64, stride = 1 : i64}> : memref<144x16xi8> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + + func.call @check_result(%output) : (memref<144x16xi8>) -> () + memref.dealloc %input : memref<36x16xi8> + memref.dealloc %output : memref<144x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp b/examples/prefab/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp new file mode 100644 index 000000000..67f7d416a --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/mlir_tests/ball/im2col_6x6_k3_main.cpp @@ -0,0 +1,41 @@ +#include +#include +#include + +// 6x6 input, k3, stride 1: 16 windows; nine K tiles (one per kernel +// position, all sixteen channels) in M tiles of eight rows. +enum { SIDE = 6, K = 3, OUT = 4, TILE_ROWS = 8, TILES = 9, LINES = 144 }; + +static void fail(void) { +#ifdef BAREMETAL + *(volatile uint32_t *)0x60000000 = 1; + while (1) { + } +#else + exit(1); +#endif +} + +extern "C" void check_result(int8_t *allocated, int8_t *aligned, int64_t offset, + int64_t rows, int64_t lanes, int64_t row_stride, + int64_t lane_stride) { + (void)allocated; + if (rows != LINES || lanes != 16 || row_stride != 16 || lane_stride != 1) + fail(); + int8_t *out = aligned + offset; + for (int w = 0; w < OUT * OUT; ++w) + for (int t = 0; t < TILES; ++t) + for (int ch = 0; ch < 16; ++ch) { + int y = w / OUT + t / K, x = w % OUT + t % K; + int line = ((w / TILE_ROWS) * TILES + t) * TILE_ROWS + w % TILE_ROWS; + int8_t expected = (int8_t)(y + x + ch); + int8_t actual = out[line * row_stride + ch]; + if (actual != expected) { + printf("FAILED: im2col 6x6 k3 w=%d t=%d ch=%d expected=%d " + "actual=%d\n", + w, t, ch, expected, actual); + fail(); + } + } + printf("PASSED: im2col 6x6 k3\n"); +} diff --git a/examples/components/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt similarity index 60% rename from examples/components/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt index 441f2843b..0cfd943de 100644 --- a/examples/components/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt +++ b/examples/prefab/balls/im2col/workloads/mlir_tests/bank/CMakeLists.txt @@ -2,5 +2,5 @@ if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) endif() set(BUCKYBALL_MLIR_TEST_PREFIX bank) -add_buckyball_mlir_test(vecmat16_atranspose_bank_depth - TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) + +add_buckyball_mlir_test(im2col_k3_two_blocks TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/im2col/workloads/mlir_tests/bank/im2col_k3_two_blocks.mlir b/examples/prefab/balls/im2col/workloads/mlir_tests/bank/im2col_k3_two_blocks.mlir new file mode 100644 index 000000000..f568240c2 --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/mlir_tests/bank/im2col_k3_two_blocks.mlir @@ -0,0 +1,38 @@ +// Virtual-bank im2col_i8: an 8x8 map of 32 channels (two groups), k3 with +// padding 1, 64 windows spread over three output groups of six K tiles. + +func.func private @check_result(memref<384x48xi8>) -> () + +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c3 = arith.constant 3 : index + %c8 = arith.constant 8 : index + %c32 = arith.constant 32 : index + %input = memref.alloc() alignment = 64 : memref<64x32xi8> + %output = memref.alloc() alignment = 64 : memref<384x48xi8> + scf.for %y = %c0 to %c8 step %c1 { + scf.for %x = %c0 to %c8 step %c1 { + scf.for %ch = %c0 to %c32 step %c1 { + %row = arith.muli %y, %c8 : index + %pixel = arith.addi %row, %x : index + %scaled = arith.muli %ch, %c3 : index + %value = arith.addi %pixel, %scaled : index + %v8 = arith.index_cast %value : index to i8 + memref.store %v8, %input[%pixel, %ch] : memref<64x32xi8> + } + } + } + %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 2 : i64}> + %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 3 : i64}> + %loaded = buckyball.bank_mvin %input, %input_bank <{depth = 64 : i64, stride = 1 : i64}> : memref<64x32xi8> + %matrix = im2col_ball.bank_im2col_i8 %loaded -> %output_bank <{input_side = 8 : i64, kernel = 3 : i64, stride = 1 : i64, padding = 1 : i64, tile_rows = 8 : i64}> + %stored = buckyball.bank_mvout %output, %matrix <{depth = 384 : i64, stride = 1 : i64}> : memref<384x48xi8> + buckyball.bank_release %loaded + buckyball.bank_release %stored + func.call @check_result(%output) : (memref<384x48xi8>) -> () + memref.dealloc %input : memref<64x32xi8> + memref.dealloc %output : memref<384x48xi8> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/im2col/workloads/mlir_tests/bank/im2col_k3_two_blocks_main.cpp b/examples/prefab/balls/im2col/workloads/mlir_tests/bank/im2col_k3_two_blocks_main.cpp new file mode 100644 index 000000000..896ce5133 --- /dev/null +++ b/examples/prefab/balls/im2col/workloads/mlir_tests/bank/im2col_k3_two_blocks_main.cpp @@ -0,0 +1,29 @@ +#include +#include +#include + +// 8x8 map, two channel blocks, k3 padding 1: K tile t = (ky * 3 + kx) * 2 + +// block; output group j holds tiles 6j..6j+5, line ((w / 8) * 6 + t % 6) * 8 +// + w % 8. A whole-bank DMA puts the three groups side by side in a row. +extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != 384 || lanes != 48 || stride != 48 || lane_stride != 1) + exit(1); + for (int w = 0; w < 64; ++w) + for (int t = 0; t < 18; ++t) + for (int ch = 0; ch < 16; ++ch) { + int position = t / 2, block = t % 2; + int y = w / 8 + position / 3 - 1, x = w % 8 + position % 3 - 1; + int expected = 0; + if (y >= 0 && y < 8 && x >= 0 && x < 8) + expected = y * 8 + x + (block * 16 + ch) * 3; + int line = ((w / 8) * 6 + t % 6) * 8 + w % 8; + int lane = (t / 6) * 16 + ch; + if (data[offset + line * stride + lane] != (int8_t)expected) { + printf("im2col two blocks mismatch w=%d t=%d ch=%d\n", w, t, ch); + exit(1); + } + } + printf("im2col k3 two blocks PASS\n"); +} diff --git a/examples/components/balls/int2fp/arch/Fp32Mul.scala b/examples/prefab/balls/int2fp/arch/Fp32Mul.scala similarity index 100% rename from examples/components/balls/int2fp/arch/Fp32Mul.scala rename to examples/prefab/balls/int2fp/arch/Fp32Mul.scala diff --git a/examples/components/balls/int2fp/arch/Int2Fp.scala b/examples/prefab/balls/int2fp/arch/Int2Fp.scala similarity index 100% rename from examples/components/balls/int2fp/arch/Int2Fp.scala rename to examples/prefab/balls/int2fp/arch/Int2Fp.scala diff --git a/examples/components/balls/int2fp/arch/Int2FpBall.scala b/examples/prefab/balls/int2fp/arch/Int2FpBall.scala similarity index 100% rename from examples/components/balls/int2fp/arch/Int2FpBall.scala rename to examples/prefab/balls/int2fp/arch/Int2FpBall.scala diff --git a/examples/components/balls/int2fp/configs/Int2FpBallParam.scala b/examples/prefab/balls/int2fp/configs/Int2FpBallParam.scala similarity index 100% rename from examples/components/balls/int2fp/configs/Int2FpBallParam.scala rename to examples/prefab/balls/int2fp/configs/Int2FpBallParam.scala diff --git a/examples/components/balls/int2fp/configs/default.toml b/examples/prefab/balls/int2fp/configs/default.toml similarity index 100% rename from examples/components/balls/int2fp/configs/default.toml rename to examples/prefab/balls/int2fp/configs/default.toml diff --git a/examples/components/balls/int2fp/configs/int2fp_params.proto b/examples/prefab/balls/int2fp/configs/int2fp_params.proto similarity index 100% rename from examples/components/balls/int2fp/configs/int2fp_params.proto rename to examples/prefab/balls/int2fp/configs/int2fp_params.proto diff --git a/examples/components/balls/int2fp/emu/src/dispatch.rs b/examples/prefab/balls/int2fp/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/int2fp/emu/src/dispatch.rs rename to examples/prefab/balls/int2fp/emu/src/dispatch.rs diff --git a/examples/components/balls/int2fp/emu/src/int2fp.rs b/examples/prefab/balls/int2fp/emu/src/int2fp.rs similarity index 96% rename from examples/components/balls/int2fp/emu/src/int2fp.rs rename to examples/prefab/balls/int2fp/emu/src/int2fp.rs index f550c5012..c8d25791b 100644 --- a/examples/components/balls/int2fp/emu/src/int2fp.rs +++ b/examples/prefab/balls/int2fp/emu/src/int2fp.rs @@ -1,8 +1,9 @@ -use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::ExecContext; use bebop_bemu::thread::banks::bank_lines; pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "int2fp: reads no old destination; rs1[29] must be clear"); let input_bank = rs1_b0(xs1); let scale_bank = rs1_b1(xs1); let output_bank = rs1_b2(xs1); diff --git a/examples/components/balls/int2fp/emu/src/lib.rs b/examples/prefab/balls/int2fp/emu/src/lib.rs similarity index 100% rename from examples/components/balls/int2fp/emu/src/lib.rs rename to examples/prefab/balls/int2fp/emu/src/lib.rs diff --git a/examples/components/balls/int2fp/emu/src/model.rs b/examples/prefab/balls/int2fp/emu/src/model.rs similarity index 100% rename from examples/components/balls/int2fp/emu/src/model.rs rename to examples/prefab/balls/int2fp/emu/src/model.rs diff --git a/examples/prefab/balls/int2fp/stack/src/int2fpOps.cpp b/examples/prefab/balls/int2fp/stack/src/int2fpOps.cpp new file mode 100644 index 000000000..198e63d64 --- /dev/null +++ b/examples/prefab/balls/int2fp/stack/src/int2fpOps.cpp @@ -0,0 +1,66 @@ +//===- int2fpOps.cpp - Int2FpBall operations ------------------------------===// +// +// The dialect of Int2FpBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "int2fpOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::int2fp_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "int2fpDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "int2fpOps.cpp.inc" + +void Int2FpBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "int2fpOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankInt32ToFp32Op::getSourceBanks() { + return {getValues(), getScales()}; +} +Value BankInt32ToFp32Op::getDestinationBank() { return getOutput(); } +bool BankInt32ToFp32Op::readsDestination() { return false; } +Operation *BankInt32ToFp32Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return Int32ToFp32Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getRelu()); +} + +SmallVector Int32ToFp32Op::getSourceBanks() { + return {int64_t(getValues()), int64_t(getScales())}; +} +int64_t Int32ToFp32Op::getDestinationBank() { return getOutput(); } +LogicalResult Int32ToFp32Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("INT32_TO_FP32"), + constantI64(builder, loc, + bankFields({int64_t(getValues()), int64_t(getScales())}, + getOutput(), false, rows * 4)), + constantI64(builder, loc, field(getRelu(), 0, 0))); + return success(); +} diff --git a/examples/prefab/balls/int2fp/stack/src/int2fpOps.h b/examples/prefab/balls/int2fp/stack/src/int2fpOps.h new file mode 100644 index 000000000..2358922d6 --- /dev/null +++ b/examples/prefab/balls/int2fp/stack/src/int2fpOps.h @@ -0,0 +1,20 @@ +//===- int2fpOps.h - Int2FpBall operations --------------------------------===// + +#ifndef INT2FP_OPS_H +#define INT2FP_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "int2fpDialect.h.inc" + +#define GET_OP_CLASSES +#include "int2fpOps.h.inc" + +#endif // INT2FP_OPS_H diff --git a/examples/prefab/balls/int2fp/stack/src/int2fpOps.td b/examples/prefab/balls/int2fp/stack/src/int2fpOps.td new file mode 100644 index 000000000..f4fb98859 --- /dev/null +++ b/examples/prefab/balls/int2fp/stack/src/int2fpOps.td @@ -0,0 +1,44 @@ +//===- int2fpOps.td - Int2FpBall operations ----------------*- tablegen -*-===// +// +// int32_to_fp32: `values` holds `rows` rows of sixteen i32 (four bank rows +// each), `scales` the sixteen f32 scales of the columns (four bank rows); the +// output receives the f32 value * scale of every element, with `relu` +// negative results replaced by zero. +// +//===----------------------------------------------------------------------===// + +#ifndef INT2FP_OPS +#define INT2FP_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def Int2FpBall_Dialect : Dialect { + let name = "int2fp_ball"; + let cppNamespace = "::buddy::int2fp_ball"; + let summary = "The operations of Int2FpBall"; +} + +class Int2FpBall_Op traits = []> + : Op])>; + +def BankInt32ToFp32Op : Int2FpBall_Op<"bank_int32_to_fp32", + [DeclareOpInterfaceMethods]> { + let summary = "int32_to_fp32 on virtual banks"; + let arguments = (ins I64:$values, I64:$scales, I64:$output, I64Attr:$rows, + UnitAttr:$relu); + let results = (outs I64:$outputOut); + let assemblyFormat = "$values `,` $scales `->` $output prop-dict attr-dict"; +} + +def Int32ToFp32Op : Int2FpBall_Op<"int32_to_fp32", + [DeclareOpInterfaceMethods]> { + let summary = "int32_to_fp32 on physical banks"; + let arguments = (ins I64Attr:$values, I64Attr:$scales, I64Attr:$output, + I64Attr:$rows, UnitAttr:$relu); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // INT2FP_OPS diff --git a/examples/components/balls/int2fp/verify/README.md b/examples/prefab/balls/int2fp/verify/README.md similarity index 100% rename from examples/components/balls/int2fp/verify/README.md rename to examples/prefab/balls/int2fp/verify/README.md diff --git a/examples/components/balls/int2fp/verify/casegen/Cargo.lock b/examples/prefab/balls/int2fp/verify/casegen/Cargo.lock similarity index 100% rename from examples/components/balls/int2fp/verify/casegen/Cargo.lock rename to examples/prefab/balls/int2fp/verify/casegen/Cargo.lock diff --git a/examples/components/balls/int2fp/verify/casegen/Cargo.toml b/examples/prefab/balls/int2fp/verify/casegen/Cargo.toml similarity index 100% rename from examples/components/balls/int2fp/verify/casegen/Cargo.toml rename to examples/prefab/balls/int2fp/verify/casegen/Cargo.toml diff --git a/examples/components/balls/int2fp/verify/casegen/src/casegen.rs b/examples/prefab/balls/int2fp/verify/casegen/src/casegen.rs similarity index 100% rename from examples/components/balls/int2fp/verify/casegen/src/casegen.rs rename to examples/prefab/balls/int2fp/verify/casegen/src/casegen.rs diff --git a/examples/components/balls/int2fp/verify/casegen/src/lib.rs b/examples/prefab/balls/int2fp/verify/casegen/src/lib.rs similarity index 100% rename from examples/components/balls/int2fp/verify/casegen/src/lib.rs rename to examples/prefab/balls/int2fp/verify/casegen/src/lib.rs diff --git a/examples/components/balls/int2fp/verify/filelists/int2fp_ball.f b/examples/prefab/balls/int2fp/verify/filelists/int2fp_ball.f similarity index 100% rename from examples/components/balls/int2fp/verify/filelists/int2fp_ball.f rename to examples/prefab/balls/int2fp/verify/filelists/int2fp_ball.f diff --git a/examples/components/balls/int2fp/verify/src/common/int2fp_defs.svh b/examples/prefab/balls/int2fp/verify/src/common/int2fp_defs.svh similarity index 100% rename from examples/components/balls/int2fp/verify/src/common/int2fp_defs.svh rename to examples/prefab/balls/int2fp/verify/src/common/int2fp_defs.svh diff --git a/examples/components/balls/int2fp/verify/src/common/int2fp_items.svh b/examples/prefab/balls/int2fp/verify/src/common/int2fp_items.svh similarity index 100% rename from examples/components/balls/int2fp/verify/src/common/int2fp_items.svh rename to examples/prefab/balls/int2fp/verify/src/common/int2fp_items.svh diff --git a/examples/components/balls/int2fp/verify/src/cov/int2fp_cov.svh b/examples/prefab/balls/int2fp/verify/src/cov/int2fp_cov.svh similarity index 100% rename from examples/components/balls/int2fp/verify/src/cov/int2fp_cov.svh rename to examples/prefab/balls/int2fp/verify/src/cov/int2fp_cov.svh diff --git a/examples/components/balls/int2fp/verify/src/env/int2fp_env.svh b/examples/prefab/balls/int2fp/verify/src/env/int2fp_env.svh similarity index 100% rename from examples/components/balls/int2fp/verify/src/env/int2fp_env.svh rename to examples/prefab/balls/int2fp/verify/src/env/int2fp_env.svh diff --git a/examples/components/balls/int2fp/verify/src/env/int2fp_scoreboard.svh b/examples/prefab/balls/int2fp/verify/src/env/int2fp_scoreboard.svh similarity index 100% rename from examples/components/balls/int2fp/verify/src/env/int2fp_scoreboard.svh rename to examples/prefab/balls/int2fp/verify/src/env/int2fp_scoreboard.svh diff --git a/examples/components/balls/int2fp/verify/src/pkg/int2fp_pkg.sv b/examples/prefab/balls/int2fp/verify/src/pkg/int2fp_pkg.sv similarity index 100% rename from examples/components/balls/int2fp/verify/src/pkg/int2fp_pkg.sv rename to examples/prefab/balls/int2fp/verify/src/pkg/int2fp_pkg.sv diff --git a/examples/components/balls/int2fp/verify/src/seq/int2fp_sequences.svh b/examples/prefab/balls/int2fp/verify/src/seq/int2fp_sequences.svh similarity index 100% rename from examples/components/balls/int2fp/verify/src/seq/int2fp_sequences.svh rename to examples/prefab/balls/int2fp/verify/src/seq/int2fp_sequences.svh diff --git a/examples/components/balls/int2fp/verify/src/tb_top.sv b/examples/prefab/balls/int2fp/verify/src/tb_top.sv similarity index 100% rename from examples/components/balls/int2fp/verify/src/tb_top.sv rename to examples/prefab/balls/int2fp/verify/src/tb_top.sv diff --git a/examples/components/balls/int2fp/verify/src/tests/int2fp_test.svh b/examples/prefab/balls/int2fp/verify/src/tests/int2fp_test.svh similarity index 100% rename from examples/components/balls/int2fp/verify/src/tests/int2fp_test.svh rename to examples/prefab/balls/int2fp/verify/src/tests/int2fp_test.svh diff --git a/examples/components/balls/int2fp/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/int2fp/workloads/ctests/CMakeLists.txt similarity index 100% rename from examples/components/balls/int2fp/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/int2fp/workloads/ctests/CMakeLists.txt diff --git a/examples/components/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c similarity index 100% rename from examples/components/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c rename to examples/prefab/balls/int2fp/workloads/ctests/int2fp_channel_base_test.c diff --git a/examples/components/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c similarity index 100% rename from examples/components/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c rename to examples/prefab/balls/int2fp/workloads/ctests/int2fp_channel_lanes_test.c diff --git a/examples/components/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c similarity index 100% rename from examples/components/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c rename to examples/prefab/balls/int2fp/workloads/ctests/int2fp_channel_two_rows_test.c diff --git a/examples/components/balls/int2fp/workloads/ctests/int2fp_groups_test.c b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_groups_test.c similarity index 100% rename from examples/components/balls/int2fp/workloads/ctests/int2fp_groups_test.c rename to examples/prefab/balls/int2fp/workloads/ctests/int2fp_groups_test.c diff --git a/examples/components/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c similarity index 100% rename from examples/components/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c rename to examples/prefab/balls/int2fp/workloads/ctests/int2fp_tensor_groups_test.c diff --git a/examples/components/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c similarity index 100% rename from examples/components/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c rename to examples/prefab/balls/int2fp/workloads/ctests/int2fp_tensor_rows_test.c diff --git a/examples/components/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c b/examples/prefab/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c similarity index 100% rename from examples/components/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c rename to examples/prefab/balls/int2fp/workloads/ctests/int2fp_tensor_values_test.c diff --git a/examples/components/balls/int2fp/workloads/isa/int2fp.h b/examples/prefab/balls/int2fp/workloads/isa/int2fp.h similarity index 92% rename from examples/components/balls/int2fp/workloads/isa/int2fp.h rename to examples/prefab/balls/int2fp/workloads/isa/int2fp.h index 3e2a87e8d..542714e21 100644 --- a/examples/components/balls/int2fp/workloads/isa/int2fp.h +++ b/examples/prefab/balls/int2fp/workloads/isa/int2fp.h @@ -13,6 +13,6 @@ BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(input_bank) | BB_BANK1(scale_bank) | \ BB_BANK2(output_bank) | BB_ITER(iter), \ FIELD((uint64_t)(relu), 0, 0), \ - BB_FUNC7(INT32_TO_FP32)) + FUNCT7(int2fp, INT32_TO_FP32)) #endif // _BB_INT2FP_H_ diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/int2fp/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/int2fp/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/int2fp/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/components/balls/int2fp/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/int2fp/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir b/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir similarity index 92% rename from examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir rename to examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir index 3c85da332..6b78184a3 100644 --- a/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir +++ b/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4.mlir @@ -17,7 +17,7 @@ func.func @main() -> i8 { buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mvin %input <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> buckyball.mvin %scales <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - buckyball.custom "int32_to_fp32" <{sources = array, destination = 3 : i64}> {rows = 1 : i64, relu = false} + int2fp_ball.int32_to_fp32 <{values = 1 : i64, scales = 2 : i64, output = 3 : i64, rows = 1 : i64}> buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> func.call @check_result(%output) : (memref<4x4xf32>) -> () buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4_main.cpp b/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4_main.cpp similarity index 100% rename from examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4_main.cpp rename to examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_4x4_main.cpp diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir b/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir similarity index 92% rename from examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir rename to examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir index 87601fee7..43295b799 100644 --- a/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir +++ b/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4.mlir @@ -16,7 +16,7 @@ func.func @main() -> i8 { buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mvin %input <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> buckyball.mvin %scales <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - buckyball.custom "int32_to_fp32" <{sources = array, destination = 3 : i64}> {rows = 1 : i64, relu = true} + int2fp_ball.int32_to_fp32 <{values = 1 : i64, scales = 2 : i64, output = 3 : i64, rows = 1 : i64, relu}> buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> func.call @check_result(%output) : (memref<4x4xf32>) -> () buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4_main.cpp b/examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4_main.cpp similarity index 100% rename from examples/components/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4_main.cpp rename to examples/prefab/balls/int2fp/workloads/mlir_tests/ball/int2fp_relu_4x4_main.cpp diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt similarity index 62% rename from examples/components/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt index e977427a0..e33d64409 100644 --- a/examples/components/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt +++ b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/CMakeLists.txt @@ -3,9 +3,7 @@ if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) endif() set(BUCKYBALL_MLIR_TEST_PREFIX bank) -set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") -file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") -string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") +set(_lines ${DESIGN_PROFILES_${BUCKYBALL_MLIR_ACTIVE_TARGET}_THREAD_MEM_BANK_ENTRIES}) # int32_to_fp32 rows hold sixteen i32, four bank rows each. math(EXPR BB_INT2FP_ROWS "${_lines} / 4") add_buckyball_mlir_test(int2fp_unit_scale TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir similarity index 90% rename from examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir rename to examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir index 6da48fada..88c67b989 100644 --- a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir +++ b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4.mlir @@ -17,7 +17,7 @@ func.func @main() -> i8 { %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> %loaded_scales = buckyball.bank_mvin %scales, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - %result = buckyball.bank_custom "int32_to_fp32" (%loaded_input, %loaded_scales) -> %output_bank {rows = 1 : i64, relu = false} + %result = int2fp_ball.bank_int32_to_fp32 %loaded_input, %loaded_scales -> %output_bank <{rows = 1 : i64}> %stored = buckyball.bank_mvout %output, %result <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> func.call @check_result(%output) : (memref<4x4xf32>) -> () buckyball.bank_release %loaded_input diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4_main.cpp b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4_main.cpp similarity index 100% rename from examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4_main.cpp rename to examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_4x4_main.cpp diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir similarity index 90% rename from examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir rename to examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir index 3372b743d..dd8aeee10 100644 --- a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir +++ b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4.mlir @@ -16,7 +16,7 @@ func.func @main() -> i8 { %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> %loaded_scales = buckyball.bank_mvin %scales, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - %result = buckyball.bank_custom "int32_to_fp32" (%loaded_input, %loaded_scales) -> %output_bank {rows = 1 : i64, relu = true} + %result = int2fp_ball.bank_int32_to_fp32 %loaded_input, %loaded_scales -> %output_bank <{rows = 1 : i64, relu}> %stored = buckyball.bank_mvout %output, %result <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> func.call @check_result(%output) : (memref<4x4xf32>) -> () buckyball.bank_release %loaded_input diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4_main.cpp b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4_main.cpp similarity index 100% rename from examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4_main.cpp rename to examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_relu_4x4_main.cpp diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in similarity index 93% rename from examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in rename to examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in index b2882d24a..b99e22b34 100644 --- a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in +++ b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale.mlir.in @@ -23,7 +23,7 @@ func.func @main() -> i8 { %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> %loaded_scale = buckyball.bank_mvin %scale, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - %result = buckyball.bank_custom "int32_to_fp32" (%loaded_input, %loaded_scale) -> %output_bank {rows = @BB_INT2FP_ROWS@ : i64, relu = false} + %result = int2fp_ball.bank_int32_to_fp32 %loaded_input, %loaded_scale -> %output_bank <{rows = @BB_INT2FP_ROWS@ : i64}> %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32> func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xf32>) -> () buckyball.bank_release %loaded_input diff --git a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp similarity index 73% rename from examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp rename to examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp index 578c4e1f5..ec0ee48e4 100644 --- a/examples/components/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp +++ b/examples/prefab/balls/int2fp/workloads/mlir_tests/bank/int2fp_unit_scale_main.cpp @@ -1,12 +1,11 @@ -#include #include #include #include extern "C" void check_result(float *, float *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { - if (rows != BANK_LINES || lanes != BANK_WIDTH / 32 || stride != lanes || - lane_stride != 1) + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != THREAD_MEM_BANK_WIDTH / 32 || + stride != lanes || lane_stride != 1) exit(1); for (int row = 0; row < rows; ++row) for (int lane = 0; lane < lanes; ++lane) @@ -14,5 +13,5 @@ extern "C" void check_result(float *, float *data, int64_t offset, int64_t rows, printf("int2fp bank mismatch row=%d lane=%d\n", row, lane); exit(1); } - printf("int2fp bank PASS rows=%d\n", BANK_LINES); + printf("int2fp bank PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); } diff --git a/examples/components/balls/int8add/arch/Int8AddBall.scala b/examples/prefab/balls/int8add/arch/Int8AddBall.scala similarity index 100% rename from examples/components/balls/int8add/arch/Int8AddBall.scala rename to examples/prefab/balls/int8add/arch/Int8AddBall.scala diff --git a/examples/components/balls/int8add/arch/PipelinedAddRawFN.scala b/examples/prefab/balls/int8add/arch/PipelinedAddRawFN.scala similarity index 100% rename from examples/components/balls/int8add/arch/PipelinedAddRawFN.scala rename to examples/prefab/balls/int8add/arch/PipelinedAddRawFN.scala diff --git a/examples/components/balls/int8add/configs/default.toml b/examples/prefab/balls/int8add/configs/default.toml similarity index 100% rename from examples/components/balls/int8add/configs/default.toml rename to examples/prefab/balls/int8add/configs/default.toml diff --git a/examples/components/balls/int8add/configs/int8add_params.proto b/examples/prefab/balls/int8add/configs/int8add_params.proto similarity index 100% rename from examples/components/balls/int8add/configs/int8add_params.proto rename to examples/prefab/balls/int8add/configs/int8add_params.proto diff --git a/examples/components/balls/int8add/emu/src/dispatch.rs b/examples/prefab/balls/int8add/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/int8add/emu/src/dispatch.rs rename to examples/prefab/balls/int8add/emu/src/dispatch.rs diff --git a/examples/components/balls/int8add/emu/src/int8add.rs b/examples/prefab/balls/int8add/emu/src/int8add.rs similarity index 96% rename from examples/components/balls/int8add/emu/src/int8add.rs rename to examples/prefab/balls/int8add/emu/src/int8add.rs index 70379adac..c54c9b09e 100644 --- a/examples/components/balls/int8add/emu/src/int8add.rs +++ b/examples/prefab/balls/int8add/emu/src/int8add.rs @@ -1,11 +1,12 @@ use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; -use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::ExecContext; pub struct Int8Add; impl Int8Add { pub fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext, relu: bool) -> u64 { + assert!(!rs1_read_old(xs1), "int8add: reads no old destination; rs1[29] must be clear"); let lhs_bank = rs1_b0(xs1); let rhs_bank = rs1_b1(xs1); let output_bank = rs1_b2(xs1); diff --git a/examples/components/balls/int8add/emu/src/lib.rs b/examples/prefab/balls/int8add/emu/src/lib.rs similarity index 100% rename from examples/components/balls/int8add/emu/src/lib.rs rename to examples/prefab/balls/int8add/emu/src/lib.rs diff --git a/examples/prefab/balls/int8add/stack/src/int8addOps.cpp b/examples/prefab/balls/int8add/stack/src/int8addOps.cpp new file mode 100644 index 000000000..ab66f875b --- /dev/null +++ b/examples/prefab/balls/int8add/stack/src/int8addOps.cpp @@ -0,0 +1,78 @@ +//===- int8addOps.cpp - Int8AddBall operations ----------------------------===// +// +// The dialect of Int8AddBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "int8addOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" + +using namespace mlir; +using namespace ::buddy::int8add_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "int8addDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "int8addOps.cpp.inc" + +void Int8AddBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "int8addOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankAddI8Op::getSourceBanks() { return {getA(), getB()}; } +Value BankAddI8Op::getDestinationBank() { return getC(); } +bool BankAddI8Op::readsDestination() { return false; } +Operation *BankAddI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return AddI8Op::create(builder, getLoc(), getLhsRatio(), getRhsRatio(), + sources[0], sources[1], destination, getRows(), + getRelu()); +} + +SmallVector AddI8Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t AddI8Op::getDestinationBank() { return getC(); } +LogicalResult AddI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), a = getA(), b = getB(), c = getC(); + if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || + a == b || a == c || b == c) + return emitOpError("add_i8 requires distinct banks and rows within a bank"); + // rs2: lhs_ratio bits in [31:0], rhs_ratio bits in [63:32]. + auto ratioBits = [&](Value ratio) -> Value { + Value bits = + arith::BitcastOp::create(builder, loc, builder.getI32Type(), ratio); + return arith::ExtUIOp::create(builder, loc, builder.getI64Type(), bits); + }; + Value rhs = arith::ShLIOp::create(builder, loc, ratioBits(getRhsRatio()), + constantI64(builder, loc, 32)); + Value rs2 = arith::OrIOp::create(builder, loc, ratioBits(getLhsRatio()), rhs); + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7( + getRelu() ? "INT8ADD_RELU" : "INT8ADD"), + constantI64(builder, loc, bankFields({a, b}, c, false, rows)), + rs2); + return success(); +} diff --git a/examples/prefab/balls/int8add/stack/src/int8addOps.h b/examples/prefab/balls/int8add/stack/src/int8addOps.h new file mode 100644 index 000000000..ce6a56a64 --- /dev/null +++ b/examples/prefab/balls/int8add/stack/src/int8addOps.h @@ -0,0 +1,20 @@ +//===- int8addOps.h - Int8AddBall operations ------------------------------===// + +#ifndef INT8ADD_OPS_H +#define INT8ADD_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "int8addDialect.h.inc" + +#define GET_OP_CLASSES +#include "int8addOps.h.inc" + +#endif // INT8ADD_OPS_H diff --git a/examples/prefab/balls/int8add/stack/src/int8addOps.td b/examples/prefab/balls/int8add/stack/src/int8addOps.td new file mode 100644 index 000000000..673fc926e --- /dev/null +++ b/examples/prefab/balls/int8add/stack/src/int8addOps.td @@ -0,0 +1,46 @@ +//===- int8addOps.td - Int8AddBall operations --------------*- tablegen -*-===// +// +// add_i8: c receives `rows` rows of sixteen i8, +// round_half_even(a * lhs_ratio + b * rhs_ratio) saturated to i8, and with +// `relu` clamped at zero. lhs_ratio and rhs_ratio are f32, both finite and +// positive. +// +//===----------------------------------------------------------------------===// + +#ifndef INT8ADD_OPS +#define INT8ADD_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def Int8AddBall_Dialect : Dialect { + let name = "int8add_ball"; + let cppNamespace = "::buddy::int8add_ball"; + let summary = "The operations of Int8AddBall"; +} + +class Int8AddBall_Op traits = []> + : Op])>; + +def BankAddI8Op : Int8AddBall_Op<"bank_add_i8", + [DeclareOpInterfaceMethods]> { + let summary = "add_i8 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, F32:$lhs_ratio, + F32:$rhs_ratio, I64Attr:$rows, UnitAttr:$relu); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c `ratios` `(` $lhs_ratio `,` " + "$rhs_ratio `)` prop-dict attr-dict"; +} + +def AddI8Op : Int8AddBall_Op<"add_i8", + [DeclareOpInterfaceMethods]> { + let summary = "add_i8 on physical banks"; + let arguments = (ins F32:$lhs_ratio, F32:$rhs_ratio, I64Attr:$a, + I64Attr:$b, I64Attr:$c, I64Attr:$rows, UnitAttr:$relu); + let assemblyFormat = "`ratios` `(` $lhs_ratio `,` $rhs_ratio `)` prop-dict " + "attr-dict"; +} + +#endif // INT8ADD_OPS diff --git a/examples/components/balls/int8add/verify/casegen/Cargo.lock b/examples/prefab/balls/int8add/verify/casegen/Cargo.lock similarity index 100% rename from examples/components/balls/int8add/verify/casegen/Cargo.lock rename to examples/prefab/balls/int8add/verify/casegen/Cargo.lock diff --git a/examples/components/balls/int8add/verify/casegen/Cargo.toml b/examples/prefab/balls/int8add/verify/casegen/Cargo.toml similarity index 100% rename from examples/components/balls/int8add/verify/casegen/Cargo.toml rename to examples/prefab/balls/int8add/verify/casegen/Cargo.toml diff --git a/examples/components/balls/int8add/verify/casegen/src/casegen.rs b/examples/prefab/balls/int8add/verify/casegen/src/casegen.rs similarity index 100% rename from examples/components/balls/int8add/verify/casegen/src/casegen.rs rename to examples/prefab/balls/int8add/verify/casegen/src/casegen.rs diff --git a/examples/components/balls/int8add/verify/casegen/src/lib.rs b/examples/prefab/balls/int8add/verify/casegen/src/lib.rs similarity index 100% rename from examples/components/balls/int8add/verify/casegen/src/lib.rs rename to examples/prefab/balls/int8add/verify/casegen/src/lib.rs diff --git a/examples/components/balls/int8add/verify/filelists/int8add_ball.f b/examples/prefab/balls/int8add/verify/filelists/int8add_ball.f similarity index 100% rename from examples/components/balls/int8add/verify/filelists/int8add_ball.f rename to examples/prefab/balls/int8add/verify/filelists/int8add_ball.f diff --git a/examples/components/balls/int8add/verify/src/common/int8add_defs.svh b/examples/prefab/balls/int8add/verify/src/common/int8add_defs.svh similarity index 100% rename from examples/components/balls/int8add/verify/src/common/int8add_defs.svh rename to examples/prefab/balls/int8add/verify/src/common/int8add_defs.svh diff --git a/examples/components/balls/int8add/verify/src/common/int8add_items.svh b/examples/prefab/balls/int8add/verify/src/common/int8add_items.svh similarity index 100% rename from examples/components/balls/int8add/verify/src/common/int8add_items.svh rename to examples/prefab/balls/int8add/verify/src/common/int8add_items.svh diff --git a/examples/components/balls/int8add/verify/src/cov/int8add_cov.svh b/examples/prefab/balls/int8add/verify/src/cov/int8add_cov.svh similarity index 100% rename from examples/components/balls/int8add/verify/src/cov/int8add_cov.svh rename to examples/prefab/balls/int8add/verify/src/cov/int8add_cov.svh diff --git a/examples/components/balls/int8add/verify/src/env/int8add_env.svh b/examples/prefab/balls/int8add/verify/src/env/int8add_env.svh similarity index 100% rename from examples/components/balls/int8add/verify/src/env/int8add_env.svh rename to examples/prefab/balls/int8add/verify/src/env/int8add_env.svh diff --git a/examples/components/balls/int8add/verify/src/env/int8add_scoreboard.svh b/examples/prefab/balls/int8add/verify/src/env/int8add_scoreboard.svh similarity index 100% rename from examples/components/balls/int8add/verify/src/env/int8add_scoreboard.svh rename to examples/prefab/balls/int8add/verify/src/env/int8add_scoreboard.svh diff --git a/examples/components/balls/int8add/verify/src/pkg/int8add_pkg.sv b/examples/prefab/balls/int8add/verify/src/pkg/int8add_pkg.sv similarity index 100% rename from examples/components/balls/int8add/verify/src/pkg/int8add_pkg.sv rename to examples/prefab/balls/int8add/verify/src/pkg/int8add_pkg.sv diff --git a/examples/components/balls/int8add/verify/src/seq/int8add_sequences.svh b/examples/prefab/balls/int8add/verify/src/seq/int8add_sequences.svh similarity index 100% rename from examples/components/balls/int8add/verify/src/seq/int8add_sequences.svh rename to examples/prefab/balls/int8add/verify/src/seq/int8add_sequences.svh diff --git a/examples/components/balls/int8add/verify/src/tb_top.sv b/examples/prefab/balls/int8add/verify/src/tb_top.sv similarity index 100% rename from examples/components/balls/int8add/verify/src/tb_top.sv rename to examples/prefab/balls/int8add/verify/src/tb_top.sv diff --git a/examples/components/balls/int8add/verify/src/tests/int8add_test.svh b/examples/prefab/balls/int8add/verify/src/tests/int8add_test.svh similarity index 100% rename from examples/components/balls/int8add/verify/src/tests/int8add_test.svh rename to examples/prefab/balls/int8add/verify/src/tests/int8add_test.svh diff --git a/examples/components/balls/int8add/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/int8add/workloads/ctests/CMakeLists.txt similarity index 100% rename from examples/components/balls/int8add/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/int8add/workloads/ctests/CMakeLists.txt diff --git a/examples/components/balls/int8add/workloads/ctests/int8add_relu_test.c b/examples/prefab/balls/int8add/workloads/ctests/int8add_relu_test.c similarity index 94% rename from examples/components/balls/int8add/workloads/ctests/int8add_relu_test.c rename to examples/prefab/balls/int8add/workloads/ctests/int8add_relu_test.c index 394b9f6b6..4b70596fa 100644 --- a/examples/components/balls/int8add/workloads/ctests/int8add_relu_test.c +++ b/examples/prefab/balls/int8add/workloads/ctests/int8add_relu_test.c @@ -5,7 +5,7 @@ #include #include -enum { ROWS = 7, LANES = BANK_WIDTH / 8, VALUES = ROWS * LANES }; +enum { ROWS = 7, LANES = THREAD_MEM_BANK_WIDTH / 8, VALUES = ROWS * LANES }; static int8_t lhs[VALUES] __attribute__((aligned(64))); static int8_t rhs[VALUES] __attribute__((aligned(64))); static int8_t output[VALUES] __attribute__((aligned(64))); diff --git a/examples/components/balls/int8add/workloads/ctests/int8add_scaled_test.c b/examples/prefab/balls/int8add/workloads/ctests/int8add_scaled_test.c similarity index 94% rename from examples/components/balls/int8add/workloads/ctests/int8add_scaled_test.c rename to examples/prefab/balls/int8add/workloads/ctests/int8add_scaled_test.c index 6dfabf33e..bb0bf8ea8 100644 --- a/examples/components/balls/int8add/workloads/ctests/int8add_scaled_test.c +++ b/examples/prefab/balls/int8add/workloads/ctests/int8add_scaled_test.c @@ -5,7 +5,7 @@ #include #include -enum { ROWS = 7, LANES = BANK_WIDTH / 8, VALUES = ROWS * LANES }; +enum { ROWS = 7, LANES = THREAD_MEM_BANK_WIDTH / 8, VALUES = ROWS * LANES }; static int8_t lhs[VALUES] __attribute__((aligned(64))); static int8_t rhs[VALUES] __attribute__((aligned(64))); static int8_t output[VALUES] __attribute__((aligned(64))); diff --git a/examples/components/balls/int8add/workloads/isa/int8add.h b/examples/prefab/balls/int8add/workloads/isa/int8add.h similarity index 91% rename from examples/components/balls/int8add/workloads/isa/int8add.h rename to examples/prefab/balls/int8add/workloads/isa/int8add.h index 63ce07e3a..0bc519669 100644 --- a/examples/components/balls/int8add/workloads/isa/int8add.h +++ b/examples/prefab/balls/int8add/workloads/isa/int8add.h @@ -18,13 +18,13 @@ static inline uint64_t bb_int8add_rs2(float lhs_ratio, float rhs_ratio) { BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(lhs_bank) | BB_BANK1(rhs_bank) | \ BB_BANK2(output_bank) | BB_ITER(iter)), \ bb_int8add_rs2(lhs_ratio, rhs_ratio), \ - BB_FUNC7(INT8ADD)) + FUNCT7(int8add, INT8ADD)) #define bb_int8add_relu(lhs_bank, rhs_bank, output_bank, iter, lhs_ratio, \ rhs_ratio) \ BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(lhs_bank) | BB_BANK1(rhs_bank) | \ BB_BANK2(output_bank) | BB_ITER(iter)), \ bb_int8add_rs2(lhs_ratio, rhs_ratio), \ - BB_FUNC7(INT8ADD_RELU)) + FUNCT7(int8add, INT8ADD_RELU)) #endif // _BB_INT8ADD_H_ diff --git a/examples/components/balls/int8add/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/int8add/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/int8add/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/int8add/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/int8add/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/int8add/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/components/balls/int8add/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/int8add/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir b/examples/prefab/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir similarity index 92% rename from examples/components/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir rename to examples/prefab/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir index fb57444c5..f7d6c59ed 100644 --- a/examples/components/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir +++ b/examples/prefab/balls/int8add/workloads/mlir_tests/ball/int8add_relu.mlir @@ -27,7 +27,7 @@ func.func @main() -> i8 { buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mvin %a <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> buckyball.mvin %b <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> - buckyball.custom "add_i8" args(%ratio, %ratio : f32, f32) <{sources = array, destination = 3 : i64}> {rows = 4 : i64, relu = true} + int8add_ball.add_i8 ratios(%ratio, %ratio) <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 4 : i64, relu}> buckyball.mvout %out <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> diff --git a/examples/components/balls/int8add/workloads/mlir_tests/ball/int8add_relu_main.cpp b/examples/prefab/balls/int8add/workloads/mlir_tests/ball/int8add_relu_main.cpp similarity index 100% rename from examples/components/balls/int8add/workloads/mlir_tests/ball/int8add_relu_main.cpp rename to examples/prefab/balls/int8add/workloads/mlir_tests/ball/int8add_relu_main.cpp diff --git a/examples/components/balls/int8add/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/int8add/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/components/balls/int8add/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/int8add/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in b/examples/prefab/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in similarity index 93% rename from examples/components/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in rename to examples/prefab/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in index 904ac8775..ecc83f9b1 100644 --- a/examples/components/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in +++ b/examples/prefab/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale.mlir.in @@ -29,7 +29,7 @@ func.func @main() -> i8 { %out_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %a_loaded = buckyball.bank_mvin %a, %a_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> %b_loaded = buckyball.bank_mvin %b, %b_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %result = buckyball.bank_custom "add_i8" (%a_loaded, %b_loaded) -> %out_bank args(%ratio, %ratio : f32, f32) {rows = @BB_BANK_LINES@ : i64, relu = false} + %result = int8add_ball.bank_add_i8 %a_loaded, %b_loaded -> %out_bank ratios(%ratio, %ratio) <{rows = @BB_BANK_LINES@ : i64}> %stored = buckyball.bank_mvout %out, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> buckyball.bank_release %a_loaded buckyball.bank_release %b_loaded diff --git a/examples/components/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp b/examples/prefab/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp similarity index 68% rename from examples/components/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp rename to examples/prefab/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp index cea5836c7..be1087dd1 100644 --- a/examples/components/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp +++ b/examples/prefab/balls/int8add/workloads/mlir_tests/bank/int8add_identity_scale_main.cpp @@ -1,17 +1,16 @@ -#include #include #include #include extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { - if (rows != BANK_LINES || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != THREAD_MEM_BANK_WIDTH / 8 || + stride != lanes || lane_stride != 1) exit(1); for (int i = 0; i < rows * lanes; ++i) if (data[offset + i] != 0) { printf("int8add bank mismatch index=%d\n", i); exit(1); } - printf("int8add bank PASS rows=%d\n", BANK_LINES); + printf("int8add bank PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); } diff --git a/examples/components/balls/int8mul/arch/Int8MulBall.scala b/examples/prefab/balls/int8mul/arch/Int8MulBall.scala similarity index 100% rename from examples/components/balls/int8mul/arch/Int8MulBall.scala rename to examples/prefab/balls/int8mul/arch/Int8MulBall.scala diff --git a/examples/components/balls/int8mul/configs/default.toml b/examples/prefab/balls/int8mul/configs/default.toml similarity index 100% rename from examples/components/balls/int8mul/configs/default.toml rename to examples/prefab/balls/int8mul/configs/default.toml diff --git a/examples/components/balls/int8mul/configs/int8mul_params.proto b/examples/prefab/balls/int8mul/configs/int8mul_params.proto similarity index 100% rename from examples/components/balls/int8mul/configs/int8mul_params.proto rename to examples/prefab/balls/int8mul/configs/int8mul_params.proto diff --git a/examples/components/balls/int8mul/emu/src/dispatch.rs b/examples/prefab/balls/int8mul/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/int8mul/emu/src/dispatch.rs rename to examples/prefab/balls/int8mul/emu/src/dispatch.rs diff --git a/examples/components/balls/int8mul/emu/src/int8mul.rs b/examples/prefab/balls/int8mul/emu/src/int8mul.rs similarity index 96% rename from examples/components/balls/int8mul/emu/src/int8mul.rs rename to examples/prefab/balls/int8mul/emu/src/int8mul.rs index 00efae535..bfe7822f3 100644 --- a/examples/components/balls/int8mul/emu/src/int8mul.rs +++ b/examples/prefab/balls/int8mul/emu/src/int8mul.rs @@ -1,5 +1,5 @@ use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; -use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::ExecContext; pub struct Int8Mul; @@ -32,6 +32,7 @@ impl Int8Mul { } pub fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "int8mul: reads no old destination; rs1[29] must be clear"); let gate_bank = rs1_b0(xs1); let input_bank = rs1_b1(xs1); let output_bank = rs1_b2(xs1); diff --git a/examples/components/balls/int8mul/emu/src/lib.rs b/examples/prefab/balls/int8mul/emu/src/lib.rs similarity index 100% rename from examples/components/balls/int8mul/emu/src/lib.rs rename to examples/prefab/balls/int8mul/emu/src/lib.rs diff --git a/examples/prefab/balls/int8mul/stack/src/int8mulOps.cpp b/examples/prefab/balls/int8mul/stack/src/int8mulOps.cpp new file mode 100644 index 000000000..2b0f0fb58 --- /dev/null +++ b/examples/prefab/balls/int8mul/stack/src/int8mulOps.cpp @@ -0,0 +1,75 @@ +//===- int8mulOps.cpp - Int8MulBall operations ----------------------------===// +// +// The dialect of Int8MulBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "int8mulOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" + +using namespace mlir; +using namespace ::buddy::int8mul_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "int8mulDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "int8mulOps.cpp.inc" + +void Int8MulBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "int8mulOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankGateMulI8Op::getSourceBanks() { + return {getGate(), getInput()}; +} +Value BankGateMulI8Op::getDestinationBank() { return getOutput(); } +bool BankGateMulI8Op::readsDestination() { return false; } +Operation *BankGateMulI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return GateMulI8Op::create(builder, getLoc(), getRatio(), sources[0], + sources[1], destination, getRows()); +} + +SmallVector GateMulI8Op::getSourceBanks() { + return {int64_t(getGate()), int64_t(getInput())}; +} +int64_t GateMulI8Op::getDestinationBank() { return getOutput(); } +LogicalResult GateMulI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), gate = getGate(), input = getInput(), + output = getOutput(); + if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || + gate == input || gate == output || input == output) + return emitOpError( + "gate_mul_i8 requires distinct banks and rows within a bank"); + // rs2: ratio bits in [31:0]; the gate row in [63:32] is the first row. + Value bits = + arith::BitcastOp::create(builder, loc, builder.getI32Type(), getRatio()); + Value rs2 = arith::ExtUIOp::create(builder, loc, builder.getI64Type(), bits); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("INT8MUL"), + constantI64(builder, loc, bankFields({gate, input}, output, false, rows)), + rs2); + return success(); +} diff --git a/examples/prefab/balls/int8mul/stack/src/int8mulOps.h b/examples/prefab/balls/int8mul/stack/src/int8mulOps.h new file mode 100644 index 000000000..8143c2631 --- /dev/null +++ b/examples/prefab/balls/int8mul/stack/src/int8mulOps.h @@ -0,0 +1,20 @@ +//===- int8mulOps.h - Int8MulBall operations ------------------------------===// + +#ifndef INT8MUL_OPS_H +#define INT8MUL_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "int8mulDialect.h.inc" + +#define GET_OP_CLASSES +#include "int8mulOps.h.inc" + +#endif // INT8MUL_OPS_H diff --git a/examples/prefab/balls/int8mul/stack/src/int8mulOps.td b/examples/prefab/balls/int8mul/stack/src/int8mulOps.td new file mode 100644 index 000000000..0d56c5034 --- /dev/null +++ b/examples/prefab/balls/int8mul/stack/src/int8mulOps.td @@ -0,0 +1,46 @@ +//===- int8mulOps.td - Int8MulBall operations --------------*- tablegen -*-===// +// +// gate_mul_i8: `gate` holds the gate, one row of sixteen i8; `input` holds +// `rows` rows of sixteen i8. The output receives +// clamp(round_half_even(gate * x * ratio), -128, 127) per column, in every +// group of the banks; the three banks have the same number of groups. `ratio` +// is f32, finite and positive. +// +//===----------------------------------------------------------------------===// + +#ifndef INT8MUL_OPS +#define INT8MUL_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def Int8MulBall_Dialect : Dialect { + let name = "int8mul_ball"; + let cppNamespace = "::buddy::int8mul_ball"; + let summary = "The operations of Int8MulBall"; +} + +class Int8MulBall_Op traits = []> + : Op])>; + +def BankGateMulI8Op : Int8MulBall_Op<"bank_gate_mul_i8", + [DeclareOpInterfaceMethods]> { + let summary = "gate_mul_i8 on virtual banks"; + let arguments = (ins I64:$gate, I64:$input, I64:$output, F32:$ratio, + I64Attr:$rows); + let results = (outs I64:$outputOut); + let assemblyFormat = "$gate `,` $input `->` $output `ratio` `(` $ratio `)` " + "prop-dict attr-dict"; +} + +def GateMulI8Op : Int8MulBall_Op<"gate_mul_i8", + [DeclareOpInterfaceMethods]> { + let summary = "gate_mul_i8 on physical banks"; + let arguments = (ins F32:$ratio, I64Attr:$gate, I64Attr:$input, + I64Attr:$output, I64Attr:$rows); + let assemblyFormat = "`ratio` `(` $ratio `)` prop-dict attr-dict"; +} + +#endif // INT8MUL_OPS diff --git a/examples/components/balls/int8mul/verify/casegen/Cargo.lock b/examples/prefab/balls/int8mul/verify/casegen/Cargo.lock similarity index 100% rename from examples/components/balls/int8mul/verify/casegen/Cargo.lock rename to examples/prefab/balls/int8mul/verify/casegen/Cargo.lock diff --git a/examples/components/balls/int8mul/verify/casegen/Cargo.toml b/examples/prefab/balls/int8mul/verify/casegen/Cargo.toml similarity index 100% rename from examples/components/balls/int8mul/verify/casegen/Cargo.toml rename to examples/prefab/balls/int8mul/verify/casegen/Cargo.toml diff --git a/examples/components/balls/int8mul/verify/casegen/src/casegen.rs b/examples/prefab/balls/int8mul/verify/casegen/src/casegen.rs similarity index 100% rename from examples/components/balls/int8mul/verify/casegen/src/casegen.rs rename to examples/prefab/balls/int8mul/verify/casegen/src/casegen.rs diff --git a/examples/components/balls/int8mul/verify/casegen/src/config.rs b/examples/prefab/balls/int8mul/verify/casegen/src/config.rs similarity index 100% rename from examples/components/balls/int8mul/verify/casegen/src/config.rs rename to examples/prefab/balls/int8mul/verify/casegen/src/config.rs diff --git a/examples/components/balls/int8mul/verify/casegen/src/lib.rs b/examples/prefab/balls/int8mul/verify/casegen/src/lib.rs similarity index 100% rename from examples/components/balls/int8mul/verify/casegen/src/lib.rs rename to examples/prefab/balls/int8mul/verify/casegen/src/lib.rs diff --git a/examples/components/balls/int8mul/verify/filelists/int8mul_ball.f b/examples/prefab/balls/int8mul/verify/filelists/int8mul_ball.f similarity index 100% rename from examples/components/balls/int8mul/verify/filelists/int8mul_ball.f rename to examples/prefab/balls/int8mul/verify/filelists/int8mul_ball.f diff --git a/examples/components/balls/int8mul/verify/src/common/int8mul_defs.svh b/examples/prefab/balls/int8mul/verify/src/common/int8mul_defs.svh similarity index 100% rename from examples/components/balls/int8mul/verify/src/common/int8mul_defs.svh rename to examples/prefab/balls/int8mul/verify/src/common/int8mul_defs.svh diff --git a/examples/components/balls/int8mul/verify/src/common/int8mul_items.svh b/examples/prefab/balls/int8mul/verify/src/common/int8mul_items.svh similarity index 100% rename from examples/components/balls/int8mul/verify/src/common/int8mul_items.svh rename to examples/prefab/balls/int8mul/verify/src/common/int8mul_items.svh diff --git a/examples/components/balls/int8mul/verify/src/cov/int8mul_cov.svh b/examples/prefab/balls/int8mul/verify/src/cov/int8mul_cov.svh similarity index 100% rename from examples/components/balls/int8mul/verify/src/cov/int8mul_cov.svh rename to examples/prefab/balls/int8mul/verify/src/cov/int8mul_cov.svh diff --git a/examples/components/balls/int8mul/verify/src/env/int8mul_env.svh b/examples/prefab/balls/int8mul/verify/src/env/int8mul_env.svh similarity index 100% rename from examples/components/balls/int8mul/verify/src/env/int8mul_env.svh rename to examples/prefab/balls/int8mul/verify/src/env/int8mul_env.svh diff --git a/examples/components/balls/int8mul/verify/src/env/int8mul_scoreboard.svh b/examples/prefab/balls/int8mul/verify/src/env/int8mul_scoreboard.svh similarity index 100% rename from examples/components/balls/int8mul/verify/src/env/int8mul_scoreboard.svh rename to examples/prefab/balls/int8mul/verify/src/env/int8mul_scoreboard.svh diff --git a/examples/components/balls/int8mul/verify/src/pkg/int8mul_pkg.sv b/examples/prefab/balls/int8mul/verify/src/pkg/int8mul_pkg.sv similarity index 100% rename from examples/components/balls/int8mul/verify/src/pkg/int8mul_pkg.sv rename to examples/prefab/balls/int8mul/verify/src/pkg/int8mul_pkg.sv diff --git a/examples/components/balls/int8mul/verify/src/seq/int8mul_sequences.svh b/examples/prefab/balls/int8mul/verify/src/seq/int8mul_sequences.svh similarity index 100% rename from examples/components/balls/int8mul/verify/src/seq/int8mul_sequences.svh rename to examples/prefab/balls/int8mul/verify/src/seq/int8mul_sequences.svh diff --git a/examples/components/balls/int8mul/verify/src/tb_top.sv b/examples/prefab/balls/int8mul/verify/src/tb_top.sv similarity index 100% rename from examples/components/balls/int8mul/verify/src/tb_top.sv rename to examples/prefab/balls/int8mul/verify/src/tb_top.sv diff --git a/examples/components/balls/int8mul/verify/src/tests/int8mul_test.svh b/examples/prefab/balls/int8mul/verify/src/tests/int8mul_test.svh similarity index 100% rename from examples/components/balls/int8mul/verify/src/tests/int8mul_test.svh rename to examples/prefab/balls/int8mul/verify/src/tests/int8mul_test.svh diff --git a/examples/components/balls/int8mul/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/int8mul/workloads/ctests/CMakeLists.txt similarity index 100% rename from examples/components/balls/int8mul/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/int8mul/workloads/ctests/CMakeLists.txt diff --git a/examples/components/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c b/examples/prefab/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c similarity index 90% rename from examples/components/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c rename to examples/prefab/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c index d5e58b1cc..b118287b9 100644 --- a/examples/components/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c +++ b/examples/prefab/balls/int8mul/workloads/ctests/int8mul_large_gate_row_test.c @@ -12,8 +12,9 @@ static int8_t input[ROWS * 16] __attribute__((aligned(64))); static int8_t output[ROWS * 16] __attribute__((aligned(64))); int main(void) { - if (BANK_LINES <= GATE_ROW) { - printf("int8mul large gate row SKIP bank_lines=%d\n", BANK_LINES); + if (THREAD_MEM_BANK_ENTRIES <= GATE_ROW) { + printf("int8mul large gate row SKIP bank_lines=%d\n", + THREAD_MEM_BANK_ENTRIES); return 0; } diff --git a/examples/components/balls/int8mul/workloads/ctests/int8mul_per_channel_gate_test.c b/examples/prefab/balls/int8mul/workloads/ctests/int8mul_per_channel_gate_test.c similarity index 100% rename from examples/components/balls/int8mul/workloads/ctests/int8mul_per_channel_gate_test.c rename to examples/prefab/balls/int8mul/workloads/ctests/int8mul_per_channel_gate_test.c diff --git a/examples/components/balls/int8mul/workloads/isa/int8mul.h b/examples/prefab/balls/int8mul/workloads/isa/int8mul.h similarity index 91% rename from examples/components/balls/int8mul/workloads/isa/int8mul.h rename to examples/prefab/balls/int8mul/workloads/isa/int8mul.h index f71d104ce..a308411d3 100644 --- a/examples/components/balls/int8mul/workloads/isa/int8mul.h +++ b/examples/prefab/balls/int8mul/workloads/isa/int8mul.h @@ -3,7 +3,6 @@ #include #include -#include #include static inline uint64_t bb_int8mul_rs2(float ratio, uint32_t gate_row) { @@ -19,6 +18,6 @@ static inline uint64_t bb_int8mul_rs2(float ratio, uint32_t gate_row) { BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(gate_bank) | BB_BANK1(input_bank) | \ BB_BANK2(output_bank) | BB_ITER(iter)), \ bb_int8mul_rs2(ratio, gate_row), \ - BB_FUNC7(INT8MUL)) + FUNCT7(int8mul, INT8MUL)) #endif // _BB_INT8MUL_H_ diff --git a/examples/components/balls/int8mul/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/int8mul/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/int8mul/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/int8mul/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/int8mul/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/int8mul/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/components/balls/int8mul/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/int8mul/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir b/examples/prefab/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir similarity index 93% rename from examples/components/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir rename to examples/prefab/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir index ddbd6f1f0..7e8f98361 100644 --- a/examples/components/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir +++ b/examples/prefab/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate.mlir @@ -24,7 +24,7 @@ func.func @main() -> i8 { buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mvin %gate <{bank = 1 : i64, depth = 1 : i64, stride = 1 : i64}> : memref<1x16xi8> buckyball.mvin %input <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> - buckyball.custom "gate_mul_i8" args(%ratio : f32) <{sources = array, destination = 3 : i64}> {rows = 4 : i64} + int8mul_ball.gate_mul_i8 ratio(%ratio) <{gate = 1 : i64, input = 2 : i64, output = 3 : i64, rows = 4 : i64}> buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> func.call @check_result(%output) : (memref<4x16xi8>) -> () buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> diff --git a/examples/components/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate_main.cpp b/examples/prefab/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate_main.cpp similarity index 100% rename from examples/components/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate_main.cpp rename to examples/prefab/balls/int8mul/workloads/mlir_tests/ball/int8mul_gate_main.cpp diff --git a/examples/components/balls/int8mul/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/int8mul/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/components/balls/int8mul/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/int8mul/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in b/examples/prefab/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in similarity index 93% rename from examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in rename to examples/prefab/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in index 3dd8ace9e..cbc2a8bcc 100644 --- a/examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in +++ b/examples/prefab/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate.mlir.in @@ -24,7 +24,7 @@ func.func @main() -> i8 { %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded_gate = buckyball.bank_mvin %gate, %gate_bank <{depth = 1 : i64, stride = 1 : i64}> : memref<1x@BB_BANK_I8_LANES@xi8> %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %result = buckyball.bank_custom "gate_mul_i8" (%loaded_gate, %loaded_input) -> %output_bank args(%ratio : f32) {rows = @BB_BANK_LINES@ : i64} + %result = int8mul_ball.bank_gate_mul_i8 %loaded_gate, %loaded_input -> %output_bank ratio(%ratio) <{rows = @BB_BANK_LINES@ : i64}> %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> func.call @check_result(%output) : (memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8>) -> () buckyball.bank_release %loaded_gate diff --git a/examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp b/examples/prefab/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp similarity index 72% rename from examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp rename to examples/prefab/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp index 805bb757d..048f32e2b 100644 --- a/examples/components/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp +++ b/examples/prefab/balls/int8mul/workloads/mlir_tests/bank/int8mul_unit_gate_main.cpp @@ -1,12 +1,11 @@ -#include #include #include #include extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { - if (rows != BANK_LINES || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != THREAD_MEM_BANK_WIDTH / 8 || + stride != lanes || lane_stride != 1) exit(1); for (int row = 0; row < rows; ++row) for (int lane = 0; lane < lanes; ++lane) @@ -14,5 +13,5 @@ extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, printf("int8mul bank mismatch row=%d lane=%d\n", row, lane); exit(1); } - printf("int8mul bank PASS rows=%d\n", BANK_LINES); + printf("int8mul bank PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); } diff --git a/examples/components/balls/lut/arch/LutBall.scala b/examples/prefab/balls/lut/arch/LutBall.scala similarity index 100% rename from examples/components/balls/lut/arch/LutBall.scala rename to examples/prefab/balls/lut/arch/LutBall.scala diff --git a/examples/components/balls/lut/configs/default.toml b/examples/prefab/balls/lut/configs/default.toml similarity index 100% rename from examples/components/balls/lut/configs/default.toml rename to examples/prefab/balls/lut/configs/default.toml diff --git a/examples/components/balls/lut/configs/lut_params.proto b/examples/prefab/balls/lut/configs/lut_params.proto similarity index 100% rename from examples/components/balls/lut/configs/lut_params.proto rename to examples/prefab/balls/lut/configs/lut_params.proto diff --git a/examples/components/balls/lut/emu/src/dispatch.rs b/examples/prefab/balls/lut/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/lut/emu/src/dispatch.rs rename to examples/prefab/balls/lut/emu/src/dispatch.rs diff --git a/examples/components/balls/lut/emu/src/lib.rs b/examples/prefab/balls/lut/emu/src/lib.rs similarity index 100% rename from examples/components/balls/lut/emu/src/lib.rs rename to examples/prefab/balls/lut/emu/src/lib.rs diff --git a/examples/components/balls/lut/emu/src/lut.rs b/examples/prefab/balls/lut/emu/src/lut.rs similarity index 95% rename from examples/components/balls/lut/emu/src/lut.rs rename to examples/prefab/balls/lut/emu/src/lut.rs index b253922ec..bd19ba924 100644 --- a/examples/components/balls/lut/emu/src/lut.rs +++ b/examples/prefab/balls/lut/emu/src/lut.rs @@ -1,11 +1,12 @@ use bebop_bemu::thread::banks::bank_lines; -use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::{BallModel, ExecContext}; pub struct Lut; impl BallModel for Lut { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "lut: reads no old destination; rs1[29] must be clear"); let input_bank = rs1_b0(xs1); let lut_bank = rs1_b1(xs1); let output_bank = rs1_b2(xs1); diff --git a/examples/prefab/balls/lut/stack/src/lutOps.cpp b/examples/prefab/balls/lut/stack/src/lutOps.cpp new file mode 100644 index 000000000..1296b863f --- /dev/null +++ b/examples/prefab/balls/lut/stack/src/lutOps.cpp @@ -0,0 +1,68 @@ +//===- lutOps.cpp - LutBall operations ------------------------------------===// +// +// The dialect of LutBall, its virtual operations building their physical ones +// and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "lutOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::lut_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "lutDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "lutOps.cpp.inc" + +void LutBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "lutOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect( + [](DialectRegistry ®istry) { registry.insert(); }), + true); +} // namespace + +SmallVector BankLookupI8Op::getSourceBanks() { + return {getCodes(), getTable()}; +} +Value BankLookupI8Op::getDestinationBank() { return getOutput(); } +bool BankLookupI8Op::readsDestination() { return false; } +Operation *BankLookupI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return LookupI8Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows()); +} + +SmallVector LookupI8Op::getSourceBanks() { + return {int64_t(getCodes()), int64_t(getTable())}; +} +int64_t LookupI8Op::getDestinationBank() { return getOutput(); } +LogicalResult LookupI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), codes = getCodes(), table = getTable(), + output = getOutput(); + if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || + codes == table || codes == output || table == output) + return emitOpError( + "lookup_i8 requires distinct banks and rows within bank depth"); + emitInstruction(builder, loc, buckyball_target::getBuckyballFunct7("LUT"), + constantI64(builder, loc, + bankFields({codes, table}, output, false, rows)), + constantI64(builder, loc, 0)); + return success(); +} diff --git a/examples/prefab/balls/lut/stack/src/lutOps.h b/examples/prefab/balls/lut/stack/src/lutOps.h new file mode 100644 index 000000000..d4c719ad7 --- /dev/null +++ b/examples/prefab/balls/lut/stack/src/lutOps.h @@ -0,0 +1,20 @@ +//===- lutOps.h - LutBall operations --------------------------------------===// + +#ifndef LUT_OPS_H +#define LUT_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "lutDialect.h.inc" + +#define GET_OP_CLASSES +#include "lutOps.h.inc" + +#endif // LUT_OPS_H diff --git a/examples/prefab/balls/lut/stack/src/lutOps.td b/examples/prefab/balls/lut/stack/src/lutOps.td new file mode 100644 index 000000000..8a9c19d87 --- /dev/null +++ b/examples/prefab/balls/lut/stack/src/lutOps.td @@ -0,0 +1,43 @@ +//===- lutOps.td - LutBall operations ----------------------*- tablegen -*-===// +// +// lookup_i8: `codes` holds `rows` rows of sixteen i8 codes, `table` the +// table: one group of 256 entries shared by every channel, or four groups with +// 256 entries per channel (channel * 256 + code, 1024 bytes per group). The +// output receives table[code] for every element. +// +//===----------------------------------------------------------------------===// + +#ifndef LUT_OPS +#define LUT_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def LutBall_Dialect : Dialect { + let name = "lut_ball"; + let cppNamespace = "::buddy::lut_ball"; + let summary = "The operations of LutBall"; +} + +class LutBall_Op traits = []> + : Op])>; + +def BankLookupI8Op : LutBall_Op<"bank_lookup_i8", + [DeclareOpInterfaceMethods]> { + let summary = "lookup_i8 on virtual banks"; + let arguments = (ins I64:$codes, I64:$table, I64:$output, I64Attr:$rows); + let results = (outs I64:$outputOut); + let assemblyFormat = "$codes `,` $table `->` $output prop-dict attr-dict"; +} + +def LookupI8Op : LutBall_Op<"lookup_i8", + [DeclareOpInterfaceMethods]> { + let summary = "lookup_i8 on physical banks"; + let arguments = (ins I64Attr:$codes, I64Attr:$table, I64Attr:$output, + I64Attr:$rows); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // LUT_OPS diff --git a/examples/components/balls/lut/verify/casegen/Cargo.lock b/examples/prefab/balls/lut/verify/casegen/Cargo.lock similarity index 100% rename from examples/components/balls/lut/verify/casegen/Cargo.lock rename to examples/prefab/balls/lut/verify/casegen/Cargo.lock diff --git a/examples/components/balls/lut/verify/casegen/Cargo.toml b/examples/prefab/balls/lut/verify/casegen/Cargo.toml similarity index 100% rename from examples/components/balls/lut/verify/casegen/Cargo.toml rename to examples/prefab/balls/lut/verify/casegen/Cargo.toml diff --git a/examples/components/balls/lut/verify/casegen/src/casegen.rs b/examples/prefab/balls/lut/verify/casegen/src/casegen.rs similarity index 100% rename from examples/components/balls/lut/verify/casegen/src/casegen.rs rename to examples/prefab/balls/lut/verify/casegen/src/casegen.rs diff --git a/examples/components/balls/lut/verify/casegen/src/lib.rs b/examples/prefab/balls/lut/verify/casegen/src/lib.rs similarity index 100% rename from examples/components/balls/lut/verify/casegen/src/lib.rs rename to examples/prefab/balls/lut/verify/casegen/src/lib.rs diff --git a/examples/components/balls/lut/verify/filelists/lut_ball.f b/examples/prefab/balls/lut/verify/filelists/lut_ball.f similarity index 100% rename from examples/components/balls/lut/verify/filelists/lut_ball.f rename to examples/prefab/balls/lut/verify/filelists/lut_ball.f diff --git a/examples/components/balls/lut/verify/src/common/lut_defs.svh b/examples/prefab/balls/lut/verify/src/common/lut_defs.svh similarity index 100% rename from examples/components/balls/lut/verify/src/common/lut_defs.svh rename to examples/prefab/balls/lut/verify/src/common/lut_defs.svh diff --git a/examples/components/balls/lut/verify/src/common/lut_items.svh b/examples/prefab/balls/lut/verify/src/common/lut_items.svh similarity index 100% rename from examples/components/balls/lut/verify/src/common/lut_items.svh rename to examples/prefab/balls/lut/verify/src/common/lut_items.svh diff --git a/examples/components/balls/lut/verify/src/cov/lut_cov.svh b/examples/prefab/balls/lut/verify/src/cov/lut_cov.svh similarity index 100% rename from examples/components/balls/lut/verify/src/cov/lut_cov.svh rename to examples/prefab/balls/lut/verify/src/cov/lut_cov.svh diff --git a/examples/components/balls/lut/verify/src/env/lut_env.svh b/examples/prefab/balls/lut/verify/src/env/lut_env.svh similarity index 100% rename from examples/components/balls/lut/verify/src/env/lut_env.svh rename to examples/prefab/balls/lut/verify/src/env/lut_env.svh diff --git a/examples/components/balls/lut/verify/src/env/lut_scoreboard.svh b/examples/prefab/balls/lut/verify/src/env/lut_scoreboard.svh similarity index 100% rename from examples/components/balls/lut/verify/src/env/lut_scoreboard.svh rename to examples/prefab/balls/lut/verify/src/env/lut_scoreboard.svh diff --git a/examples/components/balls/lut/verify/src/pkg/lut_pkg.sv b/examples/prefab/balls/lut/verify/src/pkg/lut_pkg.sv similarity index 100% rename from examples/components/balls/lut/verify/src/pkg/lut_pkg.sv rename to examples/prefab/balls/lut/verify/src/pkg/lut_pkg.sv diff --git a/examples/components/balls/lut/verify/src/seq/lut_sequences.svh b/examples/prefab/balls/lut/verify/src/seq/lut_sequences.svh similarity index 100% rename from examples/components/balls/lut/verify/src/seq/lut_sequences.svh rename to examples/prefab/balls/lut/verify/src/seq/lut_sequences.svh diff --git a/examples/components/balls/lut/verify/src/tb_top.sv b/examples/prefab/balls/lut/verify/src/tb_top.sv similarity index 100% rename from examples/components/balls/lut/verify/src/tb_top.sv rename to examples/prefab/balls/lut/verify/src/tb_top.sv diff --git a/examples/components/balls/lut/verify/src/tests/lut_test.svh b/examples/prefab/balls/lut/verify/src/tests/lut_test.svh similarity index 100% rename from examples/components/balls/lut/verify/src/tests/lut_test.svh rename to examples/prefab/balls/lut/verify/src/tests/lut_test.svh diff --git a/examples/components/balls/lut/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/lut/workloads/ctests/CMakeLists.txt similarity index 100% rename from examples/components/balls/lut/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/lut/workloads/ctests/CMakeLists.txt diff --git a/examples/components/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c b/examples/prefab/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c similarity index 94% rename from examples/components/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c rename to examples/prefab/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c index 333632749..bb7fbf0eb 100644 --- a/examples/components/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c +++ b/examples/prefab/balls/lut/workloads/ctests/lut_channel_table_lookup_test.c @@ -5,7 +5,7 @@ #include #include -enum { LANES = BANK_WIDTH / 8, ROWS = 4, VALUES = ROWS * LANES }; +enum { LANES = THREAD_MEM_BANK_WIDTH / 8, ROWS = 4, VALUES = ROWS * LANES }; static int8_t input[VALUES] __attribute__((aligned(64))); static int8_t table[64][64] __attribute__((aligned(64))); static int8_t output[VALUES] __attribute__((aligned(64))); diff --git a/examples/components/balls/lut/workloads/ctests/lut_uniform_lookup_test.c b/examples/prefab/balls/lut/workloads/ctests/lut_uniform_lookup_test.c similarity index 92% rename from examples/components/balls/lut/workloads/ctests/lut_uniform_lookup_test.c rename to examples/prefab/balls/lut/workloads/ctests/lut_uniform_lookup_test.c index 06fa85965..83290c503 100644 --- a/examples/components/balls/lut/workloads/ctests/lut_uniform_lookup_test.c +++ b/examples/prefab/balls/lut/workloads/ctests/lut_uniform_lookup_test.c @@ -5,7 +5,7 @@ #include #include -enum { LANES = BANK_WIDTH / 8, ROWS = 4, VALUES = ROWS * LANES }; +enum { LANES = THREAD_MEM_BANK_WIDTH / 8, ROWS = 4, VALUES = ROWS * LANES }; static int8_t input[VALUES] __attribute__((aligned(64))); static int8_t table[256] __attribute__((aligned(64))); static int8_t output[VALUES] __attribute__((aligned(64))); diff --git a/examples/components/balls/lut/workloads/isa/lut.h b/examples/prefab/balls/lut/workloads/isa/lut.h similarity index 88% rename from examples/components/balls/lut/workloads/isa/lut.h rename to examples/prefab/balls/lut/workloads/isa/lut.h index 8710978b5..33781bb1a 100644 --- a/examples/components/balls/lut/workloads/isa/lut.h +++ b/examples/prefab/balls/lut/workloads/isa/lut.h @@ -7,6 +7,6 @@ #define bb_lut(input_bank, lut_bank, output_bank, iter) \ BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(input_bank) | BB_BANK1(lut_bank) | \ BB_BANK2(output_bank) | BB_ITER(iter), \ - 0, BB_FUNC7(LUT)) + 0, FUNCT7(lut, LUT)) #endif // _BB_LUT_H_ diff --git a/examples/components/balls/lut/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/lut/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/lut/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/lut/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/lut/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/lut/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/components/balls/lut/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/lut/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir b/examples/prefab/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir similarity index 95% rename from examples/components/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir rename to examples/prefab/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir index a374cd87a..492b60cba 100644 --- a/examples/components/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir +++ b/examples/prefab/balls/lut/workloads/mlir_tests/ball/lut_identity.mlir @@ -35,7 +35,7 @@ func.func @main() -> i8 { buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mvin %input <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> buckyball.mvin %table <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - buckyball.custom "lookup_i8" <{sources = array, destination = 3 : i64}> {rows = 4 : i64} + lut_ball.lookup_i8 <{codes = 1 : i64, table = 2 : i64, output = 3 : i64, rows = 4 : i64}> buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> diff --git a/examples/components/balls/lut/workloads/mlir_tests/ball/lut_identity_main.cpp b/examples/prefab/balls/lut/workloads/mlir_tests/ball/lut_identity_main.cpp similarity index 100% rename from examples/components/balls/lut/workloads/mlir_tests/ball/lut_identity_main.cpp rename to examples/prefab/balls/lut/workloads/mlir_tests/ball/lut_identity_main.cpp diff --git a/examples/components/balls/lut/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/lut/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/components/balls/lut/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/lut/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in b/examples/prefab/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in similarity index 95% rename from examples/components/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in rename to examples/prefab/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in index e0f7a147e..202afc25e 100644 --- a/examples/components/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in +++ b/examples/prefab/balls/lut/workloads/mlir_tests/bank/lut_identity_table.mlir.in @@ -37,7 +37,7 @@ func.func @main() -> i8 { %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> %loaded_table = buckyball.bank_mvin %table, %table_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - %result = buckyball.bank_custom "lookup_i8" (%loaded_input, %loaded_table) -> %output_bank {rows = @BB_BANK_LINES@ : i64} + %result = lut_ball.bank_lookup_i8 %loaded_input, %loaded_table -> %output_bank <{rows = @BB_BANK_LINES@ : i64}> %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> buckyball.bank_release %loaded_input buckyball.bank_release %loaded_table diff --git a/examples/components/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp b/examples/prefab/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp similarity index 69% rename from examples/components/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp rename to examples/prefab/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp index b7c91f060..eadceb075 100644 --- a/examples/components/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp +++ b/examples/prefab/balls/lut/workloads/mlir_tests/bank/lut_identity_table_main.cpp @@ -1,17 +1,16 @@ -#include #include #include #include extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { - if (rows != BANK_LINES || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != THREAD_MEM_BANK_WIDTH / 8 || + stride != lanes || lane_stride != 1) exit(1); for (int i = 0; i < rows * lanes; ++i) if (data[offset + i] != (int8_t)i) { printf("lut bank mismatch index=%d\n", i); exit(1); } - printf("lut bank PASS rows=%d\n", BANK_LINES); + printf("lut bank PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); } diff --git a/examples/components/balls/matadd/README.md b/examples/prefab/balls/matadd/README.md similarity index 100% rename from examples/components/balls/matadd/README.md rename to examples/prefab/balls/matadd/README.md diff --git a/examples/components/balls/matadd/arch/MatAdd.scala b/examples/prefab/balls/matadd/arch/MatAdd.scala similarity index 100% rename from examples/components/balls/matadd/arch/MatAdd.scala rename to examples/prefab/balls/matadd/arch/MatAdd.scala diff --git a/examples/components/balls/matadd/arch/MatAddBall.scala b/examples/prefab/balls/matadd/arch/MatAddBall.scala similarity index 100% rename from examples/components/balls/matadd/arch/MatAddBall.scala rename to examples/prefab/balls/matadd/arch/MatAddBall.scala diff --git a/examples/components/balls/matadd/configs/default.toml b/examples/prefab/balls/matadd/configs/default.toml similarity index 100% rename from examples/components/balls/matadd/configs/default.toml rename to examples/prefab/balls/matadd/configs/default.toml diff --git a/examples/components/balls/matadd/configs/matadd_params.proto b/examples/prefab/balls/matadd/configs/matadd_params.proto similarity index 100% rename from examples/components/balls/matadd/configs/matadd_params.proto rename to examples/prefab/balls/matadd/configs/matadd_params.proto diff --git a/examples/components/balls/matadd/emu/src/64_matadd.rs b/examples/prefab/balls/matadd/emu/src/64_matadd.rs similarity index 95% rename from examples/components/balls/matadd/emu/src/64_matadd.rs rename to examples/prefab/balls/matadd/emu/src/64_matadd.rs index fdd71258e..f38d59b8f 100644 --- a/examples/components/balls/matadd/emu/src/64_matadd.rs +++ b/examples/prefab/balls/matadd/emu/src/64_matadd.rs @@ -1,11 +1,12 @@ use bebop_bemu::thread::banks::bank_size; -use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::{BallModel, ExecContext}; pub struct MatAdd; impl BallModel for MatAdd { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "matadd: reads no old destination; rs1[29] must be clear"); let a_bank = rs1_b0(xs1); let b_bank = rs1_b1(xs1); let c_bank = rs1_b2(xs1); diff --git a/examples/components/balls/matadd/emu/src/dispatch.rs b/examples/prefab/balls/matadd/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/matadd/emu/src/dispatch.rs rename to examples/prefab/balls/matadd/emu/src/dispatch.rs diff --git a/examples/components/balls/matadd/emu/src/lib.rs b/examples/prefab/balls/matadd/emu/src/lib.rs similarity index 100% rename from examples/components/balls/matadd/emu/src/lib.rs rename to examples/prefab/balls/matadd/emu/src/lib.rs diff --git a/examples/prefab/balls/matadd/stack/src/mataddOps.cpp b/examples/prefab/balls/matadd/stack/src/mataddOps.cpp new file mode 100644 index 000000000..922d29ce1 --- /dev/null +++ b/examples/prefab/balls/matadd/stack/src/mataddOps.cpp @@ -0,0 +1,67 @@ +//===- mataddOps.cpp - MatAddBall operations ------------------------------===// +// +// The dialect of MatAddBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "mataddOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::matadd_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "mataddDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "mataddOps.cpp.inc" + +void MatAddBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "mataddOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankAddI32Op::getSourceBanks() { + return {getA(), getB()}; +} +Value BankAddI32Op::getDestinationBank() { return getC(); } +bool BankAddI32Op::readsDestination() { return false; } +Operation *BankAddI32Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return AddI32Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows()); +} + +SmallVector AddI32Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t AddI32Op::getDestinationBank() { return getC(); } +LogicalResult AddI32Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), a = getA(), b = getB(), c = getC(); + if (rows <= 0 || rows > buckyball_target::getBuckyballTarget().bankDepth || + a == b || a == c || b == c) + return emitOpError( + "add_i32 requires distinct banks and rows within a bank"); + emitInstruction(builder, loc, buckyball_target::getBuckyballFunct7("MATADD"), + constantI64(builder, loc, bankFields({a, b}, c, false, rows)), + constantI64(builder, loc, 0)); + return success(); +} diff --git a/examples/prefab/balls/matadd/stack/src/mataddOps.h b/examples/prefab/balls/matadd/stack/src/mataddOps.h new file mode 100644 index 000000000..ce07f9267 --- /dev/null +++ b/examples/prefab/balls/matadd/stack/src/mataddOps.h @@ -0,0 +1,20 @@ +//===- mataddOps.h - MatAddBall operations --------------------------------===// + +#ifndef MATADD_OPS_H +#define MATADD_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "mataddDialect.h.inc" + +#define GET_OP_CLASSES +#include "mataddOps.h.inc" + +#endif // MATADD_OPS_H diff --git a/examples/prefab/balls/matadd/stack/src/mataddOps.td b/examples/prefab/balls/matadd/stack/src/mataddOps.td new file mode 100644 index 000000000..81265507d --- /dev/null +++ b/examples/prefab/balls/matadd/stack/src/mataddOps.td @@ -0,0 +1,40 @@ +//===- mataddOps.td - MatAddBall operations ----------------*- tablegen -*-===// +// +// add_i32: C = A + B modulo 2^32 over `rows` rows of four i32 in every group +// of the banks; A, B and C have the same number of groups. +// +//===----------------------------------------------------------------------===// + +#ifndef MATADD_OPS +#define MATADD_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def MatAddBall_Dialect : Dialect { + let name = "matadd_ball"; + let cppNamespace = "::buddy::matadd_ball"; + let summary = "The operations of MatAddBall"; +} + +class MatAddBall_Op traits = []> + : Op])>; + +def BankAddI32Op : MatAddBall_Op<"bank_add_i32", + [DeclareOpInterfaceMethods]> { + let summary = "add_i32 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def AddI32Op : MatAddBall_Op<"add_i32", + [DeclareOpInterfaceMethods]> { + let summary = "add_i32 on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // MATADD_OPS diff --git a/examples/components/balls/matadd/verify/casegen/Cargo.lock b/examples/prefab/balls/matadd/verify/casegen/Cargo.lock similarity index 100% rename from examples/components/balls/matadd/verify/casegen/Cargo.lock rename to examples/prefab/balls/matadd/verify/casegen/Cargo.lock diff --git a/examples/components/balls/matadd/verify/casegen/Cargo.toml b/examples/prefab/balls/matadd/verify/casegen/Cargo.toml similarity index 100% rename from examples/components/balls/matadd/verify/casegen/Cargo.toml rename to examples/prefab/balls/matadd/verify/casegen/Cargo.toml diff --git a/examples/components/balls/matadd/verify/casegen/src/lib.rs b/examples/prefab/balls/matadd/verify/casegen/src/lib.rs similarity index 100% rename from examples/components/balls/matadd/verify/casegen/src/lib.rs rename to examples/prefab/balls/matadd/verify/casegen/src/lib.rs diff --git a/examples/components/balls/matadd/verify/filelists/matadd_ball.f b/examples/prefab/balls/matadd/verify/filelists/matadd_ball.f similarity index 100% rename from examples/components/balls/matadd/verify/filelists/matadd_ball.f rename to examples/prefab/balls/matadd/verify/filelists/matadd_ball.f diff --git a/examples/components/balls/matadd/verify/src/common/matadd_defs.svh b/examples/prefab/balls/matadd/verify/src/common/matadd_defs.svh similarity index 100% rename from examples/components/balls/matadd/verify/src/common/matadd_defs.svh rename to examples/prefab/balls/matadd/verify/src/common/matadd_defs.svh diff --git a/examples/components/balls/matadd/verify/src/common/matadd_items.svh b/examples/prefab/balls/matadd/verify/src/common/matadd_items.svh similarity index 100% rename from examples/components/balls/matadd/verify/src/common/matadd_items.svh rename to examples/prefab/balls/matadd/verify/src/common/matadd_items.svh diff --git a/examples/components/balls/matadd/verify/src/env/matadd_env.svh b/examples/prefab/balls/matadd/verify/src/env/matadd_env.svh similarity index 100% rename from examples/components/balls/matadd/verify/src/env/matadd_env.svh rename to examples/prefab/balls/matadd/verify/src/env/matadd_env.svh diff --git a/examples/components/balls/matadd/verify/src/env/matadd_scoreboard.svh b/examples/prefab/balls/matadd/verify/src/env/matadd_scoreboard.svh similarity index 100% rename from examples/components/balls/matadd/verify/src/env/matadd_scoreboard.svh rename to examples/prefab/balls/matadd/verify/src/env/matadd_scoreboard.svh diff --git a/examples/components/balls/matadd/verify/src/pkg/matadd_pkg.sv b/examples/prefab/balls/matadd/verify/src/pkg/matadd_pkg.sv similarity index 100% rename from examples/components/balls/matadd/verify/src/pkg/matadd_pkg.sv rename to examples/prefab/balls/matadd/verify/src/pkg/matadd_pkg.sv diff --git a/examples/components/balls/matadd/verify/src/seq/matadd_sequences.svh b/examples/prefab/balls/matadd/verify/src/seq/matadd_sequences.svh similarity index 100% rename from examples/components/balls/matadd/verify/src/seq/matadd_sequences.svh rename to examples/prefab/balls/matadd/verify/src/seq/matadd_sequences.svh diff --git a/examples/components/balls/matadd/verify/src/tb_top.sv b/examples/prefab/balls/matadd/verify/src/tb_top.sv similarity index 100% rename from examples/components/balls/matadd/verify/src/tb_top.sv rename to examples/prefab/balls/matadd/verify/src/tb_top.sv diff --git a/examples/components/balls/matadd/verify/src/tests/matadd_test.svh b/examples/prefab/balls/matadd/verify/src/tests/matadd_test.svh similarity index 100% rename from examples/components/balls/matadd/verify/src/tests/matadd_test.svh rename to examples/prefab/balls/matadd/verify/src/tests/matadd_test.svh diff --git a/examples/components/balls/matadd/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/matadd/workloads/ctests/CMakeLists.txt similarity index 100% rename from examples/components/balls/matadd/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/matadd/workloads/ctests/CMakeLists.txt diff --git a/examples/components/balls/matadd/workloads/ctests/matadd_i32_16row_test.c b/examples/prefab/balls/matadd/workloads/ctests/matadd_i32_16row_test.c similarity index 100% rename from examples/components/balls/matadd/workloads/ctests/matadd_i32_16row_test.c rename to examples/prefab/balls/matadd/workloads/ctests/matadd_i32_16row_test.c diff --git a/examples/components/balls/matadd/workloads/isa/matadd.h b/examples/prefab/balls/matadd/workloads/isa/matadd.h similarity index 87% rename from examples/components/balls/matadd/workloads/isa/matadd.h rename to examples/prefab/balls/matadd/workloads/isa/matadd.h index 08b701742..196478d46 100644 --- a/examples/components/balls/matadd/workloads/isa/matadd.h +++ b/examples/prefab/balls/matadd/workloads/isa/matadd.h @@ -7,6 +7,6 @@ #define bb_matadd(a_bank, b_bank, c_bank, iter) \ BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(a_bank) | BB_BANK1(b_bank) | \ BB_BANK2(c_bank) | BB_ITER(iter)), \ - 0, BB_FUNC7(MATADD)) + 0, FUNCT7(matadd, MATADD)) #endif // _BB_MATADD_H_ diff --git a/examples/components/balls/matadd/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/matadd/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/matadd/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/matadd/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/matadd/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/matadd/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/components/balls/matadd/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/matadd/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir b/examples/prefab/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir similarity index 94% rename from examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir rename to examples/prefab/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir index 36444bc3c..de019547a 100644 --- a/examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir +++ b/examples/prefab/balls/matadd/workloads/mlir_tests/ball/matadd_i32.mlir @@ -24,7 +24,7 @@ func.func @main() -> i8 { buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mvin %a <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> buckyball.mvin %b <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> - buckyball.custom "add_i32" <{sources = array, destination = 3 : i64}> {rows = 16 : i64} + matadd_ball.add_i32 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 16 : i64}> buckyball.mvout %out <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> diff --git a/examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp b/examples/prefab/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp similarity index 96% rename from examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp rename to examples/prefab/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp index a8a1752f6..961c81a41 100644 --- a/examples/components/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp +++ b/examples/prefab/balls/matadd/workloads/mlir_tests/ball/matadd_i32_main.cpp @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/examples/components/balls/matadd/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/matadd/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/components/balls/matadd/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/matadd/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in b/examples/prefab/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in similarity index 94% rename from examples/components/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in rename to examples/prefab/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in index fad61675d..21f60eb25 100644 --- a/examples/components/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in +++ b/examples/prefab/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise.mlir.in @@ -26,7 +26,7 @@ func.func @main() -> i8 { : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> %b_loaded = buckyball.bank_mvin %b, %b_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - %result = buckyball.bank_custom "add_i32" (%a_loaded, %b_loaded) -> %out_bank {rows = @BB_BANK_LINES@ : i64} + %result = matadd_ball.bank_add_i32 %a_loaded, %b_loaded -> %out_bank <{rows = @BB_BANK_LINES@ : i64}> %stored = buckyball.bank_mvout %out, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> buckyball.bank_release %a_loaded diff --git a/examples/components/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp b/examples/prefab/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp similarity index 68% rename from examples/components/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp rename to examples/prefab/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp index 9676e005b..d76c1b3e1 100644 --- a/examples/components/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp +++ b/examples/prefab/balls/matadd/workloads/mlir_tests/bank/matadd_i32_elementwise_main.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -6,8 +5,8 @@ extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { - if (rows != BANK_LINES || lanes != BANK_WIDTH / 32 || stride != lanes || - lane_stride != 1) + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != THREAD_MEM_BANK_WIDTH / 32 || + stride != lanes || lane_stride != 1) exit(1); for (int row = 0; row < rows; ++row) for (int lane = 0; lane < lanes; ++lane) @@ -15,5 +14,6 @@ extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, printf("matadd bank mismatch row=%d lane=%d\n", row, lane); exit(1); } - printf("matadd bank PASS rows=%d lanes=%d\n", BANK_LINES, BANK_WIDTH / 32); + printf("matadd bank PASS rows=%d lanes=%d\n", THREAD_MEM_BANK_ENTRIES, + THREAD_MEM_BANK_WIDTH / 32); } diff --git a/examples/components/balls/maxpool/arch/MaxPoolBall.scala b/examples/prefab/balls/maxpool/arch/MaxPoolBall.scala similarity index 100% rename from examples/components/balls/maxpool/arch/MaxPoolBall.scala rename to examples/prefab/balls/maxpool/arch/MaxPoolBall.scala diff --git a/examples/components/balls/maxpool/configs/default.toml b/examples/prefab/balls/maxpool/configs/default.toml similarity index 100% rename from examples/components/balls/maxpool/configs/default.toml rename to examples/prefab/balls/maxpool/configs/default.toml diff --git a/examples/components/balls/maxpool/configs/maxpool_params.proto b/examples/prefab/balls/maxpool/configs/maxpool_params.proto similarity index 100% rename from examples/components/balls/maxpool/configs/maxpool_params.proto rename to examples/prefab/balls/maxpool/configs/maxpool_params.proto diff --git a/examples/components/balls/maxpool/emu/src/dispatch.rs b/examples/prefab/balls/maxpool/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/maxpool/emu/src/dispatch.rs rename to examples/prefab/balls/maxpool/emu/src/dispatch.rs diff --git a/examples/components/balls/maxpool/emu/src/lib.rs b/examples/prefab/balls/maxpool/emu/src/lib.rs similarity index 100% rename from examples/components/balls/maxpool/emu/src/lib.rs rename to examples/prefab/balls/maxpool/emu/src/lib.rs diff --git a/examples/components/balls/maxpool/emu/src/maxpool.rs b/examples/prefab/balls/maxpool/emu/src/maxpool.rs similarity index 94% rename from examples/components/balls/maxpool/emu/src/maxpool.rs rename to examples/prefab/balls/maxpool/emu/src/maxpool.rs index 4b3bd12e5..cde90097c 100644 --- a/examples/components/balls/maxpool/emu/src/maxpool.rs +++ b/examples/prefab/balls/maxpool/emu/src/maxpool.rs @@ -1,4 +1,4 @@ -use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old, rs1_update}; use super::instruction::{BallModel, ExecContext}; use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; @@ -66,6 +66,12 @@ impl BallModel for MaxPool { let output_stride = ((xs2 >> 32) & 0x3f) as usize; let start_row = ((xs2 >> 38) & 0xf) as usize; let start_col = ((xs2 >> 42) & 0xf) as usize; + // The RTL writes only the output window at output_base/output_stride; + // the rest of the destination survives only an in-place update (rs1[63]). + assert!( + !rs1_read_old(xs1) && rs1_update(xs1), + "maxpool: updates its output window in place; rs1[63] must be set and rs1[29] clear" + ); if rs1_b1(xs1) != 0 { panic!("maxpool: input bank 1 must be zero"); diff --git a/examples/prefab/balls/maxpool/stack/src/maxpoolOps.cpp b/examples/prefab/balls/maxpool/stack/src/maxpoolOps.cpp new file mode 100644 index 000000000..eae4c0044 --- /dev/null +++ b/examples/prefab/balls/maxpool/stack/src/maxpoolOps.cpp @@ -0,0 +1,80 @@ +//===- maxpoolOps.cpp - MaxPoolBall operations ----------------------------===// +// +// The dialect of MaxPoolBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "maxpoolOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::maxpool_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "maxpoolDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "maxpoolOps.cpp.inc" + +void MaxPoolBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "maxpoolOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankMaxpoolI8Op::getSourceBanks() { return {getInput()}; } +Value BankMaxpoolI8Op::getDestinationBank() { return getOutput(); } +bool BankMaxpoolI8Op::readsDestination() { return false; } +Operation *BankMaxpoolI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MaxpoolI8Op::create(builder, getLoc(), sources[0], destination, + getInputSide(), getKernel(), getStride(), + getPadding()); +} + +SmallVector MaxpoolI8Op::getSourceBanks() { + return {int64_t(getInput())}; +} +int64_t MaxpoolI8Op::getDestinationBank() { return getOutput(); } +LogicalResult MaxpoolI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t input = getInput(), output = getOutput(); + if (input == output) + return emitOpError("maxpool_i8 requires distinct input and output banks"); + int64_t inputSide = getInputSide(), kernel = getKernel(), + stride = getStride(), padding = getPadding(); + int64_t span = inputSide + 2 * padding - kernel; + if (inputSide <= 0 || inputSide > 15 || kernel <= 0 || kernel > 15 || + stride <= 0 || stride > 15 || padding < 0 || padding > 15 || span < 0 || + span % stride || span / stride + 1 > 15 || + inputSide * inputSide > buckyball_target::getBuckyballTarget().bankDepth) + return emitOpError("maxpool_i8 geometry exceeds the ISA field ranges"); + int64_t outputSide = span / stride + 1; + // Both maps start at the base of their groups; output rows are packed. + uint64_t rs2 = field(inputSide, 0, 3) | field(outputSide, 4, 7) | + field(kernel, 8, 11) | field(stride, 12, 15) | + field(padding, 16, 19) | field(outputSide, 32, 37); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("MAXPOOL"), + constantI64(builder, loc, + bankFields({input}, output, false, outputSide * outputSide) | + (1ULL << 63)), + constantI64(builder, loc, rs2)); + return success(); +} diff --git a/examples/prefab/balls/maxpool/stack/src/maxpoolOps.h b/examples/prefab/balls/maxpool/stack/src/maxpoolOps.h new file mode 100644 index 000000000..4152c7f03 --- /dev/null +++ b/examples/prefab/balls/maxpool/stack/src/maxpoolOps.h @@ -0,0 +1,20 @@ +//===- maxpoolOps.h - MaxPoolBall operations ------------------------------===// + +#ifndef MAXPOOL_OPS_H +#define MAXPOOL_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "maxpoolDialect.h.inc" + +#define GET_OP_CLASSES +#include "maxpoolOps.h.inc" + +#endif // MAXPOOL_OPS_H diff --git a/examples/prefab/balls/maxpool/stack/src/maxpoolOps.td b/examples/prefab/balls/maxpool/stack/src/maxpoolOps.td new file mode 100644 index 000000000..7c5cf50c5 --- /dev/null +++ b/examples/prefab/balls/maxpool/stack/src/maxpoolOps.td @@ -0,0 +1,45 @@ +//===- maxpoolOps.td - MaxPoolBall operations --------------*- tablegen -*-===// +// +// maxpool_i8: the input bank holds a square feature map of `input_side` +// pixels per side; group g is channel block g, line y * input_side + x holds +// the sixteen i8 channels of pixel (y, x). The output, with the same groups, +// receives the `kernel` x `kernel` maxima with `stride` and `padding` in the +// same layout, output_side = (input_side + 2 * padding - kernel) / stride + 1. +// +//===----------------------------------------------------------------------===// + +#ifndef MAXPOOL_OPS +#define MAXPOOL_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def MaxPoolBall_Dialect : Dialect { + let name = "maxpool_ball"; + let cppNamespace = "::buddy::maxpool_ball"; + let summary = "The operations of MaxPoolBall"; +} + +class MaxPoolBall_Op traits = []> + : Op])>; + +def BankMaxpoolI8Op : MaxPoolBall_Op<"bank_maxpool_i8", + [DeclareOpInterfaceMethods]> { + let summary = "maxpool_i8 on virtual banks"; + let arguments = (ins I64:$input, I64:$output, I64Attr:$input_side, + I64Attr:$kernel, I64Attr:$stride, I64Attr:$padding); + let results = (outs I64:$outputOut); + let assemblyFormat = "$input `->` $output prop-dict attr-dict"; +} + +def MaxpoolI8Op : MaxPoolBall_Op<"maxpool_i8", + [DeclareOpInterfaceMethods]> { + let summary = "maxpool_i8 on physical banks"; + let arguments = (ins I64Attr:$input, I64Attr:$output, I64Attr:$input_side, + I64Attr:$kernel, I64Attr:$stride, I64Attr:$padding); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // MAXPOOL_OPS diff --git a/examples/components/balls/maxpool/verify/casegen/Cargo.lock b/examples/prefab/balls/maxpool/verify/casegen/Cargo.lock similarity index 100% rename from examples/components/balls/maxpool/verify/casegen/Cargo.lock rename to examples/prefab/balls/maxpool/verify/casegen/Cargo.lock diff --git a/examples/components/balls/maxpool/verify/casegen/Cargo.toml b/examples/prefab/balls/maxpool/verify/casegen/Cargo.toml similarity index 100% rename from examples/components/balls/maxpool/verify/casegen/Cargo.toml rename to examples/prefab/balls/maxpool/verify/casegen/Cargo.toml diff --git a/examples/components/balls/maxpool/verify/casegen/src/casegen.rs b/examples/prefab/balls/maxpool/verify/casegen/src/casegen.rs similarity index 100% rename from examples/components/balls/maxpool/verify/casegen/src/casegen.rs rename to examples/prefab/balls/maxpool/verify/casegen/src/casegen.rs diff --git a/examples/components/balls/maxpool/verify/casegen/src/lib.rs b/examples/prefab/balls/maxpool/verify/casegen/src/lib.rs similarity index 100% rename from examples/components/balls/maxpool/verify/casegen/src/lib.rs rename to examples/prefab/balls/maxpool/verify/casegen/src/lib.rs diff --git a/examples/components/balls/maxpool/verify/casegen/src/model.rs b/examples/prefab/balls/maxpool/verify/casegen/src/model.rs similarity index 100% rename from examples/components/balls/maxpool/verify/casegen/src/model.rs rename to examples/prefab/balls/maxpool/verify/casegen/src/model.rs diff --git a/examples/components/balls/maxpool/verify/filelists/maxpool_ball.f b/examples/prefab/balls/maxpool/verify/filelists/maxpool_ball.f similarity index 100% rename from examples/components/balls/maxpool/verify/filelists/maxpool_ball.f rename to examples/prefab/balls/maxpool/verify/filelists/maxpool_ball.f diff --git a/examples/components/balls/maxpool/verify/src/common/maxpool_defs.svh b/examples/prefab/balls/maxpool/verify/src/common/maxpool_defs.svh similarity index 100% rename from examples/components/balls/maxpool/verify/src/common/maxpool_defs.svh rename to examples/prefab/balls/maxpool/verify/src/common/maxpool_defs.svh diff --git a/examples/components/balls/maxpool/verify/src/common/maxpool_items.svh b/examples/prefab/balls/maxpool/verify/src/common/maxpool_items.svh similarity index 100% rename from examples/components/balls/maxpool/verify/src/common/maxpool_items.svh rename to examples/prefab/balls/maxpool/verify/src/common/maxpool_items.svh diff --git a/examples/components/balls/maxpool/verify/src/cov/maxpool_cov.svh b/examples/prefab/balls/maxpool/verify/src/cov/maxpool_cov.svh similarity index 100% rename from examples/components/balls/maxpool/verify/src/cov/maxpool_cov.svh rename to examples/prefab/balls/maxpool/verify/src/cov/maxpool_cov.svh diff --git a/examples/components/balls/maxpool/verify/src/env/maxpool_env.svh b/examples/prefab/balls/maxpool/verify/src/env/maxpool_env.svh similarity index 100% rename from examples/components/balls/maxpool/verify/src/env/maxpool_env.svh rename to examples/prefab/balls/maxpool/verify/src/env/maxpool_env.svh diff --git a/examples/components/balls/maxpool/verify/src/env/maxpool_scoreboard.svh b/examples/prefab/balls/maxpool/verify/src/env/maxpool_scoreboard.svh similarity index 100% rename from examples/components/balls/maxpool/verify/src/env/maxpool_scoreboard.svh rename to examples/prefab/balls/maxpool/verify/src/env/maxpool_scoreboard.svh diff --git a/examples/components/balls/maxpool/verify/src/pkg/maxpool_pkg.sv b/examples/prefab/balls/maxpool/verify/src/pkg/maxpool_pkg.sv similarity index 100% rename from examples/components/balls/maxpool/verify/src/pkg/maxpool_pkg.sv rename to examples/prefab/balls/maxpool/verify/src/pkg/maxpool_pkg.sv diff --git a/examples/components/balls/maxpool/verify/src/seq/maxpool_sequences.svh b/examples/prefab/balls/maxpool/verify/src/seq/maxpool_sequences.svh similarity index 100% rename from examples/components/balls/maxpool/verify/src/seq/maxpool_sequences.svh rename to examples/prefab/balls/maxpool/verify/src/seq/maxpool_sequences.svh diff --git a/examples/components/balls/maxpool/verify/src/tb_top.sv b/examples/prefab/balls/maxpool/verify/src/tb_top.sv similarity index 100% rename from examples/components/balls/maxpool/verify/src/tb_top.sv rename to examples/prefab/balls/maxpool/verify/src/tb_top.sv diff --git a/examples/components/balls/maxpool/verify/src/tests/maxpool_test.svh b/examples/prefab/balls/maxpool/verify/src/tests/maxpool_test.svh similarity index 100% rename from examples/components/balls/maxpool/verify/src/tests/maxpool_test.svh rename to examples/prefab/balls/maxpool/verify/src/tests/maxpool_test.svh diff --git a/examples/components/balls/maxpool/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/maxpool/workloads/ctests/CMakeLists.txt similarity index 100% rename from examples/components/balls/maxpool/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/maxpool/workloads/ctests/CMakeLists.txt diff --git a/examples/components/balls/maxpool/workloads/ctests/maxpool_groups_test.c b/examples/prefab/balls/maxpool/workloads/ctests/maxpool_groups_test.c similarity index 100% rename from examples/components/balls/maxpool/workloads/ctests/maxpool_groups_test.c rename to examples/prefab/balls/maxpool/workloads/ctests/maxpool_groups_test.c diff --git a/examples/components/balls/maxpool/workloads/ctests/maxpool_offset_test.c b/examples/prefab/balls/maxpool/workloads/ctests/maxpool_offset_test.c similarity index 93% rename from examples/components/balls/maxpool/workloads/ctests/maxpool_offset_test.c rename to examples/prefab/balls/maxpool/workloads/ctests/maxpool_offset_test.c index 9d13e6171..f4eaffcb7 100644 --- a/examples/components/balls/maxpool/workloads/ctests/maxpool_offset_test.c +++ b/examples/prefab/balls/maxpool/workloads/ctests/maxpool_offset_test.c @@ -5,7 +5,7 @@ #include #include -enum { LANES = BANK_WIDTH / 8, ROW = 3 }; +enum { LANES = THREAD_MEM_BANK_WIDTH / 8, ROW = 3 }; static int8_t input[(ROW + 1) * LANES] __attribute__((aligned(64))); static int8_t output[LANES] __attribute__((aligned(64))); diff --git a/examples/components/balls/maxpool/workloads/ctests/maxpool_window_test.c b/examples/prefab/balls/maxpool/workloads/ctests/maxpool_window_test.c similarity index 94% rename from examples/components/balls/maxpool/workloads/ctests/maxpool_window_test.c rename to examples/prefab/balls/maxpool/workloads/ctests/maxpool_window_test.c index 6ae329a42..5da9ee4aa 100644 --- a/examples/components/balls/maxpool/workloads/ctests/maxpool_window_test.c +++ b/examples/prefab/balls/maxpool/workloads/ctests/maxpool_window_test.c @@ -5,7 +5,12 @@ #include #include -enum { LANES = BANK_WIDTH / 8, INPUT_BASE = 1, OUTPUT_BASE = 2, STRIDE = 4 }; +enum { + LANES = THREAD_MEM_BANK_WIDTH / 8, + INPUT_BASE = 1, + OUTPUT_BASE = 2, + STRIDE = 4 +}; enum { INPUT_ROWS = INPUT_BASE + 36, OUTPUT_ROWS = OUTPUT_BASE + 11 }; static int8_t input[INPUT_ROWS * LANES] __attribute__((aligned(64))); static int8_t output[OUTPUT_ROWS * LANES] __attribute__((aligned(64))); diff --git a/examples/components/balls/maxpool/workloads/isa/maxpool.h b/examples/prefab/balls/maxpool/workloads/isa/maxpool.h similarity index 97% rename from examples/components/balls/maxpool/workloads/isa/maxpool.h rename to examples/prefab/balls/maxpool/workloads/isa/maxpool.h index 5926ee7dc..925f3b372 100644 --- a/examples/components/balls/maxpool/workloads/isa/maxpool.h +++ b/examples/prefab/balls/maxpool/workloads/isa/maxpool.h @@ -19,6 +19,6 @@ FIELD(padding, 16, 19) | FIELD(input_base, 20, 25) | \ FIELD(output_base, 26, 31) | FIELD(output_stride, 32, 37) | \ FIELD(start_row, 38, 41) | FIELD(start_col, 42, 45), \ - BB_FUNC7(MAXPOOL)) + FUNCT7(maxpool, MAXPOOL)) #endif // _BB_MAXPOOL_H_ diff --git a/examples/components/balls/maxpool/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/maxpool/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/maxpool/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/maxpool/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/maxpool/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/maxpool/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/components/balls/maxpool/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/maxpool/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir b/examples/prefab/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir similarity index 81% rename from examples/components/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir rename to examples/prefab/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir index e3e6d65be..c10b4f317 100644 --- a/examples/components/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir +++ b/examples/prefab/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2.mlir @@ -16,7 +16,7 @@ func.func @main() -> i8 { buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mvin %input <{bank = 1 : i64, depth = 36 : i64, stride = 1 : i64}> : memref<36x16xi8> - buckyball.custom "maxpool_i8" <{sources = array, destination = 2 : i64}> {input_side = 6 : i64, output_side = 3 : i64, kernel = 2 : i64, stride = 2 : i64, padding = 0 : i64, op1_base = 0 : i64, wr_base = 0 : i64, output_stride = 3 : i64, start_row = 0 : i64, start_col = 0 : i64} + maxpool_ball.maxpool_i8 <{input = 1 : i64, output = 2 : i64, input_side = 6 : i64, kernel = 2 : i64, stride = 2 : i64, padding = 0 : i64}> buckyball.mvout %output <{bank = 2 : i64, depth = 9 : i64, stride = 1 : i64}> : memref<9x16xi8> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> diff --git a/examples/components/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2_main.cpp b/examples/prefab/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2_main.cpp similarity index 100% rename from examples/components/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2_main.cpp rename to examples/prefab/balls/maxpool/workloads/mlir_tests/ball/maxpool_6x6_s2_main.cpp diff --git a/examples/prefab/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..dcdfb3a23 --- /dev/null +++ b/examples/prefab/balls/maxpool/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) + set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +endif() +set(BUCKYBALL_MLIR_TEST_PREFIX bank) +set(_lines ${DESIGN_PROFILES_${BUCKYBALL_MLIR_ACTIVE_TARGET}_THREAD_MEM_BANK_ENTRIES}) +math(EXPR _remainder "${_lines} % 64") +if(_lines LESS 64 OR NOT _remainder EQUAL 0) + message(FATAL_ERROR + "MaxPool K3 bank batches require bank entries divisible by 64 (got ${_lines})") +endif() +add_buckyball_mlir_test(maxpool_k3_padded_spatial TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/components/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in b/examples/prefab/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in similarity index 89% rename from examples/components/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in rename to examples/prefab/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in index 415ab022b..6d110e979 100644 --- a/examples/components/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in +++ b/examples/prefab/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial.mlir.in @@ -25,7 +25,7 @@ func.func @main() -> i8 { %in_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %out_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded = buckyball.bank_mvin %in, %in_bank <{depth = 64 : i64, stride = 1 : i64}> : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> - %result = buckyball.bank_custom "maxpool_i8" (%loaded) -> %out_bank {input_side = 8 : i64, output_side = 8 : i64, kernel = 3 : i64, stride = 1 : i64, padding = 1 : i64, op1_base = 0 : i64, wr_base = 0 : i64, output_stride = 8 : i64, start_row = 0 : i64, start_col = 0 : i64} + %result = maxpool_ball.bank_maxpool_i8 %loaded -> %out_bank <{input_side = 8 : i64, kernel = 3 : i64, stride = 1 : i64, padding = 1 : i64}> %stored = buckyball.bank_mvout %out, %result <{depth = 64 : i64, stride = 1 : i64}> : memref<64x@BB_BANK_I8_LANES@xi8, strided<[@BB_BANK_I8_LANES@, 1], offset: ?>> buckyball.bank_release %loaded buckyball.bank_release %stored diff --git a/examples/components/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp b/examples/prefab/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp similarity index 85% rename from examples/components/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp rename to examples/prefab/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp index 254b73ca1..c7db0e0ce 100644 --- a/examples/components/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp +++ b/examples/prefab/balls/maxpool/workloads/mlir_tests/bank/maxpool_k3_padded_spatial_main.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -7,10 +6,10 @@ extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { constexpr int side = 8; - if (rows != BANK_LINES || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != THREAD_MEM_BANK_WIDTH / 8 || + stride != lanes || lane_stride != 1) exit(1); - for (int batch = 0; batch < BANK_LINES / 64; ++batch) + for (int batch = 0; batch < THREAD_MEM_BANK_ENTRIES / 64; ++batch) for (int y = 0; y < side; ++y) for (int x = 0; x < side; ++x) { int8_t expected = -128; diff --git a/examples/components/balls/mxfp2int/arch/Mxfp2Int.scala b/examples/prefab/balls/mxfp2int/arch/Mxfp2Int.scala similarity index 85% rename from examples/components/balls/mxfp2int/arch/Mxfp2Int.scala rename to examples/prefab/balls/mxfp2int/arch/Mxfp2Int.scala index 4ae592bcb..64a5eb3ad 100644 --- a/examples/components/balls/mxfp2int/arch/Mxfp2Int.scala +++ b/examples/prefab/balls/mxfp2int/arch/Mxfp2Int.scala @@ -13,11 +13,10 @@ import chisel3.experimental.hierarchy.{instantiable, public} import framework.acc.balldomain.ball.blink.BallStatus import framework.acc.memdomain.{BankRead, BankWrite} -import framework.acc.balldomain.mmio_bank.MmioRead import examples.balls.mxfp2int.configs.Mxfp2IntBallParam /** - * PipelinedMxfp2Int - MXFP to INT8 dequantizer with MMIO per-block scale. + * PipelinedMxfp2Int - MXFP to INT8 dequantizer with a per-block scale bank. * * Supports MXFP4, MXFP6, MXFP8 formats (32 elements per block). * @@ -26,7 +25,8 @@ import examples.balls.mxfp2int.configs.Mxfp2IntBallParam * MXFP6: 32 × 6-bit = 192 bits, padded to 2 words (256 bits, high 64 bits = 0) * MXFP8: 32 × 8-bit = 256 bits = 2 words * - * Scale (MMIO bank): per-block E8M0 (8-bit biased exponent, bias=127). + * Scale (op2 bank): E8M0 of block b (8-bit biased exponent, bias=127) in + * byte b % 16 of row b / 16. * * FP4/6/8 decoding (v1, simplified): signed_val = raw - (1 << (bits-1)) * FP4: raw - 8 (range -8..+7) @@ -76,6 +76,7 @@ class PipelinedMxfp2Int( val inBW = ballMapping.inBW val outBW = ballMapping.outBW + require(inBW == 2 && outBW == 1, "Mxfp2IntBall reads data and scale banks and writes one bank") @public val io = IO(new Bundle { @@ -100,8 +101,7 @@ class PipelinedMxfp2Int( )) ) - val status = new BallStatus - val mmioRead = Flipped(new MmioRead(memParam)) + val status = new BallStatus }) // ROB bookkeeping @@ -112,14 +112,15 @@ class PipelinedMxfp2Int( } // FSM - val idle :: sReadReq :: sReadResp :: sMmioReq :: sMmioResp :: sWriteLoReq :: sWriteLoResp :: sWriteHiReq :: sWriteHiResp :: sComplete :: Nil = + val idle :: sReadReq :: sReadResp :: sScaleReq :: sScaleResp :: sWriteLoReq :: sWriteLoResp :: sWriteHiReq :: sWriteHiResp :: sComplete :: Nil = Enum(10) - val state = RegInit(idle) + val state = RegInit(idle) // Per-instruction registers val raddr_reg = RegInit(0.U(frontendParam.iter_len.W)) val waddr_reg = RegInit(0.U(frontendParam.iter_len.W)) val rbank_reg = RegInit(0.U(frontendParam.bank_id_len.W)) + val sbank_reg = RegInit(0.U(frontendParam.bank_id_len.W)) val wbank_reg = RegInit(0.U(frontendParam.bank_id_len.W)) val bases = Reg(Vec(3, UInt(memParam.groupIdWidth.W))) val iter_reg = RegInit(0.U(frontendParam.iter_len.W)) @@ -168,8 +169,8 @@ class PipelinedMxfp2Int( io.bankRead(i).io.req.bits.addr := 0.U io.bankRead(i).io.resp.ready := false.B io.bankRead(i).rob_id := rob_id_reg - io.bankRead(i).bank_id := rbank_reg - io.bankRead(i).group_id := bases(0) + io.bankRead(i).bank_id := (if (i == 0) rbank_reg else sbank_reg) + io.bankRead(i).group_id := bases(i) } for (i <- 0 until outBW) { io.bankWrite(i).io.req.valid := false.B @@ -184,18 +185,18 @@ class PipelinedMxfp2Int( io.bankWrite(i).group_id := bases(2) } - io.mmioRead.req.valid := false.B - io.mmioRead.req.bits.addr := 0.U - io.mmioRead.resp.ready := false.B - io.cmdReq.ready := state === idle - io.cmdResp.valid := false.B - io.cmdResp.bits.rob_id := rob_id_reg + io.cmdReq.ready := state === idle + io.cmdResp.valid := false.B + io.cmdResp.bits.rob_id := rob_id_reg // === FSM === switch(state) { is(idle) { when(io.cmdReq.fire) { rbank_reg := io.cmdReq.bits.op1_bank + sbank_reg := io.cmdReq.bits.op2_bank + assert(io.cmdReq.bits.iter =/= 0.U, "Mxfp2IntBall iter must be positive") + assert(io.cmdReq.bits.rs2 === 0.U, "Mxfp2IntBall rs2 is reserved") bases := VecInit(io.cmdReq.bits.op1_group, io.cmdReq.bits.op2_group, io.cmdReq.bits.wr_group) wbank_reg := io.cmdReq.bits.wr_bank iter_reg := io.cmdReq.bits.iter @@ -229,7 +230,7 @@ class PipelinedMxfp2Int( inputWordsReg(word_idx_reg) := io.bankRead(0).io.resp.bits.data when(word_idx_reg === (wordsPerBlock - 1).U) { word_idx_reg := 0.U - state := sMmioReq + state := sScaleReq }.otherwise { word_idx_reg := word_idx_reg + 1.U state := sReadReq @@ -237,18 +238,18 @@ class PipelinedMxfp2Int( } } - is(sMmioReq) { - io.mmioRead.req.valid := true.B - io.mmioRead.req.bits.addr := block_idx_reg - when(io.mmioRead.req.fire) { - state := sMmioResp + is(sScaleReq) { + io.bankRead(1).io.req.valid := true.B + io.bankRead(1).io.req.bits.addr := block_idx_reg >> 4 + when(io.bankRead(1).io.req.fire) { + state := sScaleResp } } - is(sMmioResp) { - io.mmioRead.resp.ready := true.B - when(io.mmioRead.resp.fire) { - scaleReg := io.mmioRead.resp.bits.data(7, 0) + is(sScaleResp) { + io.bankRead(1).io.resp.ready := true.B + when(io.bankRead(1).io.resp.fire) { + scaleReg := io.bankRead(1).io.resp.bits.data.asTypeOf(Vec(INT8_PER_WORD, UInt(8.W)))(block_idx_reg(3, 0)) state := sWriteLoReq } } diff --git a/examples/components/balls/mxfp2int/arch/Mxfp2IntBall.scala b/examples/prefab/balls/mxfp2int/arch/Mxfp2IntBall.scala similarity index 88% rename from examples/components/balls/mxfp2int/arch/Mxfp2IntBall.scala rename to examples/prefab/balls/mxfp2int/arch/Mxfp2IntBall.scala index 0728eb398..30c4b0638 100644 --- a/examples/components/balls/mxfp2int/arch/Mxfp2IntBall.scala +++ b/examples/prefab/balls/mxfp2int/arch/Mxfp2IntBall.scala @@ -17,8 +17,8 @@ import framework.acc.balldomain.ball.blink.{BlinkIO, HasBlink} /** * Mxfp2IntBall - MXFP4 to INT8 dequantizer. * - * Wraps PipelinedMxfp2Int and connects to the Blink protocol, - * including the MMIO read channel for per-block E8M0 scales. + * Wraps PipelinedMxfp2Int and connects to the Blink protocol; the per-block + * E8M0 scales come from the second source bank. */ @instantiable class Mxfp2IntBall( @@ -37,7 +37,7 @@ class Mxfp2IntBall( val inBW = ballCommonConfig.inBW val outBW = ballCommonConfig.outBW - require(ballCommonConfig.mmioReadBW == 1 && ballCommonConfig.mmioWriteBW == 0) + require(ballCommonConfig.mmioReadBW == 0 && ballCommonConfig.mmioWriteBW == 0) @public val blink = IO(new BlinkIO(fp, mp, bp, id)) @@ -58,8 +58,6 @@ class Mxfp2IntBall( } blink.status <> mxfp2intUnit.io.status - - blink.mmioRead(0) <> mxfp2intUnit.io.mmioRead } object Mxfp2IntBall extends BallFactory { diff --git a/examples/components/balls/mxfp2int/arch/Mxfp4Decoder.scala b/examples/prefab/balls/mxfp2int/arch/Mxfp4Decoder.scala similarity index 100% rename from examples/components/balls/mxfp2int/arch/Mxfp4Decoder.scala rename to examples/prefab/balls/mxfp2int/arch/Mxfp4Decoder.scala diff --git a/examples/components/balls/mxfp2int/arch/Mxfp6Decoder.scala b/examples/prefab/balls/mxfp2int/arch/Mxfp6Decoder.scala similarity index 100% rename from examples/components/balls/mxfp2int/arch/Mxfp6Decoder.scala rename to examples/prefab/balls/mxfp2int/arch/Mxfp6Decoder.scala diff --git a/examples/components/balls/mxfp2int/arch/Mxfp8Decoder.scala b/examples/prefab/balls/mxfp2int/arch/Mxfp8Decoder.scala similarity index 100% rename from examples/components/balls/mxfp2int/arch/Mxfp8Decoder.scala rename to examples/prefab/balls/mxfp2int/arch/Mxfp8Decoder.scala diff --git a/examples/components/balls/mxfp2int/arch/MxfpDecoder.scala b/examples/prefab/balls/mxfp2int/arch/MxfpDecoder.scala similarity index 85% rename from examples/components/balls/mxfp2int/arch/MxfpDecoder.scala rename to examples/prefab/balls/mxfp2int/arch/MxfpDecoder.scala index 19fb356f7..e5692beda 100644 --- a/examples/components/balls/mxfp2int/arch/MxfpDecoder.scala +++ b/examples/prefab/balls/mxfp2int/arch/MxfpDecoder.scala @@ -49,9 +49,11 @@ object MxfpDecoder { * Positive shift = left, negative shift = right (arithmetic). */ def scaleShift(value: SInt, scale: UInt, valueWidth: Int): SInt = { - val shift = scale.zext - 127.S(10.W) - val absShift = Mux(shift >= 0.S, shift, -shift)(4, 0) - val widened = value.pad(32) + val shift = scale.zext - 127.S(10.W) + val magnitude = Mux(shift >= 0.S, shift, -shift).asUInt + // Shifts past 31 saturate a left shift and fill a right shift with the sign. + val absShift = Mux(magnitude > 31.U, 31.U, magnitude(4, 0)) + val widened = value.pad(32) Mux( shift >= 0.S, (widened << absShift).asSInt, diff --git a/examples/components/balls/mxfp2int/configs/Mxfp2IntBallParam.scala b/examples/prefab/balls/mxfp2int/configs/Mxfp2IntBallParam.scala similarity index 100% rename from examples/components/balls/mxfp2int/configs/Mxfp2IntBallParam.scala rename to examples/prefab/balls/mxfp2int/configs/Mxfp2IntBallParam.scala diff --git a/examples/components/balls/mxfp2int/configs/default.toml b/examples/prefab/balls/mxfp2int/configs/default.toml similarity index 100% rename from examples/components/balls/mxfp2int/configs/default.toml rename to examples/prefab/balls/mxfp2int/configs/default.toml diff --git a/examples/components/balls/mxfp2int/configs/mxfp2int_params.proto b/examples/prefab/balls/mxfp2int/configs/mxfp2int_params.proto similarity index 100% rename from examples/components/balls/mxfp2int/configs/mxfp2int_params.proto rename to examples/prefab/balls/mxfp2int/configs/mxfp2int_params.proto diff --git a/examples/prefab/balls/mxfp2int/emu/src/55_mxfp2int.rs b/examples/prefab/balls/mxfp2int/emu/src/55_mxfp2int.rs new file mode 100644 index 000000000..85ccd59f6 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/emu/src/55_mxfp2int.rs @@ -0,0 +1,69 @@ +//===- 55_mxfp2int.rs - MXFP2INT instruction (MXFP4 to INT8) ---------------===// +// +// Dequantizes iter MXFP4 blocks of 32 elements to int8. +// +// op1 bank: block b in row b, element e in byte e / 2, low nibble first; the +// element value is raw - 8. +// op2 bank: E8M0 scale of block b in byte b % 16 of row b / 16. +// wr bank: elements 0..15 of block b in row 2b, 16..31 in row 2b + 1, +// saturate(value * 2^(scale - 127)); the right shift rounds toward +// minus infinity. +// rs2 is reserved and zero. +// +//===----------------------------------------------------------------------===// + +use bebop_bemu::thread::banks::bank_size; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old, rs1_update}; +use super::instruction::{BallModel, ExecContext}; + +pub struct Mxfp2Int; + +const ROW_BYTES: usize = 16; +const ELEMENTS: usize = 32; + +impl BallModel for Mxfp2Int { + fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!( + !rs1_read_old(xs1) && !rs1_update(xs1), + "mxfp2int: reads no old destination; rs1[29] and rs1[63] must be clear" + ); + assert!(xs2 == 0, "mxfp2int: rs2 is reserved and must be zero"); + let data = rs1_b0(xs1); + let scales = rs1_b1(xs1); + let out = rs1_b2(xs1); + let blocks = rs1_iter(xs1) as usize; + for bank in [data, scales, out] { + let config = ctx.config(bank); + assert!(config.allocated && config.cols == 1, "mxfp2int: bank {bank} must be one allocated group"); + } + let lines = bank_size() / ROW_BYTES; + assert!(blocks > 0 && blocks * 2 <= lines, "mxfp2int: iter must be 1..half the bank depth"); + + let data = pbank(ctx, data); + let scales = pbank(ctx, scales); + let out = pbank(ctx, out); + for block in 0..blocks { + let scale = ctx.banks[scales][(block / 16) * ROW_BYTES + block % 16]; + let shift = scale as i32 - 127; + let row = &ctx.banks[data][block * ROW_BYTES..(block + 1) * ROW_BYTES]; + let mut result = [0u8; ELEMENTS]; + for (element, byte) in result.iter_mut().enumerate() { + let value = ((row[element / 2] >> ((element % 2) * 4)) & 0x0f) as i32 - 8; + let scaled = if shift >= 0 { + // |value| <= 8, so shifts past 7 saturate. + (value as i64) << shift.min(8) + } else { + (value >> (-shift).min(31)) as i64 + }; + *byte = scaled.clamp(-128, 127) as i8 as u8; + } + let start = block * 2 * ROW_BYTES; + ctx.banks[out][start..start + ELEMENTS].copy_from_slice(&result); + } + 0 + } + + fn latency(xs1: u64, _xs2: u64) -> u64 { + rs1_iter(xs1) * 4 + } +} diff --git a/examples/components/balls/mxfp2int/emu/src/dispatch.rs b/examples/prefab/balls/mxfp2int/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/mxfp2int/emu/src/dispatch.rs rename to examples/prefab/balls/mxfp2int/emu/src/dispatch.rs diff --git a/examples/components/balls/mxfp2int/emu/src/lib.rs b/examples/prefab/balls/mxfp2int/emu/src/lib.rs similarity index 100% rename from examples/components/balls/mxfp2int/emu/src/lib.rs rename to examples/prefab/balls/mxfp2int/emu/src/lib.rs diff --git a/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.cpp b/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.cpp new file mode 100644 index 000000000..c3ee27bc7 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.cpp @@ -0,0 +1,72 @@ +//===- mxfp2intOps.cpp - Mxfp2IntBall operations --------------------------===// +// +// The dialect of Mxfp2IntBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "mxfp2intOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::mxfp2int_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "mxfp2intDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "mxfp2intOps.cpp.inc" + +void Mxfp2IntBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "mxfp2intOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankMxfp4I8Op::getSourceBanks() { + return {getData(), getScales()}; +} +Value BankMxfp4I8Op::getDestinationBank() { return getOutput(); } +bool BankMxfp4I8Op::readsDestination() { return false; } +Operation *BankMxfp4I8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return Mxfp4I8Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows()); +} + +SmallVector Mxfp4I8Op::getSourceBanks() { + return {int64_t(getData()), int64_t(getScales())}; +} +int64_t Mxfp4I8Op::getDestinationBank() { return getOutput(); } +LogicalResult Mxfp4I8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), data = getData(), scales = getScales(), + output = getOutput(); + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || rows <= 0 || rows * 2 > target.bankDepth) + return emitOpError("mxfp4_i8 rows exceed half the bank"); + if (data == scales || data == output || scales == output) + return emitOpError( + "mxfp4_i8 requires distinct data, scale and output banks"); + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7("MXFP2INT"), + constantI64(builder, loc, + bankFields({data, scales}, output, false, rows)), + constantI64(builder, loc, 0)); + return success(); +} diff --git a/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.h b/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.h new file mode 100644 index 000000000..9e26f1c5f --- /dev/null +++ b/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.h @@ -0,0 +1,20 @@ +//===- mxfp2intOps.h - Mxfp2IntBall operations ----------------------------===// + +#ifndef MXFP2INT_OPS_H +#define MXFP2INT_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "mxfp2intDialect.h.inc" + +#define GET_OP_CLASSES +#include "mxfp2intOps.h.inc" + +#endif // MXFP2INT_OPS_H diff --git a/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.td b/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.td new file mode 100644 index 000000000..5e619d794 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/stack/src/mxfp2intOps.td @@ -0,0 +1,50 @@ +//===- mxfp2intOps.td - Mxfp2IntBall operations ------------*- tablegen -*-===// +// +// mxfp4_i8 (data, scales) -> output: dequantizes `rows` MXFP4 blocks of 32 +// elements to i8, one MXFP2INT instruction. Banks are one group of sixteen +// bytes per row: +// data: block b in row b, element e in byte e / 2, low nibble first; the +// element value is raw - 8. +// scales: the E8M0 scale of block b in byte b % 16 of row b / 16. +// output: elements 0..15 of block b in row 2b, 16..31 in row 2b + 1, +// saturate(value * 2^(scale - 127)), the right shift rounding toward minus +// infinity. +// `rows` is the number of blocks, 1..half the bank depth. The Ball must be +// configured for MXFP4. +// +//===----------------------------------------------------------------------===// + +#ifndef MXFP2INT_OPS +#define MXFP2INT_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def Mxfp2IntBall_Dialect : Dialect { + let name = "mxfp2int_ball"; + let cppNamespace = "::buddy::mxfp2int_ball"; + let summary = "The operations of Mxfp2IntBall"; +} + +class Mxfp2IntBall_Op traits = []> + : Op])>; + +def BankMxfp4I8Op : Mxfp2IntBall_Op<"bank_mxfp4_i8", + [DeclareOpInterfaceMethods]> { + let summary = "mxfp4_i8 on virtual banks"; + let arguments = (ins I64:$data, I64:$scales, I64:$output, I64Attr:$rows); + let results = (outs I64:$outputOut); + let assemblyFormat = "$data `,` $scales `->` $output prop-dict attr-dict"; +} + +def Mxfp4I8Op : Mxfp2IntBall_Op<"mxfp4_i8", + [DeclareOpInterfaceMethods]> { + let summary = "mxfp4_i8 on physical banks"; + let arguments = (ins I64Attr:$data, I64Attr:$scales, I64Attr:$output, + I64Attr:$rows); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // MXFP2INT_OPS diff --git a/examples/prefab/balls/mxfp2int/verify/casegen/Cargo.lock b/examples/prefab/balls/mxfp2int/verify/casegen/Cargo.lock new file mode 100644 index 000000000..61f560c0f --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "mxfp2int-casegen" +version = "0.1.0" diff --git a/examples/prefab/balls/mxfp2int/verify/casegen/Cargo.toml b/examples/prefab/balls/mxfp2int/verify/casegen/Cargo.toml new file mode 100644 index 000000000..e2a6e2794 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mxfp2int-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["rlib", "cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/prefab/balls/mxfp2int/verify/casegen/src/casegen.rs b/examples/prefab/balls/mxfp2int/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..6ae1366ae --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/casegen/src/casegen.rs @@ -0,0 +1,231 @@ +// MXFP2INT reference model (MXFP4 to INT8), the contract shared with the +// BEMU model (emu/src/55_mxfp2int.rs) and Mxfp2IntBall: +// +// op1 bank: block b in row b, element e in byte e / 2, low nibble first; the +// element value is raw - 8. +// op2 bank: E8M0 scale of block b in byte b % 16 of row b / 16. +// wr bank: elements 0..15 of block b in row 2b, 16..31 in row 2b + 1, +// saturate(value * 2^(scale - 127)); the right shift rounds toward +// minus infinity. +// rs2 is reserved and zero; 1 <= iter <= bank_entries / 2. + +pub const ROW_BYTES: usize = 16; +pub const ELEMENTS: usize = 32; +pub const NUM_CASES: u32 = 6; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Mxfp2IntCmd { + pub rob_id: u32, + pub iter: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, +} + +#[derive(Clone, Debug)] +pub struct Preload { + pub bank: u32, + pub group: u32, + pub addr: u32, + pub data: u128, +} + +#[derive(Clone, Debug)] +pub struct Write { + pub addr: u32, + pub mask: u16, + pub data: u128, +} + +#[derive(Clone, Debug)] +pub struct Step { + pub cmd: Mxfp2IntCmd, + pub preload: Vec, + pub reads: [Vec; 2], + pub writes: Vec, +} + +#[derive(Clone, Debug)] +pub struct Case { + pub steps: Vec, +} + +struct Rng(u32); + +impl Rng { + fn next(&mut self) -> u32 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 17; + self.0 ^= self.0 << 5; + self.0 + } +} + +fn word(bytes: &[u8]) -> u128 { + let mut out = 0u128; + for (lane, byte) in bytes.iter().enumerate() { + out |= u128::from(*byte) << (lane * 8); + } + out +} + +fn dequant(raw: u8, scale: u8) -> u8 { + let value = raw as i32 - 8; + let shift = scale as i32 - 127; + let scaled = if shift >= 0 { + // |value| <= 8, so shifts past 7 saturate. + (value as i64) << shift.min(8) + } else { + (value >> (-shift).min(31)) as i64 + }; + scaled.clamp(-128, 127) as i8 as u8 +} + +// `data` holds iter rows of packed nibbles, `scales` one byte per block. +fn step( + bank_entries: u32, + rob_id: u32, + banks: [u32; 3], + groups: [u32; 3], + data: &[u8], + scales: &[u8], +) -> Step { + let blocks = scales.len(); + if blocks == 0 || blocks * 2 > bank_entries as usize || data.len() != blocks * ROW_BYTES { + panic!("mxfp2int: iter must be 1..half the bank depth"); + } + if banks[0] == banks[1] || banks[0] == banks[2] || banks[1] == banks[2] { + panic!("mxfp2int: banks must differ"); + } + let mut scale_rows = vec![0u8; blocks.div_ceil(ROW_BYTES) * ROW_BYTES]; + scale_rows[..blocks].copy_from_slice(scales); + let mut preload = Vec::new(); + for (row, bytes) in data.chunks(ROW_BYTES).enumerate() { + preload.push(Preload { bank: banks[0], group: groups[0], addr: row as u32, data: word(bytes) }); + } + for (row, bytes) in scale_rows.chunks(ROW_BYTES).enumerate() { + preload.push(Preload { bank: banks[1], group: groups[1], addr: row as u32, data: word(bytes) }); + } + let mut reads = [Vec::new(), Vec::new()]; + let mut writes = Vec::new(); + for block in 0..blocks { + reads[0].push(block as u32); + reads[1].push((block / ROW_BYTES) as u32); + let row = &data[block * ROW_BYTES..(block + 1) * ROW_BYTES]; + let mut result = [0u8; ELEMENTS]; + for (element, byte) in result.iter_mut().enumerate() { + *byte = dequant((row[element / 2] >> ((element % 2) * 4)) & 0x0f, scales[block]); + } + writes.push(Write { addr: (block * 2) as u32, mask: 0xffff, data: word(&result[..ROW_BYTES]) }); + writes.push(Write { addr: (block * 2 + 1) as u32, mask: 0xffff, data: word(&result[ROW_BYTES..]) }); + } + Step { + cmd: Mxfp2IntCmd { + rob_id, + iter: blocks as u32, + rs2_lo: 0, + rs2_hi: 0, + op1_bank: banks[0], + op2_bank: banks[1], + wr_bank: banks[2], + op1_group: groups[0], + op2_group: groups[1], + wr_group: groups[2], + op1_col: 1, + op2_col: 1, + wr_col: 1, + }, + preload, + reads, + writes, + } +} + +fn pattern_data(blocks: usize) -> Vec { + let raw = |block: usize, elem: usize| ((elem * 7 + block * 3) & 0xf) as u8; + let mut data = vec![0u8; blocks * ROW_BYTES]; + for block in 0..blocks { + for elem in (0..ELEMENTS).step_by(2) { + data[block * ROW_BYTES + elem / 2] = (raw(block, elem + 1) << 4) | raw(block, elem); + } + } + data +} + +fn random_data(rng: &mut Rng, blocks: usize) -> Vec { + (0..blocks * ROW_BYTES).map(|_| rng.next() as u8).collect() +} + +pub fn gen_case(index: u32, bank_entries: u32) -> Case { + let mut rng = Rng(0x6d2b_79f5 ^ index.wrapping_mul(0x9e37_79b9)); + let steps = match index { + // Two blocks at the identity scale (mxfp4_e8m0_two_block_test). + 0 => vec![step(bank_entries, 1, [1, 2, 3], [0, 0, 0], &pattern_data(2), &[127, 127])], + // Scales across right shifts, saturating left shifts and shifts past 31, + // spanning two scale rows (mxfp4_e8m0_scale_range_test). + 1 => { + let scales = [0, 1, 90, 120, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 150, 158, 159, 200, 255]; + vec![step(bank_entries, 2, [1, 2, 3], [0, 0, 0], &pattern_data(scales.len()), &scales)] + } + // Random nibbles and scales near the identity, non-zero group bases. + 2 => { + let scales: Vec = (0..100).map(|_| 120 + (rng.next() % 16) as u8).collect(); + let data = random_data(&mut rng, scales.len()); + vec![step(bank_entries, 5, [4, 7, 9], [3, 11, 6], &data, &scales)] + } + // One block, highest banks and groups. + 3 => vec![step(bank_entries, 15, [1021, 1022, 1023], [15, 15, 15], &random_data(&mut rng, 1), &[129])], + // Full capacity: iter = bank_entries / 2. + 4 => { + let blocks = (bank_entries / 2) as usize; + let scales: Vec = (0..blocks).map(|_| rng.next() as u8).collect(); + let data = random_data(&mut rng, blocks); + vec![step(bank_entries, 9, [0, 512, 768], [1, 2, 4], &data, &scales)] + } + // Back-to-back commands without reset. + 5 => (0..3) + .map(|i| { + let blocks = 1 + (rng.next() % 18) as usize; + let scales: Vec = (0..blocks).map(|_| 118 + (rng.next() % 20) as u8).collect(); + let data = random_data(&mut rng, blocks); + step(bank_entries, 6 + i, [8 + i, 16 + i, 24 + i], [i, i + 4, i + 8], &data, &scales) + }) + .collect(), + _ => panic!("mxfp2int: unsupported case {index}"), + }; + Case { steps } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cases_build() { + for index in 0..NUM_CASES { + gen_case(index, 1024); + } + } + + // Golden values of mxfp4_e8m0_scale_range_test. + #[test] + fn scale_range_golden() { + assert_eq!(dequant(15, 127) as i8, 7); + assert_eq!(dequant(0, 127) as i8, -8); + assert_eq!(dequant(9, 0) as i8, 0); + assert_eq!(dequant(7, 0) as i8, -1); + assert_eq!(dequant(15, 255) as i8, 127); + assert_eq!(dequant(0, 255) as i8, -128); + assert_eq!(dequant(1, 126) as i8, -4); + assert_eq!(dequant(15, 130) as i8, 56); + } +} diff --git a/examples/prefab/balls/mxfp2int/verify/casegen/src/config.rs b/examples/prefab/balls/mxfp2int/verify/casegen/src/config.rs new file mode 100644 index 000000000..241755f0f --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/casegen/src/config.rs @@ -0,0 +1,9 @@ +pub fn bank_entries() -> u32 { + let path = std::env::var("BB_VERIFY_CONFIG").expect("BB_VERIFY_CONFIG is required"); + let text = std::fs::read_to_string(path).expect("failed to read BB_VERIFY_CONFIG"); + text.lines() + .find_map(|line| line.strip_prefix("bank_entries=")) + .unwrap_or_else(|| panic!("bank_entries missing from BB_VERIFY_CONFIG")) + .parse() + .expect("invalid bank_entries in BB_VERIFY_CONFIG") +} diff --git a/examples/prefab/balls/mxfp2int/verify/casegen/src/lib.rs b/examples/prefab/balls/mxfp2int/verify/casegen/src/lib.rs new file mode 100644 index 000000000..129919ffc --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/casegen/src/lib.rs @@ -0,0 +1,116 @@ +mod casegen; +mod config; + +use std::cell::RefCell; + +use casegen::{Case, Mxfp2IntCmd, Step}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn mxfp2int_num_cases() -> u32 { + casegen::NUM_CASES +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_load(index: u32) -> u32 { + let case = casegen::gen_case(index, config::bank_entries()); + let steps = case.steps.len() as u32; + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + steps +} + +fn current_step R, R>(step: u32, f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => match case.steps.get(step as usize) { + Some(s) => f(s), + None => panic!("mxfp2int_case: step {step} out of range {}", case.steps.len()), + }, + None => panic!("mxfp2int_case: no case loaded; call mxfp2int_case_load first"), + }) +} + +fn idx(index: u32, limit: usize, what: &str) -> usize { + if index as usize >= limit { + panic!("mxfp2int_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_cmd(step: u32, out_ptr: *mut Mxfp2IntCmd) { + if out_ptr.is_null() { + panic!("mxfp2int_case_cmd: null out_ptr"); + } + current_step(step, |s| unsafe { *out_ptr = s.cmd }); +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_preload_count(step: u32) -> u32 { + current_step(step, |s| s.preload.len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_preload_bank(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].bank) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_preload_group(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].group) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_preload_addr(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].addr) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_preload_lo(step: u32, i: u32) -> u64 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].data as u64) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_preload_hi(step: u32, i: u32) -> u64 { + current_step(step, |s| (s.preload[idx(i, s.preload.len(), "preload")].data >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_read_count(step: u32, port: u32) -> u32 { + current_step(step, |s| s.reads[idx(port, 2, "read_port")].len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_read_addr(step: u32, port: u32, i: u32) -> u32 { + current_step(step, |s| { + let reads = &s.reads[idx(port, 2, "read_port")]; + reads[idx(i, reads.len(), "read")] + }) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_write_count(step: u32) -> u32 { + current_step(step, |s| s.writes.len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_write_addr(step: u32, i: u32) -> u32 { + current_step(step, |s| s.writes[idx(i, s.writes.len(), "write")].addr) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_write_mask(step: u32, i: u32) -> u32 { + current_step(step, |s| u32::from(s.writes[idx(i, s.writes.len(), "write")].mask)) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_write_lo(step: u32, i: u32) -> u64 { + current_step(step, |s| s.writes[idx(i, s.writes.len(), "write")].data as u64) +} + +#[no_mangle] +pub extern "C" fn mxfp2int_case_write_hi(step: u32, i: u32) -> u64 { + current_step(step, |s| (s.writes[idx(i, s.writes.len(), "write")].data >> 64) as u64) +} diff --git a/examples/prefab/balls/mxfp2int/verify/filelists/mxfp2int_ball.f b/examples/prefab/balls/mxfp2int/verify/filelists/mxfp2int_ball.f new file mode 100644 index 000000000..44ff7dfe1 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/filelists/mxfp2int_ball.f @@ -0,0 +1,25 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+src ++incdir+src/pkg ++incdir+src/common ++incdir+src/seq ++incdir+src/cov ++incdir+src/env ++incdir+src/tests ++incdir+@RTL@ + +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv + +@RTL@/PipelinedMxfp2Int.sv +@RTL@/Mxfp2IntBall.sv + +src/pkg/mxfp2int_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/mxfp2int/verify/src/common/mxfp2int_defs.svh b/examples/prefab/balls/mxfp2int/verify/src/common/mxfp2int_defs.svh new file mode 100644 index 000000000..28fd50e0f --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/common/mxfp2int_defs.svh @@ -0,0 +1,39 @@ +typedef struct { + int unsigned rob_id; + int unsigned iter; + int unsigned rs2_lo; + int unsigned rs2_hi; + int unsigned op1_bank; + int unsigned op2_bank; + int unsigned wr_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; + int unsigned op1_col; + int unsigned op2_col; + int unsigned wr_col; +} mxfp2int_cmd_dpi_t; + +import "DPI-C" function int unsigned mxfp2int_num_cases(); +import "DPI-C" function int unsigned mxfp2int_case_load(input int unsigned index); +import "DPI-C" function void mxfp2int_case_cmd(input int unsigned step, output mxfp2int_cmd_dpi_t cmd); +import "DPI-C" function int unsigned mxfp2int_case_preload_count(input int unsigned step); +import "DPI-C" function int unsigned mxfp2int_case_preload_bank(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxfp2int_case_preload_group(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxfp2int_case_preload_addr(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxfp2int_case_preload_lo(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxfp2int_case_preload_hi(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxfp2int_case_read_count(input int unsigned step, input int unsigned port); +import "DPI-C" function int unsigned mxfp2int_case_read_addr(input int unsigned step, input int unsigned port, input int unsigned i); +import "DPI-C" function int unsigned mxfp2int_case_write_count(input int unsigned step); +import "DPI-C" function int unsigned mxfp2int_case_write_addr(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxfp2int_case_write_mask(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxfp2int_case_write_lo(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxfp2int_case_write_hi(input int unsigned step, input int unsigned i); + +`ifndef MXFP2INT_FUNCT7 +`error "MXFP2INT_FUNCT7 must be provided by the selected ballISA" +`endif + +localparam int MXFP2INT_THREAD_FUNCT7 = `MXFP2INT_FUNCT7; +localparam int MXFP2INT_TIMEOUT_CYCLES = 100000; diff --git a/examples/prefab/balls/mxfp2int/verify/src/common/mxfp2int_items.svh b/examples/prefab/balls/mxfp2int/verify/src/common/mxfp2int_items.svh new file mode 100644 index 000000000..44c31936e --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/common/mxfp2int_items.svh @@ -0,0 +1,85 @@ +class mxfp2int_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(mxfp2int_cmd_item) + + int unsigned preload_bank[]; + int unsigned preload_group[]; + int unsigned preload_addr[]; + bit [127:0] preload_data[]; + int unsigned read_addr0[]; + int unsigned read_addr1[]; + int unsigned write_addr[]; + bit [15:0] write_mask[]; + bit [127:0] write_data[]; + + function new(string name = "mxfp2int_cmd_item"); + super.new(name); + endfunction + + function void load_rust_step(int unsigned step); + mxfp2int_cmd_dpi_t cmd; + int unsigned n; + + mxfp2int_case_cmd(step, cmd); + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = MXFP2INT_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; + if (rob_id != cmd.rob_id || op1_bank != cmd.op1_bank || op2_bank != cmd.op2_bank || + wr_bank != cmd.wr_bank || op1_group != cmd.op1_group || op2_group != cmd.op2_group || + wr_group != cmd.wr_group) + `uvm_fatal("CASE", "command field exceeds the configured BLink width") + + n = mxfp2int_case_preload_count(step); + preload_bank = new[n]; + preload_group = new[n]; + preload_addr = new[n]; + preload_data = new[n]; + for (int unsigned i = 0; i < n; i++) begin + preload_bank[i] = mxfp2int_case_preload_bank(step, i); + preload_group[i] = mxfp2int_case_preload_group(step, i); + preload_addr[i] = mxfp2int_case_preload_addr(step, i); + preload_data[i] = {mxfp2int_case_preload_hi(step, i), mxfp2int_case_preload_lo(step, i)}; + end + n = mxfp2int_case_read_count(step, 0); + read_addr0 = new[n]; + for (int unsigned i = 0; i < n; i++) read_addr0[i] = mxfp2int_case_read_addr(step, 0, i); + n = mxfp2int_case_read_count(step, 1); + read_addr1 = new[n]; + for (int unsigned i = 0; i < n; i++) read_addr1[i] = mxfp2int_case_read_addr(step, 1, i); + n = mxfp2int_case_write_count(step); + write_addr = new[n]; + write_mask = new[n]; + write_data = new[n]; + for (int unsigned i = 0; i < n; i++) begin + write_addr[i] = mxfp2int_case_write_addr(step, i); + write_mask[i] = mxfp2int_case_write_mask(step, i); + write_data[i] = {mxfp2int_case_write_hi(step, i), mxfp2int_case_write_lo(step, i)}; + end + endfunction + + function void do_copy(uvm_object rhs); + mxfp2int_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not mxfp2int_cmd_item") + preload_bank = rhs_.preload_bank; + preload_group = rhs_.preload_group; + preload_addr = rhs_.preload_addr; + preload_data = rhs_.preload_data; + read_addr0 = rhs_.read_addr0; + read_addr1 = rhs_.read_addr1; + write_addr = rhs_.write_addr; + write_mask = rhs_.write_mask; + write_data = rhs_.write_data; + endfunction +endclass diff --git a/examples/prefab/balls/mxfp2int/verify/src/cov/mxfp2int_cov.svh b/examples/prefab/balls/mxfp2int/verify/src/cov/mxfp2int_cov.svh new file mode 100644 index 000000000..c285f5dba --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/cov/mxfp2int_cov.svh @@ -0,0 +1,64 @@ +class mxfp2int_cov extends uvm_component; + `uvm_component_utils(mxfp2int_cov) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, mxfp2int_cov) stim_imp; + + int unsigned cur_iter; + int unsigned cur_scale; + int unsigned cmd_count; + + covergroup cmd_cg; + coverpoint cur_iter { + bins one = {1}; + bins row = {[2 : 16]}; + bins rows = {[17 : 64]}; + bins many = {[65 : 511]}; + bins capacity = {512}; + } + endgroup + + covergroup scale_cg; + coverpoint cur_scale { + bins zero = {0}; + bins shift_past_31 = {[1 : 95]}; + bins right = {[96 : 126]}; + bins identity = {127}; + bins left = {[128 : 134]}; + bins saturate = {[135 : 254]}; + bins top = {255}; + } + endgroup + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_cg = new(); + scale_cg = new(); + endfunction + + function void write_stim(bb_blink_cmd_item item); + mxfp2int_cmd_item stim; + if (!$cast(stim, item)) `uvm_fatal("COV", "stim item is not mxfp2int_cmd_item") + cur_iter = stim.iter; + cmd_cg.sample(); + foreach (stim.preload_data[i]) begin + if (stim.preload_bank[i] != stim.op2_bank) continue; + for (int b = 0; b < 16; b++) begin + if (stim.preload_addr[i] * 16 + b >= stim.iter) break; + cur_scale = stim.preload_data[i][b*8+:8]; + scale_cg.sample(); + end + end + cmd_count++; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("mxfp2int iter coverage %0.2f%%", cmd_cg.get_coverage())) + if (scale_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("mxfp2int scale coverage %0.2f%%", scale_cg.get_coverage())) + endfunction +endclass diff --git a/examples/prefab/balls/mxfp2int/verify/src/env/mxfp2int_env.svh b/examples/prefab/balls/mxfp2int/verify/src/env/mxfp2int_env.svh new file mode 100644 index 000000000..32222694b --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/env/mxfp2int_env.svh @@ -0,0 +1,27 @@ +class mxfp2int_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(mxfp2int_env) + + mxfp2int_scoreboard scb; + mxfp2int_cov icov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scb = mxfp2int_scoreboard::type_id::create("scb", this); + icov = mxfp2int_cov::type_id::create("icov", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + scb.mem_model = mem_model; + cmd_agent.stim_ap.connect(scb.stim_imp); + cmd_agent.stim_ap.connect(icov.stim_imp); + cmd_agent.cmd_ap.connect(scb.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction +endclass diff --git a/examples/prefab/balls/mxfp2int/verify/src/env/mxfp2int_scoreboard.svh b/examples/prefab/balls/mxfp2int/verify/src/env/mxfp2int_scoreboard.svh new file mode 100644 index 000000000..b9f01fc91 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/env/mxfp2int_scoreboard.svh @@ -0,0 +1,196 @@ +class mxfp2int_scoreboard extends uvm_scoreboard; + `uvm_component_utils(mxfp2int_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, mxfp2int_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, mxfp2int_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, mxfp2int_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, mxfp2int_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, mxfp2int_scoreboard) resp_imp; + + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + + mxfp2int_cmd_item stim_q[$]; + int unsigned cmd_count; + int unsigned read_count[2]; + int unsigned write_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + mxfp2int_cmd_item got; + mxfp2int_cmd_item clone; + + if (stim_q.size() != 0) `uvm_fatal("SCB", "single outstanding command supported") + if (!$cast(got, item)) `uvm_fatal("SCB", "stim item is not mxfp2int_cmd_item") + if (!$cast(clone, got.clone())) `uvm_fatal("SCB", "failed to clone stimulus item") + stim_q.push_back(clone); + if (mem_model == null) `uvm_fatal("SCB", "mem_model handle not set") + mem_model.clear_mem(); + foreach (clone.preload_data[i]) + mem_model.load_word_g(clone.preload_bank[i], clone.preload_group[i], clone.preload_addr[i], + clone.preload_data[i]); + mem_model.arm(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + mxfp2int_cmd_item stim; + bb_blink_cmd_item want; + stim = current_stim("CMD"); + // Compare as plain BLink commands; the comparer rejects differing types. + want = bb_blink_cmd_item::type_id::create("want"); + want.copy(stim); + verification_pkg::evidence::checks++; + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) + cmd_count++; + endfunction + + function void write_read(bb_blink_read_item item); + mxfp2int_cmd_item stim; + int unsigned exp_bank; + int unsigned exp_group; + int unsigned exp_addr; + + stim = current_stim("READ"); + if (item.port != 0 && item.port != 1) + `uvm_fatal("READ", $sformatf("Mxfp2IntBall has two read ports, got port %0d", item.port)) + if (item.port == 0) begin + if (read_count[0] >= stim.read_addr0.size()) + `uvm_fatal("READ", $sformatf("extra read on port 0 at %0d", read_count[0])) + exp_bank = stim.op1_bank; + exp_group = stim.op1_group; + exp_addr = stim.read_addr0[read_count[0]]; + end else begin + if (read_count[1] >= stim.read_addr1.size()) + `uvm_fatal("READ", $sformatf("extra read on port 1 at %0d", read_count[1])) + exp_bank = stim.op2_bank; + exp_group = stim.op2_group; + exp_addr = stim.read_addr1[read_count[1]]; + end + verification_pkg::evidence::checks++; + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.bank_id !== exp_bank[BB_BLINK_BANK_ID_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d bank mismatch: got %0d exp %0d", item.port, item.bank_id, exp_bank)) + if (item.group_id !== exp_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d group mismatch: got %0d exp %0d", item.port, item.group_id, exp_group)) + if (item.addr !== exp_addr[BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d read %0d addr mismatch: got %0d exp %0d", + item.port, + read_count[item.port], + item.addr, + exp_addr + )) + read_count[item.port]++; + endfunction + + function void write_write(bb_blink_write_item item); + mxfp2int_cmd_item stim; + bit [127:0] bytes; + + stim = current_stim("WRITE"); + if (item.port != 0) + `uvm_fatal("WRITE", $sformatf("Mxfp2IntBall has one write port, got port %0d", item.port)) + if (write_count >= stim.write_addr.size()) + `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) + verification_pkg::evidence::checks++; + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.group_id !== stim.wr_group) + `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + )) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.addr !== stim.write_addr[write_count][BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("WRITE", $sformatf( + "write %0d addr mismatch: got %0d exp %0d", + write_count, + item.addr, + stim.write_addr[write_count] + )) + if (item.mask !== stim.write_mask[write_count]) + `uvm_fatal("WRITE", $sformatf( + "write %0d mask mismatch: got 0x%04h exp 0x%04h", + write_count, + item.mask, + stim.write_mask[write_count] + )) + for (int b = 0; b < 16; b++) bytes[b*8+:8] = {8{item.mask[b]}}; + if ((item.data & bytes) !== (stim.write_data[write_count] & bytes)) + `uvm_fatal("SCB", $sformatf( + "write %0d data mismatch at addr %0d: got 0x%032h exp 0x%032h", + write_count, + item.addr, + item.data, + stim.write_data[write_count] + )) + write_count++; + endfunction + + function void write_resp(bb_blink_resp_item item); + mxfp2int_cmd_item stim; + stim = current_stim("RESP"); + verification_pkg::evidence::checks++; + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (!complete_io()) + `uvm_fatal("RESP", $sformatf( + "response before all traffic: reads=%0d,%0d writes=%0d", + read_count[0], + read_count[1], + write_count + )) + resp_count++; + endfunction + + function mxfp2int_cmd_item current_stim(string tag); + if (stim_q.size() == 0) `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + return stim_q[0]; + endfunction + + function bit complete_io(); + mxfp2int_cmd_item stim = stim_q[0]; + return read_count[0] == stim.read_addr0.size() && read_count[1] == stim.read_addr1.size() && + write_count == stim.write_addr.size(); + endfunction + + function bit done(); + return stim_q.size() == 1 && cmd_count == 1 && resp_count == 1 && complete_io(); + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + read_count[0] = 0; + read_count[1] = 0; + write_count = 0; + resp_count = 0; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) + `uvm_fatal("SCB", $sformatf( + "incomplete: stim=%0d cmds=%0d reads=%0d,%0d writes=%0d resp=%0d", + stim_q.size(), + cmd_count, + read_count[0], + read_count[1], + write_count, + resp_count + )) + stim_q.delete(); + endfunction +endclass diff --git a/examples/prefab/balls/mxfp2int/verify/src/pkg/mxfp2int_pkg.sv b/examples/prefab/balls/mxfp2int/verify/src/pkg/mxfp2int_pkg.sv new file mode 100644 index 000000000..b29568d25 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/pkg/mxfp2int_pkg.sv @@ -0,0 +1,13 @@ +package mxfp2int_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + + `include "common/mxfp2int_defs.svh" + `include "common/mxfp2int_items.svh" + `include "seq/mxfp2int_sequences.svh" + `include "cov/mxfp2int_cov.svh" + `include "env/mxfp2int_scoreboard.svh" + `include "env/mxfp2int_env.svh" + `include "tests/mxfp2int_test.svh" +endpackage diff --git a/examples/prefab/balls/mxfp2int/verify/src/seq/mxfp2int_sequences.svh b/examples/prefab/balls/mxfp2int/verify/src/seq/mxfp2int_sequences.svh new file mode 100644 index 000000000..2affff9b6 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/seq/mxfp2int_sequences.svh @@ -0,0 +1,17 @@ +class mxfp2int_step_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(mxfp2int_step_seq) + + int unsigned step; + + function new(string name = "mxfp2int_step_seq"); + super.new(name); + endfunction + + task body(); + mxfp2int_cmd_item item; + item = mxfp2int_cmd_item::type_id::create("item"); + start_item(item); + item.load_rust_step(step); + finish_item(item); + endtask +endclass diff --git a/examples/prefab/balls/mxfp2int/verify/src/tb_top.sv b/examples/prefab/balls/mxfp2int/verify/src/tb_top.sv new file mode 100644 index 000000000..2c01c4374 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/tb_top.sv @@ -0,0 +1,87 @@ +module tb_top; + import uvm_pkg::*; + import mxfp2int_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + Mxfp2IntBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .blink_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .blink_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .blink_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .blink_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .blink_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .blink_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .blink_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .blink_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .blink_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .blink_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .blink_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .blink_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .blink_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .blink_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .blink_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .blink_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]) + ); + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("mxfp2int_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/mxfp2int/verify/src/tests/mxfp2int_test.svh b/examples/prefab/balls/mxfp2int/verify/src/tests/mxfp2int_test.svh new file mode 100644 index 000000000..0101a6217 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/verify/src/tests/mxfp2int_test.svh @@ -0,0 +1,56 @@ +class mxfp2int_ball_test extends uvm_test; + `uvm_component_utils(mxfp2int_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + mxfp2int_env env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + env = mxfp2int_env::type_id::create("env", this); + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + for (int unsigned index = 0; index < mxfp2int_num_cases(); index++) run_case(index); + phase.drop_objection(this); + endtask + + // Steps of one case run back to back on the same reset. + task run_case(int unsigned index); + int unsigned steps; + apply_reset(); + steps = mxfp2int_case_load(index); + for (int unsigned step = 0; step < steps; step++) run_step(index, step); + endtask + + task run_step(int unsigned index, int unsigned step); + mxfp2int_step_seq seq; + int cycles = 0; + env.scb.reset_counters(); + seq = mxfp2int_step_seq::type_id::create("seq"); + seq.step = step; + seq.start(env.cmd_agent.seqr); + while (!env.scb.done()) begin + @(posedge vif.clock); + cycles++; + if (cycles > MXFP2INT_TIMEOUT_CYCLES) + `uvm_fatal("TIMEOUT", $sformatf("Mxfp2IntBall case %0d step %0d timeout", index, step)) + end + `uvm_info("MXFP2INT", $sformatf("Mxfp2IntBall case %0d step %0d passed in %0d cycles", index, + step, cycles), UVM_LOW) + endtask + + task apply_reset(); + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/prefab/balls/mxfp2int/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/mxfp2int/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..e7e94955e --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/ctests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_buckyball_ctest(mxfp4_e8m0_two_block_test.c) +add_buckyball_ctest(mxfp4_e8m0_scale_range_test.c) diff --git a/examples/prefab/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_scale_range_test.c b/examples/prefab/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_scale_range_test.c new file mode 100644 index 000000000..d9b3c34c3 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_scale_range_test.c @@ -0,0 +1,72 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include + +// 20 blocks, so the scales span two rows of the scale bank. The scales cover +// right shifts (rounding toward minus infinity), left shifts that saturate +// and shifts of more than 31 in both directions. +#define NUM_BLOCKS 20 +#define FP4_PER_BLOCK 32 +#define BYTES_PER_BLOCK 16 + +static uint8_t input_mxfp4[NUM_BLOCKS * BYTES_PER_BLOCK] + __attribute__((aligned(64))); +static uint8_t scales_e8m0[32] __attribute__((aligned(64))); +static int8_t output_int8[NUM_BLOCKS * FP4_PER_BLOCK] + __attribute__((aligned(64))); +static const uint8_t scale_of[NUM_BLOCKS] = {0, 1, 90, 120, 124, 125, 126, + 127, 128, 129, 130, 131, 132, 133, + 134, 150, 158, 159, 200, 255}; + +static int raw_fp4(int block, int elem) { return (elem * 7 + block * 3) & 0xF; } + +static int8_t golden(int block, int elem) { + int value = raw_fp4(block, elem) - 8; + int shift = scale_of[block] - 127; + long scaled; + if (shift >= 0) + scaled = (long)value << (shift > 8 ? 8 : shift); + else + scaled = value >> (-shift > 31 ? 31 : -shift); + if (scaled > 127) + scaled = 127; + if (scaled < -128) + scaled = -128; + return (int8_t)scaled; +} + +int main(void) { + for (int b = 0; b < NUM_BLOCKS; b++) { + for (int e = 0; e < FP4_PER_BLOCK; e += 2) + input_mxfp4[b * BYTES_PER_BLOCK + e / 2] = + (uint8_t)((raw_fp4(b, e + 1) << 4) | raw_fp4(b, e)); + scales_e8m0[b] = scale_of[b]; + } + + bb_mem_alloc(1, 1, 1); + bb_mem_alloc(2, 1, 1); + bb_mem_alloc(3, 1, 1); + bb_mvin((uintptr_t)input_mxfp4, 1, NUM_BLOCKS, 1); + bb_mvin((uintptr_t)scales_e8m0, 2, 2, 1); + bb_mxfp2int(1, 2, 3, NUM_BLOCKS); + bb_mvout((uintptr_t)output_int8, 3, NUM_BLOCKS * 2, 1); + bb_mem_release(1); + bb_mem_release(2); + bb_mem_release(3); + + for (int b = 0; b < NUM_BLOCKS; b++) + for (int e = 0; e < FP4_PER_BLOCK; e++) { + int8_t got = output_int8[b * FP4_PER_BLOCK + e]; + int8_t exp = golden(b, e); + if (got != exp) { + printf("mxfp4 scale range mismatch b=%d e=%d got=%d exp=%d\n", b, e, + (int)got, (int)exp); + return 1; + } + } + printf("mxfp4 e8m0 scale range PASSED\n"); + return 0; +} diff --git a/examples/components/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c b/examples/prefab/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c similarity index 71% rename from examples/components/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c rename to examples/prefab/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c index 708c9b18a..cf2e5db7e 100644 --- a/examples/components/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c +++ b/examples/prefab/balls/mxfp2int/workloads/ctests/mxfp4_e8m0_two_block_test.c @@ -1,7 +1,5 @@ #include "buckyball.h" #include -#include -#include #include #include #include @@ -14,7 +12,7 @@ static uint8_t input_mxfp4[NUM_BLOCKS * BYTES_PER_BLOCK] __attribute__((aligned(64))); -static uint8_t scales_e8m0[16] __attribute__((aligned(16))); +static uint8_t scales_e8m0[16] __attribute__((aligned(64))); static int8_t output_int8[NUM_BLOCKS * FP4_PER_BLOCK] __attribute__((aligned(64))); @@ -33,10 +31,6 @@ static int8_t golden(int block, int elem) { } int main(void) { -#ifdef MULTITHREAD - multithread(MULTITHREAD); -#endif - for (int b = 0; b < NUM_BLOCKS; b++) { for (int e = 0; e < FP4_PER_BLOCK; e += 2) { uint8_t lo = gen_fp4(b, e); @@ -47,22 +41,15 @@ int main(void) { scales_e8m0[b] = (uint8_t)(E8M0_BIAS + b); } - mmio_allocator_t mmio_alloc; - mmio_allocator_init(&mmio_alloc); - uint16_t mmio_addr = mmio_allocator_alloc(&mmio_alloc, 1); - if (mmio_addr == (uint16_t)-1) { - printf("FAIL: MMIO alloc\n"); - return 1; - } - - const uint32_t in_bank = 0; - const uint32_t out_bank = 1; + const uint32_t in_bank = 1; + const uint32_t scale_bank = 2; + const uint32_t out_bank = 3; bb_mem_alloc(in_bank, 1, 1); + bb_mem_alloc(scale_bank, 1, 1); bb_mem_alloc(out_bank, 1, 1); - // Publish scales and packed input once before the NPU command chain. - bb_mvin_mmio((uintptr_t)scales_e8m0, mmio_addr, 1, NUM_BLOCKS); bb_mvin((uintptr_t)input_mxfp4, in_bank, NUM_BLOCKS, 1); - bb_mxfp2int(in_bank, out_bank, NUM_BLOCKS); + bb_mvin((uintptr_t)scales_e8m0, scale_bank, 1, 1); + bb_mxfp2int(in_bank, scale_bank, out_bank, NUM_BLOCKS); bb_mvout((uintptr_t)output_int8, out_bank, NUM_BLOCKS * 2, 1); int passed = 1; @@ -78,8 +65,8 @@ int main(void) { } bb_mem_release(in_bank); + bb_mem_release(scale_bank); bb_mem_release(out_bank); - mmio_allocator_free(&mmio_alloc, mmio_addr, 1); printf("mxfp4 e8m0 two-block %s\n", passed ? "PASSED" : "FAILED"); return passed ? 0 : 1; } diff --git a/examples/prefab/balls/mxfp2int/workloads/isa/mxfp2int.h b/examples/prefab/balls/mxfp2int/workloads/isa/mxfp2int.h new file mode 100644 index 000000000..76687e45d --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/isa/mxfp2int.h @@ -0,0 +1,21 @@ +#ifndef _BB_MXFP2INT_H_ +#define _BB_MXFP2INT_H_ + +#include +#include + +// Dequantizes iter MXFP4 blocks of 32 elements to int8. +// op1 bank: block b in row b, element e in byte e / 2, low nibble first; +// the element value is raw - 8. +// op2 bank: E8M0 scale of block b in byte b % 16 of row b / 16. +// wr bank: elements 0..15 of block b in row 2b, 16..31 in row 2b + 1, +// saturate(value * 2^(scale - 127)) with the right shift +// rounding toward minus infinity. +// iter: the number of blocks, 1..half the bank depth. rs2 is reserved and +// zero. +#define bb_mxfp2int(bank_id, scale_bank_id, wr_bank_id, iter) \ + BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(bank_id) | BB_BANK1(scale_bank_id) | \ + BB_BANK2(wr_bank_id) | BB_ITER(iter)), \ + 0, FUNCT7(mxfp2int, MXFP2INT)) + +#endif // _BB_MXFP2INT_H_ diff --git a/examples/components/balls/mxfp2int/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/mxfp2int/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/mxfp2int/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt new file mode 100644 index 000000000..93a1280da --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/CMakeLists.txt @@ -0,0 +1,4 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +set(BUCKYBALL_MLIR_TEST_PREFIX ball) + +add_buckyball_mlir_test(mxfp4_i8_2blocks TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_i8_2blocks.mlir b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_i8_2blocks.mlir new file mode 100644 index 000000000..1f75ffa8b --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_i8_2blocks.mlir @@ -0,0 +1,28 @@ +func.func private @check_result(memref<4x16xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + // 0x99: both nibbles 9, the value 1. + %raw = arith.constant -103 : i8 + %scale = arith.constant 127 : i8 + %input = memref.alloc() : memref<2x16xi8> + %scales = memref.alloc() : memref<1x16xi8> + %output = memref.alloc() : memref<4x16xi8> + linalg.fill ins(%raw : i8) outs(%input : memref<2x16xi8>) + linalg.fill ins(%scale : i8) outs(%scales : memref<1x16xi8>) + linalg.fill ins(%zero_i8 : i8) outs(%output : memref<4x16xi8>) + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mvin %input <{bank = 1 : i64, depth = 2 : i64, stride = 1 : i64}> : memref<2x16xi8> + buckyball.mvin %scales <{bank = 2 : i64, depth = 1 : i64, stride = 1 : i64}> : memref<1x16xi8> + mxfp2int_ball.mxfp4_i8 <{data = 1 : i64, scales = 2 : i64, output = 3 : i64, rows = 2 : i64}> + buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + func.call @check_result(%output) : (memref<4x16xi8>) -> () + memref.dealloc %input : memref<2x16xi8> + memref.dealloc %scales : memref<1x16xi8> + memref.dealloc %output : memref<4x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/components/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks_main.cpp b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_i8_2blocks_main.cpp similarity index 73% rename from examples/components/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks_main.cpp rename to examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_i8_2blocks_main.cpp index ea93e39af..bafe2eb2c 100644 --- a/examples/components/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_unit_scale_2blocks_main.cpp +++ b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/ball/mxfp4_i8_2blocks_main.cpp @@ -1,15 +1,16 @@ #include #include #include + extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { if (rows != 4 || lanes != 16 || stride != 16 || lane_stride != 1) exit(1); - for (int i = 0; i < 64; ++i) + for (int i = 0; i < rows * lanes; ++i) if (data[offset + i] != 1) { - printf("mxfp4 unit-scale mismatch index=%d\n", i); + printf("mxfp4_i8 unit-scale mismatch index=%d\n", i); exit(1); } - printf("mxfp4 unit-scale 2block PASS\n"); + printf("mxfp4_i8 2-block PASS\n"); } diff --git a/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..580170347 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,8 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +set(BUCKYBALL_MLIR_TEST_PREFIX bank) + +set(_lines ${DESIGN_PROFILES_${BUCKYBALL_MLIR_ACTIVE_TARGET}_THREAD_MEM_BANK_ENTRIES}) +# mxfp4_i8 writes two bank rows per block; one scale row holds sixteen blocks. +math(EXPR BB_MXFP_BLOCKS "${_lines} / 2") +math(EXPR BB_MXFP_SCALE_ROWS "(${BB_MXFP_BLOCKS} + 15) / 16") +add_buckyball_mlir_test(mxfp4_i8_bank_blocks TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_i8_bank_blocks.mlir.in b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_i8_bank_blocks.mlir.in new file mode 100644 index 000000000..1a4fe0a99 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_i8_bank_blocks.mlir.in @@ -0,0 +1,59 @@ +func.func private @check_result(memref<@BB_BANK_LINES@x16xi8>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %blocks = arith.constant @BB_MXFP_BLOCKS@ : index + %sixteen = arith.constant 16 : index + %scaleRows = arith.constant @BB_MXFP_SCALE_ROWS@ : index + %bias = arith.constant 125 : index + %eight = arith.constant 8 : index + %input = memref.alloc() alignment = 64 : memref<@BB_MXFP_BLOCKS@x16xi8> + %scales = memref.alloc() alignment = 64 : memref<@BB_MXFP_SCALE_ROWS@x16xi8> + %output = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x16xi8> + // Byte j of block b holds the nibbles (b + 2j) % 16 (low) and + // (b + 2j + 1) % 16 (high); element e has the raw value (b + e) % 16. + scf.for %b = %zero to %blocks step %one { + scf.for %j = %zero to %sixteen step %one { + %j2 = arith.addi %j, %j : index + %e0 = arith.addi %b, %j2 : index + %lo = arith.remui %e0, %sixteen : index + %e1 = arith.addi %e0, %one : index + %hi = arith.remui %e1, %sixteen : index + %hiShifted = arith.muli %hi, %sixteen : index + %byte = arith.addi %hiShifted, %lo : index + %v = arith.index_cast %byte : index to i8 + memref.store %v, %input[%b, %j] : memref<@BB_MXFP_BLOCKS@x16xi8> + } + } + // The scale of block b is 125 + b % 8: shifts -2..5. + scf.for %r = %zero to %scaleRows step %one { + scf.for %j = %zero to %sixteen step %one { + %r16 = arith.muli %r, %sixteen : index + %b = arith.addi %r16, %j : index + %m = arith.remui %b, %eight : index + %s = arith.addi %bias, %m : index + %v = arith.index_cast %s : index to i8 + memref.store %v, %scales[%r, %j] : memref<@BB_MXFP_SCALE_ROWS@x16xi8> + } + } + %ib = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %sb = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %ob = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %il = buckyball.bank_mvin %input, %ib <{depth = @BB_MXFP_BLOCKS@ : i64, stride = 1 : i64}> + : memref<@BB_MXFP_BLOCKS@x16xi8> + %sl = buckyball.bank_mvin %scales, %sb <{depth = @BB_MXFP_SCALE_ROWS@ : i64, stride = 1 : i64}> + : memref<@BB_MXFP_SCALE_ROWS@x16xi8> + %result = mxfp2int_ball.bank_mxfp4_i8 %il, %sl -> %ob + <{rows = @BB_MXFP_BLOCKS@ : i64}> + %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x16xi8> + buckyball.bank_release %il + buckyball.bank_release %sl + buckyball.bank_release %stored + func.call @check_result(%output) : (memref<@BB_BANK_LINES@x16xi8>) -> () + memref.dealloc %input : memref<@BB_MXFP_BLOCKS@x16xi8> + memref.dealloc %scales : memref<@BB_MXFP_SCALE_ROWS@x16xi8> + memref.dealloc %output : memref<@BB_BANK_LINES@x16xi8> + return %zero_i8 : i8 +} diff --git a/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_i8_bank_blocks_main.cpp b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_i8_bank_blocks_main.cpp new file mode 100644 index 000000000..98d2b9de9 --- /dev/null +++ b/examples/prefab/balls/mxfp2int/workloads/mlir_tests/bank/mxfp4_i8_bank_blocks_main.cpp @@ -0,0 +1,28 @@ +#include +#include +#include + +extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, + int64_t rows, int64_t lanes, int64_t stride, + int64_t lane_stride) { + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != 16 || stride != 16 || + lane_stride != 1) + exit(1); + for (int block = 0; block < THREAD_MEM_BANK_ENTRIES / 2; ++block) + for (int element = 0; element < 32; ++element) { + int value = (block + element) % 16 - 8; + int shift = block % 8 - 2; + int expected = shift >= 0 ? value << shift : value >> -shift; + if (expected > 127) + expected = 127; + if (expected < -128) + expected = -128; + if (data[offset + block * 32 + element] != expected) { + printf("mxfp4_i8 bank mismatch block=%d element=%d got=%d " + "expected=%d\n", + block, element, data[offset + block * 32 + element], expected); + exit(1); + } + } + printf("mxfp4_i8 bank PASS blocks=%d\n", THREAD_MEM_BANK_ENTRIES / 2); +} diff --git a/examples/components/balls/mxmm/arch/AddPipe.scala b/examples/prefab/balls/mxmm/arch/AddPipe.scala similarity index 100% rename from examples/components/balls/mxmm/arch/AddPipe.scala rename to examples/prefab/balls/mxmm/arch/AddPipe.scala diff --git a/examples/components/balls/mxmm/arch/Array.scala b/examples/prefab/balls/mxmm/arch/Array.scala similarity index 100% rename from examples/components/balls/mxmm/arch/Array.scala rename to examples/prefab/balls/mxmm/arch/Array.scala diff --git a/examples/components/balls/mxmm/arch/InputWindow.scala b/examples/prefab/balls/mxmm/arch/InputWindow.scala similarity index 100% rename from examples/components/balls/mxmm/arch/InputWindow.scala rename to examples/prefab/balls/mxmm/arch/InputWindow.scala diff --git a/examples/components/balls/mxmm/arch/Mxfp8Decode.scala b/examples/prefab/balls/mxmm/arch/Mxfp8Decode.scala similarity index 100% rename from examples/components/balls/mxmm/arch/Mxfp8Decode.scala rename to examples/prefab/balls/mxmm/arch/Mxfp8Decode.scala diff --git a/examples/components/balls/mxmm/arch/MxmmBall.scala b/examples/prefab/balls/mxmm/arch/MxmmBall.scala similarity index 100% rename from examples/components/balls/mxmm/arch/MxmmBall.scala rename to examples/prefab/balls/mxmm/arch/MxmmBall.scala diff --git a/examples/components/balls/mxmm/arch/PE.scala b/examples/prefab/balls/mxmm/arch/PE.scala similarity index 100% rename from examples/components/balls/mxmm/arch/PE.scala rename to examples/prefab/balls/mxmm/arch/PE.scala diff --git a/examples/components/balls/mxmm/arch/Panels.scala b/examples/prefab/balls/mxmm/arch/Panels.scala similarity index 100% rename from examples/components/balls/mxmm/arch/Panels.scala rename to examples/prefab/balls/mxmm/arch/Panels.scala diff --git a/examples/components/balls/mxmm/arch/Unit.scala b/examples/prefab/balls/mxmm/arch/Unit.scala similarity index 100% rename from examples/components/balls/mxmm/arch/Unit.scala rename to examples/prefab/balls/mxmm/arch/Unit.scala diff --git a/examples/components/balls/mxmm/configs/MxmmBallParam.scala b/examples/prefab/balls/mxmm/configs/MxmmBallParam.scala similarity index 100% rename from examples/components/balls/mxmm/configs/MxmmBallParam.scala rename to examples/prefab/balls/mxmm/configs/MxmmBallParam.scala diff --git a/examples/components/balls/mxmm/configs/default.toml b/examples/prefab/balls/mxmm/configs/default.toml similarity index 100% rename from examples/components/balls/mxmm/configs/default.toml rename to examples/prefab/balls/mxmm/configs/default.toml diff --git a/examples/components/balls/mxmm/configs/mxmm_params.proto b/examples/prefab/balls/mxmm/configs/mxmm_params.proto similarity index 100% rename from examples/components/balls/mxmm/configs/mxmm_params.proto rename to examples/prefab/balls/mxmm/configs/mxmm_params.proto diff --git a/examples/components/balls/mxmm/emu/src/accumulator.rs b/examples/prefab/balls/mxmm/emu/src/accumulator.rs similarity index 100% rename from examples/components/balls/mxmm/emu/src/accumulator.rs rename to examples/prefab/balls/mxmm/emu/src/accumulator.rs diff --git a/examples/components/balls/mxmm/emu/src/dispatch.rs b/examples/prefab/balls/mxmm/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/mxmm/emu/src/dispatch.rs rename to examples/prefab/balls/mxmm/emu/src/dispatch.rs diff --git a/examples/components/balls/mxmm/emu/src/fp32.rs b/examples/prefab/balls/mxmm/emu/src/fp32.rs similarity index 96% rename from examples/components/balls/mxmm/emu/src/fp32.rs rename to examples/prefab/balls/mxmm/emu/src/fp32.rs index 458ae3593..b5d1ffd08 100644 --- a/examples/components/balls/mxmm/emu/src/fp32.rs +++ b/examples/prefab/balls/mxmm/emu/src/fp32.rs @@ -1,10 +1,11 @@ use std::collections::HashMap; use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; use super::accumulator::{Chain, Format}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::ExecContext; pub(crate) fn execute(xs1: u64, xs2: u64, fused: bool, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "mxmm: reads no old destination; rs1[29] must be clear"); let ball_state = ctx.state::>(); let format = if fused { Format::Fma32 } else { Format::F32 }; let rows = (xs2 & 0xfff) as usize; diff --git a/examples/components/balls/mxmm/emu/src/lib.rs b/examples/prefab/balls/mxmm/emu/src/lib.rs similarity index 100% rename from examples/components/balls/mxmm/emu/src/lib.rs rename to examples/prefab/balls/mxmm/emu/src/lib.rs diff --git a/examples/components/balls/mxmm/emu/src/mxfp8.rs b/examples/prefab/balls/mxmm/emu/src/mxfp8.rs similarity index 97% rename from examples/components/balls/mxmm/emu/src/mxfp8.rs rename to examples/prefab/balls/mxmm/emu/src/mxfp8.rs index 6debeb4d6..a02908dc2 100644 --- a/examples/components/balls/mxmm/emu/src/mxfp8.rs +++ b/examples/prefab/balls/mxmm/emu/src/mxfp8.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; use super::accumulator::{Chain, Format}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::ExecContext; fn value(code: u8, factor: f32) -> f32 { @@ -15,6 +15,7 @@ fn value(code: u8, factor: f32) -> f32 { } pub(crate) fn execute(xs1: u64, xs2: u64, window: bool, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "mxmm: reads no old destination; rs1[29] must be clear"); let ball_state = ctx.state::>(); let rows = (xs2 & 0xfff) as usize; let cols = ((xs2 >> 12) & 0xfff) as usize; diff --git a/examples/components/balls/mxmm/stack/kernels/CMakeLists.txt b/examples/prefab/balls/mxmm/stack/kernels/CMakeLists.txt similarity index 98% rename from examples/components/balls/mxmm/stack/kernels/CMakeLists.txt rename to examples/prefab/balls/mxmm/stack/kernels/CMakeLists.txt index ac40de7bd..e1fd04aac 100644 --- a/examples/components/balls/mxmm/stack/kernels/CMakeLists.txt +++ b/examples/prefab/balls/mxmm/stack/kernels/CMakeLists.txt @@ -15,7 +15,7 @@ foreach(SOURCE OUTPUT ${NAME}.o COMMAND "${RVV_CXX}" --target=riscv64 -march=rv64imf_zve32f_zvl128b -mabi=lp64f -mcmodel=medany -ffreestanding -O2 -ffp-contract=off -ffunction-sections -fdata-sections - "-I${DESIGN_REPO}/stack/kernels" "-I${DESIGN_REPO}/bb-tests/workloads/lib" "-I${ISA}" + "-I${DESIGN_REPO}/stack/kernels" "-I${DESIGN_REPO}/bb-tests/workloads/lib" ${THREAD_DEFS} -MD -MF ${NAME}.d -c "${SOURCE}" -o ${NAME}.o DEPENDS "${SOURCE}" DEPFILE ${NAME}.d diff --git a/examples/components/balls/mxmm/stack/kernels/dispatch.cpp b/examples/prefab/balls/mxmm/stack/kernels/dispatch.cpp similarity index 96% rename from examples/components/balls/mxmm/stack/kernels/dispatch.cpp rename to examples/prefab/balls/mxmm/stack/kernels/dispatch.cpp index b8d0f8406..feca99585 100644 --- a/examples/components/balls/mxmm/stack/kernels/dispatch.cpp +++ b/examples/prefab/balls/mxmm/stack/kernels/dispatch.cpp @@ -4,7 +4,6 @@ #include #include #include -#include extern "C" void _mlir_ciface_rvv_norm_window(uint64_t readBank, uint64_t writeBank, @@ -12,7 +11,7 @@ _mlir_ciface_rvv_norm_window(uint64_t readBank, uint64_t writeBank, uint64_t count, uint64_t full, uint64_t start, uint64_t first, uint64_t last, uint64_t bankBytes, uint32_t meanBits, uint32_t epsilonBits) { - constexpr unsigned programBank = VIRTUAL_BANK_NUM; + constexpr unsigned programBank = CORE_VIRTUAL_BANK_COUNT; if (phase == 2) { release_kernel(programBank); return; diff --git a/examples/components/balls/mxmm/stack/kernels/norm_window.cpp b/examples/prefab/balls/mxmm/stack/kernels/norm_window.cpp similarity index 76% rename from examples/components/balls/mxmm/stack/kernels/norm_window.cpp rename to examples/prefab/balls/mxmm/stack/kernels/norm_window.cpp index fab5e461a..f300be84c 100644 --- a/examples/components/balls/mxmm/stack/kernels/norm_window.cpp +++ b/examples/prefab/balls/mxmm/stack/kernels/norm_window.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -14,9 +13,10 @@ extern "C" void norm_window(const Window *parameters, uint32_t *flags) { uint64_t banks = UINT64_C(3) | (UINT64_C(4) << 20) | (uint64_t(parameters->width) << 30); uint64_t zero = 0; - asm volatile(".insn r 0x7b, 3, %c2, x0, %0, %1" ::"r"(banks), "r"(zero), - "i"(BB_FUNC7_MXQUANT) - : "memory"); + asm volatile( + ".insn r 0x7b, 3, %c2, x0, %0, %1" ::"r"(banks), "r"(zero), + "i"(THREAD_BALLDOMAIN_BALLS_mxquant_INSTRUCTIONS_MXQUANT_FUNCT7) + : "memory"); } else if (parameters->phase == 1) { uint64_t banks = UINT64_C(4) | (UINT64_C(3) << 20) | (uint64_t(parameters->count) << 30); @@ -25,9 +25,10 @@ extern "C" void norm_window(const Window *parameters, uint32_t *flags) { (uint64_t(parameters->last) << 25) | (uint64_t(parameters->full) << 32) | (uint64_t(parameters->start) << 48); - asm volatile(".insn r 0x7b, 3, %c2, x0, %0, %1" ::"r"(banks), "r"(config), - "i"(BB_FUNC7_MXMM_MXFP8_WINDOW) - : "memory"); + asm volatile( + ".insn r 0x7b, 3, %c2, x0, %0, %1" ::"r"(banks), "r"(config), + "i"(THREAD_BALLDOMAIN_BALLS_mxmm_INSTRUCTIONS_MXMM_MXFP8_WINDOW_FUNCT7) + : "memory"); } else { asm volatile("ebreak"); } diff --git a/examples/components/balls/mxmm/stack/kernels/norm_window_image.h b/examples/prefab/balls/mxmm/stack/kernels/norm_window_image.h similarity index 100% rename from examples/components/balls/mxmm/stack/kernels/norm_window_image.h rename to examples/prefab/balls/mxmm/stack/kernels/norm_window_image.h diff --git a/examples/components/balls/mxmm/stack/kernels/pack_dispatch.cpp b/examples/prefab/balls/mxmm/stack/kernels/pack_dispatch.cpp similarity index 95% rename from examples/components/balls/mxmm/stack/kernels/pack_dispatch.cpp rename to examples/prefab/balls/mxmm/stack/kernels/pack_dispatch.cpp index 1a1445c09..6b9450458 100644 --- a/examples/components/balls/mxmm/stack/kernels/pack_dispatch.cpp +++ b/examples/prefab/balls/mxmm/stack/kernels/pack_dispatch.cpp @@ -3,7 +3,6 @@ #include #include #include -#include extern "C" void _mlir_ciface_rvv_packmm(uint64_t readBank, uint64_t writeBank, uint64_t phase, uint64_t fullK, @@ -12,7 +11,7 @@ extern "C" void _mlir_ciface_rvv_packmm(uint64_t readBank, uint64_t writeBank, uint64_t columns, uint64_t rows, uint64_t first, uint64_t last, uint64_t fused) { - constexpr unsigned programBank = VIRTUAL_BANK_NUM; + constexpr unsigned programBank = CORE_VIRTUAL_BANK_COUNT; if (phase == 0) { mvin_kernel(packmm_image, sizeof(packmm_image), programBank); return; diff --git a/examples/components/balls/mxmm/stack/kernels/packmm.cpp b/examples/prefab/balls/mxmm/stack/kernels/packmm.cpp similarity index 71% rename from examples/components/balls/mxmm/stack/kernels/packmm.cpp rename to examples/prefab/balls/mxmm/stack/kernels/packmm.cpp index ab8a6a026..c8d33df11 100644 --- a/examples/components/balls/mxmm/stack/kernels/packmm.cpp +++ b/examples/prefab/balls/mxmm/stack/kernels/packmm.cpp @@ -1,5 +1,4 @@ #include "packmm.h" -#include #include extern "C" void rvv_pack(uint32_t *, const uint32_t *, size_t, size_t, size_t, @@ -17,15 +16,19 @@ extern "C" void packmm(const PackMM *parameters, uint32_t *flags) { uint64_t(parameters->rows) | (uint64_t(parameters->columns) << 12) | (uint64_t(parameters->first) << 24) | (uint64_t(parameters->last) << 25); if (parameters->fused) - asm volatile(".insn r 0x7b, 3, %c2, x0, %0, %1" - : - : "r"(banks), "r"(config), "i"(BB_FUNC7_MXMM_FMA32) - : "memory"); + asm volatile( + ".insn r 0x7b, 3, %c2, x0, %0, %1" + : + : "r"(banks), "r"(config), + "i"(THREAD_BALLDOMAIN_BALLS_mxmm_INSTRUCTIONS_MXMM_FMA32_FUNCT7) + : "memory"); else - asm volatile(".insn r 0x7b, 3, %c2, x0, %0, %1" - : - : "r"(banks), "r"(config), "i"(BB_FUNC7_MXMM_F32) - : "memory"); + asm volatile( + ".insn r 0x7b, 3, %c2, x0, %0, %1" + : + : "r"(banks), "r"(config), + "i"(THREAD_BALLDOMAIN_BALLS_mxmm_INSTRUCTIONS_MXMM_F32_FUNCT7) + : "memory"); uint64_t result; asm volatile("csrr %0, fflags" : "=r"(result) : : "memory"); *flags = uint32_t(result); diff --git a/examples/components/balls/mxmm/stack/kernels/packmm.h b/examples/prefab/balls/mxmm/stack/kernels/packmm.h similarity index 100% rename from examples/components/balls/mxmm/stack/kernels/packmm.h rename to examples/prefab/balls/mxmm/stack/kernels/packmm.h diff --git a/examples/components/balls/mxmm/stack/kernels/packmm_image.h b/examples/prefab/balls/mxmm/stack/kernels/packmm_image.h similarity index 100% rename from examples/components/balls/mxmm/stack/kernels/packmm_image.h rename to examples/prefab/balls/mxmm/stack/kernels/packmm_image.h diff --git a/examples/components/balls/mxmm/stack/kernels/window.h b/examples/prefab/balls/mxmm/stack/kernels/window.h similarity index 100% rename from examples/components/balls/mxmm/stack/kernels/window.h rename to examples/prefab/balls/mxmm/stack/kernels/window.h diff --git a/examples/prefab/balls/mxmm/stack/src/mxmmOps.cpp b/examples/prefab/balls/mxmm/stack/src/mxmmOps.cpp new file mode 100644 index 000000000..ce922759b --- /dev/null +++ b/examples/prefab/balls/mxmm/stack/src/mxmmOps.cpp @@ -0,0 +1,167 @@ +//===- mxmmOps.cpp - MxmmBall operations ----------------------------------===// +// +// The dialect of MxmmBall, its virtual operations building their physical ones +// and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "mxmmOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::mxmm_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "mxmmDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "mxmmOps.cpp.inc" + +void MxmmBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "mxmmOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect( + [](DialectRegistry ®istry) { registry.insert(); }), + true); + +// Emits one chain step of `mnemonic`. An MXFP8 operand takes one byte per +// element and one scale per 32; an FP32 operand four bytes per element. A +// spans fullK columns, of which the step reads k from startK; only the window +// format encodes fullK and startK. +LogicalResult emitMatmul(Operation *op, OpBuilder &builder, StringRef mnemonic, + bool mx, bool window, int64_t a, int64_t b, int64_t c, + int64_t rows, int64_t cols, int64_t k, int64_t fullK, + int64_t startK, int64_t base, bool first, bool last) { + const auto &target = buckyball_target::getBuckyballTarget(); + int64_t bytes = target.bankDepth * 16; + int64_t aElements = rows * fullK, bElements = cols * k; + if (target.bankWidthBits != 128 || + !(rows == 1 || (rows > 0 && rows % 16 == 0)) || rows > 4095 || + cols <= 0 || cols % 16 || cols > 4095 || k <= 0 || k % (mx ? 32 : 4) || + k >= (1LL << 33) || base < 0 || base > 63) + return op->emitOpError("shape exceeds the ISA geometry"); + if (window && (fullK % 32 || startK % 32 || startK + k > fullK || + fullK > 65535 || startK > 65535)) + return op->emitOpError("A window exceeds full_k"); + if ((mx ? aElements + aElements / 32 : aElements * 4) > bytes || + (mx ? bElements + bElements / 32 : bElements * 4) > bytes || + base * 16 + rows * cols * 4 > bytes) + return op->emitOpError("operands exceed the bank"); + if (a == b || a == c || b == c) + return op->emitOpError("requires distinct A, B and C banks"); + uint64_t rs2 = field(rows, 0, 11) | field(cols, 12, 23) | + field(first, 24, 24) | field(last, 25, 25) | + field(base, 26, 31); + if (window) + rs2 |= field(fullK, 32, 47) | field(startK, 48, 63); + emitInstruction( + builder, op->getLoc(), buckyball_target::getBuckyballFunct7(mnemonic), + constantI64(builder, op->getLoc(), bankFields({a, b}, c, false, k)), + constantI64(builder, op->getLoc(), rs2)); + return success(); +} +} // namespace + +SmallVector BankMatmulMxfp8Op::getSourceBanks() { + return {getA(), getB()}; +} +Value BankMatmulMxfp8Op::getDestinationBank() { return getC(); } +bool BankMatmulMxfp8Op::readsDestination() { return false; } +Operation *BankMatmulMxfp8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulMxfp8Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getCols(), getK(), + getBase(), getFirst(), getLast()); +} + +SmallVector MatmulMxfp8Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t MatmulMxfp8Op::getDestinationBank() { return getC(); } +LogicalResult MatmulMxfp8Op::lowerToInstructions(OpBuilder &builder) { + return emitMatmul(*this, builder, "MXMM_MXFP8", true, false, getA(), getB(), + getC(), getRows(), getCols(), getK(), getK(), 0, getBase(), + getFirst(), getLast()); +} + +SmallVector BankMatmulMxfp8WindowOp::getSourceBanks() { + return {getA(), getB()}; +} +Value BankMatmulMxfp8WindowOp::getDestinationBank() { return getC(); } +bool BankMatmulMxfp8WindowOp::readsDestination() { return false; } +Operation *BankMatmulMxfp8WindowOp::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulMxfp8WindowOp::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getCols(), getK(), + getFullK(), getStartK(), getBase(), + getFirst(), getLast()); +} + +SmallVector MatmulMxfp8WindowOp::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t MatmulMxfp8WindowOp::getDestinationBank() { return getC(); } +LogicalResult MatmulMxfp8WindowOp::lowerToInstructions(OpBuilder &builder) { + return emitMatmul(*this, builder, "MXMM_MXFP8_WINDOW", true, true, getA(), + getB(), getC(), getRows(), getCols(), getK(), getFullK(), + getStartK(), getBase(), getFirst(), getLast()); +} + +SmallVector BankMatmulFma32Op::getSourceBanks() { + return {getA(), getB()}; +} +Value BankMatmulFma32Op::getDestinationBank() { return getC(); } +bool BankMatmulFma32Op::readsDestination() { return false; } +Operation *BankMatmulFma32Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulFma32Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getCols(), getK(), + getBase(), getFirst(), getLast()); +} + +SmallVector MatmulFma32Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t MatmulFma32Op::getDestinationBank() { return getC(); } +LogicalResult MatmulFma32Op::lowerToInstructions(OpBuilder &builder) { + return emitMatmul(*this, builder, "MXMM_FMA32", false, false, getA(), getB(), + getC(), getRows(), getCols(), getK(), getK(), 0, getBase(), + getFirst(), getLast()); +} + +SmallVector BankMatmulF32Op::getSourceBanks() { + return {getA(), getB()}; +} +Value BankMatmulF32Op::getDestinationBank() { return getC(); } +bool BankMatmulF32Op::readsDestination() { return false; } +Operation *BankMatmulF32Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulF32Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getCols(), getK(), + getBase(), getFirst(), getLast()); +} + +SmallVector MatmulF32Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t MatmulF32Op::getDestinationBank() { return getC(); } +LogicalResult MatmulF32Op::lowerToInstructions(OpBuilder &builder) { + return emitMatmul(*this, builder, "MXMM_F32", false, false, getA(), getB(), + getC(), getRows(), getCols(), getK(), getK(), 0, getBase(), + getFirst(), getLast()); +} diff --git a/examples/prefab/balls/mxmm/stack/src/mxmmOps.h b/examples/prefab/balls/mxmm/stack/src/mxmmOps.h new file mode 100644 index 000000000..4d2203273 --- /dev/null +++ b/examples/prefab/balls/mxmm/stack/src/mxmmOps.h @@ -0,0 +1,20 @@ +//===- mxmmOps.h - MxmmBall operations ------------------------------------===// + +#ifndef MXMM_OPS_H +#define MXMM_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "mxmmDialect.h.inc" + +#define GET_OP_CLASSES +#include "mxmmOps.h.inc" + +#endif // MXMM_OPS_H diff --git a/examples/prefab/balls/mxmm/stack/src/mxmmOps.td b/examples/prefab/balls/mxmm/stack/src/mxmmOps.td new file mode 100644 index 000000000..6a1390cc4 --- /dev/null +++ b/examples/prefab/balls/mxmm/stack/src/mxmmOps.td @@ -0,0 +1,118 @@ +//===- mxmmOps.td - MxmmBall operations --------------------*- tablegen -*-===// +// +// matmul_ (a, b) -> c: one step of a chained FP32 accumulation +// C[rows x cols] += A[rows x k] @ B[k x cols]. `rows` is one or a multiple of +// sixteen, `cols` a multiple of sixteen. The chain lives in the Ball, one per +// hart: `first` starts it from +0, the steps between continue it, `last` +// writes it to C as `rows` x `cols` FP32 in row-major order from row `base` +// (0..63) of the bank; nothing else of C is written, and no step reads C. A +// step without `first` continues the live chain with the same format, rows, +// cols, C and base. +// mxfp8: k is a multiple of 32. A holds rows x k E4M3 codes row-major, then +// the rows x k/32 E8M0 scales row-major. B holds the cols x k codes, column +// c the k codes from byte c * k, then the cols x k/32 scales, column c from +// byte cols * k + c * k/32. +// mxfp8_window: mxfp8 on columns start_k .. start_k + k of an A of +// full_k columns; full_k and start_k are multiples of 32. +// fma32: k is a multiple of four. A holds rows x k FP32 row-major, B column c +// the k FP32 from byte 4 * c * k; every product is fused into the sum. +// f32: fma32 with the product rounded before the sum. +// A, B and C are distinct banks of one 128-bit wide bank each. +// +//===----------------------------------------------------------------------===// + +#ifndef MXMM_OPS +#define MXMM_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def MxmmBall_Dialect : Dialect { + let name = "mxmm_ball"; + let cppNamespace = "::buddy::mxmm_ball"; + let summary = "The operations of MxmmBall"; +} + +class MxmmBall_Op traits = []> + : Op])>; + +def BankMatmulMxfp8Op : MxmmBall_Op<"bank_matmul_mxfp8", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_mxfp8 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$cols, + I64Attr:$k, I64Attr:$base, UnitAttr:$first, + UnitAttr:$last); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulMxfp8Op : MxmmBall_Op<"matmul_mxfp8", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_mxfp8 on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows, + I64Attr:$cols, I64Attr:$k, I64Attr:$base, + UnitAttr:$first, UnitAttr:$last); + let assemblyFormat = "prop-dict attr-dict"; +} + +def BankMatmulMxfp8WindowOp : MxmmBall_Op<"bank_matmul_mxfp8_window", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_mxfp8_window on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$cols, + I64Attr:$k, I64Attr:$full_k, I64Attr:$start_k, + I64Attr:$base, UnitAttr:$first, UnitAttr:$last); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulMxfp8WindowOp : MxmmBall_Op<"matmul_mxfp8_window", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_mxfp8_window on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows, + I64Attr:$cols, I64Attr:$k, I64Attr:$full_k, + I64Attr:$start_k, I64Attr:$base, UnitAttr:$first, + UnitAttr:$last); + let assemblyFormat = "prop-dict attr-dict"; +} + +def BankMatmulFma32Op : MxmmBall_Op<"bank_matmul_fma32", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_fma32 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$cols, + I64Attr:$k, I64Attr:$base, UnitAttr:$first, + UnitAttr:$last); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulFma32Op : MxmmBall_Op<"matmul_fma32", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_fma32 on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows, + I64Attr:$cols, I64Attr:$k, I64Attr:$base, + UnitAttr:$first, UnitAttr:$last); + let assemblyFormat = "prop-dict attr-dict"; +} + +def BankMatmulF32Op : MxmmBall_Op<"bank_matmul_f32", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_f32 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$cols, + I64Attr:$k, I64Attr:$base, UnitAttr:$first, + UnitAttr:$last); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulF32Op : MxmmBall_Op<"matmul_f32", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_f32 on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows, + I64Attr:$cols, I64Attr:$k, I64Attr:$base, + UnitAttr:$first, UnitAttr:$last); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // MXMM_OPS diff --git a/examples/prefab/balls/mxmm/verify/casegen/Cargo.lock b/examples/prefab/balls/mxmm/verify/casegen/Cargo.lock new file mode 100644 index 000000000..4937a413f --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "mxmm-casegen" +version = "0.1.0" diff --git a/examples/prefab/balls/mxmm/verify/casegen/Cargo.toml b/examples/prefab/balls/mxmm/verify/casegen/Cargo.toml new file mode 100644 index 000000000..492e99d44 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mxmm-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["rlib", "cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/prefab/balls/mxmm/verify/casegen/src/casegen.rs b/examples/prefab/balls/mxmm/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..5ed603413 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/casegen/src/casegen.rs @@ -0,0 +1,408 @@ +// MXMM reference model, the contract shared with the BEMU model +// (emu/src/mxfp8.rs, emu/src/fp32.rs) and MxmmBall: +// +// rs2[11:0] = M (1 or a multiple of 16), rs2[23:12] = N (multiple of 16), +// rs2[24] = first, rs2[25] = last, rs2[31:26] = C base row, iter = K. +// MXMM_MXFP8_WINDOW adds rs2[47:32] = fullK and rs2[63:48] = startK; the other +// formats reserve rs2[63:32]. +// +// MXFP8 (K multiple of 32): A holds M rows of fullK E4M3 codes followed by the +// E8M0 scales (row r, block j at byte M * fullK + r * fullK / 32 + j); B holds +// N columns of K codes followed by their scales. FP32 (K multiple of four): A +// is M rows of K floats, B is N columns of K floats. +// C[r][c] accumulates sum_k A[r][k] * B[c][k] in k order with a fused +// multiply-add (MXFP8, FMA32) or a rounded product then a rounded add (F32), +// starting from zero on first and from the chain otherwise. On last the chain +// is written to C as row-major FP32 from row base; otherwise nothing is +// written. + +pub const ROW_BYTES: usize = 16; +pub const NUM_CASES: u32 = 8; + +pub const FORMAT_MXFP8: u32 = 0; +pub const FORMAT_WINDOW: u32 = 1; +pub const FORMAT_FMA32: u32 = 2; +pub const FORMAT_F32: u32 = 3; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MxmmCmd { + pub format: u32, + pub rob_id: u32, + pub iter: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, +} + +#[derive(Clone, Debug)] +pub struct Preload { + pub bank: u32, + pub group: u32, + pub addr: u32, + pub data: u128, +} + +#[derive(Clone, Debug)] +pub struct Write { + pub addr: u32, + pub mask: u16, + pub data: u128, +} + +#[derive(Clone, Debug)] +pub struct Step { + pub cmd: MxmmCmd, + pub preload: Vec, + pub reads: [Vec; 2], + pub writes: Vec, +} + +#[derive(Clone, Debug)] +pub struct Case { + pub steps: Vec, +} + +struct Rng(u32); + +impl Rng { + fn next(&mut self) -> u32 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 17; + self.0 ^= self.0 << 5; + self.0 + } + + // E4M3 code other than NaN (0x7f, 0xff). + fn code(&mut self) -> u8 { + loop { + let code = self.next() as u8; + if code & 0x7f != 0x7f { + return code; + } + } + } + + // E8M0 scale near one, sometimes the smallest scale. + fn scale(&mut self) -> u8 { + if self.next() % 8 == 0 { + 0 + } else { + 120 + (self.next() % 14) as u8 + } + } + + fn float(&mut self) -> f32 { + let bits = (self.next() & 0x807f_ffff) | ((120 + self.next() % 12) << 23); + f32::from_bits(bits) + } +} + +fn word(bytes: &[u8]) -> u128 { + let mut out = 0u128; + for (lane, byte) in bytes.iter().enumerate() { + out |= u128::from(*byte) << (lane * 8); + } + out +} + +fn decode(code: u8, scale: u8) -> f32 { + assert!(code & 0x7f != 0x7f && scale != 255, "mxfp8: NaN encoding"); + let factor = if scale == 0 { f32::from_bits(1 << 22) } else { f32::from_bits((scale as u32) << 23) }; + let exponent = (code >> 3) & 15; + let mantissa = (code & 7) as u32; + let normal = ((exponent as u32 + 120) << 23) | (mantissa << 20); + let subnormal = (mantissa as f32 / 512.0).to_bits(); + let element = f32::from_bits(if exponent == 0 { subnormal } else { normal }); + let value = f32::from_bits((element * factor).to_bits() | ((code as u32 & 128) << 24)); + assert!(value.is_finite(), "mxfp8: decoded value overflows FP32"); + value +} + +// One operand bank: its byte image and the rows the Ball reads, in order. +struct Operand { + bytes: Vec, + reads: Vec, + // values[row][k] (A) or values[col][k] (B) seen by the command. + values: Vec>, +} + +fn sequential_reads(words: usize, start: usize) -> Vec { + (start..start + words).map(|w| w as u32).collect() +} + +fn mx_operand(rng: &mut Rng, rows: usize, k: usize) -> Operand { + let codes: Vec = (0..rows * k).map(|_| rng.code()).collect(); + let scales: Vec = (0..rows * k / 32).map(|_| rng.scale()).collect(); + let values = (0..rows) + .map(|r| (0..k).map(|i| decode(codes[r * k + i], scales[r * (k / 32) + i / 32])).collect()) + .collect(); + let mut bytes = codes; + bytes.extend_from_slice(&scales); + let code_words = rows * k / ROW_BYTES; + let mut reads = sequential_reads(code_words, 0); + reads.extend(sequential_reads((rows * k / 32).div_ceil(ROW_BYTES), code_words)); + Operand { bytes, reads, values } +} + +fn window_operand(image: &[u8], rows: usize, full_k: usize, start_k: usize, k: usize) -> Operand { + let scales = rows * full_k; + let values = (0..rows) + .map(|r| { + (0..k) + .map(|i| { + let col = start_k + i; + decode(image[r * full_k + col], image[scales + r * (full_k / 32) + col / 32]) + }) + .collect() + }) + .collect(); + let mut reads = Vec::new(); + for r in 0..rows { + for o in 0..k / ROW_BYTES { + reads.push((r * (full_k / ROW_BYTES) + start_k / ROW_BYTES + o) as u32); + } + } + for r in 0..rows { + let first = scales + r * (full_k / 32) + start_k / 32; + for o in 0..k / 32 { + if o == 0 || (first + o) % ROW_BYTES == 0 { + reads.push(((first + o) / ROW_BYTES) as u32); + } + } + } + Operand { bytes: image.to_vec(), reads, values } +} + +fn f32_operand(rng: &mut Rng, rows: usize, k: usize, zero_rows: &[usize]) -> Operand { + let values: Vec> = (0..rows) + .map(|r| (0..k).map(|_| if zero_rows.contains(&r) { 0.0 } else { rng.float() }).collect()) + .collect(); + let bytes: Vec = values.iter().flatten().flat_map(|x| x.to_le_bytes()).collect(); + let reads = sequential_reads(rows * k * 4 / ROW_BYTES, 0); + Operand { bytes, reads, values } +} + +fn mx_window_image(rng: &mut Rng, rows: usize, full_k: usize) -> Vec { + let mut image: Vec = (0..rows * full_k).map(|_| rng.code()).collect(); + image.extend((0..rows * full_k / 32).map(|_| rng.scale())); + image +} + +struct Command { + format: u32, + rob_id: u32, + banks: [u32; 3], + groups: [u32; 3], + m: usize, + n: usize, + k: usize, + first: bool, + last: bool, + base: usize, + full_k: usize, + start_k: usize, +} + +struct Chain { + format: u32, + m: usize, + n: usize, + bank: u32, + base: usize, + values: Vec, +} + +fn step(bank_entries: u32, chain: &mut Option, c: Command, a: Operand, b: Operand) -> Step { + let bank_bytes = bank_entries as usize * ROW_BYTES; + let mx = c.format == FORMAT_MXFP8 || c.format == FORMAT_WINDOW; + if !(c.m == 1 || (c.m > 0 && c.m % 16 == 0)) || c.n == 0 || c.n % 16 != 0 || c.k == 0 || c.k % (if mx { 32 } else { 4 }) != 0 { + panic!("mxmm: invalid shape"); + } + if c.format == FORMAT_WINDOW && (c.full_k == 0 || c.full_k % 32 != 0 || c.start_k % 32 != 0 || c.start_k + c.k > c.full_k) { + panic!("mxmm: invalid activation window"); + } + if c.format != FORMAT_WINDOW && (c.full_k != c.k || c.start_k != 0) { + panic!("mxmm: window fields are reserved"); + } + if c.banks[0] == c.banks[1] || c.banks[0] == c.banks[2] || c.banks[1] == c.banks[2] { + panic!("mxmm: banks must differ"); + } + if a.bytes.len() > bank_bytes || b.bytes.len() > bank_bytes || c.base * ROW_BYTES + c.m * c.n * 4 > bank_bytes { + panic!("mxmm: bank footprint exceeds depth"); + } + // The accumulator buffer holds bank_entries / 4 lines of 16 results. + if c.m * c.n > bank_entries as usize * 4 { + panic!("mxmm: result exceeds the accumulator buffer"); + } + if a.values.len() != c.m || b.values.len() != c.n || a.values.iter().chain(b.values.iter()).any(|v| v.len() != c.k) { + panic!("mxmm: operand shape mismatch"); + } + let mut live = if c.first { + if chain.is_some() { + panic!("mxmm: first while chain is live"); + } + Chain { format: c.format, m: c.m, n: c.n, bank: c.banks[2], base: c.base, values: vec![0f32; c.m * c.n] } + } else { + chain.take().expect("mxmm: continuation without first") + }; + if live.format != c.format || live.m != c.m || live.n != c.n || live.bank != c.banks[2] || live.base != c.base { + panic!("mxmm: continuation changed destination"); + } + for row in 0..c.m { + for col in 0..c.n { + let mut sum = live.values[row * c.n + col]; + for inner in 0..c.k { + let (x, y) = (a.values[row][inner], b.values[col][inner]); + sum = if c.format == FORMAT_F32 { (x * y) + sum } else { x.mul_add(y, sum) }; + } + assert!(sum.is_finite(), "mxmm: accumulator overflow"); + live.values[row * c.n + col] = sum; + } + } + let mut writes = Vec::new(); + if c.last { + let bytes: Vec = live.values.iter().flat_map(|x| x.to_le_bytes()).collect(); + for (w, chunk) in bytes.chunks(ROW_BYTES).enumerate() { + writes.push(Write { addr: (c.base + w) as u32, mask: 0xffff, data: word(chunk) }); + } + } else { + *chain = Some(live); + } + let mut preload = Vec::new(); + for (operand, image) in [&a.bytes, &b.bytes].into_iter().enumerate() { + let mut padded = image.clone(); + padded.resize(image.len().div_ceil(ROW_BYTES) * ROW_BYTES, 0); + for (row, chunk) in padded.chunks(ROW_BYTES).enumerate() { + preload.push(Preload { bank: c.banks[operand], group: c.groups[operand], addr: row as u32, data: word(chunk) }); + } + } + let mut rs2 = (c.m as u64) | ((c.n as u64) << 12) | (u64::from(c.first) << 24) | (u64::from(c.last) << 25) | ((c.base as u64) << 26); + if c.format == FORMAT_WINDOW { + rs2 |= ((c.full_k as u64) << 32) | ((c.start_k as u64) << 48); + } + Step { + cmd: MxmmCmd { + format: c.format, + rob_id: c.rob_id, + iter: c.k as u32, + rs2_lo: rs2 as u32, + rs2_hi: (rs2 >> 32) as u32, + op1_bank: c.banks[0], + op2_bank: c.banks[1], + wr_bank: c.banks[2], + op1_group: c.groups[0], + op2_group: c.groups[1], + wr_group: c.groups[2], + op1_col: 1, + op2_col: 1, + wr_col: 1, + }, + preload, + reads: [a.reads, b.reads], + writes, + } +} + +fn command(format: u32, rob_id: u32, banks: [u32; 3], groups: [u32; 3], m: usize, n: usize, k: usize, first: bool, last: bool, base: usize) -> Command { + Command { format, rob_id, banks, groups, m, n, k, first, last, base, full_k: k, start_k: 0 } +} + +pub fn gen_case(index: u32, bank_entries: u32) -> Case { + let mut rng = Rng(0x1234_5679 ^ index.wrapping_mul(0x9e37_79b9)); + let mut chain = None; + let mut steps = Vec::new(); + match index { + // One MXFP8 tile. + 0 => { + let (a, b) = (mx_operand(&mut rng, 16, 32), mx_operand(&mut rng, 16, 32)); + steps.push(step(bank_entries, &mut chain, command(FORMAT_MXFP8, 1, [1, 2, 3], [0, 0, 0], 16, 16, 32, true, true, 0), a, b)); + } + // MXFP8 vector (M = 1) over two scale blocks. + 1 => { + let (a, b) = (mx_operand(&mut rng, 1, 64), mx_operand(&mut rng, 32, 64)); + steps.push(step(bank_entries, &mut chain, command(FORMAT_MXFP8, 2, [4, 5, 6], [1, 2, 3], 1, 32, 64, true, true, 3), a, b)); + } + // FMA32 with two column groups. + 2 => { + let (a, b) = (f32_operand(&mut rng, 16, 8, &[]), f32_operand(&mut rng, 32, 8, &[])); + steps.push(step(bank_entries, &mut chain, command(FORMAT_FMA32, 3, [7, 8, 9], [4, 5, 6], 16, 32, 8, true, true, 1), a, b)); + } + // F32 (rounded product, rounded add) with two row groups and zero rows. + 3 => { + let (a, b) = (f32_operand(&mut rng, 32, 4, &[0, 17]), f32_operand(&mut rng, 16, 4, &[])); + steps.push(step(bank_entries, &mut chain, command(FORMAT_F32, 4, [10, 11, 12], [7, 8, 9], 32, 16, 4, true, true, 2), a, b)); + } + // MXFP8 chain: first, continuation, last. + 4 => { + for (i, (first, last)) in [(true, false), (false, false), (false, true)].into_iter().enumerate() { + let (a, b) = (mx_operand(&mut rng, 16, 32), mx_operand(&mut rng, 16, 32)); + steps.push(step(bank_entries, &mut chain, command(FORMAT_MXFP8, 5 + i as u32, [13, 14, 15], [10, 11, 12], 16, 16, 32, first, last, 5), a, b)); + } + } + // Sliding activation window over fullK = 96 accumulated as one chain. + 5 => { + let image = mx_window_image(&mut rng, 16, 96); + for (i, start_k) in [0usize, 32, 64].into_iter().enumerate() { + let a = window_operand(&image, 16, 96, start_k, 32); + let b = mx_operand(&mut rng, 16, 32); + let c = Command { full_k: 96, start_k, ..command(FORMAT_WINDOW, 8 + i as u32, [16, 17, 18], [13, 14, 15], 16, 16, 32, i == 0, i == 2, 7) }; + steps.push(step(bank_entries, &mut chain, c, a, b)); + } + // A single-command vector window over the upper half of fullK = 64. + let image = mx_window_image(&mut rng, 1, 64); + let a = window_operand(&image, 1, 64, 32, 32); + let b = mx_operand(&mut rng, 32, 32); + let c = Command { full_k: 64, start_k: 32, ..command(FORMAT_WINDOW, 15, [16, 17, 18], [13, 14, 15], 1, 32, 32, true, true, 4) }; + steps.push(step(bank_entries, &mut chain, c, a, b)); + } + // Eight MXFP8 panels (two batches of contexts) filling the C bank. + 6 => { + let (a, b) = (mx_operand(&mut rng, 32, 32), mx_operand(&mut rng, 64, 32)); + steps.push(step(bank_entries, &mut chain, command(FORMAT_MXFP8, 11, [1021, 1022, 1023], [31, 30, 29], 32, 64, 32, true, true, 0), a, b)); + } + // F32 chain, then an FMA32 vector after the chain closes. + 7 => { + for (i, (first, last)) in [(true, false), (false, true)].into_iter().enumerate() { + let (a, b) = (f32_operand(&mut rng, 16, 8, &[3]), f32_operand(&mut rng, 16, 8, &[])); + steps.push(step(bank_entries, &mut chain, command(FORMAT_F32, 12 + i as u32, [19, 20, 21], [16, 17, 18], 16, 16, 8, first, last, 9), a, b)); + } + let (a, b) = (f32_operand(&mut rng, 1, 12, &[]), f32_operand(&mut rng, 48, 12, &[])); + steps.push(step(bank_entries, &mut chain, command(FORMAT_FMA32, 14, [22, 23, 24], [19, 20, 21], 1, 48, 12, true, true, 0), a, b)); + } + _ => panic!("mxmm: unsupported case {index}"), + } + if chain.is_some() { + panic!("mxmm: case {index} leaves a live chain"); + } + Case { steps } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cases_build() { + for index in 0..NUM_CASES { + gen_case(index, 512); + } + } + + #[test] + fn decode_edges() { + assert_eq!(decode(0x38, 127), 1.0); + assert_eq!(decode(0x7e, 127), 448.0); + assert_eq!(decode(0x01, 127), 1.0 / 512.0); + assert_eq!(decode(0x80, 127).to_bits(), 0x8000_0000); + assert_eq!(decode(0x01, 0), f32::from_bits(1 << 22) / 512.0); + } +} diff --git a/examples/prefab/balls/mxmm/verify/casegen/src/config.rs b/examples/prefab/balls/mxmm/verify/casegen/src/config.rs new file mode 100644 index 000000000..241755f0f --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/casegen/src/config.rs @@ -0,0 +1,9 @@ +pub fn bank_entries() -> u32 { + let path = std::env::var("BB_VERIFY_CONFIG").expect("BB_VERIFY_CONFIG is required"); + let text = std::fs::read_to_string(path).expect("failed to read BB_VERIFY_CONFIG"); + text.lines() + .find_map(|line| line.strip_prefix("bank_entries=")) + .unwrap_or_else(|| panic!("bank_entries missing from BB_VERIFY_CONFIG")) + .parse() + .expect("invalid bank_entries in BB_VERIFY_CONFIG") +} diff --git a/examples/prefab/balls/mxmm/verify/casegen/src/lib.rs b/examples/prefab/balls/mxmm/verify/casegen/src/lib.rs new file mode 100644 index 000000000..93d67090b --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/casegen/src/lib.rs @@ -0,0 +1,116 @@ +mod casegen; +mod config; + +use std::cell::RefCell; + +use casegen::{Case, MxmmCmd, Step}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn mxmm_num_cases() -> u32 { + casegen::NUM_CASES +} + +#[no_mangle] +pub extern "C" fn mxmm_case_load(index: u32) -> u32 { + let case = casegen::gen_case(index, config::bank_entries()); + let steps = case.steps.len() as u32; + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + steps +} + +fn current_step R, R>(step: u32, f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => match case.steps.get(step as usize) { + Some(s) => f(s), + None => panic!("mxmm_case: step {step} out of range {}", case.steps.len()), + }, + None => panic!("mxmm_case: no case loaded; call mxmm_case_load first"), + }) +} + +fn idx(index: u32, limit: usize, what: &str) -> usize { + if index as usize >= limit { + panic!("mxmm_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn mxmm_case_cmd(step: u32, out_ptr: *mut MxmmCmd) { + if out_ptr.is_null() { + panic!("mxmm_case_cmd: null out_ptr"); + } + current_step(step, |s| unsafe { *out_ptr = s.cmd }); +} + +#[no_mangle] +pub extern "C" fn mxmm_case_preload_count(step: u32) -> u32 { + current_step(step, |s| s.preload.len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_preload_bank(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].bank) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_preload_group(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].group) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_preload_addr(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].addr) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_preload_lo(step: u32, i: u32) -> u64 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].data as u64) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_preload_hi(step: u32, i: u32) -> u64 { + current_step(step, |s| (s.preload[idx(i, s.preload.len(), "preload")].data >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_read_count(step: u32, port: u32) -> u32 { + current_step(step, |s| s.reads[idx(port, 2, "read_port")].len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_read_addr(step: u32, port: u32, i: u32) -> u32 { + current_step(step, |s| { + let reads = &s.reads[idx(port, 2, "read_port")]; + reads[idx(i, reads.len(), "read")] + }) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_write_count(step: u32) -> u32 { + current_step(step, |s| s.writes.len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_write_addr(step: u32, i: u32) -> u32 { + current_step(step, |s| s.writes[idx(i, s.writes.len(), "write")].addr) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_write_mask(step: u32, i: u32) -> u32 { + current_step(step, |s| u32::from(s.writes[idx(i, s.writes.len(), "write")].mask)) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_write_lo(step: u32, i: u32) -> u64 { + current_step(step, |s| s.writes[idx(i, s.writes.len(), "write")].data as u64) +} + +#[no_mangle] +pub extern "C" fn mxmm_case_write_hi(step: u32, i: u32) -> u64 { + current_step(step, |s| (s.writes[idx(i, s.writes.len(), "write")].data >> 64) as u64) +} diff --git a/examples/prefab/balls/mxmm/verify/filelists/mxmm_ball.f b/examples/prefab/balls/mxmm/verify/filelists/mxmm_ball.f new file mode 100644 index 000000000..fe5169644 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/filelists/mxmm_ball.f @@ -0,0 +1,40 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+src ++incdir+src/pkg ++incdir+src/common ++incdir+src/seq ++incdir+src/cov ++incdir+src/env ++incdir+src/tests ++incdir+@RTL@ + +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv + +@RTL@/MulAddRecFNToRaw_preMul_e8_s24.sv +@RTL@/MulAddRecFNToRaw_postMul_e8_s24.sv +@RTL@/RoundAnyRawFNToRecFN_ie8_is26_oe8_os24.sv +@RTL@/RoundRawFNToRecFN_e8_s24.sv +@RTL@/Fma_l3_e8_s24.sv +@RTL@/AddRawFN.sv +@RTL@/AddPipe.sv +@RTL@/PE.sv +@RTL@/Array.sv +@RTL@/Mxfp8Decode.sv +@RTL@/codes_32x128.sv +@RTL@/scales_16x8.sv +@RTL@/Panels.sv +@RTL@/InputWindow.sv +@RTL@/accumulator_128x512.sv +@RTL@/Unit.sv +@RTL@/MxmmBall.sv + +src/pkg/mxmm_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/mxmm/verify/src/common/mxmm_defs.svh b/examples/prefab/balls/mxmm/verify/src/common/mxmm_defs.svh new file mode 100644 index 000000000..d58c20f3c --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/common/mxmm_defs.svh @@ -0,0 +1,51 @@ +typedef struct { + int unsigned format; + int unsigned rob_id; + int unsigned iter; + int unsigned rs2_lo; + int unsigned rs2_hi; + int unsigned op1_bank; + int unsigned op2_bank; + int unsigned wr_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; + int unsigned op1_col; + int unsigned op2_col; + int unsigned wr_col; +} mxmm_cmd_dpi_t; + +import "DPI-C" function int unsigned mxmm_num_cases(); +import "DPI-C" function int unsigned mxmm_case_load(input int unsigned index); +import "DPI-C" function void mxmm_case_cmd(input int unsigned step, output mxmm_cmd_dpi_t cmd); +import "DPI-C" function int unsigned mxmm_case_preload_count(input int unsigned step); +import "DPI-C" function int unsigned mxmm_case_preload_bank(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxmm_case_preload_group(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxmm_case_preload_addr(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxmm_case_preload_lo(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxmm_case_preload_hi(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxmm_case_read_count(input int unsigned step, input int unsigned port); +import "DPI-C" function int unsigned mxmm_case_read_addr(input int unsigned step, input int unsigned port, input int unsigned i); +import "DPI-C" function int unsigned mxmm_case_write_count(input int unsigned step); +import "DPI-C" function int unsigned mxmm_case_write_addr(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxmm_case_write_mask(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxmm_case_write_lo(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxmm_case_write_hi(input int unsigned step, input int unsigned i); + +`ifndef MXMM_MXFP8_FUNCT7 +`error "MXMM_MXFP8_FUNCT7 must be provided by the selected ballISA" +`endif +`ifndef MXMM_MXFP8_WINDOW_FUNCT7 +`error "MXMM_MXFP8_WINDOW_FUNCT7 must be provided by the selected ballISA" +`endif +`ifndef MXMM_FMA32_FUNCT7 +`error "MXMM_FMA32_FUNCT7 must be provided by the selected ballISA" +`endif +`ifndef MXMM_F32_FUNCT7 +`error "MXMM_F32_FUNCT7 must be provided by the selected ballISA" +`endif + +// Formats of the reference model, in the order of mxmm_case_cmd.format. +localparam int MXMM_FUNCT7[4] = '{`MXMM_MXFP8_FUNCT7, `MXMM_MXFP8_WINDOW_FUNCT7, `MXMM_FMA32_FUNCT7, + `MXMM_F32_FUNCT7}; +localparam int MXMM_TIMEOUT_CYCLES = 100000; diff --git a/examples/prefab/balls/mxmm/verify/src/common/mxmm_items.svh b/examples/prefab/balls/mxmm/verify/src/common/mxmm_items.svh new file mode 100644 index 000000000..f4219db6c --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/common/mxmm_items.svh @@ -0,0 +1,89 @@ +class mxmm_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(mxmm_cmd_item) + + int unsigned format; + int unsigned preload_bank[]; + int unsigned preload_group[]; + int unsigned preload_addr[]; + bit [127:0] preload_data[]; + int unsigned read_addr0[]; + int unsigned read_addr1[]; + int unsigned write_addr[]; + bit [15:0] write_mask[]; + bit [127:0] write_data[]; + + function new(string name = "mxmm_cmd_item"); + super.new(name); + endfunction + + function void load_rust_step(int unsigned step); + mxmm_cmd_dpi_t cmd; + int unsigned n; + + mxmm_case_cmd(step, cmd); + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + if (cmd.format > 3) `uvm_fatal("CASE", $sformatf("invalid format %0d", cmd.format)) + format = cmd.format; + funct = MXMM_FUNCT7[cmd.format]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; + if (rob_id != cmd.rob_id || op1_bank != cmd.op1_bank || op2_bank != cmd.op2_bank || + wr_bank != cmd.wr_bank || op1_group != cmd.op1_group || op2_group != cmd.op2_group || + wr_group != cmd.wr_group) + `uvm_fatal("CASE", "command field exceeds the configured BLink width") + + n = mxmm_case_preload_count(step); + preload_bank = new[n]; + preload_group = new[n]; + preload_addr = new[n]; + preload_data = new[n]; + for (int unsigned i = 0; i < n; i++) begin + preload_bank[i] = mxmm_case_preload_bank(step, i); + preload_group[i] = mxmm_case_preload_group(step, i); + preload_addr[i] = mxmm_case_preload_addr(step, i); + preload_data[i] = {mxmm_case_preload_hi(step, i), mxmm_case_preload_lo(step, i)}; + end + n = mxmm_case_read_count(step, 0); + read_addr0 = new[n]; + for (int unsigned i = 0; i < n; i++) read_addr0[i] = mxmm_case_read_addr(step, 0, i); + n = mxmm_case_read_count(step, 1); + read_addr1 = new[n]; + for (int unsigned i = 0; i < n; i++) read_addr1[i] = mxmm_case_read_addr(step, 1, i); + n = mxmm_case_write_count(step); + write_addr = new[n]; + write_mask = new[n]; + write_data = new[n]; + for (int unsigned i = 0; i < n; i++) begin + write_addr[i] = mxmm_case_write_addr(step, i); + write_mask[i] = mxmm_case_write_mask(step, i); + write_data[i] = {mxmm_case_write_hi(step, i), mxmm_case_write_lo(step, i)}; + end + endfunction + + function void do_copy(uvm_object rhs); + mxmm_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not mxmm_cmd_item") + format = rhs_.format; + preload_bank = rhs_.preload_bank; + preload_group = rhs_.preload_group; + preload_addr = rhs_.preload_addr; + preload_data = rhs_.preload_data; + read_addr0 = rhs_.read_addr0; + read_addr1 = rhs_.read_addr1; + write_addr = rhs_.write_addr; + write_mask = rhs_.write_mask; + write_data = rhs_.write_data; + endfunction +endclass diff --git a/examples/prefab/balls/mxmm/verify/src/cov/mxmm_cov.svh b/examples/prefab/balls/mxmm/verify/src/cov/mxmm_cov.svh new file mode 100644 index 000000000..3fb16b408 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/cov/mxmm_cov.svh @@ -0,0 +1,59 @@ +class mxmm_cov extends uvm_component; + `uvm_component_utils(mxmm_cov) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, mxmm_cov) stim_imp; + + int unsigned cur_format; + bit [1:0] cur_chain; + int unsigned cur_m; + int unsigned cur_panels; + int unsigned cmd_count; + + covergroup cmd_cg; + coverpoint cur_format {bins mxfp8 = {0}; bins window = {1}; bins fma32 = {2}; bins f32 = {3};} + // {first, last} + coverpoint cur_chain { + bins single = {2'b11}; bins open = {2'b10}; bins middle = {2'b00}; bins close = {2'b01}; + } + coverpoint cur_m {bins vector = {1}; bins tile = {16}; bins tiles = {[32 : 4095]};} + // Panels beyond the seven contexts need a second batch. + coverpoint cur_panels { + bins one = {1}; bins batch = {[2 : 7]}; bins batches = {[8 : 4095]}; + } + format_chain: cross cur_format, cur_chain{ + ignore_bins fused_chain = binsof (cur_format) intersect {2} && !binsof (cur_chain) intersect { + 2'b11 + }; + ignore_bins f32_middle = binsof (cur_format) intersect {3} && binsof (cur_chain) intersect { + 2'b00 + }; + } + endgroup + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_cg = new(); + endfunction + + function void write_stim(bb_blink_cmd_item item); + mxmm_cmd_item stim; + int unsigned n; + if (!$cast(stim, item)) `uvm_fatal("COV", "stim item is not mxmm_cmd_item") + cur_format = stim.format; + cur_chain = {stim.rs2[24], stim.rs2[25]}; + cur_m = stim.rs2[11:0]; + n = stim.rs2[23:12]; + cur_panels = (cur_m == 1 ? 1 : cur_m / 16) * (n / 16); + cmd_cg.sample(); + cmd_count++; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("mxmm command coverage %0.2f%%", cmd_cg.get_coverage())) + endfunction +endclass diff --git a/examples/prefab/balls/mxmm/verify/src/env/mxmm_env.svh b/examples/prefab/balls/mxmm/verify/src/env/mxmm_env.svh new file mode 100644 index 000000000..f8a602322 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/env/mxmm_env.svh @@ -0,0 +1,27 @@ +class mxmm_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(mxmm_env) + + mxmm_scoreboard scb; + mxmm_cov icov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scb = mxmm_scoreboard::type_id::create("scb", this); + icov = mxmm_cov::type_id::create("icov", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + scb.mem_model = mem_model; + cmd_agent.stim_ap.connect(scb.stim_imp); + cmd_agent.stim_ap.connect(icov.stim_imp); + cmd_agent.cmd_ap.connect(scb.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction +endclass diff --git a/examples/prefab/balls/mxmm/verify/src/env/mxmm_scoreboard.svh b/examples/prefab/balls/mxmm/verify/src/env/mxmm_scoreboard.svh new file mode 100644 index 000000000..82085ca03 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/env/mxmm_scoreboard.svh @@ -0,0 +1,196 @@ +class mxmm_scoreboard extends uvm_scoreboard; + `uvm_component_utils(mxmm_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, mxmm_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, mxmm_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, mxmm_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, mxmm_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, mxmm_scoreboard) resp_imp; + + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + + mxmm_cmd_item stim_q[$]; + int unsigned cmd_count; + int unsigned read_count[2]; + int unsigned write_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + mxmm_cmd_item got; + mxmm_cmd_item clone; + + if (stim_q.size() != 0) `uvm_fatal("SCB", "single outstanding command supported") + if (!$cast(got, item)) `uvm_fatal("SCB", "stim item is not mxmm_cmd_item") + if (!$cast(clone, got.clone())) `uvm_fatal("SCB", "failed to clone stimulus item") + stim_q.push_back(clone); + if (mem_model == null) `uvm_fatal("SCB", "mem_model handle not set") + mem_model.clear_mem(); + foreach (clone.preload_data[i]) + mem_model.load_word_g(clone.preload_bank[i], clone.preload_group[i], clone.preload_addr[i], + clone.preload_data[i]); + mem_model.arm(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + mxmm_cmd_item stim; + bb_blink_cmd_item want; + stim = current_stim("CMD"); + // Compare as plain BLink commands; the comparer rejects differing types. + want = bb_blink_cmd_item::type_id::create("want"); + want.copy(stim); + verification_pkg::evidence::checks++; + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) + cmd_count++; + endfunction + + function void write_read(bb_blink_read_item item); + mxmm_cmd_item stim; + int unsigned exp_bank; + int unsigned exp_group; + int unsigned exp_addr; + + stim = current_stim("READ"); + if (item.port != 0 && item.port != 1) + `uvm_fatal("READ", $sformatf("MxmmBall has two read ports, got port %0d", item.port)) + if (item.port == 0) begin + if (read_count[0] >= stim.read_addr0.size()) + `uvm_fatal("READ", $sformatf("extra read on port 0 at %0d", read_count[0])) + exp_bank = stim.op1_bank; + exp_group = stim.op1_group; + exp_addr = stim.read_addr0[read_count[0]]; + end else begin + if (read_count[1] >= stim.read_addr1.size()) + `uvm_fatal("READ", $sformatf("extra read on port 1 at %0d", read_count[1])) + exp_bank = stim.op2_bank; + exp_group = stim.op2_group; + exp_addr = stim.read_addr1[read_count[1]]; + end + verification_pkg::evidence::checks++; + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.bank_id !== exp_bank[BB_BLINK_BANK_ID_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d bank mismatch: got %0d exp %0d", item.port, item.bank_id, exp_bank)) + if (item.group_id !== exp_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d group mismatch: got %0d exp %0d", item.port, item.group_id, exp_group)) + if (item.addr !== exp_addr[BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d read %0d addr mismatch: got %0d exp %0d", + item.port, + read_count[item.port], + item.addr, + exp_addr + )) + read_count[item.port]++; + endfunction + + function void write_write(bb_blink_write_item item); + mxmm_cmd_item stim; + bit [127:0] bytes; + + stim = current_stim("WRITE"); + if (item.port != 0) + `uvm_fatal("WRITE", $sformatf("MxmmBall has one write port, got port %0d", item.port)) + if (write_count >= stim.write_addr.size()) + `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) + verification_pkg::evidence::checks++; + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.group_id !== stim.wr_group) + `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + )) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.addr !== stim.write_addr[write_count][BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("WRITE", $sformatf( + "write %0d addr mismatch: got %0d exp %0d", + write_count, + item.addr, + stim.write_addr[write_count] + )) + if (item.mask !== stim.write_mask[write_count]) + `uvm_fatal("WRITE", $sformatf( + "write %0d mask mismatch: got 0x%04h exp 0x%04h", + write_count, + item.mask, + stim.write_mask[write_count] + )) + for (int b = 0; b < 16; b++) bytes[b*8+:8] = {8{item.mask[b]}}; + if ((item.data & bytes) !== (stim.write_data[write_count] & bytes)) + `uvm_fatal("SCB", $sformatf( + "write %0d data mismatch at addr %0d: got 0x%032h exp 0x%032h", + write_count, + item.addr, + item.data, + stim.write_data[write_count] + )) + write_count++; + endfunction + + function void write_resp(bb_blink_resp_item item); + mxmm_cmd_item stim; + stim = current_stim("RESP"); + verification_pkg::evidence::checks++; + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (!complete_io()) + `uvm_fatal("RESP", $sformatf( + "response before all traffic: reads=%0d,%0d writes=%0d", + read_count[0], + read_count[1], + write_count + )) + resp_count++; + endfunction + + function mxmm_cmd_item current_stim(string tag); + if (stim_q.size() == 0) `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + return stim_q[0]; + endfunction + + function bit complete_io(); + mxmm_cmd_item stim = stim_q[0]; + return read_count[0] == stim.read_addr0.size() && read_count[1] == stim.read_addr1.size() && + write_count == stim.write_addr.size(); + endfunction + + function bit done(); + return stim_q.size() == 1 && cmd_count == 1 && resp_count == 1 && complete_io(); + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + read_count[0] = 0; + read_count[1] = 0; + write_count = 0; + resp_count = 0; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) + `uvm_fatal("SCB", $sformatf( + "incomplete: stim=%0d cmds=%0d reads=%0d,%0d writes=%0d resp=%0d", + stim_q.size(), + cmd_count, + read_count[0], + read_count[1], + write_count, + resp_count + )) + stim_q.delete(); + endfunction +endclass diff --git a/examples/prefab/balls/mxmm/verify/src/pkg/mxmm_pkg.sv b/examples/prefab/balls/mxmm/verify/src/pkg/mxmm_pkg.sv new file mode 100644 index 000000000..2f70bd272 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/pkg/mxmm_pkg.sv @@ -0,0 +1,13 @@ +package mxmm_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + + `include "common/mxmm_defs.svh" + `include "common/mxmm_items.svh" + `include "seq/mxmm_sequences.svh" + `include "cov/mxmm_cov.svh" + `include "env/mxmm_scoreboard.svh" + `include "env/mxmm_env.svh" + `include "tests/mxmm_test.svh" +endpackage diff --git a/examples/prefab/balls/mxmm/verify/src/seq/mxmm_sequences.svh b/examples/prefab/balls/mxmm/verify/src/seq/mxmm_sequences.svh new file mode 100644 index 000000000..b73f95b80 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/seq/mxmm_sequences.svh @@ -0,0 +1,17 @@ +class mxmm_step_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(mxmm_step_seq) + + int unsigned step; + + function new(string name = "mxmm_step_seq"); + super.new(name); + endfunction + + task body(); + mxmm_cmd_item item; + item = mxmm_cmd_item::type_id::create("item"); + start_item(item); + item.load_rust_step(step); + finish_item(item); + endtask +endclass diff --git a/examples/prefab/balls/mxmm/verify/src/tb_top.sv b/examples/prefab/balls/mxmm/verify/src/tb_top.sv new file mode 100644 index 000000000..5ee20f2e4 --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/tb_top.sv @@ -0,0 +1,89 @@ +module tb_top; + import uvm_pkg::*; + import mxmm_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + MxmmBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("mxmm_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/mxmm/verify/src/tests/mxmm_test.svh b/examples/prefab/balls/mxmm/verify/src/tests/mxmm_test.svh new file mode 100644 index 000000000..6af64328d --- /dev/null +++ b/examples/prefab/balls/mxmm/verify/src/tests/mxmm_test.svh @@ -0,0 +1,56 @@ +class mxmm_ball_test extends uvm_test; + `uvm_component_utils(mxmm_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + mxmm_env env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + env = mxmm_env::type_id::create("env", this); + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + for (int unsigned index = 0; index < mxmm_num_cases(); index++) run_case(index); + phase.drop_objection(this); + endtask + + // Steps of one case run back to back on the same reset. + task run_case(int unsigned index); + int unsigned steps; + apply_reset(); + steps = mxmm_case_load(index); + for (int unsigned step = 0; step < steps; step++) run_step(index, step); + endtask + + task run_step(int unsigned index, int unsigned step); + mxmm_step_seq seq; + int cycles = 0; + env.scb.reset_counters(); + seq = mxmm_step_seq::type_id::create("seq"); + seq.step = step; + seq.start(env.cmd_agent.seqr); + while (!env.scb.done()) begin + @(posedge vif.clock); + cycles++; + if (cycles > MXMM_TIMEOUT_CYCLES) + `uvm_fatal("TIMEOUT", $sformatf("MxmmBall case %0d step %0d timeout", index, step)) + end + `uvm_info("MXMM", $sformatf("MxmmBall case %0d step %0d passed in %0d cycles", index, step, + cycles), UVM_LOW) + endtask + + task apply_reset(); + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/components/balls/mxmm/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/mxmm/workloads/ctests/CMakeLists.txt similarity index 56% rename from examples/components/balls/mxmm/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/mxmm/workloads/ctests/CMakeLists.txt index f01d58c48..4c98c223a 100644 --- a/examples/components/balls/mxmm/workloads/ctests/CMakeLists.txt +++ b/examples/prefab/balls/mxmm/workloads/ctests/CMakeLists.txt @@ -2,11 +2,10 @@ add_buckyball_ctest(mxmm_fp32_test.c) add_buckyball_ctest(mxmm_mxfp8_test.c) add_buckyball_ctest(mxmm_bank_capacity_test.c) -file(READ "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h" MXMM_PARAMS) -string(REGEX MATCH "#define BANK_LINES ([0-9]+)" _ "${MXMM_PARAMS}") -set(MXMM_BANK_LINES ${CMAKE_MATCH_1}) -string(REGEX MATCH "#define BANK_WIDTH ([0-9]+)" _ "${MXMM_PARAMS}") -math(EXPR MXMM_BANK_BYTES "${MXMM_BANK_LINES} * ${CMAKE_MATCH_1} / 8") +math( + EXPR MXMM_BANK_BYTES + "${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_ENTRIES} * ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_WIDTH} / 8" +) if(MXMM_BANK_BYTES LESS 15840) message( FATAL_ERROR diff --git a/examples/components/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c similarity index 77% rename from examples/components/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c rename to examples/prefab/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c index 316cbc764..21eec3810 100644 --- a/examples/components/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c +++ b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_bank_capacity_test.c @@ -5,7 +5,10 @@ #include #include -enum { BANK_BYTES = BANK_LINES * BANK_WIDTH / 8, ELEMENTS = BANK_BYTES / 4 }; +enum { + BANK_BYTES = THREAD_MEM_BANK_ENTRIES * THREAD_MEM_BANK_WIDTH / 8, + ELEMENTS = BANK_BYTES / 4 +}; static float a[ELEMENTS] __attribute__((aligned(BANK_BYTES))); static float b[ELEMENTS] __attribute__((aligned(BANK_BYTES))); static uint32_t result[ELEMENTS] __attribute__((aligned(BANK_BYTES))); @@ -23,14 +26,14 @@ int main(void) { } for (int bank = 3; bank <= 5; ++bank) bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)b, 4, BANK_LINES, 1); - bb_mvin((uintptr_t)result, 5, BANK_LINES, 1); + bb_mvin((uintptr_t)a, 3, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)b, 4, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)result, 5, THREAD_MEM_BANK_ENTRIES, 1); if (fused) bb_mxmm_fma32(3, 4, 5, m, n, k, 1, 1, 0); else bb_mxmm_f32(3, 4, 5, m, n, k, 1, 1, 0); - bb_mvout((uintptr_t)result, 5, BANK_LINES, 1); + bb_mvout((uintptr_t)result, 5, THREAD_MEM_BANK_ENTRIES, 1); union { float value; uint32_t bits; diff --git a/examples/components/balls/mxmm/workloads/ctests/mxmm_fp32_test.c b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_fp32_test.c similarity index 100% rename from examples/components/balls/mxmm/workloads/ctests/mxmm_fp32_test.c rename to examples/prefab/balls/mxmm/workloads/ctests/mxmm_fp32_test.c diff --git a/examples/components/balls/mxmm/workloads/ctests/mxmm_mxfp8_test.c b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_mxfp8_test.c similarity index 100% rename from examples/components/balls/mxmm/workloads/ctests/mxmm_mxfp8_test.c rename to examples/prefab/balls/mxmm/workloads/ctests/mxmm_mxfp8_test.c diff --git a/examples/components/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c similarity index 78% rename from examples/components/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c rename to examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c index 6b4d737d1..497583569 100644 --- a/examples/components/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c +++ b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_chain_test.c @@ -5,7 +5,7 @@ #include #include -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; +enum { BYTES = THREAD_MEM_BANK_ENTRIES * THREAD_MEM_BANK_WIDTH / 8 }; static uint8_t a[2][BYTES] __attribute__((aligned(64))); static uint8_t packed[BYTES] __attribute__((aligned(64))); static uint8_t w[BYTES] __attribute__((aligned(64))); @@ -25,11 +25,11 @@ int main(void) { memset(expected, 0x5a, BYTES); for (int bank = 3; bank <= 8; ++bank) bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a[0], 3, BANK_LINES, 1); - bb_mvin((uintptr_t)a[1], 4, BANK_LINES, 1); - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 7, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 8, BANK_LINES, 1); + bb_mvin((uintptr_t)a[0], 3, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)a[1], 4, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)w, 6, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)expected, 7, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)actual, 8, THREAD_MEM_BANK_ENTRIES, 1); for (int mode = 0; mode < 2; ++mode) for (int part = 0; part < 2; ++part) { if (mode == 0) { @@ -37,11 +37,11 @@ int main(void) { packed[32] = 127; bb_mvin((uintptr_t)packed, 5, 3, 1); bb_mxmm_mxfp8(5, 6, 7, 1, 16, 32, part == 0, part == 1, 1); - bb_mvout((uintptr_t)expected, 7, BANK_LINES, 1); + bb_mvout((uintptr_t)expected, 7, THREAD_MEM_BANK_ENTRIES, 1); } else { bb_mxmm_mxfp8_window(3 + part, 6, 8, 1, 16, 32, part == 0, part == 1, 1, part ? 96 : 64, part * 32); - bb_mvout((uintptr_t)actual, 8, BANK_LINES, 1); + bb_mvout((uintptr_t)actual, 8, THREAD_MEM_BANK_ENTRIES, 1); } if (part == 0) for (int i = 0; i < BYTES; ++i) diff --git a/examples/components/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c similarity index 85% rename from examples/components/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c rename to examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c index 65f443ce9..00f36e1d0 100644 --- a/examples/components/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c +++ b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_rows_test.c @@ -5,7 +5,7 @@ #include #include -enum { BYTES = BANK_LINES * BANK_WIDTH / 8 }; +enum { BYTES = THREAD_MEM_BANK_ENTRIES * THREAD_MEM_BANK_WIDTH / 8 }; static uint8_t a[BYTES] __attribute__((aligned(64))); static uint8_t packed[BYTES] __attribute__((aligned(64))); static uint8_t w[BYTES] __attribute__((aligned(64))); @@ -32,9 +32,9 @@ int main(void) { memset(expected, 0x5a, sizeof(expected)); for (int bank = 3; bank <= 7; ++bank) bb_mem_alloc(bank, 1, 1); - bb_mvin((uintptr_t)a, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)expected, 5, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 7, BANK_LINES, 1); + bb_mvin((uintptr_t)a, 3, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)expected, 5, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)actual, 7, THREAD_MEM_BANK_ENTRIES, 1); /* Complete old-71 chain before new-75 chain: one chain slot per hart. */ for (int mode = 0; mode < 2; ++mode) { for (int k = start; k < full; k += 480) { @@ -55,15 +55,15 @@ int main(void) { a + m * full + row * (full / 32) + k / 32, count / 32); } } - bb_mvin((uintptr_t)w, 6, BANK_LINES, 1); + bb_mvin((uintptr_t)w, 6, THREAD_MEM_BANK_ENTRIES, 1); if (mode == 0) { - bb_mvin((uintptr_t)packed, 4, BANK_LINES, 1); + bb_mvin((uintptr_t)packed, 4, THREAD_MEM_BANK_ENTRIES, 1); bb_mxmm_mxfp8(4, 6, 5, m, n, count, k == start, k + count == full, 1); - bb_mvout((uintptr_t)expected, 5, BANK_LINES, 1); + bb_mvout((uintptr_t)expected, 5, THREAD_MEM_BANK_ENTRIES, 1); } else { bb_mxmm_mxfp8_window(3, 6, 7, m, n, count, k == start, k + count == full, 1, full, k); - bb_mvout((uintptr_t)actual, 7, BANK_LINES, 1); + bb_mvout((uintptr_t)actual, 7, THREAD_MEM_BANK_ENTRIES, 1); } if (k + count != full) for (int i = 0; i < BYTES; ++i) diff --git a/examples/components/balls/mxmm/workloads/ctests/mxmm_window_test.c b/examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_test.c similarity index 100% rename from examples/components/balls/mxmm/workloads/ctests/mxmm_window_test.c rename to examples/prefab/balls/mxmm/workloads/ctests/mxmm_window_test.c diff --git a/examples/components/balls/mxmm/workloads/isa/mxmm.h b/examples/prefab/balls/mxmm/workloads/isa/mxmm.h similarity index 87% rename from examples/components/balls/mxmm/workloads/isa/mxmm.h rename to examples/prefab/balls/mxmm/workloads/isa/mxmm.h index 7eafd0738..c676e83d8 100644 --- a/examples/components/balls/mxmm/workloads/isa/mxmm.h +++ b/examples/prefab/balls/mxmm/workloads/isa/mxmm.h @@ -14,7 +14,7 @@ static inline void bb_mxmm_mxfp8(uint32_t a, uint32_t b, uint32_t c, uint32_t m, uint32_t last, uint32_t base) { BUCKYBALL_INSTRUCTION_R_R( BB_BANK0(a) | BB_BANK1(b) | BB_BANK2(c) | BB_ITER(k), - bb_mxmm_cfg(m, n, first, last, base), BB_FUNC7(MXMM_MXFP8)); + bb_mxmm_cfg(m, n, first, last, base), FUNCT7(mxmm, MXMM_MXFP8)); } static inline void bb_mxmm_fma32(uint32_t a, uint32_t b, uint32_t c, uint32_t m, @@ -22,7 +22,7 @@ static inline void bb_mxmm_fma32(uint32_t a, uint32_t b, uint32_t c, uint32_t m, uint32_t last, uint32_t base) { BUCKYBALL_INSTRUCTION_R_R( BB_BANK0(a) | BB_BANK1(b) | BB_BANK2(c) | BB_ITER(k), - bb_mxmm_cfg(m, n, first, last, base), BB_FUNC7(MXMM_FMA32)); + bb_mxmm_cfg(m, n, first, last, base), FUNCT7(mxmm, MXMM_FMA32)); } static inline void bb_mxmm_f32(uint32_t a, uint32_t b, uint32_t c, uint32_t m, @@ -30,7 +30,7 @@ static inline void bb_mxmm_f32(uint32_t a, uint32_t b, uint32_t c, uint32_t m, uint32_t last, uint32_t base) { BUCKYBALL_INSTRUCTION_R_R( BB_BANK0(a) | BB_BANK1(b) | BB_BANK2(c) | BB_ITER(k), - bb_mxmm_cfg(m, n, first, last, base), BB_FUNC7(MXMM_F32)); + bb_mxmm_cfg(m, n, first, last, base), FUNCT7(mxmm, MXMM_F32)); } static inline void bb_mxmm_mxfp8_window(uint32_t a, uint32_t b, uint32_t c, @@ -42,6 +42,6 @@ static inline void bb_mxmm_mxfp8_window(uint32_t a, uint32_t b, uint32_t c, BB_ITER(k), bb_mxmm_cfg(m, n, first, last, base) | FIELD(fullK, 32, 47) | FIELD(startK, 48, 63), - BB_FUNC7(MXMM_MXFP8_WINDOW)); + FUNCT7(mxmm, MXMM_MXFP8_WINDOW)); } #endif diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir new file mode 100644 index 000000000..0ce7f4321 --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_f32.mlir @@ -0,0 +1,10 @@ +// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @single +// CHECK-DAG: arith.constant 4298115073 : i64 +// CHECK-DAG: arith.constant 50397200 : i64 +// The caller loads A into bank 1, B into bank 2 and C into bank 3. One step +// that both starts and ends its chain. +func.func @single() attributes {llvm.emit_c_interface} { + mxmm_ball.matmul_f32 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 16 : i64, cols = 16 : i64, k = 4 : i64, base = 0 : i64, first, last}> + return +} diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir new file mode 100644 index 000000000..2ce380e7f --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_fma32.mlir @@ -0,0 +1,10 @@ +// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @single +// CHECK-DAG: arith.constant 4298115073 : i64 +// CHECK-DAG: arith.constant 50397200 : i64 +// The caller loads A into bank 1, B into bank 2 and C into bank 3. One step +// that both starts and ends its chain. +func.func @single() attributes {llvm.emit_c_interface} { + mxmm_ball.matmul_fma32 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 16 : i64, cols = 16 : i64, k = 4 : i64, base = 0 : i64, first, last}> + return +} diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_chain.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_chain.mlir new file mode 100644 index 000000000..03519134f --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_chain.mlir @@ -0,0 +1,18 @@ +// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @first +// CHECK-DAG: arith.constant 34362886145 : i64 +// CHECK-DAG: arith.constant 83951632 : i64 +// CHECK-LABEL: func.func @last +// CHECK-DAG: arith.constant 34362886145 : i64 +// CHECK-DAG: arith.constant 100728848 : i64 +// The caller loads A into bank 1, B into bank 2 and C into bank 3. Two steps of +// one chain over the same A and B: the first only accumulates, the last +// accumulates and writes the 16x16 sum to C from row 1. +func.func @first() attributes {llvm.emit_c_interface} { + mxmm_ball.matmul_mxfp8 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 16 : i64, cols = 16 : i64, k = 32 : i64, base = 1 : i64, first}> + return +} +func.func @last() attributes {llvm.emit_c_interface} { + mxmm_ball.matmul_mxfp8 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 16 : i64, cols = 16 : i64, k = 32 : i64, base = 1 : i64, last}> + return +} diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_window.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_window.mlir new file mode 100644 index 000000000..f719efe3b --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/ball/mxmm_mxfp8_window.mlir @@ -0,0 +1,10 @@ +// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @window +// CHECK-DAG: arith.constant 34362886145 : i64 +// CHECK-DAG: arith.constant 9007474183045121 : i64 +// The caller loads A into bank 1, B into bank 2 and C into bank 3. One step +// reads columns 32..63 of a one-row A of 64 columns. +func.func @window() attributes {llvm.emit_c_interface} { + mxmm_ball.matmul_mxfp8_window <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 1 : i64, cols = 16 : i64, k = 32 : i64, full_k = 64 : i64, start_k = 32 : i64, base = 0 : i64, first, last}> + return +} diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir new file mode 100644 index 000000000..cb83c388e --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_f32.mlir @@ -0,0 +1,17 @@ +// One unfused FP32 step that both starts and ends its chain. C comes in holding +// a guard pattern and goes back with the 16x16 sum. +func.func @single(%a: memref<16x4xi32>, %b: memref<16x4xi32>, + %c: memref<65x4xi32>) attributes {llvm.emit_c_interface} { + %a_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %c_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %la = buckyball.bank_mvin %a, %a_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + %lb = buckyball.bank_mvin %b, %b_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + %lc = buckyball.bank_mvin %c, %c_bank <{depth = 65 : i64, stride = 1 : i64}> : memref<65x4xi32> + %sum = mxmm_ball.bank_matmul_f32 %la, %lb -> %lc <{rows = 16 : i64, cols = 16 : i64, k = 4 : i64, base = 0 : i64, first, last}> + %stored = buckyball.bank_mvout %c, %sum <{depth = 65 : i64, stride = 1 : i64}> : memref<65x4xi32> + buckyball.bank_release %la + buckyball.bank_release %lb + buckyball.bank_release %stored + return +} diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir new file mode 100644 index 000000000..c9198fca8 --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_fma32.mlir @@ -0,0 +1,17 @@ +// One fused FP32 step that both starts and ends its chain. C comes in holding +// a guard pattern and goes back with the 16x16 sum. +func.func @single(%a: memref<16x4xi32>, %b: memref<16x4xi32>, + %c: memref<65x4xi32>) attributes {llvm.emit_c_interface} { + %a_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %c_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %la = buckyball.bank_mvin %a, %a_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + %lb = buckyball.bank_mvin %b, %b_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + %lc = buckyball.bank_mvin %c, %c_bank <{depth = 65 : i64, stride = 1 : i64}> : memref<65x4xi32> + %sum = mxmm_ball.bank_matmul_fma32 %la, %lb -> %lc <{rows = 16 : i64, cols = 16 : i64, k = 4 : i64, base = 0 : i64, first, last}> + %stored = buckyball.bank_mvout %c, %sum <{depth = 65 : i64, stride = 1 : i64}> : memref<65x4xi32> + buckyball.bank_release %la + buckyball.bank_release %lb + buckyball.bank_release %stored + return +} diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_chain.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_chain.mlir new file mode 100644 index 000000000..188c029d0 --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_chain.mlir @@ -0,0 +1,19 @@ +// Two steps of one MXFP8 chain over the same A and B: the first only +// accumulates, the last accumulates and writes the 16x16 sum to C from row 1. +// C comes in holding a guard pattern and goes back with the sum. +func.func @chain(%a: memref<33x16xi8>, %b: memref<33x16xi8>, + %c: memref<65x16xi8>) attributes {llvm.emit_c_interface} { + %a_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %c_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %la = buckyball.bank_mvin %a, %a_bank <{depth = 33 : i64, stride = 1 : i64}> : memref<33x16xi8> + %lb = buckyball.bank_mvin %b, %b_bank <{depth = 33 : i64, stride = 1 : i64}> : memref<33x16xi8> + %lc = buckyball.bank_mvin %c, %c_bank <{depth = 65 : i64, stride = 1 : i64}> : memref<65x16xi8> + %partial = mxmm_ball.bank_matmul_mxfp8 %la, %lb -> %lc <{rows = 16 : i64, cols = 16 : i64, k = 32 : i64, base = 1 : i64, first}> + %sum = mxmm_ball.bank_matmul_mxfp8 %la, %lb -> %partial <{rows = 16 : i64, cols = 16 : i64, k = 32 : i64, base = 1 : i64, last}> + %stored = buckyball.bank_mvout %c, %sum <{depth = 65 : i64, stride = 1 : i64}> : memref<65x16xi8> + buckyball.bank_release %la + buckyball.bank_release %lb + buckyball.bank_release %stored + return +} diff --git a/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_window.mlir b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_window.mlir new file mode 100644 index 000000000..65ccf71a8 --- /dev/null +++ b/examples/prefab/balls/mxmm/workloads/mlir_tests/bank/mxmm_mxfp8_window.mlir @@ -0,0 +1,17 @@ +// One MXFP8 step on columns 32..63 of a one-row A of 64 columns. C comes in +// holding a guard pattern and goes back with the row of 16 sums. +func.func @window(%a: memref<5x16xi8>, %b: memref<33x16xi8>, + %c: memref<5x16xi8>) attributes {llvm.emit_c_interface} { + %a_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %b_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %c_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %la = buckyball.bank_mvin %a, %a_bank <{depth = 5 : i64, stride = 1 : i64}> : memref<5x16xi8> + %lb = buckyball.bank_mvin %b, %b_bank <{depth = 33 : i64, stride = 1 : i64}> : memref<33x16xi8> + %lc = buckyball.bank_mvin %c, %c_bank <{depth = 5 : i64, stride = 1 : i64}> : memref<5x16xi8> + %sum = mxmm_ball.bank_matmul_mxfp8_window %la, %lb -> %lc <{rows = 1 : i64, cols = 16 : i64, k = 32 : i64, full_k = 64 : i64, start_k = 32 : i64, base = 0 : i64, first, last}> + %stored = buckyball.bank_mvout %c, %sum <{depth = 5 : i64, stride = 1 : i64}> : memref<5x16xi8> + buckyball.bank_release %la + buckyball.bank_release %lb + buckyball.bank_release %stored + return +} diff --git a/examples/components/balls/mxquant/arch/Encode.scala b/examples/prefab/balls/mxquant/arch/Encode.scala similarity index 100% rename from examples/components/balls/mxquant/arch/Encode.scala rename to examples/prefab/balls/mxquant/arch/Encode.scala diff --git a/examples/components/balls/mxquant/arch/MxquantBall.scala b/examples/prefab/balls/mxquant/arch/MxquantBall.scala similarity index 100% rename from examples/components/balls/mxquant/arch/MxquantBall.scala rename to examples/prefab/balls/mxquant/arch/MxquantBall.scala diff --git a/examples/components/balls/mxquant/configs/default.toml b/examples/prefab/balls/mxquant/configs/default.toml similarity index 100% rename from examples/components/balls/mxquant/configs/default.toml rename to examples/prefab/balls/mxquant/configs/default.toml diff --git a/examples/components/balls/mxquant/configs/mxquant_params.proto b/examples/prefab/balls/mxquant/configs/mxquant_params.proto similarity index 100% rename from examples/components/balls/mxquant/configs/mxquant_params.proto rename to examples/prefab/balls/mxquant/configs/mxquant_params.proto diff --git a/examples/components/balls/mxquant/emu/src/dispatch.rs b/examples/prefab/balls/mxquant/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/mxquant/emu/src/dispatch.rs rename to examples/prefab/balls/mxquant/emu/src/dispatch.rs diff --git a/examples/components/balls/mxquant/emu/src/lib.rs b/examples/prefab/balls/mxquant/emu/src/lib.rs similarity index 100% rename from examples/components/balls/mxquant/emu/src/lib.rs rename to examples/prefab/balls/mxquant/emu/src/lib.rs diff --git a/examples/components/balls/mxquant/emu/src/quant.rs b/examples/prefab/balls/mxquant/emu/src/quant.rs similarity index 94% rename from examples/components/balls/mxquant/emu/src/quant.rs rename to examples/prefab/balls/mxquant/emu/src/quant.rs index eb638d8d1..3a46a9ad1 100644 --- a/examples/components/balls/mxquant/emu/src/quant.rs +++ b/examples/prefab/balls/mxquant/emu/src/quant.rs @@ -1,4 +1,4 @@ -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::ExecContext; use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; @@ -32,6 +32,7 @@ fn encode(bits: u32, block_exponent: i32) -> u8 { } pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "mxquant: reads no old destination; rs1[29] must be clear"); let input = rs1_b0(xs1); let output = rs1_b2(xs1); let count = rs1_iter(xs1) as usize; diff --git a/examples/components/balls/mxquant/stack/kernels/CMakeLists.txt b/examples/prefab/balls/mxquant/stack/kernels/CMakeLists.txt similarity index 88% rename from examples/components/balls/mxquant/stack/kernels/CMakeLists.txt rename to examples/prefab/balls/mxquant/stack/kernels/CMakeLists.txt index aa8104386..1d7c5b1e7 100644 --- a/examples/components/balls/mxquant/stack/kernels/CMakeLists.txt +++ b/examples/prefab/balls/mxquant/stack/kernels/CMakeLists.txt @@ -6,5 +6,5 @@ set(KERNEL_SOURCES ${KERNEL_SOURCES} "${CMAKE_CURRENT_SOURCE_DIR}/dispatch.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/pack.cpp" PARENT_SCOPE) set(KERNEL_INCLUDES ${KERNEL_INCLUDES} - "${DESIGN_REPO}/examples/components/balls/mxquant/workloads" PARENT_SCOPE) + "${DESIGN_REPO}/examples/prefab/balls/mxquant/workloads" PARENT_SCOPE) set(KERNEL_EXPORTS ${KERNEL_EXPORTS} mxfp8_quant _mlir_ciface_mxfp8_quant PARENT_SCOPE) diff --git a/examples/components/balls/mxquant/stack/kernels/dispatch.cpp b/examples/prefab/balls/mxquant/stack/kernels/dispatch.cpp similarity index 100% rename from examples/components/balls/mxquant/stack/kernels/dispatch.cpp rename to examples/prefab/balls/mxquant/stack/kernels/dispatch.cpp diff --git a/examples/components/balls/mxquant/stack/kernels/pack.cpp b/examples/prefab/balls/mxquant/stack/kernels/pack.cpp similarity index 100% rename from examples/components/balls/mxquant/stack/kernels/pack.cpp rename to examples/prefab/balls/mxquant/stack/kernels/pack.cpp diff --git a/examples/prefab/balls/mxquant/stack/src/mxquantOps.cpp b/examples/prefab/balls/mxquant/stack/src/mxquantOps.cpp new file mode 100644 index 000000000..521fb1558 --- /dev/null +++ b/examples/prefab/balls/mxquant/stack/src/mxquantOps.cpp @@ -0,0 +1,69 @@ +//===- mxquantOps.cpp - MxquantBall operations ----------------------------===// +// +// The dialect of MxquantBall, its virtual operation building its physical one +// and the instruction of the physical operation. +// +//===----------------------------------------------------------------------===// + +#include "mxquantOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::mxquant_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; + +#include "mxquantDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "mxquantOps.cpp.inc" + +void MxquantBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "mxquantOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankQuantMxfp8Op::getSourceBanks() { + return {getInput()}; +} +Value BankQuantMxfp8Op::getDestinationBank() { return getOutput(); } +bool BankQuantMxfp8Op::readsDestination() { return false; } +Operation *BankQuantMxfp8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return QuantMxfp8Op::create(builder, getLoc(), sources[0], destination, + getCount()); +} + +SmallVector QuantMxfp8Op::getSourceBanks() { + return {int64_t(getInput())}; +} +int64_t QuantMxfp8Op::getDestinationBank() { return getOutput(); } +LogicalResult QuantMxfp8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t count = getCount(), input = getInput(), output = getOutput(); + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || count <= 0 || count % 32 || + count > target.bankDepth * 4) + return emitOpError("quant_mxfp8 count exceeds the bank"); + if (input == output) + return emitOpError("quant_mxfp8 requires distinct input and output banks"); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("MXQUANT"), + constantI64(builder, loc, bankFields({input}, output, false, count)), + constantI64(builder, loc, 0)); + return success(); +} diff --git a/examples/prefab/balls/mxquant/stack/src/mxquantOps.h b/examples/prefab/balls/mxquant/stack/src/mxquantOps.h new file mode 100644 index 000000000..d03f09cd4 --- /dev/null +++ b/examples/prefab/balls/mxquant/stack/src/mxquantOps.h @@ -0,0 +1,20 @@ +//===- mxquantOps.h - MxquantBall operations ------------------------------===// + +#ifndef MXQUANT_OPS_H +#define MXQUANT_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "mxquantDialect.h.inc" + +#define GET_OP_CLASSES +#include "mxquantOps.h.inc" + +#endif // MXQUANT_OPS_H diff --git a/examples/prefab/balls/mxquant/stack/src/mxquantOps.td b/examples/prefab/balls/mxquant/stack/src/mxquantOps.td new file mode 100644 index 000000000..44e9dfe7a --- /dev/null +++ b/examples/prefab/balls/mxquant/stack/src/mxquantOps.td @@ -0,0 +1,47 @@ +//===- mxquantOps.td - MxquantBall operations --------------*- tablegen -*-===// +// +// quant_mxfp8 (input) -> output: quantizes `count` FP32 (a multiple of 32) to +// block32 MXFP8, one MXQUANT instruction. Input holds the `count` finite FP32 +// from byte 0; output receives the `count` E4M3 codes from byte 0, then one +// E8M0 scale per block of 32 from byte `count`. The scale is the block +// exponent: the largest magnitude's exponent minus 8, at least -127, biased by +// 127 (0 for an all-zero block); a code rounds to nearest even, saturates at +// magnitude code 126 and keeps the sign of zero. Nothing else of the output +// is written. +// Input and output are distinct banks of one 128-bit wide bank each. +// +//===----------------------------------------------------------------------===// + +#ifndef MXQUANT_OPS +#define MXQUANT_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def MxquantBall_Dialect : Dialect { + let name = "mxquant_ball"; + let cppNamespace = "::buddy::mxquant_ball"; + let summary = "The operations of MxquantBall"; +} + +class MxquantBall_Op traits = []> + : Op])>; + +def BankQuantMxfp8Op : MxquantBall_Op<"bank_quant_mxfp8", + [DeclareOpInterfaceMethods]> { + let summary = "quant_mxfp8 on virtual banks"; + let arguments = (ins I64:$input, I64:$output, I64Attr:$count); + let results = (outs I64:$outputOut); + let assemblyFormat = "$input `->` $output prop-dict attr-dict"; +} + +def QuantMxfp8Op : MxquantBall_Op<"quant_mxfp8", + [DeclareOpInterfaceMethods]> { + let summary = "quant_mxfp8 on physical banks"; + let arguments = (ins I64Attr:$input, I64Attr:$output, I64Attr:$count); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // MXQUANT_OPS diff --git a/examples/prefab/balls/mxquant/verify/casegen/Cargo.lock b/examples/prefab/balls/mxquant/verify/casegen/Cargo.lock new file mode 100644 index 000000000..5923911d5 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "mxquant-casegen" +version = "0.1.0" diff --git a/examples/prefab/balls/mxquant/verify/casegen/Cargo.toml b/examples/prefab/balls/mxquant/verify/casegen/Cargo.toml new file mode 100644 index 000000000..369a9d49f --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mxquant-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["rlib", "cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/prefab/balls/mxquant/verify/casegen/src/casegen.rs b/examples/prefab/balls/mxquant/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..fcad20c62 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/casegen/src/casegen.rs @@ -0,0 +1,265 @@ +// MXQUANT reference model, the contract shared with the BEMU model +// (emu/src/quant.rs) and MxquantBall: +// +// op1 bank: count FP32 activations, four per row; block g of 32 is rows +// 8g..8g+7. Activations must be finite. +// wr bank: E4M3 code of activation i in byte i (block g in rows 2g, 2g+1), +// the E8M0 scale of block g in byte count + g. +// The block exponent is max(exponent(max |x|) - 8, -127) (0 for an all-zero +// block); codes round to nearest even and clip to 126 (448). +// rs2 and op2 bank are zero; count is a positive multiple of 32 with +// count <= 4 * bank_entries and count + count / 32 <= 16 * bank_entries. + +pub const ROW_BYTES: usize = 16; +pub const BLOCK: usize = 32; +pub const NUM_CASES: u32 = 6; + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MxquantCmd { + pub rob_id: u32, + pub iter: u32, + pub rs2_lo: u32, + pub rs2_hi: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, + pub op1_col: u32, + pub op2_col: u32, + pub wr_col: u32, +} + +#[derive(Clone, Debug)] +pub struct Preload { + pub bank: u32, + pub group: u32, + pub addr: u32, + pub data: u128, +} + +#[derive(Clone, Debug)] +pub struct Write { + pub addr: u32, + pub mask: u16, + pub data: u128, +} + +#[derive(Clone, Debug)] +pub struct Step { + pub cmd: MxquantCmd, + pub preload: Vec, + pub reads: Vec, + pub writes: Vec, +} + +#[derive(Clone, Debug)] +pub struct Case { + pub steps: Vec, +} + +struct Rng(u32); + +impl Rng { + fn next(&mut self) -> u32 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 17; + self.0 ^= self.0 << 5; + self.0 + } +} + +fn word(bytes: &[u8]) -> u128 { + let mut out = 0u128; + for (lane, byte) in bytes.iter().enumerate() { + out |= u128::from(*byte) << (lane * 8); + } + out +} + +fn rounded_shift(value: u32, shift: u32) -> u32 { + if shift > 31 { + return 0; + } + let half = 1 << (shift - 1); + (value + half - 1 + ((value >> shift) & 1)) >> shift +} + +fn encode(bits: u32, block_exponent: i32) -> u8 { + let sign = ((bits >> 24) & 128) as u8; + if bits & 0x7fff_ffff == 0 { + return sign; + } + let mut exponent = ((bits >> 23) & 255) as i32; + let mut fraction = bits & 0x7f_ffff; + if exponent == 0 { + let shift = fraction.leading_zeros() - 8; + fraction = (fraction << shift) & 0x7f_ffff; + exponent = 1 - shift as i32; + } + exponent -= block_exponent + 120; + let code = if exponent <= 0 { + rounded_shift(fraction | 0x80_0000, (21 - exponent) as u32) + } else { + exponent as u32 * 8 + rounded_shift(fraction, 20) + }; + sign | code.min(126) as u8 +} + +fn step(bank_entries: u32, rob_id: u32, banks: [u32; 2], groups: [u32; 2], input: &[u32]) -> Step { + let count = input.len(); + let bytes = bank_entries as usize * ROW_BYTES; + if count == 0 || count % BLOCK != 0 || count > bytes / 4 || count + count / BLOCK > bytes { + panic!("mxquant: count must be a positive multiple of 32 within the bank"); + } + if banks[0] == banks[1] { + panic!("mxquant: input and output banks must differ"); + } + if input.iter().any(|x| x & 0x7fff_ffff >= 0x7f80_0000) { + panic!("mxquant: non-finite activation"); + } + let mut preload = Vec::new(); + let mut reads = Vec::new(); + for (row, values) in input.chunks(4).enumerate() { + let bytes: Vec = values.iter().flat_map(|x| x.to_le_bytes()).collect(); + preload.push(Preload { bank: banks[0], group: groups[0], addr: row as u32, data: word(&bytes) }); + reads.push(row as u32); + } + let mut writes = Vec::new(); + for (block, values) in input.chunks(BLOCK).enumerate() { + let maximum = values.iter().map(|x| x & 0x7fff_ffff).max().unwrap(); + let exponent = if maximum == 0 { 0 } else { ((maximum >> 23) as i32 - 135).max(-127) }; + let codes: Vec = values.iter().map(|x| encode(*x, exponent)).collect(); + writes.push(Write { addr: (block * 2) as u32, mask: 0xffff, data: word(&codes[..ROW_BYTES]) }); + writes.push(Write { addr: (block * 2 + 1) as u32, mask: 0xffff, data: word(&codes[ROW_BYTES..]) }); + let byte = count + block; + let lane = byte % ROW_BYTES; + writes.push(Write { + addr: (byte / ROW_BYTES) as u32, + mask: 1 << lane, + data: u128::from((exponent + 127) as u8) << (lane * 8), + }); + } + Step { + cmd: MxquantCmd { + rob_id, + iter: count as u32, + rs2_lo: 0, + rs2_hi: 0, + op1_bank: banks[0], + op2_bank: 0, + wr_bank: banks[1], + op1_group: groups[0], + op2_group: 0, + wr_group: groups[1], + op1_col: 1, + op2_col: 0, + wr_col: 1, + }, + preload, + reads, + writes, + } +} + +// Activations of mxquant_values_test: test 0..3 are signed zeros, edges, +// rounding ties and subnormals; later tests sweep the exponent range. +fn values_test(rng: &mut u32, test: u32) -> Vec { + let edges = [0, 0x8000_0000, 1, 0x8000_0001, 0x007f_ffff, 0x0080_0000, 0x3f88_0000, 0x3f98_0000, 0x43e0_0000, 0x7f7f_ffff, 0xff7f_ffff]; + let subnormals = [0, 1, 0x1000, 0x2000, 0x3000, 0x7f_ffff, 0x8000_0001, 0x807f_ffff]; + (0..BLOCK as u32) + .map(|i| { + *rng ^= *rng << 13; + *rng ^= *rng >> 17; + *rng ^= *rng << 5; + let random = *rng; + match test { + 0 => if i & 1 != 0 { 0x8000_0000 } else { 0 }, + 1 => edges[(i % 11) as usize], + 2 => if i == 0 { 0x43e0_0000 } else if i & 1 != 0 { 0x3f88_0000 } else { 0x3f98_0000 }, + 3 => subnormals[(i % 8) as usize], + _ => (random & 0x807f_ffff) | ((((test - 4) * 4 + (random >> 23 & 3)) % 255) << 23), + } + }) + .collect() +} + +pub fn gen_case(index: u32, bank_entries: u32) -> Case { + let mut rng = Rng(0x2545_f491 ^ index.wrapping_mul(0x9e37_79b9)); + let steps = match index { + // Signed zeros, edges, rounding ties and subnormals, one block each. + 0 => { + let mut state = 0x6d2b_79f5u32; + let input: Vec = (0..4).flat_map(|test| values_test(&mut state, test)).collect(); + vec![step(bank_entries, 1, [3, 4], [0, 0], &input)] + } + // The exponent sweep of mxquant_values_test: 64 blocks, the full input + // capacity of a 512-row bank. + 1 => { + let mut state = 0x6d2b_79f5u32; + let input: Vec = (0..68).flat_map(|test| values_test(&mut state, test)).skip(4 * BLOCK).collect(); + vec![step(bank_entries, 2, [3, 4], [1, 2], &input)] + } + // One block, random magnitudes around one, highest group bases. + 2 => { + let input: Vec = (0..BLOCK).map(|_| (rng.next() & 0x807f_ffff) | ((120 + rng.next() % 16) << 23)).collect(); + vec![step(bank_entries, 15, [1022, 1023], [31, 31], &input)] + } + // Random exponents over the whole finite range, 3 / 4 of the input capacity. + 3 => { + let count = (bank_entries * 3) as usize; + let input: Vec = (0..count).map(|_| (rng.next() & 0x807f_ffff) | ((rng.next() % 255) << 23)).collect(); + vec![step(bank_entries, 7, [0, 1], [5, 6], &input)] + } + // Blocks whose maximum is subnormal or tiny (block exponent clamps to -127). + 4 => { + let input: Vec = (0..2 * BLOCK) + .map(|i| if i < BLOCK { rng.next() & 0x803f_ffff } else { (rng.next() & 0x807f_ffff) | ((rng.next() % 3) << 23) }) + .collect(); + vec![step(bank_entries, 4, [9, 10], [3, 4], &input)] + } + // Back-to-back commands without reset. + 5 => (0..3u32) + .map(|i| { + let count = BLOCK * (1 + (rng.next() % 5) as usize); + let input: Vec = (0..count).map(|_| (rng.next() & 0x807f_ffff) | ((100 + rng.next() % 60) << 23)).collect(); + step(bank_entries, 8 + i, [11 + i, 20 + i], [i, i + 7], &input) + }) + .collect(), + _ => panic!("mxquant: unsupported case {index}"), + }; + Case { steps } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cases_build() { + for index in 0..NUM_CASES { + gen_case(index, 512); + } + } + + // Golden bytes checked by mxquant_values_test. + #[test] + fn values_test_golden() { + let mut state = 0x6d2b_79f5u32; + let block = |state: &mut u32, test| { + let input = values_test(state, test); + let maximum = input.iter().map(|x| x & 0x7fff_ffff).max().unwrap(); + let exponent = if maximum == 0 { 0 } else { ((maximum >> 23) as i32 - 135).max(-127) }; + let codes: Vec = input.iter().map(|x| encode(*x, exponent)).collect(); + (codes, (exponent + 127) as u8) + }; + let (codes, scale) = block(&mut state, 0); + assert_eq!((codes[0], codes[1], scale), (0, 128, 127)); + let (codes, scale) = block(&mut state, 1); + assert_eq!((codes[9], codes[10], scale), (126, 254, 246)); + let (codes, _) = block(&mut state, 2); + assert_eq!((codes[0], codes[1], codes[2]), (126, 56, 58)); + } +} diff --git a/examples/prefab/balls/mxquant/verify/casegen/src/config.rs b/examples/prefab/balls/mxquant/verify/casegen/src/config.rs new file mode 100644 index 000000000..241755f0f --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/casegen/src/config.rs @@ -0,0 +1,9 @@ +pub fn bank_entries() -> u32 { + let path = std::env::var("BB_VERIFY_CONFIG").expect("BB_VERIFY_CONFIG is required"); + let text = std::fs::read_to_string(path).expect("failed to read BB_VERIFY_CONFIG"); + text.lines() + .find_map(|line| line.strip_prefix("bank_entries=")) + .unwrap_or_else(|| panic!("bank_entries missing from BB_VERIFY_CONFIG")) + .parse() + .expect("invalid bank_entries in BB_VERIFY_CONFIG") +} diff --git a/examples/prefab/balls/mxquant/verify/casegen/src/lib.rs b/examples/prefab/balls/mxquant/verify/casegen/src/lib.rs new file mode 100644 index 000000000..5659150db --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/casegen/src/lib.rs @@ -0,0 +1,113 @@ +mod casegen; +mod config; + +use std::cell::RefCell; + +use casegen::{Case, MxquantCmd, Step}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn mxquant_num_cases() -> u32 { + casegen::NUM_CASES +} + +#[no_mangle] +pub extern "C" fn mxquant_case_load(index: u32) -> u32 { + let case = casegen::gen_case(index, config::bank_entries()); + let steps = case.steps.len() as u32; + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + steps +} + +fn current_step R, R>(step: u32, f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => match case.steps.get(step as usize) { + Some(s) => f(s), + None => panic!("mxquant_case: step {step} out of range {}", case.steps.len()), + }, + None => panic!("mxquant_case: no case loaded; call mxquant_case_load first"), + }) +} + +fn idx(index: u32, limit: usize, what: &str) -> usize { + if index as usize >= limit { + panic!("mxquant_case_{what}: index {index} out of range {limit}"); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn mxquant_case_cmd(step: u32, out_ptr: *mut MxquantCmd) { + if out_ptr.is_null() { + panic!("mxquant_case_cmd: null out_ptr"); + } + current_step(step, |s| unsafe { *out_ptr = s.cmd }); +} + +#[no_mangle] +pub extern "C" fn mxquant_case_preload_count(step: u32) -> u32 { + current_step(step, |s| s.preload.len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_preload_bank(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].bank) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_preload_group(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].group) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_preload_addr(step: u32, i: u32) -> u32 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].addr) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_preload_lo(step: u32, i: u32) -> u64 { + current_step(step, |s| s.preload[idx(i, s.preload.len(), "preload")].data as u64) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_preload_hi(step: u32, i: u32) -> u64 { + current_step(step, |s| (s.preload[idx(i, s.preload.len(), "preload")].data >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_read_count(step: u32) -> u32 { + current_step(step, |s| s.reads.len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_read_addr(step: u32, i: u32) -> u32 { + current_step(step, |s| s.reads[idx(i, s.reads.len(), "read")]) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_write_count(step: u32) -> u32 { + current_step(step, |s| s.writes.len() as u32) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_write_addr(step: u32, i: u32) -> u32 { + current_step(step, |s| s.writes[idx(i, s.writes.len(), "write")].addr) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_write_mask(step: u32, i: u32) -> u32 { + current_step(step, |s| u32::from(s.writes[idx(i, s.writes.len(), "write")].mask)) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_write_lo(step: u32, i: u32) -> u64 { + current_step(step, |s| s.writes[idx(i, s.writes.len(), "write")].data as u64) +} + +#[no_mangle] +pub extern "C" fn mxquant_case_write_hi(step: u32, i: u32) -> u64 { + current_step(step, |s| (s.writes[idx(i, s.writes.len(), "write")].data >> 64) as u64) +} diff --git a/examples/prefab/balls/mxquant/verify/filelists/mxquant_ball.f b/examples/prefab/balls/mxquant/verify/filelists/mxquant_ball.f new file mode 100644 index 000000000..a27788ab5 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/filelists/mxquant_ball.f @@ -0,0 +1,25 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+src ++incdir+src/pkg ++incdir+src/common ++incdir+src/seq ++incdir+src/cov ++incdir+src/env ++incdir+src/tests ++incdir+@RTL@ + +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv + +@RTL@/Encode.sv +@RTL@/MxquantBall.sv + +src/pkg/mxquant_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/mxquant/verify/src/common/mxquant_defs.svh b/examples/prefab/balls/mxquant/verify/src/common/mxquant_defs.svh new file mode 100644 index 000000000..54c498bbe --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/common/mxquant_defs.svh @@ -0,0 +1,39 @@ +typedef struct { + int unsigned rob_id; + int unsigned iter; + int unsigned rs2_lo; + int unsigned rs2_hi; + int unsigned op1_bank; + int unsigned op2_bank; + int unsigned wr_bank; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; + int unsigned op1_col; + int unsigned op2_col; + int unsigned wr_col; +} mxquant_cmd_dpi_t; + +import "DPI-C" function int unsigned mxquant_num_cases(); +import "DPI-C" function int unsigned mxquant_case_load(input int unsigned index); +import "DPI-C" function void mxquant_case_cmd(input int unsigned step, output mxquant_cmd_dpi_t cmd); +import "DPI-C" function int unsigned mxquant_case_preload_count(input int unsigned step); +import "DPI-C" function int unsigned mxquant_case_preload_bank(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxquant_case_preload_group(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxquant_case_preload_addr(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxquant_case_preload_lo(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxquant_case_preload_hi(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxquant_case_read_count(input int unsigned step); +import "DPI-C" function int unsigned mxquant_case_read_addr(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxquant_case_write_count(input int unsigned step); +import "DPI-C" function int unsigned mxquant_case_write_addr(input int unsigned step, input int unsigned i); +import "DPI-C" function int unsigned mxquant_case_write_mask(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxquant_case_write_lo(input int unsigned step, input int unsigned i); +import "DPI-C" function longint unsigned mxquant_case_write_hi(input int unsigned step, input int unsigned i); + +`ifndef MXQUANT_FUNCT7 +`error "MXQUANT_FUNCT7 must be provided by the selected ballISA" +`endif + +localparam int MXQUANT_THREAD_FUNCT7 = `MXQUANT_FUNCT7; +localparam int MXQUANT_TIMEOUT_CYCLES = 100000; diff --git a/examples/prefab/balls/mxquant/verify/src/common/mxquant_items.svh b/examples/prefab/balls/mxquant/verify/src/common/mxquant_items.svh new file mode 100644 index 000000000..f70b5a4fc --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/common/mxquant_items.svh @@ -0,0 +1,80 @@ +class mxquant_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(mxquant_cmd_item) + + int unsigned preload_bank[]; + int unsigned preload_group[]; + int unsigned preload_addr[]; + bit [127:0] preload_data[]; + int unsigned read_addr[]; + int unsigned write_addr[]; + bit [15:0] write_mask[]; + bit [127:0] write_data[]; + + function new(string name = "mxquant_cmd_item"); + super.new(name); + endfunction + + function void load_rust_step(int unsigned step); + mxquant_cmd_dpi_t cmd; + int unsigned n; + + mxquant_case_cmd(step, cmd); + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = MXQUANT_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = cmd.op1_col; + op2_col = cmd.op2_col; + wr_col = cmd.wr_col; + wr_read_bank = '0; + wr_arch_bank = cmd.wr_bank; + if (rob_id != cmd.rob_id || op1_bank != cmd.op1_bank || op2_bank != cmd.op2_bank || + wr_bank != cmd.wr_bank || op1_group != cmd.op1_group || op2_group != cmd.op2_group || + wr_group != cmd.wr_group) + `uvm_fatal("CASE", "command field exceeds the configured BLink width") + + n = mxquant_case_preload_count(step); + preload_bank = new[n]; + preload_group = new[n]; + preload_addr = new[n]; + preload_data = new[n]; + for (int unsigned i = 0; i < n; i++) begin + preload_bank[i] = mxquant_case_preload_bank(step, i); + preload_group[i] = mxquant_case_preload_group(step, i); + preload_addr[i] = mxquant_case_preload_addr(step, i); + preload_data[i] = {mxquant_case_preload_hi(step, i), mxquant_case_preload_lo(step, i)}; + end + n = mxquant_case_read_count(step); + read_addr = new[n]; + for (int unsigned i = 0; i < n; i++) read_addr[i] = mxquant_case_read_addr(step, i); + n = mxquant_case_write_count(step); + write_addr = new[n]; + write_mask = new[n]; + write_data = new[n]; + for (int unsigned i = 0; i < n; i++) begin + write_addr[i] = mxquant_case_write_addr(step, i); + write_mask[i] = mxquant_case_write_mask(step, i); + write_data[i] = {mxquant_case_write_hi(step, i), mxquant_case_write_lo(step, i)}; + end + endfunction + + function void do_copy(uvm_object rhs); + mxquant_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not mxquant_cmd_item") + preload_bank = rhs_.preload_bank; + preload_group = rhs_.preload_group; + preload_addr = rhs_.preload_addr; + preload_data = rhs_.preload_data; + read_addr = rhs_.read_addr; + write_addr = rhs_.write_addr; + write_mask = rhs_.write_mask; + write_data = rhs_.write_data; + endfunction +endclass diff --git a/examples/prefab/balls/mxquant/verify/src/cov/mxquant_cov.svh b/examples/prefab/balls/mxquant/verify/src/cov/mxquant_cov.svh new file mode 100644 index 000000000..498310be0 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/cov/mxquant_cov.svh @@ -0,0 +1,76 @@ +class mxquant_cov extends uvm_component; + `uvm_component_utils(mxquant_cov) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, mxquant_cov) stim_imp; + + int unsigned cur_count; + int unsigned cur_scale; + bit [6:0] cur_magnitude; + bit cur_sign; + int unsigned cmd_count; + + covergroup cmd_cg; + coverpoint cur_count { + bins one_block = {32}; + bins blocks = {[33 : 1024]}; + bins many = {[1025 : 2047]}; + bins capacity = {2048}; + } + endgroup + + covergroup scale_cg; + coverpoint cur_scale { + bins clamped = {0}; + bins low = {[1 : 118]}; + bins unit = {[119 : 135]}; + bins high = {[136 : 246]}; + } + endgroup + + covergroup code_cg; + coverpoint cur_magnitude { + bins zero = {0}; bins subnormal = {[1 : 7]}; bins normal = {[8 : 125]}; bins clipped = {126}; + } + coverpoint cur_sign; + endgroup + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_cg = new(); + scale_cg = new(); + code_cg = new(); + endfunction + + function void write_stim(bb_blink_cmd_item item); + mxquant_cmd_item stim; + if (!$cast(stim, item)) `uvm_fatal("COV", "stim item is not mxquant_cmd_item") + cur_count = stim.iter; + cmd_cg.sample(); + foreach (stim.write_data[i]) begin + for (int b = 0; b < 16; b++) begin + if (!stim.write_mask[i][b]) continue; + if (stim.write_mask[i] == 16'hFFFF) begin + {cur_sign, cur_magnitude} = stim.write_data[i][b*8+:8]; + code_cg.sample(); + end else begin + cur_scale = stim.write_data[i][b*8+:8]; + scale_cg.sample(); + end + end + end + cmd_count++; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("mxquant count coverage %0.2f%%", cmd_cg.get_coverage())) + if (scale_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("mxquant scale coverage %0.2f%%", scale_cg.get_coverage())) + if (code_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("mxquant code coverage %0.2f%%", code_cg.get_coverage())) + endfunction +endclass diff --git a/examples/prefab/balls/mxquant/verify/src/env/mxquant_env.svh b/examples/prefab/balls/mxquant/verify/src/env/mxquant_env.svh new file mode 100644 index 000000000..5cbc20981 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/env/mxquant_env.svh @@ -0,0 +1,27 @@ +class mxquant_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(mxquant_env) + + mxquant_scoreboard scb; + mxquant_cov icov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scb = mxquant_scoreboard::type_id::create("scb", this); + icov = mxquant_cov::type_id::create("icov", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + scb.mem_model = mem_model; + cmd_agent.stim_ap.connect(scb.stim_imp); + cmd_agent.stim_ap.connect(icov.stim_imp); + cmd_agent.cmd_ap.connect(scb.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction +endclass diff --git a/examples/prefab/balls/mxquant/verify/src/env/mxquant_scoreboard.svh b/examples/prefab/balls/mxquant/verify/src/env/mxquant_scoreboard.svh new file mode 100644 index 000000000..cdd3fbb01 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/env/mxquant_scoreboard.svh @@ -0,0 +1,173 @@ +class mxquant_scoreboard extends uvm_scoreboard; + `uvm_component_utils(mxquant_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, mxquant_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, mxquant_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, mxquant_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, mxquant_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, mxquant_scoreboard) resp_imp; + + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + + mxquant_cmd_item stim_q[$]; + int unsigned cmd_count; + int unsigned read_count; + int unsigned write_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + mxquant_cmd_item got; + mxquant_cmd_item clone; + + if (stim_q.size() != 0) `uvm_fatal("SCB", "single outstanding command supported") + if (!$cast(got, item)) `uvm_fatal("SCB", "stim item is not mxquant_cmd_item") + if (!$cast(clone, got.clone())) `uvm_fatal("SCB", "failed to clone stimulus item") + stim_q.push_back(clone); + if (mem_model == null) `uvm_fatal("SCB", "mem_model handle not set") + mem_model.clear_mem(); + foreach (clone.preload_data[i]) + mem_model.load_word_g(clone.preload_bank[i], clone.preload_group[i], clone.preload_addr[i], + clone.preload_data[i]); + mem_model.arm(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + mxquant_cmd_item stim; + bb_blink_cmd_item want; + stim = current_stim("CMD"); + // Compare as plain BLink commands; the comparer rejects differing types. + want = bb_blink_cmd_item::type_id::create("want"); + want.copy(stim); + verification_pkg::evidence::checks++; + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) + cmd_count++; + endfunction + + function void write_read(bb_blink_read_item item); + mxquant_cmd_item stim; + + stim = current_stim("READ"); + if (item.port != 0) + `uvm_fatal("READ", $sformatf("MxquantBall has one read port, got port %0d", item.port)) + if (read_count >= stim.read_addr.size()) + `uvm_fatal("READ", $sformatf("extra read %0d", read_count)) + verification_pkg::evidence::checks++; + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.bank_id !== stim.op1_bank) + `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.op1_bank)) + if (item.group_id !== stim.op1_group) + `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.op1_group + )) + if (item.addr !== stim.read_addr[read_count][BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("READ", $sformatf( + "read %0d addr mismatch: got %0d exp %0d", + read_count, + item.addr, + stim.read_addr[read_count] + )) + read_count++; + endfunction + + function void write_write(bb_blink_write_item item); + mxquant_cmd_item stim; + bit [127:0] bytes; + + stim = current_stim("WRITE"); + if (item.port != 0) + `uvm_fatal("WRITE", $sformatf("MxquantBall has one write port, got port %0d", item.port)) + if (write_count >= stim.write_addr.size()) + `uvm_fatal("WRITE", $sformatf("extra write %0d", write_count)) + verification_pkg::evidence::checks++; + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.group_id !== stim.wr_group) + `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + )) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.addr !== stim.write_addr[write_count][BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal("WRITE", $sformatf( + "write %0d addr mismatch: got %0d exp %0d", + write_count, + item.addr, + stim.write_addr[write_count] + )) + if (item.mask !== stim.write_mask[write_count]) + `uvm_fatal("WRITE", $sformatf( + "write %0d mask mismatch: got 0x%04h exp 0x%04h", + write_count, + item.mask, + stim.write_mask[write_count] + )) + for (int b = 0; b < 16; b++) bytes[b*8+:8] = {8{item.mask[b]}}; + if ((item.data & bytes) !== (stim.write_data[write_count] & bytes)) + `uvm_fatal("SCB", $sformatf( + "write %0d data mismatch at addr %0d: got 0x%032h exp 0x%032h", + write_count, + item.addr, + item.data, + stim.write_data[write_count] + )) + write_count++; + endfunction + + function void write_resp(bb_blink_resp_item item); + mxquant_cmd_item stim; + stim = current_stim("RESP"); + verification_pkg::evidence::checks++; + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (!complete_io()) + `uvm_fatal("RESP", $sformatf( + "response before all traffic: reads=%0d writes=%0d", read_count, write_count)) + resp_count++; + endfunction + + function mxquant_cmd_item current_stim(string tag); + if (stim_q.size() == 0) `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + return stim_q[0]; + endfunction + + function bit complete_io(); + mxquant_cmd_item stim = stim_q[0]; + return read_count == stim.read_addr.size() && write_count == stim.write_addr.size(); + endfunction + + function bit done(); + return stim_q.size() == 1 && cmd_count == 1 && resp_count == 1 && complete_io(); + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + read_count = 0; + write_count = 0; + resp_count = 0; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) + `uvm_fatal("SCB", $sformatf( + "incomplete: stim=%0d cmds=%0d reads=%0d writes=%0d resp=%0d", + stim_q.size(), + cmd_count, + read_count, + write_count, + resp_count + )) + stim_q.delete(); + endfunction +endclass diff --git a/examples/prefab/balls/mxquant/verify/src/pkg/mxquant_pkg.sv b/examples/prefab/balls/mxquant/verify/src/pkg/mxquant_pkg.sv new file mode 100644 index 000000000..938c14267 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/pkg/mxquant_pkg.sv @@ -0,0 +1,13 @@ +package mxquant_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + + `include "common/mxquant_defs.svh" + `include "common/mxquant_items.svh" + `include "seq/mxquant_sequences.svh" + `include "cov/mxquant_cov.svh" + `include "env/mxquant_scoreboard.svh" + `include "env/mxquant_env.svh" + `include "tests/mxquant_test.svh" +endpackage diff --git a/examples/prefab/balls/mxquant/verify/src/seq/mxquant_sequences.svh b/examples/prefab/balls/mxquant/verify/src/seq/mxquant_sequences.svh new file mode 100644 index 000000000..7b2e461ba --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/seq/mxquant_sequences.svh @@ -0,0 +1,17 @@ +class mxquant_step_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(mxquant_step_seq) + + int unsigned step; + + function new(string name = "mxquant_step_seq"); + super.new(name); + endfunction + + task body(); + mxquant_cmd_item item; + item = mxquant_cmd_item::type_id::create("item"); + start_item(item); + item.load_rust_step(step); + finish_item(item); + endtask +endclass diff --git a/examples/prefab/balls/mxquant/verify/src/tb_top.sv b/examples/prefab/balls/mxquant/verify/src/tb_top.sv new file mode 100644 index 000000000..89b105c1d --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/tb_top.sv @@ -0,0 +1,78 @@ +module tb_top; + import uvm_pkg::*; + import mxquant_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + MxquantBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .blink_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .blink_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .blink_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .blink_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .blink_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .blink_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .blink_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .blink_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .blink_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .blink_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .blink_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .blink_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .blink_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .blink_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .blink_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .blink_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]) + ); + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("mxquant_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/mxquant/verify/src/tests/mxquant_test.svh b/examples/prefab/balls/mxquant/verify/src/tests/mxquant_test.svh new file mode 100644 index 000000000..7b6e27088 --- /dev/null +++ b/examples/prefab/balls/mxquant/verify/src/tests/mxquant_test.svh @@ -0,0 +1,56 @@ +class mxquant_ball_test extends uvm_test; + `uvm_component_utils(mxquant_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + mxquant_env env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + env = mxquant_env::type_id::create("env", this); + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + for (int unsigned index = 0; index < mxquant_num_cases(); index++) run_case(index); + phase.drop_objection(this); + endtask + + // Steps of one case run back to back on the same reset. + task run_case(int unsigned index); + int unsigned steps; + apply_reset(); + steps = mxquant_case_load(index); + for (int unsigned step = 0; step < steps; step++) run_step(index, step); + endtask + + task run_step(int unsigned index, int unsigned step); + mxquant_step_seq seq; + int cycles = 0; + env.scb.reset_counters(); + seq = mxquant_step_seq::type_id::create("seq"); + seq.step = step; + seq.start(env.cmd_agent.seqr); + while (!env.scb.done()) begin + @(posedge vif.clock); + cycles++; + if (cycles > MXQUANT_TIMEOUT_CYCLES) + `uvm_fatal("TIMEOUT", $sformatf("MxquantBall case %0d step %0d timeout", index, step)) + end + `uvm_info("MXQUANT", $sformatf("MxquantBall case %0d step %0d passed in %0d cycles", index, + step, cycles), UVM_LOW) + endtask + + task apply_reset(); + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/components/balls/mxquant/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/mxquant/workloads/ctests/CMakeLists.txt similarity index 54% rename from examples/components/balls/mxquant/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/mxquant/workloads/ctests/CMakeLists.txt index 017bd44ca..b1358b742 100644 --- a/examples/components/balls/mxquant/workloads/ctests/CMakeLists.txt +++ b/examples/prefab/balls/mxquant/workloads/ctests/CMakeLists.txt @@ -1,11 +1,7 @@ -file( - READ "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h" - MXQUANT_PARAMS +math( + EXPR MXQUANT_BANK_BYTES + "${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_ENTRIES} * ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_WIDTH} / 8" ) -string(REGEX MATCH "#define BANK_LINES ([0-9]+)" _ "${MXQUANT_PARAMS}") -set(MXQUANT_BANK_LINES ${CMAKE_MATCH_1}) -string(REGEX MATCH "#define BANK_WIDTH ([0-9]+)" _ "${MXQUANT_PARAMS}") -math(EXPR MXQUANT_BANK_BYTES "${MXQUANT_BANK_LINES} * ${CMAKE_MATCH_1} / 8") math(EXPR MXQUANT_BANK_TAIL "${MXQUANT_BANK_BYTES} % 128") if(MXQUANT_BANK_BYTES LESS 3840 OR MXQUANT_BANK_TAIL) message( @@ -16,7 +12,7 @@ endif() add_buckyball_ctest(mxquant_values_test.c) add_buckyball_ctest(mxquant_capacity_test.c) set(TEST_NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-mxquant_values_test" + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-mxquant_values_test" ) add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}-baremetal diff --git a/examples/components/balls/mxquant/workloads/ctests/mxquant_capacity_test.c b/examples/prefab/balls/mxquant/workloads/ctests/mxquant_capacity_test.c similarity index 77% rename from examples/components/balls/mxquant/workloads/ctests/mxquant_capacity_test.c rename to examples/prefab/balls/mxquant/workloads/ctests/mxquant_capacity_test.c index 0e7104131..40e811612 100644 --- a/examples/components/balls/mxquant/workloads/ctests/mxquant_capacity_test.c +++ b/examples/prefab/balls/mxquant/workloads/ctests/mxquant_capacity_test.c @@ -5,7 +5,10 @@ #include #include -enum { BANK_BYTES = BANK_LINES * BANK_WIDTH / 8, MAX_COUNT = BANK_BYTES / 4 }; +enum { + BANK_BYTES = THREAD_MEM_BANK_ENTRIES * THREAD_MEM_BANK_WIDTH / 8, + MAX_COUNT = BANK_BYTES / 4 +}; static uint32_t input[MAX_COUNT] __attribute__((aligned(64))); static uint8_t actual[BANK_BYTES] __attribute__((aligned(64))); @@ -18,10 +21,10 @@ int main(void) { memset(actual, 0x5a, sizeof(actual)); bb_mem_alloc(3, 1, 1); bb_mem_alloc(4, 1, 1); - bb_mvin((uintptr_t)input, 3, BANK_LINES, 1); - bb_mvin((uintptr_t)actual, 4, BANK_LINES, 1); + bb_mvin((uintptr_t)input, 3, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvin((uintptr_t)actual, 4, THREAD_MEM_BANK_ENTRIES, 1); bb_mxquant(3, 4, count); - bb_mvout((uintptr_t)actual, 4, BANK_LINES, 1); + bb_mvout((uintptr_t)actual, 4, THREAD_MEM_BANK_ENTRIES, 1); for (int i = 0; i < BANK_BYTES; ++i) { uint8_t expected = i < count ? ((i & 1) ? 0xf8 : 0x78) : i < count + count / 32 ? 119 diff --git a/examples/components/balls/mxquant/workloads/ctests/mxquant_values_test.c b/examples/prefab/balls/mxquant/workloads/ctests/mxquant_values_test.c similarity index 100% rename from examples/components/balls/mxquant/workloads/ctests/mxquant_values_test.c rename to examples/prefab/balls/mxquant/workloads/ctests/mxquant_values_test.c diff --git a/examples/components/balls/mxquant/workloads/isa/mxquant.h b/examples/prefab/balls/mxquant/workloads/isa/mxquant.h similarity index 86% rename from examples/components/balls/mxquant/workloads/isa/mxquant.h rename to examples/prefab/balls/mxquant/workloads/isa/mxquant.h index bdf29e232..04eed1988 100644 --- a/examples/components/balls/mxquant/workloads/isa/mxquant.h +++ b/examples/prefab/balls/mxquant/workloads/isa/mxquant.h @@ -6,5 +6,5 @@ #define bb_mxquant(input, output, count) \ BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(input) | BB_BANK2(output) | \ BB_ITER(count), \ - 0, BB_FUNC7(MXQUANT)) + 0, FUNCT7(mxquant, MXQUANT)) #endif diff --git a/examples/prefab/balls/mxquant/workloads/mlir_tests/ball/mxquant_two_blocks.mlir b/examples/prefab/balls/mxquant/workloads/mlir_tests/ball/mxquant_two_blocks.mlir new file mode 100644 index 000000000..0024cc141 --- /dev/null +++ b/examples/prefab/balls/mxquant/workloads/mlir_tests/ball/mxquant_two_blocks.mlir @@ -0,0 +1,9 @@ +// RUN: buddy-opt %s -lower-buckyball -canonicalize | FileCheck %s +// CHECK-LABEL: func.func @quant +// CHECK-DAG: arith.constant 68721573889 : i64 +// CHECK-DAG: arith.constant 0 : i64 +// The caller loads 64 FP32 into bank 1 and the output guard into bank 2. +func.func @quant() attributes {llvm.emit_c_interface} { + mxquant_ball.quant_mxfp8 <{input = 1 : i64, output = 2 : i64, count = 64 : i64}> + return +} diff --git a/examples/prefab/balls/mxquant/workloads/mlir_tests/bank/mxquant_two_blocks.mlir b/examples/prefab/balls/mxquant/workloads/mlir_tests/bank/mxquant_two_blocks.mlir new file mode 100644 index 000000000..9b00fcc6f --- /dev/null +++ b/examples/prefab/balls/mxquant/workloads/mlir_tests/bank/mxquant_two_blocks.mlir @@ -0,0 +1,14 @@ +// Quantizes two blocks of 32 FP32. The output comes in holding a guard +// pattern and goes back with the codes and scales. +func.func @quant(%input: memref<16x4xi32>, %output: memref<5x16xi8>) + attributes {llvm.emit_c_interface} { + %in_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %out_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %li = buckyball.bank_mvin %input, %in_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> + %lo = buckyball.bank_mvin %output, %out_bank <{depth = 5 : i64, stride = 1 : i64}> : memref<5x16xi8> + %packed = mxquant_ball.bank_quant_mxfp8 %li -> %lo <{count = 64 : i64}> + %stored = buckyball.bank_mvout %output, %packed <{depth = 5 : i64, stride = 1 : i64}> : memref<5x16xi8> + buckyball.bank_release %li + buckyball.bank_release %stored + return +} diff --git a/examples/components/balls/mxquant/workloads/reference.h b/examples/prefab/balls/mxquant/workloads/reference.h similarity index 100% rename from examples/components/balls/mxquant/workloads/reference.h rename to examples/prefab/balls/mxquant/workloads/reference.h diff --git a/examples/components/balls/redmule/arch/RedmuleBall.scala b/examples/prefab/balls/redmule/arch/RedmuleBall.scala similarity index 99% rename from examples/components/balls/redmule/arch/RedmuleBall.scala rename to examples/prefab/balls/redmule/arch/RedmuleBall.scala index a009ad89e..bed8c7bc0 100644 --- a/examples/components/balls/redmule/arch/RedmuleBall.scala +++ b/examples/prefab/balls/redmule/arch/RedmuleBall.scala @@ -193,7 +193,7 @@ class RedmuleBall( (cmd.wr_bank, cmd.rs2(41, 32) +& m) ) ) { - val shared = bank >= fp.shared_bank_id_base.U + val shared = bank >= fp.shared_vbank_base.U val capacity = Mux( shared, mp.sharedBankEntries.U, diff --git a/examples/components/balls/redmule/arch/RedmuleNative.scala b/examples/prefab/balls/redmule/arch/RedmuleNative.scala similarity index 79% rename from examples/components/balls/redmule/arch/RedmuleNative.scala rename to examples/prefab/balls/redmule/arch/RedmuleNative.scala index acaa36f24..6ac4f1506 100644 --- a/examples/components/balls/redmule/arch/RedmuleNative.scala +++ b/examples/prefab/balls/redmule/arch/RedmuleNative.scala @@ -84,13 +84,29 @@ class RedmuleNative extends BlackBox with HasBlackBoxInline { !Seq("redmule_xif_wrap.sv", "redmule_mm_wrap.sv", "redmule_inst_decoder.sv").contains(p.last) ) + // redmule_pkg.sv imports its dependencies at file scope, before the package. + // vvac gathers every file-scope item into one $unit placed at the first of + // them, so the later `import redmule_pkg::*` would precede redmule_pkg. + // Move these imports inside the package; everything else is copied verbatim. + val redmulePkgHead = + "import fpnew_pkg::*;\nimport hci_package::*;\nimport hwpe_stream_package::*;\n\npackage redmule_pkg;\n" + val redmulePkg = os.read(native / "redmule_pkg.sv") + require(redmulePkg.contains(redmulePkgHead), "redmule_pkg.sv header changed") + + val redmulePkgFixed = redmulePkg.replace( + redmulePkgHead, + "package redmule_pkg;\n import fpnew_pkg::*;\n import hci_package::*;\n import hwpe_stream_package::*;\n" + ) + val wrapper = sourceDir / "RedmuleNative.sv" - // Copy the upstream files verbatim; only the external wrapper is authored here. + // Only the external wrapper is authored here. setInline( "RedmuleNative.sv", "/* verilator lint_off ENUMVALUE */\n" + (packages ++ primitives ++ streamSources ++ hciSources ++ - controlSources ++ nativeSources).map(os.read(_)).mkString("\n") + + controlSources ++ nativeSources).map(p => + if (p == native / "redmule_pkg.sv") redmulePkgFixed else os.read(p) + ).mkString("\n") + "\n/* verilator lint_on ENUMVALUE */\n" + os.read(wrapper) ) } diff --git a/examples/components/balls/redmule/arch/RedmuleNative.sv b/examples/prefab/balls/redmule/arch/RedmuleNative.sv similarity index 100% rename from examples/components/balls/redmule/arch/RedmuleNative.sv rename to examples/prefab/balls/redmule/arch/RedmuleNative.sv diff --git a/examples/components/balls/redmule/configs/default.toml b/examples/prefab/balls/redmule/configs/default.toml similarity index 100% rename from examples/components/balls/redmule/configs/default.toml rename to examples/prefab/balls/redmule/configs/default.toml diff --git a/examples/components/balls/redmule/configs/redmule_params.proto b/examples/prefab/balls/redmule/configs/redmule_params.proto similarity index 100% rename from examples/components/balls/redmule/configs/redmule_params.proto rename to examples/prefab/balls/redmule/configs/redmule_params.proto diff --git a/examples/components/balls/redmule/emu/src/fp16.rs b/examples/prefab/balls/redmule/emu/src/fp16.rs similarity index 100% rename from examples/components/balls/redmule/emu/src/fp16.rs rename to examples/prefab/balls/redmule/emu/src/fp16.rs diff --git a/examples/components/balls/redmule/emu/src/lib.rs b/examples/prefab/balls/redmule/emu/src/lib.rs similarity index 100% rename from examples/components/balls/redmule/emu/src/lib.rs rename to examples/prefab/balls/redmule/emu/src/lib.rs diff --git a/examples/components/balls/redmule/emu/src/matmul.rs b/examples/prefab/balls/redmule/emu/src/matmul.rs similarity index 90% rename from examples/components/balls/redmule/emu/src/matmul.rs rename to examples/prefab/balls/redmule/emu/src/matmul.rs index 2855ef6c2..0bdc174b3 100644 --- a/examples/components/balls/redmule/emu/src/matmul.rs +++ b/examples/prefab/balls/redmule/emu/src/matmul.rs @@ -1,4 +1,4 @@ -use bebop_bemu::thread::instructions::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use bebop_bemu::thread::instructions::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use bebop_bemu::ball::ExecContext; pub(super) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let a = rs1_b0(xs1); @@ -11,6 +11,12 @@ pub(super) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { let c_base = ((xs2 >> 32) & 1023) as usize; let first = xs2 & (1 << 42) != 0; assert_eq!(xs2 >> 43, 0, "redmule: reserved bits must be zero"); + // A continuation reads the old C from wr_read_bank, which rs1[29] provides. + assert_eq!( + rs1_read_old(xs1), + !first, + "redmule: rs1[29] (read old destination) must be set exactly for a continuation (rs2[42] clear)" + ); assert!( rows > 0 && k > 0 && k % 8 == 0, "redmule: M > 0, K must be a positive multiple of eight" diff --git a/examples/prefab/balls/redmule/stack/src/redmuleOps.cpp b/examples/prefab/balls/redmule/stack/src/redmuleOps.cpp new file mode 100644 index 000000000..48887b348 --- /dev/null +++ b/examples/prefab/balls/redmule/stack/src/redmuleOps.cpp @@ -0,0 +1,76 @@ +//===- redmuleOps.cpp - RedmuleBall operations ----------------------------===// +// +// The dialect of RedmuleBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "redmuleOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::redmule_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "redmuleDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "redmuleOps.cpp.inc" + +void RedmuleBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "redmuleOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankMatmulF16Op::getSourceBanks() { + return {getA(), getB()}; +} +Value BankMatmulF16Op::getDestinationBank() { return getC(); } +bool BankMatmulF16Op::readsDestination() { return getReadDestination(); } +Operation *BankMatmulF16Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulF16Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getK(), + getReadDestination()); +} + +SmallVector MatmulF16Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t MatmulF16Op::getDestinationBank() { return getC(); } +LogicalResult MatmulF16Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), k = getK(), a = getA(), b = getB(), c = getC(); + bool readDestination = getReadDestination(); + const auto &target = buckyball_target::getBuckyballTarget(); + if (a == b || a == c || b == c) + return emitOpError("FP16 matmul requires distinct A, B and C banks"); + if (target.bankWidthBits != 128 || rows <= 0 || rows > 4095 || k <= 0 || + k % 8 || k >= (1LL << 33) || rows * (k / 8) > target.bankDepth || + k > target.bankDepth || rows > target.bankDepth) + return emitOpError("FP16 matmul shape exceeds the bank or ISA geometry"); + // rs2: rows in [11:0], the A, B and C row bases (all 0) in [41:12], and in + // bit 42 whether the FMAs start from positive zero. + uint64_t rs2 = field(rows, 0, 11) | field(!readDestination, 42, 42); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("REDMULE_F16"), + constantI64(builder, loc, bankFields({a, b}, c, readDestination, k)), + constantI64(builder, loc, rs2)); + return success(); +} diff --git a/examples/prefab/balls/redmule/stack/src/redmuleOps.h b/examples/prefab/balls/redmule/stack/src/redmuleOps.h new file mode 100644 index 000000000..45ba8ebe8 --- /dev/null +++ b/examples/prefab/balls/redmule/stack/src/redmuleOps.h @@ -0,0 +1,20 @@ +//===- redmuleOps.h - RedmuleBall operations ------------------------------===// + +#ifndef REDMULE_OPS_H +#define REDMULE_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "redmuleDialect.h.inc" + +#define GET_OP_CLASSES +#include "redmuleOps.h.inc" + +#endif // REDMULE_OPS_H diff --git a/examples/prefab/balls/redmule/stack/src/redmuleOps.td b/examples/prefab/balls/redmule/stack/src/redmuleOps.td new file mode 100644 index 000000000..c5d69c5ad --- /dev/null +++ b/examples/prefab/balls/redmule/stack/src/redmuleOps.td @@ -0,0 +1,49 @@ +//===- redmuleOps.td - RedmuleBall operations --------------*- tablegen -*-===// +// +// matmul_f16: C = A @ B in FP16 with ordered FMAs, or C = C + A @ B with +// read_destination. `k` is a positive multiple of eight. +// A: `rows` rows of `k` f16, k / 8 bank rows each. +// B: `k` bank rows of eight f16. +// C: `rows` bank rows of eight f16. +// +// The Ball places every operand at bank row 0. With read_destination rs1 marks +// the old C, which the Ball reads from the bank the destination held before +// this write, and the FMAs start from it instead of positive zero. +// +//===----------------------------------------------------------------------===// + +#ifndef REDMULE_OPS +#define REDMULE_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def RedmuleBall_Dialect : Dialect { + let name = "redmule_ball"; + let cppNamespace = "::buddy::redmule_ball"; + let summary = "The operations of RedmuleBall"; +} + +class RedmuleBall_Op traits = []> + : Op])>; + +def BankMatmulF16Op : RedmuleBall_Op<"bank_matmul_f16", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_f16 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$k, + UnitAttr:$read_destination); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulF16Op : RedmuleBall_Op<"matmul_f16", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_f16 on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows, + I64Attr:$k, UnitAttr:$read_destination); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // REDMULE_OPS diff --git a/examples/components/balls/redmule/verify/casegen/Cargo.lock b/examples/prefab/balls/redmule/verify/casegen/Cargo.lock similarity index 100% rename from examples/components/balls/redmule/verify/casegen/Cargo.lock rename to examples/prefab/balls/redmule/verify/casegen/Cargo.lock diff --git a/examples/components/balls/redmule/verify/casegen/Cargo.toml b/examples/prefab/balls/redmule/verify/casegen/Cargo.toml similarity index 100% rename from examples/components/balls/redmule/verify/casegen/Cargo.toml rename to examples/prefab/balls/redmule/verify/casegen/Cargo.toml diff --git a/examples/components/balls/redmule/verify/casegen/src/lib.rs b/examples/prefab/balls/redmule/verify/casegen/src/lib.rs similarity index 100% rename from examples/components/balls/redmule/verify/casegen/src/lib.rs rename to examples/prefab/balls/redmule/verify/casegen/src/lib.rs diff --git a/examples/components/balls/redmule/verify/filelists/redmule_ball.f b/examples/prefab/balls/redmule/verify/filelists/redmule_ball.f similarity index 100% rename from examples/components/balls/redmule/verify/filelists/redmule_ball.f rename to examples/prefab/balls/redmule/verify/filelists/redmule_ball.f diff --git a/examples/components/balls/redmule/verify/parameters.toml b/examples/prefab/balls/redmule/verify/parameters.toml similarity index 100% rename from examples/components/balls/redmule/verify/parameters.toml rename to examples/prefab/balls/redmule/verify/parameters.toml diff --git a/examples/components/balls/redmule/verify/src/pkg/redmule_pkg.sv b/examples/prefab/balls/redmule/verify/src/pkg/redmule_pkg.sv similarity index 100% rename from examples/components/balls/redmule/verify/src/pkg/redmule_pkg.sv rename to examples/prefab/balls/redmule/verify/src/pkg/redmule_pkg.sv diff --git a/examples/components/balls/redmule/verify/src/reference/redmule_reference.sv b/examples/prefab/balls/redmule/verify/src/reference/redmule_reference.sv similarity index 100% rename from examples/components/balls/redmule/verify/src/reference/redmule_reference.sv rename to examples/prefab/balls/redmule/verify/src/reference/redmule_reference.sv diff --git a/examples/components/balls/redmule/verify/src/tb_top.sv b/examples/prefab/balls/redmule/verify/src/tb_top.sv similarity index 100% rename from examples/components/balls/redmule/verify/src/tb_top.sv rename to examples/prefab/balls/redmule/verify/src/tb_top.sv diff --git a/examples/prefab/balls/redmule/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/redmule/workloads/ctests/CMakeLists.txt new file mode 100644 index 000000000..829a0b259 --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/ctests/CMakeLists.txt @@ -0,0 +1,12 @@ +set(REDMULE_BANK_WIDTH ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_WIDTH}) +set(REDMULE_BANK_LINES ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_ENTRIES}) +if(NOT REDMULE_BANK_WIDTH EQUAL 128 OR REDMULE_BANK_LINES LESS 18) + message( + FATAL_ERROR + "RedmuleBall tests require 128-bit rows and at least 18 bank rows, got ${REDMULE_BANK_WIDTH} bits and ${REDMULE_BANK_LINES} rows" + ) +endif() +add_buckyball_ctest(redmule_f16_test.c) +add_buckyball_ctest(redmule_f16_chain_test.c) +add_buckyball_ctest(redmule_f16_edges_test.c) +add_buckyball_ctest(redmule_f16_fused_test.c) diff --git a/examples/components/balls/redmule/workloads/ctests/redmule_f16_chain_test.c b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_chain_test.c similarity index 93% rename from examples/components/balls/redmule/workloads/ctests/redmule_f16_chain_test.c rename to examples/prefab/balls/redmule/workloads/ctests/redmule_f16_chain_test.c index 393ae7620..9fcc371d1 100644 --- a/examples/components/balls/redmule/workloads/ctests/redmule_f16_chain_test.c +++ b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_chain_test.c @@ -1,5 +1,4 @@ #include -#include #include static uint16_t a[16] __attribute__((aligned(64))); @@ -7,7 +6,7 @@ static uint16_t b[64] __attribute__((aligned(64))); static uint16_t c[16] __attribute__((aligned(64))); int main(void) { - unsigned ab = VIRTUAL_BANK_NUM - 6, bb = ab + 1, cb = ab + 2; + unsigned ab = CORE_VIRTUAL_BANK_COUNT - 6, bb = ab + 1, cb = ab + 2; for (unsigned i = 0; i < 16; ++i) { a[i] = 0x3c00; c[i] = 0x3555; diff --git a/examples/components/balls/redmule/workloads/ctests/redmule_f16_edges_test.c b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_edges_test.c similarity index 98% rename from examples/components/balls/redmule/workloads/ctests/redmule_f16_edges_test.c rename to examples/prefab/balls/redmule/workloads/ctests/redmule_f16_edges_test.c index 60ad0c7d8..a36bc9a3f 100644 --- a/examples/components/balls/redmule/workloads/ctests/redmule_f16_edges_test.c +++ b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_edges_test.c @@ -1,5 +1,4 @@ #include -#include #include static const uint16_t a[8] diff --git a/examples/components/balls/redmule/workloads/ctests/redmule_f16_fused_test.c b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_fused_test.c similarity index 97% rename from examples/components/balls/redmule/workloads/ctests/redmule_f16_fused_test.c rename to examples/prefab/balls/redmule/workloads/ctests/redmule_f16_fused_test.c index 0d36b9b52..939021f87 100644 --- a/examples/components/balls/redmule/workloads/ctests/redmule_f16_fused_test.c +++ b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_fused_test.c @@ -1,5 +1,4 @@ #include -#include #include static const uint16_t a[8] diff --git a/examples/components/balls/redmule/workloads/ctests/redmule_f16_test.c b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_test.c similarity index 94% rename from examples/components/balls/redmule/workloads/ctests/redmule_f16_test.c rename to examples/prefab/balls/redmule/workloads/ctests/redmule_f16_test.c index ba1e3f0b0..18e4cf00f 100644 --- a/examples/components/balls/redmule/workloads/ctests/redmule_f16_test.c +++ b/examples/prefab/balls/redmule/workloads/ctests/redmule_f16_test.c @@ -1,5 +1,4 @@ #include -#include #include static uint16_t a[8 * 8] __attribute__((aligned(64))); @@ -7,7 +6,7 @@ static uint16_t b[18 * 8] __attribute__((aligned(64))); static uint16_t c[7 * 8] __attribute__((aligned(64))); int main(void) { - unsigned ab = VIRTUAL_BANK_NUM - 3, bb = ab + 1, cb = ab + 2; + unsigned ab = CORE_VIRTUAL_BANK_COUNT - 3, bb = ab + 1, cb = ab + 2; for (unsigned i = 0; i < 8 * 8; ++i) a[i] = 0x7e00; for (unsigned i = 0; i < 18 * 8; ++i) diff --git a/examples/components/balls/redmule/workloads/isa/redmule.h b/examples/prefab/balls/redmule/workloads/isa/redmule.h similarity index 94% rename from examples/components/balls/redmule/workloads/isa/redmule.h rename to examples/prefab/balls/redmule/workloads/isa/redmule.h index 963c37a89..5b06bce40 100644 --- a/examples/components/balls/redmule/workloads/isa/redmule.h +++ b/examples/prefab/balls/redmule/workloads/isa/redmule.h @@ -19,6 +19,6 @@ static inline void bb_redmule(uint32_t a, uint32_t b, uint32_t c, uint32_t m, (uint64_t)first << 42; BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(a) | BB_BANK1(b) | BB_BANK2(c) | BB_ITER(k) | (first ? 0 : BB_READ_WR_OLD), - special, BB_FUNC7(REDMULE_F16)); + special, FUNCT7(redmule, REDMULE_F16)); } #endif diff --git a/examples/components/balls/redmule/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/redmule/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/redmule/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/redmule/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/redmule/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/redmule/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/components/balls/redmule/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/redmule/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir b/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir new file mode 100644 index 000000000..ce47c740b --- /dev/null +++ b/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain.mlir @@ -0,0 +1,6 @@ +// The banks 1 (A), 2 (B) and 3 (C) are allocated and loaded by the caller. +func.func @redmule_chain() attributes {llvm.emit_c_interface} { + redmule_ball.matmul_f16 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 2 : i64, k = 8 : i64}> + redmule_ball.matmul_f16 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 2 : i64, k = 8 : i64, read_destination}> + return +} diff --git a/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp b/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp similarity index 96% rename from examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp rename to examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp index 0c3e6ddb2..9adc1d8d4 100644 --- a/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp +++ b/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_chain_main.cpp @@ -1,6 +1,5 @@ #include #include -#include extern "C" void _mlir_ciface_redmule_chain(); alignas(64) static uint16_t a[16], b[64], c[16]; diff --git a/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir b/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir similarity index 59% rename from examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir rename to examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir index 0f1cf5852..d482cf813 100644 --- a/examples/components/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir +++ b/examples/prefab/balls/redmule/workloads/mlir_tests/ball/redmule_f16_invalid_k.mlir @@ -1,6 +1,6 @@ // RUN: buddy-opt %s -lower-buckyball -verify-diagnostics func.func @invalid_k() { // expected-error @+1 {{FP16 matmul shape exceeds the bank or ISA geometry}} - buckyball.custom "matmul_f16" <{sources = array, destination = 3 : i64}> {rows = 1 : i64, k = 7 : i64} + redmule_ball.matmul_f16 <{a = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 1 : i64, k = 7 : i64}> return } diff --git a/examples/components/balls/redmule/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/redmule/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/components/balls/redmule/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/redmule/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir b/examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir similarity index 81% rename from examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir rename to examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir index be089fca3..9d53f0db9 100644 --- a/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir +++ b/examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_invalid_alias.mlir @@ -3,7 +3,7 @@ func.func @invalid_alias() { %a = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %b = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> // expected-error @+1 {{FP16 matmul requires distinct A, B and C banks}} - %c = buckyball.bank_custom "matmul_f16" (%a, %b) -> %a {rows = 1 : i64, k = 8 : i64} + %c = redmule_ball.bank_matmul_f16 %a, %b -> %a <{rows = 1 : i64, k = 8 : i64}> buckyball.bank_release %c buckyball.bank_release %b return diff --git a/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in b/examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in similarity index 94% rename from examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in rename to examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in index 121181a92..09af1805d 100644 --- a/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in +++ b/examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows.mlir.in @@ -34,8 +34,8 @@ func.func @main() -> i8 { %c_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded_a = buckyball.bank_mvin %a, %a_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x8xf16> %loaded_b = buckyball.bank_mvin %b, %b_bank <{depth = 8 : i64, stride = 1 : i64}> : memref<8x8xf16> - %result = buckyball.bank_custom "matmul_f16" (%loaded_a, %loaded_b) -> %c_bank - {rows = @BB_BANK_LINES@ : i64, k = 8 : i64} + %result = redmule_ball.bank_matmul_f16 %loaded_a, %loaded_b -> %c_bank + <{rows = @BB_BANK_LINES@ : i64, k = 8 : i64}> %stored = buckyball.bank_mvout %c, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x8xf16> buckyball.bank_release %loaded_a buckyball.bank_release %loaded_b diff --git a/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows_main.cpp b/examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows_main.cpp similarity index 86% rename from examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows_main.cpp rename to examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows_main.cpp index 7fd11260c..0e0c9735f 100644 --- a/examples/components/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows_main.cpp +++ b/examples/prefab/balls/redmule/workloads/mlir_tests/bank/redmule_f16_rows_main.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -6,7 +5,8 @@ extern "C" void check_result(uint16_t *, uint16_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { - if (rows != BANK_LINES || lanes != 8 || stride != 8 || lane_stride != 1) + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != 8 || stride != 8 || + lane_stride != 1) exit(1); for (int row = 0; row < rows; ++row) for (int lane = 0; lane < lanes; ++lane) diff --git a/examples/components/balls/relu/README.md b/examples/prefab/balls/relu/README.md similarity index 100% rename from examples/components/balls/relu/README.md rename to examples/prefab/balls/relu/README.md diff --git a/examples/components/balls/relu/arch/Relu.scala b/examples/prefab/balls/relu/arch/Relu.scala similarity index 100% rename from examples/components/balls/relu/arch/Relu.scala rename to examples/prefab/balls/relu/arch/Relu.scala diff --git a/examples/components/balls/relu/arch/ReluBall.scala b/examples/prefab/balls/relu/arch/ReluBall.scala similarity index 100% rename from examples/components/balls/relu/arch/ReluBall.scala rename to examples/prefab/balls/relu/arch/ReluBall.scala diff --git a/examples/components/balls/relu/configs/default.toml b/examples/prefab/balls/relu/configs/default.toml similarity index 100% rename from examples/components/balls/relu/configs/default.toml rename to examples/prefab/balls/relu/configs/default.toml diff --git a/examples/components/balls/relu/configs/relu_params.proto b/examples/prefab/balls/relu/configs/relu_params.proto similarity index 100% rename from examples/components/balls/relu/configs/relu_params.proto rename to examples/prefab/balls/relu/configs/relu_params.proto diff --git a/examples/components/balls/relu/emu/src/50_relu.rs b/examples/prefab/balls/relu/emu/src/50_relu.rs similarity index 85% rename from examples/components/balls/relu/emu/src/50_relu.rs rename to examples/prefab/balls/relu/emu/src/50_relu.rs index 68ee6c847..7ac68263c 100644 --- a/examples/components/balls/relu/emu/src/50_relu.rs +++ b/examples/prefab/balls/relu/emu/src/50_relu.rs @@ -1,5 +1,5 @@ use bebop_bemu::thread::banks::bank_size; -use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old, rs1_update}; use super::instruction::{BallModel, ExecContext}; pub struct Relu; @@ -10,6 +10,12 @@ impl BallModel for Relu { let group = rs1_b1(xs1); let iter = rs1_iter(xs1) as usize; let stride = xs2 as usize; + // The RTL requires op1 and wr to name one renamed bank: only an in-place + // update (rs1[63]) keeps the destination alias. + assert!( + !rs1_read_old(xs1) && rs1_update(xs1), + "relu: updates its bank in place; rs1[63] must be set and rs1[29] clear" + ); if rs1_b2(xs1) != bank { panic!("relu: source and destination banks must match"); diff --git a/examples/components/balls/relu/emu/src/dispatch.rs b/examples/prefab/balls/relu/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/relu/emu/src/dispatch.rs rename to examples/prefab/balls/relu/emu/src/dispatch.rs diff --git a/examples/components/balls/relu/emu/src/lib.rs b/examples/prefab/balls/relu/emu/src/lib.rs similarity index 100% rename from examples/components/balls/relu/emu/src/lib.rs rename to examples/prefab/balls/relu/emu/src/lib.rs diff --git a/examples/prefab/balls/relu/stack/src/reluOps.cpp b/examples/prefab/balls/relu/stack/src/reluOps.cpp new file mode 100644 index 000000000..f6171fe82 --- /dev/null +++ b/examples/prefab/balls/relu/stack/src/reluOps.cpp @@ -0,0 +1,66 @@ +//===- reluOps.cpp - ReluBall operations ----------------------------------===// +// +// The dialect of ReluBall, its virtual operations building their physical ones +// and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "reluOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::relu_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "reluDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "reluOps.cpp.inc" + +void ReluBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "reluOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect( + [](DialectRegistry ®istry) { registry.insert(); }), + true); +} // namespace + +SmallVector BankReluI32Op::getSourceBanks() { return {getBank()}; } +Value BankReluI32Op::getDestinationBank() { return getBank(); } +bool BankReluI32Op::readsDestination() { return false; } +Operation *BankReluI32Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return ReluI32Op::create(builder, getLoc(), destination, getRows()); +} + +SmallVector ReluI32Op::getSourceBanks() { + return {int64_t(getBank())}; +} +int64_t ReluI32Op::getDestinationBank() { return getBank(); } +LogicalResult ReluI32Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), bank = getBank(); + int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; + if (rows <= 0 || rows > depth || depth % 16) + return emitOpError("relu_i32 rows exceed the bank"); + // Group 0 of the bank in the bank1 field; bit 63 marks the in-place + // update. rs2 is the segment stride. + uint64_t rs1 = + bankFields({bank}, bank, false, (rows + 15) / 16 * 16) | (1ULL << 63); + emitInstruction(builder, loc, buckyball_target::getBuckyballFunct7("RELU"), + constantI64(builder, loc, rs1), + constantI64(builder, loc, depth)); + return success(); +} diff --git a/examples/prefab/balls/relu/stack/src/reluOps.h b/examples/prefab/balls/relu/stack/src/reluOps.h new file mode 100644 index 000000000..268b7e24c --- /dev/null +++ b/examples/prefab/balls/relu/stack/src/reluOps.h @@ -0,0 +1,20 @@ +//===- reluOps.h - ReluBall operations ------------------------------------===// + +#ifndef RELU_OPS_H +#define RELU_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "reluDialect.h.inc" + +#define GET_OP_CLASSES +#include "reluOps.h.inc" + +#endif // RELU_OPS_H diff --git a/examples/prefab/balls/relu/stack/src/reluOps.td b/examples/prefab/balls/relu/stack/src/reluOps.td new file mode 100644 index 000000000..bb17a2a08 --- /dev/null +++ b/examples/prefab/balls/relu/stack/src/reluOps.td @@ -0,0 +1,44 @@ +//===- reluOps.td - ReluBall operations --------------------*- tablegen -*-===// +// +// relu_i32: max(x, 0) on the first `rows` rows of four i32 of a one-group +// bank, in place: the one source is the destination. +// +// The Ball updates whole groups of sixteen rows: it rounds `rows` up to a +// multiple of sixteen, which only touches rows past the valid data, and runs +// one segment as deep as the bank. +// +//===----------------------------------------------------------------------===// + +#ifndef RELU_OPS +#define RELU_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def ReluBall_Dialect : Dialect { + let name = "relu_ball"; + let cppNamespace = "::buddy::relu_ball"; + let summary = "The operations of ReluBall"; +} + +class ReluBall_Op traits = []> + : Op])>; + +def BankReluI32Op : ReluBall_Op<"bank_relu_i32", + [DeclareOpInterfaceMethods]> { + let summary = "relu_i32 on a virtual bank, in place"; + let arguments = (ins I64:$bank, I64Attr:$rows); + let results = (outs I64:$bankOut); + let assemblyFormat = "$bank prop-dict attr-dict"; +} + +def ReluI32Op : ReluBall_Op<"relu_i32", + [DeclareOpInterfaceMethods]> { + let summary = "relu_i32 on a physical bank, in place"; + let arguments = (ins I64Attr:$bank, I64Attr:$rows); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // RELU_OPS diff --git a/examples/prefab/balls/relu/verify/casegen/Cargo.lock b/examples/prefab/balls/relu/verify/casegen/Cargo.lock new file mode 100644 index 000000000..31c865ecd --- /dev/null +++ b/examples/prefab/balls/relu/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "relu-casegen" +version = "0.1.0" diff --git a/examples/prefab/balls/relu/verify/casegen/Cargo.toml b/examples/prefab/balls/relu/verify/casegen/Cargo.toml new file mode 100644 index 000000000..facc2aa63 --- /dev/null +++ b/examples/prefab/balls/relu/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "relu-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["rlib", "cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/prefab/balls/relu/verify/casegen/src/casegen.rs b/examples/prefab/balls/relu/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..bc5cf613a --- /dev/null +++ b/examples/prefab/balls/relu/verify/casegen/src/casegen.rs @@ -0,0 +1,89 @@ +use crate::model; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ReluCmd { + pub iter: u32, + pub stride: u32, + pub bank: u32, + pub group_base: u32, + pub group: u32, + pub cols: u32, + pub rob_id: u32, + pub num_words: u32, +} + +#[derive(Clone, Debug)] +pub struct ReluCase { + pub cmd: ReluCmd, + pub rows: Vec, + pub src: Vec, + pub dst: Vec, +} + +struct Lcg(u64); + +impl Lcg { + fn next(&mut self) -> u32 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (self.0 >> 32) as u32 + } +} + +// Lane values cycle through the edges (zero, -1, INT_MIN, INT_MAX, 1) before +// falling back to pseudo-random values of both signs. +fn lane_value(rng: &mut Lcg, index: usize) -> u32 { + match index % 8 { + 0 => 0, + 1 => 0xffff_ffff, + 2 => 0x8000_0000, + 3 => 0x7fff_ffff, + 4 => 1, + _ => rng.next(), + } +} + +pub fn gen_case(index: u32) -> ReluCase { + let entries = model::bank_entries(); + // (rob_id, bank, iter, stride, cols, group, group_base) + let (rob_id, bank, iter, stride, cols, group, group_base) = match index { + 0 => (1, 0, 16, entries, 1, 0, 0), + 1 => (2, 3, 32, entries / 2, 1, 0, 0), + 2 => (3, 5, 16, 16, 1, 0, 0), + 3 => (4, 7, 48, 256, 2, 1, 2), + 4 => (5, 9, 64, 64, 4, 3, 0), + 5 => (6, 2, entries, entries, 3, 2, 1), + _ => panic!("relu: unsupported case {index}"), + }; + model::validate(entries, iter, u64::from(stride), group, cols); + let rows = model::rows(entries, iter, stride); + let mut rng = Lcg(0x5eed_0000 + u64::from(index)); + let mut lane = 0usize; + let src: Vec = rows + .iter() + .map(|_| { + let mut word = 0u128; + for l in 0..4 { + word |= u128::from(lane_value(&mut rng, lane)) << (32 * l); + lane += 1; + } + word + }) + .collect(); + let dst = src.iter().map(|&w| model::relu_word(w)).collect(); + ReluCase { + cmd: ReluCmd { + iter, + stride, + bank, + group_base, + group, + cols, + rob_id, + num_words: rows.len() as u32, + }, + rows, + src, + dst, + } +} diff --git a/examples/prefab/balls/relu/verify/casegen/src/lib.rs b/examples/prefab/balls/relu/verify/casegen/src/lib.rs new file mode 100644 index 000000000..adf14a206 --- /dev/null +++ b/examples/prefab/balls/relu/verify/casegen/src/lib.rs @@ -0,0 +1,71 @@ +mod casegen; +mod model; + +use std::cell::RefCell; + +use casegen::{ReluCase, ReluCmd}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn relu_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("relu_case: no case loaded; call relu_case_load first"), + }) +} + +fn idx(index: u32, case: &ReluCase, what: &str) -> usize { + if index >= case.cmd.num_words { + panic!("relu_case_{what}: index {index} out of range {}", case.cmd.num_words); + } + index as usize +} + +#[no_mangle] +pub extern "C" fn relu_case_cmd(out_ptr: *mut ReluCmd) { + if out_ptr.is_null() { + panic!("relu_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +#[no_mangle] +pub extern "C" fn relu_case_row(index: u32) -> u32 { + current_case(|case| case.rows[idx(index, case, "row")]) +} + +#[no_mangle] +pub extern "C" fn relu_case_src_word_lo(index: u32) -> u64 { + current_case(|case| case.src[idx(index, case, "src_word_lo")] as u64) +} + +#[no_mangle] +pub extern "C" fn relu_case_src_word_hi(index: u32) -> u64 { + current_case(|case| (case.src[idx(index, case, "src_word_hi")] >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn relu_case_dst_word_lo(index: u32) -> u64 { + current_case(|case| case.dst[idx(index, case, "dst_word_lo")] as u64) +} + +#[no_mangle] +pub extern "C" fn relu_case_dst_word_hi(index: u32) -> u64 { + current_case(|case| (case.dst[idx(index, case, "dst_word_hi")] >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn relu_bank_entries() -> u32 { + model::bank_entries() +} diff --git a/examples/prefab/balls/relu/verify/casegen/src/model.rs b/examples/prefab/balls/relu/verify/casegen/src/model.rs new file mode 100644 index 000000000..942757bf9 --- /dev/null +++ b/examples/prefab/balls/relu/verify/casegen/src/model.rs @@ -0,0 +1,48 @@ +// ReluBall contract: max(x, 0) on four i32 lanes per row, in place on one +// bank group. The Ball walks segments 0, stride, 2 * stride, ... up to the +// physical bank depth and updates the first `iter` rows of each segment. + +pub fn bank_entries() -> u32 { + let path = std::env::var("BB_VERIFY_CONFIG").expect("BB_VERIFY_CONFIG is required"); + let text = std::fs::read_to_string(path).expect("failed to read BB_VERIFY_CONFIG"); + text.lines() + .find_map(|line| line.strip_prefix("bank_entries=")) + .expect("bank_entries missing from BB_VERIFY_CONFIG") + .parse() + .expect("invalid bank_entries in BB_VERIFY_CONFIG") +} + +pub fn validate(entries: u32, iter: u32, stride: u64, group: u32, cols: u32) { + if iter == 0 || iter % 16 != 0 { + panic!("relu: iter must be a positive multiple of 16"); + } + if stride == 0 || stride > u64::from(entries) || u64::from(entries) % stride != 0 { + panic!("relu: stride must divide the physical bank depth"); + } + if u64::from(iter) > stride { + panic!("relu: iter must not exceed stride"); + } + if cols == 0 || group >= cols { + panic!("relu: group is not allocated in bank"); + } +} + +// Rows in the order the Ball reads and rewrites them. +pub fn rows(entries: u32, iter: u32, stride: u32) -> Vec { + let mut out = Vec::new(); + for segment in (0..entries).step_by(stride as usize) { + for line in 0..iter { + out.push(segment + line); + } + } + out +} + +pub fn relu_word(word: u128) -> u128 { + let mut out = 0u128; + for lane in 0..4 { + let value = (word >> (32 * lane)) as u32 as i32; + out |= u128::from(value.max(0) as u32) << (32 * lane); + } + out +} diff --git a/examples/prefab/balls/relu/verify/filelists/relu_ball.f b/examples/prefab/balls/relu/verify/filelists/relu_ball.f new file mode 100644 index 000000000..f4539fab9 --- /dev/null +++ b/examples/prefab/balls/relu/verify/filelists/relu_ball.f @@ -0,0 +1,25 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+src ++incdir+src/pkg ++incdir+src/common ++incdir+src/seq ++incdir+src/cov ++incdir+src/env ++incdir+src/tests ++incdir+@RTL@ + +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv + +@RTL@/Relu.sv +@RTL@/ReluBall.sv + +src/pkg/relu_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/relu/verify/src/common/relu_defs.svh b/examples/prefab/balls/relu/verify/src/common/relu_defs.svh new file mode 100644 index 000000000..f06395a2d --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/common/relu_defs.svh @@ -0,0 +1,27 @@ +typedef struct { + int unsigned iter; + int unsigned stride; + int unsigned bank; + int unsigned group_base; + int unsigned group; + int unsigned cols; + int unsigned rob_id; + int unsigned num_words; +} relu_cmd_dpi_t; + +import "DPI-C" function int unsigned relu_bank_entries(); +import "DPI-C" function int relu_case_load(input int unsigned index); +import "DPI-C" function void relu_case_cmd(output relu_cmd_dpi_t cmd); +import "DPI-C" function int unsigned relu_case_row(input int unsigned word_index); +import "DPI-C" function longint unsigned relu_case_src_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned relu_case_src_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned relu_case_dst_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned relu_case_dst_word_hi(input int unsigned word_index); + +`ifndef RELU_FUNCT7 +`error "RELU_FUNCT7 must be provided by the selected ballISA" +`endif + +localparam int RELU_THREAD_FUNCT7 = `RELU_FUNCT7; +localparam int RELU_NUM_CASES = 6; +localparam int RELU_TIMEOUT_CYCLES = 100000; diff --git a/examples/prefab/balls/relu/verify/src/common/relu_items.svh b/examples/prefab/balls/relu/verify/src/common/relu_items.svh new file mode 100644 index 000000000..3f5acec87 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/common/relu_items.svh @@ -0,0 +1,64 @@ +class relu_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(relu_cmd_item) + + bit [31:0] num_words; + bit [31:0] stride; + bit [31:0] rows[]; + bit [127:0] src_words[]; + bit [127:0] dst_words[]; + + function new(string name = "relu_cmd_item"); + super.new(name); + endfunction + + function void load_rust_case(int unsigned index); + relu_cmd_dpi_t cmd; + int unsigned rc; + + rc = relu_case_load(index); + if (rc != 0) + `uvm_fatal("CASE", $sformatf("relu_case_load returned %0d for index %0d", rc, index)) + relu_case_cmd(cmd); + + // bb_relu: one bank is both source and destination; rs1 bank1 names the + // group inside it, which arrives in the op2 bank field. rs2 is the stride. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = RELU_THREAD_FUNCT7[6:0]; + rs2 = cmd.stride; + iter = cmd.iter; + op1_bank = cmd.bank; + op2_bank = cmd.group; + wr_bank = cmd.bank; + op1_group = cmd.group_base; + op2_group = '0; + wr_group = cmd.group_base; + op1_col = cmd.cols; + op2_col = '0; + wr_col = cmd.cols; + wr_read_bank = '0; + wr_arch_bank = cmd.bank; + stride = cmd.stride; + num_words = cmd.num_words; + + if (num_words == 0) `uvm_fatal("CASE", "ReluBall case has no rows") + rows = new[num_words]; + src_words = new[num_words]; + dst_words = new[num_words]; + for (int i = 0; i < num_words; i++) begin + rows[i] = relu_case_row(i); + src_words[i] = {relu_case_src_word_hi(i), relu_case_src_word_lo(i)}; + dst_words[i] = {relu_case_dst_word_hi(i), relu_case_dst_word_lo(i)}; + end + endfunction + + function void do_copy(uvm_object rhs); + relu_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not relu_cmd_item") + num_words = rhs_.num_words; + stride = rhs_.stride; + rows = rhs_.rows; + src_words = rhs_.src_words; + dst_words = rhs_.dst_words; + endfunction +endclass diff --git a/examples/prefab/balls/relu/verify/src/cov/relu_cov.svh b/examples/prefab/balls/relu/verify/src/cov/relu_cov.svh new file mode 100644 index 000000000..46400dbf7 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/cov/relu_cov.svh @@ -0,0 +1,47 @@ +class relu_cov extends uvm_component; + `uvm_component_utils(relu_cov) + + uvm_analysis_imp_cmd #(bb_blink_cmd_item, relu_cov) cmd_imp; + + int unsigned bank_entries; + int unsigned cur_segments; + int unsigned cur_group; + int unsigned cur_group_base; + + covergroup cmd_cg; + // One segment, several segments, every row of the bank in segments of 16. + coverpoint cur_segments { + bins one = {1}; bins some = {[2 : 63]}; bins many = {[64 : $]}; + } + coverpoint cur_group {bins first = {0}; bins later = {[1 : $]};} + coverpoint cur_group_base {bins zero = {0}; bins offset = {[1 : $]};} + endgroup + + int cmd_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + cmd_imp = new("cmd_imp", this); + cmd_cg = new(); + bank_entries = relu_bank_entries(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + if (item.funct != RELU_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) + if (item.rs2 == 0) `uvm_fatal("COV", "zero stride") + cur_segments = bank_entries / int'(item.rs2); + cur_group = item.op2_bank; + cur_group_base = item.op1_group; + cmd_count++; + cmd_cg.sample(); + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("relu_cov %0.2f%%", cmd_cg.get_coverage())) + endfunction +endclass diff --git a/examples/prefab/balls/relu/verify/src/env/relu_env.svh b/examples/prefab/balls/relu/verify/src/env/relu_env.svh new file mode 100644 index 000000000..0614f14f4 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/env/relu_env.svh @@ -0,0 +1,27 @@ +class relu_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(relu_env) + + relu_scoreboard scb; + relu_cov icov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scb = relu_scoreboard::type_id::create("scb", this); + icov = relu_cov::type_id::create("icov", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + scb.mem_model = mem_model; + cmd_agent.stim_ap.connect(scb.stim_imp); + cmd_agent.cmd_ap.connect(scb.cmd_imp); + cmd_agent.cmd_ap.connect(icov.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction +endclass diff --git a/examples/prefab/balls/relu/verify/src/env/relu_scoreboard.svh b/examples/prefab/balls/relu/verify/src/env/relu_scoreboard.svh new file mode 100644 index 000000000..a6699d4c2 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/env/relu_scoreboard.svh @@ -0,0 +1,157 @@ +class relu_scoreboard extends uvm_scoreboard; + `uvm_component_utils(relu_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, relu_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, relu_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, relu_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, relu_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, relu_scoreboard) resp_imp; + + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + + relu_cmd_item stim_q[$]; + int unsigned cmd_count; + int unsigned read_count; + int unsigned write_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + relu_cmd_item got; + relu_cmd_item clone; + + if (stim_q.size() != 0) `uvm_fatal("SCB", "single outstanding command supported") + if (!$cast(got, item)) `uvm_fatal("SCB", "stim item is not relu_cmd_item") + if (!$cast(clone, got.clone())) `uvm_fatal("SCB", "failed to clone stimulus item") + stim_q.push_back(clone); + + if (mem_model == null) `uvm_fatal("SCB", "mem_model handle not set") + mem_model.clear_mem(); + // Each row is read exactly once before it is rewritten. + for (int i = 0; i < clone.num_words; i++) + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group + clone.op2_bank), + clone.rows[i], clone.src_words[i]); + mem_model.arm(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + bb_blink_cmd_item want; + relu_cmd_item stim; + stim = current_stim("CMD"); + // Compare as plain BLink commands; the comparer rejects differing types. + want = bb_blink_cmd_item::type_id::create("want"); + want.copy(stim); + verification_pkg::evidence::checks++; + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) + cmd_count++; + endfunction + + function void write_read(bb_blink_read_item item); + relu_cmd_item stim; + int unsigned group; + + stim = current_stim("READ"); + group = stim.op1_group + stim.op2_bank; + if (item.port != 0) + `uvm_fatal("READ", $sformatf("ReluBall has one read port, got %0d", item.port)) + if (read_count >= stim.num_words) `uvm_fatal("READ", $sformatf("extra read %0d", read_count)) + if (read_count != write_count) + `uvm_fatal("READ", $sformatf("read %0d issued before write %0d", read_count, write_count)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.bank_id !== stim.op1_bank) + `uvm_fatal("READ", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.op1_bank)) + if (item.group_id !== group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf("group mismatch: got %0d exp %0d", item.group_id, group)) + if (item.addr !== stim.rows[read_count][BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal( + "READ", $sformatf( + "addr mismatch at %0d: got %0d exp %0d", read_count, item.addr, stim.rows[read_count])) + read_count++; + endfunction + + function void write_write(bb_blink_write_item item); + relu_cmd_item stim; + int unsigned group; + + stim = current_stim("WRITE"); + group = stim.op1_group + stim.op2_bank; + if (item.port != 0) + `uvm_fatal("WRITE", $sformatf("ReluBall has one write port, got %0d", item.port)) + if (write_count >= read_count) + `uvm_fatal("WRITE", $sformatf("write %0d before its read", write_count)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.group_id !== group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("WRITE", $sformatf("group mismatch: got %0d exp %0d", item.group_id, group)) + if (item.mask !== 16'hFFFF) + `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) + if (item.addr !== stim.rows[write_count][BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal( + "WRITE", $sformatf( + "addr mismatch at %0d: got %0d exp %0d", write_count, item.addr, stim.rows[write_count])) + verification_pkg::evidence::checks++; + if (item.data !== stim.dst_words[write_count]) + `uvm_fatal("SCB", $sformatf( + "data mismatch at row %0d: got 0x%032h exp 0x%032h", + item.addr, + item.data, + stim.dst_words[write_count] + )) + write_count++; + endfunction + + function void write_resp(bb_blink_resp_item item); + relu_cmd_item stim; + stim = current_stim("RESP"); + if (write_count != stim.num_words) + `uvm_fatal("RESP", $sformatf("response after %0d of %0d writes", write_count, stim.num_words)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + resp_count++; + endfunction + + function relu_cmd_item current_stim(string tag); + if (stim_q.size() == 0) `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + return stim_q[0]; + endfunction + + function bit done(); + if (stim_q.size() == 0) return 1'b0; + return cmd_count == 1 && read_count == stim_q[0].num_words && + write_count == stim_q[0].num_words && resp_count == 1; + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + read_count = 0; + write_count = 0; + resp_count = 0; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) + `uvm_fatal("SCB", $sformatf( + "incomplete: cmds=%0d reads=%0d writes=%0d resp=%0d", + cmd_count, + read_count, + write_count, + resp_count + )) + stim_q.delete(); + endfunction +endclass diff --git a/examples/prefab/balls/relu/verify/src/pkg/relu_pkg.sv b/examples/prefab/balls/relu/verify/src/pkg/relu_pkg.sv new file mode 100644 index 000000000..62c87a849 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/pkg/relu_pkg.sv @@ -0,0 +1,13 @@ +package relu_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + + `include "common/relu_defs.svh" + `include "common/relu_items.svh" + `include "seq/relu_sequences.svh" + `include "cov/relu_cov.svh" + `include "env/relu_scoreboard.svh" + `include "env/relu_env.svh" + `include "tests/relu_test.svh" +endpackage diff --git a/examples/prefab/balls/relu/verify/src/seq/relu_sequences.svh b/examples/prefab/balls/relu/verify/src/seq/relu_sequences.svh new file mode 100644 index 000000000..c90a4ccf8 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/seq/relu_sequences.svh @@ -0,0 +1,17 @@ +class relu_basic_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(relu_basic_seq) + + int unsigned case_index; + + function new(string name = "relu_basic_seq"); + super.new(name); + endfunction + + task body(); + relu_cmd_item item; + item = relu_cmd_item::type_id::create("item"); + start_item(item); + item.load_rust_case(case_index); + finish_item(item); + endtask +endclass diff --git a/examples/prefab/balls/relu/verify/src/tb_top.sv b/examples/prefab/balls/relu/verify/src/tb_top.sv new file mode 100644 index 000000000..b4f7cc075 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/tb_top.sv @@ -0,0 +1,80 @@ +module tb_top; + import uvm_pkg::*; + import relu_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + ReluBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("relu_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/relu/verify/src/tests/relu_test.svh b/examples/prefab/balls/relu/verify/src/tests/relu_test.svh new file mode 100644 index 000000000..144e7a848 --- /dev/null +++ b/examples/prefab/balls/relu/verify/src/tests/relu_test.svh @@ -0,0 +1,48 @@ +class relu_ball_test extends uvm_test; + `uvm_component_utils(relu_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + relu_env env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + env = relu_env::type_id::create("env", this); + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + for (int unsigned index = 0; index < RELU_NUM_CASES; index++) run_case(index); + phase.drop_objection(this); + endtask + + task run_case(int unsigned index); + relu_basic_seq seq; + int cycles = 0; + apply_reset(); + env.scb.reset_counters(); + seq = relu_basic_seq::type_id::create("seq"); + seq.case_index = index; + seq.start(env.cmd_agent.seqr); + while (!env.scb.done()) begin + @(posedge vif.clock); + cycles++; + if (cycles > RELU_TIMEOUT_CYCLES) + `uvm_fatal("TIMEOUT", $sformatf("ReluBall case %0d timeout", index)) + end + `uvm_info("RELU", $sformatf("ReluBall case %0d passed in %0d cycles", index, cycles), UVM_LOW) + endtask + + task apply_reset(); + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/components/balls/relu/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/relu/workloads/ctests/CMakeLists.txt similarity index 100% rename from examples/components/balls/relu/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/relu/workloads/ctests/CMakeLists.txt diff --git a/examples/components/balls/relu/workloads/ctests/relu_i32_16x16_test.c b/examples/prefab/balls/relu/workloads/ctests/relu_i32_16x16_test.c similarity index 100% rename from examples/components/balls/relu/workloads/ctests/relu_i32_16x16_test.c rename to examples/prefab/balls/relu/workloads/ctests/relu_i32_16x16_test.c diff --git a/examples/components/balls/relu/workloads/isa/relu.h b/examples/prefab/balls/relu/workloads/isa/relu.h similarity index 88% rename from examples/components/balls/relu/workloads/isa/relu.h rename to examples/prefab/balls/relu/workloads/isa/relu.h index 3cbf226eb..e0c6efe9b 100644 --- a/examples/components/balls/relu/workloads/isa/relu.h +++ b/examples/prefab/balls/relu/workloads/isa/relu.h @@ -8,6 +8,6 @@ BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(bank_id) | BB_BANK1(group) | \ BB_BANK2(bank_id) | BB_ITER(iter) | \ BB_UPDATE_WR), \ - (stride), BB_FUNC7(RELU)) + (stride), FUNCT7(relu, RELU)) #endif // _BB_RELU_H_ diff --git a/examples/components/balls/mxmm/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/relu/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/mxmm/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/relu/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/relu/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/relu/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/components/balls/relu/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/relu/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir b/examples/prefab/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir similarity index 93% rename from examples/components/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir rename to examples/prefab/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir index 8626513d0..22b913271 100644 --- a/examples/components/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir +++ b/examples/prefab/balls/relu/workloads/mlir_tests/ball/relu_i32_16row.mlir @@ -21,7 +21,7 @@ func.func @main() -> i8 { } buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mvin %input <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> - buckyball.custom "relu_i32" <{sources = array, destination = 1 : i64}> {rows = 16 : i64} + relu_ball.relu_i32 <{bank = 1 : i64, rows = 16 : i64}> buckyball.mvout %output <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> func.call @check_result(%output) : (memref<16x4xi32>) -> () diff --git a/examples/components/balls/relu/workloads/mlir_tests/ball/relu_i32_16row_main.cpp b/examples/prefab/balls/relu/workloads/mlir_tests/ball/relu_i32_16row_main.cpp similarity index 100% rename from examples/components/balls/relu/workloads/mlir_tests/ball/relu_i32_16row_main.cpp rename to examples/prefab/balls/relu/workloads/mlir_tests/ball/relu_i32_16row_main.cpp diff --git a/examples/components/balls/relu/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/relu/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/components/balls/relu/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/relu/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in b/examples/prefab/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in similarity index 94% rename from examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in rename to examples/prefab/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in index a37f6f0fc..3edb6a75d 100644 --- a/examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in +++ b/examples/prefab/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign.mlir.in @@ -28,8 +28,8 @@ func.func @main() -> i8 { %bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded = buckyball.bank_mvin %input, %bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> - %result = buckyball.bank_custom "relu_i32" (%loaded) -> %loaded - {rows = @BB_BANK_LINES@ : i64} + %result = relu_ball.bank_relu_i32 %loaded + <{rows = @BB_BANK_LINES@ : i64}> %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I32_LANES@xi32> buckyball.bank_release %stored diff --git a/examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp b/examples/prefab/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp similarity index 68% rename from examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp rename to examples/prefab/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp index 02bea257c..9bb53f9f1 100644 --- a/examples/components/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp +++ b/examples/prefab/balls/relu/workloads/mlir_tests/bank/relu_i32_mixed_sign_main.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -21,17 +20,17 @@ extern "C" int64_t size0, int64_t size1, int64_t stride0, int64_t stride1) { (void)allocated; - if (size0 != BANK_LINES || size1 != BANK_WIDTH / 32 || - stride0 != BANK_WIDTH / 32 || stride1 != 1) { + if (size0 != THREAD_MEM_BANK_ENTRIES || size1 != THREAD_MEM_BANK_WIDTH / 32 || + stride0 != THREAD_MEM_BANK_WIDTH / 32 || stride1 != 1) { printf("FAILED: relu bank shape %dx%d stride %dx%d\n", (int)size0, (int)size1, (int)stride0, (int)stride1); fail(); } int32_t *output = aligned + offset; - for (int line = 0; line < BANK_LINES; ++line) { - for (int lane = 0; lane < BANK_WIDTH / 32; ++lane) { - int32_t input = (line * (BANK_WIDTH / 32) + lane) % 17 - 8; + for (int line = 0; line < THREAD_MEM_BANK_ENTRIES; ++line) { + for (int lane = 0; lane < THREAD_MEM_BANK_WIDTH / 32; ++lane) { + int32_t input = (line * (THREAD_MEM_BANK_WIDTH / 32) + lane) % 17 - 8; int32_t expected = input < 0 ? 0 : input; int32_t actual = output[line * stride0 + lane * stride1]; if (actual != expected) { @@ -41,5 +40,5 @@ extern "C" } } } - printf("relu i32 mixed-sign PASS rows=%d\n", BANK_LINES); + printf("relu i32 mixed-sign PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); } diff --git a/examples/components/balls/smatmul/README.md b/examples/prefab/balls/smatmul/README.md similarity index 100% rename from examples/components/balls/smatmul/README.md rename to examples/prefab/balls/smatmul/README.md diff --git a/examples/components/balls/smatmul/arch/Array.scala b/examples/prefab/balls/smatmul/arch/Array.scala similarity index 100% rename from examples/components/balls/smatmul/arch/Array.scala rename to examples/prefab/balls/smatmul/arch/Array.scala diff --git a/examples/components/balls/smatmul/arch/PE.scala b/examples/prefab/balls/smatmul/arch/PE.scala similarity index 100% rename from examples/components/balls/smatmul/arch/PE.scala rename to examples/prefab/balls/smatmul/arch/PE.scala diff --git a/examples/components/balls/smatmul/arch/SMatMulBall.scala b/examples/prefab/balls/smatmul/arch/SMatMulBall.scala similarity index 100% rename from examples/components/balls/smatmul/arch/SMatMulBall.scala rename to examples/prefab/balls/smatmul/arch/SMatMulBall.scala diff --git a/examples/components/balls/smatmul/arch/SMatMulUnit.scala b/examples/prefab/balls/smatmul/arch/SMatMulUnit.scala similarity index 100% rename from examples/components/balls/smatmul/arch/SMatMulUnit.scala rename to examples/prefab/balls/smatmul/arch/SMatMulUnit.scala diff --git a/examples/components/balls/smatmul/configs/SMatMulBallParam.scala b/examples/prefab/balls/smatmul/configs/SMatMulBallParam.scala similarity index 100% rename from examples/components/balls/smatmul/configs/SMatMulBallParam.scala rename to examples/prefab/balls/smatmul/configs/SMatMulBallParam.scala diff --git a/examples/components/balls/smatmul/configs/default.toml b/examples/prefab/balls/smatmul/configs/default.toml similarity index 100% rename from examples/components/balls/smatmul/configs/default.toml rename to examples/prefab/balls/smatmul/configs/default.toml diff --git a/examples/components/balls/smatmul/configs/lane16.toml b/examples/prefab/balls/smatmul/configs/lane16.toml similarity index 100% rename from examples/components/balls/smatmul/configs/lane16.toml rename to examples/prefab/balls/smatmul/configs/lane16.toml diff --git a/examples/components/balls/smatmul/configs/smatmul_params.proto b/examples/prefab/balls/smatmul/configs/smatmul_params.proto similarity index 100% rename from examples/components/balls/smatmul/configs/smatmul_params.proto rename to examples/prefab/balls/smatmul/configs/smatmul_params.proto diff --git a/examples/components/balls/smatmul/emu/src/dispatch.rs b/examples/prefab/balls/smatmul/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/smatmul/emu/src/dispatch.rs rename to examples/prefab/balls/smatmul/emu/src/dispatch.rs diff --git a/examples/components/balls/smatmul/emu/src/fp32.rs b/examples/prefab/balls/smatmul/emu/src/fp32.rs similarity index 95% rename from examples/components/balls/smatmul/emu/src/fp32.rs rename to examples/prefab/balls/smatmul/emu/src/fp32.rs index 354b010d4..b731ba17b 100644 --- a/examples/components/balls/smatmul/emu/src/fp32.rs +++ b/examples/prefab/balls/smatmul/emu/src/fp32.rs @@ -1,5 +1,5 @@ use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::ExecContext; use std::collections::HashMap; @@ -13,6 +13,7 @@ struct Chain { pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "smatmul_f32: reads no old destination; rs1[29] must be clear"); let ball_state = ctx.state::>(); let rows = (xs2 & 0xfff) as usize; let cols = ((xs2 >> 12) & 0xfff) as usize; diff --git a/examples/components/balls/smatmul/emu/src/lib.rs b/examples/prefab/balls/smatmul/emu/src/lib.rs similarity index 100% rename from examples/components/balls/smatmul/emu/src/lib.rs rename to examples/prefab/balls/smatmul/emu/src/lib.rs diff --git a/examples/components/balls/smatmul/emu/src/mxfp8.rs b/examples/prefab/balls/smatmul/emu/src/mxfp8.rs similarity index 97% rename from examples/components/balls/smatmul/emu/src/mxfp8.rs rename to examples/prefab/balls/smatmul/emu/src/mxfp8.rs index 8f39fed1e..1cc211375 100644 --- a/examples/components/balls/smatmul/emu/src/mxfp8.rs +++ b/examples/prefab/balls/smatmul/emu/src/mxfp8.rs @@ -1,5 +1,5 @@ use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; -use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::ExecContext; use std::collections::HashMap; @@ -24,6 +24,7 @@ fn value(code: u8, factor: f32) -> f32 { } pub(crate) fn execute(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "smatmul_mxfp8: reads no old destination; rs1[29] must be clear"); let ball_state = ctx.state::>(); let rows = (xs2 & 0xfff) as usize; let cols = ((xs2 >> 12) & 0xfff) as usize; diff --git a/examples/components/balls/smatmul/emu/src/smatmul.rs b/examples/prefab/balls/smatmul/emu/src/smatmul.rs similarity index 97% rename from examples/components/balls/smatmul/emu/src/smatmul.rs rename to examples/prefab/balls/smatmul/emu/src/smatmul.rs index 1decf2c74..34101a48b 100644 --- a/examples/components/balls/smatmul/emu/src/smatmul.rs +++ b/examples/prefab/balls/smatmul/emu/src/smatmul.rs @@ -1,4 +1,4 @@ -use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::ExecContext; use bebop_bemu::thread::banks::{bank_lines, bank_row_bytes}; use std::collections::HashMap; @@ -38,6 +38,7 @@ fn read_i32(bank: &[u8], row: usize, lane: usize) -> i32 { } pub(crate) fn exec_bias(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "smatmul_bias: reads no old destination; rs1[29] must be clear"); let ball_state = ctx.state::>(); let bank = rs1_b0(xs1); let input_base = (xs2 & 0x3f) as usize; @@ -69,6 +70,7 @@ pub(crate) fn exec_bias(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { } pub(crate) fn exec_smatmul(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "smatmul: reads no old destination; rs1[29] must be clear"); let ball_state = ctx.state::>(); let m_tile = tile_rows(ctx.ball_id.expect("Ball instance")); let k_tile = tile_cols(ctx.ball_id.expect("Ball instance")); diff --git a/examples/prefab/balls/smatmul/stack/src/smatmulOps.cpp b/examples/prefab/balls/smatmul/stack/src/smatmulOps.cpp new file mode 100644 index 000000000..c580ee03f --- /dev/null +++ b/examples/prefab/balls/smatmul/stack/src/smatmulOps.cpp @@ -0,0 +1,80 @@ +//===- smatmulOps.cpp - SMatMulBall operations ----------------------------===// +// +// The dialect of SMatMulBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "smatmulOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::smatmul_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "smatmulDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "smatmulOps.cpp.inc" + +void SMatMulBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "smatmulOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankMatmulI8Op::getSourceBanks() { + return {getA(), getB()}; +} +Value BankMatmulI8Op::getDestinationBank() { return getC(); } +bool BankMatmulI8Op::readsDestination() { return true; } +Operation *BankMatmulI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulI8Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getK()); +} + +SmallVector MatmulI8Op::getSourceBanks() { + return {int64_t(getA()), int64_t(getB())}; +} +int64_t MatmulI8Op::getDestinationBank() { return getC(); } +LogicalResult MatmulI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), k = getK(), a = getA(), b = getB(), c = getC(); + const auto &target = buckyball_target::getBuckyballTarget(); + int64_t tileRows = + buckyball_target::getBuckyballBallParam("SMATMUL_OS", "tileRows"); + if (target.bankWidthBits != 128 || rows <= 0 || rows * 4 > target.bankDepth || + (rows != 1 && rows % tileRows) || k <= 0 || k > target.bankDepth || + k % 16 || rows * (k / 16) > target.bankDepth) + return emitOpError("INT8 matmul shape exceeds the bank or ISA geometry"); + if (a == b || a == c || b == c) + return emitOpError("INT8 matmul requires distinct A, B and C banks"); + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7("SMATMUL_BIAS"), + constantI64(builder, loc, bankFields({c}, 0, false, 4)), + constantI64(builder, loc, 0)); + // One accumulation over every group: first and last in one instruction. + uint64_t rs2 = field(rows, 0, 11) | field(16, 12, 23) | field(1, 24, 24) | + field(1, 25, 25); + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7("SMATMUL_OS"), + constantI64(builder, loc, bankFields({a, b}, c, false, k)), + constantI64(builder, loc, rs2)); + return success(); +} diff --git a/examples/prefab/balls/smatmul/stack/src/smatmulOps.h b/examples/prefab/balls/smatmul/stack/src/smatmulOps.h new file mode 100644 index 000000000..05a631588 --- /dev/null +++ b/examples/prefab/balls/smatmul/stack/src/smatmulOps.h @@ -0,0 +1,20 @@ +//===- smatmulOps.h - SMatMulBall operations ------------------------------===// + +#ifndef SMATMUL_OPS_H +#define SMATMUL_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "smatmulDialect.h.inc" + +#define GET_OP_CLASSES +#include "smatmulOps.h.inc" + +#endif // SMATMUL_OPS_H diff --git a/examples/prefab/balls/smatmul/stack/src/smatmulOps.td b/examples/prefab/balls/smatmul/stack/src/smatmulOps.td new file mode 100644 index 000000000..d457e9931 --- /dev/null +++ b/examples/prefab/balls/smatmul/stack/src/smatmulOps.td @@ -0,0 +1,49 @@ +//===- smatmulOps.td - SMatMulBall operations --------------*- tablegen -*-===// +// +// matmul_i8 (a, b) -> c: c = bias + a @ b for `rows` rows (one, or a multiple +// of tileRows up to a quarter of the bank depth) and sixteen columns; it always +// reads the bias in the old c. A and B have the same number of groups; group g +// holds reduction block g of `k` columns (a multiple of sixteen), so +// K = k * groups. In group g, B holds k rows of sixteen i8. A holds k / 16 rows +// of sixteen i8 for rows = 1; otherwise M tiles of tileRows rows, each tile the +// k / 16 slices of sixteen columns one after another: line +// ((r / tileRows) * (k / 16) + c / 16) * tileRows + r % tileRows. +// C starts with the bias, one row of sixteen i32 in its first four lines, and +// receives `rows` rows of sixteen i32 (four lines each). +// +//===----------------------------------------------------------------------===// + +#ifndef SMATMUL_OPS +#define SMATMUL_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def SMatMulBall_Dialect : Dialect { + let name = "smatmul_ball"; + let cppNamespace = "::buddy::smatmul_ball"; + let summary = "The operations of SMatMulBall"; +} + +class SMatMulBall_Op traits = []> + : Op])>; + +def BankMatmulI8Op : SMatMulBall_Op<"bank_matmul_i8", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_i8 on virtual banks"; + let arguments = (ins I64:$a, I64:$b, I64:$c, I64Attr:$rows, I64Attr:$k); + let results = (outs I64:$cOut); + let assemblyFormat = "$a `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulI8Op : SMatMulBall_Op<"matmul_i8", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_i8 on physical banks"; + let arguments = (ins I64Attr:$a, I64Attr:$b, I64Attr:$c, I64Attr:$rows, + I64Attr:$k); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // SMATMUL_OPS diff --git a/examples/components/balls/smatmul/verify/README.md b/examples/prefab/balls/smatmul/verify/README.md similarity index 100% rename from examples/components/balls/smatmul/verify/README.md rename to examples/prefab/balls/smatmul/verify/README.md diff --git a/examples/components/balls/smatmul/verify/casegen/Cargo.lock b/examples/prefab/balls/smatmul/verify/casegen/Cargo.lock similarity index 100% rename from examples/components/balls/smatmul/verify/casegen/Cargo.lock rename to examples/prefab/balls/smatmul/verify/casegen/Cargo.lock diff --git a/examples/components/balls/smatmul/verify/casegen/Cargo.toml b/examples/prefab/balls/smatmul/verify/casegen/Cargo.toml similarity index 100% rename from examples/components/balls/smatmul/verify/casegen/Cargo.toml rename to examples/prefab/balls/smatmul/verify/casegen/Cargo.toml diff --git a/examples/components/balls/smatmul/verify/casegen/src/casegen.rs b/examples/prefab/balls/smatmul/verify/casegen/src/casegen.rs similarity index 100% rename from examples/components/balls/smatmul/verify/casegen/src/casegen.rs rename to examples/prefab/balls/smatmul/verify/casegen/src/casegen.rs diff --git a/examples/components/balls/smatmul/verify/casegen/src/lib.rs b/examples/prefab/balls/smatmul/verify/casegen/src/lib.rs similarity index 100% rename from examples/components/balls/smatmul/verify/casegen/src/lib.rs rename to examples/prefab/balls/smatmul/verify/casegen/src/lib.rs diff --git a/examples/components/balls/smatmul/verify/casegen/src/model.rs b/examples/prefab/balls/smatmul/verify/casegen/src/model.rs similarity index 100% rename from examples/components/balls/smatmul/verify/casegen/src/model.rs rename to examples/prefab/balls/smatmul/verify/casegen/src/model.rs diff --git a/examples/components/balls/smatmul/verify/filelists/smatmul_ball.f b/examples/prefab/balls/smatmul/verify/filelists/smatmul_ball.f similarity index 100% rename from examples/components/balls/smatmul/verify/filelists/smatmul_ball.f rename to examples/prefab/balls/smatmul/verify/filelists/smatmul_ball.f diff --git a/examples/components/balls/smatmul/verify/parameters.toml b/examples/prefab/balls/smatmul/verify/parameters.toml similarity index 100% rename from examples/components/balls/smatmul/verify/parameters.toml rename to examples/prefab/balls/smatmul/verify/parameters.toml diff --git a/examples/components/balls/smatmul/verify/src/common/smatmul_defs.svh b/examples/prefab/balls/smatmul/verify/src/common/smatmul_defs.svh similarity index 100% rename from examples/components/balls/smatmul/verify/src/common/smatmul_defs.svh rename to examples/prefab/balls/smatmul/verify/src/common/smatmul_defs.svh diff --git a/examples/components/balls/smatmul/verify/src/common/smatmul_items.svh b/examples/prefab/balls/smatmul/verify/src/common/smatmul_items.svh similarity index 100% rename from examples/components/balls/smatmul/verify/src/common/smatmul_items.svh rename to examples/prefab/balls/smatmul/verify/src/common/smatmul_items.svh diff --git a/examples/components/balls/smatmul/verify/src/cov/smatmul_cov.svh b/examples/prefab/balls/smatmul/verify/src/cov/smatmul_cov.svh similarity index 100% rename from examples/components/balls/smatmul/verify/src/cov/smatmul_cov.svh rename to examples/prefab/balls/smatmul/verify/src/cov/smatmul_cov.svh diff --git a/examples/components/balls/smatmul/verify/src/env/smatmul_env.svh b/examples/prefab/balls/smatmul/verify/src/env/smatmul_env.svh similarity index 100% rename from examples/components/balls/smatmul/verify/src/env/smatmul_env.svh rename to examples/prefab/balls/smatmul/verify/src/env/smatmul_env.svh diff --git a/examples/components/balls/smatmul/verify/src/env/smatmul_scoreboard.svh b/examples/prefab/balls/smatmul/verify/src/env/smatmul_scoreboard.svh similarity index 100% rename from examples/components/balls/smatmul/verify/src/env/smatmul_scoreboard.svh rename to examples/prefab/balls/smatmul/verify/src/env/smatmul_scoreboard.svh diff --git a/examples/components/balls/smatmul/verify/src/pkg/smatmul_pkg.sv b/examples/prefab/balls/smatmul/verify/src/pkg/smatmul_pkg.sv similarity index 100% rename from examples/components/balls/smatmul/verify/src/pkg/smatmul_pkg.sv rename to examples/prefab/balls/smatmul/verify/src/pkg/smatmul_pkg.sv diff --git a/examples/components/balls/smatmul/verify/src/seq/smatmul_sequences.svh b/examples/prefab/balls/smatmul/verify/src/seq/smatmul_sequences.svh similarity index 100% rename from examples/components/balls/smatmul/verify/src/seq/smatmul_sequences.svh rename to examples/prefab/balls/smatmul/verify/src/seq/smatmul_sequences.svh diff --git a/examples/components/balls/smatmul/verify/src/tb_top.sv b/examples/prefab/balls/smatmul/verify/src/tb_top.sv similarity index 100% rename from examples/components/balls/smatmul/verify/src/tb_top.sv rename to examples/prefab/balls/smatmul/verify/src/tb_top.sv diff --git a/examples/components/balls/smatmul/verify/src/tests/smatmul_test.svh b/examples/prefab/balls/smatmul/verify/src/tests/smatmul_test.svh similarity index 100% rename from examples/components/balls/smatmul/verify/src/tests/smatmul_test.svh rename to examples/prefab/balls/smatmul/verify/src/tests/smatmul_test.svh diff --git a/examples/components/balls/smatmul/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/smatmul/workloads/ctests/CMakeLists.txt similarity index 74% rename from examples/components/balls/smatmul/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/smatmul/workloads/ctests/CMakeLists.txt index 356a7c2c9..c1b5d370b 100644 --- a/examples/components/balls/smatmul/workloads/ctests/CMakeLists.txt +++ b/examples/prefab/balls/smatmul/workloads/ctests/CMakeLists.txt @@ -6,11 +6,9 @@ add_buckyball_ctest(smatmul_16x16x32_test.c) add_buckyball_ctest(smatmul_nondefault_banks_test.c) add_buckyball_ctest(smatmul_k_groups_test.c) add_buckyball_ctest(smatmul_c_groups_test.c) -file( - STRINGS "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/ballISA.h" - F32_ISA - REGEX "^#define BB_FUNC7_SMATMUL_F32 " +if( + DEFINED + DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_BALLDOMAIN_BALLS_smatmul_INSTRUCTIONS_SMATMUL_F32_FUNCT7 ) -if(F32_ISA) add_buckyball_ctest(smatmul_f32_test.c) endif() diff --git a/examples/components/balls/smatmul/workloads/ctests/smatmul_16x16_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_16x16_test.c similarity index 100% rename from examples/components/balls/smatmul/workloads/ctests/smatmul_16x16_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_16x16_test.c diff --git a/examples/components/balls/smatmul/workloads/ctests/smatmul_16x16x32_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_16x16x32_test.c similarity index 100% rename from examples/components/balls/smatmul/workloads/ctests/smatmul_16x16x32_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_16x16x32_test.c diff --git a/examples/components/balls/smatmul/workloads/ctests/smatmul_1x16x64_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_1x16x64_test.c similarity index 100% rename from examples/components/balls/smatmul/workloads/ctests/smatmul_1x16x64_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_1x16x64_test.c diff --git a/examples/components/balls/smatmul/workloads/ctests/smatmul_8x16_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_8x16_test.c similarity index 100% rename from examples/components/balls/smatmul/workloads/ctests/smatmul_8x16_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_8x16_test.c diff --git a/examples/components/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c similarity index 100% rename from examples/components/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_bias_accumulate_test.c diff --git a/examples/components/balls/smatmul/workloads/ctests/smatmul_c_groups_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_c_groups_test.c similarity index 76% rename from examples/components/balls/smatmul/workloads/ctests/smatmul_c_groups_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_c_groups_test.c index 657e899f4..6e16cb27d 100644 --- a/examples/components/balls/smatmul/workloads/ctests/smatmul_c_groups_test.c +++ b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_c_groups_test.c @@ -1,18 +1,17 @@ #include "buckyball.h" #include #include -#include #include #include /* M=128, N=16, K=32 with output_base 32: C occupies lines 32..543, so its rows - * continue from group 0 into group 1 (line L is group L / BANK_LINES). A and B - * have two reduction groups. */ + * continue from group 0 into group 1 (line L is group L / + * THREAD_MEM_BANK_ENTRIES). A and B have two reduction groups. */ enum { M = 128, N = 16, K = 32, BASE = 32 }; static int8_t a[M * K] __attribute__((aligned(64))); static int8_t b[K * N] __attribute__((aligned(64))); static int32_t bias[N] __attribute__((aligned(64))); -static int32_t c[2][BANK_LINES * 4] __attribute__((aligned(64))); +static int32_t c[2][THREAD_MEM_BANK_ENTRIES * 4] __attribute__((aligned(64))); int main(void) { for (int row = 0; row < M; ++row) @@ -34,8 +33,8 @@ int main(void) { bb_mvin_group((uintptr_t)(b + 16 * N), 2, 1, 16, 1); bb_smatmul_bias(0, 0); bb_smatmul_os(1, 2, 3, M, N, 16, 1, 1, BASE); - bb_mvout_group((uintptr_t)c[0], 3, 0, BANK_LINES, 1); - bb_mvout_group((uintptr_t)c[1], 3, 1, BANK_LINES, 1); + bb_mvout_group((uintptr_t)c[0], 3, 0, THREAD_MEM_BANK_ENTRIES, 1); + bb_mvout_group((uintptr_t)c[1], 3, 1, THREAD_MEM_BANK_ENTRIES, 1); for (int row = 0; row < M; ++row) for (int col = 0; col < N; ++col) { @@ -43,7 +42,8 @@ int main(void) { for (int k = 0; k < K; ++k) expected += a[row * K + k] * b[k * N + col]; int line = BASE + row * 4 + col / 4; - int actual = c[line / BANK_LINES][line % BANK_LINES * 4 + col % 4]; + int actual = c[line / THREAD_MEM_BANK_ENTRIES] + [line % THREAD_MEM_BANK_ENTRIES * 4 + col % 4]; if (actual != expected) { printf("smatmul_c_groups FAIL row=%d col=%d expected=%d actual=%d\n", row, col, expected, actual); diff --git a/examples/components/balls/smatmul/workloads/ctests/smatmul_f32_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_f32_test.c similarity index 100% rename from examples/components/balls/smatmul/workloads/ctests/smatmul_f32_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_f32_test.c diff --git a/examples/components/balls/smatmul/workloads/ctests/smatmul_k_groups_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_k_groups_test.c similarity index 100% rename from examples/components/balls/smatmul/workloads/ctests/smatmul_k_groups_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_k_groups_test.c diff --git a/examples/components/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c similarity index 90% rename from examples/components/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c rename to examples/prefab/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c index 566f3c54f..3b6dcf41c 100644 --- a/examples/components/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c +++ b/examples/prefab/balls/smatmul/workloads/ctests/smatmul_nondefault_banks_test.c @@ -2,16 +2,15 @@ #include #include #include -#include #include #include enum { M = 16, N = 16, K = 16 }; enum { - BIAS_BANK = VIRTUAL_BANK_NUM - 4, - A_BANK = VIRTUAL_BANK_NUM - 3, - B_BANK = VIRTUAL_BANK_NUM - 2, - C_BANK = VIRTUAL_BANK_NUM - 1 + BIAS_BANK = CORE_VIRTUAL_BANK_COUNT - 4, + A_BANK = CORE_VIRTUAL_BANK_COUNT - 3, + B_BANK = CORE_VIRTUAL_BANK_COUNT - 2, + C_BANK = CORE_VIRTUAL_BANK_COUNT - 1 }; static int8_t input[M * K] __attribute__((aligned(64))); static int8_t weight[K * N] __attribute__((aligned(64))); diff --git a/examples/components/balls/smatmul/workloads/isa/smatmul.h b/examples/prefab/balls/smatmul/workloads/isa/smatmul.h similarity index 86% rename from examples/components/balls/smatmul/workloads/isa/smatmul.h rename to examples/prefab/balls/smatmul/workloads/isa/smatmul.h index 621a7af38..e4fd2d09f 100644 --- a/examples/components/balls/smatmul/workloads/isa/smatmul.h +++ b/examples/prefab/balls/smatmul/workloads/isa/smatmul.h @@ -8,21 +8,23 @@ // block g, so the instruction accumulates over K = k * groups with k per group // (A[:, g*k : (g+1)*k] and B[:, g*k : (g+1)*k] in the single-group layouts). // C rows continue across groups: line L = output_base + row * 4 + word lives -// in group L / BANK_LINES, line L % BANK_LINES. The bias bank has one group. +// in group L / THREAD_MEM_BANK_ENTRIES, line L % THREAD_MEM_BANK_ENTRIES. The +// bias bank has one group. #define BB_SMATMUL_CFG(rows, cols, first, last, output_base) \ (FIELD((rows), 0, 11) | FIELD((cols), 12, 23) | FIELD((first), 24, 24) | \ FIELD((last), 25, 25) | FIELD((output_base), 26, 31)) #define bb_smatmul_bias(bias_bank, input_base) \ BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(bias_bank) | BB_ITER(4), \ - FIELD((input_base), 0, 5), BB_FUNC7(SMATMUL_BIAS)) + FIELD((input_base), 0, 5), \ + FUNCT7(smatmul, SMATMUL_BIAS)) #define bb_smatmul_os(a_bank, b_bank, c_bank, rows, cols, k, first, last, \ output_base) \ BUCKYBALL_INSTRUCTION_R_R( \ BB_BANK0(a_bank) | BB_BANK1(b_bank) | BB_BANK2(c_bank) | BB_ITER(k), \ BB_SMATMUL_CFG(rows, cols, first, last, output_base), \ - BB_FUNC7(SMATMUL_OS)) + FUNCT7(smatmul, SMATMUL_OS)) // E4M3 codes in row-major A[M,K] and B[N,K], followed by E8M0[M,K/32] // and E8M0[N,K/32]. C is row-major FP32. K is a multiple of 32. @@ -31,7 +33,7 @@ BUCKYBALL_INSTRUCTION_R_R( \ BB_BANK0(a_bank) | BB_BANK1(b_bank) | BB_BANK2(c_bank) | BB_ITER(k), \ BB_SMATMUL_CFG(rows, cols, first, last, output_base), \ - BB_FUNC7(SMATMUL_MXFP8)) + FUNCT7(smatmul, SMATMUL_MXFP8)) // Row-major FP32 A[M,K], B[N,K], C[M,N]. K is a multiple of four. #define bb_smatmul_f32(a_bank, b_bank, c_bank, rows, cols, k, first, last, \ @@ -39,6 +41,6 @@ BUCKYBALL_INSTRUCTION_R_R( \ BB_BANK0(a_bank) | BB_BANK1(b_bank) | BB_BANK2(c_bank) | BB_ITER(k), \ BB_SMATMUL_CFG(rows, cols, first, last, output_base), \ - BB_FUNC7(SMATMUL_F32)) + FUNCT7(smatmul, SMATMUL_F32)) #endif // _BB_SMATMUL_H_ diff --git a/examples/components/balls/smatmul/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/smatmul/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/smatmul/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/smatmul/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/smatmul/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/smatmul/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/components/balls/smatmul/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/smatmul/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir b/examples/prefab/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir similarity index 87% rename from examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir rename to examples/prefab/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir index 6966ff1cf..14df63d16 100644 --- a/examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir +++ b/examples/prefab/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate.mlir @@ -51,14 +51,14 @@ func.func @main() -> i8 { buckyball.mvin %weight1 <{bank = 4 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> // The bias occupies the first four rows of C. buckyball.mvin %bias <{bank = 5 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> - buckyball.custom "matmul_i8_block" <{sources = array, destination = 5 : i64, read_destination}> {rows = 16 : i64, k = 16 : i64, first = true, last = false} - buckyball.custom "matmul_i8_block" <{sources = array, destination = 5 : i64, read_destination}> {rows = 16 : i64, k = 16 : i64, first = false, last = true} + // Two reduction blocks: A = (input0, input1) and B = (weight0, weight1). + buckyball.mset_transfer <{source = 3 : i64, target = 1 : i64}> + buckyball.mset_transfer <{source = 4 : i64, target = 2 : i64}> + smatmul_ball.matmul_i8 <{a = 1 : i64, b = 2 : i64, c = 5 : i64, rows = 16 : i64, k = 16 : i64}> buckyball.mvout %output <{bank = 5 : i64, depth = 64 : i64, stride = 1 : i64}> : memref<64x4xi32> func.call @check_result(%output) : (memref<64x4xi32>) -> () buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> - buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> - buckyball.mset <{bank = 4 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 5 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> memref.dealloc %input0 : memref<16x16xi8> memref.dealloc %input1 : memref<16x16xi8> diff --git a/examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp b/examples/prefab/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp similarity index 94% rename from examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp rename to examples/prefab/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp index 14d4a7062..ba053c808 100644 --- a/examples/components/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp +++ b/examples/prefab/balls/smatmul/workloads/mlir_tests/ball/smatmul_bias_accumulate_main.cpp @@ -35,5 +35,5 @@ extern "C" void check_result(int32_t *allocated, int32_t *aligned, } } } - printf("PASSED: smatmul bias and two-step accumulation\n"); + printf("PASSED: smatmul bias and two reduction blocks\n"); } diff --git a/examples/components/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt similarity index 54% rename from examples/components/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt index 67abe72b7..03d62d5a1 100644 --- a/examples/components/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt +++ b/examples/prefab/balls/smatmul/workloads/mlir_tests/bank/CMakeLists.txt @@ -3,16 +3,13 @@ if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) endif() set(BUCKYBALL_MLIR_TEST_PREFIX bank) -set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") -file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") -file(STRINGS "${_params}" _bank_width REGEX "^#define BANK_WIDTH ") -string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") -string(REGEX REPLACE ".* " "" _width "${_bank_width}") +set(_lines ${DESIGN_PROFILES_${BUCKYBALL_MLIR_ACTIVE_TARGET}_THREAD_MEM_BANK_ENTRIES}) +set(_width ${DESIGN_PROFILES_${BUCKYBALL_MLIR_ACTIVE_TARGET}_THREAD_MEM_BANK_WIDTH}) math(EXPR _line_remainder "${_lines} % 64") if(NOT (_width EQUAL 128) OR _lines LESS 64 OR NOT (_line_remainder EQUAL 0)) message(FATAL_ERROR - "SMatMul BankOp test requires BANK_WIDTH=128 and BANK_LINES a multiple " + "SMatMul BankOp test requires 128-bit bank rows and bank entries a multiple " "of 64 (got width=${_width}, lines=${_lines})") endif() math(EXPR BB_SMM_M "${_lines} / 4") diff --git a/examples/components/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in b/examples/prefab/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in similarity index 88% rename from examples/components/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in rename to examples/prefab/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in index 2876766dd..9ad6fefcb 100644 --- a/examples/components/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in +++ b/examples/prefab/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms.mlir.in @@ -51,14 +51,14 @@ func.func @main() -> i8 { %b1 = buckyball.bank_mvin %w1, %b1_bank <{depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> // The bias occupies the first four rows of C. %c = buckyball.bank_mvin %bias, %c_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xi32> - %partial = buckyball.bank_custom "matmul_i8_block" (%a0, %b0) -> %c <{read_destination}> {rows = @BB_SMM_M@ : i64, k = 16 : i64, first = true, last = false} - %result = buckyball.bank_custom "matmul_i8_block" (%a1, %b1) -> %partial <{read_destination}> {rows = @BB_SMM_M@ : i64, k = 16 : i64, first = false, last = true} + // Two reduction blocks: the second A and B become the second groups. + %a_groups = buckyball.bank_transfer %a1, %a0 + %b_groups = buckyball.bank_transfer %b1, %b0 + %result = smatmul_ball.bank_matmul_i8 %a_groups, %b_groups -> %c <{rows = @BB_SMM_M@ : i64, k = 16 : i64}> %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x4xi32> func.call @check_result(%output) : (memref<@BB_BANK_LINES@x4xi32>) -> () - buckyball.bank_release %a0 - buckyball.bank_release %b0 - buckyball.bank_release %a1 - buckyball.bank_release %b1 + buckyball.bank_release %a_groups + buckyball.bank_release %b_groups buckyball.bank_release %stored memref.dealloc %a : memref<@BB_SMM_M@x16xi8> memref.dealloc %w0 : memref<16x16xi8> diff --git a/examples/components/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp b/examples/prefab/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp similarity index 74% rename from examples/components/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp rename to examples/prefab/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp index 5a3eff2ea..0d10c7a27 100644 --- a/examples/components/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp +++ b/examples/prefab/balls/smatmul/workloads/mlir_tests/bank/smatmul_bias_two_terms_main.cpp @@ -1,13 +1,14 @@ -#include #include #include #include extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { - if (rows != BANK_LINES || lanes != 4 || stride != 4 || lane_stride != 1) + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != 4 || stride != 4 || + lane_stride != 1) exit(1); - for (int matrix_row = 0; matrix_row < BANK_LINES / 4; ++matrix_row) + for (int matrix_row = 0; matrix_row < THREAD_MEM_BANK_ENTRIES / 4; + ++matrix_row) for (int column = 0; column < 16; ++column) { int bank_row = matrix_row * 4 + column / 4; int bank_lane = column % 4; @@ -19,5 +20,5 @@ extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, exit(1); } } - printf("smatmul bank PASS rows=%d\n", BANK_LINES); + printf("smatmul bank PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); } diff --git a/examples/components/balls/toint8/arch/ToInt8Ball.scala b/examples/prefab/balls/toint8/arch/ToInt8Ball.scala similarity index 100% rename from examples/components/balls/toint8/arch/ToInt8Ball.scala rename to examples/prefab/balls/toint8/arch/ToInt8Ball.scala diff --git a/examples/components/balls/toint8/configs/default.toml b/examples/prefab/balls/toint8/configs/default.toml similarity index 100% rename from examples/components/balls/toint8/configs/default.toml rename to examples/prefab/balls/toint8/configs/default.toml diff --git a/examples/components/balls/toint8/configs/toint8_params.proto b/examples/prefab/balls/toint8/configs/toint8_params.proto similarity index 100% rename from examples/components/balls/toint8/configs/toint8_params.proto rename to examples/prefab/balls/toint8/configs/toint8_params.proto diff --git a/examples/components/balls/toint8/emu/src/dispatch.rs b/examples/prefab/balls/toint8/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/toint8/emu/src/dispatch.rs rename to examples/prefab/balls/toint8/emu/src/dispatch.rs diff --git a/examples/components/balls/toint8/emu/src/lib.rs b/examples/prefab/balls/toint8/emu/src/lib.rs similarity index 100% rename from examples/components/balls/toint8/emu/src/lib.rs rename to examples/prefab/balls/toint8/emu/src/lib.rs diff --git a/examples/components/balls/toint8/emu/src/model.rs b/examples/prefab/balls/toint8/emu/src/model.rs similarity index 100% rename from examples/components/balls/toint8/emu/src/model.rs rename to examples/prefab/balls/toint8/emu/src/model.rs diff --git a/examples/components/balls/toint8/emu/src/quant.rs b/examples/prefab/balls/toint8/emu/src/quant.rs similarity index 96% rename from examples/components/balls/toint8/emu/src/quant.rs rename to examples/prefab/balls/toint8/emu/src/quant.rs index 13da7e92e..117290472 100644 --- a/examples/components/balls/toint8/emu/src/quant.rs +++ b/examples/prefab/balls/toint8/emu/src/quant.rs @@ -1,8 +1,9 @@ -use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::ExecContext; use bebop_bemu::thread::banks::bank_lines; pub(crate) fn exec_f32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "quant_f32_to_i8: reads no old destination; rs1[29] must be clear"); let src = rs1_b0(xs1); let unused = rs1_b1(xs1); let dst = rs1_b2(xs1); @@ -56,6 +57,7 @@ pub(crate) fn exec_f32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { } pub(crate) fn exec_i32_to_i8(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "quant_i32_to_i8: reads no old destination; rs1[29] must be clear"); let src = rs1_b0(xs1); let scale_bank = rs1_b1(xs1); let dst = rs1_b2(xs1); diff --git a/examples/prefab/balls/toint8/stack/src/toint8Ops.cpp b/examples/prefab/balls/toint8/stack/src/toint8Ops.cpp new file mode 100644 index 000000000..ae3cba72c --- /dev/null +++ b/examples/prefab/balls/toint8/stack/src/toint8Ops.cpp @@ -0,0 +1,104 @@ +//===- toint8Ops.cpp - ToInt8Ball operations ------------------------------===// +// +// The dialect of ToInt8Ball, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "toint8Ops.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" + +using namespace mlir; +using namespace ::buddy::toint8_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "toint8Dialect.cpp.inc" + +#define GET_OP_CLASSES +#include "toint8Ops.cpp.inc" + +void ToInt8BallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "toint8Ops.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankFp32ToInt8Op::getSourceBanks() { + return {getInput()}; +} +Value BankFp32ToInt8Op::getDestinationBank() { return getOutput(); } +bool BankFp32ToInt8Op::readsDestination() { return false; } +Operation *BankFp32ToInt8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return Fp32ToInt8Op::create(builder, getLoc(), getMultiplier(), sources[0], + destination, getRows()); +} + +SmallVector Fp32ToInt8Op::getSourceBanks() { + return {int64_t(getInput())}; +} +int64_t Fp32ToInt8Op::getDestinationBank() { return getOutput(); } +LogicalResult Fp32ToInt8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + uint64_t rs1 = + bankFields({int64_t(getInput())}, getOutput(), false, getRows()); + Value bits = arith::BitcastOp::create(builder, loc, builder.getI32Type(), + getMultiplier()); + Value rs2 = arith::ExtUIOp::create(builder, loc, builder.getI64Type(), bits); + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7("QUANT_F32_TO_I8"), + constantI64(builder, loc, rs1), rs2); + return success(); +} + +SmallVector BankInt32ToInt8Op::getSourceBanks() { + return {getValues(), getScales()}; +} +Value BankInt32ToInt8Op::getDestinationBank() { return getOutput(); } +bool BankInt32ToInt8Op::readsDestination() { return false; } +Operation *BankInt32ToInt8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return Int32ToInt8Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows(), getRelu()); +} + +SmallVector Int32ToInt8Op::getSourceBanks() { + return {int64_t(getValues()), int64_t(getScales())}; +} +int64_t Int32ToInt8Op::getDestinationBank() { return getOutput(); } +LogicalResult Int32ToInt8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(); + int64_t depth = buckyball_target::getBuckyballTarget().bankDepth; + if (rows <= 0 || rows * 4 > depth) + return emitOpError("INT32-to-INT8 rows exceed the bank depth"); + // The output rows start at the base of the destination, packed; the + // seven-bit height takes 128 rows as two columns of 64. + int64_t width = rows > 127 ? 2 : 1; + uint64_t rs1 = bankFields({int64_t(getValues()), int64_t(getScales())}, + getOutput(), false, rows * 4); + uint64_t rs2 = uint64_t(getRelu()) | (uint64_t(width) << 8) | + (uint64_t(rows / width) << 15) | (uint64_t(width) << 22); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("QUANT_I32_TO_I8"), + constantI64(builder, loc, rs1), constantI64(builder, loc, rs2)); + return success(); +} diff --git a/examples/prefab/balls/toint8/stack/src/toint8Ops.h b/examples/prefab/balls/toint8/stack/src/toint8Ops.h new file mode 100644 index 000000000..a4813ed0f --- /dev/null +++ b/examples/prefab/balls/toint8/stack/src/toint8Ops.h @@ -0,0 +1,20 @@ +//===- toint8Ops.h - ToInt8Ball operations --------------------------------===// + +#ifndef TOINT8_OPS_H +#define TOINT8_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "toint8Dialect.h.inc" + +#define GET_OP_CLASSES +#include "toint8Ops.h.inc" + +#endif // TOINT8_OPS_H diff --git a/examples/prefab/balls/toint8/stack/src/toint8Ops.td b/examples/prefab/balls/toint8/stack/src/toint8Ops.td new file mode 100644 index 000000000..7eb70d541 --- /dev/null +++ b/examples/prefab/balls/toint8/stack/src/toint8Ops.td @@ -0,0 +1,67 @@ +//===- toint8Ops.td - ToInt8Ball operations ----------------*- tablegen -*-===// +// +// fp32_to_int8: the input bank holds `rows` rows of four f32, a multiple of +// four; the output receives rows / 4 rows of sixteen i8, +// clamp(round_half_even(x * multiplier), -128, 127). `multiplier` is f32. +// int32_to_int8: `values` holds `rows` rows of sixteen i32 (four lines each), +// `scales` the sixteen f32 scales of the columns (four lines); the output +// receives `rows` rows of sixteen i8, clamp(round_half_even(x * scale), -128, +// 127), with `relu` at least zero. +// Both work on every group: group g is channel block g, and the banks have the +// same number of groups. +// +//===----------------------------------------------------------------------===// + +#ifndef TOINT8_OPS +#define TOINT8_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def ToInt8Ball_Dialect : Dialect { + let name = "toint8_ball"; + let cppNamespace = "::buddy::toint8_ball"; + let summary = "The operations of ToInt8Ball"; +} + +class ToInt8Ball_Op traits = []> + : Op])>; + +def BankFp32ToInt8Op : ToInt8Ball_Op<"bank_fp32_to_int8", + [DeclareOpInterfaceMethods]> { + let summary = "fp32_to_int8 on virtual banks"; + let arguments = (ins I64:$input, I64:$output, F32:$multiplier, + I64Attr:$rows); + let results = (outs I64:$outputOut); + let assemblyFormat = "$input `->` $output `multiplier` `(` $multiplier `)` " + "prop-dict attr-dict"; +} + +def Fp32ToInt8Op : ToInt8Ball_Op<"fp32_to_int8", + [DeclareOpInterfaceMethods]> { + let summary = "fp32_to_int8 on physical banks"; + let arguments = (ins F32:$multiplier, I64Attr:$input, I64Attr:$output, + I64Attr:$rows); + let assemblyFormat = "`multiplier` `(` $multiplier `)` prop-dict attr-dict"; +} + +def BankInt32ToInt8Op : ToInt8Ball_Op<"bank_int32_to_int8", + [DeclareOpInterfaceMethods]> { + let summary = "int32_to_int8 on virtual banks"; + let arguments = (ins I64:$values, I64:$scales, I64:$output, I64Attr:$rows, + UnitAttr:$relu); + let results = (outs I64:$outputOut); + let assemblyFormat = "$values `,` $scales `->` $output prop-dict attr-dict"; +} + +def Int32ToInt8Op : ToInt8Ball_Op<"int32_to_int8", + [DeclareOpInterfaceMethods]> { + let summary = "int32_to_int8 on physical banks"; + let arguments = (ins I64Attr:$values, I64Attr:$scales, I64Attr:$output, + I64Attr:$rows, UnitAttr:$relu); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // TOINT8_OPS diff --git a/examples/components/balls/toint8/verify/README.md b/examples/prefab/balls/toint8/verify/README.md similarity index 100% rename from examples/components/balls/toint8/verify/README.md rename to examples/prefab/balls/toint8/verify/README.md diff --git a/examples/components/balls/toint8/verify/casegen/Cargo.lock b/examples/prefab/balls/toint8/verify/casegen/Cargo.lock similarity index 100% rename from examples/components/balls/toint8/verify/casegen/Cargo.lock rename to examples/prefab/balls/toint8/verify/casegen/Cargo.lock diff --git a/examples/components/balls/toint8/verify/casegen/Cargo.toml b/examples/prefab/balls/toint8/verify/casegen/Cargo.toml similarity index 100% rename from examples/components/balls/toint8/verify/casegen/Cargo.toml rename to examples/prefab/balls/toint8/verify/casegen/Cargo.toml diff --git a/examples/components/balls/toint8/verify/casegen/src/casegen.rs b/examples/prefab/balls/toint8/verify/casegen/src/casegen.rs similarity index 100% rename from examples/components/balls/toint8/verify/casegen/src/casegen.rs rename to examples/prefab/balls/toint8/verify/casegen/src/casegen.rs diff --git a/examples/components/balls/toint8/verify/casegen/src/lib.rs b/examples/prefab/balls/toint8/verify/casegen/src/lib.rs similarity index 100% rename from examples/components/balls/toint8/verify/casegen/src/lib.rs rename to examples/prefab/balls/toint8/verify/casegen/src/lib.rs diff --git a/examples/components/balls/toint8/verify/filelists/toint8_ball.f b/examples/prefab/balls/toint8/verify/filelists/toint8_ball.f similarity index 100% rename from examples/components/balls/toint8/verify/filelists/toint8_ball.f rename to examples/prefab/balls/toint8/verify/filelists/toint8_ball.f diff --git a/examples/components/balls/toint8/verify/src/common/toint8_defs.svh b/examples/prefab/balls/toint8/verify/src/common/toint8_defs.svh similarity index 100% rename from examples/components/balls/toint8/verify/src/common/toint8_defs.svh rename to examples/prefab/balls/toint8/verify/src/common/toint8_defs.svh diff --git a/examples/components/balls/toint8/verify/src/common/toint8_items.svh b/examples/prefab/balls/toint8/verify/src/common/toint8_items.svh similarity index 100% rename from examples/components/balls/toint8/verify/src/common/toint8_items.svh rename to examples/prefab/balls/toint8/verify/src/common/toint8_items.svh diff --git a/examples/components/balls/toint8/verify/src/cov/toint8_cov.svh b/examples/prefab/balls/toint8/verify/src/cov/toint8_cov.svh similarity index 100% rename from examples/components/balls/toint8/verify/src/cov/toint8_cov.svh rename to examples/prefab/balls/toint8/verify/src/cov/toint8_cov.svh diff --git a/examples/components/balls/toint8/verify/src/env/toint8_env.svh b/examples/prefab/balls/toint8/verify/src/env/toint8_env.svh similarity index 100% rename from examples/components/balls/toint8/verify/src/env/toint8_env.svh rename to examples/prefab/balls/toint8/verify/src/env/toint8_env.svh diff --git a/examples/components/balls/toint8/verify/src/env/toint8_scoreboard.svh b/examples/prefab/balls/toint8/verify/src/env/toint8_scoreboard.svh similarity index 100% rename from examples/components/balls/toint8/verify/src/env/toint8_scoreboard.svh rename to examples/prefab/balls/toint8/verify/src/env/toint8_scoreboard.svh diff --git a/examples/components/balls/toint8/verify/src/pkg/toint8_pkg.sv b/examples/prefab/balls/toint8/verify/src/pkg/toint8_pkg.sv similarity index 100% rename from examples/components/balls/toint8/verify/src/pkg/toint8_pkg.sv rename to examples/prefab/balls/toint8/verify/src/pkg/toint8_pkg.sv diff --git a/examples/components/balls/toint8/verify/src/seq/toint8_sequences.svh b/examples/prefab/balls/toint8/verify/src/seq/toint8_sequences.svh similarity index 100% rename from examples/components/balls/toint8/verify/src/seq/toint8_sequences.svh rename to examples/prefab/balls/toint8/verify/src/seq/toint8_sequences.svh diff --git a/examples/components/balls/toint8/verify/src/tb_top.sv b/examples/prefab/balls/toint8/verify/src/tb_top.sv similarity index 100% rename from examples/components/balls/toint8/verify/src/tb_top.sv rename to examples/prefab/balls/toint8/verify/src/tb_top.sv diff --git a/examples/components/balls/toint8/verify/src/tests/toint8_test.svh b/examples/prefab/balls/toint8/verify/src/tests/toint8_test.svh similarity index 100% rename from examples/components/balls/toint8/verify/src/tests/toint8_test.svh rename to examples/prefab/balls/toint8/verify/src/tests/toint8_test.svh diff --git a/examples/components/balls/toint8/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/toint8/workloads/ctests/CMakeLists.txt similarity index 100% rename from examples/components/balls/toint8/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/toint8/workloads/ctests/CMakeLists.txt diff --git a/examples/components/balls/toint8/workloads/ctests/quant_f32_groups_test.c b/examples/prefab/balls/toint8/workloads/ctests/quant_f32_groups_test.c similarity index 100% rename from examples/components/balls/toint8/workloads/ctests/quant_f32_groups_test.c rename to examples/prefab/balls/toint8/workloads/ctests/quant_f32_groups_test.c diff --git a/examples/components/balls/toint8/workloads/ctests/quant_f32_to_i8_test.c b/examples/prefab/balls/toint8/workloads/ctests/quant_f32_to_i8_test.c similarity index 100% rename from examples/components/balls/toint8/workloads/ctests/quant_f32_to_i8_test.c rename to examples/prefab/balls/toint8/workloads/ctests/quant_f32_to_i8_test.c diff --git a/examples/components/balls/toint8/workloads/ctests/quant_i32_groups_test.c b/examples/prefab/balls/toint8/workloads/ctests/quant_i32_groups_test.c similarity index 100% rename from examples/components/balls/toint8/workloads/ctests/quant_i32_groups_test.c rename to examples/prefab/balls/toint8/workloads/ctests/quant_i32_groups_test.c diff --git a/examples/components/balls/toint8/workloads/ctests/quant_i32_to_i8_test.c b/examples/prefab/balls/toint8/workloads/ctests/quant_i32_to_i8_test.c similarity index 100% rename from examples/components/balls/toint8/workloads/ctests/quant_i32_to_i8_test.c rename to examples/prefab/balls/toint8/workloads/ctests/quant_i32_to_i8_test.c diff --git a/examples/components/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c b/examples/prefab/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c similarity index 100% rename from examples/components/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c rename to examples/prefab/balls/toint8/workloads/ctests/toint8_i8_rounding_test.c diff --git a/examples/components/balls/toint8/workloads/ctests/toint8_i8_rows_test.c b/examples/prefab/balls/toint8/workloads/ctests/toint8_i8_rows_test.c similarity index 100% rename from examples/components/balls/toint8/workloads/ctests/toint8_i8_rows_test.c rename to examples/prefab/balls/toint8/workloads/ctests/toint8_i8_rows_test.c diff --git a/examples/components/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c b/examples/prefab/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c similarity index 100% rename from examples/components/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c rename to examples/prefab/balls/toint8/workloads/ctests/toint8_i8_scale_rows_test.c diff --git a/examples/components/balls/toint8/workloads/ctests/toint8_i8_signed_test.c b/examples/prefab/balls/toint8/workloads/ctests/toint8_i8_signed_test.c similarity index 100% rename from examples/components/balls/toint8/workloads/ctests/toint8_i8_signed_test.c rename to examples/prefab/balls/toint8/workloads/ctests/toint8_i8_signed_test.c diff --git a/examples/components/balls/toint8/workloads/isa/quant.h b/examples/prefab/balls/toint8/workloads/isa/quant.h similarity index 92% rename from examples/components/balls/toint8/workloads/isa/quant.h rename to examples/prefab/balls/toint8/workloads/isa/quant.h index 3d6f4ae31..655c9e7c1 100644 --- a/examples/components/balls/toint8/workloads/isa/quant.h +++ b/examples/prefab/balls/toint8/workloads/isa/quant.h @@ -19,7 +19,7 @@ static inline void bb_quant_f32_to_i8(uint64_t input_bank, uint64_t output_bank, } bits = {.f = scale}; BUCKYBALL_INSTRUCTION_R_R(BB_BANK0(input_bank) | BB_BANK2(output_bank) | BB_ITER(iter), - bits.u, BB_FUNC7(QUANT_F32_TO_I8)); + bits.u, FUNCT7(toint8, QUANT_F32_TO_I8)); } static inline void bb_quant_i32_to_i8(uint64_t input_bank, uint64_t scale_bank, @@ -33,7 +33,7 @@ static inline void bb_quant_i32_to_i8(uint64_t input_bank, uint64_t scale_bank, (output_base << 1) | (output_width << 8) | (output_height << 15) | (output_stride << 22) | (input_base << 29) | (uint64_t)(relu != 0), - BB_FUNC7(QUANT_I32_TO_I8)); + FUNCT7(toint8, QUANT_I32_TO_I8)); } #endif // _BB_QUANT_H_ diff --git a/examples/components/balls/toint8/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/toint8/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/toint8/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/toint8/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/toint8/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/toint8/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/components/balls/toint8/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/toint8/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir b/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir similarity index 91% rename from examples/components/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir rename to examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir index 8c3bdd7cb..215ea0b8a 100644 --- a/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir +++ b/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8.mlir @@ -15,7 +15,7 @@ func.func @main() -> i8 { buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mvin %input <{bank = 1 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - buckyball.custom "fp32_to_int8" args(%scale : f32) <{sources = array, destination = 2 : i64}> {rows = 4 : i64} + toint8_ball.fp32_to_int8 multiplier(%scale) <{input = 1 : i64, output = 2 : i64, rows = 4 : i64}> buckyball.mvout %output <{bank = 2 : i64, depth = 1 : i64, stride = 1 : i64}> : memref<1x16xi8> func.call @check_result(%output) : (memref<1x16xi8>) -> () buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> diff --git a/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8_main.cpp b/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8_main.cpp similarity index 100% rename from examples/components/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8_main.cpp rename to examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_f32_to_i8_main.cpp diff --git a/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir b/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir similarity index 95% rename from examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir rename to examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir index 940886a22..41d7d08de 100644 --- a/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir +++ b/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8.mlir @@ -32,7 +32,7 @@ func.func @main() -> i8 { buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mvin %input <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x4xi32> buckyball.mvin %scales <{bank = 2 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - buckyball.custom "int32_to_int8" <{sources = array, destination = 3 : i64}> {rows = 4 : i64, relu = true} + toint8_ball.int32_to_int8 <{values = 1 : i64, scales = 2 : i64, output = 3 : i64, rows = 4 : i64, relu}> buckyball.mvout %output <{bank = 3 : i64, depth = 4 : i64, stride = 1 : i64}> : memref<4x16xi8> func.call @check_result(%output) : (memref<4x16xi8>) -> () buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> diff --git a/examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp b/examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp similarity index 100% rename from examples/components/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp rename to examples/prefab/balls/toint8/workloads/mlir_tests/ball/quant_i32_to_i8_main.cpp diff --git a/examples/components/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt similarity index 72% rename from examples/components/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt index fc5a32057..1f1ce2ee5 100644 --- a/examples/components/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt +++ b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/CMakeLists.txt @@ -3,13 +3,11 @@ if(NOT DEFINED BUCKYBALL_MLIR_GROUP_TARGET) endif() set(BUCKYBALL_MLIR_TEST_PREFIX bank) -set(_params "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_MLIR_ACTIVE_TARGET}/params.h") -file(STRINGS "${_params}" _bank_lines REGEX "^#define BANK_LINES ") -string(REGEX REPLACE ".* " "" _lines "${_bank_lines}") +set(_lines ${DESIGN_PROFILES_${BUCKYBALL_MLIR_ACTIVE_TARGET}_THREAD_MEM_BANK_ENTRIES}) math(EXPR _remainder "${_lines} % 4") if(_lines LESS 4 OR NOT (_remainder EQUAL 0)) message(FATAL_ERROR - "ToInt8 INT32 BankOp test requires BANK_LINES divisible by four " + "ToInt8 INT32 BankOp test requires bank entries divisible by four " "(got lines=${_lines})") endif() # One QUANT_I32_TO_I8 instruction accepts at most 508 packed INT32 rows. diff --git a/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in similarity index 93% rename from examples/components/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in rename to examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in index 481456bcd..834cbcce0 100644 --- a/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in +++ b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16.mlir.in @@ -25,7 +25,7 @@ func.func @main() -> i8 { %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x4xf32> - %result = buckyball.bank_custom "fp32_to_int8" (%loaded_input) -> %output_bank args(%scale : f32) {rows = @BB_BANK_LINES@ : i64} + %result = toint8_ball.bank_fp32_to_int8 %loaded_input -> %output_bank multiplier(%scale) <{rows = @BB_BANK_LINES@ : i64}> %stored = buckyball.bank_mvout %output, %result <{depth = @BB_QUANT_I32_OUTPUT_ROWS@ : i64, stride = 1 : i64}> : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> func.call @check_result(%output) : (memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () buckyball.bank_release %loaded_input diff --git a/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp similarity index 75% rename from examples/components/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp rename to examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp index ecb134ef2..35caaee66 100644 --- a/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp +++ b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_f32_pack16_main.cpp @@ -1,13 +1,13 @@ -#include #include #include #include extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { - constexpr int outputRows = BANK_LINES / 4 < 127 ? BANK_LINES / 4 : 127; - if (rows != outputRows || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) + constexpr int outputRows = + THREAD_MEM_BANK_ENTRIES / 4 < 127 ? THREAD_MEM_BANK_ENTRIES / 4 : 127; + if (rows != outputRows || lanes != THREAD_MEM_BANK_WIDTH / 8 || + stride != lanes || lane_stride != 1) exit(1); for (int row = 0; row < rows; ++row) for (int lane = 0; lane < lanes; ++lane) { diff --git a/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in similarity index 93% rename from examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in rename to examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in index 00f7759df..af2a34393 100644 --- a/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in +++ b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16.mlir.in @@ -28,7 +28,7 @@ func.func @main() -> i8 { %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded_input = buckyball.bank_mvin %input, %input_bank <{depth = @BB_QUANT_I32_INPUT_ROWS@ : i64, stride = 1 : i64}> : memref<@BB_QUANT_I32_INPUT_ROWS@x4xi32> %loaded_scale = buckyball.bank_mvin %scale, %scale_bank <{depth = 4 : i64, stride = 1 : i64}> : memref<4x4xf32> - %result = buckyball.bank_custom "int32_to_int8" (%loaded_input, %loaded_scale) -> %output_bank {rows = @BB_QUANT_I32_OUTPUT_ROWS@ : i64, relu = false} + %result = toint8_ball.bank_int32_to_int8 %loaded_input, %loaded_scale -> %output_bank <{rows = @BB_QUANT_I32_OUTPUT_ROWS@ : i64}> %stored = buckyball.bank_mvout %output, %result <{depth = @BB_QUANT_I32_OUTPUT_ROWS@ : i64, stride = 1 : i64}> : memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8> func.call @check_result(%output) : (memref<@BB_QUANT_I32_OUTPUT_ROWS@x@BB_BANK_I8_LANES@xi8>) -> () buckyball.bank_release %loaded_input diff --git a/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp similarity index 68% rename from examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp rename to examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp index 6d965d0e7..574fcb98f 100644 --- a/examples/components/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp +++ b/examples/prefab/balls/toint8/workloads/mlir_tests/bank/quant_i32_pack16_main.cpp @@ -1,13 +1,13 @@ -#include #include #include #include extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { - const int expected_rows = (BANK_LINES < 508 ? BANK_LINES : 508) / 4; - if (rows != expected_rows || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) + const int expected_rows = + (THREAD_MEM_BANK_ENTRIES < 508 ? THREAD_MEM_BANK_ENTRIES : 508) / 4; + if (rows != expected_rows || lanes != THREAD_MEM_BANK_WIDTH / 8 || + stride != lanes || lane_stride != 1) exit(1); for (int row = 0; row < rows; ++row) for (int lane = 0; lane < lanes; ++lane) { @@ -19,5 +19,5 @@ extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, exit(1); } } - printf("quant i32 bank PASS rows=%d\n", BANK_LINES); + printf("quant i32 bank PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); } diff --git a/examples/prefab/balls/trace/arch/README.md b/examples/prefab/balls/trace/arch/README.md new file mode 100644 index 000000000..e1626e2ce --- /dev/null +++ b/examples/prefab/balls/trace/arch/README.md @@ -0,0 +1,64 @@ +# TraceBall — debug/trace Ball + +## Overview + +TraceBall is a non-computing Ball that measures elapsed cycles over arbitrary +code regions with up to sixteen independent cycle counters. It touches no bank. +The DPI-C hook lives inside TraceBall only and does not affect other modules. + +The framework requires every Ball to own at least one read and one write bank +port, so the design config gives TraceBall `inBW = 1, outBW = 1`; TraceBall +ties those ports idle and never issues a bank request. + +--- + +## Instruction: `bdb_counter` + +The selected Core's `ballISA` assigns the `funct7` of `BDB_COUNTER`. +Cycle counter control; completes in one cycle after acceptance. + +rs1 is unused and zero. + +rs2 layout (64-bit): +``` +rs2[3:0] = subcmd subcommand (0=START, 1=STOP, 2=READ; 3..15 are illegal) +rs2[7:4] = ctr_id counter id (0–15) +rs2[63:8] = payload +``` + +| subcmd | name | behavior | payload | +|--------|------|----------|---------| +| 0 | `CTR_START` | Start counter `ctr_id`, record the current cycle and the tag | tag, echoed in the trace | +| 1 | `CTR_STOP` | Emit elapsed cycles of an active `ctr_id`, then free it | ignored | +| 2 | `CTR_READ` | Emit elapsed cycles of an active `ctr_id`; it keeps running | ignored | + +Illegal inputs stop the simulation: an RTL assertion and a BEMU panic. +- `subcmd` 3..15. +- `CTR_STOP` or `CTR_READ` of a counter that is not started. + +## Counter trace + +With `--ctrace`, both Verilator (`dpi_ctrace`) and BEMU append one NDJSON +record per command to `bdb.ndjson`, so the same tools read both: +``` +{"type":"ctrace","clk":10042,"event":"ctr_start","ctr_id":0,"tag":"0xDEAD","cycle":10042} +{"type":"ctrace","clk":10429,"event":"ctr_stop","ctr_id":0,"tag":"0xDEAD","elapsed":387,"cycle":10429} +{"type":"ctrace","clk":10242,"event":"ctr_read","ctr_id":0,"current":200,"cycle":10242} +``` +`cycle` is the Ball's cycle count and `elapsed`/`current` is `cycle` minus the +cycle of the counter's START. `clk` is the simulator clock. BEMU uses the issue +cycle of the instruction for both `clk` and `cycle`. + +## Example + +```c +bdb_counter_start(0, 0xB001); // outer: whole conv + bdb_counter_start(1, 0xB002); // inner: im2col + bb_im2col(...); + bdb_counter_stop(1); + + bdb_counter_start(2, 0xB003); // inner: matmul + bb_vecmat16(...); + bdb_counter_stop(2); +bdb_counter_stop(0); +``` diff --git a/examples/components/balls/trace/arch/Trace.scala b/examples/prefab/balls/trace/arch/Trace.scala similarity index 97% rename from examples/components/balls/trace/arch/Trace.scala rename to examples/prefab/balls/trace/arch/Trace.scala index 6c297f762..979d842a7 100644 --- a/examples/components/balls/trace/arch/Trace.scala +++ b/examples/prefab/balls/trace/arch/Trace.scala @@ -151,6 +151,9 @@ class Trace( val cmd = io.cmdReq.bits val rs2 = cmd.rs2 + assert(rs2(3, 0) <= CTR_READ, "TraceBall subcmd 3..15 is illegal") + assert(rs2(3, 0) === CTR_START || ctrActive(rs2(7, 4)), "TraceBall STOP/READ of a counter that is not started") + // Counter fields from rs2 subcmd_reg := rs2(3, 0) ctr_id_reg := rs2(7, 4) diff --git a/examples/components/balls/trace/arch/TraceBall.scala b/examples/prefab/balls/trace/arch/TraceBall.scala similarity index 100% rename from examples/components/balls/trace/arch/TraceBall.scala rename to examples/prefab/balls/trace/arch/TraceBall.scala diff --git a/examples/components/balls/trace/arch/TraceDPI.scala b/examples/prefab/balls/trace/arch/TraceDPI.scala similarity index 100% rename from examples/components/balls/trace/arch/TraceDPI.scala rename to examples/prefab/balls/trace/arch/TraceDPI.scala diff --git a/examples/components/balls/trace/configs/default.toml b/examples/prefab/balls/trace/configs/default.toml similarity index 100% rename from examples/components/balls/trace/configs/default.toml rename to examples/prefab/balls/trace/configs/default.toml diff --git a/examples/components/balls/trace/configs/trace_params.proto b/examples/prefab/balls/trace/configs/trace_params.proto similarity index 100% rename from examples/components/balls/trace/configs/trace_params.proto rename to examples/prefab/balls/trace/configs/trace_params.proto diff --git a/examples/prefab/balls/trace/emu/src/04_bdb_counter.rs b/examples/prefab/balls/trace/emu/src/04_bdb_counter.rs new file mode 100644 index 000000000..e00f4a233 --- /dev/null +++ b/examples/prefab/balls/trace/emu/src/04_bdb_counter.rs @@ -0,0 +1,51 @@ +//===- 04_bdb_counter.rs - BDB_COUNTER instruction -------------------------===// +// +// rs2[3:0] is the subcommand, rs2[7:4] the counter and rs2[63:8] the payload; +// the Ball touches no bank. Each command emits one counter trace record: +// START (0): the issue cycle becomes the counter's start, the payload its tag. +// STOP (1): elapsed = cycle - start of a started counter, which is freed. +// READ (2): like STOP but the counter keeps running. +// Subcommands 3..15 and STOP/READ of a counter that is not started are illegal. +// +//===----------------------------------------------------------------------===// + +use super::instruction::{BallModel, ExecContext}; + +const CTR_START: u32 = 0; +const CTR_STOP: u32 = 1; +const CTR_READ: u32 = 2; +const NUM_COUNTERS: usize = 16; + +/// (start cycle, tag) of each started counter. +#[derive(Default)] +struct Counters([Option<(u64, u64)>; NUM_COUNTERS]); + +pub struct BdbCounter; + +impl BallModel for BdbCounter { + fn exec(_xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + let subcmd = (xs2 & 0xf) as u32; + let ctr = ((xs2 >> 4) & 0xf) as u32; + let payload = xs2 >> 8; + assert!(subcmd <= CTR_READ, "bdb_counter: subcmd {subcmd} is illegal"); + let cycle = ctx.cycle; + let state = ctx.state::(); + let mut counters = state.lock().expect("bdb_counter state poisoned"); + let slot = &mut counters.0[ctr as usize]; + if subcmd == CTR_START { + *slot = Some((cycle, payload)); + bebop_bemu::thread::trace::ctrace(CTR_START, ctr, payload, 0, cycle); + return 0; + } + let (start, tag) = slot.unwrap_or_else(|| panic!("bdb_counter: counter {ctr} is not started")); + if subcmd == CTR_STOP { + *slot = None; + } + bebop_bemu::thread::trace::ctrace(subcmd, ctr, tag, cycle - start, cycle); + 0 + } + + fn latency(_xs1: u64, _xs2: u64) -> u64 { + 1 + } +} diff --git a/examples/components/balls/trace/emu/src/dispatch.rs b/examples/prefab/balls/trace/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/trace/emu/src/dispatch.rs rename to examples/prefab/balls/trace/emu/src/dispatch.rs diff --git a/examples/components/balls/trace/emu/src/lib.rs b/examples/prefab/balls/trace/emu/src/lib.rs similarity index 100% rename from examples/components/balls/trace/emu/src/lib.rs rename to examples/prefab/balls/trace/emu/src/lib.rs diff --git a/examples/prefab/balls/trace/stack/src/traceOps.cpp b/examples/prefab/balls/trace/stack/src/traceOps.cpp new file mode 100644 index 000000000..b3c87d380 --- /dev/null +++ b/examples/prefab/balls/trace/stack/src/traceOps.cpp @@ -0,0 +1,77 @@ +//===- traceOps.cpp - TraceBall operations --------------------------------===// +// +// The dialect of TraceBall, its virtual operations building their physical ones +// and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "traceOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +#include "mlir/Dialect/Arith/IR/Arith.h" + +using namespace mlir; +using namespace ::buddy::trace_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "traceDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "traceOps.cpp.inc" + +void TraceBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "traceOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +// One BDB_COUNTER instruction: the subcommand in rs2[3:0], the counter in +// rs2[7:4] and the tag of counter_start from rs2[8]. +static LogicalResult counter(Operation *op, OpBuilder &builder, + uint64_t subcommand, int64_t counter, Value tag) { + Location loc = op->getLoc(); + if (counter < 0 || counter > 15) + return op->emitOpError("BDB counter needs a counter in [0, 15]"); + Value rs2 = + constantI64(builder, loc, field(subcommand, 0, 3) | field(counter, 4, 7)); + if (tag) + rs2 = arith::OrIOp::create( + builder, loc, rs2, + arith::ShLIOp::create(builder, loc, tag, constantI64(builder, loc, 8))); + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7("BDB_COUNTER"), + constantI64(builder, loc, 0), rs2); + return success(); +} + +SmallVector CounterStartOp::getSourceBanks() { return {}; } +int64_t CounterStartOp::getDestinationBank() { return 0; } +LogicalResult CounterStartOp::lowerToInstructions(OpBuilder &builder) { + return counter(*this, builder, 0, getCounter(), getTag()); +} + +SmallVector CounterStopOp::getSourceBanks() { return {}; } +int64_t CounterStopOp::getDestinationBank() { return 0; } +LogicalResult CounterStopOp::lowerToInstructions(OpBuilder &builder) { + return counter(*this, builder, 1, getCounter(), Value()); +} + +SmallVector CounterReadOp::getSourceBanks() { return {}; } +int64_t CounterReadOp::getDestinationBank() { return 0; } +LogicalResult CounterReadOp::lowerToInstructions(OpBuilder &builder) { + return counter(*this, builder, 2, getCounter(), Value()); +} diff --git a/examples/prefab/balls/trace/stack/src/traceOps.h b/examples/prefab/balls/trace/stack/src/traceOps.h new file mode 100644 index 000000000..58cfa9dc9 --- /dev/null +++ b/examples/prefab/balls/trace/stack/src/traceOps.h @@ -0,0 +1,20 @@ +//===- traceOps.h - TraceBall operations ----------------------------------===// + +#ifndef TRACE_OPS_H +#define TRACE_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "traceDialect.h.inc" + +#define GET_OP_CLASSES +#include "traceOps.h.inc" + +#endif // TRACE_OPS_H diff --git a/examples/prefab/balls/trace/stack/src/traceOps.td b/examples/prefab/balls/trace/stack/src/traceOps.td new file mode 100644 index 000000000..911a3508d --- /dev/null +++ b/examples/prefab/balls/trace/stack/src/traceOps.td @@ -0,0 +1,49 @@ +//===- traceOps.td - TraceBall operations ------------------*- tablegen -*-===// +// +// The BDB counter touches no bank: its operations exist only on physical +// banks, with no sources and destination 0, and rs1 is zero. `counter` (0 to +// 15) selects the counter. counter_start starts it with the i64 tag; +// counter_read traces its current value; counter_stop traces the elapsed +// cycles and stops it. +// +//===----------------------------------------------------------------------===// + +#ifndef TRACE_OPS +#define TRACE_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def TraceBall_Dialect : Dialect { + let name = "trace_ball"; + let cppNamespace = "::buddy::trace_ball"; + let summary = "The operations of TraceBall"; +} + +class TraceBall_Op traits = []> + : Op])>; + +def CounterStartOp : TraceBall_Op<"counter_start", + [DeclareOpInterfaceMethods]> { + let summary = "Start a BDB counter with a tag"; + let arguments = (ins I64:$tag, I64Attr:$counter); + let assemblyFormat = "$tag prop-dict attr-dict"; +} + +def CounterStopOp : TraceBall_Op<"counter_stop", + [DeclareOpInterfaceMethods]> { + let summary = "Trace the elapsed cycles of a BDB counter and stop it"; + let arguments = (ins I64Attr:$counter); + let assemblyFormat = "prop-dict attr-dict"; +} + +def CounterReadOp : TraceBall_Op<"counter_read", + [DeclareOpInterfaceMethods]> { + let summary = "Trace the current value of a BDB counter"; + let arguments = (ins I64Attr:$counter); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // TRACE_OPS diff --git a/examples/prefab/balls/trace/verify/casegen/Cargo.lock b/examples/prefab/balls/trace/verify/casegen/Cargo.lock new file mode 100644 index 000000000..3c7929614 --- /dev/null +++ b/examples/prefab/balls/trace/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "trace-casegen" +version = "0.1.0" diff --git a/examples/prefab/balls/trace/verify/casegen/Cargo.toml b/examples/prefab/balls/trace/verify/casegen/Cargo.toml new file mode 100644 index 000000000..de87ef0c7 --- /dev/null +++ b/examples/prefab/balls/trace/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "trace-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["rlib", "cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/prefab/balls/trace/verify/casegen/src/casegen.rs b/examples/prefab/balls/trace/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..370ec5aa3 --- /dev/null +++ b/examples/prefab/balls/trace/verify/casegen/src/casegen.rs @@ -0,0 +1,60 @@ +use crate::model::{Command, CTR_READ, CTR_START, CTR_STOP, NUM_COUNTERS, PAYLOAD_MASK}; + +fn start(ctr: u32, payload: u64) -> Command { + Command { subcmd: CTR_START, ctr, payload } +} + +fn stop(ctr: u32, payload: u64) -> Command { + Command { subcmd: CTR_STOP, ctr, payload } +} + +fn read(ctr: u32, payload: u64) -> Command { + Command { subcmd: CTR_READ, ctr, payload } +} + +pub fn gen_case(index: u32) -> Vec { + match index { + // One measured region with a read inside. + 0 => vec![start(0, 0xa001), read(0, 0), stop(0, 0)], + // Nested and sibling regions, as in the README example. + 1 => vec![ + start(0, 0xb001), + start(1, 0xb002), + stop(1, 0), + start(2, 0xb003), + read(0, 0), + stop(2, 0), + stop(0, 0), + ], + // Every counter, full-width tags; STOP and READ payloads are ignored. + 2 => { + let mut cmds = Vec::new(); + for ctr in 0..NUM_COUNTERS as u32 { + let tag = if ctr % 2 == 0 { + PAYLOAD_MASK ^ (u64::from(ctr) << 40) + } else { + 0x0055_aa55_aa55_aa55 ^ u64::from(ctr) + }; + cmds.push(start(ctr, tag)); + } + for ctr in 0..NUM_COUNTERS as u32 { + cmds.push(read(ctr, PAYLOAD_MASK)); + } + for ctr in (0..NUM_COUNTERS as u32).rev() { + cmds.push(stop(ctr, PAYLOAD_MASK ^ u64::from(ctr))); + } + cmds + } + // A restarted counter measures from the new START with the new tag; + // a zero tag is a valid tag. + 3 => vec![ + start(5, 0), + stop(5, 0), + start(5, 0x2222), + read(5, 0), + read(5, 0), + stop(5, 0), + ], + _ => panic!("trace: unsupported case {index}"), + } +} diff --git a/examples/prefab/balls/trace/verify/casegen/src/lib.rs b/examples/prefab/balls/trace/verify/casegen/src/lib.rs new file mode 100644 index 000000000..0c8502e21 --- /dev/null +++ b/examples/prefab/balls/trace/verify/casegen/src/lib.rs @@ -0,0 +1,110 @@ +mod casegen; +mod model; + +use std::cell::RefCell; + +use model::{Command, Counters, Record}; + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct TraceCmd { + pub rs2_lo: u32, + pub rs2_hi: u32, + pub rob_id: u32, +} + +struct Case { + cmds: Vec, + counters: Counters, + records: usize, + errors: usize, +} + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn trace_case_load(index: u32) -> u32 { + let cmds = casegen::gen_case(index); + let count = cmds.len() as u32; + CURRENT.with(|current| { + *current.borrow_mut() = Some(Case { cmds, counters: Counters::default(), records: 0, errors: 0 }) + }); + count +} + +fn with_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow_mut().as_mut() { + Some(case) => f(case), + None => panic!("trace_case: no case loaded; call trace_case_load first"), + }) +} + +#[no_mangle] +pub extern "C" fn trace_case_cmd(index: u32, out_ptr: *mut TraceCmd) { + if out_ptr.is_null() { + panic!("trace_case_cmd: null out_ptr"); + } + with_case(|case| { + let cmd = match case.cmds.get(index as usize) { + Some(cmd) => *cmd, + None => panic!("trace_case_cmd: index {index} out of range {}", case.cmds.len()), + }; + let rs2 = cmd.rs2(); + unsafe { + *out_ptr = TraceCmd { rs2_lo: rs2 as u32, rs2_hi: (rs2 >> 32) as u32, rob_id: index % 16 }; + } + }); +} + +// Records observed so far; -1 once any record mismatched the model. +#[no_mangle] +pub extern "C" fn trace_case_records() -> i32 { + with_case(|case| if case.errors != 0 { -1 } else { case.records as i32 }) +} + +// The trace hook the RTL calls once per counter command. +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn dpi_ctrace( + subcmd: u32, + ctr_id: u32, + tag_lo: u32, + tag_hi: u32, + elapsed_lo: u32, + elapsed_hi: u32, + cycle_lo: u32, + cycle_hi: u32, +) { + let got = Record { + subcmd, + ctr: ctr_id, + tag: u64::from(tag_lo) | (u64::from(tag_hi) << 32), + elapsed: u64::from(elapsed_lo) | (u64::from(elapsed_hi) << 32), + cycle: u64::from(cycle_lo) | (u64::from(cycle_hi) << 32), + }; + with_case(|case| { + let index = case.records; + case.records += 1; + let cmd = match case.cmds.get(index) { + Some(cmd) => *cmd, + None => { + eprintln!("[TRACE-REF] extra record {index}: {got:?}"); + case.errors += 1; + return; + } + }; + match case.counters.expect(&cmd, got.cycle) { + Ok(want) if want == got => {} + Ok(want) => { + eprintln!("[TRACE-REF] record {index} of {cmd:?}: got {got:?} expected {want:?}"); + case.errors += 1; + } + Err(reason) => { + eprintln!("[TRACE-REF] record {index} of {cmd:?}: {reason}"); + case.errors += 1; + } + } + }); +} diff --git a/examples/prefab/balls/trace/verify/casegen/src/model.rs b/examples/prefab/balls/trace/verify/casegen/src/model.rs new file mode 100644 index 000000000..89e91d673 --- /dev/null +++ b/examples/prefab/balls/trace/verify/casegen/src/model.rs @@ -0,0 +1,69 @@ +// TraceBall BDB_COUNTER contract. rs2[3:0] is the subcommand, rs2[7:4] the +// counter and rs2[63:8] the payload; the Ball touches no bank. Each command +// emits one trace record (subcmd, counter, tag, elapsed, cycle): +// START (0): records cycle as the counter's start and the payload as its +// tag; the record carries the payload and elapsed 0. +// STOP (1): elapsed = cycle - start; the payload is ignored. +// READ (2): like STOP but the counter keeps running. + +pub const CTR_START: u32 = 0; +pub const CTR_STOP: u32 = 1; +pub const CTR_READ: u32 = 2; +pub const NUM_COUNTERS: usize = 16; +pub const PAYLOAD_MASK: u64 = (1 << 56) - 1; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Command { + pub subcmd: u32, + pub ctr: u32, + pub payload: u64, +} + +impl Command { + pub fn rs2(&self) -> u64 { + if self.subcmd > CTR_READ || self.ctr as usize >= NUM_COUNTERS || self.payload > PAYLOAD_MASK { + panic!("trace: invalid command {self:?}"); + } + (self.payload << 8) | (u64::from(self.ctr) << 4) | u64::from(self.subcmd) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Record { + pub subcmd: u32, + pub ctr: u32, + pub tag: u64, + pub elapsed: u64, + pub cycle: u64, +} + +#[derive(Default)] +pub struct Counters { + start: [Option<(u64, u64)>; NUM_COUNTERS], + last_cycle: Option, +} + +impl Counters { + // The expected record of `cmd` given the cycle the Ball reported for it. + pub fn expect(&mut self, cmd: &Command, cycle: u64) -> Result { + if let Some(last) = self.last_cycle { + if cycle <= last { + return Err(format!("cycle {cycle} does not advance past {last}")); + } + } + self.last_cycle = Some(cycle); + let ctr = cmd.ctr as usize; + if cmd.subcmd == CTR_START { + self.start[ctr] = Some((cycle, cmd.payload)); + return Ok(Record { subcmd: CTR_START, ctr: cmd.ctr, tag: cmd.payload, elapsed: 0, cycle }); + } + let (start, tag) = match self.start[ctr] { + Some(entry) => entry, + None => return Err(format!("counter {ctr} used before START")), + }; + if cmd.subcmd == CTR_STOP { + self.start[ctr] = None; + } + Ok(Record { subcmd: cmd.subcmd, ctr: cmd.ctr, tag, elapsed: cycle - start, cycle }) + } +} diff --git a/examples/prefab/balls/trace/verify/filelists/trace_ball.f b/examples/prefab/balls/trace/verify/filelists/trace_ball.f new file mode 100644 index 000000000..e7d579bea --- /dev/null +++ b/examples/prefab/balls/trace/verify/filelists/trace_ball.f @@ -0,0 +1,26 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+src ++incdir+src/pkg ++incdir+src/common ++incdir+src/seq ++incdir+src/cov ++incdir+src/env ++incdir+src/tests ++incdir+@RTL@ + +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv + +@RTL@/CTraceDPI.v +@RTL@/Trace.sv +@RTL@/TraceBall.sv + +src/pkg/trace_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/trace/verify/src/common/trace_defs.svh b/examples/prefab/balls/trace/verify/src/common/trace_defs.svh new file mode 100644 index 000000000..97790ad61 --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/common/trace_defs.svh @@ -0,0 +1,17 @@ +typedef struct { + int unsigned rs2_lo; + int unsigned rs2_hi; + int unsigned rob_id; +} trace_cmd_dpi_t; + +import "DPI-C" function int unsigned trace_case_load(input int unsigned index); +import "DPI-C" function void trace_case_cmd(input int unsigned index, output trace_cmd_dpi_t cmd); +import "DPI-C" function int trace_case_records(); + +`ifndef BDB_COUNTER_FUNCT7 +`error "BDB_COUNTER_FUNCT7 must be provided by the selected ballISA" +`endif + +localparam int BDB_COUNTER_THREAD_FUNCT7 = `BDB_COUNTER_FUNCT7; +localparam int TRACE_NUM_CASES = 4; +localparam int TRACE_TIMEOUT_CYCLES = 2000; diff --git a/examples/prefab/balls/trace/verify/src/common/trace_items.svh b/examples/prefab/balls/trace/verify/src/common/trace_items.svh new file mode 100644 index 000000000..d5d41ad37 --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/common/trace_items.svh @@ -0,0 +1,29 @@ +class trace_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(trace_cmd_item) + + function new(string name = "trace_cmd_item"); + super.new(name); + endfunction + + // bdb_counter: rs1 is zero, so every bank field is zero; rs2 carries the + // subcommand, counter and payload. + function void load_rust_case(int unsigned index); + trace_cmd_dpi_t cmd; + trace_case_cmd(index, cmd); + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = BDB_COUNTER_THREAD_FUNCT7[6:0]; + rs2 = {cmd.rs2_hi, cmd.rs2_lo}; + iter = '0; + op1_bank = '0; + op2_bank = '0; + wr_bank = '0; + op1_group = '0; + op2_group = '0; + wr_group = '0; + op1_col = '0; + op2_col = '0; + wr_col = '0; + wr_read_bank = '0; + wr_arch_bank = '0; + endfunction +endclass diff --git a/examples/prefab/balls/trace/verify/src/cov/trace_cov.svh b/examples/prefab/balls/trace/verify/src/cov/trace_cov.svh new file mode 100644 index 000000000..2cbbbe44b --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/cov/trace_cov.svh @@ -0,0 +1,64 @@ +// The counter Ball never uses its bank ports: the common BLink coverage would +// require them to fire, so the test replaces it with this one. +class trace_blink_cov extends bb_blink_cov #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(trace_blink_cov) + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void check_phase(uvm_phase phase); + if (cmd_fire_count == 0) `uvm_fatal("COV", "cmd handshake never fired") + if (resp_fire_count != cmd_fire_count) + `uvm_fatal("COV", $sformatf("%0d responses for %0d commands", resp_fire_count, cmd_fire_count + )) + for (int i = 0; i < `BB_IN_BW; i++) + if (read_fire_count[i] != 0) `uvm_fatal("COV", $sformatf("read port %0d fired", i)) + for (int i = 0; i < `BB_OUT_BW; i++) + if (write_fire_count[i] != 0) `uvm_fatal("COV", $sformatf("write port %0d fired", i)) + endfunction +endclass + +class trace_cov extends uvm_component; + `uvm_component_utils(trace_cov) + + uvm_analysis_imp_cmd #(bb_blink_cmd_item, trace_cov) cmd_imp; + + int unsigned cur_subcmd; + int unsigned cur_ctr; + bit cur_payload; + + covergroup cmd_cg; + coverpoint cur_subcmd {bins start = {0}; bins stop = {1}; bins read = {2};} + coverpoint cur_ctr {bins ctr[] = {[0 : 15]};} + coverpoint cur_payload {bins zero = {0}; bins nonzero = {1};} + cross cur_subcmd, cur_ctr; + cross cur_subcmd, cur_payload; + endgroup + + int cmd_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + cmd_imp = new("cmd_imp", this); + cmd_cg = new(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + if (item.funct != BDB_COUNTER_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) + cur_subcmd = item.rs2[3:0]; + cur_ctr = item.rs2[7:4]; + cur_payload = |item.rs2[63:8]; + cmd_count++; + cmd_cg.sample(); + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("trace_cov %0.2f%%", cmd_cg.get_coverage())) + endfunction +endclass diff --git a/examples/prefab/balls/trace/verify/src/env/trace_env.svh b/examples/prefab/balls/trace/verify/src/env/trace_env.svh new file mode 100644 index 000000000..5d414e635 --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/env/trace_env.svh @@ -0,0 +1,26 @@ +class trace_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(trace_env) + + trace_scoreboard scb; + trace_cov icov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scb = trace_scoreboard::type_id::create("scb", this); + icov = trace_cov::type_id::create("icov", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + cmd_agent.stim_ap.connect(scb.stim_imp); + cmd_agent.cmd_ap.connect(scb.cmd_imp); + cmd_agent.cmd_ap.connect(icov.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction +endclass diff --git a/examples/prefab/balls/trace/verify/src/env/trace_scoreboard.svh b/examples/prefab/balls/trace/verify/src/env/trace_scoreboard.svh new file mode 100644 index 000000000..443cbd9ab --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/env/trace_scoreboard.svh @@ -0,0 +1,98 @@ +class trace_scoreboard extends uvm_scoreboard; + `uvm_component_utils(trace_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, trace_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, trace_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, trace_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, trace_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, trace_scoreboard) resp_imp; + + bb_blink_cmd_item stim_q[$]; + bb_blink_cmd_item issued_q[$]; + int unsigned expected_cmds; + int unsigned cmd_count; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + bb_blink_cmd_item want; + // Compare as plain BLink commands; the comparer rejects differing types. + want = bb_blink_cmd_item::type_id::create("want"); + want.copy(item); + stim_q.push_back(want); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + bb_blink_cmd_item want; + if (stim_q.size() == 0) `uvm_fatal("CMD", "command observed before stimulus") + if (issued_q.size() != 0) `uvm_fatal("CMD", "command accepted before the previous response") + want = stim_q.pop_front(); + verification_pkg::evidence::checks++; + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) + issued_q.push_back(want); + cmd_count++; + endfunction + + function void write_read(bb_blink_read_item item); + `uvm_fatal("READ", $sformatf("TraceBall touches no bank, got a read of bank %0d", item.bank_id)) + endfunction + + function void write_write(bb_blink_write_item item); + `uvm_fatal("WRITE", $sformatf("TraceBall touches no bank, got a write of bank %0d", item.bank_id + )) + endfunction + + // The reference model in the DPI library checks each trace record as the + // RTL emits it; the record count is checked once the case has drained. + function void write_resp(bb_blink_resp_item item); + bb_blink_cmd_item want; + int records; + if (issued_q.size() == 0) `uvm_fatal("RESP", "response without a command") + want = issued_q.pop_front(); + if (item.rob_id !== want.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, want.rob_id)) + resp_count++; + records = trace_case_records(); + verification_pkg::evidence::checks++; + if (records < 0) `uvm_fatal("TRACE", "trace record mismatch, see [TRACE-REF]") + endfunction + + function bit done(); + return expected_cmds != 0 && cmd_count == expected_cmds && resp_count == expected_cmds; + endfunction + + function void reset_counters(int unsigned num_cmds); + stim_q.delete(); + issued_q.delete(); + expected_cmds = num_cmds; + cmd_count = 0; + resp_count = 0; + endfunction + + function void check_records(); + int records; + records = trace_case_records(); + verification_pkg::evidence::checks++; + if (records < 0) `uvm_fatal("TRACE", "trace record mismatch, see [TRACE-REF]") + if (records != expected_cmds) + `uvm_fatal("TRACE", $sformatf("%0d trace records for %0d commands", records, expected_cmds)) + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) + `uvm_fatal("SCB", $sformatf( + "incomplete: cmds=%0d resp=%0d of %0d", cmd_count, resp_count, expected_cmds)) + check_records(); + endfunction +endclass diff --git a/examples/prefab/balls/trace/verify/src/pkg/trace_pkg.sv b/examples/prefab/balls/trace/verify/src/pkg/trace_pkg.sv new file mode 100644 index 000000000..11e5856e0 --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/pkg/trace_pkg.sv @@ -0,0 +1,13 @@ +package trace_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + + `include "common/trace_defs.svh" + `include "common/trace_items.svh" + `include "seq/trace_sequences.svh" + `include "cov/trace_cov.svh" + `include "env/trace_scoreboard.svh" + `include "env/trace_env.svh" + `include "tests/trace_test.svh" +endpackage diff --git a/examples/prefab/balls/trace/verify/src/seq/trace_sequences.svh b/examples/prefab/balls/trace/verify/src/seq/trace_sequences.svh new file mode 100644 index 000000000..43a7187e1 --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/seq/trace_sequences.svh @@ -0,0 +1,19 @@ +class trace_basic_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(trace_basic_seq) + + int unsigned num_cmds; + + function new(string name = "trace_basic_seq"); + super.new(name); + endfunction + + task body(); + trace_cmd_item item; + for (int unsigned i = 0; i < num_cmds; i++) begin + item = trace_cmd_item::type_id::create("item"); + start_item(item); + item.load_rust_case(i); + finish_item(item); + end + endtask +endclass diff --git a/examples/prefab/balls/trace/verify/src/tb_top.sv b/examples/prefab/balls/trace/verify/src/tb_top.sv new file mode 100644 index 000000000..a747607ab --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/tb_top.sv @@ -0,0 +1,80 @@ +module tb_top; + import uvm_pkg::*; + import trace_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + TraceBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]) + ); + + // Constant outputs removed from the port list by firtool. + assign intf.bank_read_bank_id[0] = dut.blink_bankRead_0_bank_id; + assign intf.bank_read_group_id[0] = dut.blink_bankRead_0_group_id; + assign intf.bank_read_req_valid[0] = dut.blink_bankRead_0_io_req_valid; + assign intf.bank_read_req_addr[0] = dut.blink_bankRead_0_io_req_bits_addr; + assign intf.bank_read_resp_ready[0] = dut.blink_bankRead_0_io_resp_ready; + assign intf.bank_write_bank_id[0] = dut.blink_bankWrite_0_bank_id; + assign intf.bank_write_group_id[0] = dut.blink_bankWrite_0_group_id; + assign intf.bank_write_req_valid[0] = dut.blink_bankWrite_0_io_req_valid; + assign intf.bank_write_req_addr[0] = dut.blink_bankWrite_0_io_req_bits_addr; + assign intf.bank_write_req_data[0] = dut.blink_bankWrite_0_io_req_bits_data; + assign intf.bank_write_resp_ready[0] = dut.blink_bankWrite_0_io_resp_ready; + assign intf.bank_write_req_mask[0][0] = dut.blink_bankWrite_0_io_req_bits_mask_0; + assign intf.bank_write_req_mask[0][1] = dut.blink_bankWrite_0_io_req_bits_mask_1; + assign intf.bank_write_req_mask[0][2] = dut.blink_bankWrite_0_io_req_bits_mask_2; + assign intf.bank_write_req_mask[0][3] = dut.blink_bankWrite_0_io_req_bits_mask_3; + assign intf.bank_write_req_mask[0][4] = dut.blink_bankWrite_0_io_req_bits_mask_4; + assign intf.bank_write_req_mask[0][5] = dut.blink_bankWrite_0_io_req_bits_mask_5; + assign intf.bank_write_req_mask[0][6] = dut.blink_bankWrite_0_io_req_bits_mask_6; + assign intf.bank_write_req_mask[0][7] = dut.blink_bankWrite_0_io_req_bits_mask_7; + assign intf.bank_write_req_mask[0][8] = dut.blink_bankWrite_0_io_req_bits_mask_8; + assign intf.bank_write_req_mask[0][9] = dut.blink_bankWrite_0_io_req_bits_mask_9; + assign intf.bank_write_req_mask[0][10] = dut.blink_bankWrite_0_io_req_bits_mask_10; + assign intf.bank_write_req_mask[0][11] = dut.blink_bankWrite_0_io_req_bits_mask_11; + assign intf.bank_write_req_mask[0][12] = dut.blink_bankWrite_0_io_req_bits_mask_12; + assign intf.bank_write_req_mask[0][13] = dut.blink_bankWrite_0_io_req_bits_mask_13; + assign intf.bank_write_req_mask[0][14] = dut.blink_bankWrite_0_io_req_bits_mask_14; + assign intf.bank_write_req_mask[0][15] = dut.blink_bankWrite_0_io_req_bits_mask_15; + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("trace_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/trace/verify/src/tests/trace_test.svh b/examples/prefab/balls/trace/verify/src/tests/trace_test.svh new file mode 100644 index 000000000..8f386594e --- /dev/null +++ b/examples/prefab/balls/trace/verify/src/tests/trace_test.svh @@ -0,0 +1,53 @@ +class trace_ball_test extends uvm_test; + `uvm_component_utils(trace_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + trace_env env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + bb_blink_cov#(`BB_IN_BW, `BB_OUT_BW)::type_id::set_type_override(trace_blink_cov::get_type()); + env = trace_env::type_id::create("env", this); + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + for (int unsigned index = 0; index < TRACE_NUM_CASES; index++) run_case(index); + phase.drop_objection(this); + endtask + + task run_case(int unsigned index); + trace_basic_seq seq; + int cycles = 0; + apply_reset(); + seq = trace_basic_seq::type_id::create("seq"); + seq.num_cmds = trace_case_load(index); + env.scb.reset_counters(seq.num_cmds); + seq.start(env.cmd_agent.seqr); + while (!env.scb.done()) begin + @(posedge vif.clock); + cycles++; + if (cycles > TRACE_TIMEOUT_CYCLES) + `uvm_fatal("TIMEOUT", $sformatf("TraceBall case %0d timeout", index)) + end + // The trace hook runs one cycle after the command; let it drain. + repeat (2) @(posedge vif.clock); + env.scb.check_records(); + `uvm_info("TRACE", $sformatf("TraceBall case %0d passed: %0d commands in %0d cycles", index, + seq.num_cmds, cycles), UVM_LOW) + endtask + + task apply_reset(); + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/components/balls/trace/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/trace/workloads/ctests/CMakeLists.txt similarity index 100% rename from examples/components/balls/trace/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/trace/workloads/ctests/CMakeLists.txt diff --git a/examples/components/balls/trace/workloads/ctests/bdb_counter_multi_id_test.c b/examples/prefab/balls/trace/workloads/ctests/bdb_counter_multi_id_test.c similarity index 100% rename from examples/components/balls/trace/workloads/ctests/bdb_counter_multi_id_test.c rename to examples/prefab/balls/trace/workloads/ctests/bdb_counter_multi_id_test.c diff --git a/examples/components/balls/trace/workloads/ctests/bdb_counter_nested_pair_test.c b/examples/prefab/balls/trace/workloads/ctests/bdb_counter_nested_pair_test.c similarity index 100% rename from examples/components/balls/trace/workloads/ctests/bdb_counter_nested_pair_test.c rename to examples/prefab/balls/trace/workloads/ctests/bdb_counter_nested_pair_test.c diff --git a/examples/components/balls/trace/workloads/ctests/bdb_counter_read_active_test.c b/examples/prefab/balls/trace/workloads/ctests/bdb_counter_read_active_test.c similarity index 100% rename from examples/components/balls/trace/workloads/ctests/bdb_counter_read_active_test.c rename to examples/prefab/balls/trace/workloads/ctests/bdb_counter_read_active_test.c diff --git a/examples/components/balls/trace/workloads/ctests/bdb_counter_start_stop_test.c b/examples/prefab/balls/trace/workloads/ctests/bdb_counter_start_stop_test.c similarity index 100% rename from examples/components/balls/trace/workloads/ctests/bdb_counter_start_stop_test.c rename to examples/prefab/balls/trace/workloads/ctests/bdb_counter_start_stop_test.c diff --git a/examples/components/balls/trace/workloads/isa/bdb_counter.h b/examples/prefab/balls/trace/workloads/isa/bdb_counter.h similarity index 81% rename from examples/components/balls/trace/workloads/isa/bdb_counter.h rename to examples/prefab/balls/trace/workloads/isa/bdb_counter.h index 0f37555a7..ac6963983 100644 --- a/examples/components/balls/trace/workloads/isa/bdb_counter.h +++ b/examples/prefab/balls/trace/workloads/isa/bdb_counter.h @@ -4,7 +4,7 @@ #include #include -// subcmd values +// subcmd values; 3..15 are illegal. STOP and READ need a started counter. #define BDB_CTR_START 0 #define BDB_CTR_STOP 1 #define BDB_CTR_READ 2 @@ -17,16 +17,16 @@ // Start counter ctr_id with user tag #define bdb_counter_start(ctr_id, tag) \ BUCKYBALL_INSTRUCTION_R_R(0, BDB_CTR_RS2(BDB_CTR_START, ctr_id, tag), \ - BB_FUNC7(BDB_COUNTER)) + FUNCT7(trace, BDB_COUNTER)) // Stop counter ctr_id, output elapsed to trace #define bdb_counter_stop(ctr_id) \ BUCKYBALL_INSTRUCTION_R_R(0, BDB_CTR_RS2(BDB_CTR_STOP, ctr_id, 0), \ - BB_FUNC7(BDB_COUNTER)) + FUNCT7(trace, BDB_COUNTER)) // Read counter ctr_id current value (non-destructive), output to trace #define bdb_counter_read(ctr_id) \ BUCKYBALL_INSTRUCTION_R_R(0, BDB_CTR_RS2(BDB_CTR_READ, ctr_id, 0), \ - BB_FUNC7(BDB_COUNTER)) + FUNCT7(trace, BDB_COUNTER)) #endif // _BDB_COUNTER_H_ diff --git a/examples/components/balls/trace/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/trace/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/trace/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/trace/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/trace/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/trace/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/components/balls/trace/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/trace/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/prefab/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir b/examples/prefab/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir new file mode 100644 index 000000000..a7dbb5b6a --- /dev/null +++ b/examples/prefab/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop.mlir @@ -0,0 +1,8 @@ +func.func @main() -> i8 { + %zero8 = arith.constant 0 : i8 + %tag = arith.constant 48879 : i64 + trace_ball.counter_start %tag <{counter = 3 : i64}> + trace_ball.counter_read <{counter = 3 : i64}> + trace_ball.counter_stop <{counter = 3 : i64}> + return %zero8 : i8 +} diff --git a/examples/components/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop_main.cpp b/examples/prefab/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop_main.cpp similarity index 100% rename from examples/components/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop_main.cpp rename to examples/prefab/balls/trace/workloads/mlir_tests/ball/bdb_counter_start_read_stop_main.cpp diff --git a/examples/components/balls/transpose/arch/README.md b/examples/prefab/balls/transpose/arch/README.md similarity index 95% rename from examples/components/balls/transpose/arch/README.md rename to examples/prefab/balls/transpose/arch/README.md index 4d0628d1a..34a9d6b90 100644 --- a/examples/components/balls/transpose/arch/README.md +++ b/examples/prefab/balls/transpose/arch/README.md @@ -2,7 +2,7 @@ ## Overview -This directory implements Buckyball's matrix transpose accelerator for matrix transpose operations. Located at `examples/components/balls/transpose/arch`, it serves as a matrix transpose accelerator supporting pipelined transpose operations. +This directory implements Buckyball's matrix transpose accelerator for matrix transpose operations. Located at `examples/prefab/balls/transpose/arch`, it serves as a matrix transpose accelerator supporting pipelined transpose operations. Core components: - **Transpose.scala**: Pipelined transposer implementation diff --git a/examples/components/balls/transpose/arch/Transpose.scala b/examples/prefab/balls/transpose/arch/Transpose.scala similarity index 100% rename from examples/components/balls/transpose/arch/Transpose.scala rename to examples/prefab/balls/transpose/arch/Transpose.scala diff --git a/examples/components/balls/transpose/arch/TransposeBall.scala b/examples/prefab/balls/transpose/arch/TransposeBall.scala similarity index 100% rename from examples/components/balls/transpose/arch/TransposeBall.scala rename to examples/prefab/balls/transpose/arch/TransposeBall.scala diff --git a/examples/components/balls/transpose/configs/TransposeBallParam.scala b/examples/prefab/balls/transpose/configs/TransposeBallParam.scala similarity index 100% rename from examples/components/balls/transpose/configs/TransposeBallParam.scala rename to examples/prefab/balls/transpose/configs/TransposeBallParam.scala diff --git a/examples/components/balls/transpose/configs/default.toml b/examples/prefab/balls/transpose/configs/default.toml similarity index 100% rename from examples/components/balls/transpose/configs/default.toml rename to examples/prefab/balls/transpose/configs/default.toml diff --git a/examples/components/balls/transpose/configs/transpose_params.proto b/examples/prefab/balls/transpose/configs/transpose_params.proto similarity index 100% rename from examples/components/balls/transpose/configs/transpose_params.proto rename to examples/prefab/balls/transpose/configs/transpose_params.proto diff --git a/examples/components/balls/transpose/emu/src/dispatch.rs b/examples/prefab/balls/transpose/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/transpose/emu/src/dispatch.rs rename to examples/prefab/balls/transpose/emu/src/dispatch.rs diff --git a/examples/components/balls/transpose/emu/src/lib.rs b/examples/prefab/balls/transpose/emu/src/lib.rs similarity index 100% rename from examples/components/balls/transpose/emu/src/lib.rs rename to examples/prefab/balls/transpose/emu/src/lib.rs diff --git a/examples/components/balls/transpose/emu/src/transpose.rs b/examples/prefab/balls/transpose/emu/src/transpose.rs similarity index 94% rename from examples/components/balls/transpose/emu/src/transpose.rs rename to examples/prefab/balls/transpose/emu/src/transpose.rs index fe91ead13..d088b3284 100644 --- a/examples/components/balls/transpose/emu/src/transpose.rs +++ b/examples/prefab/balls/transpose/emu/src/transpose.rs @@ -1,8 +1,9 @@ use bebop_bemu::thread::banks::{bank_row_bytes, bank_size, bank_width}; -use super::decode::{pbank_group, rs1_b0, rs1_b2, rs1_iter}; +use super::decode::{pbank_group, rs1_b0, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::ExecContext; pub(crate) fn exec_transpose(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(!rs1_read_old(xs1), "transpose: reads no old destination; rs1[29] must be clear"); let op1 = rs1_b0(xs1); let wr = rs1_b2(xs1); let iter = rs1_iter(xs1) as usize; diff --git a/examples/prefab/balls/transpose/stack/src/transposeOps.cpp b/examples/prefab/balls/transpose/stack/src/transposeOps.cpp new file mode 100644 index 000000000..3cf6f0d9a --- /dev/null +++ b/examples/prefab/balls/transpose/stack/src/transposeOps.cpp @@ -0,0 +1,73 @@ +//===- transposeOps.cpp - TransposeBall operations ------------------------===// +// +// The dialect of TransposeBall, its virtual operations building their physical +// ones and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "transposeOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::transpose_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "transposeDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "transposeOps.cpp.inc" + +void TransposeBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "transposeOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect([](DialectRegistry ®istry) { + registry.insert(); + }), + true); +} // namespace + +SmallVector BankTransposeI8Op::getSourceBanks() { + return {getInput()}; +} +Value BankTransposeI8Op::getDestinationBank() { return getOutput(); } +bool BankTransposeI8Op::readsDestination() { return false; } +Operation *BankTransposeI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return TransposeI8Op::create(builder, getLoc(), sources[0], destination, + getRows(), getCols()); +} + +SmallVector TransposeI8Op::getSourceBanks() { + return {int64_t(getInput())}; +} +int64_t TransposeI8Op::getDestinationBank() { return getOutput(); } +LogicalResult TransposeI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t input = getInput(), output = getOutput(), rows = getRows(), + cols = getCols(); + if (input == output) + return emitOpError("transpose_i8 requires distinct input and output banks"); + const auto &target = buckyball_target::getBuckyballTarget(); + int64_t lanes = target.bankWidthBits / 8; + if (lanes <= 0 || target.bankWidthBits % 8 || rows <= 0 || + rows >= (1LL << 33) || rows > target.bankDepth || cols <= 0 || + cols % lanes || cols / lanes > 31) + return emitOpError("transpose_i8 shape exceeds the bank or ISA geometry"); + emitInstruction( + builder, loc, buckyball_target::getBuckyballFunct7("TRANSPOSE"), + constantI64(builder, loc, bankFields({input}, output, false, rows)), + constantI64(builder, loc, 8)); + return success(); +} diff --git a/examples/prefab/balls/transpose/stack/src/transposeOps.h b/examples/prefab/balls/transpose/stack/src/transposeOps.h new file mode 100644 index 000000000..36ffea777 --- /dev/null +++ b/examples/prefab/balls/transpose/stack/src/transposeOps.h @@ -0,0 +1,20 @@ +//===- transposeOps.h - TransposeBall operations --------------------------===// + +#ifndef TRANSPOSE_OPS_H +#define TRANSPOSE_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "transposeDialect.h.inc" + +#define GET_OP_CLASSES +#include "transposeOps.h.inc" + +#endif // TRANSPOSE_OPS_H diff --git a/examples/prefab/balls/transpose/stack/src/transposeOps.td b/examples/prefab/balls/transpose/stack/src/transposeOps.td new file mode 100644 index 000000000..f18341b58 --- /dev/null +++ b/examples/prefab/balls/transpose/stack/src/transposeOps.td @@ -0,0 +1,42 @@ +//===- transposeOps.td - TransposeBall operations ----------*- tablegen -*-===// +// +// transpose_i8: both banks have cols / (bankWidthBits / 8) columns. The +// instruction packs the transposed bytes into the same bank geometry, with +// `rows` bank rows. +// +//===----------------------------------------------------------------------===// + +#ifndef TRANSPOSE_OPS +#define TRANSPOSE_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def TransposeBall_Dialect : Dialect { + let name = "transpose_ball"; + let cppNamespace = "::buddy::transpose_ball"; + let summary = "The operations of TransposeBall"; +} + +class TransposeBall_Op traits = []> + : Op])>; + +def BankTransposeI8Op : TransposeBall_Op<"bank_transpose_i8", + [DeclareOpInterfaceMethods]> { + let summary = "transpose_i8 on virtual banks"; + let arguments = (ins I64:$input, I64:$output, I64Attr:$rows, I64Attr:$cols); + let results = (outs I64:$outputOut); + let assemblyFormat = "$input `->` $output prop-dict attr-dict"; +} + +def TransposeI8Op : TransposeBall_Op<"transpose_i8", + [DeclareOpInterfaceMethods]> { + let summary = "transpose_i8 on physical banks"; + let arguments = (ins I64Attr:$input, I64Attr:$output, I64Attr:$rows, + I64Attr:$cols); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // TRANSPOSE_OPS diff --git a/examples/components/balls/transpose/verify/README.md b/examples/prefab/balls/transpose/verify/README.md similarity index 100% rename from examples/components/balls/transpose/verify/README.md rename to examples/prefab/balls/transpose/verify/README.md diff --git a/examples/components/balls/transpose/verify/casegen/Cargo.lock b/examples/prefab/balls/transpose/verify/casegen/Cargo.lock similarity index 100% rename from examples/components/balls/transpose/verify/casegen/Cargo.lock rename to examples/prefab/balls/transpose/verify/casegen/Cargo.lock diff --git a/examples/components/balls/transpose/verify/casegen/Cargo.toml b/examples/prefab/balls/transpose/verify/casegen/Cargo.toml similarity index 100% rename from examples/components/balls/transpose/verify/casegen/Cargo.toml rename to examples/prefab/balls/transpose/verify/casegen/Cargo.toml diff --git a/examples/components/balls/transpose/verify/casegen/src/casegen.rs b/examples/prefab/balls/transpose/verify/casegen/src/casegen.rs similarity index 100% rename from examples/components/balls/transpose/verify/casegen/src/casegen.rs rename to examples/prefab/balls/transpose/verify/casegen/src/casegen.rs diff --git a/examples/components/balls/transpose/verify/casegen/src/lib.rs b/examples/prefab/balls/transpose/verify/casegen/src/lib.rs similarity index 100% rename from examples/components/balls/transpose/verify/casegen/src/lib.rs rename to examples/prefab/balls/transpose/verify/casegen/src/lib.rs diff --git a/examples/components/balls/transpose/verify/casegen/src/model.rs b/examples/prefab/balls/transpose/verify/casegen/src/model.rs similarity index 100% rename from examples/components/balls/transpose/verify/casegen/src/model.rs rename to examples/prefab/balls/transpose/verify/casegen/src/model.rs diff --git a/examples/components/balls/transpose/verify/filelists/transpose_ball.f b/examples/prefab/balls/transpose/verify/filelists/transpose_ball.f similarity index 100% rename from examples/components/balls/transpose/verify/filelists/transpose_ball.f rename to examples/prefab/balls/transpose/verify/filelists/transpose_ball.f diff --git a/examples/components/balls/transpose/verify/src/common/transpose_defs.svh b/examples/prefab/balls/transpose/verify/src/common/transpose_defs.svh similarity index 100% rename from examples/components/balls/transpose/verify/src/common/transpose_defs.svh rename to examples/prefab/balls/transpose/verify/src/common/transpose_defs.svh diff --git a/examples/components/balls/transpose/verify/src/common/transpose_items.svh b/examples/prefab/balls/transpose/verify/src/common/transpose_items.svh similarity index 100% rename from examples/components/balls/transpose/verify/src/common/transpose_items.svh rename to examples/prefab/balls/transpose/verify/src/common/transpose_items.svh diff --git a/examples/components/balls/transpose/verify/src/cov/transpose_cov.svh b/examples/prefab/balls/transpose/verify/src/cov/transpose_cov.svh similarity index 100% rename from examples/components/balls/transpose/verify/src/cov/transpose_cov.svh rename to examples/prefab/balls/transpose/verify/src/cov/transpose_cov.svh diff --git a/examples/components/balls/transpose/verify/src/env/transpose_env.svh b/examples/prefab/balls/transpose/verify/src/env/transpose_env.svh similarity index 100% rename from examples/components/balls/transpose/verify/src/env/transpose_env.svh rename to examples/prefab/balls/transpose/verify/src/env/transpose_env.svh diff --git a/examples/components/balls/transpose/verify/src/env/transpose_scoreboard.svh b/examples/prefab/balls/transpose/verify/src/env/transpose_scoreboard.svh similarity index 100% rename from examples/components/balls/transpose/verify/src/env/transpose_scoreboard.svh rename to examples/prefab/balls/transpose/verify/src/env/transpose_scoreboard.svh diff --git a/examples/components/balls/transpose/verify/src/pkg/transpose_pkg.sv b/examples/prefab/balls/transpose/verify/src/pkg/transpose_pkg.sv similarity index 100% rename from examples/components/balls/transpose/verify/src/pkg/transpose_pkg.sv rename to examples/prefab/balls/transpose/verify/src/pkg/transpose_pkg.sv diff --git a/examples/components/balls/transpose/verify/src/seq/transpose_sequences.svh b/examples/prefab/balls/transpose/verify/src/seq/transpose_sequences.svh similarity index 100% rename from examples/components/balls/transpose/verify/src/seq/transpose_sequences.svh rename to examples/prefab/balls/transpose/verify/src/seq/transpose_sequences.svh diff --git a/examples/components/balls/transpose/verify/src/tb_top.sv b/examples/prefab/balls/transpose/verify/src/tb_top.sv similarity index 100% rename from examples/components/balls/transpose/verify/src/tb_top.sv rename to examples/prefab/balls/transpose/verify/src/tb_top.sv diff --git a/examples/components/balls/transpose/verify/src/tests/transpose_test.svh b/examples/prefab/balls/transpose/verify/src/tests/transpose_test.svh similarity index 100% rename from examples/components/balls/transpose/verify/src/tests/transpose_test.svh rename to examples/prefab/balls/transpose/verify/src/tests/transpose_test.svh diff --git a/examples/components/balls/transpose/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/transpose/workloads/ctests/CMakeLists.txt similarity index 100% rename from examples/components/balls/transpose/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/transpose/workloads/ctests/CMakeLists.txt diff --git a/examples/components/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c b/examples/prefab/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c similarity index 97% rename from examples/components/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c rename to examples/prefab/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c index 6b28564cb..b60c6a901 100644 --- a/examples/components/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c +++ b/examples/prefab/balls/transpose/workloads/ctests/transpose_i32_8x16_test.c @@ -8,7 +8,7 @@ #define ROWS 8 #define N_GROUPS 4 -#define COLS (N_GROUPS * ((BANK_WIDTH) / 32)) +#define COLS (N_GROUPS * ((THREAD_MEM_BANK_WIDTH) / 32)) #define ELEM_BITS 32 static int32_t input_matrix[ROWS * COLS] __attribute__((aligned(64))) = { diff --git a/examples/components/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c b/examples/prefab/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c similarity index 97% rename from examples/components/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c rename to examples/prefab/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c index 55d6cf435..16372179c 100644 --- a/examples/components/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c +++ b/examples/prefab/balls/transpose/workloads/ctests/transpose_i8_16x16_test.c @@ -6,7 +6,7 @@ #define ROWS 16 #define N_GROUPS 1 -#define COLS (N_GROUPS * ((BANK_WIDTH) / 8)) +#define COLS (N_GROUPS * ((THREAD_MEM_BANK_WIDTH) / 8)) #define ELEM_BITS 8 static elem_t input_matrix[ROWS * COLS] __attribute__((aligned(64))) = { diff --git a/examples/components/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c b/examples/prefab/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c similarity index 98% rename from examples/components/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c rename to examples/prefab/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c index 4aa21aab2..9c10139a8 100644 --- a/examples/components/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c +++ b/examples/prefab/balls/transpose/workloads/ctests/transpose_i8_16x32_test.c @@ -6,7 +6,7 @@ #define ROWS 16 #define N_GROUPS 2 -#define COLS (N_GROUPS * ((BANK_WIDTH) / 8)) +#define COLS (N_GROUPS * ((THREAD_MEM_BANK_WIDTH) / 8)) #define ELEM_BITS 8 static elem_t input_matrix[ROWS * COLS] __attribute__((aligned(64))) = { diff --git a/examples/components/balls/transpose/workloads/isa/transpose.h b/examples/prefab/balls/transpose/workloads/isa/transpose.h similarity index 87% rename from examples/components/balls/transpose/workloads/isa/transpose.h rename to examples/prefab/balls/transpose/workloads/isa/transpose.h index 4c219ae72..2b0b07060 100644 --- a/examples/components/balls/transpose/workloads/isa/transpose.h +++ b/examples/prefab/balls/transpose/workloads/isa/transpose.h @@ -10,6 +10,6 @@ #define bb_transpose(op1_bank_id, wr_bank_id, iter, elem_bits) \ BUCKYBALL_INSTRUCTION_R_R( \ (BB_BANK0(op1_bank_id) | BB_BANK2(wr_bank_id) | BB_ITER(iter)), \ - (FIELD((uint64_t)(elem_bits), 0, 7)), BB_FUNC7(TRANSPOSE)) + (FIELD((uint64_t)(elem_bits), 0, 7)), FUNCT7(transpose, TRANSPOSE)) #endif // _BB_TRANSPOSE_H_ diff --git a/examples/components/balls/transpose/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/transpose/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/transpose/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/transpose/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/components/balls/transpose/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/transpose/workloads/mlir_tests/ball/CMakeLists.txt similarity index 100% rename from examples/components/balls/transpose/workloads/mlir_tests/ball/CMakeLists.txt rename to examples/prefab/balls/transpose/workloads/mlir_tests/ball/CMakeLists.txt diff --git a/examples/components/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir b/examples/prefab/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir similarity index 92% rename from examples/components/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir rename to examples/prefab/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir index f8c7f99da..7c2fbf3b2 100644 --- a/examples/components/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir +++ b/examples/prefab/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8.mlir @@ -26,7 +26,7 @@ func.func @main() -> i8 { buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> buckyball.mvin %input <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> - buckyball.custom "transpose_i8" <{sources = array, destination = 2 : i64}> {rows = 16 : i64, cols = 16 : i64} + transpose_ball.transpose_i8 <{input = 1 : i64, output = 2 : i64, rows = 16 : i64, cols = 16 : i64}> buckyball.mvout %output <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> diff --git a/examples/components/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8_main.cpp b/examples/prefab/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8_main.cpp similarity index 100% rename from examples/components/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8_main.cpp rename to examples/prefab/balls/transpose/workloads/mlir_tests/ball/transpose_16x16_i8_main.cpp diff --git a/examples/components/balls/transpose/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/transpose/workloads/mlir_tests/bank/CMakeLists.txt similarity index 100% rename from examples/components/balls/transpose/workloads/mlir_tests/bank/CMakeLists.txt rename to examples/prefab/balls/transpose/workloads/mlir_tests/bank/CMakeLists.txt diff --git a/examples/components/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in b/examples/prefab/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in similarity index 92% rename from examples/components/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in rename to examples/prefab/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in index fdeab0431..2b691dc07 100644 --- a/examples/components/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in +++ b/examples/prefab/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix.mlir.in @@ -21,7 +21,7 @@ func.func @main() -> i8 { %input_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %output_bank = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> %loaded = buckyball.bank_mvin %input, %input_bank <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> - %result = buckyball.bank_custom "transpose_i8" (%loaded) -> %output_bank {rows = @BB_BANK_LINES@ : i64, cols = @BB_BANK_I8_LANES@ : i64} + %result = transpose_ball.bank_transpose_i8 %loaded -> %output_bank <{rows = @BB_BANK_LINES@ : i64, cols = @BB_BANK_I8_LANES@ : i64}> %stored = buckyball.bank_mvout %output, %result <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> : memref<@BB_BANK_LINES@x@BB_BANK_I8_LANES@xi8> buckyball.bank_release %loaded buckyball.bank_release %stored diff --git a/examples/components/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp b/examples/prefab/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp similarity index 67% rename from examples/components/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp rename to examples/prefab/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp index ef75bf830..efe2b5607 100644 --- a/examples/components/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp +++ b/examples/prefab/balls/transpose/workloads/mlir_tests/bank/transpose_i8_packed_matrix_main.cpp @@ -1,23 +1,22 @@ -#include #include #include #include extern "C" void check_result(int8_t *, int8_t *data, int64_t offset, int64_t rows, int64_t lanes, int64_t stride, int64_t lane_stride) { - if (rows != BANK_LINES || lanes != BANK_WIDTH / 8 || stride != lanes || - lane_stride != 1) + if (rows != THREAD_MEM_BANK_ENTRIES || lanes != THREAD_MEM_BANK_WIDTH / 8 || + stride != lanes || lane_stride != 1) exit(1); for (int row = 0; row < rows; ++row) for (int lane = 0; lane < lanes; ++lane) { int flat = row * lanes + lane; - int column = flat / BANK_LINES; - int source_row = flat % BANK_LINES; + int column = flat / THREAD_MEM_BANK_ENTRIES; + int source_row = flat % THREAD_MEM_BANK_ENTRIES; int8_t expected = (int8_t)(source_row * lanes + column); if (data[offset + flat] != expected) { printf("transpose bank mismatch row=%d lane=%d\n", row, lane); exit(1); } } - printf("transpose bank PASS rows=%d\n", BANK_LINES); + printf("transpose bank PASS rows=%d\n", THREAD_MEM_BANK_ENTRIES); } diff --git a/examples/components/balls/vector/arch/README.md b/examples/prefab/balls/vector/arch/README.md similarity index 100% rename from examples/components/balls/vector/arch/README.md rename to examples/prefab/balls/vector/arch/README.md diff --git a/examples/components/balls/vector/arch/VecBall.scala b/examples/prefab/balls/vector/arch/VecBall.scala similarity index 90% rename from examples/components/balls/vector/arch/VecBall.scala rename to examples/prefab/balls/vector/arch/VecBall.scala index fd7c2af59..2b137a5eb 100644 --- a/examples/components/balls/vector/arch/VecBall.scala +++ b/examples/prefab/balls/vector/arch/VecBall.scala @@ -39,10 +39,6 @@ class VecBall( Instantiate(new VecUnit(fp, mp, bp, ball, ballParam)) vecUnit.io.cmdReq <> blink.cmdReq - when(blink.cmdReq.fire) { - val cmd = blink.cmdReq.bits - assert(cmd.op1_group === 0.U && cmd.op2_group === 0.U && cmd.wr_group === 0.U, "VecBall has no group offsets") - } vecUnit.io.cmdResp <> blink.cmdResp for (i <- 0 until inBW) { diff --git a/examples/components/balls/vector/arch/VecCtrlUnit.scala b/examples/prefab/balls/vector/arch/VecCtrlUnit.scala similarity index 100% rename from examples/components/balls/vector/arch/VecCtrlUnit.scala rename to examples/prefab/balls/vector/arch/VecCtrlUnit.scala diff --git a/examples/components/balls/vector/arch/VecEXUnit.scala b/examples/prefab/balls/vector/arch/VecEXUnit.scala similarity index 100% rename from examples/components/balls/vector/arch/VecEXUnit.scala rename to examples/prefab/balls/vector/arch/VecEXUnit.scala diff --git a/examples/components/balls/vector/arch/VecLoadUnit.scala b/examples/prefab/balls/vector/arch/VecLoadUnit.scala similarity index 100% rename from examples/components/balls/vector/arch/VecLoadUnit.scala rename to examples/prefab/balls/vector/arch/VecLoadUnit.scala diff --git a/examples/components/balls/vector/arch/VecMat16.scala b/examples/prefab/balls/vector/arch/VecMat16.scala similarity index 75% rename from examples/components/balls/vector/arch/VecMat16.scala rename to examples/prefab/balls/vector/arch/VecMat16.scala index 490a6e713..6ecd095e9 100644 --- a/examples/components/balls/vector/arch/VecMat16.scala +++ b/examples/prefab/balls/vector/arch/VecMat16.scala @@ -68,31 +68,39 @@ class VecMat16( Enum(6) val state = RegInit(sIdle) - val robIdReg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) - val op1Bank = RegInit(0.U(frontendParam.bank_id_len.W)) - val op2Bank = RegInit(0.U(frontendParam.bank_id_len.W)) - val wrBank = RegInit(0.U(frontendParam.bank_id_len.W)) - val iterReg = RegInit(0.U(frontendParam.iter_len.W)) - val readRow = RegInit(0.U(frontendParam.iter_len.W)) - val aReqDone = RegInit(false.B) - val bReqDone = RegInit(false.B) - val aRespDone = RegInit(false.B) - val bRespDone = RegInit(false.B) - val aWord = RegInit(0.U(memParam.bankWidth.W)) - val bWord = RegInit(0.U(memParam.bankWidth.W)) - val acc = Reg(Vec(lane, Vec(lane, UInt(accW.W)))) + val robIdReg = RegInit(0.U(log2Up(frontendParam.rob_entries).W)) + val op1Bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val op2Bank = RegInit(0.U(frontendParam.bank_id_len.W)) + val wrBank = RegInit(0.U(frontendParam.bank_id_len.W)) + // Renamed bank holding the old C, which the instruction accumulates into. + val wrReadBank = RegInit(0.U(frontendParam.bank_id_len.W)) + // Group bases of each operand; the old C shares the destination layout. + val op1Group = RegInit(0.U(memParam.groupIdWidth.W)) + val op2Group = RegInit(0.U(memParam.groupIdWidth.W)) + val wrGroup = RegInit(0.U(memParam.groupIdWidth.W)) + val iterReg = RegInit(0.U(frontendParam.iter_len.W)) + val readRow = RegInit(0.U(frontendParam.iter_len.W)) + val aReqDone = RegInit(false.B) + val bReqDone = RegInit(false.B) + val aRespDone = RegInit(false.B) + val bRespDone = RegInit(false.B) + val aWord = RegInit(0.U(memParam.bankWidth.W)) + val bWord = RegInit(0.U(memParam.bankWidth.W)) + val acc = Reg(Vec(lane, Vec(lane, UInt(accW.W)))) val accLoad = Instantiate(new VecMat16AccLoad(frontendParam, memParam, routingParam, accW, lane, ballParam)) val store = Instantiate(new VecMat16Store(frontendParam, memParam, routingParam, accW, lane, ballParam)) - accLoad.io.start := false.B - accLoad.io.wrBank := wrBank - accLoad.io.robId := robIdReg - store.io.start := false.B - store.io.wrBank := wrBank - store.io.robId := robIdReg - store.io.acc := acc + accLoad.io.start := false.B + accLoad.io.wrBank := wrReadBank + accLoad.io.wrGroup := wrGroup + accLoad.io.robId := robIdReg + store.io.start := false.B + store.io.wrBank := wrBank + store.io.wrGroup := wrGroup + store.io.robId := robIdReg + store.io.acc := acc for (p <- 0 until 2) { accLoad.io.bankRead(p).io.req.ready := io.bankRead(p).io.req.ready accLoad.io.bankRead(p).io.resp.valid := io.bankRead(p).io.resp.valid @@ -110,7 +118,7 @@ class VecMat16( io.status.idle := state === sIdle io.status.running := state =/= sIdle && state =/= sDone VecMat16Tie.offRead(io.bankRead, robIdReg) - VecMat16Tie.offWrite(io.bankWrite, robIdReg, wrBank, memParam) + VecMat16Tie.offWrite(io.bankWrite, robIdReg, wrBank, wrGroup, memParam) switch(state) { is(sIdle) { @@ -120,13 +128,21 @@ class VecMat16( op1Bank := cmd.op1_bank op2Bank := cmd.op2_bank wrBank := cmd.wr_bank + wrReadBank := cmd.wr_read_bank + op1Group := cmd.op1_group + op2Group := cmd.op2_group + wrGroup := cmd.wr_group iterReg := cmd.iter readRow := 0.U aReqDone := false.B bReqDone := false.B aRespDone := false.B bRespDone := false.B - assert(cmd.iter > 0.U, "VecMat16 iter must be > 0") + assert( + cmd.iter =/= 0.U && cmd.iter <= memParam.bankEntries.U, + "VecMat16 iter must be 1..bank depth" + ) + assert(cmd.rs2 === 0.U, "VecMat16 rs2 is reserved") assert( cmd.op1_col === 1.U && cmd.op2_col === 1.U && cmd.wr_col === 4.U, "VecMat16 unsupported bank layout" @@ -150,9 +166,11 @@ class VecMat16( is(sReadReq) { io.bankRead(0).bank_id := op1Bank + io.bankRead(0).group_id := op1Group io.bankRead(0).io.req.valid := !aReqDone io.bankRead(0).io.req.bits.addr := readRow io.bankRead(1).bank_id := op2Bank + io.bankRead(1).group_id := op2Group io.bankRead(1).io.req.valid := !bReqDone io.bankRead(1).io.req.bits.addr := readRow when(io.bankRead(0).io.req.fire)(aReqDone := true.B) @@ -172,7 +190,9 @@ class VecMat16( is(sReadResp) { io.bankRead(0).bank_id := op1Bank + io.bankRead(0).group_id := op1Group io.bankRead(1).bank_id := op2Bank + io.bankRead(1).group_id := op2Group io.bankRead(0).io.resp.ready := !aRespDone io.bankRead(1).io.resp.ready := !bRespDone val aFire = io.bankRead(0).io.resp.fire diff --git a/examples/components/balls/vector/arch/VecMat16AccLoad.scala b/examples/prefab/balls/vector/arch/VecMat16AccLoad.scala similarity index 91% rename from examples/components/balls/vector/arch/VecMat16AccLoad.scala rename to examples/prefab/balls/vector/arch/VecMat16AccLoad.scala index d2c148e09..4603dc242 100644 --- a/examples/components/balls/vector/arch/VecMat16AccLoad.scala +++ b/examples/prefab/balls/vector/arch/VecMat16AccLoad.scala @@ -24,9 +24,10 @@ class VecMat16AccLoad( @public val io = IO(new Bundle { - val start = Input(Bool()) - val wrBank = Input(UInt(frontendParam.bank_id_len.W)) - val robId = Input(UInt(log2Up(frontendParam.rob_entries).W)) + val start = Input(Bool()) + val wrBank = Input(UInt(frontendParam.bank_id_len.W)) + val wrGroup = Input(UInt(memParam.groupIdWidth.W)) + val robId = Input(UInt(log2Up(frontendParam.rob_entries).W)) val bankRead = Vec( 2, @@ -80,7 +81,7 @@ class VecMat16AccLoad( is(sReq) { for (p <- 0 until 2) { io.bankRead(p).bank_id := io.wrBank - io.bankRead(p).group_id := loadGroup(p) + io.bankRead(p).group_id := io.wrGroup + loadGroup(p) io.bankRead(p).io.req.valid := !reqDone(p) io.bankRead(p).io.req.bits.addr := loadRow when(io.bankRead(p).io.req.fire) { @@ -100,7 +101,7 @@ class VecMat16AccLoad( is(sResp) { for (p <- 0 until 2) { io.bankRead(p).bank_id := io.wrBank - io.bankRead(p).group_id := loadGroup(p) + io.bankRead(p).group_id := io.wrGroup + loadGroup(p) io.bankRead(p).io.resp.ready := !respDone(p) when(io.bankRead(p).io.resp.fire) { val g = loadGroup(p) diff --git a/examples/components/balls/vector/arch/VecMat16Store.scala b/examples/prefab/balls/vector/arch/VecMat16Store.scala similarity index 88% rename from examples/components/balls/vector/arch/VecMat16Store.scala rename to examples/prefab/balls/vector/arch/VecMat16Store.scala index f64010b60..2517f4dbe 100644 --- a/examples/components/balls/vector/arch/VecMat16Store.scala +++ b/examples/prefab/balls/vector/arch/VecMat16Store.scala @@ -24,10 +24,11 @@ class VecMat16Store( @public val io = IO(new Bundle { - val start = Input(Bool()) - val wrBank = Input(UInt(frontendParam.bank_id_len.W)) - val robId = Input(UInt(log2Up(frontendParam.rob_entries).W)) - val acc = Input(Vec(lane, Vec(lane, UInt(accWidth.W)))) + val start = Input(Bool()) + val wrBank = Input(UInt(frontendParam.bank_id_len.W)) + val wrGroup = Input(UInt(memParam.groupIdWidth.W)) + val robId = Input(UInt(log2Up(frontendParam.rob_entries).W)) + val acc = Input(Vec(lane, Vec(lane, UInt(accWidth.W)))) val bankWrite = Vec( 4, @@ -62,7 +63,7 @@ class VecMat16Store( for (i <- 0 until 4) { io.bankWrite(i).rob_id := io.robId io.bankWrite(i).bank_id := io.wrBank - io.bankWrite(i).group_id := i.U + io.bankWrite(i).group_id := io.wrGroup + i.U io.bankWrite(i).io.req.valid := false.B io.bankWrite(i).io.req.bits.addr := 0.U io.bankWrite(i).io.req.bits.data := 0.U @@ -85,7 +86,7 @@ class VecMat16Store( is(sReq) { for (g <- 0 until 4) { io.bankWrite(g).bank_id := io.wrBank - io.bankWrite(g).group_id := g.U + io.bankWrite(g).group_id := io.wrGroup + g.U io.bankWrite(g).io.req.valid := !writeIssued(g) io.bankWrite(g).io.req.bits.addr := writeRow io.bankWrite(g).io.req.bits.data := writeData(g) @@ -109,7 +110,7 @@ class VecMat16Store( is(sResp) { for (g <- 0 until 4) { io.bankWrite(g).bank_id := io.wrBank - io.bankWrite(g).group_id := g.U + io.bankWrite(g).group_id := io.wrGroup + g.U io.bankWrite(g).io.resp.ready := !writeAcked(g) when(io.bankWrite(g).io.resp.fire) { writeAcked(g) := true.B diff --git a/examples/components/balls/vector/arch/VecMat16Tie.scala b/examples/prefab/balls/vector/arch/VecMat16Tie.scala similarity index 96% rename from examples/components/balls/vector/arch/VecMat16Tie.scala rename to examples/prefab/balls/vector/arch/VecMat16Tie.scala index 430399f73..b88c6467c 100644 --- a/examples/components/balls/vector/arch/VecMat16Tie.scala +++ b/examples/prefab/balls/vector/arch/VecMat16Tie.scala @@ -23,12 +23,13 @@ object VecMat16Tie { ports: Vec[BankWrite], robId: UInt, wrBank: UInt, + wrGroup: UInt, memParam: MemDomainParam ): Unit = { for ((p, i) <- ports.zipWithIndex) { p.rob_id := robId p.bank_id := wrBank - p.group_id := i.U + p.group_id := wrGroup + i.U p.io.req.valid := false.B p.io.req.bits.addr := 0.U p.io.req.bits.data := 0.U diff --git a/examples/components/balls/vector/arch/VecStoreUnit.scala b/examples/prefab/balls/vector/arch/VecStoreUnit.scala similarity index 100% rename from examples/components/balls/vector/arch/VecStoreUnit.scala rename to examples/prefab/balls/vector/arch/VecStoreUnit.scala diff --git a/examples/components/balls/vector/arch/VecUnit.scala b/examples/prefab/balls/vector/arch/VecUnit.scala similarity index 100% rename from examples/components/balls/vector/arch/VecUnit.scala rename to examples/prefab/balls/vector/arch/VecUnit.scala diff --git a/examples/components/balls/vector/arch/bond/BondWrapper.scala b/examples/prefab/balls/vector/arch/bond/BondWrapper.scala similarity index 100% rename from examples/components/balls/vector/arch/bond/BondWrapper.scala rename to examples/prefab/balls/vector/arch/bond/BondWrapper.scala diff --git a/examples/components/balls/vector/arch/bond/README.md b/examples/prefab/balls/vector/arch/bond/README.md similarity index 100% rename from examples/components/balls/vector/arch/bond/README.md rename to examples/prefab/balls/vector/arch/bond/README.md diff --git a/examples/components/balls/vector/arch/bond/vvv.scala b/examples/prefab/balls/vector/arch/bond/vvv.scala similarity index 100% rename from examples/components/balls/vector/arch/bond/vvv.scala rename to examples/prefab/balls/vector/arch/bond/vvv.scala diff --git a/examples/components/balls/vector/arch/op/README.md b/examples/prefab/balls/vector/arch/op/README.md similarity index 100% rename from examples/components/balls/vector/arch/op/README.md rename to examples/prefab/balls/vector/arch/op/README.md diff --git a/examples/components/balls/vector/arch/op/cascade.scala b/examples/prefab/balls/vector/arch/op/cascade.scala similarity index 100% rename from examples/components/balls/vector/arch/op/cascade.scala rename to examples/prefab/balls/vector/arch/op/cascade.scala diff --git a/examples/components/balls/vector/arch/op/mul.scala b/examples/prefab/balls/vector/arch/op/mul.scala similarity index 100% rename from examples/components/balls/vector/arch/op/mul.scala rename to examples/prefab/balls/vector/arch/op/mul.scala diff --git a/examples/components/balls/vector/arch/thread/BaseThread.scala b/examples/prefab/balls/vector/arch/thread/BaseThread.scala similarity index 100% rename from examples/components/balls/vector/arch/thread/BaseThread.scala rename to examples/prefab/balls/vector/arch/thread/BaseThread.scala diff --git a/examples/components/balls/vector/arch/thread/CasThread.scala b/examples/prefab/balls/vector/arch/thread/CasThread.scala similarity index 100% rename from examples/components/balls/vector/arch/thread/CasThread.scala rename to examples/prefab/balls/vector/arch/thread/CasThread.scala diff --git a/examples/components/balls/vector/arch/thread/MulThread.scala b/examples/prefab/balls/vector/arch/thread/MulThread.scala similarity index 100% rename from examples/components/balls/vector/arch/thread/MulThread.scala rename to examples/prefab/balls/vector/arch/thread/MulThread.scala diff --git a/examples/components/balls/vector/arch/thread/README.md b/examples/prefab/balls/vector/arch/thread/README.md similarity index 100% rename from examples/components/balls/vector/arch/thread/README.md rename to examples/prefab/balls/vector/arch/thread/README.md diff --git a/examples/components/balls/vector/arch/warp/MeshWarp.scala b/examples/prefab/balls/vector/arch/warp/MeshWarp.scala similarity index 100% rename from examples/components/balls/vector/arch/warp/MeshWarp.scala rename to examples/prefab/balls/vector/arch/warp/MeshWarp.scala diff --git a/examples/components/balls/vector/arch/warp/README.md b/examples/prefab/balls/vector/arch/warp/README.md similarity index 100% rename from examples/components/balls/vector/arch/warp/README.md rename to examples/prefab/balls/vector/arch/warp/README.md diff --git a/examples/components/balls/vector/configs/VectorBallParam.scala b/examples/prefab/balls/vector/configs/VectorBallParam.scala similarity index 100% rename from examples/components/balls/vector/configs/VectorBallParam.scala rename to examples/prefab/balls/vector/configs/VectorBallParam.scala diff --git a/examples/components/balls/vector/configs/default.toml b/examples/prefab/balls/vector/configs/default.toml similarity index 100% rename from examples/components/balls/vector/configs/default.toml rename to examples/prefab/balls/vector/configs/default.toml diff --git a/examples/components/balls/vector/configs/vector_params.proto b/examples/prefab/balls/vector/configs/vector_params.proto similarity index 100% rename from examples/components/balls/vector/configs/vector_params.proto rename to examples/prefab/balls/vector/configs/vector_params.proto diff --git a/examples/components/balls/vector/emu/src/64_vecmat16.rs b/examples/prefab/balls/vector/emu/src/64_vecmat16.rs similarity index 76% rename from examples/components/balls/vector/emu/src/64_vecmat16.rs rename to examples/prefab/balls/vector/emu/src/64_vecmat16.rs index eac7ff2da..cfd4264e8 100644 --- a/examples/components/balls/vector/emu/src/64_vecmat16.rs +++ b/examples/prefab/balls/vector/emu/src/64_vecmat16.rs @@ -1,7 +1,13 @@ -//===- 64_vecmat16.rs - MUL_WARP16 instruction ---------------------------===// +//===- 64_vecmat16.rs - VECMAT16 instruction ------------------------------===// +// +// C += A^T * B: op1 holds A^T and op2 holds B, iter rows of sixteen int8 each; +// the four-group destination holds C, sixteen rows of sixteen int32, and is +// read as the old value (rs1[29]). rs2 is reserved and zero. +// +//===----------------------------------------------------------------------===// use super::bank_matrix::{read_i32_nn_groups, read_i8_k_rows, write_i32_nn_groups}; -use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter}; +use super::decode::{pbank, pbank_group, rs1_b0, rs1_b1, rs1_b2, rs1_iter, rs1_read_old}; use super::instruction::{BallModel, ExecContext}; const WARP_M: usize = 16; @@ -11,11 +17,12 @@ pub struct VecMat16; impl BallModel for VecMat16 { fn exec(xs1: u64, xs2: u64, ctx: &mut ExecContext) -> u64 { + assert!(rs1_read_old(xs1), "vecmat16: accumulates into the old destination; rs1[29] must be set"); + assert!(xs2 == 0, "vecmat16: rs2 is reserved and must be zero"); let op1 = rs1_b0(xs1); let op2 = rs1_b1(xs1); let wr = rs1_b2(xs1); let iter = rs1_iter(xs1); - let _ = xs2; let c1 = ctx.config(op1).cols; let c2 = ctx.config(op2).cols; diff --git a/examples/components/balls/vector/emu/src/dispatch.rs b/examples/prefab/balls/vector/emu/src/dispatch.rs similarity index 100% rename from examples/components/balls/vector/emu/src/dispatch.rs rename to examples/prefab/balls/vector/emu/src/dispatch.rs diff --git a/examples/components/balls/vector/emu/src/lib.rs b/examples/prefab/balls/vector/emu/src/lib.rs similarity index 100% rename from examples/components/balls/vector/emu/src/lib.rs rename to examples/prefab/balls/vector/emu/src/lib.rs diff --git a/examples/prefab/balls/vector/stack/src/vectorOps.cpp b/examples/prefab/balls/vector/stack/src/vectorOps.cpp new file mode 100644 index 000000000..8d38f899d --- /dev/null +++ b/examples/prefab/balls/vector/stack/src/vectorOps.cpp @@ -0,0 +1,68 @@ +//===- vectorOps.cpp - VecBall operations ---------------------------------===// +// +// The dialect of VecBall, its virtual operations building their physical ones +// and the instructions of the physical operations. +// +//===----------------------------------------------------------------------===// + +#include "vectorOps.h" + +#include "Conversion/Buckyball.h" +#include "Target/BuckyballTargetRegistry.h" + +using namespace mlir; +using namespace ::buddy::vector_ball; +using mlir::buddy::bankFields; +using mlir::buddy::constantI64; +using mlir::buddy::emitInstruction; +using mlir::buddy::field; + +#include "vectorDialect.cpp.inc" + +#define GET_OP_CLASSES +#include "vectorOps.cpp.inc" + +void VecBallDialect::initialize() { + addOperations< +#define GET_OP_LIST +#include "vectorOps.cpp.inc" + >(); +} + +namespace { +[[maybe_unused]] const bool registered = + (mlir::buddy::addBallDialect( + [](DialectRegistry ®istry) { registry.insert(); }), + true); +} // namespace + +SmallVector BankMatmulAtI8Op::getSourceBanks() { + return {getAt(), getB()}; +} +Value BankMatmulAtI8Op::getDestinationBank() { return getC(); } +bool BankMatmulAtI8Op::readsDestination() { return true; } +Operation *BankMatmulAtI8Op::buildPhysical(OpBuilder &builder, + ArrayRef sources, + int64_t destination) { + return MatmulAtI8Op::create(builder, getLoc(), sources[0], sources[1], + destination, getRows()); +} + +SmallVector MatmulAtI8Op::getSourceBanks() { + return {int64_t(getAt()), int64_t(getB())}; +} +int64_t MatmulAtI8Op::getDestinationBank() { return getC(); } +LogicalResult MatmulAtI8Op::lowerToInstructions(OpBuilder &builder) { + Location loc = getLoc(); + int64_t rows = getRows(), at = getAt(), b = getB(), c = getC(); + const auto &target = buckyball_target::getBuckyballTarget(); + if (target.bankWidthBits != 128 || rows <= 0 || rows > target.bankDepth) + return emitOpError("matmul_at_i8 rows exceed the bank"); + if (at == c || b == c) + return emitOpError("matmul_at_i8 requires C apart from A^T and B"); + emitInstruction(builder, loc, + buckyball_target::getBuckyballFunct7("VECMAT16"), + constantI64(builder, loc, bankFields({at, b}, c, true, rows)), + constantI64(builder, loc, 0)); + return success(); +} diff --git a/examples/prefab/balls/vector/stack/src/vectorOps.h b/examples/prefab/balls/vector/stack/src/vectorOps.h new file mode 100644 index 000000000..4aff1c5e6 --- /dev/null +++ b/examples/prefab/balls/vector/stack/src/vectorOps.h @@ -0,0 +1,20 @@ +//===- vectorOps.h - VecBall operations -----------------------------------===// + +#ifndef VECTOR_OPS_H +#define VECTOR_OPS_H + +#include "Interfaces/BankInterfaces.h" + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/OpImplementation.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "vectorDialect.h.inc" + +#define GET_OP_CLASSES +#include "vectorOps.h.inc" + +#endif // VECTOR_OPS_H diff --git a/examples/prefab/balls/vector/stack/src/vectorOps.td b/examples/prefab/balls/vector/stack/src/vectorOps.td new file mode 100644 index 000000000..908dacce1 --- /dev/null +++ b/examples/prefab/balls/vector/stack/src/vectorOps.td @@ -0,0 +1,47 @@ +//===- vectorOps.td - VecBall operations -------------------*- tablegen -*-===// +// +// matmul_at_i8 (at, b) -> c: c += at^T * b, one VECMAT16 instruction; it +// always reads the old c. Banks are one group of sixteen i8 per row unless +// noted: +// at: A^T, `rows` rows; row k is column k of the 16 x rows matrix A. +// b: `rows` rows of sixteen i8. +// c: a bank of four groups holding sixteen rows of sixteen i32; group g +// holds columns 4g..4g+3 of each row, so C moves to and from memory as a +// plain 16 x 16 i32 matrix. +// `rows` is the reduction depth K, 1..bank depth. +// +//===----------------------------------------------------------------------===// + +#ifndef VECTOR_OPS +#define VECTOR_OPS + +include "mlir/IR/OpBase.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Interfaces/BankInterfaces.td" + +def VecBall_Dialect : Dialect { + let name = "vector_ball"; + let cppNamespace = "::buddy::vector_ball"; + let summary = "The operations of VecBall"; +} + +class VecBall_Op traits = []> + : Op])>; + +def BankMatmulAtI8Op : VecBall_Op<"bank_matmul_at_i8", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_at_i8 on virtual banks"; + let arguments = (ins I64:$at, I64:$b, I64:$c, I64Attr:$rows); + let results = (outs I64:$cOut); + let assemblyFormat = "$at `,` $b `->` $c prop-dict attr-dict"; +} + +def MatmulAtI8Op : VecBall_Op<"matmul_at_i8", + [DeclareOpInterfaceMethods]> { + let summary = "matmul_at_i8 on physical banks"; + let arguments = (ins I64Attr:$at, I64Attr:$b, I64Attr:$c, I64Attr:$rows); + let assemblyFormat = "prop-dict attr-dict"; +} + +#endif // VECTOR_OPS diff --git a/examples/prefab/balls/vector/verify/casegen/Cargo.lock b/examples/prefab/balls/vector/verify/casegen/Cargo.lock new file mode 100644 index 000000000..5f4dc2de3 --- /dev/null +++ b/examples/prefab/balls/vector/verify/casegen/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "vector-casegen" +version = "0.1.0" diff --git a/examples/prefab/balls/vector/verify/casegen/Cargo.toml b/examples/prefab/balls/vector/verify/casegen/Cargo.toml new file mode 100644 index 000000000..f5380573e --- /dev/null +++ b/examples/prefab/balls/vector/verify/casegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "vector-casegen" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["rlib", "cdylib"] + +[lints.rust] +unused_must_use = "deny" diff --git a/examples/prefab/balls/vector/verify/casegen/src/casegen.rs b/examples/prefab/balls/vector/verify/casegen/src/casegen.rs new file mode 100644 index 000000000..9dd98c8e8 --- /dev/null +++ b/examples/prefab/balls/vector/verify/casegen/src/casegen.rs @@ -0,0 +1,126 @@ +use crate::model::{self, GROUPS, LANE}; + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct VecMat16Cmd { + pub iter: u32, + pub op1_bank: u32, + pub op2_bank: u32, + pub wr_bank: u32, + pub wr_read_bank: u32, + pub rob_id: u32, + // Group bases; the old C uses the wr layout, so it shares wr_group. + pub op1_group: u32, + pub op2_group: u32, + pub wr_group: u32, +} + +#[derive(Clone, Debug)] +pub struct VecMat16Case { + pub cmd: VecMat16Cmd, + pub at: Vec, + pub b: Vec, + // Indexed group * LANE + row. + pub c_old: Vec, + pub c_new: Vec, +} + +struct Lcg(u64); + +impl Lcg { + fn next(&mut self) -> u32 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (self.0 >> 32) as u32 + } +} + +#[derive(Clone, Copy)] +enum Fill { + Random, + // Every int8 is -128 or 127: the largest products of both signs. + Extremes, +} + +fn i8_word(rng: &mut Lcg, fill: Fill) -> u128 { + let mut word = 0u128; + for lane in 0..LANE { + let byte = match fill { + Fill::Random => rng.next() as u8, + Fill::Extremes => { + if rng.next() & 1 == 0 { + 0x80 + } else { + 0x7f + } + } + }; + word |= u128::from(byte) << (8 * lane); + } + word +} + +pub fn gen_case(index: u32) -> VecMat16Case { + let entries = model::bank_entries(); + // (rob_id, iter, [op1, op2, wr, wr_read], [op1, op2, wr] group bases, fill, + // old C near the int32 limits) + let (rob_id, iter, banks, groups, fill, wrap) = match index { + 0 => (1, 1, [0, 1, 2, 3], [0, 0, 0], Fill::Random, false), + 1 => (2, 16, [4, 5, 6, 7], [0, 0, 0], Fill::Extremes, false), + 2 => (3, 5, [1, 0, 3, 2], [0, 0, 0], Fill::Extremes, true), + 3 => (4, entries, [8, 9, 10, 11], [0, 0, 0], Fill::Random, false), + 4 => (5, 7, [1023, 512, 700, 701], [0, 0, 0], Fill::Random, true), + 5 => (6, 9, [12, 13, 14, 15], [3, 7, 12], Fill::Random, false), + 6 => (7, 16, [20, 20, 21, 22], [1, 9, 5], Fill::Extremes, true), + _ => panic!("vecmat16: unsupported case {index}"), + }; + model::validate(entries, iter); + let mut rng = Lcg(0x5ec0_0000 + u64::from(index)); + let at: Vec = (0..iter).map(|_| i8_word(&mut rng, fill)).collect(); + let b: Vec = (0..iter).map(|_| i8_word(&mut rng, fill)).collect(); + let mut c = [[0i32; LANE]; LANE]; + for row in c.iter_mut() { + for value in row.iter_mut() { + let r = rng.next(); + *value = if wrap { + // Near i32::MAX or i32::MIN so the accumulation wraps. + if r & 1 == 0 { + i32::MAX - (r >> 20) as i32 + } else { + i32::MIN + (r >> 20) as i32 + } + } else { + r as i32 + }; + } + } + let mut c_old = Vec::with_capacity(GROUPS * LANE); + for g in 0..GROUPS { + for row in 0..LANE { + c_old.push(model::c_word(&c, g, row)); + } + } + model::vecmat16(&at, &b, &mut c); + let mut c_new = Vec::with_capacity(GROUPS * LANE); + for g in 0..GROUPS { + for row in 0..LANE { + c_new.push(model::c_word(&c, g, row)); + } + } + VecMat16Case { + cmd: VecMat16Cmd { + iter, + op1_bank: banks[0], + op2_bank: banks[1], + wr_bank: banks[2], + wr_read_bank: banks[3], + rob_id, + op1_group: groups[0], + op2_group: groups[1], + wr_group: groups[2], + }, + at, + b, + c_old, + c_new, + } +} diff --git a/examples/prefab/balls/vector/verify/casegen/src/lib.rs b/examples/prefab/balls/vector/verify/casegen/src/lib.rs new file mode 100644 index 000000000..b6d895a46 --- /dev/null +++ b/examples/prefab/balls/vector/verify/casegen/src/lib.rs @@ -0,0 +1,81 @@ +mod casegen; +mod model; + +use std::cell::RefCell; + +use casegen::{VecMat16Case, VecMat16Cmd}; + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +#[no_mangle] +pub extern "C" fn vector_case_load(index: u32) -> i32 { + let case = casegen::gen_case(index); + CURRENT.with(|current| *current.borrow_mut() = Some(case)); + 0 +} + +fn current_case R, R>(f: F) -> R { + CURRENT.with(|current| match current.borrow().as_ref() { + Some(case) => f(case), + None => panic!("vector_case: no case loaded; call vector_case_load first"), + }) +} + +fn word(words: &[u128], index: u32, what: &str) -> u128 { + match words.get(index as usize) { + Some(word) => *word, + None => panic!("vector_case_{what}: index {index} out of range {}", words.len()), + } +} + +#[no_mangle] +pub extern "C" fn vector_case_cmd(out_ptr: *mut VecMat16Cmd) { + if out_ptr.is_null() { + panic!("vector_case_cmd: null out_ptr"); + } + current_case(|case| unsafe { + *out_ptr = case.cmd; + }); +} + +#[no_mangle] +pub extern "C" fn vector_case_at_word_lo(index: u32) -> u64 { + current_case(|case| word(&case.at, index, "at_word_lo") as u64) +} + +#[no_mangle] +pub extern "C" fn vector_case_at_word_hi(index: u32) -> u64 { + current_case(|case| (word(&case.at, index, "at_word_hi") >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn vector_case_b_word_lo(index: u32) -> u64 { + current_case(|case| word(&case.b, index, "b_word_lo") as u64) +} + +#[no_mangle] +pub extern "C" fn vector_case_b_word_hi(index: u32) -> u64 { + current_case(|case| (word(&case.b, index, "b_word_hi") >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn vector_case_c_old_word_lo(index: u32) -> u64 { + current_case(|case| word(&case.c_old, index, "c_old_word_lo") as u64) +} + +#[no_mangle] +pub extern "C" fn vector_case_c_old_word_hi(index: u32) -> u64 { + current_case(|case| (word(&case.c_old, index, "c_old_word_hi") >> 64) as u64) +} + +#[no_mangle] +pub extern "C" fn vector_case_c_new_word_lo(index: u32) -> u64 { + current_case(|case| word(&case.c_new, index, "c_new_word_lo") as u64) +} + +#[no_mangle] +pub extern "C" fn vector_case_c_new_word_hi(index: u32) -> u64 { + current_case(|case| (word(&case.c_new, index, "c_new_word_hi") >> 64) as u64) +} diff --git a/examples/prefab/balls/vector/verify/casegen/src/model.rs b/examples/prefab/balls/vector/verify/casegen/src/model.rs new file mode 100644 index 000000000..b996d58bf --- /dev/null +++ b/examples/prefab/balls/vector/verify/casegen/src/model.rs @@ -0,0 +1,50 @@ +// VecBall VECMAT16 contract: C += A^T * B with int8 inputs and int32 +// accumulators that wrap. op1 holds A^T and op2 holds B, `iter` rows of +// sixteen int8 each; the old C is read from the renamed bank wr_read_bank and +// the new C written to wr_bank, sixteen rows of sixteen int32 in four groups: +// group g holds columns 4g..4g+3 of each row. rs2 is reserved and zero. + +pub const LANE: usize = 16; +pub const GROUPS: usize = 4; + +pub fn bank_entries() -> u32 { + let path = std::env::var("BB_VERIFY_CONFIG").expect("BB_VERIFY_CONFIG is required"); + let text = std::fs::read_to_string(path).expect("failed to read BB_VERIFY_CONFIG"); + text.lines() + .find_map(|line| line.strip_prefix("bank_entries=")) + .expect("bank_entries missing from BB_VERIFY_CONFIG") + .parse() + .expect("invalid bank_entries in BB_VERIFY_CONFIG") +} + +pub fn validate(entries: u32, iter: u32) { + if iter == 0 || iter > entries { + panic!("vecmat16: iter must be 1..bank depth"); + } +} + +pub fn i8_lane(word: u128, lane: usize) -> i32 { + ((word >> (8 * lane)) as u8 as i8) as i32 +} + +// c[i][j] += sum_k at[k][i] * b[k][j], wrapping like the hardware. +pub fn vecmat16(at: &[u128], b: &[u128], c: &mut [[i32; LANE]; LANE]) { + for i in 0..LANE { + for j in 0..LANE { + let mut acc = c[i][j]; + for k in 0..at.len() { + acc = acc.wrapping_add(i8_lane(at[k], i).wrapping_mul(i8_lane(b[k], j))); + } + c[i][j] = acc; + } + } +} + +// Word of group `g`, row `row`: columns 4g..4g+3, lowest column in the low bits. +pub fn c_word(c: &[[i32; LANE]; LANE], group: usize, row: usize) -> u128 { + let mut word = 0u128; + for lane in 0..4 { + word |= u128::from(c[row][group * 4 + lane] as u32) << (32 * lane); + } + word +} diff --git a/examples/prefab/balls/vector/verify/filelists/vector_ball.f b/examples/prefab/balls/vector/verify/filelists/vector_ball.f new file mode 100644 index 000000000..2ba66193b --- /dev/null +++ b/examples/prefab/balls/vector/verify/filelists/vector_ball.f @@ -0,0 +1,28 @@ ++incdir+@UVM@/src/core ++incdir+@UVM@/src/ball ++incdir+@UVM@/src/ball/agents/cmd ++incdir+@UVM@/src/ball/agents/mem ++incdir+@UVM@/src/ball/agents/resp ++incdir+@UVM@/src/ball/cov ++incdir+@UVM@/src/ball/env ++incdir+src ++incdir+src/pkg ++incdir+src/common ++incdir+src/seq ++incdir+src/cov ++incdir+src/env ++incdir+src/tests ++incdir+@RTL@ + +@UVM@/src/core/package.sv +@UVM@/src/ball/bb_blink_if.sv +@UVM@/src/ball/bb_uvm_pkg.sv + +@RTL@/VecMat16AccLoad.sv +@RTL@/VecMat16Store.sv +@RTL@/VecMat16.sv +@RTL@/VecUnit.sv +@RTL@/VecBall.sv + +src/pkg/vector_pkg.sv +src/tb_top.sv diff --git a/examples/prefab/balls/vector/verify/src/common/vector_defs.svh b/examples/prefab/balls/vector/verify/src/common/vector_defs.svh new file mode 100644 index 000000000..b2b0df2b7 --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/common/vector_defs.svh @@ -0,0 +1,33 @@ +typedef struct { + int unsigned iter; + int unsigned op1_bank; + int unsigned op2_bank; + int unsigned wr_bank; + int unsigned wr_read_bank; + int unsigned rob_id; + int unsigned op1_group; + int unsigned op2_group; + int unsigned wr_group; +} vector_cmd_dpi_t; + +import "DPI-C" function int vector_case_load(input int unsigned index); +import "DPI-C" function void vector_case_cmd(output vector_cmd_dpi_t cmd); +import "DPI-C" function longint unsigned vector_case_at_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned vector_case_at_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned vector_case_b_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned vector_case_b_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned vector_case_c_old_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned vector_case_c_old_word_hi(input int unsigned word_index); +import "DPI-C" function longint unsigned vector_case_c_new_word_lo(input int unsigned word_index); +import "DPI-C" function longint unsigned vector_case_c_new_word_hi(input int unsigned word_index); + +`ifndef VECMAT16_FUNCT7 +`error "VECMAT16_FUNCT7 must be provided by the selected ballISA" +`endif + +localparam int VECMAT16_THREAD_FUNCT7 = `VECMAT16_FUNCT7; +// C is sixteen rows in four groups of four int32 columns. +localparam int VECMAT16_ROWS = 16; +localparam int VECMAT16_GROUPS = 4; +localparam int VECMAT16_NUM_CASES = 7; +localparam int VECMAT16_TIMEOUT_CYCLES = 50000; diff --git a/examples/prefab/balls/vector/verify/src/common/vector_items.svh b/examples/prefab/balls/vector/verify/src/common/vector_items.svh new file mode 100644 index 000000000..65974870b --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/common/vector_items.svh @@ -0,0 +1,63 @@ +class vector_cmd_item extends bb_blink_cmd_item; + `uvm_object_utils(vector_cmd_item) + + bit [127:0] at_words[]; + bit [127:0] b_words[]; + // Indexed group * VECMAT16_ROWS + row. + bit [127:0] c_old_words[VECMAT16_GROUPS * VECMAT16_ROWS]; + bit [127:0] c_new_words[VECMAT16_GROUPS * VECMAT16_ROWS]; + + function new(string name = "vector_cmd_item"); + super.new(name); + endfunction + + function void load_rust_case(int unsigned index); + vector_cmd_dpi_t cmd; + int unsigned rc; + + rc = vector_case_load(index); + if (rc != 0) + `uvm_fatal("CASE", $sformatf("vector_case_load returned %0d for index %0d", rc, index)) + vector_case_cmd(cmd); + + // bb_vecmat16: A^T, B and C from rs1 banks 0..2, K in iter; the old C + // arrives as the renamed wr_read_bank at the wr group base. rs2 is + // reserved and zero. + rob_id = cmd.rob_id[BB_BLINK_ROB_ID_W-1:0]; + funct = VECMAT16_THREAD_FUNCT7[6:0]; + rs2 = '0; + iter = cmd.iter; + op1_bank = cmd.op1_bank; + op2_bank = cmd.op2_bank; + wr_bank = cmd.wr_bank; + op1_group = cmd.op1_group; + op2_group = cmd.op2_group; + wr_group = cmd.wr_group; + op1_col = 1; + op2_col = 1; + wr_col = VECMAT16_GROUPS; + wr_read_bank = cmd.wr_read_bank; + wr_arch_bank = cmd.wr_bank; + + at_words = new[cmd.iter]; + b_words = new[cmd.iter]; + for (int i = 0; i < cmd.iter; i++) begin + at_words[i] = {vector_case_at_word_hi(i), vector_case_at_word_lo(i)}; + b_words[i] = {vector_case_b_word_hi(i), vector_case_b_word_lo(i)}; + end + for (int i = 0; i < VECMAT16_GROUPS * VECMAT16_ROWS; i++) begin + c_old_words[i] = {vector_case_c_old_word_hi(i), vector_case_c_old_word_lo(i)}; + c_new_words[i] = {vector_case_c_new_word_hi(i), vector_case_c_new_word_lo(i)}; + end + endfunction + + function void do_copy(uvm_object rhs); + vector_cmd_item rhs_; + super.do_copy(rhs); + if (!$cast(rhs_, rhs)) `uvm_fatal("COPY", "rhs is not vector_cmd_item") + at_words = rhs_.at_words; + b_words = rhs_.b_words; + c_old_words = rhs_.c_old_words; + c_new_words = rhs_.c_new_words; + endfunction +endclass diff --git a/examples/prefab/balls/vector/verify/src/cov/vector_cov.svh b/examples/prefab/balls/vector/verify/src/cov/vector_cov.svh new file mode 100644 index 000000000..3560cc1ab --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/cov/vector_cov.svh @@ -0,0 +1,44 @@ +class vector_cov extends uvm_component; + `uvm_component_utils(vector_cov) + + uvm_analysis_imp_cmd #(bb_blink_cmd_item, vector_cov) cmd_imp; + + int unsigned cur_iter; + int unsigned cur_op1_group; + int unsigned cur_op2_group; + int unsigned cur_wr_group; + + covergroup cmd_cg; + coverpoint cur_iter {bins one = {1}; bins short = {[2 : 16]}; bins deep = {[17 : $]};} + coverpoint cur_op1_group {bins zero = {0}; bins based = {[1 : $]};} + coverpoint cur_op2_group {bins zero = {0}; bins based = {[1 : $]};} + coverpoint cur_wr_group {bins zero = {0}; bins based = {[1 : $]};} + endgroup + + int cmd_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + cmd_imp = new("cmd_imp", this); + cmd_cg = new(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + if (item.funct != VECMAT16_THREAD_FUNCT7[6:0]) + `uvm_fatal("COV", $sformatf("unexpected funct %0d", item.funct)) + cur_iter = item.iter; + cur_op1_group = item.op1_group; + cur_op2_group = item.op2_group; + cur_wr_group = item.wr_group; + cmd_count++; + cmd_cg.sample(); + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (cmd_count == 0) `uvm_fatal("COV", "no cmd observed") + if (cmd_cg.get_coverage() != 100.0) + `uvm_fatal("COV", $sformatf("vector_cov %0.2f%%", cmd_cg.get_coverage())) + endfunction +endclass diff --git a/examples/prefab/balls/vector/verify/src/env/vector_env.svh b/examples/prefab/balls/vector/verify/src/env/vector_env.svh new file mode 100644 index 000000000..14f26e4be --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/env/vector_env.svh @@ -0,0 +1,27 @@ +class vector_env extends bb_blink_env #(`BB_IN_BW, `BB_OUT_BW); + `uvm_component_utils(vector_env) + + vector_scoreboard scb; + vector_cov icov; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + scb = vector_scoreboard::type_id::create("scb", this); + icov = vector_cov::type_id::create("icov", this); + endfunction + + function void connect_phase(uvm_phase phase); + super.connect_phase(phase); + scb.mem_model = mem_model; + cmd_agent.stim_ap.connect(scb.stim_imp); + cmd_agent.cmd_ap.connect(scb.cmd_imp); + cmd_agent.cmd_ap.connect(icov.cmd_imp); + read_mon.read_ap.connect(scb.read_imp); + write_mon.write_ap.connect(scb.write_imp); + resp_mon.resp_ap.connect(scb.resp_imp); + endfunction +endclass diff --git a/examples/prefab/balls/vector/verify/src/env/vector_scoreboard.svh b/examples/prefab/balls/vector/verify/src/env/vector_scoreboard.svh new file mode 100644 index 000000000..cc5e11669 --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/env/vector_scoreboard.svh @@ -0,0 +1,200 @@ +class vector_scoreboard extends uvm_scoreboard; + `uvm_component_utils(vector_scoreboard) + + uvm_analysis_imp_stim #(bb_blink_cmd_item, vector_scoreboard) stim_imp; + uvm_analysis_imp_cmd #(bb_blink_cmd_item, vector_scoreboard) cmd_imp; + uvm_analysis_imp_read #(bb_blink_read_item, vector_scoreboard) read_imp; + uvm_analysis_imp_write #(bb_blink_write_item, vector_scoreboard) write_imp; + uvm_analysis_imp_resp #(bb_blink_resp_item, vector_scoreboard) resp_imp; + + bb_blink_mem_model #(`BB_IN_BW, `BB_OUT_BW) mem_model; + + vector_cmd_item stim_q[$]; + int unsigned cmd_count; + int unsigned read_count[2]; + int unsigned write_count[VECMAT16_GROUPS]; + int unsigned resp_count; + + function new(string name, uvm_component parent); + super.new(name, parent); + stim_imp = new("stim_imp", this); + cmd_imp = new("cmd_imp", this); + read_imp = new("read_imp", this); + write_imp = new("write_imp", this); + resp_imp = new("resp_imp", this); + endfunction + + function void write_stim(bb_blink_cmd_item item); + vector_cmd_item got; + vector_cmd_item clone; + + if (stim_q.size() != 0) `uvm_fatal("SCB", "single outstanding command supported") + if (!$cast(got, item)) `uvm_fatal("SCB", "stim item is not vector_cmd_item") + if (!$cast(clone, got.clone())) `uvm_fatal("SCB", "failed to clone stimulus item") + stim_q.push_back(clone); + + if (mem_model == null) `uvm_fatal("SCB", "mem_model handle not set") + mem_model.clear_mem(); + for (int g = 0; g < VECMAT16_GROUPS; g++) + for (int row = 0; row < VECMAT16_ROWS; row++) + mem_model.load_word_g(int'(clone.wr_read_bank), int'(clone.wr_group) + g, row, + clone.c_old_words[g*VECMAT16_ROWS+row]); + for (int k = 0; k < clone.iter; k++) begin + mem_model.load_word_g(int'(clone.op1_bank), int'(clone.op1_group), k, clone.at_words[k]); + mem_model.load_word_g(int'(clone.op2_bank), int'(clone.op2_group), k, clone.b_words[k]); + end + mem_model.arm(); + endfunction + + function void write_cmd(bb_blink_cmd_item item); + bb_blink_cmd_item want; + vector_cmd_item stim; + stim = current_stim("CMD"); + // Compare as plain BLink commands; the comparer rejects differing types. + want = bb_blink_cmd_item::type_id::create("want"); + want.copy(stim); + verification_pkg::evidence::checks++; + if (!item.compare(want)) + `uvm_fatal("CMD", $sformatf("command mismatch: got %s exp %s", item.sprint(), want.sprint())) + cmd_count++; + endfunction + + // Port p first reads the old C, row by row, group p then group 2 + p past + // the wr base; then port 0 streams A^T and port 1 streams B, one row per + // reduction step, at their own group bases. + function void write_read(bb_blink_read_item item); + vector_cmd_item stim; + int unsigned n; + int unsigned expect_bank; + int unsigned expect_group; + int unsigned expect_addr; + + stim = current_stim("READ"); + if (item.port != 0 && item.port != 1) + `uvm_fatal("READ", $sformatf("VecBall reads on two ports, got port %0d", item.port)) + n = read_count[item.port]; + if (n >= 2 * VECMAT16_ROWS + stim.iter) + `uvm_fatal("READ", $sformatf("extra read on port %0d", item.port)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("READ", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (n < 2 * VECMAT16_ROWS) begin + expect_bank = stim.wr_read_bank; + expect_group = stim.wr_group + 2 * (n % 2) + item.port; + expect_addr = n / 2; + end else begin + expect_bank = item.port == 0 ? stim.op1_bank : stim.op2_bank; + expect_group = item.port == 0 ? stim.op1_group : stim.op2_group; + expect_addr = n - 2 * VECMAT16_ROWS; + end + if (item.bank_id !== expect_bank[BB_BLINK_BANK_ID_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d read %0d bank mismatch: got %0d exp %0d", + item.port, + n, + item.bank_id, + expect_bank + )) + if (item.group_id !== expect_group[BB_BLINK_GROUP_ID_W-1:0]) + `uvm_fatal("READ", $sformatf( + "port %0d read %0d group mismatch: got %0d exp %0d", + item.port, + n, + item.group_id, + expect_group + )) + if (item.addr !== expect_addr[BB_BLINK_BANK_ADDR_W-1:0]) + `uvm_fatal( + "READ", $sformatf( + "port %0d read %0d addr mismatch: got %0d exp %0d", item.port, n, item.addr, expect_addr)) + read_count[item.port]++; + endfunction + + // Write port g stores group g of C past the wr base, rows in order, after + // every operand read. + function void write_write(bb_blink_write_item item); + vector_cmd_item stim; + int unsigned g; + int unsigned row; + + stim = current_stim("WRITE"); + if (item.port < 0 || item.port >= VECMAT16_GROUPS) + `uvm_fatal("WRITE", $sformatf("VecBall writes on four ports, got port %0d", item.port)) + g = item.port; + row = write_count[g]; + if (row >= VECMAT16_ROWS) `uvm_fatal("WRITE", $sformatf("extra write on port %0d", g)) + for (int p = 0; p < 2; p++) + if (read_count[p] != 2 * VECMAT16_ROWS + stim.iter) + `uvm_fatal("WRITE", $sformatf("write before port %0d finished reading", p)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("WRITE", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + if (item.bank_id !== stim.wr_bank) + `uvm_fatal("WRITE", $sformatf("bank mismatch: got %0d exp %0d", item.bank_id, stim.wr_bank)) + if (item.group_id !== stim.wr_group + g) + `uvm_fatal("WRITE", $sformatf( + "group mismatch: got %0d exp %0d", item.group_id, stim.wr_group + g)) + if (item.mask !== 16'hFFFF) + `uvm_fatal("WRITE", $sformatf("mask mismatch: got 0x%04h", item.mask)) + if (item.addr !== row) + `uvm_fatal("WRITE", $sformatf("port %0d addr mismatch: got %0d exp %0d", g, item.addr, row)) + verification_pkg::evidence::checks++; + if (item.data !== stim.c_new_words[g*VECMAT16_ROWS+row]) + `uvm_fatal("SCB", $sformatf( + "C mismatch at row %0d group %0d: got 0x%032h exp 0x%032h", + row, + g, + item.data, + stim.c_new_words[g*VECMAT16_ROWS+row] + )) + write_count[g]++; + endfunction + + function void write_resp(bb_blink_resp_item item); + vector_cmd_item stim; + stim = current_stim("RESP"); + for (int g = 0; g < VECMAT16_GROUPS; g++) + if (write_count[g] != VECMAT16_ROWS) + `uvm_fatal("RESP", $sformatf("response after %0d writes on port %0d", write_count[g], g)) + if (item.rob_id !== stim.rob_id) + `uvm_fatal("RESP", $sformatf("rob_id mismatch: got %0d exp %0d", item.rob_id, stim.rob_id)) + resp_count++; + endfunction + + function vector_cmd_item current_stim(string tag); + if (stim_q.size() == 0) `uvm_fatal("SCB", $sformatf("%s observed before stimulus", tag)) + return stim_q[0]; + endfunction + + function bit done(); + if (stim_q.size() == 0 || cmd_count != 1 || resp_count != 1) return 1'b0; + for (int p = 0; p < 2; p++) + if (read_count[p] != 2 * VECMAT16_ROWS + stim_q[0].iter) return 1'b0; + for (int g = 0; g < VECMAT16_GROUPS; g++) if (write_count[g] != VECMAT16_ROWS) return 1'b0; + return 1'b1; + endfunction + + function void reset_counters(); + stim_q.delete(); + cmd_count = 0; + resp_count = 0; + for (int p = 0; p < 2; p++) read_count[p] = 0; + for (int g = 0; g < VECMAT16_GROUPS; g++) write_count[g] = 0; + endfunction + + function void check_phase(uvm_phase phase); + verification_pkg::evidence::checks++; + super.check_phase(phase); + if (!done()) + `uvm_fatal("SCB", $sformatf( + "incomplete: cmds=%0d reads=%0d,%0d writes=%0d,%0d,%0d,%0d resp=%0d", + cmd_count, + read_count[0], + read_count[1], + write_count[0], + write_count[1], + write_count[2], + write_count[3], + resp_count + )) + stim_q.delete(); + endfunction +endclass diff --git a/examples/prefab/balls/vector/verify/src/pkg/vector_pkg.sv b/examples/prefab/balls/vector/verify/src/pkg/vector_pkg.sv new file mode 100644 index 000000000..dc94fef37 --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/pkg/vector_pkg.sv @@ -0,0 +1,13 @@ +package vector_pkg; + import uvm_pkg::*; + import bb_uvm_pkg::*; + `include "uvm_macros.svh" + + `include "common/vector_defs.svh" + `include "common/vector_items.svh" + `include "seq/vector_sequences.svh" + `include "cov/vector_cov.svh" + `include "env/vector_scoreboard.svh" + `include "env/vector_env.svh" + `include "tests/vector_test.svh" +endpackage diff --git a/examples/prefab/balls/vector/verify/src/seq/vector_sequences.svh b/examples/prefab/balls/vector/verify/src/seq/vector_sequences.svh new file mode 100644 index 000000000..d63048f51 --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/seq/vector_sequences.svh @@ -0,0 +1,17 @@ +class vector_basic_seq extends uvm_sequence #(bb_blink_cmd_item); + `uvm_object_utils(vector_basic_seq) + + int unsigned case_index; + + function new(string name = "vector_basic_seq"); + super.new(name); + endfunction + + task body(); + vector_cmd_item item; + item = vector_cmd_item::type_id::create("item"); + start_item(item); + item.load_rust_case(case_index); + finish_item(item); + endtask +endclass diff --git a/examples/prefab/balls/vector/verify/src/tb_top.sv b/examples/prefab/balls/vector/verify/src/tb_top.sv new file mode 100644 index 000000000..a0c1322e6 --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/tb_top.sv @@ -0,0 +1,165 @@ +module tb_top; + import uvm_pkg::*; + import vector_pkg::*; + + bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) intf (); + + VecBall dut ( + .clock(intf.clock), + .reset(intf.reset), + .blink_status_running(intf.status_running), + .blink_cmdReq_ready(intf.cmd_req_ready), + .blink_cmdReq_valid(intf.cmd_req_valid), + .blink_cmdReq_bits_rob_id(intf.cmd_req_bits_rob_id), + .blink_cmdReq_bits_funct(intf.cmd_req_bits_funct), + .blink_cmdReq_bits_rs2(intf.cmd_req_bits_rs2), + .blink_cmdReq_bits_iter(intf.cmd_req_bits_iter), + .blink_cmdReq_bits_op1_bank(intf.cmd_req_bits_op1_bank), + .blink_cmdReq_bits_op2_bank(intf.cmd_req_bits_op2_bank), + .blink_cmdReq_bits_wr_bank(intf.cmd_req_bits_wr_bank), + .blink_cmdReq_bits_op1_group(intf.cmd_req_bits_op1_group), + .blink_cmdReq_bits_op2_group(intf.cmd_req_bits_op2_group), + .blink_cmdReq_bits_wr_group(intf.cmd_req_bits_wr_group), + .blink_cmdReq_bits_op1_col(intf.cmd_req_bits_op1_col), + .blink_cmdReq_bits_op2_col(intf.cmd_req_bits_op2_col), + .blink_cmdReq_bits_wr_col(intf.cmd_req_bits_wr_col), + .blink_cmdReq_bits_wr_read_bank(intf.cmd_req_bits_wr_read_bank), + .blink_cmdReq_bits_wr_arch_bank(intf.cmd_req_bits_wr_arch_bank), + .blink_cmdResp_ready(intf.cmd_resp_ready), + .blink_cmdResp_valid(intf.cmd_resp_valid), + .blink_cmdResp_bits_rob_id(intf.cmd_resp_bits_rob_id), + .blink_bankRead_0_bank_id(intf.bank_read_bank_id[0]), + .blink_bankRead_0_rob_id(intf.bank_read_rob_id[0]), + .blink_bankRead_0_group_id(intf.bank_read_group_id[0]), + .blink_bankRead_0_io_req_ready(intf.bank_read_req_ready[0]), + .blink_bankRead_0_io_req_valid(intf.bank_read_req_valid[0]), + .blink_bankRead_0_io_req_bits_addr(intf.bank_read_req_addr[0]), + .blink_bankRead_0_io_resp_ready(intf.bank_read_resp_ready[0]), + .blink_bankRead_0_io_resp_valid(intf.bank_read_resp_valid[0]), + .blink_bankRead_0_io_resp_bits_data(intf.bank_read_resp_data[0]), + .blink_bankRead_1_bank_id(intf.bank_read_bank_id[1]), + .blink_bankRead_1_rob_id(intf.bank_read_rob_id[1]), + .blink_bankRead_1_group_id(intf.bank_read_group_id[1]), + .blink_bankRead_1_io_req_ready(intf.bank_read_req_ready[1]), + .blink_bankRead_1_io_req_valid(intf.bank_read_req_valid[1]), + .blink_bankRead_1_io_req_bits_addr(intf.bank_read_req_addr[1]), + .blink_bankRead_1_io_resp_ready(intf.bank_read_resp_ready[1]), + .blink_bankRead_1_io_resp_valid(intf.bank_read_resp_valid[1]), + .blink_bankRead_1_io_resp_bits_data(intf.bank_read_resp_data[1]), + .blink_bankWrite_0_bank_id(intf.bank_write_bank_id[0]), + .blink_bankWrite_0_rob_id(intf.bank_write_rob_id[0]), + .blink_bankWrite_0_group_id(intf.bank_write_group_id[0]), + .blink_bankWrite_0_io_req_ready(intf.bank_write_req_ready[0]), + .blink_bankWrite_0_io_req_valid(intf.bank_write_req_valid[0]), + .blink_bankWrite_0_io_req_bits_addr(intf.bank_write_req_addr[0]), + .blink_bankWrite_0_io_req_bits_mask_0(intf.bank_write_req_mask[0][0]), + .blink_bankWrite_0_io_req_bits_mask_1(intf.bank_write_req_mask[0][1]), + .blink_bankWrite_0_io_req_bits_mask_2(intf.bank_write_req_mask[0][2]), + .blink_bankWrite_0_io_req_bits_mask_3(intf.bank_write_req_mask[0][3]), + .blink_bankWrite_0_io_req_bits_mask_4(intf.bank_write_req_mask[0][4]), + .blink_bankWrite_0_io_req_bits_mask_5(intf.bank_write_req_mask[0][5]), + .blink_bankWrite_0_io_req_bits_mask_6(intf.bank_write_req_mask[0][6]), + .blink_bankWrite_0_io_req_bits_mask_7(intf.bank_write_req_mask[0][7]), + .blink_bankWrite_0_io_req_bits_mask_8(intf.bank_write_req_mask[0][8]), + .blink_bankWrite_0_io_req_bits_mask_9(intf.bank_write_req_mask[0][9]), + .blink_bankWrite_0_io_req_bits_mask_10(intf.bank_write_req_mask[0][10]), + .blink_bankWrite_0_io_req_bits_mask_11(intf.bank_write_req_mask[0][11]), + .blink_bankWrite_0_io_req_bits_mask_12(intf.bank_write_req_mask[0][12]), + .blink_bankWrite_0_io_req_bits_mask_13(intf.bank_write_req_mask[0][13]), + .blink_bankWrite_0_io_req_bits_mask_14(intf.bank_write_req_mask[0][14]), + .blink_bankWrite_0_io_req_bits_mask_15(intf.bank_write_req_mask[0][15]), + .blink_bankWrite_0_io_req_bits_data(intf.bank_write_req_data[0]), + .blink_bankWrite_0_io_resp_ready(intf.bank_write_resp_ready[0]), + .blink_bankWrite_0_io_resp_valid(intf.bank_write_resp_valid[0]), + .blink_bankWrite_0_io_resp_bits_ok(intf.bank_write_resp_ok[0]), + .blink_bankWrite_1_bank_id(intf.bank_write_bank_id[1]), + .blink_bankWrite_1_rob_id(intf.bank_write_rob_id[1]), + .blink_bankWrite_1_group_id(intf.bank_write_group_id[1]), + .blink_bankWrite_1_io_req_ready(intf.bank_write_req_ready[1]), + .blink_bankWrite_1_io_req_valid(intf.bank_write_req_valid[1]), + .blink_bankWrite_1_io_req_bits_addr(intf.bank_write_req_addr[1]), + .blink_bankWrite_1_io_req_bits_mask_0(intf.bank_write_req_mask[1][0]), + .blink_bankWrite_1_io_req_bits_mask_1(intf.bank_write_req_mask[1][1]), + .blink_bankWrite_1_io_req_bits_mask_2(intf.bank_write_req_mask[1][2]), + .blink_bankWrite_1_io_req_bits_mask_3(intf.bank_write_req_mask[1][3]), + .blink_bankWrite_1_io_req_bits_mask_4(intf.bank_write_req_mask[1][4]), + .blink_bankWrite_1_io_req_bits_mask_5(intf.bank_write_req_mask[1][5]), + .blink_bankWrite_1_io_req_bits_mask_6(intf.bank_write_req_mask[1][6]), + .blink_bankWrite_1_io_req_bits_mask_7(intf.bank_write_req_mask[1][7]), + .blink_bankWrite_1_io_req_bits_mask_8(intf.bank_write_req_mask[1][8]), + .blink_bankWrite_1_io_req_bits_mask_9(intf.bank_write_req_mask[1][9]), + .blink_bankWrite_1_io_req_bits_mask_10(intf.bank_write_req_mask[1][10]), + .blink_bankWrite_1_io_req_bits_mask_11(intf.bank_write_req_mask[1][11]), + .blink_bankWrite_1_io_req_bits_mask_12(intf.bank_write_req_mask[1][12]), + .blink_bankWrite_1_io_req_bits_mask_13(intf.bank_write_req_mask[1][13]), + .blink_bankWrite_1_io_req_bits_mask_14(intf.bank_write_req_mask[1][14]), + .blink_bankWrite_1_io_req_bits_mask_15(intf.bank_write_req_mask[1][15]), + .blink_bankWrite_1_io_req_bits_data(intf.bank_write_req_data[1]), + .blink_bankWrite_1_io_resp_ready(intf.bank_write_resp_ready[1]), + .blink_bankWrite_1_io_resp_valid(intf.bank_write_resp_valid[1]), + .blink_bankWrite_1_io_resp_bits_ok(intf.bank_write_resp_ok[1]), + .blink_bankWrite_2_bank_id(intf.bank_write_bank_id[2]), + .blink_bankWrite_2_rob_id(intf.bank_write_rob_id[2]), + .blink_bankWrite_2_group_id(intf.bank_write_group_id[2]), + .blink_bankWrite_2_io_req_ready(intf.bank_write_req_ready[2]), + .blink_bankWrite_2_io_req_valid(intf.bank_write_req_valid[2]), + .blink_bankWrite_2_io_req_bits_addr(intf.bank_write_req_addr[2]), + .blink_bankWrite_2_io_req_bits_mask_0(intf.bank_write_req_mask[2][0]), + .blink_bankWrite_2_io_req_bits_mask_1(intf.bank_write_req_mask[2][1]), + .blink_bankWrite_2_io_req_bits_mask_2(intf.bank_write_req_mask[2][2]), + .blink_bankWrite_2_io_req_bits_mask_3(intf.bank_write_req_mask[2][3]), + .blink_bankWrite_2_io_req_bits_mask_4(intf.bank_write_req_mask[2][4]), + .blink_bankWrite_2_io_req_bits_mask_5(intf.bank_write_req_mask[2][5]), + .blink_bankWrite_2_io_req_bits_mask_6(intf.bank_write_req_mask[2][6]), + .blink_bankWrite_2_io_req_bits_mask_7(intf.bank_write_req_mask[2][7]), + .blink_bankWrite_2_io_req_bits_mask_8(intf.bank_write_req_mask[2][8]), + .blink_bankWrite_2_io_req_bits_mask_9(intf.bank_write_req_mask[2][9]), + .blink_bankWrite_2_io_req_bits_mask_10(intf.bank_write_req_mask[2][10]), + .blink_bankWrite_2_io_req_bits_mask_11(intf.bank_write_req_mask[2][11]), + .blink_bankWrite_2_io_req_bits_mask_12(intf.bank_write_req_mask[2][12]), + .blink_bankWrite_2_io_req_bits_mask_13(intf.bank_write_req_mask[2][13]), + .blink_bankWrite_2_io_req_bits_mask_14(intf.bank_write_req_mask[2][14]), + .blink_bankWrite_2_io_req_bits_mask_15(intf.bank_write_req_mask[2][15]), + .blink_bankWrite_2_io_req_bits_data(intf.bank_write_req_data[2]), + .blink_bankWrite_2_io_resp_ready(intf.bank_write_resp_ready[2]), + .blink_bankWrite_2_io_resp_valid(intf.bank_write_resp_valid[2]), + .blink_bankWrite_2_io_resp_bits_ok(intf.bank_write_resp_ok[2]), + .blink_bankWrite_3_bank_id(intf.bank_write_bank_id[3]), + .blink_bankWrite_3_rob_id(intf.bank_write_rob_id[3]), + .blink_bankWrite_3_group_id(intf.bank_write_group_id[3]), + .blink_bankWrite_3_io_req_ready(intf.bank_write_req_ready[3]), + .blink_bankWrite_3_io_req_valid(intf.bank_write_req_valid[3]), + .blink_bankWrite_3_io_req_bits_addr(intf.bank_write_req_addr[3]), + .blink_bankWrite_3_io_req_bits_mask_0(intf.bank_write_req_mask[3][0]), + .blink_bankWrite_3_io_req_bits_mask_1(intf.bank_write_req_mask[3][1]), + .blink_bankWrite_3_io_req_bits_mask_2(intf.bank_write_req_mask[3][2]), + .blink_bankWrite_3_io_req_bits_mask_3(intf.bank_write_req_mask[3][3]), + .blink_bankWrite_3_io_req_bits_mask_4(intf.bank_write_req_mask[3][4]), + .blink_bankWrite_3_io_req_bits_mask_5(intf.bank_write_req_mask[3][5]), + .blink_bankWrite_3_io_req_bits_mask_6(intf.bank_write_req_mask[3][6]), + .blink_bankWrite_3_io_req_bits_mask_7(intf.bank_write_req_mask[3][7]), + .blink_bankWrite_3_io_req_bits_mask_8(intf.bank_write_req_mask[3][8]), + .blink_bankWrite_3_io_req_bits_mask_9(intf.bank_write_req_mask[3][9]), + .blink_bankWrite_3_io_req_bits_mask_10(intf.bank_write_req_mask[3][10]), + .blink_bankWrite_3_io_req_bits_mask_11(intf.bank_write_req_mask[3][11]), + .blink_bankWrite_3_io_req_bits_mask_12(intf.bank_write_req_mask[3][12]), + .blink_bankWrite_3_io_req_bits_mask_13(intf.bank_write_req_mask[3][13]), + .blink_bankWrite_3_io_req_bits_mask_14(intf.bank_write_req_mask[3][14]), + .blink_bankWrite_3_io_req_bits_mask_15(intf.bank_write_req_mask[3][15]), + .blink_bankWrite_3_io_req_bits_data(intf.bank_write_req_data[3]), + .blink_bankWrite_3_io_resp_ready(intf.bank_write_resp_ready[3]), + .blink_bankWrite_3_io_resp_valid(intf.bank_write_resp_valid[3]), + .blink_bankWrite_3_io_resp_bits_ok(intf.bank_write_resp_ok[3]) + ); + + initial begin + intf.clock = 1'b0; + intf.reset = 1'b1; + forever #5 intf.clock = ~intf.clock; + end + + initial begin + uvm_config_db#(virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW))::set(null, "*", "vif", intf); + run_test("vector_ball_test"); + end +endmodule diff --git a/examples/prefab/balls/vector/verify/src/tests/vector_test.svh b/examples/prefab/balls/vector/verify/src/tests/vector_test.svh new file mode 100644 index 000000000..b9d6a1dfd --- /dev/null +++ b/examples/prefab/balls/vector/verify/src/tests/vector_test.svh @@ -0,0 +1,49 @@ +class vector_ball_test extends uvm_test; + `uvm_component_utils(vector_ball_test) + + typedef virtual bb_blink_if #(`BB_IN_BW, `BB_OUT_BW) vif_t; + vif_t vif; + vector_env env; + + function new(string name, uvm_component parent); + super.new(name, parent); + endfunction + + function void build_phase(uvm_phase phase); + super.build_phase(phase); + if (!uvm_config_db#(vif_t)::get(this, "", "vif", vif)) + `uvm_fatal("NOVIF", "bb_blink_if not found") + env = vector_env::type_id::create("env", this); + endfunction + + task run_phase(uvm_phase phase); + phase.raise_objection(this); + for (int unsigned index = 0; index < VECMAT16_NUM_CASES; index++) run_case(index); + phase.drop_objection(this); + endtask + + task run_case(int unsigned index); + vector_basic_seq seq; + int cycles = 0; + apply_reset(); + env.scb.reset_counters(); + seq = vector_basic_seq::type_id::create("seq"); + seq.case_index = index; + seq.start(env.cmd_agent.seqr); + while (!env.scb.done()) begin + @(posedge vif.clock); + cycles++; + if (cycles > VECMAT16_TIMEOUT_CYCLES) + `uvm_fatal("TIMEOUT", $sformatf("VecBall case %0d timeout", index)) + end + `uvm_info("VECMAT16", $sformatf("VecBall case %0d passed in %0d cycles", index, cycles), + UVM_LOW) + endtask + + task apply_reset(); + vif.reset = 1'b1; + repeat (5) @(posedge vif.clock); + vif.reset = 1'b0; + repeat (2) @(posedge vif.clock); + endtask +endclass diff --git a/examples/components/balls/vector/workloads/ctests/CMakeLists.txt b/examples/prefab/balls/vector/workloads/ctests/CMakeLists.txt similarity index 51% rename from examples/components/balls/vector/workloads/ctests/CMakeLists.txt rename to examples/prefab/balls/vector/workloads/ctests/CMakeLists.txt index c5de3fc66..a140d5a75 100644 --- a/examples/components/balls/vector/workloads/ctests/CMakeLists.txt +++ b/examples/prefab/balls/vector/workloads/ctests/CMakeLists.txt @@ -1 +1,2 @@ add_buckyball_ctest(vecmat16_atranspose_16x16_test.c) +add_buckyball_ctest(vecmat16_accumulate_k4_test.c) diff --git a/examples/prefab/balls/vector/workloads/ctests/vecmat16_accumulate_k4_test.c b/examples/prefab/balls/vector/workloads/ctests/vecmat16_accumulate_k4_test.c new file mode 100644 index 000000000..f252f64df --- /dev/null +++ b/examples/prefab/balls/vector/workloads/ctests/vecmat16_accumulate_k4_test.c @@ -0,0 +1,51 @@ +#include "buckyball.h" +#include +#include +#include +#include +#include + +// C = bias + A^T * B with a reduction depth of four rows, then a second +// instruction accumulating the same product again. +enum { DIM = 16, K = 4 }; +static int8_t at[K * DIM] __attribute__((aligned(64))); +static int8_t b[K * DIM] __attribute__((aligned(64))); +static int32_t c[DIM * DIM] __attribute__((aligned(64))); +static int32_t expected[DIM * DIM]; + +int main(void) { + for (int k = 0; k < K; ++k) + for (int lane = 0; lane < DIM; ++lane) { + at[k * DIM + lane] = (int8_t)((k * 37 + lane * 11) % 255 - 127); + b[k * DIM + lane] = (int8_t)((k * 53 + lane * 29) % 255 - 128); + } + for (int i = 0; i < DIM; ++i) + for (int j = 0; j < DIM; ++j) { + int32_t bias = (i * DIM + j) * 1009 - 100000; + int32_t product = 0; + for (int k = 0; k < K; ++k) + product += at[k * DIM + i] * b[k * DIM + j]; + c[i * DIM + j] = bias; + expected[i * DIM + j] = bias + 2 * product; + } + bb_mem_alloc(1, 1, 1); + bb_mem_alloc(2, 1, 1); + bb_mem_alloc(3, 1, 4); + bb_mvin((uintptr_t)at, 1, K, 1); + bb_mvin((uintptr_t)b, 2, K, 1); + bb_mvin((uintptr_t)c, 3, DIM, 1); + bb_vecmat16(1, 2, 3, K); + bb_vecmat16(1, 2, 3, K); + bb_mvout((uintptr_t)c, 3, DIM, 1); + bb_mem_release(1); + bb_mem_release(2); + bb_mem_release(3); + for (int i = 0; i < DIM * DIM; ++i) + if (c[i] != expected[i]) { + printf("vecmat16 accumulate mismatch at %d: got %d expected %d\n", i, + c[i], expected[i]); + return 1; + } + printf("vecmat16_accumulate_k4 PASS\n"); + return 0; +} diff --git a/examples/components/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c b/examples/prefab/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c similarity index 76% rename from examples/components/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c rename to examples/prefab/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c index 1c934a288..303f886f3 100644 --- a/examples/components/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c +++ b/examples/prefab/balls/vector/workloads/ctests/vecmat16_atranspose_16x16_test.c @@ -20,20 +20,22 @@ int main(void) { at[col * DIM + row] = a[row * DIM + col]; } cpu_matmul(a, b, expected, DIM, DIM, DIM); - bb_mem_alloc(0, 1, 1); bb_mem_alloc(1, 1, 1); - bb_mem_alloc(2, 1, 4); - bb_mvin((uintptr_t)at, 0, DIM, 1); - bb_mvin((uintptr_t)b, 1, DIM, 1); - bb_vecmat16(0, 1, 2, DIM, 0); - bb_mvout((uintptr_t)actual, 2, DIM, 1); + bb_mem_alloc(2, 1, 1); + bb_mem_alloc(3, 1, 4); + bb_mvin((uintptr_t)at, 1, DIM, 1); + bb_mvin((uintptr_t)b, 2, DIM, 1); + // C starts at zero: actual is zero-initialized. + bb_mvin((uintptr_t)actual, 3, DIM, 1); + bb_vecmat16(1, 2, 3, DIM); + bb_mvout((uintptr_t)actual, 3, DIM, 1); if (!compare_i32_matrices(actual, expected, DIM, DIM)) { printf("vecmat16_atranspose_16x16 FAIL\n"); return 1; } - bb_mem_release(0); bb_mem_release(1); bb_mem_release(2); + bb_mem_release(3); printf("vecmat16_atranspose_16x16 PASS\n"); return 0; } diff --git a/examples/prefab/balls/vector/workloads/isa/vecmat16.h b/examples/prefab/balls/vector/workloads/isa/vecmat16.h new file mode 100644 index 000000000..d11a93035 --- /dev/null +++ b/examples/prefab/balls/vector/workloads/isa/vecmat16.h @@ -0,0 +1,21 @@ +#ifndef _BB_VECMAT16_H_ +#define _BB_VECMAT16_H_ + +#include +#include + +// C += A^T * B with int8 inputs and int32 accumulators. +// op1 bank (one group): A^T, iter rows of sixteen int8; row k is column k +// of the 16 x iter matrix A. +// op2 bank (one group): B, iter rows of sixteen int8. +// wr bank (four groups): C, sixteen rows of sixteen int32; group g holds +// columns 4g..4g+3 of each row. The old C is read +// and accumulated, so rs1[29] is always set. +// iter: the reduction depth K, 1..bank depth. rs2 is reserved and zero. +#define bb_vecmat16(op1_bank_id, op2_bank_id, wr_bank_id, iter) \ + BUCKYBALL_INSTRUCTION_R_R((BB_BANK0(op1_bank_id) | BB_BANK1(op2_bank_id) | \ + BB_BANK2(wr_bank_id) | BB_READ_WR_OLD | \ + BB_ITER(iter)), \ + 0, FUNCT7(vector, VECMAT16)) + +#endif // _BB_VECMAT16_H_ diff --git a/examples/components/balls/relu/workloads/mlir_tests/CMakeLists.txt b/examples/prefab/balls/vector/workloads/mlir_tests/CMakeLists.txt similarity index 100% rename from examples/components/balls/relu/workloads/mlir_tests/CMakeLists.txt rename to examples/prefab/balls/vector/workloads/mlir_tests/CMakeLists.txt diff --git a/examples/prefab/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt b/examples/prefab/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt new file mode 100644 index 000000000..1463bb1ef --- /dev/null +++ b/examples/prefab/balls/vector/workloads/mlir_tests/ball/CMakeLists.txt @@ -0,0 +1,4 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +set(BUCKYBALL_MLIR_TEST_PREFIX ball) + +add_buckyball_mlir_test(matmul_at_i8_16x16 TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/vector/workloads/mlir_tests/ball/matmul_at_i8_16x16.mlir b/examples/prefab/balls/vector/workloads/mlir_tests/ball/matmul_at_i8_16x16.mlir new file mode 100644 index 000000000..c68189646 --- /dev/null +++ b/examples/prefab/balls/vector/workloads/mlir_tests/ball/matmul_at_i8_16x16.mlir @@ -0,0 +1,51 @@ +func.func private @check_result(memref<16x16xi32>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %initial = arith.constant 7 : i32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %sixteen = arith.constant 16 : index + %at = memref.alloc() : memref<16x16xi8> + %b = memref.alloc() : memref<16x16xi8> + %c = memref.alloc() : memref<16x16xi32> + linalg.fill ins(%initial : i32) outs(%c : memref<16x16xi32>) + scf.for %k = %zero to %sixteen step %one { + scf.for %lane = %zero to %sixteen step %one { + %k3 = arith.muli %k, %three : index + %l5 = arith.muli %lane, %five : index + %asum = arith.addi %k3, %l5 : index + %arem = arith.remui %asum, %seven : index + %aval = arith.subi %arem, %three : index + %av = arith.index_cast %aval : index to i8 + %k2 = arith.muli %k, %two : index + %l3 = arith.muli %lane, %three : index + %bsum = arith.addi %k2, %l3 : index + %brem = arith.remui %bsum, %five : index + %bval = arith.subi %brem, %two : index + %bv = arith.index_cast %bval : index to i8 + memref.store %av, %at[%k, %lane] : memref<16x16xi8> + memref.store %bv, %b[%k, %lane] : memref<16x16xi8> + } + } + buckyball.mset <{bank = 1 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = true, row = 1 : i64, col = 1 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = true, row = 1 : i64, col = 4 : i64}> + buckyball.mvin %at <{bank = 1 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %b <{bank = 2 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi8> + buckyball.mvin %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + // C += A^T * B with K = 16. + vector_ball.matmul_at_i8 <{at = 1 : i64, b = 2 : i64, c = 3 : i64, rows = 16 : i64}> + buckyball.mvout %c <{bank = 3 : i64, depth = 16 : i64, stride = 1 : i64}> : memref<16x16xi32> + buckyball.mset <{bank = 1 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 2 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + buckyball.mset <{bank = 3 : i64, alloc = false, row = 0 : i64, col = 0 : i64}> + func.call @check_result(%c) : (memref<16x16xi32>) -> () + memref.dealloc %at : memref<16x16xi8> + memref.dealloc %b : memref<16x16xi8> + memref.dealloc %c : memref<16x16xi32> + return %zero_i8 : i8 +} diff --git a/examples/components/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16_main.cpp b/examples/prefab/balls/vector/workloads/mlir_tests/ball/matmul_at_i8_16x16_main.cpp similarity index 80% rename from examples/components/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16_main.cpp rename to examples/prefab/balls/vector/workloads/mlir_tests/ball/matmul_at_i8_16x16_main.cpp index 5c5c1e9cb..02dafa458 100644 --- a/examples/components/balls/vector/workloads/mlir_tests/ball/vecmat16_atranspose_16x16_main.cpp +++ b/examples/prefab/balls/vector/workloads/mlir_tests/ball/matmul_at_i8_16x16_main.cpp @@ -9,14 +9,14 @@ extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, exit(1); for (int row = 0; row < 16; ++row) for (int column = 0; column < 16; ++column) { - int32_t expected = 0; + int32_t expected = 7; for (int k = 0; k < 16; ++k) expected += ((3 * k + 5 * row) % 7 - 3) * ((2 * k + 3 * column) % 5 - 2); if (data[offset + row * stride + column] != expected) { - printf("vecmat16 A-transpose mismatch row=%d column=%d\n", row, column); + printf("matmul_at_i8 mismatch row=%d column=%d\n", row, column); exit(1); } } - printf("vecmat16 A-transpose 16x16 PASS\n"); + printf("matmul_at_i8 16x16 PASS\n"); } diff --git a/examples/prefab/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt b/examples/prefab/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt new file mode 100644 index 000000000..c2b433ede --- /dev/null +++ b/examples/prefab/balls/vector/workloads/mlir_tests/bank/CMakeLists.txt @@ -0,0 +1,4 @@ +set(BUCKYBALL_MLIR_GROUP_TARGET balls-mlir-tests-build) +set(BUCKYBALL_MLIR_TEST_PREFIX bank) + +add_buckyball_mlir_test(matmul_at_i8_bank_depth TARGET ${BUCKYBALL_MLIR_ACTIVE_TARGET}) diff --git a/examples/prefab/balls/vector/workloads/mlir_tests/bank/matmul_at_i8_bank_depth.mlir.in b/examples/prefab/balls/vector/workloads/mlir_tests/bank/matmul_at_i8_bank_depth.mlir.in new file mode 100644 index 000000000..d331d2c54 --- /dev/null +++ b/examples/prefab/balls/vector/workloads/mlir_tests/bank/matmul_at_i8_bank_depth.mlir.in @@ -0,0 +1,57 @@ +func.func private @check_result(memref<16x16xi32>) -> () +func.func @main() -> i8 { + %zero_i8 = arith.constant 0 : i8 + %initial = arith.constant -5 : i32 + %zero = arith.constant 0 : index + %one = arith.constant 1 : index + %two = arith.constant 2 : index + %three = arith.constant 3 : index + %five = arith.constant 5 : index + %seven = arith.constant 7 : index + %sixteen = arith.constant 16 : index + %depth = arith.constant @BB_BANK_LINES@ : index + %at = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x16xi8> + %b = memref.alloc() alignment = 64 : memref<@BB_BANK_LINES@x16xi8> + %c = memref.alloc() alignment = 64 : memref<16x16xi32> + linalg.fill ins(%initial : i32) outs(%c : memref<16x16xi32>) + scf.for %k = %zero to %depth step %one { + scf.for %lane = %zero to %sixteen step %one { + %k3 = arith.muli %k, %three : index + %l5 = arith.muli %lane, %five : index + %asum = arith.addi %k3, %l5 : index + %arem = arith.remui %asum, %seven : index + %aval = arith.subi %arem, %three : index + %av = arith.index_cast %aval : index to i8 + %k2 = arith.muli %k, %two : index + %l3 = arith.muli %lane, %three : index + %bsum = arith.addi %k2, %l3 : index + %brem = arith.remui %bsum, %five : index + %bval = arith.subi %brem, %two : index + %bv = arith.index_cast %bval : index to i8 + memref.store %av, %at[%k, %lane] : memref<@BB_BANK_LINES@x16xi8> + memref.store %bv, %b[%k, %lane] : memref<@BB_BANK_LINES@x16xi8> + } + } + %ab = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %bb = buckyball.bank_alloc <{row = 1 : i64, col = 1 : i64}> + %cb = buckyball.bank_alloc <{row = 1 : i64, col = 4 : i64}> + %al = buckyball.bank_mvin %at, %ab <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x16xi8> + %bl = buckyball.bank_mvin %b, %bb <{depth = @BB_BANK_LINES@ : i64, stride = 1 : i64}> + : memref<@BB_BANK_LINES@x16xi8> + %cl = buckyball.bank_mvin %c, %cb <{depth = 16 : i64, stride = 1 : i64}> + : memref<16x16xi32> + // C += A^T * B with K the bank depth. + %result = vector_ball.bank_matmul_at_i8 %al, %bl -> %cl + <{rows = @BB_BANK_LINES@ : i64}> + %stored = buckyball.bank_mvout %c, %result <{depth = 16 : i64, stride = 1 : i64}> + : memref<16x16xi32> + buckyball.bank_release %al + buckyball.bank_release %bl + buckyball.bank_release %stored + func.call @check_result(%c) : (memref<16x16xi32>) -> () + memref.dealloc %at : memref<@BB_BANK_LINES@x16xi8> + memref.dealloc %b : memref<@BB_BANK_LINES@x16xi8> + memref.dealloc %c : memref<16x16xi32> + return %zero_i8 : i8 +} diff --git a/examples/components/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth_main.cpp b/examples/prefab/balls/vector/workloads/mlir_tests/bank/matmul_at_i8_bank_depth_main.cpp similarity index 70% rename from examples/components/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth_main.cpp rename to examples/prefab/balls/vector/workloads/mlir_tests/bank/matmul_at_i8_bank_depth_main.cpp index c987cedfb..3b5f06d39 100644 --- a/examples/components/balls/vector/workloads/mlir_tests/bank/vecmat16_atranspose_bank_depth_main.cpp +++ b/examples/prefab/balls/vector/workloads/mlir_tests/bank/matmul_at_i8_bank_depth_main.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -10,14 +9,15 @@ extern "C" void check_result(int32_t *, int32_t *data, int64_t offset, exit(1); for (int row = 0; row < 16; ++row) for (int column = 0; column < 16; ++column) { - int32_t expected = 0; - for (int k = 0; k < BANK_LINES; ++k) + int32_t expected = -5; + for (int k = 0; k < THREAD_MEM_BANK_ENTRIES; ++k) expected += ((3 * k + 5 * row) % 7 - 3) * ((2 * k + 3 * column) % 5 - 2); if (data[offset + row * stride + column] != expected) { - printf("vecmat16 bank-depth mismatch row=%d column=%d\n", row, column); + printf("matmul_at_i8 bank-depth mismatch row=%d column=%d\n", row, + column); exit(1); } } - printf("vecmat16 A-transpose bank-depth PASS K=%d\n", BANK_LINES); + printf("matmul_at_i8 bank-depth PASS K=%d\n", THREAD_MEM_BANK_ENTRIES); } diff --git a/examples/components/cores/boom/LICENSE b/examples/prefab/cores/boom/LICENSE similarity index 100% rename from examples/components/cores/boom/LICENSE rename to examples/prefab/cores/boom/LICENSE diff --git a/examples/components/cores/boom/arch/Boom.scala b/examples/prefab/cores/boom/arch/Boom.scala similarity index 100% rename from examples/components/cores/boom/arch/Boom.scala rename to examples/prefab/cores/boom/arch/Boom.scala diff --git a/examples/components/cores/boom/arch/Native.scala b/examples/prefab/cores/boom/arch/Native.scala similarity index 100% rename from examples/components/cores/boom/arch/Native.scala rename to examples/prefab/cores/boom/arch/Native.scala diff --git a/examples/components/cores/boom/arch/build.sc b/examples/prefab/cores/boom/arch/build.sc similarity index 87% rename from examples/components/cores/boom/arch/build.sc rename to examples/prefab/cores/boom/arch/build.sc index 8e1c39f52..aff61a1bd 100644 --- a/examples/components/cores/boom/arch/build.sc +++ b/examples/prefab/cores/boom/arch/build.sc @@ -28,7 +28,7 @@ object rocketchip extends ScalaBuild { override def ivyDeps = super.ivyDeps() ++ Agg(ivy"com.lihaoyi::mainargs:0.5.0", ivy"org.json4s::json4s-jackson:4.0.5") } object native extends ScalaBuild { - override def sources = T.sources(boomRoot / os.RelPath("src/main/scala/v4"), boomRoot / os.RelPath("src/main/scala/buckyball")) + override def sources = T.sources(boomRoot / os.RelPath("src/main/scala/v4"), root / "arch" / "native") override def resources = T.sources(boomRoot / os.RelPath("src/main/resources")) override def moduleDeps = Seq(rocketchip) override def forkArgs = Seq("-Xmx32G", "-Xss64m") @@ -36,11 +36,14 @@ object native extends ScalaBuild { def nativeRtl = T { val cp = native.runClasspath().map(_.path) os.proc("java", "-Xmx32G", "-Xss64m", "-cp", cp.mkString(java.io.File.pathSeparator), - "buckyball.Emit", "--target-dir", T.dest / "boom").call(cwd = root) + "examples.core.boom.native.Emit", "--target-dir", T.dest / "boom").call(cwd = root) PathRef(T.dest) } object library extends ScalaBuild { override def moduleDeps = Seq(framework.library) - override def sources = T.sources(root / "arch", root / "configs") + override def sources = T.sources { + os.walk(root / "arch").filter(p => p.ext == "scala" && !p.startsWith(root / "arch" / "native")).map(PathRef(_)) ++ + Seq(PathRef(root / "configs")) + } override def resources = T.sources(Seq(nativeRtl())) } diff --git a/examples/prefab/cores/boom/arch/native/Core.scala b/examples/prefab/cores/boom/arch/native/Core.scala new file mode 100644 index 000000000..8f3b41089 --- /dev/null +++ b/examples/prefab/cores/boom/arch/native/Core.scala @@ -0,0 +1,1516 @@ +// Copied from riscv-boom src/main/scala/v4/exu/core.scala and modified for Buckyball. +// Copyright (c) The Regents of the University of California (Regents). +// All Rights Reserved. See LICENSE and LICENSE.SiFive in riscv-boom for license details. + +package examples.core.boom.native + +import java.nio.file.{Paths} + +import chisel3._ +import chisel3.util._ + +import org.chipsalliance.cde.config.Parameters +import freechips.rocketchip.rocket.Instructions._ +import freechips.rocketchip.rocket.{CSR, CSRs, Causes, PRV, TracedInstruction} +import freechips.rocketchip.tile.{CustomCSR, HasFPUParameters, TraceBundle} +import freechips.rocketchip.util.{CoreMonitorBundle, PlusArg, Str, UIntIsOneOf} +import freechips.rocketchip.devices.tilelink.{CLINTConsts, PLICConsts} +import freechips.rocketchip.trace.{TraceCoreIngress, TraceCoreInterface, TraceCoreParams} + +import boom.v4.common._ +import boom.v4.ifu.{GlobalHistory, HasBoomFrontendParameters} +import boom.v4.util._ +import boom.v4.exu._ +import examples.core.boom.native.{DecodeUnit, UniqueExeUnit} + +/** + * Top level core object that connects the Frontend to the rest of the pipeline. + */ +class BoomCore(roccCSRs: Seq[Seq[CustomCSR]])(implicit p: Parameters) + extends BoomModule + with HasBoomFrontendParameters // TODO: Don't add this trait + with HasFPUParameters { + val nTotalRoCCCSRs = roccCSRs.flatten.size + + val traceIngressParams = TraceCoreParams( + nGroups = boomParams.retireWidth, + iretireWidth = 1, + xlen = coreParams.xLen, + iaddrWidth = vaddrBitsExtended + ) + + val io = IO(new freechips.rocketchip.tile.CoreBundle { + val hartid = Input(UInt(hartIdLen.W)) + val interrupts = Input(new freechips.rocketchip.rocket.CoreInterrupts(false)) + val ifu = new boom.v4.ifu.BoomFrontendIO + val ptw = Flipped(new freechips.rocketchip.rocket.DatapathPTWIO()) + val rocc = Flipped(new freechips.rocketchip.tile.RoCCCoreIO(nTotalRoCCCSRs)) + val lsu = Flipped(new boom.v4.lsu.LSUCoreIO) + val ptw_tlb = new freechips.rocketchip.rocket.TLBPTWIO() + val trace = Output(new TraceBundle) + val fcsr_rm = UInt(freechips.rocketchip.tile.FPConstants.RM_SZ.W) + val traceStall = Input(Bool()) + val trace_core_ingress = + if (boomParams.enableTraceCoreIngress) Some(Output(new TraceCoreInterface(traceIngressParams))) else None + }) + + io.ptw_tlb := DontCare + io.ptw := DontCare + io.ifu := DontCare + + //********************************** + // construct all of the modules + + val mem_exe_units: Seq[MemExeUnit] = (0 until memWidth) map { w => + Module(new MemExeUnit( + hasAGen = w >= (memWidth - lsuWidth), + hasDGen = true + )).suggestName(s"mem_exe_unit_$w") + } + + val agen_exe_units: Seq[MemExeUnit] = mem_exe_units.filter(_.hasAGen) + + val unq_exe_units: Seq[UniqueExeUnit] = Seq( + Module(new UniqueExeUnit( + hasCSR = true, + hasRocc = usingRoCC, + hasMul = true, + hasDiv = true, + hasIfpu = true + )).suggestName(s"unique_exe_unit_0") + ) + + val unq_exe_unit = unq_exe_units(0) + val csr_resp = unq_exe_unit.io_csr_resp.get + + val alu_exe_units: Seq[ALUExeUnit] = (0 until aluWidth) map { w => + Module(new ALUExeUnit( + id = w + )).suggestName(s"alu_exe_unit_$w") + } + + val all_exe_units = alu_exe_units ++ mem_exe_units ++ unq_exe_units + + // Meanwhile, the FP pipeline holds the FP issue window, FP regfile, and FP arithmetic units. + val fp_pipeline = Module(new FpPipeline) + + // ******************************************************** + // Clear fp_pipeline before use + fp_pipeline.io.ll_wports := DontCare + + val numIrfWritePorts = aluWidth + lsuWidth + 1 + val numIrfLogicalReadPorts = all_exe_units.map(_.nReaders).reduce(_ + _) + + val numIntWakeups = coreWidth + lsuWidth + 1 + val numFpWakeupPorts = fp_pipeline.io.wakeups.length + + val numImmReaders = aluWidth + memWidth + 1 // "Wakeup" immediates when they are read + + val decode_units = (0 until decodeWidth) map { w => Module(new DecodeUnit).suggestName(s"decode_$w") } + val dec_brmask_logic = Module(new BranchMaskGenerationLogic(coreWidth)) + val rename_stage = Module(new RenameStage(coreWidth, numIntPhysRegs, numIntWakeups, false)) + val fp_rename_stage = Module(new RenameStage(coreWidth, numFpPhysRegs, numFpWakeupPorts, true)) + val pred_rename_stage = Module(new PredRenameStage(coreWidth, 1)) + val imm_rename_stage = Module(new ImmRenameStage(coreWidth, numImmReaders)) // wakeup ports used when insts read imm + val rename_stages = + Seq(rename_stage, pred_rename_stage, imm_rename_stage) ++ (if (usingFPU) Seq(fp_rename_stage) else Nil) + + val mem_iss_unit = IssueUnit(memIssueParam, numIntWakeups, false, false) + val unq_iss_unit = IssueUnit(unqIssueParam, numIntWakeups, false, false) + val alu_iss_unit = IssueUnit(aluIssueParam, numIntWakeups, enableColumnALUIssue, enableALUSingleWideDispatch) + val dispatcher = Module(new BasicDispatcher) + + val iregfileBankedWriteArray = Seq.fill(lsuWidth + 1)(None) ++ (0 until aluWidth).map { w => + if (enableColumnALUWrites) Some(w) else None + } + + val iregfile = Module(new BankedRF( + UInt(xLen.W), + numIrfBanks, + numIrfLogicalReadPorts, + numIntPhysRegs, + numIrfLogicalReadPorts, + numIrfReadPorts, + numIrfWritePorts, + iregfileBankedWriteArray, + "Integer" + )) + + val pregfile = Module(new FullyPortedRF( + Bool(), + ftqSz, + aluWidth, + 1, + "Predicate" + )) + + val immregfile = Module(new FullyPortedRF( + UInt(LONGEST_IMM_SZ.W), + numImmPhysRegs, + numImmReaders, + coreWidth, + "Immediate" + )) + + val bregfile = Module(new FullyPortedRF( + new BrInfoBundle, + maxBrCount, + aluWidth, + coreWidth, + "Branch" + )) + + val rob = Module(new Rob( + numIrfWritePorts + numFpWakeupPorts, + trace + )) + + // Used to wakeup registers in rename and issue. ROB needs to listen to something else. + val int_wakeups = Wire(Vec(numIntWakeups, Valid(new Wakeup))) + val pred_wakeups = Wire(Vec(aluWidth, Valid(new Wakeup))) + + // The arb stage guarantees only 1 pred wakeup per cycle + assert(PopCount(pred_wakeups.map(_.valid)) <= 1.U) + val pred_wakeup = Wire(Valid(new Wakeup)) + pred_wakeup.valid := pred_wakeups.map(_.valid).reduce(_ || _) + pred_wakeup.bits := DontCare + pred_wakeup.bits.uop.pdst := Mux1H(pred_wakeups.map(_.valid), pred_wakeups.map(_.bits.uop.pdst)) + + val int_bypasses = Wire(Vec(coreWidth + lsuWidth, Valid(new ExeUnitResp(xLen)))) + + //*********************************** + // Pipeline State Registers and Wires + + // Decode/Rename1 Stage + val dec_valids = Wire(Vec(coreWidth, Bool())) // are the decoded instruction valid? It may be held up though. + val dec_uops = Wire(Vec(coreWidth, new MicroOp())) + val dec_fire = Wire(Vec(coreWidth, Bool())) // can the instruction fire beyond decode? + // (can still be stopped in ren or dis) + val dec_ready = Wire(Bool()) + val dec_xcpts = Wire(Vec(coreWidth, Bool())) + val ren_stalls = Wire(Vec(coreWidth, Bool())) + + // Rename2/Dispatch stage + val dis_valids = Wire(Vec(coreWidth, Bool())) + val dis_uops = Wire(Vec(coreWidth, new MicroOp)) + val dis_fire = Wire(Vec(coreWidth, Bool())) + val dis_ready = Wire(Bool()) + + // Issue Stage/Register Read + val mem_iss_uops = mem_iss_unit.io.iss_uops + val alu_iss_uops = alu_iss_unit.io.iss_uops + val unq_iss_uops = unq_iss_unit.io.iss_uops + + // -------------------------------------- + // Dealing with branch resolutions + + // The individual branch resolutions from each ALU + val brinfos = Reg(Vec(coreWidth, Valid(new BrResolutionInfo))) + + // "Merged" branch update info from all ALUs + // brmask contains masks for rapidly clearing mispredicted instructions + // brindices contains indices to reset pointers for allocated structures + // brindices is delayed a cycle + val brupdate = Wire(new BrUpdateInfo) + val b1 = Wire(new BrUpdateMasks) + val b2 = Reg(new BrResolutionInfo) + + brupdate.b1 := b1 + brupdate.b2 := b2 + + for ((b, a) <- brinfos zip alu_exe_units) { + b.bits := UpdateBrMask(brupdate, a.io_brinfo.bits) + b.valid := a.io_brinfo.valid && !rob.io.flush.valid && !IsKilledByBranch( + brupdate, + RegNext(rob.io.flush.valid), + a.io_brinfo.bits + ) + } + b1.resolve_mask := brinfos.map(x => x.valid << x.bits.uop.br_tag).reduce(_ | _) + b1.mispredict_mask := brinfos.map(x => (x.valid && x.bits.mispredict) << x.bits.uop.br_tag).reduce(_ | _) + + // Find the oldest mispredict and use it to update indices + val live_brinfos = brinfos.map(br => + br.valid && br.bits.mispredict && !IsKilledByBranch(brupdate, RegNext(rob.io.flush.valid), br.bits.uop) + ) + + val mispredict_val = live_brinfos.reduce(_ || _) + + b2.mispredict := mispredict_val + b2.cfi_type := Mux1H(live_brinfos, brinfos.map(_.bits.cfi_type)) + b2.taken := Mux1H(live_brinfos, brinfos.map(_.bits.taken)) + b2.pc_sel := Mux1H(live_brinfos, brinfos.map(_.bits.pc_sel)) + b2.uop := UpdateBrMask(brupdate, PriorityMux(live_brinfos, brinfos.map(_.bits.uop))) + b2.jalr_target := Mux1H(live_brinfos, brinfos.map(_.bits.jalr_target)) + b2.target_offset := Mux1H(live_brinfos, brinfos.map(_.bits.target_offset)) + + val oldest_mispredict_ftq_idx = Mux1H(live_brinfos, brinfos.map(_.bits.uop.ftq_idx)) + + assert( + !((brupdate.b1.mispredict_mask =/= 0.U || brupdate.b2.mispredict) + && rob.io.rollback), + "Can't have a mispredict during rollback." + ) + + io.ifu.brupdate := brupdate + + for (eu <- all_exe_units) { + eu.io_brupdate := brupdate + } + + fp_pipeline.io.brupdate := brupdate + + // Load/Store Unit & ExeUnits + // val mem_resps = io.lsu.iresp + var agen_idx = 0 + var dgen_idx = 0 + for (eu <- mem_exe_units) { + if (eu.hasAGen) { + io.lsu.agen(agen_idx) := eu.io_agen.get + agen_idx += 1 + } + if (eu.hasDGen) { + io.lsu.dgen(dgen_idx) := eu.io_dgen.get + dgen_idx += 1 + } + } + io.lsu.dgen(dgen_idx) := fp_pipeline.io.dgen + + //------------------------------------------------------------- + // Uarch Hardware Performance Events (HPEs) + + val perfEvents = new freechips.rocketchip.rocket.EventSets(Seq( + new freechips.rocketchip.rocket.EventSet( + (mask, hits) => (mask & hits).orR, + Seq( + ("exception", () => rob.io.com_xcpt.valid), + ("nop", () => false.B), + ("nop", () => false.B), + ("nop", () => false.B) + ) + ), + new freechips.rocketchip.rocket.EventSet( + (mask, hits) => (mask & hits).orR, + Seq( +// ("I$ blocked", () => icache_blocked), + ("nop", () => false.B), + ("branch misprediction", () => b2.mispredict), + ( + "control-flow target misprediction", + () => + b2.mispredict && + b2.cfi_type === CFI_JALR + ), + ("flush", () => rob.io.flush.valid) +// ("branch resolved", () => b2.valid) + ) + ), + new freechips.rocketchip.rocket.EventSet( + (mask, hits) => (mask & hits).orR, + Seq( + ("I$ miss", () => io.ifu.perf.acquire), + ("D$ miss", () => io.lsu.perf.acquire), + ("D$ release", () => io.lsu.perf.release), + ("ITLB miss", () => io.ifu.perf.tlbMiss), + ("DTLB miss", () => io.lsu.perf.tlbMiss), + ("L2 TLB miss", () => io.ptw.perf.l2miss) + ) + ) + )) + + val csr = Module(new freechips.rocketchip.rocket.CSRFile(perfEvents, boomParams.customCSRs.decls, roccCSRs.flatten)) + csr.io.inst foreach { c => c := DontCare } + csr.io.rocc_interrupt := io.rocc.interrupt + csr.io.gva := DontCare + csr.io.htval := DontCare + csr.io.mhtinst_read_pseudo := false.B + + val custom_csrs = Wire(new BoomCustomCSRs) + custom_csrs.csrs.foreach { c => c.stall := false.B; c.set := false.B; c.sdata := DontCare } + (custom_csrs.csrs zip csr.io.customCSRs).map { case (lhs, rhs) => lhs <> rhs } + io.ifu.enable_bpd := custom_csrs.enableBPD + + //val icache_blocked = !(io.ifu.fetchpacket.valid || RegNext(io.ifu.fetchpacket.valid)) + val icache_blocked = false.B + csr.io.counters foreach { c => c.inc := RegNext(perfEvents.evaluate(c.eventSel)) } + + //**************************************** + // Time Stamp Counter & Retired Instruction Counter + // (only used for printf and vcd dumps - the actual counters are in the CSRFile) + val debug_tsc_reg = RegInit(0.U(xLen.W)) + val debug_irt_reg = RegInit(0.U(xLen.W)) + val debug_brs = RegInit(VecInit(Seq.fill(5)(0.U(xLen.W)))) + val debug_jals = RegInit(VecInit(Seq.fill(5)(0.U(xLen.W)))) + val debug_jalrs = RegInit(VecInit(Seq.fill(5)(0.U(xLen.W)))) + + for (j <- 0 until 5) { + debug_brs(j) := debug_brs(j) + PopCount(VecInit((0 until coreWidth) map { i => + rob.io.commit.arch_valids(i) && + (rob.io.commit.uops(i).debug_fsrc === j.U) && + rob.io.commit.uops(i).is_br + })) + debug_jals(j) := debug_jals(j) + PopCount(VecInit((0 until coreWidth) map { i => + rob.io.commit.arch_valids(i) && + (rob.io.commit.uops(i).debug_fsrc === j.U) && + rob.io.commit.uops(i).is_jal + })) + debug_jalrs(j) := debug_jalrs(j) + PopCount(VecInit((0 until coreWidth) map { i => + rob.io.commit.arch_valids(i) && + (rob.io.commit.uops(i).debug_fsrc === j.U) && + rob.io.commit.uops(i).is_jalr + })) + } + + dontTouch(debug_brs) + dontTouch(debug_jals) + dontTouch(debug_jalrs) + + debug_tsc_reg := debug_tsc_reg + 1.U + debug_irt_reg := debug_irt_reg + PopCount(rob.io.commit.arch_valids.asUInt) + dontTouch(debug_tsc_reg) + dontTouch(debug_irt_reg) + + //**************************************** + // Print-out information about the machine + + val issStr = "(Age-based Priority)" + + val fpPipelineStr = fp_pipeline.toString + + override def toString: String = + (BoomCoreStringPrefix("====Overall Core Params====") + "\n\n" + + mem_exe_units.map(_.toString).mkString("") + "\n" + + unq_exe_units.map(_.toString).mkString("") + "\n\n" + + alu_exe_units.map(_.toString).mkString("") + "\n\n" + + fpPipelineStr + "\n\n" + + rob.toString + "\n\n" + + BoomCoreStringPrefix( + "===Other Core Params===", + "Fetch Width : " + fetchWidth, + "Decode Width : " + coreWidth, + "Issue Width : " + issueParams.map(_.issueWidth).sum, + "ROB Size : " + numRobEntries, + "Issue Window Size : " + issueParams.map(_.numEntries) + issStr, + "Load/Store Unit Size : " + numLdqEntries + "/" + numStqEntries, + "Num Int Phys Registers: " + numIntPhysRegs, + "Num FP Phys Registers: " + numFpPhysRegs, + "Load-to-use delay : " + (if (enableFastLoadUse) 4 else 5), + "Max Branch Count : " + maxBrCount + ) + + "\n\n" + iregfile.toString + "\n\n" + + BoomCoreStringPrefix( + "Num Wakeup Ports : " + numIntWakeups, + "Num Bypass Ports : " + int_bypasses.length + ) + "\n" + + BoomCoreStringPrefix( + "DCache Ways : " + dcacheParams.nWays, + "DCache Sets : " + dcacheParams.nSets, + "DCache nMSHRs : " + dcacheParams.nMSHRs, + "ICache Ways : " + icacheParams.nWays, + "ICache Sets : " + icacheParams.nSets, + "D-TLB Ways : " + dcacheParams.nTLBWays, + "I-TLB Ways : " + icacheParams.nTLBWays, + "Paddr Bits : " + paddrBits, + "Vaddr Bits : " + vaddrBits + ) + "\n" + + BoomCoreStringPrefix( + "Using FPU Unit? : " + usingFPU.toString, + "Using FDivSqrt? : " + usingFDivSqrt.toString, + "Using VM? : " + usingVM.toString + ) + "\n") + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Fetch Stage/Frontend **** + //------------------------------------------------------------- + //------------------------------------------------------------- + io.ifu.redirect_val := false.B + io.ifu.redirect_flush := false.B + + // Breakpoint info + io.ifu.status := csr.io.status + io.ifu.bp := csr.io.bp + io.ifu.mcontext := csr.io.mcontext + io.ifu.scontext := csr.io.scontext + + io.ifu.flush_icache := (0 until coreWidth).map { i => + (rob.io.commit.arch_valids(i) && rob.io.commit.uops(i).is_fencei) || + (RegNext(dec_valids(i) && dec_uops(i).is_jalr && csr.io.status.debug)) + }.reduce(_ || _) + + // TODO FIX THIS HACK + // The below code works because of two quirks with the flush mechanism + // 1 ) All flush_on_commit instructions are also is_unique, + // In the future, this constraint will be relaxed. + // 2 ) We send out flush signals one cycle after the commit signal. We need to + // mux between one/two cycle delay for the following cases: + // ERETs are reported to the CSR two cycles before we send the flush + // Exceptions are reported to the CSR on the cycle we send the flush + // This discrepency should be resolved elsewhere. + when(RegNext(rob.io.flush.valid)) { + io.ifu.redirect_val := true.B + io.ifu.redirect_flush := true.B + val flush_typ = RegNext(rob.io.flush.bits.flush_typ) + // Clear the global history when we flush the ROB (exceptions, AMOs, unique instructions, etc.) + val new_ghist = WireInit((0.U).asTypeOf(new GlobalHistory)) + new_ghist.current_saw_branch_not_taken := true.B + new_ghist.ras_idx := io.ifu.rrd_ftq_resps(0).entry.ras_idx + io.ifu.redirect_ghist := new_ghist + when(FlushTypes.useCsrEvec(flush_typ)) { + io.ifu.redirect_pc := Mux(flush_typ === FlushTypes.eret, ShiftRegister(csr.io.evec, 3), csr.io.evec) + }.otherwise { + val flush_pc = (AlignPCToBoundary(io.ifu.rrd_ftq_resps(0).pc, icBlockBytes) + + RegNext(rob.io.flush.bits.pc_lob) + - Mux(RegNext(rob.io.flush.bits.edge_inst), 2.U, 0.U)) + val flush_pc_next = flush_pc + Mux(RegNext(rob.io.flush.bits.is_rvc), 2.U, 4.U) + io.ifu.redirect_pc := Mux(FlushTypes.useSamePC(flush_typ), flush_pc, flush_pc_next) + + } + io.ifu.redirect_ftq_idx := RegNext(rob.io.flush.bits.ftq_idx) + }.elsewhen(brupdate.b2.mispredict && !RegNext(rob.io.flush.valid)) { + val block_pc = AlignPCToBoundary(io.ifu.rrd_ftq_resps(0).pc, icBlockBytes) + val uop_maybe_pc = block_pc | brupdate.b2.uop.pc_lob + val npc = uop_maybe_pc + Mux(brupdate.b2.uop.is_rvc || brupdate.b2.uop.edge_inst, 2.U, 4.U) + val jal_br_target = Wire(UInt(vaddrBitsExtended.W)) + jal_br_target := (uop_maybe_pc.asSInt + brupdate.b2.target_offset + + (Fill(vaddrBitsExtended - 1, brupdate.b2.uop.edge_inst) << 1).asSInt).asUInt + val bj_addr = Mux(brupdate.b2.cfi_type === CFI_JALR, brupdate.b2.jalr_target, jal_br_target) + val mispredict_target = Mux(brupdate.b2.pc_sel === PC_PLUS4, npc, bj_addr) + io.ifu.redirect_val := true.B + io.ifu.redirect_pc := mispredict_target + io.ifu.redirect_flush := true.B + io.ifu.redirect_ftq_idx := brupdate.b2.uop.ftq_idx + val use_same_ghist = brupdate.b2.cfi_type === CFI_BR && + !brupdate.b2.taken && + bankAlign(block_pc) === bankAlign(npc) + val ftq_entry = io.ifu.rrd_ftq_resps(0).entry + val cfi_idx = (brupdate.b2.uop.pc_lob ^ + Mux(ftq_entry.start_bank === 1.U, 1.U << log2Ceil(bankBytes), 0.U))(log2Ceil(fetchWidth), 1) + val ftq_ghist = io.ifu.rrd_ftq_resps(0).ghist + val next_ghist = ftq_ghist.update( + ftq_entry.br_mask.asUInt, + brupdate.b2.taken, + brupdate.b2.cfi_type === CFI_BR, + cfi_idx, + true.B, + io.ifu.rrd_ftq_resps(0).pc, + ftq_entry.cfi_is_call && ftq_entry.cfi_idx.bits === cfi_idx, + ftq_entry.cfi_is_ret && ftq_entry.cfi_idx.bits === cfi_idx + ) + + io.ifu.redirect_ghist := Mux( + use_same_ghist, + ftq_ghist, + next_ghist + ) + io.ifu.redirect_ghist.current_saw_branch_not_taken := use_same_ghist + }.elsewhen(rob.io.flush_frontend || brupdate.b1.mispredict_mask =/= 0.U) { + io.ifu.redirect_flush := true.B + } + + // Tell the FTQ it can deallocate entries by passing youngest ftq_idx. + val youngest_com_idx = (coreWidth - 1).U - PriorityEncoder(rob.io.commit.valids.reverse) + io.ifu.commit.valid := rob.io.commit.valids.reduce(_ | _) || rob.io.com_xcpt.valid + io.ifu.commit.bits := Mux( + rob.io.com_xcpt.valid, + rob.io.com_xcpt.bits.ftq_idx, + rob.io.commit.uops(youngest_com_idx).ftq_idx + ) + + assert(!(rob.io.commit.valids.reduce(_ | _) && rob.io.com_xcpt.valid), "ROB can't commit and except in same cycle!") + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Decode Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + // track mask of finished instructions in the bundle + // use this to mask out insts coming from FetchBuffer that have been finished + // for example, back pressure may cause us to only issue some instructions from FetchBuffer + // but on the next cycle, we only want to retry a subset + val dec_finished_mask = RegInit(0.U(coreWidth.W)) + + //------------------------------------------------------------- + // Pull out instructions and send to the Decoders + + io.ifu.fetchpacket.ready := dec_ready + val dec_fbundle = io.ifu.fetchpacket.bits + + //------------------------------------------------------------- + // Decoders + + for (w <- 0 until coreWidth) { + dec_valids(w) := io.ifu.fetchpacket.valid && dec_fbundle.uops(w).valid && + !dec_finished_mask(w) + decode_units(w).io.enq.uop := dec_fbundle.uops(w).bits + decode_units(w).io.status := csr.io.status + decode_units(w).io.csr_decode <> csr.io.decode(w) + decode_units(w).io.interrupt := RegNext(csr.io.interrupt) + decode_units(w).io.interrupt_cause := RegNext(csr.io.interrupt_cause) + decode_units(w).io.fcsr_rm := csr.io.fcsr_rm + + dec_uops(w) := decode_units(w).io.deq.uop + } + + //------------------------------------------------------------- + // FTQ GetPC Port Arbitration + // 3 ports + // port0 goes to flush,mispredict,xcpt + // port1/2 goes to jmp unit + + val xcpt_pc_req = Wire(Decoupled(UInt(log2Ceil(ftqSz).W))) + val mispredict_pc_req = Wire(Decoupled(UInt(log2Ceil(ftqSz).W))) + val flush_pc_req = Wire(Decoupled(UInt(log2Ceil(ftqSz).W))) + + val ftq_arb = Module(new Arbiter(UInt(log2Ceil(ftqSz).W), 3)) + + // Order by the oldest. Flushes come from the oldest instructions in pipe + // Decoding exceptions come from youngest + ftq_arb.io.in(0) <> flush_pc_req + ftq_arb.io.in(1) <> mispredict_pc_req + ftq_arb.io.in(2) <> xcpt_pc_req + + io.ifu.arb_ftq_reqs(0) := ftq_arb.io.out.bits + ftq_arb.io.out.ready := true.B + + val ftq_port_issued = Array.fill(2)(false.B) + val ftq_port_addrs = Array.fill(2)(0.U(log2Ceil(ftqSz).W)) + for (i <- 0 until aluWidth) { + for (w <- 0 until 2) { + val req = alu_exe_units(i).io_arb_ftq_reqs(w) + var read_issued = false.B + val data_sel = WireInit(0.U(2.W)) + for (j <- 0 until 2) { + val issue_read = WireInit(false.B) + val use_port = WireInit(false.B) + when(!read_issued && !ftq_port_issued(j) && req.valid) { + issue_read := true.B + use_port := true.B + data_sel := UIntToOH(j.U) + } + val was_port_issued_yet = ftq_port_issued(j) + ftq_port_issued(j) = use_port || ftq_port_issued(j) + ftq_port_addrs(j) = ftq_port_addrs(j) | Mux(was_port_issued_yet || !use_port, 0.U, req.bits) + read_issued = issue_read || read_issued + } + req.ready := read_issued + alu_exe_units(i).io_rrd_ftq_resps(w) := Mux(RegNext(data_sel(0)), io.ifu.rrd_ftq_resps(1), io.ifu.rrd_ftq_resps(2)) + + } + } + for (j <- 0 until 2) { + io.ifu.arb_ftq_reqs(j + 1) := ftq_port_addrs(j) + } + + // Frontend Exception Requests + val xcpt_idx = PriorityEncoder(dec_xcpts) + xcpt_pc_req.valid := dec_xcpts.reduce(_ || _) + xcpt_pc_req.bits := dec_uops(xcpt_idx).ftq_idx + rob.io.xcpt_fetch_pc := io.ifu.rrd_ftq_resps(0).pc + + flush_pc_req.valid := rob.io.flush.valid + flush_pc_req.bits := rob.io.flush.bits.ftq_idx + + // Mispredict requests (to get the correct target) + mispredict_pc_req.valid := mispredict_val + mispredict_pc_req.bits := oldest_mispredict_ftq_idx + + //------------------------------------------------------------- + // Decode/Rename1 pipeline logic + + dec_xcpts := dec_uops zip dec_valids map { case (u, v) => u.exception && v } + // Frontend exceptions need to shoot straight past dec/dis without stall, to match + // the timing of the FTQ resp which provides badaddr. + // Wait for pipeline to empty before letting an exception past dec + val dec_prior_slot_valid = dec_valids.scanLeft(false.B)((s, v) => s || v) + + val dec_xcpt_stall = (0 until coreWidth).map(w => + dec_xcpts(w) && + (!rob.io.empty || !io.lsu.fencei_rdy || dec_prior_slot_valid(w) || dis_valids.reduce(_ || _) || !xcpt_pc_req.ready) + ) + + // stall fetch/dcode because we ran out of branch tags + val branch_mask_full = Wire(Vec(coreWidth, Bool())) + + val dec_hazards = (0 until coreWidth).map(w => + dec_valids(w) && + (!dis_ready + || rob.io.rollback + || dec_xcpt_stall(w) + || branch_mask_full(w) + || brupdate.b1.mispredict_mask =/= 0.U + || brupdate.b2.mispredict + || io.ifu.redirect_flush) + ) + + val dec_stalls = dec_hazards.scanLeft(false.B)((s, h) => s || h).takeRight(coreWidth) + dec_fire := (0 until coreWidth).map(w => dec_valids(w) && !dec_stalls(w)) + + // all decoders are empty and ready for new instructions + dec_ready := dec_fire.last + + when(dec_ready || io.ifu.redirect_flush) { + dec_finished_mask := 0.U + }.otherwise { + dec_finished_mask := dec_fire.asUInt | dec_finished_mask + } + + //------------------------------------------------------------- + // Branch Mask Logic + + dec_brmask_logic.io.brupdate := brupdate + dec_brmask_logic.io.flush_pipeline := RegNext(rob.io.flush.valid) + + for (w <- 0 until coreWidth) { + dec_brmask_logic.io.is_branch(w) := !dec_finished_mask(w) && dec_uops(w).allocate_brtag + dec_brmask_logic.io.will_fire(w) := dec_fire(w) && + dec_uops(w).allocate_brtag // ren, dis can back pressure us + dec_uops(w).br_tag := dec_brmask_logic.io.br_tag(w) + dec_uops(w).br_mask := dec_brmask_logic.io.br_mask(w) + } + + branch_mask_full := dec_brmask_logic.io.is_full + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Register Rename Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + // Inputs + for (rename <- rename_stages) { + rename.io.kill := io.ifu.redirect_flush + rename.io.brupdate := brupdate + + rename.io.debug_rob_empty := rob.io.empty + + rename.io.dec_fire := dec_fire + rename.io.dec_uops := dec_uops + + rename.io.dis_fire := dis_fire + rename.io.dis_ready := dis_ready + + rename.io.com_valids := rob.io.commit.valids + rename.io.com_uops := rob.io.commit.uops + rename.io.rollback := rob.io.rollback + } + + // Outputs + dis_uops := rename_stage.io.ren2_uops + dis_valids := rename_stage.io.ren2_mask + ren_stalls := rename_stage.io.ren_stalls + + /** + * TODO This is a bit nasty, but it's currently necessary to + * split the INT/FP rename pipelines into separate instantiations. + * Won't have to do this anymore with a properly decoupled FP pipeline. + */ + for (w <- 0 until coreWidth) { + val i_uop = rename_stage.io.ren2_uops(w) + val f_uop = fp_rename_stage.io.ren2_uops(w) + val p_uop = pred_rename_stage.io.ren2_uops(w) + val imm_uop = imm_rename_stage.io.ren2_uops(w) + val f_stall = fp_rename_stage.io.ren_stalls(w) + val p_stall = pred_rename_stage.io.ren_stalls(w) + val imm_stall = imm_rename_stage.io.ren_stalls(w) + + // lrs1 can "pass through" to prs1. Used solely to index the csr file. + dis_uops(w).prs1 := Mux( + dis_uops(w).lrs1_rtype === RT_FLT, + f_uop.prs1, + Mux(dis_uops(w).lrs1_rtype === RT_FIX, i_uop.prs1, dis_uops(w).lrs1) + ) + dis_uops(w).prs2 := Mux(dis_uops(w).lrs2_rtype === RT_FLT, f_uop.prs2, i_uop.prs2) + dis_uops(w).prs3 := f_uop.prs3 + dis_uops(w).ppred := p_uop.ppred + dis_uops(w).pdst := Mux( + dis_uops(w).dst_rtype === RT_FLT, + f_uop.pdst, + Mux( + dis_uops(w).dst_rtype === RT_FIX, + i_uop.pdst, + Mux( + dis_uops(w).is_sfb_br, + p_uop.pdst, + random.LFSR(maxPregSz) // Random set the PDST so the ALU banked IQ is balanced + ) + ) + ) + dis_uops(w).imm_sel := imm_uop.imm_sel + dis_uops(w).pimm := imm_uop.pimm + + dis_uops(w).stale_pdst := Mux(dis_uops(w).dst_rtype === RT_FLT, f_uop.stale_pdst, i_uop.stale_pdst) + + dis_uops(w).prs1_busy := i_uop.prs1_busy && (dis_uops(w).lrs1_rtype === RT_FIX) || + f_uop.prs1_busy && (dis_uops(w).lrs1_rtype === RT_FLT) + dis_uops(w).prs2_busy := i_uop.prs2_busy && (dis_uops(w).lrs2_rtype === RT_FIX) || + f_uop.prs2_busy && (dis_uops(w).lrs2_rtype === RT_FLT) + dis_uops(w).prs3_busy := f_uop.prs3_busy && dis_uops(w).frs3_en + dis_uops(w).ppred_busy := p_uop.ppred_busy && dis_uops(w).is_sfb_shadow + + ren_stalls(w) := rename_stage.io.ren_stalls(w) || f_stall || p_stall || imm_stall + } + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Dispatch Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + //------------------------------------------------------------- + // Rename2/Dispatch pipeline logic + + val dis_prior_slot_valid = dis_valids.scanLeft(false.B)((s, v) => s || v) + val dis_prior_slot_unique = (dis_uops zip dis_valids).scanLeft(false.B) { case (s, (u, v)) => s || v && u.is_unique } + + val wait_for_empty_pipeline = (0 until coreWidth).map(w => + (dis_uops(w).is_unique || !custom_csrs.enableOOO) && + (!rob.io.empty || !io.lsu.fencei_rdy || dis_prior_slot_valid(w)) + ) + + val rocc_shim_busy = if (usingRoCC) !unq_exe_unit.io_rocc_core.get.rxq_empty else false.B + val wait_for_rocc = (0 until coreWidth).map(w => (io.rocc.busy || rocc_shim_busy)) + val rxq_full = if (usingRoCC) unq_exe_unit.io_rocc_core.get.rxq_full else false.B + val block_rocc = (dis_uops zip dis_valids).map { case (u, v) => v && u.is_rocc }.scanLeft(rxq_full)(_ || _) + + val dis_rocc_alloc_stall = (dis_uops.map(_.is_rocc) zip block_rocc) map { case (p, r) => + if (usingRoCC) p && r else false.B + } + + // Only 1 branch-tag allocating insruction allowed to proceed per cycle unless enableSuperscalarSnapshots + // This reduces checkpointing complexity + val block_brtag = (dis_uops zip dis_valids).map { case (u, v) => v && u.allocate_brtag }.scanLeft(false.B)(_ || _) + + val brtag_stall = (dis_uops.map(_.allocate_brtag) zip block_brtag) map { case (p, r) => + if (enableSuperscalarSnapshots) false.B else (p && r) + } + + val dis_hazards = (0 until coreWidth).map(w => + dis_valids(w) && + (!rob.io.ready + || brtag_stall(w) + || ren_stalls(w) + || io.lsu.ldq_full(w) && dis_uops(w).uses_ldq + || io.lsu.stq_full(w) && dis_uops(w).uses_stq + || !dispatcher.io.ren_uops(w).ready + || wait_for_empty_pipeline(w) + || wait_for_rocc(w) + || dis_prior_slot_unique(w) + || dis_rocc_alloc_stall(w) + || brupdate.b1.mispredict_mask =/= 0.U + || brupdate.b2.mispredict + || io.ifu.redirect_flush) + ) + + io.lsu.fence_dmem := (dis_valids zip wait_for_empty_pipeline).map { case (v, w) => v && w }.reduce(_ || _) + + val dis_stalls = dis_hazards.scanLeft(false.B)((s, h) => s || h).takeRight(coreWidth) + dis_fire := dis_valids zip dis_stalls map { case (v, s) => v && !s } + dis_ready := !dis_stalls.last + + //------------------------------------------------------------- + // LDQ/STQ Allocation Logic + + for (w <- 0 until coreWidth) { + // Dispatching instructions request load/store queue entries when they can proceed. + dis_uops(w).ldq_idx := io.lsu.dis_ldq_idx(w) + dis_uops(w).stq_idx := io.lsu.dis_stq_idx(w) + } + + //------------------------------------------------------------- + // Rob Allocation Logic + + rob.io.enq_valids := dis_fire + rob.io.enq_uops := dis_uops + rob.io.enq_partial_stall := dis_stalls.last // TODO come up with better ROB compacting scheme. + rob.io.debug_tsc := debug_tsc_reg + rob.io.csr_stall := csr.io.csr_stall + rob.io.trace_stall := io.traceStall + + // Minor hack: ecall and breaks need to increment the FTQ deq ptr earlier than commit, since + // they write their PC into the CSR the cycle before they commit. + // Since these are also unique, increment the FTQ ptr when they are dispatched + when(RegNext(dis_fire.reduce(_ || _) && dis_uops(PriorityEncoder(dis_fire)).is_sys_pc2epc)) { + io.ifu.commit.valid := true.B + io.ifu.commit.bits := RegNext(dis_uops(PriorityEncoder(dis_valids)).ftq_idx) + } + + for (w <- 0 until coreWidth) { + // note: this assumes uops haven't been shifted - there's a 1:1 match between PC's LSBs and "w" here + // (thus the LSB of the rob_idx gives part of the PC) + if (coreWidth == 1) { + dis_uops(w).rob_idx := rob.io.rob_tail_idx + } else { + dis_uops(w).rob_idx := Cat(rob.io.rob_tail_idx >> log2Ceil(coreWidth).U, w.U(log2Ceil(coreWidth).W)) + } + } + + //------------------------------------------------------------- + // RoCC allocation logic + if (usingRoCC) { + val rocc_unit = unq_exe_unit + for (w <- 0 until coreWidth) { + // We guarantee only decoding 1 RoCC instruction per cycle + dis_uops(w).rxq_idx := rocc_unit.io_rocc_core.get.rxq_idx(w) + } + } + + //------------------------------------------------------------- + // Dispatch to issue queues + + // Get uops from rename2 + for (w <- 0 until coreWidth) { + dispatcher.io.ren_uops(w).valid := dis_fire(w) + dispatcher.io.ren_uops(w).bits := dis_uops(w) + } + + var iu_idx = 0 + // Send dispatched uops to correct issue queues + // Backpressure through dispatcher if necessary + for (i <- 0 until issueParams.size) { + if (issueParams(i).iqType == IQ_FP) { + fp_pipeline.io.dis_uops <> dispatcher.io.dis_uops(i) + } else if (issueParams(i).iqType == IQ_MEM) { + mem_iss_unit.io.dis_uops <> dispatcher.io.dis_uops(i) + } else if (issueParams(i).iqType == IQ_ALU) { + alu_iss_unit.io.dis_uops <> dispatcher.io.dis_uops(i) + } else if (issueParams(i).iqType == IQ_UNQ) { + unq_iss_unit.io.dis_uops <> dispatcher.io.dis_uops(i) + } else { + require(false) + } + } + + //------------------------------------------------------------- + // Write immediates, branches into immediate file + for (w <- 0 until coreWidth) { + val uop = RegNext(dis_uops(w)) + + immregfile.io.write_ports(w).valid := RegNext(dis_fire(w)) && !uop.imm_sel.isOneOf(IS_N, IS_SH) + immregfile.io.write_ports(w).bits.addr := uop.pimm + immregfile.io.write_ports(w).bits.data := uop.imm_packed + + bregfile.io.write_ports(w).valid := RegNext(dis_fire(w)) && uop.allocate_brtag + bregfile.io.write_ports(w).bits.addr := uop.br_tag + bregfile.io.write_ports(w).bits.data.ldq_idx := uop.ldq_idx + bregfile.io.write_ports(w).bits.data.stq_idx := uop.stq_idx + bregfile.io.write_ports(w).bits.data.rxq_idx := uop.rxq_idx + } + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Issue Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + var wu_idx = 0 + var wb_idx = 0 + var bypass_idx = 0 + for (i <- 0 until lsuWidth) { + int_wakeups(wu_idx) := io.lsu.iwakeups(i) + rob.io.wb_resps(wb_idx) := RegNext(UpdateBrMask(brupdate, RegNext(rob.io.flush.valid), io.lsu.iresp(i))) + iregfile.io.write_ports(wb_idx).valid := RegNext(io.lsu.iresp(i).valid) + iregfile.io.write_ports(wb_idx).bits.addr := RegNext(io.lsu.iresp(i).bits.uop.pdst) + iregfile.io.write_ports(wb_idx).bits.data := RegNext(io.lsu.iresp(i).bits.data) + + int_bypasses(bypass_idx).valid := RegNext(io.lsu.iresp(i).valid) + int_bypasses(bypass_idx).bits := RegNext(io.lsu.iresp(i).bits) + wu_idx += 1 + wb_idx += 1 + bypass_idx += 1 + } + + val ll_arb = Module(new Arbiter( + new ExeUnitResp(xLen), + 1 + // Mul + 1 + // F2I + (if (unq_exe_unit.hasDiv) 1 else 0) + + (if (unq_exe_unit.hasCSR) 1 else 0) + + (if (unq_exe_unit.hasRocc) 1 else 0) + )) + + var arb_idx = 0 + ll_arb.io.in(arb_idx).valid := unq_exe_unit.io_mul_resp.get.valid + ll_arb.io.in(arb_idx).bits := unq_exe_unit.io_mul_resp.get.bits + arb_idx += 1 + ll_arb.io.in(arb_idx) <> fp_pipeline.io.to_int + arb_idx += 1 + if (unq_exe_unit.hasDiv) { + ll_arb.io.in(arb_idx) <> unq_exe_unit.io_div_resp.get + arb_idx += 1 + } + if (unq_exe_unit.hasCSR) { + ll_arb.io.in(arb_idx).valid := csr_resp.valid + ll_arb.io.in(arb_idx).bits.uop := csr_resp.bits.uop + val rdata = WireInit(csr.io.rw.rdata) + if (enableBPDHPMs) { + var hpmcounter = CSRs.hpmcounter16 + var mhpmcounter = CSRs.mhpmcounter16 + for (i <- 0 until 5) { + when(csr.io.rw.addr === hpmcounter.U || csr.io.rw.addr === mhpmcounter.U) { + rdata := debug_brs(i) + } + hpmcounter = hpmcounter + 1 + mhpmcounter = mhpmcounter + 1 + when(csr.io.rw.addr === hpmcounter.U || csr.io.rw.addr === mhpmcounter.U) { + rdata := debug_jals(i) + } + hpmcounter = hpmcounter + 1 + mhpmcounter = mhpmcounter + 1 + when(csr.io.rw.addr === hpmcounter.U || csr.io.rw.addr === mhpmcounter.U) { + rdata := debug_jalrs(i) + } + hpmcounter = hpmcounter + 1 + mhpmcounter = mhpmcounter + 1 + } + } + ll_arb.io.in(arb_idx).bits.data := rdata + ll_arb.io.in(arb_idx).bits.predicated := false.B + ll_arb.io.in(arb_idx).bits.fflags.valid := false.B + ll_arb.io.in(arb_idx).bits.fflags.bits := false.B + assert(!(ll_arb.io.in(arb_idx).valid && !ll_arb.io.in(arb_idx).ready)) + arb_idx += 1 + } + if (unq_exe_unit.hasRocc) { + ll_arb.io.in(arb_idx) <> unq_exe_unit.io_rocc_resp.get + arb_idx += 1 + } + ll_arb.io.out.ready := true.B + int_wakeups(wu_idx).valid := ll_arb.io.out.valid && ll_arb.io.out.bits.uop.dst_rtype === RT_FIX + int_wakeups(wu_idx).bits.uop := ll_arb.io.out.bits.uop + int_wakeups(wu_idx).bits.speculative_mask := 0.U + int_wakeups(wu_idx).bits.rebusy := false.B + int_wakeups(wu_idx).bits.bypassable := false.B + wu_idx += 1 + + rob.io.wb_resps(wb_idx).valid := RegNext(ll_arb.io.out.valid && !IsKilledByBranch( + brupdate, + RegNext(rob.io.flush.valid), + ll_arb.io.out.bits + )) + rob.io.wb_resps(wb_idx).bits := RegNext(ll_arb.io.out.bits) + + iregfile.io.write_ports(wb_idx).valid := ll_arb.io.out.valid && ll_arb.io.out.bits.uop.dst_rtype === RT_FIX + iregfile.io.write_ports(wb_idx).bits.addr := ll_arb.io.out.bits.uop.pdst + iregfile.io.write_ports(wb_idx).bits.data := ll_arb.io.out.bits.data + wb_idx += 1 + + // loop through each issue-port (exe_units are statically connected to an issue-port) + for (i <- 0 until aluWidth) { + val unit = alu_exe_units(i) + val fast_wakeup = unit.io_fast_wakeup + + int_bypasses(bypass_idx).valid := unit.io_alu_resp.valid && unit.io_alu_resp.bits.uop.dst_rtype === RT_FIX + int_bypasses(bypass_idx).bits := unit.io_alu_resp.bits + bypass_idx += 1 + + int_wakeups(wu_idx) := fast_wakeup + wu_idx += 1 + + rob.io.wb_resps(wb_idx).valid := RegNext(unit.io_alu_resp.valid && !IsKilledByBranch( + brupdate, + RegNext(rob.io.flush.valid), + unit.io_alu_resp.bits + )) + rob.io.wb_resps(wb_idx).bits := RegNext(unit.io_alu_resp.bits) + + iregfile.io.write_ports(wb_idx).valid := unit.io_alu_resp.valid && unit.io_alu_resp.bits.uop.dst_rtype === RT_FIX + iregfile.io.write_ports(wb_idx).bits.addr := unit.io_alu_resp.bits.uop.pdst + iregfile.io.write_ports(wb_idx).bits.data := unit.io_alu_resp.bits.data + wb_idx += 1 + + pred_wakeups(i) := unit.io_fast_pred_wakeup + + } + require(wu_idx == numIntWakeups) + require(wb_idx == numIrfWritePorts) + + // arb stage guarantees 1 preg writer per cycle + val pregfile_write_valids = alu_exe_units.map(u => u.io_alu_resp.valid && u.io_alu_resp.bits.uop.is_sfb_br) + assert(PopCount(pregfile_write_valids) <= 1.U) + pregfile.io.write_ports(0).valid := pregfile_write_valids.reduce(_ || _) + pregfile.io.write_ports(0).bits.addr := Mux1H(pregfile_write_valids, alu_exe_units.map(_.io_alu_resp.bits.uop.pdst)) + pregfile.io.write_ports(0).bits.data := Mux1H(pregfile_write_valids, alu_exe_units.map(_.io_alu_resp.bits.data)) + + // Connect the predicate wakeup port + alu_iss_unit.io.pred_wakeup_port.valid := pred_wakeup.valid + alu_iss_unit.io.pred_wakeup_port.bits := pred_wakeup.bits.uop.pdst + mem_iss_unit.io.pred_wakeup_port.valid := false.B + mem_iss_unit.io.pred_wakeup_port.bits := DontCare + unq_iss_unit.io.pred_wakeup_port.valid := false.B + unq_iss_unit.io.pred_wakeup_port.bits := DontCare + + // ---------------------------------------------------------------- + // Connect the wakeup ports to the busy tables in the rename stages + + for ((renport, intport) <- rename_stage.io.wakeups zip int_wakeups) { + renport <> intport + } + if (usingFPU) { + fp_rename_stage.io.wakeups := fp_pipeline.io.wakeups + } + + pred_rename_stage.io.wakeups(0) := pred_wakeup + imm_rename_stage.io.wakeups := all_exe_units.map(_.io_rrd_immrf_wakeup) + + rename_stage.io.child_rebusys := alu_exe_units.map(_.io_child_rebusy).reduce(_ | _) + imm_rename_stage.io.child_rebusys := 0.U + pred_rename_stage.io.child_rebusys := 0.U + fp_rename_stage.io.child_rebusys := 0.U + + mem_iss_unit.io.fu_types := mem_exe_units.map(_.io_ready_fu_types) + alu_iss_unit.io.fu_types := alu_exe_units.map(_.io_ready_fu_types) + unq_iss_unit.io.fu_types := unq_exe_units.map(_.io_ready_fu_types) + + for (iss_unit <- Seq(mem_iss_unit, alu_iss_unit, unq_iss_unit)) { + iss_unit.io.tsc_reg := debug_tsc_reg + iss_unit.io.brupdate := brupdate + iss_unit.io.flush_pipeline := RegNext(rob.io.flush.valid) + + // Rebusy children of misspeculated wakeups + iss_unit.io.child_rebusys := alu_exe_units.map(_.io_child_rebusy).reduce(_ | _) + + iss_unit.io.wakeup_ports := int_wakeups + + iss_unit.io.rob_pnr_idx := rob.io.rob_pnr_idx + iss_unit.io.rob_head := rob.io.rob_head_idx + } + + mem_iss_unit.io.squash_grant := ( + mem_exe_units.map(_.io_squash_iss).reduce(_ || _) || + alu_exe_units.map(_.io_squash_iss).reduce(_ || _) || + io.lsu.iwakeups.map(_.bits.rebusy).reduce(_ || _) + ) + unq_iss_unit.io.squash_grant := ( + unq_exe_units.map(_.io_squash_iss).reduce(_ || _) || + alu_exe_units.map(_.io_squash_iss).reduce(_ || _) || + io.lsu.iwakeups.map(_.bits.rebusy).reduce(_ || _) + ) + alu_iss_unit.io.squash_grant := ( + alu_exe_units.map(_.io_squash_iss).reduce(_ || _) || + io.lsu.iwakeups.map(_.bits.rebusy).reduce(_ || _) + ) + + mem_iss_unit.io.iss_uops zip mem_exe_units map { case (i, u) => u.io_iss_uop := i } + alu_iss_unit.io.iss_uops zip alu_exe_units map { case (i, u) => u.io_iss_uop := i } + unq_iss_unit.io.iss_uops zip unq_exe_units map { case (i, u) => u.io_iss_uop := i } + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Register Read Arbitrate Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + arb_idx = 0 + for ((unit, w) <- all_exe_units.zipWithIndex) { + for (i <- 0 until unit.nReaders) { + iregfile.io.arb_read_reqs(arb_idx) <> unit.io_arb_irf_reqs(i) + arb_idx += 1 + } + immregfile.io.arb_read_reqs(w) <> unit.io_arb_immrf_req + unit.io_arb_rebusys := io.lsu.iwakeups + } + require(arb_idx == numIrfLogicalReadPorts) + for ((unit, w) <- alu_exe_units.zipWithIndex) { + pregfile.io.arb_read_reqs(w) <> unit.io_arb_prf_req + bregfile.io.arb_read_reqs(w) <> unit.io_arb_brf_req + } + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Register Read Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + // Register Read <- Issue (rrd <- iss) + var rd_idx = 0 + for ((unit, w) <- all_exe_units.zipWithIndex) { + for (i <- 0 until unit.nReaders) { + unit.io_rrd_irf_resps(i) := iregfile.io.rrd_read_resps(rd_idx) + rd_idx += 1 + } + unit.io_rrd_immrf_resp := immregfile.io.rrd_read_resps(w) + unit.io_rrd_irf_bypasses := int_bypasses + } + require(rd_idx == numIrfLogicalReadPorts) + for ((unit, w) <- alu_exe_units.zipWithIndex) { + unit.io_rrd_prf_resp := pregfile.io.rrd_read_resps(w) + unit.io_rrd_brf_resp := bregfile.io.rrd_read_resps(w) + } + + //------------------------------------------------------------- + // Privileged Co-processor 0 Register File + // Note: Normally this would be bad in that I'm writing state before + // committing, so to get this to work I stall the entire pipeline for + // CSR instructions so I never speculate these instructions. + + io.lsu.sfence := unq_exe_unit.io_sfence.get + io.ifu.sfence := unq_exe_unit.io_sfence.get + + // for critical path reasons, we aren't zero'ing this out if resp is not valid + csr.io.rw.addr := csr_resp.bits.addr + csr.io.rw.cmd := CSR.maskCmd(csr_resp.valid, csr_resp.bits.uop.csr_cmd) + csr.io.rw.wdata := csr_resp.bits.data + + rob.io.csr_replay.valid := csr_resp.valid && csr.io.rw_stall + rob.io.csr_replay.bits.uop := csr_resp.bits.uop + rob.io.csr_replay.bits.cause := MINI_EXCEPTION_CSR_REPLAY + rob.io.csr_replay.bits.badvaddr := DontCare + + // Extra I/O + // Delay retire/exception 1 cycle + csr.io.retire := RegNext(PopCount(rob.io.commit.arch_valids.asUInt)) + csr.io.exception := RegNext(rob.io.com_xcpt.valid) + // csr.io.pc used for setting EPC during exception or CSR.io.trace. + + csr.io.pc := (boom.v4.util.AlignPCToBoundary(io.ifu.com_pc, icBlockBytes) + + RegNext(rob.io.com_xcpt.bits.pc_lob) + - Mux(RegNext(rob.io.com_xcpt.bits.edge_inst), 2.U, 0.U)) + // Cause not valid for for CALL or BREAKPOINTs (CSRFile will override it). + csr.io.cause := RegNext(rob.io.com_xcpt.bits.cause) + csr.io.ungated_clock := clock + + val tval_valid = csr.io.exception && + csr.io.cause.isOneOf( + //Causes.illegal_instruction.U, we currently only write 0x0 for illegal instructions + Causes.breakpoint.U, + Causes.misaligned_load.U, + Causes.misaligned_store.U, + Causes.load_access.U, + Causes.store_access.U, + Causes.fetch_access.U, + Causes.load_page_fault.U, + Causes.store_page_fault.U, + Causes.fetch_page_fault.U + ) + + csr.io.tval := Mux( + tval_valid, + RegNext(encodeVirtualAddress(rob.io.com_xcpt.bits.badvaddr, rob.io.com_xcpt.bits.badvaddr)), + 0.U + ) + + // TODO move this function to some central location (since this is used elsewhere). + def encodeVirtualAddress(a0: UInt, ea: UInt) = + if (vaddrBitsExtended == vaddrBits) { + ea + } else { + // Efficient means to compress 64-bit VA into vaddrBits+1 bits. + // (VA is bad if VA(vaddrBits) != VA(vaddrBits-1)). + val a = a0.asSInt >> vaddrBits + val msb = Mux(a === 0.S || a === -1.S, ea(vaddrBits), !ea(vaddrBits - 1)) + Cat(msb, ea(vaddrBits - 1, 0)) + } + + // reading requires serializing the entire pipeline + csr.io.fcsr_flags.valid := rob.io.commit.fflags.valid + csr.io.fcsr_flags.bits := rob.io.commit.fflags.bits + csr.io.set_fs_dirty.get := rob.io.commit.fflags.valid + + all_exe_units.map(i => i.io_fcsr_rm := csr.io.fcsr_rm) + io.fcsr_rm := csr.io.fcsr_rm + + fp_pipeline.io.fcsr_rm := csr.io.fcsr_rm + + csr.io.hartid := io.hartid + csr.io.interrupts := io.interrupts + + // we do not support the H-extension + csr.io.htval := DontCare + csr.io.gva := DontCare + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Execute Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Load/Store Unit **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + // enqueue basic load/store info in Decode + for (w <- 0 until coreWidth) { + io.lsu.dis_uops(w).valid := dis_fire(w) + io.lsu.dis_uops(w).bits := dis_uops(w) + } + + // tell LSU about committing loads and stores to clear entries + io.lsu.commit := rob.io.commit + + // tell LSU that it should fire a load that waits for the rob to clear + io.lsu.commit_load_at_rob_head := rob.io.com_load_is_at_rob_head + + //com_xcpt.valid comes too early, will fight against a branch that resolves same cycle as an exception + io.lsu.exception := RegNext(rob.io.flush.valid) + + // Handle Branch Mispeculations + io.lsu.brupdate := brupdate + io.lsu.rob_head_idx := rob.io.rob_head_idx + io.lsu.rob_pnr_idx := rob.io.rob_pnr_idx + + io.lsu.tsc_reg := debug_tsc_reg + + // Connect IFPU + fp_pipeline.io.from_int <> unq_exe_unit.io_ifpu_resp.get + + // Connect FLDs + fp_pipeline.io.ll_wports <> io.lsu.fresp + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Commit Stage **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + var cnt = numIrfWritePorts + for (wb <- fp_pipeline.io.wb) { + rob.io.wb_resps(cnt) := wb + rob.io.wb_resps(cnt).bits.data := ieee(wb.bits.data) + cnt += 1 + } + + require(cnt == rob.numWakeupPorts) + + // branch resolution + rob.io.brupdate <> brupdate + + io.lsu.status := csr.io.status + io.lsu.bp := csr.io.bp + io.lsu.mcontext := csr.io.mcontext + io.lsu.scontext := csr.io.scontext + + all_exe_units.map(u => u.io_status := csr.io.status) + fp_pipeline.io.status := csr.io.status + + // LSU <> ROB + rob.io.lsu_clr_bsy := io.lsu.clr_bsy + rob.io.lsu_clr_unsafe := io.lsu.clr_unsafe + rob.io.lxcpt <> io.lsu.lxcpt + + assert(!(csr.io.singleStep), "[core] single-step is unsupported.") + + //------------------------------------------------------------- + // **** Flush Pipeline **** + //------------------------------------------------------------- + // flush on exceptions, miniexeptions, and after some special instructions + + fp_pipeline.io.flush_pipeline := RegNext(rob.io.flush.valid) + + for (eu <- all_exe_units) + eu.io_kill := RegNext(rob.io.flush.valid) + + assert( + !(rob.io.com_xcpt.valid && !rob.io.flush.valid), + "[core] exception occurred, but pipeline flush signal not set!" + ) + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Outputs to the External World **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + // detect pipeline freezes and throw error + val idle_cycles = freechips.rocketchip.util.WideCounter(32) + when(rob.io.commit.valids.asUInt.orR || + csr.io.csr_stall || + io.rocc.busy || + reset.asBool) { + idle_cycles := 0.U + } + assert(!(idle_cycles.value(PlusArg("boom_timeout", 13, width = 5))), "Pipeline has hung.") + + fp_pipeline.io.debug_tsc_reg := debug_tsc_reg + + //------------------------------------------------------------- + //------------------------------------------------------------- + // **** Handle Cycle-by-Cycle Printouts **** + //------------------------------------------------------------- + //------------------------------------------------------------- + + if (COMMIT_LOG_PRINTF) { + var new_commit_cnt = 0.U + + for (w <- 0 until coreWidth) { + val priv = ShiftRegister(csr.io.status.prv, 2) // erets change the privilege. Get the old one + + // To allow for diffs against spike :/ + def printf_inst(uop: MicroOp) = { + when(uop.is_rvc) { + printf("(0x%x)", uop.debug_inst(15, 0)) + }.otherwise { + printf("(0x%x)", uop.debug_inst) + } + } + + when(rob.io.commit.arch_valids(w)) { + printf("%d 0x%x ", priv, Sext(rob.io.commit.uops(w).debug_pc(vaddrBits - 1, 0), xLen)) + printf_inst(rob.io.commit.uops(w)) + when(rob.io.commit.uops(w).dst_rtype === RT_FIX && rob.io.commit.uops(w).ldst =/= 0.U) { + printf(" x%d 0x%x\n", rob.io.commit.uops(w).ldst, rob.io.commit.debug_wdata(w)) + }.elsewhen(rob.io.commit.uops(w).dst_rtype === RT_FLT) { + printf(" f%d 0x%x\n", rob.io.commit.uops(w).ldst, rob.io.commit.debug_wdata(w)) + }.otherwise { + printf("\n") + } + } + } + } else if (BRANCH_PRINTF) { + val debug_ghist = RegInit(0.U(globalHistoryLength.W)) + when(rob.io.flush.valid && FlushTypes.useCsrEvec(rob.io.flush.bits.flush_typ)) { + debug_ghist := 0.U + } + + var new_ghist = debug_ghist + + for (w <- 0 until coreWidth) { + when(rob.io.commit.arch_valids(w) && + (rob.io.commit.uops(w).is_br || rob.io.commit.uops(w).is_jal || rob.io.commit.uops(w).is_jalr)) { + // for (i <- 0 until globalHistoryLength) { + // printf("%x", new_ghist(globalHistoryLength-i-1)) + // } + // printf("\n") + printf( + "%x %x %x %x %x %x\n", + rob.io.commit.uops(w).debug_fsrc, + rob.io.commit.uops(w).taken, + rob.io.commit.uops(w).is_br, + rob.io.commit.uops(w).is_jal, + rob.io.commit.uops(w).is_jalr, + Sext(rob.io.commit.uops(w).debug_pc(vaddrBits - 1, 0), xLen) + ) + + } + new_ghist = Mux( + rob.io.commit.arch_valids(w) && rob.io.commit.uops(w).is_br, + Mux(rob.io.commit.uops(w).taken, new_ghist << 1 | 1.U(1.W), new_ghist << 1), + new_ghist + ) + } + debug_ghist := new_ghist + } + + // TODO: Does anyone want this debugging functionality? + val coreMonitorBundle = Wire(new CoreMonitorBundle(xLen, fLen)) + coreMonitorBundle := DontCare + coreMonitorBundle.clock := clock + coreMonitorBundle.reset := reset + + //------------------------------------------------------------- + //------------------------------------------------------------- + // Page Table Walker + + io.ptw.ptbr := csr.io.ptbr + io.ptw.status := csr.io.status + io.ptw.pmp := csr.io.pmp + io.ptw.sfence := io.ifu.sfence + + //------------------------------------------------------------- + //------------------------------------------------------------- + + io.rocc := DontCare + io.rocc.exception := csr.io.exception && csr.io.status.xs.orR + io.rocc.csrs <> csr.io.roccCSRs + if (usingRoCC) { + val rocc_unit = unq_exe_unit + rocc_unit.io_rocc_core.get.rocc <> io.rocc + rocc_unit.io_rocc_core.get.dis_uops := dis_uops + rocc_unit.io_rocc_core.get.rob_head_idx := rob.io.rob_head_idx + rocc_unit.io_rocc_core.get.rob_pnr_idx := rob.io.rob_pnr_idx + + for (w <- 0 until coreWidth) { + rocc_unit.io_rocc_core.get.dis_rocc_vals(w) := ( + dis_fire(w) && + dis_uops(w).is_rocc && + !dis_uops(w).exception + ) + } + } + + io.trace := DontCare + io.trace.time := csr.io.time + io.trace.insns map (t => t.valid := false.B) + io.trace.custom.get.asInstanceOf[BoomTraceBundle].rob_empty := rob.io.empty + + if (trace) { + for (w <- 0 until coreWidth) { + // Delay the trace so we have a cycle to pull PCs out of the FTQ + io.trace.insns(w).valid := RegNext(rob.io.commit.arch_valids(w)) + + // Recalculate the PC + io.ifu.debug_ftq_idx(w) := rob.io.commit.uops(w).ftq_idx + val iaddr = (AlignPCToBoundary(io.ifu.debug_fetch_pc(w), icBlockBytes) + + RegNext(rob.io.commit.uops(w).pc_lob) + - Mux(RegNext(rob.io.commit.uops(w).edge_inst), 2.U, 0.U))(vaddrBits - 1, 0) + io.trace.insns(w).iaddr := Sext(iaddr, xLen) + + def getInst(uop: MicroOp, inst: UInt): UInt = + Mux(uop.is_rvc, Cat(0.U(16.W), inst(15, 0)), inst) + + def getWdata(uop: MicroOp, wdata: UInt): UInt = + Mux((uop.dst_rtype === RT_FIX && uop.ldst =/= 0.U) || (uop.dst_rtype === RT_FLT), wdata, 0.U(xLen.W)) + + // use debug_insts instead of uop.debug_inst to use the rob's debug_inst_mem + // note: rob.debug_insts comes 1 cycle later + io.trace.insns(w).insn := getInst(RegNext(rob.io.commit.uops(w)), rob.io.commit.debug_insts(w)) + io.trace.insns(w).wdata.map(_ := RegNext(getWdata(rob.io.commit.uops(w), rob.io.commit.debug_wdata(w)))) + + // Comment out this assert because it blows up FPGA synth-asserts + // This tests correctedness of the debug_inst mem + // when (RegNext(rob.io.commit.valids(w))) { + // assert(rob.io.commit.debug_insts(w) === RegNext(rob.io.commit.uops(w).debug_inst)) + // } + // This tests correctedness of recovering pcs through ftq debug ports + // when (RegNext(rob.io.commit.valids(w))) { + // assert(Sext(io.trace.insns(w).iaddr, xLen) === + // RegNext(Sext(rob.io.commit.uops(w).debug_pc(vaddrBits-1,0), xLen))) + // } + + // These csr signals do not exactly match up with the ROB commit signals. + io.trace.insns(w).priv := RegNext(Cat(RegNext(RegNext(csr.io.status.debug)), csr.io.status.prv)) + // Can determine if it is an interrupt or not based on the MSB of the cause + io.trace.insns(w).exception := RegNext( + rob.io.com_xcpt.valid && !rob.io.com_xcpt.bits.cause(xLen - 1) + ) && (w == 0).B + io.trace.insns(w).interrupt := RegNext( + rob.io.com_xcpt.valid && rob.io.com_xcpt.bits.cause(xLen - 1) + ) && (w == 0).B + io.trace.insns(w).cause := RegNext(rob.io.com_xcpt.bits.cause) + io.trace.insns(w).tval := RegNext(csr.io.tval) + } + dontTouch(io.trace) + } else { + io.ifu.debug_ftq_idx := DontCare + } + + if (boomParams.enableTraceCoreIngress) { + for (w <- 0 until coreWidth) { + val trace_ingress = Module(new TraceCoreIngress(traceIngressParams)) + trace_ingress.io.in.valid := RegNext(rob.io.commit.arch_valids(w)) + // this is predicted to be taken, but since it is commited, it must have been predicted correctly + trace_ingress.io.in.taken := RegNext(rob.io.commit.uops(w).taken) + trace_ingress.io.in.is_branch := RegNext(rob.io.commit.uops(w).is_br) + trace_ingress.io.in.is_jal := RegNext(rob.io.commit.uops(w).is_jal) + trace_ingress.io.in.is_jalr := RegNext(rob.io.commit.uops(w).is_jalr) + trace_ingress.io.in.insn := RegNext(rob.io.commit.uops(w).debug_inst) + trace_ingress.io.in.pc := RegNext(rob.io.commit.uops(w).debug_pc) + trace_ingress.io.in.is_compressed := RegNext(rob.io.commit.uops(w).is_rvc) + trace_ingress.io.in.interrupt := RegNext( + rob.io.com_xcpt.valid && rob.io.com_xcpt.bits.cause(xLen - 1) + ) && (w == 0).B + trace_ingress.io.in.exception := RegNext( + rob.io.com_xcpt.valid && !rob.io.com_xcpt.bits.cause(xLen - 1) + ) && (w == 0).B + trace_ingress.io.in.trap_return := RegNext(rob.io.commit.uops(w).is_eret) + io.trace_core_ingress.get.group(w) <> trace_ingress.io.out + } + io.trace_core_ingress.get.ctx := RegNext(csr.io.ptbr.asid) + io.trace_core_ingress.get.tval := RegNext(csr.io.tval) + io.trace_core_ingress.get.cause := RegNext(csr.io.cause) + io.trace_core_ingress.get.time := RegNext(csr.io.time) + io.trace_core_ingress.get.priv := RegNext(csr.io.status.prv) + } +} diff --git a/examples/prefab/cores/boom/arch/native/Decode.scala b/examples/prefab/cores/boom/arch/native/Decode.scala new file mode 100644 index 000000000..8be025b9b --- /dev/null +++ b/examples/prefab/cores/boom/arch/native/Decode.scala @@ -0,0 +1,216 @@ +// Copied from riscv-boom src/main/scala/v4/exu/decode.scala and modified for Buckyball. +// Copyright (c) The Regents of the University of California (Regents). +// All Rights Reserved. See LICENSE and LICENSE.SiFive in riscv-boom for license details. + +package examples.core.boom.native + +import chisel3._ +import chisel3.util._ + +import freechips.rocketchip.tile.FPConstants +import org.chipsalliance.cde.config.Parameters +import freechips.rocketchip.rocket.Instructions._ +import freechips.rocketchip.rocket.Instructions32 +import freechips.rocketchip.rocket.CustomInstructions._ +import freechips.rocketchip.rocket.RVCExpander +import freechips.rocketchip.rocket.ALU._ +import freechips.rocketchip.rocket.{CSR, Causes, DecodeLogic} +import freechips.rocketchip.util._ + +import boom.v4.common._ +import boom.v4.util._ +import boom.v4.exu._ + +/** + * Decode unit that takes in a single instruction and generates a MicroOp. + */ +class DecodeUnit(implicit p: Parameters) + extends BoomModule + with freechips.rocketchip.rocket.constants.MemoryOpConstants + with freechips.rocketchip.rocket.constants.ScalarOpConstants { + val io = IO(new DecodeUnitIo) + + val uop = Wire(new MicroOp()) + uop := io.enq.uop + + val decode_table = + DecodeTables.X_table ++ + DecodeTables.F_table ++ + DecodeTables.FDivSqrt_table ++ + DecodeTables.X64_table ++ + DecodeTables.B_table ++ + (if (usingRoCC) DecodeTables.RoCC_table else Nil) + + val inst = uop.inst + val LDST = inst(RD_MSB, RD_LSB) + val LRS1 = inst(RS1_MSB, RS1_LSB) + val LRS2 = inst(RS2_MSB, RS2_LSB) + val LRS3 = inst(RS3_MSB, RS3_LSB) + + val cs = Wire(new CtrlSigs()).decode(inst, decode_table) + + // Exception Handling + io.csr_decode.inst := inst + val csr_en = cs.csr_cmd.isOneOf(CSR.S, CSR.C, CSR.W) + val csr_ren = cs.csr_cmd.isOneOf(CSR.S, CSR.C) && uop.lrs1 === 0.U + val system_insn = cs.csr_cmd === CSR.I + val sfence = inst === SFENCE_VMA + + val cs_legal = cs.legal +// dontTouch(cs_legal) + + require(fLen >= 64) + val illegal_rm = inst(14, 12).isOneOf(5.U, 6.U) || (inst(14, 12) === 7.U && io.fcsr_rm >= 5.U) + + val id_illegal_insn = !cs_legal || + (cs.fp_val && (io.csr_decode.fp_illegal || illegal_rm)) || + (uop.is_rocc && io.csr_decode.rocc_illegal) || + (cs.is_amo && !io.status.isa('a' - 'a')) || + (csr_en && (io.csr_decode.read_illegal || !csr_ren && io.csr_decode.write_illegal)) || + ((sfence || system_insn) && io.csr_decode.system_illegal) + +// cs.div && !csr.io.status.isa('m'-'a') || TODO check for illegal div instructions + + def checkExceptions(x: Seq[(Bool, UInt)]) = + (x.map(_._1).reduce(_ || _), PriorityMux(x)) + + val (xcpt_valid, xcpt_cause) = checkExceptions(List( + (io.interrupt && !io.enq.uop.is_sfb, io.interrupt_cause), // Disallow interrupts while we are handling a SFB + (uop.bp_debug_if, (CSR.debugTriggerCause).U), + (uop.bp_xcpt_if, (Causes.breakpoint).U), + (uop.xcpt_pf_if, (Causes.fetch_page_fault).U), + (uop.xcpt_ae_if, (Causes.fetch_access).U), + (id_illegal_insn, (Causes.illegal_instruction).U) + )) + + uop.exception := xcpt_valid + uop.exc_cause := xcpt_cause + + //------------------------------------------------------------- + + uop.is_mov := inst === ADD && LRS1 === 0.U + + uop.iq_type(IQ_UNQ) := Seq(FC_MUL, FC_DIV, FC_CSR, FC_I2F).map(c => cs.fu_code(c)).reduce(_ || _) + uop.iq_type(IQ_ALU) := Seq(FC_ALU).map(c => cs.fu_code(c)).reduce(_ || _) + uop.iq_type(IQ_MEM) := Seq(FC_AGEN, FC_DGEN).map(c => cs.fu_code(c)).reduce(_ || _) + uop.iq_type(IQ_FP) := Seq(FC_FPU, FC_FDV, FC_F2I).map(c => cs.fu_code(c)).reduce(_ || _) + + uop.fu_code := cs.fu_code.asBools + + uop.ldst := LDST + uop.lrs1 := LRS1 + uop.lrs2 := LRS2 + uop.lrs3 := LRS3 + + uop.dst_rtype := cs.dst_type + uop.lrs1_rtype := Mux(cs.rs1_type === RT_FIX && LRS1 === 0.U, RT_ZERO, cs.rs1_type) + uop.lrs2_rtype := Mux(cs.rs2_type === RT_FIX && LRS2 === 0.U, RT_ZERO, cs.rs2_type) + uop.frs3_en := cs.frs3_en + + uop.ldst_is_rs1 := uop.is_sfb_shadow + // SFB optimization + when(uop.is_sfb_shadow && cs.rs2_type === RT_X) { + uop.lrs2_rtype := Mux(LDST === 0.U, RT_ZERO, RT_FIX) + uop.lrs2 := LDST + uop.ldst_is_rs1 := false.B + }.elsewhen(uop.is_sfb_shadow && uop.is_mov) { + uop.lrs1 := LDST + uop.lrs1_rtype := Mux(LDST === 0.U, RT_ZERO, RT_FIX) + uop.ldst_is_rs1 := true.B + } + + uop.fp_val := cs.fp_val + uop.fp_ctrl := cs.fp + + uop.mem_cmd := cs.mem_cmd + uop.mem_size := Mux(cs.mem_cmd.isOneOf(M_SFENCE, M_FLUSH_ALL), Cat(LRS2 =/= 0.U, LRS1 =/= 0.U), inst(13, 12)) + uop.mem_signed := !inst(14) + uop.uses_ldq := cs.uses_ldq + uop.uses_stq := cs.uses_stq + uop.is_amo := cs.is_amo + uop.is_fence := inst === FENCE + uop.is_fencei := inst === FENCE_I + uop.is_sfence := inst === SFENCE_VMA + uop.is_sys_pc2epc := inst === EBREAK || inst === ECALL + uop.is_eret := inst === ECALL || inst === EBREAK || inst === SRET || inst === MRET || inst === DRET + uop.is_unique := cs.inst_unique || uop.is_rocc + uop.is_rocc := inst(6, 0).isOneOf("b0001011".U, "b0101011".U, "b1111011".U) && inst(14, 12).isOneOf( + 0.U, + 2.U, + 3.U, + 4.U, + 6.U, + 7.U + ) + uop.flush_on_commit := cs.flush_on_commit || (csr_en && !csr_ren && io.csr_decode.write_flush) + + //------------------------------------------------------------- + // immediates + + // repackage the immediate, and then pass the fewest number of bits around + val di24_20 = Mux(cs.imm_sel === IS_B || cs.imm_sel === IS_S, inst(11, 7), inst(24, 20)) + val imm_packed = Cat(inst(31, 25), di24_20, inst(19, 12)) + val imm = ImmGen(imm_packed, cs.imm_sel) + val imm_hi = imm >> (immPregSz - 1) + val imm_lo = imm(immPregSz - 1, 0) + val short_imm = imm_hi === 0.U || ~imm_hi === 0.U || cs.imm_sel === IS_F3 + + uop.imm_rename := cs.imm_sel =/= IS_N && cs.imm_sel =/= IS_F3 + uop.imm_packed := imm_packed + uop.imm_sel := cs.imm_sel + when(short_imm) { + uop.imm_rename := false.B + uop.imm_sel := IS_SH + uop.pimm := Mux(cs.imm_sel === IS_F3, inst(14, 12), imm_lo) + } + + uop.fp_rm := Mux(inst(14, 12) === 7.U, io.fcsr_rm, inst(14, 12)) + uop.fp_typ := inst(21, 20) + + //------------------------------------------------------------- + + uop.csr_cmd := cs.csr_cmd + when((cs.csr_cmd === CSR.S || cs.csr_cmd === CSR.C) && LRS1 === 0.U) { + uop.csr_cmd := CSR.R + } + + uop.fcn_dw := cs.fcn_dw + uop.fcn_op := cs.fcn_op + + uop.op1_sel := OP1_RS1 + when(inst === LUI || inst === CSRRWI || inst === CSRRSI || inst === CSRRCI || + inst === WFI || inst === SRET || inst === MRET || inst === DRET) { + uop.op1_sel := OP1_ZERO + }.elsewhen(inst === JAL || inst === JALR || inst === AUIPC) { + uop.op1_sel := OP1_PC + }.elsewhen(Seq(SH1ADD, SH2ADD, SH3ADD, SH1ADD_UW, SH2ADD_UW, SH3ADD_UW, ADD_UW, SLLI_UW).map(_ === inst).orR) { + uop.op1_sel := OP1_RS1SHL + } + + uop.op2_sel := OP2_RS2 + when(cs.is_amo || inst === CSRRW || inst === CSRRS || inst === CSRRC) { + uop.op2_sel := OP2_ZERO + }.elsewhen(inst === CSRRWI || inst === CSRRSI || inst === CSRRCI || + inst === WFI || inst === SRET || inst === DRET || inst === MRET) { + uop.op2_sel := OP2_IMMC + }.elsewhen(inst === JAL || inst === JALR) { + uop.op2_sel := OP2_NEXT + }.elsewhen(Seq(BCLR, BCLRI, BINV, BINVI, BSET, BSETI).map(_ === inst).orR) { + uop.op2_sel := Mux(uop.lrs2_rtype === RT_FIX, OP2_RS2OH, OP2_IMMOH) + }.elsewhen(cs.imm_sel === IS_U || cs.imm_sel === IS_I || cs.imm_sel === IS_S) { + uop.op2_sel := OP2_IMM + } + + uop.br_type := Seq( + (BEQ, B_EQ), + (BNE, B_NE), + (BGE, B_GE), + (BGEU, B_GEU), + (BLT, B_LT), + (BLTU, B_LTU), + (JAL, B_J), + (JALR, B_JR) + ).map { case (c, b) => Mux(inst === c, b, 0.U) }.reduce(_ | _) + + io.deq.uop := uop +} diff --git a/examples/prefab/cores/boom/arch/native/ExecutionUnit.scala b/examples/prefab/cores/boom/arch/native/ExecutionUnit.scala new file mode 100644 index 000000000..0483df5a2 --- /dev/null +++ b/examples/prefab/cores/boom/arch/native/ExecutionUnit.scala @@ -0,0 +1,202 @@ +// Copied from riscv-boom src/main/scala/v4/exu/execution-units/execution-unit.scala and modified for Buckyball. +// Copyright (c) The Regents of the University of California (Regents). +// All Rights Reserved. See LICENSE and LICENSE.SiFive in riscv-boom for license details. + +package examples.core.boom.native + +//------------------------------------------------------------------------------ +// Execution Units +//------------------------------------------------------------------------------ +//------------------------------------------------------------------------------ +// +// The issue window schedules micro-ops onto a specific execution pipeline +// A given execution pipeline may contain multiple functional units; one or more +// read ports, and one or more writeports. + +import scala.collection.mutable.{ArrayBuffer} + +import chisel3._ +import chisel3.util._ + +import freechips.rocketchip.rocket.{BP, CSR, SFenceReq} +import freechips.rocketchip.rocket.constants.{MemoryOpConstants} +import org.chipsalliance.cde.config.{Parameters} +import freechips.rocketchip.tile +import freechips.rocketchip.util._ + +import boom.v4.common._ +import boom.v4.ifu.FTQInfo +import boom.v4.util._ + +import boom.v4.exu._ +import examples.core.boom.native.RoCCShim + +class UniqueExeUnit( + val hasCSR: Boolean = false, + val hasMul: Boolean = false, + val hasDiv: Boolean = false, + val hasIfpu: Boolean = false, + val hasRocc: Boolean = false +)( + implicit p: Parameters) + extends ExecutionUnit("Unq") + with HasIrfReadPorts + with HasImmrfReadPort + with MemoryOpConstants { + def nReaders = 2 + + val io_squash_iss = IO(Output(Bool())) + io_squash_iss := ((io_arb_irf_reqs(0).valid && !io_arb_irf_reqs(0).ready) || + (io_arb_irf_reqs(1).valid && !io_arb_irf_reqs(1).ready)) + + when(io_squash_iss || arb_rebusied) { + val will_replay = arb_uop.valid && !IsKilledByBranch(io_brupdate, io_kill, arb_uop.bits) && !arb_rebusied + arb_uop.valid := will_replay + arb_uop.bits := UpdateBrMask(io_brupdate, arb_uop.bits) + arb_uop.bits.iw_p1_bypass_hint := false.B + arb_uop.bits.iw_p2_bypass_hint := false.B + + rrd_uop.valid := false.B + } + + val exe_int_req = Wire(new FuncUnitReq(xLen)) + exe_int_req.uop := exe_uop.bits + exe_int_req.rs1_data := exe_rs1_data + exe_int_req.rs2_data := exe_rs2_data + exe_int_req.rs3_data := DontCare + exe_int_req.imm_data := exe_imm_data + exe_int_req.pred_data := DontCare + exe_int_req.ftq_info := DontCare + + val io_mul_resp = + if (hasMul) { + fu_types += ((FC_MUL, true.B, "IMul")) + val imul = Module(new PipelinedMulUnit(imulLatency, xLen)) + imul.io.req.valid := exe_uop.valid && exe_uop.bits.fu_code(FC_MUL) + imul.io.req.bits := exe_int_req + imul.io.brupdate := io_brupdate + imul.io.kill := io_kill + imul.io.resp.ready := true.B + + val resp = IO(Output(Valid(new ExeUnitResp(xLen)))) + resp := imul.io.resp + Some(resp) + } else { + assert(!(exe_uop.valid && exe_uop.bits.fu_code(FC_MUL))) + None + } + + val (io_csr_resp, io_sfence) = + if (hasCSR) { + fu_types += ((FC_CSR, true.B, "CSR")) + val alu = Module(new ALUUnit(dataWidth = xLen)) + alu.io.req.valid := exe_uop.valid && exe_uop.bits.fu_code(FC_CSR) + alu.io.req.bits := exe_int_req + alu.io.resp.ready := true.B + alu.io.brupdate := io_brupdate + alu.io.kill := io_kill + + val c = IO(Output(Valid(new CSRResp))) + c.valid := RegNext(alu.io.resp.valid && exe_uop.bits.csr_cmd =/= CSR.N) + c.bits.uop := RegNext(alu.io.resp.bits.uop) + c.bits.data := RegNext(alu.io.resp.bits.data) + c.bits.addr := RegNext(exe_imm_data) + + val s = IO(Valid(new SFenceReq)) + s.valid := RegNext(exe_uop.valid && exe_uop.bits.is_sfence) + s.bits.rs1 := RegNext(exe_uop.bits.pimm(0)) + s.bits.rs2 := RegNext(exe_uop.bits.pimm(1)) + s.bits.addr := RegNext(exe_rs1_data) + s.bits.asid := RegNext(exe_rs2_data) + s.bits.hv := RegNext(exe_uop.bits.mem_cmd === M_HFENCEV) + s.bits.hg := RegNext(exe_uop.bits.mem_cmd === M_HFENCEG) + + (Some(c), Some(s)) + } else { + assert(!(exe_uop.valid && exe_uop.bits.fu_code(FC_CSR))) + assert(!(exe_uop.valid && exe_uop.bits.mem_cmd === M_SFENCE)) + (None, None) + } + + val (io_rocc_resp, io_rocc_core) = + if (hasRocc) { + require(hasCSR) + val rocc_core = IO(new RoCCShimCoreIO) + val rocc_resp = IO(Decoupled(new ExeUnitResp(xLen))) + val rocc = Module(new RoCCShim) + rocc.io.req.valid := exe_uop.valid && exe_uop.bits.is_rocc + rocc.io.req.bits := exe_int_req + rocc.io.brupdate := io_brupdate // We should assert on this somewhere + rocc.io.status := io_status + rocc.io.exception := io_kill + rocc_core <> rocc.io.core + + rocc_resp <> rocc.io.resp + (Some(rocc_resp), Some(rocc_core)) + } else { + assert(!(exe_uop.valid && exe_uop.bits.is_rocc)) + (None, None) + } + + val (io_ifpu_resp) = + if (hasIfpu) { + val ifpu_ready = Wire(Bool()) + fu_types += ((FC_I2F, ifpu_ready, "IFPU")) + + val ifpu = Module(new IntToFPUnit(latency = intToFpLatency)) + ifpu.io.req.valid := exe_uop.valid && exe_uop.bits.fu_code(FC_I2F) + ifpu.io.req.bits := exe_int_req + ifpu.io.req.bits.uop.fp_rm := exe_uop.bits.prs2(4, 2) + ifpu.io.req.bits.uop.fp_typ := exe_uop.bits.prs2(1, 0) + ifpu.io.fcsr_rm := io_fcsr_rm + ifpu.io.brupdate := io_brupdate + ifpu.io.kill := io_kill + + // buffer up results since we share write-port on integer regfile. + val queue = + Module(new BranchKillableQueue( + new ExeUnitResp(xLen + 1), + entries = intToFpLatency + 6 + )) // TODO being overly conservative + queue.io.enq <> ifpu.io.resp + queue.io.brupdate := io_brupdate + queue.io.flush := io_kill + assert(!(queue.io.enq.valid && !queue.io.enq.ready)) + ifpu_ready := RegNext(queue.io.count < 2.U) + + val ifpu_resp = IO(Decoupled(new ExeUnitResp(xLen + 1))) + ifpu_resp <> queue.io.deq + (Some(ifpu_resp)) + } else { + assert(!(exe_uop.valid && exe_uop.bits.fu_code(FC_I2F))) + None + } + + val (io_div_resp) = + if (hasDiv) { + val div_ready = Wire(Bool()) + fu_types += ((FC_DIV, div_ready, "IDiv")) + + val div = Module(new DivUnit(xLen)) + assert(!(div.io.req.valid && !div.io.req.ready)) + div.io.req.valid := exe_uop.valid && exe_uop.bits.fu_code(FC_DIV) + div.io.req.bits := exe_int_req + div.io.brupdate := io_brupdate + div.io.kill := io_kill + + div_ready := (div.io.req.ready && + !(exe_uop.valid && exe_uop.bits.fu_code(FC_DIV)) && + !(rrd_uop.valid && rrd_uop.bits.fu_code(FC_DIV)) && + !(arb_uop.valid && arb_uop.bits.fu_code(FC_DIV))) + + val div_resp = IO(Decoupled(new ExeUnitResp(xLen))) + div_resp <> div.io.resp + Some(div_resp) + } else { + assert(!(exe_uop.valid && exe_uop.bits.fu_code(FC_DIV))) + None + } + + io_ready_fu_types := get_ready_fu_types() + +} diff --git a/examples/prefab/cores/boom/arch/native/Native.scala b/examples/prefab/cores/boom/arch/native/Native.scala new file mode 100644 index 000000000..b91ba97b3 --- /dev/null +++ b/examples/prefab/cores/boom/arch/native/Native.scala @@ -0,0 +1,179 @@ +package examples.core.boom.native + +import chisel3._ +import chisel3.util._ +import chisel3.experimental.annotate +import sifive.enterprise.firrtl.NestedPrefixModulesAnnotation +import org.chipsalliance.cde.config._ +import freechips.rocketchip.diplomacy._ +import freechips.rocketchip.resources.BindingScope +import freechips.rocketchip.devices.tilelink.{DevNullParams, TLError} +import freechips.rocketchip.amba.axi4._ +import freechips.rocketchip.tilelink._ +import freechips.rocketchip.tile._ +import freechips.rocketchip.subsystem._ +import freechips.rocketchip.prci._ +import freechips.rocketchip.interrupts._ +import boom.v4.common._ +import examples.core.boom.native.BoomTile + +class Context extends Bundle { + val satp = UInt(64.W) + val pmp = Vec(8, new Bundle { val cfg = UInt(8.W); val addr = UInt(42.W) }) +} + +class Commands extends Bundle { + + val request = Decoupled(new Bundle { + val instruction = UInt(32.W) + val rs1 = UInt(64.W) + val rs2 = UInt(64.W) + val privilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + }) + + val busy = Input(Bool()) + val fault = Input(Bool()) +} + +class Accelerator(implicit p: Parameters) extends LazyRoCC(OpcodeSet.custom3) { + val command = BundleBridgeSource(() => new Commands) + + override lazy val module = new LazyRoCCModuleImp(this) { + val c = command.bundle + c.request.valid := io.cmd.valid + c.request.bits.instruction := io.cmd.bits.inst.asUInt + c.request.bits.rs1 := io.cmd.bits.rs1 + c.request.bits.rs2 := io.cmd.bits.rs2 + c.request.bits.privilege := io.cmd.bits.status.dprv + c.request.bits.sum := io.cmd.bits.status.sum + c.request.bits.mxr := io.cmd.bits.status.mxr + io.cmd.ready := c.request.ready + when(io.cmd.fire) { assert(!io.cmd.bits.inst.xd); assert(!io.cmd.bits.status.dv) } + io.busy := c.busy + io.interrupt := c.fault + io.resp.valid := false.B + io.resp.bits := 0.U.asTypeOf(io.resp.bits) + io.mem.req.valid := false.B + io.mem.req.bits := 0.U.asTypeOf(io.mem.req.bits) + io.mem.s1_kill := true.B + io.mem.s2_kill := true.B + io.mem.s1_data := 0.U.asTypeOf(io.mem.s1_data) + io.mem.keep_clock_enabled := false.B + io.fpu_req.valid := false.B + io.fpu_req.bits := 0.U.asTypeOf(io.fpu_req.bits) + io.fpu_resp.ready := true.B + } + +} + +class Native(implicit p: Parameters) extends LazyModule with BindingScope { + val params = p(TilesLocated(InSubsystem)).head.asInstanceOf[BoomTileAttachParams] + val tile = LazyModule(new BoomTile(params.tileParams, params.crossingParams, params.lookup)) + val trace = tile.traceSourceNode.makeSink() + val hart = BundleBridgeSource(() => UInt(1.W)) + val resetVector = BundleBridgeSource(() => UInt(44.W)) + tile.hartIdNode := hart + tile.resetVectorNode := resetVector + val irqs = IntSourceNode(IntSourcePortSimple(5, ports = 1, sources = 1)) + tile.intInwardNode := irqs + + val notifications = Seq(tile.wfiNode, tile.haltNode, tile.ceaseNode).map { source => + val sink = IntSinkNode(IntSinkPortSimple(1, 1)); sink := source; sink + } + + val commands = BundleBridgeSink[Commands]() + commands := tile.roccs.head.asInstanceOf[Accelerator].command + val context = BundleBridgeSink[Context]() + context := tile.buckyballContext + + val dma = AXI4MasterNode(Seq(AXI4MasterPortParameters(Seq(AXI4MasterParameters( + name = "accelerator", + id = IdRange(0, 16), + aligned = true, + maxFlight = Some(4) + ))))) + + val bus = LazyModule(new TLXbar) + + val error = LazyModule(new TLError( + DevNullParams(Seq(AddressSet(0x3000, 0xfff)), maxAtomic = 8, maxTransfer = 64), + beatBytes = 16 + )) + + error.node := bus.node + bus.node := tile.masterNode + bus.node := TLFIFOFixer(TLFIFOFixer.all) := AXI4ToTL() := AXI4UserYanker(Some(4)) := AXI4Fragmenter() := dma + + val memory = AXI4SlaveNode(Seq(AXI4SlavePortParameters( + Seq(AXI4SlaveParameters( + address = AddressSet(0, (BigInt(1) << 44) - 1).subtract(AddressSet(0, 0x7fffffffL)), + regionType = RegionType.UNCACHED, + executable = true, + supportsRead = TransferSizes(1, 64), + supportsWrite = TransferSizes(1, 64), + interleavedId = Some(0) + )), + beatBytes = 16 + ))) + + memory := AXI4Buffer() := AXI4UserYanker() := AXI4IdIndexer(4) := AXI4Deinterleaver(64) := + TLToAXI4() := TLWidthWidget(16) := TLBroadcast(64, 4) := bus.node + + val peripherals = AXI4SlaveNode(Seq(AXI4SlavePortParameters( + Seq(AXI4SlaveParameters( + address = AddressSet(0, 0x7fffffffL).subtract(AddressSet(0x3000, 0xfff)), + regionType = RegionType.UNCACHED, + executable = true, + supportsRead = TransferSizes(1, 16), + supportsWrite = TransferSizes(1, 16), + interleavedId = Some(0) + )), + beatBytes = 16 + ))) + + peripherals := AXI4Buffer() := AXI4UserYanker() := AXI4IdIndexer(4) := AXI4Deinterleaver(64) := + TLToAXI4() := TLWidthWidget(16) := bus.node + + lazy val module = new LazyModuleImp(this) { + annotate(this)(Seq(NestedPrefixModulesAnnotation(this.toNamed, "Boom", true))) + + val io = IO(new Bundle { + val resetVector = Input(UInt(44.W)) + val interrupts = Input(Vec(4, Bool())) + val commands = new Commands + val context = Output(new Context) + }) + + val memoryAXI = memory.makeIOs() + val peripheralsAXI = peripherals.makeIOs() + val dmaAXI = dma.makeIOs() + hart.bundle := 0.U + resetVector.bundle := io.resetVector + irqs.out.head._1 := VecInit(false.B +: io.interrupts.toSeq) + io.commands <> commands.bundle + io.context := context.bundle + } + +} + +object Emit extends App { + + implicit val p: Parameters = + new Config((site, here, up) => { + case TilesLocated(InSubsystem) => up(TilesLocated(InSubsystem)).map { + case tile: BoomTileAttachParams => + tile.copy(tileParams = tile.tileParams.copy(core = tile.tileParams.core.copy(pgLevels = 4))) + } + case BuildRoCC => Seq((p: Parameters) => LazyModule(new Accelerator()(p))) + case MaxHartIdBits => 1 + }) ++ new WithNSmallBooms(1) ++ new BaseSubsystemConfig + + val native = LazyModule(new Native) + _root_.circt.stage.ChiselStage.emitSystemVerilogFile( + native.module, + args ++ Array("--split-verilog"), + firtoolOpts = Array("--disable-all-randomization") + ) +} diff --git a/examples/prefab/cores/boom/arch/native/NestedPrefixModulesAnnotation.scala b/examples/prefab/cores/boom/arch/native/NestedPrefixModulesAnnotation.scala new file mode 100644 index 000000000..9f4c4a379 --- /dev/null +++ b/examples/prefab/cores/boom/arch/native/NestedPrefixModulesAnnotation.scala @@ -0,0 +1,10 @@ +// Hacked from CIRCT source code. Look like some SiFive internal annotations. +package sifive.enterprise.firrtl + +case class NestedPrefixModulesAnnotation( + target: firrtl.annotations.ModuleTarget, + prefix: String, + inclusive: Boolean = false) + extends firrtl.annotations.SingleTargetAnnotation[firrtl.annotations.ModuleTarget] { + def duplicate(n: firrtl.annotations.ModuleTarget): NestedPrefixModulesAnnotation = this.copy(n) +} diff --git a/examples/prefab/cores/boom/arch/native/Rocc.scala b/examples/prefab/cores/boom/arch/native/Rocc.scala new file mode 100644 index 000000000..2f2cf8861 --- /dev/null +++ b/examples/prefab/cores/boom/arch/native/Rocc.scala @@ -0,0 +1,235 @@ +// Copied from riscv-boom src/main/scala/v4/exu/execution-units/rocc.scala and modified for Buckyball. +// Copyright (c) The Regents of the University of California (Regents). +// All Rights Reserved. See LICENSE and LICENSE.SiFive in riscv-boom for license details. + +package examples.core.boom.native + +//------------------------------------------------------------------------------ + +import chisel3._ +import chisel3.util._ + +import org.chipsalliance.cde.config.Parameters +import freechips.rocketchip.tile.{RoCCCoreIO, RoCCInstruction} +import freechips.rocketchip.rocket._ + +import boom.v4.common._ +import boom.v4.util._ +import boom.v4.exu._ + +/** + * Structure similar to LSU + * - Holds instruction and operand bits prior to issuing RoCC inst to + * accelerator + * - After issue, holds queue of translations between logical and physical + * specifiers to handle RoCC responses + */ +class RoCCShim(implicit p: Parameters) extends BoomModule { + val io = IO(new RoCCShimIO) + + io.req.ready := true.B + io.core.rocc.exception := false.B + io.core.rocc.mem.store_pending := false.B + io.core.rocc.mem.req.ready := false.B + io.core.rocc.mem.s2_nack := false.B + io.core.rocc.mem.s2_nack_cause_raw := false.B + io.core.rocc.mem.s2_uncached := false.B + io.core.rocc.mem.s2_paddr := DontCare + io.core.rocc.mem.resp.valid := false.B + io.core.rocc.mem.resp.bits := DontCare + io.core.rocc.mem.replay_next := false.B + io.core.rocc.mem.s2_xcpt.ma.ld := false.B + io.core.rocc.mem.s2_xcpt.ma.st := false.B + io.core.rocc.mem.s2_xcpt.pf.ld := false.B + io.core.rocc.mem.s2_xcpt.pf.st := false.B + io.core.rocc.mem.s2_xcpt.gf.ld := false.B + io.core.rocc.mem.s2_xcpt.gf.st := false.B + io.core.rocc.mem.s2_xcpt.ae.ld := false.B + io.core.rocc.mem.s2_xcpt.ae.st := false.B + io.core.rocc.mem.s2_gpa := DontCare + io.core.rocc.mem.s2_gpa_is_pte := false.B + io.core.rocc.mem.uncached_resp.map { r => + r.valid := false.B + r.bits := DontCare + } + io.core.rocc.mem.ordered := false.B + io.core.rocc.mem.perf.acquire := false.B + io.core.rocc.mem.perf.release := false.B + io.core.rocc.mem.perf.grant := false.B + io.core.rocc.mem.perf.tlbMiss := false.B + io.core.rocc.mem.perf.blocked := false.B + io.core.rocc.mem.perf.canAcceptStoreThenLoad := false.B + io.core.rocc.mem.perf.canAcceptStoreThenRMW := false.B + io.core.rocc.mem.perf.canAcceptLoadThenLoad := false.B + io.core.rocc.mem.perf.storeBufferEmptyAfterLoad := false.B + io.core.rocc.mem.perf.storeBufferEmptyAfterStore := false.B + io.core.rocc.mem.clock_enabled := false.B + + // RoCC execute queue. Wait for PNR, holds operands and inst bits + val rxq_val = Reg(Vec(numRxqEntries, Bool())) + val rxq_op_val = Reg(Vec(numRxqEntries, Bool())) + val rxq_committed = Reg(Vec(numRxqEntries, Bool())) + val rxq_uop = Reg(Vec(numRxqEntries, new MicroOp())) + val rxq_inst = Reg(Vec(numRxqEntries, UInt(32.W))) + val rxq_rs1 = Reg(Vec(numRxqEntries, UInt(xLen.W))) + val rxq_rs2 = Reg(Vec(numRxqEntries, UInt(xLen.W))) + + // RoCC commit queue. Wait for response, or immediate unbusy + val rcq = Reg(Vec(numRcqEntries, Valid(new RCQEntry))) + val rcq_head = RegInit(0.U(log2Ceil(numRcqEntries).W)) + val rcq_tail = RegInit(0.U(log2Ceil(numRcqEntries).W)) + val rcq_full = WrapInc(rcq_tail, numRcqEntries) === rcq_head + + // The instruction we are waiting for response from + val rxq_head = RegInit(0.U(log2Ceil(numRxqEntries).W)) + // The next instruction we are waiting to "commit" through PNR + val rxq_com_head = RegInit(0.U(log2Ceil(numRxqEntries).W)) + val rxq_tail = RegInit(0.U(log2Ceil(numRxqEntries).W)) + + // Decode + val rocc_idx = WireInit(0.U) + val br_mask = WireInit(0.U(maxBrCount.W)) + var enq_val = false.B + + assert(PopCount(io.core.dis_rocc_vals) <= 1.U) + for (w <- 0 until coreWidth) { + val enq_this = !enq_val && io.core.dis_rocc_vals(w) && io.core.dis_uops(w).is_rocc + when(enq_this) { + rocc_idx := w.U + } + + io.core.rxq_idx(w) := Mux(enq_val, WrapInc(rxq_tail, numRxqEntries), rxq_tail) + + enq_val = enq_val || enq_this + } + + when(enq_val) { + rxq_val(rxq_tail) := true.B + rxq_op_val(rxq_tail) := false.B + rxq_committed(rxq_tail) := false.B + rxq_uop(rxq_tail) := io.core.dis_uops(rocc_idx) + rxq_inst(rxq_tail) := io.core.dis_uops(rocc_idx).inst + rxq_tail := WrapInc(rxq_tail, numRxqEntries) + } + + // Wait for operands + when(io.req.valid && !IsKilledByBranch(io.brupdate, io.exception, io.req.bits.uop)) { + val rxq_idx = io.req.bits.uop.rxq_idx + assert( + io.req.bits.uop.rob_idx === rxq_uop(rxq_idx).rob_idx, + "Mismatch between RoCCUnit request and RoCC execute head" + ) + assert(rxq_val(rxq_idx), "Trying to execute rocc inst without the instruction bits") + + rxq_op_val(rxq_idx) := true.B + rxq_uop(rxq_idx).pdst := io.req.bits.uop.pdst + rxq_rs1(rxq_idx) := io.req.bits.rs1_data + rxq_rs2(rxq_idx) := io.req.bits.rs2_data + } + + // Wait for ROB to OK us to execute + when(rxq_val(rxq_com_head) && + IsOlder(rxq_uop(rxq_com_head).rob_idx, io.core.rob_pnr_idx, io.core.rob_head_idx)) { + rxq_committed(rxq_com_head) := true.B + rxq_com_head := WrapInc(rxq_com_head, numRxqEntries) + } + + // Execute + io.core.rocc.cmd.valid := false.B + io.core.rocc.cmd.bits.inst := rxq_inst(rxq_head).asTypeOf(new RoCCInstruction) + io.core.rocc.cmd.bits.rs1 := rxq_rs1(rxq_head) + io.core.rocc.cmd.bits.rs2 := rxq_rs2(rxq_head) + io.core.rocc.cmd.bits.status := io.status + + val resp_arb = Module(new Arbiter(new ExeUnitResp(xLen), 2)) + resp_arb.io.in(0).valid := false.B + resp_arb.io.in(0).bits.uop := rxq_uop(rxq_head) + resp_arb.io.in(0).bits.data := DontCare + resp_arb.io.in(0).bits.predicated := false.B + resp_arb.io.in(0).bits.fflags.valid := false.B + resp_arb.io.in(0).bits.fflags.bits := DontCare + when(rxq_op_val(rxq_head) && + rxq_val(rxq_head) && + rxq_committed(rxq_head) && + io.core.rocc.cmd.ready && + resp_arb.io.in(0).ready && + !rcq_full) { + io.core.rocc.cmd.valid := true.B + + when(rxq_uop(rxq_head).dst_rtype =/= RT_X) { + rcq_tail := WrapInc(rcq_tail, numRcqEntries) + rcq(rcq_tail).valid := true.B + rcq(rcq_tail).bits.uop := rxq_uop(rxq_head) + }.otherwise { + resp_arb.io.in(0).valid := true.B + } + + rxq_val(rxq_head) := false.B + rxq_head := WrapInc(rxq_head, numRxqEntries) + } + + io.core.rxq_full := WrapInc(rxq_tail, numRxqEntries) === rxq_head + io.core.rxq_empty := rxq_tail === rxq_head + + //-------------------------- + // Branches + for (i <- 0 until numRxqEntries) { + when(rxq_val(i)) { + rxq_uop(i).br_mask := GetNewBrMask(io.brupdate, rxq_uop(i)) + when(IsKilledByBranch(io.brupdate, io.exception, rxq_uop(i))) { + rxq_val(i) := false.B + rxq_op_val(i) := false.B + } + } + } + when(io.brupdate.b2.mispredict && !io.exception) { + rxq_tail := io.brupdate.b2.uop.rxq_idx + } + + //-------------------------- + // Exception / Reset + + when(reset.asBool) { + rxq_tail := 0.U + rxq_head := 0.U + rxq_com_head := 0.U + for (i <- 0 until numRxqEntries) { + rxq_val(i) := false.B + rxq_op_val(i) := false.B + rxq_committed(i) := false.B + } + }.elsewhen(io.exception) { + rxq_tail := rxq_com_head + for (i <- 0 until numRxqEntries) { + when(!rxq_committed(i)) { + rxq_val(i) := false.B + rxq_op_val(i) := false.B + } + } + } + + //------------------ + // Handle responses + + resp_arb.io.in(1).valid := io.core.rocc.resp.valid + resp_arb.io.in(1).bits.uop := rcq(rcq_head).bits.uop + resp_arb.io.in(1).bits.data := io.core.rocc.resp.bits.data + resp_arb.io.in(1).bits.predicated := io.core.rocc.resp.bits.data + resp_arb.io.in(1).bits.fflags.valid := false.B + resp_arb.io.in(1).bits.fflags.bits := DontCare + io.core.rocc.resp.ready := resp_arb.io.in(1).ready + io.resp <> resp_arb.io.out + + when(io.core.rocc.resp.fire) { + assert(rcq(rcq_head).valid && rcq(rcq_head).bits.uop.ldst === io.core.rocc.resp.bits.rd) + rcq(rcq_head).valid := false.B + rcq_head := WrapInc(rcq_head, numRcqEntries) + } + + when(reset.asBool) { + for (i <- 0 until numRcqEntries) { + rcq(i).valid := false.B + } + } + +} diff --git a/examples/prefab/cores/boom/arch/native/Tile.scala b/examples/prefab/cores/boom/arch/native/Tile.scala new file mode 100644 index 000000000..3ab55004e --- /dev/null +++ b/examples/prefab/cores/boom/arch/native/Tile.scala @@ -0,0 +1,295 @@ +// Copied from riscv-boom src/main/scala/v4/common/tile.scala and modified for Buckyball. +// Copyright (c) The Regents of the University of California (Regents). +// All Rights Reserved. See LICENSE and LICENSE.SiFive in riscv-boom for license details. + +package examples.core.boom.native + +import chisel3._ +import chisel3.util.{Queue, RRArbiter} + +import scala.collection.mutable.{ListBuffer} + +import org.chipsalliance.cde.config._ +import freechips.rocketchip.subsystem._ +import freechips.rocketchip.devices.tilelink._ +import freechips.rocketchip.diplomacy._ +import freechips.rocketchip.prci._ +import freechips.rocketchip.rocket._ +import freechips.rocketchip.subsystem.{RocketCrossingParams} +import freechips.rocketchip.tilelink._ +import freechips.rocketchip.interrupts._ +import freechips.rocketchip.util._ +import freechips.rocketchip.tile._ + +import boom.v4.exu._ +import boom.v4.ifu._ +import boom.v4.lsu._ +import boom.v4.util.{BoomCoreStringPrefix} +import freechips.rocketchip.prci.ClockSinkParameters +import freechips.rocketchip.trace.{TraceEncoderController, TraceEncoderParams, TraceSinkArbiter} +import boom.v4.common._ +import examples.core.boom.native.{BoomCore, Context} + +/** + * BOOM tile + */ +class BoomTile private ( + val boomParams: BoomTileParams, + crossing: ClockCrossingType, + lookup: LookupByHartIdImpl, + q: Parameters) + extends BaseTile(boomParams, crossing, lookup, q) + with SinksExternalInterrupts + with SourcesExternalNotifications { + + // Private constructor ensures altered LazyModule.p is used implicitly + def this( + params: BoomTileParams, + crossing: HierarchicalElementCrossingParamsLike, + lookup: LookupByHartIdImpl + )( + implicit p: Parameters + ) = + this(params, crossing.crossingType, lookup, p) + + val buckyballContext = BundleBridgeSource(() => new Context) + val intOutwardNode = None + val masterNode = TLIdentityNode() + val slaveNode = TLIdentityNode() + + val tile_master_blocker = + tileParams.blockerCtrlAddr + .map(BasicBusBlockerParams(_, xBytes, masterPortBeatBytes, deadlock = true)) + .map(bp => LazyModule(new BasicBusBlocker(bp))) + + tile_master_blocker.foreach(lm => connectTLSlave(lm.controlNode, xBytes)) + + // TODO: this doesn't block other masters, e.g. RoCCs + tlOtherMastersNode := TLBuffer() := tile_master_blocker.map(_.node := tlMasterXbar.node) getOrElse { + tlMasterXbar.node + } + masterNode :=* tlOtherMastersNode + + val cpuDevice: SimpleDevice = new SimpleDevice("cpu", Seq("ucb-bar,boom0", "riscv")) { + override def parent = Some(ResourceAnchors.cpus) + + override def describe(resources: ResourceBindings): Description = { + val Description(name, mapping) = super.describe(resources) + Description( + name, + mapping ++ + cpuProperties ++ + nextLevelCacheProperty ++ + tileProperties + ) + } + + } + + ResourceBinding { + Resource(cpuDevice, "reg").bind(ResourceAddress(tileId)) + } + + override def makeMasterBoundaryBuffers(crossing: ClockCrossingType)(implicit p: Parameters) = crossing match { + case _: RationalCrossing => + if (!boomParams.boundaryBuffers) TLBuffer(BufferParams.none) + else TLBuffer(BufferParams.none, BufferParams.flow, BufferParams.none, BufferParams.flow, BufferParams(1)) + case _ => TLBuffer(BufferParams.none) + } + + override def makeSlaveBoundaryBuffers(crossing: ClockCrossingType)(implicit p: Parameters) = crossing match { + case _: RationalCrossing => + if (!boomParams.boundaryBuffers) TLBuffer(BufferParams.none) + else TLBuffer(BufferParams.flow, BufferParams.none, BufferParams.none, BufferParams.none, BufferParams.none) + case _ => TLBuffer(BufferParams.none) + } + + override lazy val module = new BoomTileModuleImp(this) + + // DCache + lazy val dcache: BoomNonBlockingDCache = LazyModule(new BoomNonBlockingDCache(tileId)) + val dCacheTap = TLIdentityNode() + tlMasterXbar.node := dCacheTap := TLBuffer() := TLWidthWidget( + tileParams.dcache.get.rowBits / 8 + ) := visibilityNode := dcache.node + + // Frontend/ICache + val frontend = LazyModule(new BoomFrontend(tileParams.icache.get, tileId)) + frontend.resetVectorSinkNode := resetVectorNexusNode + tlMasterXbar.node := TLBuffer() := TLWidthWidget(tileParams.icache.get.fetchBytes) := frontend.masterNode + + require( + tileParams.dcache.get.rowBits == tileParams.icache.get.fetchBytes * 8, + s"${tileParams.dcache.get.rowBits} != ${tileParams.icache.get.rowBits}" + ) + + // ROCC + val roccs = p(BuildRoCC).map(_(p)) + val roccCSRs = roccs.map(_.roccCSRs) + require( + roccCSRs.flatten.map(_.id).toSet.size == roccCSRs.flatten.size, + "LazyRoCC instantiations require overlapping CSRs" + ) + roccs.map(_.atlNode).foreach(atl => tlMasterXbar.node :=* atl) + roccs.map(_.tlNode).foreach(tl => tlOtherMastersNode :=* tl) + + val trace_encoder_controller = boomParams.traceParams.map { t => + val trace_encoder_controller = LazyModule(new TraceEncoderController(t.encoderBaseAddr, xBytes, tileId)) + connectTLSlave(trace_encoder_controller.node, xBytes) + trace_encoder_controller + } + + val trace_encoder = boomParams.traceParams match { + case Some(t) => Some(t.buildEncoder(p)) + case None => None + } + + val (trace_sinks, traceSinkIds) = boomParams.traceParams match { + case Some(t) => t.buildSinks.map(_(p)).unzip + case None => (Nil, Nil) + } + + DisableMonitors(implicit p => tlSlaveXbar.node :*= slaveNode) +} + +/** + * BOOM tile implementation + * + * @param outer top level BOOM tile + */ +class BoomTileModuleImp(outer: BoomTile) extends BaseTileModuleImp(outer) { + + val core = Module(new BoomCore(outer.roccCSRs)(outer.p)) + outer.buckyballContext.bundle.satp := core.io.ptw.ptbr.asUInt + require(core.io.ptw.pmp.size == 8) + outer.buckyballContext.bundle.pmp.zip(core.io.ptw.pmp).foreach { case (out, in) => + out.cfg := in.cfg.asUInt + out.addr := in.addr + } + val lsu = Module(new LSU()(outer.p, outer.dcache.module.edge)) + + val ptwPorts = ListBuffer(lsu.io.ptw, outer.frontend.module.io.ptw, core.io.ptw_tlb) + + val hellaCachePorts = ListBuffer[HellaCacheIO]() + + outer.reportWFI(None) // TODO: actually report this? + + outer.decodeCoreInterrupts(core.io.interrupts) // Decode the interrupt vector + + // Pass through various external constants and reports + outer.traceSourceNode.bundle <> core.io.trace + outer.bpwatchSourceNode.bundle <> DontCare // core.io.bpwatch + core.io.hartid := outer.hartIdSinkNode.bundle + + // Connect the core pipeline to other intra-tile modules + outer.frontend.module.io.cpu <> core.io.ifu + core.io.lsu <> lsu.io.core + + //fpuOpt foreach { fpu => core.io.fpu <> fpu.io } RocketFpu - not needed in boom + core.io.rocc := DontCare + + // RoCC + if (outer.roccs.size > 0) { + val (respArb, cmdRouter) = { + val respArb = Module(new RRArbiter(new RoCCResponse()(outer.p), outer.roccs.size)) + val cmdRouter = Module(new RoccCommandRouter(outer.roccs.map(_.opcodes))(outer.p)) + outer.roccs.zipWithIndex.foreach { case (rocc, i) => + ptwPorts ++= rocc.module.io.ptw + rocc.module.io.cmd <> cmdRouter.io.out(i) + val dcIF = Module(new SimpleHellaCacheIF()(outer.p)) + dcIF.io.requestor <> rocc.module.io.mem + hellaCachePorts += dcIF.io.cache + respArb.io.in(i) <> Queue(rocc.module.io.resp) + } + // first keep fpu ios unconnected + val fp_ios = outer.roccs.map { r => + val roccio = r.module.io + roccio.fpu_req.ready := true.B + roccio.fpu_resp.valid := false.B + roccio.fpu_resp.bits := DontCare + } + // Create this FPU just for RoCC + val nFPUPorts = outer.roccs.filter(_.usesFPU).size + if (nFPUPorts > 0) { + val fpuOpt = outer.tileParams.core.fpu.map(params => Module(new freechips.rocketchip.tile.FPU(params)(outer.p))) + // TODO: Check this FPU works properly + fpuOpt foreach { fpu => + // This FPU does not get CPU requests + fpu.io := DontCare + fpu.io.fcsr_rm := core.io.fcsr_rm + fpu.io.ll_resp_val := false.B + fpu.io.valid := false.B + fpu.io.killx := false.B + fpu.io.killm := false.B + + val fpArb = Module(new InOrderArbiter(new FPInput()(outer.p), new FPResult()(outer.p), nFPUPorts)) + val fp_rocc_ios = outer.roccs.filter(_.usesFPU).map(_.module.io) + fpArb.io.in_req <> fp_rocc_ios.map(_.fpu_req) + fp_rocc_ios.zip(fpArb.io.in_resp).foreach { + case (rocc, arb) => rocc.fpu_resp <> arb + } + fpu.io.cp_req <> fpArb.io.out_req + fpArb.io.out_resp <> fpu.io.cp_resp + } + } + (respArb, cmdRouter) + } + + cmdRouter.io.in <> core.io.rocc.cmd + outer.roccs.foreach(_.module.io.exception := core.io.rocc.exception) + core.io.rocc.resp <> respArb.io.out + core.io.rocc.busy <> (cmdRouter.io.busy || outer.roccs.map(_.module.io.busy).reduce(_ || _)) + core.io.rocc.interrupt := outer.roccs.map(_.module.io.interrupt).reduce(_ || _) + (core.io.rocc.csrs zip outer.roccs.map(_.module.io.csrs).flatten).foreach(t => t._2 <> t._1) + } + + // PTW + val ptw = Module(new PTW(ptwPorts.length)(outer.dcache.node.edges.out(0), outer.p)) + core.io.ptw <> ptw.io.dpath + ptw.io.requestor <> ptwPorts.toSeq + ptw.io.mem +=: hellaCachePorts + + // LSU IO + val hellaCacheArb = Module(new HellaCacheArbiter(hellaCachePorts.length)(outer.p)) + hellaCacheArb.io.requestor <> hellaCachePorts.toSeq + lsu.io.hellacache <> hellaCacheArb.io.mem + outer.dcache.module.io.lsu <> lsu.io.dmem + + if (outer.boomParams.traceParams.isDefined) { + core.io.trace_core_ingress.get <> outer.trace_encoder.get.module.io.in + outer.trace_encoder_controller.foreach { lm => + outer.trace_encoder.get.module.io.control <> lm.module.io.control + } + + val trace_sink_arbiter = Module(new TraceSinkArbiter( + outer.traceSinkIds, + use_monitor = outer.boomParams.traceParams.get.useArbiterMonitor, + monitor_name = outer.boomParams.uniqueName + )) + + trace_sink_arbiter.io.target := outer.trace_encoder.get.module.io.control.target + trace_sink_arbiter.io.in <> outer.trace_encoder.get.module.io.out + + core.io.traceStall := outer.trace_encoder.get.module.io.stall + + outer.trace_sinks.zip(outer.traceSinkIds).foreach { case (sink, id) => + val index = outer.traceSinkIds.indexOf(id) + sink.module.io.trace_in <> trace_sink_arbiter.io.out(index) + } + } else { + core.io.traceStall := false.B // no trace encoder, so no stall + } + + // Generate a descriptive string + val frontendStr = outer.frontend.module.toString + val coreStr = core.toString + + val boomTileStr = + (BoomCoreStringPrefix(s"======BOOM Tile ${outer.tileId} Params======") + "\n" + + frontendStr + + coreStr + "\n") + + override def toString: String = boomTileStr + + print(boomTileStr) +} diff --git a/examples/components/cores/boom/configs/Params.scala b/examples/prefab/cores/boom/configs/Params.scala similarity index 100% rename from examples/components/cores/boom/configs/Params.scala rename to examples/prefab/cores/boom/configs/Params.scala diff --git a/examples/components/cores/boom/configs/boom_params.proto b/examples/prefab/cores/boom/configs/boom_params.proto similarity index 100% rename from examples/components/cores/boom/configs/boom_params.proto rename to examples/prefab/cores/boom/configs/boom_params.proto diff --git a/examples/components/cores/boom/stack/CMakeLists.txt b/examples/prefab/cores/boom/stack/CMakeLists.txt similarity index 89% rename from examples/components/cores/boom/stack/CMakeLists.txt rename to examples/prefab/cores/boom/stack/CMakeLists.txt index 7d00c1146..3f5c0900c 100644 --- a/examples/components/cores/boom/stack/CMakeLists.txt +++ b/examples/prefab/cores/boom/stack/CMakeLists.txt @@ -17,7 +17,7 @@ find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) add_custom_command( OUTPUT execute.o COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 - "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" -c "${RUNTIME}/execute/direct.c" -o execute.o DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" VERBATIM) diff --git a/examples/components/cores/boom/thirdparty/riscv-boom b/examples/prefab/cores/boom/thirdparty/riscv-boom similarity index 100% rename from examples/components/cores/boom/thirdparty/riscv-boom rename to examples/prefab/cores/boom/thirdparty/riscv-boom diff --git a/examples/components/cores/boom/thirdparty/rocket-chip b/examples/prefab/cores/boom/thirdparty/rocket-chip similarity index 100% rename from examples/components/cores/boom/thirdparty/rocket-chip rename to examples/prefab/cores/boom/thirdparty/rocket-chip diff --git a/examples/components/cores/boom/workloads/ctests/CMakeLists.txt b/examples/prefab/cores/boom/workloads/ctests/CMakeLists.txt similarity index 100% rename from examples/components/cores/boom/workloads/ctests/CMakeLists.txt rename to examples/prefab/cores/boom/workloads/ctests/CMakeLists.txt diff --git a/examples/components/cores/boom/workloads/ctests/memory_matrix.c b/examples/prefab/cores/boom/workloads/ctests/memory_matrix.c similarity index 100% rename from examples/components/cores/boom/workloads/ctests/memory_matrix.c rename to examples/prefab/cores/boom/workloads/ctests/memory_matrix.c diff --git a/examples/components/cores/reduce/arch/ReduceCore.scala b/examples/prefab/cores/reduce/arch/ReduceCore.scala similarity index 100% rename from examples/components/cores/reduce/arch/ReduceCore.scala rename to examples/prefab/cores/reduce/arch/ReduceCore.scala diff --git a/examples/components/cores/reduce/arch/acc.scala b/examples/prefab/cores/reduce/arch/acc.scala similarity index 100% rename from examples/components/cores/reduce/arch/acc.scala rename to examples/prefab/cores/reduce/arch/acc.scala diff --git a/examples/components/cores/reduce/arch/build.sc b/examples/prefab/cores/reduce/arch/build.sc similarity index 100% rename from examples/components/cores/reduce/arch/build.sc rename to examples/prefab/cores/reduce/arch/build.sc diff --git a/examples/components/cores/reduce/arch/loom.scala b/examples/prefab/cores/reduce/arch/loom.scala similarity index 100% rename from examples/components/cores/reduce/arch/loom.scala rename to examples/prefab/cores/reduce/arch/loom.scala diff --git a/examples/components/cores/reduce/emu/Cargo.lock b/examples/prefab/cores/reduce/emu/Cargo.lock similarity index 100% rename from examples/components/cores/reduce/emu/Cargo.lock rename to examples/prefab/cores/reduce/emu/Cargo.lock diff --git a/examples/components/cores/reduce/emu/Cargo.toml b/examples/prefab/cores/reduce/emu/Cargo.toml similarity index 100% rename from examples/components/cores/reduce/emu/Cargo.toml rename to examples/prefab/cores/reduce/emu/Cargo.toml diff --git a/examples/components/cores/reduce/emu/src/lib.rs b/examples/prefab/cores/reduce/emu/src/lib.rs similarity index 100% rename from examples/components/cores/reduce/emu/src/lib.rs rename to examples/prefab/cores/reduce/emu/src/lib.rs diff --git a/examples/components/cores/reduce/stack/CMakeLists.txt b/examples/prefab/cores/reduce/stack/CMakeLists.txt similarity index 86% rename from examples/components/cores/reduce/stack/CMakeLists.txt rename to examples/prefab/cores/reduce/stack/CMakeLists.txt index 0d00827cb..7af3b899b 100644 --- a/examples/components/cores/reduce/stack/CMakeLists.txt +++ b/examples/prefab/cores/reduce/stack/CMakeLists.txt @@ -14,7 +14,7 @@ project(reduce-core LANGUAGES C CXX) include("${DESIGN_CMAKE}") set(RUNTIME "${DESIGN_REPO}/stack/runtime") -set(TARGET ${DESIGN_CORE_TARGET_${CORE}}) +set(TARGET ${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_PROFILE}) find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) # ===----------------------------------------------------------------------=== @@ -23,7 +23,7 @@ find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) add_custom_command( OUTPUT execute.o COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 - "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" -c "${RUNTIME}/execute/direct.c" -o execute.o DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" VERBATIM) @@ -33,9 +33,9 @@ add_custom_target(execute ALL DEPENDS execute.o) # Compiler: the Buckyball dialect and passes with the Balls of the accelerator. # ===----------------------------------------------------------------------=== set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${TARGET}}) -set(BUCKYBALL_TARGET_TABLE "${DESIGN_ISA_DIR}/${TARGET}/target.inc") add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) -add_library(${TARGET}-compiler MODULE $) +add_library(${TARGET}-compiler MODULE) +target_link_libraries(${TARGET}-compiler PRIVATE buckyball-compiler) set_target_properties(${TARGET}-compiler PROPERTIES LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") diff --git a/examples/components/cores/rocket/LICENSE.Berkeley b/examples/prefab/cores/rocket/LICENSE.Berkeley similarity index 100% rename from examples/components/cores/rocket/LICENSE.Berkeley rename to examples/prefab/cores/rocket/LICENSE.Berkeley diff --git a/examples/components/cores/rocket/LICENSE.SiFive b/examples/prefab/cores/rocket/LICENSE.SiFive similarity index 100% rename from examples/components/cores/rocket/LICENSE.SiFive rename to examples/prefab/cores/rocket/LICENSE.SiFive diff --git a/examples/components/cores/rocket/arch/Execution.scala b/examples/prefab/cores/rocket/arch/Execution.scala similarity index 99% rename from examples/components/cores/rocket/arch/Execution.scala rename to examples/prefab/cores/rocket/arch/Execution.scala index 73d9501c8..39272c5a3 100644 --- a/examples/components/cores/rocket/arch/Execution.scala +++ b/examples/prefab/cores/rocket/arch/Execution.scala @@ -55,7 +55,7 @@ class Execution(config: Params, commandsEnabled: Boolean) extends Module with Ha csr.set := false.B csr.sdata := 0.U } - val fpu = coreParams.fpu.map(cfg => Instantiate(new FPU(cfg))) + val fpu = coreParams.fpu.map(cfg => Module(new FPU(cfg))) fpu match { case Some(unit) => cpu.fpu :<>= unit.io.waiveAs[FPUCoreIO](_.cp_req, _.cp_resp) @@ -152,7 +152,7 @@ class Execution(config: Params, commandsEnabled: Boolean) extends Module with Ha val fetchPmp = Reg(Vec(nPMPs, new PMP)) when(io.fetch.request.fire)(fetchPmp := cpu.ptw.pmp) for ((port, pmps) <- Seq(io.fetch -> fetchPmp, io.data -> lsu.io.capturedPmp)) { - val checker = Instantiate(new PMPChecker(3)) + val checker = Module(new PMPChecker(3)) checker.io.prv := port.authorizationRequest.bits.privilege checker.io.pmp := pmps checker.io.addr := port.authorizationRequest.bits.paddr diff --git a/examples/components/cores/rocket/arch/Lsu.scala b/examples/prefab/cores/rocket/arch/Lsu.scala similarity index 100% rename from examples/components/cores/rocket/arch/Lsu.scala rename to examples/prefab/cores/rocket/arch/Lsu.scala diff --git a/examples/components/cores/rocket/arch/Rocket.scala b/examples/prefab/cores/rocket/arch/Rocket.scala similarity index 100% rename from examples/components/cores/rocket/arch/Rocket.scala rename to examples/prefab/cores/rocket/arch/Rocket.scala diff --git a/examples/components/cores/rocket/arch/build.sc b/examples/prefab/cores/rocket/arch/build.sc similarity index 100% rename from examples/components/cores/rocket/arch/build.sc rename to examples/prefab/cores/rocket/arch/build.sc diff --git a/examples/components/cores/rocket/arch/pipeline/Config.scala b/examples/prefab/cores/rocket/arch/pipeline/Config.scala similarity index 100% rename from examples/components/cores/rocket/arch/pipeline/Config.scala rename to examples/prefab/cores/rocket/arch/pipeline/Config.scala diff --git a/examples/components/cores/rocket/arch/pipeline/Pipeline.scala b/examples/prefab/cores/rocket/arch/pipeline/Pipeline.scala similarity index 99% rename from examples/components/cores/rocket/arch/pipeline/Pipeline.scala rename to examples/prefab/cores/rocket/arch/pipeline/Pipeline.scala index cab1b82a1..373cb3599 100644 --- a/examples/components/cores/rocket/arch/pipeline/Pipeline.scala +++ b/examples/prefab/cores/rocket/arch/pipeline/Pipeline.scala @@ -3,10 +3,8 @@ package examples.core.rocket.pipeline { - import chisel3.experimental.hierarchy.Definition - import framework.lib.arbiter.{Arbiter} - import chisel3.experimental.hierarchy.{Instance, Instantiate} + import chisel3.experimental.hierarchy.Instantiate import chisel3._ import chisel3.experimental.hierarchy.{instantiable, public} @@ -280,7 +278,7 @@ package examples.core.rocket.pipeline { val take_pc = take_pc_mem_wb // decode stage - val ibuf = Instantiate(new IBuf) + val ibuf = Module(new IBuf) val id_expanded_inst = ibuf.io.inst.map(_.bits.inst) val id_raw_inst = ibuf.io.inst.map(_.bits.raw) val id_inst = id_expanded_inst.map(_.bits) @@ -314,14 +312,14 @@ package examples.core.rocket.pipeline { if (usingRoCC) p else p.alterPartial { case CoreRoCCCount => 0 } - val csr = Instance(Definition(new CSRFile( + val csr = Module(new CSRFile( perfEvents, coreParams.customCSRs.decls, roccCSRs, hasBusError )(csrP) { override def usingRoCC: Boolean = RocketBB.this.usingRoCC - })) + }) val id_csr_en = id_ctrl.csr.isOneOf(CSR.S, CSR.C, CSR.W) val id_system_insn = id_ctrl.csr === CSR.I @@ -401,7 +399,7 @@ package examples.core.rocket.pipeline { id_vec_busy && id_ctrl.fence || id_mem_busy && (id_ctrl.amo && id_amo_rl || id_ctrl.fence_i || id_reg_fence && (id_ctrl.mem || id_ctrl.rocc))) - val bpu = Instantiate(new BreakpointUnit(nBreakpoints)) + val bpu = Module(new BreakpointUnit(nBreakpoints)) bpu.io.status := csr.io.status bpu.io.bp := csr.io.bp bpu.io.pc := ibuf.io.pc @@ -518,14 +516,14 @@ package examples.core.rocket.pipeline { (Some(ex_new_vl), Some(ex_new_vconfig)) } else { (None, None) } - val alu = Instantiate(new ALU) + val alu = Module(new ALU) alu.io.dw := ex_ctrl.alu_dw alu.io.fn := ex_ctrl.alu_fn alu.io.in2 := ex_op2.asUInt alu.io.in1 := ex_op1.asUInt // multiplier and divider - val div = Instantiate(new MulDiv(if (pipelinedMul) mulDivParams.copy(mulUnroll = 0) else mulDivParams, xLen)) + val div = Module(new MulDiv(if (pipelinedMul) mulDivParams.copy(mulUnroll = 0) else mulDivParams, xLen)) div.io.req.valid := ex_reg_valid && ex_ctrl.div div.io.req.bits.dw := ex_ctrl.alu_dw div.io.req.bits.fn := ex_ctrl.alu_fn @@ -534,7 +532,7 @@ package examples.core.rocket.pipeline { div.io.req.bits.tag := ex_waddr val mul = pipelinedMul.option { - val m = Instantiate(new PipelinedMultiplier(xLen, 2)) + val m = Module(new PipelinedMultiplier(xLen, 2)) m.io.req.valid := ex_reg_valid && ex_ctrl.mul m.io.req.bits := div.io.req.bits m @@ -1013,7 +1011,7 @@ package examples.core.rocket.pipeline { val csr_trace_with_wdata = WireInit(csr.io.trace(0)) csr_trace_with_wdata.wdata.get := rf_wdata - val debug_rob = Instantiate(new HardDebugROB(sz, 32)) + val debug_rob = Module(new HardDebugROB(sz, 32)) debug_rob.io.i_insn := csr_trace_with_wdata debug_rob.io.should_wb := (wb_ctrl.wfd || (wb_ctrl.wxd && wb_waddr =/= 0.U)) && !csr.io.trace(0).exception diff --git a/examples/components/cores/rocket/arch/pipeline/Utility.scala b/examples/prefab/cores/rocket/arch/pipeline/Utility.scala similarity index 100% rename from examples/components/cores/rocket/arch/pipeline/Utility.scala rename to examples/prefab/cores/rocket/arch/pipeline/Utility.scala diff --git a/examples/components/cores/rocket/configs/Params.scala b/examples/prefab/cores/rocket/configs/Params.scala similarity index 100% rename from examples/components/cores/rocket/configs/Params.scala rename to examples/prefab/cores/rocket/configs/Params.scala diff --git a/examples/components/cores/rocket/configs/rocket_params.proto b/examples/prefab/cores/rocket/configs/rocket_params.proto similarity index 100% rename from examples/components/cores/rocket/configs/rocket_params.proto rename to examples/prefab/cores/rocket/configs/rocket_params.proto diff --git a/examples/components/cores/rocket/emu/Cargo.lock b/examples/prefab/cores/rocket/emu/Cargo.lock similarity index 100% rename from examples/components/cores/rocket/emu/Cargo.lock rename to examples/prefab/cores/rocket/emu/Cargo.lock diff --git a/examples/components/cores/rocket/emu/Cargo.toml b/examples/prefab/cores/rocket/emu/Cargo.toml similarity index 100% rename from examples/components/cores/rocket/emu/Cargo.toml rename to examples/prefab/cores/rocket/emu/Cargo.toml diff --git a/examples/components/cores/rocket/emu/src/lib.rs b/examples/prefab/cores/rocket/emu/src/lib.rs similarity index 100% rename from examples/components/cores/rocket/emu/src/lib.rs rename to examples/prefab/cores/rocket/emu/src/lib.rs diff --git a/examples/components/cores/rocket/stack/CMakeLists.txt b/examples/prefab/cores/rocket/stack/CMakeLists.txt similarity index 89% rename from examples/components/cores/rocket/stack/CMakeLists.txt rename to examples/prefab/cores/rocket/stack/CMakeLists.txt index 389428cd5..5c50671ae 100644 --- a/examples/components/cores/rocket/stack/CMakeLists.txt +++ b/examples/prefab/cores/rocket/stack/CMakeLists.txt @@ -17,7 +17,7 @@ find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) add_custom_command( OUTPUT execute.o COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 - "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" -c "${RUNTIME}/execute/direct.c" -o execute.o DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" VERBATIM) diff --git a/examples/components/cores/rocket/thirdparty/rocket-chip b/examples/prefab/cores/rocket/thirdparty/rocket-chip similarity index 100% rename from examples/components/cores/rocket/thirdparty/rocket-chip rename to examples/prefab/cores/rocket/thirdparty/rocket-chip diff --git a/examples/components/cores/xiangshan/LICENSE b/examples/prefab/cores/xiangshan/LICENSE similarity index 100% rename from examples/components/cores/xiangshan/LICENSE rename to examples/prefab/cores/xiangshan/LICENSE diff --git a/examples/components/cores/xiangshan/arch/Chi.scala b/examples/prefab/cores/xiangshan/arch/Chi.scala similarity index 100% rename from examples/components/cores/xiangshan/arch/Chi.scala rename to examples/prefab/cores/xiangshan/arch/Chi.scala diff --git a/examples/components/cores/xiangshan/arch/Native.scala b/examples/prefab/cores/xiangshan/arch/Native.scala similarity index 100% rename from examples/components/cores/xiangshan/arch/Native.scala rename to examples/prefab/cores/xiangshan/arch/Native.scala diff --git a/examples/components/cores/xiangshan/arch/XiangShan.scala b/examples/prefab/cores/xiangshan/arch/XiangShan.scala similarity index 100% rename from examples/components/cores/xiangshan/arch/XiangShan.scala rename to examples/prefab/cores/xiangshan/arch/XiangShan.scala diff --git a/examples/prefab/cores/xiangshan/arch/build.sc b/examples/prefab/cores/xiangshan/arch/build.sc new file mode 100644 index 000000000..df61d4f66 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/build.sc @@ -0,0 +1,88 @@ +import mill._ +import mill.scalalib._ +import $file.^.^.^.^.^.arch.src.main.scala.framework.{build => framework} + +val componentRoot: os.Path = millSourcePath / os.up +val vendorRoot: os.Path = componentRoot / "thirdparty" / "xiangshan" +// Our XiangShan sources; files under native/xiangshan replace the upstream file at the same path. +val nativeRoot: os.Path = componentRoot / "arch" / "native" +val vendorHead: os.Path = + os.Path(os.proc("git", "rev-parse", "--git-path", "HEAD").call(cwd = vendorRoot).out.text().trim, vendorRoot) + +def upstreamInputs = T.sources { + val paths = Seq( + "rocket-chip/src", "rocket-chip/macros/src", "rocket-chip/cde/cde/src", "rocket-chip/hardfloat/hardfloat/src", + "difftest/src", "utility/src", "huancun/src", "fudian/src", "yunsuan/src", "coupledL2/src", "openLLC/src", + "openLLC/openNCB/src", "ChiselAIA/src", "ChiselIOPMP/src", "macros/src", "build.sc", ".mill-version", + "rocket-chip/common.sc", "rocket-chip/cde/common.sc", "rocket-chip/hardfloat/common.sc", "huancun/common.sc", + "coupledL2/common.sc", "openLLC/common.sc" + ) + paths.map(path => PathRef(vendorRoot / os.RelPath(path))) :+ PathRef(vendorHead) +} + +// Libraries XiangShan depends on (rocket-chip, utility, coupledL2, ...) with their resources, built by XiangShan's own build. +def upstreamClasspath = T { + upstreamInputs() + Seq("xiangshan.transitiveLocalClasspath", "xiangshan.compileClasspath").flatMap { task => + val shown = os.proc("mill", "--no-server", "show", task).call(cwd = vendorRoot).out.text() + ujson.read(shown).arr.map(entry => os.Path(entry.str.split(":").last)) + }.distinct.map(PathRef(_)) +} + +def gitState = T.input { + val sha = os.proc("git", "rev-parse", "HEAD").call(cwd = vendorRoot).out.trim() + val dirty = os.proc("git", "status", "-uno", "--porcelain").call(cwd = vendorRoot).out.text().nonEmpty + (sha, dirty) +} + +// Resources XiangShan's own build generates: version string and the commit id embedded in the CSR. +def versionResources = T { + val (sha, dirty) = gitState() + os.write(T.dest / "publishVersion", sha.take(12) + "\n") + os.write(T.dest / "gitStatus", s"SHA=$sha\ndirty=${if (dirty) 1 else 0}\n") + PathRef(T.dest) +} + +def vendorResources = T.sources(vendorRoot / "src" / "main" / "resources") + +object native extends ScalaModule { + override def scalaVersion = "2.13.15" + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations", "-Ytasty-reader") + override def unmanagedClasspath = T(Agg.from(upstreamClasspath())) + + override def sources = T.sources { + val upstream = vendorRoot / "src" / "main" / "scala" + os.walk(upstream) + .filter(p => p.ext == "scala" && !os.exists(nativeRoot / p.relativeTo(upstream))) + .map(PathRef(_)) :+ PathRef(nativeRoot) + } + + override def resources = T(versionResources() +: vendorResources()) +} + +def nativeRtl = T { + val output = T.dest / "xiangshan" + os.proc( + "java", "-Xmx40G", "-Xss256m", "-cp", native.runClasspath().map(_.path).mkString(java.io.File.pathSeparator), + "examples.core.xiangshan.native.Emit", "--target-dir", output, "--target", "systemverilog" + ).call(cwd = T.dest, env = Map("PATH" -> s"${sys.env("PATH")}:${vendorRoot / "src" / "main" / "resources"}")) + os.move(output / "extern_modules.sv", output / "XiangShanExtern.sv") + os.write.over(output / "filelist.f", os.read(output / "filelist.f").replace("extern_modules.sv", "XiangShanExtern.sv")) + PathRef(T.dest) +} + +trait ScalaBuild extends SbtModule { + override def scalaVersion = "2.13.16" + override def ivyDeps = Agg(ivy"org.chipsalliance::chisel:6.7.0") + override def scalacPluginIvyDeps = Agg(ivy"org.chipsalliance:::chisel-plugin:6.7.0") + override def scalacOptions = Seq("-language:reflectiveCalls", "-Ymacro-annotations") +} +object library extends ScalaBuild { + override def moduleDeps = Seq(framework.library) + override def sources = T.sources { + os.walk(componentRoot / "arch").filter(p => p.ext == "scala" && !p.startsWith(nativeRoot)).map(PathRef(_)) ++ + Seq(PathRef(componentRoot / "configs")) + } + override def resources = T.sources(Seq(nativeRtl())) +} diff --git a/examples/prefab/cores/xiangshan/arch/native/Backend.scala b/examples/prefab/cores/xiangshan/arch/native/Backend.scala new file mode 100644 index 000000000..c31a6e255 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/Backend.scala @@ -0,0 +1,1151 @@ +// Copied from XiangShan src/main/scala/xiangshan/backend/Backend.scala and modified for Buckyball. +// XiangShan is licensed under Mulan PSL v2 (http://license.coscl.org.cn/MulanPSL2). + +package examples.core.xiangshan.native + +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import chisel3.util._ +import difftest._ +import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} +import system.HasSoCParameter +import utility._ +import utility.sram.SramBroadcastBundle +import xiangshan._ +import xiangshan.backend.Bundles.{ + DynInst, + IssueQueueIQWakeUpBundle, + LoadShouldCancel, + MemExuInput, + MemExuOutput, + VPUCtrlSignals +} +import xiangshan.backend.ctrlblock.{DebugLSIO, LsTopdownInfo} +import xiangshan.backend.datapath.DataConfig.{FpData, IntData, VecData} +import xiangshan.backend.datapath.RdConfig.{IntRD, VfRD} +import xiangshan.backend.datapath.WbConfig._ +import xiangshan.backend.datapath.DataConfig._ +import xiangshan.backend.datapath._ +import xiangshan.backend.dispatch.CoreDispatchTopDownIO +import xiangshan.backend.exu.ExuBlock +import xiangshan.backend.fu.vector.Bundles.{VConfig, VType} +import xiangshan.backend.fu.{FenceIO, FenceToSbuffer, FuConfig, FuType, PerfCounterIO} +import xiangshan.backend.fu.NewCSR.PFEvent +import xiangshan.backend.issue.EntryBundles._ +import xiangshan.backend.issue.{Scheduler, SchedulerArithImp, SchedulerImpBase, SchedulerMemImp} +import xiangshan.backend.rob.{RobCoreTopDownIO, RobDebugRollingIO, RobLsqIO, RobPtr} +import xiangshan.backend.trace.TraceCoreInterface +import xiangshan.frontend.{FtqPtr, FtqRead, PreDecodeInfo} +import xiangshan.mem.{LqPtr, LsqEnqIO, SqPtr} +import scala.collection.mutable +import xiangshan.backend._ +import examples.core.xiangshan.native.{Commands, CtrlBlock} + +class Backend(val params: BackendParams)(implicit p: Parameters) extends LazyModule with HasXSParameter { + override def shouldBeInlined: Boolean = false + val inner = LazyModule(new BackendInlined(params)) + lazy val module = new BackendImp(this) +} + +class BackendImp(wrapper: Backend)(implicit p: Parameters) extends LazyModuleImp(wrapper) { + val io = IO(new BackendIO()(p, wrapper.params)) + io <> wrapper.inner.module.io + if (p(DebugOptionsKey).ResetGen) { + ResetGen(ResetGenNode(Seq(ModuleNode(wrapper.inner.module))), reset, sim = false, io.dft_reset) + } +} + +class BackendInlined(val params: BackendParams)(implicit p: Parameters) extends LazyModule with HasXSParameter { + + override def shouldBeInlined: Boolean = true + + // check read & write port config + params.configChecks + + /* Only update the idx in mem-scheduler here Idx in other schedulers can be updated the same way if needed + * + * Also note that we filter out the 'stData issue-queues' when counting */ + for ((ibp, idx) <- params.memSchdParams.get.issueBlockParams.filter(iq => iq.StdCnt == 0).zipWithIndex) { + ibp.updateIdx(idx) + } + + println(params.iqWakeUpParams) + + for ((schdCfg, i) <- params.allSchdParams.zipWithIndex) { + schdCfg.bindBackendParam(params) + } + + for ((iqCfg, i) <- params.allIssueParams.zipWithIndex) { + iqCfg.bindBackendParam(params) + } + + for ((exuCfg, i) <- params.allExuParams.zipWithIndex) { + exuCfg.bindBackendParam(params) + exuCfg.updateIQWakeUpConfigs(params.iqWakeUpParams) + exuCfg.updateExuIdx(i) + } + + println("[Backend] ExuConfigs:") + for (exuCfg <- params.allExuParams) { + val fuConfigs = exuCfg.fuConfigs + val wbPortConfigs = exuCfg.wbPortConfigs + val immType = exuCfg.immType + + println("[Backend] " + + s"${exuCfg.name}: " + + (if (exuCfg.fakeUnit) "fake, " else "") + + (if (exuCfg.hasLoadFu || exuCfg.hasHyldaFu) s"LdExuIdx(${backendParams.getLdExuIdx(exuCfg)})" else "") + + s"${fuConfigs.map(_.name).mkString("fu(s): {", ",", "}")}, " + + s"${wbPortConfigs.mkString("wb: {", ",", "}")}, " + + s"${immType.map(SelImm.mkString(_)).mkString("imm: {", ",", "}")}, " + + s"latMax(${exuCfg.latencyValMax}), ${exuCfg.fuLatancySet.mkString("lat: {", ",", "}")}, " + + s"srcReg(${exuCfg.numRegSrc})") + require( + wbPortConfigs.collectFirst { case x: IntWB => x }.nonEmpty == + fuConfigs.map(_.writeIntRf).reduce(_ || _), + s"${exuCfg.name} int wb port has no priority" + ) + require( + wbPortConfigs.collectFirst { case x: FpWB => x }.nonEmpty == + fuConfigs.map(x => x.writeFpRf).reduce(_ || _), + s"${exuCfg.name} fp wb port has no priority" + ) + require( + wbPortConfigs.collectFirst { case x: VfWB => x }.nonEmpty == + fuConfigs.map(x => x.writeVecRf).reduce(_ || _), + s"${exuCfg.name} vec wb port has no priority" + ) + } + + println(s"[Backend] all fu configs") + for (cfg <- FuConfig.allConfigs) { + println(s"[Backend] $cfg") + } + + println(s"[Backend] Int RdConfigs: ExuName(Priority)") + for ((port, seq) <- params.getRdPortParams(IntData())) { + println( + s"[Backend] port($port): ${seq.map(x => params.getExuName(x._1) + "(" + x._2.toString + ")").mkString(",")}" + ) + } + + println(s"[Backend] Int WbConfigs: ExuName(Priority)") + for ((port, seq) <- params.getWbPortParams(IntData())) { + println( + s"[Backend] port($port): ${seq.map(x => params.getExuName(x._1) + "(" + x._2.toString + ")").mkString(",")}" + ) + } + + println(s"[Backend] Fp RdConfigs: ExuName(Priority)") + for ((port, seq) <- params.getRdPortParams(FpData())) { + println( + s"[Backend] port($port): ${seq.map(x => params.getExuName(x._1) + "(" + x._2.toString + ")").mkString(",")}" + ) + } + + println(s"[Backend] Fp WbConfigs: ExuName(Priority)") + for ((port, seq) <- params.getWbPortParams(FpData())) { + println( + s"[Backend] port($port): ${seq.map(x => params.getExuName(x._1) + "(" + x._2.toString + ")").mkString(",")}" + ) + } + + println(s"[Backend] Vf RdConfigs: ExuName(Priority)") + for ((port, seq) <- params.getRdPortParams(VecData())) { + println( + s"[Backend] port($port): ${seq.map(x => params.getExuName(x._1) + "(" + x._2.toString + ")").mkString(",")}" + ) + } + + println(s"[Backend] Vf WbConfigs: ExuName(Priority)") + for ((port, seq) <- params.getWbPortParams(VecData())) { + println( + s"[Backend] port($port): ${seq.map(x => params.getExuName(x._1) + "(" + x._2.toString + ")").mkString(",")}" + ) + } + + println(s"[Backend] Dispatch Configs:") + println(s"[Backend] Load IQ enq width(${params.numLoadDp}), Store IQ enq width(${params.numStoreDp})") + println(s"[Backend] Load DP width($LSQLdEnqWidth), Store DP width($LSQStEnqWidth)") + + params.updateCopyPdestInfo + println(s"[Backend] copyPdestInfo ${params.copyPdestInfo}") + params.allExuParams.map(_.copyNum) + val ctrlBlock = LazyModule(new CtrlBlock(params)) + val intScheduler = params.intSchdParams.map(x => LazyModule(new Scheduler(x))) + val fpScheduler = params.fpSchdParams.map(x => LazyModule(new Scheduler(x))) + val vfScheduler = params.vfSchdParams.map(x => LazyModule(new Scheduler(x))) + val memScheduler = params.memSchdParams.map(x => LazyModule(new Scheduler(x))) + val dataPath = LazyModule(new DataPath(params)) + val intExuBlock = params.intSchdParams.map(x => LazyModule(new ExuBlock(x))) + val fpExuBlock = params.fpSchdParams.map(x => LazyModule(new ExuBlock(x))) + val vfExuBlock = params.vfSchdParams.map(x => LazyModule(new ExuBlock(x))) + val wbFuBusyTable = LazyModule(new WbFuBusyTable(params)) + + lazy val module = new BackendInlinedImp(this) +} + +class BackendInlinedImp(override val wrapper: BackendInlined)(implicit p: Parameters) + extends LazyModuleImp(wrapper) + with HasXSParameter + with HasPerfEvents + with HasCriticalErrors { + private implicit val params: BackendParams = wrapper.params + + val io = IO(new BackendIO()(p, wrapper.params)) + + private val ctrlBlock = wrapper.ctrlBlock.module + private val intScheduler: SchedulerImpBase = wrapper.intScheduler.get.module + private val fpScheduler = wrapper.fpScheduler.get.module + private val vfScheduler = wrapper.vfScheduler.get.module + private val memScheduler = wrapper.memScheduler.get.module + private val dataPath = wrapper.dataPath.module + private val intExuBlock = wrapper.intExuBlock.get.module + private val fpExuBlock = wrapper.fpExuBlock.get.module + private val vfExuBlock = wrapper.vfExuBlock.get.module + private val og2ForVector = Module(new Og2ForVector(params)) + private val bypassNetwork = Module(new BypassNetwork) + private val wbDataPath = Module(new WbDataPath(params)) + private val wbFuBusyTable = wrapper.wbFuBusyTable.module + private val vecExcpMod = Module(new VecExcpDataMergeModule) + + private val iqWakeUpMappedBundle: Map[Int, ValidIO[IssueQueueIQWakeUpBundle]] = ( + intScheduler.io.toSchedulers.wakeupVec ++ + fpScheduler.io.toSchedulers.wakeupVec ++ + vfScheduler.io.toSchedulers.wakeupVec ++ + memScheduler.io.toSchedulers.wakeupVec + ).map(x => (x.bits.exuIdx, x)).toMap + + private val iqWakeUpMappedBundleDelayed: Map[Int, ValidIO[IssueQueueIQWakeUpBundle]] = ( + intScheduler.io.toSchedulers.wakeupVec ++ + fpScheduler.io.toSchedulers.wakeupVec ++ + vfScheduler.io.toSchedulers.wakeupVec ++ + memScheduler.io.toSchedulers.wakeupVec + ).map { case x => + val delayed = Wire(chiselTypeOf(x)) + // TODO: add clock gate use Wen, remove issuequeue wakeupToIQ logic Wen = Wen && valid + delayed := RegNext(x) + (x.bits.exuIdx, delayed) + }.toMap + + println(s"[Backend] iq wake up keys: ${iqWakeUpMappedBundle.keys}") + + wbFuBusyTable.io.in.intSchdBusyTable := intScheduler.io.wbFuBusyTable + wbFuBusyTable.io.in.fpSchdBusyTable := fpScheduler.io.wbFuBusyTable + wbFuBusyTable.io.in.vfSchdBusyTable := vfScheduler.io.wbFuBusyTable + wbFuBusyTable.io.in.memSchdBusyTable := memScheduler.io.wbFuBusyTable + intScheduler.io.fromWbFuBusyTable.fuBusyTableRead := wbFuBusyTable.io.out.intRespRead + fpScheduler.io.fromWbFuBusyTable.fuBusyTableRead := wbFuBusyTable.io.out.fpRespRead + vfScheduler.io.fromWbFuBusyTable.fuBusyTableRead := wbFuBusyTable.io.out.vfRespRead + memScheduler.io.fromWbFuBusyTable.fuBusyTableRead := wbFuBusyTable.io.out.memRespRead + dataPath.io.wbConfictRead := wbFuBusyTable.io.out.wbConflictRead + + private val og1Cancel = dataPath.io.og1Cancel + private val og0Cancel = dataPath.io.og0Cancel + private val vlFromIntIsZero = intExuBlock.io.vlIsZero.get + private val vlFromIntIsVlmax = intExuBlock.io.vlIsVlmax.get + private val vlFromVfIsZero = vfExuBlock.io.vlIsZero.get + private val vlFromVfIsVlmax = vfExuBlock.io.vlIsVlmax.get + + private val backendCriticalError = Wire(Bool()) + + ctrlBlock.io.fromTop.hartId := io.fromTop.hartId + ctrlBlock.io.frontend <> io.frontend + ctrlBlock.io.fromCSR.toDecode := intExuBlock.io.csrToDecode.get + ctrlBlock.io.fromCSR.traceCSR := intExuBlock.io.csrio.get.traceCSR + ctrlBlock.io.fromCSR.instrAddrTransType := RegNext(intExuBlock.io.csrio.get.instrAddrTransType) + ctrlBlock.io.fromWB.wbData <> wbDataPath.io.toCtrlBlock.writeback + ctrlBlock.io.fromMem.stIn <> io.mem.stIn + ctrlBlock.io.fromMem.violation <> io.mem.memoryViolation + ctrlBlock.io.lqCanAccept := io.mem.lqCanAccept + ctrlBlock.io.sqCanAccept := io.mem.sqCanAccept + + io.mem.wfi <> ctrlBlock.io.toMem.wfi + + io.mem.lsqEnqIO <> ctrlBlock.io.toMem.lsqEnqIO + ctrlBlock.io.fromMemToDispatch.scommit := io.mem.sqDeq + ctrlBlock.io.fromMemToDispatch.lcommit := io.mem.lqDeq + ctrlBlock.io.fromMemToDispatch.sqDeqPtr := io.mem.sqDeqPtr + ctrlBlock.io.fromMemToDispatch.lqDeqPtr := io.mem.lqDeqPtr + ctrlBlock.io.fromMemToDispatch.sqCancelCnt := io.mem.sqCancelCnt + ctrlBlock.io.fromMemToDispatch.lqCancelCnt := io.mem.lqCancelCnt + ctrlBlock.io.toDispatch.wakeUpInt := intScheduler.io.toSchedulers.wakeupVec + ctrlBlock.io.toDispatch.wakeUpFp := fpScheduler.io.toSchedulers.wakeupVec + ctrlBlock.io.toDispatch.wakeUpVec := vfScheduler.io.toSchedulers.wakeupVec + ctrlBlock.io.toDispatch.wakeUpMem := memScheduler.io.toSchedulers.wakeupVec + ctrlBlock.io.toDispatch.IQValidNumVec := intScheduler.io.IQValidNumVec ++ fpScheduler.io.IQValidNumVec ++ vfScheduler.io.IQValidNumVec ++ memScheduler.io.IQValidNumVec + ctrlBlock.io.toDispatch.ldCancel := io.mem.ldCancel + ctrlBlock.io.toDispatch.og0Cancel := og0Cancel + ctrlBlock.io.toDispatch.wbPregsInt.zip(wbDataPath.io.toIntPreg).map { x => + x._1.valid := x._2.wen && x._2.intWen + x._1.bits := x._2.addr + } + ctrlBlock.io.toDispatch.wbPregsFp.zip(wbDataPath.io.toFpPreg).map { x => + x._1.valid := x._2.wen && x._2.fpWen + x._1.bits := x._2.addr + } + ctrlBlock.io.toDispatch.wbPregsVec.zip(wbDataPath.io.toVfPreg).map { x => + x._1.valid := x._2.wen && x._2.vecWen + x._1.bits := x._2.addr + } + ctrlBlock.io.toDispatch.wbPregsV0.zip(wbDataPath.io.toV0Preg).map { x => + x._1.valid := x._2.wen && x._2.v0Wen + x._1.bits := x._2.addr + } + ctrlBlock.io.toDispatch.wbPregsVl.zip(wbDataPath.io.toVlPreg).map { x => + x._1.valid := x._2.wen && x._2.vlWen + x._1.bits := x._2.addr + } + ctrlBlock.io.toDispatch.vlWriteBackInfo.vlFromIntIsZero := vlFromIntIsZero + ctrlBlock.io.toDispatch.vlWriteBackInfo.vlFromIntIsVlmax := vlFromIntIsVlmax + ctrlBlock.io.toDispatch.vlWriteBackInfo.vlFromVfIsZero := vlFromVfIsZero + ctrlBlock.io.toDispatch.vlWriteBackInfo.vlFromVfIsVlmax := vlFromVfIsVlmax + ctrlBlock.io.csrCtrl <> intExuBlock.io.csrio.get.customCtrl + ctrlBlock.io.robio.csr.intrBitSet := intExuBlock.io.csrio.get.interrupt + ctrlBlock.io.robio.csr.trapTarget := intExuBlock.io.csrio.get.trapTarget + ctrlBlock.io.robio.csr.isXRet := intExuBlock.io.csrio.get.isXRet + ctrlBlock.io.robio.csr.wfiEvent := intExuBlock.io.csrio.get.wfi_event + ctrlBlock.io.robio.csr.criticalErrorState := intExuBlock.io.csrio.get.criticalErrorState + ctrlBlock.io.robio.lsq <> io.mem.robLsqIO + ctrlBlock.io.robio.lsTopdownInfo <> io.mem.lsTopdownInfo + ctrlBlock.io.robio.debug_ls <> io.mem.debugLS + ctrlBlock.io.debugEnqLsq.canAccept := io.mem.lsqEnqIO.canAccept + ctrlBlock.io.debugEnqLsq.resp := io.mem.lsqEnqIO.resp + ctrlBlock.io.debugEnqLsq.req := ctrlBlock.io.toMem.lsqEnqIO.req + ctrlBlock.io.debugEnqLsq.needAlloc := ctrlBlock.io.toMem.lsqEnqIO.needAlloc + ctrlBlock.io.debugEnqLsq.iqAccept := ctrlBlock.io.toMem.lsqEnqIO.iqAccept + ctrlBlock.io.fromVecExcpMod.busy := vecExcpMod.o.status.busy + + val intWriteBackDelayed = Wire(chiselTypeOf(wbDataPath.io.toIntPreg)) + intWriteBackDelayed.zip(wbDataPath.io.toIntPreg).map { case (sink, source) => + sink := DontCare + sink.wen := RegNext(source.wen) + sink.intWen := RegNext(source.intWen) + sink.addr := RegEnable(source.addr, source.wen) + } + val fpWriteBackDelayed = Wire(chiselTypeOf(wbDataPath.io.toFpPreg)) + fpWriteBackDelayed.zip(wbDataPath.io.toFpPreg).map { case (sink, source) => + sink := DontCare + sink.wen := RegNext(source.wen) + sink.fpWen := RegNext(source.fpWen) + sink.addr := RegEnable(source.addr, source.wen) + } + val vfWriteBackDelayed = Wire(chiselTypeOf(wbDataPath.io.toVfPreg)) + vfWriteBackDelayed.zip(wbDataPath.io.toVfPreg).map { case (sink, source) => + sink := DontCare + sink.wen := RegNext(source.wen) + sink.vecWen := RegNext(source.vecWen) + sink.addr := RegEnable(source.addr, source.wen) + } + val v0WriteBackDelayed = Wire(chiselTypeOf(wbDataPath.io.toV0Preg)) + v0WriteBackDelayed.zip(wbDataPath.io.toV0Preg).map { case (sink, source) => + sink := DontCare + sink.wen := RegNext(source.wen) + sink.v0Wen := RegNext(source.v0Wen) + sink.addr := RegEnable(source.addr, source.wen) + } + val vlWriteBackDelayed = Wire(chiselTypeOf(wbDataPath.io.toVlPreg)) + vlWriteBackDelayed.zip(wbDataPath.io.toVlPreg).map { case (sink, source) => + sink := DontCare + sink.wen := RegNext(source.wen) + sink.vlWen := RegNext(source.vlWen) + sink.addr := RegEnable(source.addr, source.wen) + } + intScheduler.io.fromTop.hartId := io.fromTop.hartId + intScheduler.io.fromCtrlBlock.flush := ctrlBlock.io.toIssueBlock.flush + intScheduler.io.fromDispatch.uops <> ctrlBlock.io.toIssueBlock.intUops + intScheduler.io.intWriteBack := wbDataPath.io.toIntPreg + intScheduler.io.fpWriteBack := 0.U.asTypeOf(intScheduler.io.fpWriteBack) + intScheduler.io.vfWriteBack := 0.U.asTypeOf(intScheduler.io.vfWriteBack) + intScheduler.io.v0WriteBack := 0.U.asTypeOf(intScheduler.io.v0WriteBack) + intScheduler.io.vlWriteBack := 0.U.asTypeOf(intScheduler.io.vlWriteBack) + intScheduler.io.intWriteBackDelayed := intWriteBackDelayed + intScheduler.io.fpWriteBackDelayed := 0.U.asTypeOf(intScheduler.io.fpWriteBackDelayed) + intScheduler.io.vfWriteBackDelayed := 0.U.asTypeOf(intScheduler.io.vfWriteBackDelayed) + intScheduler.io.v0WriteBackDelayed := 0.U.asTypeOf(intScheduler.io.v0WriteBackDelayed) + intScheduler.io.vlWriteBackDelayed := 0.U.asTypeOf(intScheduler.io.vlWriteBackDelayed) + intScheduler.io.fromDataPath.resp := dataPath.io.toIntIQ + intScheduler.io.fromSchedulers.wakeupVec.foreach(wakeup => wakeup := iqWakeUpMappedBundle(wakeup.bits.exuIdx)) + intScheduler.io.fromSchedulers.wakeupVecDelayed.foreach { wakeup => + wakeup := iqWakeUpMappedBundleDelayed(wakeup.bits.exuIdx) + } + intScheduler.io.fromDataPath.og0Cancel := og0Cancel + intScheduler.io.fromDataPath.og1Cancel := og1Cancel + intScheduler.io.ldCancel := io.mem.ldCancel + intScheduler.io.fromDataPath.replaceRCIdx.get := dataPath.io.toWakeupQueueRCIdx.take(params.getIntExuRCWriteSize) + intScheduler.io.vlWriteBackInfo.vlFromIntIsZero := false.B + intScheduler.io.vlWriteBackInfo.vlFromIntIsVlmax := false.B + intScheduler.io.vlWriteBackInfo.vlFromVfIsZero := false.B + intScheduler.io.vlWriteBackInfo.vlFromVfIsVlmax := false.B + + fpScheduler.io.fromTop.hartId := io.fromTop.hartId + fpScheduler.io.fromCtrlBlock.flush := ctrlBlock.io.toIssueBlock.flush + fpScheduler.io.fromDispatch.uops <> ctrlBlock.io.toIssueBlock.fpUops + fpScheduler.io.intWriteBack := 0.U.asTypeOf(fpScheduler.io.intWriteBack) + fpScheduler.io.fpWriteBack := wbDataPath.io.toFpPreg + fpScheduler.io.vfWriteBack := 0.U.asTypeOf(fpScheduler.io.vfWriteBack) + fpScheduler.io.v0WriteBack := 0.U.asTypeOf(fpScheduler.io.v0WriteBack) + fpScheduler.io.vlWriteBack := 0.U.asTypeOf(fpScheduler.io.vlWriteBack) + fpScheduler.io.intWriteBackDelayed := 0.U.asTypeOf(intWriteBackDelayed) + fpScheduler.io.fpWriteBackDelayed := fpWriteBackDelayed + fpScheduler.io.vfWriteBackDelayed := 0.U.asTypeOf(intScheduler.io.vfWriteBackDelayed) + fpScheduler.io.v0WriteBackDelayed := 0.U.asTypeOf(intScheduler.io.v0WriteBackDelayed) + fpScheduler.io.vlWriteBackDelayed := 0.U.asTypeOf(intScheduler.io.vlWriteBackDelayed) + fpScheduler.io.fromDataPath.resp := dataPath.io.toFpIQ + fpScheduler.io.fromSchedulers.wakeupVec.foreach(wakeup => wakeup := iqWakeUpMappedBundle(wakeup.bits.exuIdx)) + fpScheduler.io.fromSchedulers.wakeupVecDelayed.foreach { wakeup => + wakeup := iqWakeUpMappedBundleDelayed(wakeup.bits.exuIdx) + } + fpScheduler.io.fromDataPath.og0Cancel := og0Cancel + fpScheduler.io.fromDataPath.og1Cancel := og1Cancel + fpScheduler.io.ldCancel := io.mem.ldCancel + fpScheduler.io.vlWriteBackInfo.vlFromIntIsZero := false.B + fpScheduler.io.vlWriteBackInfo.vlFromIntIsVlmax := false.B + fpScheduler.io.vlWriteBackInfo.vlFromVfIsZero := false.B + fpScheduler.io.vlWriteBackInfo.vlFromVfIsVlmax := false.B + + memScheduler.io.fromTop.hartId := io.fromTop.hartId + memScheduler.io.fromCtrlBlock.flush := ctrlBlock.io.toIssueBlock.flush + memScheduler.io.fromDispatch.uops <> ctrlBlock.io.toIssueBlock.memUops + memScheduler.io.intWriteBack := wbDataPath.io.toIntPreg + memScheduler.io.fpWriteBack := wbDataPath.io.toFpPreg + memScheduler.io.vfWriteBack := wbDataPath.io.toVfPreg + memScheduler.io.v0WriteBack := wbDataPath.io.toV0Preg + memScheduler.io.vlWriteBack := wbDataPath.io.toVlPreg + memScheduler.io.intWriteBackDelayed := intWriteBackDelayed + memScheduler.io.fpWriteBackDelayed := fpWriteBackDelayed + memScheduler.io.vfWriteBackDelayed := vfWriteBackDelayed + memScheduler.io.v0WriteBackDelayed := v0WriteBackDelayed + memScheduler.io.vlWriteBackDelayed := vlWriteBackDelayed + memScheduler.io.fromMem.get.scommit := io.mem.sqDeq + memScheduler.io.fromMem.get.lcommit := io.mem.lqDeq + memScheduler.io.fromMem.get.wakeup := io.mem.wakeup + memScheduler.io.fromMem.get.sqDeqPtr := io.mem.sqDeqPtr + memScheduler.io.fromMem.get.lqDeqPtr := io.mem.lqDeqPtr + memScheduler.io.fromMem.get.sqCancelCnt := io.mem.sqCancelCnt + memScheduler.io.fromMem.get.lqCancelCnt := io.mem.lqCancelCnt + memScheduler.io.fromMem.get.stIssuePtr := io.mem.stIssuePtr + require(memScheduler.io.fromMem.get.memWaitUpdateReq.robIdx.length == io.mem.stIn.length) + memScheduler.io.fromMem.get.memWaitUpdateReq.robIdx.zip(io.mem.stIn).foreach { case (sink, source) => + sink.valid := source.valid + sink.bits := source.bits.robIdx + } + memScheduler.io.fromMem.get.memWaitUpdateReq.sqIdx := DontCare // TODO + memScheduler.io.fromDataPath.resp := dataPath.io.toMemIQ + memScheduler.io.fromMem.get.ldaFeedback := io.mem.ldaIqFeedback + memScheduler.io.fromMem.get.staFeedback := io.mem.staIqFeedback + memScheduler.io.fromMem.get.hyuFeedback := io.mem.hyuIqFeedback + memScheduler.io.fromMem.get.vstuFeedback := io.mem.vstuIqFeedback + memScheduler.io.fromMem.get.vlduFeedback := io.mem.vlduIqFeedback + memScheduler.io.fromSchedulers.wakeupVec.foreach(wakeup => wakeup := iqWakeUpMappedBundle(wakeup.bits.exuIdx)) + memScheduler.io.fromSchedulers.wakeupVecDelayed.foreach { wakeup => + wakeup := iqWakeUpMappedBundleDelayed(wakeup.bits.exuIdx) + } + memScheduler.io.fromDataPath.og0Cancel := og0Cancel + memScheduler.io.fromDataPath.og1Cancel := og1Cancel + memScheduler.io.ldCancel := io.mem.ldCancel + memScheduler.io.fromDataPath.replaceRCIdx.get := dataPath.io.toWakeupQueueRCIdx.takeRight(params.getMemExuRCWriteSize) + memScheduler.io.vlWriteBackInfo.vlFromIntIsZero := vlFromIntIsZero + memScheduler.io.vlWriteBackInfo.vlFromIntIsVlmax := vlFromIntIsVlmax + memScheduler.io.vlWriteBackInfo.vlFromVfIsZero := vlFromVfIsZero + memScheduler.io.vlWriteBackInfo.vlFromVfIsVlmax := vlFromVfIsVlmax + memScheduler.io.fromOg2Resp.get := og2ForVector.io.toMemIQOg2Resp + + vfScheduler.io.fromTop.hartId := io.fromTop.hartId + vfScheduler.io.fromCtrlBlock.flush := ctrlBlock.io.toIssueBlock.flush + vfScheduler.io.fromDispatch.uops <> ctrlBlock.io.toIssueBlock.vfUops + vfScheduler.io.intWriteBack := 0.U.asTypeOf(vfScheduler.io.intWriteBack) + vfScheduler.io.fpWriteBack := 0.U.asTypeOf(vfScheduler.io.fpWriteBack) + vfScheduler.io.vfWriteBack := wbDataPath.io.toVfPreg + vfScheduler.io.v0WriteBack := wbDataPath.io.toV0Preg + vfScheduler.io.vlWriteBack := wbDataPath.io.toVlPreg + vfScheduler.io.intWriteBackDelayed := 0.U.asTypeOf(intWriteBackDelayed) + vfScheduler.io.fpWriteBackDelayed := 0.U.asTypeOf(fpWriteBackDelayed) + vfScheduler.io.vfWriteBackDelayed := vfWriteBackDelayed + vfScheduler.io.v0WriteBackDelayed := v0WriteBackDelayed + vfScheduler.io.vlWriteBackDelayed := vlWriteBackDelayed + vfScheduler.io.fromDataPath.resp := dataPath.io.toVfIQ + vfScheduler.io.fromSchedulers.wakeupVec.foreach(wakeup => wakeup := iqWakeUpMappedBundle(wakeup.bits.exuIdx)) + vfScheduler.io.fromSchedulers.wakeupVecDelayed.foreach { wakeup => + wakeup := iqWakeUpMappedBundleDelayed(wakeup.bits.exuIdx) + } + vfScheduler.io.fromDataPath.og0Cancel := og0Cancel + vfScheduler.io.fromDataPath.og1Cancel := og1Cancel + vfScheduler.io.ldCancel := io.mem.ldCancel + vfScheduler.io.vlWriteBackInfo.vlFromIntIsZero := vlFromIntIsZero + vfScheduler.io.vlWriteBackInfo.vlFromIntIsVlmax := vlFromIntIsVlmax + vfScheduler.io.vlWriteBackInfo.vlFromVfIsZero := vlFromVfIsZero + vfScheduler.io.vlWriteBackInfo.vlFromVfIsVlmax := vlFromVfIsVlmax + vfScheduler.io.fromOg2Resp.get := og2ForVector.io.toVfIQOg2Resp + + dataPath.io.hartId := io.fromTop.hartId + dataPath.io.flush := ctrlBlock.io.toDataPath.flush + + dataPath.io.fromIntIQ <> intScheduler.io.toDataPathAfterDelay + dataPath.io.fromFpIQ <> fpScheduler.io.toDataPathAfterDelay + dataPath.io.fromVfIQ <> vfScheduler.io.toDataPathAfterDelay + dataPath.io.fromMemIQ <> memScheduler.io.toDataPathAfterDelay + + dataPath.io.ldCancel := io.mem.ldCancel + + println( + s"[Backend] wbDataPath.io.toIntPreg: ${wbDataPath.io.toIntPreg.size}, dataPath.io.fromIntWb: ${dataPath.io.fromIntWb.size}" + ) + println( + s"[Backend] wbDataPath.io.toVfPreg: ${wbDataPath.io.toVfPreg.size}, dataPath.io.fromFpWb: ${dataPath.io.fromVfWb.size}" + ) + dataPath.io.fromIntWb := wbDataPath.io.toIntPreg + dataPath.io.fromFpWb := wbDataPath.io.toFpPreg + dataPath.io.fromVfWb := wbDataPath.io.toVfPreg + dataPath.io.fromV0Wb := wbDataPath.io.toV0Preg + dataPath.io.fromVlWb := wbDataPath.io.toVlPreg + dataPath.io.diffVlRat.foreach(_ := ctrlBlock.io.diff_vl_rat.get) + dataPath.io.fromBypassNetwork := bypassNetwork.io.toDataPath + dataPath.io.fromVecExcpMod.r := vecExcpMod.o.toVPRF.r + dataPath.io.fromVecExcpMod.w := vecExcpMod.o.toVPRF.w + dataPath.io.topDownInfo.replayAllocate := DelayN(io.topDownInfo.replayAllocate, 2) + dataPath.io.topDownInfo.sqFull := DelayN(io.topDownInfo.sqFull, 2) + dataPath.io.topDownInfo.sbFull := DelayN(io.topDownInfo.sbFull, 2) + dataPath.io.topDownInfo.l1Miss := RegNext(io.topDownInfo.l1Miss) + dataPath.io.topDownInfo.l2TopMiss.l2Miss := io.topDownInfo.l2TopMiss.l2Miss + dataPath.io.topDownInfo.l2TopMiss.l3Miss := io.topDownInfo.l2TopMiss.l3Miss + + og2ForVector.io.flush := ctrlBlock.io.toDataPath.flush + og2ForVector.io.ldCancel := io.mem.ldCancel + og2ForVector.io.fromOg1VfArith <> dataPath.io.toVecExu + og2ForVector.io.fromOg1VecMem.zip( + dataPath.io.toMemExu.zip(params.memSchdParams.get.issueBlockParams).filter(_._2.needOg2Resp).map(_._1) + ) + .foreach { + case (og1Mem, datapathMem) => og1Mem <> datapathMem + } + og2ForVector.io.fromOg1ImmInfo := dataPath.io.og1ImmInfo.zip(params.allExuParams).filter(_._2.needOg2).map(_._1) + + println( + s"[Backend] BypassNetwork OG1 Mem Size: ${bypassNetwork.io.fromDataPath.mem.zip(params.memSchdParams.get.issueBlockParams).filterNot(_._2.needOg2Resp).size}" + ) + println( + s"[Backend] BypassNetwork OG2 Mem Size: ${bypassNetwork.io.fromDataPath.mem.zip(params.memSchdParams.get.issueBlockParams).filter(_._2.needOg2Resp).size}" + ) + println( + s"[Backend] bypassNetwork.io.fromDataPath.mem: ${bypassNetwork.io.fromDataPath.mem.size}, dataPath.io.toMemExu: ${dataPath.io.toMemExu.size}" + ) + bypassNetwork.io.fromDataPath.int <> dataPath.io.toIntExu + bypassNetwork.io.fromDataPath.fp <> dataPath.io.toFpExu + bypassNetwork.io.fromDataPath.vf <> og2ForVector.io.toVfArithExu + bypassNetwork.io.fromDataPath.mem.lazyZip(params.memSchdParams.get.issueBlockParams).lazyZip( + dataPath.io.toMemExu + ).filterNot(_._2.needOg2Resp) + .map(x => (x._1, x._3)).foreach { + case (bypassMem, datapathMem) => bypassMem <> datapathMem + } + bypassNetwork.io.fromDataPath.mem.zip(params.memSchdParams.get.issueBlockParams).filter(_._2.needOg2Resp).map(_._1) + .zip(og2ForVector.io.toVecMemExu).foreach { + case (bypassMem, og2Mem) => bypassMem <> og2Mem + } + bypassNetwork.io.fromDataPath.immInfo := dataPath.io.og1ImmInfo + bypassNetwork.io.fromDataPath.immInfo.zip(params.allExuParams).filter(_._2.needOg2).map(_._1) + .zip(og2ForVector.io.toBypassNetworkImmInfo).foreach { + case (immInfo, og2ImmInfo) => immInfo := og2ImmInfo + } + bypassNetwork.io.fromDataPath.rcData := dataPath.io.toBypassNetworkRCData + bypassNetwork.io.fromExus.connectExuOutput(_.int)(intExuBlock.io.out) + bypassNetwork.io.fromExus.connectExuOutput(_.fp)(fpExuBlock.io.out) + bypassNetwork.io.fromExus.connectExuOutput(_.vf)(vfExuBlock.io.out) + + require( + bypassNetwork.io.fromExus.mem.flatten.size == io.mem.writeBack.size, + s"bypassNetwork.io.fromExus.mem.flatten.size(${bypassNetwork.io.fromExus.mem.flatten.size}: ${bypassNetwork.io.fromExus.mem.map(_.size)}, " + + s"io.mem.writeback(${io.mem.writeBack.size})" + ) + bypassNetwork.io.fromExus.mem.flatten.zip(io.mem.writeBack).foreach { case (sink, source) => + sink.valid := source.valid + sink.bits.intWen := source.bits.uop.rfWen && source.bits.isFromLoadUnit + sink.bits.pdest := source.bits.uop.pdest + sink.bits.data := source.bits.data + } + + intExuBlock.io.flush := ctrlBlock.io.toExuBlock.flush + for (i <- 0 until intExuBlock.io.in.length) { + for (j <- 0 until intExuBlock.io.in(i).length) { + val shouldLdCancel = LoadShouldCancel(bypassNetwork.io.toExus.int(i)(j).bits.loadDependency, io.mem.ldCancel) + NewPipelineConnect( + bypassNetwork.io.toExus.int(i)(j), + intExuBlock.io.in(i)(j), + intExuBlock.io.in(i)(j).fire, + Mux( + bypassNetwork.io.toExus.int(i)(j).fire, + bypassNetwork.io.toExus.int(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) || shouldLdCancel, + intExuBlock.io.in(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) + ), + Option("bypassNetwork2intExuBlock") + ) + } + } + + ctrlBlock.io.toDataPath.pcToDataPathIO <> dataPath.io.fromPcTargetMem + + private val csrin = intExuBlock.io.csrin.get + csrin.hartId := io.fromTop.hartId + csrin.msiInfo.valid := RegNext(io.fromTop.msiInfo.valid) + csrin.msiInfo.bits := RegEnable(io.fromTop.msiInfo.bits, io.fromTop.msiInfo.valid) + csrin.teemsiInfo zip io.fromTop.teemsiInfo foreach { case (csrin_teemsiInfo, fromTop_teemsiInfo) => + csrin_teemsiInfo.valid := RegNext(fromTop_teemsiInfo.valid) + csrin_teemsiInfo.bits := RegEnable(fromTop_teemsiInfo.bits, fromTop_teemsiInfo.valid) + } + csrin.clintTime.valid := RegNext(io.fromTop.clintTime.valid) + csrin.clintTime.bits := RegEnable(io.fromTop.clintTime.bits, io.fromTop.clintTime.valid) + csrin.l2FlushDone := RegNext(io.fromTop.l2FlushDone) + csrin.reset_mtvec.foreach(_ := io.fromTop.reset_mtvec.get) + csrin.trapInstInfo := ctrlBlock.io.toCSR.trapInstInfo + csrin.fromVecExcpMod.busy := vecExcpMod.o.status.busy + csrin.criticalErrorState := backendCriticalError + + private val csrio = intExuBlock.io.csrio.get + io.commands <> csrio.commands + io.timeRequest <> csrio.timeRequest + csrio.hartId := io.fromTop.hartId + csrio.fpu.fflags := ctrlBlock.io.robio.csr.fflags + csrio.fpu.isIllegal := false.B // Todo: remove it + csrio.fpu.dirty_fs := ctrlBlock.io.robio.csr.dirty_fs + csrio.vpu <> WireDefault(0.U.asTypeOf(csrio.vpu)) // Todo + + val fromIntExuVsetVType = intExuBlock.io.vtype.getOrElse(0.U.asTypeOf((Valid(new VType)))) + val fromVfExuVsetVType = vfExuBlock.io.vtype.getOrElse(0.U.asTypeOf((Valid(new VType)))) + val fromVsetVType = Mux(fromIntExuVsetVType.valid, fromIntExuVsetVType.bits, fromVfExuVsetVType.bits) + val vsetvlVType = + RegEnable(fromVsetVType, 0.U.asTypeOf(new VType), fromIntExuVsetVType.valid || fromVfExuVsetVType.valid) + ctrlBlock.io.toDecode.vsetvlVType := vsetvlVType + + val commitVType = ctrlBlock.io.robio.commitVType.vtype + val hasVsetvl = ctrlBlock.io.robio.commitVType.hasVsetvl + val vtype = VType.toVtypeStruct(Mux(hasVsetvl, vsetvlVType, commitVType.bits)).asUInt + + // csr not store the value of vl, so when using difftest we assign the value of vl to debugVl + val debugVl_s0 = WireInit(UInt(VlData().dataWidth.W), 0.U) + val debugVl_s1 = WireInit(UInt(VlData().dataWidth.W), 0.U) + debugVl_s0 := dataPath.io.diffVl.getOrElse(0.U.asTypeOf(UInt(VlData().dataWidth.W))) + debugVl_s1 := RegNext(debugVl_s0) + csrio.vpu.set_vxsat := ctrlBlock.io.robio.csr.vxsat + csrio.vpu.set_vstart.valid := ctrlBlock.io.robio.csr.vstart.valid + csrio.vpu.set_vstart.bits := ctrlBlock.io.robio.csr.vstart.bits + ctrlBlock.io.toDecode.vstart := csrio.vpu.vstart + //Todo here need change design + csrio.vpu.set_vtype.valid := commitVType.valid + csrio.vpu.set_vtype.bits := ZeroExt(vtype, XLEN) + csrio.vpu.vl := ZeroExt(debugVl_s1, XLEN) + csrio.vpu.dirty_vs := ctrlBlock.io.robio.csr.dirty_vs + csrio.exception := ctrlBlock.io.robio.exception + csrio.robDeqPtr := ctrlBlock.io.robio.robDeqPtr + csrio.memExceptionVAddr := io.mem.exceptionAddr.vaddr + csrio.memExceptionGPAddr := io.mem.exceptionAddr.gpaddr + csrio.memExceptionIsForVSnonLeafPTE := io.mem.exceptionAddr.isForVSnonLeafPTE + csrio.externalInterrupt := RegNext(io.fromTop.externalInterrupt) + csrio.perf <> io.perf + csrio.perf.retiredInstr <> ctrlBlock.io.robio.csr.perfinfo.retiredInstr + csrio.perf.ctrlInfo <> ctrlBlock.io.perfInfo.ctrlInfo + private val fenceio = intExuBlock.io.fenceio.get + io.fenceio <> fenceio + io.fenceio.sbuffer.flushSb := fenceio.sbuffer.flushSb || csrio.commandFlushStores + csrio.commandStoresEmpty := io.fenceio.sbuffer.sbIsEmpty + + // to fpExuBlock + fpExuBlock.io.flush := ctrlBlock.io.toExuBlock.flush + for (i <- 0 until fpExuBlock.io.in.length) { + for (j <- 0 until fpExuBlock.io.in(i).length) { + val shouldLdCancel = LoadShouldCancel(bypassNetwork.io.toExus.fp(i)(j).bits.loadDependency, io.mem.ldCancel) + NewPipelineConnect( + bypassNetwork.io.toExus.fp(i)(j), + fpExuBlock.io.in(i)(j), + fpExuBlock.io.in(i)(j).fire, + Mux( + bypassNetwork.io.toExus.fp(i)(j).fire, + bypassNetwork.io.toExus.fp(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) || shouldLdCancel, + fpExuBlock.io.in(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) + ), + Option("bypassNetwork2fpExuBlock") + ) + } + } + + vfExuBlock.io.flush := ctrlBlock.io.toExuBlock.flush + for (i <- 0 until vfExuBlock.io.in.size) { + for (j <- 0 until vfExuBlock.io.in(i).size) { + val shouldLdCancel = LoadShouldCancel(bypassNetwork.io.toExus.vf(i)(j).bits.loadDependency, io.mem.ldCancel) + NewPipelineConnect( + bypassNetwork.io.toExus.vf(i)(j), + vfExuBlock.io.in(i)(j), + vfExuBlock.io.in(i)(j).fire, + Mux( + bypassNetwork.io.toExus.vf(i)(j).fire, + bypassNetwork.io.toExus.vf(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) || shouldLdCancel, + vfExuBlock.io.in(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) + ), + Option("bypassNetwork2vfExuBlock") + ) + + } + } + + intExuBlock.io.frm.foreach(_ := csrio.fpu.frm) + fpExuBlock.io.frm.foreach(_ := csrio.fpu.frm) + fpExuBlock.io.vxrm.foreach(_ := csrio.vpu.vxrm) + vfExuBlock.io.frm.foreach(_ := csrio.fpu.frm) + vfExuBlock.io.vxrm.foreach(_ := csrio.vpu.vxrm) + + wbDataPath.io.flush := ctrlBlock.io.redirect + wbDataPath.io.fromTop.hartId := io.fromTop.hartId + wbDataPath.io.fromIntExu <> intExuBlock.io.out + wbDataPath.io.fromFpExu <> fpExuBlock.io.out + wbDataPath.io.fromVfExu <> vfExuBlock.io.out + wbDataPath.io.fromMemExu.flatten.zip(io.mem.writeBack).foreach { case (sink, source) => + sink.valid := source.valid + source.ready := sink.ready + sink.bits.data := VecInit(Seq.fill(sink.bits.params.wbPathNum)(source.bits.data)) + sink.bits.pdest := source.bits.uop.pdest + sink.bits.robIdx := source.bits.uop.robIdx + sink.bits.intWen.foreach(_ := source.bits.uop.rfWen) + sink.bits.fpWen.foreach(_ := source.bits.uop.fpWen) + sink.bits.vecWen.foreach(_ := source.bits.uop.vecWen) + sink.bits.v0Wen.foreach(_ := source.bits.uop.v0Wen) + sink.bits.vlWen.foreach(_ := source.bits.uop.vlWen) + sink.bits.exceptionVec.foreach(_ := source.bits.uop.exceptionVec) + sink.bits.flushPipe.foreach(_ := source.bits.uop.flushPipe) + sink.bits.replay.foreach(_ := source.bits.uop.replayInst) + sink.bits.debug := source.bits.debug + sink.bits.debugInfo := source.bits.uop.debugInfo + sink.bits.debug_seqNum := source.bits.uop.debug_seqNum + sink.bits.lqIdx.foreach(_ := source.bits.uop.lqIdx) + sink.bits.sqIdx.foreach(_ := source.bits.uop.sqIdx) + sink.bits.predecodeInfo.foreach(_ := source.bits.uop.preDecodeInfo) + sink.bits.vls.foreach { x => + x.vdIdx := source.bits.vdIdx.get + x.vdIdxInField := source.bits.vdIdxInField.get + x.vpu := source.bits.uop.vpu + x.oldVdPsrc := source.bits.uop.psrc(2) + x.isIndexed := VlduType.isIndexed(source.bits.uop.fuOpType) + x.isMasked := VlduType.isMasked(source.bits.uop.fuOpType) + x.isStrided := VlduType.isStrided(source.bits.uop.fuOpType) + x.isWhole := VlduType.isWhole(source.bits.uop.fuOpType) + x.isVecLoad := VlduType.isVecLd(source.bits.uop.fuOpType) + x.isVlm := VlduType.isMasked(source.bits.uop.fuOpType) && VlduType.isVecLd(source.bits.uop.fuOpType) + } + sink.bits.trigger.foreach(_ := source.bits.uop.trigger) + } + wbDataPath.io.fromCSR.vstart := csrio.vpu.vstart + + vecExcpMod.i.fromExceptionGen := ctrlBlock.io.toVecExcpMod.excpInfo + vecExcpMod.i.fromRab.logicPhyRegMap := ctrlBlock.io.toVecExcpMod.logicPhyRegMap + vecExcpMod.i.fromRat := ctrlBlock.io.toVecExcpMod.ratOldPest + vecExcpMod.i.fromVprf := dataPath.io.toVecExcpMod + + // to mem + private val memIssueParams = params.memSchdParams.get.issueBlockParams + private val memExuBlocksHasLDU = memIssueParams.map(_.exuBlockParams.map(x => x.hasLoadFu || x.hasHyldaFu)) + private val memExuBlocksHasVecLoad = memIssueParams.map(_.exuBlockParams.map(x => x.hasVLoadFu)) + println(s"[Backend] memExuBlocksHasLDU: $memExuBlocksHasLDU") + println(s"[Backend] memExuBlocksHasVecLoad: $memExuBlocksHasVecLoad") + + private val toMem = Wire(bypassNetwork.io.toExus.mem.cloneType) + for (i <- toMem.indices) { + for (j <- toMem(i).indices) { + val shouldLdCancel = LoadShouldCancel(bypassNetwork.io.toExus.mem(i)(j).bits.loadDependency, io.mem.ldCancel) + val needIssueTimeout = memExuBlocksHasVecLoad(i)(j) + val olderUopComing = + if (needIssueTimeout && bypassNetwork.io.toExus.mem(i)(j).bits.lqIdx.nonEmpty) + bypassNetwork.io.toExus.mem(i)(j).valid && toMem(i)(j).valid && !toMem(i)(j).fire && + (bypassNetwork.io.toExus.mem(i)(j).bits.lqIdx.get < toMem(i)(j).bits.lqIdx.get || bypassNetwork.io.toExus.mem( + i + )(j).bits.sqIdx.get < toMem(i)(j).bits.sqIdx.get) // Older inst come from iq. + else + false.B + + val issueTimeout = + if (needIssueTimeout) + Counter(0 until 14, toMem(i)(j).valid && !toMem(i)(j).fire, bypassNetwork.io.toExus.mem(i)(j).fire)._2 + else + false.B + + if (memScheduler.io.loadFinalIssueResp(i).nonEmpty && memExuBlocksHasLDU(i)(j)) { + memScheduler.io.loadFinalIssueResp(i)(j).valid := issueTimeout || olderUopComing + memScheduler.io.loadFinalIssueResp(i)(j).bits.fuType := toMem(i)(j).bits.fuType + memScheduler.io.loadFinalIssueResp(i)(j).bits.resp := RespType.block + memScheduler.io.loadFinalIssueResp(i)(j).bits.robIdx := toMem(i)(j).bits.robIdx + memScheduler.io.loadFinalIssueResp(i)(j).bits.uopIdx.foreach(_ := toMem(i)(j).bits.vpu.get.vuopIdx) + memScheduler.io.loadFinalIssueResp(i)(j).bits.sqIdx.foreach(_ := toMem(i)(j).bits.sqIdx.get) + memScheduler.io.loadFinalIssueResp(i)(j).bits.lqIdx.foreach(_ := toMem(i)(j).bits.lqIdx.get) + } + + if (memScheduler.io.vecLoadFinalIssueResp(i).nonEmpty && memExuBlocksHasVecLoad(i)(j)) { + memScheduler.io.vecLoadFinalIssueResp(i)(j).valid := issueTimeout || olderUopComing + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.fuType := toMem(i)(j).bits.fuType + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.resp := RespType.block + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.robIdx := toMem(i)(j).bits.robIdx + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.uopIdx.foreach(_ := toMem(i)(j).bits.vpu.get.vuopIdx) + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.sqIdx.foreach(_ := toMem(i)(j).bits.sqIdx.get) + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.lqIdx.foreach(_ := toMem(i)(j).bits.lqIdx.get) + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.isVecPartReplay.foreach(_ := toMem(i)( + j + ).bits.isVecPartReplay.get) + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.vecReplayMask.foreach(_ := toMem(i)(j).bits.vecReplayMask.get) + memScheduler.io.vecLoadFinalIssueResp(i)(j).bits.vecReplayMbIdx.foreach(_ := toMem(i)(j).bits.vecReplayMbIdx.get) + } + + NewPipelineConnect( + bypassNetwork.io.toExus.mem(i)(j), + toMem(i)(j), + toMem(i)(j).fire, + Mux( + bypassNetwork.io.toExus.mem(i)(j).fire, + bypassNetwork.io.toExus.mem(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) || shouldLdCancel, + toMem(i)(j).bits.robIdx.needFlush(ctrlBlock.io.toExuBlock.flush) || issueTimeout + ), + Option("bypassNetwork2toMemExus"), + isOlder = olderUopComing + ) + if (memScheduler.io.vecLoadIssueResp(i).nonEmpty && memExuBlocksHasVecLoad(i)(j)) { + memScheduler.io.vecLoadIssueResp(i)(j) match { + case resp => + resp.valid := toMem(i)(j).fire && VlduType.isVecLd(toMem(i)(j).bits.fuOpType) + resp.bits.fuType := toMem(i)(j).bits.fuType + resp.bits.robIdx := toMem(i)(j).bits.robIdx + resp.bits.uopIdx.get := toMem(i)(j).bits.vpu.get.vuopIdx + resp.bits.sqIdx.get := toMem(i)(j).bits.sqIdx.get + resp.bits.lqIdx.get := toMem(i)(j).bits.lqIdx.get + resp.bits.resp := RespType.success + resp.bits.isVecPartReplay.foreach(_ := false.B) + resp.bits.vecReplayMask.foreach(_ := 0.U) + resp.bits.vecReplayMbIdx.foreach(_ := 0.U) + } + if (backendParams.debugEn) { + dontTouch(memScheduler.io.vecLoadIssueResp(i)(j)) + } + } + } + } + + io.mem.redirect := ctrlBlock.io.redirect + io.mem.issueUops.zip(toMem.flatten).foreach { case (sink, source) => + val enableMdp = Constantin.createRecord("EnableMdp", true) + sink.valid := source.valid + source.ready := sink.ready + sink.bits.iqIdx := source.bits.iqIdx + sink.bits.isFirstIssue := source.bits.isFirstIssue + sink.bits.isVecPartReplay.foreach(_ := source.bits.isVecPartReplay.get) + sink.bits.vecReplayMask.foreach(_ := source.bits.vecReplayMask.get) + sink.bits.vecReplayMbIdx.foreach(_ := source.bits.vecReplayMbIdx.get) + sink.bits.uop := 0.U.asTypeOf(sink.bits.uop) + sink.bits.src := 0.U.asTypeOf(sink.bits.src) + sink.bits.src.zip(source.bits.src).foreach { case (l, r) => l := r } + sink.bits.uop.fuType := source.bits.fuType + sink.bits.uop.fuOpType := source.bits.fuOpType + sink.bits.uop.imm := source.bits.imm + sink.bits.uop.robIdx := source.bits.robIdx + sink.bits.uop.pdest := source.bits.pdest + sink.bits.uop.rfWen := source.bits.rfWen.getOrElse(false.B) + sink.bits.uop.fpWen := source.bits.fpWen.getOrElse(false.B) + sink.bits.uop.vecWen := source.bits.vecWen.getOrElse(false.B) + sink.bits.uop.v0Wen := source.bits.v0Wen.getOrElse(false.B) + sink.bits.uop.vlWen := source.bits.vlWen.getOrElse(false.B) + sink.bits.uop.flushPipe := source.bits.flushPipe.getOrElse(false.B) + sink.bits.uop.pc := source.bits.pc.getOrElse(0.U) + (source.bits.ftqOffset.getOrElse(0.U) << instOffsetBits) + sink.bits.uop.loadWaitBit := Mux(enableMdp, source.bits.loadWaitBit.getOrElse(false.B), false.B) + sink.bits.uop.waitForRobIdx := Mux( + enableMdp, + source.bits.waitForRobIdx.getOrElse(0.U.asTypeOf(new RobPtr)), + 0.U.asTypeOf(new RobPtr) + ) + sink.bits.uop.storeSetHit := Mux(enableMdp, source.bits.storeSetHit.getOrElse(false.B), false.B) + sink.bits.uop.loadWaitStrict := Mux(enableMdp, source.bits.loadWaitStrict.getOrElse(false.B), false.B) + sink.bits.uop.ssid := Mux(enableMdp, source.bits.ssid.getOrElse(0.U(SSIDWidth.W)), 0.U(SSIDWidth.W)) + sink.bits.uop.lqIdx := source.bits.lqIdx.getOrElse(0.U.asTypeOf(new LqPtr)) + sink.bits.uop.sqIdx := source.bits.sqIdx.getOrElse(0.U.asTypeOf(new SqPtr)) + sink.bits.uop.ftqPtr := source.bits.ftqIdx.getOrElse(0.U.asTypeOf(new FtqPtr)) + sink.bits.uop.ftqOffset := source.bits.ftqOffset.getOrElse(0.U) + sink.bits.uop.debugInfo := source.bits.perfDebugInfo + sink.bits.uop.debug_seqNum := source.bits.debug_seqNum + sink.bits.uop.vpu := source.bits.vpu.getOrElse(0.U.asTypeOf(new VPUCtrlSignals)) + sink.bits.uop.preDecodeInfo := source.bits.preDecode.getOrElse(0.U.asTypeOf(new PreDecodeInfo)) + sink.bits.uop.numLsElem := source.bits.numLsElem.getOrElse(0.U) // Todo: remove this bundle, keep only the one below + sink.bits.flowNum.foreach(_ := source.bits.numLsElem.get) + } + io.mem.loadFastMatch := memScheduler.io.toMem.get.loadFastMatch.map(_.fastMatch) + io.mem.loadFastImm := memScheduler.io.toMem.get.loadFastMatch.map(_.fastImm) + io.mem.tlbCsr := csrio.tlb + io.mem.csrCtrl := csrio.customCtrl + io.mem.sfence := fenceio.sfence + io.mem.isStoreException := CommitType.lsInstIsStore(ctrlBlock.io.robio.exception.bits.commitType) + io.mem.isVlsException := ctrlBlock.io.robio.exception.bits.vls + + io.mem.storePcRead.zipWithIndex.foreach { case (storePcRead, i) => + storePcRead := ctrlBlock.io.memStPcRead(i).data + ctrlBlock.io.memStPcRead(i).valid := io.mem.issueSta(i).valid + ctrlBlock.io.memStPcRead(i).ptr := io.mem.issueSta(i).bits.uop.ftqPtr + ctrlBlock.io.memStPcRead(i).offset := io.mem.issueSta(i).bits.uop.ftqOffset + } + + io.mem.hyuPcRead.zipWithIndex.foreach({ case (hyuPcRead, i) => + hyuPcRead := ctrlBlock.io.memHyPcRead(i).data + ctrlBlock.io.memHyPcRead(i).valid := io.mem.issueHylda(i).valid + ctrlBlock.io.memHyPcRead(i).ptr := io.mem.issueHylda(i).bits.uop.ftqPtr + ctrlBlock.io.memHyPcRead(i).offset := io.mem.issueHylda(i).bits.uop.ftqOffset + }) + + ctrlBlock.io.robio.robHeadLsIssue := io.mem.issueUops.map(deq => + deq.fire && deq.bits.uop.robIdx === ctrlBlock.io.robio.robDeqPtr + ).reduce(_ || _) + + // mem io + io.mem.robLsqIO <> ctrlBlock.io.robio.lsq + io.mem.storeDebugInfo <> ctrlBlock.io.robio.storeDebugInfo + + io.frontendSfence := fenceio.sfence + io.frontendTlbCsr := csrio.tlb + io.frontendCsrCtrl := csrio.customCtrl + + io.tlb <> csrio.tlb + + io.csrCustomCtrl := csrio.customCtrl + + io.toTop.cpuHalted := ctrlBlock.io.toTop.cpuHalt + + io.traceCoreInterface <> ctrlBlock.io.traceCoreInterface + + io.debugTopDown.fromRob := ctrlBlock.io.debugTopDown.fromRob + ctrlBlock.io.debugTopDown.fromCore := io.debugTopDown.fromCore + + io.debugRolling := ctrlBlock.io.debugRolling + + private val cg = ClockGate.genTeSrc + dontTouch(cg) + if (hasMbist) { + cg.cgen := io.dft.get.cgen + } else { + cg.cgen := false.B + } + + if (backendParams.debugEn) { + dontTouch(memScheduler.io) + dontTouch(dataPath.io.toMemExu) + dontTouch(wbDataPath.io.fromMemExu) + } + + // reset tree + if (p(DebugOptionsKey).ResetGen) { + val rightResetTree = ResetGenNode(Seq( + ModuleNode(dataPath), + ModuleNode(intExuBlock), + ModuleNode(fpExuBlock), + ModuleNode(vfExuBlock), + ModuleNode(bypassNetwork), + ModuleNode(wbDataPath) + )) + val leftResetTree = ResetGenNode(Seq( + ModuleNode(intScheduler), + ModuleNode(fpScheduler), + ModuleNode(vfScheduler), + ModuleNode(memScheduler), + ModuleNode(og2ForVector), + ModuleNode(wbFuBusyTable), + ResetGenNode(Seq( + ModuleNode(ctrlBlock), + // ResetGenNode(Seq( + CellNode(io.frontendReset) + // )) + )) + )) + ResetGen(leftResetTree, reset, sim = false, io.dft_reset) + ResetGen(rightResetTree, reset, sim = false, io.dft_reset) + } else { + io.frontendReset := DontCare + } + + // perf events + val pfevent = Module(new PFEvent) + pfevent.io.distribute_csr := RegNext(csrio.customCtrl.distribute_csr) + val csrevents = pfevent.io.hpmevent.slice(8, 16) + + val ctrlBlockPerf = ctrlBlock.getPerfEvents + val intSchedulerPerf = intScheduler.asInstanceOf[SchedulerArithImp].getPerfEvents + val fpSchedulerPerf = fpScheduler.asInstanceOf[SchedulerArithImp].getPerfEvents + val vecSchedulerPerf = vfScheduler.asInstanceOf[SchedulerArithImp].getPerfEvents + val memSchedulerPerf = memScheduler.asInstanceOf[SchedulerMemImp].getPerfEvents + val dataPathPerf = dataPath.getPerfEvents + + XSPerfAccumulate("cpu_cycle", true.B) + XSPerfAccumulate("ref_cpu_cycle", io.fromTop.clintTime.valid) + + val perfBackend = Seq( + ("cpu_cycle", true.B), + ("ref_cpu_cycle", io.fromTop.clintTime.valid) + ) + + // let index = 0 be no event + val allPerfEvents = Seq(("noEvent", 0.U)) ++ ctrlBlockPerf ++ dataPathPerf ++ + intSchedulerPerf ++ fpSchedulerPerf ++ vecSchedulerPerf ++ memSchedulerPerf ++ perfBackend + + if (printEventCoding) { + for (((name, inc), i) <- allPerfEvents.zipWithIndex) { + println("backend perfEvents Set", name, inc, i) + } + } + + val allPerfInc = allPerfEvents.map(_._2.asTypeOf(new PerfEvent)) + val perfEvents = HPerfMonitor(csrevents, allPerfInc).getPerfEvents + csrio.perf.perfEventsBackend := VecInit(perfEvents.map(_._2.asTypeOf(new PerfEvent))) + + val ctrlBlockError = ctrlBlock.getCriticalErrors + val intExuBlockError = intExuBlock.getCriticalErrors + val criticalErrors = ctrlBlockError ++ intExuBlockError + + if (printCriticalError) { + for (((name, error), _) <- criticalErrors.zipWithIndex) { + XSError(error, s"critical error: $name \n") + } + } + + // expand to collect frontend/memblock/L2 critical errors + backendCriticalError := criticalErrors.map(_._2).reduce(_ || _) + + io.toTop.cpuCriticalError := csrio.criticalErrorState + io.toTop.msiAck := csrio.msiAck + io.toTop.teemsiAck zip csrio.teemsiAck foreach { case (toTop_teemsiAck, csrio_teemsiAck) => + toTop_teemsiAck := csrio_teemsiAck + } +} + +class BackendMemIO(implicit p: Parameters, params: BackendParams) extends XSBundle { + // Since fast load replay always use load unit 0, Backend flips two load port to avoid conflicts + val flippedLda = true + // params alias + private val LoadQueueSize = VirtualLoadQueueSize + // In/Out // Todo: split it into one-direction bundle + val lsqEnqIO = Flipped(new LsqEnqIO) + val robLsqIO = new RobLsqIO + val ldaIqFeedback = Vec(params.LduCnt, Flipped(new MemRSFeedbackIO)) + val staIqFeedback = Vec(params.StaCnt, Flipped(new MemRSFeedbackIO)) + val hyuIqFeedback = Vec(params.HyuCnt, Flipped(new MemRSFeedbackIO)) + val vstuIqFeedback = Flipped(Vec(params.VstuCnt, new MemRSFeedbackIO(isVector = true))) + val vlduIqFeedback = Flipped(Vec(params.VlduCnt, new MemRSFeedbackIO(isVector = true))) + val ldCancel = Vec(params.LdExuCnt, Input(new LoadCancelIO)) + val wakeup = Vec(params.LdExuCnt, Flipped(Valid(new DynInst))) + val storePcRead = Vec(params.StaCnt, Output(UInt(VAddrBits.W))) + val hyuPcRead = Vec(params.HyuCnt, Output(UInt(VAddrBits.W))) + // Input + val writebackLda = Vec(params.LduCnt, Flipped(DecoupledIO(new MemExuOutput))) + val writebackSta = Vec(params.StaCnt, Flipped(DecoupledIO(new MemExuOutput))) + val writebackStd = Vec(params.StdCnt, Flipped(DecoupledIO(new MemExuOutput))) + val writebackHyuLda = Vec(params.HyuCnt, Flipped(DecoupledIO(new MemExuOutput))) + val writebackHyuSta = Vec(params.HyuCnt, Flipped(DecoupledIO(new MemExuOutput))) + val writebackVldu = Vec(params.VlduCnt, Flipped(DecoupledIO(new MemExuOutput(true)))) + + val s3_delayed_load_error = Input(Vec(LoadPipelineWidth, Bool())) + val stIn = Input(Vec(params.StaExuCnt, ValidIO(new DynInst()))) + val memoryViolation = Flipped(ValidIO(new Redirect)) + + val exceptionAddr = Input(new Bundle { + val vaddr = UInt(XLEN.W) + val gpaddr = UInt(XLEN.W) + val isForVSnonLeafPTE = Bool() + }) + + val sqDeq = Input(UInt(log2Ceil(EnsbufferWidth + 1).W)) + val lqDeq = Input(UInt(log2Up(CommitWidth + 1).W)) + val sqDeqPtr = Input(new SqPtr) + val lqDeqPtr = Input(new LqPtr) + + val lqCancelCnt = Input(UInt(log2Up(VirtualLoadQueueSize + 1).W)) + val sqCancelCnt = Input(UInt(log2Up(StoreQueueSize + 1).W)) + + val lqCanAccept = Input(Bool()) + val sqCanAccept = Input(Bool()) + + val otherFastWakeup = Flipped(Vec(params.LduCnt + params.HyuCnt, ValidIO(new DynInst))) + val stIssuePtr = Input(new SqPtr()) + + val debugLS = Flipped(Output(new DebugLSIO)) + + val lsTopdownInfo = Vec(params.LduCnt + params.HyuCnt, Flipped(Output(new LsTopdownInfo))) + // Output + val redirect = ValidIO(new Redirect) // rob flush MemBlock + val issueLda = MixedVec(Seq.fill(params.LduCnt)(DecoupledIO(new MemExuInput()))) + val issueSta = MixedVec(Seq.fill(params.StaCnt)(DecoupledIO(new MemExuInput()))) + val issueStd = MixedVec(Seq.fill(params.StdCnt)(DecoupledIO(new MemExuInput()))) + val issueHylda = MixedVec(Seq.fill(params.HyuCnt)(DecoupledIO(new MemExuInput()))) + val issueHysta = MixedVec(Seq.fill(params.HyuCnt)(DecoupledIO(new MemExuInput()))) + val issueVldu = MixedVec(Seq.fill(params.VlduCnt)(DecoupledIO(new MemExuInput(true)))) + + val loadFastMatch = Vec(params.LduCnt, Output(UInt(params.LduCnt.W))) + val loadFastImm = Vec(params.LduCnt, Output(UInt(12.W))) // Imm_I + + val tlbCsr = Output(new TlbCsrBundle) + val csrCtrl = Output(new CustomCSRCtrlIO) + val sfence = Output(new SfenceBundle) + val isStoreException = Output(Bool()) + val isVlsException = Output(Bool()) + + val wfi = new WfiReqBundle + + // ATTENTION: The issue ports' sequence order should be the same as IQs' deq config + def issueUops: Seq[DecoupledIO[MemExuInput]] = { + issueSta ++ + issueHylda ++ issueHysta ++ + issueLda ++ + issueVldu ++ + issueStd + }.toSeq + + // ATTENTION: The writeback ports' sequence order should be the same as IQs' deq config + def writeBack: Seq[DecoupledIO[MemExuOutput]] = { + writebackSta ++ + writebackHyuLda ++ writebackHyuSta ++ + writebackLda ++ + writebackVldu ++ + writebackStd + } + + // store event difftest information + val storeDebugInfo = Vec( + EnsbufferWidth, + new Bundle { + val robidx = Input(new RobPtr) + val pc = Output(UInt(VAddrBits.W)) + } + ) + +} + +class BackendIO(implicit p: Parameters, params: BackendParams) extends XSBundle with HasSoCParameter { + val commands = new Commands + val timeRequest = Decoupled(Bool()) + val fromTop = Flipped(new TopToBackendBundle) + + val toTop = new BackendToTopBundle + + val traceCoreInterface = new TraceCoreInterface(hasOffset = true) + val fenceio = new FenceIO + // Todo: merge these bundles into BackendFrontendIO + val frontend = Flipped(new FrontendToCtrlIO) + val frontendSfence = Output(new SfenceBundle) + val frontendCsrCtrl = Output(new CustomCSRCtrlIO) + val frontendTlbCsr = Output(new TlbCsrBundle) + val frontendReset = Output(Reset()) + + val mem = new BackendMemIO + + val perf = Input(new PerfCounterIO) + + val tlb = Output(new TlbCsrBundle) + + val csrCustomCtrl = Output(new CustomCSRCtrlIO) + + val debugTopDown = new Bundle { + val fromRob = new RobCoreTopDownIO + val fromCore = new CoreDispatchTopDownIO + } + + val debugRolling = new RobDebugRollingIO + val topDownInfo = new TopDownInfo + val dft = Option.when(hasDFT)(Input(new SramBroadcastBundle)) + val dft_reset = Option.when(hasMbist)(Input(new DFTResetSignals())) +} diff --git a/examples/prefab/cores/xiangshan/arch/native/CSR.scala b/examples/prefab/cores/xiangshan/arch/native/CSR.scala new file mode 100644 index 000000000..dccae052e --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/CSR.scala @@ -0,0 +1,486 @@ +// Copied from XiangShan src/main/scala/xiangshan/backend/fu/wrapper/CSR.scala and modified for Buckyball. +// XiangShan is licensed under Mulan PSL v2 (http://license.coscl.org.cn/MulanPSL2). + +package examples.core.xiangshan.native + +import chisel3._ +import freechips.rocketchip.rocket.CSRs +import chisel3.util._ +import org.chipsalliance.cde.config.Parameters +import utility._ +import xiangshan._ +import xiangshan.backend.fu.NewCSR._ +import xiangshan.backend.fu.util._ +import xiangshan.backend.fu.{FuConfig, FuncUnit} +import device._ +import system.HasSoCParameter +import xiangshan.ExceptionNO._ +import xiangshan.backend.Bundles._ +import xiangshan.backend.decode.Imm_Z +import xiangshan.backend.fu.NewCSR.CSRBundles.PrivState +import xiangshan.backend.fu.NewCSR.CSRDefines.PrivMode +import xiangshan.backend.rob.RobPtr +import xiangshan.frontend.FtqPtr +import CSRConst._ +import xiangshan.backend.fu.wrapper._ +import examples.core.xiangshan.native.Command + +class CSR(cfg: FuConfig)(implicit p: Parameters) + extends FuncUnit(cfg) + with HasCircularQueuePtrHelper + with HasCriticalErrors + with HasSoCParameter { + val csrIn = io.csrio.get + val csrOut = io.csrio.get + val csrToDecode = io.csrToDecode.get + + val setFsDirty = csrIn.fpu.dirty_fs + val setFflags = csrIn.fpu.fflags + + val setVsDirty = csrIn.vpu.dirty_vs + val setVstart = csrIn.vpu.set_vstart + val setVtype = csrIn.vpu.set_vtype + val setVxsat = csrIn.vpu.set_vxsat + val vlFromPreg = csrIn.vpu.vl + + val flushPipe = Wire(Bool()) + val flush = io.flush.valid + + val timeAllowed = WireDefault(true.B) + val isBall = io.in.bits.ctrl.fuOpType === Command.opType + + /** Alias of input signals */ + val (valid, src1, imm, func) = ( + io.in.valid && timeAllowed && !isBall, + io.in.bits.data.src(0), + io.in.bits.data.imm(Imm_Z().len - 1, 0), + io.in.bits.ctrl.fuOpType + ) + + // split imm/src1/rd from IMM_Z: src1/rd for tval + val addr = Imm_Z().getCSRAddr(imm) + val rd = Imm_Z().getRD(imm) + val rs1 = Imm_Z().getRS1(imm) + val imm5 = Imm_Z().getImm5(imm) + val csri = ZeroExt(imm5, XLEN) + + private val isEcall = CSROpType.isSystemOp(func) && addr === privEcall + private val isEbreak = CSROpType.isSystemOp(func) && addr === privEbreak + private val isMNret = CSROpType.isSystemOp(func) && addr === privMNret + private val isMret = CSROpType.isSystemOp(func) && addr === privMret + private val isSret = CSROpType.isSystemOp(func) && addr === privSret + private val isDret = CSROpType.isSystemOp(func) && addr === privDret + private val isWfi = CSROpType.isWfi(func) + private val isCSRAcc = CSROpType.isCsrAccess(func) + + val csrMod = Module(new NewCSR) + val trapInstMod = Module(new TrapInstMod) + val trapTvalMod = Module(new TrapTvalMod) + val satpFlushMod = Module(new SatpFlushMod) + + private val privState = csrMod.io.status.privState + // The real reg value in CSR, with no read mask + private val regOut = csrMod.io.out.bits.regOut + private val src = Mux(CSROpType.needImm(func), csri, src1) + + private val wdata = LookupTree( + func, + Seq( + CSROpType.wrt -> src1, + CSROpType.set -> (regOut | src1), + CSROpType.clr -> (regOut & (~src1).asUInt), + CSROpType.wrti -> csri, + CSROpType.seti -> (regOut | csri), + CSROpType.clri -> (regOut & (~csri).asUInt) + ) + ) + + private val csrAccess = valid && CSROpType.isCsrAccess(func) + + private val csrWen = valid && ( + CSROpType.isCSRRW(func) || + CSROpType.isCSRRSorRC(func) && rs1 =/= 0.U + ) + + private val csrRen = valid && ( + CSROpType.isCSRRW(func) && rd =/= 0.U || + CSROpType.isCSRRSorRC(func) + ) + + private val waddrReg = RegEnable(addr, 0.U(12.W), io.in.fire) + private val wdataReg = RegEnable(wdata, 0.U(64.W), io.in.fire) + + private val robIdxReg = RegEnable(io.in.bits.ctrl.robIdx, io.in.fire) + private val thisRobIdx = Wire(new RobPtr) + when(io.in.valid) { + thisRobIdx := io.in.bits.ctrl.robIdx + }.otherwise { + thisRobIdx := robIdxReg + } + private val redirectFlush = thisRobIdx.needFlush(io.flush) + + val readsTime = CSROpType.isCsrAccess(func) && addr === CSRs.time.U + val timePending = RegInit(false.B) + val timeFresh = RegInit(false.B) + val timeKilled = RegInit(false.B) + val timeOwner = Reg(new RobPtr) + timeAllowed := !timePending && (!readsTime || timeFresh) + csrOut.timeRequest.valid := io.in.valid && readsTime && !timePending && !timeFresh && !redirectFlush + csrOut.timeRequest.bits := true.B + when(csrOut.timeRequest.fire) { + timePending := true.B + timeKilled := false.B + timeOwner := io.in.bits.ctrl.robIdx + } + when(timePending && timeOwner.needFlush(io.flush))(timeKilled := true.B) + when(io.csrin.get.clintTime.valid && timePending) { + timePending := false.B + timeFresh := !timeKilled && !timeOwner.needFlush(io.flush) + } + when(io.in.fire && readsTime || redirectFlush)(timeFresh := false.B) + + csrMod.io.in match { + case in => + in.valid := valid + in.bits.wen := csrWen + in.bits.ren := csrRen + in.bits.op := CSROpType.getCSROp(func) + in.bits.addr := addr + in.bits.src := src + in.bits.wdata := wdataReg + in.bits.mret := isMret + in.bits.mnret := isMNret + in.bits.sret := isSret + in.bits.dret := isDret + in.bits.redirectFlush := redirectFlush + } + csrMod.io.trapInst := trapInstMod.io.currentTrapInst + csrMod.io.fetchMalTval := trapTvalMod.io.tval + csrMod.io.oldPrivSate := satpFlushMod.out.oldPrivState + csrMod.io.oldSatpMode := satpFlushMod.out.oldSatpMode + csrMod.io.oldVsatpMode := satpFlushMod.out.oldVsatpMode + csrMod.io.fromMem.excpVA := csrIn.memExceptionVAddr + csrMod.io.fromMem.excpGPA := csrIn.memExceptionGPAddr + csrMod.io.fromMem.excpIsForVSnonLeafPTE := csrIn.memExceptionIsForVSnonLeafPTE + + csrMod.io.fromRob.trap.valid := csrIn.exception.valid + csrMod.io.fromRob.trap.bits.pc := csrIn.exception.bits.pc + csrMod.io.fromRob.trap.bits.instr := csrIn.exception.bits.instr + csrMod.io.fromRob.trap.bits.pcGPA := csrIn.exception.bits.gpaddr + // Todo: shrink the width of trap vector. + // We use 64bits trap vector in CSR, and 24 bits exceptionVec in exception bundle. + csrMod.io.fromRob.trap.bits.trapVec := csrIn.exception.bits.exceptionVec.asUInt + csrMod.io.fromRob.trap.bits.isFetchBkpt := csrIn.exception.bits.isPcBkpt + csrMod.io.fromRob.trap.bits.singleStep := csrIn.exception.bits.singleStep + csrMod.io.fromRob.trap.bits.crossPageIPFFix := csrIn.exception.bits.crossPageIPFFix + csrMod.io.fromRob.trap.bits.isInterrupt := csrIn.exception.bits.isInterrupt + csrMod.io.fromRob.trap.bits.trigger := csrIn.exception.bits.trigger + csrMod.io.fromRob.trap.bits.isHls := csrIn.exception.bits.isHls + csrMod.io.fromRob.trap.bits.isFetchMalAddr := csrIn.exception.bits.isFetchMalAddr + csrMod.io.fromRob.trap.bits.satpFlushFirstFetchFault := csrIn.exception.bits.satpFlushFirstFetchFault + csrMod.io.fromRob.trap.bits.isForVSnonLeafPTE := csrIn.exception.bits.isForVSnonLeafPTE + + csrMod.io.fromRob.commit.fflags := setFflags + csrMod.io.fromRob.commit.fsDirty := setFsDirty + csrMod.io.fromRob.commit.vxsat.valid := setVxsat.valid + csrMod.io.fromRob.commit.vxsat.bits := setVxsat.bits + csrMod.io.fromRob.commit.vsDirty := setVsDirty + csrMod.io.fromRob.commit.vstart := setVstart + csrMod.io.fromRob.commit.vl := vlFromPreg + // Todo: correct vtype + csrMod.io.fromRob.commit.vtype.valid := setVtype.valid + csrMod.io.fromRob.commit.vtype.bits.VILL := setVtype.bits(XLEN - 1) + csrMod.io.fromRob.commit.vtype.bits.VMA := setVtype.bits(7) + csrMod.io.fromRob.commit.vtype.bits.VTA := setVtype.bits(6) + csrMod.io.fromRob.commit.vtype.bits.VSEW := setVtype.bits(5, 3) + csrMod.io.fromRob.commit.vtype.bits.VLMUL := setVtype.bits(2, 0) + + csrMod.io.fromRob.commit.instNum.valid := true.B // Todo: valid control signal + csrMod.io.fromRob.commit.instNum.bits := csrIn.perf.retiredInstr + + csrMod.io.fromRob.robDeqPtr := csrIn.robDeqPtr + + csrMod.io.fromVecExcpMod.busy := io.csrin.get.fromVecExcpMod.busy + + csrMod.io.perf := csrIn.perf + + csrMod.platformIRP.MEIP := csrIn.externalInterrupt.meip + csrMod.platformIRP.MTIP := csrIn.externalInterrupt.mtip + csrMod.platformIRP.MSIP := csrIn.externalInterrupt.msip + csrMod.platformIRP.SEIP := csrIn.externalInterrupt.seip + csrMod.platformIRP.STIP := false.B + csrMod.platformIRP.VSEIP := false.B // Todo + csrMod.platformIRP.VSTIP := false.B // Todo + csrMod.platformIRP.debugIP := csrIn.externalInterrupt.debug + csrMod.nonMaskableIRP.NMI_43 := csrIn.externalInterrupt.nmi.nmi_43 + csrMod.nonMaskableIRP.NMI_31 := csrIn.externalInterrupt.nmi.nmi_31 + + csrMod.io.fromTop.hartId := io.csrin.get.hartId + csrMod.io.fromTop.clintTime := io.csrin.get.clintTime + csrMod.io.fromTop.l2FlushDone := io.csrin.get.l2FlushDone + csrMod.io.fromTop.reset_mtvec.foreach(_ := io.csrin.get.reset_mtvec.get) + csrMod.io.fromTop.criticalErrorState := io.csrin.get.criticalErrorState + private val csrModOutValid = csrMod.io.out.valid + private val csrModOut = csrMod.io.out.bits + + trapInstMod.io.fromDecode.trapInstInfo := RegNextWithEnable(io.csrin.get.trapInstInfo, hasInit = true) + trapInstMod.io.fromRob.flush.valid := io.flush.valid + trapInstMod.io.fromRob.flush.bits.ftqPtr := io.flush.bits.ftqIdx + trapInstMod.io.fromRob.flush.bits.ftqOffset := io.flush.bits.ftqOffset + trapInstMod.io.fromRob.isInterrupt.valid := csrIn.exception.valid + trapInstMod.io.fromRob.isInterrupt.bits := csrIn.exception.bits.isInterrupt + trapInstMod.io.faultCsrUop.valid := csrMod.io.out.valid && (csrMod.io.out.bits.EX_II || csrMod.io.out.bits.EX_VI) + trapInstMod.io.faultCsrUop.bits.fuOpType := DataHoldBypass(io.in.bits.ctrl.fuOpType, io.in.fire) + trapInstMod.io.faultCsrUop.bits.imm := DataHoldBypass(io.in.bits.data.imm, io.in.fire) + trapInstMod.io.faultCsrUop.bits.ftqInfo.ftqPtr := DataHoldBypass(io.in.bits.ctrl.ftqIdx.get, io.in.fire) + trapInstMod.io.faultCsrUop.bits.ftqInfo.ftqOffset := DataHoldBypass(io.in.bits.ctrl.ftqOffset.get, io.in.fire) + // Clear trap instruction when instruction fault trap(EX_II, EX_VI) occurs. + trapInstMod.io.readClear := (csrMod.io.fromRob.trap match { + case t => + t.valid && !t.bits.isInterrupt && (t.bits.trapVec(EX_II) || t.bits.trapVec(EX_VI)) + }) + + trapTvalMod.io.targetPc.valid := csrMod.io.out.bits.targetPcUpdate + trapTvalMod.io.targetPc.bits := csrMod.io.out.bits.targetPc + trapTvalMod.io.clear := csrIn.exception.valid && csrIn.exception.bits.isFetchMalAddr + trapTvalMod.io.fromCtrlBlock.flush := io.flush + trapTvalMod.io.fromCtrlBlock.robDeqPtr := io.csrio.get.robDeqPtr + + satpFlushMod.in.satp.valid := csrMod.io.status.satp.valid + satpFlushMod.in.satp.bits := csrMod.io.status.satp.bits + satpFlushMod.in.vsatp.valid := csrMod.io.status.vsatp.valid + satpFlushMod.in.vsatp.bits := csrMod.io.status.vsatp.bits + satpFlushMod.in.privState := csrMod.io.status.privState + + val imsic = Module(new aia.IMSIC_WRAP(soc.IMSICParams)) + imsic.fromCSR.addr.valid := csrMod.toAIA.addr.valid + imsic.fromCSR.addr.bits.addr := csrMod.toAIA.addr.bits.addr + imsic.fromCSR.addr.bits.virt := csrMod.toAIA.addr.bits.v.asUInt.asBool + imsic.fromCSR.addr.bits.priv := aia.PrivType(csrMod.toAIA.addr.bits.prvm.asUInt) + imsic.fromCSR.vgein := csrMod.toAIA.vgein + imsic.fromCSR.wdata.valid := csrMod.toAIA.wdata.valid + imsic.fromCSR.wdata.bits.op := aia.OpType(csrMod.toAIA.wdata.bits.op) + imsic.fromCSR.wdata.bits.data := csrMod.toAIA.wdata.bits.data + imsic.fromCSR.claims(0) := csrMod.toAIA.mClaim + imsic.fromCSR.claims(1) := csrMod.toAIA.sClaim + imsic.fromCSR.claims(2) := csrMod.toAIA.vsClaim + + csrMod.fromAIA.rdata.valid := imsic.toCSR.rdata.valid + csrMod.fromAIA.rdata.bits.data := imsic.toCSR.rdata.bits + csrMod.fromAIA.rdata.bits.illegal := imsic.toCSR.illegal + csrMod.fromAIA.meip := imsic.toCSR.pendings(0) + csrMod.fromAIA.seip := imsic.toCSR.pendings(1) + csrMod.fromAIA.vseip := imsic.toCSR.pendings(soc.IMSICParams.intFilesNum - 1, 2) + csrMod.fromAIA.mtopei := imsic.toCSR.topeis(0) + csrMod.fromAIA.stopei := imsic.toCSR.topeis(1) + csrMod.fromAIA.vstopei := imsic.toCSR.topeis(2) + + imsic.msiio.vld_req := io.csrin.get.msiInfo.valid + imsic.msiio.data := io.csrin.get.msiInfo.bits + io.csrio.get.msiAck := imsic.msiio.vld_ack + + imsic.teemsiio.foreach { teemsiio => + teemsiio.vld_req := io.csrin.get.teemsiInfo.get.valid + teemsiio.data := io.csrin.get.teemsiInfo.get.bits + io.csrio.get.teemsiAck.get := teemsiio.vld_ack + } + csrMod.fromAIA.notice_pending := false.B + imsic.sec.foreach { imsicsec => + imsicsec.cmode := csrMod.io.tlb.mbmc.CMODE.asBool + csrMod.fromAIA.notice_pending := imsicsec.notice_pending + } + + private val exceptionVec = WireInit(0.U.asTypeOf(ExceptionVec())) // Todo: + + exceptionVec(EX_BP) := DataHoldBypass(isEbreak, false.B, io.in.fire) + exceptionVec(EX_MCALL) := DataHoldBypass(isEcall && privState.isModeM, false.B, io.in.fire) + exceptionVec(EX_HSCALL) := DataHoldBypass(isEcall && privState.isModeHS, false.B, io.in.fire) + exceptionVec(EX_VSCALL) := DataHoldBypass(isEcall && privState.isModeVS, false.B, io.in.fire) + exceptionVec(EX_UCALL) := DataHoldBypass(isEcall && privState.isModeHUorVU, false.B, io.in.fire) + exceptionVec(EX_II) := csrMod.io.out.bits.EX_II + exceptionVec(EX_VI) := csrMod.io.out.bits.EX_VI + + val isXRet = valid && func === CSROpType.jmp && !isEcall && !isEbreak + + flushPipe := csrMod.io.out.bits.flushPipe + + // tlb + val tlb = Wire(new TlbCsrBundle) + tlb.satp.changed := csrMod.io.tlb.satpASIDChanged + tlb.satp.mode := csrMod.io.tlb.satp.MODE.asUInt + tlb.satp.asid := csrMod.io.tlb.satp.ASID.asUInt + tlb.satp.ppn := csrMod.io.tlb.satp.PPN.asUInt + tlb.vsatp.changed := csrMod.io.tlb.vsatpASIDChanged + tlb.vsatp.mode := csrMod.io.tlb.vsatp.MODE.asUInt + tlb.vsatp.asid := csrMod.io.tlb.vsatp.ASID.asUInt + tlb.vsatp.ppn := csrMod.io.tlb.vsatp.PPN.asUInt + tlb.hgatp.changed := csrMod.io.tlb.hgatpVMIDChanged + tlb.hgatp.mode := csrMod.io.tlb.hgatp.MODE.asUInt + tlb.hgatp.vmid := csrMod.io.tlb.hgatp.VMID.asUInt + tlb.hgatp.ppn := csrMod.io.tlb.hgatp.PPN.asUInt + tlb.mbmc.KEYIDEN := csrMod.io.tlb.mbmc.KEYIDEN.asUInt + tlb.mbmc.BME := csrMod.io.tlb.mbmc.BME.asUInt + tlb.mbmc.CMODE := csrMod.io.tlb.mbmc.CMODE.asUInt + tlb.mbmc.BCLEAR := csrMod.io.tlb.mbmc.BCLEAR.asUInt + tlb.mbmc.BMA := csrMod.io.tlb.mbmc.BMA.asUInt + + // expose several csr bits for tlb + tlb.priv.mxr := csrMod.io.tlb.mxr + tlb.priv.sum := csrMod.io.tlb.sum + tlb.priv.vmxr := csrMod.io.tlb.vmxr + tlb.priv.vsum := csrMod.io.tlb.vsum + tlb.priv.spvp := csrMod.io.tlb.spvp + tlb.priv.virt := csrMod.io.tlb.dvirt + tlb.priv.virt_changed := DataChanged(tlb.priv.virt) + tlb.priv.imode := csrMod.io.tlb.imode + tlb.priv.dmode := csrMod.io.tlb.dmode + tlb.priv.debug := csrMod.io.tlb.debug + + // Svpbmt extension enable + tlb.mPBMTE := csrMod.io.tlb.mPBMTE + tlb.hPBMTE := csrMod.io.tlb.hPBMTE + + // pointer masking extension + tlb.pmm := csrMod.io.tlb.pmm + + /** Since some CSR read instructions are allowed to be pipelined, ready/valid signals should be modified */ + io.in.ready := csrMod.io.in.ready && timeAllowed // Includes asynchronous time reads. + io.out.valid := csrModOutValid + io.out.bits.ctrl.exceptionVec.get := exceptionVec + io.out.bits.ctrl.flushPipe.get := flushPipe + io.out.bits.ctrl.satpFlushPipe.get := csrMod.io.status.satp.valid || csrMod.io.status.vsatp.valid + io.out.bits.res.data := csrMod.io.out.bits.rData + + /** initialize NewCSR's io_out_ready from wrapper's io */ + csrMod.io.out.ready := io.out.ready + + io.out.bits.res.redirect.get.valid := io.out.valid && RegEnable(isXRet, false.B, io.in.fire) + val redirect = io.out.bits.res.redirect.get.bits + redirect := 0.U.asTypeOf(redirect) + redirect.level := RedirectLevel.flushAfter + redirect.robIdx := robIdxReg + redirect.ftqIdx := RegEnable(io.in.bits.ctrl.ftqIdx.get, io.in.fire) + redirect.ftqOffset := RegEnable(io.in.bits.ctrl.ftqOffset.get, io.in.fire) + redirect.cfiUpdate.predTaken := true.B + redirect.cfiUpdate.taken := true.B + redirect.cfiUpdate.target := csrMod.io.out.bits.targetPc.pc + redirect.cfiUpdate.backendIPF := csrMod.io.out.bits.targetPc.raiseIPF + redirect.cfiUpdate.backendIAF := csrMod.io.out.bits.targetPc.raiseIAF + redirect.cfiUpdate.backendIGPF := csrMod.io.out.bits.targetPc.raiseIGPF + // Only mispred will send redirect to frontend + redirect.cfiUpdate.isMisPred := false.B + + connectNonPipedCtrlSingal + + val commandIdle :: commandDrain :: commandSend :: commandWait :: commandDone :: Nil = Enum(5) + val commandState = RegInit(commandIdle) + val command = Reg(chiselTypeOf(csrOut.commands.request.bits)) + val commandError = RegInit(false.B) + val commandOwner = Reg(new RobPtr) + csrOut.commandFlushStores := commandState === commandDrain + csrOut.commands.request.valid := commandState === commandSend + csrOut.commands.request.bits := command + when(isBall)(io.in.ready := commandState === commandIdle && !timePending && !timeFresh) + when(io.in.fire && isBall) { + assert(!csrMod.io.out.valid) + command.instruction := Cat(Imm_Z().getCSRAddr(imm), Imm_Z().getRS1(imm), 3.U(3.W), 0.U(5.W), "h7b".U(7.W)) + command.pc := io.in.bits.data.pc.get + command.rs1 := io.in.bits.data.src(0) + command.rs2 := io.in.bits.data.src(1) + command.context := csrMod.io.accelerator + commandOwner := io.in.bits.ctrl.robIdx + commandError := csrMod.io.accelerator.virt + commandState := Mux(csrMod.io.accelerator.virt, commandDone, commandDrain) + } + when(commandState =/= commandIdle)(io.in.ready := false.B) + when(commandState === commandDrain && csrOut.commandStoresEmpty)(commandState := commandSend) + when(csrOut.commands.request.fire)(commandState := commandWait) + when(commandState === commandWait && csrOut.commands.complete.valid) { + commandError := csrOut.commands.complete.bits + commandState := commandDone + } + when(commandState === commandDone) { + io.out.valid := true.B + io.out.bits.res.data := 0.U + io.out.bits.res.redirect.get.valid := false.B + io.out.bits.ctrl.exceptionVec.get := 0.U.asTypeOf(io.out.bits.ctrl.exceptionVec.get) + // Failed accelerator operations trap instead of retiring as successful commands. + io.out.bits.ctrl.exceptionVec.get(illegalInstr) := commandError + io.out.bits.ctrl.flushPipe.get := false.B + io.out.bits.ctrl.satpFlushPipe.get := false.B + when(io.out.fire)(commandState := commandIdle) + } + when(commandState =/= commandIdle)(assert(!commandOwner.needFlush(io.flush))) + + override val criticalErrors = csrMod.getCriticalErrors + generateCriticalErrors() + + // Todo: summerize all difftest skip condition + csrOut.isPerfCnt := io.out.valid && csrMod.io.out.bits.isPerfCnt && RegEnable( + func =/= CSROpType.jmp, + false.B, + io.in.fire + ) + csrOut.fpu.frm := csrMod.io.status.fpState.frm.asUInt + csrOut.vpu.vstart := csrMod.io.status.vecState.vstart.asUInt + csrOut.vpu.vxrm := csrMod.io.status.vecState.vxrm.asUInt + + csrOut.isXRet := isXRet + + csrOut.trapTarget := csrMod.io.out.bits.targetPc + csrOut.interrupt := csrMod.io.status.interrupt + csrOut.wfi_event := csrMod.io.status.wfiEvent + + csrOut.tlb := tlb + + csrOut.debugMode := csrMod.io.status.debugMode + + csrOut.traceCSR := csrMod.io.status.traceCSR + + csrOut.customCtrl match { + case custom => + custom.pf_ctrl := csrMod.io.status.custom.pf_ctrl + // Load violation predictor + custom.lvpred_disable := csrMod.io.status.custom.lvpred_disable + custom.no_spec_load := csrMod.io.status.custom.no_spec_load + custom.storeset_wait_store := csrMod.io.status.custom.storeset_wait_store + custom.storeset_no_fast_wakeup := csrMod.io.status.custom.storeset_no_fast_wakeup + custom.lvpred_timeout := csrMod.io.status.custom.lvpred_timeout + // Branch predictor + custom.bp_ctrl := csrMod.io.status.custom.bp_ctrl + // Memory Block + custom.sbuffer_timeout := csrMod.io.status.custom.sbuffer_timeout + custom.sbuffer_threshold := csrMod.io.status.custom.sbuffer_threshold + custom.ldld_vio_check_enable := csrMod.io.status.custom.ldld_vio_check_enable + custom.soft_prefetch_enable := csrMod.io.status.custom.soft_prefetch_enable + custom.cache_error_enable := csrMod.io.status.custom.cache_error_enable + custom.uncache_write_outstanding_enable := csrMod.io.status.custom.uncache_write_outstanding_enable + custom.hd_misalign_st_enable := csrMod.io.status.custom.hd_misalign_st_enable + custom.hd_misalign_ld_enable := csrMod.io.status.custom.hd_misalign_ld_enable + custom.power_down_enable := csrMod.io.status.custom.power_down_enable + custom.commit_stuck_check_enable := csrMod.io.status.custom.commit_stuck_check_enable + custom.flush_l2_enable := csrMod.io.status.custom.flush_l2_enable + // Rename + custom.fusion_enable := csrMod.io.status.custom.fusion_enable + custom.wfi_enable := csrMod.io.status.custom.wfi_enable + // distribute csr write signal + // write to frontend and memory + custom.distribute_csr.w.valid := csrMod.io.distributedWenLegal + custom.distribute_csr.w.bits.addr := waddrReg + custom.distribute_csr.w.bits.data := wdataReg + // rename single step + custom.singlestep := csrMod.io.status.singleStepFlag + // trigger + custom.frontend_trigger := csrMod.io.status.frontendTrigger + custom.mem_trigger := csrMod.io.status.memTrigger + // virtual mode + custom.virtMode := csrMod.io.status.privState.V.asBool + // xstatus.fs field is off + custom.fsIsOff := csrMod.io.toDecode.illegalInst.fsIsOff + } + + csrOut.instrAddrTransType := csrMod.io.status.instrAddrTransType + csrOut.criticalErrorState := csrMod.io.status.criticalErrorState + + csrToDecode := csrMod.io.toDecode +} diff --git a/examples/prefab/cores/xiangshan/arch/native/Core.scala b/examples/prefab/cores/xiangshan/arch/native/Core.scala new file mode 100644 index 000000000..7a9f8b758 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/Core.scala @@ -0,0 +1,398 @@ +// Copied from XiangShan src/main/scala/xiangshan/XSCore.scala and modified for Buckyball. +// XiangShan is licensed under Mulan PSL v2 (http://license.coscl.org.cn/MulanPSL2). + +package examples.core.xiangshan.native + +import org.chipsalliance.cde.config +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import chisel3.util._ +import coupledL2.PrefetchCtrlFromCore +import freechips.rocketchip.diplomacy.{BundleBridgeSource, LazyModule, LazyModuleImp} +import freechips.rocketchip.diplomacy._ +import freechips.rocketchip.tile.HasFPUParameters +import system.{HasSoCParameter, SoCParamsKey} +import utils._ +import utility._ +import utility.mbist.{MbistInterface, MbistPipeline} +import utility.sram.{SramBroadcastBundle, SramHelper} +import xiangshan.frontend._ +import xiangshan.backend._ +import xiangshan.backend.fu.PMPRespBundle +import xiangshan.backend.trace.TraceCoreInterface +import xiangshan.mem._ +import xiangshan.cache.mmu._ +import xiangshan.cache.mmu.TlbRequestIO +import scala.collection.mutable.ListBuffer +import xiangshan._ +import examples.core.xiangshan.native.{Backend, Commands} + +abstract class XSCoreBase()(implicit p: config.Parameters) extends LazyModule with HasXSParameter { + override def shouldBeInlined: Boolean = false + // outer facing nodes + val frontend = LazyModule(new Frontend()) + val csrOut = BundleBridgeSource(Some(() => new DistributedCSRIO())) + val backend = LazyModule(new Backend(backendParams)) + + val memBlock = LazyModule(new MemBlock) + + memBlock.inner.frontendBridge.icache_node := frontend.inner.icache.clientNode + memBlock.inner.frontendBridge.instr_uncache_node := frontend.inner.instrUncache.clientNode + if (p(SoCParamsKey).EnableICacheCtrl) { + frontend.inner.icache.ctrlUnitOpt.get.node := memBlock.inner.frontendBridge.icachectrl_node + } +} + +class XSCore()(implicit p: config.Parameters) extends XSCoreBase with HasXSDts { + lazy val module = new XSCoreImp(this) +} + +class XSCoreImp(outer: XSCoreBase) extends LazyModuleImp(outer) with HasXSParameter with HasSoCParameter { + + val io = IO(new Bundle { + val hartId = Input(UInt(hartIdLen.W)) + val msiInfo = Input(ValidIO(UInt(soc.IMSICParams.MSI_INFO_WIDTH.W))) + val msiAck = Output(Bool()) + val teemsiInfo = Option.when(soc.IMSICParams.HasTEEIMSIC)(Input(ValidIO(UInt(soc.IMSICParams.MSI_INFO_WIDTH.W)))) + val teemsiAck = Option.when(soc.IMSICParams.HasTEEIMSIC)(Output(Bool())) + val commands = new Commands + val timeRequest = Decoupled(Bool()) + val clintTime = Input(ValidIO(UInt(64.W))) + val reset_vector = Input(UInt(PAddrBits.W)) + val reset_mtvec = Option.when(enableResetMtvec)(Input(UInt(PAddrBits.W))) + val cpu_halt = Output(Bool()) + val l2_flush_done = Input(Bool()) + val l2_flush_en = Output(Bool()) + val power_down_en = Output(Bool()) + val cpu_critical_error = Output(Bool()) + val resetInFrontend = Output(Bool()) + val traceCoreInterface = new TraceCoreInterface + val l2PfCtrl = Output(new PrefetchCtrlFromCore) + val perfEvents = Input(Vec(numPCntHc * coreParams.L2NBanks + 1, new PerfEvent)) + val beu_errors = Output(new XSL1BusErrors()) + val l2_hint = Input(Valid(new L2ToL1Hint())) + val l2_tlb_req = Flipped(new TlbRequestIO(nRespDups = 2)) + val l2_pmp_resp = new PMPRespBundle + val l2PfqBusy = Input(Bool()) + + val debugTopDown = new Bundle { + val robTrueCommit = Output(UInt(64.W)) + val robHeadPaddr = Valid(UInt(PAddrBits.W)) + val l2MissMatch = Input(Bool()) + val l3MissMatch = Input(Bool()) + } + + val topDownInfo = Input(new Bundle { + val l2Miss = Bool() + val l3Miss = Bool() + }) + + val dft = Option.when(hasDFT)(Input(new SramBroadcastBundle)) + val dft_reset = Option.when(hasDFT)(Input(new DFTResetSignals())) + }) + + dontTouch(io.l2_flush_done) + dontTouch(io.l2_flush_en) + dontTouch(io.power_down_en) + + println(s"FPGAPlatform:${env.FPGAPlatform} EnableDebug:${env.EnableDebug}") + + val frontend = outer.frontend.module + val backend = outer.backend.module + io.commands <> backend.io.commands + io.timeRequest <> backend.io.timeRequest + val memBlock = outer.memBlock.module + + frontend.io.hartId := memBlock.io.inner_hartId + frontend.io.reset_vector := memBlock.io.inner_reset_vector + frontend.io.softPrefetch <> memBlock.io.ifetchPrefetch + frontend.io.backend <> backend.io.frontend + frontend.io.sfence <> backend.io.frontendSfence + frontend.io.tlbCsr <> backend.io.frontendTlbCsr + frontend.io.csrCtrl <> backend.io.frontendCsrCtrl + frontend.io.fencei <> backend.io.fenceio.fencei + + backend.io.fromTop := memBlock.io.mem_to_ooo.topToBackendBypass + + require(backend.io.mem.stIn.length == memBlock.io.mem_to_ooo.stIn.length) + backend.io.mem.stIn.zip(memBlock.io.mem_to_ooo.stIn).foreach { case (sink, source) => + sink.valid := source.valid + sink.bits := 0.U.asTypeOf(sink.bits) + sink.bits.robIdx := source.bits.uop.robIdx + sink.bits.ssid := source.bits.uop.ssid + sink.bits.storeSetHit := source.bits.uop.storeSetHit + // The other signals have not been used + } + backend.io.mem.memoryViolation := memBlock.io.mem_to_ooo.memoryViolation + backend.io.mem.lsqEnqIO <> memBlock.io.ooo_to_mem.enqLsq + backend.io.mem.sqDeq := memBlock.io.mem_to_ooo.sqDeq + backend.io.mem.lqDeq := memBlock.io.mem_to_ooo.lqDeq + backend.io.mem.sqDeqPtr := memBlock.io.mem_to_ooo.sqDeqPtr + backend.io.mem.lqDeqPtr := memBlock.io.mem_to_ooo.lqDeqPtr + backend.io.mem.lqCancelCnt := memBlock.io.mem_to_ooo.lqCancelCnt + backend.io.mem.sqCancelCnt := memBlock.io.mem_to_ooo.sqCancelCnt + backend.io.mem.otherFastWakeup := memBlock.io.mem_to_ooo.otherFastWakeup + backend.io.mem.stIssuePtr := memBlock.io.mem_to_ooo.stIssuePtr + backend.io.mem.ldaIqFeedback := memBlock.io.mem_to_ooo.ldaIqFeedback + backend.io.mem.staIqFeedback := memBlock.io.mem_to_ooo.staIqFeedback + backend.io.mem.hyuIqFeedback := memBlock.io.mem_to_ooo.hyuIqFeedback + backend.io.mem.vstuIqFeedback := memBlock.io.mem_to_ooo.vstuIqFeedback + backend.io.mem.vlduIqFeedback := memBlock.io.mem_to_ooo.vlduIqFeedback + backend.io.mem.ldCancel := memBlock.io.mem_to_ooo.ldCancel + backend.io.mem.wakeup := memBlock.io.mem_to_ooo.wakeup + backend.io.mem.writebackLda <> memBlock.io.mem_to_ooo.writebackLda + backend.io.mem.writebackSta <> memBlock.io.mem_to_ooo.writebackSta + backend.io.mem.writebackHyuLda <> memBlock.io.mem_to_ooo.writebackHyuLda + backend.io.mem.writebackHyuSta <> memBlock.io.mem_to_ooo.writebackHyuSta + backend.io.mem.writebackStd <> memBlock.io.mem_to_ooo.writebackStd + backend.io.mem.writebackVldu <> memBlock.io.mem_to_ooo.writebackVldu + backend.io.mem.robLsqIO.loadMmio := memBlock.io.mem_to_ooo.lsqio.loadMmio + backend.io.mem.robLsqIO.loadMmioUop := memBlock.io.mem_to_ooo.lsqio.loadMmioUop + backend.io.mem.robLsqIO.storeMmio := memBlock.io.mem_to_ooo.lsqio.storeMmio + backend.io.mem.robLsqIO.storeMmioUop := memBlock.io.mem_to_ooo.lsqio.storeMmioUop + + // memblock error exception writeback, 1 cycle after normal writeback + backend.io.mem.s3_delayed_load_error := memBlock.io.mem_to_ooo.s3_delayed_load_error + + backend.io.mem.exceptionAddr.vaddr := memBlock.io.mem_to_ooo.lsqio.vaddr + backend.io.mem.exceptionAddr.gpaddr := memBlock.io.mem_to_ooo.lsqio.gpaddr + backend.io.mem.exceptionAddr.isForVSnonLeafPTE := memBlock.io.mem_to_ooo.lsqio.isForVSnonLeafPTE + backend.io.mem.debugLS := memBlock.io.debug_ls + backend.io.mem.lsTopdownInfo := memBlock.io.mem_to_ooo.lsTopdownInfo + backend.io.mem.lqCanAccept := memBlock.io.mem_to_ooo.lsqio.lqCanAccept + backend.io.mem.sqCanAccept := memBlock.io.mem_to_ooo.lsqio.sqCanAccept + backend.io.fenceio.sbuffer.sbIsEmpty := memBlock.io.mem_to_ooo.sbIsEmpty + + backend.io.perf.frontendInfo := frontend.io.frontendInfo + backend.io.perf.memInfo := memBlock.io.memInfo + backend.io.perf.perfEventsFrontend := frontend.io_perf + backend.io.perf.perfEventsLsu := memBlock.io_perf + backend.io.perf.perfEventsHc := memBlock.io.inner_hc_perfEvents + backend.io.perf.perfEventsBackend := DontCare + backend.io.perf.retiredInstr := DontCare + backend.io.perf.ctrlInfo := DontCare + + backend.io.mem.storeDebugInfo <> memBlock.io.mem_to_ooo.storeDebugInfo + + // top -> memBlock + memBlock.io.fromTopToBackend.clintTime := io.clintTime + memBlock.io.fromTopToBackend.msiInfo := io.msiInfo + memBlock.io.fromTopToBackend.teemsiInfo zip io.teemsiInfo foreach { case (memBlock_teemsiInfo, io_teemsiInfo) => + memBlock_teemsiInfo := io_teemsiInfo + } + memBlock.io.hartId := io.hartId + memBlock.io.l2_flush_done := io.l2_flush_done + memBlock.io.outer_reset_vector := io.reset_vector + memBlock.io.fromTopToBackend.reset_mtvec.foreach(_ := io.reset_mtvec.get) + memBlock.io.outer_hc_perfEvents := io.perfEvents + // frontend -> memBlock + memBlock.io.inner_beu_errors_icache <> frontend.io.error + memBlock.io.ooo_to_mem.backendToTopBypass := backend.io.toTop + memBlock.io.ooo_to_mem.issueLda <> backend.io.mem.issueLda + memBlock.io.ooo_to_mem.issueSta <> backend.io.mem.issueSta + memBlock.io.ooo_to_mem.issueStd <> backend.io.mem.issueStd + memBlock.io.ooo_to_mem.issueHya <> backend.io.mem.issueHylda + backend.io.mem.issueHysta.foreach(_.ready := false.B) // this fake port should not be used + memBlock.io.ooo_to_mem.issueVldu <> backend.io.mem.issueVldu + + // By default, instructions do not have exceptions when they enter the function units. + memBlock.io.ooo_to_mem.issueUops.map(_.bits.uop.clearExceptions()) + memBlock.io.ooo_to_mem.storePc := backend.io.mem.storePcRead + memBlock.io.ooo_to_mem.hybridPc := backend.io.mem.hyuPcRead + memBlock.io.ooo_to_mem.flushSb := backend.io.fenceio.sbuffer.flushSb + memBlock.io.ooo_to_mem.loadFastMatch := 0.U.asTypeOf(memBlock.io.ooo_to_mem.loadFastMatch) + memBlock.io.ooo_to_mem.loadFastImm := 0.U.asTypeOf(memBlock.io.ooo_to_mem.loadFastImm) + memBlock.io.ooo_to_mem.loadFastFuOpType := 0.U.asTypeOf(memBlock.io.ooo_to_mem.loadFastFuOpType) + + memBlock.io.ooo_to_mem.sfence <> backend.io.mem.sfence + + memBlock.io.redirect := backend.io.mem.redirect + memBlock.io.ooo_to_mem.csrCtrl := backend.io.mem.csrCtrl + memBlock.io.ooo_to_mem.tlbCsr := backend.io.mem.tlbCsr + memBlock.io.ooo_to_mem.lsqio.lcommit := backend.io.mem.robLsqIO.lcommit + memBlock.io.ooo_to_mem.lsqio.scommit := backend.io.mem.robLsqIO.scommit + memBlock.io.ooo_to_mem.lsqio.pendingMMIOld := backend.io.mem.robLsqIO.pendingMMIOld + memBlock.io.ooo_to_mem.lsqio.pendingld := backend.io.mem.robLsqIO.pendingld + memBlock.io.ooo_to_mem.lsqio.pendingst := backend.io.mem.robLsqIO.pendingst + memBlock.io.ooo_to_mem.lsqio.pendingVst := backend.io.mem.robLsqIO.pendingVst + memBlock.io.ooo_to_mem.lsqio.commit := backend.io.mem.robLsqIO.commit + memBlock.io.ooo_to_mem.lsqio.pendingPtr := backend.io.mem.robLsqIO.pendingPtr + memBlock.io.ooo_to_mem.lsqio.pendingPtrNext := backend.io.mem.robLsqIO.pendingPtrNext + memBlock.io.ooo_to_mem.isStoreException := backend.io.mem.isStoreException + memBlock.io.ooo_to_mem.isVlsException := backend.io.mem.isVlsException + + memBlock.io.fetch_to_mem.itlb <> frontend.io.ptw + memBlock.io.l2_hint.valid := io.l2_hint.valid + memBlock.io.l2_hint.bits.sourceId := io.l2_hint.bits.sourceId + memBlock.io.l2_tlb_req <> io.l2_tlb_req + memBlock.io.l2_pmp_resp <> io.l2_pmp_resp + memBlock.io.l2_hint.bits.isKeyword := io.l2_hint.bits.isKeyword + memBlock.io.l2PfqBusy := io.l2PfqBusy + + // if l2 prefetcher use stream prefetch, it should be placed in XSCore + + // top-down info + memBlock.io.debugTopDown.robHeadVaddr := backend.io.debugTopDown.fromRob.robHeadVaddr + frontend.io.debugTopDown.robHeadVaddr := backend.io.debugTopDown.fromRob.robHeadVaddr + io.debugTopDown.robHeadPaddr := backend.io.debugTopDown.fromRob.robHeadPaddr + io.debugTopDown.robTrueCommit := backend.io.debugRolling.robTrueCommit + backend.io.debugTopDown.fromCore.l2MissMatch := io.debugTopDown.l2MissMatch + backend.io.debugTopDown.fromCore.l3MissMatch := io.debugTopDown.l3MissMatch + backend.io.debugTopDown.fromCore.fromMem := memBlock.io.debugTopDown.toCore + memBlock.io.debugRolling := backend.io.debugRolling + + io.cpu_halt := memBlock.io.outer_cpu_halt + io.l2_flush_en := memBlock.io.outer_l2_flush_en + io.power_down_en := memBlock.io.outer_power_down_en + io.cpu_critical_error := memBlock.io.outer_cpu_critical_error + io.msiAck := memBlock.io.outer_msi_ack + io.teemsiAck zip memBlock.io.outer_teemsi_ack foreach { case (io_teemsiAck, memBlock_outer_teemsi_ack) => + io_teemsiAck := memBlock_outer_teemsi_ack + } + io.beu_errors.icache <> memBlock.io.outer_beu_errors_icache + io.beu_errors.dcache <> memBlock.io.dcacheError + io.beu_errors.uncache <> memBlock.io.uncacheError + io.beu_errors.l2 <> DontCare + io.l2PfCtrl := memBlock.io.outer_l2PfCtrl + + memBlock.io.resetInFrontendBypass.fromFrontend := frontend.io.resetInFrontend + io.resetInFrontend := memBlock.io.resetInFrontendBypass.toL2Top + memBlock.io.traceCoreInterfaceBypass.fromBackend <> backend.io.traceCoreInterface + io.traceCoreInterface <> memBlock.io.traceCoreInterfaceBypass.toL2Top + memBlock.io.wfi <> backend.io.mem.wfi + memBlock.io.topDownInfo.fromL2Top.l2Miss := io.topDownInfo.l2Miss + memBlock.io.topDownInfo.fromL2Top.l3Miss := io.topDownInfo.l3Miss + backend.io.topDownInfo.replayAllocate := memBlock.io.topDownInfo.toBackend.replayAllocate + backend.io.topDownInfo.sqFull := memBlock.io.topDownInfo.toBackend.sqFull + backend.io.topDownInfo.sbFull := memBlock.io.topDownInfo.toBackend.sbFull + backend.io.topDownInfo.l1Miss := memBlock.io.topDownInfo.toBackend.l1Miss + backend.io.topDownInfo.l2TopMiss.l2Miss := memBlock.io.topDownInfo.toBackend.l2TopMiss.l2Miss + backend.io.topDownInfo.l2TopMiss.l3Miss := memBlock.io.topDownInfo.toBackend.l2TopMiss.l3Miss + + if (debugOpts.ResetGen) { + backend.reset := memBlock.io.reset_backend + frontend.reset := backend.io.frontendReset + } + + memBlock.io.dft.zip(io.dft).foreach({ case (a, b) => a := b }) + memBlock.io.dft_reset.zip(io.dft_reset).foreach({ case (a, b) => a := b }) + frontend.io.dft.zip(memBlock.io.dft_frnt).foreach({ case (a, b) => a := b }) + frontend.io.dft_reset.zip(memBlock.io.dft_reset_frnt).foreach({ case (a, b) => a := b }) + backend.io.dft.zip(memBlock.io.dft_bcknd).foreach({ case (a, b) => a := b }) + backend.io.dft_reset.zip(memBlock.io.dft_reset_bcknd).foreach({ case (a, b) => a := b }) +} + +trait HasXSDts { + this: XSCore => + + val device: SimpleDevice = new SimpleDevice("cpu", Seq("ICT,xiangshan", "riscv")) { + override def parent: Some[Device] = Some(ResourceAnchors.cpus) + + def cpuProperties: PropertyMap = Map( + "device_type" -> "cpu".asProperty, + "status" -> "okay".asProperty, + "clock-frequency" -> 0.asProperty, + "riscv,isa" -> "rv64imafdcvh".asProperty, // deprecated + "riscv,isa-base" -> ISABase.asProperty, + "riscv,isa-extensions" -> ISAExtensions.map(ResourceString), + "timebase-frequency" -> 1000000.asProperty + ) + + def tileProperties: PropertyMap = { + val dcache = + if (coreParams.dcacheParametersOpt.nonEmpty) Map( + "d-cache-block-size" -> dcacheParameters.blockBytes.asProperty, + "d-cache-sets" -> dcacheParameters.nSets.asProperty, + "d-cache-size" -> (dcacheParameters.nSets * dcacheParameters.nWays * dcacheParameters.blockBytes).asProperty + ) + else Map() + + val icache = Map( + "i-cache-block-size" -> icacheParameters.blockBytes.asProperty, + "i-cache-sets" -> icacheParameters.nSets.asProperty, + "i-cache-size" -> (icacheParameters.nSets * icacheParameters.nWays * icacheParameters.blockBytes).asProperty + ) + + val dtlb = Map( + "d-tlb-size" -> (ldtlbParams.NSets * ldtlbParams.NWays).asProperty, + "d-tlb-sets" -> 1.asProperty + ) + + val itlb = Map( + "i-tlb-size" -> (itlbParams.NSets * itlbParams.NWays).asProperty, + "i-tlb-sets" -> 1.asProperty + ) + + val mmu = Map( + "tlb-split" -> Nil, + "mmu-type" -> s"riscv,sv$VAddrBits".asProperty + ) + + val pmp = Nil + + dcache ++ icache ++ dtlb ++ itlb ++ mmu ++ pmp + } + + def nextLevelCacheProperty: PropertyOption = { + if (coreParams.dcacheParametersOpt.nonEmpty) { + val outer = memBlock.inner.dcache.clientNode.edges.out.flatMap(_.manager.managers) + .filter(_.supportsAcquireB) + .flatMap(_.resources.headOption) + .map(_.owner.label) + .distinct + if (outer.isEmpty) None + else Some("next-level-cache" -> outer.map(l => ResourceReference(l)).toList) + } else None + } + + override def describe(resources: ResourceBindings): Description = { + val Description(name, mapping) = super.describe(resources) + Description(name, mapping ++ cpuProperties ++ nextLevelCacheProperty ++ tileProperties) + } + + } + + val intcDevice = new DeviceSnippet { + override def parent = Some(device) + + def describe(): Description = { + Description( + "interrupt-controller", + Map( + "compatible" -> "riscv,cpu-intc".asProperty, + "interrupt-controller" -> Nil, + "#interrupt-cells" -> 1.asProperty + ) + ) + } + + } + + ResourceBinding { + Resource(device, "reg").bind(ResourceAddress(coreParams.HartId)) + val int_resources = ( + memBlock.inner.clint_int_sink.edges.in.flatMap(_.source.sources) ++ + memBlock.inner.plic_int_sink.edges.in.flatMap(_.source.sources) ++ + memBlock.inner.debug_int_sink.edges.in.flatMap(_.source.sources) ++ + memBlock.inner.nmi_int_sink.edges.in.flatMap(_.source.sources) + ).flatMap { + s => + (s.range.start until s.range.`end`).map(_ => s.resources) + } + val int_ids = Seq( + 3, // msip [clint] + 7, // mtip [clint] + 11, // meip [plic] + 9, // seip [plic] + 65535, // debug [debug] + 31, // nmi_31 [nmi] + 43 // nmi_43 [nmi] + ) + assert(int_resources.size == int_ids.size) + for ((resources, id) <- int_resources.zip(int_ids)) { + for (r <- resources) { + r.bind(intcDevice, ResourceInt(id)) + } + } + } +} diff --git a/examples/prefab/cores/xiangshan/arch/native/CtrlBlock.scala b/examples/prefab/cores/xiangshan/arch/native/CtrlBlock.scala new file mode 100644 index 000000000..5a61c5e05 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/CtrlBlock.scala @@ -0,0 +1,926 @@ +// Copied from XiangShan src/main/scala/xiangshan/backend/CtrlBlock.scala and modified for Buckyball. +// XiangShan is licensed under Mulan PSL v2 (http://license.coscl.org.cn/MulanPSL2). + +package examples.core.xiangshan.native + +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import chisel3.util._ +import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp} +import utility._ +import utils._ +import xiangshan.ExceptionNO._ +import xiangshan._ +import xiangshan.backend.Bundles.{DecodedInst, DynInst, ExceptionInfo, ExuOutput, ExuVec, StaticInst, TrapInstInfo} +import xiangshan.backend.ctrlblock.{DebugLSIO, DebugLsInfoBundle, LsTopdownInfo, MemCtrl, RedirectGenerator} +import xiangshan.backend.datapath.DataConfig.{FpData, IntData, V0Data, VAddrData, VecData, VlData} +import xiangshan.backend.decode.FusionDecoder +import xiangshan.backend.dispatch.{CoreDispatchTopDownIO} +import xiangshan.backend.dispatch.NewDispatch +import xiangshan.backend.fu.vector.Bundles.{VType, Vl} +import xiangshan.backend.fu.wrapper.CSRToDecode +import xiangshan.backend.rename.{Rename, RenameTableWrapper, SnapshotGenerator} +import xiangshan.backend.rob.{Rob, RobCSRIO, RobCoreTopDownIO, RobDebugRollingIO, RobLsqIO, RobPtr} +import xiangshan.frontend.{FtqPcMemEntry, FtqPtr, FtqRead} +import xiangshan.mem.{LqPtr, LsqEnqIO, SqPtr} +import xiangshan.backend.issue.{FpScheduler, IntScheduler, MemScheduler, VfScheduler} +import xiangshan.backend.trace._ +import xiangshan.Redirect.findOldestRedirect +import xiangshan.backend._ +import examples.core.xiangshan.native.{DecodeStage} + +class CtrlBlock(params: BackendParams)(implicit p: Parameters) extends LazyModule { + override def shouldBeInlined: Boolean = false + + val rob = LazyModule(new Rob(params)) + + lazy val module = new CtrlBlockImp(this)(p, params) + + val gpaMem = LazyModule(new GPAMem()) +} + +class CtrlBlockImp( + override val wrapper: CtrlBlock +)( + implicit + p: Parameters, + params: BackendParams) + extends LazyModuleImp(wrapper) + with HasXSParameter + with HasCircularQueuePtrHelper + with HasPerfEvents + with HasCriticalErrors { + + val pcMemRdIndexes = new NamedIndexes(Seq( + "redirect" -> 1, + "memPred" -> 1, + "robFlush" -> 1, + "executePc" -> params.numPcMemReadPort, + "bjuTarget" -> params.BrhCnt, + "hybrid" -> params.HyuCnt, + "store" -> (if (EnableStorePrefetchSMS) params.StaCnt else 0), + "trace" -> TraceGroupNum + )) + + private val numPcMemRead = pcMemRdIndexes.maxIdx + + // now pcMem read for exu is moved to PcTargetMem (OG0) + println(s"pcMem read num: $numPcMemRead") + + val io = IO(new CtrlBlockIO()) + + val dispatch = Module(new NewDispatch) + val gpaMem = wrapper.gpaMem.module + val decode = Module(new DecodeStage) + val fusionDecoder = Module(new FusionDecoder) + val rat = Module(new RenameTableWrapper) + val rename = Module(new Rename) + val redirectGen = Module(new RedirectGenerator) + private def hasRen: Boolean = true + private val pcMem = + Module(new SyncDataModuleTemplate(new FtqPcMemEntry, FtqSize, numPcMemRead, 1, "BackendPC", hasRen = hasRen)) + private val rob = wrapper.rob.module + private val memCtrl = Module(new MemCtrl(params)) + + private val disableFusion = decode.io.csrCtrl.singlestep || !decode.io.csrCtrl.fusion_enable + + private val s0_robFlushRedirect = rob.io.flushOut + private val s1_robFlushRedirect = Wire(Valid(new Redirect)) + s1_robFlushRedirect.valid := GatedValidRegNext(s0_robFlushRedirect.valid, false.B) + s1_robFlushRedirect.bits := RegEnable(s0_robFlushRedirect.bits, s0_robFlushRedirect.valid) + + pcMem.io.ren.get(pcMemRdIndexes("robFlush").head) := s0_robFlushRedirect.valid + pcMem.io.raddr(pcMemRdIndexes("robFlush").head) := s0_robFlushRedirect.bits.ftqIdx.value + + private val s1_robFlushPc = pcMem.io.rdata(pcMemRdIndexes("robFlush").head).startAddr + (RegEnable( + s0_robFlushRedirect.bits.ftqOffset, + s0_robFlushRedirect.valid + ) << instOffsetBits) + + private val s3_redirectGen = redirectGen.io.stage2Redirect + private val s1_s3_redirect = Mux(s1_robFlushRedirect.valid, s1_robFlushRedirect, s3_redirectGen) + private val s2_s4_pendingRedirectValid = RegInit(false.B) + when(s1_s3_redirect.valid) { + s2_s4_pendingRedirectValid := true.B + }.elsewhen(GatedValidRegNext(io.frontend.toFtq.redirect.valid)) { + s2_s4_pendingRedirectValid := false.B + } + + // Redirect will be RegNext at ExuBlocks and IssueBlocks + val s2_s4_redirect = RegNextWithEnable(s1_s3_redirect) + val s3_s5_redirect = RegNextWithEnable(s2_s4_redirect) + + private val delayedNotFlushedWriteBack = io.fromWB.wbData.map { x => + val valid = x.valid + val killedByOlder = x.bits.robIdx.needFlush(Seq(s1_s3_redirect, s2_s4_redirect)) + val delayed = Wire(Valid(new ExuOutput(x.bits.params))) + delayed.valid := GatedValidRegNext(valid && !killedByOlder) + delayed.bits := RegEnable(x.bits, x.valid) + delayed.bits.debugInfo.writebackTime := GTimer() + delayed + }.toSeq + + private val delayedWriteBack = Wire(chiselTypeOf(io.fromWB.wbData)) + delayedWriteBack.zipWithIndex.map { case (x, i) => + x.valid := GatedValidRegNext(io.fromWB.wbData(i).valid) + x.bits := delayedNotFlushedWriteBack(i).bits + } + val delayedNotFlushedWriteBackNeedFlush = Wire(Vec(params.allExuParams.filter(_.needExceptionGen).length, Bool())) + delayedNotFlushedWriteBackNeedFlush := delayedNotFlushedWriteBack.filter(_.bits.params.needExceptionGen).map { x => + x.bits.exceptionVec.get.asUInt.orR || x.bits.flushPipe.getOrElse(false.B) || x.bits.replay.getOrElse(false.B) || + (if (x.bits.trigger.nonEmpty) TriggerAction.isDmode(x.bits.trigger.get) else false.B) + } + + val wbDataNoStd = io.fromWB.wbData.filter(!_.bits.params.hasStdFu) + val intScheWbData = io.fromWB.wbData.filter(_.bits.params.schdType.isInstanceOf[IntScheduler]) + val fpScheWbData = io.fromWB.wbData.filter(_.bits.params.schdType.isInstanceOf[FpScheduler]) + val vfScheWbData = io.fromWB.wbData.filter(_.bits.params.schdType.isInstanceOf[VfScheduler]) + val intCanCompress = intScheWbData.filter(_.bits.params.CanCompress) + val i2vWbData = intScheWbData.filter(_.bits.params.writeVecRf) + val f2vWbData = fpScheWbData.filter(_.bits.params.writeVecRf) + val memVloadWbData = + io.fromWB.wbData.filter(x => x.bits.params.schdType.isInstanceOf[MemScheduler] && x.bits.params.hasVLoadFu) + + private val delayedNotFlushedWriteBackNums = wbDataNoStd.map { x => + val valid = x.valid + val oldestRedirect = findOldestRedirect(findOldestRedirect(s2_s4_redirect, s3_s5_redirect), s1_s3_redirect) + val killedByOlder = x.bits.robIdx.needFlush(oldestRedirect) + val delayed = Wire(Valid(UInt(io.fromWB.wbData.size.U.getWidth.W))) + delayed.valid := GatedValidRegNext(valid && !killedByOlder) + val isIntSche = intCanCompress.contains(x) + val isFpSche = fpScheWbData.contains(x) + val isVfSche = vfScheWbData.contains(x) + val isMemVload = memVloadWbData.contains(x) + val isi2v = i2vWbData.contains(x) + val isf2v = f2vWbData.contains(x) + val canSameRobidxWbData = + if (isVfSche) { + i2vWbData ++ f2vWbData ++ vfScheWbData + } else if (isi2v) { + intCanCompress ++ fpScheWbData ++ vfScheWbData + } else if (isf2v) { + intCanCompress ++ fpScheWbData ++ vfScheWbData + } else if (isIntSche) { + intCanCompress ++ fpScheWbData + } else if (isFpSche) { + intCanCompress ++ fpScheWbData + } else if (isMemVload) { + memVloadWbData + } else { + Seq(x) + } + val sameRobidxBools = VecInit(canSameRobidxWbData.map { wb => + val killedByOlderThat = wb.bits.robIdx.needFlush(oldestRedirect) + (wb.bits.robIdx === x.bits.robIdx) && wb.valid && x.valid && !killedByOlderThat && !killedByOlder + }.toSeq) + delayed.bits := RegEnable(PopCount(sameRobidxBools), x.valid) + delayed + }.toSeq + + private val exuPredecode = VecInit( + io.fromWB.wbData.filter(_.bits.redirect.nonEmpty).map(x => x.bits.predecodeInfo.get).toSeq + ) + + private val exuRedirects: Seq[ValidIO[Redirect]] = io.fromWB.wbData.filter(_.bits.redirect.nonEmpty).map { x => + val hasCSR = x.bits.params.hasCSR + val out = Wire(Valid(new Redirect())) + out.valid := x.valid && x.bits.redirect.get.valid && !x.bits.robIdx.needFlush(Seq(s1_s3_redirect, s2_s4_redirect)) + out.bits := x.bits.redirect.get.bits + out.bits.debugIsCtrl := true.B + out.bits.debugIsMemVio := false.B + // for fix timing, next cycle assgin + if (!hasCSR) { + out.bits.cfiUpdate.backendIAF := false.B + out.bits.cfiUpdate.backendIPF := false.B + out.bits.cfiUpdate.backendIGPF := false.B + } + out + }.toSeq + + private val oldestOneHot = Redirect.selectOldestRedirect(exuRedirects) + private val CSROH = VecInit(io.fromWB.wbData.filter(_.bits.redirect.nonEmpty).map(x => x.bits.params.hasCSR.B)) + private val oldestExuRedirectIsCSR = oldestOneHot === CSROH + private val oldestExuRedirect = Mux1H(oldestOneHot, exuRedirects) + private val oldestExuPredecode = Mux1H(oldestOneHot, exuPredecode) + + private val memViolation = io.fromMem.violation + val loadReplay = Wire(ValidIO(new Redirect)) + loadReplay.valid := GatedValidRegNext(memViolation.valid) + loadReplay.bits := RegEnable(memViolation.bits, memViolation.valid) + loadReplay.bits.debugIsCtrl := false.B + loadReplay.bits.debugIsMemVio := true.B + + pcMem.io.ren.get(pcMemRdIndexes("redirect").head) := memViolation.valid + pcMem.io.raddr(pcMemRdIndexes("redirect").head) := memViolation.bits.ftqIdx.value + pcMem.io.ren.get(pcMemRdIndexes("memPred").head) := memViolation.valid + pcMem.io.raddr(pcMemRdIndexes("memPred").head) := memViolation.bits.stFtqIdx.value + redirectGen.io.memPredPcRead.data := pcMem.io.rdata(pcMemRdIndexes("memPred").head).startAddr + (RegEnable( + memViolation.bits.stFtqOffset, + memViolation.valid + ) << instOffsetBits) + + for ((pcMemIdx, i) <- pcMemRdIndexes("executePc").zipWithIndex) { + val ren = io.toDataPath.pcToDataPathIO.fromDataPathValid(i) + val raddr = io.toDataPath.pcToDataPathIO.fromDataPathFtqPtr(i).value + val roffset = io.toDataPath.pcToDataPathIO.fromDataPathFtqOffset(i) + pcMem.io.ren.get(pcMemIdx) := ren + pcMem.io.raddr(pcMemIdx) := raddr + io.toDataPath.pcToDataPathIO.toDataPathPC(i) := pcMem.io.rdata(pcMemIdx).startAddr + } + + val newestEn = RegNext(io.frontend.fromFtq.newest_entry_en) + val newestTarget = RegEnable(io.frontend.fromFtq.newest_entry_target, io.frontend.fromFtq.newest_entry_en) + val newestPtr = RegEnable(io.frontend.fromFtq.newest_entry_ptr, io.frontend.fromFtq.newest_entry_en) + val newestTargetNext = RegEnable(newestTarget, newestEn) + for ((pcMemIdx, i) <- pcMemRdIndexes("bjuTarget").zipWithIndex) { + val ren = io.toDataPath.pcToDataPathIO.fromDataPathValid(i) + val baseAddr = io.toDataPath.pcToDataPathIO.fromDataPathFtqPtr(i).value + val raddr = io.toDataPath.pcToDataPathIO.fromDataPathFtqPtr(i).value + 1.U + pcMem.io.ren.get(pcMemIdx) := ren + pcMem.io.raddr(pcMemIdx) := raddr + val needNewest = RegNext(baseAddr === newestPtr.value) + io.toDataPath.pcToDataPathIO.toDataPathTargetPC(i) := Mux( + needNewest, + newestTargetNext, + pcMem.io.rdata(pcMemIdx).startAddr + ) + } + + for ((pcMemIdx, i) <- pcMemRdIndexes("hybrid").zipWithIndex) { + // load read pcMem (s0) -> get rdata (s1) -> reg next in Memblock (s2) -> reg next in Memblock (s3) -> consumed by + // pf (s3) + pcMem.io.ren.get(pcMemIdx) := io.memHyPcRead(i).valid + pcMem.io.raddr(pcMemIdx) := io.memHyPcRead(i).ptr.value + io.memHyPcRead(i).data := pcMem.io.rdata(pcMemIdx).startAddr + (RegEnable( + io.memHyPcRead(i).offset, + io.memHyPcRead(i).valid + ) << instOffsetBits) + } + + if (EnableStorePrefetchSMS) { + for ((pcMemIdx, i) <- pcMemRdIndexes("store").zipWithIndex) { + pcMem.io.ren.get(pcMemIdx) := io.memStPcRead(i).valid + pcMem.io.raddr(pcMemIdx) := io.memStPcRead(i).ptr.value + io.memStPcRead(i).data := pcMem.io.rdata(pcMemIdx).startAddr + (RegEnable( + io.memStPcRead(i).offset, + io.memStPcRead(i).valid + ) << instOffsetBits) + } + } else { + io.memStPcRead.foreach(_.data := 0.U) + } + + /** + * trace begin + */ + val trace = Module(new Trace) + trace.io.in.fromEncoder.stall := io.traceCoreInterface.fromEncoder.stall + trace.io.in.fromEncoder.enable := io.traceCoreInterface.fromEncoder.enable + trace.io.in.fromRob := rob.io.trace.traceCommitInfo + rob.io.trace.blockCommit := trace.io.out.blockRobCommit + val tracePcStart = Wire(Vec(TraceGroupNum, UInt(IaddrWidth.W))) + for ((pcMemIdx, i) <- pcMemRdIndexes("trace").zipWithIndex) { + val traceValid = trace.toPcMem.blocks(i).valid + pcMem.io.ren.get(pcMemIdx) := traceValid + pcMem.io.raddr(pcMemIdx) := trace.toPcMem.blocks(i).bits.ftqIdx.get.value + tracePcStart(i) := pcMem.io.rdata(pcMemIdx).startAddr + } + + // Trap/Xret only occur in block(0). + val tracePriv = Mux( + Itype.isTrapOrXret(trace.toEncoder.blocks(0).bits.tracePipe.itype), + io.fromCSR.traceCSR.lastPriv, + io.fromCSR.traceCSR.currentPriv + ) + + io.traceCoreInterface.toEncoder.trap.cause := io.fromCSR.traceCSR.cause + io.traceCoreInterface.toEncoder.trap.tval := io.fromCSR.traceCSR.tval + io.traceCoreInterface.toEncoder.priv := tracePriv + io.traceCoreInterface.toEncoder.mstatus := io.fromCSR.traceCSR.mstatus + (0 until TraceGroupNum).foreach { i => + io.traceCoreInterface.toEncoder.groups(i).valid := trace.io.out.toEncoder.blocks(i).valid + io.traceCoreInterface.toEncoder.groups(i).bits.iaddr := tracePcStart(i) + io.traceCoreInterface.toEncoder.groups(i).bits.ftqOffset.foreach( + _ := trace.io.out.toEncoder.blocks(i).bits.ftqOffset.getOrElse(0.U) + ) + io.traceCoreInterface.toEncoder.groups(i).bits.itype := trace.io.out.toEncoder.blocks(i).bits.tracePipe.itype + io.traceCoreInterface.toEncoder.groups(i).bits.iretire := trace.io.out.toEncoder.blocks(i).bits.tracePipe.iretire + io.traceCoreInterface.toEncoder.groups(i).bits.ilastsize := trace.io.out.toEncoder.blocks( + i + ).bits.tracePipe.ilastsize + } + + /** + * trace end + */ + + redirectGen.io.hartId := io.fromTop.hartId + redirectGen.io.oldestExuRedirect.valid := RegNext(oldestExuRedirect.valid) + redirectGen.io.oldestExuRedirect.bits := RegNext(oldestExuRedirect.bits) + redirectGen.io.oldestExuRedirectIsCSR := RegNext(oldestExuRedirectIsCSR) + redirectGen.io.instrAddrTransType := RegNext(io.fromCSR.instrAddrTransType) + redirectGen.io.oldestExuOutPredecode.valid := RegNext(oldestExuPredecode.valid) + redirectGen.io.oldestExuOutPredecode := RegNext(oldestExuPredecode) + redirectGen.io.loadReplay <> loadReplay + val loadRedirectOffset = Mux(memViolation.bits.flushItself(), 0.U, Mux(memViolation.bits.isRVC, 2.U, 4.U)) + val loadRedirectPcFtqOffset = + RegEnable((memViolation.bits.ftqOffset << instOffsetBits).asUInt +& loadRedirectOffset, memViolation.valid) + val loadRedirectPcRead = pcMem.io.rdata(pcMemRdIndexes("redirect").head).startAddr + loadRedirectPcFtqOffset + + redirectGen.io.loadReplay.bits.cfiUpdate.pc := loadRedirectPcRead + val load_target = loadRedirectPcRead + redirectGen.io.loadReplay.bits.cfiUpdate.target := SignExt(load_target, VAddrBits + 1) + + redirectGen.io.robFlush := s1_robFlushRedirect + + val s5_flushFromRobValidAhead = DelayN(s1_robFlushRedirect.valid, 4) + val s6_flushFromRobValid = GatedValidRegNext(s5_flushFromRobValidAhead) + val frontendFlushBits = RegEnable(s1_robFlushRedirect.bits, s1_robFlushRedirect.valid) // ?? + // When ROB commits an instruction with a flush, we notify the frontend of the flush without the commit. + // Flushes to frontend may be delayed by some cycles and commit before flush causes errors. + // Thus, we make all flush reasons to behave the same as exceptions for frontend. + for (i <- 0 until CommitWidth) { + // why flushOut: instructions with flushPipe are not commited to frontend + // If we commit them to frontend, it will cause flush after commit, which is not acceptable by frontend. + val s1_isCommit = rob.io.commits.commitValid(i) && rob.io.commits.isCommit && !s0_robFlushRedirect.valid + io.frontend.toFtq.rob_commits(i).valid := GatedValidRegNext(s1_isCommit) + io.frontend.toFtq.rob_commits(i).bits := RegEnable(rob.io.commits.info(i), s1_isCommit) + } + io.frontend.toFtq.redirect.valid := s6_flushFromRobValid || s3_redirectGen.valid + io.frontend.toFtq.redirect.bits := Mux(s6_flushFromRobValid, frontendFlushBits, s3_redirectGen.bits) + io.frontend.toFtq.ftqIdxSelOH.valid := s6_flushFromRobValid || redirectGen.io.stage2Redirect.valid + io.frontend.toFtq.ftqIdxSelOH.bits := Cat( + s6_flushFromRobValid, + redirectGen.io.stage2oldestOH & Fill(NumRedirect + 1, !s6_flushFromRobValid) + ) + + //jmp/brh, sel oldest first, only use one read port + io.frontend.toFtq.ftqIdxAhead(0).valid := RegNext( + oldestExuRedirect.valid + ) && !s1_robFlushRedirect.valid && !s5_flushFromRobValidAhead + io.frontend.toFtq.ftqIdxAhead(0).bits := RegEnable(oldestExuRedirect.bits.ftqIdx, oldestExuRedirect.valid) + //loadreplay + io.frontend.toFtq.ftqIdxAhead( + NumRedirect + ).valid := loadReplay.valid && !s1_robFlushRedirect.valid && !s5_flushFromRobValidAhead + io.frontend.toFtq.ftqIdxAhead(NumRedirect).bits := loadReplay.bits.ftqIdx + //exception + io.frontend.toFtq.ftqIdxAhead.last.valid := s5_flushFromRobValidAhead + io.frontend.toFtq.ftqIdxAhead.last.bits := frontendFlushBits.ftqIdx + + // Be careful here: + // T0: rob.io.flushOut, s0_robFlushRedirect + // T1: s1_robFlushRedirect, rob.io.exception.valid + // T2: csr.redirect.valid + // T3: csr.exception.valid + // T4: csr.trapTarget + // T5: ctrlBlock.trapTarget + // T6: io.frontend.toFtq.stage2Redirect.valid + val s2_robFlushPc = RegEnable( + Mux( + s1_robFlushRedirect.bits.flushItself(), + s1_robFlushPc, // replay inst + s1_robFlushPc + Mux(s1_robFlushRedirect.bits.isRVC, 2.U, 4.U) // flush pipe + ), + s1_robFlushRedirect.valid + ) + + private val s5_csrIsTrap = DelayN(rob.io.exception.valid, 4) + private val s5_trapTargetFromCsr = io.robio.csr.trapTarget + + val flushTarget = Mux(s5_csrIsTrap, s5_trapTargetFromCsr.pc, SignExt(s2_robFlushPc, XLEN)) + val s5_trapTargetIAF = Mux(s5_csrIsTrap, s5_trapTargetFromCsr.raiseIAF, false.B) + val s5_trapTargetIPF = Mux(s5_csrIsTrap, s5_trapTargetFromCsr.raiseIPF, false.B) + val s5_trapTargetIGPF = Mux(s5_csrIsTrap, s5_trapTargetFromCsr.raiseIGPF, false.B) + when(s6_flushFromRobValid) { + io.frontend.toFtq.redirect.bits.level := RedirectLevel.flush + io.frontend.toFtq.redirect.bits.cfiUpdate.target := RegEnable(flushTarget, s5_flushFromRobValidAhead) + io.frontend.toFtq.redirect.bits.cfiUpdate.backendIAF := RegEnable(s5_trapTargetIAF, s5_flushFromRobValidAhead) + io.frontend.toFtq.redirect.bits.cfiUpdate.backendIPF := RegEnable(s5_trapTargetIPF, s5_flushFromRobValidAhead) + io.frontend.toFtq.redirect.bits.cfiUpdate.backendIGPF := RegEnable(s5_trapTargetIGPF, s5_flushFromRobValidAhead) + } + + for (i <- 0 until DecodeWidth) { + gpaMem.io.fromIFU := io.frontend.fromIfu + gpaMem.io.exceptionReadAddr.valid := rob.io.readGPAMemAddr.valid + gpaMem.io.exceptionReadAddr.bits.ftqPtr := rob.io.readGPAMemAddr.bits.ftqPtr + gpaMem.io.exceptionReadAddr.bits.ftqOffset := rob.io.readGPAMemAddr.bits.ftqOffset + } + + // vtype commit + decode.io.fromCSR := io.fromCSR.toDecode + decode.io.fromRob.isResumeVType := rob.io.toDecode.isResumeVType + decode.io.fromRob.walkToArchVType := rob.io.toDecode.walkToArchVType + decode.io.fromRob.commitVType := rob.io.toDecode.commitVType + decode.io.fromRob.walkVType := rob.io.toDecode.walkVType + + decode.io.redirect := s1_s3_redirect.valid || s2_s4_pendingRedirectValid + + // add decode Buf for in.ready better timing + /** + * Decode buffer: when decode.io.in cannot accept all insts, use this buffer to temporarily store insts that cannot + * be sent to DecodeStage. + * + * Decode buffer is a "DecodeWidth"-element long register Vector of StaticInst (in decodeBufBits), with valid signals + * (in decodeBufValid). At the same time, fetch insts input from frontend and their valid bits. All valid elements + * in these two vector of insts are at the beginning, with all invalid vector elements followed. + * + * After dealing with redirection, try to use all insts in decode buffer to fulfill decoder.io.in. If decode buffer + * has no valid insts, use insts from frontend to supply decoder. + */ + + /** Insts to be decoded, Registers in vector of DecodeWidth */ + val decodeBufBits = Reg(Vec(DecodeWidth, new StaticInst)) + + /** Valid receiving signals of instructions to be decoded, Registers in vector of DecodeWidth */ + val decodeBufValid = RegInit(VecInit(Seq.fill(DecodeWidth)(false.B))) + + /** Insts input from frontend, in vector of DecodeWidth */ + val decodeFromFrontend = io.frontend.cfVec + + /** Insts in buffer that is not ready but valid in decodeBufValid */ + val decodeBufNotAccept = VecInit(decodeBufValid.zip(decode.io.in).map(x => x._1 && !x._2.ready)) + + /** Number of insts in decode buffer that is accepted. All accepted insts are before the first unaccepted one. */ + val decodeBufAcceptNum = PriorityMuxDefault(decodeBufNotAccept.zip(Seq.tabulate(DecodeWidth)(i => i.U)), DecodeWidth.U) + + /** Input valid insts from frontend that is not ready to be accepted, or decoder prefer insts in decode buffer */ + val decodeFromFrontendNotAccept = + VecInit(decodeFromFrontend.zip(decode.io.in).map(x => decodeBufValid(0) || x._1.valid && !x._2.ready)) + + /** + * Number of input insts that is accepted. + * All accepted insts are before the first unaccepted one. + */ + val decodeFromFrontendAcceptNum = + PriorityMuxDefault(decodeFromFrontendNotAccept.zip(Seq.tabulate(DecodeWidth)(i => i.U)), DecodeWidth.U) + + if (backendParams.debugEn) { + dontTouch(decodeBufNotAccept) + dontTouch(decodeBufAcceptNum) + dontTouch(decodeFromFrontendNotAccept) + dontTouch(decodeFromFrontendAcceptNum) + } + + /** + * State machine of "decodeBufValid(i)": + * redirect || decodeBufValid(i) is the last accepted instr in decodeBuf: + * false + * decodeBufValid(i) is true, decodeBufNotAccept.drop(i) has some true signals + * (decodeBufAcceptNum > DecodeWidth-1-i) ? false + * if not : decodeBufValid(i+decodeBufAcceptNum) + * Pop "decodeBufAcceptNum" insts out of the decodeBufValid, and move others forward + * decodeBufValid(0) is false, decodeFromFrontendNotAccept.drop(i) has some true signals + * (decodeFromFrontendAcceptNum > DecodeWidth-1-i) ? false + * if not : decodeFromFrontend(i+decodeFromFrontendAcceptNum).valid + * Get first "decodeFromFrontendAcceptNum" insts from decodeFromFrontend, and move others to decodeBufValid + * + * State machine of "decodeBufBits(i)": + * decodeBufValid(i) is true, decodeBufNotAccept.drop(i) has some true signals + * decodeBufBits(i+decodeBufAcceptNum) + * decodeBufValid(0) is false, decodeFromFrontendNotAccept.drop(i) has some true signals + * decodeFromFrontend(i+decodeFromFrontendAcceptNum) + */ + for (i <- 0 until DecodeWidth) { + // decodeBufValid update + when(decode.io.redirect || decodeBufValid(0) && decodeBufValid(i) && decode.io.in(i).ready && !VecInit( + decodeBufNotAccept.drop(i) + ).asUInt.orR) { + decodeBufValid(i) := false.B + }.elsewhen(decodeBufValid(i) && VecInit(decodeBufNotAccept.drop(i)).asUInt.orR) { + decodeBufValid(i) := Mux( + decodeBufAcceptNum > DecodeWidth.U - 1.U - i.U, + false.B, + decodeBufValid(i.U + decodeBufAcceptNum) + ) + }.elsewhen(!decodeBufValid(0) && VecInit(decodeFromFrontendNotAccept.drop(i)).asUInt.orR) { + decodeBufValid(i) := Mux( + decodeFromFrontendAcceptNum > DecodeWidth.U - 1.U - i.U, + false.B, + decodeFromFrontend(i.U + decodeFromFrontendAcceptNum).valid + ) + } + // decodeBufBits update + when(decodeBufValid(i) && VecInit(decodeBufNotAccept.drop(i)).asUInt.orR) { + decodeBufBits(i) := decodeBufBits(i.U + decodeBufAcceptNum) + }.elsewhen(!decodeBufValid(0) && VecInit(decodeFromFrontendNotAccept.drop(i)).asUInt.orR) { + decodeBufBits(i).connectCtrlFlow(decodeFromFrontend(i.U + decodeFromFrontendAcceptNum).bits) + } + } + + /** Insts input from frontend, in vector of DecodeWidth */ + val decodeConnectFromFrontend = Wire(Vec(DecodeWidth, new StaticInst)) + decodeConnectFromFrontend.zip(decodeFromFrontend).map(x => x._1.connectCtrlFlow(x._2.bits)) + + /** + * DecodeStage's input: + * decode.io.in(i).valid: + * decodeBufValid(0) is true : decodeBufValid(i) | from decode buffer + * false : decodeFromFrontend(i).valid | from frontend + * + * decodeFromFrontend(i).ready: + * decodeFromFrontend(0).valid && !decodeBufValid(0) && decodeFromFrontend(i).valid && !decode.io.redirect + * valid instr in input, no instr in decode buffer, decodeFromFrontend(i) is valid, no redirection + * + * decode.io.in(i).bits: + * decodeBufValid(i) is true : decodeBufBits(i) | from decode buffer + * false : decodeConnectFromFrontend(i) | from frontend + */ + decode.io.in.zipWithIndex.foreach { case (decodeIn, i) => + decodeIn.valid := Mux(decodeBufValid(0), decodeBufValid(i), decodeFromFrontend(i).valid) + decodeFromFrontend(i).ready := decodeFromFrontend(0).valid && !decodeBufValid(0) && decodeFromFrontend( + i + ).valid && !decode.io.redirect + decodeIn.bits := Mux(decodeBufValid(i), decodeBufBits(i), decodeConnectFromFrontend(i)) + } + + /** no valid instr in decode buffer && no valid instr from frontend --> can accept new instr from frontend */ + io.frontend.canAccept := !decodeBufValid(0) || !decodeFromFrontend(0).valid + decode.io.csrCtrl := RegNext(io.csrCtrl) + decode.io.intRat <> rat.io.intReadPorts + decode.io.fpRat <> rat.io.fpReadPorts + decode.io.vecRat <> rat.io.vecReadPorts + decode.io.v0Rat <> rat.io.v0ReadPorts + decode.io.vlRat <> rat.io.vlReadPorts + decode.io.fusion := 0.U.asTypeOf(decode.io.fusion) // Todo + decode.io.stallReason.in <> io.frontend.stallReason + decode.io.backendCanAccept := io.frontend.canAccept + + // snapshot check + class CFIRobIdx extends Bundle { + val robIdx = Vec(RenameWidth, new RobPtr) + val isCFI = Vec(RenameWidth, Bool()) + } + + val genSnapshot = Cat(rename.io.out.map(out => out.fire && out.bits.snapshot)).orR + val snpt = Module(new SnapshotGenerator(0.U.asTypeOf(new CFIRobIdx))) + snpt.io.enq := genSnapshot + snpt.io.enqData.robIdx := rename.io.out.map(_.bits.robIdx) + snpt.io.enqData.isCFI := rename.io.out.map(_.bits.snapshot) + snpt.io.deq := snpt.io.valids(snpt.io.deqPtr.value) && rob.io.commits.isCommit && + Cat(rob.io.commits.commitValid.zip(rob.io.commits.robIdx).map(x => + x._1 && x._2 === snpt.io.snapshots(snpt.io.deqPtr.value).robIdx.head + )).orR + snpt.io.redirect := s1_s3_redirect.valid + + val flushVec = VecInit(snpt.io.snapshots.map { snapshot => + val notCFIMask = snapshot.isCFI.map(~_) + val shouldFlush = snapshot.robIdx.map(robIdx => + robIdx >= s1_s3_redirect.bits.robIdx || robIdx.value === s1_s3_redirect.bits.robIdx.value + ) + val shouldFlushMask = (1 to RenameWidth).map(shouldFlush take _ reduce (_ || _)) + s1_s3_redirect.valid && Cat(shouldFlushMask.zip(notCFIMask).map(x => x._1 | x._2)).andR + }) + + val flushVecNext = flushVec zip snpt.io.valids map (x => GatedValidRegNext(x._1 && x._2, false.B)) + snpt.io.flushVec := flushVecNext + + val redirectRobidx = s1_s3_redirect.bits.robIdx + + val useSnpt = VecInit.tabulate(RenameSnapshotNum) { case idx => + val snptRobidx = snpt.io.snapshots(idx).robIdx.head + // (redirectRobidx.value =/= snptRobidx.value) for only flag diffrence + snpt.io.valids(idx) && ((redirectRobidx > snptRobidx) && (redirectRobidx.value =/= snptRobidx.value) || + !s1_s3_redirect.bits.flushItself() && redirectRobidx === snptRobidx) + }.reduceTree(_ || _) + + val snptSelect = MuxCase( + 0.U(log2Ceil(RenameSnapshotNum).W), + (1 to RenameSnapshotNum).map(i => (snpt.io.enqPtr - i.U).value).map { case idx => + val thisSnapRobidx = snpt.io.snapshots(idx).robIdx.head + ( + snpt.io.valids(idx) && (redirectRobidx > thisSnapRobidx && (redirectRobidx.value =/= thisSnapRobidx.value) || + !s1_s3_redirect.bits.flushItself() && redirectRobidx === thisSnapRobidx), + idx + ) + } + ) + + rob.io.snpt.snptEnq := DontCare + rob.io.snpt.snptDeq := snpt.io.deq + rob.io.snpt.useSnpt := useSnpt + rob.io.snpt.snptSelect := snptSelect + rob.io.snpt.flushVec := flushVecNext + rat.io.snpt.snptEnq := genSnapshot + rat.io.snpt.snptDeq := snpt.io.deq + rat.io.snpt.useSnpt := useSnpt + rat.io.snpt.snptSelect := snptSelect + rat.io.snpt.flushVec := flushVec + + val decodeHasException = + decode.io.out.map(x => x.bits.exceptionVec.asUInt.orR || (!TriggerAction.isNone(x.bits.trigger))) + // fusion decoder + fusionDecoder.io.disableFusion := disableFusion + for (i <- 0 until DecodeWidth) { + fusionDecoder.io.in(i).valid := decode.io.out(i).valid && !decodeHasException(i) + fusionDecoder.io.in(i).bits := decode.io.out(i).bits.instr + if (i > 0) { + fusionDecoder.io.inReady(i - 1) := decode.io.out(i).ready + } + } + + private val decodePipeRename = Wire(Vec(RenameWidth, DecoupledIO(new DecodedInst))) + for (i <- 0 until RenameWidth) { + PipelineConnect( + decode.io.out(i), + decodePipeRename(i), + rename.io.in(i).ready, + s1_s3_redirect.valid || s2_s4_pendingRedirectValid, + moduleName = Some("decodePipeRenameModule") + ) + + decodePipeRename(i).ready := rename.io.in(i).ready + rename.io.in(i).valid := decodePipeRename(i).valid && !fusionDecoder.io.clear(i) + rename.io.in(i).bits := decodePipeRename(i).bits + dispatch.io.renameIn(i).valid := decodePipeRename(i).valid && !fusionDecoder.io.clear(i) && !decodePipeRename( + i + ).bits.isMove + dispatch.io.renameIn(i).bits := decodePipeRename(i).bits + } + + for (i <- 0 until RenameWidth - 1) { + fusionDecoder.io.dec(i) := decodePipeRename(i).bits + rename.io.fusionInfo(i) := fusionDecoder.io.info(i) + + // update the first RenameWidth - 1 instructions + decode.io.fusion(i) := fusionDecoder.io.out(i).valid && rename.io.out(i).fire + // TODO: remove this dirty code for ftq update + val sameFtqPtr = rename.io.in(i).bits.ftqPtr.value === rename.io.in(i + 1).bits.ftqPtr.value + val ftqOffset0 = rename.io.in(i).bits.ftqOffset + val ftqOffset1 = rename.io.in(i + 1).bits.ftqOffset + val ftqOffsetDiff = ftqOffset1 - ftqOffset0 + val cond1 = sameFtqPtr && ftqOffsetDiff === 1.U + val cond2 = sameFtqPtr && ftqOffsetDiff === 2.U + val cond3 = !sameFtqPtr && ftqOffset1 === 0.U + val cond4 = !sameFtqPtr && ftqOffset1 === 1.U + when(fusionDecoder.io.out(i).valid) { + fusionDecoder.io.out(i).bits.update(rename.io.in(i).bits) + fusionDecoder.io.out(i).bits.update(dispatch.io.renameIn(i).bits) + rename.io.in(i).bits.commitType := Mux(cond1, 4.U, Mux(cond2, 5.U, Mux(cond3, 6.U, 7.U))) + } + XSError( + fusionDecoder.io.out(i).valid && !cond1 && !cond2 && !cond3 && !cond4, + p"new condition $sameFtqPtr $ftqOffset0 $ftqOffset1\n" + ) + } + + // memory dependency predict + // when decode, send fold pc to mdp + private val mdpFlodPcVecVld = Wire(Vec(DecodeWidth, Bool())) + private val mdpFlodPcVec = Wire(Vec(DecodeWidth, UInt(MemPredPCWidth.W))) + for (i <- 0 until DecodeWidth) { + mdpFlodPcVecVld(i) := decode.io.out(i).fire || GatedValidRegNext(decode.io.out(i).fire) + mdpFlodPcVec(i) := Mux( + decode.io.out(i).fire, + decode.io.in(i).bits.foldpc, + rename.io.in(i).bits.foldpc + ) + } + + // currently, we only update mdp info when isReplay + memCtrl.io.redirect := s1_s3_redirect + memCtrl.io.csrCtrl := io.csrCtrl // RegNext in memCtrl + memCtrl.io.stIn := io.fromMem.stIn // RegNext in memCtrl + memCtrl.io.memPredUpdate := redirectGen.io.memPredUpdate // RegNext in memCtrl + memCtrl.io.mdpFoldPcVecVld := mdpFlodPcVecVld + memCtrl.io.mdpFlodPcVec := mdpFlodPcVec + memCtrl.io.dispatchLFSTio <> dispatch.io.lfst + + rat.io.hartId := io.fromTop.hartId + rat.io.redirect := s1_s3_redirect.valid + rat.io.rabCommits := rob.io.rabCommits + rat.io.diffRatCommitRobIdx.foreach(_ := rob.io.diffRatCommitRobIdx.get) + rat.io.renameUpdates.foreach { updates => + updates.zip(rename.io.out).foreach { case (update, renameOut) => + update.valid := renameOut.fire && !s1_s3_redirect.valid + update.bits.robIdx := renameOut.bits.robIdx + update.bits.lastUop := renameOut.bits.lastUop + update.bits.ldest := renameOut.bits.ldest + update.bits.pdest := renameOut.bits.pdest + update.bits.rfWen := renameOut.bits.rfWen + update.bits.fpWen := renameOut.bits.fpWen + update.bits.vecWen := renameOut.bits.vecWen + update.bits.v0Wen := renameOut.bits.v0Wen + update.bits.vlWen := renameOut.bits.vlWen + } + } + rat.io.intRenamePorts := rename.io.intRenamePorts + rat.io.fpRenamePorts := rename.io.fpRenamePorts + rat.io.vecRenamePorts := rename.io.vecRenamePorts + rat.io.v0RenamePorts := rename.io.v0RenamePorts + rat.io.vlRenamePorts := rename.io.vlRenamePorts + + rename.io.redirect := s1_s3_redirect + rename.io.rabCommits := rob.io.rabCommits + rename.io.singleStep := GatedValidRegNext(io.csrCtrl.singlestep) + rename.io.waittable := (memCtrl.io.waitTable2Rename zip decode.io.out).map { case (waittable2rename, decodeOut) => + RegEnable(waittable2rename, decodeOut.fire) + } + rename.io.ssit := memCtrl.io.ssit2Rename + // disble mdp + dispatch.io.lfst.resp := 0.U.asTypeOf(dispatch.io.lfst.resp) + rename.io.waittable := 0.U.asTypeOf(rename.io.waittable) + rename.io.ssit := 0.U.asTypeOf(rename.io.ssit) + rename.io.intReadPorts := VecInit(rat.io.intReadPorts.map(x => VecInit(x.map(_.data)))) + rename.io.fpReadPorts := VecInit(rat.io.fpReadPorts.map(x => VecInit(x.map(_.data)))) + rename.io.vecReadPorts := VecInit(rat.io.vecReadPorts.map(x => VecInit(x.map(_.data)))) + rename.io.v0ReadPorts := VecInit(rat.io.v0ReadPorts.map(x => VecInit(x.data))) + rename.io.vlReadPorts := VecInit(rat.io.vlReadPorts.map(x => VecInit(x.data))) + rename.io.int_need_free := rat.io.int_need_free + rename.io.int_old_pdest := rat.io.int_old_pdest + rename.io.fp_old_pdest := rat.io.fp_old_pdest + rename.io.vec_old_pdest := rat.io.vec_old_pdest + rename.io.v0_old_pdest := rat.io.v0_old_pdest + rename.io.vl_old_pdest := rat.io.vl_old_pdest + rename.io.debug_int_rat.foreach(_ := rat.io.debug_int_rat.get) + rename.io.debug_fp_rat.foreach(_ := rat.io.debug_fp_rat.get) + rename.io.debug_vec_rat.foreach(_ := rat.io.debug_vec_rat.get) + rename.io.debug_v0_rat.foreach(_ := rat.io.debug_v0_rat.get) + rename.io.debug_vl_rat.foreach(_ := rat.io.debug_vl_rat.get) + rename.io.stallReason.in <> decode.io.stallReason.out + rename.io.snpt.snptEnq := DontCare + rename.io.snpt.snptDeq := snpt.io.deq + rename.io.snpt.useSnpt := useSnpt + rename.io.snpt.snptSelect := snptSelect + rename.io.snptIsFull := snpt.io.valids.asUInt.andR + rename.io.snpt.flushVec := flushVecNext + rename.io.snptLastEnq.valid := !isEmpty(snpt.io.enqPtr, snpt.io.deqPtr) + rename.io.snptLastEnq.bits := snpt.io.snapshots((snpt.io.enqPtr - 1.U).value).robIdx.head + + val renameOut = Wire(chiselTypeOf(rename.io.out)) + renameOut <> rename.io.out + // pass all snapshot in the first element for correctness of blockBackward + renameOut.tail.foreach(_.bits.snapshot := false.B) + renameOut.head.bits.snapshot := Mux( + isFull(snpt.io.enqPtr, snpt.io.deqPtr), + false.B, + Cat(rename.io.out.map(out => out.valid && out.bits.snapshot)).orR + ) + + // pipeline between rename and dispatch + PipeGroupConnect( + renameOut, + dispatch.io.fromRename, + s1_s3_redirect.valid, + dispatch.io.toRenameAllFire, + "renamePipeDispatch" + ) + + dispatch.io.redirect := s1_s3_redirect + val enqRob = Wire(chiselTypeOf(rob.io.enq)) + enqRob.canAccept := rob.io.enq.canAccept + enqRob.canAcceptForDispatch := rob.io.enq.canAcceptForDispatch + enqRob.isEmpty := rob.io.enq.isEmpty + enqRob.resp := rob.io.enq.resp + enqRob.needAlloc := RegNext(dispatch.io.enqRob.needAlloc) + enqRob.req.zip(dispatch.io.enqRob.req).map { case (sink, source) => + sink.valid := RegNext(source.valid && !rob.io.redirect.valid) + sink.bits := RegEnable(source.bits, source.valid) + } + dispatch.io.enqRob.canAccept := enqRob.canAcceptForDispatch && !enqRob.req.map(x => + x.valid && x.bits.blockBackward && enqRob.canAccept + ).reduce(_ || _) + dispatch.io.enqRob.canAcceptForDispatch := enqRob.canAcceptForDispatch + dispatch.io.enqRob.isEmpty := enqRob.isEmpty && !enqRob.req.map(_.valid).reduce(_ || _) + dispatch.io.enqRob.resp := enqRob.resp + rob.io.enq.needAlloc := enqRob.needAlloc + rob.io.enq.req := enqRob.req + dispatch.io.robHead := rob.io.debugRobHead + dispatch.io.stallReason <> rename.io.stallReason.out + dispatch.io.lqCanAccept := io.lqCanAccept + dispatch.io.sqCanAccept := io.sqCanAccept + dispatch.io.fromMem.lcommit := io.fromMemToDispatch.lcommit + dispatch.io.fromMem.scommit := io.fromMemToDispatch.scommit + dispatch.io.fromMem.lqDeqPtr := io.fromMemToDispatch.lqDeqPtr + dispatch.io.fromMem.sqDeqPtr := io.fromMemToDispatch.sqDeqPtr + dispatch.io.fromMem.lqCancelCnt := io.fromMemToDispatch.lqCancelCnt + dispatch.io.fromMem.sqCancelCnt := io.fromMemToDispatch.sqCancelCnt + io.toMem.lsqEnqIO <> dispatch.io.toMem.lsqEnqIO + dispatch.io.wakeUpAll.wakeUpInt := io.toDispatch.wakeUpInt + dispatch.io.wakeUpAll.wakeUpFp := io.toDispatch.wakeUpFp + dispatch.io.wakeUpAll.wakeUpVec := io.toDispatch.wakeUpVec + dispatch.io.wakeUpAll.wakeUpMem := io.toDispatch.wakeUpMem + dispatch.io.IQValidNumVec := io.toDispatch.IQValidNumVec + dispatch.io.ldCancel := io.toDispatch.ldCancel + dispatch.io.og0Cancel := io.toDispatch.og0Cancel + dispatch.io.wbPregsInt := io.toDispatch.wbPregsInt + dispatch.io.wbPregsFp := io.toDispatch.wbPregsFp + dispatch.io.wbPregsVec := io.toDispatch.wbPregsVec + dispatch.io.wbPregsV0 := io.toDispatch.wbPregsV0 + dispatch.io.wbPregsVl := io.toDispatch.wbPregsVl + dispatch.io.vlWriteBackInfo := io.toDispatch.vlWriteBackInfo + dispatch.io.robHeadNotReady := rob.io.headNotReady + dispatch.io.robFull := rob.io.robFull + dispatch.io.singleStep := GatedValidRegNext(io.csrCtrl.singlestep) + + val toIssueBlockUops = + Seq(io.toIssueBlock.intUops, io.toIssueBlock.fpUops, io.toIssueBlock.vfUops, io.toIssueBlock.memUops).flatten + toIssueBlockUops.zip(dispatch.io.toIssueQueues).map(x => x._1 <> x._2) + io.toIssueBlock.flush <> s2_s4_redirect + + pcMem.io.wen.head := GatedValidRegNext(io.frontend.fromFtq.pc_mem_wen) + pcMem.io.waddr.head := RegEnable(io.frontend.fromFtq.pc_mem_waddr, io.frontend.fromFtq.pc_mem_wen) + pcMem.io.wdata.head := RegEnable(io.frontend.fromFtq.pc_mem_wdata, io.frontend.fromFtq.pc_mem_wen) + + io.toDataPath.flush := s2_s4_redirect + io.toExuBlock.flush := s2_s4_redirect + + rob.io.hartId := io.fromTop.hartId + rob.io.redirect := s1_s3_redirect + rob.io.writeback := delayedNotFlushedWriteBack + rob.io.exuWriteback := delayedWriteBack + rob.io.writebackNums := VecInit(delayedNotFlushedWriteBackNums) + rob.io.writebackNeedFlush := delayedNotFlushedWriteBackNeedFlush + rob.io.readGPAMemData := gpaMem.io.exceptionReadData + rob.io.fromVecExcpMod.busy := io.fromVecExcpMod.busy + + io.redirect := s1_s3_redirect + + // rob to int block + io.robio.csr <> rob.io.csr + // When wfi is disabled, it will not block ROB commit. + rob.io.csr.wfiEvent := io.robio.csr.wfiEvent + rob.io.wfi_enable := decode.io.csrCtrl.wfi_enable + rob.io.commit_stuck_check_enable := decode.io.csrCtrl.commit_stuck_check_enable + + io.toTop.cpuHalt := DelayN(rob.io.cpu_halt, 5) + + io.robio.csr.perfinfo.retiredInstr <> RegNext(rob.io.csr.perfinfo.retiredInstr) + io.robio.exception := rob.io.exception + io.robio.exception.bits.pc := s1_robFlushPc + + // wfi + io.frontend.wfi.wfiReq := rob.io.wfi.wfiReq + rob.io.wfi.safeFromFrontend := io.frontend.wfi.wfiSafe + io.toMem.wfi.wfiReq := rob.io.wfi.wfiReq + rob.io.wfi.safeFromMem := io.toMem.wfi.wfiSafe + + // rob to mem block + io.robio.lsq <> rob.io.lsq + + io.diff_vl_rat.foreach(_ := rat.io.diff_vl_rat.get) + + rob.io.debug_ls := io.robio.debug_ls + rob.io.debugHeadLsIssue := io.robio.robHeadLsIssue + rob.io.lsTopdownInfo := io.robio.lsTopdownInfo + rob.io.csr.criticalErrorState := io.robio.csr.criticalErrorState + rob.io.debugEnqLsq := io.debugEnqLsq + rob.io.debugInstrAddrTransType := io.fromCSR.instrAddrTransType + + io.robio.robDeqPtr := rob.io.robDeqPtr + + io.robio.storeDebugInfo <> rob.io.storeDebugInfo + + // rob to backend + io.robio.commitVType := rob.io.toDecode.commitVType + // exu block to decode + decode.io.vsetvlVType := io.toDecode.vsetvlVType + // backend to decode + decode.io.vstart := io.toDecode.vstart + // backend to rob + rob.io.vstartIsZero := io.toDecode.vstart === 0.U + + io.toCSR.trapInstInfo := decode.io.toCSR.trapInstInfo + + io.toVecExcpMod.logicPhyRegMap := rob.io.toVecExcpMod.logicPhyRegMap + io.toVecExcpMod.excpInfo := rob.io.toVecExcpMod.excpInfo + // T : rat receive rabCommit + // T+1: rat return oldPdest + io.toVecExcpMod.ratOldPest match { + case fromRat => + (0 until RabCommitWidth).foreach { idx => + val v0Valid = RegNext( + rat.io.rabCommits.isCommit && + rat.io.rabCommits.isWalk && + rat.io.rabCommits.commitValid(idx) && + rat.io.rabCommits.info(idx).v0Wen + ) + fromRat.v0OldVdPdest(idx).valid := RegNext(v0Valid) + fromRat.v0OldVdPdest(idx).bits := RegEnable(rat.io.v0_old_pdest(idx), v0Valid) + val vecValid = RegNext( + rat.io.rabCommits.isCommit && + rat.io.rabCommits.isWalk && + rat.io.rabCommits.commitValid(idx) && + rat.io.rabCommits.info(idx).vecWen + ) + fromRat.vecOldVdPdest(idx).valid := RegNext(vecValid) + fromRat.vecOldVdPdest(idx).bits := RegEnable(rat.io.vec_old_pdest(idx), vecValid) + } + } + + io.debugTopDown.fromRob := rob.io.debugTopDown.toCore + dispatch.io.debugTopDown.fromRob := rob.io.debugTopDown.toDispatch + dispatch.io.debugTopDown.fromCore := io.debugTopDown.fromCore + io.debugRolling := rob.io.debugRolling + + io.perfInfo.ctrlInfo.robFull := GatedValidRegNext(rob.io.robFull) + io.perfInfo.ctrlInfo.intdqFull := false.B + io.perfInfo.ctrlInfo.fpdqFull := false.B + io.perfInfo.ctrlInfo.lsdqFull := false.B + + val perfEvents = Seq(decode, rename, dispatch, rob).flatMap(_.getPerfEvents) + generatePerfEvent() + + val criticalErrors = rob.getCriticalErrors + generateCriticalErrors() +} diff --git a/examples/prefab/cores/xiangshan/arch/native/DecodeStage.scala b/examples/prefab/cores/xiangshan/arch/native/DecodeStage.scala new file mode 100644 index 000000000..c15c157ac --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/DecodeStage.scala @@ -0,0 +1,320 @@ +// Copied from XiangShan src/main/scala/xiangshan/backend/decode/DecodeStage.scala and modified for Buckyball. +// XiangShan is licensed under Mulan PSL v2 (http://license.coscl.org.cn/MulanPSL2). + +package examples.core.xiangshan.native + +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import chisel3.util._ +import utility._ +import utils._ +import xiangshan._ +import xiangshan.backend.rename.RatReadPort +import xiangshan.backend.Bundles._ +import xiangshan.backend.fu.vector.Bundles.{VType, Vl} +import xiangshan.backend.fu.FuType +import xiangshan.backend.fu.wrapper.CSRToDecode +import yunsuan.VpermType +import xiangshan.ExceptionNO.{illegalInstr, virtualInstr} +import xiangshan.frontend.FtqPtr +import xiangshan.backend.decode._ +import examples.core.xiangshan.native.{DecodeUnit} + +class DecodeStage(implicit p: Parameters) extends XSModule with HasPerfEvents with VectorConstants { + + val io = IO(new DecodeStageIO) + + io.in.zipWithIndex.foreach { case (d, i) => + PerfCCT.updateInstPos(d.bits.debug_seqNum, PerfCCT.InstPos.AtDecode.id.U, d.valid, clock, reset) + } + + // io alias + private val outReadys = io.out.map(_.ready) + private val inValids = io.in.map(_.valid) + private val inValid = VecInit(inValids).asUInt.orR + private val outValids = io.out.map(_.valid) + private val outValid = VecInit(outValids).asUInt.orR + //readyFromRename Counter + /** Assume number of ready channels be "RenameWidth" if the first output channel is ready. If not, assume that be 0 */ + val readyCounter = Mux(outReadys.head, RenameWidth.U, 0.U) + + /** complex decoder */ + val decoderComp = Module(new DecodeUnitComp) + + /** simple decoders in Seq of DecodeWidth */ + val decoders = Seq.fill(DecodeWidth)(Module(new DecodeUnit)) + + /** vtype generation module */ + val vtypeGen = Module(new VTypeGen) + + val debug_globalCounter = RegInit(0.U(XLEN.W)) + + /** whether DecodeStage can accept new requests from frontend (CtrlBlock) */ + val canAccept = Wire(Bool()) + + //Simple 6 + decoders.zip(io.in).foreach { case (dst, src) => + dst.io.enq.ctrlFlow := src.bits + dst.io.csrCtrl := io.csrCtrl + dst.io.fromCSR := io.fromCSR + dst.io.enq.vtype := vtypeGen.io.vtype + dst.io.enq.vstart := io.vstart + } + + /** whether instructions decoded by simple decoders require complex decoding */ + val isComplexVec = VecInit(inValids.zip(decoders.map(_.io.deq.isComplex)).map { case (valid, isComplex) => + valid && isComplex + }) + + /** whether instructions decoded by simple decoders don't require complex decoding */ + val isSimpleVec = VecInit(inValids.zip(decoders.map(_.io.deq.isComplex)).map { case (valid, isComplex) => + valid && !isComplex + }) + + /** instructions decoded by simple decoders */ + val simpleDecodedInst = VecInit(decoders.map(_.io.deq.decodedInst)) + + /** whether instructions decoded by simple decoders are illegal */ + val isIllegalInstVec = VecInit((outValids lazyZip outReadys lazyZip io.out.map(_.bits)).map { + case (valid, ready, decodedInst) => + valid && ready && (decodedInst.exceptionVec(ExceptionNO.EX_II) || decodedInst.exceptionVec(ExceptionNO.EX_VI)) + }) + + /** at least 1 instruction decoded by simple decoders is illegal */ + val hasIllegalInst = Cat(isIllegalInstVec).orR + + /** at least 1 instruction decoded by simple decoders is illegal */ + val illegalInst = PriorityMuxDefault(isIllegalInstVec.zip(io.out.map(_.bits)), 0.U.asTypeOf(new DecodedInst)) + + /** number of instructions generated by complex decoder */ + val complexNum = Wire(UInt(3.W)) + // (0, 1, 2, 3, 4, 5) + complexNum + /** + * Order of simple decoders' result (in output of DecodeStage) considering complex decoder's. Since complex decoder's + * results will be arranged before simple decoders' + */ + val complexNumAddLocation: Vec[UInt] = VecInit((0 until DecodeWidth).map(x => (x.U +& complexNum))) + + /** mask off decoded instructions that can not be accepted */ + val noMoreThanRenameReady: Vec[Bool] = VecInit(complexNumAddLocation.map(x => x <= readyCounter)) + + /** existance of complex instructions among first few simple decoders' results, which needs decoding */ + val complexValid = VecInit((isComplexVec zip noMoreThanRenameReady).map(x => x._1 & x._2)).asUInt.orR + + /** selected complex instruction for complex decoder */ + val complexInst = + PriorityMuxDefault(isComplexVec.zip(decoders.map(_.io.deq.decodedInst)), 0.U.asTypeOf(new DecodedInst)) + + /** selected complex micro operation information for complex decoder */ + val complexUopInfo = PriorityMuxDefault(isComplexVec.zip(decoders.map(_.io.deq.uopInfo)), 0.U.asTypeOf(new UopInfo)) + + vtypeGen.io.insts.zip(io.in).foreach { case (inst, in) => + inst.valid := in.valid + inst.bits := in.bits.instr + } + // when io.redirect is True, never update vtype + vtypeGen.io.canUpdateVType := decoderComp.io.in.fire && decoderComp.io.in.bits.simpleDecodedInst.isVset && !io.redirect + vtypeGen.io.walkToArchVType := io.fromRob.walkToArchVType + vtypeGen.io.commitVType := io.fromRob.commitVType + vtypeGen.io.walkVType := io.fromRob.walkVType + vtypeGen.io.vsetvlVType := io.vsetvlVType + + //Comp 1 + decoderComp.io.redirect := io.redirect + decoderComp.io.csrCtrl := io.csrCtrl + decoderComp.io.vtypeBypass := vtypeGen.io.vtype + // The input inst of decoderComp is latched last cycle. + // Set input empty, if there is no complex inst latched last cycle. + decoderComp.io.in.valid := complexValid && !io.fromRob.isResumeVType + decoderComp.io.in.bits.simpleDecodedInst := complexInst + decoderComp.io.in.bits.uopInfo := complexUopInfo + decoderComp.io.out.complexDecodedInsts.zipWithIndex.foreach { case (out, i) => out.ready := io.out(i).ready } + + /** instructions decoded by complex decoders */ + val complexDecodedInst = VecInit(decoderComp.io.out.complexDecodedInsts.map(_.bits)) + + /** whether instructions decoded by complex decoders are valid */ + val complexDecodedInstValid = VecInit(decoderComp.io.out.complexDecodedInsts.map(_.valid)) + complexNum := decoderComp.io.complexNum + + // Vec(S,S,S,C,S,C) -> Vec(1,1,1,0,0,0) + /** whether a complex instruction is before or at this position of simple decoders' result */ + val simplePrefixVec = VecInit((0 until DecodeWidth).map(i => VecInit(isSimpleVec.take(i + 1)).asUInt.andR)) + + // Vec(S,S,S,C,S,S) -> Vec(0,0,0,1,0,0) + /** one-hot representation of the first complex instruction */ + val firstComplexOH: Vec[Bool] = VecInit(PriorityEncoderOH(isComplexVec)) + + // block vector inst when vtype is resuming + val hasVectorInst = VecInit(decoders.map(x => + FuType.FuTypeOrR(x.io.deq.decodedInst.fuType, FuType.vecArithOrMem ++ FuType.vecVSET) + )).asUInt.orR + + /** condition of acceptation: no redirection, ready from rename/complex decoder, no resumeVType */ + canAccept := !io.redirect && (io.out.head.ready || decoderComp.io.in.ready) && !io.fromRob.isResumeVType + + io.canAccept := canAccept + + /** + * Assign ready signal for DecodeStage's input. Ready signal in i-th channel: + * + * It must hold that no redirection and no isResumeVType is caught. + * One situation for set up ready signal is that first "i" instructions are all simple instructions, and these "i" + * instructions can be passed down to rename together with complex decoder's result. + * Another situation is that first "i-1" instructions are all simple instructions, and the "i-th" instructions needs + * to be sent to complex decoder, with complex decoder ready for new input. + */ + io.in.zipWithIndex.foreach { case (in, i) => + in.ready := !io.redirect && ( + simplePrefixVec(i) && (i.U +& complexNum) < readyCounter || + firstComplexOH(i) && (i.U +& complexNum) <= readyCounter && decoderComp.io.in.ready + ) && !io.fromRob.isResumeVType + } + + /** final instruction decoding result */ + val finalDecodedInst = Wire(Vec(DecodeWidth, new DecodedInst)) + + /** valid signs of final instruction decoding result */ + val finalDecodedInstValid = Wire(Vec(DecodeWidth, Bool())) + + /** + * Select final result of DecodeStage. Select all complex decoded insts results at the beginning of final result, and + * use simple decoded insts to fill the rest space in DecodeWidth. + */ + for (i <- 0 until DecodeWidth) { + finalDecodedInst(i) := Mux(complexNum > i.U, complexDecodedInst(i), simpleDecodedInst(i.U - complexNum)) + finalDecodedInstValid(i) := Mux(complexNum > i.U, complexDecodedInstValid(i), simplePrefixVec(i.U - complexNum)) + } + + /** + * Generate output of DecodeStage. Pass finalDecodedInst to output as decoded instructions. + * Note that finalDecodedInst is generated in order. + */ + io.out.zipWithIndex.foreach { case (inst, i) => + inst.valid := finalDecodedInstValid(i) && !io.fromRob.isResumeVType + inst.bits := finalDecodedInst(i) + inst.bits.lsrc(0) := Mux(finalDecodedInst(i).vpu.isReverse, finalDecodedInst(i).lsrc(1), finalDecodedInst(i).lsrc(0)) + inst.bits.lsrc(1) := Mux(finalDecodedInst(i).vpu.isReverse, finalDecodedInst(i).lsrc(0), finalDecodedInst(i).lsrc(1)) + inst.bits.srcType(0) := Mux( + finalDecodedInst(i).vpu.isReverse, + finalDecodedInst(i).srcType(1), + finalDecodedInst(i).srcType(0) + ) + inst.bits.srcType(1) := Mux( + finalDecodedInst(i).vpu.isReverse, + finalDecodedInst(i).srcType(0), + finalDecodedInst(i).srcType(1) + ) + inst.bits.v0Wen := finalDecodedInst(i).vecWen && finalDecodedInst(i).ldest === 0.U || finalDecodedInst(i).v0Wen + inst.bits.vecWen := finalDecodedInst(i).vecWen && finalDecodedInst(i).ldest =/= 0.U + // when src0/src1/src2 read V0, src3 read V0 + val srcType0123HasV0 = finalDecodedInst(i).srcType.zip(finalDecodedInst(i).lsrc).take(4).map { case (s, l) => + SrcType.isVp(s) && (l === 0.U) + }.reduce(_ || _) + inst.bits.srcType(3) := Mux(srcType0123HasV0, SrcType.v0, finalDecodedInst(i).srcType(3)) + when(inst.bits.uopIdx =/= 0.U) { + inst.bits.debug_seqNum := 0.U.asTypeOf(new InstSeqNum) + } + } + + io.out.map(x => + when(x.valid) { + assert( + PopCount(VecInit(x.bits.rfWen, x.bits.fpWen, x.bits.vecWen, x.bits.v0Wen, x.bits.vlWen)) < 2.U, + "DecodeOut: can't wirte two regfile in one uop/instruction" + ) + } + ) + + /** + * Prepare address and hold for output to Rat (Rename Alias Table) + */ + for (i <- 0 until DecodeWidth) { + + // We use the lsrc/ldest before fusion decoder to read RAT for better timing. + io.intRat(i)(0).addr := io.out(i).bits.lsrc(0) + io.intRat(i)(1).addr := io.out(i).bits.lsrc(1) + io.intRat(i).foreach(_.hold := !io.out(i).ready) + + // Floating-point instructions can not be fused now. + io.fpRat(i)(0).addr := io.out(i).bits.lsrc(0) + io.fpRat(i)(1).addr := io.out(i).bits.lsrc(1) + io.fpRat(i)(2).addr := io.out(i).bits.lsrc(2) + io.fpRat(i).foreach(_.hold := !io.out(i).ready) + + // Vec instructions + // TODO: vec uop dividers need change this + io.vecRat(i)(0).addr := io.out(i).bits.lsrc(0) // vs1 + io.vecRat(i)(1).addr := io.out(i).bits.lsrc(1) // vs2 + io.vecRat(i)(2).addr := io.out(i).bits.lsrc(2) // old_vd + io.vecRat(i).foreach(_.hold := !io.out(i).ready) + + io.v0Rat(i).addr := V0_IDX.U // v0 + io.v0Rat(i).hold := !io.out(i).ready + + io.vlRat(i).addr := Vl_IDX.U // vl + io.vlRat(i).hold := !io.out(i).ready + } + + /** whether valid input requests from frontend exists */ + val hasValid = VecInit(io.in.map(_.valid)).asUInt.orR + + debug_globalCounter := debug_globalCounter + PopCount(io.out.map(_.fire)) + + io.stallReason.in.backReason := io.stallReason.out.backReason + io.stallReason.out.reason.zip(io.stallReason.in.reason).zip(io.in.map(_.valid)).foreach { case ((out, in), valid) => + out := Mux(io.stallReason.out.backReason.valid, io.stallReason.out.backReason.bits, in) + } + + io.toCSR.trapInstInfo.valid := RegNext(hasIllegalInst && !io.redirect) + io.toCSR.trapInstInfo.bits.fromDecodedInst(RegNext(illegalInst)) + + val recoveryFlag = RegInit(false.B) + when(io.redirect) { + recoveryFlag := true.B + }.elsewhen(io.in.map(_.fire).reduce(_ || _)) { + recoveryFlag := false.B + } + + XSPerfAccumulate("in_valid_count", PopCount(io.in.map(_.valid))) + XSPerfAccumulate("in_fire_count", PopCount(io.in.map(_.fire))) + XSPerfAccumulate("in_valid_not_ready_count", PopCount(io.in.map(x => x.valid && !x.ready))) + XSPerfAccumulate("stall_cycle", io.in.head match { case x => x.valid && !x.ready }) + XSPerfAccumulate("wait_cycle", !io.in.head.valid && io.out.head.ready) + XSPerfAccumulate("inst_spec", PopCount(io.in.map(_.fire))) + XSPerfAccumulate("recovery_bubble", recoveryFlag) + XSPerfAccumulate("frontend_stall_cycle", GatedValidRegNext(!io.in.head.valid && io.backendCanAccept)) + XSPerfAccumulate("backend_stall_cycle", GatedValidRegNext(io.in.head.valid && !io.in.head.ready)) + + XSPerfHistogram("in_valid_range", PopCount(io.in.map(_.valid)), true.B, 0, DecodeWidth + 1, 1) + XSPerfHistogram("in_fire_range", PopCount(io.in.map(_.fire)), true.B, 0, DecodeWidth + 1, 1) + XSPerfHistogram("out_valid_range", PopCount(io.out.map(_.valid)), true.B, 0, DecodeWidth + 1, 1) + XSPerfHistogram("out_fire_range", PopCount(io.out.map(_.fire)), true.B, 0, DecodeWidth + 1, 1) + + val fusionValid = VecInit(io.fusion.map(x => GatedValidRegNext(x))) + val inValidNotReady = io.in.map(in => GatedValidRegNext(in.valid && !in.ready)) + val frontendStall = GatedValidRegNext(!io.in.head.valid && io.backendCanAccept) + val backendStall = GatedValidRegNext(io.in.head.valid && !io.in.head.ready) + + val perfEvents = Seq( + ("decoder_fused_instr", PopCount(fusionValid)), + ("decoder_waitInstr", PopCount(inValidNotReady)), + ("decoder_stall_cycle", hasValid && !io.out(0).ready), + ("decoder_utilization", PopCount(io.in.map(_.valid))), + ("frontend_stall_cycle", frontendStall), + ("backend_stall_cycle", backendStall), + ("INST_SPEC", PopCount(io.in.map(_.fire))), + ("RECOVERY_BUBBLE", recoveryFlag) + ) + + generatePerfEvent() + + // for more readable verilog + dontTouch(isSimpleVec) + dontTouch(isComplexVec) + dontTouch(simplePrefixVec) + dontTouch(complexValid) + dontTouch(complexNum) + dontTouch(readyCounter) + dontTouch(firstComplexOH) +} diff --git a/examples/prefab/cores/xiangshan/arch/native/DecodeUnit.scala b/examples/prefab/cores/xiangshan/arch/native/DecodeUnit.scala new file mode 100644 index 000000000..f0fa090e1 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/DecodeUnit.scala @@ -0,0 +1,758 @@ +// Copied from XiangShan src/main/scala/xiangshan/backend/decode/DecodeUnit.scala and modified for Buckyball. +// XiangShan is licensed under Mulan PSL v2 (http://license.coscl.org.cn/MulanPSL2). + +package examples.core.xiangshan.native + +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import chisel3.util._ +import freechips.rocketchip.rocket.CSRs +import freechips.rocketchip.rocket.Instructions._ +import freechips.rocketchip.rocket.CustomInstructions._ +import freechips.rocketchip.util.uintToBitPat +import utility._ +import utils._ +import xiangshan.ExceptionNO.{breakPoint, illegalInstr, virtualInstr, EX_II} +import xiangshan._ +import xiangshan.backend.fu.FuType +import xiangshan.backend.Bundles.{DecodedInst, DynInst, StaticInst} +import xiangshan.backend.decode.isa.PseudoInstructions +import xiangshan.backend.decode.isa.bitfield.{InstVType, OPCODE5Bit, XSInstBitFields} +import xiangshan.backend.fu.vector.Bundles.{VType, Vl} +import xiangshan.backend.fu.wrapper.CSRToDecode +import xiangshan.backend.decode.Zimop._ +import yunsuan.{VfaluType, VfcvtType} +import xiangshan.backend.decode._ +import examples.core.xiangshan.native.{Command} + +/** + * Decode unit that takes in a single CtrlFlow and generates a CfCtrl. + */ +class DecodeUnit(implicit p: Parameters) extends XSModule with DecodeUnitConstants { + val io = IO(new DecodeUnitIO) + + val ctrl_flow = io.enq.ctrlFlow // input with RVC Expanded + + private val inst: XSInstBitFields = io.enq.ctrlFlow.instr.asTypeOf(new XSInstBitFields) + + // The accelerator command sits right before CSRRW in the integer table. + val decode_table: Array[(BitPat, List[BitPat])] = + XDecode.table.patch(XDecode.decodeArray.indexWhere(_._1 == CSRRW), Seq(Command.decode), 0) ++ + FpDecode.table ++ +// FDivSqrtDecode.table ++ + BitmanipDecode.table ++ + ScalarCryptoDecode.table ++ + XSDebugDecode.table ++ + CBODecode.table ++ + SvinvalDecode.table ++ + HypervisorDecode.table ++ + VecDecoder.table ++ + ZicondDecode.table ++ + ZimopDecode.table ++ + ZfaDecode.table + + require(decode_table.forall(_._1.width == 32), "Decoded instructions must be 32 bits after RVC expansion") + require(decode_table.map(_._2.length == 15).reduce(_ && _), "Decode tables have different column size") + // assertion for LUI: only LUI should be assigned `selImm === SelImm.IMM_U && fuType === FuType.alu` + val luiMatch = (t: Seq[BitPat]) => t(3).value == FuType.alu.ohid && t.reverse.head.value == SelImm.IMM_U.litValue + val luiTable = decode_table.filter(t => luiMatch(t._2)).map(_._1).distinct + assert(luiTable.length == 1 && luiTable.head == LUI, "Conflicts: LUI is determined by FuType and SelImm in Dispatch") + + // output + val decodedInst: DecodedInst = Wire(new DecodedInst()).decode(ctrl_flow.instr, decode_table) + + val fpDecoder = Module(new FPDecoder) + fpDecoder.io.instr := ctrl_flow.instr + decodedInst.fpu := fpDecoder.io.fpCtrl + decodedInst.fpu.wflags := fpDecoder.io.fpCtrl.wflags || decodedInst.wfflags + + decodedInst.connectStaticInst(io.enq.ctrlFlow) + + decodedInst.uopIdx := 0.U + decodedInst.firstUop := true.B + decodedInst.lastUop := true.B + decodedInst.numUops := 1.U + decodedInst.numWB := 1.U + + val isZimop = (BitPat("b1?00??0111??_?????_100_?????_1110011") === ctrl_flow.instr) || + (BitPat("b1?00??1?????_?????_100_?????_1110011") === ctrl_flow.instr) + + val isMove = BitPat("b000000000000_?????_000_?????_0010011") === ctrl_flow.instr + // temp decode zimop as move + decodedInst.isMove := (isMove || isZimop) && ctrl_flow.instr(RD_MSB, RD_LSB) =/= 0.U && !io.csrCtrl.singlestep + + // fmadd - b1000011 + // fmsub - b1000111 + // fnmsub- b1001011 + // fnmadd- b1001111 + private val isFMA = inst.OPCODE === BitPat("b100??11") + private val isVppu = FuType.isVppu(decodedInst.fuType) + private val isVecOPF = FuType.isVecOPF(decodedInst.fuType) + + // read src1~3 location + decodedInst.lsrc(0) := inst.RS1 + decodedInst.lsrc(1) := inst.RS2 + // src(2) of fma is fs3, src(2) of vector inst is old vd + decodedInst.lsrc(2) := Mux(isFMA, inst.FS3, inst.VD) + decodedInst.lsrc(3) := V0_IDX.U + decodedInst.lsrc(4) := Vl_IDX.U + + // read dest location + decodedInst.ldest := inst.RD + + // init v0Wen vlWen + decodedInst.v0Wen := false.B + decodedInst.vlWen := false.B + + val isCsr = inst.OPCODE5Bit === OPCODE5Bit.SYSTEM && inst.FUNCT3(1, 0) =/= 0.U + val isCsrr = isCsr && inst.FUNCT3 === BitPat("b?1?") && inst.RS1 === 0.U + val isCsrw = isCsr && inst.FUNCT3 === BitPat("b?01") && inst.RD === 0.U + dontTouch(isCsrr) + dontTouch(isCsrw) + + // for csrr vl instruction, convert to vsetvl + val isCsrrVlenb = isCsrr && inst.CSRIDX === CSRs.vlenb.U + val isCsrrVl = isCsrr && inst.CSRIDX === CSRs.vl.U + + private val isCboClean = CBO_CLEAN === io.enq.ctrlFlow.instr + private val isCboFlush = CBO_FLUSH === io.enq.ctrlFlow.instr + private val isCboInval = CBO_INVAL === io.enq.ctrlFlow.instr + private val isCboZero = CBO_ZERO === io.enq.ctrlFlow.instr + + // Note that rnum of aes64ks1i must be in the range 0x0..0xA. The values 0xB..0xF are reserved. + private val isAes64ks1iIllegal = + FuType.FuTypeOrR( + decodedInst.fuType, + FuType.bku + ) && (decodedInst.fuOpType === BKUOpType.aes64ks1i) && inst.isRnumIllegal + + private val isAmocasQ = + FuType.FuTypeOrR(decodedInst.fuType, FuType.mou) && decodedInst.fuOpType === LSUOpType.amocas_q + private val isAmocasQIllegal = isAmocasQ && (inst.RD(0) === 1.U || inst.RS2(0) === 1.U) + + private val exceptionII = + decodedInst.selImm === SelImm.INVALID_INSTR || + io.fromCSR.illegalInst.sfenceVMA && FuType.FuTypeOrR( + decodedInst.fuType, + FuType.fence + ) && decodedInst.fuOpType === FenceOpType.sfence || + io.fromCSR.illegalInst.sfencePart && FuType.FuTypeOrR( + decodedInst.fuType, + FuType.fence + ) && decodedInst.fuOpType === FenceOpType.nofence || + io.fromCSR.illegalInst.hfenceGVMA && FuType.FuTypeOrR( + decodedInst.fuType, + FuType.fence + ) && decodedInst.fuOpType === FenceOpType.hfence_g || + io.fromCSR.illegalInst.hfenceVVMA && FuType.FuTypeOrR( + decodedInst.fuType, + FuType.fence + ) && decodedInst.fuOpType === FenceOpType.hfence_v || + io.fromCSR.illegalInst.hlsv && FuType.FuTypeOrR(decodedInst.fuType, FuType.ldu) && (LSUOpType.isHlv( + decodedInst.fuOpType + ) || LSUOpType.isHlvx(decodedInst.fuOpType)) || + io.fromCSR.illegalInst.hlsv && FuType.FuTypeOrR(decodedInst.fuType, FuType.stu) && LSUOpType.isHsv( + decodedInst.fuOpType + ) || + io.fromCSR.illegalInst.fsIsOff && ( + FuType.FuTypeOrR(decodedInst.fuType, FuType.fpOP ++ Seq(FuType.f2v)) || + (FuType.FuTypeOrR( + decodedInst.fuType, + FuType.ldu + ) && (decodedInst.fuOpType === LSUOpType.lh || decodedInst.fuOpType === LSUOpType.lw || decodedInst.fuOpType === LSUOpType.ld) || + FuType.FuTypeOrR( + decodedInst.fuType, + FuType.stu + ) && (decodedInst.fuOpType === LSUOpType.sh || decodedInst.fuOpType === LSUOpType.sw || decodedInst.fuOpType === LSUOpType.sd)) && decodedInst.instr( + 2 + ) || + inst.isOPFVF || inst.isOPFVV + ) || + io.fromCSR.illegalInst.vsIsOff && (FuType.FuTypeOrR( + decodedInst.fuType, + FuType.vecAll + ) || isCsrrVl || isCsrrVlenb) || + io.fromCSR.illegalInst.wfi && FuType.FuTypeOrR(decodedInst.fuType, FuType.csr) && CSROpType.isWfi( + decodedInst.fuOpType + ) || + io.fromCSR.illegalInst.wrs_nto && FuType.FuTypeOrR(decodedInst.fuType, FuType.csr) && CSROpType.isWrsNto( + decodedInst.fuOpType + ) || + (decodedInst.needFrm.scalaNeedFrm || FuType.isScalaNeedFrm( + decodedInst.fuType + )) && (((decodedInst.fpu.rm === 5.U) || (decodedInst.fpu.rm === 6.U)) || ((decodedInst.fpu.rm === 7.U) && io.fromCSR.illegalInst.frm)) || + (decodedInst.needFrm.vectorNeedFrm || FuType.isVectorNeedFrm(decodedInst.fuType)) && io.fromCSR.illegalInst.frm || + (io.fromCSR.illegalInst.cboZ || !HasCMO.B) && isCboZero || + (io.fromCSR.illegalInst.cboCF || !HasCMO.B) && (isCboClean || isCboFlush) || + (io.fromCSR.illegalInst.cboI || !HasCMO.B) && isCboInval || + isAes64ks1iIllegal || + isAmocasQIllegal + + private val exceptionVI = + io.fromCSR.virtualInst.sfenceVMA && FuType.FuTypeOrR( + decodedInst.fuType, + FuType.fence + ) && decodedInst.fuOpType === FenceOpType.sfence || + io.fromCSR.virtualInst.sfencePart && FuType.FuTypeOrR( + decodedInst.fuType, + FuType.fence + ) && decodedInst.fuOpType === FenceOpType.nofence || + io.fromCSR.virtualInst.hfence && FuType.FuTypeOrR( + decodedInst.fuType, + FuType.fence + ) && (decodedInst.fuOpType === FenceOpType.hfence_g || decodedInst.fuOpType === FenceOpType.hfence_v) || + io.fromCSR.virtualInst.hlsv && FuType.FuTypeOrR(decodedInst.fuType, FuType.ldu) && (LSUOpType.isHlv( + decodedInst.fuOpType + ) || LSUOpType.isHlvx(decodedInst.fuOpType)) || + io.fromCSR.virtualInst.hlsv && FuType.FuTypeOrR(decodedInst.fuType, FuType.stu) && LSUOpType.isHsv( + decodedInst.fuOpType + ) || + io.fromCSR.virtualInst.wfi && FuType.FuTypeOrR(decodedInst.fuType, FuType.csr) && CSROpType.isWfi( + decodedInst.fuOpType + ) || + io.fromCSR.virtualInst.wrs_nto && FuType.FuTypeOrR(decodedInst.fuType, FuType.csr) && CSROpType.isWrsNto( + decodedInst.fuOpType + ) || + io.fromCSR.virtualInst.cboZ && isCboZero || + io.fromCSR.virtualInst.cboCF && (isCboClean || isCboFlush) || + io.fromCSR.virtualInst.cboI && isCboInval + + decodedInst.exceptionVec(illegalInstr) := exceptionII || io.enq.ctrlFlow.exceptionVec(EX_II) + decodedInst.exceptionVec(virtualInstr) := exceptionVI + + //update exceptionVec: from frontend trigger's breakpoint exception. To reduce 1 bit of overhead in ibuffer entry. + decodedInst.exceptionVec(breakPoint) := TriggerAction.isExp(ctrl_flow.trigger) + + decodedInst.imm := LookupTree( + decodedInst.selImm, + ImmUnion.immSelMap.map { + x => + val minBits = x._2.minBitsFromInstr(ctrl_flow.instr) + require(minBits.getWidth == x._2.len) + x._1 -> minBits + } + ) + + private val isLs = FuType.isLoadStore(decodedInst.fuType) + private val isVls = inst.isVecStore || inst.isVecLoad + private val isStore = FuType.isStore(decodedInst.fuType) + private val isAMO = FuType.isAMO(decodedInst.fuType) + private val isVStore = FuType.isVStore(decodedInst.fuType) + private val isBranch = !decodedInst.preDecodeInfo.notCFI || FuType.isJump(decodedInst.fuType) + + decodedInst.commitType := Cat(isLs | isVls, (isStore && !isAMO) | isVStore | isBranch) + + decodedInst.isVset := FuType.isVset(decodedInst.fuType) + + private val needReverseInsts = Seq(VRSUB_VI, VRSUB_VX, VFRDIV_VF, VFRSUB_VF, VFMV_F_S) + private val vextInsts = Seq(VZEXT_VF2, VZEXT_VF4, VZEXT_VF8, VSEXT_VF2, VSEXT_VF4, VSEXT_VF8) + + private val narrowInsts = Seq( + VNSRA_WV, + VNSRA_WX, + VNSRA_WI, + VNSRL_WV, + VNSRL_WX, + VNSRL_WI, + VNCLIP_WV, + VNCLIP_WX, + VNCLIP_WI, + VNCLIPU_WV, + VNCLIPU_WX, + VNCLIPU_WI + ) + + private val maskDstInsts = Seq( + VMADC_VV, + VMADC_VX, + VMADC_VI, + VMADC_VVM, + VMADC_VXM, + VMADC_VIM, + VMSBC_VV, + VMSBC_VX, + VMSBC_VVM, + VMSBC_VXM, + VMAND_MM, + VMNAND_MM, + VMANDN_MM, + VMXOR_MM, + VMOR_MM, + VMNOR_MM, + VMORN_MM, + VMXNOR_MM, + VMSEQ_VV, + VMSEQ_VX, + VMSEQ_VI, + VMSNE_VV, + VMSNE_VX, + VMSNE_VI, + VMSLE_VV, + VMSLE_VX, + VMSLE_VI, + VMSLEU_VV, + VMSLEU_VX, + VMSLEU_VI, + VMSLT_VV, + VMSLT_VX, + VMSLTU_VV, + VMSLTU_VX, + VMSGT_VX, + VMSGT_VI, + VMSGTU_VX, + VMSGTU_VI, + VMFEQ_VV, + VMFEQ_VF, + VMFNE_VV, + VMFNE_VF, + VMFLT_VV, + VMFLT_VF, + VMFLE_VV, + VMFLE_VF, + VMFGT_VF, + VMFGE_VF + ) + + private val maskOpInsts = Seq( + VMAND_MM, + VMNAND_MM, + VMANDN_MM, + VMXOR_MM, + VMOR_MM, + VMNOR_MM, + VMORN_MM, + VMXNOR_MM + ) + + private val vmaInsts = Seq( + VMACC_VV, + VMACC_VX, + VNMSAC_VV, + VNMSAC_VX, + VMADD_VV, + VMADD_VX, + VNMSUB_VV, + VNMSUB_VX, + VWMACCU_VV, + VWMACCU_VX, + VWMACC_VV, + VWMACC_VX, + VWMACCSU_VV, + VWMACCSU_VX, + VWMACCUS_VX + ) + + private val wfflagsInsts = Seq( + // opfff + FADD_S, + FSUB_S, + FADD_D, + FSUB_D, + FADD_H, + FSUB_H, + FEQ_S, + FLT_S, + FLE_S, + FEQ_D, + FLT_D, + FLE_D, + FEQ_H, + FLT_H, + FLE_H, + FMIN_S, + FMAX_S, + FMIN_D, + FMAX_D, + FMIN_H, + FMAX_H, + FMUL_S, + FMUL_D, + FMUL_H, + FDIV_S, + FDIV_D, + FSQRT_S, + FSQRT_D, + FDIV_H, + FSQRT_H, + FMADD_S, + FMSUB_S, + FNMADD_S, + FNMSUB_S, + FMADD_D, + FMSUB_D, + FNMADD_D, + FNMSUB_D, + FMADD_H, + FMSUB_H, + FNMADD_H, + FNMSUB_H, + FSGNJ_S, + FSGNJN_S, + FSGNJX_S, + FSGNJ_H, + FSGNJN_H, + FSGNJX_H, + // opfvv + VFADD_VV, + VFSUB_VV, + VFWADD_VV, + VFWSUB_VV, + VFWADD_WV, + VFWSUB_WV, + VFMUL_VV, + VFDIV_VV, + VFWMUL_VV, + VFMACC_VV, + VFNMACC_VV, + VFMSAC_VV, + VFNMSAC_VV, + VFMADD_VV, + VFNMADD_VV, + VFMSUB_VV, + VFNMSUB_VV, + VFWMACC_VV, + VFWNMACC_VV, + VFWMSAC_VV, + VFWNMSAC_VV, + VFSQRT_V, + VFMIN_VV, + VFMAX_VV, + VMFEQ_VV, + VMFNE_VV, + VMFLT_VV, + VMFLE_VV, + VFSGNJ_VV, + VFSGNJN_VV, + VFSGNJX_VV, + // opfvf + VFADD_VF, + VFSUB_VF, + VFRSUB_VF, + VFWADD_VF, + VFWSUB_VF, + VFWADD_WF, + VFWSUB_WF, + VFMUL_VF, + VFDIV_VF, + VFRDIV_VF, + VFWMUL_VF, + VFMACC_VF, + VFNMACC_VF, + VFMSAC_VF, + VFNMSAC_VF, + VFMADD_VF, + VFNMADD_VF, + VFMSUB_VF, + VFNMSUB_VF, + VFWMACC_VF, + VFWNMACC_VF, + VFWMSAC_VF, + VFWNMSAC_VF, + VFMIN_VF, + VFMAX_VF, + VMFEQ_VF, + VMFNE_VF, + VMFLT_VF, + VMFLE_VF, + VMFGT_VF, + VMFGE_VF, + VFSGNJ_VF, + VFSGNJN_VF, + VFSGNJX_VF, + // vfred + VFREDOSUM_VS, + VFREDUSUM_VS, + VFREDMAX_VS, + VFREDMIN_VS, + VFWREDOSUM_VS, + VFWREDUSUM_VS, + // fcvt & vfcvt + FCVT_S_W, + FCVT_S_WU, + FCVT_S_L, + FCVT_S_LU, + FCVT_W_S, + FCVT_WU_S, + FCVT_L_S, + FCVT_LU_S, + FCVT_D_W, + FCVT_D_WU, + FCVT_D_L, + FCVT_D_LU, + FCVT_W_D, + FCVT_WU_D, + FCVT_L_D, + FCVT_LU_D, + FCVT_S_D, + FCVT_D_S, + FCVT_S_H, + FCVT_H_S, + FCVT_H_D, + FCVT_D_H, + FCVT_H_W, + FCVT_H_WU, + FCVT_H_L, + FCVT_H_LU, + FCVT_W_H, + FCVT_WU_H, + FCVT_L_H, + FCVT_LU_H, + VFCVT_XU_F_V, + VFCVT_X_F_V, + VFCVT_RTZ_XU_F_V, + VFCVT_RTZ_X_F_V, + VFCVT_F_XU_V, + VFCVT_F_X_V, + VFWCVT_XU_F_V, + VFWCVT_X_F_V, + VFWCVT_RTZ_XU_F_V, + VFWCVT_RTZ_X_F_V, + VFWCVT_F_XU_V, + VFWCVT_F_X_V, + VFWCVT_F_F_V, + VFNCVT_XU_F_W, + VFNCVT_X_F_W, + VFNCVT_RTZ_XU_F_W, + VFNCVT_RTZ_X_F_W, + VFNCVT_F_XU_W, + VFNCVT_F_X_W, + VFNCVT_F_F_W, + VFNCVT_ROD_F_F_W, + VFRSQRT7_V, + VFREC7_V, + // zfa + FLEQ_H, + FLEQ_S, + FLEQ_D, + FLTQ_H, + FLTQ_S, + FLTQ_D, + FMINM_H, + FMINM_S, + FMINM_D, + FMAXM_H, + FMAXM_S, + FMAXM_D, + FROUND_H, + FROUND_S, + FROUND_D, + FROUNDNX_H, + FROUNDNX_S, + FROUNDNX_D, + FCVTMOD_W_D + ) + + private val scalaNeedFrmInsts = Seq( + FADD_S, + FSUB_S, + FADD_D, + FSUB_D, + FADD_H, + FSUB_H, + FCVT_W_S, + FCVT_WU_S, + FCVT_L_S, + FCVT_LU_S, + FCVT_W_D, + FCVT_WU_D, + FCVT_L_D, + FCVT_LU_D, + FCVT_S_D, + FCVT_D_S, + FCVT_W_H, + FCVT_WU_H, + FCVT_L_H, + FCVT_LU_H, + FCVT_S_H, + FCVT_H_S, + FCVT_H_D, + FCVT_D_H, + FROUND_H, + FROUND_S, + FROUND_D, + FROUNDNX_H, + FROUNDNX_S, + FROUNDNX_D + ) + + private val vectorNeedFrmInsts = Seq( + VFSLIDE1UP_VF, + VFSLIDE1DOWN_VF + ) + + decodedInst.wfflags := wfflagsInsts.map(_ === inst.ALL).reduce(_ || _) + decodedInst.needFrm.scalaNeedFrm := scalaNeedFrmInsts.map(_ === inst.ALL).reduce(_ || _) + decodedInst.needFrm.vectorNeedFrm := vectorNeedFrmInsts.map(_ === inst.ALL).reduce(_ || _) + decodedInst.vpu := 0.U.asTypeOf(decodedInst.vpu) // Todo: Connect vpu decoder + decodedInst.vpu.vill := io.enq.vtype.illegal + decodedInst.vpu.vma := io.enq.vtype.vma + decodedInst.vpu.vta := io.enq.vtype.vta + decodedInst.vpu.vsew := io.enq.vtype.vsew + decodedInst.vpu.vlmul := io.enq.vtype.vlmul + decodedInst.vpu.vm := inst.VM + decodedInst.vpu.nf := inst.NF + decodedInst.vpu.veew := inst.WIDTH + decodedInst.vpu.isReverse := needReverseInsts.map(_ === inst.ALL).reduce(_ || _) + decodedInst.vpu.isExt := vextInsts.map(_ === inst.ALL).reduce(_ || _) + val isNarrow = narrowInsts.map(_ === inst.ALL).reduce(_ || _) + val isDstMask = maskDstInsts.map(_ === inst.ALL).reduce(_ || _) + val isOpMask = maskOpInsts.map(_ === inst.ALL).reduce(_ || _) + val isVload = FuType.isVLoad(decodedInst.fuType) + val isVlx = isVload && (decodedInst.fuOpType === VlduType.vloxe || decodedInst.fuOpType === VlduType.vluxe) + val isVle = + isVload && (decodedInst.fuOpType === VlduType.vle || decodedInst.fuOpType === VlduType.vleff || decodedInst.fuOpType === VlduType.vlse) + val isVlm = isVload && (decodedInst.fuOpType === VlduType.vlm) + val isFof = isVload && (decodedInst.fuOpType === VlduType.vleff) + val isWritePartVd = + decodedInst.uopSplitType === UopSplitType.VEC_VRED || decodedInst.uopSplitType === UopSplitType.VEC_0XV || decodedInst.uopSplitType === UopSplitType.VEC_VWW + val isVma = vmaInsts.map(_ === inst.ALL).reduce(_ || _) + val emulIsFrac = + Cat(~decodedInst.vpu.vlmul(2), decodedInst.vpu.vlmul(1, 0)) +& decodedInst.vpu.veew < 4.U +& decodedInst.vpu.vsew + val vstartIsNotZero = io.enq.vstart =/= 0.U + decodedInst.vpu.isNarrow := isNarrow + decodedInst.vpu.isDstMask := isDstMask + decodedInst.vpu.isOpMask := isOpMask + decodedInst.vpu.isDependOldVd := isVppu || isVecOPF || isVStore || (isDstMask && !isOpMask) || isNarrow || isVlx || isVma || isFof || vstartIsNotZero + decodedInst.vpu.isWritePartVd := isWritePartVd || isVlm || isVle && emulIsFrac + decodedInst.vpu.vstart := io.enq.vstart + decodedInst.vpu.isVleff := isFof && inst.NF === 0.U + decodedInst.vpu.specVill := io.enq.vtype.illegal + decodedInst.vpu.specVma := io.enq.vtype.vma + decodedInst.vpu.specVta := io.enq.vtype.vta + decodedInst.vpu.specVsew := io.enq.vtype.vsew + decodedInst.vpu.specVlmul := io.enq.vtype.vlmul + + decodedInst.vlsInstr := isVls + + decodedInst.srcType(3) := Mux(inst.VM === 0.U, SrcType.vp, SrcType.DC) // mask src + decodedInst.srcType(4) := SrcType.vp // vconfig + + val uopInfoGen = Module(new UopInfoGen) + uopInfoGen.io.in.preInfo.isVecArith := inst.isVecArith + uopInfoGen.io.in.preInfo.isVecMem := inst.isVecStore || inst.isVecLoad + uopInfoGen.io.in.preInfo.isAmoCAS := inst.isAMOCAS + + uopInfoGen.io.in.preInfo.typeOfSplit := decodedInst.uopSplitType + uopInfoGen.io.in.preInfo.vsew := decodedInst.vpu.vsew + uopInfoGen.io.in.preInfo.vlmul := decodedInst.vpu.vlmul + uopInfoGen.io.in.preInfo.vwidth := inst.RM + uopInfoGen.io.in.preInfo.vmvn := inst.IMM5_OPIVI(2, 0) + uopInfoGen.io.in.preInfo.nf := inst.NF + uopInfoGen.io.in.preInfo.isVlsr := decodedInst.fuOpType === VlduType.vlr || decodedInst.fuOpType === VstuType.vsr + uopInfoGen.io.in.preInfo.isVlsm := decodedInst.fuOpType === VlduType.vlm || decodedInst.fuOpType === VstuType.vsm + io.deq.isComplex := uopInfoGen.io.out.isComplex + io.deq.uopInfo.numOfUop := uopInfoGen.io.out.uopInfo.numOfUop + io.deq.uopInfo.numOfWB := uopInfoGen.io.out.uopInfo.numOfWB + io.deq.uopInfo.lmul := uopInfoGen.io.out.uopInfo.lmul + + // decode for SoftPrefetch instructions (prefetch.w / prefetch.r / prefetch.i) + val isSoftPrefetch = inst.OPCODE === BitPat("b0010011") && inst.FUNCT3 === BitPat("b110") && inst.RD === 0.U + val isPreW = isSoftPrefetch && inst.RS2 === 3.U(5.W) + val isPreR = isSoftPrefetch && inst.RS2 === 1.U(5.W) + val isPreI = isSoftPrefetch && inst.RS2 === 0.U(5.W) + + // for fli.s|fli.d instruction + val isFLI = + inst.FUNCT7 === BitPat("b11110??") && inst.RS2 === 1.U && inst.RM === 0.U && inst.OPCODE5Bit === OPCODE5Bit.OP_FP + + when(isCsrrVl) { + // convert to vsetvl instruction + decodedInst.srcType(0) := SrcType.no + decodedInst.srcType(1) := SrcType.no + decodedInst.srcType(2) := SrcType.no + decodedInst.srcType(3) := SrcType.no + decodedInst.srcType(4) := SrcType.vp + decodedInst.lsrc(4) := Vl_IDX.U + decodedInst.waitForward := false.B + decodedInst.blockBackward := false.B + }.elsewhen(isCsrrVlenb) { + // convert to addi instruction + decodedInst.srcType(0) := SrcType.reg + decodedInst.srcType(1) := SrcType.imm + decodedInst.srcType(2) := SrcType.no + decodedInst.srcType(3) := SrcType.no + decodedInst.srcType(4) := SrcType.no + decodedInst.selImm := SelImm.IMM_I + decodedInst.waitForward := false.B + decodedInst.blockBackward := false.B + decodedInst.canRobCompress := true.B + }.elsewhen(isPreW || isPreR || isPreI) { + decodedInst.selImm := SelImm.IMM_S + decodedInst.fuType := FuType.ldu.U + decodedInst.canRobCompress := false.B + }.elsewhen(isZimop) { + // set srcType for zimop + decodedInst.srcType(0) := SrcType.reg + decodedInst.srcType(1) := SrcType.imm + // use x0 as src1 + decodedInst.lsrc(0) := 0.U + } + + io.deq.decodedInst := decodedInst + io.deq.decodedInst.rfWen := (decodedInst.ldest =/= 0.U) && decodedInst.rfWen + io.deq.decodedInst.fuType := Mux1H(Seq( + // keep condition + (!FuType.FuTypeOrR(decodedInst.fuType, FuType.vldu, FuType.vstu) && !isCsrrVl && !isCsrrVlenb) -> decodedInst.fuType, + isCsrrVl -> FuType.vsetfwf.U, + isCsrrVlenb -> FuType.alu.U, + // change vlsu to vseglsu when NF =/= 0.U + (FuType.FuTypeOrR( + decodedInst.fuType, + FuType.vldu, + FuType.vstu + ) && inst.NF === 0.U || (inst.NF =/= 0.U && (inst.MOP === "b00".U && inst.SUMOP === "b01000".U))) -> decodedInst.fuType, + // MOP === b00 && SUMOP === b01000: unit-stride whole register store + // MOP =/= b00 : strided and indexed store + (FuType.FuTypeOrR( + decodedInst.fuType, + FuType.vstu + ) && inst.NF =/= 0.U && ((inst.MOP === "b00".U && inst.SUMOP =/= "b01000".U) || inst.MOP =/= "b00".U)) -> FuType.vsegstu.U, + // MOP === b00 && LUMOP === b01000: unit-stride whole register load + // MOP =/= b00 : strided and indexed load + (FuType.FuTypeOrR( + decodedInst.fuType, + FuType.vldu + ) && inst.NF =/= 0.U && ((inst.MOP === "b00".U && inst.LUMOP =/= "b01000".U) || inst.MOP =/= "b00".U)) -> FuType.vsegldu.U + )) + io.deq.decodedInst.imm := MuxCase( + decodedInst.imm, + Seq( + isCsrrVlenb -> (VLEN / 8).U, + isZimop -> 0.U + ) + ) + + io.deq.decodedInst.fuOpType := MuxCase( + decodedInst.fuOpType, + Seq( + isCsrrVl -> VSETOpType.csrrvl, + isCsrrVlenb -> ALUOpType.add, + isFLI -> Cat(1.U, inst.FMT, inst.RS1), + (isPreW || isPreR || isPreI) -> Mux1H(Seq( + isPreW -> LSUOpType.prefetch_w, + isPreR -> LSUOpType.prefetch_r, + isPreI -> LSUOpType.prefetch_i + )), + (isCboInval && io.fromCSR.special.cboI2F) -> LSUOpType.cbo_flush + ) + ) + + // Don't compress in the same Rob entry when crossing Ftq entry boundary + io.deq.decodedInst.canRobCompress := decodedInst.canRobCompress && !io.enq.ctrlFlow.isLastInFtqEntry + + //------------------------------------------------------------- + // Debug Info +// XSDebug("in: instr=%x pc=%x excepVec=%b crossPageIPFFix=%d\n", +// io.enq.ctrl_flow.instr, io.enq.ctrl_flow.pc, io.enq.ctrl_flow.exceptionVec.asUInt, +// io.enq.ctrl_flow.crossPageIPFFix) +// XSDebug("out: srcType(0)=%b srcType(1)=%b srcType(2)=%b lsrc(0)=%d lsrc(1)=%d lsrc(2)=%d ldest=%d fuType=%b + // fuOpType=%b\n", +// io.deq.cf_ctrl.ctrl.srcType(0), io.deq.cf_ctrl.ctrl.srcType(1), io.deq.cf_ctrl.ctrl.srcType(2), +// io.deq.cf_ctrl.ctrl.lsrc(0), io.deq.cf_ctrl.ctrl.lsrc(1), io.deq.cf_ctrl.ctrl.lsrc(2), +// io.deq.cf_ctrl.ctrl.ldest, io.deq.cf_ctrl.ctrl.fuType, io.deq.cf_ctrl.ctrl.fuOpType) +// XSDebug("out: rfWen=%d fpWen=%d isXSTrap=%d noSpecExec=%d isBlocked=%d flushPipe=%d imm=%x\n", +// io.deq.cf_ctrl.ctrl.rfWen, io.deq.cf_ctrl.ctrl.fpWen, io.deq.cf_ctrl.ctrl.isXSTrap, +// io.deq.cf_ctrl.ctrl.noSpecExec, io.deq.cf_ctrl.ctrl.blockBackward, io.deq.cf_ctrl.ctrl.flushPipe, +// io.deq.cf_ctrl.ctrl.imm) +// XSDebug("out: excepVec=%b\n", io.deq.cf_ctrl.cf.exceptionVec.asUInt) +} diff --git a/examples/prefab/cores/xiangshan/arch/native/Native.scala b/examples/prefab/cores/xiangshan/arch/native/Native.scala new file mode 100644 index 000000000..84e364ebf --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/Native.scala @@ -0,0 +1,204 @@ +package examples.core.xiangshan.native + +import chisel3._ +import chisel3.experimental.annotate +import sifive.enterprise.firrtl.NestedPrefixModulesAnnotation +import chisel3.util._ +import org.chipsalliance.cde.config.{Config, Parameters} +import freechips.rocketchip.amba.axi4._ +import freechips.rocketchip.diplomacy._ +import freechips.rocketchip.interrupts._ +import freechips.rocketchip.tilelink._ +import coupledL2.{EnableCHI, L2ParamKey} +import coupledL2.tl2chi.{CHIAddrWidthKey, CHIIssue, PortIO} +import system.SoCParamsKey +import top.{ArgParser, Generator, SeperatedBusType} +import utility.{ChiselDB, Constantin} +import freechips.rocketchip.util.uintToBitPat +import xiangshan._ +import xiangshan.backend.decode.XSDecode +import xiangshan.backend.fu.FuType +import examples.core.xiangshan.native.XSTile + +class Context(implicit p: Parameters) extends XSBundle with HasPMParameters { + val satp = UInt(64.W) + val privilege = UInt(2.W) + val sum = Bool() + val mxr = Bool() + val virt = Bool() + + val pmp = Vec( + NumPMP, + new Bundle { + val cfg = UInt(8.W) + val addr = UInt((PMPAddrBits - 2).W) + } + ) + +} + +/** Accelerator command: custom-3 with funct3 = 3 and rd = 0, executed by the CSR unit. */ +object Command { + def opType = 0.U(FuOpTypeWidth.W) + + def decode = BitPat("b???????_?????_?????_011_00000_1111011") -> + XSDecode( + SrcType.reg, + SrcType.reg, + SrcType.X, + FuType.csr, + opType, + SelImm.IMM_Z, + xWen = true, + noSpec = true, + blockBack = true + ) + .generate() + +} + +class Commands(implicit p: Parameters) extends XSBundle { + + val request = Decoupled(new Bundle { + val instruction = UInt(32.W) + val pc = UInt(64.W) + val rs1 = UInt(64.W) + val rs2 = UInt(64.W) + val context = new Context + }) + + val complete = Flipped(Valid(Bool())) +} + +/** Native XiangShan tile boundary; its caches retain the upstream CHI endpoint. */ +class Native(implicit p: Parameters) extends LazyModule with BindingScope { + val tile = LazyModule(new XSTile) + val resetSource = BundleBridgeSource(() => Reset()) + tile.core_reset_sink := resetSource + val clint = IntSourceNode(IntSourcePortSimple(2, ports = 1, sources = 1)) + val plic = IntSourceNode(IntSourcePortSimple(1, ports = 2, sources = 1)) + val debug = IntSourceNode(IntSourcePortSimple(1, ports = 1, sources = 1)) + val nmi = IntSourceNode(IntSourcePortSimple(2, ports = 1, sources = 1)) + val busError = IntSinkNode(IntSinkPortSimple(1, 1)) + tile.clint_int_node := clint + tile.plic_int_node :*= plic + tile.debug_int_node := debug + tile.nmi_int_node := nmi + busError := tile.beu_int_source + + val uncached = AXI4SlaveNode(Seq(AXI4SlavePortParameters( + slaves = Seq(AXI4SlaveParameters( + address = AddressSet(0, 0x7fffffffL).subtract(p(SoCParamsKey).BEURange), + regionType = RegionType.UNCACHED, + executable = true, + supportsRead = TransferSizes(1, 16), + supportsWrite = TransferSizes(1, 16), + interleavedId = Some(0) + )), + beatBytes = 16 + ))) + + uncached := AXI4Buffer() := AXI4UserYanker() := AXI4IdIndexer(4) := + AXI4Deinterleaver(64) := TLToAXI4() := TLWidthWidget(8) := tile.sep_tl_opt.get + + lazy val module = new LazyModuleImp(this) with RequireAsyncReset { + annotate(this)(Seq(NestedPrefixModulesAnnotation(this.toNamed, "XiangShan", true))) + + val io = IO(new Bundle { + val hartId = Input(UInt(p(freechips.rocketchip.tile.MaxHartIdBits).W)) + val resetVector = Input(UInt(p(SoCParamsKey).PAddrBits.W)) + val nodeId = Input(UInt(7.W)) + val interrupts = Input(Vec(4, Bool())) + val commands = new Commands + val timeRequest = Decoupled(Bool()) + val time = Flipped(Valid(UInt(64.W))) + val chi = new PortIO + }) + + val uncachedAXI = uncached.makeIOs() + io.chi <> tile.module.io.chi.get + tile.module.io.nodeID.get := io.nodeId + tile.module.io.hartId := io.hartId + tile.module.io.reset_vector := io.resetVector + tile.module.io.reset_mtvec.foreach(_ := io.resetVector) + tile.module.io.msiInfo.valid := false.B + tile.module.io.msiInfo.bits := 0.U + tile.module.io.teemsiInfo.foreach { x => x.valid := false.B; x.bits := 0.U } + tile.module.io.traceCoreInterface.fromEncoder.enable := false.B + tile.module.io.traceCoreInterface.fromEncoder.stall := false.B + tile.module.io.debugTopDown.l3MissMatch := false.B + tile.module.io.l3Miss := false.B + tile.module.io.clintTime <> io.time + io.commands <> tile.module.io.commands + io.timeRequest <> tile.module.io.timeRequest + resetSource.out.head._1 := false.B.asAsyncReset + clint.out.head._1(0) := io.interrupts(0) + clint.out.head._1(1) := io.interrupts(1) + plic.out(0)._1(0) := io.interrupts(2) || busError.in.head._1(0) + plic.out(1)._1(0) := io.interrupts(3) + debug.out.head._1(0) := false.B + nmi.out.head._1.foreach(_ := false.B) + } + +} + +object Emit extends App { + + val (base, stageArgs, upstreamOptions) = ArgParser.parse( + Array("--config", "KunminghuV2Config", "--fpga-platform", "--disable-perf", "--disable-xmr") ++ args + ) + + val profile = base.alter((site, _, up) => { + case EnableCHI => true + case CHIIssue => "B" + case CHIAddrWidthKey => 48 + case SoCParamsKey => up(SoCParamsKey).copy( + PAddrBits = 48, + EnableICacheCtrl = false, + EnableDCacheCtrl = false, + SeperateBus = SeperatedBusType.TL, + SeperateBusRanges = Seq(AddressSet(0, 0x7fffffffL)) + ) + case DFTOptionsKey => up(DFTOptionsKey).copy(EnableMbist = false, EnableSramCtl = false) + case DebugOptionsKey => up(DebugOptionsKey).copy( + FPGAPlatform = true, + EnableDifftest = false, + AlwaysBasicDiff = false, + EnableDebug = false, + EnablePerfDebug = false, + EnableChiselDB = false, + AlwaysBasicDB = false + ) + }) + + val selected = profile(XSTileKey).head + + val core = selected.copy(L2CacheParamsOpt = + selected.L2CacheParamsOpt.map( + _.copy(dataCheck = None, enablePoison = false) + ) + ) + + val soc = profile(SoCParamsKey) + + val config = profile.alterPartial { + case XSCoreParamsKey => core + case SoCParamsKey => soc + case L2ParamKey => core.L2CacheParamsOpt.get + } + + Constantin.init(false) + ChiselDB.init(false) + val native = LazyModule(new Native()(config)) + val output = args.sliding(2).collectFirst { case Array("--target-dir", path) => path }.get + Generator.execute( + stageArgs, + native.module, + upstreamOptions ++ Array( + "--split-verilog", + s"-o=$output", + "--disable-annotation-unknown", + "--lowering-options=explicitBitcast,disallowLocalVariables,disallowPortDeclSharing" + ) + ) +} diff --git a/examples/prefab/cores/xiangshan/arch/native/Tile.scala b/examples/prefab/cores/xiangshan/arch/native/Tile.scala new file mode 100644 index 000000000..892ec44a8 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/Tile.scala @@ -0,0 +1,231 @@ +// Copied from XiangShan src/main/scala/xiangshan/XSTile.scala and modified for Buckyball. +// XiangShan is licensed under Mulan PSL v2 (http://license.coscl.org.cn/MulanPSL2). + +package examples.core.xiangshan.native + +import org.chipsalliance.cde.config.{Config, Parameters} +import chisel3._ +import chisel3.util.{log2Up, Decoupled, Valid, ValidIO} +import freechips.rocketchip.diplomacy._ +import freechips.rocketchip.interrupts._ +import freechips.rocketchip.tile.{BusErrorUnit, BusErrorUnitParams, BusErrors} +import freechips.rocketchip.tilelink._ +import freechips.rocketchip.amba.axi4._ +import system.HasSoCParameter +import top.{ArgParser, BusPerfMonitor, Generator} +import utility._ +import utility.sram.SramBroadcastBundle +import coupledL2.{EnableCHI, L2ParamKey} +import coupledL2.tl2chi.{LCrdyIn, PortIO} +import xiangshan.backend.trace.TraceCoreInterface +import xiangshan._ +import examples.core.xiangshan.native.{Commands, XSCore} + +class XSTile()(implicit p: Parameters) extends LazyModule with HasXSParameter with HasSoCParameter { + override def shouldBeInlined: Boolean = false + val core = LazyModule(new XSCore()) + val l2top = LazyModule(new L2Top()) + + val enableL2 = coreParams.L2CacheParamsOpt.isDefined + val txSourceReady = p(L2ParamKey).txSourceReady +// =========== Public Ports ============ + val memBlock = core.memBlock.inner + val core_l3_pf_port = memBlock.l3_pf_sender_opt + val memory_port = if (enableCHI && enableL2) None else Some(l2top.inner.memory_port.get) + val tl_uncache = l2top.inner.mmio_port + val sep_tl_opt = l2top.inner.sep_tl_port_opt + val beu_int_source = l2top.inner.beu.intNode + val core_reset_sink = BundleBridgeSink(Some(() => Reset())) + val clint_int_node = l2top.inner.clint_int_node + val plic_int_node = l2top.inner.plic_int_node + val debug_int_node = l2top.inner.debug_int_node + val nmi_int_node = l2top.inner.nmi_int_node + memBlock.clint_int_sink := IntBuffer() := clint_int_node + memBlock.plic_int_sink :*= IntBuffer() :*= plic_int_node + memBlock.debug_int_sink := IntBuffer() := debug_int_node + memBlock.nmi_int_sink := IntBuffer() := nmi_int_node + memBlock.beu_local_int_sink := l2top.inner.beu_local_int_source_buffer + + // =========== Components' Connection ============ + // L1 to l1_xbar + coreParams.dcacheParametersOpt.map { _ => + l2top.inner.misc_l2_pmu := l2top.inner.l1d_logger := memBlock.dcache_port := + memBlock.l1d_to_l2_buffer.node := memBlock.dcache.clientNode + } + + l2top.inner.misc_l2_pmu := l2top.inner.l1i_logger := memBlock.frontendBridge.icache_node + if (!coreParams.softPTW) { + l2top.inner.misc_l2_pmu := l2top.inner.ptw_logger := l2top.inner.ptw_to_l2_buffer.node := memBlock.ptw_to_l2_buffer.node + } + + // L2 Prefetch + l2top.inner.l2cache match { + case Some(l2) => + l2.pf_recv_node.foreach { recv => + println("Connecting L1 prefetcher to L2!") + recv := memBlock.l2_pf_sender_opt.get + } + case None => + } + + val core_l3_tpmeta_source_port = l2top.inner.l2cache match { + case Some(l2) => l2.tpmeta_source_node + case None => None + } + + val core_l3_tpmeta_sink_port = l2top.inner.l2cache match { + case Some(l2) => l2.tpmeta_sink_node + case None => None + } + + // mmio + l2top.inner.i_mmio_port := l2top.inner.i_mmio_buffer.node := memBlock.frontendBridge.instr_uncache_node + if (soc.EnableICacheCtrl) { + memBlock.frontendBridge.icachectrl_node := l2top.inner.icachectrl_port_opt.get + } + l2top.inner.d_mmio_port := memBlock.uncache_port + + // =========== IO Connection ============ + class XSTileImp(wrapper: LazyModule) extends LazyModuleImp(wrapper) { + + val io = IO(new Bundle { + val hartId = Input(UInt(hartIdLen.W)) + val msiInfo = Input(ValidIO(UInt(soc.IMSICParams.MSI_INFO_WIDTH.W))) + val msiAck = Output(Bool()) + val teemsiInfo = Option.when(soc.IMSICParams.HasTEEIMSIC)(Input(ValidIO(UInt(soc.IMSICParams.MSI_INFO_WIDTH.W)))) + val teemsiAck = Option.when(soc.IMSICParams.HasTEEIMSIC)(Output(Bool())) + val reset_vector = Input(UInt(PAddrBits.W)) + val reset_mtvec = Option.when(enableResetMtvec)(Input(UInt(PAddrBits.W))) + val cpu_halt = Output(Bool()) + val cpu_crtical_error = Output(Bool()) + val hartIsInReset = Output(Bool()) + val traceCoreInterface = new TraceCoreInterface + + val debugTopDown = new Bundle { + val robHeadPaddr = Valid(UInt(PAddrBits.W)) + val l3MissMatch = Input(Bool()) + } + + val l3Miss = Input(Bool()) + val chi = if (enableCHI) Some(new PortIO) else None + val nodeID = if (enableCHI) Some(Input(UInt(NodeIDWidth.W))) else None + val commands = new Commands + val timeRequest = Decoupled(Bool()) + val clintTime = Input(ValidIO(UInt(64.W))) + val dft = Option.when(hasDFT)(Input(new SramBroadcastBundle)) + val dft_reset = Option.when(hasMbist)(Input(new DFTResetSignals())) + val l2_flush_en = Option.when(EnablePowerDown)(Output(Bool())) + val l2_flush_done = Option.when(EnablePowerDown)(Output(Bool())) + val lcrdy = Option.when(txSourceReady)(new LCrdyIn) + }) + + dontTouch(io.hartId) + dontTouch(io.msiInfo) + io.teemsiInfo.foreach(dontTouch(_)) + if (!io.chi.isEmpty) { dontTouch(io.chi.get) } + + val core_soft_rst = core_reset_sink.in.head._1 // unused + + l2top.module.io.hartId.fromTile := io.hartId + core.module.io.hartId := l2top.module.io.hartId.toCore + core.module.io.reset_vector := l2top.module.io.reset_vector.toCore + core.module.io.reset_mtvec.foreach(_ := l2top.module.io.reset_mtvec.get.toCore) + core.module.io.msiInfo := l2top.module.io.msiInfo.toCore + l2top.module.io.msiInfo.fromTile := io.msiInfo + core.module.io.teemsiInfo zip l2top.module.io.teemsiInfo foreach { case (core_teemsiInfo, l2top_teemsiInfo) => + core_teemsiInfo := l2top_teemsiInfo.toCore + } + l2top.module.io.teemsiInfo zip io.teemsiInfo foreach { case (l2top_teemsiInfo, io_teemsiInfo) => + l2top_teemsiInfo.fromTile := io_teemsiInfo + } + io.commands <> core.module.io.commands + io.timeRequest <> core.module.io.timeRequest + core.module.io.clintTime := l2top.module.io.clintTime.toCore + l2top.module.io.clintTime.fromTile := io.clintTime + l2top.module.io.reset_vector.fromTile := io.reset_vector + l2top.module.io.reset_mtvec.foreach(x => x.fromTile := io.reset_mtvec.get) + l2top.module.io.cpu_halt.fromCore := core.module.io.cpu_halt + io.cpu_halt := l2top.module.io.cpu_halt.toTile + l2top.module.io.cpu_critical_error.fromCore := core.module.io.cpu_critical_error + io.cpu_crtical_error := l2top.module.io.cpu_critical_error.toTile + l2top.module.io.msiAck.fromCore := core.module.io.msiAck + io.msiAck := l2top.module.io.msiAck.toTile + l2top.module.io.teemsiAck zip core.module.io.teemsiAck foreach { case (l2top_teemsiAck, core_teemsiAck) => + l2top_teemsiAck.fromCore := core_teemsiAck + } + io.teemsiAck zip l2top.module.io.teemsiAck foreach { case (io_teemsiAck, l2top_teemsiAck) => + io_teemsiAck := l2top_teemsiAck.toTile + } + + l2top.module.io.hartIsInReset.resetInFrontend := core.module.io.resetInFrontend + io.hartIsInReset := l2top.module.io.hartIsInReset.toTile + l2top.module.io.traceCoreInterface.fromCore <> core.module.io.traceCoreInterface + io.traceCoreInterface <> l2top.module.io.traceCoreInterface.toTile + + l2top.module.io.beu_errors.icache <> core.module.io.beu_errors.icache + l2top.module.io.beu_errors.dcache <> core.module.io.beu_errors.dcache + l2top.module.io.beu_errors.uncache <> core.module.io.beu_errors.uncache + + l2top.module.io.l2_flush_en.foreach(_ := core.module.io.l2_flush_en) + io.l2_flush_en.foreach(_ := core.module.io.l2_flush_en) + core.module.io.l2_flush_done := l2top.module.io.l2_flush_done.getOrElse(false.B) + io.l2_flush_done.foreach(_ := l2top.module.io.l2_flush_done.getOrElse(false.B)) + io.lcrdy.foreach(in => l2top.module.io.lcrdy.foreach(out => out := in)) + + l2top.module.io.dft.zip(io.dft).foreach({ case (a, b) => a := b }) + l2top.module.io.dft_reset.zip(io.dft_reset).foreach({ case (a, b) => a := b }) + core.module.io.dft.zip(io.dft).foreach({ case (a, b) => a := b }) + core.module.io.dft_reset.zip(io.dft_reset).foreach({ case (a, b) => a := b }) + + if (enableL2) { + // TODO: add ECC interface of L2 + l2top.module.io.pfCtrlFromCore := core.module.io.l2PfCtrl + + l2top.module.io.beu_errors.l2 <> 0.U.asTypeOf(l2top.module.io.beu_errors.l2) + core.module.io.l2_hint.bits.sourceId := l2top.module.io.l2_hint.bits.sourceId + core.module.io.l2_hint.bits.isKeyword := l2top.module.io.l2_hint.bits.isKeyword + core.module.io.l2_hint.valid := l2top.module.io.l2_hint.valid + + core.module.io.l2PfqBusy := false.B + core.module.io.debugTopDown.l2MissMatch := l2top.module.io.debugTopDown.l2MissMatch + l2top.module.io.debugTopDown.robHeadPaddr := core.module.io.debugTopDown.robHeadPaddr + l2top.module.io.debugTopDown.robTrueCommit := core.module.io.debugTopDown.robTrueCommit + l2top.module.io.l2_pmp_resp := core.module.io.l2_pmp_resp + core.module.io.l2_tlb_req <> l2top.module.io.l2_tlb_req + core.module.io.topDownInfo.l2Miss := l2top.module.io.l2Miss + + core.module.io.perfEvents <> l2top.module.io.perfEvents + } else { + + l2top.module.io.beu_errors.l2 <> 0.U.asTypeOf(l2top.module.io.beu_errors.l2) + core.module.io.l2_hint.bits.sourceId := l2top.module.io.l2_hint.bits.sourceId + core.module.io.l2_hint.bits.isKeyword := l2top.module.io.l2_hint.bits.isKeyword + core.module.io.l2_hint.valid := l2top.module.io.l2_hint.valid + + core.module.io.l2PfqBusy := false.B + core.module.io.debugTopDown.l2MissMatch := false.B + core.module.io.topDownInfo.l2Miss := false.B + + core.module.io.l2_tlb_req.req.valid := false.B + core.module.io.l2_tlb_req.req.bits := DontCare + core.module.io.l2_tlb_req.req_kill := DontCare + core.module.io.l2_tlb_req.resp.ready := true.B + + core.module.io.perfEvents <> DontCare + } + + io.debugTopDown.robHeadPaddr := core.module.io.debugTopDown.robHeadPaddr + core.module.io.debugTopDown.l3MissMatch := io.debugTopDown.l3MissMatch + l2top.module.io.l3Miss.fromTile := io.l3Miss + core.module.io.topDownInfo.l3Miss := l2top.module.io.l3Miss.toCore + + io.chi.foreach(_ <> l2top.module.io.chi.get) + l2top.module.io.nodeID.foreach(_ := io.nodeID.get) + + if (debugOpts.ResetGen && enableL2) { + core.module.reset := l2top.module.reset_core + } + } + + lazy val module = new XSTileImp(this) +} diff --git a/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/CSR.scala b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/CSR.scala new file mode 100644 index 000000000..a15e9219c --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/CSR.scala @@ -0,0 +1,1685 @@ +// Replaces XiangShan src/main/scala/xiangshan/backend/fu/CSR.scala (same package) with Buckyball changes. + +/** + * ************************************************************************************* + * Copyright (c) 2020-2021 Institute of Computing Technology, Chinese Academy of Sciences + * Copyright (c) 2020-2021 Peng Cheng Laboratory + * + * XiangShan is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * + * See the Mulan PSL v2 for more details. + * ************************************************************************************* + */ + +package xiangshan.backend.fu + +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import chisel3.util._ +import difftest._ +import freechips.rocketchip.util._ +import utility.MaskedRegMap.WritableMask +import utils._ +import utility._ +import xiangshan.ExceptionNO._ +import xiangshan._ +import xiangshan.backend.fu.util._ +import xiangshan.cache._ +import xiangshan.backend.Bundles.{ExceptionInfo, TrapInstInfo} +import xiangshan.backend.fu.NewCSR.CSREvents.TargetPCBundle +import xiangshan.backend.fu.NewCSR.CSRNamedConstant.ContextStatus +import xiangshan.backend.rob.RobPtr +import utils.MathUtils.{BigIntGenMask, BigIntNot} +import xiangshan.backend.trace._ +import freechips.rocketchip.rocket.CSRs +import system.HasSoCParameter + +class FpuCsrIO extends Bundle { + val fflags = Output(Valid(UInt(5.W))) + val isIllegal = Output(Bool()) + val dirty_fs = Output(Bool()) + val frm = Input(UInt(3.W)) +} + +class VpuCsrIO(implicit p: Parameters) extends XSBundle { + val vstart = Input(UInt(XLEN.W)) + val vxrm = Input(UInt(2.W)) + + val vl = Output(UInt(XLEN.W)) + + val set_vstart = Output(Valid(UInt(XLEN.W))) + val set_vtype = Output(Valid(UInt(XLEN.W))) + val set_vxsat = Output(Valid(UInt(1.W))) + + val dirty_vs = Output(Bool()) +} + +class PerfCounterIO(implicit p: Parameters) extends XSBundle { + val perfEventsFrontend = Vec(numCSRPCntFrontend, new PerfEvent) + val perfEventsBackend = Vec(numCSRPCntCtrl, new PerfEvent) + val perfEventsLsu = Vec(numCSRPCntLsu, new PerfEvent) + val perfEventsHc = Vec(numPCntHc * coreParams.L2NBanks + 1, new PerfEvent) + val retiredInstr = UInt(7.W) + + val frontendInfo = new Bundle { + val ibufFull = Bool() + + val bpuInfo = new Bundle { + val bpRight = UInt(XLEN.W) + val bpWrong = UInt(XLEN.W) + } + + } + + val ctrlInfo = new Bundle { + val robFull = Bool() + val intdqFull = Bool() + val fpdqFull = Bool() + val lsdqFull = Bool() + } + + val memInfo = new Bundle { + val sqFull = Bool() + val lqFull = Bool() + val dcacheMSHRFull = Bool() + } + +} + +class CSRFileIO(implicit p: Parameters) extends XSBundle with HasSoCParameter { + val commands = new examples.core.xiangshan.native.Commands + val commandFlushStores = Output(Bool()) + val commandStoresEmpty = Input(Bool()) + val timeRequest = Decoupled(Bool()) + val hartId = Input(UInt(hartIdLen.W)) + // output (for func === CSROpType.jmp) + val perf = Input(new PerfCounterIO) + val criticalErrorState = Output(Bool()) + val isPerfCnt = Output(Bool()) + // to FPU + val fpu = Flipped(new FpuCsrIO) + // to VPU + val vpu = Flipped(new VpuCsrIO) + // from rob + val exception = Flipped(ValidIO(new ExceptionInfo)) + val robDeqPtr = Input(new RobPtr) + // to ROB + val isXRet = Output(Bool()) + val trapTarget = Output(new TargetPCBundle) + val interrupt = Output(Bool()) + val wfi_event = Output(Bool()) + //trace + val traceCSR = Output(new TraceCSR) + // from LSQ + val memExceptionVAddr = Input(UInt(XLEN.W)) + val memExceptionGPAddr = Input(UInt(XLEN.W)) + val memExceptionIsForVSnonLeafPTE = Input(Bool()) + // from outside cpu,externalInterrupt + val externalInterrupt = Input(new ExternalInterruptIO) + // TLB + val tlb = Output(new TlbCsrBundle) + // Debug Mode + // val singleStep = Output(Bool()) + val debugMode = Output(Bool()) + // Custom microarchiture ctrl signal + val customCtrl = Output(new CustomCSRCtrlIO) + // instruction fetch address translation type + val instrAddrTransType = Output(new AddrTransType) + // ack for axireg from imsic. which indicates imsic can work actively + val msiAck = Output(Bool()) + val teemsiAck = Option.when(soc.IMSICParams.HasTEEIMSIC)(Output(Bool())) +} + +class VtypeStruct(implicit p: Parameters) extends XSBundle { + val vill = UInt(1.W) + val reserved = UInt((XLEN - 9).W) + val vma = UInt(1.W) + val vta = UInt(1.W) + val vsew = UInt(3.W) + val vlmul = UInt(3.W) +} + +/* +class CSR(cfg: FuConfig)(implicit p: Parameters) extends FuncUnit(cfg) + with HasCSRConst + with PMPMethod + with PMAMethod + with HasXSParameter + with SdtrigExt + with DebugCSR +{ + val csrio = io.csrio.get + + val flushPipe = Wire(Bool()) + + val (valid, src1, src2, func) = ( + io.in.valid, + io.in.bits.data.src(0), + io.in.bits.data.imm, + io.in.bits.ctrl.fuOpType + ) + + // CSR define + val virtMode = RegInit(false.B) + csrio.customCtrl.virtMode := virtMode + + class Priv extends Bundle { + val m = Output(Bool()) + val h = Output(Bool()) // unused + val s = Output(Bool()) + val u = Output(Bool()) + } + + class MstatusStruct extends Bundle { + val sd = Output(UInt(1.W)) + + val pad1 = if (XLEN == 64 && HasHExtension) Output(UInt(23.W)) else if (XLEN == 64) Output(UInt(25.W)) else null + val mpv = if (XLEN == 64 && HasHExtension) Output(UInt(1.W)) else null + val gva = if (XLEN == 64 && HasHExtension) Output(UInt(1.W)) else null + val mbe = if (XLEN == 64) Output(UInt(1.W)) else null + val sbe = if (XLEN == 64) Output(UInt(1.W)) else null + val sxl = if (XLEN == 64) Output(UInt(2.W)) else null + val uxl = if (XLEN == 64) Output(UInt(2.W)) else null + val pad0 = if (XLEN == 64) Output(UInt(9.W)) else Output(UInt(8.W)) + + val tsr = Output(UInt(1.W)) + val tw = Output(UInt(1.W)) + val tvm = Output(UInt(1.W)) + val mxr = Output(UInt(1.W)) + val sum = Output(UInt(1.W)) + val mprv = Output(UInt(1.W)) + val xs = Output(UInt(2.W)) + val fs = Output(UInt(2.W)) + val mpp = Output(UInt(2.W)) + val vs = Output(UInt(2.W)) + val spp = Output(UInt(1.W)) + val pie = new Priv + val ie = new Priv + assert(this.getWidth == XLEN) + + def ube = pie.h // a little ugly + def ube_(r: UInt): Unit = { + pie.h := r(0) + } + } + + class HstatusStruct extends Bundle { + val pad4 = if (HSXLEN == 64) Output(UInt(30.W)) else null + val vsxl = if (HSXLEN == 64) Output(UInt(2.W)) else null + val pad3 = Output(UInt(9.W)) + val vtsr = Output(UInt(1.W)) + val vtw = Output(UInt(1.W)) + val vtvm = Output(UInt(1.W)) + val pad2 = Output(UInt(2.W)) + val vgein = Output(UInt(6.W)) + val pad1 = Output(UInt(2.W)) + val hu = Output(UInt(1.W)) + val spvp = Output(UInt(1.W)) + val spv = Output(UInt(1.W)) + val gva = Output(UInt(1.W)) + val vsbe = Output(UInt(1.W)) + val pad0 = Output(UInt(5.W)) + assert(this.getWidth == XLEN) + } + + class Interrupt extends Bundle { +// val d = Output(Bool()) // Debug + val e = new Priv + val t = new Priv + val s = new Priv + } + + // Debug CSRs + val dcsr = RegInit(UInt(32.W), DcsrStruct.init) + val dpc = Reg(UInt(64.W)) + val dscratch0 = Reg(UInt(64.W)) + val dscratch1 = Reg(UInt(64.W)) + val debugMode = RegInit(false.B) + val debugIntrEnable = RegInit(true.B) // debug interrupt will be handle only when debugIntrEnable + csrio.debugMode := debugMode + + val dpcPrev = RegNext(dpc) + XSDebug(dpcPrev =/= dpc, "Debug Mode: dpc is altered! Current is %x, previous is %x\n", dpc, dpcPrev) + + val dcsrData = Wire(new DcsrStruct) + dcsrData := dcsr.asTypeOf(new DcsrStruct) + val dcsrMask = ZeroExt(GenMask(15) | GenMask(13, 11) | GenMask(4) | GenMask(2, 0), XLEN)// Dcsr write mask + def dcsrUpdateSideEffect(dcsr: UInt): UInt = { + val dcsrOld = WireInit(dcsr.asTypeOf(new DcsrStruct)) + val dcsrNew = dcsr | (dcsrOld.prv(0) | dcsrOld.prv(1)).asUInt // turn 10 priv into 11 + dcsrNew + } + // csrio.singleStep := dcsrData.step + csrio.customCtrl.singlestep := dcsrData.step && !debugMode + + // Trigger CSRs + private val tselectPhy = RegInit(0.U(log2Up(TriggerNum).W)) + + private val tdata1RegVec = RegInit(VecInit(Seq.fill(TriggerNum)(Tdata1Bundle.default))) + private val tdata2RegVec = RegInit(VecInit(Seq.fill(TriggerNum)(0.U(64.W)))) + private val tdata1WireVec = tdata1RegVec.map(_.asTypeOf(new Tdata1Bundle)) + private val tdata2WireVec = tdata2RegVec + private val tdata1Selected = tdata1RegVec(tselectPhy).asTypeOf(new Tdata1Bundle) + private val tdata2Selected = tdata2RegVec(tselectPhy) + private val newTriggerChainVec = UIntToOH(tselectPhy, TriggerNum).asBools | tdata1WireVec.map(_.data.asTypeOf(new MControlData).chain) + private val newTriggerChainIsLegal = TriggerCheckChainLegal(newTriggerChainVec, TriggerChainMaxLength) + val tinfo = RegInit((BigInt(1) << TrigTypeEnum.MCONTROL.litValue.toInt).U(XLEN.W)) // This value should be 4.U + + + def WriteTselect(wdata: UInt) = { + Mux(wdata < TriggerNum.U, wdata(log2Up(TriggerNum) - 1, 0), tselectPhy) + } + + def GenTdataDistribute(tdata1: Tdata1Bundle, tdata2: UInt): MatchTriggerIO = { + val res = Wire(new MatchTriggerIO) + val mcontrol: MControlData = WireInit(tdata1.data.asTypeOf(new MControlData)) + res.matchType := mcontrol.match_.asUInt + res.select := mcontrol.select + res.timing := mcontrol.timing + res.action := mcontrol.action.asUInt + res.chain := mcontrol.chain + res.execute := mcontrol.execute + res.load := mcontrol.load + res.store := mcontrol.store + res.tdata2 := tdata2 + res + } + + csrio.customCtrl.frontend_trigger.tUpdate.bits.addr := tselectPhy + csrio.customCtrl.mem_trigger.tUpdate.bits.addr := tselectPhy + csrio.customCtrl.frontend_trigger.tUpdate.bits.tdata := GenTdataDistribute(tdata1Selected, tdata2Selected) + csrio.customCtrl.mem_trigger.tUpdate.bits.tdata := GenTdataDistribute(tdata1Selected, tdata2Selected) + + // Machine-Level CSRs + // mtvec: {BASE (WARL), MODE (WARL)} where mode is 0 or 1 + val mtvecMask = ~(0x2.U(XLEN.W)) + val mtvec = RegInit(UInt(XLEN.W), 0.U) + val mcounteren = RegInit(UInt(XLEN.W), 0.U) + // Currently, XiangShan don't support Unprivileged Counter/Timers CSRs ("Zicntr" and "Zihpm") + val mcounterenMask = 0.U(XLEN.W) + val mcause = RegInit(UInt(XLEN.W), 0.U) + val mtval = RegInit(UInt(XLEN.W), 0.U) + val mtval2 = RegInit(UInt(XLEN.W), 0.U) + val mtinst = RegInit(UInt(XLEN.W), 0.U) + val mepc = RegInit(UInt(XLEN.W), 0.U) + // Page 36 in riscv-priv: The low bit of mepc (mepc[0]) is always zero. + val mepcMask = ~(0x1.U(XLEN.W)) + + val mie = RegInit(0.U(XLEN.W)) + val mipWire = WireInit(0.U.asTypeOf(new Interrupt)) + val mipReg = RegInit(0.U(XLEN.W)) + val mipMask = ZeroExt(Array( + 1, // SSIP + 2, // VSSIP + 3, // MSIP + 5, // STIP + 6, // VSTIP + 7, // MTIP + 9, // SEIP + 10, // VSEIP + 11, // MEIP + 12, // SGEIP + ).map(GenMask(_)).reduce(_ | _), XLEN) + val mip = (mipWire.asUInt | mipReg).asTypeOf(new Interrupt) + + val mip_mie_WMask_H = if(HasHExtension){((1 << 2) | (1 << 6) | (1 << 10) | (1 << 12)).U(XLEN.W)}else{0.U(XLEN.W)} + val vssip_Mask = (1 << 2).U(XLEN.W) + + val mipWMask = vssip_Mask | ((1 << 9) | (1 << 5) | (1 << 1)).U(XLEN.W) + val mieWMask = mip_mie_WMask_H | "haaa".U(XLEN.W) + + def getMisaMxl(mxl: BigInt): BigInt = mxl << (XLEN - 2) + def getMisaExt(ext: Char): Long = 1 << (ext.toInt - 'a'.toInt) + var extList = List('a', 's', 'i', 'u') + if (HasMExtension) { extList = extList :+ 'm' } + if (HasCExtension) { extList = extList :+ 'c' } + if (HasHExtension) { extList = extList :+ 'h' } + if (HasFPU) { extList = extList ++ List('f', 'd') } + if (HasVPU) { extList = extList :+ 'v' } + val misaInitVal = getMisaMxl(2) | extList.foldLeft(0L)((sum, i) => sum | getMisaExt(i)) //"h8000000000141185".U + val misa = RegInit(UInt(XLEN.W), misaInitVal.U) + println(s"[CSR] supported isa ext: $extList") + + // MXL = 2 | 0 | EXT = b 00 0000 0100 0001 0001 0000 0101 + // (XLEN-1, XLEN-2) | |(25, 0) ZY XWVU TSRQ PONM LKJI HGFE DCBA + + // Machine Configuration + val menvcfg = RegInit(UInt(XLEN.W), 0.U) + + val mvendorid = RegInit(UInt(XLEN.W), 0.U) // this is a non-commercial implementation + val marchid = RegInit(UInt(XLEN.W), 25.U) // architecture id for XiangShan is 25; see https://github.com/riscv/riscv-isa-manual/blob/master/marchid.md + val mimpid = RegInit(UInt(XLEN.W), 0.U) // provides a unique encoding of the version of the processor implementation + val mhartid = Reg(UInt(XLEN.W)) // the hardware thread running the code + when (RegNext(RegNext(reset.asBool) && !reset.asBool)) { + mhartid := csrio.hartId + } + val mconfigptr = RegInit(UInt(XLEN.W), 0.U) // the read-only pointer pointing to the platform config structure, 0 for not supported. + val mstatus = RegInit("ha00002200".U(XLEN.W)) + + // mstatus Value Table + // | sd | Read Only + // | pad1 | WPRI + // | sxl | hardlinked to 10, use 00 to pass xv6 test + // | uxl | hardlinked to 10 + // | pad0 | + // | tsr | + // | tw | + // | tvm | + // | mxr | + // | sum | + // | mprv | + // | xs | 00 | + // | fs | 01 | + // | mpp | 00 | + // | vs | 01 | + // | spp | 0 | + // | pie | 0000 | pie.h is used as UBE + // | ie | 0000 | uie hardlinked to 0, as N ext is not implemented + + val mstatusStruct = mstatus.asTypeOf(new MstatusStruct) + def mstatusUpdateSideEffect(mstatus: UInt): UInt = { + val mstatusOld = WireInit(mstatus.asTypeOf(new MstatusStruct)) + // Cat(sd, other) + val mstatusNew = Cat( + mstatusOld.xs === ContextStatus.dirty || mstatusOld.fs === ContextStatus.dirty || mstatusOld.vs === ContextStatus.dirty, + mstatus(XLEN-2, 0) + ) + mstatusNew + } + def vsstatusUpdateSideEffect(vsstatus: UInt): UInt = { + val vsstatusOld = WireInit(vsstatus.asTypeOf(new MstatusStruct)) + val vsstatusNew = Cat(vsstatusOld.xs === "b11".U || vsstatusOld.fs === "b11".U, vsstatus(XLEN-2, 0)) + vsstatusNew + } + val mstatusWMask = (~ZeroExt(( + GenMask(63) | // SD is read-only + (if(HasHExtension) + GenMask(62, 40) // WPRI + else + GenMask(62, 38) )| // WPRI + GenMask(35, 32) | // SXL and UXL cannot be changed + GenMask(31, 23) | // WPRI + GenMask(16, 15) | // XS is read-only + GenMask(6) | // UBE, always little-endian (0) + GenMask(4) | // WPRI + GenMask(2) | // WPRI + GenMask(0) // WPRI + ), 64)).asUInt + + val medeleg = RegInit(UInt(XLEN.W), 0.U) + val midelegInit = if(HasHExtension){((1 << 12) | (1 << 10) | (1 << 6) | (1 << 2)).U}else{0.U} + val medelegWMask = if(HasHExtension) { + "hf0b7ff".U(XLEN.W) + }else { + "hb3ff".U(XLEN.W) + } + + + val mideleg = RegInit(UInt(XLEN.W), midelegInit) + val mscratch = RegInit(UInt(XLEN.W), 0.U) + + val midelegWMask = "h222".U(XLEN.W) + // PMP Mapping + val pmp = Wire(Vec(NumPMP, new PMPEntry())) // just used for method parameter + val pma = Wire(Vec(NumPMA, new PMPEntry())) // just used for method parameter + val pmpMapping = pmp_gen_mapping(pmp_init, NumPMP, PmpcfgBase, PmpaddrBase, pmp) + val pmaMapping = pmp_gen_mapping(pma_init, NumPMA, PmacfgBase, PmaaddrBase, pma) + // !WARNNING: pmp and pma CSRs are not checked in difftest. + + // Supervisor-Level CSRs + + val sstatusWNmask: BigInt = ( + BigIntGenMask(63) | // SD is read-only + BigIntGenMask(62, 34) | // WPRI + BigIntGenMask(33, 32) | // UXL is hard-wired to 64(b10) + BigIntGenMask(31, 20) | // WPRI + BigIntGenMask(17) | // WPRI + BigIntGenMask(16, 15) | // XS is read-only to zero + BigIntGenMask(12, 11) | // WPRI + BigIntGenMask(7) | // WPRI + BigIntGenMask(6) | // UBE is always little-endian (0) + BigIntGenMask(4, 2) | // WPRI + BigIntGenMask(0) // WPRI + ) + + val sstatusWmask = BigIntNot(sstatusWNmask).U(XLEN.W) + val sstatusRmask = ( + BigIntGenMask(63) | // SD + BigIntGenMask(33, 32) | // UXL + BigIntGenMask(19) | // MXR + BigIntGenMask(18) | // SUM + BigIntGenMask(16, 15) | // XS + BigIntGenMask(14, 13) | // FS + BigIntGenMask(10, 9 ) | // VS + BigIntGenMask(8) | // SPP + BigIntGenMask(6) | // UBE: hard wired to 0 + BigIntGenMask(5) | // SPIE + BigIntGenMask(1) + ).U(XLEN.W) + + println(s"sstatusWNmask: 0x${sstatusWNmask.toString(16)}") + println(s"sstatusWmask: 0x${sstatusWmask.litValue.toString(16)}") + println(s"sstatusRmask: 0x${sstatusRmask.litValue.toString(16)}") + + // stvec: {BASE (WARL), MODE (WARL)} where mode is 0 or 1 + val stvecMask = ~(0x2.U(XLEN.W)) + val stvec = RegInit(UInt(XLEN.W), 0.U) + // val sie = RegInit(0.U(XLEN.W)) + val sieMask = "h222".U & mideleg + val sipMask = "h222".U & mideleg + val sipWMask = "h2".U(XLEN.W) // ssip is writeable in smode + val satp = if(EnbaleTlbDebug) RegInit(UInt(XLEN.W), "h8000000000087fbe".U) else RegInit(0.U(XLEN.W)) + // val satp = RegInit(UInt(XLEN.W), "h8000000000087fbe".U) // only use for tlb naive debug + // val satpMask = "h80000fffffffffff".U(XLEN.W) // disable asid, mode can only be 8 / 0 + // TODO: use config to control the length of asid + // val satpMask = "h8fffffffffffffff".U(XLEN.W) // enable asid, mode can only be 8 / 0 + val satpMask = Cat("h8".U(Satp_Mode_len.W), satp_part_wmask(Satp_Asid_len, AsidLength), satp_part_wmask(Satp_Addr_len, PAddrBits-12)) + val sepc = RegInit(UInt(XLEN.W), 0.U) + // Page 60 in riscv-priv: The low bit of sepc (sepc[0]) is always zero. + val sepcMask = ~(0x1.U(XLEN.W)) + val scause = RegInit(UInt(XLEN.W), 0.U) + val stval = RegInit(UInt(XLEN.W), 0.U) + val sscratch = RegInit(UInt(XLEN.W), 0.U) + val scounteren = RegInit(UInt(XLEN.W), 0.U) + val senvcfg = RegInit(UInt(XLEN.W), 0.U) // !WARNING: there is no logic about this CSR. + // Currently, XiangShan don't support Unprivileged Counter/Timers CSRs ("Zicntr" and "Zihpm") + val scounterenMask = 0.U(XLEN.W) + + // sbpctl + // Bits 0-7: {LOOP, RAS, SC, TAGE, BIM, BTB, uBTB} + val sbpctl = RegInit(UInt(XLEN.W), "h7f".U) + csrio.customCtrl.bp_ctrl.ubtb_enable := sbpctl(0) + csrio.customCtrl.bp_ctrl.btb_enable := sbpctl(1) + csrio.customCtrl.bp_ctrl.bim_enable := sbpctl(2) + csrio.customCtrl.bp_ctrl.tage_enable := sbpctl(3) + csrio.customCtrl.bp_ctrl.sc_enable := sbpctl(4) + csrio.customCtrl.bp_ctrl.ras_enable := sbpctl(5) + csrio.customCtrl.bp_ctrl.loop_enable := sbpctl(6) + + // spfctl Bit 0: L1I Cache Prefetcher Enable + // spfctl Bit 1: L2Cache Prefetcher Enable + // spfctl Bit 2: L1D Cache Prefetcher Enable + // spfctl Bit 3: L1D train prefetch on hit + // spfctl Bit 4: L1D prefetch enable agt + // spfctl Bit 5: L1D prefetch enable pht + // spfctl Bit [9:6]: L1D prefetch active page threshold + // spfctl Bit [15:10]: L1D prefetch active page stride + // turn off L2 BOP, turn on L1 SMS by default + val spfctl = RegInit(UInt(XLEN.W), Seq( + 0 << 17, // L2 pf store only [17] init: false + 1 << 16, // L1D pf enable stride [16] init: true + 30 << 10, // L1D active page stride [15:10] init: 30 + 12 << 6, // L1D active page threshold [9:6] init: 12 + 1 << 5, // L1D enable pht [5] init: true + 1 << 4, // L1D enable agt [4] init: true + 0 << 3, // L1D train on hit [3] init: false + 1 << 2, // L1D pf enable [2] init: true + 1 << 1, // L2 pf enable [1] init: true + 1 << 0, // L1I pf enable [0] init: true + ).reduce(_|_).U(XLEN.W)) + csrio.customCtrl.l1I_pf_enable := spfctl(0) + csrio.customCtrl.l2_pf_enable := spfctl(1) + csrio.customCtrl.l1D_pf_enable := spfctl(2) + csrio.customCtrl.l1D_pf_train_on_hit := spfctl(3) + csrio.customCtrl.l1D_pf_enable_agt := spfctl(4) + csrio.customCtrl.l1D_pf_enable_pht := spfctl(5) + csrio.customCtrl.l1D_pf_active_threshold := spfctl(9, 6) + csrio.customCtrl.l1D_pf_active_stride := spfctl(15, 10) + csrio.customCtrl.l1D_pf_enable_stride := spfctl(16) + csrio.customCtrl.l2_pf_store_only := spfctl(17) + + // slvpredctl: load violation predict settings + // Default reset period: 2^16 + // Why this number: reset more frequently while keeping the overhead low + // Overhead: extra two redirections in every 64K cycles => ~0.1% overhead + val slvpredctl = Reg(UInt(XLEN.W)) + when(reset.asBool) { + slvpredctl := Constantin.createRecord("slvpredctl", 0x60) + } + csrio.customCtrl.lvpred_disable := slvpredctl(0) + csrio.customCtrl.no_spec_load := slvpredctl(1) + csrio.customCtrl.storeset_wait_store := slvpredctl(2) + csrio.customCtrl.storeset_no_fast_wakeup := slvpredctl(3) + csrio.customCtrl.lvpred_timeout := slvpredctl(8, 4) + + // smblockctl: memory block configurations + // +------------------------+---+---+---+----+----+-----+--------+ + // |XLEN-1 10| 9 | 8 | 7 | 6 | 5 | 4 |3 0| + // +------------------------+---+---+---+----+----+-----+--------+ + // | Reserved | L | S | O | CE | SP | LVC | Th | + // +------------------------+---+---+---+----+----+-----+--------+ + // Description: + // Bit 3-0 : Store buffer flush threshold (Th). + // Bit 4 : Enable load violation check after reset (LVC). + // Bit 5 : Enable soft-prefetch after reset (SP). + // Bit 6 : Enable cache error after reset (CE). + // Bit 7 : Enable uncache write outstanding (O). + // Bit 8 : Enable unaligned store (S). + // Bit 9 : Enable unaligned load (L). + // Others : Reserved. + + val smblockctl_init_val = + (0xf & StoreBufferThreshold) | + (EnableLdVioCheckAfterReset.toInt << 4) | + (EnableSoftPrefetchAfterReset.toInt << 5) | + (EnableCacheErrorAfterReset.toInt << 6) | + (EnableUncacheWriteOutstanding.toInt << 7) | + (EnableHardwareStoreMisalign.toInt << 8) | + (EnableHardwareLoadMisalign.toInt << 9) + val smblockctl = RegInit(UInt(XLEN.W), smblockctl_init_val.U) + csrio.customCtrl.sbuffer_threshold := smblockctl(3, 0) + // bits 4: enable load load violation check + csrio.customCtrl.ldld_vio_check_enable := smblockctl(4) + csrio.customCtrl.soft_prefetch_enable := smblockctl(5) + csrio.customCtrl.cache_error_enable := smblockctl(6) + csrio.customCtrl.uncache_write_outstanding_enable := smblockctl(7) + csrio.customCtrl.hd_misalign_st_enable := smblockctl(8) + csrio.customCtrl.hd_misalign_ld_enable := smblockctl(9) + + println("CSR smblockctl init value:") + println(" Store buffer replace threshold: " + StoreBufferThreshold) + println(" Enable ld-ld vio check after reset: " + EnableLdVioCheckAfterReset) + println(" Enable soft prefetch after reset: " + EnableSoftPrefetchAfterReset) + println(" Enable cache error after reset: " + EnableCacheErrorAfterReset) + println(" Enable uncache write outstanding: " + EnableUncacheWriteOutstanding) + println(" Enable unaligned store: " + EnableHardwareStoreMisalign) + println(" Enable unaligned load: " + EnableHardwareLoadMisalign) + + val srnctl = RegInit(UInt(XLEN.W), "h7".U) + csrio.customCtrl.fusion_enable := srnctl(0) + csrio.customCtrl.wfi_enable := srnctl(2) + + // Hypervisor CSRs + val hstatusWMask = "h7003c0".U(XLEN.W) + // hstatus: vtsr, vtw, vtvm, hu, spvp, spv, gva, + val hstatus = RegInit("h200000000".U(XLEN.W)) + val hstatusStruct = hstatus.asTypeOf(new HstatusStruct) + val hedeleg = RegInit(UInt(XLEN.W), 0.U) + val hideleg = RegInit(UInt(XLEN.W), 0.U) + val hidelegRMask = mideleg + val hidelegWMask = ((1 << 10) | (1 << 6) | (1 << 2)).U(XLEN.W) + val hgeie = RegInit(UInt(XLEN.W), 0.U) + val htval = RegInit(UInt(XLEN.W), 0.U) + // hvip hip hie is part of mip or mie + val hvipMask = ((1 << 10) | (1 << 6) | (1 << 2)).U(XLEN.W) + val hipRMask = (((1 << 12).U | hvipMask) & mideleg) + val hipWMask = ((1 << 2).U & mideleg)// vssip + val hieMask = hipRMask + val htinst = RegInit(UInt(XLEN.W), 0.U) + val hgeip = RegInit(UInt(XLEN.W), 0.U) + val henvcfg = RegInit(UInt(XLEN.W), 0.U) + val hgatp = RegInit(UInt(XLEN.W), 0.U) + val hgatpMask = Cat("h8".U(Hgatp_Mode_len.W), satp_part_wmask(Hgatp_Vmid_len, VmidLength), satp_part_wmask(Hgatp_Addr_len, PAddrBits-12)) + // val htimedelta = RegInit(UInt(XLEN.W), 0.U) + val hcounteren = RegInit(UInt(XLEN.W), 0.U) + // Currently, XiangShan don't support Unprivileged Counter/Timers CSRs ("Zicntr" and "Zihpm") + val hcounterenMask = 0.U(XLEN.W) + + val vsstatus = RegInit("h200002000".U(XLEN.W)) + val vsstatusStruct = vsstatus.asTypeOf(new MstatusStruct) + //vsie vsip + val vsMask = ((1 << 10) | (1 << 6) | (1 << 2)).U(XLEN.W) + val vsip_ie_Mask = ZeroExt((hideleg & mideleg & vsMask), XLEN) + val vsip_WMask = ZeroExt((hideleg & mideleg & vssip_Mask), XLEN) + val vstvec = RegInit(UInt(XLEN.W), 0.U) + val vsscratch = RegInit(UInt(XLEN.W), 0.U) + val vsepc = RegInit(UInt(XLEN.W), 0.U) + val vscause = RegInit(UInt(XLEN.W), 0.U) + val vstval = RegInit(UInt(XLEN.W), 0.U) + val vsatp = RegInit(UInt(XLEN.W), 0.U) + val tlbBundle = Wire(new TlbCsrBundle) + tlbBundle.satp.apply(satp) + tlbBundle.vsatp.apply(vsatp) + tlbBundle.hgatp.apply(hgatp) + csrio.tlb := tlbBundle + + // User-Level CSRs + val uepc = Reg(UInt(XLEN.W)) + + // fcsr + class FcsrStruct extends Bundle { + val reserved = UInt((XLEN-3-5).W) + val frm = UInt(3.W) + val fflags = UInt(5.W) + assert(this.getWidth == XLEN) + } + val fcsr = RegInit(0.U(XLEN.W)) + // set mstatus->sd and mstatus->fs when true + val csrw_dirty_fp_state = WireInit(false.B) + + def frm_wfn(wdata: UInt): UInt = { + val fcsrOld = WireInit(fcsr.asTypeOf(new FcsrStruct)) + csrw_dirty_fp_state := true.B + fcsrOld.frm := wdata(2,0) + fcsrOld.asUInt + } + def frm_rfn(rdata: UInt): UInt = rdata(7,5) + + def fflags_wfn(update: Boolean)(wdata: UInt): UInt = { + val fcsrOld = fcsr.asTypeOf(new FcsrStruct) + val fcsrNew = WireInit(fcsrOld) + if (update) { + fcsrNew.fflags := wdata(4,0) | fcsrOld.fflags + } else { + fcsrNew.fflags := wdata(4,0) + } + fcsrNew.asUInt + } + def fflags_rfn(rdata:UInt): UInt = rdata(4,0) + + def fcsr_wfn(wdata: UInt): UInt = { + val fcsrOld = WireInit(fcsr.asTypeOf(new FcsrStruct)) + csrw_dirty_fp_state := true.B + Cat(fcsrOld.reserved, wdata.asTypeOf(fcsrOld).frm, wdata.asTypeOf(fcsrOld).fflags) + } + + val fcsrMapping = Map( + MaskedRegMap(Fflags, fcsr, wfn = fflags_wfn(update = false), rfn = fflags_rfn), + MaskedRegMap(Frm, fcsr, wfn = frm_wfn, rfn = frm_rfn), + MaskedRegMap(Fcsr, fcsr, wfn = fcsr_wfn) + ) + + // Vector extension CSRs + val vstart = RegInit(0.U(XLEN.W)) + val vcsr = RegInit(0.U(XLEN.W)) + val vl = Reg(UInt(XLEN.W)) + val vtype = Reg(UInt(XLEN.W)) + val vlenb = RegInit(VDataBytes.U(XLEN.W)) + + // set mstatus->sd and mstatus->vs when true + val csrw_dirty_vs_state = WireInit(false.B) + + // vcsr is mapped to vxrm and vxsat + class VcsrStruct extends Bundle { + val reserved = UInt((XLEN-3).W) + val vxrm = UInt(2.W) + val vxsat = UInt(1.W) + assert(this.getWidth == XLEN) + } + + def vxrm_wfn(wdata: UInt): UInt = { + val vcsrOld = WireInit(vcsr.asTypeOf(new VcsrStruct)) + csrw_dirty_vs_state := true.B + vcsrOld.vxrm := wdata(1,0) + vcsrOld.asUInt + } + def vxrm_rfn(rdata: UInt): UInt = rdata(2,1) + + def vxsat_wfn(update: Boolean)(wdata: UInt): UInt = { + val vcsrOld = WireInit(vcsr.asTypeOf(new VcsrStruct)) + val vcsrNew = WireInit(vcsrOld) + csrw_dirty_vs_state := true.B + if (update) { + vcsrNew.vxsat := wdata(0) | vcsrOld.vxsat + } else { + vcsrNew.vxsat := wdata(0) + } + vcsrNew.asUInt + } + def vxsat_rfn(rdata: UInt): UInt = rdata(0) + + def vcsr_wfn(wdata: UInt): UInt = { + val vcsrOld = WireInit(vcsr.asTypeOf(new VcsrStruct)) + csrw_dirty_vs_state := true.B + vcsrOld.vxrm := wdata.asTypeOf(vcsrOld).vxrm + vcsrOld.vxsat := wdata.asTypeOf(vcsrOld).vxsat + vcsrOld.asUInt + } + + val vcsrMapping = Map( + MaskedRegMap(Vstart, vstart), + MaskedRegMap(Vxrm, vcsr, wfn = vxrm_wfn, rfn = vxrm_rfn), + MaskedRegMap(Vxsat, vcsr, wfn = vxsat_wfn(false), rfn = vxsat_rfn), + MaskedRegMap(Vcsr, vcsr, wfn = vcsr_wfn), + MaskedRegMap(Vl, vl), + MaskedRegMap(Vtype, vtype), + MaskedRegMap(Vlenb, vlenb), + ) + + // Hart Privilege Mode + val privilegeMode = RegInit(UInt(2.W), ModeM) + + //val perfEventscounten = List.fill(nrPerfCnts)(RegInit(false(Bool()))) + // Perf Counter + val nrPerfCnts = 29 // 3...31 + val privilegeModeOH = UIntToOH(privilegeMode) + val perfEventscounten = RegInit(0.U.asTypeOf(Vec(nrPerfCnts, Bool()))) + val perfCnts = List.fill(nrPerfCnts)(RegInit(0.U(XLEN.W))) + val perfEvents = List.fill(8)(RegInit("h0000000000".U(XLEN.W))) ++ + List.fill(8)(RegInit("h4010040100".U(XLEN.W))) ++ + List.fill(8)(RegInit("h8020080200".U(XLEN.W))) ++ + List.fill(5)(RegInit("hc0300c0300".U(XLEN.W))) + for (i <-0 until nrPerfCnts) { + perfEventscounten(i) := (perfEvents(i)(63,60) & privilegeModeOH).orR + } + + val hpmEvents = Wire(Vec(numPCntHc * coreParams.L2NBanks, new PerfEvent)) + for (i <- 0 until numPCntHc * coreParams.L2NBanks) { + hpmEvents(i) := csrio.perf.perfEventsHc(i) + } + + // print perfEvents + val allPerfEvents = hpmEvents.map(x => (s"Hc", x.value)) + if (printEventCoding) { + for (((name, inc), i) <- allPerfEvents.zipWithIndex) { + println("CSR perfEvents Set", name, inc, i) + } + } + + val csrevents = perfEvents.slice(24, 29) + val hpm_hc = HPerfMonitor(csrevents, hpmEvents) + val mcountinhibit = RegInit(0.U(XLEN.W)) + val mcycle = RegInit(0.U(XLEN.W)) + mcycle := mcycle + 1.U + val minstret = RegInit(0.U(XLEN.W)) + val perf_events = csrio.perf.perfEventsFrontend ++ + csrio.perf.perfEventsBackend ++ + csrio.perf.perfEventsLsu ++ + hpm_hc.getPerf + minstret := minstret + RegNext(csrio.perf.retiredInstr) + for(i <- 0 until 29){ + perfCnts(i) := Mux(mcountinhibit(i+3) | !perfEventscounten(i), perfCnts(i), perfCnts(i) + perf_events(i).value) + } + + // CSR reg map + val basicPrivMapping = Map( + + // Unprivileged Floating-Point CSRs + // Has been mapped above + + // TODO: support Unprivileged Counter/Timers CSRs ("Zicntr" and "Zihpm") + // Unprivileged Counter/Timers + MaskedRegMap(Cycle, mcycle), + // We don't support read time CSR. + // MaskedRegMap(Time, mtime), + MaskedRegMap(Instret, minstret), + + //--- Supervisor Trap Setup --- + MaskedRegMap(Sstatus, mstatus, sstatusWmask, mstatusUpdateSideEffect, sstatusRmask), + // MaskedRegMap(Sedeleg, Sedeleg), + // MaskedRegMap(Sideleg, Sideleg), + MaskedRegMap(Sie, mie, sieMask, MaskedRegMap.NoSideEffect, sieMask), + MaskedRegMap(Stvec, stvec, stvecMask, MaskedRegMap.NoSideEffect, stvecMask), + MaskedRegMap(Scounteren, scounteren, scounterenMask), + + //--- Supervisor Configuration --- + MaskedRegMap(Senvcfg, senvcfg), + + //--- Supervisor Trap Handling --- + MaskedRegMap(Sscratch, sscratch), + MaskedRegMap(Sepc, sepc, sepcMask, MaskedRegMap.NoSideEffect, sepcMask), + MaskedRegMap(Scause, scause), + MaskedRegMap(Stval, stval), + MaskedRegMap(Sip, mipReg.asUInt, sipWMask, MaskedRegMap.NoSideEffect, sipMask, x => (mipWire.asUInt | x) & sipMask), + + //--- Supervisor Protection and Translation --- + MaskedRegMap(Satp, satp, satpMask, MaskedRegMap.NoSideEffect, satpMask), + + //--- Supervisor Custom Read/Write Registers + MaskedRegMap(Sbpctl, sbpctl), + MaskedRegMap(Spfctl, spfctl), + MaskedRegMap(Slvpredctl, slvpredctl), + MaskedRegMap(Smblockctl, smblockctl), + MaskedRegMap(Srnctl, srnctl), + + //--- Machine Information Registers --- + MaskedRegMap(Mvendorid, mvendorid, 0.U(XLEN.W), MaskedRegMap.Unwritable), + MaskedRegMap(Marchid, marchid, 0.U(XLEN.W), MaskedRegMap.Unwritable), + MaskedRegMap(Mimpid, mimpid, 0.U(XLEN.W), MaskedRegMap.Unwritable), + MaskedRegMap(Mhartid, mhartid, 0.U(XLEN.W), MaskedRegMap.Unwritable), + MaskedRegMap(Mconfigptr, mconfigptr, 0.U(XLEN.W), MaskedRegMap.Unwritable), + + //--- Machine Configuration Registers --- + MaskedRegMap(Menvcfg, menvcfg), + + //--- Machine Trap Setup --- + MaskedRegMap(Mstatus, mstatus, mstatusWMask, mstatusUpdateSideEffect), + MaskedRegMap(Misa, misa, 0.U, MaskedRegMap.Unwritable), // now whole misa is unchangeable + MaskedRegMap(Medeleg, medeleg, medelegWMask), + MaskedRegMap(Mideleg, mideleg, midelegWMask), + MaskedRegMap(Mie, mie, mieWMask), + MaskedRegMap(Mtvec, mtvec, mtvecMask, MaskedRegMap.NoSideEffect, mtvecMask), + MaskedRegMap(Mcounteren, mcounteren, mcounterenMask), + + //--- Machine Trap Handling --- + MaskedRegMap(Mscratch, mscratch), + MaskedRegMap(Mepc, mepc, mepcMask, MaskedRegMap.NoSideEffect, mepcMask), + MaskedRegMap(Mcause, mcause), + MaskedRegMap(Mtval, mtval), + MaskedRegMap(Mip, mipReg.asUInt, mipWMask, MaskedRegMap.NoSideEffect, mipMask, x => (mipWire.asUInt | x) & mipMask), + + //--- Trigger --- + MaskedRegMap(Tselect, tselectPhy, WritableMask, WriteTselect), + // Todo: support chain length = 2 + MaskedRegMap(Tdata1, tdata1RegVec(tselectPhy), + WritableMask, + x => Tdata1Bundle.Write(x, tdata1RegVec(tselectPhy), newTriggerChainIsLegal, debug_mode = debugMode), + WritableMask, + x => Tdata1Bundle.Read(x)), + MaskedRegMap(Tdata2, tdata2RegVec(tselectPhy)), + MaskedRegMap(Tinfo, tinfo, 0.U(XLEN.W), MaskedRegMap.Unwritable), + + //--- Debug Mode --- + MaskedRegMap(Dcsr, dcsr, dcsrMask, dcsrUpdateSideEffect), + MaskedRegMap(Dpc, dpc), + MaskedRegMap(Dscratch0, dscratch0), + MaskedRegMap(Dscratch1, dscratch1), + MaskedRegMap(Mcountinhibit, mcountinhibit), + MaskedRegMap(Mcycle, mcycle), + MaskedRegMap(Minstret, minstret), + ) + + // hypervisor csr map + val hcsrMapping = Map( + //--- Hypervisor Trap Setup --- + MaskedRegMap(Hstatus, hstatus, hstatusWMask), + MaskedRegMap(Hedeleg, hedeleg), + MaskedRegMap(Hideleg, hideleg, hidelegWMask, MaskedRegMap.NoSideEffect, hidelegRMask), + MaskedRegMap(Hie, mie, hieMask, MaskedRegMap.NoSideEffect, hieMask), + MaskedRegMap(Hcounteren, hcounteren, hcounterenMask), + MaskedRegMap(Hgeie, hgeie), + + //--- Hypervisor Trap Handling --- + MaskedRegMap(Htval, htval), + MaskedRegMap(Hip, mipReg.asUInt, hipWMask, MaskedRegMap.NoSideEffect, hipRMask, x => (mipWire.asUInt | x) & hipRMask), + MaskedRegMap(Hvip, mipReg.asUInt, hvipMask, MaskedRegMap.NoSideEffect, hvipMask, x => (mipWire.asUInt | x) & hvipMask), + MaskedRegMap(Htinst, htinst), + MaskedRegMap(Hgeip, hgeip), + + //--- Hypervisor Configuration --- + MaskedRegMap(Henvcfg, henvcfg), + + //--- Hypervisor Protection and Translation --- + MaskedRegMap(Hgatp, hgatp, hgatpMask, MaskedRegMap.NoSideEffect, hgatpMask), + + //--- Hypervisor Counter/Timer Virtualization Registers --- + // MaskedRegMap(Htimedelta, htimedelta), + + //--- Virtual Supervisor Registers --- + MaskedRegMap(Vsstatus, vsstatus, rmask = sstatusRmask, wmask = sstatusWmask, wfn = vsstatusUpdateSideEffect), + MaskedRegMap(Vsie, mie, rmask = vsip_ie_Mask, wmask = vsip_ie_Mask), + MaskedRegMap(Vstvec, vstvec), + MaskedRegMap(Vsscratch, vsscratch), + MaskedRegMap(Vsepc, vsepc), + MaskedRegMap(Vscause, vscause), + MaskedRegMap(Vstval, vstval), + MaskedRegMap(Vsip, mipReg.asUInt, vsip_WMask, MaskedRegMap.NoSideEffect, vsip_ie_Mask, x => mipWire.asUInt | x), + MaskedRegMap(Vsatp, vsatp, satpMask, MaskedRegMap.NoSideEffect, satpMask), + + //--- Machine Registers --- + MaskedRegMap(Mtval2, mtval2), + MaskedRegMap(Mtinst, mtinst), + ) + + val perfCntMapping = (0 until 29).map(i => {Map( + MaskedRegMap(addr = Mhpmevent3 +i, + reg = perfEvents(i), + wmask = "hf87fff3fcff3fcff".U(XLEN.W)), + MaskedRegMap(addr = Mhpmcounter3 +i, + reg = perfCnts(i)), + MaskedRegMap(addr = Hpmcounter3 + i, + reg = perfCnts(i)) + )}).fold(Map())((a,b) => a ++ b) + // TODO: mechanism should be implemented later + // val MhpmcounterStart = Mhpmcounter3 + // val MhpmeventStart = Mhpmevent3 + // for (i <- 0 until nrPerfCnts) { + // perfCntMapping += MaskedRegMap(MhpmcounterStart + i, perfCnts(i)) + // perfCntMapping += MaskedRegMap(MhpmeventStart + i, perfEvents(i)) + // } + + val cacheopRegs = CacheInstrucion.CacheInsRegisterList.map{case (name, attribute) => { + name -> RegInit(0.U(attribute("width").toInt.W)) + }} + val cacheopMapping = CacheInstrucion.CacheInsRegisterList.map{case (name, attribute) => { + MaskedRegMap( + Scachebase + attribute("offset").toInt, + cacheopRegs(name) + ) + }} + + val mapping = basicPrivMapping ++ + perfCntMapping ++ + pmpMapping ++ + pmaMapping ++ + (if (HasFPU) fcsrMapping else Nil) ++ + (if (HasVPU) vcsrMapping else Nil) ++ + (if (HasCustomCSRCacheOp) cacheopMapping else Nil) ++ + (if (HasHExtension) hcsrMapping else Nil) + + + println("XiangShan CSR Lists") + + for (addr <- mapping.keys.toSeq.sorted) { + println(f"$addr%#03x ${mapping(addr)._1}") + } + + val vs_s_csr_map = List( + Sstatus.U -> Vsstatus.U, + Sie.U -> Vsie.U, + Stvec.U -> Vstvec.U, + Sscratch.U -> Vsscratch.U, + Sepc.U -> Vsepc.U, + Scause.U -> Vscause.U, + Stval.U -> Vstval.U, + Sip.U -> Vsip.U, + Satp.U -> Vsatp.U + ) + val addr = Wire(UInt(12.W)) + val vscsr_addr = LookupTreeDefault(src2(11, 0), src2(11, 0), vs_s_csr_map) + when(virtMode){ + addr := vscsr_addr + }.otherwise{ + addr := src2(11, 0) + } + val csri = ZeroExt(src2(16, 12), XLEN) + val rdata = Wire(UInt(XLEN.W)) + val rdata_tmp = Wire(UInt(XLEN.W)) + val wdata_tmp = LookupTree(func, List( + CSROpType.wrt -> src1, + CSROpType.set -> (rdata | src1), + CSROpType.clr -> (rdata & (~src1).asUInt), + CSROpType.wrti -> csri, + CSROpType.seti -> (rdata | csri), + CSROpType.clri -> (rdata & (~csri).asUInt) + )) + val is_vsip_ie = addr === Vsip.U || addr === Vsie.U + // for the difftest with NEMU(stay consistent with Spike) + val is_satp = addr === Satp.U + val is_vsatp = addr === Vsatp.U + val is_hgatp = addr === Hgatp.U + val check_apt_mode = wdata_tmp(wdata_tmp.getWidth-1, 64-Satp_Mode_len) === 8.U || wdata_tmp(wdata_tmp.getWidth-1, 64-Satp_Mode_len) === 0.U + val wdata = MuxCase(wdata_tmp, Seq( + is_vsip_ie -> ZeroExt(wdata_tmp << 1, XLEN), + (is_satp && !check_apt_mode) -> satp, + (is_vsatp && !check_apt_mode) -> vsatp, + (is_hgatp && !check_apt_mode) -> hgatp + )) + val addrInPerfCnt = (addr >= Mcycle.U) && (addr <= Mhpmcounter31.U) || + (addr >= Mcountinhibit.U) && (addr <= Mhpmevent31.U) || + (addr >= Cycle.U) && (addr <= Hpmcounter31.U) || + addr === Mip.U + csrio.isPerfCnt := addrInPerfCnt && valid && func =/= CSROpType.jmp + + // satp wen check + val satpLegalMode = (wdata.asTypeOf(new SatpStruct).mode===0.U) || (wdata.asTypeOf(new SatpStruct).mode===8.U) + + // csr access check, special case + val tvmNotPermit = (privilegeMode === ModeS && !virtMode && mstatusStruct.tvm.asBool) + val accessPermitted = !(addr === Satp.U && tvmNotPermit) + val vtvmNotPermit = (privilegeMode === ModeS && virtMode && hstatusStruct.vtvm.asBool) + val vaccessPermitted = !(addr === Vsatp.U && vtvmNotPermit) +// csrio.disableSfence := (tvmNotPermit || !virtMode && privilegeMode < ModeS) || (vtvmNotPermit || virtMode && privilegeMode < ModeS) +// csrio.disableHfenceg := !((!virtMode && privilegeMode === ModeS && !mstatusStruct.tvm.asBool) || (privilegeMode === ModeM)) // only valid in HS and mstatus.tvm == 0 or in M +// csrio.disableHfencev := !(privilegeMode === ModeM || (!virtMode && privilegeMode === ModeS)) + + // general CSR wen check + val wen = valid && CSROpType.isCsrAccess(func) && ((addr=/=Satp.U && addr =/= Vsatp.U) || satpLegalMode) + val dcsrPermitted = dcsrPermissionCheck(addr, false.B, debugMode) + val triggerPermitted = triggerPermissionCheck(addr, true.B, debugMode) // todo dmode + val HasH = (HasHExtension == true).asBool + val csrAccess = csrAccessPermissionCheck(addr, false.B, privilegeMode, virtMode, HasH) + val modePermitted = csrAccess === 0.U && dcsrPermitted && triggerPermitted + val perfcntPermitted = perfcntPermissionCheck(addr, privilegeMode, mcounteren, scounteren) + val permitted = Mux(addrInPerfCnt, perfcntPermitted, modePermitted) && Mux(virtMode, vaccessPermitted, accessPermitted) + MaskedRegMap.generate(mapping, addr, rdata_tmp, wen && permitted, wdata) + rdata := Mux(is_vsip_ie, ZeroExt(rdata_tmp >> 1, XLEN), rdata_tmp) + io.out.bits.res.data := rdata + io.out.bits.ctrl.flushPipe.get := flushPipe + connect0LatencyCtrlSingal + + // send distribute csr a w signal + csrio.customCtrl.distribute_csr.w.valid := wen && permitted + csrio.customCtrl.distribute_csr.w.bits.data := wdata + csrio.customCtrl.distribute_csr.w.bits.addr := addr + + when (RegNext(csrio.fpu.fflags.valid)) { + fcsr := fflags_wfn(update = true)(RegEnable(csrio.fpu.fflags.bits, csrio.fpu.fflags.valid)) + } + when(RegNext(csrio.vpu.set_vxsat.valid)) { + vcsr := vxsat_wfn(update = true)(RegEnable(csrio.vpu.set_vxsat.bits, csrio.vpu.set_vxsat.valid)) + } + + // set fs and sd in mstatus + when (csrw_dirty_fp_state || RegNext(csrio.fpu.dirty_fs)) { + val mstatusNew = WireInit(mstatus.asTypeOf(new MstatusStruct)) + mstatusNew.fs := "b11".U + mstatusNew.sd := true.B + mstatus := mstatusNew.asUInt + when(virtMode){ + val vsstatusNew = WireInit(vsstatus.asTypeOf(new MstatusStruct)) + vsstatusNew.fs := "b11".U + vsstatusNew.sd := true.B + vsstatus := vsstatusNew.asUInt + } + } + csrio.fpu.frm := fcsr.asTypeOf(new FcsrStruct).frm + + when (RegNext(csrio.vpu.set_vstart.valid)) { + vstart := RegEnable(csrio.vpu.set_vstart.bits, csrio.vpu.set_vstart.valid) + } + when (RegNext(csrio.vpu.set_vtype.valid)) { + vtype := RegEnable(csrio.vpu.set_vtype.bits, csrio.vpu.set_vtype.valid) + } + vl := csrio.vpu.vl + // set vs and sd in mstatus + when(csrw_dirty_vs_state || RegNext(csrio.vpu.dirty_vs)) { + val mstatusNew = WireInit(mstatus.asTypeOf(new MstatusStruct)) + mstatusNew.vs := ContextStatus.dirty + mstatusNew.sd := true.B + mstatus := mstatusNew.asUInt + } + + csrio.vpu.vstart := vstart + csrio.vpu.vxrm := vcsr.asTypeOf(new VcsrStruct).vxrm + + // Trigger Ctrl + val triggerEnableVec = tdata1RegVec.map { tdata1 => + val mcontrolData = tdata1.asTypeOf(new Tdata1Bundle).data.asTypeOf(new MControlData) + tdata1.asTypeOf(new Tdata1Bundle).type_.asUInt === TrigTypeEnum.MCONTROL && ( + mcontrolData.m && privilegeMode === ModeM || + mcontrolData.s && privilegeMode === ModeS || + mcontrolData.u && privilegeMode === ModeU) + } + val fetchTriggerEnableVec = triggerEnableVec.zip(tdata1WireVec).map { + case (tEnable, tdata1) => tEnable && tdata1.asTypeOf(new Tdata1Bundle).data.asTypeOf(new MControlData).isFetchTrigger + } + val memAccTriggerEnableVec = triggerEnableVec.zip(tdata1WireVec).map { + case (tEnable, tdata1) => tEnable && tdata1.asTypeOf(new Tdata1Bundle).data.asTypeOf(new MControlData).isMemAccTrigger + } + csrio.customCtrl.frontend_trigger.tEnableVec := fetchTriggerEnableVec + csrio.customCtrl.mem_trigger.tEnableVec := memAccTriggerEnableVec + + val tdata1Update = wen && (addr === Tdata1.U) + val tdata2Update = wen && (addr === Tdata2.U) + val triggerUpdate = wen && (addr === Tdata1.U || addr === Tdata2.U) + val frontendTriggerUpdate = + tdata1Update && wdata.asTypeOf(new Tdata1Bundle).type_.asUInt === TrigTypeEnum.MCONTROL && + wdata.asTypeOf(new Tdata1Bundle).data.asTypeOf(new MControlData).isFetchTrigger || + tdata1Selected.data.asTypeOf(new MControlData).isFetchTrigger && triggerUpdate + val memTriggerUpdate = + tdata1Update && wdata.asTypeOf(new Tdata1Bundle).type_.asUInt === TrigTypeEnum.MCONTROL && + wdata.asTypeOf(new Tdata1Bundle).data.asTypeOf(new MControlData).isMemAccTrigger || + tdata1Selected.data.asTypeOf(new MControlData).isMemAccTrigger && triggerUpdate + + csrio.customCtrl.frontend_trigger.tUpdate.valid := RegNext(RegNext(frontendTriggerUpdate)) + csrio.customCtrl.mem_trigger.tUpdate.valid := RegNext(RegNext(memTriggerUpdate)) + XSDebug(triggerEnableVec.reduce(_ || _), p"Debug Mode: At least 1 trigger is enabled," + + p"trigger enable is ${Binary(triggerEnableVec.asUInt)}\n") + + // CSR inst decode + val isEbreak = addr === privEbreak && func === CSROpType.jmp + val isEcall = addr === privEcall && func === CSROpType.jmp + val isMret = addr === privMret && func === CSROpType.jmp + val isSret = addr === privSret && func === CSROpType.jmp + val isUret = addr === privUret && func === CSROpType.jmp + val isDret = addr === privDret && func === CSROpType.jmp + val isWFI = func === CSROpType.wfi + + // Illegal privileged operation list + val illegalMret = valid && isMret && privilegeMode < ModeM + val illegalSret = valid && isSret && privilegeMode < ModeS + val illegalSModeSret = valid && isSret && privilegeMode === ModeS && virtMode === false.B && mstatusStruct.tsr.asBool + // when hstatus.vtsr == 1, if sret is executed in VS-mode, it will cause virtual instruction + val illegalVSModeSret = valid && isSret && privilegeMode === ModeS && virtMode && hstatusStruct.vtsr.asBool + // When TW=1, then if WFI is executed in any less-privileged mode, + // and it does not complete within an implementation-specific, bounded time limit, + // the WFI instruction causes an illegal instruction exception. + // The time limit may always be 0, in which case WFI always causes + // an illegal instruction exception in less-privileged modes when TW=1. + val illegalWFI = valid && isWFI && (privilegeMode < ModeM && mstatusStruct.tw === 1.U || privilegeMode === ModeU && !virtMode) + val illegalVWFI = valid && isWFI && ((virtMode && privilegeMode === ModeS && hstatusStruct.vtw === 1.U && mstatusStruct.tw === 0.U)|| + (virtMode && privilegeMode === ModeU && mstatusStruct.tw === 0.U)) + // Illegal privileged instruction check + val isIllegalAddr = valid && CSROpType.isCsrAccess(func) && MaskedRegMap.isIllegalAddr(mapping, addr) + val isIllegalAccess = !virtMode && wen && !(Mux(addrInPerfCnt, perfcntPermitted, csrAccess === 0.U && dcsrPermitted && triggerPermitted) && accessPermitted) + val isIllegalPrivOp = illegalMret || illegalSret || illegalSModeSret || illegalWFI + + val isIllegalVAccess = virtMode && wen && (csrAccess === 2.U || !vaccessPermitted) + val isIllegalVPrivOp = illegalVSModeSret || illegalVWFI + // expose several csr bits for tlb + tlbBundle.priv.mxr := mstatusStruct.mxr.asBool + tlbBundle.priv.sum := mstatusStruct.sum.asBool + tlbBundle.priv.vmxr := vsstatusStruct.mxr.asBool + tlbBundle.priv.vsum := vsstatusStruct.sum.asBool + tlbBundle.priv.spvp := hstatusStruct.spvp + tlbBundle.priv.virt := Mux(mstatusStruct.mprv.asBool, mstatusStruct.mpv & (mstatusStruct.mpp =/= ModeM), virtMode) + tlbBundle.priv.imode := privilegeMode + tlbBundle.priv.dmode := Mux((debugMode && dcsr.asTypeOf(new DcsrStruct).mprven || !debugMode) && mstatusStruct.mprv.asBool, mstatusStruct.mpp, privilegeMode) + + // Branch control + val retTarget = WireInit(0.U) + val resetSatp = (addr === Satp.U || addr === Hgatp.U || addr === Vsatp.U) && wen // write to satp will cause the pipeline be flushed + val writeVstart = addr === Vstart.U && wen // write to vstart will cause the pipeline be flushed + dontTouch(writeVstart) + + val w_fcsr_change_rm = wen && addr === Fcsr.U && wdata(7, 5) =/= fcsr(7, 5) + val w_frm_change_rm = wen && addr === Frm.U && wdata(2, 0) =/= fcsr(7, 5) + val frm_change = w_fcsr_change_rm || w_frm_change_rm + val isXRet = valid && func === CSROpType.jmp && !isEcall && !isEbreak + flushPipe := resetSatp || frm_change || isXRet || frontendTriggerUpdate || writeVstart + + private val illegalRetTarget = WireInit(false.B) + when(valid) { + when(isDret) { + retTarget := dpc(VAddrBits - 1, 0) + }.elsewhen(isMret && !illegalMret) { + retTarget := mepc(VAddrBits - 1, 0) + }.elsewhen(isSret && !illegalSret && !illegalSModeSret && !illegalVSModeSret) { + retTarget := Mux(virtMode, vsepc(VAddrBits - 1, 0), sepc(VAddrBits - 1, 0)) + }.elsewhen(isUret) { + retTarget := uepc(VAddrBits - 1, 0) + }.otherwise { + illegalRetTarget := true.B + } + }.otherwise { + illegalRetTarget := true.B // when illegalRetTarget setted, retTarget should never be used + } + + // Mux tree for regs + when(valid) { + when(isDret) { + val mstatusNew = WireInit(mstatus.asTypeOf(new MstatusStruct)) + val debugModeNew = WireInit(debugMode) + when(dcsr.asTypeOf(new DcsrStruct).prv =/= ModeM) { + mstatusNew.mprv := 0.U + } //If the new privilege mode is less privileged than M-mode, MPRV in mstatus is cleared. + mstatus := mstatusNew.asUInt + privilegeMode := dcsr.asTypeOf(new DcsrStruct).prv + debugModeNew := false.B + debugIntrEnable := true.B + debugMode := debugModeNew + XSDebug("Debug Mode: Dret executed, returning to %x.", retTarget) + }.elsewhen(isMret && !illegalMret) { + val mstatusOld = WireInit(mstatus.asTypeOf(new MstatusStruct)) + val mstatusNew = WireInit(mstatus.asTypeOf(new MstatusStruct)) + mstatusNew.ie.m := mstatusOld.pie.m + privilegeMode := mstatusOld.mpp + if (HasHExtension) { + virtMode := mstatusOld.mpv + mstatusNew.mpv := 0.U + } + mstatusNew.pie.m := true.B + mstatusNew.mpp := ModeU + when(mstatusOld.mpp =/= ModeM) { + mstatusNew.mprv := 0.U + } + mstatus := mstatusNew.asUInt + }.elsewhen(isSret && !illegalSret && !illegalSModeSret && !illegalVSModeSret) { + val mstatusOld = WireInit(mstatus.asTypeOf(new MstatusStruct)) + val mstatusNew = WireInit(mstatus.asTypeOf(new MstatusStruct)) + val hstatusOld = WireInit(hstatus.asTypeOf(new HstatusStruct)) + val hstatusNew = WireInit(hstatus.asTypeOf(new HstatusStruct)) + val vsstatusOld = WireInit(vsstatus.asTypeOf(new MstatusStruct)) + val vsstatusNew = WireInit(vsstatus.asTypeOf(new MstatusStruct)) + when(virtMode === 0.U) { + virtMode := hstatusOld.spv + hstatusNew.spv := 0.U + mstatusNew.ie.s := mstatusOld.pie.s + privilegeMode := Cat(0.U(1.W), mstatusOld.spp) + mstatusNew.pie.s := true.B + mstatusNew.spp := ModeU + when(mstatusOld.spp =/= ModeM) { + mstatusNew.mprv := 0.U + } + mstatus := mstatusNew.asUInt + hstatus := hstatusNew.asUInt + }.otherwise { + privilegeMode := vsstatusOld.spp + vsstatusNew.spp := ModeU + vsstatusNew.ie.s := vsstatusOld.pie.s + vsstatusNew.pie.s := 1.U + vsstatus := vsstatusNew.asUInt + } + }.elsewhen(isUret) { + val mstatusOld = WireInit(mstatus.asTypeOf(new MstatusStruct)) + val mstatusNew = WireInit(mstatus.asTypeOf(new MstatusStruct)) + // mstatusNew.mpp.m := ModeU //TODO: add mode U + mstatusNew.ie.u := mstatusOld.pie.u + privilegeMode := ModeU + mstatusNew.pie.u := true.B + mstatus := mstatusNew.asUInt + } + } + + io.in.ready := true.B + io.out.valid := valid + + // In this situation, hart will enter debug mode instead of handling a breakpoint exception simply. + // Ebreak block instructions backwards, so it's ok to not keep extra info to distinguish between breakpoint + // exception and enter-debug-mode exception. + val ebreakEnterDebugMode = + (privilegeMode === ModeM && dcsrData.ebreakm) || + (privilegeMode === ModeS && dcsrData.ebreaks) || + (privilegeMode === ModeU && dcsrData.ebreaku) + + // raise a debug exception waiting to enter debug mode, instead of a breakpoint exception + val raiseDebugException = !debugMode && isEbreak && ebreakEnterDebugMode + + val csrExceptionVec = WireInit(0.U.asTypeOf(ExceptionVec())) + csrExceptionVec(breakPoint) := io.in.valid && isEbreak + csrExceptionVec(ecallM) := privilegeMode === ModeM && io.in.valid && isEcall + csrExceptionVec(ecallVS) := privilegeMode === ModeS && virtMode && io.in.valid && isEcall + csrExceptionVec(ecallS) := privilegeMode === ModeS && !virtMode && io.in.valid && isEcall + csrExceptionVec(ecallU) := privilegeMode === ModeU && io.in.valid && isEcall + // Trigger an illegal instr exception when: + // * unimplemented csr is being read/written + // * csr access is illegal + csrExceptionVec(illegalInstr) := isIllegalAddr || isIllegalAccess || isIllegalPrivOp + csrExceptionVec(virtualInstr) := isIllegalVAccess || isIllegalVPrivOp + io.out.bits.ctrl.exceptionVec.get := csrExceptionVec + + XSDebug(io.in.valid, s"Debug Mode: an Ebreak is executed, ebreak cause enter-debug-mode exception ? ${raiseDebugException}\n") + + /** + * Exception and Intr + */ + val idelegS = (mideleg & mip.asUInt) + val idelegVS = (hideleg & mideleg & mip.asUInt) + def privilegedEnableDetect(idelegS: Bool, idelegVS: Bool): Bool = Mux(idelegS, + Mux(idelegVS, (virtMode && privilegeMode === ModeS && vsstatusStruct.ie.s) || (virtMode && privilegeMode < ModeS), + ((privilegeMode === ModeS) && mstatusStruct.ie.s) || (privilegeMode < ModeS) || virtMode), + ((privilegeMode === ModeM) && mstatusStruct.ie.m) || (privilegeMode < ModeM)) + + val debugIntr = csrio.externalInterrupt.debug & debugIntrEnable + XSDebug(debugIntr, "Debug Mode: debug interrupt is asserted and valid!") + // send interrupt information to ROB + val intrVecEnable = Wire(Vec(13, Bool())) + val disableInterrupt = debugMode || (dcsrData.step && !dcsrData.stepie) + intrVecEnable.zip(idelegS.asBools).zip(idelegVS.asBools).map{case((x,y),z) => x := privilegedEnableDetect(y, z) && !disableInterrupt} + val intrVec = Cat(debugIntr && !debugMode, (mie(11,0) & mip.asUInt & intrVecEnable.asUInt)) + val intrBitSet = intrVec.orR + csrio.interrupt := intrBitSet + // Page 45 in RISC-V Privileged Specification + // The WFI instruction can also be executed when interrupts are disabled. The operation of WFI + // must be unaffected by the global interrupt bits in mstatus (MIE and SIE) and the delegation + // register mideleg, but should honor the individual interrupt enables (e.g, MTIE). + csrio.wfi_event := debugIntr || (mie(11, 0) & mip.asUInt).orR + mipWire.t.m := csrio.externalInterrupt.mtip + mipWire.s.m := csrio.externalInterrupt.msip + mipWire.e.m := csrio.externalInterrupt.meip + mipWire.e.s := csrio.externalInterrupt.seip + + // interrupts + val intrNO = IntPriority.foldRight(0.U)((i: Int, sum: UInt) => Mux(intrVec(i), i.U, sum)) + val hasIntr = csrio.exception.valid && csrio.exception.bits.isInterrupt + val ivmEnable = tlbBundle.priv.imode < ModeM && satp.asTypeOf(new SatpStruct).mode === 8.U + val iexceptionPC = Mux(ivmEnable, SignExt(csrio.exception.bits.pc, XLEN), csrio.exception.bits.pc) + val iexceptionGPAddr = Mux(ivmEnable, SignExt(csrio.exception.bits.gpaddr, XLEN), csrio.exception.bits.gpaddr) + val dvmEnable = tlbBundle.priv.dmode < ModeM && satp.asTypeOf(new SatpStruct).mode === 8.U + val dexceptionPC = Mux(dvmEnable, SignExt(csrio.exception.bits.pc, XLEN), csrio.exception.bits.pc) + XSDebug(hasIntr, "interrupt: pc=0x%x, %d\n", dexceptionPC, intrNO) + val hasDebugIntr = intrNO === IRQ_DEBUG.U && hasIntr + + // exceptions from rob need to handle + val exceptionVecFromRob = csrio.exception.bits.exceptionVec + val hasException = csrio.exception.valid && !csrio.exception.bits.isInterrupt + val hasInstrPageFault = hasException && exceptionVecFromRob(instrPageFault) + val hasLoadPageFault = hasException && exceptionVecFromRob(loadPageFault) + val hasStorePageFault = hasException && exceptionVecFromRob(storePageFault) + val hasStoreAddrMisalign = hasException && exceptionVecFromRob(storeAddrMisaligned) + val hasLoadAddrMisalign = hasException && exceptionVecFromRob(loadAddrMisaligned) + val hasInstrAccessFault = hasException && exceptionVecFromRob(instrAccessFault) + val hasLoadAccessFault = hasException && exceptionVecFromRob(loadAccessFault) + val hasStoreAccessFault = hasException && exceptionVecFromRob(storeAccessFault) + val hasBreakPoint = hasException && exceptionVecFromRob(breakPoint) + val hasInstGuestPageFault = hasException && exceptionVecFromRob(instrGuestPageFault) + val hasLoadGuestPageFault = hasException && exceptionVecFromRob(loadGuestPageFault) + val hasStoreGuestPageFault = hasException && exceptionVecFromRob(storeGuestPageFault) + val hasSingleStep = hasException && csrio.exception.bits.singleStep + val hasTriggerFire = hasException && csrio.exception.bits.trigger.canFire + val triggerFrontendHitVec = csrio.exception.bits.trigger.frontendHit + val triggerMemHitVec = csrio.exception.bits.trigger.backendHit + val triggerHitVec = triggerFrontendHitVec | triggerMemHitVec // Todo: update mcontrol.hit + val triggerCanFireVec = csrio.exception.bits.trigger.frontendCanFire | csrio.exception.bits.trigger.backendCanFire + // More than one triggers can hit at the same time, but only fire one + // We select the first hit trigger to fire + val triggerFireOH = PriorityEncoderOH(triggerCanFireVec) + val triggerFireAction = PriorityMux(triggerFireOH, tdata1WireVec.map(_.getTriggerAction)).asUInt + + + XSDebug(hasSingleStep, "Debug Mode: single step exception\n") + XSDebug(hasTriggerFire, p"Debug Mode: trigger fire, frontend hit vec ${Binary(csrio.exception.bits.trigger.frontendHit.asUInt)} " + + p"backend hit vec ${Binary(csrio.exception.bits.trigger.backendHit.asUInt)}\n") + + val hasExceptionVec = csrio.exception.bits.exceptionVec + val regularExceptionNO = ExceptionNO.priorities.foldRight(0.U)((i: Int, sum: UInt) => Mux(hasExceptionVec(i), i.U, sum)) + val exceptionNO = Mux(hasSingleStep || hasTriggerFire, 3.U, regularExceptionNO) + val causeNO = (hasIntr << (XLEN - 1)).asUInt | Mux(hasIntr, intrNO, exceptionNO) + + val hasExceptionIntr = csrio.exception.valid + + val hasDebugEbreakException = hasBreakPoint && ebreakEnterDebugMode + val hasDebugTriggerException = hasTriggerFire && triggerFireAction === TrigActionEnum.DEBUG_MODE + val hasDebugException = hasDebugEbreakException || hasDebugTriggerException || hasSingleStep + val hasDebugTrap = hasDebugException || hasDebugIntr + val ebreakEnterParkLoop = debugMode && hasExceptionIntr + + XSDebug(hasExceptionIntr, "int/exc: pc %x int (%d):%x exc: (%d):%x\n", + dexceptionPC, intrNO, intrVec, exceptionNO, hasExceptionVec.asUInt + ) + XSDebug(hasExceptionIntr, + "pc %x mstatus %x mideleg %x medeleg %x mode %x\n", + dexceptionPC, + mstatus, + mideleg, + medeleg, + privilegeMode + ) + + // mtval write logic + // Due to timing reasons of memExceptionVAddr, we delay the write of mtval and stval + val memExceptionAddr = SignExt(csrio.memExceptionVAddr, XLEN) + val memExceptionGPAddr = SignExt(csrio.memExceptionGPAddr, XLEN) + val updateTval = VecInit(Seq( + hasInstrPageFault, + hasLoadPageFault, + hasStorePageFault, + hasInstrAccessFault, + hasLoadAccessFault, + hasStoreAccessFault, + hasLoadAddrMisalign, + hasStoreAddrMisalign, + hasInstGuestPageFault, + hasLoadGuestPageFault, + hasStoreGuestPageFault, + hasBreakPoint, + )).asUInt.orR + val updateTval_h = VecInit(Seq( + hasInstGuestPageFault, + hasLoadGuestPageFault, + hasStoreGuestPageFault + )).asUInt.orR + when (RegNext(RegNext(updateTval))) { + val tval = Mux( + RegNext(RegNext(hasInstrPageFault || hasInstrAccessFault || hasInstGuestPageFault || hasBreakPoint)), + RegNext(RegNext(Mux( + csrio.exception.bits.crossPageIPFFix, + SignExt(csrio.exception.bits.pc + 2.U, XLEN), + iexceptionPC + ))), + memExceptionAddr + ) + // because we update tval two beats later, we can choose xtval according to the privilegeMode which has been updated + when (RegNext(privilegeMode === ModeM)) { + mtval := tval + }.otherwise { + when (virtMode){ + vstval := tval + }.otherwise{ + stval := tval + } + } + } + + when(RegNext(RegNext(updateTval_h))) { + val tval_tmp = Mux( + RegNext(RegNext(hasInstGuestPageFault)), + RegNext(RegNext(Mux( + csrio.exception.bits.crossPageIPFFix, + SignExt(csrio.exception.bits.gpaddr + 2.U, XLEN), + iexceptionGPAddr + ))), + memExceptionGPAddr + ) + val tval = tval_tmp >> 2 + when(RegNext(privilegeMode === ModeM)) { + mtval2 := tval + }.otherwise { + htval := tval + } + } + + val debugTrapTarget = Mux(!isEbreak && debugMode, 0x38020808.U, 0x38020800.U) // 0x808 is when an exception occurs in debug mode prog buf exec + val deleg = Mux(hasIntr, mideleg , medeleg) + val hdeleg = Mux(hasIntr, hideleg, hedeleg) + // val delegS = ((deleg & (1 << (causeNO & 0xf))) != 0) && (privilegeMode < ModeM); + val delegS = deleg(causeNO(7,0)) && (privilegeMode < ModeM) + val delegVS = virtMode && delegS && hdeleg(causeNO(7, 0)) && (privilegeMode < ModeM) + val clearTval = !updateTval || hasIntr + + val clearTval_h = !updateTval_h || hasIntr + val isHyperInst = csrio.exception.bits.isHls + // ctrl block will use theses later for flush + val isXRetFlag = RegInit(false.B) + when (DelayN(io.flush.valid, 5)) { + isXRetFlag := false.B + }.elsewhen (isXRet) { + isXRetFlag := true.B + } + csrio.isXRet := isXRetFlag + private val retTargetReg = RegEnable(retTarget, isXRet && !illegalRetTarget) + private val illegalXret = RegEnable(illegalMret || illegalSret || illegalSModeSret || illegalVSModeSret, isXRet) + + private val xtvec = Mux(delegS, Mux(delegVS, vstvec, stvec), mtvec) + private val xtvecBase = xtvec(VAddrBits - 1, 2) + // When MODE=Vectored, all synchronous exceptions into M/S mode + // cause the pc to be set to the address in the BASE field, whereas + // interrupts cause the pc to be set to the address in the BASE field + // plus four times the interrupt cause number. + private val pcFromXtvec = Cat(xtvecBase + Mux(xtvec(0) && hasIntr, causeNO(3, 0), 0.U), 0.U(2.W)) + + // XRet sends redirect instead of Flush and isXRetFlag is true.B before redirect.valid. + // ROB sends exception at T0 while CSR receives at T2. + // We add a RegNext here and trapTarget is valid at T3. + csrio.trapTarget := RegEnable( + MuxCase(pcFromXtvec, Seq( + (isXRetFlag && !illegalXret) -> retTargetReg, + ((hasDebugTrap && !debugMode) || ebreakEnterParkLoop) -> debugTrapTarget + )), + isXRetFlag || csrio.exception.valid) + + when(hasExceptionIntr) { + val mstatusOld = WireInit(mstatus.asTypeOf(new MstatusStruct)) + val mstatusNew = WireInit(mstatus.asTypeOf(new MstatusStruct)) + val hstatusOld = WireInit(hstatus.asTypeOf(new HstatusStruct)) + val hstatusNew = WireInit(hstatus.asTypeOf(new HstatusStruct)) + val vsstatusOld = WireInit(vsstatus.asTypeOf(new MstatusStruct)) + val vsstatusNew = WireInit(vsstatus.asTypeOf(new MstatusStruct)) + val dcsrNew = WireInit(dcsr.asTypeOf(new DcsrStruct)) + val debugModeNew = WireInit(debugMode) + when(hasDebugTrap && !debugMode) { + import DcsrStruct._ + debugModeNew := true.B + dcsrNew.prv := privilegeMode + privilegeMode := ModeM + when(hasDebugIntr) { + dpc := iexceptionPC + dcsrNew.cause := CAUSE_HALTREQ + XSDebug(hasDebugIntr, "Debug Mode: Trap to %x at pc %x\n", debugTrapTarget, dpc) + }.otherwise { // hasDebugException + dpc := iexceptionPC // TODO: check it when hasSingleStep + dcsrNew.cause := MuxCase(0.U, Seq( + hasTriggerFire -> CAUSE_TRIGGER, + raiseDebugException -> CAUSE_EBREAK, + hasBreakPoint -> CAUSE_HALTREQ, + hasSingleStep -> CAUSE_STEP + )) + } + dcsr := dcsrNew.asUInt + debugIntrEnable := false.B + }.elsewhen (debugMode) { + //do nothing + }.elsewhen (delegVS) { + vscause := (hasIntr << (XLEN-1)).asUInt | Mux(hasIntr, intrNO - 1.U, exceptionNO) + vsepc := Mux(hasInstrPageFault || hasInstrAccessFault, iexceptionPC, dexceptionPC) + vsstatusNew.spp := privilegeMode + vsstatusNew.pie.s := vsstatusOld.ie.s + vsstatusNew.ie.s := false.B + when (clearTval) {vstval := 0.U} + virtMode := true.B + privilegeMode := ModeS + }.elsewhen (delegS) { + val virt = Mux(mstatusOld.mprv.asBool, mstatusOld.mpv, virtMode) + // to do hld st + hstatusNew.gva := (hasInstGuestPageFault || hasLoadGuestPageFault || hasStoreGuestPageFault || + ((virt.asBool || isHyperInst) && ((hasException && 0.U <= exceptionNO && exceptionNO <= 7.U && exceptionNO =/= 2.U) + || hasInstrPageFault || hasLoadPageFault || hasStorePageFault))) + hstatusNew.spv := virtMode + when(virtMode){ + hstatusNew.spvp := privilegeMode + } + virtMode := false.B + scause := causeNO + sepc := Mux(hasInstrPageFault || hasInstrAccessFault, iexceptionPC, dexceptionPC) + mstatusNew.spp := privilegeMode + mstatusNew.pie.s := mstatusOld.ie.s + mstatusNew.ie.s := false.B + privilegeMode := ModeS + when (clearTval) { stval := 0.U } + when (clearTval_h) {htval := 0.U} + }.otherwise { + val virt = Mux(mstatusOld.mprv.asBool, mstatusOld.mpv, virtMode) + // to do hld st + mstatusNew.gva := (hasInstGuestPageFault || hasLoadGuestPageFault || hasStoreGuestPageFault || + ((virt.asBool || isHyperInst) && ((hasException && 0.U <= exceptionNO && exceptionNO <= 7.U && exceptionNO =/= 2.U) + || hasInstrPageFault || hasLoadPageFault || hasStorePageFault))) + mstatusNew.mpv := virtMode + virtMode := false.B + mcause := causeNO + mepc := Mux(hasInstrPageFault || hasInstrAccessFault, iexceptionPC, dexceptionPC) + mstatusNew.mpp := privilegeMode + mstatusNew.pie.m := mstatusOld.ie.m + mstatusNew.ie.m := false.B + privilegeMode := ModeM + when (clearTval) { mtval := 0.U } + when (clearTval_h) {mtval2 := 0.U} + } + mstatus := mstatusNew.asUInt + vsstatus := vsstatusNew.asUInt + hstatus := hstatusNew.asUInt + debugMode := debugModeNew + } + + // Cache error debug support + if(HasCustomCSRCacheOp){ + val cache_error_decoder = Module(new CSRCacheErrorDecoder) + cache_error_decoder.io.encoded_cache_error := cacheopRegs("CACHE_ERROR") + } + + // Implicit add reset values for mepc[0] and sepc[0] + // TODO: rewrite mepc and sepc using a struct-like style with the LSB always being 0 + when (RegNext(RegNext(reset.asBool) && !reset.asBool)) { + mepc := Cat(mepc(XLEN - 1, 1), 0.U(1.W)) + sepc := Cat(sepc(XLEN - 1, 1), 0.U(1.W)) + vsepc := Cat(vsepc(XLEN - 1, 1), 0.U(1.W)) + } + + def readWithScala(addr: Int): UInt = mapping(addr)._1 + + val difftestIntrNO = Mux(hasIntr, causeNO, 0.U) + + // Always instantiate basic difftest modules. + if (env.AlwaysBasicDiff || env.EnableDifftest) { + val difftest = DifftestModule(new DiffArchEvent, delay = 3, dontCare = true) + difftest.coreid := csrio.hartId + difftest.valid := csrio.exception.valid + difftest.interrupt := Mux(hasIntr, causeNO, 0.U) + difftest.exception := Mux(hasException, causeNO, 0.U) + difftest.exceptionPC := dexceptionPC + if (env.EnableDifftest || env.FullBasicDiff) { + difftest.exceptionInst := csrio.exception.bits.instr + } + } + + // Always instantiate basic difftest modules. + if (env.AlwaysBasicDiff || env.EnableDifftest) { + val difftest = DifftestModule(new DiffCSRState) + difftest.coreid := csrio.hartId + difftest.privilegeMode := privilegeMode + difftest.mstatus := mstatus + difftest.sstatus := mstatus & sstatusRmask + difftest.mepc := mepc + difftest.sepc := sepc + difftest.mtval:= mtval + difftest.stval:= stval + difftest.mtvec := mtvec + difftest.stvec := stvec + difftest.mcause := mcause + difftest.scause := scause + difftest.satp := satp + difftest.mip := mipReg + difftest.mie := mie + difftest.mscratch := mscratch + difftest.sscratch := sscratch + difftest.mideleg := mideleg + difftest.medeleg := medeleg + } + + if (env.AlwaysBasicDiff || env.EnableDifftest) { + val difftest = DifftestModule(new DiffHCSRState) + difftest.coreid := csrio.hartId + difftest.virtMode := virtMode + difftest.mtval2 := mtval2 + difftest.mtinst := mtinst + difftest.hstatus := hstatus + difftest.hideleg := hideleg + difftest.hedeleg := hedeleg + difftest.hcounteren := hcounteren + difftest.htval := htval + difftest.htinst := htinst + difftest.hgatp := hgatp + difftest.vsstatus := vsstatus + difftest.vstvec := vstvec + difftest.vsepc := vsepc + difftest.vscause := vscause + difftest.vstval := vstval + difftest.vsatp := vsatp + difftest.vsscratch := vsscratch + } + + if(env.AlwaysBasicDiff || env.EnableDifftest) { + val difftest = DifftestModule(new DiffDebugMode) + difftest.coreid := csrio.hartId + difftest.debugMode := debugMode + difftest.dcsr := dcsr + difftest.dpc := dpc + difftest.dscratch0 := dscratch0 + difftest.dscratch1 := dscratch1 + } + + if (env.AlwaysBasicDiff || env.EnableDifftest) { + val difftest = DifftestModule(new DiffVecCSRState) + difftest.coreid := csrio.hartId + difftest.vstart := vstart + difftest.vxsat := vcsr.asTypeOf(new VcsrStruct).vxsat + difftest.vxrm := vcsr.asTypeOf(new VcsrStruct).vxrm + difftest.vcsr := vcsr + difftest.vl := vl + difftest.vtype := vtype + difftest.vlenb := vlenb + } +} +class PFEvent(implicit p: Parameters) extends XSModule { + val io = IO(new Bundle { + val distribute_csr = Flipped(new DistributedCSRIO()) + val hpmevent = Output(Vec(29, UInt(XLEN.W))) + }) + + val w = io.distribute_csr.w + + val perfEvents = List.fill(8)(RegInit("h0000000000".U(XLEN.W))) ++ + List.fill(8)(RegInit("h4010040100".U(XLEN.W))) ++ + List.fill(8)(RegInit("h8020080200".U(XLEN.W))) ++ + List.fill(5)(RegInit("hc0300c0300".U(XLEN.W))) + + val perfEventMapping = (0 until 29).map(i => {Map( + MaskedRegMap(addr = CSRs.mhpmevent3 + i, + reg = perfEvents(i), + wmask = "hf87fff3fcff3fcff".U(XLEN.W)) + )}).fold(Map())((a,b) => a ++ b) + + val rdata = Wire(UInt(XLEN.W)) + MaskedRegMap.generate(perfEventMapping, w.bits.addr, rdata, w.valid, w.bits.data) + for(i <- 0 until 29){ + io.hpmevent(i) := perfEvents(i) + } +} + */ diff --git a/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/FuConfig.scala b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/FuConfig.scala new file mode 100644 index 000000000..7ba5fc090 --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/FuConfig.scala @@ -0,0 +1,894 @@ +// Replaces XiangShan src/main/scala/xiangshan/backend/fu/FuConfig.scala (same package) with Buckyball changes. + +package xiangshan.backend.fu + +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import utils.EnumUtils.OHEnumeration +import xiangshan.ExceptionNO._ +import xiangshan.SelImm +import xiangshan.backend.fu.fpu.{IntFPToVec, IntToFP} +import xiangshan.backend.fu.wrapper._ +import xiangshan.backend.Bundles.ExuInput +import xiangshan.backend.datapath.DataConfig._ +import xiangshan.mem.Std + +/** + * @param name [[String]] name of fuConfig + * @param fuType [[Int]] type of func, select from [[xiangshan.backend.fu.FuType]] + * @param fuGen how to create $fu + * @param srcData type of src data used by this $fu + * @param piped if the $fu is pipelined + * @param maybeBlock the $fu need ready signal to block internal pipeline + * @param writeIntRf the $fu write int regfiles + * @param writeFpRf the $fu write float regfiles + * @param writeVecRf the $fu write vector regfiles + * @param writeV0Rf the $fu write v0 regfiles + * @param writeVlRf the $fu write vl regfiles + * @param writeFflags the $fu write fflags csr + * @param writeVxsat the $fu write vxsat csr + * @param destDataBits the width of output data in the $fu + * @param srcDataBits the width of input data in the $fu, the default value is destDataBits + * @param latency the latency of instuction executed in the $fu + * @param hasInputBuffer if the $fu has input buffer + * @param exceptionOut the $fu can produce these exception + * @param hasLoadError if the $fu has load error out + * @param flushPipe if the instuction executed in the $fu need flush out + * @param replayInst if the instuction executed in the $fu can replay in some condition + * @param trigger if the $fu need trigger out + * @param needSrcFrm if the $fu need float rounding mode signal + * @param needSrcVxrm if the $fu need vector fixed-point rounding mode signal + * @param immType the immediate type of this $fu + * @param vconfigWakeUp + * @param maskWakeUp + * @param writeVType if the $fu need write vtype + * @param readOldVtype if the $fu need read old vtype + * + * @define fu function unit + */ +case class FuConfig( + name: String, + fuType: FuType.OHType, + fuGen: (Parameters, FuConfig) => FuncUnit, + srcData: Seq[Seq[DataConfig]], + piped: Boolean, + maybeBlock: Boolean = false, + writeIntRf: Boolean = false, + writeFpRf: Boolean = false, + writeVecRf: Boolean = false, + writeV0Rf: Boolean = false, + writeVlRf: Boolean = false, + writeFakeIntRf: Boolean = false, + writeFflags: Boolean = false, + writeVxsat: Boolean = false, + destDataBits: Int = 64, + srcDataBits: Option[Int] = None, + srcNeedCopy: Boolean = false, + latency: HasFuLatency = CertainLatency(0), // two field (base latency, extra latency(option)) + hasInputBuffer: (Boolean, Int, Boolean) = (false, 0, false), + exceptionOut: Seq[Int] = Seq(), + hasLoadError: Boolean = false, + flushPipe: Boolean = false, + replayInst: Boolean = false, + trigger: Boolean = false, + needSrcFrm: Boolean = false, + needSrcVxrm: Boolean = false, + writeVType: Boolean = false, + immType: Set[UInt] = Set(), + // vector + vconfigWakeUp: Boolean = false, + maskWakeUp: Boolean = false, + readOldVtype: Boolean = false) { + def needIntWen: Boolean = writeIntRf || writeFakeIntRf + def needFpWen: Boolean = writeFpRf + def needVecWen: Boolean = writeVecRf + def needV0Wen: Boolean = writeV0Rf + def needVlWen: Boolean = writeVlRf + var vconfigIdx = -1 + var maskSrcIdx = -1 + if (vconfigWakeUp) { + vconfigIdx = getSpecialSrcIdx(VlData(), "when vconfigWakeUp is true, srcData must always contains VlData()") + } + if (maskWakeUp) { + maskSrcIdx = getSpecialSrcIdx(V0Data(), "when maskWakeUp is true, srcData must always contains V0Data()") + } + + require(!piped || piped && latency.latencyVal.isDefined, "The latency value must be set when piped is enable") + require(!vconfigWakeUp || vconfigWakeUp && vconfigIdx >= 0, "The index of vl src must be set when vlWakeUp is enable") + require(!maskWakeUp || maskWakeUp && maskSrcIdx >= 0, "The index of mask src must be set when vlWakeUp is enable") + + def numIntSrc: Int = srcData.map(_.count(x => IntRegSrcDataSet.contains(x))).fold(0)(_ max _) + def numFpSrc: Int = srcData.map(_.count(x => FpRegSrcDataSet.contains(x))).fold(0)(_ max _) + def numVecSrc: Int = srcData.map(_.count(x => VecRegSrcDataSet.contains(x))).fold(0)(_ max _) + def numVfSrc: Int = srcData.map(_.count(x => VecRegSrcDataSet.contains(x))).fold(0)(_ max _) + def numV0Src: Int = srcData.map(_.count(x => V0RegSrcDataSet.contains(x))).fold(0)(_ max _) + def numVlSrc: Int = srcData.map(_.count(x => VlRegSrcDataSet.contains(x))).fold(0)(_ max _) + def numRegSrc: Int = srcData.map(_.count(x => RegSrcDataSet.contains(x))).fold(0)(_ max _) + def numSrc: Int = srcData.map(_.length).fold(0)(_ max _) + + def readFp: Boolean = numFpSrc > 0 + + def fuSel(uop: ExuInput): Bool = + // Don't add more shit here!!! + // Todo: add new FuType to distinguish f2i, f2f + uop.fuType === this.fuType.U + + /** + * params(i): data type set of the ith src port + * @return + */ + def getRfReadDataCfgSet: Seq[Set[DataConfig]] = { + val numSrcMax = srcData.map(_.length).fold(0)(_ max _) + // make srcData is uniform sized to avoid exception when transpose + val alignedSrcData: Seq[Seq[DataConfig]] = srcData.map(x => x ++ Seq.fill(numSrcMax - x.length)(null)) + alignedSrcData.transpose.map(_.toSet.intersect(RegSrcDataSet)) + } + + def getSrcDataType(srcIdx: Int): Set[DataConfig] = { + srcData + .map((x: Seq[DataConfig]) => if (x.isDefinedAt(srcIdx)) Some(x(srcIdx)) else None) + .filter(_.nonEmpty) + .map(_.get) + .toSet + } + + def hasNoDataWB: Boolean = + !(writeIntRf || writeFpRf || writeVecRf || writeV0Rf || writeVlRf) + + def getSrcMaxWidthVec = + getRfReadDataCfgSet.map(_.map(_.dataWidth).max) + + def genSrcDataVec: Seq[UInt] = + getSrcMaxWidthVec.map(w => UInt(w.W)) + + // csr's redirect also uses redirect bundle + def hasRedirect: Boolean = Seq(FuType.jmp, FuType.brh, FuType.csr).contains(fuType) + + def hasPredecode: Boolean = Seq(FuType.jmp, FuType.brh, FuType.csr, FuType.ldu).contains(fuType) + + def needTargetPc: Boolean = Seq(FuType.jmp, FuType.brh).contains(fuType) + + // predict info + def needPdInfo: Boolean = Seq(FuType.jmp, FuType.brh, FuType.csr).contains(fuType) + + def needPc: Boolean = Seq(FuType.jmp, FuType.brh, FuType.ldu, FuType.csr).contains(fuType) + + def needFPUCtrl: Boolean = { + import FuType._ + Seq(fmac, fDivSqrt, i2f).contains(fuType) + } + + def needVecCtrl: Boolean = { + import FuType._ + Seq(vipu, vialuF, vimac, vidiv, vfpu, vppu, vfalu, vfma, vfdiv, vfcvt, vldu, vstu, vsetfwf).contains(fuType) + } + + def needCriticalErrors: Boolean = Seq(FuType.csr).contains(fuType) + + def isMul: Boolean = fuType == FuType.mul + + def isDiv: Boolean = fuType == FuType.div + + def isCsr: Boolean = fuType == FuType.csr + + def isBrh: Boolean = fuType == FuType.brh + + def isJmp: Boolean = fuType == FuType.jmp + + def isFence: Boolean = fuType == FuType.fence + + def isVecArith: Boolean = fuType == FuType.vialuF || fuType == FuType.vimac || + fuType == FuType.vppu || fuType == FuType.vipu || + fuType == FuType.vfalu || fuType == FuType.vfma || + fuType == FuType.vfdiv || fuType == FuType.vfcvt || + fuType == FuType.vidiv + + def isVecMem: Boolean = fuType == FuType.vldu || fuType == FuType.vstu || + fuType == FuType.vsegldu || fuType == FuType.vsegstu + + def needOg2: Boolean = isVecArith || fuType == FuType.vsetfwf || isVecMem + + def isSta: Boolean = name.contains("sta") + + def isStd: Boolean = name.contains("std") + + def ckAlwaysEn: Boolean = isCsr || isFence + + /** + * Get index of special src data, like [[VlData]], [[V0Data]] + * + * @param data [[DataConfig]] + * @param tips tips if get failed + * @return the index of special src data + */ + protected def getSpecialSrcIdx(data: DataConfig, tips: String): Int = { + val srcIdxVec = srcData.map(x => x.indexOf(data)) + val idx0 = srcIdxVec.head + for (idx <- srcIdxVec) { + require(idx >= 0 && idx == idx0, tips + ", and at the same index.") + } + idx0 + } + + override def toString: String = { + var str = s"${this.name}: " + if (vconfigWakeUp) str += s"vconfigIdx($vconfigIdx), " + if (maskWakeUp) str += s"maskSrcIdx($maskSrcIdx), " + str += s"latency($latency)" + str += s"src($srcData)" + str + } + +} + +object FuConfig { + + val JmpCfg: FuConfig = FuConfig( + name = "jmp", + fuType = FuType.jmp, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new JumpUnit(cfg)(p)).suggestName("jmp"), + srcData = Seq( + Seq(IntData()) // jal + ), + piped = true, + writeIntRf = true, + immType = Set(SelImm.IMM_I, SelImm.IMM_UJ, SelImm.IMM_U) + ) + + val BrhCfg: FuConfig = FuConfig( + name = "brh", + fuType = FuType.brh, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new BranchUnit(cfg)(p).suggestName("brh")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = true, + immType = Set(SelImm.IMM_SB) + ) + + val I2fCfg: FuConfig = FuConfig( + name = "i2f", + FuType.i2f, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new IntToFP(cfg)(p).suggestName("i2f")), + srcData = Seq( + Seq(IntData()) + ), + piped = true, + writeFpRf = true, + writeFflags = true, + latency = CertainLatency(2), + needSrcFrm = true + ) + + val I2vCfg: FuConfig = FuConfig( + name = "i2v", + FuType.i2v, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new IntFPToVec(cfg)(p).suggestName("i2v")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = true, + writeFpRf = true, + writeVecRf = true, + writeV0Rf = true, + latency = CertainLatency(0), + destDataBits = 128, + srcDataBits = Some(64), + immType = Set(SelImm.IMM_OPIVIU, SelImm.IMM_OPIVIS, SelImm.IMM_VRORVI) + ) + + val F2vCfg: FuConfig = FuConfig( + name = "f2v", + FuType.f2v, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new IntFPToVec(cfg)(p).suggestName("f2v")), + srcData = Seq( + Seq(FpData(), FpData()), + Seq(FpData()) + ), + piped = true, + writeFpRf = true, + writeVecRf = true, + writeV0Rf = true, + latency = CertainLatency(0), + destDataBits = 128, + srcDataBits = Some(64) + ) + + val CsrCfg: FuConfig = FuConfig( + name = "csr", + fuType = FuType.csr, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new examples.core.xiangshan.native.CSR(cfg)(p).suggestName("csr")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = false, + writeIntRf = true, + latency = UncertainLatency(), + exceptionOut = Seq(illegalInstr, virtualInstr, breakPoint, ecallU, ecallS, ecallVS, ecallM), + flushPipe = true + ) + + val AluCfg: FuConfig = FuConfig( + name = "alu", + fuType = FuType.alu, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new Alu(cfg)(p).suggestName("Alu")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = true, + writeIntRf = true, + immType = Set(SelImm.IMM_I, SelImm.IMM_U, SelImm.IMM_LUI32) + ) + + val MulCfg: FuConfig = FuConfig( + name = "mul", + fuType = FuType.mul, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new MulUnit(cfg)(p).suggestName("Mul")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = true, + writeIntRf = true, + latency = CertainLatency(2) + ) + + val DivCfg: FuConfig = FuConfig( + name = "div", + fuType = FuType.div, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new DivUnit(cfg)(p).suggestName("Div")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = false, + writeIntRf = true, + latency = UncertainLatency(), + hasInputBuffer = (true, 4, true) + ) + + val FenceCfg: FuConfig = FuConfig( + name = "fence", + FuType.fence, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new Fence(cfg)(p).suggestName("Fence")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = false, + latency = UncertainLatency(), + exceptionOut = Seq(illegalInstr, virtualInstr), + flushPipe = true + ) + + // Todo: split it to simple bitmap exu and complex bku + val BkuCfg: FuConfig = FuConfig( + name = "bku", + fuType = FuType.bku, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new Bku(cfg)(p).suggestName("Bku")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = true, + writeIntRf = true, + latency = CertainLatency(2) + ) + + val VSetRvfWvfCfg: FuConfig = FuConfig( + name = "vsetrvfwvf", + fuType = FuType.vsetfwf, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VSetRvfWvf(cfg)(p).suggestName("VSetRvfWvf")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = true, + writeVlRf = true, + writeVType = true, + writeIntRf = true, + latency = CertainLatency(0), + immType = Set(SelImm.IMM_VSETVLI, SelImm.IMM_VSETIVLI), + readOldVtype = true + ) + + val VSetRiWvfCfg: FuConfig = FuConfig( + name = "vsetriwvf", + fuType = FuType.vsetiwf, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VSetRiWvf(cfg)(p).suggestName("VSetRiWvf")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = true, + writeVlRf = true, + writeVType = true, + latency = CertainLatency(0), + immType = Set(SelImm.IMM_VSETVLI, SelImm.IMM_VSETIVLI) + ) + + val VSetRiWiCfg: FuConfig = FuConfig( + name = "vsetriwi", + fuType = FuType.vsetiwi, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VSetRiWi(cfg)(p).suggestName("VSetRiWi")), + srcData = Seq( + Seq(IntData(), IntData()) + ), + piped = true, + writeIntRf = true, + latency = CertainLatency(0), + immType = Set(SelImm.IMM_VSETVLI, SelImm.IMM_VSETIVLI) + ) + + val LduCfg: FuConfig = FuConfig( + name = "ldu", + fuType = FuType.ldu, + fuGen = null, // Todo + srcData = Seq( + Seq(IntData()) + ), + piped = false, // Todo: check it + writeIntRf = true, + writeFpRf = true, + latency = UncertainLatency(3), + exceptionOut = + Seq(loadAddrMisaligned, loadAccessFault, loadPageFault, loadGuestPageFault, breakPoint, hardwareError), + flushPipe = true, + replayInst = true, + hasLoadError = true, + trigger = true, + immType = Set(SelImm.IMM_I) + ) + + val StaCfg: FuConfig = FuConfig( + name = "sta", + fuType = FuType.stu, + fuGen = null, // Todo + srcData = Seq( + Seq(IntData()) + ), + piped = false, + latency = UncertainLatency(), + exceptionOut = + Seq(storeAddrMisaligned, storeAccessFault, storePageFault, storeGuestPageFault, breakPoint, hardwareError), + flushPipe = true, + trigger = true, + immType = Set(SelImm.IMM_S) + ) + + val StdCfg: FuConfig = FuConfig( + name = "std", + fuType = FuType.stu, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new Std(cfg)(p).suggestName("Std")), + srcData = Seq( + Seq(IntData()), + Seq(FpData()) + ), + piped = true, + latency = CertainLatency(0) + ) + + val HyldaCfg = FuConfig( + name = "hylda", + fuType = FuType.ldu, + fuGen = null, // Todo + srcData = Seq( + Seq(IntData()) + ), + piped = false, // Todo: check it + writeIntRf = true, + writeFpRf = true, + latency = UncertainLatency(3), + exceptionOut = + Seq(loadAddrMisaligned, loadAccessFault, loadPageFault, loadGuestPageFault, breakPoint, hardwareError), + flushPipe = true, + replayInst = true, + hasLoadError = true, + immType = Set(SelImm.IMM_I) + ) + + val HystaCfg = FuConfig( + name = "hysta", + fuType = FuType.stu, + fuGen = null, // Todo + srcData = Seq( + Seq(IntData()) + ), + piped = false, + latency = UncertainLatency(), + exceptionOut = + Seq(storeAddrMisaligned, storeAccessFault, storePageFault, storeGuestPageFault, breakPoint, hardwareError), + immType = Set(SelImm.IMM_S) + ) + + val FakeHystaCfg = FuConfig( + name = "hysta", + fuType = FuType.stu, + fuGen = null, // Todo + srcData = Seq(), + piped = false, + latency = UncertainLatency(), + exceptionOut = Seq(storeAddrMisaligned, storeAccessFault, storePageFault, storeGuestPageFault), + immType = Set() + ) + + val MouCfg: FuConfig = FuConfig( + name = "mou", + fuType = FuType.mou, + fuGen = null, // Todo + srcData = Seq( + Seq(IntData()) + ), + piped = false, // Todo: check it + writeFakeIntRf = true, + latency = UncertainLatency(), + exceptionOut = (LduCfg.exceptionOut ++ StaCfg.exceptionOut ++ StdCfg.exceptionOut).distinct, + trigger = true + ) + + val MoudCfg: FuConfig = FuConfig( + name = "moud", + fuType = FuType.mou, + fuGen = null, // Todo + srcData = Seq( + Seq(IntData()) + ), + piped = true, + latency = CertainLatency(0) + ) + + val VialuCfg = FuConfig( + name = "vialuFix", + fuType = FuType.vialuF, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VIAluFix(cfg)(p).suggestName("VialuFix")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = true, + writeVecRf = true, + writeV0Rf = true, + writeVxsat = true, + needSrcVxrm = true, + latency = CertainLatency(1), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr) + ) + + val VimacCfg = FuConfig( + name = "vimac", + fuType = FuType.vimac, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VIMacU(cfg)(p).suggestName("Vimac")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = true, + writeVecRf = true, + writeV0Rf = true, + writeVxsat = true, + needSrcVxrm = true, + latency = CertainLatency(2), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr) + ) + + val VidivCfg = FuConfig( + name = "vidiv", + fuType = FuType.vidiv, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VIDiv(cfg)(p).suggestName("Vidiv")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = false, + writeVecRf = true, + writeV0Rf = true, + latency = UncertainLatency(), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr) + ) + + val VppuCfg = FuConfig( + name = "vppu", + fuType = FuType.vppu, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VPPU(cfg)(p).suggestName("Vppu")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = true, + writeVecRf = true, + writeV0Rf = true, + latency = CertainLatency(2), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr) + ) + + val VipuCfg: FuConfig = FuConfig( + name = "vipu", + fuType = FuType.vipu, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VIPU(cfg)(p).suggestName("Vipu")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0 + ), + piped = true, + writeIntRf = true, + writeVecRf = true, + writeV0Rf = true, + latency = CertainLatency(2), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr) + ) + + val VfaluCfg = FuConfig( + name = "vfalu", + fuType = FuType.vfalu, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VFAlu(cfg)(p).suggestName("Vfalu")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = true, + writeVecRf = true, + writeV0Rf = true, + writeFpRf = true, + writeFflags = true, + latency = CertainLatency(1), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr), + needSrcFrm = true + ) + + val VfmaCfg = FuConfig( + name = "vfma", + fuType = FuType.vfma, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VFMA(cfg)(p).suggestName("Vfma")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = true, + writeVecRf = true, + writeV0Rf = true, + writeFflags = true, + latency = CertainLatency(3), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr), + needSrcFrm = true + ) + + val VfdivCfg = FuConfig( + name = "vfdiv", + fuType = FuType.vfdiv, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VFDivSqrt(cfg)(p).suggestName("Vfdiv")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = false, + writeVecRf = true, + writeV0Rf = true, + writeFflags = true, + latency = UncertainLatency(), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr), + needSrcFrm = true + ) + + val VfcvtCfg = FuConfig( + name = "vfcvt", + fuType = FuType.vfcvt, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new VCVT(cfg)(p).suggestName("Vfcvt")), + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) // vs1, vs2, vd_old, v0, vtype&vl + ), + piped = true, + writeVecRf = true, + writeV0Rf = true, + writeFflags = true, + latency = CertainLatency(2), + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128, + exceptionOut = Seq(illegalInstr), + needSrcFrm = true + ) + + val FaluCfg = FuConfig( + name = "falu", + fuType = FuType.falu, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new FAlu(cfg)(p).suggestName("Falu")), + srcData = Seq( + Seq(FpData(), FpData()) + ), + piped = true, + writeFpRf = true, + writeIntRf = true, + writeFflags = true, + latency = CertainLatency(1), + destDataBits = 64, + needSrcFrm = true + ) + + val FmacCfg = FuConfig( + name = "fmac", + fuType = FuType.fmac, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new FMA(cfg)(p).suggestName("Fmac")), + srcData = Seq( + Seq(FpData(), FpData(), FpData()) + ), + piped = true, + writeFpRf = true, + writeFflags = true, + latency = CertainLatency(3), + destDataBits = 64, + needSrcFrm = true + ) + + val FdivCfg = FuConfig( + name = "fdiv", + fuType = FuType.fDivSqrt, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new FDivSqrt(cfg)(p).suggestName("Fdiv")), + srcData = Seq( + Seq(FpData(), FpData()) + ), + piped = false, + writeFpRf = true, + writeFflags = true, + latency = UncertainLatency(), + destDataBits = 64, + needSrcFrm = true + ) + + val FcvtCfg = FuConfig( + name = "fcvt", + fuType = FuType.fcvt, + fuGen = (p: Parameters, cfg: FuConfig) => Module(new FCVT(cfg)(p).suggestName("Fcvt")), + srcData = Seq( + Seq(FpData()) + ), + piped = true, + writeFpRf = true, + writeIntRf = true, + writeFflags = true, + latency = CertainLatency(2), + destDataBits = 64, + needSrcFrm = true + ) + + val VlduCfg: FuConfig = FuConfig( + name = "vldu", + fuType = FuType.vldu, + fuGen = null, + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) //vs1, vs2, vd_old, v0, vconfig + ), + piped = false, // Todo: check it + writeVecRf = true, + writeV0Rf = true, + writeVlRf = true, + latency = UncertainLatency(), + exceptionOut = + Seq(loadAddrMisaligned, loadAccessFault, loadPageFault, loadGuestPageFault, breakPoint, hardwareError), + flushPipe = true, + replayInst = true, + trigger = true, + hasLoadError = true, + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128 + ) + + val VstuCfg: FuConfig = FuConfig( + name = "vstu", + fuType = FuType.vstu, + fuGen = null, + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) //vs1, vs2, vd_old, v0, vconfig + ), + piped = false, + latency = UncertainLatency(), + exceptionOut = + Seq(storeAddrMisaligned, storeAccessFault, storePageFault, storeGuestPageFault, breakPoint, hardwareError), + flushPipe = true, + replayInst = true, + trigger = true, + hasLoadError = true, + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128 + ) + + val VseglduSeg: FuConfig = FuConfig( + name = "vsegldu", + fuType = FuType.vsegldu, + fuGen = null, + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) //vs1, vs2, vd_old, v0, vconfig + ), + piped = false, // Todo: check it + writeVecRf = true, + writeV0Rf = true, + writeVlRf = true, + latency = UncertainLatency(), + exceptionOut = Seq(loadAddrMisaligned, loadAccessFault, loadPageFault, breakPoint, hardwareError), + flushPipe = true, + replayInst = true, + trigger = true, + hasLoadError = true, + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128 + ) + + val VsegstuCfg: FuConfig = FuConfig( + name = "vsegstu", + fuType = FuType.vsegstu, + fuGen = null, + srcData = Seq( + Seq(VecData(), VecData(), VecData(), V0Data(), VlData()) //vs1, vs2, vd_old, v0, vconfig + ), + piped = false, + latency = UncertainLatency(), + exceptionOut = Seq(storeAddrMisaligned, storeAccessFault, storePageFault, breakPoint, hardwareError), + flushPipe = true, + replayInst = true, + trigger = true, + hasLoadError = true, + vconfigWakeUp = true, + maskWakeUp = true, + destDataBits = 128 + ) + + def allConfigs = Seq( + JmpCfg, + BrhCfg, + I2fCfg, + I2vCfg, + F2vCfg, + CsrCfg, + AluCfg, + MulCfg, + DivCfg, + FenceCfg, + BkuCfg, + VSetRvfWvfCfg, + VSetRiWvfCfg, + VSetRiWiCfg, + LduCfg, + StaCfg, + StdCfg, + MouCfg, + MoudCfg, + VialuCfg, + VipuCfg, + VlduCfg, + VstuCfg, + VseglduSeg, + VsegstuCfg, + FaluCfg, + FmacCfg, + FcvtCfg, + FdivCfg, + VfaluCfg, + VfmaCfg, + VfcvtCfg, + HyldaCfg, + HystaCfg + ) + + def VecArithFuConfigs = Seq( + VialuCfg, + VimacCfg, + VppuCfg, + VipuCfg, + VfaluCfg, + VfmaCfg, + VfcvtCfg + ) + +} diff --git a/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/FuncUnit.scala b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/FuncUnit.scala new file mode 100644 index 000000000..6c05d0a4a --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/FuncUnit.scala @@ -0,0 +1,315 @@ +// Replaces XiangShan src/main/scala/xiangshan/backend/fu/FuncUnit.scala (same package) with Buckyball changes. + +package xiangshan.backend.fu + +import org.chipsalliance.cde.config.Parameters +import chisel3._ +import chisel3.util._ +import utility._ +import utils.OptionWrapper +import xiangshan._ +import xiangshan.backend.Bundles.VPUCtrlSignals +import xiangshan.backend.rob.RobPtr +import xiangshan.frontend.{FtqPtr, PreDecodeInfo} +import xiangshan.backend.datapath.DataConfig._ +import xiangshan.backend.fu.vector.Bundles.Vxsat +import xiangshan.ExceptionNO.illegalInstr +import xiangshan.backend.fu.vector.Bundles.VType +import xiangshan.backend.fu.wrapper.{CSRInput, CSRToDecode} + +class FuncUnitCtrlInput(cfg: FuConfig)(implicit p: Parameters) extends XSBundle { + val fuOpType = FuOpType() + val robIdx = new RobPtr + val pdest = UInt(PhyRegIdxWidth.W) + val rfWen = OptionWrapper(cfg.needIntWen, Bool()) + val fpWen = OptionWrapper(cfg.needFpWen, Bool()) + val vecWen = OptionWrapper(cfg.needVecWen, Bool()) + val v0Wen = OptionWrapper(cfg.needV0Wen, Bool()) + val vlWen = OptionWrapper(cfg.needVlWen, Bool()) + val flushPipe = OptionWrapper(cfg.flushPipe, Bool()) + val preDecode = OptionWrapper(cfg.hasPredecode, new PreDecodeInfo) + val ftqIdx = OptionWrapper(cfg.needPc || cfg.replayInst || cfg.isSta || cfg.isCsr, new FtqPtr) + val ftqOffset = OptionWrapper(cfg.needPc || cfg.replayInst || cfg.isSta || cfg.isCsr, UInt(log2Up(PredictWidth).W)) + + val predictInfo = OptionWrapper( + cfg.needPdInfo, + new Bundle { + val target = UInt(VAddrData().dataWidth.W) + val taken = Bool() + } + ) + + val fpu = OptionWrapper(cfg.writeFflags, new FPUCtrlSignals) + val vpu = OptionWrapper(cfg.needVecCtrl, new VPUCtrlSignals) +} + +class FuncUnitCtrlOutput(cfg: FuConfig)(implicit p: Parameters) extends XSBundle { + val robIdx = new RobPtr + val pdest = UInt(PhyRegIdxWidth.W) // Todo: use maximum of pregIdxWidth of different pregs + val rfWen = OptionWrapper(cfg.needIntWen, Bool()) + val fpWen = OptionWrapper(cfg.needFpWen, Bool()) + val vecWen = OptionWrapper(cfg.needVecWen, Bool()) + val v0Wen = OptionWrapper(cfg.needV0Wen, Bool()) + val vlWen = OptionWrapper(cfg.needVlWen, Bool()) + val exceptionVec = OptionWrapper(cfg.exceptionOut.nonEmpty, ExceptionVec()) + val flushPipe = OptionWrapper(cfg.flushPipe, Bool()) + val satpFlushPipe = OptionWrapper(cfg.isCsr, Bool()) + val replay = OptionWrapper(cfg.replayInst, Bool()) + val preDecode = OptionWrapper(cfg.hasPredecode, new PreDecodeInfo) + val fpu = OptionWrapper(cfg.writeFflags, new FPUCtrlSignals) + val vpu = OptionWrapper(cfg.needVecCtrl, new VPUCtrlSignals) +} + +class FuncUnitDataInput(cfg: FuConfig)(implicit p: Parameters) extends XSBundle { + val src = MixedVec(cfg.genSrcDataVec) + val imm = UInt(cfg.destDataBits.W) + val pc = OptionWrapper(cfg.needPc, UInt(VAddrData().dataWidth.W)) + val nextPcOffset = OptionWrapper(cfg.needPc && !cfg.isCsr, UInt((log2Up(PredictWidth) + 1).W)) + + def getSrcVConfig: UInt = src(cfg.vconfigIdx) + def getSrcMask: UInt = src(cfg.maskSrcIdx) +} + +class FuncUnitDataOutput(cfg: FuConfig)(implicit p: Parameters) extends XSBundle { + val data = UInt(cfg.destDataBits.W) + val fflags = OptionWrapper(cfg.writeFflags, UInt(5.W)) + val vxsat = OptionWrapper(cfg.writeVxsat, Vxsat()) + val redirect = OptionWrapper(cfg.hasRedirect, ValidIO(new Redirect)) +} + +class FuncUnitInput(cfg: FuConfig)(implicit p: Parameters) extends XSBundle { + val needCtrlPipe = cfg.latency.latencyVal.nonEmpty && (!cfg.isStd) + val ctrl = new FuncUnitCtrlInput(cfg) + val ctrlPipe = OptionWrapper(needCtrlPipe, Vec(cfg.latency.latencyVal.get + 1, new FuncUnitCtrlInput(cfg))) + val validPipe = OptionWrapper(needCtrlPipe, Vec(cfg.latency.latencyVal.get + 1, Bool())) + val data = new FuncUnitDataInput(cfg) + val dataPipe = OptionWrapper(needCtrlPipe, Vec(cfg.latency.latencyVal.get + 1, new FuncUnitDataInput(cfg))) + val perfDebugInfo = new PerfDebugInfo() + val debug_seqNum = InstSeqNum() +} + +class FuncUnitOutput(cfg: FuConfig)(implicit p: Parameters) extends XSBundle { + val ctrl = new FuncUnitCtrlOutput(cfg) + val res = new FuncUnitDataOutput(cfg) + val perfDebugInfo = new PerfDebugInfo() + val debug_seqNum = InstSeqNum() +} + +class FuncUnitIO(cfg: FuConfig)(implicit p: Parameters) extends XSBundle { + val flush = Flipped(ValidIO(new Redirect)) + val in = Flipped(DecoupledIO(new FuncUnitInput(cfg))) + val out = DecoupledIO(new FuncUnitOutput(cfg)) + val csrin = OptionWrapper(cfg.isCsr, new CSRInput) + val csrio = OptionWrapper(cfg.isCsr, new CSRFileIO) + val csrToDecode = OptionWrapper(cfg.isCsr, Output(new CSRToDecode)) + val fenceio = OptionWrapper(cfg.isFence, new FenceIO) + val frm = OptionWrapper(cfg.needSrcFrm, Input(UInt(3.W))) + val vxrm = OptionWrapper(cfg.needSrcVxrm, Input(UInt(2.W))) + val vtype = OptionWrapper(cfg.writeVlRf, (Valid(new VType))) + val vlIsZero = OptionWrapper(cfg.writeVlRf, Output(Bool())) + val vlIsVlmax = OptionWrapper(cfg.writeVlRf, Output(Bool())) + val instrAddrTransType = Option.when(cfg.isJmp || cfg.isBrh)(Input(new AddrTransType)) +} + +abstract class FuncUnit(val cfg: FuConfig)(implicit p: Parameters) extends XSModule with HasCriticalErrors { + val io = IO(new FuncUnitIO(cfg)) + PerfCCT.updateInstPos(io.in.bits.debug_seqNum, PerfCCT.InstPos.AtFU.id.U, io.in.valid, clock, reset) + PerfCCT.updateInstPos(io.out.bits.debug_seqNum, PerfCCT.InstPos.AtBypassVal.id.U, io.out.valid, clock, reset) + val criticalErrors = Seq(("none", false.B)) + + // should only be used in non-piped fu + def connectNonPipedCtrlSingal: Unit = { + io.out.bits.ctrl.robIdx := RegEnable(io.in.bits.ctrl.robIdx, io.in.fire) + io.out.bits.ctrl.pdest := RegEnable(io.in.bits.ctrl.pdest, io.in.fire) + io.out.bits.ctrl.rfWen.foreach(_ := RegEnable(io.in.bits.ctrl.rfWen.get, io.in.fire)) + io.out.bits.ctrl.fpWen.foreach(_ := RegEnable(io.in.bits.ctrl.fpWen.get, io.in.fire)) + io.out.bits.ctrl.vecWen.foreach(_ := RegEnable(io.in.bits.ctrl.vecWen.get, io.in.fire)) + io.out.bits.ctrl.v0Wen.foreach(_ := RegEnable(io.in.bits.ctrl.v0Wen.get, io.in.fire)) + io.out.bits.ctrl.vlWen.foreach(_ := RegEnable(io.in.bits.ctrl.vlWen.get, io.in.fire)) + // io.out.bits.ctrl.flushPipe should be connected in fu + io.out.bits.ctrl.preDecode.foreach(_ := RegEnable(io.in.bits.ctrl.preDecode.get, io.in.fire)) + io.out.bits.ctrl.fpu.foreach(_ := RegEnable(io.in.bits.ctrl.fpu.get, io.in.fire)) + io.out.bits.ctrl.vpu.foreach(_ := RegEnable(io.in.bits.ctrl.vpu.get, io.in.fire)) + io.out.bits.perfDebugInfo := RegEnable(io.in.bits.perfDebugInfo, io.in.fire) + io.out.bits.debug_seqNum := RegEnable(io.in.bits.debug_seqNum, io.in.fire) + } + + def connectNonPipedCtrlSingalForCSR: Unit = { + io.out.bits.ctrl.robIdx := DataHoldBypass(io.in.bits.ctrl.robIdx, io.in.fire) + io.out.bits.ctrl.pdest := DataHoldBypass(io.in.bits.ctrl.pdest, io.in.fire) + io.out.bits.ctrl.rfWen.foreach(_ := DataHoldBypass(io.in.bits.ctrl.rfWen.get, io.in.fire)) + io.out.bits.ctrl.fpWen.foreach(_ := DataHoldBypass(io.in.bits.ctrl.fpWen.get, io.in.fire)) + io.out.bits.ctrl.vecWen.foreach(_ := DataHoldBypass(io.in.bits.ctrl.vecWen.get, io.in.fire)) + io.out.bits.ctrl.v0Wen.foreach(_ := DataHoldBypass(io.in.bits.ctrl.v0Wen.get, io.in.fire)) + io.out.bits.ctrl.vlWen.foreach(_ := DataHoldBypass(io.in.bits.ctrl.vlWen.get, io.in.fire)) + // io.out.bits.ctrl.flushPipe should be connected in fu + io.out.bits.ctrl.preDecode.foreach(_ := DataHoldBypass(io.in.bits.ctrl.preDecode.get, io.in.fire)) + io.out.bits.ctrl.fpu.foreach(_ := DataHoldBypass(io.in.bits.ctrl.fpu.get, io.in.fire)) + io.out.bits.ctrl.vpu.foreach(_ := DataHoldBypass(io.in.bits.ctrl.vpu.get, io.in.fire)) + io.out.bits.perfDebugInfo := DataHoldBypass(io.in.bits.perfDebugInfo, io.in.fire) + io.out.bits.debug_seqNum := DataHoldBypass(io.in.bits.debug_seqNum, io.in.fire) + } + + def connect0LatencyCtrlSingal: Unit = { + io.out.bits.ctrl.robIdx := io.in.bits.ctrl.robIdx + io.out.bits.ctrl.pdest := io.in.bits.ctrl.pdest + io.out.bits.ctrl.rfWen.foreach(_ := io.in.bits.ctrl.rfWen.get) + io.out.bits.ctrl.fpWen.foreach(_ := io.in.bits.ctrl.fpWen.get) + io.out.bits.ctrl.vecWen.foreach(_ := io.in.bits.ctrl.vecWen.get) + io.out.bits.ctrl.v0Wen.foreach(_ := io.in.bits.ctrl.v0Wen.get) + io.out.bits.ctrl.vlWen.foreach(_ := io.in.bits.ctrl.vlWen.get) + // io.out.bits.ctrl.flushPipe should be connected in fu + io.out.bits.ctrl.preDecode.foreach(_ := io.in.bits.ctrl.preDecode.get) + io.out.bits.ctrl.fpu.foreach(_ := io.in.bits.ctrl.fpu.get) + io.out.bits.ctrl.vpu.foreach(_ := io.in.bits.ctrl.vpu.get) + io.out.bits.perfDebugInfo := io.in.bits.perfDebugInfo + io.out.bits.debug_seqNum := io.in.bits.debug_seqNum + } + +} + +/** + * @author LinJiaWei, Yinan Xu + */ +trait HasPipelineReg { this: FuncUnit => + def latency: Int + + val latdiff: Int = cfg.latency.extraLatencyVal.getOrElse(0) + val preLat: Int = latency - latdiff + require(latency >= 0 && latdiff >= 0) + + def pipelineReg( + init: FuncUnitInput, + valid: Bool, + ready: Bool, + latency: Int, + flush: ValidIO[Redirect] + ): (Seq[FuncUnitInput], Seq[Bool], Seq[Bool]) = { + val rdyVec = Seq.fill(latency)(Wire(Bool())) :+ ready + val validVec = valid +: Seq.fill(latency)(RegInit(false.B)) + val ctrlVec = init.ctrl +: Seq.fill(latency)(Reg(chiselTypeOf(io.in.bits.ctrl))) + val dataVec = init.data +: Seq.fill(latency)(Reg(chiselTypeOf(io.in.bits.data))) + val perfVec = init.perfDebugInfo +: Seq.fill(latency)(Reg(chiselTypeOf(io.in.bits.perfDebugInfo))) + val seqNumVec = init.debug_seqNum +: Seq.fill(latency)(Reg(chiselTypeOf(io.in.bits.debug_seqNum))) + + val robIdxVec = ctrlVec.map(_.robIdx) + + // if flush(0), valid 0 will not given, so set flushVec(0) to false.B + val flushVec = validVec.zip(robIdxVec).map(x => x._1 && x._2.needFlush(flush)) + + for (i <- 0 until latency) { + rdyVec(i) := !validVec(i + 1) || rdyVec(i + 1).asTypeOf(Bool()) + } + for (i <- 1 to latency) { + validVec(i) := validVec(i - 1) + when(rdyVec(i - 1) && validVec(i - 1)) { + ctrlVec(i) := ctrlVec(i - 1) + dataVec(i) := dataVec(i - 1) + perfVec(i) := perfVec(i - 1) + seqNumVec(i) := seqNumVec(i - 1) + } + } + + ( + ctrlVec.zip(dataVec).zip(perfVec).zip(seqNumVec).map { + case (((ctrl, data), perf), debug_seqNum) => { + val out = Wire(new FuncUnitInput(cfg)) + out.ctrl := ctrl + out.ctrlPipe.foreach(_ := 0.U.asTypeOf(out.ctrlPipe.get)) + out.validPipe.foreach(_ := 0.U.asTypeOf(out.validPipe.get)) + out.dataPipe.foreach(_ := 0.U.asTypeOf(out.dataPipe.get)) + out.data := data + out.perfDebugInfo := perf + out.debug_seqNum := debug_seqNum + out + } + }, + validVec, + rdyVec + ) + } + + val (pipeReg: Seq[FuncUnitInput], validVecThisFu, rdyVec) = + pipelineReg(io.in.bits, io.in.valid, io.out.ready, preLat, io.flush) + val validVec = io.in.bits.validPipe.get.zip(validVecThisFu).map(x => x._1 && x._2) + val ctrlVec = io.in.bits.ctrlPipe.get + val dataVec = io.in.bits.dataPipe.get + val perfVec = pipeReg.map(_.perfDebugInfo) + val seqNumVec = pipeReg.map(_.debug_seqNum) + + val fixtiminginit = Wire(new FuncUnitInput(cfg)) + fixtiminginit.ctrl := ctrlVec.last + fixtiminginit.ctrlPipe.foreach(_ := 0.U.asTypeOf(fixtiminginit.ctrlPipe.get)) + fixtiminginit.validPipe.foreach(_ := 0.U.asTypeOf(fixtiminginit.validPipe.get)) + fixtiminginit.dataPipe.foreach(_ := 0.U.asTypeOf(fixtiminginit.dataPipe.get)) + fixtiminginit.data := dataVec.last + fixtiminginit.perfDebugInfo := perfVec.last + fixtiminginit.debug_seqNum := seqNumVec.last + + // fixtiming pipelinereg + val (fixpipeReg: Seq[FuncUnitInput], fixValidVec, fixRdyVec) = + pipelineReg(fixtiminginit, validVec.last, rdyVec.head, latdiff, io.flush) + val fixDataVec = fixpipeReg.map(_.data) + val fixPerfVec = fixpipeReg.map(_.perfDebugInfo) + val fixSeqNumVec = fixpipeReg.map(_.debug_seqNum) + val pcVec = fixDataVec.map(_.pc) + + io.in.ready := fixRdyVec.head + io.out.valid := fixValidVec.last + + io.out.bits.ctrl.robIdx := ctrlVec.last.robIdx + io.out.bits.ctrl.pdest := ctrlVec.last.pdest + io.out.bits.ctrl.rfWen.foreach(_ := ctrlVec.last.rfWen.get) + io.out.bits.ctrl.fpWen.foreach(_ := ctrlVec.last.fpWen.get) + io.out.bits.ctrl.vecWen.foreach(_ := ctrlVec.last.vecWen.get) + io.out.bits.ctrl.v0Wen.foreach(_ := ctrlVec.last.v0Wen.get) + io.out.bits.ctrl.vlWen.foreach(_ := ctrlVec.last.vlWen.get) + io.out.bits.ctrl.fpu.foreach(_ := ctrlVec.last.fpu.get) + io.out.bits.ctrl.vpu.foreach(_ := ctrlVec.last.vpu.get) + io.out.bits.perfDebugInfo := fixPerfVec.last + io.out.bits.debug_seqNum := fixSeqNumVec.last + + // vstart illegal + if (cfg.exceptionOut.nonEmpty) { + val outVstart = ctrlVec.last.vpu.get.vstart + val vstartIllegal = outVstart =/= 0.U + io.out.bits.ctrl.exceptionVec.get := 0.U.asTypeOf(io.out.bits.ctrl.exceptionVec.get) + io.out.bits.ctrl.exceptionVec.get(illegalInstr) := vstartIllegal + } + + def regEnable(i: Int): Bool = validVec(i - 1) && rdyVec(i - 1) + + def PipelineReg[TT <: Data](i: Int)(next: TT) = { + val lat = preLat min i + RegEnable( + next, + regEnable(lat) + ) + } + + def SNReg[TT <: Data](in: TT, n: Int): TT = { + val lat = preLat min n + var next = in + for (i <- 1 to lat) { + next = PipelineReg[TT](i)(next) + } + next + } + + def S1Reg[TT <: Data](next: TT): TT = PipelineReg[TT](1)(next) + + def S2Reg[TT <: Data](next: TT): TT = PipelineReg[TT](2)(next) + + def S3Reg[TT <: Data](next: TT): TT = PipelineReg[TT](3)(next) + + def S4Reg[TT <: Data](next: TT): TT = PipelineReg[TT](4)(next) + + def S5Reg[TT <: Data](next: TT): TT = PipelineReg[TT](5)(next) + +} + +abstract class PipedFuncUnit(override val cfg: FuConfig)(implicit p: Parameters) + extends FuncUnit(cfg) + with HasPipelineReg { + override def latency: Int = cfg.latency.latencyVal.get +} diff --git a/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/NewCSR/NewCSR.scala b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/NewCSR/NewCSR.scala new file mode 100644 index 000000000..ce5f2098e --- /dev/null +++ b/examples/prefab/cores/xiangshan/arch/native/xiangshan/backend/fu/NewCSR/NewCSR.scala @@ -0,0 +1,1853 @@ +// Replaces XiangShan src/main/scala/xiangshan/backend/fu/NewCSR/NewCSR.scala (same package) with Buckyball changes. + +package xiangshan.backend.fu.NewCSR + +import chisel3._ +import chisel3.util._ +import difftest._ +import freechips.rocketchip.rocket.CSRs +import org.chipsalliance.cde.config.Parameters +import top.{ArgParser, Generator} +import utility._ +import utils.OptionWrapper +import xiangshan.backend.fu.NewCSR.CSRBundles._ +import xiangshan.backend.fu.NewCSR.CSRDefines._ +import xiangshan.backend.fu.NewCSR.CSREnumTypeImplicitCast._ +import xiangshan.backend.fu.NewCSR.CSRFunc._ +import xiangshan.backend.fu.NewCSR.CSREvents.{ + CSREvents, + DretEventSinkBundle, + EventUpdatePrivStateOutput, + MNretEventSinkBundle, + MretEventSinkBundle, + SretEventSDTSinkBundle, + SretEventSinkBundle, + TargetPCBundle, + TrapEntryDEventSinkBundle, + TrapEntryEventInput, + TrapEntryHSEventSinkBundle, + TrapEntryMEventSinkBundle, + TrapEntryMNEventSinkBundle, + TrapEntryVSEventSinkBundle +} +import xiangshan.backend.fu.fpu.Bundles.Frm +import xiangshan.backend.fu.vector.Bundles.{Vl, Vstart, Vxrm, Vxsat} +import xiangshan.backend.fu.wrapper.CSRToDecode +import xiangshan.backend.rob.RobPtr +import xiangshan._ +import xiangshan.backend.fu.PerfCounterIO +import xiangshan.backend.fu.util.CSRConst +import xiangshan.ExceptionNO._ +import xiangshan.backend.trace._ + +import scala.collection.immutable.SeqMap + +object CSRConfig { + final val ASIDLEN = 16 // the length of ASID of XS implementation + + final val ASIDMAX = 16 // the max value of ASIDLEN defined by spec + + final val HIIDWidth = 12 // support Hvictl[27:16](IID) + + final val VMIDLEN = 14 // the length of VMID of XS implementation + + final val VMIDMAX = 14 // the max value of VMIDLEN defined by spec + + final val VaddrMaxWidth = 48 + 2 // support Sv39/Sv48/Sv39x4/Sv48x4 + + final val InstWidth = 32 + + final val XLEN = 64 // Todo: use XSParams + + final val VLEN = 128 + + // Since we need macro to compute the width of CSR field, the input of macro should be the value that can be computed + // at compile time. The log2Up function cannot be used as meta-programming function, so we use litral value here + // log2Up(128 + 1), hold 0~128 + final val VlWidth = 8 + + final val PAddrWidth = 48 + + final val AddrWidthInPage = 12 + + final val PMPAddrWidth = 48 + + final val PMPOffBits = 2 + + final val PMPAddrBits = PMPAddrWidth - PMPOffBits + + // perf + final val perfCntNum = 29 // in Spec + + final val EXT_SSTC = true + + final val EXT_DBLTRP = true + + final val PPNLength = 44 + +} + +class NewCSRInput(implicit p: Parameters) extends Bundle { + val wen = Bool() + val ren = Bool() + val op = UInt(2.W) + val addr = UInt(12.W) + val src = UInt(64.W) + val wdata = UInt(64.W) + val mnret = Input(Bool()) + val mret = Input(Bool()) + val sret = Input(Bool()) + val dret = Input(Bool()) + val redirectFlush = Input(Bool()) +} + +class NewCSROutput(implicit p: Parameters) extends Bundle { + val EX_II = Bool() + val EX_VI = Bool() + val flushPipe = Bool() + val rData = UInt(64.W) + val targetPcUpdate = Bool() + val targetPc = new TargetPCBundle + val regOut = UInt(64.W) + // perf + val isPerfCnt = Bool() +} + +class NewCSR(implicit val p: Parameters) + extends Module + with HasXSParameter + with MachineLevel + with SupervisorLevel + with HypervisorLevel + with VirtualSupervisorLevel + with Unprivileged + with CSRAIA + with CSRIND + with HasExternalInterruptBundle + with HasNonMaskableIRPBundle + with CSREvents + with DebugLevel + with CSRCustom + with CSRPMP + with CSRPMA + with CSRDocDump + with HasCriticalErrors + with IpIeAliasConnect { + + import CSRConfig._ + + val io = IO(new Bundle { + + val fromTop = Input(new Bundle { + val hartId = UInt(hartIdLen.W) + val clintTime = Input(ValidIO(UInt(64.W))) + val l2FlushDone = Input(Bool()) + val reset_mtvec = Option.when(enableResetMtvec)(Input(UInt(PAddrBits.W))) + val criticalErrorState = Input(Bool()) + }) + + val in = Flipped(DecoupledIO(new NewCSRInput)) + val trapInst = Input(ValidIO(UInt(InstWidth.W))) + + val fromMem = Input(new Bundle { + val excpVA = UInt(XLEN.W) + val excpGPA = UInt(XLEN.W) + val excpIsForVSnonLeafPTE = Bool() + }) + + val fromRob = Input(new Bundle { + + val trap = ValidIO(new Bundle { + val pc = UInt((VaddrMaxWidth + 1).W) + val pcGPA = UInt(PAddrBitsMax.W) + val instr = UInt(InstWidth.W) + val trapVec = UInt(64.W) + val isFetchBkpt = Bool() + val singleStep = Bool() + val trigger = TriggerAction() + val crossPageIPFFix = Bool() + val isInterrupt = Bool() + val isHls = Bool() + val isFetchMalAddr = Bool() + val isForVSnonLeafPTE = Bool() + val satpFlushFirstFetchFault = Bool() + }) + + val commit = Input(new RobCommitCSR) + val robDeqPtr = Input(new RobPtr) + }) + + val fromVecExcpMod = Input(new Bundle { + val busy = Bool() + }) + + val perf = Input(new PerfCounterIO) + + /** Output should be a DecoupledIO, since now CSR writing to integer register file might be blocked (by arbiter) */ + val out = DecoupledIO(new NewCSROutput) + val accelerator = Output(new examples.core.xiangshan.native.Context) + + val status = Output(new Bundle { + val privState = new PrivState + val interrupt = Bool() + val wfiEvent = Bool() + val satp = ValidIO(UInt(SatpMode.getWidth.W)) + val vsatp = ValidIO(UInt(SatpMode.getWidth.W)) + + // fp + val fpState = new Bundle { + val off = Bool() + val frm = Frm() + } + + // vec + val vecState = new Bundle { + val vstart = Vstart() + val vxsat = Vxsat() + val vxrm = Vxrm() + val vcsr = UInt(XLEN.W) + val vl = Vl() + val vtype = UInt(XLEN.W) + val vlenb = UInt(XLEN.W) + val off = Bool() + } + + // debug + val debugMode = Bool() + val singleStepFlag = Bool() + // trigger + val frontendTrigger = new FrontendTdataDistributeIO() + val memTrigger = new MemTdataDistributeIO() + // Instruction fetch address translation type + val instrAddrTransType = new AddrTransType + // trace + val traceCSR = Output(new TraceCSR) + // custom + val custom = new CSRCustomState + val criticalErrorState = Bool() + }) + + // tlb + val tlb = Output(new Bundle { + val satpASIDChanged = Bool() + val vsatpASIDChanged = Bool() + val hgatpVMIDChanged = Bool() + val satp = new SatpBundle + val vsatp = new SatpBundle + val hgatp = new HgatpBundle + val mbmc = new MbmcBundle + val mxr = Bool() + val sum = Bool() + val vmxr = Bool() + val vsum = Bool() + val spvp = Bool() + val imode = UInt(2.W) + val dmode = UInt(2.W) + val debug = Bool() + val dvirt = Bool() + val mPBMTE = Bool() + val hPBMTE = Bool() + + val pmm = new Bundle { + val mseccfg = UInt(2.W) + val menvcfg = UInt(2.W) + val henvcfg = UInt(2.W) + val hstatus = UInt(2.W) + val senvcfg = UInt(2.W) + } + + }) + + val toDecode = new CSRToDecode + + val fetchMalTval = Input(UInt(XLEN.W)) + + val oldPrivSate = Input(new PrivState) + val oldSatpMode = Input(UInt(SatpMode.getWidth.W)) + val oldVsatpMode = Input(UInt(SatpMode.getWidth.W)) + + val distributedWenLegal = Output(Bool()) + }) + + val toAIA = IO(Output(new CSRToAIABundle)) + val fromAIA = IO(Flipped(Output(new AIAToCSRBundle))) + + dontTouch(toAIA) + dontTouch(fromAIA) + dontTouch(io.fromTop.clintTime) + + /* Alias of input valid/ready */ + val valid = io.in.valid + + /* Alias of input signals */ + val wen = io.in.bits.wen && valid + val addr = io.in.bits.addr + val wdata = io.in.bits.wdata + + val ren = io.in.bits.ren && valid + val raddr = io.in.bits.addr + + // flush + val redirectFlush = io.in.bits.redirectFlush + + val hasTrap = io.fromRob.trap.valid + val trapVec = io.fromRob.trap.bits.trapVec + val trapPC = io.fromRob.trap.bits.pc + val trapPCGPA = io.fromRob.trap.bits.pcGPA + val trapIsInterrupt = io.fromRob.trap.bits.isInterrupt + val trapIsCrossPageIPF = io.fromRob.trap.bits.crossPageIPFFix + val trigger = io.fromRob.trap.bits.trigger + val singleStep = io.fromRob.trap.bits.singleStep + val trapIsHls = io.fromRob.trap.bits.isHls + val trapIsFetchMalAddr = io.fromRob.trap.bits.isFetchMalAddr + val trapIsFetchBkpt = io.fromRob.trap.bits.isFetchBkpt + val trapIsForVSnonLeafPTE = io.fromRob.trap.bits.isForVSnonLeafPTE + val trapIsSatpFlushFirstFetchFault = io.fromRob.trap.bits.satpFlushFirstFetchFault + + val oldPrivState = io.oldPrivSate + val oldSatpMode = io.oldSatpMode + val oldVsatpMode = io.oldVsatpMode + + // debug_intrrupt + val debugIntrEnable = RegInit(true.B) // debug interrupt will be handle only when debugIntrEnable + val debugIntr = platformIRP.debugIP && debugIntrEnable + + // CSR Privilege State + val PRVM = RegInit(PrivMode(1, 0), PrivMode.M) + val V = RegInit(VirtMode(0), VirtMode.Off) + val debugMode = RegInit(false.B) + private val nextV = WireInit(VirtMode(0), VirtMode.Off) + V := nextV + // dcsr stopcount + val debugModeStopCountNext = debugMode && dcsr.regOut.STOPCOUNT + val debugModeStopTimeNext = debugMode && dcsr.regOut.STOPTIME + val debugModeStopCount = RegNext(debugModeStopCountNext) + val unprivCountUpdate = !debugModeStopCount && debugModeStopCountNext + + val criticalErrorStateInCSR = Wire(Bool()) + val criticalErrorState = RegEnable(true.B, false.B, io.fromTop.criticalErrorState || criticalErrorStateInCSR) + // When cetrig is 1, resuming from DebugMode following an entry due to a critical + // error will result in an immediate re-entry into Debug Mode due to the critical error. + // Ensure that dpc remains unchanged when criticalErrorState causes a re-entry into dmode, + // since the PC fetched from pcmem for updating dpc is random in this case. + val holdDpc = RegEnable(criticalErrorState && dcsr.regOut.CETRIG, false.B, dretEvent.valid) + + private val privState = Wire(new PrivState) + privState.PRVM := PRVM + privState.V := V + + private val isModeM = privState.isModeM + private val (isModeHS, isModeHU) = (privState.isModeHS, privState.isModeHU) + private val (isModeVS, isModeVU) = (privState.isModeVS, privState.isModeVU) + + val permitMod = Module(new CSRPermitModule) + val sstcIRGen = Module(new SstcInterruptGen) + val commidIdMod = Module(new CommitIDModule(40, hartIdLen)) + + commidIdMod.io.hartId := io.fromTop.hartId + val gitCommitSHA = WireInit(commidIdMod.io.commitID) + val gitDirty = WireInit(commidIdMod.io.dirty) + dontTouch(gitCommitSHA) + dontTouch(gitDirty) + + private val wenLegal = permitMod.io.out.hasLegalWen + + val legalSret = permitMod.io.out.hasLegalSret + val legalMret = permitMod.io.out.hasLegalMret + val legalMNret = permitMod.io.out.hasLegalMNret + val legalDret = permitMod.io.out.hasLegalDret + + private val wenLegalReg = GatedValidRegNext(wenLegal) + + var csrRwMap: SeqMap[Int, (CSRAddrWriteBundle[_], UInt)] = + machineLevelCSRMap ++ + supervisorLevelCSRMap ++ + hypervisorCSRMap ++ + virtualSupervisorCSRMap ++ + unprivilegedCSRMap ++ + debugCSRMap ++ + aiaCSRMap ++ + customCSRMap ++ + indCSRMap ++ + pmpCSRMap ++ + pmaCSRMap + + val csrMods: Seq[CSRModule[_]] = + machineLevelCSRMods ++ + supervisorLevelCSRMods ++ + hypervisorCSRMods ++ + virtualSupervisorCSRMods ++ + unprivilegedCSRMods ++ + debugCSRMods ++ + aiaCSRMods ++ + indCSRMods ++ + customCSRMods ++ + pmpCSRMods ++ + pmaCSRMods + + var csrOutMap: SeqMap[Int, UInt] = + machineLevelCSROutMap ++ + supervisorLevelCSROutMap ++ + hypervisorCSROutMap ++ + virtualSupervisorCSROutMap ++ + unprivilegedCSROutMap ++ + debugCSROutMap ++ + aiaCSROutMap ++ + indCSROutMap ++ + customCSROutMap ++ + pmpCSROutMap ++ + pmaCSROutMap + + // interrupt + val nmip = RegInit(new NonMaskableIRPendingBundle, (new NonMaskableIRPendingBundle).init) + nmip.NMI_43 := nonMaskableIRP.NMI_43 + nmip.NMI_31 := nonMaskableIRP.NMI_31 + + val intrMod = Module(new InterruptFilter) + intrMod.io.in.privState := privState + intrMod.io.in.mstatusMIE := mstatus.regOut.MIE.asBool + intrMod.io.in.sstatusSIE := mstatus.regOut.SIE.asBool + intrMod.io.in.vsstatusSIE := vsstatus.regOut.SIE.asBool + intrMod.io.in.mip := mip.rdataFields + intrMod.io.in.mie := mie.regOut + intrMod.io.in.mideleg := mideleg.regOut + intrMod.io.in.sip := sip.regOut + intrMod.io.in.sie := sie.regOut + intrMod.io.in.hip := hip.regOut + intrMod.io.in.hie := hie.regOut + intrMod.io.in.hideleg := hideleg.regOut + intrMod.io.in.vsip := vsip.regOut + intrMod.io.in.vsie := vsie.regOut + intrMod.io.in.hvictl := hvictl.regOut + intrMod.io.in.hstatus := hstatus.regOut + intrMod.io.in.mtopei := mtopei.regOut + intrMod.io.in.stopei := stopei.regOut + intrMod.io.in.vstopei := vstopei.regOut + intrMod.io.in.hviprio1 := hviprio1.regOut + intrMod.io.in.hviprio2 := hviprio2.regOut + intrMod.io.in.miprios := Cat(miregiprios.map(_.rdata).reverse) + intrMod.io.in.hsiprios := Cat(siregiprios.map(_.rdata).reverse) + intrMod.io.in.mnstatusNMIE := mnstatus.regOut.NMIE.asBool + intrMod.io.in.nmi := nmip.asUInt.orR + intrMod.io.in.nmiVec := nmip.asUInt + intrMod.io.in.debugMode := debugMode + intrMod.io.in.debugIntr := debugIntr + intrMod.io.in.dcsr := dcsr.regOut + intrMod.io.in.platform.meip := platformIRP.MEIP + intrMod.io.in.platform.seip := platformIRP.SEIP + intrMod.io.in.fromAIA.meip := fromAIA.meip + intrMod.io.in.fromAIA.seip := fromAIA.seip + intrMod.io.in.fromAIA.notice_pending := fromAIA.notice_pending + intrMod.io.in.mvienSEIE := mvien.regOut.SEIE.asBool + intrMod.io.in.mvipSEIP := mvip.regOut.SEIP.asBool + + val intrVec = RegEnable(intrMod.io.out.interruptVec.bits, 0.U, intrMod.io.out.interruptVec.valid) + val debug = RegEnable(intrMod.io.out.debug, false.B, intrMod.io.out.interruptVec.valid) + val nmi = intrMod.io.out.nmi + val virtualInterruptIsHvictlInject = intrMod.io.out.virtualInterruptIsHvictlInject + val irToHS = RegEnable(intrMod.io.out.irToHS, false.B, intrMod.io.out.interruptVec.valid) + val irToVS = RegEnable(intrMod.io.out.irToVS, false.B, intrMod.io.out.interruptVec.valid) + + val trapHandleMod = Module(new TrapHandleModule) + + trapHandleMod.io.in.trapInfo.valid := hasTrap + trapHandleMod.io.in.trapInfo.bits.trapVec := trapVec.asUInt + trapHandleMod.io.in.trapInfo.bits.intrVec := intrVec + trapHandleMod.io.in.trapInfo.bits.isInterrupt := trapIsInterrupt + trapHandleMod.io.in.trapInfo.bits.irToHS := irToHS + trapHandleMod.io.in.trapInfo.bits.irToVS := irToVS + trapHandleMod.io.in.privState := privState + trapHandleMod.io.in.mstatus := mstatus.regOut + trapHandleMod.io.in.sstatus := mstatus.sstatus + trapHandleMod.io.in.vsstatus := vsstatus.regOut + trapHandleMod.io.in.mnstatus := mnstatus.regOut + trapHandleMod.io.in.mideleg := mideleg.regOut + trapHandleMod.io.in.medeleg := medeleg.regOut + trapHandleMod.io.in.hideleg := hideleg.regOut + trapHandleMod.io.in.hedeleg := hedeleg.regOut + trapHandleMod.io.in.mvien := mvien.regOut + trapHandleMod.io.in.hvien := hvien.regOut + trapHandleMod.io.in.mtvec := mtvec.regOut + trapHandleMod.io.in.stvec := stvec.regOut + trapHandleMod.io.in.vstvec := vstvec.regOut + trapHandleMod.io.in.trapInfo.bits.singleStep := hasTrap && !trapIsInterrupt && singleStep + + val entryPrivState = trapHandleMod.io.out.entryPrivState + val entryDebugMode = WireInit(false.B) + val dbltrpToMN = trapHandleMod.io.out.dbltrpToMN + val hasDTExcp = trapHandleMod.io.out.hasDTExcp + + val NumPMPReal = p(PMParameKey).NumPMPReal + io.accelerator.satp := satp.regOut.asUInt + io.accelerator.privilege := io.tlb.dmode + io.accelerator.sum := io.tlb.sum + io.accelerator.mxr := io.tlb.mxr + io.accelerator.virt := io.tlb.dvirt + for (i <- 0 until p(PMParameKey).NumPMP) { + val cfg = if (i < NumPMPReal) pmpcfgs(i).rdata(7, 0) else 0.U(8.W) + val nextTor = if (i + 1 < NumPMPReal) pmpcfgs(i + 1).rdata(4, 3) === 1.U else false.B + io.accelerator.pmp(i).cfg := cfg + io.accelerator.pmp(i).addr := Mux(cfg(4, 3) =/= 0.U || nextTor, pmpaddr(i).rdata(45, 0), 0.U) + } + + // PMP + val pmpEntryMod = Module(new PMPEntryHandleModule) + pmpEntryMod.io.in.pmpCfg := pmpcfgs.map(_.regOut.asInstanceOf[PMPCfgBundle]) + pmpEntryMod.io.in.ren := ren + pmpEntryMod.io.in.wen := wenLegalReg + pmpEntryMod.io.in.addr := addr + pmpEntryMod.io.in.wdata := wdata + + // PMA + val pmaEntryMod = Module(new PMAEntryHandleModule) + pmaEntryMod.io.in.pmaCfg := pmacfgs.map(_.regOut.asInstanceOf[PMACfgBundle]) + pmaEntryMod.io.in.ren := ren + pmaEntryMod.io.in.wen := wenLegalReg + pmaEntryMod.io.in.addr := addr + pmaEntryMod.io.in.wdata := wdata + + // Todo: all wen and wdata of CSRModule assigned in this for loop + for ((id, (wBundle, _)) <- csrRwMap) { + if (vsMapS.contains(id)) { + // VS access CSR by S: privState.isModeVS && addrMappedToVS === sMapVS(id).U + wBundle.wen := wenLegalReg && ((isModeVS && addr === vsMapS(id).U) || (!isModeVS && addr === id.U)) + wBundle.wdata := wdata + } else if (sMapVS.contains(id)) { + wBundle.wen := wenLegalReg && !isModeVS && addr === id.U + wBundle.wdata := wdata + } else { + wBundle.wen := wenLegalReg && addr === id.U + wBundle.wdata := wdata + } + } + + private val writeFpLegal = permitMod.io.out.hasLegalWriteFcsr + private val writeVecLegal = permitMod.io.out.hasLegalWriteVcsr + + permitMod.io.in.csrAccess.ren := ren && valid + permitMod.io.in.csrAccess.wen := wen + permitMod.io.in.csrAccess.addr := addr + + permitMod.io.in.privState := privState + permitMod.io.in.debugMode := debugMode + + permitMod.io.in.xRet.mnret := io.in.bits.mnret && valid + permitMod.io.in.xRet.mret := io.in.bits.mret && valid + permitMod.io.in.xRet.sret := io.in.bits.sret && valid + permitMod.io.in.xRet.dret := io.in.bits.dret && valid + + permitMod.io.in.status.tsr := mstatus.regOut.TSR.asBool + permitMod.io.in.status.vtsr := hstatus.regOut.VTSR.asBool + + permitMod.io.in.status.tvm := mstatus.regOut.TVM.asBool + permitMod.io.in.status.vtvm := hstatus.regOut.VTVM.asBool + + permitMod.io.in.status.vgein := hstatus.regOut.VGEIN.asUInt + + permitMod.io.in.xcounteren.mcounteren := mcounteren.rdata + permitMod.io.in.xcounteren.hcounteren := hcounteren.rdata + permitMod.io.in.xcounteren.scounteren := scounteren.rdata + + permitMod.io.in.xstateen.mstateen0 := mstateen0.rdata + permitMod.io.in.xstateen.mstateen1 := mstateen1.rdata + permitMod.io.in.xstateen.mstateen2 := mstateen2.rdata + permitMod.io.in.xstateen.mstateen3 := mstateen3.rdata + permitMod.io.in.xstateen.hstateen0 := hstateen0.rdata + permitMod.io.in.xstateen.hstateen1 := hstateen1.rdata + permitMod.io.in.xstateen.hstateen2 := hstateen2.rdata + permitMod.io.in.xstateen.hstateen3 := hstateen3.rdata + permitMod.io.in.xstateen.sstateen0 := sstateen0.rdata + + permitMod.io.in.xenvcfg.menvcfg := menvcfg.rdata + permitMod.io.in.xenvcfg.henvcfg := henvcfg.rdata + + permitMod.io.in.status.mstatusFSOff := mstatus.regOut.FS === ContextStatus.Off + permitMod.io.in.status.mstatusVSOff := mstatus.regOut.VS === ContextStatus.Off + permitMod.io.in.status.vsstatusFSOff := vsstatus.regOut.FS === ContextStatus.Off + permitMod.io.in.status.vsstatusVSOff := vsstatus.regOut.VS === ContextStatus.Off + + permitMod.io.in.aia.miselect := miselect.rdata + permitMod.io.in.aia.siselect := siselect.rdata + permitMod.io.in.aia.vsiselect := vsiselect.rdata + permitMod.io.in.aia.mvienSEIE := mvien.regOut.SEIE.asBool + permitMod.io.in.aia.hvictlVTI := hvictl.regOut.VTI.asBool + + sstcIRGen.i.stime.valid := time.updated + sstcIRGen.i.stime.bits := time.stime + sstcIRGen.i.vstime.valid := time.updated + sstcIRGen.i.vstime.bits := time.vstime + sstcIRGen.i.stimecmp.wen := GatedValidRegNext(stimecmp.w.wen) + sstcIRGen.i.stimecmp.rdata := stimecmp.rdata + sstcIRGen.i.vstimecmp.wen := GatedValidRegNext(vstimecmp.w.wen) + sstcIRGen.i.vstimecmp.rdata := vstimecmp.rdata + sstcIRGen.i.menvcfg.wen := GatedValidRegNext(menvcfg.w.wen) + sstcIRGen.i.menvcfg.STCE := menvcfg.regOut.STCE.asBool + sstcIRGen.i.henvcfg.wen := GatedValidRegNext(henvcfg.w.wen) + sstcIRGen.i.henvcfg.STCE := henvcfg.regOut.STCE.asBool + sstcIRGen.i.htimedeltaWen := GatedValidRegNext(htimedelta.w.wen) + + miregiprios.foreach { mod => + mod.w.wen := mireg.w.wen && (miselect.regOut.ALL.asUInt === mod.addr.U) + mod.w.wdata := wdata + } + + siregiprios.foreach { mod => + mod.w.wen := sireg.w.wen && (siselect.regOut.ALL.asUInt === mod.addr.U) + mod.w.wdata := wdata + } + + siregiprios.foreach { mod => + mod match { + case m: HasSiprios => + m.mideleg := mideleg.regOut + m.mvien := mvien.regOut + case _ => + } + } + + mhartid.hartid := this.io.fromTop.hartId + mhartid.dmode := debugMode + mtvec.reset_mtvec.foreach(_ := this.io.fromTop.reset_mtvec.get) + + pmpcfgs.zipWithIndex.foreach { case (mod, i) => + mod.w.wen := wenLegalReg && (addr === (CSRs.pmpcfg0 + i / 8 * 2).U) + mod.w.wdata := pmpEntryMod.io.out.pmpCfgWData(8 * ((i % 8) + 1) - 1, 8 * (i % 8)) + } + + pmacfgs.zipWithIndex.foreach { case (mod, i) => + mod.w.wen := wenLegalReg && (addr === (CSRConst.PmacfgBase + i / 8 * 2).U) + mod.w.wdata := pmaEntryMod.io.out.pmaCfgWdata(8 * ((i % 8) + 1) - 1, 8 * (i % 8)) + } + + csrMods.foreach { mod => + mod match { + case m: HypervisorBundle => + m.hstatus := hstatus.regOut + case _ => + } + mod match { + case m: VirtualSupervisorBundle => + m.v := V.asUInt.asBool + m.hgatp := hgatp.regOut + case _ => + } + mod match { + case m: HasMachineDelegBundle => + m.mideleg := mideleg.regOut + m.medeleg := medeleg.regOut + case _ => + } + mod match { + case m: HasMachineCounterControlBundle => + m.mcountinhibit := mcountinhibit.regOut + case _ => + } + mod match { + case m: HasExternalInterruptBundle => + m.platformIRP := this.platformIRP + m.platformIRP.STIP := sstcIRGen.o.STIP + m.platformIRP.VSTIP := sstcIRGen.o.VSTIP + case _ => + } + mod match { + case m: HasRobCommitBundle => + // Todo: move RegNext from ROB to CSR + m.robCommit.instNum := io.fromRob.commit.instNum + m.robCommit.fflags := RegNextWithEnable(io.fromRob.commit.fflags) + m.robCommit.fsDirty := GatedValidRegNext(io.fromRob.commit.fsDirty) + m.robCommit.vsDirty := GatedValidRegNext(io.fromRob.commit.vsDirty) + m.robCommit.vxsat := RegNextWithEnable(io.fromRob.commit.vxsat) + m.robCommit.vtype := RegNextWithEnable(io.fromRob.commit.vtype) + m.robCommit.vl := RegNext(io.fromRob.commit.vl) + m.robCommit.vstart := RegNextWithEnable(io.fromRob.commit.vstart) + m.writeFCSR := writeFpLegal + m.writeVCSR := writeVecLegal + m.isVirtMode := V.asUInt.asBool + case _ => + } + mod match { + case m: TrapEntryDEventSinkBundle => + m.trapToD := trapEntryDEvent.out + case _ => + } + mod match { + case m: TrapEntryMEventSinkBundle => + m.trapToM := trapEntryMEvent.out + case _ => + } + mod match { + case m: TrapEntryMNEventSinkBundle => + m.trapToMN := trapEntryMNEvent.out + case _ => + } + mod match { + case m: TrapEntryHSEventSinkBundle => + m.trapToHS := trapEntryHSEvent.out + case _ => + } + mod match { + case m: TrapEntryVSEventSinkBundle => + m.trapToVS := trapEntryVSEvent.out + case _ => + } + mod match { + case m: MretEventSinkBundle => + m.retFromM := mretEvent.out + case _ => + } + mod match { + case m: MNretEventSinkBundle => + m.retFromMN := mnretEvent.out + case _ => + } + mod match { + case m: SretEventSinkBundle => + m.retFromS := sretEvent.out + case _ => + } + mod match { + case m: SretEventSDTSinkBundle => + m.retFromSSDT := sretEvent.outSDT + case _ => + } + mod match { + case m: DretEventSinkBundle => + m.retFromD := dretEvent.out + case _ => + } + mod match { + case m: HasAIABundle => + m.aiaToCSR.rdata.valid := fromAIA.rdata.valid + m.aiaToCSR.rdata.bits.data := fromAIA.rdata.bits.data + m.aiaToCSR.rdata.bits.illegal := fromAIA.rdata.bits.illegal + m.aiaToCSR.meip := fromAIA.meip + m.aiaToCSR.seip := fromAIA.seip + m.aiaToCSR.vseip := fromAIA.vseip + m.aiaToCSR.mtopei := fromAIA.mtopei + m.aiaToCSR.stopei := fromAIA.stopei + m.aiaToCSR.vstopei := fromAIA.vstopei + m.aiaToCSR.notice_pending := fromAIA.notice_pending + case _ => + } + mod match { + case m: HasInterruptFilterSink => + m.topIR.mtopi := intrMod.io.out.mtopi + m.topIR.stopi := intrMod.io.out.stopi + m.topIR.vstopi := intrMod.io.out.vstopi + case _ => + } + mod match { + case m: HasPMPAddrSink => + m.addrRData := pmpEntryMod.io.out.pmpAddrRData + case _ => + } + mod match { + case m: HasPMAAddrSink => + m.addrRData := pmaEntryMod.io.out.pmaAddrRData + case _ => + } + mod match { + case m: HasMHPMSink => + // cycle from mcycle + m.mHPM.cycle := mcycle.rdata + // time from clint + m.mHPM.time := io.fromTop.clintTime + // instret from minstret + m.mHPM.instret := minstret.rdata + // VS-Mode or VU-Mode + m.v := privState.isVirtual + m.nextV := nextV.isOneOf(VirtMode.On) + m.htimedelta := htimedelta.rdata + m.mHPM.hpmcounters.zip(mhpmcounters).map { + case (counter, mcounter) => counter := mcounter.rdata + } + case _ => + } + mod match { + case m: HasMachineEnvBundle => + m.menvcfg := menvcfg.regOut + case _ => + } + mod match { + case m: HasHypervisorEnvBundle => + m.menvcfg := menvcfg.regOut + case _ => + } + mod match { + case m: HasVirtualSupervisorEnvBundle => + m.henvcfg := henvcfg.regOut + m.menvcfg := menvcfg.regOut + case _ => + } + mod match { + case m: HasIpIeBundle => + m.mideleg := mideleg.regOut + m.mip := mip.rdata + m.mie := mie.regOut + m.mvip := mvip.regOut + m.mvien := mvien.regOut + m.hideleg := hideleg.regOut + m.hip := hip.regOut + m.hie := hie.regOut + m.hvien := hvien.regOut + m.hvip := hvip.regOut + m.sip := sip.regOut + m.sie := sie.regOut + m.vsip := vsip.regOut + m.vsie := vsie.regOut + m.hgeip := hgeip.regOut + m.hgeie := hgeie.regOut + m.hstatusVGEIN := hstatus.regOut.VGEIN + case _ => + } + mod match { + case m: HasMhpmeventOfBundle => + m.ofVec := VecInit(mhpmevents.map { event => + val mhpmevent = Wire(new MhpmeventBundle) + mhpmevent := event.rdata + mhpmevent.OF.asBool + }).asUInt + m.privState := privState + m.mcounteren := mcounteren.rdata + m.hcounteren := hcounteren.rdata + case _ => + } + mod match { + case m: HasStateenBundle => + m.fromMstateen0 := mstateen0.regOut + m.fromMstateen1 := mstateen1.regOut + m.fromMstateen2 := mstateen2.regOut + m.fromMstateen3 := mstateen3.regOut + m.fromHstateen0 := hstateen0.regOut + m.privState := privState + case _ => + } + mod match { + case m: HasDebugStopBundle => + m.debugModeStopCount := debugModeStopCount + m.debugModeStopTime := debugModeStopTimeNext + m.unprivCountUpdate := unprivCountUpdate + case _ => + } + mod match { + case m: HasNmipBundle => + m.nmip := nmip.asUInt.orR + case _ => + } + mod match { + case m: HasMachineFlushL2Bundle => + m.l2FlushDone := io.fromTop.l2FlushDone + case _ => + } + } + + trapEntryMNEvent.valid := ((hasTrap && nmi) || dbltrpToMN) && !entryDebugMode && !debugMode && mnstatus.regOut.NMIE + trapEntryMEvent.valid := hasTrap && entryPrivState.isModeM && !dbltrpToMN && !entryDebugMode && !debugMode && !nmi && mnstatus.regOut.NMIE + trapEntryHSEvent.valid := hasTrap && entryPrivState.isModeHS && !entryDebugMode && !debugMode && !nmi && mnstatus.regOut.NMIE + trapEntryVSEvent.valid := hasTrap && entryPrivState.isModeVS && !entryDebugMode && !debugMode && !nmi && mnstatus.regOut.NMIE + + Seq(trapEntryMEvent, trapEntryMNEvent, trapEntryHSEvent, trapEntryVSEvent, trapEntryDEvent).foreach { eMod => + eMod.in match { + case in: TrapEntryEventInput => + in.causeNO := trapHandleMod.io.out.causeNO + in.trapPc := trapPC + in.trapPcGPA := trapPCGPA // only used by trapEntryMEvent & trapEntryHSEvent + in.trapInst := io.trapInst + in.fetchMalTval := io.fetchMalTval + in.isCrossPageIPF := trapIsCrossPageIPF + in.isHls := trapIsHls + in.isFetchMalAddr := trapIsFetchMalAddr + in.satpFlushFirstFetchFault := trapIsSatpFlushFirstFetchFault + in.isFetchBkpt := trapIsFetchBkpt + in.trapIsForVSnonLeafPTE := trapIsForVSnonLeafPTE + in.hasDTExcp := hasDTExcp + + in.iMode.PRVM := Mux(trapIsSatpFlushFirstFetchFault, oldPrivState.PRVM, PRVM) + in.iMode.V := Mux(trapIsSatpFlushFirstFetchFault, oldPrivState.V, V) + // when NMIE is zero, force to behave as MPRV is zero + in.dMode.PRVM := Mux(mstatus.regOut.MPRV.asBool && mnstatus.regOut.NMIE.asBool, mstatus.regOut.MPP, PRVM) + in.dMode.V := V.asUInt.asBool || mstatus.regOut.MPRV && mnstatus.regOut.NMIE.asBool && (mstatus.regOut.MPP =/= PrivMode.M) && mstatus.regOut.MPV + + in.privState := privState + in.mstatus := mstatus.regOut + in.hstatus := hstatus.regOut + in.sstatus := mstatus.sstatus + in.vsstatus := vsstatus.regOut + in.pcFromXtvec := trapHandleMod.io.out.pcFromXtvec + + in.menvcfg := menvcfg.regOut + in.henvcfg := henvcfg.regOut + + in.satp := satp.regOut + in.vsatp := vsatp.regOut + in.hgatp := hgatp.regOut + in.oldSatp := Mux( + trapIsSatpFlushFirstFetchFault, + Cat(oldSatpMode, 0.U((XLEN - SatpMode.getWidth).W)).asTypeOf(in.oldSatp), + satp.regOut + ) + in.oldVsatp := Mux( + trapIsSatpFlushFirstFetchFault, + Cat(oldVsatpMode, 0.U((XLEN - SatpMode.getWidth).W)).asTypeOf(in.oldVsatp), + vsatp.regOut + ) + if (HasBitmapCheck) { + in.mbmc := mbmc.get.regOut + } else { + in.mbmc := DontCare + } + + in.memExceptionVAddr := io.fromMem.excpVA + in.memExceptionGPAddr := io.fromMem.excpGPA + in.memExceptionIsForVSnonLeafPTE := io.fromMem.excpIsForVSnonLeafPTE + + in.virtualInterruptIsHvictlInject := virtualInterruptIsHvictlInject + in.hvictlIID := hvictl.regOut.IID.asUInt + } + } + + mnretEvent.valid := legalMNret + mnretEvent.in match { + case in => + in.mstatus := mstatus.regOut + in.vsstatus := vsstatus.regOut + in.mnepc := mnepc.regOut + in.mnstatus := mnstatus.regOut + in.satp := satp.regOut + in.vsatp := vsatp.regOut + in.hgatp := hgatp.regOut + } + + mretEvent.valid := legalMret + mretEvent.in match { + case in => + in.mstatus := mstatus.regOut + in.vsstatus := vsstatus.regOut + in.mepc := mepc.regOut + in.satp := satp.regOut + in.vsatp := vsatp.regOut + in.hgatp := hgatp.regOut + } + + sretEvent.valid := legalSret + sretEvent.in match { + case in => + in.privState := privState + in.mstatus := mstatus.regOut + in.hstatus := hstatus.regOut + in.vsstatus := vsstatus.regOut + in.sepc := sepc.regOut + in.vsepc := vsepc.regOut + in.satp := satp.regOut + in.vsatp := vsatp.regOut + in.hgatp := hgatp.regOut + } + + dretEvent.valid := legalDret + dretEvent.in match { + case in => + in.dcsr := dcsr.regOut + in.dpc := dpc.regOut + in.mstatus := mstatus.regOut + in.vsstatus := vsstatus.regOut + in.satp := satp.regOut + in.vsatp := vsatp.regOut + in.hgatp := hgatp.regOut + } + + PRVM := MuxCase( + PRVM, + events.filter(_.out.isInstanceOf[EventUpdatePrivStateOutput]).map { + x => + x.out match { + case xx: EventUpdatePrivStateOutput => (xx.privState.valid -> xx.privState.bits.PRVM) + } + } + ) + + nextV := MuxCase( + V, + events.filter(_.out.isInstanceOf[EventUpdatePrivStateOutput]).map { + x => + x.out match { + case xx: EventUpdatePrivStateOutput => (xx.privState.valid -> xx.privState.bits.V) + } + } + ) + + debugMode := MuxCase( + debugMode, + Seq( + dretEvent.out.debugMode.valid -> dretEvent.out.debugMode.bits, + trapEntryDEvent.out.debugMode.valid -> trapEntryDEvent.out.debugMode.bits + ) + ) + + debugIntrEnable := MuxCase( + debugIntrEnable, + Seq( + dretEvent.out.debugIntrEnable.valid -> dretEvent.out.debugIntrEnable.bits, + trapEntryDEvent.out.debugIntrEnable.valid -> trapEntryDEvent.out.debugIntrEnable.bits + ) + ) + + // perf + val addrInPerfCnt = (wenLegal || ren) && ( + (addr >= CSRs.mcycle.U) && (addr <= CSRs.mhpmcounter31.U) || + (addr >= CSRs.cycle.U) && (addr <= CSRs.hpmcounter31.U) + ) || + ren && ( + (addr === CSRs.vstopi.U) || (addr === CSRs.vstopei.U) || + (addr === CSRs.stopi.U) || (addr === CSRs.stopei.U) || + (addr === CSRs.mtopi.U) || (addr === CSRs.mtopei.U) + ) + + val resetSatp = WireInit(false.B) + // flush + if (HasBitmapCheck) { + resetSatp := Cat( + Seq(satp, vsatp, hgatp, mbmc.get).map(_.addr.U === addr) + ).orR && wenLegalReg // write to satp will cause the pipeline be flushed + } else { + resetSatp := Cat( + Seq(satp, vsatp, hgatp).map(_.addr.U === addr) + ).orR && wenLegalReg // write to satp will cause the pipeline be flushed + } + + val floatStatusOnOff = mstatus.w.wen && ( + mstatus.w.wdataFields.FS === ContextStatus.Off && mstatus.regOut.FS =/= ContextStatus.Off || + mstatus.w.wdataFields.FS =/= ContextStatus.Off && mstatus.regOut.FS === ContextStatus.Off + ) || mstatus.wAliasSstatus.wen && ( + mstatus.wAliasSstatus.wdataFields.FS === ContextStatus.Off && mstatus.regOut.FS =/= ContextStatus.Off || + mstatus.wAliasSstatus.wdataFields.FS =/= ContextStatus.Off && mstatus.regOut.FS === ContextStatus.Off + ) || vsstatus.w.wen && ( + vsstatus.w.wdataFields.FS === ContextStatus.Off && vsstatus.regOut.FS =/= ContextStatus.Off || + vsstatus.w.wdataFields.FS =/= ContextStatus.Off && vsstatus.regOut.FS === ContextStatus.Off + ) + + val vectorStatusOnOff = mstatus.w.wen && ( + mstatus.w.wdataFields.VS === ContextStatus.Off && mstatus.regOut.VS =/= ContextStatus.Off || + mstatus.w.wdataFields.VS =/= ContextStatus.Off && mstatus.regOut.VS === ContextStatus.Off + ) || mstatus.wAliasSstatus.wen && ( + mstatus.wAliasSstatus.wdataFields.VS === ContextStatus.Off && mstatus.regOut.VS =/= ContextStatus.Off || + mstatus.wAliasSstatus.wdataFields.VS =/= ContextStatus.Off && mstatus.regOut.VS === ContextStatus.Off + ) || vsstatus.w.wen && ( + vsstatus.w.wdataFields.VS === ContextStatus.Off && vsstatus.regOut.VS =/= ContextStatus.Off || + vsstatus.w.wdataFields.VS =/= ContextStatus.Off && vsstatus.regOut.VS === ContextStatus.Off + ) + + val triggerFrontendChange = Wire(Bool()) + + val vstartChange = vstart.w.wen && ( + vstart.w.wdata === 0.U && vstart.regOut.vstart.asUInt =/= 0.U || + vstart.w.wdata =/= 0.U + ) + + // flush pipe when write frm and data > 4 or write fcsr and data[7:5] > 4 or write frm/fcsr and frm is reserved + val frmIsReserved = fcsr.frm(2) && fcsr.frm(1, 0).orR + val frmWdataReserved = fcsr.wAliasFfm.wdata(2) && fcsr.wAliasFfm.wdata(1, 0).orR + val fcsrWdataReserved = fcsr.w.wdata(7) && fcsr.w.wdata(6, 5).orR + val frmChange = fcsr.wAliasFfm.wen && (!frmIsReserved && frmWdataReserved || frmIsReserved && !frmWdataReserved) || + fcsr.w.wen && (!frmIsReserved && fcsrWdataReserved || frmIsReserved && !fcsrWdataReserved) + + val flushPipe = resetSatp || + triggerFrontendChange || floatStatusOnOff || vectorStatusOnOff || + vstartChange || frmChange + + /** + * Look up id in vsMapS and sMapVS. + * If id is in vsMapS, use vsMapS(id) when under VS mode, + * id under other modes + * Else If id is in sMapVS, use 0 when under VS mode, + * id under modes except VS + * Else, use id as read address + * Use read address to look up rdata in csrRwMap + */ + private val rdata = Mux1H(csrRwMap.map { case (id, (_, rdata)) => + if (vsMapS.contains(id)) { + ((isModeVS && addr === vsMapS(id).U) || !isModeVS && addr === id.U) -> rdata + } else if (sMapVS.contains(id)) { + (!isModeVS && addr === id.U) -> rdata + } else { + (raddr === id.U) -> rdata + } + }) + + private val rwMask = 0xc00 + private val csrOutMapFilter = csrOutMap.filter { case (id, _) => (id & rwMask) != rwMask } + + private val regOut = Mux1H(csrOutMapFilter.map { case (id, regOut) => + if (vsMapS.contains(id)) { + ((isModeVS && addr === vsMapS(id).U) || !isModeVS && addr === id.U) -> regOut + } else if (sMapVS.contains(id)) { + (!isModeVS && addr === id.U) -> regOut + } else { + (raddr === id.U) -> regOut + } + }) + + private val needTargetUpdate = + mnretEvent.out.targetPc.valid || mretEvent.out.targetPc.valid || sretEvent.out.targetPc.valid || dretEvent.out.targetPc.valid || + trapEntryMEvent.out.targetPc.valid || trapEntryMNEvent.out.targetPc.valid || trapEntryHSEvent.out.targetPc.valid || trapEntryVSEvent.out.targetPc.valid || trapEntryDEvent.out.targetPc.valid + + private val noCSRIllegal = (ren || wen) && Cat(csrRwMap.keys.toSeq.sorted.map(csrAddr => !(addr === csrAddr.U))).andR + + private val noCSRIllegalReg = RegEnable(noCSRIllegal, ren || wen) + + private val s_idle :: s_waitIMSIC :: s_finish :: Nil = Enum(3) + + /** the state machine of newCSR module */ + private val state = RegInit(s_idle) + + /** the next state of newCSR */ + private val stateNext = WireInit(state) + state := stateNext + + /** + * Asynchronous access operation of CSR. Check whether an access is asynchronous when read/write-enable is high. + * AIA registers are designed to be access asynchronously, so newCSR will wait for response. + */ + private val asyncAccess = (wen || ren) && !(permitMod.io.out.EX_II || permitMod.io.out.EX_VI) && ( + mireg.addr.U === addr && miselect.inIMSICRange || + sireg.addr.U === addr && ((!V.asUInt.asBool && siselect.inIMSICRange) || (V.asUInt.asBool && vsiselect.inIMSICRange)) || + vsireg.addr.U === addr && vsiselect.inIMSICRange + ) + + /** State machine of newCSR */ + switch(state) { + is(s_idle) { + when(valid && redirectFlush) { + stateNext := s_idle + }.elsewhen(valid && asyncAccess) { + stateNext := s_waitIMSIC + }.elsewhen(valid) { + stateNext := s_finish + } + } + is(s_waitIMSIC) { + when(redirectFlush) { + stateNext := s_idle + }.elsewhen(fromAIA.rdata.valid) { + when(io.out.ready) { + stateNext := s_idle + }.otherwise { + stateNext := s_finish + } + } + } + is(s_finish) { + when(redirectFlush || io.out.ready) { + stateNext := s_idle + } + } + } + + // Todo: check IMSIC EX_II and EX_VI + private val imsicIllegal = fromAIA.rdata.valid && fromAIA.rdata.bits.illegal + private val imsic_EX_II = imsicIllegal && !V.asUInt.asBool + private val imsic_EX_VI = imsicIllegal && V.asUInt.asBool + + /** Set io.in.ready when state machine is ready to receive a new request synchronously */ + io.in.ready := (state === s_idle) + + /** + * Valid signal of newCSR output. + * When in IDLE state, when input_valid is high, we set it. + * When in waitIMSIC state, and the next state is IDLE, we set it. + */ + + /** Data that have been read before,and should be stored because output not fired */ + val normalCSRValid = state === s_idle && valid && !asyncAccess + val waitIMSICValid = state === s_waitIMSIC && fromAIA.rdata.valid + val claimAIA = mtopei.w.wen | stopei.w.wen | vstopei.w.wen + + io.out.valid := (waitIMSICValid || state === s_finish) && !redirectFlush + io.out.bits.EX_II := DataHoldBypass( + Mux1H(Seq( + normalCSRValid -> (permitMod.io.out.EX_II || noCSRIllegal), + waitIMSICValid -> imsic_EX_II + )), + false.B, + normalCSRValid || waitIMSICValid + ) + io.out.bits.EX_VI := DataHoldBypass( + Mux1H(Seq( + normalCSRValid -> permitMod.io.out.EX_VI, + waitIMSICValid -> imsic_EX_VI + )), + false.B, + normalCSRValid || waitIMSICValid + ) + io.out.bits.flushPipe := flushPipe + + /** Prepare read data for output */ + io.out.bits.rData := DataHoldBypass( + Mux1H(Seq( + (io.in.fire || claimAIA) -> rdata, + fromAIA.rdata.valid -> fromAIA.rdata.bits.data + )), + 0.U(64.W), + io.in.fire || fromAIA.rdata.valid || claimAIA + ) + io.out.bits.regOut := regOut + io.out.bits.targetPc := DataHoldBypass( + Mux( + trapEntryDEvent.out.targetPc.valid, + trapEntryDEvent.out.targetPc.bits, + Mux1H(Seq( + mnretEvent.out.targetPc.valid -> mnretEvent.out.targetPc.bits, + mretEvent.out.targetPc.valid -> mretEvent.out.targetPc.bits, + sretEvent.out.targetPc.valid -> sretEvent.out.targetPc.bits, + dretEvent.out.targetPc.valid -> dretEvent.out.targetPc.bits, + trapEntryMEvent.out.targetPc.valid -> trapEntryMEvent.out.targetPc.bits, + trapEntryMNEvent.out.targetPc.valid -> trapEntryMNEvent.out.targetPc.bits, + trapEntryHSEvent.out.targetPc.valid -> trapEntryHSEvent.out.targetPc.bits, + trapEntryVSEvent.out.targetPc.valid -> trapEntryVSEvent.out.targetPc.bits + )) + ), + needTargetUpdate + ) + io.out.bits.targetPcUpdate := needTargetUpdate + io.out.bits.isPerfCnt := DataHoldBypass(addrInPerfCnt, false.B, io.in.fire) + + io.status.privState := privState + io.status.fpState.frm := fcsr.frm + io.status.fpState.off := mstatus.regOut.FS === ContextStatus.Off + io.status.vecState.vstart := vstart.rdata.asUInt + io.status.vecState.vxsat := vcsr.vxsat + io.status.vecState.vxrm := vcsr.vxrm + io.status.vecState.vcsr := vcsr.rdata.asUInt + io.status.vecState.vl := vl.rdata.asUInt + io.status.vecState.vtype := vtype.rdata.asUInt // Todo: check correct + io.status.vecState.vlenb := vlenb.rdata.asUInt + io.status.vecState.off := mstatus.regOut.VS === ContextStatus.Off + io.status.interrupt := intrMod.io.out.interruptVec.valid + io.status.wfiEvent := debugIntr || nmip.asUInt.orR || + (mtopi.regOut.IID.asUInt =/= 0.U) || + (stopi.regOut.IID.asUInt =/= 0.U) || + (vstopi.regOut.IID.asUInt =/= 0.U) + io.status.debugMode := debugMode + io.status.singleStepFlag := !debugMode && dcsr.regOut.STEP + io.status.satp.valid := satp.w.wen + io.status.satp.bits := satp.regOut.MODE.asUInt + io.status.vsatp.valid := vsatp.w.wen + io.status.vsatp.bits := vsatp.regOut.MODE.asUInt + + /** + * debug_begin + */ + val tdata1Selected = Wire(new Tdata1Bundle) + tdata1Selected := tdata1.rdata + val dmodeInSelectedTrigger = tdata1Selected.DMODE.asBool + val triggerCanWrite = dmodeInSelectedTrigger && debugMode || !dmodeInSelectedTrigger + val tdata1Update = tdata1.w.wen && triggerCanWrite + val tdata2Update = tdata2.w.wen && triggerCanWrite + + val tdata1Vec = tdata1RegVec.map { mod => + val tdata1Wire = Wire(new Tdata1Bundle) + tdata1Wire := mod.rdata + tdata1Wire + } + + val triggerCanRaiseBpExp = !(privState.isModeM && !mstatus.regOut.MIE || + medeleg.regOut.EX_BP && privState.isModeHS && !mstatus.sstatus.SIE || + medeleg.regOut.EX_BP && hedeleg.regOut.EX_BP && privState.isModeVS && !vsstatus.regOut.SIE) + + val debugMod = Module(new Debug) + debugMod.io.in.trapInfo.valid := hasTrap + debugMod.io.in.trapInfo.bits.trapVec := trapVec.asUInt + debugMod.io.in.trapInfo.bits.isDebugIntr := debug + debugMod.io.in.trapInfo.bits.isInterrupt := trapIsInterrupt + debugMod.io.in.trapInfo.bits.trigger := trigger + debugMod.io.in.trapInfo.bits.singleStep := singleStep + debugMod.io.in.trapInfo.bits.criticalErrorState := criticalErrorState + debugMod.io.in.privState := privState + debugMod.io.in.debugMode := debugMode + debugMod.io.in.dcsr := dcsr.regOut + debugMod.io.in.tselect := tselect.regOut + debugMod.io.in.tdata1Vec := tdata1Vec + debugMod.io.in.tdata1Selected := tdata1.rdata + debugMod.io.in.tdata2Selected := tdata2.rdata + debugMod.io.in.tdata1Update := tdata1Update + debugMod.io.in.tdata2Update := tdata2Update + debugMod.io.in.tdata1Wdata := wdata + debugMod.io.in.triggerCanRaiseBpExp := triggerCanRaiseBpExp + + entryDebugMode := debugMod.io.out.hasDebugTrap && !debugMode + + trapEntryDEvent.valid := entryDebugMode + trapEntryDEvent.in.hasDebugIntr := debugMod.io.out.hasDebugIntr + trapEntryDEvent.in.debugMode := debugMode + trapEntryDEvent.in.hasTrap := hasTrap + trapEntryDEvent.in.hasSingleStep := debugMod.io.out.hasSingleStep + trapEntryDEvent.in.triggerEnterDebugMode := debugMod.io.out.triggerEnterDebugMode + trapEntryDEvent.in.hasDebugEbreakException := debugMod.io.out.hasDebugEbreakException + trapEntryDEvent.in.breakPoint := debugMod.io.out.breakPoint + trapEntryDEvent.in.criticalErrorStateEnterDebug := debugMod.io.out.criticalErrorStateEnterDebug + trapEntryDEvent.in.holdDpc := holdDpc + + for (idx <- 0 until TriggerNum) { + val tdata1Pre = Wire(new Tdata1Bundle) + val mcontrol6Pre = Wire(new Mcontrol6) + tdata1Pre := (if (idx > 0) tdata1RegVec(idx - 1) else tdata1RegVec(idx)).rdata.asUInt + mcontrol6Pre := tdata1Pre.DATA.asUInt + val canWriteDmode = WireInit(false.B) + canWriteDmode := (if (idx > 0)(Mux( + mcontrol6Pre.CHAIN.asBool, + tdata1Pre.DMODE.asBool && tdata1Pre.TYPE.isLegal, + true.B + )) && debugMode + else debugMode).asBool + + val tdata1Next = Wire(new Tdata1Bundle) + tdata1Next := (if (idx < TriggerNum - 1) tdata1RegVec(idx + 1) else tdata1RegVec(idx)).rdata.asUInt + val dmodeNextTrigger = WireInit(false.B) + dmodeNextTrigger := (if (idx < TriggerNum - 1) tdata1Next.TYPE.isLegal && tdata1Next.DMODE.asBool else false.B) + + tdata1RegVec(idx) match { + case m: HasTriggerBundle => + m.canWriteDmode := canWriteDmode + m.chainable := debugMod.io.out.newTriggerChainIsLegal + m.dmodeNextTrigger := dmodeNextTrigger + case _ => + } + } + + tdata1RegVec.zip(tdata2RegVec).zipWithIndex.map { + case ((mod1, mod2), idx) => { + mod1.w.wen := tdata1Update && (tselect.rdata === idx.U) + mod1.w.wdata := wdata + mod2.w.wen := tdata2Update && (tselect.rdata === idx.U) + mod2.w.wdata := wdata + } + } + + triggerFrontendChange := debugMod.io.out.triggerFrontendChange + + io.status.frontendTrigger := debugMod.io.out.frontendTrigger + io.status.memTrigger := debugMod.io.out.memTrigger + + mcontext.fromHcontext <> hcontext.toMcontext + hcontext.fromMcontext <> mcontext.toHcontext + + /** + * debug_end + */ + + // trace + val privForTrace = Mux( + debugMode, + Priv.D, + Mux1H( + Seq(privState.isModeM, privState.isModeHS, privState.isModeVS, privState.isModeHU, privState.isModeVU), + Seq(Priv.M, Priv.HS, Priv.VS, Priv.HU, Priv.VU) + ) + ) + + val xret = legalDret || legalMNret || legalMret || legalSret + val currentPriv = privForTrace + val lastPriv = RegEnable(privForTrace, Priv.M, (xret || io.fromRob.trap.valid)) + + io.status.traceCSR.lastPriv := lastPriv + io.status.traceCSR.currentPriv := privForTrace + io.status.traceCSR.cause := Mux1H( + Seq(privState.isModeM, privState.isModeHS, privState.isModeVS), + Seq(mcause.rdata, scause.rdata, vscause.rdata) + ) + io.status.traceCSR.tval := Mux1H( + Seq(privState.isModeM, privState.isModeHS, privState.isModeVS), + Seq(mtval.rdata, stval.rdata, vstval.rdata) + ) + io.status.traceCSR.mstatus := mstatus.regOut.asUInt + + /** + * perf_begin + * perf number: 29 (frontend 8, ctrlblock 8, memblock 8, huancun 5) + */ + val csrevents = mhpmevents.slice(24, 29).map(_.rdata) + + val hcEvents = Wire(Vec(numPCntHc * coreParams.L2NBanks, new PerfEvent)) + for (i <- 0 until numPCntHc * coreParams.L2NBanks) { + hcEvents(i) := io.perf.perfEventsHc(i) + } + + val hpmHc = HPerfMonitor(csrevents, hcEvents) + + val allPerfEvents = io.perf.perfEventsFrontend ++ + io.perf.perfEventsBackend ++ + io.perf.perfEventsLsu ++ + hpmHc.getPerf + + val countingEn = RegInit(0.U.asTypeOf(Vec(perfCntNum, Bool()))) + val ofFromPerfCntVec = Wire(Vec(perfCntNum, Bool())) + val lcofiReqVec = Wire(Vec(perfCntNum, Bool())) + + for (i <- 0 until perfCntNum) { + mhpmcounters(i) match { + case m: HasPerfCounterBundle => + m.countingEn := countingEn(i) + m.perf := allPerfEvents(i) + ofFromPerfCntVec(i) := m.toMhpmeventOF + case _ => + } + + mhpmevents(i) match { + case m: HasOfFromPerfCntBundle => + m.ofFromPerfCnt := ofFromPerfCntVec(i) + case _ => + } + + val mhpmevent = Wire(new MhpmeventBundle) + mhpmevent := mhpmevents(i).rdata + lcofiReqVec(i) := ofFromPerfCntVec(i) && !mhpmevent.OF.asBool + + countingEn(i) := (privState.isModeM && !mhpmevent.MINH) || + (privState.isModeHS && !mhpmevent.SINH) || + (privState.isModeHU && !mhpmevent.UINH) || + (privState.isModeVS && !mhpmevent.VSINH) || + (privState.isModeVU && !mhpmevent.VUINH) + } + + val lcofiReq = lcofiReqVec.asUInt.orR + mip match { + case m: HasLocalInterruptReqBundle => + m.lcofiReq := lcofiReq + case _ => + } + + /** + * perf_end + */ + + /** + * [[io.status.custom]] connection + */ + io.status.custom.pf_ctrl.l1I_pf_enable := spfctl.regOut.L1I_PF_ENABLE.asBool + io.status.custom.pf_ctrl.l2_pf_enable := spfctl.regOut.L2_PF_ENABLE.asBool + io.status.custom.pf_ctrl.l1D_pf_enable := spfctl.regOut.L1D_PF_ENABLE.asBool + io.status.custom.pf_ctrl.l1D_pf_train_on_hit := spfctl.regOut.L1D_PF_TRAIN_ON_HIT.asBool + io.status.custom.pf_ctrl.l1D_pf_enable_agt := spfctl.regOut.L1D_PF_ENABLE_AGT.asBool + io.status.custom.pf_ctrl.l1D_pf_enable_pht := spfctl.regOut.L1D_PF_ENABLE_PHT.asBool + io.status.custom.pf_ctrl.l1D_pf_active_threshold := spfctl.regOut.L1D_PF_ACTIVE_THRESHOLD.asUInt + io.status.custom.pf_ctrl.l1D_pf_active_stride := spfctl.regOut.L1D_PF_ACTIVE_STRIDE.asUInt + io.status.custom.pf_ctrl.l1D_pf_enable_stride := spfctl.regOut.L1D_PF_ENABLE_STRIDE.asBool + io.status.custom.pf_ctrl.l2_pf_store_only := spfctl.regOut.L2_PF_STORE_ONLY.asBool + io.status.custom.pf_ctrl.l2_pf_recv_enable := spfctl.regOut.L2_PF_RECV_ENABLE.asBool + io.status.custom.pf_ctrl.l2_pf_pbop_enable := spfctl.regOut.L2_PF_PBOP_ENABLE.asBool + io.status.custom.pf_ctrl.l2_pf_vbop_enable := spfctl.regOut.L2_PF_VBOP_ENABLE.asBool + io.status.custom.pf_ctrl.l2_pf_tp_enable := spfctl.regOut.L2_PF_TP_ENABLE.asBool + io.status.custom.pf_ctrl.l2_pf_delay_latency := spfctl.regOut.L2_PF_DELAY_LATENCY.asUInt + + io.status.custom.lvpred_disable := slvpredctl.regOut.LVPRED_DISABLE.asBool + io.status.custom.no_spec_load := slvpredctl.regOut.NO_SPEC_LOAD.asBool + io.status.custom.storeset_wait_store := slvpredctl.regOut.STORESET_WAIT_STORE.asBool + io.status.custom.storeset_no_fast_wakeup := slvpredctl.regOut.STORESET_NO_FAST_WAKEUP.asBool + io.status.custom.lvpred_timeout := slvpredctl.regOut.LVPRED_TIMEOUT.asUInt + + io.status.custom.bp_ctrl.ubtb_enable := sbpctl.regOut.UBTB_ENABLE.asBool + io.status.custom.bp_ctrl.btb_enable := sbpctl.regOut.BTB_ENABLE.asBool + io.status.custom.bp_ctrl.bim_enable := sbpctl.regOut.BIM_ENABLE.asBool + io.status.custom.bp_ctrl.tage_enable := sbpctl.regOut.TAGE_ENABLE.asBool + io.status.custom.bp_ctrl.sc_enable := sbpctl.regOut.SC_ENABLE.asBool + io.status.custom.bp_ctrl.ras_enable := sbpctl.regOut.RAS_ENABLE.asBool + io.status.custom.bp_ctrl.loop_enable := sbpctl.regOut.LOOP_ENABLE.asBool + + io.status.custom.sbuffer_timeout := smblockctl.regOut.SBUFFER_TIMEOUT.asUInt + io.status.custom.sbuffer_threshold := smblockctl.regOut.SBUFFER_THRESHOLD.asUInt + io.status.custom.ldld_vio_check_enable := smblockctl.regOut.LDLD_VIO_CHECK_ENABLE.asBool + io.status.custom.soft_prefetch_enable := smblockctl.regOut.SOFT_PREFETCH_ENABLE.asBool + io.status.custom.cache_error_enable := smblockctl.regOut.CACHE_ERROR_ENABLE.asBool + io.status.custom.uncache_write_outstanding_enable := smblockctl.regOut.UNCACHE_WRITE_OUTSTANDING_ENABLE.asBool + io.status.custom.hd_misalign_st_enable := smblockctl.regOut.HD_MISALIGN_ST_ENABLE.asBool + io.status.custom.hd_misalign_ld_enable := smblockctl.regOut.HD_MISALIGN_LD_ENABLE.asBool + + io.status.custom.fusion_enable := srnctl.regOut.FUSION_ENABLE.asBool + io.status.custom.wfi_enable := srnctl.regOut.WFI_ENABLE.asBool && (!io.status.singleStepFlag) && !debugMode + + io.status.custom.power_down_enable := mcorepwr.regOut.POWER_DOWN_ENABLE.asBool + io.status.custom.commit_stuck_check_enable := mcorepwr.regOut.COMMIT_STUCK_CHECK_ENABLE.asBool + + io.status.custom.flush_l2_enable := mflushpwr.regOut.FLUSH_L2_ENABLE.asBool + + io.status.instrAddrTransType.bare := privState.isModeM || + (!privState.isVirtual && satp.regOut.MODE === SatpMode.Bare) || + (privState.isVirtual && vsatp.regOut.MODE === SatpMode.Bare && hgatp.regOut.MODE === HgatpMode.Bare) + io.status.instrAddrTransType.sv39 := !privState.isModeM && !privState.isVirtual && satp.regOut.MODE === SatpMode.Sv39 || + privState.isVirtual && vsatp.regOut.MODE === SatpMode.Sv39 + io.status.instrAddrTransType.sv48 := !privState.isModeM && !privState.isVirtual && satp.regOut.MODE === SatpMode.Sv48 || + privState.isVirtual && vsatp.regOut.MODE === SatpMode.Sv48 + io.status.instrAddrTransType.sv39x4 := privState.isVirtual && vsatp.regOut.MODE === SatpMode.Bare && hgatp.regOut.MODE === HgatpMode.Sv39x4 + io.status.instrAddrTransType.sv48x4 := privState.isVirtual && vsatp.regOut.MODE === SatpMode.Bare && hgatp.regOut.MODE === HgatpMode.Sv48x4 + assert(PopCount(io.status.instrAddrTransType.asUInt) === 1.U, "Exactly one inst trans type should be asserted") + + private val csrAccess = wenLegalReg || RegNext(ren) + + private val imsicAddrValid = + csrAccess && addr === CSRs.mireg.U && miselect.inIMSICRange || + csrAccess && addr === CSRs.sireg.U && !isModeVS && siselect.inIMSICRange || + csrAccess && (addr === CSRs.sireg.U && isModeVS || addr === CSRs.vsireg.U) && vsiselect.inIMSICRange + + private val imsicAddr = Mux1H(Seq( + (csrAccess && addr === CSRs.mireg.U) -> miselect.rdata, + (csrAccess && addr === CSRs.sireg.U && !isModeVS) -> siselect.rdata, + (csrAccess && (addr === CSRs.sireg.U && isModeVS || addr === CSRs.vsireg.U)) -> vsiselect.rdata + )) + + private val imsicAddrPrivState = Mux1H(Seq( + (csrAccess && addr === CSRs.mireg.U) -> PrivState.ModeM, + (csrAccess && addr === CSRs.sireg.U && !isModeVS) -> PrivState.ModeHS, + (csrAccess && (addr === CSRs.sireg.U && isModeVS || addr === CSRs.vsireg.U)) -> PrivState.ModeVS + )) + + private val imsicWdataValid = + mireg.w.wen && miselect.inIMSICRange || + sireg.w.wen && siselect.inIMSICRange || + vsireg.w.wen && vsiselect.inIMSICRange + + toAIA.addr.valid := imsicAddrValid + toAIA.addr.bits.addr := imsicAddr + toAIA.addr.bits.prvm := imsicAddrPrivState.PRVM + toAIA.addr.bits.v := imsicAddrPrivState.V + + toAIA.wdata.valid := imsicWdataValid + toAIA.wdata.bits.op := RegNext(io.in.bits.op) + toAIA.wdata.bits.data := RegNext(io.in.bits.src) + toAIA.vgein := hstatus.regOut.VGEIN.asUInt + toAIA.mClaim := mtopei.w.wen + toAIA.sClaim := stopei.w.wen + toAIA.vsClaim := vstopei.w.wen + + // tlb + io.tlb.satpASIDChanged := GatedValidRegNext(satp.w.wen && satp.regOut.ASID =/= satp.w.wdataFields.ASID) + io.tlb.vsatpASIDChanged := GatedValidRegNext(vsatp.w.wen && vsatp.regOut.ASID =/= vsatp.w.wdataFields.ASID) + io.tlb.hgatpVMIDChanged := GatedValidRegNext(hgatp.w.wen && hgatp.regOut.VMID =/= hgatp.w.wdataFields.VMID) + io.tlb.satp := satp.rdata + io.tlb.vsatp := vsatp.rdata + io.tlb.hgatp := hgatp.rdata + if (HasBitmapCheck) { + io.tlb.mbmc := mbmc.get.rdata + } else { + io.tlb.mbmc := DontCare + } + io.tlb.mxr := mstatus.regOut.MXR.asBool + io.tlb.sum := mstatus.regOut.SUM.asBool + io.tlb.vmxr := vsstatus.regOut.MXR.asBool + io.tlb.vsum := vsstatus.regOut.SUM.asBool + io.tlb.spvp := hstatus.regOut.SPVP.asBool + + io.tlb.imode := PRVM.asUInt + // when NMIE is zero, force to behave as MPRV is zero + io.tlb.dmode := Mux( + (debugMode && dcsr.regOut.MPRVEN || !debugMode) && mstatus.regOut.MPRV && mnstatus.regOut.NMIE, + mstatus.regOut.MPP.asUInt, + PRVM.asUInt + ) + io.tlb.dvirt := Mux( + (debugMode && dcsr.regOut.MPRVEN || !debugMode) && mstatus.regOut.MPRV && mnstatus.regOut.NMIE && mstatus.regOut.MPP =/= PrivMode.M, + mstatus.regOut.MPV.asUInt, + V.asUInt + ) + io.tlb.debug := debugMode + io.tlb.mPBMTE := RegNext(menvcfg.regOut.PBMTE.asBool) + io.tlb.hPBMTE := RegNext(henvcfg.regOut.PBMTE.asBool) + io.tlb.pmm.mseccfg := RegNext(mseccfg.regOut.PMM.asUInt) + io.tlb.pmm.menvcfg := RegNext(menvcfg.regOut.PMM.asUInt) + io.tlb.pmm.henvcfg := RegNext(henvcfg.regOut.PMM.asUInt) + io.tlb.pmm.hstatus := RegNext(hstatus.regOut.HUPMM.asUInt) + io.tlb.pmm.senvcfg := RegNext(senvcfg.regOut.PMM.asUInt) + + io.toDecode.illegalInst.sfenceVMA := isModeHS && mstatus.regOut.TVM || isModeHU + io.toDecode.virtualInst.sfenceVMA := isModeVS && hstatus.regOut.VTVM || isModeVU + io.toDecode.illegalInst.sfencePart := isModeHU + io.toDecode.virtualInst.sfencePart := isModeVU + io.toDecode.illegalInst.hfenceGVMA := isModeHS && mstatus.regOut.TVM || isModeHU + io.toDecode.illegalInst.hfenceVVMA := isModeHU + io.toDecode.virtualInst.hfence := isModeVS || isModeVU + io.toDecode.illegalInst.hlsv := isModeHU && !hstatus.regOut.HU + io.toDecode.virtualInst.hlsv := isModeVS || isModeVU + io.toDecode.illegalInst.fsIsOff := mstatus.regOut.FS === ContextStatus.Off || (isModeVS || isModeVU) && vsstatus.regOut.FS === ContextStatus.Off + io.toDecode.illegalInst.vsIsOff := mstatus.regOut.VS === ContextStatus.Off || (isModeVS || isModeVU) && vsstatus.regOut.VS === ContextStatus.Off + io.toDecode.illegalInst.wfi := isModeHU || !isModeM && mstatus.regOut.TW + io.toDecode.virtualInst.wfi := isModeVS && !mstatus.regOut.TW && hstatus.regOut.VTW || isModeVU && !mstatus.regOut.TW + io.toDecode.illegalInst.wrs_nto := !isModeM && mstatus.regOut.TW + io.toDecode.virtualInst.wrs_nto := privState.V && !mstatus.regOut.TW && hstatus.regOut.VTW + io.toDecode.illegalInst.frm := frmIsReserved + // Ref: The RISC-V Instruction Set Manual Volume I - 20.5. Control and Status Register State + io.toDecode.illegalInst.cboZ := !isModeM && !menvcfg.regOut.CBZE || isModeHU && !senvcfg.regOut.CBZE + io.toDecode.virtualInst.cboZ := menvcfg.regOut.CBZE && ( + isModeVS && !henvcfg.regOut.CBZE || + isModeVU && !(henvcfg.regOut.CBZE && senvcfg.regOut.CBZE) + ) + io.toDecode.illegalInst.cboCF := !isModeM && !menvcfg.regOut.CBCFE || isModeHU && !senvcfg.regOut.CBCFE + io.toDecode.virtualInst.cboCF := menvcfg.regOut.CBCFE && ( + isModeVS && !henvcfg.regOut.CBCFE || + isModeVU && !(henvcfg.regOut.CBCFE && senvcfg.regOut.CBCFE) + ) + io.toDecode.illegalInst.cboI := + !isModeM && menvcfg.regOut.CBIE === EnvCBIE.Off || + isModeHU && senvcfg.regOut.CBIE === EnvCBIE.Off + io.toDecode.virtualInst.cboI := menvcfg.regOut.CBIE =/= EnvCBIE.Off && ( + isModeVS && henvcfg.regOut.CBIE === EnvCBIE.Off || + isModeVU && (henvcfg.regOut.CBIE === EnvCBIE.Off || senvcfg.regOut.CBIE === EnvCBIE.Off) + ) + io.toDecode.special.cboI2F := !io.toDecode.illegalInst.cboI && !io.toDecode.virtualInst.cboI && ( + menvcfg.regOut.CBIE === EnvCBIE.Flush && !isModeM || + senvcfg.regOut.CBIE === EnvCBIE.Flush && (isModeHU || isModeVU) || + henvcfg.regOut.CBIE === EnvCBIE.Flush && (isModeVS || isModeVU) + ) + + io.distributedWenLegal := wenLegalReg && !noCSRIllegalReg + io.status.criticalErrorState := criticalErrorState && !dcsr.regOut.CETRIG.asBool + + val criticalErrors = Seq( + ("csr_dbltrp_inMN", !mnstatus.regOut.NMIE && hasTrap && !(entryDebugMode || debugMode)) + ) + + criticalErrorStateInCSR := criticalErrors.map(criticalError => criticalError._2).reduce(_ || _).asBool + generateCriticalErrors() + + // --------------------------------------------------------------------------- + // CSR documentation export + // --------------------------------------------------------------------------- + + if (env.DumpCSR) { + dumpCSRDoc() + } + + // --------------------------------------------------------------------------- + // Difftest + // --------------------------------------------------------------------------- + + // Always instantiate basic difftest modules. + if (env.AlwaysBasicDiff || env.EnableDifftest) { + // Delay trap passed to difftest until VecExcpMod is not busy + val pendingTrap = RegInit(false.B) + when(hasTrap) { + pendingTrap := true.B + }.elsewhen(!io.fromVecExcpMod.busy) { + pendingTrap := false.B + } + + val hartId = io.fromTop.hartId + val trapValid = pendingTrap && !io.fromVecExcpMod.busy + val interrupt = trapHandleMod.io.out.causeNO.Interrupt.asBool + val trapNO = Mux( + virtualInterruptIsHvictlInject && interrupt, + hvictl.regOut.IID.asUInt, + trapHandleMod.io.out.causeNO.ExceptionCode.asUInt + ) + val hasNMI = nmi && hasTrap + val interruptNO = Mux(interrupt, trapNO, 0.U) + val exceptionNO = Mux(!interrupt, trapNO, 0.U) + val isSv39: Bool = + (isModeHS || isModeHU) && satp.regOut.MODE === SatpMode.Sv39 || + (isModeVS || isModeVU) && vsatp.regOut.MODE === SatpMode.Sv39 + val isSv48: Bool = + (isModeHS || isModeHU) && satp.regOut.MODE === SatpMode.Sv48 || + (isModeVS || isModeVU) && vsatp.regOut.MODE === SatpMode.Sv48 + val isBare = !isSv39 && !isSv48 + val sv39PC = SignExt(trapPC.take(39), XLEN) + val sv48PC = SignExt(trapPC.take(48), XLEN) + val barePC = ZeroExt(trapPC.take(PAddrBits), XLEN) + // When enable virtual memory, the higher bit should fill with the msb of address of Sv39/Sv48/Sv57 + val exceptionPC = Mux1H(Seq( + isSv39 -> sv39PC, + isSv48 -> sv48PC, + isBare -> barePC + )) + + val diffArchEvent = DifftestModule(new DiffArchEvent, delay = 3, dontCare = true) + diffArchEvent.coreid := hartId + diffArchEvent.valid := trapValid + diffArchEvent.interrupt := RegEnable(interruptNO, hasTrap) + diffArchEvent.exception := RegEnable(exceptionNO, hasTrap) + diffArchEvent.exceptionPC := RegEnable(exceptionPC, hasTrap) + diffArchEvent.hasNMI := RegEnable(hasNMI, hasTrap) + diffArchEvent.virtualInterruptIsHvictlInject := RegNext(virtualInterruptIsHvictlInject && interrupt) + diffArchEvent.irToHS := RegEnable(irToHS, hasTrap) + diffArchEvent.irToVS := RegEnable(irToVS, hasTrap) + if (env.EnableDifftest || env.FullBasicDiff) { + diffArchEvent.exceptionInst := RegEnable(io.fromRob.trap.bits.instr, hasTrap) + } + + val diffCriticalErrorEvent = DifftestModule(new DiffCriticalErrorEvent, delay = 4, dontCare = true) + diffCriticalErrorEvent.valid := io.status.criticalErrorState && trapValid + diffCriticalErrorEvent.coreid := hartId + diffCriticalErrorEvent.criticalError := io.status.criticalErrorState + + val diffCSRState = DifftestModule(new DiffCSRState) + diffCSRState.coreid := hartId + diffCSRState.privilegeMode := privState.PRVM.asUInt + diffCSRState.mstatus := mstatus.rdata.asUInt + diffCSRState.sstatus := mstatus.sstatus.asUInt + diffCSRState.mepc := mepc.rdata.asUInt + diffCSRState.sepc := sepc.rdata.asUInt + diffCSRState.mtval := mtval.rdata.asUInt + diffCSRState.stval := stval.rdata.asUInt + diffCSRState.mtvec := mtvec.rdata.asUInt + diffCSRState.stvec := stvec.rdata.asUInt + diffCSRState.mcause := mcause.rdata.asUInt + diffCSRState.scause := scause.rdata.asUInt + diffCSRState.satp := satp.rdata.asUInt + diffCSRState.mip := mip.rdata.asUInt + diffCSRState.mie := mie.rdata.asUInt + diffCSRState.mscratch := mscratch.rdata.asUInt + diffCSRState.sscratch := sscratch.rdata.asUInt + diffCSRState.mideleg := mideleg.rdata.asUInt + diffCSRState.medeleg := medeleg.rdata.asUInt + + val diffDebugMode = DifftestModule(new DiffDebugMode) + diffDebugMode.coreid := hartId + diffDebugMode.debugMode := debugMode + diffDebugMode.dcsr := dcsr.rdata.asUInt + diffDebugMode.dpc := dpc.rdata.asUInt + diffDebugMode.dscratch0 := dscratch0.rdata.asUInt + diffDebugMode.dscratch1 := dscratch1.rdata.asUInt + + val diffTriggerCSRState = DifftestModule(new DiffTriggerCSRState) + diffTriggerCSRState.coreid := hartId + diffTriggerCSRState.tselect := tselect.rdata + diffTriggerCSRState.tdata1 := tdata1.rdata + diffTriggerCSRState.tinfo := tinfo.rdata + + val diffVecCSRState = DifftestModule(new DiffVecCSRState) + diffVecCSRState.coreid := hartId + diffVecCSRState.vstart := vstart.rdata.asUInt + diffVecCSRState.vxsat := vcsr.vxsat.asUInt + diffVecCSRState.vxrm := vcsr.vxrm.asUInt + diffVecCSRState.vcsr := vcsr.rdata.asUInt + diffVecCSRState.vl := RegNext(io.fromRob.commit.vl) + diffVecCSRState.vtype := vtype.rdata.asUInt + diffVecCSRState.vlenb := vlenb.rdata.asUInt + + val diffFpCSRState = DifftestModule(new DiffFpCSRState) + diffFpCSRState.coreid := hartId + diffFpCSRState.fcsr := fcsr.rdata.asUInt + + val diffHCSRState = DifftestModule(new DiffHCSRState) + diffHCSRState.coreid := hartId + diffHCSRState.virtMode := privState.V.asBool + diffHCSRState.mtval2 := mtval2.rdata.asUInt + diffHCSRState.mtinst := mtinst.rdata.asUInt + diffHCSRState.hstatus := hstatus.rdata.asUInt + diffHCSRState.hideleg := hideleg.rdata.asUInt + diffHCSRState.hedeleg := hedeleg.rdata.asUInt + diffHCSRState.hcounteren := hcounteren.rdata.asUInt + diffHCSRState.htval := htval.rdata.asUInt + diffHCSRState.htinst := htinst.rdata.asUInt + diffHCSRState.hgatp := hgatp.rdata.asUInt + diffHCSRState.vsstatus := vsstatus.rdata.asUInt + diffHCSRState.vstvec := vstvec.rdata.asUInt + diffHCSRState.vsepc := vsepc.rdata.asUInt + diffHCSRState.vscause := vscause.rdata.asUInt + diffHCSRState.vstval := vstval.rdata.asUInt + diffHCSRState.vsatp := vsatp.rdata.asUInt + diffHCSRState.vsscratch := vsscratch.rdata.asUInt + + val platformIRPMeipChange = + !platformIRP.MEIP && RegNext(platformIRP.MEIP) || platformIRP.MEIP && !RegNext(platformIRP.MEIP) + val platformIRPMtipChange = + !platformIRP.MTIP && RegNext(platformIRP.MTIP) || platformIRP.MTIP && !RegNext(platformIRP.MTIP) + val platformIRPMsipChange = + !platformIRP.MSIP && RegNext(platformIRP.MSIP) || platformIRP.MSIP && !RegNext(platformIRP.MSIP) + val platformIRPSeipChange = + !platformIRP.SEIP && RegNext(platformIRP.SEIP) || platformIRP.SEIP && !RegNext(platformIRP.SEIP) + val platformIRPStipChange = + !sstcIRGen.o.STIP && RegNext(sstcIRGen.o.STIP) || sstcIRGen.o.STIP && !RegNext(sstcIRGen.o.STIP) + val platformIRPVseipChange = !platformIRP.VSEIP && RegNext(platformIRP.VSEIP) || + platformIRP.VSEIP && !RegNext(platformIRP.VSEIP) || + !hgeip.rdata.asUInt(hstatus.regOut.VGEIN.asUInt) && RegNext(hgeip.rdata.asUInt(hstatus.regOut.VGEIN.asUInt)) || + hgeip.rdata.asUInt(hstatus.regOut.VGEIN.asUInt) && !RegNext(hgeip.rdata.asUInt(hstatus.regOut.VGEIN.asUInt)) + val platformIRPVstipChange = + !sstcIRGen.o.VSTIP && RegNext(sstcIRGen.o.VSTIP) || sstcIRGen.o.VSTIP && !RegNext(sstcIRGen.o.VSTIP) + val fromAIAMeipChange = !fromAIA.meip && RegNext(fromAIA.meip) || fromAIA.meip && !RegNext(fromAIA.meip) + val fromAIASeipChange = !fromAIA.seip && RegNext(fromAIA.seip) || fromAIA.seip && !RegNext(fromAIA.seip) + val lcofiReqChange = !lcofiReq && RegNext(lcofiReq) || lcofiReq && !RegNext(lcofiReq) + + val diffNonRegInterruptPendingEvent = DifftestModule(new DiffNonRegInterruptPendingEvent) + diffNonRegInterruptPendingEvent.coreid := hartId + diffNonRegInterruptPendingEvent.valid := (platformIRPMeipChange || platformIRPMtipChange || platformIRPMsipChange || + platformIRPSeipChange || platformIRPStipChange || + platformIRPVseipChange || platformIRPVstipChange || + fromAIAMeipChange || fromAIASeipChange || + lcofiReqChange || RegNext(reset.asBool)) & !reset.asBool + diffNonRegInterruptPendingEvent.platformIRPMeip := platformIRP.MEIP + diffNonRegInterruptPendingEvent.platformIRPMtip := platformIRP.MTIP + diffNonRegInterruptPendingEvent.platformIRPMsip := platformIRP.MSIP + diffNonRegInterruptPendingEvent.platformIRPSeip := platformIRP.SEIP + diffNonRegInterruptPendingEvent.platformIRPStip := sstcIRGen.o.STIP + diffNonRegInterruptPendingEvent.platformIRPVseip := platformIRP.VSEIP || hgeip.rdata.asUInt( + hstatus.regOut.VGEIN.asUInt + ) + diffNonRegInterruptPendingEvent.platformIRPVstip := sstcIRGen.o.VSTIP + diffNonRegInterruptPendingEvent.fromAIAMeip := fromAIA.meip + diffNonRegInterruptPendingEvent.fromAIASeip := fromAIA.seip + diffNonRegInterruptPendingEvent.localCounterOverflowInterruptReq := mip.regOut.LCOFIP.asBool + + val diffMhpmeventOverflowEvent = DifftestModule(new DiffMhpmeventOverflowEvent) + diffMhpmeventOverflowEvent.coreid := hartId + diffMhpmeventOverflowEvent.valid := Cat(mhpmevents.zipWithIndex.map { case (event, i) => + !ofFromPerfCntVec(i) && RegNext(ofFromPerfCntVec(i)) || ofFromPerfCntVec(i) && !RegNext(ofFromPerfCntVec(i)) + }).orR + diffMhpmeventOverflowEvent.mhpmeventOverflow := VecInit( + mhpmevents.map(_.regOut.asInstanceOf[MhpmeventBundle].OF.asBool) + ).asUInt + + val mtopeiChange = RegNext(fromAIA.mtopei.asUInt) =/= fromAIA.mtopei.asUInt + val stopeiChange = RegNext(fromAIA.stopei.asUInt) =/= fromAIA.stopei.asUInt + val vstopeiChange = RegNext(hstatus.regOut.VGEIN.asUInt) =/= hstatus.regOut.VGEIN.asUInt + val hgeipChange = RegNext(fromAIA.vseip) =/= fromAIA.vseip + + val diffSyncAIAEvent = DifftestModule(new DiffSyncAIAEvent) + diffSyncAIAEvent.coreid := hartId + diffSyncAIAEvent.valid := mtopeiChange || stopeiChange || vstopeiChange || hgeipChange + diffSyncAIAEvent.mtopei := mtopei.rdata + diffSyncAIAEvent.stopei := stopei.rdata + diffSyncAIAEvent.vstopei := vstopei.rdata + diffSyncAIAEvent.hgeip := hgeip.rdata + + val diffCustomMflushpwr = DifftestModule(new DiffSyncCustomMflushpwrEvent) + diffCustomMflushpwr.coreid := hartId + diffCustomMflushpwr.valid := RegNext(io.fromTop.l2FlushDone) =/= io.fromTop.l2FlushDone + diffCustomMflushpwr.l2FlushDone := io.fromTop.l2FlushDone + } +} + +trait IpIeAliasConnect { + self: NewCSR with MachineLevel with SupervisorLevel with VirtualSupervisorLevel with HypervisorLevel => + + mip.fromMvip := mvip.toMip + mip.fromSip := sip.toMip + mip.fromVSip := vsip.toMip + mvip.fromMip := mip.toMvip + mvip.fromSip := sip.toMvip + mvip.fromVSip := vsip.toMvip + hvip.fromMip := mip.toHvip + hvip.fromHip := hip.toHvip + hvip.fromVSip := vsip.toHvip + + mie.fromHie := hie.toMie + mie.fromSie := sie.toMie + mie.fromVSie := vsie.toMie + sie.fromVSie := vsie.toSie +} + +object NewCSRMain extends App { + + val (config, firrtlOpts, firtoolOpts) = ArgParser.parse( + args :+ "--disable-always-basic-diff" :+ "--dump-fir" :+ "--fpga-platform" :+ "--target" :+ "verilog" + ) + + val defaultConfig = config.alterPartial({ + // Get XSCoreParams and pass it to the "small module" + case XSCoreParamsKey => config(XSTileKey).head + }) + + Generator.execute( + firrtlOpts :+ "--full-stacktrace" :+ "--target-dir" :+ "backend", + new NewCSR()(defaultConfig), + firtoolOpts + ) + + println("done") +} diff --git a/examples/components/cores/xiangshan/configs/Params.scala b/examples/prefab/cores/xiangshan/configs/Params.scala similarity index 100% rename from examples/components/cores/xiangshan/configs/Params.scala rename to examples/prefab/cores/xiangshan/configs/Params.scala diff --git a/examples/components/cores/xiangshan/configs/xiangshan_params.proto b/examples/prefab/cores/xiangshan/configs/xiangshan_params.proto similarity index 100% rename from examples/components/cores/xiangshan/configs/xiangshan_params.proto rename to examples/prefab/cores/xiangshan/configs/xiangshan_params.proto diff --git a/examples/components/cores/xiangshan/stack/CMakeLists.txt b/examples/prefab/cores/xiangshan/stack/CMakeLists.txt similarity index 89% rename from examples/components/cores/xiangshan/stack/CMakeLists.txt rename to examples/prefab/cores/xiangshan/stack/CMakeLists.txt index bb4434ab2..ed2655a76 100644 --- a/examples/components/cores/xiangshan/stack/CMakeLists.txt +++ b/examples/prefab/cores/xiangshan/stack/CMakeLists.txt @@ -17,7 +17,7 @@ find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) add_custom_command( OUTPUT execute.o COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 - "-DCORE_SIGNATURE=${DESIGN_CORE_SIGNATURE_${CORE}}ULL" "-I${RUNTIME}/scheduler" + "-DCORE_SIGNATURE=${DESIGN_DEVICE_CORES_${DESIGN_CORE_HART_${CORE}}_SIGNATURE}ULL" "-I${RUNTIME}/scheduler" -c "${RUNTIME}/execute/direct.c" -o execute.o DEPENDS "${RUNTIME}/execute/direct.c" "${RUNTIME}/scheduler/runtime.h" VERBATIM) diff --git a/examples/components/cores/xiangshan/thirdparty/xiangshan b/examples/prefab/cores/xiangshan/thirdparty/xiangshan similarity index 100% rename from examples/components/cores/xiangshan/thirdparty/xiangshan rename to examples/prefab/cores/xiangshan/thirdparty/xiangshan diff --git a/examples/components/cores/xiangshan/workloads/ctests/CMakeLists.txt b/examples/prefab/cores/xiangshan/workloads/ctests/CMakeLists.txt similarity index 100% rename from examples/components/cores/xiangshan/workloads/ctests/CMakeLists.txt rename to examples/prefab/cores/xiangshan/workloads/ctests/CMakeLists.txt diff --git a/examples/components/cores/xiangshan/workloads/ctests/memory_matrix.c b/examples/prefab/cores/xiangshan/workloads/ctests/memory_matrix.c similarity index 100% rename from examples/components/cores/xiangshan/workloads/ctests/memory_matrix.c rename to examples/prefab/cores/xiangshan/workloads/ctests/memory_matrix.c diff --git a/examples/components/threads/attention/arch/AttentionThread.scala b/examples/prefab/threads/attention/arch/AttentionThread.scala similarity index 100% rename from examples/components/threads/attention/arch/AttentionThread.scala rename to examples/prefab/threads/attention/arch/AttentionThread.scala diff --git a/examples/components/threads/attention/arch/acc.scala b/examples/prefab/threads/attention/arch/acc.scala similarity index 100% rename from examples/components/threads/attention/arch/acc.scala rename to examples/prefab/threads/attention/arch/acc.scala diff --git a/examples/components/threads/attention/configs/balldomains/default.toml b/examples/prefab/threads/attention/configs/balldomains/default.toml similarity index 100% rename from examples/components/threads/attention/configs/balldomains/default.toml rename to examples/prefab/threads/attention/configs/balldomains/default.toml diff --git a/examples/components/threads/attention/configs/cpu/default.toml b/examples/prefab/threads/attention/configs/cpu/default.toml similarity index 100% rename from examples/components/threads/attention/configs/cpu/default.toml rename to examples/prefab/threads/attention/configs/cpu/default.toml diff --git a/examples/components/threads/attention/configs/default.toml b/examples/prefab/threads/attention/configs/default.toml similarity index 100% rename from examples/components/threads/attention/configs/default.toml rename to examples/prefab/threads/attention/configs/default.toml diff --git a/examples/components/threads/attention/configs/frontends/default.toml b/examples/prefab/threads/attention/configs/frontends/default.toml similarity index 74% rename from examples/components/threads/attention/configs/frontends/default.toml rename to examples/prefab/threads/attention/configs/frontends/default.toml index 5c58b1cb2..0f78452a5 100644 --- a/examples/components/threads/attention/configs/frontends/default.toml +++ b/examples/prefab/threads/attention/configs/frontends/default.toml @@ -2,8 +2,8 @@ bankAliasEnable = true robEntries = 16 rsOutOfOrderResponse = true bankIdLen = 10 -vbankIdUpperBound = 31 -sharedBankIdBase = 32 +maxPrivateVbank = 31 +sharedVbankBase = 32 iterLen = 34 subRobEnable = false subRobDepth = 64 diff --git a/examples/components/threads/attention/configs/gpdomain/default.toml b/examples/prefab/threads/attention/configs/gpdomain/default.toml similarity index 100% rename from examples/components/threads/attention/configs/gpdomain/default.toml rename to examples/prefab/threads/attention/configs/gpdomain/default.toml diff --git a/examples/components/threads/attention/configs/memdomains/default.toml b/examples/prefab/threads/attention/configs/memdomains/default.toml similarity index 98% rename from examples/components/threads/attention/configs/memdomains/default.toml rename to examples/prefab/threads/attention/configs/memdomains/default.toml index 0f187340e..bca02f7ec 100644 --- a/examples/components/threads/attention/configs/memdomains/default.toml +++ b/examples/prefab/threads/attention/configs/memdomains/default.toml @@ -1,6 +1,8 @@ # Attention Thread Memory Domain Configuration (default) # Private (per-thread) memory parameters. Shared memory lives in the core TOML. +addrLen = 39 + [bank] layoutEnable = true num = 20 @@ -18,6 +20,3 @@ bankNum = 16 bankEntries = 64 bankWidth = 128 readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/components/threads/attention/stack/CMakeLists.txt b/examples/prefab/threads/attention/stack/CMakeLists.txt similarity index 92% rename from examples/components/threads/attention/stack/CMakeLists.txt rename to examples/prefab/threads/attention/stack/CMakeLists.txt index 086ec8497..173381303 100644 --- a/examples/components/threads/attention/stack/CMakeLists.txt +++ b/examples/prefab/threads/attention/stack/CMakeLists.txt @@ -19,7 +19,9 @@ project(attention-thread LANGUAGES C CXX) include("${DESIGN_CMAKE}") set(ENGINE "${DESIGN_REPO}/stack/engine") -set(ISA "${DESIGN_ISA_DIR}/${THREAD}") +# The banks, thread and Ball ISA of this thread, as compile definitions. +set(THREAD_DEFS ${DESIGN_TARGET_DEFS_${THREAD}}) +list(TRANSFORM THREAD_DEFS PREPEND "-D") find_program(ELF_CC riscv64-unknown-elf-gcc REQUIRED) find_program(ELF_OBJCOPY riscv64-unknown-elf-objcopy REQUIRED) find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) @@ -28,9 +30,9 @@ find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) # Compiler: the Buckyball dialect and passes with the Balls of this thread. # ===----------------------------------------------------------------------=== set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${THREAD}}) -set(BUCKYBALL_TARGET_TABLE "${ISA}/target.inc") add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) -add_library(${THREAD}-compiler MODULE $) +add_library(${THREAD}-compiler MODULE) +target_link_libraries(${THREAD}-compiler PRIVATE buckyball-compiler) set_target_properties(${THREAD}-compiler PROPERTIES LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") @@ -44,10 +46,10 @@ add_custom_command( -ffreestanding -fno-builtin -fno-pic -fno-stack-protector -fno-jump-tables -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax -nostdlib -nostartfiles -Wl,--no-relax "-Wl,-T,${ENGINE}/program/code.ld" - "-I${ENGINE}/include" "-I${ISA}" + "-I${ENGINE}/include" ${THREAD_DEFS} "${ENGINE}/program/start.S" "${ENGINE}/program/stream.c" -o program.elf DEPENDS "${ENGINE}/program/stream.c" "${ENGINE}/program/instructions.inc" - "${ENGINE}/program/start.S" "${ENGINE}/program/code.ld" "${ISA}/params.h" + "${ENGINE}/program/start.S" "${ENGINE}/program/code.ld" VERBATIM) add_custom_command( OUTPUT program.bin @@ -61,9 +63,9 @@ add_custom_command( add_custom_command( OUTPUT loader.o COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 - "--embed-dir=${CMAKE_CURRENT_BINARY_DIR}" "-I${ENGINE}/include" "-I${ISA}" + "--embed-dir=${CMAKE_CURRENT_BINARY_DIR}" "-I${ENGINE}/include" ${THREAD_DEFS} -c "${ENGINE}/program/stream_prepare.c" -o loader.o - DEPENDS "${ENGINE}/program/stream_prepare.c" program.bin "${ISA}/params.h" + DEPENDS "${ENGINE}/program/stream_prepare.c" program.bin VERBATIM) add_custom_target(program ALL DEPENDS program.bin loader.o) @@ -93,8 +95,6 @@ set(KERNEL_SOURCES "${RVV}/flash_attention_dispatch.cpp" "${RVV}/flash_attention_mxfp8_dispatch.cpp") set(KERNEL_INCLUDES - "${ISA}" - "${DESIGN_ISA_DIR}/design" "${DESIGN_REPO}/bb-tests/workloads/lib" "${DESIGN_REPO}/stack/kernels" "${DESIGN_REPO}/stack/runtime/scheduler" @@ -140,7 +140,7 @@ foreach(LIBRARY kernels kernels-stream) add_custom_command( OUTPUT ${OBJECT} COMMAND "${LINUX_CXX}" -std=c++20 -O2 -fPIC -ffp-contract=off ${DEFINES} - ${KERNEL_INCLUDES} ${KERNEL_EMBED_DIRS} -MD -MF ${OBJECT}.d -c "${SOURCE}" -o ${OBJECT} + ${THREAD_DEFS} ${KERNEL_INCLUDES} ${KERNEL_EMBED_DIRS} -MD -MF ${OBJECT}.d -c "${SOURCE}" -o ${OBJECT} DEPENDS "${SOURCE}" ${KERNEL_DEPENDS} DEPFILE ${OBJECT}.d VERBATIM) diff --git a/examples/components/threads/build.sc b/examples/prefab/threads/build.sc similarity index 100% rename from examples/components/threads/build.sc rename to examples/prefab/threads/build.sc diff --git a/examples/components/threads/ffn/arch/FfnThread.scala b/examples/prefab/threads/ffn/arch/FfnThread.scala similarity index 100% rename from examples/components/threads/ffn/arch/FfnThread.scala rename to examples/prefab/threads/ffn/arch/FfnThread.scala diff --git a/examples/components/threads/ffn/arch/acc.scala b/examples/prefab/threads/ffn/arch/acc.scala similarity index 100% rename from examples/components/threads/ffn/arch/acc.scala rename to examples/prefab/threads/ffn/arch/acc.scala diff --git a/examples/components/threads/ffn/configs/balldomains/default.toml b/examples/prefab/threads/ffn/configs/balldomains/default.toml similarity index 100% rename from examples/components/threads/ffn/configs/balldomains/default.toml rename to examples/prefab/threads/ffn/configs/balldomains/default.toml diff --git a/examples/components/threads/ffn/configs/cpu/default.toml b/examples/prefab/threads/ffn/configs/cpu/default.toml similarity index 100% rename from examples/components/threads/ffn/configs/cpu/default.toml rename to examples/prefab/threads/ffn/configs/cpu/default.toml diff --git a/examples/components/threads/ffn/configs/default.toml b/examples/prefab/threads/ffn/configs/default.toml similarity index 100% rename from examples/components/threads/ffn/configs/default.toml rename to examples/prefab/threads/ffn/configs/default.toml diff --git a/examples/components/threads/matmul/configs/frontends/default.toml b/examples/prefab/threads/ffn/configs/frontends/default.toml similarity index 74% rename from examples/components/threads/matmul/configs/frontends/default.toml rename to examples/prefab/threads/ffn/configs/frontends/default.toml index c3e3d5e71..0f78452a5 100644 --- a/examples/components/threads/matmul/configs/frontends/default.toml +++ b/examples/prefab/threads/ffn/configs/frontends/default.toml @@ -2,8 +2,8 @@ bankAliasEnable = true robEntries = 16 rsOutOfOrderResponse = true bankIdLen = 10 -vbankIdUpperBound = 15 -sharedBankIdBase = 32 +maxPrivateVbank = 31 +sharedVbankBase = 32 iterLen = 34 subRobEnable = false subRobDepth = 64 diff --git a/examples/components/threads/ffn/configs/gpdomain/default.toml b/examples/prefab/threads/ffn/configs/gpdomain/default.toml similarity index 100% rename from examples/components/threads/ffn/configs/gpdomain/default.toml rename to examples/prefab/threads/ffn/configs/gpdomain/default.toml diff --git a/examples/components/threads/ffn/configs/memdomains/default.toml b/examples/prefab/threads/ffn/configs/memdomains/default.toml similarity index 98% rename from examples/components/threads/ffn/configs/memdomains/default.toml rename to examples/prefab/threads/ffn/configs/memdomains/default.toml index ff0bddb0e..e3a4ba22d 100644 --- a/examples/components/threads/ffn/configs/memdomains/default.toml +++ b/examples/prefab/threads/ffn/configs/memdomains/default.toml @@ -1,6 +1,8 @@ # FFN Thread Memory Domain Configuration (default) # Private (per-thread) memory parameters. Shared memory lives in the core TOML. +addrLen = 39 + [bank] layoutEnable = true num = 32 @@ -18,6 +20,3 @@ bankNum = 16 bankEntries = 64 bankWidth = 128 readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/components/threads/ffn/stack/CMakeLists.txt b/examples/prefab/threads/ffn/stack/CMakeLists.txt similarity index 92% rename from examples/components/threads/ffn/stack/CMakeLists.txt rename to examples/prefab/threads/ffn/stack/CMakeLists.txt index 042abb44d..c2f943ba6 100644 --- a/examples/components/threads/ffn/stack/CMakeLists.txt +++ b/examples/prefab/threads/ffn/stack/CMakeLists.txt @@ -19,7 +19,9 @@ project(ffn-thread LANGUAGES C CXX) include("${DESIGN_CMAKE}") set(ENGINE "${DESIGN_REPO}/stack/engine") -set(ISA "${DESIGN_ISA_DIR}/${THREAD}") +# The banks, thread and Ball ISA of this thread, as compile definitions. +set(THREAD_DEFS ${DESIGN_TARGET_DEFS_${THREAD}}) +list(TRANSFORM THREAD_DEFS PREPEND "-D") find_program(ELF_CC riscv64-unknown-elf-gcc REQUIRED) find_program(ELF_OBJCOPY riscv64-unknown-elf-objcopy REQUIRED) find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) @@ -28,9 +30,9 @@ find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) # Compiler: the Buckyball dialect and passes with the Balls of this thread. # ===----------------------------------------------------------------------=== set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${THREAD}}) -set(BUCKYBALL_TARGET_TABLE "${ISA}/target.inc") add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) -add_library(${THREAD}-compiler MODULE $) +add_library(${THREAD}-compiler MODULE) +target_link_libraries(${THREAD}-compiler PRIVATE buckyball-compiler) set_target_properties(${THREAD}-compiler PROPERTIES LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") @@ -44,10 +46,10 @@ add_custom_command( -ffreestanding -fno-builtin -fno-pic -fno-stack-protector -fno-jump-tables -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax -nostdlib -nostartfiles -Wl,--no-relax "-Wl,-T,${ENGINE}/program/code.ld" - "-I${ENGINE}/include" "-I${ISA}" + "-I${ENGINE}/include" ${THREAD_DEFS} "${ENGINE}/program/start.S" "${ENGINE}/program/stream.c" -o program.elf DEPENDS "${ENGINE}/program/stream.c" "${ENGINE}/program/instructions.inc" - "${ENGINE}/program/start.S" "${ENGINE}/program/code.ld" "${ISA}/params.h" + "${ENGINE}/program/start.S" "${ENGINE}/program/code.ld" VERBATIM) add_custom_command( OUTPUT program.bin @@ -61,9 +63,9 @@ add_custom_command( add_custom_command( OUTPUT loader.o COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 - "--embed-dir=${CMAKE_CURRENT_BINARY_DIR}" "-I${ENGINE}/include" "-I${ISA}" + "--embed-dir=${CMAKE_CURRENT_BINARY_DIR}" "-I${ENGINE}/include" ${THREAD_DEFS} -c "${ENGINE}/program/stream_prepare.c" -o loader.o - DEPENDS "${ENGINE}/program/stream_prepare.c" program.bin "${ISA}/params.h" + DEPENDS "${ENGINE}/program/stream_prepare.c" program.bin VERBATIM) add_custom_target(program ALL DEPENDS program.bin loader.o) @@ -93,8 +95,6 @@ set(KERNEL_SOURCES "${RVV}/flash_attention_dispatch.cpp" "${RVV}/flash_attention_mxfp8_dispatch.cpp") set(KERNEL_INCLUDES - "${ISA}" - "${DESIGN_ISA_DIR}/design" "${DESIGN_REPO}/bb-tests/workloads/lib" "${DESIGN_REPO}/stack/kernels" "${DESIGN_REPO}/stack/runtime/scheduler" @@ -140,7 +140,7 @@ foreach(LIBRARY kernels kernels-stream) add_custom_command( OUTPUT ${OBJECT} COMMAND "${LINUX_CXX}" -std=c++20 -O2 -fPIC -ffp-contract=off ${DEFINES} - ${KERNEL_INCLUDES} ${KERNEL_EMBED_DIRS} -MD -MF ${OBJECT}.d -c "${SOURCE}" -o ${OBJECT} + ${THREAD_DEFS} ${KERNEL_INCLUDES} ${KERNEL_EMBED_DIRS} -MD -MF ${OBJECT}.d -c "${SOURCE}" -o ${OBJECT} DEPENDS "${SOURCE}" ${KERNEL_DEPENDS} DEPFILE ${OBJECT}.d VERBATIM) diff --git a/examples/components/threads/matmul/arch/MatMulThread.scala b/examples/prefab/threads/matmul/arch/MatMulThread.scala similarity index 100% rename from examples/components/threads/matmul/arch/MatMulThread.scala rename to examples/prefab/threads/matmul/arch/MatMulThread.scala diff --git a/examples/components/threads/matmul/arch/acc.scala b/examples/prefab/threads/matmul/arch/acc.scala similarity index 100% rename from examples/components/threads/matmul/arch/acc.scala rename to examples/prefab/threads/matmul/arch/acc.scala diff --git a/examples/components/threads/matmul/configs/ant.toml b/examples/prefab/threads/matmul/configs/ant.toml similarity index 100% rename from examples/components/threads/matmul/configs/ant.toml rename to examples/prefab/threads/matmul/configs/ant.toml diff --git a/examples/components/threads/matmul/configs/balldomains/default.toml b/examples/prefab/threads/matmul/configs/balldomains/default.toml similarity index 100% rename from examples/components/threads/matmul/configs/balldomains/default.toml rename to examples/prefab/threads/matmul/configs/balldomains/default.toml diff --git a/examples/components/threads/matmul/configs/cpu/default.toml b/examples/prefab/threads/matmul/configs/cpu/default.toml similarity index 100% rename from examples/components/threads/matmul/configs/cpu/default.toml rename to examples/prefab/threads/matmul/configs/cpu/default.toml diff --git a/examples/components/threads/matmul/configs/default.toml b/examples/prefab/threads/matmul/configs/default.toml similarity index 100% rename from examples/components/threads/matmul/configs/default.toml rename to examples/prefab/threads/matmul/configs/default.toml diff --git a/examples/components/threads/ffn/configs/frontends/default.toml b/examples/prefab/threads/matmul/configs/frontends/default.toml similarity index 74% rename from examples/components/threads/ffn/configs/frontends/default.toml rename to examples/prefab/threads/matmul/configs/frontends/default.toml index 5c58b1cb2..5464f2d95 100644 --- a/examples/components/threads/ffn/configs/frontends/default.toml +++ b/examples/prefab/threads/matmul/configs/frontends/default.toml @@ -2,8 +2,8 @@ bankAliasEnable = true robEntries = 16 rsOutOfOrderResponse = true bankIdLen = 10 -vbankIdUpperBound = 31 -sharedBankIdBase = 32 +maxPrivateVbank = 15 +sharedVbankBase = 32 iterLen = 34 subRobEnable = false subRobDepth = 64 diff --git a/examples/components/threads/matmul/configs/gpdomain/default.toml b/examples/prefab/threads/matmul/configs/gpdomain/default.toml similarity index 100% rename from examples/components/threads/matmul/configs/gpdomain/default.toml rename to examples/prefab/threads/matmul/configs/gpdomain/default.toml diff --git a/examples/components/threads/matmul/configs/memdomains/default.toml b/examples/prefab/threads/matmul/configs/memdomains/default.toml similarity index 98% rename from examples/components/threads/matmul/configs/memdomains/default.toml rename to examples/prefab/threads/matmul/configs/memdomains/default.toml index 94c9b9ef3..d469cceb5 100644 --- a/examples/components/threads/matmul/configs/memdomains/default.toml +++ b/examples/prefab/threads/matmul/configs/memdomains/default.toml @@ -1,6 +1,8 @@ # MatMul Thread Memory Domain Configuration (default) # Private (per-thread) memory parameters. Shared memory lives in the core TOML. +addrLen = 39 + [bank] layoutEnable = true num = 20 @@ -18,6 +20,3 @@ bankNum = 16 bankEntries = 64 bankWidth = 128 readWidth = 8 - -[mem] -addrLen = 39 diff --git a/examples/components/threads/matmul/stack/CMakeLists.txt b/examples/prefab/threads/matmul/stack/CMakeLists.txt similarity index 91% rename from examples/components/threads/matmul/stack/CMakeLists.txt rename to examples/prefab/threads/matmul/stack/CMakeLists.txt index dbd540556..26d28e640 100644 --- a/examples/components/threads/matmul/stack/CMakeLists.txt +++ b/examples/prefab/threads/matmul/stack/CMakeLists.txt @@ -17,7 +17,9 @@ project(matmul-thread LANGUAGES C CXX) include("${DESIGN_CMAKE}") set(ENGINE "${DESIGN_REPO}/stack/engine") -set(ISA "${DESIGN_ISA_DIR}/${THREAD}") +# The banks, thread and Ball ISA of this thread, as compile definitions. +set(THREAD_DEFS ${DESIGN_TARGET_DEFS_${THREAD}}) +list(TRANSFORM THREAD_DEFS PREPEND "-D") find_program(ELF_CC riscv64-unknown-elf-gcc REQUIRED) find_program(ELF_OBJCOPY riscv64-unknown-elf-objcopy REQUIRED) find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) @@ -26,9 +28,9 @@ find_program(LINUX_CC riscv64-unknown-linux-gnu-gcc REQUIRED) # Compiler: the Buckyball dialect and passes with the Balls of this thread. # ===----------------------------------------------------------------------=== set(BUCKYBALL_BALLS ${DESIGN_TARGET_BALL_STACKS_${THREAD}}) -set(BUCKYBALL_TARGET_TABLE "${ISA}/target.inc") add_subdirectory("${DESIGN_REPO}/stack/compiler/midend" compiler) -add_library(${THREAD}-compiler MODULE $) +add_library(${THREAD}-compiler MODULE) +target_link_libraries(${THREAD}-compiler PRIVATE buckyball-compiler) set_target_properties(${THREAD}-compiler PROPERTIES LINKER_LANGUAGE CXX LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") @@ -41,12 +43,12 @@ add_custom_command( -ffreestanding -fno-builtin -fno-pic -fno-stack-protector -fno-jump-tables -fno-asynchronous-unwind-tables -msmall-data-limit=0 -mno-relax -nostdlib -nostartfiles -Wl,--no-relax "-Wl,-T,${ENGINE}/program/code.ld" - "-I${ENGINE}/include" "-I${ISA}" "-I${DESIGN_REPO}/bb-tests/workloads/lib" - "-I${DESIGN_REPO}/examples/components/balls/smatmul/workloads" + "-I${ENGINE}/include" ${THREAD_DEFS} "-I${DESIGN_REPO}/bb-tests/workloads/lib" + "-I${DESIGN_REPO}/examples/prefab/balls/smatmul/workloads" "${ENGINE}/program/start.S" "${CMAKE_CURRENT_SOURCE_DIR}/program.c" -o program.elf DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/program.c" "${CMAKE_CURRENT_SOURCE_DIR}/matmul.h" - "${ENGINE}/program/start.S" "${ENGINE}/program/code.ld" "${ISA}/params.h" + "${ENGINE}/program/start.S" "${ENGINE}/program/code.ld" VERBATIM) add_custom_command( OUTPUT program.bin @@ -60,10 +62,10 @@ add_custom_command( add_custom_command( OUTPUT loader.o COMMAND "${LINUX_CC}" -O2 -fPIC -ffreestanding -fno-builtin -msmall-data-limit=0 - "--embed-dir=${CMAKE_CURRENT_BINARY_DIR}" "-I${ENGINE}/include" "-I${ISA}" + "--embed-dir=${CMAKE_CURRENT_BINARY_DIR}" "-I${ENGINE}/include" ${THREAD_DEFS} -c "${CMAKE_CURRENT_SOURCE_DIR}/prepare.c" -o loader.o DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/prepare.c" "${CMAKE_CURRENT_SOURCE_DIR}/matmul.h" - program.bin "${ISA}/params.h" + program.bin VERBATIM) add_custom_target(program ALL DEPENDS program.bin loader.o) @@ -93,8 +95,6 @@ set(KERNEL_SOURCES "${RVV}/flash_attention_dispatch.cpp" "${RVV}/flash_attention_mxfp8_dispatch.cpp") set(KERNEL_INCLUDES - "${ISA}" - "${DESIGN_ISA_DIR}/design" "${DESIGN_REPO}/bb-tests/workloads/lib" "${DESIGN_REPO}/stack/kernels" "${DESIGN_REPO}/stack/runtime/scheduler" @@ -140,7 +140,7 @@ foreach(LIBRARY kernels kernels-stream) add_custom_command( OUTPUT ${OBJECT} COMMAND "${LINUX_CXX}" -std=c++20 -O2 -fPIC -ffp-contract=off ${DEFINES} - ${KERNEL_INCLUDES} ${KERNEL_EMBED_DIRS} -MD -MF ${OBJECT}.d -c "${SOURCE}" -o ${OBJECT} + ${THREAD_DEFS} ${KERNEL_INCLUDES} ${KERNEL_EMBED_DIRS} -MD -MF ${OBJECT}.d -c "${SOURCE}" -o ${OBJECT} DEPENDS "${SOURCE}" ${KERNEL_DEPENDS} DEPFILE ${OBJECT}.d VERBATIM) diff --git a/examples/components/threads/matmul/stack/matmul.h b/examples/prefab/threads/matmul/stack/matmul.h similarity index 100% rename from examples/components/threads/matmul/stack/matmul.h rename to examples/prefab/threads/matmul/stack/matmul.h diff --git a/examples/components/threads/matmul/stack/prepare.c b/examples/prefab/threads/matmul/stack/prepare.c similarity index 98% rename from examples/components/threads/matmul/stack/prepare.c rename to examples/prefab/threads/matmul/stack/prepare.c index b2fd721dc..21b165aec 100644 --- a/examples/components/threads/matmul/stack/prepare.c +++ b/examples/prefab/threads/matmul/stack/prepare.c @@ -1,5 +1,4 @@ #include "matmul.h" -#include "params.h" #include static const unsigned char image[] __attribute__((aligned(8))) = { diff --git a/examples/components/threads/matmul/stack/program.c b/examples/prefab/threads/matmul/stack/program.c similarity index 100% rename from examples/components/threads/matmul/stack/program.c rename to examples/prefab/threads/matmul/stack/program.c diff --git a/examples/components/threads/matmul/workloads/ctests/CMakeLists.txt b/examples/prefab/threads/matmul/workloads/ctests/CMakeLists.txt similarity index 83% rename from examples/components/threads/matmul/workloads/ctests/CMakeLists.txt rename to examples/prefab/threads/matmul/workloads/ctests/CMakeLists.txt index 609450e62..841569277 100644 --- a/examples/components/threads/matmul/workloads/ctests/CMakeLists.txt +++ b/examples/prefab/threads/matmul/workloads/ctests/CMakeLists.txt @@ -8,7 +8,7 @@ file( ) add_buckyball_ctest(${RVV_KERNEL_SOURCE}) set(RVV_KERNEL_TEST - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_kernel" + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_kernel" ) add_dependencies(component-ctests-build ${RVV_KERNEL_TEST}-ctest-build) add_subdirectory(rvv) diff --git a/examples/components/threads/matmul/workloads/ctests/rvv/CMakeLists.txt b/examples/prefab/threads/matmul/workloads/ctests/rvv/CMakeLists.txt similarity index 79% rename from examples/components/threads/matmul/workloads/ctests/rvv/CMakeLists.txt rename to examples/prefab/threads/matmul/workloads/ctests/rvv/CMakeLists.txt index d8f3d530e..8035cb31e 100644 --- a/examples/components/threads/matmul/workloads/ctests/rvv/CMakeLists.txt +++ b/examples/prefab/threads/matmul/workloads/ctests/rvv/CMakeLists.txt @@ -1,16 +1,14 @@ # Images: built by the thread stack (stack/kernels/rvv); dispatch code: built here. set(KERNEL_DIR "${BUCKYBALL_REPO_ROOT}/stack/kernels/rvv") set(IMAGE_DIR "${DESIGN_THREAD_BUILD_${BUCKYBALL_CTEST_TARGET}}/rvv") -file(READ "${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET}/params.h" PARAMS) -foreach(PARAM BANK_NUM BANK_WIDTH BANK_LINES) - string(REGEX MATCH "#define ${PARAM} ([0-9]+)" MATCH "${PARAMS}") - set(${PARAM} "${CMAKE_MATCH_1}") -endforeach() -if(BANK_NUM LESS 4 OR NOT BANK_WIDTH EQUAL 128 OR BANK_LINES LESS 32) +set(_BANK_NUM ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_NUM}) +set(_BANK_WIDTH ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_WIDTH}) +set(_BANK_ENTRIES ${DESIGN_PROFILES_${BUCKYBALL_CTEST_TARGET}_THREAD_MEM_BANK_ENTRIES}) +if(_BANK_NUM LESS 4 OR NOT _BANK_WIDTH EQUAL 128 OR _BANK_ENTRIES LESS 32) message( FATAL_ERROR "RVV kernel CTests require 4 banks, 128-bit rows and 512 bytes per bank; " - "${BUCKYBALL_CTEST_TARGET} has ${BANK_NUM} banks, ${BANK_WIDTH}-bit rows, ${BANK_LINES} lines" + "${BUCKYBALL_CTEST_TARGET} has ${_BANK_NUM} banks, ${_BANK_WIDTH}-bit rows, ${_BANK_ENTRIES} lines" ) endif() foreach( @@ -24,7 +22,7 @@ foreach( ) buckyball_enforce_ctest_line_limit("${KERNEL}.cpp") set(NAME - "${BUCKYBALL_WORKLOAD_DESIGN}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_${KERNEL}" + "${DESIGN_NAME}-${BUCKYBALL_CTEST_TARGET}-ctest-rvv_${KERNEL}" ) add_buckyball_linux_ctest(${NAME} ${CMAKE_CURRENT_SOURCE_DIR} ${KERNEL}.cpp) if(KERNEL STREQUAL "matmul") @@ -47,6 +45,8 @@ foreach( set_target_properties(${NAME}-linux PROPERTIES LINKER_LANGUAGE CXX) target_link_libraries(${NAME}-linux PRIVATE m) buckyball_ctest_deps(DEPS ${CMAKE_CURRENT_SOURCE_DIR} ${KERNEL}.cpp) + set(TARGET_DEFINES ${DESIGN_TARGET_DEFS_${BUCKYBALL_CTEST_TARGET}}) + list(TRANSFORM TARGET_DEFINES PREPEND "-D") add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${NAME}-baremetal" COMMAND @@ -55,8 +55,7 @@ foreach( -I${BUCKYBALL_REPO_ROOT}/stack/runtime/services -I${WORKLOAD_LIB_DIR} -I${KERNEL_DIR} -I${BUCKYBALL_REPO_ROOT}/stack/kernels --embed-dir=${IMAGE_DIR} - -I${BUCKYBALL_ISA_DIR}/design - -I${BUCKYBALL_ISA_DIR}/${BUCKYBALL_CTEST_TARGET} + ${DESIGN_DEFINES} ${TARGET_DEFINES} -I${BUCKYBALL_REPO_ROOT}/stack/thirdparty/buddy-mlir/llvm/mlir/include/mlir/ExecutionEngine -o "${NAME}-baremetal" ${BAREMETAL_DIR}/crt0.S -x c ${WORKLOAD_LIB_DIR}/buckyball.c ${BBHW_MEM_C} diff --git a/examples/components/threads/matmul/workloads/ctests/rvv/matmul.cpp b/examples/prefab/threads/matmul/workloads/ctests/rvv/matmul.cpp similarity index 100% rename from examples/components/threads/matmul/workloads/ctests/rvv/matmul.cpp rename to examples/prefab/threads/matmul/workloads/ctests/rvv/matmul.cpp diff --git a/examples/components/threads/matmul/workloads/ctests/rvv/mxfp8_quant.cpp b/examples/prefab/threads/matmul/workloads/ctests/rvv/mxfp8_quant.cpp similarity index 100% rename from examples/components/threads/matmul/workloads/ctests/rvv/mxfp8_quant.cpp rename to examples/prefab/threads/matmul/workloads/ctests/rvv/mxfp8_quant.cpp diff --git a/examples/components/threads/matmul/workloads/ctests/rvv/norm.cpp b/examples/prefab/threads/matmul/workloads/ctests/rvv/norm.cpp similarity index 97% rename from examples/components/threads/matmul/workloads/ctests/rvv/norm.cpp rename to examples/prefab/threads/matmul/workloads/ctests/rvv/norm.cpp index 27f693da4..17a077cb5 100644 --- a/examples/components/threads/matmul/workloads/ctests/rvv/norm.cpp +++ b/examples/prefab/threads/matmul/workloads/ctests/rvv/norm.cpp @@ -2,7 +2,6 @@ #include #include #include -#include extern "C" void _mlir_ciface_rvv_norm(UnrankedMemRefType *, UnrankedMemRefType *, @@ -11,7 +10,7 @@ extern "C" void _mlir_ciface_rvv_norm(UnrankedMemRefType *, extern "C" void _mlir_ciface_rvv_norm_no_weight(UnrankedMemRefType *, UnrankedMemRefType *, uint32_t, uint32_t); -constexpr int capacity = BANK_LINES * BANK_WIDTH / 32; +constexpr int capacity = THREAD_MEM_BANK_ENTRIES * THREAD_MEM_BANK_WIDTH / 32; constexpr int maximum = 2 * capacity + 5; alignas(16) static float input[2 * (maximum + 4) + 4]; alignas(16) static float output[2 * maximum + 4], expected[2 * maximum]; diff --git a/examples/components/threads/matmul/workloads/ctests/rvv/silu.cpp b/examples/prefab/threads/matmul/workloads/ctests/rvv/silu.cpp similarity index 100% rename from examples/components/threads/matmul/workloads/ctests/rvv/silu.cpp rename to examples/prefab/threads/matmul/workloads/ctests/rvv/silu.cpp diff --git a/examples/components/threads/matmul/workloads/ctests/rvv/snake.cpp b/examples/prefab/threads/matmul/workloads/ctests/rvv/snake.cpp similarity index 100% rename from examples/components/threads/matmul/workloads/ctests/rvv/snake.cpp rename to examples/prefab/threads/matmul/workloads/ctests/rvv/snake.cpp diff --git a/examples/components/threads/matmul/workloads/ctests/rvv/swiglu.cpp b/examples/prefab/threads/matmul/workloads/ctests/rvv/swiglu.cpp similarity index 100% rename from examples/components/threads/matmul/workloads/ctests/rvv/swiglu.cpp rename to examples/prefab/threads/matmul/workloads/ctests/rvv/swiglu.cpp diff --git a/examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt b/examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt similarity index 100% rename from examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt rename to examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/CMakeLists.txt diff --git a/examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c b/examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c similarity index 77% rename from examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c rename to examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c index 3c8f4afe8..499b900da 100644 --- a/examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c +++ b/examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/goban.c @@ -7,16 +7,16 @@ #include #include -#if BANK_WIDTH != 128 || BANK_LINES < 64 +#if THREAD_MEM_BANK_WIDTH != 128 || THREAD_MEM_BANK_ENTRIES < 64 #error "Goban thread pipeline requires 128-bit banks with at least 64 rows" #endif enum { DIM = 16 }; -static int8_t input[BB_CORE_NUM][DIM * DIM] __attribute__((aligned(64))); -static int8_t weight[BB_CORE_NUM][DIM * DIM] __attribute__((aligned(64))); -static int32_t bias[BB_CORE_NUM][DIM] __attribute__((aligned(64))); -static float scale[BB_CORE_NUM][DIM] __attribute__((aligned(64))); -static int8_t output[BB_CORE_NUM][DIM * DIM] __attribute__((aligned(64))); +static int8_t input[BB_NUM_CORES][DIM * DIM] __attribute__((aligned(64))); +static int8_t weight[BB_NUM_CORES][DIM * DIM] __attribute__((aligned(64))); +static int32_t bias[BB_NUM_CORES][DIM] __attribute__((aligned(64))); +static float scale[BB_NUM_CORES][DIM] __attribute__((aligned(64))); +static int8_t output[BB_NUM_CORES][DIM * DIM] __attribute__((aligned(64))); #ifdef __cplusplus extern "C" diff --git a/examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp b/examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp similarity index 95% rename from examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp rename to examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp index 32fd7c88a..607167b47 100644 --- a/examples/components/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp +++ b/examples/prefab/threads/matmul/workloads/ctests/transpose_smatmul_quant_pipeline/transpose_smatmul_quant_pipeline-main.cpp @@ -12,7 +12,7 @@ int main() { __ATOMIC_RELEASE); __atomic_add_fetch(&completed, 1, __ATOMIC_RELEASE); } - if (hart == BB_REPORT_HART) { + if (hart == WORKLOAD_REPORT_HART) { while (__atomic_load_n(&completed, __ATOMIC_ACQUIRE) != expected) asm volatile("nop"); return __atomic_load_n(&failed, __ATOMIC_ACQUIRE) != 0; diff --git a/scripts/nix/download.sh b/scripts/nix/download.sh index 602218f2e..a265cb6a7 100755 --- a/scripts/nix/download.sh +++ b/scripts/nix/download.sh @@ -24,7 +24,7 @@ begin_step "0-2" "submodules init" cd ${BBDIR} git submodule sync git submodule update --init --progress -git -C ${BBDIR}/stack submodule update --init --progress engine os/linux +git -C ${BBDIR}/stack submodule update --init --progress os/linux git -C ${BBDIR}/stack submodule update --init --depth 1 --single-branch --recommend-shallow --progress \ os/thirdparty/linux \ os/thirdparty/busybox \ diff --git a/stack b/stack index 28dafe976..3d48a0709 160000 --- a/stack +++ b/stack @@ -1 +1 @@ -Subproject commit 28dafe976e5d7826eb1706bbbd5274073c555a95 +Subproject commit 3d48a07097bc48032e2a6d11528319bac7e57757